diff --git a/Documentation/core-api/floating-point.rst b/Documentation/core-api/floating-point.rst
new file mode 100644
index 000000000000..a8d0d4b05052
--- /dev/null
+++ b/Documentation/core-api/floating-point.rst
@@ -0,0 +1,78 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+Floating-point API
+==================
+
+Kernel code is normally prohibited from using floating-point (FP) registers or
+instructions, including the C float and double data types. This rule reduces
+system call overhead, because the kernel does not need to save and restore the
+userspace floating-point register state.
+
+However, occasionally drivers or library functions may need to include FP code.
+This is supported by isolating the functions containing FP code to a separate
+translation unit (a separate source file), and saving/restoring the FP register
+state around calls to those functions. This creates "critical sections" of
+floating-point usage.
+
+The reason for this isolation is to prevent the compiler from generating code
+touching the FP registers outside these critical sections. Compilers sometimes
+use FP registers to optimize inlined ``memcpy`` or variable assignment, as
+floating-point registers may be wider than general-purpose registers.
+
+Usability of floating-point code within the kernel is architecture-specific.
+Additionally, because a single kernel may be configured to support platforms
+both with and without a floating-point unit, FPU availability must be checked
+both at build time and at run time.
+
+Several architectures implement the generic kernel floating-point API from
+``linux/fpu.h``, as described below. Some other architectures implement their
+own unique APIs, which are documented separately.
+
+Build-time API
+--------------
+
+Floating-point code may be built if the option ``ARCH_HAS_KERNEL_FPU_SUPPORT``
+is enabled. For C code, such code must be placed in a separate file, and that
+file must have its compilation flags adjusted using the following pattern::
+
+    CFLAGS_foo.o += $(CC_FLAGS_FPU)
+    CFLAGS_REMOVE_foo.o += $(CC_FLAGS_NO_FPU)
+
+Architectures are expected to define one or both of these variables in their
+top-level Makefile as needed. For example::
+
+    CC_FLAGS_FPU := -mhard-float
+
+or::
+
+    CC_FLAGS_NO_FPU := -msoft-float
+
+Normal kernel code is assumed to use the equivalent of ``CC_FLAGS_NO_FPU``.
+
+Runtime API
+-----------
+
+The runtime API is provided in ``linux/fpu.h``. This header cannot be included
+from files implementing FP code (those with their compilation flags adjusted as
+above). Instead, it must be included when defining the FP critical sections.
+
+.. c:function:: bool kernel_fpu_available( void )
+
+        This function reports if floating-point code can be used on this CPU or
+        platform. The value returned by this function is not expected to change
+        at runtime, so it only needs to be called once, not before every
+        critical section.
+
+.. c:function:: void kernel_fpu_begin( void )
+                void kernel_fpu_end( void )
+
+        These functions create a floating-point critical section. It is only
+        valid to call ``kernel_fpu_begin()`` after a previous call to
+        ``kernel_fpu_available()`` returned ``true``. These functions are only
+        guaranteed to be callable from (preemptible or non-preemptible) process
+        context.
+
+        Preemption may be disabled inside critical sections, so their size
+        should be minimized. They are *not* required to be reentrant. If the
+        caller expects to nest critical sections, it must implement its own
+        reference counting.
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index 7a3a08d81f11..974beccd671f 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -48,6 +48,7 @@ Library functionality that is used throughout the kernel.
    errseq
    wrappers/atomic_t
    wrappers/atomic_bitops
+   floating-point
 
 Low level entry and exit
 ========================
diff --git a/Documentation/devicetree/bindings/clock/eswin,eic7700-clock.yaml b/Documentation/devicetree/bindings/clock/eswin,eic7700-clock.yaml
new file mode 100644
index 000000000000..f5b6c197e648
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/eswin,eic7700-clock.yaml
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/eswin,eic7700-clock.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Eswin EIC7700 SOC Clock Controller
+
+maintainers:
+  - huangyifeng <huangyifeng@eswincomputing.com>
+
+description: |
+  The clock controller registers are part of the syscrg block on
+  	eic7700 SoC.
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - const: eswin,eic7700-clock
+
+  '#clock-cells':
+    const: 1
+
+required:
+  - compatible
+  - '#clock-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/eswin,eic7700-clock.h>
+
+    d0_sys_crg: sys-crg@51828000 {
+        compatible = "syscon", "simple-mfd";
+        reg = <0x000000 0x51828000 0x000000 0x80000>;
+        numa-node-id = <0>;
+        d0_clock: clock-controller {
+            compatible = "eswin,eic7700-clock";
+            #clock-cells = <1>;
+        };
+    };
+    d0_i2c9: i2c@509e0000 {
+        compatible = "snps,designware-i2c";
+        clocks = <&d0_clock EIC7700_CLK_LSP_I2C9_PCLK>;
+        clock-names = "pclk";
+        #address-cells = <1>;
+        #size-cells = <0>;
+        reg = <0x0 0x509e0000 0x0 0x8000>;
+        interrupts = <114>;
+        interrupt-parent = <&plic0>;
+        numa-node-id = <0>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/reset/eswin,eic7700-reset.yaml b/Documentation/devicetree/bindings/reset/eswin,eic7700-reset.yaml
new file mode 100644
index 000000000000..a452bce02852
--- /dev/null
+++ b/Documentation/devicetree/bindings/reset/eswin,eic7700-reset.yaml
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/reset/eswin,eic7700-reset.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Eswin EIC7700 SOC Reset Controller
+
+maintainers:
+  - huangyifeng <huangyifeng@eswincomputing.com>
+
+description: |
+  Please also refer to reset.txt in this directory for common reset
+  controller binding usage.
+  The reset controller registers are part of the syscrg block on
+  the EIC7700 SoC.
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - const: eswin,eic7700-reset
+
+  '#reset-cells':
+    description: |
+      Specifies the number of cells needed to encode a reset source.
+      Cell #1 : module id of the reset consumer
+      Cell #2 : bit position of the reset in each reset consumer module
+    const: 2
+
+required:
+  - compatible
+  - '#reset-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/reset/eswin,eic7700-reset.h>
+
+    d0_sys_crg: sys-crg@51828000 {
+        compatible = "syscon", "simple-mfd";
+        reg = <0x000000 0x51828000 0x000000 0x80000>;
+        numa-node-id = <0>;
+        d0_reset: reset-controller {
+            compatible = "eswin,eic7700-reset";
+            #reset-cells = <2>;
+        };
+    };
+    d0_i2c9: i2c@509e0000 {
+        compatible = "snps,designware-i2c";
+        resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_9>;
+        reset-names = "rst";
+        #address-cells = <1>;
+        #size-cells = <0>;
+        reg = <0x0 0x509e0000 0x0 0x8000>;
+        interrupts = <114>;
+        interrupt-parent = <&plic0>;
+        numa-node-id = <0>;
+    };
diff --git a/Documentation/features/sched/membarrier-sync-core/arch-support.txt b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
index 23260ca44946..76597adfb7d5 100644
--- a/Documentation/features/sched/membarrier-sync-core/arch-support.txt
+++ b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
@@ -10,6 +10,22 @@
 # Rely on implicit context synchronization as a result of exception return
 # when returning from IPI handler, and when returning to user-space.
 #
+# * riscv
+#
+# riscv uses xRET as return from interrupt and to return to user-space.
+#
+# Given that xRET is not core serializing, we rely on FENCE.I for providing
+# core serialization:
+#
+#  - by calling sync_core_before_usermode() on return from interrupt (cf.
+#    ipi_sync_core()),
+#
+#  - via switch_mm() and sync_core_before_usermode() (respectively, for
+#    uthread->uthread and kthread->uthread transitions) before returning
+#    to user-space.
+#
+#  The serialization in switch_mm() is activated by prepare_sync_core_cmd().
+#
 # * x86
 #
 # x86-32 uses IRET as return from interrupt, which takes care of the IPI.
@@ -44,7 +60,7 @@
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/riscv/vm-layout.rst b/Documentation/riscv/vm-layout.rst
index 69ff6da1dbf8..5462c84f4723 100644
--- a/Documentation/riscv/vm-layout.rst
+++ b/Documentation/riscv/vm-layout.rst
@@ -133,25 +133,3 @@ RISC-V Linux Kernel SV57
    ffffffff00000000 |  -4     GB | ffffffff7fffffff |    2 GB | modules, BPF
    ffffffff80000000 |  -2     GB | ffffffffffffffff |    2 GB | kernel
   __________________|____________|__________________|_________|____________________________________________________________
-
-
-Userspace VAs
---------------------
-To maintain compatibility with software that relies on the VA space with a
-maximum of 48 bits the kernel will, by default, return virtual addresses to
-userspace from a 48-bit range (sv48). This default behavior is achieved by
-passing 0 into the hint address parameter of mmap. On CPUs with an address space
-smaller than sv48, the CPU maximum supported address space will be the default.
-
-Software can "opt-in" to receiving VAs from another VA space by providing
-a hint address to mmap. A hint address passed to mmap will cause the largest
-address space that fits entirely into the hint to be used, unless there is no
-space left in the address space. If there is no space available in the requested
-address space, an address in the next smallest available address space will be
-returned.
-
-For example, in order to obtain 48-bit VA space, a hint address greater than
-:code:`1 << 47` must be provided. Note that this is 47 due to sv48 userspace
-ending at :code:`1 << 47` and the addresses beyond this are reserved for the
-kernel. Similarly, to obtain 57-bit VA space addresses, a hint address greater
-than or equal to :code:`1 << 56` must be provided.
diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst
index 3170747226f6..43bd8a145b7a 100644
--- a/Documentation/scheduler/index.rst
+++ b/Documentation/scheduler/index.rst
@@ -7,6 +7,7 @@ Scheduler
 
 
     completion
+    membarrier
     sched-arch
     sched-bwc
     sched-deadline
diff --git a/Documentation/scheduler/membarrier.rst b/Documentation/scheduler/membarrier.rst
new file mode 100644
index 000000000000..2387804b1c63
--- /dev/null
+++ b/Documentation/scheduler/membarrier.rst
@@ -0,0 +1,39 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+========================
+membarrier() System Call
+========================
+
+MEMBARRIER_CMD_{PRIVATE,GLOBAL}_EXPEDITED - Architecture requirements
+=====================================================================
+
+Memory barriers before updating rq->curr
+----------------------------------------
+
+The commands MEMBARRIER_CMD_PRIVATE_EXPEDITED and MEMBARRIER_CMD_GLOBAL_EXPEDITED
+require each architecture to have a full memory barrier after coming from
+user-space, before updating rq->curr.  This barrier is implied by the sequence
+rq_lock(); smp_mb__after_spinlock() in __schedule().  The barrier matches a full
+barrier in the proximity of the membarrier system call exit, cf.
+membarrier_{private,global}_expedited().
+
+Memory barriers after updating rq->curr
+---------------------------------------
+
+The commands MEMBARRIER_CMD_PRIVATE_EXPEDITED and MEMBARRIER_CMD_GLOBAL_EXPEDITED
+require each architecture to have a full memory barrier after updating rq->curr,
+before returning to user-space.  The schemes providing this barrier on the various
+architectures are as follows.
+
+ - alpha, arc, arm, hexagon, mips rely on the full barrier implied by
+   spin_unlock() in finish_lock_switch().
+
+ - arm64 relies on the full barrier implied by switch_to().
+
+ - powerpc, riscv, s390, sparc, x86 rely on the full barrier implied by
+   switch_mm(), if mm is not NULL; they rely on the full barrier implied
+   by mmdrop(), otherwise.  On powerpc and riscv, switch_mm() relies on
+   membarrier_arch_switch_mm().
+
+The barrier matches a full barrier in the proximity of the membarrier system call
+entry, cf. membarrier_{private,global}_expedited().
diff --git a/MAINTAINERS b/MAINTAINERS
index 294d2ce29b73..45778c833e51 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -13703,7 +13703,9 @@ M:	Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 M:	"Paul E. McKenney" <paulmck@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Supported
+F:	Documentation/scheduler/membarrier.rst
 F:	arch/*/include/asm/membarrier.h
+F:	arch/*/include/asm/sync_core.h
 F:	include/uapi/linux/membarrier.h
 F:	kernel/sched/membarrier.c
 
diff --git a/Makefile b/Makefile
index 685a65992449..50dc52b9bc56 100644
--- a/Makefile
+++ b/Makefile
@@ -980,6 +980,11 @@ KBUILD_CFLAGS	+= $(CC_FLAGS_CFI)
 export CC_FLAGS_CFI
 endif
 
+# Architectures can define flags to add/remove for floating-point support
+CC_FLAGS_FPU	+= -D_LINUX_FPU_COMPILATION_UNIT
+export CC_FLAGS_FPU
+export CC_FLAGS_NO_FPU
+
 ifneq ($(CONFIG_FUNCTION_ALIGNMENT),0)
 KBUILD_CFLAGS += -falign-functions=$(CONFIG_FUNCTION_ALIGNMENT)
 endif
diff --git a/arch/Kconfig b/arch/Kconfig
index 09603e0bc2cc..ae26a98428cd 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1488,6 +1488,12 @@ config ARCH_HAS_NONLEAF_PMD_YOUNG
 	  address translations. Page table walkers that clear the accessed bit
 	  may use this capability to reduce their search space.
 
+config ARCH_HAS_KERNEL_FPU_SUPPORT
+	bool
+	help
+	  Architectures that select this option can run floating-point code in
+	  the kernel, as described in Documentation/core-api/floating-point.rst.
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 1304992232ad..7d26860e5616 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -28,14 +28,17 @@ config RISCV
 	select ARCH_HAS_GIGANTIC_PAGE
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_MEMBARRIER_CALLBACKS
+	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_MMIOWB
 	select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 	select ARCH_HAS_PMEM_API
+	select ARCH_HAS_PREPARE_SYNC_CORE_CMD
 	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_SET_DIRECT_MAP if MMU
 	select ARCH_HAS_SET_MEMORY if MMU
 	select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
 	select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL
+	select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE
 	select ARCH_HAS_SYSCALL_WRAPPER
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
@@ -64,7 +67,7 @@ config RISCV
 	select CLINT_TIMER if !MMU
 	select CLONE_BACKWARDS
 	select COMMON_CLK
-	select CPU_PM if CPU_IDLE || HIBERNATION
+	select CPU_PM if CPU_IDLE || HIBERNATION || SUSPEND
 	select EDAC_SUPPORT
 	select FRAME_POINTER if PERF_EVENTS || (FUNCTION_TRACER && !DYNAMIC_FTRACE)
 	select GENERIC_ARCH_TOPOLOGY
@@ -150,6 +153,7 @@ config RISCV
 	select MODULE_SECTIONS if MODULES
 	select OF
 	select OF_EARLY_FLATTREE
+	select HAVE_SETUP_PER_CPU_AREA if NUMA
 	select OF_IRQ
 	select PCI_DOMAINS_GENERIC if PCI
 	select PCI_MSI if PCI
@@ -334,7 +338,7 @@ config ARCH_RV64I
 	bool "RV64I"
 	select 64BIT
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
-	select SWIOTLB if MMU
+	select SWIOTLB if MMU && !ARCH_ESWIN_EIC770X_SOC_FAMILY
 
 endchoice
 
@@ -377,6 +381,22 @@ config SCHED_MC
 	  making when dealing with multi-core CPU chips at a cost of slightly
 	  increased overhead in some places. If unsure say N here.
 
+config SCHED_CLUSTER
+	bool "Cluster scheduler support"
+	help
+	  Cluster scheduler support improves the CPU scheduler's decision
+	  making when dealing with machines that have clusters of CPUs.
+	  Cluster usually means a couple of CPUs which are placed closely
+	  by sharing mid-level caches, last-level cache tags or internal
+	  busses.
+
+config SCHED_SMT
+	bool "SMT scheduler support"
+	help
+	  Improves the CPU scheduler's decision making when dealing with
+	  MultiThreading at a cost of slightly increased overhead in some
+	  places. If unsure say N here.
+
 config NR_CPUS
 	int "Maximum number of CPUs (2-512)"
 	depends on SMP
@@ -412,8 +432,10 @@ config NUMA
 	depends on SMP && MMU
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select GENERIC_ARCH_NUMA
-	select NEED_PER_CPU_EMBED_FIRST_CHUNK
 	select OF_NUMA
+	select HAVE_SETUP_PER_CPU_AREA
+	select NEED_PER_CPU_EMBED_FIRST_CHUNK
+	select NEED_PER_CPU_PAGE_FIRST_CHUNK
 	select USE_PERCPU_NUMA_NODE_ID
 	help
 	  Enable NUMA (Non-Uniform Memory Access) support.
@@ -424,7 +446,7 @@ config NUMA
 config NODES_SHIFT
 	int "Maximum NUMA Nodes (as a power of 2)"
 	range 1 10
-	default "2"
+	default "1"
 	depends on NUMA
 	help
 	  Specify the maximum number of NUMA Nodes available on the target
diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs
index 30fd6a512828..01bf493e0d80 100644
--- a/arch/riscv/Kconfig.socs
+++ b/arch/riscv/Kconfig.socs
@@ -49,6 +49,15 @@ config ARCH_THEAD
 	help
 	  This enables support for the RISC-V based T-HEAD SoCs.
 
+config ARCH_ESWIN
+	def_bool SOC_ESWIN
+
+config SOC_ESWIN
+	bool "Eswin SoCs"
+	select ERRATA_SIFIVE if !XIP_KERNEL
+	help
+	  This enables support for Eswin SoC platform hardware.
+
 config ARCH_VIRT
 	def_bool SOC_VIRT
 
diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index 4d06f3402674..a040fd5f83ad 100644
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -54,22 +54,7 @@ endif
 endif
 endif
 
-# ISA string setting
-riscv-march-$(CONFIG_ARCH_RV32I)	:= rv32ima
-riscv-march-$(CONFIG_ARCH_RV64I)	:= rv64ima
-riscv-march-$(CONFIG_FPU)		:= $(riscv-march-y)fd
-riscv-march-$(CONFIG_RISCV_ISA_C)	:= $(riscv-march-y)c
-riscv-march-$(CONFIG_RISCV_ISA_V)	:= $(riscv-march-y)v
-
-ifdef CONFIG_TOOLCHAIN_NEEDS_OLD_ISA_SPEC
-KBUILD_CFLAGS += -Wa,-misa-spec=2.2
-KBUILD_AFLAGS += -Wa,-misa-spec=2.2
-else
-riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei
-endif
-
-# Check if the toolchain supports Zihintpause extension
-riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE) := $(riscv-march-y)_zihintpause
+include $(srctree)/arch/riscv/Makefile.isa
 
 # Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by
 # matching non-v and non-multi-letter extensions out with the filter ([^v_]*)
diff --git a/arch/riscv/Makefile.isa b/arch/riscv/Makefile.isa
new file mode 100644
index 000000000000..322a83958b96
--- /dev/null
+++ b/arch/riscv/Makefile.isa
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+# ISA string setting
+riscv-march-$(CONFIG_ARCH_RV32I)	:= rv32ima
+riscv-march-$(CONFIG_ARCH_RV64I)	:= rv64ima
+riscv-march-$(CONFIG_FPU)		:= $(riscv-march-y)fd
+riscv-march-$(CONFIG_RISCV_ISA_C)	:= $(riscv-march-y)c
+riscv-march-$(CONFIG_RISCV_ISA_V)	:= $(riscv-march-y)v
+
+ifdef CONFIG_TOOLCHAIN_NEEDS_OLD_ISA_SPEC
+KBUILD_CFLAGS += -Wa,-misa-spec=2.2
+KBUILD_AFLAGS += -Wa,-misa-spec=2.2
+else
+riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei
+endif
+
+# Check if the toolchain supports Zihintpause extension
+riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE) := $(riscv-march-y)_zihintpause
diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile
index f60a280abb15..5c3eda88e8af 100644
--- a/arch/riscv/boot/dts/Makefile
+++ b/arch/riscv/boot/dts/Makefile
@@ -6,5 +6,6 @@ subdir-y += renesas
 subdir-y += sifive
 subdir-y += starfive
 subdir-y += thead
+subdir-y += eswin
 
 obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y))
diff --git a/arch/riscv/boot/dts/eswin/Makefile b/arch/riscv/boot/dts/eswin/Makefile
new file mode 100644
index 000000000000..5a248d86b6e7
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/Makefile
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: GPL-2.0
+dtb-$(CONFIG_SOC_ESWIN) += eswin-win2030.dtb \
+							eic7700-evb.dtb \
+							eic7700-evb-a2.dtb \
+							eic7700-evb-a2-mipi.dtb \
+							eic7700-evb-a3.dtb \
+							eic7700-evb-a3-mipi.dtb \
+							eic7700-hifive-premier-p550.dtb \
+                                                        eic7700-milkv-megrez.dtb \
+                                                        eic7700-milkv-megrez-nx.dtb \
+                                                        eic7700-pine64-starpro64.dtb \
+							eic7700-hifive-premier-p550-npu.dtb \
+							eic7700-milkv-megrez-npu.dtb \
+							eic7700-pine64-starpro64-npu.dtb \
+							eic7702-evb-a1-d0.dtb \
+							eic7702-evb-a1-d1.dtb \
+							eic7702-evb-a1.dtb\
+							eic7700-z530.dtb \
+							eic7700-d314.dtb \
+							eic7702-evb-a1-interleave.dtb \
+							eic7702-tb.dtb \
+							eic7700-sbc-a1.dtb \
+							eic7700-sbc-a1-mipi.dtb\
+							eic7700-osm.dtb \
+							eic7702-d560.dtb \
+							eic7702-d560-interleave.dtb \
+							vpu7702-pcie.dtb \
+							vpu7702-evb.dtb	\
+							eic7700-ce1.dtb \
+							u2-evb.dtb
+obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y))
diff --git a/arch/riscv/boot/dts/eswin/eic7700-ce1.dts b/arch/riscv/boot/dts/eswin/eic7700-ce1.dts
new file mode 100644
index 000000000000..61c94fcb4b0a
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-ce1.dts
@@ -0,0 +1,980 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include "eic7700-evb-fccsp-camera-imx219.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 FCCSP EVB A1";
+	compatible = "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	vdd_soc: regulator-0 {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio106_default>;
+		regulator-name = "soc-adjust";
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <900000>;
+		regulator-boot-on;
+		regulator-always-on;
+		gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+		states = <800000 0x0>,
+				<900000 0x1>;
+	};
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio95_default>;
+	cpu-voltage-gpios = <&portc 31 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1600M>;
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0xA0205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0xA0205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46204620 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* pin header */
+	status = "okay";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q64jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* mpq8785 & temp sensor & rtc*/
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "d1_npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+};
+
+&d0_i2c2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 & mipi dsi fpc? */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "okay";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_aon_i2c1 {
+	/* ina226x3 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@48 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "dc_vdd";
+		reg = <0x48>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio71  : csi0/1 fpc io ctrl(O)
+	gpio72  : csi0/1 fpc io ctrl(O)?
+	gpio77  : csi2/3 fpc io ctrl(O)
+	gpio78  : csi2/3 fpc io ctrl(O)?
+	gpio83  : csi4/5 fpc io ctrl(O)
+	gpio84  : csi4/5 fpc io ctrl(O)?
+	gpio86  : level shifter oe ctrl(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : soc power boost(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
+
+
+
diff --git a/arch/riscv/boot/dts/eswin/eic7700-d314-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7700-d314-camera.dtsi
new file mode 100644
index 000000000000..a2ecef0ec3c8
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-d314-camera.dtsi
@@ -0,0 +1,569 @@
+&vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+	phy_mode = <5>; //5: 6x2lanes, 2: 3x4lanes
+    vvcam_isp_0: vvcam_isp@0 {
+        compatible = "verisilicon,isp-v4l2";
+        id = <0>;
+        status = "okay";
+		dma-noncoherent;
+		
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					remote-endpoint = <&mipi_csi2_0_output2>;
+				};
+			};
+
+            port@1 {
+                reg = <1>;
+                vvcam_isp_port0_mp: endpoint {
+                    remote-endpoint = <&vvcam_video0>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                vvcam_isp_port0_sp1: endpoint {
+                    remote-endpoint = <&vvcam_video1>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                vvcam_isp_port0_sp2: endpoint {
+                    remote-endpoint = <&vvcam_video2>;
+                };
+            };
+
+            port@4 {
+                reg = <4>;
+                vvcam_isp_port0_raw: endpoint {
+                    remote-endpoint = <&vvcam_video3>;
+                };
+            };
+        };
+    };
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+    vvcam_video_0: vvcam_video@0 {
+        compatible = "verisilicon,video";
+        id = <0>;
+        status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                vvcam_video0: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_mp>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                vvcam_video1: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_sp1>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                vvcam_video2: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_sp2>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                vvcam_video3: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_raw>;
+                };
+            };
+        };
+    };
+};
+
+&d0_i2c5 {
+	status = "okay";
+	/* i2c-sda-hold-time-ns = <1000>; */
+	/* clock-frequency = <0x186a0>; */
+
+	i2c-mux@70 {
+		/*compatible = "ti,tca9548"; */
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+
+		i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0>;
+
+			camera_imx219: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_gpio72_default>;
+				mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+				xvs-gpios = <&portc 6 GPIO_ACTIVE_HIGH>;
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out0: endpoint {
+						remote-endpoint = <&mipidphy0_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+
+		i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x1>;
+
+			camera_imx219_1: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out1: endpoint {
+						remote-endpoint = <&mipidphy1_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+
+		i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x2>;
+
+			camera_imx219_2: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out2: endpoint {
+						remote-endpoint = <&mipidphy2_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+		
+		i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x3>;
+
+			camera_imx219_3: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out3: endpoint {
+						remote-endpoint = <&mipidphy3_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+
+	};
+};
+
+
+&csi2_dphy_hw {
+	lanes = <1 2>;
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out0>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_0_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw1 {
+	lanes = <1 2>;
+	status = "okay";
+};
+&csi2_dphy1 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy1_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out1>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy1_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_1_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw2 {
+	lanes = <1 2>;
+	status = "okay";
+};
+&csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out2>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw3 {
+	lanes = <1 2>;
+	status = "okay";
+};
+&csi2_dphy3 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy3_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out3>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy3_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_3_input>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_0 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_0_input: endpoint@0 {
+				remote-endpoint = <&csidphy0_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_0_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+		
+		port@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_0_output2: endpoint@2 {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+
+&mipi_csi2_1 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_1_input: endpoint@0 {
+				remote-endpoint = <&csidphy1_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_1_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in1>;
+			};
+		};
+	};
+};
+
+
+&mipi_csi2_2 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+
+		
+	};
+};
+
+
+&mipi_csi2_3 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_3_input: endpoint@0 {
+				remote-endpoint = <&csidphy3_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_3_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in3>;
+			};
+		};
+	};
+};
+
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	dvp2axi_id = <0>;
+	port@0 {
+		axi2dvp_mipi2_in0: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_0_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds1 {
+	status = "okay";
+	dvp2axi_id = <1>;
+
+	port@0 {
+		axi2dvp_mipi2_in1: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_1_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds3 {
+	status = "okay";
+	dvp2axi_id = <3>;
+
+	port@0 {
+		axi2dvp_mipi2_in3: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_3_output0>;
+		};
+	};
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-d314.dts b/arch/riscv/boot/dts/eswin/eic7700-d314.dts
new file mode 100644
index 000000000000..685702bdb4f4
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-d314.dts
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#include "eic7700-som314.dtsi"
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN 7700D314";
+
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&pcie {
+	/* GPIO12 PCIE PRSNT input */
+};
+
+&d0_usbdrd_dwc3_1 {
+	usb-hub {
+		gpio-hog;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio43_default>;
+		gpios = <&portb 11 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "usb-hub-reset";
+	};
+};
+
+/*
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio82_default &pinctrl_gpio85_default>;
+	backlight0-gpios = <&portc 18 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portc 21 GPIO_ACTIVE_HIGH>;
+};
+*/
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	/* connect WIFI module */
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x33>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	non-removable;
+	keep-power-in-suspend;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio79_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-1 = <&pinctrl_gpio79_default>;
+		gpio_wl_reg_on = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_uart1 {
+	/* M.2 KEY E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* connect MCU */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* pin header mux with GPIO 92,93 */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* pin header mux with GPIO 35,36,37,38,39,40 */
+	status = "disabled";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		eswin-plat = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio0_default &pinctrl_gpio28_default>;
+		front-jack-gpios = <&porta 0 GPIO_ACTIVE_HIGH>;
+		back-jack-gpios = <&porta 28 GPIO_ACTIVE_HIGH>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* pin header mux with GPIO 46,47 */
+	status = "disabled";
+};
+
+&d0_i2c2 {
+	/* mipi dsi touch ctrl con */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* FUSB303B cc logic */
+	status = "okay";
+	fusb303b@21 {
+		compatible = "fcs,fusb303b";
+		status = "okay";
+		reg = <0x21>;
+		eswin,syscfg = <&d0_sys_con 0x3C0 12>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio5_default>;
+		int-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "host";
+		};
+	};
+
+};
+
+&d0_i2c4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* PCA9548 */
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* AT24C02C EEPROM */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	aon_eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* PCA9450 & SiC451 & INA226 & PAC1934 */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 15>;
+	i2c-sda-hold-time-ns = <0x40>;
+	pac1934:pmic@10 {
+		compatible = "microchip,pac1934";
+		/*update all register data*/
+		update_time_ms = <1000>;
+		eswin,chan_label = "som vdd", "soc vdd", "cpu vdd", "ddr lpvdd";
+		label = "som_info";
+		/*The update number of times the energy accumulates*/
+		energy_acc_count = <0>;
+		shunt_resistors=<1 1 1 1>;
+		reg = <0x10>;
+	};
+
+	sys_power:ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "sys_power";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+
+	mpq8785@12 {
+		compatible = "mps,mpq8785";
+		reg = <0x12>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6_default &pinctrl_gpio7_default &pinctrl_gpio8_default &pinctrl_gpio9_default
+			&pinctrl_gpio10_default &pinctrl_gpio17_default &pinctrl_gpio35_default &pinctrl_gpio36_default
+			&pinctrl_gpio37_default &pinctrl_gpio38_default &pinctrl_gpio39_default &pinctrl_gpio40_default
+			&pinctrl_gpio41_default &pinctrl_gpio46_default &pinctrl_gpio52_default
+			&pinctrl_gpio53_default &pinctrl_gpio64_default &pinctrl_gpio65_default &pinctrl_gpio66_default
+			&pinctrl_gpio67_default &pinctrl_gpio70_default &pinctrl_gpio73_default &pinctrl_gpio83_default
+			&pinctrl_gpio86_default &pinctrl_gpio87_default &pinctrl_gpio92_default &pinctrl_gpio93_default
+			&pinctrl_gpio94_default>;
+
+    /* pin header default function for GPIO
+       SPI1 (CS0,SCLK,MOSI,MISO,D2,D3): GPIO 35,36,37,38,39,40
+       I2C1 (SCL,SDA): GPIO 46,47
+       UART3(TX,RX): GPIO 92,93
+    */
+};
+
+/*
+GPIO USED ON CarrierBoard:
+	gpio0   : FP Audio Jack Sense(I), active low
+	gpio5   : TYPE C cc logic interrupt(I), active low
+	gpio11  : BT WAKE HOST(I), active low
+	gpio12  : PCIE present(I), active low
+	gpio14  : DSI FPC CON CTRL(J10&J11)
+	gpio15  : Wlan wake host(I), active low
+	gpio28  : RP audio jack sense(I), active low
+	gpio29  : EMMC active led ctrl(O)
+
+	gpio43  : USB3.2 Gen1 hub Resetn(O), active low
+	gpio71  : CSI fpc con ctrl(O)
+	gpio74  : CSI fpc con ctrl(O)
+	gpio77  : CSI fpc con ctrl(O)
+	gpio76  : HOST WAKE BT(O), active low
+	gpio79  : WLAN POWER ON(O), active high
+	gpio80  : CSI fpc con ctrl(O)
+	gpio82  : DSI FPC CON CTRL(J10)
+	gpio85  : DSI FPC CON CTRL(J11)
+	gpio84  : GPIO LED CTRL(O), active high
+*/
diff --git a/arch/riscv/boot/dts/eswin/eic7700-dvb-dsi-panel.dtsi b/arch/riscv/boot/dts/eswin/eic7700-dvb-dsi-panel.dtsi
new file mode 100644
index 000000000000..e520a4e89746
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-dvb-dsi-panel.dtsi
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for pin control of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+&SOC {
+	panel_disp: panel_disp {
+		compatible = "raspberrypi,7inch-dsi";
+		status = "okay";
+		backlight = <&reg_display>;
+		power-supply = <&reg_display>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&bridge_out>;
+			};
+		};
+	};
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+	};
+};
+
+&dsi_controller {
+	status = "okay";
+	dependencies = <&reg_display>;
+	dependencies = <&toshiba_bridge>;
+	dependencies = <&panel_disp>;
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&bridge_in>;
+			};
+		};
+	};
+
+	toshiba_bridge: bridge@0 {
+		reg = <0>;
+		compatible = "toshiba,tc358762";
+		status = "okay";
+		vddc-supply = <&reg_display>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				bridge_in: endpoint {
+					remote-endpoint = <&mipi_dsi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				bridge_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+	};
+};
+
+&video_output {
+	status = "okay";
+	power-supply = <&reg_display>;
+};
+
+&d0_i2c5 {
+	status = "okay";
+	/* i2c-sda-hold-time-ns = <1000>; */
+	/* clock-frequency = <0x186a0>; */
+
+	pca9548: i2c-mux@70 {
+		/*compatible = "ti,tca9548"; */
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+
+		i2c@7 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x7>;
+			// reg_display = &reg_display;
+
+			reg_display: regulator@45 {
+				compatible = "raspberrypi,7inch-touchscreen-panel-regulator";
+				reg = <0x45>;
+				gpio-controller;
+				#gpio-cells = <2>;
+			};
+
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-a2-mipi.dts b/arch/riscv/boot/dts/eswin/eic7700-evb-a2-mipi.dts
new file mode 100644
index 000000000000..23a690e1a4d5
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-a2-mipi.dts
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#include "eic7700-evb-a2.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+	};
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "disabled";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			dai-format = "i2s";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-a2.dts b/arch/riscv/boot/dts/eswin/eic7700-evb-a2.dts
new file mode 100644
index 000000000000..3498a1434764
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-a2.dts
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#include "eic7700-evb-a2.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dpi1: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+
+&dw_hdmi_hdcp2 {
+   status = "okay";
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-a2.dtsi b/arch/riscv/boot/dts/eswin/eic7700-evb-a2.dtsi
new file mode 100644
index 000000000000..a6bc1ae50b9e
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-a2.dtsi
@@ -0,0 +1,908 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include "eic7700-evb-camera.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 EVB A2";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			//alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+//&d0_graphcard2 {
+//	status = "okay";
+//	label = "HDMI Audio";
+//	dais = <&d0_i2s0_port>;
+//};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+// &mipi_dphy_rx {
+// 	status = "okay";
+// };
+
+// &csi_dma0 {
+// 	status = "okay";
+// };
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+// &csi2_0 {
+// 	status = "okay";
+// };
+
+// &csi2_1 {
+// 	status = "disabled";
+// };
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio34_default>;
+		gpio_wl_reg_on = <&portb 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0xA0205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0xA0205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46204620 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* mpq8785 */
+	status = "okay";
+	mpq8785@10 {
+		compatible = "mps,mpq8785";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+			
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <10 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio10_default>;
+		irq-gpios = <&porta 10 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+// &d0_i2c3 {
+// 	/* mipi csi0/csi1 */
+// 	status = "okay";
+// };
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "okay";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@48 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "dc_vdd";
+			reg = <0x48>;
+			shunt-resistor = <1000>;
+		};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
\ No newline at end of file
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-a3-mipi.dts b/arch/riscv/boot/dts/eswin/eic7700-evb-a3-mipi.dts
new file mode 100644
index 000000000000..8d5029ac2aa1
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-a3-mipi.dts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eic7700-evb-a2-mipi.dts"
+
+/ {
+	model = "ESWIN EIC7700 EVB A3";
+};
\ No newline at end of file
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-a3.dts b/arch/riscv/boot/dts/eswin/eic7700-evb-a3.dts
new file mode 100644
index 000000000000..4f511e5d7369
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-a3.dts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eic7700-evb-a2.dts"
+
+/ {
+	model = "ESWIN EIC7700 EVB A3";
+};
\ No newline at end of file
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7700-evb-camera.dtsi
new file mode 100644
index 000000000000..c4dc49598f39
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-camera.dtsi
@@ -0,0 +1,534 @@
+// #include "eswin-win2030-die0-soc.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+&vi_subsys {
+	board_compat = "eic7700,evb_vi";
+	eswin,isp_dvp0_hor = <1920>;
+	eswin,isp_dvp0_ver = <1080>;
+	phy_mode = <2>; //5: 6x2lanes, 2: 3x4lanes
+	vvcam_isp_0: vvcam_isp@0 {
+		compatible = "verisilicon,isp-v4l2";
+		id = <0>;
+		status = "okay";
+		dma-noncoherent;
+		depends-on = <&imx327_0>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&mipi_csi2_output1>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+				vvcam_isp_port0_mp: endpoint {
+					remote-endpoint = <&vvcam_video0>;
+				};
+			};
+
+			port@2 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <2>;
+				vvcam_isp_port0_sp1: endpoint {
+					remote-endpoint = <&vvcam_video1>;
+				};
+			};
+
+			port@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+				vvcam_isp_port0_sp2: endpoint {
+					remote-endpoint = <&vvcam_video2>;
+				};
+			};
+
+			port@4 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <4>;
+				vvcam_isp_port0_raw: endpoint {
+					remote-endpoint = <&vvcam_video3>;
+				};
+			};
+		};
+	};
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "okay";
+		depends-on = <&imx327_0>;
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+	vvcam_video_0: vvcam_video@0 {
+		compatible = "verisilicon,video";
+		id = <0>;
+		status = "okay";
+		#size-cells = <2>;
+		depends-on = <&imx327_0>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_video0: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_mp>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+				vvcam_video1: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp1>;
+				};
+			};
+
+			port@2 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <2>;
+				vvcam_video2: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp2>;
+				};
+			};
+
+			port@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+				vvcam_video3: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_raw>;
+				};
+			};
+		};
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+	imx327_0: imx327@36 {
+			compatible = "sony,imx327";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			#reg = <0x0 0x36 0x0 0x0>;
+			reg = <0x36>;
+			clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>; 
+			clock-frequency = <37125000>;
+			#assigned-clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clock-rates = <37125000>;
+			clock-names = "xvclk";
+			/* Reset and power down GPIOs controlled by tca6416_0 */
+			reset-gpios = <&tca6416_0 1 GPIO_ACTIVE_HIGH>;  // GPIO number 1 for MIPI_CSI0_RESET
+			pwdn-gpios = <&tca6416_0 0 GPIO_ACTIVE_HIGH>; // GPIO number 0 for MIPI_CSI0_PWDN
+			enable-gpios = <&tca6416_0 7 GPIO_ACTIVE_HIGH>;
+			// reset-gpios = <&portd 1 GPIO_ACTIVE_HIGH>;  // GPIO number 1 for MIPI_CSI0_RESET
+			// pwdn-gpios = <&portd 6 GPIO_ACTIVE_HIGH>; // GPIO number 0 for MIPI_CSI0_PWDN
+			//pinctrl-names = "default";
+			//pinctrl-0 = <&cif_clkout>;
+			port {
+				imx327_out0: endpoint {
+					// remote-endpoint = <&mipi_in_ucam0>;
+					remote-endpoint = <&mipidphy0_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <445500000>;
+				};
+			};
+		};
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "okay";
+	imx327_2: imx327@36 {
+			compatible = "sony,imx327";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			#reg = <0x0 0x36 0x0 0x0>;
+			reg = <0x36>;
+			clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>; 
+			clock-frequency = <37125000>;
+			#assigned-clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clock-rates = <37125000>;
+			clock-names = "xvclk";
+			/* Reset and power down GPIOs controlled by tca6416_0 */
+			reset-gpios = <&tca6416_0 1 GPIO_ACTIVE_HIGH>;  // GPIO number 1 for MIPI_CSI0_RESET
+			pwdn-gpios = <&tca6416_0 0 GPIO_ACTIVE_HIGH>; // GPIO number 0 for MIPI_CSI0_PWDN
+			enable-gpios = <&tca6416_0 7 GPIO_ACTIVE_HIGH>;
+			// reset-gpios = <&portd 1 GPIO_ACTIVE_HIGH>;  // GPIO number 1 for MIPI_CSI0_RESET
+			// pwdn-gpios = <&portd 6 GPIO_ACTIVE_HIGH>; // GPIO number 0 for MIPI_CSI0_PWDN
+			//pinctrl-names = "default";
+			//pinctrl-0 = <&cif_clkout>;
+			
+			port {
+				imx327_out2: endpoint {
+					// remote-endpoint = <&mipi_in_ucam0>;
+					remote-endpoint = <&mipidphy2_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <445500000>;
+				};
+			};
+		};
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "okay";
+	imx327_4: imx327@36 {
+			compatible = "sony,imx327";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			#reg = <0x0 0x36 0x0 0x0>;
+			reg = <0x36>;
+			clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>; 
+			clock-frequency = <37125000>;
+			#assigned-clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clock-rates = <37125000>;
+			clock-names = "xvclk";
+			/* Reset and power down GPIOs controlled by tca6416_0 */
+			reset-gpios = <&tca6416_0 1 GPIO_ACTIVE_HIGH>;  // GPIO number 1 for MIPI_CSI0_RESET
+			pwdn-gpios = <&tca6416_0 0 GPIO_ACTIVE_HIGH>; // GPIO number 0 for MIPI_CSI0_PWDN
+			enable-gpios = <&tca6416_0 7 GPIO_ACTIVE_HIGH>;
+			// reset-gpios = <&portd 1 GPIO_ACTIVE_HIGH>;  // GPIO number 1 for MIPI_CSI0_RESET
+			// pwdn-gpios = <&portd 6 GPIO_ACTIVE_HIGH>; // GPIO number 0 for MIPI_CSI0_PWDN
+			//pinctrl-names = "default";
+			//pinctrl-0 = <&cif_clkout>;
+			
+			port {
+				imx327_out4: endpoint {
+					// remote-endpoint = <&mipi_in_ucam0>;
+					remote-endpoint = <&mipidphy4_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <445500000>;
+				};
+			};
+		};
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+	tca6416_0: gpio@20 {
+		compatible = "ti,tca6416";
+		clock-frequency = <100000>;
+		reg = <0x20>;
+		gpio-controller; /* IRQ not connected */
+		#gpio-cells = <2>;
+		gpio-line-names = "MIPI_CSI0_PWDN", "MIPI_CSI0_RESET", "MIPI_CSI1_FBC", "MIPI_CSI1_ENB",
+				"MIPI_CSI1_RESET", "MIPI_CSI1_PWDN", "FREX_GP0", "",
+				"MIPI_CSI0_ENB", "MIPI_CSI0_FBC", "FREX_GP2", "MIPI_CSI2_FBC",
+				"MIPI_CSI2_ENB", "FREX_GP1", "MIPI_CSI2_RESET", "MIPI_CSI2_PWDN";
+		//vcc = <&gpio_vcc0>;
+		status = "okay";
+	};
+	tca6416_1: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		/* IRQ not connected */
+		gpio-line-names = "MIPI_CSI3_PWDN", "MIPI_CSI3_RESET", "MIPI_CSI3_ENB", "MIPI_CSI3_FBC",
+				"MIPI_CSI4_PWDN", "MIPI_CSI4_RESET", "MIPI_CSI4_ENB", "MIPI_CSI4_FBC",
+				"MIPI_CSI5_FBC", "MIPI_CSI5_ENB", "MIPI_CSI5_RESET", "MIPI_CSI5_PWDN",
+				"", "", "", "";
+		//vcc = <&gpio_vcc1>;
+		status = "okay";
+	};
+};
+
+&csi2_dphy_hw {
+	lanes = <1 2 3 4>;
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint {
+				remote-endpoint = <&imx327_out0>;
+				data-lanes = <1 2>;
+			};
+		};
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint {
+				remote-endpoint = <&mipi_csi2_input>;
+			};
+		};
+
+	};
+};
+
+&csi2_dphy_hw2 {
+	lanes = <1 2 3 4>;
+	status = "okay";
+};
+&csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx327_out2>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw4 {
+	lanes = <1 2 3 4>;
+	status = "okay";
+};
+&csi2_dphy4 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy4_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx327_out4>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy4_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_4_input>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_0 {
+	status = "okay";
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;	
+			reg = <0>;
+			mipi_csi2_input: endpoint {
+				remote-endpoint = <&csidphy0_out>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_csi2_output: endpoint {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+
+		port@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+			mipi_csi2_output1: endpoint {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_2 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+	};
+};
+
+
+&mipi_csi2_4 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_4_input: endpoint@0 {
+				remote-endpoint = <&csidphy4_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_4_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in4>;
+			};
+		};
+	};
+};
+
+
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	dvp2axi_id = <0>;
+	ports{
+		#address-cells = <1>;
+        #size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+        	#size-cells = <0>;
+			reg = <0>;
+			axi2dvp_mipi2_in0: endpoint {
+				remote-endpoint = <&mipi_csi2_output>;
+			};
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds1 {
+	dvp2axi_id = <1>;
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds3 {
+	dvp2axi_id = <3>;
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds4 {
+	dvp2axi_id = <4>;
+	status = "okay";
+	port@0 {
+		axi2dvp_mipi2_in4: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_4_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds5 {
+	dvp2axi_id = <5>;
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+	depends-on = <&imx327_0>;
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb-fccsp-camera-imx219.dtsi b/arch/riscv/boot/dts/eswin/eic7700-evb-fccsp-camera-imx219.dtsi
new file mode 100644
index 000000000000..87da239d0bcb
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb-fccsp-camera-imx219.dtsi
@@ -0,0 +1,487 @@
+// #include "eswin-win2030-die0-soc.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+&vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+	phy_mode = <2>; //5: 6x2lanes, 2: 3x4lanes
+	vvcam_isp_0: vvcam_isp@0 {
+		compatible = "verisilicon,isp-v4l2";
+		id = <0>;
+		status = "okay";
+		dma-noncoherent;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&mipi_csi2_output1>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+				vvcam_isp_port0_mp: endpoint {
+					remote-endpoint = <&vvcam_video0>;
+				};
+			};
+
+			port@2 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <2>;
+				vvcam_isp_port0_sp1: endpoint {
+					remote-endpoint = <&vvcam_video1>;
+				};
+			};
+
+			port@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+				vvcam_isp_port0_sp2: endpoint {
+					remote-endpoint = <&vvcam_video2>;
+				};
+			};
+
+			port@4 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <4>;
+				vvcam_isp_port0_raw: endpoint {
+					remote-endpoint = <&vvcam_video3>;
+				};
+			};
+		};
+	};
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+	vvcam_video_0: vvcam_video@0 {
+		compatible = "verisilicon,video";
+		id = <0>;
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_video0: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_mp>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+				vvcam_video1: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp1>;
+				};
+			};
+
+			port@2 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <2>;
+				vvcam_video2: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp2>;
+				};
+			};
+
+			port@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+				vvcam_video3: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_raw>;
+				};
+			};
+		};
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+	imx219_0: imx219@10 {
+			compatible = "sony,imx219";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			reg = <0x10>;
+			clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clock-rates = <24000000>;
+			clock-names = "xvclk";
+			mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+			eswin,camera-module-index = <0>;
+			eswin,camera-module-facing = "back";
+			eswin,camera-module-name = "rpi-camera-v2";
+			eswin,camera-module-lens-name = "default";
+
+			port {
+				imx219_out0: endpoint {
+					remote-endpoint = <&mipidphy0_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <364000000>;
+				};
+			};
+		};
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "okay";
+	imx219_2: imx219@10 {
+			compatible = "sony,imx219";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			reg = <0x10>;
+			clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clock-rates = <24000000>;
+			clock-names = "xvclk";
+			mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+			eswin,camera-module-index = <0>;
+			eswin,camera-module-facing = "back";
+			eswin,camera-module-name = "rpi-camera-v2";
+			eswin,camera-module-lens-name = "default";
+
+			port {
+				imx219_out2: endpoint {
+					remote-endpoint = <&mipidphy2_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <364000000>;
+				};
+			};
+		};
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "okay";
+	imx219_4: imx219@10 {
+			compatible = "sony,imx219";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			reg = <0x10>;
+			clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clocks = <&d0_clock EIC7700_CLK_VI_SHUTTER_0>;
+			assigned-clock-rates = <24000000>;
+			clock-names = "xvclk";
+			mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+			eswin,camera-module-index = <0>;
+			eswin,camera-module-facing = "back";
+			eswin,camera-module-name = "rpi-camera-v2";
+			eswin,camera-module-lens-name = "default";
+
+			port {
+				imx219_out4: endpoint {
+					remote-endpoint = <&mipidphy4_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <364000000>;
+				};
+			};
+		};
+};
+
+&csi2_dphy_hw {
+	lanes = <1 2 3 4>;
+	lanes-dp-dn = <0 0 0 0>;	// 4lanes: positive on dn/ negative on dp to hs and lp mode
+	rate = <720>;				//720Mbps
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint {
+				remote-endpoint = <&imx219_out0>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint {
+				remote-endpoint = <&mipi_csi2_input>;
+			};
+		};
+
+	};
+};
+
+&csi2_dphy_hw2 {
+	lanes = <1 2 3 4>;
+	lanes-dp-dn = <0 0 0 0>;	// 4lanes: positive on dn/ negative on dp to hs and lp mode
+	rate = <720>; 				//720Mbps
+	status = "okay";
+};
+&csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out2>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw4 {
+	lanes = <1 2 3 4>;
+	lanes-dp-dn = <0 0 0 0>;	// 4lanes: positive on dn/ negative on dp to hs and lp mode
+	rate = <720>; 				//720Mbps
+	status = "okay";
+};
+&csi2_dphy4 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy4_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out4>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy4_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_4_input>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_0 {
+	status = "okay";
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;	
+			reg = <0>;
+			mipi_csi2_input: endpoint {
+				remote-endpoint = <&csidphy0_out>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_csi2_output: endpoint {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+
+		port@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+			mipi_csi2_output1: endpoint {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_2 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_4 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_4_input: endpoint@0 {
+				remote-endpoint = <&csidphy4_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_4_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in4>;
+			};
+		};
+	};
+};
+
+
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	dvp2axi_id = <0>;
+	ports{
+		#address-cells = <1>;
+        #size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+        	#size-cells = <0>;
+			reg = <0>;
+			axi2dvp_mipi2_in0: endpoint {
+				remote-endpoint = <&mipi_csi2_output>;
+			};
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds1 {
+	dvp2axi_id = <1>;
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds3 {
+	dvp2axi_id = <3>;
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds4 {
+	dvp2axi_id = <4>;
+	status = "okay";
+	port@0 {
+		axi2dvp_mipi2_in4: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_4_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds5 {
+	dvp2axi_id = <5>;
+	status = "okay";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+
diff --git a/arch/riscv/boot/dts/eswin/eic7700-evb.dts b/arch/riscv/boot/dts/eswin/eic7700-evb.dts
new file mode 100644
index 000000000000..6a21f7ff0c43
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-evb.dts
@@ -0,0 +1,944 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 EVB A1";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio106_default>;
+	pci-socket-gpios = <&portd 10 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+    status = "disabled";
+};
+
+&virtual_display {
+    status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio7_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 7 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint0>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	io_reuse_enable;
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d0_i2s0_port0: port@0 {
+			reg = <0>;
+			d0_i2s0_endpoint0: endpoint {
+				remote-endpoint = <&hdmi_in_i2s>;
+				dai-format = "i2s";
+			};
+		};
+		d0_i2s0_port1: port@1 {
+			reg = <1>;
+			d0_i2s0_endpoint1: endpoint {
+				remote-endpoint = <&d0_codec0_endpoint>;
+				dai-format = "i2s";
+			};
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec2_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port0>, <&d0_i2s0_port1>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6251 0x6251 0x6251>;
+	dly-param-1000m = <0x20202020 0xAA205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6251 0x6251 0x6251>;
+	dly-param-1000m = <0x20202020 0x46204620 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* have cts & rts */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* rs485 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	status = "disabled";
+};
+
+&d0_uart4 {
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	eswin,pwm_inverted;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint1>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_2: es8388-2@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec2_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* mpq8785 & ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	mpq8785@10 {
+		compatible = "mps,mpq8785";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@48 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "dc_vdd";
+			reg = <0x48>;
+			shunt-resistor = <1000>;
+		};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/*
+	gpio function unused
+	group A: gpio1~6,12~15, 17~27, 29~31
+	group B: gpio32~63
+	group C: gpio64~93, 95
+	group D: gpio96~105
+
+	gpio0   : head phone plug/unplug detection0(I)
+	gpio7   : led back light power on/off(O)
+	gpio8   : dsi touch interrupt(I)
+	gpio9   : pcie present(I)
+	gpio10  : software define key input(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio16  : gphy1 resern(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio94  : gphy0 resern(O)
+	gpio106 : pcie socket power on/off(O)
+	gpio107 : tf card power on/off(O)
+	gpio108 : system led0(O)
+	gpio109 : system led1(O)
+	gpio110 : system led2(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550-camera.dtsi
new file mode 100644
index 000000000000..a2ecef0ec3c8
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550-camera.dtsi
@@ -0,0 +1,569 @@
+&vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+	phy_mode = <5>; //5: 6x2lanes, 2: 3x4lanes
+    vvcam_isp_0: vvcam_isp@0 {
+        compatible = "verisilicon,isp-v4l2";
+        id = <0>;
+        status = "okay";
+		dma-noncoherent;
+		
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					remote-endpoint = <&mipi_csi2_0_output2>;
+				};
+			};
+
+            port@1 {
+                reg = <1>;
+                vvcam_isp_port0_mp: endpoint {
+                    remote-endpoint = <&vvcam_video0>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                vvcam_isp_port0_sp1: endpoint {
+                    remote-endpoint = <&vvcam_video1>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                vvcam_isp_port0_sp2: endpoint {
+                    remote-endpoint = <&vvcam_video2>;
+                };
+            };
+
+            port@4 {
+                reg = <4>;
+                vvcam_isp_port0_raw: endpoint {
+                    remote-endpoint = <&vvcam_video3>;
+                };
+            };
+        };
+    };
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+    vvcam_video_0: vvcam_video@0 {
+        compatible = "verisilicon,video";
+        id = <0>;
+        status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                vvcam_video0: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_mp>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                vvcam_video1: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_sp1>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                vvcam_video2: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_sp2>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                vvcam_video3: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_raw>;
+                };
+            };
+        };
+    };
+};
+
+&d0_i2c5 {
+	status = "okay";
+	/* i2c-sda-hold-time-ns = <1000>; */
+	/* clock-frequency = <0x186a0>; */
+
+	i2c-mux@70 {
+		/*compatible = "ti,tca9548"; */
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+
+		i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0>;
+
+			camera_imx219: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_gpio72_default>;
+				mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+				xvs-gpios = <&portc 6 GPIO_ACTIVE_HIGH>;
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out0: endpoint {
+						remote-endpoint = <&mipidphy0_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+
+		i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x1>;
+
+			camera_imx219_1: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out1: endpoint {
+						remote-endpoint = <&mipidphy1_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+
+		i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x2>;
+
+			camera_imx219_2: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out2: endpoint {
+						remote-endpoint = <&mipidphy2_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+		
+		i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x3>;
+
+			camera_imx219_3: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				port {
+					imx219_out3: endpoint {
+						remote-endpoint = <&mipidphy3_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+		};
+
+	};
+};
+
+
+&csi2_dphy_hw {
+	lanes = <1 2>;
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out0>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_0_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw1 {
+	lanes = <1 2>;
+	status = "okay";
+};
+&csi2_dphy1 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy1_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out1>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy1_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_1_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw2 {
+	lanes = <1 2>;
+	status = "okay";
+};
+&csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out2>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw3 {
+	lanes = <1 2>;
+	status = "okay";
+};
+&csi2_dphy3 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy3_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out3>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy3_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_3_input>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_0 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_0_input: endpoint@0 {
+				remote-endpoint = <&csidphy0_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_0_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+		
+		port@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_0_output2: endpoint@2 {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+
+&mipi_csi2_1 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_1_input: endpoint@0 {
+				remote-endpoint = <&csidphy1_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_1_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in1>;
+			};
+		};
+	};
+};
+
+
+&mipi_csi2_2 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+
+		
+	};
+};
+
+
+&mipi_csi2_3 {
+	num_lanes = <2>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_3_input: endpoint@0 {
+				remote-endpoint = <&csidphy3_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_3_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in3>;
+			};
+		};
+	};
+};
+
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	dvp2axi_id = <0>;
+	port@0 {
+		axi2dvp_mipi2_in0: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_0_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds1 {
+	status = "okay";
+	dvp2axi_id = <1>;
+
+	port@0 {
+		axi2dvp_mipi2_in1: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_1_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds3 {
+	status = "okay";
+	dvp2axi_id = <3>;
+
+	port@0 {
+		axi2dvp_mipi2_in3: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_3_output0>;
+		};
+	};
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550-npu.dts b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550-npu.dts
new file mode 100644
index 000000000000..91d3346d761d
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550-npu.dts
@@ -0,0 +1,902 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+//#include "eic7700-dvb-dsi-panel.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "SiFive HiFive Premier P550";
+	compatible = "sifive,hifive-premier-p550", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+		 	alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	/* GPIO12 PCIE PRSNT input */
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dpi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	/* connect M.2 KEY E */
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	/* connect WIFI module */
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	non-removable;
+	keep-power-in-suspend;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio79_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio79_default>;
+		gpio_wl_reg_on = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_gmac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio106_default>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_gmac1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio111_default>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46204620 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+	usb-hub {
+		gpio-hog;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio43_default>;
+		gpios = <&portb 11 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "usb-hub-reset";
+	};
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* M.2 KEY E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* connect MCU */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* pin header mux with GPIO 92,93 */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* pin header mux with GPIO 35,36,37,38,39,40 */
+	status = "disabled";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	pwms = <&pwm0 0 100000 PWM_POLARITY_INVERTED>;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		eswin-plat = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio0_default &pinctrl_gpio28_default>;
+		front-jack-gpios = <&porta 0 GPIO_ACTIVE_HIGH>;
+		back-jack-gpios = <&porta 28 GPIO_ACTIVE_HIGH>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* pin header mux with GPIO 46,47 */
+	status = "disabled";
+};
+
+&d0_i2c2 {
+	/* mipi dsi touch ctrl con */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* FUSB303B cc logic */
+	status = "okay";
+	fusb303b@21 {
+		compatible = "fcs,fusb303b";
+		status = "okay";
+		reg = <0x21>;
+		eswin,syscfg = <&d0_sys_con 0x3C0 12>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio5_default>;
+		int-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "host";
+		};
+	};
+
+};
+
+&d0_i2c4 {
+	/* unused */
+	status = "disabled";
+};
+
+/* &d0_i2c5 {
+	// PCA9548
+	status = "okay";
+}; */
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* AT24C02C EEPROM */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	aon_eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* PCA9450 & SiC451 & INA226 & PAC1934 */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 15>;
+	i2c-sda-hold-time-ns = <0x40>;
+	pac1934:pmic@10 {
+		compatible = "microchip,pac1934";
+		/*update all register data*/
+		update_time_ms = <1000>;
+		eswin,chan_label = "som vdd", "soc vdd", "cpu vdd", "ddr lpvdd";
+		label = "som_info";
+		/*The update number of times the energy accumulates*/
+		energy_acc_count = <0>;
+		shunt_resistors=<1 1 1 1>;
+		reg = <0x10>;
+	};
+
+	sys_power:ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "sys_power";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+
+	mpq8785@12 {
+		compatible = "mps,mpq8785";
+		reg = <0x12>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6_default &pinctrl_gpio7_default &pinctrl_gpio8_default &pinctrl_gpio9_default
+			&pinctrl_gpio10_default &pinctrl_gpio17_default &pinctrl_gpio35_default &pinctrl_gpio36_default
+			&pinctrl_gpio37_default &pinctrl_gpio38_default &pinctrl_gpio39_default &pinctrl_gpio40_default
+			&pinctrl_gpio41_default &pinctrl_gpio46_default &pinctrl_gpio52_default
+			&pinctrl_gpio53_default &pinctrl_gpio64_default &pinctrl_gpio65_default &pinctrl_gpio66_default
+			&pinctrl_gpio67_default &pinctrl_gpio70_default &pinctrl_gpio73_default &pinctrl_gpio83_default
+			&pinctrl_gpio86_default &pinctrl_gpio87_default &pinctrl_gpio92_default &pinctrl_gpio93_default
+			&pinctrl_gpio94_default>;
+
+    /* pin header default function for GPIO
+       SPI1 (CS0,SCLK,MOSI,MISO,D2,D3): GPIO 35,36,37,38,39,40
+       I2C1 (SCL,SDA): GPIO 46,47
+       UART3(TX,RX): GPIO 92,93
+    */
+};
+
+/*
+GPIO USED ON CarrierBoard:
+	gpio0   : FP Audio Jack Sense(I), active low
+	gpio5   : TYPE C cc logic interrupt(I), active low
+	gpio11  : BT WAKE HOST(I), active low
+	gpio12  : PCIE present(I), active low
+	gpio14  : DSI FPC CON CTRL(J10&J11)
+	gpio15  : Wlan wake host(I), active low
+	gpio28  : RP audio jack sense(I), active low
+	gpio29  : EMMC active led ctrl(O)
+
+	gpio43  : USB3.2 Gen1 hub Resetn(O), active low
+	gpio71  : CSI fpc con ctrl(O)
+	gpio74  : CSI fpc con ctrl(O)
+	gpio77  : CSI fpc con ctrl(O)
+	gpio76  : HOST WAKE BT(O), active low
+	gpio79  : WLAN POWER ON(O), active high
+	gpio80  : CSI fpc con ctrl(O)
+	gpio82  : DSI FPC CON CTRL(J10)
+	gpio85  : DSI FPC CON CTRL(J11)
+	gpio84  : GPIO LED CTRL(O), active high
+
+ GPIO USED ON SOM:
+	gpio18  : HOST WAKE WLAN(O), active low
+	gpio19  : HOST WAKE BT(O), active low
+	gpio20  : WLAN WAKE HOST(I), active low
+	gpio21  : BT WAKE HOST(I), active low
+	gpio106 : gphy0 resern(O), active low
+	gpio111 : gphy1 resern(O), active low
+
+*/
+
+&gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
+&d0_clock {
+	status = "okay";
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts
new file mode 100644
index 000000000000..2127b9dc3838
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts
@@ -0,0 +1,898 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+//#include "eic7700-dvb-dsi-panel.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include "eic7700-hifive-premier-p550-camera.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "SiFive HiFive Premier P550";
+	compatible = "sifive,hifive-premier-p550", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+		 	alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+	};
+
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	/* GPIO12 PCIE PRSNT input */
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dpi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	/* connect M.2 KEY E */
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	/* connect WIFI module */
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	non-removable;
+	keep-power-in-suspend;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio79_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio79_default>;
+		gpio_wl_reg_on = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_gmac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio106_default>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_gmac1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio111_default>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46204620 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+	usb-hub {
+		gpio-hog;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio43_default>;
+		gpios = <&portb 11 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "usb-hub-reset";
+	};
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* M.2 KEY E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* connect MCU */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* pin header mux with GPIO 92,93 */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* pin header mux with GPIO 35,36,37,38,39,40 */
+	status = "disabled";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	pwms = <&pwm0 0 100000 PWM_POLARITY_INVERTED>;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		eswin-plat = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio0_default &pinctrl_gpio28_default>;
+		front-jack-gpios = <&porta 0 GPIO_ACTIVE_HIGH>;
+		back-jack-gpios = <&porta 28 GPIO_ACTIVE_HIGH>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* pin header mux with GPIO 46,47 */
+	status = "disabled";
+};
+
+&d0_i2c2 {
+	/* mipi dsi touch ctrl con */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* FUSB303B cc logic */
+	status = "okay";
+	fusb303b@21 {
+		compatible = "fcs,fusb303b";
+		status = "okay";
+		reg = <0x21>;
+		eswin,syscfg = <&d0_sys_con 0x3C0 12>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio5_default>;
+		int-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "host";
+		};
+	};
+
+};
+
+&d0_i2c4 {
+	/* unused */
+	status = "disabled";
+};
+
+/* &d0_i2c5 {
+	// PCA9548
+	status = "okay";
+}; */
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* AT24C02C EEPROM */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	aon_eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* PCA9450 & SiC451 & INA226 & PAC1934 */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 15>;
+	i2c-sda-hold-time-ns = <0x40>;
+	pac1934:pmic@10 {
+		compatible = "microchip,pac1934";
+		/*update all register data*/
+		update_time_ms = <1000>;
+		eswin,chan_label = "som vdd", "soc vdd", "cpu vdd", "ddr lpvdd";
+		label = "som_info";
+		/*The update number of times the energy accumulates*/
+		energy_acc_count = <0>;
+		shunt_resistors=<1 1 1 1>;
+		reg = <0x10>;
+	};
+
+	sys_power:ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "sys_power";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+
+	mpq8785@12 {
+		compatible = "mps,mpq8785";
+		reg = <0x12>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6_default &pinctrl_gpio7_default &pinctrl_gpio8_default &pinctrl_gpio9_default
+			&pinctrl_gpio10_default &pinctrl_gpio17_default &pinctrl_gpio35_default &pinctrl_gpio36_default
+			&pinctrl_gpio37_default &pinctrl_gpio38_default &pinctrl_gpio39_default &pinctrl_gpio40_default
+			&pinctrl_gpio41_default &pinctrl_gpio46_default &pinctrl_gpio52_default
+			&pinctrl_gpio53_default &pinctrl_gpio64_default &pinctrl_gpio65_default &pinctrl_gpio66_default
+			&pinctrl_gpio67_default &pinctrl_gpio70_default &pinctrl_gpio73_default &pinctrl_gpio83_default
+			&pinctrl_gpio86_default &pinctrl_gpio87_default &pinctrl_gpio92_default &pinctrl_gpio93_default
+			&pinctrl_gpio94_default>;
+
+    /* pin header default function for GPIO
+       SPI1 (CS0,SCLK,MOSI,MISO,D2,D3): GPIO 35,36,37,38,39,40
+       I2C1 (SCL,SDA): GPIO 46,47
+       UART3(TX,RX): GPIO 92,93
+    */
+};
+
+/*
+GPIO USED ON CarrierBoard:
+	gpio0   : FP Audio Jack Sense(I), active low
+	gpio5   : TYPE C cc logic interrupt(I), active low
+	gpio11  : BT WAKE HOST(I), active low
+	gpio12  : PCIE present(I), active low
+	gpio14  : DSI FPC CON CTRL(J10&J11)
+	gpio15  : Wlan wake host(I), active low
+	gpio28  : RP audio jack sense(I), active low
+	gpio29  : EMMC active led ctrl(O)
+
+	gpio43  : USB3.2 Gen1 hub Resetn(O), active low
+	gpio71  : CSI fpc con ctrl(O)
+	gpio74  : CSI fpc con ctrl(O)
+	gpio77  : CSI fpc con ctrl(O)
+	gpio76  : HOST WAKE BT(O), active low
+	gpio79  : WLAN POWER ON(O), active high
+	gpio80  : CSI fpc con ctrl(O)
+	gpio82  : DSI FPC CON CTRL(J10)
+	gpio85  : DSI FPC CON CTRL(J11)
+	gpio84  : GPIO LED CTRL(O), active high
+
+ GPIO USED ON SOM:
+	gpio18  : HOST WAKE WLAN(O), active low
+	gpio19  : HOST WAKE BT(O), active low
+	gpio20  : WLAN WAKE HOST(I), active low
+	gpio21  : BT WAKE HOST(I), active low
+	gpio106 : gphy0 resern(O), active low
+	gpio111 : gphy1 resern(O), active low
+
+*/
+
+&gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
+&d0_clock {
+	status = "okay";
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez-npu.dts b/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez-npu.dts
new file mode 100644
index 000000000000..aa4741af7aba
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez-npu.dts
@@ -0,0 +1,987 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Milk-V Megrez";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+	apply_npu_1G_freq;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+        status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x3b>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	disable-wp;
+	broken-cd;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5430@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+	tca6416_0: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller; /* IRQ not connected */
+		#gpio-cells = <2>;
+		gpio-line-names = "MIPI_CSI0_PWDN", "MIPI_CSI0_RESET", "MIPI_CSI1_FBC", "MIPI_CSI1_ENB",
+				"MIPI_CSI1_RESET", "MIPI_CSI1_PWDN", "FREX_GP0", "",
+				"MIPI_CSI0_ENB", "MIPI_CSI0_FBC", "FREX_GP2", "MIPI_CSI2_FBC",
+				"MIPI_CSI2_ENB", "FREX_GP1", "MIPI_CSI2_RESET", "MIPI_CSI2_PWDN";
+	};
+	tca6416_1: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		/* IRQ not connected */
+		gpio-line-names = "MIPI_CSI3_PWDN", "MIPI_CSI3_RESET", "MIPI_CSI3_ENB", "MIPI_CSI3_FBC",
+				"MIPI_CSI4_PWDN", "MIPI_CSI4_RESET", "MIPI_CSI4_ENB", "MIPI_CSI4_FBC",
+				"MIPI_CSI5_FBC", "MIPI_CSI5_ENB", "MIPI_CSI5_RESET", "MIPI_CSI5_PWDN",
+				"", "", "", "";
+	};
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+
+	rtc_m41t11: m41t11@68 {
+		compatible = "st,m41t11";
+		reg = <0x68>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@48 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "dc_vdd";
+			reg = <0x48>;
+			shunt-resistor = <1000>;
+		};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	apply_npu_high_freq;
+};
+
+&d0_clock {
+	status = "okay";
+	force-1_8ghz;
+};
+
+&d0_cpu_opp_table {
+        opp-1500000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1600000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1700000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1800000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez-nx.dts b/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez-nx.dts
new file mode 100644
index 000000000000..f65925c4b0a3
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez-nx.dts
@@ -0,0 +1,964 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Milk-V Megrez NX";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &rtc_m41t11;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio0_default &pinctrl_gpio23_default &pinctrl_gpio25_default &pinctrl_gpio91_default>;
+
+		gpio-0 {
+			gpios = <&porta 0 GPIO_ACTIVE_HIGH>;
+			label = "gpio-0";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-23 {
+			gpios = <&porta 23 GPIO_ACTIVE_HIGH>;
+			label = "gpio-23";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-25 {
+			gpios = <&porta 25 GPIO_ACTIVE_HIGH>;
+			label = "gpio-25";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-91 {
+			gpios = <&portc 27 GPIO_ACTIVE_HIGH>;
+			label = "gpio-91";
+			linux,default-trigger = "default-on";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+	apply_npu_1G_freq;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio44_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&portb 12 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x3b>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	disable-wp;
+	broken-cd;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	eswin,pwm_inverted;
+};
+
+&d0_i2c0 {
+	status = "okay";
+};
+
+&d0_i2c1 {
+	status = "okay";
+
+	es5430@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	status = "ok";
+
+	touchscreen@14 {
+		compatible = "goodix,gt9271";
+		reg = <0x14>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio28_default>;
+		interrupt-parent = <&porta>;
+		interrupts = <28 IRQ_TYPE_EDGE_RISING>;
+		irq-gpios = <&porta 28 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portc 28 GPIO_ACTIVE_HIGH>;
+
+		touchscreen-size-x = <1200>;
+		touchscreen-size-y = <1920>;
+
+		status = "okay";
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	status = "okay";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+
+	rtc_m41t11: m41t11@68 {
+		compatible = "st,m41t11";
+		reg = <0x68>;
+	};
+};
+
+&d0_aon_i2c1 {
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	apply_npu_high_freq;
+};
+
+&d0_clock {
+	status = "okay";
+	force-1_8ghz;
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <
+		&pinctrl_gpio1_default
+		&pinctrl_gpio2_default
+		&pinctrl_gpio3_default
+		&pinctrl_gpio6_default
+		&pinctrl_gpio12_default
+		&pinctrl_gpio18_default
+		&pinctrl_gpio19_default
+		&pinctrl_gpio20_default
+		&pinctrl_gpio21_default
+		&pinctrl_gpio22_default
+		&pinctrl_gpio24_default
+		&pinctrl_gpio26_default
+		&pinctrl_gpio35_default
+		&pinctrl_gpio36_default
+		&pinctrl_gpio37_default
+		&pinctrl_gpio38_default
+		&pinctrl_gpio41_default
+		&pinctrl_gpio89_default
+		&pinctrl_gpio93_default
+		>;
+
+	pinctrl_gpio28_default: gpio28-default{
+		mux {
+			groups = "gpio28_group";
+			function = "gpio28_func";
+		};
+		conf {
+			groups = "gpio28_group";
+			input-enable = <1>;
+			bias-pull-up = <1>;
+		};
+	};
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez.dts b/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez.dts
new file mode 100644
index 000000000000..1c3894b3da8a
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-milkv-megrez.dts
@@ -0,0 +1,981 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Milk-V Megrez";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+	apply_npu_1G_freq;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+        status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x3b>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	disable-wp;
+	broken-cd;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5430@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+	tca6416_0: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller; /* IRQ not connected */
+		#gpio-cells = <2>;
+		gpio-line-names = "MIPI_CSI0_PWDN", "MIPI_CSI0_RESET", "MIPI_CSI1_FBC", "MIPI_CSI1_ENB",
+				"MIPI_CSI1_RESET", "MIPI_CSI1_PWDN", "FREX_GP0", "",
+				"MIPI_CSI0_ENB", "MIPI_CSI0_FBC", "FREX_GP2", "MIPI_CSI2_FBC",
+				"MIPI_CSI2_ENB", "FREX_GP1", "MIPI_CSI2_RESET", "MIPI_CSI2_PWDN";
+	};
+	tca6416_1: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		/* IRQ not connected */
+		gpio-line-names = "MIPI_CSI3_PWDN", "MIPI_CSI3_RESET", "MIPI_CSI3_ENB", "MIPI_CSI3_FBC",
+				"MIPI_CSI4_PWDN", "MIPI_CSI4_RESET", "MIPI_CSI4_ENB", "MIPI_CSI4_FBC",
+				"MIPI_CSI5_FBC", "MIPI_CSI5_ENB", "MIPI_CSI5_RESET", "MIPI_CSI5_PWDN",
+				"", "", "", "";
+	};
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+
+	rtc_m41t11: m41t11@68 {
+		compatible = "st,m41t11";
+		reg = <0x68>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@48 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "dc_vdd";
+			reg = <0x48>;
+			shunt-resistor = <1000>;
+		};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	apply_npu_high_freq;
+};
+
+&d0_clock {
+	status = "okay";
+	force-1_8ghz;
+};
+
+&d0_cpu_opp_table {
+        opp-1500000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1600000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1700000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1800000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-osm-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7700-osm-camera.dtsi
new file mode 100644
index 000000000000..0580f139021a
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-osm-camera.dtsi
@@ -0,0 +1,344 @@
+&vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+	phy_mode = <2>; //5: 6x2lanes, 2: 3x4lanes
+	vvcam_isp_0: vvcam_isp@0 {
+		compatible = "verisilicon,isp-v4l2";
+		id = <0>;
+		status = "okay";
+		dma-noncoherent;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					remote-endpoint = <&mipi_csi2_output1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				vvcam_isp_port0_mp: endpoint {
+					remote-endpoint = <&vvcam_video0>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				vvcam_isp_port0_sp1: endpoint {
+					remote-endpoint = <&vvcam_video1>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				vvcam_isp_port0_sp2: endpoint {
+					remote-endpoint = <&vvcam_video2>;
+				};
+			};
+
+			port@4 {
+				reg = <4>;
+				vvcam_isp_port0_raw: endpoint {
+					remote-endpoint = <&vvcam_video3>;
+				};
+			};
+		};
+	};
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+	vvcam_video_0: vvcam_video@0 {
+		compatible = "verisilicon,video";
+		id = <0>;
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				vvcam_video0: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_mp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				vvcam_video1: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp1>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				vvcam_video2: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp2>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				vvcam_video3: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_raw>;
+				};
+			};
+		};
+	};
+};
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	dvp2axi_id = <0>;
+	port@0 {
+		axi2dvp_mipi2_in0: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds1 {
+	status = "disabled";
+	dvp2axi_id = <1>;
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "disabled";
+	dvp2axi_id = <2>;
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+
+
+&mipi_csi2_0 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi0_csi2_input: endpoint@0 {
+				remote-endpoint = <&csidphy0_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_output1: endpoint@2 {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_2 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+	};
+};
+
+
+
+
+&csi2_dphy_hw {
+	lanes = <1 2 3 4>;
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out0>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi0_csi2_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw2 {
+	lanes = <1 2 3 4>;
+	status = "disabled";
+};
+
+&csi2_dphy2 {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out2>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	imx219_0: imx219@10 {
+		compatible = "sony,imx219";
+		status = "okay";
+		address-cells = <1>;
+		size-cells = <0>;
+		reg = <0x10>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio72_default &pinctrl_gpio70_default>;
+		mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+		xvs-gpios = <&portc 6 GPIO_ACTIVE_HIGH>;
+
+		eswin,camera-module-index = <0>;
+		eswin,camera-module-facing = "back";
+		eswin,camera-module-name = "rpi-camera-v2";
+		eswin,camera-module-lens-name = "default";
+
+		port {
+			imx219_out0: endpoint {
+				remote-endpoint = <&mipidphy0_in_ucam0>;
+				data-lanes = <1 2 3 4>;
+				link-frequencies = /bits/ 64 <364000000>;
+			};
+		};
+	};
+};
+
+&d0_aon_i2c0 {
+	status = "disabled";
+	imx219_1: imx219@10 {
+		compatible = "sony,imx219";
+		status = "disabled";
+		address-cells = <1>;
+		size-cells = <0>;
+		reg = <0x10>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio78_default>;
+		mclk-gpios = <&portc 14 GPIO_ACTIVE_HIGH>;
+
+		eswin,camera-module-index = <0>;
+		eswin,camera-module-facing = "back";
+		eswin,camera-module-name = "rpi-camera-v2";
+		eswin,camera-module-lens-name = "default";
+
+		port {
+			imx219_out2: endpoint {
+				remote-endpoint = <&mipidphy2_in_ucam0>;
+				data-lanes = <1 2 3 4>;
+				link-frequencies = /bits/ 64 <364000000>;
+			};
+		};
+	};
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-osm.dts b/arch/riscv/boot/dts/eswin/eic7700-osm.dts
new file mode 100644
index 000000000000..982c906c2c35
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-osm.dts
@@ -0,0 +1,891 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include "eic7700-osm-camera.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 OSM";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		dvp2axi_mem: dvp2axi-mem@90000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x90000000 0x0 0x10000000>;
+			no-map;
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio28_default>;
+	cpu-voltage-gpios = <&porta 28 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1600M>;
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec3106_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	disable-wp;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	disable-wp;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio34_default>;
+		gpio_wl_reg_on = <&portb 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_gmac0 {
+	/* PHY DP83867 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio90_default>;
+	rst-gpios = <&portc 26 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	eswin,phyaddr = <1>;
+	dly-param-1000m = <0x20202020 0x64207820 0x20202020>;
+	dly-param-100m = <0x20202020 0x6E206E20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	/* PHY DP83867 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio55_default>;
+	hub-rst-gpios = <&portb 23 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* mikroBUS */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	eswin,pwm_inverted;
+};
+
+&d0_i2c0 {
+	status = "disabled";
+};
+
+&d0_i2c1 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_i2c2 {
+	status = "okay";
+	d0_tlv320aic3106: tlv320aic3106@1b {
+		compatible = "ti,tlv320aic3106";
+		reg = <0x1b>;
+		status = "okay";
+		ai3x-ocmv = <0>; /* Common-Mode Voltage 1.35V */
+		ai3x-micbias-vg = <2>; /* MICBIAS_2_5V */
+		#sound-dai-cells = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio82_default>;
+		reset-gpios = <&portc 18 GPIO_ACTIVE_LOW>;
+		port {
+			d0_codec3106_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c3 {
+	/* unused */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+		/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c1 {
+	/* pmic es5340 & rtc */
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf85063a";
+		reg = <0x51>;
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio7_default &pinctrl_gpio8_default &pinctrl_gpio9_default &pinctrl_gpio10_default
+			&pinctrl_gpio71_default &pinctrl_gpio73_default &pinctrl_gpio74_default &pinctrl_gpio75_default
+			&pinctrl_gpio76_default &pinctrl_gpio77_default &pinctrl_gpio79_default &pinctrl_gpio80_default
+			&pinctrl_gpio83_default &pinctrl_gpio84_default &pinctrl_gpio85_default &pinctrl_gpio86_default
+			&pinctrl_gpio87_default &pinctrl_gpio89_default &pinctrl_gpio91_default &pinctrl_gpio92_default>;
+};
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
+&dvp2axi {
+	memory-region = <&dvp2axi_mem>;
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-pinctrl.dtsi b/arch/riscv/boot/dts/eswin/eic7700-pinctrl.dtsi
new file mode 100644
index 000000000000..5ccd4dfcf817
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-pinctrl.dtsi
@@ -0,0 +1,1362 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for pin control of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "eswin-win2030-die0-soc.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+    #address-cells = <2>;
+	#size-cells = <2>;
+    soc {
+        pinctrl: pinctrl@0x51600080 {
+			compatible = "eswin,eic7x-pinctrl", "syscon";
+			reg = <0x0 0x51600080 0x0 0x1FFF80>;
+			status = "disabled";
+            //func0
+            pinctrl_sdio0_default: sdio0-default{
+				mux {
+					groups = "sdio0_group";
+					function = "sdio0_func";
+				};
+			};
+            pinctrl_sdio1_default: sdio1-default{
+				mux {
+					groups = "sdio1_group";
+					function = "sdio1_func";
+				};
+			};
+            pinctrl_jtag0_default: jtag0-default{
+				mux {
+					groups = "jtag0_group";
+					function = "jtag0_func";
+				};
+			};
+            pinctrl_jtag1_default: jtag1-default{
+				mux {
+					groups = "jtag1_group";
+					function = "jtag1_func";
+				};
+			};
+            pinctrl_jtag2_default: jtag2-default{
+				mux {
+					groups = "jtag2_group";
+					function = "jtag2_func";
+				};
+			};
+            pinctrl_pcie_default: pcie-default{
+				mux{
+					groups = "pcie_group";
+					function = "pcie_func";
+				};
+			};
+            pinctrl_hdmi_default: hdmi-default{
+				mux{
+					groups = "hdmi_group";
+					function = "hdmi_func";
+				};
+			};
+            pinctrl_rgmii0_default: rgmii0-default{
+				mux {
+					groups = "rgmii0_group";
+					function = "rgmii0_func";
+				};
+			};
+            pinctrl_rgmii1_default: rgmii1-default{
+				mux{
+					groups = "rgmii1_group";
+					function = "rgmii1_func";
+				};
+			};
+            pinctrl_spi0_default: spi0-default{
+				mux {
+					groups = "spi0_group";
+					function = "spi0_func";
+				};
+			};
+            pinctrl_spi1_default: spi1-default{
+				mux{
+					groups = "spi1_group";
+					function = "spi1_func";
+				};
+			};
+            pinctrl_spi3_default: spi3-default{
+				mux {
+					groups = "spi3_group";
+					function = "spi3_func";
+				};
+			};
+            pinctrl_por_time_sel0_default: por_time_sel0-default{
+				mux {
+					groups = "por_time_sel0_group";
+					function = "por_time_sel0_func";
+				};
+			};
+            pinctrl_por_time_sel1_default: por_time_sel1-default{
+				mux {
+					groups = "por_time_sel1_group";
+					function = "por_time_sel1_func";
+				};
+			};
+            pinctrl_i2s0_default: i2s0-default{
+				mux {
+					groups = "i2s0_group";
+					function = "i2s0_func";
+				};
+			};
+            pinctrl_i2s1_default: i2s1-default{
+				mux {
+					groups = "i2s1_group";
+					function = "i2s1_func";
+				};
+			};
+            pinctrl_i2s2_default: i2s2-default{
+				mux {
+					groups = "i2s2_group";
+					function = "i2s2_func";
+				};
+			};
+            pinctrl_usb0_pwren_default: usb0_pwren-default{
+				mux {
+					groups = "usb0_pwren_group";
+					function = "usb0_pwren_func";
+				};
+			};
+            pinctrl_usb1_pwren_default: usb1_pwren-default{
+				mux {
+					groups = "usb1_pwren_group";
+					function = "usb1_pwren_func";
+				};
+			};
+            pinctrl_i2c0_default: i2c0-default{
+				mux {
+					groups = "i2c0_group";
+					function = "i2c0_func";
+				};
+			};
+            pinctrl_i2c1_default: i2c1-default{
+				mux {
+					groups = "i2c1_group";
+					function = "i2c1_func";
+				};
+			};
+            pinctrl_i2c2_default: i2c2-default{
+				mux {
+					groups = "i2c2_group";
+					function = "i2c2_func";
+				};
+			};
+            pinctrl_i2c3_default: i2c3-default{
+				mux {
+					groups = "i2c3_group";
+					function = "i2c3_func";
+				};
+			};
+            pinctrl_i2c4_default: i2c4-default{
+				mux {
+					groups = "i2c4_group";
+					function = "i2c4_func";
+				};
+			};
+            pinctrl_i2c5_default: i2c5-default{
+				mux {
+					groups = "i2c5_group";
+					function = "i2c5_func";
+				};
+			};
+            pinctrl_i2c8_default: i2c8-default{
+				mux {
+					groups = "i2c8_group";
+					function = "i2c8_func";
+				};
+			};
+            pinctrl_i2c10_default: i2c10-default{
+				mux {
+					groups = "i2c10_group";
+					function = "i2c10_func";
+				};
+			};
+            pinctrl_i2c11_default: i2c11-default{
+				mux {
+					groups = "i2c11_group";
+					function = "i2c11_func";
+				};
+			};
+            pinctrl_uart0_default: uart0-default{
+				mux {
+					groups = "uart0_group";
+					function = "uart0_func";
+				};
+			};
+            pinctrl_uart1_default: uart1-default{
+				mux {
+					groups = "uart1_group";
+					function = "uart1_func";
+				};
+			};
+            pinctrl_uart2_default: uart2-default{
+				mux {
+					groups = "uart2_group";
+					function = "uart2_func";
+				};
+			};
+			pinctrl_pwm0_default: pwm0-default{
+				mux {
+					groups = "pwm0_group";
+					function = "pwm0_func";
+				};
+			};
+            pinctrl_fan_tach_default: fan_tach-default{
+				mux {
+					groups = "fan_tach_group";
+					function = "fan_tach_func";
+				};
+			};
+            pinctrl_mipi_csi0_default: mipi_csi0-default{
+				mux {
+					groups = "mipi_csi0_group";
+					function = "mipi_csi0_func";
+				};
+			};
+            pinctrl_mipi_csi1_default: mipi_csi1-default{
+				mux {
+					groups = "mipi_csi1_group";
+					function = "mipi_csi1_func";
+				};
+			};
+            pinctrl_mipi_csi2_default: mipi_csi2-default{
+				mux {
+					groups = "mipi_csi2_group";
+					function = "mipi_csi2_func";
+				};
+			};
+            pinctrl_mipi_csi3_default: mipi_csi3-default{
+				mux {
+					groups = "mipi_csi3_group";
+					function = "mipi_csi3_func";
+				};
+			};
+            pinctrl_mipi_csi4_default: mipi_csi4-default{
+				mux {
+					groups = "mipi_csi4_group";
+					function = "mipi_csi4_func";
+				};
+			};
+            pinctrl_mipi_csi5_default: mipi_csi5-default{
+				mux {
+					groups = "mipi_csi5_group";
+					function = "mipi_csi5_func";
+				};
+			};
+            pinctrl_s_mode_default: s_mode-default{
+				mux {
+					groups = "s_mode_group";
+					function = "s_mode_func";
+				};
+			};
+            pinctrl_ddr_refclk_sel_default: ddr_refclk_sel-default{
+				mux {
+					groups = "ddr_refclk_sel_group";
+					function = "ddr_refclk_sel_func";
+				};
+			};
+            pinctrl_boot_sel_default: boot_sel-default{
+				mux {
+					groups = "boot_sel_group";
+					function = "boot_sel_func";
+				};
+			};
+            pinctrl_lpddr_ref_clk_default: lpddr_ref_clk-default{
+				mux {
+					groups = "lpddr_ref_clk_group";
+					function = "lpddr_ref_clk_func";
+				};
+			};
+
+            //func1
+            pinctrl_spi2_default: spi2-default{
+				mux1 {
+					groups = "spi2_clk_group";
+					function = "spi2_clk_func";
+				};
+				conf1 {
+					groups = "spi2_clk_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux2 {
+					groups = "spi2_d0_group";
+					function = "spi2_d0_func";
+				};
+				conf2 {
+					groups = "spi2_d0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux3 {
+					groups = "spi2_d1_d2_d3_group";
+					function = "spi2_d1_d2_d3_func";
+				};
+				conf3 {
+					groups = "spi2_d1_d2_d3_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux4 {
+					groups = "spi2_cs_group";
+					function = "spi2_cs_func";
+				};
+			};
+
+            pinctrl_sata_act_led_default: sata_act_led-default{
+				mux {
+					groups = "sata_act_led_group";
+					function = "sata_act_led_func";
+				};
+				conf {
+					groups = "sata_act_led_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_emmc_led_control_default: emmc_led_control-default{
+				mux {
+					groups = "emmc_led_control_group";
+					function = "emmc_led_control_func";
+				};
+				conf {
+					groups = "emmc_led_control_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_sd0_led_control_default: sd0_led_control-default{
+				mux {
+					groups = "sd0_led_control_group";
+					function = "sd0_led_control_func";
+				};
+			};
+            pinctrl_sd1_led_control_default: sd1_led_control-default{
+				mux {
+					groups = "sd1_led_control_group";
+					function = "sd1_led_control_func";
+				};
+			};
+            pinctrl_i2c6_default: i2c6-default{
+				mux {
+					groups = "i2c6_group";
+					function = "i2c6_func";
+				};
+			};
+            pinctrl_i2c7_default: i2c7-default{
+				mux {
+					groups = "i2c7_group";
+					function = "i2c7_func";
+				};
+			};
+            pinctrl_i2c9_default: i2c9-default{
+				mux {
+					groups = "i2c9_group";
+					function = "i2c9_func";
+				};
+			};
+            pinctrl_pwm1_default: pwm1-default{
+				mux {
+					groups = "pwm1_group";
+					function = "pwm1_func";
+				};
+			};
+            pinctrl_pwm2_default: pwm2-default{
+				mux {
+					groups = "pwm2_group";
+					function = "pwm2_func";
+				};
+			};
+            pinctrl_mipi_csi_xtrig_default: mipi_csi_xtrig-default{
+				mux {
+					groups = "mipi_csi_xtrig_group";
+					function = "mipi_csi_xtrig_func";
+				};
+			};
+
+            //func3
+            pinctrl_uart3_default: uart3-default{
+				mux {
+					groups = "uart3_group";
+					function = "uart3_func";
+				};
+			};
+            pinctrl_uart4_default: uart4-default{
+				mux {
+					groups = "uart4_group";
+					function = "uart4_func";
+				};
+			};
+
+            //gpio
+            pinctrl_gpio0_default: gpio0-default{
+				mux {
+					groups = "gpio0_group";
+					function = "gpio0_func";
+				};
+				conf {
+					groups = "gpio0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio1_default: gpio1-default{
+				mux {
+					groups = "gpio1_group";
+					function = "gpio1_func";
+				};
+			};
+            pinctrl_gpio2_default: gpio2-default{
+				mux {
+					groups = "gpio2_group";
+					function = "gpio2_func";
+				};
+			};
+            pinctrl_gpio3_default: gpio3-default{
+				mux {
+					groups = "gpio3_group";
+					function = "gpio3_func";
+				};
+			};
+            pinctrl_gpio4_default: gpio4-default{
+				mux {
+					groups = "gpio4_group";
+					function = "gpio4_func";
+				};
+			};
+            pinctrl_gpio5_default: gpio5-default{
+				mux {
+					groups = "gpio5_group";
+					function = "gpio5_func";
+				};
+				conf {
+					groups = "gpio5_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio6_default: gpio6-default{
+				mux {
+					groups = "gpio6_group";
+					function = "gpio6_func";
+				};
+				conf {
+					groups = "gpio6_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio7_default: gpio7-default{
+				mux {
+					groups = "gpio7_group";
+					function = "gpio7_func";
+				};
+				conf {
+					groups = "gpio7_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio8_default: gpio8-default{
+				mux {
+					groups = "gpio8_group";
+					function = "gpio8_func";
+				};
+				conf {
+					groups = "gpio8_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio9_default: gpio9-default{
+				mux {
+					groups = "gpio9_group";
+					function = "gpio9_func";
+				};
+				conf {
+					groups = "gpio9_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio10_default: gpio10-default{
+				mux {
+					groups = "gpio10_group";
+					function = "gpio10_func";
+				};
+				conf {
+					groups = "gpio10_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio11_default: gpio11-default{
+				mux {
+					groups = "gpio11_group";
+					function = "gpio11_func";
+				};
+				conf {
+					groups = "gpio11_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio12_default: gpio12-default{
+				mux {
+					groups = "gpio12_group";
+					function = "gpio12_func";
+				};
+			};
+            pinctrl_gpio13_default: gpio13-default{
+				mux {
+					groups = "gpio13_group";
+					function = "gpio13_func";
+				};
+			};
+            pinctrl_gpio14_default: gpio14-default{
+				mux {
+					groups = "gpio14_group";
+					function = "gpio14_func";
+				};
+			};
+            pinctrl_gpio15_default: gpio15-default{
+				mux {
+					groups = "gpio15_group";
+					function = "gpio15_func";
+				};
+			};
+            pinctrl_gpio16_default: gpio16-default{
+				mux {
+					groups = "gpio16_group";
+					function = "gpio16_func";
+				};
+				conf {
+					groups = "gpio16_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio17_default: gpio17-default{
+				mux {
+					groups = "gpio17_group";
+					function = "gpio17_func";
+				};
+				conf {
+					groups = "gpio17_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio18_default: gpio18-default{
+				mux {
+					groups = "gpio18_group";
+					function = "gpio18_func";
+				};
+				conf {
+					groups = "gpio18_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio19_default: gpio19-default{
+				mux {
+					groups = "gpio19_group";
+					function = "gpio19_func";
+				};
+				conf {
+					groups = "gpio19_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio20_default: gpio20-default{
+				mux {
+					groups = "gpio20_group";
+					function = "gpio20_func";
+				};
+				conf {
+					groups = "gpio20_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio21_default: gpio21-default{
+				mux {
+					groups = "gpio21_group";
+					function = "gpio21_func";
+				};
+				conf {
+					groups = "gpio21_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio22_default: gpio22-default{
+				mux {
+					groups = "gpio22_group";
+					function = "gpio22_func";
+				};
+			};
+            pinctrl_gpio23_default: gpio23-default{
+				mux {
+					groups = "gpio23_group";
+					function = "gpio23_func";
+				};
+				conf {
+					groups = "gpio23_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio24_default: gpio24-default{
+				mux {
+					groups = "gpio24_group";
+					function = "gpio24_func";
+				};
+			};
+            pinctrl_gpio25_default: gpio25-default{
+				mux {
+					groups = "gpio25_group";
+					function = "gpio25_func";
+				};
+			};
+            pinctrl_gpio26_default: gpio26-default{
+				mux {
+					groups = "gpio26_group";
+					function = "gpio26_func";
+				};
+			};
+            pinctrl_gpio27_default: gpio27-default{
+				mux {
+					groups = "gpio27_group";
+					function = "gpio27_func";
+				};
+			};
+            pinctrl_gpio28_default: gpio28-default{
+				mux {
+					groups = "gpio28_group";
+					function = "gpio28_func";
+				};
+				conf {
+					groups = "gpio28_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio29_default: gpio29-default{
+				mux {
+					groups = "gpio29_group";
+					function = "gpio29_func";
+				};
+			};
+            pinctrl_gpio30_default: gpio30-default{
+				mux {
+					groups = "gpio30_group";
+					function = "gpio30_func";
+				};
+			};
+            pinctrl_gpio31_default: gpio31-default{
+				mux {
+					groups = "gpio31_group";
+					function = "gpio31_func";
+				};
+			};
+            pinctrl_gpio32_default: gpio32-default{
+				mux {
+					groups = "gpio32_group";
+					function = "gpio32_func";
+				};
+			};
+            pinctrl_gpio33_default: gpio33-default{
+				mux {
+					groups = "gpio33_group";
+					function = "gpio33_func";
+				};
+			};
+            pinctrl_gpio34_default: gpio34-default{
+				mux {
+					groups = "gpio34_group";
+					function = "gpio34_func";
+				};
+			};
+            pinctrl_gpio35_default: gpio35-default{
+				mux {
+					groups = "gpio35_group";
+					function = "gpio35_func";
+				};
+				conf {
+					groups = "gpio35_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio36_default: gpio36-default{
+				mux {
+					groups = "gpio36_group";
+					function = "gpio36_func";
+				};
+				conf {
+					groups = "gpio36_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio37_default: gpio37-default{
+				mux {
+					groups = "gpio37_group";
+					function = "gpio37_func";
+				};
+				conf {
+					groups = "gpio37_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio38_default: gpio38-default{
+				mux {
+					groups = "gpio38_group";
+					function = "gpio38_func";
+				};
+				conf {
+					groups = "gpio38_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio39_default: gpio39-default{
+				mux {
+					groups = "gpio39_group";
+					function = "gpio39_func";
+				};
+				conf {
+					groups = "gpio39_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio40_default: gpio40-default{
+				mux {
+					groups = "gpio40_group";
+					function = "gpio40_func";
+				};
+				conf {
+					groups = "gpio40_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio41_default: gpio41-default{
+				mux {
+					groups = "gpio41_group";
+					function = "gpio41_func";
+				};
+			};
+            pinctrl_gpio42_default: gpio42-default{
+				mux {
+					groups = "gpio42_group";
+					function = "gpio42_func";
+				};
+				conf {
+					groups = "gpio42_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio43_default: gpio43-default{
+				mux {
+					groups = "gpio43_group";
+					function = "gpio43_func";
+				};
+			};
+            pinctrl_gpio44_default: gpio44-default{
+				mux {
+					groups = "gpio44_group";
+					function = "gpio44_func";
+				};
+			};
+            pinctrl_gpio45_default: gpio45-default{
+				mux {
+					groups = "gpio45_group";
+					function = "gpio45_func";
+				};
+			};
+            pinctrl_gpio46_default: gpio46-default{
+				mux {
+					groups = "gpio46_group";
+					function = "gpio46_func";
+				};
+				conf {
+					groups = "gpio46_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio47_default: gpio47-default{
+				mux {
+					groups = "gpio47_group";
+					function = "gpio47_func";
+				};
+				conf {
+					groups = "gpio47_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio48_default: gpio48-default{
+				mux {
+					groups = "gpio48_group";
+					function = "gpio48_func";
+				};
+			};
+            pinctrl_gpio49_default: gpio49-default{
+				mux {
+					groups = "gpio49_group";
+					function = "gpio49_func";
+				};
+			};
+            pinctrl_gpio50_default: gpio50-default{
+				mux {
+					groups = "gpio50_group";
+					function = "gpio50_func";
+				};
+			};
+            pinctrl_gpio51_default: gpio51-default{
+				mux {
+					groups = "gpio51_group";
+					function = "gpio51_func";
+				};
+			};
+            pinctrl_gpio52_default: gpio52-default{
+				mux {
+					groups = "gpio52_group";
+					function = "gpio52_func";
+				};
+				conf {
+					groups = "gpio52_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio53_default: gpio53-default{
+				mux {
+					groups = "gpio53_group";
+					function = "gpio53_func";
+				};
+				conf {
+					groups = "gpio53_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio54_default: gpio54-default{
+				mux {
+					groups = "gpio54_group";
+					function = "gpio54_func";
+				};
+			};
+            pinctrl_gpio55_default: gpio55-default{
+				mux {
+					groups = "gpio55_group";
+					function = "gpio55_func";
+				};
+				conf {
+					groups = "gpio55_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio56_default: gpio56-default{
+				mux {
+					groups = "gpio56_group";
+					function = "gpio56_func";
+				};
+			};
+            pinctrl_gpio57_default: gpio57-default{
+				mux {
+					groups = "gpio57_group";
+					function = "gpio57_func";
+				};
+			};
+            pinctrl_gpio58_default: gpio58-default{
+				mux {
+					groups = "gpio58_group";
+					function = "gpio58_func";
+				};
+			};
+            pinctrl_gpio59_default: gpio59-default{
+				mux {
+					groups = "gpio59_group";
+					function = "gpio59_func";
+				};
+			};
+            pinctrl_gpio60_default: gpio60-default{
+				mux {
+					groups = "gpio60_group";
+					function = "gpio60_func";
+				};
+			};
+            pinctrl_gpio61_default: gpio61-default{
+				mux {
+					groups = "gpio61_group";
+					function = "gpio61_func";
+				};
+			};
+            pinctrl_gpio62_default: gpio62-default{
+				mux {
+					groups = "gpio62_group";
+					function = "gpio62_func";
+				};
+			};
+            pinctrl_gpio63_default: gpio63-default{
+				mux {
+					groups = "gpio63_group";
+					function = "gpio63_func";
+				};
+			};
+            pinctrl_gpio64_default: gpio64-default{
+				mux {
+					groups = "gpio64_group";
+					function = "gpio64_func";
+				};
+				conf {
+					groups = "gpio64_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio65_default: gpio65-default{
+				mux {
+					groups = "gpio65_group";
+					function = "gpio65_func";
+				};
+				conf {
+					groups = "gpio65_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio66_default: gpio66-default{
+				mux {
+					groups = "gpio66_group";
+					function = "gpio66_func";
+				};
+				conf {
+					groups = "gpio66_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio67_default: gpio67-default{
+				mux {
+					groups = "gpio67_group";
+					function = "gpio67_func";
+				};
+				conf {
+					groups = "gpio67_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio68_default: gpio68-default{
+				mux {
+					groups = "gpio68_group";
+					function = "gpio68_func";
+				};
+			};
+            pinctrl_gpio69_default: gpio69-default{
+				mux {
+					groups = "gpio69_group";
+					function = "gpio69_func";
+				};
+			};
+            pinctrl_gpio70_default: gpio70-default{
+				mux {
+					groups = "gpio70_group";
+					function = "gpio70_func";
+				};
+			};
+            pinctrl_gpio71_default: gpio71-default{
+				mux {
+					groups = "gpio71_group";
+					function = "gpio71_func";
+				};
+			};
+            pinctrl_gpio72_default: gpio72-default{
+				mux {
+					groups = "gpio72_group";
+					function = "gpio72_func";
+				};
+			};
+            pinctrl_gpio73_default: gpio73-default{
+				mux {
+					groups = "gpio73_group";
+					function = "gpio73_func";
+				};
+			};
+            pinctrl_gpio74_default: gpio74-default{
+				mux {
+					groups = "gpio74_group";
+					function = "gpio74_func";
+				};
+			};
+            pinctrl_gpio75_default: gpio75-default{
+				mux {
+					groups = "gpio75_group";
+					function = "gpio75_func";
+				};
+			};
+            pinctrl_gpio76_default: gpio76-default{
+				mux {
+					groups = "gpio76_group";
+					function = "gpio76_func";
+				};
+			};
+            pinctrl_gpio77_default: gpio77-default{
+				mux {
+					groups = "gpio77_group";
+					function = "gpio77_func";
+				};
+			};
+            pinctrl_gpio78_default: gpio78-default{
+				mux {
+					groups = "gpio78_group";
+					function = "gpio78_func";
+				};
+			};
+            pinctrl_gpio79_default: gpio79-default{
+				mux {
+					groups = "gpio79_group";
+					function = "gpio79_func";
+				};
+			};
+            pinctrl_gpio80_default: gpio80-default{
+				mux {
+					groups = "gpio80_group";
+					function = "gpio80_func";
+				};
+			};
+            pinctrl_gpio81_default: gpio81-default{
+				mux {
+					groups = "gpio81_group";
+					function = "gpio81_func";
+				};
+			};
+            pinctrl_gpio82_default: gpio82-default{
+				mux {
+					groups = "gpio82_group";
+					function = "gpio82_func";
+				};
+			};
+            pinctrl_gpio83_default: gpio83-default{
+				mux {
+					groups = "gpio83_group";
+					function = "gpio83_func";
+				};
+			};
+            pinctrl_gpio84_default: gpio84-default{
+				mux {
+					groups = "gpio84_group";
+					function = "gpio84_func";
+				};
+			};
+            pinctrl_gpio85_default: gpio85-default{
+				mux {
+					groups = "gpio85_group";
+					function = "gpio85_func";
+				};
+			};
+            pinctrl_gpio86_default: gpio86-default{
+				mux {
+					groups = "gpio86_group";
+					function = "gpio86_func";
+				};
+			};
+            pinctrl_gpio87_default: gpio87-default{
+				mux {
+					groups = "gpio87_group";
+					function = "gpio87_func";
+				};
+			};
+            pinctrl_gpio88_default: gpio88-default{
+				mux {
+					groups = "gpio88_group";
+					function = "gpio88_func";
+				};
+			};
+            pinctrl_gpio89_default: gpio89-default{
+				mux {
+					groups = "gpio89_group";
+					function = "gpio89_func";
+				};
+			};
+            pinctrl_gpio90_default: gpio90-default{
+				mux {
+					groups = "gpio90_group";
+					function = "gpio90_func";
+				};
+			};
+            pinctrl_gpio91_default: gpio91-default{
+				mux {
+					groups = "gpio91_group";
+					function = "gpio91_func";
+				};
+			};
+            pinctrl_gpio92_default: gpio92-default{
+				mux {
+					groups = "gpio92_group";
+					function = "gpio92_func";
+				};
+				conf {
+					groups = "gpio92_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio93_default: gpio93-default{
+				mux {
+					groups = "gpio93_group";
+					function = "gpio93_func";
+				};
+				conf {
+					groups = "gpio93_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio94_default: gpio94-default{
+				mux {
+					groups = "gpio94_group";
+					function = "gpio94_func";
+				};
+				conf {
+					groups = "gpio94_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio95_default: gpio95-default{
+				mux {
+					groups = "gpio95_group";
+					function = "gpio95_func";
+				};
+			};
+            pinctrl_gpio96_default: gpio96-default{
+				mux {
+					groups = "gpio96_group";
+					function = "gpio96_func";
+				};
+			};
+            pinctrl_gpio97_default: gpio97-default{
+				mux {
+					groups = "gpio97_group";
+					function = "gpio97_func";
+				};
+			};
+            pinctrl_gpio98_default: gpio98-default{
+				mux {
+					groups = "gpio98_group";
+					function = "gpio98_func";
+				};
+			};
+            pinctrl_gpio99_default: gpio99-default{
+				mux {
+					groups = "gpio99_group";
+					function = "gpio99_func";
+				};
+			};
+            pinctrl_gpio100_default: gpio100-default{
+				mux {
+					groups = "gpio100_group";
+					function = "gpio100_func";
+				};
+			};
+            pinctrl_gpio101_default: gpio101-default{
+				mux {
+					groups = "gpio101_group";
+					function = "gpio101_func";
+				};
+			};
+            pinctrl_gpio102_default: gpio102-default{
+				mux {
+					groups = "gpio102_group";
+					function = "gpio102_func";
+				};
+			};
+            pinctrl_gpio103_default: gpio103-default{
+				mux {
+					groups = "gpio103_group";
+					function = "gpio103_func";
+				};
+			};
+            pinctrl_gpio104_default: gpio104-default{
+				mux {
+					groups = "gpio104_group";
+					function = "gpio104_func";
+				};
+			};
+            pinctrl_gpio105_default: gpio105-default{
+				mux {
+					groups = "gpio105_group";
+					function = "gpio105_func";
+				};
+			};
+            pinctrl_gpio106_default: gpio106-default{
+				mux {
+					groups = "gpio106_group";
+					function = "gpio106_func";
+				};
+				conf {
+					groups = "gpio106_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio107_default: gpio107-default{
+				mux {
+					groups = "gpio107_group";
+					function = "gpio107_func";
+				};
+				conf {
+					groups = "gpio107_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio108_default: gpio108-default{
+				mux {
+					groups = "gpio108_group";
+					function = "gpio108_func";
+				};
+				conf {
+					groups = "gpio108_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio109_default: gpio109-default{
+				mux {
+					groups = "gpio109_group";
+					function = "gpio109_func";
+				};
+				conf {
+					groups = "gpio109_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio110_default: gpio110-default{
+				mux {
+					groups = "gpio110_group";
+					function = "gpio110_func";
+				};
+				conf {
+					groups = "gpio110_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio111_default: gpio111-default{
+				mux {
+					groups = "gpio111_group";
+					function = "gpio111_func";
+				};
+				conf {
+					groups = "gpio111_group";
+					input-enable = <0>;
+					bias-pull-up = <1>;
+				};
+			};
+
+            //func6
+            pinctrl_csi_mon_out_default: csi_mon_out-default{
+				mux {
+					groups = "csi_mon_out_group";
+					function = "csi_mon_out_func";
+				};
+			};
+            pinctrl_csi_ocla_clk_default: csi_ocla_clk-default{
+				mux {
+					groups = "csi_ocla_clk_group";
+					function = "csi_ocla_clk_func";
+				};
+			};
+            pinctrl_csi_mon_out_valid_default: csi_mon_out_valid-default{
+				mux {
+					groups = "csi_mon_out_valid_group";
+					function = "csi_mon_out_valid_func";
+				};
+			};
+            pinctrl_csi_parity_error_default: csi_parity_error-default{
+				mux {
+					groups = "csi_parity_error_group";
+					function = "csi_parity_error_func";
+				};
+			};
+            pinctrl_csi_dtb_out_default: csi_dtb_out-default{
+				mux {
+					groups = "csi_dtb_out_group";
+					function = "csi_dtb_out_func";
+				};
+			};
+            pinctrl_csi_phy_sel_default: csi_phy_sel-default{
+				mux {
+					groups = "csi_phy_sel_group";
+					function = "csi_phy_sel_func";
+				};
+			};
+            pinctrl_vc_g2d0_debug_out_default: vc_g2d0_debug_out-default{
+				mux {
+					groups = "vc_g2d0_debug_out_group";
+					function = "vc_g2d0_debug_out_func";
+				};
+			};
+            pinctrl_vc_g2d1_debug_out_default: vc_g2d1_debug_out-default{
+				mux {
+					groups = "vc_g2d1_debug_out_group";
+					function = "vc_g2d1_debug_out_func";
+				};
+			};
+            pinctrl_sata_mpll_clk_default: sata_mpll_clk-default{
+				mux {
+					groups = "sata_mpll_clk_group";
+					function = "sata_mpll_clk_func";
+				};
+			};
+            pinctrl_sata_ref_repeat_clk_m_default: sata_ref_repeat_clk_m-default{
+				mux {
+					groups = "sata_ref_repeat_clk_m_group";
+					function = "sata_ref_repeat_clk_m_func";
+				};
+			};
+            pinctrl_sata_ref_repeat_clk_p_default: sata_ref_repeat_clk_p-default{
+				mux {
+					groups = "sata_ref_repeat_clk_p_group";
+					function = "sata_ref_repeat_clk_p_func";
+				};
+			};
+		};
+    };
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-pine64-starpro64-npu.dts b/arch/riscv/boot/dts/eswin/eic7700-pine64-starpro64-npu.dts
new file mode 100644
index 000000000000..58ec5768451e
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-pine64-starpro64-npu.dts
@@ -0,0 +1,893 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * Copyright 2024 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Pine64 StarPro64";
+	/* TODO: use appropriate compatible strings */
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio27_default &pinctrl_gpio62_default>;
+
+		power {
+			gpios = <&portb 30 GPIO_ACTIVE_HIGH>;
+			function = LED_FUNCTION_POWER;
+			color = <LED_COLOR_ID_WHITE>;
+			linux,default-trigger = "default-on";
+		};
+
+		status {
+			gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_RED>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+        status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	/delete-property/ mmc-hs400-1_8v;
+	no-mmc-hs400;
+	delay_code = <0x37>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x70>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	disable-wp;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x80068023 0x06060606>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "disabled";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb0_pwren_default>;
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio43_default>;
+	status = "okay";
+	hub-rst-gpios = <&portb 11 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	eswin,pwm_inverted;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* mpq8785 */
+	status = "okay";
+	mpq8785@10 {
+		compatible = "mps,mpq8785";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio6   : system key(broken now)(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : host wake wlan(O)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio27  : status LED(originally SATA LED, SATA not used)(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio62  : power LED(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	apply_npu_high_freq;
+};
+
+&d0_clock {
+	status = "okay";
+	force-1_8ghz;
+};
+
+&d0_cpu_opp_table {
+        opp-1500000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1600000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1700000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1800000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-pine64-starpro64.dts b/arch/riscv/boot/dts/eswin/eic7700-pine64-starpro64.dts
new file mode 100644
index 000000000000..4614f55ca71c
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-pine64-starpro64.dts
@@ -0,0 +1,887 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * Copyright 2024 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Pine64 StarPro64";
+	/* TODO: use appropriate compatible strings */
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio27_default &pinctrl_gpio62_default>;
+
+		power {
+			gpios = <&portb 30 GPIO_ACTIVE_HIGH>;
+			function = LED_FUNCTION_POWER;
+			color = <LED_COLOR_ID_WHITE>;
+			linux,default-trigger = "default-on";
+		};
+
+		status {
+			gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_RED>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard2 {
+        status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	/delete-property/ mmc-hs400-1_8v;
+	no-mmc-hs400;
+	delay_code = <0x37>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x70>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	disable-wp;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x80068023 0x06060606>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "disabled";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb0_pwren_default>;
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio43_default>;
+	status = "okay";
+	hub-rst-gpios = <&portb 11 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	eswin,pwm_inverted;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* mpq8785 */
+	status = "okay";
+	mpq8785@10 {
+		compatible = "mps,mpq8785";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio6   : system key(broken now)(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : host wake wlan(O)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio27  : status LED(originally SATA LED, SATA not used)(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio62  : power LED(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	apply_npu_high_freq;
+};
+
+&d0_clock {
+	status = "okay";
+	force-1_8ghz;
+};
+
+&d0_cpu_opp_table {
+        opp-1500000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1600000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1700000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+        opp-1800000000 {
+                opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+                opp-microvolt = <900000>;
+                clock-latency-ns = <70000>;
+        };
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-sbc-a1-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1-camera.dtsi
new file mode 100644
index 000000000000..095958061da5
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1-camera.dtsi
@@ -0,0 +1,336 @@
+&vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+	phy_mode = <2>; //5: 6x2lanes, 2: 3x4lanes
+	vvcam_isp_0: vvcam_isp@0 {
+		compatible = "verisilicon,isp-v4l2";
+		id = <0>;
+		status = "okay";
+		dma-noncoherent;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					remote-endpoint = <&mipi_csi2_output1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				vvcam_isp_port0_mp: endpoint {
+					remote-endpoint = <&vvcam_video0>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				vvcam_isp_port0_sp1: endpoint {
+					remote-endpoint = <&vvcam_video1>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				vvcam_isp_port0_sp2: endpoint {
+					remote-endpoint = <&vvcam_video2>;
+				};
+			};
+
+			port@4 {
+				reg = <4>;
+				vvcam_isp_port0_raw: endpoint {
+					remote-endpoint = <&vvcam_video3>;
+				};
+			};
+		};
+	};
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+	vvcam_video_0: vvcam_video@0 {
+		compatible = "verisilicon,video";
+		id = <0>;
+		status = "okay";
+		#size-cells = <2>;
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				vvcam_video0: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_mp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				vvcam_video1: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp1>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				vvcam_video2: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_sp2>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				vvcam_video3: endpoint {
+					remote-endpoint = <&vvcam_isp_port0_raw>;
+				};
+			};
+		};
+	};
+};
+
+&mipi_csi2_0 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi0_csi2_input: endpoint@0 {
+				remote-endpoint = <&csidphy0_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_output1: endpoint@2 {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+&mipi_csi2_2 {
+	num_lanes = <4>;
+	snps,en-ppi-width = <1>; // 1: 16-bit, 0: 8-bit
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+	};
+};
+
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	dvp2axi_id = <0>;
+	port@0 {
+		axi2dvp_mipi2_in0: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_output0>;
+		};
+	};
+};
+
+&dvp2axi_mipi_lvds1 {
+	status = "okay";
+	dvp2axi_id = <1>;
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+
+&csi2_dphy_hw {
+	lanes = <1 2 3 4>;
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out0>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi0_csi2_input>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw2 {
+	lanes = <1 2 3 4>;
+	status = "okay";
+};
+
+&csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out2>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi csi2/csi3 */
+	status = "okay";
+	imx219_0: imx219@10 {
+			compatible = "sony,imx219";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			reg = <0x10>;
+			mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+			eswin,camera-module-index = <0>;
+			eswin,camera-module-facing = "back";
+			eswin,camera-module-name = "rpi-camera-v2";
+			eswin,camera-module-lens-name = "default";
+
+			port {
+				imx219_out2: endpoint {
+					remote-endpoint = <&mipidphy2_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <364000000>;
+				};
+			};
+		};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+	imx219_1: imx219@10 {
+			compatible = "sony,imx219";
+			status = "okay";
+			address-cells = <1>;
+			size-cells = <0>;
+			reg = <0x10>;
+
+			pinctrl-names = "default";
+			mclk-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+			eswin,camera-module-index = <0>;
+			eswin,camera-module-facing = "back";
+			eswin,camera-module-name = "rpi-camera-v2";
+			eswin,camera-module-lens-name = "default";
+
+			port {
+				imx219_out0: endpoint {
+					remote-endpoint = <&mipidphy0_in_ucam0>;
+					data-lanes = <1 2 3 4>;
+					link-frequencies = /bits/ 64 <364000000>;
+				};
+			};
+		};
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-sbc-a1-mipi.dts b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1-mipi.dts
new file mode 100755
index 000000000000..51db3caaa956
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1-mipi.dts
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eic7700-sbc-a1.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+
+&video_output {
+	status = "okay";
+	power-supply = <&reg_display>;
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+                dc_out_dsi: endpoint@0 {
+                        reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	dependencies = <&reg_display>;
+	dependencies = <&rasp_panel>;
+	dependencies = <&toshiba_bridge>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio28_default>;
+	dsi-mux-gpio = <&porta 28 GPIO_ACTIVE_HIGH>;
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&bridge_in>;
+			};
+		};
+	};
+
+	toshiba_bridge: bridge@0 {
+		reg = <0>;
+		compatible = "toshiba,tc358762";
+		status = "okay";
+		vddc-supply = <&reg_display>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				bridge_in: endpoint {
+					remote-endpoint = <&mipi_dsi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				bridge_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+	};
+
+};
+
+
+&rasp_panel {
+        status = "okay";
+        backlight = <&reg_display>;
+        power-supply = <&reg_display>;
+
+        port {
+                panel_in: endpoint {
+                        remote-endpoint = <&bridge_out>;
+                };
+        };
+};
+
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "okay";
+	reg_display: regulator@45 {
+		status = "okay";
+		compatible = "raspberrypi,7inch-touchscreen-panel-regulator";
+		reg = <0x45>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&dw_hdmi {
+        status = "disabled";
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eic7700-sbc-a1.dts b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1.dts
new file mode 100644
index 000000000000..5bba96c792d8
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1.dts
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eic7700-sbc-a1.dtsi"
+
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dpi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dw_hdmi {
+        status = "okay";
+        ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+                port@0 {
+                        reg = <0>;
+                        hdmi_in_dc: endpoint@0 {
+                                remote-endpoint = <&dc_out_hdmi>;
+                        };
+                };
+                port@2 {
+                        reg = <2>;
+                        hdmi_in_i2s: endpoint@1 {
+                                system-clock-frequency = <12288000>;
+                                remote-endpoint = <&d0_i2s0_endpoint>;
+                        };
+                };
+        };
+};
+
+&d0_i2s0 {
+        status = "okay";
+        d0_i2s0_port: port {
+                d0_i2s0_endpoint: endpoint {
+                        remote-endpoint = <&hdmi_in_i2s>;
+                        dai-format = "i2s";
+                };
+        };
+};
+
+&d0_graphcard0 {
+        status = "okay";
+        label = "HDMI Audio";
+        dais = <&d0_i2s0_port>;
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "okay";
+};
+
+&sdio1 {
+        /* disable this port */
+        status = "okay";
+};
\ No newline at end of file
diff --git a/arch/riscv/boot/dts/eswin/eic7700-sbc-a1.dtsi b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1.dtsi
new file mode 100755
index 000000000000..7c48511703f3
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-sbc-a1.dtsi
@@ -0,0 +1,804 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include "eic7700-sbc-a1-camera.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 SBC A1";
+	compatible = "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+		dvp2axi_mem: dvp2axi-mem@90000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x90000000 0x0 0x10000000>;
+			no-map;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio24_default &pinctrl_gpio25_default &pinctrl_gpio26_default>;
+
+		gpio-24 {
+			gpios = <&porta 24 GPIO_ACTIVE_HIGH>;
+			label = "gpio-24";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-25 {
+			gpios = <&porta 25 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-26 {
+			gpios = <&porta 26 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			/* remote-endpoint = <&d0_codec0_endpoint>; */
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "disabled";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* unused */
+	status = "disabled";
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	disable-wp;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio34_default>;
+		gpio_wl_reg_on = <&portb 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio23_default>;
+	hub-rst-gpios = <&porta 23 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart3 {
+	/* pin header */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3_default>;
+	status = "okay";
+};
+
+&d0_uart4 {
+	/* pin header */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* pin header */
+	status = "okay";
+	num-cs = <1>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* pin header */
+	status = "okay";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* micro hdmi */
+	status = "okay";
+};
+
+&d0_i2c1 {
+	/* pmic & usb pd sink */
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+	usb_pd_sink@8 {
+		label = "usb_pd_sink";
+		reg = <0x8>;
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "okay";
+};
+&d0_i2c3 {
+	/* mipi csi0 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* pin header */
+	status = "okay";
+	sys_power:ina226@40 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "sys_power";
+		reg = <0x40>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&d0_i2c5 {
+	/* pin header */
+	status = "okay";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf85063a";
+		reg = <0x51>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* unused */
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1_default &pinctrl_gpio2_default &pinctrl_gpio3_default &pinctrl_gpio4_default
+			&pinctrl_gpio5_default &pinctrl_gpio6_default &pinctrl_gpio7_default &pinctrl_gpio8_default
+			&pinctrl_gpio9_default &pinctrl_gpio10_default &pinctrl_gpio17_default &pinctrl_gpio27_default
+			&pinctrl_gpio64_default	&pinctrl_gpio65_default &pinctrl_gpio66_default &pinctrl_gpio67_default
+			&pinctrl_gpio72_default >;
+
+    /* pin header default function for GPIO
+       SPI1 (CS0,SCLK,MOSI,MISO,D2,D3,CS1): GPIO 35,36,37,38,39,40,41
+	   SPI2 (SCLK,D0,D1,D2,D3,CS0): GPIO 1,2,3,4,5,6
+       I2C4 (SCL,SDA): GPIO 52,53
+	   I2C5 (SCL,SDA): GPIO 54,55
+       UART3(TX,RX): GPIO 92,93
+	   UART4(TX,RX): GPIO 37,38
+    */
+};
+
+/* GPIO Function Description
+	gpio11  : usb hub reset(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : host wake(O)
+	gpio24  : led(O)
+	gpio25  : led(O)
+	gpio26  : led(O)
+	gpio28  : dsi/csi_n mux(O)
+	gpio34  : wlan power on(O)
+	gpio70  : csi0 led(O)
+	gpio72  : csi0 power(O)
+	gpio77  : csi1 power(O)
+	gpio78  : csi1 led(O)
+	gpio94  : gphy0 resetn(O)
+	gpio95  : host wake wlan(O)
+	gpio106 : vcc cpu adjust(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
+&dvp2axi {
+	memory-region = <&dvp2axi_mem>;
+};
+
+&SOC {
+       rasp_panel: rasp_panel {
+             compatible = "raspberrypi,7inch-dsi";
+	     status = "disabled";
+       };
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eic7700-som260-a1.dtsi b/arch/riscv/boot/dts/eswin/eic7700-som260-a1.dtsi
new file mode 100644
index 000000000000..24b21d774b69
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-som260-a1.dtsi
@@ -0,0 +1,750 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+// #include "eic7700-z530-dsi-panel.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN 7700S260";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+};
+
+/****************************************************
+	Carrier board device capability, need be adapted by user according to the carrier board implemented.
+	All of below modules are configured to okay by default, and need be reconfigured by user in dts of carrier board.
+		pcie
+		d0_sata
+		d0_usbdrd3_0
+		d0_usbdrd3_1
+		dw_hdmi
+		mipi csi(2 lanes) x4
+		mipi dsi(4 lanes)
+		sdio0
+		sdio1
+		d0_i2c0
+		d0_i2c1
+		d0_i2c2
+		aon_i2c0
+		d0_uart0
+		d0_uart1
+		d0_uart2
+		d0_uart4
+		ssi0
+		ssi1
+		pwm1
+		pwm2
+		gpio
+		d0_gmac0
+		d0_gmac1
+****************************************************/
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dpi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	status = "okay";
+};
+
+&sdio1 {
+	status = "okay";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	status = "okay";
+};
+
+&d0_uart2 {
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	status = "okay";
+};
+
+&ssi0 {
+	status = "okay";
+};
+
+&ssi1 {
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	pwms = <&pwm0 0 100000 PWM_POLARITY_INVERTED>;
+};
+
+&d0_i2c0 {
+	status = "okay";
+};
+
+&d0_i2c1 {
+	status = "okay";
+};
+
+&d0_i2c2 {
+	status = "okay";
+};
+
+&d0_i2c3 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* tmp102 */
+	status = "okay";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_i2c6 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* no provide */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	status = "okay";
+};
+
+&d0_aon_i2c1 {
+	/* pmic es5340 */
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-som314.dtsi b/arch/riscv/boot/dts/eswin/eic7700-som314.dtsi
new file mode 100644
index 000000000000..eb49fa122fce
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-som314.dtsi
@@ -0,0 +1,732 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include "eic7700-d314-camera.dtsi"
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN 7700S314";
+	compatible = "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		dvp2axi_mem: dvp2axi-mem@90000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x90000000 0x0 0x10000000>;
+			no-map;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dpi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+};
+
+/*
+&dsi_panel {
+	status = "okay";
+};
+*/
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+};
+
+&d0_i2s2 {
+	status = "okay";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	status = "okay";
+};
+
+&sdio1 {
+	status = "okay";
+};
+
+&d0_gmac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio106_default>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0xA0205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_gmac1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio111_default>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46204620 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	status = "okay";
+};
+
+&d0_uart2 {
+	status = "okay";
+};
+
+&d0_uart3 {
+	status = "okay";
+};
+
+&d0_uart4 {
+	status = "okay";
+};
+
+&ssi0 {
+	status = "okay";
+};
+
+&ssi1 {
+	status = "okay";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+	pwms = <&pwm0 0 100000 PWM_POLARITY_INVERTED>;
+};
+
+&d0_i2c0 {
+	status = "okay";
+};
+
+&d0_i2c1 {
+	status = "okay";
+};
+
+&d0_i2c2 {
+	status = "okay";
+};
+
+&d0_i2c3 {
+	status = "okay";
+};
+
+&d0_i2c4 {
+	status = "okay";
+};
+
+&d0_i2c5 {
+	status = "okay";
+};
+
+&d0_i2c6 {
+	status = "okay";
+};
+
+&d0_i2c7 {
+	status = "okay";
+};
+
+&d0_i2c8 {
+	status = "okay";
+};
+
+&d0_i2c9 {
+	status = "okay";
+};
+
+&d0_aon_i2c0 {
+	status = "okay";
+};
+
+&d0_aon_i2c1 {
+	status = "okay";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/*
+ GPIO USED ON SOM:
+	gpio18  : HOST WAKE WLAN(O), active low
+	gpio19  : HOST WAKE BT(O), active low
+	gpio20  : WLAN WAKE HOST(I), active low
+	gpio21  : BT WAKE HOST(I), active low
+	gpio106 : gphy0 resern(O), active low
+	gpio111 : gphy1 resern(O), active low
+
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
+&d0_clock {
+	status = "okay";
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&dvp2axi {
+	memory-region = <&dvp2axi_mem>;
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-z530-dsi-panel.dtsi b/arch/riscv/boot/dts/eswin/eic7700-z530-dsi-panel.dtsi
new file mode 100644
index 000000000000..dfa297dfc44f
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-z530-dsi-panel.dtsi
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for pin control of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+&SOC {
+	panel_disp: panel_disp {
+		compatible = "raspberrypi,7inch-dsi";
+		status = "okay";
+		backlight = <&reg_display>;
+		power-supply = <&reg_display>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&bridge_out>;
+			};
+		};
+	};
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		dc_out_dsi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&dsi_input>;
+		};
+	};
+};
+
+&dsi_controller {
+	status = "okay";
+	dependencies = <&reg_display>;
+	dependencies = <&panel_disp>;
+	dependencies = <&toshiba_bridge>;
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&bridge_in>;
+			};
+		};
+	};
+
+	toshiba_bridge: bridge@0 {
+		reg = <0>;
+		compatible = "toshiba,tc358762";
+		status = "okay";
+		vddc-supply = <&reg_display>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				bridge_in: endpoint {
+					remote-endpoint = <&mipi_dsi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				bridge_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+	};
+};
+
+&video_output {
+	status = "okay";
+	power-supply = <&reg_display>;
+};
+
+&d0_i2c1 {
+	status = "okay";
+	reg_display: regulator@45 {
+		compatible = "raspberrypi,7inch-touchscreen-panel-regulator";
+		reg = <0x45>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-z530.dts b/arch/riscv/boot/dts/eswin/eic7700-z530.dts
new file mode 100644
index 000000000000..fbcc6647868a
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-z530.dts
@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eic7700-som260-a1.dtsi"
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 Z530";
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ap_backup {
+			label = "ap-backup";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio26_default>;
+			gpios = <&porta 26 GPIO_ACTIVE_LOW>;
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&d0_gmac0 {
+	dly-param-1000m = <0x23232323 0xa0209023 0x20202020>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+};
+
+&d0_gmac1 {
+	dly-param-1000m = <0x00000000 0xe002e000 0x04040404>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+};
+
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio10_default>;
+	pci-prsnt = <&porta 10 GPIO_ACTIVE_LOW>;
+};
+
+&sdio0 {
+	/* sd card */
+	delay_code = <0x3b>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&sdio1 {
+	/* wifi module */
+	delay_code = <0x32>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio79_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portc 15 GPIO_ACTIVE_HIGH>;
+
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio79_default>;
+		gpio_wl_reg_on = <&portc 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&ssi0 {
+	/* lora */
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		eswin-plat = <3>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* rtc rs4c411 & mipi dsi */
+
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio0_default &pinctrl_gpio92_default>;
+		irq-gpios = <&porta 0 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portc 28 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+/* GPIO Function Description
+	gpio0   : touch int(I)
+	gpio5   : 5g wake on host(I)
+	gpio6   : 5g disable2(O)
+	gpio7   : mipi dsi ctrl(O)
+	gpio8   : 5g disable1(O)
+	gpio9   : rk628 plugin detect(I)
+	gpio10  : pcie prsnt(I)
+	gpio12  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio26  : ap backup(I)
+	gpio39  : lora busy(I)
+	gpio41  : host wake bt(I)
+	gpio70  : rk628 int(I)
+	gpio71  : rk628 reset(O)
+	gpio73  : rk628 power enable(O)
+	gpio74  : 5g pwren(O)
+	gpio76  : eth rset(O)
+	gpio77  : lora nreset(O)
+	gpio79  : m2 power on(O)
+	gpio80  : lora ctrl2(O)
+	gpio81  : lora ctrl1(O)
+	gpio92  : touch rstn(O)
+	gpio93  : 5g reset(O)
+*/
\ No newline at end of file
diff --git a/arch/riscv/boot/dts/eswin/eic7702-d560-die0-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7702-d560-die0-camera.dtsi
new file mode 100644
index 000000000000..aa903a98238e
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-d560-die0-camera.dtsi
@@ -0,0 +1,353 @@
+&vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+    vvcam_isp_0: vvcam_isp@0 {
+        compatible = "verisilicon,isp-v4l2";
+        id = <0>;
+        status = "disabled";
+		dma-noncoherent;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				vvcam_isp_input_port: endpoint@0 {
+					remote-endpoint = <&mipi2_csi2_output1>;
+				};
+			};
+
+            port@1 {
+                reg = <1>;
+                vvcam_isp_port0_mp: endpoint {
+                    remote-endpoint = <&vvcam_video0>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                vvcam_isp_port0_sp1: endpoint {
+                    remote-endpoint = <&vvcam_video1>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                vvcam_isp_port0_sp2: endpoint {
+                    remote-endpoint = <&vvcam_video2>;
+                };
+            };
+
+            port@4 {
+                reg = <4>;
+                vvcam_isp_port0_raw: endpoint {
+                    remote-endpoint = <&vvcam_video3>;
+                };
+            };
+        };
+    };
+
+	vvcam_vb: vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "disabled";
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+	};
+
+    vvcam_video_0: vvcam_video@0 {
+        compatible = "verisilicon,video";
+        id = <0>;
+        status = "disabled";
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu0 WIN2030_SID_ISP0>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                vvcam_video0: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_mp>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                vvcam_video1: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_sp1>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                vvcam_video2: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_sp2>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                vvcam_video3: endpoint {
+                    remote-endpoint = <&vvcam_isp_port0_raw>;
+                };
+            };
+        };
+    };
+};
+
+
+
+
+
+&mipi_csi2_0 {
+	data-lanes = <1 2>;
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi2_csi2_input: endpoint@0 {
+				remote-endpoint = <&csidphy0_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi2_csi2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in0>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi2_csi2_output1: endpoint@2 {
+				remote-endpoint = <&vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+
+&dvp2axi {
+	status = "okay";
+};
+
+&dvp2axi_mipi_lvds {
+	status = "okay";
+	port@0 {
+		axi2dvp_mipi2_in0: endpoint@0 {
+			remote-endpoint = <&mipi2_csi2_output0>;
+		};
+	};
+};
+
+&csi2_dphy_hw {
+	status = "okay";
+};
+
+&csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy0_in_ucam0: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&imx219_out0>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy0_out: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&mipi2_csi2_input>;
+			};
+		};
+	};
+};
+
+
+
+&d0_i2c3 {
+	status = "okay";
+
+
+	/* i2c-sda-hold-time-ns = <1000>; */
+	/* clock-frequency = <0x186a0>; */
+			camera_imx219: camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+                eswin,board-type = "d560";
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				power-en-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+				power-level-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+				mclk-en-gpios = <&portc 14 GPIO_ACTIVE_HIGH>;
+
+				port {
+					imx219_out0: endpoint {
+						remote-endpoint = <&mipidphy0_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+};
+
+&d0_i2c4 {
+	status = "okay";
+	/* d0 csi2-3 */
+
+			camera_imx219_1: camera-imx219-1@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				power-en-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+				power-level-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+				mclk-en-gpios = <&portc 14 GPIO_ACTIVE_HIGH>;
+
+				port {
+					imx219_out2: endpoint {
+						remote-endpoint = <&mipidphy2_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+};
+
+
+&mipi_csi2_2 {
+	data-lanes = <1 2>;
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&axi2dvp_mipi2_in2>;
+			};
+		};
+	};
+};
+
+&csi2_dphy_hw2 {
+	status = "okay";
+};
+&csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&imx219_out2>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+
+
+&dvp2axi_mipi_lvds1 {
+	status = "okay";
+	dvp2axi_id = <1>;
+};
+
+&dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+
+	port@0 {
+		axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&mipi_csi2_2_output0>;
+		};
+	};
+};
+
+
+
+&isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7702-d560-die1-camera.dtsi b/arch/riscv/boot/dts/eswin/eic7702-d560-die1-camera.dtsi
new file mode 100644
index 000000000000..6fefc408236c
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-d560-die1-camera.dtsi
@@ -0,0 +1,353 @@
+&d1_vi_subsys {
+	board_compat = "eic7700,dvb_vi";
+	eswin,isp_dvp0_hor = <3280>;
+	eswin,isp_dvp0_ver = <2464>;
+    d1_vvcam_isp_0: d1-vvcam_isp@0 {
+        compatible = "verisilicon,isp-v4l2";
+        id = <0>;
+        status = "disabled";
+		dma-noncoherent;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				d1_vvcam_isp_input_port: endpoint@0 {
+					remote-endpoint = <&d1_mipi2_csi2_output1>;
+				};
+			};
+
+            port@1 {
+                reg = <1>;
+                d1_vvcam_isp_port0_mp: endpoint {
+                    remote-endpoint = <&d1_vvcam_video0>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                d1_vvcam_isp_port0_sp1: endpoint {
+                    remote-endpoint = <&d1_vvcam_video1>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                d1_vvcam_isp_port0_sp2: endpoint {
+                    remote-endpoint = <&d1_vvcam_video2>;
+                };
+            };
+
+            port@4 {
+                reg = <4>;
+                d1_vvcam_isp_port0_raw: endpoint {
+                    remote-endpoint = <&d1_vvcam_video3>;
+                };
+            };
+        };
+    };
+
+	d1_vvcam_vb: d1-vvcam-vb@0 {
+		compatible = "verisilicon,vb";
+		status = "disabled";
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu1 WIN2030_SID_ISP0>;
+	};
+
+    d1_vvcam_video_0: d1-vvcam_video@0 {
+        compatible = "verisilicon,video";
+        id = <0>;
+        status = "disabled";
+		dma-noncoherent;
+		dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+		iommus = <&smmu1 WIN2030_SID_ISP0>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                d1_vvcam_video0: endpoint {
+                    remote-endpoint = <&d1_vvcam_isp_port0_mp>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                d1_vvcam_video1: endpoint {
+                    remote-endpoint = <&d1_vvcam_isp_port0_sp1>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                d1_vvcam_video2: endpoint {
+                    remote-endpoint = <&d1_vvcam_isp_port0_sp2>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                d1_vvcam_video3: endpoint {
+                    remote-endpoint = <&d1_vvcam_isp_port0_raw>;
+                };
+            };
+        };
+    };
+};
+
+
+
+
+
+&d1_mipi_csi2_0 {
+	data-lanes = <1 2>;
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipi2_csi2_input: endpoint@0 {
+				remote-endpoint = <&d1_csidphy0_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipi2_csi2_output0: endpoint@1 {
+				remote-endpoint = <&d1_axi2dvp_mipi2_in0>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipi2_csi2_output1: endpoint@2 {
+				remote-endpoint = <&d1_vvcam_isp_input_port>;
+			};
+		};
+	};
+};
+
+
+&d1_dvp2axi {
+	status = "okay";
+};
+
+&d1_dvp2axi_mipi_lvds {
+	status = "okay";
+	port@0 {
+		d1_axi2dvp_mipi2_in0: endpoint@0 {
+			remote-endpoint = <&d1_mipi2_csi2_output0>;
+		};
+	};
+};
+
+&d1_csi2_dphy_hw {
+	status = "okay";
+};
+
+&d1_csi2_dphy0 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipidphy0_in_ucam0: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&d1_imx219_out0>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_csidphy0_out: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&d1_mipi2_csi2_input>;
+			};
+		};
+	};
+};
+
+
+
+&d1_i2c3 {
+	status = "okay";
+
+
+	/* i2c-sda-hold-time-ns = <1000>; */
+	/* clock-frequency = <0x186a0>; */
+			d1_camera_imx219: d1-camera-imx219@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+                eswin,board-type = "d560";
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				power-en-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+				power-level-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+				mclk-en-gpios = <&portc 14 GPIO_ACTIVE_HIGH>;
+				port {
+					d1_imx219_out0: endpoint {
+						remote-endpoint = <&d1_mipidphy0_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+};
+
+&d1_i2c4 {
+	status = "okay";
+	/* d0 csi2-3 */
+
+			d1_camera_imx219_1: d1-camera-imx219-1@10 {
+				status = "okay";
+				compatible = "sony,imx219";
+				reg = <0x10>;
+
+				//clocks = <&clk_cam_24m>;
+				//clock-names = "xvclk";
+
+                eswin,board-type = "d560";
+				eswin,camera-module-index = <0>;
+				eswin,camera-module-facing = "back";
+				eswin,camera-module-name = "rpi-camera-v2";
+				eswin,camera-module-lens-name = "default";
+
+				power-en-gpios = <&portc 8 GPIO_ACTIVE_HIGH>;
+				power-level-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+				mclk-en-gpios = <&portc 14 GPIO_ACTIVE_HIGH>;
+
+				port {
+					d1_imx219_out2: endpoint {
+						remote-endpoint = <&d1_mipidphy2_in_ucam0>;
+						data-lanes = <1 2>;
+						link-frequencies = /bits/ 64 <456000000>;
+					};
+				};
+			};
+};
+
+
+&d1_mipi_csi2_2 {
+	data-lanes = <1 2>;
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipi_csi2_2_input: endpoint@0 {
+				remote-endpoint = <&d1_csidphy2_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipi_csi2_2_output0: endpoint@1 {
+				remote-endpoint = <&d1_axi2dvp_mipi2_in2>;
+			};
+		};
+	};
+};
+
+&d1_csi2_dphy_hw2 {
+	status = "okay";
+};
+&d1_csi2_dphy2 {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_mipidphy2_in_ucam0: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&d1_imx219_out2>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			d1_csidphy2_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&d1_mipi_csi2_2_input>;
+			};
+		};
+	};
+};
+
+
+
+&d1_dvp2axi_mipi_lvds1 {
+	status = "okay";
+	dvp2axi_id = <1>;
+};
+
+&d1_dvp2axi_mipi_lvds2 {
+	status = "okay";
+	dvp2axi_id = <2>;
+
+	port@0 {
+		d1_axi2dvp_mipi2_in2: endpoint@0 {
+			remote-endpoint = <&d1_mipi_csi2_2_output0>;
+		};
+	};
+};
+
+
+
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&d1_isp_1 {
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7702-d560-interleave.dts b/arch/riscv/boot/dts/eswin/eic7702-d560-interleave.dts
new file mode 100644
index 000000000000..ee7b89eb368f
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-d560-interleave.dts
@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7702 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#include "eic7702-som560-interleave.dtsi"
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN 7702 D560";
+
+	/* die0 gpio111 system led */
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio111_default>;
+
+		gpio-111 {
+			gpios = <&portd 15 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	disable-wp;
+	no-sdio;
+	no-mmc;
+};
+
+&d1_sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw315:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_die1_gpio18_default>;
+		pinctrl-1 = <&pinctrl_die1_gpio16_default>;
+		irq-gpios = <&d1_porta 18 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio13_default>;
+		gpio_wl_reg_on = <&d1_porta 13 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio14_default>;
+	hub-rst-gpios = <&porta 14 GPIO_ACTIVE_HIGH>;
+};
+
+&d1_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* connect mcu */
+	status = "okay";
+};
+
+&d1_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&ssi0 {
+	/* pin header */
+	status = "okay";
+};
+&d1_ssi0 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d1_i2c0 {
+	/* eeprom(at24c02c) & rtc(pcf8563) & power monitor(ina236) */
+	status = "okay";
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	aon_eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_i2c1 {
+	/* pin header */
+	status = "okay";
+};
+&d1_i2c1 {
+	/* ch347 */
+	status = "okay";
+};
+
+&d0_i2c2 {
+	/* d0 dsi*/
+	status = "ok";
+};
+
+&d1_i2c2 {
+	/* d1 dsi */
+	status = "ok";
+};
+
+&d0_i2c3 {
+	/* d0 csi0-1 */
+	status = "okay";
+};
+&d1_i2c3 {
+	/* d1 csi0-1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* d0 csi2-3 */
+};
+&d1_i2c4 {
+	/* d1 csi2-3 */
+};
+
+&d0_i2c5 {
+	/* pin header */
+};
+&d1_i2c5 {
+	/* cc logic fusb303b */
+	fusb303b@21 {
+		compatible = "fcs,fusb303b";
+		status = "okay";
+		reg = <0x21>;
+		eswin,syscfg = <&d1_sys_con 0x3C0 12>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio30_default>;
+		int-gpios = <&d1_porta 30 GPIO_ACTIVE_HIGH>;
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "host";
+		};
+	};
+};
+
+&d0_i2c8 {
+	/* pin header */
+};
+&d1_i2c8 {
+	/* pin header */
+};
+
+&d0_i2c9 {
+	/* pin header */
+};
+&d1_i2c9 {
+	/* pin header */
+};
+
+&wdt0 {
+	status = "disabled";
+};
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&timer1 {
+	status = "disabled";
+};
+&d1_timer1 {
+	status = "disabled";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio5_default &pinctrl_die0_gpio6_default &pinctrl_die0_gpio12_default &pinctrl_die0_gpio35_default
+				 &pinctrl_die0_gpio39_default &pinctrl_die0_gpio40_default &pinctrl_die0_gpio41_default &pinctrl_die0_gpio46_default
+				 &pinctrl_die0_gpio47_default &pinctrl_die0_gpio54_default &pinctrl_die0_gpio55_default &pinctrl_die0_gpio73_default
+				 &pinctrl_die0_gpio75_default &pinctrl_die0_gpio79_default &pinctrl_die0_gpio81_default &pinctrl_die0_gpio36_default
+				 &pinctrl_die0_gpio37_default &pinctrl_die0_gpio38_default &pinctrl_die0_gpio74_default &pinctrl_die0_gpio92_default
+				 &pinctrl_die0_gpio93_default>;
+	
+
+    /* pin header default function for GPIO
+       D0_SPI1 (CS0,SCLK,MOSI,MISO,D2,D3): GPIO 35,36,37,38,39,40
+	   D0_I2C1 (SCL,SDA): GPIO 46,47
+       D0_I2C5 (SCL,SDA): GPIO 54,55
+	   D1_SPI1 (CS0,SCLK,D0,D1,D2): GPIO 35,36,37,38,39
+       D1_UART3(TX,RX): GPIO 92,93
+	   D0_UART3(TX,RX): GPIO 92,93
+    */
+};
+
+&d1_pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio35_default &pinctrl_die1_gpio36_default &pinctrl_die1_gpio37_default &pinctrl_die1_gpio38_default
+				 &pinctrl_die1_gpio39_default &pinctrl_die1_gpio92_default &pinctrl_die1_gpio93_default>;
+};
+
+/* die0 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio18  : ina226 alert(I)
+	gpio25  : front panel audio insert detection(I)
+	gpio27  : sata act(O)
+	gpio28  : to mcu pb5
+	gpio29  : gphy0 resetn(O)
+	gpio34  : gphy1 resetn(O)
+	gpio42  : usb0 pwren(O)
+	gpio43  : usb1 pwren(O)
+	gpio111 : sys run led(O)
+*/
+/* die1 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : classis fan1 fantach
+	gpio6   : classis fan2 fantach
+	gpio27  : to mcu pb3
+	gpio28  : to mcu pb4
+	gpio29  : gphy0 resetn(O)
+	gpio34  : gphy1 resetn(O)
+	gpio42  : usb0 pwren(O)
+	gpio43  : usb1 pwren(O)
+*/
diff --git a/arch/riscv/boot/dts/eswin/eic7702-d560.dts b/arch/riscv/boot/dts/eswin/eic7702-d560.dts
new file mode 100644
index 000000000000..190cc76df82a
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-d560.dts
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7702 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#include "eic7702-som560.dtsi"
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN 7702 D560";
+
+	/* die0 gpio111 system led */
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio111_default>;
+
+		gpio-111 {
+			gpios = <&portd 15 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	disable-wp;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&d1_sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw315:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_die1_gpio18_default>;
+		pinctrl-1 = <&pinctrl_die1_gpio16_default>;
+		irq-gpios = <&d1_porta 18 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio13_default>;
+		gpio_wl_reg_on = <&d1_porta 13 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio14_default>;
+	hub-rst-gpios = <&porta 14 GPIO_ACTIVE_HIGH>;
+};
+
+&d1_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* connect mcu */
+	status = "okay";
+};
+
+&d1_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&ssi0 {
+	/* pin header */
+	status = "okay";
+};
+&d1_ssi0 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		eswin-plat = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio25_default &pinctrl_die0_gpio26_default>;
+		front-jack-gpios = <&porta 25 GPIO_ACTIVE_HIGH>;
+		back-jack-gpios = <&porta 26 GPIO_ACTIVE_HIGH>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d1_i2c0 {
+	/* eeprom(at24c02c) & rtc(pcf8563) & power monitor(ina236) */
+	status = "okay";
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	aon_eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c1 {
+	/* pin header */
+	status = "okay";
+};
+&d1_i2c1 {
+	/* ch347 */
+	status = "okay";
+};
+
+&d0_i2c2 {
+	/* d0 dsi*/
+	status = "ok";
+};
+
+&d1_i2c2 {
+	/* d1 dsi */
+	status = "ok";
+};
+
+&d0_i2c3 {
+	/* d0 csi0-1 */
+	status = "okay";
+};
+&d1_i2c3 {
+	/* d1 csi0-1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* d0 csi2-3 */
+};
+&d1_i2c4 {
+	/* d1 csi2-3 */
+};
+
+&d0_i2c5 {
+	/* pin header */
+};
+&d1_i2c5 {
+	/* cc logic fusb303b */
+	fusb303b@21 {
+		compatible = "fcs,fusb303b";
+		status = "okay";
+		reg = <0x21>;
+		eswin,syscfg = <&d1_sys_con 0x3C0 12>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio30_default>;
+		int-gpios = <&d1_porta 30 GPIO_ACTIVE_HIGH>;
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "host";
+		};
+	};
+};
+
+&d0_i2c8 {
+	/* pin header */
+};
+&d1_i2c8 {
+	/* pin header */
+};
+
+&d0_i2c9 {
+	/* pin header */
+};
+&d1_i2c9 {
+	/* pin header */
+};
+
+&wdt0 {
+	status = "disabled";
+};
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&timer1 {
+	status = "disabled";
+};
+&d1_timer1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio5_default &pinctrl_die0_gpio6_default &pinctrl_die0_gpio12_default &pinctrl_die0_gpio35_default
+				 &pinctrl_die0_gpio39_default &pinctrl_die0_gpio40_default &pinctrl_die0_gpio41_default &pinctrl_die0_gpio46_default
+				 &pinctrl_die0_gpio47_default &pinctrl_die0_gpio54_default &pinctrl_die0_gpio55_default &pinctrl_die0_gpio73_default
+				 &pinctrl_die0_gpio75_default &pinctrl_die0_gpio79_default &pinctrl_die0_gpio81_default &pinctrl_die0_gpio36_default
+				 &pinctrl_die0_gpio37_default &pinctrl_die0_gpio38_default &pinctrl_die0_gpio74_default &pinctrl_die0_gpio92_default
+				 &pinctrl_die0_gpio93_default>;
+
+
+    /* pin header default function for GPIO
+       D0_SPI1 (CS0,SCLK,MOSI,MISO,D2,D3): GPIO 35,36,37,38,39,40
+	   D0_I2C1 (SCL,SDA): GPIO 46,47
+       D0_I2C5 (SCL,SDA): GPIO 54,55
+	   D1_SPI1 (CS0,SCLK,D0,D1,D2): GPIO 35,36,37,38,39
+       D1_UART3(TX,RX): GPIO 92,93
+	   D0_UART3(TX,RX): GPIO 92,93
+    */
+};
+
+&d1_pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio35_default &pinctrl_die1_gpio36_default &pinctrl_die1_gpio37_default &pinctrl_die1_gpio38_default
+				 &pinctrl_die1_gpio39_default &pinctrl_die1_gpio92_default &pinctrl_die1_gpio93_default>;
+};
+
+/* die0 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio18  : ina226 alert(I)
+	gpio25  : front panel audio insert detection(I)
+	gpio27  : sata act(O)
+	gpio28  : to mcu pb5
+	gpio29  : gphy0 resetn(O)
+	gpio34  : gphy1 resetn(O)
+	gpio42  : usb0 pwren(O)
+	gpio43  : usb1 pwren(O)
+	gpio111 : sys run led(O)
+*/
+/* die1 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : classis fan1 fantach
+	gpio6   : classis fan2 fantach
+	gpio27  : to mcu pb3
+	gpio28  : to mcu pb4
+	gpio29  : gphy0 resetn(O)
+	gpio34  : gphy1 resetn(O)
+	gpio42  : usb0 pwren(O)
+	gpio43  : usb1 pwren(O)
+*/
diff --git a/arch/riscv/boot/dts/eswin/eic7702-evb-a1-d0.dts b/arch/riscv/boot/dts/eswin/eic7702-evb-a1-d0.dts
new file mode 100644
index 000000000000..633a760351ed
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-evb-a1-d0.dts
@@ -0,0 +1,862 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7702 SoC only active die0.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+    status = "disabled";
+};
+
+&virtual_display {
+    status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio27_default &pinctrl_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio15_default>;
+	rst-gpios = <&porta 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio14_default>;
+	rst-gpios = <&porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ina226x3 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x40>;
+		shunt-resistor = <1000>;
+	};
+	ina226@41 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_lpddr_vdd";
+		reg = <0x41>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* serdes clk buffer rc22514a */
+	status = "okay";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : gphy0 resetn(O)
+	gpio18  : ina226 alert(I)
+	gpio20  : lcd touch int(I)
+	gpio21  : head phone plug/unplug detection2(I)
+	gpio27  : black light pwr_en(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : fan pwm(O)
+	gpio69  : fan touch(I)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : mipi dsi resetn(O)
+	gpio111 : lcd touch resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eic7702-evb-a1-d1.dts b/arch/riscv/boot/dts/eswin/eic7702-evb-a1-d1.dts
new file mode 100644
index 000000000000..0c5eb06ed4a4
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-evb-a1-d1.dts
@@ -0,0 +1,878 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7702 SoC only active die1.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio68_default>;
+
+		gpio-68 {
+			gpios = <&portc 4 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+    status = "disabled";
+};
+
+&virtual_display {
+    status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio69_default &pinctrl_gpio21_default>;
+	backlight0-gpios = <&portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&sdio1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio13_default>;
+	rst-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio14_default>;
+	rst-gpios = <&porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* m.2 key */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* mipi dsi */
+	status = "disabled";
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ina226x4 & tmp102aidrlr */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+	ina226@41 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "lpddr_vdd";
+		reg = <0x41>;
+		shunt-resistor = <1000>;
+	};
+	ina226@48 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "dc_vdd";
+		reg = <0x48>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* syr838pkc */
+	status = "okay";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : lcd touch int(I)
+	gpio13  : gphy0 resetn(O)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : bt wake host(I)
+	gpio16  : sdio wifi disable(O)
+	gpio18  : sdio wifi wake host(I)
+	gpio19  : pcie prstn(I)
+	gpio20  : head phone plug/unplug detection2(I)
+	gpio21  : mipi dsi resetn(O)
+	gpio27  : tmp alert(I)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : system led(O)
+	gpio69  : black light pwr_en(O)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : host wak bt(O)
+	gpio111 : lcd touch resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eic7702-evb-a1-interleave.dts b/arch/riscv/boot/dts/eswin/eic7702-evb-a1-interleave.dts
new file mode 100644
index 000000000000..628f066512f4
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-evb-a1-interleave.dts
@@ -0,0 +1,1685 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eswin-win2030-die1-soc.dtsi"
+
+#include "eic7x-die0-pinctrl.dtsi"
+#include "eic7x-die1-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		ethernet2 = &d1_gmac0;
+		ethernet3 = &d1_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &d1_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	memory@4000000000 {
+		device_type = "memory";
+		reg = <0x40 0x00000000 0x0 0x80000000>;
+		numa-node-id = <0>;
+	};
+	memory@4100000000 {
+		device_type = "memory";
+		reg = <0x41 0x00000000 0x1 0xe0000000>;
+		numa-node-id = <0>;
+	};
+	memory@42e0000000 {
+		device_type = "memory";
+		reg = <0x42 0xe0000000 0x1 0xe0000000>;
+		numa-node-id = <1>;
+	};
+	memory@c0000000 {
+		device_type = "memory";
+		reg = <0x00 0xc0000000 0x0 0x40000000>;
+		numa-node-id = <0>;
+	};
+	memory@2040000000 {
+		device_type = "memory";
+		reg = <0x20 0x40000000 0x0 0x40000000>;
+		numa-node-id = <1>;
+	};
+	memory@300000000 {
+		device_type = "memory";
+		reg = <0x3 0x0 0x1 0x80000000>;
+		numa-node-id = <0>;
+	};
+	memory@2280000000 {
+		device_type = "memory";
+		reg = <0x22 0x80000000 0x1 0x80000000>;
+		numa-node-id = <1>;
+	};
+	memory@2e00000000 {
+		device_type = "memory";
+		reg = <0x2 0xe0000000 0x0 0x20000000>;
+		numa-node-id = <0>;
+	};
+	memory@2260000000 {
+		device_type = "memory";
+		reg = <0x22 0x60000000 0x0 0x20000000>;
+		numa-node-id = <1>;
+	};
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x41 0x00000000 0x1 0x0>,<0x43 0x00000000 0x1 0x0>;
+			linux,cma-default;
+		};
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x22 0x80000000 0x1 0x80000000>;
+			no-map;
+		};
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		d0_dc_reserved: d0_dc_reserved@2e00000000 {
+			compatible = "shared-dma-pool";
+			reusable;
+			reg = <0x2 0xe0000000 0x0 0x20000000>;
+		};
+
+		d1_dc_reserved: d1_dc_reserved@2260000000 {
+			compatible = "shared-dma-pool";
+			reusable;
+			reg = <0x22 0x60000000 0x0 0x20000000>;
+		};
+
+		// g2d_4GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x0 0xfffff000 0x0 0x1000>;
+		// };
+
+		// g2d_8GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x1 0xfffff000 0x0 0x1000>;
+		// };
+
+		// g2d_12GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x2 0xfffff000 0x0 0x1000>;
+		// };
+
+		nid_1_zero_device_simu: nid_1_zero_device_simu@2040000000 {
+			no-map;
+			reg = <0x20 0x40000000 0x0 0x400000>;
+		};
+		// d1_g2d_4GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x20 0xfffff000 0x0 0x1000>;
+		// };
+
+		// d1_g2d_8GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x21 0xfffff000 0x0 0x1000>;
+		// };
+
+
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 100>,
+				<1 0 100>,
+				<1 1 10>;
+	};
+
+	/* die1 gpio68 system led */
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio68_default>;
+
+		gpio-68 {
+			gpios = <&d1_portc 4 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&D1CACHE {
+	status = "okay";
+	d1_zero_device: zero-device@3a000000 {
+		compatible = "l3,zero-device";
+		/*reg = <0x0 0x3a000000 0x0 0x400000>;*/
+		memory-region = <&nid_1_zero_device_simu>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+&d1_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+&d1_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+&d1_pmu {
+	status = "disabled";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&d1_ddr0 {
+	status = "okay";
+};
+
+&d1_ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+&d1_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&pcie {
+	status = "okay";
+};
+&d1_pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+};
+&d1_npu {
+	status = "okay";
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+&d1_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+&d1_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+&d1_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+&d1_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+&d1_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+&d1_gpu {
+	status = "disabled";
+};
+
+&gc820 {
+	status = "okay";
+};
+&d1_gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+&vdec1 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+&venc1 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+	memory-region = <&d0_dc_reserved>;
+};
+
+&d1_video_output {
+	status = "okay";
+	memory-region = <&d1_dc_reserved>;
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&d1_dc {
+	status = "okay";
+	d1_dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d1_dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&d1_dsi_input>;
+		};
+		d1_dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&d1_vd_input>;
+		};
+		d1_dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&d1_hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+&d1_dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&d1_virtual_display {
+	status = "okay";
+	port {
+		d1_vd_input: endpoint {
+			remote-endpoint = <&d1_dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+&d1_dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			d1_dsi_input: endpoint {
+				remote-endpoint = <&d1_dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			d1_mipi_dsi_out: endpoint {
+				remote-endpoint = <&d1_panel_in>;
+			};
+		};
+	};
+	d1_dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			d1_panel_in: endpoint {
+				remote-endpoint = <&d1_mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio27_default &pinctrl_die0_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+&d1_dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio69_default &pinctrl_die1_gpio21_default>;
+	backlight0-gpios = <&d1_portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+&d1_dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			d1_hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&d1_dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			d1_hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+&d1_dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+
+&d0_i2s0 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s0 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s0_port: port {
+		d1_i2s0_endpoint: endpoint {
+			remote-endpoint = <&d1_hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s1 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s1_port: port {
+		d1_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d1_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s2 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s2_port: port {
+		d1_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d1_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-0";
+	dais = <&d0_i2s0_port>;
+};
+
+&d1_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-2";
+	dais = <&d1_i2s1_port>;
+};
+
+&d1_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-3";
+	dais = <&d1_i2s2_port>;
+};
+
+&d1_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-1";
+	dais = <&d1_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "okay";
+};
+&d1_isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+&d1_dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+&d1_mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+&d1_csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+&d1_csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+&d1_csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+&d1_csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&d1_sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&d1_sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw315:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_die1_gpio18_default>;
+		pinctrl-1 = <&pinctrl_die1_gpio16_default>;
+		irq-gpios = <&d1_porta 18 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio16_default>;
+		gpio_wl_reg_on = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&sdio1 {
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio15_default>;
+	rst-gpios = <&porta 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio13_default>;
+	rst-gpios = <&d1_porta 13 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio14_default>;
+	rst-gpios = <&porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio14_default>;
+	rst-gpios = <&d1_porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d1_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+&d1_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d1_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "disabled";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d1_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d1_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+&d1_ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d1_bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&d1_portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&d1_portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+&d1_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+&d1_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+&d1_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+&d1_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+&d1_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+&d1_fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d1_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d1_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s1_endpoint>;
+			};
+		};
+	};
+	d1_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			d0_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+&d1_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "d1_npu_vdd";
+		regulators{
+			d1_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC2";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "okay";
+	d0_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio20_default>;
+		irq-gpios = <&porta 20 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d1_i2c2 {
+	/* touch screen*/
+	status = "okay";
+	d1_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio0_default>;
+		irq-gpios = <&d1_porta 0 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&d1_portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+&d1_i2c3 {
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+&d1_i2c4 {
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+&d1_i2c5 {
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c6 {
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c7 {
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+&d1_i2c8 {
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c9 {
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ina226x3 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x40>;
+		shunt-resistor = <1000>;
+	};
+	ina226@41 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_lpddr_vdd";
+		reg = <0x41>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+
+&d1_aon_i2c0 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "vdd_dc_in";
+			reg = <0x40>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d1_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+&d1_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+};
+&d1_pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+&d1_pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+&d1_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+&d1_timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+&d1_timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+&d1_timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+&d1_timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+&d1_pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&d1_gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d0_npu_vcc>;
+	status = "okay";
+};
+&d1_llc_dev{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d1_npu_vcc>;
+	status = "okay";
+};
+
+&d2d {
+	status = "disabled";
+};
+
+&d1_d2d {
+	status = "disabled";
+};
+
+/* die0 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : gphy0 resetn(O)
+	gpio18  : ina226 alert(I)
+	gpio20  : lcd touch int(I)
+	gpio21  : head phone plug/unplug detection2(I)
+	gpio27  : black light pwr_en(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : fan pwm(O)
+	gpio69  : fan touch(I)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : mipi dsi resetn(O)
+	gpio111 : lcd touch resetn(O)
+*/
+/* die1 GPIO Function Description
+
+	gpio0   : lcd touch int(I)
+	gpio13  : gphy0 resetn(O)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : bt wake host(I)
+	gpio16  : sdio wifi disable(O)
+	gpio18  : sdio wifi wake host(I)
+	gpio19  : pcie prstn(I)
+	gpio20  : head phone plug/unplug detection2(I)
+	gpio21  : mipi dsi resetn(O)
+	gpio27  : tmp alert(I)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : system led(O)
+	gpio69  : black light pwr_en(O)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : host wak bt(O)
+	gpio111 : lcd touch resetn(O)
+*/
diff --git a/arch/riscv/boot/dts/eswin/eic7702-evb-a1.dts b/arch/riscv/boot/dts/eswin/eic7702-evb-a1.dts
new file mode 100644
index 000000000000..f5cd7bf83afe
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-evb-a1.dts
@@ -0,0 +1,1654 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eswin-win2030-die1-soc.dtsi"
+
+#include "eic7x-die0-pinctrl.dtsi"
+#include "eic7x-die1-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 EVB A1";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		ethernet2 = &d1_gmac0;
+		ethernet3 = &d1_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &d1_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@2000000000 {
+		device_type = "memory";
+		reg = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <1>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		nid_1_zero_device_simu: nid_1_zero_device_simu@2040000000 {
+			no-map;
+			reg = <0x20 0x40000000 0x0 0x400000>;
+		};
+
+		d1_g2d_4GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x20 0xfffff000 0x0 0x1000>;
+		};
+
+		d1_g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x21 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x22 0x80000000 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 100>,
+				<1 0 100>,
+				<1 1 10>;
+	};
+
+	/* die1 gpio68 system led */
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio68_default>;
+
+		gpio-68 {
+			gpios = <&d1_portc 4 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&D1CACHE {
+	status = "okay";
+	d1_zero_device: zero-device@3a000000 {
+		compatible = "l3,zero-device";
+		/*reg = <0x0 0x3a000000 0x0 0x400000>;*/
+		memory-region = <&nid_1_zero_device_simu>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+&d1_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+&d1_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+&d1_pmu {
+	status = "disabled";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&d1_ddr0 {
+	status = "okay";
+};
+
+&d1_ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+&d1_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&pcie {
+	status = "okay";
+};
+&d1_pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&d0_npu_vcc>;
+};
+&d1_npu {
+	status = "okay";
+	npu-supply=<&d1_npu_vcc>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+&d1_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+&d1_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+&d1_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+&d1_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+&d1_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+&d1_gpu {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+&d1_gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+&vdec1 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+&venc1 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&d1_video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&d1_dc {
+	status = "okay";
+	d1_dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d1_dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&d1_dsi_input>;
+		};
+		d1_dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&d1_vd_input>;
+		};
+		d1_dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&d1_hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+&d1_dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&d1_virtual_display {
+	status = "okay";
+	port {
+		d1_vd_input: endpoint {
+			remote-endpoint = <&d1_dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+&d1_dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			d1_dsi_input: endpoint {
+				remote-endpoint = <&d1_dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			d1_mipi_dsi_out: endpoint {
+				remote-endpoint = <&d1_panel_in>;
+			};
+		};
+	};
+	d1_dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			d1_panel_in: endpoint {
+				remote-endpoint = <&d1_mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio27_default &pinctrl_die0_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+&d1_dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio69_default &pinctrl_die1_gpio21_default>;
+	backlight0-gpios = <&d1_portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+&d1_dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			d1_hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&d1_dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			d1_hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+&d1_dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+
+&d0_i2s0 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s0 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s0_port: port {
+		d1_i2s0_endpoint: endpoint {
+			remote-endpoint = <&d1_hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s1 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s1_port: port {
+		d1_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d1_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s2 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s2_port: port {
+		d1_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d1_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-0";
+	dais = <&d0_i2s0_port>;
+};
+
+&d1_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-2";
+	dais = <&d1_i2s1_port>;
+};
+
+&d1_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-3";
+	dais = <&d1_i2s2_port>;
+};
+
+&d1_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-1";
+	dais = <&d1_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "okay";
+};
+&d1_isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+&d1_dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+&d1_mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+&d1_csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+&d1_csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+&d1_csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+&d1_csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&d1_sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&d1_sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw315:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_die1_gpio18_default>;
+		pinctrl-1 = <&pinctrl_die1_gpio16_default>;
+		irq-gpios = <&d1_porta 18 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio16_default>;
+		gpio_wl_reg_on = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&sdio1 {
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio15_default>;
+	rst-gpios = <&porta 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio13_default>;
+	rst-gpios = <&d1_porta 13 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio14_default>;
+	rst-gpios = <&porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio14_default>;
+	rst-gpios = <&d1_porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d1_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+&d1_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d1_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "disabled";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d1_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d1_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+&d1_ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d1_bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&d1_portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&d1_portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+&d1_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+&d1_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+&d1_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+&d1_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+&d1_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+&d1_fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d1_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d1_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s1_endpoint>;
+			};
+		};
+	};
+	d1_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			d0_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+&d1_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "d1_npu_vdd";
+		regulators{
+			d1_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC2";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	d0_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio20_default>;
+		irq-gpios = <&porta 20 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d1_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	d1_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio0_default>;
+		irq-gpios = <&d1_porta 0 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&d1_portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+&d1_i2c3 {
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+&d1_i2c4 {
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+&d1_i2c5 {
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c6 {
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c7 {
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+&d1_i2c8 {
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c9 {
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ina226x3 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x40>;
+		shunt-resistor = <1000>;
+	};
+	ina226@41 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_lpddr_vdd";
+		reg = <0x41>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+
+&d1_aon_i2c0 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "vdd_dc_in";
+			reg = <0x40>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d1_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+&d1_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+	compatible = "eswin,eswin-dummy-pvt-cpu";
+	eswin,syscon = <&d0_sys_con 0x0668>;
+};
+&d1_pvt0 {
+	status = "okay";
+	compatible = "eswin,eswin-dummy-pvt-cpu";
+	eswin,syscon = <&d1_sys_con 0x0668>;
+};
+
+&pvt1 {
+	status = "okay";
+};
+&d1_pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+&d1_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+&d1_timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "disabled";
+};
+&d1_timer1 {
+	status = "disabled";
+};
+
+&timer2 {
+	status = "okay";
+};
+&d1_timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+&d1_timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+&d1_pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&d1_gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d0_npu_vcc>;
+	status = "okay";
+};
+&d1_llc_dev{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d1_npu_vcc>;
+	status = "okay";
+};
+
+&d2d {
+	status = "disabled";
+};
+
+&d1_d2d {
+	status = "disabled";
+};
+
+/* die0 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : gphy0 resetn(O)
+	gpio18  : ina226 alert(I)
+	gpio20  : lcd touch int(I)
+	gpio21  : head phone plug/unplug detection2(I)
+	gpio27  : black light pwr_en(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : fan pwm(O)
+	gpio69  : fan touch(I)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : mipi dsi resetn(O)
+	gpio111 : lcd touch resetn(O)
+*/
+/* die1 GPIO Function Description
+
+	gpio0   : lcd touch int(I)
+	gpio13  : gphy0 resetn(O)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : bt wake host(I)
+	gpio16  : sdio wifi disable(O)
+	gpio18  : sdio wifi wake host(I)
+	gpio19  : pcie prstn(I)
+	gpio20  : head phone plug/unplug detection2(I)
+	gpio21  : mipi dsi resetn(O)
+	gpio27  : tmp alert(I)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : system led(O)
+	gpio69  : black light pwr_en(O)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : host wak bt(O)
+	gpio111 : lcd touch resetn(O)
+*/
diff --git a/arch/riscv/boot/dts/eswin/eic7702-som560-interleave.dtsi b/arch/riscv/boot/dts/eswin/eic7702-som560-interleave.dtsi
new file mode 100644
index 000000000000..1f684879fe10
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-som560-interleave.dtsi
@@ -0,0 +1,1482 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7702 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eswin-win2030-die1-soc.dtsi"
+
+#include "eic7x-die0-pinctrl.dtsi"
+#include "eic7x-die1-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 SOM560";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		ethernet2 = &d1_gmac0;
+		ethernet3 = &d1_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	memory@4000000000 {
+		device_type = "memory";
+		reg = <0x40 0x00000000 0x0 0x80000000>;
+		numa-node-id = <0>;
+	};
+	memory@4100000000 {
+		device_type = "memory";
+		reg = <0x41 0x00000000 0x1 0xe0000000>;
+		numa-node-id = <0>;
+	};
+	memory@42e0000000 {
+		device_type = "memory";
+		reg = <0x42 0xe0000000 0x1 0xe0000000>;
+		numa-node-id = <1>;
+	};
+	memory@c0000000 {
+		device_type = "memory";
+		reg = <0x00 0xc0000000 0x0 0x40000000>;
+		numa-node-id = <0>;
+	};
+	memory@2040000000 {
+		device_type = "memory";
+		reg = <0x20 0x40000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+	memory@300000000 {
+		device_type = "memory";
+		reg = <0x3 0x0 0x1 0x80000000>;
+		numa-node-id = <0>;
+	};
+	memory@2280000000 {
+		device_type = "memory";
+		reg = <0x22 0x80000000 0x1 0x80000000>;
+		numa-node-id = <1>;
+	};
+	memory@2e00000000 {
+		device_type = "memory";
+		reg = <0x2 0xe0000000 0x0 0x20000000>;
+		numa-node-id = <0>;
+	};
+	memory@2260000000 {
+		device_type = "memory";
+		reg = <0x22 0x60000000 0x0 0x20000000>;
+		numa-node-id = <1>;
+	};
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x41 0x00000000 0x1 0x0>,<0x43 0x00000000 0x1 0x0>;
+			linux,cma-default;
+		};
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x22 0x80000000 0x1 0x80000000>;
+			no-map;
+		};
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		d0_dc_reserved: d0_dc_reserved@2e0000000 {
+			compatible = "shared-dma-pool";
+			reusable;
+			reg = <0x2 0xe0000000 0x0 0x20000000>;
+		};
+
+		d1_dc_reserved: d1_dc_reserved@2260000000 {
+			compatible = "shared-dma-pool";
+			reusable;
+			reg = <0x22 0x60000000 0x0 0x20000000>;
+		};
+
+		// g2d_4GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x0 0xfffff000 0x0 0x1000>;
+		// };
+
+		// g2d_8GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x1 0xfffff000 0x0 0x1000>;
+		// };
+
+		// g2d_12GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x2 0xfffff000 0x0 0x1000>;
+		// };
+
+		nid_1_zero_device_simu: nid_1_zero_device_simu@2040000000 {
+			no-map;
+			reg = <0x20 0x40000000 0x0 0x400000>;
+		};
+		// d1_g2d_4GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x20 0xfffff000 0x0 0x1000>;
+		// };
+
+		// d1_g2d_8GB_boundary_reserved_4k {
+		// 	no-map;
+		// 	reg = <0x21 0xfffff000 0x0 0x1000>;
+		// };
+
+
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 100>,
+				<1 0 100>,
+				<1 1 10>;
+	};
+
+	vdd_soc: regulator-0 {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio21_default>;
+		regulator-name = "soc-adjust";
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <900000>;
+		regulator-boot-on;
+		regulator-always-on;
+		enable-active-high;
+		gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+		states = <800000 0x0>, <900000 0x1>;
+	};
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d1_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+// &cpu_0 {
+//	cpu-supply = <&vdd_soc>;
+// };
+
+&d0_clock {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio94_default>;
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1800M>;
+};
+
+&d1_clock {
+	status = "okay";
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1800M>;
+};
+
+&D1CACHE {
+	status = "okay";
+	d1_zero_device: zero-device@3a000000 {
+		compatible = "l3,zero-device";
+		/*reg = <0x0 0x3a000000 0x0 0x400000>;*/
+		memory-region = <&nid_1_zero_device_simu>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+&d1_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+&d1_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+&d1_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&d1_ddr0 {
+	status = "okay";
+};
+
+&d1_ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+&d1_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "disabled";
+};
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&pcie {
+	status = "okay";
+};
+&d1_pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&d0_npu_vcc>;
+};
+&d1_npu {
+	status = "okay";
+	npu-supply=<&d1_npu_vcc>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+&d1_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+&d1_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+&d1_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+&d1_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+&d1_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+&d1_gpu {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+&d1_gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+&vdec1 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+&venc1 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+	memory-region = <&d0_dc_reserved>;
+};
+
+&d1_video_output {
+	status = "okay";
+	memory-region = <&d1_dc_reserved>;
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&d1_dc {
+	status = "okay";
+	d1_dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d1_dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&d1_dsi_input>;
+		};
+		d1_dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&d1_vd_input>;
+		};
+		d1_dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&d1_hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+&d1_dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&d1_virtual_display {
+	status = "okay";
+	port {
+		d1_vd_input: endpoint {
+			remote-endpoint = <&d1_dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+&d1_dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			d1_dsi_input: endpoint {
+				remote-endpoint = <&d1_dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			d1_mipi_dsi_out: endpoint {
+				remote-endpoint = <&d1_panel_in>;
+			};
+		};
+	};
+	d1_dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			d1_panel_in: endpoint {
+				remote-endpoint = <&d1_mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio27_default &pinctrl_die0_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+&d1_dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio69_default &pinctrl_die1_gpio21_default>;
+	backlight0-gpios = <&d1_portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+&d1_dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			d1_hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&d1_dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			d1_hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+&d1_dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+
+&d0_i2s0 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s0 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s0_port: port {
+		d1_i2s0_endpoint: endpoint {
+			remote-endpoint = <&d1_hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+};
+
+&d1_i2s1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_i2s2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio-0";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+};
+
+&d0_graphcard2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio-1";
+	dais = <&d1_i2s0_port>;
+};
+
+&d1_graphcard1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_graphcard2 {
+	/* unused */
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "okay";
+};
+&d1_isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+&d1_isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+&d1_dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+&d1_mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+&d1_csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "okay";
+};
+&d1_csi_dma1 {
+	status = "okay";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+&d1_csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "okay";
+};
+&d1_csi2_1 {
+	status = "okay";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&d1_sdhci_emmc {
+	/* unused */
+	status = "disabled";
+};
+
+&sdio0 {
+	status = "okay";
+};
+
+&d1_sdio0 {
+	status = "okay";
+};
+
+&sdio1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio29_default>;
+	rst-gpios = <&porta 29 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x96206420 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio29_default>;
+	rst-gpios = <&d1_porta 29 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x8C205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio34_default>;
+	rst-gpios = <&portb 2 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46205020 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio34_default>;
+	rst-gpios = <&d1_portb 2 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d1_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+&d1_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d1_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart1 {
+	status = "okay";
+};
+
+&d0_uart2 {
+	status = "okay";
+};
+
+&d1_uart2 {
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	status = "okay";
+};
+&d1_ssi0 {
+	status = "okay";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d1_bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&d1_portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&d1_portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+&d1_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+&d1_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+&d1_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+&d1_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+&d1_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+&d1_fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	status = "okay";
+};
+
+&d1_i2c0 {
+	status = "okay";
+};
+
+&d0_i2c1 {
+	status = "okay";
+};
+&d1_i2c1 {
+	status = "okay";
+};
+
+&d0_i2c2 {
+	status = "okay";
+};
+
+&d1_i2c2 {
+	status = "okay";
+};
+
+&d0_i2c3 {
+	status = "okay";
+};
+&d1_i2c3 {
+	status = "okay";
+};
+
+&d0_i2c4 {
+	status = "okay";
+};
+&d1_i2c4 {
+	status = "okay";
+};
+
+&d0_i2c5 {
+	status = "okay";
+};
+&d1_i2c5 {
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	status = "okay";
+};
+&d1_i2c8 {
+	status = "okay";
+};
+
+&d0_i2c9 {
+	status = "okay";
+};
+&d1_i2c9 {
+	status = "okay";
+};
+
+&d0_aon_i2c0 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_aon_i2c0 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c1 {
+	/* d2d vp pmic(5p49v6965) & temp sensor(tmp102) & d0 npu pmic(tps549d22) */
+	status = "okay";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "d0_npu_vdd";
+		regulators{
+			d0_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d1_aon_i2c1 {
+	/* d1 npu pmic */
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "d1_npu_vdd";
+		regulators{
+			d1_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC2";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+	compatible = "eswin,eswin-dummy-pvt-cpu";
+	eswin,syscon = <&d0_sys_con 0x0668>;
+};
+&d1_pvt0 {
+	status = "okay";
+	compatible = "eswin,eswin-dummy-pvt-cpu";
+	eswin,syscon = <&d1_sys_con 0x0668>;
+};
+
+&pvt1 {
+	status = "okay";
+};
+&d1_pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "okay";
+};
+&d1_wdt0 {
+	status = "okay";
+};
+
+&wdt1 {
+	status = "okay";
+};
+&d1_wdt1 {
+	status = "okay";
+};
+
+&wdt2 {
+	status = "okay";
+};
+&d1_wdt2 {
+	status = "okay";
+};
+
+&wdt3 {
+	status = "okay";
+};
+&d1_wdt3 {
+	status = "okay";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+&d1_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+&d1_timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+&d1_timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+&d1_timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+&d1_timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+&d1_pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&d1_gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d0_npu_vcc>;
+	status = "okay";
+};
+&d1_llc_dev{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d1_npu_vcc>;
+	status = "okay";
+};
+
+&d2d {
+	status = "okay";
+};
+
+&d1_d2d {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7702-som560.dtsi b/arch/riscv/boot/dts/eswin/eic7702-som560.dtsi
new file mode 100644
index 000000000000..5f83942fd303
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-som560.dtsi
@@ -0,0 +1,1443 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7702 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eswin-win2030-die1-soc.dtsi"
+
+#include "eic7x-die0-pinctrl.dtsi"
+#include "eic7x-die1-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 SOM560";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		ethernet2 = &d1_gmac0;
+		ethernet3 = &d1_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@2000000000 {
+		device_type = "memory";
+		reg = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <1>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		nid_1_zero_device_simu: nid_1_zero_device_simu@2040000000 {
+			no-map;
+			reg = <0x20 0x40000000 0x0 0x400000>;
+		};
+
+		d1_g2d_4GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x20 0xfffff000 0x0 0x1000>;
+		};
+
+		d1_g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x21 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x22 0x80000000 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 100>,
+				<1 0 100>,
+				<1 1 10>;
+	};
+
+	vdd_soc: regulator-0 {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio21_default>;
+		regulator-name = "soc-adjust";
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <900000>;
+		regulator-boot-on;
+		regulator-always-on;
+		enable-active-high;
+		gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+		states = <800000 0x0>, <900000 0x1>;
+	};
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d1_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+// &cpu_0 {
+//	cpu-supply = <&vdd_soc>;
+// };
+
+&d0_clock {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio94_default>;
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1800M>;
+};
+
+&d1_clock {
+	status = "okay";
+	cpu-voltage-gpios = <&portc 30 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1800M>;
+};
+
+&D1CACHE {
+	status = "okay";
+	d1_zero_device: zero-device@3a000000 {
+		compatible = "l3,zero-device";
+		/*reg = <0x0 0x3a000000 0x0 0x400000>;*/
+		memory-region = <&nid_1_zero_device_simu>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+&d1_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+&d1_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+&d1_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&d1_ddr0 {
+	status = "okay";
+};
+
+&d1_ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+&d1_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "disabled";
+};
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&pcie {
+	status = "okay";
+};
+&d1_pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&d0_npu_vcc>;
+};
+&d1_npu {
+	status = "okay";
+	npu-supply=<&d1_npu_vcc>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+&d1_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+&d1_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+&d1_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+&d1_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+&d1_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+&d1_gpu {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+&d1_gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+&vdec1 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+&venc1 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&d1_video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&d1_dc {
+	status = "okay";
+	d1_dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d1_dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&d1_dsi_input>;
+		};
+		d1_dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&d1_vd_input>;
+		};
+		d1_dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&d1_hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+&d1_dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&d1_virtual_display {
+	status = "okay";
+	port {
+		d1_vd_input: endpoint {
+			remote-endpoint = <&d1_dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+&d1_dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			d1_dsi_input: endpoint {
+				remote-endpoint = <&d1_dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			d1_mipi_dsi_out: endpoint {
+				remote-endpoint = <&d1_panel_in>;
+			};
+		};
+	};
+	d1_dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			d1_panel_in: endpoint {
+				remote-endpoint = <&d1_mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio27_default &pinctrl_die0_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+&d1_dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio69_default &pinctrl_die1_gpio21_default>;
+	backlight0-gpios = <&d1_portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+&d1_dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			d1_hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&d1_dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			d1_hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+&d1_dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+
+&d0_i2s0 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s0 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s0_port: port {
+		d1_i2s0_endpoint: endpoint {
+			remote-endpoint = <&d1_hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+};
+
+&d1_i2s1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_i2s2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio-0";
+	dais = <&d0_i2s0_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+};
+
+&d0_graphcard2 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_graphcard0 {
+	status = "okay";
+	label = "HDMI Audio-1";
+	dais = <&d1_i2s0_port>;
+};
+
+&d1_graphcard1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_graphcard2 {
+	/* unused */
+	status = "disabled";
+};
+
+&isp_0 {
+	status = "disabled";
+};
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+&d1_isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+&d1_dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+&d1_mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+&d1_csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "okay";
+};
+&d1_csi_dma1 {
+	status = "okay";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+&d1_csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "okay";
+};
+&d1_csi2_1 {
+	status = "okay";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&d1_sdhci_emmc {
+	/* unused */
+	status = "disabled";
+};
+
+&sdio0 {
+	status = "okay";
+};
+
+&d1_sdio0 {
+	status = "okay";
+};
+
+&sdio1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio29_default>;
+	rst-gpios = <&porta 29 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x96206420 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio29_default>;
+	rst-gpios = <&d1_porta 29 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x8C205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x96205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio34_default>;
+	rst-gpios = <&portb 2 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46205020 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205020 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio34_default>;
+	rst-gpios = <&d1_portb 2 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x20202020 0x46205A20 0x20202020>;
+	dly-param-100m = <0x20202020 0x50205A20 0x20202020>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d1_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+&d1_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d1_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart1 {
+	status = "okay";
+};
+
+&d0_uart2 {
+	status = "okay";
+};
+
+&d1_uart2 {
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	status = "okay";
+};
+&d1_ssi0 {
+	status = "okay";
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d1_bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&d1_portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&d1_portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+&d1_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+&d1_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+&d1_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+&d1_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+&d1_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+&d1_fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	status = "okay";
+};
+
+&d1_i2c0 {
+	status = "okay";
+};
+
+&d0_i2c1 {
+	status = "okay";
+};
+&d1_i2c1 {
+	status = "okay";
+};
+
+&d0_i2c2 {
+	status = "okay";
+};
+
+&d1_i2c2 {
+	status = "okay";
+};
+
+&d0_i2c3 {
+	status = "okay";
+};
+&d1_i2c3 {
+	status = "okay";
+};
+
+&d0_i2c4 {
+	status = "okay";
+};
+&d1_i2c4 {
+	status = "okay";
+};
+
+&d0_i2c5 {
+	status = "okay";
+};
+&d1_i2c5 {
+	status = "okay";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	status = "okay";
+};
+&d1_i2c8 {
+	status = "okay";
+};
+
+&d0_i2c9 {
+	status = "okay";
+};
+&d1_i2c9 {
+	status = "okay";
+};
+
+&d0_aon_i2c0 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_aon_i2c0 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c1 {
+	/* d2d vp pmic(5p49v6965) & temp sensor(tmp102) & d0 npu pmic(tps549d22) */
+	status = "okay";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "d0_npu_vdd";
+		regulators{
+			d0_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d1_aon_i2c1 {
+	/* d1 npu pmic */
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "d1_npu_vdd";
+		regulators{
+			d1_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC2";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+	compatible = "eswin,eswin-dummy-pvt-cpu";
+	eswin,syscon = <&d0_sys_con 0x0668>;
+};
+&d1_pvt0 {
+	status = "okay";
+	compatible = "eswin,eswin-dummy-pvt-cpu";
+	eswin,syscon = <&d1_sys_con 0x0668>;
+};
+
+&pvt1 {
+	status = "okay";
+};
+&d1_pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "okay";
+};
+&d1_wdt0 {
+	status = "okay";
+};
+
+&wdt1 {
+	status = "okay";
+};
+&d1_wdt1 {
+	status = "okay";
+};
+
+&wdt2 {
+	status = "okay";
+};
+&d1_wdt2 {
+	status = "okay";
+};
+
+&wdt3 {
+	status = "okay";
+};
+&d1_wdt3 {
+	status = "okay";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+&d1_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+&d1_timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+&d1_timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+&d1_timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+&d1_timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+&d1_pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&d1_gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d0_npu_vcc>;
+	status = "okay";
+};
+&d1_llc_dev{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d1_npu_vcc>;
+	status = "okay";
+};
+
+&d2d {
+	status = "disabled";
+};
+
+&d1_d2d {
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7702-tb.dts b/arch/riscv/boot/dts/eswin/eic7702-tb.dts
new file mode 100644
index 000000000000..767eccef2e04
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7702-tb.dts
@@ -0,0 +1,1674 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eswin-win2030-die1-soc.dtsi"
+
+#include "eic7x-die0-pinctrl.dtsi"
+#include "eic7x-die1-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 TB";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		ethernet2 = &d1_gmac0;
+		ethernet3 = &d1_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &d1_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@2000000000 {
+		device_type = "memory";
+		reg = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <1>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		nid_1_zero_device_simu: nid_1_zero_device_simu@2040000000 {
+			no-map;
+			reg = <0x20 0x40000000 0x0 0x400000>;
+		};
+
+		d1_g2d_4GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x20 0xfffff000 0x0 0x1000>;
+		};
+
+		d1_g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x21 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x22 0x80000000 0x1 0x80000000>;
+			no-map;
+		};
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 100>,
+				<1 0 100>,
+				<1 1 10>;
+	};
+
+	/* die1 gpio68 system led */
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio68_default>;
+
+		gpio-68 {
+			gpios = <&d1_portc 4 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&D1CACHE {
+	status = "okay";
+	d1_zero_device: zero-device@3a000000 {
+		compatible = "l3,zero-device";
+		/*reg = <0x0 0x3a000000 0x0 0x400000>;*/
+		memory-region = <&nid_1_zero_device_simu>;
+	};
+};
+
+&d0_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d1_cpu_opp_table {
+	opp-1500000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1500M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1600000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1600M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1700000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1700M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+	opp-1800000000 {
+		opp-hz = /bits/ 64 <CLK_FREQ_1800M>;
+		opp-microvolt = <900000>;
+		clock-latency-ns = <70000>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio19_default>;
+	cpu-voltage-gpios = <&porta 19 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1800M>;
+};
+
+&d1_clock {
+	status = "okay";
+	cpu-voltage-gpios = <&porta 19 GPIO_ACTIVE_HIGH>;
+	cpu-default-frequency = <CLK_FREQ_1800M>;
+};
+
+&d0_reset {
+	status = "okay";
+};
+&d1_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+&d1_pmu {
+	status = "disabled";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&d1_ddr0 {
+	status = "okay";
+};
+
+&d1_ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+&d1_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&pcie {
+	status = "okay";
+};
+&d1_pcie {
+	status = "okay";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&d0_npu_vcc>;
+	npu_def_high_vol = <1100000>;
+};
+&d1_npu {
+	status = "okay";
+	npu-supply=<&d1_npu_vcc>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+&d1_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+&d1_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+&d1_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+&d1_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+&d1_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+&d1_gpu {
+	status = "disabled";
+};
+
+&gc820 {
+	status = "okay";
+};
+&d1_gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+&vdec1 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+&venc1 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&d1_video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&d1_dc {
+	status = "okay";
+	d1_dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d1_dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&d1_dsi_input>;
+		};
+		d1_dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&d1_vd_input>;
+		};
+		d1_dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&d1_hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+&d1_dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&d1_virtual_display {
+	status = "okay";
+	port {
+		d1_vd_input: endpoint {
+			remote-endpoint = <&d1_dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+&d1_dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			d1_dsi_input: endpoint {
+				remote-endpoint = <&d1_dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			d1_mipi_dsi_out: endpoint {
+				remote-endpoint = <&d1_panel_in>;
+			};
+		};
+	};
+	d1_dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			d1_panel_in: endpoint {
+				remote-endpoint = <&d1_mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio27_default &pinctrl_die0_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+&d1_dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio69_default &pinctrl_die1_gpio21_default>;
+	backlight0-gpios = <&d1_portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+&d1_dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			d1_hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&d1_dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			d1_hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+&d1_dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+
+&d0_i2s0 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s0 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s0_port: port {
+		d1_i2s0_endpoint: endpoint {
+			remote-endpoint = <&d1_hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s1 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s1_port: port {
+		d1_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d1_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s2 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s2_port: port {
+		d1_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d1_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-0";
+	dais = <&d0_i2s0_port>;
+};
+
+&d1_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-2";
+	dais = <&d1_i2s1_port>;
+};
+
+&d1_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-3";
+	dais = <&d1_i2s2_port>;
+};
+
+&d1_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-1";
+	dais = <&d1_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "okay";
+};
+&d1_isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+&d1_dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+&d1_mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+&d1_csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+&d1_csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+&d1_csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+&d1_csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&d1_sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&d1_sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw315:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_die1_gpio18_default>;
+		pinctrl-1 = <&pinctrl_die1_gpio16_default>;
+		irq-gpios = <&d1_porta 18 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio16_default>;
+		gpio_wl_reg_on = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&sdio1 {
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio15_default>;
+	rst-gpios = <&porta 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio13_default>;
+	rst-gpios = <&d1_porta 13 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio14_default>;
+	rst-gpios = <&porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio14_default>;
+	rst-gpios = <&d1_porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d1_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+&d1_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d1_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "disabled";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d1_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d1_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+&d1_ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d1_bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&d1_portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&d1_portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+&d1_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "okay";
+};
+&d1_mbox1 {
+	status = "okay";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+&d1_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+&d1_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+&d1_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+&d1_fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d1_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d1_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s1_endpoint>;
+			};
+		};
+	};
+	d1_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			d0_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+&d1_i2c1 {
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "d1_npu_vdd";
+		regulators{
+			d1_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC2";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	d0_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio20_default>;
+		irq-gpios = <&porta 20 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d1_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	d1_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio0_default>;
+		irq-gpios = <&d1_porta 0 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&d1_portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+&d1_i2c3 {
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+&d1_i2c4 {
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+&d1_i2c5 {
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c6 {
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c7 {
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+&d1_i2c8 {
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c9 {
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d0_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+
+&d1_aon_i2c0 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_vdd";
+			reg = <0x40>;
+			shunt-resistor = <1000>;
+		};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d1_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+&d1_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "disabled";
+};
+&d1_pvt0 {
+	status = "disabled";
+};
+
+&pvt1 {
+	status = "okay";
+};
+&d1_pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+&d1_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+&d1_timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "disabled";
+};
+&d1_timer1 {
+	status = "disabled";
+};
+
+&timer2 {
+	status = "okay";
+};
+&d1_timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+&d1_timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+&d1_pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&d1_gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d0_npu_vcc>;
+	status = "okay";
+};
+&d1_llc_dev{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d1_npu_vcc>;
+	status = "okay";
+};
+
+&d2d {
+	status = "disabled";
+};
+
+&d1_d2d {
+	status = "disabled";
+};
+
+/* die0 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : gphy0 resetn(O)
+	gpio18  : ina226 alert(I)
+	gpio20  : lcd touch int(I)
+	gpio21  : head phone plug/unplug detection2(I)
+	gpio27  : black light pwr_en(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : fan pwm(O)
+	gpio69  : fan touch(I)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : mipi dsi resetn(O)
+	gpio111 : lcd touch resetn(O)
+*/
+/* die1 GPIO Function Description
+
+	gpio0   : lcd touch int(I)
+	gpio13  : gphy0 resetn(O)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : bt wake host(I)
+	gpio16  : sdio wifi disable(O)
+	gpio18  : sdio wifi wake host(I)
+	gpio19  : pcie prstn(I)
+	gpio20  : head phone plug/unplug detection2(I)
+	gpio21  : mipi dsi resetn(O)
+	gpio27  : tmp alert(I)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : system led(O)
+	gpio69  : black light pwr_en(O)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : host wak bt(O)
+	gpio111 : lcd touch resetn(O)
+*/
diff --git a/arch/riscv/boot/dts/eswin/eic770x-opptable.dtsi b/arch/riscv/boot/dts/eswin/eic770x-opptable.dtsi
new file mode 100644
index 000000000000..6198424e38d5
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic770x-opptable.dtsi
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for pin control of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <dt-bindings/clock/eswin,eic7700-clock.h>
+
+#define CLK_FREQ_1800M					1800000000
+#define CLK_FREQ_1700M					1700000000
+#define CLK_FREQ_1600M					1600000000
+#define CLK_FREQ_1500M					1500000000
+#define CLK_FREQ_1400M					1400000000
+#define CLK_FREQ_1300M					1300000000
+#define CLK_FREQ_1200M					1200000000
+#define CLK_FREQ_1000M					1000000000
+#define CLK_FREQ_900M					900000000
+#define CLK_FREQ_800M					800000000
+#define CLK_FREQ_700M					700000000
+#define CLK_FREQ_600M					600000000
+#define CLK_FREQ_500M					500000000
+#define CLK_FREQ_400M					400000000
+#define CLK_FREQ_200M					200000000
+#define CLK_FREQ_100M					100000000
+#define CLK_FREQ_24M					24000000
+
+/ {
+	d0_cpu_opp_table: opp-table-d0@cpu {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-24000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_24M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-100000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_100M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-200000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_200M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-400000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_400M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-500000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_500M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-600000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_600M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-700000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_700M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-800000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_800M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-900000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_900M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1000M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1200M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1300000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1300M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1400000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1400M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+	};
+
+	d1_cpu_opp_table: opp-table-d1@cpu {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-24000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_24M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-100000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_100M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-200000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_200M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-400000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_400M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-500000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_500M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-600000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_600M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-700000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_700M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-800000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_800M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-900000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_900M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1000M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1200M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1300000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1300M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1400000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1400M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+	};
+
+	dsp_opp_table: opp-table@dsp {
+		compatible = "operating-points-v2";
+		opp@520000000 {
+			opp-hz = /bits/ 64 <520000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@1040000000 {
+			opp-hz = /bits/ 64 <1040000000>;
+			opp-microvolt = <800000>;
+		};
+	};
+	npu_opp_table: opp-table@npu {
+		compatible = "operating-points-v2";
+		opp@1040000000 {
+			opp-hz = /bits/ 64 <1040000000>;
+			opp-microvolt = <900000>;
+		};
+		opp@1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-microvolt = <1050000>;
+		};
+
+	};
+
+	g2d_opp_table: opp-table@g2d {
+		compatible = "operating-points-v2";
+		opp@260000000 {
+			opp-hz = /bits/ 64 <260000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@520000000 {
+			opp-hz = /bits/ 64 <520000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@693333334 {
+			opp-hz = /bits/ 64 <693333334>;
+			opp-microvolt = <800000>;
+		};
+		opp@1040000000 {
+			opp-hz = /bits/ 64 <1040000000>;
+			opp-microvolt = <800000>;
+		};
+	};
+
+	gpu_opp_table: opp-table@gpu {
+		compatible = "operating-points-v2";
+		opp@200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <800000>;
+		};
+	};
+
+	vi_opp_table: opp-table@vi {
+		compatible = "operating-points-v2";
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-microvolt = <800000>;
+		};
+	};
+
+	venc_opp_table: opp-table@venc {
+		compatible = "operating-points-v2";
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-microvolt = <800000>;
+		};
+	};
+
+	vdec_opp_table: opp-table@vdec {
+		compatible = "operating-points-v2";
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <800000>;
+		};
+		opp@200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-microvolt = <800000>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7x-die0-pinctrl.dtsi b/arch/riscv/boot/dts/eswin/eic7x-die0-pinctrl.dtsi
new file mode 100644
index 000000000000..8409a0886b7d
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7x-die0-pinctrl.dtsi
@@ -0,0 +1,1447 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for pin control of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "eswin-win2030-die0-soc.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+    #address-cells = <2>;
+	#size-cells = <2>;
+    soc {
+        pinctrl: pinctrl@0x51600080 {
+			compatible = "eswin,eic7x-pinctrl", "syscon";
+			reg = <0x0 0x51600080 0x0 0x1FFF80>;
+			status = "disabled";
+
+			//Dual Die Configuration
+			//func0
+			/* used with pinctrl_die1_jtag1_off_default*/
+			pinctrl_die0_jtag1_on_default: jtag1_on-default{
+				mux {
+					groups = "jtag1_on_group";
+					function = "jtag1_on_func";
+				};
+			};
+			/* used with pinctrl_die1_jtag1_on_default*/
+			pinctrl_die0_jtag1_off_default: jtag1_off-default{
+				mux {
+					groups = "jtag1_off_group";
+					function = "jtag1_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_jtag2_off_default*/
+			pinctrl_die0_jtag2_on_default: jtag2_on-default{
+				mux {
+					groups = "jtag2_on_group";
+					function = "jtag2_on_func";
+				};
+			};
+			/* used with pinctrl_die1_jtag2_on_default*/
+			pinctrl_die0_jtag2_off_default: jtag2_off-default{
+				mux {
+					groups = "jtag2_off_group";
+					function = "jtag2_off_func";
+				};
+			};
+
+			//func2
+			/* used with pinctrl_die1_gpio7_off_default*/
+			pinctrl_die0_gpio7_on_default: gpio7_on-default{
+				mux {
+					groups = "gpio7_on_group";
+					function = "gpio7_on_func";
+				};
+				conf {
+					groups = "gpio7_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio7_on_default*/
+			pinctrl_die0_gpio7_off_default: gpio7_off-default{
+				mux {
+					groups = "gpio7_off_group";
+					function = "gpio7_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_gpio8_off_default*/
+			pinctrl_die0_gpio8_on_default: gpio8_on-default{
+				mux {
+					groups = "gpio8_on_group";
+					function = "gpio8_on_func";
+				};
+				conf {
+					groups = "gpio8_on_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio8_on_default*/
+			pinctrl_die0_gpio8_off_default: gpio8_off-default{
+				mux {
+					groups = "gpio8_off_group";
+					function = "gpio8_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_gpio9_off_default*/
+			pinctrl_die0_gpio9_on_default: gpio9_on-default{
+				mux {
+					groups = "gpio9_on_group";
+					function = "gpio9_on_func";
+				};
+				conf {
+					groups = "gpio9_on_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio9_on_default*/
+			pinctrl_die0_gpio9_off_default: gpio9_off-default{
+				mux {
+					groups = "gpio9_off_group";
+					function = "gpio9_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_gpio17_off_default*/
+			pinctrl_die0_gpio17_on_default: gpio17_on-default{
+				mux {
+					groups = "gpio17_on_group";
+					function = "gpio17_on_func";
+				};
+				conf {
+					groups = "gpio17_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio17_on_default*/
+			pinctrl_die0_gpio17_off_default: gpio17_off-default{
+				mux {
+					groups = "gpio17_off_group";
+					function = "gpio17_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_gpio64_off_default*/
+			pinctrl_die0_gpio64_on_default: gpio64_on-default{
+				mux {
+					groups = "gpio64_on_group";
+					function = "gpio64_on_func";
+				};
+				conf {
+					groups = "gpio64_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio64_on_default*/
+			pinctrl_die0_gpio64_off_default: gpio64_off-default{
+				mux {
+					groups = "gpio64_off_group";
+					function = "gpio64_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_gpio65_off_default*/
+			pinctrl_die0_gpio65_on_default: gpio65_on-default{
+				mux {
+					groups = "gpio65_on_group";
+					function = "gpio65_on_func";
+				};
+				conf {
+					groups = "gpio65_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio65_on_default*/
+			pinctrl_die0_gpio65_off_default: gpio65_off-default{
+				mux {
+					groups = "gpio65_off_group";
+					function = "gpio65_off_func";
+				};
+			};
+
+			/* used with pinctrl_die1_gpio66_off_default*/
+			pinctrl_die0_gpio66_on_default: gpio66_on-default{
+				mux {
+					groups = "gpio66_on_group";
+					function = "gpio66_on_func";
+				};
+				conf {
+					groups = "gpio66_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die1_gpio66_on_default*/
+			pinctrl_die0_gpio66_off_default: gpio66_off-default{
+				mux {
+					groups = "gpio66_off_group";
+					function = "gpio66_off_func";
+				};
+			};
+
+			//func6
+			/* used with pinctrl_die1_vc_g2d0_debug_out_off_default*/
+			pinctrl_die0_vc_g2d0_debug_out_on_default: vc_g2d0_debug_out_on-default{
+				mux {
+					groups = "vc_g2d0_debug_out_on_group";
+					function = "vc_g2d0_debug_out_on_func";
+				};
+			};
+			/* used with pinctrl_die1_vc_g2d0_debug_out_on_default*/
+			pinctrl_die0_vc_g2d0_debug_out_off_default: vc_g2d0_debug_out_off-default{
+				mux {
+					groups = "vc_g2d0_debug_out_off_group";
+					function = "vc_g2d0_debug_out_off_func";
+				};
+			};
+
+			//func7
+			/* used with pinctrl_die1_ftm_test_out_off_default*/
+			pinctrl_die0_ftm_test_out_on_default: ftm_test_out_on-default{
+				mux {
+					groups = "ftm_test_out_on_group";
+					function = "ftm_test_out_on_func";
+				};
+			};
+			/* used with pinctrl_die1_ftm_test_out_on_default*/
+			pinctrl_die0_ftm_test_out_off_default: ftm_test_out_off-default{
+				mux {
+					groups = "ftm_test_out_off_group";
+					function = "ftm_test_out_off_func";
+				};
+			};
+
+            //func0
+            pinctrl_die0_sdio0_default: sdio0-default{
+				mux {
+					groups = "sdio0_group";
+					function = "sdio0_func";
+				};
+			};
+            pinctrl_die0_sdio1_default: sdio1-default{
+				mux {
+					groups = "sdio1_group";
+					function = "sdio1_func";
+				};
+			};
+            pinctrl_die0_jtag0_default: jtag0-default{
+				mux {
+					groups = "jtag0_group";
+					function = "jtag0_func";
+				};
+			};
+            pinctrl_die0_pcie_default: pcie-default{
+				mux{
+					groups = "pcie_group";
+					function = "pcie_func";
+				};
+			};
+            pinctrl_die0_hdmi_default: hdmi-default{
+				mux{
+					groups = "hdmi_group";
+					function = "hdmi_func";
+				};
+			};
+            pinctrl_die0_rgmii0_default: rgmii0-default{
+				mux {
+					groups = "rgmii0_group";
+					function = "rgmii0_func";
+				};
+			};
+            pinctrl_die0_rgmii1_default: rgmii1-default{
+				mux{
+					groups = "rgmii1_group";
+					function = "rgmii1_func";
+				};
+			};
+            pinctrl_die0_spi0_default: spi0-default{
+				mux {
+					groups = "spi0_group";
+					function = "spi0_func";
+				};
+			};
+            pinctrl_die0_spi1_default: spi1-default{
+				mux{
+					groups = "spi1_group";
+					function = "spi1_func";
+				};
+			};
+            pinctrl_die0_spi3_default: spi3-default{
+				mux {
+					groups = "spi3_group";
+					function = "spi3_func";
+				};
+			};
+            pinctrl_die0_por_time_sel0_default: por_time_sel0-default{
+				mux {
+					groups = "por_time_sel0_group";
+					function = "por_time_sel0_func";
+				};
+			};
+            pinctrl_die0_por_time_sel1_default: por_time_sel1-default{
+				mux {
+					groups = "por_time_sel1_group";
+					function = "por_time_sel1_func";
+				};
+			};
+            pinctrl_die0_i2s0_default: i2s0-default{
+				mux {
+					groups = "i2s0_group";
+					function = "i2s0_func";
+				};
+			};
+            pinctrl_die0_i2s1_default: i2s1-default{
+				mux {
+					groups = "i2s1_group";
+					function = "i2s1_func";
+				};
+			};
+            pinctrl_die0_i2s2_default: i2s2-default{
+				mux {
+					groups = "i2s2_group";
+					function = "i2s2_func";
+				};
+			};
+            pinctrl_die0_usb0_pwren_default: usb0_pwren-default{
+				mux {
+					groups = "usb0_pwren_group";
+					function = "usb0_pwren_func";
+				};
+			};
+            pinctrl_die0_usb1_pwren_default: usb1_pwren-default{
+				mux {
+					groups = "usb1_pwren_group";
+					function = "usb1_pwren_func";
+				};
+			};
+            pinctrl_die0_i2c0_default: i2c0-default{
+				mux {
+					groups = "i2c0_group";
+					function = "i2c0_func";
+				};
+			};
+            pinctrl_die0_i2c1_default: i2c1-default{
+				mux {
+					groups = "i2c1_group";
+					function = "i2c1_func";
+				};
+			};
+            pinctrl_die0_i2c2_default: i2c2-default{
+				mux {
+					groups = "i2c2_group";
+					function = "i2c2_func";
+				};
+			};
+            pinctrl_die0_i2c3_default: i2c3-default{
+				mux {
+					groups = "i2c3_group";
+					function = "i2c3_func";
+				};
+			};
+            pinctrl_die0_i2c4_default: i2c4-default{
+				mux {
+					groups = "i2c4_group";
+					function = "i2c4_func";
+				};
+			};
+            pinctrl_die0_i2c5_default: i2c5-default{
+				mux {
+					groups = "i2c5_group";
+					function = "i2c5_func";
+				};
+			};
+            pinctrl_die0_i2c8_default: i2c8-default{
+				mux {
+					groups = "i2c8_group";
+					function = "i2c8_func";
+				};
+			};
+            pinctrl_die0_i2c10_default: i2c10-default{
+				mux {
+					groups = "i2c10_group";
+					function = "i2c10_func";
+				};
+			};
+            pinctrl_die0_i2c11_default: i2c11-default{
+				mux {
+					groups = "i2c11_group";
+					function = "i2c11_func";
+				};
+			};
+            pinctrl_die0_uart0_default: uart0-default{
+				mux {
+					groups = "uart0_group";
+					function = "uart0_func";
+				};
+			};
+            pinctrl_die0_uart1_default: uart1-default{
+				mux {
+					groups = "uart1_group";
+					function = "uart1_func";
+				};
+			};
+            pinctrl_die0_uart2_default: uart2-default{
+				mux {
+					groups = "uart2_group";
+					function = "uart2_func";
+				};
+			};
+			pinctrl_die0_pwm0_default: pwm0-default{
+				mux {
+					groups = "pwm0_group";
+					function = "pwm0_func";
+				};
+			};
+            pinctrl_die0_fan_tach_default: fan_tach-default{
+				mux {
+					groups = "fan_tach_group";
+					function = "fan_tach_func";
+				};
+			};
+            pinctrl_die0_mipi_csi0_default: mipi_csi0-default{
+				mux {
+					groups = "mipi_csi0_group";
+					function = "mipi_csi0_func";
+				};
+			};
+            pinctrl_die0_mipi_csi1_default: mipi_csi1-default{
+				mux {
+					groups = "mipi_csi1_group";
+					function = "mipi_csi1_func";
+				};
+			};
+            pinctrl_die0_mipi_csi2_default: mipi_csi2-default{
+				mux {
+					groups = "mipi_csi2_group";
+					function = "mipi_csi2_func";
+				};
+			};
+            pinctrl_die0_mipi_csi3_default: mipi_csi3-default{
+				mux {
+					groups = "mipi_csi3_group";
+					function = "mipi_csi3_func";
+				};
+			};
+            pinctrl_die0_mipi_csi4_default: mipi_csi4-default{
+				mux {
+					groups = "mipi_csi4_group";
+					function = "mipi_csi4_func";
+				};
+			};
+            pinctrl_die0_mipi_csi5_default: mipi_csi5-default{
+				mux {
+					groups = "mipi_csi5_group";
+					function = "mipi_csi5_func";
+				};
+			};
+            pinctrl_die0_s_mode_default: s_mode-default{
+				mux {
+					groups = "s_mode_group";
+					function = "s_mode_func";
+				};
+			};
+            pinctrl_die0_ddr_refclk_sel_default: ddr_refclk_sel-default{
+				mux {
+					groups = "ddr_refclk_sel_group";
+					function = "ddr_refclk_sel_func";
+				};
+			};
+            pinctrl_die0_boot_sel_default: boot_sel-default{
+				mux {
+					groups = "boot_sel_group";
+					function = "boot_sel_func";
+				};
+			};
+            pinctrl_die0_lpddr_ref_clk_default: lpddr_ref_clk-default{
+				mux {
+					groups = "lpddr_ref_clk_group";
+					function = "lpddr_ref_clk_func";
+				};
+			};
+
+            //func1
+            pinctrl_die0_spi2_default: spi2-default{
+				mux1 {
+					groups = "spi2_clk_group";
+					function = "spi2_clk_func";
+				};
+				conf1 {
+					groups = "spi2_clk_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux2 {
+					groups = "spi2_d0_group";
+					function = "spi2_d0_func";
+				};
+				conf2 {
+					groups = "spi2_d0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux3 {
+					groups = "spi2_d1_d2_d3_group";
+					function = "spi2_d1_d2_d3_func";
+				};
+				conf3 {
+					groups = "spi2_d1_d2_d3_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux4 {
+					groups = "spi2_cs_group";
+					function = "spi2_cs_func";
+				};
+			};
+
+            pinctrl_die0_sata_act_led_default: sata_act_led-default{
+				mux {
+					groups = "sata_act_led_group";
+					function = "sata_act_led_func";
+				};
+				conf {
+					groups = "sata_act_led_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_emmc_led_control_default: emmc_led_control-default{
+				mux {
+					groups = "emmc_led_control_group";
+					function = "emmc_led_control_func";
+				};
+				conf {
+					groups = "emmc_led_control_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_sd0_led_control_default: sd0_led_control-default{
+				mux {
+					groups = "sd0_led_control_group";
+					function = "sd0_led_control_func";
+				};
+			};
+            pinctrl_die0_sd1_led_control_default: sd1_led_control-default{
+				mux {
+					groups = "sd1_led_control_group";
+					function = "sd1_led_control_func";
+				};
+			};
+            pinctrl_die0_i2c6_default: i2c6-default{
+				mux {
+					groups = "i2c6_group";
+					function = "i2c6_func";
+				};
+			};
+            pinctrl_die0_i2c7_default: i2c7-default{
+				mux {
+					groups = "i2c7_group";
+					function = "i2c7_func";
+				};
+			};
+            pinctrl_die0_i2c9_default: i2c9-default{
+				mux {
+					groups = "i2c9_group";
+					function = "i2c9_func";
+				};
+			};
+            pinctrl_die0_pwm1_default: pwm1-default{
+				mux {
+					groups = "pwm1_group";
+					function = "pwm1_func";
+				};
+			};
+            pinctrl_die0_pwm2_default: pwm2-default{
+				mux {
+					groups = "pwm2_group";
+					function = "pwm2_func";
+				};
+			};
+            pinctrl_die0_mipi_csi_xtrig_default: mipi_csi_xtrig-default{
+				mux {
+					groups = "mipi_csi_xtrig_group";
+					function = "mipi_csi_xtrig_func";
+				};
+			};
+
+            //func3
+            pinctrl_die0_uart3_default: uart3-default{
+				mux {
+					groups = "uart3_group";
+					function = "uart3_func";
+				};
+			};
+            pinctrl_die0_uart4_default: uart4-default{
+				mux {
+					groups = "uart4_group";
+					function = "uart4_func";
+				};
+			};
+
+            //gpio
+            pinctrl_die0_gpio0_default: gpio0-default{
+				mux {
+					groups = "gpio0_group";
+					function = "gpio0_func";
+				};
+				conf {
+					groups = "gpio0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die0_gpio1_default: gpio1-default{
+				mux {
+					groups = "gpio1_group";
+					function = "gpio1_func";
+				};
+			};
+            pinctrl_die0_gpio2_default: gpio2-default{
+				mux {
+					groups = "gpio2_group";
+					function = "gpio2_func";
+				};
+			};
+            pinctrl_die0_gpio3_default: gpio3-default{
+				mux {
+					groups = "gpio3_group";
+					function = "gpio3_func";
+				};
+			};
+            pinctrl_die0_gpio4_default: gpio4-default{
+				mux {
+					groups = "gpio4_group";
+					function = "gpio4_func";
+				};
+			};
+            pinctrl_die0_gpio5_default: gpio5-default{
+				mux {
+					groups = "gpio5_group";
+					function = "gpio5_func";
+				};
+				conf {
+					groups = "gpio5_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die0_gpio6_default: gpio6-default{
+				mux {
+					groups = "gpio6_group";
+					function = "gpio6_func";
+				};
+				conf {
+					groups = "gpio6_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio10_default: gpio10-default{
+				mux {
+					groups = "gpio10_group";
+					function = "gpio10_func";
+				};
+				conf {
+					groups = "gpio10_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die0_gpio11_default: gpio11-default{
+				mux {
+					groups = "gpio11_group";
+					function = "gpio11_func";
+				};
+				conf {
+					groups = "gpio11_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die0_gpio12_default: gpio12-default{
+				mux {
+					groups = "gpio12_group";
+					function = "gpio12_func";
+				};
+			};
+            pinctrl_die0_gpio13_default: gpio13-default{
+				mux {
+					groups = "gpio13_group";
+					function = "gpio13_func";
+				};
+			};
+            pinctrl_die0_gpio14_default: gpio14-default{
+				mux {
+					groups = "gpio14_group";
+					function = "gpio14_func";
+				};
+			};
+            pinctrl_die0_gpio15_default: gpio15-default{
+				mux {
+					groups = "gpio15_group";
+					function = "gpio15_func";
+				};
+			};
+            pinctrl_die0_gpio16_default: gpio16-default{
+				mux {
+					groups = "gpio16_group";
+					function = "gpio16_func";
+				};
+				conf {
+					groups = "gpio16_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio18_default: gpio18-default{
+				mux {
+					groups = "gpio18_group";
+					function = "gpio18_func";
+				};
+				conf {
+					groups = "gpio18_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio19_default: gpio19-default{
+				mux {
+					groups = "gpio19_group";
+					function = "gpio19_func";
+				};
+				conf {
+					groups = "gpio19_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio20_default: gpio20-default{
+				mux {
+					groups = "gpio20_group";
+					function = "gpio20_func";
+				};
+				conf {
+					groups = "gpio20_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio21_default: gpio21-default{
+				mux {
+					groups = "gpio21_group";
+					function = "gpio21_func";
+				};
+				conf {
+					groups = "gpio21_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio22_default: gpio22-default{
+				mux {
+					groups = "gpio22_group";
+					function = "gpio22_func";
+				};
+			};
+            pinctrl_die0_gpio23_default: gpio23-default{
+				mux {
+					groups = "gpio23_group";
+					function = "gpio23_func";
+				};
+			};
+            pinctrl_die0_gpio24_default: gpio24-default{
+				mux {
+					groups = "gpio24_group";
+					function = "gpio24_func";
+				};
+			};
+            pinctrl_die0_gpio25_default: gpio25-default{
+				mux {
+					groups = "gpio25_group";
+					function = "gpio25_func";
+				};
+			};
+            pinctrl_die0_gpio26_default: gpio26-default{
+				mux {
+					groups = "gpio26_group";
+					function = "gpio26_func";
+				};
+			};
+            pinctrl_die0_gpio27_default: gpio27-default{
+				mux {
+					groups = "gpio27_group";
+					function = "gpio27_func";
+				};
+			};
+            pinctrl_die0_gpio28_default: gpio28-default{
+				mux {
+					groups = "gpio28_group";
+					function = "gpio28_func";
+				};
+				conf {
+					groups = "gpio28_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die0_gpio29_default: gpio29-default{
+				mux {
+					groups = "gpio29_group";
+					function = "gpio29_func";
+				};
+			};
+            pinctrl_die0_gpio30_default: gpio30-default{
+				mux {
+					groups = "gpio30_group";
+					function = "gpio30_func";
+				};
+			};
+            pinctrl_die0_gpio31_default: gpio31-default{
+				mux {
+					groups = "gpio31_group";
+					function = "gpio31_func";
+				};
+			};
+            pinctrl_die0_gpio32_default: gpio32-default{
+				mux {
+					groups = "gpio32_group";
+					function = "gpio32_func";
+				};
+			};
+            pinctrl_die0_gpio33_default: gpio33-default{
+				mux {
+					groups = "gpio33_group";
+					function = "gpio33_func";
+				};
+			};
+            pinctrl_die0_gpio34_default: gpio34-default{
+				mux {
+					groups = "gpio34_group";
+					function = "gpio34_func";
+				};
+			};
+            pinctrl_die0_gpio35_default: gpio35-default{
+				mux {
+					groups = "gpio35_group";
+					function = "gpio35_func";
+				};
+				conf {
+					groups = "gpio35_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio36_default: gpio36-default{
+				mux {
+					groups = "gpio36_group";
+					function = "gpio36_func";
+				};
+				conf {
+					groups = "gpio36_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio37_default: gpio37-default{
+				mux {
+					groups = "gpio37_group";
+					function = "gpio37_func";
+				};
+				conf {
+					groups = "gpio37_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio38_default: gpio38-default{
+				mux {
+					groups = "gpio38_group";
+					function = "gpio38_func";
+				};
+				conf {
+					groups = "gpio38_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio39_default: gpio39-default{
+				mux {
+					groups = "gpio39_group";
+					function = "gpio39_func";
+				};
+				conf {
+					groups = "gpio39_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio40_default: gpio40-default{
+				mux {
+					groups = "gpio40_group";
+					function = "gpio40_func";
+				};
+				conf {
+					groups = "gpio40_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio41_default: gpio41-default{
+				mux {
+					groups = "gpio41_group";
+					function = "gpio41_func";
+				};
+			};
+            pinctrl_die0_gpio42_default: gpio42-default{
+				mux {
+					groups = "gpio42_group";
+					function = "gpio42_func";
+				};
+				conf {
+					groups = "gpio42_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio43_default: gpio43-default{
+				mux {
+					groups = "gpio43_group";
+					function = "gpio43_func";
+				};
+			};
+            pinctrl_die0_gpio44_default: gpio44-default{
+				mux {
+					groups = "gpio44_group";
+					function = "gpio44_func";
+				};
+			};
+            pinctrl_die0_gpio45_default: gpio45-default{
+				mux {
+					groups = "gpio45_group";
+					function = "gpio45_func";
+				};
+			};
+            pinctrl_die0_gpio46_default: gpio46-default{
+				mux {
+					groups = "gpio46_group";
+					function = "gpio46_func";
+				};
+				conf {
+					groups = "gpio46_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio47_default: gpio47-default{
+				mux {
+					groups = "gpio47_group";
+					function = "gpio47_func";
+				};
+				conf {
+					groups = "gpio47_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio48_default: gpio48-default{
+				mux {
+					groups = "gpio48_group";
+					function = "gpio48_func";
+				};
+			};
+            pinctrl_die0_gpio49_default: gpio49-default{
+				mux {
+					groups = "gpio49_group";
+					function = "gpio49_func";
+				};
+			};
+            pinctrl_die0_gpio50_default: gpio50-default{
+				mux {
+					groups = "gpio50_group";
+					function = "gpio50_func";
+				};
+			};
+            pinctrl_die0_gpio51_default: gpio51-default{
+				mux {
+					groups = "gpio51_group";
+					function = "gpio51_func";
+				};
+			};
+            pinctrl_die0_gpio52_default: gpio52-default{
+				mux {
+					groups = "gpio52_group";
+					function = "gpio52_func";
+				};
+				conf {
+					groups = "gpio52_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio53_default: gpio53-default{
+				mux {
+					groups = "gpio53_group";
+					function = "gpio53_func";
+				};
+				conf {
+					groups = "gpio53_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio54_default: gpio54-default{
+				mux {
+					groups = "gpio54_group";
+					function = "gpio54_func";
+				};
+			};
+            pinctrl_die0_gpio55_default: gpio55-default{
+				mux {
+					groups = "gpio55_group";
+					function = "gpio55_func";
+				};
+			};
+            pinctrl_die0_gpio56_default: gpio56-default{
+				mux {
+					groups = "gpio56_group";
+					function = "gpio56_func";
+				};
+			};
+            pinctrl_die0_gpio57_default: gpio57-default{
+				mux {
+					groups = "gpio57_group";
+					function = "gpio57_func";
+				};
+			};
+            pinctrl_die0_gpio58_default: gpio58-default{
+				mux {
+					groups = "gpio58_group";
+					function = "gpio58_func";
+				};
+			};
+            pinctrl_die0_gpio59_default: gpio59-default{
+				mux {
+					groups = "gpio59_group";
+					function = "gpio59_func";
+				};
+			};
+            pinctrl_die0_gpio60_default: gpio60-default{
+				mux {
+					groups = "gpio60_group";
+					function = "gpio60_func";
+				};
+			};
+            pinctrl_die0_gpio61_default: gpio61-default{
+				mux {
+					groups = "gpio61_group";
+					function = "gpio61_func";
+				};
+			};
+            pinctrl_die0_gpio62_default: gpio62-default{
+				mux {
+					groups = "gpio62_group";
+					function = "gpio62_func";
+				};
+			};
+            pinctrl_die0_gpio63_default: gpio63-default{
+				mux {
+					groups = "gpio63_group";
+					function = "gpio63_func";
+				};
+			};
+            pinctrl_die0_gpio67_default: gpio67-default{
+				mux {
+					groups = "gpio67_group";
+					function = "gpio67_func";
+				};
+				conf {
+					groups = "gpio67_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio68_default: gpio68-default{
+				mux {
+					groups = "gpio68_group";
+					function = "gpio68_func";
+				};
+			};
+            pinctrl_die0_gpio69_default: gpio69-default{
+				mux {
+					groups = "gpio69_group";
+					function = "gpio69_func";
+				};
+			};
+            pinctrl_die0_gpio70_default: gpio70-default{
+				mux {
+					groups = "gpio70_group";
+					function = "gpio70_func";
+				};
+			};
+            pinctrl_die0_gpio71_default: gpio71-default{
+				mux {
+					groups = "gpio71_group";
+					function = "gpio71_func";
+				};
+			};
+            pinctrl_die0_gpio72_default: gpio72-default{
+				mux {
+					groups = "gpio72_group";
+					function = "gpio72_func";
+				};
+			};
+            pinctrl_die0_gpio73_default: gpio73-default{
+				mux {
+					groups = "gpio73_group";
+					function = "gpio73_func";
+				};
+			};
+            pinctrl_die0_gpio74_default: gpio74-default{
+				mux {
+					groups = "gpio74_group";
+					function = "gpio74_func";
+				};
+			};
+            pinctrl_die0_gpio75_default: gpio75-default{
+				mux {
+					groups = "gpio75_group";
+					function = "gpio75_func";
+				};
+			};
+            pinctrl_die0_gpio76_default: gpio76-default{
+				mux {
+					groups = "gpio76_group";
+					function = "gpio76_func";
+				};
+			};
+            pinctrl_die0_gpio77_default: gpio77-default{
+				mux {
+					groups = "gpio77_group";
+					function = "gpio77_func";
+				};
+			};
+            pinctrl_die0_gpio78_default: gpio78-default{
+				mux {
+					groups = "gpio78_group";
+					function = "gpio78_func";
+				};
+			};
+            pinctrl_die0_gpio79_default: gpio79-default{
+				mux {
+					groups = "gpio79_group";
+					function = "gpio79_func";
+				};
+			};
+            pinctrl_die0_gpio80_default: gpio80-default{
+				mux {
+					groups = "gpio80_group";
+					function = "gpio80_func";
+				};
+			};
+            pinctrl_die0_gpio81_default: gpio81-default{
+				mux {
+					groups = "gpio81_group";
+					function = "gpio81_func";
+				};
+			};
+            pinctrl_die0_gpio88_default: gpio88-default{
+				mux {
+					groups = "gpio88_group";
+					function = "gpio88_func";
+				};
+			};
+            pinctrl_die0_gpio89_default: gpio89-default{
+				mux {
+					groups = "gpio89_group";
+					function = "gpio89_func";
+				};
+			};
+            pinctrl_die0_gpio90_default: gpio90-default{
+				mux {
+					groups = "gpio90_group";
+					function = "gpio90_func";
+				};
+			};
+            pinctrl_die0_gpio91_default: gpio91-default{
+				mux {
+					groups = "gpio91_group";
+					function = "gpio91_func";
+				};
+			};
+            pinctrl_die0_gpio92_default: gpio92-default{
+				mux {
+					groups = "gpio92_group";
+					function = "gpio92_func";
+				};
+				conf {
+					groups = "gpio92_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio93_default: gpio93-default{
+				mux {
+					groups = "gpio93_group";
+					function = "gpio93_func";
+				};
+				conf {
+					groups = "gpio93_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio94_default: gpio94-default{
+				mux {
+					groups = "gpio94_group";
+					function = "gpio94_func";
+				};
+				conf {
+					groups = "gpio94_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio95_default: gpio95-default{
+				mux {
+					groups = "gpio95_group";
+					function = "gpio95_func";
+				};
+			};
+            pinctrl_die0_gpio96_default: gpio96-default{
+				mux {
+					groups = "gpio96_group";
+					function = "gpio96_func";
+				};
+			};
+            pinctrl_die0_gpio97_default: gpio97-default{
+				mux {
+					groups = "gpio97_group";
+					function = "gpio97_func";
+				};
+			};
+            pinctrl_die0_gpio98_default: gpio98-default{
+				mux {
+					groups = "gpio98_group";
+					function = "gpio98_func";
+				};
+			};
+            pinctrl_die0_gpio99_default: gpio99-default{
+				mux {
+					groups = "gpio99_group";
+					function = "gpio99_func";
+				};
+			};
+            pinctrl_die0_gpio100_default: gpio100-default{
+				mux {
+					groups = "gpio100_group";
+					function = "gpio100_func";
+				};
+			};
+            pinctrl_die0_gpio101_default: gpio101-default{
+				mux {
+					groups = "gpio101_group";
+					function = "gpio101_func";
+				};
+			};
+            pinctrl_die0_gpio102_default: gpio102-default{
+				mux {
+					groups = "gpio102_group";
+					function = "gpio102_func";
+				};
+			};
+            pinctrl_die0_gpio103_default: gpio103-default{
+				mux {
+					groups = "gpio103_group";
+					function = "gpio103_func";
+				};
+			};
+            pinctrl_die0_gpio104_default: gpio104-default{
+				mux {
+					groups = "gpio104_group";
+					function = "gpio104_func";
+				};
+			};
+            pinctrl_die0_gpio105_default: gpio105-default{
+				mux {
+					groups = "gpio105_group";
+					function = "gpio105_func";
+				};
+			};
+            pinctrl_die0_gpio106_default: gpio106-default{
+				mux {
+					groups = "gpio106_group";
+					function = "gpio106_func";
+				};
+				conf {
+					groups = "gpio106_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio107_default: gpio107-default{
+				mux {
+					groups = "gpio107_group";
+					function = "gpio107_func";
+				};
+				conf {
+					groups = "gpio107_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio108_default: gpio108-default{
+				mux {
+					groups = "gpio108_group";
+					function = "gpio108_func";
+				};
+				conf {
+					groups = "gpio108_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio109_default: gpio109-default{
+				mux {
+					groups = "gpio109_group";
+					function = "gpio109_func";
+				};
+				conf {
+					groups = "gpio109_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio110_default: gpio110-default{
+				mux {
+					groups = "gpio110_group";
+					function = "gpio110_func";
+				};
+				conf {
+					groups = "gpio110_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die0_gpio111_default: gpio111-default{
+				mux {
+					groups = "gpio111_group";
+					function = "gpio111_func";
+				};
+				conf {
+					groups = "gpio111_group";
+					input-enable = <0>;
+					bias-pull-up = <1>;
+				};
+			};
+
+            //func6
+            pinctrl_die0_csi_mon_out_default: csi_mon_out-default{
+				mux {
+					groups = "csi_mon_out_group";
+					function = "csi_mon_out_func";
+				};
+			};
+            pinctrl_die0_csi_ocla_clk_default: csi_ocla_clk-default{
+				mux {
+					groups = "csi_ocla_clk_group";
+					function = "csi_ocla_clk_func";
+				};
+			};
+            pinctrl_die0_csi_mon_out_valid_default: csi_mon_out_valid-default{
+				mux {
+					groups = "csi_mon_out_valid_group";
+					function = "csi_mon_out_valid_func";
+				};
+			};
+            pinctrl_die0_csi_parity_error_default: csi_parity_error-default{
+				mux {
+					groups = "csi_parity_error_group";
+					function = "csi_parity_error_func";
+				};
+			};
+            pinctrl_die0_csi_dtb_out_default: csi_dtb_out-default{
+				mux {
+					groups = "csi_dtb_out_group";
+					function = "csi_dtb_out_func";
+				};
+			};
+            pinctrl_die0_csi_phy_sel_default: csi_phy_sel-default{
+				mux {
+					groups = "csi_phy_sel_group";
+					function = "csi_phy_sel_func";
+				};
+			};
+            pinctrl_die0_vc_g2d0_debug_out_default: vc_g2d0_debug_out-default{
+				mux {
+					groups = "vc_g2d0_debug_out_group";
+					function = "vc_g2d0_debug_out_func";
+				};
+			};
+            pinctrl_die0_vc_g2d1_debug_out_default: vc_g2d1_debug_out-default{
+				mux {
+					groups = "vc_g2d1_debug_out_group";
+					function = "vc_g2d1_debug_out_func";
+				};
+			};
+            pinctrl_die0_sata_mpll_clk_default: sata_mpll_clk-default{
+				mux {
+					groups = "sata_mpll_clk_group";
+					function = "sata_mpll_clk_func";
+				};
+			};
+            pinctrl_die0_sata_ref_repeat_clk_m_default: sata_ref_repeat_clk_m-default{
+				mux {
+					groups = "sata_ref_repeat_clk_m_group";
+					function = "sata_ref_repeat_clk_m_func";
+				};
+			};
+            pinctrl_die0_sata_ref_repeat_clk_p_default: sata_ref_repeat_clk_p-default{
+				mux {
+					groups = "sata_ref_repeat_clk_p_group";
+					function = "sata_ref_repeat_clk_p_func";
+				};
+			};
+
+			//func7
+			pinctrl_die0_clk_d2d_test_out_default: clk_d2d_test_out-default{
+				mux {
+					groups = "clk_d2d_test_out_group";
+					function = "clk_d2d_test_out_func";
+				};
+			};
+			pinctrl_die0_clk_spll0_test_out_default: clk_spll0_test_out-default{
+				mux {
+					groups = "clk_spll0_test_out_group";
+					function = "clk_spll0_test_out_func";
+				};
+			};
+		};
+    };
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7x-die1-pinctrl.dtsi b/arch/riscv/boot/dts/eswin/eic7x-die1-pinctrl.dtsi
new file mode 100644
index 000000000000..88b7a0e850b0
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7x-die1-pinctrl.dtsi
@@ -0,0 +1,1457 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for pin control of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "eswin-win2030-die1-soc.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+    #address-cells = <2>;
+	#size-cells = <2>;
+    soc {
+        d1_pinctrl: pinctrl@0x71600080 {
+			compatible = "eswin,eic7x-pinctrl", "syscon";
+			reg = <0x0 0x71600080 0x0 0x1FFF80>;
+			status = "disabled";
+
+			//Dual Die Configuration
+			//func0
+			/* used with pinctrl_die0_jtag1_off_default*/
+			pinctrl_die1_jtag1_on_default: jtag1_on-default{
+				mux {
+					groups = "jtag1_on_group";
+					function = "jtag1_on_func";
+				};
+			};
+			/* used with pinctrl_die0_jtag1_on_default*/
+			pinctrl_die1_jtag1_off_default: jtag1_off-default{
+				mux {
+					groups = "jtag1_off_group";
+					function = "jtag1_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_jtag2_off_default*/
+			pinctrl_die1_jtag2_on_default: jtag2_on-default{
+				mux {
+					groups = "jtag2_on_group";
+					function = "jtag2_on_func";
+				};
+			};
+			/* used with pinctrl_die0_jtag2_on_default*/
+			pinctrl_die1_jtag2_off_default: jtag2_off-default{
+				mux {
+					groups = "jtag2_off_group";
+					function = "jtag2_off_func";
+				};
+			};
+
+			//func2
+			/* used with pinctrl_die0_gpio7_off_default*/
+			pinctrl_die1_gpio7_on_default: gpio7_on-default{
+				mux {
+					groups = "gpio7_on_group";
+					function = "gpio7_on_func";
+				};
+				conf {
+					groups = "gpio7_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio7_on_default*/
+			pinctrl_die1_gpio7_off_default: gpio7_off-default{
+				mux {
+					groups = "gpio7_off_group";
+					function = "gpio7_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_gpio8_off_default*/
+			pinctrl_die1_gpio8_on_default: gpio8_on-default{
+				mux {
+					groups = "gpio8_on_group";
+					function = "gpio8_on_func";
+				};
+				conf {
+					groups = "gpio8_on_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio8_on_default*/
+			pinctrl_die1_gpio8_off_default: gpio8_off-default{
+				mux {
+					groups = "gpio8_off_group";
+					function = "gpio8_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_gpio9_off_default*/
+			pinctrl_die1_gpio9_on_default: gpio9_on-default{
+				mux {
+					groups = "gpio9_on_group";
+					function = "gpio9_on_func";
+				};
+				conf {
+					groups = "gpio9_on_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio9_on_default*/
+			pinctrl_die1_gpio9_off_default: gpio9_off-default{
+				mux {
+					groups = "gpio9_off_group";
+					function = "gpio9_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_gpio17_off_default*/
+			pinctrl_die1_gpio17_on_default: gpio17_on-default{
+				mux {
+					groups = "gpio17_on_group";
+					function = "gpio17_on_func";
+				};
+				conf {
+					groups = "gpio17_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio17_on_default*/
+			pinctrl_die1_gpio17_off_default: gpio17_off-default{
+				mux {
+					groups = "gpio17_off_group";
+					function = "gpio17_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_gpio64_off_default*/
+			pinctrl_die1_gpio64_on_default: gpio64_on-default{
+				mux {
+					groups = "gpio64_on_group";
+					function = "gpio64_on_func";
+				};
+				conf {
+					groups = "gpio64_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio64_on_default*/
+			pinctrl_die1_gpio64_off_default: gpio64_off-default{
+				mux {
+					groups = "gpio64_off_group";
+					function = "gpio64_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_gpio65_off_default*/
+			pinctrl_die1_gpio65_on_default: gpio65_on-default{
+				mux {
+					groups = "gpio65_on_group";
+					function = "gpio65_on_func";
+				};
+				conf {
+					groups = "gpio65_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio65_on_default*/
+			pinctrl_die1_gpio65_off_default: gpio65_off-default{
+				mux {
+					groups = "gpio65_off_group";
+					function = "gpio65_off_func";
+				};
+			};
+
+			/* used with pinctrl_die0_gpio66_off_default*/
+			pinctrl_die1_gpio66_on_default: gpio66_on-default{
+				mux {
+					groups = "gpio66_on_group";
+					function = "gpio66_on_func";
+				};
+				conf {
+					groups = "gpio66_on_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+			/* used with pinctrl_die0_gpio66_on_default*/
+			pinctrl_die1_gpio66_off_default: gpio66_off-default{
+				mux {
+					groups = "gpio66_off_group";
+					function = "gpio66_off_func";
+				};
+			};
+
+			//func6
+			/* used with pinctrl_die0_vc_g2d0_debug_out_off_default*/
+			pinctrl_die1_vc_g2d0_debug_out_on_default: vc_g2d0_debug_out_on-default{
+				mux {
+					groups = "vc_g2d0_debug_out_on_group";
+					function = "vc_g2d0_debug_out_on_func";
+				};
+			};
+			/* used with pinctrl_die0_vc_g2d0_debug_out_on_default*/
+			pinctrl_die1_vc_g2d0_debug_out_off_default: vc_g2d0_debug_out_off-default{
+				mux {
+					groups = "vc_g2d0_debug_out_off_group";
+					function = "vc_g2d0_debug_out_off_func";
+				};
+			};
+
+			//func7
+			/* used with pinctrl_die0_ftm_test_out_off_default*/
+			pinctrl_die1_ftm_test_out_on_default: ftm_test_out_on-default{
+				mux {
+					groups = "ftm_test_out_on_group";
+					function = "ftm_test_out_on_func";
+				};
+			};
+			/* used with pinctrl_die0_ftm_test_out_on_default*/
+			pinctrl_die1_ftm_test_out_off_default: ftm_test_out_off-default{
+				mux {
+					groups = "ftm_test_out_off_group";
+					function = "ftm_test_out_off_func";
+				};
+			};
+
+            //func0
+            pinctrl_die1_sdio0_default: sdio0-default{
+				mux {
+					groups = "sdio0_group";
+					function = "sdio0_func";
+				};
+			};
+            pinctrl_die1_sdio1_default: sdio1-default{
+				mux {
+					groups = "sdio1_group";
+					function = "sdio1_func";
+				};
+			};
+            pinctrl_die1_jtag0_default: jtag0-default{
+				mux {
+					groups = "jtag0_group";
+					function = "jtag0_func";
+				};
+			};
+            pinctrl_die1_pcie_default: pcie-default{
+				mux{
+					groups = "pcie_group";
+					function = "pcie_func";
+				};
+			};
+            pinctrl_die1_hdmi_default: hdmi-default{
+				mux{
+					groups = "hdmi_group";
+					function = "hdmi_func";
+				};
+			};
+            pinctrl_die1_rgmii0_default: rgmii0-default{
+				mux {
+					groups = "rgmii0_group";
+					function = "rgmii0_func";
+				};
+			};
+            pinctrl_die1_rgmii1_default: rgmii1-default{
+				mux{
+					groups = "rgmii1_group";
+					function = "rgmii1_func";
+				};
+			};
+            pinctrl_die1_spi0_default: spi0-default{
+				mux {
+					groups = "spi0_group";
+					function = "spi0_func";
+				};
+			};
+            pinctrl_die1_spi1_default: spi1-default{
+				mux{
+					groups = "spi1_group";
+					function = "spi1_func";
+				};
+			};
+            pinctrl_die1_spi3_default: spi3-default{
+				mux {
+					groups = "spi3_group";
+					function = "spi3_func";
+				};
+			};
+            pinctrl_die1_por_time_sel0_default: por_time_sel0-default{
+				mux {
+					groups = "por_time_sel0_group";
+					function = "por_time_sel0_func";
+				};
+			};
+            pinctrl_die1_por_time_sel1_default: por_time_sel1-default{
+				mux {
+					groups = "por_time_sel1_group";
+					function = "por_time_sel1_func";
+				};
+			};
+            pinctrl_die1_i2s0_default: i2s0-default{
+				mux {
+					groups = "i2s0_group";
+					function = "i2s0_func";
+				};
+			};
+            pinctrl_die1_i2s1_default: i2s1-default{
+				mux {
+					groups = "i2s1_group";
+					function = "i2s1_func";
+				};
+			};
+            pinctrl_die1_i2s2_default: i2s2-default{
+				mux {
+					groups = "i2s2_group";
+					function = "i2s2_func";
+				};
+			};
+            pinctrl_die1_usb0_pwren_default: usb0_pwren-default{
+				mux {
+					groups = "usb0_pwren_group";
+					function = "usb0_pwren_func";
+				};
+			};
+            pinctrl_die1_usb1_pwren_default: usb1_pwren-default{
+				mux {
+					groups = "usb1_pwren_group";
+					function = "usb1_pwren_func";
+				};
+			};
+            pinctrl_die1_i2c0_default: i2c0-default{
+				mux {
+					groups = "i2c0_group";
+					function = "i2c0_func";
+				};
+			};
+            pinctrl_die1_i2c1_default: i2c1-default{
+				mux {
+					groups = "i2c1_group";
+					function = "i2c1_func";
+				};
+			};
+            pinctrl_die1_i2c2_default: i2c2-default{
+				mux {
+					groups = "i2c2_group";
+					function = "i2c2_func";
+				};
+			};
+            pinctrl_die1_i2c3_default: i2c3-default{
+				mux {
+					groups = "i2c3_group";
+					function = "i2c3_func";
+				};
+			};
+            pinctrl_die1_i2c4_default: i2c4-default{
+				mux {
+					groups = "i2c4_group";
+					function = "i2c4_func";
+				};
+			};
+            pinctrl_die1_i2c5_default: i2c5-default{
+				mux {
+					groups = "i2c5_group";
+					function = "i2c5_func";
+				};
+			};
+            pinctrl_die1_i2c8_default: i2c8-default{
+				mux {
+					groups = "i2c8_group";
+					function = "i2c8_func";
+				};
+			};
+            pinctrl_die1_i2c10_default: i2c10-default{
+				mux {
+					groups = "i2c10_group";
+					function = "i2c10_func";
+				};
+			};
+            pinctrl_die1_i2c11_default: i2c11-default{
+				mux {
+					groups = "i2c11_group";
+					function = "i2c11_func";
+				};
+			};
+            pinctrl_die1_uart0_default: uart0-default{
+				mux {
+					groups = "uart0_group";
+					function = "uart0_func";
+				};
+			};
+            pinctrl_die1_uart1_default: uart1-default{
+				mux {
+					groups = "uart1_group";
+					function = "uart1_func";
+				};
+			};
+            pinctrl_die1_uart2_default: uart2-default{
+				mux {
+					groups = "uart2_group";
+					function = "uart2_func";
+				};
+			};
+			pinctrl_die1_pwm0_default: pwm0-default{
+				mux {
+					groups = "pwm0_group";
+					function = "pwm0_func";
+				};
+			};
+            pinctrl_die1_fan_tach_default: fan_tach-default{
+				mux {
+					groups = "fan_tach_group";
+					function = "fan_tach_func";
+				};
+			};
+            pinctrl_die1_mipi_csi0_default: mipi_csi0-default{
+				mux {
+					groups = "mipi_csi0_group";
+					function = "mipi_csi0_func";
+				};
+			};
+            pinctrl_die1_mipi_csi1_default: mipi_csi1-default{
+				mux {
+					groups = "mipi_csi1_group";
+					function = "mipi_csi1_func";
+				};
+			};
+            pinctrl_die1_mipi_csi2_default: mipi_csi2-default{
+				mux {
+					groups = "mipi_csi2_group";
+					function = "mipi_csi2_func";
+				};
+			};
+            pinctrl_die1_mipi_csi3_default: mipi_csi3-default{
+				mux {
+					groups = "mipi_csi3_group";
+					function = "mipi_csi3_func";
+				};
+			};
+            pinctrl_die1_mipi_csi4_default: mipi_csi4-default{
+				mux {
+					groups = "mipi_csi4_group";
+					function = "mipi_csi4_func";
+				};
+			};
+            pinctrl_die1_mipi_csi5_default: mipi_csi5-default{
+				mux {
+					groups = "mipi_csi5_group";
+					function = "mipi_csi5_func";
+				};
+			};
+            pinctrl_die1_s_mode_default: s_mode-default{
+				mux {
+					groups = "s_mode_group";
+					function = "s_mode_func";
+				};
+			};
+            pinctrl_die1_ddr_refclk_sel_default: ddr_refclk_sel-default{
+				mux {
+					groups = "ddr_refclk_sel_group";
+					function = "ddr_refclk_sel_func";
+				};
+			};
+            pinctrl_die1_boot_sel_default: boot_sel-default{
+				mux {
+					groups = "boot_sel_group";
+					function = "boot_sel_func";
+				};
+			};
+            pinctrl_die1_lpddr_ref_clk_default: lpddr_ref_clk-default{
+				mux {
+					groups = "lpddr_ref_clk_group";
+					function = "lpddr_ref_clk_func";
+				};
+			};
+
+            //func1
+            pinctrl_die1_spi2_default: spi2-default{
+				mux1 {
+					groups = "spi2_clk_group";
+					function = "spi2_clk_func";
+				};
+				conf1 {
+					groups = "spi2_clk_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux2 {
+					groups = "spi2_d0_group";
+					function = "spi2_d0_func";
+				};
+				conf2 {
+					groups = "spi2_d0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux3 {
+					groups = "spi2_d1_d2_d3_group";
+					function = "spi2_d1_d2_d3_func";
+				};
+				conf3 {
+					groups = "spi2_d1_d2_d3_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux4 {
+					groups = "spi2_cs_group";
+					function = "spi2_cs_func";
+				};
+			};
+
+            pinctrl_die1_sata_act_led_default: sata_act_led-default{
+				mux {
+					groups = "sata_act_led_group";
+					function = "sata_act_led_func";
+				};
+				conf {
+					groups = "sata_act_led_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_emmc_led_control_default: emmc_led_control-default{
+				mux {
+					groups = "emmc_led_control_group";
+					function = "emmc_led_control_func";
+				};
+				conf {
+					groups = "emmc_led_control_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_sd0_led_control_default: sd0_led_control-default{
+				mux {
+					groups = "sd0_led_control_group";
+					function = "sd0_led_control_func";
+				};
+			};
+            pinctrl_die1_sd1_led_control_default: sd1_led_control-default{
+				mux {
+					groups = "sd1_led_control_group";
+					function = "sd1_led_control_func";
+				};
+			};
+            pinctrl_die1_i2c6_default: i2c6-default{
+				mux {
+					groups = "i2c6_group";
+					function = "i2c6_func";
+				};
+			};
+            pinctrl_die1_i2c7_default: i2c7-default{
+				mux {
+					groups = "i2c7_group";
+					function = "i2c7_func";
+				};
+			};
+            pinctrl_die1_i2c9_default: i2c9-default{
+				mux {
+					groups = "i2c9_group";
+					function = "i2c9_func";
+				};
+			};
+            pinctrl_die1_pwm1_default: pwm1-default{
+				mux {
+					groups = "pwm1_group";
+					function = "pwm1_func";
+				};
+			};
+            pinctrl_die1_pwm2_default: pwm2-default{
+				mux {
+					groups = "pwm2_group";
+					function = "pwm2_func";
+				};
+			};
+            pinctrl_die1_mipi_csi_xtrig_default: mipi_csi_xtrig-default{
+				mux {
+					groups = "mipi_csi_xtrig_group";
+					function = "mipi_csi_xtrig_func";
+				};
+			};
+
+            //func3
+            pinctrl_die1_uart3_default: uart3-default{
+				mux {
+					groups = "uart3_group";
+					function = "uart3_func";
+				};
+			};
+            pinctrl_die1_uart4_default: uart4-default{
+				mux {
+					groups = "uart4_group";
+					function = "uart4_func";
+				};
+			};
+
+            //gpio
+            pinctrl_die1_gpio0_default: gpio0-default{
+				mux {
+					groups = "gpio0_group";
+					function = "gpio0_func";
+				};
+				conf {
+					groups = "gpio0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die1_gpio1_default: gpio1-default{
+				mux {
+					groups = "gpio1_group";
+					function = "gpio1_func";
+				};
+			};
+            pinctrl_die1_gpio2_default: gpio2-default{
+				mux {
+					groups = "gpio2_group";
+					function = "gpio2_func";
+				};
+			};
+            pinctrl_die1_gpio3_default: gpio3-default{
+				mux {
+					groups = "gpio3_group";
+					function = "gpio3_func";
+				};
+			};
+            pinctrl_die1_gpio4_default: gpio4-default{
+				mux {
+					groups = "gpio4_group";
+					function = "gpio4_func";
+				};
+			};
+            pinctrl_die1_gpio5_default: gpio5-default{
+				mux {
+					groups = "gpio5_group";
+					function = "gpio5_func";
+				};
+				conf {
+					groups = "gpio5_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die1_gpio6_default: gpio6-default{
+				mux {
+					groups = "gpio6_group";
+					function = "gpio6_func";
+				};
+				conf {
+					groups = "gpio6_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio10_default: gpio10-default{
+				mux {
+					groups = "gpio10_group";
+					function = "gpio10_func";
+				};
+				conf {
+					groups = "gpio10_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die1_gpio11_default: gpio11-default{
+				mux {
+					groups = "gpio11_group";
+					function = "gpio11_func";
+				};
+				conf {
+					groups = "gpio11_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die1_gpio12_default: gpio12-default{
+				mux {
+					groups = "gpio12_group";
+					function = "gpio12_func";
+				};
+			};
+            pinctrl_die1_gpio13_default: gpio13-default{
+				mux {
+					groups = "gpio13_group";
+					function = "gpio13_func";
+				};
+			};
+            pinctrl_die1_gpio14_default: gpio14-default{
+				mux {
+					groups = "gpio14_group";
+					function = "gpio14_func";
+				};
+			};
+            pinctrl_die1_gpio15_default: gpio15-default{
+				mux {
+					groups = "gpio15_group";
+					function = "gpio15_func";
+				};
+			};
+            pinctrl_die1_gpio16_default: gpio16-default{
+				mux {
+					groups = "gpio16_group";
+					function = "gpio16_func";
+				};
+				conf {
+					groups = "gpio16_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio18_default: gpio18-default{
+				mux {
+					groups = "gpio18_group";
+					function = "gpio18_func";
+				};
+				conf {
+					groups = "gpio18_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio19_default: gpio19-default{
+				mux {
+					groups = "gpio19_group";
+					function = "gpio19_func";
+				};
+				conf {
+					groups = "gpio19_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio20_default: gpio20-default{
+				mux {
+					groups = "gpio20_group";
+					function = "gpio20_func";
+				};
+				conf {
+					groups = "gpio20_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio21_default: gpio21-default{
+				mux {
+					groups = "gpio21_group";
+					function = "gpio21_func";
+				};
+				conf {
+					groups = "gpio21_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio22_default: gpio22-default{
+				mux {
+					groups = "gpio22_group";
+					function = "gpio22_func";
+				};
+			};
+            pinctrl_die1_gpio23_default: gpio23-default{
+				mux {
+					groups = "gpio23_group";
+					function = "gpio23_func";
+				};
+			};
+            pinctrl_die1_gpio24_default: gpio24-default{
+				mux {
+					groups = "gpio24_group";
+					function = "gpio24_func";
+				};
+			};
+            pinctrl_die1_gpio25_default: gpio25-default{
+				mux {
+					groups = "gpio25_group";
+					function = "gpio25_func";
+				};
+			};
+            pinctrl_die1_gpio26_default: gpio26-default{
+				mux {
+					groups = "gpio26_group";
+					function = "gpio26_func";
+				};
+			};
+            pinctrl_die1_gpio27_default: gpio27-default{
+				mux {
+					groups = "gpio27_group";
+					function = "gpio27_func";
+				};
+			};
+            pinctrl_die1_gpio28_default: gpio28-default{
+				mux {
+					groups = "gpio28_group";
+					function = "gpio28_func";
+				};
+				conf {
+					groups = "gpio28_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_die1_gpio29_default: gpio29-default{
+				mux {
+					groups = "gpio29_group";
+					function = "gpio29_func";
+				};
+			};
+            pinctrl_die1_gpio30_default: gpio30-default{
+				mux {
+					groups = "gpio30_group";
+					function = "gpio30_func";
+				};
+				conf {
+					groups = "gpio30_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio31_default: gpio31-default{
+				mux {
+					groups = "gpio31_group";
+					function = "gpio31_func";
+				};
+			};
+            pinctrl_die1_gpio32_default: gpio32-default{
+				mux {
+					groups = "gpio32_group";
+					function = "gpio32_func";
+				};
+			};
+            pinctrl_die1_gpio33_default: gpio33-default{
+				mux {
+					groups = "gpio33_group";
+					function = "gpio33_func";
+				};
+			};
+            pinctrl_die1_gpio34_default: gpio34-default{
+				mux {
+					groups = "gpio34_group";
+					function = "gpio34_func";
+				};
+			};
+            pinctrl_die1_gpio35_default: gpio35-default{
+				mux {
+					groups = "gpio35_group";
+					function = "gpio35_func";
+				};
+				conf {
+					groups = "gpio35_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio36_default: gpio36-default{
+				mux {
+					groups = "gpio36_group";
+					function = "gpio36_func";
+				};
+				conf {
+					groups = "gpio36_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio37_default: gpio37-default{
+				mux {
+					groups = "gpio37_group";
+					function = "gpio37_func";
+				};
+				conf {
+					groups = "gpio37_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio38_default: gpio38-default{
+				mux {
+					groups = "gpio38_group";
+					function = "gpio38_func";
+				};
+				conf {
+					groups = "gpio38_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio39_default: gpio39-default{
+				mux {
+					groups = "gpio39_group";
+					function = "gpio39_func";
+				};
+				conf {
+					groups = "gpio39_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio40_default: gpio40-default{
+				mux {
+					groups = "gpio40_group";
+					function = "gpio40_func";
+				};
+				conf {
+					groups = "gpio40_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio41_default: gpio41-default{
+				mux {
+					groups = "gpio41_group";
+					function = "gpio41_func";
+				};
+			};
+            pinctrl_die1_gpio42_default: gpio42-default{
+				mux {
+					groups = "gpio42_group";
+					function = "gpio42_func";
+				};
+				conf {
+					groups = "gpio42_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio43_default: gpio43-default{
+				mux {
+					groups = "gpio43_group";
+					function = "gpio43_func";
+				};
+			};
+            pinctrl_die1_gpio44_default: gpio44-default{
+				mux {
+					groups = "gpio44_group";
+					function = "gpio44_func";
+				};
+			};
+            pinctrl_die1_gpio45_default: gpio45-default{
+				mux {
+					groups = "gpio45_group";
+					function = "gpio45_func";
+				};
+			};
+            pinctrl_die1_gpio46_default: gpio46-default{
+				mux {
+					groups = "gpio46_group";
+					function = "gpio46_func";
+				};
+				conf {
+					groups = "gpio46_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio47_default: gpio47-default{
+				mux {
+					groups = "gpio47_group";
+					function = "gpio47_func";
+				};
+				conf {
+					groups = "gpio47_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio48_default: gpio48-default{
+				mux {
+					groups = "gpio48_group";
+					function = "gpio48_func";
+				};
+			};
+            pinctrl_die1_gpio49_default: gpio49-default{
+				mux {
+					groups = "gpio49_group";
+					function = "gpio49_func";
+				};
+			};
+            pinctrl_die1_gpio50_default: gpio50-default{
+				mux {
+					groups = "gpio50_group";
+					function = "gpio50_func";
+				};
+			};
+            pinctrl_die1_gpio51_default: gpio51-default{
+				mux {
+					groups = "gpio51_group";
+					function = "gpio51_func";
+				};
+			};
+            pinctrl_die1_gpio52_default: gpio52-default{
+				mux {
+					groups = "gpio52_group";
+					function = "gpio52_func";
+				};
+				conf {
+					groups = "gpio52_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio53_default: gpio53-default{
+				mux {
+					groups = "gpio53_group";
+					function = "gpio53_func";
+				};
+				conf {
+					groups = "gpio53_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio54_default: gpio54-default{
+				mux {
+					groups = "gpio54_group";
+					function = "gpio54_func";
+				};
+			};
+            pinctrl_die1_gpio55_default: gpio55-default{
+				mux {
+					groups = "gpio55_group";
+					function = "gpio55_func";
+				};
+			};
+            pinctrl_die1_gpio56_default: gpio56-default{
+				mux {
+					groups = "gpio56_group";
+					function = "gpio56_func";
+				};
+			};
+            pinctrl_die1_gpio57_default: gpio57-default{
+				mux {
+					groups = "gpio57_group";
+					function = "gpio57_func";
+				};
+			};
+            pinctrl_die1_gpio58_default: gpio58-default{
+				mux {
+					groups = "gpio58_group";
+					function = "gpio58_func";
+				};
+			};
+            pinctrl_die1_gpio59_default: gpio59-default{
+				mux {
+					groups = "gpio59_group";
+					function = "gpio59_func";
+				};
+			};
+            pinctrl_die1_gpio60_default: gpio60-default{
+				mux {
+					groups = "gpio60_group";
+					function = "gpio60_func";
+				};
+			};
+            pinctrl_die1_gpio61_default: gpio61-default{
+				mux {
+					groups = "gpio61_group";
+					function = "gpio61_func";
+				};
+			};
+            pinctrl_die1_gpio62_default: gpio62-default{
+				mux {
+					groups = "gpio62_group";
+					function = "gpio62_func";
+				};
+			};
+            pinctrl_die1_gpio63_default: gpio63-default{
+				mux {
+					groups = "gpio63_group";
+					function = "gpio63_func";
+				};
+			};
+            pinctrl_die1_gpio67_default: gpio67-default{
+				mux {
+					groups = "gpio67_group";
+					function = "gpio67_func";
+				};
+				conf {
+					groups = "gpio67_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio68_default: gpio68-default{
+				mux {
+					groups = "gpio68_group";
+					function = "gpio68_func";
+				};
+				conf {
+					groups = "gpio68_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio69_default: gpio69-default{
+				mux {
+					groups = "gpio69_group";
+					function = "gpio69_func";
+				};
+			};
+            pinctrl_die1_gpio70_default: gpio70-default{
+				mux {
+					groups = "gpio70_group";
+					function = "gpio70_func";
+				};
+			};
+            pinctrl_die1_gpio71_default: gpio71-default{
+				mux {
+					groups = "gpio71_group";
+					function = "gpio71_func";
+				};
+			};
+            pinctrl_die1_gpio72_default: gpio72-default{
+				mux {
+					groups = "gpio72_group";
+					function = "gpio72_func";
+				};
+			};
+            pinctrl_die1_gpio73_default: gpio73-default{
+				mux {
+					groups = "gpio73_group";
+					function = "gpio73_func";
+				};
+			};
+            pinctrl_die1_gpio74_default: gpio74-default{
+				mux {
+					groups = "gpio74_group";
+					function = "gpio74_func";
+				};
+			};
+            pinctrl_die1_gpio75_default: gpio75-default{
+				mux {
+					groups = "gpio75_group";
+					function = "gpio75_func";
+				};
+			};
+            pinctrl_die1_gpio76_default: gpio76-default{
+				mux {
+					groups = "gpio76_group";
+					function = "gpio76_func";
+				};
+			};
+            pinctrl_die1_gpio77_default: gpio77-default{
+				mux {
+					groups = "gpio77_group";
+					function = "gpio77_func";
+				};
+			};
+            pinctrl_die1_gpio78_default: gpio78-default{
+				mux {
+					groups = "gpio78_group";
+					function = "gpio78_func";
+				};
+			};
+            pinctrl_die1_gpio79_default: gpio79-default{
+				mux {
+					groups = "gpio79_group";
+					function = "gpio79_func";
+				};
+			};
+            pinctrl_die1_gpio80_default: gpio80-default{
+				mux {
+					groups = "gpio80_group";
+					function = "gpio80_func";
+				};
+			};
+            pinctrl_die1_gpio81_default: gpio81-default{
+				mux {
+					groups = "gpio81_group";
+					function = "gpio81_func";
+				};
+			};
+            pinctrl_die1_gpio88_default: gpio88-default{
+				mux {
+					groups = "gpio88_group";
+					function = "gpio88_func";
+				};
+			};
+            pinctrl_die1_gpio89_default: gpio89-default{
+				mux {
+					groups = "gpio89_group";
+					function = "gpio89_func";
+				};
+			};
+            pinctrl_die1_gpio90_default: gpio90-default{
+				mux {
+					groups = "gpio90_group";
+					function = "gpio90_func";
+				};
+			};
+            pinctrl_die1_gpio91_default: gpio91-default{
+				mux {
+					groups = "gpio91_group";
+					function = "gpio91_func";
+				};
+			};
+            pinctrl_die1_gpio92_default: gpio92-default{
+				mux {
+					groups = "gpio92_group";
+					function = "gpio92_func";
+				};
+				conf {
+					groups = "gpio92_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio93_default: gpio93-default{
+				mux {
+					groups = "gpio93_group";
+					function = "gpio93_func";
+				};
+				conf {
+					groups = "gpio93_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio94_default: gpio94-default{
+				mux {
+					groups = "gpio94_group";
+					function = "gpio94_func";
+				};
+				conf {
+					groups = "gpio94_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio95_default: gpio95-default{
+				mux {
+					groups = "gpio95_group";
+					function = "gpio95_func";
+				};
+			};
+            pinctrl_die1_gpio96_default: gpio96-default{
+				mux {
+					groups = "gpio96_group";
+					function = "gpio96_func";
+				};
+			};
+            pinctrl_die1_gpio97_default: gpio97-default{
+				mux {
+					groups = "gpio97_group";
+					function = "gpio97_func";
+				};
+			};
+            pinctrl_die1_gpio98_default: gpio98-default{
+				mux {
+					groups = "gpio98_group";
+					function = "gpio98_func";
+				};
+			};
+            pinctrl_die1_gpio99_default: gpio99-default{
+				mux {
+					groups = "gpio99_group";
+					function = "gpio99_func";
+				};
+			};
+            pinctrl_die1_gpio100_default: gpio100-default{
+				mux {
+					groups = "gpio100_group";
+					function = "gpio100_func";
+				};
+			};
+            pinctrl_die1_gpio101_default: gpio101-default{
+				mux {
+					groups = "gpio101_group";
+					function = "gpio101_func";
+				};
+			};
+            pinctrl_die1_gpio102_default: gpio102-default{
+				mux {
+					groups = "gpio102_group";
+					function = "gpio102_func";
+				};
+			};
+            pinctrl_die1_gpio103_default: gpio103-default{
+				mux {
+					groups = "gpio103_group";
+					function = "gpio103_func";
+				};
+			};
+            pinctrl_die1_gpio104_default: gpio104-default{
+				mux {
+					groups = "gpio104_group";
+					function = "gpio104_func";
+				};
+			};
+            pinctrl_die1_gpio105_default: gpio105-default{
+				mux {
+					groups = "gpio105_group";
+					function = "gpio105_func";
+				};
+			};
+            pinctrl_die1_gpio106_default: gpio106-default{
+				mux {
+					groups = "gpio106_group";
+					function = "gpio106_func";
+				};
+				conf {
+					groups = "gpio106_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio107_default: gpio107-default{
+				mux {
+					groups = "gpio107_group";
+					function = "gpio107_func";
+				};
+				conf {
+					groups = "gpio107_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio108_default: gpio108-default{
+				mux {
+					groups = "gpio108_group";
+					function = "gpio108_func";
+				};
+				conf {
+					groups = "gpio108_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio109_default: gpio109-default{
+				mux {
+					groups = "gpio109_group";
+					function = "gpio109_func";
+				};
+				conf {
+					groups = "gpio109_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio110_default: gpio110-default{
+				mux {
+					groups = "gpio110_group";
+					function = "gpio110_func";
+				};
+				conf {
+					groups = "gpio110_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_die1_gpio111_default: gpio111-default{
+				mux {
+					groups = "gpio111_group";
+					function = "gpio111_func";
+				};
+				conf {
+					groups = "gpio111_group";
+					input-enable = <0>;
+					bias-pull-up = <1>;
+				};
+			};
+
+            //func6
+            pinctrl_die1_csi_mon_out_default: csi_mon_out-default{
+				mux {
+					groups = "csi_mon_out_group";
+					function = "csi_mon_out_func";
+				};
+			};
+            pinctrl_die1_csi_ocla_clk_default: csi_ocla_clk-default{
+				mux {
+					groups = "csi_ocla_clk_group";
+					function = "csi_ocla_clk_func";
+				};
+			};
+            pinctrl_die1_csi_mon_out_valid_default: csi_mon_out_valid-default{
+				mux {
+					groups = "csi_mon_out_valid_group";
+					function = "csi_mon_out_valid_func";
+				};
+			};
+            pinctrl_die1_csi_parity_error_default: csi_parity_error-default{
+				mux {
+					groups = "csi_parity_error_group";
+					function = "csi_parity_error_func";
+				};
+			};
+            pinctrl_die1_csi_dtb_out_default: csi_dtb_out-default{
+				mux {
+					groups = "csi_dtb_out_group";
+					function = "csi_dtb_out_func";
+				};
+			};
+            pinctrl_die1_csi_phy_sel_default: csi_phy_sel-default{
+				mux {
+					groups = "csi_phy_sel_group";
+					function = "csi_phy_sel_func";
+				};
+			};
+            pinctrl_die1_vc_g2d0_debug_out_default: vc_g2d0_debug_out-default{
+				mux {
+					groups = "vc_g2d0_debug_out_group";
+					function = "vc_g2d0_debug_out_func";
+				};
+			};
+            pinctrl_die1_vc_g2d1_debug_out_default: vc_g2d1_debug_out-default{
+				mux {
+					groups = "vc_g2d1_debug_out_group";
+					function = "vc_g2d1_debug_out_func";
+				};
+			};
+            pinctrl_die1_sata_mpll_clk_default: sata_mpll_clk-default{
+				mux {
+					groups = "sata_mpll_clk_group";
+					function = "sata_mpll_clk_func";
+				};
+			};
+            pinctrl_die1_sata_ref_repeat_clk_m_default: sata_ref_repeat_clk_m-default{
+				mux {
+					groups = "sata_ref_repeat_clk_m_group";
+					function = "sata_ref_repeat_clk_m_func";
+				};
+			};
+            pinctrl_die1_sata_ref_repeat_clk_p_default: sata_ref_repeat_clk_p-default{
+				mux {
+					groups = "sata_ref_repeat_clk_p_group";
+					function = "sata_ref_repeat_clk_p_func";
+				};
+			};
+
+			//func7
+			pinctrl_die1_clk_d2d_test_out_default: clk_d2d_test_out-default{
+				mux {
+					groups = "clk_d2d_test_out_group";
+					function = "clk_d2d_test_out_func";
+				};
+			};
+			pinctrl_die1_clk_spll0_test_out_default: clk_spll0_test_out-default{
+				mux {
+					groups = "clk_spll0_test_out_group";
+					function = "clk_spll0_test_out_func";
+				};
+			};
+		};
+    };
+};
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-arch-d2d.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-arch-d2d.dtsi
new file mode 100644
index 000000000000..df4b8dac0ec1
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-arch-d2d.dtsi
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Eswin EIC7702 SoC's cpu.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#define CHIPLET_AND_DIE (0x2)
+#include "eswin-win2030-arch.dtsi"
+#include "eic770x-opptable.dtsi"
+
+&L64 {
+	cpu-map {
+		//socket0 {
+			#if (CHIPLET_AND_DIE & 0x2)
+			cluster0 {
+				core0 {
+					cpu = <&cpu_0>;
+				};
+				core1 {
+					cpu = <&cpu_1>;
+				};
+				core2 {
+					cpu = <&cpu_2>;
+				};
+				core3 {
+					cpu = <&cpu_3>;
+				};
+			};
+			#endif
+		//};
+
+		//socket1 {
+			cluster1 {
+				core0 {
+					cpu = <&cpu_4>;
+				};
+				core1 {
+					cpu = <&cpu_5>;
+				};
+				core2 {
+					cpu = <&cpu_6>;
+				};
+				core3 {
+					cpu = <&cpu_7>;
+				};
+			};
+		//};
+	};
+
+	cpu_4: cpu@4 {
+		clock-frequency = <0>;
+		compatible = "eswin,eic770x", "riscv";
+		d-cache-block-size = <64>;
+		d-cache-sets = <128>;
+		d-cache-size = <32768>;
+		d-tlb-sets = <1>;
+		d-tlb-size = <32>;
+		device_type = "cpu";
+		hardware-exec-breakpoint-count = <4>;
+		hwpf-distanceBits = <6>;
+		hwpf-hitCacheThrdBits = <5>;
+		hwpf-hitMSHRThrdBits = <4>;
+		hwpf-l2pfPoolSize = <10>;
+		hwpf-nIssQEnt = <6>;
+		hwpf-nPrefetchQueueEntries = <8>;
+		hwpf-nStreams = <16>;
+		hwpf-qFullnessThrdBits = <4>;
+		hwpf-windowBits = <6>;
+		i-cache-block-size = <64>;
+		i-cache-sets = <128>;
+		i-cache-size = <32768>;
+		i-tlb-sets = <1>;
+		i-tlb-size = <32>;
+		mmu-type = "riscv,sv48";
+		next-level-cache = <&d1_l2_cache_0>;
+	#if (CHIPLET_AND_DIE == 1)
+	#ifdef PLATFORM_HAPS
+		reg = <0x1>;
+	#else
+		reg = <0x4>;
+	#endif
+	#else
+		reg = <0x4>;
+	#endif
+		riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+		riscv,pmpgranularity = <4096>;
+		riscv,pmpregions = <8>;
+		#if (CHIPLET_AND_DIE & 0x2)
+		sifive,buserror = <&d1_bus_err0>;
+		#endif
+		status = "okay";
+		timebase-frequency = <RTCCLK_FREQ>;
+		tlb-split;
+		numa-node-id = <1>;
+		clocks = <&d1_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_0>;
+		operating-points-v2 = <&d1_cpu_opp_table>;
+		#cooling-cells = <2>;
+		dynamic-power-coefficient = <800>;
+		cpu-idle-states = <&CPU_RET>;
+		cpu4_intc: interrupt-controller {
+			#interrupt-cells = <1>;
+			compatible = "riscv,cpu-intc";
+			interrupt-controller;
+		};
+
+	};
+#ifndef PLATFORM_HAPS
+	cpu_5: cpu@5 {
+		clock-frequency = <0>;
+		compatible = "eswin,eic770x", "riscv";
+		d-cache-block-size = <64>;
+		d-cache-sets = <128>;
+		d-cache-size = <32768>;
+		d-tlb-sets = <1>;
+		d-tlb-size = <32>;
+		device_type = "cpu";
+		hardware-exec-breakpoint-count = <4>;
+		hwpf-distanceBits = <6>;
+		hwpf-hitCacheThrdBits = <5>;
+		hwpf-hitMSHRThrdBits = <4>;
+		hwpf-l2pfPoolSize = <10>;
+		hwpf-nIssQEnt = <6>;
+		hwpf-nPrefetchQueueEntries = <8>;
+		hwpf-nStreams = <16>;
+		hwpf-qFullnessThrdBits = <4>;
+		hwpf-windowBits = <6>;
+		i-cache-block-size = <64>;
+		i-cache-sets = <128>;
+		i-cache-size = <32768>;
+		i-tlb-sets = <1>;
+		i-tlb-size = <32>;
+		mmu-type = "riscv,sv48";
+		next-level-cache = <&d1_l2_cache_1>;
+		reg = <0x5>;
+		riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+		riscv,pmpgranularity = <4096>;
+		riscv,pmpregions = <8>;
+		#if (CHIPLET_AND_DIE & 0x2)
+		sifive,buserror = <&d1_bus_err1>;
+		#endif
+		status = "okay";
+		timebase-frequency = <RTCCLK_FREQ>;
+		tlb-split;
+		numa-node-id = <1>;
+		clocks = <&d1_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_1>;
+		operating-points-v2 = <&d1_cpu_opp_table>;
+		#cooling-cells = <2>;
+		dynamic-power-coefficient = <800>;
+		cpu-idle-states = <&CPU_RET>;
+		cpu5_intc: interrupt-controller {
+			#interrupt-cells = <1>;
+			compatible = "riscv,cpu-intc";
+			interrupt-controller;
+		};
+
+	};
+	cpu_6: cpu@6 {
+		clock-frequency = <0>;
+		compatible = "eswin,eic770x", "riscv";
+		d-cache-block-size = <64>;
+		d-cache-sets = <128>;
+		d-cache-size = <32768>;
+		d-tlb-sets = <1>;
+		d-tlb-size = <32>;
+		device_type = "cpu";
+		hardware-exec-breakpoint-count = <4>;
+		hwpf-distanceBits = <6>;
+		hwpf-hitCacheThrdBits = <5>;
+		hwpf-hitMSHRThrdBits = <4>;
+		hwpf-l2pfPoolSize = <10>;
+		hwpf-nIssQEnt = <6>;
+		hwpf-nPrefetchQueueEntries = <8>;
+		hwpf-nStreams = <16>;
+		hwpf-qFullnessThrdBits = <4>;
+		hwpf-windowBits = <6>;
+		i-cache-block-size = <64>;
+		i-cache-sets = <128>;
+		i-cache-size = <32768>;
+		i-tlb-sets = <1>;
+		i-tlb-size = <32>;
+		mmu-type = "riscv,sv48";
+		next-level-cache = <&d1_l2_cache_2>;
+		reg = <0x6>;
+		riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+		riscv,pmpgranularity = <4096>;
+		riscv,pmpregions = <8>;
+		#if (CHIPLET_AND_DIE & 0x2)
+		sifive,buserror = <&d1_bus_err2>;
+		#endif
+		status = "okay";
+		timebase-frequency = <RTCCLK_FREQ>;
+		tlb-split;
+		numa-node-id = <1>;
+		clocks = <&d1_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_2>;
+		operating-points-v2 = <&d1_cpu_opp_table>;
+		#cooling-cells = <2>;
+		dynamic-power-coefficient = <800>;
+		cpu-idle-states = <&CPU_RET>;
+		cpu6_intc: interrupt-controller {
+			#interrupt-cells = <1>;
+			compatible = "riscv,cpu-intc";
+			interrupt-controller;
+		};
+
+	};
+	cpu_7: cpu@7 {
+		clock-frequency = <0>;
+		compatible = "eswin,eic770x", "riscv";
+		d-cache-block-size = <64>;
+		d-cache-sets = <128>;
+		d-cache-size = <32768>;
+		d-tlb-sets = <1>;
+		d-tlb-size = <32>;
+		device_type = "cpu";
+		hardware-exec-breakpoint-count = <4>;
+		hwpf-distanceBits = <6>;
+		hwpf-hitCacheThrdBits = <5>;
+		hwpf-hitMSHRThrdBits = <4>;
+		hwpf-l2pfPoolSize = <10>;
+		hwpf-nIssQEnt = <6>;
+		hwpf-nPrefetchQueueEntries = <8>;
+		hwpf-nStreams = <16>;
+		hwpf-qFullnessThrdBits = <4>;
+		hwpf-windowBits = <6>;
+		i-cache-block-size = <64>;
+		i-cache-sets = <128>;
+		i-cache-size = <32768>;
+		i-tlb-sets = <1>;
+		i-tlb-size = <32>;
+		mmu-type = "riscv,sv48";
+		next-level-cache = <&d1_l2_cache_3>;
+		reg = <0x7>;
+		riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+		riscv,pmpgranularity = <4096>;
+		riscv,pmpregions = <8>;
+		#if (CHIPLET_AND_DIE & 0x2)
+		sifive,buserror = <&d1_bus_err3>;
+		#endif
+		status = "okay";
+		timebase-frequency = <RTCCLK_FREQ>;
+		tlb-split;
+		numa-node-id = <1>;
+		clocks = <&d1_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_3>;
+		operating-points-v2 = <&d1_cpu_opp_table>;
+		#cooling-cells = <2>;
+		dynamic-power-coefficient = <800>;
+		cpu-idle-states = <&CPU_RET>;
+		cpu7_intc: interrupt-controller {
+			#interrupt-cells = <1>;
+			compatible = "riscv,cpu-intc";
+			interrupt-controller;
+		};
+
+	};
+#endif
+};
+
+&SOC {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "SiFive,FU800-soc", "fu800-soc", "sifive-soc", "simple-bus";
+	ranges;
+	PMU1: pmu@1 {
+		riscv,raw-event-to-mhpmcounters = <0x0 0x0  0xffffffff 0xff          0x1f8
+											0x0 0x1  0xffffffff 0xfff800ff    0x1f8
+											0x0 0x2  0xffffffff 0xffffc0ff    0x1f8>;
+		riscv,event-to-mhpmcounters = <0x01 0x01 0x01 0x02 0x02 0x02 0x4 0x6 0x1f8 0x10009 0x10009 0x1f8 0x10019 0x10019 0x1f8 0x10021 0x10021 0x1f8>;
+		riscv,event-to-mhpmevent = <0x4 0x0 0x202 0x5 0x0 0x4000 0x6 0x0 0x2001 0x10009 0x0 0x102 0x10019 0x0 0x1002 0x10021 0x0 0x802>;
+		compatible = "riscv,pmu0", "riscv,pmu";
+		interrupts-extended = <&cpu4_intc 13 &cpu5_intc 13 &cpu6_intc 13 &cpu7_intc 13>;
+		numa-node-id = <1>;
+	};
+
+	d1_bus_err0: bus-error-unit@21700000 {
+		compatible = "sifive,buserror";
+		interrupt-parent = <&plic1>;
+		interrupts = <517>;
+		reg = <0x0 0x21700000 0x0 0x1000>;
+		reg-names = "control";
+		numa-node-id = <1>;
+	};
+	d1_bus_err1: bus-error-unit@21701000 {
+		compatible = "sifive,buserror";
+		interrupt-parent = <&plic1>;
+		interrupts = <518>;
+		reg = <0x0 0x21701000 0x0 0x1000>;
+		reg-names = "control";
+		numa-node-id = <1>;
+	};
+	d1_bus_err2: bus-error-unit@21702000 {
+		compatible = "sifive,buserror";
+		interrupt-parent = <&plic1>;
+		interrupts = <519>;
+		reg = <0x0 0x21702000 0x0 0x1000>;
+		reg-names = "control";
+		numa-node-id = <1>;
+	};
+	d1_bus_err3: bus-error-unit@21703000 {
+		compatible = "sifive,buserror";
+		interrupt-parent = <&plic1>;
+		interrupts = <520>;
+		reg = <0x0 0x21703000 0x0 0x1000>;
+		reg-names = "control";
+		numa-node-id = <1>;
+	};
+
+	D1CACHE: cache-controller@22010000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		cache-block-size = <64>;
+		cache-level = <3>;
+		cache-sets = <4096>;
+		cache-size = <4194304>;
+		cache-unified;
+		compatible = "sifive,ccache1", "cache", "sifive,fu740-c000-ccache";
+		interrupt-parent = <&plic1>;
+		interrupts = <1>, <3>, <4>, <2>;
+		reg = <0x0 0x22010000 0x0 0x4000 0x0 0x8000000 0x0 0x400000>;
+		reg-names = "control", "sideband";
+		sifive,a-mshr-count = <60>;
+		sifive,bank-count = <4>;
+		sifive,ecc-granularity = <8>;
+		sifive,max-master-id = <13>;
+		sifive,perfmon-counters = <6>;
+		numa-node-id = <1>;
+		ranges;
+	};
+
+
+	plic1: interrupt-controller@2c000000 {
+		#interrupt-cells = <1>;
+		compatible = "sifive,plic-1.0.0";
+		interrupt-controller;
+		interrupts-extended = <
+			&cpu4_intc 0xffffffff &cpu4_intc 9
+		#ifndef PLATFORM_HAPS
+			&cpu5_intc 0xffffffff &cpu5_intc 9
+			&cpu6_intc 0xffffffff &cpu6_intc 9
+			&cpu7_intc 0xffffffff &cpu7_intc 9
+		#endif
+			>;
+		reg = <0x0 0x2c000000 0x0 0x4000000>;
+		reg-names = "control";
+		riscv,max-priority = <7>;
+		riscv,ndev = <520>;
+		numa-node-id = <1>;
+	};
+
+	d1_l2_cache_0: pl2@20104000 {
+		cache-block-size = <64>;
+		cache-level = <2>;
+		cache-sets = <512>;
+		cache-size = <262144>;
+		cache-unified;
+		compatible = "sifive,pL2Cache0", "cache";
+		next-level-cache = <&D1CACHE>;
+		reg = <0x0 0x20104000 0x0 0x4000>;
+		reg-names = "control";
+		sifive,ecc-granularity = <16>;
+		sifive,perfmon-counters = <6>;
+		numa-node-id = <1>;
+	};
+	d1_l2_cache_1: pl2@20108000 {
+		cache-block-size = <64>;
+		cache-level = <2>;
+		cache-sets = <512>;
+		cache-size = <262144>;
+		cache-unified;
+		compatible = "sifive,pL2Cache0", "cache";
+		next-level-cache = <&D1CACHE>;
+		reg = <0x0 0x20108000 0x0 0x4000>;
+		reg-names = "control";
+		sifive,ecc-granularity = <16>;
+		sifive,perfmon-counters = <6>;
+		numa-node-id = <1>;
+	};
+	d1_l2_cache_2: pl2@2010c000 {
+		cache-block-size = <64>;
+		cache-level = <2>;
+		cache-sets = <512>;
+		cache-size = <262144>;
+		cache-unified;
+		compatible = "sifive,pL2Cache0", "cache";
+		next-level-cache = <&D1CACHE>;
+		reg = <0x0 0x2010c000 0x0 0x4000>;
+		reg-names = "control";
+		sifive,ecc-granularity = <16>;
+		sifive,perfmon-counters = <6>;
+		numa-node-id = <1>;
+	};
+	d1_l2_cache_3: pl2@20110000 {
+		cache-block-size = <64>;
+		cache-level = <2>;
+		cache-sets = <512>;
+		cache-size = <262144>;
+		cache-unified;
+		compatible = "sifive,pL2Cache0", "cache";
+		next-level-cache = <&D1CACHE>;
+		reg = <0x0 0x20110000 0x0 0x4000>;
+		reg-names = "control";
+		sifive,ecc-granularity = <16>;
+		sifive,perfmon-counters = <6>;
+		numa-node-id = <1>;
+	};
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-arch.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-arch.dtsi
new file mode 100644
index 000000000000..98c4689b99a6
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-arch.dtsi
@@ -0,0 +1,499 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Eswin EIC7700 SoC's cpu.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <dt-bindings/clock/eswin,eic7700-clock.h>
+#include "eic770x-opptable.dtsi"
+
+#define UART0_INT	100
+#define UART1_INT	101
+#define UART2_INT	102
+#define UART3_INT	103
+#define UART4_INT	104
+#define I2C_BITRATE_STANDARD    100000	/* 100 Kbit/s */
+#define I2C_BITRATE_FAST        400000	/* 400 Kbit/s */
+#define I2C_BITRATE_FAST_PLUS   1000000 /* 1 Mbit/s */
+#define I2C_BITRATE_HIGH        3400000	/* 3.4 Mbit/s */
+#define I2C_BITRATE_ULTRA       5000000 /* 5 Mbit/s */
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "SiFive,FU800-dev", "fu800-dev", "sifive-dev", "eic7700-dev";
+
+	L64: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <RTCCLK_FREQ>;
+		idle-states {
+			CPU_RET: cpu-retentive {
+				compatible = "riscv,idle-state";
+				riscv,sbi-suspend-param = <0x00000000>;
+				entry-latency-us = <20>;
+				exit-latency-us = <40>;
+				min-residency-us = <80>;
+			};
+		};
+		cpu_0: cpu@0 {
+			clock-frequency = <0>;
+			compatible = "eswin,eic770x", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			hardware-exec-breakpoint-count = <4>;
+			hwpf-distanceBits = <6>;
+			hwpf-hitCacheThrdBits = <5>;
+			hwpf-hitMSHRThrdBits = <4>;
+			hwpf-l2pfPoolSize = <10>;
+			hwpf-nIssQEnt = <6>;
+			hwpf-nPrefetchQueueEntries = <8>;
+			hwpf-nStreams = <16>;
+			hwpf-qFullnessThrdBits = <4>;
+			hwpf-windowBits = <6>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&d0_l2_cache_0>;
+			reg = <0x0>;
+			riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+			riscv,pmpgranularity = <4096>;
+			riscv,pmpregions = <8>;
+			sifive,buserror = <&d0_bus_err0>;
+			status = "okay";
+			timebase-frequency = <RTCCLK_FREQ>;
+			tlb-split;
+			numa-node-id = <0>;
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_0>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <800>;
+			cpu-idle-states = <&CPU_RET>;
+			cpu0_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+		cpu_1: cpu@1 {
+			clock-frequency = <0>;
+			compatible = "eswin,eic770x", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			hardware-exec-breakpoint-count = <4>;
+			hwpf-distanceBits = <6>;
+			hwpf-hitCacheThrdBits = <5>;
+			hwpf-hitMSHRThrdBits = <4>;
+			hwpf-l2pfPoolSize = <10>;
+			hwpf-nIssQEnt = <6>;
+			hwpf-nPrefetchQueueEntries = <8>;
+			hwpf-nStreams = <16>;
+			hwpf-qFullnessThrdBits = <4>;
+			hwpf-windowBits = <6>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&d0_l2_cache_1>;
+			reg = <0x1>;
+			riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+			riscv,pmpgranularity = <4096>;
+			riscv,pmpregions = <8>;
+			sifive,buserror = <&d0_bus_err1>;
+			status = "okay";
+			timebase-frequency = <RTCCLK_FREQ>;
+			tlb-split;
+			numa-node-id = <0>;
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_1>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <800>;
+			cpu-idle-states = <&CPU_RET>;
+			cpu1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+		cpu_2: cpu@2 {
+			clock-frequency = <0>;
+			compatible = "eswin,eic770x", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			hardware-exec-breakpoint-count = <4>;
+			hwpf-distanceBits = <6>;
+			hwpf-hitCacheThrdBits = <5>;
+			hwpf-hitMSHRThrdBits = <4>;
+			hwpf-l2pfPoolSize = <10>;
+			hwpf-nIssQEnt = <6>;
+			hwpf-nPrefetchQueueEntries = <8>;
+			hwpf-nStreams = <16>;
+			hwpf-qFullnessThrdBits = <4>;
+			hwpf-windowBits = <6>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&d0_l2_cache_2>;
+			reg = <0x2>;
+			riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+			riscv,pmpgranularity = <4096>;
+			riscv,pmpregions = <8>;
+			sifive,buserror = <&d0_bus_err2>;
+			status = "okay";
+			timebase-frequency = <RTCCLK_FREQ>;
+			tlb-split;
+			numa-node-id = <0>;
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_2>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <800>;
+			cpu-idle-states = <&CPU_RET>;
+			cpu2_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+		cpu_3: cpu@3 {
+			clock-frequency = <0>;
+			compatible = "eswin,eic770x", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			hardware-exec-breakpoint-count = <4>;
+			hwpf-distanceBits = <6>;
+			hwpf-hitCacheThrdBits = <5>;
+			hwpf-hitMSHRThrdBits = <4>;
+			hwpf-l2pfPoolSize = <10>;
+			hwpf-nIssQEnt = <6>;
+			hwpf-nPrefetchQueueEntries = <8>;
+			hwpf-nStreams = <16>;
+			hwpf-qFullnessThrdBits = <4>;
+			hwpf-windowBits = <6>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&d0_l2_cache_3>;
+			reg = <0x3>;
+			riscv,isa = "rv64imafdch_zicsr_zifencei_zba_zbb_sscofpmf";
+			riscv,pmpgranularity = <4096>;
+			riscv,pmpregions = <8>;
+			sifive,buserror = <&d0_bus_err3>;
+			status = "okay";
+			timebase-frequency = <RTCCLK_FREQ>;
+			tlb-split;
+			numa-node-id = <0>;
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_3>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <800>;
+			cpu-idle-states = <&CPU_RET>;
+			cpu3_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+	};
+
+	SOC: soc {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		compatible = "SiFive,FU800-soc", "fu800-soc", "sifive-soc", "simple-bus";
+		ranges;
+		PMU0: pmu@0 {
+			riscv,raw-event-to-mhpmcounters = <0x0 0x0  0xffffffff 0xff          0x1f8
+											   0x0 0x1  0xffffffff 0xfff800ff    0x1f8
+											   0x0 0x2  0xffffffff 0xffffc0ff    0x1f8>;
+			riscv,event-to-mhpmcounters = <0x01 0x01 0x01 0x02 0x02 0x02 0x4 0x6 0x1f8 0x10009 0x10009 0x1f8 0x10019 0x10019 0x1f8 0x10021 0x10021 0x1f8>;
+			riscv,event-to-mhpmevent = <0x4 0x0 0x202 0x5 0x0 0x4000 0x6 0x0 0x2001 0x10009 0x0 0x102 0x10019 0x0 0x1002 0x10021 0x0 0x802>;
+			compatible = "riscv,pmu0", "riscv,pmu";
+			interrupts-extended = <&cpu0_intc 13 &cpu1_intc 13 &cpu2_intc 13 &cpu3_intc 13>;
+			numa-node-id = <0>;
+		};
+		L40: authentication-controller {
+			compatible = "sifive,authentication0";
+			sifive,auth-types = "fuse";
+		};
+		L51: axi4-sys-port@40000000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "sifive,axi4-sys-port", "sifive,axi4-port", "sifive,sys-port", "simple-external-bus", "simple-bus";
+			ranges = <0x40000000 0x0 0x40000000 0x40000000>;
+			sifive,port-width-bytes = <16>;
+		};
+		L52: axi4-sys-port@8000000000 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "sifive,axi4-sys-port", "sifive,axi4-port", "sifive,sys-port", "simple-external-bus", "simple-bus";
+			ranges = <0x80 0x0 0x80 0x0 0x180 0x0>;
+			sifive,port-width-bytes = <16>;
+		};
+		L46: basic-bus-blocker@200000 {
+			compatible = "sifive,basic-bus-blocker1";
+			reg = <0x0 0x200000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L43: basic-bus-blocker@202000 {
+			compatible = "sifive,basic-bus-blocker1";
+			reg = <0x0 0x202000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L48: basic-bus-blocker@204000 {
+			compatible = "sifive,basic-bus-blocker1";
+			reg = <0x0 0x204000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L54: burst-bundler@10010000 {
+			compatible = "sifive,burst-bundler0";
+			reg = <0x0 0x10010000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		d0_bus_err0: bus-error-unit@hart0 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <517>;
+			reg = <0x0 0x1700000 0x0 0x1000>;
+			reg-names = "control";
+			numa-node-id = <0>;
+		};
+		d0_bus_err1: bus-error-unit@hart1 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <518>;
+			reg = <0x0 0x1701000 0x0 0x1000>;
+			reg-names = "control";
+			numa-node-id = <0>;
+		};
+		d0_bus_err2: bus-error-unit@hart2 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <519>;
+			reg = <0x0 0x1702000 0x0 0x1000>;
+			reg-names = "control";
+			numa-node-id = <0>;
+		};
+		d0_bus_err3: bus-error-unit@hart3 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <520>;
+			reg = <0x0 0x1703000 0x0 0x1000>;
+			reg-names = "control";
+			numa-node-id = <0>;
+		};
+		D0CACHE: cache-controller@2010000 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			cache-block-size = <64>;
+			cache-level = <3>;
+			cache-sets = <4096>;
+			cache-size = <4194304>;
+			cache-unified;
+			compatible = "sifive,ccache1", "cache", "sifive,fu740-c000-ccache";
+			interrupt-parent = <&plic0>;
+			interrupts = <1>, <3>, <4>, <2>;
+			reg = <0x0 0x2010000 0x0 0x4000 0x0 0x8000000 0x0 0x400000>;
+			reg-names = "control", "sideband";
+			sifive,a-mshr-count = <60>;
+			sifive,bank-count = <4>;
+			sifive,ecc-granularity = <8>;
+			sifive,max-master-id = <13>;
+			sifive,perfmon-counters = <6>;
+			numa-node-id = <0>;
+			ranges;
+			zero-device@1a000000 {
+				compatible = "l3,zero-device";
+				reg = <0x0 0x1a000000 0x0 0x400000>;
+			};
+		};
+		L34: debug-controller@0 {
+			compatible = "sifive,debug-100", "riscv,debug-100";
+			debug-attach = "jtag";
+			reg = <0x0 0x0 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L8: error-device@1000 {
+			compatible = "sifive,error0";
+			reg = <0x0 0x1000 0x0 0x3000 0x0 0x5000 0x0 0x13000 0x0 0x19000 0x0 0xe7000 0x0 0x114000 0x0 0xec000 0x0 0x201000 0x0 0x1000 0x0 0x203000 0x0 0x1000 0x0 0x205000 0x0 0x14fb000 0x0 0x1704000 0x0 0x8fc000 0x0 0x2014000 0x0 0x5fec000 0x0 0x8400000 0x0 0x3c00000 0x0 0x10000000 0x0 0x3000 0x0 0x10004000 0x0 0xc000 0x0 0x10011000 0x0 0x1f000 0x0 0x10034000 0x0 0x9fcc000 0x0 0x1a400000 0x0 0x5c00000>;
+		};
+		L9: error-device@10003000 {
+			compatible = "sifive,error0";
+			reg = <0x0 0x10003000 0x0 0x1000>;
+		};
+		plic0: interrupt-controller@c000000 {
+			#interrupt-cells = <1>;
+			compatible = "sifive,plic-1.0.0";
+			interrupt-controller;
+			interrupts-extended = <
+				&cpu0_intc 0xffffffff &cpu0_intc 9
+				&cpu1_intc 0xffffffff &cpu1_intc 9
+				&cpu2_intc 0xffffffff &cpu2_intc 9
+				&cpu3_intc 0xffffffff &cpu3_intc 9>;
+			reg = <0x0 0xc000000 0x0 0x4000000>;
+			reg-names = "control";
+			riscv,max-priority = <7>;
+			riscv,ndev = <520>;
+			numa-node-id = <0>;
+		};
+		L53: order-obliterator@10030000 {
+			compatible = "sifive,order-obliterator0";
+			interrupt-parent = <&plic0>;
+			interrupts = <516>;
+			reg = <0x0 0x10030000 0x0 0x4000>;
+			reg-names = "control";
+		};
+		d0_l2_cache_0: pl2@104000 {
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-sets = <512>;
+			cache-size = <262144>;
+			cache-unified;
+			compatible = "sifive,pL2Cache0", "cache";
+			next-level-cache = <&D0CACHE>;
+			reg = <0x0 0x104000 0x0 0x4000>;
+			reg-names = "control";
+			sifive,ecc-granularity = <16>;
+			sifive,perfmon-counters = <6>;
+			numa-node-id = <0>;
+		};
+		d0_l2_cache_1: pl2@108000 {
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-sets = <512>;
+			cache-size = <262144>;
+			cache-unified;
+			compatible = "sifive,pL2Cache0", "cache";
+			next-level-cache = <&D0CACHE>;
+			reg = <0x0 0x108000 0x0 0x4000>;
+			reg-names = "control";
+			sifive,ecc-granularity = <16>;
+			sifive,perfmon-counters = <6>;
+			numa-node-id = <0>;
+		};
+		d0_l2_cache_2: pl2@10c000 {
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-sets = <512>;
+			cache-size = <262144>;
+			cache-unified;
+			compatible = "sifive,pL2Cache0", "cache";
+			next-level-cache = <&D0CACHE>;
+			reg = <0x0 0x10c000 0x0 0x4000>;
+			reg-names = "control";
+			sifive,ecc-granularity = <16>;
+			sifive,perfmon-counters = <6>;
+			numa-node-id = <0>;
+		};
+		d0_l2_cache_3: pl2@110000 {
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-sets = <512>;
+			cache-size = <262144>;
+			cache-unified;
+			compatible = "sifive,pL2Cache0", "cache";
+			next-level-cache = <&D0CACHE>;
+			reg = <0x0 0x110000 0x0 0x4000>;
+			reg-names = "control";
+			sifive,ecc-granularity = <16>;
+			sifive,perfmon-counters = <6>;
+			numa-node-id = <0>;
+		};
+		L10: rom@1a000000 {
+			compatible = "ucbbar,cacheable-zero0";
+			reg = <0x0 0x1a000000 0x0 0x400000>;
+		};
+		L11: rom@3a000000 {
+			compatible = "ucbbar,cacheable-zero0";
+			reg = <0x0 0x3a000000 0x0 0x400000>;
+		};
+		L6: subsystem_pbus_clock {
+			#clock-cells = <0>;
+			clock-frequency = <10000000>;
+			clock-output-names = "subsystem_pbus_clock";
+			compatible = "fixed-clock";
+		};
+		L61: teststatus@4000 {
+			compatible = "sifive,test0";
+			reg = <0x0 0x4000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L45: tl-address-adjuster@20000000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "sifive,tl-inter-sys-port", "sifive,tl-port", "simple-external-bus", "simple-bus";
+			ranges = <0x20000000 0x0 0x20000000 0x1a000000 0x3a400000 0x0 0x3a400000 0x5c00000>;
+			sifive,port-width-bytes = <8>;
+		};
+		L42: tl-inter-mem-master-port@80000000 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "sifive,tl-inter-mem-master-port", "sifive,tl-port", "sifive,inter-mem-master-port", "simple-external-bus", "simple-bus";
+			ranges = <0x0 0x80000000 0x0 0x80000000 0x7f 0x80000000>;
+			sifive,port-width-bytes = <32>;
+		};
+		L55: trace-encoder-0@100000 {
+			compatible = "sifive,trace0";
+			reg = <0x0 0x100000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L56: trace-encoder-1@101000 {
+			compatible = "sifive,trace0";
+			reg = <0x0 0x101000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L57: trace-encoder-2@102000 {
+			compatible = "sifive,trace0";
+			reg = <0x0 0x102000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L58: trace-encoder-3@103000 {
+			compatible = "sifive,trace0";
+			reg = <0x0 0x103000 0x0 0x1000>;
+			reg-names = "control";
+		};
+		L59: trace-funnel-0@18000 {
+			compatible = "sifive,trace0";
+			reg = <0x0 0x18000 0x0 0x1000>;
+			reg-names = "control";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-die0-noc.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-die0-noc.dtsi
new file mode 100644
index 000000000000..644210d666c2
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-die0-noc.dtsi
@@ -0,0 +1,2809 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Die0 NOC monitor of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+d0_cfg_noc:d0_cfg_noc{
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x52060000 0 0x4000>;
+	numa-node-id = <0>;
+
+	interrupts = <446>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 5>;
+
+	sideband_manager@52061000{
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52061000 0 0x10>;
+		SenseIn0 =
+			<SBM_CLMM 	OFFSET0>,
+			<SBM_CNOC_AON 	OFFSET1>,
+			<SBM_CNOC_DDRT0_CTRL 	OFFSET2>,
+			<SBM_CNOC_DDRT0_PHY 	OFFSET3>,
+			<SBM_CNOC_DDRT1_CTRL 	OFFSET4>,
+			<SBM_CNOC_DDRT1_PHY 	OFFSET5>,
+			<SBM_CNOC_DSPT 	OFFSET6>,
+			<SBM_CNOC_GPU 	OFFSET7>,
+			<SBM_CNOC_HSP 	OFFSET8>,
+			<SBM_CNOC_LSP_APB2 	OFFSET9>,
+			<SBM_CNOC_LSP_APB3	OFFSET10>,
+			<SBM_CNOC_LSP_APB4 	OFFSET11>,
+			<SBM_CNOC_LSP_APB6 	OFFSET12>,
+			<SBM_CNOC_MCPUT_D2D 	OFFSET13>,
+			<SBM_CNOC_NPU 	OFFSET14>,
+			<SBM_CNOC_PCIET_P 	OFFSET15>,
+			<SBM_CNOC_PCIET_X 	OFFSET16>,
+			<SBM_CNOC_TCU 	OFFSET17>,
+			<SBM_CNOC_VC 	OFFSET18>,
+			<SBM_CNOC_VI 	OFFSET19>,
+			<SBM_CNOC_VO 	OFFSET20>;
+		bf-name =
+			"SBM_CLMM",
+			"SBM_CNOC_AON",
+			"SBM_CNOC_DDRT0_CTRL",
+			"SBM_CNOC_DDRT0_PHY ",
+			"SBM_CNOC_DDRT1_CTRL",
+			"SBM_CNOC_DDRT1_PHY",
+			"SBM_CNOC_DSPT",
+			"SBM_CNOC_GPU",
+			"SBM_CNOC_HSP",
+			"SBM_CNOC_LSP_APB2",
+			"SBM_CNOC_LSP_APB3",
+			"SBM_CNOC_LSP_APB4",
+			"SBM_CNOC_LSP_APB6",
+			"SBM_CNOC_MCPUT_D2D",
+			"SBM_CNOC_NPU",
+			"SBM_CNOC_PCIET_P",
+			"SBM_CNOC_PCIET_X",
+			"SBM_CNOC_TCU",
+			"SBM_CNOC_VC",
+			"SBM_CNOC_VI",
+			"SBM_CNOC_VO";
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1>;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4>;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 15 1 >;  /*bit 15 will aloways be 0, then we will always get "snoc_cnoc/I/0"*/
+			lut =
+				"snoc_cnoc/I/0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 10 5 >;
+			lut =
+				"clmm/T/0",
+				"cnoc_aon/T/0",
+				"cnoc_ddrt0_ctrl/T/0",
+				"cnoc_ddrt0_phy/T/0",
+				"cnoc_ddrt1_ctrl/T/0",
+				"cnoc_ddrt1_phy/T/0",
+				"cnoc_dspt/T/0",
+				"cnoc_gpu/T/0",
+				"cnoc_hsp/T/0",
+				"cnoc_lsp_apb2/T/0",
+				"cnoc_lsp_apb3/T/0",
+				"cnoc_lsp_apb4/T/0",
+				"cnoc_lsp_apb6/T/0",
+				"cnoc_mcput_d2d/T/0",
+				"cnoc_npu/T/0",
+				"cnoc_pciet_p/T/0",
+				"cnoc_pciet_x/T/0",
+				"cnoc_service/T/0",
+				"cnoc_tcu/T/0",
+				"cnoc_vc/T/0",
+				"cnoc_vi/T/0",
+				"cnoc_vo/T/0",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 2 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 8 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger1 whose information are required to calculate real absolute address */
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 8 >;	/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 46 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <0x0 0x0    0x0  0x51600000>,
+				/bits/ 64  <0x0 0x0    0x1  0x71600000>,
+				/bits/ 64  <0x0 0x1    0x0  0x51800000>,
+				/bits/ 64  <0x0 0x1    0x1  0x71800000>,
+				/bits/ 64  <0x0 0x1    0x2  0x0       >,
+				/bits/ 64  <0x0 0x1    0x3  0x0       >,
+				/bits/ 64  <0x0 0x2    0x0  0x52300000>,
+				/bits/ 64  <0x0 0x2    0x1  0x72300000>,
+				/bits/ 64  <0x0 0x3    0x0  0x53000000>,
+				/bits/ 64  <0x0 0x3    0x1  0x73000000>,
+				/bits/ 64  <0x0 0x4    0x0  0x52380000>,
+				/bits/ 64  <0x0 0x4    0x1  0x72380000>,
+				/bits/ 64  <0x0 0x5    0x0  0x53800000>,
+				/bits/ 64  <0x0 0x5    0x1  0x73800000>,
+				/bits/ 64  <0x0 0x6    0x0  0x52200000>,
+				/bits/ 64  <0x0 0x6    0x1  0x72200000>,
+				/bits/ 64  <0x0 0x7    0x0  0x51400000>,
+				/bits/ 64  <0x0 0x7    0x1  0x71400000>,
+				/bits/ 64  <0x0 0x8    0x0  0x50400000>,
+				/bits/ 64  <0x0 0x8    0x1  0x70400000>,
+				/bits/ 64  <0x0 0x9    0x0  0x50800000>,
+				/bits/ 64  <0x0 0x9    0x1  0x70800000>,
+				/bits/ 64  <0x0 0xa    0x0  0x50900000>,
+				/bits/ 64  <0x0 0xa    0x1  0x70900000>,
+				/bits/ 64  <0x0 0xb    0x0  0x50a00000>,
+				/bits/ 64  <0x0 0xb    0x1  0x70a00000>,
+				/bits/ 64  <0x0 0xc    0x0  0x50b00000>,
+				/bits/ 64  <0x0 0xc    0x1  0x70b00000>,
+				/bits/ 64  <0x0 0xd    0x0  0x52100000>,
+				/bits/ 64  <0x0 0xd    0x1  0x72100000>,
+				/bits/ 64  <0x0 0xe    0x0  0x51c00000>,
+				/bits/ 64  <0x0 0xe    0x1  0x71c00000>,
+				/bits/ 64  <0x0 0xf    0x0  0x50000000>,
+				/bits/ 64  <0x0 0xf    0x1  0x70000000>,
+				/bits/ 64  <0x0 0x10   0x0  0x54000000>,
+				/bits/ 64  <0x0 0x10   0x1  0x74000000>,
+				/bits/ 64  <0x0 0x11   0x0  0x52060000>,
+				/bits/ 64  <0x0 0x11   0x1  0x72060000>,
+				/bits/ 64  <0x0 0x12   0x0  0x50c00000>,
+				/bits/ 64  <0x0 0x12   0x1  0x70c00000>,
+				/bits/ 64  <0x0 0x13   0x0  0x50100000>,
+				/bits/ 64  <0x0 0x13   0x1  0x70100000>,
+				/bits/ 64  <0x0 0x14   0x0  0x51000000>,
+				/bits/ 64  <0x0 0x14   0x1  0x71000000>,
+				/bits/ 64  <0x0 0x15   0x0  0x50200000>,
+				/bits/ 64  <0x0 0x15   0x1  0x70200000>;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32>;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 7 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0",
+				"Prot_1",
+				"Prot_2";
+		};
+	};
+};
+
+d0_llc_noc:d0_llc_noc@52081400 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <0>;
+	reg = <0 0x52081400 0 0x4000>;
+	interrupts = <441>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+#ifdef PLATFORM_HAPS
+	clock,rate = <5000000>;  /*haps ddr controller clk*/
+#endif
+	sideband_manager@52082000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52082000 0 0x10>;
+		SenseIn0 =
+			<SBM_LNOC_NPU_LLC0 	OFFSET0>,
+			<SBM_LNOC_NPU_LLC1 	OFFSET1>,
+			<SBM_LNOC_DDRT0_P0 	OFFSET2>,
+			<SBM_LNOC_DDRT1_P0 	OFFSET3>;
+		bf-name =
+			"SBM_LNOC_NPU_LLC0",
+			"SBM_LNOC_NPU_LLC1",
+			"SBM_LNOC_DDRT0_P0",
+			"SBM_LNOC_DDRT1_P0";
+	};
+
+	llcnoc_packet_ddr0_p0_req_probe@52080000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52080000 0 0x4000>;
+		clocks = <&d0_clock EIC7700_CLK_DDRT0_P0_ACLK>;
+		clock-names = "clk";
+		interrupts = <445>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p0_req";
+	};
+	llcnoc_packet_ddr1_p0_req_probe@52080800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52080800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P0_ACLK>;
+		clock-names = "clk";
+		interrupts = <443>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p0_req";
+	};
+	llcnoc_trans_probe@52081000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52081000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <441>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "npu_llc0", "npu_llc1";
+		llcnoc_trans_npu_llc0_filter@52081480 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52081480 0 0x80>;
+		};
+		llcnoc_trans_npu_llc1_filter@52081500 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52081500 0 0x80>;
+		};
+		llcnoc_trans_profiler@52081580 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52081580 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"npu_lnoc_llc0/I/0",
+				"npu_lnoc_llc1/I/0",
+				"snoc_lnoc/I/0",
+				"RESERVED";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"lnoc_ddrt0_p0/T/0",
+				"lnoc_ddrt1_p0/T/0",
+				"lnoc_service/T/0",
+				"RESERVED";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 4 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 8 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 8 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 53 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =		/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <0x0 0x0 0x0 0x80000000      >,
+				/bits/ 64  <0x0 0x0 0x1 0x100000000     >,
+				/bits/ 64  <0x0 0x0 0x2 0x200000000     >,
+				/bits/ 64  <0x0 0x0 0x3 0x400000000     >,
+				/bits/ 64  <0x0 0x0 0x4 0x800000000     >,
+				/bits/ 64  <0x0 0x0 0x5 0x2000000000    >,
+				/bits/ 64  <0x0 0x0 0x6 0xc000000000    >,
+				/bits/ 64  <0x0 0x0 0x7 0xe000000000    >,
+				/bits/ 64  <0x0 0x0 0x8 0x4000000000    >,
+				/bits/ 64  <0x0 0x0 0x9 0x4000000100    >,
+				/bits/ 64  <0x0 0x0 0xa 0x10000000000   >,
+				/bits/ 64  <0x0 0x0 0xb 0x10000000100   >,
+				/bits/ 64  <0x0 0x0 0xc 0x0             >,
+				/bits/ 64  <0x0 0x1 0x0 0x80000080      >,
+				/bits/ 64  <0x0 0x1 0x1 0x100000080     >,
+				/bits/ 64  <0x0 0x1 0x2 0x200000080     >,
+				/bits/ 64  <0x0 0x1 0x3 0x400000080     >,
+				/bits/ 64  <0x0 0x1 0x4 0x800000080     >,
+				/bits/ 64  <0x0 0x1 0x5 0x2000000080    >,
+				/bits/ 64  <0x0 0x1 0x6 0xc000000080    >,
+				/bits/ 64  <0x0 0x1 0x7 0xe000000080    >,
+				/bits/ 64  <0x0 0x1 0x8 0x4000000080    >,
+				/bits/ 64  <0x0 0x1 0x9 0x4000000180    >,
+				/bits/ 64  <0x0 0x1 0xa 0x10000000080   >,
+				/bits/ 64  <0x0 0x1 0xb 0x10000000180   >,
+				/bits/ 64  <0x1 0x0 0x0 0x80000000      >,
+				/bits/ 64  <0x1 0x0 0x1 0x100000000     >,
+				/bits/ 64  <0x1 0x0 0x2 0x200000000     >,
+				/bits/ 64  <0x1 0x0 0x3 0x400000000     >,
+				/bits/ 64  <0x1 0x0 0x4 0x800000000     >,
+				/bits/ 64  <0x1 0x0 0x5 0x2000000000    >,
+				/bits/ 64  <0x1 0x0 0x6 0xc000000000    >,
+				/bits/ 64  <0x1 0x0 0x7 0xe000000000    >,
+				/bits/ 64  <0x1 0x0 0x8 0x4000000000    >,
+				/bits/ 64  <0x1 0x0 0x9 0x4000000100    >,
+				/bits/ 64  <0x1 0x0 0xa 0x10000000000   >,
+				/bits/ 64  <0x1 0x0 0xb 0x10000000100   >,
+				/bits/ 64  <0x1 0x0 0xc 0x0             >,
+				/bits/ 64  <0x1 0x1 0x0 0x80000080      >,
+				/bits/ 64  <0x1 0x1 0x1 0x100000080     >,
+				/bits/ 64  <0x1 0x1 0x2 0x200000080     >,
+				/bits/ 64  <0x1 0x1 0x3 0x400000080     >,
+				/bits/ 64  <0x1 0x1 0x4 0x800000080     >,
+				/bits/ 64  <0x1 0x1 0x5 0x2000000080    >,
+				/bits/ 64  <0x1 0x1 0x6 0xc000000080    >,
+				/bits/ 64  <0x1 0x1 0x7 0xe000000080    >,
+				/bits/ 64  <0x1 0x1 0x8 0x4000000080    >,
+				/bits/ 64  <0x1 0x1 0x9 0x4000000180    >,
+				/bits/ 64  <0x1 0x1 0xa 0x10000000080   >,
+				/bits/ 64  <0x1 0x1 0xb 0x10000000180   >,
+				/bits/ 64  <0x2 0x2 0x0 0x52080000      >,
+				/bits/ 64  <0x2 0x2 0x1 0x72080000      >,
+				/bits/ 64  <0x2 0x2 0x2 0x0             >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 16 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0",
+				"Prot_1",
+				"Prot_2",
+				"Qos_0",
+				"Qos_1",
+				"Qos_2",
+				"Qos_3",
+				"User_0",
+				"User_1",
+				"User_2",
+				"User_3",
+				"User_4";
+		};
+	};
+};
+
+d0_sys_noc:d0_sys_noc@52002C00 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <0>;
+	reg = <0 0x52002C00 0 0x4000>;
+	interrupts = <431>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+
+	eswin,qos-configs = "DSPT", "NPU", "SPISLV_TBU3";
+#ifdef PLATFORM_HAPS
+	eswin,DSPT-qos-base = <0x52002C80>;
+	eswin,DSPT-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x1        /* mode 0:fixed 1:limiter 2:bypass 3:regulator*/
+		/* a number of (1/256)th of Bytes/cycle.
+		   Ex:DSP AXI clk=5MHz, BW=1MB/s, register value = (1/5)*256 = 0x33
+		*/
+		0x10 0x33    /* bandwidth. 1MB/s */
+		/*
+		   Saturation(B) = ((Requried Bandwidth)*(Windows Time of Bandwidth Calculation))/16
+		   Ex:16 byte saturation for BW=1MB/s means 16us window time.
+		   The desired value is number of saturation bytes divided by 16(ex,1 for 16byte B)
+		*/
+		0x14 0x1     /* saturation, 16us*/
+		0x18 0x0>;   /* QoSEn */
+
+	eswin,NPU-qos-base = <0x52002D00>;
+	eswin,NPU-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x1      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x0>;   /* QoSEn */
+
+	eswin,SPISLV_TBU3-qos-base = <0x52002D80>;
+	eswin,SPISLV_TBU3-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x1      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x0>;   /* QoSEn */
+	clock,rate = <5000000>; /*haps ddr controller axi clk*/
+#else
+	eswin,DSPT-qos-base = <0x52002C80>;
+	eswin,DSPT-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode 0:fixed 1:limiter 2:bypass 3:regulator*/
+		/* a number of (1/256)th of Bytes/cycle.
+		   Ex:zebu zdfi design feature, dsp AXI Clk=1040MHz, BW=12.1875MB/s, register value = (9.375/1040)*256 = 0x03
+		*/
+		0x10 0x03    /* bandwidth. 12.1875MB/s */
+		/*
+		   Saturation(B) = ((Requried Bandwidth)*(Windows Time of Bandwidth Calculation))/16
+		   Ex:16 byte saturation for BW=12.1875MB/s means 1.313us window time.
+		   The desired value is number of saturation bytes divided by 16(ex,1 for 16byte B)
+		*/
+		0x14 0x1     /* saturation, 1.313us*/
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,NPU-qos-base = <0x52002D00>;
+	eswin,NPU-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,SPISLV_TBU3-qos-base = <0x52002D80>;
+	eswin,SPISLV_TBU3-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+#endif
+	sideband_manager@52004000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52004000 0 0x10>;
+		SenseIn0 =
+			<SBM_AON_SNOC_SP0 	OFFSET0>,
+			<SBM_DSPT_SNOC 		OFFSET1>,
+			<SBM_JTAG_SNOC 		OFFSET2>,
+			<SBM_MCPUT_SNOC_D2D 	OFFSET3>,
+			<SBM_MCPUT_SNOC_MP 	OFFSET4>,
+			<SBM_MCPUT_SNOC_SP0 	OFFSET5>,
+			<SBM_MCPUT_SNOC_SP1 	OFFSET6>,
+			<SBM_NPU_SNOC_SP0 	OFFSET7>,
+			<SBM_NPU_SNOC_SP1 	OFFSET8>,
+			<SBM_PCIET_SNOC_P 	OFFSET9>,
+			<SBM_SPISLV_PCIET_SNOC	OFFSET10>,
+			<SBM_TBU4_SNOC 		OFFSET11>,
+			<SBM_TCU_SNOC 		OFFSET12>,
+			<SBM_SNOC_AON 		OFFSET13>,
+			<SBM_SNOC_DDR0_P1 	OFFSET14>,
+			<SBM_SNOC_DDR0_P2 	OFFSET15>,
+			<SBM_SNOC_DDR1_P1 	OFFSET16>,
+			<SBM_SNOC_DDR1_P2 	OFFSET17>,
+			<SBM_SNOC_DSPT 		OFFSET18>,
+			<SBM_SNOC_MCPUT_D2D 	OFFSET19>,
+			<SBM_SNOC_NPU 		OFFSET20>,
+			<SBM_SNOC_PCIET 	OFFSET21>;
+		bf-name =
+			"SBM_AON_SNOC_SP0",
+			"SBM_DSPT_SNOC",
+			"SBM_JTAG_SNOC",
+			"SBM_MCPUT_SNOC_D2D ",
+			"SBM_MCPUT_SNOC_MP",
+			"SBM_MCPUT_SNOC_SP0",
+			"SBM_MCPUT_SNOC_SP1",
+			"SBM_NPU_SNOC_SP0",
+			"SBM_NPU_SNOC_SP1",
+			"SBM_PCIET_SNOC_P",
+			"SBM_SPISLV_PCIET_SNOC",
+			"SBM_TBU4_SNOC",
+			"SBM_TCU_SNOC",
+			"SBM_SNOC_AON",
+			"SBM_SNOC_DDR0_P1",
+			"SBM_SNOC_DDR0_P2",
+			"SBM_SNOC_DDR1_P1",
+			"SBM_SNOC_DDR1_P2",
+			"SBM_SNOC_DSPT",
+			"SBM_SNOC_MCPUT_D2D",
+			"SBM_SNOC_NPU",
+			"SBM_SNOC_PCIET";
+	};
+	sysnoc_packet_ddr0_p1_req_probe@52000000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52000000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT0_P1_ACLK>;
+		clock-names = "clk";
+		interrupts = <439>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p1_req";
+	};
+	sysnoc_packet_ddr0_p2_req_probe@52000800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52000800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT0_P2_ACLK>;
+		clock-names = "clk";
+		interrupts = <437>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p2_req";
+	};
+	sysnoc_packet_ddr1_p1_req_probe@52001000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52001000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P1_ACLK>;
+		clock-names = "clk";
+		interrupts = <435>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p1_req";
+	};
+	sysnoc_packet_ddr1_p2_req_probe@52001800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52001800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P2_ACLK>;
+		clock-names = "clk";
+		interrupts = <433>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p2_req";
+	};
+	sysnoc_trans_probe_0@52002000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52002000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <430>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "dspt_snoc", "npu_sp1";
+		sysnoc_trans_dspt_filter@52002E00 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002E00 0 0x80>;
+		};
+		sysnoc_trans_npu_sp1_filter@52002F80 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002F80 0 0x80>;
+		};
+		sysnoc_trans_profiler@52003180 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52003180 0 0x80>;
+		};
+	};
+	sysnoc_trans_probe_1@52002400 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52002400 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <429>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <3>;
+		counter,nr = <12>;
+		profiler,nr = <1>;
+		portsel = "mcput_mp", "mcput_sp1", "tcu";
+		sysnoc_trans_mcput_mp_filter@52002E80 {
+			status = "okay";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002E80 0 0x80>;
+		};
+		sysnoc_trans_mcput_sp1_filter@52002F00 {
+			status = "okay";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002F00 0 0x80>;
+		};
+		sysnoc_trans_tcu_filter@52003100 {
+			status = "okay";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52003100 0 0x80>;
+		};
+		sysnoc_trans_profiler@52003200 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52003200 0 0x80>;
+		};
+	};
+	sysnoc_trans_probe_2@52002800 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52002800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <428>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "spislv_tbu3", "tbu4_snoc";
+		sysnoc_trans_spislv_tbu3_filter@52003000 {	/*pcie subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52003000 0 0x80>;
+		};
+		sysnoc_trans_tbu4_filter@52003080 {	/*aon subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52003080 0 0x80>;
+		};
+		sysnoc_trans_profiler@52003280 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52003280 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1>;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4>;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 19 4 >;
+			lut =
+				"aon_snoc_sp0/I/0",
+				"dspt_snoc/I/0",
+#ifdef PLATFORM_HAPS
+				"fpga_snoc/I/0",
+#endif
+				"jtag_snoc/I/0",
+				"mcput_snoc_d2d/I/0",
+				"mcput_snoc_mp/I/0",
+				"mcput_snoc_sp0/I/0",
+				"mcput_snoc_sp1/I/0",
+				"mnoc_snoc/I/0",
+				"npu_snoc_sp0/I/0",
+				"npu_snoc_sp1/I/0",
+				"pciet_snoc_p/I/0",
+				"rnoc_snoc/I/0",
+				"spislv_tbu3_snoc/I/0",
+				"tbu4_snoc/I/0",
+				"tcu_snoc/I/0",
+				"RESERVED0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 15 4 >;
+			lut =
+				"snoc_aon/T/0",
+				"snoc_cnoc/T/0",
+				"snoc_ddrt0_p1/T/0",
+				"snoc_ddrt0_p2/T/0",
+				"snoc_ddrt1_p1/T/0",
+				"snoc_ddrt1_p2/T/0",
+				"snoc_dspt/T/0",
+				"snoc_lnoc/T/0",
+				"snoc_mcput_d2d/T/0",
+				"snoc_mnoc/T/0",
+				"snoc_npu/T/0",
+				"snoc_pciet/T/0",
+				"snoc_rnoc/T/0",
+				"snoc_service/T/0",
+				"RESERVED1",
+				"RESERVED2";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 6 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 14 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 1181 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =		/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+#ifdef PLATFORM_HAPS
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_service_T_O   0x1             0x72000000        >,
+#endif
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x6             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x7             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x6             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x7             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x6             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x7             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x8             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x9             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0xa             0x0               >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0xb             0x0               >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x4             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x5             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x6             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x7             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xf             0x0               >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x4             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x5             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x6             0x5c000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x7             0x7c000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xe             0x0               >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xf             0x0               >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x13            0x0               >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x13            0x0               >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x13            0x0               >;
+			};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32>;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 18 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0 ",
+				"Prot_1 ",
+				"Prot_2 ",
+				"User_0 ",
+				"User_1 ",
+				"User_2 ",
+				"User_3 ",
+				"User_4 ",
+				"User_5 ",
+				"User_6 ",
+				"qos0",
+				"qos1",
+				"qos2",
+				"qos3";
+		};
+	};
+};
+
+d0_media_noc:d0_media_noc@52021400 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <0>;
+	reg = <0 0x52021400 0 0x4000>;
+	interrupts = <454>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+	eswin,qos-configs = "GPU", "TBU2", "VC";
+	eswin,GPU-qos-base = <0x52021480>;
+	eswin,GPU-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode */
+		0x10 0x1E0    /* bandwidth*/
+		0x14 0x1     /* saturation*/
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,TBU2-qos-base = <0x52021500>;
+	eswin,TBU2-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,VC-qos-base = <0x52021580>;
+	eswin,VC-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+#ifdef PLATFORM_HAPS
+	clock,rate = <5000000>;  /*haps ddr controller clk*/
+#endif
+	sideband_manager@52022000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52022000 0 0x10>;
+		SenseIn0 =
+			<SBM_MNOC_GPU 	OFFSET0>,
+			<SBM_MNOC_TBU2 	OFFSET1>,
+			<SBM_MNOC_VC 	OFFSET2>,
+			<SBM_MNOC_DDRT0_P3 	OFFSET3>,
+			<SBM_MNOC_DDRT1_P3 	OFFSET4>;
+		bf-name =
+			"SBM_MNOC_GPU",
+			"SBM_MNOC_TBU2",
+			"SBM_MNOC_VC",
+			"SBM_MNOC_DDRT0_P3",
+			"SBM_MNOC_DDRT1_P3";
+	};
+
+	mnoc_packet_ddr0_p3_req_probe@52020000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52020000 0 0x4000>;
+		clocks = <&d0_clock EIC7700_CLK_DDRT0_P3_ACLK>;
+		clock-names = "clk";
+		interrupts = <458>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p3_req";
+	};
+	mnoc_packet_ddr1_p3_req_probe@52020800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52020800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P3_ACLK>;
+		clock-names = "clk";
+		interrupts = <456>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p3_req";
+	};
+	mnoc_trans_probe@52021000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52021000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <453>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <3>;
+		counter,nr = <12>;
+		profiler,nr = <1>;
+		portsel = "gpu", "tbu2", "vc";
+		mnoc_trans_gpu_filter@52021600 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52021600 0 0x80>;
+		};
+		mnoc_trans_tbu2_filter@52021680 {	/*hsp subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52021680 0 0x80>;
+		};
+		mnoc_trans_vc_filter@52021700 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52021700 0 0x80>;
+		};
+		mnoc_trans_profiler@52021780 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52021780 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"gpu_mnoc/I/0",
+				"snoc_mnoc/I/0",
+				"tbu2_mnoc/I/0",
+				"vc_mnoc/I/0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"mnoc_ddrt0_p3/T/0",
+				"mnoc_ddrt1_p3/T/0",
+				"mnoc_service/T/0",
+				"mnoc_snoc/T/0";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 3 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 7 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 55 >;			/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+					/bits/ 64  <0x0  0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x0  0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x0  0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x0  0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x0  0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x0  0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x0  0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x0  0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x0  0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x0  0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x0  0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x0  0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x0  0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x0  0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x0  0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x0  0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x0  0x3 0x0           0x0              >,
+					/bits/ 64  <0x1 0x2  0x0           0x52020000       >,
+					/bits/ 64  <0x1 0x2  0x1           0x72020000       >,
+					/bits/ 64  <0x1 0x2  0x2           0x0              >,
+					/bits/ 64  <0x1 0x2  0x3           0x0              >,
+					/bits/ 64  <0x2 0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x2 0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x2 0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x2 0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x2 0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x2 0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x2 0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x2 0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x2 0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x2 0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x2 0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x2 0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x2 0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x2 0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x2 0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x2 0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x2 0x3   0x0           0x0              >,
+					/bits/ 64  <0x3   0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x3   0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x3   0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x3   0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x3   0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x3   0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x3   0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x3   0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x3   0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x3   0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x3   0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x3   0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x3   0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x3   0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x3   0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x3   0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x3   0x3 0x0           0x0              >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 18 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0 ",
+				"Prot_1 ",
+				"Prot_2 ",
+				"User_0 ",
+				"User_1 ",
+				"User_2 ",
+				"User_3 ",
+				"User_4 ",
+				"User_5 ",
+				"User_6 ",
+				"qos0",
+				"qos1",
+				"qos2",
+				"qos3";
+		};
+	};
+};
+
+d0_realtime_noc:d0_realtime_noc@52041400 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <0>;
+	reg = <0 0x52041400 0 0x4000>;
+	interrupts = <448>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+	eswin,qos-configs = "TBU0", "VO";
+	eswin,TBU0-qos-base = <0x52041480>;
+	eswin,TBU0-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode */
+		0x10 0x1E0    /* bandwidth */
+		0x14 0x1     /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,VO-qos-base = <0x52041500>;
+	eswin,VO-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+#ifdef PLATFORM_HAPS
+	clock,rate = <5000000>;  /*haps ddr controller clk*/
+#endif
+	sideband_manager@52042000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52042000 0 0x10>;
+		SenseIn0 =
+			<SBM_RNOC_TBU0 	OFFSET0>,
+			<SBM_RNOC_VO 	OFFSET1>,
+			<SBM_RNOC_DDRT0_P4 	OFFSET2>,
+			<SBM_RNOC_DDRT1_P4 	OFFSET3>;
+		bf-name =
+			"SBM_RNOC_TBU0",
+			"SBM_RNOC_VO",
+			"SBM_RNOC_DDRT0_P4",
+			"SBM_RNOC_DDRT1_P4";
+	};
+
+	rnoc_packet_ddr0_p4_req_probe@52040000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52040000 0 0x4000>;
+		clocks = <&d0_clock EIC7700_CLK_DDRT0_P4_ACLK>;
+		clock-names = "clk";
+		interrupts = <452>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p4_req";
+	};
+	rnoc_packet_ddr1_p4_req_probe@52040800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52040800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P4_ACLK>;
+		clock-names = "clk";
+		interrupts = <450>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p4_req";
+	};
+	rnoc_trans_probe@52041000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52041000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <447>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "tbu0", "vo";
+		rnoc_trans_tbu0_filter@52041580 {	/*vi subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52041580 0 0x80>;
+		};
+		rnoc_trans_vo_filter@52041600 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52041600 0 0x80>;
+		};
+		rnoc_trans_profiler@52041680 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52041680 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"snoc_rnoc/I/0",
+				"tbu0_rnoc/I/0",
+				"vo_rnoc/I/0",
+				"RESERVED0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"rnoc_ddrt0_p4/T/0",
+				"rnoc_ddrt1_p4/T/0",
+				"rnoc_service/T/0",
+				"rnoc_snoc/T/0";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 3 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 7 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 40 >;			/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+					/bits/ 64  <0x0 0x2  0x0           0x52040000       >,
+					/bits/ 64  <0x0 0x2  0x1           0x72040000       >,
+					/bits/ 64  <0x0 0x2  0x2           0x0              >,
+					/bits/ 64  <0x0 0x2  0x3           0x0              >,
+					/bits/ 64  <0x1 0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x1 0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x1 0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x1 0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x1 0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x1 0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x1 0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x1 0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x1 0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x1 0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x1 0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x1 0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x1 0x3     0x0           0x59000000       >,
+					/bits/ 64  <0x1 0x3     0x1           0x79000000       >,
+					/bits/ 64  <0x1 0x3     0x2           0x14001000000    >,
+					/bits/ 64  <0x1 0x3     0x3           0x14009000000    >,
+					/bits/ 64  <0x1 0x3     0x4           0x0              >,
+					/bits/ 64  <0x1 0x3     0x5           0x0              >,
+					/bits/ 64  <0x2   0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x2   0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x2   0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x2   0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x2   0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x2   0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x2   0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x2   0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x2   0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x2   0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x2   0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x2   0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x2   0x3     0x0           0x59000000       >,
+					/bits/ 64  <0x2   0x3     0x1           0x79000000       >,
+					/bits/ 64  <0x2   0x3     0x2           0x14001000000    >,
+					/bits/ 64  <0x2   0x3     0x3           0x14009000000    >,
+					/bits/ 64  <0x2   0x3     0x4           0x0              >,
+					/bits/ 64  <0x2   0x3     0x5           0x0              >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 16 >;
+			lut =
+				 "Cache_0",
+				 "Cache_1",
+				 "Cache_2",
+				 "Cache_3",
+				 "Prot_0 ",
+				 "Prot_1 ",
+				 "Prot_2 ",
+				 "User_0 ",
+				 "User_1 ",
+				 "User_2 ",
+				 "User_3 ",
+				 "User_4 ",
+				 "qos0",
+				 "qos1",
+				 "qos2",
+				 "qos3";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-die0-soc.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-die0-soc.dtsi
new file mode 100644
index 000000000000..981ef91ef58f
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-die0-soc.dtsi
@@ -0,0 +1,2624 @@
+﻿// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Die0 System peripherals of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eswin-win2030-arch.dtsi"
+
+
+#include <dt-bindings/memory/eswin-win2030-sid.h>
+#include <dt-bindings/mailbox/eswin-mailbox.h>
+#include <dt-bindings/reset/eswin,eic7700-reset.h>
+#include <dt-bindings/clock/eswin,eic7700-clock.h>
+#include <dt-bindings/i2c/i2c.h>
+#include <dt-bindings/interconnect/eswin,win2030.h>
+#include <dt-bindings/power/eswin,eic770x-pmu.h>
+#include <dt-bindings/thermal/thermal.h>
+#include <dt-bindings/pwm/pwm.h>
+/ {
+    compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000","sifive,fu740";
+	thermal-zones {
+		thermal0 {
+			polling-delay-passive = <100>; /*ms*/
+			polling-delay = <1000>; /*ms*/
+			sustainable-power = <30000>; /*mW*/
+			thermal-sensors = <&pvt0>;
+
+			trips {
+				threshold: trip-point0 {
+					temperature = <65000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				target: trip-point1 {
+					temperature = <75000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				crit: trip-point2 {
+					temperature     = <85000>;
+					hysteresis      = <0>;
+					type            = "critical";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device =
+						<&cpu_0 0 10>,
+						<&cpu_1 0 10>,
+						<&cpu_2 0 10>,
+						<&cpu_3 0 10>;
+				};
+				map1 {
+					trip = <&target>;
+					contribution = <4096>;
+					cooling-device = <&d0_npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				map2 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device = <&d0_dsp0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+									<&d0_dsp1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+									<&d0_dsp2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+									<&d0_dsp3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				map3 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device = <&gpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+		};
+	};
+};
+
+&SOC {
+	
+
+		d0_uart0: serial@0x50900000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x50900000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic0>;
+			interrupts = <UART0_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <0>;
+		};
+
+		d0_uart1: serial@0x50910000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x50910000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic0>;
+			interrupts = <UART1_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <0>;
+		};
+
+		d0_uart2: serial@0x50920000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x50920000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic0>;
+			interrupts = <UART2_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+
+		d0_uart3: serial@0x50930000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x50930000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic0>;
+			interrupts = <UART3_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+
+		d0_uart4: serial@0x50940000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x50940000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic0>;
+			interrupts = <UART4_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+
+		d0_sys_con: scu_sys_con@0x51810000 {
+			compatible = "eswin,win2030-scu-sys-con", "syscon", "simple-mfd";
+			#syscon-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x51810000 0x0 0x8000>;
+			numa-node-id = <0>;
+			d0_noc_wdt:noc@51810324 {
+				compatible = "eswin,win2030-noc-wdt";
+				interrupt-parent = <&plic0>;
+				interrupts = <392>, <393>, <394>, <395>,
+						<396>, <397>, <398>, <399>, <400>,
+						<401>, <402>, <403>, <404>, <405>,
+						<406>, <407>, <408>, <409>, <410>,
+						<411>, <412>, <413>, <414>, <415>,
+						<416>, <417>, <418>, <419>, <420>,
+						<421>, <422>, <423>, <424>, <425>,
+						<426>;
+				eswin,syscrg_csr = <&d0_sys_crg 0x100 0xffff>; //timeout paramerter
+				status = "disabled";
+			};
+		};
+
+		d0_sys_crg: sys-crg@51828000 {
+			compatible = "syscon", "simple-mfd";
+			reg = <0x000000 0x51828000 0x000000 0x80000>;
+			numa-node-id = <0>;
+			d0_reset: reset-controller {
+				compatible = "eswin,eic7700-reset";
+				#reset-cells = <2>;
+				numa-node-id = <0>;
+			};
+			d0_clock: clock-controller {
+				compatible = "eswin,eic7700-clock";
+				#clock-cells = <1>;
+				numa-node-id = <0>;
+			};
+		};
+
+		hfclk: hfclk {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <LSPCLK_FREQ>;
+			clock-output-names = "hfclk";
+			numa-node-id = <0>;
+		};
+
+		d0_hsp_sp_csr: hsp_sp_top_csr@0x50440000 {
+			compatible = "eswin,win2030-hsp-sp-csr", "syscon";
+			numa-node-id = <0>;
+			#size-cells = <2>;
+			reg = <0x0 0x50440000 0x0 0x2000>;
+		};
+
+		smmu0: iommu@50c00000 {
+			compatible = "arm,smmu-v3";
+			reg = <0x0 0x50c00000 0x0 0x100000>;
+			eswin,syscfg = <&d0_sys_con 0x3fc>;
+			interrupt-parent = <&plic0>;
+			interrupts = <356>,
+				<360>,
+				<357>,
+				<358>;
+			interrupt-names = "eventq", "gerror", "priq", "cmdq-sync";
+			#iommu-cells = <1>;
+			resets = <&d0_reset TCU_RST_CTRL SW_TCU_AXI_RSTN>,
+				<&d0_reset TCU_RST_CTRL SW_TCU_CFG_RSTN>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_0>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_1>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_2>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_3>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_4>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_5>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_6>,
+				<&d0_reset TCU_RST_CTRL TBU_RSTN_7>;
+			reset-names = "axi_rst", "cfg_rst", "tbu0_rst", "tbu1_rst", "tbu2_rst", "tbu3_rst",
+				"tbu4_rst", "tbu5_rst", "tbu6_rst", "tbu7_rst";
+			status = "disabled";
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		smmu_pmu0: pmu@50c02000 {
+			compatible = "arm,smmu-v3-pmcg";
+			reg = <0x0 0x50c02000 0x0 0x1000>,
+					<0x0 0x50c22000 0x0 0x1000>;
+			eswin,syscfg = <&d0_sys_con 0x3fc>;
+			interrupt-parent = <&plic0>;
+			interrupts = <363>;
+			status = "disabled";
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		dev_foo_a: E21@0 {
+			compatible = "riscv,dev-foo-a";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_A>;
+			tbus = <WIN2030_TBUID_0xF00>;
+			/*
+			tbus = <WIN2030_TBUID_0x0>,
+				<WIN2030_TBUID_0x10>, <WIN2030_TBUID_0x11>, <WIN2030_TBUID_0x12>, <WIN2030_TBUID_0x13>,
+				<WIN2030_TBUID_0x2>, <WIN2030_TBUID_0x3>, <WIN2030_TBUID_0x4>, <WIN2030_TBUID_0x5>,
+				<WIN2030_TBUID_0x70>, <WIN2030_TBUID_0x71>, <WIN2030_TBUID_0x72>, <WIN2030_TBUID_0x73>;
+			*/
+			status = "disabled";
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		d0_pmu: power-controller@51808000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "eswin,win2030-pmu-controller";
+			#power-domain-cells = <1>;
+			reg = <0x0 0x51808000 0x0 0x8000>;
+			numa-node-id = <0>;
+			d0_pmu_pcie: win2030-pmu-controller-port@0 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_PCIE>;
+				reg_base = <0x0>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D0_PCIE";
+				tbus = <WIN2030_TBUID_PCIE>;
+			};
+			d0_pmu_dsp1: win2030-pmu-controller-port@40 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_DSP1>;
+				reg_base = <0x40>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				tbus = <WIN2030_TBUID_DSP1>;
+				label = "D0_DSP1";
+			};
+			d0_pmu_vi: win2030-pmu-controller-port@80 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_VI>;
+				reg_base = <0x80>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				tbus = <WIN2030_TBUID_ISP>,<WIN2030_TBUID_DW>;
+				label = "D0_VI";
+			};
+			d0_pmu_vo: win2030-pmu-controller-port@c0 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_VO>;
+				reg_base = <0xc0>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D0_VO";
+			};
+			d0_pmu_codec: win2030-pmu-controller-port@140 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_CODEC>;
+				reg_base = <0x140>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				tbus = <WIN2030_TBUID_VDEC>, <WIN2030_TBUID_JDEC>,<WIN2030_TBUID_VENC>, <WIN2030_TBUID_JENC>;
+				label = "D0_CODEC";
+			};
+			d0_pmu_dsp2: win2030-pmu-controller-port@200 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_DSP2>;
+				reg_base = <0x200>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				tbus = <WIN2030_TBUID_DSP2>;
+				label = "D0_DSP2";
+			};
+			d0_pmu_dsp3: win2030-pmu-controller-port@240 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				id = <EIC770X_PD_DSP3>;
+				reg_base = <0x240>;
+				power_status = <2>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				tbus = <WIN2030_TBUID_DSP3>;
+				label = "D0_DSP3";
+			};
+		};
+
+		d0_aon_dmac: dma-controller-aon@0x518c0000 {
+			compatible = "eswin,eic770x-axi-dma";
+			reg = <0x0 0x518c0000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <289>;
+			#dma-cells = <3>; // dma-cells <request channel id, handshake number, dma sel bit>;
+			clocks = <&d0_clock EIC7700_CLK_AONDMA_ACLK>,
+				 <&d0_clock EIC7700_CLK_AONDMA_CFG>;
+			clock-names = "core-clk", "cfgr-clk";
+			resets = <&d0_reset DMA1_RST_CTRL SW_DMA1_ARSTN>,
+				<&d0_reset DMA1_RST_CTRL SW_DMA1_HRSTN>;
+			reset-names = "arst", "prst";
+			dma-channels = <16>;
+			snps,dma-masters = <2>;
+			snps,priority = <0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15>;
+			snps,data-width = <3>;
+			snps,block-size = <0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000>;
+			snps,axi-max-burst-len = <32>;
+			snps,max-msize = <64>;
+			#size-cells = <2>;
+			#address-cells = <2>;
+			dma-ranges = <0x0 0x80000000 0x0 0x80000000 0x100 0x0>;
+			iommus = <&smmu0 WIN2030_SID_DMA1>;
+			tbus = <WIN2030_TBUID_DMA1>;
+			eswin,syscfg = <&d0_sys_con DMA1_SID_REG_OFFSET 0x370>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		d0_dmac0: dma-controller-hsp@0x50430000 {
+			compatible = "eswin,eic770x-axi-dma";
+			reg = <0x0 0x50430000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <57>;
+			#dma-cells = <3>; // dma-cells <request channel id, handshake number, dma sel bit>;
+			clocks = <&d0_clock EIC7700_CLK_HSP_DMA0_CLK>,
+					 <&d0_clock EIC7700_CLK_HSP_DMA0_CLK_TEST>;
+			clock-names = "core-clk", "cfgr-clk";
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_DMA0_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_DMA_PRSTN>;
+			reset-names = "arst", "prst";
+			dma-channels = <12>;
+			snps,dma-masters = <1>;
+			snps,priority = <0 1 2 3 4 5 6 7 8 9 10 11>;
+			snps,data-width = <2>;
+			snps,block-size = <0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000>;
+			snps,axi-max-burst-len = <16>;
+			snps,max-msize = <64>;
+			iommus = <&smmu0 WIN2030_SID_DMA0>;
+			tbus = <WIN2030_TBUID_DMA0>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x104c>;
+			eswin,syscfg = <&d0_sys_con DMA1_SID_REG_OFFSET 0x370>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		d0_gmac0: ethernet@50400000 {
+			compatible = "eswin,win2030-qos-eth";
+			reg = <0x0 0x50400000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupt-names = "macirq";
+			interrupts = <61>;
+			phy-mode = "rgmii-txid";
+			numa-node-id = <0>;
+			id = <0>;
+			status = "disabled";
+			clocks = <&d0_clock EIC7700_CLK_HSP_ETH_APP_CLK>,
+					 <&d0_clock EIC7700_CLK_HSP_ETH_CSR_CLK>,
+					 <&d0_clock EIC7700_CLK_HSP_ETH0_CORE_CLK>;
+			clock-names = "app", "stmmaceth","tx";
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_ETH0_ARSTN>;
+			reset-names = "ethrst";
+			//iommus = <&smmu0 WIN2030_SID_ETH0>;
+			tbus = <WIN2030_TBUID_ETH>;
+			dma-noncoherent;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1030 0x100 0x108>;
+			eswin,syscrg_csr = <&d0_sys_crg 0x148 0x14c>;
+			eswin,dly_hsp_reg = <0x114 0x118 0x11c>;
+			snps,axi-config = <&d0_stmmac_axi_setup>;
+			d0_stmmac_axi_setup: stmmac-axi-config {
+				snps,blen = <0 0 0 0 16 8 4>;
+				snps,rd_osr_lmt = <2>;
+				snps,wr_osr_lmt = <2>;
+				snps,lpi_en = <0>;
+			};
+		};
+
+		d0_gmac1: ethernet@50410000 {
+			compatible = "eswin,win2030-qos-eth";
+			reg = <0x0 0x50410000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupt-names = "macirq";
+			interrupts = <70>;
+			phy-mode = "rgmii-txid";
+			numa-node-id = <0>;
+			id = <1>;
+			status = "disabled";
+			clocks = <&d0_clock EIC7700_CLK_HSP_ETH_APP_CLK>,
+					 <&d0_clock EIC7700_CLK_HSP_ETH_CSR_CLK>,
+					 <&d0_clock EIC7700_CLK_HSP_ETH1_CORE_CLK>;
+			clock-names = "app", "stmmaceth","tx";
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_ETH1_ARSTN>;
+			reset-names = "ethrst";
+			//iommus = <&smmu0 WIN2030_SID_ETH1>;
+			tbus = <WIN2030_TBUID_ETH>;
+			dma-noncoherent;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1034 0x200 0x208>;
+			eswin,syscrg_csr = <&d0_sys_crg 0x148 0x14c>;
+			eswin,dly_hsp_reg = <0x214 0x218 0x21c>;
+			snps,axi-config = <&d0_stmmac_axi_setup_gmac1>;
+			d0_stmmac_axi_setup_gmac1: stmmac-axi-config {
+				snps,blen = <0 0 0 0 16 8 4>;
+				snps,rd_osr_lmt = <2>;
+				snps,wr_osr_lmt = <2>;
+				snps,lpi_en = <0>;
+			};
+		};
+		noc {
+			compatible = "eswin,noc","simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			numa-node-id = <0>;
+			#include "eswin-win2030-die0-noc.dtsi"
+		};
+
+		d0_npu: eswin-npu@51c00000 {
+			compatible = "eswin,npu";
+			reg = <0x0 0x51c00000 0x0 0x400000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <387 16>;
+			/*spram-region = <&npu0_reserved>;*/
+			#size-cells = <2>;
+			dma-ranges = <0x1 0x0 0x0 0xc0000000 0x1ff 0x0>;
+			iommus = <&smmu0 WIN2030_SID_NPU_DMA>;
+			tbus = <WIN2030_TBUID_NPU>;
+			dsp-avail-num = <1>;
+			spram-size = <0x400000>;
+			npu_mbox = <&d0_mbox2>;
+			clocks = <&d0_clock EIC7700_CLK_NPU_ACLK>,
+					<&d0_clock EIC7700_CLK_NPU_CFG_CLK>,
+					<&d0_clock EIC7700_CLK_NPU_CLK>,
+					<&d0_clock EIC7700_CLK_NPU_LLC_ACLK>,
+					<&d0_clock EIC7700_CLK_NPU_E31_CLK>,
+					<&d0_clock EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE>,
+					<&d0_clock EIC7700_MUX_U_NPU_LLCLK_3MUX1_GFREE>,
+					<&d0_clock EIC7700_SPLL2_FOUT2>,
+					<&d0_clock EIC7700_SPLL1_FOUT1>,
+					<&d0_clock EIC7700_CLK_CLK_NPU_LLC_SRC0>,
+					<&d0_clock EIC7700_VPLL_FOUT1>;
+
+			clock-names = 	"aclk", "cfg_clk", "core_clk", "clk_npu_llc_aclk", "e31_core_clk",
+					"mux_u_npu_core_3mux1_gfree", "mux_u_npu_llclk_3mux1_gfree",
+					"fixed_rate_clk_spll2_fout2", "fixed_rate_clk_spll1_fout1", "clk_clk_npu_llc_src0",
+					"fixed_rate_clk_vpll_fout1";
+
+			resets = <&d0_reset NPU_RST_CTRL SW_NPU_E31CORE_RSTN>;
+			reset-names = "e31_core";
+			operating-points-v2 = <&npu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <16000>; /*TBD*/
+			numa-node-id = <0>;
+			firmware-name = "eic7700_die0_e31_fw";
+			dma-noncoherent;
+		};
+
+		dev_llc_d0: llc@51c00000 {
+			compatible = "eswin,llc";
+			reg = <0x0 0x51c00000 0x0 0x400000>;
+			eswin,syscfg = <&d0_sys_con 0x324>;
+			eswin,syscrg_csr = <&d0_sys_crg>;
+			clocks = <&d0_clock EIC7700_CLK_NPU_ACLK>,
+				 <&d0_clock EIC7700_CLK_NPU_CFG_CLK>,
+				 <&d0_clock EIC7700_CLK_NPU_LLC_ACLK>,
+				 <&d0_clock EIC7700_CLK_NPU_CLK>,
+				 <&d0_clock EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE>,
+				 <&d0_clock EIC7700_SPLL2_FOUT2>,
+				 <&d0_clock EIC7700_SPLL1_FOUT1>;
+			clock-names = "aclk", "cfg_clk", "llc_clk", "core_clk",
+					"mux_u_npu_core_3mux1_gfree", "fixed_rate_clk_spll2_fout2",
+					"fixed_rate_clk_spll1_fout1";
+			resets = <&d0_reset NPU_RST_CTRL SW_NPU_AXI_RSTN>,
+				<&d0_reset NPU_RST_CTRL SW_NPU_CFG_RSTN>,
+				<&d0_reset NPU_RST_CTRL SW_NPU_CORE_RSTN>,
+				<&d0_reset NPU_RST_CTRL SW_NPU_LLC_RSTN>;
+			reset-names = "axi", "cfg", "core", "llc";
+			numa-node-id = <0>;
+			spram-region = <&npu0_reserved>;
+		};
+
+		d0_dsp_subsys:dsp_subsys@52280400 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x52280400 0x0 0x10000>,
+					<0x0 0x51810000 0x0 0x8000>;
+			ranges;
+			numa-node-id = <0>;
+			dma-ranges = <0x0 0x30000000 0x0 0xc0000000 0x0 0xce000000>;
+			compatible = "es-dsp-subsys", "simple-bus";
+			clocks = <&d0_clock EIC7700_CLK_DSPT_CFG_CLK>, <&d0_clock EIC7700_CLK_DSPT_ACLK>;
+			clock-names = "cfg_clk", "aclk";
+			resets = <&d0_reset DSP_RST_CTRL SW_DSP_AXI_RSTN>,
+					<&d0_reset DSP_RST_CTRL SW_DSP_CFG_RSTN>,
+					<&d0_reset DSP_RST_CTRL SW_DSP_DIV4_RSTN>,
+					<&d0_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_0>,
+					<&d0_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_1>,
+					<&d0_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_2>,
+					<&d0_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_3>;
+			reset-names = "axi", "cfg", "div4", "div_0", "div_1", "div_2","div_3";
+			d0_dsp0:es_dsp@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x5b000000 0x8000
+					0x28100000 0 0x5b100000 0x20000
+					0x28120000 0 0x5b120000 0x20000>;
+				clocks = <&d0_clock EIC7700_CLK_DSP_ACLK_0>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d0_mbox4>;
+				device-irq = <11
+					ESWIN_MAILBOX_DSP_0_TO_U84_REG_BASE
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_0
+					ESWIN_MAILBOX_U84_TO_DSP_0_REG_BASE
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE>;
+
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <0>;
+				iommus = <&smmu0 WIN2030_SID_DSP_0>;
+				tbus = <WIN2030_TBUID_DSP0>;
+				dma-noncoherent;
+				numa-node-id = <0>;
+				aux-e31-dtim = <0x5a110000>;
+				dsp@0 {
+				};
+			};
+			d0_dsp1:es_dsp@1 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x5b008000 0x8000
+					0x28100000 0 0x5b140000 0x20000
+					0x28120000 0 0x5b160000 0x20000>;
+				clocks = <&d0_clock EIC7700_CLK_DSP_ACLK_1>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d0_mbox5>;
+				device-irq = <13
+					ESWIN_MAILBOX_DSP_1_TO_U84_REG_BASE
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_1
+					ESWIN_MAILBOX_U84_TO_DSP_1_REG_BASE
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <1>;
+				iommus = <&smmu0 WIN2030_SID_DSP_1>;
+				tbus = <WIN2030_TBUID_DSP1>;
+				dma-noncoherent;
+				numa-node-id = <0>;
+				aux-e31-dtim = <0x5a110000>;
+				dsp@0 {
+				};
+			};
+			d0_dsp2:es_dsp@2 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x5b010000 0x8000
+					0x28100000 0 0x5b180000 0x20000
+					0x28120000 0 0x5b1a0000 0x20000>;
+				clocks = <&d0_clock EIC7700_CLK_DSP_ACLK_2>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d0_mbox6>;
+				device-irq = <15
+					ESWIN_MAILBOX_DSP_2_TO_U84_REG_BASE
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_2
+					ESWIN_MAILBOX_U84_TO_DSP_2_REG_BASE
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <2>;
+				iommus = <&smmu0 WIN2030_SID_DSP_2>;
+				tbus = <WIN2030_TBUID_DSP2>;
+				dma-noncoherent;
+				numa-node-id = <0>;
+				aux-e31-dtim = <0x5a110000>;
+				dsp@0 {
+				};
+			};
+			d0_dsp3:es_dsp@3 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x5b018000 0x8000
+					0x28100000 0 0x5b1c0000 0x20000
+					0x28120000 0 0x5b1e0000 0x20000>;
+				clocks = <&d0_clock EIC7700_CLK_DSP_ACLK_3>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d0_mbox7>;
+				device-irq = <17
+					ESWIN_MAILBOX_DSP_3_TO_U84_REG_BASE
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_3
+					ESWIN_MAILBOX_U84_TO_DSP_3_REG_BASE
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <3>;
+				iommus = <&smmu0 WIN2030_SID_DSP_3>;
+				tbus = <WIN2030_TBUID_DSP3>;
+				dma-noncoherent;
+				numa-node-id = <0>;
+				aux-e31-dtim = <0x5a110000>;
+				dsp@0 {
+				};
+			};
+			d0_sofdsp3: sofdsp@4 {
+				#sound-dai-cells = <1>;
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,sof-dsp";
+				reg = <0x0 0x5b018000 0x0 0x8000>,
+					  <0x0 0x5b1c0000 0x0 0x40000>;
+				mbox-names = "dsp-mbox";
+				mboxes = <&d0_mbox7 0>;
+				clocks = <&d0_clock EIC7700_CLK_DSP_ACLK_3>;
+				clock-names = "aclk";
+				process-id = <3>;
+				iommus = <&smmu0 WIN2030_SID_DSP_3>;
+				tbus = <WIN2030_TBUID_DSP3>;
+				dma-noncoherent;
+				mailbox-dsp-to-u84-addr = <ESWIN_MAILBOX_DSP_3_TO_U84_REG_BASE>;
+				mailbox-u84-to-dsp-addr = <ESWIN_MAILBOX_U84_TO_DSP_3_REG_BASE>;
+				dsp-uart = <&d0_uart0>;
+				device-uart-mutex = <0x51820000>;
+				numa-node-id = <0>;
+			};
+		};
+
+		gc820: g2d@50140000 {
+			compatible = "eswin,galcore_d0";
+			clocks = <&d0_clock EIC7700_CLK_VC_ACLK>,
+					<&d0_clock EIC7700_CLK_VC_CFG_CLK>,
+					<&d0_clock EIC7700_CLK_G2D_CFG_CLK>,
+					<&d0_clock EIC7700_CLK_CLK_G2D_ST2>,
+					<&d0_clock EIC7700_CLK_G2D_CLK>,
+					<&d0_clock EIC7700_CLK_G2D_ACLK>,
+					<&d0_clock EIC7700_CLK_VC_MON_PCLK>;
+			clock-names = "vc_aclk", "vc_cfg", "g2d_cfg", "g2d_st2", "g2d_clk", "g2d_aclk", "mon_pclk";
+			resets = <&d0_reset VC_RST_CTRL SW_VC_AXI_RSTN>,
+					<&d0_reset VC_RST_CTRL SW_VC_CFG_RSTN>,
+					<&d0_reset VC_RST_CTRL SW_VC_MONCFG_RSTN>,
+					<&d0_reset G2D_RST_CTRL SW_G2D_CORE_RSTN>,
+					<&d0_reset G2D_RST_CTRL SW_G2D_CFG_RSTN>,
+					<&d0_reset G2D_RST_CTRL SW_G2D_AXI_RSTN>;
+			reset-names = "axi", "cfg", "moncfg", "g2d_core", "g2d_cfg", "g2d_axi";
+			reg = <0 0x50140000 0 0x40000>, <0 0x50180000 0 0x40000>;
+			reg-names = "core_2d", "core_2d1";
+			fe-apb-offset = <0x800>;
+			interrupt-parent = <&plic0>;
+			interrupts = <49>, <50>;
+			interrupt-names = "core_2d", "core_2d1";
+			enable-mmu = <1>;
+			contiguous-size = <0xa00000>;
+			recovery = <0>;
+			dma-noncoherent;
+			operating-points-v2 = <&g2d_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <1000>; /*TBD*/
+			numa-node-id = <0>;
+		};
+
+		gpu0: gpu@51400000 {
+			compatible = "img,gpu";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x51400000 0x0 0xFFFFF>;
+			clocks = <&d0_clock EIC7700_CLK_GPU_ACLK>,
+					<&d0_clock EIC7700_CLK_GPU_GRAY_CLK>,
+					<&d0_clock EIC7700_CLK_GPU_CFG_CLK>;
+			clock-names = "aclk", "gray_clk", "cfg_clk";
+			resets =<&d0_reset GPU_RST_CTRL SW_GPU_AXI_RSTN>,
+					<&d0_reset GPU_RST_CTRL SW_GPU_CFG_RSTN>,
+					<&d0_reset GPU_RST_CTRL SW_GPU_GRAY_RSTN>,
+					<&d0_reset GPU_RST_CTRL SW_GPU_JONES_RSTN>,
+					<&d0_reset GPU_RST_CTRL SW_GPU_SPU_RSTN>;
+			reset-names = "axi", "cfg", "gray", "jones","spu";
+			interrupt-parent = <&plic0>;
+			interrupts = <15>;
+			dma-noncoherent;
+			numa-node-id = <0>;
+
+			operating-points-v2 = <&gpu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <4000>; /*TBD*/
+		};
+
+		d0_sata: sata@0x50420000{
+			compatible = "snps,eswin-ahci";
+			reg = <0x0 0x50420000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupt-names = "intrq", "msi", "pme";
+			interrupts = <58>, <59>, <60>;
+			ports-implemented = <0x1>;
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_SATA_ARSTN>;
+			reset-names = "apb";
+			#size-cells = <2>;
+			iommus = <&smmu0 WIN2030_SID_SATA>;
+			tbus = <WIN2030_TBUID_SATA>;
+			dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1050>;
+			eswin,syscrg_csr = <&d0_sys_crg 0x41c>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		pcie: pcie@0x54000000 {
+			compatible = "eswin,win2030-pcie";
+			clocks = <&d0_clock EIC7700_CLK_PCIET_ACLK>,
+				 <&d0_clock EIC7700_CLK_PCIET_CFG_CLK>,
+				 <&d0_clock EIC7700_CLK_PCIET_CR_CLK>,
+				 <&d0_clock EIC7700_CLK_PCIET_AUX_CLK>;
+			clock-names = "pcie_aclk", "pcie_cfg_clk", "pcie_cr_clk", "pcie_aux_clk";
+
+			reset-names = "pcie_cfg", "pcie_powerup", "pcie_pwren";
+			resets = <&d0_reset PCIE_RST_CTRL SW_PCIE_CFG_RSTN>,
+				 <&d0_reset PCIE_RST_CTRL SW_PCIE_POWERUP_RSTN>,
+				 <&d0_reset PCIE_RST_CTRL SW_PCIE_PERST_N>;
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			#interrupt-cells = <1>;
+			reg = <0x0 0x54000000 0x0 0x4000000>,	/* IP registers */
+			      <0x0 0x40000000 0x0 0x800000>,	/* Configuration space */
+			      <0x0 0x50000000 0x0 0x100000>;
+			reg-names = "dbi", "config", "mgmt";
+			device_type = "pci";
+			/* dma-coherent; */
+			bus-range = <0x0 0xff>;
+
+			ranges = <0x81000000  0x0 0x40800000  0x0 0x40800000 0x0 0x800000>,      /* I/O */
+				 <0x82000000  0x0 0x41000000  0x0 0x41000000 0x0 0xf000000>,    /* mem */
+				 <0xc3000000 0x80 0x00000000 0x80 0x00000000 0x2 0x00000000>;  /* mem prefetchable */
+
+			num-lanes = <0x4>;
+			/**********************************
+			  	msi_ctrl_io[0~31] : 188~219
+			  	msi_ctrl_int : 220
+			**********************************/
+			interrupts = <220>, <179>, <180>, <181>, <182>, <183>, <184>, <185>, <186>;
+			interrupt-names = "msi", "inta", "intb", "intc", "intd";
+			interrupt-parent = <&plic0>;
+			interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+			interrupt-map = <0x0 0x0 0x0 0x1 &plic0 179>,
+							<0x0 0x0 0x0 0x2 &plic0 180>,
+							<0x0 0x0 0x0 0x3 &plic0 181>,
+							<0x0 0x0 0x0 0x4 &plic0 182>;
+			iommus = <&smmu0 0xfe0000>;
+			iommu-map = <0x0 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "disabled";
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		ssi0: spi@50810000 {
+			compatible = "snps,eic770x-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x50810000 0x0 0x4000>;
+			spi-max-frequency = <4800000>;
+			clocks = <&d0_clock EIC7700_CLK_LSP_SSI0_PCLK>;
+			clock-names = "clk";
+			interrupt-parent = <&plic0>;
+			interrupts = <91>;
+			resets = <&d0_reset SSI_RST_CTRL SW_SSI_RST_N_0>;
+			reset-names = "spi";
+			numa-node-id = <0>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		ssi1: spi@50814000 {
+			compatible = "snps,eic770x-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x50814000 0x0 0x4000>;
+			spi-max-frequency = <4800000>;
+			clocks = <&d0_clock EIC7700_CLK_LSP_SSI1_PCLK>;
+			clock-names = "clk";
+			interrupt-parent = <&plic0>;
+			interrupts = <92>;
+			resets = <&d0_reset SSI_RST_CTRL SW_SSI_RST_N_1>;
+			reset-names = "spi";
+			numa-node-id = <0>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		bootspi: spi@51800000 {
+			compatible = "eswin,bootspi";
+			reg = <0x0 0x51800000 0x0 0x8000>,
+				<0x0 0x51828000 0x0 0x8000>,
+				<0x0 0x5c000000 0x0 0x8000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&d0_clock EIC7700_CLK_CLK_BOOTSPI_CFG>,
+					<&d0_clock EIC7700_CLK_CLK_BOOTSPI>;
+			clock-names = "cfg_clk", "clk";
+			resets = <&d0_reset BOOTSPI_RST_CTRL SW_BOOTSPI_RSTN>;
+			reset-names = "rst";
+			spi-max-frequency = <4800000>;
+			reg-io-width = <4>;
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+		sdhci_emmc: mmc@50450000 {
+			compatible = "eswin,emmc-sdhci-5.1";
+			reg = <0x0 0x50450000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <79>;
+			assigned-clocks = <&d0_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>;
+			assigned-clock-rates = <200000000>;
+			clocks = <&d0_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>, <&d0_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "clk_xin", "clk_ahb";
+			clock-output-names = "emmc_cardclock";
+			#clock-cells = <0>;
+
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_MSHC0_TXRX_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_MSHC0_PHY_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_EMMC_PRSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_EMMC_ARSTN>;
+			reset-names = "txrx_rst", "phy_rst", "prstn", "arstn";
+
+			core-clk-reg = <0x51828160>;
+			disable-cqe-dcmd;
+			bus-width = <8>;
+			non-removable;
+			/*mmc-ddr-1_8v;*/
+			mmc-hs400-1_8v;
+			max-frequency = <200000000>;
+			/* sdhci-caps-mask =  <0x0 0x3200000>; */
+			/* sdhci-caps-mask =  <0x2 0x3000000>; */
+			/* smmu */
+			#size-cells = <2>;
+			iommus = <&smmu0 WIN2030_SID_EMMC0>;
+			tbus = <WIN2030_TBUID_EMMC>;
+			dma-ranges = <0x0 0x00000000 0x0 0xc0000000 0x1 0x0>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1038 0x508 0x50c>;
+			eswin,syscrg_csr = <&d0_sys_crg 0x160 0x148 0x14c>;
+			status = "disabled";
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		sdio0: mmc@0x50460000{
+			compatible = "eswin,sdhci-sdio";
+			reg = <0x0 0x50460000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <81>;
+			clocks =<&d0_clock EIC7700_CLK_HSP_MSHC1_CORE_CLK>,
+				<&d0_clock EIC7700_CLK_HSP_CFG_CLK>,
+				<&d0_clock EIC7700_SPLL2_FOUT3>,
+				<&d0_clock EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1>;
+			clock-names ="clk_xin","clk_ahb","clk_spll2_fout3","clk_mux1_1";
+			clock-output-names = "sdio0_cardclock";
+			#clock-cells = <0>;
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_MSHC1_TXRX_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_MSHC1_PHY_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD0_PRSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD0_ARSTN>;
+			reset-names = "txrx_rst","phy_rst","prstn","arstn";
+
+			clock-frequency = <208000000>;
+			max-frequency = <208000000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+			iommus = <&smmu0 WIN2030_SID_SD0>;
+			tbus = <WIN2030_TBUID_SD>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x103c 0x608 0x60c>;
+			eswin,syscrg_csr = <&d0_sys_crg 0x164 0x148 0x14c>;
+			bus-width = <4>;
+			sdio-id = <0>;
+			numa-node-id = <0>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		sdio1: mmc@0x50470000{
+			compatible = "eswin,sdhci-sdio";
+			reg = <0x0 0x50470000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <83>;
+			clocks =<&d0_clock EIC7700_CLK_HSP_MSHC2_CORE_CLK>,
+				<&d0_clock EIC7700_CLK_HSP_CFG_CLK>,
+				<&d0_clock EIC7700_SPLL2_FOUT3>,
+				<&d0_clock EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1>;
+			clock-names ="clk_xin","clk_ahb","clk_spll2_fout3","clk_mux1_1";
+			clock-output-names = "sdio1_cardclock";
+			#clock-cells = <0>;
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_MSHC2_TXRX_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_MSHC2_PHY_RSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD1_PRSTN>,
+				<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD1_ARSTN>;
+			reset-names = "txrx_rst","phy_rst","prstn","arstn";
+
+			clock-frequency = <208000000>;
+			max-frequency = <208000000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+			iommus = <&smmu0 WIN2030_SID_SD1>;
+			tbus = <WIN2030_TBUID_SD>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1040 0x708 0x70c>;
+			eswin,syscrg_csr = <&d0_sys_crg 0x168 0x148 0x14c>;
+			bus-width = <4>;
+			sdio-id = <1>;
+			numa-node-id = <0>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		vdec0: video-decoder0@50100000 {
+			compatible = "eswin,video-decoder0";
+			clocks = <&d0_clock EIC7700_CLK_VC_ACLK>,
+					 <&d0_clock EIC7700_CLK_VC_CFG_CLK>,
+					 <&d0_clock EIC7700_CLK_VC_JD_CLK>,
+					 <&d0_clock EIC7700_CLK_VC_VD_CLK>,
+					 <&d0_clock EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE>,
+					 <&d0_clock EIC7700_SPLL0_FOUT1>,
+					 <&d0_clock EIC7700_SPLL2_FOUT1>,
+					 <&d0_clock EIC7700_CLK_VC_JD_PCLK>,
+					 <&d0_clock EIC7700_CLK_VC_VD_PCLK>,
+					 <&d0_clock EIC7700_CLK_VC_MON_PCLK>;
+			clock-names = "aclk", "cfg_clk", "jd_clk", "vd_clk", "vc_mux", "spll0_fout1", "spll2_fout1", "jd_pclk", "vd_pclk", "mon_pclk";
+			resets = <&d0_reset VC_RST_CTRL SW_VC_AXI_RSTN>,
+					 <&d0_reset VC_RST_CTRL SW_VC_CFG_RSTN>,
+					 <&d0_reset VC_RST_CTRL SW_VC_MONCFG_RSTN>,
+					 <&d0_reset JD_RST_CTRL SW_JD_CFG_RSTN>,
+					 <&d0_reset JD_RST_CTRL SW_JD_AXI_RSTN>,
+					 <&d0_reset VD_RST_CTRL SW_VD_CFG_RSTN>,
+					 <&d0_reset VD_RST_CTRL SW_VD_AXI_RSTN>;
+			reset-names = "axi", "cfg", "moncfg", "jd_cfg", "jd_axi", "vd_cfg", "vd_axi";
+			eswin,syscfg = <&d0_sys_con 0x0 0x4>;
+			operating-points-v2 = <&vdec_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <1000>; /*TBD*/
+			vcmd-core = <0 0x6c>;
+			axife-core = <0x200 0x100>;
+			vdec-core = <0x800 0xc00>;
+			interrupt-parent = <&plic0>;
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x0 0x0 0x80000000 0x200 0x0>;
+			iommus = <&smmu0 WIN2030_SID_VDEC>;
+			vccsr-reg = <0x0 0x501c0000 0x0 0x1000>;
+			numa-node-id = <0>;
+			tbus = <WIN2030_TBUID_VDEC>, <WIN2030_TBUID_JDEC>;
+			dma-noncoherent;
+
+			vdec_0: vdec0@50100000 {
+				core-name = "video-dec0";
+				base-addr = <0x50100000>;
+				interrupts = <236>;
+			};
+
+			jdec_0: jdec0@50120000 {
+				core-name = "jpeg-dec0";
+				base-addr = <0x50120000>;
+				interrupts = <237>;
+			};
+		};
+
+		venc0: video-encoder@50110000 {
+			compatible = "eswin,video-encoder0";
+			clocks = <&d0_clock EIC7700_CLK_VC_ACLK>,
+					 <&d0_clock EIC7700_CLK_VC_CFG_CLK>,
+					 <&d0_clock EIC7700_CLK_VC_JE_CLK>,
+					 <&d0_clock EIC7700_CLK_VC_VE_CLK>,
+					 <&d0_clock EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE>,
+					 <&d0_clock EIC7700_SPLL0_FOUT1>,
+					 <&d0_clock EIC7700_SPLL2_FOUT1>,
+					 <&d0_clock EIC7700_CLK_VC_JE_PCLK>,
+					 <&d0_clock EIC7700_CLK_VC_VE_PCLK>,
+					 <&d0_clock EIC7700_CLK_VC_MON_PCLK>;
+			clock-names = "aclk", "cfg_clk", "je_clk", "ve_clk", "vc_mux", "spll0_fout1", "spll2_fout1", "je_pclk", "ve_pclk", "mon_pclk";
+			resets = <&d0_reset VC_RST_CTRL SW_VC_AXI_RSTN>,
+					 <&d0_reset VC_RST_CTRL SW_VC_CFG_RSTN>,
+					 <&d0_reset VC_RST_CTRL SW_VC_MONCFG_RSTN>,
+					 <&d0_reset JE_RST_CTRL SW_JE_CFG_RSTN>,
+					 <&d0_reset JE_RST_CTRL SW_JE_AXI_RSTN>,
+					 <&d0_reset VE_RST_CTRL SW_VE_CFG_RSTN>,
+					 <&d0_reset VE_RST_CTRL SW_VE_AXI_RSTN>;
+			reset-names = "axi", "cfg", "moncfg", "je_cfg", "je_axi", "ve_cfg", "ve_axi";
+			eswin,syscfg = <&d0_sys_con 0x0 0x4>;
+			operating-points-v2 = <&venc_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <1000>; /*TBD*/
+			vcmd-core = <0 0x6c>;
+			axife-core = <0x2000 0x7d0>;
+			venc-core = <0x1000 0x87c>;
+			interrupt-parent = <&plic0>;
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x0 0x0 0x80000000 0x200 0x0>;
+			iommus = <&smmu0 WIN2030_SID_VENC>;
+			vccsr-reg = <0x0 0x501c0000 0x0 0x1000>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+
+			venc_0: venc0@50110000 {
+				core-name = "video-enc0";
+				base-addr = <0x50110000>;
+				interrupts = <229>;
+				tbus = <WIN2030_TBUID_VENC>;
+			};
+
+			jenc_0: jenc0@50130000 {
+				core-name = "jpeg-enc0";
+				base-addr = <0x50130000>;
+				interrupts = <232>;
+				tbus = <WIN2030_TBUID_JENC>;
+			};
+		};
+
+		/*mailbox between u84 & scpu*/
+		d0_mbox0: mbox@50a00000 {
+			compatible = "eswin,win2030-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_SCPU_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_SCPU_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <117>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_0>,
+				<&d0_clock EIC7700_CLK_MAILBOX_1>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_0>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_1>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit = <ESWIN_MAIBOX_SCPU_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & lpcpu*/
+		d0_mbox1: mbox@50a20000 {
+			compatible = "eswin,win2030-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_LPCPU_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_LPCPU_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_2>,
+				<&d0_clock EIC7700_CLK_MAILBOX_3>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_2>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_3>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_LPCPU_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & npu_0*/
+		d0_mbox2: mbox@50a40000 {
+			compatible = "eswin,npu0-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_NPU_0_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <121>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_4>,
+				<&d0_clock EIC7700_CLK_MAILBOX_5>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_4>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_5>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_NPU_0_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & npu_1*/
+		d0_mbox3: mbox@50a60000 {
+			compatible = "eswin,win2030-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_NPU_1_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_NP1_0_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <123>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_6>,
+				<&d0_clock EIC7700_CLK_MAILBOX_7>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_6>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_7>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_NPU_1_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & dsp_0*/
+		d0_mbox4: mbox@50a80000 {
+			compatible = "eswin,dsp0-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_DSP_0_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_DSP_0_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <125>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_8>,
+				<&d0_clock EIC7700_CLK_MAILBOX_9>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_8>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_9>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_0_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & dsp_1*/
+		d0_mbox5: mbox@50aa0000 {
+			compatible = "eswin,dsp1-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_DSP_1_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_DSP_1_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <127>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_10>,
+				<&d0_clock EIC7700_CLK_MAILBOX_11>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_10>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_11>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_1_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & dsp_2*/
+		d0_mbox6: mbox@50ac0000 {
+			compatible = "eswin,dsp2-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_DSP_2_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_DSP_2_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <129>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_12>,
+				<&d0_clock EIC7700_CLK_MAILBOX_13>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_12>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_13>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_2_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		/*mailbox between u84 & dsp_3*/
+		d0_mbox7: mbox@50ae0000 {
+			compatible = "eswin,dsp3-mailbox";
+			reg = <0 ESWIN_MAILBOX_U84_TO_DSP_3_REG_BASE 0 0x10000>,
+				  <0 ESWIN_MAILBOX_DSP_3_TO_U84_REG_BASE 0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <131>;
+			#mbox-cells = <1>;
+			clocks = <&d0_clock EIC7700_CLK_MAILBOX_14>,
+				<&d0_clock EIC7700_CLK_MAILBOX_15>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_14>,
+				<&d0_reset MBOX_RST_CTRL SW_MBOX_RST_N_15>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_3_IRQ_BIT>;
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		d0_ipc_scpu:ipc@0 {
+			compatible = "eswin,win2030-ipc";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x80000000 0x0 0xc0000000 0x0 0x80000000>;
+			iommus = <&smmu0 WIN2030_SID_SCPU>, <&smmu0 WIN2030_SID_CRYPT>;
+			tbus = <WIN2030_TBUID_SCPU>;
+			eswin,syscfg = <&d0_sys_con SCPU_SID_REG_OFFSET 0>,
+			               <&d0_sys_con CRYPT_SID_REG_OFFSET 0>;
+			mboxes = <&d0_mbox0 0>;
+			mbox-names = "u84_scpu";
+			numa-node-id = <0>;
+			dma-noncoherent;
+		};
+
+		d0_lpcpu:lpcpu@0 {
+			compatible = "eswin,win2030-lpcpu";
+			/* fw-region = <&lpcpu0_reserved>; */
+			clocks = <&d0_clock EIC7700_CLK_CLK_LPCPU_CORE>,
+					<&d0_clock EIC7700_CLK_CLK_LPCPU_BUS>;
+			clock-names = "core_clk", "bus_clk";
+
+			reset-names = "core_rst", "bus_rst", "dbg_rst";
+			resets = <&d0_reset LPCPU_RST_CTRL SW_LPCPU_CORE_RSTN>,
+					<&d0_reset LPCPU_RST_CTRL SW_LPCPU_BUS_RSTN>,
+					<&d0_reset LPCPU_RST_CTRL SW_LPCPU_DBG_RSTN>;
+			#size-cells = <2>;
+
+			dma-ranges = <0x0 0xb0000000 0x0 0xc0000000 0x0 0x50000000>;
+			iommus = <&smmu0 WIN2030_SID_LCPU>;
+			eswin,syscfg = <&d0_sys_con LCPU_SID_REG_OFFSET 0>;
+			tbus = <WIN2030_TBUID_LPCPU>;
+			mboxes = <&d0_mbox1 0>;
+			mbox-names = "u84_lpcpu";
+			numa-node-id = <0>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		pvt0: pvt@0x50b00000 {
+			compatible = "eswin,eswin-pvt-cpu";
+			clocks = <&d0_clock EIC7700_CLK_PVT_CLK_0>;
+			clock-names = "pvt_clk";
+			resets = <&d0_reset PVT_RST_CTRL SW_PVT_RST_N_0>;
+			reset-names = "pvt_rst";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x50b00000 0x0 0x10000>;
+			interrupts = <349>;
+			interrupt-parent = <&plic0>;
+			#thermal-sensor-cells = <0>;
+			status = "disabled";
+			label = "pvt0";
+			numa-node-id = <0>;
+		};
+		pvt1: pvt@0x52360000 {
+			compatible = "eswin,eswin-pvt-ddr";
+			clocks = <&d0_clock EIC7700_CLK_PVT_CLK_1>;
+			clock-names = "pvt_clk";
+			resets = <&d0_reset PVT_RST_CTRL SW_PVT_RST_N_1>;
+			reset-names = "pvt_rst";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x52360000 0x0 0x20000>;
+			interrupts = <350>;
+			interrupt-parent = <&plic0>;
+			#thermal-sensor-cells = <0>;
+			status = "disabled";
+			label = "pvt1";
+			numa-node-id = <0>;
+		};
+
+		fan_control: fan_control@50b50000 {
+			compatible = "eswin-fan-control";
+			reg = <0x0 0x50b50000 0x0 0x10000>,
+					<0x0 0x51810668 0x0 0x10>,
+					<0x0 0x71810668 0x0 0x10>;
+			clocks = <&d0_clock EIC7700_CLK_LSP_FAN_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset FAN_RST_CTRL SW_FAN_RST_N>;
+			reset-names = "fan_rst";
+			interrupt-parent = <&plic0>;
+			interrupt-names = "fanirq";
+			interrupts = <354>;
+			pulses-per-revolution = <2>;
+			pwm-minimum-period = <1000>;
+			pwms = <&pwm0 0 100000 0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_die0_fan_tach_default &pinctrl_die0_pwm0_default>;
+			status = "disabled";
+			label = "fan_control";
+			numa-node-id = <0>;
+		};
+
+		d0_i2c0: i2c@50950000 {
+			compatible = "snps,designware-i2c";
+			clock-frequency = <I2C_BITRATE_STANDARD>;
+			clocks = <&d0_clock EIC7700_CLK_LSP_I2C0_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_0>;
+			reset-names = "rst";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x50950000 0x0 0x8000>;
+			interrupts = <105>;
+			interrupt-parent = <&plic0>;
+			numa-node-id = <0>;
+		};
+		d0_i2c1: i2c@50960000 {
+			compatible = "snps,designware-i2c";
+			clock-frequency = <I2C_BITRATE_STANDARD>;
+			clocks = <&d0_clock EIC7700_CLK_LSP_I2C1_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_1>;
+			reset-names = "rst";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x50960000 0x0 0x8000>;
+			interrupts = <106>;
+			interrupt-parent = <&plic0>;
+			numa-node-id = <0>;
+		};
+		d0_i2c2: i2c@50970000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C2_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_2>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x50970000 0x0 0x8000>;
+				interrupts = <107>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c3: i2c@50980000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C3_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_3>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x50980000 0x0 0x8000>;
+				interrupts = <108>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c4: i2c@50990000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C4_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_4>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x50990000 0x0 0x8000>;
+				interrupts = <109>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c5: i2c@509a0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C5_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_5>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x509a0000 0x0 0x8000>;
+				interrupts = <110>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c6: i2c@509b0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C6_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_6>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x509b0000 0x0 0x8000>;
+				interrupts = <111>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c7: i2c@509c0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C7_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_7>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x509c0000 0x0 0x8000>;
+				interrupts = <112>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c8: i2c@509d0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C8_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_8>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x509d0000 0x0 0x8000>;
+				interrupts = <113>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_i2c9: i2c@509e0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_LSP_I2C9_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_9>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x509e0000 0x0 0x8000>;
+				interrupts = <114>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		d0_aon_i2c0: i2c@51830000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_AON_I2C0_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C0_RST_CTRL SW_I2C0_PRSTN>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x51830000 0x0 0x8000>;
+				interrupts = <290>;
+				interrupt-parent = <&plic0>;
+				eswin,i2c_dma = <&d0_aon_dmac>;
+				dma-names = "rx", "tx";
+				/*
+				* dmas : DMA specifiers
+				* &d0_aon_dmac : dma controller
+				* 41 : i2c0 aon dma handshake number
+				* 0xff : no need to select to dma controller
+				*/
+				dmas = <&d0_aon_dmac 10 41 0xff>, <&d0_aon_dmac 11 42 0xff>;
+				numa-node-id = <0>;
+		};
+		d0_aon_i2c1: i2c@51838000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d0_clock EIC7700_CLK_AON_I2C1_PCLK>;
+				clock-names = "pclk";
+				resets = <&d0_reset I2C1_RST_CTRL SW_I2C1_PRSTN>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x51838000 0x0 0x8000>;
+				interrupts = <291>;
+				interrupt-parent = <&plic0>;
+				numa-node-id = <0>;
+		};
+		pinctrl: pinctrl@0x51600080 {
+			compatible = "eswin,eic7x-pinctrl";
+			reg = <0x0 0x51600080 0x0 0x1FFF80>;
+			numa-node-id = <0>;
+			status = "disabled";
+			pinctrl_die0_pwm0_default: pwm0-default{
+				mux {
+					groups = "pwm0_group";
+					function = "pwm0_func";
+				};
+			};
+            pinctrl_die0_fan_tach_default: fan_tach-default{
+				mux {
+					groups = "fan_tach_group";
+					function = "fan_tach_func";
+				};
+			};
+		};
+
+		gpio0: gpio@51600000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "snps,dw-apb-gpio";
+			reg = <0x0 0x51600000 0x0 0x80>;
+			numa-node-id = <0>;
+
+			porta: gpio-port@0 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <32>;
+				reg = <0>;
+				interrupt-parent = <&plic0>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
+								319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334>;
+			};
+
+			portb: gpio-port@1 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <32>;
+				reg = <1>;
+			};
+
+			portc: gpio-port@2 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <32>;
+				reg = <2>;
+			};
+
+			portd: gpio-port@3 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <16>;
+				reg = <3>;
+			};
+		};
+
+		pwm0: pwm@0x50818000 {
+			compatible = "eswin,pwm-eswin";
+			#pwm-cells = <3>;
+			reg = <0x0 0x50818000 0x0 0x4000>;
+			clock-names = "pclk";
+			clocks = <&d0_clock EIC7700_CLK_LSP_TIMER_PCLK>;
+			clock-frequency = <200000000>;
+			resets = <&d0_reset TIMER_RST_CTRL SW_TIMER_RST_N>;
+			reset-names = "rst";
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		wdt0: watchdog@0x50800000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x50800000 0x0 0x4000>;
+			clocks =<&d0_clock EIC7700_CLK_LSP_WDT0_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_0>;
+			reset-names = "rst";
+			interrupts = <87>;
+			interrupt-parent = <&plic0>;
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		wdt1: watchdog@0x50804000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x50804000 0x0 0x4000>;
+			clocks =<&d0_clock EIC7700_CLK_LSP_WDT1_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_1>;
+			reset-names = "rst";
+			interrupts = <88>;
+			interrupt-parent = <&plic0>;
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		wdt2: watchdog@0x50808000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x50808000 0x0 0x4000>;
+			clocks =<&d0_clock EIC7700_CLK_LSP_WDT2_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_2>;
+			reset-names = "rst";
+			interrupts = <89>;
+			interrupt-parent = <&plic0>;
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		wdt3: watchdog@0x5080c000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x5080c000 0x0 0x4000>;
+			clocks =<&d0_clock EIC7700_CLK_LSP_WDT3_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_3>;
+			reset-names = "rst";
+			interrupts = <90>;
+			interrupt-parent = <&plic0>;
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		timer0: timer@0x51840000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x51840000 0x0 0x8000>;
+			perf_count = <7>;
+			numa-node-id = <0>;
+			interrupt-parent = <&plic0>;
+			interrupts = <345>;
+			clock-names = "pclk","timer_aclk";
+			clocks = <&d0_clock EIC7700_CLK_TIMER_PCLK_0>,
+					 <&d0_clock EIC7700_CLK_TIMER_CLK_0>;
+			resets = <&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_0>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_1>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_2>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_3>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_4>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_5>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_6>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_7>,
+				<&d0_reset TIMER0_RST_CTRL SW_TIMER0_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		timer1: timer@0x51848000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x51848000 0x0 0x8000>;
+			numa-node-id = <0>;
+			interrupt-parent = <&plic0>;
+			interrupts = <346>;
+			clock-names = "pclk","timer_aclk";
+			clocks = <&d0_clock EIC7700_CLK_TIMER_PCLK_1>,
+					 <&d0_clock EIC7700_CLK_TIMER_CLK_1>;
+			resets = <&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_0>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_1>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_2>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_3>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_4>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_5>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_6>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_7>,
+				<&d0_reset TIMER1_RST_CTRL SW_TIMER1_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		timer2: timer@0x51850000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x51850000 0x0 0x8000>;
+			numa-node-id = <0>;
+			interrupt-parent = <&plic0>;
+			interrupts = <347>;
+			clock-names = "pclk","timer_aclk";
+			clocks = <&d0_clock EIC7700_CLK_TIMER_PCLK_2>,
+					 <&d0_clock EIC7700_CLK_TIMER_CLK_2>;
+			resets = <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_0>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_1>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_2>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_3>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_4>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_5>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_6>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_7>,
+			   <&d0_reset TIMER2_RST_CTRL SW_TIMER2_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		timer3: timer@0x51858000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x51858000 0x0 0x8000>;
+			numa-node-id = <0>;
+			interrupt-parent = <&plic0>;
+			interrupts = <348>;
+			clock-names = "pclk","timer_aclk","timer3_clk8";
+			clocks = <&d0_clock EIC7700_CLK_TIMER_PCLK_3>,
+					 <&d0_clock EIC7700_CLK_TIMER_CLK_3>,
+					 <&d0_clock EIC7700_CLK_TIMER3_CLK8>;
+			resets = <&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_0>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_1>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_2>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_3>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_4>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_5>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_6>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_7>,
+				<&d0_reset TIMER3_RST_CTRL SW_TIMER3_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		die0_rtc: rtc@51818000 {
+			compatible = "eswin,win2030-rtc";
+			reg = <0x0 0x51818000 0x0 0x400>;
+			eswin,syscfg = <&d0_sys_con 0x3c0>;
+			interrupt-parent = <&plic0>;
+			interrupts = <292>;
+			clocks = <&d0_clock EIC7700_CLK_CLK_RTC>;
+			clock-names = "rtcclk";
+			clock-frequency = <15625>;
+			resets = <&d0_reset RTC_RST_CTRL SW_RTC_RSTN>;
+			reset-names = "rtcrst";
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		d0_i2s0: i2s0@50200000 {
+			compatible = "snps,i2s";
+			clocks = <&d0_clock EIC7700_CLK_VO_I2S_MCLK>,
+					<&d0_clock EIC7700_APLL_FOUT1>;
+			clock-names = "mclk", "apll";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#sound-dai-cells = <0x00000000>;
+			reg = <0x0 0x50200000 0x0 0x10000>;
+			dma-names = "rx", "tx";
+			dmas = <&d0_aon_dmac 4 4 0>, <&d0_aon_dmac 5 5 0>;
+			vo_mclk_sel,syscrg = <&d0_sys_crg 0x1bc>;
+			resets = <&d0_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+				<&d0_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+				<&d0_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+			reset-names = "i2srst", "i2sprst", "voprst";
+			dma-noncoherent;
+			numa-node-id = <0>;
+		};
+
+		d0_i2s1: i2s1@50210000 {
+			compatible = "snps,i2s";
+			clocks = <&d0_clock EIC7700_CLK_VO_I2S_MCLK>,
+					<&d0_clock EIC7700_APLL_FOUT1>;
+			clock-names = "mclk", "apll";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#sound-dai-cells = <0x00000000>;
+			reg = <0x0 0x50210000 0x0 0x10000>;
+			dma-names = "rx", "tx";
+			dmas = <&d0_aon_dmac 6 2 1>, <&d0_aon_dmac 7 3 1>;
+			vo_mclk_sel,syscrg = <&d0_sys_crg 0x1bc>;
+			resets = <&d0_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+				<&d0_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+				<&d0_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+			reset-names = "i2srst", "i2sprst", "voprst";
+			dma-noncoherent;
+			numa-node-id = <0>;
+		};
+
+		d0_i2s2: i2s2@50220000 {
+			compatible = "snps,i2s";
+			clocks = <&d0_clock EIC7700_CLK_VO_I2S_MCLK>,
+					<&d0_clock EIC7700_APLL_FOUT1>;
+			clock-names = "mclk", "apll";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#sound-dai-cells = <0x00000000>;
+			reg = <0x0 0x50220000 0x0 0x10000>;
+			dma-names = "rx", "tx";
+			dmas = <&d0_aon_dmac 8 0 2>, <&d0_aon_dmac 9 1 2>;
+			vo_mclk_sel,syscrg = <&d0_sys_crg 0x1bc>;
+			resets = <&d0_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+				<&d0_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+				<&d0_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+			reset-names = "i2srst", "i2sprst", "voprst";
+			dma-noncoherent;
+			numa-node-id = <0>;
+		};
+
+		d0_soundcard: soundcard {
+			compatible = "simple-audio-card";
+			simple-audio-card,name = "Eswin sound card";
+			numa-node-id = <0>;
+		};
+
+		d0_graphcard0: graphcard0 {
+			compatible = "audio-graph-card";
+			numa-node-id = <0>;
+		};
+
+		d0_graphcard1: graphcard1 {
+			compatible = "audio-graph-card";
+			numa-node-id = <0>;
+		};
+
+		d0_graphcard2: graphcard2 {
+			compatible = "audio-graph-card";
+			numa-node-id = <0>;
+		};
+
+		video_output: display-subsystem@0 {
+			compatible = "eswin,display-subsystem";
+			numa-node-id = <0>;
+			ports = <&dc_out>;
+			dma-noncoherent;
+		};
+
+		dvb_widgets: dvb-subsystem {
+			compatible = "amlogic,dvb_widgets";
+			status = "disabled";
+			numa-node-id = <0>;
+		};
+
+		dc: display_control@502c0000 {
+			compatible = "eswin,dc";
+			reg = <0x0 0x502c0000 0x0 0x100>, <0x0 0x502c0180 0x0 0x700>, <0x0 0x502c1400 0x0 0x1400>;
+			interrupt-parent = <&plic0>;
+			interrupts = <238>;
+
+			clocks = <&d0_clock EIC7700_CLK_VO_CFG_CLK>,
+			         <&d0_clock EIC7700_CLK_VO_PIXEL_CLK>,
+			         <&d0_clock EIC7700_CLK_VO_ACLK>,
+			         <&d0_clock EIC7700_SPLL0_FOUT1>,
+			         <&d0_clock EIC7700_MUX_U_VO_ACLK_ROOT_2MUX1_GFREE>,
+					 <&d0_clock EIC7700_MUX_U_VO_PIXEL_ROOT_2MUX1>,
+					 <&d0_clock EIC7700_SPLL2_FOUT2>,
+					 <&d0_clock EIC7700_VPLL_FOUT1>;
+			clock-names = "cfg_clk", "pix_clk", "axi_clk", "spll0_fout1", "vo_mux",
+						  "pix_mux", "spll2_fout2", "vpll_fout1";
+			resets = <&d0_reset VO_RST_CTRL SW_VO_AXI_RSTN>,
+					 <&d0_reset VO_RST_CTRL SW_VO_CFG_RSTN>,
+					 <&d0_reset VO_RST_CTRL SW_VO_DC_RSTN>,
+					 <&d0_reset VO_RST_CTRL SW_VO_DC_PRSTN>;
+			reset-names = "vo_arst", "vo_prst", "dc_arst", "dc_prst";
+			dma-noncoherent;
+			numa-node-id = <0>;
+
+			dc_out: port {
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		virtual_display: es_wb@0 {
+			compatible = "eswin,virtual_display";
+			bpp = /bits/ 8 <8>;
+			numa-node-id = <0>;
+		};
+
+		dsi_output: dsi-output@0 {
+			compatible = "eswin,dsi-encoder";
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+
+		dsi_controller: mipi_dsi@50270000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "eswin,dsi";
+			reg = <0x0 0x50270000 0x0 0x10000>;
+			clocks = <&d0_clock EIC7700_CLK_CLK_MIPI_TXESC>;
+			clock-names = "pclk";
+			resets = <&d0_reset VO_PHYRST_CTRL SW_VO_MIPI_PRSTN>;
+			reset-names ="phyrstn";
+			numa-node-id = <0>;
+			/*
+			phys = <&dphy>;
+			phy-names = "dphy";
+			*/
+		};
+
+		dc_test: dctest@502c0000 {
+			compatible = "eswin,dc";
+			reg = <0x0 0x502c0000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <238>;
+			numa-node-id = <0>;
+		};
+
+		dw_hdmi: hdmi@502a0000 {
+			compatible = "eswin,eswin-dw-hdmi";
+			reg = <0x0 0x502a0000 0x0 0x20000>;
+			pinctrl-names = "default";
+			//pinctrl-0 = <&hdmi_i2c_xfer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <274>;
+			clocks = <&d0_clock EIC7700_CLK_VO_CFG_CLK>, <&d0_clock EIC7700_CLK_VO_CEC_CLK>,
+					 <&d0_clock EIC7700_CLK_VO_CR_CLK>;
+			clock-names = "iahb", "cec", "isfr";
+			//power-domains = <&power WIN2030_PD_HDCP>;
+			reg-io-width = <4>;
+			ddc-i2c-scl-high-time-ns = <4708>;
+			ddc-i2c-scl-low-time-ns = <4916>;
+			#sound-dai-cells = <0>;
+			resets = <&d0_reset VO_PHYRST_CTRL SW_VO_HDMI_PRSTN>,
+					 <&d0_reset VO_PHYRST_CTRL SW_HDMI_PHYCTRL_RSTN>,
+					 <&d0_reset VO_PHYRST_CTRL SW_VO_HDMI_RSTN>;
+			reset-names = "prstn", "phyrstn", "rstn";
+			numa-node-id = <0>;
+		};
+
+		dw_hdmi_hdcp2: hdmi-hdcp2@50290000 {
+				compatible = "eswin,dw-hdmi-hdcp2";
+				reg = <0x0 0x50290000 0x0 0x10000>;
+				interrupt-parent = <&plic0>;
+				interrupts = <275>;
+				clocks = <&d0_clock EIC7700_CLK_VO_ACLK>,
+					    <&d0_clock EIC7700_CLK_VO_HDMI_IESMCLK>;
+				clock-names ="aclk", "iesmclk";
+				dma-noncoherent;
+				numa-node-id = <0>;
+		};
+
+		d0_usbdrd3_0: usb0@50480000 {
+			compatible = "eswin,win2030-dwc3";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			clocks =<&d0_clock EIC7700_GATE_HSP_USB0_SUSPEND_CLK>,
+					<&d0_clock EIC7700_CLK_HSP_ACLK>,
+					<&d0_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "suspend","aclk", "cfg_clk";
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x800 0x808 0x83c 0x840>;
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_USB0_VAUX_RSTN>;
+			reset-names = "vaux";
+			ranges;
+			numa-node-id = <0>;
+			status = "disabled";
+			d0_usbdrd_dwc3_0: dwc3@50480000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x50480000 0x0 0x10000>;
+				#address-cells = <2>;
+				#size-cells = <2>;
+				interrupt-parent = <&plic0>;
+				interrupts = <85>;
+				interrupt-names = "peripheral";
+				dr_mode = "peripheral";
+				phy_type = "utmi";
+				maximum-speed = "high-speed";
+				iommus = <&smmu0 WIN2030_SID_USB0>;
+				eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1044>;
+				dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+				snps,dis_enblslpm_quirk;
+				snps,dis-u2-freeclk-exists-quirk;
+				snps,dis_u2_susphy_quirk;
+				snps,dis-del-phy-power-chg-quirk;
+				snps,parkmode-disable-ss-quirk;
+				status = "disabled";
+				numa-node-id = <0>;
+				tbus = <WIN2030_TBUID_USB>;
+				dma-noncoherent;
+			};
+		};
+
+		d0_usbdrd3_1: usb1@50490000  {
+			compatible = "eswin,win2030-dwc3";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			clocks =<&d0_clock EIC7700_GATE_HSP_USB1_SUSPEND_CLK>,
+					<&d0_clock EIC7700_CLK_HSP_ACLK>,
+					<&d0_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "suspend","aclk", "cfg_clk";
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x900 0x908 0x93c 0x940>;
+			resets = <&d0_reset HSPDMA_RST_CTRL SW_USB1_VAUX_RSTN>;
+			reset-names = "vaux";
+			ranges;
+			numa-node-id = <0>;
+			status = "disabled";
+			d0_usbdrd_dwc3_1: dwc3@50490000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x50490000 0x0 0x10000>;
+				#address-cells = <2>;
+				#size-cells = <2>;
+				interrupt-parent = <&plic0>;
+				interrupts = <86>;
+				interrupt-names = "host";
+				dr_mode = "host";
+				phy_type = "utmi";
+				maximum-speed = "high-speed";
+				iommus = <&smmu0 WIN2030_SID_USB1>;
+				eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1048>;
+				dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+				snps,dis_enblslpm_quirk;
+				snps,dis-u2-freeclk-exists-quirk;
+				snps,dis_u2_susphy_quirk;
+				snps,dis-del-phy-power-chg-quirk;
+				snps,parkmode-disable-ss-quirk;
+				status = "disabled";
+				numa-node-id = <0>;
+				tbus = <WIN2030_TBUID_USB>;
+				dma-noncoherent;
+			};
+		};
+
+		
+
+
+
+		vi_subsys: vi-subsys@51000000 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "esw,vi_subsys";	
+			reg = <0x0 0x51000000 0x0 0x200000>, <0x0 0x51030000 0x0 0x2000>;
+			eswin,syscrg_csr = <&d0_sys_crg>;
+			eswin,vitop_csr = <&vi_top_csr>;
+			clocks = <&d0_clock EIC7700_CLK_VI_ACLK>,
+						<&d0_clock EIC7700_CLK_VI_CFG_CLK>,
+						<&d0_clock EIC7700_CLK_VI_DIG_ISP_CLK>,
+						<&d0_clock EIC7700_CLK_VI_DVP_CLK>,
+						<&d0_clock EIC7700_CLK_VI_PHY_CFG>,
+						<&d0_clock EIC7700_CLK_VI_PHY_TXCLKESC>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_0>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_1>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_2>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_3>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_4>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_5>,
+						<&d0_clock EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_SPLL0_FOUT1>,
+						<&d0_clock EIC7700_VPLL_FOUT1>;
+			clock-names = "aclk", "cfg_clk", "isp_aclk", "dvp_clk", "phy_cfg",
+							"phy_escclk", "sht0", "sht1", "sht2", "sht3", "sht4",
+							"sht5", "aclk_mux", "dvp_mux", "isp_mux", "spll0_fout1", "vpll_fout1";
+			resets = <&d0_reset VI_RST_CTRL SW_VI_AXI_RSTN>,
+						<&d0_reset VI_RST_CTRL SW_VI_CFG_RSTN>,
+						<&d0_reset ISP0_RST_CTRL SW_VI_ISP0_RSTN>,
+						<&d0_reset ISP1_RST_CTRL SW_VI_ISP1_RSTN>,
+						<&d0_reset DVP_RST_CTRL SW_VI_DVP_RSTN>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_0>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_1>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_2>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_3>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_4>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_5>;
+			reset-names = "axi", "cfg", "isp0", "isp1", "dvp", "sht0", "sht1", "sht2", "sht3", "sht4", "sht5";
+			tbus = <WIN2030_TBUID_DW>;
+			numa-node-id = <0>;
+			ranges;
+			status = "okay";
+			
+			vi_top_csr: vi_common_top_csr@51030000 {
+				compatible ="syscon", "esw,vi-common-csr";
+				status="okay";
+				clocks = <&d0_clock EIC7700_CLK_VI_ACLK>,
+						<&d0_clock EIC7700_CLK_VI_CFG_CLK>,
+						<&d0_clock EIC7700_CLK_VI_DIG_ISP_CLK>,
+						<&d0_clock EIC7700_CLK_VI_DVP_CLK>,
+						<&d0_clock EIC7700_CLK_VI_PHY_CFG>,
+						<&d0_clock EIC7700_CLK_VI_PHY_TXCLKESC>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_0>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_1>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_2>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_3>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_4>,
+						<&d0_clock EIC7700_CLK_VI_SHUTTER_5>,
+						<&d0_clock EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_SPLL0_FOUT1>,
+						<&d0_clock EIC7700_VPLL_FOUT1>;
+				clock-names = "aclk", "cfg_clk", "isp_aclk", "dvp_clk", "phy_cfg",
+							"phy_escclk", "sht0", "sht1", "sht2", "sht3", "sht4",
+							"sht5", "aclk_mux", "dvp_mux", "isp_mux", "spll0_fout1", "vpll_fout1";
+				resets = <&d0_reset VI_RST_CTRL SW_VI_AXI_RSTN>,
+						<&d0_reset VI_RST_CTRL SW_VI_CFG_RSTN>,
+						<&d0_reset ISP0_RST_CTRL SW_VI_ISP0_RSTN>,
+						<&d0_reset ISP1_RST_CTRL SW_VI_ISP1_RSTN>,
+						<&d0_reset DVP_RST_CTRL SW_VI_DVP_RSTN>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_0>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_1>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_2>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_3>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_4>,
+						<&d0_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_5>;
+				reset-names = "axi", "cfg", "isp0", "isp1", "dvp", "sht0", "sht1", "sht2", "sht3", "sht4", "sht5";
+				interrupt-parent = <&plic0>;
+				interrupts = <368 369 370 371 372 373 374 375>;
+				id = <0>;
+				#size-cells = <2>;
+				reg = <0x0 0x51030000 0x0 0x2000>;
+				numa-node-id = <0>;
+				tbus = <WIN2030_TBUID_ISP>;
+			};
+
+			isp_0: isp@51000000 {
+				compatible = "esw,win2030-isp";
+
+				reg = <0x0 0x51000000 0x0 0x10000>;
+				interrupts = <21 19 20>;
+				interrupt-parent = <&plic0>;
+				id = <0>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu0 WIN2030_SID_ISP0>;
+				tbus = <WIN2030_TBUID_ISP>;
+				eswin,vi_top_csr = <&vi_top_csr 0x1000>;
+				numa-node-id = <0>;
+				dma-noncoherent;
+			};
+
+			isp_1: isp@51010000 {
+				compatible = "esw,win2030-isp";
+
+				reg = <0x0 0x51010000 0x0 0x10000>;
+				interrupts = <24 22 23>;
+				interrupt-parent = <&plic0>;
+				id = <1>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu0 WIN2030_SID_ISP1>;
+				tbus = <WIN2030_TBUID_ISP>;
+				eswin,vi_top_csr = <&vi_top_csr 0x1004>;
+				numa-node-id = <0>;
+				dma-noncoherent;
+			};
+
+			dewarp: dewarp@51020000 {
+				compatible = "eswin,dewarp";
+				clocks = <&d0_clock EIC7700_CLK_VI_ACLK>,
+						<&d0_clock EIC7700_CLK_VI_CFG_CLK>,
+						<&d0_clock EIC7700_CLK_VI_DIG_DW_CLK>,
+						<&d0_clock EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE>,
+						<&d0_clock EIC7700_MUX_U_VI_DW_ROOT_2MUX1>,
+						<&d0_clock EIC7700_SPLL0_FOUT1>,
+						<&d0_clock EIC7700_VPLL_FOUT1>;
+				clock-names = "aclk", "cfg_clk", "dw_aclk", "aclk_mux", "dw_mux", "spll0_fout1", "vpll_fout1";
+				resets = <&d0_reset VI_RST_CTRL SW_VI_AXI_RSTN>,
+						<&d0_reset VI_RST_CTRL SW_VI_CFG_RSTN>,
+						<&d0_reset VI_RST_CTRL SW_VI_DWE_RSTN>;
+				reset-names = "axi", "cfg", "dwe";
+
+				operating-points-v2 = <&vi_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				interrupt-parent = <&plic0>;
+				interrupts = <26 25>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu0 WIN2030_SID_DW>;
+				tbus = <WIN2030_TBUID_DW>;
+				eswin,vi_top_csr = <&vi_top_csr 0x1008>;
+				reg = <0x0 0x51020000 0x0 0xc00>, <0x0 0x51020c00 0x0 0x120>;
+				numa-node-id = <0>;
+				dma-noncoherent;
+			};
+
+			dvp2axi: dvp2axi@51034000 {
+				compatible = "eswin,dvp2axi";
+				reg = <0x0 0x51034000 0x0 0x8000>;
+				reg-names = "dvp2axi_regs";
+				interrupt-parent = <&plic0>;
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr 0x100c>;
+				interrupts = <368 369 370 371 372 373 374 375>;
+				interrupt-names = "dvp0_frame_int", "dvp1_frame_int", "dvp2_frame_int",
+									"dvp3_frame_int", "dvp4_frame_int", "dvp5_frame_int",
+									"dvp2axi_error_int","dvp2axi_afull_int";
+				dma-noncoherent;
+				numa-node-id = <0>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu0 WIN2030_SID_DW>;
+				status = "disabled";
+			};
+
+			dvp2axi_mipi_lvds: dvp2axi_mipi_lvds {
+				compatible = "eswin,dvp2axi-mipi-lvds";
+				eswin,hw = <&dvp2axi>;
+				status = "disabled";
+			};
+			dvp2axi_mipi_lvds1: dvp2axi_mipi_lvds1 {
+				compatible = "eswin,dvp2axi-mipi-lvds";
+				eswin,hw = <&dvp2axi>;
+				status = "disabled";
+			};
+
+			dvp2axi_mipi_lvds2: dvp2axi_mipi_lvds2 {
+				compatible = "eswin,dvp2axi-mipi-lvds";
+				eswin,hw = <&dvp2axi>;
+				status = "disabled";
+			};
+			
+			dvp2axi_mipi_lvds3: dvp2axi_mipi_lvds3 {
+				compatible = "eswin,dvp2axi-mipi-lvds";
+				eswin,hw = <&dvp2axi>;
+				status = "disabled";
+			};
+
+			dvp2axi_mipi_lvds4: dvp2axi_mipi_lvds4 {
+				compatible = "eswin,dvp2axi-mipi-lvds";
+				eswin,hw = <&dvp2axi>;
+				status = "disabled";
+			};
+
+			dvp2axi_mipi_lvds5: dvp2axi_mipi_lvds5 {
+				compatible = "eswin,dvp2axi-mipi-lvds";
+				eswin,hw = <&dvp2axi>;
+				status = "disabled";
+			};
+
+			mipi_csi2_0: mipi-csi2-0@51050000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x51050000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr>;
+				interrupt-parent = <&plic0>;
+				interrupts = <29>;
+				interrupt-names = "csi-intr1";
+				index = <0>;
+
+				ipi2_en = <0>;/* for virtual channel */
+				ipi2_vcid = <0>;/* virtual channel id */
+				ipi3_en = <0>;
+				ipi3_vcid = <0>;
+				status = "disabled";
+			};
+
+			mipi_csi2_1: mipi-csi2-1@51060000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x51060000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr>;
+				interrupt-parent = <&plic0>;
+				interrupts = <30>;
+				interrupt-names = "csi-intr1";
+				index = <1>;
+				status = "disabled";
+			};
+
+			mipi_csi2_2: mipi-csi2-2@51070000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x51070000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr>;
+				interrupt-parent = <&plic0>;
+				interrupts = <31>;
+				interrupt-names = "csi-intr1";
+				index = <2>;
+				status = "disabled";
+			};
+
+			mipi_csi2_3: mipi-csi2-3@51080000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x51080000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr>;
+				interrupt-parent = <&plic0>;
+				interrupts = <32>;
+				interrupt-names = "csi-intr1";
+				index = <3>;
+				status = "disabled";
+			};
+
+			mipi_csi2_4: mipi-csi2-4@51090000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x51090000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr>;
+				interrupt-parent = <&plic0>;
+				interrupts = <33>;
+				interrupt-names = "csi-intr1";
+				index = <4>;
+				status = "disabled";
+			};
+
+			mipi_csi2_5: mipi-csi2-5@510a0000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x510a0000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d0_sys_crg>;
+				eswin,vi_top_csr = <&vi_top_csr>;
+				interrupt-parent = <&plic0>;
+				interrupts = <34>;
+				interrupt-names = "csi-intr1";
+				index = <5>;
+				status = "disabled";
+			};
+
+			csi2_dphy_hw: csi2-dphy-hw@510c0000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x510c0000 0x0 0x20000>;
+				phy_cfg = <0x510d8000>;
+				csi_base = <0x51050000>;
+				combine_phy = <0x510e0000>;
+				status = "disabled";
+			};
+
+			csi2_dphy_hw1: csi2-dphy-hw1@510e0000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x510e0000 0x0 0x20000>;
+				phy_cfg = <0x510f8000>;
+				csi_base = <0x51060000>;
+				status = "disabled";
+			};
+
+			csi2_dphy_hw2: csi2-dphy-hw2@51100000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x51100000 0x0 0x20000>;
+				phy_cfg = <0x51118000>;
+				csi_base = <0x51070000>;
+				combine_phy = <0x51120000>;
+				status = "disabled";
+			};
+
+			csi2_dphy_hw3: csi2-dphy-hw3@51120000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x51120000 0x0 0x20000>;
+				phy_cfg = <0x51138000>;
+				csi_base = <0x51080000>;
+				status = "disabled";
+			};
+
+			csi2_dphy_hw4: csi2-dphy-hw4@51140000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x51140000 0x0 0x20000>;
+				phy_cfg = <0x51158000>;
+				csi_base = <0x51090000>;
+				combine_phy = <0x51160000>;
+				status = "disabled";
+			};
+
+			csi2_dphy_hw5: csi2-dphy-hw5@51160000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x51160000 0x0 0x20000>;
+				phy_cfg = <0x51178000>;
+				csi_base = <0x510a0000>;
+				status = "disabled";
+			};
+
+			csi2_dphy0: csi2-dphy0 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&csi2_dphy_hw>;
+				index = <0>;
+				status = "disabled";
+			};
+			
+			csi2_dphy1: csi2-dphy1 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&csi2_dphy_hw1>;
+				index = <1>;
+				status = "disabled";
+			};
+
+			csi2_dphy2: csi2-dphy2 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&csi2_dphy_hw2>;
+				index = <2>;
+				status = "disabled";
+			};
+
+			csi2_dphy3: csi2-dphy3 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&csi2_dphy_hw3>;
+				index = <3>;
+				status = "disabled";
+			};
+
+			csi2_dphy4: csi2-dphy4 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&csi2_dphy_hw4>;
+				index = <4>;
+				status = "disabled";
+			};
+
+			csi2_dphy5: csi2-dphy5 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&csi2_dphy_hw5>;
+				index = <5>;
+				status = "disabled";
+			};
+
+			mipi_dphy_rx: dphy@510c0000 {
+				compatible = "snps,dw-dphy-rx";
+				#phy-cells = <1>;
+				bus-width = <8>;
+				snps,dphy-frequency = <300000>;
+				snps,phy_type = <8>;
+				reg = <0x0 0x510c0000 0x0 0x20000>;
+				numa-node-id = <0>;
+			};
+
+			csi_dma0: csidma@52048000 {
+				compatible = "eswin,csi-video";
+				interrupt-parent = <&plic0>;
+				interrupts = <29>;
+				reg = <0x0 0x52048000 0x0 0x1000>;
+				numa-node-id = <0>;
+
+				port {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					csi_dmar_0: endpoint@0 {
+						reg = <0>;
+						bus-type = <4>;
+						remote-endpoint = <&csi2_dma_0_3>;
+					};
+				};
+			};
+
+			csi_dma1: csidma@52058000 {
+				compatible = "eswin,csi-video";
+				interrupt-parent = <&plic0>;
+				interrupts = <30>;
+				reg = <0x0 0x52058000 0x0 0x1000>;
+				numa-node-id = <0>;
+
+				port {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					csi_dmar_1: endpoint@0 {
+						reg = <0>;
+						bus-type = <4>;
+						remote-endpoint = <&csi2_dma_1_3>;
+					};
+				};
+			};
+
+			csi2_0: csi2@51050000 {
+				compatible = "snps,dw-csi";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				interrupt-parent = <&plic0>;
+				interrupts = <29>;
+				snps,output-type = <0>;
+				reg = <0x0 0x51050000 0x0 0x1000>;
+				/*phys = <&mipi_dphy_rx 1>;*/
+				numa-node-id = <0>;
+
+				ipi2_en = <0>;/* for virtual channel */
+				ipi2_vcid = <0>;/* virtual channel id */
+				ipi3_en = <0>;
+				ipi3_vcid = <0>;
+
+				#ifdef MIPI_CSI2_IMX290_ENBALE
+				port@1 {
+					reg = <1>;
+					csi2_0_1: endpoint {
+						bus-type = <4>;
+						clock-lanes = <0>;
+						#ifdef MIPI_CSI2_2LINE_ENABLE
+						data-lanes = <1 2>;
+						#else
+						data-lanes = <1 2 3 4>;
+						#endif
+						//remote-endpoint = <&imx290_csi2>;
+					};
+				};
+				#endif
+
+				port@2 {
+					reg = <2>;
+					csi2_dma_0_3: endpoint {
+						bus-type = <4>;
+						clock-lanes = <0>;
+											data-lanes = <1 2>;
+						remote-endpoint = <&csi_dmar_0>;
+					};
+				};
+			};
+
+			csi2_1: csi2@51060000 {
+				compatible = "snps,dw-csi";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				interrupt-parent = <&plic0>;
+				interrupts = <30>;
+				snps,output-type = <0>;
+				reg = <0x0 0x51060000 0x0 0x1000>;
+				/*phys = <&mipi_dphy_rx 1>;*/
+				numa-node-id = <0>;
+
+				ipi2_en = <0>;/* for virtual channel */
+				ipi2_vcid = <0>;/* virtual channel id */
+				ipi3_en = <0>;
+				ipi3_vcid = <0>;
+
+				#ifdef MIPI_CSI2_IMX290_ENBALE
+				port@1 {
+					reg = <1>;
+					csi2_1_1: endpoint {
+						bus-type = <4>;
+						clock-lanes = <0>;
+						#ifdef MIPI_CSI2_2LINE_ENABLE
+						data-lanes = <1 2>;
+						#else
+						data-lanes = <1 2 3 4>;
+						#endif
+						//remote-endpoint = <&imx290_csi2>;
+					};
+				};
+				#endif
+
+				port@2 {
+					reg = <2>;
+					csi2_dma_1_3: endpoint {
+						bus-type = <4>;
+						remote-endpoint = <&csi_dmar_1>;
+					};
+				};
+			};
+		};
+		dc_test: dctest@502c0000 {
+			compatible = "eswin,dc";
+			reg = <0x0 0x502c0000 0x0 0x10000>;
+			interrupt-parent = <&plic0>;
+			interrupts = <238>;
+			numa-node-id = <0>;
+		};
+
+		d0_numa_sample:numa_sample@0 {
+			compatible = "eswin,numa-sample","simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			dma-ranges = <0x0 0x80000000 0x0 0xc0000000 0x0 0x80000000>;
+			iommus = <&smmu0 WIN2030_SID_SCPU>;
+			tbus = <WIN2030_TBUID_0xF00>;
+			numa-node-id = <0>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		ddr0: ddr-controller@52300000 {
+			compatible = "eswin,ddrc-1.20a";
+			interrupt-parent = <&plic0>;
+			interrupts = <10>;
+			interrupt-names = "ddr-ecc";
+			reg = <0x0 0x52300000 0x0 0x40000>;
+			ctrl-id = <0>;
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+
+		ddr1: ddr-controller@52380000 {
+			compatible = "eswin,ddrc-1.20a";
+			interrupt-parent = <&plic0>;
+			interrupts = <299>;
+			interrupt-names = "ddr-ecc";
+			reg = <0x0 0x52380000 0x0 0x40000>;
+			ctrl-id = <1>;
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+
+		d2d: d2d-unit@52100000 {
+			compatible = "eswin,eic7x-d2d";
+			interrupt-parent = <&plic0>;
+			interrupts = <287>, <288>;
+			reg = <0x0 0x52100000 0x0 0x50000>;
+			reg-names = "control";
+			numa-node-id = <0>;
+			status = "disabled";
+		};
+};
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-die1-noc.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-die1-noc.dtsi
new file mode 100644
index 000000000000..375a4f3ea279
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-die1-noc.dtsi
@@ -0,0 +1,2808 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Die1 NOC monitor of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+d1_cfg_noc:d1_cfg_noc{
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x72060000 0 0x4000>;
+
+	interrupts = <446>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic1>;
+	errlogger,idx = <0 1 3 5>;
+	numa-node-id = <1>;
+	sideband_manager@72061000{
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x72061000 0 0x10>;
+		SenseIn0 =
+			<SBM_CLMM 	OFFSET0>,
+			<SBM_CNOC_AON 	OFFSET1>,
+			<SBM_CNOC_DDRT0_CTRL 	OFFSET2>,
+			<SBM_CNOC_DDRT0_PHY 	OFFSET3>,
+			<SBM_CNOC_DDRT1_CTRL 	OFFSET4>,
+			<SBM_CNOC_DDRT1_PHY 	OFFSET5>,
+			<SBM_CNOC_DSPT 	OFFSET6>,
+			<SBM_CNOC_GPU 	OFFSET7>,
+			<SBM_CNOC_HSP 	OFFSET8>,
+			<SBM_CNOC_LSP_APB2 	OFFSET9>,
+			<SBM_CNOC_LSP_APB3	OFFSET10>,
+			<SBM_CNOC_LSP_APB4 	OFFSET11>,
+			<SBM_CNOC_LSP_APB6 	OFFSET12>,
+			<SBM_CNOC_MCPUT_D2D 	OFFSET13>,
+			<SBM_CNOC_NPU 	OFFSET14>,
+			<SBM_CNOC_PCIET_P 	OFFSET15>,
+			<SBM_CNOC_PCIET_X 	OFFSET16>,
+			<SBM_CNOC_TCU 	OFFSET17>,
+			<SBM_CNOC_VC 	OFFSET18>,
+			<SBM_CNOC_VI 	OFFSET19>,
+			<SBM_CNOC_VO 	OFFSET20>;
+		bf-name =
+			"SBM_CLMM",
+			"SBM_CNOC_AON",
+			"SBM_CNOC_DDRT0_CTRL",
+			"SBM_CNOC_DDRT0_PHY ",
+			"SBM_CNOC_DDRT1_CTRL",
+			"SBM_CNOC_DDRT1_PHY",
+			"SBM_CNOC_DSPT",
+			"SBM_CNOC_GPU",
+			"SBM_CNOC_HSP",
+			"SBM_CNOC_LSP_APB2",
+			"SBM_CNOC_LSP_APB3",
+			"SBM_CNOC_LSP_APB4",
+			"SBM_CNOC_LSP_APB6",
+			"SBM_CNOC_MCPUT_D2D",
+			"SBM_CNOC_NPU",
+			"SBM_CNOC_PCIET_P",
+			"SBM_CNOC_PCIET_X",
+			"SBM_CNOC_TCU",
+			"SBM_CNOC_VC",
+			"SBM_CNOC_VI",
+			"SBM_CNOC_VO";
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1>;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4>;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 15 1 >;  /*bit 15 will aloways be 0, then we will always get "snoc_cnoc/I/0"*/
+			lut =
+				"snoc_cnoc/I/0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 10 5 >;
+			lut =
+				"clmm/T/0",
+				"cnoc_aon/T/0",
+				"cnoc_ddrt0_ctrl/T/0",
+				"cnoc_ddrt0_phy/T/0",
+				"cnoc_ddrt1_ctrl/T/0",
+				"cnoc_ddrt1_phy/T/0",
+				"cnoc_dspt/T/0",
+				"cnoc_gpu/T/0",
+				"cnoc_hsp/T/0",
+				"cnoc_lsp_apb2/T/0",
+				"cnoc_lsp_apb3/T/0",
+				"cnoc_lsp_apb4/T/0",
+				"cnoc_lsp_apb6/T/0",
+				"cnoc_mcput_d2d/T/0",
+				"cnoc_npu/T/0",
+				"cnoc_pciet_p/T/0",
+				"cnoc_pciet_x/T/0",
+				"cnoc_service/T/0",
+				"cnoc_tcu/T/0",
+				"cnoc_vc/T/0",
+				"cnoc_vi/T/0",
+				"cnoc_vo/T/0",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 2 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 8 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger1 whose information are required to calculate real absolute address */
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 8 >;	/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 46 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <0x0 0x0    0x0  0x51600000>,
+				/bits/ 64  <0x0 0x0    0x1  0x71600000>,
+				/bits/ 64  <0x0 0x1    0x0  0x51800000>,
+				/bits/ 64  <0x0 0x1    0x1  0x71800000>,
+				/bits/ 64  <0x0 0x1    0x2  0x0       >,
+				/bits/ 64  <0x0 0x1    0x3  0x0       >,
+				/bits/ 64  <0x0 0x2    0x0  0x52300000>,
+				/bits/ 64  <0x0 0x2    0x1  0x72300000>,
+				/bits/ 64  <0x0 0x3    0x0  0x53000000>,
+				/bits/ 64  <0x0 0x3    0x1  0x73000000>,
+				/bits/ 64  <0x0 0x4    0x0  0x52380000>,
+				/bits/ 64  <0x0 0x4    0x1  0x72380000>,
+				/bits/ 64  <0x0 0x5    0x0  0x53800000>,
+				/bits/ 64  <0x0 0x5    0x1  0x73800000>,
+				/bits/ 64  <0x0 0x6    0x0  0x52200000>,
+				/bits/ 64  <0x0 0x6    0x1  0x72200000>,
+				/bits/ 64  <0x0 0x7    0x0  0x51400000>,
+				/bits/ 64  <0x0 0x7    0x1  0x71400000>,
+				/bits/ 64  <0x0 0x8    0x0  0x50400000>,
+				/bits/ 64  <0x0 0x8    0x1  0x70400000>,
+				/bits/ 64  <0x0 0x9    0x0  0x50800000>,
+				/bits/ 64  <0x0 0x9    0x1  0x70800000>,
+				/bits/ 64  <0x0 0xa    0x0  0x50900000>,
+				/bits/ 64  <0x0 0xa    0x1  0x70900000>,
+				/bits/ 64  <0x0 0xb    0x0  0x50a00000>,
+				/bits/ 64  <0x0 0xb    0x1  0x70a00000>,
+				/bits/ 64  <0x0 0xc    0x0  0x50b00000>,
+				/bits/ 64  <0x0 0xc    0x1  0x70b00000>,
+				/bits/ 64  <0x0 0xd    0x0  0x52100000>,
+				/bits/ 64  <0x0 0xd    0x1  0x72100000>,
+				/bits/ 64  <0x0 0xe    0x0  0x51c00000>,
+				/bits/ 64  <0x0 0xe    0x1  0x71c00000>,
+				/bits/ 64  <0x0 0xf    0x0  0x50000000>,
+				/bits/ 64  <0x0 0xf    0x1  0x70000000>,
+				/bits/ 64  <0x0 0x10   0x0  0x54000000>,
+				/bits/ 64  <0x0 0x10   0x1  0x74000000>,
+				/bits/ 64  <0x0 0x11   0x0  0x52060000>,
+				/bits/ 64  <0x0 0x11   0x1  0x72060000>,
+				/bits/ 64  <0x0 0x12   0x0  0x50c00000>,
+				/bits/ 64  <0x0 0x12   0x1  0x70c00000>,
+				/bits/ 64  <0x0 0x13   0x0  0x50100000>,
+				/bits/ 64  <0x0 0x13   0x1  0x70100000>,
+				/bits/ 64  <0x0 0x14   0x0  0x51000000>,
+				/bits/ 64  <0x0 0x14   0x1  0x71000000>,
+				/bits/ 64  <0x0 0x15   0x0  0x50200000>,
+				/bits/ 64  <0x0 0x15   0x1  0x70200000>;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32>;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 7 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0",
+				"Prot_1",
+				"Prot_2";
+		};
+	};
+};
+
+d1_llc_noc:d1_llc_noc@72081400 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <1>;
+	reg = <0 0x72081400 0 0x4000>;
+	interrupts = <441>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic1>;
+	errlogger,idx = <0 1 3 4 5>;
+#ifdef PLATFORM_HAPS
+	clock,rate = <5000000>;  /*haps ddr controller clk*/
+#endif
+	sideband_manager@72082000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x72082000 0 0x10>;
+		SenseIn0 =
+			<SBM_LNOC_NPU_LLC0 	OFFSET0>,
+			<SBM_LNOC_NPU_LLC1 	OFFSET1>,
+			<SBM_LNOC_DDRT0_P0 	OFFSET2>,
+			<SBM_LNOC_DDRT1_P0 	OFFSET3>;
+		bf-name =
+			"SBM_LNOC_NPU_LLC0",
+			"SBM_LNOC_NPU_LLC1",
+			"SBM_LNOC_DDRT0_P0",
+			"SBM_LNOC_DDRT1_P0";
+	};
+
+	llcnoc_packet_ddr0_p0_req_probe@72080000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72080000 0 0x4000>;
+		clocks = <&d1_clock EIC7700_CLK_DDRT0_P0_ACLK>;
+		clock-names = "clk";
+		interrupts = <445>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p0_req";
+	};
+	llcnoc_packet_ddr1_p0_req_probe@72080800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72080800 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT1_P0_ACLK>;
+		clock-names = "clk";
+		interrupts = <443>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p0_req";
+	};
+	llcnoc_trans_probe@72081000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x72081000 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <441>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "npu_llc0", "npu_llc1";
+		llcnoc_trans_npu_llc0_filter@72081480 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72081480 0 0x80>;
+		};
+		llcnoc_trans_npu_llc1_filter@72081500 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72081500 0 0x80>;
+		};
+		llcnoc_trans_profiler@72081580 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x72081580 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"npu_lnoc_llc0/I/0",
+				"npu_lnoc_llc1/I/0",
+				"snoc_lnoc/I/0",
+				"RESERVED";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"lnoc_ddrt0_p0/T/0",
+				"lnoc_ddrt1_p0/T/0",
+				"lnoc_service/T/0",
+				"RESERVED";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 4 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 8 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 8 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 53 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =		/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <0x0 0x0 0x0 0x80000000      >,
+				/bits/ 64  <0x0 0x0 0x1 0x100000000     >,
+				/bits/ 64  <0x0 0x0 0x2 0x200000000     >,
+				/bits/ 64  <0x0 0x0 0x3 0x400000000     >,
+				/bits/ 64  <0x0 0x0 0x4 0x800000000     >,
+				/bits/ 64  <0x0 0x0 0x5 0x2000000000    >,
+				/bits/ 64  <0x0 0x0 0x6 0xc000000000    >,
+				/bits/ 64  <0x0 0x0 0x7 0xe000000000    >,
+				/bits/ 64  <0x0 0x0 0x8 0x4000000000    >,
+				/bits/ 64  <0x0 0x0 0x9 0x4000000100    >,
+				/bits/ 64  <0x0 0x0 0xa 0x10000000000   >,
+				/bits/ 64  <0x0 0x0 0xb 0x10000000100   >,
+				/bits/ 64  <0x0 0x0 0xc 0x0             >,
+				/bits/ 64  <0x0 0x1 0x0 0x80000080      >,
+				/bits/ 64  <0x0 0x1 0x1 0x100000080     >,
+				/bits/ 64  <0x0 0x1 0x2 0x200000080     >,
+				/bits/ 64  <0x0 0x1 0x3 0x400000080     >,
+				/bits/ 64  <0x0 0x1 0x4 0x800000080     >,
+				/bits/ 64  <0x0 0x1 0x5 0x2000000080    >,
+				/bits/ 64  <0x0 0x1 0x6 0xc000000080    >,
+				/bits/ 64  <0x0 0x1 0x7 0xe000000080    >,
+				/bits/ 64  <0x0 0x1 0x8 0x4000000080    >,
+				/bits/ 64  <0x0 0x1 0x9 0x4000000180    >,
+				/bits/ 64  <0x0 0x1 0xa 0x10000000080   >,
+				/bits/ 64  <0x0 0x1 0xb 0x10000000180   >,
+				/bits/ 64  <0x1 0x0 0x0 0x80000000      >,
+				/bits/ 64  <0x1 0x0 0x1 0x100000000     >,
+				/bits/ 64  <0x1 0x0 0x2 0x200000000     >,
+				/bits/ 64  <0x1 0x0 0x3 0x400000000     >,
+				/bits/ 64  <0x1 0x0 0x4 0x800000000     >,
+				/bits/ 64  <0x1 0x0 0x5 0x2000000000    >,
+				/bits/ 64  <0x1 0x0 0x6 0xc000000000    >,
+				/bits/ 64  <0x1 0x0 0x7 0xe000000000    >,
+				/bits/ 64  <0x1 0x0 0x8 0x4000000000    >,
+				/bits/ 64  <0x1 0x0 0x9 0x4000000100    >,
+				/bits/ 64  <0x1 0x0 0xa 0x10000000000   >,
+				/bits/ 64  <0x1 0x0 0xb 0x10000000100   >,
+				/bits/ 64  <0x1 0x0 0xc 0x0             >,
+				/bits/ 64  <0x1 0x1 0x0 0x80000080      >,
+				/bits/ 64  <0x1 0x1 0x1 0x100000080     >,
+				/bits/ 64  <0x1 0x1 0x2 0x200000080     >,
+				/bits/ 64  <0x1 0x1 0x3 0x400000080     >,
+				/bits/ 64  <0x1 0x1 0x4 0x800000080     >,
+				/bits/ 64  <0x1 0x1 0x5 0x2000000080    >,
+				/bits/ 64  <0x1 0x1 0x6 0xc000000080    >,
+				/bits/ 64  <0x1 0x1 0x7 0xe000000080    >,
+				/bits/ 64  <0x1 0x1 0x8 0x4000000080    >,
+				/bits/ 64  <0x1 0x1 0x9 0x4000000180    >,
+				/bits/ 64  <0x1 0x1 0xa 0x10000000080   >,
+				/bits/ 64  <0x1 0x1 0xb 0x10000000180   >,
+				/bits/ 64  <0x2 0x2 0x0 0x52080000      >,
+				/bits/ 64  <0x2 0x2 0x1 0x72080000      >,
+				/bits/ 64  <0x2 0x2 0x2 0x0             >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 16 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0",
+				"Prot_1",
+				"Prot_2",
+				"Qos_0",
+				"Qos_1",
+				"Qos_2",
+				"Qos_3",
+				"User_0",
+				"User_1",
+				"User_2",
+				"User_3",
+				"User_4";
+		};
+	};
+};
+
+d1_sys_noc:d1_sys_noc@72002C00 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <1>;
+	reg = <0 0x72002C00 0 0x4000>;
+	interrupts = <431>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic1>;
+	errlogger,idx = <0 1 3 4 5>;
+
+	eswin,qos-configs = "DSPT", "NPU", "SPISLV_TBU3";
+#ifdef PLATFORM_HAPS
+	eswin,DSPT-qos-base = <0x72002C80>;
+	eswin,DSPT-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x1        /* mode 0:fixed 1:limiter 2:bypass 3:regulator*/
+		/* a number of (1/256)th of Bytes/cycle.
+		   Ex:DSP AXI clk=5MHz, BW=1MB/s, register value = (1/5)*256 = 0x33
+		*/
+		0x10 0x33    /* bandwidth. 1MB/s */
+		/*
+		   Saturation(B) = ((Requried Bandwidth)*(Windows Time of Bandwidth Calculation))/16
+		   Ex:16 byte saturation for BW=1MB/s means 16us window time.
+		   The desired value is number of saturation bytes divided by 16(ex,1 for 16byte B)
+		*/
+		0x14 0x1     /* saturation, 16us*/
+		0x18 0x0>;   /* QoSEn */
+
+	eswin,NPU-qos-base = <0x72002D00>;
+	eswin,NPU-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x1      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x0>;   /* QoSEn */
+
+	eswin,SPISLV_TBU3-qos-base = <0x72002D80>;
+	eswin,SPISLV_TBU3-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x1      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x0>;   /* QoSEn */
+	clock,rate = <5000000>; /*haps ddr controller axi clk*/
+#else
+	eswin,DSPT-qos-base = <0x52002C80>;
+	eswin,DSPT-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode 0:fixed 1:limiter 2:bypass 3:regulator*/
+		/* a number of (1/256)th of Bytes/cycle.
+		   Ex:zebu zdfi design feature, dsp AXI Clk=1040MHz, BW=12.1875MB/s, register value = (9.375/1040)*256 = 0x03
+		*/
+		0x10 0x03    /* bandwidth. 12.1875MB/s */
+		/*
+		   Saturation(B) = ((Requried Bandwidth)*(Windows Time of Bandwidth Calculation))/16
+		   Ex:16 byte saturation for BW=12.1875MB/s means 1.313us window time.
+		   The desired value is number of saturation bytes divided by 16(ex,1 for 16byte B)
+		*/
+		0x14 0x1     /* saturation, 1.313us*/
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,NPU-qos-base = <0x72002D00>;
+	eswin,NPU-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,SPISLV_TBU3-qos-base = <0x72002D80>;
+	eswin,SPISLV_TBU3-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+#endif
+	sideband_manager@72004000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x72004000 0 0x10>;
+		SenseIn0 =
+			<SBM_AON_SNOC_SP0 	OFFSET0>,
+			<SBM_DSPT_SNOC 		OFFSET1>,
+			<SBM_JTAG_SNOC 		OFFSET2>,
+			<SBM_MCPUT_SNOC_D2D 	OFFSET3>,
+			<SBM_MCPUT_SNOC_MP 	OFFSET4>,
+			<SBM_MCPUT_SNOC_SP0 	OFFSET5>,
+			<SBM_MCPUT_SNOC_SP1 	OFFSET6>,
+			<SBM_NPU_SNOC_SP0 	OFFSET7>,
+			<SBM_NPU_SNOC_SP1 	OFFSET8>,
+			<SBM_PCIET_SNOC_P 	OFFSET9>,
+			<SBM_SPISLV_PCIET_SNOC	OFFSET10>,
+			<SBM_TBU4_SNOC 		OFFSET11>,
+			<SBM_TCU_SNOC 		OFFSET12>,
+			<SBM_SNOC_AON 		OFFSET13>,
+			<SBM_SNOC_DDR0_P1 	OFFSET14>,
+			<SBM_SNOC_DDR0_P2 	OFFSET15>,
+			<SBM_SNOC_DDR1_P1 	OFFSET16>,
+			<SBM_SNOC_DDR1_P2 	OFFSET17>,
+			<SBM_SNOC_DSPT 		OFFSET18>,
+			<SBM_SNOC_MCPUT_D2D 	OFFSET19>,
+			<SBM_SNOC_NPU 		OFFSET20>,
+			<SBM_SNOC_PCIET 	OFFSET21>;
+		bf-name =
+			"SBM_AON_SNOC_SP0",
+			"SBM_DSPT_SNOC",
+			"SBM_JTAG_SNOC",
+			"SBM_MCPUT_SNOC_D2D ",
+			"SBM_MCPUT_SNOC_MP",
+			"SBM_MCPUT_SNOC_SP0",
+			"SBM_MCPUT_SNOC_SP1",
+			"SBM_NPU_SNOC_SP0",
+			"SBM_NPU_SNOC_SP1",
+			"SBM_PCIET_SNOC_P",
+			"SBM_SPISLV_PCIET_SNOC",
+			"SBM_TBU4_SNOC",
+			"SBM_TCU_SNOC",
+			"SBM_SNOC_AON",
+			"SBM_SNOC_DDR0_P1",
+			"SBM_SNOC_DDR0_P2",
+			"SBM_SNOC_DDR1_P1",
+			"SBM_SNOC_DDR1_P2",
+			"SBM_SNOC_DSPT",
+			"SBM_SNOC_MCPUT_D2D",
+			"SBM_SNOC_NPU",
+			"SBM_SNOC_PCIET";
+	};
+	sysnoc_packet_ddr0_p1_req_probe@72000000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72000000 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT0_P1_ACLK>;
+		clock-names = "clk";
+		interrupts = <439>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p1_req";
+	};
+	sysnoc_packet_ddr0_p2_req_probe@72000800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72000800 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT0_P2_ACLK>;
+		clock-names = "clk";
+		interrupts = <437>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p2_req";
+	};
+	sysnoc_packet_ddr1_p1_req_probe@72001000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72001000 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT1_P1_ACLK>;
+		clock-names = "clk";
+		interrupts = <435>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p1_req";
+	};
+	sysnoc_packet_ddr1_p2_req_probe@72001800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72001800 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT1_P2_ACLK>;
+		clock-names = "clk";
+		interrupts = <433>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p2_req";
+	};
+	sysnoc_trans_probe_0@72002000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x72002000 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <430>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "dspt_snoc", "npu_sp1";
+		sysnoc_trans_dspt_filter@72002E00 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72002E00 0 0x80>;
+		};
+		sysnoc_trans_npu_sp1_filter@72002F80 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72002F80 0 0x80>;
+		};
+		sysnoc_trans_profiler@72003180 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x72003180 0 0x80>;
+		};
+	};
+	sysnoc_trans_probe_1@72002400 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x72002400 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <429>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <3>;
+		counter,nr = <12>;
+		profiler,nr = <1>;
+		portsel = "mcput_mp", "mcput_sp1", "tcu";
+		sysnoc_trans_mcput_mp_filter@72002E80 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72002E80 0 0x80>;
+		};
+		sysnoc_trans_mcput_sp1_filter@72002F00 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72002F00 0 0x80>;
+		};
+		sysnoc_trans_tcu_filter@72003100 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72003100 0 0x80>;
+		};
+		sysnoc_trans_profiler@72003200 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x72003200 0 0x80>;
+		};
+	};
+	sysnoc_trans_probe_2@72002800 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x72002800 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <428>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "spislv_tbu3", "tbu4_snoc";
+		sysnoc_trans_spislv_tbu3_filter@72003000 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72003000 0 0x80>;
+		};
+		sysnoc_trans_tbu4_filter@72003080 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72003080 0 0x80>;
+		};
+		sysnoc_trans_profiler@72003280 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x72003280 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1>;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4>;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 19 4 >;
+			lut =
+				"aon_snoc_sp0/I/0",
+				"dspt_snoc/I/0",
+#ifdef PLATFORM_HAPS
+				"fpga_snoc/I/0",
+#endif
+				"jtag_snoc/I/0",
+				"mcput_snoc_d2d/I/0",
+				"mcput_snoc_mp/I/0",
+				"mcput_snoc_sp0/I/0",
+				"mcput_snoc_sp1/I/0",
+				"mnoc_snoc/I/0",
+				"npu_snoc_sp0/I/0",
+				"npu_snoc_sp1/I/0",
+				"pciet_snoc_p/I/0",
+				"rnoc_snoc/I/0",
+				"spislv_tbu3_snoc/I/0",
+				"tbu4_snoc/I/0",
+				"tcu_snoc/I/0",
+				"RESERVED0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 15 4 >;
+			lut =
+				"snoc_aon/T/0",
+				"snoc_cnoc/T/0",
+				"snoc_ddrt0_p1/T/0",
+				"snoc_ddrt0_p2/T/0",
+				"snoc_ddrt1_p1/T/0",
+				"snoc_ddrt1_p2/T/0",
+				"snoc_dspt/T/0",
+				"snoc_lnoc/T/0",
+				"snoc_mcput_d2d/T/0",
+				"snoc_mnoc/T/0",
+				"snoc_npu/T/0",
+				"snoc_pciet/T/0",
+				"snoc_rnoc/T/0",
+				"snoc_service/T/0",
+				"RESERVED1",
+				"RESERVED2";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 6 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 14 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 1181 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =		/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+#ifdef PLATFORM_HAPS
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <fpga_snoc_I_O        snoc_service_T_O   0x1             0x72000000        >,
+#endif
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x6             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x7             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x6             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x7             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x6             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x7             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x8             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x9             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0xa             0x0               >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0xb             0x0               >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x4             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x5             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x6             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x7             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xf             0x0               >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x4             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x5             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x6             0x5c000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x7             0x7c000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xe             0x0               >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xf             0x0               >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x13            0x0               >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x13            0x0               >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x13            0x0               >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32>;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 18 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0 ",
+				"Prot_1 ",
+				"Prot_2 ",
+				"User_0 ",
+				"User_1 ",
+				"User_2 ",
+				"User_3 ",
+				"User_4 ",
+				"User_5 ",
+				"User_6 ",
+				"qos0",
+				"qos1",
+				"qos2",
+				"qos3";
+		};
+	};
+};
+
+d1_media_noc:d1_media_noc@72021400 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <1>;
+	reg = <0 0x72021400 0 0x4000>;
+	interrupts = <454>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic1>;
+	errlogger,idx = <0 1 3 4 5>;
+	eswin,qos-configs = "GPU", "TBU2", "VC";
+	eswin,GPU-qos-base = <0x72021480>;
+	eswin,GPU-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode */
+		0x10 0x1E0    /* bandwidth */
+		0x14 0x1     /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,TBU2-qos-base = <0x72021500>;
+	eswin,TBU2-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,VC-qos-base = <0x72021580>;
+	eswin,VC-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+#ifdef PLATFORM_HAPS
+	clock,rate = <5000000>;  /*haps ddr controller clk*/
+#endif
+	sideband_manager@72022000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x72022000 0 0x10>;
+		SenseIn0 =
+			<SBM_MNOC_GPU 	OFFSET0>,
+			<SBM_MNOC_TBU2 	OFFSET1>,
+			<SBM_MNOC_VC 	OFFSET2>,
+			<SBM_MNOC_DDRT0_P3 	OFFSET3>,
+			<SBM_MNOC_DDRT1_P3 	OFFSET4>;
+		bf-name =
+			"SBM_MNOC_GPU",
+			"SBM_MNOC_TBU2",
+			"SBM_MNOC_VC",
+			"SBM_MNOC_DDRT0_P3",
+			"SBM_MNOC_DDRT1_P3";
+	};
+
+	mnoc_packet_ddr0_p3_req_probe@72020000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72020000 0 0x4000>;
+		clocks = <&d1_clock EIC7700_CLK_DDRT0_P3_ACLK>;
+		clock-names = "clk";
+		interrupts = <458>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p3_req";
+	};
+	mnoc_packet_ddr1_p3_req_probe@72020800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72020800 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT1_P3_ACLK>;
+		clock-names = "clk";
+		interrupts = <456>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p3_req";
+	};
+	mnoc_trans_probe@72021000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x72021000 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <453>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <3>;
+		counter,nr = <12>;
+		profiler,nr = <1>;
+		portsel = "gpu", "tbu2", "vc";
+		mnoc_trans_gpu_filter@72021600 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72021600 0 0x80>;
+		};
+		mnoc_trans_tbu2_filter@72021680 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72021680 0 0x80>;
+		};
+		mnoc_trans_vc_filter@72021700 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72021700 0 0x80>;
+		};
+		mnoc_trans_profiler@72021780 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x72021780 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"gpu_mnoc/I/0",
+				"snoc_mnoc/I/0",
+				"tbu2_mnoc/I/0",
+				"vc_mnoc/I/0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"mnoc_ddrt0_p3/T/0",
+				"mnoc_ddrt1_p3/T/0",
+				"mnoc_service/T/0",
+				"mnoc_snoc/T/0";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 3 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 7 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 55 >;			/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+					/bits/ 64  <0x0  0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x0  0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x0  0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x0  0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x0  0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x0  0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x0  0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x0  0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x0  0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x0  0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x0  0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x0  0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x0  0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x0  0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x0  0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x0  0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x0  0x3 0x0           0x0              >,
+					/bits/ 64  <0x1 0x2  0x0           0x52020000       >,
+					/bits/ 64  <0x1 0x2  0x1           0x72020000       >,
+					/bits/ 64  <0x1 0x2  0x2           0x0              >,
+					/bits/ 64  <0x1 0x2  0x3           0x0              >,
+					/bits/ 64  <0x2 0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x2 0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x2 0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x2 0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x2 0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x2 0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x2 0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x2 0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x2 0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x2 0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x2 0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x2 0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x2 0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x2 0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x2 0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x2 0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x2 0x3   0x0           0x0              >,
+					/bits/ 64  <0x3   0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x3   0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x3   0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x3   0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x3   0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x3   0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x3   0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x3   0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x3   0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x3   0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x3   0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x3   0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x3   0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x3   0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x3   0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x3   0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x3   0x3 0x0           0x0              >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 18 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0 ",
+				"Prot_1 ",
+				"Prot_2 ",
+				"User_0 ",
+				"User_1 ",
+				"User_2 ",
+				"User_3 ",
+				"User_4 ",
+				"User_5 ",
+				"User_6 ",
+				"qos0",
+				"qos1",
+				"qos2",
+				"qos3";
+		};
+	};
+};
+
+d1_realtime_noc:d1_realtime_noc@72041400 {
+	compatible = "eswin,win2030-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	numa-node-id = <1>;
+	reg = <0 0x72041400 0 0x4000>;
+	interrupts = <448>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic1>;
+	errlogger,idx = <0 1 3 4 5>;
+	eswin,qos-configs = "TBU0", "VO";
+	eswin,TBU0-qos-base = <0x72041480>;
+	eswin,TBU0-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode */
+		0x10 0x1E0    /* bandwidth */
+		0x14 0x1     /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,VO-qos-base = <0x72041500>;
+	eswin,VO-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+#ifdef PLATFORM_HAPS
+	clock,rate = <5000000>;  /*haps ddr controller clk*/
+#endif
+	sideband_manager@72042000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x72042000 0 0x10>;
+		SenseIn0 =
+			<SBM_RNOC_TBU0 	OFFSET0>,
+			<SBM_RNOC_VO 	OFFSET1>,
+			<SBM_RNOC_DDRT0_P4 	OFFSET2>,
+			<SBM_RNOC_DDRT1_P4 	OFFSET3>;
+		bf-name =
+			"SBM_RNOC_TBU0",
+			"SBM_RNOC_VO",
+			"SBM_RNOC_DDRT0_P4",
+			"SBM_RNOC_DDRT1_P4";
+	};
+
+	rnoc_packet_ddr0_p4_req_probe@72040000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72040000 0 0x4000>;
+		clocks = <&d1_clock EIC7700_CLK_DDRT0_P4_ACLK>;
+		clock-names = "clk";
+		interrupts = <452>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p4_req";
+	};
+	rnoc_packet_ddr1_p4_req_probe@72040800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x72040800 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_DDRT1_P4_ACLK>;
+		clock-names = "clk";
+		interrupts = <450>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p4_req";
+	};
+	rnoc_trans_probe@72041000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x72041000 0 0x4000>;
+		clocks =<&d1_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <447>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic1>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "tbu0", "vo";
+		rnoc_trans_tbu0_filter@72041580 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72041580 0 0x80>;
+		};
+		rnoc_trans_vo_filter@72041600 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x72041600 0 0x80>;
+		};
+		rnoc_trans_profiler@72041680 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x72041680 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,win2030,register", "eswin,win2030,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"snoc_rnoc/I/0",
+				"tbu0_rnoc/I/0",
+				"vo_rnoc/I/0",
+				"RESERVED0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"rnoc_ddrt0_p4/T/0",
+				"rnoc_ddrt1_p4/T/0",
+				"rnoc_service/T/0",
+				"rnoc_snoc/T/0";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 3 >;
+		};
+
+		SeqId {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,win2030,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 9 7 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 40 >;			/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+					/bits/ 64  <0x0 0x2  0x0           0x52040000       >,
+					/bits/ 64  <0x0 0x2  0x1           0x72040000       >,
+					/bits/ 64  <0x0 0x2  0x2           0x0              >,
+					/bits/ 64  <0x0 0x2  0x3           0x0              >,
+					/bits/ 64  <0x1 0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x1 0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x1 0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x1 0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x1 0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x1 0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x1 0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x1 0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x1 0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x1 0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x1 0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x1 0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x1 0x3     0x0           0x59000000       >,
+					/bits/ 64  <0x1 0x3     0x1           0x79000000       >,
+					/bits/ 64  <0x1 0x3     0x2           0x14001000000    >,
+					/bits/ 64  <0x1 0x3     0x3           0x14009000000    >,
+					/bits/ 64  <0x1 0x3     0x4           0x0              >,
+					/bits/ 64  <0x1 0x3     0x5           0x0              >,
+					/bits/ 64  <0x2   0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x2   0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x2   0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x2   0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x2   0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x2   0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x2   0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x2   0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x2   0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x2   0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x2   0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x2   0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x2   0x3     0x0           0x59000000       >,
+					/bits/ 64  <0x2   0x3     0x1           0x79000000       >,
+					/bits/ 64  <0x2   0x3     0x2           0x14001000000    >,
+					/bits/ 64  <0x2   0x3     0x3           0x14009000000    >,
+					/bits/ 64  <0x2   0x3     0x4           0x0              >,
+					/bits/ 64  <0x2   0x3     0x5           0x0              >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,win2030,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,win2030,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,win2030,bitfield";
+			offset,length = < 0x0 16 >;
+			lut =
+				 "Cache_0",
+				 "Cache_1",
+				 "Cache_2",
+				 "Cache_3",
+				 "Prot_0 ",
+				 "Prot_1 ",
+				 "Prot_2 ",
+				 "User_0 ",
+				 "User_1 ",
+				 "User_2 ",
+				 "User_3 ",
+				 "User_4 ",
+				 "qos0",
+				 "qos1",
+				 "qos2",
+				 "qos3";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-die1-soc.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-die1-soc.dtsi
new file mode 100644
index 000000000000..c2c51545ae2a
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-die1-soc.dtsi
@@ -0,0 +1,2653 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Die1 System peripherals of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "eswin-win2030-arch-d2d.dtsi"
+
+#include <dt-bindings/memory/eswin-win2030-sid.h>
+#include <dt-bindings/mailbox/eswin-mailbox.h>
+#include <dt-bindings/reset/eswin,eic7700-reset.h>
+#include <dt-bindings/clock/eswin,eic7700-clock.h>
+#include <dt-bindings/i2c/i2c.h>
+#include <dt-bindings/interconnect/eswin,win2030.h>
+#include <dt-bindings/thermal/thermal.h>
+#include <dt-bindings/power/eswin,eic770x-pmu.h>
+#include <dt-bindings/pwm/pwm.h>
+/ {
+	thermal-zones {
+		d1_thermal0 {
+			polling-delay-passive = <100>; /*ms*/
+			polling-delay = <1000>; /*ms*/
+			sustainable-power = <30000>; /*mW*/
+			thermal-sensors = <&d1_pvt0>;
+
+			trips {
+				d1_threshold: trip-point0 {
+					temperature = <65000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				d1_target: trip-point1 {
+					temperature = <75000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				d1_crit: trip-point2 {
+					temperature     = <105000>;
+					hysteresis      = <0>;
+					type            = "critical";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					trip = <&d1_target>;
+					contribution = <1024>;
+					cooling-device =
+						<&cpu_4 0 10>,
+						<&cpu_5 0 10>,
+						<&cpu_6 0 10>,
+						<&cpu_7 0 10>;
+				};
+				map1 {
+					trip = <&d1_target>;
+					contribution = <1024>;
+					cooling-device = <&d1_npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				/*
+				map2 {
+					trip = <&d1_target>;
+					contribution = <1024>;
+					cooling-device = <&d1_gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				*/
+			};
+		};
+	};
+};
+
+&SOC {
+		d1_uart0: serial@0x70900000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x70900000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic1>;
+			interrupts = <UART0_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <1>;
+		};
+
+		d1_uart1: serial@0x70910000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x70910000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic1>;
+			interrupts = <UART1_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <1>;
+		};
+
+		d1_uart2: serial@0x70920000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x70920000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic1>;
+			interrupts = <UART2_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+
+		d1_uart3: serial@0x70930000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x70930000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic1>;
+			interrupts = <UART3_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+
+		d1_uart4: serial@0x70940000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x70940000 0x0 0x10000>;
+			clock-frequency = <LSPCLK_FREQ>;
+			interrupt-parent = <&plic1>;
+			interrupts = <UART4_INT>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+
+		d1_sys_con: scu_sys_con@0x71810000 {
+			compatible = "eswin,win2030-scu-sys-con", "syscon";
+			#syscon-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x71810000 0x0 0x8000>;
+			numa-node-id = <1>;
+			d1_noc_wdt:noc@71810324 {
+				compatible = "eswin,win2030-noc-wdt";
+				interrupt-parent = <&plic1>;
+				interrupts = <392>, <393>, <394>, <395>,
+						<396>, <397>, <398>, <399>, <400>,
+						<401>, <402>, <403>, <404>, <405>,
+						<406>, <407>, <408>, <409>, <410>,
+						<411>, <412>, <413>, <414>, <415>,
+						<416>, <417>, <418>, <419>, <420>,
+						<421>, <422>, <423>, <424>, <425>,
+						<426>;
+				eswin,syscrg_csr = <&d1_sys_crg 0x100 0xffff>; //timeout paramerter
+				numa-node-id = <1>;
+			};
+		};
+
+		d1_sys_crg: sys-crg@71828000 {
+			compatible = "syscon", "simple-mfd";
+			reg = <0x000000 0x71828000 0x000000 0x80000>;
+			numa-node-id = <1>;
+			d1_reset: reset-controller {
+				compatible = "eswin,eic7700-reset";
+				#reset-cells = <2>;
+				numa-node-id = <1>;
+			};
+			d1_clock: clock-controller {
+				compatible = "eswin,eic7700-clock";
+				#clock-cells = <1>;
+				numa-node-id = <1>;
+			};
+		};
+
+		d1_hsp_sp_csr: hsp_sp_top_csr@0x70440000 {
+			compatible = "eswin,win2030-hsp-sp-csr", "syscon";
+			#size-cells = <2>;
+			reg = <0x0 0x70440000 0x0 0x2000>;
+			numa-node-id = <1>;
+		};
+
+		smmu1: iommu@70c00000 {
+			compatible = "arm,smmu-v3";
+			reg = <0x0 0x70c00000 0x0 0x100000>;
+			eswin,syscfg = <&d1_sys_con 0x3fc>;
+			interrupt-parent = <&plic1>;
+			interrupts = <356>,
+				<360>,
+				<357>,
+				<358>;
+			interrupt-names = "eventq", "gerror", "priq", "cmdq-sync";
+			#iommu-cells = <1>;
+			resets = <&d1_reset TCU_RST_CTRL SW_TCU_AXI_RSTN>,
+				<&d1_reset TCU_RST_CTRL SW_TCU_CFG_RSTN>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_0>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_1>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_2>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_3>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_4>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_5>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_6>,
+				<&d1_reset TCU_RST_CTRL TBU_RSTN_7>;
+			reset-names = "axi_rst", "cfg_rst", "tbu0_rst", "tbu1_rst", "tbu2_rst", "tbu3_rst",
+				"tbu4_rst", "tbu5_rst", "tbu6_rst", "tbu7_rst";
+			status = "disabled";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		smmu_pmu1: pmu@70c02000 {
+			compatible = "arm,smmu-v3-pmcg";
+			reg = <0x0 0x70c02000 0x0 0x1000>,
+					<0x0 0x70c22000 0x0 0x1000>;
+			eswin,syscfg = <&d1_sys_con 0x3fc>;
+			interrupt-parent = <&plic1>;
+			interrupts = <363>;
+			status = "disabled";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		dev_foo_for_die1_mapping: E21@2 {
+			compatible = "riscv,dev-foo-die1";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+			iommus = <&smmu1 WIN2030_SID_DEV_FOO_FOR_DIE1>;
+			status = "disabled";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_pmu: power-controller@71808000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "eswin,win2030-pmu-controller";
+			reg = <0x0 0x71808000 0x0 0x8000>;
+			numa-node-id = <1>;
+
+			d1_pmu_pcie: win2030-pmu-controller-port@0 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0x0>;
+				power_status = <1>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_PCIE";
+			};
+			d1_pmu_dsp1: win2030-pmu-controller-port@40 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0x40>;
+				power_status = <0>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_DSP1";
+			};
+			d1_pmu_vi: win2030-pmu-controller-port@80 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0x80>;
+				power_status = <1>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_VI";
+			};
+			d1_pmu_vo: win2030-pmu-controller-port@c0 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0xc0>;
+				power_status = <0>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_VO";
+			};
+			d1_pmu_codec: win2030-pmu-controller-port@140 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0x140>;
+				power_status = <0>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_CODEC";
+			};
+			d1_pmu_dsp2: win2030-pmu-controller-port@200 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0x200>;
+				power_status = <0>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_DSP2";
+			};
+			d1_pmu_dsp3: win2030-pmu-controller-port@240 {
+				compatible = "eswin,win2030-pmu-controller-port";
+				reg_base = <0x240>;
+				power_status = <0>;
+				power_delay = <6 6 3 3>;
+				clock_delay = <4 2 2 2>;
+				reset_delay = <2 4 2 2>;
+				clamp_delay = <3 3 2 2>;
+				label = "D1_DSP3";
+			};
+		};
+
+		d1_aon_dmac: dma-controller-aon@0x718c0000 {
+			compatible = "eswin,eic770x-axi-dma";
+			reg = <0x0 0x718c0000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <289>;
+			#dma-cells = <3>; // dma-cells <request channel id, handshake number, dma sel bit>;
+			clocks = <&d1_clock EIC7700_CLK_AONDMA_ACLK>,
+				 <&d1_clock EIC7700_CLK_AONDMA_CFG>;
+			clock-names = "core-clk", "cfgr-clk";
+			resets = <&d1_reset DMA1_RST_CTRL SW_DMA1_ARSTN>,
+				<&d1_reset DMA1_RST_CTRL SW_DMA1_HRSTN>;
+			reset-names = "arst", "prst";
+			dma-channels = <16>;
+			snps,dma-masters = <2>;
+			snps,priority = <0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15>;
+			snps,data-width = <3>;
+			snps,block-size = <0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000>;
+			snps,axi-max-burst-len = <32>;
+			snps,max-msize = <64>;
+			#size-cells = <2>;
+			#address-cells = <2>;
+			dma-ranges = <0x0 0x80000000 0x0 0x80000000 0x100 0x0>;
+			iommus = <&smmu1 WIN2030_SID_DMA1>;
+			tbus = <WIN2030_TBUID_DMA1>;
+			eswin,syscfg = <&d1_sys_con DMA1_SID_REG_OFFSET 0x370>;
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_dmac0: dma-controller-hsp@0x70430000 {
+			compatible = "eswin,eic770x-axi-dma";
+			reg = <0x0 0x70430000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <57>;
+			#dma-cells = <3>; // dma-cells <request channel id, handshake number, dma sel bit>;
+			clocks = <&d1_clock EIC7700_CLK_HSP_DMA0_CLK>,
+					 <&d1_clock EIC7700_CLK_HSP_DMA0_CLK_TEST>;
+			clock-names = "core-clk", "cfgr-clk";
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_HSP_DMA0_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_DMA_PRSTN>;
+			reset-names = "arst", "prst";
+			dma-channels = <12>;
+			snps,dma-masters = <1>;
+			snps,priority = <0 1 2 3 4 5 6 7 8 9 10 11>;
+			snps,data-width = <2>;
+			snps,block-size = <0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000 0x80000>;
+			snps,axi-max-burst-len = <16>;
+			snps,max-msize = <64>;
+			iommus = <&smmu1 WIN2030_SID_DMA0>;
+			tbus = <WIN2030_TBUID_DMA0>;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x104c>;
+			eswin,syscfg = <&d1_sys_con DMA1_SID_REG_OFFSET 0x370>;
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		noc {
+			compatible = "eswin,noc","simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			numa-node-id = <1>;
+			#include "eswin-win2030-die1-noc.dtsi"
+		};
+		vdec1: video-decoder1@70100000 {
+			compatible = "eswin,video-decoder1";
+			clocks = <&d1_clock EIC7700_CLK_VC_ACLK>,
+					 <&d1_clock EIC7700_CLK_VC_CFG_CLK>,
+					 <&d1_clock EIC7700_CLK_VC_JD_CLK>,
+					 <&d1_clock EIC7700_CLK_VC_VD_CLK>,
+					 <&d1_clock EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE>,
+					 <&d1_clock EIC7700_SPLL0_FOUT1>,
+					 <&d1_clock EIC7700_SPLL2_FOUT1>,
+					 <&d1_clock EIC7700_CLK_VC_JD_PCLK>,
+					 <&d1_clock EIC7700_CLK_VC_VD_PCLK>,
+					 <&d1_clock EIC7700_CLK_VC_MON_PCLK>;
+			clock-names = "aclk", "cfg_clk", "jd_clk", "vd_clk", "vc_mux", "spll0_fout1", "spll2_fout1", "jd_pclk", "vd_pclk", "mon_pclk";
+			resets = <&d1_reset VC_RST_CTRL SW_VC_AXI_RSTN>,
+					 <&d1_reset VC_RST_CTRL SW_VC_CFG_RSTN>,
+					 <&d1_reset VC_RST_CTRL SW_VC_MONCFG_RSTN>,
+					 <&d1_reset JD_RST_CTRL SW_JD_CFG_RSTN>,
+					 <&d1_reset JD_RST_CTRL SW_JD_AXI_RSTN>,
+					 <&d1_reset VD_RST_CTRL SW_VD_CFG_RSTN>,
+					 <&d1_reset VD_RST_CTRL SW_VD_AXI_RSTN>;
+			reset-names = "axi", "cfg", "moncfg", "jd_cfg", "jd_axi", "vd_cfg", "vd_axi";
+			eswin,syscfg = <&d1_sys_con 0x0 0x4>;
+			operating-points-v2 = <&vdec_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <1000>; /*TBD*/
+			vcmd-core = <0 0x6c>;
+			axife-core = <0x200 0x100>;
+			vdec-core = <0x800 0xc00>;
+			interrupt-parent = <&plic1>;
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x0 0x0 0x80000000 0x200 0x0>;
+			iommus = <&smmu1 WIN2030_SID_VDEC>;
+			vccsr-reg = <0x0 0x701c0000 0x0 0x1000>;
+			numa-node-id = <1>;
+			tbus = <WIN2030_TBUID_VDEC>, <WIN2030_TBUID_JDEC>;
+			dma-noncoherent;
+
+			vdec_1: vdec1@70100000 {
+				core-name = "video-dec0";
+				base-addr = <0x70100000>;
+				interrupts = <236>;
+			};
+
+			jdec_1: jdec1@70120000 {
+				core-name = "jpeg-dec0";
+				base-addr = <0x70120000>;
+				interrupts = <237>;
+			};
+		};
+
+		venc1: video-encoder@70110000 {
+			compatible = "eswin,video-encoder1";
+			clocks = <&d1_clock EIC7700_CLK_VC_ACLK>,
+					 <&d1_clock EIC7700_CLK_VC_CFG_CLK>,
+					 <&d1_clock EIC7700_CLK_VC_JE_CLK>,
+					 <&d1_clock EIC7700_CLK_VC_VE_CLK>,
+					 <&d1_clock EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE>,
+					 <&d1_clock EIC7700_SPLL0_FOUT1>,
+					 <&d1_clock EIC7700_SPLL2_FOUT1>,
+					 <&d1_clock EIC7700_CLK_VC_JE_PCLK>,
+					 <&d1_clock EIC7700_CLK_VC_VE_PCLK>,
+					 <&d1_clock EIC7700_CLK_VC_MON_PCLK>;
+			clock-names = "aclk", "cfg_clk", "je_clk", "ve_clk", "vc_mux", "spll0_fout1", "spll2_fout1", "je_pclk", "ve_pclk", "mon_pclk";
+			resets = <&d1_reset VC_RST_CTRL SW_VC_AXI_RSTN>,
+					 <&d1_reset VC_RST_CTRL SW_VC_CFG_RSTN>,
+					 <&d1_reset VC_RST_CTRL SW_VC_MONCFG_RSTN>,
+					 <&d1_reset JE_RST_CTRL SW_JE_CFG_RSTN>,
+					 <&d1_reset JE_RST_CTRL SW_JE_AXI_RSTN>,
+					 <&d1_reset VE_RST_CTRL SW_VE_CFG_RSTN>,
+					 <&d1_reset VE_RST_CTRL SW_VE_AXI_RSTN>;
+			reset-names = "axi", "cfg", "moncfg", "je_cfg", "je_axi", "ve_cfg", "ve_axi";
+			eswin,syscfg = <&d1_sys_con 0x0 0x4>;
+			operating-points-v2 = <&venc_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <1000>; /*TBD*/
+			vcmd-core = <0 0x6c>;
+			axife-core = <0x2000 0x7d0>;
+			venc-core = <0x1000 0x87c>;
+			interrupt-parent = <&plic1>;
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x0 0x0 0x80000000 0x200 0x0>;
+			iommus = <&smmu1 WIN2030_SID_VENC>;
+			vccsr-reg = <0x0 0x701c0000 0x0 0x1000>;
+			numa-node-id = <1>;
+			dma-noncoherent;
+
+			venc_1: venc0@70110000 {
+				core-name = "video-enc0";
+				base-addr = <0x70110000>;
+				interrupts = <229>;
+				tbus = <WIN2030_TBUID_VENC>;
+			};
+
+			jenc_1: jenc0@70130000 {
+				core-name = "jpeg-enc0";
+				base-addr = <0x70130000>;
+				interrupts = <232>;
+				tbus = <WIN2030_TBUID_JENC>;
+			};
+		};
+		/*mailbox between u84 & scpu*/
+		d1_mbox0: mbox@70a00000 {
+			compatible = "eswin,win2030-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_SCPU_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_SCPU_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <117>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_0>,
+				<&d1_clock EIC7700_CLK_MAILBOX_1>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_0>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_1>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit = <ESWIN_MAIBOX_SCPU_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & lpcpu*/
+		d1_mbox1: mbox@70a20000 {
+			compatible = "eswin,win2030-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_LPCPU_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_LPCPU_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <119>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_2>,
+				<&d1_clock EIC7700_CLK_MAILBOX_3>;
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_2>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_3>;
+			reset-names = "rst", "rst_device";
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_LPCPU_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & npu_0*/
+		d1_mbox2: mbox@70a40000 {
+			compatible = "eswin,npu1-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_NPU_0_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <121>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_4>,
+				<&d1_clock EIC7700_CLK_MAILBOX_5>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_4>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_5>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_NPU_0_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & npu_1*/
+		d1_mbox3: mbox@70a60000 {
+			compatible = "eswin,win2030-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_NPU_1_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_NP1_0_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <123>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_6>,
+				<&d1_clock EIC7700_CLK_MAILBOX_7>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_6>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_7>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_NPU_1_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & dsp_0*/
+		d1_mbox4: mbox@70a80000 {
+			compatible = "eswin,dsp0-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_DSP_0_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_DSP_0_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <125>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_8>,
+				<&d1_clock EIC7700_CLK_MAILBOX_9>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_8>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_9>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_0_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & dsp_1*/
+		d1_mbox5: mbox@70aa0000 {
+			compatible = "eswin,dsp1-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_DSP_1_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_DSP_1_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <127>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_10>,
+				<&d1_clock EIC7700_CLK_MAILBOX_11>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_10>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_11>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_1_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & dsp_2*/
+		d1_mbox6: mbox@70ac0000 {
+			compatible = "eswin,dsp2-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_DSP_2_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_DSP_2_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <129>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_12>,
+				<&d1_clock EIC7700_CLK_MAILBOX_13>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_12>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_13>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_2_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		/*mailbox between u84 & dsp_3*/
+		d1_mbox7: mbox@70ae0000 {
+			compatible = "eswin,dsp3-mailbox";
+			reg = <0 (ESWIN_MAILBOX_U84_TO_DSP_3_REG_BASE + 0x20000000) 0 0x10000>,
+				  <0 (ESWIN_MAILBOX_DSP_3_TO_U84_REG_BASE + 0x20000000) 0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <131>;
+			#mbox-cells = <1>;
+			clocks = <&d1_clock EIC7700_CLK_MAILBOX_14>,
+				<&d1_clock EIC7700_CLK_MAILBOX_15>;
+			clock-names = "pclk_mailbox_host", "pclk_mailbox_device";
+			resets = <&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_14>,
+				<&d1_reset MBOX_RST_CTRL SW_MBOX_RST_N_15>;
+			reset-names = "rst", "rst_device";
+			lock-bit = <ESWIN_MAILBOX_WR_LOCK_BIT_U84>;
+			irq-bit  = <ESWIN_MAIBOX_DSP_3_IRQ_BIT>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		d1_ipc_scpu:ipc@1 {
+			compatible = "eswin,win2030-ipc";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x80000000 0x0 0xc0000000 0x0 0x80000000>;
+			iommus = <&smmu1 WIN2030_SID_SCPU>, <&smmu1 WIN2030_SID_CRYPT>;
+			tbus = <WIN2030_TBUID_SCPU>;
+			eswin,syscfg = <&d1_sys_con SCPU_SID_REG_OFFSET 0>,
+						   <&d1_sys_con CRYPT_SID_REG_OFFSET 0>;
+			mboxes = <&d1_mbox0 0>;
+			mbox-names = "u84_scpu";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_lpcpu:lpcpu@1 {
+			compatible = "eswin,win2030-lpcpu";
+			/* fw-region = <&lpcpu1_reserved>; */
+			clocks = <&d1_clock EIC7700_CLK_CLK_LPCPU_CORE>,
+					<&d1_clock EIC7700_CLK_CLK_LPCPU_BUS>;
+			clock-names = "core_clk", "bus_clk";
+
+			reset-names = "core_rst", "bus_rst", "dbg_rst";
+			resets = <&d1_reset LPCPU_RST_CTRL SW_LPCPU_CORE_RSTN>,
+					<&d1_reset LPCPU_RST_CTRL SW_LPCPU_BUS_RSTN>,
+					<&d1_reset LPCPU_RST_CTRL SW_LPCPU_DBG_RSTN>;
+			#size-cells = <2>;
+
+			dma-ranges = <0x0 0xb0000000 0x0 0xc0000000 0x0 0x50000000>;
+			iommus = <&smmu1 WIN2030_SID_LCPU>;
+			eswin,syscfg = <&d1_sys_con LCPU_SID_REG_OFFSET 0>;
+			tbus = <WIN2030_TBUID_LPCPU>;
+			mboxes = <&d1_mbox1 0>;
+			mbox-names = "u84_lpcpu";
+			numa-node-id = <1>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		d1_pvt0: pvt@0x70b00000 {
+			compatible = "eswin,eswin-pvt-cpu";
+			clocks = <&d1_clock EIC7700_CLK_PVT_CLK_0>;
+			clock-names = "pvt_clk";
+			resets = <&d1_reset PVT_RST_CTRL SW_PVT_RST_N_0>;
+			reset-names = "pvt_rst";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x70b00000 0x0 0x10000>;
+			interrupts = <349>;
+			interrupt-parent = <&plic1>;
+			#thermal-sensor-cells = <0>;
+			status = "disabled";
+			label = "d1_pvt0";
+			numa-node-id = <1>;
+		};
+		d1_pvt1: pvt@0x72360000 {
+			compatible = "eswin,eswin-pvt-ddr";
+			clocks = <&d1_clock EIC7700_CLK_PVT_CLK_1>;
+			clock-names = "pvt_clk";
+			resets = <&d1_reset PVT_RST_CTRL SW_PVT_RST_N_1>;
+			reset-names = "pvt_rst";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x72360000 0x0 0x20000>;
+			interrupts = <350>;
+			interrupt-parent = <&plic1>;
+			#thermal-sensor-cells = <0>;
+			status = "disabled";
+			label = "d1_pvt1";
+			numa-node-id = <1>;
+		};
+
+		d1_fan_control: fan_control@70b50000 {
+			compatible = "eswin-fan-control";
+			reg = <0x0 0x50b50000 0x0 0x10000>,
+					<0x0 0x71810668 0x0 0x10>,
+					<0x0 0x51810668 0x0 0x10>;
+			clocks = <&d1_clock EIC7700_CLK_LSP_FAN_PCLK>;
+			clock-names = "pclk";
+			resets = <&d1_reset FAN_RST_CTRL SW_FAN_RST_N>;
+			reset-names = "fan_rst";
+			interrupt-parent = <&plic1>;
+			interrupt-names = "fanirq";
+			interrupts = <354>;
+			pulses-per-revolution = <1>;
+			pwm-minimum-period = <1000>;
+			pwms = <&d1_pwm0 0 100000 0>;
+			status = "disabled";
+			label = "d1_fan_control";
+			numa-node-id = <1>;
+		};
+
+		d1_i2c0: i2c@70950000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C0_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_0>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x70950000 0x0 0x8000>;
+				interrupts = <105>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				eswin,i2c_dma = <&d1_aon_dmac>;
+				dma-names = "rx", "tx";
+				/*
+				* dmas : DMA specifiers
+				* &d1_aon_dmac : dma controller
+				* 24 : i2c0  aon dma handshake number
+				* 6 : i2c0 dma controller sel bit in sys_son dma_cfg reg(offset 0x370)
+				*/
+				dmas = <&d1_aon_dmac 10 24 6>, <&d1_aon_dmac 11 25 6>;
+				numa-node-id = <1>;
+		};
+		d1_i2c1: i2c@70960000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C1_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_1>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x70960000 0x0 0x8000>;
+				interrupts = <106>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c2: i2c@70970000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C2_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_2>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x70970000 0x0 0x8000>;
+				interrupts = <107>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+				/*
+				d1_es8316: es8316@10 {
+					compatible = "everest,es8316";
+					reg = <0x10>;
+					interrupts = <107>;
+					interrupt-parent = <&plic1>;
+					#sound-dai-cells = <0>;
+					port {
+						d1_codec_endpoint: endpoint {
+							system-clock-frequency = <12288000>;
+							remote-endpoint = <&d1_i2s0_endpoint>;
+						};
+					};
+				};
+				*/
+		};
+		d1_i2c3: i2c@70980000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C3_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_3>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x70980000 0x0 0x8000>;
+				interrupts = <108>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c4: i2c@70990000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C4_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_4>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x70990000 0x0 0x8000>;
+				interrupts = <109>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c5: i2c@709a0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C5_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_5>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x709a0000 0x0 0x8000>;
+				interrupts = <110>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c6: i2c@709b0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C6_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_6>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x709b0000 0x0 0x8000>;
+				interrupts = <111>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c7: i2c@709c0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C7_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_7>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x709c0000 0x0 0x8000>;
+				interrupts = <112>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c8: i2c@709d0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C8_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_8>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x709d0000 0x0 0x8000>;
+				interrupts = <113>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+		d1_i2c9: i2c@709e0000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_LSP_I2C9_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C_RST_CTRL SW_I2C_RST_N_9>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x709e0000 0x0 0x8000>;
+				interrupts = <114>;
+				interrupt-parent = <&plic1>;
+				status = "disabled";
+				numa-node-id = <1>;
+		};
+
+		d1_aon_i2c0: i2c@71830000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_AON_I2C0_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C0_RST_CTRL SW_I2C0_PRSTN>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x71830000 0x0 0x8000>;
+				interrupts = <290>;
+				interrupt-parent = <&plic1>;
+				eswin,i2c_dma = <&d1_aon_dmac>;
+				dma-names = "rx", "tx";
+				/*
+				* dmas : DMA specifiers
+				* &d1_aon_dmac : dma controller
+				* 41 : i2c0 aon dma handshake number
+				* 0xff : no need to select to dma controller
+				*/
+				dmas = <&d1_aon_dmac 12 41 0xff>, <&d1_aon_dmac 13 42 0xff>;
+				numa-node-id = <1>;
+		};
+		d1_aon_i2c1: i2c@71838000 {
+				compatible = "snps,designware-i2c";
+				clock-frequency = <I2C_BITRATE_STANDARD>;
+				clocks = <&d1_clock EIC7700_CLK_AON_I2C1_PCLK>;
+				clock-names = "pclk";
+				resets = <&d1_reset I2C1_RST_CTRL SW_I2C1_PRSTN>;
+				reset-names = "rst";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x0 0x71838000 0x0 0x8000>;
+				interrupts = <291>;
+				interrupt-parent = <&plic1>;
+				numa-node-id = <1>;
+		};
+
+		d1_npu: eswin-npu@71c00000 {
+			compatible = "eswin,npu";
+			reg = <0x0 0x71c00000 0x0 0x400000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <387 16>;
+			/* spram-region = <&npu1_reserved>;*/
+			#size-cells = <2>;
+			dma-ranges = <0x1 0x0 0x0 0xc0000000 0x1ff 0x0>;
+			iommus = <&smmu1 WIN2030_SID_NPU_DMA>;
+			tbus = <WIN2030_TBUID_NPU>;
+			dsp-avail-num = <1>;
+			spram-size = <0x400000>;
+			npu_mbox = <&d1_mbox2>;
+			clocks = <&d1_clock EIC7700_CLK_NPU_ACLK>,
+					<&d1_clock EIC7700_CLK_NPU_CFG_CLK>,
+					<&d1_clock EIC7700_CLK_NPU_CLK>,
+					<&d1_clock EIC7700_CLK_NPU_LLC_ACLK>,
+					<&d1_clock EIC7700_CLK_NPU_E31_CLK>,
+					<&d1_clock EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE>,
+					<&d1_clock EIC7700_MUX_U_NPU_LLCLK_3MUX1_GFREE>,
+					<&d1_clock EIC7700_SPLL2_FOUT2>,
+					<&d1_clock EIC7700_SPLL1_FOUT1>,
+					<&d1_clock EIC7700_CLK_CLK_NPU_LLC_SRC0>,
+					<&d1_clock EIC7700_VPLL_FOUT1>;
+
+
+			clock-names = "aclk", "cfg_clk", "core_clk", "clk_npu_llc_aclk", "e31_core_clk",
+					"mux_u_npu_core_3mux1_gfree", "mux_u_npu_llclk_3mux1_gfree",
+					"fixed_rate_clk_spll2_fout2", "fixed_rate_clk_spll1_fout1", "clk_clk_npu_llc_src0",
+					"fixed_rate_clk_vpll_fout1";
+
+
+			resets = <&d1_reset NPU_RST_CTRL SW_NPU_E31CORE_RSTN>;
+			reset-names = "e31_core";
+
+			numa-node-id = <1>;
+			firmware-name = "eic7700_die1_e31_fw";
+			dma-noncoherent;
+
+			operating-points-v2 = <&npu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <16000>; /*TBD*/
+		};
+
+		d1_llc_dev: llc@71c00000 {
+			compatible = "eswin,llc";
+			reg = <0x0 0x71c00000 0x0 0x400000>;
+			eswin,syscfg = <&d1_sys_con 0x324>;
+			eswin,syscrg_csr = <&d1_sys_crg>;
+			clocks = <&d1_clock EIC7700_CLK_NPU_ACLK>,
+				 <&d1_clock EIC7700_CLK_NPU_CFG_CLK>,
+				 <&d1_clock EIC7700_CLK_NPU_LLC_ACLK>,
+				 <&d1_clock EIC7700_CLK_NPU_CLK>,
+				 <&d1_clock EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE>,
+				 <&d1_clock EIC7700_SPLL2_FOUT2>,
+				 <&d1_clock EIC7700_SPLL1_FOUT1>;
+			clock-names = "aclk", "cfg_clk", "llc_clk", "core_clk",
+					"mux_u_npu_core_3mux1_gfree", "fixed_rate_clk_spll2_fout2",
+					"fixed_rate_clk_spll1_fout1";
+			resets = <&d1_reset NPU_RST_CTRL SW_NPU_AXI_RSTN>,
+				 <&d1_reset NPU_RST_CTRL SW_NPU_CFG_RSTN>,
+				 <&d1_reset NPU_RST_CTRL SW_NPU_CORE_RSTN>,
+				 <&d1_reset NPU_RST_CTRL SW_NPU_LLC_RSTN>;
+			reset-names = "axi", "cfg", "core", "llc";
+			numa-node-id = <1>;
+			spram-region = <&npu1_reserved>;
+		};
+
+		d1_gpu: gpu@71400000 {
+			compatible = "img,gpu";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x71400000 0x0 0xFFFFF>;
+			clocks = <&d1_clock EIC7700_CLK_GPU_ACLK>,
+					<&d1_clock EIC7700_CLK_GPU_GRAY_CLK>,
+					<&d1_clock EIC7700_CLK_GPU_CFG_CLK>;
+			clock-names = "aclk", "gray_clk", "cfg_clk";
+			resets =<&d1_reset GPU_RST_CTRL SW_GPU_AXI_RSTN>,
+					<&d1_reset GPU_RST_CTRL SW_GPU_CFG_RSTN>,
+					<&d1_reset GPU_RST_CTRL SW_GPU_GRAY_RSTN>,
+					<&d1_reset GPU_RST_CTRL SW_GPU_JONES_RSTN>,
+					<&d1_reset GPU_RST_CTRL SW_GPU_SPU_RSTN>;
+			reset-names = "axi", "cfg", "gray", "jones","spu";
+			interrupt-parent = <&plic1>;
+			interrupts = <15>;
+			dma-noncoherent;
+			numa-node-id = <1>;
+
+			operating-points-v2 = <&gpu_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <4000>; /*TBD*/
+		};
+
+		d1_sata: sata@0x70420000 {
+				compatible = "snps,eswin-ahci";
+				reg = <0x0 0x70420000 0x0 0x10000>;
+				interrupt-parent = <&plic1>;
+				interrupt-names = "intrq", "msi", "pme";
+				interrupts = <58>, <59>, <60>;
+				ports-implemented = <0x1>;
+				resets = <&d1_reset HSPDMA_RST_CTRL SW_SATA_ASIC0_RSTN>,
+ 					<&d1_reset HSPDMA_RST_CTRL SW_SATA_OOB_RSTN>,
+ 					<&d1_reset HSPDMA_RST_CTRL SW_SATA_PMALIVE_RSTN>,
+					<&d1_reset HSPDMA_RST_CTRL SW_SATA_RBC_RSTN>,
+					<&d1_reset HSPDMA_RST_CTRL SW_HSP_SATA_ARSTN>;
+				reset-names = "asic0", "oob", "pmalive", "rbc", "apb";
+				#size-cells = <2>;
+				iommus = <&smmu1 WIN2030_SID_SATA>;
+				tbus = <WIN2030_TBUID_SATA>;
+				dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+				eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1050>;
+				eswin,syscrg_csr = <&d1_sys_crg 0x41c>;
+				numa-node-id = <1>;
+				dma-noncoherent;
+		};
+
+		d1_pinctrl: pinctrl@0x71600080 {
+			compatible = "eswin,eic7x-pinctrl";
+			reg = <0x0 0x71600080 0x0 0x1FFF80>;
+			status = "disabled";
+			numa-node-id = <1>;
+			d1_pinctrl_pwm0_default: pwm0-default{
+				mux{
+					groups = "pwm0_group";
+					function = "pwm0_func";
+				};
+				conf {
+					groups = "pwm0_group";
+					drive-strength = <5>;
+					bias-pull-up = <1>;
+					input-enable = <0>;
+				};
+			};
+			d1_pinctrl_pwm1_default: pwm1-default{
+				mux{
+					groups = "pwm1_group";
+					function = "pwm1_func";
+				};
+			conf {
+				groups = "pwm1_group";
+				drive-strength = <6>;
+				bias-pull-up = <0>;
+				input-enable = <1>;
+				};
+			};
+			d1_pinctrl_pwm2_default: pwm2-default{
+				mux{
+					groups = "pwm2_group";
+					function = "pwm2_func";
+				};
+				conf {
+					groups = "pwm2_group";
+					drive-strength = <7>;
+					bias-pull-down = <0>;
+					input-enable = <0>;
+				};
+			};
+		};
+
+		d1_gpio0: gpio@71600000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "snps,dw-apb-gpio";
+			reg = <0x0 0x71600000 0x0 0x80>;
+			numa-node-id = <1>;
+
+			d1_porta: gpio-port@0 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <32>;
+				reg = <0>;
+				interrupt-parent = <&plic1>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
+								319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334>;
+			};
+
+			d1_portb: gpio-port@1 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <32>;
+				reg = <1>;
+			};
+
+			d1_portc: gpio-port@2 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <32>;
+				reg = <2>;
+			};
+
+			d1_portd: gpio-port@3 {
+				compatible = "snps,dw-apb-gpio-port";
+				gpio-controller;
+				#gpio-cells = <2>;
+				ngpios = <16>;
+				reg = <3>;
+			};
+		};
+
+		d1_timer0: timer@0x71840000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x71840000 0x0 0x8000>;
+			perf_count = <7>;
+			numa-node-id = <1>;
+			interrupt-parent = <&plic1>;
+			interrupts = <345>;
+			clock-names = "pclk","timer_aclk";
+			clocks = <&d1_clock EIC7700_CLK_TIMER_PCLK_0>,
+					 <&d1_clock EIC7700_CLK_TIMER_CLK_0>;
+			resets = <&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_0>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_1>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_2>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_3>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_4>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_5>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_6>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_RSTN_7>,
+				<&d1_reset TIMER0_RST_CTRL SW_TIMER0_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		d1_timer1: timer@0x71848000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x71848000 0x0 0x8000>;
+			numa-node-id = <1>;
+			interrupt-parent = <&plic1>;
+			interrupts = <346>;
+			clock-names = "pclk","timer_aclk";
+			clocks = <&d1_clock EIC7700_CLK_TIMER_PCLK_1>,
+					 <&d1_clock EIC7700_CLK_TIMER_CLK_1>;
+			resets = <&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_0>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_1>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_2>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_3>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_4>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_5>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_6>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_RSTN_7>,
+				<&d1_reset TIMER1_RST_CTRL SW_TIMER1_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+
+		d1_timer2: timer@0x71850000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x71850000 0x0 0x8000>;
+			numa-node-id = <1>;
+			interrupt-parent = <&plic1>;
+			interrupts = <347>;
+			clock-names = "pclk","timer_aclk";
+			clocks = <&d1_clock EIC7700_CLK_TIMER_PCLK_2>,
+					 <&d1_clock EIC7700_CLK_TIMER_CLK_2>;
+			resets = <&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_0>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_1>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_2>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_3>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_4>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_5>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_6>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_RSTN_7>,
+				<&d1_reset TIMER2_RST_CTRL SW_TIMER2_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		d1_timer3: timer@0x71858000 {
+			compatible = "eswin,eswin-timer";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x71858000 0x0 0x8000>;
+			numa-node-id = <1>;
+			interrupt-parent = <&plic1>;
+			interrupts = <348>;
+			clock-names = "pclk","timer_aclk","timer3_clk8";
+			clocks = <&d1_clock EIC7700_CLK_TIMER_PCLK_3>,
+					 <&d1_clock EIC7700_CLK_TIMER_CLK_3>,
+					 <&d1_clock EIC7700_CLK_TIMER3_CLK8>;
+			resets = <&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_0>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_1>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_2>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_3>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_4>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_5>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_6>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_RSTN_7>,
+				<&d1_reset TIMER3_RST_CTRL SW_TIMER3_PRSTN>;
+			reset-names = "trst0","trst1","trst2","trst3","trst4","trst5","trst6","trst7","prst";
+		};
+
+		d1_pwm0: pwm@0x70818000 {
+			compatible = "eswin,pwm-eswin";
+			reg = <0x0 0x70818000 0x0 0x4000>;
+			clock-names = "pwm","pclk";
+			clocks = <&d1_clock EIC7700_CLK_LSP_TIMER_PCLK>;
+			clock-frequency = <200000000>;
+			resets = <&d1_reset TIMER_RST_CTRL SW_TIMER_RST_N>;
+			reset-names = "pwmrst";
+			#pwm-cells = <3>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&d1_pinctrl_pwm0_default &d1_pinctrl_pwm1_default &d1_pinctrl_pwm2_default>;
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_i2s0: i2s0@70200000 {
+			compatible = "snps,i2s";
+			clocks = <&d1_clock EIC7700_CLK_VO_I2S_MCLK>,
+					<&d1_clock EIC7700_APLL_FOUT1>;
+			clock-names = "mclk", "apll";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#sound-dai-cells = <0x00000000>;
+			reg = <0x0 0x70200000 0x0 0x10000>;
+			dma-names = "rx", "tx";
+			dmas = <&d1_aon_dmac 4 4 0>, <&d1_aon_dmac 5 5 0>;
+			vo_mclk_sel,syscrg = <&d1_sys_crg 0x1bc>;
+			resets = <&d1_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+				<&d1_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+				<&d1_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+			reset-names = "i2srst", "i2sprst", "voprst";
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		d1_i2s1: i2s1@70210000 {
+			compatible = "snps,i2s";
+			clocks = <&d1_clock EIC7700_CLK_VO_I2S_MCLK>,
+					<&d1_clock EIC7700_APLL_FOUT1>;
+			clock-names = "mclk", "apll";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#sound-dai-cells = <0x00000000>;
+			reg = <0x0 0x70210000 0x0 0x10000>;
+			dma-names = "rx", "tx";
+			dmas = <&d1_aon_dmac 6 2 1>, <&d1_aon_dmac 7 3 1>;
+			vo_mclk_sel,syscrg = <&d1_sys_crg 0x1bc>;
+			resets = <&d1_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+				<&d1_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+				<&d1_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+			reset-names = "i2srst", "i2sprst", "voprst";
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		d1_i2s2: i2s@70220000 {
+			compatible = "snps,i2s";
+			clocks = <&d1_clock EIC7700_CLK_VO_I2S_MCLK>,
+					<&d1_clock EIC7700_APLL_FOUT1>;
+			clock-names = "mclk", "apll";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#sound-dai-cells = <0x00000000>;
+			reg = <0x0 0x70220000 0x0 0x10000>;
+			dma-names = "rx", "tx";
+			dmas = <&d1_aon_dmac 8 0 2>, <&d1_aon_dmac 9 1 2>;
+			vo_mclk_sel,syscrg = <&d1_sys_crg 0x1bc>;
+			resets = <&d1_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+				<&d1_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+				<&d1_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+			reset-names = "i2srst", "i2sprst", "voprst";
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		d1_graphcard0: graphcard4 {
+			compatible = "audio-graph-card";
+			numa-node-id = <1>;
+		};
+
+		d1_graphcard1: graphcard5 {
+			compatible = "audio-graph-card";
+			numa-node-id = <1>;
+		};
+
+		d1_graphcard2: graphcard6 {
+			compatible = "audio-graph-card";
+			numa-node-id = <1>;
+		};
+
+		d1_dsp_subsys:dsp_subsys@72280400 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			reg = <0x0 0x72280400 0x0 0x10000>,
+				<0x0 0x71810000 0x0 0x8000>;
+			ranges;
+			dma-ranges = <0x0 0x30000000 0x0 0xc0000000 0x0 0xce000000>;
+			compatible = "es-dsp-subsys", "simple-bus";
+			clocks = <&d1_clock EIC7700_CLK_DSPT_CFG_CLK>,  <&d1_clock EIC7700_CLK_DSPT_ACLK>;
+			clock-names = "cfg_clk", "aclk";
+			resets = <&d1_reset DSP_RST_CTRL SW_DSP_AXI_RSTN>,
+				<&d1_reset DSP_RST_CTRL SW_DSP_CFG_RSTN>,
+				<&d1_reset DSP_RST_CTRL SW_DSP_DIV4_RSTN>,
+				<&d1_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_0>,
+				<&d1_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_1>,
+				<&d1_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_2>,
+				<&d1_reset DSP_RST_CTRL SW_DSP_DIV_RSTN_3>;
+			reset-names = "axi", "cfg", "div4", "div_0", "div_1", "div_2","div_3";
+			numa-node-id = <1>;
+			d1_dsp0:es_dsp@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x7b000000 0x8000
+					0x28100000 0 0x7b100000 0x20000
+					0x28120000 0 0x7b120000 0x20000>;
+				clocks = <&d1_clock EIC7700_CLK_DSP_ACLK_0>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d1_mbox4>;
+				device-irq = <11
+					(ESWIN_MAILBOX_DSP_0_TO_U84_REG_BASE + 0x20000000)
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_0
+					(ESWIN_MAILBOX_U84_TO_DSP_0_REG_BASE + 0x20000000)
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					(ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE + 0x20000000)>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <0>;
+				iommus = <&smmu1 WIN2030_SID_DSP_0>;
+				tbus = <WIN2030_TBUID_DSP0>;
+				dma-noncoherent;
+				numa-node-id = <1>;
+				aux-e31-dtim = <0x7a110000>;
+				dsp@0 {
+				};
+			};
+			d1_dsp1:es_dsp@1 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x7b008000 0x8000
+					0x28100000 0 0x7b140000 0x20000
+					0x28120000 0 0x7b160000 0x20000>;
+				clocks = <&d1_clock EIC7700_CLK_DSP_ACLK_1>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d1_mbox5>;
+				device-irq = <13
+					(ESWIN_MAILBOX_DSP_1_TO_U84_REG_BASE + 0x20000000)
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_1
+					(ESWIN_MAILBOX_U84_TO_DSP_1_REG_BASE + 0x20000000)
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					(ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE + 0x20000000)>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <1>;
+				iommus = <&smmu1 WIN2030_SID_DSP_1>;
+				tbus = <WIN2030_TBUID_DSP1>;
+				dma-noncoherent;
+				numa-node-id = <1>;
+				aux-e31-dtim = <0x7a110000>;
+				dsp@0 {
+				};
+			};
+			d1_dsp2:es_dsp@2 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x7b010000 0x8000
+					0x28100000 0 0x7b180000 0x20000
+					0x28120000 0 0x7b1a0000 0x20000>;
+				clocks = <&d1_clock EIC7700_CLK_DSP_ACLK_2>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d1_mbox6>;
+				device-irq = <15
+					(ESWIN_MAILBOX_DSP_2_TO_U84_REG_BASE + 0x20000000)
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_2
+					(ESWIN_MAILBOX_U84_TO_DSP_2_REG_BASE + 0x20000000)
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					(ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE + 0x20000000)>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <2>;
+				iommus = <&smmu1 WIN2030_SID_DSP_2>;
+				tbus = <WIN2030_TBUID_DSP2>;
+				dma-noncoherent;
+				numa-node-id = <1>;
+				aux-e31-dtim = <0x7a110000>;
+				dsp@0 {
+				};
+			};
+			d1_dsp3:es_dsp@3 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "eswin-dsp", "cdns,xrp-hw-eswin";
+				ranges = <0x28000000 0 0x7b018000 0x8000
+					0x28100000 0 0x7b1c0000 0x20000
+					0x28120000 0 0x7b1e0000 0x20000>;
+				clocks = <&d1_clock EIC7700_CLK_DSP_ACLK_3>;
+				clock-names = "aclk";
+				operating-points-v2 = <&dsp_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				dsp_mbox = <&d1_mbox7>;
+				device-irq = <17
+					(ESWIN_MAILBOX_DSP_3_TO_U84_REG_BASE + 0x20000000)
+					ESWIN_MAILBOX_WR_LOCK_BIT_DSP_3
+					(ESWIN_MAILBOX_U84_TO_DSP_3_REG_BASE + 0x20000000)
+					ESWIN_MAIBOX_U84_IRQ_BIT
+					(ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE + 0x20000000)>;
+				device-uart = <0x50900000>;
+				device-irq-mode = <1>;
+				host-irq-mode = <1>;
+				firmware-name = "eic7700_dsp_fw";
+				process-id = <3>;
+				iommus = <&smmu1 WIN2030_SID_DSP_3>;
+				tbus = <WIN2030_TBUID_DSP3>;
+				dma-noncoherent;
+				numa-node-id = <1>;
+				aux-e31-dtim = <0x7a110000>;
+				dsp@0 {
+				};
+			};
+		};
+		d1_rtc: rtc@71818000 {
+			compatible = "eswin,win2030-rtc";
+			reg = <0x0 0x71818000 0x0 0x400>;
+			eswin,syscfg = <&d1_sys_con 0x3c0>;
+			interrupt-parent = <&plic1>;
+			interrupts = <292>;
+			clocks = <&d1_clock EIC7700_CLK_CLK_RTC>;
+			clock-names = "rtcclk";
+			clock-frequency = <15625>;
+			resets = <&d1_reset RTC_RST_CTRL SW_RTC_RSTN>;
+			reset-names = "rtcrst";
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_pcie: pcie@0x74000000 {
+			compatible = "eswin,win2030-pcie";
+			clocks = <&d1_clock EIC7700_CLK_PCIET_ACLK>,
+				 <&d1_clock EIC7700_CLK_PCIET_CFG_CLK>,
+				 <&d1_clock EIC7700_CLK_PCIET_CR_CLK>,
+				 <&d1_clock EIC7700_CLK_PCIET_AUX_CLK>;
+			clock-names = "pcie_aclk", "pcie_cfg_clk", "pcie_cr_clk", "pcie_aux_clk";
+
+			reset-names = "pcie_cfg", "pcie_powerup", "pcie_pwren";
+			resets = <&d1_reset PCIE_RST_CTRL SW_PCIE_CFG_RSTN>,
+				 <&d1_reset PCIE_RST_CTRL SW_PCIE_POWERUP_RSTN>,
+				 <&d1_reset PCIE_RST_CTRL SW_PCIE_PERST_N>;
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			#interrupt-cells = <1>;
+			reg = <0x0 0x74000000 0x0 0x4000000>,	/* IP registers */
+				  <0x0 0x60000000 0x0 0x800000>,	/* Configuration space */
+				  <0x0 0x70000000 0x0 0x100000>;
+			reg-names = "dbi", "config", "mgmt";
+			device_type = "pci";
+			/* dma-coherent; */
+			bus-range = <0x0 0xff>;
+
+			ranges = <0x81000000  0x0 0x60800000  0x0 0x60800000 0x0 0x800000>,	  /* I/O */
+				 <0x82000000  0x0 0x61000000  0x0 0x61000000 0x0 0xf000000>,	/* mem */
+				 <0xc3000000 0xa0 0x00000000 0xa0 0x00000000 0x2 0x00000000>;  /* mem prefetchable */
+
+			num-lanes = <0x4>;
+			/**********************************
+			  	msi_ctrl_io[0~31] : 188~219
+			  	msi_ctrl_int : 220
+			**********************************/
+			interrupts = <220>, <179>, <180>, <181>, <182>, <183>, <184>, <185>, <186>;
+			interrupt-names = "msi", "inta", "intb", "intc", "intd";
+			interrupt-parent = <&plic1>;
+			interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+			interrupt-map = <0x0 0x0 0x0 0x1 &plic1 179>,
+							<0x0 0x0 0x0 0x2 &plic1 180>,
+							<0x0 0x0 0x0 0x3 &plic1 181>,
+							<0x0 0x0 0x0 0x4 &plic1 182>;
+			iommus = <&smmu1 0xfe0000>;
+			iommu-map = <0x0 &smmu1 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "disabled";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_gmac0: ethernet@70400000 {
+			compatible = "eswin,win2030-qos-eth";
+			reg = <0x0 0x70400000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupt-names = "macirq";
+			interrupts = <61>;
+			phy-mode = "rgmii-txid";
+			numa-node-id = <1>;
+			id = <2>;
+			status = "disabled";
+			clocks = <&d1_clock EIC7700_CLK_HSP_ETH_APP_CLK>,
+					 <&d1_clock EIC7700_CLK_HSP_ETH_CSR_CLK>,
+					 <&d1_clock EIC7700_CLK_HSP_ETH0_CORE_CLK>;
+			clock-names = "app", "stmmaceth","tx";
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_HSP_ETH0_ARSTN>;
+			reset-names = "ethrst";
+			//iommus = <&smmu1 WIN2030_SID_ETH0>;
+			tbus = <WIN2030_TBUID_ETH>;
+			dma-noncoherent;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1030 0x100 0x108>;
+			eswin,syscrg_csr = <&d1_sys_crg 0x148 0x14c>;
+			eswin,dly_hsp_reg = <0x114 0x118 0x11c>;
+			snps,axi-config = <&d1_stmmac_axi_setup>;
+			d1_stmmac_axi_setup: stmmac-axi-config {
+				snps,blen = <0 0 0 0 16 8 4>;
+				snps,rd_osr_lmt = <2>;
+				snps,wr_osr_lmt = <2>;
+				snps,lpi_en = <0>;
+			};
+		};
+
+		d1_gmac1: ethernet@70410000 {
+			compatible = "eswin,win2030-qos-eth";
+			reg = <0x0 0x70410000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupt-names = "macirq";
+			interrupts = <70>;
+			phy-mode = "rgmii-txid";
+			numa-node-id = <1>;
+			id = <3>;
+			status = "disabled";
+			clocks = <&d1_clock EIC7700_CLK_HSP_ETH_APP_CLK>,
+					 <&d1_clock EIC7700_CLK_HSP_ETH_CSR_CLK>,
+					 <&d1_clock EIC7700_CLK_HSP_ETH1_CORE_CLK>;
+			clock-names = "app", "stmmaceth","tx";
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_HSP_ETH1_ARSTN>;
+			reset-names = "ethrst";
+			//iommus = <&smmu1 WIN2030_SID_ETH1>;
+			tbus = <WIN2030_TBUID_ETH>;
+			dma-noncoherent;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1034 0x200 0x208>;
+			eswin,syscrg_csr = <&d1_sys_crg 0x148 0x14c>;
+			eswin,dly_hsp_reg = <0x214 0x218 0x21c>;
+			snps,axi-config = <&d1_stmmac_axi_setup_gmac1>;
+			d1_stmmac_axi_setup_gmac1: stmmac-axi-config {
+				snps,blen = <0 0 0 0 16 8 4>;
+				snps,rd_osr_lmt = <2>;
+				snps,wr_osr_lmt = <2>;
+				snps,lpi_en = <0>;
+			};
+		};
+
+		d1_sdhci_emmc: mmc@70450000 {
+			compatible = "eswin,emmc-sdhci-5.1";
+			reg = <0x0 0x70450000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <79>;
+			assigned-clocks = <&d1_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>;
+			assigned-clock-rates = <200000000>;
+			clocks = <&d1_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>, <&d1_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "clk_xin", "clk_ahb";
+			clock-output-names = "d1_emmc_cardclock";
+			#clock-cells = <0>;
+
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_MSHC0_TXRX_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_MSHC0_PHY_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_EMMC_PRSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_EMMC_ARSTN>;
+			reset-names = "txrx_rst", "phy_rst", "prstn", "arstn";
+
+			disable-cqe-dcmd;
+			bus-width = <8>;
+			non-removable;
+			/* mmc-ddr-1_8v; */
+			mmc-hs400-1_8v;
+			max-frequency = <200000000>;
+			/* sdhci-caps-mask =  <0x0 0x3200000>; */
+			/* smmu */
+			#size-cells = <2>;
+			iommus = <&smmu1 WIN2030_SID_EMMC0>;
+			tbus = <WIN2030_TBUID_EMMC>;
+			dma-ranges = <0x0 0x00000000 0x0 0xc0000000 0x1 0x0>;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1038 0x508 0x50c>;
+			eswin,syscrg_csr = <&d1_sys_crg 0x160 0x148 0x14c>;
+			status = "disabled";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_sdio0: mmc@0x70460000{
+			compatible = "eswin,sdhci-sdio";
+			reg = <0x0 0x70460000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <81>;
+			clocks =<&d1_clock EIC7700_CLK_HSP_MSHC1_CORE_CLK>,
+				<&d1_clock EIC7700_CLK_HSP_CFG_CLK>,
+				<&d1_clock EIC7700_SPLL2_FOUT3>,
+				<&d1_clock EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1>;
+			clock-names ="clk_xin","clk_ahb","clk_spll2_fout3","clk_mux1_1";
+			clock-output-names = "sdio_cardclock";
+			#clock-cells = <0>;
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_MSHC1_TXRX_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_MSHC1_PHY_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_SD0_PRSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_SD0_ARSTN>;
+			reset-names = "txrx_rst","phy_rst","prstn","arstn";
+
+			core-clk-reg = <0x71828164>;
+			clock-frequency = <208000000>;
+			max-frequency = <208000000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+			iommus = <&smmu1 WIN2030_SID_SD0>;
+			tbus = <WIN2030_TBUID_SD>;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x103c 0x608 0x60c>;
+			eswin,syscrg_csr = <&d1_sys_crg 0x164 0x148 0x14c>;
+			bus-width = <4>;
+			sdio-id = <0>;
+			numa-node-id = <1>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		d1_sdio1: mmc@0x70470000{
+			compatible = "eswin,sdhci-sdio";
+			reg = <0x0 0x70470000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <83>;
+			clocks =<&d1_clock EIC7700_CLK_HSP_MSHC2_CORE_CLK>,
+				<&d1_clock EIC7700_CLK_HSP_CFG_CLK>,
+				<&d1_clock EIC7700_SPLL2_FOUT3>,
+				<&d1_clock EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1>;
+			clock-names ="clk_xin","clk_ahb","clk_spll2_fout3","clk_mux1_1";
+			clock-output-names = "sdio_cardclock";
+			#clock-cells = <0>;
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_MSHC2_TXRX_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_MSHC2_PHY_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_SD1_PRSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_SD1_ARSTN>;
+			reset-names = "txrx_rst","phy_rst","prstn","arstn";
+
+			core-clk-reg = <0x71828168>;
+			clock-frequency = <208000000>;
+			max-frequency = <208000000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+			iommus = <&smmu1 WIN2030_SID_SD1>;
+			tbus = <WIN2030_TBUID_SD>;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1040 0x708 0x70c>;
+			eswin,syscrg_csr = <&d1_sys_crg 0x168 0x148 0x14c>;
+			bus-width = <4>;
+			sdio-id = <1>;
+			numa-node-id = <1>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		d1_ssi0: spi@70810000 {
+			compatible = "snps,eic770x-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x70810000 0x0 0x4000>;
+			spi-max-frequency = <4800000>;
+			clocks = <&d1_clock EIC7700_CLK_LSP_SSI0_PCLK>;
+			clock-names = "clk";
+			interrupt-parent = <&plic1>;
+			interrupts = <91>;
+			resets = <&d1_reset SSI_RST_CTRL SW_SSI_RST_N_0>;
+			reset-names = "spi";
+			numa-node-id = <1>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		d1_ssi1: spi@70814000 {
+			compatible = "snps,eic770x-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x70814000 0x0 0x4000>;
+			spi-max-frequency = <4800000>;
+			clocks = <&d1_clock EIC7700_CLK_LSP_SSI1_PCLK>;
+			clock-names = "clk";
+			interrupt-parent = <&plic1>;
+			interrupts = <92>;
+			resets = <&d1_reset SSI_RST_CTRL SW_SSI_RST_N_1>;
+			reset-names = "spi";
+			numa-node-id = <1>;
+			status = "disabled";
+			dma-noncoherent;
+		};
+
+		d1_bootspi: spi@71800000 {
+			compatible = "eswin,bootspi";
+			reg = <0x0 0x71800000 0x0 0x8000>,
+				<0x0 0x71828000 0x0 0x8000>,
+				<0x0 0x7c000000 0x0 0x8000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&d1_clock EIC7700_CLK_CLK_BOOTSPI_CFG>,
+					<&d1_clock EIC7700_CLK_CLK_BOOTSPI>;
+			clock-names = "cfg_clk", "clk";
+			resets = <&d1_reset BOOTSPI_RST_CTRL SW_BOOTSPI_RSTN>;
+			reset-names = "rst";
+			spi-max-frequency = <4800000>;
+			reg-io-width = <4>;
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_video_output: display-subsystem@1 {
+			compatible = "eswin,display-subsystem";
+			ports = <&d1_dc_out>;
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_dc: display_control@702c0000 {
+			compatible = "eswin,dc";
+			reg = <0x0 0x702c0000 0x0 0x100>, <0x0 0x702c0180 0x0 0x700>, <0x0 0x702c1400 0x0 0x1400>;
+			interrupt-parent = <&plic1>;
+			interrupts = <238>;
+
+			clocks = <&d1_clock EIC7700_CLK_VO_CFG_CLK>,
+					 <&d1_clock EIC7700_CLK_VO_PIXEL_CLK>,
+					 <&d1_clock EIC7700_CLK_VO_ACLK>,
+					 <&d1_clock EIC7700_SPLL0_FOUT1>,
+					 <&d1_clock EIC7700_MUX_U_VO_ACLK_ROOT_2MUX1_GFREE>,
+					 <&d1_clock EIC7700_MUX_U_VO_PIXEL_ROOT_2MUX1>,
+					 <&d1_clock EIC7700_SPLL2_FOUT2>,
+					 <&d1_clock EIC7700_VPLL_FOUT1>;
+			clock-names = "cfg_clk", "pix_clk", "axi_clk", "spll0_fout1", "vo_mux",
+						  "pix_mux", "spll2_fout2", "vpll_fout1";
+			resets = <&d1_reset VO_RST_CTRL SW_VO_AXI_RSTN>,
+					 <&d1_reset VO_RST_CTRL SW_VO_CFG_RSTN>,
+					 <&d1_reset VO_RST_CTRL SW_VO_DC_RSTN>,
+					 <&d1_reset VO_RST_CTRL SW_VO_DC_PRSTN>;
+			reset-names = "vo_arst", "vo_prst", "dc_arst", "dc_prst";
+			dma-noncoherent;
+			numa-node-id = <1>;
+
+			d1_dc_out: port {
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		d1_virtual_display: es_wb@1 {
+			compatible = "eswin,virtual_display";
+			bpp = /bits/ 8 <8>;
+			numa-node-id = <1>;
+		};
+
+		d1_dsi_output: dsi-output@1 {
+			compatible = "eswin,dsi-encoder";
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+
+		d1_dsi_controller: mipi_dsi@70270000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "eswin,dsi";
+			reg = <0x0 0x70270000 0x0 0x10000>;
+			clocks = <&d1_clock EIC7700_CLK_CLK_MIPI_TXESC>;
+			clock-names = "pclk";
+			resets = <&d1_reset VO_PHYRST_CTRL SW_VO_MIPI_PRSTN>;
+			reset-names ="phyrstn";
+			numa-node-id = <1>;
+			/*
+			phys = <&dphy>;
+			phy-names = "dphy";
+			*/
+		};
+
+		d1_dc_test: dctest@702c0000 {
+			compatible = "eswin,dc";
+			reg = <0x0 0x702c0000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <238>;
+			numa-node-id = <1>;
+		};
+
+		d1_dw_hdmi: hdmi@702a0000 {
+			compatible = "eswin,eswin-dw-hdmi";
+			reg = <0x0 0x702a0000 0x0 0x20000>;
+			pinctrl-names = "default";
+			//pinctrl-0 = <&hdmi_i2c_xfer>;
+			interrupt-parent = <&plic1>;
+			interrupts = <274>;
+			clocks = <&d1_clock EIC7700_CLK_VO_CFG_CLK>, <&d1_clock EIC7700_CLK_VO_CEC_CLK>,
+					 <&d1_clock EIC7700_CLK_VO_CR_CLK>;
+			clock-names = "iahb", "cec", "isfr";
+			//power-domains = <&power WIN2030_PD_HDCP>;
+			reg-io-width = <4>;
+			ddc-i2c-scl-high-time-ns = <4708>;
+			ddc-i2c-scl-low-time-ns = <4916>;
+			#sound-dai-cells = <0>;
+			resets = <&d1_reset VO_PHYRST_CTRL SW_VO_HDMI_PRSTN>,
+					 <&d1_reset VO_PHYRST_CTRL SW_HDMI_PHYCTRL_RSTN>,
+					 <&d1_reset VO_PHYRST_CTRL SW_VO_HDMI_RSTN>;
+			reset-names = "prstn", "phyrstn", "rstn";
+			numa-node-id = <1>;
+		};
+
+		d1_dw_hdmi_hdcp2: hdmi-hdcp2@70290000 {
+			compatible = "eswin,dw-hdmi-hdcp2";
+			reg = <0x0 0x70290000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <275>;
+			clocks = <&d1_clock EIC7700_CLK_VO_ACLK>,
+				 <&d1_clock EIC7700_CLK_VO_HDMI_IESMCLK>;
+			clock-names ="aclk", "iesmclk";
+			dma-noncoherent;
+			numa-node-id = <1>;
+		};
+
+		d1_wdt0: watchdog@0x70800000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x70800000 0x0 0x4000>;
+			clocks =<&d1_clock EIC7700_CLK_LSP_WDT0_PCLK>;
+			clock-names = "pclk";
+			resets = <&d1_reset WDT_RST_CTRL SW_WDT_RST_N_0>;
+			reset-names = "rst";
+			interrupts = <87>;
+			interrupt-parent = <&plic1>;
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_wdt1: watchdog@0x70804000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x70804000 0x0 0x4000>;
+			clocks =<&d1_clock EIC7700_CLK_LSP_WDT1_PCLK>;
+			clock-names = "pclk";
+			resets = <&d1_reset WDT_RST_CTRL SW_WDT_RST_N_1>;
+			reset-names = "rst";
+			interrupts = <88>;
+			interrupt-parent = <&plic1>;
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_wdt2: watchdog@0x70808000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x70808000 0x0 0x4000>;
+			clocks =<&d1_clock EIC7700_CLK_LSP_WDT2_PCLK>;
+			clock-names = "pclk";
+			resets = <&d1_reset WDT_RST_CTRL SW_WDT_RST_N_2>;
+			reset-names = "rst";
+			interrupts = <89>;
+			interrupt-parent = <&plic1>;
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_wdt3: watchdog@0x7080c000 {
+			compatible = "snps,dw-wdt";
+			reg = <0x0 0x7080c000 0x0 0x4000>;
+			clocks =<&d1_clock EIC7700_CLK_LSP_WDT3_PCLK>;
+			clock-names = "pclk";
+			resets = <&d1_reset WDT_RST_CTRL SW_WDT_RST_N_3>;
+			reset-names = "rst";
+			interrupts = <90>;
+			interrupt-parent = <&plic1>;
+			status = "disabled";
+			numa-node-id = <1>;
+		};
+
+		d1_gc820: g2d@70140000 {
+			compatible = "eswin,galcore_d1";
+			clocks = <&d1_clock EIC7700_CLK_VC_ACLK>,
+					<&d1_clock EIC7700_CLK_VC_CFG_CLK>,
+					<&d1_clock EIC7700_CLK_G2D_CFG_CLK>,
+					<&d1_clock EIC7700_CLK_CLK_G2D_ST2>,
+					<&d1_clock EIC7700_CLK_G2D_CLK>,
+					<&d1_clock EIC7700_CLK_G2D_ACLK>,
+					<&d1_clock EIC7700_CLK_VC_MON_PCLK>;
+			clock-names = "vc_aclk", "vc_cfg", "g2d_cfg", "g2d_st2", "g2d_clk", "g2d_aclk","mon_pclk";
+			resets = <&d1_reset VC_RST_CTRL SW_VC_AXI_RSTN>,
+					<&d1_reset VC_RST_CTRL SW_VC_CFG_RSTN>,
+					<&d1_reset VC_RST_CTRL SW_VC_MONCFG_RSTN>,
+					<&d1_reset G2D_RST_CTRL SW_G2D_CORE_RSTN>,
+					<&d1_reset G2D_RST_CTRL SW_G2D_CFG_RSTN>,
+					<&d1_reset G2D_RST_CTRL SW_G2D_AXI_RSTN>;
+			reset-names = "axi", "cfg", "moncfg", "g2d_core", "g2d_cfg", "g2d_axi";
+			reg = <0 0x70140000 0 0x40000>, <0 0x70180000 0 0x40000>;
+			reg-names = "core_2d", "core_2d1";
+			fe-apb-offset = <0x800>;
+			interrupt-parent = <&plic1>;
+			interrupts = <49>, <50>;
+			interrupt-names = "core_2d", "core_2d1";
+			enable-mmu = <1>;
+			contiguous-size = <0xa00000>;
+			recovery = <0>;
+			dma-noncoherent;
+			operating-points-v2 = <&g2d_opp_table>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <1000>; /*TBD*/
+			numa-node-id = <1>;
+		};
+
+		d1_sdhci_emmc: mmc@70450000 {
+			compatible = "eswin,emmc-sdhci-5.1";
+			reg = <0x0 0x70450000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <79>;
+			assigned-clocks = <&d1_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>;
+			assigned-clock-rates = <200000000>;
+			clocks = <&d1_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>, <&d1_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "clk_xin", "clk_ahb";
+			clock-output-names = "d1_emmc_cardclock";
+			#clock-cells = <0>;
+
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_MSHC0_TXRX_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_MSHC0_PHY_RSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_EMMC_PRSTN>,
+				<&d1_reset HSPDMA_RST_CTRL SW_HSP_EMMC_ARSTN>;
+			reset-names = "txrx_rst", "phy_rst", "prstn", "arstn";
+
+			core-clk-reg = <0x71828160>;
+			disable-cqe-dcmd;
+			bus-width = <8>;
+			non-removable;
+			/* mmc-ddr-1_8v; */
+			mmc-hs400-1_8v;
+			max-frequency = <200000000>;
+			/* sdhci-caps-mask =  <0x0 0x3200000>; */
+			/* smmu */
+			#size-cells = <2>;
+			iommus = <&smmu1 WIN2030_SID_EMMC0>;
+			tbus = <WIN2030_TBUID_EMMC>;
+			dma-ranges = <0x0 0x00000000 0x0 0xc0000000 0x1 0x0>;
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1038>;
+			status = "disabled";
+			numa-node-id = <1>;
+			dma-noncoherent;
+		};
+
+		d1_graphcard: graphcard {
+			compatible = "audio-graph-card";
+			numa-node-id = <1>;
+		};
+
+		d1_usbdrd3_0: usb0@70480000 {
+			compatible = "eswin,win2030-dwc3";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			clocks =<&d1_clock EIC7700_GATE_HSP_USB0_SUSPEND_CLK>,
+					<&d1_clock EIC7700_CLK_HSP_ACLK>,
+					<&d1_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "suspend","aclk", "cfg_clk";
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x800 0x808 0x83c 0x840>;
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_USB0_VAUX_RSTN>;
+			reset-names = "vaux";
+			ranges;
+			status = "disabled";
+			numa-node-id = <1>;
+			d1_usbdrd_dwc3_0: dwc3@70480000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x70480000 0x0 0x10000>;
+				#address-cells = <2>;
+				#size-cells = <2>;
+				interrupt-parent = <&plic1>;
+				interrupts = <85>;
+				interrupt-names = "peripheral";
+				dr_mode = "peripheral";
+				phy_type = "utmi";
+				maximum-speed = "high-speed";
+ 				iommus = <&smmu1 WIN2030_SID_USB0>;
+				eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1044>;
+				dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+				snps,dis_enblslpm_quirk;
+				snps,dis-u2-freeclk-exists-quirk;
+				snps,dis_u2_susphy_quirk;
+				snps,dis-del-phy-power-chg-quirk;
+				snps,parkmode-disable-ss-quirk;
+				status = "disabled";
+				numa-node-id = <1>;
+				tbus = <WIN2030_TBUID_USB>;
+				dma-noncoherent;
+			};
+		};
+
+		d1_usbdrd3_1: usb1@70490000 {
+			compatible = "eswin,win2030-dwc3";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			clocks =<&d1_clock EIC7700_GATE_HSP_USB1_SUSPEND_CLK>,
+					<&d1_clock EIC7700_CLK_HSP_ACLK>,
+					<&d1_clock EIC7700_CLK_HSP_CFG_CLK>;
+			clock-names = "suspend","aclk", "cfg_clk";
+			eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x900 0x908 0x93c 0x940>;
+			resets = <&d1_reset HSPDMA_RST_CTRL SW_USB1_VAUX_RSTN>;
+			reset-names = "vaux";
+			ranges;
+			status = "disabled";
+			numa-node-id = <1>;
+			d1_usbdrd_dwc3_1: dwc3@70490000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x70490000 0x0 0x10000>;
+				#address-cells = <2>;
+				#size-cells = <2>;
+				interrupt-parent = <&plic1>;
+				interrupts = <86>;
+				interrupt-names = "host";
+				dr_mode = "host";
+				phy_type = "utmi";
+				maximum-speed = "high-speed";
+				iommus = <&smmu1 WIN2030_SID_USB1>;
+				eswin,hsp_sp_csr = <&d1_hsp_sp_csr 0x1048>;
+				dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+				snps,dis_enblslpm_quirk;
+				snps,dis-u2-freeclk-exists-quirk;
+				snps,dis_u2_susphy_quirk;
+				snps,dis-del-phy-power-chg-quirk;
+				snps,parkmode-disable-ss-quirk;
+				status = "disabled";
+				numa-node-id = <1>;
+				tbus = <WIN2030_TBUID_USB>;
+				dma-noncoherent;
+			};
+		};
+		d1_vi_subsys: d1-vi-subsys@71000000 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "esw,vi_subsys";
+			reg = <0x0 0x71000000 0x0 0x200000>, <0x0 0x71030000 0x0 0x2000>;
+			eswin,syscrg_csr = <&d1_sys_crg>;
+			eswin,vitop_csr = <&d1_vi_top_csr>;
+			clocks = <&d1_clock EIC7700_CLK_VI_ACLK>,
+					 <&d1_clock EIC7700_CLK_VI_CFG_CLK>,
+					 <&d1_clock EIC7700_CLK_VI_DIG_ISP_CLK>,
+					 <&d1_clock EIC7700_CLK_VI_DVP_CLK>,
+					 <&d1_clock EIC7700_CLK_VI_PHY_CFG>,
+					 <&d1_clock EIC7700_CLK_VI_PHY_TXCLKESC>,
+					 <&d1_clock EIC7700_CLK_VI_SHUTTER_0>,
+					 <&d1_clock EIC7700_CLK_VI_SHUTTER_1>,
+					 <&d1_clock EIC7700_CLK_VI_SHUTTER_2>,
+					 <&d1_clock EIC7700_CLK_VI_SHUTTER_3>,
+					 <&d1_clock EIC7700_CLK_VI_SHUTTER_4>,
+					 <&d1_clock EIC7700_CLK_VI_SHUTTER_5>,
+					 <&d1_clock EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE>,
+					 <&d1_clock EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE>,
+					 <&d1_clock EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE>,
+					 <&d1_clock EIC7700_SPLL0_FOUT1>,
+					 <&d1_clock EIC7700_VPLL_FOUT1>;
+			clock-names = "aclk", "cfg_clk", "isp_aclk", "dvp_clk", "phy_cfg",
+							"phy_escclk", "sht0", "sht1", "sht2", "sht3", "sht4",
+							"sht5", "aclk_mux", "dvp_mux", "isp_mux", "spll0_fout1", "vpll_fout1";
+			resets = <&d1_reset VI_RST_CTRL SW_VI_AXI_RSTN>,
+					 <&d1_reset VI_RST_CTRL SW_VI_CFG_RSTN>,
+					 <&d1_reset ISP0_RST_CTRL SW_VI_ISP0_RSTN>,
+					 <&d1_reset ISP1_RST_CTRL SW_VI_ISP1_RSTN>,
+					 <&d1_reset DVP_RST_CTRL SW_VI_DVP_RSTN>,
+					 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_0>,
+					 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_1>,
+					 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_2>,
+					 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_3>,
+					 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_4>,
+					 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_5>;
+			reset-names = "axi", "cfg", "isp0", "isp1", "dvp", "sht0", "sht1", "sht2", "sht3", "sht4", "sht5";
+			tbus = <WIN2030_TBUID_DW>;
+			numa-node-id = <1>;
+			ranges;
+			status = "okay";
+
+			d1_vi_top_csr: vi_common_top_csr@71030000 {
+				compatible = "esw,vi-common-csr", "syscon";
+				clocks = <&d1_clock EIC7700_CLK_VI_ACLK>,
+						 <&d1_clock EIC7700_CLK_VI_CFG_CLK>,
+						 <&d1_clock EIC7700_CLK_VI_DIG_ISP_CLK>,
+						 <&d1_clock EIC7700_CLK_VI_DVP_CLK>,
+						 <&d1_clock EIC7700_CLK_VI_PHY_CFG>,
+						 <&d1_clock EIC7700_CLK_VI_PHY_TXCLKESC>,
+						 <&d1_clock EIC7700_CLK_VI_SHUTTER_0>,
+						 <&d1_clock EIC7700_CLK_VI_SHUTTER_1>,
+						 <&d1_clock EIC7700_CLK_VI_SHUTTER_2>,
+						 <&d1_clock EIC7700_CLK_VI_SHUTTER_3>,
+						 <&d1_clock EIC7700_CLK_VI_SHUTTER_4>,
+						 <&d1_clock EIC7700_CLK_VI_SHUTTER_5>,
+						 <&d1_clock EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE>,
+						 <&d1_clock EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE>,
+						 <&d1_clock EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE>,
+						 <&d1_clock EIC7700_SPLL0_FOUT1>,
+						 <&d1_clock EIC7700_VPLL_FOUT1>;
+				clock-names = "aclk", "cfg_clk", "isp_aclk", "dvp_clk", "phy_cfg",
+							"phy_escclk", "sht0", "sht1", "sht2", "sht3", "sht4",
+							"sht5", "aclk_mux", "dvp_mux", "isp_mux", "spll0_fout1", "vpll_fout1";
+				resets = <&d1_reset VI_RST_CTRL SW_VI_AXI_RSTN>,
+						 <&d1_reset VI_RST_CTRL SW_VI_CFG_RSTN>,
+						 <&d1_reset ISP0_RST_CTRL SW_VI_ISP0_RSTN>,
+						 <&d1_reset ISP1_RST_CTRL SW_VI_ISP1_RSTN>,
+						 <&d1_reset DVP_RST_CTRL SW_VI_DVP_RSTN>,
+						 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_0>,
+						 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_1>,
+						 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_2>,
+						 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_3>,
+						 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_4>,
+						 <&d1_reset SHUTTER_RST_CTRL SW_VI_SHUTTER_RSTN_5>;
+				reset-names = "axi", "cfg", "isp0", "isp1", "dvp", "sht0", "sht1", "sht2", "sht3", "sht4", "sht5";
+
+				id = <0>;
+				#size-cells = <2>;
+				reg = <0x0 0x71030000 0x0 0x10000>;
+				numa-node-id = <1>;
+			};
+
+			d1_isp_0: isp@71000000 {
+				compatible = "esw,win2030-isp";
+				reg = <0x0 0x71000000 0x0 0x10000>;
+				interrupts = <21 19 20>;
+				interrupt-parent = <&plic1>;
+				id = <0>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu1 WIN2030_SID_ISP0>;
+				eswin,vi_top_csr = <&d1_vi_top_csr 0x1000>;
+				numa-node-id = <1>;
+				dma-noncoherent;
+			};
+
+			d1_isp_1: isp@71010000 {
+				compatible = "esw,win2030-isp";
+				reg = <0x0 0x71010000 0x0 0x10000>;
+				interrupts = <24 22 23>;
+				interrupt-parent = <&plic1>;
+				id = <1>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu1 WIN2030_SID_ISP1>;
+				eswin,vi_top_csr = <&d1_vi_top_csr 0x1004>;
+				numa-node-id = <1>;
+				dma-noncoherent;
+			};
+
+			d1_dewarp: dewarp@71020000 {
+				compatible = "eswin,dewarp";
+				clocks = <&d1_clock EIC7700_CLK_VI_ACLK>,
+						<&d1_clock EIC7700_CLK_VI_CFG_CLK>,
+						<&d1_clock EIC7700_CLK_VI_DIG_DW_CLK>,
+						<&d1_clock EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE>,
+						<&d1_clock EIC7700_MUX_U_VI_DW_ROOT_2MUX1>,
+						<&d1_clock EIC7700_SPLL0_FOUT1>,
+						<&d1_clock EIC7700_VPLL_FOUT1>;
+				clock-names = "aclk", "cfg_clk", "dw_aclk", "aclk_mux", "dw_mux", "spll0_fout1", "vpll_fout1";
+				resets = <&d1_reset VI_RST_CTRL SW_VI_AXI_RSTN>,
+						 <&d1_reset VI_RST_CTRL SW_VI_CFG_RSTN>,
+						 <&d1_reset VI_RST_CTRL SW_VI_DWE_RSTN>;
+				reset-names = "axi", "cfg", "dwe";
+
+				operating-points-v2 = <&vi_opp_table>;
+				#cooling-cells = <2>;
+				dynamic-power-coefficient = <1000>; /*TBD*/
+				interrupt-parent = <&plic1>;
+				interrupts = <26 25>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu1 WIN2030_SID_DW>;
+				tbus = <WIN2030_TBUID_DW>;
+				eswin,vi_top_csr = <&d1_vi_top_csr 0x1008>;
+				reg = <0x0 0x71020000 0x0 0xc00>, <0x0 0x71020c00 0x0 0x120>;
+				numa-node-id = <1>;
+				dma-noncoherent;
+			};
+
+			d1_dvp2axi: d1-dvp2axi@71034000 {
+				compatible = "eswin,dvp2axi";
+				reg = <0x0 0x71034000 0x0 0x8000>;
+				reg-names = "dvp2axi_regs";
+				interrupt-parent = <&plic1>;
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr 0x100c>;
+				interrupts = <368 369 370 371 372 373 374 375>;
+				interrupt-names = "dvp0_frame_int", "dvp1_frame_int", "dvp2_frame_int",
+									"dvp3_frame_int", "dvp4_frame_int", "dvp5_frame_int",
+									"dvp2axi_error_int","dvp2axi_afull_int";
+				dma-noncoherent;
+				numa-node-id = <1>;
+				#size-cells = <2>;
+				dma-ranges = <0x0 0x20000000 0x0 0x80000000 0x0 0x40000000>;
+				iommus = <&smmu1 WIN2030_SID_DW>;
+				tbus = <WIN2030_TBUID_DW>;
+				status = "disabled";
+			};
+
+			d1_dvp2axi_mipi_lvds: d1-dvp2axi_mipi_lvds {
+				compatible = "eswin,dvp2axi-mipi-lvds_d1";
+				eswin,hw = <&d1_dvp2axi>;
+				status = "disabled";
+			};
+
+			d1_dvp2axi_mipi_lvds1: d1-dvp2axi_mipi_lvds1 {
+				compatible = "eswin,dvp2axi-mipi-lvds_d1";
+				eswin,hw = <&d1_dvp2axi>;
+				status = "disabled";
+			};
+
+			d1_dvp2axi_mipi_lvds2: d1-dvp2axi_mipi_lvds2 {
+				compatible = "eswin,dvp2axi-mipi-lvds_d1";
+				eswin,hw = <&d1_dvp2axi>;
+				status = "disabled";
+			};
+
+			d1_dvp2axi_mipi_lvds3: d1-dvp2axi_mipi_lvds3 {
+				compatible = "eswin,dvp2axi-mipi-lvds_d1";
+				eswin,hw = <&d1_dvp2axi>;
+				status = "disabled";
+			};
+
+			d1_dvp2axi_mipi_lvds4: d1-dvp2axi_mipi_lvds4 {
+				compatible = "eswin,dvp2axi-mipi-lvds_d1";
+				eswin,hw = <&d1_dvp2axi>;
+				status = "disabled";
+			};
+
+			d1_dvp2axi_mipi_lvds5: d1-dvp2axi_mipi_lvds5 {
+				compatible = "eswin,dvp2axi-mipi-lvds_d1";
+				eswin,hw = <&d1_dvp2axi>;
+				status = "disabled";
+			};
+
+			d1_mipi_csi2_0: d1-mipi-csi2-0@71050000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x71050000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr>;
+				interrupt-parent = <&plic1>;
+				interrupts = <29>;
+				interrupt-names = "csi-intr1";
+				index = <0>;
+
+				ipi2_en = <0>;/* for virtual channel */
+				ipi2_vcid = <0>;/* virtual channel id */
+				ipi3_en = <0>;
+				ipi3_vcid = <0>;
+				status = "disabled";
+			};
+
+			d1_mipi_csi2_1: d1-mipi-csi2-1@71060000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x71060000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr>;
+				interrupt-parent = <&plic1>;
+				interrupts = <30>;
+				interrupt-names = "csi-intr1";
+				index = <1>;
+				status = "disabled";
+			};
+
+			d1_mipi_csi2_2: d1-mipi-csi2-2@71070000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x71070000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr>;
+				interrupt-parent = <&plic1>;
+				interrupts = <31>;
+				interrupt-names = "csi-intr1";
+				index = <2>;
+				status = "disabled";
+			};
+
+			d1_mipi_csi2_3: d1-mipi-csi2-3@71080000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x71080000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr>;
+				interrupt-parent = <&plic1>;
+				interrupts = <32>;
+				interrupt-names = "csi-intr1";
+				index = <3>;
+				status = "disabled";
+			};
+
+			d1_mipi_csi2_4: d1-mipi-csi2-4@71090000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x71090000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr>;
+				interrupt-parent = <&plic1>;
+				interrupts = <33>;
+				interrupt-names = "csi-intr1";
+				index = <4>;
+				status = "disabled";
+			};
+
+			d1_mipi_csi2_5: d1-mipi-csi2-5@710a0000 {
+				#address-cells = <2>;
+				#size-cells = <2>;
+				compatible = "eswin,eic770x-mipi-csi2";
+				reg = <0x0 0x710a0000 0x0 0x10000>;
+				reg-names = "csihost_regs";
+				eswin,syscrg_csr = <&d1_sys_crg>;
+				eswin,vi_top_csr = <&d1_vi_top_csr>;
+				interrupt-parent = <&plic1>;
+				interrupts = <34>;
+				interrupt-names = "csi-intr1";
+				index = <5>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy_hw: d1-csi2-dphy-hw@710c0000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x710c0000 0x0 0x20000>;
+				phy_cfg = <0x710d8000>;
+				csi_base = <0x71050000>;
+				combine_phy = <0x710e0000>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy_hw1: d1-csi2-dphy-hw1@710e0000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x710e0000 0x0 0x20000>;
+				phy_cfg = <0x710f8000>;
+				csi_base = <0x71060000>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy_hw2: d1-csi2-dphy-hw2@71100000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x71100000 0x0 0x20000>;
+				phy_cfg = <0x71118000>;
+				csi_base = <0x71070000>;
+				combine_phy = <0x71120000>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy_hw3: d1-csi2-dphy-hw3@71120000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x71120000 0x0 0x20000>;
+				phy_cfg = <0x71138000>;
+				csi_base = <0x71080000>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy_hw4: d1-csi2-dphy-hw4@71140000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x71140000 0x0 0x20000>;
+				phy_cfg = <0x71158000>;
+				csi_base = <0x71090000>;
+				combine_phy = <0x71160000>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy_hw5: d1-csi2-dphy-hw5@71160000 {
+				compatible = "eswin,eic770x-csi2-dphy-hw";
+				reg = <0x0 0x71160000 0x0 0x20000>;
+				phy_cfg = <0x71178000>;
+				csi_base = <0x710a0000>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy0: d1-csi2-dphy0 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&d1_csi2_dphy_hw>;
+				index = <0>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy1: d1-csi2-dphy1 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&d1_csi2_dphy_hw1>;
+				index = <1>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy2: d1-csi2-dphy2 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&d1_csi2_dphy_hw2>;
+				index = <2>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy3: d1-csi2-dphy3 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&d1_csi2_dphy_hw3>;
+				index = <3>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy4: d1-csi2-dphy4 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&d1_csi2_dphy_hw4>;
+				index = <4>;
+				status = "disabled";
+			};
+
+			d1_csi2_dphy5: d1-csi2-dphy5 {
+				compatible = "eswin,eic770x-csi2-dphy";
+				eswin,hw = <&d1_csi2_dphy_hw5>;
+				index = <5>;
+				status = "disabled";
+			};
+
+
+			d1_mipi_dphy_rx: dphy@710c0000 {
+				compatible = "snps,dw-dphy-rx";
+				#phy-cells = <1>;
+				bus-width = <8>;
+				snps,dphy-frequency = <300000>;
+				snps,phy_type = <8>;
+				reg = <0x0 0x710c0000 0x0 0x20000>;
+				numa-node-id = <1>;
+			};
+
+			d1_csi_dma0: csidma@72048000 {
+				compatible = "eswin,csi-video";
+				interrupt-parent = <&plic1>;
+				interrupts = <29>;
+				reg = <0x0 0x72048000 0x0 0x1000>;
+				numa-node-id = <1>;
+
+				port {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					d1_csi_dmar_0: endpoint@0 {
+						reg = <0>;
+						bus-type = <4>;
+						remote-endpoint = <&d1_csi2_dma_0_3>;
+					};
+				};
+			};
+
+			d1_csi_dma1: csidma@72058000 {
+				compatible = "eswin,csi-video";
+				interrupt-parent = <&plic1>;
+				interrupts = <30>;
+				reg = <0x0 0x72058000 0x0 0x1000>;
+				numa-node-id = <1>;
+
+				port {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					d1_csi_dmar_1: endpoint@0 {
+						reg = <0>;
+						bus-type = <4>;
+						remote-endpoint = <&d1_csi2_dma_1_3>;
+					};
+				};
+			};
+
+			d1_csi2_0: csi2@71050000 {
+				compatible = "snps,dw-csi";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				interrupt-parent = <&plic1>;
+				interrupts = <29>;
+				snps,output-type = <0>;
+				reg = <0x0 0x71050000 0x0 0x1000>;
+				/*phys = <&mipi_dphy_rx 1>;*/
+				numa-node-id = <1>;
+
+				ipi2_en = <0>;/* for virtual channel */
+				ipi2_vcid = <0>;/* virtual channel id */
+				ipi3_en = <0>;
+				ipi3_vcid = <0>;
+
+				#ifdef MIPI_CSI2_IMX290_ENBALE
+				port@1 {
+					reg = <1>;
+					d1_csi2_0_1: endpoint {
+						bus-type = <4>;
+						clock-lanes = <0>;
+						#ifdef MIPI_CSI2_2LINE_ENABLE
+						data-lanes = <1 2>;
+						#else
+						data-lanes = <1 2 3 4>;
+						#endif
+						//remote-endpoint = <&imx290_csi2>;
+					};
+				};
+				#endif
+
+				port@2 {
+					reg = <2>;
+					d1_csi2_dma_0_3: endpoint {
+						bus-type = <5>;
+						remote-endpoint = <&d1_csi_dmar_0>;
+					};
+				};
+			};
+
+			d1_csi2_1: csi2@71060000 {
+				compatible = "snps,dw-csi";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				interrupt-parent = <&plic1>;
+				interrupts = <30>;
+				snps,output-type = <0>;
+				reg = <0x0 0x71060000 0x0 0x1000>;
+				/*phys = <&mipi_dphy_rx 1>;*/
+				numa-node-id = <1>;
+
+				ipi2_en = <0>;/* for virtual channel */
+				ipi2_vcid = <0>;/* virtual channel id */
+				ipi3_en = <0>;
+				ipi3_vcid = <0>;
+
+				#ifdef MIPI_CSI2_IMX290_ENBALE
+				port@1 {
+					reg = <1>;
+					d1_csi2_1_1: endpoint {
+						bus-type = <4>;
+						clock-lanes = <0>;
+						#ifdef MIPI_CSI2_2LINE_ENABLE
+						data-lanes = <1 2>;
+						#else
+						data-lanes = <1 2 3 4>;
+						#endif
+						//remote-endpoint = <&imx290_csi2>;
+					};
+				};
+				#endif
+
+				port@2 {
+					reg = <2>;
+					d1_csi2_dma_1_3: endpoint {
+						bus-type = <5>;
+						remote-endpoint = <&d1_csi_dmar_1>;
+					};
+				};
+			};
+
+		};
+
+		d1_dc_test: dctest@702c0000 {
+			compatible = "eswin,dc";
+			reg = <0x0 0x702c0000 0x0 0x10000>;
+			interrupt-parent = <&plic1>;
+			interrupts = <238>;
+			numa-node-id = <1>;
+		};
+
+		d1_numa_sample:numa_sample@1 {
+			compatible = "eswin,numa-sample","simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			dma-ranges = <0x0 0x80000000 0x0 0xc0000000 0x0 0x80000000>;
+			iommus = <&smmu1 WIN2030_SID_SCPU>;
+			numa-node-id = <1>;
+			dma-noncoherent;
+			status = "disabled";
+		};
+
+		d1_ddr0: ddr-controller@72300000 {
+			compatible = "eswin,ddrc-1.20a";
+			interrupt-parent = <&plic1>;
+			interrupts = <10>;
+			interrupt-names = "ddr-ecc";
+			reg = <0x0 0x72300000 0x0 0x40000>;
+			ctrl-id = <0>;
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+
+		d1_ddr1: ddr-controller@72380000 {
+			compatible = "eswin,ddrc-1.20a";
+			interrupt-parent = <&plic1>;
+			interrupts = <299>;
+			interrupt-names = "ddr-ecc";
+			reg = <0x0 0x72380000 0x0 0x40000>;
+			ctrl-id = <1>;
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+
+		d1_d2d: d2d-unit@72100000 {
+			compatible = "eswin,eic7x-d2d";
+			interrupt-parent = <&plic1>;
+			interrupts = <287>, <288>;
+			reg = <0x0 0x72100000 0x0 0x50000>;
+			reg-names = "control";
+			numa-node-id = <1>;
+			status = "disabled";
+		};
+};
+
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030-platform.dtsi b/arch/riscv/boot/dts/eswin/eswin-win2030-platform.dtsi
new file mode 100644
index 000000000000..0754fce5b9d7
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030-platform.dtsi
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Configuration of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/*
+CHIPLET_AND_DIE = ((CHIPLET_NUM << 1) | AVAILABLE_DIE)
+BIT1  BIT0
+0	0	CHIPLET_1 | DIE0
+0	1	CHIPLET_1 | DIE1
+1	X       CHIPLET_2 | X
+*/
+
+#define CHIPLET_NUM 0
+#define PLATFORM_HAPS
+#define MEMMODE_FLAT
+#define AVAILABLE_DIE 0
+
+//#define CHIPLET_AND_DIE		((CHIPLET_NUM << 1) | AVAILABLE_DIE)
+
+/* VI related Macros */
+#define MIPI_CSI2_IMX290_ENBALE
+#define MIPI_CSI2_2LINE_ENABLE
+
+#if (CHIPLET_NUM==0)
+#if defined PLATFORM_HAPS
+#define RTCCLK_FREQ		1000000
+#define CPUCLK_FREQ		7500000
+#define LSPCLK_FREQ		5000000
+#define MEMORY_SIZE_H		0x2
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x10000000
+#elif defined PLATFORM_ZEBU
+#define RTCCLK_FREQ		1000000
+#define CPUCLK_FREQ		5000000
+#define LSPCLK_FREQ		5000000
+#define MEMORY_SIZE_H		0x2
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x10000000
+#elif defined PLATFORM_EIC7700_E_LX_G3_A1
+#define RTCCLK_FREQ		1000000
+#define CPUCLK_FREQ		5000000
+#define LSPCLK_FREQ		5000000
+#define MEMORY_SIZE_H		0x2
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x10000000
+#elif defined PLATFORM_EIC7700_E_L5_G2_A1
+#define RTCCLK_FREQ		1000000
+#define CPUCLK_FREQ		5000000
+#define LSPCLK_FREQ		5000000
+#define MEMORY_SIZE_H		0x1
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x10000000
+#endif
+#else /* CHIPLET_2 */
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		5000000
+#define CPUCLK_FREQ		5000000
+#ifdef MEMMODE_INTERLEAVE
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x8000000
+#else /* non-interleaving */
+#define MEMORY_SIZE_H		0x2
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x10000000
+#endif /* end of MEMMODE_INTERLEAVE */
+#endif /* end of (CHIPLET_NUM==0) */
diff --git a/arch/riscv/boot/dts/eswin/eswin-win2030.dts b/arch/riscv/boot/dts/eswin/eswin-win2030.dts
new file mode 100644
index 000000000000..766dcb95bb71
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin-win2030.dts
@@ -0,0 +1,1379 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#include "eswin-win2030-platform.dtsi"
+#include "eic7700-pinctrl.dtsi"
+
+#if ((CHIPLET_AND_DIE & 0x2) || (CHIPLET_AND_DIE == 0x0))
+#include "eswin-win2030-die0-soc.dtsi"
+#endif
+
+#if ((CHIPLET_AND_DIE & 0x2) || (CHIPLET_AND_DIE == 0x1))
+#include "eswin-win2030-die1-soc.dtsi"
+#endif
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	/*model = "SiFive HiFive Unmatched A00";*/
+	model = "ESWIN WIN2030";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740";
+
+#if ((CHIPLET_AND_DIE & 0x2) || (CHIPLET_AND_DIE == 0x0))
+    aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+	};
+#endif
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+#if (CHIPLET_NUM == 1)
+#ifdef MEMMODE_INTERLEAVE
+	memory@4000000000 {
+		device_type = "memory";
+		reg = <0x40 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x40 0x0 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		smpmemtest_rsv0@4011200000 {
+			reg = <0x40 0x11200000 0 0x2000000>;
+			no-map;
+		};
+
+		smpmemtest_rsv1@4013200000 {
+			reg = <0x40 0x13200000 0 0x2000000>;
+			no-map;
+		};
+
+		lpcpu0_reserved: lpcpu@4020000000 {
+			no-map;
+			reg = <0x40 0x20000000 0x0 0x100000>;
+		};
+
+		lpcpu1_reserved: lpcpu@4020100000 {
+			no-map;
+			reg = <0x40 0x20100000 0x0 0x100000>;
+		};
+
+		secure_memory_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x40 0xb0000000 0x0 0x10000000>;
+			no-map;
+		};
+
+		secure_memory_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x40 0xc0000000 0x0 0x10000000>;
+			no-map;
+		};
+	};
+#else
+	memory@2000000000 {
+		device_type = "memory";
+		reg = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <1>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		smpmemtest_rsv0@91200000 {
+			reg = <0 0x91200000 0 0x2000000>;
+			no-map;
+		};
+
+		secure_memory_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x0 0xb0000000 0x0 0x8000000>;
+			no-map;
+		};
+
+		secure_memory_nid_0_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x0 (0xb0000000 + 0x8000000) 0x0 0x8000000>;
+			no-map;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x0 0xc0000000 0x0 0x40000000>;
+			no-map;
+		};
+
+		mmz_nid_0_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x1 0x0 0x0 0x40000000>;
+			no-map;
+		};
+
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 0x0 0x0 0x40000000>;
+			no-map;
+		};
+
+		mmz_nid_1_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 0x40000000 0x0 0x40000000>;
+			no-map;
+		};
+
+		smpmemtest_rsv1@2000000000 {
+			reg = <0x20 0x80000000 0 0x2000000>;
+			no-map;
+		};
+
+		lpcpu0_reserved: lpcpu@a0000000 {
+			no-map;
+			reg = <0x0 0xa0000000 0x0 0x100000>;
+		};
+
+		lpcpu1_reserved: lpcpu@20a0000000 {
+			no-map;
+			reg = <0x20 0xa0000000 0x0 0x100000>;
+		};
+
+		secure_memory_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 0xd0000000 0x0 0x8000000>;
+			no-map;
+		};
+
+		secure_memory_nid_1_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 (0xd0000000 + 0x8000000) 0x0 0x8000000>;
+			no-map;
+		};
+	};
+#endif
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 15>,
+				<1 0 15>,
+				<1 1 10>;
+	};
+#else //CHIPLET_1
+#if (AVAILABLE_DIE == 0)
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		smpmemtest_rsv0@91200000 {
+			reg = <0 0x91200000 0 0x2000000>;
+			no-map;
+		};
+
+		lpcpu0_reserved: lpcpu@a0000000 {
+			no-map;
+			reg = <0x0 0xa0000000 0x0 0x100000>;
+		};
+
+		secure_memory_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x0 0xb0000000 0x0 0x8000000>;
+			no-map;
+		};
+
+		secure_memory_nid_0_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x0 (0xb0000000 + 0x8000000) 0x0 0x8000000>;
+			no-map;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x0 0xc0000000 0x1 0x40000000>;
+			no-map;
+		};
+
+		mmz_nid_0_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x2 0x0 0x0 0x8000000>;
+			no-map;
+		};
+	};
+
+	soc {
+		reset_test@1e00e000 {
+			compatible	= "reset_test";
+			resets = <&d0_reset SCPU_RST_CTRL SW_SCPU_BUS_RSTN>,
+					 <&d0_reset SCPU_RST_CTRL SW_SCPU_CORE_RSTN>,
+					 <&d0_reset SCPU_RST_CTRL SW_SCPU_DBG_RSTN>;
+			reset-names = "bus", "core", "dbg";
+		};
+	};
+#else
+	memory@2000000000 {
+		device_type = "memory";
+		reg = <0x20 0x0 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x20 0x0 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		smpmemtest_rsv0@2011200000 {
+			reg = <0x20 0x11200000 0 0x2000000>;
+			no-map;
+		};
+
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 0x40000000 0x0 0x40000000>;
+			no-map;
+		};
+
+		mmz_nid_1_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 0x80000000 0x0 0x40000000>;
+			no-map;
+		};
+
+		lpcpu1_reserved: lpcpu@20c0000000 {
+			no-map;
+			reg = <0x20 0xc0000000 0x0 0x100000>;
+		};
+
+		secure_memory_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 0xd0000000 0x0 0x8000000>;
+			no-map;
+		};
+
+		secure_memory_nid_1_part_1 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x20 (0xd0000000 + 0x8000000) 0x0 0x8000000>;
+			no-map;
+		};
+	};
+#endif /* end of (AVAILABLE_DIE == 0) */
+#endif /* end of (CHIPLET_NUM==1) */
+};
+
+#if ((CHIPLET_AND_DIE & 0x2) || (CHIPLET_AND_DIE == 0x0))
+&d0_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	status = "okay";
+};
+
+&d0_uart2 {
+	status = "okay";
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_dmac0 {
+	status = "disabled";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_gmac0 {
+	status = "disabled";
+};
+
+&d0_gmac1 {
+	status = "disabled";
+};
+
+
+&d0_npu {
+	status = "okay";
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "disabled";
+};
+
+&d0_dsp2 {
+	status = "disabled";
+};
+
+&d0_dsp3 {
+	status = "disabled";
+};
+
+&d0_i2s0 {
+	status = "disabled";
+	eswin-plat = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "disabled";
+	eswin-plat = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "disabled";
+	dais = <&d0_i2s0_port
+			&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "disabled";
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+};
+
+&gc820 {
+	status = "disabled";
+};
+
+&gpu0 {
+	status = "disabled";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&ssi0 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	status = "disabled";
+};
+
+&sdio0 {
+	status = "disabled";
+};
+
+&sdio1 {
+	status = "disabled";
+};
+
+&vdec0 {
+	status = "disabled";
+};
+
+&venc0 {
+	status = "disabled";
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "disabled";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "disabled";
+};
+
+&dvb_widgets {
+	status = "disabled";
+	dw_name = "dvb-widgets";
+	dw_num = <8>;
+	led-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+};
+
+&pvt0 {
+	status = "disabled";
+};
+
+&pvt1 {
+	status = "disabled";
+};
+
+&fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	status = "disabled";
+	eeprom@50 {
+		compatible = "atmel,24c1024";
+		reg = <0x50>;
+	};
+};
+
+&d0_pmu {
+	status = "disabled";
+};
+
+&d0_i2c1 {
+	status = "disabled";
+};
+
+&d0_i2c2 {
+	status = "disabled";
+	d0_es8316: es8316@10 {
+		compatible = "everest,es8316";
+		reg = <0x10>;
+		interrupts = <107>;
+		interrupt-parent = <&plic0>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c3 {
+	status = "disabled";
+	eeprom@51 {
+		compatible = "atmel,24c02";
+		reg = <0x51>;
+	};
+};
+
+&d0_i2c4 {
+	status = "disabled";
+	i2c-slave-eeprom@51 {
+		compatible = "slave-24c02";
+		reg = <(I2C_OWN_SLAVE_ADDRESS | 0x51)>;
+	};
+};
+
+&d0_i2c5 {
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	status = "disabled";
+	aon_eeprom@50 {
+		compatible = "atmel,24c1024";
+		reg = <0x50>;
+	};
+};
+
+&d0_aon_i2c1 {
+	status = "disabled";
+};
+
+&pinctrl {
+	status = "disabled";
+	pinctrl_gpio27_default: gpio27-default{
+		mux{
+			groups = "gpio27_group";
+			function = "gpio27_func";
+		};
+		conf {
+			groups = "gpio27_group";
+			input-enable = <1>;
+		};
+	};
+};
+
+&gpio0 {
+	status = "disabled";
+	porta: gpio-port@0 {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio27_default>;
+		gpio-ranges = <&pinctrl 27 55 111>;
+		interrupts = <303>,<311>,<312>,<313>,<314>,<331>;
+		interrupt-state = <0 1 1 1>,<8 0 1 1>,<9 0 1 1>,
+							<10 0 1 1>,<11 0 1 1>,<28 0 1 1>;
+		direction-input = <0 8 9 10 11 28>;
+		direction-output = <7 1 16 0>;
+		gpio-state = <11 1 12 1>;
+	};
+	portb: gpio-port@1 {
+		direction-input = <5 13 9 25>;
+		direction-output = <26 0 3 1>;
+		gpio-state = <11 1 17 1>;
+	};
+
+	portc: gpio-port@2 {
+		direction-input = <5 13 9 25>;
+		direction-output = <30 0 3 1>;
+		gpio-state = <11 1 17 1>;
+	};
+
+	portd: gpio-port@3 {
+		direction-input = <9 1 8 1>;
+		direction-output = <10 1 11 1 12 1 13 1 14 1 15 0>;
+		gpio-state = <6 1 5 1>;
+	};
+};
+
+&pwm0 {
+	status = "disabled";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&timer0 {
+	status = "disabled";
+};
+
+&timer1 {
+	status = "disabled";
+};
+
+&timer2 {
+	status = "disabled";
+};
+
+&timer3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "disabled";
+};
+
+&d0_sata {
+	status = "disabled";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&video_output {
+	status = "disabled";
+};
+
+&dc {
+	status = "disabled";
+};
+
+&virtual_display {
+    status = "disabled";
+};
+
+&dsi_output {
+    status = "disabled";
+};
+
+&dsi_controller {
+    status = "disabled";
+};
+
+&dc_test {
+    status = "disabled";
+};
+
+&dw_hdmi {
+    status = "disabled";
+    eswin-plat = <0>;
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "disabled";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_usbdrd3_0 {
+	status = "disabled";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "disabled";
+	dr_mode = "peripheral";
+};
+
+&d0_usbdrd3_1 {
+	status = "disabled";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "disabled";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "disabled";
+};
+
+&mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&d0_numa_sample {
+	status = "disabled";
+};
+
+#endif
+
+/* Devices on Die 1 */
+#if ((CHIPLET_AND_DIE & 0x2) || (CHIPLET_AND_DIE == 0x1))
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+&dev_foo_for_die1_mapping {
+	status = "okay";
+};
+
+&d1_reset {
+	status = "okay";
+};
+
+&d1_clock {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "disabled";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&vdec1 {
+	status = "disabled";
+};
+
+&venc1 {
+	status = "disabled";
+};
+
+&d1_mbox0 {
+	status = "disabled";
+};
+
+&d1_mbox1 {
+	status = "disabled";
+};
+
+&d1_mbox2 {
+	status = "disabled";
+};
+
+&d1_mbox3 {
+	status = "disabled";
+};
+
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d1_mbox7 {
+	status = "okay";
+};
+
+&d1_ipc_scpu {
+	status = "disabled";
+};
+
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&d1_i2c0 {
+	status = "disabled";
+	eeprom@50 {
+		compatible = "atmel,24c1024";
+		reg = <0x50>;
+	};
+};
+
+&d1_pmu {
+	status = "disabled";
+};
+
+&d1_i2c1 {
+	status = "disabled";
+};
+
+&d1_i2c2 {
+	status = "disabled";
+};
+
+&d1_i2c3 {
+	status = "disabled";
+};
+
+&d1_i2c4 {
+	status = "disabled";
+};
+
+&d1_i2c5 {
+	status = "disabled";
+};
+
+&d1_i2c6 {
+	status = "disabled";
+};
+
+&d1_i2c7 {
+	status = "disabled";
+};
+
+&d1_i2c8 {
+	status = "disabled";
+};
+
+&d1_i2c9 {
+	status = "disabled";
+};
+
+&d1_i2s0 {
+	status = "disabled";
+};
+
+&d1_i2s1 {
+	status = "disabled";
+};
+
+&d1_i2s2 {
+	status = "disabled";
+};
+
+&d1_graphcard {
+	status = "disabled";
+};
+
+&d1_aon_i2c0 {
+	status = "disabled";
+	aon_eeprom@50 {
+		compatible = "atmel,24c1024";
+		reg = <0x50>;
+	};
+};
+
+&d1_aon_i2c1 {
+	status = "disabled";
+};
+
+&d1_npu {
+	status = "disabled";
+};
+
+&d1_dsp_subsys {
+	status = "disabled";
+};
+
+&d1_dsp0 {
+	status = "disabled";
+};
+
+&d1_dsp1 {
+	status = "disabled";
+};
+
+&d1_dsp2 {
+	status = "disabled";
+};
+
+&d1_dsp3 {
+	status = "disabled";
+};
+
+&d0_pcie {
+	status = "disabled";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "ddr0_p0_req";
+	stat,1 = "ddr1_p0_req";
+	//latency,0 = "llcnoc_trans_probe";
+	//pending,0 = "llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req";
+	stat,1 = "TracePort:ddr0_p2_req";
+	stat,2 = "TracePort:ddr1_p1_req";
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	//latency,0 = "TracePort:sysnoc_trans_probe_0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1",
+		    "Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1", "Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,2 = "TracePort:sysnoc_trans_probe_2";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_gmac0 {
+	status = "disabled";
+};
+
+&d1_gmac1 {
+	status = "disabled";
+};
+
+&d1_sdio0 {
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	status = "disabled";
+};
+
+&d1_ssi0 {
+	status = "disabled";
+};
+
+&d1_gpu {
+	status = "disabled";
+};
+
+&d1_sata {
+	status = "disabled";
+};
+
+&d1_pinctrl {
+	status = "disabled";
+};
+
+&d1_gpio0 {
+	status = "disabled";
+};
+
+&d1_timer0 {
+	status = "disabled";
+};
+
+&d1_timer1 {
+	status = "disabled";
+};
+
+&d1_timer2 {
+	status = "disabled";
+};
+
+&d1_timer3 {
+	status = "disabled";
+};
+
+&d1_rtc {
+	status = "disabled";
+};
+
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&d1_pvt0 {
+	status = "disabled";
+};
+
+&d1_pvt1 {
+	status = "disabled";
+};
+
+&d1_fan_control {
+       status = "disabled";
+};
+
+&d1_sdhci_emmc {
+	status = "disabled";
+};
+
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&d1_video_output {
+	status = "disabled";
+};
+
+&d1_dc {
+	status = "disabled";
+};
+
+&d1_virtual_display {
+    status = "disabled";
+};
+
+&d1_dsi_output {
+    status = "disabled";
+};
+
+&d1_dsi_controller {
+    status = "disabled";
+};
+
+&d1_dc_test {
+    status = "disabled";
+};
+
+&d1_dw_hdmi {
+    status = "disabled";
+};
+
+&d1_dw_hdmi_hdcp2 {
+    status = "disabled";
+};
+
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&d1_gc820 {
+	status = "disabled";
+};
+
+&d1_usbdrd3_0 {
+	status = "disabled";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "disabled";
+};
+
+&d1_usbdrd3_1 {
+	status = "disabled";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "disabled";
+};
+
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&d1_isp_1 {
+	status = "disabled";
+};
+
+&d1_dewarp {
+	status = "disabled";
+};
+
+&d1_mipi_dphy_rx {
+	status = "disabled";
+};
+
+&d1_csi_dma0 {
+	status = "disabled";
+};
+
+&d1_csi_dma1 {
+	status = "disabled";
+};
+
+&d1_csi2_0 {
+	status = "disabled";
+};
+
+&d1_csi2_1 {
+	status = "disabled";
+};
+
+&d1_numa_sample {
+	status = "disabled";
+};
+
+#endif
diff --git a/arch/riscv/boot/dts/eswin/eswin_aicard.dts b/arch/riscv/boot/dts/eswin/eswin_aicard.dts
new file mode 100644
index 000000000000..0449d80e39c7
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eswin_aicard.dts
@@ -0,0 +1,1121 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x3
+#define MEMORY_SIZE_L		0x80000000
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 VPU A1";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x0>;
+			no-map;
+		};
+
+		rpc_reserved:rpc_reserved@100000000 {
+			reg = <0x1 0x0 0 0x1000>;
+			no-map;
+		};
+
+		commu_reserved:commu_reserved@100001000 {
+			reg = <0x1 0x1000 0 0x10000>;
+			no-map;
+		};
+
+		kernel_debug_reserved:kernel_debug_reserved@100011000 {
+			reg = <0x1 0x11000 0 0x100000>;
+			no-map;
+		};
+
+		pgretire_reserved:pgretire_reserved@100111000 {
+			reg = <0x1 0x111000 0 0x100000>;
+			no-map;
+		};
+
+		configs_reserved:configs_reserved@100211000 {
+			reg = <0x1 0x211000 0 0x1000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring0:vdev0vring0@100212000 {
+			reg = <0x1 0x212000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring1:vdev0vring1@100312000 {
+			reg = <0x1 0x312000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0buffer:vdev0buffer@100412000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x412000 0 0x4000000>; // pre is rx
+			no-map;
+		};
+
+		rsc_table0:rsc_table0@104412000 {
+			reg = <0x1 0x4412000 0 0x1000>;
+			no-map;
+		};
+
+		version_table:version_table@104413000 {
+			reg = <0x1 0x4413000 0 0x1000>;
+			no-map;
+		};
+
+		bd_table:bd_table@104414000 {
+			reg = <0x1 0x4414000 0 0x1000>;
+			no-map;
+		};
+
+		share_memory {
+			compatible = "eswin,shm";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			phys-base = <0x1 0x4415000>;
+			virt-base = <0x1 0x4415000>;
+			shm-size = <0xbbeb000>;
+			reg = <0x1 0x4415000 0 0xbbeb000>;
+		};
+
+	};
+
+	soc {
+		d0_ipcm:ipcm@0 {
+			compatible = "eswin,ipcm", "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			vf_id = <0>;
+			numa-node-id = <0>;
+			memory-region = <&ipcm_vdev0vring0>,<&ipcm_vdev0vring1>,<&ipcm_vdev0buffer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+		  	smmu_dev = <&smmu_cfg_0>;
+		};
+
+		dev_smmu_cfg: smmu_cfg@0 {
+			compatible = "eswin,vpu7702_dev_smmu_cfg";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0xc0000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_B>;
+			iommu-map = <0x000000 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "okay";
+			numa-node-id = <0>;
+			dma-noncoherent;
+                        #include "smmu_cfg.dtsi"
+
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	/delete-property/ mboxes;
+	status = "okay";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&d0_npu{
+	status = "okay";
+	apply_npu_1G_freq;
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "disabled";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "disabled";
+};
+
+&dc {
+	status = "disabled";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "disabled";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "disabled";
+};
+
+&dsi_controller {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+&d0_i2s0 {
+	status = "disabled";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "disabled";
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "disabled";
+};
+
+&mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "disabled";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "disabled";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "disabled";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_sata {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "disabled";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "disabled";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* 12V_DC_IN */
+	status = "disabled";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&d0_i2c1 {
+	/* NPU Power Control */
+	status = "disabled";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* Temperature Sensor */
+	status = "disabled";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_i2c3 {
+	/* Die 2 Die Serdes */
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* SMBUS */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* AT24C02C EEPROM */
+	status = "disabled";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ADC */
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c10_default>;
+	status = "disabled";
+
+	adc@48 {
+		compatible = "ti,ads1115";
+		reg = <0x48>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@4 {
+			reg = <4>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@5 {
+			reg = <5>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@6 {
+			reg = <6>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@7 {
+			reg = <7>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+	};
+};
+
+&d0_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+
+	pinctrl_gpio12_default: gpio12-default{
+		mux {
+			groups = "gpio12_group";
+			function = "gpio12_func";
+		};
+		conf {
+			groups = "gpio12_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio27_default: gpio27-default{
+		mux {
+			groups = "gpio27_group";
+			function = "gpio27_func";
+		};
+		conf {
+			groups = "gpio27_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio58_default: gpio58-default {
+		mux {
+			groups = "gpio58_group";
+			function = "gpio58_func";
+		};
+		conf {
+			groups = "gpio58_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio59_default: gpio59-default {
+		mux {
+			groups = "gpio59_group";
+			function = "gpio59_func";
+		};
+		conf {
+			groups = "gpio59_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio60_default: gpio60-default {
+		mux {
+			groups = "gpio60_group";
+			function = "gpio60_func";
+		};
+		conf {
+			groups = "gpio60_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio61_default: gpio61-default {
+		mux {
+			groups = "gpio61_group";
+			function = "gpio61_func";
+		};
+		conf {
+			groups = "gpio61_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio12_default
+	&pinctrl_gpio27_default
+	&pinctrl_gpio58_default
+	&pinctrl_gpio59_default
+	&pinctrl_gpio60_default
+	&pinctrl_gpio61_default>;
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/smmu_cfg.dtsi b/arch/riscv/boot/dts/eswin/smmu_cfg.dtsi
new file mode 100644
index 000000000000..f067df5a1014
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/smmu_cfg.dtsi
@@ -0,0 +1,256 @@
+smmu_cfg_0: smmu_cfg0@0 {};
+smmu_cfg_1: smmu_cfg1@1 {};
+smmu_cfg_2: smmu_cfg2@2 {};
+smmu_cfg_3: smmu_cfg3@3 {};
+smmu_cfg_4: smmu_cfg4@4 {};
+smmu_cfg_5: smmu_cfg5@5 {};
+smmu_cfg_6: smmu_cfg6@6 {};
+smmu_cfg_7: smmu_cfg7@7 {};
+smmu_cfg_8: smmu_cfg8@8 {};
+smmu_cfg_9: smmu_cfg9@9 {};
+smmu_cfg_10: smmu_cfg10@10 {};
+smmu_cfg_11: smmu_cfg11@11 {};
+smmu_cfg_12: smmu_cfg12@12 {};
+smmu_cfg_13: smmu_cfg13@13 {};
+smmu_cfg_14: smmu_cfg14@14 {};
+smmu_cfg_15: smmu_cfg15@15 {};
+smmu_cfg_16: smmu_cfg16@16 {};
+smmu_cfg_17: smmu_cfg17@17 {};
+smmu_cfg_18: smmu_cfg18@18 {};
+smmu_cfg_19: smmu_cfg19@19 {};
+smmu_cfg_20: smmu_cfg20@20 {};
+smmu_cfg_21: smmu_cfg21@21 {};
+smmu_cfg_22: smmu_cfg22@22 {};
+smmu_cfg_23: smmu_cfg23@23 {};
+smmu_cfg_24: smmu_cfg24@24 {};
+smmu_cfg_25: smmu_cfg25@25 {};
+smmu_cfg_26: smmu_cfg26@26 {};
+smmu_cfg_27: smmu_cfg27@27 {};
+smmu_cfg_28: smmu_cfg28@28 {};
+smmu_cfg_29: smmu_cfg29@29 {};
+smmu_cfg_30: smmu_cfg30@30 {};
+smmu_cfg_31: smmu_cfg31@31 {};
+smmu_cfg_32: smmu_cfg32@32 {};
+smmu_cfg_33: smmu_cfg33@33 {};
+smmu_cfg_34: smmu_cfg34@34 {};
+smmu_cfg_35: smmu_cfg35@35 {};
+smmu_cfg_36: smmu_cfg36@36 {};
+smmu_cfg_37: smmu_cfg37@37 {};
+smmu_cfg_38: smmu_cfg38@38 {};
+smmu_cfg_39: smmu_cfg39@39 {};
+smmu_cfg_40: smmu_cfg40@40 {};
+smmu_cfg_41: smmu_cfg41@41 {};
+smmu_cfg_42: smmu_cfg42@42 {};
+smmu_cfg_43: smmu_cfg43@43 {};
+smmu_cfg_44: smmu_cfg44@44 {};
+smmu_cfg_45: smmu_cfg45@45 {};
+smmu_cfg_46: smmu_cfg46@46 {};
+smmu_cfg_47: smmu_cfg47@47 {};
+smmu_cfg_48: smmu_cfg48@48 {};
+smmu_cfg_49: smmu_cfg49@49 {};
+smmu_cfg_50: smmu_cfg50@50 {};
+smmu_cfg_51: smmu_cfg51@51 {};
+smmu_cfg_52: smmu_cfg52@52 {};
+smmu_cfg_53: smmu_cfg53@53 {};
+smmu_cfg_54: smmu_cfg54@54 {};
+smmu_cfg_55: smmu_cfg55@55 {};
+smmu_cfg_56: smmu_cfg56@56 {};
+smmu_cfg_57: smmu_cfg57@57 {};
+smmu_cfg_58: smmu_cfg58@58 {};
+smmu_cfg_59: smmu_cfg59@59 {};
+smmu_cfg_60: smmu_cfg60@60 {};
+smmu_cfg_61: smmu_cfg61@61 {};
+smmu_cfg_62: smmu_cfg62@62 {};
+smmu_cfg_63: smmu_cfg63@63 {};
+smmu_cfg_64: smmu_cfg64@64 {};
+smmu_cfg_65: smmu_cfg65@65 {};
+smmu_cfg_66: smmu_cfg66@66 {};
+smmu_cfg_67: smmu_cfg67@67 {};
+smmu_cfg_68: smmu_cfg68@68 {};
+smmu_cfg_69: smmu_cfg69@69 {};
+smmu_cfg_70: smmu_cfg70@70 {};
+smmu_cfg_71: smmu_cfg71@71 {};
+smmu_cfg_72: smmu_cfg72@72 {};
+smmu_cfg_73: smmu_cfg73@73 {};
+smmu_cfg_74: smmu_cfg74@74 {};
+smmu_cfg_75: smmu_cfg75@75 {};
+smmu_cfg_76: smmu_cfg76@76 {};
+smmu_cfg_77: smmu_cfg77@77 {};
+smmu_cfg_78: smmu_cfg78@78 {};
+smmu_cfg_79: smmu_cfg79@79 {};
+smmu_cfg_80: smmu_cfg80@80 {};
+smmu_cfg_81: smmu_cfg81@81 {};
+smmu_cfg_82: smmu_cfg82@82 {};
+smmu_cfg_83: smmu_cfg83@83 {};
+smmu_cfg_84: smmu_cfg84@84 {};
+smmu_cfg_85: smmu_cfg85@85 {};
+smmu_cfg_86: smmu_cfg86@86 {};
+smmu_cfg_87: smmu_cfg87@87 {};
+smmu_cfg_88: smmu_cfg88@88 {};
+smmu_cfg_89: smmu_cfg89@89 {};
+smmu_cfg_90: smmu_cfg90@90 {};
+smmu_cfg_91: smmu_cfg91@91 {};
+smmu_cfg_92: smmu_cfg92@92 {};
+smmu_cfg_93: smmu_cfg93@93 {};
+smmu_cfg_94: smmu_cfg94@94 {};
+smmu_cfg_95: smmu_cfg95@95 {};
+smmu_cfg_96: smmu_cfg96@96 {};
+smmu_cfg_97: smmu_cfg97@97 {};
+smmu_cfg_98: smmu_cfg98@98 {};
+smmu_cfg_99: smmu_cfg99@99 {};
+smmu_cfg_100: smmu_cfg100@100 {};
+smmu_cfg_101: smmu_cfg101@101 {};
+smmu_cfg_102: smmu_cfg102@102 {};
+smmu_cfg_103: smmu_cfg103@103 {};
+smmu_cfg_104: smmu_cfg104@104 {};
+smmu_cfg_105: smmu_cfg105@105 {};
+smmu_cfg_106: smmu_cfg106@106 {};
+smmu_cfg_107: smmu_cfg107@107 {};
+smmu_cfg_108: smmu_cfg108@108 {};
+smmu_cfg_109: smmu_cfg109@109 {};
+smmu_cfg_110: smmu_cfg110@110 {};
+smmu_cfg_111: smmu_cfg111@111 {};
+smmu_cfg_112: smmu_cfg112@112 {};
+smmu_cfg_113: smmu_cfg113@113 {};
+smmu_cfg_114: smmu_cfg114@114 {};
+smmu_cfg_115: smmu_cfg115@115 {};
+smmu_cfg_116: smmu_cfg116@116 {};
+smmu_cfg_117: smmu_cfg117@117 {};
+smmu_cfg_118: smmu_cfg118@118 {};
+smmu_cfg_119: smmu_cfg119@119 {};
+smmu_cfg_120: smmu_cfg120@120 {};
+smmu_cfg_121: smmu_cfg121@121 {};
+smmu_cfg_122: smmu_cfg122@122 {};
+smmu_cfg_123: smmu_cfg123@123 {};
+smmu_cfg_124: smmu_cfg124@124 {};
+smmu_cfg_125: smmu_cfg125@125 {};
+smmu_cfg_126: smmu_cfg126@126 {};
+smmu_cfg_127: smmu_cfg127@127 {};
+smmu_cfg_128: smmu_cfg128@128 {};
+smmu_cfg_129: smmu_cfg129@129 {};
+smmu_cfg_130: smmu_cfg130@130 {};
+smmu_cfg_131: smmu_cfg131@131 {};
+smmu_cfg_132: smmu_cfg132@132 {};
+smmu_cfg_133: smmu_cfg133@133 {};
+smmu_cfg_134: smmu_cfg134@134 {};
+smmu_cfg_135: smmu_cfg135@135 {};
+smmu_cfg_136: smmu_cfg136@136 {};
+smmu_cfg_137: smmu_cfg137@137 {};
+smmu_cfg_138: smmu_cfg138@138 {};
+smmu_cfg_139: smmu_cfg139@139 {};
+smmu_cfg_140: smmu_cfg140@140 {};
+smmu_cfg_141: smmu_cfg141@141 {};
+smmu_cfg_142: smmu_cfg142@142 {};
+smmu_cfg_143: smmu_cfg143@143 {};
+smmu_cfg_144: smmu_cfg144@144 {};
+smmu_cfg_145: smmu_cfg145@145 {};
+smmu_cfg_146: smmu_cfg146@146 {};
+smmu_cfg_147: smmu_cfg147@147 {};
+smmu_cfg_148: smmu_cfg148@148 {};
+smmu_cfg_149: smmu_cfg149@149 {};
+smmu_cfg_150: smmu_cfg150@150 {};
+smmu_cfg_151: smmu_cfg151@151 {};
+smmu_cfg_152: smmu_cfg152@152 {};
+smmu_cfg_153: smmu_cfg153@153 {};
+smmu_cfg_154: smmu_cfg154@154 {};
+smmu_cfg_155: smmu_cfg155@155 {};
+smmu_cfg_156: smmu_cfg156@156 {};
+smmu_cfg_157: smmu_cfg157@157 {};
+smmu_cfg_158: smmu_cfg158@158 {};
+smmu_cfg_159: smmu_cfg159@159 {};
+smmu_cfg_160: smmu_cfg160@160 {};
+smmu_cfg_161: smmu_cfg161@161 {};
+smmu_cfg_162: smmu_cfg162@162 {};
+smmu_cfg_163: smmu_cfg163@163 {};
+smmu_cfg_164: smmu_cfg164@164 {};
+smmu_cfg_165: smmu_cfg165@165 {};
+smmu_cfg_166: smmu_cfg166@166 {};
+smmu_cfg_167: smmu_cfg167@167 {};
+smmu_cfg_168: smmu_cfg168@168 {};
+smmu_cfg_169: smmu_cfg169@169 {};
+smmu_cfg_170: smmu_cfg170@170 {};
+smmu_cfg_171: smmu_cfg171@171 {};
+smmu_cfg_172: smmu_cfg172@172 {};
+smmu_cfg_173: smmu_cfg173@173 {};
+smmu_cfg_174: smmu_cfg174@174 {};
+smmu_cfg_175: smmu_cfg175@175 {};
+smmu_cfg_176: smmu_cfg176@176 {};
+smmu_cfg_177: smmu_cfg177@177 {};
+smmu_cfg_178: smmu_cfg178@178 {};
+smmu_cfg_179: smmu_cfg179@179 {};
+smmu_cfg_180: smmu_cfg180@180 {};
+smmu_cfg_181: smmu_cfg181@181 {};
+smmu_cfg_182: smmu_cfg182@182 {};
+smmu_cfg_183: smmu_cfg183@183 {};
+smmu_cfg_184: smmu_cfg184@184 {};
+smmu_cfg_185: smmu_cfg185@185 {};
+smmu_cfg_186: smmu_cfg186@186 {};
+smmu_cfg_187: smmu_cfg187@187 {};
+smmu_cfg_188: smmu_cfg188@188 {};
+smmu_cfg_189: smmu_cfg189@189 {};
+smmu_cfg_190: smmu_cfg190@190 {};
+smmu_cfg_191: smmu_cfg191@191 {};
+smmu_cfg_192: smmu_cfg192@192 {};
+smmu_cfg_193: smmu_cfg193@193 {};
+smmu_cfg_194: smmu_cfg194@194 {};
+smmu_cfg_195: smmu_cfg195@195 {};
+smmu_cfg_196: smmu_cfg196@196 {};
+smmu_cfg_197: smmu_cfg197@197 {};
+smmu_cfg_198: smmu_cfg198@198 {};
+smmu_cfg_199: smmu_cfg199@199 {};
+smmu_cfg_200: smmu_cfg200@200 {};
+smmu_cfg_201: smmu_cfg201@201 {};
+smmu_cfg_202: smmu_cfg202@202 {};
+smmu_cfg_203: smmu_cfg203@203 {};
+smmu_cfg_204: smmu_cfg204@204 {};
+smmu_cfg_205: smmu_cfg205@205 {};
+smmu_cfg_206: smmu_cfg206@206 {};
+smmu_cfg_207: smmu_cfg207@207 {};
+smmu_cfg_208: smmu_cfg208@208 {};
+smmu_cfg_209: smmu_cfg209@209 {};
+smmu_cfg_210: smmu_cfg210@210 {};
+smmu_cfg_211: smmu_cfg211@211 {};
+smmu_cfg_212: smmu_cfg212@212 {};
+smmu_cfg_213: smmu_cfg213@213 {};
+smmu_cfg_214: smmu_cfg214@214 {};
+smmu_cfg_215: smmu_cfg215@215 {};
+smmu_cfg_216: smmu_cfg216@216 {};
+smmu_cfg_217: smmu_cfg217@217 {};
+smmu_cfg_218: smmu_cfg218@218 {};
+smmu_cfg_219: smmu_cfg219@219 {};
+smmu_cfg_220: smmu_cfg220@220 {};
+smmu_cfg_221: smmu_cfg221@221 {};
+smmu_cfg_222: smmu_cfg222@222 {};
+smmu_cfg_223: smmu_cfg223@223 {};
+smmu_cfg_224: smmu_cfg224@224 {};
+smmu_cfg_225: smmu_cfg225@225 {};
+smmu_cfg_226: smmu_cfg226@226 {};
+smmu_cfg_227: smmu_cfg227@227 {};
+smmu_cfg_228: smmu_cfg228@228 {};
+smmu_cfg_229: smmu_cfg229@229 {};
+smmu_cfg_230: smmu_cfg230@230 {};
+smmu_cfg_231: smmu_cfg231@231 {};
+smmu_cfg_232: smmu_cfg232@232 {};
+smmu_cfg_233: smmu_cfg233@233 {};
+smmu_cfg_234: smmu_cfg234@234 {};
+smmu_cfg_235: smmu_cfg235@235 {};
+smmu_cfg_236: smmu_cfg236@236 {};
+smmu_cfg_237: smmu_cfg237@237 {};
+smmu_cfg_238: smmu_cfg238@238 {};
+smmu_cfg_239: smmu_cfg239@239 {};
+smmu_cfg_240: smmu_cfg240@240 {};
+smmu_cfg_241: smmu_cfg241@241 {};
+smmu_cfg_242: smmu_cfg242@242 {};
+smmu_cfg_243: smmu_cfg243@243 {};
+smmu_cfg_244: smmu_cfg244@244 {};
+smmu_cfg_245: smmu_cfg245@245 {};
+smmu_cfg_246: smmu_cfg246@246 {};
+smmu_cfg_247: smmu_cfg247@247 {};
+smmu_cfg_248: smmu_cfg248@248 {};
+smmu_cfg_249: smmu_cfg249@249 {};
+smmu_cfg_250: smmu_cfg250@250 {};
+smmu_cfg_251: smmu_cfg251@251 {};
+smmu_cfg_252: smmu_cfg252@252 {};
+smmu_cfg_253: smmu_cfg253@253 {};
+smmu_cfg_254: smmu_cfg254@254 {};
+smmu_cfg_255: smmu_cfg255@255 {};
diff --git a/arch/riscv/boot/dts/eswin/u2-evb.dts b/arch/riscv/boot/dts/eswin/u2-evb.dts
new file mode 100644
index 000000000000..7309a3496b29
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/u2-evb.dts
@@ -0,0 +1,1745 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eswin-win2030-die1-soc.dtsi"
+
+#include "eic7x-die0-pinctrl.dtsi"
+#include "eic7x-die1-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN U2 EVB";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		ethernet2 = &d1_gmac0;
+		ethernet3 = &d1_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &d1_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@2000000000 {
+		device_type = "memory";
+		reg = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <1>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	memory@79000000 {
+		device_type = "memory";
+		reg = <0x0 0x79000000 0x0 0x400000>;
+		numa-node-id = <1>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x20 0x00000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		npu1_reserved: sprammemory@79000000 {
+			no-map;
+			reg = <0x0 0x79000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		nid_1_zero_device_simu: nid_1_zero_device_simu@2040000000 {
+			no-map;
+			reg = <0x20 0x40000000 0x0 0x400000>;
+		};
+
+		d1_g2d_4GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x20 0xfffff000 0x0 0x1000>;
+		};
+
+		d1_g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x21 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_1_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x22 0x80000000 0x1 0x80000000>;
+			no-map;
+		};
+
+		rpc_reserved:rpc_reserved@100000000 {
+			reg = <0x1 0x0 0 0x1000>;
+			no-map;
+		};
+
+		commu_reserved:commu_reserved@100001000 {
+			reg = <0x1 0x1000 0 0x10000>;
+			no-map;
+		};
+
+		kernel_debug_reserved:kernel_debug_reserved@100011000 {
+			reg = <0x1 0x11000 0 0x100000>;
+			no-map;
+		};
+
+		pgretire_reserved:pgretire_reserved@100111000 {
+			reg = <0x1 0x111000 0 0x100000>;
+			no-map;
+		};
+
+		configs_reserved:configs_reserved@100211000 {
+			reg = <0x10 0x211000 0 0x1000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring0:vdev0vring0@100212000 {
+			reg = <0x1 0x212000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring1:vdev0vring1@100312000 {
+			reg = <0x1 0x312000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0buffer:vdev0buffer@100412000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x412000 0 0x4000000>; // pre is rx
+			no-map;
+		};
+
+		rsc_table0:rsc_table0@104412000 {
+			reg = <0x1 0x4412000 0 0x1000>;
+			no-map;
+		};
+
+		version_table:version_table@104413000 {
+			reg = <0x1 0x4413000 0 0x1000>;
+			no-map;
+		};
+
+		bd_table:bd_table@104414000 {
+			reg = <0x1 0x4414000 0 0x1000>;
+			no-map;
+		};
+
+		share_memory {
+			compatible = "eswin,shm";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			phys-base = <0x1 0x4415000>;
+			virt-base = <0x1 0x4415000>;
+			shm-size = <0xbbeb000>;
+			reg = <0x1 0x4415000 0 0xbbeb000>;
+		};
+
+	};
+
+	soc {
+		d0_ipcm:ipcm@0 {
+			compatible = "eswin,ipcm", "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			vf_id = <0>;
+			numa-node-id = <0>;
+			memory-region = <&ipcm_vdev0vring0>,<&ipcm_vdev0vring1>,<&ipcm_vdev0buffer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+		  	smmu_dev = <&smmu_cfg_0>;
+		};
+
+		dev_smmu_cfg: smmu_cfg@0 {
+			compatible = "eswin,vpu7702_dev_smmu_cfg";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0xc0000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_B>;
+			iommu-map = <0x000000 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "okay";
+			numa-node-id = <0>;
+			dma-noncoherent;
+                        #include "smmu_cfg.dtsi"
+		};
+	};
+
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				<0 1 100>,
+				<1 0 100>,
+				<1 1 10>;
+	};
+
+	/* die1 gpio68 system led */
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio68_default>;
+
+		gpio-68 {
+			gpios = <&d1_portc 4 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+	reg_vcc1v8: vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&D1CACHE {
+	status = "okay";
+	d1_zero_device: zero-device@3a000000 {
+		compatible = "l3,zero-device";
+		/*reg = <0x0 0x3a000000 0x0 0x400000>;*/
+		memory-region = <&nid_1_zero_device_simu>;
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+&d1_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+&d1_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+&d1_pmu {
+	status = "disabled";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&d1_ddr0 {
+	status = "okay";
+};
+
+&d1_ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+&smmu1 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+&smmu_pmu1 {
+	status = "disabled";
+};
+
+
+&dev_foo_a {
+	status = "okay";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d1_cfg_noc {
+	status = "okay";
+};
+
+&d1_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d1_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d1_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d1_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d1_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+&d1_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	status = "okay";
+};
+&d1_lpcpu {
+	status = "disabled";
+};
+
+&pcie {
+	status = "disabled";
+};
+&d1_pcie {
+	status = "disabled";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&d0_npu_vcc>;
+};
+&d1_npu {
+	status = "okay";
+	npu-supply=<&d1_npu_vcc>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+&d1_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+&d1_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+&d1_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+&d1_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+&d1_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+&d1_gpu {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+&d1_gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+&vdec1 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+&venc1 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&d1_video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&d1_dc {
+	status = "okay";
+	d1_dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		d1_dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&d1_dsi_input>;
+		};
+		d1_dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&d1_vd_input>;
+		};
+		d1_dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&d1_hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+&d1_dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&d1_virtual_display {
+	status = "okay";
+	port {
+		d1_vd_input: endpoint {
+			remote-endpoint = <&d1_dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_controller {
+    status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+&d1_dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			d1_dsi_input: endpoint {
+				remote-endpoint = <&d1_dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			d1_mipi_dsi_out: endpoint {
+				remote-endpoint = <&d1_panel_in>;
+			};
+		};
+	};
+	d1_dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			d1_panel_in: endpoint {
+				remote-endpoint = <&d1_mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio27_default &pinctrl_die0_gpio106_default>;
+	backlight0-gpios = <&porta 27 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+};
+&d1_dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio69_default &pinctrl_die1_gpio21_default>;
+	backlight0-gpios = <&d1_portc 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&d1_porta 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+&d1_dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			d1_hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&d1_dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			d1_hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "okay";
+};
+&d1_dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+
+&d0_i2s0 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s0 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s0_port: port {
+		d1_i2s0_endpoint: endpoint {
+			remote-endpoint = <&d1_hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s1 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s1_port: port {
+		d1_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d1_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	numa-node-id = <0>;
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d1_i2s2 {
+	status = "okay";
+	numa-node-id = <1>;
+	d1_i2s2_port: port {
+		d1_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d1_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-0";
+	dais = <&d0_i2s0_port>;
+};
+
+&d1_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-2";
+	dais = <&d1_i2s1_port>;
+};
+
+&d1_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-3";
+	dais = <&d1_i2s2_port>;
+};
+
+&d1_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio-1";
+	dais = <&d1_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+&d1_isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "okay";
+};
+&d1_isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "okay";
+};
+&d1_dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+&d1_mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+&d1_csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+&d1_csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+&d1_csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+&d1_csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&d1_sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio13_default>;
+	cd-gpios = <&porta 13 GPIO_ACTIVE_LOW>;
+};
+
+&d1_sdio0 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	vqmmc-supply = <&reg_vcc1v8>;
+	aw315:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_die1_gpio18_default>;
+		pinctrl-1 = <&pinctrl_die1_gpio16_default>;
+		irq-gpios = <&d1_porta 18 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+	mdhd_wlan {
+		compatible = "android,bcmdhd_wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio16_default>;
+		gpio_wl_reg_on = <&d1_porta 16 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&sdio1 {
+	status = "disabled";
+};
+
+&d1_sdio1 {
+	status = "disabled";
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio15_default>;
+	rst-gpios = <&porta 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio13_default>;
+	rst-gpios = <&d1_porta 13 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x5a0c6423 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die0_gpio14_default>;
+	rst-gpios = <&porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d1_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_die1_gpio14_default>;
+	rst-gpios = <&d1_porta 14 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&d1_pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x5a266425 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+};
+
+&d1_sata {
+	status = "okay";
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+&d1_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d1_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d1_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d1_dmac0 {
+	status = "disabled";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d1_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d1_uart0 {
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d1_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d1_uart2 {
+	/* pin header */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+&d1_ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&d1_ssi1 {
+	/* unused */
+	status = "disabled";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d1_bootspi {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <1>;
+	cs-gpios = <&d1_portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&d1_portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+&d1_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+&d1_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+&d1_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+&d1_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+&d1_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+&d1_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+&d1_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+&d1_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+&d1_fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d1_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d1_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s1_endpoint>;
+			};
+		};
+	};
+	d1_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d1_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d1_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			d0_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+&d1_i2c1 {
+	status = "okay";
+	es5340@f {
+		compatible = "einno,es5340";
+		reg = <0xf>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "d1_npu_vdd";
+		regulators{
+			d1_npu_vcc:npu_svcc{
+				regulator-name="NPU_SVCC2";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	d0_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die0_gpio20_default>;
+		irq-gpios = <&porta 20 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d1_i2c2 {
+	/* touch screen*/
+	status = "ok";
+	d1_gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&d1_porta>;
+		interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_die1_gpio0_default>;
+		irq-gpios = <&d1_porta 0 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&d1_portd 15 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+&d1_i2c3 {
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+&d1_i2c4 {
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+&d1_i2c5 {
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c6 {
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c7 {
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+};
+&d1_i2c8 {
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+&d1_i2c9 {
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ina226x3 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x40>;
+		shunt-resistor = <1000>;
+	};
+	ina226@41 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_lpddr_vdd";
+		reg = <0x41>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "d0_soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+
+&d1_aon_i2c0 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@40 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "vdd_dc_in";
+			reg = <0x40>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "d1_soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d1_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+&d1_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+&d1_pwm0 {
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "disabled";
+};
+&d1_pvt0 {
+	status = "disabled";
+};
+
+&pvt1 {
+	status = "okay";
+};
+&d1_pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+&d1_wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+&d1_wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+&d1_wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+&d1_wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+&d1_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+&d1_timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "disabled";
+};
+&d1_timer1 {
+	status = "disabled";
+};
+
+&timer2 {
+	status = "okay";
+};
+&d1_timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+&d1_timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+&d1_pinctrl {
+	status = "okay";
+};
+
+&gpio0 {
+	status = "okay";
+};
+&d1_gpio0 {
+	status = "okay";
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d0_npu_vcc>;
+	status = "okay";
+};
+&d1_llc_dev{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&d1_npu_vcc>;
+	status = "okay";
+};
+
+&d2d {
+	status = "okay";
+};
+
+&d1_d2d {
+	status = "okay";
+};
+
+/* die0 GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio11  : tmp alert(I)
+	gpio13  : tf card detect(I)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : gphy0 resetn(O)
+	gpio18  : ina226 alert(I)
+	gpio20  : lcd touch int(I)
+	gpio21  : head phone plug/unplug detection2(I)
+	gpio27  : black light pwr_en(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : fan pwm(O)
+	gpio69  : fan touch(I)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : mipi dsi resetn(O)
+	gpio111 : lcd touch resetn(O)
+*/
+/* die1 GPIO Function Description
+
+	gpio0   : lcd touch int(I)
+	gpio13  : gphy0 resetn(O)
+	gpio14  : gphy1 resetn(O)
+	gpio15  : bt wake host(I)
+	gpio16  : sdio wifi disable(O)
+	gpio18  : sdio wifi wake host(I)
+	gpio19  : pcie prstn(I)
+	gpio20  : head phone plug/unplug detection2(I)
+	gpio21  : mipi dsi resetn(O)
+	gpio27  : tmp alert(I)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio68  : system led(O)
+	gpio69  : black light pwr_en(O)
+	gpio92  : frex gp0(O)
+	gpio93  : frex gp1(O)
+	gpio106 : host wak bt(O)
+	gpio111 : lcd touch resetn(O)
+*/
diff --git a/arch/riscv/boot/dts/eswin/vpu7702-evb.dts b/arch/riscv/boot/dts/eswin/vpu7702-evb.dts
new file mode 100644
index 000000000000..bd5395e60d17
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/vpu7702-evb.dts
@@ -0,0 +1,1095 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x4
+#define MEMORY_SIZE_L		0x0
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7700 EVB A2";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+		rtc1 = &pcf_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x80000000>;
+			no-map;
+		};
+
+		rpc_reserved:rpc_reserved@100000000 {
+			reg = <0x1 0x0 0 0x1000>;
+			no-map;
+		};
+
+		commu_reserved:commu_reserved@100001000 {
+			reg = <0x1 0x1000 0 0x10000>;
+			no-map;
+		};
+
+		kernel_debug_reserved:kernel_debug_reserved@100011000 {
+			reg = <0x1 0x11000 0 0x100000>;
+			no-map;
+		};
+
+		pgretire_reserved:pgretire_reserved@100111000 {
+			reg = <0x1 0x111000 0 0x100000>;
+			no-map;
+		};
+
+		configs_reserved:configs_reserved@100211000 {
+			reg = <0x10 0x211000 0 0x1000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring0:vdev0vring0@100212000 {
+			reg = <0x1 0x212000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring1:vdev0vring1@100312000 {
+			reg = <0x1 0x312000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0buffer:vdev0buffer@100412000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x412000 0 0x4000000>; // pre is rx
+			no-map;
+		};
+
+		rsc_table0:rsc_table0@104412000 {
+			reg = <0x1 0x4412000 0 0x1000>;
+			no-map;
+		};
+
+		version_table:version_table@104413000 {
+			reg = <0x1 0x4413000 0 0x1000>;
+			no-map;
+		};
+
+		bd_table:bd_table@104414000 {
+			reg = <0x1 0x4414000 0 0x1000>;
+			no-map;
+		};
+
+		share_memory {
+			compatible = "eswin,shm";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			phys-base = <0x1 0x4415000>;
+			virt-base = <0x1 0x4415000>;
+			shm-size = <0xbbeb000>;
+			reg = <0x1 0x4415000 0 0xbbeb000>;
+		};
+
+
+	};
+
+	soc {
+		d0_ipcm:ipcm@0 {
+			compatible = "eswin,ipcm", "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			vf_id = <0>;
+			numa-node-id = <0>;
+			memory-region = <&ipcm_vdev0vring0>,<&ipcm_vdev0vring1>,<&ipcm_vdev0buffer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+		  	smmu_dev = <&smmu_cfg_0>;
+		};
+
+		dev_smmu_cfg: smmu_cfg@0 {
+			compatible = "eswin,vpu7702_dev_smmu_cfg";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0xc0000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_B>;
+			iommu-map = <0x000000 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "okay";
+			numa-node-id = <0>;
+			dma-noncoherent;
+                        #include "smmu_cfg.dtsi"
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	/delete-property/ mboxes;
+	status = "okay";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&d0_npu{
+	status = "okay";
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s2 {
+	status = "okay";
+	d0_i2s2_port: port {
+		d0_i2s2_endpoint: endpoint {
+			remote-endpoint = <&d0_codec1_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	label = "Analog Audio-0";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	label = "Analog Audio-1";
+	dais = <&d0_i2s2_port>;
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@10 {
+		compatible = "eswin,es8388";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+	d0_es8388_1: es8388-1@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		port {
+			d0_codec1_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s2_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	/* mpq8785 */
+	status = "okay";
+	mpq8785@10 {
+		compatible = "mps,mpq8785";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		eswin,regulator_label = "supply vdd1", "npu vdd1", "npu current1", "npu temperature1";
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1100000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* touch screen*/
+	status = "okay";
+	gt911:touchscreen@14 {
+		compatible = "goodix,gt911";
+		reg = <0x14>;
+		interrupt-parent = <&porta>;
+		interrupts = <10 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio10_default>;
+		irq-gpios = <&porta 10 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&portd 10 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&d0_i2c3 {
+	/* mipi csi0/csi1 */
+	status = "okay";
+};
+
+&d0_i2c4 {
+	/* mipi csi2/csi3 */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* mipi csi4/csi5 */
+	status = "disabled";
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* io extended for mipi csi */
+	status = "okay";
+	tca6416_0: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller; /* IRQ not connected */
+		#gpio-cells = <2>;
+		gpio-line-names = "MIPI_CSI0_PWDN", "MIPI_CSI0_RESET", "MIPI_CSI1_FBC", "MIPI_CSI1_ENB",
+				"MIPI_CSI1_RESET", "MIPI_CSI1_PWDN", "FREX_GP0", "",
+				"MIPI_CSI0_ENB", "MIPI_CSI0_FBC", "FREX_GP2", "MIPI_CSI2_FBC",
+				"MIPI_CSI2_ENB", "FREX_GP1", "MIPI_CSI2_RESET", "MIPI_CSI2_PWDN";
+	};
+	tca6416_1: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		/* IRQ not connected */
+		gpio-line-names = "MIPI_CSI3_PWDN", "MIPI_CSI3_RESET", "MIPI_CSI3_ENB", "MIPI_CSI3_FBC",
+				"MIPI_CSI4_PWDN", "MIPI_CSI4_RESET", "MIPI_CSI4_ENB", "MIPI_CSI4_FBC",
+				"MIPI_CSI5_FBC", "MIPI_CSI5_ENB", "MIPI_CSI5_RESET", "MIPI_CSI5_PWDN",
+				"", "", "", "";
+	};
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* temp sensor & rtc */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	pcf_rtc:rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* ina226x4 */
+	status = "okay";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@45 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "cpu_vdd";
+		reg = <0x45>;
+		shunt-resistor = <1000>;
+	};
+	ina226@44 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "soc_vdd";
+			reg = <0x44>;
+			shunt-resistor = <1000>;
+		};
+	ina226@41 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "lpddr_vdd";
+			reg = <0x41>;
+			shunt-resistor = <1000>;
+		};
+	ina226@48 {
+			compatible = "ti,ina226";
+			#io-channel-cells = <1>;
+			label = "dc_vdd";
+			reg = <0x48>;
+			shunt-resistor = <1000>;
+		};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+};
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/vpu7702-pcie-a2.dts b/arch/riscv/boot/dts/eswin/vpu7702-pcie-a2.dts
new file mode 100644
index 000000000000..bca9411e1d19
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/vpu7702-pcie-a2.dts
@@ -0,0 +1,1181 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x3
+#define MEMORY_SIZE_L		0x80000000
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 VPU A2";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x0>;
+			no-map;
+		};
+
+		rpc_reserved:rpc_reserved@100000000 {
+			reg = <0x1 0x0 0 0x1000>;
+			no-map;
+		};
+
+		commu_reserved:commu_reserved@100001000 {
+			reg = <0x1 0x1000 0 0x10000>;
+			no-map;
+		};
+
+		kernel_debug_reserved:kernel_debug_reserved@100011000 {
+			reg = <0x1 0x11000 0 0x100000>;
+			no-map;
+		};
+
+		pgretire_reserved:pgretire_reserved@100111000 {
+			reg = <0x1 0x111000 0 0x100000>;
+			no-map;
+		};
+
+		configs_reserved:configs_reserved@100211000 {
+			reg = <0x1 0x211000 0 0x1000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring0:vdev0vring0@100212000 {
+			reg = <0x1 0x212000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring1:vdev0vring1@100312000 {
+			reg = <0x1 0x312000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0buffer:vdev0buffer@100412000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x412000 0 0x4000000>; // pre is rx
+			no-map;
+		};
+
+		rsc_table0:rsc_table0@104412000 {
+			reg = <0x1 0x4412000 0 0x1000>;
+			no-map;
+		};
+
+		version_table:version_table@104413000 {
+			reg = <0x1 0x4413000 0 0x1000>;
+			no-map;
+		};
+
+		bd_table:bd_table@104414000 {
+			reg = <0x1 0x4414000 0 0x1000>;
+			no-map;
+		};
+
+		share_memory {
+			compatible = "eswin,shm";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			phys-base = <0x1 0x4415000>;
+			virt-base = <0x1 0x4415000>;
+			shm-size = <0xbbeb000>;
+			reg = <0x1 0x4415000 0 0xbbeb000>;
+		};
+
+	};
+
+	soc {
+		d0_ipcm:ipcm@0 {
+			compatible = "eswin,ipcm", "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			vf_id = <0>;
+			numa-node-id = <0>;
+			memory-region = <&ipcm_vdev0vring0>,<&ipcm_vdev0vring1>,<&ipcm_vdev0buffer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+		  	smmu_dev = <&smmu_cfg_0>;
+		};
+
+		dev_smmu_cfg: smmu_cfg@0 {
+			compatible = "eswin,vpu7702_dev_smmu_cfg";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0xc0000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_B>;
+			iommu-map = <0x000000 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "okay";
+			numa-node-id = <0>;
+			dma-noncoherent;
+                        #include "smmu_cfg.dtsi"
+
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+/ {
+	thermal-zones {
+		thermal0 {
+			trips {
+				threshold: trip-point0 {
+					temperature = <65000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				target: trip-point1 {
+					temperature = <75000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				crit: trip-point2 {
+					temperature     = <95000>;
+					hysteresis      = <0>;
+					type            = "critical";
+				};
+			};
+
+			cooling-maps {
+				/*
+				map0 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device =
+						<&cpu_0 0 10>,
+						<&cpu_1 0 10>,
+						<&cpu_2 0 10>,
+						<&cpu_3 0 10>;
+				};
+				map1 {
+					trip = <&target>;
+					contribution = <4096>;
+					cooling-device = <&g2d_opp_table THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				map2 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device = <&venc_opp_table THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				map3 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device = <&vdec_opp_table THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				*/
+			};
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+
+	d0_pmu_vi: win2030-pmu-controller-port@80 {
+		power_status = <0>;
+	};
+	d0_pmu_vo: win2030-pmu-controller-port@c0 {
+		power_status = <0>;
+	};
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	/delete-property/ mboxes;
+	status = "okay";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&d0_npu{
+	status = "disabled";
+	apply_npu_1G_freq;
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "disabled";
+};
+
+&d0_dsp0 {
+	status = "disabled";
+};
+
+&d0_dsp1 {
+	status = "disabled";
+};
+
+&d0_dsp2 {
+	status = "disabled";
+};
+
+&d0_dsp3 {
+	status = "disabled";
+};
+
+&gpu0 {
+	status = "disabled";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "disabled";
+};
+
+&dc {
+	status = "disabled";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "disabled";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "disabled";
+};
+
+&dsi_controller {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+&d0_i2s0 {
+	status = "disabled";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "disabled";
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "disabled";
+};
+
+&mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "disabled";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "disabled";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "disabled";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_sata {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "disabled";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "disabled";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "disabled";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "disabled";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "disabled";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "disabled";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* 12V_DC_IN */
+	status = "disabled";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&d0_i2c1 {
+	/* NPU Power Control */
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* Temperature Sensor */
+	status = "disabled";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_i2c3 {
+	/* Die 2 Die Serdes */
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* ADC */
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c10_default>;
+	status = "disabled";
+
+	adc@48 {
+		compatible = "ti,ads1115";
+		reg = <0x48>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@4 {
+			reg = <4>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@5 {
+			reg = <5>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@6 {
+			reg = <6>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@7 {
+			reg = <7>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+	};
+};
+
+&d0_i2c5 {
+	/* AT24C02C EEPROM */
+	status = "disabled";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* SMBUS */
+	status = "disabled";
+};
+
+&d0_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+
+	pinctrl_gpio12_default: gpio12-default{
+		mux {
+			groups = "gpio12_group";
+			function = "gpio12_func";
+		};
+		conf {
+			groups = "gpio12_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio27_default: gpio27-default{
+		mux {
+			groups = "gpio27_group";
+			function = "gpio27_func";
+		};
+		conf {
+			groups = "gpio27_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio58_default: gpio58-default {
+		mux {
+			groups = "gpio58_group";
+			function = "gpio58_func";
+		};
+		conf {
+			groups = "gpio58_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio59_default: gpio59-default {
+		mux {
+			groups = "gpio59_group";
+			function = "gpio59_func";
+		};
+		conf {
+			groups = "gpio59_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio60_default: gpio60-default {
+		mux {
+			groups = "gpio60_group";
+			function = "gpio60_func";
+		};
+		conf {
+			groups = "gpio60_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio61_default: gpio61-default {
+		mux {
+			groups = "gpio61_group";
+			function = "gpio61_func";
+		};
+		conf {
+			groups = "gpio61_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio12_default
+	&pinctrl_gpio27_default
+	&pinctrl_gpio58_default
+	&pinctrl_gpio59_default
+	&pinctrl_gpio60_default
+	&pinctrl_gpio61_default>;
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/vpu7702-pcie.dts b/arch/riscv/boot/dts/eswin/vpu7702-pcie.dts
new file mode 100644
index 000000000000..4f5cdc089386
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/vpu7702-pcie.dts
@@ -0,0 +1,1181 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x3
+#define MEMORY_SIZE_L		0x80000000
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 VPU A1";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x0>;
+			no-map;
+		};
+
+		rpc_reserved:rpc_reserved@100000000 {
+			reg = <0x1 0x0 0 0x1000>;
+			no-map;
+		};
+
+		commu_reserved:commu_reserved@100001000 {
+			reg = <0x1 0x1000 0 0x10000>;
+			no-map;
+		};
+
+		kernel_debug_reserved:kernel_debug_reserved@100011000 {
+			reg = <0x1 0x11000 0 0x100000>;
+			no-map;
+		};
+
+		pgretire_reserved:pgretire_reserved@100111000 {
+			reg = <0x1 0x111000 0 0x100000>;
+			no-map;
+		};
+
+		configs_reserved:configs_reserved@100211000 {
+			reg = <0x1 0x211000 0 0x1000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring0:vdev0vring0@100212000 {
+			reg = <0x1 0x212000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring1:vdev0vring1@100312000 {
+			reg = <0x1 0x312000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0buffer:vdev0buffer@100412000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x412000 0 0x4000000>; // pre is rx
+			no-map;
+		};
+
+		rsc_table0:rsc_table0@104412000 {
+			reg = <0x1 0x4412000 0 0x1000>;
+			no-map;
+		};
+
+		version_table:version_table@104413000 {
+			reg = <0x1 0x4413000 0 0x1000>;
+			no-map;
+		};
+
+		bd_table:bd_table@104414000 {
+			reg = <0x1 0x4414000 0 0x1000>;
+			no-map;
+		};
+
+		share_memory {
+			compatible = "eswin,shm";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			phys-base = <0x1 0x4415000>;
+			virt-base = <0x1 0x4415000>;
+			shm-size = <0xbbeb000>;
+			reg = <0x1 0x4415000 0 0xbbeb000>;
+		};
+
+	};
+
+	soc {
+		d0_ipcm:ipcm@0 {
+			compatible = "eswin,ipcm", "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			vf_id = <0>;
+			numa-node-id = <0>;
+			memory-region = <&ipcm_vdev0vring0>,<&ipcm_vdev0vring1>,<&ipcm_vdev0buffer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+		  	smmu_dev = <&smmu_cfg_0>;
+		};
+
+		dev_smmu_cfg: smmu_cfg@0 {
+			compatible = "eswin,vpu7702_dev_smmu_cfg";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0xc0000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_B>;
+			iommu-map = <0x000000 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "okay";
+			numa-node-id = <0>;
+			dma-noncoherent;
+                        #include "smmu_cfg.dtsi"
+
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+/ {
+	thermal-zones {
+		thermal0 {
+			trips {
+				threshold: trip-point0 {
+					temperature = <65000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				target: trip-point1 {
+					temperature = <75000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+				crit: trip-point2 {
+					temperature     = <95000>;
+					hysteresis      = <0>;
+					type            = "critical";
+				};
+			};
+
+			cooling-maps {
+				/*
+				map0 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device =
+						<&cpu_0 0 10>,
+						<&cpu_1 0 10>,
+						<&cpu_2 0 10>,
+						<&cpu_3 0 10>;
+				};
+				map1 {
+					trip = <&target>;
+					contribution = <4096>;
+					cooling-device = <&g2d_opp_table THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				map2 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device = <&venc_opp_table THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				map3 {
+					trip = <&target>;
+					contribution = <1024>;
+					cooling-device = <&vdec_opp_table THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+				*/
+			};
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+
+	d0_pmu_vi: win2030-pmu-controller-port@80 {
+		power_status = <0>;
+	};
+	d0_pmu_vo: win2030-pmu-controller-port@c0 {
+		power_status = <0>;
+	};
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	/delete-property/ mboxes;
+	status = "okay";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&d0_npu{
+	status = "disabled";
+	apply_npu_1G_freq;
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "disabled";
+};
+
+&d0_dsp0 {
+	status = "disabled";
+};
+
+&d0_dsp1 {
+	status = "disabled";
+};
+
+&d0_dsp2 {
+	status = "disabled";
+};
+
+&d0_dsp3 {
+	status = "disabled";
+};
+
+&gpu0 {
+	status = "disabled";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "disabled";
+};
+
+&dc {
+	status = "disabled";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "disabled";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "disabled";
+};
+
+&dsi_controller {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "disabled";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+	status = "disabled";
+};
+
+&d0_i2s0 {
+	status = "disabled";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "disabled";
+};
+
+&d0_graphcard2 {
+	status = "disabled";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "disabled";
+};
+
+&isp_1 {
+	status = "disabled";
+};
+
+&dewarp {
+	status = "disabled";
+};
+
+&mipi_dphy_rx {
+	status = "disabled";
+};
+
+&csi_dma0 {
+	status = "disabled";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "disabled";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "disabled";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "disabled";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "disabled";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "disabled";
+};
+
+&d0_sata {
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "disabled";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "disabled";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "disabled";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "disabled";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "disabled";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "disabled";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "disabled";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "disabled";
+};
+
+&d0_i2c0 {
+	/* 12V_DC_IN */
+	status = "disabled";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&d0_i2c1 {
+	/* NPU Power Control */
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* Temperature Sensor */
+	status = "disabled";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_i2c3 {
+	/* Die 2 Die Serdes */
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* SMBUS */
+	status = "disabled";
+};
+
+&d0_i2c5 {
+	/* AT24C02C EEPROM */
+	status = "disabled";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* ADC */
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c10_default>;
+	status = "disabled";
+
+	adc@48 {
+		compatible = "ti,ads1115";
+		reg = <0x48>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@4 {
+			reg = <4>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@5 {
+			reg = <5>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@6 {
+			reg = <6>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@7 {
+			reg = <7>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+	};
+};
+
+&d0_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "disabled";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+
+	pinctrl_gpio12_default: gpio12-default{
+		mux {
+			groups = "gpio12_group";
+			function = "gpio12_func";
+		};
+		conf {
+			groups = "gpio12_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio27_default: gpio27-default{
+		mux {
+			groups = "gpio27_group";
+			function = "gpio27_func";
+		};
+		conf {
+			groups = "gpio27_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio58_default: gpio58-default {
+		mux {
+			groups = "gpio58_group";
+			function = "gpio58_func";
+		};
+		conf {
+			groups = "gpio58_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio59_default: gpio59-default {
+		mux {
+			groups = "gpio59_group";
+			function = "gpio59_func";
+		};
+		conf {
+			groups = "gpio59_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio60_default: gpio60-default {
+		mux {
+			groups = "gpio60_group";
+			function = "gpio60_func";
+		};
+		conf {
+			groups = "gpio60_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio61_default: gpio61-default {
+		mux {
+			groups = "gpio61_group";
+			function = "gpio61_func";
+		};
+		conf {
+			groups = "gpio61_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio12_default
+	&pinctrl_gpio27_default
+	&pinctrl_gpio58_default
+	&pinctrl_gpio59_default
+	&pinctrl_gpio60_default
+	&pinctrl_gpio61_default>;
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/eswin/vpu7702_pcie_factory.dts b/arch/riscv/boot/dts/eswin/vpu7702_pcie_factory.dts
new file mode 100644
index 000000000000..acbcf170b5e6
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/vpu7702_pcie_factory.dts
@@ -0,0 +1,1121 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+
+/* If wanna enable ECC capability of DDR, should reserve highest zone of 1/8 all space for it */
+#define MEMORY_SIZE_H		0x3
+#define MEMORY_SIZE_L		0x80000000
+#define CMA_SIZE		0x20000000
+
+#include "eswin-win2030-die0-soc.dtsi"
+#include "eic7700-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "ESWIN EIC7702 VPU FACTORY A2";
+	compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000",
+		     "sifive,fu740", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+		rtc0 = &die0_rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		opensbi_domain_config: domain-config {
+			compatible = "opensbi,domain,config";
+			system-suspend-test;
+		};
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+		numa-node-id = <0>;
+	};
+
+	memory@59000000 {
+		device_type = "memory";
+		reg = <0x0 0x59000000 0x0 0x400000>;
+		numa-node-id = <0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			alloc-ranges = <0x0 0x80000000 MEMORY_SIZE_H MEMORY_SIZE_L>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		ramoops@fff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0xfff00000 0x0 0x100000>;
+			console-size = <0x40000>;
+			record-size = <0x40000>;
+			ftrace-size = <0x40000>;
+			pmsg-size = <0x40000>;
+			no-map;
+		};
+
+		g2d_8GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x1 0xfffff000 0x0 0x1000>;
+		};
+
+		g2d_12GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x2 0xfffff000 0x0 0x1000>;
+		};
+
+		mmz_nid_0_part_0 {
+			compatible = "eswin-reserve-memory";
+			reg = <0x3 0x0 0x1 0x0>;
+			no-map;
+		};
+
+		rpc_reserved:rpc_reserved@100000000 {
+			reg = <0x1 0x0 0 0x1000>;
+			no-map;
+		};
+
+		commu_reserved:commu_reserved@100001000 {
+			reg = <0x1 0x1000 0 0x10000>;
+			no-map;
+		};
+
+		kernel_debug_reserved:kernel_debug_reserved@100011000 {
+			reg = <0x1 0x11000 0 0x100000>;
+			no-map;
+		};
+
+		pgretire_reserved:pgretire_reserved@100111000 {
+			reg = <0x1 0x111000 0 0x100000>;
+			no-map;
+		};
+
+		configs_reserved:configs_reserved@100211000 {
+			reg = <0x1 0x211000 0 0x1000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring0:vdev0vring0@100212000 {
+			reg = <0x1 0x212000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0vring1:vdev0vring1@100312000 {
+			reg = <0x1 0x312000 0 0x100000>;
+			no-map;
+		};
+
+		ipcm_vdev0buffer:vdev0buffer@100412000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x412000 0 0x4000000>; // pre is rx
+			no-map;
+		};
+
+		rsc_table0:rsc_table0@104412000 {
+			reg = <0x1 0x4412000 0 0x1000>;
+			no-map;
+		};
+
+		version_table:version_table@104413000 {
+			reg = <0x1 0x4413000 0 0x1000>;
+			no-map;
+		};
+
+		bd_table:bd_table@104414000 {
+			reg = <0x1 0x4414000 0 0x1000>;
+			no-map;
+		};
+
+		share_memory {
+			compatible = "eswin,shm";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			phys-base = <0x1 0x4415000>;
+			virt-base = <0x1 0x4415000>;
+			shm-size = <0xbbeb000>;
+			reg = <0x1 0x4415000 0 0xbbeb000>;
+		};
+
+	};
+
+	soc {
+		d0_ipcm:ipcm@0 {
+			compatible = "eswin,ipcm", "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			vf_id = <0>;
+			numa-node-id = <0>;
+			memory-region = <&ipcm_vdev0vring0>,<&ipcm_vdev0vring1>,<&ipcm_vdev0buffer>;
+			interrupt-parent = <&plic0>;
+			interrupts = <119>;
+		  	smmu_dev = <&smmu_cfg_0>;
+		};
+
+		dev_smmu_cfg: smmu_cfg@0 {
+			compatible = "eswin,vpu7702_dev_smmu_cfg";
+			#size-cells = <2>;
+			dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0xc0000000>;
+			iommus = <&smmu0 WIN2030_SID_DEV_FOO_B>;
+			iommu-map = <0x000000 &smmu0 0xff0000 0xffffff>;
+			tbus = <WIN2030_TBUID_PCIE>;
+			status = "okay";
+			numa-node-id = <0>;
+			dma-noncoherent;
+                        #include "smmu_cfg.dtsi"
+
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio107_default &pinctrl_gpio108_default &pinctrl_gpio109_default>;
+
+		gpio-107 {
+			gpios = <&portd 11 GPIO_ACTIVE_HIGH>;
+			label = "power";
+			linux,default-trigger = "default-on";
+		};
+
+		gpio-108 {
+			gpios = <&portd 12 GPIO_ACTIVE_LOW>;
+			label = "heartbeat";
+			linux,default-trigger = "heartbeat";
+		};
+
+		gpio-109 {
+			gpios = <&portd 13 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+
+		gpio-110 {
+			gpios = <&portd 14 GPIO_ACTIVE_HIGH>;
+			label = "gpio-109";
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		ok_key {
+			label = "OK";
+			linux,code = <KEY_OK>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_gpio6_default>;
+			gpios = <&porta 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&d0_pmu {
+	status = "okay";
+};
+
+&ddr0 {
+	status = "okay";
+};
+
+&ddr1 {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&smmu_pmu0 {
+	status = "disabled";
+};
+
+&d0_cfg_noc {
+	status = "okay";
+};
+
+&d0_llc_noc {
+	status = "okay";
+	stat,0 = "TracePort:ddr0_p0_req";
+	stat,1 = "TracePort:ddr1_p0_req";
+	//latency,0 = "TracePort:llcnoc_trans_probe";
+	//pending,0 = "TracePort:llcnoc_trans_probe";
+};
+
+&d0_sys_noc {
+	status = "okay";
+
+	//eswin,DSPT-qos-owner;
+	//eswin,NPU-qos-owner;
+	//eswin,SPISLV_TBU3-qos-owner;
+
+#if 0
+	stat,0 = "TracePort:ddr0_p1_req",
+		 "InitFlow:mcput_snoc_mp/I/0";
+
+	stat,1 = "TracePort:ddr0_p2_req",
+		 "InitFlow:dspt_snoc/I/0",
+		 "AddrBase:0x81000000", "AddrSize:0x30",
+		 "Opcode:RdWrLockUrg", "Status:ReqRsp", "Length:0x8000", "Urgency:0x0";
+
+	stat,2 = "TracePort:ddr1_p1_req",
+		 "Status:Req", "AddrSize:0x28";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#else
+	stat,0 = "TracePort:ddr0_p1_req";
+
+	stat,1 = "TracePort:ddr0_p2_req";
+
+	stat,2 = "TracePort:ddr1_p1_req";
+
+	stat,3 = "TracePort:ddr1_p2_req";
+#endif
+
+	latency,0 = "TracePort:sysnoc_trans_probe_0", "AddrSize:0x0";
+	latency,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x28","Opcode:RdWr";
+	//latency,2 = "TracePort:sysnoc_trans_probe_2";
+
+	//pending,0 = "TracePort:sysnoc_trans_probe_0";
+	//pending,1 = "TracePort:sysnoc_trans_probe_1","Mode:latency","AddrBase:0x82000000","AddrSize:0x0","Opcode:RdWr";
+	pending,0 = "TracePort:sysnoc_trans_probe_2", "AddrSize:0x3";
+};
+
+&d0_media_noc {
+	status = "okay";
+
+	//eswin,GPU-qos-owner;
+	//eswin,TBU2-qos-owner;
+	//eswin,VC-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p3_req";
+	stat,1 = "TracePort:ddr1_p3_req";
+	//latency,0 = "TracePort:mnoc_trans_probe";
+	//pending,0 = "TracePort:mnoc_trans_probe";
+};
+
+&d0_realtime_noc {
+	status = "okay";
+
+	//eswin,TBU0-qos-owner;
+	//eswin,VO-qos-owner;
+
+	stat,0 = "TracePort:ddr0_p4_req";
+	stat,1 = "TracePort:ddr1_p4_req";
+	//latency,0 = "TracePort:rnoc_trans_probe";
+	//pending,0 = "TracePort:rnoc_trans_probe";
+};
+
+&d0_noc_wdt {
+	status = "okay";
+};
+
+&d0_ipc_scpu {
+	status = "okay";
+};
+
+&d0_lpcpu {
+	/delete-property/ mboxes;
+	status = "okay";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&d0_npu{
+	status = "okay";
+	apply_npu_1G_freq;
+	npu-supply=<&npu_vcc1>;
+};
+
+&d0_dsp_subsys {
+	status = "okay";
+};
+
+&d0_dsp0 {
+	status = "okay";
+};
+
+&d0_dsp1 {
+	status = "okay";
+};
+
+&d0_dsp2 {
+	status = "okay";
+};
+
+&d0_dsp3 {
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&gc820 {
+	status = "okay";
+};
+
+&vdec0 {
+	status = "okay";
+};
+
+&venc0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+	dc_out: port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dc_out_dsi: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&dsi_input>;
+		};
+		dc_out_dpi1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&vd_input>;
+		};
+		dc_out_hdmi: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&hdmi_in_dc>;
+		};
+	};
+};
+
+&dc_test {
+	status = "disabled";
+};
+
+&virtual_display {
+	status = "okay";
+	port {
+		vd_input: endpoint {
+			remote-endpoint = <&dc_out_dpi1>;
+		};
+	};
+};
+
+&dsi_output {
+	status = "okay";
+};
+
+&dsi_controller {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			dsi_input: endpoint {
+				remote-endpoint = <&dc_out_dsi>;
+			};
+		};
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			mipi_dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	dsi_panel:dsi_panel@0 {
+		compatible = "eswin,generic-panel";
+		reg = <0>;
+		dsi,format = <0>; //RGB888:0, RGB666:1, RGB565:3
+		dsi,lanes = <4>;
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5_default &pinctrl_gpio111_default>;
+	backlight0-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			hdmi_in_dc: endpoint@0 {
+				remote-endpoint = <&dc_out_hdmi>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&dw_hdmi_hdcp2 {
+    status = "okay";
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "disabled";
+};
+
+&d0_i2s2 {
+	status = "disabled";
+};
+
+&d0_graphcard0 {
+	status = "disabled";
+};
+
+&d0_graphcard1 {
+	status = "disabled";
+};
+
+&d0_graphcard2 {
+	status = "okay";
+	label = "HDMI Audio";
+	dais = <&d0_i2s0_port>;
+};
+
+&isp_0 {
+	status = "okay";
+};
+
+&isp_1 {
+	status = "okay";
+};
+
+&dewarp {
+	status = "okay";
+};
+
+&mipi_dphy_rx {
+	status = "okay";
+};
+
+&csi_dma0 {
+	status = "okay";
+};
+
+&csi_dma1 {
+	status = "disabled";
+};
+
+&csi2_0 {
+	status = "okay";
+};
+
+&csi2_1 {
+	status = "disabled";
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+	no-sdio;
+	no-sd;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x55>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	aw3155:wifi_aw3155@0 {
+		compatible = "aml_w1_sdio";
+		reg = <0x0>;
+		interrupt-parent = <&porta>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default", "default";
+		pinctrl-0 = <&pinctrl_gpio15_default>;
+		pinctrl-1 = <&pinctrl_gpio34_default>;
+		irq-gpios = <&porta 15 GPIO_ACTIVE_HIGH>;
+		rst-gpios = <&portb 2 GPIO_ACTIVE_HIGH>;
+
+	};
+};
+
+&d0_gmac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio94_default>;
+	rst-gpios = <&portc 30 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x23232323 0x800c8023 0x0c0c0c0c>;
+	dly-param-100m = <0x50505050 0x803f8050 0x3f3f3f3f>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_gmac1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio16_default>;
+	rst-gpios = <&porta 16 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+	dly-param-1000m = <0x25252525 0x80268025 0x26262626>;
+	dly-param-100m = <0x48484848 0x80588048 0x58585858>;
+	dly-param-10m = <0 0 0>;
+	status = "okay";
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* BT M.2 KEY-E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* RS232 DB9 */
+	status = "okay";
+};
+
+&d0_uart3 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_uart4 {
+	/* unused */
+	status = "disabled";
+};
+
+&ssi0 {
+	/* spi flash */
+	status = "okay";
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&ssi1 {
+	/* spi flash */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2_default>;
+	num-cs = <2>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+	spi-flash@1 {
+		compatible = "winbond,w25q128fw",
+					"jedec,spi-nor";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&portd 4 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&d0_mbox0 {
+	status = "okay";
+};
+
+&d0_mbox1 {
+	status = "disabled";
+};
+
+&d0_mbox2 {
+	status = "okay";
+};
+
+&d0_mbox3 {
+	status = "okay";
+};
+
+&d0_mbox4 {
+	status = "okay";
+};
+
+&d0_mbox5 {
+	status = "okay";
+};
+
+&d0_mbox6 {
+	status = "okay";
+};
+
+&d0_mbox7 {
+	status = "okay";
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* 12V_DC_IN */
+	status = "disabled";
+	i2c-sda-hold-time-ns = <0x40>;
+	ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "soc_vdd";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&d0_i2c1 {
+	/* NPU Power Control */
+	status = "okay";
+	tps549d22@10 {
+		compatible = "ti,tps549d22";
+		reg = <0x10>;
+		eswin,regulator_default-microvolt=<1000000>;
+		label = "npu_vdd";
+		regulators{
+			npu_vcc1:npu_svcc{
+				regulator-name="NPU_SVCC";
+				regulator-min-microvolt=<700000>;
+				regulator-max-microvolt=<1120000>;
+				regulator-min-microamp=<20000000>;
+				regulator-max-microamp=<40000000>;
+				regulator-ov-protection-microvolt=<1100000>;
+			};
+		};
+	};
+};
+
+&d0_i2c2 {
+	/* Temperature Sensor */
+	status = "disabled";
+	tmp102@48 {
+		compatible = "ti,tmp102";
+		reg = <0x48>;
+		label = "d0_board_tmp";
+		#thermal-sensor-cells = <1>;
+	};
+};
+
+&d0_i2c3 {
+	/* Die 2 Die Serdes */
+	status = "disabled";
+};
+
+&d0_i2c4 {
+	/* ADC */
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c10_default>;
+	status = "disabled";
+
+	adc@48 {
+		compatible = "ti,ads1115";
+		reg = <0x48>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@4 {
+			reg = <4>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@5 {
+			reg = <5>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@6 {
+			reg = <6>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+
+		channel@7 {
+			reg = <7>;
+			ti,gain = <0>;
+			ti,datarate = <7>;
+		};
+	};
+};
+
+&d0_i2c5 {
+	/* AT24C02C EEPROM */
+	status = "disabled";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_i2c6 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c7 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c8 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_i2c9 {
+	/* unused */
+	status = "disabled";
+};
+
+&d0_aon_i2c0 {
+	/* SMBUS */
+	status = "disabled";
+};
+
+&d0_aon_i2c1 {
+	status = "disabled";
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "disabled";
+};
+
+&wdt1 {
+	status = "disabled";
+};
+
+&wdt2 {
+	status = "disabled";
+};
+
+&wdt3 {
+	status = "disabled";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&timer0 {
+	status = "okay";
+};
+
+&timer1 {
+	status = "okay";
+};
+
+&timer2 {
+	status = "okay";
+};
+
+&timer3 {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+
+	pinctrl_gpio12_default: gpio12-default{
+		mux {
+			groups = "gpio12_group";
+			function = "gpio12_func";
+		};
+		conf {
+			groups = "gpio12_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio27_default: gpio27-default{
+		mux {
+			groups = "gpio27_group";
+			function = "gpio27_func";
+		};
+		conf {
+			groups = "gpio27_group";
+			bias-pull-down = <1>;
+		};
+	};
+
+	pinctrl_gpio58_default: gpio58-default {
+		mux {
+			groups = "gpio58_group";
+			function = "gpio58_func";
+		};
+		conf {
+			groups = "gpio58_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio59_default: gpio59-default {
+		mux {
+			groups = "gpio59_group";
+			function = "gpio59_func";
+		};
+		conf {
+			groups = "gpio59_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio60_default: gpio60-default {
+		mux {
+			groups = "gpio60_group";
+			function = "gpio60_func";
+		};
+		conf {
+			groups = "gpio60_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+
+	pinctrl_gpio61_default: gpio61-default {
+		mux {
+			groups = "gpio61_group";
+			function = "gpio61_func";
+		};
+		conf {
+			groups = "gpio61_group";
+			input-enable = <1>;
+			bias-pull-down = <0>;
+		};
+	};
+};
+
+/* GPIO Function Description
+
+	gpio0   : pcie prstn(I)
+	gpio5   : led back light power on/off(O)
+	gpio6   : system key(I)
+	gpio10  : dsi touch interrupt(I)
+	gpio11  : head phone plug/unplug detection2(I)
+	gpio12  : tf card power on/off(O)
+	gpio14  : bt wake host(I)
+	gpio15  : wlan wake host(I)
+	gpio16  : gphy1 resetn(O)
+	gpio28  : head phone plug/unplug detection1(I)
+	gpio34  : m.2 power on/off(O)
+	gpio94  : gphy0 resetn(O)
+	gpio106 : touch reset(O)
+	gpio107 : system led0(O)
+	gpio108 : system led1(O)
+	gpio109 : system led2(O)
+	gpio110 : system led3(O)
+	gpio111 : mipi dsi resetn(O)
+*/
+
+&gpio0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio12_default
+	&pinctrl_gpio27_default
+	&pinctrl_gpio58_default
+	&pinctrl_gpio59_default
+	&pinctrl_gpio60_default
+	&pinctrl_gpio61_default>;
+};
+
+&dev_llc_d0{
+	/* apply_npu_1G_freq; */
+	npu-supply=<&npu_vcc1>;
+	status = "okay";
+};
diff --git a/arch/riscv/configs/eic7700_dbg_defconfig b/arch/riscv/configs/eic7700_dbg_defconfig
new file mode 100644
index 000000000000..fdbad29cf77e
--- /dev/null
+++ b/arch/riscv/configs/eic7700_dbg_defconfig
@@ -0,0 +1,914 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_KEXEC=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_HOTPLUG_CPU=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false firmware_class.path=/lib/firmware/eic7x/ kgdboc=ttyS0,115200 ftrace_dump_on_oops=1"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_JUMP_LABEL=y
+# CONFIG_GCC_PLUGINS is not set
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_INET_DIAG=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=y
+CONFIG_BT_RFCOMM=y
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HS=y
+CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_MCTP=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+CONFIG_REALTEK_PHY=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_GOODIX=y
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_8250_NR_UARTS=5
+CONFIG_SERIAL_8250_RUNTIME_UARTS=5
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_REGULATOR_ES5340=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_PWC=m
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_XILINX=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_DRM_TOSHIBA_TC358762=y
+CONFIG_DRM_DW_MIPI_DSI=y
+CONFIG_ESWIN_MIPI_DSI=y
+CONFIG_ESWIN_RASPBERRYPI_7INCH_PANEL=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_VGA_CONSOLE=y
+CONFIG_DUMMY_CONSOLE=y
+CONFIG_DUMMY_CONSOLE_COLUMNS=80
+CONFIG_DUMMY_CONSOLE_ROWS=25
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
+CONFIG_LOGO=y
+CONFIG_LOGO_LINUX_MONO=y
+CONFIG_LOGO_LINUX_VGA16=y
+CONFIG_LOGO_LINUX_CLUT224=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_ESWIN_SND_TLV320AIC3106_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_UHID=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF85063=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_RSTKDUMP=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=15
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_KGDB=y
+CONFIG_KGDB_KDB=y
+CONFIG_KDB_KEYBOARD=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_DEBUG_KMEMLEAK=y
+CONFIG_DEBUG_OBJECTS=y
+CONFIG_DEBUG_OBJECTS_SELFTEST=y
+CONFIG_DEBUG_OBJECTS_FREE=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_KASAN=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+CONFIG_RCU_CPU_STALL_CPUTIME=y
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_VIDEO_ESWIN_VI_TOP=y
+CONFIG_VIDEO_ESWIN_DVP2AXI_DEPENDENCES=y
+CONFIG_VIDEO_ESWIN_DVP2AXI=y
+CONFIG_DWC_MIPI_CSI2_HOST=y
+CONFIG_PHY_ESWIN_CSI2_DPHY=y
+CONFIG_VIDEO_IMX219=y
+CONFIG_I2C_MUX=y
+CONFIG_I2C_MUX_PCA954x=y
+CONFIG_VIDEO_IMX327=y
+CONFIG_VIDEO_ESWIN_ISP=m
+CONFIG_EIC7700_EVB_VI=n
+CONFIG_BCMDHD=m
+CONFIG_BCMDHD_SDIO_IRQ=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_BCMDHD_FW_PATH="/lib/firmware/fw_bcm43456c5_ag.bin"
+CONFIG_BCMDHD_NVRAM_PATH="/lib/firmware/nvram_ap6256.txt"
+CONFIG_BCMDHD_CLM_PATH="/lib/firmware/clm_bcm43752a2_ag.blob"
diff --git a/arch/riscv/configs/eic7700_defconfig b/arch/riscv/configs/eic7700_defconfig
new file mode 100644
index 000000000000..81721d25cce9
--- /dev/null
+++ b/arch/riscv/configs/eic7700_defconfig
@@ -0,0 +1,910 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_KEXEC=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_HOTPLUG_CPU=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false firmware_class.path=/lib/firmware/eic7x/"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_JUMP_LABEL=y
+# CONFIG_GCC_PLUGINS is not set
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_INET_DIAG=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=y
+CONFIG_BT_RFCOMM=y
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HS=y
+CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_MCTP=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+CONFIG_REALTEK_PHY=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_GOODIX=y
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_8250_NR_UARTS=5
+CONFIG_SERIAL_8250_RUNTIME_UARTS=5
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_REGULATOR_ES5340=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_PWC=m
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_XILINX=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_DRM_TOSHIBA_TC358762=y
+CONFIG_DRM_DW_MIPI_DSI=y
+CONFIG_ESWIN_MIPI_DSI=y
+CONFIG_ESWIN_RASPBERRYPI_7INCH_PANEL=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_VGA_CONSOLE=y
+CONFIG_DUMMY_CONSOLE=y
+CONFIG_DUMMY_CONSOLE_COLUMNS=80
+CONFIG_DUMMY_CONSOLE_ROWS=25
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
+CONFIG_LOGO=y
+CONFIG_LOGO_LINUX_MONO=y
+CONFIG_LOGO_LINUX_VGA16=y
+CONFIG_LOGO_LINUX_CLUT224=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_ESWIN_SND_TLV320AIC3106_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_UHID=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF85063=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=7
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_FS=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_VIDEO_ESWIN_VI_TOP=y
+CONFIG_VIDEO_ESWIN_DVP2AXI_DEPENDENCES=y
+CONFIG_VIDEO_ESWIN_DVP2AXI=y
+CONFIG_DWC_MIPI_CSI2_HOST=y
+CONFIG_PHY_ESWIN_CSI2_DPHY=y
+CONFIG_VIDEO_IMX219=y
+CONFIG_I2C_MUX=y
+CONFIG_I2C_MUX_PCA954x=y
+CONFIG_VIDEO_IMX327=y
+CONFIG_VIDEO_ESWIN_ISP=m
+CONFIG_EIC7700_EVB_VI=n
+CONFIG_BCMDHD=m
+CONFIG_BCMDHD_SDIO_IRQ=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_BCMDHD_FW_PATH="/lib/firmware/fw_bcm43456c5_ag.bin"
+CONFIG_BCMDHD_NVRAM_PATH="/lib/firmware/nvram_ap6256.txt"
+CONFIG_BCMDHD_CLM_PATH="/lib/firmware/clm_bcm43752a2_ag.blob"
diff --git a/arch/riscv/configs/eic7702_dbg_defconfig b/arch/riscv/configs/eic7702_dbg_defconfig
new file mode 100644
index 000000000000..f8c0ab4d33ac
--- /dev/null
+++ b/arch/riscv/configs/eic7702_dbg_defconfig
@@ -0,0 +1,890 @@
+CONFIG_LOCALVERSION="-eic7702"
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_NUMA_BALANCING=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_KEXEC=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_SCHED_CLUSTER=y
+CONFIG_NR_CPUS=8
+CONFIG_HOTPLUG_CPU=y
+CONFIG_NUMA=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x20000000 disable_bypass=false firmware_class.path=/lib/firmware/eic7x/ kgdboc=ttyS0,115200 ftrace_dump_on_oops=1"
+CONFIG_CMDLINE_EXTEND=y
+# CONFIG_SUSPEND is not set
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_JUMP_LABEL=y
+# CONFIG_GCC_PLUGINS is not set
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=y
+CONFIG_BT_RFCOMM=y
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HS=y
+CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_MCTP=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+CONFIG_REALTEK_PHY=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_GOODIX=y
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_8250_NR_UARTS=10
+CONFIG_SERIAL_8250_RUNTIME_UARTS=10
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_MUX_PCA954x=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_REGULATOR_ES5340=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_PWC=m
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_XILINX=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_VGA_CONSOLE=y
+CONFIG_DUMMY_CONSOLE=y
+CONFIG_DUMMY_CONSOLE_COLUMNS=80
+CONFIG_DUMMY_CONSOLE_ROWS=25
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
+CONFIG_LOGO=y
+CONFIG_LOGO_LINUX_MONO=y
+CONFIG_LOGO_LINUX_VGA16=y
+CONFIG_LOGO_LINUX_CLUT224=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_UHID=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ARCH_ESWIN_EIC7702_SOC=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_EIC7X_D2D=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_RSTKDUMP=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_DMA_NUMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=512
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=7
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_KGDB=y
+CONFIG_KGDB_KDB=y
+CONFIG_KDB_KEYBOARD=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_SCHEDSTATS=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+CONFIG_RCU_CPU_STALL_CPUTIME=y
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_BCMDHD=m
+CONFIG_BCMDHD_SDIO_IRQ=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_BCMDHD_FW_PATH="/lib/firmware/fw_bcm43456c5_ag.bin"
+CONFIG_BCMDHD_NVRAM_PATH="/lib/firmware/nvram_ap6256.txt"
+CONFIG_BCMDHD_CLM_PATH="/lib/firmware/clm_bcm43752a2_ag.blob"
+CONFIG_VIDEO_ESWIN_VI_TOP=y
+CONFIG_VIDEO_ESWIN_DVP2AXI=y
diff --git a/arch/riscv/configs/eic7702_defconfig b/arch/riscv/configs/eic7702_defconfig
new file mode 100644
index 000000000000..32b36f7a820b
--- /dev/null
+++ b/arch/riscv/configs/eic7702_defconfig
@@ -0,0 +1,903 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_NUMA_BALANCING=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_KEXEC=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_SCHED_CLUSTER=y
+CONFIG_NR_CPUS=8
+CONFIG_HOTPLUG_CPU=y
+CONFIG_NUMA=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x20000000 disable_bypass=false firmware_class.path=/lib/firmware/eic7x/"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=y
+CONFIG_BT_RFCOMM=y
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HS=y
+CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_MCTP=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+CONFIG_REALTEK_PHY=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_GOODIX=y
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_8250_NR_UARTS=10
+CONFIG_SERIAL_8250_RUNTIME_UARTS=10
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_MUX_PCA954x=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_ESWIN_DUMMY_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_REGULATOR_ES5340=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_PWC=m
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_XILINX=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_VGA_CONSOLE=y
+CONFIG_DUMMY_CONSOLE=y
+CONFIG_DUMMY_CONSOLE_COLUMNS=80
+CONFIG_DUMMY_CONSOLE_ROWS=25
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
+CONFIG_LOGO=y
+CONFIG_LOGO_LINUX_MONO=y
+CONFIG_LOGO_LINUX_VGA16=y
+CONFIG_LOGO_LINUX_CLUT224=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_UHID=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ARCH_ESWIN_EIC7702_SOC=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_EIC7X_D2D=y
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_DMA_NUMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=512
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=7
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_SCHEDSTATS=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_BCMDHD=m
+CONFIG_BCMDHD_SDIO_IRQ=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_BCMDHD_FW_PATH="/lib/firmware/fw_bcm43456c5_ag.bin"
+CONFIG_BCMDHD_NVRAM_PATH="/lib/firmware/nvram_ap6256.txt"
+CONFIG_BCMDHD_CLM_PATH="/lib/firmware/clm_bcm43752a2_ag.blob"
+CONFIG_VIDEO_ESWIN_VI_TOP=y
+CONFIG_VIDEO_ESWIN_DVP2AXI=y
diff --git a/arch/riscv/configs/eswin_aicard_defconfig b/arch/riscv/configs/eswin_aicard_defconfig
new file mode 100644
index 000000000000..920d04bd4f03
--- /dev/null
+++ b/arch/riscv/configs/eswin_aicard_defconfig
@@ -0,0 +1,857 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_AUDIT=m
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=tty1 console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false no_console_suspend"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_CPU_IDLE=y
+CONFIG_RISCV_SBI_CPUIDLE=y
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_OBJREF=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_CLUSTERIP=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=m
+CONFIG_MCTP=y
+# CONFIG_WIRELESS is not set
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_USB_PWC=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+CONFIG_VIDEO_XILINX=y
+# CONFIG_DRM=y
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_AMD_DC_HDCP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_AUTOFS4_FS=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_CIFS=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_AES=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_PRINTK_TO_SHMEM=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=15
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_ESWIN_AICARD=y
diff --git a/arch/riscv/configs/u2_evb_defconfig b/arch/riscv/configs/u2_evb_defconfig
new file mode 100644
index 000000000000..63c898d93c77
--- /dev/null
+++ b/arch/riscv/configs/u2_evb_defconfig
@@ -0,0 +1,906 @@
+CONFIG_LOCALVERSION="-eic7702"
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_NUMA_BALANCING=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_KEXEC=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_SCHED_CLUSTER=y
+CONFIG_NR_CPUS=8
+CONFIG_HOTPLUG_CPU=y
+CONFIG_NUMA=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x20000000 disable_bypass=false firmware_class.path=/lib/firmware/eic7x/"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_CPU_IDLE=y
+CONFIG_RISCV_SBI_CPUIDLE=y
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=y
+CONFIG_BT_RFCOMM=y
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HS=y
+CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_MCTP=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_ESWIN_PCIE_VPU=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+CONFIG_REALTEK_PHY=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_8250_NR_UARTS=10
+CONFIG_SERIAL_8250_RUNTIME_UARTS=10
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_MUX_PCA954x=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_REGULATOR_ES5340=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_PWC=m
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_XILINX=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_VGA_CONSOLE=y
+CONFIG_DUMMY_CONSOLE=y
+CONFIG_DUMMY_CONSOLE_COLUMNS=80
+CONFIG_DUMMY_CONSOLE_ROWS=25
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
+CONFIG_LOGO=y
+CONFIG_LOGO_LINUX_MONO=y
+CONFIG_LOGO_LINUX_VGA16=y
+CONFIG_LOGO_LINUX_CLUT224=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_UHID=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ARCH_ESWIN_EIC7702_SOC=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_EIC7X_D2D=y
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_DMA_NUMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=512
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=7
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_SCHEDSTATS=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_BCMDHD=m
+CONFIG_BCMDHD_SDIO_IRQ=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_BCMDHD_FW_PATH="/lib/firmware/fw_bcm43456c5_ag.bin"
+CONFIG_BCMDHD_NVRAM_PATH="/lib/firmware/nvram_ap6256.txt"
+CONFIG_BCMDHD_CLM_PATH="/lib/firmware/clm_bcm43752a2_ag.blob"
+CONFIG_VIDEO_ESWIN_VI_TOP=y
+CONFIG_VIDEO_ESWIN_DVP2AXI=y
diff --git a/arch/riscv/configs/vpu7702_evb_defconfig b/arch/riscv/configs/vpu7702_evb_defconfig
new file mode 100644
index 000000000000..06307be90b44
--- /dev/null
+++ b/arch/riscv/configs/vpu7702_evb_defconfig
@@ -0,0 +1,876 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_AUDIT=m
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=tty1 console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false no_console_suspend"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_CPU_IDLE=y
+CONFIG_RISCV_SBI_CPUIDLE=y
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_OBJREF=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_CLUSTERIP=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=m
+CONFIG_MCTP=y
+# CONFIG_WIRELESS is not set
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_ESWIN_PCIE_VPU=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_USB_PWC=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+CONFIG_VIDEO_XILINX=y
+# CONFIG_DRM=y
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_AMD_DC_HDCP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_AUTOFS4_FS=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_CIFS=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_AES=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=15
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
diff --git a/arch/riscv/configs/vpu7702_pcie_a2_defconfig b/arch/riscv/configs/vpu7702_pcie_a2_defconfig
new file mode 100644
index 000000000000..781a13eb6f3d
--- /dev/null
+++ b/arch/riscv/configs/vpu7702_pcie_a2_defconfig
@@ -0,0 +1,880 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_AUDIT=m
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=tty1 console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false no_console_suspend"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_CPU_IDLE=y
+CONFIG_RISCV_SBI_CPUIDLE=y
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_OBJREF=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_CLUSTERIP=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=m
+CONFIG_MCTP=y
+# CONFIG_WIRELESS is not set
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_USB_PWC=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+CONFIG_VIDEO_XILINX=y
+# CONFIG_DRM=y
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_AMD_DC_HDCP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_ESWIN_PCIE_VPU=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_AUTOFS4_FS=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_CIFS=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_AES=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=15
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_IIO=y
+CONFIG_TI_ADS1015=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
diff --git a/arch/riscv/configs/vpu7702_pcie_defconfig b/arch/riscv/configs/vpu7702_pcie_defconfig
new file mode 100644
index 000000000000..e5b63d83ce1a
--- /dev/null
+++ b/arch/riscv/configs/vpu7702_pcie_defconfig
@@ -0,0 +1,880 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_AUDIT=m
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=tty1 console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false no_console_suspend"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_SUSPEND=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_CPU_IDLE=y
+CONFIG_RISCV_SBI_CPUIDLE=y
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BINFMT_MISC=m
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_OBJREF=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_L2TP=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_CLUSTERIP=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=m
+CONFIG_MCTP=y
+# CONFIG_WIRELESS is not set
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8192SE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_USB_PWC=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+CONFIG_VIDEO_XILINX=y
+# CONFIG_DRM=y
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_NOUVEAU=y
+CONFIG_DRM_AMD_DC_HDCP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_PM=y
+CONFIG_PM_SLEEP=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_ESWIN_PCIE_VPU=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_AUTOFS4_FS=y
+CONFIG_FANOTIFY=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_CIFS=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+CONFIG_CRYPTO_ECB=m
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_HMAC=m
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_SHA512=m
+CONFIG_CRYPTO_AES=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=15
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_REGULATOR_TPS549D22=y
+CONFIG_IIO=y
+CONFIG_TI_ADS1015=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
diff --git a/arch/riscv/configs/win2030_defconfig b/arch/riscv/configs/win2030_defconfig
new file mode 100644
index 000000000000..a3d3b8f07f15
--- /dev/null
+++ b/arch/riscv/configs/win2030_defconfig
@@ -0,0 +1,1076 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_TASKSTATS=y
+CONFIG_TASK_DELAY_ACCT=y
+CONFIG_TASK_XACCT=y
+CONFIG_TASK_IO_ACCOUNTING=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_PERF_EVENTS=y
+CONFIG_KEXEC=y
+CONFIG_SOC_ESWIN=y
+CONFIG_SOC_VIRT=y
+CONFIG_SMP=y
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_RISCV_BOOT_SPINWAIT is not set
+CONFIG_CMDLINE="earlycon=sbi console=ttyS0,115200n8 clk_ignore_unused cma_pernuma=0x2000000 disable_bypass=false"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BLK_DEV_ZONED=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BLK_WBT=y
+CONFIG_BLK_CGROUP_IOCOST=y
+CONFIG_BLK_SED_OPAL=y
+CONFIG_BINFMT_MISC=m
+CONFIG_ZSWAP=y
+CONFIG_ZSWAP_ZPOOL_DEFAULT_ZSMALLOC=y
+CONFIG_ZBUD=y
+CONFIG_Z3FOLD_DEPRECATED=m
+CONFIG_SLAB_FREELIST_RANDOM=y
+CONFIG_SLAB_FREELIST_HARDENED=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_TLS_TOE=y
+CONFIG_XFRM_USER=y
+CONFIG_NET_KEY=m
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPVTI=m
+CONFIG_INET_AH=m
+CONFIG_INET_ESP=m
+CONFIG_INET_IPCOMP=m
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_IPV6=m
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_RPL_LWTUNNEL=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_NFCT=y
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_IP_DCCP=m
+CONFIG_SCTP_DEFAULT_COOKIE_HMAC_SHA1=y
+CONFIG_SCTP_COOKIE_HMAC_MD5=y
+CONFIG_RDS=m
+CONFIG_RDS_RDMA=m
+CONFIG_RDS_TCP=m
+CONFIG_TIPC=m
+CONFIG_ATM=m
+CONFIG_ATM_CLIP=m
+CONFIG_ATM_LANE=m
+CONFIG_ATM_MPOA=m
+CONFIG_ATM_BR2684=m
+CONFIG_L2TP=m
+CONFIG_L2TP_DEBUGFS=m
+CONFIG_L2TP_V3=y
+CONFIG_L2TP_IP=m
+CONFIG_L2TP_ETH=m
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_VLAN_8021Q_MVRP=y
+CONFIG_LLC2=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NETLINK_DIAG=m
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BT=y
+CONFIG_BT_RFCOMM=y
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_MCTP=y
+CONFIG_CFG80211=y
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCIEASPM_PERFORMANCE=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_COUNT=2
+CONFIG_BLK_DEV_RAM_SIZE=32768
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_ENCLOSURE_SERVICES=m
+CONFIG_EEPROM_AT24=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=m
+CONFIG_CHR_DEV_ST=m
+CONFIG_BLK_DEV_SR=m
+CONFIG_CHR_DEV_SG=m
+# CONFIG_BLK_DEV_BSG is not set
+CONFIG_CHR_DEV_SCH=m
+CONFIG_SCSI_ENCLOSURE=m
+CONFIG_SCSI_CONSTANTS=y
+CONFIG_SCSI_LOGGING=y
+CONFIG_SCSI_SCAN_ASYNC=y
+CONFIG_SCSI_FC_ATTRS=m
+CONFIG_SCSI_SAS_ATA=y
+CONFIG_ISCSI_TCP=m
+CONFIG_SCSI_CXGB3_ISCSI=m
+CONFIG_SCSI_CXGB4_ISCSI=m
+CONFIG_SCSI_BNX2_ISCSI=m
+CONFIG_SCSI_BNX2X_FCOE=m
+CONFIG_BE2ISCSI=m
+CONFIG_BLK_DEV_3W_XXXX_RAID=m
+CONFIG_SCSI_HPSA=m
+CONFIG_SCSI_3W_9XXX=m
+CONFIG_SCSI_3W_SAS=m
+CONFIG_SCSI_ACARD=m
+CONFIG_SCSI_AACRAID=m
+CONFIG_SCSI_AIC7XXX=m
+CONFIG_AIC7XXX_RESET_DELAY_MS=15000
+CONFIG_SCSI_AIC79XX=m
+CONFIG_AIC79XX_RESET_DELAY_MS=15000
+CONFIG_SCSI_AIC94XX=m
+# CONFIG_AIC94XX_DEBUG is not set
+CONFIG_SCSI_MVSAS=m
+# CONFIG_SCSI_MVSAS_DEBUG is not set
+CONFIG_SCSI_MVUMI=m
+CONFIG_SCSI_ADVANSYS=m
+CONFIG_SCSI_ARCMSR=m
+CONFIG_SCSI_ESAS2R=m
+CONFIG_MEGARAID_NEWGEN=y
+CONFIG_MEGARAID_LEGACY=m
+CONFIG_MEGARAID_SAS=m
+CONFIG_SCSI_MPT2SAS=m
+CONFIG_SCSI_MPI3MR=m
+CONFIG_SCSI_SMARTPQI=m
+CONFIG_SCSI_HPTIOP=m
+CONFIG_LIBFC=m
+CONFIG_LIBFCOE=m
+CONFIG_FCOE=m
+CONFIG_SCSI_SNIC=m
+CONFIG_SCSI_STEX=m
+CONFIG_SCSI_SYM53C8XX_2=m
+CONFIG_SCSI_QLA_FC=m
+CONFIG_TCM_QLA2XXX=m
+CONFIG_SCSI_QLA_ISCSI=m
+CONFIG_QEDI=m
+CONFIG_QEDF=m
+CONFIG_SCSI_LPFC=m
+CONFIG_SCSI_WD719X=m
+CONFIG_SCSI_PMCRAID=m
+CONFIG_SCSI_PM8001=m
+CONFIG_SCSI_BFA_FC=m
+CONFIG_SCSI_VIRTIO=m
+CONFIG_SCSI_CHELSIO_FCOE=m
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_HP_SW=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_SCSI_DH_ALUA=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_AHCI_ESWIN=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_RAID=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_VERITY=m
+CONFIG_TARGET_CORE=m
+CONFIG_TCM_IBLOCK=m
+CONFIG_TCM_FILEIO=m
+CONFIG_TCM_PSCSI=m
+CONFIG_TCM_USER2=m
+CONFIG_LOOPBACK_TARGET=m
+CONFIG_TCM_FC=m
+CONFIG_ISCSI_TARGET=m
+CONFIG_ISCSI_TARGET_CXGB4=m
+CONFIG_FUSION=y
+CONFIG_FUSION_SPI=m
+CONFIG_FUSION_FC=m
+CONFIG_FUSION_SAS=m
+CONFIG_FUSION_CTL=m
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_NET_FC=y
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_TUN=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+CONFIG_E100=m
+CONFIG_E1000=m
+CONFIG_E1000E=m
+CONFIG_IGB=m
+CONFIG_IGBVF=m
+CONFIG_IXGBE=m
+CONFIG_IXGBEVF=m
+CONFIG_I40E=m
+CONFIG_I40EVF=m
+CONFIG_ICE=m
+CONFIG_FM10K=m
+CONFIG_IGC=m
+# CONFIG_NET_VENDOR_LITEX is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+CONFIG_MLX4_EN=m
+CONFIG_MLX5_CORE=m
+CONFIG_MLX5_FPGA=y
+CONFIG_MLX5_CORE_EN=y
+CONFIG_MLX5_CORE_IPOIB=y
+CONFIG_MLX5_EN_TLS=y
+CONFIG_MLX5_SF=y
+CONFIG_MLXSW_CORE=m
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_PENSANDO is not set
+CONFIG_QLA3XXX=m
+CONFIG_QLCNIC=m
+CONFIG_NETXEN_NIC=m
+CONFIG_QED=m
+CONFIG_QEDE=m
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_WIN2030=y
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+CONFIG_NGBE=m
+CONFIG_TXGBE=m
+# CONFIG_NET_VENDOR_WIZNET is not set
+# CONFIG_NET_VENDOR_XILINX is not set
+CONFIG_MOTORCOMM_PHY=y
+CONFIG_REALTEK_PHY=y
+CONFIG_PPP=m
+CONFIG_PPP_BSDCOMP=m
+CONFIG_PPP_DEFLATE=m
+CONFIG_PPP_MPPE=m
+CONFIG_PPPOL2TP=m
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+CONFIG_IWLWIFI=m
+CONFIG_IWLDVM=m
+CONFIG_IWLMVM=m
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_MICROCHIP is not set
+# CONFIG_WLAN_VENDOR_PURELIFI is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+CONFIG_RTL8180=m
+CONFIG_RTL8187=m
+CONFIG_RTL8192CE=m
+CONFIG_RTL8192SE=m
+CONFIG_RTL8192DE=m
+CONFIG_RTL8723AE=m
+CONFIG_RTL8723BE=m
+CONFIG_RTL8188EE=m
+CONFIG_RTL8192EE=m
+CONFIG_RTL8821AE=m
+CONFIG_RTL8192CU=m
+# CONFIG_RTLWIFI_DEBUG is not set
+CONFIG_RTL8XXXU=m
+CONFIG_RTW88=m
+CONFIG_RTW88_8822BE=m
+CONFIG_RTW88_8822BS=m
+CONFIG_RTW88_8822BU=m
+CONFIG_RTW88_8822CE=m
+CONFIG_RTW88_8822CS=m
+CONFIG_RTW88_8822CU=m
+CONFIG_RTW88_8723DE=m
+CONFIG_RTW88_8723DS=m
+CONFIG_RTW88_8723DU=m
+CONFIG_RTW88_8821CE=m
+CONFIG_RTW88_8821CS=m
+CONFIG_RTW88_8821CU=m
+CONFIG_RTW89=m
+CONFIG_RTW89_8851BE=m
+CONFIG_RTW89_8852AE=m
+CONFIG_RTW89_8852BE=m
+CONFIG_RTW89_8852CE=m
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_SILABS is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_BCMDHD_SDIO_IRQ=y
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_GOODIX=y
+CONFIG_SERIO_LIBPS2=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_HVC_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+# CONFIG_I2C_COMPAT is not set
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_MUX_PCA954x=y
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I2C_ALGOBIT=y
+CONFIG_I2C_DESIGNWARE_SLAVE=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_SLAVE_EEPROM=y
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=y
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_MMIO=y
+CONFIG_SPI_ESWIN_BOOTSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7X=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=y
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_INA2XX=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_PAC1934=y
+CONFIG_THERMAL=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y
+CONFIG_THERMAL_GOV_FAIR_SHARE=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_BANG_BANG=y
+CONFIG_THERMAL_GOV_USER_SPACE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_DW_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_MPQ8785=y
+CONFIG_REGULATOR_ES5340=y
+CONFIG_REGULATOR_PCA9450=y
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_PWC=m
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_MEDIA_PCI_SUPPORT=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_XILINX=y
+CONFIG_CXD2880_SPI_DRV=m
+CONFIG_MEDIA_TUNER_E4000=m
+CONFIG_MEDIA_TUNER_FC0011=m
+CONFIG_MEDIA_TUNER_FC0012=m
+CONFIG_MEDIA_TUNER_FC0013=m
+CONFIG_MEDIA_TUNER_FC2580=m
+CONFIG_MEDIA_TUNER_IT913X=m
+CONFIG_MEDIA_TUNER_M88RS6000T=m
+CONFIG_MEDIA_TUNER_MAX2165=m
+CONFIG_MEDIA_TUNER_MSI001=m
+CONFIG_MEDIA_TUNER_MT2060=m
+CONFIG_MEDIA_TUNER_MT2063=m
+CONFIG_MEDIA_TUNER_MT2131=m
+CONFIG_MEDIA_TUNER_MT2266=m
+CONFIG_MEDIA_TUNER_MXL301RF=m
+CONFIG_MEDIA_TUNER_MXL5005S=m
+CONFIG_MEDIA_TUNER_MXL5007T=m
+CONFIG_MEDIA_TUNER_QM1D1B0004=m
+CONFIG_MEDIA_TUNER_QM1D1C0042=m
+CONFIG_MEDIA_TUNER_QT1010=m
+CONFIG_MEDIA_TUNER_R820T=m
+CONFIG_MEDIA_TUNER_SI2157=m
+CONFIG_MEDIA_TUNER_TDA18212=m
+CONFIG_MEDIA_TUNER_TDA18218=m
+CONFIG_MEDIA_TUNER_TDA18250=m
+CONFIG_MEDIA_TUNER_TUA9001=m
+CONFIG_DVB_MXL5XX=m
+CONFIG_DVB_STB0899=m
+CONFIG_DVB_STB6100=m
+CONFIG_DVB_STV090x=m
+CONFIG_DVB_STV0910=m
+CONFIG_DVB_STV6110x=m
+CONFIG_DVB_STV6111=m
+CONFIG_DVB_DRXK=m
+CONFIG_DVB_MN88472=m
+CONFIG_DVB_MN88473=m
+CONFIG_DVB_SI2165=m
+CONFIG_DVB_TDA18271C2DD=m
+CONFIG_DVB_CX24110=m
+CONFIG_DVB_CX24116=m
+CONFIG_DVB_CX24117=m
+CONFIG_DVB_CX24120=m
+CONFIG_DVB_CX24123=m
+CONFIG_DVB_DS3000=m
+CONFIG_DVB_MB86A16=m
+CONFIG_DVB_MT312=m
+CONFIG_DVB_S5H1420=m
+CONFIG_DVB_SI21XX=m
+CONFIG_DVB_STB6000=m
+CONFIG_DVB_STV0288=m
+CONFIG_DVB_STV0299=m
+CONFIG_DVB_STV0900=m
+CONFIG_DVB_STV6110=m
+CONFIG_DVB_TDA10071=m
+CONFIG_DVB_TDA10086=m
+CONFIG_DVB_TDA8083=m
+CONFIG_DVB_TDA8261=m
+CONFIG_DVB_TDA826X=m
+CONFIG_DVB_TS2020=m
+CONFIG_DVB_TUA6100=m
+CONFIG_DVB_TUNER_CX24113=m
+CONFIG_DVB_TUNER_ITD1000=m
+CONFIG_DVB_VES1X93=m
+CONFIG_DVB_ZL10036=m
+CONFIG_DVB_ZL10039=m
+CONFIG_DVB_CX22700=m
+CONFIG_DVB_CX22702=m
+CONFIG_DVB_CXD2820R=m
+CONFIG_DVB_CXD2841ER=m
+CONFIG_DVB_DIB3000MB=m
+CONFIG_DVB_DIB3000MC=m
+CONFIG_DVB_DIB7000M=m
+CONFIG_DVB_DIB7000P=m
+CONFIG_DVB_DIB9000=m
+CONFIG_DVB_DRXD=m
+CONFIG_DVB_EC100=m
+CONFIG_DVB_L64781=m
+CONFIG_DVB_MT352=m
+CONFIG_DVB_NXT6000=m
+CONFIG_DVB_S5H1432=m
+CONFIG_DVB_SP887X=m
+CONFIG_DVB_STV0367=m
+CONFIG_DVB_TDA10048=m
+CONFIG_DVB_TDA1004X=m
+CONFIG_DVB_ZD1301_DEMOD=m
+CONFIG_DVB_ZL10353=m
+CONFIG_DVB_CXD2880=m
+CONFIG_DVB_STV0297=m
+CONFIG_DVB_TDA10021=m
+CONFIG_DVB_TDA10023=m
+CONFIG_DVB_VES1820=m
+CONFIG_DVB_AU8522_DTV=m
+CONFIG_DVB_AU8522_V4L=m
+CONFIG_DVB_BCM3510=m
+CONFIG_DVB_LG2160=m
+CONFIG_DVB_LGDT3305=m
+CONFIG_DVB_LGDT330X=m
+CONFIG_DVB_MXL692=m
+CONFIG_DVB_NXT200X=m
+CONFIG_DVB_OR51132=m
+CONFIG_DVB_OR51211=m
+CONFIG_DVB_S5H1409=m
+CONFIG_DVB_S5H1411=m
+CONFIG_DVB_DIB8000=m
+CONFIG_DVB_MB86A20S=m
+CONFIG_DVB_S921=m
+CONFIG_DVB_MN88443X=m
+CONFIG_DVB_TC90522=m
+CONFIG_DVB_PLL=m
+CONFIG_DVB_TUNER_DIB0070=m
+CONFIG_DVB_TUNER_DIB0090=m
+CONFIG_DVB_A8293=m
+CONFIG_DVB_AF9033=m
+CONFIG_DVB_ASCOT2E=m
+CONFIG_DVB_ATBM8830=m
+CONFIG_DVB_HELENE=m
+CONFIG_DVB_HORUS3A=m
+CONFIG_DVB_ISL6405=m
+CONFIG_DVB_ISL6421=m
+CONFIG_DVB_ISL6423=m
+CONFIG_DVB_IX2505V=m
+CONFIG_DVB_LGS8GL5=m
+CONFIG_DVB_LGS8GXX=m
+CONFIG_DVB_LNBH25=m
+CONFIG_DVB_LNBH29=m
+CONFIG_DVB_LNBP21=m
+CONFIG_DVB_LNBP22=m
+CONFIG_DVB_M88RS2000=m
+CONFIG_DVB_TDA665x=m
+CONFIG_DVB_DRX39XYJ=m
+CONFIG_DVB_CXD2099=m
+CONFIG_DVB_SP2=m
+# CONFIG_DRM_I2C_CH7006 is not set
+# CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_I2C_NXP_TDA9950=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_AMDGPU_USERPTR=y
+CONFIG_DRM_AMD_ACP=y
+CONFIG_DRM_AMD_DC_SI=y
+CONFIG_DRM_NOUVEAU=m
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_TOSHIBA_TC358768=m
+CONFIG_DRM_SIMPLEDRM=m
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_SIMPLE=m
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=y
+CONFIG_SND_ESWIN_DW_I2S=y
+CONFIG_ESWIN_SND_SOC_CODECS=y
+CONFIG_ESWIN_SND_ES8388_CODEC=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_AUDIO_GRAPH_CARD=y
+CONFIG_USB_ULPI_BUS=y
+CONFIG_USB_CONN_GPIO=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+# CONFIG_USB_DEFAULT_PERSIST is not set
+CONFIG_USB_MON=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=y
+CONFIG_USB_WDM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_ULPI=y
+CONFIG_USB_DWC2=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_CP210X=y
+CONFIG_USB_SERIAL_FTDI_SIO=y
+CONFIG_USB_SERIAL_KEYSPAN=y
+CONFIG_USB_SERIAL_PL2303=y
+CONFIG_USB_SERIAL_OTI6858=y
+CONFIG_USB_SERIAL_QUALCOMM=y
+CONFIG_USB_SERIAL_OPTION=y
+CONFIG_USB_TEST=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_DEBUG_FILES=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB303B=y
+CONFIG_MMC=y
+CONFIG_MMC_TEST=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_INFINIBAND=m
+CONFIG_INFINIBAND_USER_MAD=m
+CONFIG_INFINIBAND_USER_ACCESS=m
+CONFIG_INFINIBAND_CXGB4=m
+CONFIG_INFINIBAND_IRDMA=m
+CONFIG_MLX4_INFINIBAND=m
+CONFIG_MLX5_INFINIBAND=m
+CONFIG_INFINIBAND_MTHCA=m
+# CONFIG_INFINIBAND_MTHCA_DEBUG is not set
+CONFIG_EDAC=y
+CONFIG_EDAC_ESWIN=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_DS1307=y
+CONFIG_RTC_DRV_DS1307_CENTURY=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=y
+CONFIG_DMATEST=m
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_STAGING=y
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_ESWIN=y
+CONFIG_VIDEO_ESWIN_VDEC=y
+CONFIG_VIDEO_ESWIN_VENC=y
+CONFIG_VIDEO_ESWIN_HAE=y
+CONFIG_VIDEO_ESWIN_DEWARP=y
+CONFIG_VIDEO_ESWIN_MEDIA_EXT_DRIVER=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_TIMER_ESWIN=y
+CONFIG_MAILBOX=y
+CONFIG_ESWIN_MBOX=y
+CONFIG_ESWIN_LPCPU=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY=y
+CONFIG_ESWIN_DSP=m
+CONFIG_ESWIN_NPU=m
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_POWERSAVE=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PASSIVE=y
+CONFIG_EXTCON=y
+CONFIG_MEMORY=y
+CONFIG_ESWIN_BUDDY=y
+CONFIG_ESWIN_PROC=y
+CONFIG_ESWIN_RSVMEM_HEAP=y
+CONFIG_ESWIN_MMZ_VB=y
+CONFIG_ESWIN_DEV_DMA_BUF=y
+CONFIG_ESWIN_IOMMU_RSV=y
+CONFIG_ESWIN_DMA_MEMCP=y
+CONFIG_ESWIN_MALLOC_DMABUF=y
+CONFIG_PWM=y
+CONFIG_PWM_ESWIN=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_REISERFS_FS=m
+CONFIG_REISERFS_FS_XATTR=y
+CONFIG_REISERFS_FS_POSIX_ACL=y
+CONFIG_REISERFS_FS_SECURITY=y
+CONFIG_JFS_FS=m
+CONFIG_JFS_POSIX_ACL=y
+CONFIG_JFS_SECURITY=y
+CONFIG_XFS_FS=m
+CONFIG_XFS_QUOTA=y
+CONFIG_XFS_POSIX_ACL=y
+CONFIG_XFS_RT=y
+CONFIG_GFS2_FS=m
+CONFIG_GFS2_FS_LOCKING_DLM=y
+CONFIG_OCFS2_FS=m
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_F2FS_FS=m
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_ZONEFS_FS=m
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_FUSE_FS=y
+CONFIG_CUSE=y
+CONFIG_VIRTIO_FS=y
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
+CONFIG_OVERLAY_FS_INDEX=y
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS_RW=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_FTRACE=y
+CONFIG_PSTORE_RAM=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_DLM=m
+CONFIG_SECURITY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_SECURITY_LANDLOCK=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_CRYPTO_CBC=m
+# CONFIG_CRYPTO_CTS is not set
+# CONFIG_CRYPTO_XTS is not set
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_CRYPTO_DEV_ESWIN_EIC770x=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
+CONFIG_CONSOLE_LOGLEVEL_QUIET=7
+CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+# CONFIG_FTRACE is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_MEMTEST=y
diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c
index 3d9a32d791f7..57fac9c2ccba 100644
--- a/arch/riscv/errata/sifive/errata.c
+++ b/arch/riscv/errata/sifive/errata.c
@@ -12,6 +12,7 @@
 #include <asm/alternative.h>
 #include <asm/vendorid_list.h>
 #include <asm/errata_list.h>
+#include <asm/cacheflush.h>
 
 struct errata_info_t {
 	char name[32];
@@ -94,6 +95,14 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
 	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 		return;
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	/* Set this just to make core cbo code happy */
+	if (stage == RISCV_ALTERNATIVES_BOOT) {
+		riscv_cbom_block_size = 1;
+		riscv_noncoherent_supported();
+	}
+#endif
+
 	cpu_req_errata = sifive_errata_probe(archid, impid);
 
 	for (alt = begin; alt < end; alt++) {
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index 110752594228..600abcec3631 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -71,6 +71,10 @@ do {									\
  */
 #define smp_mb__after_spinlock()	RISCV_FENCE(iorw,iorw)
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#define wfe()	do { } while (0)
+#endif
+
 #include <asm-generic/barrier.h>
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/riscv/include/asm/dma-noncoherent.h b/arch/riscv/include/asm/dma-noncoherent.h
index 312cfa0858fb..b1e0cba77c41 100644
--- a/arch/riscv/include/asm/dma-noncoherent.h
+++ b/arch/riscv/include/asm/dma-noncoherent.h
@@ -7,7 +7,7 @@
 #define __ASM_DMA_NONCOHERENT_H
 
 #include <linux/dma-direct.h>
-
+#include <uapi/linux/es_vb_user.h>
 /*
  * struct riscv_nonstd_cache_ops - Structure for non-standard CMO function pointers
  *
@@ -21,8 +21,52 @@ struct riscv_nonstd_cache_ops {
 	void (*wback_inv)(phys_addr_t paddr, size_t size);
 };
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+typedef enum {
+	FLAT_DDR_MEM = 0,
+	INTERLEAVE_DDR_MEM,
+	SPRAM,
+} eic770x_memory_type_t;
+
+void _do_arch_sync_cache_all(EIC770X_LOGICAL_MEM_NODE_E nid);
+#endif
+
 extern struct riscv_nonstd_cache_ops noncoherent_cache_ops;
 
 void riscv_noncoherent_register_cache_ops(const struct riscv_nonstd_cache_ops *ops);
 
+
+static inline void arch_get_mem_node_and_type(unsigned long pfn,
+	EIC770X_LOGICAL_MEM_NODE_E *nid, eic770x_memory_type_t *p_mem_type)
+{
+#ifdef CONFIG_NUMA
+	if (unlikely(PFN_IN_SPRAM_DIE0(pfn))) {
+		*nid = EIC770X_LOGICAL_SPRAM_NODE_0;
+		*p_mem_type = SPRAM;
+	}
+	else if(unlikely(PFN_IN_SPRAM_DIE1(pfn))) {
+		*nid = EIC770X_LOGICAL_SPRAM_NODE_1;
+		*p_mem_type = SPRAM;
+	}
+	else {
+		if (CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, INTPART0) ||
+			CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, INTPART1)) {
+			*p_mem_type = INTERLEAVE_DDR_MEM;
+			*nid = EIC770X_LOGICAL_INTERLEAVE_MEM_NODE;
+		} else {
+			*p_mem_type = FLAT_DDR_MEM;
+			*nid = pfn_to_nid(pfn);
+		}
+	}
+#else
+	if (unlikely(PFN_IN_SPRAM_DIE0(pfn))) {
+		*nid = EIC770X_LOGICAL_SPRAM_NODE_0;
+		*p_mem_type = SPRAM;
+	}
+	else {
+		*nid = EIC770X_LOGICAL_FLAT_MEM_NODE_0;
+		*p_mem_type = FLAT_DDR_MEM;
+	}
+#endif
+}
 #endif	/* __ASM_DMA_NONCOHERENT_H */
diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
index 42497d487a17..0c3c9507dca1 100644
--- a/arch/riscv/include/asm/io.h
+++ b/arch/riscv/include/asm/io.h
@@ -52,6 +52,9 @@
 #define __io_pbw()	__asm__ __volatile__ ("fence iow,o" : : : "memory");
 #define __io_paw()	__asm__ __volatile__ ("fence o,io"  : : : "memory");
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#define __iomb()	mb()
+#endif
 /*
  * Accesses from a single hart to a single I/O address must be ordered.  This
  * allows us to use the raw read macros, but we still need to fence before and
diff --git a/arch/riscv/include/asm/membarrier.h b/arch/riscv/include/asm/membarrier.h
index 6c016ebb5020..47b240d0d596 100644
--- a/arch/riscv/include/asm/membarrier.h
+++ b/arch/riscv/include/asm/membarrier.h
@@ -22,6 +22,25 @@ static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
 	/*
 	 * The membarrier system call requires a full memory barrier
 	 * after storing to rq->curr, before going back to user-space.
+	 *
+	 * This barrier is also needed for the SYNC_CORE command when
+	 * switching between processes; in particular, on a transition
+	 * from a thread belonging to another mm to a thread belonging
+	 * to the mm for which a membarrier SYNC_CORE is done on CPU0:
+	 *
+	 *   - [CPU0] sets all bits in the mm icache_stale_mask (in
+	 *     prepare_sync_core_cmd());
+	 *
+	 *   - [CPU1] stores to rq->curr (by the scheduler);
+	 *
+	 *   - [CPU0] loads rq->curr within membarrier and observes
+	 *     cpu_rq(1)->curr->mm != mm, so the IPI is skipped on
+	 *     CPU1; this means membarrier relies on switch_mm() to
+	 *     issue the sync-core;
+	 *
+	 *   - [CPU1] switch_mm() loads icache_stale_mask; if the bit
+	 *     is zero, switch_mm() may incorrectly skip the sync-core.
+	 *
 	 * Matches a full barrier in the proximity of the membarrier
 	 * system call entry.
 	 */
diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h
index f896708e8331..dbe56bc01230 100644
--- a/arch/riscv/include/asm/pgtable-bits.h
+++ b/arch/riscv/include/asm/pgtable-bits.h
@@ -18,6 +18,13 @@
 #define _PAGE_DIRTY     (1 << 7)    /* Set by hardware on any write */
 #define _PAGE_SOFT      (1 << 8)    /* Reserved for software */
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#define _PAGE_UNCACHE   (1 << 9)    /* Map to system port .i.e, uncached*/
+
+/* Map to system port llc .i.e, through sys port llc space */
+#define _PAGE_LLC       (1 << 12)
+#endif
+
 #define _PAGE_SPECIAL   _PAGE_SOFT
 #define _PAGE_TABLE     _PAGE_PRESENT
 
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 332a6bf72b1d..90e1fd734f7d 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -205,9 +205,194 @@ extern struct pt_alloc_ops pt_ops __initdata;
 
 #define PAGE_TABLE		__pgprot(_PAGE_TABLE)
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#define _PAGE_IOREMAP (_PAGE_KERNEL | _PAGE_UNCACHE)
+#else
 #define _PAGE_IOREMAP	((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_IO)
+#endif
 #define PAGE_KERNEL_IO		__pgprot(_PAGE_IOREMAP)
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+/* DIE0 */
+#define DIE0_MEM_PORT_PFN_START				(CONFIG_RISCV_DIE0_CACHED_OFFSET >> PAGE_SHIFT)
+#define DIE0_MEM_PORT_PFN_END				((CONFIG_RISCV_DIE0_CACHED_OFFSET + CONFIG_RISCV_DIE0_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+#define DIE0_SYS_PORT_PFN_START				(CONFIG_RISCV_DIE0_UNCACHED_OFFSET >> PAGE_SHIFT)
+#define DIE0_MEM_TO_SYS_PFN_ADDRESS(a)			(DIE0_SYS_PORT_PFN_START + ((u64)(a) - DIE0_MEM_PORT_PFN_START))
+#define DIE0_SYS_TO_MEM_PFN_ADDRESS(a)			(DIE0_MEM_PORT_PFN_START + ((u64)(a) - DIE0_SYS_PORT_PFN_START))
+#define DIE0_SYS_PORT_PFN_END				((CONFIG_RISCV_DIE0_UNCACHED_OFFSET + CONFIG_RISCV_DIE0_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+#define DIE0_SYS_PORT_LLC_PFN_START			((CONFIG_RISCV_DIE0_UNCACHED_OFFSET + 0x1800000000)>> PAGE_SHIFT)
+#define DIE0_MEM_TO_SYS_LLC_PFN_ADDRESS(a)		(DIE0_SYS_PORT_LLC_PFN_START + ((u64)(a) - DIE0_MEM_PORT_PFN_START))
+#define DIE0_SYS_LLC_TO_MEM_PFN_ADDRESS(a)		(DIE0_MEM_PORT_PFN_START + ((u64)(a) - DIE0_SYS_PORT_LLC_PFN_START))
+#define DIE0_SYS_PORT_LLC_PFN_END			(((CONFIG_RISCV_DIE0_UNCACHED_OFFSET + 0x1800000000) + CONFIG_RISCV_DIE0_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+/* DIE1 */
+#define DIE1_MEM_PORT_PFN_START				(CONFIG_RISCV_DIE1_CACHED_OFFSET >> PAGE_SHIFT)
+#define DIE1_MEM_PORT_PFN_END				((CONFIG_RISCV_DIE1_CACHED_OFFSET + CONFIG_RISCV_DIE1_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+#define DIE1_SYS_PORT_PFN_START				(CONFIG_RISCV_DIE1_UNCACHED_OFFSET >> PAGE_SHIFT)
+#define DIE1_MEM_TO_SYS_PFN_ADDRESS(a)			(DIE1_SYS_PORT_PFN_START + ((u64)(a) - DIE1_MEM_PORT_PFN_START))
+#define DIE1_SYS_TO_MEM_PFN_ADDRESS(a)			(DIE1_MEM_PORT_PFN_START + ((u64)(a) - DIE1_SYS_PORT_PFN_START))
+#define DIE1_SYS_PORT_PFN_END				((CONFIG_RISCV_DIE1_UNCACHED_OFFSET + CONFIG_RISCV_DIE1_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+#define DIE1_SYS_PORT_LLC_PFN_START			((CONFIG_RISCV_DIE1_UNCACHED_OFFSET + 0x1800000000) >> PAGE_SHIFT)
+#define DIE1_MEM_TO_SYS_LLC_PFN_ADDRESS(a)		(DIE1_SYS_PORT_LLC_PFN_START + ((u64)(a) - DIE1_MEM_PORT_PFN_START))
+#define DIE1_SYS_LLC_TO_MEM_PFN_ADDRESS(a)		(DIE1_MEM_PORT_PFN_START + ((u64)(a) - DIE1_SYS_PORT_LLC_PFN_START))
+#define DIE1_SYS_PORT_LLC_PFN_END			(((CONFIG_RISCV_DIE1_UNCACHED_OFFSET + 0x1800000000) + CONFIG_RISCV_DIE1_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+/* interleave */
+#define INTERLEAVE_MEM_PORT_PFN_START			(CONFIG_RISCV_INTERLEAVE_CACHED_OFFSET >> PAGE_SHIFT)
+#define INTERLEAVE_MEM_PORT_PFN_END			((CONFIG_RISCV_INTERLEAVE_CACHED_OFFSET + CONFIG_RISCV_INTERLEAVE_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+#define INTERLEAVE_SYS_PORT_PFN_START			(CONFIG_RISCV_INTERLEAVE_UNCACHED_OFFSET >> PAGE_SHIFT)
+#define INTERLEAVE_MEM_TO_SYS_PFN_ADDRESS(a)		(INTERLEAVE_SYS_PORT_PFN_START + ((u64)(a) - INTERLEAVE_MEM_PORT_PFN_START))
+#define INTERLEAVE_SYS_TO_MEM_PFN_ADDRESS(a)		(INTERLEAVE_MEM_PORT_PFN_START + ((u64)(a) - INTERLEAVE_SYS_PORT_PFN_START))
+#define INTERLEAVE_SYS_PORT_PFN_END			((CONFIG_RISCV_INTERLEAVE_UNCACHED_OFFSET + CONFIG_RISCV_INTERLEAVE_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+#define INTERLEAVE_SYS_PORT_LLC_PFN_START		((CONFIG_RISCV_INTERLEAVE_UNCACHED_OFFSET + 0x3000000000) >> PAGE_SHIFT)
+#define INTERLEAVE_MEM_TO_SYS_LLC_PFN_ADDRESS(a)	(INTERLEAVE_SYS_PORT_LLC_PFN_START + ((u64)(a) - INTERLEAVE_MEM_PORT_PFN_START))
+#define INTERLEAVE_SYS_LLC_TO_MEM_PFN_ADDRESS(a)	(INTERLEAVE_MEM_PORT_PFN_START + ((u64)(a) - INTERLEAVE_SYS_PORT_LLC_PFN_START))
+#define INTERLEAVE_SYS_PORT_LLC_PFN_END			(((CONFIG_RISCV_INTERLEAVE_UNCACHED_OFFSET + 0x3000000000) + CONFIG_RISCV_INTERLEAVE_MEM_MAX_SIZE) >> PAGE_SHIFT)
+
+/* --------new conversion------- */
+/* DIE0 MEM PORT Address range 0x8000 0000 --- 0xF FFFF FFFF, 62GB */
+#define _MEM_PORT_D0_ADDR_RANGE_VAL	((0x1FUL << 31) >> PAGE_SHIFT)
+#define _MEM_PORT_D0_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 31) >> PAGE_SHIFT)
+
+/* DIE1 MEM PORT Address range 0x20 0000 0000 --- 0x2F FFFF FFFF, 64GB */
+#define _MEM_PORT_D1_ADDR_RANGE_VAL	((0x2UL << 36) >> PAGE_SHIFT)
+#define _MEM_PORT_D1_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+
+/* Dual die, MEM PORT Interleave Address range:0x40 0000 0000 --- 0x5F FFFF FFFF, 128GB
+   part 0: 0x40 0000 0000 --- 0x4F FFFF FFFF, 64GB
+   part 1: 0x50 0000 0000 --- 0x5F FFFF FFFF, 64GB
+ */
+#define _MEM_PORT_INTPART0_ADDR_RANGE_VAL	((0x4UL << 36) >> PAGE_SHIFT)
+#define _MEM_PORT_INTPART0_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 36)>> PAGE_SHIFT)
+#define _MEM_PORT_INTPART1_ADDR_RANGE_VAL	((0x5UL << 36) >> PAGE_SHIFT)
+#define _MEM_PORT_INTPART1_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+
+
+/* DIE0 SYS PORT Address range 0xC0 0000 0000 --- 0xCF FFFF FFFF, 64GB */
+#define _SYS_PORT_D0_ADDR_RANGE_VAL		((0xCUL << 36) >> PAGE_SHIFT)
+#define _SYS_PORT_D0_ADDR_RANGE_BITMASK		(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+
+/* DIE0 SYS LLC PORT Address range 0xD8 0000 0000 --- 0xDF FFFF FFFF, 32GB */
+#define _SYSLLC_PORT_D0_ADDR_RANGE_VAL		((0x1BUL << 35) >> PAGE_SHIFT)
+#define _SYSLLC_PORT_D0_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 35)  >> PAGE_SHIFT)
+
+/* DIE1 SYS PORT Address range 0xE0 0000 0000 --- 0xEF FFFF FFFF, 64GB */
+#define _SYS_PORT_D1_ADDR_RANGE_VAL		((0xEUL << 36) >> PAGE_SHIFT)
+#define _SYS_PORT_D1_ADDR_RANGE_BITMASK		(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+
+/* DIE1 SYS LLC PORT Address range 0xF8 0000 0000 --- 0xFF FFFF FFFF, 32GB */
+#define _SYSLLC_PORT_D1_ADDR_RANGE_VAL		((0x1FUL << 35) >> PAGE_SHIFT)
+#define _SYSLLC_PORT_D1_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 35) >> PAGE_SHIFT)
+
+/* Dual die, SYS PORT Interleave Address range:0x100 0000 0000 --- 0x11F FFFF FFFF, 128GB
+   part 0: 0x100 0000 0000 --- 0x10F FFFF FFFF, 64GB
+   part 1: 0x110 0000 0000 --- 0x11F FFFF FFFF, 64GB
+ */
+#define _SYS_PORT_INTPART0_ADDR_RANGE_VAL	((0x10UL << 36) >> PAGE_SHIFT)
+#define _SYS_PORT_INTPART0_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+#define _SYS_PORT_INTPART1_ADDR_RANGE_VAL	((0x11UL << 36) >> PAGE_SHIFT)
+#define _SYS_PORT_INTPART1_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+
+/* Dual die, SYS LLC PORT Interleave Address range:0x130 0000 0000 --- 0x13F FFFF FFFF, 64GB */
+#define _SYSLLC_PORT_INT_ADDR_RANGE_VAL		((0x13UL << 36) >> PAGE_SHIFT)
+#define _SYSLLC_PORT_INT_ADDR_RANGE_BITMASK	(GENMASK_ULL(63, 36) >> PAGE_SHIFT)
+
+#define CHECK_MEMORY_RANGE_OPFUNC(pfn, range, die)	((pfn & _##range##_PORT_##die##_ADDR_RANGE_BITMASK) == _##range##_PORT_##die##_ADDR_RANGE_VAL)
+
+#define PFN_IN_SPRAM_DIE0(pfn) ((pfn_to_phys(pfn)>=0x59000000) && (pfn_to_phys(pfn)<0x59400000))
+#define PFN_IN_SPRAM_DIE1(pfn) ((pfn_to_phys(pfn)>=0x79000000) && (pfn_to_phys(pfn)<0x79400000))
+/* pha conversion between mem port and sys port or sysllc_port */
+static inline unsigned long convert_pfn_from_mem_to_sys_port(unsigned long pfn)
+{
+	if (((pfn & _MEM_PORT_D0_ADDR_RANGE_BITMASK) >= DIE0_MEM_PORT_PFN_START) && ((pfn & _MEM_PORT_D0_ADDR_RANGE_BITMASK) <= DIE0_MEM_PORT_PFN_END)) {
+		return DIE0_MEM_TO_SYS_PFN_ADDRESS(pfn);
+	}
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC7702_SOC)
+	else if (pfn < DIE0_MEM_PORT_PFN_START) {
+		return pfn;
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, D1)) {
+		return DIE1_MEM_TO_SYS_PFN_ADDRESS(pfn);
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, INTPART0) || CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, INTPART1)) {
+		return INTERLEAVE_MEM_TO_SYS_PFN_ADDRESS(pfn);
+	}
+#endif
+	else
+		return pfn;
+}
+#define convert_pha_from_mem_to_sys_port(pha) \
+			(convert_pfn_from_mem_to_sys_port(pha >> PAGE_SHIFT) << PAGE_SHIFT)
+
+static inline unsigned long convert_pfn_from_mem_to_sys_port_llc(unsigned long pfn)
+{
+	if (((pfn & _MEM_PORT_D0_ADDR_RANGE_BITMASK) >= DIE0_MEM_PORT_PFN_START) && ((pfn & _MEM_PORT_D0_ADDR_RANGE_BITMASK) <= DIE0_MEM_PORT_PFN_END)) {
+		return DIE0_MEM_TO_SYS_LLC_PFN_ADDRESS(pfn);
+	}
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC7702_SOC)
+	else if (pfn < DIE0_MEM_PORT_PFN_START) {
+		return pfn;
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, D1)) {
+		return DIE1_MEM_TO_SYS_LLC_PFN_ADDRESS(pfn);
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, INTPART0) || CHECK_MEMORY_RANGE_OPFUNC(pfn, MEM, INTPART1)) {
+		return INTERLEAVE_MEM_TO_SYS_LLC_PFN_ADDRESS(pfn);
+	}
+#endif
+	else
+		return pfn;
+}
+#define convert_pha_from_mem_to_sys_port_llc(pha) \
+			(convert_pfn_from_mem_to_sys_port_llc(pha >> PAGE_SHIFT) << PAGE_SHIFT)
+
+static inline unsigned long convert_pfn_from_sys_to_mem_port(unsigned long pfn)
+{
+	if (likely(CHECK_MEMORY_RANGE_OPFUNC(pfn, SYS, D0))) {
+		return DIE0_SYS_TO_MEM_PFN_ADDRESS(pfn);
+	}
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC7702_SOC)
+	else if (pfn < DIE0_MEM_PORT_PFN_START) {
+		return pfn;
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, SYS, D1)) {
+		return DIE1_SYS_TO_MEM_PFN_ADDRESS(pfn);
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, SYS, INTPART0) || CHECK_MEMORY_RANGE_OPFUNC(pfn, SYS, INTPART1)) {
+		return INTERLEAVE_SYS_TO_MEM_PFN_ADDRESS(pfn);
+	}
+#endif
+	else
+		return pfn;
+}
+
+static inline unsigned long convert_pfn_from_sys_llc_to_mem_port(unsigned long pfn)
+{
+	if (likely(CHECK_MEMORY_RANGE_OPFUNC(pfn, SYSLLC, D0))) {
+		return DIE0_SYS_LLC_TO_MEM_PFN_ADDRESS(pfn);
+	}
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC7702_SOC)
+	else if (pfn < DIE0_MEM_PORT_PFN_START) {
+		return pfn;
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, SYSLLC, D1)) {
+		return DIE1_SYS_LLC_TO_MEM_PFN_ADDRESS(pfn);
+	}
+	else if (CHECK_MEMORY_RANGE_OPFUNC(pfn, SYSLLC, INT)) {
+		return INTERLEAVE_SYS_LLC_TO_MEM_PFN_ADDRESS(pfn);
+	}
+#endif
+	else
+		return pfn;
+}
+#endif
+
 extern pgd_t swapper_pg_dir[];
 extern pgd_t trampoline_pg_dir[];
 extern pgd_t early_pg_dir[];
@@ -325,12 +510,21 @@ static inline unsigned long pte_napot(pte_t pte)
 /* Yields the page frame number (PFN) of a page table entry */
 static inline unsigned long pte_pfn(pte_t pte)
 {
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	if (unlikely(pte_val(pte) & _PAGE_UNCACHE)) {
+		return convert_pfn_from_sys_to_mem_port(__page_val_to_pfn(pte_val(pte)));
+	}
+	else {
+		return convert_pfn_from_sys_llc_to_mem_port(__page_val_to_pfn(pte_val(pte)));
+	}
+#else
 	unsigned long res  = __page_val_to_pfn(pte_val(pte));
 
 	if (has_svnapot() && pte_napot(pte))
 		res = res & (res - 1UL);
 
 	return res;
+#endif
 }
 
 #define pte_page(x)     pfn_to_page(pte_pfn(x))
@@ -338,11 +532,27 @@ static inline unsigned long pte_pfn(pte_t pte)
 /* Constructs a page table entry */
 static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
 {
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	unsigned long pfn_new;
+
+	if (unlikely(_PAGE_UNCACHE == (pgprot_val(prot) & _PAGE_UNCACHE))) {
+		pfn_new = convert_pfn_from_mem_to_sys_port(pfn);
+		pr_debug("pfn_pte:pfn_mport 0x%lx to pfn_sysport 0x%lx\n", pfn, pfn_new);
+		return __pte((pfn_new << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+	}
+	else if (unlikely(_PAGE_LLC == (pgprot_val(prot) & _PAGE_LLC))) {
+		pfn_new = convert_pfn_from_mem_to_sys_port_llc(pfn);
+		pr_debug("pfn_pte:pfn_mport 0x%lx to pfn_llc 0x%lx\n", pfn, pfn_new);
+		return __pte((pfn_new << _PAGE_PFN_SHIFT) | (pgprot_val(prot) & ~_PAGE_LLC));
+	}
+	return __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+#else
 	unsigned long prot_val = pgprot_val(prot);
 
 	ALT_THEAD_PMA(prot_val);
 
 	return __pte((pfn << _PAGE_PFN_SHIFT) | prot_val);
+#endif
 }
 
 #define mk_pte(page, prot)       pfn_pte(page_to_pfn(page), prot)
@@ -596,6 +806,17 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
 	return ptep_test_and_clear_young(vma, address, ptep);
 }
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#define pgprot_noncached(prot) \
+			__pgprot(pgprot_val(prot) | _PAGE_UNCACHE)
+
+#define pgprot_writecombine(prot) pgprot_noncached(prot)
+
+#define pgprot_dmacoherent(prot)    pgprot_noncached(prot)
+
+#define pgprot_llc(prot) \
+			__pgprot(pgprot_val(prot) | _PAGE_LLC)
+#else
 #define pgprot_noncached pgprot_noncached
 static inline pgprot_t pgprot_noncached(pgprot_t _prot)
 {
@@ -617,6 +838,7 @@ static inline pgprot_t pgprot_writecombine(pgprot_t _prot)
 
 	return __pgprot(prot);
 }
+#endif
 
 /*
  * THP functions
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index 4f6af8c6cfa0..590e6d1cde70 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -13,42 +13,19 @@
 
 #include <asm/ptrace.h>
 
-#ifdef CONFIG_64BIT
-#define DEFAULT_MAP_WINDOW	(UL(1) << (MMAP_VA_BITS - 1))
-#define STACK_TOP_MAX		TASK_SIZE
-
 #define arch_get_mmap_end(addr, len, flags)			\
 ({								\
-	unsigned long mmap_end;					\
-	typeof(addr) _addr = (addr);				\
-	if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \
-		mmap_end = STACK_TOP_MAX;			\
-	else if ((_addr) >= VA_USER_SV57)			\
-		mmap_end = STACK_TOP_MAX;			\
-	else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \
-		mmap_end = VA_USER_SV48;			\
-	else							\
-		mmap_end = VA_USER_SV39;			\
-	mmap_end;						\
+	STACK_TOP_MAX;						\
 })
 
 #define arch_get_mmap_base(addr, base)				\
 ({								\
-	unsigned long mmap_base;				\
-	typeof(addr) _addr = (addr);				\
-	typeof(base) _base = (base);				\
-	unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base);	\
-	if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \
-		mmap_base = (_base);				\
-	else if (((_addr) >= VA_USER_SV57) && (VA_BITS >= VA_BITS_SV57)) \
-		mmap_base = VA_USER_SV57 - rnd_gap;		\
-	else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \
-		mmap_base = VA_USER_SV48 - rnd_gap;		\
-	else							\
-		mmap_base = VA_USER_SV39 - rnd_gap;		\
-	mmap_base;						\
+	base;							\
 })
 
+#ifdef CONFIG_64BIT
+#define DEFAULT_MAP_WINDOW	(UL(1) << (MMAP_VA_BITS - 1))
+#define STACK_TOP_MAX		TASK_SIZE_64
 #else
 #define DEFAULT_MAP_WINDOW	TASK_SIZE
 #define STACK_TOP_MAX		TASK_SIZE
diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
index 3ed853b8a8c8..8d261a317175 100644
--- a/arch/riscv/include/asm/sbi.h
+++ b/arch/riscv/include/asm/sbi.h
@@ -29,6 +29,7 @@ enum sbi_ext_id {
 	SBI_EXT_RFENCE = 0x52464E43,
 	SBI_EXT_HSM = 0x48534D,
 	SBI_EXT_SRST = 0x53525354,
+	SBI_EXT_SUSP = 0x53555350,
 	SBI_EXT_PMU = 0x504D55,
 
 	/* Experimentals extensions must lie within this range */
@@ -113,6 +114,14 @@ enum sbi_srst_reset_reason {
 	SBI_SRST_RESET_REASON_SYS_FAILURE,
 };
 
+enum sbi_ext_susp_fid {
+	SBI_EXT_SUSP_SYSTEM_SUSPEND = 0,
+};
+
+enum sbi_ext_susp_sleep_type {
+	SBI_SUSP_SLEEP_TYPE_SUSPEND_TO_RAM = 0,
+};
+
 enum sbi_ext_pmu_fid {
 	SBI_EXT_PMU_NUM_COUNTERS = 0,
 	SBI_EXT_PMU_COUNTER_GET_INFO,
diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h
index a727be723c56..6fd519f9e364 100644
--- a/arch/riscv/include/asm/switch_to.h
+++ b/arch/riscv/include/asm/switch_to.h
@@ -63,6 +63,20 @@ static __always_inline bool has_fpu(void)
 	return riscv_has_extension_likely(RISCV_ISA_EXT_f) ||
 		riscv_has_extension_likely(RISCV_ISA_EXT_d);
 }
+
+static inline void kernel_fpu_begin(void)
+{
+	preempt_disable();
+	fstate_save(current, task_pt_regs(current));
+	csr_set(CSR_SSTATUS, SR_FS);
+}
+
+static inline void kernel_fpu_end(void)
+{
+	csr_clear(CSR_SSTATUS, SR_FS);
+	fstate_restore(current, task_pt_regs(current));
+	preempt_enable();
+}
 #else
 static __always_inline bool has_fpu(void) { return false; }
 #define fstate_save(task, regs) do { } while (0)
diff --git a/arch/riscv/include/asm/sync_core.h b/arch/riscv/include/asm/sync_core.h
new file mode 100644
index 000000000000..9153016da8f1
--- /dev/null
+++ b/arch/riscv/include/asm/sync_core.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_RISCV_SYNC_CORE_H
+#define _ASM_RISCV_SYNC_CORE_H
+
+/*
+ * RISC-V implements return to user-space through an xRET instruction,
+ * which is not core serializing.
+ */
+static inline void sync_core_before_usermode(void)
+{
+	asm volatile ("fence.i" ::: "memory");
+}
+
+#ifdef CONFIG_SMP
+/*
+ * Ensure the next switch_mm() on every CPU issues a core serializing
+ * instruction for the given @mm.
+ */
+static inline void prepare_sync_core_cmd(struct mm_struct *mm)
+{
+	cpumask_setall(&mm->context.icache_stale_mask);
+}
+#else
+static inline void prepare_sync_core_cmd(struct mm_struct *mm)
+{
+}
+#endif /* CONFIG_SMP */
+
+#endif /* _ASM_RISCV_SYNC_CORE_H */
diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c
index e264e59e596e..a191ba7f8892 100644
--- a/arch/riscv/kernel/module-sections.c
+++ b/arch/riscv/kernel/module-sections.c
@@ -9,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleloader.h>
+#include <linux/sort.h>
 
 unsigned long module_emit_got_entry(struct module *mod, unsigned long val)
 {
@@ -55,19 +56,27 @@ unsigned long module_emit_plt_entry(struct module *mod, unsigned long val)
 	return (unsigned long)&plt[i];
 }
 
-static int is_rela_equal(const Elf_Rela *x, const Elf_Rela *y)
+#define cmp_3way(a, b)	((a) < (b) ? -1 : (a) > (b))
+
+static int cmp_rela(const void *a, const void *b)
 {
-	return x->r_info == y->r_info && x->r_addend == y->r_addend;
+	const Elf_Rela *x = a, *y = b;
+	int i;
+
+	/* sort by type, symbol index and addend */
+	i = cmp_3way(x->r_info, y->r_info);
+	if (i == 0)
+		i = cmp_3way(x->r_addend, y->r_addend);
+	return i;
 }
 
 static bool duplicate_rela(const Elf_Rela *rela, int idx)
 {
-	int i;
-	for (i = 0; i < idx; i++) {
-		if (is_rela_equal(&rela[i], &rela[idx]))
-			return true;
-	}
-	return false;
+	/*
+	 * Entries are sorted by type, symbol index and addend. That means
+	 * that, if a duplicate entry exists, it must be in the preceding slot.
+	 */
+	return idx > 0 && cmp_rela(rela + idx, rela + idx - 1) == 0;
 }
 
 static void count_max_entries(Elf_Rela *relas, int num,
@@ -87,11 +96,33 @@ static void count_max_entries(Elf_Rela *relas, int num,
 	}
 }
 
+static bool rela_needs_plt_got(const Elf_Rela *rela)
+{
+	unsigned int type = ELF_R_TYPE(rela->r_info);
+
+	return type == R_RISCV_CALL_PLT || type == R_RISCV_GOT_HI20;
+}
+
+/* Copy PLT and GOT relas to the scratch array. */
+static unsigned int partition_plt_got_relas(const Elf_Rela *relas, Elf_Rela *scratch,
+					    unsigned int num_rela)
+{
+	int j = 0;
+
+	for (int i = 0; i < num_rela; i++)
+		if (rela_needs_plt_got(&relas[i]))
+			scratch[j++] = relas[i];
+
+	return j;
+}
+
 int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
 			      char *secstrings, struct module *mod)
 {
 	unsigned int num_plts = 0;
 	unsigned int num_gots = 0;
+	Elf_Rela *scratch = NULL;
+	size_t scratch_size = 0;
 	int i;
 
 	/*
@@ -132,9 +163,26 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
 		if (!(dst_sec->sh_flags & SHF_EXECINSTR))
 			continue;
 
-		count_max_entries(relas, num_rela, &num_plts, &num_gots);
+		/*
+		 * apply_relocate_add() relies on HI20 and LO12 relocation pairs being
+		 * close together, so sort a copy of the section to avoid interfering.
+		 */
+		if (sechdrs[i].sh_size > scratch_size) {
+			scratch = kvrealloc(scratch, scratch_size, sechdrs[i].sh_size, GFP_KERNEL);
+			if (!scratch)
+				return -ENOMEM;
+			scratch_size = sechdrs[i].sh_size;
+		}
+
+		/* sort relocations requiring a PLT or GOT entry so duplicates are adjacent */
+		num_rela = partition_plt_got_relas(relas, scratch, num_rela);
+		sort(scratch, num_rela, sizeof(Elf_Rela), cmp_rela, NULL);
+		count_max_entries(scratch, num_rela, &num_plts, &num_gots);
 	}
 
+	if (scratch)
+		kvfree(scratch);
+
 	mod->arch.plt.shdr->sh_type = SHT_NOBITS;
 	mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
 	mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES;
diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c
index df4f6fec5d17..31000dacbba0 100644
--- a/arch/riscv/kernel/module.c
+++ b/arch/riscv/kernel/module.c
@@ -346,6 +346,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 	Elf_Sym *sym;
 	u32 *location;
 	unsigned int i, type;
+	unsigned int j_idx = 0;
 	Elf_Addr v;
 	int res;
 
@@ -384,9 +385,10 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 		v = sym->st_value + rel[i].r_addend;
 
 		if (type == R_RISCV_PCREL_LO12_I || type == R_RISCV_PCREL_LO12_S) {
-			unsigned int j;
+			unsigned int j = j_idx;
+			bool found = false;
 
-			for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) {
+			do {
 				unsigned long hi20_loc =
 					sechdrs[sechdrs[relsec].sh_info].sh_addr
 					+ rel[j].r_offset;
@@ -415,16 +417,26 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 					hi20 = (offset + 0x800) & 0xfffff000;
 					lo12 = offset - hi20;
 					v = lo12;
+					found = true;
 
 					break;
 				}
-			}
-			if (j == sechdrs[relsec].sh_size / sizeof(*rel)) {
+
+				j++;
+				if (j > sechdrs[relsec].sh_size / sizeof(*rel))
+					j = 0;
+
+			} while (j_idx != j);
+
+			if (!found) {
 				pr_err(
 				  "%s: Can not find HI20 relocation information\n",
 				  me->name);
 				return -EINVAL;
 			}
+
+			/* Record the previous j-loop end index */
+			j_idx = j;
 		}
 
 		res = handler(me, location, v);
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 83e223318822..dd973216e31c 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -204,3 +204,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 	p->thread.sp = (unsigned long)childregs; /* kernel sp */
 	return 0;
 }
+
+EXPORT_SYMBOL_GPL(__fstate_save);
+EXPORT_SYMBOL_GPL(__fstate_restore);
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 9584d62c96ee..78e69a4073a6 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -509,7 +509,10 @@ static void kvm_riscv_vcpu_update_config(const unsigned long *isa)
 	if (riscv_isa_extension_available(isa, ZICBOZ))
 		henvcfg |= ENVCFG_CBZE;
 
+#ifndef CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY
 	csr_write(CSR_HENVCFG, henvcfg);
+#endif
+
 #ifdef CONFIG_32BIT
 	csr_write(CSR_HENVCFGH, henvcfg >> 32);
 #endif
diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c
index e2e2a115afb5..0a8ab8c3700f 100644
--- a/arch/riscv/kvm/vcpu_pmu.c
+++ b/arch/riscv/kvm/vcpu_pmu.c
@@ -556,6 +556,8 @@ void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu)
 	 */
 	if (!riscv_isa_extension_available(NULL, SSCOFPMF))
 		return;
+	/* reject sscofpmf */
+	return;
 
 	ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs);
 	if (ret < 0 || !hpm_width || !num_hw_ctrs)
diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c
index 341bd6706b4c..86cfa1cfc493 100644
--- a/arch/riscv/mm/dma-noncoherent.c
+++ b/arch/riscv/mm/dma-noncoherent.c
@@ -7,9 +7,14 @@
 
 #include <linux/dma-direct.h>
 #include <linux/dma-map-ops.h>
+#include <linux/io.h>
 #include <linux/mm.h>
 #include <asm/cacheflush.h>
 #include <asm/dma-noncoherent.h>
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#include <soc/sifive/sifive_ccache.h>
+#include <linux/iommu.h>
+#endif
 
 static bool noncoherent_supported __ro_after_init;
 int dma_cache_alignment __ro_after_init = ARCH_DMA_MINALIGN;
@@ -67,7 +72,7 @@ static inline bool arch_sync_dma_cpu_needs_post_dma_flush(void)
 }
 
 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
-			      enum dma_data_direction dir)
+				  enum dma_data_direction dir)
 {
 	switch (dir) {
 	case DMA_TO_DEVICE:
@@ -84,7 +89,7 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
 	case DMA_BIDIRECTIONAL:
 		/* Skip the invalidate here if it's done later */
 		if (IS_ENABLED(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) &&
-		    arch_sync_dma_cpu_needs_post_dma_flush())
+			arch_sync_dma_cpu_needs_post_dma_flush())
 			arch_dma_cache_wback(paddr, size);
 		else
 			arch_dma_cache_wback_inv(paddr, size);
@@ -94,6 +99,9 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
 		break;
 	}
 }
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+EXPORT_SYMBOL(arch_sync_dma_for_device);
+#endif
 
 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 			   enum dma_data_direction dir)
@@ -113,6 +121,9 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 		break;
 	}
 }
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+EXPORT_SYMBOL(arch_sync_dma_for_cpu);
+#endif
 
 void arch_dma_prep_coherent(struct page *page, size_t size)
 {
@@ -128,6 +139,13 @@ void arch_dma_prep_coherent(struct page *page, size_t size)
 	ALT_CMO_OP(flush, flush_addr, size, riscv_cbom_block_size);
 }
 
+#ifdef CONFIG_IOMMU_DMA
+void arch_teardown_dma_ops(struct device *dev)
+{
+	dev->dma_ops = NULL;
+}
+#endif
+
 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 		const struct iommu_ops *iommu, bool coherent)
 {
@@ -142,12 +160,17 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 		   dev_driver_string(dev), dev_name(dev));
 
 	dev->dma_coherent = coherent;
+
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	if (iommu)
+		iommu_setup_dma_ops(dev, dma_base, size);
+	#endif
 }
 
 void riscv_noncoherent_supported(void)
 {
 	WARN(!riscv_cbom_block_size,
-	     "Non-coherent DMA support enabled without a block size\n");
+		 "Non-coherent DMA support enabled without a block size\n");
 	noncoherent_supported = true;
 }
 
@@ -156,3 +179,137 @@ void __init riscv_set_dma_cache_alignment(void)
 	if (!noncoherent_supported)
 		dma_cache_alignment = 1;
 }
+
+#ifdef CONFIG_ARCH_HAS_DMA_SET_UNCACHED
+static struct page **__iommu_dma_common_find_pages(void *cpu_addr)
+{
+	struct vm_struct *area = find_vm_area(cpu_addr);
+
+	if (!area || area->flags != VM_DMA_COHERENT)
+		return NULL;
+	return area->pages;
+}
+
+void arch_dma_clear_uncached(void *addr, size_t size)
+{
+	struct page **pages = NULL;
+
+	pr_debug("smmu_dbg, %s, remap addr:0x%p, size:0x%lx\n",
+		__func__, addr, size);
+	pages = __iommu_dma_common_find_pages(addr);
+	if (!pages) { // todo: supposed to handle this error
+		pr_err( "smmu_dbg, %s:%d, fail to find pages\n",
+			__func__, __LINE__);
+
+		return;
+	}
+	kvfree(pages);
+	memunmap(addr);
+}
+
+void *arch_dma_set_uncached(void *addr, size_t size)
+{
+	struct page **pages = NULL;
+	static struct page *page = NULL;
+	struct vm_struct *area = NULL;
+	phys_addr_t phys_addr = convert_pha_from_mem_to_sys_port(__pa(addr));
+	void *mem_base = NULL;
+
+	pr_debug("smmu_dbg, %s, pfn:0x%lx, pha:0x%016lx, vaddr:0x%px\n",
+		__func__, virt_to_pfn(addr), __pa(addr), addr);
+	mem_base = memremap(phys_addr, size, MEMREMAP_WT);
+	if (!mem_base) {
+		pr_err("%s memremap failed for addr %px\n", __func__, addr);
+		return ERR_PTR(-EINVAL);
+	}
+
+	pr_debug("smmu_dbg, %s, pha+offset:0x%016llx, remap vaddr:0x%px, size:0x%lx\n",
+		__func__, phys_addr, mem_base, size);
+
+	pages = kvzalloc(sizeof(*pages), GFP_KERNEL);
+	if (!pages) {
+		pr_err("smmu_dbg, %s:%d, failed to alloc memory!\n",
+			__func__, __LINE__);
+		goto err_pages_alloc;
+	}
+	page = virt_to_page(addr);
+	area = find_vm_area(mem_base);
+	if (!area) {
+		pr_err("smmu_dbg, %s:%d, failed to find vm area!\n",
+			__func__, __LINE__);
+		goto err_find_vm_area;
+	}
+	pr_debug("smmu_dbg, %s, check area-pages=0x%px\n", __func__, area->pages);
+	pages[0] = page;
+	area->pages = pages;
+	area->flags = VM_DMA_COHERENT;
+
+	return mem_base;
+
+err_find_vm_area:
+	kvfree(pages);
+
+err_pages_alloc:
+	memunmap(mem_base);
+
+	return NULL;
+}
+#endif
+
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#ifdef CONFIG_NUMA
+static inline void _do_arch_sync_cache_all_by_hartid(int hartid) {
+	int cpuid;
+	int hardid_mask = BIT(hartid);
+	cpuid = riscv_hartid_to_cpuid(hartid);
+	smp_call_function_single(cpuid, ccache_flush_all, &hardid_mask, true);
+}
+#endif
+void _do_arch_sync_cache_all(EIC770X_LOGICAL_MEM_NODE_E nid)
+{
+#ifdef CONFIG_NUMA
+switch (nid) {
+	case EIC770X_LOGICAL_FLAT_MEM_NODE_0: {
+		_do_arch_sync_cache_all_by_hartid(0);
+		break;
+	}
+	case EIC770X_LOGICAL_FLAT_MEM_NODE_1: {
+		_do_arch_sync_cache_all_by_hartid(4);
+		break;
+	}
+	case EIC770X_LOGICAL_INTERLEAVE_MEM_NODE: {
+		struct cpumask cpus = {0};
+		int hartids = BIT(0) | BIT(4);
+
+		cpumask_set_cpu(riscv_hartid_to_cpuid(0), &cpus);
+		cpumask_set_cpu(riscv_hartid_to_cpuid(4), &cpus);
+		smp_call_function_many(&cpus, ccache_flush_all, &hartids, true);
+		break;
+	}
+	default: {
+		break;
+	}
+}
+#else
+	int cpuid, hartid_mask;
+
+	cpuid = smp_processor_id();
+	hartid_mask = BIT(cpuid_to_hartid_map(cpuid));
+	ccache_flush_all(&hartid_mask);
+#endif
+}
+
+void arch_sync_cache_all(phys_addr_t phys, size_t size)
+{
+	EIC770X_LOGICAL_MEM_NODE_E nid;
+	eic770x_memory_type_t mem_type;
+
+	arch_get_mem_node_and_type(phys_to_pfn(phys), &nid, &mem_type);
+	if (nid == EIC770X_LOGICAL_SPRAM_NODE_0 ||
+		nid == EIC770X_LOGICAL_SPRAM_NODE_1)
+		return;
+
+	_do_arch_sync_cache_all(nid);
+}
+EXPORT_SYMBOL(arch_sync_cache_all);
+#endif
\ No newline at end of file
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index bdf8ac6c7e30..3d783bfb05ed 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -295,7 +295,6 @@ static void __init setup_bootmem(void)
 	if (!IS_ENABLED(CONFIG_BUILTIN_DTB))
 		memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
 
-	dma_contiguous_reserve(dma32_phys_limit);
 	if (IS_ENABLED(CONFIG_64BIT))
 		hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
 }
@@ -1512,6 +1511,8 @@ static void __init reserve_crashkernel(void)
 
 	crashk_res.start = crash_base;
 	crashk_res.end = crash_base + crash_size - 1;
+	pr_info("insert_resource:crashk_res(0x%lx, 0x%lx)\n", crashk_res.start, crashk_res.end);
+	insert_resource(&iomem_resource, &crashk_res);
 }
 
 void __init paging_init(void)
@@ -1533,6 +1534,14 @@ void __init misc_mem_init(void)
 	local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END);
 #endif
 	zone_sizes_init();
+
+	/*
+	 * Reserve the CMA area after dma32_phys_limit was initialised.
+	 * It must be called after arch_numa_init() which calls numa_init() to
+	 * initialize node_online_map that gets used by dma_contiguous_reserve()
+	 */
+	dma_contiguous_reserve(dma32_phys_limit);
+
 	reserve_crashkernel();
 	memblock_dump_all();
 }
diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
index ef887efcb679..21ea6ed76470 100644
--- a/arch/riscv/mm/pgtable.c
+++ b/arch/riscv/mm/pgtable.c
@@ -36,6 +36,7 @@ pud_t *pud_offset(p4d_t *p4d, unsigned long address)
 
 	return (pud_t *)p4d;
 }
+EXPORT_SYMBOL_GPL(pud_offset);
 
 p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
 {
@@ -44,6 +45,7 @@ p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
 
 	return (p4d_t *)pgd;
 }
+EXPORT_SYMBOL_GPL(p4d_offset);
 #endif
 
 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
diff --git a/drivers/Kconfig b/drivers/Kconfig
index efb66e25fa2d..fbcc3b7a3580 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -243,4 +243,5 @@ source "drivers/hte/Kconfig"
 
 source "drivers/cdx/Kconfig"
 
+source "drivers/reset-kdump/Kconfig"
 endmenu
diff --git a/drivers/Makefile b/drivers/Makefile
index 1bec7819a837..83f1b3ee5e5d 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -199,3 +199,4 @@ obj-$(CONFIG_DRM_ACCEL)		+= accel/
 obj-$(CONFIG_CDX_BUS)		+= cdx/
 
 obj-$(CONFIG_S390)		+= s390/
+obj-$(CONFIG_RSTKDUMP)  += reset-kdump/
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 42b51c9812a0..fcc935629d93 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -136,6 +136,15 @@ config SATA_MOBILE_LPM_POLICY
 	  Note "Minimum power" is known to cause issues, including disk
 	  corruption, with some disks and should not be used.
 
+config AHCI_ESWIN
+	tristate "Eswin AHCI SATA support"
+	select SATA_HOST
+	help
+	  This option enables support for Eswin AHCI Serial ATA
+	  controllers.
+
+	  If unsure, say N.
+
 config SATA_AHCI_PLATFORM
 	tristate "Platform AHCI SATA support"
 	select SATA_HOST
diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile
index 20e6645ab737..a596e5ea5134 100644
--- a/drivers/ata/Makefile
+++ b/drivers/ata/Makefile
@@ -13,6 +13,7 @@ obj-$(CONFIG_SATA_INIC162X)	+= sata_inic162x.o
 obj-$(CONFIG_SATA_SIL24)	+= sata_sil24.o
 obj-$(CONFIG_SATA_DWC)		+= sata_dwc_460ex.o
 obj-$(CONFIG_SATA_HIGHBANK)	+= sata_highbank.o libahci.o
+obj-$(CONFIG_AHCI_ESWIN)	+= ahci_eswin.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_BRCM)		+= ahci_brcm.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_CEVA)		+= ahci_ceva.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_DA850)	+= ahci_da850.o libahci.o libahci_platform.o
diff --git a/drivers/ata/ahci_eswin.c b/drivers/ata/ahci_eswin.c
new file mode 100644
index 000000000000..ff22d11488d7
--- /dev/null
+++ b/drivers/ata/ahci_eswin.c
@@ -0,0 +1,318 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN AHCI SATA Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/device.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/libata.h>
+#include <linux/ahci_platform.h>
+#include <linux/acpi.h>
+#include <linux/pci_ids.h>
+#include <linux/iommu.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+#include  <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include "ahci.h"
+
+#define DRV_NAME "ahci"
+
+#define AWSMMUSID                     GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID                    GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID                     GENMASK(15, 8)  // The sid of read operation
+#define ARSMMUSSID                    GENMASK(7, 0)   // The ssid of read operation
+#define SATA_REF_CTRL1                0x338
+#define SATA_PHY_CTRL0                0x328
+#define SATA_PHY_CTRL1                0x32c
+#define SATA_LOS_IDEN                 0x33c
+#define SATA_AXI_LP_CTRL              0x308
+#define SATA_REG_CTRL                 0x334
+#define SATA_MPLL_CTRL                0x320
+#define SATA_RESET_CTRL               0x340
+#define SATA_RESET_CTRL_ASSERT        0x3
+#define SATA_RESET_CTRL_DEASSERT      0x0
+#define SATA_PHY_RESET                BIT(0)
+#define SATA_P0_RESET                 BIT(1)
+#define SATA_LOS_LEVEL                0x9
+#define SATA_LOS_BIAS                 (0x02 << 16)
+#define SATA_REF_REPEATCLK_EN         BIT(0)
+#define SATA_REF_USE_PAD              BIT(20)
+#define SATA_P0_AMPLITUDE_GEN1        0x42
+#define SATA_P0_AMPLITUDE_GEN2        (0x46 << 8)
+#define SATA_P0_AMPLITUDE_GEN3        (0x73 << 16)
+#define SATA_P0_PHY_TX_PREEMPH_GEN1   0x05
+#define SATA_P0_PHY_TX_PREEMPH_GEN2   (0x05 << 8)
+#define SATA_P0_PHY_TX_PREEMPH_GEN3   (0x08 << 16)
+#define SATA_MPLL_MULTIPLIER          (0x3c << 16)
+#define SATA_M_CSYSREQ                BIT(0)
+#define SATA_S_CSYSREQ                BIT(16)
+
+struct eswin_ahci_plat {
+    struct reset_control *apb_rst;
+};
+
+static const struct ata_port_info ahci_port_info = {
+    .flags		= AHCI_FLAG_COMMON,
+    .pio_mask	= ATA_PIO4,
+    .udma_mask	= ATA_UDMA6,
+    .port_ops	= &ahci_platform_ops,
+};
+
+static const struct ata_port_info ahci_port_info_nolpm = {
+    .flags		= AHCI_FLAG_COMMON | ATA_FLAG_NO_LPM,
+    .pio_mask	= ATA_PIO4,
+    .udma_mask	= ATA_UDMA6,
+    .port_ops	= &ahci_platform_ops,
+};
+
+static struct scsi_host_template ahci_platform_sht = {
+    AHCI_SHT(DRV_NAME),
+};
+
+static int eswin_sata_sid_cfg(struct device *dev)
+{
+    int ret;
+    struct regmap *regmap;
+    int hsp_mmu_sata_reg;
+    u32 rdwr_sid_ssid;
+    u32 sid;
+    struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+    /* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+    if (fwspec == NULL) {
+        dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+        return 0;
+    }
+    sid = fwspec->ids[0];
+    regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+    if (IS_ERR(regmap)) {
+        dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+        return 0;
+    }
+    ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+            &hsp_mmu_sata_reg);
+    if (ret) {
+        dev_err(dev, "can't get sata sid cfg reg offset (%d)\n", ret);
+        return ret;
+    }
+
+    /* make the reading sid the same as writing sid, ssid is fixed to zero */
+    rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+    rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+    rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+    rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+    regmap_write(regmap, hsp_mmu_sata_reg, rdwr_sid_ssid);
+
+    ret = win2030_dynm_sid_enable(dev_to_node(dev));
+    if (ret < 0)
+        dev_err(dev, "failed to config sata streamID(%d)!\n", sid);
+     else
+        dev_dbg(dev, "success to config sata streamID(%d)!\n", sid);
+    pr_err("eswin_sata_sid_cfg success\n");
+
+    return ret;
+}
+
+static int eswin_sata_init(struct device *dev)
+{
+    struct regmap *regmap;
+    regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+    if (IS_ERR(regmap)) {
+        dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+        return -1;
+    }
+
+    regmap_write(regmap, SATA_REF_CTRL1, 0x1);
+    regmap_write(regmap, SATA_PHY_CTRL0, (SATA_P0_AMPLITUDE_GEN1|SATA_P0_AMPLITUDE_GEN2|SATA_P0_AMPLITUDE_GEN3));
+    regmap_write(regmap, SATA_PHY_CTRL1, (SATA_P0_PHY_TX_PREEMPH_GEN1|SATA_P0_PHY_TX_PREEMPH_GEN2|SATA_P0_PHY_TX_PREEMPH_GEN3));
+    regmap_write(regmap, SATA_LOS_IDEN, SATA_LOS_LEVEL|SATA_LOS_BIAS);
+    regmap_write(regmap, SATA_AXI_LP_CTRL, (SATA_M_CSYSREQ|SATA_S_CSYSREQ));
+    regmap_write(regmap, SATA_REG_CTRL, (SATA_REF_REPEATCLK_EN|SATA_REF_USE_PAD));
+    regmap_write(regmap, SATA_MPLL_CTRL, SATA_MPLL_MULTIPLIER);
+    udelay(20);
+
+    return 0;
+}
+
+static int eswin_ahci_platform_resets(struct ahci_host_priv *hpriv,
+				    struct device *dev)
+{
+	struct eswin_ahci_plat *plat = hpriv->plat_data;
+    struct regmap *regmap;
+    int ret;
+
+    regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+    if (IS_ERR(regmap)) {
+        dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+        return -1;
+    }
+
+    plat->apb_rst = devm_reset_control_get_optional(dev, "apb");
+	if (PTR_ERR(plat->apb_rst) == -EPROBE_DEFER)
+		return PTR_ERR(plat->apb_rst);
+
+    ret = reset_control_assert(plat->apb_rst);
+	if (ret) {
+		dev_err(dev, "failed to assert apb_rst\n");
+		return ret;
+	}
+    regmap_write(regmap, SATA_RESET_CTRL, SATA_RESET_CTRL_ASSERT);
+
+    regmap_write(regmap, SATA_RESET_CTRL, SATA_RESET_CTRL_DEASSERT);
+    ret = reset_control_deassert(plat->apb_rst);
+	if (ret) {
+		dev_err(dev, "failed to deassert apb_rst\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ahci_probe(struct platform_device *pdev)
+{
+    struct device *dev = &pdev->dev;
+    struct ahci_host_priv *hpriv;
+    struct eswin_ahci_plat *plat;
+    const struct ata_port_info *port;
+    int ret;
+
+    plat = devm_kzalloc(dev, sizeof(*plat), GFP_KERNEL);
+	if (!plat)
+		return -ENOMEM;
+
+    hpriv = ahci_platform_get_resources(pdev,
+                        0);
+    if (IS_ERR(hpriv))
+        return PTR_ERR(hpriv);
+
+    eswin_sata_init(dev);  //The PHY parameters need to be modified before resetting
+
+    hpriv->plat_data = plat;
+    ret = eswin_ahci_platform_resets(hpriv, dev);
+	if (ret)
+		return ret;
+
+    ret = ahci_platform_enable_resources(hpriv);
+	if (ret)
+		return ret;
+
+    eswin_sata_sid_cfg(dev);
+
+    win2030_tbu_power(&pdev->dev, true);
+
+    ret = dma_set_mask_and_coherent(dev,DMA_BIT_MASK(41));
+    if (ret)
+        return ret;
+
+    of_property_read_u32(dev->of_node,
+                "ports-implemented", &hpriv->saved_port_map);
+
+    if (of_device_is_compatible(dev->of_node, "hisilicon,hisi-ahci"))
+        hpriv->flags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ;
+
+    port = acpi_device_get_match_data(dev);
+    if (!port){
+        port = &ahci_port_info;
+    }
+
+    ret = ahci_platform_init_host(pdev, hpriv, port,
+                &ahci_platform_sht);
+    if (ret)
+        goto disable_resources;
+
+    return 0;
+disable_resources:
+    ahci_platform_disable_resources(hpriv);
+    return ret;
+}
+
+static int ahci_remove(struct platform_device *pdev)
+{
+    win2030_tbu_power(&pdev->dev, false);
+
+    ata_platform_remove_one(pdev);
+
+    return 0;
+}
+
+static int eswin_ahci_suspend(struct device *dev)
+{
+	int ret;
+
+    win2030_tbu_power(dev, false);
+
+	ret = ahci_platform_suspend(dev);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int eswin_ahci_resume(struct device *dev)
+{
+	int ret;
+
+	ret = ahci_platform_resume(dev);
+	if (ret)
+		return ret;
+
+	win2030_tbu_power(dev, true);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(ahci_pm_ops, eswin_ahci_suspend,
+            eswin_ahci_resume);
+
+static const struct of_device_id ahci_of_match[] = {
+    { .compatible = "snps,eswin-ahci", },
+    {},
+};
+MODULE_DEVICE_TABLE(of, ahci_of_match);
+
+static const struct acpi_device_id ahci_acpi_match[] = {
+    { "APMC0D33", (unsigned long)&ahci_port_info_nolpm },
+    { ACPI_DEVICE_CLASS(PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff) },
+    {},
+};
+MODULE_DEVICE_TABLE(acpi, ahci_acpi_match);
+
+static struct platform_driver ahci_driver = {
+    .probe = ahci_probe,
+    .remove = ahci_remove,
+    .shutdown = ahci_platform_shutdown,
+    .driver = {
+        .name = DRV_NAME,
+        .of_match_table = ahci_of_match,
+        .acpi_match_table = ahci_acpi_match,
+        .pm = &ahci_pm_ops,
+    },
+};
+module_platform_driver(ahci_driver);
+
+MODULE_DESCRIPTION("ESWIN AHCI SATA driver");
+MODULE_AUTHOR("Lu Yulin <luyulin@eswincomputing.com>");
+MODULE_LICENSE("GPL");
\ No newline at end of file
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index c30099866174..151de7e16525 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -471,6 +471,7 @@ source "drivers/clk/actions/Kconfig"
 source "drivers/clk/analogbits/Kconfig"
 source "drivers/clk/baikal-t1/Kconfig"
 source "drivers/clk/bcm/Kconfig"
+source "drivers/clk/eswin/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
 source "drivers/clk/imgtec/Kconfig"
 source "drivers/clk/imx/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index 18969cbd4bb1..a41095b98844 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -91,6 +91,7 @@ obj-$(CONFIG_CLK_BAIKAL_T1)		+= baikal-t1/
 obj-y					+= bcm/
 obj-$(CONFIG_ARCH_BERLIN)		+= berlin/
 obj-$(CONFIG_ARCH_DAVINCI)		+= davinci/
+obj-$(CONFIG_ARCH_ESWIN)		+= eswin/
 obj-$(CONFIG_ARCH_HISI)			+= hisilicon/
 obj-y					+= imgtec/
 obj-y					+= imx/
diff --git a/drivers/clk/eswin/Kconfig b/drivers/clk/eswin/Kconfig
new file mode 100644
index 000000000000..8e3045a15a51
--- /dev/null
+++ b/drivers/clk/eswin/Kconfig
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config COMMON_CLK_EIC7700
+	bool "EIC7700 Clock Driver"
+	help
+	  Build the Eswin EIC7700 SOC clock driver based on the common clock framework.
+	  This driver provides support for the clock control on the Eswin EIC7700 SoC,
+	  which is essential for managing clock rates and power management.
+
+if COMMON_CLK_EIC7700
+	source "drivers/clk/eswin/test/Kconfig"
+endif
diff --git a/drivers/clk/eswin/Makefile b/drivers/clk/eswin/Makefile
new file mode 100644
index 000000000000..23cc1d811bf5
--- /dev/null
+++ b/drivers/clk/eswin/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Eswin Clock specific Makefile
+#
+
+obj-y	+= clk.o
+
+obj-$(CONFIG_COMMON_CLK_EIC7700)	+= clk-eic7700.o
+obj-$(CONFIG_COMMON_CLK_EIC7700_TEST)	+= test/
\ No newline at end of file
diff --git a/drivers/clk/eswin/clk-eic7700.c b/drivers/clk/eswin/clk-eic7700.c
new file mode 100644
index 000000000000..ee01dd2d1890
--- /dev/null
+++ b/drivers/clk/eswin/clk-eic7700.c
@@ -0,0 +1,3808 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * ESWIN EIC7700 Clk Provider Driver
+ *
+ * Authors: huangyifeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk.h>
+#include <dt-bindings/clock/eswin,eic7700-clock.h>
+#include "clk_eic7700.h"
+#include "clk.h"
+
+/* clock parent list */
+static const char *const mux_u_cpu_root_3mux1_gfree_p[] = {
+	"clk_pll_cpu", "clk_clk_u84_core_lp", "fixed_rate_clk_xtal_24m"
+};
+static u32 mux_u_cpu_root_3mux1_gfree_p_table[] = { 0x000000, 0x000001,
+						    0x000002 };
+
+static const char *const mux_u_cpu_aclk_2mux1_gfree_p[] = {
+	"fixed_factor_u_cpu_div2", "mux_u_cpu_root_3mux1_gfree"
+};
+
+static const char *const dsp_aclk_root_2mux1_gfree_mux_p[] = {
+	"fixed_rate_clk_spll2_fout1", "fixed_rate_clk_spll0_fout1"
+};
+
+static const char *const d2d_aclk_root_2mux1_gfree_mux_p[] = {
+	"fixed_rate_clk_spll2_fout1",
+	"fixed_rate_clk_spll0_fout1",
+};
+
+static const char *const ddr_aclk_root_2mux1_gfree_mux_p[] = {
+	"fixed_rate_clk_spll2_fout1",
+	"fixed_rate_clk_spll0_fout1",
+};
+
+static const char *const mshcore_root_3mux1_0_mux_p[] = {
+	"fixed_rate_clk_spll0_fout3", "fixed_rate_clk_spll2_fout3",
+	"fixed_rate_clk_xtal_24m"
+};
+static u32 mshcore_root_3mux1_0_mux_p_table[] = { 0x000000, 0x000001,
+						  0x100000 };
+
+static const char *const mshcore_root_3mux1_1_mux_p[] = {
+	"fixed_rate_clk_spll0_fout3", "fixed_rate_clk_spll2_fout3",
+	"fixed_rate_clk_xtal_24m"
+};
+static u32 mshcore_root_3mux1_1_mux_p_table[] = { 0x000000, 0x000001,
+						  0x100000 };
+
+static const char *const mshcore_root_3mux1_2_mux_p[] = {
+	"fixed_rate_clk_spll0_fout3", "fixed_rate_clk_spll2_fout3",
+	"fixed_rate_clk_xtal_24m"
+};
+static u32 mshcore_root_3mux1_2_mux_p_table[] = { 0x000000, 0x000001,
+						  0x100000 };
+
+static const char *const npu_llclk_3mux1_gfree_mux_p[] = {
+	"clk_clk_npu_llc_src0", "clk_clk_npu_llc_src1",
+	"fixed_rate_clk_vpll_fout1"
+};
+static u32 npu_llclk_3mux1_gfree_mux_p_table[] = { 0x000000, 0x000001,
+						   0x000002 };
+
+static const char *const npu_core_3mux1_gfree_mux_p[] = {
+	"fixed_rate_clk_spll1_fout1", "fixed_rate_clk_vpll_fout1",
+	"fixed_rate_clk_spll2_fout2"
+};
+static u32 npu_core_3mux1_gfree_mux_p_table[] = { 0x000000, 0x000001,
+						  0x000002 };
+
+static const char *const npu_e31_3mux1_gfree_mux_p[] = {
+	"fixed_rate_clk_spll1_fout1", "fixed_rate_clk_vpll_fout1",
+	"fixed_rate_clk_spll2_fout2"
+};
+static u32 npu_e31_3mux1_gfree_mux_p_table[] = { 0x000000, 0x000100, 0x000200 };
+
+static const char *const vi_aclk_root_2mux1_gfree_mux_p[] = {
+	"fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"
+};
+
+static const char *const mux_u_vi_dw_root_2mux1_p[] = {
+	"fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll0_fout1"
+};
+
+static const char *const mux_u_vi_dvp_root_2mux1_gfree_p[] = {
+	"fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll0_fout1"
+};
+
+static const char *const mux_u_vi_dig_isp_root_2mux1_gfree_p[] = {
+	"fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll0_fout1"
+};
+
+static const char *const mux_u_vo_aclk_root_2mux1_gfree_p[] = {
+	"fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"
+};
+
+static const char *const mux_u_vo_pixel_root_2mux1_p[] = {
+	"fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll2_fout2"
+};
+
+static const char *const mux_u_vcdec_root_2mux1_gfree_p[] = {
+	"fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"
+};
+
+static const char *const mux_u_vcaclk_root_2mux1_gfree_p[] = {
+	"fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"
+};
+
+static const char *const mux_u_syscfg_clk_root_2mux1_gfree_p[] = {
+	"divder_u_sys_cfg_div_dynm", "fixed_rate_clk_xtal_24m"
+};
+
+static const char *const mux_u_bootspi_clk_2mux1_gfree_p[] = {
+	"divder_u_bootspi_div_dynm", "fixed_rate_clk_xtal_24m"
+};
+
+static const char *const mux_u_scpu_core_clk_2mux1_gfree_p[] = {
+	"divder_u_scpu_core_div_dynm", "fixed_rate_clk_xtal_24m"
+};
+
+static const char *const mux_u_lpcpu_core_clk_2mux1_gfree_p[] = {
+	"divder_u_lpcpu_core_div_dynm", "fixed_rate_clk_xtal_24m"
+};
+
+static const char *const mux_u_vo_mclk_2mux_ext_mclk_p[] = {
+	"divder_u_vo_mclk_div_dynm", "fixed_rate_ext_mclk"
+};
+
+static const char *const mux_u_aondma_axi2mux1_gfree_p[] = {
+	"divder_u_aondma_axi_div_dynm", "fixed_rate_clk_xtal_24m"
+};
+
+static const char *const mux_u_rmii_ref_2mux1_p[] = {
+	"fixed_factor_u_hsp_rmii_ref_div6", "fixed_rate_lpddr_ref_bak"
+};
+
+static const char *const mux_u_eth_core_2mux1_p[] = {
+	"fixed_rate_clk_spll1_fout3", "fixed_rate_lpddr_ref_bak"
+};
+
+static const char *const mux_u_sata_phy_2mux1_p[] = {
+	"divder_u_sata_phy_ref_div_dynm", "fixed_rate_lpddr_ref_bak"
+};
+
+/* fixed rate clocks */
+static struct eswin_fixed_rate_clock eic7700_fixed_rate_clks[] = {
+	{
+		EIC7700_XTAL_24M,
+		"fixed_rate_clk_xtal_24m",
+		NULL,
+		0,
+		24000000,
+	},
+	{
+		EIC7700_XTAL_32K,
+		"fixed_rate_clk_xtal_32k",
+		NULL,
+		0,
+		32768,
+	},
+	{
+		EIC7700_SPLL0_FOUT1,
+		"fixed_rate_clk_spll0_fout1",
+		NULL,
+		0,
+		1600000000,
+	},
+	{
+		EIC7700_SPLL0_FOUT2,
+		"fixed_rate_clk_spll0_fout2",
+		NULL,
+		0,
+		800000000,
+	},
+	{
+		EIC7700_SPLL0_FOUT3,
+		"fixed_rate_clk_spll0_fout3",
+		NULL,
+		0,
+		400000000,
+	},
+	{
+		EIC7700_SPLL1_FOUT1,
+		"fixed_rate_clk_spll1_fout1",
+		NULL,
+		0,
+		1500000000,
+	},
+	{
+		EIC7700_SPLL1_FOUT2,
+		"fixed_rate_clk_spll1_fout2",
+		NULL,
+		0,
+		300000000,
+	},
+	{
+		EIC7700_SPLL1_FOUT3,
+		"fixed_rate_clk_spll1_fout3",
+		NULL,
+		0,
+		250000000,
+	},
+	{
+		EIC7700_SPLL2_FOUT1,
+		"fixed_rate_clk_spll2_fout1",
+		NULL,
+		0,
+		2080000000,
+	},
+	{
+		EIC7700_SPLL2_FOUT2,
+		"fixed_rate_clk_spll2_fout2",
+		NULL,
+		0,
+		1040000000,
+	},
+	{
+		EIC7700_SPLL2_FOUT3,
+		"fixed_rate_clk_spll2_fout3",
+		NULL,
+		0,
+		416000000,
+	},
+	{
+		EIC7700_VPLL_FOUT1,
+		"fixed_rate_clk_vpll_fout1",
+		NULL,
+		0,
+		1188000000,
+	},
+	{
+		EIC7700_VPLL_FOUT2,
+		"fixed_rate_clk_vpll_fout2",
+		NULL,
+		0,
+		594000000,
+	},
+	{
+		EIC7700_VPLL_FOUT3,
+		"fixed_rate_clk_vpll_fout3",
+		NULL,
+		0,
+		49500000,
+	},
+	{
+		EIC7700_APLL_FOUT2,
+		"fixed_rate_clk_apll_fout2",
+		NULL,
+		0,
+		0,
+	},
+	{
+		EIC7700_APLL_FOUT3,
+		"fixed_rate_clk_apll_fout3",
+		NULL,
+		0,
+		0,
+	},
+	{
+		EIC7700_EXT_MCLK,
+		"fixed_rate_ext_mclk",
+		NULL,
+		0,
+		0,
+	},
+	{
+		EIC7700_LPDDR_REF_BAK,
+		"fixed_rate_lpddr_ref_bak",
+		NULL,
+		0,
+		50000000,
+	},
+};
+
+static struct eswin_pll_clock eic7700_pll_clks[] = {
+	{
+		EIC7700_APLL_FOUT1,
+		"clk_apll_fout1",
+		NULL,
+		EIC7700_REG_OFFSET_APLL_CFG_0,
+		0,
+		1,
+		12,
+		6,
+		20,
+		12,
+		EIC7700_REG_OFFSET_APLL_CFG_1,
+		4,
+		24,
+		EIC7700_REG_OFFSET_APLL_CFG_2,
+		1,
+		3,
+		16,
+		3,
+		EIC7700_REG_OFFSET_PLL_STATUS,
+		4,
+		1,
+	},
+	{
+		EIC7700_PLL_CPU,
+		"clk_pll_cpu",
+		NULL,
+		EIC7700_REG_OFFSET_MCPUT_PLL_CFG_0,
+		0,
+		1,
+		12,
+		6,
+		20,
+		12,
+		EIC7700_REG_OFFSET_MCPUT_PLL_CFG_1,
+		4,
+		24,
+		EIC7700_REG_OFFSET_MCPUT_PLL_CFG_2,
+		1,
+		3,
+		16,
+		3,
+		EIC7700_REG_OFFSET_PLL_STATUS,
+		5,
+		1,
+	},
+};
+
+/* fixed factor clocks */
+static struct eswin_fixed_factor_clock eic7700_fixed_factor_clks[] = {
+	{
+		EIC7700_FIXED_FACTOR_U_CPU_DIV2,
+		"fixed_factor_u_cpu_div2",
+		"mux_u_cpu_root_3mux1_gfree",
+		1,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_CLK_1M_DIV24,
+		"fixed_factor_u_clk_1m_div24",
+		"fixed_rate_clk_xtal_24m",
+		1,
+		24,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_MIPI_TXESC_DIV10,
+		"fixed_factor_u_mipi_txesc_div10",
+		"clk_clk_sys_cfg",
+		1,
+		10,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_U84_CORE_LP_DIV2,
+		"fixed_factor_u_u84_core_lp_div2",
+		"gate_clk_spll0_fout2",
+		1,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_SCPU_BUS_DIV2,
+		"fixed_factor_u_scpu_bus_div2",
+		"mux_u_scpu_core_clk_2mux1_gfree",
+		1,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_LPCPU_BUS_DIV2,
+		"fixed_factor_u_lpcpu_bus_div2",
+		"mux_u_lpcpu_core_clk_2mux1_gfree",
+		1,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_PCIE_CR_DIV2,
+		"fixed_factor_u_pcie_cr_div2",
+		"clk_clk_sys_cfg",
+		1,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_PCIE_AUX_DIV4,
+		"fixed_factor_u_pcie_aux_div4",
+		"clk_clk_sys_cfg",
+		1,
+		4,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_PVT_DIV20,
+		"fixed_factor_u_pvt_div20",
+		"fixed_rate_clk_xtal_24m",
+		1,
+		20,
+		0,
+	},
+
+	{
+		EIC7700_FIXED_FACTOR_U_HSP_RMII_REF_DIV6,
+		"fixed_factor_u_hsp_rmii_ref_div6",
+		"fixed_rate_clk_spll1_fout2",
+		1,
+		6,
+		0,
+	},
+};
+
+static struct eswin_mux_clock eic7700_mux_clks[] = {
+	{
+		EIC7700_MUX_U_CPU_ROOT_3MUX1_GFREE,
+		"mux_u_cpu_root_3mux1_gfree",
+		mux_u_cpu_root_3mux1_gfree_p,
+		ARRAY_SIZE(mux_u_cpu_root_3mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		0,
+		BIT_MASK(0) | BIT_MASK(1),
+		0,
+		mux_u_cpu_root_3mux1_gfree_p_table,
+	},
+
+	{
+		EIC7700_MUX_U_CPU_ACLK_2MUX1_GFREE,
+		"mux_u_cpu_aclk_2mux1_gfree",
+		mux_u_cpu_aclk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_cpu_aclk_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		20,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_DSP_ACLK_ROOT_2MUX1_GFREE,
+		"mux_u_dsp_aclk_root_2mux1_gfree",
+		dsp_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(dsp_aclk_root_2mux1_gfree_mux_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DSP_ACLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_D2D_ACLK_ROOT_2MUX1_GFREE,
+		"mux_u_d2d_aclk_root_2mux1_gfree",
+		d2d_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(d2d_aclk_root_2mux1_gfree_mux_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_D2D_ACLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_DDR_ACLK_ROOT_2MUX1_GFREE,
+		"mux_u_ddr_aclk_root_2mux1_gfree",
+		ddr_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(ddr_aclk_root_2mux1_gfree_mux_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		16,
+		1,
+		0,
+	},
+
+	{ EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_0, "mux_u_mshcore_root_3mux1_0",
+	  mshcore_root_3mux1_0_mux_p, ARRAY_SIZE(mshcore_root_3mux1_0_mux_p),
+	  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL, 0,
+	  BIT_MASK(0) | BIT_MASK(20), 0, mshcore_root_3mux1_0_mux_p_table },
+
+	{
+		EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1,
+		"mux_u_mshcore_root_3mux1_1",
+		mshcore_root_3mux1_1_mux_p,
+		ARRAY_SIZE(mshcore_root_3mux1_1_mux_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL,
+		0,
+		BIT_MASK(0) | BIT_MASK(20),
+		0,
+		mshcore_root_3mux1_1_mux_p_table,
+	},
+
+	{ EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_2, "mux_u_mshcore_root_3mux1_2",
+	  mshcore_root_3mux1_2_mux_p, ARRAY_SIZE(mshcore_root_3mux1_2_mux_p),
+	  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL, 0,
+	  BIT_MASK(0) | BIT_MASK(20), 0, mshcore_root_3mux1_2_mux_p_table },
+
+	{ EIC7700_MUX_U_NPU_LLCLK_3MUX1_GFREE, "mux_u_npu_llclk_3mux1_gfree",
+	  npu_llclk_3mux1_gfree_mux_p, ARRAY_SIZE(npu_llclk_3mux1_gfree_mux_p),
+	  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_NPU_LLC_CTRL, 0,
+	  BIT_MASK(0) | BIT_MASK(1), 0, npu_llclk_3mux1_gfree_mux_p_table },
+
+	{ EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE, "mux_u_npu_core_3mux1_gfree",
+	  npu_core_3mux1_gfree_mux_p, ARRAY_SIZE(npu_core_3mux1_gfree_mux_p),
+	  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_NPU_CORE_CTRL, 0,
+	  BIT_MASK(0) | BIT_MASK(1), 0, npu_core_3mux1_gfree_mux_p_table },
+
+	{ EIC7700_MUX_U_NPU_E31_3MUX1_GFREE, "mux_u_npu_e31_3mux1_gfree",
+	  npu_e31_3mux1_gfree_mux_p, ARRAY_SIZE(npu_e31_3mux1_gfree_mux_p),
+	  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_NPU_CORE_CTRL, 0,
+	  BIT_MASK(8) | BIT_MASK(9), 0, npu_e31_3mux1_gfree_mux_p_table },
+
+	{
+		EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE,
+		"mux_u_vi_aclk_root_2mux1_gfree",
+		vi_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(vi_aclk_root_2mux1_gfree_mux_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_ACLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VI_DW_ROOT_2MUX1,
+		"mux_u_vi_dw_root_2mux1",
+		mux_u_vi_dw_root_2mux1_p,
+		ARRAY_SIZE(mux_u_vi_dw_root_2mux1_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DWCLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE,
+		"mux_u_vi_dvp_root_2mux1_gfree",
+		mux_u_vi_dvp_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vi_dvp_root_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE,
+		"mux_u_vi_dig_isp_root_2mux1_gfree",
+		mux_u_vi_dig_isp_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vi_dig_isp_root_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VO_ACLK_ROOT_2MUX1_GFREE,
+		"mux_u_vo_aclk_root_2mux1_gfree",
+		mux_u_vo_aclk_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vo_aclk_root_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_ACLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VO_PIXEL_ROOT_2MUX1,
+		"mux_u_vo_pixel_root_2mux1",
+		mux_u_vo_pixel_root_2mux1_p,
+		ARRAY_SIZE(mux_u_vo_pixel_root_2mux1_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_PIXEL_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VCDEC_ROOT_2MUX1_GFREE,
+		"mux_u_vcdec_root_2mux1_gfree",
+		mux_u_vcdec_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vcdec_root_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VCDEC_ROOTCLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE,
+		"mux_u_vcaclk_root_2mux1_gfree",
+		mux_u_vcaclk_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vcaclk_root_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_ACLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_SYSCFG_CLK_ROOT_2MUX1_GFREE,
+		"mux_u_syscfg_clk_root_2mux1_gfree",
+		mux_u_syscfg_clk_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_syscfg_clk_root_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SYSCFG_CLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_BOOTSPI_CLK_2MUX1_GFREE,
+		"mux_u_bootspi_clk_2mux1_gfree",
+		mux_u_bootspi_clk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_bootspi_clk_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_SCPU_CORE_CLK_2MUX1_GFREE,
+		"mux_u_scpu_core_clk_2mux1_gfree",
+		mux_u_scpu_core_clk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_scpu_core_clk_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_LPCPU_CORE_CLK_2MUX1_GFREE,
+		"mux_u_lpcpu_core_clk_2mux1_gfree",
+		mux_u_lpcpu_core_clk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_lpcpu_core_clk_2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_VO_MCLK_2MUX_EXT_MCLK,
+		"mux_u_vo_mclk_2mux_ext_mclk",
+		mux_u_vo_mclk_2mux_ext_mclk_p,
+		ARRAY_SIZE(mux_u_vo_mclk_2mux_ext_mclk_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_MCLK_CTRL,
+		0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_AONDMA_AXI2MUX1_GFREE,
+		"mux_u_aondma_axi2mux1_gfree",
+		mux_u_aondma_axi2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_aondma_axi2mux1_gfree_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL,
+		20,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_RMII_REF_2MUX,
+		"mux_u_rmii_ref_2mux1",
+		mux_u_rmii_ref_2mux1_p,
+		ARRAY_SIZE(mux_u_rmii_ref_2mux1_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH0_CTRL,
+		2,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_ETH_CORE_2MUX1,
+		"mux_u_eth_core_2mux1",
+		mux_u_eth_core_2mux1_p,
+		ARRAY_SIZE(mux_u_eth_core_2mux1_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH0_CTRL,
+		1,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_MUX_U_SATA_PHY_2MUX1,
+		"mux_u_sata_phy_2mux1",
+		mux_u_sata_phy_2mux1_p,
+		ARRAY_SIZE(mux_u_sata_phy_2mux1_p),
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SATA_OOB_CTRL,
+		9,
+		1,
+		0,
+	},
+};
+
+/*
+ * The hardware decides value 0, 1 and 2 both means 2 divsor, so we add these tables.
+ * When using these tables, the clock framework will use the last member
+ * being 0 as a marker to indicate the end of the table,
+ * So an additional member is required.
+ */
+static struct clk_div_table u_3_bit_special_div_table[8 + 1];
+static struct clk_div_table u_4_bit_special_div_table[16 + 1];
+static struct clk_div_table u_6_bit_special_div_table[64 + 1];
+static struct clk_div_table u_7_bit_special_div_table[128 + 1];
+static struct clk_div_table u_8_bit_special_div_table[256 + 1];
+static struct clk_div_table u_11_bit_special_div_table[2048 + 1];
+static struct clk_div_table u_16_bit_special_div_table[65536 + 1];
+
+static struct eswin_divider_clock eic7700_div_clks[] = {
+	{ EIC7700_DIVDER_U_SYS_CFG_DIV_DYNM, "divder_u_sys_cfg_div_dynm",
+	  "fixed_rate_clk_spll0_fout3", 0, EIC7700_REG_OFFSET_SYSCFG_CLK_CTRL,
+	  4, 3, CLK_DIVIDER_ROUND_CLOSEST, u_3_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_NOC_NSP_DIV_DYNM, "divder_u_noc_nsp_div_dynm",
+	  "fixed_rate_clk_spll2_fout1", 0, EIC7700_REG_OFFSET_NOC_CLK_CTRL, 0,
+	  3, CLK_DIVIDER_ROUND_CLOSEST, u_3_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_BOOTSPI_DIV_DYNM, "divder_u_bootspi_div_dynm",
+	  "gate_clk_spll0_fout2", 0, EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL, 4, 6,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_6_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_SCPU_CORE_DIV_DYNM, "divder_u_scpu_core_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0, EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL,
+	  4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_LPCPU_CORE_DIV_DYNM, "divder_u_lpcpu_core_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0,
+	  EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_GPU_ACLK_DIV_DYNM, "divder_u_gpu_aclk_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0, EIC7700_REG_OFFSET_GPU_ACLK_CTRL, 4,
+	  4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_DSP_ACLK_DIV_DYNM, "divder_u_dsp_aclk_div_dynm",
+	  "clk_clk_dsp_root", 0, EIC7700_REG_OFFSET_DSP_ACLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_D2D_ACLK_DIV_DYNM, "divder_u_d2d_aclk_div_dynm",
+	  "mux_u_d2d_aclk_root_2mux1_gfree", 0,
+	  EIC7700_REG_OFFSET_D2D_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_DDR_ACLK_DIV_DYNM, "divder_u_ddr_aclk_div_dynm",
+	  "mux_u_ddr_aclk_root_2mux1_gfree", 0, EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+	  20, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_HSP_ACLK_DIV_DYNM, "divder_u_hsp_aclk_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0, EIC7700_REG_OFFSET_HSP_ACLK_CTRL, 4,
+	  4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_ETH_TXCLK_DIV_DYNM_0,
+	  "divder_u_eth_txclk_div_dynm_0", "mux_u_eth_core_2mux1", 0,
+	  EIC7700_REG_OFFSET_ETH0_CTRL, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_ETH_TXCLK_DIV_DYNM_1,
+	  "divder_u_eth_txclk_div_dynm_1", "mux_u_eth_core_2mux1", 0,
+	  EIC7700_REG_OFFSET_ETH1_CTRL, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_0,
+	  "divder_u_mshc_core_div_dynm_0", "mux_u_mshcore_root_3mux1_0", 0,
+	  EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL, 4, 12, CLK_DIVIDER_ONE_BASED },
+
+	{ EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_1,
+	  "divder_u_mshc_core_div_dynm_1", "mux_u_mshcore_root_3mux1_1", 0,
+	  EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL, 4, 12, CLK_DIVIDER_ONE_BASED },
+
+	{ EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_2,
+	  "divder_u_mshc_core_div_dynm_2", "mux_u_mshcore_root_3mux1_2", 0,
+	  EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL, 4, 12, CLK_DIVIDER_ONE_BASED },
+
+	{ EIC7700_DIVDER_U_PCIE_ACLK_DIV_DYNM, "divder_u_pcie_aclk_div_dynm",
+	  "fixed_rate_clk_spll2_fout2", 0, EIC7700_REG_OFFSET_PCIE_ACLK_CTRL, 4,
+	  4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_NPU_ACLK_DIV_DYNM, "divder_u_npu_aclk_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0, EIC7700_REG_OFFSET_NPU_ACLK_CTRL, 4,
+	  4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_NPU_LLC_SRC0_DIV_DYNM,
+	  "divder_u_npu_llc_src0_div_dynm", "fixed_rate_clk_spll0_fout1", 0,
+	  EIC7700_REG_OFFSET_NPU_LLC_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_NPU_LLC_SRC1_DIV_DYNM,
+	  "divder_u_npu_llc_src1_div_dynm", "fixed_rate_clk_spll2_fout1", 0,
+	  EIC7700_REG_OFFSET_NPU_LLC_CTRL, 8, 4, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_NPU_CORECLK_DIV_DYNM,
+	  "divder_u_npu_coreclk_div_dynm", "mux_u_npu_core_3mux1_gfree", 0,
+	  EIC7700_REG_OFFSET_NPU_CORE_CTRL, 4, 4, CLK_DIVIDER_ONE_BASED },
+
+	{ EIC7700_DIVDER_U_NPU_E31_DIV_DYNM, "divder_u_npu_e31_div_dynm",
+	  "mux_u_npu_e31_3mux1_gfree", 0, EIC7700_REG_OFFSET_NPU_CORE_CTRL, 12,
+	  4, CLK_DIVIDER_ONE_BASED },
+
+	{ EIC7700_DIVDER_U_VI_ACLK_DIV_DYNM, "divder_u_vi_aclk_div_dynm",
+	  "mux_u_vi_aclk_root_2mux1_gfree", 0, EIC7700_REG_OFFSET_VI_ACLK_CTRL,
+	  4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_DW_DIV_DYNM, "divder_u_vi_dw_div_dynm",
+	  "mux_u_vi_dw_root_2mux1", 0, EIC7700_REG_OFFSET_VI_DWCLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_DVP_DIV_DYNM, "divder_u_vi_dvp_div_dynm",
+	  "mux_u_vi_dig_root_2mux1_gfree", 0,
+	  EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_DIG_ISP_DIV_DYNM, "divder_u_vi_dig_isp_div_dynm",
+	  "mux_u_vi_dig_isp_root_2mux1_gfree", 0,
+	  EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_0,
+	  "divder_u_vi_shutter_div_dynm_0", "fixed_rate_clk_vpll_fout2", 0,
+	  EIC7700_REG_OFFSET_VI_SHUTTER0, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_1,
+	  "divder_u_vi_shutter_div_dynm_1", "fixed_rate_clk_vpll_fout2", 0,
+	  EIC7700_REG_OFFSET_VI_SHUTTER1, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_2,
+	  "divder_u_vi_shutter_div_dynm_2", "fixed_rate_clk_vpll_fout2", 0,
+	  EIC7700_REG_OFFSET_VI_SHUTTER2, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_3,
+	  "divder_u_vi_shutter_div_dynm_3", "fixed_rate_clk_vpll_fout2", 0,
+	  EIC7700_REG_OFFSET_VI_SHUTTER3, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_4,
+	  "divder_u_vi_shutter_div_dynm_4", "fixed_rate_clk_vpll_fout2", 0,
+	  EIC7700_REG_OFFSET_VI_SHUTTER4, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_5,
+	  "divder_u_vi_shutter_div_dynm_5", "fixed_rate_clk_vpll_fout2", 0,
+	  EIC7700_REG_OFFSET_VI_SHUTTER5, 4, 7, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_7_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VO_ACLK_DIV_DYNM, "divder_u_vo_aclk_div_dynm",
+	  "mux_u_vo_aclk_root_2mux1_gfree", 0, EIC7700_REG_OFFSET_VO_ACLK_CTRL,
+	  4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_IESMCLK_DIV_DYNM, "divder_u_iesmclk_div_dynm",
+	  "fixed_rate_clk_spll0_fout3", 0, EIC7700_REG_OFFSET_VO_IESMCLK_CTRL,
+	  4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VO_PIXEL_DIV_DYNM, "divder_u_vo_pixel_div_dynm",
+	  "mux_u_vo_pixel_root_2mux1", 0, EIC7700_REG_OFFSET_VO_PIXEL_CTRL, 4,
+	  6, CLK_DIVIDER_ROUND_CLOSEST, u_6_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VO_MCLK_DIV_DYNM, "divder_u_vo_mclk_div_dynm",
+	  "clk_apll_fout1", 0, EIC7700_REG_OFFSET_VO_MCLK_CTRL, 4, 8,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_8_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VO_CEC_DIV_DYNM, "divder_u_vo_cec_div_dynm",
+	  "fixed_rate_clk_vpll_fout2", 0, EIC7700_REG_OFFSET_VO_PHY_CLKCTRL, 16,
+	  16, CLK_DIVIDER_ROUND_CLOSEST, u_16_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VC_ACLK_DIV_DYNM, "divder_u_vc_aclk_div_dynm",
+	  "mux_u_vcaclk_root_2mux1_gfree", 0, EIC7700_REG_OFFSET_VC_ACLK_CTRL,
+	  4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_JD_DIV_DYNM, "divder_u_jd_div_dynm",
+	  "clk_clk_vc_root", 0, EIC7700_REG_OFFSET_JD_CLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_JE_DIV_DYNM, "divder_u_je_div_dynm",
+	  "clk_clk_vc_root", 0, EIC7700_REG_OFFSET_JE_CLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VE_DIV_DYNM, "divder_u_ve_div_dynm",
+	  "clk_clk_vc_root", 0, EIC7700_REG_OFFSET_VE_CLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_VD_DIV_DYNM, "divder_u_vd_div_dynm",
+	  "clk_clk_vc_root", 0, EIC7700_REG_OFFSET_VD_CLK_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_G2D_DIV_DYNM, "divder_u_g2d_div_dynm",
+	  "clk_clk_dsp_root", 0, EIC7700_REG_OFFSET_G2D_CTRL, 4, 4,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_AONDMA_AXI_DIV_DYNM, "divder_u_aondma_axi_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0, EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL,
+	  4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_CRYPTO_DIV_DYNM, "divder_u_crypto_div_dynm",
+	  "fixed_rate_clk_spll0_fout1", 0, EIC7700_REG_OFFSET_SPACC_CLK_CTRL, 4,
+	  4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_SATA_PHY_REF_DIV_DYNM,
+	  "divder_u_sata_phy_ref_div_dynm", "fixed_rate_clk_spll1_fout2", 0,
+	  EIC7700_REG_OFFSET_SATA_OOB_CTRL, 0, 4, CLK_DIVIDER_ROUND_CLOSEST,
+	  u_4_bit_special_div_table },
+
+	{
+		EIC7700_DIVDER_U_DSP_0_ACLK_DIV_DYNM,
+		"divder_u_dsp_0_aclk_div_dynm",
+		"gate_dspt_aclk",
+		0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL,
+		19,
+		1,
+	},
+
+	{
+		EIC7700_DIVDER_U_DSP_1_ACLK_DIV_DYNM,
+		"divder_u_dsp_1_aclk_div_dynm",
+		"gate_dspt_aclk",
+		0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL,
+		20,
+		1,
+	},
+
+	{
+		EIC7700_DIVDER_U_DSP_2_ACLK_DIV_DYNM,
+		"divder_u_dsp_2_aclk_div_dynm",
+		"gate_dspt_aclk",
+		0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL,
+		21,
+		1,
+	},
+
+	{
+		EIC7700_DIVDER_U_DSP_3_ACLK_DIV_DYNM,
+		"divder_u_dsp_3_aclk_div_dynm",
+		"gate_dspt_aclk",
+		0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL,
+		22,
+		1,
+	},
+
+	{ EIC7700_DIVDER_U_AON_RTC_DIV_DYNM, "divder_u_aon_rtc_div_dynm",
+	  "clk_clk_1m", 0, EIC7700_REG_OFFSET_RTC_CLK_CTRL, 21, 11,
+	  CLK_DIVIDER_ROUND_CLOSEST, u_11_bit_special_div_table },
+
+	{ EIC7700_DIVDER_U_U84_RTC_TOGGLE_DIV_DYNM,
+	  "divder_u_u84_rtc_toggle_dynm", "fixed_rate_clk_xtal_24m", 0,
+	  EIC7700_REG_OFFSET_RTC_CLK_CTRL, 16, 5,
+	  CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ROUND_CLOSEST },
+
+};
+
+/*
+ *	these clks should init early to cost down the whole clk module init time
+ */
+
+static struct eswin_clock eic7700_clks_early_0[] = {
+	{
+		EIC7700_CLK_CLK_DSP_ROOT,
+		"clk_clk_dsp_root",
+		"mux_u_dsp_aclk_root_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_VC_ROOT,
+		"clk_clk_vc_root",
+		"mux_u_vcdec_root_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct eswin_clock eic7700_clks_early_1[] = {
+	{
+		EIC7700_CLK_CLK_SYS_CFG,
+		"clk_clk_sys_cfg",
+		"mux_u_syscfg_clk_root_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_D2DDR_ACLK,
+		"clk_clk_d2ddr_aclk",
+		"divder_u_ddr_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_AONDMA_AXI_ST3,
+		"clk_clk_aondma_axi_st3",
+		"mux_u_aondma_axi2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_G2D_ST2,
+		"clk_clk_g2d_st2",
+		"divder_u_g2d_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_MIPI_TXESC,
+		"clk_clk_mipi_txesc",
+		"fixed_factor_u_mipi_txesc_div10",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_VI_ACLK_ST1,
+		"clk_clk_vi_aclk_st1",
+		"divder_u_vi_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct eswin_gate_clock eic7700_gate_clks[] = {
+
+	{
+		EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_0,
+		"gate_clk_cpu_ext_src_core_clk_0",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		28,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_1,
+		"gate_clk_cpu_ext_src_core_clk_1",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		29,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_2,
+		"gate_clk_cpu_ext_src_core_clk_2",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_3,
+		"gate_clk_cpu_ext_src_core_clk_3",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_TRACE_CLK_0,
+		"gate_clk_cpu_trace_clk_0",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		24,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_TRACE_CLK_1,
+		"gate_clk_cpu_trace_clk_1",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		25,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_TRACE_CLK_2,
+		"gate_clk_cpu_trace_clk_2",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		26,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_TRACE_CLK_3,
+		"gate_clk_cpu_trace_clk_3",
+		"mux_u_cpu_root_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		27,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_TRACE_COM_CLK,
+		"gate_clk_cpu_trace_com_clk",
+		"mux_u_cpu_aclk_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		23,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CPU_CLK,
+		"gate_clk_cpu_clk",
+		"mux_u_cpu_aclk_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		28,
+		0,
+	}, /*same as EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_0 */
+
+	{
+		EIC7700_GATE_CLK_SPLL0_FOUT2,
+		"gate_clk_spll0_fout2",
+		"fixed_rate_clk_spll0_fout2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SPLL0_CFG_2,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_NOC_NSP_CLK,
+		"gate_noc_nsp_clk",
+		"divder_u_noc_nsp_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NOC_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_BOOTSPI,
+		"gate_clk_bootspi",
+		"mux_u_bootspi_clk_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_BOOTSPI_CFG,
+		"gate_clk_bootspi_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_BOOTSPI_CFGCLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_SCPU_CORE,
+		"gate_clk_scpu_core",
+		"mux_u_scpu_core_clk_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_SCPU_BUS,
+		"gate_clk_scpu_bus",
+		"fixed_factor_u_scpu_bus_div2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SCPU_BUSCLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_LPCPU_CORE,
+		"gate_clk_lpcpu_core",
+		"mux_u_lpcpu_core_clk_2mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_LPCPU_BUS,
+		"gate_clk_lpcpu_bus",
+		"fixed_factor_u_lpcpu_bus_div2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LPCPU_BUSCLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_GPU_ACLK,
+		"gate_gpu_aclk",
+		"divder_u_gpu_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_GPU_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_GPU_GRAY_CLK,
+		"gate_gpu_gray_clk",
+		"fixed_rate_clk_xtal_24m",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_GPU_GRAY_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_GPU_CFG_CLK,
+		"gate_gpu_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_GPU_CFG_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DSPT_ACLK,
+		"gate_dspt_aclk",
+		"divder_u_dsp_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DSP_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DSPT_CFG_CLK,
+		"gate_dspt_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_D2D_ACLK,
+		"gate_d2d_aclk",
+		"divder_u_d2d_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_D2D_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_D2D_CFG_CLK,
+		"gate_d2d_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_D2D_CFG_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TCU_ACLK,
+		"gate_tcu_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TCU_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TCU_CFG_CLK,
+		"gate_tcu_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TCU_CFG_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT_CFG_CLK,
+		"gate_ddrt_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		9,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT0_P0_ACLK,
+		"gate_ddrt0_p0_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		4,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT0_P1_ACLK,
+		"gate_ddrt0_p1_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		5,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT0_P2_ACLK,
+		"gate_ddrt0_p2_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		6,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT0_P3_ACLK,
+		"gate_ddrt0_p3_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		7,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT0_P4_ACLK,
+		"gate_ddrt0_p4_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		8,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT1_P0_ACLK,
+		"gate_ddrt1_p0_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL,
+		4,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT1_P1_ACLK,
+		"gate_ddrt1_p1_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL,
+		5,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT1_P2_ACLK,
+		"gate_ddrt1_p2_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL,
+		6,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT1_P3_ACLK,
+		"gate_ddrt1_p3_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL,
+		7,
+		0,
+	},
+
+	{
+		EIC7700_GATE_DDRT1_P4_ACLK,
+		"gate_ddrt1_p4_aclk",
+		"clk_clk_d2ddr_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL,
+		8,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_HSP_ACLK,
+		"gate_clk_hsp_aclk",
+		"divder_u_hsp_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_HSP_CFGCLK,
+		"gate_clk_hsp_cfgclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_CFG_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_PCIET_ACLK,
+		"gate_pciet_aclk",
+		"divder_u_pcie_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_PCIET_CFG_CLK,
+		"gate_pciet_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_CFG_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_PCIET_CR_CLK,
+		"gate_pciet_cr_clk",
+		"fixed_factor_u_pcie_cr_div2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_CFG_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_PCIET_AUX_CLK,
+		"gate_pciet_aux_clk",
+		"fixed_factor_u_pcie_aux_div4",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_CFG_CTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_NPU_ACLK,
+		"gate_npu_aclk",
+		"divder_u_npu_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_NPU_CFG_CLK,
+		"gate_npu_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_ACLK_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_NPU_LLC_ACLK,
+		"gate_npu_llc_aclk",
+		"mux_u_npu_llclk_3mux1_gfree",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_LLC_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_NPU_CLK,
+		"gate_npu_clk",
+		"divder_u_npu_coreclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_CORE_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_NPU_E31_CLK,
+		"gate_npu_e31_clk",
+		"divder_u_npu_e31_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_CORE_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_ACLK,
+		"gate_vi_aclk",
+		"clk_clk_vi_aclk_st1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_CFG_CLK,
+		"gate_vi_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_ACLK_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_DIG_DW_CLK,
+		"gate_vi_dig_dw_clk",
+		"divder_u_vi_dw_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DWCLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_DVP_CLK,
+		"gate_vi_dvp_clk",
+		"divder_u_vi_dvp_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_DIG_ISP_CLK,
+		"gate_vi_dig_isp_clk",
+		"divder_u_vi_dig_isp_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_SHUTTER_0,
+		"gate_vi_shutter_0",
+		"divder_u_vi_shutter_div_dynm_0",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER0,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_SHUTTER_1,
+		"gate_vi_shutter_1",
+		"divder_u_vi_shutter_div_dynm_1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER1,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_SHUTTER_2,
+		"gate_vi_shutter_2",
+		"divder_u_vi_shutter_div_dynm_2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER2,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_SHUTTER_3,
+		"gate_vi_shutter_3",
+		"divder_u_vi_shutter_div_dynm_3",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER3,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_SHUTTER_4,
+		"gate_vi_shutter_4",
+		"divder_u_vi_shutter_div_dynm_4",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER4,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_SHUTTER_5,
+		"gate_vi_shutter_5",
+		"divder_u_vi_shutter_div_dynm_5",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER5,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_PHY_TXCLKESC,
+		"gate_vi_phy_txclkesc",
+		"clk_clk_mipi_txesc",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_PHY_CLKCTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VI_PHY_CFG,
+		"gate_vi_phy_cfg",
+		"fixed_rate_clk_xtal_24m",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_PHY_CLKCTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VO_ACLK,
+		"gate_vo_aclk",
+		"divder_u_vo_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VO_CFG_CLK,
+		"gate_vo_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_ACLK_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VO_HDMI_IESMCLK,
+		"gate_vo_hdmi_iesmclk",
+		"divder_u_iesmclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_IESMCLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VO_PIXEL_CLK,
+		"gate_vo_pixel_clk",
+		"divder_u_vo_pixel_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_PIXEL_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VO_I2S_MCLK,
+		"gate_vo_i2s_mclk",
+		"mux_u_vo_mclk_2mux_ext_mclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_MCLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VO_CR_CLK,
+		"gate_vo_cr_clk",
+		"clk_clk_mipi_txesc",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_PHY_CLKCTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_ACLK,
+		"gate_vc_aclk",
+		"divder_u_vc_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_ACLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_CFG_CLK,
+		"gate_vc_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_JE_CLK,
+		"gate_vc_je_clk",
+		"divder_u_je_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_JE_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_JD_CLK,
+		"gate_vc_jd_clk",
+		"divder_u_jd_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_JD_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_VE_CLK,
+		"gate_vc_ve_clk",
+		"divder_u_ve_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VE_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_VD_CLK,
+		"gate_vc_vd_clk",
+		"divder_u_vd_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VD_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_G2D_CFG_CLK,
+		"gate_g2d_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_G2D_CTRL,
+		28,
+		0,
+	},
+
+	{
+		EIC7700_GATE_G2D_CLK,
+		"gate_g2d_clk",
+		"clk_clk_g2d_st2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_G2D_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_G2D_ACLK,
+		"gate_g2d_aclk",
+		"clk_clk_g2d_st2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_G2D_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_AONDMA_CFG,
+		"gate_clk_aondma_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_AONDMA_ACLK,
+		"gate_aondma_aclk",
+		"clk_clk_aondma_axi_st3",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_AON_ACLK,
+		"gate_aon_aclk",
+		"clk_clk_aondma_axi_st3",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL,
+		29,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_CLK_0,
+		"gate_time_clk_0",
+		"fixed_rate_clk_xtal_24m",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_CLK_1,
+		"gate_time_clk_1",
+		"fixed_rate_clk_xtal_24m",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_CLK_2,
+		"gate_time_clk_2",
+		"fixed_rate_clk_xtal_24m",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_CLK_3,
+		"gate_time_clk_3",
+		"fixed_rate_clk_xtal_24m",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		3,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_PCLK_0,
+		"gate_timer_pclk_0",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		4,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_PCLK_1,
+		"gate_timer_pclk_1",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		5,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_PCLK_2,
+		"gate_timer_pclk_2",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		6,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER_PCLK_3,
+		"gate_timer_pclk_3",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		7,
+		0,
+	},
+
+	{
+		EIC7700_GATE_TIMER3_CLK8,
+		"gate_timer3_clk8",
+		"fixed_rate_clk_vpll_fout3",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL,
+		8,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_RTC_CFG,
+		"gate_clk_rtc_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_RTC_CLK_CTRL,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_RTC,
+		"gate_clk_rtc",
+		"divder_u_aon_rtc_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_RTC_CLK_CTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_PKA_CFG,
+		"gate_clk_pka_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PKA_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_SPACC_CFG,
+		"gate_clk_spacc_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SPACC_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_CRYPTO,
+		"gate_clk_crypto",
+		"divder_u_crypto_div_dynm",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SPACC_CLK_CTRL,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_TRNG_CFG,
+		"gate_clk_trng_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TRNG_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_OTP_CFG,
+		"gate_clk_otp_cfg",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_OTP_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_0,
+		"gate_clk_mailbox_0",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_1,
+		"gate_clk_mailbox_1",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_2,
+		"gate_clk_mailbox_2",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_3,
+		"gate_clk_mailbox_3",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		3,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_4,
+		"gate_clk_mailbox_4",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		4,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_5,
+		"gate_clk_mailbox_5",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		5,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_6,
+		"gate_clk_mailbox_6",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		6,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_7,
+		"gate_clk_mailbox_7",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		7,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_8,
+		"gate_clk_mailbox_8",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		8,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_9,
+		"gate_clk_mailbox_9",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		9,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_10,
+		"gate_clk_mailbox_10",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		10,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_11,
+		"gate_clk_mailbox_11",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		11,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_12,
+		"gate_clk_mailbox_12",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		12,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_13,
+		"gate_clk_mailbox_13",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		13,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_14,
+		"gate_clk_mailbox_14",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		14,
+		0,
+	},
+
+	{
+		EIC7700_GATE_CLK_MAILBOX_15,
+		"gate_clk_mailbox_15",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		15,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C0_PCLK,
+		"gate_i2c0_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		7,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C1_PCLK,
+		"gate_i2c1_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		8,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C2_PCLK,
+		"gate_i2c2_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		9,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C3_PCLK,
+		"gate_i2c3_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		10,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C4_PCLK,
+		"gate_i2c4_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		11,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C5_PCLK,
+		"gate_i2c5_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		12,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C6_PCLK,
+		"gate_i2c6_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		13,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C7_PCLK,
+		"gate_i2c7_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		14,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C8_PCLK,
+		"gate_i2c8_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		15,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_I2C9_PCLK,
+		"gate_i2c9_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		16,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_WDT0_PCLK,
+		"gate_lsp_wdt0_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		28,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_WDT1_PCLK,
+		"gate_lsp_wdt1_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		29,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_WDT2_PCLK,
+		"gate_lsp_wdt2_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		30,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_WDT3_PCLK,
+		"gate_lsp_wdt3_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_SSI0_PCLK,
+		"gate_lsp_ssi0_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		26,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_SSI1_PCLK,
+		"gate_lsp_ssi1_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		27,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_UART0_PCLK,
+		"gate_lsp_uart0_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		17,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_UART1_PCLK,
+		"gate_lsp_uart1_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		18,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_UART2_PCLK,
+		"gate_lsp_uart2_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		19,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_UART3_PCLK,
+		"gate_lsp_uart3_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		20,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_UART4_PCLK,
+		"gate_lsp_uart4_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		21,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_TIMER_PCLK,
+		"gate_lsp_timer_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		25,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_FAN_PCLK,
+		"gate_lsp_fan_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_PVT_PCLK,
+		"gate_lsp_pvt_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_PVT0_CLK,
+		"gate_pvt0_clk",
+		"fixed_factor_u_pvt_div20",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		16,
+		0,
+	},
+
+	{
+		EIC7700_GATE_LSP_PVT1_CLK,
+		"gate_pvt1_clk",
+		"fixed_factor_u_pvt_div20",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1,
+		17,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_JE_PCLK,
+		"gate_vc_je_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL,
+		2,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_JD_PCLK,
+		"gate_vc_jd_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_VE_PCLK,
+		"gate_vc_ve_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL,
+		5,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_VD_PCLK,
+		"gate_vc_vd_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL,
+		4,
+		0,
+	},
+
+	{
+		EIC7700_GATE_VC_MON_PCLK,
+		"gate_vc_mon_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL,
+		3,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_MSHC0_CORE_CLK,
+		"gate_hsp_mshc0_core_clk",
+		"divder_u_mshc_core_div_dynm_0",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL,
+		16,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_MSHC1_CORE_CLK,
+		"gate_hsp_mshc1_core_clk",
+		"divder_u_mshc_core_div_dynm_1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL,
+		16,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_MSHC2_CORE_CLK,
+		"gate_hsp_mshc2_core_clk",
+		"divder_u_mshc_core_div_dynm_2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL,
+		16,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_SATA_RBC_CLK,
+		"gate_hsp_sata_rbc_clk",
+		"fixed_rate_clk_spll1_fout2",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SATA_RBC_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_SATA_OOB_CLK,
+		"gate_hsp_sata_oob_clk",
+		"mux_u_sata_phy_2mux1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SATA_OOB_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_DMA0_CLK_TEST,
+		"gate_hsp_dma0_clk_test",
+		"gate_clk_hsp_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_ACLK_CTRL,
+		1,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_DMA0_CLK,
+		"gate_hsp_dma0_clk",
+		"gate_clk_hsp_aclk",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_ACLK_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_ETH0_CORE_CLK,
+		"gate_hsp_eth0_core_clk",
+		"divder_u_eth_txclk_div_dynm_0",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH0_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_ETH1_CORE_CLK,
+		"gate_hsp_eth1_core_clk",
+		"divder_u_eth_txclk_div_dynm_1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH1_CTRL,
+		0,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_RMII_REF_0,
+		"gate_hsp_rmii_ref_0",
+		"mux_u_rmii_ref_2mux1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH0_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_HSP_RMII_REF_1,
+		"gate_hsp_rmii_ref_1",
+		"mux_u_rmii_ref_2mux1",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH1_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_AON_I2C0_PCLK,
+		"gate_aon_i2c0_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_I2C0_CLK_CTRL,
+		31,
+		0,
+	},
+
+	{
+		EIC7700_GATE_AON_I2C1_PCLK,
+		"gate_aon_i2c1_pclk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_I2C1_CLK_CTRL,
+		31,
+		0,
+	},
+};
+
+static struct eswin_clock eic7700_clks[] = {
+	{
+		EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_0,
+		"clk_cpu_ext_src_core_clk_0",
+		"gate_clk_cpu_ext_src_core_clk_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_1,
+		"clk_cpu_ext_src_core_clk_1",
+		"gate_clk_cpu_ext_src_core_clk_1",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_2,
+		"clk_cpu_ext_src_core_clk_2",
+		"gate_clk_cpu_ext_src_core_clk_2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_3,
+		"clk_cpu_ext_src_core_clk_3",
+		"gate_clk_cpu_ext_src_core_clk_3",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_TRACE_CLK_0,
+		"clk_cpu_trace_clk_0",
+		"gate_clk_cpu_trace_clk_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_TRACE_CLK_1,
+		"clk_cpu_trace_clk_1",
+		"gate_clk_cpu_trace_clk_1",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_TRACE_CLK_2,
+		"clk_cpu_trace_clk_2",
+		"gate_clk_cpu_trace_clk_2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_TRACE_CLK_3,
+		"clk_cpu_trace_clk_3",
+		"gate_clk_cpu_trace_clk_3",
+		CLK_SET_RATE_PARENT,
+	},
+
+	{
+		EIC7700_CLK_CPU_TRACE_COM_CLK,
+		"clk_cpu_trace_com_clk",
+		"gate_clk_cpu_trace_com_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CPU_CLK,
+		"clk_cpu_clk",
+		"gate_clk_cpu_clk",
+		CLK_SET_RATE_PARENT,
+	},
+
+	{
+		EIC7700_CLK_CLK_1M,
+		"clk_clk_1m",
+		"fixed_factor_u_clk_1m_div24",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NOC_CFG_CLK,
+		"clk_noc_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NOC_NSP_CLK,
+		"clk_noc_nsp_clk",
+		"gate_noc_nsp_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_BOOTSPI,
+		"clk_clk_bootspi",
+		"gate_clk_bootspi",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_BOOTSPI_CFG,
+		"clk_clk_bootspi_cfg",
+		"gate_clk_bootspi_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_U84_CORE_LP,
+		"clk_clk_u84_core_lp",
+		"fixed_factor_u_u84_core_lp_div2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_SCPU_CORE,
+		"clk_clk_scpu_core",
+		"gate_clk_scpu_core",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_SCPU_BUS,
+		"clk_clk_scpu_bus",
+		"gate_clk_scpu_bus",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_LPCPU_CORE,
+		"clk_clk_lpcpu_core",
+		"gate_clk_lpcpu_core",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_LPCPU_BUS,
+		"clk_clk_lpcpu_bus",
+		"gate_clk_lpcpu_bus",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_GPU_ACLK,
+		"clk_gpu_aclk",
+		"gate_gpu_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_GPU_GRAY_CLK,
+		"clk_gpu_gray_clk",
+		"gate_gpu_gray_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_GPU_CFG_CLK,
+		"clk_gpu_cfg_clk",
+		"gate_gpu_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DSPT_ACLK,
+		"clk_dspt_aclk",
+		"gate_dspt_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DSPT_CFG_CLK,
+		"clk_dspt_cfg_clk",
+		"gate_dspt_cfg_clk",
+		CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE,
+	},
+	{
+		EIC7700_CLK_D2D_ACLK,
+		"clk_d2d_aclk",
+		"gate_d2d_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_D2D_CFG_CLK,
+		"clk_d2d_cfg_clk",
+		"gate_d2d_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TCU_ACLK,
+		"clk_tcu_aclk",
+		"gate_tcu_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TCU_CFG_CLK,
+		"clk_tcu_cfg_clk",
+		"gate_tcu_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT_CFG_CLK,
+		"clk_ddrt_cfg_clk",
+		"gate_ddrt_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT0_P0_ACLK,
+		"clk_ddrt0_p0_aclk",
+		"gate_ddrt0_p0_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT0_P1_ACLK,
+		"clk_ddrt0_p1_aclk",
+		"gate_ddrt0_p1_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT0_P2_ACLK,
+		"clk_ddrt0_p2_aclk",
+		"gate_ddrt0_p2_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT0_P3_ACLK,
+		"clk_ddrt0_p3_aclk",
+		"gate_ddrt0_p3_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT0_P4_ACLK,
+		"clk_ddrt0_p4_aclk",
+		"gate_ddrt0_p4_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT1_P0_ACLK,
+		"clk_ddrt1_p0_aclk",
+		"gate_ddrt1_p0_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT1_P1_ACLK,
+		"clk_ddrt1_p1_aclk",
+		"gate_ddrt1_p1_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT1_P2_ACLK,
+		"clk_ddrt1_p2_aclk",
+		"gate_ddrt1_p2_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT1_P3_ACLK,
+		"clk_ddrt1_p3_aclk",
+		"gate_ddrt1_p3_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DDRT1_P4_ACLK,
+		"clk_ddrt1_p4_aclk",
+		"gate_ddrt1_p4_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_ACLK,
+		"clk_hsp_aclk",
+		"gate_clk_hsp_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_CFG_CLK,
+		"clk_hsp_cfg_clk",
+		"gate_clk_hsp_cfgclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_SATA_RBC_CLK,
+		"clk_hsp_sata_rbc_clk",
+		"gate_hsp_sata_rbc_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_SATA_OOB_CLK,
+		"clk_hsp_sata_oob_clk",
+		"gate_hsp_sata_oob_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_SATA_PHY_REF,
+		"clk_hsp_sata_phy_ref",
+		"gate_hsp_sata_oob_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_SATA_PMALIVE_CLK,
+		"clk_hsp_sata_pmalive_clk",
+		"gate_hsp_sata_oob_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_ETH_APP_CLK,
+		"clk_hsp_eth_app_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_ETH_CSR_CLK,
+		"clk_hsp_eth_csr_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_ETH0_CORE_CLK,
+		"clk_hsp_eth0_core_clk",
+		"gate_hsp_eth0_core_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_ETH1_CORE_CLK,
+		"clk_hsp_eth1_core_clk",
+		"gate_hsp_eth1_core_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_MSHC0_CORE_CLK,
+		"clk_hsp_mshc0_core_clk",
+		"gate_hsp_mshc0_core_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_MSHC1_CORE_CLK,
+		"clk_hsp_mshc1_core_clk",
+		"gate_hsp_mshc1_core_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_MSHC2_CORE_CLK,
+		"clk_hsp_mshc2_core_clk",
+		"gate_hsp_mshc2_core_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_MSHC0_TMR_CLK,
+		"clk_hsp_mshc0_tmr_clk",
+		"clk_clk_1m",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_MSHC1_TMR_CLK,
+		"clk_hsp_mshc1_tmr_clk",
+		"clk_clk_1m",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_MSHC2_TMR_CLK,
+		"clk_hsp_mshc2_tmr_clk",
+		"clk_clk_1m",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_USB0_SUSPEND_CLK,
+		"clk_hsp_usb0_suspend_clk",
+		"clk_clk_1m",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_USB1_SUSPEND_CLK,
+		"clk_hsp_usb1_suspend_clk",
+		"clk_clk_1m",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_PCIET_ACLK,
+		"clk_pciet_aclk",
+		"gate_pciet_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_PCIET_CFG_CLK,
+		"clk_pciet_cfg_clk",
+		"gate_pciet_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_PCIET_CR_CLK,
+		"clk_pciet_cr_clk",
+		"gate_pciet_cr_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_PCIET_AUX_CLK,
+		"clk_pciet_aux_clk",
+		"gate_pciet_aux_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NPU_ACLK,
+		"clk_npu_aclk",
+		"gate_npu_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NPU_CFG_CLK,
+		"clk_npu_cfg_clk",
+		"gate_npu_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_NPU_LLC_SRC0,
+		"clk_clk_npu_llc_src0",
+		"divder_u_npu_llc_src0_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_NPU_LLC_SRC1,
+		"clk_clk_npu_llc_src1",
+		"divder_u_npu_llc_src1_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NPU_LLC_ACLK,
+		"clk_npu_llc_aclk",
+		"gate_npu_llc_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NPU_CLK,
+		"clk_npu_clk",
+		"gate_npu_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_NPU_E31_CLK,
+		"clk_npu_e31_clk",
+		"gate_npu_e31_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_ACLK,
+		"clk_vi_aclk",
+		"gate_vi_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_DIG_DW_CLK,
+		"clk_vi_dig_dw_clk",
+		"gate_vi_dig_dw_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_CFG_CLK,
+		"clk_vi_cfg_clk",
+		"gate_vi_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_DVP_CLK,
+		"clk_vi_dvp_clk",
+		"gate_vi_dvp_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_DIG_ISP_CLK,
+		"clk_vi_dig_isp_clk",
+		"gate_vi_dig_isp_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_SHUTTER_0,
+		"clk_vi_shutter_0",
+		"gate_vi_shutter_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_SHUTTER_1,
+		"clk_vi_shutter_1",
+		"gate_vi_shutter_1",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_SHUTTER_2,
+		"clk_vi_shutter_2",
+		"gate_vi_shutter_2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_SHUTTER_3,
+		"clk_vi_shutter_3",
+		"gate_vi_shutter_3",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_SHUTTER_4,
+		"clk_vi_shutter_4",
+		"gate_vi_shutter_4",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_SHUTTER_5,
+		"clk_vi_shutter_5",
+		"gate_vi_shutter_5",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_PHY_TXCLKESC,
+		"clk_vi_phy_txclkesc",
+		"gate_vi_phy_txclkesc",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VI_PHY_CFG,
+		"clk_vi_phy_cfg",
+		"gate_vi_phy_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_ACLK,
+		"clk_vo_aclk",
+		"gate_vo_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_CFG_CLK,
+		"clk_vo_cfg_clk",
+		"gate_vo_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_HDMI_IESMCLK,
+		"clk_vo_hdmi_iesmclk",
+		"gate_vo_hdmi_iesmclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_PIXEL_CLK,
+		"clk_vo_pixel_clk",
+		"gate_vo_pixel_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_I2S_MCLK,
+		"clk_vo_i2s_mclk",
+		"gate_vo_i2s_mclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_CR_CLK,
+		"clk_vo_cr_clk",
+		"gate_vo_cr_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VO_CEC_CLK,
+		"clk_vo_cec_clk",
+		"divder_u_vo_cec_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_ACLK,
+		"clk_vc_aclk",
+		"gate_vc_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_CFG_CLK,
+		"clk_vc_cfg_clk",
+		"gate_vc_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_JE_CLK,
+		"clk_vc_je_clk",
+		"gate_vc_je_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_JD_CLK,
+		"clk_vc_jd_clk",
+		"gate_vc_jd_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_VE_CLK,
+		"clk_vc_ve_clk",
+		"gate_vc_ve_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_VD_CLK,
+		"clk_vc_vd_clk",
+		"gate_vc_vd_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_G2D_CFG_CLK,
+		"clk_g2d_cfg_clk",
+		"gate_g2d_cfg_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_G2D_CLK,
+		"clk_g2d_clk",
+		"gate_g2d_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_G2D_ACLK,
+		"clk_g2d_aclk",
+		"gate_g2d_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_AONDMA_CFG,
+		"clk_aondma_cfg",
+		"gate_clk_aondma_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_AONDMA_ACLK,
+		"clk_aondma_aclk",
+		"gate_aondma_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_AON_ACLK,
+		"clk_aon_aclk",
+		"gate_aon_aclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_CLK_0,
+		"clk_timer_clk_0",
+		"gate_time_clk_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_CLK_1,
+		"clk_timer_clk_1",
+		"gate_time_clk_1",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_CLK_2,
+		"clk_timer_clk_2",
+		"gate_time_clk_2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_CLK_3,
+		"clk_timer_clk_3",
+		"gate_time_clk_3",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_PCLK_0,
+		"clk_timer_pclk_0",
+		"gate_timer_pclk_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_PCLK_1,
+		"clk_timer_pclk_1",
+		"gate_timer_pclk_1",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_PCLK_2,
+		"clk_timer_pclk_2",
+		"gate_timer_pclk_2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER_PCLK_3,
+		"clk_timer_pclk_3",
+		"gate_timer_pclk_3",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_TIMER3_CLK8,
+		"clk_timer3_clk8",
+		"gate_timer3_clk8",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_RTC_CFG,
+		"clk_clk_rtc_cfg",
+		"gate_clk_rtc_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_RTC,
+		"clk_clk_rtc",
+		"gate_clk_rtc",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_PKA_CFG,
+		"clk_clk_pka_cfg",
+		"gate_clk_pka_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_SPACC_CFG,
+		"clk_clk_spacc_cfg",
+		"gate_clk_spacc_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_CRYPTO,
+		"clk_clk_crypto",
+		"gate_clk_crypto",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_TRNG_CFG,
+		"clk_clk_trng_cfg",
+		"gate_clk_trng_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_OTP_CFG,
+		"clk_clk_otp_cfg",
+		"gate_clk_otp_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLMM_CFG_CLK,
+		"clk_clmm_cfg_clk",
+		"clk_clk_sys_cfg",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLMM_DEB_CLK,
+		"clk_clmm_deb_clk",
+		"clk_clk_1m",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_0,
+		"clk_mailbox0",
+		"gate_clk_mailbox_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_1,
+		"clk_mailbox1",
+		"gate_clk_mailbox_1",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_2,
+		"clk_mailbox2",
+		"gate_clk_mailbox_2",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_3,
+		"clk_mailbox3",
+		"gate_clk_mailbox_3",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_4,
+		"clk_mailbox4",
+		"gate_clk_mailbox_4",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_5,
+		"clk_mailbox5",
+		"gate_clk_mailbox_5",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_6,
+		"clk_mailbox6",
+		"gate_clk_mailbox_6",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_7,
+		"clk_mailbox7",
+		"gate_clk_mailbox_7",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_8,
+		"clk_mailbox8",
+		"gate_clk_mailbox_8",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_9,
+		"clk_mailbox9",
+		"gate_clk_mailbox_9",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_10,
+		"clk_mailbox10",
+		"gate_clk_mailbox_10",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_11,
+		"clk_mailbox11",
+		"gate_clk_mailbox_11",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_12,
+		"clk_mailbox12",
+		"gate_clk_mailbox_12",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_13,
+		"clk_mailbox13",
+		"gate_clk_mailbox_13",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_14,
+		"clk_mailbox14",
+		"gate_clk_mailbox_14",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_MAILBOX_15,
+		"clk_mailbox15",
+		"gate_clk_mailbox_15",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C0_PCLK,
+		"clk_i2c0_pclk",
+		"gate_i2c0_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C1_PCLK,
+		"clk_i2c1_pclk",
+		"gate_i2c1_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C2_PCLK,
+		"clk_i2c2_pclk",
+		"gate_i2c2_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C3_PCLK,
+		"clk_i2c3_pclk",
+		"gate_i2c3_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C4_PCLK,
+		"clk_i2c4_pclk",
+		"gate_i2c4_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C5_PCLK,
+		"clk_i2c5_pclk",
+		"gate_i2c5_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C6_PCLK,
+		"clk_i2c6_pclk",
+		"gate_i2c6_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C7_PCLK,
+		"clk_i2c7_pclk",
+		"gate_i2c7_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C8_PCLK,
+		"clk_i2c8_pclk",
+		"gate_i2c8_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_I2C9_PCLK,
+		"clk_i2c9_pclk",
+		"gate_i2c9_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+
+	{
+		EIC7700_CLK_LSP_WDT0_PCLK,
+		"clk_lsp_wdt0_pclk",
+		"gate_lsp_wdt0_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_WDT1_PCLK,
+		"clk_lsp_wdt1_pclk",
+		"gate_lsp_wdt1_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_WDT2_PCLK,
+		"clk_lsp_wdt2_pclk",
+		"gate_lsp_wdt2_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_WDT3_PCLK,
+		"clk_lsp_wdt3_pclk",
+		"gate_lsp_wdt3_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_SSI0_PCLK,
+		"clk_lsp_ssi0_pclk",
+		"gate_lsp_ssi0_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_SSI1_PCLK,
+		"clk_lsp_ssi1_pclk",
+		"gate_lsp_ssi1_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_UART0_PCLK,
+		"clk_lsp_uart0_pclk",
+		"gate_lsp_uart0_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_UART1_PCLK,
+		"clk_lsp_uart1_pclk",
+		"gate_lsp_uart1_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_UART2_PCLK,
+		"clk_lsp_uart2_pclk",
+		"gate_lsp_uart2_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_UART3_PCLK,
+		"clk_lsp_uart3_pclk",
+		"gate_lsp_uart3_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_UART4_PCLK,
+		"clk_lsp_uart4_pclk",
+		"gate_lsp_uart4_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_TIMER_PCLK,
+		"clk_lsp_timer_pclk",
+		"gate_lsp_timer_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_FAN_PCLK,
+		"clk_lsp_fan_pclk",
+		"gate_lsp_fan_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_LSP_PVT_PCLK,
+		"clk_lsp_pvt_pclk",
+		"gate_lsp_pvt_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_PVT_CLK_0,
+		"clk_pvt0_clk",
+		"gate_pvt0_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_PVT_CLK_1,
+		"clk_pvt1_clk",
+		"gate_pvt1_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_JE_PCLK,
+		"clk_vc_je_pclk",
+		"gate_vc_je_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_JD_PCLK,
+		"clk_vc_jd_pclk",
+		"gate_vc_jd_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_VE_PCLK,
+		"clk_vc_ve_pclk",
+		"gate_vc_ve_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_VD_PCLK,
+		"clk_vc_vd_pclk",
+		"gate_vc_vd_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_VC_MON_PCLK,
+		"clk_vc_mon_pclk",
+		"gate_vc_mon_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+
+	{
+		EIC7700_CLK_HSP_DMA0_CLK,
+		"clk_hsp_dma0_clk",
+		"gate_hsp_dma0_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_DMA0_CLK_TEST,
+		"clk_hsp_dma0_clk_TEST",
+		"gate_hsp_dma0_clk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_RMII_REF_0,
+		"clk_hsp_rmii_ref_0",
+		"gate_hsp_rmii_ref_0",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_HSP_RMII_REF_1,
+		"clk_hsp_rmii_ref_1",
+		"gate_hsp_rmii_ref_1",
+		CLK_SET_RATE_PARENT,
+	},
+
+	{
+		EIC7700_CLK_DSP_ACLK_0,
+		"clk_dsp_aclk_0",
+		"divder_u_dsp_0_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DSP_ACLK_1,
+		"clk_dsp_aclk_1",
+		"divder_u_dsp_1_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DSP_ACLK_2,
+		"clk_dsp_aclk_2",
+		"divder_u_dsp_2_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_DSP_ACLK_3,
+		"clk_dsp_aclk_3",
+		"divder_u_dsp_3_aclk_div_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_CLK_U84_RTC_TOGGLE,
+		"clk_clk_u84_rtc_toggle",
+		"divder_u_u84_rtc_toggle_dynm",
+		CLK_SET_RATE_PARENT,
+	},
+
+	{
+		EIC7700_CLK_AON_I2C0_PCLK,
+		"clk_aon_i2c0_pclk",
+		"gate_aon_i2c0_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		EIC7700_CLK_AON_I2C1_PCLK,
+		"clk_aon_i2c1_pclk",
+		"gate_aon_i2c1_pclk",
+		CLK_SET_RATE_PARENT,
+	},
+};
+
+static void special_div_table_init(struct clk_div_table *table, int table_size)
+{
+	int i;
+
+	if (table_size < 3)
+		return;
+
+	if (!table)
+		return;
+
+	/*The hardware decides value 0, 1 and 2 both means 2 divsor*/
+	for (i = 0; i < 3; i++) {
+		table[i].val = i;
+		table[i].div = 2;
+	}
+	for (i = 3; i < table_size - 1; i++) {
+		table[i].val = i;
+		table[i].div = i;
+	}
+	table[table_size - 1].val = 0;
+	table[table_size - 1].div = 0;
+}
+
+static int eswin_cpu_clk_init(struct platform_device *pdev)
+{
+	struct clk *cpu_clk;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	u32 default_freq;
+	int ret = 0;
+	int numa_id;
+	char name[128] = { 0 };
+
+	ret = of_property_read_u32(np, "cpu-default-frequency", &default_freq);
+	if (ret) {
+		dev_info(dev, "cpu-default-frequency not set\n");
+		return ret;
+	}
+	numa_id = dev_to_node(dev->parent);
+	if (numa_id < 0)
+		sprintf(name, "%s", "clk_cpu_ext_src_core_clk_0");
+	else
+		sprintf(name, "d%d_%s", numa_id, "clk_cpu_ext_src_core_clk_0");
+
+	cpu_clk = __clk_lookup(name);
+	if (!cpu_clk) {
+		dev_err(dev, "Failed to lookup CPU clock\n");
+		return -EINVAL;
+	}
+	ret = clk_set_rate(cpu_clk, default_freq);
+	if (ret) {
+		dev_err(dev, "Failed to set CPU frequency: %d\n", ret);
+		return ret;
+	}
+	dev_info(dev, "CPU frequency set to %u Hz\n", default_freq);
+	return 0;
+}
+
+static int eswin_clk_probe(struct platform_device *pdev)
+{
+	struct eswin_clock_data *clk_data;
+
+	clk_data = eswin_clk_init(pdev, EIC7700_NR_CLKS);
+	if (!clk_data)
+		return -EAGAIN;
+
+	special_div_table_init(u_3_bit_special_div_table,
+			       ARRAY_SIZE(u_3_bit_special_div_table));
+	special_div_table_init(u_4_bit_special_div_table,
+			       ARRAY_SIZE(u_4_bit_special_div_table));
+	special_div_table_init(u_6_bit_special_div_table,
+			       ARRAY_SIZE(u_6_bit_special_div_table));
+	special_div_table_init(u_7_bit_special_div_table,
+			       ARRAY_SIZE(u_7_bit_special_div_table));
+	special_div_table_init(u_8_bit_special_div_table,
+			       ARRAY_SIZE(u_8_bit_special_div_table));
+	special_div_table_init(u_11_bit_special_div_table,
+			       ARRAY_SIZE(u_11_bit_special_div_table));
+	special_div_table_init(u_16_bit_special_div_table,
+			       ARRAY_SIZE(u_16_bit_special_div_table));
+
+	eswin_clk_register_fixed_rate(eic7700_fixed_rate_clks,
+				      ARRAY_SIZE(eic7700_fixed_rate_clks),
+				      clk_data);
+	eswin_clk_register_pll(eic7700_pll_clks, ARRAY_SIZE(eic7700_pll_clks),
+			       clk_data, &pdev->dev);
+
+	eswin_clk_register_fixed_factor(eic7700_fixed_factor_clks,
+					ARRAY_SIZE(eic7700_fixed_factor_clks),
+					clk_data);
+	eswin_clk_register_mux(eic7700_mux_clks, ARRAY_SIZE(eic7700_mux_clks),
+			       clk_data);
+	eswin_clk_register_clk(eic7700_clks_early_0,
+			       ARRAY_SIZE(eic7700_clks_early_0), clk_data);
+	eswin_clk_register_divider(eic7700_div_clks,
+				   ARRAY_SIZE(eic7700_div_clks), clk_data);
+	eswin_clk_register_clk(eic7700_clks_early_1,
+			       ARRAY_SIZE(eic7700_clks_early_1), clk_data);
+	eswin_clk_register_gate(eic7700_gate_clks,
+				ARRAY_SIZE(eic7700_gate_clks), clk_data);
+	eswin_clk_register_clk(eic7700_clks, ARRAY_SIZE(eic7700_clks),
+			       clk_data);
+
+	eswin_cpu_clk_init(pdev);
+
+	return 0;
+}
+
+static const struct of_device_id eswin_clock_dt_ids[] = {
+	{
+		.compatible = "eswin,eic7700-clock",
+	},
+	{ /* sentinel */ },
+};
+
+static struct platform_driver eswin_clock_driver = {
+	.probe	= eswin_clk_probe,
+	.driver = {
+		.name	= "eswin-clock",
+		.of_match_table	= eswin_clock_dt_ids,
+	},
+};
+module_platform_driver(eswin_clock_driver);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("huangyifeng<huangyifeng@eswincomputing.com>");
+MODULE_DESCRIPTION("Eswin EIC7700 clock controller driver");
diff --git a/drivers/clk/eswin/clk.c b/drivers/clk/eswin/clk.c
new file mode 100644
index 000000000000..f5284cb1b611
--- /dev/null
+++ b/drivers/clk/eswin/clk.c
@@ -0,0 +1,1093 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: huangyifeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/util_macros.h>
+#include <linux/gpio/consumer.h>
+#include <dt-bindings/clock/eswin,eic7700-clock.h>
+#include "clk.h"
+
+#define NUMA_NODE_NUMS 2
+
+struct cpu_info {
+	bool cpu_no_boost_1_6ghz;
+	uint64_t cpu_freqhz[NUMA_NODE_NUMS];
+};
+struct cpu_info g_cpu_info = {.cpu_no_boost_1_6ghz = true,
+								.cpu_freqhz = {0}};
+static DEFINE_MUTEX(lock);
+
+struct clk_hw *eswin_clk_find_parent(struct eswin_clock_data *data,
+				     char *parent_name)
+{
+	int i;
+	struct clk *clks;
+
+	for (i = 0; i < data->clk_data.clk_num; i++) {
+		clks = data->clk_data.clks[i];
+		if (clks == NULL)
+			continue;
+
+		if (!strcmp(__clk_get_name(clks), parent_name))
+			return __clk_get_hw(clks);
+	}
+	return NULL;
+}
+
+struct eswin_clock_data *eswin_clk_init(struct platform_device *pdev,
+					int nr_clks)
+{
+	struct eswin_clock_data *clk_data;
+	struct clk **clk_table;
+	void __iomem *base;
+	struct device *parent;
+
+	parent = pdev->dev.parent;
+	if (!parent) {
+		dev_err(&pdev->dev, "no parent\n");
+		goto err;
+	}
+
+	base = of_iomap(parent->of_node, 0);
+	if (!base) {
+		dev_err(&pdev->dev, "failed to map clock registers\n");
+		goto err;
+	}
+	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
+	if (!clk_data)
+		goto err;
+
+	clk_data->base = base;
+	clk_table = kcalloc(nr_clks, sizeof(*clk_table), GFP_KERNEL);
+	if (!clk_table)
+		goto err_data;
+
+	clk_data->clk_data.clks = clk_table;
+	clk_data->clk_data.clk_num = nr_clks;
+	clk_data->numa_id = dev_to_node(parent);
+	spin_lock_init(&clk_data->lock);
+
+	of_clk_add_provider(pdev->dev.of_node, of_clk_src_onecell_get,
+			    &clk_data->clk_data);
+	return clk_data;
+
+err_data:
+	kfree(clk_data);
+err:
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(eswin_clk_init);
+
+int eswin_clk_register_fixed_rate(const struct eswin_fixed_rate_clock *clks,
+				  int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+		if (data->numa_id < 0)
+			sprintf(name, "%s", clks[i].name);
+		else
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+
+		clk = clk_register_fixed_rate(NULL, name, clks[i].parent_name,
+					      clks[i].flags,
+					      clks[i].fixed_rate);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__,
+			       name);
+			kfree(name);
+			goto err;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+	}
+
+	return 0;
+err:
+	while (i--)
+		clk_unregister_fixed_rate(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_fixed_rate);
+
+static int eswin_clk_set_cpu_voltage(struct gpio_desc *cpu_voltage_gpio,
+				     enum voltage_level target_voltage)
+{
+	if (!cpu_voltage_gpio)
+		return -ENODEV;
+
+	switch (target_voltage) {
+	case VOLTAGE_0_9V:
+		gpiod_set_value(cpu_voltage_gpio, 1);
+		break;
+	case VOLTAGE_0_8V:
+		gpiod_set_value(cpu_voltage_gpio, 0);
+		break;
+	default:
+		pr_err("%s %d: unsupport  voltage %d\n", __func__, __LINE__,
+		       target_voltage);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int eswin_calc_pll(u32 *frac_val, u32 *postdiv1_val, u32 *fbdiv_val,
+			  u32 *refdiv_val, u64 rate,
+			  const struct eswin_clk_pll *clk)
+{
+	int ret = 0;
+
+	switch (clk->id) {
+	case EIC7700_APLL_FOUT1:
+		switch (rate) {
+		case APLL_LOW_FREQ:
+			*frac_val = 10603200;
+			*postdiv1_val = 0;
+			*fbdiv_val = 37;
+			*refdiv_val = 1;
+			break;
+		case APLL_HIGH_FREQ:
+		default:
+			*frac_val = 14092861;
+			*postdiv1_val = 0;
+			*fbdiv_val = 163;
+			*refdiv_val = 1;
+			break;
+		}
+		break;
+	case EIC7700_PLL_CPU:
+		switch (rate) {
+		case CLK_FREQ_1800M:
+			*frac_val = 0;
+			*postdiv1_val = 0;
+			*fbdiv_val = 300;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1700M:
+			*frac_val = 5592405;
+			*postdiv1_val = 0;
+			*fbdiv_val = 283;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1600M:
+			*frac_val = 11184810;
+			*postdiv1_val = 0;
+			*fbdiv_val = 266;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1500M:
+			*frac_val = 0;
+			*postdiv1_val = 0;
+			*fbdiv_val = 250;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1300M:
+			*frac_val = 11184810;
+			*postdiv1_val = 0;
+			*fbdiv_val = 216;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1200M:
+			*frac_val = 0;
+			*postdiv1_val = 0;
+			*fbdiv_val = 200;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1000M:
+			*frac_val = 11184810;
+			*postdiv1_val = 0;
+			*fbdiv_val = 166;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_900M:
+			*frac_val = 0;
+			*postdiv1_val = 0;
+			*fbdiv_val = 150;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_800M:
+			*frac_val = 5592405;
+			*postdiv1_val = 0;
+			*fbdiv_val = 133;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_700M:
+			*frac_val = 11184810;
+			*postdiv1_val = 0;
+			*fbdiv_val = 116;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_600M:
+			*frac_val = 0;
+			*postdiv1_val = 0;
+			*fbdiv_val = 100;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_500M:
+			*frac_val = 5592405;
+			*postdiv1_val = 0;
+			*fbdiv_val = 83;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_400M:
+			*frac_val = 11184810;
+			*postdiv1_val = 0;
+			*fbdiv_val = 66;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_200M:
+			*frac_val = 5592405;
+			*postdiv1_val = 0;
+			*fbdiv_val = 33;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_100M:
+			*frac_val = 11184810;
+			*postdiv1_val = 0;
+			*fbdiv_val = 16;
+			*refdiv_val = 1;
+			break;
+		case CLK_FREQ_1400M:
+		default:
+			*frac_val = 5592405;
+			*postdiv1_val = 0;
+			*fbdiv_val = 233;
+			*refdiv_val = 1;
+			break;
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		pr_err("%s %d, Invalid pll set req, rate %lld, clk id %d\n",
+		       __func__, __LINE__, rate, clk->id);
+		break;
+	}
+	return ret;
+}
+
+static bool cpu_no_boost_1_6ghz;
+static int __init cpu_no_boost_1_6ghz_setup(char *__unused)
+{
+	cpu_no_boost_1_6ghz = true;
+	return 1;
+}
+__setup("cpu_no_boost_1_6ghz", cpu_no_boost_1_6ghz_setup);
+
+#define to_pll_clk(_hw) container_of(_hw, struct eswin_clk_pll, hw)
+static int clk_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+			    unsigned long parent_rate)
+{
+	struct eswin_clk_pll *clk = to_pll_clk(hw);
+	u32 frac_val = 0, postdiv1_val, fbdiv_val, refdiv_val;
+	u32 val;
+	int ret;
+	struct clk *clk_cpu_mux = NULL;
+	struct clk *clk_cpu_lp_pll = NULL;
+	struct clk *clk_cpu_pll = NULL;
+	int try_count = 0;
+	bool lock_flag = false;
+	char clk_cpu_mux_name[50] = { 0 };
+	char clk_cpu_lp_pll_name[50] = { 0 };
+	char clk_cpu_pll_name[50] = { 0 };
+	enum voltage_level cpu_target_voltage = VOLTAGE_0_8V;
+	bool force_1_8ghz = clk->force_1_8ghz;
+
+	ret = eswin_calc_pll(&frac_val, &postdiv1_val, &fbdiv_val, &refdiv_val,
+			     (u64)rate, clk);
+	if (ret)
+		return ret;
+
+	/*
+	 * we must switch the cpu to other clk before we change the cpu pll
+	 */
+	if (clk->id == EIC7700_PLL_CPU) {
+		if (clk->numa_id < 0) {
+			sprintf(clk_cpu_mux_name, "%s",
+				"mux_u_cpu_root_3mux1_gfree");
+			sprintf(clk_cpu_lp_pll_name, "%s",
+				"clk_clk_u84_core_lp");
+			sprintf(clk_cpu_pll_name, "%s", "clk_pll_cpu");
+		} else {
+			sprintf(clk_cpu_mux_name, "d%d_%s", clk->numa_id,
+				"mux_u_cpu_root_3mux1_gfree");
+			sprintf(clk_cpu_lp_pll_name, "d%d_%s", clk->numa_id,
+				"clk_clk_u84_core_lp");
+			sprintf(clk_cpu_pll_name, "d%d_%s", clk->numa_id,
+				"clk_pll_cpu");
+		}
+
+		clk_cpu_mux = __clk_lookup(clk_cpu_mux_name);
+		if (!clk_cpu_mux) {
+			pr_err("%s %d, failed to get %s\n", __func__, __LINE__,
+			       clk_cpu_mux_name);
+			return -EINVAL;
+		}
+		clk_cpu_lp_pll = __clk_lookup(clk_cpu_lp_pll_name);
+		if (!clk_cpu_lp_pll) {
+			pr_err("%s %d, failed to get %s\n", __func__, __LINE__,
+			       clk_cpu_lp_pll_name);
+			return -EINVAL;
+		}
+		ret = clk_prepare_enable(clk_cpu_lp_pll);
+		if (ret) {
+			pr_err("%s %d, failed to enable %s, ret %d\n", __func__,
+			       __LINE__, clk_cpu_lp_pll_name, ret);
+			return ret;
+		}
+		clk_cpu_pll = __clk_lookup(clk_cpu_pll_name);
+		if (!clk_cpu_pll) {
+			pr_err("%s %d, failed to get %s\n", __func__, __LINE__,
+			       clk_cpu_pll_name);
+			clk_disable_unprepare(clk_cpu_lp_pll);
+			return -EINVAL;
+		}
+
+		ret = clk_set_parent(clk_cpu_mux, clk_cpu_lp_pll);
+		if (ret) {
+			pr_err("%s %d, failed to switch %s to %s, ret %d\n",
+			       __func__, __LINE__, clk_cpu_mux_name,
+			       clk_cpu_lp_pll_name, ret);
+			clk_disable_unprepare(clk_cpu_lp_pll);
+			return -EPERM;
+		}
+		mutex_lock(&lock);
+		if (clk->numa_id >= 0) {
+			g_cpu_info.cpu_freqhz[clk->numa_id] = rate;
+			rate = g_cpu_info.cpu_freqhz[0] > g_cpu_info.cpu_freqhz[1] ? 
+				g_cpu_info.cpu_freqhz[0] : g_cpu_info.cpu_freqhz[1];
+		}
+		/*
+		 * The CPU clock has now switched to the LP_PLL,
+		 * so we can adjust the CPU's supply voltage
+		 * If the board cpu voltage does not support boosting to 0.9V,
+		 * then the frequency cannot exceed 1.6GHz.
+		 */
+		switch (rate) {
+		case CLK_FREQ_1800M:
+		case CLK_FREQ_1700M:
+			cpu_target_voltage = VOLTAGE_0_9V;
+			break;
+		case CLK_FREQ_1600M:
+		case CLK_FREQ_1500M:
+			cpu_target_voltage = true == g_cpu_info.cpu_no_boost_1_6ghz ?
+						     VOLTAGE_0_8V :
+						     VOLTAGE_0_9V;
+			if (force_1_8ghz)
+				cpu_target_voltage = VOLTAGE_0_9V;
+			break;
+		default:
+			cpu_target_voltage = VOLTAGE_0_8V;
+			/*
+			 * For boards that do not support voltage switching,
+			 * the voltage is maintained at 0.8V.
+			 * Therefore, this is also considered successful.
+			 */
+			break;
+		}
+	}
+	mutex_unlock(&lock);
+
+	if (clk->cpu_current_voltage !=	cpu_target_voltage) {
+		ret = eswin_clk_set_cpu_voltage(clk->cpu_voltage_gpio,
+						cpu_target_voltage);
+		if (ret && !force_1_8ghz) {
+			pr_warn("failed to change cpu voltage to %d mV, not support rate %ld\n",
+				cpu_target_voltage, rate);
+			goto switch_back;
+		} else {
+			pr_info("cpu voltage change to %d mV, target rate %ld\n",
+				cpu_target_voltage, rate);
+			clk->cpu_current_voltage =
+				cpu_target_voltage;
+		}
+	}
+
+
+	/*first disable pll */
+	val = readl_relaxed(clk->ctrl_reg0);
+	val &= ~(((1 << clk->pllen_width) - 1) << clk->pllen_shift);
+	val |= 0 << clk->pllen_shift;
+	writel_relaxed(val, clk->ctrl_reg0);
+
+	val = readl_relaxed(clk->ctrl_reg0);
+	val &= ~(((1 << clk->fbdiv_width) - 1) << clk->fbdiv_shift);
+	val &= ~(((1 << clk->refdiv_width) - 1) << clk->refdiv_shift);
+	val |= refdiv_val << clk->refdiv_shift;
+	val |= fbdiv_val << clk->fbdiv_shift;
+	writel_relaxed(val, clk->ctrl_reg0);
+
+	val = readl_relaxed(clk->ctrl_reg1);
+	val &= ~(((1 << clk->frac_width) - 1) << clk->frac_shift);
+	val |= frac_val << clk->frac_shift;
+	writel_relaxed(val, clk->ctrl_reg1);
+
+	val = readl_relaxed(clk->ctrl_reg2);
+	val &= ~(((1 << clk->postdiv1_width) - 1) << clk->postdiv1_shift);
+	val |= postdiv1_val << clk->postdiv1_shift;
+	writel_relaxed(val, clk->ctrl_reg2);
+
+	/*at last, enable pll */
+	val = readl_relaxed(clk->ctrl_reg0);
+	val &= ~(((1 << clk->pllen_width) - 1) << clk->pllen_shift);
+	val |= 1 << clk->pllen_shift;
+	writel_relaxed(val, clk->ctrl_reg0);
+
+	/*
+	 * usually the pll wil lock in 50us
+	 */
+	do {
+		usleep_range(refdiv_val * 80, refdiv_val * 80 * 2);
+		val = readl_relaxed(clk->status_reg);
+		if (val & 1 << clk->lock_shift) {
+			lock_flag = true;
+			break;
+		}
+	} while (try_count++ < 10);
+
+	if (false == lock_flag) {
+		pr_err("%s %d, failed to lock the cpu pll, cpu will work on low power pll\n",
+		       __func__, __LINE__);
+		return -EBUSY;
+	}
+
+switch_back:
+	if (clk->id == EIC7700_PLL_CPU) {
+		ret = clk_set_parent(clk_cpu_mux, clk_cpu_pll);
+		if (ret) {
+			pr_err("%s %d, failed to switch %s to %s, ret %d\n",
+			       __func__, __LINE__, clk_cpu_mux_name,
+			       clk_cpu_pll_name, ret);
+			return -EPERM;
+		}
+		clk_disable_unprepare(clk_cpu_lp_pll);
+	}
+	return ret;
+}
+
+static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
+					 unsigned long parent_rate)
+{
+	struct eswin_clk_pll *clk = to_pll_clk(hw);
+	u64 frac_val, fbdiv_val, refdiv_val;
+	u32 postdiv1_val;
+	u32 val;
+	u64 rate;
+
+	val = readl_relaxed(clk->ctrl_reg0);
+	val = val >> clk->fbdiv_shift;
+	val &= ((1 << clk->fbdiv_width) - 1);
+	fbdiv_val = val;
+
+	val = readl_relaxed(clk->ctrl_reg0);
+	val = val >> clk->refdiv_shift;
+	val &= ((1 << clk->refdiv_width) - 1);
+	refdiv_val = val;
+
+	val = readl_relaxed(clk->ctrl_reg1);
+	val = val >> clk->frac_shift;
+	val &= ((1 << clk->frac_width) - 1);
+	frac_val = val;
+
+	val = readl_relaxed(clk->ctrl_reg2);
+	val = val >> clk->postdiv1_shift;
+	val &= ((1 << clk->postdiv1_width) - 1);
+	postdiv1_val = val;
+
+	switch (clk->id) {
+	case EIC7700_APLL_FOUT1:
+		switch (frac_val) {
+		case 14092861:
+			rate = APLL_HIGH_FREQ;
+			break;
+		case 10603200:
+			rate = APLL_LOW_FREQ;
+			break;
+		default:
+			pr_err("%s %d, clk id %d, unknown frac_val %llu\n",
+			       __func__, __LINE__, clk->id, frac_val);
+			rate = 0;
+			break;
+		}
+		break;
+	case EIC7700_PLL_CPU:
+		switch (fbdiv_val) {
+		case 300:
+			rate = CLK_FREQ_1800M;
+			break;
+		case 283:
+			rate = CLK_FREQ_1700M;
+			break;
+		case 266:
+			rate = CLK_FREQ_1600M;
+			break;
+		case 250:
+			rate = CLK_FREQ_1500M;
+			break;
+		case 216:
+			rate = CLK_FREQ_1300M;
+			break;
+		case 200:
+			rate = CLK_FREQ_1200M;
+			break;
+		case 166:
+			rate = CLK_FREQ_1000M;
+			break;
+		case 150:
+			rate = CLK_FREQ_900M;
+			break;
+		case 133:
+			rate = CLK_FREQ_800M;
+			break;
+		case 116:
+			rate = CLK_FREQ_700M;
+			break;
+		case 100:
+			rate = CLK_FREQ_600M;
+			break;
+		case 83:
+			rate = CLK_FREQ_500M;
+			break;
+		case 66:
+			rate = CLK_FREQ_400M;
+			break;
+		case 33:
+			rate = CLK_FREQ_200M;
+			break;
+		case 16:
+			rate = CLK_FREQ_100M;
+			break;
+		case 233:
+			rate = CLK_FREQ_1400M;
+			break;
+		default:
+			pr_err("%s %d, clk id %d, unknown fbdiv_val %llu\n",
+			       __func__, __LINE__, clk->id, fbdiv_val);
+			rate = 0;
+			break;
+		}
+		break;
+	default:
+		pr_err("%s %d, unknown clk id %d\n", __func__, __LINE__,
+		       clk->id);
+		rate = 0;
+		break;
+	}
+	return rate;
+}
+
+static long clk_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long *parent_rate)
+{
+	struct eswin_clk_pll *clk = to_pll_clk(hw);
+	int index;
+	u64 round_rate = 0;
+
+	/*Must be sorted in ascending order*/
+	u64 apll_clk[] = { APLL_LOW_FREQ, APLL_HIGH_FREQ };
+	u64 cpu_pll_clk[] = { CLK_FREQ_100M,  CLK_FREQ_200M,  CLK_FREQ_400M,
+			      CLK_FREQ_500M,  CLK_FREQ_600M,  CLK_FREQ_700M,
+			      CLK_FREQ_800M,  CLK_FREQ_900M,  CLK_FREQ_1000M,
+			      CLK_FREQ_1200M, CLK_FREQ_1300M, CLK_FREQ_1400M,
+			      CLK_FREQ_1500M, CLK_FREQ_1600M, CLK_FREQ_1700M,
+			      CLK_FREQ_1800M };
+
+	switch (clk->id) {
+	case EIC7700_APLL_FOUT1:
+		index = find_closest(rate, apll_clk, ARRAY_SIZE(apll_clk));
+		round_rate = apll_clk[index];
+		break;
+	case EIC7700_PLL_CPU:
+		index = find_closest(rate, cpu_pll_clk,
+				     ARRAY_SIZE(cpu_pll_clk));
+		round_rate = cpu_pll_clk[index];
+		break;
+	default:
+		pr_err("%s %d, unknown clk id %d\n", __func__, __LINE__,
+		       clk->id);
+		round_rate = 0;
+		break;
+	}
+	return round_rate;
+}
+
+static const struct clk_ops eswin_clk_pll_ops = {
+	.set_rate = clk_pll_set_rate,
+	.recalc_rate = clk_pll_recalc_rate,
+	.round_rate = clk_pll_round_rate,
+};
+
+void eswin_clk_register_pll(struct eswin_pll_clock *clks, int nums,
+			    struct eswin_clock_data *data, struct device *dev)
+{
+	void __iomem *base = data->base;
+	struct eswin_clk_pll *p_clk = NULL;
+	struct clk *clk = NULL;
+	struct clk_init_data init;
+	int i;
+	static struct gpio_desc *cpu_voltage_gpio;
+	int force_1_8ghz = 0;
+
+	p_clk = devm_kzalloc(dev, sizeof(*p_clk) * nums, GFP_KERNEL);
+
+	if (!p_clk)
+		return;
+
+	/*
+	 *In the D2D system, the boost operation is performed using the GPIO on Die0.
+	 *However, the same GPIO pin cannot be acquired twice, so special handling is implemented:
+	 *Once the GPIO is acquired,the other driver simply uses it directly
+	 */
+	cpu_voltage_gpio =
+		IS_ERR_OR_NULL(cpu_voltage_gpio) ?
+			devm_gpiod_get(dev, "cpu-voltage", GPIOD_OUT_HIGH) :
+			cpu_voltage_gpio;
+	if (IS_ERR_OR_NULL(cpu_voltage_gpio)) {
+		dev_warn(dev, "failed to get cpu voltage gpio, unable to adjust CPU voltage\n");
+		cpu_voltage_gpio = NULL;
+	} else {
+		/*cpu default freq is 1400M, the voltage should be VOLTAGE_0_8V*/
+		eswin_clk_set_cpu_voltage(cpu_voltage_gpio, VOLTAGE_0_8V);
+	}
+
+	force_1_8ghz = device_property_read_bool(dev, "force-1_8ghz");
+
+	mutex_lock(&lock);
+	if (g_cpu_info.cpu_no_boost_1_6ghz)
+		g_cpu_info.cpu_no_boost_1_6ghz = true == cpu_no_boost_1_6ghz;
+
+	if(data->numa_id >= 0)
+		g_cpu_info.cpu_freqhz[data->numa_id] = CLK_FREQ_1400M;
+
+	mutex_unlock(&lock);
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+		const char *parent_name =
+			clks[i].parent_name ?
+				kzalloc(strlen(clks[i].parent_name) +
+						2 * sizeof(char) + sizeof(int),
+					GFP_KERNEL) :
+				NULL;
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			if (parent_name) {
+				sprintf((char *)parent_name, "%s",
+					clks[i].parent_name);
+			}
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			if (parent_name) {
+				sprintf((char *)parent_name, "d%d_%s",
+					data->numa_id, clks[i].parent_name);
+			}
+		}
+
+		init.name = name;
+		init.flags = 0;
+		init.parent_names = parent_name ? &parent_name : NULL;
+		init.num_parents = parent_name ? 1 : 0;
+		init.ops = &eswin_clk_pll_ops;
+
+		p_clk->id = clks[i].id;
+		p_clk->numa_id = data->numa_id;
+		p_clk->ctrl_reg0 = base + clks[i].ctrl_reg0;
+		p_clk->pllen_shift = clks[i].pllen_shift;
+		p_clk->pllen_width = clks[i].pllen_width;
+		p_clk->refdiv_shift = clks[i].refdiv_shift;
+		p_clk->refdiv_width = clks[i].refdiv_width;
+		p_clk->fbdiv_shift = clks[i].fbdiv_shift;
+		p_clk->fbdiv_width = clks[i].fbdiv_width;
+
+		p_clk->ctrl_reg1 = base + clks[i].ctrl_reg1;
+		p_clk->frac_shift = clks[i].frac_shift;
+		p_clk->frac_width = clks[i].frac_width;
+
+		p_clk->ctrl_reg2 = base + clks[i].ctrl_reg2;
+		p_clk->postdiv1_shift = clks[i].postdiv1_shift;
+		p_clk->postdiv1_width = clks[i].postdiv1_width;
+		p_clk->postdiv2_shift = clks[i].postdiv2_shift;
+		p_clk->postdiv2_width = clks[i].postdiv2_width;
+
+		p_clk->status_reg = base + clks[i].status_reg;
+		p_clk->lock_shift = clks[i].lock_shift;
+		p_clk->lock_width = clks[i].lock_width;
+
+		p_clk->hw.init = &init;
+		p_clk->cpu_voltage_gpio = cpu_voltage_gpio;
+		p_clk->cpu_current_voltage = VOLTAGE_0_8V;
+		p_clk->force_1_8ghz = force_1_8ghz;
+		clk = clk_register(dev, &p_clk->hw);
+		if (IS_ERR(clk)) {
+			devm_kfree(dev, p_clk);
+			dev_err(dev, "%s: failed to register clock %s\n",
+				__func__, clks[i].name);
+			continue;
+		}
+
+		data->clk_data.clks[clks[i].id] = clk;
+		p_clk++;
+
+		kfree(name);
+		if (parent_name)
+			kfree(parent_name);
+	}
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_pll);
+
+int eswin_clk_register_fixed_factor(const struct eswin_fixed_factor_clock *clks,
+				    int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+		char *parent_name =
+			kzalloc(strlen(clks[i].parent_name) + 2 * sizeof(char) +
+					sizeof(int),
+				GFP_KERNEL);
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id,
+				clks[i].parent_name);
+		}
+
+		clk = clk_register_fixed_factor(NULL, name, parent_name,
+						clks[i].flags, clks[i].mult,
+						clks[i].div);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__,
+			       name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+
+		kfree(name);
+		kfree(parent_name);
+	}
+
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_fixed_factor(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_fixed_factor);
+
+int eswin_clk_register_mux(const struct eswin_mux_clock *clks, int nums,
+			   struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+	int j;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+
+		char **parent_names = kzalloc(
+			sizeof(char *) * clks[i].num_parents, GFP_KERNEL);
+		if (data->numa_id < 0)
+			sprintf(name, "%s", clks[i].name);
+		else
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+
+		for (j = 0; j < clks[i].num_parents; j++) {
+			parent_names[j] =
+				kzalloc(strlen(clks[i].parent_names[j]) +
+						2 * sizeof(char) + sizeof(int),
+					GFP_KERNEL);
+			if (data->numa_id < 0) {
+				sprintf(parent_names[j], "%s",
+					clks[i].parent_names[j]);
+			} else {
+				sprintf(parent_names[j], "d%d_%s",
+					data->numa_id, clks[i].parent_names[j]);
+			}
+		}
+		clk = clk_register_mux_table(
+			NULL, name, (const char *const *)parent_names,
+			clks[i].num_parents, clks[i].flags,
+			base + clks[i].offset, clks[i].shift, clks[i].mask,
+			clks[i].mux_flags, clks[i].table, &data->lock);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__,
+			       clks[i].name);
+			kfree(name);
+			for (j = 0; j < clks[i].num_parents; j++)
+				kfree(parent_names[j]);
+
+			kfree(parent_names);
+			goto err;
+		}
+
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+
+		kfree(name);
+		for (j = 0; j < clks[i].num_parents; j++)
+			kfree(parent_names[j]);
+
+		kfree(parent_names);
+	}
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_mux(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_mux);
+
+int eswin_clk_register_divider(const struct eswin_divider_clock *clks, int nums,
+			       struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+	struct clk_hw *clk_hw;
+	struct clk_hw *parent_hw;
+	struct clk_parent_data parent_data;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+		char *parent_name =
+			kzalloc(strlen(clks[i].parent_name) + 2 * sizeof(char) +
+					sizeof(int),
+				GFP_KERNEL);
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id,
+				clks[i].parent_name);
+		}
+		parent_hw = eswin_clk_find_parent(data, parent_name);
+		parent_data.name = parent_name;
+		parent_data.hw = parent_hw;
+		parent_data.fw_name = NULL;
+		clk_hw = clk_hw_register_divider_table_parent_data(
+			NULL, name, &parent_data, clks[i].flags,
+			base + clks[i].offset, clks[i].shift, clks[i].width,
+			clks[i].div_flags, clks[i].table, &data->lock);
+		if (IS_ERR(clk_hw)) {
+			pr_err("%s: failed to register clock %s\n", __func__,
+			       clks[i].name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+		clk = clk_hw->clk;
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+		kfree(parent_name);
+	}
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_divider(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_divider);
+
+int eswin_clk_register_gate(const struct eswin_gate_clock *clks, int nums,
+			    struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+	struct clk_hw *clk_hw;
+	struct clk_hw *parent_hw;
+	struct clk_parent_data parent_data;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+		char *parent_name =
+			kzalloc(strlen(clks[i].parent_name) + 2 * sizeof(char) +
+					sizeof(int),
+				GFP_KERNEL);
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id,
+				clks[i].parent_name);
+		}
+		parent_hw = eswin_clk_find_parent(data, parent_name);
+		parent_data.name = parent_name;
+		parent_data.hw = parent_hw;
+		parent_data.fw_name = NULL;
+		clk_hw = clk_hw_register_gate_parent_data(
+			NULL, name, &parent_data, clks[i].flags,
+			base + clks[i].offset, clks[i].bit_idx,
+			clks[i].gate_flags, &data->lock);
+		if (IS_ERR(clk_hw)) {
+			pr_err("%s: failed to register clock %s\n", __func__,
+			       clks[i].name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+		clk = clk_hw->clk;
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+		kfree(parent_name);
+	}
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_gate(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_gate);
+
+static const struct clk_ops clk_dummpy_ops = {
+
+};
+
+struct clk *eswin_register_clk(struct eswin_clock_data *data,
+			       struct device *dev, const char *name,
+			       const char *parent_name, unsigned long flags,
+			       spinlock_t *lock)
+{
+	struct eswin_clock *eclk;
+	struct clk *clk;
+	struct clk_init_data init;
+	struct clk_parent_data parent_data;
+	struct clk_hw *parent_hw;
+
+	eclk = kzalloc(sizeof(*eclk), GFP_KERNEL);
+	if (!eclk)
+		return ERR_PTR(-ENOMEM);
+
+	init.ops = &clk_dummpy_ops;
+
+	init.name = name;
+	init.flags = flags;
+	init.parent_names = NULL;
+	init.num_parents = (parent_name ? 1 : 0);
+	init.parent_data = &parent_data;
+
+	parent_hw = eswin_clk_find_parent(data, (char *)parent_name);
+	parent_data.name = parent_name;
+	parent_data.hw = parent_hw;
+	parent_data.fw_name = NULL;
+
+	eclk->hw.init = &init;
+
+	clk = clk_register(dev, &eclk->hw);
+	if (IS_ERR(clk))
+		kfree(eclk);
+
+	return clk;
+}
+
+int eswin_clk_register_clk(const struct eswin_clock *clks, int nums,
+			   struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) +
+					     sizeof(int),
+				     GFP_KERNEL);
+		char *parent_name =
+			kzalloc(strlen(clks[i].parent_name) + 2 * sizeof(char) +
+					sizeof(int),
+				GFP_KERNEL);
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id,
+				clks[i].parent_name);
+		}
+		clk = eswin_register_clk(data, NULL, name, parent_name,
+					 clks[i].flags, &data->lock);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__,
+			       clks[i].name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+		kfree(parent_name);
+	}
+	return 0;
+err:
+	while (i--)
+		clk_unregister_gate(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_clk);
diff --git a/drivers/clk/eswin/clk.h b/drivers/clk/eswin/clk.h
new file mode 100644
index 000000000000..42cf07d693ec
--- /dev/null
+++ b/drivers/clk/eswin/clk.h
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * Authors: huangyifeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef __ESWIN_CLK_H__
+#define __ESWIN_CLK_H__
+
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+
+#define CLK_FREQ_1800M 1800000000
+#define CLK_FREQ_1700M 1700000000
+#define CLK_FREQ_1600M 1600000000
+#define CLK_FREQ_1500M 1500000000
+#define CLK_FREQ_1400M 1400000000
+#define CLK_FREQ_1300M 1300000000
+#define CLK_FREQ_1200M 1200000000
+#define CLK_FREQ_1000M 1000000000
+#define CLK_FREQ_900M 900000000
+#define CLK_FREQ_800M 800000000
+#define CLK_FREQ_700M 700000000
+#define CLK_FREQ_600M 600000000
+#define CLK_FREQ_500M 500000000
+#define CLK_FREQ_400M 400000000
+#define CLK_FREQ_200M 200000000
+#define CLK_FREQ_100M 100000000
+#define CLK_FREQ_24M 24000000
+
+#define APLL_HIGH_FREQ 983040000
+#define APLL_LOW_FREQ 225792000
+
+struct eswin_clock_data {
+	struct clk_onecell_data clk_data;
+	void __iomem *base;
+	int numa_id;
+	spinlock_t lock;
+};
+
+struct eswin_clock {
+	unsigned int id;
+	char *name;
+	const char *parent_name;
+	unsigned long flags;
+	struct clk_hw hw;
+	const char *alias;
+};
+
+struct eswin_fixed_rate_clock {
+	unsigned int id;
+	char *name;
+	const char *parent_name;
+	unsigned long flags;
+	unsigned long fixed_rate;
+};
+
+struct eswin_fixed_factor_clock {
+	unsigned int id;
+	char *name;
+	const char *parent_name;
+	unsigned long mult;
+	unsigned long div;
+	unsigned long flags;
+};
+
+struct eswin_mux_clock {
+	unsigned int id;
+	const char *name;
+	const char *const *parent_names;
+	u8 num_parents;
+	unsigned long flags;
+	unsigned long offset;
+	u8 shift;
+	u32 mask;
+	u8 mux_flags;
+	u32 *table;
+	const char *alias;
+};
+
+struct eswin_divider_clock {
+	unsigned int id;
+	const char *name;
+	const char *parent_name;
+	unsigned long flags;
+	unsigned long offset;
+	u8 shift;
+	u8 width;
+	u8 div_flags;
+	struct clk_div_table *table;
+	const char *alias;
+};
+
+struct eswin_gate_clock {
+	unsigned int id;
+	const char *name;
+	const char *parent_name;
+	unsigned long flags;
+	unsigned long offset;
+	u8 bit_idx;
+	u8 gate_flags;
+	const char *alias;
+};
+
+struct eswin_pll_clock {
+	u32 id;
+	const char *name;
+	const char *parent_name;
+	const u32 ctrl_reg0;
+	const u8 pllen_shift;
+	const u8 pllen_width;
+	const u8 refdiv_shift;
+	const u8 refdiv_width;
+	const u8 fbdiv_shift;
+	const u8 fbdiv_width;
+
+	const u32 ctrl_reg1;
+	const u8 frac_shift;
+	const u8 frac_width;
+
+	const u32 ctrl_reg2;
+	const u8 postdiv1_shift;
+	const u8 postdiv1_width;
+	const u8 postdiv2_shift;
+	const u8 postdiv2_width;
+
+	const u32 status_reg;
+	const u8 lock_shift;
+	const u8 lock_width;
+};
+
+enum voltage_level {
+	VOLTAGE_0_9V = 900, // Represents 0.9V in millivolts
+	VOLTAGE_0_8V = 800 // Represents 0.8V in millivolts
+};
+
+struct eswin_clk_pll {
+	struct clk_hw hw;
+	u32 id;
+	int numa_id;
+	void __iomem *ctrl_reg0;
+	u8 pllen_shift;
+	u8 pllen_width;
+	u8 refdiv_shift;
+	u8 refdiv_width;
+	u8 fbdiv_shift;
+	u8 fbdiv_width;
+
+	void __iomem *ctrl_reg1;
+	u8 frac_shift;
+	u8 frac_width;
+
+	void __iomem *ctrl_reg2;
+	u8 postdiv1_shift;
+	u8 postdiv1_width;
+	u8 postdiv2_shift;
+	u8 postdiv2_width;
+
+	void __iomem *status_reg;
+	u8 lock_shift;
+	u8 lock_width;
+	struct gpio_desc *cpu_voltage_gpio;
+	enum voltage_level cpu_current_voltage;
+	bool force_1_8ghz;
+};
+
+struct eswin_clock_data *eswin_clk_init(struct platform_device *pdev, int nr_clks);
+int eswin_clk_register_fixed_rate(const struct eswin_fixed_rate_clock *clks, int nums,
+				  struct eswin_clock_data *data);
+
+void eswin_clk_register_pll(struct eswin_pll_clock *clks, int nums,
+			    struct eswin_clock_data *data, struct device *dev);
+
+int eswin_clk_register_fixed_factor(const struct eswin_fixed_factor_clock *clks,
+				    int nums, struct eswin_clock_data *data);
+int eswin_clk_register_mux(const struct eswin_mux_clock *clks, int nums,
+			   struct eswin_clock_data *data);
+
+int eswin_clk_register_divider(const struct eswin_divider_clock *clks, int nums,
+			       struct eswin_clock_data *data);
+int eswin_clk_register_gate(const struct eswin_gate_clock *clks, int nums,
+			    struct eswin_clock_data *data);
+
+int eswin_clk_register_clk(const struct eswin_clock *clks, int nums,
+			   struct eswin_clock_data *data);
+
+#define eswin_clk_unregister(type)                                 \
+	static inline void eswin_clk_unregister_##type(            \
+		const struct eswin_##type##_clock *clks, int nums, \
+		struct eswin_clock_data *data)                     \
+	{                                                          \
+		struct clk **clocks = data->clk_data.clks;         \
+		int i;                                             \
+		for (i = 0; i < nums; i++) {                       \
+			int id = clks[i].id;                       \
+			if (clocks[id])                            \
+				clk_unregister_##type(clocks[id]); \
+		}                                                  \
+	}
+
+eswin_clk_unregister(fixed_rate) eswin_clk_unregister(fixed_factor)
+	eswin_clk_unregister(mux) eswin_clk_unregister(divider)
+		eswin_clk_unregister(gate)
+
+#endif /* __ESWIN_CLK_H__ */
diff --git a/drivers/clk/eswin/clk_eic7700.h b/drivers/clk/eswin/clk_eic7700.h
new file mode 100644
index 000000000000..fbaca02e9a4c
--- /dev/null
+++ b/drivers/clk/eswin/clk_eic7700.h
@@ -0,0 +1,191 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * Authors: huangyifeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef __CLK_EIC7700_H__
+#define __CLK_EIC7700_H__
+
+/*REG OFFSET OF SYS-CRG*/
+#define EIC7700_REG_OFFSET_SPLL0_CFG_0 0X0000
+#define EIC7700_REG_OFFSET_SPLL0_CFG_1 0X0004
+#define EIC7700_REG_OFFSET_SPLL0_CFG_2 0X0008
+#define EIC7700_REG_OFFSET_SPLL0_DSKEWCAL 0X000C
+#define EIC7700_REG_OFFSET_SPLL0_SSC 0X0010
+#define EIC7700_REG_OFFSET_SPLL1_CFG_0 0X0014
+#define EIC7700_REG_OFFSET_SPLL1_CFG_1 0X0018
+#define EIC7700_REG_OFFSET_SPLL1_CFG_2 0X001C
+#define EIC7700_REG_OFFSET_SPLL1_DSKEWCAL 0X0020
+#define EIC7700_REG_OFFSET_SPLL1_SSC 0X0024
+#define EIC7700_REG_OFFSET_SPLL2_CFG_0 0X0028
+#define EIC7700_REG_OFFSET_SPLL2_CFG_1 0X002C
+#define EIC7700_REG_OFFSET_SPLL2_CFG_2 0X0030
+#define EIC7700_REG_OFFSET_SPLL2_DSKEWCAL 0X0034
+#define EIC7700_REG_OFFSET_SPLL2_SSC 0X0038
+#define EIC7700_REG_OFFSET_VPLL_CFG_0 0X003C
+#define EIC7700_REG_OFFSET_VPLL_CFG_1 0X0040
+#define EIC7700_REG_OFFSET_VPLL_CFG_2 0X0044
+#define EIC7700_REG_OFFSET_VPLL_DSKEWCAL 0X0048
+#define EIC7700_REG_OFFSET_VPLL_SSC 0X004C
+#define EIC7700_REG_OFFSET_APLL_CFG_0 0X0050
+#define EIC7700_REG_OFFSET_APLL_CFG_1 0X0054
+#define EIC7700_REG_OFFSET_APLL_CFG_2 0X0058
+#define EIC7700_REG_OFFSET_APLL_DSKEWCAL 0X005C
+#define EIC7700_REG_OFFSET_APLL_SSC 0X0060
+#define EIC7700_REG_OFFSET_MCPUT_PLL_CFG_0 0X0064
+#define EIC7700_REG_OFFSET_MCPUT_PLL_CFG_1 0X0068
+#define EIC7700_REG_OFFSET_MCPUT_PLL_CFG_2 0X006C
+#define EIC7700_REG_OFFSET_MCPUT_PLL_DSKEWCAL 0X0070
+#define EIC7700_REG_OFFSET_MCPUT_PLL_SSC 0X0074
+#define EIC7700_REG_OFFSET_DDRT_PLL_CFG_0 0X0078
+#define EIC7700_REG_OFFSET_DDRT_PLL_CFG_1 0X007C
+#define EIC7700_REG_OFFSET_DDRT_PLL_CFG_2 0X0080
+#define EIC7700_REG_OFFSET_DDRT_PLL_DSKEWCAL 0X0084
+#define EIC7700_REG_OFFSET_DDRT_PLL_SSC 0X0088
+#define EIC7700_REG_OFFSET_PLL_STATUS 0X00A4
+#define EIC7700_REG_OFFSET_NOC_CLK_CTRL 0X100
+#define EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL 0X104
+#define EIC7700_REG_OFFSET_BOOTSPI_CFGCLK_CTRL 0X108
+#define EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL 0X10C
+#define EIC7700_REG_OFFSET_SCPU_BUSCLK_CTRL 0X110
+#define EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL 0X114
+#define EIC7700_REG_OFFSET_LPCPU_BUSCLK_CTRL 0X118
+#define EIC7700_REG_OFFSET_TCU_ACLK_CTRL 0X11C
+#define EIC7700_REG_OFFSET_TCU_CFG_CTRL 0X120
+#define EIC7700_REG_OFFSET_DDR_CLK_CTRL 0X124
+#define EIC7700_REG_OFFSET_DDR1_CLK_CTRL 0X128
+#define EIC7700_REG_OFFSET_GPU_ACLK_CTRL 0X12C
+#define EIC7700_REG_OFFSET_GPU_CFG_CTRL 0X130
+#define EIC7700_REG_OFFSET_GPU_GRAY_CTRL 0X134
+#define EIC7700_REG_OFFSET_DSP_ACLK_CTRL 0X138
+#define EIC7700_REG_OFFSET_DSP_CFG_CTRL 0X13C
+#define EIC7700_REG_OFFSET_D2D_ACLK_CTRL 0X140
+#define EIC7700_REG_OFFSET_D2D_CFG_CTRL 0X144
+#define EIC7700_REG_OFFSET_HSP_ACLK_CTRL 0X148
+#define EIC7700_REG_OFFSET_HSP_CFG_CTRL 0X14C
+#define EIC7700_REG_OFFSET_SATA_RBC_CTRL 0X150
+#define EIC7700_REG_OFFSET_SATA_OOB_CTRL 0X154
+#define EIC7700_REG_OFFSET_ETH0_CTRL 0X158
+#define EIC7700_REG_OFFSET_ETH1_CTRL 0X15C
+#define EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL 0X160
+#define EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL 0X164
+#define EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL 0X168
+#define EIC7700_REG_OFFSET_MSHC_USB_SLWCLK 0X16C
+#define EIC7700_REG_OFFSET_PCIE_ACLK_CTRL 0X170
+#define EIC7700_REG_OFFSET_PCIE_CFG_CTRL 0X174
+#define EIC7700_REG_OFFSET_NPU_ACLK_CTRL 0X178
+#define EIC7700_REG_OFFSET_NPU_LLC_CTRL 0X17C
+#define EIC7700_REG_OFFSET_NPU_CORE_CTRL 0X180
+#define EIC7700_REG_OFFSET_VI_DWCLK_CTRL 0X184
+#define EIC7700_REG_OFFSET_VI_ACLK_CTRL 0X188
+#define EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL 0X18C
+#define EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL 0X190
+#define EIC7700_REG_OFFSET_VI_SHUTTER0 0X194
+#define EIC7700_REG_OFFSET_VI_SHUTTER1 0X198
+#define EIC7700_REG_OFFSET_VI_SHUTTER2 0X19C
+#define EIC7700_REG_OFFSET_VI_SHUTTER3 0X1A0
+#define EIC7700_REG_OFFSET_VI_SHUTTER4 0X1A4
+#define EIC7700_REG_OFFSET_VI_SHUTTER5 0X1A8
+#define EIC7700_REG_OFFSET_VI_PHY_CLKCTRL 0X1AC
+#define EIC7700_REG_OFFSET_VO_ACLK_CTRL 0X1B0
+#define EIC7700_REG_OFFSET_VO_IESMCLK_CTRL 0X1B4
+#define EIC7700_REG_OFFSET_VO_PIXEL_CTRL 0X1B8
+#define EIC7700_REG_OFFSET_VO_MCLK_CTRL 0X1BC
+#define EIC7700_REG_OFFSET_VO_PHY_CLKCTRL 0X1C0
+#define EIC7700_REG_OFFSET_VC_ACLK_CTRL 0X1C4
+#define EIC7700_REG_OFFSET_VCDEC_ROOTCLK_CTRL 0X1C8
+#define EIC7700_REG_OFFSET_G2D_CTRL 0X1CC
+#define EIC7700_REG_OFFSET_VC_CLKEN_CTRL 0X1D0
+#define EIC7700_REG_OFFSET_JE_CLK_CTRL 0X1D4
+#define EIC7700_REG_OFFSET_JD_CLK_CTRL 0X1D8
+#define EIC7700_REG_OFFSET_VD_CLK_CTRL 0X1DC
+#define EIC7700_REG_OFFSET_VE_CLK_CTRL 0X1E0
+#define EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL 0X1E4
+#define EIC7700_REG_OFFSET_TIMER_CLK_CTRL 0X1E8
+#define EIC7700_REG_OFFSET_RTC_CLK_CTRL 0X1EC
+#define EIC7700_REG_OFFSET_PKA_CLK_CTRL 0X1F0
+#define EIC7700_REG_OFFSET_SPACC_CLK_CTRL 0X1F4
+#define EIC7700_REG_OFFSET_TRNG_CLK_CTRL 0X1F8
+#define EIC7700_REG_OFFSET_OTP_CLK_CTRL 0X1FC
+#define EIC7700_REG_OFFSET_LSP_CLK_EN0 0X200
+#define EIC7700_REG_OFFSET_LSP_CLK_EN1 0X204
+#define EIC7700_REG_OFFSET_U84_CLK_CTRL 0X208
+#define EIC7700_REG_OFFSET_SYSCFG_CLK_CTRL 0X20C
+#define EIC7700_REG_OFFSET_I2C0_CLK_CTRL 0X210
+#define EIC7700_REG_OFFSET_I2C1_CLK_CTRL 0X214
+#define EIC7700_REG_OFFSET_DFT_CLK_CTRL 0X280
+#define EIC7700_REG_OFFSET_SYS_SWRST_VALUE 0X300
+#define EIC7700_REG_OFFSET_CLR_RST_STATUS 0X304
+#define EIC7700_REG_OFFSET_DIE_STATUS 0X308
+#define EIC7700_REG_OFFSET_CLR_BOOT_INFO 0X30C
+#define EIC7700_REG_OFFSET_SCPU_BOOT_ADDRESS 0X310
+#define EIC7700_REG_OFFSET_LPCPU_BOOT_ADDRESS 0X314
+#define EIC7700_REG_OFFSET_NPUE31_BOOT_ADDRESS 0X318
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS0_HI 0X31C
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS0_LOW 0X320
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS1_HI 0X324
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS1_LOW 0X328
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS2_HI 0X32C
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS2_LOW 0X330
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS3_HI 0X334
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS3_LOW 0X338
+#define EIC7700_REG_OFFSET_BOOT_SEL_STAT 0X33C
+#define EIC7700_REG_OFFSET_BOOT_SPI_CFG 0X340
+#define EIC7700_REG_OFFSET_SNOC_RST_CTRL 0X400
+#define EIC7700_REG_OFFSET_GPU_RST_CTRL 0X404
+#define EIC7700_REG_OFFSET_DSP_RST_CTRL 0X408
+#define EIC7700_REG_OFFSET_D2D_RST_CTRL 0X40C
+#define EIC7700_REG_OFFSET_DDR_RST_CTRL 0X410
+#define EIC7700_REG_OFFSET_TCU_RST_CTRL 0X414
+#define EIC7700_REG_OFFSET_NPU_RST_CTRL 0X418
+#define EIC7700_REG_OFFSET_HSPDMA_RST_CTRL 0X41C
+#define EIC7700_REG_OFFSET_PCIE_RST_CTRL 0X420
+#define EIC7700_REG_OFFSET_I2C_RST_CTRL 0X424
+#define EIC7700_REG_OFFSET_FAN_RST_CTRL 0X428
+#define EIC7700_REG_OFFSET_PVT_RST_CTRL 0X42C
+#define EIC7700_REG_OFFSET_MBOX_RST_CTRL 0X430
+#define EIC7700_REG_OFFSET_UART_RST_CTRL 0X434
+#define EIC7700_REG_OFFSET_GPIO_RST_CTRL 0X438
+#define EIC7700_REG_OFFSET_TIMER_RST_CTRL 0X43C
+#define EIC7700_REG_OFFSET_SSI_RST_CTRL 0X440
+#define EIC7700_REG_OFFSET_WDT_RST_CTRL 0X444
+#define EIC7700_REG_OFFSET_LSP_CFGRST_CTRL 0X448
+#define EIC7700_REG_OFFSET_U84_RST_CTRL 0X44C
+#define EIC7700_REG_OFFSET_SCPU_RST_CTRL 0X450
+#define EIC7700_REG_OFFSET_LPCPU_RST_CTRL 0X454
+#define EIC7700_REG_OFFSET_VC_RST_CTRL 0X458
+#define EIC7700_REG_OFFSET_JD_RST_CTRL 0X45C
+#define EIC7700_REG_OFFSET_JE_RST_CTRL 0X460
+#define EIC7700_REG_OFFSET_VE_RST_CTRL 0X468
+#define EIC7700_REG_OFFSET_G2D_RST_CTRL 0X46C
+#define EIC7700_REG_OFFSET_VI_RST_CTRL 0X470
+#define EIC7700_REG_OFFSET_DVP_RST_CTRL 0X474
+#define EIC7700_REG_OFFSET_ISP0_RST_CTRL 0X478
+#define EIC7700_REG_OFFSET_ISP1_RST_CTRL 0X47C
+#define EIC7700_REG_OFFSET_SHUTTER_RST_CTRL 0X480
+#define EIC7700_REG_OFFSET_VO_PHYRST_CTRL 0X484
+#define EIC7700_REG_OFFSET_VO_I2SRST_CTRL 0X488
+#define EIC7700_REG_OFFSET_VO_RST_CTRL 0X48C
+#define EIC7700_REG_OFFSET_BOOTSPI_RST_CTRL 0X490
+#define EIC7700_REG_OFFSET_I2C1_RST_CTRL 0X494
+#define EIC7700_REG_OFFSET_I2C0_RST_CTRL 0X498
+#define EIC7700_REG_OFFSET_DMA1_RST_CTRL 0X49C
+#define EIC7700_REG_OFFSET_FPRT_RST_CTRL 0X4A0
+#define EIC7700_REG_OFFSET_HBLOCK_RST_CTRL 0X4A4
+#define EIC7700_REG_OFFSET_SECSR_RST_CTRL 0X4A8
+#define EIC7700_REG_OFFSET_OTP_RST_CTRL 0X4AC
+#define EIC7700_REG_OFFSET_PKA_RST_CTRL 0X4B0
+#define EIC7700_REG_OFFSET_SPACC_RST_CTRL 0X4B4
+#define EIC7700_REG_OFFSET_TRNG_RST_CTRL 0X4B8
+#define EIC7700_REG_OFFSET_TIMER0_RST_CTRL 0X4C0
+#define EIC7700_REG_OFFSET_TIMER1_RST_CTRL 0X4C4
+#define EIC7700_REG_OFFSET_TIMER2_RST_CTRL 0X4C8
+#define EIC7700_REG_OFFSET_TIMER3_RST_CTRL 0X4CC
+#define EIC7700_REG_OFFSET_RTC_RST_CTRL 0X4D0
+#define EIC7700_REG_OFFSET_MNOC_RST_CTRL 0X4D4
+#define EIC7700_REG_OFFSET_RNOC_RST_CTRL 0X4D8
+#define EIC7700_REG_OFFSET_CNOC_RST_CTRL 0X4DC
+#define EIC7700_REG_OFFSET_LNOC_RST_CTRL 0X4E0
+#endif
diff --git a/drivers/clk/eswin/test/Kconfig b/drivers/clk/eswin/test/Kconfig
new file mode 100755
index 000000000000..ed2c606fc8d9
--- /dev/null
+++ b/drivers/clk/eswin/test/Kconfig
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config COMMON_CLK_EIC7700_TEST
+	bool "Eswin Clock Driver Test"
+	default y
+	help
+	  Build the Eswin clock test driver
diff --git a/drivers/clk/eswin/test/Makefile b/drivers/clk/eswin/test/Makefile
new file mode 100755
index 000000000000..721c9d8ffa22
--- /dev/null
+++ b/drivers/clk/eswin/test/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Eswin Clock test specific Makefile
+#
+
+obj-$(CONFIG_COMMON_CLK_EIC7700_TEST)	+= clk_test_kmd.o
diff --git a/drivers/clk/eswin/test/clk_test_kmd.c b/drivers/clk/eswin/test/clk_test_kmd.c
new file mode 100755
index 000000000000..a330e7339382
--- /dev/null
+++ b/drivers/clk/eswin/test/clk_test_kmd.c
@@ -0,0 +1,425 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/seq_file.h>
+/*
+	1.mount -t debugfs none /sys/kernel/debug
+*/
+#define CLKDBG_ARGS_MAX			3
+#define CLKDBG_INSTR_MAX_LEN	128
+#define CLKDBG_DEFAULT_SEP_STR	" \n"
+
+enum {
+	CLKDBG_CMD_ENABLE,
+	CLKDBG_CMD_DISABLE,
+	CLKDBG_CMD_PREPARE,
+	CLKDBG_CMD_UNPREPARE,
+	CLKDBG_CMD_PREPARE_ENABLE,
+	CLKDBG_CMD_DISABLE_UNPREPARE,
+	CLKDBG_CMD_SETRATE,
+	CLKDBG_CMD_ROUNDRATE,
+	CLKDBG_CMD_GETRATE,
+	CLKDBG_CMD_SETPARENT,
+	CLKDBG_CMD_GETPARENT,
+	CLKDBG_CMD_NR,
+};
+
+enum {
+	CLKDBG_ERR_NO = 0x0,
+	CLKDBG_ERR_PARSE_CMD = 0x1,
+	CLKDBG_ERR_PARSE_SELF = 0x2,
+	CLKDBG_ERR_PARSE_ARGS = 0x4,
+	CLKDBG_ERR_EXEC_CMD = 0x10,
+};
+#define CLKDBG_ERR_PARSE	(CLKDBG_ERR_PARSE_CMD |\
+							CLKDBG_ERR_PARSE_SELF |\
+							CLKDBG_ERR_PARSE_ARGS)
+#define CLKDBG_ERR_EXEC		(CLKDBG_ERR_EXEC_CMD)
+
+enum {
+	CLKDBG_STS_COMPLETE,
+	CLKDBG_STS_START,
+};
+
+struct cmd_packet {
+	unsigned int cmd;
+	struct clk *self;
+	struct clk_hw *self_hw;
+	struct clk *parent;
+	struct clk_hw *parent_hw;
+	unsigned long rate;
+	int result;
+	int status;
+};
+
+struct clkdbg {
+	void *instr;
+	struct dentry *clkdbg_dir;
+	struct cmd_packet *cmdp;
+};
+
+static struct clkdbg *clkdbg = NULL;
+
+static const char * const cmd_str[CLKDBG_CMD_NR] = {
+	"enable",
+	"disable",
+	"prepare",
+	"unprepare",
+	"prepare_enable",
+	"disable_unprepare",
+	"set_rate",
+	"round_rate",
+	"get_rate",
+	"set_parent",
+	"get_parent",
+};
+
+static const char * const hlp_str[CLKDBG_CMD_NR] = {
+	"enable\t\t\t: echo enable {clk} > clkdbg\n",
+	"disable\t\t\t: echo disable {clk} > clkdbg\n",
+	"prepare\t\t\t: echo prepare {clk} > clkdbg\n",
+	"unprepare\t\t: echo unprepare {clk} > clkdbg\n",
+	"prepare_enable\t\t: echo prepare {clk} > clkdbg\n",
+	"disable_unprepare\t: echo unprepare {clk} > clkdbg\n",
+	"set_rate\t\t: echo set_rate {clk} {rate(Hz)} > clkdbg\n",
+	"round_rate\t\t: echo round_rate {clk} {rate(Hz)} > clkdbg\n",
+	"get_rate\t\t: echo get_rate {clk} > clkdbg\n",
+	"set_parent\t\t: echo set_parent {clk} {parent} > clkdbg\n",
+	"get_parent\t\t: echo get_parent {clk} > clkdbg\n",
+};
+
+static int clkdbg_parse_cmd(char **str, char *sep,
+							struct cmd_packet *cmdp)
+{
+	char *str_sec;
+	int ret, cmd = 0;
+
+	/* parse the first section string. */
+	str_sec = strsep(str, sep);
+	if (str_sec == NULL)
+		return -EINVAL;
+
+	/*
+	 * search request keyword to find predefined keywords:
+	 * enable, disable, prepare, unprepare, set_rate, set_parent.
+	 */
+	do {
+		ret = strcmp(str_sec, cmd_str[cmd]);
+	} while (ret && (++cmd < CLKDBG_CMD_NR));
+
+	/* assign and record command value. */
+	cmdp->cmd = cmd;
+
+	pr_info("cmd: %d\n", cmdp->cmd);
+
+	return ret;
+}
+
+static int clkdbg_parse_selfclk(char **str, char *sep,
+								  struct cmd_packet *cmdp)
+{
+	char *str_sec;
+
+	/* parse the self clock name. */
+	str_sec = strsep(str, sep);
+	if (str_sec == NULL)
+		return -EINVAL;
+
+	/*
+	 * try to lookup self clock reference, and return directly
+	 * if failed.
+	 */
+	cmdp->self = __clk_lookup(str_sec);
+	if (!cmdp->self)
+		return -EINVAL;
+
+	cmdp->self_hw = __clk_get_hw(cmdp->self);
+
+	pr_info("self: \"%s\"\n", __clk_get_name(cmdp->self));
+
+	return 0;
+}
+
+static int clkdbg_parse_args(char **str, char *sep,
+							 struct cmd_packet *cmdp)
+{
+	char *args[CLKDBG_ARGS_MAX];
+	int idx = 0, ret = 0;
+
+	/* parse argument list and store them. */
+	do {
+		args[idx] = strsep(str, sep);
+	} while (args[idx] && (++idx < CLKDBG_ARGS_MAX));
+
+	switch (cmdp->cmd) {
+	case CLKDBG_CMD_SETRATE:
+	case CLKDBG_CMD_ROUNDRATE:
+		ret = kstrtoul(args[0], 0, &cmdp->rate);
+		pr_info("rate: %ld\n", cmdp->rate);
+		break;
+	case CLKDBG_CMD_SETPARENT:
+		cmdp->parent = __clk_lookup(args[0]);
+		if (!cmdp->parent)
+			ret = -EINVAL;
+		pr_info("parent: \"%s\"\n", __clk_get_name(cmdp->parent));
+		break;
+	default:
+		ret = 0;
+		break;
+	}
+
+	return ret;
+}
+
+static int clkdbg_parse_str(char **str, char *sep,
+							struct cmd_packet *cmdp)
+{
+	int ret = 0;
+
+	ret = clkdbg_parse_cmd(str, sep, cmdp);
+	if (ret) {
+		cmdp->result = CLKDBG_ERR_PARSE_CMD;
+		goto out;
+	}
+
+	ret = clkdbg_parse_selfclk(str, sep, cmdp);
+	if (ret) {
+		cmdp->result = CLKDBG_ERR_PARSE_SELF;
+		goto out;
+	}
+
+	ret = clkdbg_parse_args(str, sep, cmdp);
+	if (ret) {
+		cmdp->result = CLKDBG_ERR_PARSE_ARGS;
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
+static int clkdbg_execute_cmd(struct clkdbg *clkdbg)
+{
+	struct cmd_packet *cmdp = clkdbg->cmdp;
+	int ret = 0;
+
+	switch (cmdp->cmd) {
+	case CLKDBG_CMD_PREPARE:
+		ret = clk_prepare(cmdp->self);
+		break;
+	case CLKDBG_CMD_UNPREPARE:
+		clk_unprepare(cmdp->self);
+		break;
+	case CLKDBG_CMD_ENABLE:
+		ret = clk_enable(cmdp->self);
+		break;
+	case CLKDBG_CMD_DISABLE:
+		clk_disable(cmdp->self);
+		break;
+	case CLKDBG_CMD_PREPARE_ENABLE:
+		ret = clk_prepare_enable(cmdp->self);
+		break;
+	case CLKDBG_CMD_DISABLE_UNPREPARE:
+		clk_disable_unprepare(cmdp->self);
+		break;
+	case CLKDBG_CMD_SETRATE:
+		ret = clk_set_rate(cmdp->self, cmdp->rate);
+		break;
+	case CLKDBG_CMD_ROUNDRATE:
+		ret = clk_round_rate(cmdp->self, cmdp->rate);
+		if (ret >= 0) {
+			cmdp->rate = ret;
+			ret = 0;
+		}
+		break;
+	case CLKDBG_CMD_GETRATE:
+		cmdp->rate = clk_get_rate(cmdp->self);
+		break;
+	case CLKDBG_CMD_SETPARENT:
+		ret = clk_set_parent(cmdp->self, cmdp->parent);
+		break;
+	case CLKDBG_CMD_GETPARENT:
+		cmdp->parent = clk_get_parent(cmdp->self);
+		cmdp->parent_hw = __clk_get_hw(cmdp->parent);
+		break;
+	default:
+		break;
+	}
+
+	if (ret)
+		cmdp->result = CLKDBG_ERR_EXEC_CMD;
+
+	return ret;
+}
+
+static ssize_t clkdbg_write(struct file *filp,
+							const char __user *userbuf,
+							size_t count, loff_t *ppos)
+{
+	struct clkdbg *clkdbg =
+			((struct seq_file *)filp->private_data)->private;
+	struct cmd_packet *cmdp = clkdbg->cmdp;
+	char *sep = CLKDBG_DEFAULT_SEP_STR;
+	char *instr = clkdbg->instr;
+	int ret;
+
+	memset(instr, 0x0, CLKDBG_INSTR_MAX_LEN);
+	memset(cmdp, 0x0, sizeof(*cmdp));
+	clkdbg->cmdp->status = CLKDBG_STS_START;
+	clkdbg->cmdp->result = CLKDBG_ERR_NO;
+
+	ret = copy_from_user(instr, userbuf, count);
+	if (ret) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	/*
+	 * parse the input string and write cmd, argument list,
+	 * self clock and parent(if exists) into cmd_packet struct.
+	 */
+	ret = clkdbg_parse_str(&instr, sep, cmdp);
+	if (ret < 0)
+		goto out;
+
+	/* execute clock command. */
+	ret = clkdbg_execute_cmd(clkdbg);
+
+out:
+	return count;
+}
+
+static int clkdbg_show(struct seq_file *s, void *data)
+{
+	struct clkdbg *clkdbg = (struct clkdbg *)s->private;
+	struct cmd_packet *cmdp = clkdbg->cmdp;
+	struct clk_hw *clk_hw;
+	//const char *pname_c, *pname;
+	int idx, num_parents;
+
+	/* default helper string printed. */
+	if (cmdp->status == CLKDBG_STS_COMPLETE) {
+		for (idx = CLKDBG_CMD_ENABLE; idx < CLKDBG_CMD_NR; idx++)
+			seq_printf(s, "%s", hlp_str[idx]);
+		return 0;
+	}
+
+	if (cmdp->result & CLKDBG_ERR_PARSE) {
+		seq_printf(s, "parse cmd string failed(%d)!\n", cmdp->result);
+		goto out;
+	}
+
+	seq_printf(s, "%s \"%s\"",
+			   cmd_str[cmdp->cmd], __clk_get_name(cmdp->self));
+
+	switch (cmdp->cmd) {
+	case CLKDBG_CMD_SETRATE:
+	case CLKDBG_CMD_ROUNDRATE:
+		seq_printf(s, " as %ld ", cmdp->rate);
+		break;
+	case CLKDBG_CMD_GETRATE:
+		seq_printf(s, ":\n\t%ld Hz\n", cmdp->rate);
+		break;
+	case CLKDBG_CMD_SETPARENT:
+		seq_printf(s, " as \"%s\" ", __clk_get_name(cmdp->parent));
+		break;
+	case CLKDBG_CMD_GETPARENT:
+		seq_puts(s, ":\n");
+		num_parents = clk_hw_get_num_parents(cmdp->self_hw);
+		for (idx = 0; idx < num_parents; idx++) {
+			clk_hw = clk_hw_get_parent_by_index(cmdp->self_hw, idx);
+			if (clk_hw == cmdp->parent_hw)
+				seq_puts(s, "\t[*]");
+			else
+				seq_puts(s, "\t[ ]");
+
+			seq_printf(s, "%s\n", __clk_get_name(clk_hw->clk));
+		}
+		break;
+	default:
+		break;
+	}
+
+	seq_printf(s, "%s\n", cmdp->result ? "failed" : "success");
+
+out:
+	cmdp->status = CLKDBG_STS_COMPLETE;
+
+	return 0;
+}
+
+static int clkdbg_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, clkdbg_show, inode->i_private);
+}
+
+static const struct file_operations clkdbg_ops = {
+	.owner		= THIS_MODULE,
+	.open		= clkdbg_open,
+	.release	= single_release,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.write		= clkdbg_write,
+};
+
+static int clkdbg_add_debugfs(void)
+{
+	if (!debugfs_initialized())
+		return 0;
+
+	clkdbg->clkdbg_dir = debugfs_create_file("clkdbg", 0600, NULL, clkdbg, &clkdbg_ops);
+
+	return 0;
+}
+
+static int __init clkdbg_init(void)
+{
+	clkdbg = kzalloc(sizeof(*clkdbg), GFP_KERNEL);
+	if (!clkdbg)
+		return -ENOMEM;
+
+	clkdbg->instr = kzalloc(CLKDBG_INSTR_MAX_LEN, GFP_KERNEL);
+	if (!clkdbg->instr)
+		return -ENOMEM;
+
+	clkdbg->cmdp = kzalloc(sizeof(struct cmd_packet), GFP_KERNEL);
+	if (!clkdbg->instr)
+		return -ENOMEM;
+
+	return clkdbg_add_debugfs();
+}
+
+static void __exit clkdbg_exit(void)
+{
+	if (clkdbg && clkdbg->clkdbg_dir)
+		debugfs_remove_recursive(clkdbg->clkdbg_dir);
+
+	if (clkdbg)
+		kfree(clkdbg);
+}
+
+module_init(clkdbg_init);
+module_exit(clkdbg_exit);
+
+MODULE_DESCRIPTION("CCF Debug Driver");
+MODULE_AUTHOR("Shunli Wang <wangshunli@cambricon.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index 8208a3d89563..db73c2254f75 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -73,6 +73,11 @@ config DW_APB_TIMER_OF
 	select DW_APB_TIMER
 	select TIMER_OF
 
+config TIMER_ESWIN
+	tristate "ESWIN timer support"
+	help
+	  Enables the support for the eswin timer.
+
 config FTTMR010_TIMER
 	bool "Faraday Technology timer driver" if COMPILE_TEST
 	depends on HAS_IOMEM
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 368c3461dab8..cee18fa560c0 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -21,6 +21,7 @@ obj-$(CONFIG_OMAP_DM_TIMER)	+= timer-ti-dm.o
 obj-$(CONFIG_OMAP_DM_SYSTIMER)	+= timer-ti-dm-systimer.o
 obj-$(CONFIG_DW_APB_TIMER)	+= dw_apb_timer.o
 obj-$(CONFIG_DW_APB_TIMER_OF)	+= dw_apb_timer_of.o
+obj-$(CONFIG_TIMER_ESWIN)	+= timer-eswin.o
 obj-$(CONFIG_FTTMR010_TIMER)	+= timer-fttmr010.o
 obj-$(CONFIG_IXP4XX_TIMER)	+= timer-ixp4xx.o
 obj-$(CONFIG_ROCKCHIP_TIMER)      += timer-rockchip.o
diff --git a/drivers/clocksource/timer-eswin.c b/drivers/clocksource/timer-eswin.c
new file mode 100644
index 000000000000..f7003459da04
--- /dev/null
+++ b/drivers/clocksource/timer-eswin.c
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * ESWIN timer driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Xuxiang <xuxiang@eswincomputing.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/clockchips.h>
+#include <linux/interrupt.h>
+#include <linux/mfd/stm32-lptimer.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_wakeirq.h>
+#include <linux/reset.h>
+#include <linux/miscdevice.h>
+
+#define APBT_MIN_PERIOD         4
+#define APBT_MIN_DELTA_USEC     200
+
+#define APBTMR_N_LOAD_COUNT     0x00
+#define APBTMR_N_CURRENT_VALUE  0x04
+#define APBTMR_N_CONTROL        0x08
+#define APBTMR_N_EOI            0x0c
+#define APBTMR_N_INT_STATUS     0x10
+
+#define APBTMRS_INT_STATUS      0xa0
+#define APBTMRS_EOI             0xa4
+#define APBTMRS_RAW_INT_STATUS  0xa8
+#define APBTMRS_COMP_VERSION    0xac
+
+#define APBTMR_CONTROL_ENABLE   (1 << 0)
+/* 1: periodic, 0:free running. */
+#define APBTMR_CONTROL_MODE_PERIODIC    (1 << 1)
+#define APBTMR_CONTROL_INT              (1 << 2)
+
+#define APBTMR_MAX_CNT  0xFFFFFFFF
+#define APBTMR_EACH_OFS 0x14
+
+struct eswin_timer {
+    /* Memory Mapped Register */
+    struct resource *mem;
+    void __iomem *mmio_base;
+    u32 perf_count;
+    u32 numa_id;
+};
+
+static struct eswin_timer *perf_timer[2] = {NULL, NULL};
+static void __iomem *perf_cnt_base[2] = {NULL, NULL};
+
+static inline u32 eswin_readl(struct eswin_timer *timer, unsigned long offs)
+{
+    return readl(timer->mmio_base + offs);
+}
+
+static inline void eswin_writel(struct eswin_timer *timer, u32 val,
+            unsigned long offs)
+{
+    writel(val, timer->mmio_base + offs);
+}
+
+static inline u32 eswin_readl_relaxed(struct eswin_timer *timer, unsigned long offs)
+{
+    return readl_relaxed(timer->mmio_base + offs);
+}
+
+static inline void eswin_writel_relaxed(struct eswin_timer *timer, u32 val,
+            unsigned long offs)
+{
+    writel_relaxed(val, timer->mmio_base + offs);
+}
+
+static irqreturn_t timer_irq_handler(int irq, void *dev_id)
+{
+    struct eswin_timer *time = dev_id;
+    pr_err("Enter timer_irq_handler  irq = %x, time->mmio_base = %lx\n",irq,(long)time->mmio_base);
+    /*
+    * clear interrupt read(0xa4)
+    * read intr status read(0xa0)
+    */
+    eswin_readl(time,APBTMRS_INT_STATUS);
+    eswin_readl(time,APBTMRS_EOI);
+    return IRQ_HANDLED;
+}
+
+static int __init timer_init(struct device_node *np)
+{
+    struct clk *pclk, *timer_aclk, *timer3_clk8;
+    struct reset_control *trstc0,*trstc1,*trstc2,*trstc3,*trstc4,*trstc5,*trstc6,*trstc7,*prstc;
+
+    /*
+    * Reset the timer if the reset control is available, wiping
+    * out the state the firmware may have left it
+    */
+    trstc0 = of_reset_control_get(np, "trst0");
+    if (!IS_ERR(trstc0)) {
+        reset_control_assert(trstc0);
+        reset_control_deassert(trstc0);
+    }
+    trstc1 = of_reset_control_get(np, "trst1");
+    if (!IS_ERR(trstc1)) {
+        reset_control_assert(trstc1);
+        reset_control_deassert(trstc1);
+    }
+    trstc2 = of_reset_control_get(np, "trst2");
+    if (!IS_ERR(trstc2)) {
+        reset_control_assert(trstc2);
+        reset_control_deassert(trstc2);
+    }
+    trstc3 = of_reset_control_get(np, "trst3");
+    if (!IS_ERR(trstc3)) {
+        reset_control_assert(trstc3);
+        reset_control_deassert(trstc3);
+    }
+    trstc4 = of_reset_control_get(np, "trst4");
+    if (!IS_ERR(trstc4)) {
+        reset_control_assert(trstc4);
+        reset_control_deassert(trstc4);
+    }
+    trstc5 = of_reset_control_get(np, "trst5");
+    if (!IS_ERR(trstc5)) {
+        reset_control_assert(trstc5);
+        reset_control_deassert(trstc5);
+    }
+    trstc6 = of_reset_control_get(np, "trst6");
+    if (!IS_ERR(trstc6)) {
+        reset_control_assert(trstc6);
+        reset_control_deassert(trstc6);
+    }
+    trstc7 = of_reset_control_get(np, "trst7");
+    if (!IS_ERR(trstc7)) {
+        reset_control_assert(trstc7);
+        reset_control_deassert(trstc7);
+    }
+    prstc = of_reset_control_get(np, "prst");
+    if (!IS_ERR(prstc)) {
+        reset_control_assert(prstc);
+        reset_control_deassert(prstc);
+    }
+
+    /*
+    * Not all implementations use a peripheral clock, so don't panic
+    * if it's not present
+    */
+    pclk = of_clk_get_by_name(np, "pclk");
+    if (!IS_ERR(pclk))
+        if (clk_prepare_enable(pclk))
+            pr_warn("pclk for %pOFn is present, but could not be activated\n",np);
+    timer_aclk = of_clk_get_by_name(np, "timer_aclk");
+    if (!IS_ERR(timer_aclk))
+        if (clk_prepare_enable(timer_aclk))
+            pr_warn("timer_aclk for %pOFn is present, but could not be activated\n",np);
+    timer3_clk8 = of_clk_get_by_name(np, "timer3_clk8");
+    if (!IS_ERR(timer3_clk8))
+        if (clk_prepare_enable(timer3_clk8))
+            pr_warn("timer3_clk8 for %pOFn is present, but could not be activated\n",np);
+
+    return 0;
+}
+
+static int timer_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    struct resource *res;
+    u64 base;
+    if (perf_timer[0] == NULL) {
+        return -EIO;
+    }
+
+    res = perf_timer[0]->mem;
+
+    base = res->start + perf_timer[0]->perf_count * 0x14;
+    remap_pfn_range(vma, vma->vm_start, base >> 12,
+            vma->vm_end - vma->vm_start, vma->vm_page_prot);
+    return 0;
+}
+
+static int timer_mmap_die1(struct file *file, struct vm_area_struct *vma)
+{
+    struct resource *res;
+    u64 base;
+    if (perf_timer[1] == NULL) {
+        return -EIO;
+    }
+
+    res = perf_timer[1]->mem;
+
+    base = res->start + perf_timer[1]->perf_count * 0x14;
+    remap_pfn_range(vma, vma->vm_start, base >> 12,
+            vma->vm_end - vma->vm_start, vma->vm_page_prot);
+    return 0;
+}
+
+static struct file_operations timer_fops[2] = {
+    {
+        .owner = THIS_MODULE,
+        .mmap = timer_mmap,
+    },
+    {
+        .owner = THIS_MODULE,
+        .mmap = timer_mmap_die1,
+    },
+};
+
+static struct miscdevice timer_misc[2] = {
+    {
+        .minor = MISC_DYNAMIC_MINOR,
+        .name = "perf_count",
+        .fops = &timer_fops[0],
+    },
+    {
+        .minor = MISC_DYNAMIC_MINOR,
+        .name = "perf_count_die1",
+        .fops = &timer_fops[1],
+    },
+};
+
+/*
+ * use timer0 channel 7 for performance statics counter.
+ *
+ * This counter freq is 24MHz, and max cnt value is 0xFFFF_FFFF, it decrease one every timer_clk, and when
+ * decrease zero, it will load the max cnt, and repeat this.
+ *
+ * resolution: about 42ns per cnt. So the max time that will not overflow is 42 * 0xFFFF_FFFF ~= 180s
+ */
+
+u32 get_perf_timer_cnt(u32 numa_id)
+{
+    return APBTMR_MAX_CNT - readl(perf_cnt_base[numa_id]);
+}
+EXPORT_SYMBOL(get_perf_timer_cnt);
+
+static int init_timer_perf_counter(struct eswin_timer *time, u32 chan)
+{
+    int ret;
+
+    time->perf_count = chan;
+    eswin_writel(time, APBTMR_MAX_CNT, chan * APBTMR_EACH_OFS + APBTMR_N_LOAD_COUNT);
+    eswin_writel(time, 0x7, chan * APBTMR_EACH_OFS + APBTMR_N_CONTROL);
+    ret = misc_register(&timer_misc[time->numa_id]);
+    perf_timer[time->numa_id] = time;
+    perf_cnt_base[time->numa_id] = perf_timer[time->numa_id]->mmio_base + chan * APBTMR_EACH_OFS + APBTMR_N_CURRENT_VALUE;
+    return 0;
+}
+
+static int eswin_timer_probe(struct platform_device *pdev)
+{
+    struct device_node *np = pdev->dev.of_node;
+    struct eswin_timer *time;
+    struct resource *res;
+    int error, irq, ret;
+    u32 val;
+
+    dev_info(&pdev->dev, "eswin_timer_probe\n");
+    /*add eswin timer*/
+    time = devm_kzalloc(&pdev->dev, sizeof(struct eswin_timer), GFP_KERNEL);
+    if (!time)
+        return -ENOMEM;
+
+    res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+    if (!res) {
+        dev_err(&pdev->dev, "failed to get register memory\n");
+        return -EINVAL;
+    }
+    time->mem = res;
+
+    ret = device_property_read_u32(&pdev->dev, "numa-node-id", &time->numa_id);
+	if (0 != ret) {
+		dev_err(&pdev->dev, "failed to get numa node id\n");
+		return ret;
+	}
+
+    time->mmio_base = devm_ioremap_resource(&pdev->dev, res);
+    if (IS_ERR(time->mmio_base))
+        return PTR_ERR(time->mmio_base);
+
+    irq = platform_get_irq(pdev, 0);
+    if (irq < 0)
+        return -ENXIO;
+
+    error = devm_request_irq(&pdev->dev, irq, timer_irq_handler, 0,
+                                pdev->name, time);
+    if (error) {
+        dev_err(&pdev->dev, "could not request IRQ %d\n", irq);
+        return error;
+    }
+
+    ret = timer_init(np);
+    if (ret)
+        return ret;
+
+    time->perf_count = 0xff;
+
+    ret = of_property_read_u32(np, "perf_count", &val);
+    if (!ret) {
+        init_timer_perf_counter(time, val);
+    }
+
+    dev_info(&pdev->dev, "eswin_timer_probe success\n");
+    return 0;
+}
+
+static int eswin_timer_remove(struct platform_device *pdev)
+{
+    return -EBUSY; /* cannot unregister clockevent */
+}
+
+static const struct of_device_id eswin_timer_of_match[] = {
+    { .compatible = "eswin,eswin-timer", },
+    {},
+};
+MODULE_DEVICE_TABLE(of, eswin_timer_of_match);
+
+static struct platform_driver eswin_timer_driver = {
+    .probe  = eswin_timer_probe,
+    .remove = eswin_timer_remove,
+    .driver = {
+        .name = "eswin-timer",
+        .of_match_table = of_match_ptr(eswin_timer_of_match),
+    },
+};
+module_platform_driver(eswin_timer_driver);
+
+MODULE_ALIAS("platform:eswin-timer");
+MODULE_DESCRIPTION("eswin timer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index c761952f0dc6..ab4df516eff4 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -796,5 +796,6 @@ config CRYPTO_DEV_SA2UL
 
 source "drivers/crypto/aspeed/Kconfig"
 source "drivers/crypto/starfive/Kconfig"
+source "drivers/crypto/eswin/Kconfig"
 
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index d859d6a5f3a4..14ce58d6cc90 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -51,3 +51,4 @@ obj-y += hisilicon/
 obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/
 obj-y += intel/
 obj-y += starfive/
+obj-y += eswin/
diff --git a/drivers/crypto/eswin/Kconfig b/drivers/crypto/eswin/Kconfig
new file mode 100644
index 000000000000..bbd9153e7368
--- /dev/null
+++ b/drivers/crypto/eswin/Kconfig
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config CRYPTO_DEV_ESWIN_EIC770x
+	tristate "Support for ESWIN EIC7X crypto block cipher accelerator"
+	depends on ARCH_ESWIN_EIC770X_SOC_FAMILY
+	help
+	  Support for ESWIN SEC Engine in EIC770X SoC
+
+	  To compile this as a module, choose M here: the module
+	  will be called es_cipher.
diff --git a/drivers/crypto/eswin/Makefile b/drivers/crypto/eswin/Makefile
new file mode 100644
index 000000000000..a77ccb139b9d
--- /dev/null
+++ b/drivers/crypto/eswin/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_ESWIN_EIC770x) += es_cipher.o
+es_cipher-y = eswin-ipc-scpu.o
diff --git a/drivers/crypto/eswin/eswin-ipc-scpu.c b/drivers/crypto/eswin/eswin-ipc-scpu.c
new file mode 100644
index 000000000000..49c770254661
--- /dev/null
+++ b/drivers/crypto/eswin/eswin-ipc-scpu.c
@@ -0,0 +1,2536 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN cipher serivce driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Min Lin <linmin@eswincomputing.com>
+ */
+
+#include <linux/types.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/random.h>
+#include <uapi/linux/dma-heap.h>
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/mem_perf_api.h>
+#include <linux/delay.h>
+#include <linux/miscdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/debugfs.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
+#include <linux/wait.h>
+#include <linux/idr.h>
+#include <linux/dma-map-ops.h>
+#include <asm/smp.h>
+#include <linux/hw_random.h>
+#include <linux/random.h>
+
+#include <linux/eswin-win2030-sid-cfg.h>
+#include <linux/dmabuf-heap-import-helper.h>
+#include <linux/eswin_cpuid_hartid_convert.h>
+#include <uapi/linux/eswin-ipc-scpu.h>
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+#define LOG_PRINT_DATA_DEBUG_EN	(0)
+
+#ifndef __UNION32
+#define __UNION32(MSB, SEC, THD, LSB)                                          \
+	((((MSB)&0xff) << 24) | (((SEC)&0xff) << 16) | (((THD)&0xff) << 8) |   \
+	 (((LSB)&0xff)))
+#endif
+
+#define MAX_RX_TIMEOUT (msecs_to_jiffies(30000))
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+
+#define IPC_SERVICE_REQ_MAX_LEN sizeof(req_service_t)
+#define IPC_SERVICE_RES_MAX_LEN sizeof(res_service_t)
+
+
+#define IPC_K_HANDLE_MAX_ID	INT_MAX
+
+
+
+typedef struct usermem_iova_info {
+	struct sg_table table;
+	struct list_head node;
+}usermem_iova_info_t;
+
+typedef struct ipc_private_data {
+	struct heap_root hp_root;
+	struct list_head usermem_iova_list;
+	struct mutex usermem_iova_lock;
+	struct idr cipher_mem_resource_idr; // protected by idr_cipher_mem_lock
+	struct mutex idr_cipher_mem_lock;
+}ipc_private_data_t;
+
+struct ipc_session {
+	struct miscdevice miscdev;
+	struct hwrng rng;
+	struct mutex lock;
+	struct mbox_client client;
+	struct mbox_chan *mbox_channel;
+	struct dma_allocation_data send_buff;
+	wait_queue_head_t waitq;
+	u32 num;
+	u8 *req_msg;
+	u32 res_size;
+	__u32 kinfo_cnt;
+	int kinfo_id[MAX_NUM_K_DMA_ALLOC_INFO];
+
+	struct mutex idr_lock;
+	struct idr handle_idr;
+
+	struct workqueue_struct *work_q;
+	struct work_struct session_work;
+	#ifdef ES_CIPHER_QEMU_DEBUG
+	struct delayed_work d_session_work;
+	#endif
+
+	ipc_private_data_t *ipc_priv;
+
+	res_service_t res_srvc;
+	atomic_t receive_data_ready;
+	atomic_t ipc_service_ready;
+};
+
+typedef struct process_data {
+	struct list_head node;
+	int id;
+}process_data_t;
+
+typedef struct process_data_list {
+	struct list_head list; // protected by lock
+	struct mutex lock;
+	struct ipc_session *session;
+}process_data_list_t;
+
+typedef struct cipher_mem_resource_info {
+	struct dma_buf *dma_buf;
+	khandle_dma_allocation_data_t k_dma_info;
+	struct ipc_session *session;
+	struct kref refcount;
+}cipher_mem_resource_info_t;
+
+// static res_service_t res_srvc;
+static DEFINE_MUTEX(dmabuf_bank_lock);
+
+static u32 req_size[SRVC_TYPE_MAX + 2] = {
+	sizeof(signature_validation_check_req_t),
+	sizeof(image_decryption_req_t),
+	sizeof(firmware_download_req_t),
+	sizeof(pubkey_download_req_t),
+	sizeof(rsa_encdec_req_t),
+	sizeof(ecdh_key_req_t),
+	sizeof(aes_sm4_encdec_req_t),
+	sizeof(sha1_sha256_sm3_digest_req_t),
+	sizeof(hmac_req_t),
+	sizeof(otp_req_t),
+	sizeof(trng_req_t),
+	sizeof(addr_region_protection_req_t),
+	sizeof(downloadable_init_req_t),
+	sizeof(basicio_req_t),
+	sizeof(axprot_req_t),
+	sizeof(downloadable_destory_req_t),
+	sizeof(downloadable_ioctl_req_t),
+};
+
+/* Declaration of local functions */
+static void ipc_release_mem_khandle_fn(struct kref *ref);
+
+static int ipc_add_cipher_mem_rsc_info_to_idr(struct ipc_session *session, cipher_mem_resource_info_t *pstCipher_mem_rsc_info);
+static void ipc_remove_cipher_mem_rsc_info_from_idr(struct ipc_session *session, u32 id);
+static cipher_mem_resource_info_t *ipc_find_cipher_mem_rsc_info(struct ipc_session *session, u32 id);
+
+static int ipc_session_mem_info_get(struct ipc_session *session, cipher_create_handle_req_t *pstCreate_handle_req);
+static void ipc_session_mem_info_put(struct ipc_session *session);
+
+/*
+static struct page *dma_common_vaddr_to_page(void *cpu_addr)
+{
+	if (is_vmalloc_addr(cpu_addr))
+		return vmalloc_to_page(cpu_addr);
+	return virt_to_page(cpu_addr);
+}
+*/
+static int ipc_usermem_iova_alloc(struct ipc_session *session, u64 addr, size_t len, dma_addr_t *dma_addr_p)
+{
+	struct device *dev = session->miscdev.parent;
+	ipc_private_data_t *ipc_priv =session->ipc_priv;
+	struct page **pages;
+	u32 offset, nr_pages, i;
+	u64 first, last;
+	usermem_iova_info_t *req_data_addr_i;
+	struct sg_table *table;
+	int ret = -ENOMEM;
+
+	if (!len) {
+		dev_err(dev, "invalid userptr size.\n");
+		return -EINVAL;
+	}
+	/* offset into first page */
+	offset = offset_in_page(addr);
+
+	/* Calculate number of pages */
+	first = (addr & PAGE_MASK) >> PAGE_SHIFT;
+	last  = ((addr + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
+	nr_pages = last - first + 1;
+	dev_dbg(dev, "%s, addr=0x%llx, nr_pages=%d(fist:0x%llx,last:0x%llx), len=0x%lx\n",
+		__func__, addr, nr_pages, first, last, len);
+
+	/* alloc array to storing the pages */
+	pages = kvmalloc_array(nr_pages,
+				   sizeof(struct page *),
+				   GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	/* alloc usermem iova node for storing the iova info(sgt) */
+	req_data_addr_i = kzalloc(sizeof(*req_data_addr_i), GFP_KERNEL);
+	if (!req_data_addr_i) {
+		ret = -ENOMEM;
+		goto free_pages_list;
+	}
+
+	/* Get user pages into memory */
+	ret = get_user_pages_fast(addr & PAGE_MASK,
+				nr_pages,
+				FOLL_FORCE | FOLL_WRITE,
+				pages);
+	if (ret != nr_pages) {
+		nr_pages = (ret >= 0) ? ret : 0;
+		dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
+			ret, nr_pages);
+		ret = ret < 0 ? ret : -EINVAL;
+		goto free_usermem_iova_i;
+	}
+
+	table = &req_data_addr_i->table;
+	ret = sg_alloc_table_from_pages(table,
+					pages,
+					nr_pages, offset,
+					len,
+					GFP_KERNEL);
+	if (ret) {
+		dev_err(dev, "sg_alloc_table_from_pages, err=%d\n", ret);
+		goto free_user_pages;
+	}
+
+	ret = dma_map_sgtable(dev, table, DMA_BIDIRECTIONAL, 0);
+	if (ret) {
+		dev_err(dev, "dma_map_sgtable, err=%d\n", ret);
+		ret = -ENOMEM;
+		goto free_sg_table;
+	}
+	*dma_addr_p = sg_dma_address(table->sgl);
+	mutex_lock(&ipc_priv->usermem_iova_lock);
+	list_add(&req_data_addr_i->node, &ipc_priv->usermem_iova_list);
+	mutex_unlock(&ipc_priv->usermem_iova_lock);
+
+	dev_dbg(dev, "%s, dmaaddr=0x%llx(phys:0x%llx)\n",
+		__func__, sg_dma_address(table->sgl), sg_phys(table->sgl));
+
+	kfree(pages);
+
+	return 0;
+
+free_sg_table:
+	sg_free_table(table);
+free_user_pages:
+	for (i = 0; i < nr_pages; i++)
+		put_page(pages[i]);
+free_usermem_iova_i:
+	kfree(req_data_addr_i);
+free_pages_list:
+	kvfree(pages);
+
+	return ret;
+}
+
+static int do_ipc_usermem_iova_free(struct ipc_session *session, struct sg_table *table)
+{
+	struct device *dev = session->miscdev.parent;
+	struct sg_page_iter piter;
+
+	dev_dbg(dev, "%s, dmaAddr=0x%llx, phys=0x%llx\n", __func__, sg_dma_address(table->sgl), sg_phys(table->sgl));
+
+	dma_unmap_sgtable(dev, table, DMA_TO_DEVICE, 0);
+
+	for_each_sgtable_page(table, &piter, 0) {
+		struct page *page = sg_page_iter_page(&piter);
+		put_page(page);
+	}
+	sg_free_table(table);
+
+	return 0;
+}
+
+static int ipc_usermem_iova_free(struct ipc_session *session, dma_addr_t dma_addr)
+{
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+	usermem_iova_info_t *req_data_addr_i, *tmp;
+	int ret = -EINVAL;
+
+	mutex_lock(&ipc_priv->usermem_iova_lock);
+	list_for_each_entry_safe(req_data_addr_i, tmp, &ipc_priv->usermem_iova_list, node) {
+		struct sg_table *table = &req_data_addr_i->table;
+		if (dma_addr == sg_dma_address(table->sgl)) {
+			do_ipc_usermem_iova_free(session, table);
+			list_del(&req_data_addr_i->node);
+			kfree(req_data_addr_i);
+			ret = 0;
+			pr_debug("%s, free dma_addr=0x%llx\n", __func__, dma_addr);
+			break;
+		}
+	}
+	mutex_unlock(&ipc_priv->usermem_iova_lock);
+
+	return ret;
+}
+
+static int ipc_dmabuf_iova_alloc(struct ipc_session *session, struct dma_buf *dma_buf, u64 offset, size_t len, dma_addr_t *dma_addr_p)
+{
+	struct device *dev = session->miscdev.parent;
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+	struct heap_mem *hmem = NULL;
+	struct heap_root *hp_root = &ipc_priv->hp_root;
+
+	/* a:hold ,hold the dmabuf untill this function exit */
+	get_dma_buf(dma_buf);
+
+	/* b: add refcount since a new dmabuf_fd was created  and associated with the user process that is calling this function.
+	 * user program must call close(dmabuf_fd) explicitly to decrease file_count(dambuf->file), so that the dmabuf can be finally released
+	*/
+	// get_dma_buf(dmabuf);
+
+	/* the refcount will be increased by common_dmabuf_heap_import_from_user, and decrease when ipc_dmabuf_iova_free is called */
+	hmem = common_dmabuf_heap_import_from_user_with_dma_buf_st(hp_root, dma_buf);
+	if(IS_ERR(hmem)) {
+		dev_err(dev, "dmabuf-heap alloc from userspace failed\n");
+		dma_buf_put(dma_buf); // a:put, put the dmabuf back
+		return -ENOMEM;
+	}
+
+	*dma_addr_p = sg_dma_address(hmem->sgt->sgl) + offset;
+
+	dma_buf_put(dma_buf); // a:put, put the dmabuf back
+
+	dev_dbg(dev, "%s, dma_addr=0x%llx(phys:0x%llx)\n", __func__, *dma_addr_p, sg_phys(hmem->sgt->sgl));
+
+	return 0;
+}
+
+static int ipc_dmabuf_iova_free(struct ipc_session *session, struct dma_buf *dma_buf)
+{
+	struct heap_mem *hmem = NULL;
+	struct heap_root *hp_root = &session->ipc_priv->hp_root;
+
+	pr_debug("---%s:%d\n", __func__, __LINE__);
+	hmem = common_dmabuf_lookup_heapobj_by_dma_buf_st(hp_root, dma_buf);
+	if (IS_ERR(hmem)) {
+		pr_err("cannot find dmabuf-heap for dma_buf 0x%px\n", dma_buf);
+		return -EINVAL;
+	}
+
+	common_dmabuf_heap_release(hmem);
+
+	return 0;
+}
+static int ipc_req_data_iova_alloc(struct ipc_session *session, cipher_mem_resource_info_t *pstCipher_mem_rsc_info)
+{
+	struct device *dev = session->miscdev.parent;
+	struct vm_area_struct *vma = NULL;
+	vm_flags_t vm_flags;
+	struct mm_struct *mm = current->mm;
+	u64 offset = 0;
+	bool is_dmabuf = false;
+	struct dma_buf *dma_buf = NULL;
+	int ret = 0;
+	struct dma_allocation_data *pstDma_alloc_info = &pstCipher_mem_rsc_info->k_dma_info.dma_alloc_info;
+	u64 addr = (u64)pstDma_alloc_info->cpu_vaddr;
+	size_t len = pstDma_alloc_info->size;
+	dma_addr_t dma_addr;
+
+	/* */
+	mmap_read_lock(mm);
+	vma = vma_lookup(mm, addr & PAGE_MASK);
+	if (!vma) {
+		dev_err(dev, "%s, vma_lookup failed!\n", __func__);
+		return -EFAULT;
+	}
+	vm_flags = vma->vm_flags;
+	mmap_read_unlock(mm);
+
+	if (vm_flags & (VM_IO | VM_PFNMAP)) {
+		dev_dbg(dev, "%s, vm_flags=0x%lx, Page-ranges managed without struct page, just pure PFN!\n", __func__, vm_flags);
+		is_dmabuf = true;
+	}
+	dev_dbg(dev, "%s, is_dmabuf = %s. vm_start=0x%lx, vm_end=0x%lx, size=0x%lx\n", __func__,
+		is_dmabuf==0?"false":"true", vma->vm_start, vma->vm_end, (vma->vm_end - vma->vm_start));
+
+	/* check the validation of addr and len */
+	if (addr + len > vma->vm_end) {
+		dev_err(dev, "%s, Err,addr +len exceed the end_addr of the buffer!\n", __func__);
+		return -EINVAL;
+	}
+
+	if(is_dmabuf == false) {
+		ret = ipc_usermem_iova_alloc(session, addr, len, &dma_addr);
+		if (ret) {
+			dev_dbg(dev, "ipc_usermem_iova_alloc, failed!, ErrCode:%d\n", ret);
+			return ret;
+		}
+	}
+	else {
+		dma_buf = vma->vm_private_data;
+		if (NULL == dma_buf) {
+			dev_err(dev, "%s, Err, Can't find dmabuf!\n", __func__);
+			return -EFAULT;
+		}
+
+		offset = addr - vma->vm_start;
+		ret = ipc_dmabuf_iova_alloc(session, dma_buf, offset, len, &dma_addr);
+		if (ret) {
+			dev_dbg(dev, "ipc_dmabuf_iova_alloc, failed!, ErrCode:%d\n", ret);
+			return ret;
+		}
+	}
+
+	pstCipher_mem_rsc_info->dma_buf = dma_buf;
+	pstDma_alloc_info->dma_addr = dma_addr;
+	return 0;
+}
+
+static int ipc_req_data_iova_free(struct ipc_session *session, cipher_mem_resource_info_t *pstCipher_mem_rsc_info)
+{
+	struct device *dev = session->miscdev.parent;
+	dma_addr_t dma_addr = pstCipher_mem_rsc_info->k_dma_info.dma_alloc_info.dma_addr;
+	struct dma_buf *dma_buf = pstCipher_mem_rsc_info->dma_buf;
+	int ret = -EINVAL;
+
+	if (!dma_buf) {
+		ret = ipc_usermem_iova_free(session, dma_addr);
+		if (ret) {
+			dev_err(dev, "faild to free usermem iova:0x%llx, ErrCode:%d!\n", dma_addr, ret);
+		}
+	}
+	else {
+		ret = ipc_dmabuf_iova_free(session, dma_buf);
+		if (ret) {
+			dev_err(dev, "faild to free dmabuf iova:0x%llx, ErrCode:%d!\n", dma_addr, ret);
+		}
+	}
+
+	return ret;
+}
+
+static int ipc_assign_handle_id(struct ipc_session *session, req_service_t *service_req, u32 *handle_id)
+{
+	int ret = 0;
+
+	mutex_lock(&session->idr_lock);
+	ret = idr_alloc(&session->handle_idr, service_req, 0, IPC_K_HANDLE_MAX_ID,
+			GFP_KERNEL);
+	if (ret >= 0) {
+		*handle_id = ret;
+	}
+	mutex_unlock(&session->idr_lock);
+
+	return ret < 0 ? ret : 0;
+}
+
+static int ipc_create_handle(struct ipc_session *session, req_service_t *in_service_req, u32 *handle_id)
+{
+	int ret = 0;
+	struct device *dev =  session->miscdev.parent;
+	req_service_t *pService_req = NULL;
+
+	if (in_service_req == NULL)
+		return -EINVAL;
+
+	pService_req = kzalloc(sizeof(*pService_req), GFP_KERNEL);
+	if (!pService_req) {
+		dev_err(dev, "failed to create handle, err(%d)\n", ret);
+		return -ENOMEM;
+	}
+	memcpy(pService_req, in_service_req, sizeof(*pService_req));
+
+	ret = ipc_assign_handle_id(session, pService_req, handle_id);
+	if (ret < 0) {
+		dev_err(dev, "failed to assign handle id, err(%d)\n", ret);
+		goto  free_service_req;
+	}
+
+	return 0;
+
+free_service_req:
+	kfree(pService_req);
+	return ret;
+}
+
+static int ipc_find_handle(struct ipc_session *session, u32 handle_id, req_service_t **ppService_req)
+{
+	int ret = 0;
+	struct device *dev =  session->miscdev.parent;
+	req_service_t *pService_req;
+
+	mutex_lock(&session->idr_lock);
+	pService_req = idr_find(&session->handle_idr, handle_id);
+	if (!pService_req) {
+		dev_err(dev, "failed to find handle %d\n", handle_id);
+		ret = -EINVAL;
+	}
+	else {
+		*ppService_req = pService_req;
+	}
+	mutex_unlock(&session->idr_lock);
+
+	return ret;
+}
+
+static int ipc_destroy_handle(struct ipc_session *session, u32 handle_id)
+{
+	int ret = 0;
+	struct device *dev =  session->miscdev.parent;
+	req_service_t *pService_req;
+	aes_sm4_encdec_req_t *pSymm_req;
+
+	mutex_lock(&session->idr_lock);
+	pService_req = idr_find(&session->handle_idr, handle_id);
+	if (!pService_req) {
+		dev_err(dev, "failed to find handle %d\n", handle_id);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	pSymm_req = &pService_req->data.symm_crypto_req;
+	dev_dbg(dev, "pSymm_req->crypto_mode=%d\n", pSymm_req->flag.crypto_mode);
+	dev_dbg(dev, "pSymm_req->keySrc=%d\n", pSymm_req->flag.keyid);
+	dev_dbg(dev, "pSymm_req->key_size=%d\n", pSymm_req->key_size);
+	dev_dbg(dev, "pSymm_req->iv_size=%d\n", pSymm_req->iv_size);
+
+	idr_remove(&session->handle_idr, handle_id);
+	mutex_unlock(&session->idr_lock);
+
+	kfree(pService_req);
+	return 0;
+
+out:
+	mutex_unlock(&session->idr_lock);
+
+	return ret;
+}
+
+static int ipc_ioctl_create_handle(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	unsigned int cmd_size = 0;
+	// cipher_create_handle_req_t create_handle_req;
+	cipher_create_handle_req_t *pstCreate_handle_req;
+	req_service_t *pService_req;
+	u32 handle_id;
+	int ret = 0;
+
+	// pr_debug("---%s:%d---\n", __func__, __LINE__);
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(cipher_create_handle_req_t)) {
+		return -EFAULT;
+	}
+
+	pstCreate_handle_req = kzalloc(sizeof(*pstCreate_handle_req), GFP_ATOMIC); // GFP_KERNEL
+	if (!pstCreate_handle_req) {
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pstCreate_handle_req, user_arg, sizeof(cipher_create_handle_req_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE;
+	}
+
+	pService_req = &pstCreate_handle_req->service_req;
+	ret = ipc_create_handle(session, pService_req, &handle_id);
+	if (ret < 0) {
+		goto OUT_FREE;
+	}
+
+	pstCreate_handle_req->handle_id = handle_id;
+
+	if (copy_to_user(user_arg, pstCreate_handle_req, sizeof(cipher_create_handle_req_t))) {
+		ipc_destroy_handle(session, handle_id);
+		ret = -EFAULT;
+	}
+
+OUT_FREE:
+	kfree(pstCreate_handle_req);
+
+	dev_dbg(dev, "%s, ret=%d!\n", __func__, ret);
+	return ret;
+}
+
+static int ipc_ioc_destroy_handle(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	unsigned int cmd_size = 0;
+	cipher_create_handle_req_t *pstCreate_handle_req;
+	int ret = 0;
+
+	// pr_debug("---%s:%d---\n", __func__, __LINE__);
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(cipher_create_handle_req_t)) {
+		return -EFAULT;
+	}
+
+	pstCreate_handle_req = kzalloc(sizeof(*pstCreate_handle_req), GFP_ATOMIC); // GFP_KERNEL
+	if (!pstCreate_handle_req) {
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pstCreate_handle_req, user_arg, sizeof(cipher_create_handle_req_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE;
+	}
+
+	ret = ipc_destroy_handle(session, pstCreate_handle_req->handle_id);
+	if (ret < 0) {
+		goto OUT_FREE;
+	}
+
+OUT_FREE:
+	kfree(pstCreate_handle_req);
+	dev_dbg(dev, "%s, ret=%d!\n", __func__, ret);
+
+	return ret;
+}
+
+static int ipc_ioc_get_handle_config(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	unsigned int cmd_size = 0;
+	cipher_create_handle_req_t *pstCreate_handle_req;
+	req_service_t *pService_req;
+	int ret = 0;
+
+	// pr_debug("---%s:%d---\n", __func__, __LINE__);
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(cipher_create_handle_req_t)) {
+		return -EFAULT;
+	}
+
+	pstCreate_handle_req = kzalloc(sizeof(*pstCreate_handle_req), GFP_ATOMIC); // GFP_KERNEL
+	if (!pstCreate_handle_req) {
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pstCreate_handle_req, user_arg, sizeof(cipher_create_handle_req_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE;
+	}
+
+	ret = ipc_find_handle(session, pstCreate_handle_req->handle_id, &pService_req);
+	if (ret < 0) {
+		return ret;
+	}
+	memcpy(&pstCreate_handle_req->service_req, pService_req, sizeof(*pService_req));
+	if (copy_to_user(user_arg, pstCreate_handle_req, sizeof(cipher_create_handle_req_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE;
+	}
+
+OUT_FREE:
+	kfree(pstCreate_handle_req);
+	dev_dbg(dev, "%s, ret=%d!\n", __func__, ret);
+
+	return ret;
+}
+
+static int ipc_ioc_update_handle_config(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	unsigned int cmd_size = 0;
+	cipher_create_handle_req_t *pstCreate_handle_req;
+	req_service_t *pService_req;
+	int ret = 0;
+
+	// pr_debug("---%s:%d---\n", __func__, __LINE__);
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(cipher_create_handle_req_t)) {
+		return -EFAULT;
+	}
+
+	pstCreate_handle_req = kzalloc(sizeof(*pstCreate_handle_req), GFP_ATOMIC); // GFP_KERNEL
+	if (!pstCreate_handle_req) {
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pstCreate_handle_req, user_arg, sizeof(cipher_create_handle_req_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE;
+	}
+
+	ret = ipc_find_handle(session, pstCreate_handle_req->handle_id, &pService_req);
+	if (ret < 0) {
+		goto OUT_FREE;
+	}
+	/* update this idr with the user configuration */
+	memcpy(pService_req, &pstCreate_handle_req->service_req, sizeof(*pService_req));
+
+OUT_FREE:
+	kfree(pstCreate_handle_req);
+	dev_dbg(dev, "%s, ret=%d!\n", __func__, ret);
+
+	return ret;
+}
+
+static char check_xor(char *buf, int size)
+{
+    char tmp = 0;
+    int i = 0;
+
+    for (i = 0; i < size; i++)
+        tmp ^= buf[i];
+
+    return tmp;
+}
+
+
+static int encode_ecdh_key_req(ecdh_key_req_t *in, u8 *out)
+{
+	int len = 0;
+	u32 keylen = 0;
+
+	if (in->flag.opt == OPT_GEN) {
+		len = sizeof(in->flag) + sizeof(ecdh_key_gen_t);
+		memcpy(out, (u8 *)in, len);
+	} else if (in->flag.opt == OPT_AGR) {
+		len = sizeof(in->flag) + sizeof(u32);
+		memcpy(out, (u8 *)in, len);
+
+		keylen = in->data.agr.pubk_remote.keylen;
+		memcpy(out + len, (u8 *)&in->data.agr.pubk_remote.keylen,
+		       sizeof(u32));
+		len += sizeof(u32);
+
+		memcpy(out + len, (u8 *)in->data.agr.pubk_remote.x, keylen);
+		len += keylen;
+
+		memcpy(out + len, (u8 *)in->data.agr.pubk_remote.y, keylen);
+		len += keylen;
+
+		keylen = in->data.agr.privk_local.keylen;
+		memcpy(out + len, (u8 *)&in->data.agr.privk_local.keylen,
+		       sizeof(u32));
+		len += sizeof(u32);
+
+		memcpy(out + len, (u8 *)in->data.agr.privk_local.z, keylen);
+		len += keylen;
+	} else if (in->flag.opt == OPT_DRIV) {
+		len = sizeof(in->flag);
+		memcpy(out, (u8 *)in, len);
+
+		keylen = in->data.driv.seck.keylen;
+		memcpy(out + len, (u8 *)&in->data.driv.seck.keylen,
+		       sizeof(u32));
+		len += sizeof(u32);
+
+		memcpy(out + len, (u8 *)in->data.driv.seck.x, keylen);
+		len += keylen;
+
+		memcpy(out + len, (u8 *)in->data.driv.seck.y, keylen);
+		len += keylen;
+
+		memcpy(out + len, (u8 *)&in->data.driv.rand, 4);
+		len += 4;
+	} else {
+		len = 0;
+	}
+
+	return len;
+}
+
+static int encode_rsa_req(rsa_encdec_req_t *in, u8 *out)
+{
+	int len;
+	int key_len = 0;
+
+	if (in->flag.keyid < EXT_PRIV_KEY)
+		key_len = 0;
+	else if (in->flag.keyid == EXT_PRIV_KEY) {
+		key_len = sizeof(in->t_key.rsa_prvkey.keylen) +
+			  2 * in->t_key.rsa_prvkey.keylen;
+	} else {
+		key_len = sizeof(in->t_key.rsa2048_pubkey.keylen) +
+			  sizeof(in->t_key.rsa2048_pubkey.exponent) +
+			  in->t_key.rsa2048_pubkey.keylen;
+	}
+
+	len = 20 + key_len;
+	memcpy(out, (u8 *)in, len);
+
+	return len;
+}
+
+static int encode_hmac_req(hmac_req_t *in, u8 *out)
+{
+	int len;
+	len = 16 + in->key_size;
+	memcpy(out, (u8 *)in, len);
+
+	return len;
+}
+
+static int encode_symm_req(aes_sm4_encdec_req_t *in, u8 *out)
+{
+	int len;
+
+	len = 20;
+	memcpy(out, (u8 *)in, len);
+	memcpy(out + len, in->key, in->key_size);
+	len += in->key_size;
+	memcpy(out + len, &in->iv_size, 4);
+	len += 4;
+	memcpy(out + len, in->iv, in->iv_size);
+	len += in->iv_size;
+
+	return len;
+}
+
+static int encode_loadable_req(downloadable_init_req_t *in, u8 *out)
+{
+	int len;
+	flag14_t *flag = (flag14_t *)in;
+	if (flag->service_type == 0) {
+		len = sizeof(downloadable_init_req_t);
+	} else if (flag->service_type == 1) {
+		len = sizeof(downloadable_destory_req_t);
+	} else if (flag->service_type == 2) {
+		downloadable_ioctl_req_t *ioctl_req =
+			(downloadable_ioctl_req_t *)in;
+		len = sizeof(downloadable_ioctl_req_t) + ioctl_req->param_size;
+	}
+
+	memcpy((void *)out, (void *)in, len);
+	return len;
+}
+
+static void u32_to_u8(u8 *u8_data, u32 *u32_data)
+{
+	u32 tmp;
+	tmp = *u32_data;
+	*u8_data = (u8)(0x000000ff & tmp);
+	*(u8_data + 1) = (u8)((0x0000ff00 & tmp) >> 8);
+	*(u8_data + 2) = (u8)((0x00ff0000 & tmp) >> 16);
+	*(u8_data + 3) = (u8)((0xff000000 & tmp) >> 24);
+}
+
+static int encode_req_data(u8 *out, SRVC_TYPE service_id, req_data_domain_t *in)
+{
+	int len = 0;
+	if (NULL == out || NULL == in)
+		return len;
+
+	switch (service_id) {
+	case SRVC_TYPE_SIGN_CHECK:
+	case SRVC_TYPE_IMG_DECRPT:
+	case SRVC_TYPE_FIRMWARE_DOWNLOAD:
+	case SRVC_TYPE_TRNG:
+	case SRVC_TYPE_DIGEST:
+	case SRVC_TYPE_ADDR_REGION_PROTECTION:
+	case SRVC_TYPE_BASIC_IO:
+	case SRVC_TYPE_AXPROT:
+	case SRVC_TYPE_PUBKEY_DOWNLOAD:
+	case SRVC_TYPE_OTP_READ_PROGRAM:
+		len = req_size[service_id];
+		memcpy((void *)out, (void *)in, len);
+		break;
+	case SRVC_TYPE_ECDH_KEY:
+		len = encode_ecdh_key_req(&(in->ecdh_key_req), out);
+		break;
+	case SRVC_TYPE_SYM_CRYPT_DECRYPT:
+		len = encode_symm_req(&(in->symm_crypto_req), out);
+		break;
+	case SRVC_TYPE_RSA_CRYPT_DECRYPT:
+		len = encode_rsa_req(&(in->rsa_crypto_req), out);
+		break;
+	case SRVC_TYPE_HMAC:
+		len = encode_hmac_req(&(in->hmac_req), out);
+		break;
+	case SRVC_TYPE_DOWNLOADABLE:
+		len = encode_loadable_req(&(in->dl_init_req), out);
+		break;
+	default:
+		break;
+	}
+	return len;
+}
+
+static int get_req_data_size(u8 service_id, u8 *req_data)
+{
+	int size = 0;
+	unsigned int keylen = 0;
+	switch (service_id) {
+	case SRVC_TYPE_SIGN_CHECK:
+	case SRVC_TYPE_IMG_DECRPT:
+	case SRVC_TYPE_FIRMWARE_DOWNLOAD:
+	case SRVC_TYPE_TRNG:
+	case SRVC_TYPE_ADDR_REGION_PROTECTION:
+	case SRVC_TYPE_BASIC_IO:
+	case SRVC_TYPE_AXPROT:
+	case SRVC_TYPE_PUBKEY_DOWNLOAD:
+	case SRVC_TYPE_DIGEST:
+	case SRVC_TYPE_OTP_READ_PROGRAM:
+		size = req_size[service_id];
+		break;
+	case SRVC_TYPE_DOWNLOADABLE: {
+		flag14_t *flag = (flag14_t *)req_data;
+		if (flag->service_type == 0) {
+			size = req_size[service_id];
+		} else if (flag->service_type == 1) {
+			size = sizeof(downloadable_destory_req_t);
+		} else if (flag->service_type == 2) {
+			downloadable_ioctl_req_t *ioctl_req =
+				(downloadable_ioctl_req_t *)req_data;
+			size = sizeof(downloadable_ioctl_req_t) +
+			       ioctl_req->param_size;
+		}
+	} break;
+
+	case SRVC_TYPE_HMAC: {
+		hmac_req_t *h_req;
+		h_req = (hmac_req_t *)req_data;
+		size = 16 + h_req->key_size;
+	} break;
+
+	case SRVC_TYPE_RSA_CRYPT_DECRYPT: {
+		rsa_encdec_req_t *rsa_req;
+		int key_len = 0;
+		rsa_req = (rsa_encdec_req_t *)req_data;
+		if (rsa_req->flag.keyid < EXT_PRIV_KEY) {
+			key_len = 0;
+		} else if (rsa_req->flag.keyid == EXT_PRIV_KEY) {
+			key_len = sizeof(rsa_req->t_key.rsa_prvkey.keylen) +
+				  2 * rsa_req->t_key.rsa_prvkey.keylen;
+		} else {
+			key_len = sizeof(rsa_req->t_key.rsa2048_pubkey.keylen) +
+				  sizeof(rsa_req->t_key.rsa2048_pubkey.exponent) +
+				  rsa_req->t_key.rsa2048_pubkey.keylen;
+		}
+		size = 20 + key_len;
+	} break;
+
+	case SRVC_TYPE_ECDH_KEY: {
+		ecdh_key_req_t *in = (ecdh_key_req_t *)req_data;
+		if (in->flag.opt == OPT_GEN) {
+			size = sizeof(in->flag) + sizeof(ecdh_key_gen_t);
+		} else if (in->flag.opt == OPT_AGR) {
+			size = sizeof(in->flag) + sizeof(u32);
+			keylen = in->data.agr.pubk_remote.keylen;
+			size += 4 + 2 * keylen;
+			keylen = in->data.agr.privk_local.keylen;
+			size += 4 + keylen;
+		} else if (in->flag.opt == OPT_DRIV) {
+			size = sizeof(in->flag);
+			keylen = in->data.driv.seck.keylen;
+			size += sizeof(u32) + 2 * keylen;
+			size += 4;
+		} else {
+			size = 0;
+		}
+	} break;
+
+	case SRVC_TYPE_SYM_CRYPT_DECRYPT: {
+		aes_sm4_encdec_req_t *in = (aes_sm4_encdec_req_t *)req_data;
+		size = 24 + in->key_size + in->iv_size;
+	} break;
+
+	default:
+		break;
+	}
+
+	pr_debug("get_req_data_size: SRVC_TYPE[%d] size = [0x%x]!\r\n",
+	       service_id, size);
+	if (0 != (size % 4)) {
+		if ((SRVC_TYPE_OTP_READ_PROGRAM != service_id) && (SRVC_TYPE_ECDH_KEY != service_id)) {
+			pr_err("get_req_data_size: SRVC_TYPE[%d] size = [0x%x], not 4 "
+			       "times, error!\r\n",
+			       service_id, size);
+			return 0;
+		}
+	}
+
+	return size;
+}
+
+static inline u32 get_union32_data(u8 *in, u32 offset)
+{
+	u32 ret = 0;
+	ret = __UNION32(*(in + offset + 3), *(in + offset + 2),
+			*(in + offset + 1), *(in + offset));
+	return ret;
+}
+
+static int get_receive_message_size(struct ipc_session *session, u8 *res_msg)
+{
+	int size = 0;
+	res_service_t *tmp;
+	tmp = (res_service_t *)res_msg;
+
+	#if LOG_PRINT_DATA_DEBUG_EN
+	int i = 0;
+
+	pr_info("response data is : \r\n");
+	for (i = 0; i < (tmp->size + 20); i++) {
+		pr_info("0x%02x ! \r\n", res_msg[i]);
+	}
+	#endif
+
+	switch (tmp->service_id) {
+	case SRVC_TYPE_SIGN_CHECK:
+		size = sizeof(signature_validation_check_res_t);
+		break;
+
+	case SRVC_TYPE_IMG_DECRPT:
+		size = sizeof(image_decryption_res_t);
+		break;
+
+	case SRVC_TYPE_FIRMWARE_DOWNLOAD:
+		size = sizeof(firmware_download_res_t);
+		break;
+
+	case SRVC_TYPE_TRNG:
+		size = MAX_TRNG_DATA_LEN;
+		break;
+
+	case SRVC_TYPE_DIGEST:
+		size = sizeof(sha1_sha256_sm3_digest_res_t);
+		break;
+
+	case SRVC_TYPE_ADDR_REGION_PROTECTION:
+		size = sizeof(addr_region_protection_res_t);
+		break;
+
+	case SRVC_TYPE_DOWNLOADABLE:
+		size = sizeof(downloadable_srvc_res);
+		break;
+
+	case SRVC_TYPE_BASIC_IO:
+		size = sizeof(basicio_res_t);
+		break;
+
+	case SRVC_TYPE_AXPROT:
+		size = sizeof(axprot_res_t);
+		break;
+
+	case SRVC_TYPE_SYM_CRYPT_DECRYPT:
+		size = sizeof(aes_sm4_encdec_res_t);
+		break;
+
+	case SRVC_TYPE_OTP_READ_PROGRAM:
+		size = sizeof(otp_res_t);
+		break;
+
+	case SRVC_TYPE_RSA_CRYPT_DECRYPT:
+		size = sizeof(rsa_encdec_res_t);
+		break;
+
+	case SRVC_TYPE_HMAC:
+		size = sizeof(hmac_res_t);
+		break;
+
+	case SRVC_TYPE_ECDH_KEY:
+		size = sizeof(ecdh_key_res_t);
+		break;
+
+	default:
+		break;
+	}
+	size = size + RES_DATA_OFFSET;
+	memcpy(&session->res_srvc, res_msg, size);
+	return size;
+}
+
+#if LOG_PRINT_DATA_DEBUG_EN
+static void msg_print(msg_send_t *msg)
+{
+	int i = 0;
+	pr_info("req num is: %02x.\r\n", msg->num);
+	pr_info("req service_id is: %02x.\r\n", msg->service_id);
+	pr_info("req size is: %02x.\r\n", msg->size);
+	pr_info("packet data is:\r\n");
+	for (i = 0; i < msg->size; i++) {
+		pr_info("0x%02X ", *((char *)msg->data + i));
+		if (((i + 1) & 0xf) == 0)
+			pr_info("\r\n");
+	}
+}
+#else
+#define msg_print(msg)
+#endif
+
+static int encode_send_buffer(u8 *buffer_addr, msg_send_t *msg)
+{
+	int ret = 0;
+	u8 tmp;
+	int i, len;
+
+	if (NULL == msg->data)
+		return -EAGAIN;
+
+	memcpy(buffer_addr + 0, (void *)&msg->fream_head, 4);
+
+	u32_to_u8(buffer_addr + 4, &msg->num);
+	u32_to_u8(buffer_addr + 8, &msg->service_id);
+	u32_to_u8(buffer_addr + 12, &msg->size);
+	len = msg->size;
+
+	memcpy(buffer_addr + 16, msg->data, len);
+	tmp = *buffer_addr;
+	len = 16 + len;
+
+	for (i = 1; i < len; i++)
+		tmp = tmp ^ (*(buffer_addr + i));
+
+	*(buffer_addr + i) = tmp;
+
+	return ret;
+}
+
+static void fill_and_slice_regdata(mbox_register_data *reg_data,
+				   dma_addr_t shm_dma_addr, u8 *out)
+{
+	u8 xor ;
+	int i;
+	int len;
+	u8 *tmp;
+
+	memset(reg_data, 0, sizeof(mbox_register_data));
+	memset(out, 0, 8);
+
+	reg_data->fream_head = FRAME_HEAD_U84;
+	reg_data->fream_len = (u8)0x04;
+	reg_data->ipc_addr[0] = (u8)(0x000000ff & shm_dma_addr);
+	reg_data->ipc_addr[1] = (u8)((0x0000ff00 & shm_dma_addr) >> 8);
+	reg_data->ipc_addr[2] = (u8)((0x00ff0000 & shm_dma_addr) >> 16);
+	reg_data->ipc_addr[3] = (u8)((0xff000000 & shm_dma_addr) >> 24);
+
+	len = sizeof(mbox_register_data) - 1;
+	tmp = (u8 *)reg_data;
+	xor = *(tmp + 0);
+	for (i = 1; i < len; i++)
+		xor = xor^(*(tmp + i));
+	*(tmp + i) = xor;
+
+	for (i = 0; i < 7; i++)
+		*(out + i) = *(tmp + i);
+	*(out + i) = 0x80;
+
+	return;
+}
+
+static int get_shm_addr(struct ipc_session *session)
+{
+	int ret = 0;
+	struct device *dev = session->miscdev.parent;
+	session->send_buff.size = sizeof(req_data_domain_t);
+	session->send_buff.cpu_vaddr =
+		dma_alloc_coherent(dev, session->send_buff.size,
+				   &session->send_buff.dma_addr, GFP_KERNEL);
+	if (NULL == session->send_buff.cpu_vaddr) {
+		dev_err(dev, "dma_alloc_coherent failed\n");
+		ret = -ENOMEM;
+	}
+
+	return ret;
+}
+
+static void free_shm_addr(struct ipc_session *session)
+{
+	struct device *dev = session->miscdev.parent;
+
+	dma_free_coherent(dev, session->send_buff.size,
+			  session->send_buff.cpu_vaddr,
+			  session->send_buff.dma_addr);
+}
+
+static int get_send_reg_data(u8 *in, u8 *out, struct ipc_session *session)
+{
+	int ret, len;
+	#if LOG_PRINT_DATA_DEBUG_EN
+	int k;
+	#endif
+	struct dma_allocation_data tmp;
+	msg_send_t msg_send;
+	req_service_t *service_req = (req_service_t *)in;
+	mbox_register_data reg_data;
+	struct device *dev = session->miscdev.parent;
+
+	session->num++;
+	msg_send.fream_head = PREAM_REQ;
+	msg_send.num = session->num;
+	msg_send.service_id = service_req->serivce_type;
+	msg_send.size = get_req_data_size(msg_send.service_id,
+					  (u8 *)(in + SERVICE_ID_OFFSET));
+
+	dev_dbg(dev, "msg_send.size:0x%d.\r\n", msg_send.size);
+	if (0 >= msg_send.size) {
+		dev_err(dev, "msg_send.size is error.\r\n");
+		return -EINVAL;
+	}
+
+	tmp.cpu_vaddr = kzalloc(msg_send.size, GFP_KERNEL);
+	if (NULL == tmp.cpu_vaddr) {
+		dev_err(dev, "kzalloc failed\n");
+		return -ENOMEM;
+	}
+
+	msg_send.data = (u8 *)tmp.cpu_vaddr;
+	len = encode_req_data(msg_send.data, msg_send.service_id,
+			      (req_data_domain_t *)(in + SERVICE_ID_OFFSET));
+	if (len != msg_send.size) {
+		dev_err(dev, "encode_req_data error!\r\n");
+		ret = -EAGAIN;
+		goto out;
+	}
+
+	msg_print(&msg_send);
+
+	ret = encode_send_buffer(session->send_buff.cpu_vaddr, &msg_send);
+	if (ret < 0) {
+		dev_err(dev, "encode_send_buffer err!\r\n");
+		ret = -EAGAIN;
+		goto out;
+	}
+
+	fill_and_slice_regdata(&reg_data, session->send_buff.dma_addr, out);
+
+	/*TODO: delete when release*/
+	#if LOG_PRINT_DATA_DEBUG_EN
+	len = 16 + msg_send.size + 1; // 1-xor byte
+	pr_debug("win2030_umbox_send: umbox_send_data is:\r\n");
+	for (k = 0; k < len; k++) {
+		pr_debug("0x%02x ",
+		       *((u8 *)(session->send_buff.cpu_vaddr) + k));
+		if ((k + 1) % 8 == 0) {
+			pr_debug("\r\n");
+		}
+	}
+	#endif
+
+out:
+	kfree(tmp.cpu_vaddr);
+
+	return ret;
+}
+
+static void eswin_ipc_tx_done(struct mbox_client *client, void *msg, int r)
+{
+	if (r)
+		dev_warn(client->dev, "Client: Message could not be sent:%d\n",
+			 r);
+	else
+		dev_dbg(client->dev, "Client: Message sent\n");
+}
+
+static void eswin_ipc_rx_callback(struct mbox_client *client, void *msg)
+{
+	struct ipc_session *session = container_of(client, struct ipc_session, client);
+	struct device *dev = session->miscdev.parent;
+	res_service_t *pstRes_srvc = &session->res_srvc;
+	u8 reg_data[8];
+	u8 *res_msg = (u8 *)session->send_buff.cpu_vaddr;
+	u32 dma_addr;
+	u32 fream_head;
+	u32 i = 0;
+	int frame_len=0;
+
+	for (i = 0; i < 8; i++)
+		reg_data[i] = *((u8 *)msg + i);
+
+	dma_addr =
+		__UNION32(reg_data[1], reg_data[2], reg_data[3], reg_data[4]);
+	if (dma_addr != session->send_buff.dma_addr) {
+		dev_err(dev, "The received dma_addr is bad \n");
+		goto out;
+	}
+
+	fream_head = get_union32_data(res_msg, 0);
+	if (PREAM_RSP == fream_head) {
+		frame_len = IPC_RES_FIXED_SIZE + \
+			*((u32 *)res_msg + IPC_RES_DATA_LEN_OFFSET);
+		if (check_xor(res_msg,frame_len))
+		{
+			pstRes_srvc->num = IPC_RES_XOR_ERR;
+			pstRes_srvc->service_id = IPC_RES_XOR_ERR;
+			pstRes_srvc->ipc_status = IPC_RES_XOR_ERR;
+			pstRes_srvc->service_status = IPC_RES_XOR_ERR;
+			pstRes_srvc->size = IPC_RES_XOR_ERR;
+			session->res_size = RES_DATA_OFFSET;
+			dev_err(dev, "check_xor err \n");
+			goto out;
+		}
+		session->res_size =
+			get_receive_message_size(session, res_msg + PREAM_RSP_LEN);
+	} else {
+		pstRes_srvc->num = IPC_RES_HEADER_ERR;
+		pstRes_srvc->service_id = IPC_RES_HEADER_ERR;
+		pstRes_srvc->ipc_status = IPC_RES_HEADER_ERR;
+		pstRes_srvc->service_status = IPC_RES_HEADER_ERR;
+		pstRes_srvc->size = IPC_RES_HEADER_ERR;
+		session->res_size = RES_DATA_OFFSET;
+		dev_err(dev, "The response header is invalid \n");
+		goto out;
+	}
+
+out:
+	/* put the cipher_mem_rsc_ifo */
+	pr_debug("%s:session->num=%d\n", __func__, session->res_srvc.num);
+	queue_work(session->work_q, &session->session_work);
+
+	return;
+}
+
+static int ipc_add_proc_priv_data_to_list(process_data_list_t *pstProc_data_list, int id)
+{
+	process_data_t *proc_data = NULL;
+
+	proc_data = kzalloc(sizeof(*proc_data), GFP_ATOMIC);
+	if (!proc_data)
+		return -ENOMEM;
+
+	proc_data->id = id;
+
+	mutex_lock(&pstProc_data_list->lock);
+	list_add_tail(&proc_data->node, &pstProc_data_list->list);
+	mutex_unlock(&pstProc_data_list->lock);
+
+	return 0;
+}
+
+static void ipc_del_proc_priv_data_from_list(process_data_list_t *pstProc_data_list, int id)
+{
+	process_data_t *proc_data, *tmp_proc_data;
+
+	mutex_lock(&pstProc_data_list->lock);
+	list_for_each_entry_safe(proc_data, tmp_proc_data, &pstProc_data_list->list, node) {
+		if (id == proc_data->id) {
+			list_del(&proc_data->node);
+			kfree(proc_data);
+		}
+	}
+	mutex_unlock(&pstProc_data_list->lock);
+}
+
+static process_data_list_t *ipc_proc_data_list_init(void)
+{
+	process_data_list_t *pstProc_data_list = NULL;
+
+	pstProc_data_list = kzalloc(sizeof(*pstProc_data_list), GFP_ATOMIC);
+	if (!pstProc_data_list)
+		return NULL;
+
+	mutex_init(&pstProc_data_list->lock);
+	INIT_LIST_HEAD(&pstProc_data_list->list);
+
+	return pstProc_data_list;
+}
+
+static void ipc_proc_data_list_destroy(process_data_list_t *pstProc_data_list)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_info = NULL;
+	process_data_t *proc_data, *tmp_proc_data;
+
+	mutex_lock(&pstProc_data_list->lock);
+	list_for_each_entry_safe(proc_data, tmp_proc_data, &pstProc_data_list->list, node) {
+		pr_debug("%s:id=%d\n", __func__, proc_data->id);
+		pstCipher_mem_rsc_info = ipc_find_cipher_mem_rsc_info(session, proc_data->id);
+		if (pstCipher_mem_rsc_info) {
+			kref_put(&pstCipher_mem_rsc_info->refcount, ipc_release_mem_khandle_fn);
+		}
+		list_del(&proc_data->node);
+		kfree(proc_data);
+	}
+	mutex_unlock(&pstProc_data_list->lock);
+
+	kfree(pstProc_data_list);
+}
+
+static int eswin_ipc_open(struct inode *inode, struct file *filp)
+{
+	int ret = 0;
+	struct miscdevice *mdev = filp->private_data;
+	struct ipc_session *session = container_of(mdev, struct ipc_session, miscdev);
+	process_data_list_t *pstProc_data_list = NULL;
+
+	pstProc_data_list = ipc_proc_data_list_init();
+	if (!pstProc_data_list)
+		return -ENOMEM;
+
+	pstProc_data_list->session = session;
+	filp->private_data = pstProc_data_list;
+
+	return ret;
+}
+
+static int eswin_receive_data_ready(struct ipc_session *session)
+{
+	int data_ready;
+	data_ready = atomic_read(&session->receive_data_ready);
+	return data_ready;
+}
+
+static int eswin_ipc_service_ready(struct ipc_session *session)
+{
+	int data_ready;
+	data_ready = atomic_read(&session->ipc_service_ready);
+	return data_ready;
+}
+
+static int eswin_ipc_release(struct inode *inode, struct file *filp)
+{
+	process_data_list_t *pstProc_data_list = filp->private_data;
+
+	pr_debug("----%s:pid[%d] called!\n", __func__, task_pid_nr(current));
+
+	ipc_proc_data_list_destroy(pstProc_data_list);
+	filp->private_data = NULL;
+
+	return 0;
+}
+
+static int do_ipc_msg_tx(struct ipc_session *session, req_service_t *pService_req)
+{
+	struct device *dev = session->miscdev.parent;
+	u8 reg_data[8];
+	int ret;
+
+	WARN_ON(!mutex_is_locked(&session->lock));
+
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	if (!session->mbox_channel) {
+		dev_err(dev, "Channel cannot do Tx\n");
+		return -EINVAL;
+	}
+	#endif
+
+	ret = get_send_reg_data((u8 *)pService_req, reg_data, session);
+	if (ret < 0) {
+		ret = -EAGAIN;
+		goto OUT;
+	}
+	dev_dbg(dev,"pid[%d]:session->num:%d\n", task_pid_nr(current), session->num);
+	atomic_set(&session->receive_data_ready, false);
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	ret = mbox_send_message(session->mbox_channel, reg_data);
+	if (ret < 0){
+		ret = -EAGAIN;
+		dev_err(dev, "Failed to send message via mailbox\r\n");
+		atomic_set(&session->receive_data_ready, true);
+		goto OUT;
+	}
+
+	#else
+	session->res_srvc.service_id = pService_req->serivce_type;
+	session->res_srvc.service_status = 1; // For QEMU simulation ,alwasy be false
+	session->res_size = sizeof(res_data_domain_t) + RES_DATA_OFFSET;
+	session->res_srvc.data_t.ecdh_key_res.ecdh_key.keylen = PKA_CURVE_BRAINPOOL_P256R1_BYTE;
+	queue_delayed_work(session->work_q, &session->d_session_work, msecs_to_jiffies(5000));
+	#endif
+
+	ret = wait_event_interruptible_timeout(session->waitq,
+					 eswin_receive_data_ready(session),
+					 MAX_RX_TIMEOUT);
+
+OUT:
+
+	return ret;
+}
+
+static int ipc_session_mem_info_get(struct ipc_session *session, cipher_create_handle_req_t *pstCreate_handle_req)
+{
+	struct device *dev = session->miscdev.parent;
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_infos[MAX_NUM_K_DMA_ALLOC_INFO] = {NULL};
+	khandle_dma_allocation_data_t *pstK_dma_info = NULL;
+	__u32 kinfo_cnt = pstCreate_handle_req->kinfo_cnt;
+	int i, j;
+
+	WARN_ON(!mutex_is_locked(&session->lock));
+	/* clear session khandle info */
+	session->kinfo_cnt = 0;
+	for (i = 0; i < MAX_NUM_K_DMA_ALLOC_INFO; i++)
+		session->kinfo_id[i] = -1;
+
+	pr_debug("%s:kinfo_cnt=%d\n", __func__, kinfo_cnt);
+	/* find the pstCipher_mem_rsc_infos by id */
+	for (i = 0; i < kinfo_cnt; i++) {
+		pstK_dma_info = &pstCreate_handle_req->k_dma_infos[i];
+		pstCipher_mem_rsc_infos[i] = ipc_find_cipher_mem_rsc_info(session, pstK_dma_info->id);
+		if (pstCipher_mem_rsc_infos[i] == NULL)
+		{
+			dev_err(dev, "Failed to find cipher_mem_rsc_info by id(%d)\n", pstK_dma_info->id);
+			for (j = 0; j < i; j++) {
+				session->kinfo_id[j] = -1;
+				kref_put(&pstCipher_mem_rsc_infos[j]->refcount, ipc_release_mem_khandle_fn);
+			}
+			return -EFAULT;
+		}
+		session->kinfo_id[i] = pstK_dma_info->id;;
+		kref_get(&pstCipher_mem_rsc_infos[i]->refcount);
+		pr_debug("%s:id[%d]=%d\n", __func__, i, session->kinfo_id[i]);
+	}
+
+	session->kinfo_cnt = pstCreate_handle_req->kinfo_cnt;
+
+	return 0;
+}
+
+static void ipc_session_mem_info_put(struct ipc_session *session)
+{
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_info = NULL;
+	__u32 kinfo_cnt = session->kinfo_cnt;
+	int i;
+
+	pr_debug("%s:kinfo_cnt=%d\n", __func__, kinfo_cnt);
+	for (i = 0; i < kinfo_cnt; i++) {
+		pr_debug("%s:id[%d]=%d\n", __func__, i, session->kinfo_id[i]);
+		pstCipher_mem_rsc_info = ipc_find_cipher_mem_rsc_info(session, session->kinfo_id[i]);
+		if (pstCipher_mem_rsc_info) {
+			kref_put(&pstCipher_mem_rsc_info->refcount, ipc_release_mem_khandle_fn);
+		}
+		session->kinfo_id[i] = -1;
+	}
+	session->kinfo_cnt = 0;
+}
+
+static int ipc_msg_mbox_tx(struct ipc_session *session, cipher_create_handle_req_t* req_handle)
+{
+	req_service_t *service_req = NULL;
+	int ret = 0;
+	unsigned long time;
+	struct device *dev = session->miscdev.parent;
+
+	if (req_handle == NULL) {
+		pr_err("req_handle is NULL\n");
+		return -EINVAL;
+	}
+
+	service_req = &req_handle->service_req;
+	/* wait for the service to be ready */
+	time = jiffies;
+	while (mutex_trylock(&session->lock) == 0) {
+		if (msleep_interruptible(10)) {
+			ret = -ERESTARTSYS; // interrupted by a signal from user-space
+			goto OUT_TX_ERR;
+		}
+		if (time_after(jiffies, time + MAX_RX_TIMEOUT)) {
+			pr_err("Time out waiting for mutex be released!\n");
+			ret = -ETIMEDOUT;
+			goto OUT_TX_ERR;
+		}
+	}
+	/* In the case that previous session was interrupted by signal from user-space,
+	   the mutex was unlocked by previous process, but service is still runnning for
+	   the previous process. So, current process needs to wait again.
+	*/
+	while (false == eswin_receive_data_ready(session)) {
+		if (msleep_interruptible(10)) {
+			ret = -ERESTARTSYS; // interrupted by a signal from user-space
+			mutex_unlock(&session->lock);
+			pr_debug("%s,pid[%d] was cancled by user!\n", __func__, task_pid_nr(current));
+			goto OUT_TX_ERR;
+		}
+		if (time_after(jiffies, time + MAX_RX_TIMEOUT)) {
+			pr_err("%s, pid[%d] Time out waiting for Cipher Service be ready!\n",
+				__func__, task_pid_nr(current));
+			ret = -ETIMEDOUT;
+			mutex_unlock(&session->lock);
+			goto OUT_TX_ERR;
+		}
+	}
+
+	pr_debug("---%s:pid[%d],SRVC_TYPE=%d get the session!\n",
+		__func__, task_pid_nr(current), service_req->serivce_type);
+	/* find the mem src info by id, and hold the infos by adding the krefcount */
+	ret = ipc_session_mem_info_get(session, req_handle);
+	if (ret) {
+		pr_err("Failed to hold the cipher_mem_src_info\n");
+		mutex_unlock(&session->lock);
+		goto OUT_TX_ERR;
+	}
+
+	ret = do_ipc_msg_tx(session, service_req);
+	if (ret == 0) {
+		ret = -ETIMEDOUT;
+		mutex_unlock(&session->lock);
+		dev_err(dev,"pid[%d],session->num:%d,timeout!!!\n",
+			task_pid_nr(current), session->num);
+		goto OUT_TX_ERR;
+	}
+	else if(ret < 0) {
+		mutex_unlock(&session->lock);
+		dev_dbg(dev,"pid[%d],session->num:%d,cancled!!!\n",
+			task_pid_nr(current), session->num);
+		goto OUT_TX_ERR;
+	}
+
+	return 0;
+
+OUT_TX_ERR:
+	return ret;
+}
+/**
+ * @brief Generate TRNG data using IPC mechanism.
+ * @param rng_data Pointer to the buffer where the generated random data will be stored.
+ * @param rng_size Size of the buffer in bytes.
+ * @return error, or trng data length.
+ */
+int eswin_ipc_trng_generator(struct ipc_session *session, void *rng_data, u32 rng_size)
+{
+	cipher_create_handle_req_t *req_handle = NULL;
+	int ret = 0;
+	int got_data_size = 0;
+	struct device *dev = NULL;
+
+	if (rng_data == NULL || rng_size == 0 || rng_size > MAX_TRNG_DATA_LEN) {
+		pr_err("Invalid input parameters: rng_data=%p, rng_size=%u\n", rng_data, rng_size);
+		return -EINVAL;
+	}
+
+	if (!session || !atomic_read(&session->ipc_service_ready)) {
+		pr_err("IPC session is not ready\n");
+		return -EBUSY;
+	}
+
+	dev = session->miscdev.parent;
+
+	req_handle = kzalloc(sizeof(*req_handle), GFP_ATOMIC);
+	if (!req_handle) {
+		pr_err("Failed to allocate memory for req_handle\n");
+		return -ENOMEM;
+	}
+
+	req_handle->service_req.serivce_type = SRVC_TYPE_TRNG;
+	req_handle->service_req.data.trng_req.flag = rng_size;
+
+	// Note: session will be locked in following ipc_msg_mbox_tx
+	ret = ipc_msg_mbox_tx(session, req_handle);
+	if (ret < 0) {
+		dev_err(dev, "Failed to send message via mailbox\r\n");
+		goto OUT_FREE;
+	}
+
+	got_data_size = session->res_srvc.size;
+	ret = rng_size < got_data_size ? rng_size : got_data_size;
+
+	memcpy(rng_data, session->res_srvc.data_t.trng_res.data, ret);
+	mutex_unlock(&session->lock);
+
+OUT_FREE:
+	kfree(req_handle);
+	return ret;
+}
+
+EXPORT_SYMBOL_GPL(eswin_ipc_trng_generator);
+
+static int ipc_ioctl_msg_commu(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	cipher_create_handle_req_t *pstCreate_handle_req;// = &stCreate_handle_req;
+	int ret = 0;
+	unsigned int cmd_size = 0;
+
+	pr_debug("---%s:%d,pid[%d]---\n",
+		__func__, __LINE__, task_pid_nr(current));
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(cipher_create_handle_req_t)) {
+		return -EFAULT;
+	}
+
+	pstCreate_handle_req = kzalloc(sizeof(*pstCreate_handle_req), GFP_ATOMIC); // GFP_KERNEL
+	if (!pstCreate_handle_req) {
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pstCreate_handle_req, user_arg, sizeof(cipher_create_handle_req_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE;
+	}
+
+	ret = ipc_msg_mbox_tx(session, pstCreate_handle_req);
+	if (ret < 0) {
+		dev_err(dev, "Failed to send message via mailbox\r\n");
+		goto OUT_FREE;
+	}
+
+	/* copy response data to user */
+	pr_debug("---%s:%d,(%d), sizeof(res_srvc)=0x%lx, res_size=0x%x\n",
+		__func__, __LINE__, task_pid_nr(current), sizeof(session->res_srvc), session->res_size);
+	pr_debug("---%s:pid[%d],SRVC_TYPE=%d release the session!\n",
+		__func__, task_pid_nr(current), pstCreate_handle_req->service_req.serivce_type);
+	memcpy(&pstCreate_handle_req->service_resp, &session->res_srvc, session->res_size);
+	mutex_unlock(&session->lock);
+
+	if (copy_to_user(user_arg, pstCreate_handle_req, sizeof(cipher_create_handle_req_t)))
+		ret = -EFAULT;
+
+OUT_FREE:
+	kfree(pstCreate_handle_req);
+
+	return ret;
+}
+
+static ssize_t eswin_ipc_message_write(struct file *filp,
+				       const char __user *userbuf, size_t count,
+				       loff_t *ppos)
+{
+	process_data_list_t *pstProc_data_list = filp->private_data;
+	struct ipc_session *session = pstProc_data_list->session;
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+	struct heap_root *hp_root = &ipc_priv->hp_root;
+	struct device *dev = hp_root->dev;
+	u8 reg_data[8];
+	int ret;
+
+	if (!session->mbox_channel) {
+		dev_err(dev, "Channel cannot do Tx\n");
+		return -EINVAL;
+	}
+
+	if (count > IPC_SERVICE_REQ_MAX_LEN) {
+		dev_err(dev,
+			"Message length %ld greater than ipc allowed %ld\n",
+			count, IPC_SERVICE_REQ_MAX_LEN);
+		return -EINVAL;
+	}
+
+	mutex_lock(&session->lock);
+
+	atomic_set(&session->ipc_service_ready, false);
+	session->req_msg = kzalloc(IPC_SERVICE_REQ_MAX_LEN, GFP_KERNEL);
+	if (!session->req_msg) {
+		ret = -ENOMEM;
+		goto out1;
+	}
+
+	ret = copy_from_user(session->req_msg, userbuf, count);
+	if (ret) {
+		ret = -EFAULT;
+		goto out2;
+	}
+
+	ret = get_send_reg_data(session->req_msg, reg_data, session);
+	if (ret < 0) {
+		ret = -EAGAIN;
+		goto out3;
+	}
+
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	ret = mbox_send_message(session->mbox_channel, reg_data);
+	if (ret < 0){
+		ret = -EAGAIN;
+		dev_err(dev, "Failed to send message via mailbox\r\n");
+		goto out3;
+	}
+
+	atomic_set(&session->receive_data_ready, false);
+	#else
+	session->res_srvc.service_status = 0; // For QEMU simulation ,alwasy be true
+	session->res_size = sizeof(res_data_domain_t) + RES_DATA_OFFSET;
+	session->res_srvc.data_t.ecdh_key_res.ecdh_key.keylen = PKA_CURVE_BRAINPOOL_P256R1_BYTE;
+	atomic_set(&session->receive_data_ready, true);
+	#endif
+
+	ret = wait_event_interruptible_timeout(session->waitq,
+					 eswin_receive_data_ready(session),
+					 MAX_RX_TIMEOUT);
+	if (ret == 0)
+		dev_err(dev, "Timeout waiting for scpu response\r\n");
+
+out3:
+	dma_free_coherent(dev, session->send_buff.size,
+			  session->send_buff.cpu_vaddr,
+			  session->send_buff.dma_addr);
+out2:
+	kfree(session->req_msg);
+
+out1:
+	atomic_set(&session->receive_data_ready, true);
+	mutex_unlock(&session->lock);
+	return ret;
+}
+
+static ssize_t eswin_ipc_message_read(struct file *filp, char __user *userbuf,
+				      size_t count, loff_t *ppos)
+{
+	process_data_list_t *pstProc_data_list = filp->private_data;
+	struct ipc_session *session = pstProc_data_list->session;
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+	struct heap_root *hp_root = &ipc_priv->hp_root;
+	struct device *dev = hp_root->dev;
+	int ret = 0;
+
+	if (copy_to_user(userbuf, &session->res_srvc, session->res_size) != 0) {
+		dev_err(dev, "copy to user failed\n");
+		ret = -EFAULT;
+		goto out;
+	}
+
+out:
+	atomic_set(&session->ipc_service_ready, true);
+	return ret;
+}
+
+static __poll_t eswin_service_status_poll(struct file *filp,
+					  struct poll_table_struct *wait)
+{
+	process_data_list_t *pstProc_data_list = filp->private_data;
+	struct ipc_session *session = pstProc_data_list->session;
+
+	poll_wait(filp, &session->waitq, wait);
+
+	if (eswin_ipc_service_ready(session))
+		return EPOLLIN | EPOLLRDNORM;
+	return 0;
+}
+
+static int ipc_ioctl_alloc_iova(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_info = NULL;
+	khandle_dma_allocation_data_t *pstK_dma_info = NULL;
+	struct dma_allocation_data *pstDma_alloc_info = NULL;
+	unsigned int cmd_size = 0;
+	u64 addr;
+	u64 len;
+	int ret = 0;
+
+	// pr_debug("---%s:%d---\n", __func__, __LINE__);
+
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(khandle_dma_allocation_data_t)) {
+		return -EFAULT;
+	}
+
+	pstCipher_mem_rsc_info = kzalloc(sizeof(*pstCipher_mem_rsc_info), GFP_KERNEL);
+	if (!pstCipher_mem_rsc_info)
+		return -ENOMEM;
+
+	pstCipher_mem_rsc_info->session = session;
+	pstK_dma_info = &pstCipher_mem_rsc_info->k_dma_info;
+	if (copy_from_user(pstK_dma_info, user_arg, sizeof(khandle_dma_allocation_data_t))) {
+		ret = -EFAULT;
+		goto OUT_FREE_CIPHER_MEM;
+	}
+
+	pstDma_alloc_info = &pstK_dma_info->dma_alloc_info;
+	addr = (u64)pstDma_alloc_info->cpu_vaddr;
+	len = pstDma_alloc_info->size;
+	dev_dbg(dev, "%s, addr=0x%llx,len=0x%llx\n", __func__, addr, len);
+	ret = ipc_req_data_iova_alloc(session, pstCipher_mem_rsc_info);
+	if (ret) {
+		dev_err(dev, "failed to alloc iova, err=%d\n", ret);
+		goto OUT_FREE_CIPHER_MEM;
+	}
+
+	/* init refcount of this mem resource info */
+	kref_init(&pstCipher_mem_rsc_info->refcount);
+
+	/* add this mem resource info into IDR(assign id simultaneously) */
+	ret = ipc_add_cipher_mem_rsc_info_to_idr(session, pstCipher_mem_rsc_info);
+	if (ret) {
+		dev_err(dev, "Failed to add cipher_mem_rsc_info into IDR\n");
+		goto OUT_UNIT_KERNEL_HANDLE;
+	}
+
+	/* alloc process priv data(node) and assign the ID that was allocated by ipc_add_cipher_mem_rsc_info_to_idr,
+	   then add this node to the pstProc_data_list. This node is used for searching and delete the mem_rsc_info
+	   with ID when eswin_ipc_release is called(i.e, the eswin_ipc file is closed bu user)*/
+	ret = ipc_add_proc_priv_data_to_list(pstProc_data_list, pstCipher_mem_rsc_info->k_dma_info.id);
+	if (ret) {
+		dev_err(dev, "Failed to add proc data!\n");
+		goto OUT_REMOVE_FROM_IDR;
+	}
+
+	if (copy_to_user(user_arg, pstK_dma_info, sizeof(khandle_dma_allocation_data_t))) {
+		ret = -EFAULT;
+		goto OUT_PROC_DATA_DEL;
+	}
+	pr_debug("%s:pid[%d]:id=%d, iova=0x%llx, dmabuf=0x%px, len=0x%llx\n",
+		__func__, task_pid_nr(current), pstK_dma_info->id,
+		pstDma_alloc_info->dma_addr, pstCipher_mem_rsc_info->dma_buf, len);
+
+	/* increase refcount of this module to prevent module from being removed when mem resource is still in use */
+	if (!try_module_get(THIS_MODULE)) {
+		ret = -ENODEV;
+		dev_err(dev, "Failed to get module!\n");
+		goto OUT_PROC_DATA_DEL;
+	}
+
+	return 0;
+
+OUT_PROC_DATA_DEL:
+	ipc_del_proc_priv_data_from_list(pstProc_data_list, pstCipher_mem_rsc_info->k_dma_info.id);
+OUT_REMOVE_FROM_IDR:
+	ipc_remove_cipher_mem_rsc_info_from_idr(session, pstCipher_mem_rsc_info->k_dma_info.id);
+OUT_UNIT_KERNEL_HANDLE:
+	/* decrease h->refcount, then the refcount is 0 here, so that the iova will be freed. */
+	kref_put(&pstCipher_mem_rsc_info->refcount, ipc_release_mem_khandle_fn);
+OUT_FREE_CIPHER_MEM:
+	kfree(pstCipher_mem_rsc_info);
+	return ret;
+}
+
+static int ipc_ioctl_free_iova(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	khandle_dma_allocation_data_t k_dma_info;
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_info = NULL;
+	unsigned int cmd_size = 0;
+	int ret = 0;
+
+	// pr_debug("---%s:%d---\n", __func__, __LINE__);
+
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(khandle_dma_allocation_data_t)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&k_dma_info, user_arg, sizeof(khandle_dma_allocation_data_t))) {
+		return -EFAULT;
+	}
+
+	pstCipher_mem_rsc_info = ipc_find_cipher_mem_rsc_info(session, k_dma_info.id);
+	if (NULL == pstCipher_mem_rsc_info) {
+		ret = -EFAULT;
+		goto OUT;
+	}
+	pr_debug("%s:free id=%d\n", __func__, pstCipher_mem_rsc_info->k_dma_info.id);
+	ipc_del_proc_priv_data_from_list(pstProc_data_list, k_dma_info.id);
+	kref_put(&pstCipher_mem_rsc_info->refcount, ipc_release_mem_khandle_fn);
+OUT:
+	return ret;
+}
+
+static int ipc_ioctl_cpu_nid(process_data_list_t *pstProc_data_list, unsigned int cmd, void __user *user_arg)
+{
+	struct ipc_session *session = pstProc_data_list->session;
+	struct device *dev = session->miscdev.parent;
+	unsigned int cmd_size = 0;
+	cipher_get_nid_req_t arg;
+	int hartid, harts = fls(cpu_online_mask->bits[0]);
+	int cpu;
+
+	cmd_size = _IOC_SIZE(cmd);
+	if (cmd_size != sizeof(arg)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&arg, user_arg, sizeof(arg))) {
+		return -EFAULT;
+	}
+
+	cpu = get_cpu();
+	// hartid = cpuid_to_hartid_map(arg.cpuid);
+	hartid = eswin_cpuid_to_hartid_map(cpu);
+	put_cpu();
+
+	arg.nid = (hartid < 4 ? 0 : 1);
+
+	dev_dbg(dev, "%s, oneline_harts:%d, cpu%d,cpuid:%d, hartid:%d, nid:%d\n",
+		__func__, harts, cpu, arg.cpuid, hartid, arg.nid);
+
+	if (copy_to_user(user_arg, &arg, sizeof(arg))) {
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static long eswin_ipc_ioctl(struct file *filp, unsigned int cmd,
+			    unsigned long arg)
+{
+	process_data_list_t *pstProc_data_list = filp->private_data;
+	struct ipc_session *session = pstProc_data_list->session;
+	struct heap_root *hp_root = &session->ipc_priv->hp_root;
+	struct device *dev = hp_root->dev;
+	void *cpu_vaddr = NULL;
+	struct dmabuf_bank_info *info, *info_free;
+	unsigned int cmd_size = 0;
+	size_t buf_size = 0;
+	int ret = 0;
+	void __user *argp = (void __user *)arg;
+
+	switch (cmd) {
+		/* alloc memory by driver using dmabuf heap helper API  */
+		case SCPU_IOC_ALLOCATE_MEM_BY_DRIVER_WITH_DMA_HEAP: {
+			mutex_lock(&dmabuf_bank_lock);
+			cmd_size = _IOC_SIZE(cmd);
+			if (cmd_size != sizeof(struct dmabuf_bank_info)) {
+				return -EFAULT;
+			}
+
+			info = kzalloc(sizeof(*info), GFP_KERNEL);
+			if (!info) {
+				return -ENOMEM;
+			}
+
+			if (copy_from_user(info, (void __user *)arg, cmd_size) != 0) {
+				kfree(info);
+				return -EFAULT;
+			}
+
+			/* alloc dmabuf via dma heap, map the pha to dma addr(iova), get dmabuf_fd*/
+			info->hmem = common_dmabuf_heap_import_from_kernel(
+				hp_root, "mmz_nid_0_part_0", info->dma_alloc_info.size, 0);
+			if (IS_ERR(info->hmem)) {
+				dev_err(dev, "dmabuf-heap alloc from kernel fail\n");
+				kfree(info);
+				return -ENOMEM;
+			}
+			info->dma_alloc_info.dmabuf_fd = info->hmem->dbuf_fd;
+
+			/* map the pha to cpu vaddr */
+			cpu_vaddr = common_dmabuf_heap_map_vaddr(info->hmem);
+			if (cpu_vaddr == NULL) {
+				dev_err(dev, "map to cpu_vaddr failed\n");
+				common_dmabuf_heap_release(info->hmem);
+				kfree(info);
+				return -ENOMEM;
+			}
+			/* get the actual buf_size (page aligned size)of the dmabuf allocated by dmabuf_heap */
+			buf_size = common_dmabuf_heap_get_size(info->hmem);
+
+			pr_debug("dmabuf info: CPU KVaddr:0x%lx, PA:0x%lx, DMA addr(iova):0x%lx, allocated size=0x%lx\n",
+				(unsigned long)info->hmem->vaddr,
+				(unsigned long)sg_phys(info->hmem->sgt->sgl),
+				(unsigned long)sg_dma_address(info->hmem->sgt->sgl),
+				(unsigned long)buf_size);
+
+			info->hmem->dir = DMA_BIDIRECTIONAL;
+			info->dma_alloc_info.phy_addr =
+				(unsigned long)sg_phys(info->hmem->sgt->sgl);
+			info->dma_alloc_info.dma_addr =
+				(unsigned long)sg_dma_address(info->hmem->sgt->sgl);
+
+			pr_debug("%s:%d, alloc size=0x%llx\n", __func__, __LINE__,
+				info->dma_alloc_info.size);
+			/*  return the dmabuf_fd to user space, so it can mmap the dmabuf to tje vaddr of the user space*/
+			if (copy_to_user((void __user *)arg, info, cmd_size) != 0) {
+				dev_err(dev, "copy to user failed\n");
+				ret = -EFAULT;
+			}
+			kfree(info);
+			mutex_unlock(&dmabuf_bank_lock);
+			break;
+		}
+		case SCPU_IOC_FREE_MEM_BY_DRIVER_WITH_DMA_HEAP: {
+			mutex_lock(&dmabuf_bank_lock);
+
+			cmd_size = _IOC_SIZE(cmd);
+			if (cmd_size != sizeof(struct dmabuf_bank_info)) {
+				return -EFAULT;
+			}
+
+			info_free = kzalloc(sizeof(*info_free), GFP_KERNEL);
+			if (!info_free) {
+				return -ENOMEM;
+			}
+
+			if (copy_from_user(info_free, (void __user *)arg, cmd_size) !=
+			0) {
+				kfree(info_free);
+				return -EFAULT;
+			}
+
+			common_dmabuf_heap_umap_vaddr(info_free->hmem);
+			common_dmabuf_heap_release(info_free->hmem);
+			kfree(info_free);
+
+			mutex_unlock(&dmabuf_bank_lock);
+			break;
+		}
+		case SCPU_IOC_ALLOC_IOVA: {
+			return ipc_ioctl_alloc_iova(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_FREE_IOVA: {
+			return ipc_ioctl_free_iova(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_CPU_NID: {
+			return ipc_ioctl_cpu_nid(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_CREATE_HANDLE: {
+			return ipc_ioctl_create_handle(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_DESTROY_HANDLE: {
+			return ipc_ioc_destroy_handle(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_GET_HANDLE_CONFIG: {
+			return ipc_ioc_get_handle_config(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_UPDATE_HANDLE_CONFIG: {
+			return ipc_ioc_update_handle_config(pstProc_data_list, cmd, argp);
+		}
+		case SCPU_IOC_MESSAGE_COMMUNICATION: {
+			return ipc_ioctl_msg_commu(pstProc_data_list, cmd, argp);
+		}
+		default: {
+			dev_err(dev, "Invalid IOCTL command %u\n", cmd);
+			return -ENOTTY;
+		}
+	}
+
+	return ret;
+}
+
+static const struct file_operations eswin_ipc_message_ops = {
+	.owner = THIS_MODULE,
+	.write = eswin_ipc_message_write,
+	.read = eswin_ipc_message_read,
+	.open = eswin_ipc_open,
+	.poll = eswin_service_status_poll,
+	.release = eswin_ipc_release,
+	.unlocked_ioctl = eswin_ipc_ioctl,
+};
+
+static int eswin_ipc_request_channel(struct platform_device *pdev,
+						   const char *name)
+{
+	struct ipc_session *session = platform_get_drvdata(pdev);
+	struct mbox_client *client = &session->client;
+	struct mbox_chan *channel = NULL;
+
+	client->dev = &pdev->dev;
+	client->rx_callback = eswin_ipc_rx_callback;
+	client->tx_prepare = NULL;
+	client->tx_done = eswin_ipc_tx_done;
+	client->tx_block = false;
+	client->knows_txdone = false;
+
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	channel = mbox_request_channel_byname(client, name);
+	if (IS_ERR(channel)) {
+		dev_warn(&pdev->dev, "Failed to request %s channel\n", name);
+		session->mbox_channel = NULL;
+		return -EFAULT;
+	}
+	#endif
+	session->mbox_channel = channel;
+	dev_dbg(&pdev->dev, "request mbox chan %s\n", name);
+
+	return 0;
+}
+
+static int eswin_ipc_dev_nid(struct device* dev, int *pNid)
+{
+	int nid;
+
+	#ifdef CONFIG_NUMA
+	nid = dev_to_node(dev);
+	if (nid == NUMA_NO_NODE) {
+		dev_err(dev, "%s:%d, numa-node-id was not defined!\n", __func__, __LINE__);
+		return -EFAULT;
+	}
+	#else
+	if (of_property_read_s32(dev->of_node, "numa-node-id", &nid)) {
+		dev_err(dev, "%s:%d, numa-node-id was not defined!\n", __func__, __LINE__);
+		return -EFAULT;
+	}
+	#endif
+
+	*pNid = nid;
+
+	return 0;
+}
+
+static int ipc_add_cipher_mem_rsc_info_to_idr(struct ipc_session *session, cipher_mem_resource_info_t *pstCipher_mem_rsc_info)
+{
+	int ret = 0;
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+
+	mutex_lock(&ipc_priv->idr_cipher_mem_lock);
+	ret = idr_alloc(&ipc_priv->cipher_mem_resource_idr, pstCipher_mem_rsc_info, 0, INT_MAX, GFP_KERNEL);
+	if (ret >= 0) {
+		pstCipher_mem_rsc_info->k_dma_info.id = ret;
+	}
+	mutex_unlock(&ipc_priv->idr_cipher_mem_lock);
+
+	return ret < 0 ? ret : 0;
+}
+
+static cipher_mem_resource_info_t *ipc_find_cipher_mem_rsc_info(struct ipc_session *session, u32 id)
+{
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_info = NULL;
+
+	mutex_lock(&ipc_priv->idr_cipher_mem_lock);
+	pstCipher_mem_rsc_info = idr_find(&ipc_priv->cipher_mem_resource_idr, id);
+	if (!pstCipher_mem_rsc_info) {
+		pr_err("Failed to find cipher_mem_src_info of id:%d\n", id);
+	}
+	mutex_unlock(&ipc_priv->idr_cipher_mem_lock);
+
+	return pstCipher_mem_rsc_info;
+}
+
+static void ipc_remove_cipher_mem_rsc_info_from_idr(struct ipc_session *session, u32 id)
+{
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+
+	mutex_lock(&ipc_priv->idr_cipher_mem_lock);
+	if (NULL == idr_remove(&ipc_priv->cipher_mem_resource_idr, id)) {
+		pr_err("Failed to remove cipher_mem_src_info of id:%d(Not in previously in the IDR)\n", id);
+	}
+	mutex_unlock(&ipc_priv->idr_cipher_mem_lock);
+}
+
+static void ipc_release_mem_khandle_fn(struct kref *ref)
+{
+	cipher_mem_resource_info_t *pstCipher_mem_rsc_info = container_of(ref, cipher_mem_resource_info_t, refcount);
+	struct ipc_session *session = pstCipher_mem_rsc_info->session;
+
+	pr_debug("%s, id=%d, iova=0x%llx\n",
+		__func__, pstCipher_mem_rsc_info->k_dma_info.id,
+		pstCipher_mem_rsc_info->k_dma_info.dma_alloc_info.dma_addr);
+	ipc_req_data_iova_free(session, pstCipher_mem_rsc_info);
+
+	ipc_remove_cipher_mem_rsc_info_from_idr(session, pstCipher_mem_rsc_info->k_dma_info.id);
+
+	kfree(pstCipher_mem_rsc_info);
+
+	/* decrease refcount of this module */
+	module_put(THIS_MODULE);
+}
+
+static void ipc_session_work(struct work_struct *work)
+{
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	struct ipc_session *session = container_of(work, struct ipc_session, session_work);
+	#else
+	struct ipc_session *session = container_of(work, struct ipc_session, d_session_work.work);
+	#endif
+
+	pr_debug("======%s, session->num:%d,service_id:%d done!\n",
+		__func__, session->num, session->res_srvc.service_id);
+	ipc_session_mem_info_put(session);
+	atomic_set(&session->receive_data_ready, 1);
+	wake_up_interruptible(&session->waitq);
+}
+
+static int ipc_cipher_mem_rsc_info_mgt_init(struct ipc_session *session)
+{
+	struct device *dev = session->miscdev.parent;
+	ipc_private_data_t *ipc_priv = NULL;
+	int ret = 0;
+
+	ipc_priv = kzalloc(sizeof(ipc_private_data_t), GFP_KERNEL);
+	if (!ipc_priv)
+		return -ENOMEM;
+	/* Init ipc_private_data_t for each process who opened this device*/
+	common_dmabuf_heap_import_init(&ipc_priv->hp_root, dev);
+	INIT_LIST_HEAD(&ipc_priv->usermem_iova_list);
+	mutex_init(&ipc_priv->usermem_iova_lock);
+
+	mutex_init(&ipc_priv->idr_cipher_mem_lock);
+	idr_init(&ipc_priv->cipher_mem_resource_idr);
+
+	session->ipc_priv = ipc_priv;
+
+	/* init work queue for putting back mem_rsc_info which was hold by session */
+	session->work_q = create_singlethread_workqueue("eswin-ipc-wq");
+	if (!session->work_q) {
+		dev_err(dev, "failed to init work queue\n");
+		ret = -ENOMEM;
+		goto OUT;
+	}
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	INIT_WORK(&session->session_work, ipc_session_work);
+	#else
+	INIT_DELAYED_WORK(&session->d_session_work, ipc_session_work);
+	#endif
+
+	return 0;
+OUT:
+	idr_destroy(&ipc_priv->cipher_mem_resource_idr);
+	common_dmabuf_heap_import_uninit(&ipc_priv->hp_root);
+	kfree(ipc_priv);
+	return ret;
+}
+
+static int ipc_cipher_mem_rsc_mgt_unit(struct ipc_session *session)
+{
+	ipc_private_data_t *ipc_priv = session->ipc_priv;
+	struct heap_root *hp_root = &ipc_priv->hp_root;
+	struct device *dev = session->miscdev.parent;
+	int ret = 0;
+
+	mutex_lock(&ipc_priv->idr_cipher_mem_lock);
+	if (!idr_is_empty(&ipc_priv->cipher_mem_resource_idr)) {
+		dev_err(dev, "Can't uninstall driver, some of the resource are still in use!!!\n");
+		ret = -EBUSY;
+		mutex_unlock(&ipc_priv->idr_cipher_mem_lock);
+		goto err;
+	}
+	idr_destroy(&ipc_priv->cipher_mem_resource_idr);
+	mutex_unlock(&ipc_priv->idr_cipher_mem_lock);
+
+	common_dmabuf_heap_import_uninit(hp_root);
+	kfree(ipc_priv);
+
+	destroy_workqueue(session->work_q);
+
+err:
+	return ret;
+}
+
+static int ipc_cipher_handle_free(int id, void *p, void *data)
+{
+	req_service_t *service_req = (req_service_t *)p;
+
+	kfree(service_req);
+
+	pr_debug("---%s: id=%d\n", __func__, id);
+	return 0;
+}
+
+static int ipc_cipher_destroy_all_handles(struct ipc_session *session)
+{
+	int ret = 0;
+
+	mutex_lock(&session->idr_lock);
+	ret = idr_for_each(&session->handle_idr, ipc_cipher_handle_free, NULL);
+	idr_destroy(&session->handle_idr);
+	mutex_unlock(&session->idr_lock);
+
+	return ret;
+}
+
+static int eswin_rng_init(struct hwrng *rng)
+{
+	struct ipc_session *session = (struct ipc_session *)rng->priv;
+	struct device *dev = session->miscdev.parent;
+
+    dev_dbg(dev, "Eswin RNG driver initialized\n");
+    return 0;
+}
+
+static void eswin_rng_cleanup(struct hwrng *rng)
+{
+	struct ipc_session *session = (struct ipc_session *)rng->priv;
+	struct device *dev = session->miscdev.parent;
+
+    dev_dbg(dev,"Eswin RNG driver cleanup\n");
+}
+
+static int eswin_rng_read(struct hwrng *rng, void *buf, size_t max_len, bool wait)
+{
+    int ret = 0;
+	struct ipc_session *session = (struct ipc_session *)rng->priv;
+	struct device *dev = session->miscdev.parent;
+
+    dev_dbg(dev, "Eswin RNG read called, max_len = %ld, wait = %d\n", max_len, wait);
+
+    if (!wait) {
+        return 0;
+    }
+
+    ret = eswin_ipc_trng_generator(session, buf, max_len);
+    if (ret < 0) {
+        dev_err(dev, "Failed to read random data: %d\n", ret);
+        return ret;
+    }
+
+    for (int i = 0; i < ret; i++) {
+        dev_dbg(dev, "0x%02x ", ((u8 *)buf)[i]);
+    }
+    dev_dbg(dev, "\n");
+
+    return ret;
+}
+
+static int eswin_ipc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const char *mbox_channel_name;
+	struct ipc_session *session = NULL;
+	int nid = 0;
+	int ret = 0;
+
+	ret = eswin_ipc_dev_nid(dev, &nid);
+	if (ret) {
+		dev_err(dev, "failed to ipc's nodeID\n");
+		return ret;
+	}
+	#ifndef ES_CIPHER_QEMU_DEBUG
+	ret = win2030_aon_sid_cfg(dev);
+	if (ret) {
+		dev_err(dev, "failed to init_streamID: %d\n", ret);
+		return -EFAULT;
+	}
+	win2030_tbu_power(dev, true);
+	#endif
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		dev_warn(dev, "Unable to set coherent mask\n");
+
+	session = devm_kzalloc(dev, sizeof(*session), GFP_KERNEL);
+	if (!session) {
+		return -ENOMEM;
+	}
+	platform_set_drvdata(pdev, session);
+
+	session->miscdev.minor = MISC_DYNAMIC_MINOR;
+	session->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "%s%d", "eswin_ipc_misc_dev", nid);
+	session->miscdev.fops = &eswin_ipc_message_ops;
+	session->miscdev.parent = dev;
+
+	/* dma_alloc_coherent memory for memory sharing between scpu and u84 */
+	ret = get_shm_addr(session);
+	if (ret < 0) {
+		devm_kfree(dev, session);
+		ret = -ENOMEM;
+		goto OUT_FREE_SESSION;
+	}
+
+	ret =  ipc_cipher_mem_rsc_info_mgt_init(session);
+	if (0 != ret) {
+		dev_err(dev, "failed to init mem resource mgt: %d\n", ret);
+		goto OUT_FREE_SESSION;
+	}
+
+	ret = misc_register(&session->miscdev);
+	if (ret) {
+		dev_err(dev, "failed to register misc device: %d\n", ret);
+		ret = -EBUSY;
+		goto OUT_UNIT_MEM_MGT;
+	}
+
+	/*use eswin mailbox0 to send msg and mailbox1 receive msg*/
+	ret = device_property_read_string(&pdev->dev, "mbox-names",
+					  &mbox_channel_name);
+	if (ret == 0) {
+		ret = eswin_ipc_request_channel(pdev, mbox_channel_name);
+		if (ret) {
+			dev_err(dev, "failed to request mbox channel\n");
+			goto OUT_MISC_DEREGISTER;
+		}
+	}else{
+		dev_err(dev, "given arguments are not valid: %d\n", ret);
+		ret = -EINVAL;
+		goto OUT_MISC_DEREGISTER;
+	}
+
+	mutex_init(&session->lock);
+	init_waitqueue_head(&session->waitq);
+	dev_info(dev, "initialized successfully\n");
+
+	mutex_init(&session->idr_lock);
+	idr_init(&session->handle_idr);
+
+	atomic_set(&session->receive_data_ready, true);
+	atomic_set(&session->ipc_service_ready, true);
+
+	session->rng.priv = (unsigned long)session;
+	session->rng.name = devm_kasprintf(dev, GFP_KERNEL, "%s%d", "eswin_rng", nid);
+	session->rng.init = eswin_rng_init;
+	session->rng.cleanup = eswin_rng_cleanup;
+	session->rng.read = eswin_rng_read;
+	ret = hwrng_register(&session->rng);
+    if (ret) {
+        pr_err("Failed to register %s\n", session->rng.name);
+		goto OUT_MISC_DEREGISTER;
+    }
+
+	pr_debug("sizeof(cipher_create_handle_req_t)=%ld, sizeof(req_service_t)=%ld, sizeof(res_service_t)=%ld, sizeof(req_data_domain_t)=%ld\n",
+		sizeof(cipher_create_handle_req_t), sizeof(req_service_t), sizeof(res_service_t), sizeof(req_data_domain_t));
+
+	return 0;
+
+OUT_MISC_DEREGISTER:
+	misc_deregister(&session->miscdev);
+OUT_UNIT_MEM_MGT:
+	ipc_cipher_mem_rsc_mgt_unit(session);
+OUT_FREE_SESSION:
+	free_shm_addr(session);
+	devm_kfree(dev, session);
+
+	return ret;
+}
+
+static int eswin_ipc_remove(struct platform_device *pdev)
+{
+	struct ipc_session *session = platform_get_drvdata(pdev);
+	struct device *dev = session->miscdev.parent;
+	int ret = 0;
+
+	hwrng_unregister(&session->rng);
+
+	ret =ipc_cipher_mem_rsc_mgt_unit(session);
+	if (ret)
+		return ret;
+
+	ipc_cipher_destroy_all_handles(session);
+
+	#ifndef QEMU_DEBU
+	mbox_free_channel(session->mbox_channel);
+	#endif
+
+	misc_deregister(&session->miscdev);
+
+	free_shm_addr(session);
+	win2030_tbu_power(dev, false);
+	dev_info(&pdev->dev, "%s removed!\n", pdev->name);
+	return 0;
+}
+
+static const struct of_device_id eswin_ipc_match[] = {
+	{
+		.compatible = "eswin,win2030-ipc",
+	},
+	{ /* Sentinel */ }
+};
+
+static struct platform_driver eswin_ipc_driver = {
+	.driver = {
+		.name = "win2030-ipc",
+		.of_match_table = eswin_ipc_match,
+		.suppress_bind_attrs	= true,
+	},
+	.probe = eswin_ipc_probe,
+	.remove = eswin_ipc_remove,
+};
+// builtin_platform_driver(eswin_ipc_driver);
+module_platform_driver(eswin_ipc_driver);
+
+MODULE_DESCRIPTION("ESWIN WIN2030 firmware protocol driver");
+MODULE_LICENSE("GPL v2");
\ No newline at end of file
diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c
index 84ae708fafe7..c5634708c9a4 100644
--- a/drivers/dma-buf/dma-heap.c
+++ b/drivers/dma-buf/dma-heap.c
@@ -300,6 +300,7 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
 	kfree(heap);
 	return err_ret;
 }
+EXPORT_SYMBOL(dma_heap_add);
 
 static char *dma_heap_devnode(const struct device *dev, umode_t *mode)
 {
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index 72fb40de58b3..470f5d0e60dc 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -6,6 +6,25 @@
  *
  * Author: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>
  */
+/*****************************************************************************
+ * ESWIN dma driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
 
 #include <linux/bitops.h>
 #include <linux/delay.h>
@@ -28,10 +47,15 @@
 #include <linux/reset.h>
 #include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/iommu.h>
 
 #include "dw-axi-dmac.h"
 #include "../dmaengine.h"
 #include "../virt-dma.h"
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eswin-win2030-sid-cfg.h>
 
 /*
  * The set of bus widths supported by the DMA controller. DW AXI DMAC supports
@@ -50,6 +74,13 @@
 #define AXI_DMA_FLAG_HAS_APB_REGS	BIT(0)
 #define AXI_DMA_FLAG_HAS_RESETS		BIT(1)
 #define AXI_DMA_FLAG_USE_CFG2		BIT(2)
+#define AXI_DMA_FLAG_HAS_2RESETS	BIT(3)
+#define AXI_DMA_FLAG_HAS_EIC7700	BIT(4)
+
+#define AWSMMUSID	GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID	GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID	GENMASK(15, 8)	// The sid of read operation
+#define ARSMMUSSID	GENMASK(7, 0)	// The ssid of read operation
 
 static inline void
 axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val)
@@ -98,6 +129,8 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 			 config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS |
 			 config->src_per << CH_CFG_H_SRC_PER_POS |
 			 config->dst_per << CH_CFG_H_DST_PER_POS |
+			 0xF <<CH_CFG_H_SRC_OSR_LMT_POS |
+			 0xF <<CH_CFG_H_DST_OSR_LMT_POS |
 			 config->prior << CH_CFG_H_PRIORITY_POS;
 	} else {
 		cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS |
@@ -105,6 +138,8 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 		cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS |
 			 config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS |
 			 config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS |
+			 0xF <<CH_CFG2_H_SRC_OSR_LMT_POS |
+			 0xF <<CH_CFG2_H_DST_OSR_LMT_POS |
 			 config->prior << CH_CFG2_H_PRIORITY_POS;
 	}
 	axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo);
@@ -114,37 +149,49 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 static inline void axi_dma_disable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val &= ~DMAC_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_dma_enable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val |= DMAC_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_dma_irq_disable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val &= ~INT_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_dma_irq_enable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val |= INT_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_chan_irq_disable(struct axi_dma_chan *chan, u32 irq_mask)
@@ -220,6 +267,7 @@ static void axi_dma_hw_init(struct axi_dma_chip *chip)
 {
 	int ret;
 	u32 i;
+	int flags;
 
 	for (i = 0; i < chip->dw->hdata->nr_channels; i++) {
 		axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL);
@@ -228,6 +276,20 @@ static void axi_dma_hw_init(struct axi_dma_chip *chip)
 	ret = dma_set_mask_and_coherent(chip->dev, DMA_BIT_MASK(64));
 	if (ret)
 		dev_warn(chip->dev, "Unable to set coherent mask\n");
+
+	flags = (uintptr_t)of_device_get_match_data(chip->dev);
+	if (flags & AXI_DMA_FLAG_HAS_EIC7700) {
+		if (of_node_name_prefix(chip->dev->of_node, "dma-controller-hsp")) {
+			win2030_dma_sid_cfg(chip->dev);
+		}
+		else {
+			win2030_aon_sid_cfg(chip->dev);
+		}
+
+		/* TBU power up */
+		win2030_tbu_power(chip->dev, true);
+	}
+
 }
 
 static u32 axi_chan_get_xfer_width(struct axi_dma_chan *chan, dma_addr_t src,
@@ -523,7 +585,7 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set)
 	unsigned long reg_value, val;
 
 	if (!chip->apb_regs) {
-		dev_err(chip->dev, "apb_regs not initialized\n");
+		dev_dbg(chip->dev, "apb_regs not initialized\n");
 		return;
 	}
 
@@ -575,25 +637,43 @@ static void write_desc_dar(struct axi_dma_hw_desc *desc, dma_addr_t adr)
 	desc->lli->dar = cpu_to_le64(adr);
 }
 
-static void set_desc_src_master(struct axi_dma_hw_desc *desc)
+static void set_desc_src_master(struct axi_dma_hw_desc *hw_desc,
+				 struct axi_dma_chan *chan)
 {
 	u32 val;
 
 	/* Select AXI0 for source master */
-	val = le32_to_cpu(desc->lli->ctl_lo);
-	val &= ~CH_CTL_L_SRC_MAST;
-	desc->lli->ctl_lo = cpu_to_le32(val);
+	val = le32_to_cpu(hw_desc->lli->ctl_lo);
+	if (chan->chip->dw->hdata->nr_masters > 1)
+	{
+		if(DMA_DEV_TO_MEM == chan->direction || DMA_DEV_TO_DEV == chan->direction) {
+			val |= CH_CTL_L_SRC_MAST;
+		}
+		else
+		{
+			val &= ~CH_CTL_L_SRC_MAST;
+		}
+	}
+	else
+		val &= ~CH_CTL_L_SRC_MAST;
+	hw_desc->lli->ctl_lo = cpu_to_le32(val);
 }
 
 static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc,
-				 struct axi_dma_desc *desc)
+				 struct axi_dma_chan *chan)
 {
 	u32 val;
 
 	/* Select AXI1 for source master if available */
 	val = le32_to_cpu(hw_desc->lli->ctl_lo);
-	if (desc->chan->chip->dw->hdata->nr_masters > 1)
-		val |= CH_CTL_L_DST_MAST;
+	if (chan->chip->dw->hdata->nr_masters > 1)
+	{
+		if(DMA_MEM_TO_DEV == chan->direction || DMA_DEV_TO_DEV == chan->direction) {
+			val |= CH_CTL_L_DST_MAST;
+		}
+		else
+			val &= ~CH_CTL_L_DST_MAST;
+	}
 	else
 		val &= ~CH_CTL_L_DST_MAST;
 
@@ -612,6 +692,8 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 	size_t block_ts;
 	u32 ctllo, ctlhi;
 	u32 burst_len;
+	u32 src_maxburst;
+	u32 dst_maxburst;
 
 	axi_block_ts = chan->chip->dw->hdata->block_size[chan->id];
 
@@ -624,6 +706,8 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 		return -EINVAL;
 	}
 
+	src_maxburst = chan->chip->dw->hdata->max_msize;
+	dst_maxburst  = chan->chip->dw->hdata->max_msize;
 	switch (chan->direction) {
 	case DMA_MEM_TO_DEV:
 		reg_width = __ffs(chan->config.dst_addr_width);
@@ -632,6 +716,7 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 			mem_width << CH_CTL_L_SRC_WIDTH_POS |
 			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS |
 			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS;
+		dst_maxburst = chan->config.dst_maxburst;
 		block_ts = len >> mem_width;
 		break;
 	case DMA_DEV_TO_MEM:
@@ -641,6 +726,7 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 			mem_width << CH_CTL_L_DST_WIDTH_POS |
 			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
 			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS;
+		src_maxburst = chan->config.src_maxburst;
 		block_ts = len >> reg_width;
 		break;
 	default:
@@ -677,10 +763,18 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 
 	ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
 		 DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS;
+	if(is_power_of_2(dst_maxburst) && is_power_of_2(src_maxburst))
+	{
+		dst_maxburst = order_base_2(dst_maxburst)? order_base_2(dst_maxburst) - 1 : 0;
+		src_maxburst = order_base_2(src_maxburst)? order_base_2(src_maxburst) - 1 : 0;
+	}else
+		dev_err(chan->chip->dev, "dst_burst or src_burst error!\n");
+	ctllo |= dst_maxburst << CH_CTL_L_DST_MSIZE_POS |
+		 src_maxburst << CH_CTL_L_SRC_MSIZE_POS;
 	hw_desc->lli->ctl_lo = cpu_to_le32(ctllo);
 
-	set_desc_src_master(hw_desc);
-
+	set_desc_src_master(hw_desc, chan);
+	set_desc_dest_master(hw_desc, chan);
 	hw_desc->len = len;
 	return 0;
 }
@@ -879,7 +973,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 	size_t block_ts, max_block_ts, xfer_len;
 	struct axi_dma_hw_desc *hw_desc = NULL;
 	struct axi_dma_desc *desc = NULL;
-	u32 xfer_width, reg, num;
+	u32 xfer_width, reg, num, max_burst_len;
 	u64 llp = 0;
 	u8 lms = 0; /* Select AXI0 master for LLI fetching */
 
@@ -887,6 +981,9 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 		axi_chan_name(chan), &src_adr, &dst_adr, len, flags);
 
 	max_block_ts = chan->chip->dw->hdata->block_size[chan->id];
+	max_burst_len = chan->chip->dw->hdata->max_msize;
+	max_burst_len = order_base_2(max_burst_len)? order_base_2(max_burst_len) - 1 : 0;
+
 	xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, len);
 	num = DIV_ROUND_UP(len, max_block_ts << xfer_width);
 	desc = axi_desc_alloc(num);
@@ -937,16 +1034,16 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 		}
 		hw_desc->lli->ctl_hi = cpu_to_le32(reg);
 
-		reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
-		       DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS |
+		reg = (max_burst_len << CH_CTL_L_DST_MSIZE_POS |
+		       max_burst_len << CH_CTL_L_SRC_MSIZE_POS |
 		       xfer_width << CH_CTL_L_DST_WIDTH_POS |
 		       xfer_width << CH_CTL_L_SRC_WIDTH_POS |
 		       DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
 		       DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS);
 		hw_desc->lli->ctl_lo = cpu_to_le32(reg);
 
-		set_desc_src_master(hw_desc);
-		set_desc_dest_master(hw_desc, desc);
+		set_desc_src_master(hw_desc, chan);
+		set_desc_dest_master(hw_desc, chan);
 
 		hw_desc->len = xfer_len;
 		desc->length += hw_desc->len;
@@ -1283,19 +1380,78 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev)
 	return axi_dma_resume(chip);
 }
 
+int win2030_dma_sel_cfg(struct axi_dma_chan *chan, u32 val)
+{
+	struct axi_dma_chip *chip = chan->chip;
+	struct device *dev = chan->chip->dev;
+	int ret = 0;
+	struct regmap *regmap;
+	int dma_sel_reg;
+	u32 dma_sel = 0;
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "No eswin,syscfg phandle specified\n");
+		return -1;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", 2,
+					&dma_sel_reg);
+	if (ret) {
+		dev_err(dev, "can't get sid cfg reg offset in sys_con(errno:%d)\n", ret);
+		return ret;
+	}
+	regmap_read(regmap, dma_sel_reg, &dma_sel);
+	
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-hsp")) {
+		if (val < 32)
+			dma_sel &= ~(1 << val);
+	}
+	else {
+		if (val < 32)
+			dma_sel |= (1 << val);
+	}
+	regmap_write(regmap, dma_sel_reg, dma_sel);
+	return 0;
+}
+
+static bool axi_dma_filter_fn(struct dma_chan *dchan, void *param)
+{
+	struct axi_dma_filter_data *fdata = param;
+
+	if (dchan->device->dev != fdata->dw_dma->dma.dev)
+		return false;
+
+	if (fdata->request != dchan->chan_id)
+		return false;
+
+	return true;
+}
+
 static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec,
 					    struct of_dma *ofdma)
 {
 	struct dw_axi_dma *dw = ofdma->of_dma_data;
 	struct axi_dma_chan *chan;
 	struct dma_chan *dchan;
+	struct axi_dma_filter_data fdata;
 
-	dchan = dma_get_any_slave_channel(&dw->dma);
-	if (!dchan)
+	fdata.dw_dma = dw,
+	fdata.request = dma_spec->args[0];
+
+	dchan = dma_request_channel(dw->dma.cap_mask, axi_dma_filter_fn, &fdata);
+	if (!dchan) {
+		dev_err(dw->dma.dev, "request chan failed\n");
 		return NULL;
+	}
 
 	chan = dchan_to_axi_dma_chan(dchan);
-	chan->hw_handshake_num = dma_spec->args[0];
+	chan->hw_handshake_num = dma_spec->args[1];
+	int flags = (uintptr_t)of_device_get_match_data(chan->chip->dev);
+	if (flags & AXI_DMA_FLAG_HAS_EIC7700) {
+		if (dma_spec->args_count > 1)
+			win2030_dma_sel_cfg(chan, dma_spec->args[2]);
+	}
 	return dchan;
 }
 
@@ -1366,9 +1522,108 @@ static int parse_device_properties(struct axi_dma_chip *chip)
 		chip->dw->hdata->axi_rw_burst_len = tmp;
 	}
 
+	/* axi-max-burst-len is optional property */
+	ret = device_property_read_u32(dev, "snps,max-msize", &tmp);
+	if (!ret) {
+		if (tmp > 1024)
+			return -EINVAL;
+		if (tmp < 1)
+			return -EINVAL;
+
+		chip->dw->hdata->max_msize = tmp;
+	}else
+		chip->dw->hdata->max_msize = 4;
+	
 	return 0;
 }
 
+static int dw_axi_dma_ctrl_show(struct seq_file *s, void *v)
+{
+	struct axi_dma_chip *chip = s->private;
+	u32 i = 0, j = 0, val = 0;
+	u32 chan_num;
+
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-aon")) {
+		chan_num = 16;
+	} else {
+		chan_num = 11;
+	}
+
+	seq_puts(s, "\n-------------------dma common reg------------------------");
+
+	for (i = 0; i < 15; i++) {
+		if (i * 8 == 0x38)
+			continue;
+
+		val = axi_dma_ioread32(chip, i * 8);
+		seq_printf(s, "\n>>>dma reg 0x%02x:0x%02x", i*8, val);
+	}
+	seq_puts(s, "\n-------------------------------------------------------\n");
+
+	seq_puts(s, "\n--------------------dma chan reg-------------------------");
+	for (i = 0; i < chan_num; i++) {
+		seq_printf(s, "\n>>>dma chan[%d]reg:", i);
+		for (j=0; j<19; j++) {
+			if (j * 8 == 0x48)
+				continue;
+			val = axi_chan_ioread32(&chip->dw->chan[i], j*8);
+			seq_printf(s, " %02x:%02x", j*8, val);
+		}
+	}
+	seq_puts(s, "\n------------------------------------------------------\n");
+	return 0;
+}
+
+static int dw_axi_dma_ctrl_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_axi_dma_ctrl_show, inode->i_private);
+}
+
+static ssize_t dw_axi_dma_ctrl_write(struct file *file, const char __user *buf,
+				  size_t count, loff_t *ppos)
+{
+	struct axi_dma_chip *chip = ((struct seq_file *)file->private_data)->private;
+	u32 reg, val;
+	char kbuf[25];
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+	if (sscanf(kbuf, "%x%x", &reg, &val) == -1)
+		return -EFAULT;
+	if ((reg < 0) || (reg > 0x78)) {
+		dev_err(chip->dev, "it is no a dma register\n");
+		return count;
+	}
+	dev_info(chip->dev, "write dma common reg=0x%x, val=0x%x\n", reg, val);
+	axi_dma_iowrite32(chip, reg, val);
+	return count;
+}
+
+static const struct file_operations dw_axi_dma_status_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_axi_dma_ctrl_open,
+	.read = seq_read,
+	.write = dw_axi_dma_ctrl_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+void dw_axi_dma_register_debugfs(struct axi_dma_chip *chip)
+{
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-aon")) {
+		chip->debugfs_dir = debugfs_create_dir("dw-axi-aon-dma", NULL);
+	} else {
+		chip->debugfs_dir = debugfs_create_dir("dw-axi-hsp-dma", NULL);
+	}
+	if (IS_ERR(chip->debugfs_dir)) {
+		dev_err(chip->dev, "failed to create debugfs dir!\n");
+		return;
+	}
+
+	debugfs_create_file("ctrl", 0400, chip->debugfs_dir, chip,
+			    &dw_axi_dma_status_fops);
+}
+
 static int dw_probe(struct platform_device *pdev)
 {
 	struct axi_dma_chip *chip;
@@ -1403,6 +1658,8 @@ static int dw_probe(struct platform_device *pdev)
 	if (IS_ERR(chip->regs))
 		return PTR_ERR(chip->regs);
 
+	spin_lock_init(&chip->lock);
+
 	flags = (uintptr_t)of_device_get_match_data(&pdev->dev);
 	if (flags & AXI_DMA_FLAG_HAS_APB_REGS) {
 		chip->apb_regs = devm_platform_ioremap_resource(pdev, 1);
@@ -1419,6 +1676,23 @@ static int dw_probe(struct platform_device *pdev)
 		if (ret)
 			return ret;
 	}
+	if (flags & AXI_DMA_FLAG_HAS_2RESETS) {
+		resets = devm_reset_control_get_optional(&pdev->dev, "arst");
+		if (IS_ERR(resets))
+			return PTR_ERR(resets);
+
+		ret = reset_control_deassert(resets);
+		if (ret)
+			return ret;
+
+		resets = devm_reset_control_get_optional(&pdev->dev, "prst");
+		if (IS_ERR(resets))
+			return PTR_ERR(resets);
+
+		ret = reset_control_deassert(resets);
+		if (ret)
+			return ret;
+	}
 
 	chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2);
 
@@ -1462,6 +1736,10 @@ static int dw_probe(struct platform_device *pdev)
 	dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
 	dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask);
 
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-aon")) {
+		dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
+	}
+
 	/* DMA capabilities */
 	dw->dma.max_burst = hdata->axi_rw_burst_len;
 	dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS;
@@ -1525,6 +1803,8 @@ static int dw_probe(struct platform_device *pdev)
 	dev_info(chip->dev, "DesignWare AXI DMA Controller, %d channels\n",
 		 dw->hdata->nr_channels);
 
+	dw_axi_dma_register_debugfs(chip);
+
 	return 0;
 
 err_pm_disable:
@@ -1539,6 +1819,7 @@ static int dw_remove(struct platform_device *pdev)
 	struct dw_axi_dma *dw = chip->dw;
 	struct axi_dma_chan *chan, *_chan;
 	u32 i;
+	unsigned int flags;
 
 	/* Enable clk before accessing to registers */
 	clk_prepare_enable(chip->cfgr_clk);
@@ -1562,6 +1843,13 @@ static int dw_remove(struct platform_device *pdev)
 		list_del(&chan->vc.chan.device_node);
 		tasklet_kill(&chan->vc.task);
 	}
+	flags = (uintptr_t)of_device_get_match_data(&pdev->dev);
+	if (flags & AXI_DMA_FLAG_HAS_EIC7700) {
+		/* TBU power down before reset */
+		win2030_tbu_power(chip->dev, false);
+	}
+
+	debugfs_remove_recursive(chip->debugfs_dir);
 
 	return 0;
 }
@@ -1579,6 +1867,9 @@ static const struct of_device_id dw_dma_of_id_table[] = {
 	}, {
 		.compatible = "starfive,jh7110-axi-dma",
 		.data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2),
+	}, {
+		.compatible = "eswin,eic770x-axi-dma",
+		.data = (void *)(AXI_DMA_FLAG_HAS_2RESETS | AXI_DMA_FLAG_USE_CFG2 | AXI_DMA_FLAG_HAS_EIC7700),
 	},
 	{}
 };
@@ -1598,3 +1889,4 @@ module_platform_driver(dw_driver);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Synopsys DesignWare AXI DMA Controller platform driver");
 MODULE_AUTHOR("Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>");
+MODULE_AUTHOR("XuXiang <xuxiang@eswincomputing.com>");
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
index 8521530a34ec..bcbc75598af5 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
@@ -6,6 +6,25 @@
  *
  * Author: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>
  */
+/*****************************************************************************
+ * ESWIN dma driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
 
 #ifndef _AXI_DMA_PLATFORM_H
 #define _AXI_DMA_PLATFORM_H
@@ -30,6 +49,7 @@ struct dw_axi_dma_hcfg {
 	u32	priority[DMAC_MAX_CHANNELS];
 	/* maximum supported axi burst length */
 	u32	axi_rw_burst_len;
+	u32	max_msize;
 	/* Register map for DMAX_NUM_CHANNELS <= 8 */
 	bool	reg_map_8_channels;
 	bool	restrict_axi_burst_len;
@@ -71,6 +91,13 @@ struct axi_dma_chip {
 	struct clk		*core_clk;
 	struct clk		*cfgr_clk;
 	struct dw_axi_dma	*dw;
+	struct dentry *debugfs_dir;
+	spinlock_t lock;
+};
+
+struct axi_dma_filter_data {
+	struct dw_axi_dma *dw_dma;
+	int request;
 };
 
 /* LLI == Linked List Item */
@@ -203,7 +230,7 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
 #define UNUSED_CHANNEL		0x3F /* Set unused DMA channel to 0x3F */
 #define DMA_APB_HS_SEL_BIT_SIZE	0x08 /* HW handshake bits per channel */
 #define DMA_APB_HS_SEL_MASK	0xFF /* HW handshake select masks */
-#define MAX_BLOCK_SIZE		0x1000 /* 1024 blocks * 4 bytes data width */
+#define MAX_BLOCK_SIZE		0x200000 /* 1024 blocks * 512 bytes data width */
 #define DMA_REG_MAP_CH_REF	0x08 /* Channel count to choose register map */
 
 /* DMAC_CFG */
@@ -283,6 +310,8 @@ enum {
 #define CH_CTL_L_SRC_MAST		BIT(0)
 
 /* CH_CFG_H */
+#define CH_CFG_H_DST_OSR_LMT_POS	27
+#define CH_CFG_H_SRC_OSR_LMT_POS	23
 #define CH_CFG_H_PRIORITY_POS		17
 #define CH_CFG_H_DST_PER_POS		12
 #define CH_CFG_H_SRC_PER_POS		7
@@ -322,8 +351,9 @@ enum {
 #define CH_CFG2_H_TT_FC_POS		0
 #define CH_CFG2_H_HS_SEL_SRC_POS	3
 #define CH_CFG2_H_HS_SEL_DST_POS	4
-#define CH_CFG2_H_PRIORITY_POS		20
-
+#define CH_CFG2_H_PRIORITY_POS		15
+#define CH_CFG2_H_SRC_OSR_LMT_POS	23
+#define CH_CFG2_H_DST_OSR_LMT_POS	27
 /**
  * DW AXI DMA channel interrupts
  *
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index 110e99b86a66..95e8f7b8e909 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -561,4 +561,11 @@ config EDAC_NPCM
 	  error detection (in-line ECC in which a section 1/8th of the memory
 	  device used to store data is used for ECC storage).
 
+config EDAC_ESWIN
+	tristate "Synopsys DDR Memory Controller for Eswin"
+	depends on SOC_ESWIN
+	help
+	  Support for error detection and correction on the Synopsys DDR
+	  memory controller.
+
 endif # EDAC
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index 446364264e2b..c6809ccef45c 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -88,3 +88,4 @@ obj-$(CONFIG_EDAC_BLUEFIELD)		+= bluefield_edac.o
 obj-$(CONFIG_EDAC_DMC520)		+= dmc520_edac.o
 obj-$(CONFIG_EDAC_NPCM)			+= npcm_edac.o
 obj-$(CONFIG_EDAC_ZYNQMP)		+= zynqmp_edac.o
+obj-$(CONFIG_EDAC_ESWIN)		+= eswin_edac.o
diff --git a/drivers/edac/eswin_edac.c b/drivers/edac/eswin_edac.c
new file mode 100644
index 000000000000..ebf42694d3c0
--- /dev/null
+++ b/drivers/edac/eswin_edac.c
@@ -0,0 +1,1416 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Synopsys DDR ECC Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ */
+
+#include <linux/edac.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+
+#include "edac_module.h"
+
+/* Number of cs_rows needed per memory controller */
+#define SYNPS_EDAC_NR_CSROWS 1
+
+/* Number of channels per memory controller */
+#define SYNPS_EDAC_NR_CHANS 1
+
+/* Granularity of reported error in bytes */
+#define SYNPS_EDAC_ERR_GRAIN 1
+
+#define SYNPS_EDAC_MSG_SIZE 256
+
+#define SYNPS_EDAC_MOD_STRING "synps_edac"
+#define SYNPS_EDAC_MOD_VER "1"
+
+/* Synopsys DDR memory controller registers that are relevant to ECC */
+#define CTRL_OFST 0x10000
+#define T_ZQ_OFST 0x100A4
+
+/* ECC control register */
+#define ECC_CTRL_OFST 0xC4
+/* ECC log register */
+#define CE_LOG_OFST 0xC8
+/* ECC address register */
+#define CE_ADDR_OFST 0xCC
+/* ECC data[31:0] register */
+#define CE_DATA_31_0_OFST 0xD0
+
+/* Uncorrectable error info registers */
+#define UE_LOG_OFST 0xDC
+#define UE_ADDR_OFST 0xE0
+#define UE_DATA_31_0_OFST 0xE4
+
+#define STAT_OFST 0xF0
+#define SCRUB_OFST 0xF4
+
+/* Control register bit field definitions */
+#define CTRL_BW_MASK 0xC
+#define CTRL_BW_SHIFT 2
+
+#define DDRCTL_WDTH_16 1
+#define DDRCTL_WDTH_32 0
+
+/* ZQ register bit field definitions */
+#define T_ZQ_DDRMODE_MASK 0x2
+
+/* ECC control register bit field definitions */
+#define ECC_CTRL_CLR_CE_ERR 0x2
+#define ECC_CTRL_CLR_UE_ERR 0x1
+
+/* ECC correctable/uncorrectable error log register definitions */
+#define LOG_VALID 0x1
+#define CE_LOG_BITPOS_MASK 0xFE
+#define CE_LOG_BITPOS_SHIFT 1
+
+/* ECC correctable/uncorrectable error address register definitions */
+#define ADDR_COL_MASK 0xFFF
+#define ADDR_ROW_MASK 0xFFFF000
+#define ADDR_ROW_SHIFT 12
+#define ADDR_BANK_MASK 0x70000000
+#define ADDR_BANK_SHIFT 28
+
+/* ECC statistic register definitions */
+#define STAT_UECNT_MASK 0xFFFF0000
+#define STAT_CECNT_MASK 0xFFFF
+#define STAT_UECNT_SHIFT 16
+
+/* ECC scrub register definitions */
+#define SCRUB_MODE_MASK 0x7
+#define SCRUB_MODE_SECDED 0x4
+
+/* DDR ECC Quirks */
+#define DDR_ECC_INTR_SUPPORT BIT(0)
+#define DDR_ECC_DATA_POISON_SUPPORT BIT(1)
+#define DDR_ECC_INTR_SELF_CLEAR BIT(2)
+
+/* ZynqMP Enhanced DDR memory controller registers that are relevant to ECC */
+/* ECC Configuration Registers */
+#define ECC_CFG0_OFST 0x10600
+#define ECC_CFG1_OFST 0x10604
+
+/* ECC Status Register */
+#define ECC_STAT_OFST 0x10608
+
+/* ECC Clear Register */
+#define ECC_CLR_OFST 0x1060C
+
+/* ECC Error count Register */
+#define ECC_ERRCNT_OFST 0x10610
+
+/* ECC Corrected Error Address Register */
+#define ECC_CEADDR0_OFST 0x10614
+#define ECC_CEADDR1_OFST 0x10618
+
+/* ECC Syndrome Registers */
+#define ECC_CSYND0_OFST 0x1061C
+#define ECC_CSYND1_OFST 0x10620
+#define ECC_CSYND2_OFST 0x10624
+
+/* ECC Bit Mask0 Address Register */
+#define ECC_BITMASK0_OFST 0x10628
+#define ECC_BITMASK1_OFST 0x1062C
+#define ECC_BITMASK2_OFST 0x10630
+
+/* ECC UnCorrected Error Address Register */
+#define ECC_UEADDR0_OFST 0x10634
+#define ECC_UEADDR1_OFST 0x10638
+
+/* ECC Syndrome Registers */
+#define ECC_UESYND0_OFST 0x1063C
+#define ECC_UESYND1_OFST 0x10640
+#define ECC_UESYND2_OFST 0x10644
+
+/* ECC Poison Address Reg */
+#define ECC_POISON0_OFST 0x10648
+#define ECC_POISON1_OFST 0x1064C
+
+#define ECC_ADDRMAP0_OFFSET 0x200
+
+/* Control register bitfield definitions */
+#define ECC_CTRL_BUSWIDTH_MASK 0x3000
+#define ECC_CTRL_BUSWIDTH_SHIFT 12
+#define ECC_CTRL_CLR_CE_ERRCNT BIT(2)
+#define ECC_CTRL_CLR_UE_ERRCNT BIT(3)
+
+/* DDR Control Register width definitions  */
+#define DDRCTL_EWDTH_16 2
+#define DDRCTL_EWDTH_32 1
+#define DDRCTL_EWDTH_64 0
+
+/* ECC status register definitions */
+#define ECC_STAT_UECNT_MASK 0xF0000
+#define ECC_STAT_UECNT_SHIFT 16
+#define ECC_STAT_CECNT_MASK 0xF00
+#define ECC_STAT_CECNT_SHIFT 8
+#define ECC_STAT_BITNUM_MASK 0x7F
+
+/* DDR QOS Interrupt register definitions */
+#define DDR_QOS_IRQ_STAT_OFST 0x20200
+#define DDR_QOSUE_MASK 0x4
+#define DDR_QOSCE_MASK 0x2
+#define ECC_CE_UE_INTR_MASK 0x6
+#define DDR_QOS_IRQ_EN_OFST 0x20208
+#define DDR_QOS_IRQ_DB_OFST 0x2020C
+
+/* DDR QOS Interrupt register definitions */
+#define DDR_UE_MASK BIT(9)
+#define DDR_CE_MASK BIT(8)
+
+/* ECC Corrected Error Register Mask and Shifts*/
+#define ECC_CEADDR0_RW_MASK 0x3FFFF
+#define ECC_CEADDR0_RNK_MASK BIT(24)
+#define ECC_CEADDR1_BNKGRP_MASK 0x3000000
+#define ECC_CEADDR1_BNKNR_MASK 0x70000
+#define ECC_CEADDR1_CID_MASK 0xF0000000
+#define ECC_CEADDR1_COL_MASK 0x7FF
+#define ECC_CEADDR1_CID_SHIFT 28
+#define ECC_CEADDR1_BNKGRP_SHIFT 24
+#define ECC_CEADDR1_BNKNR_SHIFT 16
+#define ECC_CEADDR0_RNK_SHIFT 24
+
+/* ECC Poison register shifts */
+#define ECC_POISON0_RANK_SHIFT 24
+#define ECC_POISON0_RANK_MASK BIT(24)
+#define ECC_POISON0_COLUMN_SHIFT 0
+#define ECC_POISON0_COLUMN_MASK 0xFFF
+#define ECC_POISON1_BG_SHIFT 28
+#define ECC_POISON1_BG_MASK 0x30000000
+#define ECC_POISON1_BANKNR_SHIFT 24
+#define ECC_POISON1_BANKNR_MASK 0x7000000
+#define ECC_POISON1_ROW_SHIFT 0
+#define ECC_POISON1_ROW_MASK 0x3FFFF
+
+/* DDR Memory type defines */
+#define MEM_TYPE_DDR3 0x1
+#define MEM_TYPE_LPDDR3 0x8
+#define MEM_TYPE_DDR2 0x4
+#define MEM_TYPE_DDR4 0x10
+#define MEM_TYPE_LPDDR4 0x20
+
+/* DDRC Software control register */
+#define DDRC_SWCTL 0x10C80
+
+/* DDRC ECC CE & UE poison mask */
+#define ECC_CEPOISON_MASK 0x3
+#define ECC_UEPOISON_MASK 0x1
+
+/* DDRC Device config masks */
+#define DDRC_MSTR_CFG_MASK 0xC0000000
+#define DDRC_MSTR_CFG_SHIFT 30
+#define DDRC_MSTR_CFG_X4_MASK 0x0
+#define DDRC_MSTR_CFG_X8_MASK 0x1
+#define DDRC_MSTR_CFG_X16_MASK 0x2
+#define DDRC_MSTR_CFG_X32_MASK 0x3
+
+#define DDR_MAX_ROW_SHIFT 18
+#define DDR_MAX_COL_SHIFT 14
+#define DDR_MAX_BANK_SHIFT 3
+#define DDR_MAX_BANKGRP_SHIFT 2
+
+#define ROW_MAX_VAL_MASK 0xF
+#define COL_MAX_VAL_MASK 0xF
+#define BANK_MAX_VAL_MASK 0x1F
+#define BANKGRP_MAX_VAL_MASK 0x1F
+#define RANK_MAX_VAL_MASK 0x1F
+
+#define ROW_B0_BASE 6
+#define ROW_B1_BASE 7
+#define ROW_B2_BASE 8
+#define ROW_B3_BASE 9
+#define ROW_B4_BASE 10
+#define ROW_B5_BASE 11
+#define ROW_B6_BASE 12
+#define ROW_B7_BASE 13
+#define ROW_B8_BASE 14
+#define ROW_B9_BASE 15
+#define ROW_B10_BASE 16
+#define ROW_B11_BASE 17
+#define ROW_B12_BASE 18
+#define ROW_B13_BASE 19
+#define ROW_B14_BASE 20
+#define ROW_B15_BASE 21
+#define ROW_B16_BASE 22
+#define ROW_B17_BASE 23
+
+#define COL_B2_BASE 2
+#define COL_B3_BASE 3
+#define COL_B4_BASE 4
+#define COL_B5_BASE 5
+#define COL_B6_BASE 6
+#define COL_B7_BASE 7
+#define COL_B8_BASE 8
+#define COL_B9_BASE 9
+#define COL_B10_BASE 10
+#define COL_B11_BASE 11
+#define COL_B12_BASE 12
+#define COL_B13_BASE 13
+
+#define BANK_B0_BASE 2
+#define BANK_B1_BASE 3
+#define BANK_B2_BASE 4
+
+#define BANKGRP_B0_BASE 2
+#define BANKGRP_B1_BASE 3
+
+#define RANK_B0_BASE 6
+
+/**
+ * struct ecc_error_info - ECC error log information.
+ * @row:	Row number.
+ * @col:	Column number.
+ * @rank:	Rank number.
+ * @bank:	Bank number.
+ * @bitpos:	Bit position.
+ * @data:	Data causing the error.
+ * @bankgrpnr:	Bank group number.
+ * @blknr:	Block number.
+ */
+struct ecc_error_info
+{
+	u32 row;
+	u32 col;
+	u32 rank;
+	u32 bank;
+	u32 bitpos;
+	u32 data;
+	u32 bankgrpnr;
+	u32 cid;
+};
+
+/**
+ * struct synps_ecc_status - ECC status information to report.
+ * @ce_cnt:	Correctable error count.
+ * @ue_cnt:	Uncorrectable error count.
+ * @ceinfo:	Correctable error log information.
+ * @ueinfo:	Uncorrectable error log information.
+ */
+struct synps_ecc_status
+{
+	u32 ce_cnt;
+	u32 ue_cnt;
+	struct ecc_error_info ceinfo;
+	struct ecc_error_info ueinfo;
+};
+
+/**
+ * struct synps_edac_priv - DDR memory controller private instance data.
+ * @baseaddr:		Base address of the DDR controller.
+ * @message:		Buffer for framing the event specific info.
+ * @stat:		ECC status information.
+ * @p_data:		Platform data.
+ * @ce_cnt:		Correctable Error count.
+ * @ue_cnt:		Uncorrectable Error count.
+ * @poison_addr:	Data poison address.
+ * @row_shift:		Bit shifts for row bit.
+ * @col_shift:		Bit shifts for column bit.
+ * @bank_shift:		Bit shifts for bank bit.
+ * @bankgrp_shift:	Bit shifts for bank group bit.
+ * @rank_shift:		Bit shifts for rank bit.
+ */
+struct synps_edac_priv
+{
+	void __iomem *baseaddr;
+	resource_size_t start;
+	char message[SYNPS_EDAC_MSG_SIZE];
+	struct synps_ecc_status stat;
+	const struct synps_platform_data *p_data;
+	u32 ce_cnt;
+	u32 ue_cnt;
+#ifdef CONFIG_EDAC_DEBUG
+	ulong poison_addr;
+	u32 row_shift[18];
+	u32 col_shift[14];
+	u32 bank_shift[3];
+	u32 bankgrp_shift[2];
+	u32 rank_shift[1];
+#endif
+};
+
+/**
+ * struct synps_platform_data -  synps platform data structure.
+ * @get_error_info:	Get EDAC error info.
+ * @get_mtype:		Get mtype.
+ * @get_dtype:		Get dtype.
+ * @get_ecc_state:	Get ECC state.
+ * @quirks:		To differentiate IPs.
+ */
+struct synps_platform_data
+{
+	int (*get_error_info)(struct synps_edac_priv *priv);
+	enum mem_type (*get_mtype)(const void __iomem *base);
+	enum dev_type (*get_dtype)(const void __iomem *base);
+	bool (*get_ecc_state)(void __iomem *base);
+	int quirks;
+};
+
+/**
+ * eswin_get_error_info - Get the current ECC error info.
+ * @priv:	DDR memory controller private instance data.
+ *
+ * Return: one if there is no error otherwise returns zero.
+ */
+static int eswin_get_error_info(struct synps_edac_priv *priv)
+{
+	struct synps_ecc_status *p;
+	u32 regval = 0, clearval = 0, cache = 0;
+	void __iomem *base;
+
+	base = priv->baseaddr;
+	p = &priv->stat;
+
+	regval = readl(base + ECC_STAT_OFST);
+	if (!regval)
+		return 1;
+
+	p->ce_cnt = (regval & ECC_STAT_CECNT_MASK) >> ECC_STAT_CECNT_SHIFT;
+	p->ue_cnt = (regval & ECC_STAT_UECNT_MASK) >> ECC_STAT_UECNT_SHIFT;
+	pr_err("ctrl 0x%llx ecc state:%x,sbr_read_ecc_ue:%d,sbr_read_ecc_ue:%d,"
+		   "ecc_uncorrected_err:0x%08x,ecc_corrected_err:0x%08x,ecc_corrected_bit_num:0x%x\n",
+		   priv->start, regval, (regval >> 25) & 0x1, (regval >> 24) & 0x1, p->ue_cnt,
+		   p->ce_cnt, regval & 0x7f);
+
+	cache = readl(base + ECC_ERRCNT_OFST);
+	pr_err("ctrl 0x%llx uncorr_cnt:0x%04x,corr_cnt:0x%04x\n", priv->start, (cache >> 16) & 0xffff, cache & 0xffff);
+
+	if (!p->ce_cnt)
+		goto ue_err;
+
+	pr_err("ECC_CEADDR0: 0x%08X ECC_CEADDR1: 0x%08X ECC_BITMASK0: 0x%08X ECC_BITMASK1: 0x%08X ECC_BITMASK2: 0x%08X\n",
+		   readl(base + ECC_CEADDR0_OFST), readl(base + ECC_CEADDR1_OFST),
+		   readl(base + ECC_BITMASK0_OFST), readl(base + ECC_BITMASK1_OFST),
+		   readl(base + ECC_BITMASK2_OFST));
+
+	pr_err("ECCCSYN0: 0x%08X ECCCSYN1: 0x%08X ECCCSYN2: 0x%08X\n",
+
+		   readl(base + ECC_CSYND0_OFST), readl(base + ECC_CSYND1_OFST),
+		   readl(base + ECC_CSYND2_OFST));
+ue_err:
+	if (!p->ue_cnt)
+		goto out;
+
+	pr_err("ECC_UEADDR0: 0x%08X ECC_UEADDR1: 0x%08X ECUCSYN0: 0x%08X ECUCSYN1: 0x%08X ECUCSYN2: 0x%08X\n",
+		   readl(base + ECC_UEADDR0_OFST), readl(base + ECC_UEADDR1_OFST),
+		   readl(base + ECC_UESYND0_OFST), readl(base + ECC_UESYND1_OFST),
+		   readl(base + ECC_UESYND2_OFST));
+out:
+	clearval = ECC_CTRL_CLR_CE_ERR | ECC_CTRL_CLR_CE_ERRCNT;
+	clearval |= ECC_CTRL_CLR_UE_ERR | ECC_CTRL_CLR_UE_ERRCNT;
+	writel(clearval, base + ECC_CLR_OFST);
+	writel(0x0, base + ECC_CLR_OFST);
+
+	regval = readl(base + ECC_STAT_OFST);
+
+	cache = readl(base + ECC_ERRCNT_OFST);
+
+	return 0;
+}
+
+/**
+ * handle_error - Handle Correctable and Uncorrectable errors.
+ * @mci:	EDAC memory controller instance.
+ * @p:		Synopsys ECC status structure.
+ *
+ * Handles ECC correctable and uncorrectable errors.
+ */
+static void handle_error(struct mem_ctl_info *mci, struct synps_ecc_status *p)
+{
+	struct synps_edac_priv *priv = mci->pvt_info;
+	struct ecc_error_info *pinf;
+
+	if (p->ce_cnt)
+	{
+		pinf = &p->ceinfo;
+		if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+		{
+			snprintf(priv->message, SYNPS_EDAC_MSG_SIZE,
+					 "DDR ECC error type:%s Row %d Col %d Rank %d Bank %d BankGroup Number %d Cid %d Bit Position: %d Data: 0x%08x",
+					 "CE", pinf->row, pinf->col, pinf->rank, pinf->bank,
+					 pinf->bankgrpnr, pinf->cid,
+					 pinf->bitpos, pinf->data);
+		}
+		else
+		{
+			snprintf(priv->message, SYNPS_EDAC_MSG_SIZE,
+					 "DDR ECC error type:%s Row %d Bank %d Col %d Bit Position: %d Data: 0x%08x",
+					 "CE", pinf->row, pinf->bank, pinf->col,
+					 pinf->bitpos, pinf->data);
+		}
+
+		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci,
+							 p->ce_cnt, 0, 0, 0, 0, 0, -1,
+							 priv->message, "");
+	}
+
+	if (p->ue_cnt)
+	{
+		pinf = &p->ueinfo;
+		if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+		{
+			snprintf(priv->message, SYNPS_EDAC_MSG_SIZE,
+					 "DDR ECC error type :%s Row %d Col %d Rank %d Bank %d BankGroup Number %d Cid %d",
+					 "UE", pinf->row, pinf->col, pinf->rank, pinf->bank,
+					 pinf->bankgrpnr, pinf->cid);
+		}
+		else
+		{
+			snprintf(priv->message, SYNPS_EDAC_MSG_SIZE,
+					 "DDR ECC error type :%s Row %d Bank %d Col %d ",
+					 "UE", pinf->row, pinf->bank, pinf->col);
+		}
+
+		edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci,
+							 p->ue_cnt, 0, 0, 0, 0, 0, -1,
+							 priv->message, "");
+	}
+
+	memset(p, 0, sizeof(*p));
+}
+
+/**
+ * intr_handler - Interrupt Handler for ECC interrupts.
+ * @irq:        IRQ number.
+ * @dev_id:     Device ID.
+ *
+ * Return: IRQ_NONE, if interrupt not set or IRQ_HANDLED otherwise.
+ */
+static irqreturn_t intr_handler(int irq, void *dev_id)
+{
+	const struct synps_platform_data *p_data;
+	struct mem_ctl_info *mci = dev_id;
+	struct synps_edac_priv *priv;
+	int status, regval;
+
+	priv = mci->pvt_info;
+	p_data = priv->p_data;
+
+	dev_warn(&mci->dev, "ddr ecc interrupt\n");
+
+	/*
+	 * v3.0 of the controller has the ce/ue bits cleared automatically,
+	 * so this condition does not apply.
+	 */
+	if (!(priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR))
+	{
+		regval = readl(priv->baseaddr + DDR_QOS_IRQ_STAT_OFST);
+		regval &= (DDR_QOSCE_MASK | DDR_QOSUE_MASK);
+		if (!(regval & ECC_CE_UE_INTR_MASK))
+			return IRQ_NONE;
+	}
+
+	status = p_data->get_error_info(priv);
+	/* if (status)
+		return IRQ_NONE; */
+
+	priv->ce_cnt += priv->stat.ce_cnt;
+	priv->ue_cnt += priv->stat.ue_cnt;
+	handle_error(mci, &priv->stat);
+
+	edac_dbg(3, "Total error count CE %d UE %d\n",
+			 priv->ce_cnt, priv->ue_cnt);
+	/* v3.0 of the controller does not have this register */
+	if (!(priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR))
+	{
+		writel(regval, priv->baseaddr + DDR_QOS_IRQ_STAT_OFST);
+	}
+	return IRQ_HANDLED;
+}
+
+/**
+ * check_errors - Check controller for ECC errors.
+ * @mci:	EDAC memory controller instance.
+ *
+ * Check and post ECC errors. Called by the polling thread.
+ */
+static void check_errors(struct mem_ctl_info *mci)
+{
+	const struct synps_platform_data *p_data;
+	struct synps_edac_priv *priv;
+	int status;
+
+	priv = mci->pvt_info;
+	p_data = priv->p_data;
+
+	status = p_data->get_error_info(priv);
+	if (status)
+		return;
+
+	priv->ce_cnt += priv->stat.ce_cnt;
+	priv->ue_cnt += priv->stat.ue_cnt;
+	handle_error(mci, &priv->stat);
+
+	edac_dbg(3, "Total error count CE %d UE %d\n",
+			 priv->ce_cnt, priv->ue_cnt);
+}
+
+/**
+ * eswin_get_dtype - Return the controller memory width.
+ * @base:	DDR memory controller base address.
+ *
+ * Get the EDAC device type width appropriate for the current controller
+ * configuration.
+ *
+ * Return: a device type width enumeration.
+ */
+static enum dev_type eswin_get_dtype(const void __iomem *base)
+{
+	enum dev_type dt;
+	u32 width;
+
+	width = readl(base + CTRL_OFST);
+	width = (width & ECC_CTRL_BUSWIDTH_MASK) >> ECC_CTRL_BUSWIDTH_SHIFT;
+	switch (width)
+	{
+	case DDRCTL_EWDTH_16:
+		dt = DEV_X2;
+		break;
+	case DDRCTL_EWDTH_32:
+		dt = DEV_X4;
+		break;
+	case DDRCTL_EWDTH_64:
+		dt = DEV_X8;
+		break;
+	default:
+		dt = DEV_UNKNOWN;
+	}
+
+	return dt;
+}
+
+/**
+ * eswin_get_ecc_state - Return the controller ECC enable/disable status.
+ * @base:	DDR memory controller base address.
+ *
+ * Get the ECC enable/disable status for the controller.
+ *
+ * Return: a ECC status boolean i.e true/false - enabled/disabled.
+ */
+static bool eswin_get_ecc_state(void __iomem *base)
+{
+	enum dev_type dt;
+	u32 ecctype;
+
+	dt = eswin_get_dtype(base);
+	if (dt == DEV_UNKNOWN)
+		return false;
+
+	ecctype = readl(base + ECC_CFG0_OFST) & SCRUB_MODE_MASK;
+	// if ((ecctype == SCRUB_MODE_SECDED) &&
+	//     ((dt == DEV_X2) || (dt == DEV_X4) || (dt == DEV_X8)))
+	// 	return true;
+	pr_debug("ECC mode: 1/%d memory\n", 1 << (ecctype + 3));
+	return true;
+}
+
+/**
+ * get_memsize - Read the size of the attached memory device.
+ *
+ * Return: the memory size in bytes.
+ */
+static u32 get_memsize(void)
+{
+	struct sysinfo inf;
+
+	si_meminfo(&inf);
+
+	return inf.totalram * inf.mem_unit;
+}
+
+/**
+ * eswin_get_mtype - Returns controller memory type.
+ * @base:	Synopsys ECC status structure.
+ *
+ * Get the EDAC memory type appropriate for the current controller
+ * configuration.
+ *
+ * Return: a memory type enumeration.
+ */
+static enum mem_type eswin_get_mtype(const void __iomem *base)
+{
+	enum mem_type mt;
+	u32 memtype;
+
+	memtype = readl(base + CTRL_OFST);
+
+	if ((memtype & MEM_TYPE_DDR3) || (memtype & MEM_TYPE_LPDDR3))
+		mt = MEM_DDR3;
+	else if (memtype & MEM_TYPE_DDR2)
+		mt = MEM_RDDR2;
+	else if ((memtype & MEM_TYPE_LPDDR4) || (memtype & MEM_TYPE_DDR4))
+		mt = MEM_DDR4;
+	else
+		mt = MEM_EMPTY;
+
+	return mt;
+}
+
+/**
+ * init_csrows - Initialize the csrow data.
+ * @mci:	EDAC memory controller instance.
+ *
+ * Initialize the chip select rows associated with the EDAC memory
+ * controller instance.
+ */
+static void init_csrows(struct mem_ctl_info *mci)
+{
+	struct synps_edac_priv *priv = mci->pvt_info;
+	const struct synps_platform_data *p_data;
+	struct csrow_info *csi;
+	struct dimm_info *dimm;
+	u32 size, row;
+	int j;
+
+	p_data = priv->p_data;
+
+	for (row = 0; row < mci->nr_csrows; row++)
+	{
+		csi = mci->csrows[row];
+		size = get_memsize();
+
+		for (j = 0; j < csi->nr_channels; j++)
+		{
+			dimm = csi->channels[j]->dimm;
+			dimm->edac_mode = EDAC_SECDED;
+			dimm->mtype = p_data->get_mtype(priv->baseaddr);
+			dimm->nr_pages = (size >> PAGE_SHIFT) / csi->nr_channels;
+			dimm->grain = SYNPS_EDAC_ERR_GRAIN;
+			dimm->dtype = p_data->get_dtype(priv->baseaddr);
+		}
+	}
+}
+
+/**
+ * mc_init - Initialize one driver instance.
+ * @mci:	EDAC memory controller instance.
+ * @pdev:	platform device.
+ *
+ * Perform initialization of the EDAC memory controller instance and
+ * related driver-private data associated with the memory controller the
+ * instance is bound to.
+ */
+static void mc_init(struct mem_ctl_info *mci, struct platform_device *pdev)
+{
+	struct synps_edac_priv *priv;
+
+	mci->pdev = &pdev->dev;
+	priv = mci->pvt_info;
+	platform_set_drvdata(pdev, mci);
+
+	/* Initialize controller capabilities and configuration */
+	mci->mtype_cap = MEM_FLAG_DDR3 | MEM_FLAG_DDR2;
+	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
+	mci->scrub_cap = SCRUB_HW_SRC;
+	mci->scrub_mode = SCRUB_NONE;
+
+	mci->edac_cap = EDAC_FLAG_SECDED;
+	mci->ctl_name = "synps_ddr_controller";
+	mci->dev_name = SYNPS_EDAC_MOD_STRING;
+	mci->mod_name = SYNPS_EDAC_MOD_VER;
+
+	if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+	{
+		edac_op_state = EDAC_OPSTATE_INT;
+	}
+	else
+	{
+		edac_op_state = EDAC_OPSTATE_POLL;
+		mci->edac_check = check_errors;
+	}
+
+	mci->ctl_page_to_phys = NULL;
+
+	init_csrows(mci);
+}
+
+static void enable_intr(struct synps_edac_priv *priv)
+{
+	/* Enable UE/CE Interrupts */
+	if (priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR)
+		writel(DDR_UE_MASK | DDR_CE_MASK,
+			   priv->baseaddr + ECC_CLR_OFST);
+	else
+		writel(DDR_QOSUE_MASK | DDR_QOSCE_MASK,
+			   priv->baseaddr + DDR_QOS_IRQ_EN_OFST);
+}
+
+static void disable_intr(struct synps_edac_priv *priv)
+{
+	/* Disable UE/CE Interrupts */
+	writel(DDR_QOSUE_MASK | DDR_QOSCE_MASK,
+		   priv->baseaddr + DDR_QOS_IRQ_DB_OFST);
+}
+
+static int setup_irq(struct mem_ctl_info *mci,
+					 struct platform_device *pdev)
+{
+	struct synps_edac_priv *priv = mci->pvt_info;
+	int ret, irq;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+	{
+		edac_printk(KERN_ERR, EDAC_MC,
+					"No IRQ %d in DT\n", irq);
+		return irq;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq, intr_handler,
+						   0, dev_name(&pdev->dev), mci);
+	if (ret < 0)
+	{
+		edac_printk(KERN_ERR, EDAC_MC, "Failed to request IRQ\n");
+		return ret;
+	}
+
+	enable_intr(priv);
+
+	return 0;
+}
+
+static const struct synps_platform_data eswin_edac_def = {
+	.get_error_info = eswin_get_error_info,
+	.get_mtype = eswin_get_mtype,
+	.get_dtype = eswin_get_dtype,
+	.get_ecc_state = eswin_get_ecc_state,
+	.quirks = (DDR_ECC_INTR_SUPPORT | DDR_ECC_INTR_SELF_CLEAR
+#ifdef CONFIG_EDAC_DEBUG
+			   | DDR_ECC_DATA_POISON_SUPPORT
+#endif
+			   ),
+};
+
+static const struct of_device_id synps_edac_match[] = {
+	{.compatible = "eswin,ddrc-1.20a",
+	 .data = (void *)&eswin_edac_def},
+	{
+		/* end of table */
+	}};
+
+MODULE_DEVICE_TABLE(of, synps_edac_match);
+
+#ifdef CONFIG_EDAC_DEBUG
+#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
+
+/**
+ * ddr_poison_setup -	Update poison registers.
+ * @priv:		DDR memory controller private instance data.
+ *
+ * Update poison registers as per DDR mapping.
+ * Return: none.
+ */
+static void ddr_poison_setup(struct synps_edac_priv *priv)
+{
+	int col = 0, row = 0, bank = 0, bankgrp = 0, rank = 0, regval;
+	int index;
+	ulong hif_addr = 0;
+
+	hif_addr = priv->poison_addr >> 3;
+
+	for (index = 0; index < DDR_MAX_ROW_SHIFT; index++)
+	{
+		if (priv->row_shift[index])
+			row |= (((hif_addr >> priv->row_shift[index]) &
+					 BIT(0))
+					<< index);
+		else
+			break;
+	}
+
+	for (index = 0; index < DDR_MAX_COL_SHIFT; index++)
+	{
+		if (priv->col_shift[index] || index < 3)
+			col |= (((hif_addr >> priv->col_shift[index]) &
+					 BIT(0))
+					<< index);
+		else
+			break;
+	}
+
+	for (index = 0; index < DDR_MAX_BANK_SHIFT; index++)
+	{
+		if (priv->bank_shift[index])
+			bank |= (((hif_addr >> priv->bank_shift[index]) &
+					  BIT(0))
+					 << index);
+		else
+			break;
+	}
+
+	for (index = 0; index < DDR_MAX_BANKGRP_SHIFT; index++)
+	{
+		if (priv->bankgrp_shift[index])
+			bankgrp |= (((hif_addr >> priv->bankgrp_shift[index]) & BIT(0)) << index);
+		else
+			break;
+	}
+
+	if (priv->rank_shift[0])
+		rank = (hif_addr >> priv->rank_shift[0]) & BIT(0);
+
+	regval = (rank << ECC_POISON0_RANK_SHIFT) & ECC_POISON0_RANK_MASK;
+	regval |= (col << ECC_POISON0_COLUMN_SHIFT) & ECC_POISON0_COLUMN_MASK;
+	writel(regval, priv->baseaddr + ECC_POISON0_OFST);
+
+	regval = (bankgrp << ECC_POISON1_BG_SHIFT) & ECC_POISON1_BG_MASK;
+	regval |= (bank << ECC_POISON1_BANKNR_SHIFT) & ECC_POISON1_BANKNR_MASK;
+	regval |= (row << ECC_POISON1_ROW_SHIFT) & ECC_POISON1_ROW_MASK;
+	writel(regval, priv->baseaddr + ECC_POISON1_OFST);
+}
+
+static ssize_t inject_data_error_show(struct device *dev,
+									  struct device_attribute *mattr,
+									  char *data)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct synps_edac_priv *priv = mci->pvt_info;
+
+	return sprintf(data, "Poison0 Addr: 0x%08x\n\rPoison1 Addr: 0x%08x\n\r"
+						 "Error injection Address: 0x%lx\n\r",
+				   readl(priv->baseaddr + ECC_POISON0_OFST),
+				   readl(priv->baseaddr + ECC_POISON1_OFST),
+				   priv->poison_addr);
+}
+
+static ssize_t inject_data_error_store(struct device *dev,
+									   struct device_attribute *mattr,
+									   const char *data, size_t count)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct synps_edac_priv *priv = mci->pvt_info;
+
+	if (kstrtoul(data, 0, &priv->poison_addr))
+		return -EINVAL;
+
+	ddr_poison_setup(priv);
+
+	return count;
+}
+
+static ssize_t inject_data_poison_show(struct device *dev,
+									   struct device_attribute *mattr,
+									   char *data)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct synps_edac_priv *priv = mci->pvt_info;
+
+	return sprintf(data, "Data Poisoning: %s\n\r",
+				   (((readl(priv->baseaddr + ECC_CFG1_OFST)) & 0x3) == 0x3)
+					   ? ("Correctable Error")
+					   : ("UnCorrectable Error"));
+}
+
+static ssize_t inject_data_poison_store(struct device *dev,
+										struct device_attribute *mattr,
+										const char *data, size_t count)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct synps_edac_priv *priv = mci->pvt_info;
+
+	writel(0, priv->baseaddr + DDRC_SWCTL);
+	if (strncmp(data, "CE", 2) == 0)
+		writel(ECC_CEPOISON_MASK, priv->baseaddr + ECC_CFG1_OFST);
+	else
+		writel(ECC_UEPOISON_MASK, priv->baseaddr + ECC_CFG1_OFST);
+	writel(1, priv->baseaddr + DDRC_SWCTL);
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(inject_data_error);
+static DEVICE_ATTR_RW(inject_data_poison);
+
+static int edac_create_sysfs_attributes(struct mem_ctl_info *mci)
+{
+	int rc;
+
+	rc = device_create_file(&mci->dev, &dev_attr_inject_data_error);
+	if (rc < 0)
+		return rc;
+	rc = device_create_file(&mci->dev, &dev_attr_inject_data_poison);
+	if (rc < 0)
+		return rc;
+	return 0;
+}
+
+static void edac_remove_sysfs_attributes(struct mem_ctl_info *mci)
+{
+	device_remove_file(&mci->dev, &dev_attr_inject_data_error);
+	device_remove_file(&mci->dev, &dev_attr_inject_data_poison);
+}
+
+static void setup_row_address_map(struct synps_edac_priv *priv, u32 *addrmap)
+{
+	u32 addrmap_row_b2_10;
+	int index;
+
+	priv->row_shift[0] = (addrmap[5] & ROW_MAX_VAL_MASK) + ROW_B0_BASE;
+	priv->row_shift[1] = ((addrmap[5] >> 8) &
+						  ROW_MAX_VAL_MASK) +
+						 ROW_B1_BASE;
+
+	addrmap_row_b2_10 = (addrmap[5] >> 16) & ROW_MAX_VAL_MASK;
+	if (addrmap_row_b2_10 != ROW_MAX_VAL_MASK)
+	{
+		for (index = 2; index < 11; index++)
+			priv->row_shift[index] = addrmap_row_b2_10 +
+									 index + ROW_B0_BASE;
+	}
+	else
+	{
+		priv->row_shift[2] = (addrmap[9] &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B2_BASE;
+		priv->row_shift[3] = ((addrmap[9] >> 8) &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B3_BASE;
+		priv->row_shift[4] = ((addrmap[9] >> 16) &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B4_BASE;
+		priv->row_shift[5] = ((addrmap[9] >> 24) &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B5_BASE;
+		priv->row_shift[6] = (addrmap[10] &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B6_BASE;
+		priv->row_shift[7] = ((addrmap[10] >> 8) &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B7_BASE;
+		priv->row_shift[8] = ((addrmap[10] >> 16) &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B8_BASE;
+		priv->row_shift[9] = ((addrmap[10] >> 24) &
+							  ROW_MAX_VAL_MASK) +
+							 ROW_B9_BASE;
+		priv->row_shift[10] = (addrmap[11] &
+							   ROW_MAX_VAL_MASK) +
+							  ROW_B10_BASE;
+	}
+
+	priv->row_shift[11] = (((addrmap[5] >> 24) & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : (((addrmap[5] >> 24) &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B11_BASE);
+	priv->row_shift[12] = ((addrmap[6] & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : ((addrmap[6] &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B12_BASE);
+	priv->row_shift[13] = (((addrmap[6] >> 8) & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : (((addrmap[6] >> 8) &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B13_BASE);
+	priv->row_shift[14] = (((addrmap[6] >> 16) & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : (((addrmap[6] >> 16) &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B14_BASE);
+	priv->row_shift[15] = (((addrmap[6] >> 24) & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : (((addrmap[6] >> 24) &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B15_BASE);
+	priv->row_shift[16] = ((addrmap[7] & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : ((addrmap[7] &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B16_BASE);
+	priv->row_shift[17] = (((addrmap[7] >> 8) & ROW_MAX_VAL_MASK) ==
+						   ROW_MAX_VAL_MASK)
+							  ? 0
+							  : (((addrmap[7] >> 8) &
+								  ROW_MAX_VAL_MASK) +
+								 ROW_B17_BASE);
+}
+
+static void setup_column_address_map(struct synps_edac_priv *priv, u32 *addrmap)
+{
+	u32 width, memtype;
+	int index;
+
+	memtype = readl(priv->baseaddr + CTRL_OFST);
+	width = (memtype & ECC_CTRL_BUSWIDTH_MASK) >> ECC_CTRL_BUSWIDTH_SHIFT;
+
+	priv->col_shift[0] = 0;
+	priv->col_shift[1] = 1;
+	priv->col_shift[2] = (addrmap[2] & COL_MAX_VAL_MASK) + COL_B2_BASE;
+	priv->col_shift[3] = ((addrmap[2] >> 8) &
+						  COL_MAX_VAL_MASK) +
+						 COL_B3_BASE;
+	priv->col_shift[4] = (((addrmap[2] >> 16) & COL_MAX_VAL_MASK) ==
+						  COL_MAX_VAL_MASK)
+							 ? 0
+							 : (((addrmap[2] >> 16) &
+								 COL_MAX_VAL_MASK) +
+								COL_B4_BASE);
+	priv->col_shift[5] = (((addrmap[2] >> 24) & COL_MAX_VAL_MASK) ==
+						  COL_MAX_VAL_MASK)
+							 ? 0
+							 : (((addrmap[2] >> 24) &
+								 COL_MAX_VAL_MASK) +
+								COL_B5_BASE);
+	priv->col_shift[6] = ((addrmap[3] & COL_MAX_VAL_MASK) ==
+						  COL_MAX_VAL_MASK)
+							 ? 0
+							 : ((addrmap[3] &
+								 COL_MAX_VAL_MASK) +
+								COL_B6_BASE);
+	priv->col_shift[7] = (((addrmap[3] >> 8) & COL_MAX_VAL_MASK) ==
+						  COL_MAX_VAL_MASK)
+							 ? 0
+							 : (((addrmap[3] >> 8) &
+								 COL_MAX_VAL_MASK) +
+								COL_B7_BASE);
+	priv->col_shift[8] = (((addrmap[3] >> 16) & COL_MAX_VAL_MASK) ==
+						  COL_MAX_VAL_MASK)
+							 ? 0
+							 : (((addrmap[3] >> 16) &
+								 COL_MAX_VAL_MASK) +
+								COL_B8_BASE);
+	priv->col_shift[9] = (((addrmap[3] >> 24) & COL_MAX_VAL_MASK) ==
+						  COL_MAX_VAL_MASK)
+							 ? 0
+							 : (((addrmap[3] >> 24) &
+								 COL_MAX_VAL_MASK) +
+								COL_B9_BASE);
+	if (width == DDRCTL_EWDTH_64)
+	{
+		if (memtype & MEM_TYPE_LPDDR3)
+		{
+			priv->col_shift[10] = ((addrmap[4] &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : ((addrmap[4] & COL_MAX_VAL_MASK) +
+										 COL_B10_BASE);
+			priv->col_shift[11] = (((addrmap[4] >> 8) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[4] >> 8) & COL_MAX_VAL_MASK) +
+										 COL_B11_BASE);
+		}
+		else
+		{
+			priv->col_shift[11] = ((addrmap[4] &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : ((addrmap[4] & COL_MAX_VAL_MASK) +
+										 COL_B10_BASE);
+			priv->col_shift[13] = (((addrmap[4] >> 8) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[4] >> 8) & COL_MAX_VAL_MASK) +
+										 COL_B11_BASE);
+		}
+	}
+	else if (width == DDRCTL_EWDTH_32)
+	{
+		if (memtype & MEM_TYPE_LPDDR3)
+		{
+			priv->col_shift[10] = (((addrmap[3] >> 24) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[3] >> 24) & COL_MAX_VAL_MASK) +
+										 COL_B9_BASE);
+			priv->col_shift[11] = ((addrmap[4] &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : ((addrmap[4] & COL_MAX_VAL_MASK) +
+										 COL_B10_BASE);
+		}
+		else
+		{
+			priv->col_shift[11] = (((addrmap[3] >> 24) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[3] >> 24) & COL_MAX_VAL_MASK) +
+										 COL_B9_BASE);
+			priv->col_shift[13] = ((addrmap[4] &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : ((addrmap[4] & COL_MAX_VAL_MASK) +
+										 COL_B10_BASE);
+		}
+	}
+	else
+	{
+		if (memtype & MEM_TYPE_LPDDR3)
+		{
+			priv->col_shift[10] = (((addrmap[3] >> 16) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[3] >> 16) & COL_MAX_VAL_MASK) +
+										 COL_B8_BASE);
+			priv->col_shift[11] = (((addrmap[3] >> 24) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[3] >> 24) & COL_MAX_VAL_MASK) +
+										 COL_B9_BASE);
+			priv->col_shift[13] = ((addrmap[4] &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : ((addrmap[4] & COL_MAX_VAL_MASK) +
+										 COL_B10_BASE);
+		}
+		else
+		{
+			priv->col_shift[11] = (((addrmap[3] >> 16) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[3] >> 16) & COL_MAX_VAL_MASK) +
+										 COL_B8_BASE);
+			priv->col_shift[13] = (((addrmap[3] >> 24) &
+									COL_MAX_VAL_MASK) == COL_MAX_VAL_MASK)
+									  ? 0
+									  : (((addrmap[3] >> 24) & COL_MAX_VAL_MASK) +
+										 COL_B9_BASE);
+		}
+	}
+
+	if (width)
+	{
+		for (index = 9; index > width; index--)
+		{
+			priv->col_shift[index] = priv->col_shift[index - width];
+			priv->col_shift[index - width] = 0;
+		}
+	}
+}
+
+static void setup_bank_address_map(struct synps_edac_priv *priv, u32 *addrmap)
+{
+	priv->bank_shift[0] = (addrmap[1] & BANK_MAX_VAL_MASK) + BANK_B0_BASE;
+	priv->bank_shift[1] = ((addrmap[1] >> 8) &
+						   BANK_MAX_VAL_MASK) +
+						  BANK_B1_BASE;
+	priv->bank_shift[2] = (((addrmap[1] >> 16) &
+							BANK_MAX_VAL_MASK) == BANK_MAX_VAL_MASK)
+							  ? 0
+							  : (((addrmap[1] >> 16) & BANK_MAX_VAL_MASK) +
+								 BANK_B2_BASE);
+}
+
+static void setup_bg_address_map(struct synps_edac_priv *priv, u32 *addrmap)
+{
+	priv->bankgrp_shift[0] = (addrmap[8] &
+							  BANKGRP_MAX_VAL_MASK) +
+							 BANKGRP_B0_BASE;
+	priv->bankgrp_shift[1] = (((addrmap[8] >> 8) & BANKGRP_MAX_VAL_MASK) ==
+							  BANKGRP_MAX_VAL_MASK)
+								 ? 0
+								 : (((addrmap[8] >> 8) & BANKGRP_MAX_VAL_MASK) + BANKGRP_B1_BASE);
+}
+
+static void setup_rank_address_map(struct synps_edac_priv *priv, u32 *addrmap)
+{
+	priv->rank_shift[0] = ((addrmap[1] & RANK_MAX_VAL_MASK) ==
+						   RANK_MAX_VAL_MASK)
+							  ? 0
+							  : ((addrmap[0] &
+								  RANK_MAX_VAL_MASK) +
+								 RANK_B0_BASE);
+}
+
+/**
+ * setup_address_map -	Set Address Map by querying ADDRMAP registers.
+ * @priv:		DDR memory controller private instance data.
+ *
+ * Set Address Map by querying ADDRMAP registers.
+ *
+ * Return: none.
+ */
+static void setup_address_map(struct synps_edac_priv *priv)
+{
+	u32 addrmap[12];
+	int index;
+
+	for (index = 0; index < 12; index++)
+	{
+		u32 addrmap_offset;
+
+		addrmap_offset = ECC_ADDRMAP0_OFFSET + (index * 4);
+		addrmap[index] = readl(priv->baseaddr + addrmap_offset);
+		pr_err("addrmap:%d_value:%x\n", index, addrmap[index]);
+	}
+	setup_rank_address_map(priv, addrmap);
+
+	setup_bank_address_map(priv, addrmap);
+
+	setup_bg_address_map(priv, addrmap);
+
+	setup_column_address_map(priv, addrmap);
+
+	setup_row_address_map(priv, addrmap);
+}
+#endif /* CONFIG_EDAC_DEBUG */
+
+/**
+ * mc_probe - Check controller and bind driver.
+ * @pdev:	platform device.
+ *
+ * Probe a specific controller instance for binding with the driver.
+ *
+ * Return: 0 if the controller instance was successfully bound to the
+ * driver; otherwise, < 0 on error.
+ */
+static int mc_probe(struct platform_device *pdev)
+{
+	const struct synps_platform_data *p_data;
+	struct edac_mc_layer layers[2];
+	struct synps_edac_priv *priv;
+	struct mem_ctl_info *mci;
+	void __iomem *baseaddr;
+	struct resource *res;
+	int rc;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	baseaddr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(baseaddr))
+		return PTR_ERR(baseaddr);
+
+	p_data = of_device_get_match_data(&pdev->dev);
+	if (!p_data)
+		return -ENODEV;
+
+	if (!p_data->get_ecc_state(baseaddr))
+	{
+		edac_printk(KERN_INFO, EDAC_MC, "ECC not enabled\n");
+		return -ENXIO;
+	}
+
+	layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
+	layers[0].size = SYNPS_EDAC_NR_CSROWS;
+	layers[0].is_virt_csrow = true;
+	layers[1].type = EDAC_MC_LAYER_CHANNEL;
+	layers[1].size = SYNPS_EDAC_NR_CHANS;
+	layers[1].is_virt_csrow = false;
+
+	device_property_read_u32(&pdev->dev, "ctrl-id", &rc);
+
+	mci = edac_mc_alloc(rc, ARRAY_SIZE(layers), layers,
+						sizeof(struct synps_edac_priv));
+	if (!mci)
+	{
+		edac_printk(KERN_ERR, EDAC_MC,
+					"Failed memory allocation for mc instance\n");
+		return -ENOMEM;
+	}
+
+	priv = mci->pvt_info;
+	priv->baseaddr = baseaddr;
+	priv->start = res->start;
+	priv->p_data = p_data;
+
+	mc_init(mci, pdev);
+
+	if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+	{
+		rc = setup_irq(mci, pdev);
+		if (rc)
+			goto free_edac_mc;
+	}
+
+	// rc = edac_mc_add_mc(mci);
+	// if (rc)
+	// {
+	// 	edac_printk(KERN_ERR, EDAC_MC,
+	// 				"Failed to register with EDAC core\n");
+	// 	goto free_edac_mc;
+	// }
+
+#ifdef CONFIG_EDAC_DEBUG
+	if (priv->p_data->quirks & DDR_ECC_DATA_POISON_SUPPORT)
+	{
+		rc = edac_create_sysfs_attributes(mci);
+		if (rc)
+		{
+			edac_printk(KERN_ERR, EDAC_MC,
+						"Failed to create sysfs entries\n");
+			goto free_edac_mc;
+		}
+	}
+	if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+		setup_address_map(priv);
+#endif
+
+	/*
+	 * Start capturing the correctable and uncorrectable errors. A write of
+	 * 0 starts the counters.
+	 */
+	if (!(priv->p_data->quirks & DDR_ECC_INTR_SUPPORT))
+		writel(0x0, baseaddr + ECC_CTRL_OFST);
+	edac_printk(KERN_INFO, EDAC_MC, "%s init succ\n",pdev->name);
+	return rc;
+
+free_edac_mc:
+	edac_mc_free(mci);
+
+	return rc;
+}
+
+/**
+ * mc_remove - Unbind driver from controller.
+ * @pdev:	Platform device.
+ *
+ * Return: Unconditionally 0
+ */
+static int mc_remove(struct platform_device *pdev)
+{
+	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
+	struct synps_edac_priv *priv = mci->pvt_info;
+
+	if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+		disable_intr(priv);
+
+#ifdef CONFIG_EDAC_DEBUG
+	if (priv->p_data->quirks & DDR_ECC_DATA_POISON_SUPPORT)
+		edac_remove_sysfs_attributes(mci);
+#endif
+
+	edac_mc_del_mc(&pdev->dev);
+	edac_mc_free(mci);
+
+	return 0;
+}
+
+static struct platform_driver synps_edac_mc_driver = {
+	.driver = {
+		.name = "eswin-edac",
+		.of_match_table = synps_edac_match,
+	},
+	.probe = mc_probe,
+	.remove = mc_remove,
+};
+
+module_platform_driver(synps_edac_mc_driver);
+
+MODULE_AUTHOR("Eswin Inc");
+MODULE_DESCRIPTION("Synopsys DDR ECC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index a2b4c97bcb31..0d12020ffd9e 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -388,6 +388,10 @@ source "drivers/gpu/drm/solomon/Kconfig"
 
 source "drivers/gpu/drm/sprd/Kconfig"
 
+source "drivers/gpu/drm/eswin/Kconfig"
+
+source "drivers/gpu/drm/img/Kconfig"
+
 config DRM_HYPERV
 	tristate "DRM Support for Hyper-V synthetic video device"
 	depends on DRM && PCI && MMU && HYPERV
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 215e78e79125..6067543def36 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -198,3 +198,5 @@ obj-$(CONFIG_DRM_HYPERV) += hyperv/
 obj-y			+= solomon/
 obj-$(CONFIG_DRM_SPRD) += sprd/
 obj-$(CONFIG_DRM_LOONGSON) += loongson/
+obj-$(CONFIG_DRM_ESWIN) += eswin/
+obj-$(CONFIG_DRM_IMG) += img/
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 901d1961b739..49b33b2f6701 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -8,7 +8,10 @@ config DRM_AMD_DC
 	depends on BROKEN || !CC_IS_CLANG || ARM64 || RISCV || SPARC64 || X86_64
 	select SND_HDA_COMPONENT if SND_HDA_CORE
 	# !CC_IS_CLANG: https://github.com/ClangBuiltLinux/linux/issues/1752
-	select DRM_AMD_DC_FP if (X86 || LOONGARCH || (PPC64 && ALTIVEC) || (ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG))
+	select DRM_AMD_DC_FP if ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG
+	select DRM_AMD_DC_FP if PPC64 && ALTIVEC
+	select DRM_AMD_DC_FP if RISCV && FPU
+	select DRM_AMD_DC_FP if LOONGARCH || X86
 	help
 	  Choose this option if you want to use the new display engine
 	  support for AMDGPU. This adds required support for Vega and
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
index 172aa10a8800..53a7122ba98d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
@@ -35,6 +35,8 @@
 #include <asm/neon.h>
 #elif defined(CONFIG_LOONGARCH)
 #include <asm/fpu.h>
+#elif defined(CONFIG_RISCV)
+#include <asm/switch_to.h>
 #endif
 
 /**
@@ -90,7 +92,7 @@ void dc_fpu_begin(const char *function_name, const int line)
 	*pcpu += 1;
 
 	if (*pcpu == 1) {
-#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
+#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH) || defined(CONFIG_RISCV)
 		migrate_disable();
 		kernel_fpu_begin();
 #elif defined(CONFIG_PPC64)
@@ -130,7 +132,7 @@ void dc_fpu_end(const char *function_name, const int line)
 	pcpu = get_cpu_ptr(&fpu_recursion_depth);
 	*pcpu -= 1;
 	if (*pcpu <= 0) {
-#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
+#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH) || defined(CONFIG_RISCV)
 		kernel_fpu_end();
 		migrate_enable();
 #elif defined(CONFIG_PPC64)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index 0ba9a7997d56..96f7110e8e41 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -43,6 +43,12 @@ dml_ccflags := -mfpu=64
 dml_rcflags := -msoft-float
 endif
 
+ifdef CONFIG_RISCV
+include $(srctree)/arch/riscv/Makefile.isa
+# Remove V from the ISA string, like in arch/riscv/Makefile, but keep F and D.
+dml_ccflags := -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)([^v_]*)v?/\1\2/')
+endif
+
 ifdef CONFIG_CC_IS_GCC
 ifneq ($(call gcc-min-version, 70100),y)
 IS_OLD_GCC = 1
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
index 04d4a1a10698..3f3bf54f5da3 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
@@ -29,6 +29,7 @@
 #include <drm/drm_print.h>
 
 #define HWVER_131			0x31333100	/* IP version 1.31 */
+#define HWVER_30313500			0x30313500
 
 #define DSI_VERSION			0x00
 #define VERSION				GENMASK(31, 8)
@@ -781,7 +782,7 @@ static void dw_mipi_dsi_dphy_timing_config(struct dw_mipi_dsi *dsi)
 
 	hw_version = dsi_read(dsi, DSI_VERSION) & VERSION;
 
-	if (hw_version >= HWVER_131) {
+	if (hw_version >= HWVER_131 || hw_version == HWVER_30313500)  {
 		dsi_write(dsi, DSI_PHY_TMR_CFG,
 			  PHY_HS2LP_TIME_V131(timing.data_hs2lp) |
 			  PHY_LP2HS_TIME_V131(timing.data_lp2hs));
diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c
index b67eafa55715..5ebe418bd383 100644
--- a/drivers/gpu/drm/drm_gem_vram_helper.c
+++ b/drivers/gpu/drm/drm_gem_vram_helper.c
@@ -870,7 +870,7 @@ static struct ttm_tt *bo_driver_ttm_tt_create(struct ttm_buffer_object *bo,
 	if (!tt)
 		return NULL;
 
-	ret = ttm_tt_init(tt, bo, page_flags, ttm_cached, 0);
+	ret = ttm_tt_init(tt, bo, page_flags, ttm_write_combined, 0);
 	if (ret < 0)
 		goto err_ttm_tt_init;
 
diff --git a/drivers/gpu/drm/eswin/Kconfig b/drivers/gpu/drm/eswin/Kconfig
new file mode 100644
index 000000000000..9a08d81437fd
--- /dev/null
+++ b/drivers/gpu/drm/eswin/Kconfig
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config DRM_ESWIN
+	tristate "DRM Support for Eswin"
+	depends on DRM
+	select DRM_KMS_HELPER
+	help
+	  Choose this option if you have a Eswin soc chipset.
+	  This driver provides Eswin kernel mode
+	  setting and buffer management. It does not
+	  provide 2D or 3D acceleration.
+
+config ESWIN_VIRTUAL_DISPLAY
+	bool "display content output to debugfs file"
+	depends on DRM_ESWIN
+	select DEBUG_FS
+	help
+	  This is a debug feature which capture video content output
+	  from display controller. Output path is debugfs/dri/connector/.
+	  The content format is ARGB which Alpha is 0 for 8bits.
+	  Disabled in default.
+
+config ESWIN_MMU
+	bool "MMU support for Eswin display controller"
+	depends on DRM_ESWIN
+	help
+	  This is a memory management function which translate virtual address
+	  to physical address. DPU MMU only do address translate, doesn't
+	  support security and shareable.
+
+config ESWIN_DW_HDMI
+    bool "ESWIN specific extensions for Synopsys DW HDMI"
+    depends on DRM_ESWIN
+    select CEC_CORE if CEC_NOTIFIER
+    select DRM_DISPLAY_HDMI_HELPER
+    select DRM_DISPLAY_HELPER
+    select DRM_KMS_HELPER
+    help
+      This selects support for ESWIN SoC specific extensions
+      for the Synopsys DesignWare HDMI driver. If you want to
+      enable HDMI on win2030 based SoC, you should select
+      this option.
+
+config DW_HDMI_I2S_AUDIO
+    bool "Synopsys Designware I2S Audio interface"
+    depends on SND_SOC
+    depends on ESWIN_DW_HDMI
+    select SND_SOC_HDMI_CODEC
+    help
+      Support the I2S Audio interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_CEC
+    bool "Synopsis Designware CEC interface"
+    depends on ESWIN_DW_HDMI
+    select CEC_CORE
+    select CEC_NOTIFIER
+    help
+      Support the CEC interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_HDCP
+    bool "Synopsis Designware HDCP interface"
+    depends on ESWIN_DW_HDMI
+    help
+      Support the HDCP interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_HDCP2
+    tristate "Synopsis Designware HDCP2 interface"
+    depends on DW_HDMI_HDCP
+    help
+      Support the HDCP2 interface which is part of the Synopsys
+      Designware HDMI block.
+
+config ESWIN_MIPI_DSI
+    bool "Eswin mipi dsi and 1080p panel driver"
+    depends on DRM_ESWIN
+    select DRM_DW_MIPI_DSI
+    help
+      Support mipi dsi 4-lanes panel output
+
+config ESWIN_RASPBERRYPI_7INCH_PANEL
+    bool "Support for Raspberry Pi 7-inch DSI panel"
+    depends on ESWIN_MIPI_DSI
+    select PM
+    select REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY
+    select DRM_PANEL_SIMPLE
+    select DRM_TOSHIBA_TC358762
+    help
+      Enable support for the Raspberry Pi 7-inch DSI panel
diff --git a/drivers/gpu/drm/eswin/Makefile b/drivers/gpu/drm/eswin/Makefile
new file mode 100644
index 000000000000..1f7a44346d8d
--- /dev/null
+++ b/drivers/gpu/drm/eswin/Makefile
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: GPL-2.0
+
+ccflags-y += -I$(srctree)/drivers/memory/eswin/es_dev_buf/include/linux
+
+es_drm-objs := es_dc_hw.o \
+		es_dc.o \
+		es_crtc.o \
+		es_drv.o \
+		es_fb.o \
+		es_gem.o \
+		es_plane.o \
+		es_simple_enc.o
+
+es_drm-$(CONFIG_ESWIN_VIRTUAL_DISPLAY) += es_virtual.o
+es_drm-$(CONFIG_ESWIN_MMU) += es_dc_mmu.o
+es_drm-$(CONFIG_ESWIN_DW_HDMI) += es_dw_hdmi.o dw-hdmi.o
+
+es_drm-$(CONFIG_DW_HDMI_I2S_AUDIO) += dw_hdmi_i2s_audio.o
+es_drm-$(CONFIG_DW_HDMI_CEC) += dw_hdmi_cec.o
+es_drm-$(CONFIG_DW_HDMI_HDCP) += dw_hdmi_hdcp.o
+
+dw_hdcp2-$(CONFIG_DW_HDMI_HDCP2) += dw_hdmi_hdcp2.o
+
+obj-$(CONFIG_DRM_ESWIN) += es_drm.o
+obj-$(CONFIG_DW_HDMI_HDCP2) += dw_hdcp2.o
+
+es_drm-$(CONFIG_ESWIN_MIPI_DSI)+= es_mipi_dsi.o es_panel.o
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/dw-hdmi.c b/drivers/gpu/drm/eswin/dw-hdmi.c
new file mode 100644
index 000000000000..dd244a700723
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw-hdmi.c
@@ -0,0 +1,4632 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DesignWare High-Definition Multimedia Interface (HDMI) driver
+ *
+ * Copyright (C) 2013-2015 Mentor Graphics Inc.
+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc.
+ * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
+ *****************************************************************************
+ * ESWIN hdmi driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/hdmi.h>
+#include <linux/i2c.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/regmap.h>
+#include <linux/dma-mapping.h>
+#include <linux/spinlock.h>
+
+#include <media/cec-notifier.h>
+
+#include <linux/media-bus-format.h>
+#include <linux/videodev2.h>
+#include <linux/reset.h>
+
+#include <drm/bridge/dw_hdmi.h>
+#include <drm/display/drm_hdmi_helper.h>
+#include <drm/display/drm_scdc_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_of.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+
+#include "dw_hdmi_audio.h"
+#include "dw_hdmi_cec.h"
+#include "dw-hdmi.h"
+#include "dw_hdmi_hdcp.h"
+
+#define DDC_CI_ADDR 0x37
+#define DDC_SEGMENT_ADDR 0x30
+
+#define HDMI_EDID_LEN 512
+
+/* DW-HDMI Controller >= 0x200a are at least compliant with SCDC version 1 */
+#define SCDC_MIN_SOURCE_VERSION 0x1
+
+#define HDMI14_MAX_TMDSCLK 340000000
+
+#define HDMI_CEC_CLK 32768
+
+static bool hpd_flag[2] = {0, 0};
+
+static const u16 csc_coeff_default[3][4] = {
+	{ 0x2000, 0x0000, 0x0000, 0x0000 },
+	{ 0x0000, 0x2000, 0x0000, 0x0000 },
+	{ 0x0000, 0x0000, 0x2000, 0x0000 }
+};
+
+static const u16 csc_coeff_rgb_out_eitu601[3][4] = {
+	{ 0x2000, 0x6926, 0x74fd, 0x010e },
+	{ 0x2000, 0x2cdd, 0x0000, 0x7e9a },
+	{ 0x2000, 0x0000, 0x38b4, 0x7e3b }
+};
+
+static const u16 csc_coeff_rgb_out_eitu709[3][4] = {
+	{ 0x2000, 0x7106, 0x7a02, 0x00a7 },
+	{ 0x2000, 0x3264, 0x0000, 0x7e6d },
+	{ 0x2000, 0x0000, 0x3b61, 0x7e25 }
+};
+
+static const u16 csc_coeff_rgb_in_eitu601[3][4] = {
+	{ 0x2591, 0x1322, 0x074b, 0x0000 },
+	{ 0x6535, 0x2000, 0x7acc, 0x0200 },
+	{ 0x6acd, 0x7534, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_in_eitu709[3][4] = {
+	{ 0x2dc5, 0x0d9b, 0x049e, 0x0000 },
+	{ 0x62f0, 0x2000, 0x7d11, 0x0200 },
+	{ 0x6756, 0x78ab, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_full_to_rgb_limited[3][4] = {
+	{ 0x1b7c, 0x0000, 0x0000, 0x0020 },
+	{ 0x0000, 0x1b7c, 0x0000, 0x0020 },
+	{ 0x0000, 0x0000, 0x1b7c, 0x0020 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_8bit_eitu601[3][4] = {
+	{ 0x2591, 0x1323, 0x074c, 0x0000 },
+	{ 0xe534, 0x2000, 0xfacc, 0x0200 },
+	{ 0xeacd, 0xf533, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_8bit_eitu709[3][4] = {
+	{ 0x2dc6, 0x0d9b, 0x049f, 0x0000 },
+	{ 0xe2ef, 0x2000, 0xfd11, 0x0200 },
+	{ 0xe755, 0xf8ab, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_10bit_eitu601[3][4] = {
+	{ 0x2591, 0x1323, 0x074c, 0x0000 },
+	{ 0xe534, 0x2000, 0xfacc, 0x0800 },
+	{ 0xeacd, 0xf533, 0x2000, 0x0800 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_10bit_eitu709[3][4] = {
+	{ 0x2dc6, 0x0d9b, 0x049f, 0x0000 },
+	{ 0xe2ef, 0x2000, 0xfd11, 0x0800 },
+	{ 0xe755, 0xf8ab, 0x2000, 0x0800 }
+};
+
+struct hdmi_vmode {
+	bool mdataenablepolarity;
+
+	unsigned int mpixelclock;
+	unsigned int mpixelrepetitioninput;
+	unsigned int mpixelrepetitionoutput;
+	unsigned int mtmdsclock;
+};
+
+struct hdmi_data_info {
+	unsigned int enc_in_bus_format;
+	unsigned int enc_out_bus_format;
+	unsigned int enc_in_encoding;
+	unsigned int enc_out_encoding;
+	unsigned int pix_repet_factor;
+	unsigned int hdcp_enable;
+	struct hdmi_vmode video_mode;
+	bool rgb_limited_range;
+};
+
+struct dw_hdmi_i2c {
+	struct i2c_adapter	adap;
+
+	struct mutex		lock;	/* used to serialize data transfers */
+	struct completion	cmp;
+	u8			stat;
+
+	u8			slave_reg;
+	bool			is_regaddr;
+	bool			is_segment;
+};
+
+struct dw_hdmi_phy_data {
+	enum dw_hdmi_phy_type type;
+	const char *name;
+	unsigned int gen;
+	bool has_svsret;
+	int (*configure)(struct dw_hdmi *hdmi,
+			 const struct dw_hdmi_plat_data *pdata,
+			 unsigned long mpixelclock);
+};
+
+struct dw_hdmi {
+	struct drm_connector connector;
+	struct drm_bridge bridge;
+	struct drm_bridge *next_bridge;
+	unsigned int version;
+
+	struct platform_device *hdcp_dev;
+	struct platform_device *audio;
+	struct platform_device *cec;
+	struct device *dev;
+	struct clk *isfr_clk;
+	struct clk *iahb_clk;
+	struct clk *cec_clk;
+	struct dw_hdmi_i2c *i2c;
+
+	struct hdmi_data_info hdmi_data;
+	const struct dw_hdmi_plat_data *plat_data;
+	struct dw_hdcp *hdcp;
+	int vic;
+	int numa_id;
+
+	u8 edid[HDMI_EDID_LEN];
+
+	struct {
+		const struct dw_hdmi_phy_ops *ops;
+		const char *name;
+		void *data;
+		bool enabled;
+	} phy;
+
+	struct drm_display_mode previous_mode;
+
+	struct i2c_adapter *ddc;
+	void __iomem *regs;
+	bool sink_is_hdmi;
+	bool sink_has_audio;
+	bool hpd_state;
+
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *default_state;
+	struct pinctrl_state *unwedge_state;
+
+	struct mutex mutex;		/* for state below and previous_mode */
+	enum drm_connector_force force;	/* mutex-protected force state */
+	struct drm_connector *curr_conn;/* current connector (only valid when !disabled) */
+	bool disabled;			/* DRM has disabled our bridge */
+	bool bridge_is_on;		/* indicates the bridge is on */
+	bool rxsense;			/* rxsense state */
+	u8 phy_mask;			/* desired phy int mask settings */
+	u8 mc_clkdis;			/* clock disable register */
+
+	spinlock_t audio_lock;
+	struct mutex audio_mutex;
+	unsigned int sample_non_pcm;
+	unsigned int sample_width;
+	unsigned int sample_rate;
+	unsigned int channels;
+	unsigned int audio_cts;
+	unsigned int audio_n;
+	bool audio_enable;
+	struct dentry *debugfs_dir;
+
+	unsigned int reg_shift;
+	struct regmap *regm;
+	void (*enable_audio)(struct dw_hdmi *hdmi);
+	void (*disable_audio)(struct dw_hdmi *hdmi);
+
+	struct mutex cec_notifier_mutex;
+	struct cec_notifier *cec_notifier;
+
+	hdmi_codec_plugged_cb plugged_cb;
+	struct device *codec_dev;
+	enum drm_connector_status last_connector_result;
+
+	int irq;
+	struct delayed_work work;
+	struct workqueue_struct *workqueue;
+	struct reset_control *rst_hdmi_prstn;
+	struct reset_control *rst_hdmi_phyrstn;
+	struct reset_control *rst_hdmi_rstn;
+};
+
+#define HDMI_IH_PHY_STAT0_RX_SENSE                                             \
+	(HDMI_IH_PHY_STAT0_RX_SENSE0 | HDMI_IH_PHY_STAT0_RX_SENSE1 |           \
+	 HDMI_IH_PHY_STAT0_RX_SENSE2 | HDMI_IH_PHY_STAT0_RX_SENSE3)
+
+#define HDMI_PHY_RX_SENSE \
+	(HDMI_PHY_RX_SENSE0 | HDMI_PHY_RX_SENSE1 | \
+	 HDMI_PHY_RX_SENSE2 | HDMI_PHY_RX_SENSE3)
+
+static struct edid *dw_hdmi_get_edid(struct dw_hdmi *hdmi,
+				     struct drm_connector *connector);
+static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi,
+						const struct drm_display_mode *mode);
+
+static inline void hdmi_writeb(struct dw_hdmi *hdmi, u8 val, int offset)
+{
+	regmap_write(hdmi->regm, offset << hdmi->reg_shift, val);
+}
+
+static inline u8 hdmi_readb(struct dw_hdmi *hdmi, int offset)
+{
+	unsigned int val = 0;
+
+	regmap_read(hdmi->regm, offset << hdmi->reg_shift, &val);
+
+	return val;
+}
+
+static void handle_plugged_change(struct dw_hdmi *hdmi, bool plugged)
+{
+	if (hdmi->plugged_cb && hdmi->codec_dev)
+		hdmi->plugged_cb(hdmi->codec_dev, plugged);
+}
+
+int dw_hdmi_set_plugged_cb(struct dw_hdmi *hdmi, hdmi_codec_plugged_cb fn,
+			   struct device *codec_dev)
+{
+	bool plugged;
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->plugged_cb = fn;
+	hdmi->codec_dev = codec_dev;
+	plugged = hdmi->last_connector_result == connector_status_connected;
+	handle_plugged_change(hdmi, plugged);
+	mutex_unlock(&hdmi->mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_plugged_cb);
+
+static void hdmi_modb(struct dw_hdmi *hdmi, u8 data, u8 mask, unsigned reg)
+{
+	regmap_update_bits(hdmi->regm, reg << hdmi->reg_shift, mask, data);
+}
+
+static void hdmi_mask_writeb(struct dw_hdmi *hdmi, u8 data, unsigned int reg,
+			     u8 shift, u8 mask)
+{
+	hdmi_modb(hdmi, data << shift, mask, reg);
+}
+
+static void repo_hpd_event(struct work_struct *p_work)
+{
+	struct dw_hdmi *hdmi = container_of(p_work, struct dw_hdmi, work.work);
+	u8 phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+
+	mutex_lock(&hdmi->mutex);
+	if (!(phy_stat & HDMI_PHY_RX_SENSE))
+		hdmi->rxsense = false;
+	if (phy_stat & HDMI_PHY_HPD)
+		hdmi->rxsense = true;
+	mutex_unlock(&hdmi->mutex);
+
+	if (hdmi->bridge.dev) {
+		if (drm_helper_hpd_irq_event(hdmi->bridge.dev)) {
+			dev_info(hdmi->dev, "hpd event report status:%d\n", hdmi->last_connector_result);
+		}
+	}
+
+	msleep(150);
+	if (hdmi->hpd_state) {
+		if (hdmi->hdcp && hdmi->hdcp->hdcp2 &&
+		    hdmi->hdcp->hdcp2->enable &&
+		    (tv_hdmi_hdcp2_support(hdmi)) == 1) {
+			hdmi->hdcp->hdcp2->start(hdmi->hdcp->hdcp2);
+		} else {
+			hdmi_tx_hdcp_config(hdmi, &hdmi->previous_mode);
+		}
+	}
+}
+
+static bool check_hdmi_irq(struct dw_hdmi *hdmi, int intr_stat, int phy_int_pol)
+{
+	int msecs;
+
+	/* To determine whether interrupt type is HPD */
+	if (!(intr_stat & HDMI_IH_PHY_STAT0_HPD))
+		return false;
+
+	if (phy_int_pol & HDMI_PHY_HPD) {
+		dev_info(hdmi->dev, "dw hdmi plug in\n");
+		msecs = 100;
+		hdmi->hpd_state = true;
+	} else {
+		dev_info(hdmi->dev, "dw hdmi plug out\n");
+		msecs = 1000;
+		hdmi->hpd_state = false;
+		if (hdmi->hdcp && hdmi->hdcp->hdcp2 &&
+		    hdmi->hdcp->hdcp2->enable) {
+			hdmi->hdcp->hdcp2->stop(hdmi->hdcp->hdcp2);
+		}
+		if (hdmi->hdcp && hdmi->hdcp->hdcp_stop) {
+			hdmi->hdcp->hdcp_stop(hdmi->hdcp);
+		}
+	}
+	mod_delayed_work(hdmi->workqueue, &hdmi->work, msecs_to_jiffies(msecs));
+
+	return true;
+}
+
+static void init_hpd_work(struct dw_hdmi *hdmi)
+{
+	hdmi->workqueue = create_workqueue("hpd_queue");
+	INIT_DELAYED_WORK(&hdmi->work, repo_hpd_event);
+}
+
+static void dw_hdmi_i2c_set_divs(struct dw_hdmi *hdmi)
+{
+	unsigned long clk_rate_khz;
+	unsigned long low_ns, high_ns;
+	unsigned long div_low, div_high;
+
+	/* Standard-mode */
+	high_ns = 4708;
+	low_ns = 4916;
+
+	/* Adjust to avoid overflow */
+	clk_rate_khz = DIV_ROUND_UP(clk_get_rate(hdmi->isfr_clk), 1000);
+
+	div_low = (clk_rate_khz * low_ns) / 1000000;
+	if ((clk_rate_khz * low_ns) % 1000000)
+		div_low++;
+
+	div_high = (clk_rate_khz * high_ns) / 1000000;
+	if ((clk_rate_khz * high_ns) % 1000000)
+		div_high++;
+
+	/* Maximum divider supported by hw is 0xffff */
+	if (div_low > 0xffff)
+		div_low = 0xffff;
+
+	if (div_high > 0xffff)
+		div_high = 0xffff;
+
+	hdmi_writeb(hdmi, div_high & 0xff, HDMI_I2CM_SS_SCL_HCNT_0_ADDR);
+	hdmi_writeb(hdmi, (div_high >> 8) & 0xff, HDMI_I2CM_SS_SCL_HCNT_1_ADDR);
+	hdmi_writeb(hdmi, div_low & 0xff, HDMI_I2CM_SS_SCL_LCNT_0_ADDR);
+	hdmi_writeb(hdmi, (div_low >> 8) & 0xff, HDMI_I2CM_SS_SCL_LCNT_1_ADDR);
+}
+
+static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi)
+{
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL,
+		    HDMI_PHY_I2CM_INT_ADDR);
+
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL |
+		    HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL,
+		    HDMI_PHY_I2CM_CTLINT_ADDR);
+
+	/* Software reset */
+	hdmi_writeb(hdmi, 0x00, HDMI_I2CM_SOFTRSTZ);
+
+	/* Set Standard Mode speed (determined to be 100KHz on iMX6) */
+	hdmi_writeb(hdmi, 0x00, HDMI_I2CM_DIV);
+
+	/* Set done, not acknowledged and arbitration interrupt polarities */
+	hdmi_writeb(hdmi, HDMI_I2CM_INT_DONE_POL, HDMI_I2CM_INT);
+	hdmi_writeb(hdmi, HDMI_I2CM_CTLINT_NAC_POL | HDMI_I2CM_CTLINT_ARB_POL,
+		    HDMI_I2CM_CTLINT);
+
+	/* Clear DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_I2CM_STAT0);
+
+	/* Mute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_MUTE_I2CM_STAT0);
+
+	/* set SDA high level holding time */
+	hdmi_writeb(hdmi, 0x48, HDMI_I2CM_SDA_HOLD);
+	dw_hdmi_i2c_set_divs(hdmi);
+}
+
+static bool dw_hdmi_i2c_unwedge(struct dw_hdmi *hdmi)
+{
+	/* If no unwedge state then give up */
+	if (!hdmi->unwedge_state)
+		return false;
+
+	dev_info(hdmi->dev, "Attempting to unwedge stuck i2c bus\n");
+
+	/*
+	 * This is a huge hack to workaround a problem where the dw_hdmi i2c
+	 * bus could sometimes get wedged.  Once wedged there doesn't appear
+	 * to be any way to unwedge it (including the HDMI_I2CM_SOFTRSTZ)
+	 * other than pulsing the SDA line.
+	 *
+	 * We appear to be able to pulse the SDA line (in the eyes of dw_hdmi)
+	 * by:
+	 * 1. Remux the pin as a GPIO output, driven low.
+	 * 2. Wait a little while.  1 ms seems to work, but we'll do 10.
+	 * 3. Immediately jump to remux the pin as dw_hdmi i2c again.
+	 *
+	 * At the moment of remuxing, the line will still be low due to its
+	 * recent stint as an output, but then it will be pulled high by the
+	 * (presumed) external pullup.  dw_hdmi seems to see this as a rising
+	 * edge and that seems to get it out of its jam.
+	 *
+	 * This wedging was only ever seen on one TV, and only on one of
+	 * its HDMI ports.  It happened when the TV was powered on while the
+	 * device was plugged in.  A scope trace shows the TV bringing both SDA
+	 * and SCL low, then bringing them both back up at roughly the same
+	 * time.  Presumably this confuses dw_hdmi because it saw activity but
+	 * no real STOP (maybe it thinks there's another master on the bus?).
+	 * Giving it a clean rising edge of SDA while SCL is already high
+	 * presumably makes dw_hdmi see a STOP which seems to bring dw_hdmi out
+	 * of its stupor.
+	 *
+	 * Note that after coming back alive, transfers seem to immediately
+	 * resume, so if we unwedge due to a timeout we should wait a little
+	 * longer for our transfer to finish, since it might have just started
+	 * now.
+	 */
+	pinctrl_select_state(hdmi->pinctrl, hdmi->unwedge_state);
+	msleep(10);
+	pinctrl_select_state(hdmi->pinctrl, hdmi->default_state);
+
+	return true;
+}
+
+static int dw_hdmi_i2c_wait(struct dw_hdmi *hdmi)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int stat;
+
+	stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10);
+	if (!stat) {
+		/* If we can't unwedge, return timeout */
+		if (!dw_hdmi_i2c_unwedge(hdmi))
+			return -EAGAIN;
+
+		/* We tried to unwedge; give it another chance */
+		stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10);
+		if (!stat)
+			return -EAGAIN;
+	}
+
+	/* Check for error condition on the bus */
+	if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+		return -EIO;
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_read(struct dw_hdmi *hdmi,
+			    unsigned char *buf, unsigned int length)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int ret;
+
+	if (!i2c->is_regaddr) {
+		dev_dbg(hdmi->dev, "set read register address to 0\n");
+		i2c->slave_reg = 0x00;
+		i2c->is_regaddr = true;
+	}
+
+	while (length--) {
+		reinit_completion(&i2c->cmp);
+
+		hdmi_writeb(hdmi, i2c->slave_reg++, HDMI_I2CM_ADDRESS);
+		if (i2c->is_segment)
+			hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_READ_EXT,
+				    HDMI_I2CM_OPERATION);
+		else
+			hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_READ,
+				    HDMI_I2CM_OPERATION);
+
+		ret = dw_hdmi_i2c_wait(hdmi);
+		if (ret)
+			return ret;
+
+		/* Check for error condition on the bus */
+		if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+			return -EIO;
+
+		*buf++ = hdmi_readb(hdmi, HDMI_I2CM_DATAI);
+	}
+	i2c->is_segment = false;
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_write(struct dw_hdmi *hdmi,
+			     unsigned char *buf, unsigned int length)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int ret;
+
+	if (!i2c->is_regaddr) {
+		/* Use the first write byte as register address */
+		i2c->slave_reg = buf[0];
+		length--;
+		buf++;
+		i2c->is_regaddr = true;
+	}
+
+	while (length--) {
+		reinit_completion(&i2c->cmp);
+
+		hdmi_writeb(hdmi, *buf++, HDMI_I2CM_DATAO);
+		hdmi_writeb(hdmi, i2c->slave_reg++, HDMI_I2CM_ADDRESS);
+		hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_WRITE,
+			    HDMI_I2CM_OPERATION);
+
+		ret = dw_hdmi_i2c_wait(hdmi);
+		if (ret)
+			return ret;
+
+		/* Check for error condition on the bus */
+		if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+			return -EIO;
+	}
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap,
+			    struct i2c_msg *msgs, int num)
+{
+	struct dw_hdmi *hdmi = i2c_get_adapdata(adap);
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	u8 addr = msgs[0].addr;
+	int i, ret = 0;
+
+	if (addr == DDC_CI_ADDR)
+		/*
+		 * The internal I2C controller does not support the multi-byte
+		 * read and write operations needed for DDC/CI.
+		 * TOFIX: Blacklist the DDC/CI address until we filter out
+		 * unsupported I2C operations.
+		 */
+		return -EOPNOTSUPP;
+
+	dev_vdbg(hdmi->dev, "xfer: num: %d, addr: %#x\n", num, addr);
+
+	for (i = 0; i < num; i++) {
+		if (msgs[i].len == 0) {
+			dev_dbg(hdmi->dev,
+				"unsupported transfer %d/%d, no data\n",
+				i + 1, num);
+			return -EOPNOTSUPP;
+		}
+	}
+
+	mutex_lock(&i2c->lock);
+
+	/* Unmute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, 0x00, HDMI_IH_MUTE_I2CM_STAT0);
+
+	/* Set slave device address taken from the first I2C message */
+	if (addr == DDC_SEGMENT_ADDR && msgs[0].len == 1)
+		addr = DDC_ADDR;
+	hdmi_writeb(hdmi, addr, HDMI_I2CM_SLAVE);
+
+	/* Set slave device register address on transfer */
+	i2c->is_regaddr = false;
+
+	/* Set segment pointer for I2C extended read mode operation */
+	i2c->is_segment = false;
+
+	for (i = 0; i < num; i++) {
+		dev_vdbg(hdmi->dev, "xfer: num: %d/%d, len: %d, flags: %#x\n",
+			i + 1, num, msgs[i].len, msgs[i].flags);
+		udelay(100);
+		if (msgs[i].addr == DDC_SEGMENT_ADDR && msgs[i].len == 1) {
+			i2c->is_segment = true;
+			hdmi_writeb(hdmi, DDC_SEGMENT_ADDR, HDMI_I2CM_SEGADDR);
+			hdmi_writeb(hdmi, *msgs[i].buf, HDMI_I2CM_SEGPTR);
+		} else {
+			if (msgs[i].flags & I2C_M_RD)
+				ret = dw_hdmi_i2c_read(hdmi, msgs[i].buf,
+						       msgs[i].len);
+			else
+				ret = dw_hdmi_i2c_write(hdmi, msgs[i].buf,
+							msgs[i].len);
+		}
+		if (ret < 0) {
+			dev_info(hdmi->dev, "i2c transfer fail\n");
+			udelay(200 * 1000);
+			break;
+		}
+	}
+
+	if (!ret)
+		ret = num;
+
+	/* Mute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_MUTE_I2CM_STAT0);
+
+	mutex_unlock(&i2c->lock);
+
+	return ret;
+}
+
+static u32 dw_hdmi_i2c_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm dw_hdmi_algorithm = {
+	.master_xfer = dw_hdmi_i2c_xfer,
+	.functionality = dw_hdmi_i2c_func,
+};
+
+u8 tv_hdmi_hdcp2_support(struct dw_hdmi *hdmi)
+{
+	int ret = 0;
+	int try_times = 10;
+	unsigned char start = 0x50;
+	u8 buf;
+	int i;
+
+	struct i2c_msg msgs[] = { {
+					  .addr = 0x3a,
+					  .flags = 0,
+					  .len = 1,
+					  .buf = &start,
+				  },
+				  {
+					  .addr = 0x3a,
+					  .flags = I2C_M_RD,
+					  .len = 1,
+					  .buf = &buf,
+				  } };
+
+	for (i = 0; i < try_times; i++) {
+		//if (hdmi->bridge_is_on)
+		ret = dw_hdmi_i2c_xfer(hdmi->ddc, msgs, 2);
+
+		printk("ret = %d; buf = %d\n", ret, buf);
+		if (ret == 2 && (buf & 0x4))
+			return ((buf & 0x4) >> 2);
+		udelay(100);
+	}
+
+	return -1;
+}
+EXPORT_SYMBOL_GPL(tv_hdmi_hdcp2_support);
+
+static struct i2c_adapter *dw_hdmi_i2c_adapter(struct dw_hdmi *hdmi)
+{
+	struct i2c_adapter *adap;
+	struct dw_hdmi_i2c *i2c;
+	int ret;
+
+	i2c = devm_kzalloc(hdmi->dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&i2c->lock);
+	init_completion(&i2c->cmp);
+
+	adap = &i2c->adap;
+	adap->class = I2C_CLASS_DDC;
+	adap->owner = THIS_MODULE;
+	adap->dev.parent = hdmi->dev;
+	adap->algo = &dw_hdmi_algorithm;
+	strscpy(adap->name, "DesignWare HDMI", sizeof(adap->name));
+	i2c_set_adapdata(adap, hdmi);
+
+	ret = i2c_add_adapter(adap);
+	if (ret) {
+		dev_warn(hdmi->dev, "cannot add %s I2C adapter\n", adap->name);
+		devm_kfree(hdmi->dev, i2c);
+		return ERR_PTR(ret);
+	}
+
+	hdmi->i2c = i2c;
+
+	dev_info(hdmi->dev, "registered %s I2C bus driver\n", adap->name);
+
+	return adap;
+}
+
+static void hdmi_set_cts_n(struct dw_hdmi *hdmi, unsigned int cts,
+			   unsigned int n)
+{
+	/* Must be set/cleared first */
+	hdmi_modb(hdmi, 0, HDMI_AUD_CTS3_CTS_MANUAL, HDMI_AUD_CTS3);
+
+	/* nshift factor = 0 */
+	hdmi_modb(hdmi, 0, HDMI_AUD_CTS3_N_SHIFT_MASK, HDMI_AUD_CTS3);
+
+	/* Use automatic CTS generation mode when CTS is not set */
+	if (cts)
+		hdmi_writeb(hdmi, ((cts >> 16) &
+				   HDMI_AUD_CTS3_AUDCTS19_16_MASK) |
+				  HDMI_AUD_CTS3_CTS_MANUAL,
+			    HDMI_AUD_CTS3);
+	else
+		hdmi_writeb(hdmi, 0, HDMI_AUD_CTS3);
+	hdmi_writeb(hdmi, (cts >> 8) & 0xff, HDMI_AUD_CTS2);
+	hdmi_writeb(hdmi, cts & 0xff, HDMI_AUD_CTS1);
+
+	hdmi_writeb(hdmi, (n >> 16) & 0x0f, HDMI_AUD_N3);
+	hdmi_writeb(hdmi, (n >> 8) & 0xff, HDMI_AUD_N2);
+	hdmi_writeb(hdmi, n & 0xff, HDMI_AUD_N1);
+}
+
+static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk)
+{
+	unsigned int n = (128 * freq) / 1000;
+	unsigned int mult = 1;
+
+	while (freq > 48000) {
+		mult *= 2;
+		freq /= 2;
+	}
+
+	switch (freq) {
+	case 32000:
+		if (pixel_clk == 25175000)
+			n = 4576;
+		else if (pixel_clk == 27027000)
+			n = 4096;
+		else if (pixel_clk == 74176000 || pixel_clk == 148352000)
+			n = 11648;
+		else if (pixel_clk == 297000000)
+			n = 3072;
+		else
+			n = 4096;
+		n *= mult;
+		break;
+
+	case 44100:
+		if (pixel_clk == 25175000)
+			n = 7007;
+		else if (pixel_clk == 74176000)
+			n = 17836;
+		else if (pixel_clk == 148352000)
+			n = 8918;
+		else if (pixel_clk == 297000000)
+			n = 4704;
+		else
+			n = 6272;
+		n *= mult;
+		break;
+
+	case 48000:
+		if (pixel_clk == 25175000)
+			n = 6864;
+		else if (pixel_clk == 27027000)
+			n = 6144;
+		else if (pixel_clk == 74176000)
+			n = 11648;
+		else if (pixel_clk == 148352000)
+			n = 5824;
+		else if (pixel_clk == 297000000)
+			n = 5120;
+		else
+			n = 6144;
+		n *= mult;
+		break;
+
+	default:
+		break;
+	}
+
+	return n;
+}
+
+/*
+ * When transmitting IEC60958 linear PCM audio, these registers allow to
+ * configure the channel status information of all the channel status
+ * bits in the IEC60958 frame. For the moment this configuration is only
+ * used when the I2S audio interface, General Purpose Audio (GPA),
+ * or AHB audio DMA (AHBAUDDMA) interface is active
+ * (for S/PDIF interface this information comes from the stream).
+ */
+void dw_hdmi_set_channel_status(struct dw_hdmi *hdmi,
+				u8 *channel_status)
+{
+	/*
+	 * Set channel status register for frequency and word length.
+	 * Use default values for other registers.
+	 */
+	hdmi_writeb(hdmi, channel_status[3], HDMI_FC_AUDSCHNLS7);
+	hdmi_writeb(hdmi, channel_status[4], HDMI_FC_AUDSCHNLS8);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_channel_status);
+
+static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi,
+	unsigned long pixel_clk, unsigned int sample_rate)
+{
+	unsigned long ftdms = pixel_clk;
+	unsigned int n, cts;
+	u8 config3;
+	u64 tmp;
+
+	n = hdmi_compute_n(sample_rate, pixel_clk);
+
+	config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID);
+
+	/* Compute CTS when using internal AHB audio or General Parallel audio*/
+	if ((config3 & HDMI_CONFIG3_AHBAUDDMA) || (config3 & HDMI_CONFIG3_GPAUD)) {
+		/*
+		 * Compute the CTS value from the N value.  Note that CTS and N
+		 * can be up to 20 bits in total, so we need 64-bit math.  Also
+		 * note that our TDMS clock is not fully accurate; it is
+		 * accurate to kHz.  This can introduce an unnecessary remainder
+		 * in the calculation below, so we don't try to warn about that.
+		 */
+		tmp = (u64)ftdms * n;
+		do_div(tmp, 128 * sample_rate);
+		cts = tmp;
+
+		dev_dbg(hdmi->dev, "%s: fs=%uHz ftdms=%lu.%03luMHz N=%d cts=%d\n",
+			__func__, sample_rate,
+			ftdms / 1000000, (ftdms / 1000) % 1000,
+			n, cts);
+	} else {
+		cts = 0;
+	}
+
+	spin_lock_irq(&hdmi->audio_lock);
+	hdmi->audio_n = n;
+	hdmi->audio_cts = cts;
+	hdmi_set_cts_n(hdmi, cts, hdmi->audio_enable ? n : 0);
+	spin_unlock_irq(&hdmi->audio_lock);
+}
+
+static void hdmi_init_clk_regenerator(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate);
+	mutex_unlock(&hdmi->audio_mutex);
+}
+
+static void hdmi_clk_regenerator_update_pixel_clock(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock,
+				 hdmi->sample_rate);
+	mutex_unlock(&hdmi->audio_mutex);
+}
+
+void dw_hdmi_set_sample_width(struct dw_hdmi *hdmi, unsigned int width)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->sample_width = width;
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_width);
+
+void dw_hdmi_set_sample_non_pcm(struct dw_hdmi *hdmi, unsigned int non_pcm)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->sample_non_pcm = non_pcm;
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_non_pcm);
+
+void dw_hdmi_set_sample_rate(struct dw_hdmi *hdmi, unsigned int rate)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->sample_rate = rate;
+	hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock,
+				 hdmi->sample_rate);
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_rate);
+
+void dw_hdmi_set_channel_count(struct dw_hdmi *hdmi, unsigned int cnt)
+{
+	u8 layout;
+
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->channels = cnt;
+
+	/*
+	 * For >2 channel PCM audio, we need to select layout 1
+	 * and set an appropriate channel map.
+	 */
+	if (cnt > 2)
+		layout = HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT1;
+	else
+		layout = HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT0;
+
+	hdmi_modb(hdmi, layout, HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_MASK,
+		  HDMI_FC_AUDSCONF);
+
+	/* Set the audio infoframes channel count */
+	hdmi_modb(hdmi, (cnt - 1) << HDMI_FC_AUDICONF0_CC_OFFSET,
+		  HDMI_FC_AUDICONF0_CC_MASK, HDMI_FC_AUDICONF0);
+
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_channel_count);
+
+void dw_hdmi_set_channel_allocation(struct dw_hdmi *hdmi, unsigned int ca)
+{
+	mutex_lock(&hdmi->audio_mutex);
+
+	hdmi_writeb(hdmi, ca, HDMI_FC_AUDICONF2);
+
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_channel_allocation);
+
+static void hdmi_enable_audio_clk(struct dw_hdmi *hdmi, bool enable)
+{
+	if (enable)
+		hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_AUDCLK_DISABLE;
+	else
+		hdmi->mc_clkdis |= HDMI_MC_CLKDIS_AUDCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+}
+
+static u8 *hdmi_audio_get_eld(struct dw_hdmi *hdmi)
+{
+	if (!hdmi->curr_conn)
+		return NULL;
+
+	return hdmi->curr_conn->eld;
+}
+
+static void dw_hdmi_gp_audio_enable(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+	int sample_freq = 0x2, org_sample_freq = 0xD;
+	int ch_mask = BIT(hdmi->channels) - 1;
+
+	switch (hdmi->sample_rate) {
+	case 32000:
+		sample_freq = 0x03;
+		org_sample_freq = 0x0C;
+		break;
+	case 44100:
+		sample_freq = 0x00;
+		org_sample_freq = 0x0F;
+		break;
+	case 48000:
+		sample_freq = 0x02;
+		org_sample_freq = 0x0D;
+		break;
+	case 88200:
+		sample_freq = 0x08;
+		org_sample_freq = 0x07;
+		break;
+	case 96000:
+		sample_freq = 0x0A;
+		org_sample_freq = 0x05;
+		break;
+	case 176400:
+		sample_freq = 0x0C;
+		org_sample_freq = 0x03;
+		break;
+	case 192000:
+		sample_freq = 0x0E;
+		org_sample_freq = 0x01;
+		break;
+	default:
+		break;
+	}
+
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+	hdmi_enable_audio_clk(hdmi, true);
+
+	hdmi_writeb(hdmi, 0x1, HDMI_FC_AUDSCHNLS0);
+	hdmi_writeb(hdmi, hdmi->channels, HDMI_FC_AUDSCHNLS2);
+	hdmi_writeb(hdmi, 0x22, HDMI_FC_AUDSCHNLS3);
+	hdmi_writeb(hdmi, 0x22, HDMI_FC_AUDSCHNLS4);
+	hdmi_writeb(hdmi, 0x11, HDMI_FC_AUDSCHNLS5);
+	hdmi_writeb(hdmi, 0x11, HDMI_FC_AUDSCHNLS6);
+	hdmi_writeb(hdmi, (0x3 << 4) | sample_freq, HDMI_FC_AUDSCHNLS7);
+	hdmi_writeb(hdmi, (org_sample_freq << 4) | 0xb, HDMI_FC_AUDSCHNLS8);
+
+	hdmi_writeb(hdmi, ch_mask, HDMI_GP_CONF1);
+	hdmi_writeb(hdmi, 0x02, HDMI_GP_CONF2);
+	hdmi_writeb(hdmi, 0x01, HDMI_GP_CONF0);
+
+	hdmi_modb(hdmi,  0x3, 0x3, HDMI_FC_DATAUTO3);
+
+	/* hbr */
+	if (hdmi->sample_rate == 192000 && hdmi->channels == 8 &&
+	    hdmi->sample_width == 32 && hdmi->sample_non_pcm)
+		hdmi_modb(hdmi, 0x01, 0x01, HDMI_GP_CONF2);
+
+	if (pdata->enable_audio)
+		pdata->enable_audio(hdmi,
+				    hdmi->channels,
+				    hdmi->sample_width,
+				    hdmi->sample_rate,
+				    hdmi->sample_non_pcm);
+}
+
+static void dw_hdmi_gp_audio_disable(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, 0);
+
+	hdmi_modb(hdmi,  0, 0x3, HDMI_FC_DATAUTO3);
+	if (pdata->disable_audio)
+		pdata->disable_audio(hdmi);
+
+	hdmi_enable_audio_clk(hdmi, false);
+}
+
+static void dw_hdmi_ahb_audio_enable(struct dw_hdmi *hdmi)
+{
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+}
+
+static void dw_hdmi_ahb_audio_disable(struct dw_hdmi *hdmi)
+{
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, 0);
+}
+
+static void dw_hdmi_i2s_audio_enable(struct dw_hdmi *hdmi)
+{
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+	hdmi_enable_audio_clk(hdmi, true);
+}
+
+static void dw_hdmi_i2s_audio_disable(struct dw_hdmi *hdmi)
+{
+	hdmi_enable_audio_clk(hdmi, false);
+}
+
+void dw_hdmi_audio_enable(struct dw_hdmi *hdmi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&hdmi->audio_lock, flags);
+	hdmi->audio_enable = true;
+	if (hdmi->enable_audio)
+		hdmi->enable_audio(hdmi);
+	spin_unlock_irqrestore(&hdmi->audio_lock, flags);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_audio_enable);
+
+void dw_hdmi_audio_disable(struct dw_hdmi *hdmi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&hdmi->audio_lock, flags);
+	hdmi->audio_enable = false;
+	if (hdmi->disable_audio)
+		hdmi->disable_audio(hdmi);
+	spin_unlock_irqrestore(&hdmi->audio_lock, flags);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_audio_disable);
+
+static bool hdmi_bus_fmt_is_rgb(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static bool hdmi_bus_fmt_is_yuv444(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_YUV16_1X48:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static bool hdmi_bus_fmt_is_yuv422(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static bool hdmi_bus_fmt_is_yuv420(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static int hdmi_bus_fmt_color_depth(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+		return 8;
+
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		return 10;
+
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+		return 12;
+
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+	case MEDIA_BUS_FMT_YUV16_1X48:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		return 16;
+
+	default:
+		return 0;
+	}
+}
+
+/*
+ * this submodule is responsible for the video data synchronization.
+ * for example, for RGB 4:4:4 input, the data map is defined as
+ *			pin{47~40} <==> R[7:0]
+ *			pin{31~24} <==> G[7:0]
+ *			pin{15~8}  <==> B[7:0]
+ */
+static void hdmi_video_sample(struct dw_hdmi *hdmi)
+{
+	int color_format = 0;
+	u8 val;
+
+	switch (hdmi->hdmi_data.enc_in_bus_format) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		color_format = 0x01;
+		break;
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		color_format = 0x03;
+		break;
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+		color_format = 0x05;
+		break;
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		color_format = 0x07;
+		break;
+
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+		color_format = 0x09;
+		break;
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		color_format = 0x0B;
+		break;
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+		color_format = 0x0D;
+		break;
+	case MEDIA_BUS_FMT_YUV16_1X48:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		color_format = 0x0F;
+		break;
+
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		color_format = 0x16;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		color_format = 0x14;
+		break;
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+		color_format = 0x12;
+		break;
+
+	default:
+		return;
+	}
+
+	val = HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_DISABLE |
+		((color_format << HDMI_TX_INVID0_VIDEO_MAPPING_OFFSET) &
+		HDMI_TX_INVID0_VIDEO_MAPPING_MASK);
+	hdmi_writeb(hdmi, val, HDMI_TX_INVID0);
+
+	/* Enable TX stuffing: When DE is inactive, fix the output data to 0 */
+	val = HDMI_TX_INSTUFFING_BDBDATA_STUFFING_ENABLE |
+		HDMI_TX_INSTUFFING_RCRDATA_STUFFING_ENABLE |
+		HDMI_TX_INSTUFFING_GYDATA_STUFFING_ENABLE;
+	hdmi_writeb(hdmi, val, HDMI_TX_INSTUFFING);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_GYDATA0);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_GYDATA1);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_RCRDATA0);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_RCRDATA1);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_BCBDATA0);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_BCBDATA1);
+}
+
+static int is_color_space_conversion(struct dw_hdmi *hdmi)
+{
+	struct hdmi_data_info *hdmi_data = &hdmi->hdmi_data;
+	bool is_input_rgb, is_output_rgb;
+
+	is_input_rgb = hdmi_bus_fmt_is_rgb(hdmi_data->enc_in_bus_format);
+	is_output_rgb = hdmi_bus_fmt_is_rgb(hdmi_data->enc_out_bus_format);
+
+	return (is_input_rgb != is_output_rgb) ||
+	       (is_input_rgb && is_output_rgb && hdmi_data->rgb_limited_range);
+}
+
+static int is_color_space_decimation(struct dw_hdmi *hdmi)
+{
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		return 0;
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_in_bus_format) ||
+	    hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_in_bus_format))
+		return 1;
+
+	return 0;
+}
+
+static int is_color_space_interpolation(struct dw_hdmi *hdmi)
+{
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_in_bus_format))
+		return 0;
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format) ||
+	    hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
+		return 1;
+
+	return 0;
+}
+
+static bool is_csc_needed(struct dw_hdmi *hdmi)
+{
+	return is_color_space_conversion(hdmi) ||
+	       is_color_space_decimation(hdmi) ||
+	       is_color_space_interpolation(hdmi);
+}
+
+static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi)
+{
+	const u16(*csc_coeff)[3][4] = &csc_coeff_default;
+	bool is_input_rgb, is_output_rgb, is_output_yuv;
+	unsigned i;
+	u32 csc_scale = 1;
+	unsigned int depth =
+		hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format);
+
+	is_input_rgb = hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_in_bus_format);
+	is_output_rgb = hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format);
+	is_output_yuv =
+		hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format) ||
+		hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format);
+
+	if (!is_input_rgb && is_output_rgb) {
+		if (hdmi->hdmi_data.enc_out_encoding == V4L2_YCBCR_ENC_601)
+			csc_coeff = &csc_coeff_rgb_out_eitu601;
+		else
+			csc_coeff = &csc_coeff_rgb_out_eitu709;
+	} else if (is_input_rgb && !is_output_rgb) {
+		if (hdmi->hdmi_data.enc_out_encoding == V4L2_YCBCR_ENC_601) {
+			if (is_output_yuv && depth == 8)
+				csc_coeff = &csc_coeff_rgb_to_yuv_8bit_eitu601;
+			else if (is_output_yuv && depth == 10)
+				csc_coeff = &csc_coeff_rgb_to_yuv_10bit_eitu601;
+			else
+				csc_coeff = &csc_coeff_rgb_in_eitu601;
+		} else {
+			if (is_output_yuv && depth == 8)
+				csc_coeff = &csc_coeff_rgb_to_yuv_8bit_eitu709;
+			else if (is_output_yuv && depth == 10)
+				csc_coeff = &csc_coeff_rgb_to_yuv_10bit_eitu709;
+			else
+				csc_coeff = &csc_coeff_rgb_in_eitu709;
+		}
+		csc_scale = 0;
+	} else if (is_input_rgb && is_output_rgb &&
+		   hdmi->hdmi_data.rgb_limited_range) {
+		csc_coeff = &csc_coeff_rgb_full_to_rgb_limited;
+	}
+
+	/* The CSC registers are sequential, alternating MSB then LSB */
+	for (i = 0; i < ARRAY_SIZE(csc_coeff_default[0]); i++) {
+		u16 coeff_a = (*csc_coeff)[0][i];
+		u16 coeff_b = (*csc_coeff)[1][i];
+		u16 coeff_c = (*csc_coeff)[2][i];
+
+		hdmi_writeb(hdmi, coeff_a & 0xff, HDMI_CSC_COEF_A1_LSB + i * 2);
+		hdmi_writeb(hdmi, coeff_a >> 8, HDMI_CSC_COEF_A1_MSB + i * 2);
+		hdmi_writeb(hdmi, coeff_b & 0xff, HDMI_CSC_COEF_B1_LSB + i * 2);
+		hdmi_writeb(hdmi, coeff_b >> 8, HDMI_CSC_COEF_B1_MSB + i * 2);
+		hdmi_writeb(hdmi, coeff_c & 0xff, HDMI_CSC_COEF_C1_LSB + i * 2);
+		hdmi_writeb(hdmi, coeff_c >> 8, HDMI_CSC_COEF_C1_MSB + i * 2);
+	}
+
+	hdmi_modb(hdmi, csc_scale, HDMI_CSC_SCALE_CSCSCALE_MASK,
+		  HDMI_CSC_SCALE);
+}
+
+static void hdmi_video_csc(struct dw_hdmi *hdmi)
+{
+	int color_depth = 0;
+	int interpolation = HDMI_CSC_CFG_INTMODE_DISABLE;
+	int decimation = 0;
+
+	/* YCC422 interpolation to 444 mode */
+	if (is_color_space_interpolation(hdmi))
+		interpolation = HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA1;
+	else if (is_color_space_decimation(hdmi))
+		decimation = HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA3;
+
+	switch (hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format)) {
+	case 8:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_24BPP;
+		break;
+	case 10:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_30BPP;
+		break;
+	case 12:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_36BPP;
+		break;
+	case 16:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_48BPP;
+		break;
+
+	default:
+		return;
+	}
+
+	/* Configure the CSC registers */
+	hdmi_writeb(hdmi, interpolation | decimation, HDMI_CSC_CFG);
+	hdmi_modb(hdmi, color_depth, HDMI_CSC_SCALE_CSC_COLORDE_PTH_MASK,
+		  HDMI_CSC_SCALE);
+
+	dw_hdmi_update_csc_coeffs(hdmi);
+}
+
+/*
+ * HDMI video packetizer is used to packetize the data.
+ * for example, if input is YCC422 mode or repeater is used,
+ * data should be repacked this module can be bypassed.
+ */
+static void hdmi_video_packetize(struct dw_hdmi *hdmi)
+{
+	unsigned int color_depth = 0;
+	unsigned int remap_size = HDMI_VP_REMAP_YCC422_16bit;
+	unsigned int output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_PP;
+	struct hdmi_data_info *hdmi_data = &hdmi->hdmi_data;
+	u8 val, vp_conf;
+	u8 clear_gcp_auto = 0;
+
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format) ||
+	    hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format) ||
+	    hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi_bus_fmt_color_depth(
+					hdmi->hdmi_data.enc_out_bus_format)) {
+		case 8:
+			color_depth = 4;
+			output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS;
+			clear_gcp_auto = 1;
+			break;
+		case 10:
+			color_depth = 5;
+			break;
+		case 12:
+			color_depth = 6;
+			break;
+		case 16:
+			color_depth = 7;
+			break;
+		default:
+			output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS;
+		}
+	} else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi_bus_fmt_color_depth(
+					hdmi->hdmi_data.enc_out_bus_format)) {
+		case 0:
+		case 8:
+			remap_size = HDMI_VP_REMAP_YCC422_16bit;
+			clear_gcp_auto = 1;
+			break;
+		case 10:
+			remap_size = HDMI_VP_REMAP_YCC422_20bit;
+			break;
+		case 12:
+			remap_size = HDMI_VP_REMAP_YCC422_24bit;
+			break;
+
+		default:
+			return;
+		}
+		output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_YCC422;
+	} else {
+		return;
+	}
+
+	/* set the packetizer registers */
+	val = ((color_depth << HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET) &
+		HDMI_VP_PR_CD_COLOR_DEPTH_MASK) |
+		((hdmi_data->pix_repet_factor <<
+		HDMI_VP_PR_CD_DESIRED_PR_FACTOR_OFFSET) &
+		HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK);
+	hdmi_writeb(hdmi, val, HDMI_VP_PR_CD);
+
+	/* HDMI1.4b specification section 6.5.3:
+	 * Source shall only send GCPs with non-zero CD to sinks
+	 * that indicate support for Deep Color.
+	 * GCP only transmit CD and do not handle AVMUTE, PP norDefault_Phase (yet).
+	 * Disable Auto GCP when 24-bit color for sinks that not support Deep Color.
+	 */
+	val = hdmi_readb(hdmi, HDMI_FC_DATAUTO3);
+	if (clear_gcp_auto == 1)
+		val &= ~HDMI_FC_DATAUTO3_GCP_AUTO;
+	else
+		val |= HDMI_FC_DATAUTO3_GCP_AUTO;
+	hdmi_writeb(hdmi, val, HDMI_FC_DATAUTO3);
+
+	hdmi_modb(hdmi, HDMI_VP_STUFF_PR_STUFFING_STUFFING_MODE,
+		  HDMI_VP_STUFF_PR_STUFFING_MASK, HDMI_VP_STUFF);
+
+	/* Data from pixel repeater block */
+	if (hdmi_data->pix_repet_factor > 1) {
+		vp_conf = HDMI_VP_CONF_PR_EN_ENABLE |
+			  HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER;
+	} else { /* data from packetizer block */
+		vp_conf = HDMI_VP_CONF_PR_EN_DISABLE |
+			  HDMI_VP_CONF_BYPASS_SELECT_VID_PACKETIZER;
+	}
+
+	hdmi_modb(hdmi, vp_conf,
+		  HDMI_VP_CONF_PR_EN_MASK |
+		  HDMI_VP_CONF_BYPASS_SELECT_MASK, HDMI_VP_CONF);
+
+	hdmi_modb(hdmi, 1 << HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET,
+		  HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, HDMI_VP_STUFF);
+
+	hdmi_writeb(hdmi, remap_size, HDMI_VP_REMAP);
+
+	if (output_select == HDMI_VP_CONF_OUTPUT_SELECTOR_PP) {
+		vp_conf = HDMI_VP_CONF_BYPASS_EN_DISABLE |
+			  HDMI_VP_CONF_PP_EN_ENABLE |
+			  HDMI_VP_CONF_YCC422_EN_DISABLE;
+	} else if (output_select == HDMI_VP_CONF_OUTPUT_SELECTOR_YCC422) {
+		vp_conf = HDMI_VP_CONF_BYPASS_EN_DISABLE |
+			  HDMI_VP_CONF_PP_EN_DISABLE |
+			  HDMI_VP_CONF_YCC422_EN_ENABLE;
+	} else if (output_select == HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS) {
+		vp_conf = HDMI_VP_CONF_BYPASS_EN_ENABLE |
+			  HDMI_VP_CONF_PP_EN_DISABLE |
+			  HDMI_VP_CONF_YCC422_EN_DISABLE;
+	} else {
+		return;
+	}
+
+	hdmi_modb(hdmi, vp_conf,
+		  HDMI_VP_CONF_BYPASS_EN_MASK | HDMI_VP_CONF_PP_EN_ENMASK |
+		  HDMI_VP_CONF_YCC422_EN_MASK, HDMI_VP_CONF);
+
+	hdmi_modb(hdmi, HDMI_VP_STUFF_PP_STUFFING_STUFFING_MODE |
+			HDMI_VP_STUFF_YCC422_STUFFING_STUFFING_MODE,
+		  HDMI_VP_STUFF_PP_STUFFING_MASK |
+		  HDMI_VP_STUFF_YCC422_STUFFING_MASK, HDMI_VP_STUFF);
+
+	hdmi_modb(hdmi, output_select, HDMI_VP_CONF_OUTPUT_SELECTOR_MASK,
+		  HDMI_VP_CONF);
+}
+
+/* -----------------------------------------------------------------------------
+ * Synopsys PHY Handling
+ */
+
+static inline void hdmi_phy_test_clear(struct dw_hdmi *hdmi,
+				       unsigned char bit)
+{
+	hdmi_modb(hdmi, bit << HDMI_PHY_TST0_TSTCLR_OFFSET,
+		  HDMI_PHY_TST0_TSTCLR_MASK, HDMI_PHY_TST0);
+}
+
+static bool hdmi_phy_wait_i2c_done(struct dw_hdmi *hdmi, int msec)
+{
+	u32 val;
+
+	while ((val = hdmi_readb(hdmi, HDMI_IH_I2CMPHY_STAT0) & 0x3) == 0) {
+		if (msec-- == 0)
+			return false;
+		udelay(1000);
+	}
+	hdmi_writeb(hdmi, val, HDMI_IH_I2CMPHY_STAT0);
+
+	return true;
+}
+
+void dw_hdmi_phy_i2c_write(struct dw_hdmi *hdmi, unsigned short data,
+			   unsigned char addr)
+{
+	hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0);
+	hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR);
+	hdmi_writeb(hdmi, (unsigned char)(data >> 8),
+		    HDMI_PHY_I2CM_DATAO_1_ADDR);
+	hdmi_writeb(hdmi, (unsigned char)(data >> 0),
+		    HDMI_PHY_I2CM_DATAO_0_ADDR);
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_WRITE,
+		    HDMI_PHY_I2CM_OPERATION_ADDR);
+	hdmi_phy_wait_i2c_done(hdmi, 1000);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_i2c_write);
+
+static int hdmi_phy_i2c_read(struct dw_hdmi *hdmi, unsigned char addr)
+{
+	int val;
+
+	hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0);
+	hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR);
+	hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_1_ADDR);
+	hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_0_ADDR);
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_READ,
+		    HDMI_PHY_I2CM_OPERATION_ADDR);
+	hdmi_phy_wait_i2c_done(hdmi, 1000);
+	val = hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_1_ADDR);
+	val = (val & 0xff) << 8;
+	val += hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_0_ADDR) & 0xff;
+	return val;
+}
+
+/* Filter out invalid setups to avoid configuring SCDC and scrambling */
+static bool dw_hdmi_support_scdc(struct dw_hdmi *hdmi,
+				 const struct drm_display_info *display)
+{
+	/* Completely disable SCDC support for older controllers */
+	if (hdmi->version < 0x200a)
+		return false;
+
+	/* Disable if no DDC bus */
+	if (!hdmi->ddc)
+		return false;
+
+	/* Disable if SCDC is not supported, or if an HF-VSDB block is absent */
+	if (!display->hdmi.scdc.supported ||
+	    !display->hdmi.scdc.scrambling.supported)
+		return false;
+
+	/*
+	 * Disable if display only support low TMDS rates and scrambling
+	 * for low rates is not supported either
+	 */
+	if (!display->hdmi.scdc.scrambling.low_rates &&
+	    display->max_tmds_clock <= 340000)
+		return false;
+
+	return true;
+}
+
+/*
+ * HDMI2.0 Specifies the following procedure for High TMDS Bit Rates:
+ * - The Source shall suspend transmission of the TMDS clock and data
+ * - The Source shall write to the TMDS_Bit_Clock_Ratio bit to change it
+ * from a 0 to a 1 or from a 1 to a 0
+ * - The Source shall allow a minimum of 1 ms and a maximum of 100 ms from
+ * the time the TMDS_Bit_Clock_Ratio bit is written until resuming
+ * transmission of TMDS clock and data
+ *
+ * To respect the 100ms maximum delay, the dw_hdmi_set_high_tmds_clock_ratio()
+ * helper should called right before enabling the TMDS Clock and Data in
+ * the PHY configuration callback.
+ */
+void dw_hdmi_set_high_tmds_clock_ratio(struct dw_hdmi *hdmi,
+				       const struct drm_display_info *display)
+{
+	unsigned long mtmdsclock = hdmi->hdmi_data.video_mode.mtmdsclock;
+
+	/* Control for TMDS Bit Period/TMDS Clock-Period Ratio */
+	if (dw_hdmi_support_scdc(hdmi, display)) {
+		if (mtmdsclock > HDMI14_MAX_TMDSCLK)
+			drm_scdc_set_high_tmds_clock_ratio(hdmi->curr_conn, 1);
+		else
+			drm_scdc_set_high_tmds_clock_ratio(hdmi->curr_conn, 0);
+	}
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_high_tmds_clock_ratio);
+
+static void dw_hdmi_phy_enable_powerdown(struct dw_hdmi *hdmi, bool enable)
+{
+	hdmi_mask_writeb(hdmi, !enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_PDZ_OFFSET,
+			 HDMI_PHY_CONF0_PDZ_MASK);
+}
+
+static void dw_hdmi_phy_enable_tmds(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_ENTMDS_OFFSET,
+			 HDMI_PHY_CONF0_ENTMDS_MASK);
+}
+
+static void dw_hdmi_phy_enable_svsret(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_SVSRET_OFFSET,
+			 HDMI_PHY_CONF0_SVSRET_MASK);
+}
+
+void dw_hdmi_phy_gen2_pddq(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_GEN2_PDDQ_OFFSET,
+			 HDMI_PHY_CONF0_GEN2_PDDQ_MASK);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen2_pddq);
+
+void dw_hdmi_phy_gen2_txpwron(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_GEN2_TXPWRON_OFFSET,
+			 HDMI_PHY_CONF0_GEN2_TXPWRON_MASK);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen2_txpwron);
+
+static void dw_hdmi_phy_sel_data_en_pol(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_SELDATAENPOL_OFFSET,
+			 HDMI_PHY_CONF0_SELDATAENPOL_MASK);
+}
+
+static void dw_hdmi_phy_sel_interface_control(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_SELDIPIF_OFFSET,
+			 HDMI_PHY_CONF0_SELDIPIF_MASK);
+}
+
+void dw_hdmi_phy_gen1_reset(struct dw_hdmi *hdmi)
+{
+	/* PHY reset. The reset signal is active low on Gen1 PHYs. */
+	hdmi_writeb(hdmi, 0, HDMI_MC_PHYRSTZ);
+	hdmi_writeb(hdmi, HDMI_MC_PHYRSTZ_PHYRSTZ, HDMI_MC_PHYRSTZ);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen1_reset);
+
+void dw_hdmi_phy_gen2_reset(struct dw_hdmi *hdmi)
+{
+	/* PHY reset. The reset signal is active high on Gen2 PHYs. */
+	hdmi_writeb(hdmi, HDMI_MC_PHYRSTZ_PHYRSTZ, HDMI_MC_PHYRSTZ);
+	hdmi_writeb(hdmi, 0, HDMI_MC_PHYRSTZ);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen2_reset);
+
+void dw_hdmi_phy_i2c_set_addr(struct dw_hdmi *hdmi, u8 address)
+{
+	hdmi_phy_test_clear(hdmi, 1);
+	hdmi_writeb(hdmi, address, HDMI_PHY_I2CM_SLAVE_ADDR);
+	hdmi_phy_test_clear(hdmi, 0);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_i2c_set_addr);
+
+static void dw_hdmi_phy_power_off(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_phy_data *phy = hdmi->phy.data;
+	unsigned int i;
+	u16 val;
+
+	if (phy->gen == 1) {
+		dw_hdmi_phy_enable_tmds(hdmi, 0);
+		dw_hdmi_phy_enable_powerdown(hdmi, true);
+		return;
+	}
+
+	dw_hdmi_phy_gen2_txpwron(hdmi, 0);
+
+	/*
+	 * Wait for TX_PHY_LOCK to be deasserted to indicate that the PHY went
+	 * to low power mode.
+	 */
+	for (i = 0; i < 5; ++i) {
+		val = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+		if (!(val & HDMI_PHY_TX_PHY_LOCK))
+			break;
+
+		usleep_range(1000, 2000);
+	}
+
+	if (val & HDMI_PHY_TX_PHY_LOCK)
+		dev_warn(hdmi->dev, "PHY failed to power down\n");
+	else
+		dev_dbg(hdmi->dev, "PHY powered down in %u iterations\n", i);
+
+	dw_hdmi_phy_gen2_pddq(hdmi, 1);
+}
+
+static int dw_hdmi_phy_power_on(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_phy_data *phy = hdmi->phy.data;
+	unsigned int i;
+	u8 val;
+
+	if (phy->gen == 1) {
+		dw_hdmi_phy_enable_powerdown(hdmi, false);
+
+		/* Toggle TMDS enable. */
+		dw_hdmi_phy_enable_tmds(hdmi, 0);
+		dw_hdmi_phy_enable_tmds(hdmi, 1);
+		return 0;
+	}
+
+	dw_hdmi_phy_gen2_txpwron(hdmi, 1);
+	dw_hdmi_phy_gen2_pddq(hdmi, 0);
+
+	/* Wait for PHY PLL lock */
+	for (i = 0; i < 5; ++i) {
+		val = hdmi_readb(hdmi, HDMI_PHY_STAT0) & HDMI_PHY_TX_PHY_LOCK;
+		if (val)
+			break;
+
+		usleep_range(1000, 2000);
+	}
+
+	if (!val) {
+		dev_err(hdmi->dev, "PHY PLL failed to lock\n");
+		return -ETIMEDOUT;
+	}
+
+	dev_dbg(hdmi->dev, "PHY PLL locked %u iterations\n", i);
+	return 0;
+}
+
+/*
+ * PHY configuration function for the DWC HDMI 3D TX PHY. Based on the available
+ * information the DWC MHL PHY has the same register layout and is thus also
+ * supported by this function.
+ */
+static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi,
+		const struct dw_hdmi_plat_data *pdata,
+		unsigned long mpixelclock)
+{
+	const struct dw_hdmi_mpll_config *mpll_config = pdata->mpll_cfg;
+	const struct dw_hdmi_curr_ctrl *curr_ctrl = pdata->cur_ctr;
+	const struct dw_hdmi_phy_config *phy_config = pdata->phy_config;
+
+	/* TOFIX Will need 420 specific PHY configuration tables */
+	unsigned int depth =
+		hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format);
+
+	/* PLL/MPLL Cfg - always match on final entry */
+	for (; mpll_config->mpixelclock != ~0UL; mpll_config++)
+		if (mpixelclock <= mpll_config->mpixelclock)
+			break;
+
+	for (; curr_ctrl->mpixelclock != ~0UL; curr_ctrl++)
+		if (mpixelclock <= curr_ctrl->mpixelclock)
+			break;
+
+	for (; phy_config->mpixelclock != ~0UL; phy_config++)
+		if (mpixelclock <= phy_config->mpixelclock)
+			break;
+
+	if (mpll_config->mpixelclock == ~0UL ||
+	    curr_ctrl->mpixelclock == ~0UL || phy_config->mpixelclock == ~0UL)
+		return -EINVAL;
+
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		depth = fls(depth - 8);
+	else
+		depth = 0;
+	if (depth)
+		depth--;
+
+	dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].cpce,
+			      HDMI_3D_TX_PHY_CPCE_CTRL);
+	dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].gmp,
+			      HDMI_3D_TX_PHY_DRVANACTRL);
+	dw_hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[depth],
+			      HDMI_3D_TX_PHY_CURRCTRL);
+
+	dw_hdmi_phy_i2c_write(hdmi, 0, HDMI_3D_TX_PHY_PLLPHBYCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_FB_CLK,
+			      HDMI_3D_TX_PHY_MSM_CTRL);
+
+	dw_hdmi_phy_i2c_write(hdmi, phy_config->term, HDMI_3D_TX_PHY_TXTERM);
+	dw_hdmi_phy_i2c_write(hdmi, phy_config->sym_ctr,
+			      HDMI_3D_TX_PHY_CKSYMTXCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, phy_config->vlev_ctr,
+			      HDMI_3D_TX_PHY_VLEVCTRL);
+
+	/* Override and disable clock termination. */
+	dw_hdmi_phy_i2c_write(hdmi, HDMI_3D_TX_PHY_CKCALCTRL_OVERRIDE,
+			      HDMI_3D_TX_PHY_CKCALCTRL);
+
+	return 0;
+}
+
+static int hdmi_phy_configure(struct dw_hdmi *hdmi,
+			      const struct drm_display_info *display)
+{
+	const struct dw_hdmi_phy_data *phy = hdmi->phy.data;
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+	unsigned long mpixelclock = hdmi->hdmi_data.video_mode.mpixelclock;
+	unsigned long mtmdsclock = hdmi->hdmi_data.video_mode.mtmdsclock;
+	int ret;
+
+	dw_hdmi_phy_power_off(hdmi);
+
+	dw_hdmi_set_high_tmds_clock_ratio(hdmi, display);
+
+	/* Leave low power consumption mode by asserting SVSRET. */
+	if (phy->has_svsret)
+		dw_hdmi_phy_enable_svsret(hdmi, 1);
+
+	dw_hdmi_phy_gen2_reset(hdmi);
+
+	hdmi_writeb(hdmi, HDMI_MC_HEACPHY_RST_ASSERT, HDMI_MC_HEACPHY_RST);
+
+	dw_hdmi_phy_i2c_set_addr(hdmi, HDMI_PHY_I2CM_SLAVE_ADDR_PHY_GEN2);
+
+	/* Write to the PHY as configured by the platform */
+	if (pdata->configure_phy)
+		ret = pdata->configure_phy(hdmi, pdata->priv_data, mpixelclock);
+	else
+		ret = phy->configure(hdmi, pdata, mpixelclock);
+	if (ret) {
+		dev_err(hdmi->dev, "PHY configuration failed (clock %lu)\n",
+			mpixelclock);
+		return ret;
+	}
+
+	/* Wait for resuming transmission of TMDS clock and data */
+	if (mtmdsclock > HDMI14_MAX_TMDSCLK)
+		msleep(100);
+
+	return dw_hdmi_phy_power_on(hdmi);
+}
+
+static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
+			    const struct drm_display_info *display,
+			    const struct drm_display_mode *mode)
+{
+	int i, ret;
+
+	/* HDMI Phy spec says to do the phy initialization sequence twice */
+	for (i = 0; i < 2; i++) {
+		dw_hdmi_phy_sel_data_en_pol(hdmi, 1);
+		dw_hdmi_phy_sel_interface_control(hdmi, 0);
+
+		ret = hdmi_phy_configure(hdmi, display);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void dw_hdmi_phy_disable(struct dw_hdmi *hdmi, void *data)
+{
+	dw_hdmi_phy_power_off(hdmi);
+}
+
+enum drm_connector_status dw_hdmi_phy_read_hpd(struct dw_hdmi *hdmi,
+					       void *data)
+{
+	return hdmi_readb(hdmi, HDMI_PHY_STAT0) & HDMI_PHY_HPD ?
+		connector_status_connected : connector_status_disconnected;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_read_hpd);
+
+void dw_hdmi_phy_update_hpd(struct dw_hdmi *hdmi, void *data,
+			    bool force, bool disabled, bool rxsense)
+{
+	u8 old_mask = hdmi->phy_mask;
+
+	if (force || disabled || !rxsense)
+		hdmi->phy_mask |= HDMI_PHY_RX_SENSE;
+	else
+		hdmi->phy_mask &= ~HDMI_PHY_RX_SENSE;
+
+	if (old_mask != hdmi->phy_mask)
+		hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_update_hpd);
+
+void dw_hdmi_phy_setup_hpd(struct dw_hdmi *hdmi, void *data)
+{
+	/*
+	 * Configure the PHY RX SENSE and HPD interrupts polarities and clear
+	 * any pending interrupt.
+	 */
+	hdmi_writeb(hdmi, HDMI_PHY_HPD | HDMI_PHY_RX_SENSE, HDMI_PHY_POL0);
+	hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE,
+		    HDMI_IH_PHY_STAT0);
+
+	/* Enable cable hot plug irq. */
+	hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0);
+
+	/* Clear and unmute interrupts. */
+	hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE,
+		    HDMI_IH_PHY_STAT0);
+	hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE),
+		    HDMI_IH_MUTE_PHY_STAT0);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_setup_hpd);
+
+static const struct dw_hdmi_phy_ops dw_hdmi_synopsys_phy_ops = {
+	.init = dw_hdmi_phy_init,
+	.disable = dw_hdmi_phy_disable,
+	.read_hpd = dw_hdmi_phy_read_hpd,
+	.update_hpd = dw_hdmi_phy_update_hpd,
+	.setup_hpd = dw_hdmi_phy_setup_hpd,
+};
+
+/* -----------------------------------------------------------------------------
+ * HDMI TX Setup
+ */
+
+static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi,
+				const struct drm_display_mode *mode)
+{
+	u8 de, vsync_pol, hsync_pol, hdmi_dvi;
+
+	/* Configure the video polarity */
+	if (hdmi->hdmi_data.video_mode.mdataenablepolarity)
+		de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH;
+	else
+		de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW;
+
+	vsync_pol = mode->flags & DRM_MODE_FLAG_PVSYNC ?
+			    HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_HIGH :
+			    HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_LOW;
+
+	hsync_pol = mode->flags & DRM_MODE_FLAG_PHSYNC ?
+			    HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH :
+			    HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW;
+
+	/* Config the display mode */
+	hdmi_dvi = hdmi->sink_is_hdmi ? HDMI_A_HDCPCFG0_HDMIDVI_HDMI :
+					HDMI_A_HDCPCFG0_HDMIDVI_DVI;
+
+	hdmi_modb(hdmi, hdmi_dvi, HDMI_A_HDCPCFG0_HDMIDVI_MASK,
+		  HDMI_A_HDCPCFG0);
+
+	/* disable rx detect */
+	hdmi_modb(hdmi, HDMI_A_HDCPCFG0_RXDETECT_DISABLE,
+		  HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0);
+
+	hdmi_modb(hdmi, vsync_pol | hsync_pol | de,
+		  HDMI_A_VIDPOLCFG_VSYNCPOL_MASK |
+			  HDMI_A_VIDPOLCFG_HSYNCPOL_MASK |
+			  HDMI_A_VIDPOLCFG_DATAENPOL_MASK,
+		  HDMI_A_VIDPOLCFG);
+
+	hdmi_modb(hdmi, HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE,
+		  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, HDMI_A_HDCPCFG1);
+
+	if (hdmi->hdcp && hdmi->hdcp->hdcp_start) {
+		hdmi->hdcp->hdcp_start(hdmi->hdcp);
+	}
+}
+
+static void hdmi_config_AVI(struct dw_hdmi *hdmi,
+			    const struct drm_connector *connector,
+			    const struct drm_display_mode *mode)
+{
+	struct hdmi_avi_infoframe frame;
+	u8 val;
+
+	/* Initialise info frame from DRM mode */
+	drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode);
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) {
+		drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode,
+						   hdmi->hdmi_data.rgb_limited_range ?
+						   HDMI_QUANTIZATION_RANGE_LIMITED :
+						   HDMI_QUANTIZATION_RANGE_FULL);
+	} else {
+		frame.quantization_range = HDMI_QUANTIZATION_RANGE_DEFAULT;
+		frame.ycc_quantization_range =
+			HDMI_YCC_QUANTIZATION_RANGE_LIMITED;
+	}
+
+	if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
+		frame.colorspace = HDMI_COLORSPACE_YUV444;
+	else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		frame.colorspace = HDMI_COLORSPACE_YUV422;
+	else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format))
+		frame.colorspace = HDMI_COLORSPACE_YUV420;
+	else
+		frame.colorspace = HDMI_COLORSPACE_RGB;
+
+	/* Set up colorimetry */
+	if (!hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi->hdmi_data.enc_out_encoding) {
+		case V4L2_YCBCR_ENC_601:
+			if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV601)
+				frame.colorimetry = HDMI_COLORIMETRY_EXTENDED;
+			else
+				frame.colorimetry = HDMI_COLORIMETRY_ITU_601;
+			frame.extended_colorimetry =
+					HDMI_EXTENDED_COLORIMETRY_XV_YCC_601;
+			break;
+		case V4L2_YCBCR_ENC_709:
+			if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV709)
+				frame.colorimetry = HDMI_COLORIMETRY_EXTENDED;
+			else
+				frame.colorimetry = HDMI_COLORIMETRY_ITU_709;
+			frame.extended_colorimetry =
+					HDMI_EXTENDED_COLORIMETRY_XV_YCC_709;
+			break;
+		case V4L2_YCBCR_ENC_BT2020:
+			if (hdmi->hdmi_data.enc_in_encoding ==
+			    V4L2_YCBCR_ENC_BT2020)
+				frame.colorimetry = HDMI_COLORIMETRY_EXTENDED;
+			else
+				frame.colorimetry = HDMI_COLORIMETRY_ITU_709;
+			frame.extended_colorimetry =
+				HDMI_EXTENDED_COLORIMETRY_BT2020;
+			break;
+		default: /* Carries no data */
+			frame.colorimetry = HDMI_COLORIMETRY_ITU_601;
+			frame.extended_colorimetry =
+					HDMI_EXTENDED_COLORIMETRY_XV_YCC_601;
+			break;
+		}
+	} else {
+		frame.colorimetry = HDMI_COLORIMETRY_NONE;
+		frame.extended_colorimetry =
+			HDMI_EXTENDED_COLORIMETRY_XV_YCC_601;
+	}
+
+	frame.scan_mode = HDMI_SCAN_MODE_NONE;
+
+	/*
+	 * The Designware IP uses a different byte format from standard
+	 * AVI info frames, though generally the bits are in the correct
+	 * bytes.
+	 */
+
+	/*
+	 * AVI data byte 1 differences: Colorspace in bits 0,1 rather than 5,6,
+	 * scan info in bits 4,5 rather than 0,1 and active aspect present in
+	 * bit 6 rather than 4.
+	 */
+	val = (frame.scan_mode & 3) << 4 | (frame.colorspace & 3);
+	if (frame.active_aspect & 15)
+		val |= HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT;
+	if (frame.top_bar || frame.bottom_bar)
+		val |= HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR;
+	if (frame.left_bar || frame.right_bar)
+		val |= HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR;
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF0);
+
+	/* AVI data byte 2 differences: none */
+	val = ((frame.colorimetry & 0x3) << 6) |
+	      ((frame.picture_aspect & 0x3) << 4) |
+	      (frame.active_aspect & 0xf);
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF1);
+
+	/* AVI data byte 3 differences: none */
+	val = ((frame.extended_colorimetry & 0x7) << 4) |
+	      ((frame.quantization_range & 0x3) << 2) |
+	      (frame.nups & 0x3);
+	if (frame.itc)
+		val |= HDMI_FC_AVICONF2_IT_CONTENT_VALID;
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF2);
+
+	/* AVI data byte 4 differences: none */
+	val = frame.video_code & 0x7f;
+	hdmi_writeb(hdmi, val, HDMI_FC_AVIVID);
+
+	/* AVI Data Byte 5- set up input and output pixel repetition */
+	val = (((hdmi->hdmi_data.video_mode.mpixelrepetitioninput + 1) <<
+		HDMI_FC_PRCONF_INCOMING_PR_FACTOR_OFFSET) &
+		HDMI_FC_PRCONF_INCOMING_PR_FACTOR_MASK) |
+		((hdmi->hdmi_data.video_mode.mpixelrepetitionoutput <<
+		HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_OFFSET) &
+		HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK);
+	hdmi_writeb(hdmi, val, HDMI_FC_PRCONF);
+
+	/*
+	 * AVI data byte 5 differences: content type in 0,1 rather than 4,5,
+	 * ycc range in bits 2,3 rather than 6,7
+	 */
+	val = ((frame.ycc_quantization_range & 0x3) << 2) |
+	      (frame.content_type & 0x3);
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF3);
+
+	/* AVI Data Bytes 6-13 */
+	hdmi_writeb(hdmi, frame.top_bar & 0xff, HDMI_FC_AVIETB0);
+	hdmi_writeb(hdmi, (frame.top_bar >> 8) & 0xff, HDMI_FC_AVIETB1);
+	hdmi_writeb(hdmi, frame.bottom_bar & 0xff, HDMI_FC_AVISBB0);
+	hdmi_writeb(hdmi, (frame.bottom_bar >> 8) & 0xff, HDMI_FC_AVISBB1);
+	hdmi_writeb(hdmi, frame.left_bar & 0xff, HDMI_FC_AVIELB0);
+	hdmi_writeb(hdmi, (frame.left_bar >> 8) & 0xff, HDMI_FC_AVIELB1);
+	hdmi_writeb(hdmi, frame.right_bar & 0xff, HDMI_FC_AVISRB0);
+	hdmi_writeb(hdmi, (frame.right_bar >> 8) & 0xff, HDMI_FC_AVISRB1);
+}
+
+static void hdmi_config_vendor_specific_infoframe(struct dw_hdmi *hdmi,
+						  const struct drm_connector *connector,
+						  const struct drm_display_mode *mode)
+{
+	struct hdmi_vendor_infoframe frame;
+	u8 buffer[10];
+	ssize_t err;
+
+	err = drm_hdmi_vendor_infoframe_from_display_mode(&frame, connector,
+							  mode);
+	if (err < 0)
+		/*
+		 * Going into that statement does not means vendor infoframe
+		 * fails. It just informed us that vendor infoframe is not
+		 * needed for the selected mode. Only 4k or stereoscopic 3D
+		 * mode requires vendor infoframe. So just simply return.
+		 */
+		return;
+
+	err = hdmi_vendor_infoframe_pack(&frame, buffer, sizeof(buffer));
+	if (err < 0) {
+		dev_err(hdmi->dev, "Failed to pack vendor infoframe: %zd\n",
+			err);
+		return;
+	}
+	hdmi_mask_writeb(hdmi, 0, HDMI_FC_DATAUTO0, HDMI_FC_DATAUTO0_VSD_OFFSET,
+			HDMI_FC_DATAUTO0_VSD_MASK);
+
+	/* Set the length of HDMI vendor specific InfoFrame payload */
+	hdmi_writeb(hdmi, buffer[2], HDMI_FC_VSDSIZE);
+
+	/* Set 24bit IEEE Registration Identifier */
+	hdmi_writeb(hdmi, buffer[4], HDMI_FC_VSDIEEEID0);
+	hdmi_writeb(hdmi, buffer[5], HDMI_FC_VSDIEEEID1);
+	hdmi_writeb(hdmi, buffer[6], HDMI_FC_VSDIEEEID2);
+
+	/* Set HDMI_Video_Format and HDMI_VIC/3D_Structure */
+	hdmi_writeb(hdmi, buffer[7], HDMI_FC_VSDPAYLOAD0);
+	hdmi_writeb(hdmi, buffer[8], HDMI_FC_VSDPAYLOAD1);
+
+	if (frame.s3d_struct >= HDMI_3D_STRUCTURE_SIDE_BY_SIDE_HALF)
+		hdmi_writeb(hdmi, buffer[9], HDMI_FC_VSDPAYLOAD2);
+
+	/* Packet frame interpolation */
+	hdmi_writeb(hdmi, 1, HDMI_FC_DATAUTO1);
+
+	/* Auto packets per frame and line spacing */
+	hdmi_writeb(hdmi, 0x11, HDMI_FC_DATAUTO2);
+
+	/* Configures the Frame Composer On RDRB mode */
+	hdmi_mask_writeb(hdmi, 1, HDMI_FC_DATAUTO0, HDMI_FC_DATAUTO0_VSD_OFFSET,
+			HDMI_FC_DATAUTO0_VSD_MASK);
+}
+
+static void hdmi_config_drm_infoframe(struct dw_hdmi *hdmi,
+				      const struct drm_connector *connector)
+{
+	const struct drm_connector_state *conn_state = connector->state;
+	struct hdmi_drm_infoframe frame;
+	u8 buffer[30];
+	ssize_t err;
+	int i;
+
+	if (!hdmi->plat_data->use_drm_infoframe)
+		return;
+
+	hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_DISABLE,
+		  HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN);
+
+	err = drm_hdmi_infoframe_set_hdr_metadata(&frame, conn_state);
+	if (err < 0)
+		return;
+
+	err = hdmi_drm_infoframe_pack(&frame, buffer, sizeof(buffer));
+	if (err < 0) {
+		dev_err(hdmi->dev, "Failed to pack drm infoframe: %zd\n", err);
+		return;
+	}
+
+	hdmi_writeb(hdmi, frame.version, HDMI_FC_DRM_HB0);
+	hdmi_writeb(hdmi, frame.length, HDMI_FC_DRM_HB1);
+
+	for (i = 0; i < frame.length; i++)
+		hdmi_writeb(hdmi, buffer[4 + i], HDMI_FC_DRM_PB0 + i);
+
+	hdmi_writeb(hdmi, 1, HDMI_FC_DRM_UP);
+	hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_ENABLE,
+		  HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN);
+}
+
+static void hdmi_av_composer(struct dw_hdmi *hdmi,
+			     const struct drm_display_info *display,
+			     const struct drm_display_mode *mode)
+{
+	u8 inv_val, bytes;
+	const struct drm_hdmi_info *hdmi_info = &display->hdmi;
+	struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode;
+	int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len;
+	unsigned int vdisplay, hdisplay;
+
+	vmode->mpixelclock = mode->clock * 1000;
+
+	dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock);
+
+	vmode->mtmdsclock = vmode->mpixelclock;
+
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi_bus_fmt_color_depth(
+				hdmi->hdmi_data.enc_out_bus_format)) {
+		case 16:
+			vmode->mtmdsclock = vmode->mpixelclock * 2;
+			break;
+		case 12:
+			vmode->mtmdsclock = vmode->mpixelclock * 3 / 2;
+			break;
+		case 10:
+			vmode->mtmdsclock = vmode->mpixelclock * 5 / 4;
+			break;
+		}
+	}
+
+	if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format))
+		vmode->mtmdsclock /= 2;
+
+	dev_dbg(hdmi->dev, "final tmdsclock = %d\n", vmode->mtmdsclock);
+
+	/* Set up HDMI_FC_INVIDCONF */
+	inv_val = (hdmi->hdmi_data.hdcp_enable ||
+		   (dw_hdmi_support_scdc(hdmi, display) &&
+		    (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK ||
+		     hdmi_info->scdc.scrambling.low_rates)) ?
+		HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE :
+		HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE);
+
+	inv_val |= mode->flags & DRM_MODE_FLAG_PVSYNC ?
+		HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH :
+		HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_LOW;
+
+	inv_val |= mode->flags & DRM_MODE_FLAG_PHSYNC ?
+		HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_HIGH :
+		HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_LOW;
+
+	inv_val |= (vmode->mdataenablepolarity ?
+		HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_HIGH :
+		HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_LOW);
+
+	if (hdmi->vic == 39)
+		inv_val |= HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH;
+	else
+		inv_val |= mode->flags & DRM_MODE_FLAG_INTERLACE ?
+			HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH :
+			HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_LOW;
+
+	inv_val |= mode->flags & DRM_MODE_FLAG_INTERLACE ?
+		HDMI_FC_INVIDCONF_IN_I_P_INTERLACED :
+		HDMI_FC_INVIDCONF_IN_I_P_PROGRESSIVE;
+
+	inv_val |= hdmi->sink_is_hdmi ?
+		HDMI_FC_INVIDCONF_DVI_MODEZ_HDMI_MODE :
+		HDMI_FC_INVIDCONF_DVI_MODEZ_DVI_MODE;
+
+	hdmi_writeb(hdmi, inv_val, HDMI_FC_INVIDCONF);
+
+	hdisplay = mode->hdisplay;
+	hblank = mode->htotal - mode->hdisplay;
+	h_de_hs = mode->hsync_start - mode->hdisplay;
+	hsync_len = mode->hsync_end - mode->hsync_start;
+
+	/*
+	 * When we're setting a YCbCr420 mode, we need
+	 * to adjust the horizontal timing to suit.
+	 */
+	if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) {
+		hdisplay /= 2;
+		hblank /= 2;
+		h_de_hs /= 2;
+		hsync_len /= 2;
+	}
+
+	vdisplay = mode->vdisplay;
+	vblank = mode->vtotal - mode->vdisplay;
+	v_de_vs = mode->vsync_start - mode->vdisplay;
+	vsync_len = mode->vsync_end - mode->vsync_start;
+
+	/*
+	 * When we're setting an interlaced mode, we need
+	 * to adjust the vertical timing to suit.
+	 */
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		vdisplay /= 2;
+		vblank /= 2;
+		v_de_vs /= 2;
+		vsync_len /= 2;
+	}
+
+	/* Scrambling Control */
+	if (dw_hdmi_support_scdc(hdmi, display)) {
+		if (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK ||
+		    hdmi_info->scdc.scrambling.low_rates) {
+			/*
+			 * HDMI2.0 Specifies the following procedure:
+			 * After the Source Device has determined that
+			 * SCDC_Present is set (=1), the Source Device should
+			 * write the accurate Version of the Source Device
+			 * to the Source Version field in the SCDCS.
+			 * Source Devices compliant shall set the
+			 * Source Version = 1.
+			 */
+			drm_scdc_readb(hdmi->ddc, SCDC_SINK_VERSION,
+				       &bytes);
+			drm_scdc_writeb(hdmi->ddc, SCDC_SOURCE_VERSION,
+				min_t(u8, bytes, SCDC_MIN_SOURCE_VERSION));
+
+			/* Enabled Scrambling in the Sink */
+			drm_scdc_set_scrambling(hdmi->curr_conn, 1);
+
+			/*
+			 * To activate the scrambler feature, you must ensure
+			 * that the quasi-static configuration bit
+			 * fc_invidconf.HDCP_keepout is set at configuration
+			 * time, before the required mc_swrstzreq.tmdsswrst_req
+			 * reset request is issued.
+			 */
+			hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ,
+				    HDMI_MC_SWRSTZ);
+			hdmi_writeb(hdmi, 1, HDMI_FC_SCRAMBLER_CTRL);
+		} else {
+			hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL);
+			hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ,
+				    HDMI_MC_SWRSTZ);
+			drm_scdc_set_scrambling(hdmi->curr_conn, 0);
+		}
+	} else {
+		hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL);
+	}
+
+	/* Set up horizontal active pixel width */
+	hdmi_writeb(hdmi, hdisplay >> 8, HDMI_FC_INHACTV1);
+	hdmi_writeb(hdmi, hdisplay, HDMI_FC_INHACTV0);
+
+	/* Set up vertical active lines */
+	hdmi_writeb(hdmi, vdisplay >> 8, HDMI_FC_INVACTV1);
+	hdmi_writeb(hdmi, vdisplay, HDMI_FC_INVACTV0);
+
+	/* Set up horizontal blanking pixel region width */
+	hdmi_writeb(hdmi, hblank >> 8, HDMI_FC_INHBLANK1);
+	hdmi_writeb(hdmi, hblank, HDMI_FC_INHBLANK0);
+
+	/* Set up vertical blanking pixel region width */
+	hdmi_writeb(hdmi, vblank, HDMI_FC_INVBLANK);
+
+	/* Set up HSYNC active edge delay width (in pixel clks) */
+	hdmi_writeb(hdmi, h_de_hs >> 8, HDMI_FC_HSYNCINDELAY1);
+	hdmi_writeb(hdmi, h_de_hs, HDMI_FC_HSYNCINDELAY0);
+
+	/* Set up VSYNC active edge delay (in lines) */
+	hdmi_writeb(hdmi, v_de_vs, HDMI_FC_VSYNCINDELAY);
+
+	/* Set up HSYNC active pulse width (in pixel clks) */
+	hdmi_writeb(hdmi, hsync_len >> 8, HDMI_FC_HSYNCINWIDTH1);
+	hdmi_writeb(hdmi, hsync_len, HDMI_FC_HSYNCINWIDTH0);
+
+	/* Set up VSYNC active edge delay (in lines) */
+	hdmi_writeb(hdmi, vsync_len, HDMI_FC_VSYNCINWIDTH);
+}
+
+/* HDMI Initialization Step B.4 */
+static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi)
+{
+	/* control period minimum duration */
+	hdmi_writeb(hdmi, 12, HDMI_FC_CTRLDUR);
+	hdmi_writeb(hdmi, 32, HDMI_FC_EXCTRLDUR);
+	hdmi_writeb(hdmi, 1, HDMI_FC_EXCTRLSPAC);
+
+	/* Set to fill TMDS data channels */
+	hdmi_writeb(hdmi, 0x0B, HDMI_FC_CH0PREAM);
+	hdmi_writeb(hdmi, 0x16, HDMI_FC_CH1PREAM);
+	hdmi_writeb(hdmi, 0x21, HDMI_FC_CH2PREAM);
+
+	/* Enable pixel clock and tmds data path */
+	hdmi->mc_clkdis |= HDMI_MC_CLKDIS_HDCPCLK_DISABLE |
+			   HDMI_MC_CLKDIS_CSCCLK_DISABLE |
+			   HDMI_MC_CLKDIS_AUDCLK_DISABLE |
+			   HDMI_MC_CLKDIS_PREPCLK_DISABLE |
+			   HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_PIXELCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+	/* Enable csc path */
+	if (is_csc_needed(hdmi)) {
+		hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE;
+		hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+		hdmi_writeb(hdmi, HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_IN_PATH,
+			    HDMI_MC_FLOWCTRL);
+	} else {
+		hdmi->mc_clkdis |= HDMI_MC_CLKDIS_CSCCLK_DISABLE;
+		hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+		hdmi_writeb(hdmi, HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_BYPASS,
+			    HDMI_MC_FLOWCTRL);
+	}
+}
+
+/* Workaround to clear the overflow condition */
+static void dw_hdmi_clear_overflow(struct dw_hdmi *hdmi)
+{
+	unsigned int count;
+	unsigned int i;
+	u8 val;
+
+	/*
+	 * Under some circumstances the Frame Composer arithmetic unit can miss
+	 * an FC register write due to being busy processing the previous one.
+	 * The issue can be worked around by issuing a TMDS software reset and
+	 * then write one of the FC registers several times.
+	 *
+	 * The number of iterations matters and depends on the HDMI TX revision
+	 * (and possibly on the platform).
+	 * 4 iterations for i.MX6Q(v1.30a) and 1 iteration for others.
+	 * i.MX6DL (v1.31a), Allwinner SoCs (v1.32a), Rockchip RK3288 SoC (v2.00a),
+	 * Amlogic Meson GX SoCs (v2.01a), RK3328/RK3399 SoCs (v2.11a)
+	 * and i.MX8MPlus (v2.13a) have been identified as needing the workaround
+	 * with a single iteration.
+	 */
+
+	switch (hdmi->version) {
+	case 0x130a:
+		count = 4;
+		break;
+	default:
+		count = 1;
+		break;
+	}
+
+	/* TMDS software reset */
+	hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ, HDMI_MC_SWRSTZ);
+
+	val = hdmi_readb(hdmi, HDMI_FC_INVIDCONF);
+	for (i = 0; i < count; i++)
+		hdmi_writeb(hdmi, val, HDMI_FC_INVIDCONF);
+}
+
+static void hdmi_disable_overflow_interrupts(struct dw_hdmi *hdmi)
+{
+	hdmi_writeb(hdmi, HDMI_IH_MUTE_FC_STAT2_OVERFLOW_MASK,
+		    HDMI_IH_MUTE_FC_STAT2);
+}
+
+static int dw_hdmi_setup(struct dw_hdmi *hdmi,
+			 const struct drm_connector *connector,
+			 const struct drm_display_mode *mode)
+{
+	void *data = hdmi->plat_data->phy_data;
+	int ret;
+
+	hdmi_disable_overflow_interrupts(hdmi);
+
+	hdmi->vic = drm_match_cea_mode(mode);
+
+	if (!hdmi->vic) {
+		dev_dbg(hdmi->dev, "Non-CEA mode used in HDMI\n");
+	} else {
+		dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic);
+	}
+
+	dev_dbg(hdmi->dev, "mode clock:%d, hdisplay:%d, htotal:%d, vdisplay:%d, vtotal:%d\n",
+			mode->clock, mode->hdisplay, mode->htotal, mode->vdisplay, mode->vtotal);
+
+	if (hdmi->plat_data->get_enc_out_encoding)
+		hdmi->hdmi_data.enc_out_encoding =
+			hdmi->plat_data->get_enc_out_encoding(data);
+	else if ((hdmi->vic == 6) || (hdmi->vic == 7) || (hdmi->vic == 21) ||
+		 (hdmi->vic == 22) || (hdmi->vic == 2) || (hdmi->vic == 3) ||
+		 (hdmi->vic == 17) || (hdmi->vic == 18))
+		hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601;
+	else
+		hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709;
+
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK) {
+		hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1;
+		hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1;
+	} else {
+		hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0;
+		hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0;
+	}
+
+	/* TOFIX: Get input format from plat data or fallback to RGB888 */
+	if (hdmi->plat_data->get_input_bus_format)
+		hdmi->hdmi_data.enc_in_bus_format =
+			hdmi->plat_data->get_input_bus_format(data);
+	else if (hdmi->hdmi_data.enc_in_bus_format == MEDIA_BUS_FMT_FIXED)
+		hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	else
+		hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	/* TOFIX: Get input encoding from plat data or fallback to none */
+	if (hdmi->plat_data->get_enc_in_encoding)
+		hdmi->hdmi_data.enc_in_encoding =
+			hdmi->plat_data->get_enc_in_encoding(data);
+	else
+		hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT;
+
+	/* TOFIX: Default to RGB888 output format */
+	if (hdmi->plat_data->get_output_bus_format)
+		hdmi->hdmi_data.enc_out_bus_format =
+			hdmi->plat_data->get_output_bus_format(data);
+	else if (hdmi->hdmi_data.enc_out_bus_format == MEDIA_BUS_FMT_FIXED)
+		hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	else
+		hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	hdmi->hdmi_data.rgb_limited_range = hdmi->sink_is_hdmi &&
+		drm_default_rgb_quant_range(mode) ==
+		HDMI_QUANTIZATION_RANGE_LIMITED;
+
+	hdmi->hdmi_data.pix_repet_factor = 0;
+	hdmi->hdmi_data.hdcp_enable = 0;
+	hdmi->hdmi_data.video_mode.mdataenablepolarity = true;
+
+	/* HDMI Initialization Step B.1 */
+	hdmi_av_composer(hdmi, &connector->display_info, mode);
+
+	/* HDMI Initializateion Step B.2 */
+	ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data,
+				  &connector->display_info,
+				  &hdmi->previous_mode);
+	if (ret)
+		return ret;
+	hdmi->phy.enabled = true;
+
+	/* HDMI Initialization Step B.3 */
+	dw_hdmi_enable_video_path(hdmi);
+
+	if (hdmi->sink_has_audio) {
+		dev_dbg(hdmi->dev, "sink has audio support\n");
+
+		/* HDMI Initialization Step E - Configure audio */
+		hdmi_clk_regenerator_update_pixel_clock(hdmi);
+		hdmi_enable_audio_clk(hdmi, hdmi->audio_enable);
+	}
+
+	/* not for DVI mode */
+	if (hdmi->sink_is_hdmi) {
+		dev_dbg(hdmi->dev, "%s HDMI mode\n", __func__);
+
+		/* HDMI Initialization Step F - Configure AVI InfoFrame */
+		hdmi_config_AVI(hdmi, connector, mode);
+		hdmi_config_vendor_specific_infoframe(hdmi, connector, mode);
+		hdmi_config_drm_infoframe(hdmi, connector);
+	} else {
+		dev_dbg(hdmi->dev, "%s DVI mode\n", __func__);
+	}
+
+	hdmi_video_packetize(hdmi);
+	hdmi_video_csc(hdmi);
+	hdmi_video_sample(hdmi);
+	if (!hpd_flag[hdmi->numa_id]) {
+		hdmi_tx_hdcp_config(hdmi, mode);
+	}
+	dw_hdmi_clear_overflow(hdmi);
+
+	return 0;
+}
+
+static void initialize_hdmi_ih_mutes(struct dw_hdmi *hdmi)
+{
+	u8 ih_mute;
+
+	/*
+	 * Boot up defaults are:
+	 * HDMI_IH_MUTE   = 0x03 (disabled)
+	 * HDMI_IH_MUTE_* = 0x00 (enabled)
+	 *
+	 * Disable top level interrupt bits in HDMI block
+	 */
+	ih_mute = hdmi_readb(hdmi, HDMI_IH_MUTE) |
+		  HDMI_IH_MUTE_MUTE_WAKEUP_INTERRUPT |
+		  HDMI_IH_MUTE_MUTE_ALL_INTERRUPT;
+
+	hdmi_writeb(hdmi, ih_mute, HDMI_IH_MUTE);
+
+	/* by default mask all interrupts */
+	hdmi_writeb(hdmi, 0xff, HDMI_VP_MASK);
+	hdmi_writeb(hdmi, 0xff, HDMI_FC_MASK0);
+	hdmi_writeb(hdmi, 0xff, HDMI_FC_MASK1);
+	hdmi_writeb(hdmi, 0xff, HDMI_FC_MASK2);
+	hdmi_writeb(hdmi, 0xff, HDMI_PHY_MASK0);
+	hdmi_writeb(hdmi, 0xff, HDMI_PHY_I2CM_INT_ADDR);
+	hdmi_writeb(hdmi, 0xff, HDMI_PHY_I2CM_CTLINT_ADDR);
+	hdmi_writeb(hdmi, 0xff, HDMI_AUD_INT);
+	hdmi_writeb(hdmi, 0xff, HDMI_AUD_SPDIFINT);
+	hdmi_writeb(hdmi, 0xff, HDMI_AUD_HBR_MASK);
+	hdmi_writeb(hdmi, 0xff, HDMI_GP_MASK);
+	hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK);
+	hdmi_writeb(hdmi, 0xff, HDMI_I2CM_INT);
+	hdmi_writeb(hdmi, 0xff, HDMI_I2CM_CTLINT);
+
+	/* Disable interrupts in the IH_MUTE_* registers */
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_FC_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_FC_STAT1);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_FC_STAT2);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_AS_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_PHY_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_I2CM_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_CEC_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_VP_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_I2CMPHY_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_AHBDMAAUD_STAT0);
+
+	/* Enable top level interrupt bits in HDMI block */
+	ih_mute &= ~(HDMI_IH_MUTE_MUTE_WAKEUP_INTERRUPT |
+		     HDMI_IH_MUTE_MUTE_ALL_INTERRUPT);
+	hdmi_writeb(hdmi, ih_mute, HDMI_IH_MUTE);
+}
+
+static void dw_hdmi_poweron(struct dw_hdmi *hdmi)
+{
+	hdmi->bridge_is_on = true;
+
+	/*
+	 * The curr_conn field is guaranteed to be valid here, as this function
+	 * is only be called when !hdmi->disabled.
+	 */
+	dev_dbg(hdmi->dev, "%s", __func__);
+	dw_hdmi_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode);
+}
+
+static void dw_hdmi_poweroff(struct dw_hdmi *hdmi)
+{
+	dev_dbg(hdmi->dev, "%s", __func__);
+	if (hdmi->phy.enabled) {
+		hdmi->phy.ops->disable(hdmi, hdmi->phy.data);
+		hdmi->phy.enabled = false;
+	}
+
+	hdmi->bridge_is_on = false;
+}
+
+static void dw_hdmi_update_power(struct dw_hdmi *hdmi)
+{
+	int force = hdmi->force;
+
+	if (hdmi->disabled) {
+		force = DRM_FORCE_OFF;
+	} else if (force == DRM_FORCE_UNSPECIFIED) {
+		if (hdmi->rxsense)
+			force = DRM_FORCE_ON;
+		else
+			force = DRM_FORCE_OFF;
+	}
+
+	if (force == DRM_FORCE_OFF) {
+		if (hdmi->bridge_is_on)
+			dw_hdmi_poweroff(hdmi);
+	} else {
+		if (!hdmi->bridge_is_on) {
+			struct edid *edid;
+			edid = dw_hdmi_get_edid(hdmi, &hdmi->connector);
+			dw_hdmi_poweron(hdmi);
+			kfree(edid);
+		}
+	}
+}
+
+/*
+ * Adjust the detection of RXSENSE according to whether we have a forced
+ * connection mode enabled, or whether we have been disabled.  There is
+ * no point processing RXSENSE interrupts if we have a forced connection
+ * state, or DRM has us disabled.
+ *
+ * We also disable rxsense interrupts when we think we're disconnected
+ * to avoid floating TDMS signals giving false rxsense interrupts.
+ *
+ * Note: we still need to listen for HPD interrupts even when DRM has us
+ * disabled so that we can detect a connect event.
+ */
+static void dw_hdmi_update_phy_mask(struct dw_hdmi *hdmi)
+{
+	if (hdmi->phy.ops->update_hpd)
+		hdmi->phy.ops->update_hpd(hdmi, hdmi->phy.data,
+					  hdmi->force, hdmi->disabled,
+					  hdmi->rxsense);
+}
+
+static enum drm_connector_status dw_hdmi_detect(struct dw_hdmi *hdmi)
+{
+	enum drm_connector_status result;
+
+	result = hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data);
+	hdmi->last_connector_result = result;
+
+	dev_dbg(hdmi->dev, "detect connector status:%d\n", result);
+
+	return result;
+}
+
+static struct edid *dw_hdmi_get_edid(struct dw_hdmi *hdmi,
+				     struct drm_connector *connector)
+{
+	struct edid *edid;
+
+	if (!hdmi->ddc)
+		return NULL;
+
+	edid = drm_get_edid(connector, hdmi->ddc);
+	if (!edid) {
+		dev_err(hdmi->dev, "failed to get edid\n");
+		return NULL;
+	}
+
+	dev_info(hdmi->dev, "got edid: width[%d] x height[%d]\n",
+		 edid->width_cm, edid->height_cm);
+
+	hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid);
+	hdmi->sink_has_audio = drm_detect_monitor_audio(edid);
+
+	return edid;
+}
+
+/* -----------------------------------------------------------------------------
+ * DRM Connector Operations
+ */
+
+static enum drm_connector_status
+dw_hdmi_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+					     connector);
+	return dw_hdmi_detect(hdmi);
+}
+
+static int dw_hdmi_connector_get_modes(struct drm_connector *connector)
+{
+	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+					     connector);
+	struct edid *edid;
+	int ret;
+
+	edid = dw_hdmi_get_edid(hdmi, connector);
+	if (!edid)
+		return 0;
+
+	drm_connector_update_edid_property(connector, edid);
+	cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid);
+	ret = drm_add_edid_modes(connector, edid);
+	kfree(edid);
+
+	return ret;
+}
+
+static int dw_hdmi_connector_atomic_check(struct drm_connector *connector,
+					  struct drm_atomic_state *state)
+{
+	struct drm_connector_state *old_state =
+		drm_atomic_get_old_connector_state(state, connector);
+	struct drm_connector_state *new_state =
+		drm_atomic_get_new_connector_state(state, connector);
+	struct drm_crtc *crtc = new_state->crtc;
+	struct drm_crtc_state *crtc_state;
+
+	if (!crtc)
+		return 0;
+
+	if (!drm_connector_atomic_hdr_metadata_equal(old_state, new_state)) {
+		crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+
+		crtc_state->mode_changed = true;
+	}
+
+	return 0;
+}
+
+static void dw_hdmi_connector_force(struct drm_connector *connector)
+{
+	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+					     connector);
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->force = connector->force;
+	dw_hdmi_update_power(hdmi);
+	dw_hdmi_update_phy_mask(hdmi);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static int dw_hdmi_atomic_connector_set_property(
+	struct drm_connector *connector, struct drm_connector_state *state,
+	struct drm_property *property, uint64_t val)
+{
+	struct dw_hdmi *hdmi =
+		container_of(connector, struct dw_hdmi, connector);
+	const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (ops && ops->set_property)
+		return ops->set_property(connector, state, property, val,
+					 hdmi->plat_data->phy_data);
+	else
+		return -EINVAL;
+}
+
+static int
+dw_hdmi_atomic_connector_get_property(struct drm_connector *connector,
+				      const struct drm_connector_state *state,
+				      struct drm_property *property,
+				      uint64_t *val)
+{
+	struct dw_hdmi *hdmi =
+		container_of(connector, struct dw_hdmi, connector);
+	const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (ops && ops->get_property)
+		return ops->get_property(connector, state, property, val,
+					 hdmi->plat_data->phy_data);
+	else
+		return -EINVAL;
+}
+
+static int dw_hdmi_connector_set_property(struct drm_connector *connector,
+					  struct drm_property *property,
+					  uint64_t val)
+{
+	return dw_hdmi_atomic_connector_set_property(connector, NULL, property,
+						     val);
+}
+
+static const struct drm_connector_funcs dw_hdmi_connector_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.detect = dw_hdmi_connector_detect,
+	.destroy = drm_connector_cleanup,
+	.force = dw_hdmi_connector_force,
+	.reset = drm_atomic_helper_connector_reset,
+	.set_property = dw_hdmi_connector_set_property,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.atomic_set_property = dw_hdmi_atomic_connector_set_property,
+	.atomic_get_property = dw_hdmi_atomic_connector_get_property,
+};
+
+static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = {
+	.get_modes = dw_hdmi_connector_get_modes,
+	.atomic_check = dw_hdmi_connector_atomic_check,
+};
+
+static void dw_hdmi_attatch_properties(struct dw_hdmi *hdmi)
+{
+	unsigned int color = MEDIA_BUS_FMT_RGB888_1X24;
+	int video_mapping, colorspace;
+
+	enum drm_connector_status connect_status =
+		hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data);
+		const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (connect_status == connector_status_connected) {
+		video_mapping = (hdmi_readb(hdmi, HDMI_TX_INVID0) &
+				 HDMI_TX_INVID0_VIDEO_MAPPING_MASK);
+		colorspace = (hdmi_readb(hdmi, HDMI_FC_AVICONF0) &
+			      HDMI_FC_AVICONF0_PIX_FMT_MASK);
+		switch (video_mapping) {
+		case 0x01:
+			color = MEDIA_BUS_FMT_RGB888_1X24;
+			break;
+		case 0x03:
+			color = MEDIA_BUS_FMT_RGB101010_1X30;
+			break;
+		case 0x09:
+			if (colorspace == HDMI_COLORSPACE_YUV420)
+				color = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+			else if (colorspace == HDMI_COLORSPACE_YUV422)
+				color = MEDIA_BUS_FMT_UYVY8_1X16;
+			else
+				color = MEDIA_BUS_FMT_YUV8_1X24;
+			break;
+		case 0x0b:
+			if (colorspace == HDMI_COLORSPACE_YUV420)
+				color = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+			else if (colorspace == HDMI_COLORSPACE_YUV422)
+				color = MEDIA_BUS_FMT_UYVY10_1X20;
+			else
+				color = MEDIA_BUS_FMT_YUV10_1X30;
+			break;
+		case 0x14:
+			color = MEDIA_BUS_FMT_UYVY10_1X20;
+			break;
+		case 0x16:
+			color = MEDIA_BUS_FMT_UYVY8_1X16;
+			break;
+		default:
+			color = MEDIA_BUS_FMT_RGB888_1X24;
+			dev_err(hdmi->dev, "unexpected mapping: 0x%x\n",
+				video_mapping);
+		}
+
+		hdmi->hdmi_data.enc_in_bus_format = color;
+		hdmi->hdmi_data.enc_out_bus_format = color;
+		/*
+         * input format will be set as yuv444 when output
+         * format is yuv420
+         */
+		if (color == MEDIA_BUS_FMT_UYVY10_1X20)
+			hdmi->hdmi_data.enc_in_bus_format =
+				MEDIA_BUS_FMT_YUV10_1X30;
+		else if (color == MEDIA_BUS_FMT_UYVY8_1X16)
+			hdmi->hdmi_data.enc_in_bus_format =
+				MEDIA_BUS_FMT_YUV8_1X24;
+	}
+
+	if (ops && ops->attatch_properties)
+		return ops->attatch_properties(&hdmi->connector, color,
+					       hdmi->version,
+					       hdmi->plat_data->phy_data);
+}
+
+static void dw_hdmi_destroy_properties(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (ops && ops->destroy_properties)
+		return ops->destroy_properties(&hdmi->connector,
+					       hdmi->plat_data->phy_data);
+}
+
+static int dw_hdmi_connector_create(struct dw_hdmi *hdmi)
+{
+	struct drm_connector *connector = &hdmi->connector;
+	struct cec_connector_info conn_info;
+	struct cec_notifier *notifier;
+
+	if (hdmi->version >= 0x200a)
+		connector->ycbcr_420_allowed =
+			hdmi->plat_data->ycbcr_420_allowed;
+	else
+		connector->ycbcr_420_allowed = false;
+
+	connector->interlace_allowed = 1;
+	connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+	drm_connector_helper_add(connector, &dw_hdmi_connector_helper_funcs);
+
+	drm_connector_init_with_ddc(hdmi->bridge.dev, connector,
+				    &dw_hdmi_connector_funcs,
+				    DRM_MODE_CONNECTOR_HDMIA,
+				    hdmi->ddc);
+
+	/*
+	 * drm_connector_attach_max_bpc_property() requires the
+	 * connector to have a state.
+	 */
+	drm_atomic_helper_connector_reset(connector);
+
+	drm_connector_attach_max_bpc_property(connector, 8, 16);
+
+	if (hdmi->version >= 0x200a && hdmi->plat_data->use_drm_infoframe)
+		drm_connector_attach_hdr_output_metadata_property(connector);
+
+	drm_connector_attach_encoder(connector, hdmi->bridge.encoder);
+
+	dw_hdmi_attatch_properties(hdmi);
+
+	cec_fill_conn_info_from_drm(&conn_info, connector);
+
+	notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info);
+	if (!notifier)
+		return -ENOMEM;
+
+	mutex_lock(&hdmi->cec_notifier_mutex);
+	hdmi->cec_notifier = notifier;
+	mutex_unlock(&hdmi->cec_notifier_mutex);
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * DRM Bridge Operations
+ */
+
+/*
+ * Possible output formats :
+ * - MEDIA_BUS_FMT_UYYVYY16_0_5X48,
+ * - MEDIA_BUS_FMT_UYYVYY12_0_5X36,
+ * - MEDIA_BUS_FMT_UYYVYY10_0_5X30,
+ * - MEDIA_BUS_FMT_UYYVYY8_0_5X24,
+ * - MEDIA_BUS_FMT_YUV16_1X48,
+ * - MEDIA_BUS_FMT_RGB161616_1X48,
+ * - MEDIA_BUS_FMT_UYVY12_1X24,
+ * - MEDIA_BUS_FMT_YUV12_1X36,
+ * - MEDIA_BUS_FMT_RGB121212_1X36,
+ * - MEDIA_BUS_FMT_UYVY10_1X20,
+ * - MEDIA_BUS_FMT_YUV10_1X30,
+ * - MEDIA_BUS_FMT_RGB101010_1X30,
+ * - MEDIA_BUS_FMT_UYVY8_1X16,
+ * - MEDIA_BUS_FMT_YUV8_1X24,
+ * - MEDIA_BUS_FMT_RGB888_1X24,
+ */
+
+/* Can return a maximum of 11 possible output formats for a mode/connector */
+#define MAX_OUTPUT_SEL_FORMATS	11
+
+static u32 *dw_hdmi_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					unsigned int *num_output_fmts)
+{
+	struct drm_connector *conn = conn_state->connector;
+	struct drm_display_info *info = &conn->display_info;
+	struct drm_display_mode *mode = &crtc_state->mode;
+	u8 max_bpc = conn_state->max_requested_bpc;
+	bool is_hdmi2_sink = info->hdmi.scdc.supported ||
+			     (info->color_formats & DRM_COLOR_FORMAT_YCBCR420);
+	u32 *output_fmts;
+	unsigned int i = 0;
+
+	*num_output_fmts = 0;
+
+	output_fmts = kcalloc(MAX_OUTPUT_SEL_FORMATS, sizeof(*output_fmts),
+			      GFP_KERNEL);
+	if (!output_fmts)
+		return NULL;
+
+	/* If dw-hdmi is the first or only bridge, avoid negociating with ourselves */
+	if (list_is_singular(&bridge->encoder->bridge_chain) ||
+	    list_is_first(&bridge->chain_node, &bridge->encoder->bridge_chain)) {
+		*num_output_fmts = 1;
+		output_fmts[0] = MEDIA_BUS_FMT_FIXED;
+
+		return output_fmts;
+	}
+
+	/*
+	 * If the current mode enforces 4:2:0, force the output but format
+	 * to 4:2:0 and do not add the YUV422/444/RGB formats
+	 */
+	if (conn->ycbcr_420_allowed &&
+	    (drm_mode_is_420_only(info, mode) ||
+	     (is_hdmi2_sink && drm_mode_is_420_also(info, mode)))) {
+
+		/* Order bus formats from 16bit to 8bit if supported */
+		if (max_bpc >= 16 && info->bpc == 16 &&
+		    (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48))
+			output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY16_0_5X48;
+
+		if (max_bpc >= 12 && info->bpc >= 12 &&
+		    (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36))
+			output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY12_0_5X36;
+
+		if (max_bpc >= 10 && info->bpc >= 10 &&
+		    (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30))
+			output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+
+		/* Default 8bit fallback */
+		output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+
+		if (drm_mode_is_420_only(info, mode)) {
+			*num_output_fmts = i;
+			return output_fmts;
+		}
+	}
+
+	/*
+	 * Order bus formats from 16bit to 8bit and from YUV422 to RGB
+	 * if supported. In any case the default RGB888 format is added
+	 */
+
+	/* Default 8bit RGB fallback */
+	output_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+
+	if (max_bpc >= 16 && info->bpc == 16) {
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			output_fmts[i++] = MEDIA_BUS_FMT_YUV16_1X48;
+
+		output_fmts[i++] = MEDIA_BUS_FMT_RGB161616_1X48;
+	}
+
+	if (max_bpc >= 12 && info->bpc >= 12) {
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			output_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			output_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+
+		output_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+	}
+
+	if (max_bpc >= 10 && info->bpc >= 10) {
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			output_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			output_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+
+		output_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+	}
+
+	if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+		output_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+
+	if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+		output_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+
+	*num_output_fmts = i;
+
+	return output_fmts;
+}
+
+/*
+ * Possible input formats :
+ * - MEDIA_BUS_FMT_RGB888_1X24
+ * - MEDIA_BUS_FMT_YUV8_1X24
+ * - MEDIA_BUS_FMT_UYVY8_1X16
+ * - MEDIA_BUS_FMT_UYYVYY8_0_5X24
+ * - MEDIA_BUS_FMT_RGB101010_1X30
+ * - MEDIA_BUS_FMT_YUV10_1X30
+ * - MEDIA_BUS_FMT_UYVY10_1X20
+ * - MEDIA_BUS_FMT_UYYVYY10_0_5X30
+ * - MEDIA_BUS_FMT_RGB121212_1X36
+ * - MEDIA_BUS_FMT_YUV12_1X36
+ * - MEDIA_BUS_FMT_UYVY12_1X24
+ * - MEDIA_BUS_FMT_UYYVYY12_0_5X36
+ * - MEDIA_BUS_FMT_RGB161616_1X48
+ * - MEDIA_BUS_FMT_YUV16_1X48
+ * - MEDIA_BUS_FMT_UYYVYY16_0_5X48
+ */
+
+/* Can return a maximum of 3 possible input formats for an output format */
+#define MAX_INPUT_SEL_FORMATS	3
+
+static u32 *dw_hdmi_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					u32 output_fmt,
+					unsigned int *num_input_fmts)
+{
+	u32 *input_fmts;
+	unsigned int i = 0;
+
+	*num_input_fmts = 0;
+
+	input_fmts = kcalloc(MAX_INPUT_SEL_FORMATS, sizeof(*input_fmts),
+			     GFP_KERNEL);
+	if (!input_fmts)
+		return NULL;
+
+	switch (output_fmt) {
+	/* If MEDIA_BUS_FMT_FIXED is tested, return default bus format */
+	case MEDIA_BUS_FMT_FIXED:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	/* 8bit */
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		break;
+	case MEDIA_BUS_FMT_YUV8_1X24:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+
+	/* 10bit */
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		break;
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		break;
+
+	/* 12bit */
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		break;
+	case MEDIA_BUS_FMT_YUV12_1X36:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		break;
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		break;
+
+	/* 16bit */
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB161616_1X48;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV16_1X48;
+		break;
+	case MEDIA_BUS_FMT_YUV16_1X48:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV16_1X48;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB161616_1X48;
+		break;
+
+	/*YUV 4:2:0 */
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		input_fmts[i++] = output_fmt;
+		break;
+	}
+
+	*num_input_fmts = i;
+
+	if (*num_input_fmts == 0) {
+		kfree(input_fmts);
+		input_fmts = NULL;
+	}
+
+	return input_fmts;
+}
+
+static int dw_hdmi_bridge_atomic_check(struct drm_bridge *bridge,
+				       struct drm_bridge_state *bridge_state,
+				       struct drm_crtc_state *crtc_state,
+				       struct drm_connector_state *conn_state)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	hdmi->hdmi_data.enc_out_bus_format =
+			bridge_state->output_bus_cfg.format;
+
+	hdmi->hdmi_data.enc_in_bus_format =
+			bridge_state->input_bus_cfg.format;
+
+	dev_dbg(hdmi->dev, "input format 0x%04x, output format 0x%04x\n",
+		bridge_state->input_bus_cfg.format,
+		bridge_state->output_bus_cfg.format);
+
+	return 0;
+}
+
+static int dw_hdmi_bridge_attach(struct drm_bridge *bridge,
+				 enum drm_bridge_attach_flags flags)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
+		return drm_bridge_attach(bridge->encoder, hdmi->next_bridge,
+					 bridge, flags);
+
+	return dw_hdmi_connector_create(hdmi);
+}
+
+static void dw_hdmi_bridge_detach(struct drm_bridge *bridge)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	mutex_lock(&hdmi->cec_notifier_mutex);
+	cec_notifier_conn_unregister(hdmi->cec_notifier);
+	hdmi->cec_notifier = NULL;
+	mutex_unlock(&hdmi->cec_notifier_mutex);
+}
+
+static enum drm_mode_status
+dw_hdmi_bridge_mode_valid(struct drm_bridge *bridge,
+			  const struct drm_display_info *info,
+			  const struct drm_display_mode *mode)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+	enum drm_mode_status mode_status = MODE_OK;
+
+	/* We don't support double-clocked modes */
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+		return MODE_BAD;
+
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
+
+	if (pdata->mode_valid)
+		mode_status = pdata->mode_valid(hdmi, pdata->priv_data, info,
+						mode);
+
+	return mode_status;
+}
+
+static void dw_hdmi_bridge_mode_set(struct drm_bridge *bridge,
+				    const struct drm_display_mode *orig_mode,
+				    const struct drm_display_mode *mode)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	mutex_lock(&hdmi->mutex);
+
+	/* Store the display mode for plugin/DKMS poweron events */
+	drm_mode_copy(&hdmi->previous_mode, mode);
+
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
+					  struct drm_bridge_state *old_state)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->disabled = true;
+	hdmi->curr_conn = NULL;
+	dw_hdmi_update_power(hdmi);
+	dw_hdmi_update_phy_mask(hdmi);
+	handle_plugged_change(hdmi, false);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
+					 struct drm_bridge_state *old_state)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+	struct drm_atomic_state *state = old_state->base.state;
+	struct drm_connector *connector;
+
+	connector = drm_atomic_get_new_connector_for_encoder(state,
+							     bridge->encoder);
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->disabled = false;
+	hdmi->curr_conn = connector;
+	dw_hdmi_update_power(hdmi);
+	dw_hdmi_update_phy_mask(hdmi);
+	handle_plugged_change(hdmi, true);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static enum drm_connector_status dw_hdmi_bridge_detect(struct drm_bridge *bridge)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	return dw_hdmi_detect(hdmi);
+}
+
+static struct edid *dw_hdmi_bridge_get_edid(struct drm_bridge *bridge,
+					    struct drm_connector *connector)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	return dw_hdmi_get_edid(hdmi, connector);
+}
+
+static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.attach = dw_hdmi_bridge_attach,
+	.detach = dw_hdmi_bridge_detach,
+	.atomic_check = dw_hdmi_bridge_atomic_check,
+	.atomic_get_output_bus_fmts = dw_hdmi_bridge_atomic_get_output_bus_fmts,
+	.atomic_get_input_bus_fmts = dw_hdmi_bridge_atomic_get_input_bus_fmts,
+	.atomic_enable = dw_hdmi_bridge_atomic_enable,
+	.atomic_disable = dw_hdmi_bridge_atomic_disable,
+	.mode_set = dw_hdmi_bridge_mode_set,
+	.mode_valid = dw_hdmi_bridge_mode_valid,
+	.detect = dw_hdmi_bridge_detect,
+	.get_edid = dw_hdmi_bridge_get_edid,
+};
+
+/* -----------------------------------------------------------------------------
+ * IRQ Handling
+ */
+
+static irqreturn_t dw_hdmi_i2c_irq(struct dw_hdmi *hdmi)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	unsigned int stat;
+
+	stat = hdmi_readb(hdmi, HDMI_IH_I2CM_STAT0);
+	if (!stat)
+		return IRQ_NONE;
+
+	hdmi_writeb(hdmi, stat, HDMI_IH_I2CM_STAT0);
+
+	i2c->stat = stat;
+
+	complete(&i2c->cmp);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id)
+{
+	struct dw_hdmi *hdmi = dev_id;
+	u8 intr_stat, hdcp_stat;
+	irqreturn_t ret = IRQ_NONE;
+
+	if (hdmi->i2c)
+		ret = dw_hdmi_i2c_irq(hdmi);
+	if (ret == IRQ_HANDLED)
+		return ret;
+
+	intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
+	hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT);
+	if (intr_stat | hdcp_stat) {
+		if (intr_stat) {
+			dev_info(hdmi->dev, "intr status %#x\n", intr_stat);
+			hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_PHY_STAT0);
+		}
+		if (hdcp_stat) {
+			dev_info(hdmi->dev, "HDCP status %#x\n", hdcp_stat);
+			hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK);
+		}
+		return IRQ_WAKE_THREAD;
+	}
+
+	return ret;
+}
+
+void dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense)
+{
+	mutex_lock(&hdmi->mutex);
+
+	if (!hdmi->force) {
+		/*
+		 * If the RX sense status indicates we're disconnected,
+		 * clear the software rxsense status.
+		 */
+		if (!rx_sense)
+			hdmi->rxsense = false;
+
+		/*
+		 * Only set the software rxsense status when both
+		 * rxsense and hpd indicates we're connected.
+		 * This avoids what seems to be bad behaviour in
+		 * at least iMX6S versions of the phy.
+		 */
+		if (hpd)
+			hdmi->rxsense = true;
+
+		dw_hdmi_update_power(hdmi);
+		dw_hdmi_update_phy_mask(hdmi);
+	}
+	mutex_unlock(&hdmi->mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_setup_rx_sense);
+
+static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
+{
+	struct dw_hdmi *hdmi = dev_id;
+	u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat, hdcp_stat;
+
+	intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
+	if (intr_stat) {
+		phy_int_pol = hdmi_readb(hdmi, HDMI_PHY_POL0);
+		phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+		dev_dbg(hdmi->dev, "phy_int_pol:0x%x, phy_stat:0x%x\n",
+			phy_int_pol, phy_stat);
+
+		phy_pol_mask = 0;
+		if (intr_stat & HDMI_IH_PHY_STAT0_HPD)
+			phy_pol_mask |= HDMI_PHY_HPD;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE0)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE0;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE1)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE1;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE2)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE2;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE3)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE3;
+
+		if (phy_pol_mask)
+			hdmi_modb(hdmi, ~phy_int_pol, phy_pol_mask, HDMI_PHY_POL0);
+		if (phy_int_pol & HDMI_PHY_HPD)
+			hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_BUSCLEAR,
+				    HDMI_I2CM_OPERATION);
+
+		/*
+        * RX sense tells us whether the TDMS transmitters are detecting
+        * load - in other words, there's something listening on the
+        * other end of the link.  Use this to decide whether we should
+        * power on the phy as HPD may be toggled by the sink to merely
+        * ask the source to re-read the EDID.
+        */
+		if (intr_stat &
+		    (HDMI_IH_PHY_STAT0_RX_SENSE | HDMI_IH_PHY_STAT0_HPD)) {
+			hpd_flag[hdmi->numa_id] = true;
+			dw_hdmi_setup_rx_sense(hdmi,
+							phy_stat & HDMI_PHY_HPD,
+					        phy_stat & HDMI_PHY_RX_SENSE);
+
+			if ((phy_stat & (HDMI_PHY_RX_SENSE | HDMI_PHY_HPD)) == 0) {
+				mutex_lock(&hdmi->cec_notifier_mutex);
+				cec_notifier_phys_addr_invalidate(hdmi->cec_notifier);
+				mutex_unlock(&hdmi->cec_notifier_mutex);
+			}
+		}
+
+		check_hdmi_irq(hdmi, intr_stat, phy_int_pol);
+		hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0);
+		hdmi_writeb(hdmi,
+			    ~(HDMI_IH_PHY_STAT0_HPD |
+			      HDMI_IH_PHY_STAT0_RX_SENSE),
+			    HDMI_IH_MUTE_PHY_STAT0);
+	}
+
+	hpd_flag[hdmi->numa_id] = false;
+
+	hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT);
+	if (hdcp_stat) {
+		if (hdmi->hdcp && hdmi->hdcp->hdcp_isr)
+			hdmi->hdcp->hdcp_isr(hdmi->hdcp, hdcp_stat);
+		hdmi_writeb(hdmi, hdcp_stat, HDMI_A_APIINTCLR);
+		hdmi_writeb(hdmi, 0x00, HDMI_A_APIINTMSK);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static const struct dw_hdmi_phy_data dw_hdmi_phys[] = {
+	{
+		.type = DW_HDMI_PHY_DWC_HDMI_TX_PHY,
+		.name = "DWC HDMI TX PHY",
+		.gen = 1,
+	}, {
+		.type = DW_HDMI_PHY_DWC_MHL_PHY_HEAC,
+		.name = "DWC MHL PHY + HEAC PHY",
+		.gen = 2,
+		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_MHL_PHY,
+		.name = "DWC MHL PHY",
+		.gen = 2,
+		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_HDMI_3D_TX_PHY_HEAC,
+		.name = "DWC HDMI 3D TX PHY + HEAC PHY",
+		.gen = 2,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_HDMI_3D_TX_PHY,
+		.name = "DWC HDMI 3D TX PHY",
+		.gen = 2,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_HDMI20_TX_PHY,
+		.name = "DWC HDMI 2.0 TX PHY",
+		.gen = 2,
+		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_VENDOR_PHY,
+		.name = "Vendor PHY",
+	}
+};
+
+static int dw_hdmi_detect_phy(struct dw_hdmi *hdmi)
+{
+	unsigned int i;
+	u8 phy_type;
+
+	phy_type = hdmi->plat_data->phy_force_vendor ?
+				DW_HDMI_PHY_VENDOR_PHY :
+				hdmi_readb(hdmi, HDMI_CONFIG2_ID);
+
+	if (phy_type == DW_HDMI_PHY_VENDOR_PHY) {
+		/* Vendor PHYs require support from the glue layer. */
+		if (!hdmi->plat_data->phy_ops || !hdmi->plat_data->phy_name) {
+			dev_err(hdmi->dev,
+				"Vendor HDMI PHY not supported by glue layer\n");
+			return -ENODEV;
+		}
+
+		hdmi->phy.ops = hdmi->plat_data->phy_ops;
+		hdmi->phy.data = hdmi->plat_data->phy_data;
+		hdmi->phy.name = hdmi->plat_data->phy_name;
+		return 0;
+	}
+
+	/* Synopsys PHYs are handled internally. */
+	for (i = 0; i < ARRAY_SIZE(dw_hdmi_phys); ++i) {
+		if (dw_hdmi_phys[i].type == phy_type) {
+			hdmi->phy.ops = &dw_hdmi_synopsys_phy_ops;
+			hdmi->phy.name = dw_hdmi_phys[i].name;
+			hdmi->phy.data = (void *)&dw_hdmi_phys[i];
+
+			if (!dw_hdmi_phys[i].configure &&
+			    !hdmi->plat_data->configure_phy) {
+				dev_err(hdmi->dev, "%s requires platform support\n",
+					hdmi->phy.name);
+				return -ENODEV;
+			}
+
+			return 0;
+		}
+	}
+
+	dev_err(hdmi->dev, "Unsupported HDMI PHY type (%02x)\n", phy_type);
+	return -ENODEV;
+}
+
+static void dw_hdmi_cec_enable(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->mutex);
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CECCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_cec_disable(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->mutex);
+	hdmi->mc_clkdis |= HDMI_MC_CLKDIS_CECCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static const struct dw_hdmi_cec_ops dw_hdmi_cec_ops = {
+	.write = hdmi_writeb,
+	.read = hdmi_readb,
+	.enable = dw_hdmi_cec_enable,
+	.disable = dw_hdmi_cec_disable,
+};
+
+static int dw_hdmi_status_show(struct seq_file *s, void *v)
+{
+	struct dw_hdmi *hdmi = s->private;
+	u32 val;
+
+	seq_puts(s, "PHY: ");
+	if (!hdmi->phy.enabled) {
+		seq_puts(s, "disabled\n");
+		return 0;
+	}
+	seq_puts(s, "enabled\t\t\tMode: ");
+	if (hdmi->sink_is_hdmi)
+		seq_puts(s, "HDMI\n");
+	else
+		seq_puts(s, "DVI\n");
+	if (hdmi->hdmi_data.video_mode.mtmdsclock > 340000000)
+		val = hdmi->hdmi_data.video_mode.mtmdsclock / 4;
+	else
+		val = hdmi->hdmi_data.video_mode.mtmdsclock;
+	seq_printf(s, "Pixel Clk: %uHz\t\tTMDS Clk: %uHz\n",
+		   hdmi->hdmi_data.video_mode.mpixelclock, val);
+	seq_puts(s, "Color Format: ");
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "RGB");
+	else if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "YUV444");
+	else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "YUV422");
+	else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "YUV420");
+	else
+		seq_puts(s, "UNKNOWN");
+	val = hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format);
+	seq_printf(s, "\t\tColor Depth: %d bit\n", val);
+	seq_puts(s, "Colorimetry: ");
+	switch (hdmi->hdmi_data.enc_out_encoding) {
+	case V4L2_YCBCR_ENC_601:
+		seq_puts(s, "ITU.BT601");
+		break;
+	case V4L2_YCBCR_ENC_709:
+		seq_puts(s, "ITU.BT709");
+		break;
+	case V4L2_YCBCR_ENC_BT2020:
+		seq_puts(s, "ITU.BT2020");
+		break;
+	default: /* Carries no data */
+		seq_puts(s, "ITU.BT601");
+		break;
+	}
+
+	seq_puts(s, "\t\tEOTF: ");
+
+	if (hdmi->version < 0x211a) {
+		seq_puts(s, "Unsupported\n");
+		return 0;
+	}
+
+	val = hdmi_readb(hdmi, HDMI_FC_PACKET_TX_EN);
+	if (!(val & HDMI_FC_PACKET_TX_EN_DRM_MASK)) {
+		seq_puts(s, "Off\n");
+		return 0;
+	}
+
+	switch (hdmi_readb(hdmi, HDMI_FC_DRM_PB0)) {
+	case TRADITIONAL_GAMMA_SDR:
+		seq_puts(s, "SDR");
+		break;
+	case TRADITIONAL_GAMMA_HDR:
+		seq_puts(s, "HDR");
+		break;
+	case SMPTE_ST2084:
+		seq_puts(s, "ST2084");
+		break;
+	case HLG:
+		seq_puts(s, "HLG");
+		break;
+	default:
+		seq_puts(s, "Not Defined\n");
+		return 0;
+	}
+
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB3) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB2);
+	seq_printf(s, "\nx0: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB5) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB4);
+	seq_printf(s, "\t\t\t\ty0: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB7) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB6);
+	seq_printf(s, "x1: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB9) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB8);
+	seq_printf(s, "\t\t\t\ty1: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB11) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB10);
+	seq_printf(s, "x2: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB13) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB12);
+	seq_printf(s, "\t\t\t\ty2: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB15) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB14);
+	seq_printf(s, "white x: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB17) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB16);
+	seq_printf(s, "\t\t\twhite y: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB19) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB18);
+	seq_printf(s, "max lum: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB21) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB20);
+	seq_printf(s, "\t\t\tmin lum: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB23) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB22);
+	seq_printf(s, "max cll: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB25) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB24);
+	seq_printf(s, "\t\t\tmax fall: %d\n", val);
+	return 0;
+}
+
+static int dw_hdmi_status_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_hdmi_status_show, inode->i_private);
+}
+
+static const struct file_operations dw_hdmi_status_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_hdmi_status_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+struct dw_hdmi_reg_table {
+	int reg_base;
+	int reg_end;
+};
+
+static const struct dw_hdmi_reg_table hdmi_reg_table[] = {
+	{ HDMI_DESIGN_ID, HDMI_CONFIG3_ID },
+	{ HDMI_IH_FC_STAT0, HDMI_IH_MUTE },
+	{ HDMI_TX_INVID0, HDMI_TX_BCBDATA1 },
+	{ HDMI_VP_STATUS, HDMI_VP_POL },
+	{ HDMI_FC_INVIDCONF, HDMI_FC_DBGTMDS2 },
+	{ HDMI_PHY_CONF0, HDMI_PHY_POL0 },
+	{ HDMI_PHY_I2CM_SLAVE_ADDR, HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR },
+	{ HDMI_AUD_CONF0, 0x3624 },
+	{ HDMI_MC_SFRDIV, HDMI_MC_HEACPHY_RST },
+	{ HDMI_CSC_CFG, HDMI_CSC_COEF_C4_LSB },
+	{ HDMI_A_HDCPCFG0, 0x52bb },
+	{ 0x7800, 0x7818 },
+	{ 0x7900, 0x790e },
+	{ HDMI_CEC_CTRL, HDMI_CEC_WKUPCTRL },
+	{ HDMI_I2CM_SLAVE, HDMI_I2CM_SDA_HOLD },
+};
+
+static int dw_hdmi_ctrl_show(struct seq_file *s, void *v)
+{
+	struct dw_hdmi *hdmi = s->private;
+	u32 i = 0, j = 0, val = 0;
+
+	seq_puts(s, "\n>>>hdmi_ctl reg ");
+	for (i = 0; i < 16; i++)
+		seq_printf(s, " %2x", i);
+	seq_puts(s, "\n---------------------------------------------------");
+
+	for (i = 0; i < ARRAY_SIZE(hdmi_reg_table); i++) {
+		for (j = hdmi_reg_table[i].reg_base;
+		     j <= hdmi_reg_table[i].reg_end; j++) {
+			val = hdmi_readb(hdmi, j);
+			if ((j - hdmi_reg_table[i].reg_base) % 16 == 0)
+				seq_printf(s, "\n>>>hdmi_ctl %04x:", j);
+			seq_printf(s, " %02x", val);
+		}
+	}
+	seq_puts(s, "\n---------------------------------------------------\n");
+
+	return 0;
+}
+
+static int dw_hdmi_ctrl_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_hdmi_ctrl_show, inode->i_private);
+}
+
+static ssize_t dw_hdmi_ctrl_write(struct file *file, const char __user *buf,
+				  size_t count, loff_t *ppos)
+{
+	struct dw_hdmi *hdmi = ((struct seq_file *)file->private_data)->private;
+	u32 reg, val;
+	char kbuf[25];
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+	if (sscanf(kbuf, "%x%x", &reg, &val) == -1)
+		return -EFAULT;
+	if ((reg < 0) || (reg > HDMI_I2CM_FS_SCL_LCNT_0_ADDR)) {
+		dev_err(hdmi->dev, "it is no a hdmi register\n");
+		return count;
+	}
+	dev_info(hdmi->dev, "/**********hdmi register config******/");
+	dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val);
+	hdmi_writeb(hdmi, val, reg);
+	return count;
+}
+
+static const struct file_operations dw_hdmi_ctrl_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_hdmi_ctrl_open,
+	.read = seq_read,
+	.write = dw_hdmi_ctrl_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int dw_hdmi_phy_show(struct seq_file *s, void *v)
+{
+	struct dw_hdmi *hdmi = s->private;
+	u32 i, val;
+
+	seq_puts(s, "\n>>>hdmi_phy reg\n");
+	for (i = 0; i < 0x28; i++) {
+		val = hdmi_phy_i2c_read(hdmi, i);
+		seq_printf(s, "regs %02x val %04x\n", i, val);
+	}
+	return 0;
+}
+
+static int dw_hdmi_phy_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_hdmi_phy_show, inode->i_private);
+}
+
+static ssize_t dw_hdmi_phy_write(struct file *file, const char __user *buf,
+				 size_t count, loff_t *ppos)
+{
+	struct dw_hdmi *hdmi = ((struct seq_file *)file->private_data)->private;
+	u32 reg, val;
+	char kbuf[25];
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+	if (sscanf(kbuf, "%x%x", &reg, &val) == -1)
+		return -EFAULT;
+	if ((reg < 0) || (reg > 0x100)) {
+		dev_err(hdmi->dev, "it is not a hdmi phy register\n");
+		return count;
+	}
+	dev_info(hdmi->dev, "/*******hdmi phy register config******/");
+	dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val);
+	dw_hdmi_phy_i2c_write(hdmi, val, reg);
+	return count;
+}
+
+static const struct file_operations dw_hdmi_phy_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_hdmi_phy_open,
+	.read = seq_read,
+	.write = dw_hdmi_phy_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static void dw_hdmi_register_debugfs(struct device *dev, struct dw_hdmi *hdmi)
+{
+	char name[15];
+	sprintf(name, "dw-hdmi-%d", hdmi->numa_id);
+	hdmi->debugfs_dir = debugfs_create_dir(name, NULL);
+	if (IS_ERR(hdmi->debugfs_dir)) {
+		dev_err(dev, "failed to create debugfs dir!\n");
+		return;
+	}
+
+	debugfs_create_file("status", 0400, hdmi->debugfs_dir, hdmi,
+			    &dw_hdmi_status_fops);
+
+	debugfs_create_file("ctrl", 0400, hdmi->debugfs_dir, hdmi,
+			    &dw_hdmi_ctrl_fops);
+
+	debugfs_create_file("phy", 0400, hdmi->debugfs_dir, hdmi,
+			    &dw_hdmi_phy_fops);
+}
+
+static void dw_hdmi_register_hdcp(struct device *dev, struct dw_hdmi *hdmi,
+				  u32 val, bool hdcp1x_enable)
+{
+	struct dw_hdcp hdmi_hdcp = {
+		.hdmi = hdmi,
+		.write = hdmi_writeb,
+		.read = hdmi_readb,
+		.regs = hdmi->regs,
+		.reg_io_width = val,
+		.enable = hdcp1x_enable,
+		.numa_id = hdmi->numa_id,
+	};
+	struct platform_device_info hdcp_device_info = {
+		.parent = dev,
+		.id = PLATFORM_DEVID_AUTO,
+		.res = NULL,
+		.num_res = 0,
+		.name = DW_HDCP_DRIVER_NAME,
+		.data = &hdmi_hdcp,
+		.size_data = sizeof(hdmi_hdcp),
+		.dma_mask = DMA_BIT_MASK(32),
+	};
+
+	hdmi->hdcp_dev = platform_device_register_full(&hdcp_device_info);
+	if (IS_ERR(hdmi->hdcp_dev))
+		dev_err(dev, "failed to register hdcp!\n");
+	else
+		hdmi->hdcp = hdmi->hdcp_dev->dev.platform_data;
+}
+
+static const struct regmap_config hdmi_regmap_8bit_config = {
+	.reg_bits	= 32,
+	.val_bits	= 8,
+	.reg_stride	= 1,
+	.max_register	= HDMI_I2CM_SDA_HOLD,
+};
+
+static const struct regmap_config hdmi_regmap_32bit_config = {
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+	.max_register	= HDMI_I2CM_SDA_HOLD << 2,
+};
+
+static void dw_hdmi_init_hw(struct dw_hdmi *hdmi)
+{
+	initialize_hdmi_ih_mutes(hdmi);
+
+	/*
+	 * Reset HDMI DDC I2C master controller and mute I2CM interrupts.
+	 * Even if we are using a separate i2c adapter doing this doesn't
+	 * hurt.
+	 */
+	dw_hdmi_i2c_init(hdmi);
+
+	if (hdmi->phy.ops->setup_hpd)
+		hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data);
+
+	/*
+     * HDMI status maybe incorrect in the following condition:
+     * HDMI plug in -> system sleep ->  HDMI plug out -> system wake up.
+     * At this time, cat /sys/class/drm/card 0-HDMI-A-1/status is connected.
+     * There is no hpd interrupt, because HDMI is powerdown during suspend.
+     * So we need check the current HDMI status in this case.
+     */
+	if (hdmi->connector.status == connector_status_connected)
+		if (hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data) ==
+		    connector_status_disconnected) {
+			hdmi->hpd_state = false;
+			mod_delayed_work(hdmi->workqueue, &hdmi->work,
+					 msecs_to_jiffies(20));
+		}
+}
+
+/* -----------------------------------------------------------------------------
+ * Probe/remove API, used from platforms based on the DRM bridge API.
+ */
+
+static int dw_hdmi_parse_dt(struct dw_hdmi *hdmi)
+{
+	struct device_node *endpoint;
+	struct device_node *remote;
+
+	if (!hdmi->plat_data->output_port)
+		return 0;
+
+	endpoint = of_graph_get_endpoint_by_regs(hdmi->dev->of_node,
+						 hdmi->plat_data->output_port,
+						 -1);
+	if (!endpoint) {
+		/*
+		 * On platforms whose bindings don't make the output port
+		 * mandatory (such as Rockchip) the plat_data->output_port
+		 * field isn't set, so it's safe to make this a fatal error.
+		 */
+		dev_err(hdmi->dev, "Missing endpoint in port@%u\n",
+			hdmi->plat_data->output_port);
+		return -ENODEV;
+	}
+
+	remote = of_graph_get_remote_port_parent(endpoint);
+	of_node_put(endpoint);
+	if (!remote) {
+		dev_err(hdmi->dev, "Endpoint in port@%u unconnected\n",
+			hdmi->plat_data->output_port);
+		return -ENODEV;
+	}
+
+	if (!of_device_is_available(remote)) {
+		dev_err(hdmi->dev, "port@%u remote device is disabled\n",
+			hdmi->plat_data->output_port);
+		of_node_put(remote);
+		return -ENODEV;
+	}
+
+	hdmi->next_bridge = of_drm_find_bridge(remote);
+	of_node_put(remote);
+	if (!hdmi->next_bridge)
+		return -EPROBE_DEFER;
+
+	return 0;
+}
+
+bool dw_hdmi_bus_fmt_is_420(struct dw_hdmi *hdmi)
+{
+	return hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_bus_fmt_is_420);
+
+void dw_hdmi_enable_video(struct dw_hdmi *hdmi)
+{
+	dev_info(hdmi->dev, "%s", __func__);
+	hdmi_writeb(hdmi, HDMI_FC_GCP_CLEAR_AVMUTE, HDMI_FC_GCP);
+}
+void dw_hdmi_disable_video(struct dw_hdmi *hdmi)
+{
+	dev_info(hdmi->dev, "%s", __func__);
+	hdmi_writeb(hdmi, HDMI_FC_GCP_SET_AVMUTE, HDMI_FC_GCP);
+}
+
+struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev,
+			      const struct dw_hdmi_plat_data *plat_data)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct platform_device_info pdevinfo;
+	struct device_node *ddc_node;
+	struct dw_hdmi_cec_data cec;
+	struct dw_hdmi *hdmi;
+	struct resource *iores = NULL;
+	int irq;
+	int ret;
+	u32 val = 1;
+	u8 prod_id0;
+	u8 prod_id1;
+	u8 config0;
+	u8 config3;
+	bool hdcp1x_enable = false;
+	int numa_id = 0;
+
+	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return ERR_PTR(-ENOMEM);
+
+	hdmi->plat_data = plat_data;
+	hdmi->dev = dev;
+	hdmi->sample_rate = 48000;
+	hdmi->disabled = true;
+	hdmi->rxsense = true;
+	hdmi->phy_mask = (u8) ~(HDMI_PHY_HPD | HDMI_PHY_RX_SENSE);
+	hdmi->mc_clkdis = 0x7f;
+	hdmi->last_connector_result = connector_status_disconnected;
+
+	mutex_init(&hdmi->mutex);
+	mutex_init(&hdmi->audio_mutex);
+	mutex_init(&hdmi->cec_notifier_mutex);
+	spin_lock_init(&hdmi->audio_lock);
+
+	ret = dw_hdmi_parse_dt(hdmi);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	if(of_property_read_u32(pdev->dev.of_node, "numa-node-id", &numa_id)) {
+		numa_id = 0;
+	}
+	dev_info(&pdev->dev, "numa_id=%d\n", numa_id);
+	hdmi->numa_id = numa_id;
+
+	ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0);
+	if (ddc_node) {
+		hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node);
+		of_node_put(ddc_node);
+		if (!hdmi->ddc) {
+			dev_dbg(hdmi->dev, "failed to read ddc node\n");
+			return ERR_PTR(-EPROBE_DEFER);
+		}
+
+	} else {
+		dev_dbg(hdmi->dev, "no ddc property found\n");
+	}
+
+	if (!plat_data->regm) {
+		const struct regmap_config *reg_config;
+
+		of_property_read_u32(np, "reg-io-width", &val);
+		switch (val) {
+		case 4:
+			reg_config = &hdmi_regmap_32bit_config;
+			hdmi->reg_shift = 2;
+			break;
+		case 1:
+			reg_config = &hdmi_regmap_8bit_config;
+			break;
+		default:
+			dev_err(dev, "reg-io-width must be 1 or 4\n");
+			return ERR_PTR(-EINVAL);
+		}
+
+		iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		hdmi->regs = devm_ioremap_resource(dev, iores);
+		if (IS_ERR(hdmi->regs)) {
+			ret = PTR_ERR(hdmi->regs);
+			goto err_res;
+		}
+
+		hdmi->regm = devm_regmap_init_mmio(dev, hdmi->regs, reg_config);
+		if (IS_ERR(hdmi->regm)) {
+			dev_err(dev, "Failed to configure regmap\n");
+			ret = PTR_ERR(hdmi->regm);
+			goto err_res;
+		}
+	} else {
+		hdmi->regm = plat_data->regm;
+	}
+
+	hdmi->isfr_clk = devm_clk_get(hdmi->dev, "isfr");
+	if (IS_ERR(hdmi->isfr_clk)) {
+		ret = PTR_ERR(hdmi->isfr_clk);
+		dev_err(hdmi->dev, "Unable to get HDMI isfr clk: %d\n", ret);
+		goto err_res;
+	}
+
+	ret = clk_prepare_enable(hdmi->isfr_clk);
+	if (ret) {
+		dev_err(hdmi->dev, "Cannot enable HDMI isfr clock: %d\n", ret);
+		goto err_res;
+	}
+
+	hdmi->iahb_clk = devm_clk_get(hdmi->dev, "iahb");
+	if (IS_ERR(hdmi->iahb_clk)) {
+		ret = PTR_ERR(hdmi->iahb_clk);
+		dev_err(hdmi->dev, "Unable to get HDMI iahb clk: %d\n", ret);
+		goto err_isfr;
+	}
+
+	ret = clk_prepare_enable(hdmi->iahb_clk);
+	if (ret) {
+		dev_err(hdmi->dev, "Cannot enable HDMI iahb clock: %d\n", ret);
+		goto err_isfr;
+	}
+
+	hdmi->cec_clk = devm_clk_get(hdmi->dev, "cec");
+	if (PTR_ERR(hdmi->cec_clk) == -ENOENT) {
+		hdmi->cec_clk = NULL;
+	} else if (IS_ERR(hdmi->cec_clk)) {
+		ret = PTR_ERR(hdmi->cec_clk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(hdmi->dev, "Cannot get HDMI cec clock: %d\n",
+				ret);
+
+		hdmi->cec_clk = NULL;
+		goto err_iahb;
+	} else {
+		ret = clk_prepare_enable(hdmi->cec_clk);
+		if (ret) {
+			dev_err(hdmi->dev, "Cannot enable HDMI cec clock: %d\n",
+				ret);
+			goto err_iahb;
+		}
+
+		ret = clk_set_rate(hdmi->cec_clk, HDMI_CEC_CLK);
+		if (ret) {
+			dev_err(hdmi->dev, "Cannot set CEC clock rate, error:%d\n", ret);
+			goto err_iahb;
+		}
+	}
+
+	/* hdmi prstn reset */
+	hdmi->rst_hdmi_prstn =
+		devm_reset_control_get_optional(hdmi->dev, "prstn");
+	if (IS_ERR_OR_NULL(hdmi->rst_hdmi_prstn)) {
+		dev_err(hdmi->dev, "Failed to get hdmi prstn reset handle\n");
+		ret = -EFAULT;
+		goto err_cec;
+	}
+
+	/* hdmi phyctl reset */
+	hdmi->rst_hdmi_phyrstn =
+		devm_reset_control_get_optional(hdmi->dev, "phyrstn");
+	if (IS_ERR_OR_NULL(hdmi->rst_hdmi_phyrstn)) {
+		dev_err(hdmi->dev, "Failed to get hdmi phyrstn reset handle\n");
+		ret = -EFAULT;
+		goto err_cec;
+	}
+
+	/* hdmi rstn reset */
+	hdmi->rst_hdmi_rstn =
+		devm_reset_control_get_optional(hdmi->dev, "rstn");
+	if (IS_ERR_OR_NULL(hdmi->rst_hdmi_rstn)) {
+		dev_err(hdmi->dev, "Failed to get hdmi rstn reset handle\n");
+		ret = -EFAULT;
+		goto err_cec;
+	}
+
+	if (hdmi->rst_hdmi_prstn) {
+		ret = reset_control_reset(hdmi->rst_hdmi_prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_phyrstn) {
+		ret = reset_control_reset(hdmi->rst_hdmi_phyrstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_rstn) {
+		ret = reset_control_reset(hdmi->rst_hdmi_rstn);
+		WARN_ON(0 != ret);
+	}
+
+	/* Product and revision IDs */
+	hdmi->version = (hdmi_readb(hdmi, HDMI_DESIGN_ID) << 8)
+		      | (hdmi_readb(hdmi, HDMI_REVISION_ID) << 0);
+	prod_id0 = hdmi_readb(hdmi, HDMI_PRODUCT_ID0);
+	prod_id1 = hdmi_readb(hdmi, HDMI_PRODUCT_ID1);
+
+	if (prod_id0 != HDMI_PRODUCT_ID0_HDMI_TX ||
+	    (prod_id1 & ~HDMI_PRODUCT_ID1_HDCP) != HDMI_PRODUCT_ID1_HDMI_TX) {
+		dev_err(dev, "Unsupported HDMI controller (%04x:%02x:%02x)\n",
+			hdmi->version, prod_id0, prod_id1);
+		ret = -ENODEV;
+		goto err_cec;
+	}
+
+	ret = dw_hdmi_detect_phy(hdmi);
+	if (ret < 0)
+		goto err_cec;
+
+	dev_info(dev, "Detected HDMI TX controller v%x.%03x %s HDCP (%s)\n",
+		 hdmi->version >> 12, hdmi->version & 0xfff,
+		 prod_id1 & HDMI_PRODUCT_ID1_HDCP ? "with" : "without",
+		 hdmi->phy.name);
+
+	dw_hdmi_init_hw(hdmi);
+	init_hpd_work(hdmi);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		goto err_cec;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, dw_hdmi_hardirq,
+					dw_hdmi_irq, IRQF_SHARED,
+					dev_name(dev), hdmi);
+	if (ret)
+		goto err_cec;
+
+	/*
+	 * To prevent overflows in HDMI_IH_FC_STAT2, set the clk regenerator
+	 * N and cts values before enabling phy
+	 */
+	hdmi_init_clk_regenerator(hdmi);
+
+	/* If DDC bus is not specified, try to register HDMI I2C bus */
+	if (!hdmi->ddc) {
+		/* Look for (optional) stuff related to unwedging */
+		hdmi->pinctrl = devm_pinctrl_get(dev);
+		if (!IS_ERR(hdmi->pinctrl)) {
+			hdmi->unwedge_state =
+				pinctrl_lookup_state(hdmi->pinctrl, "unwedge");
+			hdmi->default_state =
+				pinctrl_lookup_state(hdmi->pinctrl, "default");
+
+			if (IS_ERR(hdmi->default_state) ||
+			    IS_ERR(hdmi->unwedge_state)) {
+				if (!IS_ERR(hdmi->unwedge_state))
+					dev_warn(dev,
+						 "Unwedge requires default pinctrl\n");
+				hdmi->default_state = NULL;
+				hdmi->unwedge_state = NULL;
+			}
+		}
+
+		hdmi->ddc = dw_hdmi_i2c_adapter(hdmi);
+		if (IS_ERR(hdmi->ddc))
+			hdmi->ddc = NULL;
+	}
+
+	hdmi->bridge.driver_private = hdmi;
+	hdmi->bridge.funcs = &dw_hdmi_bridge_funcs;
+	hdmi->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID
+			 | DRM_BRIDGE_OP_HPD;
+	hdmi->bridge.interlace_allowed = true;
+	hdmi->bridge.ddc = hdmi->ddc;
+#ifdef CONFIG_OF
+	hdmi->bridge.of_node = pdev->dev.of_node;
+#endif
+
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+	pdevinfo.parent = dev;
+	pdevinfo.id = PLATFORM_DEVID_AUTO;
+
+	config0 = hdmi_readb(hdmi, HDMI_CONFIG0_ID);
+	config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID);
+
+	if (iores && config3 & HDMI_CONFIG3_AHBAUDDMA) {
+		struct dw_hdmi_audio_data audio;
+
+		audio.phys = iores->start;
+		audio.base = hdmi->regs;
+		audio.irq = irq;
+		audio.hdmi = hdmi;
+		audio.get_eld = hdmi_audio_get_eld;
+		hdmi->enable_audio = dw_hdmi_ahb_audio_enable;
+		hdmi->disable_audio = dw_hdmi_ahb_audio_disable;
+
+		pdevinfo.name = "dw-hdmi-ahb-audio";
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
+	} else if (config0 & HDMI_CONFIG0_I2S) {
+		struct dw_hdmi_i2s_audio_data audio;
+
+		audio.hdmi	= hdmi;
+		audio.get_eld	= hdmi_audio_get_eld;
+		audio.write	= hdmi_writeb;
+		audio.read	= hdmi_readb;
+		hdmi->enable_audio = dw_hdmi_i2s_audio_enable;
+		hdmi->disable_audio = dw_hdmi_i2s_audio_disable;
+
+		pdevinfo.name = "dw-hdmi-i2s-audio";
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
+	} else if (iores && config3 & HDMI_CONFIG3_GPAUD) {
+		struct dw_hdmi_audio_data audio;
+
+		audio.phys = iores->start;
+		audio.base = hdmi->regs;
+		audio.irq = irq;
+		audio.hdmi = hdmi;
+		audio.get_eld = hdmi_audio_get_eld;
+
+		hdmi->enable_audio = dw_hdmi_gp_audio_enable;
+		hdmi->disable_audio = dw_hdmi_gp_audio_disable;
+
+		pdevinfo.name = "dw-hdmi-gp-audio";
+		pdevinfo.id = PLATFORM_DEVID_NONE;
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
+	}
+
+	if (!plat_data->disable_cec && (config0 & HDMI_CONFIG0_CEC)) {
+		cec.hdmi = hdmi;
+		cec.ops = &dw_hdmi_cec_ops;
+		cec.irq = irq;
+
+		pdevinfo.name = "dw-hdmi-cec";
+		pdevinfo.data = &cec;
+		pdevinfo.size_data = sizeof(cec);
+		pdevinfo.dma_mask = 0;
+
+		hdmi->cec = platform_device_register_full(&pdevinfo);
+	}
+
+	drm_bridge_add(&hdmi->bridge);
+
+	dw_hdmi_register_debugfs(dev, hdmi);
+
+	if (of_property_read_bool(np, "hdcp1x-enable"))
+		hdcp1x_enable = true;
+	dw_hdmi_register_hdcp(dev, hdmi, val, hdcp1x_enable);
+
+	return hdmi;
+
+err_cec:
+	clk_disable_unprepare(hdmi->cec_clk);
+err_iahb:
+	clk_disable_unprepare(hdmi->iahb_clk);
+err_isfr:
+	clk_disable_unprepare(hdmi->isfr_clk);
+err_res:
+	i2c_put_adapter(hdmi->ddc);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_probe);
+
+void dw_hdmi_remove(struct dw_hdmi *hdmi)
+{
+	int ret;
+
+	drm_bridge_remove(&hdmi->bridge);
+
+	cancel_delayed_work(&hdmi->work);
+	flush_workqueue(hdmi->workqueue);
+	destroy_workqueue(hdmi->workqueue);
+
+	if (hdmi->audio && !IS_ERR(hdmi->audio))
+		platform_device_unregister(hdmi->audio);
+	if (!IS_ERR(hdmi->cec))
+		platform_device_unregister(hdmi->cec);
+	if (hdmi->hdcp_dev && !IS_ERR(hdmi->hdcp_dev))
+		platform_device_unregister(hdmi->hdcp_dev);
+
+	/* Disable all interrupts */
+	hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0);
+
+	dw_hdmi_destroy_properties(hdmi);
+	hdmi->connector.funcs->destroy(&hdmi->connector);
+
+	clk_disable_unprepare(hdmi->iahb_clk);
+	clk_disable_unprepare(hdmi->isfr_clk);
+	clk_disable_unprepare(hdmi->cec_clk);
+
+	if (hdmi->i2c)
+		i2c_del_adapter(&hdmi->i2c->adap);
+	else
+		i2c_put_adapter(hdmi->ddc);
+
+	debugfs_remove_recursive(hdmi->debugfs_dir);
+
+	if (hdmi->rst_hdmi_prstn) {
+		ret = reset_control_assert(hdmi->rst_hdmi_prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_phyrstn) {
+		ret = reset_control_assert(hdmi->rst_hdmi_phyrstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_rstn) {
+		ret = reset_control_assert(hdmi->rst_hdmi_rstn);
+		WARN_ON(0 != ret);
+	}
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_remove);
+
+/* -----------------------------------------------------------------------------
+ * Bind/unbind API, used from platforms based on the component framework.
+ */
+struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev,
+			     struct drm_encoder *encoder,
+			     const struct dw_hdmi_plat_data *plat_data)
+{
+	struct dw_hdmi *hdmi;
+	int ret;
+
+	hdmi = dw_hdmi_probe(pdev, plat_data);
+	if (IS_ERR(hdmi))
+		return hdmi;
+
+	ret = drm_bridge_attach(encoder, &hdmi->bridge, NULL, 0);
+	if (ret) {
+		dw_hdmi_remove(hdmi);
+		DRM_ERROR("Failed to initialize bridge with drm\n");
+		return ERR_PTR(ret);
+	}
+
+	return hdmi;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_bind);
+
+void dw_hdmi_unbind(struct dw_hdmi *hdmi)
+{
+	dw_hdmi_remove(hdmi);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_unbind);
+
+void dw_hdmi_suspend(struct dw_hdmi *hdmi)
+{
+	dev_dbg(hdmi->dev, "%s", __func__);
+	if (!hdmi) {
+		dev_warn(hdmi->dev, "HDMI has not been initialized\n");
+		return;
+	}
+
+	mutex_lock(&hdmi->mutex);
+
+	/*
+     * When system shutdown, hdmi should be disabled.
+     * When system suspend, dw_hdmi_bridge_disable will disable hdmi first.
+     * To prevent duplicate operation, we should determine whether hdmi
+     * has been disabled.
+     */
+	if (!hdmi->disabled) {
+		hdmi->disabled = true;
+		dw_hdmi_update_power(hdmi);
+		dw_hdmi_update_phy_mask(hdmi);
+	}
+	mutex_unlock(&hdmi->mutex);
+
+	if (hdmi->irq)
+		disable_irq(hdmi->irq);
+	cancel_delayed_work(&hdmi->work);
+	flush_workqueue(hdmi->workqueue);
+	pinctrl_pm_select_sleep_state(hdmi->dev);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_suspend);
+
+void dw_hdmi_resume(struct dw_hdmi *hdmi)
+{
+	dev_dbg(hdmi->dev, "%s", __func__);
+	dw_hdmi_init_hw(hdmi);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_resume);
+
+MODULE_AUTHOR("Sascha Hauer <s.hauer@pengutronix.de>");
+MODULE_AUTHOR("Andy Yan <andy.yan@rock-chips.com>");
+MODULE_AUTHOR("Yakir Yang <ykk@rock-chips.com>");
+MODULE_AUTHOR("Vladimir Zapolskiy <vladimir_zapolskiy@mentor.com>");
+MODULE_DESCRIPTION("DW HDMI transmitter driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:dw-hdmi");
diff --git a/drivers/gpu/drm/eswin/dw-hdmi.h b/drivers/gpu/drm/eswin/dw-hdmi.h
new file mode 100644
index 000000000000..9304c5865aee
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw-hdmi.h
@@ -0,0 +1,1289 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2011 Freescale Semiconductor, Inc.
+ *****************************************************************************
+ * ESWIN hdmi driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+#ifndef __DW_HDMI_H__
+#define __DW_HDMI_H__
+
+#include <sound/hdmi-codec.h>
+#include <drm/drm_property.h>
+
+extern struct platform_driver dw_hdmi_eswin_pltfm_driver;
+extern struct platform_driver snd_dw_hdmi_driver;
+extern struct platform_driver dw_hdmi_cec_driver;
+extern struct platform_driver dw_hdmi_hdcp_driver;
+extern struct platform_driver dw_hdmi_hdcp2_driver;
+
+/* Identification Registers */
+#define HDMI_DESIGN_ID                          0x0000
+#define HDMI_REVISION_ID                        0x0001
+#define HDMI_PRODUCT_ID0                        0x0002
+#define HDMI_PRODUCT_ID1                        0x0003
+#define HDMI_CONFIG0_ID                         0x0004
+#define HDMI_CONFIG1_ID                         0x0005
+#define HDMI_CONFIG2_ID                         0x0006
+#define HDMI_CONFIG3_ID                         0x0007
+
+/* Interrupt Registers */
+#define HDMI_IH_FC_STAT0                        0x0100
+#define HDMI_IH_FC_STAT1                        0x0101
+#define HDMI_IH_FC_STAT2                        0x0102
+#define HDMI_IH_AS_STAT0                        0x0103
+#define HDMI_IH_PHY_STAT0                       0x0104
+#define HDMI_IH_I2CM_STAT0                      0x0105
+#define HDMI_IH_CEC_STAT0                       0x0106
+#define HDMI_IH_VP_STAT0                        0x0107
+#define HDMI_IH_I2CMPHY_STAT0                   0x0108
+#define HDMI_IH_AHBDMAAUD_STAT0                 0x0109
+
+#define HDMI_IH_MUTE_FC_STAT0                   0x0180
+#define HDMI_IH_MUTE_FC_STAT1                   0x0181
+#define HDMI_IH_MUTE_FC_STAT2                   0x0182
+#define HDMI_IH_MUTE_AS_STAT0                   0x0183
+#define HDMI_IH_MUTE_PHY_STAT0                  0x0184
+#define HDMI_IH_MUTE_I2CM_STAT0                 0x0185
+#define HDMI_IH_MUTE_CEC_STAT0                  0x0186
+#define HDMI_IH_MUTE_VP_STAT0                   0x0187
+#define HDMI_IH_MUTE_I2CMPHY_STAT0              0x0188
+#define HDMI_IH_MUTE_AHBDMAAUD_STAT0            0x0189
+#define HDMI_IH_MUTE                            0x01FF
+
+/* Video Sample Registers */
+#define HDMI_TX_INVID0                          0x0200
+#define HDMI_TX_INSTUFFING                      0x0201
+#define HDMI_TX_GYDATA0                         0x0202
+#define HDMI_TX_GYDATA1                         0x0203
+#define HDMI_TX_RCRDATA0                        0x0204
+#define HDMI_TX_RCRDATA1                        0x0205
+#define HDMI_TX_BCBDATA0                        0x0206
+#define HDMI_TX_BCBDATA1                        0x0207
+
+/* Video Packetizer Registers */
+#define HDMI_VP_STATUS                          0x0800
+#define HDMI_VP_PR_CD                           0x0801
+#define HDMI_VP_STUFF                           0x0802
+#define HDMI_VP_REMAP                           0x0803
+#define HDMI_VP_CONF                            0x0804
+#define HDMI_VP_STAT                            0x0805
+#define HDMI_VP_INT                             0x0806
+#define HDMI_VP_MASK                            0x0807
+#define HDMI_VP_POL                             0x0808
+
+/* Frame Composer Registers */
+#define HDMI_FC_INVIDCONF                       0x1000
+#define HDMI_FC_INHACTV0                        0x1001
+#define HDMI_FC_INHACTV1                        0x1002
+#define HDMI_FC_INHBLANK0                       0x1003
+#define HDMI_FC_INHBLANK1                       0x1004
+#define HDMI_FC_INVACTV0                        0x1005
+#define HDMI_FC_INVACTV1                        0x1006
+#define HDMI_FC_INVBLANK                        0x1007
+#define HDMI_FC_HSYNCINDELAY0                   0x1008
+#define HDMI_FC_HSYNCINDELAY1                   0x1009
+#define HDMI_FC_HSYNCINWIDTH0                   0x100A
+#define HDMI_FC_HSYNCINWIDTH1                   0x100B
+#define HDMI_FC_VSYNCINDELAY                    0x100C
+#define HDMI_FC_VSYNCINWIDTH                    0x100D
+#define HDMI_FC_INFREQ0                         0x100E
+#define HDMI_FC_INFREQ1                         0x100F
+#define HDMI_FC_INFREQ2                         0x1010
+#define HDMI_FC_CTRLDUR                         0x1011
+#define HDMI_FC_EXCTRLDUR                       0x1012
+#define HDMI_FC_EXCTRLSPAC                      0x1013
+#define HDMI_FC_CH0PREAM                        0x1014
+#define HDMI_FC_CH1PREAM                        0x1015
+#define HDMI_FC_CH2PREAM                        0x1016
+#define HDMI_FC_AVICONF3                        0x1017
+#define HDMI_FC_GCP                             0x1018
+#define HDMI_FC_AVICONF0                        0x1019
+#define HDMI_FC_AVICONF1                        0x101A
+#define HDMI_FC_AVICONF2                        0x101B
+#define HDMI_FC_AVIVID                          0x101C
+#define HDMI_FC_AVIETB0                         0x101D
+#define HDMI_FC_AVIETB1                         0x101E
+#define HDMI_FC_AVISBB0                         0x101F
+#define HDMI_FC_AVISBB1                         0x1020
+#define HDMI_FC_AVIELB0                         0x1021
+#define HDMI_FC_AVIELB1                         0x1022
+#define HDMI_FC_AVISRB0                         0x1023
+#define HDMI_FC_AVISRB1                         0x1024
+#define HDMI_FC_AUDICONF0                       0x1025
+#define HDMI_FC_AUDICONF1                       0x1026
+#define HDMI_FC_AUDICONF2                       0x1027
+#define HDMI_FC_AUDICONF3                       0x1028
+#define HDMI_FC_VSDIEEEID0                      0x1029
+#define HDMI_FC_VSDSIZE                         0x102A
+#define HDMI_FC_VSDIEEEID1                      0x1030
+#define HDMI_FC_VSDIEEEID2                      0x1031
+#define HDMI_FC_VSDPAYLOAD0                     0x1032
+#define HDMI_FC_VSDPAYLOAD1                     0x1033
+#define HDMI_FC_VSDPAYLOAD2                     0x1034
+#define HDMI_FC_VSDPAYLOAD3                     0x1035
+#define HDMI_FC_VSDPAYLOAD4                     0x1036
+#define HDMI_FC_VSDPAYLOAD5                     0x1037
+#define HDMI_FC_VSDPAYLOAD6                     0x1038
+#define HDMI_FC_VSDPAYLOAD7                     0x1039
+#define HDMI_FC_VSDPAYLOAD8                     0x103A
+#define HDMI_FC_VSDPAYLOAD9                     0x103B
+#define HDMI_FC_VSDPAYLOAD10                    0x103C
+#define HDMI_FC_VSDPAYLOAD11                    0x103D
+#define HDMI_FC_VSDPAYLOAD12                    0x103E
+#define HDMI_FC_VSDPAYLOAD13                    0x103F
+#define HDMI_FC_VSDPAYLOAD14                    0x1040
+#define HDMI_FC_VSDPAYLOAD15                    0x1041
+#define HDMI_FC_VSDPAYLOAD16                    0x1042
+#define HDMI_FC_VSDPAYLOAD17                    0x1043
+#define HDMI_FC_VSDPAYLOAD18                    0x1044
+#define HDMI_FC_VSDPAYLOAD19                    0x1045
+#define HDMI_FC_VSDPAYLOAD20                    0x1046
+#define HDMI_FC_VSDPAYLOAD21                    0x1047
+#define HDMI_FC_VSDPAYLOAD22                    0x1048
+#define HDMI_FC_VSDPAYLOAD23                    0x1049
+#define HDMI_FC_SPDVENDORNAME0                  0x104A
+#define HDMI_FC_SPDVENDORNAME1                  0x104B
+#define HDMI_FC_SPDVENDORNAME2                  0x104C
+#define HDMI_FC_SPDVENDORNAME3                  0x104D
+#define HDMI_FC_SPDVENDORNAME4                  0x104E
+#define HDMI_FC_SPDVENDORNAME5                  0x104F
+#define HDMI_FC_SPDVENDORNAME6                  0x1050
+#define HDMI_FC_SPDVENDORNAME7                  0x1051
+#define HDMI_FC_SDPPRODUCTNAME0                 0x1052
+#define HDMI_FC_SDPPRODUCTNAME1                 0x1053
+#define HDMI_FC_SDPPRODUCTNAME2                 0x1054
+#define HDMI_FC_SDPPRODUCTNAME3                 0x1055
+#define HDMI_FC_SDPPRODUCTNAME4                 0x1056
+#define HDMI_FC_SDPPRODUCTNAME5                 0x1057
+#define HDMI_FC_SDPPRODUCTNAME6                 0x1058
+#define HDMI_FC_SDPPRODUCTNAME7                 0x1059
+#define HDMI_FC_SDPPRODUCTNAME8                 0x105A
+#define HDMI_FC_SDPPRODUCTNAME9                 0x105B
+#define HDMI_FC_SDPPRODUCTNAME10                0x105C
+#define HDMI_FC_SDPPRODUCTNAME11                0x105D
+#define HDMI_FC_SDPPRODUCTNAME12                0x105E
+#define HDMI_FC_SDPPRODUCTNAME13                0x105F
+#define HDMI_FC_SDPPRODUCTNAME14                0x1060
+#define HDMI_FC_SPDPRODUCTNAME15                0x1061
+#define HDMI_FC_SPDDEVICEINF                    0x1062
+#define HDMI_FC_AUDSCONF                        0x1063
+#define HDMI_FC_AUDSSTAT                        0x1064
+#define HDMI_FC_AUDSV                           0x1065
+#define HDMI_FC_AUDSU                           0x1066
+#define HDMI_FC_AUDSCHNLS0                       0x1067
+#define HDMI_FC_AUDSCHNLS1                       0x1068
+#define HDMI_FC_AUDSCHNLS2                       0x1069
+#define HDMI_FC_AUDSCHNLS3                       0x106A
+#define HDMI_FC_AUDSCHNLS4                       0x106B
+#define HDMI_FC_AUDSCHNLS5                       0x106C
+#define HDMI_FC_AUDSCHNLS6                       0x106D
+#define HDMI_FC_AUDSCHNLS7                       0x106E
+#define HDMI_FC_AUDSCHNLS8                       0x106F
+#define HDMI_FC_DATACH0FILL                     0x1070
+#define HDMI_FC_DATACH1FILL                     0x1071
+#define HDMI_FC_DATACH2FILL                     0x1072
+#define HDMI_FC_CTRLQHIGH                       0x1073
+#define HDMI_FC_CTRLQLOW                        0x1074
+#define HDMI_FC_ACP0                            0x1075
+#define HDMI_FC_ACP28                           0x1076
+#define HDMI_FC_ACP27                           0x1077
+#define HDMI_FC_ACP26                           0x1078
+#define HDMI_FC_ACP25                           0x1079
+#define HDMI_FC_ACP24                           0x107A
+#define HDMI_FC_ACP23                           0x107B
+#define HDMI_FC_ACP22                           0x107C
+#define HDMI_FC_ACP21                           0x107D
+#define HDMI_FC_ACP20                           0x107E
+#define HDMI_FC_ACP19                           0x107F
+#define HDMI_FC_ACP18                           0x1080
+#define HDMI_FC_ACP17                           0x1081
+#define HDMI_FC_ACP16                           0x1082
+#define HDMI_FC_ACP15                           0x1083
+#define HDMI_FC_ACP14                           0x1084
+#define HDMI_FC_ACP13                           0x1085
+#define HDMI_FC_ACP12                           0x1086
+#define HDMI_FC_ACP11                           0x1087
+#define HDMI_FC_ACP10                           0x1088
+#define HDMI_FC_ACP9                            0x1089
+#define HDMI_FC_ACP8                            0x108A
+#define HDMI_FC_ACP7                            0x108B
+#define HDMI_FC_ACP6                            0x108C
+#define HDMI_FC_ACP5                            0x108D
+#define HDMI_FC_ACP4                            0x108E
+#define HDMI_FC_ACP3                            0x108F
+#define HDMI_FC_ACP2                            0x1090
+#define HDMI_FC_ACP1                            0x1091
+#define HDMI_FC_ISCR1_0                         0x1092
+#define HDMI_FC_ISCR1_16                        0x1093
+#define HDMI_FC_ISCR1_15                        0x1094
+#define HDMI_FC_ISCR1_14                        0x1095
+#define HDMI_FC_ISCR1_13                        0x1096
+#define HDMI_FC_ISCR1_12                        0x1097
+#define HDMI_FC_ISCR1_11                        0x1098
+#define HDMI_FC_ISCR1_10                        0x1099
+#define HDMI_FC_ISCR1_9                         0x109A
+#define HDMI_FC_ISCR1_8                         0x109B
+#define HDMI_FC_ISCR1_7                         0x109C
+#define HDMI_FC_ISCR1_6                         0x109D
+#define HDMI_FC_ISCR1_5                         0x109E
+#define HDMI_FC_ISCR1_4                         0x109F
+#define HDMI_FC_ISCR1_3                         0x10A0
+#define HDMI_FC_ISCR1_2                         0x10A1
+#define HDMI_FC_ISCR1_1                         0x10A2
+#define HDMI_FC_ISCR2_15                        0x10A3
+#define HDMI_FC_ISCR2_14                        0x10A4
+#define HDMI_FC_ISCR2_13                        0x10A5
+#define HDMI_FC_ISCR2_12                        0x10A6
+#define HDMI_FC_ISCR2_11                        0x10A7
+#define HDMI_FC_ISCR2_10                        0x10A8
+#define HDMI_FC_ISCR2_9                         0x10A9
+#define HDMI_FC_ISCR2_8                         0x10AA
+#define HDMI_FC_ISCR2_7                         0x10AB
+#define HDMI_FC_ISCR2_6                         0x10AC
+#define HDMI_FC_ISCR2_5                         0x10AD
+#define HDMI_FC_ISCR2_4                         0x10AE
+#define HDMI_FC_ISCR2_3                         0x10AF
+#define HDMI_FC_ISCR2_2                         0x10B0
+#define HDMI_FC_ISCR2_1                         0x10B1
+#define HDMI_FC_ISCR2_0                         0x10B2
+#define HDMI_FC_DATAUTO0                        0x10B3
+#define HDMI_FC_DATAUTO1                        0x10B4
+#define HDMI_FC_DATAUTO2                        0x10B5
+#define HDMI_FC_DATMAN                          0x10B6
+#define HDMI_FC_DATAUTO3                        0x10B7
+#define HDMI_FC_RDRB0                           0x10B8
+#define HDMI_FC_RDRB1                           0x10B9
+#define HDMI_FC_RDRB2                           0x10BA
+#define HDMI_FC_RDRB3                           0x10BB
+#define HDMI_FC_RDRB4                           0x10BC
+#define HDMI_FC_RDRB5                           0x10BD
+#define HDMI_FC_RDRB6                           0x10BE
+#define HDMI_FC_RDRB7                           0x10BF
+#define HDMI_FC_STAT0                           0x10D0
+#define HDMI_FC_INT0                            0x10D1
+#define HDMI_FC_MASK0                           0x10D2
+#define HDMI_FC_POL0                            0x10D3
+#define HDMI_FC_STAT1                           0x10D4
+#define HDMI_FC_INT1                            0x10D5
+#define HDMI_FC_MASK1                           0x10D6
+#define HDMI_FC_POL1                            0x10D7
+#define HDMI_FC_STAT2                           0x10D8
+#define HDMI_FC_INT2                            0x10D9
+#define HDMI_FC_MASK2                           0x10DA
+#define HDMI_FC_POL2                            0x10DB
+#define HDMI_FC_PRCONF                          0x10E0
+#define HDMI_FC_SCRAMBLER_CTRL                  0x10E1
+#define HDMI_FC_PACKET_TX_EN                    0x10E3
+
+#define HDMI_FC_GMD_STAT                        0x1100
+#define HDMI_FC_GMD_EN                          0x1101
+#define HDMI_FC_GMD_UP                          0x1102
+#define HDMI_FC_GMD_CONF                        0x1103
+#define HDMI_FC_GMD_HB                          0x1104
+#define HDMI_FC_GMD_PB0                         0x1105
+#define HDMI_FC_GMD_PB1                         0x1106
+#define HDMI_FC_GMD_PB2                         0x1107
+#define HDMI_FC_GMD_PB3                         0x1108
+#define HDMI_FC_GMD_PB4                         0x1109
+#define HDMI_FC_GMD_PB5                         0x110A
+#define HDMI_FC_GMD_PB6                         0x110B
+#define HDMI_FC_GMD_PB7                         0x110C
+#define HDMI_FC_GMD_PB8                         0x110D
+#define HDMI_FC_GMD_PB9                         0x110E
+#define HDMI_FC_GMD_PB10                        0x110F
+#define HDMI_FC_GMD_PB11                        0x1110
+#define HDMI_FC_GMD_PB12                        0x1111
+#define HDMI_FC_GMD_PB13                        0x1112
+#define HDMI_FC_GMD_PB14                        0x1113
+#define HDMI_FC_GMD_PB15                        0x1114
+#define HDMI_FC_GMD_PB16                        0x1115
+#define HDMI_FC_GMD_PB17                        0x1116
+#define HDMI_FC_GMD_PB18                        0x1117
+#define HDMI_FC_GMD_PB19                        0x1118
+#define HDMI_FC_GMD_PB20                        0x1119
+#define HDMI_FC_GMD_PB21                        0x111A
+#define HDMI_FC_GMD_PB22                        0x111B
+#define HDMI_FC_GMD_PB23                        0x111C
+#define HDMI_FC_GMD_PB24                        0x111D
+#define HDMI_FC_GMD_PB25                        0x111E
+#define HDMI_FC_GMD_PB26                        0x111F
+#define HDMI_FC_GMD_PB27                        0x1120
+
+#define HDMI_FC_DRM_UP                          0x1167
+#define HDMI_FC_DRM_HB0                         0x1168
+#define HDMI_FC_DRM_HB1                         0x1169
+#define HDMI_FC_DRM_PB0                         0x116A
+#define HDMI_FC_DRM_PB1                         0x116B
+#define HDMI_FC_DRM_PB2                         0x116C
+#define HDMI_FC_DRM_PB3                         0x116D
+#define HDMI_FC_DRM_PB4                         0x116E
+#define HDMI_FC_DRM_PB5                         0x116F
+#define HDMI_FC_DRM_PB6                         0x1170
+#define HDMI_FC_DRM_PB7                         0x1171
+#define HDMI_FC_DRM_PB8                         0x1172
+#define HDMI_FC_DRM_PB9                         0x1173
+#define HDMI_FC_DRM_PB10                        0x1174
+#define HDMI_FC_DRM_PB11                        0x1175
+#define HDMI_FC_DRM_PB12                        0x1176
+#define HDMI_FC_DRM_PB13                        0x1177
+#define HDMI_FC_DRM_PB14                        0x1178
+#define HDMI_FC_DRM_PB15                        0x1179
+#define HDMI_FC_DRM_PB16                        0x117A
+#define HDMI_FC_DRM_PB17                        0x117B
+#define HDMI_FC_DRM_PB18                        0x117C
+#define HDMI_FC_DRM_PB19                        0x117D
+#define HDMI_FC_DRM_PB20                        0x117E
+#define HDMI_FC_DRM_PB21                        0x117F
+#define HDMI_FC_DRM_PB22                        0x1180
+#define HDMI_FC_DRM_PB23                        0x1181
+#define HDMI_FC_DRM_PB24                        0x1182
+#define HDMI_FC_DRM_PB25                        0x1183
+#define HDMI_FC_DRM_PB26                        0x1184
+
+#define HDMI_FC_DBGFORCE                        0x1200
+#define HDMI_FC_DBGAUD0CH0                      0x1201
+#define HDMI_FC_DBGAUD1CH0                      0x1202
+#define HDMI_FC_DBGAUD2CH0                      0x1203
+#define HDMI_FC_DBGAUD0CH1                      0x1204
+#define HDMI_FC_DBGAUD1CH1                      0x1205
+#define HDMI_FC_DBGAUD2CH1                      0x1206
+#define HDMI_FC_DBGAUD0CH2                      0x1207
+#define HDMI_FC_DBGAUD1CH2                      0x1208
+#define HDMI_FC_DBGAUD2CH2                      0x1209
+#define HDMI_FC_DBGAUD0CH3                      0x120A
+#define HDMI_FC_DBGAUD1CH3                      0x120B
+#define HDMI_FC_DBGAUD2CH3                      0x120C
+#define HDMI_FC_DBGAUD0CH4                      0x120D
+#define HDMI_FC_DBGAUD1CH4                      0x120E
+#define HDMI_FC_DBGAUD2CH4                      0x120F
+#define HDMI_FC_DBGAUD0CH5                      0x1210
+#define HDMI_FC_DBGAUD1CH5                      0x1211
+#define HDMI_FC_DBGAUD2CH5                      0x1212
+#define HDMI_FC_DBGAUD0CH6                      0x1213
+#define HDMI_FC_DBGAUD1CH6                      0x1214
+#define HDMI_FC_DBGAUD2CH6                      0x1215
+#define HDMI_FC_DBGAUD0CH7                      0x1216
+#define HDMI_FC_DBGAUD1CH7                      0x1217
+#define HDMI_FC_DBGAUD2CH7                      0x1218
+#define HDMI_FC_DBGTMDS0                        0x1219
+#define HDMI_FC_DBGTMDS1                        0x121A
+#define HDMI_FC_DBGTMDS2                        0x121B
+
+/* HDMI Source PHY Registers */
+#define HDMI_PHY_CONF0                          0x3000
+#define HDMI_PHY_TST0                           0x3001
+#define HDMI_PHY_TST1                           0x3002
+#define HDMI_PHY_TST2                           0x3003
+#define HDMI_PHY_STAT0                          0x3004
+#define HDMI_PHY_INT0                           0x3005
+#define HDMI_PHY_MASK0                          0x3006
+#define HDMI_PHY_POL0                           0x3007
+
+/* HDMI Master PHY Registers */
+#define HDMI_PHY_I2CM_SLAVE_ADDR                0x3020
+#define HDMI_PHY_I2CM_ADDRESS_ADDR              0x3021
+#define HDMI_PHY_I2CM_DATAO_1_ADDR              0x3022
+#define HDMI_PHY_I2CM_DATAO_0_ADDR              0x3023
+#define HDMI_PHY_I2CM_DATAI_1_ADDR              0x3024
+#define HDMI_PHY_I2CM_DATAI_0_ADDR              0x3025
+#define HDMI_PHY_I2CM_OPERATION_ADDR            0x3026
+#define HDMI_PHY_I2CM_INT_ADDR                  0x3027
+#define HDMI_PHY_I2CM_CTLINT_ADDR               0x3028
+#define HDMI_PHY_I2CM_DIV_ADDR                  0x3029
+#define HDMI_PHY_I2CM_SOFTRSTZ_ADDR             0x302a
+#define HDMI_PHY_I2CM_SS_SCL_HCNT_1_ADDR        0x302b
+#define HDMI_PHY_I2CM_SS_SCL_HCNT_0_ADDR        0x302c
+#define HDMI_PHY_I2CM_SS_SCL_LCNT_1_ADDR        0x302d
+#define HDMI_PHY_I2CM_SS_SCL_LCNT_0_ADDR        0x302e
+#define HDMI_PHY_I2CM_FS_SCL_HCNT_1_ADDR        0x302f
+#define HDMI_PHY_I2CM_FS_SCL_HCNT_0_ADDR        0x3030
+#define HDMI_PHY_I2CM_FS_SCL_LCNT_1_ADDR        0x3031
+#define HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR        0x3032
+
+/* Audio Sampler Registers */
+#define HDMI_AUD_CONF0                          0x3100
+#define HDMI_AUD_CONF1                          0x3101
+#define HDMI_AUD_INT                            0x3102
+#define HDMI_AUD_CONF2                          0x3103
+#define HDMI_AUD_N1                             0x3200
+#define HDMI_AUD_N2                             0x3201
+#define HDMI_AUD_N3                             0x3202
+#define HDMI_AUD_CTS1                           0x3203
+#define HDMI_AUD_CTS2                           0x3204
+#define HDMI_AUD_CTS3                           0x3205
+#define HDMI_AUD_INPUTCLKFS                     0x3206
+#define HDMI_AUD_SPDIFINT			0x3302
+#define HDMI_AUD_CONF0_HBR                      0x3400
+#define HDMI_AUD_HBR_STATUS                     0x3401
+#define HDMI_AUD_HBR_INT                        0x3402
+#define HDMI_AUD_HBR_POL                        0x3403
+#define HDMI_AUD_HBR_MASK                       0x3404
+
+/*
+ * Generic Parallel Audio Interface Registers
+ * Not used as GPAUD interface is not enabled in hw
+ */
+#define HDMI_GP_CONF0                           0x3500
+#define HDMI_GP_CONF1                           0x3501
+#define HDMI_GP_CONF2                           0x3502
+#define HDMI_GP_STAT                            0x3503
+#define HDMI_GP_INT                             0x3504
+#define HDMI_GP_MASK                            0x3505
+#define HDMI_GP_POL                             0x3506
+
+/* Audio DMA Registers */
+#define HDMI_AHB_DMA_CONF0                      0x3600
+#define HDMI_AHB_DMA_START                      0x3601
+#define HDMI_AHB_DMA_STOP                       0x3602
+#define HDMI_AHB_DMA_THRSLD                     0x3603
+#define HDMI_AHB_DMA_STRADDR0                   0x3604
+#define HDMI_AHB_DMA_STRADDR1                   0x3605
+#define HDMI_AHB_DMA_STRADDR2                   0x3606
+#define HDMI_AHB_DMA_STRADDR3                   0x3607
+#define HDMI_AHB_DMA_STPADDR0                   0x3608
+#define HDMI_AHB_DMA_STPADDR1                   0x3609
+#define HDMI_AHB_DMA_STPADDR2                   0x360a
+#define HDMI_AHB_DMA_STPADDR3                   0x360b
+#define HDMI_AHB_DMA_BSTADDR0                   0x360c
+#define HDMI_AHB_DMA_BSTADDR1                   0x360d
+#define HDMI_AHB_DMA_BSTADDR2                   0x360e
+#define HDMI_AHB_DMA_BSTADDR3                   0x360f
+#define HDMI_AHB_DMA_MBLENGTH0                  0x3610
+#define HDMI_AHB_DMA_MBLENGTH1                  0x3611
+#define HDMI_AHB_DMA_STAT                       0x3612
+#define HDMI_AHB_DMA_INT                        0x3613
+#define HDMI_AHB_DMA_MASK                       0x3614
+#define HDMI_AHB_DMA_POL                        0x3615
+#define HDMI_AHB_DMA_CONF1                      0x3616
+#define HDMI_AHB_DMA_BUFFSTAT                   0x3617
+#define HDMI_AHB_DMA_BUFFINT                    0x3618
+#define HDMI_AHB_DMA_BUFFMASK                   0x3619
+#define HDMI_AHB_DMA_BUFFPOL                    0x361a
+
+/* Main Controller Registers */
+#define HDMI_MC_SFRDIV                          0x4000
+#define HDMI_MC_CLKDIS                          0x4001
+#define HDMI_MC_SWRSTZ                          0x4002
+#define HDMI_MC_OPCTRL                          0x4003
+#define HDMI_MC_FLOWCTRL                        0x4004
+#define HDMI_MC_PHYRSTZ                         0x4005
+#define HDMI_MC_LOCKONCLOCK                     0x4006
+#define HDMI_MC_HEACPHY_RST                     0x4007
+
+/* Color Space  Converter Registers */
+#define HDMI_CSC_CFG                            0x4100
+#define HDMI_CSC_SCALE                          0x4101
+#define HDMI_CSC_COEF_A1_MSB                    0x4102
+#define HDMI_CSC_COEF_A1_LSB                    0x4103
+#define HDMI_CSC_COEF_A2_MSB                    0x4104
+#define HDMI_CSC_COEF_A2_LSB                    0x4105
+#define HDMI_CSC_COEF_A3_MSB                    0x4106
+#define HDMI_CSC_COEF_A3_LSB                    0x4107
+#define HDMI_CSC_COEF_A4_MSB                    0x4108
+#define HDMI_CSC_COEF_A4_LSB                    0x4109
+#define HDMI_CSC_COEF_B1_MSB                    0x410A
+#define HDMI_CSC_COEF_B1_LSB                    0x410B
+#define HDMI_CSC_COEF_B2_MSB                    0x410C
+#define HDMI_CSC_COEF_B2_LSB                    0x410D
+#define HDMI_CSC_COEF_B3_MSB                    0x410E
+#define HDMI_CSC_COEF_B3_LSB                    0x410F
+#define HDMI_CSC_COEF_B4_MSB                    0x4110
+#define HDMI_CSC_COEF_B4_LSB                    0x4111
+#define HDMI_CSC_COEF_C1_MSB                    0x4112
+#define HDMI_CSC_COEF_C1_LSB                    0x4113
+#define HDMI_CSC_COEF_C2_MSB                    0x4114
+#define HDMI_CSC_COEF_C2_LSB                    0x4115
+#define HDMI_CSC_COEF_C3_MSB                    0x4116
+#define HDMI_CSC_COEF_C3_LSB                    0x4117
+#define HDMI_CSC_COEF_C4_MSB                    0x4118
+#define HDMI_CSC_COEF_C4_LSB                    0x4119
+
+/* HDCP Encryption Engine Registers */
+#define HDMI_A_HDCPCFG0                         0x5000
+#define HDMI_A_HDCPCFG1                         0x5001
+#define HDMI_A_HDCPOBS0                         0x5002
+#define HDMI_A_HDCPOBS1                         0x5003
+#define HDMI_A_HDCPOBS2                         0x5004
+#define HDMI_A_HDCPOBS3                         0x5005
+#define HDMI_A_APIINTCLR                        0x5006
+#define HDMI_A_APIINTSTAT                       0x5007
+#define HDMI_A_APIINTMSK                        0x5008
+#define HDMI_A_VIDPOLCFG                        0x5009
+#define HDMI_A_OESSWCFG                         0x500A
+#define HDMI_A_TIMER1SETUP0                     0x500B
+#define HDMI_A_TIMER1SETUP1                     0x500C
+#define HDMI_A_TIMER2SETUP0                     0x500D
+#define HDMI_A_TIMER2SETUP1                     0x500E
+#define HDMI_A_100MSCFG                         0x500F
+#define HDMI_A_2SCFG0                           0x5010
+#define HDMI_A_2SCFG1                           0x5011
+#define HDMI_A_5SCFG0                           0x5012
+#define HDMI_A_5SCFG1                           0x5013
+#define HDMI_A_SRMVERLSB                        0x5014
+#define HDMI_A_SRMVERMSB                        0x5015
+#define HDMI_A_SRMCTRL                          0x5016
+#define HDMI_A_SFRSETUP                         0x5017
+#define HDMI_A_I2CHSETUP                        0x5018
+#define HDMI_A_INTSETUP                         0x5019
+#define HDMI_A_PRESETUP                         0x501A
+#define HDMI_A_SRM_BASE                         0x5020
+
+/* HDCP Registers */
+#define HDMI_HDCPREG_RMCTL                      0x780e
+#define HDMI_HDCPREG_RMSTS                      0x780f
+#define HDMI_HDCPREG_SEED0                      0x7810
+#define HDMI_HDCPREG_SEED1                      0x7811
+#define HDMI_HDCPREG_DPK0                       0x7812
+#define HDMI_HDCPREG_DPK1                       0x7813
+#define HDMI_HDCPREG_DPK2                       0x7814
+#define HDMI_HDCPREG_DPK3                       0x7815
+#define HDMI_HDCPREG_DPK4                       0x7816
+#define HDMI_HDCPREG_DPK5                       0x7817
+#define HDMI_HDCPREG_DPK6                       0x7818
+#define HDMI_HDCP2REG_CTRL                      0x7904
+#define HDMI_HDCP2REG_MASK                      0x790c
+#define HDMI_HDCP2REG_MUTE                      0x7912
+
+/* CEC Engine Registers */
+#define HDMI_CEC_CTRL                           0x7D00
+#define HDMI_CEC_STAT                           0x7D01
+#define HDMI_CEC_MASK                           0x7D02
+#define HDMI_CEC_POLARITY                       0x7D03
+#define HDMI_CEC_INT                            0x7D04
+#define HDMI_CEC_ADDR_L                         0x7D05
+#define HDMI_CEC_ADDR_H                         0x7D06
+#define HDMI_CEC_TX_CNT                         0x7D07
+#define HDMI_CEC_RX_CNT                         0x7D08
+#define HDMI_CEC_TX_DATA0                       0x7D10
+#define HDMI_CEC_TX_DATA1                       0x7D11
+#define HDMI_CEC_TX_DATA2                       0x7D12
+#define HDMI_CEC_TX_DATA3                       0x7D13
+#define HDMI_CEC_TX_DATA4                       0x7D14
+#define HDMI_CEC_TX_DATA5                       0x7D15
+#define HDMI_CEC_TX_DATA6                       0x7D16
+#define HDMI_CEC_TX_DATA7                       0x7D17
+#define HDMI_CEC_TX_DATA8                       0x7D18
+#define HDMI_CEC_TX_DATA9                       0x7D19
+#define HDMI_CEC_TX_DATA10                      0x7D1a
+#define HDMI_CEC_TX_DATA11                      0x7D1b
+#define HDMI_CEC_TX_DATA12                      0x7D1c
+#define HDMI_CEC_TX_DATA13                      0x7D1d
+#define HDMI_CEC_TX_DATA14                      0x7D1e
+#define HDMI_CEC_TX_DATA15                      0x7D1f
+#define HDMI_CEC_RX_DATA0                       0x7D20
+#define HDMI_CEC_RX_DATA1                       0x7D21
+#define HDMI_CEC_RX_DATA2                       0x7D22
+#define HDMI_CEC_RX_DATA3                       0x7D23
+#define HDMI_CEC_RX_DATA4                       0x7D24
+#define HDMI_CEC_RX_DATA5                       0x7D25
+#define HDMI_CEC_RX_DATA6                       0x7D26
+#define HDMI_CEC_RX_DATA7                       0x7D27
+#define HDMI_CEC_RX_DATA8                       0x7D28
+#define HDMI_CEC_RX_DATA9                       0x7D29
+#define HDMI_CEC_RX_DATA10                      0x7D2a
+#define HDMI_CEC_RX_DATA11                      0x7D2b
+#define HDMI_CEC_RX_DATA12                      0x7D2c
+#define HDMI_CEC_RX_DATA13                      0x7D2d
+#define HDMI_CEC_RX_DATA14                      0x7D2e
+#define HDMI_CEC_RX_DATA15                      0x7D2f
+#define HDMI_CEC_LOCK                           0x7D30
+#define HDMI_CEC_WKUPCTRL                       0x7D31
+
+/* I2C Master Registers (E-DDC) */
+#define HDMI_I2CM_SLAVE                         0x7E00
+#define HDMI_I2CM_ADDRESS                       0x7E01
+#define HDMI_I2CM_DATAO                         0x7E02
+#define HDMI_I2CM_DATAI                         0x7E03
+#define HDMI_I2CM_OPERATION                     0x7E04
+#define HDMI_I2CM_INT                           0x7E05
+#define HDMI_I2CM_CTLINT                        0x7E06
+#define HDMI_I2CM_DIV                           0x7E07
+#define HDMI_I2CM_SEGADDR                       0x7E08
+#define HDMI_I2CM_SOFTRSTZ                      0x7E09
+#define HDMI_I2CM_SEGPTR                        0x7E0A
+#define HDMI_I2CM_SS_SCL_HCNT_1_ADDR            0x7E0B
+#define HDMI_I2CM_SS_SCL_HCNT_0_ADDR            0x7E0C
+#define HDMI_I2CM_SS_SCL_LCNT_1_ADDR            0x7E0D
+#define HDMI_I2CM_SS_SCL_LCNT_0_ADDR            0x7E0E
+#define HDMI_I2CM_FS_SCL_HCNT_1_ADDR            0x7E0F
+#define HDMI_I2CM_FS_SCL_HCNT_0_ADDR            0x7E10
+#define HDMI_I2CM_FS_SCL_LCNT_1_ADDR            0x7E11
+#define HDMI_I2CM_FS_SCL_LCNT_0_ADDR            0x7E12
+#define HDMI_I2CM_SDA_HOLD                      0x7E13
+
+enum {
+/* PRODUCT_ID0 field values */
+	HDMI_PRODUCT_ID0_HDMI_TX = 0xa0,
+
+/* PRODUCT_ID1 field values */
+	HDMI_PRODUCT_ID1_HDCP = 0xc0,
+	HDMI_PRODUCT_ID1_HDMI_RX = 0x02,
+	HDMI_PRODUCT_ID1_HDMI_TX = 0x01,
+
+/* CONFIG0_ID field values */
+	HDMI_CONFIG0_I2S = 0x10,
+	HDMI_CONFIG0_CEC = 0x02,
+
+/* CONFIG1_ID field values */
+	HDMI_CONFIG1_AHB = 0x01,
+
+/* CONFIG3_ID field values */
+	HDMI_CONFIG3_AHBAUDDMA = 0x02,
+	HDMI_CONFIG3_GPAUD = 0x01,
+
+/* IH_FC_INT2 field values */
+	HDMI_IH_FC_INT2_OVERFLOW_MASK = 0x03,
+	HDMI_IH_FC_INT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_IH_FC_INT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* IH_FC_STAT2 field values */
+	HDMI_IH_FC_STAT2_OVERFLOW_MASK = 0x03,
+	HDMI_IH_FC_STAT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_IH_FC_STAT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* IH_PHY_STAT0 field values */
+	HDMI_IH_PHY_STAT0_RX_SENSE3 = 0x20,
+	HDMI_IH_PHY_STAT0_RX_SENSE2 = 0x10,
+	HDMI_IH_PHY_STAT0_RX_SENSE1 = 0x8,
+	HDMI_IH_PHY_STAT0_RX_SENSE0 = 0x4,
+	HDMI_IH_PHY_STAT0_TX_PHY_LOCK = 0x2,
+	HDMI_IH_PHY_STAT0_HPD = 0x1,
+
+/* IH_I2CM_STAT0 and IH_MUTE_I2CM_STAT0 field values */
+	HDMI_IH_I2CM_STAT0_DONE = 0x2,
+	HDMI_IH_I2CM_STAT0_ERROR = 0x1,
+
+/* IH_MUTE_I2CMPHY_STAT0 field values */
+	HDMI_IH_MUTE_I2CMPHY_STAT0_I2CMPHYDONE = 0x2,
+	HDMI_IH_MUTE_I2CMPHY_STAT0_I2CMPHYERROR = 0x1,
+
+/* IH_AHBDMAAUD_STAT0 field values */
+	HDMI_IH_AHBDMAAUD_STAT0_ERROR = 0x20,
+	HDMI_IH_AHBDMAAUD_STAT0_LOST = 0x10,
+	HDMI_IH_AHBDMAAUD_STAT0_RETRY = 0x08,
+	HDMI_IH_AHBDMAAUD_STAT0_DONE = 0x04,
+	HDMI_IH_AHBDMAAUD_STAT0_BUFFFULL = 0x02,
+	HDMI_IH_AHBDMAAUD_STAT0_BUFFEMPTY = 0x01,
+
+/* IH_MUTE_FC_STAT2 field values */
+	HDMI_IH_MUTE_FC_STAT2_OVERFLOW_MASK = 0x03,
+	HDMI_IH_MUTE_FC_STAT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_IH_MUTE_FC_STAT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* IH_MUTE_AHBDMAAUD_STAT0 field values */
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_ERROR = 0x20,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_LOST = 0x10,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_RETRY = 0x08,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_DONE = 0x04,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFFULL = 0x02,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFEMPTY = 0x01,
+
+/* IH_MUTE field values */
+	HDMI_IH_MUTE_MUTE_WAKEUP_INTERRUPT = 0x2,
+	HDMI_IH_MUTE_MUTE_ALL_INTERRUPT = 0x1,
+
+/* TX_INVID0 field values */
+	HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_MASK = 0x80,
+	HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_ENABLE = 0x80,
+	HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_DISABLE = 0x00,
+	HDMI_TX_INVID0_VIDEO_MAPPING_MASK = 0x1F,
+	HDMI_TX_INVID0_VIDEO_MAPPING_OFFSET = 0,
+
+/* TX_INSTUFFING field values */
+	HDMI_TX_INSTUFFING_BDBDATA_STUFFING_MASK = 0x4,
+	HDMI_TX_INSTUFFING_BDBDATA_STUFFING_ENABLE = 0x4,
+	HDMI_TX_INSTUFFING_BDBDATA_STUFFING_DISABLE = 0x0,
+	HDMI_TX_INSTUFFING_RCRDATA_STUFFING_MASK = 0x2,
+	HDMI_TX_INSTUFFING_RCRDATA_STUFFING_ENABLE = 0x2,
+	HDMI_TX_INSTUFFING_RCRDATA_STUFFING_DISABLE = 0x0,
+	HDMI_TX_INSTUFFING_GYDATA_STUFFING_MASK = 0x1,
+	HDMI_TX_INSTUFFING_GYDATA_STUFFING_ENABLE = 0x1,
+	HDMI_TX_INSTUFFING_GYDATA_STUFFING_DISABLE = 0x0,
+
+/* VP_PR_CD field values */
+	HDMI_VP_PR_CD_COLOR_DEPTH_MASK = 0xF0,
+	HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET = 4,
+	HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK = 0x0F,
+	HDMI_VP_PR_CD_DESIRED_PR_FACTOR_OFFSET = 0,
+
+/* VP_STUFF field values */
+	HDMI_VP_STUFF_IDEFAULT_PHASE_MASK = 0x20,
+	HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET = 5,
+	HDMI_VP_STUFF_IFIX_PP_TO_LAST_MASK = 0x10,
+	HDMI_VP_STUFF_IFIX_PP_TO_LAST_OFFSET = 4,
+	HDMI_VP_STUFF_ICX_GOTO_P0_ST_MASK = 0x8,
+	HDMI_VP_STUFF_ICX_GOTO_P0_ST_OFFSET = 3,
+	HDMI_VP_STUFF_YCC422_STUFFING_MASK = 0x4,
+	HDMI_VP_STUFF_YCC422_STUFFING_STUFFING_MODE = 0x4,
+	HDMI_VP_STUFF_YCC422_STUFFING_DIRECT_MODE = 0x0,
+	HDMI_VP_STUFF_PP_STUFFING_MASK = 0x2,
+	HDMI_VP_STUFF_PP_STUFFING_STUFFING_MODE = 0x2,
+	HDMI_VP_STUFF_PP_STUFFING_DIRECT_MODE = 0x0,
+	HDMI_VP_STUFF_PR_STUFFING_MASK = 0x1,
+	HDMI_VP_STUFF_PR_STUFFING_STUFFING_MODE = 0x1,
+	HDMI_VP_STUFF_PR_STUFFING_DIRECT_MODE = 0x0,
+
+/* VP_CONF field values */
+	HDMI_VP_CONF_BYPASS_EN_MASK = 0x40,
+	HDMI_VP_CONF_BYPASS_EN_ENABLE = 0x40,
+	HDMI_VP_CONF_BYPASS_EN_DISABLE = 0x00,
+	HDMI_VP_CONF_PP_EN_ENMASK = 0x20,
+	HDMI_VP_CONF_PP_EN_ENABLE = 0x20,
+	HDMI_VP_CONF_PP_EN_DISABLE = 0x00,
+	HDMI_VP_CONF_PR_EN_MASK = 0x10,
+	HDMI_VP_CONF_PR_EN_ENABLE = 0x10,
+	HDMI_VP_CONF_PR_EN_DISABLE = 0x00,
+	HDMI_VP_CONF_YCC422_EN_MASK = 0x8,
+	HDMI_VP_CONF_YCC422_EN_ENABLE = 0x8,
+	HDMI_VP_CONF_YCC422_EN_DISABLE = 0x0,
+	HDMI_VP_CONF_BYPASS_SELECT_MASK = 0x4,
+	HDMI_VP_CONF_BYPASS_SELECT_VID_PACKETIZER = 0x4,
+	HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER = 0x0,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_MASK = 0x3,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS = 0x3,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_YCC422 = 0x1,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_PP = 0x0,
+
+/* VP_REMAP field values */
+	HDMI_VP_REMAP_MASK = 0x3,
+	HDMI_VP_REMAP_YCC422_24bit = 0x2,
+	HDMI_VP_REMAP_YCC422_20bit = 0x1,
+	HDMI_VP_REMAP_YCC422_16bit = 0x0,
+
+/* FC_INVIDCONF field values */
+	HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK = 0x80,
+	HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE = 0x80,
+	HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE = 0x00,
+	HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_MASK = 0x40,
+	HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH = 0x40,
+	HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_LOW = 0x00,
+	HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_MASK = 0x20,
+	HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_HIGH = 0x20,
+	HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_LOW = 0x00,
+	HDMI_FC_INVIDCONF_DE_IN_POLARITY_MASK = 0x10,
+	HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_HIGH = 0x10,
+	HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_LOW = 0x00,
+	HDMI_FC_INVIDCONF_DVI_MODEZ_MASK = 0x8,
+	HDMI_FC_INVIDCONF_DVI_MODEZ_HDMI_MODE = 0x8,
+	HDMI_FC_INVIDCONF_DVI_MODEZ_DVI_MODE = 0x0,
+	HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_MASK = 0x2,
+	HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH = 0x2,
+	HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_LOW = 0x0,
+	HDMI_FC_INVIDCONF_IN_I_P_MASK = 0x1,
+	HDMI_FC_INVIDCONF_IN_I_P_INTERLACED = 0x1,
+	HDMI_FC_INVIDCONF_IN_I_P_PROGRESSIVE = 0x0,
+
+/* FC_AUDICONF0 field values */
+	HDMI_FC_AUDICONF0_CC_OFFSET = 4,
+	HDMI_FC_AUDICONF0_CC_MASK = 0x70,
+	HDMI_FC_AUDICONF0_CT_OFFSET = 0,
+	HDMI_FC_AUDICONF0_CT_MASK = 0xF,
+
+/* FC_AUDICONF1 field values */
+	HDMI_FC_AUDICONF1_SS_OFFSET = 3,
+	HDMI_FC_AUDICONF1_SS_MASK = 0x18,
+	HDMI_FC_AUDICONF1_SF_OFFSET = 0,
+	HDMI_FC_AUDICONF1_SF_MASK = 0x7,
+
+/* FC_AUDICONF3 field values */
+	HDMI_FC_AUDICONF3_LFEPBL_OFFSET = 5,
+	HDMI_FC_AUDICONF3_LFEPBL_MASK = 0x60,
+	HDMI_FC_AUDICONF3_DM_INH_OFFSET = 4,
+	HDMI_FC_AUDICONF3_DM_INH_MASK = 0x10,
+	HDMI_FC_AUDICONF3_LSV_OFFSET = 0,
+	HDMI_FC_AUDICONF3_LSV_MASK = 0xF,
+
+/* FC_AUDSCHNLS0 field values */
+	HDMI_FC_AUDSCHNLS0_CGMSA_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS0_CGMSA_MASK = 0x30,
+	HDMI_FC_AUDSCHNLS0_COPYRIGHT_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS0_COPYRIGHT_MASK = 0x01,
+
+/* FC_AUDSCHNLS3-6 field values */
+	HDMI_FC_AUDSCHNLS3_OIEC_CH0_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS3_OIEC_CH0_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS3_OIEC_CH1_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS3_OIEC_CH1_MASK = 0xf0,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH2_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH2_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH3_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH3_MASK = 0xf0,
+
+	HDMI_FC_AUDSCHNLS5_OIEC_CH0_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS5_OIEC_CH0_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS5_OIEC_CH1_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS5_OIEC_CH1_MASK = 0xf0,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH2_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH2_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH3_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH3_MASK = 0xf0,
+
+/* HDMI_FC_AUDSCHNLS7 field values */
+	HDMI_FC_AUDSCHNLS7_ACCURACY_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS7_ACCURACY_MASK = 0x30,
+
+/* HDMI_FC_AUDSCHNLS8 field values */
+	HDMI_FC_AUDSCHNLS8_ORIGSAMPFREQ_MASK = 0xf0,
+	HDMI_FC_AUDSCHNLS8_ORIGSAMPFREQ_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS8_WORDLEGNTH_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS8_WORDLEGNTH_OFFSET = 0,
+
+/* FC_AUDSCONF field values */
+	HDMI_FC_AUDSCONF_AUD_PACKET_SAMPFIT_MASK = 0xF0,
+	HDMI_FC_AUDSCONF_AUD_PACKET_SAMPFIT_OFFSET = 4,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_MASK = 0x1,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_OFFSET = 0,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT1 = 0x1,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT0 = 0x0,
+
+/* FC_STAT2 field values */
+	HDMI_FC_STAT2_OVERFLOW_MASK = 0x03,
+	HDMI_FC_STAT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_FC_STAT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* FC_INT2 field values */
+	HDMI_FC_INT2_OVERFLOW_MASK = 0x03,
+	HDMI_FC_INT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_FC_INT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* FC_MASK2 field values */
+	HDMI_FC_MASK2_OVERFLOW_MASK = 0x03,
+	HDMI_FC_MASK2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_FC_MASK2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* FC_PRCONF field values */
+	HDMI_FC_PRCONF_INCOMING_PR_FACTOR_MASK = 0xF0,
+	HDMI_FC_PRCONF_INCOMING_PR_FACTOR_OFFSET = 4,
+	HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK = 0x0F,
+	HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_OFFSET = 0,
+
+/* FC_PACKET_TX_EN field values */
+	HDMI_FC_PACKET_TX_EN_DRM_MASK = 0x80,
+	HDMI_FC_PACKET_TX_EN_DRM_ENABLE = 0x80,
+	HDMI_FC_PACKET_TX_EN_DRM_DISABLE = 0x00,
+
+/* FC_AVICONF0-FC_AVICONF3 field values */
+	HDMI_FC_AVICONF0_PIX_FMT_MASK = 0x03,
+	HDMI_FC_AVICONF0_PIX_FMT_RGB = 0x00,
+	HDMI_FC_AVICONF0_PIX_FMT_YCBCR422 = 0x01,
+	HDMI_FC_AVICONF0_PIX_FMT_YCBCR444 = 0x02,
+	HDMI_FC_AVICONF0_ACTIVE_FMT_MASK = 0x40,
+	HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT = 0x40,
+	HDMI_FC_AVICONF0_ACTIVE_FMT_NO_INFO = 0x00,
+	HDMI_FC_AVICONF0_BAR_DATA_MASK = 0x0C,
+	HDMI_FC_AVICONF0_BAR_DATA_NO_DATA = 0x00,
+	HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR = 0x04,
+	HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR = 0x08,
+	HDMI_FC_AVICONF0_BAR_DATA_VERT_HORIZ_BAR = 0x0C,
+	HDMI_FC_AVICONF0_SCAN_INFO_MASK = 0x30,
+	HDMI_FC_AVICONF0_SCAN_INFO_OVERSCAN = 0x10,
+	HDMI_FC_AVICONF0_SCAN_INFO_UNDERSCAN = 0x20,
+	HDMI_FC_AVICONF0_SCAN_INFO_NODATA = 0x00,
+
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_MASK = 0x0F,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_USE_CODED = 0x08,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_4_3 = 0x09,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_16_9 = 0x0A,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_14_9 = 0x0B,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_MASK = 0x30,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_NO_DATA = 0x00,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_4_3 = 0x10,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_16_9 = 0x20,
+	HDMI_FC_AVICONF1_COLORIMETRY_MASK = 0xC0,
+	HDMI_FC_AVICONF1_COLORIMETRY_NO_DATA = 0x00,
+	HDMI_FC_AVICONF1_COLORIMETRY_SMPTE = 0x40,
+	HDMI_FC_AVICONF1_COLORIMETRY_ITUR = 0x80,
+	HDMI_FC_AVICONF1_COLORIMETRY_EXTENDED_INFO = 0xC0,
+
+	HDMI_FC_AVICONF2_SCALING_MASK = 0x03,
+	HDMI_FC_AVICONF2_SCALING_NONE = 0x00,
+	HDMI_FC_AVICONF2_SCALING_HORIZ = 0x01,
+	HDMI_FC_AVICONF2_SCALING_VERT = 0x02,
+	HDMI_FC_AVICONF2_SCALING_HORIZ_VERT = 0x03,
+	HDMI_FC_AVICONF2_RGB_QUANT_MASK = 0x0C,
+	HDMI_FC_AVICONF2_RGB_QUANT_DEFAULT = 0x00,
+	HDMI_FC_AVICONF2_RGB_QUANT_LIMITED_RANGE = 0x04,
+	HDMI_FC_AVICONF2_RGB_QUANT_FULL_RANGE = 0x08,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_MASK = 0x70,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC601 = 0x00,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC709 = 0x10,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_SYCC601 = 0x20,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_ADOBE_YCC601 = 0x30,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_ADOBE_RGB = 0x40,
+	HDMI_FC_AVICONF2_IT_CONTENT_MASK = 0x80,
+	HDMI_FC_AVICONF2_IT_CONTENT_NO_DATA = 0x00,
+	HDMI_FC_AVICONF2_IT_CONTENT_VALID = 0x80,
+
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_MASK = 0x03,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_GRAPHICS = 0x00,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_PHOTO = 0x01,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_CINEMA = 0x02,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_GAME = 0x03,
+	HDMI_FC_AVICONF3_QUANT_RANGE_MASK = 0x0C,
+	HDMI_FC_AVICONF3_QUANT_RANGE_LIMITED = 0x00,
+	HDMI_FC_AVICONF3_QUANT_RANGE_FULL = 0x04,
+
+/* HDMI_FC_GCP */
+	HDMI_FC_GCP_SET_AVMUTE = 0x2,
+	HDMI_FC_GCP_CLEAR_AVMUTE = 0x1,
+
+/* FC_DBGFORCE field values */
+	HDMI_FC_DBGFORCE_FORCEAUDIO = 0x10,
+	HDMI_FC_DBGFORCE_FORCEVIDEO = 0x1,
+
+/* FC_DATAUTO0 field values */
+	HDMI_FC_DATAUTO0_VSD_MASK = 0x08,
+	HDMI_FC_DATAUTO0_VSD_OFFSET = 3,
+
+/* FC_DATAUTO3 field values */
+	HDMI_FC_DATAUTO3_GCP_AUTO = 0x04,
+
+/* PHY_CONF0 field values */
+	HDMI_PHY_CONF0_PDZ_MASK = 0x80,
+	HDMI_PHY_CONF0_PDZ_OFFSET = 7,
+	HDMI_PHY_CONF0_ENTMDS_MASK = 0x40,
+	HDMI_PHY_CONF0_ENTMDS_OFFSET = 6,
+	HDMI_PHY_CONF0_SVSRET_MASK = 0x20,
+	HDMI_PHY_CONF0_SVSRET_OFFSET = 5,
+	HDMI_PHY_CONF0_GEN2_PDDQ_MASK = 0x10,
+	HDMI_PHY_CONF0_GEN2_PDDQ_OFFSET = 4,
+	HDMI_PHY_CONF0_GEN2_TXPWRON_MASK = 0x8,
+	HDMI_PHY_CONF0_GEN2_TXPWRON_OFFSET = 3,
+	HDMI_PHY_CONF0_GEN2_ENHPDRXSENSE_MASK = 0x4,
+	HDMI_PHY_CONF0_GEN2_ENHPDRXSENSE_OFFSET = 2,
+	HDMI_PHY_CONF0_SELDATAENPOL_MASK = 0x2,
+	HDMI_PHY_CONF0_SELDATAENPOL_OFFSET = 1,
+	HDMI_PHY_CONF0_SELDIPIF_MASK = 0x1,
+	HDMI_PHY_CONF0_SELDIPIF_OFFSET = 0,
+
+/* PHY_TST0 field values */
+	HDMI_PHY_TST0_TSTCLR_MASK = 0x20,
+	HDMI_PHY_TST0_TSTCLR_OFFSET = 5,
+	HDMI_PHY_TST0_TSTEN_MASK = 0x10,
+	HDMI_PHY_TST0_TSTEN_OFFSET = 4,
+	HDMI_PHY_TST0_TSTCLK_MASK = 0x1,
+	HDMI_PHY_TST0_TSTCLK_OFFSET = 0,
+
+/* PHY_STAT0 field values */
+	HDMI_PHY_RX_SENSE3 = 0x80,
+	HDMI_PHY_RX_SENSE2 = 0x40,
+	HDMI_PHY_RX_SENSE1 = 0x20,
+	HDMI_PHY_RX_SENSE0 = 0x10,
+	HDMI_PHY_HPD = 0x02,
+	HDMI_PHY_TX_PHY_LOCK = 0x01,
+
+/* PHY_I2CM_SLAVE_ADDR field values */
+	HDMI_PHY_I2CM_SLAVE_ADDR_PHY_GEN2 = 0x69,
+	HDMI_PHY_I2CM_SLAVE_ADDR_HEAC_PHY = 0x49,
+
+/* PHY_I2CM_OPERATION_ADDR field values */
+	HDMI_PHY_I2CM_OPERATION_ADDR_WRITE = 0x10,
+	HDMI_PHY_I2CM_OPERATION_ADDR_READ = 0x1,
+
+/* HDMI_PHY_I2CM_INT_ADDR */
+	HDMI_PHY_I2CM_INT_ADDR_DONE_POL = 0x08,
+	HDMI_PHY_I2CM_INT_ADDR_DONE_MASK = 0x04,
+
+/* HDMI_PHY_I2CM_CTLINT_ADDR */
+	HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL = 0x80,
+	HDMI_PHY_I2CM_CTLINT_ADDR_NAC_MASK = 0x40,
+	HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL = 0x08,
+	HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_MASK = 0x04,
+
+/* AUD_CONF0 field values */
+	HDMI_AUD_CONF0_SW_RESET = 0x80,
+	HDMI_AUD_CONF0_I2S_SELECT = 0x20,
+	HDMI_AUD_CONF0_I2S_EN3 = 0x08,
+	HDMI_AUD_CONF0_I2S_EN2 = 0x04,
+	HDMI_AUD_CONF0_I2S_EN1 = 0x02,
+	HDMI_AUD_CONF0_I2S_EN0 = 0x01,
+
+/* AUD_CONF1 field values */
+	HDMI_AUD_CONF1_MODE_I2S = 0x00,
+	HDMI_AUD_CONF1_MODE_RIGHT_J = 0x20,
+	HDMI_AUD_CONF1_MODE_LEFT_J = 0x40,
+	HDMI_AUD_CONF1_MODE_BURST_1 = 0x60,
+	HDMI_AUD_CONF1_MODE_BURST_2 = 0x80,
+	HDMI_AUD_CONF1_WIDTH_16 = 0x10,
+	HDMI_AUD_CONF1_WIDTH_24 = 0x18,
+
+/* AUD_CTS3 field values */
+	HDMI_AUD_CTS3_N_SHIFT_OFFSET = 5,
+	HDMI_AUD_CTS3_N_SHIFT_MASK = 0xe0,
+	HDMI_AUD_CTS3_N_SHIFT_1 = 0,
+	HDMI_AUD_CTS3_N_SHIFT_16 = 0x20,
+	HDMI_AUD_CTS3_N_SHIFT_32 = 0x40,
+	HDMI_AUD_CTS3_N_SHIFT_64 = 0x60,
+	HDMI_AUD_CTS3_N_SHIFT_128 = 0x80,
+	HDMI_AUD_CTS3_N_SHIFT_256 = 0xa0,
+	/* note that the CTS3 MANUAL bit has been removed
+	   from our part. Can't set it, will read as 0. */
+	HDMI_AUD_CTS3_CTS_MANUAL = 0x10,
+	HDMI_AUD_CTS3_AUDCTS19_16_MASK = 0x0f,
+
+/* HDMI_AUD_INPUTCLKFS field values */
+	HDMI_AUD_INPUTCLKFS_128FS = 0,
+	HDMI_AUD_INPUTCLKFS_256FS = 1,
+	HDMI_AUD_INPUTCLKFS_512FS = 2,
+	HDMI_AUD_INPUTCLKFS_64FS = 4,
+
+/* AHB_DMA_CONF0 field values */
+	HDMI_AHB_DMA_CONF0_SW_FIFO_RST_OFFSET = 7,
+	HDMI_AHB_DMA_CONF0_SW_FIFO_RST_MASK = 0x80,
+	HDMI_AHB_DMA_CONF0_HBR = 0x10,
+	HDMI_AHB_DMA_CONF0_EN_HLOCK_OFFSET = 3,
+	HDMI_AHB_DMA_CONF0_EN_HLOCK_MASK = 0x08,
+	HDMI_AHB_DMA_CONF0_INCR_TYPE_OFFSET = 1,
+	HDMI_AHB_DMA_CONF0_INCR_TYPE_MASK = 0x06,
+	HDMI_AHB_DMA_CONF0_INCR4 = 0x0,
+	HDMI_AHB_DMA_CONF0_INCR8 = 0x2,
+	HDMI_AHB_DMA_CONF0_INCR16 = 0x4,
+	HDMI_AHB_DMA_CONF0_BURST_MODE = 0x1,
+
+/* HDMI_AHB_DMA_START field values */
+	HDMI_AHB_DMA_START_START_OFFSET = 0,
+	HDMI_AHB_DMA_START_START_MASK = 0x01,
+
+/* HDMI_AHB_DMA_STOP field values */
+	HDMI_AHB_DMA_STOP_STOP_OFFSET = 0,
+	HDMI_AHB_DMA_STOP_STOP_MASK = 0x01,
+
+/* AHB_DMA_STAT, AHB_DMA_INT, AHB_DMA_MASK, AHB_DMA_POL field values */
+	HDMI_AHB_DMA_DONE = 0x80,
+	HDMI_AHB_DMA_RETRY_SPLIT = 0x40,
+	HDMI_AHB_DMA_LOSTOWNERSHIP = 0x20,
+	HDMI_AHB_DMA_ERROR = 0x10,
+	HDMI_AHB_DMA_FIFO_THREMPTY = 0x04,
+	HDMI_AHB_DMA_FIFO_FULL = 0x02,
+	HDMI_AHB_DMA_FIFO_EMPTY = 0x01,
+
+/* AHB_DMA_BUFFSTAT, AHB_DMA_BUFFINT,AHB_DMA_BUFFMASK,AHB_DMA_BUFFPOL values */
+	HDMI_AHB_DMA_BUFFSTAT_FULL = 0x02,
+	HDMI_AHB_DMA_BUFFSTAT_EMPTY = 0x01,
+
+/* MC_CLKDIS field values */
+	HDMI_MC_CLKDIS_HDCPCLK_DISABLE = 0x40,
+	HDMI_MC_CLKDIS_CECCLK_DISABLE = 0x20,
+	HDMI_MC_CLKDIS_CSCCLK_DISABLE = 0x10,
+	HDMI_MC_CLKDIS_AUDCLK_DISABLE = 0x8,
+	HDMI_MC_CLKDIS_PREPCLK_DISABLE = 0x4,
+	HDMI_MC_CLKDIS_TMDSCLK_DISABLE = 0x2,
+	HDMI_MC_CLKDIS_PIXELCLK_DISABLE = 0x1,
+	HDMI_MC_CLKDIS_PIXELCLK_MASK = 0x1,
+	HDMI_MC_CLKDIS_PIXELCLK_ENABLE = 0,
+
+/* MC_SWRSTZ field values */
+	HDMI_MC_SWRSTZ_I2SSWRST_REQ = 0x08,
+	HDMI_MC_SWRSTZ_TMDSSWRST_REQ = 0x02,
+
+/* MC_FLOWCTRL field values */
+	HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_MASK = 0x1,
+	HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_IN_PATH = 0x1,
+	HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_BYPASS = 0x0,
+
+/* MC_PHYRSTZ field values */
+	HDMI_MC_PHYRSTZ_PHYRSTZ = 0x01,
+
+/* MC_HEACPHY_RST field values */
+	HDMI_MC_HEACPHY_RST_ASSERT = 0x1,
+	HDMI_MC_HEACPHY_RST_DEASSERT = 0x0,
+
+/* CSC_CFG field values */
+	HDMI_CSC_CFG_INTMODE_MASK = 0x30,
+	HDMI_CSC_CFG_INTMODE_OFFSET = 4,
+	HDMI_CSC_CFG_INTMODE_DISABLE = 0x00,
+	HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA1 = 0x10,
+	HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA2 = 0x20,
+	HDMI_CSC_CFG_DECMODE_MASK = 0x3,
+	HDMI_CSC_CFG_DECMODE_OFFSET = 0,
+	HDMI_CSC_CFG_DECMODE_DISABLE = 0x0,
+	HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA1 = 0x1,
+	HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA2 = 0x2,
+	HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA3 = 0x3,
+
+/* CSC_SCALE field values */
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_MASK = 0xF0,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_24BPP = 0x00,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_30BPP = 0x50,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_36BPP = 0x60,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_48BPP = 0x70,
+	HDMI_CSC_SCALE_CSCSCALE_MASK = 0x03,
+
+/* A_HDCPCFG0 field values */
+	HDMI_A_HDCPCFG0_ELVENA_MASK = 0x80,
+	HDMI_A_HDCPCFG0_ELVENA_ENABLE = 0x80,
+	HDMI_A_HDCPCFG0_ELVENA_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_I2CFASTMODE_MASK = 0x40,
+	HDMI_A_HDCPCFG0_I2CFASTMODE_ENABLE = 0x40,
+	HDMI_A_HDCPCFG0_I2CFASTMODE_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_BYPENCRYPTION_MASK = 0x20,
+	HDMI_A_HDCPCFG0_BYPENCRYPTION_ENABLE = 0x20,
+	HDMI_A_HDCPCFG0_BYPENCRYPTION_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_SYNCRICHECK_MASK = 0x10,
+	HDMI_A_HDCPCFG0_SYNCRICHECK_ENABLE = 0x10,
+	HDMI_A_HDCPCFG0_SYNCRICHECK_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_AVMUTE_MASK = 0x8,
+	HDMI_A_HDCPCFG0_AVMUTE_ENABLE = 0x8,
+	HDMI_A_HDCPCFG0_AVMUTE_DISABLE = 0x0,
+	HDMI_A_HDCPCFG0_RXDETECT_MASK = 0x4,
+	HDMI_A_HDCPCFG0_RXDETECT_ENABLE = 0x4,
+	HDMI_A_HDCPCFG0_RXDETECT_DISABLE = 0x0,
+	HDMI_A_HDCPCFG0_EN11FEATURE_MASK = 0x2,
+	HDMI_A_HDCPCFG0_EN11FEATURE_ENABLE = 0x2,
+	HDMI_A_HDCPCFG0_EN11FEATURE_DISABLE = 0x0,
+	HDMI_A_HDCPCFG0_HDMIDVI_MASK = 0x1,
+	HDMI_A_HDCPCFG0_HDMIDVI_HDMI = 0x1,
+	HDMI_A_HDCPCFG0_HDMIDVI_DVI = 0x0,
+
+/* A_HDCPCFG1 field values */
+	HDMI_A_HDCPCFG1_DISSHA1CHECK_MASK = 0x8,
+	HDMI_A_HDCPCFG1_DISSHA1CHECK_DISABLE = 0x8,
+	HDMI_A_HDCPCFG1_DISSHA1CHECK_ENABLE = 0x0,
+	HDMI_A_HDCPCFG1_PH2UPSHFTENC_MASK = 0x4,
+	HDMI_A_HDCPCFG1_PH2UPSHFTENC_ENABLE = 0x4,
+	HDMI_A_HDCPCFG1_PH2UPSHFTENC_DISABLE = 0x0,
+	HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK = 0x2,
+	HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE = 0x2,
+	HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_ENABLE = 0x0,
+	HDMI_A_HDCPCFG1_SWRESET_MASK = 0x1,
+	HDMI_A_HDCPCFG1_SWRESET_ASSERT = 0x0,
+
+/* A_VIDPOLCFG field values */
+	HDMI_A_VIDPOLCFG_UNENCRYPTCONF_MASK = 0x60,
+	HDMI_A_VIDPOLCFG_UNENCRYPTCONF_OFFSET = 5,
+	HDMI_A_VIDPOLCFG_DATAENPOL_MASK = 0x10,
+	HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH = 0x10,
+	HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW = 0x0,
+	HDMI_A_VIDPOLCFG_VSYNCPOL_MASK = 0x8,
+	HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_HIGH = 0x8,
+	HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_LOW = 0x0,
+	HDMI_A_VIDPOLCFG_HSYNCPOL_MASK = 0x2,
+	HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH = 0x2,
+	HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW = 0x0,
+
+/* I2CM_OPERATION field values */
+	HDMI_I2CM_OPERATION_BUSCLEAR = 0x20,
+	HDMI_I2CM_OPERATION_WRITE = 0x10,
+	HDMI_I2CM_OPERATION_READ_EXT = 0x2,
+	HDMI_I2CM_OPERATION_READ = 0x1,
+
+/* I2CM_INT field values */
+	HDMI_I2CM_INT_DONE_POL = 0x8,
+	HDMI_I2CM_INT_DONE_MASK = 0x4,
+
+/* I2CM_CTLINT field values */
+	HDMI_I2CM_CTLINT_NAC_POL = 0x80,
+	HDMI_I2CM_CTLINT_NAC_MASK = 0x40,
+	HDMI_I2CM_CTLINT_ARB_POL = 0x8,
+	HDMI_I2CM_CTLINT_ARB_MASK = 0x4,
+};
+
+/*
+ * HDMI 3D TX PHY registers
+ */
+#define HDMI_3D_TX_PHY_PWRCTRL			0x00
+#define HDMI_3D_TX_PHY_SERDIVCTRL		0x01
+#define HDMI_3D_TX_PHY_SERCKCTRL		0x02
+#define HDMI_3D_TX_PHY_SERCKKILLCTRL		0x03
+#define HDMI_3D_TX_PHY_TXRESCTRL		0x04
+#define HDMI_3D_TX_PHY_CKCALCTRL		0x05
+#define HDMI_3D_TX_PHY_CPCE_CTRL		0x06
+#define HDMI_3D_TX_PHY_TXCLKMEASCTRL		0x07
+#define HDMI_3D_TX_PHY_TXMEASCTRL		0x08
+#define HDMI_3D_TX_PHY_CKSYMTXCTRL		0x09
+#define HDMI_3D_TX_PHY_CMPSEQCTRL		0x0a
+#define HDMI_3D_TX_PHY_CMPPWRCTRL		0x0b
+#define HDMI_3D_TX_PHY_CMPMODECTRL		0x0c
+#define HDMI_3D_TX_PHY_MEASCTRL			0x0d
+#define HDMI_3D_TX_PHY_VLEVCTRL			0x0e
+#define HDMI_3D_TX_PHY_D2ACTRL			0x0f
+#define HDMI_3D_TX_PHY_CURRCTRL			0x10
+#define HDMI_3D_TX_PHY_DRVANACTRL		0x11
+#define HDMI_3D_TX_PHY_PLLMEASCTRL		0x12
+#define HDMI_3D_TX_PHY_PLLPHBYCTRL		0x13
+#define HDMI_3D_TX_PHY_GRP_CTRL			0x14
+#define HDMI_3D_TX_PHY_GMPCTRL			0x15
+#define HDMI_3D_TX_PHY_MPLLMEASCTRL		0x16
+#define HDMI_3D_TX_PHY_MSM_CTRL			0x17
+#define HDMI_3D_TX_PHY_SCRPB_STATUS		0x18
+#define HDMI_3D_TX_PHY_TXTERM			0x19
+#define HDMI_3D_TX_PHY_PTRPT_ENBL		0x1a
+#define HDMI_3D_TX_PHY_PATTERNGEN		0x1b
+#define HDMI_3D_TX_PHY_SDCAP_MODE		0x1c
+#define HDMI_3D_TX_PHY_SCOPEMODE		0x1d
+#define HDMI_3D_TX_PHY_DIGTXMODE		0x1e
+#define HDMI_3D_TX_PHY_STR_STATUS		0x1f
+#define HDMI_3D_TX_PHY_SCOPECNT0		0x20
+#define HDMI_3D_TX_PHY_SCOPECNT1		0x21
+#define HDMI_3D_TX_PHY_SCOPECNT2		0x22
+#define HDMI_3D_TX_PHY_SCOPECNTCLK		0x23
+#define HDMI_3D_TX_PHY_SCOPESAMPLE		0x24
+#define HDMI_3D_TX_PHY_SCOPECNTMSB01		0x25
+#define HDMI_3D_TX_PHY_SCOPECNTMSB2CK		0x26
+
+/* HDMI_3D_TX_PHY_CKCALCTRL values */
+#define HDMI_3D_TX_PHY_CKCALCTRL_OVERRIDE		BIT(15)
+
+/* HDMI_3D_TX_PHY_MSM_CTRL values */
+#define HDMI_3D_TX_PHY_MSM_CTRL_MPLL_PH_SEL_CK		BIT(13)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_CLK_REF_MPLL	(0 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_OFF		(1 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_PCLK		(2 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_FB_CLK		(3 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_SCOPE_CK_SEL		BIT(0)
+
+/* HDMI_3D_TX_PHY_PTRPT_ENBL values */
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_OVERRIDE		BIT(15)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_PG_SKIP_BIT2		BIT(8)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_PG_SKIP_BIT1		BIT(7)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_PG_SKIP_BIT0		BIT(6)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_CK_REF_ENB		BIT(5)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_RCAL_ENB		BIT(4)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_TX_CLK_ALIGN_ENB	BIT(3)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_TX_READY		BIT(2)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_CKO_WORD_ENB		BIT(1)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_REFCLK_ENB		BIT(0)
+
+struct dw_hdmi;
+
+struct dw_hdmi_property_ops {
+	void (*attatch_properties)(struct drm_connector *connector,
+				   unsigned int color, int version, void *data);
+	void (*destroy_properties)(struct drm_connector *connector, void *data);
+	int (*set_property)(struct drm_connector *connector,
+			    struct drm_connector_state *state,
+			    struct drm_property *property, u64 val, void *data);
+	int (*get_property)(struct drm_connector *connector,
+			    const struct drm_connector_state *state,
+			    struct drm_property *property, u64 *val,
+			    void *data);
+};
+
+enum supported_eotf_type {
+	TRADITIONAL_GAMMA_SDR = 0,
+	TRADITIONAL_GAMMA_HDR,
+	SMPTE_ST2084,
+	HLG,
+	FUTURE_EOTF
+};
+
+void dw_hdmi_suspend(struct dw_hdmi *hdmi);
+void dw_hdmi_enable_video(struct dw_hdmi *hdmi);
+void dw_hdmi_disable_video(struct dw_hdmi *hdmi);
+#endif /* __DW_HDMI_H__ */
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_audio.h b/drivers/gpu/drm/eswin/dw_hdmi_audio.h
new file mode 100644
index 000000000000..105f5e71babd
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_audio.h
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN hdmi driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+
+#ifndef DW_HDMI_AUDIO_H
+#define DW_HDMI_AUDIO_H
+
+struct dw_hdmi;
+
+struct dw_hdmi_audio_data {
+	phys_addr_t phys;
+	void __iomem *base;
+	int irq;
+	struct dw_hdmi *hdmi;
+	u8 *(*get_eld)(struct dw_hdmi *hdmi);
+};
+
+struct dw_hdmi_i2s_audio_data {
+	struct dw_hdmi *hdmi;
+
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	u8 *(*get_eld)(struct dw_hdmi *hdmi);
+};
+
+#endif
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_cec.c b/drivers/gpu/drm/eswin/dw_hdmi_cec.c
new file mode 100644
index 000000000000..90e8c973ef06
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_cec.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Designware HDMI CEC driver
+ *
+ * Copyright (C) 2015-2017 Russell King.
+ *****************************************************************************
+ * ESWIN hdmi driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <drm/drm_edid.h>
+
+#include <media/cec.h>
+#include <media/cec-notifier.h>
+
+#include "dw_hdmi_cec.h"
+
+#define CEC_IO_ADDR 0x516000fc
+
+enum {
+	HDMI_IH_CEC_STAT0 = 0x0106,
+	HDMI_IH_MUTE_CEC_STAT0 = 0x0186,
+
+	HDMI_CEC_CTRL = 0x7d00,
+	CEC_CTRL_START = BIT(0),
+	CEC_CTRL_FRAME_TYP = 3 << 1,
+	CEC_CTRL_RETRY = 0 << 1,
+	CEC_CTRL_NORMAL = 1 << 1,
+	CEC_CTRL_IMMED = 2 << 1,
+
+	HDMI_CEC_STAT = 0x7d01,
+	CEC_STAT_DONE = BIT(0),
+	CEC_STAT_EOM = BIT(1),
+	CEC_STAT_NACK = BIT(2),
+	CEC_STAT_ARBLOST = BIT(3),
+	CEC_STAT_ERROR_INIT = BIT(4),
+	CEC_STAT_ERROR_FOLL = BIT(5),
+	CEC_STAT_WAKEUP = BIT(6),
+
+	HDMI_CEC_MASK = 0x7d02,
+	HDMI_CEC_POLARITY = 0x7d03,
+	HDMI_CEC_INT = 0x7d04,
+	HDMI_CEC_ADDR_L = 0x7d05,
+	HDMI_CEC_ADDR_H = 0x7d06,
+	HDMI_CEC_TX_CNT = 0x7d07,
+	HDMI_CEC_RX_CNT = 0x7d08,
+	HDMI_CEC_TX_DATA0 = 0x7d10,
+	HDMI_CEC_RX_DATA0 = 0x7d20,
+	HDMI_CEC_LOCK = 0x7d30,
+	HDMI_CEC_WKUPCTRL = 0x7d31,
+};
+
+struct dw_hdmi_cec {
+	struct dw_hdmi *hdmi;
+	const struct dw_hdmi_cec_ops *ops;
+	u32 addresses;
+	struct cec_adapter *adap;
+	struct cec_msg rx_msg;
+	unsigned int tx_status;
+	bool tx_done;
+	bool rx_done;
+	struct cec_notifier *notify;
+	int irq;
+};
+
+static void dw_hdmi_write(struct dw_hdmi_cec *cec, u8 val, int offset)
+{
+	cec->ops->write(cec->hdmi, val, offset);
+}
+
+static u8 dw_hdmi_read(struct dw_hdmi_cec *cec, int offset)
+{
+	return cec->ops->read(cec->hdmi, offset);
+}
+
+static int dw_hdmi_cec_log_addr(struct cec_adapter *adap, u8 logical_addr)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (logical_addr == CEC_LOG_ADDR_INVALID)
+		cec->addresses = 0;
+	else
+		cec->addresses |= BIT(logical_addr) | BIT(15);
+
+	dw_hdmi_write(cec, cec->addresses & 255, HDMI_CEC_ADDR_L);
+	dw_hdmi_write(cec, cec->addresses >> 8, HDMI_CEC_ADDR_H);
+
+	return 0;
+}
+
+static int dw_hdmi_cec_transmit(struct cec_adapter *adap, u8 attempts,
+				u32 signal_free_time, struct cec_msg *msg)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+	unsigned int i, ctrl;
+
+	switch (signal_free_time) {
+	case CEC_SIGNAL_FREE_TIME_RETRY:
+		ctrl = CEC_CTRL_RETRY;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEW_INITIATOR:
+	default:
+		ctrl = CEC_CTRL_NORMAL;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEXT_XFER:
+		ctrl = CEC_CTRL_IMMED;
+		break;
+	}
+
+	for (i = 0; i < msg->len; i++)
+		dw_hdmi_write(cec, msg->msg[i], HDMI_CEC_TX_DATA0 + i);
+
+	dw_hdmi_write(cec, msg->len, HDMI_CEC_TX_CNT);
+	dw_hdmi_write(cec, ctrl | CEC_CTRL_START, HDMI_CEC_CTRL);
+
+	return 0;
+}
+
+static irqreturn_t dw_hdmi_cec_hardirq(int irq, void *data)
+{
+	struct cec_adapter *adap = data;
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+	unsigned int stat = dw_hdmi_read(cec, HDMI_IH_CEC_STAT0);
+	irqreturn_t ret = IRQ_HANDLED;
+
+	if (stat == 0)
+		return IRQ_NONE;
+
+	dw_hdmi_write(cec, stat, HDMI_IH_CEC_STAT0);
+
+	if (stat & CEC_STAT_ERROR_INIT) {
+		cec->tx_status = CEC_TX_STATUS_ERROR;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	} else if (stat & CEC_STAT_DONE) {
+		cec->tx_status = CEC_TX_STATUS_OK;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	} else if (stat & CEC_STAT_NACK) {
+		cec->tx_status = CEC_TX_STATUS_NACK;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	if (stat & CEC_STAT_EOM) {
+		unsigned int len, i;
+
+		len = dw_hdmi_read(cec, HDMI_CEC_RX_CNT);
+		if (len > sizeof(cec->rx_msg.msg))
+			len = sizeof(cec->rx_msg.msg);
+
+		for (i = 0; i < len; i++)
+			cec->rx_msg.msg[i] =
+				dw_hdmi_read(cec, HDMI_CEC_RX_DATA0 + i);
+
+		dw_hdmi_write(cec, 0, HDMI_CEC_LOCK);
+
+		cec->rx_msg.len = len;
+		smp_wmb();
+		cec->rx_done = true;
+
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	return ret;
+}
+
+static irqreturn_t dw_hdmi_cec_thread(int irq, void *data)
+{
+	struct cec_adapter *adap = data;
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (cec->tx_done) {
+		cec->tx_done = false;
+		cec_transmit_attempt_done(adap, cec->tx_status);
+	}
+	if (cec->rx_done) {
+		cec->rx_done = false;
+		smp_rmb();
+		cec_received_msg(adap, &cec->rx_msg);
+	}
+	return IRQ_HANDLED;
+}
+
+static int dw_hdmi_cec_enable(struct cec_adapter *adap, bool enable)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (!enable) {
+		dw_hdmi_write(cec, ~0, HDMI_CEC_MASK);
+		dw_hdmi_write(cec, ~0, HDMI_IH_MUTE_CEC_STAT0);
+		dw_hdmi_write(cec, 0, HDMI_CEC_POLARITY);
+
+		cec->ops->disable(cec->hdmi);
+	} else {
+		unsigned int irqs;
+
+		dw_hdmi_write(cec, 0, HDMI_CEC_CTRL);
+		dw_hdmi_write(cec, ~0, HDMI_IH_CEC_STAT0);
+		dw_hdmi_write(cec, 0, HDMI_CEC_LOCK);
+
+		dw_hdmi_cec_log_addr(cec->adap, CEC_LOG_ADDR_INVALID);
+
+		cec->ops->enable(cec->hdmi);
+
+		irqs = CEC_STAT_ERROR_INIT | CEC_STAT_NACK | CEC_STAT_EOM |
+		       CEC_STAT_DONE;
+		dw_hdmi_write(cec, irqs, HDMI_CEC_POLARITY);
+		dw_hdmi_write(cec, ~irqs, HDMI_CEC_MASK);
+		dw_hdmi_write(cec, ~irqs, HDMI_IH_MUTE_CEC_STAT0);
+	}
+	return 0;
+}
+
+static const struct cec_adap_ops dw_hdmi_cec_ops = {
+	.adap_enable = dw_hdmi_cec_enable,
+	.adap_log_addr = dw_hdmi_cec_log_addr,
+	.adap_transmit = dw_hdmi_cec_transmit,
+};
+
+static void dw_hdmi_cec_del(void *data)
+{
+	struct dw_hdmi_cec *cec = data;
+
+	cec_delete_adapter(cec->adap);
+}
+
+static int dw_hdmi_cec_probe(struct platform_device *pdev)
+{
+	struct dw_hdmi_cec_data *data = dev_get_platdata(&pdev->dev);
+	struct dw_hdmi_cec *cec;
+	int ret;
+	void __iomem *cec_io_base;
+	int reg_val;
+
+	if (!data)
+		return -ENXIO;
+
+	/*
+     * Our device is just a convenience - we want to link to the real
+     * hardware device here, so that userspace can see the association
+     * between the HDMI hardware and its associated CEC chardev.
+     */
+	cec = devm_kzalloc(&pdev->dev, sizeof(*cec), GFP_KERNEL);
+	if (!cec)
+		return -ENOMEM;
+
+	cec->irq = data->irq;
+	cec->ops = data->ops;
+	cec->hdmi = data->hdmi;
+
+	platform_set_drvdata(pdev, cec);
+
+	dw_hdmi_write(cec, 0, HDMI_CEC_TX_CNT);
+	dw_hdmi_write(cec, ~0, HDMI_CEC_MASK);
+	dw_hdmi_write(cec, ~0, HDMI_IH_MUTE_CEC_STAT0);
+	dw_hdmi_write(cec, 0, HDMI_CEC_POLARITY);
+
+	cec->adap =
+		cec_allocate_adapter(&dw_hdmi_cec_ops, cec, "dw_hdmi",
+				     CEC_CAP_DEFAULTS | CEC_CAP_CONNECTOR_INFO,
+				     CEC_MAX_LOG_ADDRS);
+	if (IS_ERR(cec->adap))
+		return PTR_ERR(cec->adap);
+
+	/* override the module pointer */
+	cec->adap->owner = THIS_MODULE;
+
+	ret = devm_add_action_or_reset(&pdev->dev, dw_hdmi_cec_del, cec);
+	if (ret)
+		return ret;
+
+	ret = devm_request_threaded_irq(&pdev->dev, cec->irq,
+					dw_hdmi_cec_hardirq, dw_hdmi_cec_thread,
+					IRQF_SHARED, "dw-hdmi-cec", cec->adap);
+	if (ret < 0)
+		return ret;
+
+	cec->notify = cec_notifier_cec_adap_register(pdev->dev.parent, NULL,
+						     cec->adap);
+	if (!cec->notify)
+		return -ENOMEM;
+
+	ret = cec_register_adapter(cec->adap, pdev->dev.parent);
+	if (ret < 0) {
+		cec_notifier_cec_adap_unregister(cec->notify, cec->adap);
+		return ret;
+	}
+
+	/*
+     * CEC documentation says we must not call cec_delete_adapter
+     * after a successful call to cec_register_adapter().
+     */
+	devm_remove_action(&pdev->dev, dw_hdmi_cec_del, cec);
+
+	cec_io_base = devm_ioremap(&pdev->dev, CEC_IO_ADDR, 4);
+	if (!cec_io_base) {
+		dev_err(&pdev->dev, "failed to remap cec io ctl\n");
+		return -ENOMEM;
+	}
+
+	/* set the cec io to output */
+	reg_val = readl((char *)cec_io_base);
+	reg_val |= 0x1;
+	writel(reg_val, (char *)cec_io_base);
+
+	return 0;
+}
+
+static int dw_hdmi_cec_remove(struct platform_device *pdev)
+{
+	struct dw_hdmi_cec *cec = platform_get_drvdata(pdev);
+
+	cec_notifier_cec_adap_unregister(cec->notify, cec->adap);
+	cec_unregister_adapter(cec->adap);
+
+	return 0;
+}
+
+struct platform_driver dw_hdmi_cec_driver = {
+    .probe  = dw_hdmi_cec_probe,
+    .remove = dw_hdmi_cec_remove,
+    .driver = {
+        .name = "dw-hdmi-cec",
+    },
+};
+//module_platform_driver(dw_hdmi_cec_driver);
+
+MODULE_DESCRIPTION("Synopsys Designware HDMI CEC driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS(PLATFORM_MODULE_PREFIX "dw-hdmi-cec");
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_cec.h b/drivers/gpu/drm/eswin/dw_hdmi_cec.h
new file mode 100644
index 000000000000..a43854725d88
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_cec.h
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN hdmi driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+
+#ifndef DW_HDMI_CEC_H
+#define DW_HDMI_CEC_H
+
+struct dw_hdmi;
+
+struct dw_hdmi_cec_ops {
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	void (*enable)(struct dw_hdmi *hdmi);
+	void (*disable)(struct dw_hdmi *hdmi);
+};
+
+struct dw_hdmi_cec_data {
+	struct dw_hdmi *hdmi;
+	const struct dw_hdmi_cec_ops *ops;
+	int irq;
+};
+#endif
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_hdcp.c b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.c
new file mode 100644
index 000000000000..8147044d12c9
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.c
@@ -0,0 +1,735 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN hdcp driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/hdmi.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/kthread.h>
+#include <linux/mutex.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include "dw-hdmi.h"
+#include "dw_hdmi_hdcp.h"
+
+#define HDCP_KEY_PATH "/usr/hdcp1.4_key/Tx_A2_TestDPK_encrypted"
+
+#define HDCP_KEY_SIZE 308
+#define HDCP_KEY_SEED_SIZE 2
+
+#define KSV_LEN 5
+#define HEADER 10
+#define SHAMAX 20
+
+#define MAX_DOWNSTREAM_DEVICE_NUM 5
+#define DPK_WR_OK_TIMEOUT_US 30000
+#define HDMI_HDCP1X_ID 5
+
+/* HDCP Registers */
+#define HDMI_A_KSVMEMCTRL 0x5016
+#define HDMI_HDCPREG_ANCONF 0x7805
+#define HDMI_HDCPREG_AN0 0x7806
+#define HDMI_HDCPREG_AN1 0x7807
+#define HDMI_HDCPREG_AN2 0x7808
+#define HDMI_HDCPREG_AN3 0x7809
+#define HDMI_HDCPREG_AN4 0x780a
+#define HDMI_HDCPREG_AN5 0x780b
+#define HDMI_HDCPREG_AN6 0x780c
+#define HDMI_HDCPREG_AN7 0x780d
+#define HDMI_HDCPREG_RMCTL 0x780e
+#define HDMI_HDCPREG_RMSTS 0x780f
+#define HDMI_HDCPREG_SEED0 0x7810
+#define HDMI_HDCPREG_SEED1 0x7811
+#define HDMI_HDCPREG_DPK0 0x7812
+#define HDMI_HDCPREG_DPK1 0x7813
+#define HDMI_HDCPREG_DPK2 0x7814
+#define HDMI_HDCPREG_DPK3 0x7815
+#define HDMI_HDCPREG_DPK4 0x7816
+#define HDMI_HDCPREG_DPK5 0x7817
+#define HDMI_HDCPREG_DPK6 0x7818
+#define HDMI_HDCP2REG_CTRL 0x7904
+#define HDMI_HDCP2REG_MASK 0x790c
+#define HDMI_HDCP2REG_MUTE 0x7912
+
+enum dw_hdmi_hdcp_state {
+	DW_HDCP_DISABLED,
+	DW_HDCP_AUTH_START,
+	DW_HDCP_AUTH_SUCCESS,
+	DW_HDCP_AUTH_FAIL,
+};
+
+enum {
+	DW_HDMI_HDCP_KSV_LEN = 8,
+	DW_HDMI_HDCP_SHA_LEN = 20,
+	DW_HDMI_HDCP_DPK_LEN = 280,
+	DW_HDMI_HDCP_KEY_LEN = 308,
+	DW_HDMI_HDCP_SEED_LEN = 2,
+};
+
+enum {
+	HDMI_MC_CLKDIS_HDCPCLK_MASK = 0x40,
+	HDMI_MC_CLKDIS_HDCPCLK_ENABLE = 0x00,
+
+	HDMI_A_SRMCTRL_SHA1_FAIL_MASK = 0X08,
+	HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_SHA1_FAIL_ENABLE = 0X08,
+
+	HDMI_A_SRMCTRL_KSV_UPDATE_MASK = 0X04,
+	HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE = 0X04,
+
+	HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK = 0X01,
+	HDMI_A_SRMCTRL_KSV_MEM_REQ_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_KSV_MEM_REQ_ENABLE = 0X01,
+
+	HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK = 0X02,
+	HDMI_A_SRMCTRL_KSV_MEM_ACCESS_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_KSV_MEM_ACCESS_ENABLE = 0X02,
+
+	HDMI_A_SRM_BASE_MAX_DEVS_EXCEEDED = 0x80,
+	HDMI_A_SRM_BASE_DEVICE_COUNT = 0x7f,
+
+	HDMI_A_SRM_BASE_MAX_CASCADE_EXCEEDED = 0x08,
+
+	HDMI_A_APIINTSTAT_KSVSHA1_CALC_INT = 0x02,
+	HDMI_A_APIINTSTAT_KSVSHA1_CALC_DONE_INT = 0x20,
+	/* HDCPREG_RMSTS field values */
+	DPK_WR_OK_STS = 0x40,
+
+	HDMI_A_HDCP22_MASK = 0x40,
+
+	HDMI_HDCP2_OVR_EN_MASK = 0x02,
+	HDMI_HDCP2_OVR_ENABLE = 0x02,
+	HDMI_HDCP2_OVR_DISABLE = 0x00,
+
+	HDMI_HDCP2_FORCE_MASK = 0x04,
+	HDMI_HDCP2_FORCE_ENABLE = 0x04,
+	HDMI_HDCP2_FORCE_DISABLE = 0x00,
+	HDMI_A_KSVMEMCTRL_KSV_SHA1_STATUS = 0x08,
+};
+
+static struct dw_hdcp *g_hdcp[2];
+static int trytimes = 0;
+
+static void hdcp_modb(struct dw_hdcp *hdcp, u8 data, u8 mask, unsigned int reg)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	u8 val = hdcp->read(hdmi, reg) & ~mask;
+
+	val |= data & mask;
+	hdcp->write(hdmi, val, reg);
+}
+
+static int hdcp_load_keys_cb(struct dw_hdcp *hdcp)
+{
+	u8 hdcp_vendor_data[320];
+	struct file *fp;
+	loff_t pos = 0;
+	ssize_t nread;
+
+	hdcp->keys = kmalloc(HDCP_KEY_SIZE, GFP_KERNEL);
+	if (!hdcp->keys)
+		return -ENOMEM;
+
+	hdcp->seeds = kmalloc(HDCP_KEY_SEED_SIZE, GFP_KERNEL);
+	if (!hdcp->seeds) {
+		kfree(hdcp->keys);
+		return -ENOMEM;
+	}
+
+	fp = filp_open(HDCP_KEY_PATH, O_RDONLY, 0644);
+	if (IS_ERR(fp)) {
+		printk("Error, Tx_A2_TestDPK_encrypted doesn't exist.\n");
+		goto err;
+	}
+
+	nread = kernel_read(fp, hdcp_vendor_data, sizeof(hdcp_vendor_data),
+			    &pos);
+
+	if (nread != sizeof(hdcp_vendor_data)) {
+		printk("Error, failed to read %ld bytes to non volatile memory area,ret %ld\n",
+		       sizeof(hdcp_vendor_data), nread);
+		filp_close(fp, NULL);
+		goto err;
+	}
+
+	memcpy(hdcp->keys, hdcp_vendor_data, HDCP_KEY_SIZE);
+	memcpy(hdcp->seeds, hdcp_vendor_data + HDCP_KEY_SIZE,
+	       HDCP_KEY_SEED_SIZE);
+
+	filp_close(fp, NULL);
+	return 0;
+err:
+	kfree(hdcp->keys);
+	kfree(hdcp->seeds);
+	hdcp->keys = NULL;
+	hdcp->seeds = NULL;
+	return -EIO;
+}
+
+static int dw_hdmi_hdcp_load_key(struct dw_hdcp *hdcp)
+{
+	int i, j;
+	int ret, val;
+	void __iomem *reg_rmsts_addr;
+	struct hdcp_keys *hdcp_keys;
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+
+	if (!hdcp->keys) {
+		ret = hdcp_load_keys_cb(hdcp);
+		if (ret)
+			return ret;
+	}
+	hdcp_keys = hdcp->keys;
+
+	if (hdcp->reg_io_width == 4)
+		reg_rmsts_addr = hdcp->regs + (HDMI_HDCPREG_RMSTS << 2);
+	else if (hdcp->reg_io_width == 1)
+		reg_rmsts_addr = hdcp->regs + HDMI_HDCPREG_RMSTS;
+	else
+		return -EPERM;
+
+	/* Disable decryption logic */
+	hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL);
+	ret = readx_poll_timeout(readl, reg_rmsts_addr, val,
+				 val & DPK_WR_OK_STS, 1000,
+				 DPK_WR_OK_TIMEOUT_US);
+	if (ret)
+		return ret;
+
+	hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK6);
+	hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK5);
+
+	/* The useful data in ksv should be 5 byte */
+	for (i = 4; i >= 0; i--)
+		hdcp->write(hdmi, hdcp_keys->KSV[i], HDMI_HDCPREG_DPK0 + i);
+	ret = readx_poll_timeout(readl, reg_rmsts_addr, val,
+				 val & DPK_WR_OK_STS, 1000,
+				 DPK_WR_OK_TIMEOUT_US);
+
+	if (ret)
+		return ret;
+
+	/* Enable decryption logic */
+	if (hdcp->seeds) {
+		hdcp->write(hdmi, 1, HDMI_HDCPREG_RMCTL);
+		hdcp->write(hdmi, hdcp->seeds[0], HDMI_HDCPREG_SEED1);
+		hdcp->write(hdmi, hdcp->seeds[1], HDMI_HDCPREG_SEED0);
+	} else {
+		hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL);
+	}
+
+	/* Write encrypt device private key */
+	for (i = 0; i < DW_HDMI_HDCP_DPK_LEN - 6; i += 7) {
+		for (j = 6; j >= 0; j--)
+			hdcp->write(hdmi, hdcp_keys->devicekey[i + j],
+				    HDMI_HDCPREG_DPK0 + j);
+		ret = readx_poll_timeout(readl, reg_rmsts_addr, val,
+					 val & DPK_WR_OK_STS, 1000,
+					 DPK_WR_OK_TIMEOUT_US);
+
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+static int dw_hdmi_hdcp1x_start(struct dw_hdcp *hdcp)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	int i;
+	int val;
+	u8 An[8];
+	int ret;
+
+	if (!hdcp->enable)
+		return -EPERM;
+
+	if (hdcp->status == DW_HDCP_AUTH_START ||
+	    hdcp->status == DW_HDCP_AUTH_SUCCESS)
+		return 0;
+
+	/* Update An */
+	get_random_bytes(&An, sizeof(An));
+	for (i = 0; i < 8; i++)
+		hdcp->write(hdmi, An[i], HDMI_HDCPREG_AN0 + i);
+
+	hdcp->write(hdmi, 0x01, HDMI_HDCPREG_ANCONF);
+
+	if (!(hdcp->read(hdmi, HDMI_HDCPREG_RMSTS) & 0x3f)) {
+		ret = dw_hdmi_hdcp_load_key(hdcp);
+		if (ret) {
+			dev_err(hdcp->dev, "load hdcp key failed, ret=%d\n", ret);
+			return ret;
+		}
+	}
+
+	/* disable the pixel clock*/
+	dev_dbg(hdcp->dev, "start hdcp with disable hdmi pixel clock\n");
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_DISABLE,
+		  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+
+	if (hdcp->hdcp2) {
+		for (i = 0; i < 100; i++) {
+			if (hdcp->hdcp2->wait_hdcp2_reset) {
+				msleep(80);
+			} else {
+				break;
+			}
+		}
+		printk("wait_hdcp2_reset i = %d\n", i);
+	}
+	val = hdcp->read(hdmi, HDMI_HDCP2REG_CTRL);
+	dev_dbg(hdcp->dev, "before set HDMI_HDCP2REG_CTRL val = %d\n", val);
+
+	hdcp_modb(hdcp, HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE,
+		  HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK, HDMI_FC_INVIDCONF);
+
+	hdcp->remaining_times = hdcp->retry_times;
+	if (hdcp->read(hdmi, HDMI_CONFIG1_ID) & HDMI_A_HDCP22_MASK) {
+		hdcp_modb(hdcp,
+			  HDMI_HDCP2_OVR_ENABLE | HDMI_HDCP2_FORCE_DISABLE,
+			  HDMI_HDCP2_OVR_EN_MASK | HDMI_HDCP2_FORCE_MASK,
+			  HDMI_HDCP2REG_CTRL);
+		hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MASK);
+		hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MUTE);
+	}
+
+	hdcp->write(hdmi, 0x40, HDMI_A_OESSWCFG);
+	hdcp_modb(hdcp,
+		  HDMI_A_HDCPCFG0_BYPENCRYPTION_DISABLE |
+			  HDMI_A_HDCPCFG0_EN11FEATURE_DISABLE |
+			  HDMI_A_HDCPCFG0_SYNCRICHECK_ENABLE,
+		  HDMI_A_HDCPCFG0_BYPENCRYPTION_MASK |
+			  HDMI_A_HDCPCFG0_EN11FEATURE_MASK |
+			  HDMI_A_HDCPCFG0_SYNCRICHECK_MASK,
+		  HDMI_A_HDCPCFG0);
+
+	hdcp_modb(hdcp,
+		  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_ENABLE |
+			  HDMI_A_HDCPCFG1_PH2UPSHFTENC_ENABLE,
+		  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK |
+			  HDMI_A_HDCPCFG1_PH2UPSHFTENC_MASK,
+		  HDMI_A_HDCPCFG1);
+
+	/* Reset HDCP Engine */
+	if (hdcp->read(hdmi, HDMI_MC_CLKDIS) & HDMI_MC_CLKDIS_HDCPCLK_MASK) {
+		hdcp_modb(hdcp, HDMI_A_HDCPCFG1_SWRESET_ASSERT,
+			  HDMI_A_HDCPCFG1_SWRESET_MASK, HDMI_A_HDCPCFG1);
+	}
+
+	hdcp->write(hdmi, 0x00, HDMI_A_APIINTMSK);
+	hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_ENABLE,
+		  HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0);
+
+	/*
+     * XXX: to sleep 500ms here between output hdmi and enable hdcpclk,
+     * otherwise hdcp auth fail when Connect to repeater
+     */
+	msleep(500);
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_ENABLE,
+		  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+
+	hdcp->status = DW_HDCP_AUTH_START;
+	dev_info(hdcp->dev, "%s success\n", __func__);
+
+	/* enable the pixel clock*/
+	dev_dbg(hdcp->dev, "start hdcp with enable hdmi pixel clock\n");
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_ENABLE,
+		  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+
+	return 0;
+}
+
+static int dw_hdmi_hdcp1x_stop(struct dw_hdcp *hdcp)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	u8 val;
+	bool phy_enable = false;
+
+	if (!hdcp->enable)
+		return -EPERM;
+
+	val = hdcp->read(hdmi, HDMI_PHY_CONF0);
+	if (val & HDMI_PHY_CONF0_GEN2_TXPWRON_MASK) {
+		phy_enable = true;
+	}
+
+	dev_dbg(hdcp->dev, "dw_hdmi_hdcp1x_stop\n");
+	if (phy_enable) {
+		dev_dbg(hdcp->dev, "stop hdcp with disable hdmi pixel clock\n");
+		hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_DISABLE,
+			  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+	}
+
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_DISABLE,
+		  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+	hdcp->write(hdmi, 0xff, HDMI_A_APIINTMSK);
+
+	hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_DISABLE,
+		  HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0);
+
+	hdcp_modb(hdcp,
+		  HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE |
+			  HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE,
+		  HDMI_A_SRMCTRL_SHA1_FAIL_MASK |
+			  HDMI_A_SRMCTRL_KSV_UPDATE_MASK,
+		  HDMI_A_SRMCTRL);
+
+	hdcp->status = DW_HDCP_DISABLED;
+
+	if (phy_enable) {
+		dev_dbg(hdcp->dev, "stop hdcp with enable hdmi pixel clock\n");
+		hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_ENABLE,
+			  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+	}
+
+	return 0;
+}
+
+void dw_hdmi_hdcp2_init(struct dw_hdcp2 *hdcp2)
+{
+	if (g_hdcp[hdcp2->numa_id])
+		g_hdcp[hdcp2->numa_id]->hdcp2 = hdcp2;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_hdcp2_init);
+
+void dw_hdmi_hdcp2_remove(struct dw_hdcp2 *hdcp2)
+{
+	dev_info(hdcp2->dev, "%s numa id:%d\n", __func__, hdcp2->numa_id);
+	if (g_hdcp[hdcp2->numa_id]->hdcp2)
+		g_hdcp[hdcp2->numa_id]->hdcp2->stop(hdcp2);
+	g_hdcp[hdcp2->numa_id]->hdcp2 = NULL;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_hdcp2_remove);
+
+void dw_hdmi_hdcp2_start(int enable, int numa_id)
+{
+	int val;
+
+	if (!(g_hdcp[numa_id]->hdcp2))
+		return;
+
+	dev_dbg(g_hdcp[numa_id]->dev, "%s enable = %d\n", __func__, enable);
+	if (enable == 0) {
+		hdcp_modb(g_hdcp[numa_id],
+			  HDMI_HDCP2_OVR_ENABLE | HDMI_HDCP2_FORCE_DISABLE,
+			  HDMI_HDCP2_OVR_EN_MASK | HDMI_HDCP2_FORCE_MASK,
+			  HDMI_HDCP2REG_CTRL);
+		hdcp_modb(g_hdcp[numa_id], HDMI_MC_CLKDIS_HDCPCLK_DISABLE,
+			  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+	} else if (enable == 1) {
+		hdcp_modb(g_hdcp[numa_id], HDMI_MC_CLKDIS_HDCPCLK_ENABLE,
+			  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+		hdcp_modb(g_hdcp[numa_id],
+			  HDMI_HDCP2_OVR_ENABLE | HDMI_HDCP2_FORCE_ENABLE,
+			  HDMI_HDCP2_OVR_EN_MASK | HDMI_HDCP2_FORCE_MASK,
+			  HDMI_HDCP2REG_CTRL);
+		hdcp_modb(g_hdcp[numa_id], HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE,
+			  HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK,
+			  HDMI_FC_INVIDCONF);
+	} else if (enable == 2) {
+		val = g_hdcp[numa_id]->read(g_hdcp[numa_id]->hdmi, HDMI_PHY_STAT0);
+		if (val & HDMI_PHY_HPD)
+			dw_hdmi_hdcp1x_start(g_hdcp[numa_id]);
+	} else if (enable == 3) {
+		if (g_hdcp[numa_id]->hdcp2 && g_hdcp[numa_id]->hdcp2->enable &&
+		    (tv_hdmi_hdcp2_support(g_hdcp[numa_id]->hdmi) == 1)) {
+			if (g_hdcp[numa_id]->status != DW_HDCP_DISABLED)
+				dw_hdmi_hdcp1x_stop(g_hdcp[numa_id]);
+			g_hdcp[numa_id]->hdcp2->start(g_hdcp[numa_id]->hdcp2);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_hdcp2_start);
+
+static int dw_hdmi_hdcp_start(struct dw_hdcp *hdcp)
+{
+	if (hdcp->hdcp2 && hdcp->hdcp2->enable &&
+	    (tv_hdmi_hdcp2_support(hdcp->hdmi) == 1)) {
+		// hdcp->hdcp2->start();
+		return 0;
+	}
+	return dw_hdmi_hdcp1x_start(hdcp);
+}
+
+static int dw_hdmi_hdcp_stop(struct dw_hdcp *hdcp)
+{
+	if (hdcp->hdcp2 && hdcp->hdcp2->hot_plug) {
+		// g_hdcp->hdcp2->stop();
+		printk("func: %s; line: %d\n", __func__, __LINE__);
+	}
+
+	return dw_hdmi_hdcp1x_stop(hdcp);
+}
+
+static void dw_hdmi_hdcp_isr(struct dw_hdcp *hdcp, int hdcp_int)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	int val;
+
+	dev_info(hdcp->dev, "hdcp_int is 0x%02x\n", hdcp_int);
+
+	if (hdcp_int & HDMI_A_APIINTSTAT_KSVSHA1_CALC_DONE_INT) {
+		dev_dbg(hdcp->dev, "hdcp sink is a repeater\n");
+		val = hdcp->read(hdmi, HDMI_A_KSVMEMCTRL);
+		if (val | HDMI_A_KSVMEMCTRL_KSV_SHA1_STATUS) {
+			dev_dbg(hdcp->dev,
+				"hdcp verifivation failed, waiting hdmi controller re-authentication!\n");
+		} else {
+			dev_dbg(hdcp->dev, "hdcp verifivation succeeded!\n");
+			/* reset HDCP */
+			hdcp_modb(hdcp, HDMI_A_HDCPCFG1_SWRESET_ASSERT,
+				  HDMI_A_HDCPCFG1_SWRESET_MASK,
+				  HDMI_A_HDCPCFG1);
+		}
+	}
+
+	if (hdcp_int & 0x40) {
+		hdcp->status = DW_HDCP_AUTH_FAIL;
+		dev_info(hdcp->dev, "hdcp auth fail\n");
+		if (hdcp->remaining_times > 1)
+			hdcp->remaining_times--;
+		else if (hdcp->remaining_times == 1)
+			hdcp_modb(hdcp,
+				  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE,
+				  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK,
+				  HDMI_A_HDCPCFG1);
+	} else if (hdcp_int & 0x80) {
+		dev_info(hdcp->dev, "hdcp auth success\n");
+		hdcp->status = DW_HDCP_AUTH_SUCCESS;
+	} else if (hdcp_int & 0x10) {
+		dev_info(hdcp->dev, "i2c nack\n");
+		trytimes++;
+		if (trytimes == 20) {
+			trytimes = 0;
+			dw_hdmi_hdcp1x_stop(hdcp);
+		}
+	}
+}
+
+static ssize_t hdcp_enable_read(struct device *device,
+				struct device_attribute *attr, char *buf)
+{
+	bool enable = 0;
+	struct miscdevice *mdev= dev_get_drvdata(device);
+	struct dw_hdcp *hdcp = mdev->parent->platform_data;
+
+	if (hdcp)
+		enable = hdcp->enable;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", enable);
+}
+
+static ssize_t hdcp_enable_write(struct device *device,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	bool enable;
+	struct miscdevice *mdev= dev_get_drvdata(device);
+	struct dw_hdcp *hdcp = mdev->parent->platform_data;
+	int ret;
+
+	if (!hdcp)
+		return -EINVAL;
+
+	if (kstrtobool(buf, &enable))
+		return -EINVAL;
+
+	if (hdcp->enable != enable) {
+		if (enable) {
+			hdcp->enable = enable;
+			if (hdcp->hdcp2 && hdcp->hdcp2->hot_plug) {
+				return count;
+			}
+
+			if (hdcp->read(hdcp->hdmi, HDMI_PHY_STAT0) &
+			    HDMI_PHY_HPD) {
+				ret = dw_hdmi_hdcp1x_start(hdcp);
+				if (ret) {
+					hdcp->enable = 0;
+				}
+			}
+		} else {
+			if (hdcp->status != DW_HDCP_DISABLED) {
+				dw_hdmi_hdcp1x_stop(hdcp);
+			}
+			hdcp->enable = enable;
+		}
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR(enable, 0644, hdcp_enable_read, hdcp_enable_write);
+
+static ssize_t hdcp_trytimes_read(struct device *device,
+				  struct device_attribute *attr, char *buf)
+{
+	int trytimes = 0;
+	struct miscdevice *mdev= dev_get_drvdata(device);
+	struct dw_hdcp *hdcp = mdev->parent->platform_data;
+
+	if (hdcp)
+		trytimes = hdcp->retry_times;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", trytimes);
+}
+
+static ssize_t hdcp_trytimes_write(struct device *device,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	int trytimes;
+	struct miscdevice *mdev= dev_get_drvdata(device);
+	struct dw_hdcp *hdcp = mdev->parent->platform_data;
+
+	if (!hdcp)
+		return -EINVAL;
+
+	if (kstrtoint(buf, 0, &trytimes))
+		return -EINVAL;
+
+	if (hdcp->retry_times != trytimes) {
+		hdcp->retry_times = trytimes;
+		hdcp->remaining_times = hdcp->retry_times;
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR(trytimes, 0644, hdcp_trytimes_read, hdcp_trytimes_write);
+
+static ssize_t hdcp_status_read(struct device *device,
+				struct device_attribute *attr, char *buf)
+{
+	int status = DW_HDCP_DISABLED;
+	struct miscdevice *mdev= dev_get_drvdata(device);
+	struct dw_hdcp *hdcp = mdev->parent->platform_data;
+
+	if (hdcp)
+		status = hdcp->status;
+
+	if (status == DW_HDCP_DISABLED)
+		return snprintf(buf, PAGE_SIZE, "hdcp disable\n");
+	else if (status == DW_HDCP_AUTH_START)
+		return snprintf(buf, PAGE_SIZE, "hdcp_auth_start\n");
+	else if (status == DW_HDCP_AUTH_SUCCESS)
+		return snprintf(buf, PAGE_SIZE, "hdcp_auth_success\n");
+	else if (status == DW_HDCP_AUTH_FAIL)
+		return snprintf(buf, PAGE_SIZE, "hdcp_auth_fail\n");
+	else
+		return snprintf(buf, PAGE_SIZE, "unknown status\n");
+}
+
+static DEVICE_ATTR(status, 0444, hdcp_status_read, NULL);
+
+static int dw_hdmi_hdcp_probe(struct platform_device *pdev)
+{
+	int ret = 0;
+	struct dw_hdcp *hdcp = pdev->dev.platform_data;
+
+	dev_info(&pdev->dev, "%s numa id:%d\n", __func__, hdcp->numa_id);
+	g_hdcp[hdcp->numa_id] = hdcp;
+	char name[15];
+	sprintf(name, "hdmi_%d_hdcp1x", hdcp->numa_id);
+	hdcp->mdev.minor = MISC_DYNAMIC_MINOR;
+	hdcp->mdev.name = name;
+	hdcp->mdev.mode = 0666;
+	hdcp->mdev.parent = &pdev->dev;
+
+	if (misc_register(&hdcp->mdev)) {
+		dev_err(&pdev->dev, "HDCP: Could not add character driver\n");
+		return -EINVAL;
+	}
+
+	ret = device_create_file(hdcp->mdev.this_device, &dev_attr_enable);
+	if (ret) {
+		dev_err(&pdev->dev, "HDCP: Could not add sys file enable\n");
+		ret = -EINVAL;
+		goto error0;
+	}
+
+	ret = device_create_file(hdcp->mdev.this_device, &dev_attr_trytimes);
+	if (ret) {
+		dev_err(&pdev->dev, "HDCP: Could not add sys file trytimes\n");
+		ret = -EINVAL;
+		goto error1;
+	}
+
+	ret = device_create_file(hdcp->mdev.this_device, &dev_attr_status);
+	if (ret) {
+		dev_err(&pdev->dev, "HDCP: Could not add sys file status\n");
+		ret = -EINVAL;
+		goto error2;
+	}
+
+	/* retry time if hdcp auth fail. unlimited time if set 0 */
+	hdcp->retry_times = 0;
+	hdcp->dev = &pdev->dev;
+	hdcp->hdcp_start = dw_hdmi_hdcp_start;
+	hdcp->hdcp_stop = dw_hdmi_hdcp_stop;
+	hdcp->hdcp_isr = dw_hdmi_hdcp_isr;
+
+	dev_dbg(hdcp->dev, "%s success\n", __func__);
+	return 0;
+
+error2:
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes);
+error1:
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_enable);
+error0:
+	misc_deregister(&hdcp->mdev);
+	return ret;
+}
+
+static int dw_hdmi_hdcp_remove(struct platform_device *pdev)
+{
+	struct dw_hdcp *hdcp = pdev->dev.platform_data;
+
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes);
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_enable);
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_status);
+	misc_deregister(&hdcp->mdev);
+
+	kfree(hdcp->keys);
+	kfree(hdcp->seeds);
+
+	return 0;
+}
+
+struct platform_driver dw_hdmi_hdcp_driver = {
+    .probe  = dw_hdmi_hdcp_probe,
+    .remove = dw_hdmi_hdcp_remove,
+    .driver = {
+        .name = DW_HDCP_DRIVER_NAME,
+    },
+};
+
+//module_platform_driver(dw_hdmi_hdcp_driver);
+MODULE_DESCRIPTION("DW HDMI transmitter HDCP driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_hdcp.h b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.h
new file mode 100644
index 000000000000..8d8c386515f4
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.h
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN hdcp driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+
+#ifndef DW_HDMI_HDCP_H
+#define DW_HDMI_HDCP_H
+
+#include <linux/miscdevice.h>
+#include "dsp_dma_buf.h"
+
+#define DW_HDCP_DRIVER_NAME "dw-hdmi-hdcp"
+#define HDCP_PRIVATE_KEY_SIZE 280
+#define HDCP_KEY_SHA_SIZE 20
+
+struct hdcp_keys {
+	u8 KSV[8];
+	u8 devicekey[HDCP_PRIVATE_KEY_SIZE];
+	u8 sha1[HDCP_KEY_SHA_SIZE];
+};
+
+typedef struct {
+	int allocated, initialized;
+	int code_loaded;
+
+	int code_is_phys_mem;
+	dma_addr_t code_base;
+	uint32_t code_size;
+	uint8_t *code;
+	int data_is_phys_mem;
+	dma_addr_t data_base;
+	uint32_t data_size;
+	uint8_t *data;
+
+	struct resource *hpi_resource;
+	uint8_t __iomem *hpi;
+	struct device *dev;
+} hl_device;
+
+struct dw_hdcp2 {
+	int numa_id;
+	int enable;
+	void (*start)(struct dw_hdcp2* hdcp2);
+	void (*stop)(struct dw_hdcp2* hdcp2);
+
+	struct device *dev;
+	int wait_hdcp2_reset;
+	int hot_plug;
+	struct miscdevice mdev;
+	int auth_sucess;
+	hl_device *hld;
+	struct miscdevice mics_hld;
+	struct clk *aclk;
+	struct clk *iesmclk;
+
+	struct dev_buffer_desc code_buffer;
+	struct dev_buffer_desc data_buffer;
+	struct dma_buf_attachment *code_attach;
+	struct dma_buf_attachment *data_attach;
+};
+
+struct dw_hdcp {
+	bool enable;
+	int retry_times;
+	int remaining_times;
+	char *seeds;
+	int invalidkey;
+	char *invalidkeys;
+	int hdcp2_enable;
+	int status;
+	u32 reg_io_width;
+	int numa_id;
+
+	struct dw_hdcp2 *hdcp2;
+	struct miscdevice mdev;
+	struct hdcp_keys *keys;
+	struct device *dev;
+	struct dw_hdmi *hdmi;
+	void __iomem *regs;
+
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	int (*hdcp_start)(struct dw_hdcp *hdcp);
+	int (*hdcp_stop)(struct dw_hdcp *hdcp);
+	void (*hdcp_isr)(struct dw_hdcp *hdcp, int hdcp_int);
+};
+
+extern u8 tv_hdmi_hdcp2_support(struct dw_hdmi *hdmi);
+extern void dw_hdmi_hdcp2_init(struct dw_hdcp2 *hdcp2);
+extern void dw_hdmi_hdcp2_remove(struct dw_hdcp2 *hdcp2);
+extern void dw_hdmi_hdcp2_start(int enable, int numa_id);
+#endif
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_hdcp2.c b/drivers/gpu/drm/eswin/dw_hdmi_hdcp2.c
new file mode 100644
index 000000000000..aa71bc7cce8e
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_hdcp2.c
@@ -0,0 +1,893 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN hdcp driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/miscdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/platform_device.h>
+
+#include <linux/version.h>
+#include <linux/netlink.h>
+#include <linux/proc_fs.h>
+#include <linux/moduleparam.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+
+#include "host_lib_driver_linux_if.h"
+#include "dw_hdmi_hdcp.h"
+
+/**
+ * \file
+ * \ingroup HL_Driver_Kernel
+ * \brief Sample Linux Host Library Driver
+ * \copydoc HL_Driver_Kernel
+ */
+
+/**
+ * \defgroup HL_Driver_Linux Sample Linux Host Library Driver
+ * \ingroup HL_Driver
+ * \brief Sample code for the Linux Host Library Driver.
+ * The Linux Host Library Driver is composed of 2 parts:
+ * 1. A kernel driver.
+ * 2. A file access instance.
+ *
+ * The kernel driver is the kernel executable code enabling the firmware to execute.
+ * It provides the access to the hardware register to interact with the firmware.
+ *
+ * The file access instance initializes the #hl_driver_t structure for the
+ * host library access.  The Host Library references the file access to request the
+ * kernel operations.
+ */
+
+/**
+ * \defgroup HL_Driver_Kernel Sample Linux Kernel Host Library Driver
+ * \ingroup HL_Driver_Linux
+ * \brief Example code for the Linux Kernel Host Library Driver.
+ *
+ * The Sample Linux Kernel Driver operates on the linux kernel.
+ * To install (requires root access):
+ * \code
+ insmod bin/linux_hld_module.ko verbose=0
+ * \endcode
+ *
+ * To remove (requires root access):
+ * \code
+ rmmod linux_hld_module
+ * \endcode
+ *
+ * Example Linux Host Library Code:
+ * \code
+ */
+
+#define MAX_HL_DEVICES 16
+#define TROOT_GRIFFIN
+
+static hl_device hl_devices[2][MAX_HL_DEVICES];
+
+static bool randomize_mem = false;
+module_param(randomize_mem, bool, 0);
+MODULE_PARM_DESC(noverify, "Wipe memory allocations on startup (for debug)");
+
+static void dw_hdcp2_stop(struct dw_hdcp2 *hdcp2)
+{
+	dev_info(hdcp2->dev, "%s\n", __func__);
+	hdcp2->hot_plug = 0;
+	dw_hdmi_hdcp2_start(0, hdcp2->numa_id);
+}
+
+static void dw_hdcp2_start(struct dw_hdcp2 *hdcp2)
+{
+	dev_info(hdcp2->dev, "%s\n", __func__);
+	dw_hdmi_hdcp2_start(1, hdcp2->numa_id);
+	hdcp2->hot_plug = 1;
+}
+
+/* HL_DRV_IOC_MEMINFO implementation */
+static long get_meminfo(hl_device *hl_dev, void __user *arg)
+{
+	struct hl_drv_ioc_meminfo info;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->hpi_resource == 0)) {
+		return -EFAULT;
+	}
+
+	info.hpi_base = hl_dev->hpi_resource->start;
+	info.code_base = hl_dev->code_base;
+	info.code_size = hl_dev->code_size;
+	info.data_base = hl_dev->data_base;
+	info.data_size = hl_dev->data_size;
+
+	if (copy_to_user(arg, &info, sizeof info) != 0) {
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/* HL_DRV_IOC_LOAD_CODE implementation */
+static long load_code(hl_device *hl_dev, struct hl_drv_ioc_code __user *arg)
+{
+	struct hl_drv_ioc_code head;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->code == 0) ||
+	    (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&head, arg, sizeof head) != 0)
+		return -EFAULT;
+
+	if (head.len > hl_dev->code_size)
+		return -ENOSPC;
+
+	if (hl_dev->code_loaded)
+		return -EBUSY;
+
+	if (randomize_mem) {
+		get_random_bytes(hl_dev->code, hl_dev->code_size);
+		get_random_bytes(hl_dev->data, hl_dev->data_size);
+	}
+
+	if (copy_from_user(hl_dev->code, &arg->data, head.len) != 0)
+		return -EFAULT;
+
+	hl_dev->code_loaded = 1;
+	return 0;
+}
+
+/* HL_DRV_IOC_WRITE_DATA implementation */
+static long write_data(hl_device *hl_dev, struct hl_drv_ioc_data __user *arg)
+{
+	struct hl_drv_ioc_data head;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&head, arg, sizeof head) != 0)
+		return -EFAULT;
+
+	if (hl_dev->data_size < head.len)
+		return -ENOSPC;
+	if (hl_dev->data_size - head.len < head.offset)
+		return -ENOSPC;
+
+	if (copy_from_user(hl_dev->data + head.offset, &arg->data, head.len) !=
+	    0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/* HL_DRV_IOC_READ_DATA implementation */
+static long read_data(hl_device *hl_dev, struct hl_drv_ioc_data __user *arg)
+{
+	struct hl_drv_ioc_data head;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&head, arg, sizeof head) != 0)
+		return -EFAULT;
+
+	if (hl_dev->data_size < head.len)
+		return -ENOSPC;
+	if (hl_dev->data_size - head.len < head.offset)
+		return -ENOSPC;
+
+	if (copy_to_user(&arg->data, hl_dev->data + head.offset, head.len) != 0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/* HL_DRV_IOC_MEMSET_DATA implementation */
+static long set_data(hl_device *hl_dev, void __user *arg)
+{
+	union {
+		struct hl_drv_ioc_data data;
+		unsigned char buf[sizeof(struct hl_drv_ioc_data) + 1];
+	} u;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&u.data, arg, sizeof u.buf) != 0)
+		return -EFAULT;
+
+	if (hl_dev->data_size < u.data.len)
+		return -ENOSPC;
+	if (hl_dev->data_size - u.data.len < u.data.offset)
+		return -ENOSPC;
+
+	memset(hl_dev->data + u.data.offset, u.data.data[0], u.data.len);
+	return 0;
+}
+
+/* HL_DRV_IOC_READ_HPI implementation */
+static long hpi_read(hl_device *hl_dev, void __user *arg)
+{
+	struct hl_drv_ioc_hpi_reg reg;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->hpi_resource == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&reg, arg, sizeof reg) != 0)
+		return -EFAULT;
+
+	if ((reg.offset & 3) ||
+	    reg.offset >= resource_size(hl_dev->hpi_resource))
+		return -EINVAL;
+
+	reg.value = ioread32(hl_dev->hpi + reg.offset);
+
+	if (copy_to_user(arg, &reg, sizeof reg) != 0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/* HL_DRV_IOC_WRITE_HPI implementation */
+static long hpi_write(hl_device *hl_dev, void __user *arg)
+{
+	struct hl_drv_ioc_hpi_reg reg;
+
+	if ((hl_dev == 0) || (arg == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&reg, arg, sizeof reg) != 0)
+		return -EFAULT;
+
+	if ((reg.offset & 3) ||
+	    reg.offset >= resource_size(hl_dev->hpi_resource))
+		return -EINVAL;
+
+	iowrite32(reg.value, hl_dev->hpi + reg.offset);
+
+#ifdef TROOT_GRIFFIN
+	//  If Kill command
+	//  (HL_GET_CMD_EVENT(krequest.data) == TROOT_CMD_SYSTEM_ON_EXIT_REQ))
+	//
+	if ((reg.offset == 0x38) && ((reg.value & 0x000000ff) == 0x08)) {
+		hl_dev->code_loaded = 0;
+	}
+#endif
+	return 0;
+}
+
+static hl_device *alloc_hl_dev_slot(const struct hl_drv_ioc_meminfo *info, int numa_id)
+{
+	int i;
+
+	if (info == 0) {
+		return 0;
+	}
+
+	/* Check if we have a matching device (same HPI base) */
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		hl_device *slot = &hl_devices[numa_id][i];
+		if (slot->allocated &&
+		    (info->hpi_base == slot->hpi_resource->start))
+			return slot;
+	}
+
+	/* Find unused slot */
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		hl_device *slot = &hl_devices[numa_id][i];
+		if (!slot->allocated) {
+			slot->allocated = 1;
+			return slot;
+		}
+	}
+
+	return 0;
+}
+
+static void free_dma_areas(struct dw_hdcp2 *hdcp2)
+{
+	hl_device *hl_dev =  hdcp2->hld;
+
+	if (hl_dev == 0) {
+		return;
+	}
+
+	if (!hdcp2->numa_id) {
+		if (!hl_dev->code_is_phys_mem && hl_dev->code) {
+			dma_free_coherent(0, hl_dev->code_size, hl_dev->code,
+					hl_dev->code_base);
+			hl_dev->code = 0;
+		}
+
+		if (!hl_dev->data_is_phys_mem && hl_dev->data) {
+			dma_free_coherent(0, hl_dev->data_size, hl_dev->data,
+					hl_dev->data_base);
+			hl_dev->data = 0;
+		}
+	} else {
+		if (!hl_dev->code_is_phys_mem && hl_dev->code) {
+			dev_mem_vunmap(&hdcp2->code_buffer, hl_dev->code);
+
+			if (hdcp2->code_attach) {
+				dev_mem_detach(hdcp2->code_attach, DMA_BIDIRECTIONAL);
+			}
+
+			if (hdcp2->code_buffer.buf) {
+				dev_mem_free(hdcp2->code_buffer.buf);
+			}
+		}
+
+		if (!hl_dev->data_is_phys_mem && hl_dev->data) {
+			dev_mem_vunmap(&hdcp2->data_buffer, hl_dev->data);
+
+			if (hdcp2->data_attach) {
+				dev_mem_detach(hdcp2->data_attach, DMA_BIDIRECTIONAL);
+			}
+
+			if (hdcp2->data_buffer.buf) {
+				dev_mem_free(hdcp2->data_buffer.buf);
+			}
+		}
+	}
+}
+
+static int alloc_dma_areas(struct dw_hdcp2 *hdcp2, const struct hl_drv_ioc_meminfo *info)
+{
+	int ret;
+	hl_device *hl_dev = hdcp2->hld;
+	struct device *dev = hdcp2->dev;
+
+	if ((hl_dev == 0) || (info == 0)) {
+		return -EFAULT;
+	}
+
+	hl_dev->code_size = info->code_size;
+	hl_dev->code_is_phys_mem =
+		(info->code_base != HL_DRIVER_ALLOCATE_DYNAMIC_MEM);
+
+	if (hl_dev->code_is_phys_mem && (hl_dev->code == 0)) {
+		/* TODO: support highmem */
+		hl_dev->code_base = info->code_base;
+		hl_dev->code = phys_to_virt(hl_dev->code_base);
+	} else {
+		if (!hdcp2->numa_id) {
+			hl_dev->code = dma_alloc_coherent(dev, hl_dev->code_size,
+											  &hl_dev->code_base, GFP_KERNEL);
+			if (!hl_dev->code) {
+				return -ENOMEM;
+			}
+			dev_info(dev,"code dma addr:0x%llx, size:0x%x\n", hl_dev->code_base, hl_dev->code_size);
+		} else {
+			ret = dev_mem_alloc(hl_dev->code_size, ES_MEM_ALLOC_SPRAM_DIE1,
+								&hdcp2->code_buffer.buf);
+			if (ret < 0) {
+				dev_err(dev, "dev_mem_alloc code buf failed!\n");
+				return -ENOMEM;
+			}
+
+			hl_dev->code_base = dev_mem_attach(hdcp2->code_buffer.buf, dev,
+											   DMA_BIDIRECTIONAL, &hdcp2->code_attach);
+			if (hl_dev->code_base == 0) {
+				dev_err(dev, "dev_mem_attach code buf failed!\n");
+				goto release_code_buf;
+			}
+
+			hl_dev->code = dev_mem_vmap(&hdcp2->code_buffer);
+			if (!hl_dev->code) {
+				dev_err(dev, "dev_mem_vmap code buf failed!\n");
+				goto release_code_attach;
+			}
+			dev_info(dev, "code spram addr:0x%llx, size:0x%x\n", hl_dev->code_base, hl_dev->code_size);
+		}
+	}
+
+	hl_dev->data_size = info->data_size;
+	hl_dev->data_is_phys_mem =
+		(info->data_base != HL_DRIVER_ALLOCATE_DYNAMIC_MEM);
+
+	if (hl_dev->data_is_phys_mem && (hl_dev->data == 0)) {
+		hl_dev->data_base = info->data_base;
+		hl_dev->data = phys_to_virt(hl_dev->data_base);
+	} else {
+		if (!hdcp2->numa_id) {
+			hl_dev->data = dma_alloc_coherent(dev, hl_dev->data_size,
+											  &hl_dev->data_base, GFP_KERNEL);
+			if (!hl_dev->data) {
+				free_dma_areas(hdcp2);
+				return -ENOMEM;
+			}
+			dev_info(dev,"data dma addr:0x%llx, size:0x%x\n", hl_dev->code_base, hl_dev->data_size);
+		} else {
+			ret = dev_mem_alloc(hl_dev->data_size, ES_MEM_ALLOC_SPRAM_DIE1,
+								&hdcp2->data_buffer.buf);
+			if (ret < 0) {
+				dev_err(dev, "dev_mem_alloc data buf failed!\n");
+				goto release_code_vmap;
+			}
+
+			hl_dev->data_base = dev_mem_attach(hdcp2->data_buffer.buf, dev,
+											   DMA_BIDIRECTIONAL, &hdcp2->data_attach);
+			if (hl_dev->data_base == 0) {
+				dev_err(dev, "dev_mem_attach data buf failed!\n");
+				goto release_data_buf;
+			}
+
+			hl_dev->data = dev_mem_vmap(&hdcp2->data_buffer);
+			if (!hl_dev->data) {
+				dev_err(dev, "dev_mem_vmap data buf failed!\n");
+				goto release_data_attach;
+			}
+			dev_info(dev, "data spram addr:0x%llx, size:0x%x\n", hl_dev->data_base, hl_dev->data_size);
+		}
+	}
+
+	return 0;
+
+release_data_attach:
+	dev_mem_detach(hdcp2->data_attach, DMA_BIDIRECTIONAL);
+release_data_buf:
+	dev_mem_free(hdcp2->data_buffer.buf);
+release_code_vmap:
+	dev_mem_vunmap(&hdcp2->code_buffer, hl_dev->code);
+release_code_attach:
+	dev_mem_detach(hdcp2->code_attach, DMA_BIDIRECTIONAL);
+release_code_buf:
+	dev_mem_free(hdcp2->code_buffer.buf);
+
+	return -ENOMEM;
+}
+
+/* HL_DRV_IOC_INIT implementation */
+static long hld_init(struct dw_hdcp2 *hdcp2, void __user *arg)
+{
+	struct resource *hpi_mem;
+	struct hl_drv_ioc_meminfo info;
+	int rc;
+
+	if ((arg == 0)) {
+		dev_err(hdcp2->dev, "param error!\n");
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&info, arg, sizeof info) != 0)
+		return -EFAULT;
+
+	hdcp2->hld = alloc_hl_dev_slot(&info, hdcp2->numa_id);
+	if (!hdcp2->hld)
+		return -EMFILE;
+
+	if (!hdcp2->hld->initialized) {
+		rc = alloc_dma_areas(hdcp2, &info);
+		if (rc < 0)
+			goto err_free;
+
+		hpi_mem = request_mem_region(info.hpi_base, 128, "hl_dev-hpi");
+		if (!hpi_mem) {
+			rc = -EADDRNOTAVAIL;
+			goto err_free;
+		}
+
+		hdcp2->hld->hpi = ioremap(hpi_mem->start, resource_size(hpi_mem));
+		if (!hdcp2->hld->hpi) {
+			rc = -ENOMEM;
+			goto err_release_region;
+		}
+		hdcp2->hld->hpi_resource = hpi_mem;
+		hdcp2->hld->initialized = 1;
+	}
+
+	return 0;
+
+err_release_region:
+	release_resource(hpi_mem);
+err_free:
+	free_dma_areas(hdcp2);
+	hdcp2->hld->initialized = 0;
+	hdcp2->hld->allocated = 0;
+	hdcp2->hld->hpi_resource = 0;
+	hdcp2->hld->hpi = 0;
+
+	return rc;
+}
+
+static void free_hl_dev_slot(hl_device *slot)
+{
+	if (slot == 0) {
+		return;
+	}
+
+	if (!slot->allocated)
+		return;
+
+	if (slot->initialized) {
+		if (slot->hpi) {
+			iounmap(slot->hpi);
+			slot->hpi = 0;
+		}
+
+		if (slot->hpi_resource) {
+			release_mem_region(slot->hpi_resource->start, 128);
+			slot->hpi_resource = 0;
+		}
+	}
+
+	slot->initialized = 0;
+	slot->allocated = 0;
+}
+
+static long hld_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
+{
+	struct miscdevice *misc_hld = f->private_data;
+	struct dw_hdcp2 *hdcp2 = dev_get_drvdata(misc_hld->parent);
+	void __user *data;
+
+	if (f == 0) {
+		return -EFAULT;
+	}
+
+	data = (void __user *)arg;
+
+	if (cmd == HL_DRV_IOC_INIT) {
+		return hld_init(hdcp2, data);
+	} else if (!hdcp2->hld) {
+		return -EAGAIN;
+	}
+
+	switch (cmd) {
+	case HL_DRV_IOC_INIT:
+		return hld_init(hdcp2, data);
+	case HL_DRV_IOC_MEMINFO:
+		return get_meminfo(hdcp2->hld, data);
+	case HL_DRV_IOC_READ_HPI:
+		return hpi_read(hdcp2->hld, data);
+	case HL_DRV_IOC_WRITE_HPI:
+		return hpi_write(hdcp2->hld, data);
+	case HL_DRV_IOC_LOAD_CODE:
+		return load_code(hdcp2->hld, data);
+	case HL_DRV_IOC_WRITE_DATA:
+		return write_data(hdcp2->hld, data);
+	case HL_DRV_IOC_READ_DATA:
+		return read_data(hdcp2->hld, data);
+	case HL_DRV_IOC_MEMSET_DATA:
+		return set_data(hdcp2->hld, data);
+
+	case DW_DRV_IOC_CONNECT_STATUS:
+		return hdcp2->hot_plug;
+	case DW_DRV_IOC_CONNECT_SET:
+		dev_info(hdcp2->dev, "set hdcp2 reset one\n");
+		hdcp2->wait_hdcp2_reset = 1;
+		dw_hdmi_hdcp2_start(1, hdcp2->numa_id);
+		return 0;
+	case DW_DRV_IOC_DISCONNECT_SET:
+		if (hdcp2->wait_hdcp2_reset == 1) {
+			printk("set hdcp2 reset zero\n");
+			hdcp2->wait_hdcp2_reset = 0;
+			dw_hdmi_hdcp2_start(0, hdcp2->numa_id);
+		}
+		if (hdcp2->auth_sucess == 1) {
+			hdcp2->auth_sucess = 0;
+		}
+		return 0;
+	case DW_DRV_IOC_AUTH_SUCCESS:
+		hdcp2->auth_sucess = 1;
+		return 0;
+	case DW_DRV_IOC_AUTH_FAIL:
+		hdcp2->auth_sucess = 0;
+		return 0;
+	case DW_DRV_IOC_NO_CAPACITY:
+		printk("set hdcp2 reset zero 3005\n");
+		hdcp2->hot_plug = 0;
+		hdcp2->wait_hdcp2_reset = 0;
+		dw_hdmi_hdcp2_start(0, hdcp2->numa_id);
+		dw_hdmi_hdcp2_start(2, hdcp2->numa_id);
+		return 0;
+	}
+
+	return -ENOTTY;
+}
+
+static const struct file_operations hld_file_operations = {
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = hld_ioctl,
+#endif
+	.unlocked_ioctl = hld_ioctl,
+	.owner = THIS_MODULE,
+};
+
+static int rigister_hld_device(struct dw_hdcp2 *hdcp2)
+{
+	int i;
+	int numa_id = hdcp2->numa_id;
+	char name[20];
+
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		hl_devices[numa_id][i].allocated = 0;
+		hl_devices[numa_id][i].initialized = 0;
+		hl_devices[numa_id][i].code_loaded = 0;
+		hl_devices[numa_id][i].code = 0;
+		hl_devices[numa_id][i].data = 0;
+		hl_devices[numa_id][i].hpi_resource = 0;
+		hl_devices[numa_id][i].hpi = 0;
+	}
+
+	sprintf(name, "hl_dev%d", numa_id);
+	hdcp2->mics_hld.minor = MISC_DYNAMIC_MINOR;
+	hdcp2->mics_hld.name = name;
+	hdcp2->mics_hld.fops = &hld_file_operations;
+	hdcp2->mics_hld.parent = hdcp2->dev;
+
+	dev_info(hdcp2->dev, "%s numa id:%d\n", __func__, numa_id);
+
+	if (misc_register(&hdcp2->mics_hld)) {
+		dev_err(hdcp2->dev, "Could not add character driver\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void hld_exit(struct dw_hdcp2 *hdcp2)
+{
+	int i;
+
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		free_hl_dev_slot(&hl_devices[hdcp2->numa_id][i]);
+	}
+
+	free_dma_areas(hdcp2);
+
+	misc_deregister(&hdcp2->mics_hld);
+}
+
+static int dw_hdmi2_hdcp2_clk_enable(struct dw_hdcp2 *hdcp2)
+{
+	hdcp2->aclk = devm_clk_get(hdcp2->dev, "aclk");
+	if (IS_ERR(hdcp2->aclk)) {
+		dev_err(hdcp2->dev, "Unable to get hdcp2 aclk\n");
+		return -1;
+	}
+	clk_prepare_enable(hdcp2->aclk);
+
+	hdcp2->iesmclk = devm_clk_get(hdcp2->dev, "iesmclk");
+	if (IS_ERR(hdcp2->iesmclk)) {
+		dev_err(hdcp2->dev, "Unable to get hdcp2_clk_hdmi\n");
+		return -1;
+	}
+	clk_prepare_enable(hdcp2->iesmclk);
+
+	return 0;
+}
+
+static ssize_t hdcp2_show_enable(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct miscdevice *mdev= dev_get_drvdata(dev);
+	struct dw_hdcp2 *hdcp2 = dev_get_drvdata(mdev->parent);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", hdcp2->enable);
+}
+
+static ssize_t hdcp2_store_enable(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t size)
+{
+	int enable;
+	struct miscdevice *mdev= dev_get_drvdata(dev);
+	struct dw_hdcp2 *hdcp2 = dev_get_drvdata(mdev->parent);
+
+	if (kstrtoint(buf, 0, &enable))
+		return size;
+
+	if (hdcp2->enable != enable) {
+		hdcp2->enable = enable;
+		if (enable) {
+			dw_hdmi_hdcp2_start(3, hdcp2->numa_id);
+		} else {
+			if (hdcp2->hot_plug) {
+				hdcp2->stop(hdcp2);
+				dw_hdmi_hdcp2_start(2, hdcp2->numa_id);
+			}
+		}
+	}
+	return size;
+}
+
+static ssize_t hdcp2_show_status(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct miscdevice *mdev= dev_get_drvdata(dev);
+	struct dw_hdcp2 *hdcp2 = dev_get_drvdata(mdev->parent);
+
+	if (hdcp2->enable != 1) {
+		return snprintf(buf, PAGE_SIZE, "%s\n", "no enable hdcp2");
+	} else if (!hdcp2->hot_plug) {
+		return snprintf(buf, PAGE_SIZE, "%s\n", "hdcp2 no auth");
+	} else {
+		if (hdcp2->auth_sucess)
+			return snprintf(buf, PAGE_SIZE, "%s\n",
+					"hdcp2 auth sucess");
+		else
+			return snprintf(buf, PAGE_SIZE, "%s\n",
+					"no already auth sucess");
+	}
+}
+
+static DEVICE_ATTR(enable, 0644, hdcp2_show_enable, hdcp2_store_enable);
+static DEVICE_ATTR(status, 0444, hdcp2_show_status, NULL);
+
+static int create_device_node(struct dw_hdcp2 *hdcp2)
+{
+	int ret;
+	char name[20];
+
+	sprintf(name, "hdcp2-%d-node", hdcp2->numa_id);
+	hdcp2->mdev.minor = MISC_DYNAMIC_MINOR;
+	hdcp2->mdev.name = name;
+	hdcp2->mdev.mode = 0666;
+	hdcp2->mdev.parent = hdcp2->dev;
+	if (misc_register(&(hdcp2->mdev))) {
+		dev_err(hdcp2->dev, "HDCP2: Could not add character driver\n");
+		return -1;
+	}
+
+	ret = device_create_file(hdcp2->mdev.this_device,
+				 &dev_attr_enable);
+	if (ret) {
+		dev_err(hdcp2->dev, "HDCP: Could not add sys file enable\n");
+		ret = -EINVAL;
+		goto error0;
+	}
+
+	ret = device_create_file(hdcp2->mdev.this_device,
+				 &dev_attr_status);
+	if (ret) {
+		dev_err(hdcp2->dev, "HDCP: Could not add sys file status\n");
+		ret = -EINVAL;
+		goto error1;
+	}
+
+	return 0;
+
+error1:
+	device_remove_file(hdcp2->mdev.this_device, &dev_attr_enable);
+error0:
+	misc_deregister(&hdcp2->mdev);
+	return ret;
+}
+
+static void end_device_node(struct dw_hdcp2 *hdcp2)
+{
+	device_remove_file(hdcp2->mdev.this_device, &dev_attr_enable);
+	device_remove_file(hdcp2->mdev.this_device, &dev_attr_status);
+	misc_deregister(&(hdcp2->mdev));
+}
+
+static int eswin_hdmi_hdcp2_probe(struct platform_device *pdev)
+{
+	struct dw_hdcp2 *hdcp2;
+	struct device *dev = &pdev->dev;
+	int numa_id;
+
+	hdcp2 = devm_kzalloc(dev, sizeof(*hdcp2), GFP_KERNEL);
+	if (!hdcp2)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, hdcp2);
+	if(of_property_read_u32(dev->of_node, "numa-node-id", &numa_id)) {
+		numa_id = 0;
+	}
+	dev_info(dev, "%s numa_id:%d\n", __func__, numa_id);
+	hdcp2->numa_id = numa_id;
+
+	hdcp2->dev = dev;
+	hdcp2->stop = dw_hdcp2_stop;
+	hdcp2->start = dw_hdcp2_start;
+	rigister_hld_device(hdcp2);
+	dw_hdmi2_hdcp2_clk_enable(hdcp2);
+	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	dw_hdmi_hdcp2_init(hdcp2);
+	dw_hdmi_hdcp2_start(3, numa_id);
+
+	create_device_node(hdcp2);
+	return 0;
+}
+
+static int eswin_hdmi_hdcp2_remove(struct platform_device *pdev)
+{
+	struct dw_hdcp2 *hdcp2 = platform_get_drvdata(pdev);
+	dev_info(hdcp2->dev, "%s numa id:%d\n", __func__, hdcp2->numa_id);
+	dw_hdmi_hdcp2_remove(hdcp2);
+	end_device_node(hdcp2);
+	hld_exit(hdcp2);
+
+	clk_disable_unprepare(hdcp2->aclk);
+	clk_disable_unprepare(hdcp2->iesmclk);
+
+	return 0;
+}
+
+static int __maybe_unused dw_hdcp2_suspend(struct device *dev)
+{
+	dev_dbg(dev, "%s\n", __func__);
+	struct dw_hdcp2 *hdcp2 = dev_get_drvdata(dev);
+
+	clk_disable(hdcp2->aclk);
+	clk_disable(hdcp2->iesmclk);
+
+	return 0;
+}
+
+static int __maybe_unused dw_hdcp2_resume(struct device *dev)
+{
+	dev_dbg(dev, "%s\n", __func__);
+	struct dw_hdcp2 *hdcp2 = dev_get_drvdata(dev);
+
+	clk_enable(hdcp2->aclk);
+	clk_enable(hdcp2->iesmclk);
+
+	return 0;
+}
+
+static const struct dev_pm_ops dw_hdcp2_pm = {
+	.resume = dw_hdcp2_resume,
+	.suspend = dw_hdcp2_suspend,
+};
+
+static const struct of_device_id dw_hdmi_hdcp2_dt_ids[] = {
+	{
+		.compatible = "eswin,dw-hdmi-hdcp2",
+	},
+};
+MODULE_DEVICE_TABLE(of, dw_hdmi_hdcp2_dt_ids);
+
+struct platform_driver dw_hdmi_hdcp2_driver = {
+	.probe = eswin_hdmi_hdcp2_probe,
+	.remove = eswin_hdmi_hdcp2_remove,
+	.driver = {
+        .name = "dw-hdmi-hdcp2",
+        .owner = THIS_MODULE,
+		.pm = &dw_hdcp2_pm,
+        .of_match_table = of_match_ptr(dw_hdmi_hdcp2_dt_ids),
+    },
+};
+
+static int __init hdmi_hdcp2_init(void)
+{
+	return platform_driver_register(&dw_hdmi_hdcp2_driver);
+}
+
+static void hdmi_hdcp2_exit(void)
+{
+	platform_driver_unregister(&dw_hdmi_hdcp2_driver);
+}
+
+late_initcall_sync(hdmi_hdcp2_init);
+module_exit(hdmi_hdcp2_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Synopsys, Inc.");
+MODULE_DESCRIPTION("Linux Host Library Driver");
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_i2s_audio.c b/drivers/gpu/drm/eswin/dw_hdmi_i2s_audio.c
new file mode 100644
index 000000000000..33af4db6fa4c
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_i2s_audio.c
@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * dw-hdmi-i2s-audio.c
+ *
+ * Copyright (c) 2017 Renesas Solutions Corp.
+ * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+ *****************************************************************************
+ * ESWIN hdmi driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+
+#include <drm/bridge/dw_hdmi.h>
+#include <drm/drm_crtc.h>
+
+#include <sound/hdmi-codec.h>
+
+#include "dw-hdmi.h"
+#include "dw_hdmi_audio.h"
+
+#define DRIVER_NAME "dw-hdmi-i2s-audio"
+
+static inline void hdmi_write(struct dw_hdmi_i2s_audio_data *audio,
+			      u8 val, int offset)
+{
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	audio->write(hdmi, val, offset);
+}
+
+static inline u8 hdmi_read(struct dw_hdmi_i2s_audio_data *audio, int offset)
+{
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	return audio->read(hdmi, offset);
+}
+
+static int dw_hdmi_i2s_hw_params(struct device *dev, void *data,
+				 struct hdmi_codec_daifmt *fmt,
+				 struct hdmi_codec_params *hparms)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+	u8 conf0 = 0;
+	u8 conf1 = 0;
+	u8 inputclkfs = 0;
+
+	/* it cares I2S only */
+	if (fmt->bit_clk_provider | fmt->frame_clk_provider) {
+		dev_err(dev, "unsupported clock settings\n");
+		return -EINVAL;
+	}
+
+	/* Reset the FIFOs before applying new params */
+	hdmi_write(audio, HDMI_AUD_CONF0_SW_RESET, HDMI_AUD_CONF0);
+	hdmi_write(audio, (u8)~HDMI_MC_SWRSTZ_I2SSWRST_REQ, HDMI_MC_SWRSTZ);
+
+	inputclkfs	= HDMI_AUD_INPUTCLKFS_64FS;
+	conf0		= (HDMI_AUD_CONF0_I2S_SELECT | HDMI_AUD_CONF0_I2S_EN0);
+
+	/* Enable the required i2s lanes */
+	switch (hparms->channels) {
+	case 7 ... 8:
+		conf0 |= HDMI_AUD_CONF0_I2S_EN3;
+		fallthrough;
+	case 5 ... 6:
+		conf0 |= HDMI_AUD_CONF0_I2S_EN2;
+		fallthrough;
+	case 3 ... 4:
+		conf0 |= HDMI_AUD_CONF0_I2S_EN1;
+		/* Fall-thru */
+	}
+
+	switch (hparms->sample_width) {
+	case 16:
+		conf1 = HDMI_AUD_CONF1_WIDTH_16;
+		break;
+	case 24:
+	case 32:
+		conf1 = HDMI_AUD_CONF1_WIDTH_24;
+		break;
+	}
+
+	switch (fmt->fmt) {
+	case HDMI_I2S:
+		conf1 |= HDMI_AUD_CONF1_MODE_I2S;
+		break;
+	case HDMI_RIGHT_J:
+		conf1 |= HDMI_AUD_CONF1_MODE_RIGHT_J;
+		break;
+	case HDMI_LEFT_J:
+		conf1 |= HDMI_AUD_CONF1_MODE_LEFT_J;
+		break;
+	case HDMI_DSP_A:
+		conf1 |= HDMI_AUD_CONF1_MODE_BURST_1;
+		break;
+	case HDMI_DSP_B:
+		conf1 |= HDMI_AUD_CONF1_MODE_BURST_2;
+		break;
+	default:
+		dev_err(dev, "unsupported format\n");
+		return -EINVAL;
+	}
+
+	dw_hdmi_set_sample_rate(hdmi, hparms->sample_rate);
+	dw_hdmi_set_channel_status(hdmi, hparms->iec.status);
+	dw_hdmi_set_channel_count(hdmi, hparms->channels);
+	dw_hdmi_set_channel_allocation(hdmi, hparms->cea.channel_allocation);
+
+	hdmi_write(audio, inputclkfs, HDMI_AUD_INPUTCLKFS);
+	hdmi_write(audio, conf0, HDMI_AUD_CONF0);
+	hdmi_write(audio, conf1, HDMI_AUD_CONF1);
+
+	return 0;
+}
+
+static int dw_hdmi_i2s_audio_startup(struct device *dev, void *data)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	dw_hdmi_audio_enable(hdmi);
+
+	return 0;
+}
+
+static void dw_hdmi_i2s_audio_shutdown(struct device *dev, void *data)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	dw_hdmi_audio_disable(hdmi);
+}
+
+static int dw_hdmi_i2s_get_eld(struct device *dev, void *data, uint8_t *buf,
+			       size_t len)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	u8 *eld;
+
+	eld = audio->get_eld(audio->hdmi);
+	if (eld)
+		memcpy(buf, eld, min_t(size_t, MAX_ELD_BYTES, len));
+	else
+		/* Pass en empty ELD if connector not available */
+		memset(buf, 0, len);
+
+	return 0;
+}
+
+static int dw_hdmi_i2s_get_dai_id(struct snd_soc_component *component,
+				  struct device_node *endpoint)
+{
+	struct of_endpoint of_ep;
+	int ret;
+
+	ret = of_graph_parse_endpoint(endpoint, &of_ep);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * HDMI sound should be located as reg = <2>
+	 * Then, it is sound port 0
+	 */
+	if (of_ep.port == 2)
+		return 0;
+
+	return -EINVAL;
+}
+
+static int dw_hdmi_i2s_hook_plugged_cb(struct device *dev, void *data,
+				       hdmi_codec_plugged_cb fn,
+				       struct device *codec_dev)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	return dw_hdmi_set_plugged_cb(hdmi, fn, codec_dev);
+}
+
+static const struct hdmi_codec_ops dw_hdmi_i2s_ops = {
+	.hw_params	= dw_hdmi_i2s_hw_params,
+	.audio_startup  = dw_hdmi_i2s_audio_startup,
+	.audio_shutdown	= dw_hdmi_i2s_audio_shutdown,
+	.get_eld	= dw_hdmi_i2s_get_eld,
+	.get_dai_id	= dw_hdmi_i2s_get_dai_id,
+	.hook_plugged_cb = dw_hdmi_i2s_hook_plugged_cb,
+};
+
+static int snd_dw_hdmi_probe(struct platform_device *pdev)
+{
+	struct dw_hdmi_i2s_audio_data *audio = pdev->dev.platform_data;
+	struct platform_device_info pdevinfo;
+	struct hdmi_codec_pdata pdata;
+	struct platform_device *platform;
+
+	memset(&pdata, 0, sizeof(pdata));
+	pdata.ops		= &dw_hdmi_i2s_ops;
+	pdata.i2s		= 1;
+	pdata.max_i2s_channels	= 8;
+	pdata.data		= audio;
+
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+	pdevinfo.parent		= pdev->dev.parent;
+	pdevinfo.id		= PLATFORM_DEVID_AUTO;
+	pdevinfo.name		= HDMI_CODEC_DRV_NAME;
+	pdevinfo.data		= &pdata;
+	pdevinfo.size_data	= sizeof(pdata);
+	pdevinfo.dma_mask	= DMA_BIT_MASK(32);
+
+	platform = platform_device_register_full(&pdevinfo);
+	if (IS_ERR(platform))
+		return PTR_ERR(platform);
+
+	dev_set_drvdata(&pdev->dev, platform);
+
+	return 0;
+}
+
+static void snd_dw_hdmi_remove(struct platform_device *pdev)
+{
+	struct platform_device *platform = dev_get_drvdata(&pdev->dev);
+
+	platform_device_unregister(platform);
+}
+
+struct platform_driver snd_dw_hdmi_driver = {
+	.probe	= snd_dw_hdmi_probe,
+	.remove_new = snd_dw_hdmi_remove,
+	.driver	= {
+		.name = DRIVER_NAME,
+	},
+};
+//module_platform_driver(snd_dw_hdmi_driver);
+
+MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>");
+MODULE_DESCRIPTION("Synopsis Designware HDMI I2S ALSA SoC interface");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/gpu/drm/eswin/es_crtc.c b/drivers/gpu/drm/eswin/es_crtc.c
new file mode 100644
index 000000000000..54b74b638069
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_crtc.c
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_vblank.h>
+
+#include "es_drm.h"
+#include "es_crtc.h"
+
+void es_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	drm_crtc_cleanup(crtc);
+	kfree(es_crtc);
+}
+
+static void es_crtc_reset(struct drm_crtc *crtc)
+{
+	struct es_crtc_state *state;
+
+	if (crtc->state) {
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+
+		state = to_es_crtc_state(crtc->state);
+		kfree(state);
+		crtc->state = NULL;
+	}
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state == NULL)
+		return;
+
+	__drm_atomic_helper_crtc_reset(crtc, &state->base);
+
+	state->sync_mode = ES_SINGLE_DC;
+	state->output_fmt = MEDIA_BUS_FMT_RBG888_1X24;
+	state->encoder_type = DRM_MODE_ENCODER_NONE;
+#ifdef CONFIG_ESWIN_MMU
+	state->mmu_prefetch = ES_MMU_PREFETCH_DISABLE;
+#endif
+}
+
+static struct drm_crtc_state *
+es_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
+{
+	struct es_crtc_state *ori_state;
+	struct es_crtc_state *state;
+
+	if (WARN_ON(!crtc->state))
+		return NULL;
+
+	ori_state = to_es_crtc_state(crtc->state);
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &state->base);
+
+	state->sync_mode = ori_state->sync_mode;
+	state->output_fmt = ori_state->output_fmt;
+	state->encoder_type = ori_state->encoder_type;
+	state->bpp = ori_state->bpp;
+	state->dither_enable = ori_state->dither_enable;
+	state->underflow = ori_state->underflow;
+	state->bg_color = ori_state->bg_color;
+#ifdef CONFIG_ESWIN_MMU
+	state->mmu_prefetch = ori_state->mmu_prefetch;
+#endif
+
+	return &state->base;
+}
+
+static void es_crtc_atomic_destroy_state(struct drm_crtc *crtc,
+					 struct drm_crtc_state *state)
+{
+	__drm_atomic_helper_crtc_destroy_state(state);
+	kfree(to_es_crtc_state(state));
+}
+
+static int es_crtc_atomic_set_property(struct drm_crtc *crtc,
+				       struct drm_crtc_state *state,
+				       struct drm_property *property,
+				       uint64_t val)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(state);
+
+	if (property == es_crtc->sync_mode)
+		es_crtc_state->sync_mode = val;
+	else if (property == es_crtc->mmu_prefetch)
+		es_crtc_state->mmu_prefetch = val;
+	else if (property == es_crtc->dither)
+		es_crtc_state->dither_enable = val;
+	else if (property == es_crtc->bg_color)
+		es_crtc_state->bg_color = val;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int es_crtc_atomic_get_property(struct drm_crtc *crtc,
+				       const struct drm_crtc_state *state,
+				       struct drm_property *property,
+				       uint64_t *val)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	const struct es_crtc_state *es_crtc_state =
+		container_of(state, const struct es_crtc_state, base);
+
+	if (property == es_crtc->sync_mode)
+		*val = es_crtc_state->sync_mode;
+	else if (property == es_crtc->mmu_prefetch)
+		*val = es_crtc_state->mmu_prefetch;
+	else if (property == es_crtc->dither)
+		*val = es_crtc_state->dither_enable;
+	else if (property == es_crtc->bg_color)
+		*val = es_crtc_state->bg_color;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int es_crtc_debugfs_show(struct seq_file *s, void *data)
+{
+	struct drm_crtc *crtc = s->private;
+	struct es_crtc_state *crtc_state = to_es_crtc_state(crtc->state);
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+
+	seq_printf(s, "crtc[%u]: %s\n", crtc->base.id, crtc->name);
+	seq_printf(s, "\tactive = %d\n", crtc->state->active);
+	seq_printf(s, "\tsize = %dx%d\n", mode->hdisplay, mode->vdisplay);
+	seq_printf(s, "\tbpp = %u\n", crtc_state->bpp);
+	seq_printf(s, "\tunderflow = %d\n", crtc_state->underflow);
+
+	return 0;
+}
+
+static int es_crtc_debugfs_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, es_crtc_debugfs_show, inode->i_private);
+}
+
+static const struct file_operations es_crtc_debugfs_fops = {
+	.open = es_crtc_debugfs_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int es_crtc_debugfs_init(struct drm_crtc *crtc)
+{
+	debugfs_create_file("status", 0444, crtc->debugfs_entry, crtc,
+			    &es_crtc_debugfs_fops);
+
+	return 0;
+}
+#else
+static int es_crtc_debugfs_init(struct drm_crtc *crtc)
+{
+	return 0;
+}
+#endif /* CONFIG_DEBUG_FS */
+
+static int es_crtc_late_register(struct drm_crtc *crtc)
+{
+	return es_crtc_debugfs_init(crtc);
+}
+
+static int es_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	es_crtc->funcs->enable_vblank(es_crtc->dev, true);
+
+	return 0;
+}
+
+static void es_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	es_crtc->funcs->enable_vblank(es_crtc->dev, false);
+}
+
+static const struct drm_crtc_funcs es_crtc_funcs = {
+	.set_config = drm_atomic_helper_set_config,
+	.destroy = es_crtc_destroy,
+	.page_flip = drm_atomic_helper_page_flip,
+	.reset = es_crtc_reset,
+	.atomic_duplicate_state = es_crtc_atomic_duplicate_state,
+	.atomic_destroy_state = es_crtc_atomic_destroy_state,
+	.atomic_set_property = es_crtc_atomic_set_property,
+	.atomic_get_property = es_crtc_atomic_get_property,
+	// .gamma_set      = drm_atomic_helper_legacy_gamma_set, /* TODO: */
+	.late_register = es_crtc_late_register,
+	.enable_vblank = es_crtc_enable_vblank,
+	.disable_vblank = es_crtc_disable_vblank,
+};
+
+static u8 cal_pixel_bits(u32 bus_format)
+{
+	u8 bpp;
+
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB565_1X16:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		bpp = 16;
+		break;
+	case MEDIA_BUS_FMT_RGB666_1X18:
+	case MEDIA_BUS_FMT_RGB666_1X24_CPADHI:
+		bpp = 18;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		bpp = 20;
+		break;
+	case MEDIA_BUS_FMT_BGR888_1X24:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+		bpp = 24;
+		break;
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		bpp = 30;
+		break;
+	default:
+		bpp = 24;
+		break;
+	}
+
+	return bpp;
+}
+
+static bool es_crtc_mode_fixup(struct drm_crtc *crtc,
+			       const struct drm_display_mode *mode,
+			       struct drm_display_mode *adjusted_mode)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	return es_crtc->funcs->mode_fixup(es_crtc->dev, mode, adjusted_mode);
+}
+
+static void es_crtc_atomic_enable(struct drm_crtc *crtc,
+				  struct drm_atomic_state *old_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc->state);
+
+	es_crtc_state->bpp = cal_pixel_bits(es_crtc_state->output_fmt);
+
+	es_crtc->funcs->enable(es_crtc->dev, crtc);
+
+	drm_crtc_vblank_on(crtc);
+}
+
+static void es_crtc_atomic_disable(struct drm_crtc *crtc,
+				   struct drm_atomic_state *old_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	drm_crtc_vblank_off(crtc);
+
+	es_crtc->funcs->disable(es_crtc->dev);
+
+	if (crtc->state->event && !crtc->state->active) {
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		spin_unlock_irq(&crtc->dev->event_lock);
+
+		crtc->state->event = NULL;
+	}
+}
+
+static void es_crtc_atomic_begin(struct drm_crtc *crtc,
+				 struct drm_atomic_state *old_crtc_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct device *dev = es_crtc->dev;
+	struct drm_property_blob *blob = crtc->state->gamma_lut;
+	struct drm_color_lut *lut;
+
+	if (crtc->state->color_mgmt_changed) {
+		if ((blob) && (blob->length)) {
+			lut = blob->data;
+			es_crtc->funcs->set_gamma(dev, lut,
+						  blob->length / sizeof(*lut));
+			es_crtc->funcs->enable_gamma(dev, true);
+		} else {
+			es_crtc->funcs->enable_gamma(dev, false);
+		}
+	}
+}
+
+static void es_crtc_atomic_flush(struct drm_crtc *crtc,
+				 struct drm_atomic_state *old_crtc_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct drm_pending_vblank_event *event = crtc->state->event;
+
+	es_crtc->funcs->commit(es_crtc->dev);
+
+	if (event) {
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_arm_vblank_event(crtc, event);
+		spin_unlock_irq(&crtc->dev->event_lock);
+		crtc->state->event = NULL;
+	}
+}
+
+static const struct drm_crtc_helper_funcs es_crtc_helper_funcs = {
+	.mode_fixup = es_crtc_mode_fixup,
+	.atomic_enable = es_crtc_atomic_enable,
+	.atomic_disable = es_crtc_atomic_disable,
+	.atomic_begin = es_crtc_atomic_begin,
+	.atomic_flush = es_crtc_atomic_flush,
+};
+
+static const struct drm_prop_enum_list es_sync_mode_enum_list[] = {
+	{ ES_SINGLE_DC, "single dc mode" },
+	{ ES_MULTI_DC_PRIMARY, "primary dc for multi dc mode" },
+	{ ES_MULTI_DC_SECONDARY, "secondary dc for multi dc mode" },
+};
+
+#ifdef CONFIG_ESWIN_MMU
+static const struct drm_prop_enum_list es_mmu_prefetch_enum_list[] = {
+	{ ES_MMU_PREFETCH_DISABLE, "disable mmu prefetch" },
+	{ ES_MMU_PREFETCH_ENABLE, "enable mmu prefetch" },
+};
+#endif
+
+struct es_crtc *es_crtc_create(struct drm_device *drm_dev,
+			       struct es_dc_info *info)
+{
+	struct es_crtc *crtc;
+	int ret;
+
+	if (!info)
+		return NULL;
+
+	crtc = kzalloc(sizeof(struct es_crtc), GFP_KERNEL);
+	if (!crtc)
+		return NULL;
+
+	ret = drm_crtc_init_with_planes(drm_dev, &crtc->base, NULL, NULL,
+					&es_crtc_funcs,
+					info->name ? info->name : NULL);
+	if (ret)
+		goto err_free_crtc;
+
+	drm_crtc_helper_add(&crtc->base, &es_crtc_helper_funcs);
+
+	/* Set up the crtc properties */
+	if (info->pipe_sync) {
+		crtc->sync_mode = drm_property_create_enum(
+			drm_dev, 0, "SYNC_MODE", es_sync_mode_enum_list,
+			ARRAY_SIZE(es_sync_mode_enum_list));
+
+		if (!crtc->sync_mode)
+			goto err_cleanup_crts;
+
+		drm_object_attach_property(&crtc->base.base, crtc->sync_mode,
+					   ES_SINGLE_DC);
+	}
+
+	if (info->gamma_size) {
+		ret = drm_mode_crtc_set_gamma_size(&crtc->base,
+						   info->gamma_size);
+		if (ret)
+			goto err_cleanup_crts;
+
+		drm_crtc_enable_color_mgmt(&crtc->base, 0, false,
+					   info->gamma_size);
+	}
+
+	if (info->background) {
+		crtc->bg_color = drm_property_create_range(
+			drm_dev, 0, "BG_COLOR", 0, 0xffffffff);
+
+		if (!crtc->bg_color)
+			goto err_cleanup_crts;
+
+		drm_object_attach_property(&crtc->base.base, crtc->bg_color, 0);
+	}
+
+	crtc->dither = drm_property_create_bool(drm_dev, 0, "DITHER_ENABLED");
+	if (!crtc->dither)
+		goto err_cleanup_crts;
+
+	drm_object_attach_property(&crtc->base.base, crtc->dither, 0);
+
+#ifdef CONFIG_ESWIN_MMU
+	if (info->mmu_prefetch) {
+		crtc->mmu_prefetch = drm_property_create_enum(
+			drm_dev, 0, "MMU_PREFETCH", es_mmu_prefetch_enum_list,
+			ARRAY_SIZE(es_mmu_prefetch_enum_list));
+		if (!crtc->mmu_prefetch)
+			goto err_cleanup_crts;
+
+		drm_object_attach_property(&crtc->base.base, crtc->mmu_prefetch,
+					   ES_MMU_PREFETCH_DISABLE);
+	}
+#endif
+
+	crtc->max_bpc = info->max_bpc;
+	crtc->color_formats = info->color_formats;
+	return crtc;
+
+err_cleanup_crts:
+	drm_crtc_cleanup(&crtc->base);
+
+err_free_crtc:
+	kfree(crtc);
+	return NULL;
+}
+
+void es_crtc_handle_vblank(struct drm_crtc *crtc, bool underflow)
+{
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc->state);
+
+	drm_crtc_handle_vblank(crtc);
+
+	es_crtc_state->underflow = underflow;
+}
diff --git a/drivers/gpu/drm/eswin/es_crtc.h b/drivers/gpu/drm/eswin/es_crtc.h
new file mode 100644
index 000000000000..3c8ea4c17d95
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_crtc.h
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_CRTC_H__
+#define __ES_CRTC_H__
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "es_type.h"
+
+struct es_crtc_funcs {
+	void (*enable)(struct device *dev, struct drm_crtc *crtc);
+	void (*disable)(struct device *dev);
+	bool (*mode_fixup)(struct device *dev,
+			   const struct drm_display_mode *mode,
+			   struct drm_display_mode *adjusted_mode);
+	void (*set_gamma)(struct device *dev, struct drm_color_lut *lut,
+			  unsigned int size);
+	void (*enable_gamma)(struct device *dev, bool enable);
+	void (*enable_vblank)(struct device *dev, bool enable);
+	void (*commit)(struct device *dev);
+};
+
+struct es_crtc_state {
+	struct drm_crtc_state base;
+
+	u32 sync_mode;
+	u32 output_fmt;
+	u32 bg_color;
+	u8 encoder_type;
+	u8 mmu_prefetch;
+	u8 bpp;
+	bool dither_enable;
+	bool underflow;
+};
+
+struct es_crtc {
+	struct drm_crtc base;
+	struct device *dev;
+	struct drm_pending_vblank_event *event;
+	unsigned int max_bpc;
+	unsigned int color_formats; /* supported color format */
+
+	struct drm_property *sync_mode;
+	struct drm_property *mmu_prefetch;
+	struct drm_property *dither;
+	struct drm_property *bg_color;
+
+	const struct es_crtc_funcs *funcs;
+};
+
+void es_crtc_destroy(struct drm_crtc *crtc);
+
+struct es_crtc *es_crtc_create(struct drm_device *drm_dev,
+			       struct es_dc_info *info);
+void es_crtc_handle_vblank(struct drm_crtc *crtc, bool underflow);
+
+static inline struct es_crtc *to_es_crtc(struct drm_crtc *crtc)
+{
+	return container_of(crtc, struct es_crtc, base);
+}
+
+static inline struct es_crtc_state *
+to_es_crtc_state(struct drm_crtc_state *state)
+{
+	return container_of(state, struct es_crtc_state, base);
+}
+#endif /* __ES_CRTC_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dc.c b/drivers/gpu/drm/eswin/es_dc.c
new file mode 100644
index 000000000000..9ea5560062aa
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc.c
@@ -0,0 +1,1219 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/component.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/reset.h>
+#include <linux/delay.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_blend.h>
+
+#include "es_drm.h"
+#include "es_type.h"
+#include "es_dc_hw.h"
+#include "es_dc.h"
+#include "es_crtc.h"
+#include "es_drv.h"
+
+#define VO_ACLK_HIGHEST 800000000
+
+static inline void update_format(u32 format, struct dc_hw_fb *fb)
+{
+	u8 f;
+
+	switch (format) {
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_BGRX4444:
+		f = FORMAT_X4R4G4B4;
+		break;
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_BGRA4444:
+		f = FORMAT_A4R4G4B4;
+		break;
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_BGRX5551:
+		f = FORMAT_X1R5G5B5;
+		break;
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_BGRA5551:
+		f = FORMAT_A1R5G5B5;
+		break;
+	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_RGB565:
+		f = FORMAT_R5G6B5;
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_BGRX8888:
+		f = FORMAT_X8R8G8B8;
+		break;
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_BGRA8888:
+		f = FORMAT_A8R8G8B8;
+		break;
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+		f = FORMAT_YUY2;
+		break;
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+		f = FORMAT_UYVY;
+		break;
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV420:
+		f = FORMAT_YV12;
+		break;
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+		f = FORMAT_NV12;
+		break;
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+		f = FORMAT_NV16;
+		break;
+	case DRM_FORMAT_P010:
+		f = FORMAT_P010;
+		break;
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_BGRA1010102:
+		f = FORMAT_A2R10G10B10;
+		break;
+	default:
+		f = FORMAT_A8R8G8B8;
+		break;
+	}
+	fb->format = f;
+}
+
+static inline void update_swizzle(u32 format, struct dc_hw_fb *fb)
+{
+	fb->swizzle = SWIZZLE_ARGB;
+	fb->uv_swizzle = 0;
+
+	switch (format) {
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_RGBA1010102:
+		fb->swizzle = SWIZZLE_RGBA;
+		break;
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ABGR2101010:
+		fb->swizzle = SWIZZLE_ABGR;
+		break;
+	case DRM_FORMAT_BGRA4444:
+	case DRM_FORMAT_BGRX4444:
+	case DRM_FORMAT_BGRX5551:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRA5551:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_BGRA1010102:
+		fb->swizzle = SWIZZLE_BGRA;
+		break;
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV61:
+		fb->uv_swizzle = 1;
+		break;
+	default:
+		break;
+	}
+}
+
+static inline u8 to_es_rotation(unsigned int rotation)
+{
+	u8 rot;
+
+	switch (rotation & DRM_MODE_REFLECT_MASK) {
+	case DRM_MODE_REFLECT_X:
+		rot = FLIP_X;
+		return rot;
+	case DRM_MODE_REFLECT_Y:
+		rot = FLIP_Y;
+		return rot;
+	case DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y:
+		rot = FLIP_XY;
+		return rot;
+	default:
+		break;
+	}
+
+	switch (rotation & DRM_MODE_ROTATE_MASK) {
+	case DRM_MODE_ROTATE_0:
+		rot = ROT_0;
+		break;
+	case DRM_MODE_ROTATE_90:
+		rot = ROT_90;
+		break;
+	case DRM_MODE_ROTATE_180:
+		rot = ROT_180;
+		break;
+	case DRM_MODE_ROTATE_270:
+		rot = ROT_270;
+		break;
+	default:
+		rot = ROT_0;
+		break;
+	}
+
+	return rot;
+}
+
+static inline u8 to_es_yuv_color_space(u32 color_space)
+{
+	u8 cs;
+
+	switch (color_space) {
+	case DRM_COLOR_YCBCR_BT601:
+		cs = COLOR_SPACE_601;
+		break;
+	case DRM_COLOR_YCBCR_BT709:
+		cs = COLOR_SPACE_709;
+		break;
+	case DRM_COLOR_YCBCR_BT2020:
+		cs = COLOR_SPACE_2020;
+		break;
+	default:
+		cs = COLOR_SPACE_601;
+		break;
+	}
+
+	return cs;
+}
+
+static inline u8 to_es_tile_mode(u64 modifier)
+{
+	// DRM_FORMAT_MOD_ES_NORM_MODE_MASK 0x1F
+	return (u8)(modifier & 0x1F);
+}
+
+static int es_dc_clk_configs(struct device *dev, bool enable)
+{
+	int ret = 0;
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	if (enable) {
+		if (dc->dc_clkon) {
+			return 0;
+		}
+
+		ret = clk_prepare_enable(dc->cfg_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare/enable cfg_clk\n");
+			return ret;
+		}
+
+		ret = clk_prepare_enable(dc->pix_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare/enable pix_clk\n");
+			return ret;
+		}
+
+		ret = clk_prepare_enable(dc->axi_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare/enable axi_clk\n");
+			return ret;
+		}
+
+		dc->pix_clk_rate = clk_get_rate(dc->pix_clk);
+		dc->dc_clkon = true;
+	} else {
+		if (!dc->dc_clkon) {
+			return 0;
+		}
+		clk_disable_unprepare(dc->pix_clk);
+		clk_disable_unprepare(dc->axi_clk);
+		clk_disable_unprepare(dc->cfg_clk);
+		dc->dc_clkon = false;
+	}
+
+	return 0;
+}
+
+static void dc_deinit(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_interrupt(&dc->hw, 0);
+	dc_hw_deinit(&dc->hw);
+	es_dc_clk_configs(dev, false);
+}
+
+static int dc_init(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	int ret;
+	long rate;
+
+	dc->first_frame = true;
+
+	ret = clk_set_parent(dc->vo_mux, dc->spll0_fout1);
+	if (ret < 0) {
+		pr_err("DC: failed to set core clk parent: %d\n", ret);
+		return ret;
+	}
+
+	rate = clk_round_rate(dc->axi_clk, VO_ACLK_HIGHEST);
+	if (rate > 0) {
+		ret = clk_set_rate(dc->axi_clk, rate);
+		if (ret) {
+			pr_err("DC: failed to set axi clk: %d\n", ret);
+			return ret;
+		}
+	}
+
+	ret = es_dc_clk_configs(dev, true);
+	if (ret < 0) {
+		dev_err(dev, "failed to prepare/enable clks\n");
+		return ret;
+	}
+
+	ret = dc_hw_init(&dc->hw);
+	if (ret) {
+		dev_err(dev, "failed to init DC HW\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void es_dc_dump_enable(struct device *dev, dma_addr_t addr,
+			      unsigned int pitch)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_dump(&dc->hw, addr, pitch);
+}
+
+static void es_dc_dump_disable(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_disable_dump(&dc->hw);
+}
+
+static void es_dc_enable(struct device *dev, struct drm_crtc *crtc)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct es_crtc_state *crtc_state = to_es_crtc_state(crtc->state);
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct dc_hw_display display;
+	struct dc_hw_cursor cursor = { 0 };
+	int ret;
+
+	display.bus_format = crtc_state->output_fmt;
+	display.h_active = mode->hdisplay;
+	display.h_total = mode->htotal;
+	display.h_sync_start = mode->hsync_start;
+	display.h_sync_end = mode->hsync_end;
+	if (mode->flags & DRM_MODE_FLAG_PHSYNC)
+		display.h_sync_polarity = true;
+	else
+		display.h_sync_polarity = false;
+
+	display.v_active = mode->vdisplay;
+	display.v_total = mode->vtotal;
+	display.v_sync_start = mode->vsync_start;
+	display.v_sync_end = mode->vsync_end;
+	if (mode->flags & DRM_MODE_FLAG_PVSYNC)
+		display.v_sync_polarity = true;
+	else
+		display.v_sync_polarity = false;
+
+	display.sync_mode = crtc_state->sync_mode;
+	display.bg_color = crtc_state->bg_color;
+	display.dither_enable = crtc_state->dither_enable;
+
+	display.enable = true;
+	es_dc_clk_configs(dev, true);
+
+	if (dc->pix_clk_rate != mode->clock) {
+		if (mode->clock == 513820) {
+			ret = clk_set_parent(dc->pix_mux, dc->spll2_fout2);
+			if (ret < 0) {
+				dev_err(dev, "failed to set pix mux parent spll2, err:%d\n", ret);
+			}
+			/* set mode (2880*1800@90hz) clock to 520M */
+			clk_set_rate(dc->pix_clk, 520000000);
+		} else {
+			ret = clk_set_parent(dc->pix_mux, dc->vpll_fout1);
+			if (ret < 0) {
+				dev_err(dev, "failed to set pix mux parent vpll, err:%d\n", ret);
+			}
+			clk_set_rate(dc->pix_clk, mode->clock * 1000);
+		}
+		dc->pix_clk_rate = mode->clock;
+	}
+
+	if (crtc_state->encoder_type == DRM_MODE_ENCODER_DSI ||
+	    crtc_state->encoder_type == DRM_MODE_ENCODER_VIRTUAL ||
+	    crtc_state->encoder_type == DRM_MODE_ENCODER_NONE)
+		dc_hw_set_out(&dc->hw, OUT_DPI);
+	else
+		dc_hw_set_out(&dc->hw, OUT_DP);
+
+#ifdef CONFIG_ESWIN_MMU
+	if (crtc_state->mmu_prefetch == ES_MMU_PREFETCH_ENABLE)
+		dc_hw_enable_mmu_prefetch(&dc->hw, true);
+	else
+		dc_hw_enable_mmu_prefetch(&dc->hw, false);
+#endif
+
+	dc_hw_setup_display(&dc->hw, &display);
+	cursor.enable = false;
+	dc_hw_update_cursor(&dc->hw, &cursor);
+}
+
+static void es_dc_disable(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct dc_hw_display display;
+
+	display.enable = false;
+
+	dc_hw_setup_display(&dc->hw, &display);
+	es_dc_clk_configs(dev, false);
+}
+
+static bool es_dc_mode_fixup(struct device *dev,
+			     const struct drm_display_mode *mode,
+			     struct drm_display_mode *adjusted_mode)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	long clk_rate;
+
+	dev_dbg(dev, "before adjust mode clock:%d\n", adjusted_mode->clock);
+	if (adjusted_mode->clock == 513820)
+		return true;
+
+	if (dc->pix_clk) {
+		clk_rate = clk_round_rate(dc->pix_clk,
+					  adjusted_mode->clock * 1000);
+		adjusted_mode->clock = DIV_ROUND_UP(clk_rate, 1000);
+		dev_dbg(dev, "adjust mode clock:%d\n", adjusted_mode->clock);
+	}
+
+	return true;
+}
+
+static void es_dc_set_gamma(struct device *dev, struct drm_color_lut *lut,
+			    unsigned int size)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	u16 i, r, g, b;
+	u8 bits;
+
+	if (size != dc->hw.info->gamma_size) {
+		dev_err(dev, "gamma size does not match!\n");
+		return;
+	}
+
+	bits = dc->hw.info->gamma_bits;
+	for (i = 0; i < size; i++) {
+		r = drm_color_lut_extract(lut[i].red, bits);
+		g = drm_color_lut_extract(lut[i].green, bits);
+		b = drm_color_lut_extract(lut[i].blue, bits);
+		dc_hw_update_gamma(&dc->hw, i, r, g, b);
+	}
+}
+
+static void es_dc_enable_gamma(struct device *dev, bool enable)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_gamma(&dc->hw, enable);
+}
+
+static void es_dc_enable_vblank(struct device *dev, bool enable)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_interrupt(&dc->hw, enable);
+}
+
+static u32 calc_factor(u32 src, u32 dest)
+{
+	u32 factor = 1 << 16;
+
+	if ((src > 1) && (dest > 1))
+		factor = ((src - 1) << 16) / (dest - 1);
+
+	return factor;
+}
+
+static void update_scale(struct dc_hw_scale *scale, int width, int height,
+			 int dst_w, int dst_h, unsigned int rotation,
+			 struct dc_hw_roi *roi)
+{
+	int src_w, src_h, temp;
+
+	scale->enable = false;
+	if (roi->enable) {
+		src_w = roi->width;
+		src_h = roi->height;
+	} else {
+		src_w = width;
+		src_h = height;
+	}
+
+	if (drm_rotation_90_or_270(rotation)) {
+		temp = src_w;
+		src_w = src_h;
+		src_h = temp;
+	}
+
+	if (src_w != dst_w) {
+		scale->scale_factor_x = calc_factor(src_w, dst_w);
+		scale->enable = true;
+	} else {
+		scale->scale_factor_x = 1 << 16;
+	}
+	if (src_h != dst_h) {
+		scale->scale_factor_y = calc_factor(src_h, dst_h);
+		scale->enable = true;
+	} else {
+		scale->scale_factor_y = 1 << 16;
+	}
+}
+
+static void update_fb(struct es_plane *plane, struct dc_hw_fb *fb)
+{
+	struct drm_plane_state *state = plane->base.state;
+	struct drm_framebuffer *drm_fb = state->fb;
+	struct drm_rect *src = &state->src;
+
+	fb->y_address = plane->dma_addr[0];
+	fb->y_stride = drm_fb->pitches[0];
+	if (drm_fb->format->format == DRM_FORMAT_YVU420) {
+		fb->u_address = plane->dma_addr[2];
+		fb->v_address = plane->dma_addr[1];
+		fb->u_stride = drm_fb->pitches[2];
+		fb->v_stride = drm_fb->pitches[1];
+	} else {
+		fb->u_address = plane->dma_addr[1];
+		fb->v_address = plane->dma_addr[2];
+		fb->u_stride = drm_fb->pitches[1];
+		fb->v_stride = drm_fb->pitches[2];
+	}
+	fb->width = drm_rect_width(src) >> 16;
+	fb->height = drm_rect_height(src) >> 16;
+	fb->tile_mode = to_es_tile_mode(drm_fb->modifier);
+	fb->rotation = to_es_rotation(state->rotation);
+	fb->yuv_color_space = to_es_yuv_color_space(state->color_encoding);
+	fb->enable = state->visible;
+	update_format(drm_fb->format->format, fb);
+	update_swizzle(drm_fb->format->format, fb);
+}
+
+static void update_degamma(struct es_dc *dc, struct es_plane *plane,
+			   struct es_plane_state *plane_state)
+{
+	dc_hw_update_degamma(&dc->hw, plane->id, plane_state->degamma);
+	plane_state->degamma_changed = false;
+}
+
+void update_roi(struct es_dc *dc, enum dc_hw_plane_id id,
+		struct es_plane_state *plane_state, struct dc_hw_roi *roi,
+		struct dc_hw_fb *fb)
+{
+	struct drm_es_roi *data;
+	u16 src_w = fb->width;
+	u16 src_h = fb->height;
+
+	if (plane_state->roi) {
+		data = (struct drm_es_roi *)plane_state->roi->data;
+		if (data->enable) {
+			roi->x = data->roi_x;
+			roi->y = data->roi_y;
+			roi->width = (data->roi_w + data->roi_x > src_w) ?
+					     (src_w - data->roi_x) :
+					     data->roi_w;
+			roi->height = (data->roi_h + data->roi_y > src_h) ?
+					      (src_h - data->roi_y) :
+					      data->roi_h;
+			roi->enable = true;
+		} else {
+			roi->enable = false;
+			roi->width = src_w;
+			roi->height = src_h;
+		}
+
+		dc_hw_update_roi(&dc->hw, id, roi);
+	} else {
+		roi->enable = false;
+	}
+}
+
+static void update_color_mgmt(struct es_dc *dc, u8 id, struct dc_hw_fb *fb,
+			      struct es_plane_state *plane_state)
+{
+	struct drm_es_color_mgmt *data;
+	struct dc_hw_colorkey colorkey;
+
+	if (plane_state->color_mgmt) {
+		data = plane_state->color_mgmt->data;
+
+		fb->clear_enable = data->clear_enable;
+		fb->clear_value = data->clear_value;
+
+		if (data->colorkey > data->colorkey_high)
+			data->colorkey = data->colorkey_high;
+
+		colorkey.colorkey = data->colorkey;
+		colorkey.colorkey_high = data->colorkey_high;
+		colorkey.transparency = (data->transparency) ?
+						DC_TRANSPARENCY_KEY :
+						DC_TRANSPARENCY_OPAQUE;
+		dc_hw_update_colorkey(&dc->hw, id, &colorkey);
+	}
+}
+
+static void update_primary_plane(struct es_dc *dc, struct es_plane *plane)
+{
+	struct dc_hw_fb fb = { 0 };
+	struct dc_hw_scale scale;
+	struct drm_plane_state *state = plane->base.state;
+	struct es_plane_state *plane_state = to_es_plane_state(state);
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct dc_hw_roi roi;
+
+	update_fb(plane, &fb);
+
+	update_roi(dc, plane->id, plane_state, &roi, &fb);
+
+	update_scale(&scale, fb.width, fb.height, mode->hdisplay,
+		     mode->vdisplay, state->rotation, &roi);
+
+	if (plane_state->degamma_changed)
+		update_degamma(dc, plane, plane_state);
+
+	update_color_mgmt(dc, plane->id, &fb, plane_state);
+
+	dc_hw_update_plane(&dc->hw, plane->id, &fb, &scale);
+}
+
+static void update_overlay_plane(struct es_dc *dc, struct es_plane *plane)
+{
+	struct dc_hw_fb fb = { 0 };
+	struct dc_hw_scale scale;
+	struct dc_hw_position pos;
+	struct dc_hw_blend blend;
+	struct drm_plane_state *state = plane->base.state;
+	struct es_plane_state *plane_state = to_es_plane_state(state);
+	struct drm_rect *dest = &state->dst;
+	struct dc_hw_roi roi;
+
+	update_fb(plane, &fb);
+	update_roi(dc, plane->id, plane_state, &roi, &fb);
+	update_scale(&scale, fb.width, fb.height, drm_rect_width(dest),
+		     drm_rect_height(dest), state->rotation, &roi);
+
+	if (plane_state->degamma_changed)
+		update_degamma(dc, plane, plane_state);
+
+	update_color_mgmt(dc, plane->id, &fb, plane_state);
+
+	dc_hw_update_plane(&dc->hw, plane->id, &fb, &scale);
+
+	pos.start_x = dest->x1;
+	pos.start_y = dest->y1;
+	pos.end_x = dest->x2;
+	pos.end_y = dest->y2;
+	dc_hw_set_position(&dc->hw, &pos);
+
+	blend.alpha = (u8)(state->alpha >> 8);
+	blend.blend_mode = (u8)(state->pixel_blend_mode);
+	dc_hw_set_blend(&dc->hw, &blend);
+}
+
+static void update_cursor_size(struct drm_plane_state *state,
+			       struct dc_hw_cursor *cursor)
+{
+	u8 size_type;
+
+	switch (state->crtc_w) {
+	case 32:
+		size_type = CURSOR_SIZE_32X32;
+		break;
+	case 64:
+		size_type = CURSOR_SIZE_64X64;
+		break;
+	case 128:
+		size_type = CURSOR_SIZE_128X128;
+		break;
+	case 256:
+		size_type = CURSOR_SIZE_256X256;
+		break;
+	default:
+		size_type = CURSOR_SIZE_32X32;
+		break;
+	}
+
+	cursor->size = size_type;
+}
+
+static void update_cursor_plane(struct es_dc *dc, struct es_plane *plane)
+{
+	struct drm_plane_state *state = plane->base.state;
+	struct dc_hw_cursor cursor;
+
+	cursor.address = plane->dma_addr[0];
+
+	if (state->crtc_x > 0) {
+		cursor.x = state->crtc_x;
+		cursor.hot_x = 0;
+	} else {
+		cursor.hot_x = -state->crtc_x;
+		cursor.x = 0;
+	}
+	if (state->crtc_y > 0) {
+		cursor.y = state->crtc_y;
+		cursor.hot_y = 0;
+	} else {
+		cursor.hot_y = -state->crtc_y;
+		cursor.y = 0;
+	}
+
+	update_cursor_size(state, &cursor);
+	cursor.enable = true;
+
+	dc_hw_update_cursor(&dc->hw, &cursor);
+}
+
+static void es_dc_update_plane(struct device *dev, struct es_plane *plane)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	enum drm_plane_type type = plane->base.type;
+
+	switch (type) {
+	case DRM_PLANE_TYPE_PRIMARY:
+		update_primary_plane(dc, plane);
+		break;
+	case DRM_PLANE_TYPE_OVERLAY:
+		update_overlay_plane(dc, plane);
+		break;
+	case DRM_PLANE_TYPE_CURSOR:
+		update_cursor_plane(dc, plane);
+		break;
+	default:
+		break;
+	}
+}
+
+static void es_dc_disable_plane(struct device *dev, struct es_plane *plane)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	enum drm_plane_type type = plane->base.type;
+	struct dc_hw_fb fb = { 0 };
+	struct dc_hw_cursor cursor = { 0 };
+
+	switch (type) {
+	case DRM_PLANE_TYPE_PRIMARY:
+	case DRM_PLANE_TYPE_OVERLAY:
+		fb.enable = false;
+		dc_hw_update_plane(&dc->hw, plane->id, &fb, NULL);
+		break;
+	case DRM_PLANE_TYPE_CURSOR:
+		cursor.enable = false;
+		dc_hw_update_cursor(&dc->hw, &cursor);
+		break;
+	default:
+		break;
+	}
+}
+
+static bool es_dc_mod_supported(const struct es_plane_info *plane_info,
+				u64 modifier)
+{
+	const u64 *mods;
+
+	if (plane_info->modifiers == NULL)
+		return false;
+
+	for (mods = plane_info->modifiers; *mods != DRM_FORMAT_MOD_INVALID;
+	     mods++) {
+		if (*mods == modifier)
+			return true;
+	}
+
+	return false;
+}
+
+static int es_dc_check_plane(struct device *dev, struct es_plane *plane,
+			     struct drm_plane_state *state)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct drm_framebuffer *fb = state->fb;
+	const struct es_plane_info *plane_info;
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_crtc_state *crtc_state;
+
+	plane_info = &dc->hw.info->planes[plane->id];
+	if (plane_info == NULL)
+		return -EINVAL;
+
+	if (fb->width < plane_info->min_width ||
+	    fb->width > plane_info->max_width ||
+	    fb->height < plane_info->min_height ||
+	    fb->height > plane_info->max_height)
+		dev_err_once(dev, "buffer size may not support on plane%d.\n",
+			     plane->id);
+
+	if ((plane->base.type != DRM_PLANE_TYPE_CURSOR) &&
+	    (!es_dc_mod_supported(plane_info, fb->modifier))) {
+		dev_err(dev, "unsupported modifier on plane%d.\n", plane->id);
+		return -EINVAL;
+	}
+
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
+	if (IS_ERR(crtc_state))
+		return -EINVAL;
+
+	return drm_atomic_helper_check_plane_state(state, crtc_state,
+						   plane_info->min_scale,
+						   plane_info->max_scale, true,
+						   true);
+}
+
+static irqreturn_t dc_isr(int irq, void *data)
+{
+	struct es_dc *dc = data;
+
+	dc_hw_get_interrupt(&dc->hw);
+
+	if (!dc->dc_initialized) {
+		return IRQ_HANDLED;
+	}
+
+	es_crtc_handle_vblank(&dc->crtc->base, dc_hw_check_underflow(&dc->hw));
+
+	return IRQ_HANDLED;
+}
+
+static void es_dc_commit(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+#ifdef CONFIG_ESWIN_MMU
+	dc_mmu_flush(&dc->hw);
+#endif
+
+	if (!dc->first_frame) {
+		if (dc_hw_flip_in_progress(&dc->hw))
+			udelay(100);
+
+		dc_hw_enable_shadow_register(&dc->hw, false);
+	}
+
+	dc_hw_commit(&dc->hw);
+
+	if (dc->first_frame)
+		dc->first_frame = false;
+
+	if (!dc->dc_initialized)
+		dc->dc_initialized = true;
+
+	dc_hw_enable_shadow_register(&dc->hw, true);
+}
+
+static const struct es_crtc_funcs dc_crtc_funcs = {
+	.enable = es_dc_enable,
+	.disable = es_dc_disable,
+	.mode_fixup = es_dc_mode_fixup,
+	.set_gamma = es_dc_set_gamma,
+	.enable_gamma = es_dc_enable_gamma,
+	.enable_vblank = es_dc_enable_vblank,
+	.commit = es_dc_commit,
+};
+
+static const struct es_plane_funcs dc_plane_funcs = {
+	.update = es_dc_update_plane,
+	.disable = es_dc_disable_plane,
+	.check = es_dc_check_plane,
+};
+
+static const struct es_dc_funcs dc_funcs = {
+	.dump_enable = es_dc_dump_enable,
+	.dump_disable = es_dc_dump_disable,
+};
+
+static int dc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+#ifdef CONFIG_ESWIN_MMU
+	struct es_drm_private *priv = drm_dev->dev_private;
+#endif
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct device_node *port;
+	struct es_crtc *crtc;
+	struct es_dc_info *dc_info;
+	struct es_plane *plane;
+	struct drm_plane *drm_plane, *tmp;
+	struct es_plane_info *plane_info;
+	int i, ret;
+
+	if (!drm_dev || !dc) {
+		dev_err(dev, "devices are not created.\n");
+		return -ENODEV;
+	}
+
+	ret = dc_init(dev);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize DC hardware.\n");
+		return ret;
+	}
+
+#ifdef CONFIG_ESWIN_MMU
+	if (priv->mmu_constructed == false) {
+		ret = dc_mmu_construct(priv->dma_dev, &priv->mmu);
+		if (ret) {
+			dev_err(dev, "failed to construct DC MMU\n");
+			goto err_clean_dc;
+		}
+		priv->mmu_constructed = true;
+	}
+	ret = dc_hw_mmu_init(&dc->hw, priv->mmu);
+	if (ret) {
+		dev_err(dev, "failed to init DC MMU\n");
+		goto err_clean_dc;
+	}
+#endif
+
+	ret = es_drm_iommu_attach_device(drm_dev, dev);
+	if (ret < 0) {
+		dev_err(dev, "Failed to attached iommu device.\n");
+		goto err_clean_dc;
+	}
+
+	port = of_get_child_by_name(dev->of_node, "port");
+	if (!port) {
+		dev_err(dev, "no port node found\n");
+		goto err_detach_dev;
+	}
+	of_node_put(port);
+
+	dc_info = dc->hw.info;
+	crtc = es_crtc_create(drm_dev, dc_info);
+	if (!crtc) {
+		dev_err(dev, "Failed to create CRTC.\n");
+		ret = -ENOMEM;
+		goto err_detach_dev;
+	}
+
+	crtc->base.port = port;
+	crtc->dev = dev;
+	crtc->funcs = &dc_crtc_funcs;
+
+	for (i = 0; i < dc_info->plane_num; i++) {
+		plane_info = (struct es_plane_info *)&dc_info->planes[i];
+
+		plane = es_plane_create(drm_dev, plane_info,
+					drm_crtc_mask(&crtc->base));
+		if (!plane)
+			goto err_cleanup_planes;
+
+		plane->funcs = &dc_plane_funcs;
+
+		if (plane_info->type == DRM_PLANE_TYPE_PRIMARY) {
+			crtc->base.primary = &plane->base;
+			drm_dev->mode_config.min_width = plane_info->min_width;
+			drm_dev->mode_config.min_height =
+				plane_info->min_height;
+			drm_dev->mode_config.max_width = plane_info->max_width;
+			drm_dev->mode_config.max_height =
+				plane_info->max_height;
+		}
+
+		if (plane_info->type == DRM_PLANE_TYPE_CURSOR) {
+			crtc->base.cursor = &plane->base;
+			drm_dev->mode_config.cursor_width =
+				plane_info->max_width;
+			drm_dev->mode_config.cursor_height =
+				plane_info->max_height;
+		}
+	}
+
+	dc->crtc = crtc;
+	dc->funcs = &dc_funcs;
+
+	es_drm_update_pitch_alignment(drm_dev, dc_info->pitch_alignment);
+
+	es_dc_clk_configs(dev, false);
+
+	return 0;
+
+err_cleanup_planes:
+	list_for_each_entry_safe(drm_plane, tmp,
+				 &drm_dev->mode_config.plane_list, head)
+		if (drm_plane->possible_crtcs == drm_crtc_mask(&crtc->base))
+			es_plane_destory(drm_plane);
+
+	es_crtc_destroy(&crtc->base);
+err_detach_dev:
+	es_drm_iommu_detach_device(drm_dev, dev);
+err_clean_dc:
+	dc_deinit(dev);
+	return ret;
+}
+
+static void dc_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+
+	dc_deinit(dev);
+
+	es_drm_iommu_detach_device(drm_dev, dev);
+}
+
+const struct component_ops dc_component_ops = {
+	.bind = dc_bind,
+	.unbind = dc_unbind,
+};
+
+static void vo_qos_cfg(void)
+{
+	void __iomem *qos;
+
+#define VO_QOS_CSR 0x50281050UL
+	qos = ioremap(VO_QOS_CSR, 8);
+	if (!qos) {
+		printk("qos ioremap fail---------------\n");
+		return;
+	}
+	writel(0x9, qos);
+	writel(0x9, (char *)qos + 4);
+
+	iounmap(qos);
+	return;
+}
+
+static const struct of_device_id dc_driver_dt_match[] = {
+	{
+		.compatible = "eswin,dc",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, dc_driver_dt_match);
+
+static int dc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct es_dc *dc;
+	int irq, ret;
+
+	dc = devm_kzalloc(dev, sizeof(*dc), GFP_KERNEL);
+	if (!dc)
+		return -ENOMEM;
+
+	dc->hw.hi_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(dc->hw.hi_base))
+		return PTR_ERR(dc->hw.hi_base);
+
+#ifdef CONFIG_ESWIN_MMU
+	dc->hw.mmu_base = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(dc->hw.mmu_base))
+		return PTR_ERR(dc->hw.mmu_base);
+#endif
+
+	dc->hw.reg_base = devm_platform_ioremap_resource(pdev, 2);
+	if (IS_ERR(dc->hw.reg_base))
+		return PTR_ERR(dc->hw.reg_base);
+
+	irq = platform_get_irq(pdev, 0);
+	ret = devm_request_irq(dev, irq, dc_isr, 0, dev_name(dev), dc);
+	if (ret < 0) {
+		dev_err(dev, "Failed to install irq:%u.\n", irq);
+		return ret;
+	}
+
+	dc->vo_mux = devm_clk_get(dev, "vo_mux");
+	if (IS_ERR(dc->vo_mux)) {
+		ret = PTR_ERR(dc->vo_mux);
+		dev_err(dev, "failed to get vo_mux: %d\n", ret);
+		return ret;
+	}
+
+	dc->spll0_fout1 = devm_clk_get_optional(dev, "spll0_fout1");
+	if (IS_ERR(dc->spll0_fout1)) {
+		dev_err(dev, "failed to get spll0_fout1 source\n");
+		return PTR_ERR(dc->spll0_fout1);
+	}
+
+	dc->cfg_clk = devm_clk_get_optional(dev, "cfg_clk");
+	if (IS_ERR(dc->cfg_clk)) {
+		dev_err(dev, "failed to get cfg_clk source\n");
+		return PTR_ERR(dc->cfg_clk);
+	}
+
+	dc->pix_clk = devm_clk_get_optional(dev, "pix_clk");
+	if (IS_ERR(dc->pix_clk)) {
+		dev_err(dev, "failed to get pix_clk source\n");
+		return PTR_ERR(dc->pix_clk);
+	}
+
+	dc->axi_clk = devm_clk_get_optional(dev, "axi_clk");
+	if (IS_ERR(dc->axi_clk)) {
+		dev_err(dev, "failed to get axi_clk source\n");
+		return PTR_ERR(dc->axi_clk);
+	}
+
+	dc->pix_mux = devm_clk_get_optional(dev, "pix_mux");
+	if (IS_ERR(dc->pix_mux)) {
+		dev_err(dev, "failed to get pix mux clk source\n");
+		return PTR_ERR(dc->pix_mux);
+	}
+
+	dc->spll2_fout2 = devm_clk_get_optional(dev, "spll2_fout2");
+	if (IS_ERR(dc->spll2_fout2)) {
+		dev_err(dev, "failed to get spll2 fout2 clk source\n");
+		return PTR_ERR(dc->spll2_fout2);
+	}
+
+	dc->vpll_fout1 = devm_clk_get_optional(dev, "vpll_fout1");
+	if (IS_ERR(dc->vpll_fout1)) {
+		dev_err(dev, "failed to get vpll fout1 clk source\n");
+		return PTR_ERR(dc->vpll_fout1);
+	}
+
+	dc->vo_arst = devm_reset_control_get_optional(dev, "vo_arst");
+	if (IS_ERR_OR_NULL(dc->vo_arst)) {
+		dev_err(dev, "Failed to vo_arst handle\n");
+		return PTR_ERR(dc->vo_arst);
+	}
+
+	dc->vo_prst = devm_reset_control_get_optional(dev, "vo_prst");
+	if (IS_ERR_OR_NULL(dc->vo_prst)) {
+		dev_err(dev, "Failed to vo_prst handle\n");
+		return PTR_ERR(dc->vo_prst);
+	}
+
+	dc->dc_arst = devm_reset_control_get_optional(dev, "dc_arst");
+	if (IS_ERR_OR_NULL(dc->dc_arst)) {
+		dev_err(dev, "Failed to dc_arst handle\n");
+		return PTR_ERR(dc->dc_arst);
+	}
+
+	dc->dc_prst = devm_reset_control_get_optional(dev, "dc_prst");
+	if (IS_ERR_OR_NULL(dc->dc_prst)) {
+		dev_err(dev, "Failed to dc_prst handle\n");
+		return PTR_ERR(dc->dc_prst);
+	}
+
+	/* reset dc first to ensure no data on axi bus */
+	if (dc->dc_arst) {
+		ret = reset_control_reset(dc->dc_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->dc_prst) {
+		ret = reset_control_reset(dc->dc_prst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_arst) {
+		ret = reset_control_reset(dc->vo_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_prst) {
+		ret = reset_control_reset(dc->vo_prst);
+		WARN_ON(0 != ret);
+	}
+
+	dev_set_drvdata(dev, dc);
+	vo_qos_cfg();
+	return component_add(dev, &dc_component_ops);
+}
+
+static int dc_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	component_del(dev, &dc_component_ops);
+
+	if (dc->dc_arst) {
+		ret = reset_control_assert(dc->dc_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->dc_prst) {
+		ret = reset_control_assert(dc->dc_prst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_arst) {
+		ret = reset_control_assert(dc->vo_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_prst) {
+		ret = reset_control_assert(dc->vo_prst);
+		WARN_ON(0 != ret);
+	}
+
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+struct platform_driver dc_platform_driver = {
+    .probe = dc_probe,
+    .remove = dc_remove,
+    .driver = {
+        .name = "es-dc",
+        .of_match_table = of_match_ptr(dc_driver_dt_match),
+    },
+};
+
+MODULE_DESCRIPTION("Eswin DC Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_dc.h b/drivers/gpu/drm/eswin/es_dc.h
new file mode 100644
index 000000000000..63c5c8d79aa4
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc.h
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_DC_H__
+#define __ES_DC_H__
+
+#include <linux/mm_types.h>
+#include <drm/drm_modes.h>
+
+#include "es_plane.h"
+#include "es_crtc.h"
+#include "es_dc_hw.h"
+#ifdef CONFIG_ESWIN_MMU
+#include "es_dc_mmu.h"
+#endif
+
+struct es_dc_funcs {
+	void (*dump_enable)(struct device *dev, dma_addr_t addr,
+			    unsigned int pitch);
+	void (*dump_disable)(struct device *dev);
+};
+
+struct es_dc {
+	struct es_crtc *crtc;
+	struct dc_hw hw;
+
+	struct clk *vo_mux;
+	struct clk *spll0_fout1;
+	struct clk *cfg_clk;
+	struct clk *pix_clk;
+	struct clk *axi_clk;
+	struct clk *pix_mux;
+	struct clk *spll2_fout2;
+	struct clk *vpll_fout1;
+	unsigned int pix_clk_rate; /* in KHz */
+
+	struct reset_control *vo_arst;
+	struct reset_control *vo_prst;
+	struct reset_control *dc_arst;
+	struct reset_control *dc_prst;
+
+	bool first_frame;
+	bool dc_initialized;
+	bool dc_clkon;
+
+	const struct es_dc_funcs *funcs;
+};
+
+extern struct platform_driver dc_platform_driver;
+#endif /* __ES_DC_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dc_hw.c b/drivers/gpu/drm/eswin/es_dc_hw.c
new file mode 100644
index 000000000000..4ba66b5bba2e
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_hw.c
@@ -0,0 +1,1631 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/io.h>
+#include <linux/bits.h>
+#include <linux/media-bus-format.h>
+#include <linux/delay.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_blend.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
+
+#include "es_drm.h"
+#include "es_type.h"
+#include "es_dc_hw.h"
+
+static const u32 horKernel[] = {
+	0x00000000, 0x20000000, 0x00002000, 0x00000000, 0x00000000, 0x00000000,
+	0x23fd1c03, 0x00000000, 0x00000000, 0x00000000, 0x181f0000, 0x000027e1,
+	0x00000000, 0x00000000, 0x00000000, 0x2b981468, 0x00000000, 0x00000000,
+	0x00000000, 0x10f00000, 0x00002f10, 0x00000000, 0x00000000, 0x00000000,
+	0x32390dc7, 0x00000000, 0x00000000, 0x00000000, 0x0af50000, 0x0000350b,
+	0x00000000, 0x00000000, 0x00000000, 0x3781087f, 0x00000000, 0x00000000,
+	0x00000000, 0x06660000, 0x0000399a, 0x00000000, 0x00000000, 0x00000000,
+	0x3b5904a7, 0x00000000, 0x00000000, 0x00000000, 0x033c0000, 0x00003cc4,
+	0x00000000, 0x00000000, 0x00000000, 0x3de1021f, 0x00000000, 0x00000000,
+	0x00000000, 0x01470000, 0x00003eb9, 0x00000000, 0x00000000, 0x00000000,
+	0x3f5300ad, 0x00000000, 0x00000000, 0x00000000, 0x00480000, 0x00003fb8,
+	0x00000000, 0x00000000, 0x00000000, 0x3fef0011, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00004000, 0x00000000, 0x00000000, 0x00000000,
+	0x20002000, 0x00000000, 0x00000000, 0x00000000, 0x1c030000, 0x000023fd,
+	0x00000000, 0x00000000, 0x00000000, 0x27e1181f, 0x00000000, 0x00000000,
+	0x00000000, 0x14680000, 0x00002b98, 0x00000000, 0x00000000, 0x00000000,
+	0x2f1010f0, 0x00000000, 0x00000000, 0x00000000, 0x0dc70000, 0x00003239,
+	0x00000000, 0x00000000, 0x00000000, 0x350b0af5, 0x00000000, 0x00000000,
+	0x00000000, 0x087f0000, 0x00003781, 0x00000000, 0x00000000, 0x00000000,
+	0x399a0666, 0x00000000, 0x00000000, 0x00000000, 0x04a70000, 0x00003b59,
+	0x00000000, 0x00000000, 0x00000000, 0x3cc4033c, 0x00000000, 0x00000000,
+	0x00000000, 0x021f0000,
+};
+#define H_COEF_SIZE (sizeof(horKernel) / sizeof(u32))
+
+static const u32 verKernel[] = {
+	0x00000000, 0x20000000, 0x00002000, 0x00000000, 0x00000000, 0x00000000,
+	0x23fd1c03, 0x00000000, 0x00000000, 0x00000000, 0x181f0000, 0x000027e1,
+	0x00000000, 0x00000000, 0x00000000, 0x2b981468, 0x00000000, 0x00000000,
+	0x00000000, 0x10f00000, 0x00002f10, 0x00000000, 0x00000000, 0x00000000,
+	0x32390dc7, 0x00000000, 0x00000000, 0x00000000, 0x0af50000, 0x0000350b,
+	0x00000000, 0x00000000, 0x00000000, 0x3781087f, 0x00000000, 0x00000000,
+	0x00000000, 0x06660000, 0x0000399a, 0x00000000, 0x00000000, 0x00000000,
+	0x3b5904a7, 0x00000000, 0x00000000, 0x00000000, 0x033c0000, 0x00003cc4,
+	0x00000000, 0x00000000, 0x00000000, 0x3de1021f, 0x00000000, 0x00000000,
+	0x00000000, 0x01470000, 0x00003eb9, 0x00000000, 0x00000000, 0x00000000,
+	0x3f5300ad, 0x00000000, 0x00000000, 0x00000000, 0x00480000, 0x00003fb8,
+	0x00000000, 0x00000000, 0x00000000, 0x3fef0011, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00004000, 0x00000000, 0xcdcd0000, 0xfdfdfdfd,
+	0xabababab, 0xabababab, 0x00000000, 0x00000000, 0x5ff5f456, 0x000f5f58,
+	0x02cc6c78, 0x02cc0c28, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee,
+};
+#define V_COEF_SIZE (sizeof(verKernel) / sizeof(u32))
+
+/*
+ * RGB 709->2020 conversion parameters
+ */
+static u16 RGB2RGB[RGB_TO_RGB_TABLE_SIZE] = { 10279, 5395, 709,	 1132, 15065,
+					      187,   269,  1442, 14674 };
+
+/*
+ * YUV601 to RGB conversion parameters
+ * YUV2RGB[0]  - [8] : C0 - C8;
+ * YUV2RGB[9]  - [11]: D0 - D2;
+ * YUV2RGB[12] - [13]: Y clamp min & max calue;
+ * YUV2RGB[14] - [15]: UV clamp min & max calue;
+ */
+static s32 YUV601_2RGB[YUV_TO_RGB_TABLE_SIZE] = {
+	1196, 0,       1640,   1196,	 -404, -836, 1196, 2076,
+	0,    -916224, 558336, -1202944, 64,   940,  64,   960
+};
+
+/*
+ * YUV709 to RGB conversion parameters
+ * YUV2RGB[0]  - [8] : C0 - C8;
+ * YUV2RGB[9]  - [11]: D0 - D2;
+ * YUV2RGB[12] - [13]: Y clamp min & max calue;
+ * YUV2RGB[14] - [15]: UV clamp min & max calue;
+ */
+static s32 YUV709_2RGB[YUV_TO_RGB_TABLE_SIZE] = {
+	1196, 0,	1844,	1196,	  -220, -548, 1196, 2172,
+	0,    -1020672, 316672, -1188608, 64,	940,  64,   960
+};
+
+/*
+ * YUV2020 to RGB conversion parameters
+ * YUV2RGB[0]  - [8] : C0 - C8;
+ * YUV2RGB[9]  - [11]: D0 - D2;
+ * YUV2RGB[12] - [13]: Y clamp min & max calue;
+ * YUV2RGB[14] - [15]: UV clamp min & max calue;
+ */
+static s32 YUV2020_2RGB[YUV_TO_RGB_TABLE_SIZE] = {
+	1196, 0,       1724,   1196,	 -192, -668, 1196, 2200,
+	0,    -959232, 363776, -1202944, 64,   940,  64,   960
+};
+
+/*
+ * RGB to YUV2020 conversion parameters
+ * RGB2YUV[0] - [8] : C0 - C8;
+ * RGB2YUV[9] - [11]: D0 - D2;
+ */
+static s16 RGB2YUV[RGB_TO_YUV_TABLE_SIZE] = { 230, 594,	 52,  -125, -323, 448,
+					      448, -412, -36, 64,   512,  512 };
+
+/*
+ * Degamma table for 709 color space data.
+ */
+static u16 DEGAMMA_709[DEGAMMA_SIZE] = {
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0005, 0x0007,
+	0x000a, 0x000d, 0x0011, 0x0015, 0x0019, 0x001e, 0x0024, 0x002a, 0x0030,
+	0x0038, 0x003f, 0x0048, 0x0051, 0x005a, 0x0064, 0x006f, 0x007b, 0x0087,
+	0x0094, 0x00a1, 0x00af, 0x00be, 0x00ce, 0x00de, 0x00ef, 0x0101, 0x0114,
+	0x0127, 0x013b, 0x0150, 0x0166, 0x017c, 0x0193, 0x01ac, 0x01c4, 0x01de,
+	0x01f9, 0x0214, 0x0230, 0x024d, 0x026b, 0x028a, 0x02aa, 0x02ca, 0x02ec,
+	0x030e, 0x0331, 0x0355, 0x037a, 0x03a0, 0x03c7, 0x03ef, 0x0418, 0x0441,
+	0x046c, 0x0498, 0x04c4, 0x04f2, 0x0520, 0x0550, 0x0581, 0x05b2, 0x05e5,
+	0x0618, 0x064d, 0x0682, 0x06b9, 0x06f0, 0x0729, 0x0763, 0x079d, 0x07d9,
+	0x0816, 0x0854, 0x0893, 0x08d3, 0x0914, 0x0956, 0x0999, 0x09dd, 0x0a23,
+	0x0a69, 0x0ab1, 0x0afa, 0x0b44, 0x0b8f, 0x0bdb, 0x0c28, 0x0c76, 0x0cc6,
+	0x0d17, 0x0d69, 0x0dbb, 0x0e10, 0x0e65, 0x0ebb, 0x0f13, 0x0f6c, 0x0fc6,
+	0x1021, 0x107d, 0x10db, 0x113a, 0x119a, 0x11fb, 0x125d, 0x12c1, 0x1325,
+	0x138c, 0x13f3, 0x145b, 0x14c5, 0x1530, 0x159c, 0x160a, 0x1678, 0x16e8,
+	0x175a, 0x17cc, 0x1840, 0x18b5, 0x192b, 0x19a3, 0x1a1c, 0x1a96, 0x1b11,
+	0x1b8e, 0x1c0c, 0x1c8c, 0x1d0c, 0x1d8e, 0x1e12, 0x1e96, 0x1f1c, 0x1fa3,
+	0x202c, 0x20b6, 0x2141, 0x21ce, 0x225c, 0x22eb, 0x237c, 0x240e, 0x24a1,
+	0x2536, 0x25cc, 0x2664, 0x26fc, 0x2797, 0x2832, 0x28cf, 0x296e, 0x2a0e,
+	0x2aaf, 0x2b51, 0x2bf5, 0x2c9b, 0x2d41, 0x2dea, 0x2e93, 0x2f3e, 0x2feb,
+	0x3099, 0x3148, 0x31f9, 0x32ab, 0x335f, 0x3414, 0x34ca, 0x3582, 0x363c,
+	0x36f7, 0x37b3, 0x3871, 0x3930, 0x39f1, 0x3ab3, 0x3b77, 0x3c3c, 0x3d02,
+	0x3dcb, 0x3e94, 0x3f5f, 0x402c, 0x40fa, 0x41ca, 0x429b, 0x436d, 0x4442,
+	0x4517, 0x45ee, 0x46c7, 0x47a1, 0x487d, 0x495a, 0x4a39, 0x4b19, 0x4bfb,
+	0x4cde, 0x4dc3, 0x4eaa, 0x4f92, 0x507c, 0x5167, 0x5253, 0x5342, 0x5431,
+	0x5523, 0x5616, 0x570a, 0x5800, 0x58f8, 0x59f1, 0x5aec, 0x5be9, 0x5ce7,
+	0x5de6, 0x5ee7, 0x5fea, 0x60ef, 0x61f5, 0x62fc, 0x6406, 0x6510, 0x661d,
+	0x672b, 0x683b, 0x694c, 0x6a5f, 0x6b73, 0x6c8a, 0x6da2, 0x6ebb, 0x6fd6,
+	0x70f3, 0x7211, 0x7331, 0x7453, 0x7576, 0x769b, 0x77c2, 0x78ea, 0x7a14,
+	0x7b40, 0x7c6d, 0x7d9c, 0x7ecd, 0x3f65, 0x3f8c, 0x3fb2, 0x3fd8
+};
+
+/*
+ * Degamma table for 2020 color space data.
+ */
+static u16 DEGAMMA_2020[DEGAMMA_SIZE] = {
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0001,
+	0x0001, 0x0001, 0x0001, 0x0001, 0x0001, 0x0001, 0x0002, 0x0002, 0x0002,
+	0x0002, 0x0002, 0x0003, 0x0003, 0x0003, 0x0003, 0x0004, 0x0004, 0x0004,
+	0x0005, 0x0005, 0x0006, 0x0006, 0x0006, 0x0007, 0x0007, 0x0008, 0x0008,
+	0x0009, 0x000a, 0x000a, 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f,
+	0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0016, 0x0017, 0x0018,
+	0x0019, 0x001b, 0x001c, 0x001e, 0x001f, 0x0021, 0x0022, 0x0024, 0x0026,
+	0x0028, 0x002a, 0x002c, 0x002e, 0x0030, 0x0033, 0x0035, 0x0038, 0x003a,
+	0x003d, 0x0040, 0x0043, 0x0046, 0x0049, 0x004d, 0x0050, 0x0054, 0x0057,
+	0x005b, 0x005f, 0x0064, 0x0068, 0x006d, 0x0071, 0x0076, 0x007c, 0x0081,
+	0x0086, 0x008c, 0x0092, 0x0098, 0x009f, 0x00a5, 0x00ac, 0x00b4, 0x00bb,
+	0x00c3, 0x00cb, 0x00d3, 0x00dc, 0x00e5, 0x00ee, 0x00f8, 0x0102, 0x010c,
+	0x0117, 0x0123, 0x012e, 0x013a, 0x0147, 0x0154, 0x0161, 0x016f, 0x017e,
+	0x018d, 0x019c, 0x01ac, 0x01bd, 0x01ce, 0x01e0, 0x01f3, 0x0206, 0x021a,
+	0x022f, 0x0244, 0x025a, 0x0272, 0x0289, 0x02a2, 0x02bc, 0x02d6, 0x02f2,
+	0x030f, 0x032c, 0x034b, 0x036b, 0x038b, 0x03ae, 0x03d1, 0x03f5, 0x041b,
+	0x0443, 0x046b, 0x0495, 0x04c1, 0x04ee, 0x051d, 0x054e, 0x0580, 0x05b4,
+	0x05ea, 0x0622, 0x065c, 0x0698, 0x06d6, 0x0717, 0x075a, 0x079f, 0x07e7,
+	0x0831, 0x087e, 0x08cd, 0x0920, 0x0976, 0x09ce, 0x0a2a, 0x0a89, 0x0aec,
+	0x0b52, 0x0bbc, 0x0c2a, 0x0c9b, 0x0d11, 0x0d8b, 0x0e0a, 0x0e8d, 0x0f15,
+	0x0fa1, 0x1033, 0x10ca, 0x1167, 0x120a, 0x12b2, 0x1360, 0x1415, 0x14d1,
+	0x1593, 0x165d, 0x172e, 0x1806, 0x18e7, 0x19d0, 0x1ac1, 0x1bbb, 0x1cbf,
+	0x1dcc, 0x1ee3, 0x2005, 0x2131, 0x2268, 0x23ab, 0x24fa, 0x2656, 0x27be,
+	0x2934, 0x2ab8, 0x2c4a, 0x2dec, 0x2f9d, 0x315f, 0x3332, 0x3516, 0x370d,
+	0x3916, 0x3b34, 0x3d66, 0x3fad, 0x420b, 0x4480, 0x470d, 0x49b3, 0x4c73,
+	0x4f4e, 0x5246, 0x555a, 0x588e, 0x5be1, 0x5f55, 0x62eb, 0x66a6, 0x6a86,
+	0x6e8c, 0x72bb, 0x7714, 0x7b99, 0x3dcb, 0x3e60, 0x3ef5, 0x3f8c
+};
+
+/* one is for primary plane and the other is for all overlay planes */
+static const struct dc_hw_plane_reg dc_plane_reg[] = {
+	{
+		.y_address = DC_FRAMEBUFFER_ADDRESS,
+		.u_address = DC_FRAMEBUFFER_U_ADDRESS,
+		.v_address = DC_FRAMEBUFFER_V_ADDRESS,
+		.y_stride = DC_FRAMEBUFFER_STRIDE,
+		.u_stride = DC_FRAMEBUFFER_U_STRIDE,
+		.v_stride = DC_FRAMEBUFFER_V_STRIDE,
+		.size = DC_FRAMEBUFFER_SIZE,
+		.scale_factor_x = DC_FRAMEBUFFER_SCALE_FACTOR_X,
+		.scale_factor_y = DC_FRAMEBUFFER_SCALE_FACTOR_Y,
+		.h_filter_coef_index = DC_FRAMEBUFFER_H_FILTER_COEF_INDEX,
+		.h_filter_coef_data = DC_FRAMEBUFFER_H_FILTER_COEF_DATA,
+		.v_filter_coef_index = DC_FRAMEBUFFER_V_FILTER_COEF_INDEX,
+		.v_filter_coef_data = DC_FRAMEBUFFER_V_FILTER_COEF_DATA,
+		.init_offset = DC_FRAMEBUFFER_INIT_OFFSET,
+		.color_key = DC_FRAMEBUFFER_COLOR_KEY,
+		.color_key_high = DC_FRAMEBUFFER_COLOR_KEY_HIGH,
+		.clear_value = DC_FRAMEBUFFER_CLEAR_VALUE,
+		.color_table_index = DC_FRAMEBUFFER_COLOR_TABLE_INDEX,
+		.color_table_data = DC_FRAMEBUFFER_COLOR_TABLE_DATA,
+		.scale_config = DC_FRAMEBUFFER_SCALE_CONFIG,
+		.degamma_index = DC_FRAMEBUFFER_DEGAMMA_INDEX,
+		.degamma_data = DC_FRAMEBUFFER_DEGAMMA_DATA,
+		.degamma_ex_data = DC_FRAMEBUFFER_DEGAMMA_EX_DATA,
+		.roi_origin = DC_FRAMEBUFFER_ROI_ORIGIN,
+		.roi_size = DC_FRAMEBUFFER_ROI_SIZE,
+		.YUVToRGBCoef0 = DC_FRAMEBUFFER_YUVTORGB_COEF0,
+		.YUVToRGBCoef1 = DC_FRAMEBUFFER_YUVTORGB_COEF1,
+		.YUVToRGBCoef2 = DC_FRAMEBUFFER_YUVTORGB_COEF2,
+		.YUVToRGBCoef3 = DC_FRAMEBUFFER_YUVTORGB_COEF3,
+		.YUVToRGBCoef4 = DC_FRAMEBUFFER_YUVTORGB_COEF4,
+		.YUVToRGBCoefD0 = DC_FRAMEBUFFER_YUVTORGB_COEFD0,
+		.YUVToRGBCoefD1 = DC_FRAMEBUFFER_YUVTORGB_COEFD1,
+		.YUVToRGBCoefD2 = DC_FRAMEBUFFER_YUVTORGB_COEFD2,
+		.YClampBound = DC_FRAMEBUFFER_Y_CLAMP_BOUND,
+		.UVClampBound = DC_FRAMEBUFFER_UV_CLAMP_BOUND,
+		.RGBToRGBCoef0 = DC_FRAMEBUFFER_RGBTORGB_COEF0,
+		.RGBToRGBCoef1 = DC_FRAMEBUFFER_RGBTORGB_COEF1,
+		.RGBToRGBCoef2 = DC_FRAMEBUFFER_RGBTORGB_COEF2,
+		.RGBToRGBCoef3 = DC_FRAMEBUFFER_RGBTORGB_COEF3,
+		.RGBToRGBCoef4 = DC_FRAMEBUFFER_RGBTORGB_COEF4,
+	},
+	{
+		.y_address = DC_OVERLAY_ADDRESS,
+		.u_address = DC_OVERLAY_U_ADDRESS,
+		.v_address = DC_OVERLAY_V_ADDRESS,
+		.y_stride = DC_OVERLAY_STRIDE,
+		.u_stride = DC_OVERLAY_U_STRIDE,
+		.v_stride = DC_OVERLAY_V_STRIDE,
+		.size = DC_OVERLAY_SIZE,
+		.scale_factor_x = DC_OVERLAY_SCALE_FACTOR_X,
+		.scale_factor_y = DC_OVERLAY_SCALE_FACTOR_Y,
+		.h_filter_coef_index = DC_OVERLAY_H_FILTER_COEF_INDEX,
+		.h_filter_coef_data = DC_OVERLAY_H_FILTER_COEF_DATA,
+		.v_filter_coef_index = DC_OVERLAY_V_FILTER_COEF_INDEX,
+		.v_filter_coef_data = DC_OVERLAY_V_FILTER_COEF_DATA,
+		.init_offset = DC_OVERLAY_INIT_OFFSET,
+		.color_key = DC_OVERLAY_COLOR_KEY,
+		.color_key_high = DC_OVERLAY_COLOR_KEY_HIGH,
+		.clear_value = DC_OVERLAY_CLEAR_VALUE,
+		.color_table_index = DC_OVERLAY_COLOR_TABLE_INDEX,
+		.color_table_data = DC_OVERLAY_COLOR_TABLE_DATA,
+		.scale_config = DC_OVERLAY_SCALE_CONFIG,
+		.degamma_index = DC_OVERLAY_DEGAMMA_INDEX,
+		.degamma_data = DC_OVERLAY_DEGAMMA_DATA,
+		.degamma_ex_data = DC_OVERLAY_DEGAMMA_EX_DATA,
+		.roi_origin = DC_OVERLAY_ROI_ORIGIN,
+		.roi_size = DC_OVERLAY_ROI_SIZE,
+		.YUVToRGBCoef0 = DC_OVERLAY_YUVTORGB_COEF0,
+		.YUVToRGBCoef1 = DC_OVERLAY_YUVTORGB_COEF1,
+		.YUVToRGBCoef2 = DC_OVERLAY_YUVTORGB_COEF2,
+		.YUVToRGBCoef3 = DC_OVERLAY_YUVTORGB_COEF3,
+		.YUVToRGBCoef4 = DC_OVERLAY_YUVTORGB_COEF4,
+		.YUVToRGBCoefD0 = DC_OVERLAY_YUVTORGB_COEFD0,
+		.YUVToRGBCoefD1 = DC_OVERLAY_YUVTORGB_COEFD1,
+		.YUVToRGBCoefD2 = DC_OVERLAY_YUVTORGB_COEFD2,
+		.YClampBound = DC_OVERLAY_Y_CLAMP_BOUND,
+		.UVClampBound = DC_OVERLAY_UV_CLAMP_BOUND,
+		.RGBToRGBCoef0 = DC_OVERLAY_RGBTORGB_COEF0,
+		.RGBToRGBCoef1 = DC_OVERLAY_RGBTORGB_COEF1,
+		.RGBToRGBCoef2 = DC_OVERLAY_RGBTORGB_COEF2,
+		.RGBToRGBCoef3 = DC_OVERLAY_RGBTORGB_COEF3,
+		.RGBToRGBCoef4 = DC_OVERLAY_RGBTORGB_COEF4,
+	},
+};
+
+static const u32 read_reg[] = {
+	DC_FRAMEBUFFER_CONFIG,
+	DC_OVERLAY_CONFIG,
+	DC_OVERLAY_SCALE_CONFIG,
+};
+#define READ_REG_COUNT ARRAY_SIZE(read_reg)
+
+static const u32 primary_overlay_formats[] = {
+	DRM_FORMAT_XRGB4444, DRM_FORMAT_ARGB4444,    DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555, DRM_FORMAT_RGB565,	     DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_ARGB2101010, DRM_FORMAT_RGBA4444,
+	DRM_FORMAT_RGBA5551, DRM_FORMAT_RGBA8888,    DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_ABGR4444, DRM_FORMAT_ABGR1555,    DRM_FORMAT_BGR565,
+	DRM_FORMAT_ABGR8888, DRM_FORMAT_ABGR2101010, DRM_FORMAT_BGRA4444,
+	DRM_FORMAT_BGRA5551, DRM_FORMAT_BGRA8888,    DRM_FORMAT_BGRA1010102,
+	DRM_FORMAT_XBGR4444, DRM_FORMAT_RGBX4444,    DRM_FORMAT_BGRX4444,
+	DRM_FORMAT_XBGR1555, DRM_FORMAT_RGBX5551,    DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_XBGR8888, DRM_FORMAT_RGBX8888,    DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_YUYV,     DRM_FORMAT_YVYU,	     DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,     DRM_FORMAT_NV12,	     DRM_FORMAT_NV21,
+	DRM_FORMAT_NV16,     DRM_FORMAT_NV61,	     DRM_FORMAT_P010,
+	/* TODO. */
+};
+
+static const u32 cursor_formats[] = { DRM_FORMAT_ARGB8888 };
+
+static const u64 cursor_dumy_modifiers[] = { DRM_FORMAT_MOD_LINEAR,
+					     DRM_FORMAT_MOD_INVALID };
+
+static const u64 format_modifiers[] = { DRM_FORMAT_MOD_LINEAR,
+					DRM_FORMAT_MOD_INVALID };
+
+#define FRAC_16_16(mult, div) (((mult) << 16) / (div))
+
+static const struct es_plane_info dc_hw_planes[][PLANE_NUM] = {
+	{
+		/* DC_REV_5551 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = NULL,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 32,
+			.max_height = 32,
+			.rotation = DRM_MODE_ROTATE_0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+	},
+	{
+		/* DC_REV_5551_306 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+				    DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
+				    DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = NULL,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 32,
+			.max_height = 32,
+			.rotation = 0,
+			.color_encoding = 0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+				    DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
+				    DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+	},
+	{
+		/* DC_REV_5701_303 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = NULL,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 256,
+			.max_height = 256,
+			.rotation = DRM_MODE_ROTATE_0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+	},
+	{
+		/* DC_REV_5701_309 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096 * 2, //support dual 4K extended mode
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 32),
+			.max_scale = FRAC_16_16(2, 1),
+			.roi = true,
+			.color_mgmt = true,
+			.background = true,
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = cursor_dumy_modifiers,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 256,
+			.max_height = 256,
+			.rotation = DRM_MODE_ROTATE_0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 32),
+			.max_scale = FRAC_16_16(2, 1),
+			.roi = true,
+			.color_mgmt = true,
+		},
+	},
+};
+
+static const struct es_dc_info dc_info[] = {
+	{
+		/* DC_REV_5551 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5551],
+		.max_bpc = 10,
+		.color_formats = DRM_COLOR_FORMAT_RGB444,
+		.gamma_size = GAMMA_SIZE,
+		.gamma_bits = 10,
+		.pitch_alignment = 128,
+		.pipe_sync = false,
+		.mmu_prefetch = false,
+	},
+	{
+		/* DC_REV_5551_306 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5551_306],
+		.max_bpc = 10,
+		.color_formats = DRM_COLOR_FORMAT_RGB444,
+		.gamma_size = GAMMA_SIZE,
+		.gamma_bits = 10,
+		.pitch_alignment = 128,
+		.pipe_sync = false,
+		.mmu_prefetch = false,
+	},
+	{
+		/* DC_REV_5701_303 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5701_303],
+		.max_bpc = 10,
+		.color_formats =
+			DRM_COLOR_FORMAT_RGB444 | DRM_COLOR_FORMAT_YCBCR444 |
+			DRM_COLOR_FORMAT_YCBCR422 | DRM_COLOR_FORMAT_YCBCR420,
+		.gamma_size = GAMMA_EX_SIZE,
+		.gamma_bits = 12,
+		.pitch_alignment = 128,
+		.pipe_sync = false,
+		.mmu_prefetch = false,
+	},
+	{
+		/* DC_REV_5701_309 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5701_309],
+		.max_bpc = 10,
+		.color_formats =
+			DRM_COLOR_FORMAT_RGB444 | DRM_COLOR_FORMAT_YCBCR444 |
+			DRM_COLOR_FORMAT_YCBCR422 | DRM_COLOR_FORMAT_YCBCR420,
+		.gamma_size = GAMMA_EX_SIZE,
+		.gamma_bits = 12,
+		.pitch_alignment = 128,
+		.pipe_sync = true,
+		.mmu_prefetch = true,
+		.background = true,
+	},
+};
+
+static const struct dc_hw_funcs hw_func[];
+
+static inline u32 hi_write(struct dc_hw *hw, u32 reg, u32 val)
+{
+	writel(val, hw->hi_base + reg);
+
+	return 0;
+}
+
+static inline u32 hi_read(struct dc_hw *hw, u32 reg)
+{
+	u32 value;
+
+	value = readl(hw->hi_base + reg);
+	return value;
+}
+
+static inline int dc_get_read_index(struct dc_hw *hw, u32 reg)
+{
+	int i;
+
+	for (i = 0; i < READ_REG_COUNT; i++) {
+		if (hw->read_block[i].reg == reg)
+			return i;
+	}
+	return i;
+}
+
+static inline void dc_write(struct dc_hw *hw, u32 reg, u32 value)
+{
+	writel(value, hw->reg_base + reg - DC_REG_BASE);
+
+	if (hw->read_block) {
+		int i = dc_get_read_index(hw, reg);
+
+		if (i < READ_REG_COUNT)
+			hw->read_block[i].value = value;
+	}
+}
+
+static inline u32 dc_read(struct dc_hw *hw, u32 reg)
+{
+	u32 value = readl(hw->reg_base + reg - DC_REG_BASE);
+
+	if (hw->read_block) {
+		int i = dc_get_read_index(hw, reg);
+
+		if (i < READ_REG_COUNT) {
+			value &= ~BIT(0);
+			value |= hw->read_block[i].value & BIT(0);
+		}
+	}
+
+	return value;
+}
+
+static inline void dc_set_clear(struct dc_hw *hw, u32 reg, u32 set, u32 clear)
+{
+	u32 value = dc_read(hw, reg);
+
+	value &= ~clear;
+	value |= set;
+	dc_write(hw, reg, value);
+}
+
+static void load_default_filter(struct dc_hw *hw,
+				const struct dc_hw_plane_reg *reg)
+{
+	u8 i;
+
+	dc_write(hw, reg->scale_config, 0x33);
+	dc_write(hw, reg->init_offset, 0x80008000);
+	dc_write(hw, reg->h_filter_coef_index, 0x00);
+	for (i = 0; i < H_COEF_SIZE; i++)
+		dc_write(hw, reg->h_filter_coef_data, horKernel[i]);
+
+	dc_write(hw, reg->v_filter_coef_index, 0x00);
+	for (i = 0; i < V_COEF_SIZE; i++)
+		dc_write(hw, reg->v_filter_coef_data, verKernel[i]);
+}
+
+static void load_rgb_to_rgb(struct dc_hw *hw, const struct dc_hw_plane_reg *reg,
+			    u16 *table)
+{
+	dc_write(hw, reg->RGBToRGBCoef0, table[0] | (table[1] << 16));
+	dc_write(hw, reg->RGBToRGBCoef1, table[2] | (table[3] << 16));
+	dc_write(hw, reg->RGBToRGBCoef2, table[4] | (table[5] << 16));
+	dc_write(hw, reg->RGBToRGBCoef3, table[6] | (table[7] << 16));
+	dc_write(hw, reg->RGBToRGBCoef4, table[8]);
+}
+
+static void load_yuv_to_rgb(struct dc_hw *hw, const struct dc_hw_plane_reg *reg,
+			    s32 *table)
+{
+	dc_write(hw, reg->YUVToRGBCoef0,
+		 (0xFFFF & table[0]) | (table[1] << 16));
+	dc_write(hw, reg->YUVToRGBCoef1,
+		 (0xFFFF & table[2]) | (table[3] << 16));
+	dc_write(hw, reg->YUVToRGBCoef2,
+		 (0xFFFF & table[4]) | (table[5] << 16));
+	dc_write(hw, reg->YUVToRGBCoef3,
+		 (0xFFFF & table[6]) | (table[7] << 16));
+	dc_write(hw, reg->YUVToRGBCoef4, table[8]);
+	dc_write(hw, reg->YUVToRGBCoefD0, table[9]);
+	dc_write(hw, reg->YUVToRGBCoefD1, table[10]);
+	dc_write(hw, reg->YUVToRGBCoefD2, table[11]);
+	dc_write(hw, reg->YClampBound, table[12] | (table[13] << 16));
+	dc_write(hw, reg->UVClampBound, table[14] | (table[15] << 16));
+}
+
+static void load_rgb_to_yuv(struct dc_hw *hw, s16 *table)
+{
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF0, table[0] | (table[1] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF1, table[2] | (table[3] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF2, table[4] | (table[5] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF3, table[6] | (table[7] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF4, table[8]);
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEFD0, table[9]);
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEFD1, table[10]);
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEFD2, table[11]);
+}
+
+static bool is_rgb(enum dc_hw_color_format format)
+{
+	switch (format) {
+	case FORMAT_X4R4G4B4:
+	case FORMAT_A4R4G4B4:
+	case FORMAT_X1R5G5B5:
+	case FORMAT_A1R5G5B5:
+	case FORMAT_R5G6B5:
+	case FORMAT_X8R8G8B8:
+	case FORMAT_A8R8G8B8:
+	case FORMAT_A2R10G10B10:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static void load_degamma_table(struct dc_hw *hw,
+			       const struct dc_hw_plane_reg *reg, u16 *table)
+{
+	u16 i;
+	u32 value;
+
+	for (i = 0; i < DEGAMMA_SIZE; i++) {
+		dc_write(hw, reg->degamma_index, i);
+		value = table[i] | (table[i] << 16);
+		dc_write(hw, reg->degamma_data, value);
+		dc_write(hw, reg->degamma_ex_data, table[i]);
+	}
+}
+
+int dc_hw_init(struct dc_hw *hw)
+{
+	u8 i;
+	u32 revision = hi_read(hw, DC_HW_REVISION);
+	u32 cid = hi_read(hw, DC_HW_CHIP_CID);
+
+	switch (revision) {
+	case 0x5551:
+		if (cid < 0x306)
+			hw->rev = DC_REV_5551;
+		else
+			hw->rev = DC_REV_5551_306;
+		break;
+	case 0x5701:
+		if (cid < 0x309)
+			hw->rev = DC_REV_5701_303;
+		else
+			hw->rev = DC_REV_5701_309;
+		break;
+	default:
+		return -ENXIO;
+	}
+
+	if (hw->rev == DC_REV_5551) {
+		hw->read_block = kzalloc(
+			sizeof(struct dc_hw_read) * READ_REG_COUNT, GFP_KERNEL);
+		if (!hw->read_block)
+			return -ENOMEM;
+		for (i = 0; i < READ_REG_COUNT; i++)
+			hw->read_block[i].reg = read_reg[i];
+	}
+
+	/*
+     * Do soft reset before configuring DC
+     * Sleep 50ms to ensure no data transfer on AXI bus,
+     * because dc may be initialized in uboot
+     */
+	{
+		u32 val;
+
+		val = dc_read(hw, DC_FRAMEBUFFER_CONFIG);
+		val &= ~BIT(0);
+		val |= BIT(3);
+		dc_write(hw, DC_FRAMEBUFFER_CONFIG, val);
+		mdelay(50);
+		hi_write(hw, 0x0, 0x1000);
+		mdelay(10);
+	}
+
+	hw->info = (struct es_dc_info *)&dc_info[hw->rev];
+	hw->func = (struct dc_hw_funcs *)&hw_func[hw->rev];
+
+	for (i = 0; i < ARRAY_SIZE(dc_plane_reg); i++) {
+		load_default_filter(hw, &dc_plane_reg[i]);
+		if ((hw->rev == DC_REV_5701_303) ||
+		    (hw->rev == DC_REV_5701_309))
+			load_rgb_to_rgb(hw, &dc_plane_reg[i], RGB2RGB);
+	}
+
+	if (hw->rev == DC_REV_5701_303)
+		load_rgb_to_yuv(hw, RGB2YUV);
+
+	dc_write(hw, DC_CURSOR_BACKGROUND, 0x00FFFFFF);
+	dc_write(hw, DC_CURSOR_FOREGROUND, 0x00AAAAAA);
+	dc_write(hw, DC_DISPLAY_PANEL_CONFIG, 0x111);
+
+	return 0;
+}
+
+void dc_hw_deinit(struct dc_hw *hw)
+{
+	if (hw->read_block)
+		kfree(hw->read_block);
+}
+
+void dc_hw_update_plane(struct dc_hw *hw, enum dc_hw_plane_id id,
+			struct dc_hw_fb *fb, struct dc_hw_scale *scale)
+{
+	struct dc_hw_plane *plane = NULL;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		if (fb) {
+			if (fb->enable == false)
+				plane->fb.enable = false;
+			else
+				memcpy(&plane->fb, fb,
+				       sizeof(*fb) - sizeof(fb->dirty));
+			plane->fb.dirty = true;
+		}
+		if (scale) {
+			memcpy(&plane->scale, scale,
+			       sizeof(*scale) - sizeof(scale->dirty));
+			plane->scale.dirty = true;
+		}
+	}
+}
+
+void dc_hw_update_degamma(struct dc_hw *hw, enum dc_hw_plane_id id, u32 mode)
+{
+	struct dc_hw_plane *plane = NULL;
+	int i = (int)id;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		if (hw->info->planes[i].degamma_size) {
+			plane->degamma.mode = mode;
+			plane->degamma.dirty = true;
+		} else {
+			plane->degamma.dirty = false;
+		}
+	}
+}
+
+void dc_hw_set_position(struct dc_hw *hw, struct dc_hw_position *pos)
+{
+	memcpy(&hw->overlay.pos, pos, sizeof(*pos) - sizeof(pos->dirty));
+	hw->overlay.pos.dirty = true;
+}
+
+void dc_hw_set_blend(struct dc_hw *hw, struct dc_hw_blend *blend)
+{
+	memcpy(&hw->overlay.blend, blend,
+	       sizeof(*blend) - sizeof(blend->dirty));
+	hw->overlay.blend.dirty = true;
+}
+
+void dc_hw_update_cursor(struct dc_hw *hw, struct dc_hw_cursor *cursor)
+{
+	memcpy(&hw->cursor, cursor, sizeof(*cursor) - sizeof(cursor->dirty));
+	hw->cursor.dirty = true;
+}
+
+void dc_hw_update_gamma(struct dc_hw *hw, u16 index, u16 r, u16 g, u16 b)
+{
+	if (index >= hw->info->gamma_size)
+		return;
+
+	hw->gamma.gamma[index][0] = r;
+	hw->gamma.gamma[index][1] = g;
+	hw->gamma.gamma[index][2] = b;
+	hw->gamma.dirty = true;
+}
+
+void dc_hw_enable_gamma(struct dc_hw *hw, bool enable)
+{
+	hw->gamma.enable = enable;
+	hw->gamma.dirty = true;
+}
+
+void dc_hw_enable_dump(struct dc_hw *hw, u32 addr, u32 pitch)
+{
+	dc_write(hw, 0x14F0, addr);
+	dc_write(hw, 0x14E8, addr);
+	dc_write(hw, 0x1500, pitch);
+	dc_write(hw, 0x14F8, 0x30000);
+}
+
+void dc_hw_disable_dump(struct dc_hw *hw)
+{
+	dc_write(hw, 0x14F8, 0x00);
+}
+
+void dc_hw_setup_display(struct dc_hw *hw, struct dc_hw_display *display)
+{
+	memcpy(&hw->display, display, sizeof(*display));
+
+	hw->func->display(hw, display);
+}
+
+void dc_hw_enable_interrupt(struct dc_hw *hw, bool enable)
+{
+	dc_write(hw, DC_DISPLAY_INT_ENABLE, enable);
+}
+
+u32 dc_hw_get_interrupt(struct dc_hw *hw)
+{
+	return dc_read(hw, DC_DISPLAY_INT);
+}
+
+bool dc_hw_flip_in_progress(struct dc_hw *hw)
+{
+	return dc_read(hw, DC_FRAMEBUFFER_CONFIG) & BIT(6);
+}
+
+bool dc_hw_check_underflow(struct dc_hw *hw)
+{
+	return dc_read(hw, DC_FRAMEBUFFER_CONFIG) & BIT(5);
+}
+
+void dc_hw_enable_shadow_register(struct dc_hw *hw, bool enable)
+{
+	if (enable)
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(3));
+	else
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(3), 0);
+}
+
+void dc_hw_set_out(struct dc_hw *hw, enum dc_hw_out out)
+{
+	if (out <= OUT_DP)
+		hw->out = out;
+}
+
+static void gamma_commit(struct dc_hw *hw)
+{
+	u16 i;
+	u32 value;
+
+	if (hw->gamma.dirty) {
+		if (hw->gamma.enable) {
+			dc_write(hw, DC_DISPLAY_GAMMA_INDEX, 0x00);
+			for (i = 0; i < GAMMA_SIZE; i++) {
+				value = hw->gamma.gamma[i][2] |
+					(hw->gamma.gamma[i][1] << 10) |
+					(hw->gamma.gamma[i][0] << 20);
+				dc_write(hw, DC_DISPLAY_GAMMA_DATA, value);
+			}
+
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(2), 0);
+		} else {
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(2));
+		}
+		hw->gamma.dirty = false;
+	}
+}
+
+static void gamma_ex_commit(struct dc_hw *hw)
+{
+	u16 i;
+	u32 value;
+
+	if (hw->gamma.dirty) {
+		if (hw->gamma.enable) {
+			dc_write(hw, DC_DISPLAY_GAMMA_EX_INDEX, 0x00);
+			for (i = 0; i < GAMMA_EX_SIZE; i++) {
+				value = hw->gamma.gamma[i][2] |
+					(hw->gamma.gamma[i][1] << 16);
+				dc_write(hw, DC_DISPLAY_GAMMA_EX_DATA, value);
+				dc_write(hw, DC_DISPLAY_GAMMA_EX_ONE_DATA,
+					 hw->gamma.gamma[i][0]);
+			}
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(2), 0);
+		} else {
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(2));
+		}
+		hw->gamma.dirty = false;
+	}
+}
+
+static void plane_commit(struct dc_hw *hw)
+{
+	struct dc_hw_plane *plane;
+	const struct dc_hw_plane_reg *reg;
+	u16 i;
+
+	for (i = 0; i < PLANE_NUM; i++) {
+		if (i == PRIMARY_PLANE) {
+			plane = &hw->primary;
+			reg = &dc_plane_reg[0];
+		} else if (i == OVERLAY_PLANE) {
+			plane = &hw->overlay.plane;
+			reg = &dc_plane_reg[1];
+		} else {
+			continue;
+		}
+
+		if (plane->fb.dirty) {
+			if (plane->fb.enable) {
+				// sifive_l2_flush64_range(plane->fb.y_address, plane->fb.width * plane->fb.height * 4);
+
+				dc_write(hw, reg->y_address,
+					 plane->fb.y_address);
+				dc_write(hw, reg->u_address,
+					 plane->fb.u_address);
+				dc_write(hw, reg->v_address,
+					 plane->fb.v_address);
+				dc_write(hw, reg->y_stride, plane->fb.y_stride);
+				dc_write(hw, reg->u_stride, plane->fb.u_stride);
+				dc_write(hw, reg->v_stride, plane->fb.v_stride);
+				dc_write(hw, reg->size,
+					 plane->fb.width |
+						 (plane->fb.height << 15));
+				if (plane->fb.clear_enable)
+					dc_write(hw, reg->clear_value,
+						 plane->fb.clear_value);
+			} else {
+				dc_hw_enable_shadow_register(hw, true);
+				if (i == PRIMARY_PLANE) {
+					u32 reg_clr = BIT(24), reg_set = 0;
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     reg_set, reg_clr);
+
+					reg_clr = (0x1F << 26),
+					reg_set = BIT(8) | (05 << 26);
+					dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+						     reg_set, reg_clr);
+				}
+				mdelay(50);
+				dc_hw_enable_shadow_register(hw, false);
+			}
+
+			if (i == PRIMARY_PLANE) {
+				u32 reg_clr = 0, reg_set = 0;
+
+				reg_clr = (0x1F << 26) | BIT(25) |
+					  (0x03 << 23) | (0x1F << 17) |
+					  (0x07 << 14) | (0x07 << 11) | BIT(0);
+				reg_set = (plane->fb.format << 26) |
+					  (plane->fb.uv_swizzle << 25) |
+					  (plane->fb.swizzle << 23) |
+					  (plane->fb.tile_mode << 17) |
+					  (plane->fb.yuv_color_space << 14) |
+					  (plane->fb.rotation << 11) |
+					  (plane->fb.clear_enable << 8) |
+					  plane->fb.enable;
+				if (plane->fb.enable) {
+					reg_clr |= BIT(8);
+					reg_set |= BIT(4);
+				}
+
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, reg_set,
+					     reg_clr);
+			} else
+				dc_set_clear(
+					hw, DC_OVERLAY_CONFIG,
+					(plane->fb.format << 16) |
+						(plane->fb.uv_swizzle << 15) |
+						(plane->fb.swizzle << 13) |
+						(plane->fb.tile_mode << 8) |
+						(plane->fb.yuv_color_space
+						 << 5) |
+						(plane->fb.rotation << 2) |
+						(plane->fb.enable << 24) |
+						(plane->fb.clear_enable << 25),
+					(0x1F << 16) | BIT(15) | (0x03 << 13) |
+						(0x1F << 8) | (0x07 << 5) |
+						(0x07 << 2) | BIT(24));
+			plane->fb.dirty = false;
+		}
+
+		if (plane->scale.dirty) {
+			if (plane->scale.enable) {
+				dc_write(hw, reg->scale_factor_x,
+					 plane->scale.scale_factor_x);
+				dc_write(hw, reg->scale_factor_y,
+					 plane->scale.scale_factor_y);
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+						     BIT(22), 0);
+				else
+					dc_set_clear(hw,
+						     DC_OVERLAY_SCALE_CONFIG,
+						     BIT(8), 0);
+
+			} else {
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+						     0, BIT(22));
+				else
+					dc_set_clear(hw,
+						     DC_OVERLAY_SCALE_CONFIG, 0,
+						     BIT(8));
+			}
+			plane->scale.dirty = false;
+		}
+
+		if (plane->colorkey.dirty) {
+			dc_write(hw, reg->color_key, plane->colorkey.colorkey);
+			dc_write(hw, reg->color_key_high,
+				 plane->colorkey.colorkey_high);
+
+			if (i == PRIMARY_PLANE)
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+					     plane->colorkey.transparency << 9,
+					     0x03 << 9);
+			else
+				dc_set_clear(hw, DC_OVERLAY_CONFIG,
+					     plane->colorkey.transparency,
+					     0x03);
+
+			plane->colorkey.dirty = false;
+		}
+
+		if (plane->roi.dirty) {
+			if (plane->roi.enable) {
+				dc_write(hw, reg->roi_origin,
+					 plane->roi.x | (plane->roi.y << 16));
+				dc_write(hw, reg->roi_size,
+					 plane->roi.width |
+						 (plane->roi.height << 16));
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(0), 0);
+			} else {
+				// if (i == PRIMARY_PLANE)
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, 0,
+					     BIT(0));
+			}
+			plane->roi.dirty = false;
+		}
+	}
+}
+
+static void plane_ex_commit(struct dc_hw *hw)
+{
+	struct dc_hw_plane *plane;
+	const struct dc_hw_plane_reg *reg;
+	u16 i;
+
+	for (i = 0; i < PLANE_NUM; i++) {
+		if (i == PRIMARY_PLANE) {
+			plane = &hw->primary;
+			reg = &dc_plane_reg[0];
+		} else if (i == OVERLAY_PLANE) {
+			plane = &hw->overlay.plane;
+			reg = &dc_plane_reg[1];
+		} else {
+			continue;
+		}
+
+		if (plane->fb.dirty) {
+			if (is_rgb(plane->fb.format)) {
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     0, BIT(8));
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     0, BIT(30));
+			} else {
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(8), BIT(6));
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     BIT(30), BIT(29));
+				switch (plane->fb.yuv_color_space) {
+				case COLOR_SPACE_601:
+					load_yuv_to_rgb(hw, reg, YUV601_2RGB);
+					break;
+				case COLOR_SPACE_709:
+					load_yuv_to_rgb(hw, reg, YUV709_2RGB);
+					break;
+				case COLOR_SPACE_2020:
+					load_yuv_to_rgb(hw, reg, YUV2020_2RGB);
+					break;
+				default:
+					break;
+				}
+			}
+		}
+		if (plane->degamma.dirty) {
+			switch (plane->degamma.mode) {
+			case ES_DEGAMMA_DISABLE:
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     0, BIT(5));
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG, 0,
+						     BIT(28));
+				break;
+			case ES_DEGAMMA_BT709:
+				load_degamma_table(hw, reg, DEGAMMA_709);
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(5), 0);
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     BIT(28), 0);
+				break;
+			case ES_DEGAMMA_BT2020:
+				load_degamma_table(hw, reg, DEGAMMA_2020);
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(5), 0);
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     BIT(28), 0);
+				break;
+			default:
+				break;
+			}
+			plane->degamma.dirty = false;
+		}
+	}
+	plane_commit(hw);
+}
+
+static void setup_display(struct dc_hw *hw, struct dc_hw_display *display)
+{
+	u32 dpi_cfg;
+
+	if (hw->display.enable) {
+		switch (display->bus_format) {
+		case MEDIA_BUS_FMT_RGB565_1X16:
+			dpi_cfg = 0;
+			break;
+		case MEDIA_BUS_FMT_RGB666_1X18:
+			dpi_cfg = 3;
+			break;
+		case MEDIA_BUS_FMT_RGB666_1X24_CPADHI:
+			dpi_cfg = 4;
+			break;
+		case MEDIA_BUS_FMT_RGB888_1X24:
+			dpi_cfg = 5;
+			break;
+		case MEDIA_BUS_FMT_RGB101010_1X30:
+			dpi_cfg = 6;
+			break;
+		default:
+			dpi_cfg = 5;
+			break;
+		}
+		dc_write(hw, DC_DISPLAY_DPI_CONFIG, dpi_cfg);
+
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(4));
+
+		dc_write(hw, DC_DISPLAY_H,
+			 hw->display.h_active | (hw->display.h_total << 16));
+		dc_write(hw, DC_DISPLAY_H_SYNC,
+			 hw->display.h_sync_start |
+				 (hw->display.h_sync_end << 15) |
+				 (hw->display.h_sync_polarity ? 0 : BIT(31)) |
+				 BIT(30));
+		dc_write(hw, DC_DISPLAY_V,
+			 hw->display.v_active | (hw->display.v_total << 16));
+		dc_write(hw, DC_DISPLAY_V_SYNC,
+			 hw->display.v_sync_start |
+				 (hw->display.v_sync_end << 15) |
+				 (hw->display.v_sync_polarity ? 0 : BIT(31)) |
+				 BIT(30));
+
+		if (hw->info->pipe_sync) {
+			switch (display->sync_mode) {
+			case ES_SINGLE_DC:
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, 0,
+					     BIT(3) | BIT(4));
+				break;
+			case ES_MULTI_DC_PRIMARY:
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX,
+					     BIT(3) | BIT(4), 0);
+				break;
+			case ES_MULTI_DC_SECONDARY:
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX,
+					     BIT(3), BIT(4));
+				break;
+			default:
+				break;
+			}
+		}
+
+		if (hw->display.dither_enable) {
+			dc_write(hw, DC_DISPLAY_DITHER_TABLE_LOW,
+				 DC_DISPLAY_DITHERTABLE_LOW);
+			dc_write(hw, DC_DISPLAY_DITHER_TABLE_HIGH,
+				 DC_DISPLAY_DITHERTABLE_HIGH);
+			dc_write(hw, DC_DISPLAY_DITHER_CONFIG, BIT(31));
+		} else {
+			dc_write(hw, DC_DISPLAY_DITHER_CONFIG, 0);
+		}
+
+		if (hw->info->background)
+			dc_write(hw, DC_FRAMEBUFFER_BG_COLOR,
+				 hw->display.bg_color);
+
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(4), 0);
+	} else {
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(4) | BIT(0));
+	}
+}
+
+static void setup_display_ex(struct dc_hw *hw, struct dc_hw_display *display)
+{
+	u32 dp_cfg;
+	bool is_yuv = false;
+
+	if (hw->display.enable && hw->out == OUT_DP) {
+		switch (display->bus_format) {
+		case MEDIA_BUS_FMT_RGB565_1X16:
+			dp_cfg = 0;
+			break;
+		case MEDIA_BUS_FMT_RGB666_1X18:
+			dp_cfg = 1;
+			break;
+		case MEDIA_BUS_FMT_RGB888_1X24:
+			dp_cfg = 2;
+			break;
+		case MEDIA_BUS_FMT_RGB101010_1X30:
+			dp_cfg = 3;
+			break;
+		case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+			dp_cfg = 0;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_UYVY8_1X16:
+			dp_cfg = 2 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_YUV8_1X24:
+			dp_cfg = 4 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+			dp_cfg = 6 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_UYVY10_1X20:
+			dp_cfg = 8 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_YUV10_1X30:
+			dp_cfg = 10 << 4;
+			is_yuv = true;
+			break;
+		default:
+			dp_cfg = 2;
+			break;
+		}
+		if (is_yuv)
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, BIT(7), 0);
+		else
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, 0, BIT(7));
+		dc_write(hw, DC_DISPLAY_DP_CONFIG, dp_cfg | BIT(3));
+	}
+
+	if (hw->out == OUT_DPI)
+		dc_set_clear(hw, DC_DISPLAY_DP_CONFIG, 0, BIT(3));
+
+	setup_display(hw, display);
+}
+
+static const struct dc_hw_funcs hw_func[] = {
+	{
+		.gamma = &gamma_commit,
+		.plane = &plane_commit,
+		.display = setup_display,
+	},
+	{
+		.gamma = &gamma_commit,
+		.plane = &plane_commit,
+		.display = setup_display,
+	},
+	{
+		.gamma = &gamma_ex_commit,
+		.plane = &plane_ex_commit,
+		.display = setup_display_ex,
+	},
+	{
+		.gamma = &gamma_ex_commit,
+		.plane = &plane_ex_commit,
+		.display = setup_display_ex,
+	},
+};
+
+void dc_hw_commit(struct dc_hw *hw)
+{
+	hw->func->gamma(hw);
+	hw->func->plane(hw);
+
+	if (hw->overlay.pos.dirty) {
+		dc_write(hw, DC_OVERLAY_TOP_LEFT,
+			 hw->overlay.pos.start_x |
+				 (hw->overlay.pos.start_y << 15));
+		dc_write(hw, DC_OVERLAY_BOTTOM_RIGHT,
+			 hw->overlay.pos.end_x | (hw->overlay.pos.end_y << 15));
+		hw->overlay.pos.dirty = false;
+	}
+
+	if (hw->overlay.blend.dirty) {
+		dc_write(hw, DC_OVERLAY_SRC_GLOBAL_COLOR,
+			 hw->overlay.blend.alpha << 24);
+		dc_write(hw, DC_OVERLAY_DST_GLOBAL_COLOR,
+			 hw->overlay.blend.alpha << 24);
+		switch (hw->overlay.blend.blend_mode) {
+		case BLEND_PREMULTI:
+			dc_write(hw, DC_OVERLAY_BLEND_CONFIG, 0x3450);
+			break;
+		case BLEND_COVERAGE:
+			dc_write(hw, DC_OVERLAY_BLEND_CONFIG, 0x3950);
+			break;
+		case BLEND_PIXEL_NONE:
+			dc_write(hw, DC_OVERLAY_BLEND_CONFIG, 0x3548);
+			break;
+		default:
+			break;
+		}
+		hw->overlay.blend.dirty = false;
+	}
+
+	if (hw->cursor.dirty) {
+		if (hw->cursor.enable) {
+			dc_write(hw, DC_CURSOR_ADDRESS, hw->cursor.address);
+			dc_write(hw, DC_CURSOR_LOCATION,
+				 hw->cursor.x | (hw->cursor.y << 16));
+			dc_write(hw, DC_CURSOR_CONFIG,
+				  (hw->cursor.hot_x << 16) |
+				  (hw->cursor.hot_y << 8) |
+				  (hw->cursor.size << 5) | 0x06);
+		} else {
+			dc_write(hw, DC_CURSOR_CONFIG, 0x00);
+		}
+		hw->cursor.dirty = false;
+	}
+}
+
+#ifdef CONFIG_ESWIN_MMU
+static u32 mmu_read(struct dc_hw *hw, u32 reg)
+{
+	return readl(hw->mmu_base + reg - MMU_REG_BASE);
+}
+
+static void mmu_write(struct dc_hw *hw, u32 reg, u32 value)
+{
+	writel(value, hw->mmu_base + reg - MMU_REG_BASE);
+}
+
+static void mmu_set_clear(struct dc_hw *hw, u32 reg, u32 set, u32 clear)
+{
+	u32 value = mmu_read(hw, reg);
+
+	value &= ~clear;
+	value |= set;
+	mmu_write(hw, reg, value);
+}
+
+int dc_hw_mmu_init(struct dc_hw *hw, dc_mmu_pt mmu)
+{
+	u32 mtlb = 0, ext_mtlb = 0;
+	u32 safe_addr = 0, ext_safe_addr = 0;
+	u32 config = 0;
+
+	mtlb = (u32)(mmu->mtlb_physical & 0xFFFFFFFF);
+	ext_mtlb = (u32)(mmu->mtlb_physical >> 32);
+
+	/* more than 40bit physical address */
+	if (ext_mtlb & 0xFFFFFF00) {
+		pr_err("Mtlb address out of range.\n");
+		return -EFAULT;
+	}
+
+	config = (ext_mtlb << 20) | (mtlb >> 12);
+	if (mmu->mode == MMU_MODE_1K)
+		mmu_set_clear(hw, MMU_REG_CONTEXT_PD, (config << 4) | BIT(0),
+			      (0xFFFFFFF << 4) | (0x03));
+	else
+		mmu_set_clear(hw, MMU_REG_CONTEXT_PD, (config << 4),
+			      (0xFFFFFFF << 4) | (0x03));
+
+	safe_addr = (u32)(mmu->safe_page_physical & 0xFFFFFFFF);
+	ext_safe_addr = (u32)(mmu->safe_page_physical >> 32);
+
+	if ((safe_addr & 0x3F) || (ext_safe_addr & 0xFFFFFF00)) {
+		pr_err("Invalid safe_address.\n");
+		return -EFAULT;
+	}
+
+	mmu_write(hw, MMU_REG_TABLE_ARRAY_SIZE, 1);
+	mmu_write(hw, MMU_REG_SAFE_SECURE, safe_addr);
+	mmu_write(hw, MMU_REG_SAFE_NON_SECURE, safe_addr);
+
+	mmu_set_clear(hw, MMU_REG_SAFE_EXT_ADDRESS,
+		      (ext_safe_addr << 16) | ext_safe_addr,
+		      BIT(31) | (0xFF << 16) | BIT(15) | 0xFF);
+
+	mmu_write(hw, MMU_REG_CONTROL, BIT(5) | BIT(0));
+
+	mmu_write(hw, DEC_REG_CONTROL, DEC_REG_CONTROL_VALUE);
+
+	return 0;
+}
+
+void dc_hw_enable_mmu_prefetch(struct dc_hw *hw, bool enable)
+{
+	if (enable)
+		dc_write(hw, DC_MMU_PREFETCH, BIT(0));
+	else
+		dc_write(hw, DC_MMU_PREFETCH, 0);
+}
+
+void dc_mmu_flush(struct dc_hw *hw)
+{
+	u32 config, read;
+
+	read = mmu_read(hw, MMU_REG_CONFIG);
+	config = read | BIT(4);
+
+	mmu_write(hw, MMU_REG_CONFIG, config);
+	mmu_write(hw, MMU_REG_CONFIG, read);
+}
+#endif
+
+void dc_hw_update_roi(struct dc_hw *hw, enum dc_hw_plane_id id,
+		      struct dc_hw_roi *roi)
+{
+	struct dc_hw_plane *plane = NULL;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		memcpy(&plane->roi, roi, sizeof(*roi) - sizeof(roi->dirty));
+		plane->roi.dirty = true;
+	}
+}
+
+void dc_hw_update_colorkey(struct dc_hw *hw, enum dc_hw_plane_id id,
+			   struct dc_hw_colorkey *colorkey)
+{
+	struct dc_hw_plane *plane = NULL;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		memcpy(&plane->colorkey, colorkey,
+		       sizeof(*colorkey) - sizeof(colorkey->dirty));
+		plane->colorkey.dirty = true;
+	}
+}
diff --git a/drivers/gpu/drm/eswin/es_dc_hw.h b/drivers/gpu/drm/eswin/es_dc_hw.h
new file mode 100644
index 000000000000..8999e0c382b4
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_hw.h
@@ -0,0 +1,497 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_DC_HW_H__
+#define __ES_DC_HW_H__
+
+#ifdef CONFIG_ESWIN_MMU
+#include "es_dc_mmu.h"
+#endif
+
+#define DC_HW_REVISION 0x24
+#define DC_HW_CHIP_CID 0x30
+
+#define DC_REG_BASE 0x1400
+
+#define DC_FRAMEBUFFER_CONFIG 0x1518
+#define DC_FRAMEBUFFER_CONFIG_EX 0x1CC0
+#define DC_FRAMEBUFFER_SCALE_CONFIG 0x1520
+#define DC_FRAMEBUFFER_ADDRESS 0x1400
+#define DC_FRAMEBUFFER_U_ADDRESS 0x1530
+#define DC_FRAMEBUFFER_V_ADDRESS 0x1538
+#define DC_FRAMEBUFFER_STRIDE 0x1408
+#define DC_FRAMEBUFFER_U_STRIDE 0x1800
+#define DC_FRAMEBUFFER_V_STRIDE 0x1808
+#define DC_FRAMEBUFFER_SIZE 0x1810
+#define DC_FRAMEBUFFER_SCALE_FACTOR_X 0x1828
+#define DC_FRAMEBUFFER_SCALE_FACTOR_Y 0x1830
+#define DC_FRAMEBUFFER_H_FILTER_COEF_INDEX 0x1838
+#define DC_FRAMEBUFFER_H_FILTER_COEF_DATA 0x1A00
+#define DC_FRAMEBUFFER_V_FILTER_COEF_INDEX 0x1A08
+#define DC_FRAMEBUFFER_V_FILTER_COEF_DATA 0x1A10
+#define DC_FRAMEBUFFER_INIT_OFFSET 0x1A20
+#define DC_FRAMEBUFFER_COLOR_KEY 0x1508
+#define DC_FRAMEBUFFER_COLOR_KEY_HIGH 0x1510
+#define DC_FRAMEBUFFER_CLEAR_VALUE 0x1A18
+#define DC_FRAMEBUFFER_COLOR_TABLE_INDEX 0x1818
+#define DC_FRAMEBUFFER_COLOR_TABLE_DATA 0x1820
+#define DC_FRAMEBUFFER_BG_COLOR 0x1528
+#define DC_FRAMEBUFFER_ROI_ORIGIN 0x1CB0
+#define DC_FRAMEBUFFER_ROI_SIZE 0x1CB8
+#define DC_FRAMEBUFFER_DEGAMMA_INDEX 0x1D88
+#define DC_FRAMEBUFFER_DEGAMMA_DATA 0x1D90
+#define DC_FRAMEBUFFER_DEGAMMA_EX_DATA 0x1D98
+#define DC_FRAMEBUFFER_YUVTORGB_COEF0 0x1DA0
+#define DC_FRAMEBUFFER_YUVTORGB_COEF1 0x1DA8
+#define DC_FRAMEBUFFER_YUVTORGB_COEF2 0x1DB0
+#define DC_FRAMEBUFFER_YUVTORGB_COEF3 0x1DB8
+#define DC_FRAMEBUFFER_YUVTORGB_COEF4 0x1E00
+#define DC_FRAMEBUFFER_YUVTORGB_COEFD0 0x1E08
+#define DC_FRAMEBUFFER_YUVTORGB_COEFD1 0x1E10
+#define DC_FRAMEBUFFER_YUVTORGB_COEFD2 0x1E18
+#define DC_FRAMEBUFFER_Y_CLAMP_BOUND 0x1E88
+#define DC_FRAMEBUFFER_UV_CLAMP_BOUND 0x1E90
+#define DC_FRAMEBUFFER_RGBTORGB_COEF0 0x1E20
+#define DC_FRAMEBUFFER_RGBTORGB_COEF1 0x1E28
+#define DC_FRAMEBUFFER_RGBTORGB_COEF2 0x1E30
+#define DC_FRAMEBUFFER_RGBTORGB_COEF3 0x1E38
+#define DC_FRAMEBUFFER_RGBTORGB_COEF4 0x1E40
+
+#define DC_OVERLAY_CONFIG 0x1540
+#define DC_OVERLAY_SCALE_CONFIG 0x1C00
+#define DC_OVERLAY_BLEND_CONFIG 0x1580
+#define DC_OVERLAY_TOP_LEFT 0x1640
+#define DC_OVERLAY_BOTTOM_RIGHT 0x1680
+#define DC_OVERLAY_ADDRESS 0x15C0
+#define DC_OVERLAY_U_ADDRESS 0x1840
+#define DC_OVERLAY_V_ADDRESS 0x1880
+#define DC_OVERLAY_STRIDE 0x1600
+#define DC_OVERLAY_U_STRIDE 0x18C0
+#define DC_OVERLAY_V_STRIDE 0x1900
+#define DC_OVERLAY_SIZE 0x17C0
+#define DC_OVERLAY_SCALE_FACTOR_X 0x1A40
+#define DC_OVERLAY_SCALE_FACTOR_Y 0x1A80
+#define DC_OVERLAY_H_FILTER_COEF_INDEX 0x1AC0
+#define DC_OVERLAY_H_FILTER_COEF_DATA 0x1B00
+#define DC_OVERLAY_V_FILTER_COEF_INDEX 0x1B40
+#define DC_OVERLAY_V_FILTER_COEF_DATA 0x1B80
+#define DC_OVERLAY_INIT_OFFSET 0x1BC0
+#define DC_OVERLAY_COLOR_KEY 0x1740
+#define DC_OVERLAY_COLOR_KEY_HIGH 0x1780
+#define DC_OVERLAY_CLEAR_VALUE 0x1940
+#define DC_OVERLAY_COLOR_TABLE_INDEX 0x1980
+#define DC_OVERLAY_COLOR_TABLE_DATA 0x19C0
+#define DC_OVERLAY_SRC_GLOBAL_COLOR 0x16C0
+#define DC_OVERLAY_DST_GLOBAL_COLOR 0x1700
+#define DC_OVERLAY_ROI_ORIGIN 0x1D00
+#define DC_OVERLAY_ROI_SIZE 0x1D40
+#define DC_OVERLAY_DEGAMMA_INDEX 0x2200
+#define DC_OVERLAY_DEGAMMA_DATA 0x2240
+#define DC_OVERLAY_DEGAMMA_EX_DATA 0x2280
+#define DC_OVERLAY_YUVTORGB_COEF0 0x1EC0
+#define DC_OVERLAY_YUVTORGB_COEF1 0x1F00
+#define DC_OVERLAY_YUVTORGB_COEF2 0x1F40
+#define DC_OVERLAY_YUVTORGB_COEF3 0x1F80
+#define DC_OVERLAY_YUVTORGB_COEF4 0x1FC0
+#define DC_OVERLAY_YUVTORGB_COEFD0 0x2000
+#define DC_OVERLAY_YUVTORGB_COEFD1 0x2040
+#define DC_OVERLAY_YUVTORGB_COEFD2 0x2080
+#define DC_OVERLAY_Y_CLAMP_BOUND 0x22C0
+#define DC_OVERLAY_UV_CLAMP_BOUND 0x2300
+#define DC_OVERLAY_RGBTORGB_COEF0 0x20C0
+#define DC_OVERLAY_RGBTORGB_COEF1 0x2100
+#define DC_OVERLAY_RGBTORGB_COEF2 0x2140
+#define DC_OVERLAY_RGBTORGB_COEF3 0x2180
+#define DC_OVERLAY_RGBTORGB_COEF4 0x21C0
+
+#define DC_CURSOR_CONFIG 0x1468
+#define DC_CURSOR_ADDRESS 0x146C
+#define DC_CURSOR_LOCATION 0x1470
+#define DC_CURSOR_BACKGROUND 0x1474
+#define DC_CURSOR_FOREGROUND 0x1478
+#define DC_CURSOR_CLK_GATING 0x1484
+
+#define DC_DISPLAY_DITHER_CONFIG 0x1410
+#define DC_DISPLAY_PANEL_CONFIG 0x1418
+#define DC_DISPLAY_DITHER_TABLE_LOW 0x1420
+#define DC_DISPLAY_DITHER_TABLE_HIGH 0x1428
+#define DC_DISPLAY_H 0x1430
+#define DC_DISPLAY_H_SYNC 0x1438
+#define DC_DISPLAY_V 0x1440
+#define DC_DISPLAY_V_SYNC 0x1448
+#define DC_DISPLAY_CURRENT_LOCATION 0x1450
+#define DC_DISPLAY_GAMMA_INDEX 0x1458
+#define DC_DISPLAY_GAMMA_DATA 0x1460
+#define DC_DISPLAY_INT 0x147C
+#define DC_DISPLAY_INT_ENABLE 0x1480
+#define DC_DISPLAY_DBI_CONFIG 0x1488
+#define DC_DISPLAY_GENERAL_CONFIG 0x14B0
+#define DC_DISPLAY_DPI_CONFIG 0x14B8
+#define DC_DISPLAY_DEBUG_COUNTER_SELECT 0x14D0
+#define DC_DISPLAY_DEBUG_COUNTER_VALUE 0x14D8
+#define DC_DISPLAY_DP_CONFIG 0x1CD0
+#define DC_DISPLAY_GAMMA_EX_INDEX 0x1CF0
+#define DC_DISPLAY_GAMMA_EX_DATA 0x1CF8
+#define DC_DISPLAY_GAMMA_EX_ONE_DATA 0x1D80
+#define DC_DISPLAY_RGBTOYUV_COEF0 0x1E48
+#define DC_DISPLAY_RGBTOYUV_COEF1 0x1E50
+#define DC_DISPLAY_RGBTOYUV_COEF2 0x1E58
+#define DC_DISPLAY_RGBTOYUV_COEF3 0x1E60
+#define DC_DISPLAY_RGBTOYUV_COEF4 0x1E68
+#define DC_DISPLAY_RGBTOYUV_COEFD0 0x1E70
+#define DC_DISPLAY_RGBTOYUV_COEFD1 0x1E78
+#define DC_DISPLAY_RGBTOYUV_COEFD2 0x1E80
+
+#define DC_CLK_GATTING 0x1A28
+
+#define DC_TRANSPARENCY_OPAQUE 0x00
+#define DC_TRANSPARENCY_KEY 0x02
+#define DC_DISPLAY_DITHERTABLE_LOW 0x7B48F3C0
+#define DC_DISPLAY_DITHERTABLE_HIGH 0x596AD1E2
+
+#define GAMMA_SIZE 256
+#define GAMMA_EX_SIZE 300
+#define DEGAMMA_SIZE 260
+
+#define RGB_TO_RGB_TABLE_SIZE 9
+#define YUV_TO_RGB_TABLE_SIZE 16
+#define RGB_TO_YUV_TABLE_SIZE 12
+
+#ifdef CONFIG_ESWIN_MMU
+#define DC_MMU_PREFETCH 0x1E98
+
+#define MMU_REG_BASE 0x0180
+
+#define MMU_REG_CONFIG 0x0184
+#define MMU_REG_CONTROL 0x0388
+#define MMU_REG_TABLE_ARRAY_SIZE 0x0394
+#define MMU_REG_SAFE_NON_SECURE 0x0398
+#define MMU_REG_SAFE_SECURE 0x039C
+#define MMU_REG_SAFE_EXT_ADDRESS 0x03A0
+#define MMU_REG_CONTEXT_PD 0x03B4
+
+#define DEC_REG_CONTROL 0x0800
+#define DEC_REG_CONTROL_VALUE 0x02010188
+#endif
+
+enum dc_hw_chip_rev {
+	DC_REV_5551,
+	DC_REV_5551_306,
+	DC_REV_5701_303,
+	DC_REV_5701_309,
+};
+
+enum dc_hw_plane_id { PRIMARY_PLANE, CURSOR_PLANE, OVERLAY_PLANE, PLANE_NUM };
+
+enum dc_hw_color_format {
+	FORMAT_X4R4G4B4,
+	FORMAT_A4R4G4B4,
+	FORMAT_X1R5G5B5,
+	FORMAT_A1R5G5B5,
+	FORMAT_R5G6B5,
+	FORMAT_X8R8G8B8,
+	FORMAT_A8R8G8B8,
+	FORMAT_YUY2,
+	FORMAT_UYVY,
+	FORMAT_INDEX8,
+	FORMAT_MONOCHROME,
+	FORMAT_YV12 = 0xf,
+	FORMAT_A8,
+	FORMAT_NV12,
+	FORMAT_NV16,
+	FORMAT_RG16,
+	FORMAT_R8,
+	FORMAT_NV12_10BIT,
+	FORMAT_A2R10G10B10,
+	FORMAT_NV16_10BIT,
+	FORMAT_INDEX1,
+	FORMAT_INDEX2,
+	FORMAT_INDEX4,
+	FORMAT_P010,
+};
+
+enum dc_hw_yuv_color_space {
+	COLOR_SPACE_601 = 0,
+	COLOR_SPACE_709 = 1,
+	COLOR_SPACE_2020 = 3,
+};
+
+enum dc_hw_rotation {
+	ROT_0 = 0,
+	ROT_90 = 4,
+	ROT_180 = 5,
+	ROT_270 = 6,
+	FLIP_X = 1,
+	FLIP_Y = 2,
+	FLIP_XY = 3,
+};
+
+enum dc_hw_swizzle {
+	SWIZZLE_ARGB = 0,
+	SWIZZLE_RGBA,
+	SWIZZLE_ABGR,
+	SWIZZLE_BGRA,
+};
+
+enum dc_hw_cursor_size {
+	CURSOR_SIZE_32X32 = 0,
+	CURSOR_SIZE_64X64,
+	CURSOR_SIZE_128X128,
+	CURSOR_SIZE_256X256,
+};
+
+enum dc_hw_out {
+	OUT_DPI,
+	OUT_DP,
+};
+
+enum dc_hw_blend_mode {
+	/* out.rgb = plane_alpha * fg.rgb +
+     *      (1 - (plane_alpha * fg.alpha)) * bg.rgb
+     */
+	BLEND_PREMULTI,
+	/* out.rgb = plane_alpha * fg.alpha * fg.rgb +
+     *      (1 - (plane_alpha * fg.alpha)) * bg.rgb
+     */
+	BLEND_COVERAGE,
+	/* out.rgb = plane_alpha * fg.rgb +
+     *      (1 - plane_alpha) * bg.rgb
+     */
+	BLEND_PIXEL_NONE,
+};
+
+struct dc_hw_plane_reg {
+	u32 y_address;
+	u32 u_address;
+	u32 v_address;
+	u32 y_stride;
+	u32 u_stride;
+	u32 v_stride;
+	u32 size;
+	u32 scale_factor_x;
+	u32 scale_factor_y;
+	u32 h_filter_coef_index;
+	u32 h_filter_coef_data;
+	u32 v_filter_coef_index;
+	u32 v_filter_coef_data;
+	u32 init_offset;
+	u32 color_key;
+	u32 color_key_high;
+	u32 clear_value;
+	u32 color_table_index;
+	u32 color_table_data;
+	u32 scale_config;
+	u32 degamma_index;
+	u32 degamma_data;
+	u32 degamma_ex_data;
+	u32 roi_origin;
+	u32 roi_size;
+	u32 YUVToRGBCoef0;
+	u32 YUVToRGBCoef1;
+	u32 YUVToRGBCoef2;
+	u32 YUVToRGBCoef3;
+	u32 YUVToRGBCoef4;
+	u32 YUVToRGBCoefD0;
+	u32 YUVToRGBCoefD1;
+	u32 YUVToRGBCoefD2;
+	u32 YClampBound;
+	u32 UVClampBound;
+	u32 RGBToRGBCoef0;
+	u32 RGBToRGBCoef1;
+	u32 RGBToRGBCoef2;
+	u32 RGBToRGBCoef3;
+	u32 RGBToRGBCoef4;
+};
+
+struct dc_hw_fb {
+	u32 y_address;
+	u32 u_address;
+	u32 v_address;
+	u32 clear_value;
+	u16 y_stride;
+	u16 u_stride;
+	u16 v_stride;
+	u16 width;
+	u16 height;
+	u8 format;
+	u8 tile_mode;
+	u8 rotation;
+	u8 yuv_color_space;
+	u8 swizzle;
+	u8 uv_swizzle;
+	bool clear_enable;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_scale {
+	u32 scale_factor_x;
+	u32 scale_factor_y;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_position {
+	u16 start_x;
+	u16 start_y;
+	u16 end_x;
+	u16 end_y;
+	bool dirty;
+};
+
+struct dc_hw_blend {
+	u8 alpha;
+	u8 blend_mode;
+	bool dirty;
+};
+
+struct dc_hw_colorkey {
+	u32 colorkey;
+	u32 colorkey_high;
+	u8 transparency;
+	bool dirty;
+};
+
+struct dc_hw_roi {
+	u16 x;
+	u16 y;
+	u16 width;
+	u16 height;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_cursor {
+	u32 address;
+	u16 x;
+	u16 y;
+	u16 hot_x;
+	u16 hot_y;
+	u8 size;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_display {
+	u32 bus_format;
+	u16 h_active;
+	u16 h_total;
+	u16 h_sync_start;
+	u16 h_sync_end;
+	u16 v_active;
+	u16 v_total;
+	u16 v_sync_start;
+	u16 v_sync_end;
+	u16 sync_mode;
+	u32 bg_color;
+	bool h_sync_polarity;
+	bool v_sync_polarity;
+	bool enable;
+	bool dither_enable;
+};
+
+struct dc_hw_gamma {
+	u16 gamma[GAMMA_EX_SIZE][3];
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_degamma {
+	u16 degamma[DEGAMMA_SIZE][3];
+	u32 mode;
+	bool dirty;
+};
+
+struct dc_hw_plane {
+	struct dc_hw_fb fb;
+	struct dc_hw_scale scale;
+	struct dc_hw_roi roi;
+	struct dc_hw_colorkey colorkey;
+	struct dc_hw_degamma degamma;
+};
+
+struct dc_hw_overlay {
+	struct dc_hw_plane plane;
+	struct dc_hw_position pos;
+	struct dc_hw_blend blend;
+};
+
+struct dc_hw_read {
+	u32 reg;
+	u32 value;
+};
+
+struct dc_hw;
+struct dc_hw_funcs {
+	void (*gamma)(struct dc_hw *hw);
+	void (*plane)(struct dc_hw *hw);
+	void (*display)(struct dc_hw *hw, struct dc_hw_display *display);
+};
+
+struct dc_hw {
+	enum dc_hw_chip_rev rev;
+	enum dc_hw_out out;
+	void *hi_base;
+	void *reg_base;
+#ifdef CONFIG_ESWIN_MMU
+	void *mmu_base;
+#endif
+	struct dc_hw_read *read_block;
+	struct dc_hw_display display;
+	struct dc_hw_gamma gamma;
+	struct dc_hw_plane primary;
+	struct dc_hw_overlay overlay;
+	struct dc_hw_cursor cursor;
+	struct dc_hw_funcs *func;
+	struct es_dc_info *info;
+};
+
+int dc_hw_init(struct dc_hw *hw);
+void dc_hw_deinit(struct dc_hw *hw);
+void dc_hw_update_plane(struct dc_hw *hw, enum dc_hw_plane_id id,
+			struct dc_hw_fb *fb, struct dc_hw_scale *scale);
+void dc_hw_update_degamma(struct dc_hw *hw, enum dc_hw_plane_id id, u32 mode);
+void dc_hw_set_position(struct dc_hw *hw, struct dc_hw_position *pos);
+void dc_hw_set_blend(struct dc_hw *hw, struct dc_hw_blend *blend);
+void dc_hw_update_cursor(struct dc_hw *hw, struct dc_hw_cursor *cursor);
+void dc_hw_update_gamma(struct dc_hw *hw, u16 index, u16 r, u16 g, u16 b);
+void dc_hw_enable_gamma(struct dc_hw *hw, bool enable);
+void dc_hw_enable_dump(struct dc_hw *hw, u32 addr, u32 pitch);
+void dc_hw_disable_dump(struct dc_hw *hw);
+void dc_hw_setup_display(struct dc_hw *hw, struct dc_hw_display *display);
+void dc_hw_enable_interrupt(struct dc_hw *hw, bool enable);
+u32 dc_hw_get_interrupt(struct dc_hw *hw);
+bool dc_hw_flip_in_progress(struct dc_hw *hw);
+bool dc_hw_check_underflow(struct dc_hw *hw);
+void dc_hw_enable_shadow_register(struct dc_hw *hw, bool enable);
+void dc_hw_set_out(struct dc_hw *hw, enum dc_hw_out out);
+void dc_hw_commit(struct dc_hw *hw);
+#ifdef CONFIG_ESWIN_MMU
+int dc_hw_mmu_init(struct dc_hw *hw, dc_mmu_pt mmu);
+void dc_hw_enable_mmu_prefetch(struct dc_hw *hw, bool enable);
+void dc_mmu_flush(struct dc_hw *hw);
+#endif
+void dc_hw_update_roi(struct dc_hw *hw, enum dc_hw_plane_id id,
+		      struct dc_hw_roi *roi);
+void dc_hw_update_colorkey(struct dc_hw *hw, enum dc_hw_plane_id id,
+			   struct dc_hw_colorkey *colorkey);
+
+#endif /* __ES_DC_HW_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dc_mmu.c b/drivers/gpu/drm/eswin/es_dc_mmu.c
new file mode 100644
index 000000000000..4e7e6a6be107
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_mmu.c
@@ -0,0 +1,788 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+
+#include <asm/io.h>
+#include <asm/delay.h>
+
+#include "es_dc_mmu.h"
+
+int _allocate_memory(u32 bytes, void **memory)
+{
+	void *mem = NULL;
+
+	if (bytes == 0 || memory == NULL) {
+		pr_err("%s has invalid arguments.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	if (bytes > PAGE_SIZE) {
+		mem = vmalloc(bytes);
+	} else {
+		mem = kmalloc(bytes, GFP_KERNEL);
+	}
+
+	if (!mem) {
+		pr_err("%s out of memory.\n", __FUNCTION__);
+		return -ENOMEM;
+	}
+
+	memset((u8 *)mem, 0, bytes);
+	*memory = mem;
+
+	return 0;
+}
+
+static int _create_mutex(void **mutex)
+{
+	int ret = 0;
+
+	if (mutex == NULL) {
+		return -EINVAL;
+	}
+
+	ret = _allocate_memory(sizeof(struct mutex), mutex);
+	if (ret)
+		return ret;
+
+	mutex_init(*(struct mutex **)mutex);
+
+	return 0;
+}
+
+static int _acquire_mutex(void *mutex, u32 timeout)
+{
+	if (mutex == NULL) {
+		pr_err("%s has invalid argument.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	if (timeout == DC_INFINITE) {
+		mutex_lock(mutex);
+		return 0;
+	}
+
+	for (;;) {
+		/* Try to acquire the mutex. */
+		if (mutex_trylock(mutex)) {
+			/* Success. */
+			return 0;
+		}
+
+		if (timeout-- == 0) {
+			break;
+		}
+
+		/* Wait for 1 millisecond. */
+		udelay(1000);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int _release_mutex(void *mutex)
+{
+	if (mutex == NULL) {
+		pr_err("%s has invalid argument.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	mutex_unlock(mutex);
+
+	return 0;
+}
+
+static u32 _mtlb_offset(u32 address)
+{
+	return (address & MMU_MTLB_MASK) >> MMU_MTLB_SHIFT;
+}
+
+static u32 _stlb_offset(u32 address)
+{
+	return (address & MMU_STLB_4K_MASK) >> MMU_STLB_4K_SHIFT;
+}
+
+static u32 _address_to_index(dc_mmu_pt mmu, u32 address)
+{
+	return _mtlb_offset(address) * MMU_STLB_4K_ENTRY_NUM +
+	       _stlb_offset(address);
+}
+
+static u32 _set_page(u32 page_address, u32 page_address_ext, bool writable)
+{
+	u32 entry = page_address
+		    /* AddressExt */
+		    | (page_address_ext << 4)
+		    /* Ignore exception */
+		    | (0 << 1)
+		    /* Present */
+		    | (1 << 0);
+
+	if (writable) {
+		/* writable */
+		entry |= (1 << 2);
+	}
+
+	return entry;
+}
+
+static void _write_page_entry(u32 *page_entry, u32 entry_value)
+{
+	*page_entry = entry_value;
+}
+
+static u32 _read_page_entry(u32 *page_entry)
+{
+	return *page_entry;
+}
+
+int _allocate_stlb(dc_mmu_stlb_pt *stlb)
+{
+	dc_mmu_stlb_pt stlb_t = NULL;
+	void *mem = NULL;
+
+	mem = kzalloc(sizeof(dc_mmu_stlb), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	stlb_t = (dc_mmu_stlb_pt)mem;
+
+	stlb_t->size = MMU_STLB_4K_SIZE;
+
+	*stlb = stlb_t;
+
+	return 0;
+}
+
+int _allocate_all_stlb(struct device *dev, dc_mmu_stlb_pt *stlb)
+{
+	dc_mmu_stlb_pt stlb_t = NULL;
+	void *mem = NULL;
+	void *cookie = NULL;
+	dma_addr_t dma_addr;
+	size_t size;
+
+	mem = kzalloc(sizeof(dc_mmu_stlb), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	stlb_t = (dc_mmu_stlb_pt)mem;
+
+	stlb_t->size = MMU_STLB_4K_SIZE * MMU_MTLB_ENTRY_NUM;
+	size = PAGE_ALIGN(stlb_t->size);
+
+	cookie = dma_alloc_wc(dev, size, &dma_addr, GFP_KERNEL);
+	if (!cookie) {
+		dev_err(dev, "Failed to alloc stlb buffer.\n");
+		return -ENOMEM;
+	}
+
+	stlb_t->logical = cookie;
+	stlb_t->physBase = (u64)dma_addr;
+	memset(stlb_t->logical, 0, size);
+
+	*stlb = stlb_t;
+
+	return 0;
+}
+
+int _setup_process_address_space(struct device *dev, dc_mmu_pt mmu)
+{
+	u32 *map = NULL;
+	u32 free, i;
+	u32 dynamic_mapping_entries, address;
+	dc_mmu_stlb_pt all_stlb;
+	int ret = 0;
+
+	dynamic_mapping_entries = MMU_MTLB_ENTRY_NUM;
+	mmu->dynamic_mapping_start = 0;
+	mmu->page_table_size = dynamic_mapping_entries * MMU_STLB_4K_SIZE;
+
+	mmu->page_table_entries = mmu->page_table_size / sizeof(u32);
+
+	ret = _allocate_memory(mmu->page_table_size,
+			       (void **)&mmu->map_logical);
+	if (ret) {
+		pr_err("Failed to alloc mmu map buffer.\n");
+		return ret;
+		;
+	}
+
+	ret = _allocate_memory(mmu->page_table_size,
+			       (void **)&mmu->page_entry_info);
+	if (ret) {
+		pr_err("Failed to alloc entry info buffer.\n");
+		return ret;
+		;
+	}
+
+	map = mmu->map_logical;
+
+	/* Initialize free area */
+	free = mmu->page_table_entries;
+	_write_page_entry(map, (free << 8) | DC_MMU_FREE);
+	_write_page_entry(map + 1, ~0U);
+
+	mmu->heap_list = 0;
+	mmu->free_nodes = false;
+
+	ret = _allocate_all_stlb(dev, &all_stlb);
+	if (ret)
+		return ret;
+
+	BUG_ON((all_stlb->physBase & (0xffUL << 32)) !=
+	       (mmu->mtlb_physical & (0xffUL << 32)));
+
+	mmu->stlb_physical = all_stlb->physBase;
+	mmu->stlb_bytes = all_stlb->size;
+
+	for (i = 0; i < dynamic_mapping_entries; i++) {
+		dc_mmu_stlb_pt stlb;
+		dc_mmu_stlb_pt *stlbs = (dc_mmu_stlb_pt *)mmu->stlbs;
+
+		ret = _allocate_stlb(&stlb);
+		if (ret)
+			return ret;
+
+		stlb->physBase = all_stlb->physBase + i * MMU_STLB_4K_SIZE;
+		stlb->logical =
+			all_stlb->logical + i * MMU_STLB_4K_SIZE / sizeof(u32);
+
+		stlbs[i] = stlb;
+	}
+
+	address = (u32)all_stlb->physBase;
+
+	ret = _acquire_mutex(mmu->page_table_mutex, DC_INFINITE);
+	if (ret)
+		return ret;
+
+	for (i = mmu->dynamic_mapping_start;
+	     i < mmu->dynamic_mapping_start + dynamic_mapping_entries; i++) {
+		u32 mtlb_entry;
+
+		mtlb_entry = address | MMU_MTLB_4K_PAGE | MMU_MTLB_PRESENT;
+
+		address += MMU_STLB_4K_SIZE;
+
+		/* Insert Slave TLB address to Master TLB entry.*/
+		_write_page_entry(mmu->mtlb_logical + i, mtlb_entry);
+	}
+
+	kfree(all_stlb);
+	_release_mutex(mmu->page_table_mutex);
+
+	return 0;
+}
+
+/* MMU Construct */
+int dc_mmu_construct(struct device *dev, dc_mmu_pt *mmu)
+{
+	dc_mmu_pt mmu_t = NULL;
+	void *mem = NULL;
+	void *cookie = NULL, *cookie_safe = NULL;
+	dma_addr_t dma_addr, dma_addr_safe;
+	u32 size = 0;
+	int ret = 0;
+
+	mem = kzalloc(sizeof(dc_mmu), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	mmu_t = (dc_mmu_pt)mem;
+	mmu_t->mtlb_bytes = MMU_MTLB_SIZE;
+	size = PAGE_ALIGN(mmu_t->mtlb_bytes);
+	if (size == PAGE_SIZE) {
+		size = PAGE_SIZE * 2;
+	}
+
+	/* Allocate MTLB */
+	cookie = dma_alloc_wc(dev, size, &dma_addr, GFP_KERNEL);
+	if (!cookie) {
+		dev_err(dev, "Failed to alloc mtlb buffer.\n");
+		return -ENOMEM;
+	}
+
+	mmu_t->mtlb_logical = cookie;
+	mmu_t->mtlb_physical = (u64)dma_addr;
+	memset(mmu_t->mtlb_logical, 0, size);
+
+	size = MMU_MTLB_ENTRY_NUM * sizeof(dc_mmu_stlb_pt);
+
+	ret = _allocate_memory(size, &mmu_t->stlbs);
+	if (ret)
+		return ret;
+
+	ret = _create_mutex(&mmu_t->page_table_mutex);
+	if (ret)
+		return ret;
+
+	mmu_t->mode = MMU_MODE_1K;
+
+	ret = _setup_process_address_space(dev, mmu_t);
+	if (ret)
+		return ret;
+
+	/* Allocate safe page */
+	cookie_safe = dma_alloc_wc(dev, 4096, &dma_addr_safe, GFP_KERNEL);
+	if (!cookie_safe) {
+		dev_err(dev, "Failed to alloc safe page.\n");
+		return -ENOMEM;
+	}
+
+	mmu_t->safe_page_logical = cookie_safe;
+	mmu_t->safe_page_physical = (u64)dma_addr_safe;
+	memset(mmu_t->safe_page_logical, 0, size);
+
+	*mmu = mmu_t;
+
+	return 0;
+}
+
+int dc_mmu_get_page_entry(dc_mmu_pt mmu, u32 address, u32 **page_table)
+{
+	dc_mmu_stlb_pt stlb;
+	dc_mmu_stlb_pt *stlbs = (dc_mmu_stlb_pt *)mmu->stlbs;
+	u32 mtlb_offset = _mtlb_offset(address);
+	u32 stlb_offset = _stlb_offset(address);
+
+	stlb = stlbs[mtlb_offset - mmu->dynamic_mapping_start];
+	if (stlb == NULL) {
+		pr_err("BUG: invalid stlb,  mmu=%p stlbs=%p  mtlb_offset=0x%x %s(%d)\n",
+		       mmu, stlbs, mtlb_offset, __FUNCTION__, __LINE__);
+		return -ENXIO;
+	}
+
+	*page_table = &stlb->logical[stlb_offset];
+
+	return 0;
+}
+
+int _link(dc_mmu_pt mmu, u32 index, u32 node)
+{
+	if (index >= mmu->page_table_entries) {
+		mmu->heap_list = node;
+	} else {
+		u32 *map = mmu->map_logical;
+
+		switch (DC_ENTRY_TYPE(_read_page_entry(&map[index]))) {
+		case DC_MMU_SINGLE:
+			/* Previous is a single node, link to it*/
+			_write_page_entry(&map[index],
+					  (node << 8) | DC_MMU_SINGLE);
+			break;
+		case DC_MMU_FREE:
+			/* Link to FREE TYPE node */
+			_write_page_entry(&map[index + 1], node);
+			break;
+		default:
+			pr_err("MMU table corrupted at index %u!", index);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int _add_free(dc_mmu_pt mmu, u32 index, u32 node, u32 count)
+{
+	u32 *map = mmu->map_logical;
+
+	if (count == 1) {
+		/* Initialize a single page node */
+		_write_page_entry(map + node, DC_SINGLE_PAGE_NODE_INITIALIZE |
+						      DC_MMU_SINGLE);
+	} else {
+		/* Initialize the FREE node*/
+		_write_page_entry(map + node, (count << 8) | DC_MMU_FREE);
+		_write_page_entry(map + node + 1, ~0U);
+	}
+
+	return _link(mmu, index, node);
+}
+
+/* Collect free nodes */
+int _collect(dc_mmu_pt mmu)
+{
+	u32 *map = mmu->map_logical;
+	u32 count = 0, start = 0, i = 0;
+	u32 previous = ~0U;
+	int ret = 0;
+
+	mmu->heap_list = ~0U;
+	mmu->free_nodes = false;
+
+	/* Walk the entire page table */
+	for (i = 0; i < mmu->page_table_entries; i++) {
+		switch (DC_ENTRY_TYPE(_read_page_entry(&map[i]))) {
+		case DC_MMU_SINGLE:
+			if (count++ == 0) {
+				/* Set new start node */
+				start = i;
+			}
+			break;
+		case DC_MMU_FREE:
+			if (count == 0) {
+				/* Set new start node */
+				start = i;
+			}
+
+			count += _read_page_entry(&map[i]) >> 8;
+			/* Advance the index of the page table */
+			i += (_read_page_entry(&map[i]) >> 8) - 1;
+			break;
+		case DC_MMU_USED:
+			/* Meet used node, start to collect */
+			if (count > 0) {
+				/* Add free node to list*/
+				ret = _add_free(mmu, previous, start, count);
+				if (ret)
+					return ret;
+				/* Reset previous unused node index */
+				previous = start;
+				count = 0;
+			}
+			break;
+		default:
+			pr_err("MMU page table corrupted at index %u!", i);
+			return -EINVAL;
+		}
+	}
+
+	/* If left node is an open node. */
+	if (count > 0) {
+		ret = _add_free(mmu, previous, start, count);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int _fill_page_table(u32 *page_table, u32 page_count, u32 entry_value)
+{
+	u32 i;
+
+	for (i = 0; i < page_count; i++) {
+		_write_page_entry(page_table + i, entry_value);
+	}
+
+	return 0;
+}
+
+int dc_mmu_allocate_pages(dc_mmu_pt mmu, u32 page_count, u32 *address)
+{
+	bool got = false, acquired = false;
+	u32 *map;
+	u32 index = 0, vaddr, left;
+	u32 previous = ~0U;
+	u32 mtlb_offset, stlb_offset;
+	int ret = 0;
+
+	if (page_count == 0 || page_count > mmu->page_table_entries) {
+		pr_err("%s has invalid arguments.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	_acquire_mutex(mmu->page_table_mutex, DC_INFINITE);
+	acquired = true;
+
+	for (map = mmu->map_logical; !got;) {
+		for (index = mmu->heap_list;
+		     !got && (index < mmu->page_table_entries);) {
+			switch (DC_ENTRY_TYPE(_read_page_entry(&map[index]))) {
+			case DC_MMU_SINGLE:
+				if ((index != 0) && (page_count == 1)) {
+					got = true;
+				} else {
+					/* Move to next node */
+					previous = index;
+					index = _read_page_entry(&map[index]) >>
+						8;
+				}
+				break;
+			case DC_MMU_FREE:
+				if ((index == 0) &&
+				    (page_count <
+				     ((_read_page_entry(&map[index]) >> 8)))) {
+					got = true;
+				} else if ((index != 0) &&
+					   (page_count <=
+					    ((_read_page_entry(&map[index]) >>
+					      8)))) {
+					got = true;
+				} else {
+					/* Move to next node */
+					previous = index;
+					index = _read_page_entry(
+						&map[index + 1]);
+				}
+				break;
+			default:
+				/* Only link SINGLE and FREE node */
+				pr_err("MMU table corrupted at index %u!",
+				       index);
+				ret = -EINVAL;
+				goto OnError;
+			}
+		}
+
+		/* If out of index */
+		if (index >= mmu->page_table_entries) {
+			if (mmu->free_nodes) {
+				/* Collect the free node */
+				ret = _collect(mmu);
+				if (ret)
+					goto OnError;
+				previous = ~0U;
+			} else {
+				ret = -ENODATA;
+				goto OnError;
+			}
+		}
+	}
+
+	switch (DC_ENTRY_TYPE(_read_page_entry(&map[index]))) {
+	case DC_MMU_SINGLE:
+		/* Unlink single node from node list */
+		ret = _link(mmu, previous, _read_page_entry(&map[index]) >> 8);
+		if (ret)
+			goto OnError;
+		break;
+
+	case DC_MMU_FREE:
+		left = (_read_page_entry(&map[index]) >> 8) - page_count;
+		switch (left) {
+		case 0:
+			/* Unlink the entire FREE type node */
+			ret = _link(mmu, previous,
+				    _read_page_entry(&map[index + 1]));
+			if (ret)
+				goto OnError;
+			break;
+		case 1:
+			/* Keep the map[index] as a single node,
+             * mark the left as used
+             */
+			_write_page_entry(&map[index],
+					  (_read_page_entry(&map[index + 1])
+					   << 8) | DC_MMU_SINGLE);
+			index++;
+			break;
+		default:
+			/* FREE type node left */
+			_write_page_entry(&map[index],
+					  (left << 8) | DC_MMU_FREE);
+			index += left;
+			if (index + page_count > mmu->page_table_entries) {
+				pr_err("Error: index %d , left %d\n", index,
+				       left);
+				ret = -EINVAL;
+				goto OnError;
+			}
+
+			break;
+		}
+		break;
+	default:
+		/* Only link SINGLE and FREE node */
+		pr_err("MMU table corrupted at index %u!", index);
+		ret = -EINVAL;
+		goto OnError;
+	}
+
+	if (index + page_count > mmu->page_table_entries) {
+		pr_err("Fatal ERROR: This may caused by free pages more than allocated pages\n");
+		_write_page_entry(mmu->map_logical,
+				  (mmu->page_table_entries << 8) | DC_MMU_FREE);
+		_write_page_entry(mmu->map_logical + 1, ~0U);
+		previous = ~0U;
+		mmu->heap_list = 0;
+		memset(mmu->page_entry_info, 0,
+		       mmu->page_table_entries * sizeof(u32));
+
+		ret = -EINVAL;
+		goto OnError;
+	}
+
+	/* Mark node as used */
+	ret = _fill_page_table(&map[index], page_count, DC_MMU_USED);
+	if (ret)
+		goto OnError;
+
+	_release_mutex(mmu->page_table_mutex);
+
+	mtlb_offset =
+		index / MMU_STLB_4K_ENTRY_NUM + mmu->dynamic_mapping_start;
+	stlb_offset = index % MMU_STLB_4K_ENTRY_NUM;
+
+	vaddr = (mtlb_offset << MMU_MTLB_SHIFT) |
+		(stlb_offset << MMU_STLB_4K_SHIFT);
+
+	if (vaddr == 0) {
+		pr_err("Error occured on alloc vaddr\n");
+		ret = -ENODATA;
+		goto OnError;
+	}
+
+	mmu->page_entry_info[index] = (index << 12) | (page_count & 0xfff);
+
+	if (address != NULL) {
+		*address = vaddr;
+	}
+
+	return 0;
+
+OnError:
+	if (acquired) {
+		_release_mutex(mmu->page_table_mutex);
+	}
+
+	return ret;
+}
+
+int dc_mmu_free_pages(dc_mmu_pt mmu, u32 address, u32 page_count)
+{
+	u32 *node, index;
+
+	if (page_count == 0)
+		return -EINVAL;
+
+	index = _address_to_index(mmu, address);
+	node = mmu->map_logical + _address_to_index(mmu, address);
+
+	_acquire_mutex(mmu->page_table_mutex, DC_INFINITE);
+
+	if (((mmu->page_entry_info[index] & 0xfff) != (page_count & 0xfff)) ||
+	    ((mmu->page_entry_info[index] >> 12) != index)) {
+		pr_err("index %d, page_count %d: free pages not equal to alloc\n",
+		       index, page_count);
+		_release_mutex(mmu->page_table_mutex);
+		return -EINVAL;
+	} else {
+		mmu->page_entry_info[index] = 0;
+	}
+
+	if (page_count == 1) {
+		/* Mark the Single page node free */
+		_write_page_entry(node, DC_SINGLE_PAGE_NODE_INITIALIZE |
+						DC_MMU_SINGLE);
+	} else {
+		/* Mark the FREE type node free */
+		_write_page_entry(node, (page_count << 8) | DC_MMU_FREE);
+		_write_page_entry(node + 1, ~0U);
+	}
+
+	mmu->free_nodes = true;
+
+	_release_mutex(mmu->page_table_mutex);
+
+	return 0;
+}
+
+int dc_mmu_set_page(dc_mmu_pt mmu, u64 page_address, u32 *page_entry)
+{
+	u32 address_ext;
+	u32 address;
+
+	if (page_entry == NULL || (page_address & 0xFFF)) {
+		return -EINVAL;
+	}
+
+	/* [31:0]. */
+	address = (u32)(page_address & 0xFFFFFFFF);
+	/* [39:32]. */
+	address_ext = (u32)((page_address >> 32) & 0xFF);
+
+	_write_page_entry(page_entry, _set_page(address, address_ext, true));
+
+	return 0;
+}
+
+int dc_mmu_map_memory(dc_mmu_pt mmu, u64 physical, u32 page_count, u32 *address,
+		      bool continuous)
+{
+	u32 virutal_address, i = 0;
+	bool allocated = false;
+	int ret = 0;
+
+	ret = dc_mmu_allocate_pages(mmu, page_count, &virutal_address);
+	if (ret)
+		goto OnError;
+
+	if (continuous)
+		*address = virutal_address | (physical & MMU_PAGE_4K_MASK);
+	else
+		*address = virutal_address;
+
+	allocated = true;
+
+	/* Fill in page table */
+	for (i = 0; i < page_count; i++) {
+		u64 page_phy;
+		u32 *page_entry;
+		struct page **pages;
+
+		if (continuous == true) {
+			page_phy = physical + i * MMU_PAGE_4K_SIZE;
+		} else {
+			pages = (struct page **)physical;
+			page_phy = page_to_phys(pages[i]);
+		}
+
+		ret = dc_mmu_get_page_entry(mmu, virutal_address, &page_entry);
+		if (ret)
+			goto OnError;
+
+		/* Write the page address to the page entry */
+		ret = dc_mmu_set_page(mmu, page_phy, page_entry);
+		if (ret)
+			goto OnError;
+
+		/* Get next page */
+		virutal_address += MMU_PAGE_4K_SIZE;
+	}
+
+	wmb();
+
+	return 0;
+
+OnError:
+	if (allocated)
+		dc_mmu_free_pages(mmu, virutal_address, page_count);
+	pr_info("%s fail!\n", __FUNCTION__);
+
+	return ret;
+}
+
+int dc_mmu_unmap_memory(dc_mmu_pt mmu, u32 gpu_address, u32 page_count)
+{
+	return dc_mmu_free_pages(mmu, gpu_address, page_count);
+}
diff --git a/drivers/gpu/drm/eswin/es_dc_mmu.h b/drivers/gpu/drm/eswin/es_dc_mmu.h
new file mode 100644
index 000000000000..ad5dc4e3dc1a
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_mmu.h
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef _ES_DC_MMU_H_
+#define _ES_DC_MMU_H_
+
+#include "es_type.h"
+
+#define DC_INFINITE ((u32)(~0U))
+
+#define DC_ENTRY_TYPE(x) (x & 0xF0)
+#define DC_SINGLE_PAGE_NODE_INITIALIZE (~((1U << 8) - 1))
+
+#define DC_INVALID_PHYSICAL_ADDRESS ~0ULL
+#define DC_INVALID_ADDRESS ~0U
+
+/* 1k mode */
+#define MMU_MTLB_SHIFT 24
+#define MMU_STLB_4K_SHIFT 12
+
+#define MMU_MTLB_BITS (32 - MMU_MTLB_SHIFT)
+#define MMU_PAGE_4K_BITS MMU_STLB_4K_SHIFT
+#define MMU_STLB_4K_BITS (32 - MMU_MTLB_BITS - MMU_PAGE_4K_BITS)
+
+#define MMU_MTLB_ENTRY_NUM (1 << MMU_MTLB_BITS)
+#define MMU_MTLB_SIZE (MMU_MTLB_ENTRY_NUM << 2)
+#define MMU_STLB_4K_ENTRY_NUM (1 << MMU_STLB_4K_BITS)
+#define MMU_STLB_4K_SIZE (MMU_STLB_4K_ENTRY_NUM << 2)
+#define MMU_PAGE_4K_SIZE (1 << MMU_STLB_4K_SHIFT)
+
+#define MMU_MTLB_MASK (~((1U << MMU_MTLB_SHIFT) - 1))
+#define MMU_STLB_4K_MASK ((~0U << MMU_STLB_4K_SHIFT) ^ MMU_MTLB_MASK)
+#define MMU_PAGE_4K_MASK (MMU_PAGE_4K_SIZE - 1)
+
+/* page offset definitions. */
+#define MMU_OFFSET_4K_BITS (32 - MMU_MTLB_BITS - MMU_STLB_4K_BITS)
+#define MMU_OFFSET_4K_MASK ((1U << MMU_OFFSET_4K_BITS) - 1)
+
+#define MMU_MTLB_PRESENT 0x00000001
+#define MMU_MTLB_EXCEPTION 0x00000002
+#define MMU_MTLB_4K_PAGE 0x00000000
+
+typedef enum _dc_mmu_type {
+	DC_MMU_USED = (0 << 4),
+	DC_MMU_SINGLE = (1 << 4),
+	DC_MMU_FREE = (2 << 4),
+} dc_mmu_type;
+
+typedef enum _dc_mmu_mode {
+	MMU_MODE_1K,
+	MMU_MODE_4K,
+} dc_mmu_mode;
+
+typedef struct _dc_mmu_stlb {
+	u32 *logical;
+	void *physical;
+	u32 size;
+	u64 physBase;
+	u32 pageCount;
+} dc_mmu_stlb, *dc_mmu_stlb_pt;
+
+typedef struct _dc_mmu {
+	u32 mtlb_bytes;
+	u64 mtlb_physical;
+	u32 *mtlb_logical;
+
+	void *safe_page_logical;
+	u64 safe_page_physical;
+
+	u32 dynamic_mapping_start;
+
+	void *stlbs;
+
+	// u64    stlb_physicals[MMU_MTLB_ENTRY_NUM];
+	u32 stlb_bytes;
+	u64 stlb_physical;
+
+	u32 page_table_entries;
+	u32 page_table_size;
+	u32 heap_list;
+
+	u32 *map_logical;
+	u32 *page_entry_info;
+	bool free_nodes;
+
+	void *page_table_mutex;
+
+	dc_mmu_mode mode;
+
+	void *static_stlb;
+} dc_mmu, *dc_mmu_pt;
+
+int dc_mmu_construct(struct device *dev, dc_mmu_pt *mmu);
+int dc_mmu_map_memory(dc_mmu_pt mmu, u64 physical, u32 page_count, u32 *address,
+		      bool continuous);
+int dc_mmu_unmap_memory(dc_mmu_pt mmu, u32 gpu_address, u32 page_count);
+
+#endif /* _ES_DC_MMU_H_ */
diff --git a/drivers/gpu/drm/eswin/es_drm.h b/drivers/gpu/drm/eswin/es_drm.h
new file mode 100644
index 000000000000..5e1cacfc14a0
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_drm.h
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_DRM_H__
+#define __ES_DRM_H__
+
+#include <drm/drm.h>
+
+enum drm_es_degamma_mode {
+	ES_DEGAMMA_DISABLE = 0,
+	ES_DEGAMMA_BT709 = 1,
+	ES_DEGAMMA_BT2020 = 2,
+};
+
+enum drm_es_sync_dc_mode {
+	ES_SINGLE_DC = 0,
+	ES_MULTI_DC_PRIMARY = 1,
+	ES_MULTI_DC_SECONDARY = 2,
+};
+
+enum drm_es_mmu_prefetch_mode {
+	ES_MMU_PREFETCH_DISABLE = 0,
+	ES_MMU_PREFETCH_ENABLE = 1,
+};
+
+#endif /* __ES_DRM_H__ */
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/es_drv.c b/drivers/gpu/drm/eswin/es_drv.c
new file mode 100644
index 000000000000..d29c3572857c
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_drv.c
@@ -0,0 +1,593 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/of_graph.h>
+#include <linux/component.h>
+#include <linux/iommu.h>
+#include <linux/version.h>
+#include <linux/of_address.h>
+#include <linux/dma-map-ops.h>
+#include <linux/of.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_of.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_debugfs.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_fbdev_generic.h>
+
+#include "es_drv.h"
+#include "es_fb.h"
+#include "es_gem.h"
+#include "es_plane.h"
+#include "es_crtc.h"
+#include "es_simple_enc.h"
+#include "es_dc.h"
+#include "es_virtual.h"
+#ifdef CONFIG_ESWIN_DW_HDMI
+#include "dw-hdmi.h"
+#endif
+
+#include "es_mipi_dsi.h"
+
+#define DRV_NAME "es_drm"
+#define DRV_DESC "Eswin DRM driver"
+#define DRV_DATE "20191101"
+#define DRV_MAJOR 1
+#define DRV_MINOR 0
+
+static bool has_iommu = true;
+static struct platform_driver es_drm_platform_driver;
+
+static const struct file_operations fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+	.compat_ioctl = drm_compat_ioctl,
+	.poll = drm_poll,
+	.read = drm_read,
+	.mmap = es_gem_mmap,
+};
+
+#ifdef CONFIG_DEBUG_FS
+static int es_debugfs_planes_show(struct seq_file *s, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *)s->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_plane *plane = NULL;
+
+	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+		struct drm_plane_state *state = plane->state;
+		struct es_plane_state *plane_state = to_es_plane_state(state);
+
+		seq_printf(s, "plane[%u]: %s\n", plane->base.id, plane->name);
+		seq_printf(s, "\tcrtc = %s\n",
+			   state->crtc ? state->crtc->name : "(null)");
+		seq_printf(s, "\tcrtc id = %u\n",
+			   state->crtc ? state->crtc->base.id : 0);
+		seq_printf(s, "\tcrtc-pos = " DRM_RECT_FMT "\n",
+			   DRM_RECT_ARG(&plane_state->status.dest));
+		seq_printf(s, "\tsrc-pos = " DRM_RECT_FP_FMT "\n",
+			   DRM_RECT_FP_ARG(&plane_state->status.src));
+		seq_printf(s, "\tformat = %p4cc\n",
+			   state->fb ? &state->fb->format->format : NULL);
+		seq_printf(s, "\trotation = 0x%x\n", state->rotation);
+		seq_printf(s, "\ttiling = %u\n", plane_state->status.tile_mode);
+
+		seq_puts(s, "\n");
+	}
+
+	return 0;
+}
+
+static struct drm_info_list es_debugfs_list[] = {
+	{ "planes", es_debugfs_planes_show, 0, NULL },
+};
+
+static void es_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(es_debugfs_list, ARRAY_SIZE(es_debugfs_list),
+				 minor->debugfs_root, minor);
+}
+#endif
+
+static struct drm_driver es_drm_driver = {
+	.driver_features = DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_GEM |
+			   DRIVER_SYNCOBJ,
+	.lastclose = drm_fb_helper_lastclose,
+	.gem_prime_import = es_gem_prime_import,
+	.gem_prime_import_sg_table = es_gem_prime_import_sg_table,
+	.dumb_create = es_gem_dumb_create,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init = es_debugfs_init,
+#endif
+	.fops = &fops,
+	.name = DRV_NAME,
+	.desc = DRV_DESC,
+	.date = DRV_DATE,
+	.major = DRV_MAJOR,
+	.minor = DRV_MINOR,
+};
+
+int es_drm_iommu_attach_device(struct drm_device *drm_dev, struct device *dev)
+{
+	struct es_drm_private *priv = drm_dev->dev_private;
+	int ret;
+
+	if (!has_iommu)
+		return 0;
+
+	if (!priv->domain) {
+		priv->domain = iommu_get_domain_for_dev(dev);
+		if (IS_ERR(priv->domain))
+			return PTR_ERR(priv->domain);
+		priv->dma_dev = dev;
+	}
+
+	ret = iommu_attach_device(priv->domain, dev);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "Failed to attach iommu device\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+void es_drm_iommu_detach_device(struct drm_device *drm_dev, struct device *dev)
+{
+	struct es_drm_private *priv = drm_dev->dev_private;
+
+	if (!has_iommu)
+		return;
+
+	iommu_detach_device(priv->domain, dev);
+
+	if (priv->dma_dev == dev)
+		priv->dma_dev = drm_dev->dev;
+}
+
+void es_drm_update_pitch_alignment(struct drm_device *drm_dev,
+				   unsigned int alignment)
+{
+	struct es_drm_private *priv = drm_dev->dev_private;
+
+	if (alignment > priv->pitch_alignment)
+		priv->pitch_alignment = alignment;
+}
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+static int es_drm_create_properties(struct drm_device *dev)
+{
+	struct drm_property *prop;
+	struct es_drm_private *private = dev->dev_private;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC, "EOTF", 0,
+					 5);
+	if (!prop)
+		return -ENOMEM;
+	private->eotf_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "COLOR_SPACE", 0, 12);
+	if (!prop)
+		return -ENOMEM;
+	private->color_space_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "GLOBAL_ALPHA", 0, 255);
+	if (!prop)
+		return -ENOMEM;
+	private->global_alpha_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "BLEND_MODE", 0, 1);
+	if (!prop)
+		return -ENOMEM;
+	private->blend_mode_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "ALPHA_SCALE", 0, 1);
+	if (!prop)
+		return -ENOMEM;
+	private->alpha_scale_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "ASYNC_COMMIT", 0, 1);
+	if (!prop)
+		return -ENOMEM;
+	private->async_commit_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC, "SHARE_ID",
+					 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	private->share_id_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "CONNECTOR_ID", 0, 0xf);
+	if (!prop)
+		return -ENOMEM;
+	private->connector_id_prop = prop;
+
+	return drm_mode_create_tv_properties(dev, 0);
+}
+#endif
+
+static int es_drm_bind(struct device *dev)
+{
+	struct drm_device *drm_dev;
+	struct es_drm_private *priv;
+	int ret, id;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
+	static u64 dma_mask = DMA_BIT_MASK(40);
+#else
+	static u64 dma_mask = DMA_40BIT_MASK;
+#endif
+	drm_dev = drm_dev_alloc(&es_drm_driver, dev);
+	if (IS_ERR(drm_dev))
+		return PTR_ERR(drm_dev);
+
+	dev_set_drvdata(dev, drm_dev);
+
+	priv = devm_kzalloc(drm_dev->dev, sizeof(struct es_drm_private),
+			    GFP_KERNEL);
+	if (!priv) {
+		ret = -ENOMEM;
+		goto err_put_dev;
+	}
+
+	priv->pitch_alignment = 64;
+	priv->dma_dev = drm_dev->dev;
+	priv->dma_dev->coherent_dma_mask = dma_mask;
+	dma_set_mask_and_coherent(priv->dma_dev, DMA_BIT_MASK(40));
+
+	drm_dev->dev_private = priv;
+
+	drm_mode_config_init(drm_dev);
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+	es_drm_create_properties(drm_dev);
+#endif
+
+	/* Now try and bind all our sub-components */
+	ret = component_bind_all(dev, drm_dev);
+	if (ret)
+		goto err_mode;
+
+	es_mode_config_init(drm_dev);
+
+	ret = drm_vblank_init(drm_dev, drm_dev->mode_config.num_crtc);
+	if (ret)
+		goto err_bind;
+
+	drm_mode_config_reset(drm_dev);
+
+	drm_kms_helper_poll_init(drm_dev);
+
+	ret = drm_dev_register(drm_dev, 0);
+	if (ret)
+		goto err_helper;
+
+	drm_fbdev_generic_setup(drm_dev, 32);
+
+	ret = of_property_read_u32(dev->of_node, "numa-node-id", &id);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "Failed to read index property, ret = %d\n",
+			      ret);
+		return ret;
+	}
+	DRM_INFO("drm dev is on die%d\n", id);
+	priv->die_id = id;
+	priv->mmu_constructed = false;
+
+	if (drm_dev->unique) {
+		sprintf(drm_dev->unique, "%d", id);
+	}
+	DRM_INFO("drm_dev name:%s\n", drm_dev->unique);
+	return 0;
+
+err_helper:
+	drm_kms_helper_poll_fini(drm_dev);
+err_bind:
+	component_unbind_all(drm_dev->dev, drm_dev);
+err_mode:
+	drm_mode_config_cleanup(drm_dev);
+	if (priv->domain)
+		iommu_domain_free(priv->domain);
+err_put_dev:
+	drm_dev->dev_private = NULL;
+	dev_set_drvdata(dev, NULL);
+	drm_dev_put(drm_dev);
+	return ret;
+}
+
+static void es_drm_unbind(struct device *dev)
+{
+	struct drm_device *drm_dev = dev_get_drvdata(dev);
+	struct es_drm_private *priv = drm_dev->dev_private;
+
+	drm_dev_unregister(drm_dev);
+	drm_kms_helper_poll_fini(drm_dev);
+	drm_atomic_helper_shutdown(drm_dev);
+	component_unbind_all(drm_dev->dev, drm_dev);
+	drm_mode_config_cleanup(drm_dev);
+
+	if (priv->domain) {
+		iommu_domain_free(priv->domain);
+		priv->domain = NULL;
+	}
+
+	drm_dev->dev_private = NULL;
+	dev_set_drvdata(dev, NULL);
+	drm_dev_put(drm_dev);
+}
+
+static const struct component_master_ops es_drm_ops = {
+	.bind = es_drm_bind,
+	.unbind = es_drm_unbind,
+};
+
+static int compare_of(struct device *dev, void *data)
+{
+	// DRM_INFO("Comparing of node %pOF with %pOF\n", dev->of_node, data);
+	return dev->of_node == data;
+}
+
+static int es_drm_of_component_probe(struct device *dev,
+				     int (*compare_of)(struct device *, void *),
+				     const struct component_master_ops *m_ops)
+{
+	struct device_node *ep, *port, *remote;
+	struct component_match *match = NULL;
+	int i;
+	bool found = false;
+	bool matched = false;
+	int ret;
+
+	if (!dev->of_node)
+		return -EINVAL;
+
+	ret = of_reserved_mem_device_init(dev);
+	if (ret) {
+		dev_info(dev, "No memory-region specified, use system cma, ret:%d\n", ret);
+	}
+
+	/*
+	 * Bind the crtc's ports first, so that drm_of_find_possible_crtcs()
+	 * called from encoder's .bind callbacks works as expected
+	 */
+	for (i = 0;; i++) {
+		struct device_node *iommu;
+		port = of_parse_phandle(dev->of_node, "ports", i);
+		if (!port)
+			break;
+
+		if (of_device_is_available(port->parent)) {
+			drm_of_component_match_add(dev, &match, compare_of,
+						   port->parent);
+		}
+
+		iommu = of_parse_phandle(port->parent, "iommus", 0);
+
+		/*
+         * if there is a crtc not support iommu, force set all
+         * crtc use non-iommu buffer.
+         */
+		if (!iommu || !of_device_is_available(iommu->parent))
+			has_iommu = false;
+
+		found = true;
+
+		of_node_put(iommu);
+		of_node_put(port);
+	}
+
+	if (!found) {
+		DRM_DEV_ERROR(dev, "No available DC found.\n");
+		return -ENODEV;
+	}
+
+	if (i == 0) {
+		dev_err(dev, "missing 'ports' property\n");
+		return -ENODEV;
+	}
+
+	if (!match) {
+		dev_err(dev, "no available port\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * For bound crtcs, bind the encoders attached to their remote endpoint
+	 */
+	for (i = 0;; i++) {
+		port = of_parse_phandle(dev->of_node, "ports", i);
+		if (!port)
+			break;
+
+		if (!of_device_is_available(port->parent)) {
+			of_node_put(port);
+			continue;
+		}
+
+		for_each_child_of_node(port, ep) {
+			remote = of_graph_get_remote_port_parent(ep);
+			if (!remote || !of_device_is_available(remote)) {
+				of_node_put(remote);
+				continue;
+			} else if (!of_device_is_available(remote->parent)) {
+				dev_warn(
+					dev,
+					"parent device of %pOF is not available\n",
+					remote);
+				of_node_put(remote);
+				continue;
+			}
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+			if (!strcmp(remote->name, "hdmi")) {
+				matched = true;
+			}
+#endif
+
+#ifdef CONFIG_ESWIN_VIRTUAL_DISPLAY
+			if (!strcmp(remote->name, "es_wb")) {
+				matched = true;
+			}
+#endif
+
+#ifdef CONFIG_ESWIN_MIPI_DSI
+			if (!strcmp(remote->name, "mipi_dsi")) {
+				matched = true;
+			}
+#endif
+			if (matched == true) {
+				drm_of_component_match_add(dev, &match,
+							   compare_of, remote);
+				matched = false;
+				dev_dbg(dev, "matched: %pOF, remote->name:%s\n",
+					remote, remote->name);
+			}
+
+			of_node_put(remote);
+		}
+		of_node_put(port);
+	}
+	return component_master_add_with_match(dev, m_ops, match);
+}
+
+static int es_drm_platform_probe(struct platform_device *pdev)
+{
+	DRM_INFO("drm platform probe enter\n");
+
+	return es_drm_of_component_probe(&pdev->dev, compare_of, &es_drm_ops);
+}
+
+static int es_drm_platform_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &es_drm_ops);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int es_drm_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int es_drm_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_resume(drm);
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(es_drm_pm_ops, es_drm_suspend, es_drm_resume);
+
+static const struct of_device_id es_drm_dt_ids[] = {
+
+	{
+		.compatible = "eswin,display-subsystem",
+	},
+
+	{ /* sentinel */ },
+
+};
+
+MODULE_DEVICE_TABLE(of, es_drm_dt_ids);
+
+static struct platform_driver es_drm_platform_driver = {
+    .probe = es_drm_platform_probe,
+    .remove = es_drm_platform_remove,
+
+    .driver = {
+        .name = DRV_NAME,
+        .of_match_table = es_drm_dt_ids,
+        .pm = &es_drm_pm_ops,
+    },
+};
+
+static struct platform_driver *const drivers[] = {
+	&es_drm_platform_driver,
+	/* put display control driver at start */
+	&dc_platform_driver,
+
+/* connector */
+
+/* bridge */
+#if 1
+#ifdef CONFIG_ESWIN_DW_HDMI
+	&dw_hdmi_eswin_pltfm_driver,
+#endif
+#ifdef CONFIG_DW_HDMI_I2S_AUDIO
+	&snd_dw_hdmi_driver,
+#endif
+
+#ifdef CONFIG_DW_HDMI_CEC
+	&dw_hdmi_cec_driver,
+#endif
+
+#ifdef CONFIG_DW_HDMI_HDCP
+	&dw_hdmi_hdcp_driver,
+#endif
+#endif
+
+#ifdef CONFIG_ESWIN_VIRTUAL_DISPLAY
+	&virtual_display_platform_driver,
+#endif
+
+#ifdef CONFIG_ESWIN_MIPI_DSI
+	&es_mipi_dsi_driver,
+#endif
+};
+
+static int __init es_drm_init(void)
+{
+	DRM_INFO("drm init enter\n");
+
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
+}
+
+static void __exit es_drm_fini(void)
+{
+	DRM_INFO("drm exit enter\n");
+
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
+}
+
+module_init(es_drm_init);
+module_exit(es_drm_fini);
+
+MODULE_DESCRIPTION("Eswin DRM Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_drv.h b/drivers/gpu/drm/eswin/es_drv.h
new file mode 100644
index 000000000000..b6ef6cf9548e
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_drv.h
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_DRV_H__
+#define __ES_DRV_H__
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include <drm/drm_gem.h>
+#include <drm/drm_device.h>
+
+#include "es_plane.h"
+#ifdef CONFIG_ESWIN_MMU
+#include "es_dc_mmu.h"
+#endif
+
+/*
+ *
+ * @dma_dev: device for DMA API.
+ *  - use the first attached device if support iommu
+    else use drm device (only contiguous buffer support)
+ * @domain: iommu domain for DRM.
+ *  - all DC IOMMU share same domain to reduce mapping
+ * @pitch_alignment: buffer pitch alignment required by sub-devices.
+ *
+ */
+struct es_drm_private {
+	struct device *dma_dev;
+	struct iommu_domain *domain;
+#ifdef CONFIG_ESWIN_DW_HDMI
+	struct drm_property *connector_id_prop;
+	struct drm_property *eotf_prop;
+	struct drm_property *color_space_prop;
+	struct drm_property *global_alpha_prop;
+	struct drm_property *blend_mode_prop;
+	struct drm_property *alpha_scale_prop;
+	struct drm_property *async_commit_prop;
+	struct drm_property *share_id_prop;
+#endif
+
+#ifdef CONFIG_ESWIN_MMU
+	dc_mmu *mmu;
+	bool mmu_constructed;
+#endif
+	unsigned int die_id;
+
+	unsigned int pitch_alignment;
+};
+
+int es_drm_iommu_attach_device(struct drm_device *drm_dev, struct device *dev);
+
+void es_drm_iommu_detach_device(struct drm_device *drm_dev, struct device *dev);
+
+void es_drm_update_pitch_alignment(struct drm_device *drm_dev,
+				   unsigned int alignment);
+
+static inline struct device *to_dma_dev(struct drm_device *dev)
+{
+	struct es_drm_private *priv = dev->dev_private;
+
+	return priv->dma_dev;
+}
+
+static inline bool is_iommu_enabled(struct drm_device *dev)
+{
+	struct es_drm_private *priv = dev->dev_private;
+
+	return priv->domain != NULL ? true : false;
+}
+#endif /* __ES_DRV_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dw_hdmi.c b/drivers/gpu/drm/eswin/es_dw_hdmi.c
new file mode 100644
index 000000000000..99bc87ad8e23
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dw_hdmi.c
@@ -0,0 +1,1019 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: DengLei <denglei@eswincomputing.com>
+ */
+
+#include <linux/component.h>
+#include <linux/clk.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+
+#include <drm/drm_mode_config.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_of.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/bridge/dw_hdmi.h>
+
+#include <drm/drm_mode.h>
+#include <linux/videodev2.h>
+#include <linux/media-bus-format.h>
+#include <linux/pm_runtime.h>
+
+#include "dw-hdmi.h"
+#include "es_drv.h"
+#include "es_crtc.h"
+
+#define HIWORD_UPDATE(val, mask) (val | (mask) << 16)
+#define ESWIN_HDMI_COLORIMETRY_BT2020                                          \
+	(HDMI_COLORIMETRY_EXTENDED + HDMI_EXTENDED_COLORIMETRY_BT2020)
+
+/* HDMI output pixel format */
+enum drm_hdmi_output_type {
+	DRM_HDMI_OUTPUT_DEFAULT_RGB, /* default RGB */
+	DRM_HDMI_OUTPUT_YCBCR444, /* YCBCR 444 */
+	DRM_HDMI_OUTPUT_YCBCR422, /* YCBCR 422 */
+	DRM_HDMI_OUTPUT_YCBCR420, /* YCBCR 420 */
+	DRM_HDMI_OUTPUT_YCBCR_HQ, /* Highest subsampled YUV */
+	DRM_HDMI_OUTPUT_YCBCR_LQ, /* Lowest subsampled YUV */
+	DRM_HDMI_OUTPUT_INVALID, /* Guess what ? */
+};
+
+enum dw_hdmi_eswin_color_depth {
+	ESWIN_HDMI_DEPTH_8,
+	ESWIN_HDMI_DEPTH_10,
+	ESWIN_HDMI_DEPTH_12,
+	ESWIN_HDMI_DEPTH_16,
+	ESWIN_HDMI_DEPTH_420_10,
+	ESWIN_HDMI_DEPTH_420_12,
+	ESWIN_HDMI_DEPTH_420_16
+};
+
+struct eswin_hdmi {
+	struct device *dev;
+	struct regmap *regmap;
+	struct dw_hdmi *hdmi;
+	struct drm_encoder encoder;
+	u8 id;
+	unsigned long bus_format;
+	unsigned long output_bus_format;
+	unsigned long enc_out_encoding;
+
+	struct drm_property *color_depth_property;
+	struct drm_property *output_format_property;
+	struct drm_property *colorimetry_property;
+	struct drm_property *video_enable_property;
+
+	struct drm_property *color_depth_capacity;
+	struct drm_property *output_format_capacity;
+	struct drm_property *is_hdmi_capacity;
+	struct drm_property *width_heigth_capacity;
+	struct drm_property *quant_range_select_capacity;
+	struct drm_property *max_tmds_clock_capacity;
+
+	unsigned int colordepth;
+	unsigned int colorimetry;
+	unsigned int phy_bus_width;
+	unsigned int hdmi_quant_range;
+	enum drm_hdmi_output_type hdmi_output;
+	bool video_enable;
+};
+
+#define to_eswin_hdmi(x) container_of(x, struct eswin_hdmi, x)
+
+static const struct dw_hdmi_mpll_config eswin_mpll_cfg[] = {
+	{
+		27000000,
+		{
+			{ 0x0003, 0x0628 },
+			{ 0x1003, 0x0632 },
+			{ 0x2003, 0x023c },
+		},
+	},
+	{
+		54000000,
+		{
+			{ 0x0002, 0x0614 },
+			{ 0x1002, 0x0619 },
+			{ 0x2002, 0x021e },
+		},
+	},
+	{
+		74250000,
+		{
+			{ 0x0002, 0x0214 },
+			{ 0x1009, 0x0619 },
+			{ 0x2001, 0x060f },
+		},
+	},
+	{
+		108000000,
+		{
+			{ 0x0001, 0x060a },
+			{ 0x1009, 0x0619 },
+			{ 0x2001, 0x020f },
+		},
+	},
+	{
+		148500000,
+		{
+			{ 0x0001, 0x020a },
+			{ 0x1018, 0x0619 },
+			{ 0x2008, 0x060f },
+		},
+	},
+	{
+		235690000,
+		{
+			{ 0x0000, 0x0605 },
+			{ 0x1018, 0x0219 },
+			{ 0x2648, 0x020f },
+		},
+	},
+	{
+		297000000,
+		{
+			{ 0x0000, 0x0205 },
+			{ 0x1658, 0x0219 },
+			{ 0x2648, 0x020f },
+		},
+	},
+	{
+		371370000,
+		{
+			{ 0x0640, 0x0205 },
+			{ 0x1658, 0x0019 },
+			{ 0x2648, 0x000f },
+		},
+	},
+	{
+		513820000,
+		{
+			{ 0x0640, 0x0005 },
+			{ 0x1658, 0x0019 },
+			{ 0x2648, 0x000f },
+		},
+	},
+	{
+		594000000,
+		{
+			{ 0x0640, 0x0005 },
+			{ 0x1658, 0x0019 },
+			{ 0x2648, 0x000f },
+		},
+	},
+	{
+		~0UL,
+		{
+			{ 0x0000, 0x0000 },
+			{ 0x0000, 0x0000 },
+			{ 0x0000, 0x0000 },
+		},
+	}
+};
+
+static const struct dw_hdmi_curr_ctrl eswin_cur_ctr[] = {
+	/*      pixelclk    bpp8    bpp10   bpp12 */
+	{
+		27000000,
+		{ 0x0283, 0x0281, 0x02c2 },
+	},
+	{
+		54000000,
+		{ 0x1183, 0x1203, 0x1202 },
+	},
+	{
+		74250000,
+		{ 0x1142, 0x2203, 0x2141 },
+	},
+	{
+		108000000,
+		{ 0x20c0, 0x2203, 0x2100 },
+	},
+	{
+		148500000,
+		{ 0x2080, 0x3203, 0x3141 },
+	},
+	{
+		235690000,
+		{ 0x3040, 0x3182, 0x3100 },
+	},
+	{
+		297000000,
+		{ 0x3041, 0x3182, 0x3100 },
+	},
+	{
+		371370000,
+		{ 0x3041, 0x31c0, 0x3100 },
+	},
+	{
+		513820000,
+		{ 0x3080, 0x31c0, 0x3100 },
+	},
+	{
+		594000000,
+		{ 0x3080, 0x31c0, 0x3100 },
+	},
+	{
+		~0UL,
+		{ 0x0000, 0x0000, 0x0000 },
+	}
+};
+
+static struct dw_hdmi_phy_config eswin_phy_config[] = {
+	/*pixelclk   symbol   term   vlev*/
+	{ 165000000, 0x8088, 0x0007, 0x0180 },
+	{ 297000000, 0x80c8, 0x0004, 0x0180 },
+	{ 594000000, 0x80f3, 0x0000, 0x0180 },
+	{ ~0UL, 0x0000, 0x0000, 0x0000 }
+};
+
+static enum drm_mode_status
+dw_hdmi_eswin_mode_valid(struct dw_hdmi *hdmi, void *data,
+			 const struct drm_display_info *info,
+			 const struct drm_display_mode *mode)
+{
+	const struct dw_hdmi_mpll_config *mpll_cfg = eswin_mpll_cfg;
+	int pclk = mode->clock * 1000;
+	bool valid = false;
+	int i;
+
+	for (i = 0; mpll_cfg[i].mpixelclock != (~0UL); i++) {
+		if (pclk <= mpll_cfg[i].mpixelclock) {
+			valid = true;
+			break;
+		}
+	}
+
+	return (valid) ? MODE_OK : MODE_BAD;
+}
+
+static bool
+dw_hdmi_eswin_encoder_mode_fixup(struct drm_encoder *encoder,
+				 const struct drm_display_mode *mode,
+				 struct drm_display_mode *adj_mode)
+{
+	return true;
+}
+
+static void dw_hdmi_eswin_select_output(struct drm_connector_state *conn_state,
+					struct drm_crtc_state *crtc_state,
+					struct eswin_hdmi *hdmi,
+					unsigned int *color_format,
+					unsigned int *color_depth,
+					unsigned long *enc_out_encoding,
+					unsigned int *eotf)
+{
+	struct drm_display_info *info = &conn_state->connector->display_info;
+	struct drm_display_mode *mode = &crtc_state->mode;
+	struct hdr_output_metadata *hdr_metadata;
+	u32 vic = drm_match_cea_mode(mode);
+	unsigned long tmdsclock, pixclock = mode->crtc_clock;
+	bool support_dc = false;
+	int max_tmds_clock = info->max_tmds_clock;
+	int output_eotf;
+
+	*color_format = DRM_HDMI_OUTPUT_DEFAULT_RGB;
+
+	switch (hdmi->hdmi_output) {
+	case DRM_HDMI_OUTPUT_YCBCR_HQ:
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR444;
+		else if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+		else if (conn_state->connector->ycbcr_420_allowed &&
+			 drm_mode_is_420(info, mode))
+			*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR_LQ:
+		if (conn_state->connector->ycbcr_420_allowed &&
+		    drm_mode_is_420(info, mode))
+			*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		else if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+		else if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR444;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR420:
+		if (conn_state->connector->ycbcr_420_allowed &&
+		    drm_mode_is_420(info, mode))
+			*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR422:
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR444:
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR444;
+		break;
+	case DRM_HDMI_OUTPUT_DEFAULT_RGB:
+	default:
+		break;
+	}
+
+	if (*color_format == DRM_HDMI_OUTPUT_DEFAULT_RGB &&
+	    info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30)
+		support_dc = true;
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR444 &&
+	    info->edid_hdmi_rgb444_dc_modes &
+		    (DRM_EDID_HDMI_DC_Y444 | DRM_EDID_HDMI_DC_30))
+		support_dc = true;
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR422)
+		support_dc = true;
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR420 &&
+	    info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30)
+		support_dc = true;
+
+	if (hdmi->colordepth > 8 && support_dc)
+		*color_depth = 10;
+	else
+		*color_depth = 8;
+
+	*eotf = TRADITIONAL_GAMMA_SDR;
+	if (conn_state->hdr_output_metadata) {
+		hdr_metadata = (struct hdr_output_metadata *)
+				       conn_state->hdr_output_metadata->data;
+		output_eotf = hdr_metadata->hdmi_metadata_type1.eotf;
+		if (output_eotf > TRADITIONAL_GAMMA_HDR &&
+		    output_eotf < FUTURE_EOTF)
+			*eotf = output_eotf;
+	}
+
+	if ((*eotf > TRADITIONAL_GAMMA_HDR &&
+	     conn_state->connector->hdr_sink_metadata.hdmi_type1.eotf &
+		     BIT(*eotf)) ||
+	    (hdmi->colorimetry == ESWIN_HDMI_COLORIMETRY_BT2020))
+		*enc_out_encoding = V4L2_YCBCR_ENC_BT2020;
+	else if ((vic == 6) || (vic == 7) || (vic == 21) || (vic == 22) ||
+		 (vic == 2) || (vic == 3) || (vic == 17) || (vic == 18))
+		*enc_out_encoding = V4L2_YCBCR_ENC_601;
+	else
+		*enc_out_encoding = V4L2_YCBCR_ENC_709;
+
+	if (*enc_out_encoding == V4L2_YCBCR_ENC_BT2020) {
+		/* BT2020 require color depth at lest 10bit */
+		*color_depth = 10;
+		/* We prefer use YCbCr422 to send 10bit */
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+	}
+
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+		pixclock *= 2;
+	if ((mode->flags & DRM_MODE_FLAG_3D_MASK) ==
+	    DRM_MODE_FLAG_3D_FRAME_PACKING)
+		pixclock *= 2;
+
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR422 || *color_depth == 8)
+		tmdsclock = pixclock;
+	else
+		tmdsclock = pixclock * (*color_depth) / 8;
+
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR420)
+		tmdsclock /= 2;
+
+	/* XXX: max_tmds_clock of some sink is 0, we think it is 340MHz. */
+	if (!max_tmds_clock)
+		max_tmds_clock = 340000;
+
+	max_tmds_clock = min(max_tmds_clock, 594000);
+
+	if (tmdsclock > max_tmds_clock) {
+		if (max_tmds_clock >= 594000) {
+			*color_depth = 8;
+		} else if (max_tmds_clock > 340000) {
+			if (drm_mode_is_420(info, mode) || tmdsclock >= 594000)
+				*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		} else {
+			*color_depth = 8;
+			if (drm_mode_is_420(info, mode) || tmdsclock >= 594000)
+				*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		}
+	}
+}
+
+static int
+dw_hdmi_eswin_encoder_atomic_check(struct drm_encoder *encoder,
+				   struct drm_crtc_state *crtc_state,
+				   struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *s = to_es_crtc_state(crtc_state);
+	struct eswin_hdmi *hdmi = to_eswin_hdmi(encoder);
+	unsigned int colordepth, colorformat, bus_width, eotf;
+
+	dw_hdmi_eswin_select_output(conn_state, crtc_state, hdmi, &colorformat,
+				    &colordepth, &hdmi->enc_out_encoding,
+				    &eotf);
+	if (colordepth > 8)
+		hdmi->bus_format = MEDIA_BUS_FMT_RGB101010_1X30;
+	else
+		hdmi->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	/* DC does not support YUV output */
+	s->output_fmt = hdmi->bus_format;
+
+	if (colorformat == DRM_HDMI_OUTPUT_YCBCR420) {
+		if (colordepth > 8)
+			hdmi->output_bus_format = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+		else
+			hdmi->output_bus_format = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+
+		bus_width = colordepth / 2;
+	} else {
+		if ((colordepth > 8) &&
+		    (colorformat != DRM_HDMI_OUTPUT_YCBCR422)) {
+			if (colorformat != DRM_HDMI_OUTPUT_DEFAULT_RGB)
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_YUV10_1X30;
+			else
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_RGB101010_1X30;
+		} else {
+			if (colorformat != DRM_HDMI_OUTPUT_DEFAULT_RGB)
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_YUV8_1X24;
+			else
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_RGB888_1X24;
+		}
+		if (colorformat == DRM_HDMI_OUTPUT_YCBCR422) {
+			bus_width = 8;
+			if (colordepth > 8)
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_UYVY10_1X20;
+			else
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_UYVY8_1X16;
+		} else {
+			bus_width = colordepth;
+		}
+	}
+
+	hdmi->phy_bus_width = bus_width;
+
+	s->encoder_type = DRM_MODE_ENCODER_TMDS;
+
+	return 0;
+}
+
+static unsigned long dw_hdmi_eswin_get_input_bus_format(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->bus_format;
+}
+
+static unsigned long dw_hdmi_eswin_get_output_bus_format(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->output_bus_format;
+}
+
+static unsigned long dw_hdmi_eswin_get_enc_in_encoding(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->enc_out_encoding;
+}
+
+static unsigned long dw_hdmi_eswin_get_enc_out_encoding(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->enc_out_encoding;
+}
+
+static const struct drm_prop_enum_list color_depth_enum_list[] = {
+	{ 0, "Automatic" }, /* Same as 24bit */
+	{ 8, "24bit" },
+	{ 10, "30bit" },
+};
+
+static const struct drm_prop_enum_list drm_hdmi_output_enum_list[] = {
+	{ DRM_HDMI_OUTPUT_DEFAULT_RGB, "output_rgb" },
+	{ DRM_HDMI_OUTPUT_YCBCR444, "output_ycbcr444" },
+	{ DRM_HDMI_OUTPUT_YCBCR422, "output_ycbcr422" },
+	{ DRM_HDMI_OUTPUT_YCBCR420, "output_ycbcr420" },
+	{ DRM_HDMI_OUTPUT_YCBCR_HQ, "output_ycbcr_high_subsampling" },
+	{ DRM_HDMI_OUTPUT_YCBCR_LQ, "output_ycbcr_low_subsampling" },
+	{ DRM_HDMI_OUTPUT_INVALID, "invalid_output" },
+};
+
+static const struct drm_prop_enum_list colorimetry_enum_list[] = {
+	{ HDMI_COLORIMETRY_NONE, "None" },
+	{ ESWIN_HDMI_COLORIMETRY_BT2020, "ITU_2020" },
+};
+
+static const struct drm_prop_enum_list quant_range_enum_list[] = {
+	{ HDMI_QUANTIZATION_RANGE_DEFAULT, "default" },
+	{ HDMI_QUANTIZATION_RANGE_LIMITED, "limit" },
+	{ HDMI_QUANTIZATION_RANGE_FULL, "full" },
+};
+
+static const struct drm_prop_enum_list color_depth_capacity_list[] = {
+	{ BIT(ESWIN_HDMI_DEPTH_8), "8bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_10), "10bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_12), "12bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_16), "16bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_420_10), "yuv420_10bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_420_12), "yuv420_12bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_420_16), "yuv420_16bit" },
+};
+
+static const struct drm_prop_enum_list output_format_capacity_list[] = {
+	{ BIT(DRM_HDMI_OUTPUT_DEFAULT_RGB), "rgb" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR444), "yuv444" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR422), "yuv422" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR420), "yuv420" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR_HQ), "yuv_hq" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR_LQ), "yuv_lq" },
+};
+
+static void dw_hdmi_eswin_attatch_properties(struct drm_connector *connector,
+					     unsigned int color, int version,
+					     void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+	struct drm_property *prop;
+#ifdef CONFIG_ESWIN_DW_HDMI
+	struct es_drm_private *private = connector->dev->dev_private;
+#endif
+	switch (color) {
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_DEFAULT_RGB;
+		hdmi->colordepth = 10;
+		break;
+	case MEDIA_BUS_FMT_YUV8_1X24:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR444;
+		hdmi->colordepth = 8;
+		break;
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR444;
+		hdmi->colordepth = 10;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR422;
+		hdmi->colordepth = 10;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR422;
+		hdmi->colordepth = 8;
+		break;
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR420;
+		hdmi->colordepth = 8;
+		break;
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR420;
+		hdmi->colordepth = 10;
+		break;
+	default:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_DEFAULT_RGB;
+		hdmi->colordepth = 8;
+	}
+
+	if (!hdmi->color_depth_property) {
+		prop = drm_property_create_enum(
+			connector->dev, 0, "hdmi_output_color_depth",
+			color_depth_enum_list,
+			ARRAY_SIZE(color_depth_enum_list));
+		if (prop) {
+			hdmi->color_depth_property = prop;
+			drm_object_attach_property(&connector->base, prop, 0);
+		}
+	}
+
+	prop = drm_property_create_enum(connector->dev, 0, "hdmi_output_format",
+					drm_hdmi_output_enum_list,
+					ARRAY_SIZE(drm_hdmi_output_enum_list));
+	if (prop) {
+		hdmi->output_format_property = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_enum(connector->dev, 0,
+					"hdmi_output_colorimetry",
+					colorimetry_enum_list,
+					ARRAY_SIZE(colorimetry_enum_list));
+	if (prop) {
+		hdmi->colorimetry_property = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_bool(connector->dev, 0, "video_enable");
+	if (prop) {
+		hdmi->video_enable_property = prop;
+		drm_object_attach_property(&connector->base, prop, 1);
+		hdmi->video_enable = true;
+	}
+
+	prop = drm_property_create_enum(connector->dev, 0,
+					"hdmi_color_depth_capacity",
+					color_depth_capacity_list,
+					ARRAY_SIZE(color_depth_capacity_list));
+	if (prop) {
+		hdmi->color_depth_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_enum(
+		connector->dev, 0, "hdmi_output_format_capacity",
+		output_format_capacity_list,
+		ARRAY_SIZE(output_format_capacity_list));
+	if (prop) {
+		hdmi->output_format_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_bool(connector->dev, 0, "is_hdmi_capacity");
+	if (prop) {
+		hdmi->is_hdmi_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_range(
+		connector->dev, 0, "hdmi_width_height_mm_capacity", 0, 0xff);
+	if (prop) {
+		hdmi->width_heigth_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_bool(connector->dev, 0,
+					"hdmi_quant_range_sel_capacity");
+	if (prop) {
+		hdmi->quant_range_select_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_range(
+		connector->dev, 0, "hdmi_max_tmds_clock_capacity", 0, 340000);
+	if (prop) {
+		hdmi->max_tmds_clock_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = connector->dev->mode_config.hdr_output_metadata_property;
+	if (version >= 0x211a)
+		drm_object_attach_property(&connector->base, prop, 0);
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+	drm_object_attach_property(&connector->base, private->connector_id_prop,
+				   0);
+#endif
+}
+
+static void dw_hdmi_eswin_destroy_properties(struct drm_connector *connector,
+					     void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	if (hdmi->color_depth_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->color_depth_property);
+		hdmi->color_depth_property = NULL;
+	}
+
+	if (hdmi->output_format_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->output_format_property);
+		hdmi->output_format_property = NULL;
+	}
+
+	if (hdmi->colorimetry_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->colorimetry_property);
+		hdmi->colorimetry_property = NULL;
+	}
+
+	if (hdmi->video_enable_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->video_enable_property);
+		hdmi->video_enable_property = NULL;
+	}
+
+	if (hdmi->color_depth_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->color_depth_capacity);
+		hdmi->color_depth_capacity = NULL;
+	}
+
+	if (hdmi->output_format_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->output_format_capacity);
+		hdmi->output_format_capacity = NULL;
+	}
+
+	if (hdmi->is_hdmi_capacity) {
+		drm_property_destroy(connector->dev, hdmi->is_hdmi_capacity);
+		hdmi->is_hdmi_capacity = NULL;
+	}
+
+	if (hdmi->width_heigth_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->width_heigth_capacity);
+		hdmi->width_heigth_capacity = NULL;
+	}
+
+	if (hdmi->quant_range_select_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->quant_range_select_capacity);
+		hdmi->quant_range_select_capacity = NULL;
+	}
+
+	if (hdmi->max_tmds_clock_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->max_tmds_clock_capacity);
+		hdmi->max_tmds_clock_capacity = NULL;
+	}
+}
+
+static int dw_hdmi_eswin_set_property(struct drm_connector *connector,
+				      struct drm_connector_state *state,
+				      struct drm_property *property,
+				      uint64_t val, void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	if (property == hdmi->color_depth_property) {
+		hdmi->colordepth = val;
+	} else if (property == hdmi->output_format_property) {
+		hdmi->hdmi_output = val;
+	} else if (property == hdmi->colorimetry_property) {
+		hdmi->colorimetry = val;
+	} else if (property == hdmi->video_enable_property) {
+		if (hdmi->video_enable != val) {
+			if (val == true) {
+				dw_hdmi_enable_video(hdmi->hdmi);
+			} else {
+				dw_hdmi_disable_video(hdmi->hdmi);
+			}
+			hdmi->video_enable = val;
+		}
+	} else {
+		DRM_DEBUG("don't support set %s property\n", property->name);
+		return 0;
+	}
+	return 0;
+}
+
+static int dw_hdmi_eswin_get_property(struct drm_connector *connector,
+				      const struct drm_connector_state *state,
+				      struct drm_property *property,
+				      uint64_t *val, void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+	struct drm_display_info *info = &connector->display_info;
+	struct drm_mode_config *config = &connector->dev->mode_config;
+#ifdef CONFIG_ESWIN_DW_HDMI
+	struct es_drm_private *private = connector->dev->dev_private;
+#endif
+	if (property == hdmi->color_depth_property) {
+		*val = hdmi->colordepth;
+	} else if (property == hdmi->output_format_property) {
+		*val = hdmi->hdmi_output;
+	} else if (property == hdmi->color_depth_capacity) {
+		*val = BIT(ESWIN_HDMI_DEPTH_8);
+		if (info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30)
+			*val |= BIT(ESWIN_HDMI_DEPTH_10);
+		if (info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_36)
+			*val |= BIT(ESWIN_HDMI_DEPTH_12);
+		if (info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_48)
+			*val |= BIT(ESWIN_HDMI_DEPTH_16);
+		if (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30)
+			*val |= BIT(ESWIN_HDMI_DEPTH_420_10);
+		if (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36)
+			*val |= BIT(ESWIN_HDMI_DEPTH_420_12);
+		if (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48)
+			*val |= BIT(ESWIN_HDMI_DEPTH_420_16);
+	} else if (property == hdmi->output_format_capacity) {
+		*val = BIT(DRM_HDMI_OUTPUT_DEFAULT_RGB);
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*val |= BIT(DRM_HDMI_OUTPUT_YCBCR444);
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*val |= BIT(DRM_HDMI_OUTPUT_YCBCR422);
+		if (connector->ycbcr_420_allowed &&
+		    info->color_formats & DRM_COLOR_FORMAT_YCBCR420)
+			*val |= BIT(DRM_HDMI_OUTPUT_YCBCR420);
+	} else if (property == config->hdr_output_metadata_property) {
+		*val = state->hdr_output_metadata ?
+			       state->hdr_output_metadata->base.id :
+			       0;
+	} else if (property == hdmi->colorimetry_property) {
+		*val = hdmi->colorimetry;
+	}
+#ifdef CONFIG_ESWIN_DW_HDMI
+	else if (property == private->connector_id_prop) {
+		*val = hdmi->id;
+	}
+#endif
+	else if (property == hdmi->is_hdmi_capacity) {
+		*val = info->is_hdmi;
+	} else if (property == hdmi->quant_range_select_capacity) {
+		*val = info->rgb_quant_range_selectable;
+	} else if (property == hdmi->width_heigth_capacity) {
+		property->values[0] = info->width_mm;
+		property->values[1] = info->height_mm;
+		*val = 0;
+	} else if (property == hdmi->max_tmds_clock_capacity) {
+		*val = info->max_tmds_clock;
+	} else if (property == hdmi->video_enable_property) {
+		*val = hdmi->video_enable;
+	} else {
+		DRM_ERROR("failed to get eswin hdmi connector %s property\n",
+			  property->name);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static const struct dw_hdmi_property_ops dw_hdmi_eswin_property_ops = {
+	.attatch_properties = dw_hdmi_eswin_attatch_properties,
+	.destroy_properties = dw_hdmi_eswin_destroy_properties,
+	.set_property = dw_hdmi_eswin_set_property,
+	.get_property = dw_hdmi_eswin_get_property,
+};
+
+static const struct drm_encoder_helper_funcs
+	dw_hdmi_eswin_encoder_helper_funcs = {
+		.mode_fixup = dw_hdmi_eswin_encoder_mode_fixup,
+		.atomic_check = dw_hdmi_eswin_encoder_atomic_check,
+	};
+
+static const struct dw_hdmi_plat_data win2030_hdmi_drv_data = {
+	.mode_valid = dw_hdmi_eswin_mode_valid,
+	.mpll_cfg = eswin_mpll_cfg,
+	.cur_ctr = eswin_cur_ctr,
+	.phy_config = eswin_phy_config,
+	.use_drm_infoframe = true,
+	.ycbcr_420_allowed = false,
+};
+
+static const struct of_device_id dw_hdmi_eswin_dt_ids[] = {
+	{ .compatible = "eswin,eswin-dw-hdmi", .data = &win2030_hdmi_drv_data },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dw_hdmi_eswin_dt_ids);
+
+static int dw_hdmi_eswin_bind(struct device *dev, struct device *master,
+			      void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_hdmi_plat_data *plat_data;
+	const struct of_device_id *match;
+	struct drm_device *drm = data;
+	struct drm_encoder *encoder;
+	struct eswin_hdmi *hdmi;
+	int ret = 0;
+
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
+	hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return -ENOMEM;
+
+	match = of_match_node(dw_hdmi_eswin_dt_ids, pdev->dev.of_node);
+	plat_data = devm_kmemdup(&pdev->dev, match->data, sizeof(*plat_data),
+				 GFP_KERNEL);
+	if (!plat_data)
+		return -ENOMEM;
+
+	hdmi->dev = &pdev->dev;
+
+	plat_data->phy_data = hdmi;
+	encoder = &hdmi->encoder;
+
+	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
+	/*
+     * If we failed to find the CRTC(s) which this encoder is
+     * supposed to be connected to, it's because the CRTC has
+     * not been registered yet.  Defer probing, and hope that
+     * the required CRTC is added later.
+     */
+	if (encoder->possible_crtcs == 0)
+		return -EPROBE_DEFER;
+
+	plat_data->phy_data = hdmi;
+	plat_data->get_input_bus_format = dw_hdmi_eswin_get_input_bus_format;
+	plat_data->get_output_bus_format = dw_hdmi_eswin_get_output_bus_format;
+	plat_data->get_enc_in_encoding = dw_hdmi_eswin_get_enc_in_encoding;
+	plat_data->get_enc_out_encoding = dw_hdmi_eswin_get_enc_out_encoding;
+	plat_data->property_ops = &dw_hdmi_eswin_property_ops;
+
+	drm_encoder_helper_add(encoder, &dw_hdmi_eswin_encoder_helper_funcs);
+	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+
+	platform_set_drvdata(pdev, hdmi);
+
+	hdmi->hdmi = dw_hdmi_bind(pdev, encoder, plat_data);
+
+	/*
+     * If dw_hdmi_bind() fails we'll never call dw_hdmi_unbind(),
+     * which would have called the encoder cleanup.  Do it manually.
+     */
+	if (IS_ERR(hdmi->hdmi)) {
+		ret = PTR_ERR(hdmi->hdmi);
+		drm_encoder_cleanup(encoder);
+	}
+
+	return ret;
+}
+
+static void dw_hdmi_eswin_unbind(struct device *dev, struct device *master,
+				 void *data)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dw_hdmi_unbind(hdmi->hdmi);
+}
+
+static const struct component_ops dw_hdmi_eswin_ops = {
+	.bind = dw_hdmi_eswin_bind,
+	.unbind = dw_hdmi_eswin_unbind,
+};
+
+static int dw_hdmi_eswin_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dw_hdmi_eswin_ops);
+}
+
+static void dw_hdmi_eswin_shutdown(struct platform_device *pdev)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(&pdev->dev);
+
+	if(hdmi == NULL)
+	{
+		pr_err("no hdmi!!!\n");
+		return;
+	}
+	if(hdmi->hdmi == NULL)
+	{
+		pr_err("no hdmi->hdmi!!!\n");
+		return;
+	}
+	dw_hdmi_suspend(hdmi->hdmi);
+}
+
+static int dw_hdmi_eswin_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dw_hdmi_eswin_ops);
+	return 0;
+}
+
+static int __maybe_unused dw_hdmi_eswin_suspend(struct device *dev)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dw_hdmi_suspend(hdmi->hdmi);
+
+	return 0;
+}
+
+static int __maybe_unused dw_hdmi_eswin_resume(struct device *dev)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dw_hdmi_resume(hdmi->hdmi);
+
+	return 0;
+}
+
+static const struct dev_pm_ops dw_hdmi_eswin_pm = {	SET_SYSTEM_SLEEP_PM_OPS(
+	dw_hdmi_eswin_suspend, dw_hdmi_eswin_resume) };
+
+struct platform_driver dw_hdmi_eswin_pltfm_driver = {
+    .probe  = dw_hdmi_eswin_probe,
+    .remove = dw_hdmi_eswin_remove,
+    .shutdown = dw_hdmi_eswin_shutdown,
+    .driver = {
+        .name = "dw-hdmi-eswin",
+        .pm = &dw_hdmi_eswin_pm,
+        .of_match_table = dw_hdmi_eswin_dt_ids,
+    },
+};
+
+//module_platform_driver(dw_hdmi_eswin_pltfm_driver);
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/eswin/es_fb.c b/drivers/gpu/drm/eswin/es_fb.c
new file mode 100644
index 000000000000..7b566a379d13
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_fb.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/module.h>
+
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+
+#include "es_fb.h"
+#include "es_gem.h"
+
+static struct drm_framebuffer_funcs es_fb_funcs = {
+	.create_handle = drm_gem_fb_create_handle,
+	.destroy = drm_gem_fb_destroy,
+	.dirty = drm_atomic_helper_dirtyfb,
+};
+
+static struct drm_framebuffer *
+es_fb_alloc(struct drm_device *dev, const struct drm_mode_fb_cmd2 *mode_cmd,
+	    struct es_gem_object **obj, unsigned int num_planes)
+{
+	struct drm_framebuffer *fb;
+	int ret, i;
+
+	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
+	if (!fb)
+		return ERR_PTR(-ENOMEM);
+
+	drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd);
+
+	for (i = 0; i < num_planes; i++)
+		fb->obj[i] = &obj[i]->base;
+
+	ret = drm_framebuffer_init(dev, fb, &es_fb_funcs);
+	if (ret) {
+		dev_err(dev->dev, "Failed to initialize framebuffer: %d\n",
+			ret);
+		kfree(fb);
+		return ERR_PTR(ret);
+	}
+
+	return fb;
+}
+
+static struct drm_framebuffer *
+es_fb_create(struct drm_device *dev, struct drm_file *file_priv,
+	     const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_framebuffer *fb;
+	const struct drm_format_info *info;
+	struct es_gem_object *objs[MAX_NUM_PLANES];
+	struct drm_gem_object *obj;
+	unsigned int height, size;
+	unsigned char i, num_planes;
+	int ret = 0;
+
+	info = drm_format_info(mode_cmd->pixel_format);
+	if (!info)
+		return ERR_PTR(-EINVAL);
+
+	num_planes = info->num_planes;
+	if (num_planes > MAX_NUM_PLANES)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < num_planes; i++) {
+		obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[i]);
+		if (!obj) {
+			dev_err(dev->dev, "Failed to lookup GEM object.\n");
+			ret = -ENXIO;
+			goto err;
+		}
+
+		height =
+			drm_format_info_plane_height(info, mode_cmd->height, i);
+
+		size = height * mode_cmd->pitches[i] + mode_cmd->offsets[i];
+
+		if (obj->size < size) {
+			drm_gem_object_put(obj);
+			ret = -EINVAL;
+			goto err;
+		}
+
+		objs[i] = to_es_gem_object(obj);
+	}
+
+	fb = es_fb_alloc(dev, mode_cmd, objs, i);
+	if (IS_ERR(fb)) {
+		ret = PTR_ERR(fb);
+		goto err;
+	}
+
+	return fb;
+
+err:
+	for (; i > 0; i--) {
+		drm_gem_object_put(&objs[i - 1]->base);
+	}
+
+	return ERR_PTR(ret);
+}
+
+struct es_gem_object *es_fb_get_gem_obj(struct drm_framebuffer *fb,
+					unsigned char plane)
+{
+	if (plane > MAX_NUM_PLANES)
+		return NULL;
+
+	return to_es_gem_object(fb->obj[plane]);
+}
+
+static const struct drm_mode_config_funcs es_mode_config_funcs = {
+	.fb_create = es_fb_create,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static struct drm_mode_config_helper_funcs es_mode_config_helpers = {
+	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
+};
+
+void es_mode_config_init(struct drm_device *dev)
+{
+	if (dev->mode_config.max_width == 0 ||
+	    dev->mode_config.max_height == 0) {
+		dev->mode_config.min_width = 0;
+		dev->mode_config.min_height = 0;
+		dev->mode_config.max_width = 4096;
+		dev->mode_config.max_height = 4096;
+	}
+	dev->mode_config.funcs = &es_mode_config_funcs;
+	dev->mode_config.helper_private = &es_mode_config_helpers;
+}
diff --git a/drivers/gpu/drm/eswin/es_fb.h b/drivers/gpu/drm/eswin/es_fb.h
new file mode 100644
index 000000000000..db667daf6e4d
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_fb.h
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_FB_H__
+#define __ES_FB_H__
+
+struct es_gem_object *es_fb_get_gem_obj(struct drm_framebuffer *fb,
+					unsigned char plane);
+
+void es_mode_config_init(struct drm_device *dev);
+#endif /* __ES_FB_H__ */
diff --git a/drivers/gpu/drm/eswin/es_gem.c b/drivers/gpu/drm/eswin/es_gem.c
new file mode 100644
index 000000000000..e44cfa325527
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_gem.c
@@ -0,0 +1,620 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+#include <linux/dma-buf.h>
+#include <drm/drm_prime.h>
+
+#include "es_drv.h"
+#include "es_gem.h"
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+static const struct drm_gem_object_funcs es_gem_default_funcs;
+
+static void nonseq_free(struct page **pages, unsigned int nr_page)
+{
+	u32 i;
+
+	if (!pages)
+		return;
+
+	for (i = 0; i < nr_page; i++)
+		__free_page(pages[i]);
+}
+
+#ifdef CONFIG_ESWIN_MMU
+static int get_pages(unsigned int nr_page, struct es_gem_object *es_obj)
+{
+	struct page *pages;
+	u32 i, num_page, page_count = 0;
+	int order = 0;
+	gfp_t gfp = GFP_KERNEL;
+
+	if (!es_obj->pages)
+		return -EINVAL;
+
+	gfp &= ~__GFP_HIGHMEM;
+	gfp |= __GFP_DMA32;
+
+	num_page = nr_page;
+
+	do {
+		pages = NULL;
+		order = get_order(num_page * PAGE_SIZE);
+		num_page = 1 << order;
+
+		if ((num_page + page_count > nr_page) || (order >= MAX_ORDER)) {
+			num_page = num_page >> 1;
+			continue;
+		}
+
+		pages = alloc_pages(gfp, order);
+		if (!pages) {
+			if (num_page == 1) {
+				nonseq_free(es_obj->pages, page_count);
+				return -ENOMEM;
+			}
+
+			num_page = num_page >> 1;
+		} else {
+			for (i = 0; i < num_page; i++) {
+				es_obj->pages[page_count + i] = &pages[i];
+				SetPageReserved(es_obj->pages[page_count + i]);
+			}
+
+			page_count += num_page;
+			num_page = nr_page - page_count;
+		}
+
+	} while (page_count < nr_page);
+
+	es_obj->get_pages = true;
+
+	return 0;
+}
+#endif
+
+static void put_pages(unsigned int nr_page, struct es_gem_object *es_obj)
+{
+	u32 i;
+
+	for (i = 0; i < nr_page; i++)
+		ClearPageReserved(es_obj->pages[i]);
+
+	nonseq_free(es_obj->pages, nr_page);
+
+	return;
+}
+
+static int es_gem_alloc_buf(struct es_gem_object *es_obj)
+{
+	struct drm_device *dev = es_obj->base.dev;
+	unsigned int nr_pages;
+	struct sg_table sgt;
+	int ret = -ENOMEM;
+#ifdef CONFIG_ESWIN_MMU
+	struct es_drm_private *priv = dev->dev_private;
+#endif
+
+	if (es_obj->dma_addr) {
+		DRM_DEV_DEBUG_KMS(dev->dev, "already allocated.\n");
+		return 0;
+	}
+
+	es_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;
+
+	if (!is_iommu_enabled(dev))
+		es_obj->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
+
+	nr_pages = es_obj->size >> PAGE_SHIFT;
+
+	es_obj->pages = kvmalloc_array(nr_pages, sizeof(struct page *),
+				       GFP_KERNEL | __GFP_ZERO);
+	if (!es_obj->pages) {
+		DRM_DEV_ERROR(dev->dev, "failed to allocate pages.\n");
+		return -ENOMEM;
+	}
+
+	es_obj->cookie = dma_alloc_attrs(to_dma_dev(dev), es_obj->size,
+					 &es_obj->dma_addr, GFP_KERNEL,
+					 es_obj->dma_attrs);
+	if (!es_obj->cookie) {
+#ifdef CONFIG_ESWIN_MMU
+		ret = get_pages(nr_pages, es_obj);
+		if (ret) {
+			DRM_DEV_ERROR(dev->dev, "fail to allocate buffer.\n");
+			goto err_free;
+		}
+#else
+		DRM_DEV_ERROR(dev->dev, "failed to allocate buffer.\n");
+		goto err_free;
+#endif
+	}
+
+#ifdef CONFIG_ESWIN_MMU
+	/* MMU map*/
+	if (!priv->mmu) {
+		DRM_DEV_ERROR(dev->dev, "invalid mmu.\n");
+		ret = -EINVAL;
+		goto err_mem_free;
+	}
+
+	if (!es_obj->get_pages)
+		ret = dc_mmu_map_memory(priv->mmu, (u64)es_obj->dma_addr,
+					nr_pages, &es_obj->iova, true);
+	else
+		ret = dc_mmu_map_memory(priv->mmu, (u64)es_obj->pages, nr_pages,
+					&es_obj->iova, false);
+
+	if (ret) {
+		DRM_DEV_ERROR(dev->dev, "failed to do mmu map.\n");
+		goto err_mem_free;
+	}
+#else
+	es_obj->iova = es_obj->dma_addr;
+#endif
+
+	if (!es_obj->get_pages) {
+		ret = dma_get_sgtable_attrs(to_dma_dev(dev), &sgt,
+					    es_obj->cookie, es_obj->dma_addr,
+					    es_obj->size, es_obj->dma_attrs);
+		if (ret < 0) {
+			DRM_DEV_ERROR(dev->dev, "failed to get sgtable.\n");
+			goto err_mem_free;
+		}
+
+		if (drm_prime_sg_to_page_array(&sgt, es_obj->pages, nr_pages)) {
+			DRM_DEV_ERROR(dev->dev, "invalid sgtable.\n");
+			ret = -EINVAL;
+			goto err_sgt_free;
+		}
+
+		sg_free_table(&sgt);
+	}
+
+	return 0;
+
+err_sgt_free:
+	sg_free_table(&sgt);
+err_mem_free:
+	if (!es_obj->get_pages)
+		dma_free_attrs(to_dma_dev(dev), es_obj->size, es_obj->cookie,
+			       es_obj->dma_addr, es_obj->dma_attrs);
+	else
+		put_pages(nr_pages, es_obj);
+err_free:
+	kvfree(es_obj->pages);
+
+	return ret;
+}
+
+static void es_gem_free_buf(struct es_gem_object *es_obj)
+{
+	struct drm_device *dev = es_obj->base.dev;
+#ifdef CONFIG_ESWIN_MMU
+	struct es_drm_private *priv = dev->dev_private;
+	unsigned int nr_pages;
+#endif
+
+	if ((!es_obj->get_pages) && (!es_obj->dma_addr)) {
+		DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr is invalid.\n");
+		return;
+	}
+
+#ifdef CONFIG_ESWIN_MMU
+	if (!priv->mmu) {
+		DRM_DEV_ERROR(dev->dev, "invalid mmu.\n");
+		return;
+	}
+
+	if (!es_obj->sgt) { // dumb buffer release
+		nr_pages = es_obj->size >> PAGE_SHIFT;
+		if (es_obj->iova) {
+			dc_mmu_unmap_memory(priv->mmu, es_obj->iova, nr_pages);
+		}
+	} else { // prime buffer release
+		if (es_obj->iova_list) {
+			if (es_obj->iova_list->iova) {
+				dc_mmu_unmap_memory(
+					priv->mmu, es_obj->iova_list->iova,
+					es_obj->iova_list->nr_pages);
+				kfree(es_obj->iova_list);
+			}
+		}
+	}
+#endif
+
+	if (!es_obj->get_pages) {
+		dma_free_attrs(to_dma_dev(dev), es_obj->size, es_obj->cookie,
+			       (dma_addr_t)es_obj->dma_addr, es_obj->dma_attrs);
+	} else {
+		if (!es_obj->dma_addr) {
+			DRM_DEV_ERROR(dev->dev, "No dma addr allocated, no need to free\n");
+			return;
+		}
+		put_pages(es_obj->size >> PAGE_SHIFT, es_obj);
+	}
+
+	kvfree(es_obj->pages);
+}
+
+static void es_gem_free_object(struct drm_gem_object *obj)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+
+#ifdef CONFIG_ESWIN_MMU
+	if (es_obj)
+		es_gem_free_buf(es_obj);
+#endif
+	if (obj->import_attach) {
+		drm_prime_gem_destroy(obj, es_obj->sgt);
+	}
+
+	drm_gem_object_release(obj);
+
+	kfree(es_obj);
+}
+
+static struct es_gem_object *es_gem_alloc_object(struct drm_device *dev,
+						 size_t size)
+{
+	struct es_gem_object *es_obj;
+	struct drm_gem_object *obj;
+	int ret;
+
+	es_obj = kzalloc(sizeof(*es_obj), GFP_KERNEL);
+	if (!es_obj)
+		return ERR_PTR(-ENOMEM);
+
+	es_obj->size = size;
+	obj = &es_obj->base;
+
+	ret = drm_gem_object_init(dev, obj, size);
+	if (ret)
+		goto err_free;
+
+	es_obj->base.funcs = &es_gem_default_funcs;
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret) {
+		drm_gem_object_release(obj);
+		goto err_free;
+	}
+
+	return es_obj;
+
+err_free:
+	kfree(es_obj);
+	return ERR_PTR(ret);
+}
+
+struct es_gem_object *es_gem_create_object(struct drm_device *dev, size_t size)
+{
+	struct es_gem_object *es_obj;
+	int ret;
+
+	size = PAGE_ALIGN(size);
+
+	es_obj = es_gem_alloc_object(dev, size);
+	if (IS_ERR(es_obj))
+		return es_obj;
+
+	ret = es_gem_alloc_buf(es_obj);
+	if (ret) {
+		drm_gem_object_release(&es_obj->base);
+		kfree(es_obj);
+		return ERR_PTR(ret);
+	}
+
+	return es_obj;
+}
+
+static struct es_gem_object *es_gem_create_with_handle(struct drm_device *dev,
+						       struct drm_file *file,
+						       size_t size,
+						       unsigned int *handle)
+{
+	struct es_gem_object *es_obj;
+	struct drm_gem_object *obj;
+	int ret;
+
+	es_obj = es_gem_create_object(dev, size);
+	if (IS_ERR(es_obj))
+		return es_obj;
+
+	obj = &es_obj->base;
+
+	ret = drm_gem_handle_create(file, obj, handle);
+	drm_gem_object_put(obj);
+	if (ret) {
+		pr_err("Drm GEM handle create failed\n");
+		return ERR_PTR(ret);
+	}
+
+	return es_obj;
+}
+
+static int es_gem_mmap_obj(struct drm_gem_object *obj,
+			   struct vm_area_struct *vma)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+	struct drm_device *drm_dev = es_obj->base.dev;
+	unsigned long vm_size;
+	int ret = 0;
+
+	vm_size = vma->vm_end - vma->vm_start;
+	if (vm_size > es_obj->size)
+		return -EINVAL;
+
+	vma->vm_pgoff = 0;
+
+	if (!es_obj->get_pages) {
+		vm_flags_clear(vma, VM_PFNMAP);
+
+		ret = dma_mmap_attrs(to_dma_dev(drm_dev), vma, es_obj->cookie,
+				     es_obj->dma_addr, es_obj->size,
+				     es_obj->dma_attrs);
+	} else {
+		u32 i, nr_pages, pfn = 0U;
+		unsigned long start;
+
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+		vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP);
+		start = vma->vm_start;
+		vm_size = PAGE_ALIGN(vm_size);
+		nr_pages = vm_size >> PAGE_SHIFT;
+
+		for (i = 0; i < nr_pages; i++) {
+			pfn = page_to_pfn(es_obj->pages[i]);
+
+			ret = remap_pfn_range(vma, start, pfn, PAGE_SIZE,
+					      vma->vm_page_prot);
+			if (ret < 0)
+				break;
+
+			start += PAGE_SIZE;
+		}
+	}
+
+	if (ret)
+		drm_gem_vm_close(vma);
+
+	return ret;
+}
+
+struct sg_table *es_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+
+	return drm_prime_pages_to_sg(obj->dev, es_obj->pages,
+				     es_obj->size >> PAGE_SHIFT);
+}
+
+static int es_gem_prime_vmap(struct drm_gem_object *obj,
+			     struct iosys_map *map)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+
+	void * vaddr = es_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING ?
+		       page_address(es_obj->cookie) : es_obj->cookie;
+
+	iosys_map_set_vaddr(map, vaddr);
+
+	return 0;
+}
+
+static void es_gem_prime_vunmap(struct drm_gem_object *obj,
+				struct iosys_map *map)
+{
+	/* Nothing to do */
+}
+
+static const struct vm_operations_struct es_vm_ops = {
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static const struct drm_gem_object_funcs es_gem_default_funcs = {
+	.free = es_gem_free_object,
+	.get_sg_table = es_gem_prime_get_sg_table,
+	.vmap = es_gem_prime_vmap,
+	.vunmap = es_gem_prime_vunmap,
+	.vm_ops = &es_vm_ops,
+};
+
+int es_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
+		       struct drm_mode_create_dumb *args)
+{
+	struct es_drm_private *priv = dev->dev_private;
+	struct es_gem_object *es_obj;
+	unsigned int pitch = args->width * DIV_ROUND_UP(args->bpp, 8);
+
+	args->pitch = ALIGN(pitch, priv->pitch_alignment);
+	args->size = PAGE_ALIGN(args->pitch * args->height);
+
+	es_obj =
+		es_gem_create_with_handle(dev, file, args->size, &args->handle);
+	return PTR_ERR_OR_ZERO(es_obj);
+}
+
+struct drm_gem_object *es_gem_prime_import(struct drm_device *dev,
+					   struct dma_buf *dma_buf)
+{
+	return drm_gem_prime_import_dev(dev, dma_buf, to_dma_dev(dev));
+}
+
+struct drm_gem_object *
+es_gem_prime_import_sg_table(struct drm_device *dev,
+			     struct dma_buf_attachment *attach,
+			     struct sg_table *sgt)
+{
+	struct es_gem_object *es_obj;
+	int npages;
+	int ret;
+	struct scatterlist *s = NULL;
+	u32 i = 0;
+	dma_addr_t expected;
+	size_t size = attach->dmabuf->size;
+#ifdef CONFIG_ESWIN_MMU
+	u32 iova, j;
+	struct scatterlist **splist;
+	struct es_drm_private *priv = dev->dev_private;
+
+	if (!priv->mmu) {
+		DRM_ERROR("invalid mmu.\n");
+		ret = -EINVAL;
+		return ERR_PTR(ret);
+	}
+#endif
+
+	size = PAGE_ALIGN(size);
+
+	es_obj = es_gem_alloc_object(dev, size);
+	if (IS_ERR(es_obj))
+		return ERR_CAST(es_obj);
+
+	npages = es_obj->size >> PAGE_SHIFT;
+	es_obj->pages =
+		kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!es_obj->pages) {
+		ret = -ENOMEM;
+		goto err_gemalloc;
+	}
+
+	ret = drm_prime_sg_to_page_array(sgt, es_obj->pages, npages);
+	if (ret)
+		goto err_free_page;
+
+	expected = sg_dma_address(sgt->sgl);
+#ifdef CONFIG_ESWIN_MMU
+	splist = (struct scatterlist **)kzalloc(sizeof(s) * sgt->nents,
+						GFP_KERNEL);
+	if (!splist) {
+		DRM_ERROR("Allocate splist failed");
+		ret = -ENOMEM;
+		goto err_free_page;
+	}
+
+	es_obj->iova_list =
+		(iova_info_t *)kzalloc(sizeof(iova_info_t), GFP_KERNEL);
+	if (!es_obj->iova_list) {
+		DRM_ERROR("Allocate splist failed");
+		ret = -ENOMEM;
+		goto err_sp;
+	}
+
+	for_each_sg (sgt->sgl, s, sgt->nents, i) {
+		splist[i] = s;
+	}
+	i = 0;
+	es_obj->nr_iova = sgt->nents;
+
+	for (j = sgt->nents; j > 0; j--) {
+		s = splist[j - 1];
+#else
+	for_each_sg (sgt->sgl, s, sgt->nents, i) {
+#endif
+		if (sg_dma_address(s) != expected) {
+#ifndef CONFIG_ESWIN_MMU
+			DRM_ERROR("sg_table is not contiguous");
+			ret = -EINVAL;
+			goto err;
+#endif
+		}
+
+		if (sg_dma_len(s) & (PAGE_SIZE - 1)) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+#ifdef CONFIG_ESWIN_MMU
+		iova = 0;
+
+		if (j == 1) {
+			ret = dc_mmu_map_memory(priv->mmu, (u64)es_obj->pages,
+						npages, &iova, false);
+			if (ret) {
+				DRM_ERROR("failed to do mmu map.\n");
+				goto err;
+			}
+			es_obj->iova_list->iova = iova;
+			es_obj->iova_list->nr_pages = npages;
+		}
+
+		if (i == 0)
+			es_obj->iova = iova;
+#else
+		if (i == 0)
+			es_obj->iova = sg_dma_address(s);
+#endif
+
+		expected = sg_dma_address(s) + sg_dma_len(s);
+	}
+
+	es_obj->dma_addr = sg_dma_address(sgt->sgl);
+
+	es_obj->sgt = sgt;
+#ifdef CONFIG_ESWIN_MMU
+	kfree(splist);
+#endif
+
+	return &es_obj->base;
+
+#ifdef CONFIG_ESWIN_MMU
+err:
+	kfree(es_obj->iova_list);
+err_sp:
+	kfree(splist);
+#endif
+err_free_page:
+	kvfree(es_obj->pages);
+err_gemalloc:
+	es_gem_free_object(&es_obj->base);
+
+	return ERR_PTR(ret);
+}
+
+int es_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	int ret = 0;
+
+	ret = drm_gem_mmap_obj(obj, obj->size, vma);
+	if (ret < 0)
+		return ret;
+
+	return es_gem_mmap_obj(obj, vma);
+}
+
+int es_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	obj = vma->vm_private_data;
+
+	if (obj->import_attach)
+		return dma_buf_mmap(obj->dma_buf, vma, 0);
+
+	return es_gem_mmap_obj(obj, vma);
+}
diff --git a/drivers/gpu/drm/eswin/es_gem.h b/drivers/gpu/drm/eswin/es_gem.h
new file mode 100644
index 000000000000..4dc902406052
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_gem.h
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_GEM_H__
+#define __ES_GEM_H__
+
+#include <linux/dma-buf.h>
+
+#include <drm/drm_gem.h>
+
+#include "es_drv.h"
+
+#ifdef CONFIG_ESWIN_MMU
+typedef struct _iova_info {
+	u32 iova;
+	u32 nr_pages;
+} iova_info_t;
+#endif
+
+/*
+ *
+ * @base: drm gem object.
+ * @size: size requested from user
+ * @cookie: cookie returned by dma_alloc_attrs
+ *  - not kernel virtual address with DMA_ATTR_NO_KERNEL_MAPPING
+ * @dma_addr: bus address(accessed by dma) to allocated memory region.
+ *  - this address could be physical address without IOMMU and
+ *  device address with IOMMU.
+ * @dma_attrs: attribute for DMA API
+ * @get_pages: flag for manually applying for non-contiguous memory.
+ * @pages: Array of backing pages.
+ * @sgt: Imported sg_table.
+ *
+ */
+struct es_gem_object {
+	struct drm_gem_object base;
+	size_t size;
+	void *cookie;
+	dma_addr_t dma_addr;
+	u32 iova;
+	unsigned long dma_attrs;
+	bool get_pages;
+	struct page **pages;
+	struct sg_table *sgt;
+#ifdef CONFIG_ESWIN_MMU
+	iova_info_t *iova_list;
+	u32 nr_iova;
+#endif
+};
+
+static inline struct es_gem_object *to_es_gem_object(struct drm_gem_object *obj)
+{
+	return container_of(obj, struct es_gem_object, base);
+}
+
+struct es_gem_object *es_gem_create_object(struct drm_device *dev, size_t size);
+
+int es_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+
+int es_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
+		       struct drm_mode_create_dumb *args);
+
+int es_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+
+struct sg_table *es_gem_prime_get_sg_table(struct drm_gem_object *obj);
+
+struct drm_gem_object *es_gem_prime_import(struct drm_device *dev,
+					   struct dma_buf *dma_buf);
+struct drm_gem_object *
+es_gem_prime_import_sg_table(struct drm_device *dev,
+			     struct dma_buf_attachment *attach,
+			     struct sg_table *sgt);
+
+#endif /* __ES_GEM_H__ */
diff --git a/drivers/gpu/drm/eswin/es_mipi_dsi.c b/drivers/gpu/drm/eswin/es_mipi_dsi.c
new file mode 100644
index 000000000000..cf3b74eb0479
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_mipi_dsi.c
@@ -0,0 +1,707 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/iopoll.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/mfd/syscon.h>
+#include <linux/gpio/consumer.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+#include <video/mipi_display.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/bridge/dw_mipi_dsi.h>
+#include <linux/media-bus-format.h>
+
+#include "es_crtc.h"
+#include "es_mipi_dsi.h"
+#include "es_panel.h"
+
+#define MAX_LANE_COUNT 4
+#define PLL_INPUT_REF_CLK 20000000
+
+/* Defines the offset of the dphy register */
+#define DSI_PHY_TST_CTRL0 0xb4
+#define PHY_TESTCLK BIT(1)
+#define PHY_UNTESTCLK 0
+#define PHY_TESTCLR BIT(0)
+#define PHY_UNTESTCLR 0
+
+#define DSI_PHY_TST_CTRL1 0xb8
+#define PHY_TESTEN BIT(16)
+#define PHY_UNTESTEN 0
+#define PHY_TESTDOUT(n) (((n) & 0xff) << 8)
+#define PHY_TESTDIN(n) (((n) & 0xff) << 0)
+#define INPUT_DIVIDER(val) (val & 0x7f)
+
+#define LOW_PROGRAM_EN 0
+#define HIGH_PROGRAM_EN BIT(7)
+#define LOOP_DIV_LOW_SEL(val) (val & 0x1f)
+#define LOOP_DIV_HIGH_SEL(val) ((val >> 5) & 0xf)
+#define PLL_LOOP_DIV_EN BIT(5)
+#define PLL_INPUT_DIV_EN BIT(4)
+
+#define LPF_RESISTORS_15_5KOHM 0x1
+#define LPF_RESISTORS_13KOHM 0x2
+#define LPF_RESISTORS_11_5KOHM 0x4
+#define LPF_RESISTORS_10_5KOHM 0x8
+#define LPF_RESISTORS_8KOHM 0x10
+#define LPF_PROGRAM_EN BIT(6)
+#define LPF_RESISTORS_SEL(val) ((val) & 0x3f)
+
+#define LOW_PROGRAM_EN 0
+#define HIGH_PROGRAM_EN BIT(7)
+#define LOOP_DIV_LOW_SEL(val) (val & 0x1f)
+#define LOOP_DIV_HIGH_SEL(val) ((val >> 5) & 0xf)
+#define PLL_LOOP_DIV_EN BIT(5)
+#define PLL_INPUT_DIV_EN BIT(4)
+
+#define PLL_BIAS_CUR_SEL_CAP_VCO_CONTROL 0x10
+#define PLL_CP_CONTROL_PLL_LOCK_BYPASS 0x11
+#define PLL_LPF_AND_CP_CONTROL 0x12
+#define PLL_INPUT_DIVIDER_RATIO 0x17
+#define PLL_LOOP_DIVIDER_RATIO 0x18
+#define PLL_INPUT_AND_LOOP_DIVIDER_RATIOS_CONTROL 0x19
+
+#define DSI_TO_CNT_CFG 0x78
+#define HSTX_TO_CNT(p) (((p) & 0xffff) << 16)
+#define LPRX_TO_CNT(p) ((p) & 0xffff)
+
+static bool panel_driver_registed = false;
+
+struct hstt {
+	unsigned int maxfreq;
+	struct dw_mipi_dsi_dphy_timing timing;
+};
+
+struct pll_parameter {
+	u32 max_freq;
+	u32 actual_freq;
+	u16 hs_freq_range;
+	u16 pll_n;
+	u16 pll_m;
+	u8 vco_cntrl;
+	u8 cpbias_cntrl;
+	u8 gmp_cntrl;
+	u8 int_cntrl;
+	u8 prop_cntrl;
+};
+
+#define HSTT(_maxfreq, _c_lp2hs, _c_hs2lp, _d_lp2hs, _d_hs2lp)                 \
+	{                                                                      \
+		.maxfreq = _maxfreq, .timing = {                               \
+			.clk_lp2hs = _c_lp2hs,                                 \
+			.clk_hs2lp = _c_hs2lp,                                 \
+			.data_lp2hs = _d_lp2hs,                                \
+			.data_hs2lp = _d_hs2lp,                                \
+		}                                                              \
+	}
+
+#define PLL_PARAM(_maxfreq, _vco_ctl, _cpbias_ctl, _gmp_ctl, _int_ctl,         \
+		  _prop_ctl)                                                   \
+	{                                                                      \
+		.max_freq = _maxfreq, .vco_cntrl = _vco_ctl,                   \
+		.cpbias_cntrl = _cpbias_ctl, .gmp_cntrl = _gmp_ctl,            \
+		.int_cntrl = _int_ctl, .prop_cntrl = _prop_ctl,                \
+	}
+
+struct mipi_dsi_priv {
+	u32 lanes;
+	u32 format;
+	unsigned long mode_flags;
+
+	struct pll_parameter pll_param;
+
+	void __iomem *dphy_base;
+
+	struct clk *ivideo_clk;
+	u64 pll_ref_clk;
+
+	struct dw_mipi_dsi_plat_data plat_data;
+	struct reset_control *rst_dsi_phyrstn;
+	void *data;
+	struct gpio_desc *dsi_mux_gpio;
+};
+
+struct es_mipi_dsi {
+	struct mipi_dsi_priv dsi_priv;
+	struct drm_encoder encoder;
+	struct drm_crtc *crtc;
+};
+
+/* Table A-4 High-Speed Transition Times (High-Speed Entry LP->HS DATA LANE(Considers HS_CLK_LANE_ENTRY)d)*/
+struct hstt hstt_table[] = {
+	HSTT(80, 21, 17, 35, 10),      HSTT(90, 23, 17, 39, 10),
+	HSTT(100, 22, 17, 37, 10),     HSTT(110, 25, 18, 43, 11),
+	HSTT(120, 26, 20, 46, 11),     HSTT(130, 27, 19, 46, 11),
+	HSTT(140, 27, 19, 46, 11),     HSTT(150, 28, 20, 47, 12),
+	HSTT(160, 30, 21, 53, 13),     HSTT(170, 30, 21, 55, 13),
+	HSTT(180, 31, 21, 53, 13),     HSTT(190, 32, 22, 58, 13),
+	HSTT(205, 35, 22, 58, 13),     HSTT(220, 37, 26, 63, 15),
+	HSTT(235, 38, 28, 65, 16),     HSTT(250, 41, 29, 71, 17),
+	HSTT(275, 43, 29, 74, 18),     HSTT(300, 45, 32, 80, 19),
+	HSTT(325, 48, 33, 86, 18),     HSTT(350, 51, 35, 91, 20),
+	HSTT(400, 59, 37, 102, 21),    HSTT(450, 65, 40, 115, 23),
+	HSTT(500, 71, 41, 126, 24),    HSTT(550, 77, 44, 135, 26),
+	HSTT(600, 82, 46, 147, 27),    HSTT(650, 87, 48, 156, 28),
+	HSTT(700, 94, 52, 166, 29),    HSTT(750, 99, 52, 175, 31),
+	HSTT(800, 105, 55, 187, 32),   HSTT(850, 110, 58, 196, 32),
+	HSTT(900, 115, 58, 206, 35),   HSTT(950, 120, 62, 213, 36),
+	HSTT(1000, 128, 63, 225, 38),  HSTT(1050, 132, 65, 234, 38),
+	HSTT(1100, 138, 67, 243, 39),  HSTT(1150, 146, 69, 259, 42),
+	HSTT(1200, 151, 71, 269, 43),  HSTT(1250, 153, 74, 273, 45),
+	HSTT(1300, 160, 73, 282, 46),  HSTT(1350, 165, 76, 294, 47),
+	HSTT(1400, 172, 78, 304, 49),  HSTT(1450, 177, 80, 314, 49),
+	HSTT(1500, 183, 81, 326, 52),  HSTT(1550, 191, 84, 339, 52),
+	HSTT(1600, 194, 85, 345, 52),  HSTT(1650, 201, 86, 355, 53),
+	HSTT(1700, 208, 88, 368, 53),  HSTT(1750, 212, 89, 378, 53),
+	HSTT(1800, 220, 90, 389, 54),  HSTT(1850, 223, 92, 401, 54),
+	HSTT(1900, 231, 91, 413, 55),  HSTT(1950, 236, 95, 422, 56),
+	HSTT(2000, 243, 97, 432, 56),  HSTT(2050, 248, 99, 442, 58),
+	HSTT(2100, 252, 100, 454, 59), HSTT(2150, 259, 102, 460, 61),
+	HSTT(2200, 266, 105, 476, 62), HSTT(2250, 269, 109, 481, 63),
+	HSTT(2300, 272, 109, 490, 65), HSTT(2350, 281, 112, 502, 66),
+	HSTT(2400, 283, 115, 509, 66), HSTT(2450, 282, 115, 510, 67),
+	HSTT(2500, 281, 118, 508, 67),
+};
+
+static struct pll_parameter pll_tbl[] = {
+	PLL_PARAM(55, 0x3f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(82, 0x37, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(110, 0x2f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(165, 0x27, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(220, 0x1f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(330, 0x17, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(440, 0x0f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(660, 0x07, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(1149, 0x03, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(1152, 0x01, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(1250, 0x01, 0x10, 0x01, 0x00, 0x0E),
+};
+
+static const struct of_device_id es_mipi_dsi_dt_ids[] = {
+	{
+		.compatible = "eswin,dsi",
+	},
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(of, es_mipi_dsi_dt_ids);
+
+static void dsi_write(struct mipi_dsi_priv *priv, u32 reg, u32 val)
+{
+	writel(val, priv->dphy_base + reg);
+}
+
+static u32 dsi_read(struct mipi_dsi_priv *priv, u32 reg)
+{
+	return readl(priv->dphy_base + reg);
+}
+
+static void es_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct es_mipi_dsi *es_dsi =
+		container_of(encoder, struct es_mipi_dsi, encoder);
+	struct mipi_dsi_priv *dsi_priv = &es_dsi->dsi_priv;
+
+	DRM_DEBUG("enter, encoder = 0x%px\n", encoder);
+
+	clk_disable_unprepare(dsi_priv->ivideo_clk);
+}
+
+static int
+es_dsi_encoder_atomic_check(struct drm_encoder *encoder,
+				   struct drm_crtc_state *crtc_state,
+				   struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *s = to_es_crtc_state(crtc_state);
+	struct drm_display_info *info = &conn_state->connector->display_info;
+	u32 bus_format = info->bus_formats[0];
+	s->output_fmt = bus_format;
+	return 0;
+}
+
+static bool es_dsi_encoder_mode_fixup(struct drm_encoder *encoder,
+				      const struct drm_display_mode *mode,
+				      struct drm_display_mode *adj_mode)
+{
+	struct es_mipi_dsi *es_dsi =
+		container_of(encoder, struct es_mipi_dsi, encoder);
+	struct drm_crtc_state *crtc_state =
+		container_of(mode, struct drm_crtc_state, mode);
+
+	DRM_DEBUG("enter, bind crtc%d\n", drm_crtc_index(crtc_state->crtc));
+	DRM_DEBUG("mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+	DRM_DEBUG("adj_mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(adj_mode));
+
+	es_dsi->crtc = crtc_state->crtc;
+	return true;
+}
+
+static void es_dsi_encoder_mode_set(struct drm_encoder *encoder,
+				    struct drm_display_mode *mode,
+				    struct drm_display_mode *adj_mode)
+{
+	DRM_DEBUG("mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+	DRM_DEBUG("adj_mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(adj_mode));
+}
+
+static void es_dsi_encoder_mode_enable(struct drm_encoder *encoder)
+{
+	int ret;
+	struct es_mipi_dsi *es_dsi =
+		container_of(encoder, struct es_mipi_dsi, encoder);
+	struct mipi_dsi_priv *dsi_priv = &es_dsi->dsi_priv;
+
+	DRM_DEBUG("enter, encoder = 0x%px\n", encoder);
+
+	ret = clk_prepare_enable(dsi_priv->ivideo_clk);
+	if (ret)
+		DRM_ERROR("enable ivideo clk failed, ret = %d\n", ret);
+}
+
+static const struct drm_encoder_helper_funcs es_dsi_encoder_helper_funcs = {
+	.mode_fixup = es_dsi_encoder_mode_fixup,
+	.mode_set = es_dsi_encoder_mode_set,
+	.enable = es_dsi_encoder_mode_enable,
+	.disable = es_dsi_encoder_disable,
+	.atomic_check = es_dsi_encoder_atomic_check,
+};
+
+static void es_mipi_dsi_phy_write(struct mipi_dsi_priv *dsi, u8 test_code,
+				  u8 test_data)
+{
+	/*
+     * With the falling edge on TESTCLK, the TESTDIN[7:0] signal content
+     * is latched internally as the current test code. Test data is
+     * programmed internally by rising edge on TESTCLK.
+     */
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  PHY_TESTEN | PHY_TESTDOUT(0) | PHY_TESTDIN(test_code));
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  PHY_UNTESTEN | PHY_TESTDOUT(0) | PHY_TESTDIN(test_data));
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+}
+
+static void es_mipi_dsi_phy_write_m(struct mipi_dsi_priv *dsi, u16 m)
+{
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  PHY_TESTEN | PHY_TESTDOUT(0) | PLL_LOOP_DIVIDER_RATIO);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  LOOP_DIV_HIGH_SEL(m) | HIGH_PROGRAM_EN);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL1, LOOP_DIV_LOW_SEL(m) | LOW_PROGRAM_EN);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+}
+
+static int es_dsi_phy_init(void *priv_data)
+{
+	struct mipi_dsi_priv *dsi_priv = (struct mipi_dsi_priv *)priv_data;
+	struct pll_parameter *pll_param = &dsi_priv->pll_param;
+
+	dsi_write(dsi_priv, DSI_TO_CNT_CFG, HSTX_TO_CNT(0) | LPRX_TO_CNT(0));
+
+	es_mipi_dsi_phy_write(dsi_priv,
+			      PLL_INPUT_AND_LOOP_DIVIDER_RATIOS_CONTROL,
+			      PLL_LOOP_DIV_EN | PLL_INPUT_DIV_EN);
+
+	DRM_INFO(
+		"set pll m:0x%x, n:0x%x, vco_cntrl:0x%x ,cpbias:0x%x, gmp:0x%x, int:0x%x, prop:0x%x\n",
+		pll_param->pll_m, pll_param->pll_n, pll_param->vco_cntrl,
+		pll_param->cpbias_cntrl, pll_param->gmp_cntrl,
+		pll_param->int_cntrl, pll_param->prop_cntrl);
+	//  0x17 cfg -> n
+	es_mipi_dsi_phy_write(dsi_priv, PLL_INPUT_DIVIDER_RATIO,
+			      INPUT_DIVIDER(pll_param->pll_n));
+
+	es_mipi_dsi_phy_write_m(dsi_priv, pll_param->pll_m);
+
+	// 0x12 cfg
+	es_mipi_dsi_phy_write(dsi_priv, PLL_LPF_AND_CP_CONTROL,
+			      LPF_PROGRAM_EN | pll_param->vco_cntrl);
+
+	// 0x1c cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x1c, pll_param->cpbias_cntrl);
+
+	// 0x13 cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x13, pll_param->gmp_cntrl << 4);
+
+	// 0x0f cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x0f, pll_param->int_cntrl);
+	// 0x0f cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x0e, pll_param->prop_cntrl);
+
+	return 0;
+}
+
+static int dsi_phy_pll_get_params(struct mipi_dsi_priv *dsi_priv, u64 lane_bps)
+{
+	int i;
+	unsigned int min_prediv, max_prediv;
+	unsigned int _prediv0 = 1, _prediv, best_prediv = 0;
+	unsigned long best_freq = 0;
+	unsigned long fvco_min, fvco_max, fin, fout, fvco;
+	unsigned long f_multi, best_multi;
+	unsigned long min_delta = ULONG_MAX;
+	struct pll_parameter *pll_param = &dsi_priv->pll_param;
+
+	fin = dsi_priv->pll_ref_clk;
+	fout = lane_bps / 2;
+
+	/* constraint: 2Mhz <= Fref / N <= 8MHz */
+	min_prediv = DIV_ROUND_UP(fin, 8 * USEC_PER_SEC);
+	max_prediv = fin / (2 * USEC_PER_SEC);
+
+	/* constraint: 320MHz <= Fvco <= 1250Mhz */
+	fvco_min = 320 * USEC_PER_SEC;
+	fvco_max = 1250 * USEC_PER_SEC;
+
+	if (fout * 4 < fvco_min)
+		_prediv0 = 8;
+	else if (fout * 2 < fvco_min)
+		_prediv0 = 4;
+	else if (fout < fvco_min)
+		_prediv0 = 2;
+
+	fvco = fout * _prediv0;
+
+	for (_prediv = min_prediv; _prediv <= max_prediv; _prediv++) {
+		u64 tmp;
+		u32 delta;
+
+		/* Fvco = Fref * M / N */
+		tmp = (u64)fvco * _prediv;
+		do_div(tmp, fin);
+		f_multi = tmp;
+		/* M range */
+		if (f_multi < 64 || f_multi > 625)
+			continue;
+
+		tmp = (u64)f_multi * fin;
+		do_div(tmp, _prediv); // fvco
+		if (tmp < fvco_min || tmp > fvco_max)
+			continue;
+
+		delta = abs(fvco - tmp);
+		if (delta < min_delta) {
+			best_prediv = _prediv;
+			best_multi = f_multi; // M
+			min_delta = delta;
+			best_freq = tmp / _prediv0; // fout
+		}
+	}
+
+	if (best_freq) {
+		pll_param->pll_n = best_prediv - 1;
+		pll_param->pll_m = best_multi - 2;
+		pll_param->actual_freq = best_freq;
+	} else {
+		DRM_ERROR("Can not find best_freq for DPHY\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(pll_tbl); i++) {
+		if (pll_param->actual_freq <=
+		    pll_tbl[i].max_freq * USEC_PER_SEC) {
+			pll_param->max_freq =
+				pll_tbl[i].max_freq * USEC_PER_SEC;
+			/* cfgclkfreqrange[5:0] = round[(Fcfg_clk(MHz)-17)*4] */
+			pll_param->vco_cntrl = pll_tbl[i].vco_cntrl;
+			pll_param->cpbias_cntrl = pll_tbl[i].cpbias_cntrl;
+			pll_param->gmp_cntrl = pll_tbl[i].gmp_cntrl;
+			pll_param->int_cntrl = pll_tbl[i].int_cntrl;
+			pll_param->prop_cntrl = pll_tbl[i].prop_cntrl;
+			break;
+		}
+	}
+
+	DRM_INFO(
+		"max_freq = %d, actual_freq = %d, pll_n = %d, pll_m = %d, i=%d\n",
+		pll_param->max_freq, pll_param->actual_freq, pll_param->pll_n,
+		pll_param->pll_m, i);
+	return 0;
+}
+
+static int es_dsi_get_lane_mbps(void *priv_data,
+				const struct drm_display_mode *mode,
+				unsigned long mode_flags, u32 lanes, u32 format,
+				unsigned int *lane_mbps)
+{
+	int ret, bpp;
+	u64 tmp;
+	struct mipi_dsi_priv *dsi_priv = (struct mipi_dsi_priv *)priv_data;
+
+	DRM_INFO("mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+	DRM_INFO("lanes: %d, mode_flags: 0x%lx, format: %d\n", lanes,
+		 mode_flags, format);
+
+	dsi_priv->format = format;
+	bpp = mipi_dsi_pixel_format_to_bpp(dsi_priv->format);
+	if (bpp < 0) {
+		DRM_DEV_ERROR(NULL, "failed to get bpp for pixel format %d\n",
+			      dsi_priv->format);
+		return bpp;
+	}
+
+	if (!lanes)
+		lanes = dsi_priv->plat_data.max_data_lanes;
+
+	dsi_priv->mode_flags = mode_flags;
+	dsi_priv->lanes = lanes;
+
+	tmp = (u64)mode->clock * 1000;
+
+	tmp = tmp * bpp / lanes;
+
+	ret = dsi_phy_pll_get_params(dsi_priv, tmp);
+	if (ret)
+		return ret;
+
+	*lane_mbps =
+		DIV_ROUND_UP(dsi_priv->pll_param.actual_freq * 2, USEC_PER_SEC);
+
+	DRM_INFO("bpp = %d, lane_mbps = %d\n", bpp, *lane_mbps);
+
+	return 0;
+}
+
+static int es_dsi_get_timing(void *priv_data, unsigned int lane_mbps,
+			     struct dw_mipi_dsi_dphy_timing *timing)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hstt_table); i++)
+		if (lane_mbps < hstt_table[i].maxfreq)
+			break;
+
+	if (i == ARRAY_SIZE(hstt_table))
+		i--;
+
+	*timing = hstt_table[i].timing;
+
+	DRM_INFO(
+		"lane_mbps: %d, c_lp2hs: %d, c_hs2lp: %d, d_lp2hs: %d, d_hs2lp: %d\n",
+		lane_mbps, timing->clk_lp2hs, timing->clk_hs2lp,
+		timing->data_lp2hs, timing->data_lp2hs);
+
+	return 0;
+}
+
+struct dw_mipi_dsi_phy_ops dphy_ops = {
+	.init = es_dsi_phy_init,
+	.get_lane_mbps = es_dsi_get_lane_mbps,
+	.get_timing = es_dsi_get_timing,
+};
+
+static int es_mipi_dsi_bind(struct device *dev, struct device *master,
+			    void *data)
+{
+	int ret = 0;
+	unsigned int max_data_lanes;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm_dev = (struct drm_device *)data;
+	struct es_mipi_dsi *es_dsi;
+	struct mipi_dsi_priv *dsi_priv;
+	struct drm_encoder *encoder;
+	struct resource *res;
+
+	es_dsi = devm_kzalloc(&pdev->dev, sizeof(*es_dsi), GFP_KERNEL);
+	if (!es_dsi) {
+		ret = -ENODEV;
+		goto exit0;
+	}
+
+	if (!pdev->dev.of_node) {
+		DRM_ERROR("of_node is null, dev: %px\n", dev);
+		ret = -ENODEV;
+		goto exit0;
+	}
+
+	dsi_priv = &es_dsi->dsi_priv;
+
+	ret = of_property_read_u32(pdev->dev.of_node, "max-lanes",
+				   &max_data_lanes);
+	if (ret) {
+		DRM_WARN("max-lanes not defined, default max-lanes is 4\n");
+		max_data_lanes = MAX_LANE_COUNT;
+	}
+	dsi_priv->plat_data.max_data_lanes = max_data_lanes;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dsi_priv->dphy_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(dsi_priv->dphy_base)) {
+		DRM_ERROR("get mm_sys register base failed, of_node: %pOF\n",
+			  dev->of_node);
+		ret = PTR_ERR(dsi_priv->dphy_base);
+		goto exit0;
+	}
+        // if dsi mux gpio is setting, mux to dsi
+        dsi_priv->dsi_mux_gpio = devm_gpiod_get(dev, "dsi-mux", GPIOD_OUT_LOW);
+        if (!IS_ERR(dsi_priv->dsi_mux_gpio)) {
+                gpiod_set_value(dsi_priv->dsi_mux_gpio, 1);
+                dev_info(dev, "dsi-csi-mux gpio set to dsi\n");
+        }
+
+	dsi_priv->plat_data.base = dsi_priv->dphy_base;
+
+	dsi_priv->plat_data.phy_ops = &dphy_ops;
+	dsi_priv->plat_data.priv_data = dsi_priv;
+	dsi_priv->pll_ref_clk = PLL_INPUT_REF_CLK;
+
+	dsi_priv->ivideo_clk = devm_clk_get(dev, "pclk");
+	if (IS_ERR(dsi_priv->ivideo_clk)) {
+		DRM_ERROR("get ivideo clk failed, of_node: %pOF\n",
+			  dev->of_node);
+		ret = PTR_ERR(dsi_priv->ivideo_clk);
+		goto exit0;
+	}
+
+	/* dsi phyctl reset */
+	dsi_priv->rst_dsi_phyrstn =
+		devm_reset_control_get_optional(dev, "phyrstn");
+	if (IS_ERR_OR_NULL(dsi_priv->rst_dsi_phyrstn)) {
+		DRM_ERROR("Failed to get dsi phyrstn reset handle\n");
+		goto exit0;
+	}
+	if (dsi_priv->rst_dsi_phyrstn) {
+		ret = reset_control_reset(dsi_priv->rst_dsi_phyrstn);
+		WARN_ON(0 != ret);
+	}
+
+	encoder = &es_dsi->encoder;
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm_dev, dev->of_node);
+	if (encoder->possible_crtcs == 0) {
+		DRM_ERROR("Failed to find possible_crtcs:0\n");
+	}
+
+	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_DSI);
+	if (ret) {
+		DRM_ERROR("Failed to initialize encoder with drm\n");
+		return ret;
+	}
+
+	drm_encoder_helper_add(encoder, &es_dsi_encoder_helper_funcs);
+
+	platform_set_drvdata(pdev, es_dsi);
+
+	dsi_priv->data = dw_mipi_dsi_probe(pdev, &dsi_priv->plat_data);
+	if (IS_ERR(dsi_priv->data)) {
+		DRM_ERROR("probe dw-mipi dsi failed\n");
+		ret = -ENODEV;
+		goto exit2;
+	}
+
+	ret = dw_mipi_dsi_bind((struct dw_mipi_dsi *)dsi_priv->data, encoder);
+	if (ret)
+		goto exit3;
+
+	DRM_DEBUG("mipi dsi bind done\n");
+
+	return 0;
+
+exit3:
+	dw_mipi_dsi_remove((struct dw_mipi_dsi *)dsi_priv->data);
+exit2:
+	drm_encoder_cleanup(encoder);
+exit0:
+	DRM_INFO("mipi dsi bind failed, ret = %d\n", ret);
+	return ret;
+}
+
+static void es_mipi_dsi_unbind(struct device *dev, struct device *master,
+			       void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct es_mipi_dsi *es_dsi =
+		(struct es_mipi_dsi *)platform_get_drvdata(pdev);
+	struct mipi_dsi_priv *dsi_priv = &es_dsi->dsi_priv;
+
+	DRM_INFO("mipi dsi unbind\n");
+
+	dw_mipi_dsi_unbind((struct dw_mipi_dsi *)dsi_priv->data);
+	dw_mipi_dsi_remove((struct dw_mipi_dsi *)dsi_priv->data);
+	drm_encoder_cleanup(&es_dsi->encoder);
+}
+
+static const struct component_ops es_mipi_dsi_ops = {
+	.bind = es_mipi_dsi_bind,
+	.unbind = es_mipi_dsi_unbind,
+};
+
+static int es_mipi_dsi_probe(struct platform_device *pdev)
+{
+	if(panel_driver_registed == false) {
+		mipi_dsi_driver_register(&es_panel_driver);
+		panel_driver_registed = true;
+	}
+
+	return component_add(&pdev->dev, &es_mipi_dsi_ops);
+}
+
+static int es_mipi_dsi_remove(struct platform_device *pdev)
+{
+	DRM_INFO("mipi dsi remove\n");
+	if(panel_driver_registed == true) {
+		panel_driver_registed = false;
+		mipi_dsi_driver_unregister(&es_panel_driver);
+	}
+	component_del(&pdev->dev, &es_mipi_dsi_ops);
+
+	return 0;
+}
+
+struct platform_driver es_mipi_dsi_driver = {
+	.probe = es_mipi_dsi_probe,
+	.remove = es_mipi_dsi_remove,
+	.driver = {
+		   .of_match_table = es_mipi_dsi_dt_ids,
+		   .name = "mipi-dsi-drv",
+	},
+};
+
+MODULE_AUTHOR("lilijun@eswin.com");
+MODULE_DESCRIPTION("Eswin dw dsi driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_mipi_dsi.h b/drivers/gpu/drm/eswin/es_mipi_dsi.h
new file mode 100644
index 000000000000..2c21ced10992
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_mipi_dsi.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_MIPI_DSI_H__
+#define __ES_MIPI_DSI_H__
+
+extern struct platform_driver es_mipi_dsi_driver;
+
+#endif // __ES_MIPI_DSI_H__
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/es_panel.c b/drivers/gpu/drm/eswin/es_panel.c
new file mode 100644
index 000000000000..185a5b5a8daf
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_panel.c
@@ -0,0 +1,975 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include "es_panel.h"
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_modes.h>
+#include <linux/backlight.h>
+#include <linux/of.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+#include <linux/media-bus-format.h>
+
+#include <video/mipi_display.h>
+#include <video/of_videomode.h>
+#include <video/videomode.h>
+#include <video/of_display_timing.h>
+#include <video/of_videomode.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/slab.h>
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/list.h>
+#include <linux/gpio/consumer.h>
+#include <drm/drm_print.h>
+#include <linux/version.h>
+
+#define ES_PANEL_MAJOR 0
+#define ES_PANEL_NR_DEVS 1
+#define ES_PANEL_NAME "es_panel"
+
+static int panel_major = ES_PANEL_MAJOR;
+static int panel_minor = 0;
+static int panel_nr_devs = ES_PANEL_NR_DEVS;
+static struct device *pr_dev = NULL;
+
+typedef struct user_cmd_buffer_s {
+	struct list_head head;
+	u32 sleepUs;
+	int buf_size;
+	u8 *buf;
+} user_cmd_buffer_t;
+
+struct es_panel {
+	struct device *dev;
+	struct drm_panel panel;
+	struct backlight_device *backlight;
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *bias_pos, *bias_neg;
+	bool prepared;
+	bool enabled;
+	int error;
+
+	// chr dev:
+	struct cdev cdev;
+	dev_t devt;
+	struct class *es_panel_class;
+
+	// user data:
+	MIPI_MODE_INFO_S mode_info;
+	struct drm_display_mode user_mode;
+	bool user_mode_inited;
+
+	struct list_head init_cmd_list;
+	int init_cmd_writted;
+	user_cmd_buffer_t enable_cmd_buf;
+	user_cmd_buffer_t disable_cmd_buf;
+
+	struct gpio_desc *gpio_backlight0;
+	struct gpio_desc *gpio_reset;
+	enum mipi_dsi_pixel_format format;
+	unsigned int lanes;
+};
+
+static void es_panel_dcs_write(struct es_panel *ctx, const void *data,
+			       size_t len);
+
+#define es_panel_dcs_write_seq_static(ctx, seq...)         \
+	({                                                 \
+		static const u8 d[] = { seq };             \
+		es_panel_dcs_write(ctx, d, ARRAY_SIZE(d)); \
+		udelay(10);                                \
+	})
+
+#define es_panel_check_retval(x)     \
+	do {                         \
+		if ((x))             \
+			return -EIO; \
+	} while (0)
+
+// #ifdef CONFIG_1080P_4LANES_PANEL
+#define HFP (60)
+#define HSA (60)
+#define HBP (60)
+#define VFP (8)
+#define VSA (8)
+#define VBP (8)
+#define VAC (1920)
+#define HAC (1080)
+#define LANES 4
+#define PIX_CLK 148500
+// #elif CONFIG_720P_3LANES_PANEL
+// #define HFP (30)
+// #define HSA (30)
+// #define HBP (30)
+// #define VFP (30)
+// #define VSA (30)
+// #define VBP (30)
+// #define VAC (1280)
+// #define HAC (720)
+// #define LANES 3
+// #define PIX_CLK 59400
+// #else
+// #define HFP (60)
+// #define HSA (60)
+// #define HBP (60)
+// #define VFP (8)
+// #define VSA (8)
+// #define VBP (8)
+// #define VAC (1920)
+// #define HAC (1080)
+// #define LANES 4
+// #define PIX_CLK 148500
+// #endif
+
+static struct drm_display_mode default_mode = {
+	.clock = PIX_CLK,
+	.hdisplay = HAC,
+	.hsync_start = HAC + HFP,
+	.hsync_end = HAC + HFP + HSA,
+	.htotal = HAC + HFP + HSA + HBP,
+	.vdisplay = VAC,
+	.vsync_start = VAC + VFP,
+	.vsync_end = VAC + VFP + VSA,
+	.vtotal = VAC + VFP + VSA + VBP,
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+};
+
+void GP_COMMAD_PA(struct es_panel *ctx, u8 a)
+{
+	return;
+	// u8 b[3] = {0};
+	// b[0] = 0xBC;
+	// b[1] = a;
+	// b[2] = a >> 8;
+	// es_panel_dcs_write(ctx, b, 3);
+	// es_panel_dcs_write_seq_static(ctx, 0xBf);
+}
+
+static void free_cmd_buf(user_cmd_buffer_t *pcmd_buf)
+{
+	if (pcmd_buf->buf) {
+		pcmd_buf->buf_size = 0;
+		vfree(pcmd_buf->buf);
+		pcmd_buf->buf = NULL;
+	}
+	return;
+}
+
+void free_cmd_list(struct list_head *list)
+{
+	if (!list_empty(list)) {
+		user_cmd_buffer_t *pos = NULL, *n = NULL;
+		list_for_each_entry_safe(pos, n, list, head) {
+			if (pos) {
+				list_del_init(&pos->head);
+				if (pos->buf) {
+					vfree(pos->buf);
+					pos->buf = NULL;
+				}
+				vfree(pos);
+				pos = NULL;
+			}
+		}
+	}
+}
+
+static int parser_and_creat_cmd_buf(MIPI_CMD_S *cmd, user_cmd_buffer_t *cmd_buf)
+{
+	if (!cmd) {
+		dev_err(pr_dev, "cmd invalid\n");
+		return -EPERM;
+	}
+	if (!cmd_buf) {
+		dev_err(pr_dev, "cmd_buf invalid\n");
+		return -EPERM;
+	}
+
+	if (cmd_buf->buf) {
+		vfree(cmd_buf->buf);
+		cmd_buf->buf = NULL;
+	}
+
+	if (cmd->dataType == 0x13) {
+		cmd_buf->buf_size = 1;
+		cmd_buf->buf = vmalloc(cmd_buf->buf_size);
+		cmd_buf->buf[0] = cmd->cmdSize;
+	} else if (cmd->dataType == 0x23) {
+		cmd_buf->buf_size = 2;
+		cmd_buf->buf = vmalloc(cmd_buf->buf_size);
+		cmd_buf->buf[0] = cmd->cmdSize >> 8;
+		cmd_buf->buf[1] = cmd->cmdSize & 0x00ff;
+	} else if (cmd->dataType == 0x29) {
+		cmd_buf->buf_size = cmd->cmdSize;
+		cmd_buf->buf = vmalloc(cmd_buf->buf_size);
+		if (copy_from_user(cmd_buf->buf, (u8 __user *)cmd->pCmd,
+				   cmd_buf->buf_size)) {
+			dev_err(pr_dev, "cmd buffer copy_from_user failed\n");
+			return -ENOMEM;
+		}
+	}
+	cmd_buf->sleepUs = cmd->sleepUs;
+	dev_dbg(pr_dev, "user set cmd size:%d, sleep:%dus,cmd:0x%x \n",
+		cmd_buf->buf_size, cmd_buf->sleepUs, cmd_buf->buf[0]);
+	return 0;
+}
+
+int es_panel_fops_open(struct inode *inode, struct file *filp)
+{
+	struct es_panel *ctx;
+	ctx = container_of(inode->i_cdev, struct es_panel, cdev);
+	filp->private_data = ctx;
+	return 0;
+}
+
+int es_panel_fops_release(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static long es_panel_fops_ioctl(struct file *file, unsigned int cmd,
+				unsigned long arg)
+{
+	struct es_panel *ctx;
+	struct mipi_dsi_device *dsi;
+	MIPI_CMD_S ucmd;
+	int ret = 0;
+
+	ctx = (struct es_panel *)file->private_data;
+	dsi = to_mipi_dsi_device(ctx->dev);
+
+	switch (cmd) {
+	case IOC_ES_MIPI_TX_SET_MODE:
+		es_panel_check_retval(copy_from_user(
+			&ctx->mode_info, (MIPI_MODE_INFO_S __user *)arg,
+			sizeof(MIPI_MODE_INFO_S)));
+		dev_dbg(pr_dev,
+			"cmd: IOC_ES_MIPI_TX_SET_MODE, devId:%d, hdisplay:%u, hsyncStart:%u, hsyncEnd:%u, htotal:%u, hskew:%u\n"
+			"vdisplay:%u vsyncStart:%u vsyncEnd:%u vtotal:%u vscan:%u vrefresh:%d flags:%d type:%d "
+			"outputFormat:%d outputMode:%d lanes:%d\n",
+			ctx->mode_info.devId, ctx->mode_info.hdisplay,
+			ctx->mode_info.hsyncStart, ctx->mode_info.hsyncEnd,
+			ctx->mode_info.htotal, ctx->mode_info.hskew,
+			ctx->mode_info.vdisplay, ctx->mode_info.vsyncStart,
+			ctx->mode_info.vsyncEnd, ctx->mode_info.vtotal,
+			ctx->mode_info.vscan, ctx->mode_info.vrefresh,
+			ctx->mode_info.flags, ctx->mode_info.type,
+			ctx->mode_info.outputFormat, ctx->mode_info.outputMode,
+			ctx->mode_info.lanes);
+
+		memset(&ctx->user_mode, 0, sizeof(struct drm_display_mode));
+		ctx->user_mode.clock = 12500;
+		ctx->user_mode.hdisplay = ctx->mode_info.hdisplay;
+		ctx->user_mode.hsync_start = ctx->mode_info.hsyncStart;
+		ctx->user_mode.hsync_end = ctx->mode_info.hsyncEnd;
+		ctx->user_mode.htotal = ctx->mode_info.htotal;
+		ctx->user_mode.vdisplay = ctx->mode_info.vdisplay;
+		ctx->user_mode.vsync_start = ctx->mode_info.vsyncStart;
+		ctx->user_mode.vsync_end = ctx->mode_info.vsyncEnd;
+		ctx->user_mode.vtotal = ctx->mode_info.vtotal;
+		ctx->user_mode.width_mm = 36;
+		ctx->user_mode.height_mm = 49;
+		ctx->user_mode.flags = DRM_MODE_FLAG_NVSYNC |
+				       DRM_MODE_FLAG_NHSYNC;
+		ctx->user_mode_inited = true;
+
+		dsi->lanes = ctx->mode_info.lanes;
+		if (ctx->mode_info.outputFormat == MIPI_OUT_FORMAT_RGB_24_BIT) {
+			dsi->format = MIPI_DSI_FMT_RGB888;
+		} else {
+			dev_err(pr_dev, "panel not support fmt:%d now",
+				ctx->mode_info.outputFormat);
+		}
+		dsi->mode_flags = MIPI_DSI_MODE_VIDEO;
+
+		break;
+	case IOC_ES_MIPI_TX_SET_INIT_CMD:
+		user_cmd_buffer_t *cmd_buf = NULL;
+
+		if (ctx->init_cmd_writted) {
+			dev_info(pr_dev,
+				 "init_cmd_writted : clean init cmd list\n");
+			free_cmd_list(&ctx->init_cmd_list);
+			INIT_LIST_HEAD(&ctx->init_cmd_list);
+			ctx->init_cmd_writted = 0;
+		}
+		cmd_buf = vmalloc(sizeof(user_cmd_buffer_t));
+		if (!cmd_buf) {
+			dev_err(pr_dev, "cmd buffer alloc failed\n");
+			return -ENOMEM;
+		}
+		memset(cmd_buf, 0, sizeof(user_cmd_buffer_t));
+
+		es_panel_check_retval(copy_from_user(
+			&ucmd, (MIPI_CMD_S __user *)arg, sizeof(MIPI_CMD_S)));
+		ret = parser_and_creat_cmd_buf(&ucmd, cmd_buf);
+		if (ret) {
+			break;
+		}
+		if (!cmd_buf) {
+			dev_err(pr_dev, "cmd_buf creat failed!!!!");
+			break;
+		}
+		list_add_tail(&cmd_buf->head, &ctx->init_cmd_list);
+
+		break;
+	case IOC_ES_MIPI_TX_ENABLE:
+		user_cmd_buffer_t *enable_cmd_buf = &ctx->enable_cmd_buf;
+		dev_dbg(pr_dev, "IOC_ES_MIPI_TX_ENABLE In");
+		es_panel_check_retval(copy_from_user(
+			&ucmd, (MIPI_CMD_S __user *)arg, sizeof(MIPI_CMD_S)));
+		ret = parser_and_creat_cmd_buf(&ucmd, enable_cmd_buf);
+		break;
+	case IOC_ES_MIPI_TX_DISABLE:
+		user_cmd_buffer_t *disable_cmd_buf = &ctx->disable_cmd_buf;
+		dev_dbg(pr_dev, "IOC_ES_MIPI_TX_DISABLE In");
+		es_panel_check_retval(copy_from_user(
+			&ucmd, (MIPI_CMD_S __user *)arg, sizeof(MIPI_CMD_S)));
+		ret = parser_and_creat_cmd_buf(&ucmd, disable_cmd_buf);
+		break;
+	default:
+		dev_err(pr_dev, "unsupported command %d", cmd);
+		break;
+	}
+	return ret;
+}
+
+struct file_operations es_panel_fops = {
+	.owner = THIS_MODULE,
+	.open = es_panel_fops_open,
+	.release = es_panel_fops_release,
+	.unlocked_ioctl = es_panel_fops_ioctl,
+};
+
+int es_panel_chrdev_create(struct es_panel *ctx)
+{
+	int ret = 0;
+	dev_dbg(pr_dev, "---BEGIN es_panel_chrdev_create---\n");
+
+	ret = alloc_chrdev_region(&ctx->devt, panel_minor, panel_nr_devs,
+				  ES_PANEL_NAME);
+	if (ret < 0) {
+		dev_err(pr_dev,
+			"es-panel-chrdev: can't allocate major number\n");
+		goto fail;
+	}
+	panel_major = MAJOR(ctx->devt);
+
+	cdev_init(&ctx->cdev, &es_panel_fops);
+	ctx->cdev.owner = THIS_MODULE;
+
+	ret = cdev_add(&ctx->cdev, ctx->devt, 1);
+	if (ret) {
+		dev_err(pr_dev, "failed to add cdev\n");
+		goto unregister_chrdev;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+	ctx->es_panel_class = class_create(ES_PANEL_NAME);
+#else
+	ctx->es_panel_class = class_create(THIS_MODULE, ES_PANEL_NAME);
+#endif
+	if (IS_ERR(ctx->es_panel_class)) {
+		dev_err(pr_dev, "class_create error\n");
+		ret = PTR_ERR(ctx->es_panel_class);
+		goto del_cdev;
+	}
+
+	if (device_create(ctx->es_panel_class, NULL, ctx->devt, ctx,
+			  ES_PANEL_NAME) == NULL) {
+		dev_err(pr_dev, "device_create error\n");
+		ret = -EINVAL;
+		goto destroy_class;
+	}
+
+	dev_dbg(pr_dev, "---END es_panel_chrdev_create---\n");
+	return 0;
+
+destroy_class:
+	class_destroy(ctx->es_panel_class);
+del_cdev:
+	cdev_del(&ctx->cdev);
+unregister_chrdev:
+	unregister_chrdev_region(ctx->devt, panel_nr_devs);
+fail:
+	return ret;
+}
+
+void es_panel_chrdev_destroy(struct es_panel *ctx)
+{
+	device_destroy(ctx->es_panel_class, MKDEV(panel_major, panel_minor));
+	class_destroy(ctx->es_panel_class);
+
+	cdev_del(&ctx->cdev);
+
+	unregister_chrdev_region(ctx->devt, panel_nr_devs);
+	dev_dbg(pr_dev, "es_panel_chrdev_destroy done\n");
+}
+
+static inline struct es_panel *panel_to_es_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct es_panel, panel);
+}
+
+static void es_panel_dcs_write(struct es_panel *ctx, const void *data,
+			       size_t len)
+{
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	ssize_t ret;
+	char *addr;
+
+	if (ctx->error < 0)
+		return;
+
+	addr = (char *)data;
+	if ((int)*addr < 0xB0)
+		ret = mipi_dsi_dcs_write_buffer(dsi, data, len);
+	else
+		ret = mipi_dsi_generic_write(dsi, data, len);
+	if (ret < 0) {
+		dev_err(ctx->dev, "error %zd writing seq: %ph\n", ret, data);
+		ctx->error = ret;
+	}
+	dev_dbg(ctx->dev, "write cmd size:%ld, data0[0x%x]  ret:%d\n", len,
+		addr[0], ctx->error);
+}
+
+#ifdef PANEL_SUPPORT_READBACK
+static int es_panel_dcs_read(struct es_panel *ctx, u8 cmd, void *data,
+			     size_t len)
+{
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	ssize_t ret;
+
+	if (ctx->error < 0)
+		return 0;
+
+	ret = mipi_dsi_dcs_read(dsi, cmd, data, len);
+	if (ret < 0) {
+		dev_err(ctx->dev, "error %d reading dcs seq:(%#x)\n", ret, cmd);
+		ctx->error = ret;
+	}
+
+	return ret;
+}
+
+static void es_panel_panel_get_data(struct es_panel *ctx)
+{
+	u8 buffer[3] = { 0 };
+	static int ret;
+
+	if (ret == 0) {
+		ret = es_panel_dcs_read(ctx, 0x0A, buffer, 1);
+		dev_info(ctx->dev, "return %d data(0x%08x) to dsi engine\n",
+			 ret, buffer[0] | (buffer[1] << 8));
+	}
+}
+#endif
+
+void es_panel_panel_init(struct es_panel *ctx)
+{
+	user_cmd_buffer_t *pos = NULL;
+	list_for_each_entry(pos, &ctx->init_cmd_list, head) {
+		if (pos) {
+			es_panel_dcs_write(ctx, pos->buf, pos->buf_size);
+			if (pos->sleepUs) {
+				msleep(pos->sleepUs / 1000);
+			}
+		}
+	}
+	ctx->init_cmd_writted = 1;
+}
+
+void es_panel_3_lanes_panel_init(struct es_panel *ctx)
+{
+	dev_info(ctx->dev, "init 3lanes panel\n");
+	GP_COMMAD_PA(ctx, 0x04);
+	es_panel_dcs_write_seq_static(ctx, 0xB9, 0xF1, 0x12, 0x83);
+	GP_COMMAD_PA(ctx, 0x1C);
+	es_panel_dcs_write_seq_static(ctx, 0xBA, 0x32, 0x81, 0x05, 0xF9, 0x0E,
+				      0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x44, 0x25, 0x00, 0x91, 0x0A,
+				      0x00, 0x00, 0x02, 0x4F, 0xD1, 0x00, 0x00,
+				      0x37);
+	GP_COMMAD_PA(ctx, 0x05);
+	es_panel_dcs_write_seq_static(ctx, 0xB8, 0x26, 0x22, 0x20, 0x00);
+	GP_COMMAD_PA(ctx, 0x04);
+	es_panel_dcs_write_seq_static(ctx, 0xBF, 0x02, 0x11, 0x00);
+	GP_COMMAD_PA(ctx, 0x0B);
+	es_panel_dcs_write_seq_static(ctx, 0xB3, 0x0C, 0x10, 0x0A, 0x50, 0x03,
+				      0xFF, 0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x0A);
+	es_panel_dcs_write_seq_static(ctx, 0xB3, 0x0C, 0x10, 0x0A, 0x50, 0x03,
+				      0xFF, 0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x02);
+	es_panel_dcs_write_seq_static(ctx, 0xBC, 0x46);
+	GP_COMMAD_PA(ctx, 0x02);
+	es_panel_dcs_write_seq_static(ctx, 0xCC, 0x0B);
+	GP_COMMAD_PA(ctx, 0x02);
+	es_panel_dcs_write_seq_static(ctx, 0xB4, 0x80);
+	GP_COMMAD_PA(ctx, 0x04);
+	es_panel_dcs_write_seq_static(ctx, 0xB2, 0xC8, 0x02, 0x30);
+	GP_COMMAD_PA(ctx, 0x0F);
+	es_panel_dcs_write_seq_static(ctx, 0xE3, 0x07, 0x07, 0x0B, 0x0B, 0x03,
+				      0x0B, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x80,
+				      0xC0, 0x10);
+	GP_COMMAD_PA(ctx, 0x0D);
+	es_panel_dcs_write_seq_static(ctx, 0xC1, 0x25, 0x00, 0x1E, 0x1E, 0x77,
+				      0xF1, 0xFF, 0xFF, 0xCC, 0xCC, 0x77, 0x77);
+	GP_COMMAD_PA(ctx, 0x03);
+	es_panel_dcs_write_seq_static(ctx, 0xB5, 0x0A, 0x0A);
+	GP_COMMAD_PA(ctx, 0x03);
+	es_panel_dcs_write_seq_static(ctx, 0xB6, 0x50, 0x50);
+	GP_COMMAD_PA(ctx, 0x40);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xE9, 0xC4, 0x10, 0x0F, 0x00, 0x00, 0xB2, 0xB8, 0x12, 0x31,
+		0x23, 0x48, 0x8B, 0xB2, 0xB8, 0x47, 0x20, 0x00, 0x00, 0x30,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x02,
+		0x46, 0x02, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0xF8,
+		0x13, 0x57, 0x13, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88,
+		0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x3E);
+	es_panel_dcs_write_seq_static(ctx, 0xEA, 0x00, 0x1A, 0x00, 0x00, 0x00,
+				      0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x75, 0x31, 0x31, 0x88, 0x88, 0x88,
+				      0x88, 0x88, 0x88, 0x88, 0x8F, 0x64, 0x20,
+				      0x20, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88,
+				      0x88, 0x8F, 0x23, 0x10, 0x00, 0x00, 0x02,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x30, 0x02, 0xA0, 0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x23);
+	es_panel_dcs_write_seq_static(ctx, 0xE0, 0x01, 0x11, 0x16, 0x28, 0x30,
+				      0x38, 0x45, 0x39, 0x07, 0x0B, 0x0D, 0x12,
+				      0x14, 0x12, 0x14, 0x11, 0x19, 0x01, 0x11,
+				      0x16, 0x28, 0x30, 0x38, 0x45, 0x39, 0x07,
+				      0x0B, 0x0D, 0x12, 0x14, 0x12, 0x14, 0x11,
+				      0x19);
+
+	es_panel_dcs_write_seq_static(ctx, 0xC7, 0xE7, 0x00);
+	// es_panel_dcs_write_seq_static(ctx,0xC8, 0x10, 0x40);
+	es_panel_dcs_write_seq_static(ctx, 0x51,
+				      0xFF); // brightness [0x00-> 0xFF]
+	es_panel_dcs_write_seq_static(ctx, 0x53, (1 << 2) | (1 << 5));
+	msleep(20);
+}
+
+void es_panel_4_lanes_panel_init(struct es_panel *ctx)
+{
+	dev_info(ctx->dev, "init 4lanes panel\n");
+	es_panel_dcs_write_seq_static(ctx, 0xB0, 0x98, 0x85, 0x0A);
+	es_panel_dcs_write_seq_static(ctx, 0xC1, 0x00, 0x00,
+				      0x00); // 01反扫  00正扫
+	es_panel_dcs_write_seq_static(ctx, 0xC4, 0x70, 0x19, 0x23, 0x00, 0x0F,
+				      0x0F, 0x00);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xD0, 0x33, 0x05, 0x21, 0xE7, 0x62, 0x00, 0x88, 0xA0, 0xA0,
+		0x03, 0x02, 0x80, 0xD0, 0x1B,
+		0x10); // VGH dual mode,      VGL single mode,      VGH=12V,      VGL=-12V
+	es_panel_dcs_write_seq_static(ctx, 0xD2, 0x13, 0x13, 0xEA, 0x22);
+	es_panel_dcs_write_seq_static(ctx, 0xD1, 0x09, 0x09,
+				      0xc2); ////4003 & 4003B EN
+	es_panel_dcs_write_seq_static(ctx, 0xD3, 0x44, 0x33, 0x05, 0x03, 0x4A,
+				      0x4A, 0x33, 0x17, 0x22, 0x43,
+				      0x6E); // set GVDDP=4.1V, GVDDN=-4.1V,
+	// VGHO=12V,      VGLO=-11V
+	es_panel_dcs_write_seq_static(ctx, 0xD5, 0x8B, 0x00, 0x00, 0x00, 0x01,
+				      0x7D, 0x01, 0x7D, 0x01, 0x7D, 0x00, 0x00,
+				      0x00, 0x00); // set VCOM
+	es_panel_dcs_write_seq_static(ctx, 0xD6, 0x00, 0x00, 0x08, 0x17, 0x23,
+				      0x65, 0x77, 0x44, 0x87, 0x00, 0x00,
+				      0x09); // P12_D[3] for sleep in
+	// current reduce setting
+	es_panel_dcs_write_seq_static(ctx, 0xEC, 0x76, 0x1E, 0x32, 0x00, 0x46,
+				      0x00, 0x00);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xC7, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x2D, 0x00, 0x43, 0x00,
+		0x58, 0x00, 0x6C, 0x00, 0x75, 0x00, 0x8C, 0x00, 0x9A, 0x00,
+		0xCA, 0x00, 0xF1, 0x01, 0x2F, 0x01, 0x5F, 0x01, 0xAE, 0x01,
+		0xEC, 0x01, 0xEE, 0x02, 0x25, 0x02, 0x62, 0x02, 0x8A, 0x02,
+		0xC4, 0x02, 0xEA, 0x03, 0x1F, 0x03, 0x33, 0x03, 0x3E, 0x03,
+		0x59, 0x03, 0x70, 0x03, 0x88, 0x03, 0xB4, 0x03, 0xC8, 0x03,
+		0xE8, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x2D, 0x00, 0x43, 0x00,
+		0x58, 0x00, 0x6C, 0x00, 0x75, 0x00, 0x8C, 0x00, 0x9A, 0x00,
+		0xCA, 0x00, 0xF1, 0x01, 0x2F, 0x01, 0x5F, 0x01, 0xAE, 0x01,
+		0xEC, 0x01, 0xEE, 0x02, 0x25, 0x02, 0x62, 0x02, 0x8A, 0x02,
+		0xC4, 0x02, 0xEA, 0x03, 0x1F, 0x03, 0x33, 0x03, 0x3E, 0x03,
+		0x59, 0x03, 0x70, 0x03, 0x88, 0x03, 0xB4, 0x03, 0xC8, 0x03,
+		0xE8, 0x01, 0x01);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xE5, 0x6D, 0x92, 0x80, 0x34, 0x76, 0x12, 0x12, 0x00, 0x00,
+		0x36, 0x36, 0x24, 0x24, 0x26, 0xF6, 0xF6, 0xF6, 0xF6, 0xF6,
+		0xF6, 0x6D, 0x9B, 0x89, 0x34, 0x76, 0x1B, 0x1B, 0x09, 0x09,
+		0x3F, 0x3F, 0x2D, 0x2D, 0x26, 0xF6, 0xF6, 0xF6, 0xF6, 0xF6,
+		0xF6, 0x04, 0x40, 0x90, 0x00, 0xD6, 0x00, 0x00, 0x00, 0x00,
+		0x01, 0x01, 0x44, 0x00, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x03, 0x03,
+		0x03, 0x03, 0x00, 0x07);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xEA, 0x70, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x40, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x89,
+		0x8a, 0x05, 0x05, 0x22, 0x0a, 0x0a, 0x0b, 0x00, 0x08, 0x00,
+		0x30, 0x01, 0x09, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x00, 0x01,
+		0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xDD, 0x22, 0x22,
+		0xCC, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0xCC, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33);
+	es_panel_dcs_write_seq_static(ctx, 0xEE, 0x22, 0x10, 0x02, 0x02, 0x0F,
+				      0x40, 0x00, 0x07, 0x00, 0x04, 0x00, 0x00,
+				      0x00, 0xB9, 0x77, 0x00, 0x55, 0x05,
+				      0x00); // 4003
+	es_panel_dcs_write_seq_static(ctx, 0xEB, 0xa3, 0xcf, 0x73, 0x18, 0x54,
+				      0x55, 0x55, 0x55, 0x55, 0x00, 0x55, 0x55,
+				      0x55, 0x55, 0x55, 0x25, 0x0D, 0x0F, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x55, 0x55, 0x55,
+				      0x55, 0x32, 0x3E, 0x55, 0x43, 0x55);
+
+	es_panel_dcs_write_seq_static(ctx, 0x51, 0x0F,
+				      0xFF); // brightness [0x00-> 0xFFF]
+	es_panel_dcs_write_seq_static(ctx, 0x53, (1 << 2) | (1 << 5));
+}
+
+static int es_panel_disable(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	int ret = 0;
+	user_cmd_buffer_t *cmd_buf = &ctx->disable_cmd_buf;
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(panel->dev);
+
+	dev_dbg(pr_dev, "%s:%d size:%x ctx->enabled:%d\n", __func__, __LINE__,
+		ctx->disable_cmd_buf.buf_size, ctx->enabled);
+
+	if (!ctx->enabled)
+		return 0;
+
+	if (ctx->backlight) {
+		ctx->backlight->props.power = FB_BLANK_POWERDOWN;
+		backlight_update_status(ctx->backlight);
+	}
+#if 1
+	gpiod_set_value(ctx->gpio_backlight0, 0);
+#endif
+
+	ctx->enabled = false;
+	if (cmd_buf->buf_size > 0) {
+		if (cmd_buf->sleepUs) {
+			msleep(cmd_buf->sleepUs / 1000);
+		}
+		es_panel_dcs_write(ctx, cmd_buf->buf, cmd_buf->buf_size);
+	} else {
+		dev_warn(pr_dev, "%s[%d]:Disable cmd invalid,Please set it\n",
+			 __func__, __LINE__);
+	}
+
+	dev_info(pr_dev, "[%s]display off... sleep in..\n", __func__);
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0)
+		dev_err(ctx->dev, "Failed to turn off the display: %d\n", ret);
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0)
+		dev_err(ctx->dev, "Failed to enter sleep mode: %d\n", ret);
+
+	return ret;
+}
+
+static int es_panel_unprepare(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+
+	if (!ctx->prepared)
+		return 0;
+
+	ctx->error = 0;
+	dev_dbg(pr_dev, "%s:%d\n", __func__, __LINE__);
+	ctx->prepared = false;
+	return 0;
+}
+
+static int es_panel_prepare(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	int ret;
+
+	if (ctx->prepared)
+		return 0;
+	dev_dbg(pr_dev, "%s:%d\n", __func__, __LINE__);
+
+	// #ifdef CONFIG_1080P_4LANES_PANEL
+	es_panel_4_lanes_panel_init(ctx);
+	// #elif CONFIG_720P_3LANES_PANEL
+	//     es_panel_3_lanes_panel_init(ctx);
+	// #else
+	//     es_panel_panel_init(ctx);
+	// #endif
+
+	ret = ctx->error;
+	if (ret < 0)
+		es_panel_unprepare(panel);
+	ctx->prepared = true;
+#ifdef PANEL_SUPPORT_READBACK
+	es_panel_panel_get_data(ctx);
+#endif
+
+	return ret;
+}
+
+static int es_panel_enable(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	int ret = 0;
+	user_cmd_buffer_t *cmd_buf = &ctx->enable_cmd_buf;
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(panel->dev);
+
+	if (ctx->enabled)
+		return 0;
+
+	if (ctx->backlight) {
+		ctx->backlight->props.power = FB_BLANK_UNBLANK;
+		backlight_update_status(ctx->backlight);
+	}
+#if 1
+	gpiod_set_value(ctx->gpio_backlight0, 1);
+#endif
+
+	ctx->enabled = true;
+	if (cmd_buf->buf_size > 0) {
+		if (cmd_buf->sleepUs) {
+			msleep(cmd_buf->sleepUs / 1000);
+		}
+		es_panel_dcs_write(ctx, cmd_buf->buf, cmd_buf->buf_size);
+	} else {
+		dev_warn(pr_dev, "%s[%d]:Enable cmd invalid,Please set it\n",
+			 __func__, __LINE__);
+	}
+
+	dev_info(pr_dev, "[%s] sleep out.., display on...\n", __func__);
+	msleep(20);
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(ctx->dev, "Failed to exit sleep mode: %d\n", ret);
+		return ret;
+	}
+	/* Panel is operational 120 msec after reset */
+	msleep(200);
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret)
+		return ret;
+	msleep(50);
+
+	return ret;
+}
+
+static int es_panel_get_modes(struct drm_panel *panel,
+			      struct drm_connector *connector)
+{
+	u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	struct drm_display_mode *mode;
+
+	dev_dbg(pr_dev, "%s:%d\n", __func__, __LINE__);
+	if (ctx->user_mode_inited == false) {
+		mode = drm_mode_duplicate(connector->dev, &default_mode);
+		if (!mode) {
+			dev_err(panel->dev, "failed to add mode %ux%ux\n",
+				default_mode.hdisplay, default_mode.vdisplay);
+			return -ENOMEM;
+		}
+	} else {
+		mode = drm_mode_duplicate(connector->dev, &ctx->user_mode);
+		if (!mode) {
+			dev_err(panel->dev, "failed to add mode %ux%ux\n",
+				ctx->user_mode.hdisplay,
+				ctx->user_mode.vdisplay);
+			return -ENOMEM;
+		}
+	}
+	dev_dbg(pr_dev,
+		"get modes user_mode_inited:%d, clock:%d, hdisplay:%u, hsyncStart:%u, hsyncEnd:%u, htotal:%u\n"
+		"vdisplay:%u vsyncStart:%u vsyncEnd:%u vtotal:%u width_mm:%d, height_mm:%d flags:0x%08x \n",
+		ctx->user_mode_inited, mode->clock, mode->hdisplay,
+		mode->hsync_start, mode->hsync_end, mode->htotal,
+		mode->vdisplay, mode->vsync_start, mode->vsync_end,
+		mode->vtotal, mode->width_mm, mode->height_mm, mode->flags);
+
+	drm_mode_set_name(mode);
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	drm_mode_probed_add(connector, mode);
+
+	connector->display_info.width_mm = ctx->user_mode.width_mm;
+	connector->display_info.height_mm = ctx->user_mode.height_mm;
+
+	if (ctx->format == MIPI_DSI_FMT_RGB888) {
+		bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	} else if (ctx->format == MIPI_DSI_FMT_RGB666) {
+		bus_format = MEDIA_BUS_FMT_RGB666_1X18;
+	} else if (ctx->format == MIPI_DSI_FMT_RGB565) {
+		bus_format = MEDIA_BUS_FMT_RGB565_1X16;
+	} else {
+		dev_err(pr_dev, "format:%d not support\n", ctx->format);
+	}
+	drm_display_info_set_bus_formats(&connector->display_info, &bus_format,
+					 1);
+	connector->display_info.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
+
+	return 1;
+}
+
+static const struct drm_panel_funcs es_panel_drm_funcs = {
+	.disable = es_panel_disable,
+	.unprepare = es_panel_unprepare,
+	.prepare = es_panel_prepare,
+	.enable = es_panel_enable,
+	.get_modes = es_panel_get_modes,
+};
+
+int es_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct es_panel *ctx;
+	int ret;
+	struct device_node *dsi_node, *remote_node = NULL, *endpoint = NULL;
+	int val;
+	// for print
+	pr_dev = dev;
+	dev_info(pr_dev, "[%s] Enter\n", __func__);
+
+	dsi_node = of_get_parent(dev->of_node);
+	if (dsi_node) {
+		endpoint = of_graph_get_next_endpoint(dsi_node, NULL);
+		if (endpoint) {
+			remote_node = of_graph_get_remote_port_parent(endpoint);
+			if (!remote_node) {
+				dev_err(pr_dev,
+					"No panel connected,skip probe es_panel\n");
+				return -ENODEV;
+			}
+			dev_info(pr_dev, "device node name:%s\n",
+				 remote_node->name);
+		}
+	}
+
+	ctx = devm_kzalloc(dev, sizeof(struct es_panel), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, ctx);
+	ctx->dev = dev;
+
+	// default val, if other value, set it on:IOC_ES_MIPI_TX_SET_MODE.
+	if (!of_property_read_u32(dev->of_node, "dsi,format", &val))
+		dsi->format = val;
+	if (!of_property_read_u32(dev->of_node, "dsi,lanes", &val))
+		dsi->lanes = val;
+	ctx->format = dsi->format;
+	ctx->lanes = dsi->lanes;
+
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM |
+			  MIPI_DSI_MODE_VIDEO_SYNC_PULSE;
+
+	ctx->gpio_backlight0 =
+		devm_gpiod_get(ctx->dev, "backlight0", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpio_backlight0)) {
+		dev_err(ctx->dev, "failed to get backlight0 gpio, err: %ld\n",
+			PTR_ERR(ctx->gpio_backlight0));
+		return PTR_ERR(ctx->gpio_backlight0);
+	}
+
+	ctx->gpio_reset = devm_gpiod_get(ctx->dev, "rst", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpio_reset)) {
+		dev_err(ctx->dev, "failed to get rst gpio, err: %ld\n",
+			PTR_ERR(ctx->gpio_reset));
+		return PTR_ERR(ctx->gpio_reset);
+	}
+
+	msleep(50);
+
+	gpiod_set_value(ctx->gpio_reset, 1);
+	gpiod_set_value(ctx->gpio_backlight0, 1);
+
+	ctx->prepared = false;
+	ctx->enabled = false;
+	ctx->user_mode_inited = false;
+
+	drm_panel_init(&ctx->panel, dev, &es_panel_drm_funcs,
+		       DRM_MODE_CONNECTOR_DPI);
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0)
+		drm_panel_remove(&ctx->panel);
+
+	// es_panel_chrdev_create(ctx);
+	INIT_LIST_HEAD(&ctx->init_cmd_list);
+	ctx->init_cmd_writted = 0;
+	memset(&ctx->enable_cmd_buf, 0, sizeof(user_cmd_buffer_t));
+	memset(&ctx->disable_cmd_buf, 0, sizeof(user_cmd_buffer_t));
+	dev_info(pr_dev, "[%s] Leave\n", __func__);
+	return ret;
+}
+
+void es_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct es_panel *ctx = mipi_dsi_get_drvdata(dsi);
+
+	free_cmd_list(&ctx->init_cmd_list);
+	free_cmd_buf(&ctx->enable_cmd_buf);
+	free_cmd_buf(&ctx->disable_cmd_buf);
+
+	devm_gpiod_put(ctx->dev, ctx->gpio_backlight0);
+	devm_gpiod_put(ctx->dev, ctx->gpio_reset);
+
+	// es_panel_chrdev_destroy(ctx);
+	mipi_dsi_detach(dsi);
+	drm_panel_remove(&ctx->panel);
+
+	return;
+}
+
+static const struct of_device_id es_panel_of_match[] = {
+	{
+		.compatible = "eswin,generic-panel",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, es_panel_of_match);
+
+struct mipi_dsi_driver es_panel_driver = {
+    .probe = es_panel_probe,
+    .remove = es_panel_remove,
+    .driver =
+        {
+            .name = "es-panel",
+            .of_match_table = es_panel_of_match,
+        },
+};
+
+MODULE_AUTHOR("Eswin");
+MODULE_DESCRIPTION("Eswin Generic Panel Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_panel.h b/drivers/gpu/drm/eswin/es_panel.h
new file mode 100644
index 000000000000..093ac739e5e9
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_panel.h
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef _ES_PANEL_H__
+#define _ES_PANEL_H__
+
+#include <drm/drm_mipi_dsi.h>
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+enum {
+	IOC_ES_MIPI_TX_SET_MODE = 0x100,
+	IOC_ES_MIPI_TX_SET_INIT_CMD,
+	IOC_ES_MIPI_TX_SET_CMD,
+	IOC_ES_MIPI_TX_ENABLE,
+	IOC_ES_MIPI_TX_DISABLE,
+};
+
+typedef enum MIPI_OUT_FORMAT_E {
+	MIPI_OUT_FORMAT_RGB_16_BIT = 0x0,
+	MIPI_OUT_FORMAT_RGB_18_BIT,
+	MIPI_OUT_FORMAT_RGB_24_BIT,
+	MIPI_OUT_FORMAT_BUTT
+} MIPI_OUT_FORMAT_E;
+
+typedef enum MIPI_OUT_MODE_E {
+	MIPI_OUTPUT_MODE_CSI = 0x0,
+	MIPI_OUTPUT_MODE_DSI_VIDEO,
+	MIPI_OUTPUT_MODE_DSI_CMD,
+	MIPI_OUTPUT_MODE_DSI_BUTT
+} MIPI_OUT_MODE_E;
+
+typedef struct MIPI_MODE_INFO {
+	u32 devId;
+	u16 hdisplay;
+	u16 hsyncStart;
+	u16 hsyncEnd;
+	u16 htotal;
+	u16 hskew;
+	u16 vdisplay;
+	u16 vsyncStart;
+	u16 vsyncEnd;
+	u16 vtotal;
+	u16 vscan;
+
+	u32 vrefresh;
+	u32 flags;
+	u32 type;
+
+	MIPI_OUT_FORMAT_E outputFormat;
+	MIPI_OUT_MODE_E outputMode;
+	u32 lanes;
+} MIPI_MODE_INFO_S;
+
+typedef struct MIPI_CMD_S {
+	int devId;
+	u16 dataType;
+	u16 cmdSize;
+	u32 sleepUs;
+	u8 *pCmd;
+} MIPI_CMD_S;
+
+extern struct mipi_dsi_driver es_panel_driver;
+void es_panel_remove(struct mipi_dsi_device *dsi);
+int es_panel_probe(struct mipi_dsi_device *dsi);
+
+#endif //_ES_PANEL_H__
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/es_plane.c b/drivers/gpu/drm/eswin/es_plane.c
new file mode 100644
index 000000000000..e2e01ac5fa3f
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_plane.c
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_blend.h>
+
+#include "es_drm.h"
+#include "es_type.h"
+#include "es_crtc.h"
+#include "es_plane.h"
+#include "es_gem.h"
+#include "es_fb.h"
+
+void es_plane_destory(struct drm_plane *plane)
+{
+	struct es_plane *es_plane = to_es_plane(plane);
+
+	drm_plane_cleanup(plane);
+	kfree(es_plane);
+}
+
+static void es_plane_reset(struct drm_plane *plane)
+{
+	struct es_plane_state *state;
+
+	if (plane->state) {
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+
+		state = to_es_plane_state(plane->state);
+		kfree(state);
+		plane->state = NULL;
+	}
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state == NULL)
+		return;
+
+	__drm_atomic_helper_plane_reset(plane, &state->base);
+
+	state->degamma = ES_DEGAMMA_DISABLE;
+	state->degamma_changed = false;
+	memset(&state->status, 0, sizeof(state->status));
+}
+
+static void _es_plane_duplicate_blob(struct es_plane_state *state,
+				     struct es_plane_state *ori_state)
+{
+	state->roi = ori_state->roi;
+	state->color_mgmt = ori_state->color_mgmt;
+	if (state->roi)
+		drm_property_blob_get(state->roi);
+	if (state->color_mgmt)
+		drm_property_blob_get(state->color_mgmt);
+}
+
+static struct drm_plane_state *
+es_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct es_plane_state *ori_state;
+	struct es_plane_state *state;
+
+	if (WARN_ON(!plane->state))
+		return NULL;
+
+	ori_state = to_es_plane_state(plane->state);
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &state->base);
+
+	state->degamma = ori_state->degamma;
+	state->degamma_changed = ori_state->degamma_changed;
+
+	_es_plane_duplicate_blob(state, ori_state);
+	memcpy(&state->status, &ori_state->status, sizeof(ori_state->status));
+
+	return &state->base;
+}
+
+static void es_plane_atomic_destroy_state(struct drm_plane *plane,
+					  struct drm_plane_state *state)
+{
+	struct es_plane_state *es_plane_state = to_es_plane_state(state);
+
+	__drm_atomic_helper_plane_destroy_state(state);
+	drm_property_blob_put(es_plane_state->roi);
+	drm_property_blob_put(es_plane_state->color_mgmt);
+	kfree(es_plane_state);
+}
+
+static int _es_plane_set_property_blob_from_id(struct drm_device *dev,
+					       struct drm_property_blob **blob,
+					       u64 blob_id,
+					       size_t expected_size)
+{
+	struct drm_property_blob *new_blob = NULL;
+
+	if (blob_id) {
+		new_blob = drm_property_lookup_blob(dev, blob_id);
+		if (!new_blob) {
+			return -EINVAL;
+		}
+
+		if (new_blob->length != expected_size) {
+			drm_property_blob_put(new_blob);
+			return -EINVAL;
+		}
+	}
+	drm_property_replace_blob(blob, new_blob);
+	drm_property_blob_put(new_blob);
+	return 0;
+}
+
+static int es_plane_atomic_set_property(struct drm_plane *plane,
+					struct drm_plane_state *state,
+					struct drm_property *property,
+					uint64_t val)
+{
+	struct drm_device *dev = plane->dev;
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct es_plane_state *es_plane_state = to_es_plane_state(state);
+	int ret = 0;
+
+	if (property == es_plane->degamma_mode) {
+		if (es_plane_state->degamma != val) {
+			es_plane_state->degamma = val;
+			es_plane_state->degamma_changed = true;
+		} else {
+			es_plane_state->degamma_changed = false;
+		}
+	} else if (property == es_plane->roi_prop) {
+		ret = _es_plane_set_property_blob_from_id(
+			dev, &es_plane_state->roi, val,
+			sizeof(struct drm_es_roi));
+		return ret;
+	} else if (property == es_plane->color_mgmt_prop) {
+		ret = _es_plane_set_property_blob_from_id(
+			dev, &es_plane_state->color_mgmt, val,
+			sizeof(struct drm_es_color_mgmt));
+		return ret;
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int es_plane_atomic_get_property(struct drm_plane *plane,
+					const struct drm_plane_state *state,
+					struct drm_property *property,
+					uint64_t *val)
+{
+	struct es_plane *es_plane = to_es_plane(plane);
+	const struct es_plane_state *es_plane_state =
+		container_of(state, const struct es_plane_state, base);
+
+	if (property == es_plane->degamma_mode)
+		*val = es_plane_state->degamma;
+	else if (property == es_plane->roi_prop)
+		*val = (es_plane_state->roi) ? es_plane_state->roi->base.id : 0;
+	else if (property == es_plane->color_mgmt_prop)
+		*val = (es_plane_state->color_mgmt) ?
+			       es_plane_state->color_mgmt->base.id :
+			       0;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static bool es_format_mod_supported(struct drm_plane *plane, u32 format,
+				    u64 modifier)
+{
+	int i;
+
+	/* We always have to allow these modifiers:
+     * 1. Core DRM checks for LINEAR support if userspace does not provide
+ modifiers.
+     * 2. Not passing any modifiers is the same as explicitly passing
+ INVALID.
+     */
+	if (modifier == DRM_FORMAT_MOD_LINEAR)
+		return true;
+
+	/* Check that the modifier is on the list of the plane's supported
+ modifiers. */
+	for (i = 0; i < plane->modifier_count; i++) {
+		if (modifier == plane->modifiers[i])
+			break;
+	}
+
+	if (i == plane->modifier_count)
+		return false;
+
+	return true;
+}
+
+const struct drm_plane_funcs es_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = es_plane_destory,
+	.reset = es_plane_reset,
+	.atomic_duplicate_state = es_plane_atomic_duplicate_state,
+	.atomic_destroy_state = es_plane_atomic_destroy_state,
+	.atomic_set_property = es_plane_atomic_set_property,
+	.atomic_get_property = es_plane_atomic_get_property,
+	.format_mod_supported = es_format_mod_supported,
+};
+
+static unsigned char es_get_plane_number(struct drm_framebuffer *fb)
+{
+	const struct drm_format_info *info;
+
+	if (!fb)
+		return 0;
+
+	info = drm_format_info(fb->format->format);
+	if (!info || info->num_planes > MAX_NUM_PLANES)
+		return 0;
+
+	return info->num_planes;
+}
+
+static int es_plane_atomic_check(struct drm_plane *plane,
+				 struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state =
+		drm_atomic_get_new_plane_state(state, plane);
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct drm_framebuffer *fb = new_plane_state->fb;
+	struct drm_crtc *crtc = new_plane_state->crtc;
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct drm_device *dev = plane->dev;
+	struct es_drm_private *priv = dev->dev_private;
+
+	if (!crtc || !fb)
+		return 0;
+
+	/* Reject FB with misaligned pitches, which seem to crash the DC */
+	if (fb->format->num_planes == 1 &&
+	    fb->pitches[0] & (priv->pitch_alignment - 1))
+		return -EINVAL;
+
+	return es_plane->funcs->check(es_crtc->dev, es_plane, new_plane_state);
+}
+
+static void es_plane_atomic_update(struct drm_plane *plane,
+				   struct drm_atomic_state *old_state)
+{
+	unsigned char i, num_planes;
+	struct drm_framebuffer *fb;
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct drm_plane_state *state = plane->state;
+	struct es_crtc *es_crtc = to_es_crtc(state->crtc);
+	struct es_plane_state *plane_state = to_es_plane_state(state);
+	// struct drm_format_name_buf *name = &plane_state->status.format_name;
+
+	if (!state->fb || !state->crtc)
+		return;
+
+	fb = state->fb;
+
+	num_planes = es_get_plane_number(fb);
+
+	for (i = 0; i < num_planes; i++) {
+		struct es_gem_object *es_obj;
+
+		es_obj = es_fb_get_gem_obj(fb, i);
+		es_plane->dma_addr[i] = es_obj->iova + fb->offsets[i];
+	}
+
+	plane_state->status.tile_mode = 0; /* to be updated */
+	plane_state->status.src = drm_plane_state_src(state);
+	plane_state->status.dest = drm_plane_state_dest(state);
+	// drm_get_format_name(fb->format->format, name);
+
+	es_plane->funcs->update(es_crtc->dev, es_plane);
+}
+
+static void es_plane_atomic_disable(struct drm_plane *plane,
+				    struct drm_atomic_state *state)
+{
+	struct drm_plane_state *old_plane_state =
+		drm_atomic_get_old_plane_state(state, plane);
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct es_crtc *es_crtc = to_es_crtc(old_plane_state->crtc);
+
+	if (!old_plane_state->crtc)
+		return;
+
+	if (es_plane->funcs && es_plane->funcs->disable)
+		es_plane->funcs->disable(es_crtc->dev, es_plane);
+}
+
+const struct drm_plane_helper_funcs es_plane_helper_funcs = {
+	.atomic_check = es_plane_atomic_check,
+	.atomic_update = es_plane_atomic_update,
+	.atomic_disable = es_plane_atomic_disable,
+};
+
+static const struct drm_prop_enum_list es_degamma_mode_enum_list[] = {
+	{ ES_DEGAMMA_DISABLE, "disabled" },
+	{ ES_DEGAMMA_BT709, "preset degamma for BT709" },
+	{ ES_DEGAMMA_BT2020, "preset degamma for BT2020" },
+};
+
+struct es_plane *es_plane_create(struct drm_device *drm_dev,
+				 struct es_plane_info *info,
+				 unsigned int possible_crtcs)
+{
+	struct es_plane *plane;
+	int ret;
+
+	if (!info)
+		return NULL;
+
+	plane = kzalloc(sizeof(struct es_plane), GFP_KERNEL);
+	if (!plane)
+		return NULL;
+
+	plane->id = info->id;
+
+	ret = drm_universal_plane_init(drm_dev, &plane->base, possible_crtcs,
+				       &es_plane_funcs, info->formats,
+				       info->num_formats, info->modifiers,
+				       info->type,
+				       info->name ? info->name : NULL);
+	if (ret)
+		goto err_free_plane;
+
+	drm_plane_helper_add(&plane->base, &es_plane_helper_funcs);
+
+	/* Set up the plane properties */
+	if (info->degamma_size) {
+		plane->degamma_mode = drm_property_create_enum(
+			drm_dev, 0, "DEGAMMA_MODE", es_degamma_mode_enum_list,
+			ARRAY_SIZE(es_degamma_mode_enum_list));
+
+		if (!plane->degamma_mode)
+			goto error_cleanup_plane;
+
+		drm_object_attach_property(&plane->base.base,
+					   plane->degamma_mode,
+					   ES_DEGAMMA_DISABLE);
+	}
+
+	if (info->rotation) {
+		ret = drm_plane_create_rotation_property(
+			&plane->base, DRM_MODE_ROTATE_0, info->rotation);
+		if (ret)
+			goto error_cleanup_plane;
+	}
+
+	if (info->blend_mode) {
+		ret = drm_plane_create_blend_mode_property(&plane->base,
+							   info->blend_mode);
+		if (ret)
+			goto error_cleanup_plane;
+		ret = drm_plane_create_alpha_property(&plane->base);
+		if (ret)
+			goto error_cleanup_plane;
+	}
+
+	if (info->color_encoding) {
+		ret = drm_plane_create_color_properties(
+			&plane->base, info->color_encoding,
+			BIT(DRM_COLOR_YCBCR_LIMITED_RANGE),
+			DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE);
+		if (ret)
+			goto error_cleanup_plane;
+	}
+
+	if (info->roi) {
+		plane->roi_prop = drm_property_create(
+			drm_dev, DRM_MODE_PROP_BLOB, "ROI", 0);
+		if (!plane->roi_prop)
+			goto error_cleanup_plane;
+
+		drm_object_attach_property(&plane->base.base, plane->roi_prop,
+					   0);
+	}
+
+	if (info->color_mgmt) {
+		plane->color_mgmt_prop = drm_property_create(
+			drm_dev, DRM_MODE_PROP_BLOB, "COLOR_CONFIG", 0);
+		if (!plane->color_mgmt_prop)
+			return NULL;
+
+		drm_object_attach_property(&plane->base.base,
+					   plane->color_mgmt_prop, 0);
+	}
+
+	return plane;
+
+error_cleanup_plane:
+	drm_plane_cleanup(&plane->base);
+err_free_plane:
+	kfree(plane);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/eswin/es_plane.h b/drivers/gpu/drm/eswin/es_plane.h
new file mode 100644
index 000000000000..b7e7d4617d49
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_plane.h
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_PLANE_H__
+#define __ES_PLANE_H__
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_plane_helper.h>
+
+#include "es_type.h"
+#include "es_fb.h"
+
+#define MAX_NUM_PLANES 3 /* colour format plane */
+
+struct es_plane;
+
+struct es_plane_funcs {
+	void (*update)(struct device *dev, struct es_plane *plane);
+	void (*disable)(struct device *dev, struct es_plane *plane);
+	int (*check)(struct device *dev, struct es_plane *plane,
+		     struct drm_plane_state *state);
+};
+
+struct drm_es_roi {
+	__u16 enable;
+	__u16 roi_x;
+	__u16 roi_y;
+	__u16 roi_w;
+	__u16 roi_h;
+};
+
+struct drm_es_color_mgmt {
+	__u32 colorkey;
+	__u32 colorkey_high;
+	__u32 clear_value;
+	__u16 clear_enable;
+	__u16 transparency;
+};
+
+struct es_plane_status {
+	u32 tile_mode;
+	struct drm_rect src;
+	struct drm_rect dest;
+	// struct drm_format_name_buf format_name;
+};
+
+struct es_plane_state {
+	struct drm_plane_state base;
+	struct es_plane_status status; /* for debugfs */
+	struct drm_property_blob *roi;
+	struct drm_property_blob *color_mgmt;
+
+	u32 degamma;
+	bool degamma_changed;
+};
+
+struct es_plane {
+	struct drm_plane base;
+	u8 id;
+	dma_addr_t dma_addr[MAX_NUM_PLANES];
+
+	struct drm_property *degamma_mode;
+	struct drm_property *roi_prop;
+	struct drm_property *color_mgmt_prop;
+
+	const struct es_plane_funcs *funcs;
+};
+
+void es_plane_destory(struct drm_plane *plane);
+
+struct es_plane *es_plane_create(struct drm_device *drm_dev,
+				 struct es_plane_info *info,
+				 unsigned int possible_crtcs);
+
+static inline struct es_plane *to_es_plane(struct drm_plane *plane)
+{
+	return container_of(plane, struct es_plane, base);
+}
+
+static inline struct es_plane_state *
+to_es_plane_state(struct drm_plane_state *state)
+{
+	return container_of(state, struct es_plane_state, base);
+}
+#endif /* __ES_PLANE_H__ */
diff --git a/drivers/gpu/drm/eswin/es_simple_enc.c b/drivers/gpu/drm/eswin/es_simple_enc.c
new file mode 100644
index 000000000000..e5bbb2381107
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_simple_enc.c
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/component.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_blend.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_modeset_helper_vtables.h>
+
+#include "es_crtc.h"
+#include "es_simple_enc.h"
+
+static const struct simple_encoder_priv hdmi_priv = {
+	.encoder_type = DRM_MODE_ENCODER_TMDS
+};
+
+static const struct simple_encoder_priv dsi_priv = {
+	.encoder_type = DRM_MODE_ENCODER_DSI
+};
+
+static const struct drm_encoder_funcs encoder_funcs = {
+	.destroy = drm_encoder_cleanup
+};
+
+static inline struct simple_encoder *to_simple_encoder(struct drm_encoder *enc)
+{
+	return container_of(enc, struct simple_encoder, encoder);
+}
+
+static int encoder_parse_dt(struct device *dev)
+{
+	struct simple_encoder *simple = dev_get_drvdata(dev);
+	int ret = 0;
+	int cnt, i;
+	u32 *vals;
+	u32 *masks;
+
+	simple->dss_regmap = syscon_regmap_lookup_by_phandle(
+		dev->of_node, "eswin,dss-syscon");
+
+	if (IS_ERR(simple->dss_regmap)) {
+		if (PTR_ERR(simple->dss_regmap) != -ENODEV) {
+			dev_err(dev, "failed to get dss-syscon\n");
+			ret = PTR_ERR(simple->dss_regmap);
+			goto err;
+		}
+		simple->dss_regmap = NULL;
+		goto err;
+	}
+
+	cnt = of_property_count_elems_of_size(dev->of_node, "eswin,mux-mask",
+					      4);
+	if (!cnt) {
+		ret = cnt;
+		goto err;
+	}
+
+	simple->dss_regdatas = devm_kzalloc(
+		dev, sizeof(*simple->dss_regdatas) * cnt, GFP_KERNEL);
+
+	masks = kcalloc(cnt, sizeof(*masks), GFP_KERNEL);
+	if (!masks) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	vals = kcalloc(cnt, sizeof(*vals), GFP_KERNEL);
+	if (!vals) {
+		ret = -ENOMEM;
+		goto err_free_masks;
+	}
+
+	ret = of_property_read_u32_array(dev->of_node, "eswin,mux-mask", masks,
+					 cnt);
+	if (ret)
+		goto err_free_vals;
+
+	ret = of_property_read_u32_array(dev->of_node, "eswin,mux-val", vals,
+					 cnt);
+	if (ret)
+		goto err_free_vals;
+
+	for (i = 0; i < cnt; i++) {
+		simple->dss_regdatas[i].mask = masks[i];
+		simple->dss_regdatas[i].value = vals[i];
+	}
+
+err_free_vals:
+	kfree(vals);
+err_free_masks:
+	kfree(masks);
+err:
+	return ret;
+}
+
+void encoder_atomic_enable(struct drm_encoder *encoder,
+			   struct drm_atomic_state *state)
+{
+	struct simple_encoder *simple = to_simple_encoder(encoder);
+	struct dss_data *data = simple->dss_regdatas;
+	int crtc_id;
+
+	if (!simple->dss_regmap)
+		return;
+
+	crtc_id = drm_of_encoder_active_endpoint_id(simple->dev->of_node,
+						    encoder);
+
+	regmap_update_bits(simple->dss_regmap, 0, data[crtc_id].mask,
+			   data[crtc_id].value);
+}
+
+int encoder_atomic_check(struct drm_encoder *encoder,
+			 struct drm_crtc_state *crtc_state,
+			 struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc_state);
+	struct drm_connector *connector = conn_state->connector;
+
+	es_crtc_state->encoder_type = encoder->encoder_type;
+
+	if (connector->display_info.num_bus_formats)
+		es_crtc_state->output_fmt =
+			connector->display_info.bus_formats[0];
+	else
+		es_crtc_state->output_fmt = MEDIA_BUS_FMT_RGB888_1X24;
+
+	switch (es_crtc_state->output_fmt) {
+	case MEDIA_BUS_FMT_RGB565_1X16:
+	case MEDIA_BUS_FMT_RGB666_1X18:
+	case MEDIA_BUS_FMT_RGB666_1X24_CPADHI:
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		break;
+	default:
+		es_crtc_state->output_fmt = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	}
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
+	.atomic_enable = encoder_atomic_enable,
+	.atomic_check = encoder_atomic_check,
+};
+
+static int encoder_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+	struct simple_encoder *simple = dev_get_drvdata(dev);
+	struct drm_encoder *encoder;
+	struct drm_bridge *bridge;
+	int ret;
+
+	encoder = &simple->encoder;
+
+	/* Encoder. */
+	ret = drm_encoder_init(drm_dev, encoder, &encoder_funcs,
+			       simple->priv->encoder_type, NULL);
+	if (ret)
+		return ret;
+
+	drm_encoder_helper_add(encoder, &encoder_helper_funcs);
+
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm_dev, dev->of_node);
+
+	/* output port is port1*/
+	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, -1, NULL, &bridge);
+	if (ret)
+		goto err;
+
+	ret = drm_bridge_attach(encoder, bridge, NULL, 0);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	drm_encoder_cleanup(encoder);
+
+	return ret;
+}
+
+static void encoder_unbind(struct device *dev, struct device *master,
+			   void *data)
+{
+	struct simple_encoder *simple = dev_get_drvdata(dev);
+
+	drm_encoder_cleanup(&simple->encoder);
+}
+
+static const struct component_ops encoder_component_ops = {
+	.bind = encoder_bind,
+	.unbind = encoder_unbind,
+};
+
+static const struct of_device_id simple_encoder_dt_match[] = {
+	{ .compatible = "eswin,hdmi-encoder", .data = &hdmi_priv },
+	{ .compatible = "eswin,dp-encoder", .data = &hdmi_priv },
+	{ .compatible = "eswin,dsi-encoder", .data = &dsi_priv },
+	{},
+};
+MODULE_DEVICE_TABLE(of, simple_encoder_dt_match);
+
+static int encoder_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct simple_encoder *simple;
+	int ret;
+
+	simple = devm_kzalloc(dev, sizeof(*simple), GFP_KERNEL);
+	if (!simple)
+		return -ENOMEM;
+
+	simple->priv = of_device_get_match_data(dev);
+
+	simple->dev = dev;
+
+	dev_set_drvdata(dev, simple);
+
+	ret = encoder_parse_dt(dev);
+	if (ret)
+		return ret;
+
+	return component_add(dev, &encoder_component_ops);
+}
+
+static int encoder_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	component_del(dev, &encoder_component_ops);
+
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+struct platform_driver simple_encoder_driver = {
+    .probe = encoder_probe,
+    .remove = encoder_remove,
+    .driver = {
+        .name = "es-simple-encoder",
+        .of_match_table = of_match_ptr(simple_encoder_dt_match),
+    },
+};
+
+MODULE_DESCRIPTION("Simple Encoder Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_simple_enc.h b/drivers/gpu/drm/eswin/es_simple_enc.h
new file mode 100644
index 000000000000..bd7f2212f2a2
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_simple_enc.h
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_SIMPLE_ENC_H_
+#define __ES_SIMPLE_ENC_H_
+
+struct simple_encoder_priv {
+	unsigned char encoder_type;
+};
+
+struct dss_data {
+	u32 mask;
+	u32 value;
+};
+
+struct simple_encoder {
+	struct drm_encoder encoder;
+	struct device *dev;
+	const struct simple_encoder_priv *priv;
+	struct regmap *dss_regmap;
+	struct dss_data *dss_regdatas;
+};
+
+extern struct platform_driver simple_encoder_driver;
+#endif /* __ES_SIMPLE_ENC_H_ */
diff --git a/drivers/gpu/drm/eswin/es_type.h b/drivers/gpu/drm/eswin/es_type.h
new file mode 100644
index 000000000000..c180315e5835
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_type.h
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_TYPE_H__
+#define __ES_TYPE_H__
+
+#include <drm/drm_plane.h>
+#include <drm/drm_plane_helper.h>
+
+struct es_plane_info {
+	const char *name;
+	u8 id;
+	enum drm_plane_type type;
+	unsigned int num_formats;
+	const u32 *formats;
+	const u64 *modifiers;
+	unsigned int min_width;
+	unsigned int min_height;
+	unsigned int max_width;
+	unsigned int max_height;
+	unsigned int rotation;
+	unsigned int blend_mode;
+	unsigned int color_encoding;
+
+	/* 0 means no de-gamma LUT */
+	unsigned int degamma_size;
+
+	int min_scale; /* 16.16 fixed point */
+	int max_scale; /* 16.16 fixed point */
+	bool roi;
+	bool color_mgmt;
+	bool background;
+};
+
+struct es_dc_info {
+	const char *name;
+
+	/* planes */
+	unsigned char plane_num;
+	const struct es_plane_info *planes;
+
+	unsigned int max_bpc;
+	unsigned int color_formats;
+
+	/* 0 means no gamma LUT */
+	u16 gamma_size;
+	u8 gamma_bits;
+
+	u16 pitch_alignment;
+
+	bool pipe_sync;
+	bool mmu_prefetch;
+	bool background;
+};
+
+#endif /* __ES_TYPE_H__ */
diff --git a/drivers/gpu/drm/eswin/es_virtual.c b/drivers/gpu/drm/eswin/es_virtual.c
new file mode 100644
index 000000000000..b2f2770804db
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_virtual.c
@@ -0,0 +1,1395 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#include <linux/component.h>
+#include <linux/of_platform.h>
+#include <linux/media-bus-format.h>
+#include <linux/debugfs.h>
+
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
+
+#include "es_virtual.h"
+#include "es_dc.h"
+#include "es_gem.h"
+
+static unsigned char __get_bpp(struct es_virtual_display *vd)
+{
+	if (vd->bus_format == MEDIA_BUS_FMT_RGB101010_1X30)
+		return 10;
+	return 8;
+}
+
+static void vd_dump_destroy(struct es_virtual_display *vd)
+{
+	struct drm_device *drm_dev = vd->encoder.dev;
+
+	if (vd->dump_blob.data) {
+		vunmap(vd->dump_blob.data);
+		vd->dump_blob.data = NULL;
+	}
+	vd->dump_blob.size = 0;
+
+	debugfs_remove(vd->dump_debugfs);
+	vd->dump_debugfs = NULL;
+
+	if (vd->dump_obj) {
+		mutex_lock(&drm_dev->struct_mutex);
+		drm_gem_object_put(&vd->dump_obj->base);
+		mutex_unlock(&drm_dev->struct_mutex);
+		vd->dump_obj = NULL;
+	}
+}
+
+static void vd_dump_create(struct es_virtual_display *vd,
+			   struct drm_display_mode *mode)
+{
+	struct drm_device *drm_dev = vd->encoder.dev;
+	struct es_dc *dc = dev_get_drvdata(vd->dc);
+	struct es_gem_object *obj;
+	unsigned int pitch, size;
+	void *kvaddr;
+	char *name;
+
+	if (!dc->funcs)
+		return;
+
+	vd_dump_destroy(vd);
+
+	/* dump in 4bytes XRGB format */
+	pitch = mode->hdisplay * 4;
+	pitch = ALIGN(pitch, dc->hw.info->pitch_alignment);
+	size = PAGE_ALIGN(pitch * mode->vdisplay);
+
+	obj = es_gem_create_object(drm_dev, size);
+	if (IS_ERR(obj)) {
+		return;
+	}
+
+	vd->dump_obj = obj;
+	vd->pitch = pitch;
+
+	kvaddr = vmap(obj->pages, obj->size >> PAGE_SHIFT, VM_MAP,
+		      pgprot_writecombine(PAGE_KERNEL));
+	if (!kvaddr)
+		goto err;
+
+	vd->dump_blob.data = kvaddr;
+	vd->dump_blob.size = obj->size;
+
+	name = kasprintf(GFP_KERNEL, "%dx%d-XRGB-%d.raw", mode->hdisplay,
+			 mode->vdisplay, __get_bpp(vd));
+	if (!name)
+		goto err;
+
+	vd->dump_debugfs = debugfs_create_blob(
+		name, 0444, vd->connector.debugfs_entry, &vd->dump_blob);
+	kfree(name);
+
+	return;
+
+err:
+	vd_dump_destroy(vd);
+}
+
+static void vd_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct es_virtual_display *vd;
+
+	drm_encoder_cleanup(encoder);
+	vd = to_virtual_display_with_encoder(encoder);
+	vd_dump_destroy(vd);
+}
+
+static const struct drm_encoder_funcs vd_encoder_funcs = {
+	.destroy = vd_encoder_destroy
+};
+
+static void vd_mode_set(struct drm_encoder *encoder,
+			struct drm_display_mode *mode,
+			struct drm_display_mode *adjusted_mode)
+{
+	struct es_virtual_display *vd;
+
+	vd = to_virtual_display_with_encoder(encoder);
+	vd_dump_create(vd, adjusted_mode);
+}
+
+static void vd_encoder_disable(struct drm_encoder *encoder)
+{
+	struct es_virtual_display *vd;
+	struct es_dc *dc;
+
+	vd = to_virtual_display_with_encoder(encoder);
+	dc = dev_get_drvdata(vd->dc);
+	if (dc->funcs && dc->funcs->dump_disable)
+		dc->funcs->dump_disable(vd->dc);
+}
+
+static void vd_encoder_enable(struct drm_encoder *encoder)
+{
+	struct es_virtual_display *vd;
+	struct es_dc *dc;
+
+	vd = to_virtual_display_with_encoder(encoder);
+	dc = dev_get_drvdata(vd->dc);
+	if (dc->funcs && dc->funcs->dump_enable && vd->dump_obj)
+		dc->funcs->dump_enable(vd->dc, vd->dump_obj->iova, vd->pitch);
+}
+
+int vd_encoder_atomic_check(struct drm_encoder *encoder,
+			    struct drm_crtc_state *crtc_state,
+			    struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc_state);
+
+	es_crtc_state->encoder_type = encoder->encoder_type;
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs vd_encoder_helper_funcs = {
+	.mode_set = vd_mode_set,
+	.enable = vd_encoder_enable,
+	.disable = vd_encoder_disable,
+	.atomic_check = vd_encoder_atomic_check,
+};
+
+static const struct drm_display_mode edid_cea_modes_1[] = {
+	/* 1 - 640x480@60Hz 4:3 */
+	{
+		DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656, 752,
+			 800, 0, 480, 490, 492, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 2 - 720x480@60Hz 4:3 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 3 - 720x480@60Hz 16:9 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 4 - 1280x720@60Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 5 - 1920x1080i@60Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 6 - 720(1440)x480i@60Hz 4:3 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 7 - 720(1440)x480i@60Hz 16:9 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 8 - 720(1440)x240@60Hz 4:3 */
+	{
+		DRM_MODE("720x240", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 9 - 720(1440)x240@60Hz 16:9 */
+	{
+		DRM_MODE("720x240", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 10 - 2880x480i@60Hz 4:3 */
+	{
+		DRM_MODE("2880x480i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 11 - 2880x480i@60Hz 16:9 */
+	{
+		DRM_MODE("2880x480i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 12 - 2880x240@60Hz 4:3 */
+	{
+		DRM_MODE("2880x240", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 13 - 2880x240@60Hz 16:9 */
+	{
+		DRM_MODE("2880x240", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 14 - 1440x480@60Hz 4:3 */
+	{
+		DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1472,
+			 1596, 1716, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 15 - 1440x480@60Hz 16:9 */
+	{
+		DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1472,
+			 1596, 1716, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 16 - 1920x1080@60Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 17 - 720x576@50Hz 4:3 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 18 - 720x576@50Hz 16:9 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 19 - 1280x720@50Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 20 - 1920x1080i@50Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 21 - 720(1440)x576i@50Hz 4:3 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 22 - 720(1440)x576i@50Hz 16:9 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 23 - 720(1440)x288@50Hz 4:3 */
+	{
+		DRM_MODE("720x288", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 24 - 720(1440)x288@50Hz 16:9 */
+	{
+		DRM_MODE("720x288", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 25 - 2880x576i@50Hz 4:3 */
+	{
+		DRM_MODE("2880x576i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 26 - 2880x576i@50Hz 16:9 */
+	{
+		DRM_MODE("2880x576i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 27 - 2880x288@50Hz 4:3 */
+	{
+		DRM_MODE("2880x288", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 28 - 2880x288@50Hz 16:9 */
+	{
+		DRM_MODE("2880x288", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 29 - 1440x576@50Hz 4:3 */
+	{
+		DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+			 1592, 1728, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 30 - 1440x576@50Hz 16:9 */
+	{
+		DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+			 1592, 1728, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 31 - 1920x1080@50Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 32 - 1920x1080@24Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 33 - 1920x1080@25Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 34 - 1920x1080@30Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 35 - 2880x480@60Hz 4:3 */
+	{
+		DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2944,
+			 3192, 3432, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 36 - 2880x480@60Hz 16:9 */
+	{
+		DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2944,
+			 3192, 3432, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 37 - 2880x576@50Hz 4:3 */
+	{
+		DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2928,
+			 3184, 3456, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 38 - 2880x576@50Hz 16:9 */
+	{
+		DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2928,
+			 3184, 3456, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 39 - 1920x1080i@50Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 72000, 1920, 1952,
+			 2120, 2304, 0, 1080, 1126, 1136, 1250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 40 - 1920x1080i@100Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 41 - 1280x720@100Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 42 - 720x576@100Hz 4:3 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 43 - 720x576@100Hz 16:9 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 44 - 720(1440)x576i@100Hz 4:3 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 45 - 720(1440)x576i@100Hz 16:9 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 46 - 1920x1080i@120Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 47 - 1280x720@120Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 48 - 720x480@120Hz 4:3 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 54000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 49 - 720x480@120Hz 16:9 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 54000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 50 - 720(1440)x480i@120Hz 4:3 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 27000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 51 - 720(1440)x480i@120Hz 16:9 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 27000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 52 - 720x576@200Hz 4:3 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 108000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 53 - 720x576@200Hz 16:9 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 108000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 54 - 720(1440)x576i@200Hz 4:3 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 55 - 720(1440)x576i@200Hz 16:9 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 56 - 720x480@240Hz 4:3 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 108000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 57 - 720x480@240Hz 16:9 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 108000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 58 - 720(1440)x480i@240Hz 4:3 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 54000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 59 - 720(1440)x480i@240Hz 16:9 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 54000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 60 - 1280x720@24Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 61 - 1280x720@25Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700,
+			 3740, 3960, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 62 - 1280x720@30Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 63 - 1920x1080@120Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 64 - 1920x1080@100Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 65 - 1280x720@24Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 66 - 1280x720@25Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700,
+			 3740, 3960, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 67 - 1280x720@30Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 68 - 1280x720@50Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 69 - 1280x720@60Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 70 - 1280x720@100Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 71 - 1280x720@120Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 72 - 1920x1080@24Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 73 - 1920x1080@25Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 74 - 1920x1080@30Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 75 - 1920x1080@50Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 76 - 1920x1080@60Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 77 - 1920x1080@100Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 78 - 1920x1080@120Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 79 - 1680x720@24Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 80 - 1680x720@25Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2908,
+			 2948, 3168, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 81 - 1680x720@30Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2380,
+			 2420, 2640, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 82 - 1680x720@50Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 82500, 1680, 1940,
+			 1980, 2200, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 83 - 1680x720@60Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 99000, 1680, 1940,
+			 1980, 2200, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 84 - 1680x720@100Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 165000, 1680, 1740,
+			 1780, 2000, 0, 720, 725, 730, 825, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 85 - 1680x720@120Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 198000, 1680, 1740,
+			 1780, 2000, 0, 720, 725, 730, 825, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 86 - 2560x1080@24Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 99000, 2560, 3558,
+			 3602, 3750, 0, 1080, 1084, 1089, 1100, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 87 - 2560x1080@25Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 90000, 2560, 3008,
+			 3052, 3200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 88 - 2560x1080@30Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 118800, 2560, 3328,
+			 3372, 3520, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 89 - 2560x1080@50Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 185625, 2560, 3108,
+			 3152, 3300, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 90 - 2560x1080@60Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 198000, 2560, 2808,
+			 2852, 3000, 0, 1080, 1084, 1089, 1100, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 91 - 2560x1080@100Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 371250, 2560, 2778,
+			 2822, 2970, 0, 1080, 1084, 1089, 1250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 92 - 2560x1080@120Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 495000, 2560, 3108,
+			 3152, 3300, 0, 1080, 1084, 1089, 1250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 93 - 3840x2160@24Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 94 - 3840x2160@25Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 95 - 3840x2160@30Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 96 - 3840x2160@50Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 97 - 3840x2160@60Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 98 - 4096x2160@24Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 99 - 4096x2160@25Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5064,
+			 5152, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 100 - 4096x2160@30Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 4184,
+			 4272, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 101 - 4096x2160@50Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 5064,
+			 5152, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 102 - 4096x2160@60Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 4184,
+			 4272, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 103 - 3840x2160@24Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 104 - 3840x2160@25Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 105 - 3840x2160@30Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 106 - 3840x2160@50Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 107 - 3840x2160@60Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 108 - 1280x720@48Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 90000, 1280, 2240,
+			 2280, 2500, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 109 - 1280x720@48Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 90000, 1280, 2240,
+			 2280, 2500, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 110 - 1680x720@48Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 99000, 1680, 2490,
+			 2530, 2750, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 111 - 1920x1080@48Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 112 - 1920x1080@48Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 113 - 2560x1080@48Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 198000, 2560, 3558,
+			 3602, 3750, 0, 1080, 1084, 1089, 1100, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 114 - 3840x2160@48Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 115 - 4096x2160@48Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 116 - 3840x2160@48Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 117 - 3840x2160@100Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 118 - 3840x2160@120Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 119 - 3840x2160@100Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 120 - 3840x2160@120Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 121 - 5120x2160@24Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 396000, 5120, 7116,
+			 7204, 7500, 0, 2160, 2168, 2178, 2200, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 122 - 5120x2160@25Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 396000, 5120, 6816,
+			 6904, 7200, 0, 2160, 2168, 2178, 2200, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 123 - 5120x2160@30Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 396000, 5120, 5784,
+			 5872, 6000, 0, 2160, 2168, 2178, 2200, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 124 - 5120x2160@48Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 742500, 5120, 5866,
+			 5954, 6250, 0, 2160, 2168, 2178, 2475, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 125 - 5120x2160@50Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 742500, 5120, 6216,
+			 6304, 6600, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 126 - 5120x2160@60Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 742500, 5120, 5284,
+			 5372, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 127 - 5120x2160@100Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 1485000, 5120, 6216,
+			 6304, 6600, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+};
+
+static int vd_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode = NULL;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(edid_cea_modes_1); i++) {
+		if (edid_cea_modes_1[i].clock == 594000 ||
+		    edid_cea_modes_1[i].clock == 297000 ||
+		    edid_cea_modes_1[i].clock == 148500 ||
+		    edid_cea_modes_1[i].clock == 108000 ||
+		    edid_cea_modes_1[i].clock == 74250 ||
+		    edid_cea_modes_1[i].clock == 54000 ||
+		    edid_cea_modes_1[i].clock == 27000) {
+			mode = drm_mode_duplicate(dev, &edid_cea_modes_1[i]);
+			drm_mode_probed_add(connector, mode);
+		}
+	}
+
+	return 0;
+}
+
+static struct drm_encoder *vd_best_encoder(struct drm_connector *connector)
+{
+	struct es_virtual_display *vd;
+
+	vd = to_virtual_display_with_connector(connector);
+	return &vd->encoder;
+}
+
+static enum drm_mode_status vd_mode_valid(struct drm_connector *connector,
+					  struct drm_display_mode *mode)
+{
+	if (mode->clock != 594000 && mode->clock != 297000 &&
+	    mode->clock != 148500 && mode->clock != 108000 &&
+	    mode->clock != 74250 && mode->clock != 54000 &&
+	    mode->clock != 27000) {
+		return MODE_NOCLOCK;
+	}
+
+	return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs vd_connector_helper_funcs = {
+	.get_modes = vd_get_modes,
+	.mode_valid = vd_mode_valid,
+	.best_encoder = vd_best_encoder,
+};
+
+static void vd_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static enum drm_connector_status
+vd_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct es_virtual_display *vd;
+	enum drm_connector_status status = connector_status_unknown;
+
+	vd = to_virtual_display_with_connector(connector);
+
+	if (vd->enable) {
+		status = connector_status_connected;
+	} else {
+		status = connector_status_disconnected;
+	}
+	return status;
+}
+
+static ssize_t virtual_enable_read(struct file *file, char __user *buf,
+				   size_t count, loff_t *ppos)
+{
+	struct es_virtual_display *vd = file->private_data;
+	char kbuf[16];
+	int len;
+
+	len = snprintf(kbuf, sizeof(kbuf), "%u\n", vd->enable);
+
+	return simple_read_from_buffer(buf, count, ppos, kbuf, len);
+}
+
+static ssize_t virtual_enable_write(struct file *file, const char __user *buf,
+				    size_t count, loff_t *ppos)
+{
+	struct es_virtual_display *vd = file->private_data;
+	char kbuf[16];
+	unsigned long val;
+
+	if (count >= sizeof(kbuf))
+		return -EINVAL;
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+
+	kbuf[count] = '\0';
+
+	if (kstrtoul(kbuf, 10, &val))
+		return -EINVAL;
+
+	vd->enable = val ? 1 : 0;
+
+	return count;
+}
+
+static const struct file_operations virtual_enable_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.read = virtual_enable_read,
+	.write = virtual_enable_write,
+	.open = simple_open,
+	.llseek = default_llseek,
+};
+
+static void vd_connector_debugfs_init(struct drm_connector *connector,
+				      struct dentry *root)
+{
+	struct es_virtual_display *vd;
+
+	vd = to_virtual_display_with_connector(connector);
+
+	if (!connector->debugfs_entry) {
+		DRM_WARN("The connector debugsf_entry invalid");
+	} else {
+		debugfs_create_file("enable", 0444, connector->debugfs_entry,
+				    vd, &virtual_enable_debugfs_fops);
+	}
+	DRM_INFO("Creat debugfs file for Vitual dev:%s", connector->name);
+}
+
+static const struct drm_connector_funcs vd_connector_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = vd_connector_destroy,
+	.detect = vd_connector_detect,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.reset = drm_atomic_helper_connector_reset,
+	.debugfs_init = vd_connector_debugfs_init,
+};
+static int vd_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+	struct es_virtual_display *vd = dev_get_drvdata(dev);
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	struct device_node *ep, *np;
+	struct platform_device *pdev;
+	int ret;
+
+	/* Encoder */
+	encoder = &vd->encoder;
+	ret = drm_encoder_init(drm_dev, encoder, &vd_encoder_funcs,
+			       DRM_MODE_ENCODER_VIRTUAL, NULL);
+	if (ret) {
+		return ret;
+	}
+
+	encoder->encoder_type = DRM_MODE_ENCODER_VIRTUAL;
+	drm_encoder_helper_add(encoder, &vd_encoder_helper_funcs);
+
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm_dev, dev->of_node);
+
+	/* Connector */
+	connector = &vd->connector;
+	ret = drm_connector_init(drm_dev, connector, &vd_connector_funcs,
+				 DRM_MODE_CONNECTOR_VIRTUAL);
+	if (ret)
+		goto connector_init_err;
+	drm_connector_helper_add(connector, &vd_connector_helper_funcs);
+	connector->interlace_allowed = false;
+	connector->doublescan_allowed = false;
+	connector->dpms = DRM_MODE_DPMS_OFF;
+	connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+			    DRM_CONNECTOR_POLL_DISCONNECT;
+	ret = drm_connector_register(connector);
+	if (ret)
+		goto connector_reg_err;
+
+	drm_display_info_set_bus_formats(&connector->display_info,
+					 &vd->bus_format, 1);
+
+	/* attach */
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret)
+		goto attach_err;
+
+	ep = of_graph_get_endpoint_by_regs(dev->of_node, 0, -1);
+	if (!ep) {
+		ret = -EINVAL;
+		goto attach_err;
+	}
+
+	np = of_graph_get_remote_port_parent(ep);
+	of_node_put(ep);
+	if (!np) {
+		ret = -EINVAL;
+		goto attach_err;
+	}
+
+	pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!pdev) {
+		ret = -EPROBE_DEFER;
+		goto attach_err;
+	}
+	get_device(&pdev->dev);
+	vd->dc = &pdev->dev;
+
+	return 0;
+
+attach_err:
+	drm_connector_unregister(connector);
+connector_reg_err:
+	drm_connector_cleanup(connector);
+connector_init_err:
+	drm_encoder_cleanup(encoder);
+	return ret;
+}
+
+static void vd_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct es_virtual_display *vd = dev_get_drvdata(dev);
+
+	drm_connector_unregister(&vd->connector);
+	drm_connector_cleanup(&vd->connector);
+	drm_encoder_cleanup(&vd->encoder);
+	if (vd->dump_obj) {
+		drm_gem_object_put(&vd->dump_obj->base);
+		vd->dump_obj = NULL;
+	}
+}
+
+const struct component_ops vd_component_ops = {
+	.bind = vd_bind,
+	.unbind = vd_unbind,
+};
+
+static const struct of_device_id vd_driver_dt_match[] = {
+	{
+		.compatible = "eswin,virtual_display",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, vd_driver_dt_match);
+
+static int vd_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct es_virtual_display *vd;
+	unsigned char bpp;
+
+	vd = devm_kzalloc(dev, sizeof(*vd), GFP_KERNEL);
+	if (!vd)
+		return -ENOMEM;
+
+	vd->bus_format = MEDIA_BUS_FMT_RGB101010_1X30;
+	of_property_read_u8(dev->of_node, "bpp", &bpp);
+	if (bpp == 8)
+		vd->bus_format = MEDIA_BUS_FMT_RBG888_1X24;
+
+	dev_set_drvdata(dev, vd);
+
+	return component_add(dev, &vd_component_ops);
+}
+
+static int vd_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	component_del(dev, &vd_component_ops);
+
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+struct platform_driver virtual_display_platform_driver = {
+    .probe = vd_probe,
+    .remove = vd_remove,
+    .driver = {
+        .name = "es-virtual-display",
+        .of_match_table = of_match_ptr(vd_driver_dt_match),
+    },
+};
+
+MODULE_DESCRIPTION("Eswin Virtual Display Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_virtual.h b/drivers/gpu/drm/eswin/es_virtual.h
new file mode 100644
index 000000000000..fc7fa59e586b
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_virtual.h
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef __ES_VIRTUAL_H_
+#define __ES_VIRTUAL_H_
+
+struct es_virtual_display {
+	u32 enable;
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+	struct device *dc;
+	u32 bus_format;
+
+	struct dentry *dump_debugfs;
+	struct debugfs_blob_wrapper dump_blob;
+	struct es_gem_object *dump_obj;
+	unsigned int pitch;
+};
+
+static inline struct es_virtual_display *
+to_virtual_display_with_connector(struct drm_connector *connector)
+{
+	return container_of(connector, struct es_virtual_display, connector);
+}
+
+static inline struct es_virtual_display *
+to_virtual_display_with_encoder(struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct es_virtual_display, encoder);
+}
+
+extern struct platform_driver virtual_display_platform_driver;
+#endif /* __ES_VIRTUAL_H_ */
diff --git a/drivers/gpu/drm/eswin/host_lib_driver_linux_if.h b/drivers/gpu/drm/eswin/host_lib_driver_linux_if.h
new file mode 100644
index 000000000000..bc544798575e
--- /dev/null
+++ b/drivers/gpu/drm/eswin/host_lib_driver_linux_if.h
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN drm driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin Driver team
+ */
+
+#ifndef _HOST_LIB_DRIVER_LINUX_IF_H_
+#define _HOST_LIB_DRIVER_LINUX_IF_H_
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define HL_DRIVER_ALLOCATE_DYNAMIC_MEM 0xffffffff
+// hl_drv_ioctl numbers
+enum {
+	HL_DRV_NR_MIN = 0x10,
+	HL_DRV_NR_INIT,
+	HL_DRV_NR_MEMINFO,
+	HL_DRV_NR_LOAD_CODE,
+	HL_DRV_NR_READ_DATA,
+	HL_DRV_NR_WRITE_DATA,
+	HL_DRV_NR_MEMSET_DATA,
+	HL_DRV_NR_READ_HPI,
+	HL_DRV_NR_WRITE_HPI,
+
+	DW_DRV_NR_CONNECT_STATUS,
+	DW_DRV_NR_CONNECT_SET,
+	DW_DRV_NR_DISCONNECT_SET,
+	DW_DRV_NR_AUTH_SUCCESS,
+	DW_DRV_NR_AUTH_FAIL,
+	DW_DRV_NR_NO_CAPACITY,
+
+	HL_DRV_NR_MAX
+};
+
+/*
+ * HL_DRV_IOC_INIT: associate file descriptor with the indicated memory.  This
+ * must be called before any other hl_drv_ioctl on the file descriptor.
+ *
+ *   - hpi_base = base address of HPI registers.
+ *   - code_base = base address of firmware memory (0 to allocate internally)
+ *   - data_base = base address of data memory (0 to allocate internally)
+ *   - code_len, data_len = length of firmware and data memory, respectively.
+ */
+#define HL_DRV_IOC_INIT _IOW('H', HL_DRV_NR_INIT, struct hl_drv_ioc_meminfo)
+
+/*
+ * HL_DRV_IOC_MEMINFO: retrieve memory information from file descriptor.
+ *
+ * Fills out the meminfo struct, returning the values passed to HL_DRV_IOC_INIT
+ * except that the actual base addresses of internal allocations (if any) are
+ * reported.
+ */
+#define HL_DRV_IOC_MEMINFO                                                     \
+	_IOR('H', HL_DRV_NR_MEMINFO, struct hl_drv_ioc_meminfo)
+
+struct hl_drv_ioc_meminfo {
+	__u32 hpi_base;
+	__u32 code_base;
+	__u32 code_size;
+	__u32 data_base;
+	__u32 data_size;
+};
+
+/*
+ * HL_DRV_IOC_LOAD_CODE: write the provided buffer to the firmware memory.
+ *
+ *   - len = number of bytes in data buffer
+ *   - data = data to write to firmware memory.
+ *
+ * This can only be done once (successfully).  Subsequent attempts will
+ * return -EBUSY.
+ */
+#define HL_DRV_IOC_LOAD_CODE                                                   \
+	_IOW('H', HL_DRV_NR_LOAD_CODE, struct hl_drv_ioc_code)
+
+struct hl_drv_ioc_code {
+	__u32 len;
+	__u8 data[];
+};
+
+/*
+ * HL_DRV_IOC_READ_DATA: copy from data memory.
+ * HL_DRV_IOC_WRITE_DATA: copy to data memory.
+ *
+ *   - offset = start copying at this byte offset into the data memory.
+ *   - len    = number of bytes to copy.
+ *   - data   = for write, buffer containing data to copy.
+ *              for read, buffer to which read data will be written.
+ *
+ */
+#define HL_DRV_IOC_READ_DATA                                                   \
+	_IOWR('H', HL_DRV_NR_READ_DATA, struct hl_drv_ioc_data)
+#define HL_DRV_IOC_WRITE_DATA                                                  \
+	_IOW('H', HL_DRV_NR_WRITE_DATA, struct hl_drv_ioc_data)
+
+/*
+ * HL_DRV_IOC_MEMSET_DATA: initialize data memory.
+ *
+ *   - offset  = start initializatoin at this byte offset into the data memory.
+ *   - len     = number of bytes to set.
+ *   - data[0] = byte value to write to all indicated memory locations.
+ */
+#define HL_DRV_IOC_MEMSET_DATA                                                 \
+	_IOW('H', HL_DRV_NR_MEMSET_DATA, struct hl_drv_ioc_data)
+
+struct hl_drv_ioc_data {
+	__u32 offset;
+	__u32 len;
+	__u8 data[];
+};
+
+/*
+ * HL_DRV_IOC_READ_HPI: read HPI register.
+ * HL_DRV_IOC_WRITE_HPI: write HPI register.
+ *
+ *   - offset = byte offset of HPI register to access.
+ *   - value  = for write, value to write.
+ *              for read, location to which result is stored.
+ */
+#define HL_DRV_IOC_READ_HPI                                                    \
+	_IOWR('H', HL_DRV_NR_READ_HPI, struct hl_drv_ioc_hpi_reg)
+#define HL_DRV_IOC_WRITE_HPI                                                   \
+	_IOW('H', HL_DRV_NR_WRITE_HPI, struct hl_drv_ioc_hpi_reg)
+
+struct hl_drv_ioc_hpi_reg {
+	__u32 offset;
+	__u32 value;
+};
+
+#define DW_DRV_IOC_CONNECT_STATUS                                              \
+	_IOR('H', DW_DRV_NR_CONNECT_STATUS, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_CONNECT_SET                                                 \
+	_IOW('H', DW_DRV_NR_CONNECT_SET, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_DISCONNECT_SET                                              \
+	_IOW('H', DW_DRV_NR_DISCONNECT_SET, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_AUTH_SUCCESS                                                \
+	_IOW('H', DW_DRV_NR_AUTH_SUCCESS, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_AUTH_FAIL                                                   \
+	_IOW('H', DW_DRV_NR_AUTH_FAIL, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_NO_CAPACITY                                                 \
+	_IOW('H', DW_DRV_NR_NO_CAPACITY, struct hl_drv_ioc_info)
+
+struct hl_drv_ioc_info {
+	union {
+		__u32 connect;
+		__u32 auth;
+		__u32 capacity;
+	};
+};
+
+#endif // _HOST_LIB_DRIVER_LINUX_IF_H_
diff --git a/drivers/gpu/drm/img/Kconfig b/drivers/gpu/drm/img/Kconfig
new file mode 100644
index 000000000000..a83d094127e7
--- /dev/null
+++ b/drivers/gpu/drm/img/Kconfig
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config DRM_IMG
+	bool "Imagination PowerVR GPU"
+        select DRM
+        default n
+
+source "drivers/gpu/drm/img/img-volcanic/Kconfig"
diff --git a/drivers/gpu/drm/img/Makefile b/drivers/gpu/drm/img/Makefile
new file mode 100644
index 000000000000..fa5c41d9908d
--- /dev/null
+++ b/drivers/gpu/drm/img/Makefile
@@ -0,0 +1,4 @@
+# volcanic
+obj-$(CONFIG_DRM_IMG_VOLCANIC) += img-volcanic/
+
+ccflags-y+= -I$(srctree)/mm
diff --git a/drivers/gpu/drm/img/img-volcanic/Kconfig b/drivers/gpu/drm/img/img-volcanic/Kconfig
new file mode 100644
index 000000000000..587a1c60ad8f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/Kconfig
@@ -0,0 +1,10 @@
+config DRM_IMG_VOLCANIC
+	tristate "DRM support for PowerVR GPU"
+        select DRM_IMG
+	select SYNC_FILE
+	default n
+	help
+	  Driver for PowerVR Rogue graphics hardware.
+
+	  Say Y here if your SoC contains a PowerVR Volcanic GPU. For more
+	  information, see <http://www.imgtec.com/powervr/>.
diff --git a/drivers/gpu/drm/img/img-volcanic/Makefile b/drivers/gpu/drm/img/img-volcanic/Makefile
new file mode 100644
index 000000000000..087e156a263f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/Makefile
@@ -0,0 +1,138 @@
+########################################################################### ###
+#@Title         Root kernel makefile
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+# This top-level kbuild makefile builds all the Linux kernel modules in the
+# DDK. To run kbuild, this makefile is copied to $(TARGET_PRIMARY_OUT)/kbuild/Makefile
+# and make is invoked in $(TARGET_PRIMARY_OUT)/kbuild.
+
+# This makefile doesn't define any kbuild special variables apart from
+# ccflags-y and obj-m. The variables for objects are picked up by including
+# the kbuild makefile fragments named in $(INTERNAL_KBUILD_MAKEFILES). The
+# list of objects that these fragments make is collected in
+# $(INTERNAL_KBUILD_OBJECTS) and $(INTERNAL_EXTRA_KBUILD_OBJECTS). These
+# variables are set according to the build's $(KERNEL_COMPONENTS) and
+# $(EXTRA_PVRSRVKM_COMPONENTS). To add a new kernel module to the build, edit
+# these variables in the per-build Makefile.
+
+INTERNAL_KBUILD_OBJECTS=pvrsrvkm.o
+INTERNAL_EXTRA_KBUILD_OBJECTS=""
+TOP:=$(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+TOP:=$(TOP:/=)
+BRIDGE_SOURCE_ROOT=$(TOP)/generated/volcanic
+TARGET_PRIMARY_ARCH=target_riscv64
+PVR_ARCH=volcanic
+PVR_ARCH_DEFS=volcanic
+PVR_SYSTEM := eswin_cpu
+#PDUMP ?= 1
+PDUMP ?= 0
+BUILD ?= release
+RGX_BVNC ?= 30.3.408.101
+RGX_BNC ?= 30.3.408.101
+PVRSRV_NEED_PVR_DPF=1
+PVRSRV_NEED_PVR_ASSERT=1
+PVR_SERVICES_DEBUG=1
+WINDOW_SYSTEM=nulldrmws
+
+#include $(OUT)/config_kernel.mk
+include $(srctree)/$(src)/config_kernel.mk
+
+# Use the logic from kernel's Kbuild.include to select .NOTINTERMEDIATE or .SECONDARY based on make version
+ifneq ($(and $(filter notintermediate, $(.FEATURES)),$(filter-out 4.4,$(MAKE_VERSION))),)
+.NOTINTERMEDIATE:
+else
+.SECONDARY:
+endif
+
+
+define symlink-source-file
+@if [ ! -e $(dir $@) ]; then mkdir -p $(dir $@); fi
+@if [ ! -h $@ ]; then ln -sf $< $@; fi
+endef
+
+bridge_base := $(BRIDGE_SOURCE_ROOT)
+
+#$(OUT)/$(TARGET_PRIMARY_ARCH)/kbuild/%.c: $(TOP)/%.c
+#	$(symlink-source-file)
+
+#$(OUT)/$(TARGET_PRIMARY_ARCH)/kbuild/generated/$(PVR_ARCH)/%.c: $(bridge_base)/%.c
+#	$(symlink-source-file)
+
+#$(OUT)/$(TARGET_PRIMARY_ARCH)/kbuild/external/%.c: $(abspath $(srctree))/%.c
+#	$(symlink-source-file)
+
+#ccflags-y += -D__linux__ -include $(OUT)/config_kernel.h \
+# -include kernel_config_compatibility.h \
+# -I$(OUT)/include -I$(TOP)/kernel/drivers/staging/imgtec
+ccflags-y += -D__linux__ -include $(srctree)/$(src)/config_kernel.h \
+ -include $(srctree)/drivers/gpu/drm/img/kernel_config_compatibility.h \
+ -I$(OUT)/include \
+ -I$(srctree)/drivers/gpu/drm/img \
+ -I$(srctree)/$(src)/hwdefs/volcanic \
+ -I$(srctree)/$(src)/hwdefs/volcanic/km \
+ -I$(srctree)/$(src)/include/system/rgx_tc \
+ -I$(srctree)/$(src)/include/volcanic/system/rgx_tc
+
+include $(srctree)/$(src)/services/server/env/linux/Kbuild.mk
+include $(srctree)/$(src)/services/system/eswin_cpu/Kbuild.mk
+
+#include $(INTERNAL_KBUILD_MAKEFILES)
+
+define add-file-cflags
+CFLAGS_$(notdir $(1)) := $(CFLAGS_$(1))
+endef
+
+# Define old style CFLAG_ variables for kernels older than 5.4
+$(foreach _m,$(INTERNAL_KBUILD_OBJECTS:.o=-y), \
+ $(foreach _f,$($(_m)), \
+  $(if $(CFLAGS_$(_f)), \
+   $(eval $(call add-file-cflags,$(_f))))))
+
+#ifneq ($(KERNEL_DRIVER_DIR),)
+# ccflags-y += \
+#   -I$(abspath $(srctree))/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM) \
+#   -I$(abspath $(srctree))/$(KERNEL_DRIVER_DIR)
+#endif
+
+$(if $($(PVRSRVKM_NAME)-y),,$(warning $(PVRSRVKM_NAME)-y was empty, which could mean that srvkm is missing from $$(KERNEL_COMPONENTS)))
+$(PVRSRVKM_NAME)-y += $(foreach _m,$(INTERNAL_EXTRA_KBUILD_OBJECTS:.o=),$($(_m)-y))
+
+#obj-m += $(INTERNAL_KBUILD_OBJECTS)
+obj-$(CONFIG_DRM_IMG_VOLCANIC) += $(INTERNAL_KBUILD_OBJECTS)
diff --git a/drivers/gpu/drm/img/img-volcanic/config_kernel.h b/drivers/gpu/drm/img/img-volcanic/config_kernel.h
new file mode 100644
index 000000000000..1e73467fd7bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/config_kernel.h
@@ -0,0 +1,324 @@
+#define SUPPORT_RGXRAY_BRIDGE
+#define RGX_FW_FILENAME "rgx.fw"
+#define RGX_SH_FILENAME "rgx.sh"
+#define PVR_BUILD_DIR "eswin_linux"
+#define PVR_BUILD_TYPE "release"
+#define PVRSRV_MODNAME "pvrsrvkm"
+#define PVRHMMU_MODNAME ""
+#define PVRSYNC_MODNAME "pvr_sync"
+#define SUPPORT_RGX 1
+#define DISPLAY_CONTROLLER drm_nulldisp
+#define PVRSRV_MAX_DEVICES 8
+#define PVRSRV_HWPERF_COUNTERS_PERBLK 12
+#define RELEASE
+#define RGX_BVNC_CORE_KM_HEADER "cores/rgxcore_km_30.3.408.101.h"
+#define RGX_BNC_CONFIG_KM_HEADER "configs/rgxconfig_km_30.V.408.101.h"
+#define PVRSRV_ENABLE_XD_MEM
+#define PVRSRV_NEED_PVR_DPF
+#define PVRSRV_NEED_PVR_ASSERT
+#define SUPPORT_LINUX_FDINFO
+#define PVRSRV_TRACE_ROGUE_EVENTS
+#define SUPPORT_PHYSMEM_TEST
+#define PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY 5
+#define PVRSRV_POISON_ON_ALLOC_VALUE 0xd9
+#define PVRSRV_POISON_ON_FREE_VALUE 0x63
+#define RGX_NUM_OS_SUPPORTED 1
+#define RGX_NUM_DRIVERS_SUPPORTED 1
+#define RGX_HCS_DEFAULT_DEADLINE_MS 0xFFFFFFFFU
+#define RGX_FW_HEAP_USES_FIRMWARE_OSID 0
+#define RGX_FW_HEAP_USES_HOST_OSID 1
+#define RGX_FW_HEAP_USES_DEDICATED_OSID 2
+#define RGX_FW_HEAP_OSID_ASSIGNMENT RGX_FW_HEAP_USES_FIRMWARE_OSID
+#define PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION 0
+#define RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION  512
+#define PVRSRV_APPHINT_DRIVERMODE "default"
+#define RGX_FW_HEAP_SHIFT 25
+#define PVRSRV_APPHINT_GUESTFWHEAPSTRIDE (1 << RGX_FW_HEAP_SHIFT)
+#define PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN IMG_FALSE
+#define RGX_VZ_CONNECTION_TIMEOUT_US 60000000
+#define PVRSRV_VZ_BYPASS_HMMU
+#define GPUVIRT_VALIDATION_NUM_OS 8
+#define PVRSRV_ENABLE_CCCB_GROW
+#define SUPPORT_POWMON_COMPONENT
+#define PVR_POWER_ACTOR_MEASUREMENT_PERIOD_MS 10U
+#define PVR_POWER_MONITOR_HWPERF 1
+#define SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+#define PVR_LDM_PLATFORM_PRE_REGISTERED
+#define PVR_LDM_DRIVER_REGISTRATION_NAME "pvrsrvkm"
+#define PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN 256
+#define ION_DEFAULT_HEAP_NAME "ion_system_heap"
+#define ION_DEFAULT_HEAP_ID_MASK (1 << ION_HEAP_TYPE_SYSTEM)
+#define SUPPORT_FASTRENDER_DM
+#define ASSERTONHWRTRIGGER_DEFAULT_ENABLED
+#define PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT APPHNT_BLDVAR_DBGDUMPLIMIT
+#define PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG IMG_FALSE
+#define PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE 0x4000
+#define PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE 786432
+#define PVRSRV_APPHINT_ENABLESIGNATURECHECKS APPHNT_BLDVAR_ENABLESIGNATURECHECKS
+#define PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE RGXFW_SIG_BUFFER_SIZE_MIN
+#define PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING IMG_FALSE
+#define PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG
+#define PVRSRV_APPHINT_VALIDATEIRQ 0
+#define PVRSRV_APPHINT_DISABLECLOCKGATING 0
+#define PVRSRV_APPHINT_DISABLEDMOVERLAP 0
+#define PVRSRV_APPHINT_FABRICCOHERENCYOVERRIDE 1
+#define PVRSRV_APPHINT_ENABLEDMKILLINGRANDMODE 0
+#define PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL
+#define PVRSRV_APPHINT_ENABLERDPOWERISLAND RGX_RD_POWER_ISLAND_DEFAULT
+#define PVRSRV_APPHINT_ENABLESPUCLOCKGATING IMG_FALSE
+#define PVRSRV_APPHINT_HWVALENABLESPUPOWERMASKCHANGE 0
+#define PVRSRV_APPHINT_HWVALAVAILABLESPUMASK 0xFFFFFFFF
+#define PVRSRV_APPHINT_HWVALAVAILABLERACMASK 0xFFFFFFFF
+#define PVRSRV_APPHINT_HWVALTPUF20BILPRECISION 1
+#define PVRSRV_APPHINT_KILLINGCTL 0
+#define PVRSRV_APPHINT_CDMTDM_KILLINGCTL 0
+#define PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH 0
+#define PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH 0
+#define PVRSRV_APPHINT_FIRMWAREPERF FW_PERF_CONF_NONE
+#define PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN
+#define PVRSRV_APPHINT_HWPERFDISABLECOUNTERFILTER 0
+#define PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB 2048
+#define PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB 2048
+#define PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS 50
+#define PVRSRV_APPHINT_NEWFILTERINGMODE IMG_FALSE
+#define PVRSRV_APPHINT_TRUNCATEMODE 0
+#define PVRSRV_APPHINT_EMUMAXFREQ 0
+#define PVRSRV_APPHINT_GPIOVALIDATIONMODE 0
+#define PVRSRV_APPHINT_RGXBVNC ""
+#define PVRSRV_APPHINT_RISCVDMITEST 0
+#define PVRSRV_APPHINT_RCEDISABLEMASK 0x00000000
+#define PVRSRV_APPHINT_PCGPKTDROPTHRESH  0x0000000b
+#define PVRSRV_APPHINT_RAYSLCMMUAUTOCACHEOPS  0x0000000f
+#define RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US 2000000
+#define PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE 1
+#define PVRSRV_APPHINT_VALIDATESOCUSCTIMERS 0
+#define PVRSRV_APPHINT_SHGEOMPIPEMASK_OVERRIDE 0
+#define PVRSRV_APPHINT_CLEANUPTHREADPRIORITY 5
+#define PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY 0
+#define PVRSRV_APPHINT_CACHEOPTHREADPRIORITY 1
+#define PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 11
+#define PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES 10000
+#define PVRSRV_APPHINT_ASSERTONHWRTRIGGER IMG_FALSE
+#define PVRSRV_APPHINT_ASSERTOUTOFMEMORY IMG_FALSE
+#define PVRSRV_APPHINT_CHECKMLIST APPHNT_BLDVAR_DEBUG
+#define PVRSRV_APPHINT_DISABLEFEDLOGGING IMG_FALSE
+#define PVRSRV_APPHINT_KCCB_SIZE_LOG2 10
+#define PVRSRV_APPHINT_ENABLEAPM RGX_ACTIVEPM_DEFAULT
+#define PVRSRV_APPHINT_ENABLEHTBLOGGROUP 0
+#define PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING RGXFWIF_INICFG_OS_ICS_DM_APPHINT
+#define PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC 40000
+#define PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD 90
+#define PVRSRV_APPHINT_ENABLELOGGROUP RGXFWIF_LOG_TYPE_NONE
+#define PVRSRV_APPHINT_FIRMWARELOGTYPE 0
+#define PVRSRV_APPHINT_FWTRACEBUFSIZEINDWORDS RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS
+#define PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE 1
+#define PVRSRV_APPHINT_FBCDCVERSIONOVERRIDE 0
+#define PVRSRV_APPHINT_HTBOPERATIONMODE HTB_OPMODE_DROPOLDEST
+#define PVRSRV_APPHINT_HTBUFFERSIZE 64
+#define PVRSRV_APPHINT_ENABLEFTRACEGPU IMG_FALSE
+#define PVRSRV_APPHINT_HWPERFFWFILTER 0
+#define PVRSRV_APPHINT_HWPERFHOSTFILTER 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_SERVICES 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_EGL 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGLES 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENCL 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_VULKAN 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGL 0
+#define PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE 2
+#define PVRSRV_APPHINT_ENABLEFWPOISONONFREE IMG_FALSE
+#define PVRSRV_APPHINT_FWPOISONONFREEVALUE 0xBD
+#define PVRSRV_APPHINT_ZEROFREELIST IMG_FALSE
+#define PVRSRV_APPHINT_GPUUNITSPOWERCHANGE IMG_FALSE
+#define PVRSRV_APPHINT_DISABLEPDUMPPANIC IMG_FALSE
+#define PVRSRV_APPHINT_CACHEOPCONFIG 0
+#define PVRSRV_APPHINT_CACHEOPUMKMHRESHOLDSIZE 0
+#define PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC IMG_FALSE
+#define PVRSRV_APPHINT_PHYSMEMTESTPASSES APPHNT_PHYSMEMTEST_ENABLE
+#define PVRSRV_APPHINT_MMUVAL_PARITYFLIPPMR "unset"
+#define PVRSRV_APPHINT_TESTSLRINTERVAL 0
+#define PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2 8
+#define PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2 7
+#define PVRSRV_APPHINT_CLKCTRL0 0xAAAAAAAAAAAAAAAAULL
+#define PVRSRV_APPHINT_CLKCTRL1 0xAAAAAAAAAAAAAAAAULL
+#define PVRSRV_APPHINT_CLKCTRL2 0xAAAAAAAAULL
+#define SUPPORT_SOC_TIMER
+#define PDVFS_COM_HOST 1
+#define PDVFS_COM_AP 2
+#define PDVFS_COM_PMC 3
+#define PDVFS_COM_IMG_CLKDIV 4
+#define PDVFS_COM_SYSREG 5
+#define PDVFS_COM PDVFS_COM_HOST
+#define SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY
+#define PVR_GPIO_MODE_GENERAL 1
+#define PVR_GPIO_MODE PVR_GPIO_MODE_GENERAL
+#define PVRSRV_ENABLE_PROCESS_STATS
+#define PVR_ANNOTATION_MAX_LEN 63
+#define PVRSRV_DEVICE_INIT_MODE PVRSRV_LINUX_DEV_INIT_ON_CONNECT
+#define SUPPORT_DI_BRG_IMPL
+#define PVR_LINUX_PHYSMEM_MAX_POOL_PAGES 10240
+#define PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES 20480
+#define PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES
+#define PVR_DIRTY_BYTES_FLUSH_THRESHOLD 524288
+#define PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD 256
+#define PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM 2
+#define PVR_LINUX_KMALLOC_ALLOCATION_THRESHOLD 16384
+#define SUPPORT_PMR_DEFERRED_FREE
+#define SUPPORT_PMR_PAGES_DEFERRED_FREE
+#define SUPPORT_MMU_DEFERRED_FREE
+#define SUPPORT_WRAP_EXTMEM
+#define PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC 1
+#define SUPPORT_NATIVE_FENCE_SYNC
+#define PVRSRV_STALLED_CCB_ACTION
+#define PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP
+#define UPDATE_FENCE_CHECKPOINT_COUNT 1
+#define RGX_VZ_CONNECTION_COOLDOWN_PERIOD 0
+#define PVR_DRM_NAME "pvr"
+#define DEVICE_MEMSETCPY_ALIGN_IN_BYTES 16
+#define RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS 0
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA 15
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D 16
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ3D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ2D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_CDM 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TA 16
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_3D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_KICKSYNC 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TDM 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_RDM 15
+#define RGX_DRIVERID_0_DEFAULT_PRIORITY (1 - 0)
+#define RGX_DRIVERID_1_DEFAULT_PRIORITY (1 - 1)
+#define RGX_DRIVERID_2_DEFAULT_PRIORITY (1 - 2)
+#define RGX_DRIVERID_3_DEFAULT_PRIORITY (1 - 3)
+#define RGX_DRIVERID_4_DEFAULT_PRIORITY (1 - 4)
+#define RGX_DRIVERID_5_DEFAULT_PRIORITY (1 - 5)
+#define RGX_DRIVERID_6_DEFAULT_PRIORITY (1 - 6)
+#define RGX_DRIVERID_7_DEFAULT_PRIORITY (1 - 7)
+#define RGX_DRIVERID_8_DEFAULT_PRIORITY (1 - 8)
+#define RGX_DRIVERID_9_DEFAULT_PRIORITY (1 - 9)
+#define RGX_DRIVERID_10_DEFAULT_PRIORITY (1 - 10)
+#define RGX_DRIVERID_11_DEFAULT_PRIORITY (1 - 11)
+#define RGX_DRIVERID_12_DEFAULT_PRIORITY (1 - 12)
+#define RGX_DRIVERID_13_DEFAULT_PRIORITY (1 - 13)
+#define RGX_DRIVERID_14_DEFAULT_PRIORITY (1 - 14)
+#define RGX_DRIVERID_15_DEFAULT_PRIORITY (1 - 15)
+#define RGX_DRIVERID_16_DEFAULT_PRIORITY (1 - 16)
+#define RGX_DRIVERID_17_DEFAULT_PRIORITY (1 - 17)
+#define RGX_DRIVERID_18_DEFAULT_PRIORITY (1 - 18)
+#define RGX_DRIVERID_19_DEFAULT_PRIORITY (1 - 19)
+#define RGX_DRIVERID_20_DEFAULT_PRIORITY (1 - 20)
+#define RGX_DRIVERID_21_DEFAULT_PRIORITY (1 - 21)
+#define RGX_DRIVERID_22_DEFAULT_PRIORITY (1 - 22)
+#define RGX_DRIVERID_23_DEFAULT_PRIORITY (1 - 23)
+#define RGX_DRIVERID_24_DEFAULT_PRIORITY (1 - 24)
+#define RGX_DRIVERID_25_DEFAULT_PRIORITY (1 - 25)
+#define RGX_DRIVERID_26_DEFAULT_PRIORITY (1 - 26)
+#define RGX_DRIVERID_27_DEFAULT_PRIORITY (1 - 27)
+#define RGX_DRIVERID_28_DEFAULT_PRIORITY (1 - 28)
+#define RGX_DRIVERID_29_DEFAULT_PRIORITY (1 - 29)
+#define RGX_DRIVERID_30_DEFAULT_PRIORITY (1 - 30)
+#define RGX_DRIVERID_31_DEFAULT_PRIORITY (1 - 31)
+#define RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_1_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_2_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_3_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_4_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_5_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_6_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_7_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_8_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_9_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_10_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_11_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_12_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_13_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_14_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_15_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_16_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_17_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_18_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_19_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_20_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_21_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_22_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_23_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_24_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_25_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_26_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_27_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_28_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_29_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_30_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_31_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_0_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_1_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_2_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_3_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_4_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_5_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_6_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_7_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_8_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_9_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_10_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_11_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_12_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_13_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_14_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_15_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_16_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_17_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_18_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_19_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_20_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_21_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_22_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_23_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_24_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_25_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_26_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_27_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_28_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_29_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_30_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_31_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL 0
+#define RGX_DRIVER_DEFAULT_TIME_SLICES_SUM (((((((((((((((((((((((((((((((( + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0)
+#define DRIVER0_SECURITY_SUPPORT 0
+#define DRIVER1_SECURITY_SUPPORT 0
+#define DRIVER2_SECURITY_SUPPORT 0
+#define DRIVER3_SECURITY_SUPPORT 0
+#define DRIVER4_SECURITY_SUPPORT 0
+#define DRIVER5_SECURITY_SUPPORT 0
+#define DRIVER6_SECURITY_SUPPORT 0
+#define DRIVER7_SECURITY_SUPPORT 0
+#define DRIVER8_SECURITY_SUPPORT 0
+#define DRIVER9_SECURITY_SUPPORT 0
+#define DRIVER10_SECURITY_SUPPORT 0
+#define DRIVER11_SECURITY_SUPPORT 0
+#define DRIVER12_SECURITY_SUPPORT 0
+#define DRIVER13_SECURITY_SUPPORT 0
+#define DRIVER14_SECURITY_SUPPORT 0
+#define DRIVER15_SECURITY_SUPPORT 0
+#define DRIVER16_SECURITY_SUPPORT 0
+#define DRIVER17_SECURITY_SUPPORT 0
+#define DRIVER18_SECURITY_SUPPORT 0
+#define DRIVER19_SECURITY_SUPPORT 0
+#define DRIVER20_SECURITY_SUPPORT 0
+#define DRIVER21_SECURITY_SUPPORT 0
+#define DRIVER22_SECURITY_SUPPORT 0
+#define DRIVER23_SECURITY_SUPPORT 0
+#define DRIVER24_SECURITY_SUPPORT 0
+#define DRIVER25_SECURITY_SUPPORT 0
+#define DRIVER26_SECURITY_SUPPORT 0
+#define DRIVER27_SECURITY_SUPPORT 0
+#define DRIVER28_SECURITY_SUPPORT 0
+#define DRIVER29_SECURITY_SUPPORT 0
+#define DRIVER30_SECURITY_SUPPORT 0
+#define DRIVER31_SECURITY_SUPPORT 0
+#define SUPPORT_BUFFER_SYNC 1
diff --git a/drivers/gpu/drm/img/img-volcanic/config_kernel.mk b/drivers/gpu/drm/img/img-volcanic/config_kernel.mk
new file mode 100644
index 000000000000..a1ca43790015
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/config_kernel.mk
@@ -0,0 +1,55 @@
+override HOST_PRIMARY_ARCH := host_x86_64
+override HOST_32BIT_ARCH := host_i386
+override HOST_FORCE_32BIT := -m32
+override HOST_ALL_ARCH := host_x86_64 host_i386
+override TARGET_PRIMARY_ARCH := target_riscv64
+override TARGET_SECONDARY_ARCH :=
+override TARGET_ALL_ARCH := target_riscv64
+override TARGET_FORCE_32BIT :=
+override PVR_ARCH := volcanic
+override PVRSRV_DIR := services
+override METAG_VERSION_NEEDED := 2.8.1.0.3
+override RISCV_VERSION_NEEDED := 1.7.1
+override KERNELDIR := /lib/modules/6.6.21-eic7x/build/
+override KERNEL_ID := 6.6.21-eic7x
+override PVRSRV_MODULE_BASEDIR := /lib/modules/6.6.21-eic7x/extra/
+override KERNEL_COMPONENTS := srvkm drm_nulldisp drm_nulldisp
+override KERNEL_CROSS_COMPILE := riscv64-unknown-linux-gnu-
+override WINDOW_SYSTEM := xorg
+override PVRSRV_MODNAME := pvrsrvkm
+override PVRHMMU_MODNAME :=
+override PVR_BUILD_DIR := eswin_linux
+override PVR_BUILD_TYPE := release
+override SUPPORT_RGX := 1
+override DISPLAY_CONTROLLER := drm_nulldisp
+override PVR_SYSTEM := eswin_cpu
+override PVR_LOADER :=
+override BUILD := release
+override SORT_BRIDGE_STRUCTS := 1
+override DEBUGLINK := 1
+override RGX_BNC := 30.V.408.101
+override PVRSRV_ENABLE_XD_MEM := 1
+override SUPPORT_LINUX_FDINFO := 1
+override PVRSRV_TRACE_ROGUE_EVENTS := 1
+override SUPPORT_PHYSMEM_TEST := 1
+override RGX_NUM_DRIVERS_SUPPORTED := 1
+override RGX_FW_HEAP_OSID_ASSIGNMENT := RGX_FW_HEAP_USES_FIRMWARE_OSID
+override VMM_TYPE := stub
+override SUPPORT_POWMON_COMPONENT := 1
+override RGX_SECONDARY_OS_CLOCK_SOURCE := mono 
+override PDVFS_COM_HOST := 1
+override PDVFS_COM_AP := 2
+override PDVFS_COM_PMC := 3
+override PDVFS_COM_IMG_CLKDIV := 4
+override PDVFS_COM_SYSREG := 5
+override PDVFS_COM := PDVFS_COM_HOST
+override SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY := 1
+override PVR_GPIO_MODE_GENERAL := 1
+override PVR_GPIO_MODE := PVR_GPIO_MODE_GENERAL
+override PVR_HANDLE_BACKEND := idr
+override SUPPORT_DMABUF_BRIDGE := 1
+override SUPPORT_DI_BRG_IMPL := 1
+override SUPPORT_WRAP_EXTMEM := 1
+override SUPPORT_NATIVE_FENCE_SYNC := 1
+override SUPPORT_DMA_FENCE := 1
+override SUPPORT_BUFFER_SYNC := 1
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_bridge.h
new file mode 100644
index 000000000000..1dec13fccce7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_CACHE_BRIDGE_H
+#define CLIENT_CACHE_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_cache_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp);
+
+#endif /* CLIENT_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_direct_bridge.c
new file mode 100644
index 000000000000..9691bae9352f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_direct_bridge.c
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for cache
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_cache_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "cache_ops.h"
+
+#include "cache_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PMR **psPMRInt;
+
+	psPMRInt = (PMR **) phPMR;
+
+	eError =
+	    CacheOpQueue(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			 ui32NumCacheOps,
+			 psPMRInt, pui64Address, puiOffset, puiSize, piuCacheOp, ui32OpTimeline);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = CacheOpValExec(psPMRInt, ui64Address, uiOffset, uiSize, iuCacheOp);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    CacheOpLog(psPMRInt,
+		       ui64Address, uiOffset, uiSize, i64StartTime, i64EndTime, iuCacheOp);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/common_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/common_cache_bridge.h
new file mode 100644
index 000000000000..cc848753f7fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/common_cache_bridge.h
@@ -0,0 +1,126 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CACHE_BRIDGE_H
+#define COMMON_CACHE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_CACHE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE			PVRSRV_BRIDGE_CACHE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPEXEC			PVRSRV_BRIDGE_CACHE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CACHE_CACHEOPLOG			PVRSRV_BRIDGE_CACHE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CACHE_CMD_LAST			(PVRSRV_BRIDGE_CACHE_CMD_FIRST+2)
+
+/*******************************************
+            CacheOpQueue
+ *******************************************/
+
+/* Bridge in structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPQUEUE_TAG
+{
+	PVRSRV_CACHE_OP *piuCacheOp;
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	IMG_UINT32 ui32NumCacheOps;
+	IMG_UINT32 ui32OpTimeline;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPQUEUE;
+
+/* Bridge out structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPQUEUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPQUEUE;
+
+/*******************************************
+            CacheOpExec
+ *******************************************/
+
+/* Bridge in structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPEXEC_TAG
+{
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPEXEC;
+
+/* Bridge out structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPEXEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPEXEC;
+
+/*******************************************
+            CacheOpLog
+ *******************************************/
+
+/* Bridge in structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPLOG_TAG
+{
+	IMG_INT64 i64EndTime;
+	IMG_INT64 i64StartTime;
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPLOG;
+
+/* Bridge out structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPLOG;
+
+#endif /* COMMON_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/server_cache_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/server_cache_bridge.c
new file mode 100644
index 000000000000..b2f5267cc597
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/server_cache_bridge.c
@@ -0,0 +1,452 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "cache_km.h"
+
+#include "common_cache_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(CACHE_BATCH_MAX <= IMG_UINT32_MAX,
+	      "CACHE_BATCH_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeCacheOpQueue(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psCacheOpQueueIN_UI8,
+			 IMG_UINT8 * psCacheOpQueueOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPQUEUE *psCacheOpQueueIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *psCacheOpQueueOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+	PVRSRV_CACHE_OP *iuCacheOpInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) + 0;
+
+	if (unlikely(psCacheOpQueueIN->ui32NumCacheOps > CACHE_BATCH_MAX))
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto CacheOpQueue_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto CacheOpQueue_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psCacheOpQueueIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psCacheOpQueueIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psCacheOpQueueOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *));
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psCacheOpQueueIN->phPMR,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psCacheOpQueueIN->pui64Address,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psCacheOpQueueIN->puiOffset,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psCacheOpQueueIN->puiSize,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		iuCacheOpInt =
+		    (PVRSRV_CACHE_OP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, iuCacheOpInt, (const void __user *)psCacheOpQueueIN->piuCacheOp,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+			/* Look up the address from the handle */
+			psCacheOpQueueOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psCacheOpQueueOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpQueueOUT->eError =
+	    CacheOpQueue(psConnection, OSGetDevNode(psConnection),
+			 psCacheOpQueueIN->ui32NumCacheOps,
+			 psPMRInt,
+			 ui64AddressInt,
+			 uiOffsetInt, uiSizeInt, iuCacheOpInt, psCacheOpQueueIN->ui32OpTimeline);
+
+CacheOpQueue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psCacheOpQueueOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpExec(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psCacheOpExecIN_UI8,
+			IMG_UINT8 * psCacheOpExecOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPEXEC *psCacheOpExecIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPEXEC *psCacheOpExecOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpExecIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpExecOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpExecOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpExec_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpExecOUT->eError =
+	    CacheOpValExec(psPMRInt,
+			   psCacheOpExecIN->ui64Address,
+			   psCacheOpExecIN->uiOffset,
+			   psCacheOpExecIN->uiSize, psCacheOpExecIN->iuCacheOp);
+
+CacheOpExec_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpLog(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psCacheOpLogIN_UI8,
+		       IMG_UINT8 * psCacheOpLogOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPLOG *psCacheOpLogIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPLOG *psCacheOpLogOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpLogIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpLogOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpLogOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpLog_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpLogOUT->eError =
+	    CacheOpLog(psPMRInt,
+		       psCacheOpLogIN->ui64Address,
+		       psCacheOpLogIN->uiOffset,
+		       psCacheOpLogIN->uiSize,
+		       psCacheOpLogIN->i64StartTime,
+		       psCacheOpLogIN->i64EndTime, psCacheOpLogIN->iuCacheOp);
+
+CacheOpLog_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+
+/*
+ * Register all CACHE functions with services
+ */
+PVRSRV_ERROR InitCACHEBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE,
+			      PVRSRVBridgeCacheOpQueue, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPQUEUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPQUEUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC,
+			      PVRSRVBridgeCacheOpExec, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPEXEC),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPEXEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG,
+			      PVRSRVBridgeCacheOpLog, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPLOG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cache functions with services
+ */
+void DeinitCACHEBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/common_cmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/common_cmm_bridge.h
new file mode 100644
index 000000000000..da48de3598be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/common_cmm_bridge.h
@@ -0,0 +1,114 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CMM_BRIDGE_H
+#define COMMON_CMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_CMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CMM_CMD_LAST			(PVRSRV_BRIDGE_CMM_CMD_FIRST+2)
+
+/*******************************************
+            DevmemIntExportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX;
+
+/* Bridge out structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX;
+
+/*******************************************
+            DevmemIntUnexportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX;
+
+/* Bridge out structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX;
+
+/*******************************************
+            DevmemIntAcquireRemoteCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX;
+
+/* Bridge out structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX;
+
+#endif /* COMMON_CMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/server_cmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/server_cmm_bridge.c
new file mode 100644
index 000000000000..cc658c793d3b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/server_cmm_bridge.c
@@ -0,0 +1,414 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_server.h"
+
+#include "common_cmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DevmemIntExportCtxpsContextExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnexportCtx((DEVMEMINT_CTX_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntExportCtx(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntExportCtxIN_UI8,
+			       IMG_UINT8 * psDevmemIntExportCtxOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxOUT_UI8,
+								     0);
+
+	IMG_HANDLE hContext = psDevmemIntExportCtxIN->hContext;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntExportCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX_EXPORT *psContextExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError =
+	    DevmemIntExportCtx(psContextInt, psPMRInt, &psContextExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntExportCtxOUT->
+								    hContextExport,
+								    (void *)psContextExportInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+								    PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntExportCtxpsContextExportIntRelease);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntExportCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntExportCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psContextExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DevmemIntUnexportCtx(psContextExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnexportCtx(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntUnexportCtxIN_UI8,
+				 IMG_UINT8 * psDevmemIntUnexportCtxOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntUnexportCtxIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnexportCtxOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnexportCtxOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnexportCtxIN->hContextExport,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT);
+	if (unlikely((psDevmemIntUnexportCtxOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnexportCtxOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnexportCtx_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnexportCtx_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntAcquireRemoteCtxpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntAcquireRemoteCtx(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxIN_UI8,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevmemIntAcquireRemoteCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntAcquireRemoteCtxOUT->hContext = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    DevmemIntAcquireRemoteCtx(psPMRInt, &psContextInt, &hPrivDataInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntAcquireRemoteCtxOUT->hContext,
+				      (void *)psContextInt, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntAcquireRemoteCtxpsContextIntRelease);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psDevmemIntAcquireRemoteCtxOUT->hPrivData,
+					 (void *)hPrivDataInt, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psDevmemIntAcquireRemoteCtxOUT->hContext);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntAcquireRemoteCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntAcquireRemoteCtxOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psDevmemIntAcquireRemoteCtxOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DevmemIntCtxDestroy(psContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_CMM_BRIDGE */
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+
+/*
+ * Register all CMM functions with services
+ */
+PVRSRV_ERROR InitCMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX,
+			      PVRSRVBridgeDevmemIntExportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX,
+			      PVRSRVBridgeDevmemIntUnexportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX,
+			      PVRSRVBridgeDevmemIntAcquireRemoteCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cmm functions with services
+ */
+void DeinitCMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX);
+
+}
+#else /* EXCLUDE_CMM_BRIDGE */
+/* This bridge is conditional on EXCLUDE_CMM_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitCMMBridge() \
+	PVRSRV_OK
+
+#define DeinitCMMBridge()
+
+#endif /* EXCLUDE_CMM_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/common_dc_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/common_dc_bridge.h
new file mode 100644
index 000000000000..d811fbd4cef1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/common_dc_bridge.h
@@ -0,0 +1,548 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DC_BRIDGE_H
+#define COMMON_DC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_surface.h"
+#include "dc_external.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE			PVRSRV_BRIDGE_DC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DC_DCDEVICERELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DC_DCGETINFO			PVRSRV_BRIDGE_DC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_DC_DCPANELQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_DC_DCFORMATQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_DC_DCDIMQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_DC_DCSETBLANK			PVRSRV_BRIDGE_DC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING			PVRSRV_BRIDGE_DC_CMD_FIRST+10
+#define PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+11
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+12
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+13
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE			PVRSRV_BRIDGE_DC_CMD_FIRST+14
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK			PVRSRV_BRIDGE_DC_CMD_FIRST+15
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY			PVRSRV_BRIDGE_DC_CMD_FIRST+16
+#define PVRSRV_BRIDGE_DC_DCBUFFERALLOC			PVRSRV_BRIDGE_DC_CMD_FIRST+17
+#define PVRSRV_BRIDGE_DC_DCBUFFERIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+18
+#define PVRSRV_BRIDGE_DC_DCBUFFERFREE			PVRSRV_BRIDGE_DC_CMD_FIRST+19
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+20
+#define PVRSRV_BRIDGE_DC_DCBUFFERPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+21
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+22
+#define PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+23
+#define PVRSRV_BRIDGE_DC_DCBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+24
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2			PVRSRV_BRIDGE_DC_CMD_FIRST+25
+#define PVRSRV_BRIDGE_DC_CMD_LAST			(PVRSRV_BRIDGE_DC_CMD_FIRST+25)
+
+/*******************************************
+            DCDevicesQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT;
+
+/* Bridge out structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT;
+
+/*******************************************
+            DCDevicesEnumerate
+ *******************************************/
+
+/* Bridge in structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	IMG_UINT32 ui32DeviceArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE;
+
+/* Bridge out structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE;
+
+/*******************************************
+            DCDeviceAcquire
+ *******************************************/
+
+/* Bridge in structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE_TAG
+{
+	IMG_UINT32 ui32DeviceIndex;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE;
+
+/* Bridge out structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE;
+
+/*******************************************
+            DCDeviceRelease
+ *******************************************/
+
+/* Bridge in structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICERELEASE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICERELEASE;
+
+/* Bridge out structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICERELEASE;
+
+/*******************************************
+            DCGetInfo
+ *******************************************/
+
+/* Bridge in structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_IN_DCGETINFO_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCGETINFO;
+
+/* Bridge out structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DCGETINFO_TAG
+{
+	DC_DISPLAY_INFO sDisplayInfo;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCGETINFO;
+
+/*******************************************
+            DCPanelQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT;
+
+/* Bridge out structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT;
+
+/*******************************************
+            DCPanelQuery
+ *******************************************/
+
+/* Bridge in structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	IMG_UINT32 ui32PanelsArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERY;
+
+/* Bridge out structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERY_TAG
+{
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERY;
+
+/*******************************************
+            DCFormatQuery
+ *******************************************/
+
+/* Bridge in structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCFORMATQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_FORMAT *psFormat;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumFormats;
+} __packed PVRSRV_BRIDGE_IN_DCFORMATQUERY;
+
+/* Bridge out structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCFORMATQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCFORMATQUERY;
+
+/*******************************************
+            DCDimQuery
+ *******************************************/
+
+/* Bridge in structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCDIMQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_DIMS *psDim;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumDims;
+} __packed PVRSRV_BRIDGE_IN_DCDIMQUERY;
+
+/* Bridge out structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCDIMQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDIMQUERY;
+
+/*******************************************
+            DCSetBlank
+ *******************************************/
+
+/* Bridge in structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_IN_DCSETBLANK_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETBLANK;
+
+/* Bridge out structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETBLANK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETBLANK;
+
+/*******************************************
+            DCSetVSyncReporting
+ *******************************************/
+
+/* Bridge in structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING;
+
+/* Bridge out structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING;
+
+/*******************************************
+            DCLastVSyncQuery
+ *******************************************/
+
+/* Bridge in structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY;
+
+/* Bridge out structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY_TAG
+{
+	IMG_INT64 i64Timestamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY;
+
+/*******************************************
+            DCSystemBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE;
+
+/* Bridge out structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE;
+
+/*******************************************
+            DCSystemBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE;
+
+/* Bridge out structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextCreate
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE;
+
+/* Bridge out structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE;
+
+/*******************************************
+            DCDisplayContextConfigureCheck
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/* Bridge out structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/*******************************************
+            DCDisplayContextDestroy
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	IMG_HANDLE hDisplayContext;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY;
+
+/* Bridge out structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY;
+
+/*******************************************
+            DCBufferAlloc
+ *******************************************/
+
+/* Bridge in structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	DC_BUFFER_CREATE_INFO sSurfInfo;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERALLOC;
+
+/* Bridge out structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERALLOC;
+
+/*******************************************
+            DCBufferImport
+ *******************************************/
+
+/* Bridge in structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	IMG_HANDLE *phImport;
+	DC_BUFFER_IMPORT_INFO sSurfAttrib;
+	IMG_UINT32 ui32NumPlanes;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERIMPORT;
+
+/* Bridge out structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT;
+
+/*******************************************
+            DCBufferFree
+ *******************************************/
+
+/* Bridge in structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERFREE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERFREE;
+
+/* Bridge out structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERFREE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERFREE;
+
+/*******************************************
+            DCBufferUnimport
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT;
+
+/* Bridge out structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT;
+
+/*******************************************
+            DCBufferPin
+ *******************************************/
+
+/* Bridge in structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERPIN;
+
+/* Bridge out structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERPIN;
+
+/*******************************************
+            DCBufferUnpin
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNPIN;
+
+/* Bridge out structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN;
+
+/*******************************************
+            DCBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE;
+
+/* Bridge out structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE;
+
+/*******************************************
+            DCBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERRELEASE;
+
+/* Bridge out structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextConfigure2
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	PVRSRV_FENCE hAcquireFence;
+	PVRSRV_TIMELINE hReleaseFenceTimeline;
+	IMG_UINT32 ui32DisplayPeriod;
+	IMG_UINT32 ui32MaxDepth;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2;
+
+/* Bridge out structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hReleaseFence;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2;
+
+#endif /* COMMON_DC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/server_dc_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/server_dc_bridge.c
new file mode 100644
index 000000000000..3b5b9150c2ec
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/server_dc_bridge.c
@@ -0,0 +1,2554 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dc_server.h"
+
+#include "common_dc_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeDCDevicesQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCDevicesQueryCountIN_UI8,
+				IMG_UINT8 * psDCDevicesQueryCountOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDCDevicesQueryCountIN);
+
+	psDCDevicesQueryCountOUT->eError =
+	    DCDevicesQueryCount(&psDCDevicesQueryCountOUT->ui32DeviceCount);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DEVICE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_DEVICE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDevicesEnumerate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDCDevicesEnumerateIN_UI8,
+			       IMG_UINT8 * psDCDevicesEnumerateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *psDCDevicesEnumerateIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *psDCDevicesEnumerateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateOUT_UI8,
+								     0);
+
+	IMG_UINT32 *pui32DeviceIndexInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32)) + 0;
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize > DC_MAX_DEVICE_COUNT)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	psDCDevicesEnumerateOUT->pui32DeviceIndex = psDCDevicesEnumerateIN->pui32DeviceIndex;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDevicesEnumerateIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDevicesEnumerateIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDevicesEnumerate_exit;
+			}
+		}
+	}
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize != 0)
+	{
+		pui32DeviceIndexInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32);
+	}
+
+	psDCDevicesEnumerateOUT->eError =
+	    DCDevicesEnumerate(psConnection, OSGetDevNode(psConnection),
+			       psDCDevicesEnumerateIN->ui32DeviceArraySize,
+			       &psDCDevicesEnumerateOUT->ui32DeviceCount, pui32DeviceIndexInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDevicesEnumerateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDevicesEnumerate_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32DeviceIndexInt) &&
+	    ((psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDevicesEnumerateOUT->pui32DeviceIndex,
+		      pui32DeviceIndexInt,
+		      (psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDevicesEnumerate_exit;
+		}
+	}
+
+DCDevicesEnumerate_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDevicesEnumerateOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDeviceAcquirepsDeviceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDeviceRelease((DC_DEVICE *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceAcquireIN_UI8,
+			    IMG_UINT8 * psDCDeviceAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *psDCDeviceAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *psDCDeviceAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireOUT_UI8, 0);
+
+	DC_DEVICE *psDeviceInt = NULL;
+
+	psDCDeviceAcquireOUT->eError =
+	    DCDeviceAcquire(psConnection, OSGetDevNode(psConnection),
+			    psDCDeviceAcquireIN->ui32DeviceIndex, &psDeviceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDCDeviceAcquireOUT->hDevice,
+								 (void *)psDeviceInt,
+								 PVRSRV_HANDLE_TYPE_DC_DEVICE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _DCDeviceAcquirepsDeviceIntRelease);
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceAcquire_exit:
+
+	if (psDCDeviceAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psDeviceInt)
+		{
+			DCDeviceRelease(psDeviceInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceReleaseIN_UI8,
+			    IMG_UINT8 * psDCDeviceReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICERELEASE *psDCDeviceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *psDCDeviceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDeviceReleaseIN->hDevice,
+					      PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	if (unlikely((psDCDeviceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCDeviceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCGetInfo(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psDCGetInfoIN_UI8,
+		      IMG_UINT8 * psDCGetInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCGETINFO *psDCGetInfoIN =
+	    (PVRSRV_BRIDGE_IN_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCGETINFO *psDCGetInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCGetInfoIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCGetInfoOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCGetInfoOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCGetInfo_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCGetInfoOUT->eError = DCGetInfo(psDeviceInt, &psDCGetInfoOUT->sDisplayInfo);
+
+DCGetInfo_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCPanelQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDCPanelQueryCountIN_UI8,
+			      IMG_UINT8 * psDCPanelQueryCountOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *psDCPanelQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *psDCPanelQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryCountIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryCountOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryCountOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQueryCount_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryCountOUT->eError =
+	    DCPanelQueryCount(psDeviceInt, &psDCPanelQueryCountOUT->ui32NumPanels);
+
+DCPanelQueryCount_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(DC_MAX_PANEL_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PANEL_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCPanelQuery(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCPanelQueryIN_UI8,
+			 IMG_UINT8 * psDCPanelQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERY *psDCPanelQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERY *psDCPanelQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_PANEL_INFO *psPanelInfoInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO)) + 0;
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize > DC_MAX_PANEL_COUNT)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCPanelQuery_exit;
+	}
+
+	psDCPanelQueryOUT->psPanelInfo = psDCPanelQueryIN->psPanelInfo;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCPanelQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCPanelQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCPanelQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCPanelQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCPanelQuery_exit;
+			}
+		}
+	}
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize != 0)
+	{
+		psPanelInfoInt =
+		    (PVRSRV_PANEL_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryOUT->eError =
+	    DCPanelQuery(psDeviceInt,
+			 psDCPanelQueryIN->ui32PanelsArraySize,
+			 &psDCPanelQueryOUT->ui32NumPanels, psPanelInfoInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCPanelQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psPanelInfoInt) &&
+	    ((psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCPanelQueryOUT->psPanelInfo, psPanelInfoInt,
+		      (psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO))) != PVRSRV_OK))
+		{
+			psDCPanelQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCPanelQuery_exit;
+		}
+	}
+
+DCPanelQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCPanelQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_FORMATS <= IMG_UINT32_MAX,
+	      "DC_MAX_FORMATS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCFormatQuery(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCFormatQueryIN_UI8,
+			  IMG_UINT8 * psDCFormatQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCFORMATQUERY *psDCFormatQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCFORMATQUERY *psDCFormatQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCFormatQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_FORMAT *psFormatInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) +
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCFormatQueryIN->ui32NumFormats > DC_MAX_FORMATS))
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCFormatQuery_exit;
+	}
+
+	psDCFormatQueryOUT->pui32Supported = psDCFormatQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCFormatQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCFormatQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCFormatQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCFormatQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCFormatQuery_exit;
+			}
+		}
+	}
+
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		psFormatInt =
+		    (PVRSRV_SURFACE_FORMAT *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT);
+	}
+
+	/* Copy the data over */
+	if (psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psFormatInt, (const void __user *)psDCFormatQueryIN->psFormat,
+		     psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) !=
+		    PVRSRV_OK)
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCFormatQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCFormatQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCFormatQueryOUT->eError =
+	    DCFormatQuery(psDeviceInt,
+			  psDCFormatQueryIN->ui32NumFormats, psFormatInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCFormatQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCFormatQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+
+DCFormatQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCFormatQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DIMENSIONS <= IMG_UINT32_MAX,
+	      "DC_MAX_DIMENSIONS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDimQuery(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCDimQueryIN_UI8,
+		       IMG_UINT8 * psDCDimQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDIMQUERY *psDCDimQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDIMQUERY *psDCDimQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDimQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_DIMS *psDimInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) +
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCDimQueryIN->ui32NumDims > DC_MAX_DIMENSIONS))
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDimQuery_exit;
+	}
+
+	psDCDimQueryOUT->pui32Supported = psDCDimQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDimQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDimQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDimQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDimQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDimQuery_exit;
+			}
+		}
+	}
+
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		psDimInt =
+		    (PVRSRV_SURFACE_DIMS *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS);
+	}
+
+	/* Copy the data over */
+	if (psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psDimInt, (const void __user *)psDCDimQueryIN->psDim,
+		     psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) != PVRSRV_OK)
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDimQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDimQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDimQueryOUT->eError =
+	    DCDimQuery(psDeviceInt, psDCDimQueryIN->ui32NumDims, psDimInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCDimQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDimQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+
+DCDimQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDimQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetBlank(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCSetBlankIN_UI8,
+		       IMG_UINT8 * psDCSetBlankOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETBLANK *psDCSetBlankIN =
+	    (PVRSRV_BRIDGE_IN_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSETBLANK *psDCSetBlankOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSetBlankIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetBlankOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetBlankOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetBlank_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetBlankOUT->eError = DCSetBlank(psDeviceInt, psDCSetBlankIN->bEnabled);
+
+DCSetBlank_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetVSyncReporting(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCSetVSyncReportingIN_UI8,
+				IMG_UINT8 * psDCSetVSyncReportingOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *psDCSetVSyncReportingIN =
+	    (PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *psDCSetVSyncReportingOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevice = psDCSetVSyncReportingIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetVSyncReportingOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetVSyncReportingOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetVSyncReporting_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetVSyncReportingOUT->eError =
+	    DCSetVSyncReporting(psDeviceInt, psDCSetVSyncReportingIN->bEnabled);
+
+DCSetVSyncReporting_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCLastVSyncQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCLastVSyncQueryIN_UI8,
+			     IMG_UINT8 * psDCLastVSyncQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *psDCLastVSyncQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *psDCLastVSyncQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCLastVSyncQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCLastVSyncQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCLastVSyncQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCLastVSyncQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCLastVSyncQueryOUT->eError =
+	    DCLastVSyncQuery(psDeviceInt, &psDCLastVSyncQueryOUT->i64Timestamp);
+
+DCLastVSyncQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCSystemBufferAcquirepsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCSystemBufferRelease((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferAcquireIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferAcquireOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSystemBufferAcquireIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSystemBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError =
+	    DCSystemBufferAcquire(psDeviceInt,
+				  &psDCSystemBufferAcquireOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDCSystemBufferAcquireOUT->
+								       hBuffer, (void *)psBufferInt,
+								       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DCSystemBufferAcquirepsBufferIntRelease);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCSystemBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCSystemBufferRelease(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferReleaseIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCSystemBufferReleaseIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCSystemBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCSystemBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDisplayContextCreatepsDisplayContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDisplayContextDestroy((DC_DISPLAY_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextCreate(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDCDisplayContextCreateIN_UI8,
+				   IMG_UINT8 * psDCDisplayContextCreateOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDisplayContextCreateIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError =
+	    DCDisplayContextCreate(psDeviceInt, &psDisplayContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDCDisplayContextCreateOUT->
+									hDisplayContext,
+									(void *)psDisplayContextInt,
+									PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DCDisplayContextCreatepsDisplayContextIntRelease);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCDisplayContextCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDisplayContextInt)
+		{
+			DCDisplayContextDestroy(psDisplayContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigureCheck(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckIN_UI8,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigureCheckIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) +
+	    0;
+
+	if (unlikely(psDCDisplayContextConfigureCheckIN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigureCheckOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigureCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigureCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigureCheckOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) >
+	    0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->psSurfInfo,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+			       sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->phBuffers,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigureCheckOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    DCDisplayContextConfigureCheck(psDisplayContextInt,
+					   psDCDisplayContextConfigureCheckIN->ui32PipeCount,
+					   psSurfInfoInt, psBuffersInt);
+
+DCDisplayContextConfigureCheck_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigureCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDCDisplayContextDestroyIN_UI8,
+				    IMG_UINT8 * psDCDisplayContextDestroyOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDisplayContextDestroyIN->
+					      hDisplayContext,
+					      PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	if (unlikely
+	    ((psDCDisplayContextDestroyOUT->eError != PVRSRV_OK)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCDisplayContextDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAllocpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAlloc(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferAllocIN_UI8,
+			  IMG_UINT8 * psDCBufferAllocOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERALLOC *psDCBufferAllocIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *psDCBufferAllocOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferAllocIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAllocOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError =
+	    DCBufferAlloc(psDisplayContextInt,
+			  &psDCBufferAllocIN->sSurfInfo,
+			  &psDCBufferAllocOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psDCBufferAllocOUT->hBuffer,
+							       (void *)psBufferInt,
+							       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+							       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							       (PFN_HANDLE_RELEASE) &
+							       _DCBufferAllocpsBufferIntRelease);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferAlloc_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAllocOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferImportpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static_assert(DC_MAX_PLANES <= IMG_UINT32_MAX,
+	      "DC_MAX_PLANES must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCBufferImport(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psDCBufferImportIN_UI8,
+			   IMG_UINT8 * psDCBufferImportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *psDCBufferImportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *psDCBufferImportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferImportIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PMR **psImportInt = NULL;
+	IMG_HANDLE *hImportInt2 = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCBufferImportIN->ui32NumPlanes > DC_MAX_PLANES))
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCBufferImport_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCBufferImport_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCBufferImportIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCBufferImportIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCBufferImportOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+
+	if (psDCBufferImportIN->ui32NumPlanes != 0)
+	{
+		psImportInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psImportInt, 0, psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *));
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *);
+		hImportInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hImportInt2, (const void __user *)psDCBufferImportIN->phImport,
+		     psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDCBufferImportOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCBufferImport_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferImportOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+			/* Look up the address from the handle */
+			psDCBufferImportOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psImportInt[i],
+						       hImportInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError =
+	    DCBufferImport(psDisplayContextInt,
+			   psDCBufferImportIN->ui32NumPlanes,
+			   psImportInt, &psDCBufferImportIN->sSurfAttrib, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferImport_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								&psDCBufferImportOUT->hBuffer,
+								(void *)psBufferInt,
+								PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								(PFN_HANDLE_RELEASE) &
+								_DCBufferImportpsBufferIntRelease);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferImport_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hImportInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psImportInt && psImportInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hImportInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferImportOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCBufferImportOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferFree(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCBufferFreeIN_UI8,
+			 IMG_UINT8 * psDCBufferFreeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERFREE *psDCBufferFreeIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERFREE *psDCBufferFreeOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferFreeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferFreeIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferFreeOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferFreeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferFree_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferFree_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnimport(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCBufferUnimportIN_UI8,
+			     IMG_UINT8 * psDCBufferUnimportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *psDCBufferUnimportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *psDCBufferUnimportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnimportOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnimportIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferUnimportOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnimportOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnimport_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnimport_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferPinhPinHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferUnpin((DC_PIN_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferPin(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDCBufferPinIN_UI8,
+			IMG_UINT8 * psDCBufferPinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERPIN *psDCBufferPinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERPIN *psDCBufferPinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferPinIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	DC_PIN_HANDLE hPinHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferPinOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = DCBufferPin(psBufferInt, &hPinHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferPin_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							     &psDCBufferPinOUT->hPinHandle,
+							     (void *)hPinHandleInt,
+							     PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+							     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							     (PFN_HANDLE_RELEASE) &
+							     _DCBufferPinhPinHandleIntRelease);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferPin_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferPinOUT->eError != PVRSRV_OK)
+	{
+		if (hPinHandleInt)
+		{
+			DCBufferUnpin(hPinHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnpin(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferUnpinIN_UI8,
+			  IMG_UINT8 * psDCBufferUnpinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *psDCBufferUnpinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *psDCBufferUnpinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnpinOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnpinIN->hPinHandle,
+					      PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE);
+	if (unlikely((psDCBufferUnpinOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnpinOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnpin_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnpin_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAcquirepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferRelease((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferAcquireIN_UI8,
+			    IMG_UINT8 * psDCBufferAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *psDCBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *psDCBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferAcquireIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAcquireOUT->eError = DCBufferAcquire(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psDCBufferAcquireOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _DCBufferAcquirepsExtMemIntRelease);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferRelease(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferReleaseIN_UI8,
+			    IMG_UINT8 * psDCBufferReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *psDCBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *psDCBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psDCBufferReleaseIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psDCBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferRelease_exit:
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigure2(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psDCDisplayContextConfigure2IN_UI8,
+				       IMG_UINT8 * psDCDisplayContextConfigure2OUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2IN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2OUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2OUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigure2IN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCDisplayContextConfigure2IN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigure2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigure2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigure2OUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigure2IN->psSurfInfo,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigure2IN->phBuffers,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigure2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigure2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigure2OUT->eError =
+	    DCDisplayContextConfigure(psDisplayContextInt,
+				      psDCDisplayContextConfigure2IN->ui32PipeCount,
+				      psSurfInfoInt,
+				      psBuffersInt,
+				      psDCDisplayContextConfigure2IN->ui32DisplayPeriod,
+				      psDCDisplayContextConfigure2IN->ui32MaxDepth,
+				      psDCDisplayContextConfigure2IN->hAcquireFence,
+				      psDCDisplayContextConfigure2IN->hReleaseFenceTimeline,
+				      &psDCDisplayContextConfigure2OUT->hReleaseFence);
+
+DCDisplayContextConfigure2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigure2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+
+/*
+ * Register all DC functions with services
+ */
+PVRSRV_ERROR InitDCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT,
+			      PVRSRVBridgeDCDevicesQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE,
+			      PVRSRVBridgeDCDevicesEnumerate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE,
+			      PVRSRVBridgeDCDeviceAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE,
+			      PVRSRVBridgeDCDeviceRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO, PVRSRVBridgeDCGetInfo,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT,
+			      PVRSRVBridgeDCPanelQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY,
+			      PVRSRVBridgeDCPanelQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY,
+			      PVRSRVBridgeDCFormatQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY, PVRSRVBridgeDCDimQuery,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK, PVRSRVBridgeDCSetBlank,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING,
+			      PVRSRVBridgeDCSetVSyncReporting, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY,
+			      PVRSRVBridgeDCLastVSyncQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE,
+			      PVRSRVBridgeDCSystemBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE,
+			      PVRSRVBridgeDCSystemBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE,
+			      PVRSRVBridgeDCDisplayContextCreate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK,
+			      PVRSRVBridgeDCDisplayContextConfigureCheck, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY,
+			      PVRSRVBridgeDCDisplayContextDestroy, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC,
+			      PVRSRVBridgeDCBufferAlloc, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT,
+			      PVRSRVBridgeDCBufferImport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE,
+			      PVRSRVBridgeDCBufferFree, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT,
+			      PVRSRVBridgeDCBufferUnimport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN,
+			      PVRSRVBridgeDCBufferPin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN,
+			      PVRSRVBridgeDCBufferUnpin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE,
+			      PVRSRVBridgeDCBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE,
+			      PVRSRVBridgeDCBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2,
+			      PVRSRVBridgeDCDisplayContextConfigure2, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dc functions with services
+ */
+void DeinitDCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_bridge.h
new file mode 100644
index 000000000000..bfa6bfb9c037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_bridge.h
@@ -0,0 +1,111 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+#define CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_devicememhistory_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut);
+
+#endif /* CLIENT_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
new file mode 100644
index 000000000000..a593a7e2cc7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
@@ -0,0 +1,195 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for devicememhistory
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_devicememhistory_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  uiOffset,
+				  sDevVAddr,
+				  uiSize,
+				  puiText,
+				  ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    uiOffset,
+				    sDevVAddr,
+				    uiSize,
+				    puiText,
+				    ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryMapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					sBaseDevVAddr,
+					ui32ui32StartPage,
+					ui32NumPages,
+					uiAllocSize,
+					puiText,
+					ui32Log2PageSize,
+					ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryUnmapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  sBaseDevVAddr,
+					  ui32ui32StartPage,
+					  ui32NumPages,
+					  uiAllocSize,
+					  puiText,
+					  ui32Log2PageSize,
+					  ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   uiOffset,
+					   sDevVAddr,
+					   uiSize,
+					   puiText,
+					   ui32Log2PageSize,
+					   ui32AllocPageCount,
+					   pui32AllocPageIndices,
+					   ui32FreePageCount,
+					   pui32FreePageIndices,
+					   ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/common_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/common_devicememhistory_bridge.h
new file mode 100644
index 000000000000..800f98dd2836
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/common_devicememhistory_bridge.h
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DEVICEMEMHISTORY_BRIDGE_H
+#define COMMON_DEVICEMEMHISTORY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST			(PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4)
+
+/*******************************************
+            DevicememHistoryMap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP;
+
+/* Bridge out structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP;
+
+/*******************************************
+            DevicememHistoryUnmap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP;
+
+/* Bridge out structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP;
+
+/*******************************************
+            DevicememHistoryMapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE;
+
+/*******************************************
+            DevicememHistoryUnmapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/*******************************************
+            DevicememHistorySparseChange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE;
+
+/* Bridge out structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE;
+
+#endif /* COMMON_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/server_devicememhistory_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/server_devicememhistory_bridge.c
new file mode 100644
index 000000000000..72c6a86a0ec8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/server_devicememhistory_bridge.c
@@ -0,0 +1,835 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+#include "common_devicememhistory_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMap(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevicememHistoryMapIN_UI8,
+				IMG_UINT8 * psDevicememHistoryMapOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *psDevicememHistoryMapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *psDevicememHistoryMapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryMapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryMapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryMapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryMap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryMapOUT->eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  psDevicememHistoryMapIN->uiOffset,
+				  psDevicememHistoryMapIN->sDevVAddr,
+				  psDevicememHistoryMapIN->uiSize,
+				  uiTextInt,
+				  psDevicememHistoryMapIN->ui32Log2PageSize,
+				  psDevicememHistoryMapIN->ui32AllocationIndex,
+				  &psDevicememHistoryMapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmap(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevicememHistoryUnmapIN_UI8,
+				  IMG_UINT8 * psDevicememHistoryUnmapOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryUnmapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryUnmapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryUnmapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryUnmapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryUnmap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryUnmapOUT->eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    psDevicememHistoryUnmapIN->uiOffset,
+				    psDevicememHistoryUnmapIN->sDevVAddr,
+				    psDevicememHistoryUnmapIN->uiSize,
+				    uiTextInt,
+				    psDevicememHistoryUnmapIN->ui32Log2PageSize,
+				    psDevicememHistoryUnmapIN->ui32AllocationIndex,
+				    &psDevicememHistoryUnmapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryUnmap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMapVRange(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeIN_UI8,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryMapVRangeOUT->eError =
+	    DevicememHistoryMapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					psDevicememHistoryMapVRangeIN->sBaseDevVAddr,
+					psDevicememHistoryMapVRangeIN->ui32ui32StartPage,
+					psDevicememHistoryMapVRangeIN->ui32NumPages,
+					psDevicememHistoryMapVRangeIN->uiAllocSize,
+					uiTextInt,
+					psDevicememHistoryMapVRangeIN->ui32Log2PageSize,
+					psDevicememHistoryMapVRangeIN->ui32AllocationIndex,
+					&psDevicememHistoryMapVRangeOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmapVRange(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeIN_UI8,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistoryUnmapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapVRangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryUnmapVRangeOUT->eError =
+	    DevicememHistoryUnmapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					  psDevicememHistoryUnmapVRangeIN->sBaseDevVAddr,
+					  psDevicememHistoryUnmapVRangeIN->ui32ui32StartPage,
+					  psDevicememHistoryUnmapVRangeIN->ui32NumPages,
+					  psDevicememHistoryUnmapVRangeIN->uiAllocSize,
+					  uiTextInt,
+					  psDevicememHistoryUnmapVRangeIN->ui32Log2PageSize,
+					  psDevicememHistoryUnmapVRangeIN->ui32AllocationIndex,
+					  &psDevicememHistoryUnmapVRangeOUT->
+					  ui32AllocationIndexOut);
+
+DevicememHistoryUnmapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistorySparseChange(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psDevicememHistorySparseChangeIN_UI8,
+					 IMG_UINT8 * psDevicememHistorySparseChangeOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistorySparseChangeIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32AllocPageCount *
+	     sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32FreePageCount *
+	     sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32AllocPageCount >
+	     PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistorySparseChangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistorySparseChangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistorySparseChangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistorySparseChange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32AllocPageIndices,
+		     psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32FreePageIndices,
+		     psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistorySparseChangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistorySparseChangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistorySparseChange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistorySparseChangeOUT->eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   psDevicememHistorySparseChangeIN->uiOffset,
+					   psDevicememHistorySparseChangeIN->sDevVAddr,
+					   psDevicememHistorySparseChangeIN->uiSize,
+					   uiTextInt,
+					   psDevicememHistorySparseChangeIN->ui32Log2PageSize,
+					   psDevicememHistorySparseChangeIN->ui32AllocPageCount,
+					   ui32AllocPageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32FreePageCount,
+					   ui32FreePageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32AllocationIndex,
+					   &psDevicememHistorySparseChangeOUT->
+					   ui32AllocationIndexOut);
+
+DevicememHistorySparseChange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistorySparseChangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pDEVICEMEMHISTORYBridgeLock;
+
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+
+/*
+ * Register all DEVICEMEMHISTORY functions with services
+ */
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pDEVICEMEMHISTORYBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP,
+			      PVRSRVBridgeDevicememHistoryMap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP,
+			      PVRSRVBridgeDevicememHistoryUnmap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryMapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryUnmapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE,
+			      PVRSRVBridgeDevicememHistorySparseChange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all devicememhistory functions with services
+ */
+void DeinitDEVICEMEMHISTORYBridge(void)
+{
+	OSLockDestroy(pDEVICEMEMHISTORYBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/common_di_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/common_di_bridge.h
new file mode 100644
index 000000000000..8591006140b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/common_di_bridge.h
@@ -0,0 +1,153 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DI_BRIDGE_H
+#define COMMON_DI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "pvr_dicommon.h"
+
+#define PVRSRV_BRIDGE_DI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DI_DICREATECONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DI_DIREADENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DI_DIWRITEENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DI_DILISTALLENTRIES			PVRSRV_BRIDGE_DI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DI_CMD_LAST			(PVRSRV_BRIDGE_DI_CMD_FIRST+4)
+
+/*******************************************
+            DICreateContext
+ *******************************************/
+
+/* Bridge in structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_IN_DICREATECONTEXT_TAG
+{
+	IMG_CHAR *puiStreamName;
+} __packed PVRSRV_BRIDGE_IN_DICREATECONTEXT;
+
+/* Bridge out structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_OUT_DICREATECONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_CHAR *puiStreamName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DICREATECONTEXT;
+
+/*******************************************
+            DIDestroyContext
+ *******************************************/
+
+/* Bridge in structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT;
+
+/* Bridge out structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT;
+
+/*******************************************
+            DIReadEntry
+ *******************************************/
+
+/* Bridge in structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIREADENTRY_TAG
+{
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+} __packed PVRSRV_BRIDGE_IN_DIREADENTRY;
+
+/* Bridge out structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIREADENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIREADENTRY;
+
+/*******************************************
+            DIWriteEntry
+ *******************************************/
+
+/* Bridge in structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIWRITEENTRY_TAG
+{
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+	const IMG_CHAR *puiValue;
+	IMG_UINT32 ui32ValueSize;
+} __packed PVRSRV_BRIDGE_IN_DIWRITEENTRY;
+
+/* Bridge out structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIWRITEENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIWRITEENTRY;
+
+/*******************************************
+            DIListAllEntries
+ *******************************************/
+
+/* Bridge in structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_IN_DILISTALLENTRIES_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DILISTALLENTRIES;
+
+/* Bridge out structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_OUT_DILISTALLENTRIES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DILISTALLENTRIES;
+
+#endif /* COMMON_DI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/server_di_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/server_di_bridge.c
new file mode 100644
index 000000000000..154bccdaae5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/server_di_bridge.c
@@ -0,0 +1,652 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "di_impl_brg.h"
+
+#include "common_di_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DICreateContextpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DIDestroyContextKM((DI_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDICreateContext(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDICreateContextIN_UI8,
+			    IMG_UINT8 * psDICreateContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DICREATECONTEXT *psDICreateContextIN =
+	    (PVRSRV_BRIDGE_IN_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DICREATECONTEXT *psDICreateContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextOUT_UI8, 0);
+
+	IMG_CHAR *puiStreamNameInt = NULL;
+	DI_CONTEXT *psContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	PVR_UNREFERENCED_PARAMETER(psDICreateContextIN);
+
+	psDICreateContextOUT->puiStreamName = psDICreateContextIN->puiStreamName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDICreateContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DICreateContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDICreateContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDICreateContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDICreateContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DICreateContext_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		puiStreamNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	psDICreateContextOUT->eError = DICreateContextKM(puiStreamNameInt, &psContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		goto DICreateContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDICreateContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDICreateContextOUT->hContext,
+								 (void *)psContextInt,
+								 PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								 (PFN_HANDLE_RELEASE) &
+								 _DICreateContextpsContextIntRelease);
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DICreateContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamNameInt) && ((PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDICreateContextOUT->puiStreamName, puiStreamNameInt,
+		      (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psDICreateContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DICreateContext_exit;
+		}
+	}
+
+DICreateContext_exit:
+
+	if (psDICreateContextOUT->eError != PVRSRV_OK)
+	{
+		if (psDICreateContextOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDICreateContextOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DIDestroyContextKM(psContextInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDICreateContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIDestroyContext(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIDestroyContextIN_UI8,
+			     IMG_UINT8 * psDIDestroyContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *psDIDestroyContextIN =
+	    (PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *psDIDestroyContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDIDestroyContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDIDestroyContextIN->hContext,
+					      PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	if (unlikely((psDIDestroyContextOUT->eError != PVRSRV_OK) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDIDestroyContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIDestroyContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DIDestroyContext_exit:
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIReadEntry(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDIReadEntryIN_UI8,
+			IMG_UINT8 * psDIReadEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIREADENTRY *psDIReadEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIREADENTRY *psDIReadEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIReadEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIReadEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIReadEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIReadEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIReadEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIReadEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIReadEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIReadEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIReadEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIReadEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIReadEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIReadEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIReadEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIReadEntryOUT->eError =
+	    DIReadEntryKM(psContextInt,
+			  uiEntryPathInt, psDIReadEntryIN->ui64Offset, psDIReadEntryIN->ui64Size);
+
+DIReadEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIReadEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIWriteEntry(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDIWriteEntryIN_UI8,
+			 IMG_UINT8 * psDIWriteEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIWRITEENTRY *psDIWriteEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIWRITEENTRY *psDIWriteEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIWriteEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+	IMG_CHAR *uiValueInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psDIWriteEntryIN->ui32ValueSize > DI_IMPL_BRG_PATH_LEN))
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DIWriteEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIWriteEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIWriteEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIWriteEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIWriteEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIWriteEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIWriteEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDIWriteEntryIN->ui32ValueSize != 0)
+	{
+		uiValueInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiValueInt, (const void __user *)psDIWriteEntryIN->puiValue,
+		     psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiValueInt)[(psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) -
+					  1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIWriteEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIWriteEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIWriteEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIWriteEntryOUT->eError =
+	    DIWriteEntryKM(psContextInt,
+			   uiEntryPathInt, psDIWriteEntryIN->ui32ValueSize, uiValueInt);
+
+DIWriteEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIWriteEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIListAllEntries(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIListAllEntriesIN_UI8,
+			     IMG_UINT8 * psDIListAllEntriesOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DILISTALLENTRIES *psDIListAllEntriesIN =
+	    (PVRSRV_BRIDGE_IN_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *psDIListAllEntriesOUT =
+	    (PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIListAllEntriesIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIListAllEntriesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIListAllEntriesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIListAllEntries_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIListAllEntriesOUT->eError = DIListAllEntriesKM(psContextInt);
+
+DIListAllEntries_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+
+/*
+ * Register all DI functions with services
+ */
+PVRSRV_ERROR InitDIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT,
+			      PVRSRVBridgeDICreateContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DICREATECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DICREATECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT,
+			      PVRSRVBridgeDIDestroyContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY,
+			      PVRSRVBridgeDIReadEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIREADENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIREADENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY,
+			      PVRSRVBridgeDIWriteEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIWRITEENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIWRITEENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES,
+			      PVRSRVBridgeDIListAllEntries, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DILISTALLENTRIES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DILISTALLENTRIES));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all di functions with services
+ */
+void DeinitDIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/common_dma_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/common_dma_bridge.h
new file mode 100644
index 000000000000..dafa475b9c9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/common_dma_bridge.h
@@ -0,0 +1,123 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMA_BRIDGE_H
+#define COMMON_DMA_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DMA_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMA_DMATRANSFER			PVRSRV_BRIDGE_DMA_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE			PVRSRV_BRIDGE_DMA_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS			PVRSRV_BRIDGE_DMA_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMA_CMD_LAST			(PVRSRV_BRIDGE_DMA_CMD_FIRST+2)
+
+/*******************************************
+            DmaTransfer
+ *******************************************/
+
+/* Bridge in structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_IN_DMATRANSFER_TAG
+{
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32NumDMAs;
+	IMG_UINT32 ui32uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DMATRANSFER;
+
+/* Bridge out structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_OUT_DMATRANSFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMATRANSFER;
+
+/*******************************************
+            DmaSparseMappingTable
+ *******************************************/
+
+/* Bridge in structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_BOOL *pbTable;
+	IMG_UINT32 ui32SizeInPages;
+} __packed PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE;
+
+/* Bridge out structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_BOOL *pbTable;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE;
+
+/*******************************************
+            DmaDeviceParams
+ *******************************************/
+
+/* Bridge in structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_IN_DMADEVICEPARAMS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DMADEVICEPARAMS;
+
+/* Bridge out structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DmaBuffAlign;
+	IMG_UINT32 ui32DmaTransferMult;
+} __packed PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS;
+
+#endif /* COMMON_DMA_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/server_dma_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/server_dma_bridge.c
new file mode 100644
index 000000000000..7f2c5dd495e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/server_dma_bridge.c
@@ -0,0 +1,490 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dma_km.h"
+
+#include "common_dma_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(MAX_DMA_OPS <= IMG_UINT32_MAX, "MAX_DMA_OPS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaTransfer(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDmaTransferIN_UI8,
+			IMG_UINT8 * psDmaTransferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMATRANSFER *psDmaTransferIN =
+	    (PVRSRV_BRIDGE_IN_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMATRANSFER *psDmaTransferOUT =
+	    (PVRSRV_BRIDGE_OUT_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) + 0;
+
+	if (unlikely(psDmaTransferIN->ui32NumDMAs > MAX_DMA_OPS))
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaTransfer_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaTransfer_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaTransferIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaTransferIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaTransferOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psDmaTransferIN->ui32NumDMAs * sizeof(PMR *));
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psDmaTransferIN->phPMR,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psDmaTransferIN->pui64Address,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psDmaTransferIN->puiOffset,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psDmaTransferIN->puiSize,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+			/* Look up the address from the handle */
+			psDmaTransferOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDmaTransferOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaTransferOUT->eError =
+	    DmaTransfer(psConnection, OSGetDevNode(psConnection),
+			psDmaTransferIN->ui32NumDMAs,
+			psPMRInt,
+			ui64AddressInt,
+			uiOffsetInt,
+			uiSizeInt, psDmaTransferIN->ui32uiFlags, psDmaTransferIN->hUpdateTimeline);
+
+DmaTransfer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaTransferOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(32 <= IMG_UINT32_MAX, "32 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaSparseMappingTable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDmaSparseMappingTableIN_UI8,
+				  IMG_UINT8 * psDmaSparseMappingTableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableIN =
+	    (PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDmaSparseMappingTableIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_BOOL *pbTableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) + 0;
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages > 32)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	psDmaSparseMappingTableOUT->pbTable = psDmaSparseMappingTableIN->pbTable;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaSparseMappingTableIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaSparseMappingTableIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaSparseMappingTable_exit;
+			}
+		}
+	}
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages != 0)
+	{
+		pbTableInt = (IMG_BOOL *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDmaSparseMappingTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DmaSparseMappingTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaSparseMappingTableOUT->eError =
+	    DmaSparseMappingTable(psPMRInt,
+				  psDmaSparseMappingTableIN->uiOffset,
+				  psDmaSparseMappingTableIN->ui32SizeInPages, pbTableInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		goto DmaSparseMappingTable_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pbTableInt) && ((psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDmaSparseMappingTableOUT->pbTable, pbTableInt,
+		      (psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL))) !=
+		     PVRSRV_OK))
+		{
+			psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaSparseMappingTable_exit;
+		}
+	}
+
+DmaSparseMappingTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaSparseMappingTableOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDmaDeviceParams(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDmaDeviceParamsIN_UI8,
+			    IMG_UINT8 * psDmaDeviceParamsOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *psDmaDeviceParamsIN =
+	    (PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *psDmaDeviceParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psDmaDeviceParamsIN);
+
+	psDmaDeviceParamsOUT->eError =
+	    DmaDeviceParams(psConnection, OSGetDevNode(psConnection),
+			    &psDmaDeviceParamsOUT->ui32DmaBuffAlign,
+			    &psDmaDeviceParamsOUT->ui32DmaTransferMult);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+
+/*
+ * Register all DMA functions with services
+ */
+PVRSRV_ERROR InitDMABridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER,
+			      PVRSRVBridgeDmaTransfer, NULL, sizeof(PVRSRV_BRIDGE_IN_DMATRANSFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMATRANSFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE,
+			      PVRSRVBridgeDmaSparseMappingTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS,
+			      PVRSRVBridgeDmaDeviceParams, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dma functions with services
+ */
+void DeinitDMABridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/common_dmabuf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/common_dmabuf_bridge.h
new file mode 100644
index 000000000000..cc631e3e1116
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/common_dmabuf_bridge.h
@@ -0,0 +1,145 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMABUF_BRIDGE_H
+#define COMMON_DMABUF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_DMABUF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DMABUF_CMD_LAST			(PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3)
+
+/*******************************************
+            PhysmemImportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF;
+
+/* Bridge out structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF;
+
+/*******************************************
+            PhysmemExportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF;
+
+/* Bridge out structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF;
+
+/*******************************************
+            PhysmemExportGemHandle
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE;
+
+/* Bridge out structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Handle;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE;
+
+/*******************************************
+            PhysmemImportSparseDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_SIZE_T uiChunkSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF;
+
+/* Bridge out structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF;
+
+#endif /* COMMON_DMABUF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/server_dmabuf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/server_dmabuf_bridge.c
new file mode 100644
index 000000000000..89aa1477f55a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/server_dmabuf_bridge.c
@@ -0,0 +1,574 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "physmem_dmabuf.h"
+#include "pmr.h"
+
+#include "common_dmabuf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemImportDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemImportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemImportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufOUT_UI8,
+								      0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPhysmemImportDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportDmaBufIN->puiName,
+		     psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportDmaBuf_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR))
+					 - 1] = '\0';
+	}
+
+	psPhysmemImportDmaBufOUT->eError =
+	    PhysmemImportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPhysmemImportDmaBufIN->ifd,
+				psPhysmemImportDmaBufIN->uiFlags,
+				psPhysmemImportDmaBufIN->ui32NameSize,
+				uiNameInt,
+				&psPMRPtrInt,
+				&psPhysmemImportDmaBufOUT->uiSize,
+				&psPhysmemImportDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportDmaBufOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psPhysmemImportDmaBufOUT->
+								     hPMRPtr, (void *)psPMRPtrInt,
+								     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _PhysmemImportDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportDmaBuf_exit:
+
+	if (psPhysmemImportDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemExportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemExportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psPhysmemExportDmaBufIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportDmaBufOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportDmaBuf_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportDmaBufOUT->eError =
+	    PhysmemExportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPMRInt, &psPhysmemExportDmaBufOUT->iFd);
+
+PhysmemExportDmaBuf_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportGemHandle(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemExportGemHandleIN_UI8,
+				   IMG_UINT8 * psPhysmemExportGemHandleOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPhysmemExportGemHandleIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportGemHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportGemHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportGemHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportGemHandleOUT->eError =
+	    PhysmemExportGemHandle(psConnection, OSGetDevNode(psConnection),
+				   psPMRInt, &psPhysmemExportGemHandleOUT->ui32Handle);
+
+PhysmemExportGemHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemImportSparseDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportSparseDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufIN_UI8,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufOUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (unlikely(psPhysmemImportSparseDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportSparseDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportSparseDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportSparseDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemImportSparseDmaBufIN->pui32MappingTable,
+		     psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+	}
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportSparseDmaBufIN->puiName,
+		     psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+		((IMG_CHAR *)
+		 uiNameInt)[(psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PhysmemImportSparseDmaBuf(psConnection, OSGetDevNode(psConnection),
+				      psPhysmemImportSparseDmaBufIN->ifd,
+				      psPhysmemImportSparseDmaBufIN->uiFlags,
+				      psPhysmemImportSparseDmaBufIN->uiChunkSize,
+				      psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks,
+				      psPhysmemImportSparseDmaBufIN->ui32NumVirtChunks,
+				      ui32MappingTableInt,
+				      psPhysmemImportSparseDmaBufIN->ui32NameSize,
+				      uiNameInt,
+				      &psPMRPtrInt,
+				      &psPhysmemImportSparseDmaBufOUT->uiSize,
+				      &psPhysmemImportSparseDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psPhysmemImportSparseDmaBufOUT->hPMRPtr, (void *)psPMRPtrInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PhysmemImportSparseDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportSparseDmaBuf_exit:
+
+	if (psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportSparseDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+/*
+ * Register all DMABUF functions with services
+ */
+PVRSRV_ERROR InitDMABUFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF,
+			      PVRSRVBridgePhysmemImportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF,
+			      PVRSRVBridgePhysmemExportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE,
+			      PVRSRVBridgePhysmemExportGemHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF,
+			      PVRSRVBridgePhysmemImportSparseDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dmabuf functions with services
+ */
+void DeinitDMABUFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF,
+				PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_bridge.h
new file mode 100644
index 000000000000..2c5d18843120
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_bridge.h
@@ -0,0 +1,64 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_HTBUFFER_BRIDGE_H
+#define CLIENT_HTBUFFER_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_htbuffer_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode);
+
+#endif /* CLIENT_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_direct_bridge.c
new file mode 100644
index 000000000000..3e6295cdfdf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_direct_bridge.c
@@ -0,0 +1,70 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for htbuffer
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_htbuffer_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#include "htbserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError =
+	    HTBControlKM(ui32NumGroups,
+			 pui32GroupEnable, ui32LogLevel, ui32EnablePID, ui32LogMode, ui32OpMode);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/common_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/common_htbuffer_bridge.h
new file mode 100644
index 000000000000..32be202186dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/common_htbuffer_bridge.h
@@ -0,0 +1,82 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_HTBUFFER_BRIDGE_H
+#define COMMON_HTBUFFER_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST			0
+#define PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL			PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_LAST			(PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0)
+
+/*******************************************
+            HTBControl
+ *******************************************/
+
+/* Bridge in structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_IN_HTBCONTROL_TAG
+{
+	IMG_UINT32 *pui32GroupEnable;
+	IMG_UINT32 ui32EnablePID;
+	IMG_UINT32 ui32LogLevel;
+	IMG_UINT32 ui32LogMode;
+	IMG_UINT32 ui32NumGroups;
+	IMG_UINT32 ui32OpMode;
+} __packed PVRSRV_BRIDGE_IN_HTBCONTROL;
+
+/* Bridge out structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_OUT_HTBCONTROL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HTBCONTROL;
+
+#endif /* COMMON_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/server_htbuffer_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/server_htbuffer_bridge.c
new file mode 100644
index 000000000000..9b730d28e5b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/server_htbuffer_bridge.c
@@ -0,0 +1,227 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "htbserver.h"
+
+#include "common_htbuffer_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(HTB_FLAG_NUM_EL <= IMG_UINT32_MAX,
+	      "HTB_FLAG_NUM_EL must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHTBControl(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psHTBControlIN_UI8,
+		       IMG_UINT8 * psHTBControlOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HTBCONTROL *psHTBControlIN =
+	    (PVRSRV_BRIDGE_IN_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HTBCONTROL *psHTBControlOUT =
+	    (PVRSRV_BRIDGE_OUT_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlOUT_UI8, 0);
+
+	IMG_UINT32 *ui32GroupEnableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psHTBControlIN->ui32NumGroups > HTB_FLAG_NUM_EL))
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HTBControl_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HTBControl_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHTBControlIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHTBControlIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHTBControlOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HTBControl_exit;
+			}
+		}
+	}
+
+	if (psHTBControlIN->ui32NumGroups != 0)
+	{
+		ui32GroupEnableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32GroupEnableInt,
+		     (const void __user *)psHTBControlIN->pui32GroupEnable,
+		     psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psHTBControlOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HTBControl_exit;
+		}
+	}
+
+	psHTBControlOUT->eError =
+	    HTBControlKM(psHTBControlIN->ui32NumGroups,
+			 ui32GroupEnableInt,
+			 psHTBControlIN->ui32LogLevel,
+			 psHTBControlIN->ui32EnablePID,
+			 psHTBControlIN->ui32LogMode, psHTBControlIN->ui32OpMode);
+
+HTBControl_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHTBControlOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pHTBUFFERBridgeLock;
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+
+/*
+ * Register all HTBUFFER functions with services
+ */
+PVRSRV_ERROR InitHTBUFFERBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pHTBUFFERBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL,
+			      PVRSRVBridgeHTBControl, pHTBUFFERBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_HTBCONTROL),
+			      sizeof(PVRSRV_BRIDGE_OUT_HTBCONTROL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all htbuffer functions with services
+ */
+void DeinitHTBUFFERBridge(void)
+{
+	OSLockDestroy(pHTBUFFERBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL);
+
+}
+#else /* EXCLUDE_HTBUFFER_BRIDGE */
+/* This bridge is conditional on EXCLUDE_HTBUFFER_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitHTBUFFERBridge() \
+	PVRSRV_OK
+
+#define DeinitHTBUFFERBridge()
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_bridge.h
new file mode 100644
index 000000000000..d5ba6dd74da3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_bridge.h
@@ -0,0 +1,234 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_MM_BRIDGE_H
+#define CLIENT_MM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_mm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge,
+							IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags,
+						IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats);
+
+#endif /* CLIENT_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_direct_bridge.c
new file mode 100644
index 000000000000..c1810e959ca8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_direct_bridge.c
@@ -0,0 +1,719 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for mm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_mm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRExportPMR(psPMRInt, &psPMRExportInt, pui64Size, pui32Log2Contig, pui64Password);
+
+	*phPMRExport = psPMRExportInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMR);
+	PVR_UNREFERENCED_PARAMETER(phPMRExport);
+	PVR_UNREFERENCED_PARAMETER(pui64Size);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2Contig);
+	PVR_UNREFERENCED_PARAMETER(pui64Password);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError = PMRUnexportPMR(psPMRExportInt);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRGetUID(psPMRInt, pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psBufferInt;
+	PMR *psExtMemInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psBufferInt = (PMR *) hBuffer;
+
+	eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+
+	*phExtMem = psExtMemInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtMemInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtMemInt = (PMR *) hExtMem;
+
+	eError = PMRUnmakeLocalImportHandle(psExtMemInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PMR *psPMRInt = NULL;
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError =
+	    PhysmemImportPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     psPMRExportInt,
+			     ui64uiPassword, ui64uiSize, ui32uiLog2Contig, &psPMRInt);
+
+	*phPMR = psPMRInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+	PVR_UNREFERENCED_PARAMETER(ui64uiPassword);
+	PVR_UNREFERENCED_PARAMETER(ui64uiSize);
+	PVR_UNREFERENCED_PARAMETER(ui32uiLog2Contig);
+	PVR_UNREFERENCED_PARAMETER(phPMR);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtHandleInt;
+	PMR *psPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtHandleInt = (PMR *) hExtHandle;
+
+	eError = PMRLocalImportPMR(psExtHandleInt, &psPMRInt, puiSize, puiAlign);
+
+	*phPMR = psPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRUnrefPMR(psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRPtrInt = NULL;
+
+	eError =
+	    PhysmemNewRamBackedPMR_direct(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  uiSize,
+					  ui32NumPhysChunks,
+					  ui32NumVirtChunks,
+					  pui32MappingTable,
+					  ui32Log2PageSize,
+					  uiFlags,
+					  ui32AnnotationLength,
+					  puiAnnotation,
+					  ui32PID, &psPMRPtrInt, ui32PDumpFlags, puiOutFlags);
+
+	*phPMRPtr = psPMRPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	eError =
+	    DevmemIntCtxCreate(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       bbKernelMemoryCtx,
+			       &psDevMemServerContextInt, &hPrivDataInt, pui32CPUCacheLineSize);
+
+	*phDevMemServerContext = psDevMemServerContextInt;
+	*phPrivData = hPrivDataInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError = DevmemIntCtxDestroy(psDevmemServerContextInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				ui32HeapConfigIndex, ui32HeapIndex, &psDevmemHeapPtrInt);
+
+	*phDevmemHeapPtr = psDevmemHeapPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeapInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemHeapInt = (DEVMEMINT_HEAP *) hDevmemHeap;
+
+	eError = DevmemIntHeapDestroy(psDevmemHeapInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnmapPMR(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemIntReserveRange(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psDevmemServerHeapInt,
+				  sAddress, uiLength, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	PMR *psPMRInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemIntReserveRangeAndMapPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   psDevmemServerHeapInt,
+					   sAddress,
+					   uiLength, psPMRInt, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemIntChangeSparse(ui32AllocPageCount,
+				  pui32AllocPageIndices,
+				  ui32FreePageCount,
+				  pui32FreePageIndices, ui32SparseFlags, psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntIsVDevAddrValid(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, sAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries)
+{
+#if defined(RGX_FEATURE_FBCDC)
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntInvalidateFBSCTable(psDevmemCtxInt, ui64FBSCEntries);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(ui64FBSCEntries);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   pui32NumHeapConfigs);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     ui32HeapConfigIndex, pui32NumHeaps);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigName(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  ui32HeapConfigIndex, ui32HeapConfigNameBufSz, puiHeapConfigName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapDetails(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       ui32HeapConfigIndex,
+			       ui32HeapIndex,
+			       ui32HeapNameBufSz,
+			       puiHeapNameOut,
+			       psDevVAddrBase,
+			       puiHeapLength,
+			       puiReservedRegionLength,
+			       pui32Log2DataPageSizeOut, pui32Log2ImportAlignmentOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, bRegister);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfoKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				       ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), peHeap);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntGetFaultAddress(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, psFaultAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVStatsUpdateOOMStat(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32ui32StatType, ui32pid);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32ui32StatType);
+	PVR_UNREFERENCED_PARAMETER(ui32pid);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt, sAddress, uiLength, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       ui32PageCount, ui32PhysPageOffset, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnmapPages(psReservationInt, ui32VirtPageOffset, ui32PageCount);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     ui32PageCount, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/common_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/common_mm_bridge.h
new file mode 100644
index 000000000000..3b3c8c2dc215
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/common_mm_bridge.h
@@ -0,0 +1,757 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MM_BRIDGE_H
+#define COMMON_MM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MM_PMREXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_MM_PMRGETUID			PVRSRV_BRIDGE_MM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_MM_PMRIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_MM_PMRUNREFPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+9
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+10
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+11
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+12
+#define PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+13
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+14
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+15
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+16
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+17
+#define PVRSRV_BRIDGE_MM_CHANGESPARSEMEM			PVRSRV_BRIDGE_MM_CMD_FIRST+18
+#define PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID			PVRSRV_BRIDGE_MM_CMD_FIRST+19
+#define PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE			PVRSRV_BRIDGE_MM_CMD_FIRST+20
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+21
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+22
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME			PVRSRV_BRIDGE_MM_CMD_FIRST+23
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS			PVRSRV_BRIDGE_MM_CMD_FIRST+24
+#define PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM			PVRSRV_BRIDGE_MM_CMD_FIRST+25
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO			PVRSRV_BRIDGE_MM_CMD_FIRST+26
+#define PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP			PVRSRV_BRIDGE_MM_CMD_FIRST+27
+#define PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS			PVRSRV_BRIDGE_MM_CMD_FIRST+28
+#define PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT			PVRSRV_BRIDGE_MM_CMD_FIRST+29
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+30
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+31
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+32
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+33
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE			PVRSRV_BRIDGE_MM_CMD_FIRST+34
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2			PVRSRV_BRIDGE_MM_CMD_FIRST+35
+#define PVRSRV_BRIDGE_MM_CMD_LAST			(PVRSRV_BRIDGE_MM_CMD_FIRST+35)
+
+/*******************************************
+            PMRExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMREXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMREXPORTPMR;
+
+/* Bridge out structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMREXPORTPMR_TAG
+{
+	IMG_UINT64 ui64Password;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRExport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2Contig;
+} __packed PVRSRV_BRIDGE_OUT_PMREXPORTPMR;
+
+/*******************************************
+            PMRUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMRExport;
+} __packed PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR;
+
+/* Bridge out structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR;
+
+/*******************************************
+            PMRGetUID
+ *******************************************/
+
+/* Bridge in structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_IN_PMRGETUID_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRGETUID;
+
+/* Bridge out structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_OUT_PMRGETUID_TAG
+{
+	IMG_UINT64 ui64UID;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRGETUID;
+
+/*******************************************
+            PMRMakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRUnmakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRIMPORTPMR_TAG
+{
+	IMG_UINT64 ui64uiPassword;
+	IMG_UINT64 ui64uiSize;
+	IMG_HANDLE hPMRExport;
+	IMG_UINT32 ui32uiLog2Contig;
+} __packed PVRSRV_BRIDGE_IN_PMRIMPORTPMR;
+
+/* Bridge out structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRIMPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRIMPORTPMR;
+
+/*******************************************
+            PMRLocalImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_HANDLE hExtHandle;
+} __packed PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR;
+
+/* Bridge out structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR;
+
+/*******************************************
+            PMRUnrefPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNREFPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRUNREFPMR;
+
+/* Bridge out structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNREFPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNREFPMR;
+
+/*******************************************
+            PhysmemNewRamBackedPMR
+ *******************************************/
+
+/* Bridge in structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiAnnotation;
+	IMG_UINT32 ui32AnnotationLength;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_PID ui32PID;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR;
+
+/* Bridge out structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR;
+
+/*******************************************
+            DevmemIntCtxCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_BOOL bbKernelMemoryCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE;
+
+/* Bridge out structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_HANDLE hDevMemServerContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CPUCacheLineSize;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE;
+
+/*******************************************
+            DevmemIntCtxDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY_TAG
+{
+	IMG_HANDLE hDevmemServerContext;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY;
+
+/* Bridge out structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY;
+
+/*******************************************
+            DevmemIntHeapCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE;
+
+/* Bridge out structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemHeapPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE;
+
+/*******************************************
+            DevmemIntHeapDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY_TAG
+{
+	IMG_HANDLE hDevmemHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY;
+
+/* Bridge out structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY;
+
+/*******************************************
+            DevmemIntMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR;
+
+/* Bridge out structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR;
+
+/*******************************************
+            DevmemIntUnmapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR;
+
+/* Bridge out structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR;
+
+/*******************************************
+            DevmemIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE;
+
+/* Bridge out structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE;
+
+/*******************************************
+            DevmemIntReserveRangeAndMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	IMG_HANDLE hPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/* Bridge out structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/*******************************************
+            DevmemIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE;
+
+/*******************************************
+            ChangeSparseMem
+ *******************************************/
+
+/* Bridge in structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_IN_CHANGESPARSEMEM_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32SparseFlags;
+} __packed PVRSRV_BRIDGE_IN_CHANGESPARSEMEM;
+
+/* Bridge out structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM;
+
+/*******************************************
+            DevmemIsVDevAddrValid
+ *******************************************/
+
+/* Bridge in structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID;
+
+/* Bridge out structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID;
+
+/*******************************************
+            DevmemInvalidateFBSCTable
+ *******************************************/
+
+/* Bridge in structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	IMG_UINT64 ui64FBSCEntries;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE;
+
+/* Bridge out structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE;
+
+/*******************************************
+            HeapCfgHeapConfigCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT;
+
+/* Bridge out structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeapConfigs;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT;
+
+/*******************************************
+            HeapCfgHeapCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT_TAG
+{
+	IMG_UINT32 ui32HeapConfigIndex;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT;
+
+/* Bridge out structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT;
+
+/*******************************************
+            HeapCfgHeapConfigName
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapConfigNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME;
+
+/* Bridge out structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME;
+
+/*******************************************
+            HeapCfgHeapDetails
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_CHAR *puiHeapNameOut;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+	IMG_UINT32 ui32HeapNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS;
+
+/* Bridge out structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_CHAR *puiHeapNameOut;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2DataPageSizeOut;
+	IMG_UINT32 ui32Log2ImportAlignmentOut;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS;
+
+/*******************************************
+            DevmemIntRegisterPFNotifyKM
+ *******************************************/
+
+/* Bridge in structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_BOOL bRegister;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/* Bridge out structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/*******************************************
+            PhysHeapGetMemInfo
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO;
+
+/* Bridge out structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO;
+
+/*******************************************
+            GetDefaultPhysicalHeap
+ *******************************************/
+
+/* Bridge in structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP;
+
+/* Bridge out structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP eHeap;
+} __packed PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP;
+
+/*******************************************
+            DevmemGetFaultAddress
+ *******************************************/
+
+/* Bridge in structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS;
+
+/* Bridge out structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_DEV_VIRTADDR sFaultAddress;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS;
+
+/*******************************************
+            PVRSRVStatsUpdateOOMStat
+ *******************************************/
+
+/* Bridge in structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	IMG_PID ui32pid;
+	IMG_UINT32 ui32ui32StatType;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT;
+
+/* Bridge out structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT;
+
+/*******************************************
+            DevmemXIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE;
+
+/*******************************************
+            DevmemXIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE;
+
+/*******************************************
+            DevmemXIntMapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32PhysPageOffset;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES;
+
+/* Bridge out structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES;
+
+/*******************************************
+            DevmemXIntUnmapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES;
+
+/* Bridge out structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES;
+
+/*******************************************
+            DevmemXIntMapVRangeToBackingPage
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/* Bridge out structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/*******************************************
+            PhysHeapGetMemInfo2
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2;
+
+/* Bridge out structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2;
+
+#endif /* COMMON_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/server_mm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/server_mm_bridge.c
new file mode 100644
index 000000000000..a79a92f5e9d1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/server_mm_bridge.c
@@ -0,0 +1,3237 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+#include "common_mm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleasePMRExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRExportPMRpsPMRExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnexportPMR((PMR_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRExportPMRIN_UI8,
+			 IMG_UINT8 * psPMRExportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMREXPORTPMR *psPMRExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMREXPORTPMR *psPMRExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PMRExportPMR(psPMRInt,
+			 &psPMRExportInt,
+			 &psPMRExportPMROUT->ui64Size,
+			 &psPMRExportPMROUT->ui32Log2Contig, &psPMRExportPMROUT->ui64Password);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRExportPMR_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &hPMRExportInt, (void *)psPMRExportInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _PMRExportPMRpsPMRExportIntRelease);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRExportPMROUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+							      &psPMRExportPMROUT->hPMRExport,
+							      (void *)psPMRExportInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      ReleasePMRExport);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRExportPMROUT->hPMRExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE) psPMRExportPMROUT->
+							     hPMRExport,
+							     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hPMRExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hPMRExportInt,
+							PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psPMRExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnexportPMR(psPMRExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRExportPMR NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgePMRUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psPMRUnexportPMRIN_UI8,
+			   IMG_UINT8 * psPMRUnexportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *psPMRUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *psPMRUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMROUT_UI8, 0);
+
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+				       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_FALSE);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hPMRExportInt,
+				     psPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT((psPMRUnexportPMROUT->eError == PVRSRV_OK) ||
+		   (psPMRUnexportPMROUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRUnexportPMR_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRUnexportPMR NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgePMRGetUID(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psPMRGetUIDIN_UI8,
+		      IMG_UINT8 * psPMRGetUIDOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRGETUID *psPMRGetUIDIN =
+	    (PVRSRV_BRIDGE_IN_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRGETUID *psPMRGetUIDOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRGetUIDIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRGetUIDOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRGetUIDOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRGetUID_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRGetUIDOUT->eError = PMRGetUID(psPMRInt, &psPMRGetUIDOUT->ui64UID);
+
+PMRGetUID_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRMakeLocalImportHandlepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnmakeLocalImportHandle((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRMakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleIN_UI8,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psPMRMakeLocalImportHandleIN->hBuffer;
+	PMR *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer,
+				       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE, IMG_TRUE);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psPMRMakeLocalImportHandleOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PMRMakeLocalImportHandlepsExtMemIntRelease);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRMakeLocalImportHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnmakeLocalImportHandle(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnmakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleIN_UI8,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRUnmakeLocalImportHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psPMRUnmakeLocalImportHandleIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_OK) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRUnmakeLocalImportHandleOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRUnmakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRUnmakeLocalImportHandle_exit:
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRImportPMRIN_UI8,
+			 IMG_UINT8 * psPMRImportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRIMPORTPMR *psPMRImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *psPMRImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMRExport = psPMRImportPMRIN->hPMRExport;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psPMRImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_TRUE);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRImportPMROUT->eError =
+	    PhysmemImportPMR(psConnection, OSGetDevNode(psConnection),
+			     psPMRExportInt,
+			     psPMRImportPMRIN->ui64uiPassword,
+			     psPMRImportPMRIN->ui64uiSize,
+			     psPMRImportPMRIN->ui32uiLog2Contig, &psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psPMRImportPMROUT->hPMR,
+							      (void *)psPMRInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _PMRImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRExportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psPMRImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRImportPMR NULL
+#endif
+
+static PVRSRV_ERROR _PMRLocalImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRLocalImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPMRLocalImportPMRIN_UI8,
+			      IMG_UINT8 * psPMRLocalImportPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *psPMRLocalImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *psPMRLocalImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hExtHandle = psPMRLocalImportPMRIN->hExtHandle;
+	PMR *psExtHandleInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRLocalImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psExtHandleInt,
+				       hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT, IMG_TRUE);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError =
+	    PMRLocalImportPMR(psExtHandleInt,
+			      &psPMRInt,
+			      &psPMRLocalImportPMROUT->uiSize, &psPMRLocalImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPMRLocalImportPMROUT->hPMR,
+								   (void *)psPMRInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PMRLocalImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRLocalImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psExtHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psPMRLocalImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnrefPMR(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRUnrefPMRIN_UI8,
+			IMG_UINT8 * psPMRUnrefPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNREFPMR *psPMRUnrefPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNREFPMR *psPMRUnrefPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRUnrefPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRUnrefPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (unlikely((psPMRUnrefPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnrefPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRUnrefPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRUnrefPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemNewRamBackedPMRpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemNewRamBackedPMR(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMRIN_UI8,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMROUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMRIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMROUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiAnnotationInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (unlikely(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemNewRamBackedPMRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemNewRamBackedPMRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemNewRamBackedPMR_exit;
+			}
+		}
+	}
+
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->pui32MappingTable,
+		     psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+	}
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength != 0)
+	{
+		uiAnnotationInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiAnnotationInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->puiAnnotation,
+		     psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+		((IMG_CHAR *)
+		 uiAnnotationInt)[(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength *
+				   sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psPhysmemNewRamBackedPMROUT->eError =
+	    PhysmemNewRamBackedPMR(psConnection, OSGetDevNode(psConnection),
+				   psPhysmemNewRamBackedPMRIN->uiSize,
+				   psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks,
+				   psPhysmemNewRamBackedPMRIN->ui32NumVirtChunks,
+				   ui32MappingTableInt,
+				   psPhysmemNewRamBackedPMRIN->ui32Log2PageSize,
+				   psPhysmemNewRamBackedPMRIN->uiFlags,
+				   psPhysmemNewRamBackedPMRIN->ui32AnnotationLength,
+				   uiAnnotationInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PID,
+				   &psPMRPtrInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PDumpFlags,
+				   &psPhysmemNewRamBackedPMROUT->uiOutFlags);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemNewRamBackedPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psPhysmemNewRamBackedPMROUT->
+									hPMRPtr,
+									(void *)psPMRPtrInt,
+									PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_PhysmemNewRamBackedPMRpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemNewRamBackedPMR_exit:
+
+	if (psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemNewRamBackedPMROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntCtxCreatepsDevMemServerContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxCreate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntCtxCreateIN_UI8,
+			       IMG_UINT8 * psDevmemIntCtxCreateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateOUT_UI8,
+								     0);
+
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntCtxCreateOUT->hDevMemServerContext = NULL;
+
+	psDevmemIntCtxCreateOUT->eError =
+	    DevmemIntCtxCreate(psConnection, OSGetDevNode(psConnection),
+			       psDevmemIntCtxCreateIN->bbKernelMemoryCtx,
+			       &psDevMemServerContextInt,
+			       &hPrivDataInt, &psDevmemIntCtxCreateOUT->ui32CPUCacheLineSize);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntCtxCreateOUT->
+								    hDevMemServerContext,
+								    (void *)
+								    psDevMemServerContextInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntCtxCreatepsDevMemServerContextIntRelease);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntCtxCreateOUT->
+								       hPrivData,
+								       (void *)hPrivDataInt,
+								       PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       psDevmemIntCtxCreateOUT->
+								       hDevMemServerContext);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxCreate_exit:
+
+	if (psDevmemIntCtxCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntCtxCreateOUT->hDevMemServerContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDevmemIntCtxCreateOUT->
+							     hDevMemServerContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psDevMemServerContextInt)
+		{
+			DevmemIntCtxDestroy(psDevMemServerContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntCtxDestroyIN_UI8,
+				IMG_UINT8 * psDevmemIntCtxDestroyOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyOUT_UI8,
+								      0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntCtxDestroyIN->
+					      hDevmemServerContext,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	if (unlikely
+	    ((psDevmemIntCtxDestroyOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntCtxDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntHeapDestroy((DEVMEMINT_HEAP *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapCreate(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntHeapCreateIN_UI8,
+				IMG_UINT8 * psDevmemIntHeapCreateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntHeapCreateIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntHeapCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				psDevmemIntHeapCreateIN->ui32HeapConfigIndex,
+				psDevmemIntHeapCreateIN->ui32HeapIndex, &psDevmemHeapPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psDevmemIntHeapCreateOUT->
+								     hDevmemHeapPtr,
+								     (void *)psDevmemHeapPtrInt,
+								     PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntHeapCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemHeapPtrInt)
+		{
+			DevmemIntHeapDestroy(psDevmemHeapPtrInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntHeapDestroyIN_UI8,
+				 IMG_UINT8 * psDevmemIntHeapDestroyOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *) IMG_OFFSET_ADDR(psDevmemIntHeapDestroyIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *)
+	    IMG_OFFSET_ADDR(psDevmemIntHeapDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntHeapDestroyIN->hDevmemHeap,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	if (unlikely((psDevmemIntHeapDestroyOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntHeapDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapDestroy_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDevmemIntMapPMRIN_UI8,
+			    IMG_UINT8 * psDevmemIntMapPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *psDevmemIntMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *psDevmemIntMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntMapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntMapPMROUT->eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+DevmemIntMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnmapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDevmemIntUnmapPMRIN_UI8,
+			      IMG_UINT8 * psDevmemIntUnmapPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntUnmapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntUnmapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntUnmapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnmapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnmapPMROUT->eError = DevmemIntUnmapPMR(psReservationInt);
+
+DevmemIntUnmapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIntReserveRangeIN_UI8,
+				  IMG_UINT8 * psDevmemIntReserveRangeOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError =
+	    DevmemIntReserveRange(psConnection, OSGetDevNode(psConnection),
+				  psDevmemServerHeapInt,
+				  psDevmemIntReserveRangeIN->sAddress,
+				  psDevmemIntReserveRangeIN->uiLength,
+				  psDevmemIntReserveRangeIN->uiFlags, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntReserveRangeOUT->
+								       hReservation,
+								       (void *)psReservationInt,
+								       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DevmemIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRangeAndMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMRIN_UI8,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMROUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeAndMapPMRIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntReserveRangeAndMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    DevmemIntReserveRangeAndMapPMR(psConnection, OSGetDevNode(psConnection),
+					   psDevmemServerHeapInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->sAddress,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiLength,
+					   psPMRInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiFlags,
+					   &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntReserveRangeAndMapPMROUT->hReservation,
+				      (void *)psReservationInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRangeAndMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeIN_UI8,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeChangeSparseMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psChangeSparseMemIN_UI8,
+			    IMG_UINT8 * psChangeSparseMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *psChangeSparseMemIN =
+	    (PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *psChangeSparseMemOUT =
+	    (PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+	IMG_HANDLE hReservation = psChangeSparseMemIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psChangeSparseMemIN->ui32AllocPageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (unlikely(psChangeSparseMemIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto ChangeSparseMem_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psChangeSparseMemIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psChangeSparseMemIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psChangeSparseMemOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto ChangeSparseMem_exit;
+			}
+		}
+	}
+
+	if (psChangeSparseMemIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32AllocPageIndices,
+		     psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+	if (psChangeSparseMemIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32FreePageIndices,
+		     psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psChangeSparseMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psChangeSparseMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto ChangeSparseMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psChangeSparseMemOUT->eError =
+	    DevmemIntChangeSparse(psChangeSparseMemIN->ui32AllocPageCount,
+				  ui32AllocPageIndicesInt,
+				  psChangeSparseMemIN->ui32FreePageCount,
+				  ui32FreePageIndicesInt,
+				  psChangeSparseMemIN->ui32SparseFlags, psReservationInt);
+
+ChangeSparseMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psChangeSparseMemOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIsVDevAddrValid(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidIN_UI8,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIsVDevAddrValidIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIsVDevAddrValidOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIsVDevAddrValidOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIsVDevAddrValid_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIsVDevAddrValidOUT->eError =
+	    DevmemIntIsVDevAddrValid(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, psDevmemIsVDevAddrValidIN->sAddress);
+
+DevmemIsVDevAddrValid_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+
+static IMG_INT
+PVRSRVBridgeDevmemInvalidateFBSCTable(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableIN_UI8,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemInvalidateFBSCTableIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemInvalidateFBSCTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemInvalidateFBSCTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    DevmemIntInvalidateFBSCTable(psDevmemCtxInt,
+					 psDevmemInvalidateFBSCTableIN->ui64FBSCEntries);
+
+DevmemInvalidateFBSCTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeDevmemInvalidateFBSCTable NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigCount(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountIN_UI8,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHeapCfgHeapConfigCountIN);
+
+	psHeapCfgHeapConfigCountOUT->eError =
+	    HeapCfgHeapConfigCount(psConnection, OSGetDevNode(psConnection),
+				   &psHeapCfgHeapConfigCountOUT->ui32NumHeapConfigs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapCount(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psHeapCfgHeapCountIN_UI8,
+			     IMG_UINT8 * psHeapCfgHeapCountOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountOUT_UI8, 0);
+
+	psHeapCfgHeapCountOUT->eError =
+	    HeapCfgHeapCount(psConnection, OSGetDevNode(psConnection),
+			     psHeapCfgHeapCountIN->ui32HeapConfigIndex,
+			     &psHeapCfgHeapCountOUT->ui32NumHeaps);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigName(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameIN_UI8,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameOUT_UI8, 0);
+
+	IMG_CHAR *puiHeapConfigNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	psHeapCfgHeapConfigNameOUT->puiHeapConfigName =
+	    psHeapCfgHeapConfigNameIN->puiHeapConfigName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapConfigNameIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapConfigNameIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapConfigName_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz != 0)
+	{
+		puiHeapConfigNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapConfigNameOUT->eError =
+	    HeapCfgHeapConfigName(psConnection, OSGetDevNode(psConnection),
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigIndex,
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz,
+				  puiHeapConfigNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapConfigNameOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapConfigNameInt) &&
+	    ((psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapConfigNameOUT->puiHeapConfigName,
+		      puiHeapConfigNameInt,
+		      (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapConfigName_exit;
+		}
+	}
+
+HeapCfgHeapConfigName_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapConfigNameOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapDetails(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psHeapCfgHeapDetailsIN_UI8,
+			       IMG_UINT8 * psHeapCfgHeapDetailsOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsOUT_UI8,
+								     0);
+
+	IMG_CHAR *puiHeapNameOutInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) + 0;
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	psHeapCfgHeapDetailsOUT->puiHeapNameOut = psHeapCfgHeapDetailsIN->puiHeapNameOut;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapDetailsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapDetailsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapDetails_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz != 0)
+	{
+		puiHeapNameOutInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapDetailsOUT->eError =
+	    HeapCfgHeapDetails(psConnection, OSGetDevNode(psConnection),
+			       psHeapCfgHeapDetailsIN->ui32HeapConfigIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapNameBufSz,
+			       puiHeapNameOutInt,
+			       &psHeapCfgHeapDetailsOUT->sDevVAddrBase,
+			       &psHeapCfgHeapDetailsOUT->uiHeapLength,
+			       &psHeapCfgHeapDetailsOUT->uiReservedRegionLength,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2DataPageSizeOut,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2ImportAlignmentOut);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapDetailsOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapNameOutInt) &&
+	    ((psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapDetailsOUT->puiHeapNameOut,
+		      puiHeapNameOutInt,
+		      (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapDetails_exit;
+		}
+	}
+
+HeapCfgHeapDetails_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapDetailsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntRegisterPFNotifyKM(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMIN_UI8,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntRegisterPFNotifyKMIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntRegisterPFNotifyKMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntRegisterPFNotifyKM_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, psDevmemIntRegisterPFNotifyKMIN->bRegister);
+
+DevmemIntRegisterPFNotifyKM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPhysHeapGetMemInfoIN_UI8,
+			       IMG_UINT8 * psPhysHeapGetMemInfoOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoIN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoOUT_UI8,
+								     0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V1)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfoIN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfoIN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfoIN->peaPhysHeapID,
+		     psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V1 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1);
+	}
+
+	psPhysHeapGetMemInfoOUT->eError =
+	    PVRSRVPhysHeapGetMemInfoKM(psConnection, OSGetDevNode(psConnection),
+				       psPhysHeapGetMemInfoIN->ui32PhysHeapCount,
+				       eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfoOUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V1))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+
+PhysHeapGetMemInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDefaultPhysicalHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapIN_UI8,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapIN =
+	    (PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDefaultPhysicalHeapIN);
+
+	psGetDefaultPhysicalHeapOUT->eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(psConnection, OSGetDevNode(psConnection),
+					   &psGetDefaultPhysicalHeapOUT->eHeap);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemGetFaultAddress(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemGetFaultAddressIN_UI8,
+				  IMG_UINT8 * psDevmemGetFaultAddressOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemGetFaultAddressIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemGetFaultAddressOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemGetFaultAddressOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemGetFaultAddress_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemGetFaultAddressOUT->eError =
+	    DevmemIntGetFaultAddress(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, &psDevmemGetFaultAddressOUT->sFaultAddress);
+
+DevmemGetFaultAddress_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+
+static IMG_INT
+PVRSRVBridgePVRSRVStatsUpdateOOMStat(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatIN_UI8,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatOUT_UI8, 0);
+
+	psPVRSRVStatsUpdateOOMStatOUT->eError =
+	    PVRSRVStatsUpdateOOMStat(psConnection, OSGetDevNode(psConnection),
+				     psPVRSRVStatsUpdateOOMStatIN->ui32ui32StatType,
+				     psPVRSRVStatsUpdateOOMStatIN->ui32pid);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePVRSRVStatsUpdateOOMStat NULL
+#endif
+
+static PVRSRV_ERROR _DevmemXIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemXIntUnreserveRange((DEVMEMXINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDevmemXIntReserveRangeIN_UI8,
+				   IMG_UINT8 * psDevmemXIntReserveRangeOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemXIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt,
+				   psDevmemXIntReserveRangeIN->sAddress,
+				   psDevmemXIntReserveRangeIN->uiLength, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDevmemXIntReserveRangeOUT->
+									hReservation,
+									(void *)psReservationInt,
+									PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DevmemXIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemXIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeIN_UI8,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemXIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemXIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapPages(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemXIntMapPagesIN_UI8,
+			       IMG_UINT8 * psDevmemXIntMapPagesOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesOUT_UI8,
+								     0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemXIntMapPagesIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapPagesOUT->eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       psDevmemXIntMapPagesIN->ui32PageCount,
+			       psDevmemXIntMapPagesIN->ui32PhysPageOffset,
+			       psDevmemXIntMapPagesIN->uiFlags,
+			       psDevmemXIntMapPagesIN->ui32VirtPageOffset);
+
+DevmemXIntMapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnmapPages(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesIN_UI8,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntUnmapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntUnmapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntUnmapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnmapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnmapPagesOUT->eError =
+	    DevmemXIntUnmapPages(psReservationInt,
+				 psDevmemXIntUnmapPagesIN->ui32VirtPageOffset,
+				 psDevmemXIntUnmapPagesIN->ui32PageCount);
+
+DevmemXIntUnmapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapVRangeToBackingPage(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageIN_UI8,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapVRangeToBackingPageIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapVRangeToBackingPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapVRangeToBackingPage_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     psDevmemXIntMapVRangeToBackingPageIN->ui32PageCount,
+					     psDevmemXIntMapVRangeToBackingPageIN->uiFlags,
+					     psDevmemXIntMapVRangeToBackingPageIN->
+					     ui32VirtPageOffset);
+
+DevmemXIntMapVRangeToBackingPage_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo2(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysHeapGetMemInfo2IN_UI8,
+				IMG_UINT8 * psPhysHeapGetMemInfo2OUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2IN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2IN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2OUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2OUT_UI8,
+								      0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V2)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfo2IN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfo2IN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfo2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfo2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo2_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfo2IN->peaPhysHeapID,
+		     psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V2 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2);
+	}
+
+	psPhysHeapGetMemInfo2OUT->eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(psConnection, OSGetDevNode(psConnection),
+					psPhysHeapGetMemInfo2IN->ui32PhysHeapCount,
+					eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfo2OUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V2))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+
+PhysHeapGetMemInfo2_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfo2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+
+/*
+ * Register all MM functions with services
+ */
+PVRSRV_ERROR InitMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR,
+			      PVRSRVBridgePMRExportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMREXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMREXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR,
+			      PVRSRVBridgePMRUnexportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID, PVRSRVBridgePMRGetUID,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRGETUID),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRGETUID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRMakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRUnmakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR,
+			      PVRSRVBridgePMRImportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR,
+			      PVRSRVBridgePMRLocalImportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR,
+			      PVRSRVBridgePMRUnrefPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRUNREFPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNREFPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR,
+			      PVRSRVBridgePhysmemNewRamBackedPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE,
+			      PVRSRVBridgeDevmemIntCtxCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY,
+			      PVRSRVBridgeDevmemIntCtxDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE,
+			      PVRSRVBridgeDevmemIntHeapCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY,
+			      PVRSRVBridgeDevmemIntHeapDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR,
+			      PVRSRVBridgeDevmemIntMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR,
+			      PVRSRVBridgeDevmemIntUnmapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE,
+			      PVRSRVBridgeDevmemIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR,
+			      PVRSRVBridgeDevmemIntReserveRangeAndMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM,
+			      PVRSRVBridgeChangeSparseMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_CHANGESPARSEMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID,
+			      PVRSRVBridgeDevmemIsVDevAddrValid, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE,
+			      PVRSRVBridgeDevmemInvalidateFBSCTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT,
+			      PVRSRVBridgeHeapCfgHeapConfigCount, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT,
+			      PVRSRVBridgeHeapCfgHeapCount, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME,
+			      PVRSRVBridgeHeapCfgHeapConfigName, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS,
+			      PVRSRVBridgeHeapCfgHeapDetails, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM,
+			      PVRSRVBridgeDevmemIntRegisterPFNotifyKM, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO,
+			      PVRSRVBridgePhysHeapGetMemInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP,
+			      PVRSRVBridgeGetDefaultPhysicalHeap, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS,
+			      PVRSRVBridgeDevmemGetFaultAddress, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT,
+			      PVRSRVBridgePVRSRVStatsUpdateOOMStat, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES,
+			      PVRSRVBridgeDevmemXIntMapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES,
+			      PVRSRVBridgeDevmemXIntUnmapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE,
+			      PVRSRVBridgeDevmemXIntMapVRangeToBackingPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2,
+			      PVRSRVBridgePhysHeapGetMemInfo2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mm functions with services
+ */
+void DeinitMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM,
+				PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/common_mmextmem_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/common_mmextmem_bridge.h
new file mode 100644
index 000000000000..f52637853325
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/common_mmextmem_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MMEXTMEM_BRIDGE_H
+#define COMMON_MMEXTMEM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM			PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST			(PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0)
+
+/*******************************************
+            PhysmemWrapExtMem
+ *******************************************/
+
+/* Bridge in structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_UINT64 ui64CpuVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM;
+
+/* Bridge out structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM;
+
+#endif /* COMMON_MMEXTMEM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/server_mmextmem_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/server_mmextmem_bridge.c
new file mode 100644
index 000000000000..538d04cdf0e0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/server_mmextmem_bridge.c
@@ -0,0 +1,166 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "physmem_extmem.h"
+
+#include "common_mmextmem_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemWrapExtMempsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemWrapExtMem(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPhysmemWrapExtMemIN_UI8,
+			      IMG_UINT8 * psPhysmemWrapExtMemOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemOUT_UI8, 0);
+
+	PMR *psPMRPtrInt = NULL;
+
+	psPhysmemWrapExtMemOUT->eError =
+	    PhysmemWrapExtMem(psConnection, OSGetDevNode(psConnection),
+			      psPhysmemWrapExtMemIN->uiSize,
+			      psPhysmemWrapExtMemIN->ui64CpuVAddr,
+			      psPhysmemWrapExtMemIN->uiFlags, &psPMRPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemWrapExtMemOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPhysmemWrapExtMemOUT->hPMRPtr,
+								   (void *)psPMRPtrInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PhysmemWrapExtMempsPMRPtrIntRelease);
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemWrapExtMem_exit:
+
+	if (psPhysmemWrapExtMemOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+
+/*
+ * Register all MMEXTMEM functions with services
+ */
+PVRSRV_ERROR InitMMEXTMEMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM,
+			      PVRSRVBridgePhysmemWrapExtMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mmextmem functions with services
+ */
+void DeinitMMEXTMEMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_bridge.h
new file mode 100644
index 000000000000..2bf967bfcea9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMP_BRIDGE_H
+#define CLIENT_PDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_direct_bridge.c
new file mode 100644
index 000000000000..7c012dbcf21e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_direct_bridge.c
@@ -0,0 +1,151 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpImageDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  psDevmemCtxInt,
+					  ui32StringSize,
+					  puiFileName,
+					  sDataDevAddr,
+					  ui32DataSize,
+					  ui32LogicalWidth,
+					  ui32LogicalHeight,
+					  ui32PhysicalWidth,
+					  ui32PhysicalHeight,
+					  ePixelFormat,
+					  eMemLayout,
+					  eFBCompression,
+					  pui32FBCClearColour,
+					  eeFBCSwizzle,
+					  sHeaderDevAddr, ui32HeaderSize, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			   ui32CommentSize, puiComment, ui32Flags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpSetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpDataDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 psDevmemCtxInt,
+					 ui32StringSize,
+					 puiFileName,
+					 sDataDevAddr,
+					 ui32DataSize,
+					 ui32HeaderType,
+					 ui32ElementType, ui32ElementCount, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/common_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/common_pdump_bridge.h
new file mode 100644
index 000000000000..eb6ab9488a20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/common_pdump_bridge.h
@@ -0,0 +1,155 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMP_BRIDGE_H
+#define COMMON_PDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_BRIDGE_PDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMP_CMD_LAST			(PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3)
+
+/*******************************************
+            PDumpImageDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_DEV_VIRTADDR sHeaderDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_UINT32 *pui32FBCClearColour;
+	const IMG_CHAR *puiFileName;
+	IMG_FB_COMPRESSION eFBCompression;
+	IMG_MEMLAYOUT eMemLayout;
+	PDUMP_PIXEL_FORMAT ePixelFormat;
+	PDUMP_FBC_SWIZZLE eeFBCSwizzle;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32HeaderSize;
+	IMG_UINT32 ui32LogicalHeight;
+	IMG_UINT32 ui32LogicalWidth;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PhysicalHeight;
+	IMG_UINT32 ui32PhysicalWidth;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR;
+
+/* Bridge out structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR;
+
+/*******************************************
+            PVRSRVPDumpComment
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT_TAG
+{
+	IMG_CHAR *puiComment;
+	IMG_UINT32 ui32CommentSize;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT;
+
+/* Bridge out structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT;
+
+/*******************************************
+            PVRSRVPDumpSetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME_TAG
+{
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME;
+
+/*******************************************
+            PDumpDataDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32ElementCount;
+	IMG_UINT32 ui32ElementType;
+	IMG_UINT32 ui32HeaderType;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR;
+
+/* Bridge out structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR;
+
+#endif /* COMMON_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/server_pdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/server_pdump_bridge.c
new file mode 100644
index 000000000000..6b3baf668710
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/server_pdump_bridge.c
@@ -0,0 +1,571 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+#include "common_pdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(4 <= IMG_UINT32_MAX, "4 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpImageDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpImageDescriptorIN_UI8,
+				 IMG_UINT8 * psPDumpImageDescriptorOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpImageDescriptorIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *)
+	    IMG_OFFSET_ADDR(psPDumpImageDescriptorOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpImageDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+	IMG_UINT32 *ui32FBCClearColourInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) 4 * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psPDumpImageDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpImageDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpImageDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpImageDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpImageDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psPDumpImageDescriptorIN->puiFileName,
+		     psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	{
+		ui32FBCClearColourInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 4 * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (4 * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FBCClearColourInt,
+		     (const void __user *)psPDumpImageDescriptorIN->pui32FBCClearColour,
+		     4 * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpImageDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpImageDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpImageDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpImageDescriptorOUT->eError =
+	    DevmemIntPDumpImageDescriptor(psConnection, OSGetDevNode(psConnection),
+					  psDevmemCtxInt,
+					  psPDumpImageDescriptorIN->ui32StringSize,
+					  uiFileNameInt,
+					  psPDumpImageDescriptorIN->sDataDevAddr,
+					  psPDumpImageDescriptorIN->ui32DataSize,
+					  psPDumpImageDescriptorIN->ui32LogicalWidth,
+					  psPDumpImageDescriptorIN->ui32LogicalHeight,
+					  psPDumpImageDescriptorIN->ui32PhysicalWidth,
+					  psPDumpImageDescriptorIN->ui32PhysicalHeight,
+					  psPDumpImageDescriptorIN->ePixelFormat,
+					  psPDumpImageDescriptorIN->eMemLayout,
+					  psPDumpImageDescriptorIN->eFBCompression,
+					  ui32FBCClearColourInt,
+					  psPDumpImageDescriptorIN->eeFBCSwizzle,
+					  psPDumpImageDescriptorIN->sHeaderDevAddr,
+					  psPDumpImageDescriptorIN->ui32HeaderSize,
+					  psPDumpImageDescriptorIN->ui32PDumpFlags);
+
+PDumpImageDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpImageDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_COMMENT_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_COMMENT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpComment(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPVRSRVPDumpCommentIN_UI8,
+			       IMG_UINT8 * psPVRSRVPDumpCommentOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentOUT_UI8,
+								     0);
+
+	IMG_CHAR *uiCommentInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPVRSRVPDumpCommentIN->ui32CommentSize > PVRSRV_PDUMP_MAX_COMMENT_SIZE))
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPVRSRVPDumpCommentIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPVRSRVPDumpCommentIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PVRSRVPDumpComment_exit;
+			}
+		}
+	}
+
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize != 0)
+	{
+		uiCommentInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiCommentInt, (const void __user *)psPVRSRVPDumpCommentIN->puiComment,
+		     psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PVRSRVPDumpComment_exit;
+		}
+		((IMG_CHAR *)
+		 uiCommentInt)[(psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPVRSRVPDumpCommentOUT->eError =
+	    PDumpCommentKM(psConnection, OSGetDevNode(psConnection),
+			   psPVRSRVPDumpCommentIN->ui32CommentSize,
+			   uiCommentInt, psPVRSRVPDumpCommentIN->ui32Flags);
+
+PVRSRVPDumpComment_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPVRSRVPDumpCommentOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameOUT_UI8,
+								      0);
+
+	psPVRSRVPDumpSetFrameOUT->eError =
+	    PDumpSetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    psPVRSRVPDumpSetFrameIN->ui32Frame);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpDataDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPDumpDataDescriptorIN_UI8,
+				IMG_UINT8 * psPDumpDataDescriptorOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpDataDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPDumpDataDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpDataDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpDataDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpDataDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpDataDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPDumpDataDescriptorIN->puiFileName,
+		     psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpDataDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpDataDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpDataDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpDataDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpDataDescriptorOUT->eError =
+	    DevmemIntPDumpDataDescriptor(psConnection, OSGetDevNode(psConnection),
+					 psDevmemCtxInt,
+					 psPDumpDataDescriptorIN->ui32StringSize,
+					 uiFileNameInt,
+					 psPDumpDataDescriptorIN->sDataDevAddr,
+					 psPDumpDataDescriptorIN->ui32DataSize,
+					 psPDumpDataDescriptorIN->ui32HeaderType,
+					 psPDumpDataDescriptorIN->ui32ElementType,
+					 psPDumpDataDescriptorIN->ui32ElementCount,
+					 psPDumpDataDescriptorIN->ui32PDumpFlags);
+
+PDumpDataDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpDataDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+
+/*
+ * Register all PDUMP functions with services
+ */
+PVRSRV_ERROR InitPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR,
+			      PVRSRVBridgePDumpImageDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT,
+			      PVRSRVBridgePVRSRVPDumpComment, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME,
+			      PVRSRVBridgePVRSRVPDumpSetFrame, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR,
+			      PVRSRVBridgePDumpDataDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdump functions with services
+ */
+void DeinitPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..a2ba27b88b30
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_bridge.h
@@ -0,0 +1,74 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPCTRL_BRIDGE_H
+#define CLIENT_PDUMPCTRL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpctrl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge);
+
+#endif /* CLIENT_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
new file mode 100644
index 000000000000..a2bc30360aea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
@@ -0,0 +1,109 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpctrl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpctrl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpGetStateKM(pui64State);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpGetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), pui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpSetDefaultCaptureParamsKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32Mode,
+					   ui32Start,
+					   ui32End,
+					   ui32Interval, ui32MaxParamFileSize, ui32AutoTermTimeout);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpIsLastCaptureFrameKM(pbpbIsLastCaptureFrame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpForceCaptureStopKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge));
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/common_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/common_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..f710e4c06131
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/common_pdumpctrl_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPCTRL_BRIDGE_H
+#define COMMON_PDUMPCTRL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST			(PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4)
+
+/*******************************************
+            PVRSRVPDumpGetState
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE;
+
+/* Bridge out structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT64 ui64State;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE;
+
+/*******************************************
+            PVRSRVPDumpGetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME;
+
+/*******************************************
+            PVRSRVPDumpSetDefaultCaptureParams
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	IMG_UINT32 ui32AutoTermTimeout;
+	IMG_UINT32 ui32End;
+	IMG_UINT32 ui32Interval;
+	IMG_UINT32 ui32MaxParamFileSize;
+	IMG_UINT32 ui32Mode;
+	IMG_UINT32 ui32Start;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/* Bridge out structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/*******************************************
+            PVRSRVPDumpIsLastCaptureFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/* Bridge out structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bpbIsLastCaptureFrame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/*******************************************
+            PVRSRVPDumpForceCaptureStop
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP;
+
+/* Bridge out structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP;
+
+#endif /* COMMON_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/server_pdumpctrl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/server_pdumpctrl_bridge.c
new file mode 100644
index 000000000000..2d5578cf83f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/server_pdumpctrl_bridge.c
@@ -0,0 +1,252 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pdump_km.h"
+
+#include "common_pdumpctrl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetState(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetStateIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetStateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetStateIN);
+
+	psPVRSRVPDumpGetStateOUT->eError = PDumpGetStateKM(&psPVRSRVPDumpGetStateOUT->ui64State);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetFrameIN);
+
+	psPVRSRVPDumpGetFrameOUT->eError =
+	    PDumpGetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    &psPVRSRVPDumpGetFrameOUT->ui32Frame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams(IMG_UINT32 ui32DispatchTableEntry,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8,
+					       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *psPVRSRVPDumpSetDefaultCaptureParamsIN
+	    =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS
+	    *psPVRSRVPDumpSetDefaultCaptureParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8, 0);
+
+	psPVRSRVPDumpSetDefaultCaptureParamsOUT->eError =
+	    PDumpSetDefaultCaptureParamsKM(psConnection, OSGetDevNode(psConnection),
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Mode,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Start,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32End,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Interval,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32MaxParamFileSize,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32AutoTermTimeout);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameIN_UI8,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpIsLastCaptureFrameIN);
+
+	psPVRSRVPDumpIsLastCaptureFrameOUT->eError =
+	    PDumpIsLastCaptureFrameKM(&psPVRSRVPDumpIsLastCaptureFrameOUT->bpbIsLastCaptureFrame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpForceCaptureStop(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopIN_UI8,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpForceCaptureStopIN);
+
+	psPVRSRVPDumpForceCaptureStopOUT->eError =
+	    PDumpForceCaptureStopKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pPDUMPCTRLBridgeLock;
+
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+
+/*
+ * Register all PDUMPCTRL functions with services
+ */
+PVRSRV_ERROR InitPDUMPCTRLBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pPDUMPCTRLBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE,
+			      PVRSRVBridgePVRSRVPDumpGetState, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME,
+			      PVRSRVBridgePVRSRVPDumpGetFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS,
+			      PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams, pPDUMPCTRLBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME,
+			      PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP,
+			      PVRSRVBridgePVRSRVPDumpForceCaptureStop, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpctrl functions with services
+ */
+void DeinitPDUMPCTRLBridge(void)
+{
+	OSLockDestroy(pPDUMPCTRLBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_bridge.h
new file mode 100644
index 000000000000..a4368477e225
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_bridge.h
@@ -0,0 +1,125 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPMM_BRIDGE_H
+#define CLIENT_PDUMPMM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpmm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
new file mode 100644
index 000000000000..1358f75a7537
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpmm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpmm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMem(psPMRInt, uiOffset, uiSize, ui32PDumpFlags, bbZero);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue32(psPMRInt, uiOffset, ui32Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue64(psPMRInt, uiOffset, ui64Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       uiOffset, uiSize, ui32ArraySize, puiFileName, ui32uiFileOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  uiOffset,
+				  ui32MemspaceNameLen,
+				  puiMemspaceName,
+				  ui32SymbolicAddrLen,
+				  puiSymbolicAddr, puiNewOffset, puiNextSymName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpPol32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpCheck32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpCBP(psPMRInt, uiReadOffset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError =
+	    DevmemIntPDumpSaveToFileVirtual(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    psDevmemServerContextInt,
+					    sAddress,
+					    uiSize,
+					    ui32ArraySize,
+					    puiFileName, ui32FileOffset, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/common_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/common_pdumpmm_bridge.h
new file mode 100644
index 000000000000..2020cbf5b30b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/common_pdumpmm_bridge.h
@@ -0,0 +1,259 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPMM_BRIDGE_H
+#define COMMON_PDUMPMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_LAST			(PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8)
+
+/*******************************************
+            PMRPDumpLoadMem
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_BOOL bbZero;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM;
+
+/* Bridge out structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM;
+
+/*******************************************
+            PMRPDumpLoadMemValue32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32;
+
+/* Bridge out structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32;
+
+/*******************************************
+            PMRPDumpLoadMemValue64
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	IMG_UINT64 ui64Value;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64;
+
+/* Bridge out structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64;
+
+/*******************************************
+            PMRPDumpSaveToFile
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32uiFileOffset;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE;
+
+/* Bridge out structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE;
+
+/*******************************************
+            PMRPDumpSymbolicAddr
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	IMG_UINT32 ui32MemspaceNameLen;
+	IMG_UINT32 ui32SymbolicAddrLen;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR;
+
+/* Bridge out structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiNewOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR;
+
+/*******************************************
+            PMRPDumpPol32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPPOL32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPPOL32;
+
+/* Bridge out structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32;
+
+/*******************************************
+            PMRPDumpCheck32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32;
+
+/* Bridge out structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32;
+
+/*******************************************
+            PMRPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiReadOffset;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCBP;
+
+/* Bridge out structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCBP;
+
+/*******************************************
+            DevmemIntPDumpSaveToFileVirtual
+ *******************************************/
+
+/* Bridge in structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hDevmemServerContext;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32FileOffset;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+/* Bridge out structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+#endif /* COMMON_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/server_pdumpmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/server_pdumpmm_bridge.c
new file mode 100644
index 000000000000..38e1a6610fd5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/server_pdumpmm_bridge.c
@@ -0,0 +1,958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+#include "common_pdumpmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpLoadMemIN_UI8,
+			    IMG_UINT8 * psPMRPDumpLoadMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *psPMRPDumpLoadMemIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *psPMRPDumpLoadMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemOUT->eError =
+	    PMRPDumpLoadMem(psPMRInt,
+			    psPMRPDumpLoadMemIN->uiOffset,
+			    psPMRPDumpLoadMemIN->uiSize,
+			    psPMRPDumpLoadMemIN->ui32PDumpFlags, psPMRPDumpLoadMemIN->bbZero);
+
+PMRPDumpLoadMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue32(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PMRPDumpLoadMemValue32(psPMRInt,
+				   psPMRPDumpLoadMemValue32IN->uiOffset,
+				   psPMRPDumpLoadMemValue32IN->ui32Value,
+				   psPMRPDumpLoadMemValue32IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue64(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue64IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue64OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue64_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PMRPDumpLoadMemValue64(psPMRInt,
+				   psPMRPDumpLoadMemValue64IN->uiOffset,
+				   psPMRPDumpLoadMemValue64IN->ui64Value,
+				   psPMRPDumpLoadMemValue64IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue64_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSaveToFile(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRPDumpSaveToFileIN_UI8,
+			       IMG_UINT8 * psPMRPDumpSaveToFileOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSaveToFileIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPMRPDumpSaveToFileIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSaveToFileIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSaveToFileIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSaveToFile_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPMRPDumpSaveToFileIN->puiFileName,
+		     psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSaveToFile_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSaveToFileOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSaveToFileOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSaveToFile_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSaveToFileOUT->eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       psPMRPDumpSaveToFileIN->uiOffset,
+			       psPMRPDumpSaveToFileIN->uiSize,
+			       psPMRPDumpSaveToFileIN->ui32ArraySize,
+			       uiFileNameInt, psPMRPDumpSaveToFileIN->ui32uiFileOffset);
+
+PMRPDumpSaveToFile_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSaveToFileOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSymbolicAddr(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrIN_UI8,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *) IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *)
+	    IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSymbolicAddrIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *puiMemspaceNameInt = NULL;
+	IMG_CHAR *puiSymbolicAddrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) + 0;
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen > PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen > PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	psPMRPDumpSymbolicAddrOUT->puiMemspaceName = psPMRPDumpSymbolicAddrIN->puiMemspaceName;
+	psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr = psPMRPDumpSymbolicAddrIN->puiSymbolicAddr;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSymbolicAddrIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSymbolicAddrIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSymbolicAddr_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen != 0)
+	{
+		puiMemspaceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR);
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen != 0)
+	{
+		puiSymbolicAddrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  psPMRPDumpSymbolicAddrIN->uiOffset,
+				  psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen,
+				  puiMemspaceNameInt,
+				  psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen,
+				  puiSymbolicAddrInt,
+				  &psPMRPDumpSymbolicAddrOUT->uiNewOffset,
+				  &psPMRPDumpSymbolicAddrOUT->uiNextSymName);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiMemspaceNameInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiMemspaceName,
+		      puiMemspaceNameInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiSymbolicAddrInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr,
+		      puiSymbolicAddrInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+PMRPDumpSymbolicAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSymbolicAddrOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpPol32(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psPMRPDumpPol32IN_UI8,
+			  IMG_UINT8 * psPMRPDumpPol32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *psPMRPDumpPol32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *psPMRPDumpPol32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpPol32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpPol32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpPol32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpPol32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpPol32OUT->eError =
+	    PMRPDumpPol32(psPMRInt,
+			  psPMRPDumpPol32IN->uiOffset,
+			  psPMRPDumpPol32IN->ui32Value,
+			  psPMRPDumpPol32IN->ui32Mask,
+			  psPMRPDumpPol32IN->eOperator, psPMRPDumpPol32IN->ui32PDumpFlags);
+
+PMRPDumpPol32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCheck32(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpCheck32IN_UI8,
+			    IMG_UINT8 * psPMRPDumpCheck32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *psPMRPDumpCheck32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *psPMRPDumpCheck32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCheck32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCheck32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCheck32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCheck32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCheck32OUT->eError =
+	    PMRPDumpCheck32(psPMRInt,
+			    psPMRPDumpCheck32IN->uiOffset,
+			    psPMRPDumpCheck32IN->ui32Value,
+			    psPMRPDumpCheck32IN->ui32Mask,
+			    psPMRPDumpCheck32IN->eOperator, psPMRPDumpCheck32IN->ui32PDumpFlags);
+
+PMRPDumpCheck32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRPDumpCBPIN_UI8,
+			IMG_UINT8 * psPMRPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCBP *psPMRPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *psPMRPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCBPIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCBPOUT->eError =
+	    PMRPDumpCBP(psPMRInt,
+			psPMRPDumpCBPIN->uiReadOffset,
+			psPMRPDumpCBPIN->uiWriteOffset,
+			psPMRPDumpCBPIN->uiPacketSize, psPMRPDumpCBPIN->uiBufferSize);
+
+PMRPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualIN_UI8,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerContext = psDevmemIntPDumpSaveToFileVirtualIN->hDevmemServerContext;
+	DEVMEMINT_CTX *psDevmemServerContextInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (unlikely
+	    (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevmemIntPDumpSaveToFileVirtualIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevmemIntPDumpSaveToFileVirtualIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevmemIntPDumpSaveToFileVirtual_exit;
+			}
+		}
+	}
+
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psDevmemIntPDumpSaveToFileVirtualIN->puiFileName,
+		     psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevmemIntPDumpSaveToFileVirtual_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize *
+				 sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerContextInt,
+				       hDevmemServerContext,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntPDumpSaveToFileVirtualOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    DevmemIntPDumpSaveToFileVirtual(psConnection, OSGetDevNode(psConnection),
+					    psDevmemServerContextInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->sAddress,
+					    psDevmemIntPDumpSaveToFileVirtualIN->uiSize,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize,
+					    uiFileNameInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32FileOffset,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32PDumpFlags);
+
+DevmemIntPDumpSaveToFileVirtual_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevmemIntPDumpSaveToFileVirtualOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+
+/*
+ * Register all PDUMPMM functions with services
+ */
+PVRSRV_ERROR InitPDUMPMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM,
+			      PVRSRVBridgePMRPDumpLoadMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32,
+			      PVRSRVBridgePMRPDumpLoadMemValue32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64,
+			      PVRSRVBridgePMRPDumpLoadMemValue64, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE,
+			      PVRSRVBridgePMRPDumpSaveToFile, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR,
+			      PVRSRVBridgePMRPDumpSymbolicAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32,
+			      PVRSRVBridgePMRPDumpPol32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPPOL32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32,
+			      PVRSRVBridgePMRPDumpCheck32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP,
+			      PVRSRVBridgePMRPDumpCBP, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+			      PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL,
+			      PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpmm functions with services
+ */
+void DeinitPDUMPMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_bridge.h
new file mode 100644
index 000000000000..2cfafd5a83e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PVRTL_BRIDGE_H
+#define CLIENT_PVRTL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pvrtl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams,
+						  IMG_UINT32 * pui32NumFound);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin,
+						IMG_UINT32 * pui32Available);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data);
+
+#endif /* CLIENT_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_direct_bridge.c
new file mode 100644
index 000000000000..fa2fbed9595b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_direct_bridge.c
@@ -0,0 +1,175 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pvrtl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pvrtl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#include "tlserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerOpenStreamKM(puiName, ui32Mode, &psSDInt, &psTLPMRInt);
+
+	*phSD = psSDInt;
+	*phTLPMR = psTLPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCloseStreamKM(psSDInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerAcquireDataKM(psSDInt, pui32ReadOffset, pui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerReleaseDataKM(psSDInt, ui32ReadOffset, ui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams, IMG_UINT32 * pui32NumFound)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerDiscoverStreamsKM(puiNamePattern, ui32Size, puiStreams, pui32NumFound);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin, IMG_UINT32 * pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    pui32BufferOffset, ui32Size, ui32SizeMin, pui32Available);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCommitStreamKM(psSDInt, ui32ReqSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerWriteDataKM(psSDInt, ui32Size, pui8Data);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/common_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/common_pvrtl_bridge.h
new file mode 100644
index 000000000000..edc822375222
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/common_pvrtl_bridge.h
@@ -0,0 +1,214 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PVRTL_BRIDGE_H
+#define COMMON_PVRTL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#define PVRSRV_BRIDGE_PVRTL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PVRTL_TLWRITEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PVRTL_CMD_LAST			(PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7)
+
+/*******************************************
+            TLOpenStream
+ *******************************************/
+
+/* Bridge in structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_IN_TLOPENSTREAM_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_UINT32 ui32Mode;
+} __packed PVRSRV_BRIDGE_IN_TLOPENSTREAM;
+
+/* Bridge out structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLOPENSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_HANDLE hTLPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLOPENSTREAM;
+
+/*******************************************
+            TLCloseStream
+ *******************************************/
+
+/* Bridge in structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCLOSESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLCLOSESTREAM;
+
+/* Bridge out structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCLOSESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCLOSESTREAM;
+
+/*******************************************
+            TLAcquireData
+ *******************************************/
+
+/* Bridge in structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_IN_TLACQUIREDATA_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLACQUIREDATA;
+
+/* Bridge out structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_OUT_TLACQUIREDATA_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLACQUIREDATA;
+
+/*******************************************
+            TLReleaseData
+ *******************************************/
+
+/* Bridge in structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_IN_TLRELEASEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_IN_TLRELEASEDATA;
+
+/* Bridge out structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_OUT_TLRELEASEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLRELEASEDATA;
+
+/*******************************************
+            TLDiscoverStreams
+ *******************************************/
+
+/* Bridge in structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS_TAG
+{
+	const IMG_CHAR *puiNamePattern;
+	IMG_CHAR *puiStreams;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS;
+
+/* Bridge out structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS_TAG
+{
+	IMG_CHAR *puiStreams;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumFound;
+} __packed PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS;
+
+/*******************************************
+            TLReserveStream
+ *******************************************/
+
+/* Bridge in structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_IN_TLRESERVESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32SizeMin;
+} __packed PVRSRV_BRIDGE_IN_TLRESERVESTREAM;
+
+/* Bridge out structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLRESERVESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Available;
+	IMG_UINT32 ui32BufferOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLRESERVESTREAM;
+
+/*******************************************
+            TLCommitStream
+ *******************************************/
+
+/* Bridge in structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCOMMITSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReqSize;
+} __packed PVRSRV_BRIDGE_IN_TLCOMMITSTREAM;
+
+/* Bridge out structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM;
+
+/*******************************************
+            TLWriteData
+ *******************************************/
+
+/* Bridge in structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_IN_TLWRITEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLWRITEDATA;
+
+/* Bridge out structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_OUT_TLWRITEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLWRITEDATA;
+
+#endif /* COMMON_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/server_pvrtl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/server_pvrtl_bridge.c
new file mode 100644
index 000000000000..9facf794011e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/server_pvrtl_bridge.c
@@ -0,0 +1,827 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "tlserver.h"
+
+#include "common_pvrtl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _TLOpenStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = TLServerCloseStreamKM((TL_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLOpenStream(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psTLOpenStreamIN_UI8,
+			 IMG_UINT8 * psTLOpenStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLOPENSTREAM *psTLOpenStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLOPENSTREAM *psTLOpenStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamOUT_UI8, 0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	psTLOpenStreamOUT->hSD = NULL;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLOpenStreamOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLOpenStream_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLOpenStreamIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLOpenStreamIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLOpenStreamOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLOpenStream_exit;
+			}
+		}
+	}
+
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psTLOpenStreamIN->puiName,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLOpenStreamOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLOpenStream_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) - 1] =
+		    '\0';
+	}
+
+	psTLOpenStreamOUT->eError =
+	    TLServerOpenStreamKM(uiNameInt, psTLOpenStreamIN->ui32Mode, &psSDInt, &psTLPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		goto TLOpenStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psTLOpenStreamOUT->hSD,
+							      (void *)psSDInt,
+							      PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _TLOpenStreampsSDIntRelease);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								 &psTLOpenStreamOUT->hTLPMR,
+								 (void *)psTLPMRInt,
+								 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 psTLOpenStreamOUT->hSD);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLOpenStream_exit:
+
+	if (psTLOpenStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psTLOpenStreamOUT->hSD)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psTLOpenStreamOUT->hSD,
+							     PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSDInt)
+		{
+			TLServerCloseStreamKM(psSDInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLOpenStreamOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCloseStream(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLCloseStreamIN_UI8,
+			  IMG_UINT8 * psTLCloseStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCLOSESTREAM *psTLCloseStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *psTLCloseStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLCloseStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psTLCloseStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	if (unlikely((psTLCloseStreamOUT->eError != PVRSRV_OK) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psTLCloseStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCloseStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLCloseStream_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLAcquireData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLAcquireDataIN_UI8,
+			  IMG_UINT8 * psTLAcquireDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLACQUIREDATA *psTLAcquireDataIN =
+	    (PVRSRV_BRIDGE_IN_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLACQUIREDATA *psTLAcquireDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLAcquireDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLAcquireDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLAcquireDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLAcquireData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLAcquireDataOUT->eError =
+	    TLServerAcquireDataKM(psSDInt,
+				  &psTLAcquireDataOUT->ui32ReadOffset,
+				  &psTLAcquireDataOUT->ui32ReadLen);
+
+TLAcquireData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReleaseData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLReleaseDataIN_UI8,
+			  IMG_UINT8 * psTLReleaseDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRELEASEDATA *psTLReleaseDataIN =
+	    (PVRSRV_BRIDGE_IN_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRELEASEDATA *psTLReleaseDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReleaseDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReleaseDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReleaseDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReleaseData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReleaseDataOUT->eError =
+	    TLServerReleaseDataKM(psSDInt,
+				  psTLReleaseDataIN->ui32ReadOffset,
+				  psTLReleaseDataIN->ui32ReadLen);
+
+TLReleaseData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLDiscoverStreams(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psTLDiscoverStreamsIN_UI8,
+			      IMG_UINT8 * psTLDiscoverStreamsOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *psTLDiscoverStreamsIN =
+	    (PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *psTLDiscoverStreamsOUT =
+	    (PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsOUT_UI8, 0);
+
+	IMG_CHAR *uiNamePatternInt = NULL;
+	IMG_CHAR *puiStreamsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) + 0;
+
+	if (psTLDiscoverStreamsIN->ui32Size > PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLDiscoverStreams_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psTLDiscoverStreamsOUT->puiStreams = psTLDiscoverStreamsIN->puiStreams;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLDiscoverStreams_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLDiscoverStreamsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLDiscoverStreamsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLDiscoverStreams_exit;
+			}
+		}
+	}
+
+	{
+		uiNamePatternInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNamePatternInt,
+		     (const void __user *)psTLDiscoverStreamsIN->puiNamePattern,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+		((IMG_CHAR *) uiNamePatternInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) -
+						1] = '\0';
+	}
+	if (psTLDiscoverStreamsIN->ui32Size != 0)
+	{
+		puiStreamsInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR);
+	}
+
+	psTLDiscoverStreamsOUT->eError =
+	    TLServerDiscoverStreamsKM(uiNamePatternInt,
+				      psTLDiscoverStreamsIN->ui32Size,
+				      puiStreamsInt, &psTLDiscoverStreamsOUT->ui32NumFound);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLDiscoverStreamsOUT->eError != PVRSRV_OK))
+	{
+		goto TLDiscoverStreams_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamsInt) && ((psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psTLDiscoverStreamsOUT->puiStreams, puiStreamsInt,
+		      (psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+	}
+
+TLDiscoverStreams_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLDiscoverStreamsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReserveStream(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psTLReserveStreamIN_UI8,
+			    IMG_UINT8 * psTLReserveStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRESERVESTREAM *psTLReserveStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *psTLReserveStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReserveStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReserveStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReserveStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReserveStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReserveStreamOUT->eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    &psTLReserveStreamOUT->ui32BufferOffset,
+				    psTLReserveStreamIN->ui32Size,
+				    psTLReserveStreamIN->ui32SizeMin,
+				    &psTLReserveStreamOUT->ui32Available);
+
+TLReserveStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCommitStream(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psTLCommitStreamIN_UI8,
+			   IMG_UINT8 * psTLCommitStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *psTLCommitStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *psTLCommitStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLCommitStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLCommitStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLCommitStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCommitStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLCommitStreamOUT->eError =
+	    TLServerCommitStreamKM(psSDInt, psTLCommitStreamIN->ui32ReqSize);
+
+TLCommitStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLWriteData(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psTLWriteDataIN_UI8,
+			IMG_UINT8 * psTLWriteDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLWRITEDATA *psTLWriteDataIN =
+	    (PVRSRV_BRIDGE_IN_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLWRITEDATA *psTLWriteDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLWriteDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psTLWriteDataIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLWriteData_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLWriteData_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLWriteDataIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLWriteDataIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLWriteDataOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLWriteData_exit;
+			}
+		}
+	}
+
+	if (psTLWriteDataIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psTLWriteDataIN->pui8Data,
+		     psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psTLWriteDataOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLWriteData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLWriteDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLWriteDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLWriteData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLWriteDataOUT->eError =
+	    TLServerWriteDataKM(psSDInt, psTLWriteDataIN->ui32Size, ui8DataInt);
+
+TLWriteData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLWriteDataOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+
+/*
+ * Register all PVRTL functions with services
+ */
+PVRSRV_ERROR InitPVRTLBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM,
+			      PVRSRVBridgeTLOpenStream, NULL, sizeof(PVRSRV_BRIDGE_IN_TLOPENSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLOPENSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM,
+			      PVRSRVBridgeTLCloseStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCLOSESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCLOSESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA,
+			      PVRSRVBridgeTLAcquireData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLACQUIREDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLACQUIREDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA,
+			      PVRSRVBridgeTLReleaseData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRELEASEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRELEASEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS,
+			      PVRSRVBridgeTLDiscoverStreams, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM,
+			      PVRSRVBridgeTLReserveStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRESERVESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRESERVESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM,
+			      PVRSRVBridgeTLCommitStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCOMMITSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA,
+			      PVRSRVBridgeTLWriteData, NULL, sizeof(PVRSRV_BRIDGE_IN_TLWRITEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLWRITEDATA));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pvrtl functions with services
+ */
+void DeinitPVRTLBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/common_rgxbreakpoint_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/common_rgxbreakpoint_bridge.h
new file mode 100644
index 000000000000..ca5290a88266
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/common_rgxbreakpoint_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxbreakpoint
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxbreakpoint
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXBREAKPOINT_BRIDGE_H
+#define COMMON_RGXBREAKPOINT_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_LAST			(PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXSetBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT_TAG
+{
+	IMG_UINT64 ui64TempSpillingAddr;
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 eFWDataMaster;
+	IMG_UINT32 ui32BreakpointAddr;
+	IMG_UINT32 ui32DM;
+	IMG_UINT32 ui32HandlerAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT;
+
+/* Bridge out structure for RGXSetBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT;
+
+/*******************************************
+            RGXClearBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXClearBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT;
+
+/* Bridge out structure for RGXClearBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT;
+
+/*******************************************
+            RGXEnableBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXEnableBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT;
+
+/* Bridge out structure for RGXEnableBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT;
+
+/*******************************************
+            RGXDisableBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXDisableBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT;
+
+/* Bridge out structure for RGXDisableBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT;
+
+/*******************************************
+            RGXOverallocateBPRegisters
+ *******************************************/
+
+/* Bridge in structure for RGXOverallocateBPRegisters */
+typedef struct PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS_TAG
+{
+	IMG_UINT32 ui32SharedRegs;
+	IMG_UINT32 ui32TempRegs;
+} __packed PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS;
+
+/* Bridge out structure for RGXOverallocateBPRegisters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS;
+
+#endif /* COMMON_RGXBREAKPOINT_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.c
new file mode 100644
index 000000000000..78fd2b405e92
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.c
@@ -0,0 +1,381 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxbreakpoint
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxbreakpoint
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxbreakpoint.h"
+
+#include "common_rgxbreakpoint_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXBREAKPOINT_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSetBreakpointIN_UI8,
+			     IMG_UINT8 * psRGXSetBreakpointOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT *psRGXSetBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT *) IMG_OFFSET_ADDR(psRGXSetBreakpointIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT *psRGXSetBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT *) IMG_OFFSET_ADDR(psRGXSetBreakpointOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXSetBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXSetBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetBreakpointOUT->eError =
+	    PVRSRVRGXSetBreakpointKM(psConnection, OSGetDevNode(psConnection),
+				     hPrivDataInt,
+				     psRGXSetBreakpointIN->eFWDataMaster,
+				     psRGXSetBreakpointIN->ui64TempSpillingAddr,
+				     psRGXSetBreakpointIN->ui32BreakpointAddr,
+				     psRGXSetBreakpointIN->ui32HandlerAddr,
+				     psRGXSetBreakpointIN->ui32DM);
+
+RGXSetBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXClearBreakpointIN_UI8,
+			       IMG_UINT8 * psRGXClearBreakpointOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT *psRGXClearBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT *) IMG_OFFSET_ADDR(psRGXClearBreakpointIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT *psRGXClearBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT *) IMG_OFFSET_ADDR(psRGXClearBreakpointOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPrivData = psRGXClearBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXClearBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXClearBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXClearBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXClearBreakpointOUT->eError =
+	    PVRSRVRGXClearBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXClearBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXEnableBreakpointIN_UI8,
+				IMG_UINT8 * psRGXEnableBreakpointOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT *psRGXEnableBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXEnableBreakpointIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT *psRGXEnableBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXEnableBreakpointOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPrivData = psRGXEnableBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXEnableBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXEnableBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXEnableBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXEnableBreakpointOUT->eError =
+	    PVRSRVRGXEnableBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXEnableBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXDisableBreakpointIN_UI8,
+				 IMG_UINT8 * psRGXDisableBreakpointOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT *psRGXDisableBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXDisableBreakpointIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT *psRGXDisableBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT *)
+	    IMG_OFFSET_ADDR(psRGXDisableBreakpointOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXDisableBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXDisableBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXDisableBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDisableBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXDisableBreakpointOUT->eError =
+	    PVRSRVRGXDisableBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXDisableBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOverallocateBPRegisters(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXOverallocateBPRegistersIN_UI8,
+				       IMG_UINT8 * psRGXOverallocateBPRegistersOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS *psRGXOverallocateBPRegistersIN =
+	    (PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS *)
+	    IMG_OFFSET_ADDR(psRGXOverallocateBPRegistersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS *psRGXOverallocateBPRegistersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS *)
+	    IMG_OFFSET_ADDR(psRGXOverallocateBPRegistersOUT_UI8, 0);
+
+	psRGXOverallocateBPRegistersOUT->eError =
+	    PVRSRVRGXOverallocateBPRegistersKM(psConnection, OSGetDevNode(psConnection),
+					       psRGXOverallocateBPRegistersIN->ui32TempRegs,
+					       psRGXOverallocateBPRegistersIN->ui32SharedRegs);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
+
+#if !defined(EXCLUDE_RGXBREAKPOINT_BRIDGE)
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void);
+void DeinitRGXBREAKPOINTBridge(void);
+
+/*
+ * Register all RGXBREAKPOINT functions with services
+ */
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT,
+			      PVRSRVBridgeRGXSetBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT,
+			      PVRSRVBridgeRGXClearBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT,
+			      PVRSRVBridgeRGXEnableBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT,
+			      PVRSRVBridgeRGXDisableBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS,
+			      PVRSRVBridgeRGXOverallocateBPRegisters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxbreakpoint functions with services
+ */
+void DeinitRGXBREAKPOINTBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS);
+
+}
+#else /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXBREAKPOINT_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXBREAKPOINTBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXBREAKPOINTBridge()
+
+#endif /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/common_rgxcmp_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/common_rgxcmp_bridge.h
new file mode 100644
index 000000000000..aa1d2b73985a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/common_rgxcmp_bridge.h
@@ -0,0 +1,309 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXCMP_BRIDGE_H
+#define COMMON_RGXCMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXCMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXCMP_CMD_LAST			(PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11)
+
+/*******************************************
+            RGXCreateComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticComputeContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+	IMG_UINT32 ui32StaticComputeContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT;
+
+/* Bridge out structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT;
+
+/*******************************************
+            RGXDestroyComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT;
+
+/* Bridge out structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT;
+
+/*******************************************
+            RGXFlushComputeData
+ *******************************************/
+
+/* Bridge in structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA;
+
+/* Bridge out structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA;
+
+/*******************************************
+            RGXSendCancelCmd
+ *******************************************/
+
+/* Bridge in structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32FirstIntJobRefToCancel;
+	IMG_INT32 i32LastIntJobRefToCancel;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD;
+
+/* Bridge out structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD;
+
+/*******************************************
+            RGXSetComputeContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/*******************************************
+            RGXNotifyComputeWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXSetComputeContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/*******************************************
+            RGXGetLastDeviceError
+ *******************************************/
+
+/* Bridge in structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR;
+
+/* Bridge out structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Error;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR;
+
+/*******************************************
+            RGXKickTimestampQuery
+ *******************************************/
+
+/* Bridge in structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY;
+
+/* Bridge out structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY;
+
+/*******************************************
+            RGXKickCDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKCDM_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumOfWorkgroups;
+	IMG_UINT32 ui32NumOfWorkitems;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKCDM;
+
+/* Bridge out structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKCDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	IMG_UINT32 ui32IntJobRef;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKCDM;
+
+/*******************************************
+            RGXCDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXCDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY;
+
+#endif /* COMMON_RGXCMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/server_rgxcmp_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/server_rgxcmp_bridge.c
new file mode 100644
index 000000000000..8910cfb5ba6a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/server_rgxcmp_bridge.c
@@ -0,0 +1,1580 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxcompute.h"
+
+#include "common_rgxcmp_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateComputeContextpsComputeContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyComputeContextKM((RGX_SERVER_COMPUTE_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_COMPUTECONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_COMPUTECONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXCreateComputeContextIN_UI8,
+				    IMG_UINT8 * psRGXCreateComputeContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateComputeContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticComputeContextStateInt = NULL;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateComputeContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize >
+	     RGXFWIF_STATIC_COMPUTECONTEXT_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateComputeContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateComputeContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateComputeContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateComputeContextIN->pui8FrameworkCmd,
+		     psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize != 0)
+	{
+		ui8StaticComputeContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		    sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticComputeContextStateInt,
+		     (const void __user *)psRGXCreateComputeContextIN->
+		     pui8StaticComputeContextState,
+		     psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVRGXCreateComputeContextKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXCreateComputeContextIN->i32Priority,
+					    psRGXCreateComputeContextIN->ui32FrameworkCmdSize,
+					    ui8FrameworkCmdInt,
+					    hPrivDataInt,
+					    psRGXCreateComputeContextIN->
+					    ui32StaticComputeContextStateSize,
+					    ui8StaticComputeContextStateInt,
+					    psRGXCreateComputeContextIN->ui32PackedCCBSizeU88,
+					    psRGXCreateComputeContextIN->ui32ContextFlags,
+					    psRGXCreateComputeContextIN->ui64RobustnessAddress,
+					    psRGXCreateComputeContextIN->ui32MaxDeadlineMS,
+					    &psComputeContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psRGXCreateComputeContextOUT->
+									 hComputeContext,
+									 (void *)
+									 psComputeContextInt,
+									 PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _RGXCreateComputeContextpsComputeContextIntRelease);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateComputeContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateComputeContextOUT->eError != PVRSRV_OK)
+	{
+		if (psComputeContextInt)
+		{
+			PVRSRVRGXDestroyComputeContextKM(psComputeContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateComputeContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXDestroyComputeContextIN_UI8,
+				     IMG_UINT8 * psRGXDestroyComputeContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXDestroyComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXDestroyComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyComputeContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyComputeContextIN->
+					      hComputeContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyComputeContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyComputeContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyComputeContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyComputeContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFlushComputeData(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXFlushComputeDataIN_UI8,
+				IMG_UINT8 * psRGXFlushComputeDataOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataIN =
+	    (PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataOUT_UI8,
+								      0);
+
+	IMG_HANDLE hComputeContext = psRGXFlushComputeDataIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXFlushComputeDataOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXFlushComputeData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXFlushComputeDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXFlushComputeDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXFlushComputeData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXFlushComputeDataOUT->eError = PVRSRVRGXFlushComputeDataKM(psComputeContextInt);
+
+RGXFlushComputeData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendCancelCmd(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSendCancelCmdIN_UI8,
+			     IMG_UINT8 * psRGXSendCancelCmdOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *psRGXSendCancelCmdIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *psRGXSendCancelCmdOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSendCancelCmdIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSendCancelCmdOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSendCancelCmd_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendCancelCmdOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendCancelCmd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVRGXSendCancelCmdKM(psComputeContextInt,
+				     psRGXSendCancelCmdIN->i32FirstIntJobRefToCancel,
+				     psRGXSendCancelCmdIN->i32LastIntJobRefToCancel);
+
+RGXSendCancelCmd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPriorityIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVRGXSetComputeContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						 psComputeContextInt,
+						 psRGXSetComputeContextPriorityIN->i32Priority);
+
+RGXSetComputeContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXNotifyComputeWriteOffsetUpdateIN_UI8,
+					      IMG_UINT8 *
+					      psRGXNotifyComputeWriteOffsetUpdateOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXNotifyComputeWriteOffsetUpdateIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXNotifyComputeWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXNotifyComputeWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXNotifyComputeWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXNotifyComputeWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(psComputeContextInt);
+
+RGXNotifyComputeWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPropertyIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVRGXSetComputeContextPropertyKM(psComputeContextInt,
+						 psRGXSetComputeContextPropertyIN->ui32Property,
+						 psRGXSetComputeContextPropertyIN->ui64Input,
+						 &psRGXSetComputeContextPropertyOUT->ui64Output);
+
+RGXSetComputeContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetLastDeviceError(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorIN_UI8,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXGetLastDeviceErrorOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXGetLastDeviceError_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetLastDeviceErrorIN);
+
+	psRGXGetLastDeviceErrorOUT->eError =
+	    PVRSRVRGXGetLastDeviceErrorKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetLastDeviceErrorOUT->ui32Error);
+
+RGXGetLastDeviceError_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTimestampQuery(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXKickTimestampQueryIN_UI8,
+				  IMG_UINT8 * psRGXKickTimestampQueryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickTimestampQueryIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickTimestampQueryIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTimestampQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTimestampQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTimestampQuery_exit;
+			}
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTimestampQueryIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickTimestampQueryIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickTimestampQueryIN->pui8DMCmd,
+		     psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTimestampQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTimestampQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVRGXKickTimestampQueryKM(psComputeContextInt,
+					  psRGXKickTimestampQueryIN->hCheckFenceFd,
+					  psRGXKickTimestampQueryIN->hUpdateTimeline,
+					  &psRGXKickTimestampQueryOUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXKickTimestampQueryIN->ui32CmdSize,
+					  ui8DMCmdInt, psRGXKickTimestampQueryIN->ui32ExtJobRef);
+
+RGXKickTimestampQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTimestampQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickCDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickCDMIN_UI8,
+		       IMG_UINT8 * psRGXKickCDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKCDM *psRGXKickCDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKCDM *psRGXKickCDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickCDMIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickCDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickCDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickCDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickCDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickCDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickCDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickCDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateOffset,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateValue,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickCDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickCDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickCDMIN->pui8DMCmd,
+		     psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickCDMIN->pui32SyncPMRFlags,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0, psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickCDMIN->phSyncPMRs,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickCDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickCDMOUT->eError =
+	    PVRSRVRGXKickCDMKM(psComputeContextInt,
+			       psRGXKickCDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickCDMIN->hCheckFenceFd,
+			       psRGXKickCDMIN->hUpdateTimeline,
+			       &psRGXKickCDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickCDMIN->hExportFenceToSignal,
+			       psRGXKickCDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickCDMIN->ui32PDumpFlags,
+			       psRGXKickCDMIN->ui32ExtJobRef,
+			       psRGXKickCDMIN->ui32SyncPMRCount,
+			       ui32SyncPMRFlagsInt,
+			       psSyncPMRsInt,
+			       psRGXKickCDMIN->ui32NumOfWorkgroups,
+			       psRGXKickCDMIN->ui32NumOfWorkitems,
+			       psRGXKickCDMIN->ui64DeadlineInus, &psRGXKickCDMOUT->ui32IntJobRef);
+
+RGXKickCDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickCDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXCDMGetSharedMemoryIN);
+
+	psRGXCDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXCDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXCDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMGetSharedMemory_exit:
+
+	if (psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXCDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+
+/*
+ * Register all RGXCMP functions with services
+ */
+PVRSRV_ERROR InitRGXCMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT,
+			      PVRSRVBridgeRGXCreateComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT,
+			      PVRSRVBridgeRGXDestroyComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA,
+			      PVRSRVBridgeRGXFlushComputeData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD,
+			      PVRSRVBridgeRGXSendCancelCmd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetComputeContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetComputeContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR,
+			      PVRSRVBridgeRGXGetLastDeviceError, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY,
+			      PVRSRVBridgeRGXKickTimestampQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM,
+			      PVRSRVBridgeRGXKickCDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKCDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKCDM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxcmp functions with services
+ */
+void DeinitRGXCMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
new file mode 100644
index 000000000000..534b8e9536ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
@@ -0,0 +1,339 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXFWDBG_BRIDGE_H
+#define COMMON_RGXFWDBG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "rgx_bridge.h"
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST			(PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15)
+
+/*******************************************
+            RGXFWDebugSetFWLog
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG_TAG
+{
+	IMG_UINT32 ui32RGXFWLogType;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG;
+
+/* Bridge out structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG;
+
+/*******************************************
+            RGXFWDebugDumpFreelistPageList
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/* Bridge out structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/*******************************************
+            RGXFWDebugPowerOff
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF;
+
+/* Bridge out structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF;
+
+/*******************************************
+            RGXFWDebugPowerOn
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON;
+
+/* Bridge out structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON;
+
+/*******************************************
+            RGXFWDebugSetVzConnectionCooldownPeriodInSec
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	IMG_UINT32 ui32VzConne;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/* Bridge out structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/*******************************************
+            RGXFWDebugSetHCSDeadline
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	IMG_UINT32 ui32RGXHCSDeadline;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE;
+
+/* Bridge out structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE;
+
+/*******************************************
+            RGXFWDebugSetDriverPriority
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/* Bridge out structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSlice
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32TSPercentage;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSliceInterval
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	IMG_UINT32 ui32TSIntervalMs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/*******************************************
+            RGXFWDebugSetDriverIsolationGroup
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32IsolationGroup;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/* Bridge out structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/*******************************************
+            RGXFWDebugSetOSNewOnlineState
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32OSNewState;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/* Bridge out structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/*******************************************
+            RGXFWDebugMapGuestHeap
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	IMG_UINT64 ui64ui64GuestHeapBase;
+	IMG_UINT32 ui32DriverID;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP;
+
+/* Bridge out structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP;
+
+/*******************************************
+            RGXFWDebugPHRConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32PHRMode;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE;
+
+/*******************************************
+            RGXFWDebugWdgConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32WdgPeriodUs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE;
+
+/*******************************************
+            RGXCurrentTime
+ *******************************************/
+
+/* Bridge in structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_IN_RGXCURRENTTIME_TAG
+{
+	IMG_UINT8 ui8TimerType;
+} __packed PVRSRV_BRIDGE_IN_RGXCURRENTTIME;
+
+/* Bridge out structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCURRENTTIME_TAG
+{
+	IMG_UINT64 ui64Time;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCURRENTTIME;
+
+/*******************************************
+            RGXFWDebugInjectFault
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT;
+
+/* Bridge out structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT;
+
+#endif /* COMMON_RGXFWDBG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
new file mode 100644
index 000000000000..e5646490aa2d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
@@ -0,0 +1,542 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "rgxfwdbg.h"
+#include "pmr.h"
+#include "rgxtimecorr.h"
+
+#include "common_rgxfwdbg_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetFWLog(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogOUT_UI8,
+								     0);
+
+	psRGXFWDebugSetFWLogOUT->eError =
+	    PVRSRVRGXFWDebugSetFWLogKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXFWDebugSetFWLogIN->ui32RGXFWLogType);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugDumpFreelistPageList(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListIN_UI8,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugDumpFreelistPageListIN);
+
+	psRGXFWDebugDumpFreelistPageListOUT->eError =
+	    PVRSRVRGXFWDebugDumpFreelistPageListKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOff(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugPowerOffIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugPowerOffOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOffIN);
+
+	psRGXFWDebugPowerOffOUT->eError =
+	    PVRSRVRGXFWDebugPowerOffKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOn(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXFWDebugPowerOnIN_UI8,
+			      IMG_UINT8 * psRGXFWDebugPowerOnOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOnIN);
+
+	psRGXFWDebugPowerOnOUT->eError =
+	    PVRSRVRGXFWDebugPowerOnKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec(IMG_UINT32 ui32DispatchTableEntry,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8,
+							 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8, 0);
+
+	psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT->eError =
+	    PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(psConnection,
+								 OSGetDevNode(psConnection),
+								 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN->
+								 ui32VzConne);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetHCSDeadline(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineIN_UI8,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineOUT_UI8, 0);
+
+	psRGXFWDebugSetHCSDeadlineOUT->eError =
+	    PVRSRVRGXFWDebugSetHCSDeadlineKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXFWDebugSetHCSDeadlineIN->ui32RGXHCSDeadline);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityIN_UI8,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverPriorityOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRGXFWDebugSetDriverPriorityIN->ui32DriverID,
+						psRGXFWDebugSetDriverPriorityIN->ui32Priority);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSlice(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceIN_UI8,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceKM(psConnection, OSGetDevNode(psConnection),
+						 psRGXFWDebugSetDriverTimeSliceIN->ui32DriverID,
+						 psRGXFWDebugSetDriverTimeSliceIN->
+						 ui32TSPercentage);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval(IMG_UINT32 ui32DispatchTableEntry,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8,
+						 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceIntervalOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(psConnection, OSGetDevNode(psConnection),
+							 psRGXFWDebugSetDriverTimeSliceIntervalIN->
+							 ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXFWDebugSetDriverIsolationGroupIN_UI8,
+					      IMG_UINT8 *
+					      psRGXFWDebugSetDriverIsolationGroupOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverIsolationGroupOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverIsolationGroupKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32DriverID,
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32IsolationGroup);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetOSNewOnlineState(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateIN_UI8,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateOUT_UI8, 0);
+
+	psRGXFWDebugSetOSNewOnlineStateOUT->eError =
+	    PVRSRVRGXFWDebugSetOSNewOnlineStateKM(psConnection, OSGetDevNode(psConnection),
+						  psRGXFWDebugSetOSNewOnlineStateIN->ui32DriverID,
+						  psRGXFWDebugSetOSNewOnlineStateIN->
+						  ui32OSNewState);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugMapGuestHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapOUT_UI8, 0);
+
+	psRGXFWDebugMapGuestHeapOUT->eError =
+	    PVRSRVRGXFWDebugMapGuestHeapKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugMapGuestHeapIN->ui32DriverID,
+					   psRGXFWDebugMapGuestHeapIN->ui64ui64GuestHeapBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPHRConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureOUT_UI8, 0);
+
+	psRGXFWDebugPHRConfigureOUT->eError =
+	    PVRSRVRGXFWDebugPHRConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugPHRConfigureIN->ui32ui32PHRMode);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugWdgConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureOUT_UI8, 0);
+
+	psRGXFWDebugWdgConfigureOUT->eError =
+	    PVRSRVRGXFWDebugWdgConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugWdgConfigureIN->ui32ui32WdgPeriodUs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCurrentTime(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psRGXCurrentTimeIN_UI8,
+			   IMG_UINT8 * psRGXCurrentTimeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCURRENTTIME *psRGXCurrentTimeIN =
+	    (PVRSRV_BRIDGE_IN_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *psRGXCurrentTimeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeOUT_UI8, 0);
+
+	psRGXCurrentTimeOUT->eError =
+	    PVRSRVRGXCurrentTime(psConnection, OSGetDevNode(psConnection),
+				 psRGXCurrentTimeIN->ui8TimerType, &psRGXCurrentTimeOUT->ui64Time);
+
+	return 0;
+}
+
+#define PVRSRVBridgeRGXFWDebugInjectFault NULL
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+
+/*
+ * Register all RGXFWDBG functions with services
+ */
+PVRSRV_ERROR InitRGXFWDBGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG,
+			      PVRSRVBridgeRGXFWDebugSetFWLog, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST,
+			      PVRSRVBridgeRGXFWDebugDumpFreelistPageList, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF,
+			      PVRSRVBridgeRGXFWDebugPowerOff, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON,
+			      PVRSRVBridgeRGXFWDebugPowerOn, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC,
+			      PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC),
+			      sizeof
+			      (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE,
+			      PVRSRVBridgeRGXFWDebugSetHCSDeadline, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY,
+			      PVRSRVBridgeRGXFWDebugSetDriverPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSlice, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP,
+			      PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE,
+			      PVRSRVBridgeRGXFWDebugSetOSNewOnlineState, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP,
+			      PVRSRVBridgeRGXFWDebugMapGuestHeap, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugPHRConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugWdgConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME,
+			      PVRSRVBridgeRGXCurrentTime, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCURRENTTIME),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCURRENTTIME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT,
+			      PVRSRVBridgeRGXFWDebugInjectFault, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxfwdbg functions with services
+ */
+void DeinitRGXFWDBGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/common_rgxhwperf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/common_rgxhwperf_bridge.h
new file mode 100644
index 000000000000..d1d664f2d05f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/common_rgxhwperf_bridge.h
@@ -0,0 +1,302 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXHWPERF_BRIDGE_H
+#define COMMON_RGXHWPERF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_hwperf.h"
+
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST			(PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+12)
+
+/*******************************************
+            RGXGetConfiguredHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/*******************************************
+            RGXGetEnabledHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS;
+
+/* Bridge out structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32BlockCount;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetHWPerfTimeStamp
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP;
+
+/* Bridge out structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT64 ui64TimeStamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP;
+
+/*******************************************
+            RGXCtrlHWPerf
+ *******************************************/
+
+/* Bridge in structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_IN_RGXCTRLHWPERF_TAG
+{
+	IMG_UINT64 ui64Mask;
+	IMG_UINT32 ui32StreamId;
+	IMG_BOOL bToggle;
+} __packed PVRSRV_BRIDGE_IN_RGXCTRLHWPERF;
+
+/* Bridge out structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF;
+
+/*******************************************
+            RGXGetHWPerfBvncFeatureFlags
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/* Bridge out structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	RGX_HWPERF_BVNC sBVNC;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/*******************************************
+            RGXControlHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	IMG_UINT16 *pui16BlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_BOOL bEnable;
+} __packed PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS;
+
+/* Bridge out structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS;
+
+/*******************************************
+            RGXConfigMuxHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXConfigMuxHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXConfigMuxHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS;
+
+/*******************************************
+            RGXConfigCustomCounters
+ *******************************************/
+
+/* Bridge in structure for RGXConfigCustomCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS_TAG
+{
+	IMG_UINT32 *pui32CustomCounterIDs;
+	IMG_UINT16 ui16CustomBlockID;
+	IMG_UINT16 ui16NumCustomCounters;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS;
+
+/* Bridge out structure for RGXConfigCustomCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS;
+
+/*******************************************
+            RGXConfigureHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_UINT32 ui32CtrlWord;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS;
+
+/* Bridge out structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetConfiguredHWPerfMuxCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfMuxCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfMuxCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS;
+
+/*******************************************
+            RGXOpenHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXCloseHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXWriteHWPerfClientEvent
+ *******************************************/
+
+/* Bridge in structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT;
+
+/* Bridge out structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT;
+
+#endif /* COMMON_RGXHWPERF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/server_rgxhwperf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/server_rgxhwperf_bridge.c
new file mode 100644
index 000000000000..fc8d05d6036a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/server_rgxhwperf_bridge.c
@@ -0,0 +1,1292 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxhwperf.h"
+#include "rgx_fwif_km.h"
+
+#include "common_rgxhwperf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersIN_UI8,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters =
+	    psRGXGetConfiguredHWPerfCountersIN->psConfiguredCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredCountersInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+						   psRGXGetConfiguredHWPerfCountersIN->ui32BlockID,
+						   psConfiguredCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters,
+		      psConfiguredCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetEnabledHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksIN_UI8,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT32 *pui32EnabledBlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) + 0;
+
+	psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs =
+	    psRGXGetEnabledHWPerfBlocksIN->pui32EnabledBlockIDs;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetEnabledHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXGetEnabledHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetEnabledHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		pui32EnabledBlockIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32);
+	}
+
+	psRGXGetEnabledHWPerfBlocksOUT->eError =
+	    PVRSRVRGXGetEnabledHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					      psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen,
+					      &psRGXGetEnabledHWPerfBlocksOUT->ui32BlockCount,
+					      pui32EnabledBlockIDsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetEnabledHWPerfBlocksOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32EnabledBlockIDsInt) &&
+	    ((psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs,
+		      pui32EnabledBlockIDsInt,
+		      (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetEnabledHWPerfBlocks_exit;
+		}
+	}
+
+RGXGetEnabledHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetEnabledHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfTimeStamp(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampIN_UI8,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfTimeStampIN);
+
+	psRGXGetHWPerfTimeStampOUT->eError =
+	    PVRSRVRGXGetHWPerfTimeStampKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetHWPerfTimeStampOUT->ui64TimeStamp);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCtrlHWPerf(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXCtrlHWPerfIN_UI8,
+			  IMG_UINT8 * psRGXCtrlHWPerfOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *psRGXCtrlHWPerfIN =
+	    (PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *psRGXCtrlHWPerfOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfOUT_UI8, 0);
+
+	psRGXCtrlHWPerfOUT->eError =
+	    PVRSRVRGXCtrlHWPerfKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXCtrlHWPerfIN->ui32StreamId,
+				  psRGXCtrlHWPerfIN->bToggle, psRGXCtrlHWPerfIN->ui64Mask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsIN_UI8,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfBvncFeatureFlagsIN);
+
+	psRGXGetHWPerfBvncFeatureFlagsOUT->eError =
+	    PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(psConnection, OSGetDevNode(psConnection),
+						 &psRGXGetHWPerfBvncFeatureFlagsOUT->sBVNC);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXControlHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksIN_UI8,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT16 *ui16BlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) + 0;
+
+	if (unlikely(psRGXControlHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXControlHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXControlHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXControlHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		ui16BlockIDsInt = (IMG_UINT16 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16);
+	}
+
+	/* Copy the data over */
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui16BlockIDsInt,
+		     (const void __user *)psRGXControlHWPerfBlocksIN->pui16BlockIDs,
+		     psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) != PVRSRV_OK)
+		{
+			psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXControlHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXControlHWPerfBlocksOUT->eError =
+	    PVRSRVRGXControlHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXControlHWPerfBlocksIN->bEnable,
+					   psRGXControlHWPerfBlocksIN->ui32ArrayLen,
+					   ui16BlockIDsInt);
+
+RGXControlHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXControlHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigMuxHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXConfigMuxHWPerfCountersIN_UI8,
+				       IMG_UINT8 * psRGXConfigMuxHWPerfCountersOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS *psRGXConfigMuxHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigMuxHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS *psRGXConfigMuxHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigMuxHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigMuxHWPerfCounters_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigMuxHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigMuxHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXConfigMuxHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigMuxHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigMuxHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_MUX_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								     ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+		    sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigMuxHWPerfCountersIN->psBlockConfigs,
+		     psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigMuxHWPerfCounters_exit;
+		}
+	}
+
+	psRGXConfigMuxHWPerfCountersOUT->eError =
+	    PVRSRVRGXConfigMuxHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+					       psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen,
+					       psBlockConfigsInt);
+
+RGXConfigMuxHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigMuxHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGX_HWPERF_MAX_CUSTOM_CNTRS <= IMG_UINT32_MAX,
+	      "RGX_HWPERF_MAX_CUSTOM_CNTRS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigCustomCounters(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXConfigCustomCountersIN_UI8,
+				    IMG_UINT8 * psRGXConfigCustomCountersOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS *psRGXConfigCustomCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigCustomCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS *psRGXConfigCustomCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigCustomCountersOUT_UI8, 0);
+
+	IMG_UINT32 *ui32CustomCounterIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32)) +
+	    0;
+
+	if (unlikely
+	    (psRGXConfigCustomCountersIN->ui16NumCustomCounters > RGX_HWPERF_MAX_CUSTOM_CNTRS))
+	{
+		psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigCustomCounters_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigCustomCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigCustomCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigCustomCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigCustomCounters_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigCustomCountersIN->ui16NumCustomCounters != 0)
+	{
+		ui32CustomCounterIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CustomCounterIDsInt,
+		     (const void __user *)psRGXConfigCustomCountersIN->pui32CustomCounterIDs,
+		     psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigCustomCounters_exit;
+		}
+	}
+
+	psRGXConfigCustomCountersOUT->eError =
+	    PVRSRVRGXConfigCustomCountersKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXConfigCustomCountersIN->ui16CustomBlockID,
+					    psRGXConfigCustomCountersIN->ui16NumCustomCounters,
+					    ui32CustomCounterIDsInt);
+
+RGXConfigCustomCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigCustomCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigureHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksIN_UI8,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigureHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigureHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigureHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigureHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigureHWPerfBlocksIN->psBlockConfigs,
+		     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigureHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXConfigureHWPerfBlocksOUT->eError =
+	    PVRSRVRGXConfigureHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXConfigureHWPerfBlocksIN->ui32CtrlWord,
+					     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen,
+					     psBlockConfigsInt);
+
+RGXConfigureHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigureHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfMuxCounters(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXGetConfiguredHWPerfMuxCountersIN_UI8,
+					      IMG_UINT8 *
+					      psRGXGetConfiguredHWPerfMuxCountersOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *psRGXGetConfiguredHWPerfMuxCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfMuxCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *psRGXGetConfiguredHWPerfMuxCountersOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfMuxCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfMuxCountersOUT->psConfiguredMuxCounters =
+	    psRGXGetConfiguredHWPerfMuxCountersIN->psConfiguredMuxCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfMuxCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfMuxCountersIN),
+			      sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfMuxCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfMuxCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredMuxCountersInt =
+		    (RGX_HWPERF_CONFIG_MUX_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								     ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXGetConfiguredHWPerfMuxCountersIN->
+						      ui32BlockID, psConfiguredMuxCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfMuxCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfMuxCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredMuxCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfMuxCountersOUT->
+		      psConfiguredMuxCounters, psConfiguredMuxCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+			    PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfMuxCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfMuxCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfMuxCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXOpenHWPerfClientStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCloseHWPerfClientStreamKM((HWPERF_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOpenHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamIN_UI8,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamOUT_UI8, 0);
+
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXOpenHWPerfClientStreamIN);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVRGXOpenHWPerfClientStreamKM(psConnection, OSGetDevNode(psConnection), &psSDInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXOpenHWPerfClientStreamOUT->hSD, (void *)psSDInt,
+				      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXOpenHWPerfClientStreampsSDIntRelease);
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXOpenHWPerfClientStream_exit:
+
+	if (psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psSDInt)
+		{
+			PVRSRVRGXCloseHWPerfClientStreamKM(psSDInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCloseHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamIN_UI8,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCloseHWPerfClientStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCloseHWPerfClientStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	if (unlikely((psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_OK) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCloseHWPerfClientStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCloseHWPerfClientStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCloseHWPerfClientStream_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXWriteHWPerfClientEvent(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventIN_UI8,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventIN =
+	    (PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psRGXWriteHWPerfClientEventIN->hSD;
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXWriteHWPerfClientEventIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXWriteHWPerfClientEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXWriteHWPerfClientEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXWriteHWPerfClientEvent_exit;
+			}
+		}
+	}
+
+	if (psRGXWriteHWPerfClientEventIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psRGXWriteHWPerfClientEventIN->pui8Data,
+		     psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXWriteHWPerfClientEvent_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD, IMG_TRUE);
+	if (unlikely(psRGXWriteHWPerfClientEventOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVRGXWriteHWPerfClientEventKM(psSDInt,
+					      psRGXWriteHWPerfClientEventIN->ui32Size, ui8DataInt);
+
+RGXWriteHWPerfClientEvent_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXWriteHWPerfClientEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+
+/*
+ * Register all RGXHWPERF functions with services
+ */
+PVRSRV_ERROR InitRGXHWPERFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS,
+			      PVRSRVBridgeRGXGetEnabledHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP,
+			      PVRSRVBridgeRGXGetHWPerfTimeStamp, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF,
+			      PVRSRVBridgeRGXCtrlHWPerf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCTRLHWPERF),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS,
+			      PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS,
+			      PVRSRVBridgeRGXControlHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXConfigMuxHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS,
+			      PVRSRVBridgeRGXConfigCustomCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS,
+			      PVRSRVBridgeRGXConfigureHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfMuxCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXOpenHWPerfClientStream, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXCloseHWPerfClientStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT,
+			      PVRSRVBridgeRGXWriteHWPerfClientEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxhwperf functions with services
+ */
+void DeinitRGXHWPERFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/common_rgxkicksync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/common_rgxkicksync_bridge.h
new file mode 100644
index 000000000000..afd882ca7529
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/common_rgxkicksync_bridge.h
@@ -0,0 +1,143 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXKICKSYNC_BRIDGE_H
+#define COMMON_RGXKICKSYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST			(PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3)
+
+/*******************************************
+            RGXCreateKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXDestroyKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXKickSync2
+ *******************************************/
+
+/* Bridge in structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKSYNC2_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 *pui32UpdateDevVarOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phUpdateUFODevVarBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hTimelineFenceFD;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKSYNC2;
+
+/* Bridge out structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKSYNC2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFenceFD;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKSYNC2;
+
+/*******************************************
+            RGXSetKickSyncContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+#endif /* COMMON_RGXKICKSYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/server_rgxkicksync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/server_rgxkicksync_bridge.c
new file mode 100644
index 000000000000..8d943cd536e2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/server_rgxkicksync_bridge.c
@@ -0,0 +1,593 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxkicksync.h"
+
+#include "common_rgxkicksync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateKickSyncContextpsKickSyncContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyKickSyncContextKM((RGX_SERVER_KICKSYNC_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateKickSyncContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateKickSyncContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXCreateKickSyncContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVRGXCreateKickSyncContextKM(psConnection, OSGetDevNode(psConnection),
+					     hPrivDataInt,
+					     psRGXCreateKickSyncContextIN->ui32PackedCCBSizeU88,
+					     psRGXCreateKickSyncContextIN->ui32ContextFlags,
+					     &psKickSyncContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateKickSyncContextOUT->hKickSyncContext,
+				      (void *)psKickSyncContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateKickSyncContextpsKickSyncContextIntRelease);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateKickSyncContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK)
+	{
+		if (psKickSyncContextInt)
+		{
+			PVRSRVRGXDestroyKickSyncContextKM(psKickSyncContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyKickSyncContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyKickSyncContextIN->
+					      hKickSyncContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyKickSyncContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyKickSyncContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyKickSyncContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyKickSyncContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_DEV_VARS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_DEV_VARS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickSync2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickSync2IN_UI8,
+			 IMG_UINT8 * psRGXKickSync2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *psRGXKickSync2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *psRGXKickSync2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2OUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXKickSync2IN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFODevVarBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFODevVarBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateDevVarOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRGXKickSync2IN->ui32ClientUpdateCount > PVRSRV_MAX_DEV_VARS))
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickSync2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickSync2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickSync2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickSync2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickSync2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFODevVarBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFODevVarBlockInt, 0,
+			       psRGXKickSync2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFODevVarBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFODevVarBlockInt2,
+		     (const void __user *)psRGXKickSync2IN->phUpdateUFODevVarBlock,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateDevVarOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateDevVarOffsetInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateDevVarOffset,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateValue,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickSync2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickSync2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickSync2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickSync2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFODevVarBlockInt[i],
+						       hUpdateUFODevVarBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickSync2OUT->eError =
+	    PVRSRVRGXKickSyncKM(psKickSyncContextInt,
+				psRGXKickSync2IN->ui32ClientUpdateCount,
+				psUpdateUFODevVarBlockInt,
+				ui32UpdateDevVarOffsetInt,
+				ui32UpdateValueInt,
+				psRGXKickSync2IN->hCheckFenceFD,
+				psRGXKickSync2IN->hTimelineFenceFD,
+				&psRGXKickSync2OUT->hUpdateFenceFD,
+				uiUpdateFenceNameInt, psRGXKickSync2IN->ui32ExtJobRef);
+
+RGXKickSync2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+
+	if (hUpdateUFODevVarBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFODevVarBlockInt && psUpdateUFODevVarBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFODevVarBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickSync2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetKickSyncContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXSetKickSyncContextPropertyIN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetKickSyncContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetKickSyncContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVRGXSetKickSyncContextPropertyKM(psKickSyncContextInt,
+						  psRGXSetKickSyncContextPropertyIN->ui32Property,
+						  psRGXSetKickSyncContextPropertyIN->ui64Input,
+						  &psRGXSetKickSyncContextPropertyOUT->ui64Output);
+
+RGXSetKickSyncContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+
+/*
+ * Register all RGXKICKSYNC functions with services
+ */
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXCreateKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXDestroyKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2,
+			      PVRSRVBridgeRGXKickSync2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKSYNC2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKSYNC2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetKickSyncContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxkicksync functions with services
+ */
+void DeinitRGXKICKSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY);
+
+}
+#else /* SUPPORT_RGXKICKSYNC_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXKICKSYNC_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXKICKSYNCBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXKICKSYNCBridge()
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_bridge.h
new file mode 100644
index 000000000000..ed5a664ef17e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_bridge.h
@@ -0,0 +1,73 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RGXPDUMP_BRIDGE_H
+#define CLIENT_RGXPDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_rgxpdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
new file mode 100644
index 000000000000..7ba5126053dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
@@ -0,0 +1,118 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for rgxpdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_rgxpdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "rgx_bridge.h"
+
+#include "rgxpdump.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpTraceBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpSignatureBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPreCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPostCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					     ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/common_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/common_rgxpdump_bridge.h
new file mode 100644
index 000000000000..37317acb4707
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/common_rgxpdump_bridge.h
@@ -0,0 +1,161 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXPDUMP_BRIDGE_H
+#define COMMON_RGXPDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST			(PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5)
+
+/*******************************************
+            PDumpTraceBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER;
+
+/* Bridge out structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER;
+
+/*******************************************
+            PDumpSignatureBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER;
+
+/* Bridge out structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER;
+
+/*******************************************
+            PDumpComputeCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpValCheckPreCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND;
+
+/* Bridge out structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND;
+
+/*******************************************
+            PDumpValCheckPostCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND;
+
+/* Bridge out structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND;
+
+#endif /* COMMON_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/server_rgxpdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/server_rgxpdump_bridge.c
new file mode 100644
index 000000000000..58ed4ebb037f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/server_rgxpdump_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxpdump.h"
+
+#include "common_rgxpdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePDumpTraceBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psPDumpTraceBufferIN_UI8,
+			     IMG_UINT8 * psPDumpTraceBufferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *psPDumpTraceBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *psPDumpTraceBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferOUT_UI8, 0);
+
+	psPDumpTraceBufferOUT->eError =
+	    PVRSRVPDumpTraceBufferKM(psConnection, OSGetDevNode(psConnection),
+				     psPDumpTraceBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpSignatureBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpSignatureBufferIN_UI8,
+				 IMG_UINT8 * psPDumpSignatureBufferOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *) IMG_OFFSET_ADDR(psPDumpSignatureBufferIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *)
+	    IMG_OFFSET_ADDR(psPDumpSignatureBufferOUT_UI8, 0);
+
+	psPDumpSignatureBufferOUT->eError =
+	    PVRSRVPDumpSignatureBufferKM(psConnection, OSGetDevNode(psConnection),
+					 psPDumpSignatureBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+#define PVRSRVBridgePDumpComputeCRCSignatureCheck NULL
+
+static IMG_INT
+PVRSRVBridgePDumpCRCSignatureCheck(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckIN_UI8,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckOUT_UI8, 0);
+
+	psPDumpCRCSignatureCheckOUT->eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(psConnection, OSGetDevNode(psConnection),
+					   psPDumpCRCSignatureCheckIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPreCommand(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psPDumpValCheckPreCommandIN_UI8,
+				    IMG_UINT8 * psPDumpValCheckPreCommandOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandOUT_UI8, 0);
+
+	psPDumpValCheckPreCommandOUT->eError =
+	    PVRSRVPDumpValCheckPreCommandKM(psConnection, OSGetDevNode(psConnection),
+					    psPDumpValCheckPreCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPostCommand(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPDumpValCheckPostCommandIN_UI8,
+				     IMG_UINT8 * psPDumpValCheckPostCommandOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandOUT_UI8, 0);
+
+	psPDumpValCheckPostCommandOUT->eError =
+	    PVRSRVPDumpValCheckPostCommandKM(psConnection, OSGetDevNode(psConnection),
+					     psPDumpValCheckPostCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+
+/*
+ * Register all RGXPDUMP functions with services
+ */
+PVRSRV_ERROR InitRGXPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER,
+			      PVRSRVBridgePDumpTraceBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER,
+			      PVRSRVBridgePDumpSignatureBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpComputeCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND,
+			      PVRSRVBridgePDumpValCheckPreCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND,
+			      PVRSRVBridgePDumpValCheckPostCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxpdump functions with services
+ */
+void DeinitRGXPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/common_rgxregconfig_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/common_rgxregconfig_bridge.h
new file mode 100644
index 000000000000..942b7e4b06d0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/common_rgxregconfig_bridge.h
@@ -0,0 +1,146 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXREGCONFIG_BRIDGE_H
+#define COMMON_RGXREGCONFIG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST			(PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetRegConfigType
+ *******************************************/
+
+/* Bridge in structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE_TAG
+{
+	IMG_UINT8 ui8RegPowerIsland;
+} __packed PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE;
+
+/* Bridge out structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE;
+
+/*******************************************
+            RGXAddRegconfig
+ *******************************************/
+
+/* Bridge in structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXADDREGCONFIG_TAG
+{
+	IMG_UINT64 ui64RegMask;
+	IMG_UINT64 ui64RegValue;
+	IMG_UINT32 ui32RegAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXADDREGCONFIG;
+
+/* Bridge out structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG;
+
+/*******************************************
+            RGXClearRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG;
+
+/* Bridge out structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG;
+
+/*******************************************
+            RGXEnableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG;
+
+/* Bridge out structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG;
+
+/*******************************************
+            RGXDisableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG;
+
+/* Bridge out structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG;
+
+#endif /* COMMON_RGXREGCONFIG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/server_rgxregconfig_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/server_rgxregconfig_bridge.c
new file mode 100644
index 000000000000..548401470d09
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/server_rgxregconfig_bridge.c
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxregconfig.h"
+
+#include "common_rgxregconfig_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetRegConfigType(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXSetRegConfigTypeIN_UI8,
+				IMG_UINT8 * psRGXSetRegConfigTypeOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeOUT_UI8,
+								      0);
+
+	psRGXSetRegConfigTypeOUT->eError =
+	    PVRSRVRGXSetRegConfigTypeKM(psConnection, OSGetDevNode(psConnection),
+					psRGXSetRegConfigTypeIN->ui8RegPowerIsland);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXAddRegconfig(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRGXAddRegconfigIN_UI8,
+			    IMG_UINT8 * psRGXAddRegconfigOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *psRGXAddRegconfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *psRGXAddRegconfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigOUT_UI8, 0);
+
+	psRGXAddRegconfigOUT->eError =
+	    PVRSRVRGXAddRegConfigKM(psConnection, OSGetDevNode(psConnection),
+				    psRGXAddRegconfigIN->ui32RegAddr,
+				    psRGXAddRegconfigIN->ui64RegValue,
+				    psRGXAddRegconfigIN->ui64RegMask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXClearRegConfigIN_UI8,
+			      IMG_UINT8 * psRGXClearRegConfigOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *psRGXClearRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *psRGXClearRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXClearRegConfigIN);
+
+	psRGXClearRegConfigOUT->eError =
+	    PVRSRVRGXClearRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXEnableRegConfigIN_UI8,
+			       IMG_UINT8 * psRGXEnableRegConfigOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *psRGXEnableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *psRGXEnableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEnableRegConfigIN);
+
+	psRGXEnableRegConfigOUT->eError =
+	    PVRSRVRGXEnableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXDisableRegConfigIN_UI8,
+				IMG_UINT8 * psRGXDisableRegConfigOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *psRGXDisableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *psRGXDisableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXDisableRegConfigIN);
+
+	psRGXDisableRegConfigOUT->eError =
+	    PVRSRVRGXDisableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+
+/*
+ * Register all RGXREGCONFIG functions with services
+ */
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE,
+			      PVRSRVBridgeRGXSetRegConfigType, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG,
+			      PVRSRVBridgeRGXAddRegconfig, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXADDREGCONFIG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG,
+			      PVRSRVBridgeRGXClearRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG,
+			      PVRSRVBridgeRGXEnableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG,
+			      PVRSRVBridgeRGXDisableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxregconfig functions with services
+ */
+void DeinitRGXREGCONFIGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG);
+
+}
+#else /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXREGCONFIG_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXREGCONFIGBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXREGCONFIGBridge()
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/common_rgxta3d_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/common_rgxta3d_bridge.h
new file mode 100644
index 000000000000..cf215caeddff
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/common_rgxta3d_bridge.h
@@ -0,0 +1,466 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTA3D_BRIDGE_H
+#define COMMON_RGXTA3D_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_fwif_shared.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_LAST			(PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15)
+
+/*******************************************
+            RGXDestroyHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET_TAG
+{
+	IMG_HANDLE hKmHwRTDataSet;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET;
+
+/* Bridge out structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET;
+
+/*******************************************
+            RGXCreateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER;
+
+/* Bridge out structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferMemDesc;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER;
+
+/* Bridge out structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER;
+
+/*******************************************
+            RGXPopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+} __packed PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXUnpopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+} __packed PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST;
+
+/* Bridge out structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST;
+
+/*******************************************
+            RGXDestroyRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT;
+
+/* Bridge out structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT;
+
+/*******************************************
+            RGXSendZSStoreDisable
+ *******************************************/
+
+/* Bridge in structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32ExtJobRefToDisableZSStore;
+	IMG_BOOL bDisableDepthStore;
+	IMG_BOOL bDisableStencilStore;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE;
+
+/* Bridge out structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE;
+
+/*******************************************
+            RGXSetRenderContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXRenderContextStalled
+ *******************************************/
+
+/* Bridge in structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED_TAG
+{
+	IMG_HANDLE hRenderContext;
+} __packed PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED;
+
+/* Bridge out structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED;
+
+/*******************************************
+            RGXKickTA3D2
+ *******************************************/
+
+/* Bridge in structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTA3D2_TAG
+{
+	IMG_UINT64 ui64Deadline;
+	IMG_HANDLE hKMHWRTDataSet;
+	IMG_HANDLE hMSAAScratchBuffer;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock;
+	IMG_HANDLE hRenderContext;
+	IMG_HANDLE hZSBuffer;
+	IMG_UINT32 *pui32Client3DUpdateSyncOffset;
+	IMG_UINT32 *pui32Client3DUpdateValue;
+	IMG_UINT32 *pui32ClientTAFenceSyncOffset;
+	IMG_UINT32 *pui32ClientTAFenceValue;
+	IMG_UINT32 *pui32ClientTAUpdateSyncOffset;
+	IMG_UINT32 *pui32ClientTAUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui83DCmd;
+	IMG_BYTE *pui83DPRCmd;
+	IMG_BYTE *pui8TACmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_CHAR *puiUpdateFenceName3D;
+	IMG_HANDLE *phClient3DUpdateSyncPrimBlock;
+	IMG_HANDLE *phClientTAFenceSyncPrimBlock;
+	IMG_HANDLE *phClientTAUpdateSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFence;
+	PVRSRV_FENCE hCheckFence3D;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	PVRSRV_TIMELINE hUpdateTimeline3D;
+	IMG_UINT32 ui323DCmdSize;
+	IMG_UINT32 ui323DPRCmdSize;
+	IMG_UINT32 ui32Client3DUpdateCount;
+	IMG_UINT32 ui32ClientTAFenceCount;
+	IMG_UINT32 ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumberOfDrawCalls;
+	IMG_UINT32 ui32NumberOfIndices;
+	IMG_UINT32 ui32NumberOfMRTs;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PRFenceUFOSyncOffset;
+	IMG_UINT32 ui32PRFenceValue;
+	IMG_UINT32 ui32RenderTargetSize;
+	IMG_UINT32 ui32SyncPMRCount;
+	IMG_UINT32 ui32TACmdSize;
+	IMG_BOOL bbAbort;
+	IMG_BOOL bbKick3D;
+	IMG_BOOL bbKickPR;
+	IMG_BOOL bbKickTA;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTA3D2;
+
+/* Bridge out structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTA3D2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	PVRSRV_FENCE hUpdateFence3D;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTA3D2;
+
+/*******************************************
+            RGXSetRenderContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hRenderContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXCreateHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_UINT64 ui64FlippedMultiSampleCtl;
+	IMG_UINT64 ui64MultiSampleCtl;
+	IMG_DEV_VIRTADDR *psMacrotileArrayDevVAddr;
+	IMG_DEV_VIRTADDR *psPMMlistDevVAddr;
+	IMG_DEV_VIRTADDR *psRTCDevVAddr;
+	IMG_DEV_VIRTADDR *psRgnHeaderDevVAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_DEV_VIRTADDR *psVHeapTableDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT32 ui32ISPMergeLowerX;
+	IMG_UINT32 ui32ISPMergeLowerY;
+	IMG_UINT32 ui32ISPMergeScaleX;
+	IMG_UINT32 ui32ISPMergeScaleY;
+	IMG_UINT32 ui32ISPMergeUpperX;
+	IMG_UINT32 ui32ISPMergeUpperY;
+	IMG_UINT32 ui32ISPMtileSize;
+	IMG_UINT32 ui32MTileStride;
+	IMG_UINT32 ui32PPPScreen;
+	IMG_UINT32 ui32RgnHeaderSize;
+	IMG_UINT32 ui32TEAA;
+	IMG_UINT32 ui32TEMTILE1;
+	IMG_UINT32 ui32TEMTILE2;
+	IMG_UINT32 ui32TEScreen;
+	IMG_UINT32 ui32TPCSize;
+	IMG_UINT32 ui32TPCStride;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET;
+
+/* Bridge out structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET;
+
+/*******************************************
+            RGXCreateFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hFreeListReservation;
+	IMG_HANDLE hMemCtxPrivData;
+	IMG_HANDLE hsGlobalFreeList;
+	IMG_UINT32 ui32GrowFLPages;
+	IMG_UINT32 ui32GrowParamThreshold;
+	IMG_UINT32 ui32InitFLPages;
+	IMG_UINT32 ui32MaxFLPages;
+	IMG_BOOL bbFreeListCheck;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEFREELIST;
+
+/* Bridge out structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST;
+
+/*******************************************
+            RGXCreateRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_DEV_VIRTADDR sVDMCallStackAddr;
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticRenderContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32Max3DDeadlineMS;
+	IMG_UINT32 ui32MaxTADeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+	IMG_UINT32 ui32StaticRenderContextStateSize;
+	IMG_UINT32 ui32ui32CallStackDepth;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT;
+
+/* Bridge out structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_HANDLE hRenderContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT;
+
+/*******************************************
+            RGXCreateHWRTDataSet2
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_UINT64 ui64FlippedMultiSampleCtl;
+	IMG_UINT64 ui64MultiSampleCtl;
+	IMG_HANDLE hPMMlistsReservation;
+	IMG_DEV_VIRTADDR *psMacrotileArrayDevVAddr;
+	IMG_DEV_VIRTADDR *psRTCDevVAddr;
+	IMG_DEV_VIRTADDR *psRgnHeaderDevVAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_DEV_VIRTADDR *psVHeapTableDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT32 ui32ISPMergeLowerX;
+	IMG_UINT32 ui32ISPMergeLowerY;
+	IMG_UINT32 ui32ISPMergeScaleX;
+	IMG_UINT32 ui32ISPMergeScaleY;
+	IMG_UINT32 ui32ISPMergeUpperX;
+	IMG_UINT32 ui32ISPMergeUpperY;
+	IMG_UINT32 ui32ISPMtileSize;
+	IMG_UINT32 ui32MTileStride;
+	IMG_UINT32 ui32PPPScreen;
+	IMG_UINT32 ui32RgnHeaderSize;
+	IMG_UINT32 ui32TEAA;
+	IMG_UINT32 ui32TEMTILE1;
+	IMG_UINT32 ui32TEMTILE2;
+	IMG_UINT32 ui32TEScreen;
+	IMG_UINT32 ui32TPCSize;
+	IMG_UINT32 ui32TPCStride;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2;
+
+/* Bridge out structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2;
+
+#endif /* COMMON_RGXTA3D_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/server_rgxta3d_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/server_rgxta3d_bridge.c
new file mode 100644
index 000000000000..121ec9714cd3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/server_rgxta3d_bridge.c
@@ -0,0 +1,3005 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxta3d.h"
+
+#include "common_rgxta3d_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetIN_UI8,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyHWRTDataSetOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyHWRTDataSetIN->
+					      hKmHwRTDataSet,
+					      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+	if (unlikely
+	    ((psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyHWRTDataSetOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyHWRTDataSet_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyHWRTDataSet_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateZSBufferpssZSBufferKMIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyZSBufferKM((RGX_ZSBUFFER_DATA *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateZSBufferIN_UI8,
+			      IMG_UINT8 * psRGXCreateZSBufferOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *psRGXCreateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *psRGXCreateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psRGXCreateZSBufferIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psRGXCreateZSBufferIN->hPMR;
+	PMR *psPMRInt = NULL;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError =
+	    RGXCreateZSBufferKM(psConnection, OSGetDevNode(psConnection),
+				psReservationInt,
+				psPMRInt, psRGXCreateZSBufferIN->uiMapFlags, &pssZSBufferKMInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateZSBufferOUT->
+								   hsZSBufferKM,
+								   (void *)pssZSBufferKMInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateZSBufferpssZSBufferKMIntRelease);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssZSBufferKMInt)
+		{
+			RGXDestroyZSBufferKM(pssZSBufferKMInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyZSBufferIN_UI8,
+			       IMG_UINT8 * psRGXDestroyZSBufferOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyZSBufferIN->
+					      hsZSBufferMemDesc,
+					      PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	if (unlikely
+	    ((psRGXDestroyZSBufferOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyZSBuffer_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXPopulateZSBufferpssPopulationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXUnpopulateZSBufferKM((RGX_POPULATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXPopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXPopulateZSBufferIN_UI8,
+				IMG_UINT8 * psRGXPopulateZSBufferOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferOUT_UI8,
+								      0);
+
+	IMG_HANDLE hsZSBufferKM = psRGXPopulateZSBufferIN->hsZSBufferKM;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+	RGX_POPULATION *pssPopulationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXPopulateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pssZSBufferKMInt,
+				       hsZSBufferKM,
+				       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError =
+	    RGXPopulateZSBufferKM(pssZSBufferKMInt, &pssPopulationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXPopulateZSBufferOUT->
+								     hsPopulation,
+								     (void *)pssPopulationInt,
+								     PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXPopulateZSBufferpssPopulationIntRelease);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXPopulateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pssZSBufferKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hsZSBufferKM, PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXPopulateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssPopulationInt)
+		{
+			RGXUnpopulateZSBufferKM(pssPopulationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXUnpopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferIN_UI8,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXUnpopulateZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXUnpopulateZSBufferIN->hsPopulation,
+					      PVRSRV_HANDLE_TYPE_RGX_POPULATION);
+	if (unlikely((psRGXUnpopulateZSBufferOUT->eError != PVRSRV_OK) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXUnpopulateZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXUnpopulateZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXUnpopulateZSBuffer_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyFreeListIN_UI8,
+			       IMG_UINT8 * psRGXDestroyFreeListOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *psRGXDestroyFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *psRGXDestroyFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyFreeListOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyFreeListIN->hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+	if (unlikely((psRGXDestroyFreeListOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyFreeListOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyFreeList_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyFreeList_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXDestroyRenderContextIN_UI8,
+				    IMG_UINT8 * psRGXDestroyRenderContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRenderContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRenderContextIN->
+					      hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyRenderContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRenderContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRenderContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRenderContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendZSStoreDisable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXSendZSStoreDisableIN_UI8,
+				  IMG_UINT8 * psRGXSendZSStoreDisableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSendZSStoreDisableIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendZSStoreDisableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendZSStoreDisable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVRGXSendZSStoreDisableKM(psConnection, OSGetDevNode(psConnection),
+					  psRenderContextInt,
+					  psRGXSendZSStoreDisableIN->bDisableDepthStore,
+					  psRGXSendZSStoreDisableIN->bDisableStencilStore,
+					  psRGXSendZSStoreDisableIN->i32ExtJobRefToDisableZSStore);
+
+RGXSendZSStoreDisable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPriorityIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPriorityIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVRGXSetRenderContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRenderContextInt,
+						psRGXSetRenderContextPriorityIN->i32Priority);
+
+RGXSetRenderContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXRenderContextStalled(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXRenderContextStalledIN_UI8,
+				    IMG_UINT8 * psRGXRenderContextStalledOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledIN =
+	    (PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXRenderContextStalledIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXRenderContextStalledOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXRenderContextStalledOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXRenderContextStalled_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXRenderContextStalledOUT->eError = RGXRenderContextStalledKM(psRenderContextInt);
+
+RGXRenderContextStalled_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTA3D2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickTA3D2IN_UI8,
+			 IMG_UINT8 * psRGXKickTA3D2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *psRGXKickTA3D2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *psRGXKickTA3D2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2OUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXKickTA3D2IN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAFenceSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAFenceSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAFenceSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAFenceValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClient3DUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClient3DUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32Client3DUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32Client3DUpdateValueInt = NULL;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock = psRGXKickTA3D2IN->hPRFenceUFOSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *psPRFenceUFOSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_CHAR *uiUpdateFenceName3DInt = NULL;
+	IMG_BYTE *ui8TACmdInt = NULL;
+	IMG_BYTE *ui83DPRCmdInt = NULL;
+	IMG_BYTE *ui83DCmdInt = NULL;
+	IMG_HANDLE hKMHWRTDataSet = psRGXKickTA3D2IN->hKMHWRTDataSet;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSetInt = NULL;
+	IMG_HANDLE hZSBuffer = psRGXKickTA3D2IN->hZSBuffer;
+	RGX_ZSBUFFER_DATA *psZSBufferInt = NULL;
+	IMG_HANDLE hMSAAScratchBuffer = psRGXKickTA3D2IN->hMSAAScratchBuffer;
+	RGX_ZSBUFFER_DATA *psMSAAScratchBufferInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAFenceCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32Client3DUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32TACmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DPRCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTA3D2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTA3D2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTA3D2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		psClientTAFenceSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAFenceSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAFenceSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAFenceSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAFenceSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceValue,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		psClientTAUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateValue,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		psClient3DUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClient3DUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClient3DUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClient3DUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClient3DUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateValue,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	{
+		uiUpdateFenceName3DInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceName3DInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName3D,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceName3DInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						      1] = '\0';
+	}
+	if (psRGXKickTA3D2IN->ui32TACmdSize != 0)
+	{
+		ui8TACmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8TACmdInt, (const void __user *)psRGXKickTA3D2IN->pui8TACmd,
+		     psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize != 0)
+	{
+		ui83DPRCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DPRCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DPRCmd,
+		     psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DCmdSize != 0)
+	{
+		ui83DCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DCmd,
+		     psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32SyncPMRFlags,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickTA3D2IN->phSyncPMRs,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psClientTAFenceSyncPrimBlockInt[i],
+						       hClientTAFenceSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientTAUpdateSyncPrimBlockInt[i],
+						       hClientTAUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClient3DUpdateSyncPrimBlockInt[i],
+						       hClient3DUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPRFenceUFOSyncPrimBlockInt,
+				       hPRFenceUFOSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psKMHWRTDataSetInt,
+					       hKMHWRTDataSet,
+					       PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psZSBufferInt,
+					       hZSBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psMSAAScratchBufferInt,
+					       hMSAAScratchBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVRGXKickTA3DKM(psRenderContextInt,
+				psRGXKickTA3D2IN->ui32ClientTAFenceCount,
+				psClientTAFenceSyncPrimBlockInt,
+				ui32ClientTAFenceSyncOffsetInt,
+				ui32ClientTAFenceValueInt,
+				psRGXKickTA3D2IN->ui32ClientTAUpdateCount,
+				psClientTAUpdateSyncPrimBlockInt,
+				ui32ClientTAUpdateSyncOffsetInt,
+				ui32ClientTAUpdateValueInt,
+				psRGXKickTA3D2IN->ui32Client3DUpdateCount,
+				psClient3DUpdateSyncPrimBlockInt,
+				ui32Client3DUpdateSyncOffsetInt,
+				ui32Client3DUpdateValueInt,
+				psPRFenceUFOSyncPrimBlockInt,
+				psRGXKickTA3D2IN->ui32PRFenceUFOSyncOffset,
+				psRGXKickTA3D2IN->ui32PRFenceValue,
+				psRGXKickTA3D2IN->hCheckFence,
+				psRGXKickTA3D2IN->hUpdateTimeline,
+				&psRGXKickTA3D2OUT->hUpdateFence,
+				uiUpdateFenceNameInt,
+				psRGXKickTA3D2IN->hCheckFence3D,
+				psRGXKickTA3D2IN->hUpdateTimeline3D,
+				&psRGXKickTA3D2OUT->hUpdateFence3D,
+				uiUpdateFenceName3DInt,
+				psRGXKickTA3D2IN->ui32TACmdSize,
+				ui8TACmdInt,
+				psRGXKickTA3D2IN->ui323DPRCmdSize,
+				ui83DPRCmdInt,
+				psRGXKickTA3D2IN->ui323DCmdSize,
+				ui83DCmdInt,
+				psRGXKickTA3D2IN->ui32ExtJobRef,
+				psRGXKickTA3D2IN->bbKickTA,
+				psRGXKickTA3D2IN->bbKickPR,
+				psRGXKickTA3D2IN->bbKick3D,
+				psRGXKickTA3D2IN->bbAbort,
+				psRGXKickTA3D2IN->ui32PDumpFlags,
+				psKMHWRTDataSetInt,
+				psZSBufferInt,
+				psMSAAScratchBufferInt,
+				psRGXKickTA3D2IN->ui32SyncPMRCount,
+				ui32SyncPMRFlagsInt,
+				psSyncPMRsInt,
+				psRGXKickTA3D2IN->ui32RenderTargetSize,
+				psRGXKickTA3D2IN->ui32NumberOfDrawCalls,
+				psRGXKickTA3D2IN->ui32NumberOfIndices,
+				psRGXKickTA3D2IN->ui32NumberOfMRTs, psRGXKickTA3D2IN->ui64Deadline);
+
+RGXKickTA3D2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+
+	if (hClientTAFenceSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAFenceSyncPrimBlockInt && psClientTAFenceSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAFenceSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClientTAUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAUpdateSyncPrimBlockInt && psClientTAUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClient3DUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClient3DUpdateSyncPrimBlockInt && psClient3DUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClient3DUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPRFenceUFOSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPRFenceUFOSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psKMHWRTDataSetInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hKMHWRTDataSet,
+						    PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psZSBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hZSBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psMSAAScratchBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hMSAAScratchBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTA3D2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPropertyIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPropertyOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPropertyIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVRGXSetRenderContextPropertyKM(psRenderContextInt,
+						psRGXSetRenderContextPropertyIN->ui32Property,
+						psRGXSetRenderContextPropertyIN->ui64Input,
+						&psRGXSetRenderContextPropertyOUT->ui64Output);
+
+RGXSetRenderContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetIN_UI8,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *) IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetOUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sVHeapTableDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sPMMlistDevVAddrInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sMacrotileArrayDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRgnHeaderDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRTCDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet = psRGXCreateHWRTDataSetIN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSetIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSetIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+
+	{
+		sVHeapTableDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sVHeapTableDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psVHeapTableDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sPMMlistDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMMlistDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMMlistDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sMacrotileArrayDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sMacrotileArrayDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psMacrotileArrayDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sRgnHeaderDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRgnHeaderDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psRgnHeaderDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sRTCDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRTCDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psRTCDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSetOUT->eError =
+	    RGXCreateHWRTDataSet(psConnection, OSGetDevNode(psConnection),
+				 sVHeapTableDevVAddrInt,
+				 sPMMlistDevVAddrInt,
+				 psapsFreeListsInt,
+				 psRGXCreateHWRTDataSetIN->ui32PPPScreen,
+				 psRGXCreateHWRTDataSetIN->ui64MultiSampleCtl,
+				 psRGXCreateHWRTDataSetIN->ui64FlippedMultiSampleCtl,
+				 psRGXCreateHWRTDataSetIN->ui32TPCStride,
+				 sTailPtrsDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui32TPCSize,
+				 psRGXCreateHWRTDataSetIN->ui32TEScreen,
+				 psRGXCreateHWRTDataSetIN->ui32TEAA,
+				 psRGXCreateHWRTDataSetIN->ui32TEMTILE1,
+				 psRGXCreateHWRTDataSetIN->ui32TEMTILE2,
+				 psRGXCreateHWRTDataSetIN->ui32MTileStride,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeLowerX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeLowerY,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeUpperX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeUpperY,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeScaleX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeScaleY,
+				 sMacrotileArrayDevVAddrInt,
+				 sRgnHeaderDevVAddrInt,
+				 sRTCDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui32RgnHeaderSize,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMtileSize,
+				 psRGXCreateHWRTDataSetIN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSetOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateFreeListpsCleanupCookieIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyFreeList((RGX_FREELIST *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateFreeListIN_UI8,
+			      IMG_UINT8 * psRGXCreateFreeListOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *psRGXCreateFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *psRGXCreateFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListOUT_UI8, 0);
+
+	IMG_HANDLE hMemCtxPrivData = psRGXCreateFreeListIN->hMemCtxPrivData;
+	IMG_HANDLE hMemCtxPrivDataInt = NULL;
+	IMG_HANDLE hsGlobalFreeList = psRGXCreateFreeListIN->hsGlobalFreeList;
+	RGX_FREELIST *pssGlobalFreeListInt = NULL;
+	IMG_HANDLE hFreeListReservation = psRGXCreateFreeListIN->hFreeListReservation;
+	DEVMEMINT_RESERVATION *psFreeListReservationInt = NULL;
+	RGX_FREELIST *psCleanupCookieInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hMemCtxPrivDataInt,
+				       hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+		/* Look up the address from the handle */
+		psRGXCreateFreeListOUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&pssGlobalFreeListInt,
+					       hsGlobalFreeList,
+					       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+		if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXCreateFreeList_exit;
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psFreeListReservationInt,
+				       hFreeListReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError =
+	    RGXCreateFreeList(psConnection, OSGetDevNode(psConnection),
+			      hMemCtxPrivDataInt,
+			      psRGXCreateFreeListIN->ui32MaxFLPages,
+			      psRGXCreateFreeListIN->ui32InitFLPages,
+			      psRGXCreateFreeListIN->ui32GrowFLPages,
+			      psRGXCreateFreeListIN->ui32GrowParamThreshold,
+			      pssGlobalFreeListInt,
+			      psRGXCreateFreeListIN->bbFreeListCheck,
+			      psFreeListReservationInt, &psCleanupCookieInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateFreeListOUT->
+								   hCleanupCookie,
+								   (void *)psCleanupCookieInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateFreeListpsCleanupCookieIntRelease);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateFreeList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hMemCtxPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (pssGlobalFreeListInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hsGlobalFreeList,
+						    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psFreeListReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hFreeListReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateFreeListOUT->eError != PVRSRV_OK)
+	{
+		if (psCleanupCookieInt)
+		{
+			RGXDestroyFreeList(psCleanupCookieInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateRenderContextpsRenderContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRenderContextKM((RGX_SERVER_RENDER_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_RENDERCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RENDERCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXCreateRenderContextIN_UI8,
+				   IMG_UINT8 * psRGXCreateRenderContextOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateRenderContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticRenderContextStateInt = NULL;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateRenderContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize >
+	     RGXFWIF_STATIC_RENDERCONTEXT_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRenderContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRenderContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRenderContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8FrameworkCmd,
+		     psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize != 0)
+	{
+		ui8StaticRenderContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticRenderContextStateInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8StaticRenderContextState,
+		     psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVRGXCreateRenderContextKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXCreateRenderContextIN->i32Priority,
+					   psRGXCreateRenderContextIN->sVDMCallStackAddr,
+					   psRGXCreateRenderContextIN->ui32ui32CallStackDepth,
+					   psRGXCreateRenderContextIN->ui32FrameworkCmdSize,
+					   ui8FrameworkCmdInt,
+					   hPrivDataInt,
+					   psRGXCreateRenderContextIN->
+					   ui32StaticRenderContextStateSize,
+					   ui8StaticRenderContextStateInt,
+					   psRGXCreateRenderContextIN->ui32PackedCCBSizeU8888,
+					   psRGXCreateRenderContextIN->ui32ContextFlags,
+					   psRGXCreateRenderContextIN->ui64RobustnessAddress,
+					   psRGXCreateRenderContextIN->ui32MaxTADeadlineMS,
+					   psRGXCreateRenderContextIN->ui32Max3DDeadlineMS,
+					   &psRenderContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psRGXCreateRenderContextOUT->
+									hRenderContext,
+									(void *)psRenderContextInt,
+									PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_RGXCreateRenderContextpsRenderContextIntRelease);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRenderContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRenderContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRenderContextInt)
+		{
+			PVRSRVRGXDestroyRenderContextKM(psRenderContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRenderContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2IN_UI8,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2IN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2OUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sVHeapTableDevVAddrInt = NULL;
+	IMG_HANDLE hPMMlistsReservation = psRGXCreateHWRTDataSet2IN->hPMMlistsReservation;
+	DEVMEMINT_RESERVATION *psPMMlistsReservationInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sMacrotileArrayDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRgnHeaderDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRTCDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet = psRGXCreateHWRTDataSet2IN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSet2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSet2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+
+	{
+		sVHeapTableDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sVHeapTableDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psVHeapTableDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sMacrotileArrayDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sMacrotileArrayDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psMacrotileArrayDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sRgnHeaderDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRgnHeaderDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psRgnHeaderDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sRTCDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRTCDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psRTCDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMMlistsReservationInt,
+				       hPMMlistsReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    RGXCreateHWRTDataSet2(psConnection, OSGetDevNode(psConnection),
+				  sVHeapTableDevVAddrInt,
+				  psPMMlistsReservationInt,
+				  psapsFreeListsInt,
+				  psRGXCreateHWRTDataSet2IN->ui32PPPScreen,
+				  psRGXCreateHWRTDataSet2IN->ui64MultiSampleCtl,
+				  psRGXCreateHWRTDataSet2IN->ui64FlippedMultiSampleCtl,
+				  psRGXCreateHWRTDataSet2IN->ui32TPCStride,
+				  sTailPtrsDevVAddrInt,
+				  psRGXCreateHWRTDataSet2IN->ui32TPCSize,
+				  psRGXCreateHWRTDataSet2IN->ui32TEScreen,
+				  psRGXCreateHWRTDataSet2IN->ui32TEAA,
+				  psRGXCreateHWRTDataSet2IN->ui32TEMTILE1,
+				  psRGXCreateHWRTDataSet2IN->ui32TEMTILE2,
+				  psRGXCreateHWRTDataSet2IN->ui32MTileStride,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeLowerX,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeLowerY,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeUpperX,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeUpperY,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeScaleX,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeScaleY,
+				  sMacrotileArrayDevVAddrInt,
+				  sRgnHeaderDevVAddrInt,
+				  sRTCDevVAddrInt,
+				  psRGXCreateHWRTDataSet2IN->ui32RgnHeaderSize,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMtileSize,
+				  psRGXCreateHWRTDataSet2IN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMMlistsReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMMlistsReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSet2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+
+/*
+ * Register all RGXTA3D functions with services
+ */
+PVRSRV_ERROR InitRGXTA3DBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET,
+			      PVRSRVBridgeRGXDestroyHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER,
+			      PVRSRVBridgeRGXCreateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER,
+			      PVRSRVBridgeRGXDestroyZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXPopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXUnpopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST,
+			      PVRSRVBridgeRGXDestroyFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT,
+			      PVRSRVBridgeRGXDestroyRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE,
+			      PVRSRVBridgeRGXSendZSStoreDisable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetRenderContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED,
+			      PVRSRVBridgeRGXRenderContextStalled, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2,
+			      PVRSRVBridgeRGXKickTA3D2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKTA3D2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTA3D2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetRenderContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET,
+			      PVRSRVBridgeRGXCreateHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST,
+			      PVRSRVBridgeRGXCreateFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT,
+			      PVRSRVBridgeRGXCreateRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2,
+			      PVRSRVBridgeRGXCreateHWRTDataSet2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxta3d functions with services
+ */
+void DeinitRGXTA3DBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
new file mode 100644
index 000000000000..34d7c2c56a2c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTIMERQUERY_BRIDGE_H
+#define COMMON_RGXTIMERQUERY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST			(PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2)
+
+/*******************************************
+            RGXBeginTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY;
+
+/* Bridge out structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY;
+
+/*******************************************
+            RGXEndTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY;
+
+/* Bridge out structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY;
+
+/*******************************************
+            RGXQueryTimer
+ *******************************************/
+
+/* Bridge in structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_IN_RGXQUERYTIMER_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXQUERYTIMER;
+
+/* Bridge out structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXQUERYTIMER_TAG
+{
+	IMG_UINT64 ui64EndTime;
+	IMG_UINT64 ui64StartTime;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXQUERYTIMER;
+
+#endif /* COMMON_RGXTIMERQUERY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
new file mode 100644
index 000000000000..4af3cfc35cba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
@@ -0,0 +1,171 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtimerquery.h"
+
+#include "common_rgxtimerquery_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXBeginTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXBeginTimerQueryIN_UI8,
+			       IMG_UINT8 * psRGXBeginTimerQueryOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryOUT_UI8,
+								     0);
+
+	psRGXBeginTimerQueryOUT->eError =
+	    PVRSRVRGXBeginTimerQueryKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXBeginTimerQueryIN->ui32QueryId);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEndTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXEndTimerQueryIN_UI8,
+			     IMG_UINT8 * psRGXEndTimerQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *psRGXEndTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *psRGXEndTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEndTimerQueryIN);
+
+	psRGXEndTimerQueryOUT->eError =
+	    PVRSRVRGXEndTimerQueryKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXQueryTimer(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXQueryTimerIN_UI8,
+			  IMG_UINT8 * psRGXQueryTimerOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXQUERYTIMER *psRGXQueryTimerIN =
+	    (PVRSRV_BRIDGE_IN_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *psRGXQueryTimerOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerOUT_UI8, 0);
+
+	psRGXQueryTimerOUT->eError =
+	    PVRSRVRGXQueryTimerKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXQueryTimerIN->ui32QueryId,
+				  &psRGXQueryTimerOUT->ui64StartTime,
+				  &psRGXQueryTimerOUT->ui64EndTime);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+
+/*
+ * Register all RGXTIMERQUERY functions with services
+ */
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY,
+			      PVRSRVBridgeRGXBeginTimerQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY,
+			      PVRSRVBridgeRGXEndTimerQuery, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER, PVRSRVBridgeRGXQueryTimer,
+			      NULL, sizeof(PVRSRV_BRIDGE_IN_RGXQUERYTIMER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXQUERYTIMER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtimerquery functions with services
+ */
+void DeinitRGXTIMERQUERYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/common_rgxtq2_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/common_rgxtq2_bridge.h
new file mode 100644
index 000000000000..f20fca015dc0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/common_rgxtq2_bridge.h
@@ -0,0 +1,264 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ2_BRIDGE_H
+#define COMMON_RGXTQ2_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8)
+
+/*******************************************
+            RGXTDMCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXTDMNotifyWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXTDMSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXTDMSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXTDMSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ2_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/server_rgxtq2_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/server_rgxtq2_bridge.c
new file mode 100644
index 000000000000..1129b824d850
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/server_rgxtq2_bridge.c
@@ -0,0 +1,1610 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtdmtransfer.h"
+
+#include "common_rgxtq2_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXTDMCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMDestroyTransferContextKM((RGX_SERVER_TQ_TDM_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXTDMCreateTransferContextIN_UI8,
+					IMG_UINT8 * psRGXTDMCreateTransferContextOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXTDMCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXTDMCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMCreateTransferContextOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXTDMCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVRGXTDMCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+						psRGXTDMCreateTransferContextIN->i32Priority,
+						psRGXTDMCreateTransferContextIN->
+						ui32FrameworkCmdSize, ui8FrameworkCmdInt,
+						hPrivDataInt,
+						psRGXTDMCreateTransferContextIN->
+						ui32PackedCCBSizeU88,
+						psRGXTDMCreateTransferContextIN->ui32ContextFlags,
+						psRGXTDMCreateTransferContextIN->
+						ui64RobustnessAddress,
+						psRGXTDMCreateTransferContextIN->ui32MaxDeadlineMS,
+						&psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXTDMCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXTDMCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXTDMDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextIN_UI8,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMDestroyTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMDestroyTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	if (unlikely
+	    ((psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psRGXTDMDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						     psTransferContextInt,
+						     psRGXTDMSetTransferContextPriorityIN->
+						     i32Priority);
+
+RGXTDMSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateIN_UI8,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMNotifyWriteOffsetUpdateIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMNotifyWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMNotifyWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMNotifyWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMNotifyWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(psTransferContextInt,
+						  psRGXTDMNotifyWriteOffsetUpdateIN->
+						  ui32PDumpFlags);
+
+RGXTDMNotifyWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer2IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVRGXTDMSubmitTransferKM(psTransferContextInt,
+					 psRGXTDMSubmitTransfer2IN->ui32PDumpFlags,
+					 psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount,
+					 psUpdateUFOSyncPrimBlockInt,
+					 ui32UpdateSyncOffsetInt,
+					 ui32UpdateValueInt,
+					 psRGXTDMSubmitTransfer2IN->hCheckFenceFD,
+					 psRGXTDMSubmitTransfer2IN->hUpdateTimeline,
+					 &psRGXTDMSubmitTransfer2OUT->hUpdateFence,
+					 uiUpdateFenceNameInt,
+					 psRGXTDMSubmitTransfer2IN->ui32CommandSize,
+					 ui8FWCommandInt,
+					 psRGXTDMSubmitTransfer2IN->ui32ExternalJobReference,
+					 psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount,
+					 ui32SyncPMRFlagsInt,
+					 psSyncPMRsInt,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic1,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic2,
+					 psRGXTDMSubmitTransfer2IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTDMGetSharedMemoryIN);
+
+	psRGXTDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXTDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMGetSharedMemory_exit:
+
+	if (psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPropertyKM(psTransferContextInt,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui32Property,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui64Input,
+						     &psRGXTDMSetTransferContextPropertyOUT->
+						     ui64Output);
+
+RGXTDMSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer3IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVRGXTDMSubmitTransfer3KM(psTransferContextInt,
+					  psRGXTDMSubmitTransfer3IN->ui32PDumpFlags,
+					  psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount,
+					  psUpdateUFOSyncPrimBlockInt,
+					  ui32UpdateSyncOffsetInt,
+					  ui32UpdateValueInt,
+					  psRGXTDMSubmitTransfer3IN->hCheckFenceFD,
+					  psRGXTDMSubmitTransfer3IN->hUpdateTimeline,
+					  &psRGXTDMSubmitTransfer3OUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXTDMSubmitTransfer3IN->hExportFenceToSignal,
+					  psRGXTDMSubmitTransfer3IN->ui32CommandSize,
+					  ui8FWCommandInt,
+					  psRGXTDMSubmitTransfer3IN->ui32ExternalJobReference,
+					  psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount,
+					  ui32SyncPMRFlagsInt,
+					  psSyncPMRsInt,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic1,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic2,
+					  psRGXTDMSubmitTransfer3IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+
+/*
+ * Register all RGXTQ2 functions with services
+ */
+PVRSRV_ERROR InitRGXTQ2Bridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXTDMSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXTDMSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXTDMSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXTDMSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq2 functions with services
+ */
+void DeinitRGXTQ2Bridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/common_rgxtq_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/common_rgxtq_bridge.h
new file mode 100644
index 000000000000..31d90dc0f993
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/common_rgxtq_bridge.h
@@ -0,0 +1,245 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ_BRIDGE_H
+#define COMMON_RGXTQ_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+7)
+
+/*******************************************
+            RGXCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdize;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32ClientUpdateCount;
+	IMG_UINT32 *pui32CommandSize;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32TQPrepareFlags;
+	IMG_UINT32 **pui32UpdateSyncOffset;
+	IMG_UINT32 **pui32UpdateValue;
+	IMG_UINT8 **pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE **phUpdateUFOSyncPrimBlock;
+	PVRSRV_TIMELINE h2DUpdateTimeline;
+	PVRSRV_TIMELINE h3DUpdateTimeline;
+	PVRSRV_FENCE hCheckFenceFD;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PrepareCount;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE h2DUpdateFence;
+	PVRSRV_FENCE h3DUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTQGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTQGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTQGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTQReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTQReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTQReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32ClientUpdateCount;
+	IMG_UINT32 *pui32CommandSize;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32TQPrepareFlags;
+	IMG_UINT32 **pui32UpdateSyncOffset;
+	IMG_UINT32 **pui32UpdateValue;
+	IMG_UINT8 **pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE **phUpdateUFOSyncPrimBlock;
+	PVRSRV_TIMELINE h2DUpdateTimeline;
+	PVRSRV_TIMELINE h3DUpdateTimeline;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PrepareCount;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE h2DUpdateFence;
+	PVRSRV_FENCE h3DUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/server_rgxtq_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/server_rgxtq_bridge.c
new file mode 100644
index 000000000000..f50c1184676a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/server_rgxtq_bridge.c
@@ -0,0 +1,1954 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtransfer.h"
+#include "rgx_tq_shared.h"
+
+#include "common_rgxtq_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyTransferContextKM((RGX_SERVER_TQ_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateTransferContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateTransferContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT *psRGXCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT *psRGXCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateTransferContextIN->ui32FrameworkCmdize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateTransferContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateTransferContextIN->ui32FrameworkCmdize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVRGXCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXCreateTransferContextIN->i32Priority,
+					     psRGXCreateTransferContextIN->ui32FrameworkCmdize,
+					     ui8FrameworkCmdInt,
+					     hPrivDataInt,
+					     psRGXCreateTransferContextIN->ui32PackedCCBSizeU8888,
+					     psRGXCreateTransferContextIN->ui32ContextFlags,
+					     psRGXCreateTransferContextIN->ui64RobustnessAddress,
+					     &psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyTransferContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyTransferContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT *psRGXDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT *psRGXDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyTransferContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetTransferContextPriorityIN_UI8,
+					  IMG_UINT8 * psRGXSetTransferContextPriorityOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY *psRGXSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY *psRGXSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						  psTransferContextInt,
+						  psRGXSetTransferContextPriorityIN->i32Priority);
+
+RGXSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXSubmitTransfer2IN_UI8,
+			       IMG_UINT8 * psRGXSubmitTransfer2OUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2 *psRGXSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2 *psRGXSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer2OUT_UI8,
+								     0);
+
+	IMG_HANDLE hTransferContext = psRGXSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateCountInt = NULL;
+	SYNC_PRIMITIVE_BLOCK ***psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE **hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 **ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 **ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT32 *ui32CommandSizeInt = NULL;
+	IMG_UINT8 **ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32TQPrepareFlagsInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BYTE *pArrayArgsBuffer2 = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+	IMG_UINT32 ui32BufferSize2 = 0;
+	IMG_UINT32 ui32NextOffset2 = 0;
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount *
+		     sizeof(SYNC_PRIMITIVE_BLOCK **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_HANDLE **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT8 *));
+	}
+
+	if (unlikely(psRGXSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32ClientUpdateCountInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateCountInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32ClientUpdateCount,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer for first dimension */
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK ***) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(SYNC_PRIMITIVE_BLOCK **);
+		/* Assigning hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer for first dimension */
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_HANDLE);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateSyncOffsetInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateValueInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateValueInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32CommandSizeInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CommandSizeInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32CommandSize,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui8FWCommandInt to the right offset in the pool buffer for first dimension */
+		ui8FWCommandInt = (IMG_UINT8 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT8 *);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32TQPrepareFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32TQPrepareFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32TQPrepareFlags,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXSubmitTransfer2IN->phSyncPMRs,
+		     psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		ui64BufferSize = 0;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] *
+			     sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize += ((IMG_UINT64) ui32CommandSizeInt[i] * sizeof(IMG_UINT8));
+		}
+		if (ui64BufferSize > IMG_UINT32_MAX)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			goto RGXSubmitTransfer2_exit;
+		}
+		ui32BufferSize2 = (IMG_UINT32) ui64BufferSize;
+	}
+
+	if (ui32BufferSize2 != 0)
+	{
+		pArrayArgsBuffer2 = OSAllocMemNoStats(ui32BufferSize2);
+
+		if (!pArrayArgsBuffer2)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			if (ui32ClientUpdateCountInt[i] > PVRSRV_MAX_SYNCS)
+			{
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Assigning each psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer (this is the second dimension) */
+			psUpdateUFOSyncPrimBlockInt[i] =
+			    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer2,
+								      ui32NextOffset2);
+			OSCachedMemSet(psUpdateUFOSyncPrimBlockInt[i], 0,
+				       ui32ClientUpdateCountInt[i] *
+				       sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui32NextOffset2 +=
+			    ui32ClientUpdateCountInt[i] * sizeof(SYNC_PRIMITIVE_BLOCK *);
+			/* Assigning each hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer (this is the second dimension) */
+			hUpdateUFOSyncPrimBlockInt2[i] =
+			    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateSyncOffsetInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateSyncOffsetInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateValueInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateValueInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			if (ui32CommandSizeInt[i] > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE)
+			{
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Assigning each ui8FWCommandInt to the right offset in the pool buffer (this is the second dimension) */
+			ui8FWCommandInt[i] =
+			    (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32CommandSizeInt[i] * sizeof(IMG_UINT8);
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_HANDLE **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->
+			     phUpdateUFOSyncPrimBlock[i], sizeof(IMG_HANDLE **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (hUpdateUFOSyncPrimBlockInt2[i]),
+				     (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui32UpdateSyncOffset[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateSyncOffsetInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui32UpdateValue[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateValueInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT8 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui8FWCommand[i],
+			     sizeof(IMG_UINT8 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32CommandSizeInt[i] * sizeof(IMG_UINT8)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui8FWCommandInt[i]), (const void __user *)psPtr,
+				     (ui32CommandSizeInt[i] * sizeof(IMG_UINT8))) != PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+				/* Look up the address from the handle */
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+							       (void **)
+							       &psUpdateUFOSyncPrimBlockInt[i][j],
+							       hUpdateUFOSyncPrimBlockInt2[i][j],
+							       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+							       IMG_TRUE);
+				if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+				{
+					UnlockHandle(psConnection->psHandleBase);
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSubmitTransfer2OUT->eError =
+	    PVRSRVRGXSubmitTransferKM(psTransferContextInt,
+				      psRGXSubmitTransfer2IN->ui32PrepareCount,
+				      ui32ClientUpdateCountInt,
+				      psUpdateUFOSyncPrimBlockInt,
+				      ui32UpdateSyncOffsetInt,
+				      ui32UpdateValueInt,
+				      psRGXSubmitTransfer2IN->hCheckFenceFD,
+				      psRGXSubmitTransfer2IN->h2DUpdateTimeline,
+				      &psRGXSubmitTransfer2OUT->h2DUpdateFence,
+				      psRGXSubmitTransfer2IN->h3DUpdateTimeline,
+				      &psRGXSubmitTransfer2OUT->h3DUpdateFence,
+				      uiUpdateFenceNameInt,
+				      ui32CommandSizeInt,
+				      ui8FWCommandInt,
+				      ui32TQPrepareFlagsInt,
+				      psRGXSubmitTransfer2IN->ui32ExtJobRef,
+				      psRGXSubmitTransfer2IN->ui32SyncPMRCount,
+				      ui32SyncPMRFlagsInt, psSyncPMRsInt);
+
+RGXSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+
+				/* Unreference the previously looked up handle */
+				if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i]
+				    && psUpdateUFOSyncPrimBlockInt[i][j])
+				{
+					PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+								    hUpdateUFOSyncPrimBlockInt2[i]
+								    [j],
+								    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+				}
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize2 == ui32NextOffset2);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (pArrayArgsBuffer2)
+		OSFreeMemNoStats(pArrayArgsBuffer2);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTQGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTQReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTQGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXTQGetSharedMemoryIN_UI8,
+				 IMG_UINT8 * psRGXTQGetSharedMemoryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY *psRGXTQGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY *) IMG_OFFSET_ADDR(psRGXTQGetSharedMemoryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY *psRGXTQGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTQGetSharedMemoryIN);
+
+	psRGXTQGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTQGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection), &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTQGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTQGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRGXTQGetSharedMemoryOUT->
+								      hCLIPMRMem,
+								      (void *)psCLIPMRMemInt,
+								      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RGXTQGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTQGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTQGetSharedMemory_exit:
+
+	if (psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTQReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTQReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXTQReleaseSharedMemoryIN_UI8,
+				     IMG_UINT8 * psRGXTQReleaseSharedMemoryOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY *psRGXTQReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY *psRGXTQReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQReleaseSharedMemoryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTQReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTQReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTQReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTQReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTQReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetTransferContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetTransferContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY *psRGXSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY *psRGXSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXSetTransferContextPropertyKM(psTransferContextInt,
+						  psRGXSetTransferContextPropertyIN->ui32Property,
+						  psRGXSetTransferContextPropertyIN->ui64Input,
+						  &psRGXSetTransferContextPropertyOUT->ui64Output);
+
+RGXSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXSubmitTransfer3IN_UI8,
+			       IMG_UINT8 * psRGXSubmitTransfer3OUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3 *psRGXSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3 *psRGXSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer3OUT_UI8,
+								     0);
+
+	IMG_HANDLE hTransferContext = psRGXSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateCountInt = NULL;
+	SYNC_PRIMITIVE_BLOCK ***psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE **hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 **ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 **ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT32 *ui32CommandSizeInt = NULL;
+	IMG_UINT8 **ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32TQPrepareFlagsInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BYTE *pArrayArgsBuffer2 = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+	IMG_UINT32 ui32BufferSize2 = 0;
+	IMG_UINT32 ui32NextOffset2 = 0;
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount *
+		     sizeof(SYNC_PRIMITIVE_BLOCK **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_HANDLE **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT8 *));
+	}
+
+	if (unlikely(psRGXSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32ClientUpdateCountInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateCountInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32ClientUpdateCount,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer for first dimension */
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK ***) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(SYNC_PRIMITIVE_BLOCK **);
+		/* Assigning hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer for first dimension */
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_HANDLE);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateSyncOffsetInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateValueInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateValueInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32CommandSizeInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CommandSizeInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32CommandSize,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui8FWCommandInt to the right offset in the pool buffer for first dimension */
+		ui8FWCommandInt = (IMG_UINT8 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT8 *);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32TQPrepareFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32TQPrepareFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32TQPrepareFlags,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXSubmitTransfer3IN->phSyncPMRs,
+		     psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		ui64BufferSize = 0;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] *
+			     sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize += ((IMG_UINT64) ui32CommandSizeInt[i] * sizeof(IMG_UINT8));
+		}
+		if (ui64BufferSize > IMG_UINT32_MAX)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			goto RGXSubmitTransfer3_exit;
+		}
+		ui32BufferSize2 = (IMG_UINT32) ui64BufferSize;
+	}
+
+	if (ui32BufferSize2 != 0)
+	{
+		pArrayArgsBuffer2 = OSAllocMemNoStats(ui32BufferSize2);
+
+		if (!pArrayArgsBuffer2)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			if (ui32ClientUpdateCountInt[i] > PVRSRV_MAX_SYNCS)
+			{
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Assigning each psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer (this is the second dimension) */
+			psUpdateUFOSyncPrimBlockInt[i] =
+			    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer2,
+								      ui32NextOffset2);
+			OSCachedMemSet(psUpdateUFOSyncPrimBlockInt[i], 0,
+				       ui32ClientUpdateCountInt[i] *
+				       sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui32NextOffset2 +=
+			    ui32ClientUpdateCountInt[i] * sizeof(SYNC_PRIMITIVE_BLOCK *);
+			/* Assigning each hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer (this is the second dimension) */
+			hUpdateUFOSyncPrimBlockInt2[i] =
+			    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateSyncOffsetInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateSyncOffsetInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateValueInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateValueInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			if (ui32CommandSizeInt[i] > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE)
+			{
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Assigning each ui8FWCommandInt to the right offset in the pool buffer (this is the second dimension) */
+			ui8FWCommandInt[i] =
+			    (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32CommandSizeInt[i] * sizeof(IMG_UINT8);
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_HANDLE **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->
+			     phUpdateUFOSyncPrimBlock[i], sizeof(IMG_HANDLE **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (hUpdateUFOSyncPrimBlockInt2[i]),
+				     (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui32UpdateSyncOffset[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateSyncOffsetInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui32UpdateValue[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateValueInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT8 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui8FWCommand[i],
+			     sizeof(IMG_UINT8 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32CommandSizeInt[i] * sizeof(IMG_UINT8)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui8FWCommandInt[i]), (const void __user *)psPtr,
+				     (ui32CommandSizeInt[i] * sizeof(IMG_UINT8))) != PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+				/* Look up the address from the handle */
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+							       (void **)
+							       &psUpdateUFOSyncPrimBlockInt[i][j],
+							       hUpdateUFOSyncPrimBlockInt2[i][j],
+							       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+							       IMG_TRUE);
+				if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+				{
+					UnlockHandle(psConnection->psHandleBase);
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSubmitTransfer3OUT->eError =
+	    PVRSRVRGXSubmitTransfer3KM(psTransferContextInt,
+				       psRGXSubmitTransfer3IN->ui32PrepareCount,
+				       ui32ClientUpdateCountInt,
+				       psUpdateUFOSyncPrimBlockInt,
+				       ui32UpdateSyncOffsetInt,
+				       ui32UpdateValueInt,
+				       psRGXSubmitTransfer3IN->hCheckFenceFD,
+				       psRGXSubmitTransfer3IN->h2DUpdateTimeline,
+				       &psRGXSubmitTransfer3OUT->h2DUpdateFence,
+				       psRGXSubmitTransfer3IN->h3DUpdateTimeline,
+				       &psRGXSubmitTransfer3OUT->h3DUpdateFence,
+				       uiUpdateFenceNameInt,
+				       psRGXSubmitTransfer3IN->hExportFenceToSignal,
+				       ui32CommandSizeInt,
+				       ui8FWCommandInt,
+				       ui32TQPrepareFlagsInt,
+				       psRGXSubmitTransfer3IN->ui32ExtJobRef,
+				       psRGXSubmitTransfer3IN->ui32SyncPMRCount,
+				       ui32SyncPMRFlagsInt, psSyncPMRsInt);
+
+RGXSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+
+				/* Unreference the previously looked up handle */
+				if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i]
+				    && psUpdateUFOSyncPrimBlockInt[i][j])
+				{
+					PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+								    hUpdateUFOSyncPrimBlockInt2[i]
+								    [j],
+								    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+				}
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize2 == ui32NextOffset2);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (pArrayArgsBuffer2)
+		OSFreeMemNoStats(pArrayArgsBuffer2);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXTQ_BRIDGE */
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+PVRSRV_ERROR InitRGXTQBridge(void);
+void DeinitRGXTQBridge(void);
+
+/*
+ * Register all RGXTQ functions with services
+ */
+PVRSRV_ERROR InitRGXTQBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+			      PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTQGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTQReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+			      PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq functions with services
+ */
+void DeinitRGXTQBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+				PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+				PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3);
+
+}
+#else /* SUPPORT_RGXTQ_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXTQ_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXTQBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXTQBridge()
+
+#endif /* SUPPORT_RGXTQ_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_bridge.h
new file mode 100644
index 000000000000..c419e21e9d07
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_bridge.h
@@ -0,0 +1,88 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RI_BRIDGE_H
+#define CLIENT_RI_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_ri_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner);
+
+#endif /* CLIENT_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_direct_bridge.c
new file mode 100644
index 000000000000..de037ee78be5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_direct_bridge.c
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for ri
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_ri_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "ri_typedefs.h"
+
+#include "ri_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError =
+	    RIWriteMEMDESCEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psPMRHandleInt,
+				  ui32TextBSize,
+				  puiTextB, ui64Offset, ui64Size, ui64Flags, &psRIHandleInt);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	eError =
+	    RIWriteProcListEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   ui32TextBSize, puiTextB, ui64Size, ui64DevVAddr, &psRIHandleInt);
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIUpdateMEMDESCAddrKM(psRIHandleInt, sAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIDeleteMEMDESCEntryKM(psRIHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIDumpListKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpAllKM();
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpProcessKM(ui32Pid);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRHandleInt, ui32Owner);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/common_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/common_ri_bridge.h
new file mode 100644
index 000000000000..baa41adccdfe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/common_ri_bridge.h
@@ -0,0 +1,224 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RI_BRIDGE_H
+#define COMMON_RI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "ri_typedefs.h"
+
+#define PVRSRV_BRIDGE_RI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR			PVRSRV_BRIDGE_RI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RI_RIDUMPLIST			PVRSRV_BRIDGE_RI_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RI_RIDUMPALL			PVRSRV_BRIDGE_RI_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RI_RIDUMPPROCESS			PVRSRV_BRIDGE_RI_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER			PVRSRV_BRIDGE_RI_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RI_CMD_LAST			(PVRSRV_BRIDGE_RI_CMD_FIRST+8)
+
+/*******************************************
+            RIWritePMREntry
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY;
+
+/* Bridge out structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY;
+
+/*******************************************
+            RIWriteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_UINT64 ui64Flags;
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRHandle;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY;
+
+/* Bridge out structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY;
+
+/*******************************************
+            RIWriteProcListEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_UINT64 ui64DevVAddr;
+	IMG_UINT64 ui64Size;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY;
+
+/* Bridge out structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY;
+
+/*******************************************
+            RIUpdateMEMDESCAddr
+ *******************************************/
+
+/* Bridge in structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR_TAG
+{
+	IMG_DEV_VIRTADDR sAddr;
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR;
+
+/* Bridge out structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR;
+
+/*******************************************
+            RIDeleteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY;
+
+/* Bridge out structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY;
+
+/*******************************************
+            RIDumpList
+ *******************************************/
+
+/* Bridge in structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPLIST_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPLIST;
+
+/* Bridge out structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPLIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPLIST;
+
+/*******************************************
+            RIDumpAll
+ *******************************************/
+
+/* Bridge in structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPALL_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPALL;
+
+/* Bridge out structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPALL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPALL;
+
+/*******************************************
+            RIDumpProcess
+ *******************************************/
+
+/* Bridge in structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPPROCESS_TAG
+{
+	IMG_PID ui32Pid;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPPROCESS;
+
+/* Bridge out structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPPROCESS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPPROCESS;
+
+/*******************************************
+            RIWritePMREntryWithOwner
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	IMG_HANDLE hPMRHandle;
+	IMG_PID ui32Owner;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER;
+
+/* Bridge out structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER;
+
+#endif /* COMMON_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/server_ri_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/server_ri_bridge.c
new file mode 100644
index 000000000000..f8a54d70ef0c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/server_ri_bridge.c
@@ -0,0 +1,761 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "ri_server.h"
+
+#include "common_ri_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntry(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRIWritePMREntryIN_UI8,
+			    IMG_UINT8 * psRIWritePMREntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *psRIWritePMREntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *psRIWritePMREntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryOUT->eError = RIWritePMREntryKM(psPMRHandleInt);
+
+RIWritePMREntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteMEMDESCEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIWriteMEMDESCEntryIN_UI8,
+				IMG_UINT8 * psRIWriteMEMDESCEntryOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMRHandle = psRIWriteMEMDESCEntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteMEMDESCEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteMEMDESCEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteMEMDESCEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteMEMDESCEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteMEMDESCEntryIN->puiTextB,
+		     psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteMEMDESCEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWriteMEMDESCEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError =
+	    RIWriteMEMDESCEntryKM(psConnection, OSGetDevNode(psConnection),
+				  psPMRHandleInt,
+				  psRIWriteMEMDESCEntryIN->ui32TextBSize,
+				  uiTextBInt,
+				  psRIWriteMEMDESCEntryIN->ui64Offset,
+				  psRIWriteMEMDESCEntryIN->ui64Size,
+				  psRIWriteMEMDESCEntryIN->ui64Flags, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRIWriteMEMDESCEntryOUT->
+								     hRIHandle,
+								     (void *)psRIHandleInt,
+								     PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RIWriteMEMDESCEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteMEMDESCEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteMEMDESCEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteProcListEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteProcListEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIWriteProcListEntryIN_UI8,
+				 IMG_UINT8 * psRIWriteProcListEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *) IMG_OFFSET_ADDR(psRIWriteProcListEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *)
+	    IMG_OFFSET_ADDR(psRIWriteProcListEntryOUT_UI8, 0);
+
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteProcListEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteProcListEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteProcListEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteProcListEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteProcListEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteProcListEntryIN->puiTextB,
+		     psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteProcListEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psRIWriteProcListEntryOUT->eError =
+	    RIWriteProcListEntryKM(psConnection, OSGetDevNode(psConnection),
+				   psRIWriteProcListEntryIN->ui32TextBSize,
+				   uiTextBInt,
+				   psRIWriteProcListEntryIN->ui64Size,
+				   psRIWriteProcListEntryIN->ui64DevVAddr, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteProcListEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRIWriteProcListEntryOUT->
+								      hRIHandle,
+								      (void *)psRIHandleInt,
+								      PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RIWriteProcListEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteProcListEntry_exit:
+
+	if (psRIWriteProcListEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteProcListEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIUpdateMEMDESCAddr(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrIN_UI8,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrIN =
+	    (PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrOUT_UI8,
+								      0);
+
+	IMG_HANDLE hRIHandle = psRIUpdateMEMDESCAddrIN->hRIHandle;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRIHandleInt,
+				       hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE, IMG_TRUE);
+	if (unlikely(psRIUpdateMEMDESCAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIUpdateMEMDESCAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    RIUpdateMEMDESCAddrKM(psRIHandleInt, psRIUpdateMEMDESCAddrIN->sAddr);
+
+RIUpdateMEMDESCAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRIHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDeleteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryIN_UI8,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *)
+	    IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIDeleteMEMDESCEntryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRIDeleteMEMDESCEntryIN->hRIHandle,
+					      PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	if (unlikely((psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_OK) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRIDeleteMEMDESCEntryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDeleteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIDeleteMEMDESCEntry_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpList(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRIDumpListIN_UI8,
+		       IMG_UINT8 * psRIDumpListOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPLIST *psRIDumpListIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPLIST *psRIDumpListOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIDumpListIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIDumpListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIDumpListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDumpList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIDumpListOUT->eError = RIDumpListKM(psPMRHandleInt);
+
+RIDumpList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpAll(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psRIDumpAllIN_UI8,
+		      IMG_UINT8 * psRIDumpAllOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPALL *psRIDumpAllIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPALL *psRIDumpAllOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psRIDumpAllIN);
+
+	psRIDumpAllOUT->eError = RIDumpAllKM();
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpProcess(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRIDumpProcessIN_UI8,
+			  IMG_UINT8 * psRIDumpProcessOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPPROCESS *psRIDumpProcessIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *psRIDumpProcessOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psRIDumpProcessOUT->eError = RIDumpProcessKM(psRIDumpProcessIN->ui32Pid);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntryWithOwner(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerIN_UI8,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryWithOwnerIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryWithOwnerOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntryWithOwner_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    RIWritePMREntryWithOwnerKM(psPMRHandleInt, psRIWritePMREntryWithOwnerIN->ui32Owner);
+
+RIWritePMREntryWithOwner_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+
+/*
+ * Register all RI functions with services
+ */
+PVRSRV_ERROR InitRIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY,
+			      PVRSRVBridgeRIWritePMREntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY,
+			      PVRSRVBridgeRIWriteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY,
+			      PVRSRVBridgeRIWriteProcListEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR,
+			      PVRSRVBridgeRIUpdateMEMDESCAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY,
+			      PVRSRVBridgeRIDeleteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST, PVRSRVBridgeRIDumpList,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPLIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPLIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL, PVRSRVBridgeRIDumpAll,
+			      NULL, 0, sizeof(PVRSRV_BRIDGE_OUT_RIDUMPALL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS,
+			      PVRSRVBridgeRIDumpProcess, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPPROCESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPPROCESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER,
+			      PVRSRVBridgeRIWritePMREntryWithOwner, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all ri functions with services
+ */
+void DeinitRIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/common_smm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/common_smm_bridge.h
new file mode 100644
index 000000000000..7d6603dfbe8b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/common_smm_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SMM_BRIDGE_H
+#define COMMON_SMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SMM_CMD_LAST			(PVRSRV_BRIDGE_SMM_CMD_FIRST+2)
+
+/*******************************************
+            PMRSecureExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR;
+
+/* Bridge out structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR;
+
+/*******************************************
+            PMRSecureUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR;
+
+/* Bridge out structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR;
+
+/*******************************************
+            PMRSecureImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR;
+
+/* Bridge out structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR;
+
+#endif /* COMMON_SMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/server_smm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/server_smm_bridge.c
new file mode 100644
index 000000000000..5641b1ea12f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/server_smm_bridge.c
@@ -0,0 +1,281 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "secure_export.h"
+
+#include "common_smm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRSecureExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureExportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureExportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *psPMRSecureExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *psPMRSecureExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMROUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRSecureExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR *psPMROutInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRSecureExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRSecureExportPMROUT->eError =
+	    PMRSecureExportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRInt,
+			       &psPMRSecureExportPMROUT->Export, &psPMROutInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureExportPMR_exit;
+	}
+
+PMRSecureExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRSecureExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMROutInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRSecureUnexportPMR(psPMROutInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRSecureUnexportPMRIN_UI8,
+				 IMG_UINT8 * psPMRSecureUnexportPMROUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureUnexportPMRIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *)
+	    IMG_OFFSET_ADDR(psPMRSecureUnexportPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRSecureUnexportPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT);
+	if (unlikely((psPMRSecureUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRSecureUnexportPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureUnexportPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRSecureImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureImportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureImportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *psPMRSecureImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *psPMRSecureImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMROUT_UI8,
+								     0);
+
+	PMR *psPMRInt = NULL;
+
+	psPMRSecureImportPMROUT->eError =
+	    PMRSecureImportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRSecureImportPMRIN->Export,
+			       &psPMRInt,
+			       &psPMRSecureImportPMROUT->uiSize, &psPMRSecureImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psPMRSecureImportPMROUT->hPMR,
+								    (void *)psPMRInt,
+								    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _PMRSecureImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureImportPMR_exit:
+
+	if (psPMRSecureImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+
+/*
+ * Register all SMM functions with services
+ */
+PVRSRV_ERROR InitSMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR,
+			      PVRSRVBridgePMRSecureExportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR,
+			      PVRSRVBridgePMRSecureUnexportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR,
+			      PVRSRVBridgePMRSecureImportPMR, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all smm functions with services
+ */
+void DeinitSMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/common_srvcore_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/common_srvcore_bridge.h
new file mode 100644
index 000000000000..603ba5b1d89e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/common_srvcore_bridge.h
@@ -0,0 +1,369 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SRVCORE_BRIDGE_H
+#define COMMON_SRVCORE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_device_types.h"
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_SRVCORE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SRVCORE_CONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SRVCORE_DISCONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16
+#define PVRSRV_BRIDGE_SRVCORE_CMD_LAST			(PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16)
+
+/*******************************************
+            Connect
+ *******************************************/
+
+/* Bridge in structure for Connect */
+typedef struct PVRSRV_BRIDGE_IN_CONNECT_TAG
+{
+	IMG_UINT32 ui32ClientBuildOptions;
+	IMG_UINT32 ui32ClientDDKBuild;
+	IMG_UINT32 ui32ClientDDKVersion;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_CONNECT;
+
+/* Bridge out structure for Connect */
+typedef struct PVRSRV_BRIDGE_OUT_CONNECT_TAG
+{
+	IMG_UINT64 ui64PackedBvnc;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CapabilityFlags;
+	IMG_UINT8 ui8KernelArch;
+} __packed PVRSRV_BRIDGE_OUT_CONNECT;
+
+/*******************************************
+            Disconnect
+ *******************************************/
+
+/* Bridge in structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_IN_DISCONNECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DISCONNECT;
+
+/* Bridge out structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_OUT_DISCONNECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DISCONNECT;
+
+/*******************************************
+            AcquireGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT;
+
+/* Bridge out structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT;
+
+/*******************************************
+            ReleaseGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+} __packed PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT;
+
+/* Bridge out structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT;
+
+/*******************************************
+            EventObjectOpen
+ *******************************************/
+
+/* Bridge in structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hEventObject;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN;
+
+/* Bridge out structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN;
+
+/*******************************************
+            EventObjectWait
+ *******************************************/
+
+/* Bridge in structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT;
+
+/* Bridge out structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT;
+
+/*******************************************
+            EventObjectClose
+ *******************************************/
+
+/* Bridge in structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE;
+
+/* Bridge out structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE;
+
+/*******************************************
+            DumpDebugInfo
+ *******************************************/
+
+/* Bridge in structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_IN_DUMPDEBUGINFO_TAG
+{
+	IMG_UINT32 ui32VerbLevel;
+} __packed PVRSRV_BRIDGE_IN_DUMPDEBUGINFO;
+
+/* Bridge out structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO;
+
+/*******************************************
+            GetDevClockSpeed
+ *******************************************/
+
+/* Bridge in structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED;
+
+/* Bridge out structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ClockSpeed;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED;
+
+/*******************************************
+            HWOpTimeout
+ *******************************************/
+
+/* Bridge in structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_IN_HWOPTIMEOUT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HWOPTIMEOUT;
+
+/* Bridge out structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_HWOPTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HWOPTIMEOUT;
+
+/*******************************************
+            AlignmentCheck
+ *******************************************/
+
+/* Bridge in structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_IN_ALIGNMENTCHECK_TAG
+{
+	IMG_UINT32 *pui32AlignChecks;
+	IMG_UINT32 ui32AlignChecksSize;
+} __packed PVRSRV_BRIDGE_IN_ALIGNMENTCHECK;
+
+/* Bridge out structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK;
+
+/*******************************************
+            GetDeviceStatus
+ *******************************************/
+
+/* Bridge in structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVICESTATUS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVICESTATUS;
+
+/* Bridge out structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVICESTATUS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceSatus;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVICESTATUS;
+
+/*******************************************
+            GetMultiCoreInfo
+ *******************************************/
+
+/* Bridge in structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_IN_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	IMG_UINT32 ui32CapsSize;
+} __packed PVRSRV_BRIDGE_IN_GETMULTICOREINFO;
+
+/* Bridge out structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_OUT_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumCores;
+} __packed PVRSRV_BRIDGE_OUT_GETMULTICOREINFO;
+
+/*******************************************
+            EventObjectWaitTimeout
+ *******************************************/
+
+/* Bridge in structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	IMG_UINT64 ui64uiTimeoutus;
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT;
+
+/* Bridge out structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT;
+
+/*******************************************
+            FindProcessMemStats
+ *******************************************/
+
+/* Bridge in structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	IMG_UINT32 ui32ArrSize;
+	IMG_UINT32 ui32PID;
+	IMG_BOOL bbAllProcessStats;
+} __packed PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS;
+
+/* Bridge out structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS;
+
+/*******************************************
+            AcquireInfoPage
+ *******************************************/
+
+/* Bridge in structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE;
+
+/* Bridge out structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE;
+
+/*******************************************
+            ReleaseInfoPage
+ *******************************************/
+
+/* Bridge in structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_RELEASEINFOPAGE;
+
+/* Bridge out structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE;
+
+#endif /* COMMON_SRVCORE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/server_srvcore_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/server_srvcore_bridge.c
new file mode 100644
index 000000000000..d40e53b244f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/server_srvcore_bridge.c
@@ -0,0 +1,1080 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "srvcore.h"
+#include "info_page.h"
+#include "proc_stats.h"
+#include "rgx_fwif_alignchecks.h"
+
+#include "common_srvcore_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeConnect(IMG_UINT32 ui32DispatchTableEntry,
+		    IMG_UINT8 * psConnectIN_UI8,
+		    IMG_UINT8 * psConnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CONNECT *psConnectIN =
+	    (PVRSRV_BRIDGE_IN_CONNECT *) IMG_OFFSET_ADDR(psConnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CONNECT *psConnectOUT =
+	    (PVRSRV_BRIDGE_OUT_CONNECT *) IMG_OFFSET_ADDR(psConnectOUT_UI8, 0);
+
+	psConnectOUT->eError =
+	    PVRSRVConnectKM(psConnection, OSGetDevNode(psConnection),
+			    psConnectIN->ui32Flags,
+			    psConnectIN->ui32ClientBuildOptions,
+			    psConnectIN->ui32ClientDDKVersion,
+			    psConnectIN->ui32ClientDDKBuild,
+			    &psConnectOUT->ui8KernelArch,
+			    &psConnectOUT->ui32CapabilityFlags, &psConnectOUT->ui64PackedBvnc);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDisconnect(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDisconnectIN_UI8,
+		       IMG_UINT8 * psDisconnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DISCONNECT *psDisconnectIN =
+	    (PVRSRV_BRIDGE_IN_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DISCONNECT *psDisconnectOUT =
+	    (PVRSRV_BRIDGE_OUT_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDisconnectIN);
+
+	psDisconnectOUT->eError = PVRSRVDisconnectKM();
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireGlobalEventObjecthGlobalEventObjectIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseGlobalEventObjectKM((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psAcquireGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psAcquireGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectOUT_UI8, 0);
+
+	IMG_HANDLE hGlobalEventObjectInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireGlobalEventObjectIN);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAcquireGlobalEventObjectKM(&hGlobalEventObjectInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psAcquireGlobalEventObjectOUT->hGlobalEventObject,
+				      (void *)hGlobalEventObjectInt,
+				      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _AcquireGlobalEventObjecthGlobalEventObjectIntRelease);
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AcquireGlobalEventObject_exit:
+
+	if (psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK)
+	{
+		if (hGlobalEventObjectInt)
+		{
+			PVRSRVReleaseGlobalEventObjectKM(hGlobalEventObjectInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psReleaseGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psReleaseGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psReleaseGlobalEventObjectOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psReleaseGlobalEventObjectIN->
+					      hGlobalEventObject,
+					      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	if (unlikely
+	    ((psReleaseGlobalEventObjectOUT->eError != PVRSRV_OK)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psReleaseGlobalEventObjectOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto ReleaseGlobalEventObject_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+ReleaseGlobalEventObject_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _EventObjectOpenhOSEventIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = OSEventObjectClose((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectOpen(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectOpenIN_UI8,
+			    IMG_UINT8 * psEventObjectOpenOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *psEventObjectOpenIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *psEventObjectOpenOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenOUT_UI8, 0);
+
+	IMG_HANDLE hEventObject = psEventObjectOpenIN->hEventObject;
+	IMG_HANDLE hEventObjectInt = NULL;
+	IMG_HANDLE hOSEventInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectOpenOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hEventObjectInt,
+				       hEventObject,
+				       PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT, IMG_TRUE);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = OSEventObjectOpen(hEventObjectInt, &hOSEventInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		goto EventObjectOpen_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psEventObjectOpenOUT->hOSEvent,
+								 (void *)hOSEventInt,
+								 PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _EventObjectOpenhOSEventIntRelease);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectOpen_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hEventObjectInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hEventObject, PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psEventObjectOpenOUT->eError != PVRSRV_OK)
+	{
+		if (hOSEventInt)
+		{
+			OSEventObjectClose(hOSEventInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectWaitIN_UI8,
+			    IMG_UINT8 * psEventObjectWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *psEventObjectWaitIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *psEventObjectWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitOUT->eError = OSEventObjectWait(hOSEventKMInt);
+
+EventObjectWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectClose(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psEventObjectCloseIN_UI8,
+			     IMG_UINT8 * psEventObjectCloseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *psEventObjectCloseIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *psEventObjectCloseOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectCloseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psEventObjectCloseIN->hOSEventKM,
+					      PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	if (unlikely((psEventObjectCloseOUT->eError != PVRSRV_OK) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psEventObjectCloseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectClose_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectClose_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDumpDebugInfo(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDumpDebugInfoIN_UI8,
+			  IMG_UINT8 * psDumpDebugInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *psDumpDebugInfoIN =
+	    (PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *psDumpDebugInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoOUT_UI8, 0);
+
+	psDumpDebugInfoOUT->eError =
+	    PVRSRVDumpDebugInfoKM(psConnection, OSGetDevNode(psConnection),
+				  psDumpDebugInfoIN->ui32VerbLevel);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDevClockSpeed(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetDevClockSpeedIN_UI8,
+			     IMG_UINT8 * psGetDevClockSpeedOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *psGetDevClockSpeedIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *psGetDevClockSpeedOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDevClockSpeedIN);
+
+	psGetDevClockSpeedOUT->eError =
+	    PVRSRVGetDevClockSpeedKM(psConnection, OSGetDevNode(psConnection),
+				     &psGetDevClockSpeedOUT->ui32ClockSpeed);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHWOpTimeout(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psHWOpTimeoutIN_UI8,
+			IMG_UINT8 * psHWOpTimeoutOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HWOPTIMEOUT *psHWOpTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *psHWOpTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHWOpTimeoutIN);
+
+	psHWOpTimeoutOUT->eError = PVRSRVHWOpTimeoutKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static_assert(RGXFW_ALIGN_CHECKS_UM_MAX <= IMG_UINT32_MAX,
+	      "RGXFW_ALIGN_CHECKS_UM_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeAlignmentCheck(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psAlignmentCheckIN_UI8,
+			   IMG_UINT8 * psAlignmentCheckOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *psAlignmentCheckIN =
+	    (PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *psAlignmentCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AlignChecksInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psAlignmentCheckIN->ui32AlignChecksSize > RGXFW_ALIGN_CHECKS_UM_MAX))
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto AlignmentCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto AlignmentCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psAlignmentCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psAlignmentCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psAlignmentCheckOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto AlignmentCheck_exit;
+			}
+		}
+	}
+
+	if (psAlignmentCheckIN->ui32AlignChecksSize != 0)
+	{
+		ui32AlignChecksInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AlignChecksInt,
+		     (const void __user *)psAlignmentCheckIN->pui32AlignChecks,
+		     psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psAlignmentCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto AlignmentCheck_exit;
+		}
+	}
+
+	psAlignmentCheckOUT->eError =
+	    PVRSRVAlignmentCheckKM(psConnection, OSGetDevNode(psConnection),
+				   psAlignmentCheckIN->ui32AlignChecksSize, ui32AlignChecksInt);
+
+AlignmentCheck_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psAlignmentCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDeviceStatus(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psGetDeviceStatusIN_UI8,
+			    IMG_UINT8 * psGetDeviceStatusOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVICESTATUS *psGetDeviceStatusIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *psGetDeviceStatusOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDeviceStatusIN);
+
+	psGetDeviceStatusOUT->eError =
+	    PVRSRVGetDeviceStatusKM(psConnection, OSGetDevNode(psConnection),
+				    &psGetDeviceStatusOUT->ui32DeviceSatus);
+
+	return 0;
+}
+
+static_assert(8 <= IMG_UINT32_MAX, "8 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeGetMultiCoreInfo(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetMultiCoreInfoIN_UI8,
+			     IMG_UINT8 * psGetMultiCoreInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETMULTICOREINFO *psGetMultiCoreInfoIN =
+	    (PVRSRV_BRIDGE_IN_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *psGetMultiCoreInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoOUT_UI8, 0);
+
+	IMG_UINT64 *pui64CapsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize > 8)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	psGetMultiCoreInfoOUT->pui64Caps = psGetMultiCoreInfoIN->pui64Caps;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psGetMultiCoreInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psGetMultiCoreInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto GetMultiCoreInfo_exit;
+			}
+		}
+	}
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize != 0)
+	{
+		pui64CapsInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64);
+	}
+
+	psGetMultiCoreInfoOUT->eError =
+	    PVRSRVGetMultiCoreInfoKM(psConnection, OSGetDevNode(psConnection),
+				     psGetMultiCoreInfoIN->ui32CapsSize,
+				     &psGetMultiCoreInfoOUT->ui32NumCores, pui64CapsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psGetMultiCoreInfoOUT->eError != PVRSRV_OK))
+	{
+		goto GetMultiCoreInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64CapsInt) && ((psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psGetMultiCoreInfoOUT->pui64Caps, pui64CapsInt,
+		      (psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto GetMultiCoreInfo_exit;
+		}
+	}
+
+GetMultiCoreInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psGetMultiCoreInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWaitTimeout(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psEventObjectWaitTimeoutIN_UI8,
+				   IMG_UINT8 * psEventObjectWaitTimeoutOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitTimeoutIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitTimeoutOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitTimeoutOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWaitTimeout_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitTimeoutOUT->eError =
+	    OSEventObjectWaitTimeout(hOSEventKMInt, psEventObjectWaitTimeoutIN->ui64uiTimeoutus);
+
+EventObjectWaitTimeout_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PROCESS_STAT_TYPE_COUNT <= IMG_UINT32_MAX,
+	      "PVRSRV_PROCESS_STAT_TYPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeFindProcessMemStats(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psFindProcessMemStatsIN_UI8,
+				IMG_UINT8 * psFindProcessMemStatsOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *psFindProcessMemStatsIN =
+	    (PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *psFindProcessMemStatsOUT =
+	    (PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsOUT_UI8,
+								      0);
+
+	IMG_UINT64 *pui64MemStatsArrayInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psFindProcessMemStatsIN->ui32ArrSize > PVRSRV_PROCESS_STAT_TYPE_COUNT)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto FindProcessMemStats_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psFindProcessMemStatsOUT->pui64MemStatsArray = psFindProcessMemStatsIN->pui64MemStatsArray;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto FindProcessMemStats_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psFindProcessMemStatsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psFindProcessMemStatsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto FindProcessMemStats_exit;
+			}
+		}
+	}
+
+	if (psFindProcessMemStatsIN->ui32ArrSize != 0)
+	{
+		pui64MemStatsArrayInt =
+		    (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64);
+	}
+
+	psFindProcessMemStatsOUT->eError =
+	    PVRSRVFindProcessMemStatsKM(psFindProcessMemStatsIN->ui32PID,
+					psFindProcessMemStatsIN->ui32ArrSize,
+					psFindProcessMemStatsIN->bbAllProcessStats,
+					pui64MemStatsArrayInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psFindProcessMemStatsOUT->eError != PVRSRV_OK))
+	{
+		goto FindProcessMemStats_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64MemStatsArrayInt) &&
+	    ((psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psFindProcessMemStatsOUT->pui64MemStatsArray,
+		      pui64MemStatsArrayInt,
+		      (psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto FindProcessMemStats_exit;
+		}
+	}
+
+FindProcessMemStats_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psFindProcessMemStatsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireInfoPagepsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseInfoPageKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psAcquireInfoPageIN_UI8,
+			    IMG_UINT8 * psAcquireInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *psAcquireInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *psAcquireInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageOUT_UI8, 0);
+
+	PMR *psPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireInfoPageIN);
+
+	psAcquireInfoPageOUT->eError = PVRSRVAcquireInfoPageKM(&psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psAcquireInfoPageOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psAcquireInfoPageOUT->hPMR, (void *)psPMRInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _AcquireInfoPagepsPMRIntRelease);
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+AcquireInfoPage_exit:
+
+	if (psAcquireInfoPageOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			PVRSRVReleaseInfoPageKM(psPMRInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psReleaseInfoPageIN_UI8,
+			    IMG_UINT8 * psReleaseInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *psReleaseInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *psReleaseInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psReleaseInfoPageOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psReleaseInfoPageIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psReleaseInfoPageOUT->eError != PVRSRV_OK) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psReleaseInfoPageOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto ReleaseInfoPage_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+ReleaseInfoPage_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+
+/*
+ * Register all SRVCORE functions with services
+ */
+PVRSRV_ERROR InitSRVCOREBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT,
+			      PVRSRVBridgeConnect, NULL, sizeof(PVRSRV_BRIDGE_IN_CONNECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_CONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT,
+			      PVRSRVBridgeDisconnect, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DISCONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT,
+			      PVRSRVBridgeAcquireGlobalEventObject, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT,
+			      PVRSRVBridgeReleaseGlobalEventObject, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN,
+			      PVRSRVBridgeEventObjectOpen, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT,
+			      PVRSRVBridgeEventObjectWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE,
+			      PVRSRVBridgeEventObjectClose, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO,
+			      PVRSRVBridgeDumpDebugInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DUMPDEBUGINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED,
+			      PVRSRVBridgeGetDevClockSpeed, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT,
+			      PVRSRVBridgeHWOpTimeout, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HWOPTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK,
+			      PVRSRVBridgeAlignmentCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_ALIGNMENTCHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS,
+			      PVRSRVBridgeGetDeviceStatus, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVICESTATUS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO,
+			      PVRSRVBridgeGetMultiCoreInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_GETMULTICOREINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_GETMULTICOREINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT,
+			      PVRSRVBridgeEventObjectWaitTimeout, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS,
+			      PVRSRVBridgeFindProcessMemStats, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS),
+			      sizeof(PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE,
+			      PVRSRVBridgeAcquireInfoPage, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE,
+			      PVRSRVBridgeReleaseInfoPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEINFOPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all srvcore functions with services
+ */
+void DeinitSRVCOREBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_bridge.h
new file mode 100644
index 000000000000..19f1b0ece5b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_bridge.h
@@ -0,0 +1,102 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNC_BRIDGE_H
+#define CLIENT_SYNC_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_sync_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence);
+
+#endif /* CLIENT_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_direct_bridge.c
new file mode 100644
index 000000000000..d631aea421ad
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_direct_bridge.c
@@ -0,0 +1,262 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for sync
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_sync_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    &psSyncHandleInt,
+					    pui32SyncPrimVAddr,
+					    pui32SyncPrimBlockSize, &pshSyncPMRInt);
+
+	*phSyncHandle = psSyncHandleInt;
+	*phhSyncPMR = pshSyncPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimSetKM(psSyncHandleInt, ui32Index, ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpKM(psSyncHandleInt, ui32Offset);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt, ui32Offset, ui32Value);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     ui32Offset, ui32Value, ui32Mask, eOperator, uiPDumpFlags);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     ui32Offset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVSyncAllocEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   bServerSync, ui32FWAddr, ui32ClassNameSize, puiClassName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVSyncFreeEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32FWAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PVRSRVSyncCheckpointSignalledPDumpPolKM(hFence);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hFence);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/common_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/common_sync_bridge.h
new file mode 100644
index 000000000000..753d40b64f12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/common_sync_bridge.h
@@ -0,0 +1,254 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNC_BRIDGE_H
+#define COMMON_SYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#define PVRSRV_BRIDGE_SYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMSET			PVRSRV_BRIDGE_SYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE			PVRSRV_BRIDGE_SYNC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNC_CMD_LAST			(PVRSRV_BRIDGE_SYNC_CMD_FIRST+9)
+
+/*******************************************
+            AllocSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_HANDLE hhSyncPMR;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32SyncPrimBlockSize;
+	IMG_UINT32 ui32SyncPrimVAddr;
+} __packed PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            FreeSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+} __packed PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            SyncPrimSet
+ *******************************************/
+
+/* Bridge in structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMSET_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMSET;
+
+/* Bridge out structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMSET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMSET;
+
+/*******************************************
+            SyncPrimPDump
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP;
+
+/* Bridge out structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP;
+
+/*******************************************
+            SyncPrimPDumpValue
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE;
+
+/* Bridge out structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE;
+
+/*******************************************
+            SyncPrimPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+	PDUMP_FLAGS_T uiPDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL;
+
+/* Bridge out structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL;
+
+/*******************************************
+            SyncPrimPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP;
+
+/* Bridge out structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP;
+
+/*******************************************
+            SyncAllocEvent
+ *******************************************/
+
+/* Bridge in structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCALLOCEVENT_TAG
+{
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32FWAddr;
+	IMG_BOOL bServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCALLOCEVENT;
+
+/* Bridge out structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT;
+
+/*******************************************
+            SyncFreeEvent
+ *******************************************/
+
+/* Bridge in structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFREEEVENT_TAG
+{
+	IMG_UINT32 ui32FWAddr;
+} __packed PVRSRV_BRIDGE_IN_SYNCFREEEVENT;
+
+/* Bridge out structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFREEEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFREEEVENT;
+
+/*******************************************
+            SyncCheckpointSignalledPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_FENCE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+/* Bridge out structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+#endif /* COMMON_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/server_sync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/server_sync_bridge.c
new file mode 100644
index 000000000000..a0d9210086c9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/server_sync_bridge.c
@@ -0,0 +1,755 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+#include "common_sync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _AllocSyncPrimitiveBlockpsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVFreeSyncPrimitiveBlockKM((SYNC_PRIMITIVE_BLOCK *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAllocSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockIN_UI8,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockOUT_UI8, 0);
+
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAllocSyncPrimitiveBlockIN);
+
+	psAllocSyncPrimitiveBlockOUT->hSyncHandle = NULL;
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(psConnection, OSGetDevNode(psConnection),
+					    &psSyncHandleInt,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimVAddr,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimBlockSize,
+					    &pshSyncPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAllocSyncPrimitiveBlockOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psAllocSyncPrimitiveBlockOUT->
+									 hSyncHandle,
+									 (void *)psSyncHandleInt,
+									 PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _AllocSyncPrimitiveBlockpsSyncHandleIntRelease);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psAllocSyncPrimitiveBlockOUT->hhSyncPMR,
+					 (void *)pshSyncPMRInt,
+					 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psAllocSyncPrimitiveBlockOUT->hSyncHandle);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AllocSyncPrimitiveBlock_exit:
+
+	if (psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK)
+	{
+		if (psAllocSyncPrimitiveBlockOUT->hSyncHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psAllocSyncPrimitiveBlockOUT->
+							     hSyncHandle,
+							     PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSyncHandleInt)
+		{
+			PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeFreeSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockIN_UI8,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psFreeSyncPrimitiveBlockOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psFreeSyncPrimitiveBlockIN->hSyncHandle,
+					      PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	if (unlikely((psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_OK) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psFreeSyncPrimitiveBlockOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto FreeSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+FreeSyncPrimitiveBlock_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncPrimSet(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psSyncPrimSetIN_UI8,
+			IMG_UINT8 * psSyncPrimSetOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMSET *psSyncPrimSetIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMSET *psSyncPrimSetOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimSetIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimSetOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimSetOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimSet_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimSetOUT->eError =
+	    PVRSRVSyncPrimSetKM(psSyncHandleInt,
+				psSyncPrimSetIN->ui32Index, psSyncPrimSetIN->ui32Value);
+
+SyncPrimSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDump(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncPrimPDumpIN_UI8,
+			  IMG_UINT8 * psSyncPrimPDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *psSyncPrimPDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *psSyncPrimPDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVSyncPrimPDumpKM(psSyncHandleInt, psSyncPrimPDumpIN->ui32Offset);
+
+SyncPrimPDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDump NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpValue(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncPrimPDumpValueIN_UI8,
+			       IMG_UINT8 * psSyncPrimPDumpValueOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueOUT_UI8,
+								     0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpValueIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpValueOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpValue_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt,
+				       psSyncPrimPDumpValueIN->ui32Offset,
+				       psSyncPrimPDumpValueIN->ui32Value);
+
+SyncPrimPDumpValue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpValue NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpPolIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpPolOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpPolIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpPolOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpPol_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     psSyncPrimPDumpPolIN->ui32Offset,
+				     psSyncPrimPDumpPolIN->ui32Value,
+				     psSyncPrimPDumpPolIN->ui32Mask,
+				     psSyncPrimPDumpPolIN->eOperator,
+				     psSyncPrimPDumpPolIN->uiPDumpFlags);
+
+SyncPrimPDumpPol_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpPol NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpCBPIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpCBPIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     psSyncPrimPDumpCBPIN->ui32Offset,
+				     psSyncPrimPDumpCBPIN->uiWriteOffset,
+				     psSyncPrimPDumpCBPIN->uiPacketSize,
+				     psSyncPrimPDumpCBPIN->uiBufferSize);
+
+SyncPrimPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpCBP NULL
+#endif
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncAllocEvent(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncAllocEventIN_UI8,
+			   IMG_UINT8 * psSyncAllocEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *psSyncAllocEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *psSyncAllocEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventOUT_UI8, 0);
+
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncAllocEventIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncAllocEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncAllocEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncAllocEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncAllocEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncAllocEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncAllocEvent_exit;
+			}
+		}
+	}
+
+	if (psSyncAllocEventIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncAllocEventIN->puiClassName,
+		     psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncAllocEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncAllocEvent_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psSyncAllocEventOUT->eError =
+	    PVRSRVSyncAllocEventKM(psConnection, OSGetDevNode(psConnection),
+				   psSyncAllocEventIN->bServerSync,
+				   psSyncAllocEventIN->ui32FWAddr,
+				   psSyncAllocEventIN->ui32ClassNameSize, uiClassNameInt);
+
+SyncAllocEvent_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncAllocEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFreeEvent(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncFreeEventIN_UI8,
+			  IMG_UINT8 * psSyncFreeEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFREEEVENT *psSyncFreeEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *psSyncFreeEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventOUT_UI8, 0);
+
+	psSyncFreeEventOUT->eError =
+	    PVRSRVSyncFreeEventKM(psConnection, OSGetDevNode(psConnection),
+				  psSyncFreeEventIN->ui32FWAddr);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncCheckpointSignalledPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolIN_UI8,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psSyncCheckpointSignalledPDumpPolOUT->eError =
+	    PVRSRVSyncCheckpointSignalledPDumpPolKM(psSyncCheckpointSignalledPDumpPolIN->hFence);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncCheckpointSignalledPDumpPol NULL
+#endif
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+
+/*
+ * Register all SYNC functions with services
+ */
+PVRSRV_ERROR InitSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeAllocSyncPrimitiveBlock, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeFreeSyncPrimitiveBlock, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK),
+			      sizeof(PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET,
+			      PVRSRVBridgeSyncPrimSet, NULL, sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMSET),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMSET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP,
+			      PVRSRVBridgeSyncPrimPDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE,
+			      PVRSRVBridgeSyncPrimPDumpValue, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL,
+			      PVRSRVBridgeSyncPrimPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP,
+			      PVRSRVBridgeSyncPrimPDumpCBP, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT,
+			      PVRSRVBridgeSyncAllocEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCALLOCEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT,
+			      PVRSRVBridgeSyncFreeEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFREEEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFREEEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+			      PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL,
+			      PVRSRVBridgeSyncCheckpointSignalledPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all sync functions with services
+ */
+void DeinitSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+				PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/common_syncfallback_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/common_syncfallback_bridge.h
new file mode 100644
index 000000000000..87016c4706c7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/common_syncfallback_bridge.h
@@ -0,0 +1,357 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCFALLBACK_BRIDGE_H
+#define COMMON_SYNCFALLBACK_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST			(PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15)
+
+/*******************************************
+            SyncFbTimelineCreatePVR
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR;
+
+/* Bridge out structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR;
+
+/*******************************************
+            SyncFbTimelineRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE;
+
+/* Bridge out structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE;
+
+/*******************************************
+            SyncFbFenceDup
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hInFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP;
+
+/* Bridge out structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP;
+
+/*******************************************
+            SyncFbFenceMerge
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hInFence1;
+	IMG_HANDLE hInFence2;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE;
+
+/* Bridge out structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE;
+
+/*******************************************
+            SyncFbFenceRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE;
+
+/* Bridge out structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE;
+
+/*******************************************
+            SyncFbFenceWait
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT_TAG
+{
+	IMG_HANDLE hFence;
+	IMG_UINT32 ui32Timeout;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT;
+
+/* Bridge out structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT;
+
+/*******************************************
+            SyncFbFenceDump
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP_TAG
+{
+	IMG_HANDLE hFence;
+	const IMG_CHAR *puiDescStr;
+	const IMG_CHAR *puiFileStr;
+	const IMG_CHAR *puiModuleStr;
+	IMG_UINT32 ui32DescStrLength;
+	IMG_UINT32 ui32FileStrLength;
+	IMG_UINT32 ui32Line;
+	IMG_UINT32 ui32ModuleStrLength;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP;
+
+/* Bridge out structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP;
+
+/*******************************************
+            SyncFbTimelineCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW;
+
+/* Bridge out structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW;
+
+/*******************************************
+            SyncFbFenceCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW;
+
+/* Bridge out structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW;
+
+/*******************************************
+            SyncFbTimelineAdvanceSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW;
+
+/* Bridge out structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW;
+
+/*******************************************
+            SyncFbFenceExportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroyInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/*******************************************
+            SyncFbFenceImportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hImport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroySecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/*******************************************
+            SyncFbFenceImportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Import;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE;
+
+#endif /* COMMON_SYNCFALLBACK_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/server_syncfallback_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/server_syncfallback_bridge.c
new file mode 100644
index 000000000000..68e650d89602
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/server_syncfallback_bridge.c
@@ -0,0 +1,1958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync_fallback_server.h"
+#include "pvrsrv_sync_server.h"
+
+#include "common_syncfallback_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleaseExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _SyncFbTimelineCreatePVRpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_TIMELINE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_TIMELINE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreatePVR(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVRIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVROUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVRIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVROUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVROUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize > SYNC_FB_TIMELINE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreatePVRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreatePVRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreatePVR_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreatePVRIN->puiTimelineName,
+		     psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreatePVR_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    SyncFbTimelineCreatePVR(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize,
+				    uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreatePVROUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreatePVRpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreatePVR_exit:
+
+	if (psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreatePVROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psSyncFbTimelineReleaseIN_UI8,
+				  IMG_UINT8 * psSyncFbTimelineReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbTimelineReleaseIN->hTimeline,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	if (unlikely((psSyncFbTimelineReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbTimelineReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceDuppsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDup(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncFbFenceDupIN_UI8,
+			   IMG_UINT8 * psSyncFbFenceDupOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *psSyncFbFenceDupIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *psSyncFbFenceDupOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupOUT_UI8, 0);
+
+	IMG_HANDLE hInFence = psSyncFbFenceDupIN->hInFence;
+	PVRSRV_FENCE_SERVER *psInFenceInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFenceInt,
+				       hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError = SyncFbFenceDup(psInFenceInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceDupOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceDuppsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceDup_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceDupOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceMergepsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceMerge(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncFbFenceMergeIN_UI8,
+			     IMG_UINT8 * psSyncFbFenceMergeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *psSyncFbFenceMergeIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *psSyncFbFenceMergeOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeOUT_UI8, 0);
+
+	IMG_HANDLE hInFence1 = psSyncFbFenceMergeIN->hInFence1;
+	PVRSRV_FENCE_SERVER *psInFence1Int = NULL;
+	IMG_HANDLE hInFence2 = psSyncFbFenceMergeIN->hInFence2;
+	PVRSRV_FENCE_SERVER *psInFence2Int = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceMergeIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceMergeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceMergeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceMerge_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt, (const void __user *)psSyncFbFenceMergeIN->puiFenceName,
+		     psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceMerge_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence1Int,
+				       hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence2Int,
+				       hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    SyncFbFenceMerge(psInFence1Int,
+			     psInFence2Int,
+			     psSyncFbFenceMergeIN->ui32FenceNameSize,
+			     uiFenceNameInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceMergeOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceMergepsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceMerge_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFence1Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psInFence2Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceMergeOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceMergeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncFbFenceReleaseIN_UI8,
+			       IMG_UINT8 * psSyncFbFenceReleaseOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *psSyncFbFenceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *psSyncFbFenceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceReleaseIN->hFence,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	if (unlikely((psSyncFbFenceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbFenceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceWaitIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *psSyncFbFenceWaitIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *psSyncFbFenceWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceWaitIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceWaitOUT->eError =
+	    SyncFbFenceWait(psFenceInt, psSyncFbFenceWaitIN->ui32Timeout);
+
+SyncFbFenceWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+static_assert((SYNC_FB_FILE_STRING_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_FILE_STRING_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_MODULE_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_MODULE_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_DESC_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_DESC_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDump(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceDumpIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *psSyncFbFenceDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *psSyncFbFenceDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceDumpIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	IMG_CHAR *uiFileStrInt = NULL;
+	IMG_CHAR *uiModuleStrInt = NULL;
+	IMG_CHAR *uiDescStrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32FileStrLength > (SYNC_FB_FILE_STRING_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely
+	    (psSyncFbFenceDumpIN->ui32ModuleStrLength > (SYNC_FB_MODULE_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32DescStrLength > (SYNC_FB_DESC_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceDump_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceDumpIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceDumpIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceDump_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceDumpIN->ui32FileStrLength != 0)
+	{
+		uiFileStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileStrInt, (const void __user *)psSyncFbFenceDumpIN->puiFileStr,
+		     psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileStrInt)[(psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength != 0)
+	{
+		uiModuleStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiModuleStrInt, (const void __user *)psSyncFbFenceDumpIN->puiModuleStr,
+		     psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiModuleStrInt)[(psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32DescStrLength != 0)
+	{
+		uiDescStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiDescStrInt, (const void __user *)psSyncFbFenceDumpIN->puiDescStr,
+		     psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiDescStrInt)[(psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDumpOUT->eError =
+	    SyncFbFenceDump(psFenceInt,
+			    psSyncFbFenceDumpIN->ui32Line,
+			    psSyncFbFenceDumpIN->ui32FileStrLength,
+			    uiFileStrInt,
+			    psSyncFbFenceDumpIN->ui32ModuleStrLength,
+			    uiModuleStrInt, psSyncFbFenceDumpIN->ui32DescStrLength, uiDescStrInt);
+
+SyncFbFenceDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceDumpOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbTimelineCreateSWpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWIN_UI8,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWOUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreateSWIN->puiTimelineName,
+		     psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    SyncFbTimelineCreateSW(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize,
+				   uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreateSWOUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreateSWpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreateSW_exit:
+
+	if (psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceCreateSWpsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psSyncFbFenceCreateSWIN_UI8,
+				IMG_UINT8 * psSyncFbFenceCreateSWOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWOUT_UI8,
+								      0);
+
+	IMG_HANDLE hTimeline = psSyncFbFenceCreateSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceCreateSWIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt,
+		     (const void __user *)psSyncFbFenceCreateSWIN->puiFenceName,
+		     psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    SyncFbFenceCreateSW(psConnection, OSGetDevNode(psConnection),
+				psTimelineInt,
+				psSyncFbFenceCreateSWIN->ui32FenceNameSize,
+				uiFenceNameInt,
+				&psOutFenceInt, &psSyncFbFenceCreateSWOUT->ui64SyncPtIdx);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceCreateSWOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceCreateSWpsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceCreateSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineAdvanceSW(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWOUT_UI8, 0);
+
+	IMG_HANDLE hTimeline = psSyncFbTimelineAdvanceSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbTimelineAdvanceSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineAdvanceSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    SyncFbTimelineAdvanceSW(psTimelineInt, &psSyncFbTimelineAdvanceSWOUT->ui64SyncPtIdx);
+
+SyncFbTimelineAdvanceSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceExportInsecurepsExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceExportDestroyInsecure((PVRSRV_FENCE_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportInsecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    SyncFbFenceExportInsecure(psFenceInt, &psExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase, &hExportInt,
+				      (void *)psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceExportInsecurepsExportIntRelease);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportInsecureOUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+									   &psSyncFbFenceExportInsecureOUT->
+									   hExport,
+									   (void *)psExportInt,
+									   PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+									   PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+									   (PFN_HANDLE_RELEASE) &
+									   ReleaseExport);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncFbFenceExportInsecureOUT->hExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE)
+							     psSyncFbFenceExportInsecureOUT->
+							     hExport,
+							     PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hExportInt,
+							PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psExportInt)
+		{
+			SyncFbFenceExportDestroyInsecure(psExportInt);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroyInsecure(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureIN_UI8,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psExportInt,
+				       (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->hExport,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_FALSE);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hExportInt,
+				     psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+		     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT((psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK) ||
+		   (psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportDestroyInsecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportDestroyInsecure_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportDestroyInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceImportInsecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hImport = psSyncFbFenceImportInsecureIN->hImport;
+	PVRSRV_FENCE_EXPORT *psImportInt = NULL;
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psImportInt,
+				       hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_TRUE);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    SyncFbFenceImportInsecure(psConnection, OSGetDevNode(psConnection),
+				      psImportInt, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportInsecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportInsecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psImportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceImportInsecure NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceExportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceExportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportSecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportSecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportSecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportSecureOUT->eError =
+	    SyncFbFenceExportSecure(psConnection, OSGetDevNode(psConnection),
+				    psFenceInt,
+				    &psSyncFbFenceExportSecureOUT->Export,
+				    &psExportInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportSecure_exit;
+	}
+
+SyncFbFenceExportSecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psExportInt)
+		{
+			SyncFbFenceExportDestroySecure(psExportInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroySecure(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureIN_UI8,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncFbFenceExportDestroySecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroySecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroySecureOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncFbFenceExportDestroySecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncFbFenceExportDestroySecure_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceImportSecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceImportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceImportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    SyncFbFenceImportSecure(psConnection, OSGetDevNode(psConnection),
+				    psSyncFbFenceImportSecureIN->Import, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportSecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportSecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportSecure_exit:
+
+	if (psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+
+/*
+ * Register all SYNCFALLBACK functions with services
+ */
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR,
+			      PVRSRVBridgeSyncFbTimelineCreatePVR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE,
+			      PVRSRVBridgeSyncFbTimelineRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK, PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP,
+			      PVRSRVBridgeSyncFbFenceDup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE,
+			      PVRSRVBridgeSyncFbFenceMerge, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE,
+			      PVRSRVBridgeSyncFbFenceRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT,
+			      PVRSRVBridgeSyncFbFenceWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP,
+			      PVRSRVBridgeSyncFbFenceDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW,
+			      PVRSRVBridgeSyncFbTimelineCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW,
+			      PVRSRVBridgeSyncFbFenceCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW,
+			      PVRSRVBridgeSyncFbTimelineAdvanceSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroyInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceImportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceExportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroySecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceImportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all syncfallback functions with services
+ */
+void DeinitSYNCFALLBACKBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_bridge.h
new file mode 100644
index 000000000000..544efd962834
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_bridge.h
@@ -0,0 +1,68 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNCTRACKING_BRIDGE_H
+#define CLIENT_SYNCTRACKING_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_synctracking_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName);
+
+#endif /* CLIENT_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_direct_bridge.c
new file mode 100644
index 000000000000..baeb89a6a0ae
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_direct_bridge.c
@@ -0,0 +1,92 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for synctracking
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_synctracking_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "sync.h"
+#include "sync_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	pshRecordInt = (SYNC_RECORD_HANDLE) hhRecord;
+
+	eError = PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt;
+
+	pshServerSyncPrimBlockInt = (SYNC_PRIMITIVE_BLOCK *) hhServerSyncPrimBlock;
+
+	eError =
+	    PVRSRVSyncRecordAddKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  ui32ui32FwBlockAddr,
+				  ui32ui32SyncOffset,
+				  bbServerSync, ui32ClassNameSize, puiClassName);
+
+	*phhRecord = pshRecordInt;
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/common_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/common_synctracking_bridge.h
new file mode 100644
index 000000000000..9fd9fdb796a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/common_synctracking_bridge.h
@@ -0,0 +1,97 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCTRACKING_BRIDGE_H
+#define COMMON_SYNCTRACKING_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST			(PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1)
+
+/*******************************************
+            SyncRecordRemoveByHandle
+ *******************************************/
+
+/* Bridge in structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	IMG_HANDLE hhRecord;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE;
+
+/* Bridge out structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE;
+
+/*******************************************
+            SyncRecordAdd
+ *******************************************/
+
+/* Bridge in structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhServerSyncPrimBlock;
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32ui32FwBlockAddr;
+	IMG_UINT32 ui32ui32SyncOffset;
+	IMG_BOOL bbServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDADD;
+
+/* Bridge out structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhRecord;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDADD;
+
+#endif /* COMMON_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/server_synctracking_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/server_synctracking_bridge.c
new file mode 100644
index 000000000000..06757e5ef8d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/server_synctracking_bridge.c
@@ -0,0 +1,329 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+
+#include "common_synctracking_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeSyncRecordRemoveByHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleIN_UI8,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordRemoveByHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncRecordRemoveByHandleIN->hhRecord,
+					      PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE);
+	if (unlikely((psSyncRecordRemoveByHandleOUT->eError != PVRSRV_OK) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncRecordRemoveByHandleOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordRemoveByHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordRemoveByHandle_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncRecordAddpshRecordIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVSyncRecordRemoveByHandleKM((SYNC_RECORD_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncRecordAdd(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncRecordAddIN_UI8,
+			  IMG_UINT8 * psSyncRecordAddOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDADD *psSyncRecordAddIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDADD *psSyncRecordAddOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddOUT_UI8, 0);
+
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	IMG_HANDLE hhServerSyncPrimBlock = psSyncRecordAddIN->hhServerSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncRecordAddIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncRecordAdd_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncRecordAdd_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncRecordAddIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncRecordAddIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncRecordAddOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncRecordAdd_exit;
+			}
+		}
+	}
+
+	if (psSyncRecordAddIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncRecordAddIN->puiClassName,
+		     psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncRecordAddOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncRecordAdd_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncRecordAddOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pshServerSyncPrimBlockInt,
+				       hhServerSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError =
+	    PVRSRVSyncRecordAddKM(psConnection, OSGetDevNode(psConnection),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  psSyncRecordAddIN->ui32ui32FwBlockAddr,
+				  psSyncRecordAddIN->ui32ui32SyncOffset,
+				  psSyncRecordAddIN->bbServerSync,
+				  psSyncRecordAddIN->ui32ClassNameSize, uiClassNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psSyncRecordAddOUT->hhRecord,
+							       (void *)pshRecordInt,
+							       PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+							       PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+							       (PFN_HANDLE_RELEASE) &
+							       _SyncRecordAddpshRecordIntRelease);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordAdd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pshServerSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hhServerSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psSyncRecordAddOUT->eError != PVRSRV_OK)
+	{
+		if (pshRecordInt)
+		{
+			PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncRecordAddOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+
+/*
+ * Register all SYNCTRACKING functions with services
+ */
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+			      PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE,
+			      PVRSRVBridgeSyncRecordRemoveByHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING, PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD,
+			      PVRSRVBridgeSyncRecordAdd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDADD),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDADD));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all synctracking functions with services
+ */
+void DeinitSYNCTRACKINGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_bridge.h
new file mode 100644
index 000000000000..1dec13fccce7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_CACHE_BRIDGE_H
+#define CLIENT_CACHE_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_cache_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp);
+
+#endif /* CLIENT_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_direct_bridge.c
new file mode 100644
index 000000000000..9691bae9352f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_direct_bridge.c
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for cache
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_cache_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "cache_ops.h"
+
+#include "cache_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PMR **psPMRInt;
+
+	psPMRInt = (PMR **) phPMR;
+
+	eError =
+	    CacheOpQueue(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			 ui32NumCacheOps,
+			 psPMRInt, pui64Address, puiOffset, puiSize, piuCacheOp, ui32OpTimeline);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = CacheOpValExec(psPMRInt, ui64Address, uiOffset, uiSize, iuCacheOp);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    CacheOpLog(psPMRInt,
+		       ui64Address, uiOffset, uiSize, i64StartTime, i64EndTime, iuCacheOp);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/common_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/common_cache_bridge.h
new file mode 100644
index 000000000000..cc848753f7fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/common_cache_bridge.h
@@ -0,0 +1,126 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CACHE_BRIDGE_H
+#define COMMON_CACHE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_CACHE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE			PVRSRV_BRIDGE_CACHE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPEXEC			PVRSRV_BRIDGE_CACHE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CACHE_CACHEOPLOG			PVRSRV_BRIDGE_CACHE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CACHE_CMD_LAST			(PVRSRV_BRIDGE_CACHE_CMD_FIRST+2)
+
+/*******************************************
+            CacheOpQueue
+ *******************************************/
+
+/* Bridge in structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPQUEUE_TAG
+{
+	PVRSRV_CACHE_OP *piuCacheOp;
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	IMG_UINT32 ui32NumCacheOps;
+	IMG_UINT32 ui32OpTimeline;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPQUEUE;
+
+/* Bridge out structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPQUEUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPQUEUE;
+
+/*******************************************
+            CacheOpExec
+ *******************************************/
+
+/* Bridge in structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPEXEC_TAG
+{
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPEXEC;
+
+/* Bridge out structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPEXEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPEXEC;
+
+/*******************************************
+            CacheOpLog
+ *******************************************/
+
+/* Bridge in structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPLOG_TAG
+{
+	IMG_INT64 i64EndTime;
+	IMG_INT64 i64StartTime;
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPLOG;
+
+/* Bridge out structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPLOG;
+
+#endif /* COMMON_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/server_cache_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/server_cache_bridge.c
new file mode 100644
index 000000000000..b2f5267cc597
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/server_cache_bridge.c
@@ -0,0 +1,452 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "cache_km.h"
+
+#include "common_cache_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(CACHE_BATCH_MAX <= IMG_UINT32_MAX,
+	      "CACHE_BATCH_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeCacheOpQueue(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psCacheOpQueueIN_UI8,
+			 IMG_UINT8 * psCacheOpQueueOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPQUEUE *psCacheOpQueueIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *psCacheOpQueueOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+	PVRSRV_CACHE_OP *iuCacheOpInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) + 0;
+
+	if (unlikely(psCacheOpQueueIN->ui32NumCacheOps > CACHE_BATCH_MAX))
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto CacheOpQueue_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto CacheOpQueue_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psCacheOpQueueIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psCacheOpQueueIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psCacheOpQueueOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *));
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psCacheOpQueueIN->phPMR,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psCacheOpQueueIN->pui64Address,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psCacheOpQueueIN->puiOffset,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psCacheOpQueueIN->puiSize,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		iuCacheOpInt =
+		    (PVRSRV_CACHE_OP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, iuCacheOpInt, (const void __user *)psCacheOpQueueIN->piuCacheOp,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+			/* Look up the address from the handle */
+			psCacheOpQueueOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psCacheOpQueueOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpQueueOUT->eError =
+	    CacheOpQueue(psConnection, OSGetDevNode(psConnection),
+			 psCacheOpQueueIN->ui32NumCacheOps,
+			 psPMRInt,
+			 ui64AddressInt,
+			 uiOffsetInt, uiSizeInt, iuCacheOpInt, psCacheOpQueueIN->ui32OpTimeline);
+
+CacheOpQueue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psCacheOpQueueOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpExec(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psCacheOpExecIN_UI8,
+			IMG_UINT8 * psCacheOpExecOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPEXEC *psCacheOpExecIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPEXEC *psCacheOpExecOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpExecIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpExecOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpExecOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpExec_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpExecOUT->eError =
+	    CacheOpValExec(psPMRInt,
+			   psCacheOpExecIN->ui64Address,
+			   psCacheOpExecIN->uiOffset,
+			   psCacheOpExecIN->uiSize, psCacheOpExecIN->iuCacheOp);
+
+CacheOpExec_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpLog(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psCacheOpLogIN_UI8,
+		       IMG_UINT8 * psCacheOpLogOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPLOG *psCacheOpLogIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPLOG *psCacheOpLogOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpLogIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpLogOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpLogOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpLog_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpLogOUT->eError =
+	    CacheOpLog(psPMRInt,
+		       psCacheOpLogIN->ui64Address,
+		       psCacheOpLogIN->uiOffset,
+		       psCacheOpLogIN->uiSize,
+		       psCacheOpLogIN->i64StartTime,
+		       psCacheOpLogIN->i64EndTime, psCacheOpLogIN->iuCacheOp);
+
+CacheOpLog_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+
+/*
+ * Register all CACHE functions with services
+ */
+PVRSRV_ERROR InitCACHEBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE,
+			      PVRSRVBridgeCacheOpQueue, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPQUEUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPQUEUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC,
+			      PVRSRVBridgeCacheOpExec, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPEXEC),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPEXEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG,
+			      PVRSRVBridgeCacheOpLog, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPLOG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cache functions with services
+ */
+void DeinitCACHEBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/common_cmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/common_cmm_bridge.h
new file mode 100644
index 000000000000..da48de3598be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/common_cmm_bridge.h
@@ -0,0 +1,114 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CMM_BRIDGE_H
+#define COMMON_CMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_CMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CMM_CMD_LAST			(PVRSRV_BRIDGE_CMM_CMD_FIRST+2)
+
+/*******************************************
+            DevmemIntExportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX;
+
+/* Bridge out structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX;
+
+/*******************************************
+            DevmemIntUnexportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX;
+
+/* Bridge out structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX;
+
+/*******************************************
+            DevmemIntAcquireRemoteCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX;
+
+/* Bridge out structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX;
+
+#endif /* COMMON_CMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/server_cmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/server_cmm_bridge.c
new file mode 100644
index 000000000000..cc658c793d3b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/server_cmm_bridge.c
@@ -0,0 +1,414 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_server.h"
+
+#include "common_cmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DevmemIntExportCtxpsContextExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnexportCtx((DEVMEMINT_CTX_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntExportCtx(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntExportCtxIN_UI8,
+			       IMG_UINT8 * psDevmemIntExportCtxOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxOUT_UI8,
+								     0);
+
+	IMG_HANDLE hContext = psDevmemIntExportCtxIN->hContext;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntExportCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX_EXPORT *psContextExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError =
+	    DevmemIntExportCtx(psContextInt, psPMRInt, &psContextExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntExportCtxOUT->
+								    hContextExport,
+								    (void *)psContextExportInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+								    PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntExportCtxpsContextExportIntRelease);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntExportCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntExportCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psContextExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DevmemIntUnexportCtx(psContextExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnexportCtx(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntUnexportCtxIN_UI8,
+				 IMG_UINT8 * psDevmemIntUnexportCtxOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntUnexportCtxIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnexportCtxOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnexportCtxOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnexportCtxIN->hContextExport,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT);
+	if (unlikely((psDevmemIntUnexportCtxOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnexportCtxOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnexportCtx_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnexportCtx_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntAcquireRemoteCtxpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntAcquireRemoteCtx(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxIN_UI8,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevmemIntAcquireRemoteCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntAcquireRemoteCtxOUT->hContext = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    DevmemIntAcquireRemoteCtx(psPMRInt, &psContextInt, &hPrivDataInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntAcquireRemoteCtxOUT->hContext,
+				      (void *)psContextInt, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntAcquireRemoteCtxpsContextIntRelease);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psDevmemIntAcquireRemoteCtxOUT->hPrivData,
+					 (void *)hPrivDataInt, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psDevmemIntAcquireRemoteCtxOUT->hContext);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntAcquireRemoteCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntAcquireRemoteCtxOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psDevmemIntAcquireRemoteCtxOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DevmemIntCtxDestroy(psContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_CMM_BRIDGE */
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+
+/*
+ * Register all CMM functions with services
+ */
+PVRSRV_ERROR InitCMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX,
+			      PVRSRVBridgeDevmemIntExportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX,
+			      PVRSRVBridgeDevmemIntUnexportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX,
+			      PVRSRVBridgeDevmemIntAcquireRemoteCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cmm functions with services
+ */
+void DeinitCMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX);
+
+}
+#else /* EXCLUDE_CMM_BRIDGE */
+/* This bridge is conditional on EXCLUDE_CMM_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitCMMBridge() \
+	PVRSRV_OK
+
+#define DeinitCMMBridge()
+
+#endif /* EXCLUDE_CMM_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/common_dc_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/common_dc_bridge.h
new file mode 100644
index 000000000000..d811fbd4cef1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/common_dc_bridge.h
@@ -0,0 +1,548 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DC_BRIDGE_H
+#define COMMON_DC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_surface.h"
+#include "dc_external.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE			PVRSRV_BRIDGE_DC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DC_DCDEVICERELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DC_DCGETINFO			PVRSRV_BRIDGE_DC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_DC_DCPANELQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_DC_DCFORMATQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_DC_DCDIMQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_DC_DCSETBLANK			PVRSRV_BRIDGE_DC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING			PVRSRV_BRIDGE_DC_CMD_FIRST+10
+#define PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+11
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+12
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+13
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE			PVRSRV_BRIDGE_DC_CMD_FIRST+14
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK			PVRSRV_BRIDGE_DC_CMD_FIRST+15
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY			PVRSRV_BRIDGE_DC_CMD_FIRST+16
+#define PVRSRV_BRIDGE_DC_DCBUFFERALLOC			PVRSRV_BRIDGE_DC_CMD_FIRST+17
+#define PVRSRV_BRIDGE_DC_DCBUFFERIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+18
+#define PVRSRV_BRIDGE_DC_DCBUFFERFREE			PVRSRV_BRIDGE_DC_CMD_FIRST+19
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+20
+#define PVRSRV_BRIDGE_DC_DCBUFFERPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+21
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+22
+#define PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+23
+#define PVRSRV_BRIDGE_DC_DCBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+24
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2			PVRSRV_BRIDGE_DC_CMD_FIRST+25
+#define PVRSRV_BRIDGE_DC_CMD_LAST			(PVRSRV_BRIDGE_DC_CMD_FIRST+25)
+
+/*******************************************
+            DCDevicesQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT;
+
+/* Bridge out structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT;
+
+/*******************************************
+            DCDevicesEnumerate
+ *******************************************/
+
+/* Bridge in structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	IMG_UINT32 ui32DeviceArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE;
+
+/* Bridge out structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE;
+
+/*******************************************
+            DCDeviceAcquire
+ *******************************************/
+
+/* Bridge in structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE_TAG
+{
+	IMG_UINT32 ui32DeviceIndex;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE;
+
+/* Bridge out structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE;
+
+/*******************************************
+            DCDeviceRelease
+ *******************************************/
+
+/* Bridge in structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICERELEASE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICERELEASE;
+
+/* Bridge out structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICERELEASE;
+
+/*******************************************
+            DCGetInfo
+ *******************************************/
+
+/* Bridge in structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_IN_DCGETINFO_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCGETINFO;
+
+/* Bridge out structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DCGETINFO_TAG
+{
+	DC_DISPLAY_INFO sDisplayInfo;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCGETINFO;
+
+/*******************************************
+            DCPanelQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT;
+
+/* Bridge out structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT;
+
+/*******************************************
+            DCPanelQuery
+ *******************************************/
+
+/* Bridge in structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	IMG_UINT32 ui32PanelsArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERY;
+
+/* Bridge out structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERY_TAG
+{
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERY;
+
+/*******************************************
+            DCFormatQuery
+ *******************************************/
+
+/* Bridge in structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCFORMATQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_FORMAT *psFormat;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumFormats;
+} __packed PVRSRV_BRIDGE_IN_DCFORMATQUERY;
+
+/* Bridge out structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCFORMATQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCFORMATQUERY;
+
+/*******************************************
+            DCDimQuery
+ *******************************************/
+
+/* Bridge in structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCDIMQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_DIMS *psDim;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumDims;
+} __packed PVRSRV_BRIDGE_IN_DCDIMQUERY;
+
+/* Bridge out structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCDIMQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDIMQUERY;
+
+/*******************************************
+            DCSetBlank
+ *******************************************/
+
+/* Bridge in structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_IN_DCSETBLANK_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETBLANK;
+
+/* Bridge out structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETBLANK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETBLANK;
+
+/*******************************************
+            DCSetVSyncReporting
+ *******************************************/
+
+/* Bridge in structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING;
+
+/* Bridge out structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING;
+
+/*******************************************
+            DCLastVSyncQuery
+ *******************************************/
+
+/* Bridge in structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY;
+
+/* Bridge out structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY_TAG
+{
+	IMG_INT64 i64Timestamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY;
+
+/*******************************************
+            DCSystemBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE;
+
+/* Bridge out structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE;
+
+/*******************************************
+            DCSystemBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE;
+
+/* Bridge out structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextCreate
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE;
+
+/* Bridge out structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE;
+
+/*******************************************
+            DCDisplayContextConfigureCheck
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/* Bridge out structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/*******************************************
+            DCDisplayContextDestroy
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	IMG_HANDLE hDisplayContext;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY;
+
+/* Bridge out structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY;
+
+/*******************************************
+            DCBufferAlloc
+ *******************************************/
+
+/* Bridge in structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	DC_BUFFER_CREATE_INFO sSurfInfo;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERALLOC;
+
+/* Bridge out structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERALLOC;
+
+/*******************************************
+            DCBufferImport
+ *******************************************/
+
+/* Bridge in structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	IMG_HANDLE *phImport;
+	DC_BUFFER_IMPORT_INFO sSurfAttrib;
+	IMG_UINT32 ui32NumPlanes;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERIMPORT;
+
+/* Bridge out structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT;
+
+/*******************************************
+            DCBufferFree
+ *******************************************/
+
+/* Bridge in structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERFREE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERFREE;
+
+/* Bridge out structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERFREE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERFREE;
+
+/*******************************************
+            DCBufferUnimport
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT;
+
+/* Bridge out structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT;
+
+/*******************************************
+            DCBufferPin
+ *******************************************/
+
+/* Bridge in structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERPIN;
+
+/* Bridge out structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERPIN;
+
+/*******************************************
+            DCBufferUnpin
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNPIN;
+
+/* Bridge out structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN;
+
+/*******************************************
+            DCBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE;
+
+/* Bridge out structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE;
+
+/*******************************************
+            DCBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERRELEASE;
+
+/* Bridge out structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextConfigure2
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	PVRSRV_FENCE hAcquireFence;
+	PVRSRV_TIMELINE hReleaseFenceTimeline;
+	IMG_UINT32 ui32DisplayPeriod;
+	IMG_UINT32 ui32MaxDepth;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2;
+
+/* Bridge out structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hReleaseFence;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2;
+
+#endif /* COMMON_DC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/server_dc_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/server_dc_bridge.c
new file mode 100644
index 000000000000..3b5b9150c2ec
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/server_dc_bridge.c
@@ -0,0 +1,2554 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dc_server.h"
+
+#include "common_dc_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeDCDevicesQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCDevicesQueryCountIN_UI8,
+				IMG_UINT8 * psDCDevicesQueryCountOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDCDevicesQueryCountIN);
+
+	psDCDevicesQueryCountOUT->eError =
+	    DCDevicesQueryCount(&psDCDevicesQueryCountOUT->ui32DeviceCount);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DEVICE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_DEVICE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDevicesEnumerate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDCDevicesEnumerateIN_UI8,
+			       IMG_UINT8 * psDCDevicesEnumerateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *psDCDevicesEnumerateIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *psDCDevicesEnumerateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateOUT_UI8,
+								     0);
+
+	IMG_UINT32 *pui32DeviceIndexInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32)) + 0;
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize > DC_MAX_DEVICE_COUNT)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	psDCDevicesEnumerateOUT->pui32DeviceIndex = psDCDevicesEnumerateIN->pui32DeviceIndex;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDevicesEnumerateIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDevicesEnumerateIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDevicesEnumerate_exit;
+			}
+		}
+	}
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize != 0)
+	{
+		pui32DeviceIndexInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32);
+	}
+
+	psDCDevicesEnumerateOUT->eError =
+	    DCDevicesEnumerate(psConnection, OSGetDevNode(psConnection),
+			       psDCDevicesEnumerateIN->ui32DeviceArraySize,
+			       &psDCDevicesEnumerateOUT->ui32DeviceCount, pui32DeviceIndexInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDevicesEnumerateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDevicesEnumerate_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32DeviceIndexInt) &&
+	    ((psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDevicesEnumerateOUT->pui32DeviceIndex,
+		      pui32DeviceIndexInt,
+		      (psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDevicesEnumerate_exit;
+		}
+	}
+
+DCDevicesEnumerate_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDevicesEnumerateOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDeviceAcquirepsDeviceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDeviceRelease((DC_DEVICE *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceAcquireIN_UI8,
+			    IMG_UINT8 * psDCDeviceAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *psDCDeviceAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *psDCDeviceAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireOUT_UI8, 0);
+
+	DC_DEVICE *psDeviceInt = NULL;
+
+	psDCDeviceAcquireOUT->eError =
+	    DCDeviceAcquire(psConnection, OSGetDevNode(psConnection),
+			    psDCDeviceAcquireIN->ui32DeviceIndex, &psDeviceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDCDeviceAcquireOUT->hDevice,
+								 (void *)psDeviceInt,
+								 PVRSRV_HANDLE_TYPE_DC_DEVICE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _DCDeviceAcquirepsDeviceIntRelease);
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceAcquire_exit:
+
+	if (psDCDeviceAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psDeviceInt)
+		{
+			DCDeviceRelease(psDeviceInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceReleaseIN_UI8,
+			    IMG_UINT8 * psDCDeviceReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICERELEASE *psDCDeviceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *psDCDeviceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDeviceReleaseIN->hDevice,
+					      PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	if (unlikely((psDCDeviceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCDeviceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCGetInfo(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psDCGetInfoIN_UI8,
+		      IMG_UINT8 * psDCGetInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCGETINFO *psDCGetInfoIN =
+	    (PVRSRV_BRIDGE_IN_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCGETINFO *psDCGetInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCGetInfoIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCGetInfoOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCGetInfoOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCGetInfo_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCGetInfoOUT->eError = DCGetInfo(psDeviceInt, &psDCGetInfoOUT->sDisplayInfo);
+
+DCGetInfo_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCPanelQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDCPanelQueryCountIN_UI8,
+			      IMG_UINT8 * psDCPanelQueryCountOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *psDCPanelQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *psDCPanelQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryCountIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryCountOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryCountOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQueryCount_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryCountOUT->eError =
+	    DCPanelQueryCount(psDeviceInt, &psDCPanelQueryCountOUT->ui32NumPanels);
+
+DCPanelQueryCount_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(DC_MAX_PANEL_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PANEL_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCPanelQuery(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCPanelQueryIN_UI8,
+			 IMG_UINT8 * psDCPanelQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERY *psDCPanelQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERY *psDCPanelQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_PANEL_INFO *psPanelInfoInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO)) + 0;
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize > DC_MAX_PANEL_COUNT)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCPanelQuery_exit;
+	}
+
+	psDCPanelQueryOUT->psPanelInfo = psDCPanelQueryIN->psPanelInfo;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCPanelQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCPanelQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCPanelQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCPanelQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCPanelQuery_exit;
+			}
+		}
+	}
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize != 0)
+	{
+		psPanelInfoInt =
+		    (PVRSRV_PANEL_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryOUT->eError =
+	    DCPanelQuery(psDeviceInt,
+			 psDCPanelQueryIN->ui32PanelsArraySize,
+			 &psDCPanelQueryOUT->ui32NumPanels, psPanelInfoInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCPanelQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psPanelInfoInt) &&
+	    ((psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCPanelQueryOUT->psPanelInfo, psPanelInfoInt,
+		      (psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO))) != PVRSRV_OK))
+		{
+			psDCPanelQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCPanelQuery_exit;
+		}
+	}
+
+DCPanelQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCPanelQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_FORMATS <= IMG_UINT32_MAX,
+	      "DC_MAX_FORMATS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCFormatQuery(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCFormatQueryIN_UI8,
+			  IMG_UINT8 * psDCFormatQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCFORMATQUERY *psDCFormatQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCFORMATQUERY *psDCFormatQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCFormatQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_FORMAT *psFormatInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) +
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCFormatQueryIN->ui32NumFormats > DC_MAX_FORMATS))
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCFormatQuery_exit;
+	}
+
+	psDCFormatQueryOUT->pui32Supported = psDCFormatQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCFormatQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCFormatQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCFormatQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCFormatQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCFormatQuery_exit;
+			}
+		}
+	}
+
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		psFormatInt =
+		    (PVRSRV_SURFACE_FORMAT *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT);
+	}
+
+	/* Copy the data over */
+	if (psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psFormatInt, (const void __user *)psDCFormatQueryIN->psFormat,
+		     psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) !=
+		    PVRSRV_OK)
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCFormatQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCFormatQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCFormatQueryOUT->eError =
+	    DCFormatQuery(psDeviceInt,
+			  psDCFormatQueryIN->ui32NumFormats, psFormatInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCFormatQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCFormatQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+
+DCFormatQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCFormatQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DIMENSIONS <= IMG_UINT32_MAX,
+	      "DC_MAX_DIMENSIONS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDimQuery(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCDimQueryIN_UI8,
+		       IMG_UINT8 * psDCDimQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDIMQUERY *psDCDimQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDIMQUERY *psDCDimQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDimQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_DIMS *psDimInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) +
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCDimQueryIN->ui32NumDims > DC_MAX_DIMENSIONS))
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDimQuery_exit;
+	}
+
+	psDCDimQueryOUT->pui32Supported = psDCDimQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDimQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDimQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDimQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDimQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDimQuery_exit;
+			}
+		}
+	}
+
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		psDimInt =
+		    (PVRSRV_SURFACE_DIMS *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS);
+	}
+
+	/* Copy the data over */
+	if (psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psDimInt, (const void __user *)psDCDimQueryIN->psDim,
+		     psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) != PVRSRV_OK)
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDimQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDimQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDimQueryOUT->eError =
+	    DCDimQuery(psDeviceInt, psDCDimQueryIN->ui32NumDims, psDimInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCDimQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDimQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+
+DCDimQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDimQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetBlank(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCSetBlankIN_UI8,
+		       IMG_UINT8 * psDCSetBlankOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETBLANK *psDCSetBlankIN =
+	    (PVRSRV_BRIDGE_IN_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSETBLANK *psDCSetBlankOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSetBlankIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetBlankOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetBlankOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetBlank_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetBlankOUT->eError = DCSetBlank(psDeviceInt, psDCSetBlankIN->bEnabled);
+
+DCSetBlank_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetVSyncReporting(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCSetVSyncReportingIN_UI8,
+				IMG_UINT8 * psDCSetVSyncReportingOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *psDCSetVSyncReportingIN =
+	    (PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *psDCSetVSyncReportingOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevice = psDCSetVSyncReportingIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetVSyncReportingOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetVSyncReportingOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetVSyncReporting_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetVSyncReportingOUT->eError =
+	    DCSetVSyncReporting(psDeviceInt, psDCSetVSyncReportingIN->bEnabled);
+
+DCSetVSyncReporting_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCLastVSyncQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCLastVSyncQueryIN_UI8,
+			     IMG_UINT8 * psDCLastVSyncQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *psDCLastVSyncQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *psDCLastVSyncQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCLastVSyncQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCLastVSyncQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCLastVSyncQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCLastVSyncQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCLastVSyncQueryOUT->eError =
+	    DCLastVSyncQuery(psDeviceInt, &psDCLastVSyncQueryOUT->i64Timestamp);
+
+DCLastVSyncQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCSystemBufferAcquirepsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCSystemBufferRelease((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferAcquireIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferAcquireOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSystemBufferAcquireIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSystemBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError =
+	    DCSystemBufferAcquire(psDeviceInt,
+				  &psDCSystemBufferAcquireOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDCSystemBufferAcquireOUT->
+								       hBuffer, (void *)psBufferInt,
+								       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DCSystemBufferAcquirepsBufferIntRelease);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCSystemBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCSystemBufferRelease(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferReleaseIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCSystemBufferReleaseIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCSystemBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCSystemBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDisplayContextCreatepsDisplayContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDisplayContextDestroy((DC_DISPLAY_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextCreate(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDCDisplayContextCreateIN_UI8,
+				   IMG_UINT8 * psDCDisplayContextCreateOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDisplayContextCreateIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError =
+	    DCDisplayContextCreate(psDeviceInt, &psDisplayContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDCDisplayContextCreateOUT->
+									hDisplayContext,
+									(void *)psDisplayContextInt,
+									PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DCDisplayContextCreatepsDisplayContextIntRelease);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCDisplayContextCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDisplayContextInt)
+		{
+			DCDisplayContextDestroy(psDisplayContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigureCheck(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckIN_UI8,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigureCheckIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) +
+	    0;
+
+	if (unlikely(psDCDisplayContextConfigureCheckIN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigureCheckOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigureCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigureCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigureCheckOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) >
+	    0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->psSurfInfo,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+			       sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->phBuffers,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigureCheckOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    DCDisplayContextConfigureCheck(psDisplayContextInt,
+					   psDCDisplayContextConfigureCheckIN->ui32PipeCount,
+					   psSurfInfoInt, psBuffersInt);
+
+DCDisplayContextConfigureCheck_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigureCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDCDisplayContextDestroyIN_UI8,
+				    IMG_UINT8 * psDCDisplayContextDestroyOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDisplayContextDestroyIN->
+					      hDisplayContext,
+					      PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	if (unlikely
+	    ((psDCDisplayContextDestroyOUT->eError != PVRSRV_OK)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCDisplayContextDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAllocpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAlloc(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferAllocIN_UI8,
+			  IMG_UINT8 * psDCBufferAllocOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERALLOC *psDCBufferAllocIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *psDCBufferAllocOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferAllocIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAllocOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError =
+	    DCBufferAlloc(psDisplayContextInt,
+			  &psDCBufferAllocIN->sSurfInfo,
+			  &psDCBufferAllocOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psDCBufferAllocOUT->hBuffer,
+							       (void *)psBufferInt,
+							       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+							       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							       (PFN_HANDLE_RELEASE) &
+							       _DCBufferAllocpsBufferIntRelease);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferAlloc_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAllocOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferImportpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static_assert(DC_MAX_PLANES <= IMG_UINT32_MAX,
+	      "DC_MAX_PLANES must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCBufferImport(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psDCBufferImportIN_UI8,
+			   IMG_UINT8 * psDCBufferImportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *psDCBufferImportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *psDCBufferImportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferImportIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PMR **psImportInt = NULL;
+	IMG_HANDLE *hImportInt2 = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCBufferImportIN->ui32NumPlanes > DC_MAX_PLANES))
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCBufferImport_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCBufferImport_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCBufferImportIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCBufferImportIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCBufferImportOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+
+	if (psDCBufferImportIN->ui32NumPlanes != 0)
+	{
+		psImportInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psImportInt, 0, psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *));
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *);
+		hImportInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hImportInt2, (const void __user *)psDCBufferImportIN->phImport,
+		     psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDCBufferImportOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCBufferImport_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferImportOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+			/* Look up the address from the handle */
+			psDCBufferImportOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psImportInt[i],
+						       hImportInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError =
+	    DCBufferImport(psDisplayContextInt,
+			   psDCBufferImportIN->ui32NumPlanes,
+			   psImportInt, &psDCBufferImportIN->sSurfAttrib, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferImport_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								&psDCBufferImportOUT->hBuffer,
+								(void *)psBufferInt,
+								PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								(PFN_HANDLE_RELEASE) &
+								_DCBufferImportpsBufferIntRelease);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferImport_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hImportInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psImportInt && psImportInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hImportInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferImportOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCBufferImportOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferFree(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCBufferFreeIN_UI8,
+			 IMG_UINT8 * psDCBufferFreeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERFREE *psDCBufferFreeIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERFREE *psDCBufferFreeOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferFreeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferFreeIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferFreeOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferFreeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferFree_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferFree_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnimport(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCBufferUnimportIN_UI8,
+			     IMG_UINT8 * psDCBufferUnimportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *psDCBufferUnimportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *psDCBufferUnimportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnimportOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnimportIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferUnimportOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnimportOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnimport_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnimport_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferPinhPinHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferUnpin((DC_PIN_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferPin(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDCBufferPinIN_UI8,
+			IMG_UINT8 * psDCBufferPinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERPIN *psDCBufferPinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERPIN *psDCBufferPinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferPinIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	DC_PIN_HANDLE hPinHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferPinOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = DCBufferPin(psBufferInt, &hPinHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferPin_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							     &psDCBufferPinOUT->hPinHandle,
+							     (void *)hPinHandleInt,
+							     PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+							     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							     (PFN_HANDLE_RELEASE) &
+							     _DCBufferPinhPinHandleIntRelease);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferPin_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferPinOUT->eError != PVRSRV_OK)
+	{
+		if (hPinHandleInt)
+		{
+			DCBufferUnpin(hPinHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnpin(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferUnpinIN_UI8,
+			  IMG_UINT8 * psDCBufferUnpinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *psDCBufferUnpinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *psDCBufferUnpinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnpinOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnpinIN->hPinHandle,
+					      PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE);
+	if (unlikely((psDCBufferUnpinOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnpinOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnpin_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnpin_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAcquirepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferRelease((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferAcquireIN_UI8,
+			    IMG_UINT8 * psDCBufferAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *psDCBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *psDCBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferAcquireIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAcquireOUT->eError = DCBufferAcquire(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psDCBufferAcquireOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _DCBufferAcquirepsExtMemIntRelease);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferRelease(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferReleaseIN_UI8,
+			    IMG_UINT8 * psDCBufferReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *psDCBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *psDCBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psDCBufferReleaseIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psDCBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferRelease_exit:
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigure2(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psDCDisplayContextConfigure2IN_UI8,
+				       IMG_UINT8 * psDCDisplayContextConfigure2OUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2IN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2OUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2OUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigure2IN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCDisplayContextConfigure2IN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigure2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigure2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigure2OUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigure2IN->psSurfInfo,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigure2IN->phBuffers,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigure2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigure2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigure2OUT->eError =
+	    DCDisplayContextConfigure(psDisplayContextInt,
+				      psDCDisplayContextConfigure2IN->ui32PipeCount,
+				      psSurfInfoInt,
+				      psBuffersInt,
+				      psDCDisplayContextConfigure2IN->ui32DisplayPeriod,
+				      psDCDisplayContextConfigure2IN->ui32MaxDepth,
+				      psDCDisplayContextConfigure2IN->hAcquireFence,
+				      psDCDisplayContextConfigure2IN->hReleaseFenceTimeline,
+				      &psDCDisplayContextConfigure2OUT->hReleaseFence);
+
+DCDisplayContextConfigure2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigure2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+
+/*
+ * Register all DC functions with services
+ */
+PVRSRV_ERROR InitDCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT,
+			      PVRSRVBridgeDCDevicesQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE,
+			      PVRSRVBridgeDCDevicesEnumerate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE,
+			      PVRSRVBridgeDCDeviceAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE,
+			      PVRSRVBridgeDCDeviceRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO, PVRSRVBridgeDCGetInfo,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT,
+			      PVRSRVBridgeDCPanelQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY,
+			      PVRSRVBridgeDCPanelQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY,
+			      PVRSRVBridgeDCFormatQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY, PVRSRVBridgeDCDimQuery,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK, PVRSRVBridgeDCSetBlank,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING,
+			      PVRSRVBridgeDCSetVSyncReporting, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY,
+			      PVRSRVBridgeDCLastVSyncQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE,
+			      PVRSRVBridgeDCSystemBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE,
+			      PVRSRVBridgeDCSystemBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE,
+			      PVRSRVBridgeDCDisplayContextCreate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK,
+			      PVRSRVBridgeDCDisplayContextConfigureCheck, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY,
+			      PVRSRVBridgeDCDisplayContextDestroy, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC,
+			      PVRSRVBridgeDCBufferAlloc, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT,
+			      PVRSRVBridgeDCBufferImport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE,
+			      PVRSRVBridgeDCBufferFree, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT,
+			      PVRSRVBridgeDCBufferUnimport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN,
+			      PVRSRVBridgeDCBufferPin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN,
+			      PVRSRVBridgeDCBufferUnpin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE,
+			      PVRSRVBridgeDCBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE,
+			      PVRSRVBridgeDCBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2,
+			      PVRSRVBridgeDCDisplayContextConfigure2, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dc functions with services
+ */
+void DeinitDCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_bridge.h
new file mode 100644
index 000000000000..bfa6bfb9c037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_bridge.h
@@ -0,0 +1,111 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+#define CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_devicememhistory_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut);
+
+#endif /* CLIENT_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
new file mode 100644
index 000000000000..a593a7e2cc7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
@@ -0,0 +1,195 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for devicememhistory
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_devicememhistory_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  uiOffset,
+				  sDevVAddr,
+				  uiSize,
+				  puiText,
+				  ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    uiOffset,
+				    sDevVAddr,
+				    uiSize,
+				    puiText,
+				    ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryMapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					sBaseDevVAddr,
+					ui32ui32StartPage,
+					ui32NumPages,
+					uiAllocSize,
+					puiText,
+					ui32Log2PageSize,
+					ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryUnmapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  sBaseDevVAddr,
+					  ui32ui32StartPage,
+					  ui32NumPages,
+					  uiAllocSize,
+					  puiText,
+					  ui32Log2PageSize,
+					  ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   uiOffset,
+					   sDevVAddr,
+					   uiSize,
+					   puiText,
+					   ui32Log2PageSize,
+					   ui32AllocPageCount,
+					   pui32AllocPageIndices,
+					   ui32FreePageCount,
+					   pui32FreePageIndices,
+					   ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/common_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/common_devicememhistory_bridge.h
new file mode 100644
index 000000000000..800f98dd2836
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/common_devicememhistory_bridge.h
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DEVICEMEMHISTORY_BRIDGE_H
+#define COMMON_DEVICEMEMHISTORY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST			(PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4)
+
+/*******************************************
+            DevicememHistoryMap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP;
+
+/* Bridge out structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP;
+
+/*******************************************
+            DevicememHistoryUnmap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP;
+
+/* Bridge out structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP;
+
+/*******************************************
+            DevicememHistoryMapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE;
+
+/*******************************************
+            DevicememHistoryUnmapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/*******************************************
+            DevicememHistorySparseChange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE;
+
+/* Bridge out structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE;
+
+#endif /* COMMON_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/server_devicememhistory_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/server_devicememhistory_bridge.c
new file mode 100644
index 000000000000..72c6a86a0ec8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/server_devicememhistory_bridge.c
@@ -0,0 +1,835 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+#include "common_devicememhistory_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMap(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevicememHistoryMapIN_UI8,
+				IMG_UINT8 * psDevicememHistoryMapOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *psDevicememHistoryMapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *psDevicememHistoryMapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryMapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryMapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryMapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryMap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryMapOUT->eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  psDevicememHistoryMapIN->uiOffset,
+				  psDevicememHistoryMapIN->sDevVAddr,
+				  psDevicememHistoryMapIN->uiSize,
+				  uiTextInt,
+				  psDevicememHistoryMapIN->ui32Log2PageSize,
+				  psDevicememHistoryMapIN->ui32AllocationIndex,
+				  &psDevicememHistoryMapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmap(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevicememHistoryUnmapIN_UI8,
+				  IMG_UINT8 * psDevicememHistoryUnmapOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryUnmapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryUnmapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryUnmapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryUnmapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryUnmap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryUnmapOUT->eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    psDevicememHistoryUnmapIN->uiOffset,
+				    psDevicememHistoryUnmapIN->sDevVAddr,
+				    psDevicememHistoryUnmapIN->uiSize,
+				    uiTextInt,
+				    psDevicememHistoryUnmapIN->ui32Log2PageSize,
+				    psDevicememHistoryUnmapIN->ui32AllocationIndex,
+				    &psDevicememHistoryUnmapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryUnmap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMapVRange(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeIN_UI8,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryMapVRangeOUT->eError =
+	    DevicememHistoryMapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					psDevicememHistoryMapVRangeIN->sBaseDevVAddr,
+					psDevicememHistoryMapVRangeIN->ui32ui32StartPage,
+					psDevicememHistoryMapVRangeIN->ui32NumPages,
+					psDevicememHistoryMapVRangeIN->uiAllocSize,
+					uiTextInt,
+					psDevicememHistoryMapVRangeIN->ui32Log2PageSize,
+					psDevicememHistoryMapVRangeIN->ui32AllocationIndex,
+					&psDevicememHistoryMapVRangeOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmapVRange(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeIN_UI8,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistoryUnmapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapVRangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryUnmapVRangeOUT->eError =
+	    DevicememHistoryUnmapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					  psDevicememHistoryUnmapVRangeIN->sBaseDevVAddr,
+					  psDevicememHistoryUnmapVRangeIN->ui32ui32StartPage,
+					  psDevicememHistoryUnmapVRangeIN->ui32NumPages,
+					  psDevicememHistoryUnmapVRangeIN->uiAllocSize,
+					  uiTextInt,
+					  psDevicememHistoryUnmapVRangeIN->ui32Log2PageSize,
+					  psDevicememHistoryUnmapVRangeIN->ui32AllocationIndex,
+					  &psDevicememHistoryUnmapVRangeOUT->
+					  ui32AllocationIndexOut);
+
+DevicememHistoryUnmapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistorySparseChange(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psDevicememHistorySparseChangeIN_UI8,
+					 IMG_UINT8 * psDevicememHistorySparseChangeOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistorySparseChangeIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32AllocPageCount *
+	     sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32FreePageCount *
+	     sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32AllocPageCount >
+	     PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistorySparseChangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistorySparseChangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistorySparseChangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistorySparseChange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32AllocPageIndices,
+		     psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32FreePageIndices,
+		     psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistorySparseChangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistorySparseChangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistorySparseChange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistorySparseChangeOUT->eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   psDevicememHistorySparseChangeIN->uiOffset,
+					   psDevicememHistorySparseChangeIN->sDevVAddr,
+					   psDevicememHistorySparseChangeIN->uiSize,
+					   uiTextInt,
+					   psDevicememHistorySparseChangeIN->ui32Log2PageSize,
+					   psDevicememHistorySparseChangeIN->ui32AllocPageCount,
+					   ui32AllocPageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32FreePageCount,
+					   ui32FreePageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32AllocationIndex,
+					   &psDevicememHistorySparseChangeOUT->
+					   ui32AllocationIndexOut);
+
+DevicememHistorySparseChange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistorySparseChangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pDEVICEMEMHISTORYBridgeLock;
+
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+
+/*
+ * Register all DEVICEMEMHISTORY functions with services
+ */
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pDEVICEMEMHISTORYBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP,
+			      PVRSRVBridgeDevicememHistoryMap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP,
+			      PVRSRVBridgeDevicememHistoryUnmap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryMapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryUnmapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE,
+			      PVRSRVBridgeDevicememHistorySparseChange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all devicememhistory functions with services
+ */
+void DeinitDEVICEMEMHISTORYBridge(void)
+{
+	OSLockDestroy(pDEVICEMEMHISTORYBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/common_di_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/common_di_bridge.h
new file mode 100644
index 000000000000..8591006140b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/common_di_bridge.h
@@ -0,0 +1,153 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DI_BRIDGE_H
+#define COMMON_DI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "pvr_dicommon.h"
+
+#define PVRSRV_BRIDGE_DI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DI_DICREATECONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DI_DIREADENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DI_DIWRITEENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DI_DILISTALLENTRIES			PVRSRV_BRIDGE_DI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DI_CMD_LAST			(PVRSRV_BRIDGE_DI_CMD_FIRST+4)
+
+/*******************************************
+            DICreateContext
+ *******************************************/
+
+/* Bridge in structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_IN_DICREATECONTEXT_TAG
+{
+	IMG_CHAR *puiStreamName;
+} __packed PVRSRV_BRIDGE_IN_DICREATECONTEXT;
+
+/* Bridge out structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_OUT_DICREATECONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_CHAR *puiStreamName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DICREATECONTEXT;
+
+/*******************************************
+            DIDestroyContext
+ *******************************************/
+
+/* Bridge in structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT;
+
+/* Bridge out structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT;
+
+/*******************************************
+            DIReadEntry
+ *******************************************/
+
+/* Bridge in structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIREADENTRY_TAG
+{
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+} __packed PVRSRV_BRIDGE_IN_DIREADENTRY;
+
+/* Bridge out structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIREADENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIREADENTRY;
+
+/*******************************************
+            DIWriteEntry
+ *******************************************/
+
+/* Bridge in structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIWRITEENTRY_TAG
+{
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+	const IMG_CHAR *puiValue;
+	IMG_UINT32 ui32ValueSize;
+} __packed PVRSRV_BRIDGE_IN_DIWRITEENTRY;
+
+/* Bridge out structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIWRITEENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIWRITEENTRY;
+
+/*******************************************
+            DIListAllEntries
+ *******************************************/
+
+/* Bridge in structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_IN_DILISTALLENTRIES_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DILISTALLENTRIES;
+
+/* Bridge out structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_OUT_DILISTALLENTRIES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DILISTALLENTRIES;
+
+#endif /* COMMON_DI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/server_di_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/server_di_bridge.c
new file mode 100644
index 000000000000..154bccdaae5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/server_di_bridge.c
@@ -0,0 +1,652 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "di_impl_brg.h"
+
+#include "common_di_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DICreateContextpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DIDestroyContextKM((DI_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDICreateContext(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDICreateContextIN_UI8,
+			    IMG_UINT8 * psDICreateContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DICREATECONTEXT *psDICreateContextIN =
+	    (PVRSRV_BRIDGE_IN_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DICREATECONTEXT *psDICreateContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextOUT_UI8, 0);
+
+	IMG_CHAR *puiStreamNameInt = NULL;
+	DI_CONTEXT *psContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	PVR_UNREFERENCED_PARAMETER(psDICreateContextIN);
+
+	psDICreateContextOUT->puiStreamName = psDICreateContextIN->puiStreamName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDICreateContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DICreateContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDICreateContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDICreateContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDICreateContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DICreateContext_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		puiStreamNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	psDICreateContextOUT->eError = DICreateContextKM(puiStreamNameInt, &psContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		goto DICreateContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDICreateContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDICreateContextOUT->hContext,
+								 (void *)psContextInt,
+								 PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								 (PFN_HANDLE_RELEASE) &
+								 _DICreateContextpsContextIntRelease);
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DICreateContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamNameInt) && ((PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDICreateContextOUT->puiStreamName, puiStreamNameInt,
+		      (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psDICreateContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DICreateContext_exit;
+		}
+	}
+
+DICreateContext_exit:
+
+	if (psDICreateContextOUT->eError != PVRSRV_OK)
+	{
+		if (psDICreateContextOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDICreateContextOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DIDestroyContextKM(psContextInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDICreateContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIDestroyContext(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIDestroyContextIN_UI8,
+			     IMG_UINT8 * psDIDestroyContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *psDIDestroyContextIN =
+	    (PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *psDIDestroyContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDIDestroyContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDIDestroyContextIN->hContext,
+					      PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	if (unlikely((psDIDestroyContextOUT->eError != PVRSRV_OK) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDIDestroyContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIDestroyContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DIDestroyContext_exit:
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIReadEntry(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDIReadEntryIN_UI8,
+			IMG_UINT8 * psDIReadEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIREADENTRY *psDIReadEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIREADENTRY *psDIReadEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIReadEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIReadEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIReadEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIReadEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIReadEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIReadEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIReadEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIReadEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIReadEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIReadEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIReadEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIReadEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIReadEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIReadEntryOUT->eError =
+	    DIReadEntryKM(psContextInt,
+			  uiEntryPathInt, psDIReadEntryIN->ui64Offset, psDIReadEntryIN->ui64Size);
+
+DIReadEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIReadEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIWriteEntry(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDIWriteEntryIN_UI8,
+			 IMG_UINT8 * psDIWriteEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIWRITEENTRY *psDIWriteEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIWRITEENTRY *psDIWriteEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIWriteEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+	IMG_CHAR *uiValueInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psDIWriteEntryIN->ui32ValueSize > DI_IMPL_BRG_PATH_LEN))
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DIWriteEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIWriteEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIWriteEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIWriteEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIWriteEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIWriteEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIWriteEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDIWriteEntryIN->ui32ValueSize != 0)
+	{
+		uiValueInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiValueInt, (const void __user *)psDIWriteEntryIN->puiValue,
+		     psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiValueInt)[(psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) -
+					  1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIWriteEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIWriteEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIWriteEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIWriteEntryOUT->eError =
+	    DIWriteEntryKM(psContextInt,
+			   uiEntryPathInt, psDIWriteEntryIN->ui32ValueSize, uiValueInt);
+
+DIWriteEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIWriteEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIListAllEntries(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIListAllEntriesIN_UI8,
+			     IMG_UINT8 * psDIListAllEntriesOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DILISTALLENTRIES *psDIListAllEntriesIN =
+	    (PVRSRV_BRIDGE_IN_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *psDIListAllEntriesOUT =
+	    (PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIListAllEntriesIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIListAllEntriesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIListAllEntriesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIListAllEntries_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIListAllEntriesOUT->eError = DIListAllEntriesKM(psContextInt);
+
+DIListAllEntries_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+
+/*
+ * Register all DI functions with services
+ */
+PVRSRV_ERROR InitDIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT,
+			      PVRSRVBridgeDICreateContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DICREATECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DICREATECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT,
+			      PVRSRVBridgeDIDestroyContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY,
+			      PVRSRVBridgeDIReadEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIREADENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIREADENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY,
+			      PVRSRVBridgeDIWriteEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIWRITEENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIWRITEENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES,
+			      PVRSRVBridgeDIListAllEntries, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DILISTALLENTRIES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DILISTALLENTRIES));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all di functions with services
+ */
+void DeinitDIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/common_dma_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/common_dma_bridge.h
new file mode 100644
index 000000000000..dafa475b9c9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/common_dma_bridge.h
@@ -0,0 +1,123 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMA_BRIDGE_H
+#define COMMON_DMA_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DMA_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMA_DMATRANSFER			PVRSRV_BRIDGE_DMA_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE			PVRSRV_BRIDGE_DMA_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS			PVRSRV_BRIDGE_DMA_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMA_CMD_LAST			(PVRSRV_BRIDGE_DMA_CMD_FIRST+2)
+
+/*******************************************
+            DmaTransfer
+ *******************************************/
+
+/* Bridge in structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_IN_DMATRANSFER_TAG
+{
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32NumDMAs;
+	IMG_UINT32 ui32uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DMATRANSFER;
+
+/* Bridge out structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_OUT_DMATRANSFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMATRANSFER;
+
+/*******************************************
+            DmaSparseMappingTable
+ *******************************************/
+
+/* Bridge in structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_BOOL *pbTable;
+	IMG_UINT32 ui32SizeInPages;
+} __packed PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE;
+
+/* Bridge out structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_BOOL *pbTable;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE;
+
+/*******************************************
+            DmaDeviceParams
+ *******************************************/
+
+/* Bridge in structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_IN_DMADEVICEPARAMS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DMADEVICEPARAMS;
+
+/* Bridge out structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DmaBuffAlign;
+	IMG_UINT32 ui32DmaTransferMult;
+} __packed PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS;
+
+#endif /* COMMON_DMA_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/server_dma_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/server_dma_bridge.c
new file mode 100644
index 000000000000..7f2c5dd495e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/server_dma_bridge.c
@@ -0,0 +1,490 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dma_km.h"
+
+#include "common_dma_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(MAX_DMA_OPS <= IMG_UINT32_MAX, "MAX_DMA_OPS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaTransfer(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDmaTransferIN_UI8,
+			IMG_UINT8 * psDmaTransferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMATRANSFER *psDmaTransferIN =
+	    (PVRSRV_BRIDGE_IN_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMATRANSFER *psDmaTransferOUT =
+	    (PVRSRV_BRIDGE_OUT_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) + 0;
+
+	if (unlikely(psDmaTransferIN->ui32NumDMAs > MAX_DMA_OPS))
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaTransfer_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaTransfer_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaTransferIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaTransferIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaTransferOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psDmaTransferIN->ui32NumDMAs * sizeof(PMR *));
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psDmaTransferIN->phPMR,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psDmaTransferIN->pui64Address,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psDmaTransferIN->puiOffset,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psDmaTransferIN->puiSize,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+			/* Look up the address from the handle */
+			psDmaTransferOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDmaTransferOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaTransferOUT->eError =
+	    DmaTransfer(psConnection, OSGetDevNode(psConnection),
+			psDmaTransferIN->ui32NumDMAs,
+			psPMRInt,
+			ui64AddressInt,
+			uiOffsetInt,
+			uiSizeInt, psDmaTransferIN->ui32uiFlags, psDmaTransferIN->hUpdateTimeline);
+
+DmaTransfer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaTransferOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(32 <= IMG_UINT32_MAX, "32 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaSparseMappingTable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDmaSparseMappingTableIN_UI8,
+				  IMG_UINT8 * psDmaSparseMappingTableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableIN =
+	    (PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDmaSparseMappingTableIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_BOOL *pbTableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) + 0;
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages > 32)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	psDmaSparseMappingTableOUT->pbTable = psDmaSparseMappingTableIN->pbTable;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaSparseMappingTableIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaSparseMappingTableIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaSparseMappingTable_exit;
+			}
+		}
+	}
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages != 0)
+	{
+		pbTableInt = (IMG_BOOL *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDmaSparseMappingTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DmaSparseMappingTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaSparseMappingTableOUT->eError =
+	    DmaSparseMappingTable(psPMRInt,
+				  psDmaSparseMappingTableIN->uiOffset,
+				  psDmaSparseMappingTableIN->ui32SizeInPages, pbTableInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		goto DmaSparseMappingTable_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pbTableInt) && ((psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDmaSparseMappingTableOUT->pbTable, pbTableInt,
+		      (psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL))) !=
+		     PVRSRV_OK))
+		{
+			psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaSparseMappingTable_exit;
+		}
+	}
+
+DmaSparseMappingTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaSparseMappingTableOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDmaDeviceParams(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDmaDeviceParamsIN_UI8,
+			    IMG_UINT8 * psDmaDeviceParamsOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *psDmaDeviceParamsIN =
+	    (PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *psDmaDeviceParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psDmaDeviceParamsIN);
+
+	psDmaDeviceParamsOUT->eError =
+	    DmaDeviceParams(psConnection, OSGetDevNode(psConnection),
+			    &psDmaDeviceParamsOUT->ui32DmaBuffAlign,
+			    &psDmaDeviceParamsOUT->ui32DmaTransferMult);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+
+/*
+ * Register all DMA functions with services
+ */
+PVRSRV_ERROR InitDMABridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER,
+			      PVRSRVBridgeDmaTransfer, NULL, sizeof(PVRSRV_BRIDGE_IN_DMATRANSFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMATRANSFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE,
+			      PVRSRVBridgeDmaSparseMappingTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS,
+			      PVRSRVBridgeDmaDeviceParams, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dma functions with services
+ */
+void DeinitDMABridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/common_dmabuf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/common_dmabuf_bridge.h
new file mode 100644
index 000000000000..cc631e3e1116
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/common_dmabuf_bridge.h
@@ -0,0 +1,145 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMABUF_BRIDGE_H
+#define COMMON_DMABUF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_DMABUF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DMABUF_CMD_LAST			(PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3)
+
+/*******************************************
+            PhysmemImportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF;
+
+/* Bridge out structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF;
+
+/*******************************************
+            PhysmemExportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF;
+
+/* Bridge out structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF;
+
+/*******************************************
+            PhysmemExportGemHandle
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE;
+
+/* Bridge out structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Handle;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE;
+
+/*******************************************
+            PhysmemImportSparseDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_SIZE_T uiChunkSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF;
+
+/* Bridge out structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF;
+
+#endif /* COMMON_DMABUF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/server_dmabuf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/server_dmabuf_bridge.c
new file mode 100644
index 000000000000..89aa1477f55a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/server_dmabuf_bridge.c
@@ -0,0 +1,574 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "physmem_dmabuf.h"
+#include "pmr.h"
+
+#include "common_dmabuf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemImportDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemImportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemImportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufOUT_UI8,
+								      0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPhysmemImportDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportDmaBufIN->puiName,
+		     psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportDmaBuf_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR))
+					 - 1] = '\0';
+	}
+
+	psPhysmemImportDmaBufOUT->eError =
+	    PhysmemImportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPhysmemImportDmaBufIN->ifd,
+				psPhysmemImportDmaBufIN->uiFlags,
+				psPhysmemImportDmaBufIN->ui32NameSize,
+				uiNameInt,
+				&psPMRPtrInt,
+				&psPhysmemImportDmaBufOUT->uiSize,
+				&psPhysmemImportDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportDmaBufOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psPhysmemImportDmaBufOUT->
+								     hPMRPtr, (void *)psPMRPtrInt,
+								     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _PhysmemImportDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportDmaBuf_exit:
+
+	if (psPhysmemImportDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemExportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemExportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psPhysmemExportDmaBufIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportDmaBufOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportDmaBuf_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportDmaBufOUT->eError =
+	    PhysmemExportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPMRInt, &psPhysmemExportDmaBufOUT->iFd);
+
+PhysmemExportDmaBuf_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportGemHandle(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemExportGemHandleIN_UI8,
+				   IMG_UINT8 * psPhysmemExportGemHandleOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPhysmemExportGemHandleIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportGemHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportGemHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportGemHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportGemHandleOUT->eError =
+	    PhysmemExportGemHandle(psConnection, OSGetDevNode(psConnection),
+				   psPMRInt, &psPhysmemExportGemHandleOUT->ui32Handle);
+
+PhysmemExportGemHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemImportSparseDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportSparseDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufIN_UI8,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufOUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (unlikely(psPhysmemImportSparseDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportSparseDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportSparseDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportSparseDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemImportSparseDmaBufIN->pui32MappingTable,
+		     psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+	}
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportSparseDmaBufIN->puiName,
+		     psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+		((IMG_CHAR *)
+		 uiNameInt)[(psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PhysmemImportSparseDmaBuf(psConnection, OSGetDevNode(psConnection),
+				      psPhysmemImportSparseDmaBufIN->ifd,
+				      psPhysmemImportSparseDmaBufIN->uiFlags,
+				      psPhysmemImportSparseDmaBufIN->uiChunkSize,
+				      psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks,
+				      psPhysmemImportSparseDmaBufIN->ui32NumVirtChunks,
+				      ui32MappingTableInt,
+				      psPhysmemImportSparseDmaBufIN->ui32NameSize,
+				      uiNameInt,
+				      &psPMRPtrInt,
+				      &psPhysmemImportSparseDmaBufOUT->uiSize,
+				      &psPhysmemImportSparseDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psPhysmemImportSparseDmaBufOUT->hPMRPtr, (void *)psPMRPtrInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PhysmemImportSparseDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportSparseDmaBuf_exit:
+
+	if (psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportSparseDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+/*
+ * Register all DMABUF functions with services
+ */
+PVRSRV_ERROR InitDMABUFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF,
+			      PVRSRVBridgePhysmemImportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF,
+			      PVRSRVBridgePhysmemExportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE,
+			      PVRSRVBridgePhysmemExportGemHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF,
+			      PVRSRVBridgePhysmemImportSparseDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dmabuf functions with services
+ */
+void DeinitDMABUFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF,
+				PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_bridge.h
new file mode 100644
index 000000000000..2c5d18843120
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_bridge.h
@@ -0,0 +1,64 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_HTBUFFER_BRIDGE_H
+#define CLIENT_HTBUFFER_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_htbuffer_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode);
+
+#endif /* CLIENT_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_direct_bridge.c
new file mode 100644
index 000000000000..3e6295cdfdf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_direct_bridge.c
@@ -0,0 +1,70 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for htbuffer
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_htbuffer_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#include "htbserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError =
+	    HTBControlKM(ui32NumGroups,
+			 pui32GroupEnable, ui32LogLevel, ui32EnablePID, ui32LogMode, ui32OpMode);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/common_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/common_htbuffer_bridge.h
new file mode 100644
index 000000000000..32be202186dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/common_htbuffer_bridge.h
@@ -0,0 +1,82 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_HTBUFFER_BRIDGE_H
+#define COMMON_HTBUFFER_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST			0
+#define PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL			PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_LAST			(PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0)
+
+/*******************************************
+            HTBControl
+ *******************************************/
+
+/* Bridge in structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_IN_HTBCONTROL_TAG
+{
+	IMG_UINT32 *pui32GroupEnable;
+	IMG_UINT32 ui32EnablePID;
+	IMG_UINT32 ui32LogLevel;
+	IMG_UINT32 ui32LogMode;
+	IMG_UINT32 ui32NumGroups;
+	IMG_UINT32 ui32OpMode;
+} __packed PVRSRV_BRIDGE_IN_HTBCONTROL;
+
+/* Bridge out structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_OUT_HTBCONTROL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HTBCONTROL;
+
+#endif /* COMMON_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/server_htbuffer_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/server_htbuffer_bridge.c
new file mode 100644
index 000000000000..9b730d28e5b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/server_htbuffer_bridge.c
@@ -0,0 +1,227 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "htbserver.h"
+
+#include "common_htbuffer_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(HTB_FLAG_NUM_EL <= IMG_UINT32_MAX,
+	      "HTB_FLAG_NUM_EL must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHTBControl(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psHTBControlIN_UI8,
+		       IMG_UINT8 * psHTBControlOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HTBCONTROL *psHTBControlIN =
+	    (PVRSRV_BRIDGE_IN_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HTBCONTROL *psHTBControlOUT =
+	    (PVRSRV_BRIDGE_OUT_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlOUT_UI8, 0);
+
+	IMG_UINT32 *ui32GroupEnableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psHTBControlIN->ui32NumGroups > HTB_FLAG_NUM_EL))
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HTBControl_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HTBControl_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHTBControlIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHTBControlIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHTBControlOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HTBControl_exit;
+			}
+		}
+	}
+
+	if (psHTBControlIN->ui32NumGroups != 0)
+	{
+		ui32GroupEnableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32GroupEnableInt,
+		     (const void __user *)psHTBControlIN->pui32GroupEnable,
+		     psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psHTBControlOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HTBControl_exit;
+		}
+	}
+
+	psHTBControlOUT->eError =
+	    HTBControlKM(psHTBControlIN->ui32NumGroups,
+			 ui32GroupEnableInt,
+			 psHTBControlIN->ui32LogLevel,
+			 psHTBControlIN->ui32EnablePID,
+			 psHTBControlIN->ui32LogMode, psHTBControlIN->ui32OpMode);
+
+HTBControl_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHTBControlOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pHTBUFFERBridgeLock;
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+
+/*
+ * Register all HTBUFFER functions with services
+ */
+PVRSRV_ERROR InitHTBUFFERBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pHTBUFFERBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL,
+			      PVRSRVBridgeHTBControl, pHTBUFFERBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_HTBCONTROL),
+			      sizeof(PVRSRV_BRIDGE_OUT_HTBCONTROL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all htbuffer functions with services
+ */
+void DeinitHTBUFFERBridge(void)
+{
+	OSLockDestroy(pHTBUFFERBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL);
+
+}
+#else /* EXCLUDE_HTBUFFER_BRIDGE */
+/* This bridge is conditional on EXCLUDE_HTBUFFER_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitHTBUFFERBridge() \
+	PVRSRV_OK
+
+#define DeinitHTBUFFERBridge()
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_bridge.h
new file mode 100644
index 000000000000..d5ba6dd74da3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_bridge.h
@@ -0,0 +1,234 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_MM_BRIDGE_H
+#define CLIENT_MM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_mm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge,
+							IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags,
+						IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats);
+
+#endif /* CLIENT_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_direct_bridge.c
new file mode 100644
index 000000000000..c1810e959ca8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_direct_bridge.c
@@ -0,0 +1,719 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for mm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_mm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRExportPMR(psPMRInt, &psPMRExportInt, pui64Size, pui32Log2Contig, pui64Password);
+
+	*phPMRExport = psPMRExportInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMR);
+	PVR_UNREFERENCED_PARAMETER(phPMRExport);
+	PVR_UNREFERENCED_PARAMETER(pui64Size);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2Contig);
+	PVR_UNREFERENCED_PARAMETER(pui64Password);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError = PMRUnexportPMR(psPMRExportInt);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRGetUID(psPMRInt, pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psBufferInt;
+	PMR *psExtMemInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psBufferInt = (PMR *) hBuffer;
+
+	eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+
+	*phExtMem = psExtMemInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtMemInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtMemInt = (PMR *) hExtMem;
+
+	eError = PMRUnmakeLocalImportHandle(psExtMemInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PMR *psPMRInt = NULL;
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError =
+	    PhysmemImportPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     psPMRExportInt,
+			     ui64uiPassword, ui64uiSize, ui32uiLog2Contig, &psPMRInt);
+
+	*phPMR = psPMRInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+	PVR_UNREFERENCED_PARAMETER(ui64uiPassword);
+	PVR_UNREFERENCED_PARAMETER(ui64uiSize);
+	PVR_UNREFERENCED_PARAMETER(ui32uiLog2Contig);
+	PVR_UNREFERENCED_PARAMETER(phPMR);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtHandleInt;
+	PMR *psPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtHandleInt = (PMR *) hExtHandle;
+
+	eError = PMRLocalImportPMR(psExtHandleInt, &psPMRInt, puiSize, puiAlign);
+
+	*phPMR = psPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRUnrefPMR(psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRPtrInt = NULL;
+
+	eError =
+	    PhysmemNewRamBackedPMR_direct(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  uiSize,
+					  ui32NumPhysChunks,
+					  ui32NumVirtChunks,
+					  pui32MappingTable,
+					  ui32Log2PageSize,
+					  uiFlags,
+					  ui32AnnotationLength,
+					  puiAnnotation,
+					  ui32PID, &psPMRPtrInt, ui32PDumpFlags, puiOutFlags);
+
+	*phPMRPtr = psPMRPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	eError =
+	    DevmemIntCtxCreate(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       bbKernelMemoryCtx,
+			       &psDevMemServerContextInt, &hPrivDataInt, pui32CPUCacheLineSize);
+
+	*phDevMemServerContext = psDevMemServerContextInt;
+	*phPrivData = hPrivDataInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError = DevmemIntCtxDestroy(psDevmemServerContextInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				ui32HeapConfigIndex, ui32HeapIndex, &psDevmemHeapPtrInt);
+
+	*phDevmemHeapPtr = psDevmemHeapPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeapInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemHeapInt = (DEVMEMINT_HEAP *) hDevmemHeap;
+
+	eError = DevmemIntHeapDestroy(psDevmemHeapInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnmapPMR(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemIntReserveRange(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psDevmemServerHeapInt,
+				  sAddress, uiLength, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	PMR *psPMRInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemIntReserveRangeAndMapPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   psDevmemServerHeapInt,
+					   sAddress,
+					   uiLength, psPMRInt, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemIntChangeSparse(ui32AllocPageCount,
+				  pui32AllocPageIndices,
+				  ui32FreePageCount,
+				  pui32FreePageIndices, ui32SparseFlags, psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntIsVDevAddrValid(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, sAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries)
+{
+#if defined(RGX_FEATURE_FBCDC)
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntInvalidateFBSCTable(psDevmemCtxInt, ui64FBSCEntries);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(ui64FBSCEntries);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   pui32NumHeapConfigs);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     ui32HeapConfigIndex, pui32NumHeaps);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigName(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  ui32HeapConfigIndex, ui32HeapConfigNameBufSz, puiHeapConfigName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapDetails(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       ui32HeapConfigIndex,
+			       ui32HeapIndex,
+			       ui32HeapNameBufSz,
+			       puiHeapNameOut,
+			       psDevVAddrBase,
+			       puiHeapLength,
+			       puiReservedRegionLength,
+			       pui32Log2DataPageSizeOut, pui32Log2ImportAlignmentOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, bRegister);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfoKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				       ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), peHeap);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntGetFaultAddress(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, psFaultAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVStatsUpdateOOMStat(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32ui32StatType, ui32pid);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32ui32StatType);
+	PVR_UNREFERENCED_PARAMETER(ui32pid);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt, sAddress, uiLength, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       ui32PageCount, ui32PhysPageOffset, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnmapPages(psReservationInt, ui32VirtPageOffset, ui32PageCount);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     ui32PageCount, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/common_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/common_mm_bridge.h
new file mode 100644
index 000000000000..3b3c8c2dc215
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/common_mm_bridge.h
@@ -0,0 +1,757 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MM_BRIDGE_H
+#define COMMON_MM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MM_PMREXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_MM_PMRGETUID			PVRSRV_BRIDGE_MM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_MM_PMRIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_MM_PMRUNREFPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+9
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+10
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+11
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+12
+#define PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+13
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+14
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+15
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+16
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+17
+#define PVRSRV_BRIDGE_MM_CHANGESPARSEMEM			PVRSRV_BRIDGE_MM_CMD_FIRST+18
+#define PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID			PVRSRV_BRIDGE_MM_CMD_FIRST+19
+#define PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE			PVRSRV_BRIDGE_MM_CMD_FIRST+20
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+21
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+22
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME			PVRSRV_BRIDGE_MM_CMD_FIRST+23
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS			PVRSRV_BRIDGE_MM_CMD_FIRST+24
+#define PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM			PVRSRV_BRIDGE_MM_CMD_FIRST+25
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO			PVRSRV_BRIDGE_MM_CMD_FIRST+26
+#define PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP			PVRSRV_BRIDGE_MM_CMD_FIRST+27
+#define PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS			PVRSRV_BRIDGE_MM_CMD_FIRST+28
+#define PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT			PVRSRV_BRIDGE_MM_CMD_FIRST+29
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+30
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+31
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+32
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+33
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE			PVRSRV_BRIDGE_MM_CMD_FIRST+34
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2			PVRSRV_BRIDGE_MM_CMD_FIRST+35
+#define PVRSRV_BRIDGE_MM_CMD_LAST			(PVRSRV_BRIDGE_MM_CMD_FIRST+35)
+
+/*******************************************
+            PMRExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMREXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMREXPORTPMR;
+
+/* Bridge out structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMREXPORTPMR_TAG
+{
+	IMG_UINT64 ui64Password;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRExport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2Contig;
+} __packed PVRSRV_BRIDGE_OUT_PMREXPORTPMR;
+
+/*******************************************
+            PMRUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMRExport;
+} __packed PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR;
+
+/* Bridge out structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR;
+
+/*******************************************
+            PMRGetUID
+ *******************************************/
+
+/* Bridge in structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_IN_PMRGETUID_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRGETUID;
+
+/* Bridge out structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_OUT_PMRGETUID_TAG
+{
+	IMG_UINT64 ui64UID;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRGETUID;
+
+/*******************************************
+            PMRMakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRUnmakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRIMPORTPMR_TAG
+{
+	IMG_UINT64 ui64uiPassword;
+	IMG_UINT64 ui64uiSize;
+	IMG_HANDLE hPMRExport;
+	IMG_UINT32 ui32uiLog2Contig;
+} __packed PVRSRV_BRIDGE_IN_PMRIMPORTPMR;
+
+/* Bridge out structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRIMPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRIMPORTPMR;
+
+/*******************************************
+            PMRLocalImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_HANDLE hExtHandle;
+} __packed PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR;
+
+/* Bridge out structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR;
+
+/*******************************************
+            PMRUnrefPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNREFPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRUNREFPMR;
+
+/* Bridge out structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNREFPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNREFPMR;
+
+/*******************************************
+            PhysmemNewRamBackedPMR
+ *******************************************/
+
+/* Bridge in structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiAnnotation;
+	IMG_UINT32 ui32AnnotationLength;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_PID ui32PID;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR;
+
+/* Bridge out structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR;
+
+/*******************************************
+            DevmemIntCtxCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_BOOL bbKernelMemoryCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE;
+
+/* Bridge out structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_HANDLE hDevMemServerContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CPUCacheLineSize;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE;
+
+/*******************************************
+            DevmemIntCtxDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY_TAG
+{
+	IMG_HANDLE hDevmemServerContext;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY;
+
+/* Bridge out structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY;
+
+/*******************************************
+            DevmemIntHeapCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE;
+
+/* Bridge out structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemHeapPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE;
+
+/*******************************************
+            DevmemIntHeapDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY_TAG
+{
+	IMG_HANDLE hDevmemHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY;
+
+/* Bridge out structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY;
+
+/*******************************************
+            DevmemIntMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR;
+
+/* Bridge out structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR;
+
+/*******************************************
+            DevmemIntUnmapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR;
+
+/* Bridge out structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR;
+
+/*******************************************
+            DevmemIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE;
+
+/* Bridge out structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE;
+
+/*******************************************
+            DevmemIntReserveRangeAndMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	IMG_HANDLE hPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/* Bridge out structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/*******************************************
+            DevmemIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE;
+
+/*******************************************
+            ChangeSparseMem
+ *******************************************/
+
+/* Bridge in structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_IN_CHANGESPARSEMEM_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32SparseFlags;
+} __packed PVRSRV_BRIDGE_IN_CHANGESPARSEMEM;
+
+/* Bridge out structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM;
+
+/*******************************************
+            DevmemIsVDevAddrValid
+ *******************************************/
+
+/* Bridge in structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID;
+
+/* Bridge out structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID;
+
+/*******************************************
+            DevmemInvalidateFBSCTable
+ *******************************************/
+
+/* Bridge in structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	IMG_UINT64 ui64FBSCEntries;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE;
+
+/* Bridge out structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE;
+
+/*******************************************
+            HeapCfgHeapConfigCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT;
+
+/* Bridge out structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeapConfigs;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT;
+
+/*******************************************
+            HeapCfgHeapCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT_TAG
+{
+	IMG_UINT32 ui32HeapConfigIndex;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT;
+
+/* Bridge out structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT;
+
+/*******************************************
+            HeapCfgHeapConfigName
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapConfigNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME;
+
+/* Bridge out structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME;
+
+/*******************************************
+            HeapCfgHeapDetails
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_CHAR *puiHeapNameOut;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+	IMG_UINT32 ui32HeapNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS;
+
+/* Bridge out structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_CHAR *puiHeapNameOut;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2DataPageSizeOut;
+	IMG_UINT32 ui32Log2ImportAlignmentOut;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS;
+
+/*******************************************
+            DevmemIntRegisterPFNotifyKM
+ *******************************************/
+
+/* Bridge in structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_BOOL bRegister;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/* Bridge out structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/*******************************************
+            PhysHeapGetMemInfo
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO;
+
+/* Bridge out structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO;
+
+/*******************************************
+            GetDefaultPhysicalHeap
+ *******************************************/
+
+/* Bridge in structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP;
+
+/* Bridge out structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP eHeap;
+} __packed PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP;
+
+/*******************************************
+            DevmemGetFaultAddress
+ *******************************************/
+
+/* Bridge in structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS;
+
+/* Bridge out structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_DEV_VIRTADDR sFaultAddress;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS;
+
+/*******************************************
+            PVRSRVStatsUpdateOOMStat
+ *******************************************/
+
+/* Bridge in structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	IMG_PID ui32pid;
+	IMG_UINT32 ui32ui32StatType;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT;
+
+/* Bridge out structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT;
+
+/*******************************************
+            DevmemXIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE;
+
+/*******************************************
+            DevmemXIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE;
+
+/*******************************************
+            DevmemXIntMapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32PhysPageOffset;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES;
+
+/* Bridge out structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES;
+
+/*******************************************
+            DevmemXIntUnmapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES;
+
+/* Bridge out structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES;
+
+/*******************************************
+            DevmemXIntMapVRangeToBackingPage
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/* Bridge out structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/*******************************************
+            PhysHeapGetMemInfo2
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2;
+
+/* Bridge out structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2;
+
+#endif /* COMMON_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/server_mm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/server_mm_bridge.c
new file mode 100644
index 000000000000..a79a92f5e9d1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/server_mm_bridge.c
@@ -0,0 +1,3237 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+#include "common_mm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleasePMRExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRExportPMRpsPMRExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnexportPMR((PMR_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRExportPMRIN_UI8,
+			 IMG_UINT8 * psPMRExportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMREXPORTPMR *psPMRExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMREXPORTPMR *psPMRExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PMRExportPMR(psPMRInt,
+			 &psPMRExportInt,
+			 &psPMRExportPMROUT->ui64Size,
+			 &psPMRExportPMROUT->ui32Log2Contig, &psPMRExportPMROUT->ui64Password);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRExportPMR_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &hPMRExportInt, (void *)psPMRExportInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _PMRExportPMRpsPMRExportIntRelease);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRExportPMROUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+							      &psPMRExportPMROUT->hPMRExport,
+							      (void *)psPMRExportInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      ReleasePMRExport);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRExportPMROUT->hPMRExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE) psPMRExportPMROUT->
+							     hPMRExport,
+							     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hPMRExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hPMRExportInt,
+							PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psPMRExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnexportPMR(psPMRExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRExportPMR NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgePMRUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psPMRUnexportPMRIN_UI8,
+			   IMG_UINT8 * psPMRUnexportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *psPMRUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *psPMRUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMROUT_UI8, 0);
+
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+				       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_FALSE);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hPMRExportInt,
+				     psPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT((psPMRUnexportPMROUT->eError == PVRSRV_OK) ||
+		   (psPMRUnexportPMROUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRUnexportPMR_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRUnexportPMR NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgePMRGetUID(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psPMRGetUIDIN_UI8,
+		      IMG_UINT8 * psPMRGetUIDOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRGETUID *psPMRGetUIDIN =
+	    (PVRSRV_BRIDGE_IN_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRGETUID *psPMRGetUIDOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRGetUIDIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRGetUIDOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRGetUIDOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRGetUID_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRGetUIDOUT->eError = PMRGetUID(psPMRInt, &psPMRGetUIDOUT->ui64UID);
+
+PMRGetUID_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRMakeLocalImportHandlepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnmakeLocalImportHandle((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRMakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleIN_UI8,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psPMRMakeLocalImportHandleIN->hBuffer;
+	PMR *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer,
+				       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE, IMG_TRUE);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psPMRMakeLocalImportHandleOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PMRMakeLocalImportHandlepsExtMemIntRelease);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRMakeLocalImportHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnmakeLocalImportHandle(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnmakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleIN_UI8,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRUnmakeLocalImportHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psPMRUnmakeLocalImportHandleIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_OK) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRUnmakeLocalImportHandleOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRUnmakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRUnmakeLocalImportHandle_exit:
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRImportPMRIN_UI8,
+			 IMG_UINT8 * psPMRImportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRIMPORTPMR *psPMRImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *psPMRImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMRExport = psPMRImportPMRIN->hPMRExport;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psPMRImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_TRUE);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRImportPMROUT->eError =
+	    PhysmemImportPMR(psConnection, OSGetDevNode(psConnection),
+			     psPMRExportInt,
+			     psPMRImportPMRIN->ui64uiPassword,
+			     psPMRImportPMRIN->ui64uiSize,
+			     psPMRImportPMRIN->ui32uiLog2Contig, &psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psPMRImportPMROUT->hPMR,
+							      (void *)psPMRInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _PMRImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRExportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psPMRImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRImportPMR NULL
+#endif
+
+static PVRSRV_ERROR _PMRLocalImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRLocalImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPMRLocalImportPMRIN_UI8,
+			      IMG_UINT8 * psPMRLocalImportPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *psPMRLocalImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *psPMRLocalImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hExtHandle = psPMRLocalImportPMRIN->hExtHandle;
+	PMR *psExtHandleInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRLocalImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psExtHandleInt,
+				       hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT, IMG_TRUE);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError =
+	    PMRLocalImportPMR(psExtHandleInt,
+			      &psPMRInt,
+			      &psPMRLocalImportPMROUT->uiSize, &psPMRLocalImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPMRLocalImportPMROUT->hPMR,
+								   (void *)psPMRInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PMRLocalImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRLocalImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psExtHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psPMRLocalImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnrefPMR(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRUnrefPMRIN_UI8,
+			IMG_UINT8 * psPMRUnrefPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNREFPMR *psPMRUnrefPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNREFPMR *psPMRUnrefPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRUnrefPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRUnrefPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (unlikely((psPMRUnrefPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnrefPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRUnrefPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRUnrefPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemNewRamBackedPMRpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemNewRamBackedPMR(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMRIN_UI8,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMROUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMRIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMROUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiAnnotationInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (unlikely(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemNewRamBackedPMRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemNewRamBackedPMRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemNewRamBackedPMR_exit;
+			}
+		}
+	}
+
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->pui32MappingTable,
+		     psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+	}
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength != 0)
+	{
+		uiAnnotationInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiAnnotationInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->puiAnnotation,
+		     psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+		((IMG_CHAR *)
+		 uiAnnotationInt)[(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength *
+				   sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psPhysmemNewRamBackedPMROUT->eError =
+	    PhysmemNewRamBackedPMR(psConnection, OSGetDevNode(psConnection),
+				   psPhysmemNewRamBackedPMRIN->uiSize,
+				   psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks,
+				   psPhysmemNewRamBackedPMRIN->ui32NumVirtChunks,
+				   ui32MappingTableInt,
+				   psPhysmemNewRamBackedPMRIN->ui32Log2PageSize,
+				   psPhysmemNewRamBackedPMRIN->uiFlags,
+				   psPhysmemNewRamBackedPMRIN->ui32AnnotationLength,
+				   uiAnnotationInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PID,
+				   &psPMRPtrInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PDumpFlags,
+				   &psPhysmemNewRamBackedPMROUT->uiOutFlags);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemNewRamBackedPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psPhysmemNewRamBackedPMROUT->
+									hPMRPtr,
+									(void *)psPMRPtrInt,
+									PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_PhysmemNewRamBackedPMRpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemNewRamBackedPMR_exit:
+
+	if (psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemNewRamBackedPMROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntCtxCreatepsDevMemServerContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxCreate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntCtxCreateIN_UI8,
+			       IMG_UINT8 * psDevmemIntCtxCreateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateOUT_UI8,
+								     0);
+
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntCtxCreateOUT->hDevMemServerContext = NULL;
+
+	psDevmemIntCtxCreateOUT->eError =
+	    DevmemIntCtxCreate(psConnection, OSGetDevNode(psConnection),
+			       psDevmemIntCtxCreateIN->bbKernelMemoryCtx,
+			       &psDevMemServerContextInt,
+			       &hPrivDataInt, &psDevmemIntCtxCreateOUT->ui32CPUCacheLineSize);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntCtxCreateOUT->
+								    hDevMemServerContext,
+								    (void *)
+								    psDevMemServerContextInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntCtxCreatepsDevMemServerContextIntRelease);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntCtxCreateOUT->
+								       hPrivData,
+								       (void *)hPrivDataInt,
+								       PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       psDevmemIntCtxCreateOUT->
+								       hDevMemServerContext);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxCreate_exit:
+
+	if (psDevmemIntCtxCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntCtxCreateOUT->hDevMemServerContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDevmemIntCtxCreateOUT->
+							     hDevMemServerContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psDevMemServerContextInt)
+		{
+			DevmemIntCtxDestroy(psDevMemServerContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntCtxDestroyIN_UI8,
+				IMG_UINT8 * psDevmemIntCtxDestroyOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyOUT_UI8,
+								      0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntCtxDestroyIN->
+					      hDevmemServerContext,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	if (unlikely
+	    ((psDevmemIntCtxDestroyOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntCtxDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntHeapDestroy((DEVMEMINT_HEAP *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapCreate(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntHeapCreateIN_UI8,
+				IMG_UINT8 * psDevmemIntHeapCreateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntHeapCreateIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntHeapCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				psDevmemIntHeapCreateIN->ui32HeapConfigIndex,
+				psDevmemIntHeapCreateIN->ui32HeapIndex, &psDevmemHeapPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psDevmemIntHeapCreateOUT->
+								     hDevmemHeapPtr,
+								     (void *)psDevmemHeapPtrInt,
+								     PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntHeapCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemHeapPtrInt)
+		{
+			DevmemIntHeapDestroy(psDevmemHeapPtrInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntHeapDestroyIN_UI8,
+				 IMG_UINT8 * psDevmemIntHeapDestroyOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *) IMG_OFFSET_ADDR(psDevmemIntHeapDestroyIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *)
+	    IMG_OFFSET_ADDR(psDevmemIntHeapDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntHeapDestroyIN->hDevmemHeap,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	if (unlikely((psDevmemIntHeapDestroyOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntHeapDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapDestroy_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDevmemIntMapPMRIN_UI8,
+			    IMG_UINT8 * psDevmemIntMapPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *psDevmemIntMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *psDevmemIntMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntMapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntMapPMROUT->eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+DevmemIntMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnmapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDevmemIntUnmapPMRIN_UI8,
+			      IMG_UINT8 * psDevmemIntUnmapPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntUnmapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntUnmapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntUnmapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnmapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnmapPMROUT->eError = DevmemIntUnmapPMR(psReservationInt);
+
+DevmemIntUnmapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIntReserveRangeIN_UI8,
+				  IMG_UINT8 * psDevmemIntReserveRangeOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError =
+	    DevmemIntReserveRange(psConnection, OSGetDevNode(psConnection),
+				  psDevmemServerHeapInt,
+				  psDevmemIntReserveRangeIN->sAddress,
+				  psDevmemIntReserveRangeIN->uiLength,
+				  psDevmemIntReserveRangeIN->uiFlags, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntReserveRangeOUT->
+								       hReservation,
+								       (void *)psReservationInt,
+								       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DevmemIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRangeAndMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMRIN_UI8,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMROUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeAndMapPMRIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntReserveRangeAndMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    DevmemIntReserveRangeAndMapPMR(psConnection, OSGetDevNode(psConnection),
+					   psDevmemServerHeapInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->sAddress,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiLength,
+					   psPMRInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiFlags,
+					   &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntReserveRangeAndMapPMROUT->hReservation,
+				      (void *)psReservationInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRangeAndMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeIN_UI8,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeChangeSparseMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psChangeSparseMemIN_UI8,
+			    IMG_UINT8 * psChangeSparseMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *psChangeSparseMemIN =
+	    (PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *psChangeSparseMemOUT =
+	    (PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+	IMG_HANDLE hReservation = psChangeSparseMemIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psChangeSparseMemIN->ui32AllocPageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (unlikely(psChangeSparseMemIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto ChangeSparseMem_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psChangeSparseMemIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psChangeSparseMemIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psChangeSparseMemOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto ChangeSparseMem_exit;
+			}
+		}
+	}
+
+	if (psChangeSparseMemIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32AllocPageIndices,
+		     psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+	if (psChangeSparseMemIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32FreePageIndices,
+		     psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psChangeSparseMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psChangeSparseMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto ChangeSparseMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psChangeSparseMemOUT->eError =
+	    DevmemIntChangeSparse(psChangeSparseMemIN->ui32AllocPageCount,
+				  ui32AllocPageIndicesInt,
+				  psChangeSparseMemIN->ui32FreePageCount,
+				  ui32FreePageIndicesInt,
+				  psChangeSparseMemIN->ui32SparseFlags, psReservationInt);
+
+ChangeSparseMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psChangeSparseMemOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIsVDevAddrValid(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidIN_UI8,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIsVDevAddrValidIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIsVDevAddrValidOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIsVDevAddrValidOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIsVDevAddrValid_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIsVDevAddrValidOUT->eError =
+	    DevmemIntIsVDevAddrValid(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, psDevmemIsVDevAddrValidIN->sAddress);
+
+DevmemIsVDevAddrValid_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+
+static IMG_INT
+PVRSRVBridgeDevmemInvalidateFBSCTable(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableIN_UI8,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemInvalidateFBSCTableIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemInvalidateFBSCTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemInvalidateFBSCTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    DevmemIntInvalidateFBSCTable(psDevmemCtxInt,
+					 psDevmemInvalidateFBSCTableIN->ui64FBSCEntries);
+
+DevmemInvalidateFBSCTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeDevmemInvalidateFBSCTable NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigCount(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountIN_UI8,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHeapCfgHeapConfigCountIN);
+
+	psHeapCfgHeapConfigCountOUT->eError =
+	    HeapCfgHeapConfigCount(psConnection, OSGetDevNode(psConnection),
+				   &psHeapCfgHeapConfigCountOUT->ui32NumHeapConfigs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapCount(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psHeapCfgHeapCountIN_UI8,
+			     IMG_UINT8 * psHeapCfgHeapCountOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountOUT_UI8, 0);
+
+	psHeapCfgHeapCountOUT->eError =
+	    HeapCfgHeapCount(psConnection, OSGetDevNode(psConnection),
+			     psHeapCfgHeapCountIN->ui32HeapConfigIndex,
+			     &psHeapCfgHeapCountOUT->ui32NumHeaps);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigName(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameIN_UI8,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameOUT_UI8, 0);
+
+	IMG_CHAR *puiHeapConfigNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	psHeapCfgHeapConfigNameOUT->puiHeapConfigName =
+	    psHeapCfgHeapConfigNameIN->puiHeapConfigName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapConfigNameIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapConfigNameIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapConfigName_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz != 0)
+	{
+		puiHeapConfigNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapConfigNameOUT->eError =
+	    HeapCfgHeapConfigName(psConnection, OSGetDevNode(psConnection),
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigIndex,
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz,
+				  puiHeapConfigNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapConfigNameOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapConfigNameInt) &&
+	    ((psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapConfigNameOUT->puiHeapConfigName,
+		      puiHeapConfigNameInt,
+		      (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapConfigName_exit;
+		}
+	}
+
+HeapCfgHeapConfigName_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapConfigNameOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapDetails(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psHeapCfgHeapDetailsIN_UI8,
+			       IMG_UINT8 * psHeapCfgHeapDetailsOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsOUT_UI8,
+								     0);
+
+	IMG_CHAR *puiHeapNameOutInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) + 0;
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	psHeapCfgHeapDetailsOUT->puiHeapNameOut = psHeapCfgHeapDetailsIN->puiHeapNameOut;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapDetailsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapDetailsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapDetails_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz != 0)
+	{
+		puiHeapNameOutInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapDetailsOUT->eError =
+	    HeapCfgHeapDetails(psConnection, OSGetDevNode(psConnection),
+			       psHeapCfgHeapDetailsIN->ui32HeapConfigIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapNameBufSz,
+			       puiHeapNameOutInt,
+			       &psHeapCfgHeapDetailsOUT->sDevVAddrBase,
+			       &psHeapCfgHeapDetailsOUT->uiHeapLength,
+			       &psHeapCfgHeapDetailsOUT->uiReservedRegionLength,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2DataPageSizeOut,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2ImportAlignmentOut);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapDetailsOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapNameOutInt) &&
+	    ((psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapDetailsOUT->puiHeapNameOut,
+		      puiHeapNameOutInt,
+		      (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapDetails_exit;
+		}
+	}
+
+HeapCfgHeapDetails_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapDetailsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntRegisterPFNotifyKM(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMIN_UI8,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntRegisterPFNotifyKMIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntRegisterPFNotifyKMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntRegisterPFNotifyKM_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, psDevmemIntRegisterPFNotifyKMIN->bRegister);
+
+DevmemIntRegisterPFNotifyKM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPhysHeapGetMemInfoIN_UI8,
+			       IMG_UINT8 * psPhysHeapGetMemInfoOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoIN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoOUT_UI8,
+								     0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V1)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfoIN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfoIN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfoIN->peaPhysHeapID,
+		     psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V1 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1);
+	}
+
+	psPhysHeapGetMemInfoOUT->eError =
+	    PVRSRVPhysHeapGetMemInfoKM(psConnection, OSGetDevNode(psConnection),
+				       psPhysHeapGetMemInfoIN->ui32PhysHeapCount,
+				       eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfoOUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V1))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+
+PhysHeapGetMemInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDefaultPhysicalHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapIN_UI8,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapIN =
+	    (PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDefaultPhysicalHeapIN);
+
+	psGetDefaultPhysicalHeapOUT->eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(psConnection, OSGetDevNode(psConnection),
+					   &psGetDefaultPhysicalHeapOUT->eHeap);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemGetFaultAddress(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemGetFaultAddressIN_UI8,
+				  IMG_UINT8 * psDevmemGetFaultAddressOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemGetFaultAddressIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemGetFaultAddressOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemGetFaultAddressOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemGetFaultAddress_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemGetFaultAddressOUT->eError =
+	    DevmemIntGetFaultAddress(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, &psDevmemGetFaultAddressOUT->sFaultAddress);
+
+DevmemGetFaultAddress_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+
+static IMG_INT
+PVRSRVBridgePVRSRVStatsUpdateOOMStat(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatIN_UI8,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatOUT_UI8, 0);
+
+	psPVRSRVStatsUpdateOOMStatOUT->eError =
+	    PVRSRVStatsUpdateOOMStat(psConnection, OSGetDevNode(psConnection),
+				     psPVRSRVStatsUpdateOOMStatIN->ui32ui32StatType,
+				     psPVRSRVStatsUpdateOOMStatIN->ui32pid);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePVRSRVStatsUpdateOOMStat NULL
+#endif
+
+static PVRSRV_ERROR _DevmemXIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemXIntUnreserveRange((DEVMEMXINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDevmemXIntReserveRangeIN_UI8,
+				   IMG_UINT8 * psDevmemXIntReserveRangeOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemXIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt,
+				   psDevmemXIntReserveRangeIN->sAddress,
+				   psDevmemXIntReserveRangeIN->uiLength, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDevmemXIntReserveRangeOUT->
+									hReservation,
+									(void *)psReservationInt,
+									PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DevmemXIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemXIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeIN_UI8,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemXIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemXIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapPages(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemXIntMapPagesIN_UI8,
+			       IMG_UINT8 * psDevmemXIntMapPagesOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesOUT_UI8,
+								     0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemXIntMapPagesIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapPagesOUT->eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       psDevmemXIntMapPagesIN->ui32PageCount,
+			       psDevmemXIntMapPagesIN->ui32PhysPageOffset,
+			       psDevmemXIntMapPagesIN->uiFlags,
+			       psDevmemXIntMapPagesIN->ui32VirtPageOffset);
+
+DevmemXIntMapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnmapPages(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesIN_UI8,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntUnmapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntUnmapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntUnmapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnmapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnmapPagesOUT->eError =
+	    DevmemXIntUnmapPages(psReservationInt,
+				 psDevmemXIntUnmapPagesIN->ui32VirtPageOffset,
+				 psDevmemXIntUnmapPagesIN->ui32PageCount);
+
+DevmemXIntUnmapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapVRangeToBackingPage(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageIN_UI8,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapVRangeToBackingPageIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapVRangeToBackingPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapVRangeToBackingPage_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     psDevmemXIntMapVRangeToBackingPageIN->ui32PageCount,
+					     psDevmemXIntMapVRangeToBackingPageIN->uiFlags,
+					     psDevmemXIntMapVRangeToBackingPageIN->
+					     ui32VirtPageOffset);
+
+DevmemXIntMapVRangeToBackingPage_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo2(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysHeapGetMemInfo2IN_UI8,
+				IMG_UINT8 * psPhysHeapGetMemInfo2OUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2IN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2IN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2OUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2OUT_UI8,
+								      0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V2)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfo2IN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfo2IN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfo2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfo2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo2_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfo2IN->peaPhysHeapID,
+		     psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V2 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2);
+	}
+
+	psPhysHeapGetMemInfo2OUT->eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(psConnection, OSGetDevNode(psConnection),
+					psPhysHeapGetMemInfo2IN->ui32PhysHeapCount,
+					eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfo2OUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V2))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+
+PhysHeapGetMemInfo2_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfo2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+
+/*
+ * Register all MM functions with services
+ */
+PVRSRV_ERROR InitMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR,
+			      PVRSRVBridgePMRExportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMREXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMREXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR,
+			      PVRSRVBridgePMRUnexportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID, PVRSRVBridgePMRGetUID,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRGETUID),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRGETUID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRMakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRUnmakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR,
+			      PVRSRVBridgePMRImportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR,
+			      PVRSRVBridgePMRLocalImportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR,
+			      PVRSRVBridgePMRUnrefPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRUNREFPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNREFPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR,
+			      PVRSRVBridgePhysmemNewRamBackedPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE,
+			      PVRSRVBridgeDevmemIntCtxCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY,
+			      PVRSRVBridgeDevmemIntCtxDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE,
+			      PVRSRVBridgeDevmemIntHeapCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY,
+			      PVRSRVBridgeDevmemIntHeapDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR,
+			      PVRSRVBridgeDevmemIntMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR,
+			      PVRSRVBridgeDevmemIntUnmapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE,
+			      PVRSRVBridgeDevmemIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR,
+			      PVRSRVBridgeDevmemIntReserveRangeAndMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM,
+			      PVRSRVBridgeChangeSparseMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_CHANGESPARSEMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID,
+			      PVRSRVBridgeDevmemIsVDevAddrValid, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE,
+			      PVRSRVBridgeDevmemInvalidateFBSCTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT,
+			      PVRSRVBridgeHeapCfgHeapConfigCount, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT,
+			      PVRSRVBridgeHeapCfgHeapCount, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME,
+			      PVRSRVBridgeHeapCfgHeapConfigName, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS,
+			      PVRSRVBridgeHeapCfgHeapDetails, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM,
+			      PVRSRVBridgeDevmemIntRegisterPFNotifyKM, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO,
+			      PVRSRVBridgePhysHeapGetMemInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP,
+			      PVRSRVBridgeGetDefaultPhysicalHeap, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS,
+			      PVRSRVBridgeDevmemGetFaultAddress, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT,
+			      PVRSRVBridgePVRSRVStatsUpdateOOMStat, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES,
+			      PVRSRVBridgeDevmemXIntMapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES,
+			      PVRSRVBridgeDevmemXIntUnmapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE,
+			      PVRSRVBridgeDevmemXIntMapVRangeToBackingPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2,
+			      PVRSRVBridgePhysHeapGetMemInfo2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mm functions with services
+ */
+void DeinitMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM,
+				PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/common_mmextmem_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/common_mmextmem_bridge.h
new file mode 100644
index 000000000000..f52637853325
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/common_mmextmem_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MMEXTMEM_BRIDGE_H
+#define COMMON_MMEXTMEM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM			PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST			(PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0)
+
+/*******************************************
+            PhysmemWrapExtMem
+ *******************************************/
+
+/* Bridge in structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_UINT64 ui64CpuVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM;
+
+/* Bridge out structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM;
+
+#endif /* COMMON_MMEXTMEM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/server_mmextmem_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/server_mmextmem_bridge.c
new file mode 100644
index 000000000000..538d04cdf0e0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/server_mmextmem_bridge.c
@@ -0,0 +1,166 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "physmem_extmem.h"
+
+#include "common_mmextmem_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemWrapExtMempsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemWrapExtMem(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPhysmemWrapExtMemIN_UI8,
+			      IMG_UINT8 * psPhysmemWrapExtMemOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemOUT_UI8, 0);
+
+	PMR *psPMRPtrInt = NULL;
+
+	psPhysmemWrapExtMemOUT->eError =
+	    PhysmemWrapExtMem(psConnection, OSGetDevNode(psConnection),
+			      psPhysmemWrapExtMemIN->uiSize,
+			      psPhysmemWrapExtMemIN->ui64CpuVAddr,
+			      psPhysmemWrapExtMemIN->uiFlags, &psPMRPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemWrapExtMemOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPhysmemWrapExtMemOUT->hPMRPtr,
+								   (void *)psPMRPtrInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PhysmemWrapExtMempsPMRPtrIntRelease);
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemWrapExtMem_exit:
+
+	if (psPhysmemWrapExtMemOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+
+/*
+ * Register all MMEXTMEM functions with services
+ */
+PVRSRV_ERROR InitMMEXTMEMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM,
+			      PVRSRVBridgePhysmemWrapExtMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mmextmem functions with services
+ */
+void DeinitMMEXTMEMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_bridge.h
new file mode 100644
index 000000000000..2bf967bfcea9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMP_BRIDGE_H
+#define CLIENT_PDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_direct_bridge.c
new file mode 100644
index 000000000000..7c012dbcf21e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_direct_bridge.c
@@ -0,0 +1,151 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpImageDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  psDevmemCtxInt,
+					  ui32StringSize,
+					  puiFileName,
+					  sDataDevAddr,
+					  ui32DataSize,
+					  ui32LogicalWidth,
+					  ui32LogicalHeight,
+					  ui32PhysicalWidth,
+					  ui32PhysicalHeight,
+					  ePixelFormat,
+					  eMemLayout,
+					  eFBCompression,
+					  pui32FBCClearColour,
+					  eeFBCSwizzle,
+					  sHeaderDevAddr, ui32HeaderSize, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			   ui32CommentSize, puiComment, ui32Flags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpSetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpDataDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 psDevmemCtxInt,
+					 ui32StringSize,
+					 puiFileName,
+					 sDataDevAddr,
+					 ui32DataSize,
+					 ui32HeaderType,
+					 ui32ElementType, ui32ElementCount, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/common_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/common_pdump_bridge.h
new file mode 100644
index 000000000000..eb6ab9488a20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/common_pdump_bridge.h
@@ -0,0 +1,155 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMP_BRIDGE_H
+#define COMMON_PDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_BRIDGE_PDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMP_CMD_LAST			(PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3)
+
+/*******************************************
+            PDumpImageDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_DEV_VIRTADDR sHeaderDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_UINT32 *pui32FBCClearColour;
+	const IMG_CHAR *puiFileName;
+	IMG_FB_COMPRESSION eFBCompression;
+	IMG_MEMLAYOUT eMemLayout;
+	PDUMP_PIXEL_FORMAT ePixelFormat;
+	PDUMP_FBC_SWIZZLE eeFBCSwizzle;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32HeaderSize;
+	IMG_UINT32 ui32LogicalHeight;
+	IMG_UINT32 ui32LogicalWidth;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PhysicalHeight;
+	IMG_UINT32 ui32PhysicalWidth;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR;
+
+/* Bridge out structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR;
+
+/*******************************************
+            PVRSRVPDumpComment
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT_TAG
+{
+	IMG_CHAR *puiComment;
+	IMG_UINT32 ui32CommentSize;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT;
+
+/* Bridge out structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT;
+
+/*******************************************
+            PVRSRVPDumpSetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME_TAG
+{
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME;
+
+/*******************************************
+            PDumpDataDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32ElementCount;
+	IMG_UINT32 ui32ElementType;
+	IMG_UINT32 ui32HeaderType;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR;
+
+/* Bridge out structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR;
+
+#endif /* COMMON_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/server_pdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/server_pdump_bridge.c
new file mode 100644
index 000000000000..6b3baf668710
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/server_pdump_bridge.c
@@ -0,0 +1,571 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+#include "common_pdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(4 <= IMG_UINT32_MAX, "4 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpImageDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpImageDescriptorIN_UI8,
+				 IMG_UINT8 * psPDumpImageDescriptorOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpImageDescriptorIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *)
+	    IMG_OFFSET_ADDR(psPDumpImageDescriptorOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpImageDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+	IMG_UINT32 *ui32FBCClearColourInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) 4 * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psPDumpImageDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpImageDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpImageDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpImageDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpImageDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psPDumpImageDescriptorIN->puiFileName,
+		     psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	{
+		ui32FBCClearColourInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 4 * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (4 * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FBCClearColourInt,
+		     (const void __user *)psPDumpImageDescriptorIN->pui32FBCClearColour,
+		     4 * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpImageDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpImageDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpImageDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpImageDescriptorOUT->eError =
+	    DevmemIntPDumpImageDescriptor(psConnection, OSGetDevNode(psConnection),
+					  psDevmemCtxInt,
+					  psPDumpImageDescriptorIN->ui32StringSize,
+					  uiFileNameInt,
+					  psPDumpImageDescriptorIN->sDataDevAddr,
+					  psPDumpImageDescriptorIN->ui32DataSize,
+					  psPDumpImageDescriptorIN->ui32LogicalWidth,
+					  psPDumpImageDescriptorIN->ui32LogicalHeight,
+					  psPDumpImageDescriptorIN->ui32PhysicalWidth,
+					  psPDumpImageDescriptorIN->ui32PhysicalHeight,
+					  psPDumpImageDescriptorIN->ePixelFormat,
+					  psPDumpImageDescriptorIN->eMemLayout,
+					  psPDumpImageDescriptorIN->eFBCompression,
+					  ui32FBCClearColourInt,
+					  psPDumpImageDescriptorIN->eeFBCSwizzle,
+					  psPDumpImageDescriptorIN->sHeaderDevAddr,
+					  psPDumpImageDescriptorIN->ui32HeaderSize,
+					  psPDumpImageDescriptorIN->ui32PDumpFlags);
+
+PDumpImageDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpImageDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_COMMENT_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_COMMENT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpComment(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPVRSRVPDumpCommentIN_UI8,
+			       IMG_UINT8 * psPVRSRVPDumpCommentOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentOUT_UI8,
+								     0);
+
+	IMG_CHAR *uiCommentInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPVRSRVPDumpCommentIN->ui32CommentSize > PVRSRV_PDUMP_MAX_COMMENT_SIZE))
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPVRSRVPDumpCommentIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPVRSRVPDumpCommentIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PVRSRVPDumpComment_exit;
+			}
+		}
+	}
+
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize != 0)
+	{
+		uiCommentInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiCommentInt, (const void __user *)psPVRSRVPDumpCommentIN->puiComment,
+		     psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PVRSRVPDumpComment_exit;
+		}
+		((IMG_CHAR *)
+		 uiCommentInt)[(psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPVRSRVPDumpCommentOUT->eError =
+	    PDumpCommentKM(psConnection, OSGetDevNode(psConnection),
+			   psPVRSRVPDumpCommentIN->ui32CommentSize,
+			   uiCommentInt, psPVRSRVPDumpCommentIN->ui32Flags);
+
+PVRSRVPDumpComment_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPVRSRVPDumpCommentOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameOUT_UI8,
+								      0);
+
+	psPVRSRVPDumpSetFrameOUT->eError =
+	    PDumpSetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    psPVRSRVPDumpSetFrameIN->ui32Frame);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpDataDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPDumpDataDescriptorIN_UI8,
+				IMG_UINT8 * psPDumpDataDescriptorOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpDataDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPDumpDataDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpDataDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpDataDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpDataDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpDataDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPDumpDataDescriptorIN->puiFileName,
+		     psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpDataDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpDataDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpDataDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpDataDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpDataDescriptorOUT->eError =
+	    DevmemIntPDumpDataDescriptor(psConnection, OSGetDevNode(psConnection),
+					 psDevmemCtxInt,
+					 psPDumpDataDescriptorIN->ui32StringSize,
+					 uiFileNameInt,
+					 psPDumpDataDescriptorIN->sDataDevAddr,
+					 psPDumpDataDescriptorIN->ui32DataSize,
+					 psPDumpDataDescriptorIN->ui32HeaderType,
+					 psPDumpDataDescriptorIN->ui32ElementType,
+					 psPDumpDataDescriptorIN->ui32ElementCount,
+					 psPDumpDataDescriptorIN->ui32PDumpFlags);
+
+PDumpDataDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpDataDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+
+/*
+ * Register all PDUMP functions with services
+ */
+PVRSRV_ERROR InitPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR,
+			      PVRSRVBridgePDumpImageDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT,
+			      PVRSRVBridgePVRSRVPDumpComment, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME,
+			      PVRSRVBridgePVRSRVPDumpSetFrame, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR,
+			      PVRSRVBridgePDumpDataDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdump functions with services
+ */
+void DeinitPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..a2ba27b88b30
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_bridge.h
@@ -0,0 +1,74 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPCTRL_BRIDGE_H
+#define CLIENT_PDUMPCTRL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpctrl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge);
+
+#endif /* CLIENT_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
new file mode 100644
index 000000000000..a2bc30360aea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
@@ -0,0 +1,109 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpctrl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpctrl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpGetStateKM(pui64State);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpGetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), pui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpSetDefaultCaptureParamsKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32Mode,
+					   ui32Start,
+					   ui32End,
+					   ui32Interval, ui32MaxParamFileSize, ui32AutoTermTimeout);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpIsLastCaptureFrameKM(pbpbIsLastCaptureFrame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpForceCaptureStopKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge));
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/common_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/common_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..f710e4c06131
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/common_pdumpctrl_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPCTRL_BRIDGE_H
+#define COMMON_PDUMPCTRL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST			(PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4)
+
+/*******************************************
+            PVRSRVPDumpGetState
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE;
+
+/* Bridge out structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT64 ui64State;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE;
+
+/*******************************************
+            PVRSRVPDumpGetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME;
+
+/*******************************************
+            PVRSRVPDumpSetDefaultCaptureParams
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	IMG_UINT32 ui32AutoTermTimeout;
+	IMG_UINT32 ui32End;
+	IMG_UINT32 ui32Interval;
+	IMG_UINT32 ui32MaxParamFileSize;
+	IMG_UINT32 ui32Mode;
+	IMG_UINT32 ui32Start;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/* Bridge out structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/*******************************************
+            PVRSRVPDumpIsLastCaptureFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/* Bridge out structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bpbIsLastCaptureFrame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/*******************************************
+            PVRSRVPDumpForceCaptureStop
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP;
+
+/* Bridge out structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP;
+
+#endif /* COMMON_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/server_pdumpctrl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/server_pdumpctrl_bridge.c
new file mode 100644
index 000000000000..2d5578cf83f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/server_pdumpctrl_bridge.c
@@ -0,0 +1,252 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pdump_km.h"
+
+#include "common_pdumpctrl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetState(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetStateIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetStateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetStateIN);
+
+	psPVRSRVPDumpGetStateOUT->eError = PDumpGetStateKM(&psPVRSRVPDumpGetStateOUT->ui64State);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetFrameIN);
+
+	psPVRSRVPDumpGetFrameOUT->eError =
+	    PDumpGetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    &psPVRSRVPDumpGetFrameOUT->ui32Frame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams(IMG_UINT32 ui32DispatchTableEntry,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8,
+					       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *psPVRSRVPDumpSetDefaultCaptureParamsIN
+	    =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS
+	    *psPVRSRVPDumpSetDefaultCaptureParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8, 0);
+
+	psPVRSRVPDumpSetDefaultCaptureParamsOUT->eError =
+	    PDumpSetDefaultCaptureParamsKM(psConnection, OSGetDevNode(psConnection),
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Mode,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Start,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32End,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Interval,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32MaxParamFileSize,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32AutoTermTimeout);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameIN_UI8,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpIsLastCaptureFrameIN);
+
+	psPVRSRVPDumpIsLastCaptureFrameOUT->eError =
+	    PDumpIsLastCaptureFrameKM(&psPVRSRVPDumpIsLastCaptureFrameOUT->bpbIsLastCaptureFrame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpForceCaptureStop(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopIN_UI8,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpForceCaptureStopIN);
+
+	psPVRSRVPDumpForceCaptureStopOUT->eError =
+	    PDumpForceCaptureStopKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pPDUMPCTRLBridgeLock;
+
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+
+/*
+ * Register all PDUMPCTRL functions with services
+ */
+PVRSRV_ERROR InitPDUMPCTRLBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pPDUMPCTRLBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE,
+			      PVRSRVBridgePVRSRVPDumpGetState, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME,
+			      PVRSRVBridgePVRSRVPDumpGetFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS,
+			      PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams, pPDUMPCTRLBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME,
+			      PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP,
+			      PVRSRVBridgePVRSRVPDumpForceCaptureStop, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpctrl functions with services
+ */
+void DeinitPDUMPCTRLBridge(void)
+{
+	OSLockDestroy(pPDUMPCTRLBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_bridge.h
new file mode 100644
index 000000000000..a4368477e225
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_bridge.h
@@ -0,0 +1,125 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPMM_BRIDGE_H
+#define CLIENT_PDUMPMM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpmm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
new file mode 100644
index 000000000000..1358f75a7537
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpmm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpmm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMem(psPMRInt, uiOffset, uiSize, ui32PDumpFlags, bbZero);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue32(psPMRInt, uiOffset, ui32Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue64(psPMRInt, uiOffset, ui64Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       uiOffset, uiSize, ui32ArraySize, puiFileName, ui32uiFileOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  uiOffset,
+				  ui32MemspaceNameLen,
+				  puiMemspaceName,
+				  ui32SymbolicAddrLen,
+				  puiSymbolicAddr, puiNewOffset, puiNextSymName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpPol32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpCheck32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpCBP(psPMRInt, uiReadOffset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError =
+	    DevmemIntPDumpSaveToFileVirtual(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    psDevmemServerContextInt,
+					    sAddress,
+					    uiSize,
+					    ui32ArraySize,
+					    puiFileName, ui32FileOffset, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/common_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/common_pdumpmm_bridge.h
new file mode 100644
index 000000000000..2020cbf5b30b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/common_pdumpmm_bridge.h
@@ -0,0 +1,259 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPMM_BRIDGE_H
+#define COMMON_PDUMPMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_LAST			(PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8)
+
+/*******************************************
+            PMRPDumpLoadMem
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_BOOL bbZero;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM;
+
+/* Bridge out structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM;
+
+/*******************************************
+            PMRPDumpLoadMemValue32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32;
+
+/* Bridge out structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32;
+
+/*******************************************
+            PMRPDumpLoadMemValue64
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	IMG_UINT64 ui64Value;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64;
+
+/* Bridge out structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64;
+
+/*******************************************
+            PMRPDumpSaveToFile
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32uiFileOffset;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE;
+
+/* Bridge out structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE;
+
+/*******************************************
+            PMRPDumpSymbolicAddr
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	IMG_UINT32 ui32MemspaceNameLen;
+	IMG_UINT32 ui32SymbolicAddrLen;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR;
+
+/* Bridge out structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiNewOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR;
+
+/*******************************************
+            PMRPDumpPol32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPPOL32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPPOL32;
+
+/* Bridge out structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32;
+
+/*******************************************
+            PMRPDumpCheck32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32;
+
+/* Bridge out structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32;
+
+/*******************************************
+            PMRPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiReadOffset;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCBP;
+
+/* Bridge out structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCBP;
+
+/*******************************************
+            DevmemIntPDumpSaveToFileVirtual
+ *******************************************/
+
+/* Bridge in structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hDevmemServerContext;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32FileOffset;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+/* Bridge out structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+#endif /* COMMON_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/server_pdumpmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/server_pdumpmm_bridge.c
new file mode 100644
index 000000000000..38e1a6610fd5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/server_pdumpmm_bridge.c
@@ -0,0 +1,958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+#include "common_pdumpmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpLoadMemIN_UI8,
+			    IMG_UINT8 * psPMRPDumpLoadMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *psPMRPDumpLoadMemIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *psPMRPDumpLoadMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemOUT->eError =
+	    PMRPDumpLoadMem(psPMRInt,
+			    psPMRPDumpLoadMemIN->uiOffset,
+			    psPMRPDumpLoadMemIN->uiSize,
+			    psPMRPDumpLoadMemIN->ui32PDumpFlags, psPMRPDumpLoadMemIN->bbZero);
+
+PMRPDumpLoadMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue32(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PMRPDumpLoadMemValue32(psPMRInt,
+				   psPMRPDumpLoadMemValue32IN->uiOffset,
+				   psPMRPDumpLoadMemValue32IN->ui32Value,
+				   psPMRPDumpLoadMemValue32IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue64(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue64IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue64OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue64_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PMRPDumpLoadMemValue64(psPMRInt,
+				   psPMRPDumpLoadMemValue64IN->uiOffset,
+				   psPMRPDumpLoadMemValue64IN->ui64Value,
+				   psPMRPDumpLoadMemValue64IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue64_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSaveToFile(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRPDumpSaveToFileIN_UI8,
+			       IMG_UINT8 * psPMRPDumpSaveToFileOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSaveToFileIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPMRPDumpSaveToFileIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSaveToFileIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSaveToFileIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSaveToFile_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPMRPDumpSaveToFileIN->puiFileName,
+		     psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSaveToFile_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSaveToFileOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSaveToFileOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSaveToFile_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSaveToFileOUT->eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       psPMRPDumpSaveToFileIN->uiOffset,
+			       psPMRPDumpSaveToFileIN->uiSize,
+			       psPMRPDumpSaveToFileIN->ui32ArraySize,
+			       uiFileNameInt, psPMRPDumpSaveToFileIN->ui32uiFileOffset);
+
+PMRPDumpSaveToFile_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSaveToFileOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSymbolicAddr(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrIN_UI8,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *) IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *)
+	    IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSymbolicAddrIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *puiMemspaceNameInt = NULL;
+	IMG_CHAR *puiSymbolicAddrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) + 0;
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen > PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen > PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	psPMRPDumpSymbolicAddrOUT->puiMemspaceName = psPMRPDumpSymbolicAddrIN->puiMemspaceName;
+	psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr = psPMRPDumpSymbolicAddrIN->puiSymbolicAddr;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSymbolicAddrIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSymbolicAddrIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSymbolicAddr_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen != 0)
+	{
+		puiMemspaceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR);
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen != 0)
+	{
+		puiSymbolicAddrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  psPMRPDumpSymbolicAddrIN->uiOffset,
+				  psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen,
+				  puiMemspaceNameInt,
+				  psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen,
+				  puiSymbolicAddrInt,
+				  &psPMRPDumpSymbolicAddrOUT->uiNewOffset,
+				  &psPMRPDumpSymbolicAddrOUT->uiNextSymName);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiMemspaceNameInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiMemspaceName,
+		      puiMemspaceNameInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiSymbolicAddrInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr,
+		      puiSymbolicAddrInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+PMRPDumpSymbolicAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSymbolicAddrOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpPol32(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psPMRPDumpPol32IN_UI8,
+			  IMG_UINT8 * psPMRPDumpPol32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *psPMRPDumpPol32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *psPMRPDumpPol32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpPol32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpPol32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpPol32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpPol32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpPol32OUT->eError =
+	    PMRPDumpPol32(psPMRInt,
+			  psPMRPDumpPol32IN->uiOffset,
+			  psPMRPDumpPol32IN->ui32Value,
+			  psPMRPDumpPol32IN->ui32Mask,
+			  psPMRPDumpPol32IN->eOperator, psPMRPDumpPol32IN->ui32PDumpFlags);
+
+PMRPDumpPol32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCheck32(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpCheck32IN_UI8,
+			    IMG_UINT8 * psPMRPDumpCheck32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *psPMRPDumpCheck32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *psPMRPDumpCheck32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCheck32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCheck32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCheck32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCheck32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCheck32OUT->eError =
+	    PMRPDumpCheck32(psPMRInt,
+			    psPMRPDumpCheck32IN->uiOffset,
+			    psPMRPDumpCheck32IN->ui32Value,
+			    psPMRPDumpCheck32IN->ui32Mask,
+			    psPMRPDumpCheck32IN->eOperator, psPMRPDumpCheck32IN->ui32PDumpFlags);
+
+PMRPDumpCheck32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRPDumpCBPIN_UI8,
+			IMG_UINT8 * psPMRPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCBP *psPMRPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *psPMRPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCBPIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCBPOUT->eError =
+	    PMRPDumpCBP(psPMRInt,
+			psPMRPDumpCBPIN->uiReadOffset,
+			psPMRPDumpCBPIN->uiWriteOffset,
+			psPMRPDumpCBPIN->uiPacketSize, psPMRPDumpCBPIN->uiBufferSize);
+
+PMRPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualIN_UI8,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerContext = psDevmemIntPDumpSaveToFileVirtualIN->hDevmemServerContext;
+	DEVMEMINT_CTX *psDevmemServerContextInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (unlikely
+	    (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevmemIntPDumpSaveToFileVirtualIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevmemIntPDumpSaveToFileVirtualIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevmemIntPDumpSaveToFileVirtual_exit;
+			}
+		}
+	}
+
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psDevmemIntPDumpSaveToFileVirtualIN->puiFileName,
+		     psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevmemIntPDumpSaveToFileVirtual_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize *
+				 sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerContextInt,
+				       hDevmemServerContext,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntPDumpSaveToFileVirtualOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    DevmemIntPDumpSaveToFileVirtual(psConnection, OSGetDevNode(psConnection),
+					    psDevmemServerContextInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->sAddress,
+					    psDevmemIntPDumpSaveToFileVirtualIN->uiSize,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize,
+					    uiFileNameInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32FileOffset,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32PDumpFlags);
+
+DevmemIntPDumpSaveToFileVirtual_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevmemIntPDumpSaveToFileVirtualOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+
+/*
+ * Register all PDUMPMM functions with services
+ */
+PVRSRV_ERROR InitPDUMPMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM,
+			      PVRSRVBridgePMRPDumpLoadMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32,
+			      PVRSRVBridgePMRPDumpLoadMemValue32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64,
+			      PVRSRVBridgePMRPDumpLoadMemValue64, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE,
+			      PVRSRVBridgePMRPDumpSaveToFile, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR,
+			      PVRSRVBridgePMRPDumpSymbolicAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32,
+			      PVRSRVBridgePMRPDumpPol32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPPOL32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32,
+			      PVRSRVBridgePMRPDumpCheck32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP,
+			      PVRSRVBridgePMRPDumpCBP, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+			      PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL,
+			      PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpmm functions with services
+ */
+void DeinitPDUMPMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_bridge.h
new file mode 100644
index 000000000000..2cfafd5a83e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PVRTL_BRIDGE_H
+#define CLIENT_PVRTL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pvrtl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams,
+						  IMG_UINT32 * pui32NumFound);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin,
+						IMG_UINT32 * pui32Available);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data);
+
+#endif /* CLIENT_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_direct_bridge.c
new file mode 100644
index 000000000000..fa2fbed9595b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_direct_bridge.c
@@ -0,0 +1,175 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pvrtl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pvrtl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#include "tlserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerOpenStreamKM(puiName, ui32Mode, &psSDInt, &psTLPMRInt);
+
+	*phSD = psSDInt;
+	*phTLPMR = psTLPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCloseStreamKM(psSDInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerAcquireDataKM(psSDInt, pui32ReadOffset, pui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerReleaseDataKM(psSDInt, ui32ReadOffset, ui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams, IMG_UINT32 * pui32NumFound)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerDiscoverStreamsKM(puiNamePattern, ui32Size, puiStreams, pui32NumFound);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin, IMG_UINT32 * pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    pui32BufferOffset, ui32Size, ui32SizeMin, pui32Available);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCommitStreamKM(psSDInt, ui32ReqSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerWriteDataKM(psSDInt, ui32Size, pui8Data);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/common_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/common_pvrtl_bridge.h
new file mode 100644
index 000000000000..edc822375222
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/common_pvrtl_bridge.h
@@ -0,0 +1,214 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PVRTL_BRIDGE_H
+#define COMMON_PVRTL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#define PVRSRV_BRIDGE_PVRTL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PVRTL_TLWRITEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PVRTL_CMD_LAST			(PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7)
+
+/*******************************************
+            TLOpenStream
+ *******************************************/
+
+/* Bridge in structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_IN_TLOPENSTREAM_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_UINT32 ui32Mode;
+} __packed PVRSRV_BRIDGE_IN_TLOPENSTREAM;
+
+/* Bridge out structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLOPENSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_HANDLE hTLPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLOPENSTREAM;
+
+/*******************************************
+            TLCloseStream
+ *******************************************/
+
+/* Bridge in structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCLOSESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLCLOSESTREAM;
+
+/* Bridge out structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCLOSESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCLOSESTREAM;
+
+/*******************************************
+            TLAcquireData
+ *******************************************/
+
+/* Bridge in structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_IN_TLACQUIREDATA_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLACQUIREDATA;
+
+/* Bridge out structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_OUT_TLACQUIREDATA_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLACQUIREDATA;
+
+/*******************************************
+            TLReleaseData
+ *******************************************/
+
+/* Bridge in structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_IN_TLRELEASEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_IN_TLRELEASEDATA;
+
+/* Bridge out structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_OUT_TLRELEASEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLRELEASEDATA;
+
+/*******************************************
+            TLDiscoverStreams
+ *******************************************/
+
+/* Bridge in structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS_TAG
+{
+	const IMG_CHAR *puiNamePattern;
+	IMG_CHAR *puiStreams;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS;
+
+/* Bridge out structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS_TAG
+{
+	IMG_CHAR *puiStreams;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumFound;
+} __packed PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS;
+
+/*******************************************
+            TLReserveStream
+ *******************************************/
+
+/* Bridge in structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_IN_TLRESERVESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32SizeMin;
+} __packed PVRSRV_BRIDGE_IN_TLRESERVESTREAM;
+
+/* Bridge out structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLRESERVESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Available;
+	IMG_UINT32 ui32BufferOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLRESERVESTREAM;
+
+/*******************************************
+            TLCommitStream
+ *******************************************/
+
+/* Bridge in structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCOMMITSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReqSize;
+} __packed PVRSRV_BRIDGE_IN_TLCOMMITSTREAM;
+
+/* Bridge out structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM;
+
+/*******************************************
+            TLWriteData
+ *******************************************/
+
+/* Bridge in structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_IN_TLWRITEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLWRITEDATA;
+
+/* Bridge out structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_OUT_TLWRITEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLWRITEDATA;
+
+#endif /* COMMON_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/server_pvrtl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/server_pvrtl_bridge.c
new file mode 100644
index 000000000000..9facf794011e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/server_pvrtl_bridge.c
@@ -0,0 +1,827 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "tlserver.h"
+
+#include "common_pvrtl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _TLOpenStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = TLServerCloseStreamKM((TL_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLOpenStream(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psTLOpenStreamIN_UI8,
+			 IMG_UINT8 * psTLOpenStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLOPENSTREAM *psTLOpenStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLOPENSTREAM *psTLOpenStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamOUT_UI8, 0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	psTLOpenStreamOUT->hSD = NULL;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLOpenStreamOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLOpenStream_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLOpenStreamIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLOpenStreamIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLOpenStreamOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLOpenStream_exit;
+			}
+		}
+	}
+
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psTLOpenStreamIN->puiName,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLOpenStreamOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLOpenStream_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) - 1] =
+		    '\0';
+	}
+
+	psTLOpenStreamOUT->eError =
+	    TLServerOpenStreamKM(uiNameInt, psTLOpenStreamIN->ui32Mode, &psSDInt, &psTLPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		goto TLOpenStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psTLOpenStreamOUT->hSD,
+							      (void *)psSDInt,
+							      PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _TLOpenStreampsSDIntRelease);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								 &psTLOpenStreamOUT->hTLPMR,
+								 (void *)psTLPMRInt,
+								 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 psTLOpenStreamOUT->hSD);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLOpenStream_exit:
+
+	if (psTLOpenStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psTLOpenStreamOUT->hSD)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psTLOpenStreamOUT->hSD,
+							     PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSDInt)
+		{
+			TLServerCloseStreamKM(psSDInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLOpenStreamOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCloseStream(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLCloseStreamIN_UI8,
+			  IMG_UINT8 * psTLCloseStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCLOSESTREAM *psTLCloseStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *psTLCloseStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLCloseStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psTLCloseStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	if (unlikely((psTLCloseStreamOUT->eError != PVRSRV_OK) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psTLCloseStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCloseStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLCloseStream_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLAcquireData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLAcquireDataIN_UI8,
+			  IMG_UINT8 * psTLAcquireDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLACQUIREDATA *psTLAcquireDataIN =
+	    (PVRSRV_BRIDGE_IN_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLACQUIREDATA *psTLAcquireDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLAcquireDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLAcquireDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLAcquireDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLAcquireData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLAcquireDataOUT->eError =
+	    TLServerAcquireDataKM(psSDInt,
+				  &psTLAcquireDataOUT->ui32ReadOffset,
+				  &psTLAcquireDataOUT->ui32ReadLen);
+
+TLAcquireData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReleaseData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLReleaseDataIN_UI8,
+			  IMG_UINT8 * psTLReleaseDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRELEASEDATA *psTLReleaseDataIN =
+	    (PVRSRV_BRIDGE_IN_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRELEASEDATA *psTLReleaseDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReleaseDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReleaseDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReleaseDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReleaseData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReleaseDataOUT->eError =
+	    TLServerReleaseDataKM(psSDInt,
+				  psTLReleaseDataIN->ui32ReadOffset,
+				  psTLReleaseDataIN->ui32ReadLen);
+
+TLReleaseData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLDiscoverStreams(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psTLDiscoverStreamsIN_UI8,
+			      IMG_UINT8 * psTLDiscoverStreamsOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *psTLDiscoverStreamsIN =
+	    (PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *psTLDiscoverStreamsOUT =
+	    (PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsOUT_UI8, 0);
+
+	IMG_CHAR *uiNamePatternInt = NULL;
+	IMG_CHAR *puiStreamsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) + 0;
+
+	if (psTLDiscoverStreamsIN->ui32Size > PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLDiscoverStreams_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psTLDiscoverStreamsOUT->puiStreams = psTLDiscoverStreamsIN->puiStreams;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLDiscoverStreams_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLDiscoverStreamsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLDiscoverStreamsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLDiscoverStreams_exit;
+			}
+		}
+	}
+
+	{
+		uiNamePatternInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNamePatternInt,
+		     (const void __user *)psTLDiscoverStreamsIN->puiNamePattern,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+		((IMG_CHAR *) uiNamePatternInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) -
+						1] = '\0';
+	}
+	if (psTLDiscoverStreamsIN->ui32Size != 0)
+	{
+		puiStreamsInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR);
+	}
+
+	psTLDiscoverStreamsOUT->eError =
+	    TLServerDiscoverStreamsKM(uiNamePatternInt,
+				      psTLDiscoverStreamsIN->ui32Size,
+				      puiStreamsInt, &psTLDiscoverStreamsOUT->ui32NumFound);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLDiscoverStreamsOUT->eError != PVRSRV_OK))
+	{
+		goto TLDiscoverStreams_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamsInt) && ((psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psTLDiscoverStreamsOUT->puiStreams, puiStreamsInt,
+		      (psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+	}
+
+TLDiscoverStreams_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLDiscoverStreamsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReserveStream(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psTLReserveStreamIN_UI8,
+			    IMG_UINT8 * psTLReserveStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRESERVESTREAM *psTLReserveStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *psTLReserveStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReserveStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReserveStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReserveStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReserveStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReserveStreamOUT->eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    &psTLReserveStreamOUT->ui32BufferOffset,
+				    psTLReserveStreamIN->ui32Size,
+				    psTLReserveStreamIN->ui32SizeMin,
+				    &psTLReserveStreamOUT->ui32Available);
+
+TLReserveStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCommitStream(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psTLCommitStreamIN_UI8,
+			   IMG_UINT8 * psTLCommitStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *psTLCommitStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *psTLCommitStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLCommitStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLCommitStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLCommitStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCommitStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLCommitStreamOUT->eError =
+	    TLServerCommitStreamKM(psSDInt, psTLCommitStreamIN->ui32ReqSize);
+
+TLCommitStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLWriteData(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psTLWriteDataIN_UI8,
+			IMG_UINT8 * psTLWriteDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLWRITEDATA *psTLWriteDataIN =
+	    (PVRSRV_BRIDGE_IN_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLWRITEDATA *psTLWriteDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLWriteDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psTLWriteDataIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLWriteData_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLWriteData_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLWriteDataIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLWriteDataIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLWriteDataOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLWriteData_exit;
+			}
+		}
+	}
+
+	if (psTLWriteDataIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psTLWriteDataIN->pui8Data,
+		     psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psTLWriteDataOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLWriteData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLWriteDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLWriteDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLWriteData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLWriteDataOUT->eError =
+	    TLServerWriteDataKM(psSDInt, psTLWriteDataIN->ui32Size, ui8DataInt);
+
+TLWriteData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLWriteDataOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+
+/*
+ * Register all PVRTL functions with services
+ */
+PVRSRV_ERROR InitPVRTLBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM,
+			      PVRSRVBridgeTLOpenStream, NULL, sizeof(PVRSRV_BRIDGE_IN_TLOPENSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLOPENSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM,
+			      PVRSRVBridgeTLCloseStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCLOSESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCLOSESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA,
+			      PVRSRVBridgeTLAcquireData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLACQUIREDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLACQUIREDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA,
+			      PVRSRVBridgeTLReleaseData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRELEASEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRELEASEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS,
+			      PVRSRVBridgeTLDiscoverStreams, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM,
+			      PVRSRVBridgeTLReserveStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRESERVESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRESERVESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM,
+			      PVRSRVBridgeTLCommitStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCOMMITSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA,
+			      PVRSRVBridgeTLWriteData, NULL, sizeof(PVRSRV_BRIDGE_IN_TLWRITEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLWRITEDATA));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pvrtl functions with services
+ */
+void DeinitPVRTLBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/common_rgxcmp_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/common_rgxcmp_bridge.h
new file mode 100644
index 000000000000..aa1d2b73985a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/common_rgxcmp_bridge.h
@@ -0,0 +1,309 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXCMP_BRIDGE_H
+#define COMMON_RGXCMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXCMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXCMP_CMD_LAST			(PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11)
+
+/*******************************************
+            RGXCreateComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticComputeContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+	IMG_UINT32 ui32StaticComputeContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT;
+
+/* Bridge out structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT;
+
+/*******************************************
+            RGXDestroyComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT;
+
+/* Bridge out structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT;
+
+/*******************************************
+            RGXFlushComputeData
+ *******************************************/
+
+/* Bridge in structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA;
+
+/* Bridge out structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA;
+
+/*******************************************
+            RGXSendCancelCmd
+ *******************************************/
+
+/* Bridge in structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32FirstIntJobRefToCancel;
+	IMG_INT32 i32LastIntJobRefToCancel;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD;
+
+/* Bridge out structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD;
+
+/*******************************************
+            RGXSetComputeContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/*******************************************
+            RGXNotifyComputeWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXSetComputeContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/*******************************************
+            RGXGetLastDeviceError
+ *******************************************/
+
+/* Bridge in structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR;
+
+/* Bridge out structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Error;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR;
+
+/*******************************************
+            RGXKickTimestampQuery
+ *******************************************/
+
+/* Bridge in structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY;
+
+/* Bridge out structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY;
+
+/*******************************************
+            RGXKickCDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKCDM_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumOfWorkgroups;
+	IMG_UINT32 ui32NumOfWorkitems;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKCDM;
+
+/* Bridge out structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKCDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	IMG_UINT32 ui32IntJobRef;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKCDM;
+
+/*******************************************
+            RGXCDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXCDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY;
+
+#endif /* COMMON_RGXCMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/server_rgxcmp_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/server_rgxcmp_bridge.c
new file mode 100644
index 000000000000..8910cfb5ba6a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/server_rgxcmp_bridge.c
@@ -0,0 +1,1580 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxcompute.h"
+
+#include "common_rgxcmp_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateComputeContextpsComputeContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyComputeContextKM((RGX_SERVER_COMPUTE_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_COMPUTECONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_COMPUTECONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXCreateComputeContextIN_UI8,
+				    IMG_UINT8 * psRGXCreateComputeContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateComputeContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticComputeContextStateInt = NULL;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateComputeContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize >
+	     RGXFWIF_STATIC_COMPUTECONTEXT_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateComputeContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateComputeContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateComputeContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateComputeContextIN->pui8FrameworkCmd,
+		     psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize != 0)
+	{
+		ui8StaticComputeContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		    sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticComputeContextStateInt,
+		     (const void __user *)psRGXCreateComputeContextIN->
+		     pui8StaticComputeContextState,
+		     psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVRGXCreateComputeContextKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXCreateComputeContextIN->i32Priority,
+					    psRGXCreateComputeContextIN->ui32FrameworkCmdSize,
+					    ui8FrameworkCmdInt,
+					    hPrivDataInt,
+					    psRGXCreateComputeContextIN->
+					    ui32StaticComputeContextStateSize,
+					    ui8StaticComputeContextStateInt,
+					    psRGXCreateComputeContextIN->ui32PackedCCBSizeU88,
+					    psRGXCreateComputeContextIN->ui32ContextFlags,
+					    psRGXCreateComputeContextIN->ui64RobustnessAddress,
+					    psRGXCreateComputeContextIN->ui32MaxDeadlineMS,
+					    &psComputeContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psRGXCreateComputeContextOUT->
+									 hComputeContext,
+									 (void *)
+									 psComputeContextInt,
+									 PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _RGXCreateComputeContextpsComputeContextIntRelease);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateComputeContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateComputeContextOUT->eError != PVRSRV_OK)
+	{
+		if (psComputeContextInt)
+		{
+			PVRSRVRGXDestroyComputeContextKM(psComputeContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateComputeContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXDestroyComputeContextIN_UI8,
+				     IMG_UINT8 * psRGXDestroyComputeContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXDestroyComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXDestroyComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyComputeContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyComputeContextIN->
+					      hComputeContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyComputeContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyComputeContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyComputeContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyComputeContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFlushComputeData(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXFlushComputeDataIN_UI8,
+				IMG_UINT8 * psRGXFlushComputeDataOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataIN =
+	    (PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataOUT_UI8,
+								      0);
+
+	IMG_HANDLE hComputeContext = psRGXFlushComputeDataIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXFlushComputeDataOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXFlushComputeData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXFlushComputeDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXFlushComputeDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXFlushComputeData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXFlushComputeDataOUT->eError = PVRSRVRGXFlushComputeDataKM(psComputeContextInt);
+
+RGXFlushComputeData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendCancelCmd(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSendCancelCmdIN_UI8,
+			     IMG_UINT8 * psRGXSendCancelCmdOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *psRGXSendCancelCmdIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *psRGXSendCancelCmdOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSendCancelCmdIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSendCancelCmdOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSendCancelCmd_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendCancelCmdOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendCancelCmd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVRGXSendCancelCmdKM(psComputeContextInt,
+				     psRGXSendCancelCmdIN->i32FirstIntJobRefToCancel,
+				     psRGXSendCancelCmdIN->i32LastIntJobRefToCancel);
+
+RGXSendCancelCmd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPriorityIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVRGXSetComputeContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						 psComputeContextInt,
+						 psRGXSetComputeContextPriorityIN->i32Priority);
+
+RGXSetComputeContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXNotifyComputeWriteOffsetUpdateIN_UI8,
+					      IMG_UINT8 *
+					      psRGXNotifyComputeWriteOffsetUpdateOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXNotifyComputeWriteOffsetUpdateIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXNotifyComputeWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXNotifyComputeWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXNotifyComputeWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXNotifyComputeWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(psComputeContextInt);
+
+RGXNotifyComputeWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPropertyIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVRGXSetComputeContextPropertyKM(psComputeContextInt,
+						 psRGXSetComputeContextPropertyIN->ui32Property,
+						 psRGXSetComputeContextPropertyIN->ui64Input,
+						 &psRGXSetComputeContextPropertyOUT->ui64Output);
+
+RGXSetComputeContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetLastDeviceError(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorIN_UI8,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXGetLastDeviceErrorOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXGetLastDeviceError_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetLastDeviceErrorIN);
+
+	psRGXGetLastDeviceErrorOUT->eError =
+	    PVRSRVRGXGetLastDeviceErrorKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetLastDeviceErrorOUT->ui32Error);
+
+RGXGetLastDeviceError_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTimestampQuery(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXKickTimestampQueryIN_UI8,
+				  IMG_UINT8 * psRGXKickTimestampQueryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickTimestampQueryIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickTimestampQueryIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTimestampQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTimestampQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTimestampQuery_exit;
+			}
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTimestampQueryIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickTimestampQueryIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickTimestampQueryIN->pui8DMCmd,
+		     psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTimestampQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTimestampQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVRGXKickTimestampQueryKM(psComputeContextInt,
+					  psRGXKickTimestampQueryIN->hCheckFenceFd,
+					  psRGXKickTimestampQueryIN->hUpdateTimeline,
+					  &psRGXKickTimestampQueryOUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXKickTimestampQueryIN->ui32CmdSize,
+					  ui8DMCmdInt, psRGXKickTimestampQueryIN->ui32ExtJobRef);
+
+RGXKickTimestampQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTimestampQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickCDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickCDMIN_UI8,
+		       IMG_UINT8 * psRGXKickCDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKCDM *psRGXKickCDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKCDM *psRGXKickCDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickCDMIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickCDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickCDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickCDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickCDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickCDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickCDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickCDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateOffset,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateValue,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickCDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickCDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickCDMIN->pui8DMCmd,
+		     psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickCDMIN->pui32SyncPMRFlags,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0, psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickCDMIN->phSyncPMRs,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickCDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickCDMOUT->eError =
+	    PVRSRVRGXKickCDMKM(psComputeContextInt,
+			       psRGXKickCDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickCDMIN->hCheckFenceFd,
+			       psRGXKickCDMIN->hUpdateTimeline,
+			       &psRGXKickCDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickCDMIN->hExportFenceToSignal,
+			       psRGXKickCDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickCDMIN->ui32PDumpFlags,
+			       psRGXKickCDMIN->ui32ExtJobRef,
+			       psRGXKickCDMIN->ui32SyncPMRCount,
+			       ui32SyncPMRFlagsInt,
+			       psSyncPMRsInt,
+			       psRGXKickCDMIN->ui32NumOfWorkgroups,
+			       psRGXKickCDMIN->ui32NumOfWorkitems,
+			       psRGXKickCDMIN->ui64DeadlineInus, &psRGXKickCDMOUT->ui32IntJobRef);
+
+RGXKickCDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickCDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXCDMGetSharedMemoryIN);
+
+	psRGXCDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXCDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXCDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMGetSharedMemory_exit:
+
+	if (psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXCDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+
+/*
+ * Register all RGXCMP functions with services
+ */
+PVRSRV_ERROR InitRGXCMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT,
+			      PVRSRVBridgeRGXCreateComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT,
+			      PVRSRVBridgeRGXDestroyComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA,
+			      PVRSRVBridgeRGXFlushComputeData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD,
+			      PVRSRVBridgeRGXSendCancelCmd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetComputeContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetComputeContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR,
+			      PVRSRVBridgeRGXGetLastDeviceError, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY,
+			      PVRSRVBridgeRGXKickTimestampQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM,
+			      PVRSRVBridgeRGXKickCDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKCDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKCDM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxcmp functions with services
+ */
+void DeinitRGXCMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
new file mode 100644
index 000000000000..534b8e9536ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
@@ -0,0 +1,339 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXFWDBG_BRIDGE_H
+#define COMMON_RGXFWDBG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "rgx_bridge.h"
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST			(PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15)
+
+/*******************************************
+            RGXFWDebugSetFWLog
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG_TAG
+{
+	IMG_UINT32 ui32RGXFWLogType;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG;
+
+/* Bridge out structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG;
+
+/*******************************************
+            RGXFWDebugDumpFreelistPageList
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/* Bridge out structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/*******************************************
+            RGXFWDebugPowerOff
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF;
+
+/* Bridge out structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF;
+
+/*******************************************
+            RGXFWDebugPowerOn
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON;
+
+/* Bridge out structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON;
+
+/*******************************************
+            RGXFWDebugSetVzConnectionCooldownPeriodInSec
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	IMG_UINT32 ui32VzConne;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/* Bridge out structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/*******************************************
+            RGXFWDebugSetHCSDeadline
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	IMG_UINT32 ui32RGXHCSDeadline;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE;
+
+/* Bridge out structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE;
+
+/*******************************************
+            RGXFWDebugSetDriverPriority
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/* Bridge out structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSlice
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32TSPercentage;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSliceInterval
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	IMG_UINT32 ui32TSIntervalMs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/*******************************************
+            RGXFWDebugSetDriverIsolationGroup
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32IsolationGroup;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/* Bridge out structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/*******************************************
+            RGXFWDebugSetOSNewOnlineState
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32OSNewState;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/* Bridge out structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/*******************************************
+            RGXFWDebugMapGuestHeap
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	IMG_UINT64 ui64ui64GuestHeapBase;
+	IMG_UINT32 ui32DriverID;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP;
+
+/* Bridge out structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP;
+
+/*******************************************
+            RGXFWDebugPHRConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32PHRMode;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE;
+
+/*******************************************
+            RGXFWDebugWdgConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32WdgPeriodUs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE;
+
+/*******************************************
+            RGXCurrentTime
+ *******************************************/
+
+/* Bridge in structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_IN_RGXCURRENTTIME_TAG
+{
+	IMG_UINT8 ui8TimerType;
+} __packed PVRSRV_BRIDGE_IN_RGXCURRENTTIME;
+
+/* Bridge out structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCURRENTTIME_TAG
+{
+	IMG_UINT64 ui64Time;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCURRENTTIME;
+
+/*******************************************
+            RGXFWDebugInjectFault
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT;
+
+/* Bridge out structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT;
+
+#endif /* COMMON_RGXFWDBG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
new file mode 100644
index 000000000000..e5646490aa2d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
@@ -0,0 +1,542 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "rgxfwdbg.h"
+#include "pmr.h"
+#include "rgxtimecorr.h"
+
+#include "common_rgxfwdbg_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetFWLog(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogOUT_UI8,
+								     0);
+
+	psRGXFWDebugSetFWLogOUT->eError =
+	    PVRSRVRGXFWDebugSetFWLogKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXFWDebugSetFWLogIN->ui32RGXFWLogType);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugDumpFreelistPageList(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListIN_UI8,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugDumpFreelistPageListIN);
+
+	psRGXFWDebugDumpFreelistPageListOUT->eError =
+	    PVRSRVRGXFWDebugDumpFreelistPageListKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOff(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugPowerOffIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugPowerOffOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOffIN);
+
+	psRGXFWDebugPowerOffOUT->eError =
+	    PVRSRVRGXFWDebugPowerOffKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOn(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXFWDebugPowerOnIN_UI8,
+			      IMG_UINT8 * psRGXFWDebugPowerOnOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOnIN);
+
+	psRGXFWDebugPowerOnOUT->eError =
+	    PVRSRVRGXFWDebugPowerOnKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec(IMG_UINT32 ui32DispatchTableEntry,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8,
+							 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8, 0);
+
+	psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT->eError =
+	    PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(psConnection,
+								 OSGetDevNode(psConnection),
+								 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN->
+								 ui32VzConne);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetHCSDeadline(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineIN_UI8,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineOUT_UI8, 0);
+
+	psRGXFWDebugSetHCSDeadlineOUT->eError =
+	    PVRSRVRGXFWDebugSetHCSDeadlineKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXFWDebugSetHCSDeadlineIN->ui32RGXHCSDeadline);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityIN_UI8,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverPriorityOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRGXFWDebugSetDriverPriorityIN->ui32DriverID,
+						psRGXFWDebugSetDriverPriorityIN->ui32Priority);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSlice(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceIN_UI8,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceKM(psConnection, OSGetDevNode(psConnection),
+						 psRGXFWDebugSetDriverTimeSliceIN->ui32DriverID,
+						 psRGXFWDebugSetDriverTimeSliceIN->
+						 ui32TSPercentage);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval(IMG_UINT32 ui32DispatchTableEntry,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8,
+						 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceIntervalOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(psConnection, OSGetDevNode(psConnection),
+							 psRGXFWDebugSetDriverTimeSliceIntervalIN->
+							 ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXFWDebugSetDriverIsolationGroupIN_UI8,
+					      IMG_UINT8 *
+					      psRGXFWDebugSetDriverIsolationGroupOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverIsolationGroupOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverIsolationGroupKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32DriverID,
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32IsolationGroup);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetOSNewOnlineState(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateIN_UI8,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateOUT_UI8, 0);
+
+	psRGXFWDebugSetOSNewOnlineStateOUT->eError =
+	    PVRSRVRGXFWDebugSetOSNewOnlineStateKM(psConnection, OSGetDevNode(psConnection),
+						  psRGXFWDebugSetOSNewOnlineStateIN->ui32DriverID,
+						  psRGXFWDebugSetOSNewOnlineStateIN->
+						  ui32OSNewState);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugMapGuestHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapOUT_UI8, 0);
+
+	psRGXFWDebugMapGuestHeapOUT->eError =
+	    PVRSRVRGXFWDebugMapGuestHeapKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugMapGuestHeapIN->ui32DriverID,
+					   psRGXFWDebugMapGuestHeapIN->ui64ui64GuestHeapBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPHRConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureOUT_UI8, 0);
+
+	psRGXFWDebugPHRConfigureOUT->eError =
+	    PVRSRVRGXFWDebugPHRConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugPHRConfigureIN->ui32ui32PHRMode);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugWdgConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureOUT_UI8, 0);
+
+	psRGXFWDebugWdgConfigureOUT->eError =
+	    PVRSRVRGXFWDebugWdgConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugWdgConfigureIN->ui32ui32WdgPeriodUs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCurrentTime(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psRGXCurrentTimeIN_UI8,
+			   IMG_UINT8 * psRGXCurrentTimeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCURRENTTIME *psRGXCurrentTimeIN =
+	    (PVRSRV_BRIDGE_IN_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *psRGXCurrentTimeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeOUT_UI8, 0);
+
+	psRGXCurrentTimeOUT->eError =
+	    PVRSRVRGXCurrentTime(psConnection, OSGetDevNode(psConnection),
+				 psRGXCurrentTimeIN->ui8TimerType, &psRGXCurrentTimeOUT->ui64Time);
+
+	return 0;
+}
+
+#define PVRSRVBridgeRGXFWDebugInjectFault NULL
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+
+/*
+ * Register all RGXFWDBG functions with services
+ */
+PVRSRV_ERROR InitRGXFWDBGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG,
+			      PVRSRVBridgeRGXFWDebugSetFWLog, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST,
+			      PVRSRVBridgeRGXFWDebugDumpFreelistPageList, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF,
+			      PVRSRVBridgeRGXFWDebugPowerOff, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON,
+			      PVRSRVBridgeRGXFWDebugPowerOn, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC,
+			      PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC),
+			      sizeof
+			      (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE,
+			      PVRSRVBridgeRGXFWDebugSetHCSDeadline, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY,
+			      PVRSRVBridgeRGXFWDebugSetDriverPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSlice, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP,
+			      PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE,
+			      PVRSRVBridgeRGXFWDebugSetOSNewOnlineState, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP,
+			      PVRSRVBridgeRGXFWDebugMapGuestHeap, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugPHRConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugWdgConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME,
+			      PVRSRVBridgeRGXCurrentTime, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCURRENTTIME),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCURRENTTIME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT,
+			      PVRSRVBridgeRGXFWDebugInjectFault, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxfwdbg functions with services
+ */
+void DeinitRGXFWDBGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/common_rgxhwperf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/common_rgxhwperf_bridge.h
new file mode 100644
index 000000000000..7940464c7177
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/common_rgxhwperf_bridge.h
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXHWPERF_BRIDGE_H
+#define COMMON_RGXHWPERF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_hwperf.h"
+
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST			(PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9)
+
+/*******************************************
+            RGXGetConfiguredHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/*******************************************
+            RGXGetEnabledHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS;
+
+/* Bridge out structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32BlockCount;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetHWPerfTimeStamp
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP;
+
+/* Bridge out structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT64 ui64TimeStamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP;
+
+/*******************************************
+            RGXCtrlHWPerf
+ *******************************************/
+
+/* Bridge in structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_IN_RGXCTRLHWPERF_TAG
+{
+	IMG_UINT64 ui64Mask;
+	IMG_UINT32 ui32StreamId;
+	IMG_BOOL bToggle;
+} __packed PVRSRV_BRIDGE_IN_RGXCTRLHWPERF;
+
+/* Bridge out structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF;
+
+/*******************************************
+            RGXGetHWPerfBvncFeatureFlags
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/* Bridge out structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	RGX_HWPERF_BVNC sBVNC;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/*******************************************
+            RGXControlHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	IMG_UINT16 *pui16BlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_BOOL bEnable;
+} __packed PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS;
+
+/* Bridge out structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS;
+
+/*******************************************
+            RGXConfigureHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_UINT32 ui32CtrlWord;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS;
+
+/* Bridge out structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS;
+
+/*******************************************
+            RGXOpenHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXCloseHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXWriteHWPerfClientEvent
+ *******************************************/
+
+/* Bridge in structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT;
+
+/* Bridge out structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT;
+
+#endif /* COMMON_RGXHWPERF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/server_rgxhwperf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/server_rgxhwperf_bridge.c
new file mode 100644
index 000000000000..97aa4f234367
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/server_rgxhwperf_bridge.c
@@ -0,0 +1,919 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxhwperf.h"
+#include "rgx_fwif_km.h"
+
+#include "common_rgxhwperf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersIN_UI8,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters =
+	    psRGXGetConfiguredHWPerfCountersIN->psConfiguredCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredCountersInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+						   psRGXGetConfiguredHWPerfCountersIN->ui32BlockID,
+						   psConfiguredCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters,
+		      psConfiguredCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetEnabledHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksIN_UI8,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT32 *pui32EnabledBlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) + 0;
+
+	psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs =
+	    psRGXGetEnabledHWPerfBlocksIN->pui32EnabledBlockIDs;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetEnabledHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXGetEnabledHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetEnabledHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		pui32EnabledBlockIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32);
+	}
+
+	psRGXGetEnabledHWPerfBlocksOUT->eError =
+	    PVRSRVRGXGetEnabledHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					      psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen,
+					      &psRGXGetEnabledHWPerfBlocksOUT->ui32BlockCount,
+					      pui32EnabledBlockIDsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetEnabledHWPerfBlocksOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32EnabledBlockIDsInt) &&
+	    ((psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs,
+		      pui32EnabledBlockIDsInt,
+		      (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetEnabledHWPerfBlocks_exit;
+		}
+	}
+
+RGXGetEnabledHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetEnabledHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfTimeStamp(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampIN_UI8,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfTimeStampIN);
+
+	psRGXGetHWPerfTimeStampOUT->eError =
+	    PVRSRVRGXGetHWPerfTimeStampKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetHWPerfTimeStampOUT->ui64TimeStamp);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCtrlHWPerf(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXCtrlHWPerfIN_UI8,
+			  IMG_UINT8 * psRGXCtrlHWPerfOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *psRGXCtrlHWPerfIN =
+	    (PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *psRGXCtrlHWPerfOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfOUT_UI8, 0);
+
+	psRGXCtrlHWPerfOUT->eError =
+	    PVRSRVRGXCtrlHWPerfKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXCtrlHWPerfIN->ui32StreamId,
+				  psRGXCtrlHWPerfIN->bToggle, psRGXCtrlHWPerfIN->ui64Mask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsIN_UI8,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfBvncFeatureFlagsIN);
+
+	psRGXGetHWPerfBvncFeatureFlagsOUT->eError =
+	    PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(psConnection, OSGetDevNode(psConnection),
+						 &psRGXGetHWPerfBvncFeatureFlagsOUT->sBVNC);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXControlHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksIN_UI8,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT16 *ui16BlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) + 0;
+
+	if (unlikely(psRGXControlHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXControlHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXControlHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXControlHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		ui16BlockIDsInt = (IMG_UINT16 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16);
+	}
+
+	/* Copy the data over */
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui16BlockIDsInt,
+		     (const void __user *)psRGXControlHWPerfBlocksIN->pui16BlockIDs,
+		     psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) != PVRSRV_OK)
+		{
+			psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXControlHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXControlHWPerfBlocksOUT->eError =
+	    PVRSRVRGXControlHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXControlHWPerfBlocksIN->bEnable,
+					   psRGXControlHWPerfBlocksIN->ui32ArrayLen,
+					   ui16BlockIDsInt);
+
+RGXControlHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXControlHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX + 3 <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX+3 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigureHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksIN_UI8,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigureHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX + 3))
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigureHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigureHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigureHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigureHWPerfBlocksIN->psBlockConfigs,
+		     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigureHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXConfigureHWPerfBlocksOUT->eError =
+	    PVRSRVRGXConfigureHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXConfigureHWPerfBlocksIN->ui32CtrlWord,
+					     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen,
+					     psBlockConfigsInt);
+
+RGXConfigureHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigureHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXOpenHWPerfClientStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCloseHWPerfClientStreamKM((HWPERF_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOpenHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamIN_UI8,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamOUT_UI8, 0);
+
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXOpenHWPerfClientStreamIN);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVRGXOpenHWPerfClientStreamKM(psConnection, OSGetDevNode(psConnection), &psSDInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXOpenHWPerfClientStreamOUT->hSD, (void *)psSDInt,
+				      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXOpenHWPerfClientStreampsSDIntRelease);
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXOpenHWPerfClientStream_exit:
+
+	if (psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psSDInt)
+		{
+			PVRSRVRGXCloseHWPerfClientStreamKM(psSDInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCloseHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamIN_UI8,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCloseHWPerfClientStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCloseHWPerfClientStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	if (unlikely((psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_OK) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCloseHWPerfClientStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCloseHWPerfClientStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCloseHWPerfClientStream_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXWriteHWPerfClientEvent(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventIN_UI8,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventIN =
+	    (PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psRGXWriteHWPerfClientEventIN->hSD;
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXWriteHWPerfClientEventIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXWriteHWPerfClientEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXWriteHWPerfClientEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXWriteHWPerfClientEvent_exit;
+			}
+		}
+	}
+
+	if (psRGXWriteHWPerfClientEventIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psRGXWriteHWPerfClientEventIN->pui8Data,
+		     psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXWriteHWPerfClientEvent_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD, IMG_TRUE);
+	if (unlikely(psRGXWriteHWPerfClientEventOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVRGXWriteHWPerfClientEventKM(psSDInt,
+					      psRGXWriteHWPerfClientEventIN->ui32Size, ui8DataInt);
+
+RGXWriteHWPerfClientEvent_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXWriteHWPerfClientEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+
+/*
+ * Register all RGXHWPERF functions with services
+ */
+PVRSRV_ERROR InitRGXHWPERFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS,
+			      PVRSRVBridgeRGXGetEnabledHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP,
+			      PVRSRVBridgeRGXGetHWPerfTimeStamp, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF,
+			      PVRSRVBridgeRGXCtrlHWPerf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCTRLHWPERF),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS,
+			      PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS,
+			      PVRSRVBridgeRGXControlHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS,
+			      PVRSRVBridgeRGXConfigureHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXOpenHWPerfClientStream, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXCloseHWPerfClientStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT,
+			      PVRSRVBridgeRGXWriteHWPerfClientEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxhwperf functions with services
+ */
+void DeinitRGXHWPERFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/common_rgxkicksync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/common_rgxkicksync_bridge.h
new file mode 100644
index 000000000000..afd882ca7529
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/common_rgxkicksync_bridge.h
@@ -0,0 +1,143 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXKICKSYNC_BRIDGE_H
+#define COMMON_RGXKICKSYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST			(PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3)
+
+/*******************************************
+            RGXCreateKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXDestroyKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXKickSync2
+ *******************************************/
+
+/* Bridge in structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKSYNC2_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 *pui32UpdateDevVarOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phUpdateUFODevVarBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hTimelineFenceFD;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKSYNC2;
+
+/* Bridge out structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKSYNC2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFenceFD;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKSYNC2;
+
+/*******************************************
+            RGXSetKickSyncContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+#endif /* COMMON_RGXKICKSYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/server_rgxkicksync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/server_rgxkicksync_bridge.c
new file mode 100644
index 000000000000..8d943cd536e2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/server_rgxkicksync_bridge.c
@@ -0,0 +1,593 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxkicksync.h"
+
+#include "common_rgxkicksync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateKickSyncContextpsKickSyncContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyKickSyncContextKM((RGX_SERVER_KICKSYNC_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateKickSyncContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateKickSyncContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXCreateKickSyncContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVRGXCreateKickSyncContextKM(psConnection, OSGetDevNode(psConnection),
+					     hPrivDataInt,
+					     psRGXCreateKickSyncContextIN->ui32PackedCCBSizeU88,
+					     psRGXCreateKickSyncContextIN->ui32ContextFlags,
+					     &psKickSyncContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateKickSyncContextOUT->hKickSyncContext,
+				      (void *)psKickSyncContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateKickSyncContextpsKickSyncContextIntRelease);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateKickSyncContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK)
+	{
+		if (psKickSyncContextInt)
+		{
+			PVRSRVRGXDestroyKickSyncContextKM(psKickSyncContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyKickSyncContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyKickSyncContextIN->
+					      hKickSyncContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyKickSyncContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyKickSyncContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyKickSyncContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyKickSyncContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_DEV_VARS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_DEV_VARS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickSync2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickSync2IN_UI8,
+			 IMG_UINT8 * psRGXKickSync2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *psRGXKickSync2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *psRGXKickSync2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2OUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXKickSync2IN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFODevVarBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFODevVarBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateDevVarOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRGXKickSync2IN->ui32ClientUpdateCount > PVRSRV_MAX_DEV_VARS))
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickSync2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickSync2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickSync2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickSync2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickSync2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFODevVarBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFODevVarBlockInt, 0,
+			       psRGXKickSync2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFODevVarBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFODevVarBlockInt2,
+		     (const void __user *)psRGXKickSync2IN->phUpdateUFODevVarBlock,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateDevVarOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateDevVarOffsetInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateDevVarOffset,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateValue,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickSync2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickSync2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickSync2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickSync2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFODevVarBlockInt[i],
+						       hUpdateUFODevVarBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickSync2OUT->eError =
+	    PVRSRVRGXKickSyncKM(psKickSyncContextInt,
+				psRGXKickSync2IN->ui32ClientUpdateCount,
+				psUpdateUFODevVarBlockInt,
+				ui32UpdateDevVarOffsetInt,
+				ui32UpdateValueInt,
+				psRGXKickSync2IN->hCheckFenceFD,
+				psRGXKickSync2IN->hTimelineFenceFD,
+				&psRGXKickSync2OUT->hUpdateFenceFD,
+				uiUpdateFenceNameInt, psRGXKickSync2IN->ui32ExtJobRef);
+
+RGXKickSync2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+
+	if (hUpdateUFODevVarBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFODevVarBlockInt && psUpdateUFODevVarBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFODevVarBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickSync2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetKickSyncContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXSetKickSyncContextPropertyIN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetKickSyncContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetKickSyncContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVRGXSetKickSyncContextPropertyKM(psKickSyncContextInt,
+						  psRGXSetKickSyncContextPropertyIN->ui32Property,
+						  psRGXSetKickSyncContextPropertyIN->ui64Input,
+						  &psRGXSetKickSyncContextPropertyOUT->ui64Output);
+
+RGXSetKickSyncContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+
+/*
+ * Register all RGXKICKSYNC functions with services
+ */
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXCreateKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXDestroyKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2,
+			      PVRSRVBridgeRGXKickSync2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKSYNC2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKSYNC2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetKickSyncContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxkicksync functions with services
+ */
+void DeinitRGXKICKSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY);
+
+}
+#else /* SUPPORT_RGXKICKSYNC_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXKICKSYNC_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXKICKSYNCBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXKICKSYNCBridge()
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_bridge.h
new file mode 100644
index 000000000000..ed5a664ef17e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_bridge.h
@@ -0,0 +1,73 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RGXPDUMP_BRIDGE_H
+#define CLIENT_RGXPDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_rgxpdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
new file mode 100644
index 000000000000..7ba5126053dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
@@ -0,0 +1,118 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for rgxpdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_rgxpdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "rgx_bridge.h"
+
+#include "rgxpdump.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpTraceBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpSignatureBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPreCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPostCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					     ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/common_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/common_rgxpdump_bridge.h
new file mode 100644
index 000000000000..37317acb4707
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/common_rgxpdump_bridge.h
@@ -0,0 +1,161 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXPDUMP_BRIDGE_H
+#define COMMON_RGXPDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST			(PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5)
+
+/*******************************************
+            PDumpTraceBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER;
+
+/* Bridge out structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER;
+
+/*******************************************
+            PDumpSignatureBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER;
+
+/* Bridge out structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER;
+
+/*******************************************
+            PDumpComputeCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpValCheckPreCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND;
+
+/* Bridge out structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND;
+
+/*******************************************
+            PDumpValCheckPostCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND;
+
+/* Bridge out structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND;
+
+#endif /* COMMON_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/server_rgxpdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/server_rgxpdump_bridge.c
new file mode 100644
index 000000000000..58ed4ebb037f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/server_rgxpdump_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxpdump.h"
+
+#include "common_rgxpdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePDumpTraceBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psPDumpTraceBufferIN_UI8,
+			     IMG_UINT8 * psPDumpTraceBufferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *psPDumpTraceBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *psPDumpTraceBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferOUT_UI8, 0);
+
+	psPDumpTraceBufferOUT->eError =
+	    PVRSRVPDumpTraceBufferKM(psConnection, OSGetDevNode(psConnection),
+				     psPDumpTraceBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpSignatureBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpSignatureBufferIN_UI8,
+				 IMG_UINT8 * psPDumpSignatureBufferOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *) IMG_OFFSET_ADDR(psPDumpSignatureBufferIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *)
+	    IMG_OFFSET_ADDR(psPDumpSignatureBufferOUT_UI8, 0);
+
+	psPDumpSignatureBufferOUT->eError =
+	    PVRSRVPDumpSignatureBufferKM(psConnection, OSGetDevNode(psConnection),
+					 psPDumpSignatureBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+#define PVRSRVBridgePDumpComputeCRCSignatureCheck NULL
+
+static IMG_INT
+PVRSRVBridgePDumpCRCSignatureCheck(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckIN_UI8,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckOUT_UI8, 0);
+
+	psPDumpCRCSignatureCheckOUT->eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(psConnection, OSGetDevNode(psConnection),
+					   psPDumpCRCSignatureCheckIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPreCommand(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psPDumpValCheckPreCommandIN_UI8,
+				    IMG_UINT8 * psPDumpValCheckPreCommandOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandOUT_UI8, 0);
+
+	psPDumpValCheckPreCommandOUT->eError =
+	    PVRSRVPDumpValCheckPreCommandKM(psConnection, OSGetDevNode(psConnection),
+					    psPDumpValCheckPreCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPostCommand(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPDumpValCheckPostCommandIN_UI8,
+				     IMG_UINT8 * psPDumpValCheckPostCommandOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandOUT_UI8, 0);
+
+	psPDumpValCheckPostCommandOUT->eError =
+	    PVRSRVPDumpValCheckPostCommandKM(psConnection, OSGetDevNode(psConnection),
+					     psPDumpValCheckPostCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+
+/*
+ * Register all RGXPDUMP functions with services
+ */
+PVRSRV_ERROR InitRGXPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER,
+			      PVRSRVBridgePDumpTraceBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER,
+			      PVRSRVBridgePDumpSignatureBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpComputeCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND,
+			      PVRSRVBridgePDumpValCheckPreCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND,
+			      PVRSRVBridgePDumpValCheckPostCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxpdump functions with services
+ */
+void DeinitRGXPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/common_rgxray_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/common_rgxray_bridge.h
new file mode 100644
index 000000000000..8a5ecd699492
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/common_rgxray_bridge.h
@@ -0,0 +1,133 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxray
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxray
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXRAY_BRIDGE_H
+#define COMMON_RGXRAY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXRAY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXRAY_RGXCREATERAYCONTEXT			PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXRAY_RGXDESTROYRAYCONTEXT			PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXRAY_RGXKICKRDM			PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXRAY_CMD_LAST			(PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+2)
+
+/*******************************************
+            RGXCreateRayContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRayContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8sStaticRayContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32StaticRayContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT;
+
+/* Bridge out structure for RGXCreateRayContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT_TAG
+{
+	IMG_HANDLE hRayContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT;
+
+/*******************************************
+            RGXDestroyRayContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRayContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT_TAG
+{
+	IMG_HANDLE hRayContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT;
+
+/* Bridge out structure for RGXDestroyRayContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT;
+
+/*******************************************
+            RGXKickRDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickRDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKRDM_TAG
+{
+	IMG_UINT64 ui64ui64DeadlineInus;
+	IMG_HANDLE hRayContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32ui32AccStructSizeInBytes;
+	IMG_UINT32 ui32ui32DispatchSize;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKRDM;
+
+/* Bridge out structure for RGXKickRDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKRDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKRDM;
+
+#endif /* COMMON_RGXRAY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/server_rgxray_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/server_rgxray_bridge.c
new file mode 100644
index 000000000000..c537779917bf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/server_rgxray_bridge.c
@@ -0,0 +1,634 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxray
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxray
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxray.h"
+
+#include "common_rgxray_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateRayContextpsRayContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRayContextKM((RGX_SERVER_RAY_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_STATIC_RAYCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RAYCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRayContext(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXCreateRayContextIN_UI8,
+				IMG_UINT8 * psRGXCreateRayContextOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT *psRGXCreateRayContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT *) IMG_OFFSET_ADDR(psRGXCreateRayContextIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT *psRGXCreateRayContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT *) IMG_OFFSET_ADDR(psRGXCreateRayContextOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPrivData = psRGXCreateRayContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8sStaticRayContextStateInt = NULL;
+	RGX_SERVER_RAY_CONTEXT *psRayContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRayContextIN->ui32StaticRayContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely
+	    (psRGXCreateRayContextIN->ui32StaticRayContextStateSize >
+	     RGXFWIF_STATIC_RAYCONTEXT_SIZE))
+	{
+		psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRayContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRayContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRayContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRayContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRayContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRayContextIN->ui32StaticRayContextStateSize != 0)
+	{
+		ui8sStaticRayContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRayContextIN->ui32StaticRayContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRayContextIN->ui32StaticRayContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8sStaticRayContextStateInt,
+		     (const void __user *)psRGXCreateRayContextIN->pui8sStaticRayContextState,
+		     psRGXCreateRayContextIN->ui32StaticRayContextStateSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRayContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRayContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRayContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRayContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRayContextOUT->eError =
+	    PVRSRVRGXCreateRayContextKM(psConnection, OSGetDevNode(psConnection),
+					psRGXCreateRayContextIN->i32Priority,
+					hPrivDataInt,
+					psRGXCreateRayContextIN->ui32ContextFlags,
+					psRGXCreateRayContextIN->ui32StaticRayContextStateSize,
+					ui8sStaticRayContextStateInt,
+					psRGXCreateRayContextIN->ui64RobustnessAddress,
+					psRGXCreateRayContextIN->ui32MaxDeadlineMS,
+					&psRayContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRayContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRayContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRayContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXCreateRayContextOUT->
+								     hRayContext,
+								     (void *)psRayContextInt,
+								     PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXCreateRayContextpsRayContextIntRelease);
+	if (unlikely(psRGXCreateRayContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRayContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRayContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRayContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRayContextInt)
+		{
+			PVRSRVRGXDestroyRayContextKM(psRayContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRayContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRayContext(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXDestroyRayContextIN_UI8,
+				 IMG_UINT8 * psRGXDestroyRayContextOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT *psRGXDestroyRayContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT *) IMG_OFFSET_ADDR(psRGXDestroyRayContextIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT *psRGXDestroyRayContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRayContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRayContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRayContextIN->hRayContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT);
+	if (unlikely((psRGXDestroyRayContextOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyRayContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyRayContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRayContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRayContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRayContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickRDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickRDMIN_UI8,
+		       IMG_UINT8 * psRGXKickRDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKRDM *psRGXKickRDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKRDM *) IMG_OFFSET_ADDR(psRGXKickRDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKRDM *psRGXKickRDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKRDM *) IMG_OFFSET_ADDR(psRGXKickRDMOUT_UI8, 0);
+
+	IMG_HANDLE hRayContext = psRGXKickRDMIN->hRayContext;
+	RGX_SERVER_RAY_CONTEXT *psRayContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickRDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickRDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickRDM_exit;
+	}
+
+	if (unlikely(psRGXKickRDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickRDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickRDM_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickRDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickRDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickRDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickRDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickRDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickRDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickRDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickRDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickRDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+	if (psRGXKickRDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickRDMIN->pui32ClientUpdateOffset,
+		     psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+	if (psRGXKickRDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickRDMIN->pui32ClientUpdateValue,
+		     psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickRDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickRDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickRDMIN->pui8DMCmd,
+		     psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickRDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRayContextInt,
+				       hRayContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickRDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickRDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickRDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickRDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickRDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickRDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickRDMOUT->eError =
+	    PVRSRVRGXKickRDMKM(psRayContextInt,
+			       psRGXKickRDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickRDMIN->hCheckFenceFd,
+			       psRGXKickRDMIN->hUpdateTimeline,
+			       &psRGXKickRDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickRDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickRDMIN->ui32PDumpFlags,
+			       psRGXKickRDMIN->ui32ExtJobRef,
+			       psRGXKickRDMIN->ui32ui32AccStructSizeInBytes,
+			       psRGXKickRDMIN->ui32ui32DispatchSize,
+			       psRGXKickRDMIN->ui64ui64DeadlineInus);
+
+RGXKickRDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRayContext, PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickRDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickRDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXRAYBridge(void);
+void DeinitRGXRAYBridge(void);
+
+/*
+ * Register all RGXRAY functions with services
+ */
+PVRSRV_ERROR InitRGXRAYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXCREATERAYCONTEXT,
+			      PVRSRVBridgeRGXCreateRayContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXDESTROYRAYCONTEXT,
+			      PVRSRVBridgeRGXDestroyRayContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXKICKRDM,
+			      PVRSRVBridgeRGXKickRDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKRDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKRDM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxray functions with services
+ */
+void DeinitRGXRAYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXCREATERAYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXDESTROYRAYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXKICKRDM);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/common_rgxregconfig_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/common_rgxregconfig_bridge.h
new file mode 100644
index 000000000000..942b7e4b06d0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/common_rgxregconfig_bridge.h
@@ -0,0 +1,146 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXREGCONFIG_BRIDGE_H
+#define COMMON_RGXREGCONFIG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST			(PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetRegConfigType
+ *******************************************/
+
+/* Bridge in structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE_TAG
+{
+	IMG_UINT8 ui8RegPowerIsland;
+} __packed PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE;
+
+/* Bridge out structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE;
+
+/*******************************************
+            RGXAddRegconfig
+ *******************************************/
+
+/* Bridge in structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXADDREGCONFIG_TAG
+{
+	IMG_UINT64 ui64RegMask;
+	IMG_UINT64 ui64RegValue;
+	IMG_UINT32 ui32RegAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXADDREGCONFIG;
+
+/* Bridge out structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG;
+
+/*******************************************
+            RGXClearRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG;
+
+/* Bridge out structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG;
+
+/*******************************************
+            RGXEnableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG;
+
+/* Bridge out structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG;
+
+/*******************************************
+            RGXDisableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG;
+
+/* Bridge out structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG;
+
+#endif /* COMMON_RGXREGCONFIG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/server_rgxregconfig_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/server_rgxregconfig_bridge.c
new file mode 100644
index 000000000000..548401470d09
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/server_rgxregconfig_bridge.c
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxregconfig.h"
+
+#include "common_rgxregconfig_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetRegConfigType(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXSetRegConfigTypeIN_UI8,
+				IMG_UINT8 * psRGXSetRegConfigTypeOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeOUT_UI8,
+								      0);
+
+	psRGXSetRegConfigTypeOUT->eError =
+	    PVRSRVRGXSetRegConfigTypeKM(psConnection, OSGetDevNode(psConnection),
+					psRGXSetRegConfigTypeIN->ui8RegPowerIsland);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXAddRegconfig(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRGXAddRegconfigIN_UI8,
+			    IMG_UINT8 * psRGXAddRegconfigOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *psRGXAddRegconfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *psRGXAddRegconfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigOUT_UI8, 0);
+
+	psRGXAddRegconfigOUT->eError =
+	    PVRSRVRGXAddRegConfigKM(psConnection, OSGetDevNode(psConnection),
+				    psRGXAddRegconfigIN->ui32RegAddr,
+				    psRGXAddRegconfigIN->ui64RegValue,
+				    psRGXAddRegconfigIN->ui64RegMask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXClearRegConfigIN_UI8,
+			      IMG_UINT8 * psRGXClearRegConfigOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *psRGXClearRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *psRGXClearRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXClearRegConfigIN);
+
+	psRGXClearRegConfigOUT->eError =
+	    PVRSRVRGXClearRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXEnableRegConfigIN_UI8,
+			       IMG_UINT8 * psRGXEnableRegConfigOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *psRGXEnableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *psRGXEnableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEnableRegConfigIN);
+
+	psRGXEnableRegConfigOUT->eError =
+	    PVRSRVRGXEnableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXDisableRegConfigIN_UI8,
+				IMG_UINT8 * psRGXDisableRegConfigOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *psRGXDisableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *psRGXDisableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXDisableRegConfigIN);
+
+	psRGXDisableRegConfigOUT->eError =
+	    PVRSRVRGXDisableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+
+/*
+ * Register all RGXREGCONFIG functions with services
+ */
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE,
+			      PVRSRVBridgeRGXSetRegConfigType, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG,
+			      PVRSRVBridgeRGXAddRegconfig, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXADDREGCONFIG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG,
+			      PVRSRVBridgeRGXClearRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG,
+			      PVRSRVBridgeRGXEnableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG,
+			      PVRSRVBridgeRGXDisableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxregconfig functions with services
+ */
+void DeinitRGXREGCONFIGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG);
+
+}
+#else /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXREGCONFIG_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXREGCONFIGBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXREGCONFIGBridge()
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/common_rgxta3d_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/common_rgxta3d_bridge.h
new file mode 100644
index 000000000000..9157c6895073
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/common_rgxta3d_bridge.h
@@ -0,0 +1,425 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTA3D_BRIDGE_H
+#define COMMON_RGXTA3D_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_fwif_shared.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_LAST			(PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15)
+
+/*******************************************
+            RGXDestroyHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET_TAG
+{
+	IMG_HANDLE hKmHwRTDataSet;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET;
+
+/* Bridge out structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET;
+
+/*******************************************
+            RGXCreateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER;
+
+/* Bridge out structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferMemDesc;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER;
+
+/* Bridge out structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER;
+
+/*******************************************
+            RGXPopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+} __packed PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXUnpopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+} __packed PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST;
+
+/* Bridge out structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST;
+
+/*******************************************
+            RGXDestroyRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT;
+
+/* Bridge out structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT;
+
+/*******************************************
+            RGXSendZSStoreDisable
+ *******************************************/
+
+/* Bridge in structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32ExtJobRefToDisableZSStore;
+	IMG_BOOL bDisableDepthStore;
+	IMG_BOOL bDisableStencilStore;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE;
+
+/* Bridge out structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE;
+
+/*******************************************
+            RGXSetRenderContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXRenderContextStalled
+ *******************************************/
+
+/* Bridge in structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED_TAG
+{
+	IMG_HANDLE hRenderContext;
+} __packed PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED;
+
+/* Bridge out structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED;
+
+/*******************************************
+            RGXKickTA3D2
+ *******************************************/
+
+/* Bridge in structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTA3D2_TAG
+{
+	IMG_UINT64 ui64Deadline;
+	IMG_HANDLE hKMHWRTDataSet;
+	IMG_HANDLE hMSAAScratchBuffer;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock;
+	IMG_HANDLE hRenderContext;
+	IMG_HANDLE hZSBuffer;
+	IMG_UINT32 *pui32Client3DUpdateSyncOffset;
+	IMG_UINT32 *pui32Client3DUpdateValue;
+	IMG_UINT32 *pui32ClientTAFenceSyncOffset;
+	IMG_UINT32 *pui32ClientTAFenceValue;
+	IMG_UINT32 *pui32ClientTAUpdateSyncOffset;
+	IMG_UINT32 *pui32ClientTAUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui83DCmd;
+	IMG_BYTE *pui83DPRCmd;
+	IMG_BYTE *pui8TACmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_CHAR *puiUpdateFenceName3D;
+	IMG_HANDLE *phClient3DUpdateSyncPrimBlock;
+	IMG_HANDLE *phClientTAFenceSyncPrimBlock;
+	IMG_HANDLE *phClientTAUpdateSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFence;
+	PVRSRV_FENCE hCheckFence3D;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	PVRSRV_TIMELINE hUpdateTimeline3D;
+	IMG_UINT32 ui323DCmdSize;
+	IMG_UINT32 ui323DPRCmdSize;
+	IMG_UINT32 ui32Client3DUpdateCount;
+	IMG_UINT32 ui32ClientTAFenceCount;
+	IMG_UINT32 ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumberOfDrawCalls;
+	IMG_UINT32 ui32NumberOfIndices;
+	IMG_UINT32 ui32NumberOfMRTs;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PRFenceUFOSyncOffset;
+	IMG_UINT32 ui32PRFenceValue;
+	IMG_UINT32 ui32RenderTargetSize;
+	IMG_UINT32 ui32SyncPMRCount;
+	IMG_UINT32 ui32TACmdSize;
+	IMG_BOOL bbAbort;
+	IMG_BOOL bbKick3D;
+	IMG_BOOL bbKickPR;
+	IMG_BOOL bbKickTA;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTA3D2;
+
+/* Bridge out structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTA3D2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	PVRSRV_FENCE hUpdateFence3D;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTA3D2;
+
+/*******************************************
+            RGXSetRenderContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hRenderContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXCreateHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_DEV_VIRTADDR sVHeapTableDevVAddr;
+	IMG_DEV_VIRTADDR *psPMDataAddr;
+	IMG_DEV_VIRTADDR *psPMSecureDataAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET;
+
+/* Bridge out structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET;
+
+/*******************************************
+            RGXCreateFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hFreeListAndStateReservation;
+	IMG_HANDLE hMemCtxPrivData;
+	IMG_HANDLE hsGlobalFreeList;
+	IMG_UINT32 ui32GrowFLPages;
+	IMG_UINT32 ui32GrowParamThreshold;
+	IMG_UINT32 ui32InitFLPages;
+	IMG_UINT32 ui32MaxFLPages;
+	IMG_BOOL bbFreeListCheck;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEFREELIST;
+
+/* Bridge out structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST;
+
+/*******************************************
+            RGXCreateRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticRenderContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32Max3DDeadlineMS;
+	IMG_UINT32 ui32MaxTADeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+	IMG_UINT32 ui32StaticRenderContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT;
+
+/* Bridge out structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_HANDLE hRenderContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT;
+
+/*******************************************
+            RGXCreateHWRTDataSet2
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_DEV_VIRTADDR sVHeapTableDevVAddr;
+	IMG_HANDLE hMListsReservation;
+	IMG_HANDLE hPMSecureStatesReservation;
+	IMG_HANDLE hPMStatesReservation;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2;
+
+/* Bridge out structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2;
+
+#endif /* COMMON_RGXTA3D_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/server_rgxta3d_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/server_rgxta3d_bridge.c
new file mode 100644
index 000000000000..dbb87e97fb26
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/server_rgxta3d_bridge.c
@@ -0,0 +1,2842 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxta3d.h"
+
+#include "common_rgxta3d_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetIN_UI8,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyHWRTDataSetOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyHWRTDataSetIN->
+					      hKmHwRTDataSet,
+					      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+	if (unlikely
+	    ((psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyHWRTDataSetOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyHWRTDataSet_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyHWRTDataSet_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateZSBufferpssZSBufferKMIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyZSBufferKM((RGX_ZSBUFFER_DATA *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateZSBufferIN_UI8,
+			      IMG_UINT8 * psRGXCreateZSBufferOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *psRGXCreateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *psRGXCreateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psRGXCreateZSBufferIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psRGXCreateZSBufferIN->hPMR;
+	PMR *psPMRInt = NULL;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError =
+	    RGXCreateZSBufferKM(psConnection, OSGetDevNode(psConnection),
+				psReservationInt,
+				psPMRInt, psRGXCreateZSBufferIN->uiMapFlags, &pssZSBufferKMInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateZSBufferOUT->
+								   hsZSBufferKM,
+								   (void *)pssZSBufferKMInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateZSBufferpssZSBufferKMIntRelease);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssZSBufferKMInt)
+		{
+			RGXDestroyZSBufferKM(pssZSBufferKMInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyZSBufferIN_UI8,
+			       IMG_UINT8 * psRGXDestroyZSBufferOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyZSBufferIN->
+					      hsZSBufferMemDesc,
+					      PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	if (unlikely
+	    ((psRGXDestroyZSBufferOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyZSBuffer_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXPopulateZSBufferpssPopulationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXUnpopulateZSBufferKM((RGX_POPULATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXPopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXPopulateZSBufferIN_UI8,
+				IMG_UINT8 * psRGXPopulateZSBufferOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferOUT_UI8,
+								      0);
+
+	IMG_HANDLE hsZSBufferKM = psRGXPopulateZSBufferIN->hsZSBufferKM;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+	RGX_POPULATION *pssPopulationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXPopulateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pssZSBufferKMInt,
+				       hsZSBufferKM,
+				       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError =
+	    RGXPopulateZSBufferKM(pssZSBufferKMInt, &pssPopulationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXPopulateZSBufferOUT->
+								     hsPopulation,
+								     (void *)pssPopulationInt,
+								     PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXPopulateZSBufferpssPopulationIntRelease);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXPopulateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pssZSBufferKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hsZSBufferKM, PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXPopulateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssPopulationInt)
+		{
+			RGXUnpopulateZSBufferKM(pssPopulationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXUnpopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferIN_UI8,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXUnpopulateZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXUnpopulateZSBufferIN->hsPopulation,
+					      PVRSRV_HANDLE_TYPE_RGX_POPULATION);
+	if (unlikely((psRGXUnpopulateZSBufferOUT->eError != PVRSRV_OK) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXUnpopulateZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXUnpopulateZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXUnpopulateZSBuffer_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyFreeListIN_UI8,
+			       IMG_UINT8 * psRGXDestroyFreeListOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *psRGXDestroyFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *psRGXDestroyFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyFreeListOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyFreeListIN->hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+	if (unlikely((psRGXDestroyFreeListOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyFreeListOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyFreeList_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyFreeList_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXDestroyRenderContextIN_UI8,
+				    IMG_UINT8 * psRGXDestroyRenderContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRenderContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRenderContextIN->
+					      hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyRenderContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRenderContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRenderContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRenderContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendZSStoreDisable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXSendZSStoreDisableIN_UI8,
+				  IMG_UINT8 * psRGXSendZSStoreDisableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSendZSStoreDisableIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendZSStoreDisableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendZSStoreDisable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVRGXSendZSStoreDisableKM(psConnection, OSGetDevNode(psConnection),
+					  psRenderContextInt,
+					  psRGXSendZSStoreDisableIN->bDisableDepthStore,
+					  psRGXSendZSStoreDisableIN->bDisableStencilStore,
+					  psRGXSendZSStoreDisableIN->i32ExtJobRefToDisableZSStore);
+
+RGXSendZSStoreDisable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPriorityIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPriorityIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVRGXSetRenderContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRenderContextInt,
+						psRGXSetRenderContextPriorityIN->i32Priority);
+
+RGXSetRenderContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXRenderContextStalled(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXRenderContextStalledIN_UI8,
+				    IMG_UINT8 * psRGXRenderContextStalledOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledIN =
+	    (PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXRenderContextStalledIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXRenderContextStalledOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXRenderContextStalledOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXRenderContextStalled_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXRenderContextStalledOUT->eError = RGXRenderContextStalledKM(psRenderContextInt);
+
+RGXRenderContextStalled_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTA3D2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickTA3D2IN_UI8,
+			 IMG_UINT8 * psRGXKickTA3D2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *psRGXKickTA3D2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *psRGXKickTA3D2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2OUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXKickTA3D2IN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAFenceSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAFenceSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAFenceSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAFenceValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClient3DUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClient3DUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32Client3DUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32Client3DUpdateValueInt = NULL;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock = psRGXKickTA3D2IN->hPRFenceUFOSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *psPRFenceUFOSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_CHAR *uiUpdateFenceName3DInt = NULL;
+	IMG_BYTE *ui8TACmdInt = NULL;
+	IMG_BYTE *ui83DPRCmdInt = NULL;
+	IMG_BYTE *ui83DCmdInt = NULL;
+	IMG_HANDLE hKMHWRTDataSet = psRGXKickTA3D2IN->hKMHWRTDataSet;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSetInt = NULL;
+	IMG_HANDLE hZSBuffer = psRGXKickTA3D2IN->hZSBuffer;
+	RGX_ZSBUFFER_DATA *psZSBufferInt = NULL;
+	IMG_HANDLE hMSAAScratchBuffer = psRGXKickTA3D2IN->hMSAAScratchBuffer;
+	RGX_ZSBUFFER_DATA *psMSAAScratchBufferInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAFenceCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32Client3DUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32TACmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DPRCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTA3D2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTA3D2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTA3D2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		psClientTAFenceSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAFenceSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAFenceSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAFenceSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAFenceSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceValue,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		psClientTAUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateValue,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		psClient3DUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClient3DUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClient3DUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClient3DUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClient3DUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateValue,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	{
+		uiUpdateFenceName3DInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceName3DInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName3D,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceName3DInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						      1] = '\0';
+	}
+	if (psRGXKickTA3D2IN->ui32TACmdSize != 0)
+	{
+		ui8TACmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8TACmdInt, (const void __user *)psRGXKickTA3D2IN->pui8TACmd,
+		     psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize != 0)
+	{
+		ui83DPRCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DPRCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DPRCmd,
+		     psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DCmdSize != 0)
+	{
+		ui83DCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DCmd,
+		     psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32SyncPMRFlags,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickTA3D2IN->phSyncPMRs,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psClientTAFenceSyncPrimBlockInt[i],
+						       hClientTAFenceSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientTAUpdateSyncPrimBlockInt[i],
+						       hClientTAUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClient3DUpdateSyncPrimBlockInt[i],
+						       hClient3DUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPRFenceUFOSyncPrimBlockInt,
+				       hPRFenceUFOSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psKMHWRTDataSetInt,
+					       hKMHWRTDataSet,
+					       PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psZSBufferInt,
+					       hZSBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psMSAAScratchBufferInt,
+					       hMSAAScratchBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVRGXKickTA3DKM(psRenderContextInt,
+				psRGXKickTA3D2IN->ui32ClientTAFenceCount,
+				psClientTAFenceSyncPrimBlockInt,
+				ui32ClientTAFenceSyncOffsetInt,
+				ui32ClientTAFenceValueInt,
+				psRGXKickTA3D2IN->ui32ClientTAUpdateCount,
+				psClientTAUpdateSyncPrimBlockInt,
+				ui32ClientTAUpdateSyncOffsetInt,
+				ui32ClientTAUpdateValueInt,
+				psRGXKickTA3D2IN->ui32Client3DUpdateCount,
+				psClient3DUpdateSyncPrimBlockInt,
+				ui32Client3DUpdateSyncOffsetInt,
+				ui32Client3DUpdateValueInt,
+				psPRFenceUFOSyncPrimBlockInt,
+				psRGXKickTA3D2IN->ui32PRFenceUFOSyncOffset,
+				psRGXKickTA3D2IN->ui32PRFenceValue,
+				psRGXKickTA3D2IN->hCheckFence,
+				psRGXKickTA3D2IN->hUpdateTimeline,
+				&psRGXKickTA3D2OUT->hUpdateFence,
+				uiUpdateFenceNameInt,
+				psRGXKickTA3D2IN->hCheckFence3D,
+				psRGXKickTA3D2IN->hUpdateTimeline3D,
+				&psRGXKickTA3D2OUT->hUpdateFence3D,
+				uiUpdateFenceName3DInt,
+				psRGXKickTA3D2IN->ui32TACmdSize,
+				ui8TACmdInt,
+				psRGXKickTA3D2IN->ui323DPRCmdSize,
+				ui83DPRCmdInt,
+				psRGXKickTA3D2IN->ui323DCmdSize,
+				ui83DCmdInt,
+				psRGXKickTA3D2IN->ui32ExtJobRef,
+				psRGXKickTA3D2IN->bbKickTA,
+				psRGXKickTA3D2IN->bbKickPR,
+				psRGXKickTA3D2IN->bbKick3D,
+				psRGXKickTA3D2IN->bbAbort,
+				psRGXKickTA3D2IN->ui32PDumpFlags,
+				psKMHWRTDataSetInt,
+				psZSBufferInt,
+				psMSAAScratchBufferInt,
+				psRGXKickTA3D2IN->ui32SyncPMRCount,
+				ui32SyncPMRFlagsInt,
+				psSyncPMRsInt,
+				psRGXKickTA3D2IN->ui32RenderTargetSize,
+				psRGXKickTA3D2IN->ui32NumberOfDrawCalls,
+				psRGXKickTA3D2IN->ui32NumberOfIndices,
+				psRGXKickTA3D2IN->ui32NumberOfMRTs, psRGXKickTA3D2IN->ui64Deadline);
+
+RGXKickTA3D2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+
+	if (hClientTAFenceSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAFenceSyncPrimBlockInt && psClientTAFenceSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAFenceSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClientTAUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAUpdateSyncPrimBlockInt && psClientTAUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClient3DUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClient3DUpdateSyncPrimBlockInt && psClient3DUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClient3DUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPRFenceUFOSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPRFenceUFOSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psKMHWRTDataSetInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hKMHWRTDataSet,
+						    PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psZSBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hZSBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psMSAAScratchBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hMSAAScratchBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTA3D2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPropertyIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPropertyOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPropertyIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVRGXSetRenderContextPropertyKM(psRenderContextInt,
+						psRGXSetRenderContextPropertyIN->ui32Property,
+						psRGXSetRenderContextPropertyIN->ui64Input,
+						&psRGXSetRenderContextPropertyOUT->ui64Output);
+
+RGXSetRenderContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetIN_UI8,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *) IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetOUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sPMDataAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sPMSecureDataAddrInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet = psRGXCreateHWRTDataSetIN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSetIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSetIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+
+	{
+		sPMDataAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMDataAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMDataAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sPMSecureDataAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMSecureDataAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMSecureDataAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSetOUT->eError =
+	    RGXCreateHWRTDataSet(psConnection, OSGetDevNode(psConnection),
+				 psRGXCreateHWRTDataSetIN->sVHeapTableDevVAddr,
+				 sPMDataAddrInt,
+				 sPMSecureDataAddrInt,
+				 psapsFreeListsInt,
+				 sTailPtrsDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSetOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateFreeListpsCleanupCookieIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyFreeList((RGX_FREELIST *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateFreeListIN_UI8,
+			      IMG_UINT8 * psRGXCreateFreeListOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *psRGXCreateFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *psRGXCreateFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListOUT_UI8, 0);
+
+	IMG_HANDLE hMemCtxPrivData = psRGXCreateFreeListIN->hMemCtxPrivData;
+	IMG_HANDLE hMemCtxPrivDataInt = NULL;
+	IMG_HANDLE hsGlobalFreeList = psRGXCreateFreeListIN->hsGlobalFreeList;
+	RGX_FREELIST *pssGlobalFreeListInt = NULL;
+	IMG_HANDLE hFreeListAndStateReservation =
+	    psRGXCreateFreeListIN->hFreeListAndStateReservation;
+	DEVMEMINT_RESERVATION *psFreeListAndStateReservationInt = NULL;
+	RGX_FREELIST *psCleanupCookieInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hMemCtxPrivDataInt,
+				       hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+		/* Look up the address from the handle */
+		psRGXCreateFreeListOUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&pssGlobalFreeListInt,
+					       hsGlobalFreeList,
+					       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+		if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXCreateFreeList_exit;
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psFreeListAndStateReservationInt,
+				       hFreeListAndStateReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError =
+	    RGXCreateFreeList(psConnection, OSGetDevNode(psConnection),
+			      hMemCtxPrivDataInt,
+			      psRGXCreateFreeListIN->ui32MaxFLPages,
+			      psRGXCreateFreeListIN->ui32InitFLPages,
+			      psRGXCreateFreeListIN->ui32GrowFLPages,
+			      psRGXCreateFreeListIN->ui32GrowParamThreshold,
+			      pssGlobalFreeListInt,
+			      psRGXCreateFreeListIN->bbFreeListCheck,
+			      psFreeListAndStateReservationInt, &psCleanupCookieInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateFreeListOUT->
+								   hCleanupCookie,
+								   (void *)psCleanupCookieInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateFreeListpsCleanupCookieIntRelease);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateFreeList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hMemCtxPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (pssGlobalFreeListInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hsGlobalFreeList,
+						    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psFreeListAndStateReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hFreeListAndStateReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateFreeListOUT->eError != PVRSRV_OK)
+	{
+		if (psCleanupCookieInt)
+		{
+			RGXDestroyFreeList(psCleanupCookieInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateRenderContextpsRenderContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRenderContextKM((RGX_SERVER_RENDER_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_RENDERCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RENDERCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXCreateRenderContextIN_UI8,
+				   IMG_UINT8 * psRGXCreateRenderContextOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateRenderContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticRenderContextStateInt = NULL;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateRenderContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize >
+	     RGXFWIF_STATIC_RENDERCONTEXT_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRenderContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRenderContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRenderContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8FrameworkCmd,
+		     psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize != 0)
+	{
+		ui8StaticRenderContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticRenderContextStateInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8StaticRenderContextState,
+		     psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVRGXCreateRenderContextKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXCreateRenderContextIN->i32Priority,
+					   psRGXCreateRenderContextIN->ui32FrameworkCmdSize,
+					   ui8FrameworkCmdInt,
+					   hPrivDataInt,
+					   psRGXCreateRenderContextIN->
+					   ui32StaticRenderContextStateSize,
+					   ui8StaticRenderContextStateInt,
+					   psRGXCreateRenderContextIN->ui32PackedCCBSizeU8888,
+					   psRGXCreateRenderContextIN->ui32ContextFlags,
+					   psRGXCreateRenderContextIN->ui64RobustnessAddress,
+					   psRGXCreateRenderContextIN->ui32MaxTADeadlineMS,
+					   psRGXCreateRenderContextIN->ui32Max3DDeadlineMS,
+					   &psRenderContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psRGXCreateRenderContextOUT->
+									hRenderContext,
+									(void *)psRenderContextInt,
+									PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_RGXCreateRenderContextpsRenderContextIntRelease);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRenderContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRenderContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRenderContextInt)
+		{
+			PVRSRVRGXDestroyRenderContextKM(psRenderContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRenderContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2IN_UI8,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2IN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2OUT_UI8, 0);
+
+	IMG_HANDLE hMListsReservation = psRGXCreateHWRTDataSet2IN->hMListsReservation;
+	DEVMEMINT_RESERVATION *psMListsReservationInt = NULL;
+	IMG_HANDLE hPMStatesReservation = psRGXCreateHWRTDataSet2IN->hPMStatesReservation;
+	DEVMEMINT_RESERVATION *psPMStatesReservationInt = NULL;
+	IMG_HANDLE hPMSecureStatesReservation =
+	    psRGXCreateHWRTDataSet2IN->hPMSecureStatesReservation;
+	DEVMEMINT_RESERVATION *psPMSecureStatesReservationInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet = psRGXCreateHWRTDataSet2IN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSet2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSet2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psMListsReservationInt,
+				       hMListsReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMStatesReservationInt,
+				       hPMStatesReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMSecureStatesReservationInt,
+				       hPMSecureStatesReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    RGXCreateHWRTDataSet2(psConnection, OSGetDevNode(psConnection),
+				  psRGXCreateHWRTDataSet2IN->sVHeapTableDevVAddr,
+				  psMListsReservationInt,
+				  psPMStatesReservationInt,
+				  psPMSecureStatesReservationInt,
+				  psapsFreeListsInt,
+				  sTailPtrsDevVAddrInt,
+				  psRGXCreateHWRTDataSet2IN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psMListsReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMListsReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMStatesReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMStatesReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMSecureStatesReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMSecureStatesReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSet2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+
+/*
+ * Register all RGXTA3D functions with services
+ */
+PVRSRV_ERROR InitRGXTA3DBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET,
+			      PVRSRVBridgeRGXDestroyHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER,
+			      PVRSRVBridgeRGXCreateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER,
+			      PVRSRVBridgeRGXDestroyZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXPopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXUnpopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST,
+			      PVRSRVBridgeRGXDestroyFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT,
+			      PVRSRVBridgeRGXDestroyRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE,
+			      PVRSRVBridgeRGXSendZSStoreDisable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetRenderContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED,
+			      PVRSRVBridgeRGXRenderContextStalled, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2,
+			      PVRSRVBridgeRGXKickTA3D2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKTA3D2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTA3D2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetRenderContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET,
+			      PVRSRVBridgeRGXCreateHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST,
+			      PVRSRVBridgeRGXCreateFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT,
+			      PVRSRVBridgeRGXCreateRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2,
+			      PVRSRVBridgeRGXCreateHWRTDataSet2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxta3d functions with services
+ */
+void DeinitRGXTA3DBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
new file mode 100644
index 000000000000..34d7c2c56a2c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTIMERQUERY_BRIDGE_H
+#define COMMON_RGXTIMERQUERY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST			(PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2)
+
+/*******************************************
+            RGXBeginTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY;
+
+/* Bridge out structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY;
+
+/*******************************************
+            RGXEndTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY;
+
+/* Bridge out structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY;
+
+/*******************************************
+            RGXQueryTimer
+ *******************************************/
+
+/* Bridge in structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_IN_RGXQUERYTIMER_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXQUERYTIMER;
+
+/* Bridge out structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXQUERYTIMER_TAG
+{
+	IMG_UINT64 ui64EndTime;
+	IMG_UINT64 ui64StartTime;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXQUERYTIMER;
+
+#endif /* COMMON_RGXTIMERQUERY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
new file mode 100644
index 000000000000..4af3cfc35cba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
@@ -0,0 +1,171 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtimerquery.h"
+
+#include "common_rgxtimerquery_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXBeginTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXBeginTimerQueryIN_UI8,
+			       IMG_UINT8 * psRGXBeginTimerQueryOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryOUT_UI8,
+								     0);
+
+	psRGXBeginTimerQueryOUT->eError =
+	    PVRSRVRGXBeginTimerQueryKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXBeginTimerQueryIN->ui32QueryId);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEndTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXEndTimerQueryIN_UI8,
+			     IMG_UINT8 * psRGXEndTimerQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *psRGXEndTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *psRGXEndTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEndTimerQueryIN);
+
+	psRGXEndTimerQueryOUT->eError =
+	    PVRSRVRGXEndTimerQueryKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXQueryTimer(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXQueryTimerIN_UI8,
+			  IMG_UINT8 * psRGXQueryTimerOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXQUERYTIMER *psRGXQueryTimerIN =
+	    (PVRSRV_BRIDGE_IN_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *psRGXQueryTimerOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerOUT_UI8, 0);
+
+	psRGXQueryTimerOUT->eError =
+	    PVRSRVRGXQueryTimerKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXQueryTimerIN->ui32QueryId,
+				  &psRGXQueryTimerOUT->ui64StartTime,
+				  &psRGXQueryTimerOUT->ui64EndTime);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+
+/*
+ * Register all RGXTIMERQUERY functions with services
+ */
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY,
+			      PVRSRVBridgeRGXBeginTimerQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY,
+			      PVRSRVBridgeRGXEndTimerQuery, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER, PVRSRVBridgeRGXQueryTimer,
+			      NULL, sizeof(PVRSRV_BRIDGE_IN_RGXQUERYTIMER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXQUERYTIMER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtimerquery functions with services
+ */
+void DeinitRGXTIMERQUERYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/common_rgxtq2_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/common_rgxtq2_bridge.h
new file mode 100644
index 000000000000..f20fca015dc0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/common_rgxtq2_bridge.h
@@ -0,0 +1,264 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ2_BRIDGE_H
+#define COMMON_RGXTQ2_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8)
+
+/*******************************************
+            RGXTDMCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXTDMNotifyWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXTDMSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXTDMSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXTDMSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ2_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/server_rgxtq2_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/server_rgxtq2_bridge.c
new file mode 100644
index 000000000000..1129b824d850
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/server_rgxtq2_bridge.c
@@ -0,0 +1,1610 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtdmtransfer.h"
+
+#include "common_rgxtq2_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXTDMCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMDestroyTransferContextKM((RGX_SERVER_TQ_TDM_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXTDMCreateTransferContextIN_UI8,
+					IMG_UINT8 * psRGXTDMCreateTransferContextOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXTDMCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXTDMCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMCreateTransferContextOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXTDMCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVRGXTDMCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+						psRGXTDMCreateTransferContextIN->i32Priority,
+						psRGXTDMCreateTransferContextIN->
+						ui32FrameworkCmdSize, ui8FrameworkCmdInt,
+						hPrivDataInt,
+						psRGXTDMCreateTransferContextIN->
+						ui32PackedCCBSizeU88,
+						psRGXTDMCreateTransferContextIN->ui32ContextFlags,
+						psRGXTDMCreateTransferContextIN->
+						ui64RobustnessAddress,
+						psRGXTDMCreateTransferContextIN->ui32MaxDeadlineMS,
+						&psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXTDMCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXTDMCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXTDMDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextIN_UI8,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMDestroyTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMDestroyTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	if (unlikely
+	    ((psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psRGXTDMDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						     psTransferContextInt,
+						     psRGXTDMSetTransferContextPriorityIN->
+						     i32Priority);
+
+RGXTDMSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateIN_UI8,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMNotifyWriteOffsetUpdateIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMNotifyWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMNotifyWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMNotifyWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMNotifyWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(psTransferContextInt,
+						  psRGXTDMNotifyWriteOffsetUpdateIN->
+						  ui32PDumpFlags);
+
+RGXTDMNotifyWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer2IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVRGXTDMSubmitTransferKM(psTransferContextInt,
+					 psRGXTDMSubmitTransfer2IN->ui32PDumpFlags,
+					 psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount,
+					 psUpdateUFOSyncPrimBlockInt,
+					 ui32UpdateSyncOffsetInt,
+					 ui32UpdateValueInt,
+					 psRGXTDMSubmitTransfer2IN->hCheckFenceFD,
+					 psRGXTDMSubmitTransfer2IN->hUpdateTimeline,
+					 &psRGXTDMSubmitTransfer2OUT->hUpdateFence,
+					 uiUpdateFenceNameInt,
+					 psRGXTDMSubmitTransfer2IN->ui32CommandSize,
+					 ui8FWCommandInt,
+					 psRGXTDMSubmitTransfer2IN->ui32ExternalJobReference,
+					 psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount,
+					 ui32SyncPMRFlagsInt,
+					 psSyncPMRsInt,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic1,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic2,
+					 psRGXTDMSubmitTransfer2IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTDMGetSharedMemoryIN);
+
+	psRGXTDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXTDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMGetSharedMemory_exit:
+
+	if (psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPropertyKM(psTransferContextInt,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui32Property,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui64Input,
+						     &psRGXTDMSetTransferContextPropertyOUT->
+						     ui64Output);
+
+RGXTDMSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer3IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVRGXTDMSubmitTransfer3KM(psTransferContextInt,
+					  psRGXTDMSubmitTransfer3IN->ui32PDumpFlags,
+					  psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount,
+					  psUpdateUFOSyncPrimBlockInt,
+					  ui32UpdateSyncOffsetInt,
+					  ui32UpdateValueInt,
+					  psRGXTDMSubmitTransfer3IN->hCheckFenceFD,
+					  psRGXTDMSubmitTransfer3IN->hUpdateTimeline,
+					  &psRGXTDMSubmitTransfer3OUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXTDMSubmitTransfer3IN->hExportFenceToSignal,
+					  psRGXTDMSubmitTransfer3IN->ui32CommandSize,
+					  ui8FWCommandInt,
+					  psRGXTDMSubmitTransfer3IN->ui32ExternalJobReference,
+					  psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount,
+					  ui32SyncPMRFlagsInt,
+					  psSyncPMRsInt,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic1,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic2,
+					  psRGXTDMSubmitTransfer3IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+
+/*
+ * Register all RGXTQ2 functions with services
+ */
+PVRSRV_ERROR InitRGXTQ2Bridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXTDMSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXTDMSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXTDMSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXTDMSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq2 functions with services
+ */
+void DeinitRGXTQ2Bridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_bridge.h
new file mode 100644
index 000000000000..c419e21e9d07
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_bridge.h
@@ -0,0 +1,88 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RI_BRIDGE_H
+#define CLIENT_RI_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_ri_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner);
+
+#endif /* CLIENT_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_direct_bridge.c
new file mode 100644
index 000000000000..de037ee78be5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_direct_bridge.c
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for ri
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_ri_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "ri_typedefs.h"
+
+#include "ri_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError =
+	    RIWriteMEMDESCEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psPMRHandleInt,
+				  ui32TextBSize,
+				  puiTextB, ui64Offset, ui64Size, ui64Flags, &psRIHandleInt);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	eError =
+	    RIWriteProcListEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   ui32TextBSize, puiTextB, ui64Size, ui64DevVAddr, &psRIHandleInt);
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIUpdateMEMDESCAddrKM(psRIHandleInt, sAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIDeleteMEMDESCEntryKM(psRIHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIDumpListKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpAllKM();
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpProcessKM(ui32Pid);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRHandleInt, ui32Owner);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/common_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/common_ri_bridge.h
new file mode 100644
index 000000000000..baa41adccdfe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/common_ri_bridge.h
@@ -0,0 +1,224 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RI_BRIDGE_H
+#define COMMON_RI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "ri_typedefs.h"
+
+#define PVRSRV_BRIDGE_RI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR			PVRSRV_BRIDGE_RI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RI_RIDUMPLIST			PVRSRV_BRIDGE_RI_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RI_RIDUMPALL			PVRSRV_BRIDGE_RI_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RI_RIDUMPPROCESS			PVRSRV_BRIDGE_RI_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER			PVRSRV_BRIDGE_RI_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RI_CMD_LAST			(PVRSRV_BRIDGE_RI_CMD_FIRST+8)
+
+/*******************************************
+            RIWritePMREntry
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY;
+
+/* Bridge out structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY;
+
+/*******************************************
+            RIWriteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_UINT64 ui64Flags;
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRHandle;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY;
+
+/* Bridge out structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY;
+
+/*******************************************
+            RIWriteProcListEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_UINT64 ui64DevVAddr;
+	IMG_UINT64 ui64Size;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY;
+
+/* Bridge out structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY;
+
+/*******************************************
+            RIUpdateMEMDESCAddr
+ *******************************************/
+
+/* Bridge in structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR_TAG
+{
+	IMG_DEV_VIRTADDR sAddr;
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR;
+
+/* Bridge out structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR;
+
+/*******************************************
+            RIDeleteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY;
+
+/* Bridge out structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY;
+
+/*******************************************
+            RIDumpList
+ *******************************************/
+
+/* Bridge in structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPLIST_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPLIST;
+
+/* Bridge out structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPLIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPLIST;
+
+/*******************************************
+            RIDumpAll
+ *******************************************/
+
+/* Bridge in structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPALL_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPALL;
+
+/* Bridge out structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPALL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPALL;
+
+/*******************************************
+            RIDumpProcess
+ *******************************************/
+
+/* Bridge in structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPPROCESS_TAG
+{
+	IMG_PID ui32Pid;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPPROCESS;
+
+/* Bridge out structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPPROCESS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPPROCESS;
+
+/*******************************************
+            RIWritePMREntryWithOwner
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	IMG_HANDLE hPMRHandle;
+	IMG_PID ui32Owner;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER;
+
+/* Bridge out structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER;
+
+#endif /* COMMON_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/server_ri_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/server_ri_bridge.c
new file mode 100644
index 000000000000..f8a54d70ef0c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/server_ri_bridge.c
@@ -0,0 +1,761 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "ri_server.h"
+
+#include "common_ri_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntry(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRIWritePMREntryIN_UI8,
+			    IMG_UINT8 * psRIWritePMREntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *psRIWritePMREntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *psRIWritePMREntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryOUT->eError = RIWritePMREntryKM(psPMRHandleInt);
+
+RIWritePMREntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteMEMDESCEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIWriteMEMDESCEntryIN_UI8,
+				IMG_UINT8 * psRIWriteMEMDESCEntryOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMRHandle = psRIWriteMEMDESCEntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteMEMDESCEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteMEMDESCEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteMEMDESCEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteMEMDESCEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteMEMDESCEntryIN->puiTextB,
+		     psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteMEMDESCEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWriteMEMDESCEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError =
+	    RIWriteMEMDESCEntryKM(psConnection, OSGetDevNode(psConnection),
+				  psPMRHandleInt,
+				  psRIWriteMEMDESCEntryIN->ui32TextBSize,
+				  uiTextBInt,
+				  psRIWriteMEMDESCEntryIN->ui64Offset,
+				  psRIWriteMEMDESCEntryIN->ui64Size,
+				  psRIWriteMEMDESCEntryIN->ui64Flags, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRIWriteMEMDESCEntryOUT->
+								     hRIHandle,
+								     (void *)psRIHandleInt,
+								     PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RIWriteMEMDESCEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteMEMDESCEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteMEMDESCEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteProcListEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteProcListEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIWriteProcListEntryIN_UI8,
+				 IMG_UINT8 * psRIWriteProcListEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *) IMG_OFFSET_ADDR(psRIWriteProcListEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *)
+	    IMG_OFFSET_ADDR(psRIWriteProcListEntryOUT_UI8, 0);
+
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteProcListEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteProcListEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteProcListEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteProcListEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteProcListEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteProcListEntryIN->puiTextB,
+		     psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteProcListEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psRIWriteProcListEntryOUT->eError =
+	    RIWriteProcListEntryKM(psConnection, OSGetDevNode(psConnection),
+				   psRIWriteProcListEntryIN->ui32TextBSize,
+				   uiTextBInt,
+				   psRIWriteProcListEntryIN->ui64Size,
+				   psRIWriteProcListEntryIN->ui64DevVAddr, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteProcListEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRIWriteProcListEntryOUT->
+								      hRIHandle,
+								      (void *)psRIHandleInt,
+								      PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RIWriteProcListEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteProcListEntry_exit:
+
+	if (psRIWriteProcListEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteProcListEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIUpdateMEMDESCAddr(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrIN_UI8,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrIN =
+	    (PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrOUT_UI8,
+								      0);
+
+	IMG_HANDLE hRIHandle = psRIUpdateMEMDESCAddrIN->hRIHandle;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRIHandleInt,
+				       hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE, IMG_TRUE);
+	if (unlikely(psRIUpdateMEMDESCAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIUpdateMEMDESCAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    RIUpdateMEMDESCAddrKM(psRIHandleInt, psRIUpdateMEMDESCAddrIN->sAddr);
+
+RIUpdateMEMDESCAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRIHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDeleteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryIN_UI8,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *)
+	    IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIDeleteMEMDESCEntryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRIDeleteMEMDESCEntryIN->hRIHandle,
+					      PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	if (unlikely((psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_OK) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRIDeleteMEMDESCEntryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDeleteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIDeleteMEMDESCEntry_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpList(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRIDumpListIN_UI8,
+		       IMG_UINT8 * psRIDumpListOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPLIST *psRIDumpListIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPLIST *psRIDumpListOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIDumpListIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIDumpListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIDumpListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDumpList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIDumpListOUT->eError = RIDumpListKM(psPMRHandleInt);
+
+RIDumpList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpAll(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psRIDumpAllIN_UI8,
+		      IMG_UINT8 * psRIDumpAllOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPALL *psRIDumpAllIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPALL *psRIDumpAllOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psRIDumpAllIN);
+
+	psRIDumpAllOUT->eError = RIDumpAllKM();
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpProcess(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRIDumpProcessIN_UI8,
+			  IMG_UINT8 * psRIDumpProcessOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPPROCESS *psRIDumpProcessIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *psRIDumpProcessOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psRIDumpProcessOUT->eError = RIDumpProcessKM(psRIDumpProcessIN->ui32Pid);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntryWithOwner(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerIN_UI8,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryWithOwnerIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryWithOwnerOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntryWithOwner_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    RIWritePMREntryWithOwnerKM(psPMRHandleInt, psRIWritePMREntryWithOwnerIN->ui32Owner);
+
+RIWritePMREntryWithOwner_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+
+/*
+ * Register all RI functions with services
+ */
+PVRSRV_ERROR InitRIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY,
+			      PVRSRVBridgeRIWritePMREntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY,
+			      PVRSRVBridgeRIWriteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY,
+			      PVRSRVBridgeRIWriteProcListEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR,
+			      PVRSRVBridgeRIUpdateMEMDESCAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY,
+			      PVRSRVBridgeRIDeleteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST, PVRSRVBridgeRIDumpList,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPLIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPLIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL, PVRSRVBridgeRIDumpAll,
+			      NULL, 0, sizeof(PVRSRV_BRIDGE_OUT_RIDUMPALL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS,
+			      PVRSRVBridgeRIDumpProcess, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPPROCESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPPROCESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER,
+			      PVRSRVBridgeRIWritePMREntryWithOwner, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all ri functions with services
+ */
+void DeinitRIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/common_smm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/common_smm_bridge.h
new file mode 100644
index 000000000000..7d6603dfbe8b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/common_smm_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SMM_BRIDGE_H
+#define COMMON_SMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SMM_CMD_LAST			(PVRSRV_BRIDGE_SMM_CMD_FIRST+2)
+
+/*******************************************
+            PMRSecureExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR;
+
+/* Bridge out structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR;
+
+/*******************************************
+            PMRSecureUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR;
+
+/* Bridge out structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR;
+
+/*******************************************
+            PMRSecureImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR;
+
+/* Bridge out structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR;
+
+#endif /* COMMON_SMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/server_smm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/server_smm_bridge.c
new file mode 100644
index 000000000000..5641b1ea12f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/server_smm_bridge.c
@@ -0,0 +1,281 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "secure_export.h"
+
+#include "common_smm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRSecureExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureExportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureExportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *psPMRSecureExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *psPMRSecureExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMROUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRSecureExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR *psPMROutInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRSecureExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRSecureExportPMROUT->eError =
+	    PMRSecureExportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRInt,
+			       &psPMRSecureExportPMROUT->Export, &psPMROutInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureExportPMR_exit;
+	}
+
+PMRSecureExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRSecureExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMROutInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRSecureUnexportPMR(psPMROutInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRSecureUnexportPMRIN_UI8,
+				 IMG_UINT8 * psPMRSecureUnexportPMROUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureUnexportPMRIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *)
+	    IMG_OFFSET_ADDR(psPMRSecureUnexportPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRSecureUnexportPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT);
+	if (unlikely((psPMRSecureUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRSecureUnexportPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureUnexportPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRSecureImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureImportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureImportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *psPMRSecureImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *psPMRSecureImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMROUT_UI8,
+								     0);
+
+	PMR *psPMRInt = NULL;
+
+	psPMRSecureImportPMROUT->eError =
+	    PMRSecureImportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRSecureImportPMRIN->Export,
+			       &psPMRInt,
+			       &psPMRSecureImportPMROUT->uiSize, &psPMRSecureImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psPMRSecureImportPMROUT->hPMR,
+								    (void *)psPMRInt,
+								    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _PMRSecureImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureImportPMR_exit:
+
+	if (psPMRSecureImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+
+/*
+ * Register all SMM functions with services
+ */
+PVRSRV_ERROR InitSMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR,
+			      PVRSRVBridgePMRSecureExportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR,
+			      PVRSRVBridgePMRSecureUnexportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR,
+			      PVRSRVBridgePMRSecureImportPMR, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all smm functions with services
+ */
+void DeinitSMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/common_srvcore_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/common_srvcore_bridge.h
new file mode 100644
index 000000000000..603ba5b1d89e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/common_srvcore_bridge.h
@@ -0,0 +1,369 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SRVCORE_BRIDGE_H
+#define COMMON_SRVCORE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_device_types.h"
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_SRVCORE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SRVCORE_CONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SRVCORE_DISCONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16
+#define PVRSRV_BRIDGE_SRVCORE_CMD_LAST			(PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16)
+
+/*******************************************
+            Connect
+ *******************************************/
+
+/* Bridge in structure for Connect */
+typedef struct PVRSRV_BRIDGE_IN_CONNECT_TAG
+{
+	IMG_UINT32 ui32ClientBuildOptions;
+	IMG_UINT32 ui32ClientDDKBuild;
+	IMG_UINT32 ui32ClientDDKVersion;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_CONNECT;
+
+/* Bridge out structure for Connect */
+typedef struct PVRSRV_BRIDGE_OUT_CONNECT_TAG
+{
+	IMG_UINT64 ui64PackedBvnc;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CapabilityFlags;
+	IMG_UINT8 ui8KernelArch;
+} __packed PVRSRV_BRIDGE_OUT_CONNECT;
+
+/*******************************************
+            Disconnect
+ *******************************************/
+
+/* Bridge in structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_IN_DISCONNECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DISCONNECT;
+
+/* Bridge out structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_OUT_DISCONNECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DISCONNECT;
+
+/*******************************************
+            AcquireGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT;
+
+/* Bridge out structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT;
+
+/*******************************************
+            ReleaseGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+} __packed PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT;
+
+/* Bridge out structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT;
+
+/*******************************************
+            EventObjectOpen
+ *******************************************/
+
+/* Bridge in structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hEventObject;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN;
+
+/* Bridge out structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN;
+
+/*******************************************
+            EventObjectWait
+ *******************************************/
+
+/* Bridge in structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT;
+
+/* Bridge out structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT;
+
+/*******************************************
+            EventObjectClose
+ *******************************************/
+
+/* Bridge in structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE;
+
+/* Bridge out structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE;
+
+/*******************************************
+            DumpDebugInfo
+ *******************************************/
+
+/* Bridge in structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_IN_DUMPDEBUGINFO_TAG
+{
+	IMG_UINT32 ui32VerbLevel;
+} __packed PVRSRV_BRIDGE_IN_DUMPDEBUGINFO;
+
+/* Bridge out structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO;
+
+/*******************************************
+            GetDevClockSpeed
+ *******************************************/
+
+/* Bridge in structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED;
+
+/* Bridge out structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ClockSpeed;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED;
+
+/*******************************************
+            HWOpTimeout
+ *******************************************/
+
+/* Bridge in structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_IN_HWOPTIMEOUT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HWOPTIMEOUT;
+
+/* Bridge out structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_HWOPTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HWOPTIMEOUT;
+
+/*******************************************
+            AlignmentCheck
+ *******************************************/
+
+/* Bridge in structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_IN_ALIGNMENTCHECK_TAG
+{
+	IMG_UINT32 *pui32AlignChecks;
+	IMG_UINT32 ui32AlignChecksSize;
+} __packed PVRSRV_BRIDGE_IN_ALIGNMENTCHECK;
+
+/* Bridge out structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK;
+
+/*******************************************
+            GetDeviceStatus
+ *******************************************/
+
+/* Bridge in structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVICESTATUS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVICESTATUS;
+
+/* Bridge out structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVICESTATUS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceSatus;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVICESTATUS;
+
+/*******************************************
+            GetMultiCoreInfo
+ *******************************************/
+
+/* Bridge in structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_IN_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	IMG_UINT32 ui32CapsSize;
+} __packed PVRSRV_BRIDGE_IN_GETMULTICOREINFO;
+
+/* Bridge out structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_OUT_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumCores;
+} __packed PVRSRV_BRIDGE_OUT_GETMULTICOREINFO;
+
+/*******************************************
+            EventObjectWaitTimeout
+ *******************************************/
+
+/* Bridge in structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	IMG_UINT64 ui64uiTimeoutus;
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT;
+
+/* Bridge out structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT;
+
+/*******************************************
+            FindProcessMemStats
+ *******************************************/
+
+/* Bridge in structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	IMG_UINT32 ui32ArrSize;
+	IMG_UINT32 ui32PID;
+	IMG_BOOL bbAllProcessStats;
+} __packed PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS;
+
+/* Bridge out structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS;
+
+/*******************************************
+            AcquireInfoPage
+ *******************************************/
+
+/* Bridge in structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE;
+
+/* Bridge out structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE;
+
+/*******************************************
+            ReleaseInfoPage
+ *******************************************/
+
+/* Bridge in structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_RELEASEINFOPAGE;
+
+/* Bridge out structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE;
+
+#endif /* COMMON_SRVCORE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/server_srvcore_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/server_srvcore_bridge.c
new file mode 100644
index 000000000000..d40e53b244f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/server_srvcore_bridge.c
@@ -0,0 +1,1080 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "srvcore.h"
+#include "info_page.h"
+#include "proc_stats.h"
+#include "rgx_fwif_alignchecks.h"
+
+#include "common_srvcore_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeConnect(IMG_UINT32 ui32DispatchTableEntry,
+		    IMG_UINT8 * psConnectIN_UI8,
+		    IMG_UINT8 * psConnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CONNECT *psConnectIN =
+	    (PVRSRV_BRIDGE_IN_CONNECT *) IMG_OFFSET_ADDR(psConnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CONNECT *psConnectOUT =
+	    (PVRSRV_BRIDGE_OUT_CONNECT *) IMG_OFFSET_ADDR(psConnectOUT_UI8, 0);
+
+	psConnectOUT->eError =
+	    PVRSRVConnectKM(psConnection, OSGetDevNode(psConnection),
+			    psConnectIN->ui32Flags,
+			    psConnectIN->ui32ClientBuildOptions,
+			    psConnectIN->ui32ClientDDKVersion,
+			    psConnectIN->ui32ClientDDKBuild,
+			    &psConnectOUT->ui8KernelArch,
+			    &psConnectOUT->ui32CapabilityFlags, &psConnectOUT->ui64PackedBvnc);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDisconnect(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDisconnectIN_UI8,
+		       IMG_UINT8 * psDisconnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DISCONNECT *psDisconnectIN =
+	    (PVRSRV_BRIDGE_IN_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DISCONNECT *psDisconnectOUT =
+	    (PVRSRV_BRIDGE_OUT_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDisconnectIN);
+
+	psDisconnectOUT->eError = PVRSRVDisconnectKM();
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireGlobalEventObjecthGlobalEventObjectIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseGlobalEventObjectKM((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psAcquireGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psAcquireGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectOUT_UI8, 0);
+
+	IMG_HANDLE hGlobalEventObjectInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireGlobalEventObjectIN);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAcquireGlobalEventObjectKM(&hGlobalEventObjectInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psAcquireGlobalEventObjectOUT->hGlobalEventObject,
+				      (void *)hGlobalEventObjectInt,
+				      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _AcquireGlobalEventObjecthGlobalEventObjectIntRelease);
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AcquireGlobalEventObject_exit:
+
+	if (psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK)
+	{
+		if (hGlobalEventObjectInt)
+		{
+			PVRSRVReleaseGlobalEventObjectKM(hGlobalEventObjectInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psReleaseGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psReleaseGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psReleaseGlobalEventObjectOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psReleaseGlobalEventObjectIN->
+					      hGlobalEventObject,
+					      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	if (unlikely
+	    ((psReleaseGlobalEventObjectOUT->eError != PVRSRV_OK)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psReleaseGlobalEventObjectOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto ReleaseGlobalEventObject_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+ReleaseGlobalEventObject_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _EventObjectOpenhOSEventIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = OSEventObjectClose((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectOpen(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectOpenIN_UI8,
+			    IMG_UINT8 * psEventObjectOpenOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *psEventObjectOpenIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *psEventObjectOpenOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenOUT_UI8, 0);
+
+	IMG_HANDLE hEventObject = psEventObjectOpenIN->hEventObject;
+	IMG_HANDLE hEventObjectInt = NULL;
+	IMG_HANDLE hOSEventInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectOpenOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hEventObjectInt,
+				       hEventObject,
+				       PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT, IMG_TRUE);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = OSEventObjectOpen(hEventObjectInt, &hOSEventInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		goto EventObjectOpen_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psEventObjectOpenOUT->hOSEvent,
+								 (void *)hOSEventInt,
+								 PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _EventObjectOpenhOSEventIntRelease);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectOpen_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hEventObjectInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hEventObject, PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psEventObjectOpenOUT->eError != PVRSRV_OK)
+	{
+		if (hOSEventInt)
+		{
+			OSEventObjectClose(hOSEventInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectWaitIN_UI8,
+			    IMG_UINT8 * psEventObjectWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *psEventObjectWaitIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *psEventObjectWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitOUT->eError = OSEventObjectWait(hOSEventKMInt);
+
+EventObjectWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectClose(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psEventObjectCloseIN_UI8,
+			     IMG_UINT8 * psEventObjectCloseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *psEventObjectCloseIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *psEventObjectCloseOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectCloseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psEventObjectCloseIN->hOSEventKM,
+					      PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	if (unlikely((psEventObjectCloseOUT->eError != PVRSRV_OK) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psEventObjectCloseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectClose_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectClose_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDumpDebugInfo(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDumpDebugInfoIN_UI8,
+			  IMG_UINT8 * psDumpDebugInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *psDumpDebugInfoIN =
+	    (PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *psDumpDebugInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoOUT_UI8, 0);
+
+	psDumpDebugInfoOUT->eError =
+	    PVRSRVDumpDebugInfoKM(psConnection, OSGetDevNode(psConnection),
+				  psDumpDebugInfoIN->ui32VerbLevel);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDevClockSpeed(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetDevClockSpeedIN_UI8,
+			     IMG_UINT8 * psGetDevClockSpeedOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *psGetDevClockSpeedIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *psGetDevClockSpeedOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDevClockSpeedIN);
+
+	psGetDevClockSpeedOUT->eError =
+	    PVRSRVGetDevClockSpeedKM(psConnection, OSGetDevNode(psConnection),
+				     &psGetDevClockSpeedOUT->ui32ClockSpeed);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHWOpTimeout(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psHWOpTimeoutIN_UI8,
+			IMG_UINT8 * psHWOpTimeoutOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HWOPTIMEOUT *psHWOpTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *psHWOpTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHWOpTimeoutIN);
+
+	psHWOpTimeoutOUT->eError = PVRSRVHWOpTimeoutKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static_assert(RGXFW_ALIGN_CHECKS_UM_MAX <= IMG_UINT32_MAX,
+	      "RGXFW_ALIGN_CHECKS_UM_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeAlignmentCheck(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psAlignmentCheckIN_UI8,
+			   IMG_UINT8 * psAlignmentCheckOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *psAlignmentCheckIN =
+	    (PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *psAlignmentCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AlignChecksInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psAlignmentCheckIN->ui32AlignChecksSize > RGXFW_ALIGN_CHECKS_UM_MAX))
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto AlignmentCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto AlignmentCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psAlignmentCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psAlignmentCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psAlignmentCheckOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto AlignmentCheck_exit;
+			}
+		}
+	}
+
+	if (psAlignmentCheckIN->ui32AlignChecksSize != 0)
+	{
+		ui32AlignChecksInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AlignChecksInt,
+		     (const void __user *)psAlignmentCheckIN->pui32AlignChecks,
+		     psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psAlignmentCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto AlignmentCheck_exit;
+		}
+	}
+
+	psAlignmentCheckOUT->eError =
+	    PVRSRVAlignmentCheckKM(psConnection, OSGetDevNode(psConnection),
+				   psAlignmentCheckIN->ui32AlignChecksSize, ui32AlignChecksInt);
+
+AlignmentCheck_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psAlignmentCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDeviceStatus(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psGetDeviceStatusIN_UI8,
+			    IMG_UINT8 * psGetDeviceStatusOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVICESTATUS *psGetDeviceStatusIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *psGetDeviceStatusOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDeviceStatusIN);
+
+	psGetDeviceStatusOUT->eError =
+	    PVRSRVGetDeviceStatusKM(psConnection, OSGetDevNode(psConnection),
+				    &psGetDeviceStatusOUT->ui32DeviceSatus);
+
+	return 0;
+}
+
+static_assert(8 <= IMG_UINT32_MAX, "8 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeGetMultiCoreInfo(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetMultiCoreInfoIN_UI8,
+			     IMG_UINT8 * psGetMultiCoreInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETMULTICOREINFO *psGetMultiCoreInfoIN =
+	    (PVRSRV_BRIDGE_IN_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *psGetMultiCoreInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoOUT_UI8, 0);
+
+	IMG_UINT64 *pui64CapsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize > 8)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	psGetMultiCoreInfoOUT->pui64Caps = psGetMultiCoreInfoIN->pui64Caps;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psGetMultiCoreInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psGetMultiCoreInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto GetMultiCoreInfo_exit;
+			}
+		}
+	}
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize != 0)
+	{
+		pui64CapsInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64);
+	}
+
+	psGetMultiCoreInfoOUT->eError =
+	    PVRSRVGetMultiCoreInfoKM(psConnection, OSGetDevNode(psConnection),
+				     psGetMultiCoreInfoIN->ui32CapsSize,
+				     &psGetMultiCoreInfoOUT->ui32NumCores, pui64CapsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psGetMultiCoreInfoOUT->eError != PVRSRV_OK))
+	{
+		goto GetMultiCoreInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64CapsInt) && ((psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psGetMultiCoreInfoOUT->pui64Caps, pui64CapsInt,
+		      (psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto GetMultiCoreInfo_exit;
+		}
+	}
+
+GetMultiCoreInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psGetMultiCoreInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWaitTimeout(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psEventObjectWaitTimeoutIN_UI8,
+				   IMG_UINT8 * psEventObjectWaitTimeoutOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitTimeoutIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitTimeoutOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitTimeoutOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWaitTimeout_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitTimeoutOUT->eError =
+	    OSEventObjectWaitTimeout(hOSEventKMInt, psEventObjectWaitTimeoutIN->ui64uiTimeoutus);
+
+EventObjectWaitTimeout_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PROCESS_STAT_TYPE_COUNT <= IMG_UINT32_MAX,
+	      "PVRSRV_PROCESS_STAT_TYPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeFindProcessMemStats(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psFindProcessMemStatsIN_UI8,
+				IMG_UINT8 * psFindProcessMemStatsOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *psFindProcessMemStatsIN =
+	    (PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *psFindProcessMemStatsOUT =
+	    (PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsOUT_UI8,
+								      0);
+
+	IMG_UINT64 *pui64MemStatsArrayInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psFindProcessMemStatsIN->ui32ArrSize > PVRSRV_PROCESS_STAT_TYPE_COUNT)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto FindProcessMemStats_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psFindProcessMemStatsOUT->pui64MemStatsArray = psFindProcessMemStatsIN->pui64MemStatsArray;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto FindProcessMemStats_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psFindProcessMemStatsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psFindProcessMemStatsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto FindProcessMemStats_exit;
+			}
+		}
+	}
+
+	if (psFindProcessMemStatsIN->ui32ArrSize != 0)
+	{
+		pui64MemStatsArrayInt =
+		    (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64);
+	}
+
+	psFindProcessMemStatsOUT->eError =
+	    PVRSRVFindProcessMemStatsKM(psFindProcessMemStatsIN->ui32PID,
+					psFindProcessMemStatsIN->ui32ArrSize,
+					psFindProcessMemStatsIN->bbAllProcessStats,
+					pui64MemStatsArrayInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psFindProcessMemStatsOUT->eError != PVRSRV_OK))
+	{
+		goto FindProcessMemStats_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64MemStatsArrayInt) &&
+	    ((psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psFindProcessMemStatsOUT->pui64MemStatsArray,
+		      pui64MemStatsArrayInt,
+		      (psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto FindProcessMemStats_exit;
+		}
+	}
+
+FindProcessMemStats_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psFindProcessMemStatsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireInfoPagepsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseInfoPageKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psAcquireInfoPageIN_UI8,
+			    IMG_UINT8 * psAcquireInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *psAcquireInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *psAcquireInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageOUT_UI8, 0);
+
+	PMR *psPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireInfoPageIN);
+
+	psAcquireInfoPageOUT->eError = PVRSRVAcquireInfoPageKM(&psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psAcquireInfoPageOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psAcquireInfoPageOUT->hPMR, (void *)psPMRInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _AcquireInfoPagepsPMRIntRelease);
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+AcquireInfoPage_exit:
+
+	if (psAcquireInfoPageOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			PVRSRVReleaseInfoPageKM(psPMRInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psReleaseInfoPageIN_UI8,
+			    IMG_UINT8 * psReleaseInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *psReleaseInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *psReleaseInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psReleaseInfoPageOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psReleaseInfoPageIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psReleaseInfoPageOUT->eError != PVRSRV_OK) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psReleaseInfoPageOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto ReleaseInfoPage_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+ReleaseInfoPage_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+
+/*
+ * Register all SRVCORE functions with services
+ */
+PVRSRV_ERROR InitSRVCOREBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT,
+			      PVRSRVBridgeConnect, NULL, sizeof(PVRSRV_BRIDGE_IN_CONNECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_CONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT,
+			      PVRSRVBridgeDisconnect, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DISCONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT,
+			      PVRSRVBridgeAcquireGlobalEventObject, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT,
+			      PVRSRVBridgeReleaseGlobalEventObject, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN,
+			      PVRSRVBridgeEventObjectOpen, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT,
+			      PVRSRVBridgeEventObjectWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE,
+			      PVRSRVBridgeEventObjectClose, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO,
+			      PVRSRVBridgeDumpDebugInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DUMPDEBUGINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED,
+			      PVRSRVBridgeGetDevClockSpeed, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT,
+			      PVRSRVBridgeHWOpTimeout, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HWOPTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK,
+			      PVRSRVBridgeAlignmentCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_ALIGNMENTCHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS,
+			      PVRSRVBridgeGetDeviceStatus, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVICESTATUS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO,
+			      PVRSRVBridgeGetMultiCoreInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_GETMULTICOREINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_GETMULTICOREINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT,
+			      PVRSRVBridgeEventObjectWaitTimeout, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS,
+			      PVRSRVBridgeFindProcessMemStats, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS),
+			      sizeof(PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE,
+			      PVRSRVBridgeAcquireInfoPage, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE,
+			      PVRSRVBridgeReleaseInfoPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEINFOPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all srvcore functions with services
+ */
+void DeinitSRVCOREBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_bridge.h
new file mode 100644
index 000000000000..19f1b0ece5b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_bridge.h
@@ -0,0 +1,102 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNC_BRIDGE_H
+#define CLIENT_SYNC_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_sync_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence);
+
+#endif /* CLIENT_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_direct_bridge.c
new file mode 100644
index 000000000000..d631aea421ad
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_direct_bridge.c
@@ -0,0 +1,262 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for sync
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_sync_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    &psSyncHandleInt,
+					    pui32SyncPrimVAddr,
+					    pui32SyncPrimBlockSize, &pshSyncPMRInt);
+
+	*phSyncHandle = psSyncHandleInt;
+	*phhSyncPMR = pshSyncPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimSetKM(psSyncHandleInt, ui32Index, ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpKM(psSyncHandleInt, ui32Offset);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt, ui32Offset, ui32Value);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     ui32Offset, ui32Value, ui32Mask, eOperator, uiPDumpFlags);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     ui32Offset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVSyncAllocEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   bServerSync, ui32FWAddr, ui32ClassNameSize, puiClassName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVSyncFreeEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32FWAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PVRSRVSyncCheckpointSignalledPDumpPolKM(hFence);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hFence);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/common_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/common_sync_bridge.h
new file mode 100644
index 000000000000..753d40b64f12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/common_sync_bridge.h
@@ -0,0 +1,254 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNC_BRIDGE_H
+#define COMMON_SYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#define PVRSRV_BRIDGE_SYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMSET			PVRSRV_BRIDGE_SYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE			PVRSRV_BRIDGE_SYNC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNC_CMD_LAST			(PVRSRV_BRIDGE_SYNC_CMD_FIRST+9)
+
+/*******************************************
+            AllocSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_HANDLE hhSyncPMR;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32SyncPrimBlockSize;
+	IMG_UINT32 ui32SyncPrimVAddr;
+} __packed PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            FreeSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+} __packed PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            SyncPrimSet
+ *******************************************/
+
+/* Bridge in structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMSET_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMSET;
+
+/* Bridge out structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMSET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMSET;
+
+/*******************************************
+            SyncPrimPDump
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP;
+
+/* Bridge out structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP;
+
+/*******************************************
+            SyncPrimPDumpValue
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE;
+
+/* Bridge out structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE;
+
+/*******************************************
+            SyncPrimPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+	PDUMP_FLAGS_T uiPDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL;
+
+/* Bridge out structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL;
+
+/*******************************************
+            SyncPrimPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP;
+
+/* Bridge out structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP;
+
+/*******************************************
+            SyncAllocEvent
+ *******************************************/
+
+/* Bridge in structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCALLOCEVENT_TAG
+{
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32FWAddr;
+	IMG_BOOL bServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCALLOCEVENT;
+
+/* Bridge out structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT;
+
+/*******************************************
+            SyncFreeEvent
+ *******************************************/
+
+/* Bridge in structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFREEEVENT_TAG
+{
+	IMG_UINT32 ui32FWAddr;
+} __packed PVRSRV_BRIDGE_IN_SYNCFREEEVENT;
+
+/* Bridge out structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFREEEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFREEEVENT;
+
+/*******************************************
+            SyncCheckpointSignalledPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_FENCE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+/* Bridge out structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+#endif /* COMMON_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/server_sync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/server_sync_bridge.c
new file mode 100644
index 000000000000..a0d9210086c9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/server_sync_bridge.c
@@ -0,0 +1,755 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+#include "common_sync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _AllocSyncPrimitiveBlockpsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVFreeSyncPrimitiveBlockKM((SYNC_PRIMITIVE_BLOCK *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAllocSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockIN_UI8,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockOUT_UI8, 0);
+
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAllocSyncPrimitiveBlockIN);
+
+	psAllocSyncPrimitiveBlockOUT->hSyncHandle = NULL;
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(psConnection, OSGetDevNode(psConnection),
+					    &psSyncHandleInt,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimVAddr,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimBlockSize,
+					    &pshSyncPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAllocSyncPrimitiveBlockOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psAllocSyncPrimitiveBlockOUT->
+									 hSyncHandle,
+									 (void *)psSyncHandleInt,
+									 PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _AllocSyncPrimitiveBlockpsSyncHandleIntRelease);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psAllocSyncPrimitiveBlockOUT->hhSyncPMR,
+					 (void *)pshSyncPMRInt,
+					 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psAllocSyncPrimitiveBlockOUT->hSyncHandle);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AllocSyncPrimitiveBlock_exit:
+
+	if (psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK)
+	{
+		if (psAllocSyncPrimitiveBlockOUT->hSyncHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psAllocSyncPrimitiveBlockOUT->
+							     hSyncHandle,
+							     PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSyncHandleInt)
+		{
+			PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeFreeSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockIN_UI8,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psFreeSyncPrimitiveBlockOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psFreeSyncPrimitiveBlockIN->hSyncHandle,
+					      PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	if (unlikely((psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_OK) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psFreeSyncPrimitiveBlockOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto FreeSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+FreeSyncPrimitiveBlock_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncPrimSet(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psSyncPrimSetIN_UI8,
+			IMG_UINT8 * psSyncPrimSetOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMSET *psSyncPrimSetIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMSET *psSyncPrimSetOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimSetIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimSetOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimSetOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimSet_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimSetOUT->eError =
+	    PVRSRVSyncPrimSetKM(psSyncHandleInt,
+				psSyncPrimSetIN->ui32Index, psSyncPrimSetIN->ui32Value);
+
+SyncPrimSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDump(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncPrimPDumpIN_UI8,
+			  IMG_UINT8 * psSyncPrimPDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *psSyncPrimPDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *psSyncPrimPDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVSyncPrimPDumpKM(psSyncHandleInt, psSyncPrimPDumpIN->ui32Offset);
+
+SyncPrimPDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDump NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpValue(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncPrimPDumpValueIN_UI8,
+			       IMG_UINT8 * psSyncPrimPDumpValueOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueOUT_UI8,
+								     0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpValueIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpValueOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpValue_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt,
+				       psSyncPrimPDumpValueIN->ui32Offset,
+				       psSyncPrimPDumpValueIN->ui32Value);
+
+SyncPrimPDumpValue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpValue NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpPolIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpPolOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpPolIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpPolOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpPol_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     psSyncPrimPDumpPolIN->ui32Offset,
+				     psSyncPrimPDumpPolIN->ui32Value,
+				     psSyncPrimPDumpPolIN->ui32Mask,
+				     psSyncPrimPDumpPolIN->eOperator,
+				     psSyncPrimPDumpPolIN->uiPDumpFlags);
+
+SyncPrimPDumpPol_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpPol NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpCBPIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpCBPIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     psSyncPrimPDumpCBPIN->ui32Offset,
+				     psSyncPrimPDumpCBPIN->uiWriteOffset,
+				     psSyncPrimPDumpCBPIN->uiPacketSize,
+				     psSyncPrimPDumpCBPIN->uiBufferSize);
+
+SyncPrimPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpCBP NULL
+#endif
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncAllocEvent(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncAllocEventIN_UI8,
+			   IMG_UINT8 * psSyncAllocEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *psSyncAllocEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *psSyncAllocEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventOUT_UI8, 0);
+
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncAllocEventIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncAllocEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncAllocEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncAllocEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncAllocEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncAllocEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncAllocEvent_exit;
+			}
+		}
+	}
+
+	if (psSyncAllocEventIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncAllocEventIN->puiClassName,
+		     psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncAllocEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncAllocEvent_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psSyncAllocEventOUT->eError =
+	    PVRSRVSyncAllocEventKM(psConnection, OSGetDevNode(psConnection),
+				   psSyncAllocEventIN->bServerSync,
+				   psSyncAllocEventIN->ui32FWAddr,
+				   psSyncAllocEventIN->ui32ClassNameSize, uiClassNameInt);
+
+SyncAllocEvent_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncAllocEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFreeEvent(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncFreeEventIN_UI8,
+			  IMG_UINT8 * psSyncFreeEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFREEEVENT *psSyncFreeEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *psSyncFreeEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventOUT_UI8, 0);
+
+	psSyncFreeEventOUT->eError =
+	    PVRSRVSyncFreeEventKM(psConnection, OSGetDevNode(psConnection),
+				  psSyncFreeEventIN->ui32FWAddr);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncCheckpointSignalledPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolIN_UI8,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psSyncCheckpointSignalledPDumpPolOUT->eError =
+	    PVRSRVSyncCheckpointSignalledPDumpPolKM(psSyncCheckpointSignalledPDumpPolIN->hFence);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncCheckpointSignalledPDumpPol NULL
+#endif
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+
+/*
+ * Register all SYNC functions with services
+ */
+PVRSRV_ERROR InitSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeAllocSyncPrimitiveBlock, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeFreeSyncPrimitiveBlock, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK),
+			      sizeof(PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET,
+			      PVRSRVBridgeSyncPrimSet, NULL, sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMSET),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMSET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP,
+			      PVRSRVBridgeSyncPrimPDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE,
+			      PVRSRVBridgeSyncPrimPDumpValue, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL,
+			      PVRSRVBridgeSyncPrimPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP,
+			      PVRSRVBridgeSyncPrimPDumpCBP, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT,
+			      PVRSRVBridgeSyncAllocEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCALLOCEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT,
+			      PVRSRVBridgeSyncFreeEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFREEEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFREEEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+			      PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL,
+			      PVRSRVBridgeSyncCheckpointSignalledPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all sync functions with services
+ */
+void DeinitSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+				PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/common_syncfallback_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/common_syncfallback_bridge.h
new file mode 100644
index 000000000000..87016c4706c7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/common_syncfallback_bridge.h
@@ -0,0 +1,357 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCFALLBACK_BRIDGE_H
+#define COMMON_SYNCFALLBACK_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST			(PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15)
+
+/*******************************************
+            SyncFbTimelineCreatePVR
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR;
+
+/* Bridge out structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR;
+
+/*******************************************
+            SyncFbTimelineRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE;
+
+/* Bridge out structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE;
+
+/*******************************************
+            SyncFbFenceDup
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hInFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP;
+
+/* Bridge out structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP;
+
+/*******************************************
+            SyncFbFenceMerge
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hInFence1;
+	IMG_HANDLE hInFence2;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE;
+
+/* Bridge out structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE;
+
+/*******************************************
+            SyncFbFenceRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE;
+
+/* Bridge out structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE;
+
+/*******************************************
+            SyncFbFenceWait
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT_TAG
+{
+	IMG_HANDLE hFence;
+	IMG_UINT32 ui32Timeout;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT;
+
+/* Bridge out structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT;
+
+/*******************************************
+            SyncFbFenceDump
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP_TAG
+{
+	IMG_HANDLE hFence;
+	const IMG_CHAR *puiDescStr;
+	const IMG_CHAR *puiFileStr;
+	const IMG_CHAR *puiModuleStr;
+	IMG_UINT32 ui32DescStrLength;
+	IMG_UINT32 ui32FileStrLength;
+	IMG_UINT32 ui32Line;
+	IMG_UINT32 ui32ModuleStrLength;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP;
+
+/* Bridge out structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP;
+
+/*******************************************
+            SyncFbTimelineCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW;
+
+/* Bridge out structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW;
+
+/*******************************************
+            SyncFbFenceCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW;
+
+/* Bridge out structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW;
+
+/*******************************************
+            SyncFbTimelineAdvanceSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW;
+
+/* Bridge out structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW;
+
+/*******************************************
+            SyncFbFenceExportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroyInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/*******************************************
+            SyncFbFenceImportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hImport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroySecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/*******************************************
+            SyncFbFenceImportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Import;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE;
+
+#endif /* COMMON_SYNCFALLBACK_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/server_syncfallback_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/server_syncfallback_bridge.c
new file mode 100644
index 000000000000..68e650d89602
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/server_syncfallback_bridge.c
@@ -0,0 +1,1958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync_fallback_server.h"
+#include "pvrsrv_sync_server.h"
+
+#include "common_syncfallback_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleaseExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _SyncFbTimelineCreatePVRpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_TIMELINE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_TIMELINE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreatePVR(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVRIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVROUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVRIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVROUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVROUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize > SYNC_FB_TIMELINE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreatePVRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreatePVRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreatePVR_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreatePVRIN->puiTimelineName,
+		     psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreatePVR_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    SyncFbTimelineCreatePVR(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize,
+				    uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreatePVROUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreatePVRpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreatePVR_exit:
+
+	if (psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreatePVROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psSyncFbTimelineReleaseIN_UI8,
+				  IMG_UINT8 * psSyncFbTimelineReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbTimelineReleaseIN->hTimeline,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	if (unlikely((psSyncFbTimelineReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbTimelineReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceDuppsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDup(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncFbFenceDupIN_UI8,
+			   IMG_UINT8 * psSyncFbFenceDupOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *psSyncFbFenceDupIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *psSyncFbFenceDupOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupOUT_UI8, 0);
+
+	IMG_HANDLE hInFence = psSyncFbFenceDupIN->hInFence;
+	PVRSRV_FENCE_SERVER *psInFenceInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFenceInt,
+				       hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError = SyncFbFenceDup(psInFenceInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceDupOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceDuppsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceDup_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceDupOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceMergepsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceMerge(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncFbFenceMergeIN_UI8,
+			     IMG_UINT8 * psSyncFbFenceMergeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *psSyncFbFenceMergeIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *psSyncFbFenceMergeOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeOUT_UI8, 0);
+
+	IMG_HANDLE hInFence1 = psSyncFbFenceMergeIN->hInFence1;
+	PVRSRV_FENCE_SERVER *psInFence1Int = NULL;
+	IMG_HANDLE hInFence2 = psSyncFbFenceMergeIN->hInFence2;
+	PVRSRV_FENCE_SERVER *psInFence2Int = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceMergeIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceMergeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceMergeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceMerge_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt, (const void __user *)psSyncFbFenceMergeIN->puiFenceName,
+		     psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceMerge_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence1Int,
+				       hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence2Int,
+				       hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    SyncFbFenceMerge(psInFence1Int,
+			     psInFence2Int,
+			     psSyncFbFenceMergeIN->ui32FenceNameSize,
+			     uiFenceNameInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceMergeOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceMergepsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceMerge_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFence1Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psInFence2Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceMergeOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceMergeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncFbFenceReleaseIN_UI8,
+			       IMG_UINT8 * psSyncFbFenceReleaseOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *psSyncFbFenceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *psSyncFbFenceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceReleaseIN->hFence,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	if (unlikely((psSyncFbFenceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbFenceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceWaitIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *psSyncFbFenceWaitIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *psSyncFbFenceWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceWaitIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceWaitOUT->eError =
+	    SyncFbFenceWait(psFenceInt, psSyncFbFenceWaitIN->ui32Timeout);
+
+SyncFbFenceWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+static_assert((SYNC_FB_FILE_STRING_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_FILE_STRING_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_MODULE_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_MODULE_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_DESC_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_DESC_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDump(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceDumpIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *psSyncFbFenceDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *psSyncFbFenceDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceDumpIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	IMG_CHAR *uiFileStrInt = NULL;
+	IMG_CHAR *uiModuleStrInt = NULL;
+	IMG_CHAR *uiDescStrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32FileStrLength > (SYNC_FB_FILE_STRING_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely
+	    (psSyncFbFenceDumpIN->ui32ModuleStrLength > (SYNC_FB_MODULE_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32DescStrLength > (SYNC_FB_DESC_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceDump_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceDumpIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceDumpIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceDump_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceDumpIN->ui32FileStrLength != 0)
+	{
+		uiFileStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileStrInt, (const void __user *)psSyncFbFenceDumpIN->puiFileStr,
+		     psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileStrInt)[(psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength != 0)
+	{
+		uiModuleStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiModuleStrInt, (const void __user *)psSyncFbFenceDumpIN->puiModuleStr,
+		     psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiModuleStrInt)[(psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32DescStrLength != 0)
+	{
+		uiDescStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiDescStrInt, (const void __user *)psSyncFbFenceDumpIN->puiDescStr,
+		     psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiDescStrInt)[(psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDumpOUT->eError =
+	    SyncFbFenceDump(psFenceInt,
+			    psSyncFbFenceDumpIN->ui32Line,
+			    psSyncFbFenceDumpIN->ui32FileStrLength,
+			    uiFileStrInt,
+			    psSyncFbFenceDumpIN->ui32ModuleStrLength,
+			    uiModuleStrInt, psSyncFbFenceDumpIN->ui32DescStrLength, uiDescStrInt);
+
+SyncFbFenceDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceDumpOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbTimelineCreateSWpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWIN_UI8,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWOUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreateSWIN->puiTimelineName,
+		     psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    SyncFbTimelineCreateSW(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize,
+				   uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreateSWOUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreateSWpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreateSW_exit:
+
+	if (psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceCreateSWpsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psSyncFbFenceCreateSWIN_UI8,
+				IMG_UINT8 * psSyncFbFenceCreateSWOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWOUT_UI8,
+								      0);
+
+	IMG_HANDLE hTimeline = psSyncFbFenceCreateSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceCreateSWIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt,
+		     (const void __user *)psSyncFbFenceCreateSWIN->puiFenceName,
+		     psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    SyncFbFenceCreateSW(psConnection, OSGetDevNode(psConnection),
+				psTimelineInt,
+				psSyncFbFenceCreateSWIN->ui32FenceNameSize,
+				uiFenceNameInt,
+				&psOutFenceInt, &psSyncFbFenceCreateSWOUT->ui64SyncPtIdx);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceCreateSWOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceCreateSWpsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceCreateSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineAdvanceSW(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWOUT_UI8, 0);
+
+	IMG_HANDLE hTimeline = psSyncFbTimelineAdvanceSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbTimelineAdvanceSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineAdvanceSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    SyncFbTimelineAdvanceSW(psTimelineInt, &psSyncFbTimelineAdvanceSWOUT->ui64SyncPtIdx);
+
+SyncFbTimelineAdvanceSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceExportInsecurepsExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceExportDestroyInsecure((PVRSRV_FENCE_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportInsecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    SyncFbFenceExportInsecure(psFenceInt, &psExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase, &hExportInt,
+				      (void *)psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceExportInsecurepsExportIntRelease);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportInsecureOUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+									   &psSyncFbFenceExportInsecureOUT->
+									   hExport,
+									   (void *)psExportInt,
+									   PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+									   PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+									   (PFN_HANDLE_RELEASE) &
+									   ReleaseExport);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncFbFenceExportInsecureOUT->hExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE)
+							     psSyncFbFenceExportInsecureOUT->
+							     hExport,
+							     PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hExportInt,
+							PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psExportInt)
+		{
+			SyncFbFenceExportDestroyInsecure(psExportInt);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroyInsecure(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureIN_UI8,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psExportInt,
+				       (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->hExport,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_FALSE);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hExportInt,
+				     psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+		     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT((psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK) ||
+		   (psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportDestroyInsecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportDestroyInsecure_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportDestroyInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceImportInsecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hImport = psSyncFbFenceImportInsecureIN->hImport;
+	PVRSRV_FENCE_EXPORT *psImportInt = NULL;
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psImportInt,
+				       hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_TRUE);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    SyncFbFenceImportInsecure(psConnection, OSGetDevNode(psConnection),
+				      psImportInt, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportInsecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportInsecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psImportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceImportInsecure NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceExportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceExportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportSecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportSecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportSecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportSecureOUT->eError =
+	    SyncFbFenceExportSecure(psConnection, OSGetDevNode(psConnection),
+				    psFenceInt,
+				    &psSyncFbFenceExportSecureOUT->Export,
+				    &psExportInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportSecure_exit;
+	}
+
+SyncFbFenceExportSecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psExportInt)
+		{
+			SyncFbFenceExportDestroySecure(psExportInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroySecure(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureIN_UI8,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncFbFenceExportDestroySecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroySecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroySecureOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncFbFenceExportDestroySecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncFbFenceExportDestroySecure_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceImportSecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceImportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceImportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    SyncFbFenceImportSecure(psConnection, OSGetDevNode(psConnection),
+				    psSyncFbFenceImportSecureIN->Import, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportSecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportSecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportSecure_exit:
+
+	if (psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+
+/*
+ * Register all SYNCFALLBACK functions with services
+ */
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR,
+			      PVRSRVBridgeSyncFbTimelineCreatePVR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE,
+			      PVRSRVBridgeSyncFbTimelineRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK, PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP,
+			      PVRSRVBridgeSyncFbFenceDup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE,
+			      PVRSRVBridgeSyncFbFenceMerge, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE,
+			      PVRSRVBridgeSyncFbFenceRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT,
+			      PVRSRVBridgeSyncFbFenceWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP,
+			      PVRSRVBridgeSyncFbFenceDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW,
+			      PVRSRVBridgeSyncFbTimelineCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW,
+			      PVRSRVBridgeSyncFbFenceCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW,
+			      PVRSRVBridgeSyncFbTimelineAdvanceSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroyInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceImportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceExportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroySecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceImportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all syncfallback functions with services
+ */
+void DeinitSYNCFALLBACKBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_bridge.h
new file mode 100644
index 000000000000..544efd962834
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_bridge.h
@@ -0,0 +1,68 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNCTRACKING_BRIDGE_H
+#define CLIENT_SYNCTRACKING_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_synctracking_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName);
+
+#endif /* CLIENT_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_direct_bridge.c
new file mode 100644
index 000000000000..baeb89a6a0ae
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_direct_bridge.c
@@ -0,0 +1,92 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for synctracking
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_synctracking_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "sync.h"
+#include "sync_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	pshRecordInt = (SYNC_RECORD_HANDLE) hhRecord;
+
+	eError = PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt;
+
+	pshServerSyncPrimBlockInt = (SYNC_PRIMITIVE_BLOCK *) hhServerSyncPrimBlock;
+
+	eError =
+	    PVRSRVSyncRecordAddKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  ui32ui32FwBlockAddr,
+				  ui32ui32SyncOffset,
+				  bbServerSync, ui32ClassNameSize, puiClassName);
+
+	*phhRecord = pshRecordInt;
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/common_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/common_synctracking_bridge.h
new file mode 100644
index 000000000000..9fd9fdb796a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/common_synctracking_bridge.h
@@ -0,0 +1,97 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCTRACKING_BRIDGE_H
+#define COMMON_SYNCTRACKING_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST			(PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1)
+
+/*******************************************
+            SyncRecordRemoveByHandle
+ *******************************************/
+
+/* Bridge in structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	IMG_HANDLE hhRecord;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE;
+
+/* Bridge out structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE;
+
+/*******************************************
+            SyncRecordAdd
+ *******************************************/
+
+/* Bridge in structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhServerSyncPrimBlock;
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32ui32FwBlockAddr;
+	IMG_UINT32 ui32ui32SyncOffset;
+	IMG_BOOL bbServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDADD;
+
+/* Bridge out structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhRecord;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDADD;
+
+#endif /* COMMON_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/server_synctracking_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/server_synctracking_bridge.c
new file mode 100644
index 000000000000..06757e5ef8d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/server_synctracking_bridge.c
@@ -0,0 +1,329 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+
+#include "common_synctracking_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeSyncRecordRemoveByHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleIN_UI8,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordRemoveByHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncRecordRemoveByHandleIN->hhRecord,
+					      PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE);
+	if (unlikely((psSyncRecordRemoveByHandleOUT->eError != PVRSRV_OK) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncRecordRemoveByHandleOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordRemoveByHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordRemoveByHandle_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncRecordAddpshRecordIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVSyncRecordRemoveByHandleKM((SYNC_RECORD_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncRecordAdd(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncRecordAddIN_UI8,
+			  IMG_UINT8 * psSyncRecordAddOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDADD *psSyncRecordAddIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDADD *psSyncRecordAddOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddOUT_UI8, 0);
+
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	IMG_HANDLE hhServerSyncPrimBlock = psSyncRecordAddIN->hhServerSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncRecordAddIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncRecordAdd_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncRecordAdd_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncRecordAddIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncRecordAddIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncRecordAddOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncRecordAdd_exit;
+			}
+		}
+	}
+
+	if (psSyncRecordAddIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncRecordAddIN->puiClassName,
+		     psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncRecordAddOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncRecordAdd_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncRecordAddOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pshServerSyncPrimBlockInt,
+				       hhServerSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError =
+	    PVRSRVSyncRecordAddKM(psConnection, OSGetDevNode(psConnection),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  psSyncRecordAddIN->ui32ui32FwBlockAddr,
+				  psSyncRecordAddIN->ui32ui32SyncOffset,
+				  psSyncRecordAddIN->bbServerSync,
+				  psSyncRecordAddIN->ui32ClassNameSize, uiClassNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psSyncRecordAddOUT->hhRecord,
+							       (void *)pshRecordInt,
+							       PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+							       PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+							       (PFN_HANDLE_RELEASE) &
+							       _SyncRecordAddpshRecordIntRelease);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordAdd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pshServerSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hhServerSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psSyncRecordAddOUT->eError != PVRSRV_OK)
+	{
+		if (pshRecordInt)
+		{
+			PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncRecordAddOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+
+/*
+ * Register all SYNCTRACKING functions with services
+ */
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+			      PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE,
+			      PVRSRVBridgeSyncRecordRemoveByHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING, PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD,
+			      PVRSRVBridgeSyncRecordAdd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDADD),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDADD));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all synctracking functions with services
+ */
+void DeinitSYNCTRACKINGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.2.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.2.30.h
new file mode 100644
index 000000000000..88b123174c49
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.2.30.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.2.30 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_2_30_H
+#define RGXCONFIG_KM_1_V_2_30_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 30
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_2_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.19.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.19.h
new file mode 100644
index 000000000000..5f60752630c0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.19.h
@@ -0,0 +1,82 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.4.19 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_4_19_H
+#define RGXCONFIG_KM_1_V_4_19_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 19
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_4_19_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.5.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.5.h
new file mode 100644
index 000000000000..1af637872cf9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.5.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.4.5 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_4_5_H
+#define RGXCONFIG_KM_1_V_4_5_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 5
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_4_5_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_15.V.1.64.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_15.V.1.64.h
new file mode 100644
index 000000000000..3ad2665eb9ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_15.V.1.64.h
@@ -0,0 +1,87 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 15.V.1.64 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_15_V_1_64_H
+#define RGXCONFIG_KM_15_V_1_64_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 15
+#define RGX_BNC_KM_N 1
+#define RGX_BNC_KM_C 64
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP217)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_15_V_1_64_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.18.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.18.h
new file mode 100644
index 000000000000..4594c6429129
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.18.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.104.18 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_104_18_H
+#define RGXCONFIG_KM_22_V_104_18_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 18
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_104_18_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.218.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.218.h
new file mode 100644
index 000000000000..e271c768760d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.218.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.104.218 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_104_218_H
+#define RGXCONFIG_KM_22_V_104_218_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 218
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_104_218_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.208.318.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.208.318.h
new file mode 100644
index 000000000000..8b76f2aa6cf0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.208.318.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.208.318 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_208_318_H
+#define RGXCONFIG_KM_22_V_208_318_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 318
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_208_318_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.21.16.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.21.16.h
new file mode 100644
index 000000000000..9d54fdd6a578
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.21.16.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.21.16 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_21_16_H
+#define RGXCONFIG_KM_22_V_21_16_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 21
+#define RGX_BNC_KM_C 16
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (8U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_21_16_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.25.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.25.h
new file mode 100644
index 000000000000..ffe2926153ff
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.25.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.25 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_25_H
+#define RGXCONFIG_KM_22_V_54_25_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 25
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_25_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.30.h
new file mode 100644
index 000000000000..d59101b73ad6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.30.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.30 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_30_H
+#define RGXCONFIG_KM_22_V_54_30_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 30
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.330.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.330.h
new file mode 100644
index 000000000000..fd7547bca34e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.330.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.330 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_330_H
+#define RGXCONFIG_KM_22_V_54_330_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 330
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_330_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.38.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.38.h
new file mode 100644
index 000000000000..6db0a9272d48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.38.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.38 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_38_H
+#define RGXCONFIG_KM_22_V_54_38_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 38
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_38_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.104.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.104.504.h
new file mode 100644
index 000000000000..e79b894f1299
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.104.504.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.104.504 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_104_504_H
+#define RGXCONFIG_KM_24_V_104_504_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 504
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_104_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.504.h
new file mode 100644
index 000000000000..2b92c75a2d80
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.504.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.208.504 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_208_504_H
+#define RGXCONFIG_KM_24_V_208_504_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 504
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_208_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.505.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.505.h
new file mode 100644
index 000000000000..d14c991870e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.505.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.208.505 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_208_505_H
+#define RGXCONFIG_KM_24_V_208_505_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 505
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_208_505_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.54.204.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.54.204.h
new file mode 100644
index 000000000000..0274380eafdd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.54.204.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.54.204 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_54_204_H
+#define RGXCONFIG_KM_24_V_54_204_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 204
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_54_204_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.108.208.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.108.208.h
new file mode 100644
index 000000000000..bae4231b4f9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.108.208.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 29.V.108.208 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_29_V_108_208_H
+#define RGXCONFIG_KM_29_V_108_208_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 29
+#define RGX_BNC_KM_N 108
+#define RGX_BNC_KM_C 208
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_29_V_108_208_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.52.202.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.52.202.h
new file mode 100644
index 000000000000..805e6df66ab3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.52.202.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 29.V.52.202 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_29_V_52_202_H
+#define RGXCONFIG_KM_29_V_52_202_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 29
+#define RGX_BNC_KM_N 52
+#define RGX_BNC_KM_C 202
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_29_V_52_202_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.11.3.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.11.3.h
new file mode 100644
index 000000000000..19155570aa2e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.11.3.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 33.V.11.3 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_33_V_11_3_H
+#define RGXCONFIG_KM_33_V_11_3_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 33
+#define RGX_BNC_KM_N 11
+#define RGX_BNC_KM_C 3
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (1U)
+#define RGX_FEATURE_NUM_OSIDS (2U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_33_V_11_3_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.22.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.22.1.h
new file mode 100644
index 000000000000..a375f749cf46
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.22.1.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 33.V.22.1 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_33_V_22_1_H
+#define RGXCONFIG_KM_33_V_22_1_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 33
+#define RGX_BNC_KM_N 22
+#define RGX_BNC_KM_C 1
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_PM_MMU_VFP
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_33_V_22_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.182.h
new file mode 100644
index 000000000000..48084d57d188
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.182.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_182_H
+#define RGXCONFIG_KM_36_V_104_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.183.h
new file mode 100644
index 000000000000..dc4a3590f0d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.183.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.183 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_183_H
+#define RGXCONFIG_KM_36_V_104_183_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 183
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.796.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.796.h
new file mode 100644
index 000000000000..d4f87c4ffa4d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.796.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.796 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_796_H
+#define RGXCONFIG_KM_36_V_104_796_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 796
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (2U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SAFETY_IRQ
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_796_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.52.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.52.182.h
new file mode 100644
index 000000000000..049939fbb452
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.52.182.h
@@ -0,0 +1,105 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.52.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_52_182_H
+#define RGXCONFIG_KM_36_V_52_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 52
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_52_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.103.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.103.h
new file mode 100644
index 000000000000..3f8b4ffab362
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.103.h
@@ -0,0 +1,103 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.103 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_103_H
+#define RGXCONFIG_KM_36_V_54_103_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 103
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_36_V_54_103_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.182.h
new file mode 100644
index 000000000000..78b53d52cf4a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.182.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_182_H
+#define RGXCONFIG_KM_36_V_54_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.183.h
new file mode 100644
index 000000000000..d2f5dd4bab1b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.183.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.183 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_183_H
+#define RGXCONFIG_KM_36_V_54_183_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 183
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.280.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.280.h
new file mode 100644
index 000000000000..30bbaa68dcb5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.280.h
@@ -0,0 +1,110 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.280 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_280_H
+#define RGXCONFIG_KM_36_V_54_280_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 280
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (11U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_280_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.51.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.51.h
new file mode 100644
index 000000000000..28114cad8c97
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.51.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.2.51 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_2_51_H
+#define RGXCONFIG_KM_4_V_2_51_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 51
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_2_51_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.58.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.58.h
new file mode 100644
index 000000000000..99ee53e0c197
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.58.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.2.58 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_2_58_H
+#define RGXCONFIG_KM_4_V_2_58_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 58
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_2_58_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.4.55.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.4.55.h
new file mode 100644
index 000000000000..c27adf56b55b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.4.55.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.4.55 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_4_55_H
+#define RGXCONFIG_KM_4_V_4_55_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 55
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_4_55_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.6.62.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.6.62.h
new file mode 100644
index 000000000000..2d4aa9599a2b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.6.62.h
@@ -0,0 +1,93 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.6.62 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_6_62_H
+#define RGXCONFIG_KM_4_V_6_62_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 6
+#define RGX_BNC_KM_C 62
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_6_62_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.102.389.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.102.389.h
new file mode 100644
index 000000000000..a8eab1e08e4d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.102.389.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 46.V.102.389 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_46_V_102_389_H
+#define RGXCONFIG_KM_46_V_102_389_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 46
+#define RGX_BNC_KM_N 102
+#define RGX_BNC_KM_C 389
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (20U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_VOLCANIC_TB
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_46_V_102_389_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.204.390.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.204.390.h
new file mode 100644
index 000000000000..96544ed32b48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.204.390.h
@@ -0,0 +1,114 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 46.V.204.390 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_46_V_204_390_H
+#define RGXCONFIG_KM_46_V_204_390_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 46
+#define RGX_BNC_KM_N 204
+#define RGX_BNC_KM_C 390
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (20U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_VOLCANIC_TB
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_46_V_204_390_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_5.V.1.46.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_5.V.1.46.h
new file mode 100644
index 000000000000..1c9b7db878d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_5.V.1.46.h
@@ -0,0 +1,85 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 5.V.1.46 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_5_V_1_46_H
+#define RGXCONFIG_KM_5_V_1_46_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 5
+#define RGX_BNC_KM_N 1
+#define RGX_BNC_KM_C 46
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP217)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (1U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_5_V_1_46_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_6.V.4.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_6.V.4.35.h
new file mode 100644
index 000000000000..cbf5589ce79c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_6.V.4.35.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 6.V.4.35 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_6_V_4_35_H
+#define RGXCONFIG_KM_6_V_4_35_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 6
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 35
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_6_V_4_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.39.4.19.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.39.4.19.h
new file mode 100644
index 000000000000..a64cb4f379e9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.39.4.19.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.39.4.19
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_39_4_19_H
+#define RGXCORE_KM_1_39_4_19_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2784771 */
+
+/******************************************************************************
+ * BVNC = 1.39.4.19
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 39
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 19
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344
+#define FIX_HW_BRN_42321
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_39_4_19_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.75.2.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.75.2.30.h
new file mode 100644
index 000000000000..b33edeafe674
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.75.2.30.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.75.2.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_75_2_30_H
+#define RGXCORE_KM_1_75_2_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2309075 */
+
+/******************************************************************************
+ * BVNC = 1.75.2.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 75
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_42321
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_75_2_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.82.4.5.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.82.4.5.h
new file mode 100644
index 000000000000..ae53d7ceef27
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.82.4.5.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.82.4.5
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_82_4_5_H
+#define RGXCORE_KM_1_82_4_5_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2503111 */
+
+/******************************************************************************
+ * BVNC = 1.82.4.5
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 82
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 5
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_82_4_5_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_15.5.1.64.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_15.5.1.64.h
new file mode 100644
index 000000000000..e354f5b8c8fe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_15.5.1.64.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 15.5.1.64
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_15_5_1_64_H
+#define RGXCORE_KM_15_5_1_64_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3846532 */
+
+/******************************************************************************
+ * BVNC = 15.5.1.64
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 15
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 1
+#define RGX_BVNC_KM_C 64
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_15_5_1_64_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.102.54.38.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.102.54.38.h
new file mode 100644
index 000000000000..ee279a63a255
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.102.54.38.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.102.54.38
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_102_54_38_H
+#define RGXCORE_KM_22_102_54_38_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4831550 */
+
+/******************************************************************************
+ * BVNC = 22.102.54.38
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 102
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 38
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_102_54_38_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.104.208.318.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.104.208.318.h
new file mode 100644
index 000000000000..58472828a1b0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.104.208.318.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.104.208.318
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_104_208_318_H
+#define RGXCORE_KM_22_104_208_318_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5124208 */
+
+/******************************************************************************
+ * BVNC = 22.104.208.318
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 104
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 318
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_104_208_318_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.105.208.318.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.105.208.318.h
new file mode 100644
index 000000000000..d175ac4a8a87
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.105.208.318.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.105.208.318
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_105_208_318_H
+#define RGXCORE_KM_22_105_208_318_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5324554 */
+
+/******************************************************************************
+ * BVNC = 22.105.208.318
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 105
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 318
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_105_208_318_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.30.54.25.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.30.54.25.h
new file mode 100644
index 000000000000..34d2c176900a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.30.54.25.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.30.54.25
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_30_54_25_H
+#define RGXCORE_KM_22_30_54_25_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4086500 */
+
+/******************************************************************************
+ * BVNC = 22.30.54.25
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 30
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 25
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_30_54_25_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.40.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.40.54.30.h
new file mode 100644
index 000000000000..f9dfea7676f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.40.54.30.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.40.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_40_54_30_H
+#define RGXCORE_KM_22_40_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4094817 */
+
+/******************************************************************************
+ * BVNC = 22.40.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 40
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_40_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.46.54.330.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.46.54.330.h
new file mode 100644
index 000000000000..f269fc0c8a12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.46.54.330.h
@@ -0,0 +1,78 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.46.54.330
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_46_54_330_H
+#define RGXCORE_KM_22_46_54_330_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4136505 */
+
+/******************************************************************************
+ * BVNC = 22.46.54.330
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 46
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 330
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_22_46_54_330_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.49.21.16.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.49.21.16.h
new file mode 100644
index 000000000000..f1cc4df22992
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.49.21.16.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.49.21.16
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_49_21_16_H
+#define RGXCORE_KM_22_49_21_16_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4158766 */
+
+/******************************************************************************
+ * BVNC = 22.49.21.16
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 49
+#define RGX_BVNC_KM_N 21
+#define RGX_BVNC_KM_C 16
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_49_21_16_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.67.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.67.54.30.h
new file mode 100644
index 000000000000..c60d20b87718
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.67.54.30.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.67.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_67_54_30_H
+#define RGXCORE_KM_22_67_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4339986 */
+
+/******************************************************************************
+ * BVNC = 22.67.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 67
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_67_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.68.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.68.54.30.h
new file mode 100644
index 000000000000..b5f8459a1442
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.68.54.30.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.68.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_68_54_30_H
+#define RGXCORE_KM_22_68_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4339984 */
+
+/******************************************************************************
+ * BVNC = 22.68.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 68
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_68_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.86.104.218.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.86.104.218.h
new file mode 100644
index 000000000000..a0c3001f1147
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.86.104.218.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.86.104.218
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_86_104_218_H
+#define RGXCORE_KM_22_86_104_218_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4665024 */
+
+/******************************************************************************
+ * BVNC = 22.86.104.218
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 86
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 218
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_86_104_218_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.87.104.18.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.87.104.18.h
new file mode 100644
index 000000000000..2a2c66ad95d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.87.104.18.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.87.104.18
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_87_104_18_H
+#define RGXCORE_KM_22_87_104_18_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4658768 */
+
+/******************************************************************************
+ * BVNC = 22.87.104.18
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 87
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 18
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_87_104_18_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.50.208.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.50.208.504.h
new file mode 100644
index 000000000000..1aca3f49a04c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.50.208.504.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.50.208.504
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_50_208_504_H
+#define RGXCORE_KM_24_50_208_504_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5086680 */
+
+/******************************************************************************
+ * BVNC = 24.50.208.504
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 50
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 504
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_50_208_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.56.208.505.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.56.208.505.h
new file mode 100644
index 000000000000..26060f932d47
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.56.208.505.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.56.208.505
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_56_208_505_H
+#define RGXCORE_KM_24_56_208_505_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5203837 */
+
+/******************************************************************************
+ * BVNC = 24.56.208.505
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 56
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 505
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_56_208_505_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.66.54.204.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.66.54.204.h
new file mode 100644
index 000000000000..4fb0ef0395b3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.66.54.204.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.66.54.204
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_66_54_204_H
+#define RGXCORE_KM_24_66_54_204_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5200207 */
+
+/******************************************************************************
+ * BVNC = 24.66.54.204
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 66
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 204
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_66_54_204_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.67.104.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.67.104.504.h
new file mode 100644
index 000000000000..4266f63edac9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.67.104.504.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.67.104.504
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_67_104_504_H
+#define RGXCORE_KM_24_67_104_504_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5203838 */
+
+/******************************************************************************
+ * BVNC = 24.67.104.504
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 67
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 504
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_67_104_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.14.108.208.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.14.108.208.h
new file mode 100644
index 000000000000..6636f54022c7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.14.108.208.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 29.14.108.208
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_29_14_108_208_H
+#define RGXCORE_KM_29_14_108_208_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5682134 */
+
+/******************************************************************************
+ * BVNC = 29.14.108.208
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 29
+#define RGX_BVNC_KM_V 14
+#define RGX_BVNC_KM_N 108
+#define RGX_BVNC_KM_C 208
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_68186
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_29_14_108_208_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.19.52.202.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.19.52.202.h
new file mode 100644
index 000000000000..01683cb721ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.19.52.202.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 29.19.52.202
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_29_19_52_202_H
+#define RGXCORE_KM_29_19_52_202_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5691248 */
+
+/******************************************************************************
+ * BVNC = 29.19.52.202
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 29
+#define RGX_BVNC_KM_V 19
+#define RGX_BVNC_KM_N 52
+#define RGX_BVNC_KM_C 202
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_68186
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_29_19_52_202_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.15.11.3.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.15.11.3.h
new file mode 100644
index 000000000000..9762fa901bda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.15.11.3.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 33.15.11.3
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_33_15_11_3_H
+#define RGXCORE_KM_33_15_11_3_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5820045 */
+
+/******************************************************************************
+ * BVNC = 33.15.11.3
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 33
+#define RGX_BVNC_KM_V 15
+#define RGX_BVNC_KM_N 11
+#define RGX_BVNC_KM_C 3
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_33_15_11_3_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.8.22.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.8.22.1.h
new file mode 100644
index 000000000000..272e922e0191
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.8.22.1.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 33.8.22.1
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_33_8_22_1_H
+#define RGXCORE_KM_33_8_22_1_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5544349 */
+
+/******************************************************************************
+ * BVNC = 33.8.22.1
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 33
+#define RGX_BVNC_KM_V 8
+#define RGX_BVNC_KM_N 22
+#define RGX_BVNC_KM_C 1
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_33_8_22_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.29.52.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.29.52.182.h
new file mode 100644
index 000000000000..7a5a87c84f7b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.29.52.182.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.29.52.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_29_52_182_H
+#define RGXCORE_KM_36_29_52_182_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908879 */
+
+/******************************************************************************
+ * BVNC = 36.29.52.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 29
+#define RGX_BVNC_KM_N 52
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_29_52_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.50.54.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.50.54.182.h
new file mode 100644
index 000000000000..0eccc97305ab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.50.54.182.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.50.54.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_50_54_182_H
+#define RGXCORE_KM_36_50_54_182_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5849603 */
+
+/******************************************************************************
+ * BVNC = 36.50.54.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 50
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_50_54_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.52.104.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.52.104.182.h
new file mode 100644
index 000000000000..d14b662b9730
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.52.104.182.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.52.104.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_52_104_182_H
+#define RGXCORE_KM_36_52_104_182_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5849605 */
+
+/******************************************************************************
+ * BVNC = 36.52.104.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 52
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_52_104_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.53.104.796.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.53.104.796.h
new file mode 100644
index 000000000000..c3ec305d3830
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.53.104.796.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.53.104.796
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_53_104_796_H
+#define RGXCORE_KM_36_53_104_796_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5896094 */
+
+/******************************************************************************
+ * BVNC = 36.53.104.796
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 53
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 796
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_53_104_796_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.54.54.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.54.54.183.h
new file mode 100644
index 000000000000..6cac1b26d084
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.54.54.183.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.54.54.183
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_54_54_183_H
+#define RGXCORE_KM_36_54_54_183_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908021 */
+
+/******************************************************************************
+ * BVNC = 36.54.54.183
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 54
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 183
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_54_54_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.55.54.103.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.55.54.103.h
new file mode 100644
index 000000000000..809d5808d3ea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.55.54.103.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.55.54.103
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_55_54_103_H
+#define RGXCORE_KM_36_55_54_103_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908021 */
+
+/******************************************************************************
+ * BVNC = 36.55.54.103
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 55
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 103
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_55_54_103_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.56.104.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.56.104.183.h
new file mode 100644
index 000000000000..96e875e3b1f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.56.104.183.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.56.104.183
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_56_104_183_H
+#define RGXCORE_KM_36_56_104_183_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5942195 */
+
+/******************************************************************************
+ * BVNC = 36.56.104.183
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 56
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 183
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_56_104_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.60.54.280.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.60.54.280.h
new file mode 100644
index 000000000000..5f46e75d4f06
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.60.54.280.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.60.54.280
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_60_54_280_H
+#define RGXCORE_KM_36_60_54_280_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @6040409 */
+
+/******************************************************************************
+ * BVNC = 36.60.54.280
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 60
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 280
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_60_54_280_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.31.4.55.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.31.4.55.h
new file mode 100644
index 000000000000..a39603ae35a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.31.4.55.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.31.4.55
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_31_4_55_H
+#define RGXCORE_KM_4_31_4_55_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2919104 */
+
+/******************************************************************************
+ * BVNC = 4.31.4.55
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 31
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 55
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_31_4_55_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.40.2.51.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.40.2.51.h
new file mode 100644
index 000000000000..931fa0cd445c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.40.2.51.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.40.2.51
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_40_2_51_H
+#define RGXCORE_KM_4_40_2_51_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3254374 */
+
+/******************************************************************************
+ * BVNC = 4.40.2.51
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 40
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 51
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_40_2_51_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.43.6.62.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.43.6.62.h
new file mode 100644
index 000000000000..90d35a8360bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.43.6.62.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.43.6.62
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_43_6_62_H
+#define RGXCORE_KM_4_43_6_62_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3253129 */
+
+/******************************************************************************
+ * BVNC = 4.43.6.62
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 43
+#define RGX_BVNC_KM_N 6
+#define RGX_BVNC_KM_C 62
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_43_6_62_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.45.2.58.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.45.2.58.h
new file mode 100644
index 000000000000..00e6b90f6207
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.45.2.58.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.45.2.58
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_45_2_58_H
+#define RGXCORE_KM_4_45_2_58_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3547765 */
+
+/******************************************************************************
+ * BVNC = 4.45.2.58
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 45
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 58
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_45_2_58_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.46.6.62.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.46.6.62.h
new file mode 100644
index 000000000000..909198f233b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.46.6.62.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.46.6.62
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_46_6_62_H
+#define RGXCORE_KM_4_46_6_62_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4015666 */
+
+/******************************************************************************
+ * BVNC = 4.46.6.62
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 46
+#define RGX_BVNC_KM_N 6
+#define RGX_BVNC_KM_C 62
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_46_6_62_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.72.102.389.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.72.102.389.h
new file mode 100644
index 000000000000..66eb259c0ebf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.72.102.389.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 46.72.102.389
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_46_72_102_389_H
+#define RGXCORE_KM_46_72_102_389_H
+
+/* Automatically generated file (07/02/2023 09:11:39): Do not edit manually */
+/* CS: @6385487 */
+
+/******************************************************************************
+ * BVNC = 46.72.102.389
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 46
+#define RGX_BVNC_KM_V 72
+#define RGX_BVNC_KM_N 102
+#define RGX_BVNC_KM_C 389
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_66927
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_46_72_102_389_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.94.204.390.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.94.204.390.h
new file mode 100644
index 000000000000..548989eb518d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.94.204.390.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 46.94.204.390
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_46_94_204_390_H
+#define RGXCORE_KM_46_94_204_390_H
+
+/* Automatically generated file (02/03/2023 09:11:42): Do not edit manually */
+/* CS: @6389604 */
+
+/******************************************************************************
+ * BVNC = 46.94.204.390
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 46
+#define RGX_BVNC_KM_V 94
+#define RGX_BVNC_KM_N 204
+#define RGX_BVNC_KM_C 390
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_66927
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_46_94_204_390_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_5.9.1.46.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_5.9.1.46.h
new file mode 100644
index 000000000000..cba592e7b327
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_5.9.1.46.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 5.9.1.46
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_5_9_1_46_H
+#define RGXCORE_KM_5_9_1_46_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2967148 */
+
+/******************************************************************************
+ * BVNC = 5.9.1.46
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 5
+#define RGX_BVNC_KM_V 9
+#define RGX_BVNC_KM_N 1
+#define RGX_BVNC_KM_C 46
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_5_9_1_46_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_6.34.4.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_6.34.4.35.h
new file mode 100644
index 000000000000..b45ed9e1547c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_6.34.4.35.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 6.34.4.35
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_6_34_4_35_H
+#define RGXCORE_KM_6_34_4_35_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3533654 */
+
+/******************************************************************************
+ * BVNC = 6.34.4.35
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 6
+#define RGX_BVNC_KM_V 34
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 35
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_6_34_4_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_defs_km.h
new file mode 100644
index 000000000000..73c44b77f498
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_defs_km.h
@@ -0,0 +1,341 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_defs_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_DEFS_KM_H
+#define RGX_BVNC_DEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(RGX_BVNC_DEFS_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_defs_um.h"
+#endif
+
+#define BVNC_FIELD_WIDTH (16U)
+
+#define PVR_ARCH_NAME "rogue"
+
+
+/******************************************************************************
+ * Mask and bit-position macros for features without values
+ *****************************************************************************/
+
+#define RGX_FEATURE_AXI_ACELITE_POS                                     (0U)
+#define RGX_FEATURE_AXI_ACELITE_BIT_MASK                                (IMG_UINT64_C(0x0000000000000001))
+
+#define RGX_FEATURE_CLUSTER_GROUPING_POS                                (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK                           (IMG_UINT64_C(0x0000000000000002))
+
+#define RGX_FEATURE_COMPUTE_POS                                         (2U)
+#define RGX_FEATURE_COMPUTE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000004))
+
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_POS                          (3U)
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_BIT_MASK                     (IMG_UINT64_C(0x0000000000000008))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_POS                                 (4U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_BIT_MASK                            (IMG_UINT64_C(0x0000000000000010))
+
+#define RGX_FEATURE_COREID_PER_OS_POS                                   (5U)
+#define RGX_FEATURE_COREID_PER_OS_BIT_MASK                              (IMG_UINT64_C(0x0000000000000020))
+
+#define RGX_FEATURE_DYNAMIC_DUST_POWER_POS                              (6U)
+#define RGX_FEATURE_DYNAMIC_DUST_POWER_BIT_MASK                         (IMG_UINT64_C(0x0000000000000040))
+
+#define RGX_FEATURE_FASTRENDER_DM_POS                                   (7U)
+#define RGX_FEATURE_FASTRENDER_DM_BIT_MASK                              (IMG_UINT64_C(0x0000000000000080))
+
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_POS                           (8U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_BIT_MASK                      (IMG_UINT64_C(0x0000000000000100))
+
+#define RGX_FEATURE_GPU_VIRTUALISATION_POS                              (9U)
+#define RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK                         (IMG_UINT64_C(0x0000000000000200))
+
+#define RGX_FEATURE_GS_RTA_SUPPORT_POS                                  (10U)
+#define RGX_FEATURE_GS_RTA_SUPPORT_BIT_MASK                             (IMG_UINT64_C(0x0000000000000400))
+
+#define RGX_FEATURE_IRQ_PER_OS_POS                                      (11U)
+#define RGX_FEATURE_IRQ_PER_OS_BIT_MASK                                 (IMG_UINT64_C(0x0000000000000800))
+
+#define RGX_FEATURE_MIPS_POS                                            (12U)
+#define RGX_FEATURE_MIPS_BIT_MASK                                       (IMG_UINT64_C(0x0000000000001000))
+
+#define RGX_FEATURE_PBE2_IN_XE_POS                                      (13U)
+#define RGX_FEATURE_PBE2_IN_XE_BIT_MASK                                 (IMG_UINT64_C(0x0000000000002000))
+
+#define RGX_FEATURE_PBE_CHECKSUM_2D_POS                                 (14U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D_BIT_MASK                            (IMG_UINT64_C(0x0000000000004000))
+
+#define RGX_FEATURE_PBVNC_COREID_REG_POS                                (15U)
+#define RGX_FEATURE_PBVNC_COREID_REG_BIT_MASK                           (IMG_UINT64_C(0x0000000000008000))
+
+#define RGX_FEATURE_PERFBUS_POS                                         (16U)
+#define RGX_FEATURE_PERFBUS_BIT_MASK                                    (IMG_UINT64_C(0x0000000000010000))
+
+#define RGX_FEATURE_PERF_COUNTER_BATCH_POS                              (17U)
+#define RGX_FEATURE_PERF_COUNTER_BATCH_BIT_MASK                         (IMG_UINT64_C(0x0000000000020000))
+
+#define RGX_FEATURE_PM_MMU_VFP_POS                                      (18U)
+#define RGX_FEATURE_PM_MMU_VFP_BIT_MASK                                 (IMG_UINT64_C(0x0000000000040000))
+
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_POS                              (19U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_BIT_MASK                         (IMG_UINT64_C(0x0000000000080000))
+
+#define RGX_FEATURE_ROGUEXE_POS                                         (20U)
+#define RGX_FEATURE_ROGUEXE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000100000))
+
+#define RGX_FEATURE_SAFETY_IRQ_POS                                      (21U)
+#define RGX_FEATURE_SAFETY_IRQ_BIT_MASK                                 (IMG_UINT64_C(0x0000000000200000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_POS                (22U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_BIT_MASK           (IMG_UINT64_C(0x0000000000400000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1_POS             (23U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1_BIT_MASK        (IMG_UINT64_C(0x0000000000800000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2_POS             (24U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2_BIT_MASK        (IMG_UINT64_C(0x0000000001000000))
+
+#define RGX_FEATURE_SINGLE_BIF_POS                                      (25U)
+#define RGX_FEATURE_SINGLE_BIF_BIT_MASK                                 (IMG_UINT64_C(0x0000000002000000))
+
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE_POS                           (26U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE_BIT_MASK                      (IMG_UINT64_C(0x0000000004000000))
+
+#define RGX_FEATURE_SOC_TIMER_POS                                       (27U)
+#define RGX_FEATURE_SOC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0000000008000000))
+
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_POS                            (28U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_BIT_MASK                       (IMG_UINT64_C(0x0000000010000000))
+
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_POS                                (29U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_BIT_MASK                           (IMG_UINT64_C(0x0000000020000000))
+
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION_POS                          (30U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK                     (IMG_UINT64_C(0x0000000040000000))
+
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT_POS                           (31U)
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK                      (IMG_UINT64_C(0x0000000080000000))
+
+#define RGX_FEATURE_TFBC_NATIVE_YUV10_POS                               (32U)
+#define RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK                          (IMG_UINT64_C(0x0000000100000000))
+
+#define RGX_FEATURE_TILE_REGION_PROTECTION_POS                          (33U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION_BIT_MASK                     (IMG_UINT64_C(0x0000000200000000))
+
+#define RGX_FEATURE_TLA_POS                                             (34U)
+#define RGX_FEATURE_TLA_BIT_MASK                                        (IMG_UINT64_C(0x0000000400000000))
+
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_POS             (35U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_BIT_MASK        (IMG_UINT64_C(0x0000000800000000))
+
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_POS                         (36U)
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_BIT_MASK                    (IMG_UINT64_C(0x0000001000000000))
+
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_POS                      (37U)
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK                 (IMG_UINT64_C(0x0000002000000000))
+
+#define RGX_FEATURE_VOLCANIC_TB_POS                                     (38U)
+#define RGX_FEATURE_VOLCANIC_TB_BIT_MASK                                (IMG_UINT64_C(0x0000004000000000))
+
+#define RGX_FEATURE_WATCHDOG_TIMER_POS                                  (39U)
+#define RGX_FEATURE_WATCHDOG_TIMER_BIT_MASK                             (IMG_UINT64_C(0x0000008000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_POS                            (40U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_BIT_MASK                       (IMG_UINT64_C(0x0000010000000000))
+
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY_POS                             (41U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY_BIT_MASK                        (IMG_UINT64_C(0x0000020000000000))
+
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE_POS                           (42U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE_BIT_MASK                      (IMG_UINT64_C(0x0000040000000000))
+
+
+/******************************************************************************
+ * Defines for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (2U)
+#define RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX                              (3U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX                  (2U)
+#define RGX_FEATURE_FBCDC_MAX_VALUE_IDX                                 (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX                       (6U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX                    (4U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX           (2U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX             (2U)
+#define RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_META_MAX_VALUE_IDX                                  (4U)
+#define RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX                     (3U)
+#define RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX                          (5U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX                     (9U)
+#define RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX                            (2U)
+#define RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX                             (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX                        (4U)
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX       (3U)
+#define RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX                             (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX              (2U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX                 (6U)
+#define RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX                          (2U)
+#define RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX                          (4U)
+#define RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX            (2U)
+#define RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX                       (2U)
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX           (2U)
+#define RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX                        (2U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX                (2U)
+
+/******************************************************************************
+ * Features with values indexes
+ *****************************************************************************/
+
+typedef enum _RGX_FEATURE_WITH_VALUE_INDEX_ {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_ECC_RAMS_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_IDX,
+	RGX_FEATURE_FBCDC_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_IDX,
+	RGX_FEATURE_LAYOUT_MARS_IDX,
+	RGX_FEATURE_META_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX,
+	RGX_FEATURE_NUM_MEMBUS_IDX,
+	RGX_FEATURE_NUM_OSIDS_IDX,
+	RGX_FEATURE_NUM_RASTER_PIPES_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_IDX,
+	RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_IDX,
+	RGX_FEATURE_SLC_BANKS_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_IDX,
+	RGX_FEATURE_TFBC_VERSION_IDX,
+	RGX_FEATURE_TILE_SIZE_X_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_IDX,
+	RGX_FEATURE_XE_ARCHITECTURE_IDX,
+	RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_IDX,
+	RGX_FEATURE_XPU_MAX_SLAVES_IDX,
+	RGX_FEATURE_XPU_REGISTER_BROADCAST_IDX,
+	RGX_FEATURE_WITH_VALUES_MAX_IDX,
+} RGX_FEATURE_WITH_VALUE_INDEX;
+
+
+/******************************************************************************
+ * Mask and bit-position macros for ERNs and BRNs
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344_POS                                            (0U)
+#define FIX_HW_BRN_38344_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000001))
+
+#define HW_ERN_42290_POS                                                (1U)
+#define HW_ERN_42290_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000002))
+
+#define FIX_HW_BRN_42321_POS                                            (2U)
+#define FIX_HW_BRN_42321_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000004))
+
+#define HW_ERN_42606_POS                                                (3U)
+#define HW_ERN_42606_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000008))
+
+#define HW_ERN_47025_POS                                                (4U)
+#define HW_ERN_47025_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000010))
+
+#define FIX_HW_BRN_50767_POS                                            (5U)
+#define FIX_HW_BRN_50767_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000020))
+
+#define HW_ERN_57596_POS                                                (6U)
+#define HW_ERN_57596_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000040))
+
+#define FIX_HW_BRN_60084_POS                                            (7U)
+#define FIX_HW_BRN_60084_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000080))
+
+#define HW_ERN_61389_POS                                                (8U)
+#define HW_ERN_61389_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000100))
+
+#define FIX_HW_BRN_61450_POS                                            (9U)
+#define FIX_HW_BRN_61450_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000200))
+
+#define FIX_HW_BRN_63142_POS                                            (10U)
+#define FIX_HW_BRN_63142_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000400))
+
+#define FIX_HW_BRN_63553_POS                                            (11U)
+#define FIX_HW_BRN_63553_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000800))
+
+#define FIX_HW_BRN_64502_POS                                            (12U)
+#define FIX_HW_BRN_64502_BIT_MASK                                       (IMG_UINT64_C(0x0000000000001000))
+
+#define FIX_HW_BRN_65101_POS                                            (13U)
+#define FIX_HW_BRN_65101_BIT_MASK                                       (IMG_UINT64_C(0x0000000000002000))
+
+#define FIX_HW_BRN_65273_POS                                            (14U)
+#define FIX_HW_BRN_65273_BIT_MASK                                       (IMG_UINT64_C(0x0000000000004000))
+
+#define HW_ERN_66622_POS                                                (15U)
+#define HW_ERN_66622_BIT_MASK                                           (IMG_UINT64_C(0x0000000000008000))
+
+#define FIX_HW_BRN_66927_POS                                            (16U)
+#define FIX_HW_BRN_66927_BIT_MASK                                       (IMG_UINT64_C(0x0000000000010000))
+
+#define FIX_HW_BRN_68186_POS                                            (17U)
+#define FIX_HW_BRN_68186_BIT_MASK                                       (IMG_UINT64_C(0x0000000000020000))
+
+#define FIX_HW_BRN_71242_POS                                            (18U)
+#define FIX_HW_BRN_71242_BIT_MASK                                       (IMG_UINT64_C(0x0000000000040000))
+
+#define FIX_HW_BRN_71317_POS                                            (19U)
+#define FIX_HW_BRN_71317_BIT_MASK                                       (IMG_UINT64_C(0x0000000000080000))
+
+/* Macro used for padding the unavailable values for features with values */
+#define RGX_FEATURE_VALUE_INVALID	(0xFFFFFFFEU)
+
+/* Macro used for marking a feature with value as disabled for a specific bvnc */
+#define RGX_FEATURE_VALUE_DISABLED	(0xFFFFFFFFU)
+
+#endif /* RGX_BVNC_DEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_table_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_table_km.h
new file mode 100644
index 000000000000..d10a0a1ab22d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_table_km.h
@@ -0,0 +1,458 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_table_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_TABLE_KM_H
+#define RGX_BVNC_TABLE_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+
+#ifndef RGXBVNC_C
+#error "This file should only be included from rgxbvnc.c"
+#endif
+
+#if defined(RGX_BVNC_TABLE_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_table_um.h"
+#endif
+
+
+/******************************************************************************
+ * Arrays for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_ECC_RAMS_values[RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values[RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_values[RGX_FEATURE_FBCDC_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, 50, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ALGORITHM_values[RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 50, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values[RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 7, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_LAYOUT_MARS_values[RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_values[RGX_FEATURE_META_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, LTP217, LTP218, MTP218, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_SIZE_values[RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_CLUSTERS_values[RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values[RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 6, 7, 8, 12, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_MEMBUS_values[RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_OSIDS_values[RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_RASTER_PIPES_values[RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values[RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, 36, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_values[RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_BANKS_values[RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values[RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 512, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values[RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 8, 16, 64, 128, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TB_GPU_COUNT_values[RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TFBC_VERSION_values[RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 10, 11, 20, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_X_values[RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_Y_values[RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values[RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XE_ARCHITECTURE_values[RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_values[RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 19, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_MAX_SLAVES_values[RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_REGISTER_BROADCAST_values[RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+
+/******************************************************************************
+ * Table contains pointers to each feature value array for features that have
+ * values.
+ * Indexed using enum RGX_FEATURE_WITH_VALUE_INDEX from rgx_bvnc_defs_km.h
+ *****************************************************************************/
+
+static const void * const gaFeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX] = {
+	aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_ECC_RAMS_values,
+	aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values,
+	aui16_RGX_FEATURE_FBCDC_values,
+	aui16_RGX_FEATURE_FBCDC_ALGORITHM_values,
+	aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_LAYOUT_MARS_values,
+	aui16_RGX_FEATURE_META_values,
+	aui16_RGX_FEATURE_META_COREMEM_SIZE_values,
+	aui16_RGX_FEATURE_NUM_CLUSTERS_values,
+	aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values,
+	aui16_RGX_FEATURE_NUM_MEMBUS_values,
+	aui16_RGX_FEATURE_NUM_OSIDS_values,
+	aui16_RGX_FEATURE_NUM_RASTER_PIPES_values,
+	aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values,
+	aui16_RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_values,
+	aui16_RGX_FEATURE_SLC_BANKS_values,
+	aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values,
+	aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values,
+	aui16_RGX_FEATURE_TB_GPU_COUNT_values,
+	aui16_RGX_FEATURE_TFBC_VERSION_values,
+	aui16_RGX_FEATURE_TILE_SIZE_X_values,
+	aui16_RGX_FEATURE_TILE_SIZE_Y_values,
+	aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values,
+	aui16_RGX_FEATURE_XE_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_values,
+	aui16_RGX_FEATURE_XPU_MAX_SLAVES_values,
+	aui16_RGX_FEATURE_XPU_REGISTER_BROADCAST_values,
+};
+
+
+/******************************************************************************
+ * Array containing the lengths of the arrays containing the values.
+ * Used for indexing the aui16_<FEATURE>_values defined upwards
+ *****************************************************************************/
+
+
+static const IMG_UINT16 gaFeaturesValuesMaxIndexes[] = {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX,
+	RGX_FEATURE_META_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX,
+};
+
+#define RGX_FEATURE_VALUE_TYPE_UINT16 (0x0000U)
+#define RGX_FEATURE_VALUE_TYPE_UINT32 (0x8000U)
+#define RGX_FEATURE_TYPE_BIT_SHIFT 14
+
+/******************************************************************************
+ * Bit-positions for features with values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16FeaturesWithValuesBitPositions[] = {
+	(0U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_POS */
+	(2U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_ECC_RAMS_POS */
+	(4U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FAULT_DECODE_VERSION_POS */
+	(6U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_POS */
+	(9U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ALGORITHM_POS */
+	(12U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ARCHITECTURE_POS */
+	(15U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_POS */
+	(17U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_POS */
+	(19U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_LAYOUT_MARS_POS */
+	(21U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_POS */
+	(24U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_SIZE_POS */
+	(26U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_CLUSTERS_POS */
+	(29U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_IPP_PIPES_POS */
+	(33U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_MEMBUS_POS */
+	(35U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_OSIDS_POS */
+	(37U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_RASTER_PIPES_POS */
+	(39U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PHYS_BUS_WIDTH_POS */
+	(42U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_POS */
+	(44U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_BANKS_POS */
+	(47U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_POS */
+	(49U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_POS */
+	(52U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TB_GPU_COUNT_POS */
+	(54U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TFBC_VERSION_POS */
+	(57U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_X_POS */
+	(59U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_Y_POS */
+	(61U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_POS */
+	(64U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XE_ARCHITECTURE_POS */
+	(66U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_POS */
+	(68U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_MAX_SLAVES_POS */
+	(70U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_REGISTER_BROADCAST_POS */
+};
+
+
+/******************************************************************************
+ * Bit-masks for features with values
+ *****************************************************************************/
+
+static const IMG_UINT64 aui64FeaturesWithValuesBitMasks[] = {
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_ECC_RAMS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_FAULT_DECODE_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000001C0)), /* RGX_FEATURE_FBCDC_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000E00)), /* RGX_FEATURE_FBCDC_ALGORITHM_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000007000)), /* RGX_FEATURE_FBCDC_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000018000)), /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000060000)), /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000180000)), /* RGX_FEATURE_LAYOUT_MARS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000E00000)), /* RGX_FEATURE_META_BIT_MASK */
+	(IMG_UINT64_C(0x0000000003000000)), /* RGX_FEATURE_META_COREMEM_SIZE_BIT_MASK */
+	(IMG_UINT64_C(0x000000001C000000)), /* RGX_FEATURE_NUM_CLUSTERS_BIT_MASK */
+	(IMG_UINT64_C(0x00000001E0000000)), /* RGX_FEATURE_NUM_ISP_IPP_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000000600000000)), /* RGX_FEATURE_NUM_MEMBUS_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_NUM_OSIDS_BIT_MASK */
+	(IMG_UINT64_C(0x0000006000000000)), /* RGX_FEATURE_NUM_RASTER_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000038000000000)), /* RGX_FEATURE_PHYS_BUS_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x00000C0000000000)), /* RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000700000000000)), /* RGX_FEATURE_SLC_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0001800000000000)), /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x000E000000000000)), /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_BIT_MASK */
+	(IMG_UINT64_C(0x0030000000000000)), /* RGX_FEATURE_TB_GPU_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x01C0000000000000)), /* RGX_FEATURE_TFBC_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0600000000000000)), /* RGX_FEATURE_TILE_SIZE_X_BIT_MASK */
+	(IMG_UINT64_C(0x1800000000000000)), /* RGX_FEATURE_TILE_SIZE_Y_BIT_MASK */
+	(IMG_UINT64_C(0x6000000000000000)), /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_XE_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_XPU_MAX_SLAVES_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000000C0)), /* RGX_FEATURE_XPU_REGISTER_BROADCAST_BIT_MASK */
+};
+
+
+/******************************************************************************
+ * Table mapping bitmasks for features and features with values
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaFeatures[][4]=
+{
+	{ IMG_UINT64_C(0x0001000000040005), IMG_UINT64_C(0x0000000400010414), IMG_UINT64_C(0x340a81a06d6a9211), IMG_UINT64_C(0x0000000000000000) },	/* 1.0.4.5 */
+	{ IMG_UINT64_C(0x0001000000040013), IMG_UINT64_C(0x0000000400010415), IMG_UINT64_C(0x340a81a06d6a9211), IMG_UINT64_C(0x0000000000000000) },	/* 1.0.4.19 */
+	{ IMG_UINT64_C(0x0004000000020033), IMG_UINT64_C(0x0000042c0803045f), IMG_UINT64_C(0x340a81a0ea4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.2.51 */
+	{ IMG_UINT64_C(0x000400000002003a), IMG_UINT64_C(0x0000042c0803065f), IMG_UINT64_C(0x340a81b0ea4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.2.58 */
+	{ IMG_UINT64_C(0x0004000000040037), IMG_UINT64_C(0x0000042c0803045e), IMG_UINT64_C(0x340a81a0ee4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.4.55 */
+	{ IMG_UINT64_C(0x000400000006003e), IMG_UINT64_C(0x0000042c0803065f), IMG_UINT64_C(0x340ab1d0f24aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.6.62 */
+	{ IMG_UINT64_C(0x000500000001002e), IMG_UINT64_C(0x0000000008110445), IMG_UINT64_C(0x340691a0252a9411), IMG_UINT64_C(0x0000000000000001) },	/* 5.0.1.46 */
+	{ IMG_UINT64_C(0x0006000000040023), IMG_UINT64_C(0x0000042c0803045f), IMG_UINT64_C(0x340a81a0ee4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 6.0.4.35 */
+	{ IMG_UINT64_C(0x000f000000010040), IMG_UINT64_C(0x0000000008110645), IMG_UINT64_C(0x340891b0452a9411), IMG_UINT64_C(0x0000000000000001) },	/* 15.0.1.64 */
+	{ IMG_UINT64_C(0x0016000000150010), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0494b044080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.21.16 */
+	{ IMG_UINT64_C(0x0016000000360019), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b064080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.25 */
+	{ IMG_UINT64_C(0x001600000036001e), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b084080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.30 */
+	{ IMG_UINT64_C(0x0016000000360026), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a08953084080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.38 */
+	{ IMG_UINT64_C(0x001600000036014a), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b0840a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.330 */
+	{ IMG_UINT64_C(0x0016000000680012), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a089530c4080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.104.18 */
+	{ IMG_UINT64_C(0x00160000006800da), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a089530c40a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.104.218 */
+	{ IMG_UINT64_C(0x0016000000d0013e), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a08a551080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.208.318 */
+	{ IMG_UINT64_C(0x00180000003600cc), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a089930840a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.54.204 */
+	{ IMG_UINT64_C(0x00180000006801f8), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a089930a40a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.104.504 */
+	{ IMG_UINT64_C(0x0018000000d001f8), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a0aa951080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.208.504 */
+	{ IMG_UINT64_C(0x0018000000d001f9), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a0aa951080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.208.505 */
+	{ IMG_UINT64_C(0x001d0000003400ca), IMG_UINT64_C(0x000002183951feb5), IMG_UINT64_C(0x2a069930440a9891), IMG_UINT64_C(0x0000000000000001) },	/* 29.0.52.202 */
+	{ IMG_UINT64_C(0x001d0000006c00d0), IMG_UINT64_C(0x000002183951feb5), IMG_UINT64_C(0x2a0aa950e80a9891), IMG_UINT64_C(0x0000000000000001) },	/* 29.0.108.208 */
+	{ IMG_UINT64_C(0x00210000000b0003), IMG_UINT64_C(0x000002001551ba25), IMG_UINT64_C(0x2a02992824080011), IMG_UINT64_C(0x0000000000000001) },	/* 33.0.11.3 */
+	{ IMG_UINT64_C(0x0021000000160001), IMG_UINT64_C(0x000002181955be35), IMG_UINT64_C(0x2a06993044080011), IMG_UINT64_C(0x0000000000000001) },	/* 33.0.22.1 */
+	{ IMG_UINT64_C(0x00240000003400b6), IMG_UINT64_C(0x000002001d51bf35), IMG_UINT64_C(0x2a4699304412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.52.182 */
+	{ IMG_UINT64_C(0x0024000000360067), IMG_UINT64_C(0x000002181d51b735), IMG_UINT64_C(0x2a469930840abad1), IMG_UINT64_C(0x0000000000000001) },	/* 36.0.54.103 */
+	{ IMG_UINT64_C(0x00240000003600b6), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a4699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.182 */
+	{ IMG_UINT64_C(0x00240000003600b7), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a4699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.183 */
+	{ IMG_UINT64_C(0x0024000000360118), IMG_UINT64_C(0x00000219dd51bf35), IMG_UINT64_C(0x2a8699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.280 */
+	{ IMG_UINT64_C(0x00240000006800b6), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a469930a412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.182 */
+	{ IMG_UINT64_C(0x00240000006800b7), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a469930a412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.183 */
+	{ IMG_UINT64_C(0x002400000068031c), IMG_UINT64_C(0x0000039a1d79af35), IMG_UINT64_C(0x2a469930a412bad9), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.796 */
+	{ IMG_UINT64_C(0x002e000000660185), IMG_UINT64_C(0x00000241dd51bf35), IMG_UINT64_C(0x2ad899324412bad5), IMG_UINT64_C(0x0000000000000055) },	/* 46.0.102.389 */
+	{ IMG_UINT64_C(0x002e000000cc0186), IMG_UINT64_C(0x00000259dd51bf35), IMG_UINT64_C(0x2ad89932c412bad5), IMG_UINT64_C(0x0000000000000055) },	/* 46.0.204.390 */
+};
+
+/******************************************************************************
+ * Table mapping bitmasks for ERNs/BRNs
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaErnsBrns[][2]=
+{
+	{ IMG_UINT64_C(0x0001002700040013), IMG_UINT64_C(0x0000000000000005) },	/* 1.39.4.19 */
+	{ IMG_UINT64_C(0x0001005200040005), IMG_UINT64_C(0x0000000000000000) },	/* 1.82.4.5 */
+	{ IMG_UINT64_C(0x0004001f00040037), IMG_UINT64_C(0x000000000000042a) },	/* 4.31.4.55 */
+	{ IMG_UINT64_C(0x0004002800020033), IMG_UINT64_C(0x000000000000042a) },	/* 4.40.2.51 */
+	{ IMG_UINT64_C(0x0004002b0006003e), IMG_UINT64_C(0x000000000000142a) },	/* 4.43.6.62 */
+	{ IMG_UINT64_C(0x0004002d0002003a), IMG_UINT64_C(0x000000000000140a) },	/* 4.45.2.58 */
+	{ IMG_UINT64_C(0x0004002e0006003e), IMG_UINT64_C(0x000000000000142a) },	/* 4.46.6.62 */
+	{ IMG_UINT64_C(0x000500090001002e), IMG_UINT64_C(0x0000000000000001) },	/* 5.9.1.46 */
+	{ IMG_UINT64_C(0x0006002200040023), IMG_UINT64_C(0x000000000000040a) },	/* 6.34.4.35 */
+	{ IMG_UINT64_C(0x000f000500010040), IMG_UINT64_C(0x0000000000001008) },	/* 15.5.1.64 */
+	{ IMG_UINT64_C(0x0016001e00360019), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.30.54.25 */
+	{ IMG_UINT64_C(0x001600280036001e), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.40.54.30 */
+	{ IMG_UINT64_C(0x0016002e0036014a), IMG_UINT64_C(0x0000000000087a8a) },	/* 22.46.54.330 */
+	{ IMG_UINT64_C(0x0016003100150010), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.49.21.16 */
+	{ IMG_UINT64_C(0x001600430036001e), IMG_UINT64_C(0x00000000000859c8) },	/* 22.67.54.30 */
+	{ IMG_UINT64_C(0x001600440036001e), IMG_UINT64_C(0x0000000000085948) },	/* 22.68.54.30 */
+	{ IMG_UINT64_C(0x00160056006800da), IMG_UINT64_C(0x0000000000083908) },	/* 22.86.104.218 */
+	{ IMG_UINT64_C(0x0016005700680012), IMG_UINT64_C(0x0000000000081948) },	/* 22.87.104.18 */
+	{ IMG_UINT64_C(0x0016006600360026), IMG_UINT64_C(0x0000000000081948) },	/* 22.102.54.38 */
+	{ IMG_UINT64_C(0x0016006800d0013e), IMG_UINT64_C(0x000000000008390a) },	/* 22.104.208.318 */
+	{ IMG_UINT64_C(0x0018003200d001f8), IMG_UINT64_C(0x000000000008884a) },	/* 24.50.208.504 */
+	{ IMG_UINT64_C(0x0018003800d001f9), IMG_UINT64_C(0x000000000008884a) },	/* 24.56.208.505 */
+	{ IMG_UINT64_C(0x00180042003600cc), IMG_UINT64_C(0x000000000008884a) },	/* 24.66.54.204 */
+	{ IMG_UINT64_C(0x00180043006801f8), IMG_UINT64_C(0x000000000008884a) },	/* 24.67.104.504 */
+	{ IMG_UINT64_C(0x001d000e006c00d0), IMG_UINT64_C(0x00000000000a885a) },	/* 29.14.108.208 */
+	{ IMG_UINT64_C(0x001d0013003400ca), IMG_UINT64_C(0x00000000000a885a) },	/* 29.19.52.202 */
+	{ IMG_UINT64_C(0x0021000800160001), IMG_UINT64_C(0x000000000008085a) },	/* 33.8.22.1 */
+	{ IMG_UINT64_C(0x0021000f000b0003), IMG_UINT64_C(0x00000000000c085a) },	/* 33.15.11.3 */
+	{ IMG_UINT64_C(0x0024001d003400b6), IMG_UINT64_C(0x000000000008085a) },	/* 36.29.52.182 */
+	{ IMG_UINT64_C(0x00240032003600b6), IMG_UINT64_C(0x00000000000c085a) },	/* 36.50.54.182 */
+	{ IMG_UINT64_C(0x00240034006800b6), IMG_UINT64_C(0x00000000000c085a) },	/* 36.52.104.182 */
+	{ IMG_UINT64_C(0x002400350068031c), IMG_UINT64_C(0x000000000008005a) },	/* 36.53.104.796 */
+	{ IMG_UINT64_C(0x00240036003600b7), IMG_UINT64_C(0x000000000008085a) },	/* 36.54.54.183 */
+	{ IMG_UINT64_C(0x0024003700360067), IMG_UINT64_C(0x000000000008085a) },	/* 36.55.54.103 */
+	{ IMG_UINT64_C(0x00240038006800b7), IMG_UINT64_C(0x000000000008085a) },	/* 36.56.104.183 */
+	{ IMG_UINT64_C(0x0024003c00360118), IMG_UINT64_C(0x000000000008085a) },	/* 36.60.54.280 */
+	{ IMG_UINT64_C(0x002e004800660185), IMG_UINT64_C(0x000000000009085a) },	/* 46.72.102.389 */
+	{ IMG_UINT64_C(0x002e005e00cc0186), IMG_UINT64_C(0x000000000009085a) },	/* 46.94.204.390 */
+};
+
+#if defined(DEBUG)
+
+#define FEATURE_NO_VALUES_NAMES_MAX_IDX (43U)
+
+static const IMG_CHAR * const gaszFeaturesNoValuesNames[FEATURE_NO_VALUES_NAMES_MAX_IDX] =
+{
+	"AXI_ACELITE",
+	"CLUSTER_GROUPING",
+	"COMPUTE",
+	"COMPUTE_MORTON_CAPABLE",
+	"COMPUTE_OVERLAP",
+	"COREID_PER_OS",
+	"DYNAMIC_DUST_POWER",
+	"FASTRENDER_DM",
+	"GPU_MULTICORE_SUPPORT",
+	"GPU_VIRTUALISATION",
+	"GS_RTA_SUPPORT",
+	"IRQ_PER_OS",
+	"MIPS",
+	"PBE2_IN_XE",
+	"PBE_CHECKSUM_2D",
+	"PBVNC_COREID_REG",
+	"PERFBUS",
+	"PERF_COUNTER_BATCH",
+	"PM_MMU_VFP",
+	"RISCV_FW_PROCESSOR",
+	"ROGUEXE",
+	"SAFETY_IRQ",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT_V1",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT_V2",
+	"SINGLE_BIF",
+	"SLC_SIZE_CONFIGURABLE",
+	"SOC_TIMER",
+	"SYS_BUS_SECURE_RESET",
+	"TDM_PDS_CHECKSUM",
+	"TFBC_DELTA_CORRELATION",
+	"TFBC_LOSSY_37_PERCENT",
+	"TFBC_NATIVE_YUV10",
+	"TILE_REGION_PROTECTION",
+	"TLA",
+	"TPU_CEM_DATAMASTER_GLOBAL_REGISTERS",
+	"TPU_DM_GLOBAL_REGISTERS",
+	"TPU_FILTERING_MODE_CONTROL",
+	"VOLCANIC_TB",
+	"WATCHDOG_TIMER",
+	"WORKGROUP_PROTECTION",
+	"XE_MEMORY_HIERARCHY",
+	"XT_TOP_INFRASTRUCTURE",
+};
+
+#define ERNSBRNS_IDS_MAX_IDX (20U)
+
+static const IMG_UINT32 gaui64ErnsBrnsIDs[ERNSBRNS_IDS_MAX_IDX] =
+{
+	38344,
+	42290,
+	42321,
+	42606,
+	47025,
+	50767,
+	57596,
+	60084,
+	61389,
+	61450,
+	63142,
+	63553,
+	64502,
+	65101,
+	65273,
+	66622,
+	66927,
+	68186,
+	71242,
+	71317,
+};
+
+#endif /* defined(DEBUG) */
+#endif /* RGX_BVNC_TABLE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_cr_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_cr_defs_km.h
new file mode 100644
index 000000000000..1326877b50be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_cr_defs_km.h
@@ -0,0 +1,8472 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_cr_defs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGX_CR_DEFS_KM_H
+#define RGX_CR_DEFS_KM_H
+
+#if !defined(IMG_EXPLICIT_INCLUDE_HWDEFS)
+#error This file may only be included if explicitly defined
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGX_CR_DEFS_KM_REVISION 1
+
+/*
+    Register RGX_CR_RASTERISATION_INDIRECT
+*/
+#define RGX_CR_RASTERISATION_INDIRECT                                               (0x8238U)
+#define RGX_CR_RASTERISATION_INDIRECT_MASKFULL                                      (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_RASTERISATION_INDIRECT_ADDRESS_SHIFT                                 (0U)
+#define RGX_CR_RASTERISATION_INDIRECT_ADDRESS_CLRMSK                                (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_USC_INDIRECT
+*/
+#define RGX_CR_USC_INDIRECT                                                         (0x8000U)
+#define RGX_CR_USC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_USC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_USC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_PBE_INDIRECT
+*/
+#define RGX_CR_PBE_INDIRECT                                                         (0x83E0U)
+#define RGX_CR_PBE_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_PBE_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_PBE_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_PBE_PERF_INDIRECT
+*/
+#define RGX_CR_PBE_PERF_INDIRECT                                                    (0x83D8U)
+#define RGX_CR_PBE_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_PBE_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_PBE_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_TPU_PERF_INDIRECT
+*/
+#define RGX_CR_TPU_PERF_INDIRECT                                                    (0x83F0U)
+#define RGX_CR_TPU_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TPU_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_TPU_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_INDIRECT
+*/
+#define RGX_CR_RASTERISATION_PERF_INDIRECT                                          (0x8318U)
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_ADDRESS_SHIFT                            (0U)
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_ADDRESS_CLRMSK                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_INDIRECT
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT                                             (0x8028U)
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_ADDRESS_SHIFT                               (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_ADDRESS_CLRMSK                              (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_USC_PERF_INDIRECT
+*/
+#define RGX_CR_USC_PERF_INDIRECT                                                    (0x8030U)
+#define RGX_CR_USC_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_USC_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_USC_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_INDIRECT
+*/
+#define RGX_CR_BLACKPEARL_INDIRECT                                                  (0x8388U)
+#define RGX_CR_BLACKPEARL_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BLACKPEARL_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BLACKPEARL_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_INDIRECT
+*/
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT                                             (0x83F8U)
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_ADDRESS_SHIFT                               (0U)
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_ADDRESS_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_TEXAS3_PERF_INDIRECT
+*/
+#define RGX_CR_TEXAS3_PERF_INDIRECT                                                 (0x83D0U)
+#define RGX_CR_TEXAS3_PERF_INDIRECT_MASKFULL                                        (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TEXAS3_PERF_INDIRECT_ADDRESS_SHIFT                                   (0U)
+#define RGX_CR_TEXAS3_PERF_INDIRECT_ADDRESS_CLRMSK                                  (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_INDIRECT
+*/
+#define RGX_CR_TEXAS_PERF_INDIRECT                                                  (0x8288U)
+#define RGX_CR_TEXAS_PERF_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_TEXAS_PERF_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_TEXAS_PERF_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_BX_TU_PERF_INDIRECT
+*/
+#define RGX_CR_BX_TU_PERF_INDIRECT                                                  (0xC900U)
+#define RGX_CR_BX_TU_PERF_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BX_TU_PERF_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BX_TU_PERF_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_CLK_CTRL
+*/
+#define RGX_CR_CLK_CTRL                                                             (0x0000U)
+#define RGX_CR_CLK_CTRL__PBE2_XE__MASKFULL                                          (IMG_UINT64_C(0xFFFFFF003F3FFFFF))
+#define RGX_CR_CLK_CTRL__S7_INFRA__MASKFULL                                         (IMG_UINT64_C(0xCFCF03000F3F3F0F))
+#define RGX_CR_CLK_CTRL_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFF003F3FFFFF))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_SHIFT                                             (62U)
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_CLRMSK                                            (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_ON                                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_AUTO                                              (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__ON                                     (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__AUTO                                   (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__OFF                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__ON                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__AUTO                                    (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_SHIFT                                                   (60U)
+#define RGX_CR_CLK_CTRL_IPP_CLRMSK                                                  (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_IPP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_ON                                                      (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_AUTO                                                    (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__ON                                           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__ON                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_FBC_SHIFT                                                   (58U)
+#define RGX_CR_CLK_CTRL_FBC_CLRMSK                                                  (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FBC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC_ON                                                      (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC_AUTO                                                    (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_SHIFT                                                  (56U)
+#define RGX_CR_CLK_CTRL_FBDC_CLRMSK                                                 (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FBDC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_ON                                                     (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_AUTO                                                   (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__ON                                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__AUTO                                        (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__ON                                           (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__AUTO                                         (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_SHIFT                                            (54U)
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_CLRMSK                                           (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_ON                                               (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_AUTO                                             (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__ON                                    (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__AUTO                                  (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__ON                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__AUTO                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_USCS_SHIFT                                                  (52U)
+#define RGX_CR_CLK_CTRL_USCS_CLRMSK                                                 (IMG_UINT64_C(0xFFCFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_USCS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS_ON                                                     (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS_AUTO                                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__ON                                          (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__AUTO                                        (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__ON                                           (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__AUTO                                         (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_PBE_SHIFT                                                   (50U)
+#define RGX_CR_CLK_CTRL_PBE_CLRMSK                                                  (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_PBE_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE_ON                                                      (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE_AUTO                                                    (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__ON                                           (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__ON                                            (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_SHIFT                                                (48U)
+#define RGX_CR_CLK_CTRL_MCU_L1_CLRMSK                                               (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_MCU_L1_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_ON                                                   (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_AUTO                                                 (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__ON                                        (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__AUTO                                      (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__ON                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__AUTO                                       (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_CDM_SHIFT                                                   (46U)
+#define RGX_CR_CLK_CTRL_CDM_CLRMSK                                                  (IMG_UINT64_C(0xFFFF3FFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_CDM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM_ON                                                      (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM_AUTO                                                    (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_SHIFT                                              (44U)
+#define RGX_CR_CLK_CTRL_SIDEKICK_CLRMSK                                             (IMG_UINT64_C(0xFFFFCFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_SIDEKICK_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_ON                                                 (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_AUTO                                               (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__OFF                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__ON                                      (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__AUTO                                    (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__ON                                       (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__AUTO                                     (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_SHIFT                                          (42U)
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_CLRMSK                                         (IMG_UINT64_C(0xFFFFF3FFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_ON                                             (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_AUTO                                           (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__ON                                  (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__AUTO                                (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__OFF                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__ON                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__AUTO                                 (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SHIFT                                                   (40U)
+#define RGX_CR_CLK_CTRL_BIF_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_ON                                                      (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF_AUTO                                                    (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_SHIFT                                         (28U)
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_ON                                            (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_AUTO                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__OFF                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__ON                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__AUTO                               (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__ON                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__AUTO                                (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_SHIFT                                                (26U)
+#define RGX_CR_CLK_CTRL_MCU_L0_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL_MCU_L0_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_ON                                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_AUTO                                                 (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__ON                                        (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__AUTO                                      (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__ON                                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__AUTO                                       (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_TPU_SHIFT                                                   (24U)
+#define RGX_CR_CLK_CTRL_TPU_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL_TPU_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_ON                                                      (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU_AUTO                                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_USC_SHIFT                                                   (20U)
+#define RGX_CR_CLK_CTRL_USC_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL_USC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC_ON                                                      (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC_AUTO                                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_TLA_SHIFT                                                   (18U)
+#define RGX_CR_CLK_CTRL_TLA_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL_TLA_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA_ON                                                      (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA_AUTO                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_SLC_SHIFT                                                   (16U)
+#define RGX_CR_CLK_CTRL_SLC_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL_SLC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC_ON                                                      (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC_AUTO                                                    (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_UVS_SHIFT                                                   (14U)
+#define RGX_CR_CLK_CTRL_UVS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL_UVS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS_ON                                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS_AUTO                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_PDS_SHIFT                                                   (12U)
+#define RGX_CR_CLK_CTRL_PDS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL_PDS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS_ON                                                      (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS_AUTO                                                    (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_VDM_SHIFT                                                   (10U)
+#define RGX_CR_CLK_CTRL_VDM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL_VDM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM_ON                                                      (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM_AUTO                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_PM_SHIFT                                                    (8U)
+#define RGX_CR_CLK_CTRL_PM_CLRMSK                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL_PM_OFF                                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM_ON                                                       (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM_AUTO                                                     (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__ON                                            (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__AUTO                                          (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__ON                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__AUTO                                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_GPP_SHIFT                                                   (6U)
+#define RGX_CR_CLK_CTRL_GPP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL_GPP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP_ON                                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP_AUTO                                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_TE_SHIFT                                                    (4U)
+#define RGX_CR_CLK_CTRL_TE_CLRMSK                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL_TE_OFF                                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE_ON                                                       (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE_AUTO                                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__ON                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__AUTO                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__ON                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__AUTO                                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TSP_SHIFT                                                   (2U)
+#define RGX_CR_CLK_CTRL_TSP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL_TSP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP_ON                                                      (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP_AUTO                                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_ISP_SHIFT                                                   (0U)
+#define RGX_CR_CLK_CTRL_ISP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL_ISP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP_ON                                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP_AUTO                                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS
+*/
+#define RGX_CR_CLK_STATUS                                                           (0x0008U)
+#define RGX_CR_CLK_STATUS__PBE2_XE__MASKFULL                                        (IMG_UINT64_C(0x00000001FFF077FF))
+#define RGX_CR_CLK_STATUS__S7_INFRA__MASKFULL                                       (IMG_UINT64_C(0x00000001B3101773))
+#define RGX_CR_CLK_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x00000001FFF077FF))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_SHIFT                                            (32U)
+#define RGX_CR_CLK_STATUS_MCU_FBTC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_RUNNING                                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__S7_INFRA__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__S7_INFRA__RUNNING                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__PBE2_XE__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__PBE2_XE__RUNNING                                (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_SHIFT                                           (31U)
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_RUNNING                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__S7_INFRA__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__S7_INFRA__RUNNING                              (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__PBE2_XE__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__PBE2_XE__RUNNING                               (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_IPP_SHIFT                                                 (30U)
+#define RGX_CR_CLK_STATUS_IPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CLK_STATUS_IPP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP_RUNNING                                               (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_IPP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_IPP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_FBC_SHIFT                                                 (29U)
+#define RGX_CR_CLK_STATUS_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS_FBC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC_RUNNING                                               (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBDC_SHIFT                                                (28U)
+#define RGX_CR_CLK_STATUS_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS_FBDC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC_RUNNING                                              (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FBDC__S7_INFRA__GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC__S7_INFRA__RUNNING                                   (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FBDC__PBE2_XE__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC__PBE2_XE__RUNNING                                    (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_SHIFT                                          (27U)
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_RUNNING                                        (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__S7_INFRA__GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__S7_INFRA__RUNNING                             (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__PBE2_XE__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__PBE2_XE__RUNNING                              (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_USCS_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS_USCS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS_USCS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_USCS__S7_INFRA__GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS__S7_INFRA__RUNNING                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_USCS__PBE2_XE__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS__PBE2_XE__RUNNING                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_PBE_SHIFT                                                 (25U)
+#define RGX_CR_CLK_STATUS_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS_PBE_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE_RUNNING                                               (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_PBE__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_PBE__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_MCU_L1_SHIFT                                              (24U)
+#define RGX_CR_CLK_STATUS_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS_MCU_L1_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1_RUNNING                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__S7_INFRA__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__S7_INFRA__RUNNING                                 (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__PBE2_XE__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__PBE2_XE__RUNNING                                  (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_CDM_SHIFT                                                 (23U)
+#define RGX_CR_CLK_STATUS_CDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_CLK_STATUS_CDM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM_RUNNING                                               (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_CDM__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_CDM__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_SIDEKICK_SHIFT                                            (22U)
+#define RGX_CR_CLK_STATUS_SIDEKICK_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CLK_STATUS_SIDEKICK_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK_RUNNING                                          (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__S7_INFRA__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__S7_INFRA__RUNNING                               (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__PBE2_XE__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__PBE2_XE__RUNNING                                (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_SHIFT                                        (21U)
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_RUNNING                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__S7_INFRA__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__S7_INFRA__RUNNING                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__PBE2_XE__GATED                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__PBE2_XE__RUNNING                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SHIFT                                                 (20U)
+#define RGX_CR_CLK_STATUS_BIF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_RUNNING                                               (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_BIF__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_BIF__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_SHIFT                                       (14U)
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_RUNNING                                     (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__S7_INFRA__GATED                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__S7_INFRA__RUNNING                          (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__PBE2_XE__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__PBE2_XE__RUNNING                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_MCU_L0_SHIFT                                              (13U)
+#define RGX_CR_CLK_STATUS_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS_MCU_L0_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0_RUNNING                                            (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_MCU_L0__S7_INFRA__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0__S7_INFRA__RUNNING                                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_MCU_L0__PBE2_XE__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0__PBE2_XE__RUNNING                                  (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_TPU_SHIFT                                                 (12U)
+#define RGX_CR_CLK_STATUS_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS_TPU_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_RUNNING                                               (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_TPU__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_TPU__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_USC_SHIFT                                                 (10U)
+#define RGX_CR_CLK_STATUS_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS_USC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC_RUNNING                                               (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_USC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_USC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_TLA_SHIFT                                                 (9U)
+#define RGX_CR_CLK_STATUS_TLA_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS_TLA_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA_RUNNING                                               (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_TLA__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_TLA__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_SLC_SHIFT                                                 (8U)
+#define RGX_CR_CLK_STATUS_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS_SLC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC_RUNNING                                               (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_SLC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_SLC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_UVS_SHIFT                                                 (7U)
+#define RGX_CR_CLK_STATUS_UVS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS_UVS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS_RUNNING                                               (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_UVS__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_UVS__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_PDS_SHIFT                                                 (6U)
+#define RGX_CR_CLK_STATUS_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS_PDS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS_RUNNING                                               (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_PDS__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_PDS__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_VDM_SHIFT                                                 (5U)
+#define RGX_CR_CLK_STATUS_VDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS_VDM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM_RUNNING                                               (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_VDM__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_VDM__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_PM_SHIFT                                                  (4U)
+#define RGX_CR_CLK_STATUS_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS_PM_GATED                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM_RUNNING                                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_PM__S7_INFRA__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM__S7_INFRA__RUNNING                                     (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_PM__PBE2_XE__GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM__PBE2_XE__RUNNING                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_GPP_SHIFT                                                 (3U)
+#define RGX_CR_CLK_STATUS_GPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS_GPP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP_RUNNING                                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_GPP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_GPP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_TE_SHIFT                                                  (2U)
+#define RGX_CR_CLK_STATUS_TE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS_TE_GATED                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE_RUNNING                                                (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TE__S7_INFRA__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE__S7_INFRA__RUNNING                                     (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TE__PBE2_XE__GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE__PBE2_XE__RUNNING                                      (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TSP_SHIFT                                                 (1U)
+#define RGX_CR_CLK_STATUS_TSP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS_TSP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP_RUNNING                                               (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_TSP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_TSP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_ISP_SHIFT                                                 (0U)
+#define RGX_CR_CLK_STATUS_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS_ISP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP_RUNNING                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS_ISP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS_ISP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID__PBVNC                                                       (0x0020U)
+#define RGX_CR_CORE_ID__PBVNC__MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__BRANCH_ID_SHIFT                                      (48U)
+#define RGX_CR_CORE_ID__PBVNC__BRANCH_ID_CLRMSK                                     (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__VERSION_ID_SHIFT                                     (32U)
+#define RGX_CR_CORE_ID__PBVNC__VERSION_ID_CLRMSK                                    (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_SHIFT                       (16U)
+#define RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_CORE_ID__PBVNC__CONFIG_ID_SHIFT                                      (0U)
+#define RGX_CR_CORE_ID__PBVNC__CONFIG_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID                                                              (0x0018U)
+#define RGX_CR_CORE_ID_MASKFULL                                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CORE_ID_ID_SHIFT                                                     (16U)
+#define RGX_CR_CORE_ID_ID_CLRMSK                                                    (0x0000FFFFU)
+#define RGX_CR_CORE_ID_CONFIG_SHIFT                                                 (0U)
+#define RGX_CR_CORE_ID_CONFIG_CLRMSK                                                (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_CORE_REVISION
+*/
+#define RGX_CR_CORE_REVISION                                                        (0x0020U)
+#define RGX_CR_CORE_REVISION_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CORE_REVISION_DESIGNER_SHIFT                                         (24U)
+#define RGX_CR_CORE_REVISION_DESIGNER_CLRMSK                                        (0x00FFFFFFU)
+#define RGX_CR_CORE_REVISION_MAJOR_SHIFT                                            (16U)
+#define RGX_CR_CORE_REVISION_MAJOR_CLRMSK                                           (0xFF00FFFFU)
+#define RGX_CR_CORE_REVISION_MINOR_SHIFT                                            (8U)
+#define RGX_CR_CORE_REVISION_MINOR_CLRMSK                                           (0xFFFF00FFU)
+#define RGX_CR_CORE_REVISION_MAINTENANCE_SHIFT                                      (0U)
+#define RGX_CR_CORE_REVISION_MAINTENANCE_CLRMSK                                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_DESIGNER_REV_FIELD1
+*/
+#define RGX_CR_DESIGNER_REV_FIELD1                                                  (0x0028U)
+#define RGX_CR_DESIGNER_REV_FIELD1_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_DESIGNER_REV_FIELD1_DESIGNER_REV_FIELD1_SHIFT                        (0U)
+#define RGX_CR_DESIGNER_REV_FIELD1_DESIGNER_REV_FIELD1_CLRMSK                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_DESIGNER_REV_FIELD2
+*/
+#define RGX_CR_DESIGNER_REV_FIELD2                                                  (0x0030U)
+#define RGX_CR_DESIGNER_REV_FIELD2_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_DESIGNER_REV_FIELD2_DESIGNER_REV_FIELD2_SHIFT                        (0U)
+#define RGX_CR_DESIGNER_REV_FIELD2_DESIGNER_REV_FIELD2_CLRMSK                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_CHANGESET_NUMBER
+*/
+#define RGX_CR_CHANGESET_NUMBER                                                     (0x0040U)
+#define RGX_CR_CHANGESET_NUMBER_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CHANGESET_NUMBER_CHANGESET_NUMBER_SHIFT                              (0U)
+#define RGX_CR_CHANGESET_NUMBER_CHANGESET_NUMBER_CLRMSK                             (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_GRAY
+*/
+#define RGX_CR_SOC_TIMER_GRAY                                                       (0x00E0U)
+#define RGX_CR_SOC_TIMER_GRAY_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_SHIFT                                           (0U)
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_CLRMSK                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_BINARY
+*/
+#define RGX_CR_SOC_TIMER_BINARY                                                     (0x00E8U)
+#define RGX_CR_SOC_TIMER_BINARY_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_CLRMSK                                        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_CLK_XTPLUS_CTRL
+*/
+#define RGX_CR_CLK_XTPLUS_CTRL                                                      (0x0080U)
+#define RGX_CR_CLK_XTPLUS_CTRL_MASKFULL                                             (IMG_UINT64_C(0x0000003FFFFF0000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_SHIFT                                            (36U)
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_ON                                               (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_AUTO                                             (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_SHIFT                                           (34U)
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFF3FFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_ON                                              (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_AUTO                                            (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_SHIFT                                            (32U)
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_ON                                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_AUTO                                             (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_SHIFT                                        (30U)
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_ON                                           (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_AUTO                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_SHIFT                                          (28U)
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_ON                                             (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_AUTO                                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_SHIFT                                         (26U)
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_ON                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_AUTO                                          (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_SHIFT                                          (24U)
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_ON                                             (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_AUTO                                           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_SHIFT                                     (22U)
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_ON                                        (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_AUTO                                      (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_SHIFT                                 (20U)
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_ON                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_AUTO                                  (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_SHIFT                                     (18U)
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_ON                                        (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_AUTO                                      (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_SHIFT                                       (16U)
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_ON                                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_AUTO                                        (IMG_UINT64_C(0x0000000000020000))
+
+
+/*
+    Register RGX_CR_CLK_XTPLUS_STATUS
+*/
+#define RGX_CR_CLK_XTPLUS_STATUS                                                    (0x0088U)
+#define RGX_CR_CLK_XTPLUS_STATUS_MASKFULL                                           (IMG_UINT64_C(0x00000000000007FF))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_SHIFT                                          (10U)
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_RUNNING                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_SHIFT                                          (9U)
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_RUNNING                                        (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_SHIFT                                      (8U)
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_RUNNING                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_SHIFT                                         (7U)
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_RUNNING                                       (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_SHIFT                                        (6U)
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_RUNNING                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_SHIFT                                       (5U)
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_RUNNING                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_SHIFT                                        (4U)
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_RUNNING                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_SHIFT                                   (3U)
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_RUNNING                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_SHIFT                               (2U)
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_RUNNING                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_SHIFT                                   (1U)
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_RUNNING                                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_SHIFT                                     (0U)
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_RUNNING                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET
+*/
+#define RGX_CR_SOFT_RESET                                                           (0x0100U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL                                        (IMG_UINT64_C(0xFFEFFFFFFFFFFC3D))
+#define RGX_CR_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x00E7FFFFFFFFFC3D))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_SHIFT                             (63U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_CLRMSK                            (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_EN                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_SHIFT                             (62U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_SHIFT                             (61U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_CLRMSK                            (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_EN                                (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_SHIFT                                (60U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_CLRMSK                               (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_EN                                   (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_SHIFT                               (59U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_CLRMSK                              (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_EN                                  (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_SHIFT                                       (58U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_CLRMSK                                      (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_EN                                          (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_SHIFT                                       (57U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_CLRMSK                                      (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_EN                                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_SHIFT                                       (56U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_CLRMSK                                      (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_EN                                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SOFT_RESET_DPX1_CORE_SHIFT                                           (55U)
+#define RGX_CR_SOFT_RESET_DPX1_CORE_CLRMSK                                          (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DPX1_CORE_EN                                              (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SOFT_RESET_DPX0_CORE_SHIFT                                           (54U)
+#define RGX_CR_SOFT_RESET_DPX0_CORE_CLRMSK                                          (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DPX0_CORE_EN                                              (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SOFT_RESET_FBA_SHIFT                                                 (53U)
+#define RGX_CR_SOFT_RESET_FBA_CLRMSK                                                (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBA_EN                                                    (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_SHIFT                                    (51U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_CLRMSK                                   (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_EN                                       (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SOFT_RESET_SH_SHIFT                                                  (50U)
+#define RGX_CR_SOFT_RESET_SH_CLRMSK                                                 (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_SH_EN                                                     (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SOFT_RESET_VRDM_SHIFT                                                (49U)
+#define RGX_CR_SOFT_RESET_VRDM_CLRMSK                                               (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_VRDM_EN                                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SOFT_RESET_MCU_FBTC_SHIFT                                            (48U)
+#define RGX_CR_SOFT_RESET_MCU_FBTC_CLRMSK                                           (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_MCU_FBTC_EN                                               (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_SHIFT                                       (47U)
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_EN                                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_SHIFT                                       (46U)
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_EN                                          (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_SHIFT                                       (45U)
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_EN                                          (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_SHIFT                                       (44U)
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_EN                                          (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SOFT_RESET_IPP_SHIFT                                                 (43U)
+#define RGX_CR_SOFT_RESET_IPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_IPP_EN                                                    (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_SHIFT                                           (42U)
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_EN                                              (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_SHIFT                                        (41U)
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_EN                                           (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_SHIFT                                         (40U)
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_EN                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_SHIFT                                         (39U)
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_EN                                            (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_SHIFT                                         (38U)
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_EN                                            (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_SHIFT                                         (37U)
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_EN                                            (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_SHIFT                                         (36U)
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_EN                                            (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_SHIFT                                         (35U)
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_EN                                            (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SOFT_RESET_MMU_SHIFT                                                 (34U)
+#define RGX_CR_SOFT_RESET_MMU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SOFT_RESET_MMU_EN                                                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SOFT_RESET_BIF1_SHIFT                                                (33U)
+#define RGX_CR_SOFT_RESET_BIF1_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF1_EN                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SOFT_RESET_GARTEN_SHIFT                                              (32U)
+#define RGX_CR_SOFT_RESET_GARTEN_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GARTEN_EN                                                 (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SOFT_RESET_CPU_SHIFT                                                 (32U)
+#define RGX_CR_SOFT_RESET_CPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SOFT_RESET_CPU_EN                                                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_SHIFT                                         (31U)
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_EN                                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_SHIFT                                         (30U)
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_EN                                            (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_SHIFT                                         (29U)
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_EN                                            (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_SHIFT                                          (28U)
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_EN                                             (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SOFT_RESET_SLC_SHIFT                                                 (27U)
+#define RGX_CR_SOFT_RESET_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SOFT_RESET_SLC_EN                                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SOFT_RESET_TLA_SHIFT                                                 (26U)
+#define RGX_CR_SOFT_RESET_TLA_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SOFT_RESET_TLA_EN                                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SOFT_RESET_UVS_SHIFT                                                 (25U)
+#define RGX_CR_SOFT_RESET_UVS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SOFT_RESET_UVS_EN                                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SOFT_RESET_TE_SHIFT                                                  (24U)
+#define RGX_CR_SOFT_RESET_TE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SOFT_RESET_TE_EN                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SOFT_RESET_GPP_SHIFT                                                 (23U)
+#define RGX_CR_SOFT_RESET_GPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SOFT_RESET_GPP_EN                                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SOFT_RESET_FBDC_SHIFT                                                (22U)
+#define RGX_CR_SOFT_RESET_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SOFT_RESET_FBDC_EN                                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SOFT_RESET_FBC_SHIFT                                                 (21U)
+#define RGX_CR_SOFT_RESET_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SOFT_RESET_FBC_EN                                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SOFT_RESET_PM_SHIFT                                                  (20U)
+#define RGX_CR_SOFT_RESET_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SOFT_RESET_PM_EN                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SOFT_RESET_PBE_SHIFT                                                 (19U)
+#define RGX_CR_SOFT_RESET_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SOFT_RESET_PBE_EN                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SOFT_RESET_USC_SHARED_SHIFT                                          (18U)
+#define RGX_CR_SOFT_RESET_USC_SHARED_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SOFT_RESET_USC_SHARED_EN                                             (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SOFT_RESET_MCU_L1_SHIFT                                              (17U)
+#define RGX_CR_SOFT_RESET_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SOFT_RESET_MCU_L1_EN                                                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SOFT_RESET_BIF_SHIFT                                                 (16U)
+#define RGX_CR_SOFT_RESET_BIF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SOFT_RESET_BIF_EN                                                    (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SOFT_RESET_CDM_SHIFT                                                 (15U)
+#define RGX_CR_SOFT_RESET_CDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SOFT_RESET_CDM_EN                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SOFT_RESET_VDM_SHIFT                                                 (14U)
+#define RGX_CR_SOFT_RESET_VDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SOFT_RESET_VDM_EN                                                    (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SOFT_RESET_TESS_SHIFT                                                (13U)
+#define RGX_CR_SOFT_RESET_TESS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SOFT_RESET_TESS_EN                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SOFT_RESET_PDS_SHIFT                                                 (12U)
+#define RGX_CR_SOFT_RESET_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SOFT_RESET_PDS_EN                                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SOFT_RESET_ISP_SHIFT                                                 (11U)
+#define RGX_CR_SOFT_RESET_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SOFT_RESET_ISP_EN                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SOFT_RESET_TSP_SHIFT                                                 (10U)
+#define RGX_CR_SOFT_RESET_TSP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SOFT_RESET_TSP_EN                                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SOFT_RESET_SYSARB_SHIFT                                              (5U)
+#define RGX_CR_SOFT_RESET_SYSARB_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SOFT_RESET_SYSARB_EN                                                 (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_SHIFT                                       (4U)
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_EN                                          (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SOFT_RESET_MCU_L0_SHIFT                                              (3U)
+#define RGX_CR_SOFT_RESET_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SOFT_RESET_MCU_L0_EN                                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SOFT_RESET_TPU_SHIFT                                                 (2U)
+#define RGX_CR_SOFT_RESET_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SOFT_RESET_TPU_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SOFT_RESET_USC_SHIFT                                                 (0U)
+#define RGX_CR_SOFT_RESET_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SOFT_RESET_USC_EN                                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET2
+*/
+#define RGX_CR_SOFT_RESET2                                                          (0x0108U)
+#define RGX_CR_SOFT_RESET2_MASKFULL                                                 (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_SOFT_RESET2_SPFILTER_SHIFT                                           (12U)
+#define RGX_CR_SOFT_RESET2_SPFILTER_CLRMSK                                          (0xFFE00FFFU)
+#define RGX_CR_SOFT_RESET2_TDM_SHIFT                                                (11U)
+#define RGX_CR_SOFT_RESET2_TDM_CLRMSK                                               (0xFFFFF7FFU)
+#define RGX_CR_SOFT_RESET2_TDM_EN                                                   (0x00000800U)
+#define RGX_CR_SOFT_RESET2_ASTC_SHIFT                                               (10U)
+#define RGX_CR_SOFT_RESET2_ASTC_CLRMSK                                              (0xFFFFFBFFU)
+#define RGX_CR_SOFT_RESET2_ASTC_EN                                                  (0x00000400U)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_SHIFT                                         (9U)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_EN                                            (0x00000200U)
+#define RGX_CR_SOFT_RESET2_USCPS_SHIFT                                              (8U)
+#define RGX_CR_SOFT_RESET2_USCPS_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_SOFT_RESET2_USCPS_EN                                                 (0x00000100U)
+#define RGX_CR_SOFT_RESET2_IPF_SHIFT                                                (7U)
+#define RGX_CR_SOFT_RESET2_IPF_CLRMSK                                               (0xFFFFFF7FU)
+#define RGX_CR_SOFT_RESET2_IPF_EN                                                   (0x00000080U)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_SHIFT                                           (6U)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_EN                                              (0x00000040U)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_SHIFT                                         (5U)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_EN                                            (0x00000020U)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_SHIFT                                         (4U)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_EN                                            (0x00000010U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_SHIFT                                     (3U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_EN                                        (0x00000008U)
+#define RGX_CR_SOFT_RESET2_PIXEL_SHIFT                                              (2U)
+#define RGX_CR_SOFT_RESET2_PIXEL_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_SOFT_RESET2_PIXEL_EN                                                 (0x00000004U)
+#define RGX_CR_SOFT_RESET2_CDM_SHIFT                                                (1U)
+#define RGX_CR_SOFT_RESET2_CDM_CLRMSK                                               (0xFFFFFFFDU)
+#define RGX_CR_SOFT_RESET2_CDM_EN                                                   (0x00000002U)
+#define RGX_CR_SOFT_RESET2_VERTEX_SHIFT                                             (0U)
+#define RGX_CR_SOFT_RESET2_VERTEX_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_CR_SOFT_RESET2_VERTEX_EN                                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_ENABLE
+*/
+#define RGX_CR_EVENT_ENABLE                                                         (0x0128U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__MASKFULL                                      (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_ENABLE__SIGNALS__MASKFULL                                      (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_SHIFT                                 (28U)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_EN                                    (0x10000000U)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_SHIFT                                (27U)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_CLRMSK                               (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_EN                                   (0x08000000U)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_SHIFT                                 (26U)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_EN                                    (0x04000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_SHIFT                                  (25U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_CLRMSK                                 (0xFDFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_EN                                     (0x02000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_SHIFT                                  (24U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_CLRMSK                                 (0xFEFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_EN                                     (0x01000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_SHIFT                                  (23U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_CLRMSK                                 (0xFF7FFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_EN                                     (0x00800000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_SHIFT                                  (22U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_EN                                     (0x00400000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_SHIFT                                  (21U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_CLRMSK                                 (0xFFDFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_EN                                     (0x00200000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_SHIFT                                  (20U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_SHIFT                                  (20U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_SHIFT                                  (19U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_CLRMSK                                 (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_EN                                     (0x00080000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_SHIFT                               (19U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_EN                                  (0x00080000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_SHIFT                                  (18U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_CLRMSK                                 (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_EN                                     (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_SHIFT                                      (17U)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_CLRMSK                                     (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_EN                                         (0x00020000U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_SHIFT                                      (14U)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_CLRMSK                                     (0xFFFFBFFFU)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_EN                                         (0x00004000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_SHIFT                                    (8U)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_EN                                       (0x00000100U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_SHIFT                                (3U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_EN                                   (0x00000008U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_SHIFT                                      (0U)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_STATUS
+*/
+#define RGX_CR_EVENT_STATUS                                                         (0x0130U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__MASKFULL                                      (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_STATUS__SIGNALS__MASKFULL                                      (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_SHIFT                                 (28U)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_EN                                    (0x10000000U)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_SHIFT                                (27U)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_CLRMSK                               (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_EN                                   (0x08000000U)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_SHIFT                                 (26U)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_EN                                    (0x04000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_SHIFT                                  (25U)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_CLRMSK                                 (0xFDFFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_EN                                     (0x02000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_SHIFT                                  (24U)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_CLRMSK                                 (0xFEFFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_EN                                     (0x01000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_SHIFT                                  (23U)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_CLRMSK                                 (0xFF7FFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_EN                                     (0x00800000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_SHIFT                                  (22U)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_EN                                     (0x00400000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_SHIFT                                  (21U)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_CLRMSK                                 (0xFFDFFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_EN                                     (0x00200000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_SHIFT                                  (20U)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_SHIFT                                  (20U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_SHIFT                                  (19U)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_CLRMSK                                 (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_EN                                     (0x00080000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_SHIFT                               (19U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_EN                                  (0x00080000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_SHIFT                                  (18U)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_CLRMSK                                 (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_EN                                     (0x00040000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_SHIFT                                      (17U)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_CLRMSK                                     (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_EN                                         (0x00020000U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_SHIFT                                      (14U)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_CLRMSK                                     (0xFFFFBFFFU)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_EN                                         (0x00004000U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_SHIFT                                    (8U)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_EN                                       (0x00000100U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_SHIFT                                (3U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_EN                                   (0x00000008U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_SHIFT                                      (0U)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_CLEAR
+*/
+#define RGX_CR_EVENT_CLEAR                                                          (0x0138U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__MASKFULL                                       (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_CLEAR__SIGNALS__MASKFULL                                       (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_SHIFT                                 (31U)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_CLRMSK                                (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_EN                                    (0x80000000U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_SHIFT                                   (30U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_CLRMSK                                  (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_EN                                      (0x40000000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_SHIFT                             (29U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_EN                                (0x20000000U)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_SHIFT                                  (28U)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_CLRMSK                                 (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_EN                                     (0x10000000U)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_SHIFT                                 (27U)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_CLRMSK                                (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_EN                                    (0x08000000U)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_SHIFT                                  (26U)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_CLRMSK                                 (0xFBFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_EN                                     (0x04000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_SHIFT                                   (25U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_CLRMSK                                  (0xFDFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_EN                                      (0x02000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_SHIFT                                   (24U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_CLRMSK                                  (0xFEFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_EN                                      (0x01000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_SHIFT                                   (23U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_CLRMSK                                  (0xFF7FFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_EN                                      (0x00800000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_SHIFT                                   (22U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_CLRMSK                                  (0xFFBFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_EN                                      (0x00400000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_SHIFT                                   (21U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_CLRMSK                                  (0xFFDFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_EN                                      (0x00200000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_SHIFT                                   (20U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_EN                                      (0x00100000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_SHIFT                                   (20U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_EN                                      (0x00100000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_SHIFT                                   (19U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_CLRMSK                                  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_EN                                      (0x00080000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_SHIFT                                (19U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_CLRMSK                               (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_EN                                   (0x00080000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_SHIFT                                   (18U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_CLRMSK                                  (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_EN                                      (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT               (18U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK              (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                  (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT               (18U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK              (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                  (0x00040000U)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_SHIFT                                       (17U)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_CLRMSK                                      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_EN                                          (0x00020000U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                   (17U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                  (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                      (0x00020000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_SHIFT                               (16U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_EN                                  (0x00010000U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_SHIFT                                        (15U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_CLRMSK                                       (0xFFFF7FFFU)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_EN                                           (0x00008000U)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_SHIFT                                       (14U)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_CLRMSK                                      (0xFFFFBFFFU)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_EN                                          (0x00004000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_SHIFT                                           (13U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_EN                                              (0x00002000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_SHIFT                                           (12U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_EN                                              (0x00001000U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_SHIFT                                        (11U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_CLRMSK                                       (0xFFFFF7FFU)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_EN                                           (0x00000800U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_SHIFT                                     (10U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_EN                                        (0x00000400U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_SHIFT                                     (9U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_EN                                        (0x00000200U)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_SHIFT                                     (8U)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_CLRMSK                                    (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_EN                                        (0x00000100U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_SHIFT                                   (7U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_EN                                      (0x00000080U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_SHIFT                                       (6U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_CLRMSK                                      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_EN                                          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_SHIFT                                        (5U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_EN                                           (0x00000020U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_SHIFT                                  (4U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_EN                                     (0x00000010U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_SHIFT                                 (3U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_EN                                    (0x00000008U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_SHIFT                                   (2U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_EN                                      (0x00000004U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_SHIFT                                    (1U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_EN                                       (0x00000002U)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_SHIFT                                       (0U)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_TIMER
+*/
+#define RGX_CR_TIMER                                                                (0x0160U)
+#define RGX_CR_TIMER_MASKFULL                                                       (IMG_UINT64_C(0x8000FFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_SHIFT                                                    (63U)
+#define RGX_CR_TIMER_BIT31_CLRMSK                                                   (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_EN                                                       (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_TIMER_VALUE_SHIFT                                                    (0U)
+#define RGX_CR_TIMER_VALUE_CLRMSK                                                   (IMG_UINT64_C(0xFFFF000000000000))
+
+
+/*
+    Register RGX_CR_TLA_STATUS
+*/
+#define RGX_CR_TLA_STATUS                                                           (0x0178U)
+#define RGX_CR_TLA_STATUS_MASKFULL                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_TLA_STATUS_BLIT_COUNT_SHIFT                                          (39U)
+#define RGX_CR_TLA_STATUS_BLIT_COUNT_CLRMSK                                         (IMG_UINT64_C(0x0000007FFFFFFFFF))
+#define RGX_CR_TLA_STATUS_REQUEST_SHIFT                                             (7U)
+#define RGX_CR_TLA_STATUS_REQUEST_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF800000007F))
+#define RGX_CR_TLA_STATUS_FIFO_FULLNESS_SHIFT                                       (1U)
+#define RGX_CR_TLA_STATUS_FIFO_FULLNESS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFF81))
+#define RGX_CR_TLA_STATUS_BUSY_SHIFT                                                (0U)
+#define RGX_CR_TLA_STATUS_BUSY_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_TLA_STATUS_BUSY_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_PM_PARTIAL_RENDER_ENABLE
+*/
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE                                             (0x0338U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_SHIFT                                    (0U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_SIDEKICK_IDLE
+*/
+#define RGX_CR_SIDEKICK_IDLE                                                        (0x03C8U)
+#define RGX_CR_SIDEKICK_IDLE_MASKFULL                                               (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_SHIFT                                           (6U)
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_EN                                              (0x00000040U)
+#define RGX_CR_SIDEKICK_IDLE_MMU_SHIFT                                              (5U)
+#define RGX_CR_SIDEKICK_IDLE_MMU_CLRMSK                                             (0xFFFFFFDFU)
+#define RGX_CR_SIDEKICK_IDLE_MMU_EN                                                 (0x00000020U)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_SHIFT                                           (4U)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_CLRMSK                                          (0xFFFFFFEFU)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_EN                                              (0x00000010U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_SHIFT                                              (3U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_CLRMSK                                             (0xFFFFFFF7U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_EN                                                 (0x00000008U)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_SHIFT                                           (2U)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_EN                                              (0x00000004U)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_SHIFT                                           (1U)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_EN                                              (0x00000002U)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_SHIFT                                            (0U)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_MARS_IDLE
+*/
+#define RGX_CR_MARS_IDLE                                                            (0x08F8U)
+#define RGX_CR_MARS_IDLE_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_SHIFT                                           (2U)
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_EN                                              (0x00000004U)
+#define RGX_CR_MARS_IDLE_CPU_SHIFT                                                  (1U)
+#define RGX_CR_MARS_IDLE_CPU_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_MARS_IDLE_CPU_EN                                                     (0x00000002U)
+#define RGX_CR_MARS_IDLE_SOCIF_SHIFT                                                (0U)
+#define RGX_CR_MARS_IDLE_SOCIF_CLRMSK                                               (0xFFFFFFFEU)
+#define RGX_CR_MARS_IDLE_SOCIF_EN                                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS                                             (0x0430U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_MASKFULL                                    (IMG_UINT64_C(0x00000000000000F3))
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_LAST_PIPE_SHIFT                             (4U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_LAST_PIPE_CLRMSK                            (0xFFFFFF0FU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_SHIFT                           (1U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_CLRMSK                          (0xFFFFFFFDU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_EN                              (0x00000002U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_SHIFT                              (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK0
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0                                              (0x0438U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_MASKFULL                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE1_SHIFT                             (32U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE1_CLRMSK                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE0_SHIFT                             (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE0_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK1
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1                                              (0x0440U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_PDS_STATE2_SHIFT                             (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_PDS_STATE2_CLRMSK                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK2
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2                                              (0x0448U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_MASKFULL                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT2_SHIFT                            (32U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT2_CLRMSK                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT1_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT1_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK0
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0                                             (0x0450U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_MASKFULL                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE1_SHIFT                            (32U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE1_CLRMSK                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE0_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE0_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK1
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1                                             (0x0458U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_PDS_STATE2_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_PDS_STATE2_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK2
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2                                             (0x0460U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_MASKFULL                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT2_SHIFT                           (32U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT2_CLRMSK                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT1_SHIFT                           (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT1_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS                                             (0x04A0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_SHIFT                           (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_CLRMSK                          (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_EN                              (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_SHIFT                              (0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_PDS0                                                     (0x04A8U)
+#define RGX_CR_CDM_CONTEXT_PDS0_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_SHIFT                                     (36U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_CLRMSK                                    (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSIZE                                 (16U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_SHIFT                                     (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_PDS1                                                     (0x04B0U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__MASKFULL                                (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_MASKFULL                                            (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                       (30U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                      (0xBFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                          (0x40000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_SHIFT                                   (29U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_CLRMSK                                  (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_EN                                      (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                       (29U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                      (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                          (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_SHIFT                                   (28U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_CLRMSK                                  (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_EN                                      (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_SHIFT                            (28U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_CLRMSK                           (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_EN                               (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_SHIFT                                        (27U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_CLRMSK                                       (0xF7FFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_EN                                           (0x08000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                      (22U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                     (0xF03FFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_SHIFT                                  (21U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_CLRMSK                                 (0xF81FFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                     (21U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK                    (0xFFDFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_EN                        (0x00200000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_SHIFT                                 (20U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_CLRMSK                                (0xFFEFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_EN                                    (0x00100000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                       (12U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                      (0xFFE00FFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SIZE_SHIFT                                   (11U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SIZE_CLRMSK                                  (0xFFF007FFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_SHIFT                                     (7U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_CLRMSK                                    (0xFFFFF87FU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                         (7U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                        (0xFFFFF07FU)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_SHIFT                                     (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_CLRMSK                                    (0xFFFFFF81U)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_SHIFT                                         (0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS
+*/
+#define RGX_CR_CDM_TERMINATE_PDS                                                    (0x04B8U)
+#define RGX_CR_CDM_TERMINATE_PDS_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_SHIFT                                    (36U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_CLRMSK                                   (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSIZE                                (16U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_SHIFT                                    (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSIZE                                (16U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS1
+*/
+#define RGX_CR_CDM_TERMINATE_PDS1                                                   (0x04C0U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__MASKFULL                              (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_MASKFULL                                          (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                     (30U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                    (0xBFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                        (0x40000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_SHIFT                                 (29U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_CLRMSK                                (0xDFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_EN                                    (0x20000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                     (29U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                    (0xDFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                        (0x20000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_SHIFT                                 (28U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_EN                                    (0x10000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_SHIFT                          (28U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_CLRMSK                         (0xEFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_EN                             (0x10000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_SHIFT                                      (27U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_CLRMSK                                     (0xF7FFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_EN                                         (0x08000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                    (22U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                   (0xF03FFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_SHIFT                                (21U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_CLRMSK                               (0xF81FFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                   (21U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK                  (0xFFDFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_EN                      (0x00200000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_SHIFT                               (20U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_CLRMSK                              (0xFFEFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_EN                                  (0x00100000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                     (12U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                    (0xFFE00FFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SIZE_SHIFT                                 (11U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SIZE_CLRMSK                                (0xFFF007FFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_SHIFT                                   (7U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_CLRMSK                                  (0xFFFFF87FU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                       (7U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                      (0xFFFFF07FU)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_SHIFT                                   (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_CLRMSK                                  (0xFFFFFF81U)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_SHIFT                                       (0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0                                                (0x04D8U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_SHIFT                                (36U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_CLRMSK                               (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_SHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSIZE                            (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1                                                (0x04E0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__MASKFULL                           (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                  (30U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                 (0xBFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                     (0x40000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_SHIFT                              (29U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_EN                                 (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                  (29U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                     (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_SHIFT                              (28U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_EN                                 (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_SHIFT                       (28U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_CLRMSK                      (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_EN                          (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_SHIFT                                   (27U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_CLRMSK                                  (0xF7FFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_EN                                      (0x08000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                 (22U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                (0xF03FFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_SHIFT                             (21U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_CLRMSK                            (0xF81FFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                (21U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK               (0xFFDFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_EN                   (0x00200000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_SHIFT                            (20U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_CLRMSK                           (0xFFEFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_EN                               (0x00100000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                  (12U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                 (0xFFE00FFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SIZE_SHIFT                              (11U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SIZE_CLRMSK                             (0xFFF007FFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_SHIFT                                (7U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_CLRMSK                               (0xFFFFF87FU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                    (7U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                   (0xFFFFF07FU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_SHIFT                                (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_CLRMSK                               (0xFFFFFF81U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_SHIFT                                    (0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_CONFIG
+*/
+#define RGX_CR_MIPS_WRAPPER_CONFIG                                                  (0x0810U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_MASKFULL                                         (IMG_UINT64_C(0x000001030F01FFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_SHIFT                             (40U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_CLRMSK                            (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_EN                                (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_SHIFT                               (33U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_EN                                  (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_SHIFT                               (32U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_EN                                  (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_OS_ID_SHIFT                                      (25U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_OS_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFF1FFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_SHIFT                                    (24U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_EN                                       (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_SHIFT                              (16U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MIPS32                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_REGBANK_BASE_ADDR_SHIFT                          (0U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_REGBANK_BASE_ADDR_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP1_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1                                             (0x0818U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP1_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2                                             (0x0820U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP2_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1                                             (0x0828U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP2_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2                                             (0x0830U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP3_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1                                             (0x0838U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP3_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2                                             (0x0840U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP4_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1                                             (0x0848U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP4_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2                                             (0x0850U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP5_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1                                             (0x0858U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP5_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2                                             (0x0860U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS                                      (0x0868U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_MASKFULL                             (IMG_UINT64_C(0x00000001FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_SHIFT                          (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_EN                             (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_ADDRESS_SHIFT                        (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_ADDRESS_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR                                       (0x0870U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_MASKFULL                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG                                         (0x0878U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MASKFULL                                (IMG_UINT64_C(0xFFFFFFF7FFFFFFBF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ADDR_OUT_SHIFT                          (36U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ADDR_OUT_CLRMSK                         (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_OS_ID_SHIFT                             (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_OS_ID_CLRMSK                            (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_BASE_ADDR_IN_SHIFT                      (12U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_BASE_ADDR_IN_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_SHIFT                           (11U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_EN                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_SHIFT                       (7U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFF87F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4KB                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_16KB                        (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64KB                        (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256KB                       (IMG_UINT64_C(0x0000000000000180))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_1MB                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4MB                         (IMG_UINT64_C(0x0000000000000280))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_16MB                        (IMG_UINT64_C(0x0000000000000300))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64MB                        (IMG_UINT64_C(0x0000000000000380))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256MB                       (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ENTRY_SHIFT                             (1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ENTRY_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFC1))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_EN                          (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_READ
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ                                           (0x0880U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_MASKFULL                                  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_ENTRY_SHIFT                               (1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_ENTRY_CLRMSK                              (0xFFFFFFC1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_SHIFT                             (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA                                           (0x0888U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFF7FFFFFF81))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_SHIFT                            (36U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_CLRMSK                           (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_OS_ID_SHIFT                               (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_OS_ID_CLRMSK                              (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_SHIFT                        (12U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_SHIFT                             (11U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_EN                                (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_SHIFT                         (7U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFF87F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_SHIFT                         (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_EN                            (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_ENABLE
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE                                              (0x08A0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_STATUS
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS                                              (0x08A8U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_CLEAR
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR                                               (0x08B0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_MASKFULL                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_SHIFT                                   (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_NMI_ENABLE
+*/
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE                                              (0x08B8U)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT                                               (0x08C0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_MASKFULL                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_SHIFT                                 (0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_DEBUG_CONFIG
+*/
+#define RGX_CR_MIPS_DEBUG_CONFIG                                                    (0x08C8U)
+#define RGX_CR_MIPS_DEBUG_CONFIG_MASKFULL                                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_SHIFT                          (0U)
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_CLRMSK                         (0xFFFFFFFEU)
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_EN                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_EXCEPTION_STATUS
+*/
+#define RGX_CR_MIPS_EXCEPTION_STATUS                                                (0x08D0U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_SHIFT                                 (5U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_CLRMSK                                (0xFFFFFFDFU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_EN                                    (0x00000020U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_SHIFT                             (4U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_CLRMSK                            (0xFFFFFFEFU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN                                (0x00000010U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_SHIFT                              (3U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_EN                                 (0x00000008U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_SHIFT                              (2U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_EN                                 (0x00000004U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_SHIFT                                   (1U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_EN                                      (0x00000002U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_SHIFT                                   (0U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_STATUS
+*/
+#define RGX_CR_MIPS_WRAPPER_STATUS                                                  (0x08E8U)
+#define RGX_CR_MIPS_WRAPPER_STATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MIPS_WRAPPER_STATUS_OUTSTANDING_REQUESTS_SHIFT                       (0U)
+#define RGX_CR_MIPS_WRAPPER_STATUS_OUTSTANDING_REQUESTS_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_XPU_BROADCAST
+*/
+#define RGX_CR_XPU_BROADCAST                                                        (0x0890U)
+#define RGX_CR_XPU_BROADCAST_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_XPU_BROADCAST_MASK_SHIFT                                             (0U)
+#define RGX_CR_XPU_BROADCAST_MASK_CLRMSK                                            (0xFFFFFE00U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX                                                    (0x0A00U)
+#define RGX_CR_META_SP_MSLVDATAX_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT                                                    (0x0A08U)
+#define RGX_CR_META_SP_MSLVDATAT_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0                                                    (0x0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_SHIFT                                         (2U)
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_CLRMSK                                        (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_SHIFT                                     (1U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN                                        (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_SHIFT                                           (0U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1                                                    (0x0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1_MASKFULL                                           (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_SHIFT                                 (30U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_CLRMSK                                (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_SHIFT                              (29U)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_EN                                 (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_SHIFT                             (28U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_EN                                (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_SHIFT                                 (26U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN                                    (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_SHIFT                                 (25U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_CLRMSK                                (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_EN                                    (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_SHIFT                                        (24U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_CLRMSK                                       (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_EN                                           (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_SHIFT                                     (21U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_CLRMSK                                    (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_SHIFT                                       (20U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_CLRMSK                                      (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_EN                                          (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_SHIFT                                    (18U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_EN                                       (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_SHIFT                                       (4U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_CLRMSK                                      (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_CLRMSK                                  (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_SHIFT                                   (0U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_CLRMSK                                  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE                                                 (0x0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_SHIFT                                     (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK                                                   (0x0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI                                                  (0x0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK                                                   (0x0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI                                                  (0x0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK                                                   (0x0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI                                                  (0x0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK                                                   (0x0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI                                                  (0x0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST                                                      (0x0AC0U)
+#define RGX_CR_META_SP_MSLVRST_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS                                                (0x0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_SHIFT                                (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_EN                                   (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_SHIFT                                (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN                                   (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE                                                (0x0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_SHIFT                                   (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_CLRMSK                                  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_EN                                      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_EN                                      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL                                                 (0x0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE
+*/
+#define RGX_CR_MTS_SCHEDULE                                                         (0x0B00U)
+#define RGX_CR_MTS_SCHEDULE_MASKFULL                                                (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE_HOST_SHIFT                                              (8U)
+#define RGX_CR_MTS_SCHEDULE_HOST_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE_HOST_BG_TIMER                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_HOST_HOST                                               (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_SHIFT                                          (6U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_CLRMSK                                         (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT0                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT1                                           (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT2                                           (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT3                                           (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_SHIFT                                           (5U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_CLRMSK                                          (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_BGCTX                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_INTCTX                                          (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE_TASK_SHIFT                                              (4U)
+#define RGX_CR_MTS_SCHEDULE_TASK_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED                                        (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_TASK_COUNTED                                            (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE_DM_SHIFT                                                (0U)
+#define RGX_CR_MTS_SCHEDULE_DM_CLRMSK                                               (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM0                                                  (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM1                                                  (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM2                                                  (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM3                                                  (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM4                                                  (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM5                                                  (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM6                                                  (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM7                                                  (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM_ALL                                               (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE1
+*/
+#define RGX_CR_MTS_SCHEDULE1                                                        (0x10B00U)
+#define RGX_CR_MTS_SCHEDULE1_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE1_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE1_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE1_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE1_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE2
+*/
+#define RGX_CR_MTS_SCHEDULE2                                                        (0x20B00U)
+#define RGX_CR_MTS_SCHEDULE2_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE2_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE2_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE2_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE2_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE3
+*/
+#define RGX_CR_MTS_SCHEDULE3                                                        (0x30B00U)
+#define RGX_CR_MTS_SCHEDULE3_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE3_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE3_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE3_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE3_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE4
+*/
+#define RGX_CR_MTS_SCHEDULE4                                                        (0x40B00U)
+#define RGX_CR_MTS_SCHEDULE4_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE4_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE4_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE4_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE4_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE5
+*/
+#define RGX_CR_MTS_SCHEDULE5                                                        (0x50B00U)
+#define RGX_CR_MTS_SCHEDULE5_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE5_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE5_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE5_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE5_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE6
+*/
+#define RGX_CR_MTS_SCHEDULE6                                                        (0x60B00U)
+#define RGX_CR_MTS_SCHEDULE6_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE6_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE6_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE6_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE6_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE7
+*/
+#define RGX_CR_MTS_SCHEDULE7                                                        (0x70B00U)
+#define RGX_CR_MTS_SCHEDULE7_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE7_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE7_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE7_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE7_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC                                           (0x0B30U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC                                           (0x0B38U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC                                          (0x0B40U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC                                          (0x0B48U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_GARTEN_WRAPPER_CONFIG
+*/
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG                                            (0x0B50U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__MASKFULL                        (IMG_UINT64_C(0x000FF0FFFFFFF701))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_MASKFULL                                   (IMG_UINT64_C(0x0000FFFFFFFFF001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT                        (44U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK                       (IMG_UINT64_C(0xFFFF0FFFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT             (44U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_CLRMSK            (IMG_UINT64_C(0xFFF00FFFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT                             (40U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_CLRMSK                            (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_SHIFT                           (12U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_CLRMSK                          (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PERSISTENCE_SHIFT         (9U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PERSISTENCE_CLRMSK        (IMG_UINT64_C(0xFFFFFFFFFFFFF9FF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_SHIFT        (8U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_EN           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_SHIFT                            (0U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_MTS                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S7_INFRA__META                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S7_INFRA__MTS                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MTS_DM0_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE                                             (0x0B58U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM1_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE                                             (0x0B60U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM2_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE                                             (0x0B68U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM3_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE                                             (0x0B70U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM4_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE                                             (0x0B78U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM5_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE                                             (0x0B80U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX
+*/
+#define RGX_CR_MTS_INTCTX                                                           (0x0B98U)
+#define RGX_CR_MTS_INTCTX_MASKFULL                                                  (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_SHIFT                                    (22U)
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_CLRMSK                                   (0xC03FFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_PTR_SHIFT                                              (18U)
+#define RGX_CR_MTS_INTCTX_DM_PTR_CLRMSK                                             (0xFFC3FFFFU)
+#define RGX_CR_MTS_INTCTX_THREAD_ACTIVE_SHIFT                                       (16U)
+#define RGX_CR_MTS_INTCTX_THREAD_ACTIVE_CLRMSK                                      (0xFFFCFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_SHIFT                                   (8U)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_CLRMSK                                  (0xFFFF00FFU)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX
+*/
+#define RGX_CR_MTS_BGCTX                                                            (0x0BA0U)
+#define RGX_CR_MTS_BGCTX_MASKFULL                                                   (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_MTS_BGCTX_DM_PTR_SHIFT                                               (10U)
+#define RGX_CR_MTS_BGCTX_DM_PTR_CLRMSK                                              (0xFFFFC3FFU)
+#define RGX_CR_MTS_BGCTX_THREAD_ACTIVE_SHIFT                                        (8U)
+#define RGX_CR_MTS_BGCTX_THREAD_ACTIVE_CLRMSK                                       (0xFFFFFCFFU)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE
+*/
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE                                           (0x0BA8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_SHIFT                                 (56U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_CLRMSK                                (IMG_UINT64_C(0x00FFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_SHIFT                                 (48U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_CLRMSK                                (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_SHIFT                                 (40U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_CLRMSK                                (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_SHIFT                                 (32U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_CLRMSK                                (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_SHIFT                                 (24U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_SHIFT                                 (16U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_SHIFT                                 (8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_SHIFT                                 (0U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MTS_GPU_INT_STATUS
+*/
+#define RGX_CR_MTS_GPU_INT_STATUS                                                   (0x0BB0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_SHIFT                                      (0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE_ENABLE
+*/
+#define RGX_CR_MTS_SCHEDULE_ENABLE                                                  (0x0BC8U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_SHIFT                                       (0U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_CLRMSK                                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS0_EVENT_STATUS                                                 (0x0BD8U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR                                                  (0x0BE8U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS1_EVENT_STATUS                                                 (0x10BD8U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR                                                  (0x10BE8U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS2_EVENT_STATUS                                                 (0x20BD8U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR                                                  (0x20BE8U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS3_EVENT_STATUS                                                 (0x30BD8U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR                                                  (0x30BE8U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS4_EVENT_STATUS                                                 (0x40BD8U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR                                                  (0x40BE8U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS5_EVENT_STATUS                                                 (0x50BD8U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR                                                  (0x50BE8U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS6_EVENT_STATUS                                                 (0x60BD8U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR                                                  (0x60BE8U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS7_EVENT_STATUS                                                 (0x70BD8U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR                                                  (0x70BE8U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_BOOT
+*/
+#define RGX_CR_META_BOOT                                                            (0x0BF8U)
+#define RGX_CR_META_BOOT_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_BOOT_MODE_SHIFT                                                 (0U)
+#define RGX_CR_META_BOOT_MODE_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_META_BOOT_MODE_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_GARTEN_SLC
+*/
+#define RGX_CR_GARTEN_SLC                                                           (0x0BB8U)
+#define RGX_CR_GARTEN_SLC_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_SHIFT                                     (0U)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_PPP
+*/
+#define RGX_CR_PPP                                                                  (0x0CD0U)
+#define RGX_CR_PPP_MASKFULL                                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CHECKSUM_SHIFT                                                   (0U)
+#define RGX_CR_PPP_CHECKSUM_CLRMSK                                                  (0x00000000U)
+
+
+#define RGX_CR_ISP_RENDER_DIR_TYPE_MASK                                             (0x00000003U)
+/*
+Top-left to bottom-right */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_TL2BR                                            (0x00000000U)
+/*
+Top-right to bottom-left */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_TR2BL                                            (0x00000001U)
+/*
+Bottom-left to top-right */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_BL2TR                                            (0x00000002U)
+/*
+Bottom-right to top-left */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_BR2TL                                            (0x00000003U)
+
+
+#define RGX_CR_ISP_RENDER_MODE_TYPE_MASK                                            (0x00000003U)
+/*
+Normal render     */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_NORM                                            (0x00000000U)
+/*
+Fast 2D render    */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_FAST_2D                                         (0x00000002U)
+/*
+Fast scale render */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_FAST_SCALE                                      (0x00000003U)
+
+
+/*
+    Register RGX_CR_ISP_RENDER
+*/
+#define RGX_CR_ISP_RENDER                                                           (0x0F08U)
+#define RGX_CR_ISP_RENDER_MASKFULL                                                  (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_SHIFT                           (8U)
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_CLRMSK                          (0xFFFFFEFFU)
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_EN                              (0x00000100U)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_SHIFT                             (7U)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_CLRMSK                            (0xFFFFFF7FU)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_EN                                (0x00000080U)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_SHIFT                           (6U)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_CLRMSK                          (0xFFFFFFBFU)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_EN                              (0x00000040U)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_SHIFT                                        (5U)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_EN                                           (0x00000020U)
+#define RGX_CR_ISP_RENDER_RESUME_SHIFT                                              (4U)
+#define RGX_CR_ISP_RENDER_RESUME_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_ISP_RENDER_RESUME_EN                                                 (0x00000010U)
+#define RGX_CR_ISP_RENDER_DIR_SHIFT                                                 (2U)
+#define RGX_CR_ISP_RENDER_DIR_CLRMSK                                                (0xFFFFFFF3U)
+#define RGX_CR_ISP_RENDER_DIR_TL2BR                                                 (0x00000000U)
+#define RGX_CR_ISP_RENDER_DIR_TR2BL                                                 (0x00000004U)
+#define RGX_CR_ISP_RENDER_DIR_BL2TR                                                 (0x00000008U)
+#define RGX_CR_ISP_RENDER_DIR_BR2TL                                                 (0x0000000CU)
+#define RGX_CR_ISP_RENDER_MODE_SHIFT                                                (0U)
+#define RGX_CR_ISP_RENDER_MODE_CLRMSK                                               (0xFFFFFFFCU)
+#define RGX_CR_ISP_RENDER_MODE_NORM                                                 (0x00000000U)
+#define RGX_CR_ISP_RENDER_MODE_FAST_2D                                              (0x00000002U)
+#define RGX_CR_ISP_RENDER_MODE_FAST_SCALE                                           (0x00000003U)
+
+
+/*
+    Register RGX_CR_ISP_CTL
+*/
+#define RGX_CR_ISP_CTL                                                              (0x0F38U)
+#define RGX_CR_ISP_CTL_MASKFULL                                                     (IMG_UINT64_C(0x00000000FFFFF3FF))
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_SHIFT                                         (31U)
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_EN                                            (0x80000000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SHIFT                                             (30U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_CLRMSK                                            (0xBFFFFFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_EN                                                (0x40000000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_SHIFT                                         (29U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_EN                                            (0x20000000U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_SHIFT                                        (28U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_CLRMSK                                       (0xEFFFFFFFU)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_EN                                           (0x10000000U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_SHIFT                                             (27U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_CLRMSK                                            (0xF7FFFFFFU)
+#define RGX_CR_ISP_CTL_PAIR_TILES_EN                                                (0x08000000U)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_SHIFT                                            (26U)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_CLRMSK                                           (0xFBFFFFFFU)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_EN                                               (0x04000000U)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_SHIFT                                            (25U)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_CLRMSK                                           (0xFDFFFFFFU)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_EN                                               (0x02000000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_SHIFT                                    (23U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_CLRMSK                                   (0xFE7FFFFFU)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_DX9                                      (0x00000000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_DX10                                     (0x00800000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_OGL                                      (0x01000000U)
+#define RGX_CR_ISP_CTL_NUM_TILES_PER_USC_SHIFT                                      (21U)
+#define RGX_CR_ISP_CTL_NUM_TILES_PER_USC_CLRMSK                                     (0xFF9FFFFFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_SHIFT                                           (20U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_CLRMSK                                          (0xFFEFFFFFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_EN                                              (0x00100000U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_SHIFT                                     (19U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_EN                                        (0x00080000U)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_SHIFT                               (18U)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_EN                                  (0x00040000U)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_SHIFT                                    (17U)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_CLRMSK                                   (0xFFFDFFFFU)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_EN                                       (0x00020000U)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_SHIFT                                             (16U)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_CLRMSK                                            (0xFFFEFFFFU)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_EN                                                (0x00010000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_SHIFT                                            (12U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_CLRMSK                                           (0xFFFF0FFFU)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_ONE                                         (0x00000000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TWO                                         (0x00001000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_THREE                                       (0x00002000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FOUR                                        (0x00003000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FIVE                                        (0x00004000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SIX                                         (0x00005000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SEVEN                                       (0x00006000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_EIGHT                                       (0x00007000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_NINE                                        (0x00008000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TEN                                         (0x00009000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_ELEVEN                                      (0x0000A000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TWELVE                                      (0x0000B000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_THIRTEEN                                    (0x0000C000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FOURTEEN                                    (0x0000D000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FIFTEEN                                     (0x0000E000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SIXTEEN                                     (0x0000F000U)
+#define RGX_CR_ISP_CTL_VALID_ID_SHIFT                                               (4U)
+#define RGX_CR_ISP_CTL_VALID_ID_CLRMSK                                              (0xFFFFFC0FU)
+#define RGX_CR_ISP_CTL_UPASS_START_SHIFT                                            (0U)
+#define RGX_CR_ISP_CTL_UPASS_START_CLRMSK                                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_ISP_STATUS
+*/
+#define RGX_CR_ISP_STATUS                                                           (0x1038U)
+#define RGX_CR_ISP_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_SHIFT                                           (2U)
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_EN                                              (0x00000004U)
+#define RGX_CR_ISP_STATUS_ACTIVE_SHIFT                                              (1U)
+#define RGX_CR_ISP_STATUS_ACTIVE_CLRMSK                                             (0xFFFFFFFDU)
+#define RGX_CR_ISP_STATUS_ACTIVE_EN                                                 (0x00000002U)
+#define RGX_CR_ISP_STATUS_EOR_SHIFT                                                 (0U)
+#define RGX_CR_ISP_STATUS_EOR_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_ISP_STATUS_EOR_EN                                                    (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_ISP_XTP_RESUME, with 64 repeats
+*/
+#define RGX_CR_ISP_XTP_RESUME_REPEATCOUNT                                           (64U)
+/*
+    Register RGX_CR_ISP_XTP_RESUME0
+*/
+#define RGX_CR_ISP_XTP_RESUME0                                                      (0x3A00U)
+#define RGX_CR_ISP_XTP_RESUME0_MASKFULL                                             (IMG_UINT64_C(0x00000000003FF3FF))
+#define RGX_CR_ISP_XTP_RESUME0_TILE_X_SHIFT                                         (12U)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_X_CLRMSK                                        (0xFFC00FFFU)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_Y_SHIFT                                         (0U)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_Y_CLRMSK                                        (0xFFFFFC00U)
+
+
+/*
+    Register group: RGX_CR_ISP_XTP_STORE, with 32 repeats
+*/
+#define RGX_CR_ISP_XTP_STORE_REPEATCOUNT                                            (32U)
+/*
+    Register RGX_CR_ISP_XTP_STORE0
+*/
+#define RGX_CR_ISP_XTP_STORE0                                                       (0x3C00U)
+#define RGX_CR_ISP_XTP_STORE0_MASKFULL                                              (IMG_UINT64_C(0x000000007F3FF3FF))
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_SHIFT                                          (30U)
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_CLRMSK                                         (0xBFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_EN                                             (0x40000000U)
+#define RGX_CR_ISP_XTP_STORE0_EOR_SHIFT                                             (29U)
+#define RGX_CR_ISP_XTP_STORE0_EOR_CLRMSK                                            (0xDFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_EOR_EN                                                (0x20000000U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_SHIFT                                       (28U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_CLRMSK                                      (0xEFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_EN                                          (0x10000000U)
+#define RGX_CR_ISP_XTP_STORE0_MT_SHIFT                                              (24U)
+#define RGX_CR_ISP_XTP_STORE0_MT_CLRMSK                                             (0xF0FFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_X_SHIFT                                          (12U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_X_CLRMSK                                         (0xFFC00FFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_Y_SHIFT                                          (0U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_Y_CLRMSK                                         (0xFFFFFC00U)
+
+
+/*
+    Register group: RGX_CR_BIF_CAT_BASE, with 8 repeats
+*/
+#define RGX_CR_BIF_CAT_BASE_REPEATCOUNT                                             (8U)
+/*
+    Register RGX_CR_BIF_CAT_BASE0
+*/
+#define RGX_CR_BIF_CAT_BASE0                                                        (0x1200U)
+#define RGX_CR_BIF_CAT_BASE0_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE1
+*/
+#define RGX_CR_BIF_CAT_BASE1                                                        (0x1208U)
+#define RGX_CR_BIF_CAT_BASE1_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE1_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE1_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE1_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE1_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE2
+*/
+#define RGX_CR_BIF_CAT_BASE2                                                        (0x1210U)
+#define RGX_CR_BIF_CAT_BASE2_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE2_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE2_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE2_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE2_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE3
+*/
+#define RGX_CR_BIF_CAT_BASE3                                                        (0x1218U)
+#define RGX_CR_BIF_CAT_BASE3_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE3_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE3_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE3_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE3_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE4
+*/
+#define RGX_CR_BIF_CAT_BASE4                                                        (0x1220U)
+#define RGX_CR_BIF_CAT_BASE4_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE4_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE4_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE4_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE4_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE5
+*/
+#define RGX_CR_BIF_CAT_BASE5                                                        (0x1228U)
+#define RGX_CR_BIF_CAT_BASE5_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE5_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE5_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE5_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE5_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE6
+*/
+#define RGX_CR_BIF_CAT_BASE6                                                        (0x1230U)
+#define RGX_CR_BIF_CAT_BASE6_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE6_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE6_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE6_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE6_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE7
+*/
+#define RGX_CR_BIF_CAT_BASE7                                                        (0x1238U)
+#define RGX_CR_BIF_CAT_BASE7_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE7_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE7_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE7_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE7_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE_INDEX
+*/
+#define RGX_CR_BIF_CAT_BASE_INDEX                                                   (0x1240U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_MASKFULL                                          (IMG_UINT64_C(0x00070707073F0707))
+#define RGX_CR_BIF_CAT_BASE_INDEX_RVTX_SHIFT                                        (48U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_RVTX_CLRMSK                                       (IMG_UINT64_C(0xFFF8FFFFFFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_RAY_SHIFT                                         (40U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_RAY_CLRMSK                                        (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_HOST_SHIFT                                        (32U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_HOST_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TLA_SHIFT                                         (24U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TLA_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TDM_SHIFT                                         (19U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TDM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFC7FFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_CDM_SHIFT                                         (16U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_CDM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFF8FFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_PIXEL_SHIFT                                       (8U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_PIXEL_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFF8FF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TA_SHIFT                                          (0U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF8))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_VCE0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0                                                 (0x1248U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_MASKFULL                                        (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_INIT_PAGE_SHIFT                                 (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_INIT_PAGE_CLRMSK                                (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_ADDR_SHIFT                                      (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_SHIFT                                      (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_EN                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_SHIFT                                     (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_EN                                        (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_TE0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_TE0                                                  (0x1250U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_MASKFULL                                         (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_INIT_PAGE_SHIFT                                  (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_INIT_PAGE_CLRMSK                                 (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_ADDR_SHIFT                                       (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_ADDR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_SHIFT                                       (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_SHIFT                                      (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_ALIST0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0                                               (0x1260U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_MASKFULL                                      (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_INIT_PAGE_SHIFT                               (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_INIT_PAGE_CLRMSK                              (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_ADDR_SHIFT                                    (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_SHIFT                                    (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_EN                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_SHIFT                                   (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_VCE1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1                                                 (0x1268U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_MASKFULL                                        (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_INIT_PAGE_SHIFT                                 (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_INIT_PAGE_CLRMSK                                (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_ADDR_SHIFT                                      (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_SHIFT                                      (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_EN                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_SHIFT                                     (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_EN                                        (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_TE1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_TE1                                                  (0x1270U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_MASKFULL                                         (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_INIT_PAGE_SHIFT                                  (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_INIT_PAGE_CLRMSK                                 (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_ADDR_SHIFT                                       (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_ADDR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_SHIFT                                       (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_SHIFT                                      (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_ALIST1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1                                               (0x1280U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_MASKFULL                                      (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_INIT_PAGE_SHIFT                               (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_INIT_PAGE_CLRMSK                              (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_ADDR_SHIFT                                    (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_SHIFT                                    (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_EN                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_SHIFT                                   (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_BIF_MMU_ENTRY_STATUS                                                 (0x1288U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF0F3))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_SHIFT                                   (12U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_CLRMSK                                  (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_SHIFT                                  (4U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFF0F))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_DATA_TYPE_SHIFT                                 (0U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_DATA_TYPE_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_BIF_MMU_ENTRY
+*/
+#define RGX_CR_BIF_MMU_ENTRY                                                        (0x1290U)
+#define RGX_CR_BIF_MMU_ENTRY_MASKFULL                                               (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_SHIFT                                           (1U)
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_EN                                              (0x00000002U)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_SHIFT                                          (0U)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_CTRL_INVAL
+*/
+#define RGX_CR_BIF_CTRL_INVAL                                                       (0x12A0U)
+#define RGX_CR_BIF_CTRL_INVAL_MASKFULL                                              (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_SHIFT                                            (3U)
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_EN                                               (0x00000008U)
+#define RGX_CR_BIF_CTRL_INVAL_PC_SHIFT                                              (2U)
+#define RGX_CR_BIF_CTRL_INVAL_PC_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_BIF_CTRL_INVAL_PC_EN                                                 (0x00000004U)
+#define RGX_CR_BIF_CTRL_INVAL_PD_SHIFT                                              (1U)
+#define RGX_CR_BIF_CTRL_INVAL_PD_CLRMSK                                             (0xFFFFFFFDU)
+#define RGX_CR_BIF_CTRL_INVAL_PD_EN                                                 (0x00000002U)
+#define RGX_CR_BIF_CTRL_INVAL_PT_SHIFT                                              (0U)
+#define RGX_CR_BIF_CTRL_INVAL_PT_CLRMSK                                             (0xFFFFFFFEU)
+#define RGX_CR_BIF_CTRL_INVAL_PT_EN                                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_CTRL
+*/
+#define RGX_CR_BIF_CTRL                                                             (0x12A8U)
+#define RGX_CR_BIF_CTRL__XE_MEM__MASKFULL                                           (IMG_UINT64_C(0x000000000000033F))
+#define RGX_CR_BIF_CTRL_MASKFULL                                                    (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_SHIFT                                (9U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_CLRMSK                               (0xFFFFFDFFU)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_EN                                   (0x00000200U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_SHIFT                               (8U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_CLRMSK                              (0xFFFFFEFFU)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_EN                                  (0x00000100U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_SHIFT                               (7U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_CLRMSK                              (0xFFFFFF7FU)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_EN                                  (0x00000080U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_SHIFT                              (6U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_EN                                 (0x00000040U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_SHIFT                                        (5U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_EN                                           (0x00000020U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_SHIFT                                        (4U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_CLRMSK                                       (0xFFFFFFEFU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_EN                                           (0x00000010U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_SHIFT                                            (3U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_EN                                               (0x00000008U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_SHIFT                                          (2U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_EN                                             (0x00000004U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_SHIFT                                        (1U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_EN                                           (0x00000002U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_SHIFT                                        (0U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK0_MMU_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS                                           (0x12B0U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_MASKFULL                                  (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT                            (12U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK                           (0xFFFF0FFFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT                           (8U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK                          (0xFFFFF8FFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT                           (5U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK                          (0xFFFFFF9FU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_SHIFT                            (4U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN                               (0x00000010U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_SHIFT                    (2U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN                       (0x00000004U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_SHIFT                               (0U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK0_REQ_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS                                           (0x12B8U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__MASKFULL                         (IMG_UINT64_C(0x001FFFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_MASKFULL                                  (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_SHIFT                        (52U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_CLRMSK                       (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN                           (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_SHIFT                                 (50U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_CLRMSK                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT                     (46U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK                    (IMG_UINT64_C(0xFFF03FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT                              (44U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK                             (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT                              (40U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT                     (40U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK                    (IMG_UINT64_C(0xFFFFC0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT                             (4U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK                            (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT                        (4U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSIZE                         (16U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK1_MMU_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS                                           (0x12C0U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_MASKFULL                                  (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_CAT_BASE_SHIFT                            (12U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_CAT_BASE_CLRMSK                           (0xFFFF0FFFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_PAGE_SIZE_SHIFT                           (8U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_PAGE_SIZE_CLRMSK                          (0xFFFFF8FFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_DATA_TYPE_SHIFT                           (5U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_DATA_TYPE_CLRMSK                          (0xFFFFFF9FU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_SHIFT                            (4U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_EN                               (0x00000010U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_SHIFT                    (2U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_EN                       (0x00000004U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_SHIFT                               (0U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK1_REQ_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS                                           (0x12C8U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_MASKFULL                                  (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_SHIFT                                 (50U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_CLRMSK                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_EN                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_SB_SHIFT                              (44U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_SB_CLRMSK                             (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_ID_SHIFT                              (40U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_SHIFT                             (4U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_CLRMSK                            (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_ALIGNSHIFT                        (4U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_ALIGNSIZE                         (16U)
+
+
+/*
+    Register RGX_CR_BIF_MMU_STATUS
+*/
+#define RGX_CR_BIF_MMU_STATUS                                                       (0x12D0U)
+#define RGX_CR_BIF_MMU_STATUS__XE_MEM__MASKFULL                                     (IMG_UINT64_C(0x000000001FFFFFF7))
+#define RGX_CR_BIF_MMU_STATUS_MASKFULL                                              (IMG_UINT64_C(0x000000001FFFFFF7))
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_SHIFT                                        (28U)
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_CLRMSK                                       (0xEFFFFFFFU)
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_EN                                           (0x10000000U)
+#define RGX_CR_BIF_MMU_STATUS_PC_DATA_SHIFT                                         (20U)
+#define RGX_CR_BIF_MMU_STATUS_PC_DATA_CLRMSK                                        (0xF00FFFFFU)
+#define RGX_CR_BIF_MMU_STATUS_PD_DATA_SHIFT                                         (12U)
+#define RGX_CR_BIF_MMU_STATUS_PD_DATA_CLRMSK                                        (0xFFF00FFFU)
+#define RGX_CR_BIF_MMU_STATUS_PT_DATA_SHIFT                                         (4U)
+#define RGX_CR_BIF_MMU_STATUS_PT_DATA_CLRMSK                                        (0xFFFFF00FU)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_SHIFT                                         (2U)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_EN                                            (0x00000004U)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_SHIFT                                          (1U)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_EN                                             (0x00000002U)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_SHIFT                                            (0U)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_EN                                               (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_BIF_TILING_CFG, with 8 repeats
+*/
+#define RGX_CR_BIF_TILING_CFG_REPEATCOUNT                                           (8U)
+/*
+    Register RGX_CR_BIF_TILING_CFG0
+*/
+#define RGX_CR_BIF_TILING_CFG0                                                      (0x12D8U)
+#define RGX_CR_BIF_TILING_CFG0_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG0_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG1
+*/
+#define RGX_CR_BIF_TILING_CFG1                                                      (0x12E0U)
+#define RGX_CR_BIF_TILING_CFG1_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG1_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG2
+*/
+#define RGX_CR_BIF_TILING_CFG2                                                      (0x12E8U)
+#define RGX_CR_BIF_TILING_CFG2_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG2_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG3
+*/
+#define RGX_CR_BIF_TILING_CFG3                                                      (0x12F0U)
+#define RGX_CR_BIF_TILING_CFG3_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG3_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG4
+*/
+#define RGX_CR_BIF_TILING_CFG4                                                      (0x12F8U)
+#define RGX_CR_BIF_TILING_CFG4_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG4_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG5
+*/
+#define RGX_CR_BIF_TILING_CFG5                                                      (0x1300U)
+#define RGX_CR_BIF_TILING_CFG5_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG5_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG6
+*/
+#define RGX_CR_BIF_TILING_CFG6                                                      (0x1308U)
+#define RGX_CR_BIF_TILING_CFG6_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG6_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG7
+*/
+#define RGX_CR_BIF_TILING_CFG7                                                      (0x1310U)
+#define RGX_CR_BIF_TILING_CFG7_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG7_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_READS_EXT_STATUS
+*/
+#define RGX_CR_BIF_READS_EXT_STATUS                                                 (0x1320U)
+#define RGX_CR_BIF_READS_EXT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_BIF_READS_EXT_STATUS_MMU_SHIFT                                       (16U)
+#define RGX_CR_BIF_READS_EXT_STATUS_MMU_CLRMSK                                      (0xF000FFFFU)
+#define RGX_CR_BIF_READS_EXT_STATUS_BANK1_SHIFT                                     (0U)
+#define RGX_CR_BIF_READS_EXT_STATUS_BANK1_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_READS_INT_STATUS
+*/
+#define RGX_CR_BIF_READS_INT_STATUS                                                 (0x1328U)
+#define RGX_CR_BIF_READS_INT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000007FFFFFF))
+#define RGX_CR_BIF_READS_INT_STATUS_MMU_SHIFT                                       (16U)
+#define RGX_CR_BIF_READS_INT_STATUS_MMU_CLRMSK                                      (0xF800FFFFU)
+#define RGX_CR_BIF_READS_INT_STATUS_BANK1_SHIFT                                     (0U)
+#define RGX_CR_BIF_READS_INT_STATUS_BANK1_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_READS_INT_STATUS
+*/
+#define RGX_CR_BIFPM_READS_INT_STATUS                                               (0x1330U)
+#define RGX_CR_BIFPM_READS_INT_STATUS_MASKFULL                                      (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIFPM_READS_INT_STATUS_BANK0_SHIFT                                   (0U)
+#define RGX_CR_BIFPM_READS_INT_STATUS_BANK0_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_READS_EXT_STATUS
+*/
+#define RGX_CR_BIFPM_READS_EXT_STATUS                                               (0x1338U)
+#define RGX_CR_BIFPM_READS_EXT_STATUS_MASKFULL                                      (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIFPM_READS_EXT_STATUS_BANK0_SHIFT                                   (0U)
+#define RGX_CR_BIFPM_READS_EXT_STATUS_BANK0_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_STATUS_MMU
+*/
+#define RGX_CR_BIFPM_STATUS_MMU                                                     (0x1350U)
+#define RGX_CR_BIFPM_STATUS_MMU_MASKFULL                                            (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIFPM_STATUS_MMU_REQUESTS_SHIFT                                      (0U)
+#define RGX_CR_BIFPM_STATUS_MMU_REQUESTS_CLRMSK                                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_STATUS_MMU
+*/
+#define RGX_CR_BIF_STATUS_MMU                                                       (0x1358U)
+#define RGX_CR_BIF_STATUS_MMU_MASKFULL                                              (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIF_STATUS_MMU_REQUESTS_SHIFT                                        (0U)
+#define RGX_CR_BIF_STATUS_MMU_REQUESTS_CLRMSK                                       (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_READ
+*/
+#define RGX_CR_BIF_FAULT_READ                                                       (0x13E0U)
+#define RGX_CR_BIF_FAULT_READ_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_SHIFT                                         (4U)
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_ALIGNSHIFT                                    (4U)
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_ALIGNSIZE                                     (16U)
+
+
+/*
+    Register RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS
+*/
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS                                     (0x1430U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_MASKFULL                            (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT                      (12U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK                     (0xFFFF0FFFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT                     (8U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK                    (0xFFFFF8FFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT                     (5U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK                    (0xFFFFFF9FU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_SHIFT                      (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_CLRMSK                     (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN                         (0x00000010U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_SHIFT              (2U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_CLRMSK             (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN                 (0x00000004U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_SHIFT                         (0U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_EN                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS
+*/
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS                                     (0x1438U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_MASKFULL                            (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_SHIFT                           (50U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_CLRMSK                          (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN                              (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT                        (44U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK                       (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT                        (40U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK                       (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT                       (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK                      (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT                  (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSIZE                   (16U)
+
+
+/*
+    Register RGX_CR_TFBC_COMPRESSION_CONTROL
+*/
+#define RGX_CR_TFBC_COMPRESSION_CONTROL                                             (0x14A0U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_MASKFULL                                    (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_SHIFT            (8U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_CLRMSK           (0xFFFFFEFFU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN               (0x00000100U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_SHIFT                        (7U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN                           (0x00000080U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_SHIFT_SHIFT                         (4U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_SHIFT_CLRMSK                        (0xFFFFFF8FU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_SHIFT                        (3U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_CLRMSK                       (0xFFFFFFF7U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_EN                           (0x00000008U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT                                (1U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK                               (0xFFFFFFF9U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT                              (0x00000000U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION  (0x00000002U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD                  (0x00000004U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_RESERVED                             (0x00000006U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT                         (0U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0                       (0x00000000U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_MCU_FENCE
+*/
+#define RGX_CR_MCU_FENCE                                                            (0x1740U)
+#define RGX_CR_MCU_FENCE_MASKFULL                                                   (IMG_UINT64_C(0x000007FFFFFFFFE0))
+#define RGX_CR_MCU_FENCE_DM_SHIFT                                                   (40U)
+#define RGX_CR_MCU_FENCE_DM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_MCU_FENCE_DM_VERTEX                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MCU_FENCE_DM_PIXEL                                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MCU_FENCE_DM_COMPUTE                                                 (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MCU_FENCE_DM_RAY_VERTEX                                              (IMG_UINT64_C(0x0000030000000000))
+#define RGX_CR_MCU_FENCE_DM_RAY                                                     (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MCU_FENCE_DM_FASTRENDER                                              (IMG_UINT64_C(0x0000050000000000))
+#define RGX_CR_MCU_FENCE_ADDR_SHIFT                                                 (5U)
+#define RGX_CR_MCU_FENCE_ADDR_CLRMSK                                                (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_CR_MCU_FENCE_ADDR_ALIGNSHIFT                                            (5U)
+#define RGX_CR_MCU_FENCE_ADDR_ALIGNSIZE                                             (32U)
+
+
+/*
+    Register group: RGX_CR_SCRATCH, with 16 repeats
+*/
+#define RGX_CR_SCRATCH_REPEATCOUNT                                                  (16U)
+/*
+    Register RGX_CR_SCRATCH0
+*/
+#define RGX_CR_SCRATCH0                                                             (0x1A00U)
+#define RGX_CR_SCRATCH0_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH0_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH0_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH1
+*/
+#define RGX_CR_SCRATCH1                                                             (0x1A08U)
+#define RGX_CR_SCRATCH1_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH1_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH1_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH2
+*/
+#define RGX_CR_SCRATCH2                                                             (0x1A10U)
+#define RGX_CR_SCRATCH2_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH2_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH2_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH3
+*/
+#define RGX_CR_SCRATCH3                                                             (0x1A18U)
+#define RGX_CR_SCRATCH3_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH3_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH3_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH4
+*/
+#define RGX_CR_SCRATCH4                                                             (0x1A20U)
+#define RGX_CR_SCRATCH4_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH4_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH4_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH5
+*/
+#define RGX_CR_SCRATCH5                                                             (0x1A28U)
+#define RGX_CR_SCRATCH5_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH5_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH5_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH6
+*/
+#define RGX_CR_SCRATCH6                                                             (0x1A30U)
+#define RGX_CR_SCRATCH6_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH6_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH6_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH7
+*/
+#define RGX_CR_SCRATCH7                                                             (0x1A38U)
+#define RGX_CR_SCRATCH7_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH7_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH7_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH8
+*/
+#define RGX_CR_SCRATCH8                                                             (0x1A40U)
+#define RGX_CR_SCRATCH8_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH8_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH8_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH9
+*/
+#define RGX_CR_SCRATCH9                                                             (0x1A48U)
+#define RGX_CR_SCRATCH9_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH9_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH9_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH10
+*/
+#define RGX_CR_SCRATCH10                                                            (0x1A50U)
+#define RGX_CR_SCRATCH10_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH10_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH10_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH11
+*/
+#define RGX_CR_SCRATCH11                                                            (0x1A58U)
+#define RGX_CR_SCRATCH11_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH11_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH11_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH12
+*/
+#define RGX_CR_SCRATCH12                                                            (0x1A60U)
+#define RGX_CR_SCRATCH12_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH12_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH12_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH13
+*/
+#define RGX_CR_SCRATCH13                                                            (0x1A68U)
+#define RGX_CR_SCRATCH13_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH13_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH13_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH14
+*/
+#define RGX_CR_SCRATCH14                                                            (0x1A70U)
+#define RGX_CR_SCRATCH14_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH14_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH14_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH15
+*/
+#define RGX_CR_SCRATCH15                                                            (0x1A78U)
+#define RGX_CR_SCRATCH15_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH15_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH15_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS0_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS0_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS0_SCRATCH0
+*/
+#define RGX_CR_OS0_SCRATCH0                                                         (0x1A80U)
+#define RGX_CR_OS0_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH1
+*/
+#define RGX_CR_OS0_SCRATCH1                                                         (0x1A88U)
+#define RGX_CR_OS0_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH2
+*/
+#define RGX_CR_OS0_SCRATCH2                                                         (0x1A90U)
+#define RGX_CR_OS0_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS0_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH3
+*/
+#define RGX_CR_OS0_SCRATCH3                                                         (0x1A98U)
+#define RGX_CR_OS0_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS0_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS1_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS1_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS1_SCRATCH0
+*/
+#define RGX_CR_OS1_SCRATCH0                                                         (0x11A80U)
+#define RGX_CR_OS1_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH1
+*/
+#define RGX_CR_OS1_SCRATCH1                                                         (0x11A88U)
+#define RGX_CR_OS1_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH2
+*/
+#define RGX_CR_OS1_SCRATCH2                                                         (0x11A90U)
+#define RGX_CR_OS1_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS1_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH3
+*/
+#define RGX_CR_OS1_SCRATCH3                                                         (0x11A98U)
+#define RGX_CR_OS1_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS1_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS2_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS2_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS2_SCRATCH0
+*/
+#define RGX_CR_OS2_SCRATCH0                                                         (0x21A80U)
+#define RGX_CR_OS2_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH1
+*/
+#define RGX_CR_OS2_SCRATCH1                                                         (0x21A88U)
+#define RGX_CR_OS2_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH2
+*/
+#define RGX_CR_OS2_SCRATCH2                                                         (0x21A90U)
+#define RGX_CR_OS2_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS2_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH3
+*/
+#define RGX_CR_OS2_SCRATCH3                                                         (0x21A98U)
+#define RGX_CR_OS2_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS2_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS3_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS3_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS3_SCRATCH0
+*/
+#define RGX_CR_OS3_SCRATCH0                                                         (0x31A80U)
+#define RGX_CR_OS3_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH1
+*/
+#define RGX_CR_OS3_SCRATCH1                                                         (0x31A88U)
+#define RGX_CR_OS3_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH2
+*/
+#define RGX_CR_OS3_SCRATCH2                                                         (0x31A90U)
+#define RGX_CR_OS3_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS3_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH3
+*/
+#define RGX_CR_OS3_SCRATCH3                                                         (0x31A98U)
+#define RGX_CR_OS3_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS3_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS4_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS4_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS4_SCRATCH0
+*/
+#define RGX_CR_OS4_SCRATCH0                                                         (0x41A80U)
+#define RGX_CR_OS4_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH1
+*/
+#define RGX_CR_OS4_SCRATCH1                                                         (0x41A88U)
+#define RGX_CR_OS4_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH2
+*/
+#define RGX_CR_OS4_SCRATCH2                                                         (0x41A90U)
+#define RGX_CR_OS4_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS4_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH3
+*/
+#define RGX_CR_OS4_SCRATCH3                                                         (0x41A98U)
+#define RGX_CR_OS4_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS4_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS5_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS5_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS5_SCRATCH0
+*/
+#define RGX_CR_OS5_SCRATCH0                                                         (0x51A80U)
+#define RGX_CR_OS5_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH1
+*/
+#define RGX_CR_OS5_SCRATCH1                                                         (0x51A88U)
+#define RGX_CR_OS5_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH2
+*/
+#define RGX_CR_OS5_SCRATCH2                                                         (0x51A90U)
+#define RGX_CR_OS5_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS5_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH3
+*/
+#define RGX_CR_OS5_SCRATCH3                                                         (0x51A98U)
+#define RGX_CR_OS5_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS5_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS6_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS6_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS6_SCRATCH0
+*/
+#define RGX_CR_OS6_SCRATCH0                                                         (0x61A80U)
+#define RGX_CR_OS6_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH1
+*/
+#define RGX_CR_OS6_SCRATCH1                                                         (0x61A88U)
+#define RGX_CR_OS6_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH2
+*/
+#define RGX_CR_OS6_SCRATCH2                                                         (0x61A90U)
+#define RGX_CR_OS6_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS6_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH3
+*/
+#define RGX_CR_OS6_SCRATCH3                                                         (0x61A98U)
+#define RGX_CR_OS6_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS6_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS7_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS7_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS7_SCRATCH0
+*/
+#define RGX_CR_OS7_SCRATCH0                                                         (0x71A80U)
+#define RGX_CR_OS7_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH1
+*/
+#define RGX_CR_OS7_SCRATCH1                                                         (0x71A88U)
+#define RGX_CR_OS7_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH2
+*/
+#define RGX_CR_OS7_SCRATCH2                                                         (0x71A90U)
+#define RGX_CR_OS7_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS7_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH3
+*/
+#define RGX_CR_OS7_SCRATCH3                                                         (0x71A98U)
+#define RGX_CR_OS7_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS7_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_SPFILTER_SIGNAL_DESCR
+*/
+#define RGX_CR_SPFILTER_SIGNAL_DESCR                                                (0x2700U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MASKFULL                                       (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_SHIFT                                     (0U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_CLRMSK                                    (0xFFFF0000U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_ALIGNSHIFT                                (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_SPFILTER_SIGNAL_DESCR_MIN
+*/
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN                                            (0x2708U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_SHIFT                                 (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSHIFT                            (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSIZE                             (16U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_ADDR_REMAP_CONFIG, with 16 repeats
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG_REPEATCOUNT                                 (16U)
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0                                            (0x3000U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1                                            (0x3008U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2                                            (0x3010U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3                                            (0x3018U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4                                            (0x3020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5                                            (0x3028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6                                            (0x3030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7                                            (0x3038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8                                            (0x3040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9                                            (0x3048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10                                           (0x3050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11                                           (0x3058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12                                           (0x3060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13                                           (0x3068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14                                           (0x3070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15                                           (0x3078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_BOOT
+*/
+#define RGX_CR_FWCORE_BOOT                                                          (0x3090U)
+#define RGX_CR_FWCORE_BOOT_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_BOOT_ENABLE_SHIFT                                             (0U)
+#define RGX_CR_FWCORE_BOOT_ENABLE_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_BOOT_ENABLE_EN                                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_RESET_ADDR
+*/
+#define RGX_CR_FWCORE_RESET_ADDR                                                    (0x3098U)
+#define RGX_CR_FWCORE_RESET_ADDR_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_SHIFT                                         (1U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_CLRMSK                                        (0x00000001U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_ALIGNSHIFT                                    (1U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_ALIGNSIZE                                     (2U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_ADDR
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR                                              (0x30A0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_SHIFT                                   (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_CLRMSK                                  (0x00000001U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_ALIGNSHIFT                              (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_ALIGNSIZE                               (2U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT                                             (0x30A8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_SHIFT                            (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS                                          (0x30B0U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_MASKFULL                                 (IMG_UINT64_C(0x000000000000F771))
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_CAT_BASE_SHIFT                           (12U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_CAT_BASE_CLRMSK                          (0xFFFF0FFFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_SHIFT                          (8U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_CLRMSK                         (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_DATA_TYPE_SHIFT                          (5U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_DATA_TYPE_CLRMSK                         (0xFFFFFF9FU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_SHIFT                           (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_CLRMSK                          (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_EN                              (0x00000010U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_SHIFT                              (0U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS                                          (0x30B8U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_MASKFULL                                 (IMG_UINT64_C(0x001FFFFFFFFFFFF0))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_SHIFT                                (52U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_CLRMSK                               (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN                                   (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT                             (46U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK                            (IMG_UINT64_C(0xFFF03FFFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT                             (40U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK                            (IMG_UINT64_C(0xFFFFC0FFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT                            (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK                           (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT                       (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSIZE                        (16U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CTRL_INVAL
+*/
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL                                                (0x30C0U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_MASKFULL                                       (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_SHIFT                                      (3U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_EN                                         (0x00000008U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_SHIFT                                       (2U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_CLRMSK                                      (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_EN                                          (0x00000004U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_SHIFT                                       (1U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_EN                                          (0x00000002U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_SHIFT                                       (0U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_MMU_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_MMU_STATUS                                                (0x30C8U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000FFFFFF7))
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PC_DATA_SHIFT                                  (20U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PC_DATA_CLRMSK                                 (0xF00FFFFFU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PD_DATA_SHIFT                                  (12U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PD_DATA_CLRMSK                                 (0xFFF00FFFU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PT_DATA_SHIFT                                  (4U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PT_DATA_CLRMSK                                 (0xFFFFF00FU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_SHIFT                                  (2U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_EN                                     (0x00000004U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_SHIFT                                   (1U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_EN                                      (0x00000002U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_READS_EXT_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS                                          (0x30D8U)
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MMU_SHIFT                                (0U)
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MMU_CLRMSK                               (0xFFFFF000U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_READS_INT_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS                                          (0x30E0U)
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x00000000000007FF))
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MMU_SHIFT                                (0U)
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MMU_CLRMSK                               (0xFFFFF800U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_FENCE
+*/
+#define RGX_CR_FWCORE_WRAPPER_FENCE                                                 (0x30E8U)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_EN                                           (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_MEM_CAT_BASE, with 8 repeats
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE_REPEATCOUNT                                      (8U)
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE0
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE0                                                 (0x30F0U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE1
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE1                                                 (0x30F8U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE2
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE2                                                 (0x3100U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE3
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE3                                                 (0x3108U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE4
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE4                                                 (0x3110U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE5
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE5                                                 (0x3118U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE6
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE6                                                 (0x3120U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE7
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE7                                                 (0x3128U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_RESET
+*/
+#define RGX_CR_FWCORE_WDT_RESET                                                     (0x3130U)
+#define RGX_CR_FWCORE_WDT_RESET_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WDT_RESET_EN_SHIFT                                            (0U)
+#define RGX_CR_FWCORE_WDT_RESET_EN_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_RESET_EN_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_CTRL
+*/
+#define RGX_CR_FWCORE_WDT_CTRL                                                      (0x3138U)
+#define RGX_CR_FWCORE_WDT_CTRL_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFF1F01))
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_SHIFT                                           (16U)
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_CLRMSK                                          (0x0000FFFFU)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_SHIFT                                      (8U)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_CLRMSK                                     (0xFFFFE0FFU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_COUNT
+*/
+#define RGX_CR_FWCORE_WDT_COUNT                                                     (0x3140U)
+#define RGX_CR_FWCORE_WDT_COUNT_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED0, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED0_REPEATCOUNT                                     (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00                                                (0x3400U)
+#define RGX_CR_FWCORE_DMI_RESERVED00_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01                                                (0x3408U)
+#define RGX_CR_FWCORE_DMI_RESERVED01_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02                                                (0x3410U)
+#define RGX_CR_FWCORE_DMI_RESERVED02_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03                                                (0x3418U)
+#define RGX_CR_FWCORE_DMI_RESERVED03_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0                                                     (0x3420U)
+#define RGX_CR_FWCORE_DMI_DATA0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1                                                     (0x3428U)
+#define RGX_CR_FWCORE_DMI_DATA1_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED1, with 10 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED1_REPEATCOUNT                                     (10U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10                                                (0x3430U)
+#define RGX_CR_FWCORE_DMI_RESERVED10_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11                                                (0x3438U)
+#define RGX_CR_FWCORE_DMI_RESERVED11_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12                                                (0x3440U)
+#define RGX_CR_FWCORE_DMI_RESERVED12_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13                                                (0x3448U)
+#define RGX_CR_FWCORE_DMI_RESERVED13_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14                                                (0x3450U)
+#define RGX_CR_FWCORE_DMI_RESERVED14_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL                                                 (0x3480U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_MASKFULL                                        (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_SHIFT                                   (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_CLRMSK                                  (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN                                      (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_SHIFT                                 (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_CLRMSK                                (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN                                    (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_SHIFT                              (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_EN                                 (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_SHIFT                                  (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_EN                                     (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS                                                  (0x3488U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_SHIFT                                  (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_EN                                     (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_SHIFT                               (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_EN                                  (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_SHIFT                               (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_EN                                  (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_SHIFT                               (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_CLRMSK                              (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN                                  (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_SHIFT                               (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_EN                                  (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_SHIFT                             (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_EN                                (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_SHIFT                             (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_EN                                (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_SHIFT                                 (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_CLRMSK                                (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_EN                                    (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_SHIFT                                 (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_CLRMSK                                (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_EN                                    (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_SHIFT                                 (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_CLRMSK                                (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_EN                                    (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_SHIFT                                 (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_CLRMSK                                (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_EN                                    (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_SHIFT                                  (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN                                     (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_SHIFT                                  (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_EN                                     (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_SHIFT                              (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_EN                                 (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_SHIFT                                   (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_CLRMSK                                  (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_EN                                      (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_SHIFT                            (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_CLRMSK                           (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_EN                               (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_SHIFT                            (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_EN                               (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_SHIFT                                    (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_CLRMSK                                   (0xFFFFFFF0U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED2, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED2_REPEATCOUNT                                     (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20                                                (0x3490U)
+#define RGX_CR_FWCORE_DMI_RESERVED20_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21                                                (0x3498U)
+#define RGX_CR_FWCORE_DMI_RESERVED21_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22                                                (0x34A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED22_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23                                                (0x34A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED23_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS                                                (0x34B0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_MASKFULL                                       (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_SHIFT                              (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_CLRMSK                             (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_SHIFT                                     (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN                                        (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT                                   (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK                                  (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_SHIFT                                (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_CLRMSK                               (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND                                                   (0x34B8U)
+#define RGX_CR_FWCORE_DMI_COMMAND_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT                                     (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_CLRMSK                                    (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_CLRMSK                                    (0xFF000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED3, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED3_REPEATCOUNT                                     (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30                                                (0x34C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED30_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31                                                (0x34C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED31_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS                                                      (0x35C0U)
+#define RGX_CR_FWCORE_DMI_SBCS_MASKFULL                                             (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_SHIFT                                      (29U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_CLRMSK                                     (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_SHIFT                                    (22U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_CLRMSK                                   (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_EN                                       (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_SHIFT                                         (21U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN                                            (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_SHIFT                                   (20U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN                                      (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT                                       (17U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_CLRMSK                                      (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_SHIFT                                (16U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_CLRMSK                               (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_EN                                   (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_SHIFT                                   (15U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_CLRMSK                                  (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_EN                                      (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT                                        (12U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK                                       (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_SHIFT                                        (5U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_CLRMSK                                       (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_SHIFT                                    (4U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_EN                                       (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_SHIFT                                     (3U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_EN                                        (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_SHIFT                                     (2U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_EN                                        (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_SHIFT                                     (1U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_EN                                        (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_SHIFT                                      (0U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0                                                (0x35C8U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_SBDATA, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA_REPEATCOUNT                                        (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0                                                   (0x35E0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA0_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1                                                   (0x35E8U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA1_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2                                                   (0x35F0U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA2_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3                                                   (0x35F8U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA3_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0                                                  (0x3600U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_SLC_CTRL_MISC
+*/
+#define RGX_CR_SLC_CTRL_MISC                                                        (0x3800U)
+#define RGX_CR_SLC_CTRL_MISC_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFF03FF010F))
+#define RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS_SHIFT                                    (32U)
+#define RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS_CLRMSK                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_SHIFT                             (25U)
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_EN                                (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_SHIFT                                  (24U)
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_EN                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SHIFT                                 (16U)
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_INTERLEAVED_64_BYTE                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_INTERLEAVED_128_BYTE                  (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SIMPLE_HASH1                          (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SIMPLE_HASH2                          (IMG_UINT64_C(0x0000000000110000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH2_SCRAMBLE                    (IMG_UINT64_C(0x0000000000210000))
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_SHIFT                                            (8U)
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_EN                                               (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_SHIFT                                    (3U)
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_EN                                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_SHIFT                            (2U)
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_EN                               (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_SHIFT                          (1U)
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_SHIFT                            (0U)
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC_CTRL_FLUSH_INVAL
+*/
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL                                                 (0x3818U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_MASKFULL                                        (IMG_UINT64_C(0x0000000080000FFF))
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_SHIFT                                      (31U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_CLRMSK                                     (0x7FFFFFFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_EN                                         (0x80000000U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_SHIFT                             (11U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_EN                                (0x00000800U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_SHIFT                             (10U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_CLRMSK                            (0xFFFFFBFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_EN                                (0x00000400U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_SHIFT                                    (9U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_EN                                       (0x00000200U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_SHIFT                                    (8U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_EN                                       (0x00000100U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_SHIFT                                    (7U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_CLRMSK                                   (0xFFFFFF7FU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_EN                                       (0x00000080U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_SHIFT                                    (6U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_CLRMSK                                   (0xFFFFFFBFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_EN                                       (0x00000040U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_SHIFT                              (5U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_EN                                 (0x00000020U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_SHIFT                                    (4U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_EN                                       (0x00000010U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_SHIFT                                (3U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_EN                                   (0x00000008U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_SHIFT                                  (2U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_EN                                     (0x00000004U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_SHIFT                                     (1U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_EN                                        (0x00000002U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_SHIFT                                       (0U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS0
+*/
+#define RGX_CR_SLC_STATUS0                                                          (0x3820U)
+#define RGX_CR_SLC_STATUS0_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_SHIFT                                (2U)
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_EN                                   (0x00000004U)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_SHIFT                                      (1U)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_EN                                         (0x00000002U)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_SHIFT                                      (0U)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_CTRL_BYPASS
+*/
+#define RGX_CR_SLC_CTRL_BYPASS                                                      (0x3828U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__MASKFULL                                    (IMG_UINT64_C(0x0FFFFFFFFFFF7FFF))
+#define RGX_CR_SLC_CTRL_BYPASS_MASKFULL                                             (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_SHIFT                     (59U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_CLRMSK                    (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_EN                        (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_SHIFT            (58U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_CLRMSK           (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_EN               (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_SHIFT            (57U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_CLRMSK           (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_EN               (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_SHIFT              (56U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_CLRMSK             (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_EN                 (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_SHIFT              (55U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_CLRMSK             (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_EN                 (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_SHIFT                     (54U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_CLRMSK                    (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_EN                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_SHIFT                    (53U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_CLRMSK                   (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_EN                       (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_SHIFT                      (52U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_CLRMSK                     (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_EN                         (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_SHIFT                        (51U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_CLRMSK                       (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_EN                           (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_SHIFT                               (50U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_CLRMSK                              (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_EN                                  (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_SHIFT                          (49U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_CLRMSK                         (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_EN                             (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_SHIFT                          (48U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_CLRMSK                         (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_EN                             (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_SHIFT                          (47U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_CLRMSK                         (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_EN                             (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_SHIFT                          (46U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_CLRMSK                         (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_EN                             (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_SHIFT                               (45U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_CLRMSK                              (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_EN                                  (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_SHIFT                               (44U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_CLRMSK                              (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_EN                                  (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_SHIFT                               (43U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_CLRMSK                              (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_EN                                  (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_SHIFT                                (42U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_CLRMSK                               (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_EN                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_SHIFT                               (41U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_EN                                  (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_SHIFT                               (40U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_EN                                  (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_SHIFT                    (39U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_CLRMSK                   (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_EN                       (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_SHIFT                           (38U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_CLRMSK                          (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_EN                              (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_SHIFT                      (37U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_CLRMSK                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_EN                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_SHIFT                               (36U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_EN                                  (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_SHIFT                     (35U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_CLRMSK                    (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_EN                        (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_SHIFT                     (34U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_EN                        (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_SHIFT                            (33U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_EN                               (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_SHIFT                            (32U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_EN                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_SHIFT                         (31U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_EN                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_SHIFT                           (30U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_EN                              (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_SHIFT                              (29U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_EN                                 (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_SHIFT                             (28U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_EN                                (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_SHIFT                                  (27U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_EN                                     (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_SHIFT                                         (26U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_EN                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_SHIFT                                    (25U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_EN                                       (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_SHIFT                                        (24U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_EN                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_SHIFT                                       (23U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_EN                                          (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_SHIFT                                        (22U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_EN                                           (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_SHIFT                                       (21U)
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_EN                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_SHIFT                                         (20U)
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_EN                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_SHIFT                                        (19U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_EN                                           (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_SHIFT                                        (18U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_EN                                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_SHIFT                                        (17U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_EN                                           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_SHIFT                                     (16U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_EN                                        (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_SHIFT                                    (15U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN                                       (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_SHIFT                                        (14U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_EN                                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_SHIFT                                       (13U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_SHIFT                                       (12U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_SHIFT                                     (11U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_EN                                        (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_SHIFT                                     (10U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_EN                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_SHIFT                                     (9U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_EN                                        (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_SHIFT                                         (8U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_EN                                            (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_SHIFT                                         (7U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_EN                                            (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_SHIFT                                         (6U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_EN                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_SHIFT                                   (5U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_EN                                      (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_SHIFT                                         (4U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_EN                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_SHIFT                                     (3U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_EN                                        (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_SHIFT                                       (2U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_EN                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_SHIFT                                          (1U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_SHIFT                                            (0U)
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_EN                                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC_STATUS1
+*/
+#define RGX_CR_SLC_STATUS1                                                          (0x3870U)
+#define RGX_CR_SLC_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0x800003FF03FFFFFF))
+#define RGX_CR_SLC_STATUS1_PAUSED_SHIFT                                             (63U)
+#define RGX_CR_SLC_STATUS1_PAUSED_CLRMSK                                            (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_PAUSED_EN                                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_SLC_STATUS1_READS1_SHIFT                                             (32U)
+#define RGX_CR_SLC_STATUS1_READS1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_CR_SLC_STATUS1_READS0_SHIFT                                             (16U)
+#define RGX_CR_SLC_STATUS1_READS0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_CR_SLC_STATUS1_READS1_EXT_SHIFT                                         (8U)
+#define RGX_CR_SLC_STATUS1_READS1_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_SLC_STATUS1_READS0_EXT_SHIFT                                         (0U)
+#define RGX_CR_SLC_STATUS1_READS0_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_SLC_IDLE
+*/
+#define RGX_CR_SLC_IDLE                                                             (0x3898U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MASKFULL                                           (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_SLC_IDLE_MASKFULL                                                    (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_SHIFT                                   (9U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_CLRMSK                                  (0xFFFFFDFFU)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_EN                                      (0x00000200U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_SHIFT                                   (8U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_CLRMSK                                  (0xFFFFFEFFU)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_EN                                      (0x00000100U)
+#define RGX_CR_SLC_IDLE_IMGBV4_SHIFT                                                (7U)
+#define RGX_CR_SLC_IDLE_IMGBV4_CLRMSK                                               (0xFFFFFF7FU)
+#define RGX_CR_SLC_IDLE_IMGBV4_EN                                                   (0x00000080U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_SHIFT                                           (6U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_EN                                              (0x00000040U)
+#define RGX_CR_SLC_IDLE_RBOFIFO_SHIFT                                               (5U)
+#define RGX_CR_SLC_IDLE_RBOFIFO_CLRMSK                                              (0xFFFFFFDFU)
+#define RGX_CR_SLC_IDLE_RBOFIFO_EN                                                  (0x00000020U)
+#define RGX_CR_SLC_IDLE_FRC_CONV_SHIFT                                              (4U)
+#define RGX_CR_SLC_IDLE_FRC_CONV_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_SLC_IDLE_FRC_CONV_EN                                                 (0x00000010U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_SHIFT                                              (3U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_CLRMSK                                             (0xFFFFFFF7U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_EN                                                 (0x00000008U)
+#define RGX_CR_SLC_IDLE_VXD_CONV_SHIFT                                              (2U)
+#define RGX_CR_SLC_IDLE_VXD_CONV_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_SLC_IDLE_VXD_CONV_EN                                                 (0x00000004U)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_SHIFT                                             (1U)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_CLRMSK                                            (0xFFFFFFFDU)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_EN                                                (0x00000002U)
+#define RGX_CR_SLC_IDLE_CBAR_SHIFT                                                  (0U)
+#define RGX_CR_SLC_IDLE_CBAR_CLRMSK                                                 (0xFFFFFFFEU)
+#define RGX_CR_SLC_IDLE_CBAR_EN                                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS2
+*/
+#define RGX_CR_SLC_STATUS2                                                          (0x3908U)
+#define RGX_CR_SLC_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x000003FF03FFFFFF))
+#define RGX_CR_SLC_STATUS2_READS3_SHIFT                                             (32U)
+#define RGX_CR_SLC_STATUS2_READS3_CLRMSK                                            (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_CR_SLC_STATUS2_READS2_SHIFT                                             (16U)
+#define RGX_CR_SLC_STATUS2_READS2_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_CR_SLC_STATUS2_READS3_EXT_SHIFT                                         (8U)
+#define RGX_CR_SLC_STATUS2_READS3_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_SLC_STATUS2_READS2_EXT_SHIFT                                         (0U)
+#define RGX_CR_SLC_STATUS2_READS2_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_SLC_CTRL_MISC2
+*/
+#define RGX_CR_SLC_CTRL_MISC2                                                       (0x3930U)
+#define RGX_CR_SLC_CTRL_MISC2_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC2_SCRAMBLE_BITS_SHIFT                                   (0U)
+#define RGX_CR_SLC_CTRL_MISC2_SCRAMBLE_BITS_CLRMSK                                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_SLC_CROSSBAR_LOAD_BALANCE
+*/
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE                                            (0x3938U)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_MASKFULL                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_SHIFT                               (0U)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_SIZE_IN_KB
+*/
+#define RGX_CR_SLC_SIZE_IN_KB                                                       (0x3970U)
+#define RGX_CR_SLC_SIZE_IN_KB_MASKFULL                                              (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_SLC_SIZE_IN_KB_SIZE_SHIFT                                            (0U)
+#define RGX_CR_SLC_SIZE_IN_KB_SIZE_CLRMSK                                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_USC_TIMER
+*/
+#define RGX_CR_USC_TIMER                                                            (0x46C8U)
+#define RGX_CR_USC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_TIMER_CNT_SHIFT                                                  (0U)
+#define RGX_CR_USC_TIMER_CNT_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_TIMER_CNT
+*/
+#define RGX_CR_USC_TIMER_CNT                                                        (0x46D0U)
+#define RGX_CR_USC_TIMER_CNT_MASKFULL                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_USC_TIMER_CNT_RESET_SHIFT                                            (0U)
+#define RGX_CR_USC_TIMER_CNT_RESET_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_USC_TIMER_CNT_RESET_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_USC_UVS0_CHECKSUM
+*/
+#define RGX_CR_USC_UVS0_CHECKSUM                                                    (0x5000U)
+#define RGX_CR_USC_UVS0_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS0_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS0_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS1_CHECKSUM
+*/
+#define RGX_CR_USC_UVS1_CHECKSUM                                                    (0x5008U)
+#define RGX_CR_USC_UVS1_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS1_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS1_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS2_CHECKSUM
+*/
+#define RGX_CR_USC_UVS2_CHECKSUM                                                    (0x5010U)
+#define RGX_CR_USC_UVS2_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS2_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS2_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS3_CHECKSUM
+*/
+#define RGX_CR_USC_UVS3_CHECKSUM                                                    (0x5018U)
+#define RGX_CR_USC_UVS3_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS3_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS3_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_SIGNATURE
+*/
+#define RGX_CR_PPP_SIGNATURE                                                        (0x5020U)
+#define RGX_CR_PPP_SIGNATURE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_SIGNATURE_VALUE_SHIFT                                            (0U)
+#define RGX_CR_PPP_SIGNATURE_VALUE_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_SIGNATURE
+*/
+#define RGX_CR_TE_SIGNATURE                                                         (0x5028U)
+#define RGX_CR_TE_SIGNATURE_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_SIGNATURE_VALUE_SHIFT                                             (0U)
+#define RGX_CR_TE_SIGNATURE_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_CHECKSUM
+*/
+#define RGX_CR_TE_CHECKSUM                                                          (0x5110U)
+#define RGX_CR_TE_CHECKSUM_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_CHECKSUM_VALUE_SHIFT                                              (0U)
+#define RGX_CR_TE_CHECKSUM_VALUE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVB_CHECKSUM
+*/
+#define RGX_CR_USC_UVB_CHECKSUM                                                     (0x5118U)
+#define RGX_CR_USC_UVB_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_CHECKSUM
+*/
+#define RGX_CR_VCE_CHECKSUM                                                         (0x5030U)
+#define RGX_CR_VCE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_VCE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_PDS_CHECKSUM
+*/
+#define RGX_CR_ISP_PDS_CHECKSUM                                                     (0x5038U)
+#define RGX_CR_ISP_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_TPF_CHECKSUM
+*/
+#define RGX_CR_ISP_TPF_CHECKSUM                                                     (0x5040U)
+#define RGX_CR_ISP_TPF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_PLANE0_CHECKSUM
+*/
+#define RGX_CR_TFPU_PLANE0_CHECKSUM                                                 (0x5048U)
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_VALUE_SHIFT                                     (0U)
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_VALUE_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_PLANE1_CHECKSUM
+*/
+#define RGX_CR_TFPU_PLANE1_CHECKSUM                                                 (0x5050U)
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_VALUE_SHIFT                                     (0U)
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_VALUE_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM
+*/
+#define RGX_CR_PBE_CHECKSUM                                                         (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_PBE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PDS_DOUTM_STM_SIGNATURE
+*/
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE                                              (0x5060U)
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_VALUE_SHIFT                                  (0U)
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_VALUE_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_IFPU_ISP_CHECKSUM
+*/
+#define RGX_CR_IFPU_ISP_CHECKSUM                                                    (0x5068U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS4_CHECKSUM
+*/
+#define RGX_CR_USC_UVS4_CHECKSUM                                                    (0x5100U)
+#define RGX_CR_USC_UVS4_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS4_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS4_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS5_CHECKSUM
+*/
+#define RGX_CR_USC_UVS5_CHECKSUM                                                    (0x5108U)
+#define RGX_CR_USC_UVS5_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS5_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS5_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_CLIP_CHECKSUM
+*/
+#define RGX_CR_PPP_CLIP_CHECKSUM                                                    (0x5120U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_PHASE
+*/
+#define RGX_CR_PERF_TA_PHASE                                                        (0x6008U)
+#define RGX_CR_PERF_TA_PHASE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_PHASE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_TA_PHASE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_PHASE
+*/
+#define RGX_CR_PERF_3D_PHASE                                                        (0x6010U)
+#define RGX_CR_PERF_3D_PHASE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_PHASE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_3D_PHASE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_COMPUTE_PHASE
+*/
+#define RGX_CR_PERF_COMPUTE_PHASE                                                   (0x6018U)
+#define RGX_CR_PERF_COMPUTE_PHASE_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_COMPUTE_PHASE_COUNT_SHIFT                                       (0U)
+#define RGX_CR_PERF_COMPUTE_PHASE_COUNT_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_CYCLE
+*/
+#define RGX_CR_PERF_TA_CYCLE                                                        (0x6020U)
+#define RGX_CR_PERF_TA_CYCLE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_CYCLE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_TA_CYCLE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_CYCLE
+*/
+#define RGX_CR_PERF_3D_CYCLE                                                        (0x6028U)
+#define RGX_CR_PERF_3D_CYCLE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_CYCLE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_3D_CYCLE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_COMPUTE_CYCLE
+*/
+#define RGX_CR_PERF_COMPUTE_CYCLE                                                   (0x6030U)
+#define RGX_CR_PERF_COMPUTE_CYCLE_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_COMPUTE_CYCLE_COUNT_SHIFT                                       (0U)
+#define RGX_CR_PERF_COMPUTE_CYCLE_COUNT_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_OR_3D_CYCLE
+*/
+#define RGX_CR_PERF_TA_OR_3D_CYCLE                                                  (0x6038U)
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_COUNT_SHIFT                                      (0U)
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_COUNT_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_INITIAL_TA_CYCLE
+*/
+#define RGX_CR_PERF_INITIAL_TA_CYCLE                                                (0x6040U)
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_READ_STALL
+*/
+#define RGX_CR_PERF_SLC0_READ_STALL                                                 (0x60B8U)
+#define RGX_CR_PERF_SLC0_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC0_WRITE_STALL                                                (0x60C0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_READ_STALL
+*/
+#define RGX_CR_PERF_SLC1_READ_STALL                                                 (0x60E0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC1_WRITE_STALL                                                (0x60E8U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_READ_STALL
+*/
+#define RGX_CR_PERF_SLC2_READ_STALL                                                 (0x6158U)
+#define RGX_CR_PERF_SLC2_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC2_WRITE_STALL                                                (0x6160U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_READ_STALL
+*/
+#define RGX_CR_PERF_SLC3_READ_STALL                                                 (0x6180U)
+#define RGX_CR_PERF_SLC3_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC3_WRITE_STALL                                                (0x6188U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_SPINUP
+*/
+#define RGX_CR_PERF_3D_SPINUP                                                       (0x6220U)
+#define RGX_CR_PERF_3D_SPINUP_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_SPINUP_CYCLES_SHIFT                                          (0U)
+#define RGX_CR_PERF_3D_SPINUP_CYCLES_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_AXI_ACE_LITE_CONFIGURATION
+*/
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION                                           (0x38C0U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_MASKFULL                                  (IMG_UINT64_C(0x00003FFFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT                    (45U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_CLRMSK                   (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_EN                       (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_OSID_SECURITY_SHIFT                       (37U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_OSID_SECURITY_CLRMSK                      (IMG_UINT64_C(0xFFFFE01FFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT    (36U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_CLRMSK   (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_EN       (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_SHIFT              (35U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_CLRMSK             (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_EN                 (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_SHIFT               (34U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_CLRMSK              (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_EN                  (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT           (30U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_CLRMSK          (IMG_UINT64_C(0xFFFFFFFC3FFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT                    (26U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFC3FFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT                    (22U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFC3FFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_BARRIER_SHIFT                    (20U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_BARRIER_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_BARRIER_SHIFT                    (18U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_BARRIER_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT          (16U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_CLRMSK         (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT                   (14U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT                   (12U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT               (10U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT               (8U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_NON_SNOOPING_SHIFT                (4U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_NON_SNOOPING_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFF0F))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_NON_SNOOPING_SHIFT                (0U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_NON_SNOOPING_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF0))
+
+
+/*
+    Register RGX_CR_POWER_ESTIMATE_RESULT
+*/
+#define RGX_CR_POWER_ESTIMATE_RESULT                                                (0x6328U)
+#define RGX_CR_POWER_ESTIMATE_RESULT_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_POWER_ESTIMATE_RESULT_VALUE_SHIFT                                    (0U)
+#define RGX_CR_POWER_ESTIMATE_RESULT_VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF
+*/
+#define RGX_CR_TA_PERF                                                              (0x7600U)
+#define RGX_CR_TA_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TA_PERF_CLR_3_SHIFT                                                  (4U)
+#define RGX_CR_TA_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define RGX_CR_TA_PERF_CLR_3_EN                                                     (0x00000010U)
+#define RGX_CR_TA_PERF_CLR_2_SHIFT                                                  (3U)
+#define RGX_CR_TA_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_TA_PERF_CLR_2_EN                                                     (0x00000008U)
+#define RGX_CR_TA_PERF_CLR_1_SHIFT                                                  (2U)
+#define RGX_CR_TA_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define RGX_CR_TA_PERF_CLR_1_EN                                                     (0x00000004U)
+#define RGX_CR_TA_PERF_CLR_0_SHIFT                                                  (1U)
+#define RGX_CR_TA_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_TA_PERF_CLR_0_EN                                                     (0x00000002U)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT0
+*/
+#define RGX_CR_TA_PERF_SELECT0                                                      (0x7608U)
+#define RGX_CR_TA_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT1
+*/
+#define RGX_CR_TA_PERF_SELECT1                                                      (0x7610U)
+#define RGX_CR_TA_PERF_SELECT1_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT1_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT1_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT1_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT1_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT1_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT2
+*/
+#define RGX_CR_TA_PERF_SELECT2                                                      (0x7618U)
+#define RGX_CR_TA_PERF_SELECT2_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT2_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT2_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT2_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT2_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT2_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT3
+*/
+#define RGX_CR_TA_PERF_SELECT3                                                      (0x7620U)
+#define RGX_CR_TA_PERF_SELECT3_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT3_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT3_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT3_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT3_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT3_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECTED_BITS
+*/
+#define RGX_CR_TA_PERF_SELECTED_BITS                                                (0x7648U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG3_SHIFT                                     (48U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG3_CLRMSK                                    (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG2_SHIFT                                     (32U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG2_CLRMSK                                    (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG1_SHIFT                                     (16U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG1_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG0_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG0_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_0
+*/
+#define RGX_CR_TA_PERF_COUNTER_0                                                    (0x7650U)
+#define RGX_CR_TA_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_1
+*/
+#define RGX_CR_TA_PERF_COUNTER_1                                                    (0x7658U)
+#define RGX_CR_TA_PERF_COUNTER_1_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_1_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_1_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_2
+*/
+#define RGX_CR_TA_PERF_COUNTER_2                                                    (0x7660U)
+#define RGX_CR_TA_PERF_COUNTER_2_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_2_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_2_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_3
+*/
+#define RGX_CR_TA_PERF_COUNTER_3                                                    (0x7668U)
+#define RGX_CR_TA_PERF_COUNTER_3_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_3_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_3_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF
+*/
+#define RGX_CR_RASTERISATION_PERF                                                   (0x7700U)
+#define RGX_CR_RASTERISATION_PERF_MASKFULL                                          (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_RASTERISATION_PERF_CLR_3_SHIFT                                       (4U)
+#define RGX_CR_RASTERISATION_PERF_CLR_3_CLRMSK                                      (0xFFFFFFEFU)
+#define RGX_CR_RASTERISATION_PERF_CLR_3_EN                                          (0x00000010U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_SHIFT                                       (3U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_CLRMSK                                      (0xFFFFFFF7U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_EN                                          (0x00000008U)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_SHIFT                                       (2U)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_CLRMSK                                      (0xFFFFFFFBU)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_EN                                          (0x00000004U)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_SHIFT                                       (1U)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_EN                                          (0x00000002U)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_SHIFT                                 (0U)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_SELECT0
+*/
+#define RGX_CR_RASTERISATION_PERF_SELECT0                                           (0x7708U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MASKFULL                                  (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MAX_SHIFT                           (48U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MAX_CLRMSK                          (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MIN_SHIFT                           (32U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MIN_CLRMSK                          (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_SHIFT                                (21U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_GROUP_SELECT_SHIFT                        (16U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_GROUP_SELECT_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BIT_SELECT_SHIFT                          (0U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BIT_SELECT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_COUNTER_0
+*/
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0                                         (0x7750U)
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_REG_SHIFT                               (0U)
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_REG_CLRMSK                              (0x00000000U)
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF                                                  (0x7800U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_MASKFULL                                         (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_SHIFT                                      (4U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_EN                                         (0x00000010U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_SHIFT                                      (3U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_EN                                         (0x00000008U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_SHIFT                                      (2U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_EN                                         (0x00000004U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_SHIFT                                      (1U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_EN                                         (0x00000002U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_SHIFT                                (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0                                          (0x7808U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MASKFULL                                 (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MAX_SHIFT                          (48U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MAX_CLRMSK                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MIN_SHIFT                          (32U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MIN_CLRMSK                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_SHIFT                               (21U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_EN                                  (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_GROUP_SELECT_SHIFT                       (16U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_GROUP_SELECT_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BIT_SELECT_SHIFT                         (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BIT_SELECT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0                                        (0x7850U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_REG_SHIFT                              (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_REG_CLRMSK                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF
+*/
+#define RGX_CR_TPU_MCU_L0_PERF                                                      (0x7900U)
+#define RGX_CR_TPU_MCU_L0_PERF_MASKFULL                                             (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_SHIFT                                          (4U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_EN                                             (0x00000010U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_SHIFT                                          (3U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_EN                                             (0x00000008U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_SHIFT                                          (2U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_EN                                             (0x00000004U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_SHIFT                                          (1U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_EN                                             (0x00000002U)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_SHIFT                                    (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_SELECT0
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0                                              (0x7908U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MASKFULL                                     (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MAX_SHIFT                              (48U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MAX_CLRMSK                             (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MIN_SHIFT                              (32U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MIN_CLRMSK                             (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_SHIFT                                   (21U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_EN                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_GROUP_SELECT_SHIFT                           (16U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_GROUP_SELECT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BIT_SELECT_SHIFT                             (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BIT_SELECT_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_COUNTER_0
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0                                            (0x7950U)
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_REG_SHIFT                                  (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_REG_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_PERF
+*/
+#define RGX_CR_USC_PERF                                                             (0x8100U)
+#define RGX_CR_USC_PERF_MASKFULL                                                    (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_USC_PERF_CLR_3_SHIFT                                                 (4U)
+#define RGX_CR_USC_PERF_CLR_3_CLRMSK                                                (0xFFFFFFEFU)
+#define RGX_CR_USC_PERF_CLR_3_EN                                                    (0x00000010U)
+#define RGX_CR_USC_PERF_CLR_2_SHIFT                                                 (3U)
+#define RGX_CR_USC_PERF_CLR_2_CLRMSK                                                (0xFFFFFFF7U)
+#define RGX_CR_USC_PERF_CLR_2_EN                                                    (0x00000008U)
+#define RGX_CR_USC_PERF_CLR_1_SHIFT                                                 (2U)
+#define RGX_CR_USC_PERF_CLR_1_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_USC_PERF_CLR_1_EN                                                    (0x00000004U)
+#define RGX_CR_USC_PERF_CLR_0_SHIFT                                                 (1U)
+#define RGX_CR_USC_PERF_CLR_0_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_USC_PERF_CLR_0_EN                                                    (0x00000002U)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_SHIFT                                           (0U)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_USC_PERF_SELECT0
+*/
+#define RGX_CR_USC_PERF_SELECT0                                                     (0x8108U)
+#define RGX_CR_USC_PERF_SELECT0_MASKFULL                                            (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MAX_SHIFT                                     (48U)
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MAX_CLRMSK                                    (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MIN_SHIFT                                     (32U)
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MIN_CLRMSK                                    (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_MODE_SHIFT                                          (21U)
+#define RGX_CR_USC_PERF_SELECT0_MODE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_MODE_EN                                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_USC_PERF_SELECT0_GROUP_SELECT_SHIFT                                  (16U)
+#define RGX_CR_USC_PERF_SELECT0_GROUP_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_USC_PERF_SELECT0_BIT_SELECT_SHIFT                                    (0U)
+#define RGX_CR_USC_PERF_SELECT0_BIT_SELECT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_USC_PERF_COUNTER_0
+*/
+#define RGX_CR_USC_PERF_COUNTER_0                                                   (0x8150U)
+#define RGX_CR_USC_PERF_COUNTER_0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_PERF_COUNTER_0_REG_SHIFT                                         (0U)
+#define RGX_CR_USC_PERF_COUNTER_0_REG_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_JONES_IDLE
+*/
+#define RGX_CR_JONES_IDLE                                                           (0x8328U)
+#define RGX_CR_JONES_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x0000000000007FFF))
+#define RGX_CR_JONES_IDLE_TDM_SHIFT                                                 (14U)
+#define RGX_CR_JONES_IDLE_TDM_CLRMSK                                                (0xFFFFBFFFU)
+#define RGX_CR_JONES_IDLE_TDM_EN                                                    (0x00004000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_SHIFT                                          (13U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_EN                                             (0x00002000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_SHIFT                                              (12U)
+#define RGX_CR_JONES_IDLE_FB_CDC_CLRMSK                                             (0xFFFFEFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_EN                                                 (0x00001000U)
+#define RGX_CR_JONES_IDLE_MMU_SHIFT                                                 (11U)
+#define RGX_CR_JONES_IDLE_MMU_CLRMSK                                                (0xFFFFF7FFU)
+#define RGX_CR_JONES_IDLE_MMU_EN                                                    (0x00000800U)
+#define RGX_CR_JONES_IDLE_TLA_SHIFT                                                 (10U)
+#define RGX_CR_JONES_IDLE_TLA_CLRMSK                                                (0xFFFFFBFFU)
+#define RGX_CR_JONES_IDLE_TLA_EN                                                    (0x00000400U)
+#define RGX_CR_JONES_IDLE_GARTEN_SHIFT                                              (9U)
+#define RGX_CR_JONES_IDLE_GARTEN_CLRMSK                                             (0xFFFFFDFFU)
+#define RGX_CR_JONES_IDLE_GARTEN_EN                                                 (0x00000200U)
+#define RGX_CR_JONES_IDLE_HOSTIF_SHIFT                                              (8U)
+#define RGX_CR_JONES_IDLE_HOSTIF_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_JONES_IDLE_HOSTIF_EN                                                 (0x00000100U)
+#define RGX_CR_JONES_IDLE_SOCIF_SHIFT                                               (7U)
+#define RGX_CR_JONES_IDLE_SOCIF_CLRMSK                                              (0xFFFFFF7FU)
+#define RGX_CR_JONES_IDLE_SOCIF_EN                                                  (0x00000080U)
+#define RGX_CR_JONES_IDLE_TILING_SHIFT                                              (6U)
+#define RGX_CR_JONES_IDLE_TILING_CLRMSK                                             (0xFFFFFFBFU)
+#define RGX_CR_JONES_IDLE_TILING_EN                                                 (0x00000040U)
+#define RGX_CR_JONES_IDLE_IPP_SHIFT                                                 (5U)
+#define RGX_CR_JONES_IDLE_IPP_CLRMSK                                                (0xFFFFFFDFU)
+#define RGX_CR_JONES_IDLE_IPP_EN                                                    (0x00000020U)
+#define RGX_CR_JONES_IDLE_USCS_SHIFT                                                (4U)
+#define RGX_CR_JONES_IDLE_USCS_CLRMSK                                               (0xFFFFFFEFU)
+#define RGX_CR_JONES_IDLE_USCS_EN                                                   (0x00000010U)
+#define RGX_CR_JONES_IDLE_PM_SHIFT                                                  (3U)
+#define RGX_CR_JONES_IDLE_PM_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_JONES_IDLE_PM_EN                                                     (0x00000008U)
+#define RGX_CR_JONES_IDLE_CDM_SHIFT                                                 (2U)
+#define RGX_CR_JONES_IDLE_CDM_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_JONES_IDLE_CDM_EN                                                    (0x00000004U)
+#define RGX_CR_JONES_IDLE_VDM_SHIFT                                                 (1U)
+#define RGX_CR_JONES_IDLE_VDM_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_JONES_IDLE_VDM_EN                                                    (0x00000002U)
+#define RGX_CR_JONES_IDLE_BIF_SHIFT                                                 (0U)
+#define RGX_CR_JONES_IDLE_BIF_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_JONES_IDLE_BIF_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_TORNADO_PERF
+*/
+#define RGX_CR_TORNADO_PERF                                                         (0x8228U)
+#define RGX_CR_TORNADO_PERF_MASKFULL                                                (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TORNADO_PERF_CLR_3_SHIFT                                             (4U)
+#define RGX_CR_TORNADO_PERF_CLR_3_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_TORNADO_PERF_CLR_3_EN                                                (0x00000010U)
+#define RGX_CR_TORNADO_PERF_CLR_2_SHIFT                                             (3U)
+#define RGX_CR_TORNADO_PERF_CLR_2_CLRMSK                                            (0xFFFFFFF7U)
+#define RGX_CR_TORNADO_PERF_CLR_2_EN                                                (0x00000008U)
+#define RGX_CR_TORNADO_PERF_CLR_1_SHIFT                                             (2U)
+#define RGX_CR_TORNADO_PERF_CLR_1_CLRMSK                                            (0xFFFFFFFBU)
+#define RGX_CR_TORNADO_PERF_CLR_1_EN                                                (0x00000004U)
+#define RGX_CR_TORNADO_PERF_CLR_0_SHIFT                                             (1U)
+#define RGX_CR_TORNADO_PERF_CLR_0_CLRMSK                                            (0xFFFFFFFDU)
+#define RGX_CR_TORNADO_PERF_CLR_0_EN                                                (0x00000002U)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_SHIFT                                       (0U)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_TORNADO_PERF_SELECT0
+*/
+#define RGX_CR_TORNADO_PERF_SELECT0                                                 (0x8230U)
+#define RGX_CR_TORNADO_PERF_SELECT0_MASKFULL                                        (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MAX_SHIFT                                 (48U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MAX_CLRMSK                                (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MIN_SHIFT                                 (32U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MIN_CLRMSK                                (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_SHIFT                                      (21U)
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_EN                                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TORNADO_PERF_SELECT0_GROUP_SELECT_SHIFT                              (16U)
+#define RGX_CR_TORNADO_PERF_SELECT0_GROUP_SELECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BIT_SELECT_SHIFT                                (0U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BIT_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TORNADO_PERF_COUNTER_0
+*/
+#define RGX_CR_TORNADO_PERF_COUNTER_0                                               (0x8268U)
+#define RGX_CR_TORNADO_PERF_COUNTER_0_MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TORNADO_PERF_COUNTER_0_REG_SHIFT                                     (0U)
+#define RGX_CR_TORNADO_PERF_COUNTER_0_REG_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF
+*/
+#define RGX_CR_TEXAS_PERF                                                           (0x8290U)
+#define RGX_CR_TEXAS_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_TEXAS_PERF_CLR_5_SHIFT                                               (6U)
+#define RGX_CR_TEXAS_PERF_CLR_5_CLRMSK                                              (0xFFFFFFBFU)
+#define RGX_CR_TEXAS_PERF_CLR_5_EN                                                  (0x00000040U)
+#define RGX_CR_TEXAS_PERF_CLR_4_SHIFT                                               (5U)
+#define RGX_CR_TEXAS_PERF_CLR_4_CLRMSK                                              (0xFFFFFFDFU)
+#define RGX_CR_TEXAS_PERF_CLR_4_EN                                                  (0x00000020U)
+#define RGX_CR_TEXAS_PERF_CLR_3_SHIFT                                               (4U)
+#define RGX_CR_TEXAS_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_PERF_CLR_3_EN                                                  (0x00000010U)
+#define RGX_CR_TEXAS_PERF_CLR_2_SHIFT                                               (3U)
+#define RGX_CR_TEXAS_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define RGX_CR_TEXAS_PERF_CLR_2_EN                                                  (0x00000008U)
+#define RGX_CR_TEXAS_PERF_CLR_1_SHIFT                                               (2U)
+#define RGX_CR_TEXAS_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_PERF_CLR_1_EN                                                  (0x00000004U)
+#define RGX_CR_TEXAS_PERF_CLR_0_SHIFT                                               (1U)
+#define RGX_CR_TEXAS_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_TEXAS_PERF_CLR_0_EN                                                  (0x00000002U)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_SELECT0
+*/
+#define RGX_CR_TEXAS_PERF_SELECT0                                                   (0x8298U)
+#define RGX_CR_TEXAS_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF803FFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_SHIFT                                        (31U)
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_TEXAS_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define RGX_CR_TEXAS_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFC0FFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_COUNTER_0
+*/
+#define RGX_CR_TEXAS_PERF_COUNTER_0                                                 (0x82D8U)
+#define RGX_CR_TEXAS_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define RGX_CR_TEXAS_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_JONES_PERF
+*/
+#define RGX_CR_JONES_PERF                                                           (0x8330U)
+#define RGX_CR_JONES_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_JONES_PERF_CLR_3_SHIFT                                               (4U)
+#define RGX_CR_JONES_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define RGX_CR_JONES_PERF_CLR_3_EN                                                  (0x00000010U)
+#define RGX_CR_JONES_PERF_CLR_2_SHIFT                                               (3U)
+#define RGX_CR_JONES_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define RGX_CR_JONES_PERF_CLR_2_EN                                                  (0x00000008U)
+#define RGX_CR_JONES_PERF_CLR_1_SHIFT                                               (2U)
+#define RGX_CR_JONES_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define RGX_CR_JONES_PERF_CLR_1_EN                                                  (0x00000004U)
+#define RGX_CR_JONES_PERF_CLR_0_SHIFT                                               (1U)
+#define RGX_CR_JONES_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_JONES_PERF_CLR_0_EN                                                  (0x00000002U)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_PERF_SELECT0
+*/
+#define RGX_CR_JONES_PERF_SELECT0                                                   (0x8338U)
+#define RGX_CR_JONES_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_MODE_SHIFT                                        (21U)
+#define RGX_CR_JONES_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_JONES_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define RGX_CR_JONES_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define RGX_CR_JONES_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_JONES_PERF_COUNTER_0
+*/
+#define RGX_CR_JONES_PERF_COUNTER_0                                                 (0x8368U)
+#define RGX_CR_JONES_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_JONES_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define RGX_CR_JONES_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF
+*/
+#define RGX_CR_BLACKPEARL_PERF                                                      (0x8400U)
+#define RGX_CR_BLACKPEARL_PERF_MASKFULL                                             (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_SHIFT                                          (6U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_EN                                             (0x00000040U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_SHIFT                                          (5U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_EN                                             (0x00000020U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_SHIFT                                          (4U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_EN                                             (0x00000010U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_SHIFT                                          (3U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_EN                                             (0x00000008U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_SHIFT                                          (2U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_EN                                             (0x00000004U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_SHIFT                                          (1U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_EN                                             (0x00000002U)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_SHIFT                                    (0U)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_SELECT0
+*/
+#define RGX_CR_BLACKPEARL_PERF_SELECT0                                              (0x8408U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MASKFULL                                     (IMG_UINT64_C(0x3FFF3FFF803FFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MAX_SHIFT                              (48U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MAX_CLRMSK                             (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MIN_SHIFT                              (32U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MIN_CLRMSK                             (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_SHIFT                                   (31U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_EN                                      (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_GROUP_SELECT_SHIFT                           (16U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_GROUP_SELECT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFC0FFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BIT_SELECT_SHIFT                             (0U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BIT_SELECT_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_COUNTER_0
+*/
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0                                            (0x8448U)
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_REG_SHIFT                                  (0U)
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_REG_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_PERF
+*/
+#define RGX_CR_PBE_PERF                                                             (0x8478U)
+#define RGX_CR_PBE_PERF_MASKFULL                                                    (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_PBE_PERF_CLR_3_SHIFT                                                 (4U)
+#define RGX_CR_PBE_PERF_CLR_3_CLRMSK                                                (0xFFFFFFEFU)
+#define RGX_CR_PBE_PERF_CLR_3_EN                                                    (0x00000010U)
+#define RGX_CR_PBE_PERF_CLR_2_SHIFT                                                 (3U)
+#define RGX_CR_PBE_PERF_CLR_2_CLRMSK                                                (0xFFFFFFF7U)
+#define RGX_CR_PBE_PERF_CLR_2_EN                                                    (0x00000008U)
+#define RGX_CR_PBE_PERF_CLR_1_SHIFT                                                 (2U)
+#define RGX_CR_PBE_PERF_CLR_1_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_PBE_PERF_CLR_1_EN                                                    (0x00000004U)
+#define RGX_CR_PBE_PERF_CLR_0_SHIFT                                                 (1U)
+#define RGX_CR_PBE_PERF_CLR_0_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_PBE_PERF_CLR_0_EN                                                    (0x00000002U)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_SHIFT                                           (0U)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_PBE_PERF_SELECT0
+*/
+#define RGX_CR_PBE_PERF_SELECT0                                                     (0x8480U)
+#define RGX_CR_PBE_PERF_SELECT0_MASKFULL                                            (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MAX_SHIFT                                     (48U)
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MAX_CLRMSK                                    (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MIN_SHIFT                                     (32U)
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MIN_CLRMSK                                    (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_MODE_SHIFT                                          (21U)
+#define RGX_CR_PBE_PERF_SELECT0_MODE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_MODE_EN                                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_PBE_PERF_SELECT0_GROUP_SELECT_SHIFT                                  (16U)
+#define RGX_CR_PBE_PERF_SELECT0_GROUP_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BIT_SELECT_SHIFT                                    (0U)
+#define RGX_CR_PBE_PERF_SELECT0_BIT_SELECT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_PBE_PERF_COUNTER_0
+*/
+#define RGX_CR_PBE_PERF_COUNTER_0                                                   (0x84B0U)
+#define RGX_CR_PBE_PERF_COUNTER_0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_PERF_COUNTER_0_REG_SHIFT                                         (0U)
+#define RGX_CR_PBE_PERF_COUNTER_0_REG_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_OCP_REVINFO
+*/
+#define RGX_CR_OCP_REVINFO                                                          (0x9000U)
+#define RGX_CR_OCP_REVINFO_MASKFULL                                                 (IMG_UINT64_C(0x00000007FFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_SYSBUS_SHIFT                                      (33U)
+#define RGX_CR_OCP_REVINFO_HWINFO_SYSBUS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF9FFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_SHIFT                                      (32U)
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_EN                                         (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_OCP_REVINFO_REVISION_SHIFT                                           (0U)
+#define RGX_CR_OCP_REVINFO_REVISION_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_OCP_SYSCONFIG
+*/
+#define RGX_CR_OCP_SYSCONFIG                                                        (0x9010U)
+#define RGX_CR_OCP_SYSCONFIG_MASKFULL                                               (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_CR_OCP_SYSCONFIG_DUST2_STANDBY_MODE_SHIFT                               (10U)
+#define RGX_CR_OCP_SYSCONFIG_DUST2_STANDBY_MODE_CLRMSK                              (0xFFFFF3FFU)
+#define RGX_CR_OCP_SYSCONFIG_DUST1_STANDBY_MODE_SHIFT                               (8U)
+#define RGX_CR_OCP_SYSCONFIG_DUST1_STANDBY_MODE_CLRMSK                              (0xFFFFFCFFU)
+#define RGX_CR_OCP_SYSCONFIG_DUST0_STANDBY_MODE_SHIFT                               (6U)
+#define RGX_CR_OCP_SYSCONFIG_DUST0_STANDBY_MODE_CLRMSK                              (0xFFFFFF3FU)
+#define RGX_CR_OCP_SYSCONFIG_RASCAL_STANDBYMODE_SHIFT                               (4U)
+#define RGX_CR_OCP_SYSCONFIG_RASCAL_STANDBYMODE_CLRMSK                              (0xFFFFFFCFU)
+#define RGX_CR_OCP_SYSCONFIG_STANDBY_MODE_SHIFT                                     (2U)
+#define RGX_CR_OCP_SYSCONFIG_STANDBY_MODE_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_OCP_SYSCONFIG_IDLE_MODE_SHIFT                                        (0U)
+#define RGX_CR_OCP_SYSCONFIG_IDLE_MODE_CLRMSK                                       (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_0
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_0                                                  (0x9020U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_SHIFT                        (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_1
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_1                                                  (0x9028U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_SHIFT                      (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_CLRMSK                     (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_EN                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_2
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_2                                                  (0x9030U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_SHIFT                                (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_0
+*/
+#define RGX_CR_OCP_IRQSTATUS_0                                                      (0x9038U)
+#define RGX_CR_OCP_IRQSTATUS_0_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_SHIFT                         (0U)
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_EN                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_1
+*/
+#define RGX_CR_OCP_IRQSTATUS_1                                                      (0x9040U)
+#define RGX_CR_OCP_IRQSTATUS_1_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_SHIFT                       (0U)
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_CLRMSK                      (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_EN                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_2
+*/
+#define RGX_CR_OCP_IRQSTATUS_2                                                      (0x9048U)
+#define RGX_CR_OCP_IRQSTATUS_2_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_SHIFT                                 (0U)
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_0
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_0                                                  (0x9050U)
+#define RGX_CR_OCP_IRQENABLE_SET_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_SHIFT                     (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_CLRMSK                    (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_EN                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_1
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_1                                                  (0x9058U)
+#define RGX_CR_OCP_IRQENABLE_SET_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_SHIFT                   (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_2
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_2                                                  (0x9060U)
+#define RGX_CR_OCP_IRQENABLE_SET_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_SHIFT                             (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_0
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_0                                                  (0x9068U)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_SHIFT                    (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_CLRMSK                   (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_EN                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_1
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_1                                                  (0x9070U)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_SHIFT                  (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_CLRMSK                 (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_EN                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_2
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_2                                                  (0x9078U)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_SHIFT                            (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQ_EVENT
+*/
+#define RGX_CR_OCP_IRQ_EVENT                                                        (0x9080U)
+#define RGX_CR_OCP_IRQ_EVENT_MASKFULL                                               (IMG_UINT64_C(0x00000000000FFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_SHIFT                    (19U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_EN                       (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_SHIFT                    (18U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_EN                       (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_SHIFT                    (17U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_EN                       (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_SHIFT                    (16U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_EN                       (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (15U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_SHIFT                         (14U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_SHIFT                          (13U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_SHIFT                        (12U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (11U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_SHIFT                         (10U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_SHIFT                          (9U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_SHIFT                        (8U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (7U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_SHIFT                         (6U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_SHIFT                          (5U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_SHIFT                        (4U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (3U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_SHIFT                         (2U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_SHIFT                          (1U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_SHIFT                        (0U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_OCP_DEBUG_CONFIG
+*/
+#define RGX_CR_OCP_DEBUG_CONFIG                                                     (0x9088U)
+#define RGX_CR_OCP_DEBUG_CONFIG_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_SHIFT                                           (0U)
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_DEBUG_STATUS
+*/
+#define RGX_CR_OCP_DEBUG_STATUS                                                     (0x9090U)
+#define RGX_CR_OCP_DEBUG_STATUS_MASKFULL                                            (IMG_UINT64_C(0x001F1F77FFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SDISCACK_SHIFT                              (51U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SDISCACK_CLRMSK                             (IMG_UINT64_C(0xFFE7FFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_SHIFT                              (50U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_EN                                 (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_MCONNECT_SHIFT                              (48U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_MCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SDISCACK_SHIFT                              (43U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SDISCACK_CLRMSK                             (IMG_UINT64_C(0xFFFFE7FFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_SHIFT                              (42U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_EN                                 (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_MCONNECT_SHIFT                              (40U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_MCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_SHIFT                                  (38U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_EN                                     (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_SHIFT                         (37U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_EN                            (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_SHIFT                           (36U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_CLRMSK                          (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_EN                              (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_SHIFT                                  (34U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_EN                                     (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_SHIFT                         (33U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_EN                            (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_SHIFT                           (32U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_EN                              (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_SHIFT                                (31U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_EN                                   (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_SHIFT                                   (30U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_EN                                      (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_SHIFT                                (29U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCACK_SHIFT                                (27U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFE7FFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_SHIFT                                (26U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MCONNECT_SHIFT                                (24U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_SHIFT                                (23U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_SHIFT                                   (22U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_SHIFT                                (21U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCACK_SHIFT                                (19U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE7FFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_SHIFT                                (18U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MCONNECT_SHIFT                                (16U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_SHIFT                                (15U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_SHIFT                                   (14U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_SHIFT                                (13U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCACK_SHIFT                                (11U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFE7FF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_SHIFT                                (10U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MCONNECT_SHIFT                                (8U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_SHIFT                                (7U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_SHIFT                                   (6U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_SHIFT                                (5U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCACK_SHIFT                                (3U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFE7))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_SHIFT                                (2U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MCONNECT_SHIFT                                (0U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_SHIFT                                     (6U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_CLRMSK                                    (0xFFFFFFBFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_EN                                        (0x00000040U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_SHIFT                                         (5U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_EN                                            (0x00000020U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_SHIFT                                         (4U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_EN                                            (0x00000010U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_SHIFT                                       (3U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_CLRMSK                                      (0xFFFFFFF7U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_EN                                          (0x00000008U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_SHIFT                                       (1U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_EN                                          (0x00000002U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_SHIFT                                          (0U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_EN                                             (0x00000001U)
+
+
+#define RGX_CR_BIF_TRUST_DM_MASK                                                    (0x0000007FU)
+
+
+/*
+    Register RGX_CR_BIF_TRUST
+*/
+#define RGX_CR_BIF_TRUST                                                            (0xA000U)
+#define RGX_CR_BIF_TRUST_MASKFULL                                                   (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_SHIFT                          (20U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_CLRMSK                         (0xFFEFFFFFU)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_EN                             (0x00100000U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_SHIFT                            (19U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_CLRMSK                           (0xFFF7FFFFU)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_EN                               (0x00080000U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_SHIFT                                 (18U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_CLRMSK                                (0xFFFBFFFFU)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_EN                                    (0x00040000U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_SHIFT                                   (17U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_CLRMSK                                  (0xFFFDFFFFU)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_EN                                      (0x00020000U)
+#define RGX_CR_BIF_TRUST_ENABLE_SHIFT                                               (16U)
+#define RGX_CR_BIF_TRUST_ENABLE_CLRMSK                                              (0xFFFEFFFFU)
+#define RGX_CR_BIF_TRUST_ENABLE_EN                                                  (0x00010000U)
+#define RGX_CR_BIF_TRUST_DM_TRUSTED_SHIFT                                           (9U)
+#define RGX_CR_BIF_TRUST_DM_TRUSTED_CLRMSK                                          (0xFFFF01FFU)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_SHIFT                             (8U)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_CLRMSK                            (0xFFFFFEFFU)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_EN                                (0x00000100U)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_SHIFT                               (7U)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_CLRMSK                              (0xFFFFFF7FU)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_EN                                  (0x00000080U)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_SHIFT                               (6U)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_EN                                  (0x00000040U)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_CLRMSK                              (0xFFFFFFDFU)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_EN                                  (0x00000020U)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_SHIFT                                 (4U)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_EN                                    (0x00000010U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_SHIFT                                 (3U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_EN                                    (0x00000008U)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_SHIFT                              (2U)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_EN                                 (0x00000004U)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_SHIFT                                (1U)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_EN                                   (0x00000002U)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_SHIFT                                (0U)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_SYS_BUS_SECURE
+*/
+#define RGX_CR_SYS_BUS_SECURE                                                       (0xA100U)
+#define RGX_CR_SYS_BUS_SECURE__SECR__MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_MASKFULL                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_SHIFT                                          (0U)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_FBA_FC0_CHECKSUM
+*/
+#define RGX_CR_FBA_FC0_CHECKSUM                                                     (0xD170U)
+#define RGX_CR_FBA_FC0_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC0_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC0_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC1_CHECKSUM
+*/
+#define RGX_CR_FBA_FC1_CHECKSUM                                                     (0xD178U)
+#define RGX_CR_FBA_FC1_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC1_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC1_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC2_CHECKSUM
+*/
+#define RGX_CR_FBA_FC2_CHECKSUM                                                     (0xD180U)
+#define RGX_CR_FBA_FC2_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC2_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC2_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC3_CHECKSUM
+*/
+#define RGX_CR_FBA_FC3_CHECKSUM                                                     (0xD188U)
+#define RGX_CR_FBA_FC3_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC3_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC3_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_CLK_CTRL2
+*/
+#define RGX_CR_CLK_CTRL2                                                            (0xD200U)
+#define RGX_CR_CLK_CTRL2_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000F33))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_SHIFT                                             (10U)
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_ON                                                (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_AUTO                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL2_VRDM_SHIFT                                                 (8U)
+#define RGX_CR_CLK_CTRL2_VRDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL2_VRDM_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_VRDM_ON                                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL2_VRDM_AUTO                                                  (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL2_SH_SHIFT                                                   (4U)
+#define RGX_CR_CLK_CTRL2_SH_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL2_SH_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_SH_ON                                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL2_SH_AUTO                                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL2_FBA_SHIFT                                                  (0U)
+#define RGX_CR_CLK_CTRL2_FBA_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL2_FBA_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_FBA_ON                                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL2_FBA_AUTO                                                   (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS2
+*/
+#define RGX_CR_CLK_STATUS2                                                          (0xD208U)
+#define RGX_CR_CLK_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000015))
+#define RGX_CR_CLK_STATUS2_VRDM_SHIFT                                               (4U)
+#define RGX_CR_CLK_STATUS2_VRDM_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS2_VRDM_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_VRDM_RUNNING                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS2_SH_SHIFT                                                 (2U)
+#define RGX_CR_CLK_STATUS2_SH_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS2_SH_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_SH_RUNNING                                               (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS2_FBA_SHIFT                                                (0U)
+#define RGX_CR_CLK_STATUS2_FBA_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS2_FBA_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_FBA_RUNNING                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL
+*/
+#define RGX_CR_RPM_SHF_FPL                                                          (0xD520U)
+#define RGX_CR_RPM_SHF_FPL_MASKFULL                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFC))
+#define RGX_CR_RPM_SHF_FPL_SIZE_SHIFT                                               (40U)
+#define RGX_CR_RPM_SHF_FPL_SIZE_CLRMSK                                              (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_CR_RPM_SHF_FPL_BASE_SHIFT                                               (2U)
+#define RGX_CR_RPM_SHF_FPL_BASE_CLRMSK                                              (IMG_UINT64_C(0xFFFFFF0000000003))
+#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSHIFT                                          (2U)
+#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSIZE                                           (4U)
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL_READ
+*/
+#define RGX_CR_RPM_SHF_FPL_READ                                                     (0xD528U)
+#define RGX_CR_RPM_SHF_FPL_READ_MASKFULL                                            (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_SHIFT                                        (22U)
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_CLRMSK                                       (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_EN                                           (0x00400000U)
+#define RGX_CR_RPM_SHF_FPL_READ_OFFSET_SHIFT                                        (0U)
+#define RGX_CR_RPM_SHF_FPL_READ_OFFSET_CLRMSK                                       (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL_WRITE
+*/
+#define RGX_CR_RPM_SHF_FPL_WRITE                                                    (0xD530U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_MASKFULL                                           (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_SHIFT                                       (22U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_EN                                          (0x00400000U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_OFFSET_SHIFT                                       (0U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_OFFSET_CLRMSK                                      (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL
+*/
+#define RGX_CR_RPM_SHG_FPL                                                          (0xD538U)
+#define RGX_CR_RPM_SHG_FPL_MASKFULL                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFC))
+#define RGX_CR_RPM_SHG_FPL_SIZE_SHIFT                                               (40U)
+#define RGX_CR_RPM_SHG_FPL_SIZE_CLRMSK                                              (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_CR_RPM_SHG_FPL_BASE_SHIFT                                               (2U)
+#define RGX_CR_RPM_SHG_FPL_BASE_CLRMSK                                              (IMG_UINT64_C(0xFFFFFF0000000003))
+#define RGX_CR_RPM_SHG_FPL_BASE_ALIGNSHIFT                                          (2U)
+#define RGX_CR_RPM_SHG_FPL_BASE_ALIGNSIZE                                           (4U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL_READ
+*/
+#define RGX_CR_RPM_SHG_FPL_READ                                                     (0xD540U)
+#define RGX_CR_RPM_SHG_FPL_READ_MASKFULL                                            (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_SHIFT                                        (22U)
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_CLRMSK                                       (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_EN                                           (0x00400000U)
+#define RGX_CR_RPM_SHG_FPL_READ_OFFSET_SHIFT                                        (0U)
+#define RGX_CR_RPM_SHG_FPL_READ_OFFSET_CLRMSK                                       (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL_WRITE
+*/
+#define RGX_CR_RPM_SHG_FPL_WRITE                                                    (0xD548U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_MASKFULL                                           (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_SHIFT                                       (22U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_EN                                          (0x00400000U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_OFFSET_SHIFT                                       (0U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_OFFSET_CLRMSK                                      (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_SH_PERF
+*/
+#define RGX_CR_SH_PERF                                                              (0xD5F8U)
+#define RGX_CR_SH_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_SH_PERF_CLR_3_SHIFT                                                  (4U)
+#define RGX_CR_SH_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define RGX_CR_SH_PERF_CLR_3_EN                                                     (0x00000010U)
+#define RGX_CR_SH_PERF_CLR_2_SHIFT                                                  (3U)
+#define RGX_CR_SH_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_SH_PERF_CLR_2_EN                                                     (0x00000008U)
+#define RGX_CR_SH_PERF_CLR_1_SHIFT                                                  (2U)
+#define RGX_CR_SH_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define RGX_CR_SH_PERF_CLR_1_EN                                                     (0x00000004U)
+#define RGX_CR_SH_PERF_CLR_0_SHIFT                                                  (1U)
+#define RGX_CR_SH_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_SH_PERF_CLR_0_EN                                                     (0x00000002U)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_SH_PERF_SELECT0
+*/
+#define RGX_CR_SH_PERF_SELECT0                                                      (0xD600U)
+#define RGX_CR_SH_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define RGX_CR_SH_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SH_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_SH_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_SH_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_SH_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SH_PERF_COUNTER_0
+*/
+#define RGX_CR_SH_PERF_COUNTER_0                                                    (0xD628U)
+#define RGX_CR_SH_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SH_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define RGX_CR_SH_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_SHG_CHECKSUM
+*/
+#define RGX_CR_SHF_SHG_CHECKSUM                                                     (0xD1C0U)
+#define RGX_CR_SHF_SHG_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_SHG_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SHF_SHG_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_VERTEX_BIF_CHECKSUM
+*/
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM                                              (0xD1C8U)
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_VALUE_SHIFT                                  (0U)
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_VALUE_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_VARY_BIF_CHECKSUM
+*/
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM                                                (0xD1D0U)
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_VALUE_SHIFT                                    (0U)
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_RPM_BIF_CHECKSUM
+*/
+#define RGX_CR_RPM_BIF_CHECKSUM                                                     (0xD1D8U)
+#define RGX_CR_RPM_BIF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_RPM_BIF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_RPM_BIF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHG_BIF_CHECKSUM
+*/
+#define RGX_CR_SHG_BIF_CHECKSUM                                                     (0xD1E0U)
+#define RGX_CR_SHG_BIF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHG_BIF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SHG_BIF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHG_FE_BE_CHECKSUM
+*/
+#define RGX_CR_SHG_FE_BE_CHECKSUM                                                   (0xD1E8U)
+#define RGX_CR_SHG_FE_BE_CHECKSUM_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHG_FE_BE_CHECKSUM_VALUE_SHIFT                                       (0U)
+#define RGX_CR_SHG_FE_BE_CHECKSUM_VALUE_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register DPX_CR_BF_PERF
+*/
+#define DPX_CR_BF_PERF                                                              (0xC458U)
+#define DPX_CR_BF_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BF_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_BF_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_BF_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_BF_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_BF_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_BF_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_BF_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_BF_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_BF_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_BF_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_BF_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_BF_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_BF_PERF_SELECT0
+*/
+#define DPX_CR_BF_PERF_SELECT0                                                      (0xC460U)
+#define DPX_CR_BF_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_BF_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BF_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_BF_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BF_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_BF_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BF_PERF_COUNTER_0
+*/
+#define DPX_CR_BF_PERF_COUNTER_0                                                    (0xC488U)
+#define DPX_CR_BF_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BF_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_BF_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_BT_PERF
+*/
+#define DPX_CR_BT_PERF                                                              (0xC3D0U)
+#define DPX_CR_BT_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BT_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_BT_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_BT_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_BT_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_BT_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_BT_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_BT_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_BT_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_BT_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_BT_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_BT_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_BT_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_BT_PERF_SELECT0
+*/
+#define DPX_CR_BT_PERF_SELECT0                                                      (0xC3D8U)
+#define DPX_CR_BT_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_BT_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BT_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_BT_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BT_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_BT_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BT_PERF_COUNTER_0
+*/
+#define DPX_CR_BT_PERF_COUNTER_0                                                    (0xC420U)
+#define DPX_CR_BT_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BT_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_BT_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_RQ_USC_DEBUG
+*/
+#define DPX_CR_RQ_USC_DEBUG                                                         (0xC110U)
+#define DPX_CR_RQ_USC_DEBUG_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RQ_USC_DEBUG_CHECKSUM_SHIFT                                          (0U)
+#define DPX_CR_RQ_USC_DEBUG_CHECKSUM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register DPX_CR_BIF_FAULT_BANK_MMU_STATUS
+*/
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS                                            (0xC5C8U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_MASKFULL                                   (IMG_UINT64_C(0x000000000000F775))
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_CAT_BASE_SHIFT                             (12U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_CAT_BASE_CLRMSK                            (0xFFFF0FFFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_PAGE_SIZE_SHIFT                            (8U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_PAGE_SIZE_CLRMSK                           (0xFFFFF8FFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_DATA_TYPE_SHIFT                            (5U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_DATA_TYPE_CLRMSK                           (0xFFFFFF9FU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_SHIFT                             (4U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_CLRMSK                            (0xFFFFFFEFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_EN                                (0x00000010U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_SHIFT                     (2U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                    (0xFFFFFFFBU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_EN                        (0x00000004U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_SHIFT                                (0U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_CLRMSK                               (0xFFFFFFFEU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_EN                                   (0x00000001U)
+
+
+/*
+    Register DPX_CR_BIF_FAULT_BANK_REQ_STATUS
+*/
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS                                            (0xC5D0U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_MASKFULL                                   (IMG_UINT64_C(0x03FFFFFFFFFFFFF0))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_SHIFT                                  (57U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_CLRMSK                                 (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_EN                                     (IMG_UINT64_C(0x0200000000000000))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_SB_SHIFT                               (44U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_SB_CLRMSK                              (IMG_UINT64_C(0xFE000FFFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_ID_SHIFT                               (40U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_ID_CLRMSK                              (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_SHIFT                              (4U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_CLRMSK                             (IMG_UINT64_C(0xFFFFFF000000000F))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_ALIGNSHIFT                         (4U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_ALIGNSIZE                          (16U)
+
+
+/*
+    Register DPX_CR_BIF_MMU_STATUS
+*/
+#define DPX_CR_BIF_MMU_STATUS                                                       (0xC5D8U)
+#define DPX_CR_BIF_MMU_STATUS_MASKFULL                                              (IMG_UINT64_C(0x000000000FFFFFF7))
+#define DPX_CR_BIF_MMU_STATUS_PC_DATA_SHIFT                                         (20U)
+#define DPX_CR_BIF_MMU_STATUS_PC_DATA_CLRMSK                                        (0xF00FFFFFU)
+#define DPX_CR_BIF_MMU_STATUS_PD_DATA_SHIFT                                         (12U)
+#define DPX_CR_BIF_MMU_STATUS_PD_DATA_CLRMSK                                        (0xFFF00FFFU)
+#define DPX_CR_BIF_MMU_STATUS_PT_DATA_SHIFT                                         (4U)
+#define DPX_CR_BIF_MMU_STATUS_PT_DATA_CLRMSK                                        (0xFFFFF00FU)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_SHIFT                                         (2U)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_CLRMSK                                        (0xFFFFFFFBU)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_EN                                            (0x00000004U)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_SHIFT                                          (1U)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_CLRMSK                                         (0xFFFFFFFDU)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_EN                                             (0x00000002U)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_SHIFT                                            (0U)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_RT_PERF
+*/
+#define DPX_CR_RT_PERF                                                              (0xC700U)
+#define DPX_CR_RT_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_RT_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_RT_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_RT_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_RT_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_RT_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_RT_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_RT_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_RT_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_RT_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_RT_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_RT_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_RT_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_RT_PERF_SELECT0
+*/
+#define DPX_CR_RT_PERF_SELECT0                                                      (0xC708U)
+#define DPX_CR_RT_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_RT_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_RT_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_RT_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_RT_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_RT_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_RT_PERF_COUNTER_0
+*/
+#define DPX_CR_RT_PERF_COUNTER_0                                                    (0xC730U)
+#define DPX_CR_RT_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RT_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_RT_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_BX_TU_PERF
+*/
+#define DPX_CR_BX_TU_PERF                                                           (0xC908U)
+#define DPX_CR_BX_TU_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BX_TU_PERF_CLR_3_SHIFT                                               (4U)
+#define DPX_CR_BX_TU_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define DPX_CR_BX_TU_PERF_CLR_3_EN                                                  (0x00000010U)
+#define DPX_CR_BX_TU_PERF_CLR_2_SHIFT                                               (3U)
+#define DPX_CR_BX_TU_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define DPX_CR_BX_TU_PERF_CLR_2_EN                                                  (0x00000008U)
+#define DPX_CR_BX_TU_PERF_CLR_1_SHIFT                                               (2U)
+#define DPX_CR_BX_TU_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define DPX_CR_BX_TU_PERF_CLR_1_EN                                                  (0x00000004U)
+#define DPX_CR_BX_TU_PERF_CLR_0_SHIFT                                               (1U)
+#define DPX_CR_BX_TU_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define DPX_CR_BX_TU_PERF_CLR_0_EN                                                  (0x00000002U)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register DPX_CR_BX_TU_PERF_SELECT0
+*/
+#define DPX_CR_BX_TU_PERF_SELECT0                                                   (0xC910U)
+#define DPX_CR_BX_TU_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_SHIFT                                        (21U)
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BX_TU_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define DPX_CR_BX_TU_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BX_TU_PERF_COUNTER_0
+*/
+#define DPX_CR_BX_TU_PERF_COUNTER_0                                                 (0xC938U)
+#define DPX_CR_BX_TU_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BX_TU_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define DPX_CR_BX_TU_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register DPX_CR_RS_PDS_RR_CHECKSUM
+*/
+#define DPX_CR_RS_PDS_RR_CHECKSUM                                                   (0xC0F0U)
+#define DPX_CR_RS_PDS_RR_CHECKSUM_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RS_PDS_RR_CHECKSUM_VALUE_SHIFT                                       (0U)
+#define DPX_CR_RS_PDS_RR_CHECKSUM_VALUE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT                                            (0xE140U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_MASKFULL                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_SHIFT                                   (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_CLRMSK                                  (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING                                                    (0xE148U)
+#define RGX_CR_MMU_CBASE_MAPPING_MASKFULL                                           (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT                                    (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK                                   (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT                               (12U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS
+*/
+#define RGX_CR_MMU_FAULT_STATUS                                                     (0xE150U)
+#define RGX_CR_MMU_FAULT_STATUS_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_ADDRESS_SHIFT                                       (28U)
+#define RGX_CR_MMU_FAULT_STATUS_ADDRESS_CLRMSK                                      (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_CONTEXT_SHIFT                                       (20U)
+#define RGX_CR_MMU_FAULT_STATUS_CONTEXT_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFF00FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_TAG_SB_SHIFT                                        (12U)
+#define RGX_CR_MMU_FAULT_STATUS_TAG_SB_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFF00FFF))
+#define RGX_CR_MMU_FAULT_STATUS_REQ_ID_SHIFT                                        (6U)
+#define RGX_CR_MMU_FAULT_STATUS_REQ_ID_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_CR_MMU_FAULT_STATUS_LEVEL_SHIFT                                         (4U)
+#define RGX_CR_MMU_FAULT_STATUS_LEVEL_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_MMU_FAULT_STATUS_RNW_SHIFT                                           (3U)
+#define RGX_CR_MMU_FAULT_STATUS_RNW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_RNW_EN                                              (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT                                          (1U)
+#define RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_SHIFT                                         (0U)
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_EN                                            (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS_META                                                (0xE158U)
+#define RGX_CR_MMU_FAULT_STATUS_META_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT                                  (28U)
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK                                 (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT                                  (20U)
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF00FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_SHIFT                                   (12U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFF00FFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT                                   (6U)
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT                                    (4U)
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_SHIFT                                      (3U)
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_EN                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT                                     (1U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_SHIFT                                    (0U)
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN                                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC3_CTRL_MISC
+*/
+#define RGX_CR_SLC3_CTRL_MISC                                                       (0xE200U)
+#define RGX_CR_SLC3_CTRL_MISC_MASKFULL                                              (IMG_UINT64_C(0x0000000000000107))
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_SHIFT                                  (8U)
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_EN                                     (0x00000100U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SHIFT                                (0U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_CLRMSK                               (0xFFFFFFF8U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_LINEAR                               (0x00000000U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_IN_PAGE_HASH                         (0x00000001U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_FIXED_PVR_HASH                       (0x00000002U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SCRAMBLE_PVR_HASH                    (0x00000003U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_WEAVED_HASH                          (0x00000004U)
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE
+*/
+#define RGX_CR_SLC3_SCRAMBLE                                                        (0xE208U)
+#define RGX_CR_SLC3_SCRAMBLE_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE_BITS_SHIFT                                             (0U)
+#define RGX_CR_SLC3_SCRAMBLE_BITS_CLRMSK                                            (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE2
+*/
+#define RGX_CR_SLC3_SCRAMBLE2                                                       (0xE210U)
+#define RGX_CR_SLC3_SCRAMBLE2_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE2_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE2_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE3
+*/
+#define RGX_CR_SLC3_SCRAMBLE3                                                       (0xE218U)
+#define RGX_CR_SLC3_SCRAMBLE3_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE3_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE3_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE4
+*/
+#define RGX_CR_SLC3_SCRAMBLE4                                                       (0xE260U)
+#define RGX_CR_SLC3_SCRAMBLE4_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE4_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE4_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_STATUS
+*/
+#define RGX_CR_SLC3_STATUS                                                          (0xE220U)
+#define RGX_CR_SLC3_STATUS_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_STATUS_WRITES1_SHIFT                                            (48U)
+#define RGX_CR_SLC3_STATUS_WRITES1_CLRMSK                                           (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC3_STATUS_WRITES0_SHIFT                                            (32U)
+#define RGX_CR_SLC3_STATUS_WRITES0_CLRMSK                                           (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_SLC3_STATUS_READS1_SHIFT                                             (16U)
+#define RGX_CR_SLC3_STATUS_READS1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_SLC3_STATUS_READS0_SHIFT                                             (0U)
+#define RGX_CR_SLC3_STATUS_READS0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SLC3_IDLE
+*/
+#define RGX_CR_SLC3_IDLE                                                            (0xE228U)
+#define RGX_CR_SLC3_IDLE_MASKFULL                                                   (IMG_UINT64_C(0x00000000000FFFFF))
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST2_SHIFT                                         (18U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST2_CLRMSK                                        (0xFFF3FFFFU)
+#define RGX_CR_SLC3_IDLE_MMU_SHIFT                                                  (17U)
+#define RGX_CR_SLC3_IDLE_MMU_CLRMSK                                                 (0xFFFDFFFFU)
+#define RGX_CR_SLC3_IDLE_MMU_EN                                                     (0x00020000U)
+#define RGX_CR_SLC3_IDLE_RDI_SHIFT                                                  (16U)
+#define RGX_CR_SLC3_IDLE_RDI_CLRMSK                                                 (0xFFFEFFFFU)
+#define RGX_CR_SLC3_IDLE_RDI_EN                                                     (0x00010000U)
+#define RGX_CR_SLC3_IDLE_IMGBV4_SHIFT                                               (12U)
+#define RGX_CR_SLC3_IDLE_IMGBV4_CLRMSK                                              (0xFFFF0FFFU)
+#define RGX_CR_SLC3_IDLE_CACHE_BANKS_SHIFT                                          (4U)
+#define RGX_CR_SLC3_IDLE_CACHE_BANKS_CLRMSK                                         (0xFFFFF00FU)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST_SHIFT                                          (2U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST_CLRMSK                                         (0xFFFFFFF3U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_SHIFT                                         (1U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_EN                                            (0x00000002U)
+#define RGX_CR_SLC3_IDLE_XBAR_SHIFT                                                 (0U)
+#define RGX_CR_SLC3_IDLE_XBAR_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_SLC3_IDLE_XBAR_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC3_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC3_FAULT_STOP_STATUS                                               (0xE248U)
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_MASKFULL                                      (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_BIF_SHIFT                                     (0U)
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_BIF_CLRMSK                                    (0xFFFFE000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_MODE
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_MODE                                               (0xF048U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MASKFULL                                      (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_SHIFT                                    (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_CLRMSK                                   (0xFFFFFFFCU)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INDEX                                    (0x00000000U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INSTANCE                                 (0x00000001U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_LIST                                     (0x00000002U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING0
+*/
+#define RGX_CR_CONTEXT_MAPPING0                                                     (0xF078U)
+#define RGX_CR_CONTEXT_MAPPING0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_2D_SHIFT                                            (24U)
+#define RGX_CR_CONTEXT_MAPPING0_2D_CLRMSK                                           (0x00FFFFFFU)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_SHIFT                                           (16U)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_CLRMSK                                          (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING0_3D_SHIFT                                            (8U)
+#define RGX_CR_CONTEXT_MAPPING0_3D_CLRMSK                                           (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING0_TA_SHIFT                                            (0U)
+#define RGX_CR_CONTEXT_MAPPING0_TA_CLRMSK                                           (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING1
+*/
+#define RGX_CR_CONTEXT_MAPPING1                                                     (0xF080U)
+#define RGX_CR_CONTEXT_MAPPING1_MASKFULL                                            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_CONTEXT_MAPPING1_HOST_SHIFT                                          (8U)
+#define RGX_CR_CONTEXT_MAPPING1_HOST_CLRMSK                                         (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING1_TLA_SHIFT                                           (0U)
+#define RGX_CR_CONTEXT_MAPPING1_TLA_CLRMSK                                          (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING2
+*/
+#define RGX_CR_CONTEXT_MAPPING2                                                     (0xF088U)
+#define RGX_CR_CONTEXT_MAPPING2_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING3
+*/
+#define RGX_CR_CONTEXT_MAPPING3                                                     (0xF090U)
+#define RGX_CR_CONTEXT_MAPPING3_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_JONES_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ                                           (0xF098U)
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_COUNTER_SHIFT                             (0U)
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_COUNTER_CLRMSK                            (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ                                      (0xF0A0U)
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_MASKFULL                             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_COUNTER_SHIFT                        (0U)
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_COUNTER_CLRMSK                       (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_DUST_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ                                            (0xF0A8U)
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_MASKFULL                                   (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_COUNTER_SHIFT                              (0U)
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_COUNTER_CLRMSK                             (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_JONES_FIX
+*/
+#define RGX_CR_JONES_FIX                                                            (0xF0C0U)
+#define RGX_CR_JONES_FIX__ROGUE3__MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX_MASKFULL                                                   (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX_DISABLE_SHIFT                                              (0U)
+#define RGX_CR_JONES_FIX_DISABLE_CLRMSK                                             (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING4
+*/
+#define RGX_CR_CONTEXT_MAPPING4                                                     (0xF210U)
+#define RGX_CR_CONTEXT_MAPPING4_MASKFULL                                            (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_SHIFT                                  (40U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_SHIFT                                    (32U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_SHIFT                                     (24U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_SHIFT                                  (16U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_SHIFT                                    (8U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_SHIFT                                     (0U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MULTICORE_GPU
+*/
+#define RGX_CR_MULTICORE_GPU                                                        (0xF300U)
+#define RGX_CR_MULTICORE_GPU_MASKFULL                                               (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_SHIFT                              (6U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_EN                                 (0x00000040U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_SHIFT                              (5U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_EN                                 (0x00000020U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_SHIFT                               (4U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_CLRMSK                              (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_EN                                  (0x00000010U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_SHIFT                               (3U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_CLRMSK                              (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN                                  (0x00000008U)
+#define RGX_CR_MULTICORE_GPU_ID_SHIFT                                               (0U)
+#define RGX_CR_MULTICORE_GPU_ID_CLRMSK                                              (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_MULTICORE_SYSTEM
+*/
+#define RGX_CR_MULTICORE_SYSTEM                                                     (0xF308U)
+#define RGX_CR_MULTICORE_SYSTEM_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON                                       (0xF310U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT          (8U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK         (0xC00000FFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON                                       (0xF320U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT          (8U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK         (0xC00000FFU)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON                                        (0xF330U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                    (30U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                   (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT           (8U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK          (0xC00000FFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_ERR_INJ
+*/
+#define RGX_CR_ECC_RAM_ERR_INJ                                                      (0xF340U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MASKFULL                                             (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_SHIFT                                   (4U)
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_EN                                      (0x00000010U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_SHIFT                                            (3U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_EN                                               (0x00000008U)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_SHIFT                                     (2U)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_EN                                        (0x00000004U)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_SHIFT                                         (1U)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_EN                                            (0x00000002U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_SHIFT                                           (0U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_INIT_KICK
+*/
+#define RGX_CR_ECC_RAM_INIT_KICK                                                    (0xF348U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MASKFULL                                           (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_SHIFT                                 (4U)
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_EN                                    (0x00000010U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_SHIFT                                          (3U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_EN                                             (0x00000008U)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_SHIFT                                   (2U)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_EN                                      (0x00000004U)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_SHIFT                                       (1U)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_EN                                          (0x00000002U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_SHIFT                                         (0U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_INIT_DONE
+*/
+#define RGX_CR_ECC_RAM_INIT_DONE                                                    (0xF350U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MASKFULL                                           (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_SHIFT                                 (4U)
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_EN                                    (0x00000010U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_SHIFT                                          (3U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_EN                                             (0x00000008U)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_SHIFT                                   (2U)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_EN                                      (0x00000004U)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_SHIFT                                       (1U)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_EN                                          (0x00000002U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_SHIFT                                         (0U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_ENABLE
+*/
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE                                         (0xF390U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL                               (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_SHIFT                       (7U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_CLRMSK                      (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_EN                          (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_SHIFT                   (6U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_EN                      (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                (5U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK               (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_EN                   (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                 (4U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_EN                    (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_SHIFT                         (3U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_CLRMSK                        (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_EN                            (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_SHIFT                         (2U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_EN                            (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_SHIFT                        (1U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_EN                           (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_SHIFT                   (0U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_STATUS
+*/
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE                                         (0xF398U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__MASKFULL                               (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_SHIFT                       (7U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_CLRMSK                      (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_EN                          (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_SHIFT                   (6U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_EN                      (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                (5U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK               (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_EN                   (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                 (4U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_EN                    (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_SHIFT                         (3U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_CLRMSK                        (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_EN                            (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_SHIFT                         (2U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_EN                            (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_SHIFT                        (1U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_EN                           (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_SHIFT                   (0U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_CLEAR
+*/
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE                                          (0xF3A0U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__MASKFULL                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_SHIFT                        (7U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_EN                           (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_SHIFT                    (6U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                   (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_EN                       (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                 (5U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK                (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_EN                    (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                  (4U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_EN                     (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_SHIFT                          (3U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_EN                             (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_SHIFT                          (2U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_EN                             (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_SHIFT                         (1U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_EN                            (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_SHIFT                    (0U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                   (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_EN                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_STATUS
+*/
+#define RGX_CR_FAULT_FW_STATUS                                                      (0xF3B0U)
+#define RGX_CR_FAULT_FW_STATUS_MASKFULL                                             (IMG_UINT64_C(0x0000000000010001))
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_SHIFT                                    (16U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_CLRMSK                                   (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_EN                                       (0x00010000U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_SHIFT                                     (0U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_CLEAR
+*/
+#define RGX_CR_FAULT_FW_CLEAR                                                       (0xF3B8U)
+#define RGX_CR_FAULT_FW_CLEAR_MASKFULL                                              (IMG_UINT64_C(0x0000000000010001))
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_SHIFT                                     (16U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_EN                                        (0x00010000U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_SHIFT                                      (0U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_SAFETY_EVENT_ENABLE
+*/
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE                                     (0xF3D8U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL                           (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_SHIFT                   (7U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_CLRMSK                  (0xFFFFFF7FU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_EN                      (0x00000080U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_SHIFT               (6U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_CLRMSK              (0xFFFFFFBFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_EN                  (0x00000040U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT            (5U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK           (0xFFFFFFDFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_EN               (0x00000020U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT             (4U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_EN                (0x00000010U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_SHIFT                     (3U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_CLRMSK                    (0xFFFFFFF7U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_EN                        (0x00000008U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_SHIFT                     (2U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_CLRMSK                    (0xFFFFFFFBU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_EN                        (0x00000004U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_SHIFT                    (1U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_CLRMSK                   (0xFFFFFFFDU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_EN                       (0x00000002U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_SHIFT               (0U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_CLRMSK              (0xFFFFFFFEU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_EN                  (0x00000001U)
+
+
+#endif /* RGX_CR_DEFS_KM_H */
+/*****************************************************************************
+ End of file (rgx_cr_defs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxdefs_km.h
new file mode 100644
index 000000000000..818166dff902
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxdefs_km.h
@@ -0,0 +1,381 @@
+/*************************************************************************/ /*!
+@Title          Rogue hw definitions (kernel mode)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXDEFS_KM_H
+#define RGXDEFS_KM_H
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+#include RGX_BVNC_CORE_KM_HEADER
+#include RGX_BNC_CONFIG_KM_HEADER
+#endif
+
+#define IMG_EXPLICIT_INCLUDE_HWDEFS
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#include "rgx_cr_defs_km.h"
+#endif
+#undef IMG_EXPLICIT_INCLUDE_HWDEFS
+
+#include "rgx_heap_firmware.h"
+
+/* The following Macros are picked up through BVNC headers for no hardware
+ * operations to be compatible with old build infrastructure.
+ */
+#if defined(NO_HARDWARE)
+/******************************************************************************
+ * Check for valid B.X.N.C
+ *****************************************************************************/
+#if !defined(RGX_BVNC_KM_B) || !defined(RGX_BVNC_KM_V) || !defined(RGX_BVNC_KM_N) || !defined(RGX_BVNC_KM_C)
+#error "Need to specify BVNC (RGX_BVNC_KM_B, RGX_BVNC_KM_V, RGX_BVNC_KM_N and RGX_BVNC_C)"
+#endif
+
+/* Check core/config compatibility */
+#if (RGX_BVNC_KM_B != RGX_BNC_KM_B) || (RGX_BVNC_KM_N != RGX_BNC_KM_N) || (RGX_BVNC_KM_C != RGX_BNC_KM_C)
+#error "BVNC headers are mismatching (KM core/config)"
+#endif
+#endif
+
+/******************************************************************************
+ * RGX Version name
+ *****************************************************************************/
+#define RGX_BVNC_KM_ST2(S)	#S
+#define RGX_BVNC_KM_ST(S)	RGX_BVNC_KM_ST2(S)
+#define RGX_BVNC_KM			RGX_BVNC_KM_ST(RGX_BVNC_KM_B) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_V) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_N) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_C)
+#define RGX_BVNC_KM_V_ST	RGX_BVNC_KM_ST(RGX_BVNC_KM_V)
+
+/* Maximum string size is [bb.vvvp.nnnn.cccc\0], includes null char */
+#define RGX_BVNC_STR_SIZE_MAX (2U+1U+4U+1U+4U+1U+4U+1U)
+#define RGX_BVNC_STR_FMTSPEC  "%u.%u.%u.%u"
+#define RGX_BVNC_STRP_FMTSPEC "%u.%up.%u.%u"
+
+
+/******************************************************************************
+ * RGX Defines
+ *****************************************************************************/
+
+#define BVNC_FIELD_MASK     ((1UL << BVNC_FIELD_WIDTH) - 1U)
+#define C_POSITION          (0U)
+#define N_POSITION          ((C_POSITION) + (BVNC_FIELD_WIDTH))
+#define V_POSITION          ((N_POSITION) + (BVNC_FIELD_WIDTH))
+#define B_POSITION          ((V_POSITION) + (BVNC_FIELD_WIDTH))
+
+#define B_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (B_POSITION)))
+#define V_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (V_POSITION)))
+#define N_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (N_POSITION)))
+#define C_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (C_POSITION)))
+
+#define GET_B(x)            (((x) & (B_POSTION_MASK)) >> (B_POSITION))
+#define GET_V(x)            (((x) & (V_POSTION_MASK)) >> (V_POSITION))
+#define GET_N(x)            (((x) & (N_POSTION_MASK)) >> (N_POSITION))
+#define GET_C(x)            (((x) & (C_POSTION_MASK)) >> (C_POSITION))
+
+#define BVNC_PACK(B,V,N,C)  (((((IMG_UINT64)(B))) << (B_POSITION)) | \
+                             ((((IMG_UINT64)(V))) << (V_POSITION)) | \
+                             ((((IMG_UINT64)(N))) << (N_POSITION)) | \
+                             ((((IMG_UINT64)(C))) << (C_POSITION)) \
+                            )
+
+#define RGX_CR_CORE_ID_CONFIG_N_SHIFT    (8U)
+#define RGX_CR_CORE_ID_CONFIG_C_SHIFT    (0U)
+
+#define RGX_CR_CORE_ID_CONFIG_N_CLRMSK   (0XFFFF00FFU)
+#define RGX_CR_CORE_ID_CONFIG_C_CLRMSK   (0XFFFFFF00U)
+
+#if defined(RGX_CR_CORE_ID__PBVNC)
+#define GET_PBVNC_B(X)	((X & ~RGX_CR_CORE_ID__PBVNC__BRANCH_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__BRANCH_ID_SHIFT)
+#define GET_PBVNC_V(X)	((X & ~RGX_CR_CORE_ID__PBVNC__VERSION_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__VERSION_ID_SHIFT)
+#define GET_PBVNC_N(X)	((X & ~RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_SHIFT)
+#define GET_PBVNC_C(X)	((X & ~RGX_CR_CORE_ID__PBVNC__CONFIG_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__CONFIG_ID_SHIFT)
+#endif
+
+#if defined(RGX_FEATURE_NUM_OSIDS)
+#define RGXFW_MAX_NUM_OSIDS                               (RGX_FEATURE_NUM_OSIDS)
+#else
+#define RGXFW_MAX_NUM_OSIDS                               (8U)
+#endif
+
+#define RGXFW_HOST_DRIVER_ID                              (0U)
+#define RGXFW_GUEST_DRIVER_ID_START                       (RGXFW_HOST_DRIVER_ID + 1U)
+
+#define RGXFW_THREAD_0                                    (0U)
+#define RGXFW_THREAD_1                                    (1U)
+
+/* META cores (required for the RGX_FEATURE_META) */
+#define MTP218   (1U)
+#define MTP219   (2U)
+#define LTP218   (3U)
+#define LTP217   (4U)
+
+/* META Core memory feature depending on META variants */
+#define RGX_META_COREMEM_32K      (32*1024)
+#define RGX_META_COREMEM_48K      (48*1024)
+#define RGX_META_COREMEM_64K      (64*1024)
+#define RGX_META_COREMEM_96K      (96*1024)
+#define RGX_META_COREMEM_128K     (128*1024)
+#define RGX_META_COREMEM_256K     (256*1024)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+#if (!defined(SUPPORT_TRUSTED_DEVICE) || defined(RGX_FEATURE_META_DMA)) && \
+    (defined(RGX_FEATURE_META_COREMEM_SIZE) && RGX_FEATURE_META_COREMEM_SIZE != 0)
+#define RGX_META_COREMEM_SIZE     (RGX_FEATURE_META_COREMEM_SIZE*1024U)
+#define RGX_META_COREMEM          (1)
+#define RGX_META_COREMEM_CODE     (1)
+#if !defined(FIX_HW_BRN_50767) && (!defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1))
+#define RGX_META_COREMEM_DATA     (1)
+#endif
+#else
+#undef SUPPORT_META_COREMEM
+#undef RGX_FEATURE_META_COREMEM_SIZE
+#undef RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_COREMEM_SIZE (0)
+#define RGX_META_COREMEM_SIZE         (0)
+#endif
+#endif
+
+#define GET_ROGUE_CACHE_LINE_SIZE(x)    ((((IMG_UINT32)(x)) > 0U) ? ((IMG_UINT32)(x)/8U) : (0U))
+
+#if defined(RGX_FEATURE_META_DMA)
+#define RGX_META_DMA_BLOCK_SIZE (32U)
+#else
+#define RGX_META_DMA_BLOCK_SIZE (0U)
+#endif
+
+#if defined(SUPPORT_AGP)
+#define MAX_HW_TA3DCONTEXTS	3U
+#else
+#define MAX_HW_TA3DCONTEXTS	2U
+#endif
+
+#define RGX_CR_CLK_CTRL_ALL_ON          (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL_MASKFULL)
+#define RGX_CR_CLK_CTRL_ALL_AUTO        (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_ON         (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_AUTO       (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_XTPLUS_CTRL_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_XTPLUS_CTRL_MASKFULL)
+#define RGX_CR_CLK_XTPLUS_CTRL_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_XTPLUS_CTRL_MASKFULL)
+#define DPX_CR_DPX_CLK_CTRL_ALL_ON      (IMG_UINT64_C(0x5555555555555555)&DPX_CR_DPX_CLK_CTRL_MASKFULL)
+#define DPX_CR_DPX_CLK_CTRL_ALL_AUTO    (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&DPX_CR_DPX_CLK_CTRL_MASKFULL)
+
+#define RGX_CR_SOFT_RESET_DUST_n_CORE_EN	(RGX_CR_SOFT_RESET_DUST_A_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_B_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_C_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_D_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_E_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_F_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_G_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_H_CORE_EN)
+
+/* SOFT_RESET Rascal and DUSTs bits */
+#define RGX_CR_SOFT_RESET_RASCALDUSTS_EN	(RGX_CR_SOFT_RESET_RASCAL_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_n_CORE_EN)
+
+
+
+
+/* SOFT_RESET steps as defined in the TRM */
+#define RGX_S7_SOFT_RESET_DUSTS (RGX_CR_SOFT_RESET_DUST_n_CORE_EN)
+
+#define RGX_S7_SOFT_RESET_JONES (RGX_CR_SOFT_RESET_PM_EN  | \
+                                 RGX_CR_SOFT_RESET_VDM_EN | \
+                                 RGX_CR_SOFT_RESET_ISP_EN)
+
+#define RGX_S7_SOFT_RESET_JONES_ALL (RGX_S7_SOFT_RESET_JONES  | \
+                                     RGX_CR_SOFT_RESET_BIF_EN | \
+                                     RGX_CR_SOFT_RESET_SLC_EN | \
+                                     RGX_CR_SOFT_RESET_GARTEN_EN)
+
+#define RGX_S7_SOFT_RESET2 (RGX_CR_SOFT_RESET2_BLACKPEARL_EN | \
+                            RGX_CR_SOFT_RESET2_PIXEL_EN | \
+                            RGX_CR_SOFT_RESET2_CDM_EN | \
+                            RGX_CR_SOFT_RESET2_VERTEX_EN)
+
+
+/* PM interactive mode enabled default for all rogue cores */
+#define PM_INTERACTIVE_MODE
+
+#define RGX_MLIST_ENTRY_STRIDE	(4U) /* 4 bytes */
+#define RGX_NUM_PM_ADDR_SPACES	(2U) /* VCE & TE share virtual space and Alist */
+#define RGX_PM_MAX_PB_VIRT_ADDR_SPACE  (IMG_UINT64_C(0x400000000)) /* PM Maximum addressable limit */
+
+#define RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT		(12U)
+#define RGX_BIF_PM_PHYSICAL_PAGE_SIZE			(1UL << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT		(14U)
+#define RGX_BIF_PM_VIRTUAL_PAGE_SIZE			(1UL << RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE	(16U)
+
+/* To get the number of required Dusts, divide the number of
+ * clusters by 2 and round up
+ */
+#define RGX_REQ_NUM_DUSTS(CLUSTERS)    (((CLUSTERS) + 1U) / 2U)
+
+/* To get the number of required Bernado/Phantom(s), divide
+ * the number of clusters by 4 and round up
+ */
+#define RGX_REQ_NUM_PHANTOMS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+#define RGX_REQ_NUM_BERNADOS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+#define RGX_REQ_NUM_BLACKPEARLS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+# define RGX_NUM_PHANTOMS (RGX_REQ_NUM_PHANTOMS(RGX_FEATURE_NUM_CLUSTERS))
+#endif
+
+
+/* META second thread feature depending on META variants and
+ * available CoreMem
+ */
+#if defined(RGX_FEATURE_META) && (RGX_FEATURE_META == MTP218 || RGX_FEATURE_META == MTP219) && defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && (RGX_FEATURE_META_COREMEM_SIZE == 256)
+#define RGXFW_META_SUPPORT_2ND_THREAD
+#endif
+
+/*
+ * Utility macros to calculate CAT_BASE register addresses
+ */
+#define BIF_CAT_BASEx(n) \
+	(RGX_CR_BIF_CAT_BASE0 + ((n) * (RGX_CR_BIF_CAT_BASE1 - RGX_CR_BIF_CAT_BASE0)))
+
+#define FWCORE_MEM_CAT_BASEx(n) \
+	(RGX_CR_FWCORE_MEM_CAT_BASE0 + ((n) * (RGX_CR_FWCORE_MEM_CAT_BASE1 - RGX_CR_FWCORE_MEM_CAT_BASE0)))
+
+/*
+ * FWCORE wrapper register defines
+ */
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT   RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK  RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK
+#define FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT     (12U)
+
+/******************************************************************************
+ * WA HWBRNs
+ *****************************************************************************/
+
+#if defined(RGX_CR_JONES_IDLE_MASKFULL)
+/* Workaround for HW BRN 57289 */
+#if (RGX_CR_JONES_IDLE_MASKFULL != 0x0000000000007FFF)
+#error This WA must be updated if RGX_CR_JONES_IDLE is expanded!!!
+#endif
+#undef RGX_CR_JONES_IDLE_MASKFULL
+#undef RGX_CR_JONES_IDLE_TDM_SHIFT
+#undef RGX_CR_JONES_IDLE_TDM_CLRMSK
+#undef RGX_CR_JONES_IDLE_TDM_EN
+#define RGX_CR_JONES_IDLE_MASKFULL                        (IMG_UINT64_C(0x0000000000003FFF))
+#endif
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+
+#if defined(RGX_FEATURE_ROGUEXE)
+#define RGX_NUM_RASTERISATION_MODULES	RGX_FEATURE_NUM_CLUSTERS
+#else
+#define RGX_NUM_RASTERISATION_MODULES	RGX_NUM_PHANTOMS
+#endif
+
+#endif /* !(defined(__KERNEL__) || defined(TEE_DDK)) */
+
+/* GPU CR timer tick in GPU cycles */
+#define RGX_CRTIME_TICK_IN_CYCLES					(256U)
+#define RGX_CRTIME_TICK_IN_CYCLES_SHIFT				(8U)
+
+/* for nohw multicore return max cores possible to client */
+#define RGX_MULTICORE_MAX_NOHW_CORES               (4U)
+
+/*
+  If the size of the SLC is less than this value then the TPU bypasses the SLC.
+ */
+#define RGX_TPU_CACHED_SLC_SIZE_THRESHOLD_KB			(128U)
+
+/*
+ * If the size of the SLC is bigger than this value then the TCU must not be bypassed in the SLC.
+ * In XE_MEMORY_HIERARCHY cores, the TCU is bypassed by default.
+ */
+#define RGX_TCU_CACHED_SLC_SIZE_THRESHOLD_KB			(32U)
+
+/*
+ * Register used by the FW to track the current boot stage (not used in MIPS)
+ */
+#define RGX_FW_BOOT_STAGE_REGISTER     (RGX_CR_POWER_ESTIMATE_RESULT)
+
+/*
+ * Virtualisation definitions
+ */
+#define RGX_VIRTUALISATION_REG_SIZE_PER_OS (RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE)
+
+/*
+ * Renaming MTS sideband bitfields to emphasize that the Register Bank number
+ * of the MTS register used identifies a specific Driver/VM rather than the OSID tag
+ * emitted on bus memory transactions.
+ */
+#define RGX_MTS_SBDATA_DRIVERID_CLRMSK RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_CLRMSK
+#define RGX_MTS_SBDATA_DRIVERID_SHIFT RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_SHIFT
+
+/*
+ * Register Bank containing registers secured against host access
+ */
+#define RGX_HOST_SECURE_REGBANK_OFFSET				(0xF0000U)
+#define RGX_HOST_SECURE_REGBANK_SIZE				(0x10000U)
+
+/*
+	Maximum number of render targets in array
+*/
+#define RGX_MAX_TA_RENDER_TARGETS					(2048U)
+
+/*
+ * Macro used to indicate which version of HWPerf is active
+ */
+#define RGX_FEATURE_HWPERF_ROGUE
+
+/*
+ * Maximum number of cores supported by TRP
+ */
+#define RGX_TRP_MAX_NUM_CORES                           (4U)
+
+/*
+ * Maximum number of cores supported by WGP
+ */
+#define RGX_WGP_MAX_NUM_CORES                           (8U)
+
+#if defined(RGX_FEATURE_VOLCANIC_TB)
+#define SUPPORT_VOLCANIC_TB
+#endif
+#define RGX_FEATURE_SECURITY_ROGUE
+
+/* Typically the PCI bus returns this value on error */
+#define RGX_PCI_ERROR_VALUE_BYTE (0xFFU)
+#define RGX_PCI_ERROR_VALUE_DWORD (0xFFFFFFFFU)
+
+#endif /* RGXDEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmhdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmhdefs_km.h
new file mode 100644
index 000000000000..663001dd9ebc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmhdefs_km.h
@@ -0,0 +1,286 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmhdefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_mh.def
+ */
+
+
+#ifndef RGXMHDEFS_KM_H
+#define RGXMHDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMHDEFS_KM_REVISION 0
+
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_FENCE                            (0x00000000U)
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_CONTEXT                          (0x00000001U)
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_QUEUE                            (0x00000002U)
+
+
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTL_STREAM                       (0x00000000U)
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTX_BUFFER                       (0x00000001U)
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_QUEUE_CTL                        (0x00000002U)
+
+
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAFSTACK                              (0x00000008U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMLIST                               (0x00000009U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DFSTACK                              (0x0000000aU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMLIST                               (0x0000000bU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX0                                (0x0000000cU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX1                                (0x0000002dU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_UFSTACK                               (0x0000000fU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMMUSTACK                            (0x00000012U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMMUSTACK                            (0x00000013U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAUFSTACK                             (0x00000016U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DUFSTACK                             (0x00000017U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DVFP                                 (0x00000019U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAVFP                                 (0x0000001aU)
+
+
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAFSTACK                              (0x00000000U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMLIST                               (0x00000001U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DFSTACK                              (0x00000002U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMLIST                               (0x00000003U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX0                                (0x00000004U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX1                                (0x00000025U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_MAVP                                  (0x00000006U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_UFSTACK                               (0x00000007U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMMUSTACK                            (0x00000008U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMMUSTACK                            (0x00000009U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAUFSTACK                             (0x00000014U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DUFSTACK                             (0x00000015U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAVFP                                 (0x00000018U)
+
+
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PPP                                        (0x00000008U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_VCERTC                                     (0x00000007U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_TEACRTC                                    (0x00000006U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGRTC                                     (0x00000005U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGR                                       (0x00000004U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGS                                       (0x00000003U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_TPC                                        (0x00000002U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_VCE                                        (0x00000001U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_PREQ00                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_PREQ01                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_RREQ                     (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_DBSC                     (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CPF                      (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_DELTA                    (0x00000007U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ02                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ03                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ00                   (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ01                   (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ02                   (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ03                   (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_RREQ                     (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_DBSC                     (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CPF                      (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_DELTA                    (0x0000000bU)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ02                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ03                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ04                   (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ05                   (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ06                   (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ00                   (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ01                   (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ02                   (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ03                   (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ04                   (0x0000000bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ05                   (0x0000000cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ06                   (0x0000000dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_RREQ                     (0x0000000eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_DBSC                     (0x0000000fU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CPF                      (0x00000010U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_DELTA                    (0x00000011U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ00                  (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ01                  (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ02                  (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ03                  (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ04                  (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ05                  (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ06                  (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ07                  (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ08                  (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ09                  (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ10                  (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ11                  (0x0000000bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ12                  (0x0000000cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ13                  (0x0000000dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ00                  (0x0000000eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ01                  (0x0000000fU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ02                  (0x00000010U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ03                  (0x00000011U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ04                  (0x00000012U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ05                  (0x00000013U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ06                  (0x00000014U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ07                  (0x00000015U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ08                  (0x00000016U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ09                  (0x00000017U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ10                  (0x00000018U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ11                  (0x00000019U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ12                  (0x0000001aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ13                  (0x0000001bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_RREQ                    (0x0000001cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_DBSC                    (0x0000001dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CPF                     (0x0000001eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_DELTA                   (0x0000001fU)
+
+
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_PDS_STATE                                (0x00000000U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DEPTH_BIAS                               (0x00000001U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_FLOOR_OFFSET_DATA                        (0x00000002U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DELTA_DATA                               (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_ISP_ENCODING_ISP_TAG_ZLS                                      (0x00000000U)
+#define RGX_MH_TAG_SB_ISP_ENCODING_ISP_TAG_DS                                       (0x00000001U)
+
+
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTROL                                  (0x00000000U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STATE                                    (0x00000001U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_INDEX                                    (0x00000002U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STACK                                    (0x00000004U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTEXT                                  (0x00000008U)
+
+
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTROL_STREAM                           (0x00000000U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_INDIRECT_DATA                            (0x00000001U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_EVENT_DATA                               (0x00000002U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTEXT_STATE                            (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_OPCODE_FETCH                           (0x00000002U)
+#define RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_DATA_ACCESS                            (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PT_REQUEST                               (0x00000000U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PD_REQUEST                               (0x00000001U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PC_REQUEST                               (0x00000002U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PT_REQUEST                            (0x00000003U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_REQUEST                            (0x00000004U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_REQUEST                            (0x00000005U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_WREQUEST                           (0x00000006U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_WREQUEST                           (0x00000007U)
+
+
+#define RGX_MH_TAG_ENCODING_MH_TAG_MMU                                              (0x00000000U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU                                          (0x00000001U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU                                          (0x00000002U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU                                          (0x00000003U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MIPS                                             (0x00000004U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0                                         (0x00000005U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1                                         (0x00000006U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2                                         (0x00000007U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3                                         (0x00000008U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0                                         (0x00000009U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1                                         (0x0000000aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2                                         (0x0000000bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3                                         (0x0000000cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4                                         (0x0000000dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PDS_0                                            (0x0000000eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PDS_1                                            (0x0000000fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCA                                         (0x00000010U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCB                                         (0x00000011U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCC                                         (0x00000012U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCD                                         (0x00000013U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCA                                     (0x00000014U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCB                                     (0x00000015U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCC                                     (0x00000016U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCD                                     (0x00000017U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDSRW                                        (0x00000018U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TCU_0                                            (0x00000019U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TCU_1                                            (0x0000001aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_0                                          (0x0000001bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_1                                          (0x0000001cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_2                                          (0x0000001dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_3                                          (0x0000001eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_USC                                              (0x0000001fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_ISP_ZLS                                          (0x00000020U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_ISP_DS                                           (0x00000021U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF                                              (0x00000022U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS                                     (0x00000023U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF                                          (0x00000024U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ                                         (0x00000025U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS                                       (0x00000026U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG5                                         (0x00000027U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PPP                                           (0x00000028U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPWRTC                                        (0x00000029U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TEACRTC                                       (0x0000002aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGRTC                                        (0x0000002bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGREGION                                     (0x0000002cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGSTREAM                                     (0x0000002dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPW                                           (0x0000002eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPC                                           (0x0000002fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PM_ALLOC                                         (0x00000030U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PM_DEALLOC                                       (0x00000031U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TDM_DMA                                          (0x00000032U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TDM_CTL                                          (0x00000033U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE0                                             (0x00000034U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE1                                             (0x00000035U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE2                                             (0x00000036U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE3                                             (0x00000037U)
+
+
+#endif /* RGXMHDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmhdefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmmudefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmmudefs_km.h
new file mode 100644
index 000000000000..14a6a73c84bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmmudefs_km.h
@@ -0,0 +1,216 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmmudefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_bif.def
+ */
+
+
+#ifndef RGXMMUDEFS_KM_H
+#define RGXMMUDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMMUDEFS_KM_REVISION 0
+
+#define RGX_BIF_DM_ENCODING_VERTEX                                                  (0x00000000U)
+#define RGX_BIF_DM_ENCODING_PIXEL                                                   (0x00000001U)
+#define RGX_BIF_DM_ENCODING_COMPUTE                                                 (0x00000002U)
+#define RGX_BIF_DM_ENCODING_TLA                                                     (0x00000003U)
+#define RGX_BIF_DM_ENCODING_PB_VCE                                                  (0x00000004U)
+#define RGX_BIF_DM_ENCODING_PB_TE                                                   (0x00000005U)
+#define RGX_BIF_DM_ENCODING_META                                                    (0x00000007U)
+#define RGX_BIF_DM_ENCODING_HOST                                                    (0x00000008U)
+#define RGX_BIF_DM_ENCODING_PM_ALIST                                                (0x00000009U)
+
+
+#define RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT                                            (30U)
+#define RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT                                            (21U)
+#define RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFC01FFFFF))
+#define RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT                                            (12U)
+#define RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFE00FFF))
+
+
+#define RGX_MMUCTRL_ENTRIES_PC_VALUE                                                (0x00000400U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PD_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PT_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PC_VALUE                                             (0x00000020U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PD_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PT_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_PAGE_SIZE_MASK                                                  (0x00000007U)
+#define RGX_MMUCTRL_PAGE_SIZE_4KB                                                   (0x00000000U)
+#define RGX_MMUCTRL_PAGE_SIZE_16KB                                                  (0x00000001U)
+#define RGX_MMUCTRL_PAGE_SIZE_64KB                                                  (0x00000002U)
+#define RGX_MMUCTRL_PAGE_SIZE_256KB                                                 (0x00000003U)
+#define RGX_MMUCTRL_PAGE_SIZE_1MB                                                   (0x00000004U)
+#define RGX_MMUCTRL_PAGE_SIZE_2MB                                                   (0x00000005U)
+
+
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT                                            (12U)
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_SHIFT                                           (14U)
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF0000003FFF))
+
+
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_SHIFT                                           (16U)
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_SHIFT                                          (18U)
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFF000003FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_SHIFT                                            (20U)
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00000FFFFF))
+
+
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_SHIFT                                            (21U)
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00001FFFFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_SHIFT                                         (12U)
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_SHIFT                                        (10U)
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000003FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_SHIFT                                        (8U)
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000000FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_SHIFT                                       (6U)
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF000000003F))
+
+
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_SHIFT                                   (62U)
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_CLRMSK                                  (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_SHIFT                                        (40U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_CLRMSK                                       (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PAGE_SHIFT                                              (12U)
+#define RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_SHIFT                                        (6U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_SHIFT                                     (5U)
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000000000000020))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_SHIFT                                            (4U)
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_SHIFT                                   (3U)
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN                                      (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PT_DATA_CC_SHIFT                                                (2U)
+#define RGX_MMUCTRL_PT_DATA_CC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_MMUCTRL_PT_DATA_CC_EN                                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_EN                                            (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PT_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PT_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PT_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_SHIFT                                     (40U)
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_SHIFT                                           (5U)
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF1))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB                                         (IMG_UINT64_C(0x0000000000000006))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB                                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB                                           (IMG_UINT64_C(0x000000000000000a))
+#define RGX_MMUCTRL_PD_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PD_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PD_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT                                           (4U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK                                          (0x0000000FU)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT                                      (12U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE                                       (4096U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_SHIFT                                     (1U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN                                        (0x00000002U)
+#define RGX_MMUCTRL_PC_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PC_DATA_VALID_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_MMUCTRL_PC_DATA_VALID_EN                                                (0x00000001U)
+
+
+#endif /* RGXMMUDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmmudefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_27.V.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_27.V.254.2.h
new file mode 100644
index 000000000000..3d7bc8345c05
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_27.V.254.2.h
@@ -0,0 +1,136 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 27.V.254.2 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_27_V_254_2_H
+#define RGXCONFIG_KM_27_V_254_2_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 27
+#define RGX_BNC_KM_N 254
+#define RGX_BNC_KM_C 2
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (3U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_MMU_VERSION (3U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (2U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_27_V_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.1632.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.1632.1.h
new file mode 100644
index 000000000000..caafb57427d6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.1632.1.h
@@ -0,0 +1,140 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 30.V.1632.1 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_30_V_1632_1_H
+#define RGXCONFIG_KM_30_V_1632_1_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 30
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 1
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (512U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_30_V_1632_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.408.101.h
new file mode 100644
index 000000000000..7e9a1f7b3cd0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.408.101.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 30.V.408.101 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_30_V_408_101_H
+#define RGXCONFIG_KM_30_V_408_101_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 30
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 101
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_30_V_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.816.20.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.816.20.h
new file mode 100644
index 000000000000..f09e158a5ab8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.816.20.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 30.V.816.20 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_30_V_816_20_H
+#define RGXCONFIG_KM_30_V_816_20_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 30
+#define RGX_BNC_KM_N 816
+#define RGX_BNC_KM_C 20
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (256U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_30_V_816_20_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.21.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.21.h
new file mode 100644
index 000000000000..b0faae645326
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.21.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.21 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_21_H
+#define RGXCONFIG_KM_35_V_1632_21_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 21
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (512U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_21_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.23.h
new file mode 100644
index 000000000000..07fd015bc7a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.23.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.23 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_23_H
+#define RGXCONFIG_KM_35_V_1632_23_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 23
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (512U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.34.h
new file mode 100644
index 000000000000..18e0cf4ba89b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.34.h
@@ -0,0 +1,153 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.34 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_34_H
+#define RGXCONFIG_KM_35_V_1632_34_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 34
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.35.h
new file mode 100644
index 000000000000..d359272a63f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.35.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.35 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_35_H
+#define RGXCONFIG_KM_35_V_1632_35_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 35
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.101.h
new file mode 100644
index 000000000000..ab5fde00d4d5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.101.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.101 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_101_H
+#define RGXCONFIG_KM_35_V_408_101_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 101
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.23.h
new file mode 100644
index 000000000000..718e12c8ed69
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.23.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.23 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_23_H
+#define RGXCONFIG_KM_35_V_408_23_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 23
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.33.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.33.h
new file mode 100644
index 000000000000..df85d8147f21
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.33.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.33 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_33_H
+#define RGXCONFIG_KM_35_V_408_33_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 33
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_33_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.34.h
new file mode 100644
index 000000000000..8fb939b2a65b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.34.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.34 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_34_H
+#define RGXCONFIG_KM_35_V_408_34_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 34
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_38.V.2448.402.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_38.V.2448.402.h
new file mode 100644
index 000000000000..4d4d7c7f9c1d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_38.V.2448.402.h
@@ -0,0 +1,153 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 38.V.2448.402 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_38_V_2448_402_H
+#define RGXCONFIG_KM_38_V_2448_402_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 38
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 402
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_38_V_2448_402_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1041.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1041.h
new file mode 100644
index 000000000000..3732194dda31
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1041.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.1041 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_1041_H
+#define RGXCONFIG_KM_70_V_2448_1041_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1041
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_1041_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1042.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1042.h
new file mode 100644
index 000000000000..3c8dbf11a303
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1042.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.1042 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_1042_H
+#define RGXCONFIG_KM_70_V_2448_1042_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1042
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_1042_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1360.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1360.h
new file mode 100644
index 000000000000..ec915beeaaf1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1360.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.1360 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_1360_H
+#define RGXCONFIG_KM_70_V_2448_1360_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1360
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_1360_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.418.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.418.h
new file mode 100644
index 000000000000..045e17306351
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.418.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.418 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_418_H
+#define RGXCONFIG_KM_70_V_2448_418_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 418
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (2U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_418_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1210.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1210.h
new file mode 100644
index 000000000000..bc75857064f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1210.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.2448.1210 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_2448_1210_H
+#define RGXCONFIG_KM_71_V_2448_1210_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1210
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_2448_1210_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1211.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1211.h
new file mode 100644
index 000000000000..67b2fa73b0a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1211.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.2448.1211 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_2448_1211_H
+#define RGXCONFIG_KM_71_V_2448_1211_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1211
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_2448_1211_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1212.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1212.h
new file mode 100644
index 000000000000..5d896d60fca8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1212.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.2448.1212 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_2448_1212_H
+#define RGXCONFIG_KM_71_V_2448_1212_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1212
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_2448_1212_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2080.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2080.h
new file mode 100644
index 000000000000..9b6265e018d8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2080.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2080 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2080_H
+#define RGXCONFIG_KM_71_V_3672_2080_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2080
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2080_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2081.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2081.h
new file mode 100644
index 000000000000..28ca12ce156f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2081.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2081 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2081_H
+#define RGXCONFIG_KM_71_V_3672_2081_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2081
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2081_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2082.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2082.h
new file mode 100644
index 000000000000..f7c2d401daf1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2082.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2082 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2082_H
+#define RGXCONFIG_KM_71_V_3672_2082_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2082
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2082_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2083.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2083.h
new file mode 100644
index 000000000000..2fc12a05bcad
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2083.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2083 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2083_H
+#define RGXCONFIG_KM_71_V_3672_2083_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2083
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2083_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2084.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2084.h
new file mode 100644
index 000000000000..7ecc97a74886
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2084.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2084 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2084_H
+#define RGXCONFIG_KM_71_V_3672_2084_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2084
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2084_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.5.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.5.254.2.h
new file mode 100644
index 000000000000..e0b7dfc02471
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.5.254.2.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 27.5.254.2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_27_5_254_2_H
+#define RGXCORE_KM_27_5_254_2_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5262350 */
+
+/******************************************************************************
+ * BVNC = 27.5.254.2
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 27
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 254
+#define RGX_BVNC_KM_C 2
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71422
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+
+
+
+#endif /* RGXCORE_KM_27_5_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.7.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.7.254.2.h
new file mode 100644
index 000000000000..38624611a499
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.7.254.2.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 27.7.254.2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_27_7_254_2_H
+#define RGXCORE_KM_27_7_254_2_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5941981 */
+
+/******************************************************************************
+ * BVNC = 27.7.254.2
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 27
+#define RGX_BVNC_KM_V 7
+#define RGX_BVNC_KM_N 254
+#define RGX_BVNC_KM_C 2
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+
+
+
+#endif /* RGXCORE_KM_27_7_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.8.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.8.254.2.h
new file mode 100644
index 000000000000..6719e1c5e96c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.8.254.2.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 27.8.254.2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_27_8_254_2_H
+#define RGXCORE_KM_27_8_254_2_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5975910 */
+
+/******************************************************************************
+ * BVNC = 27.8.254.2
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 27
+#define RGX_BVNC_KM_V 8
+#define RGX_BVNC_KM_N 254
+#define RGX_BVNC_KM_C 2
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+
+
+
+#endif /* RGXCORE_KM_27_8_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.408.101.h
new file mode 100644
index 000000000000..f05575d4ef65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.408.101.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 30.3.408.101
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_30_3_408_101_H
+#define RGXCORE_KM_30_3_408_101_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5653222 */
+
+/******************************************************************************
+ * BVNC = 30.3.408.101
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 30
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 101
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_30_3_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.816.20.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.816.20.h
new file mode 100644
index 000000000000..7319e86fb7bf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.816.20.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 30.3.816.20
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_30_3_816_20_H
+#define RGXCORE_KM_30_3_816_20_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5690709 */
+
+/******************************************************************************
+ * BVNC = 30.3.816.20
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 30
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 816
+#define RGX_BVNC_KM_C 20
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_30_3_816_20_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.5.1632.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.5.1632.1.h
new file mode 100644
index 000000000000..0fbbef658811
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.5.1632.1.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 30.5.1632.1
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_30_5_1632_1_H
+#define RGXCORE_KM_30_5_1632_1_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5621518 */
+
+/******************************************************************************
+ * BVNC = 30.5.1632.1
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 30
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 1
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_30_5_1632_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.21.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.21.h
new file mode 100644
index 000000000000..f2e489cfbab7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.21.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.21
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_21_H
+#define RGXCORE_KM_35_2_1632_21_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5744712 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.21
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 21
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71157
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_21_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.23.h
new file mode 100644
index 000000000000..9c19267ac08a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.23.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_23_H
+#define RGXCORE_KM_35_2_1632_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5928760 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71157
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.34.h
new file mode 100644
index 000000000000..a4c3741fb3e4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.34.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.34
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_34_H
+#define RGXCORE_KM_35_2_1632_34_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6097610 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.34
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 34
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.35.h
new file mode 100644
index 000000000000..9a959fe90e65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.35.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.35
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_35_H
+#define RGXCORE_KM_35_2_1632_35_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6327132 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.35
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 35
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.33.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.33.h
new file mode 100644
index 000000000000..6e97c78a9226
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.33.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.408.33
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_408_33_H
+#define RGXCORE_KM_35_2_408_33_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @6220991 */
+
+/******************************************************************************
+ * BVNC = 35.2.408.33
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 33
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_408_33_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.34.h
new file mode 100644
index 000000000000..ecd899c933b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.34.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.408.34
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_408_34_H
+#define RGXCORE_KM_35_2_408_34_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6252022 */
+
+/******************************************************************************
+ * BVNC = 35.2.408.34
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 34
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_408_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.1632.23.h
new file mode 100644
index 000000000000..9726e2ca1fb4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.1632.23.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.1632.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_1632_23_H
+#define RGXCORE_KM_35_3_1632_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5906056 */
+
+/******************************************************************************
+ * BVNC = 35.3.1632.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.101.h
new file mode 100644
index 000000000000..074fa3d600f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.101.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.408.101
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_408_101_H
+#define RGXCORE_KM_35_3_408_101_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5653222 */
+
+/******************************************************************************
+ * BVNC = 35.3.408.101
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 101
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.33.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.33.h
new file mode 100644
index 000000000000..a17f409b5644
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.33.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.408.33
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_408_33_H
+#define RGXCORE_KM_35_3_408_33_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6249306 */
+
+/******************************************************************************
+ * BVNC = 35.3.408.33
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 33
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_408_33_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.34.h
new file mode 100644
index 000000000000..c44fe2243220
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.34.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.408.34
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_408_34_H
+#define RGXCORE_KM_35_3_408_34_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6285885 */
+
+/******************************************************************************
+ * BVNC = 35.3.408.34
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 34
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_408_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.4.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.4.1632.23.h
new file mode 100644
index 000000000000..02899da3d8fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.4.1632.23.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.4.1632.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_4_1632_23_H
+#define RGXCORE_KM_35_4_1632_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5952322 */
+
+/******************************************************************************
+ * BVNC = 35.4.1632.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 4
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_4_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.5.408.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.5.408.23.h
new file mode 100644
index 000000000000..a5048e951f47
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.5.408.23.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.5.408.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_5_408_23_H
+#define RGXCORE_KM_35_5_408_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6080671 */
+
+/******************************************************************************
+ * BVNC = 35.5.408.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_5_408_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.4.2448.402.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.4.2448.402.h
new file mode 100644
index 000000000000..7d9e4e4ac323
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.4.2448.402.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 38.4.2448.402
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_38_4_2448_402_H
+#define RGXCORE_KM_38_4_2448_402_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5976185 */
+
+/******************************************************************************
+ * BVNC = 38.4.2448.402
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 38
+#define RGX_BVNC_KM_V 4
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 402
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72143
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_38_4_2448_402_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.6.2448.402.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.6.2448.402.h
new file mode 100644
index 000000000000..7bf85accef8a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.6.2448.402.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 38.6.2448.402
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_38_6_2448_402_H
+#define RGXCORE_KM_38_6_2448_402_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6068975 */
+
+/******************************************************************************
+ * BVNC = 38.6.2448.402
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 38
+#define RGX_BVNC_KM_V 6
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 402
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72143
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_38_6_2448_402_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1041.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1041.h
new file mode 100644
index 000000000000..c44b6d4d1e5f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1041.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.1041
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_1041_H
+#define RGXCORE_KM_70_2_2448_1041_H
+
+/* Automatically generated file (06/05/2022 11:58:47): Do not edit manually */
+/* CS: @6243633 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.1041
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1041
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_1041_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1042.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1042.h
new file mode 100644
index 000000000000..de04ac9fde5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1042.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.1042
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_1042_H
+#define RGXCORE_KM_70_2_2448_1042_H
+
+/* Automatically generated file (05/09/2023 09:10:40): Do not edit manually */
+/* CS: @6236243 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.1042
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1042
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_1042_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1360.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1360.h
new file mode 100644
index 000000000000..95c2bbad0095
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1360.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.1360
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_1360_H
+#define RGXCORE_KM_70_2_2448_1360_H
+
+/* Automatically generated file (08/06/2022 13:43:04): Do not edit manually */
+/* CS: @6208659 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.1360
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1360
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_1360_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.418.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.418.h
new file mode 100644
index 000000000000..80335f71e1aa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.418.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.418
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_418_H
+#define RGXCORE_KM_70_2_2448_418_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @6208659 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.418
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 418
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_418_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1042.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1042.h
new file mode 100644
index 000000000000..490f350681f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1042.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.3.2448.1042
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_3_2448_1042_H
+#define RGXCORE_KM_70_3_2448_1042_H
+
+/* Automatically generated file (14/09/2023 09:10:35): Do not edit manually */
+/* CS: @6458228 */
+
+/******************************************************************************
+ * BVNC = 70.3.2448.1042
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1042
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_3_2448_1042_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1360.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1360.h
new file mode 100644
index 000000000000..c07d2e70b8ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1360.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.3.2448.1360
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_3_2448_1360_H
+#define RGXCORE_KM_70_3_2448_1360_H
+
+/* Automatically generated file (28/06/2022 10:24:53): Do not edit manually */
+/* CS: @6266740 */
+
+/******************************************************************************
+ * BVNC = 70.3.2448.1360
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1360
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_3_2448_1360_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1210.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1210.h
new file mode 100644
index 000000000000..e70690a29eb8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1210.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.2448.1210
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_2448_1210_H
+#define RGXCORE_KM_71_2_2448_1210_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6317660 */
+
+/******************************************************************************
+ * BVNC = 71.2.2448.1210
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1210
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_2448_1210_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1211.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1211.h
new file mode 100644
index 000000000000..4bd717024ef8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1211.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.2448.1211
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_2448_1211_H
+#define RGXCORE_KM_71_2_2448_1211_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6347253 */
+
+/******************************************************************************
+ * BVNC = 71.2.2448.1211
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1211
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_2448_1211_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1212.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1212.h
new file mode 100644
index 000000000000..4bf4a7bb6fda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1212.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.2448.1212
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_2448_1212_H
+#define RGXCORE_KM_71_2_2448_1212_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6450859 */
+
+/******************************************************************************
+ * BVNC = 71.2.2448.1212
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1212
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_2448_1212_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2080.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2080.h
new file mode 100644
index 000000000000..1a17786ee245
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2080.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2080
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2080_H
+#define RGXCORE_KM_71_2_3672_2080_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6375375 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2080
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2080
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2080_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2081.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2081.h
new file mode 100644
index 000000000000..3b0113c6a5c5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2081.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2081
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2081_H
+#define RGXCORE_KM_71_2_3672_2081_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6422248 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2081
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2081
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2081_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2082.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2082.h
new file mode 100644
index 000000000000..651bad6b7c97
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2082.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2082
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2082_H
+#define RGXCORE_KM_71_2_3672_2082_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6449305 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2082
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2082
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2082_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2083.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2083.h
new file mode 100644
index 000000000000..e98d838c03f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2083.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2083
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2083_H
+#define RGXCORE_KM_71_2_3672_2083_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6464565 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2083
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2083
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2083_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2083.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2083.h
new file mode 100644
index 000000000000..bdc923c78d0e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2083.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.3.3672.2083
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_3_3672_2083_H
+#define RGXCORE_KM_71_3_3672_2083_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6473124 */
+
+/******************************************************************************
+ * BVNC = 71.3.3672.2083
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2083
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_3_3672_2083_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2084.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2084.h
new file mode 100644
index 000000000000..f287f89ce147
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2084.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.3.3672.2084
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_3_3672_2084_H
+#define RGXCORE_KM_71_3_3672_2084_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6495448 */
+
+/******************************************************************************
+ * BVNC = 71.3.3672.2084
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2084
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_3_3672_2084_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_defs_km.h
new file mode 100644
index 000000000000..769165e76a2e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_defs_km.h
@@ -0,0 +1,397 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_defs_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_DEFS_KM_H
+#define RGX_BVNC_DEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(RGX_BVNC_DEFS_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_defs_um.h"
+#endif
+
+#define BVNC_FIELD_WIDTH (16U)
+
+#define PVR_ARCH_NAME "volcanic"
+
+
+/******************************************************************************
+ * Mask and bit-position macros for features without values
+ *****************************************************************************/
+
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE_POS                     (0U)
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE_BIT_MASK                (IMG_UINT64_C(0x0000000000000001))
+
+#define RGX_FEATURE_AXI_ACE_POS                                         (1U)
+#define RGX_FEATURE_AXI_ACE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000002))
+
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE_POS                       (2U)
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE_BIT_MASK                  (IMG_UINT64_C(0x0000000000000004))
+
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_POS                (3U)
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_BIT_MASK           (IMG_UINT64_C(0x0000000000000008))
+
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE_POS                      (4U)
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE_BIT_MASK                 (IMG_UINT64_C(0x0000000000000010))
+
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_POS                  (5U)
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_BIT_MASK             (IMG_UINT64_C(0x0000000000000020))
+
+#define RGX_FEATURE_CLUSTER_GROUPING_POS                                (6U)
+#define RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK                           (IMG_UINT64_C(0x0000000000000040))
+
+#define RGX_FEATURE_COMPUTE_POS                                         (7U)
+#define RGX_FEATURE_COMPUTE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000080))
+
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_POS                          (8U)
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_BIT_MASK                     (IMG_UINT64_C(0x0000000000000100))
+
+#define RGX_FEATURE_COMPUTE_ONLY_POS                                    (9U)
+#define RGX_FEATURE_COMPUTE_ONLY_BIT_MASK                               (IMG_UINT64_C(0x0000000000000200))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_POS                                 (10U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_BIT_MASK                            (IMG_UINT64_C(0x0000000000000400))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS_POS                   (11U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS_BIT_MASK              (IMG_UINT64_C(0x0000000000000800))
+
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS_POS                  (12U)
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK             (IMG_UINT64_C(0x0000000000001000))
+
+#define RGX_FEATURE_COREID_PER_OS_POS                                   (13U)
+#define RGX_FEATURE_COREID_PER_OS_BIT_MASK                              (IMG_UINT64_C(0x0000000000002000))
+
+#define RGX_FEATURE_CR_PARITY_POS                                       (14U)
+#define RGX_FEATURE_CR_PARITY_BIT_MASK                                  (IMG_UINT64_C(0x0000000000004000))
+
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE_POS                       (15U)
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE_BIT_MASK                  (IMG_UINT64_C(0x0000000000008000))
+
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7_POS                            (16U)
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7_BIT_MASK                       (IMG_UINT64_C(0x0000000000010000))
+
+#define RGX_FEATURE_ERYX_TOP_INFRASTRUCTURE_POS                         (17U)
+#define RGX_FEATURE_ERYX_TOP_INFRASTRUCTURE_BIT_MASK                    (IMG_UINT64_C(0x0000000000020000))
+
+#define RGX_FEATURE_FASTRENDER_DM_POS                                   (18U)
+#define RGX_FEATURE_FASTRENDER_DM_BIT_MASK                              (IMG_UINT64_C(0x0000000000040000))
+
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS_POS                     (19U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK                (IMG_UINT64_C(0x0000000000080000))
+
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER_POS                            (20U)
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER_BIT_MASK                       (IMG_UINT64_C(0x0000000000100000))
+
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS_POS                     (21U)
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK                (IMG_UINT64_C(0x0000000000200000))
+
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_POS                           (22U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_BIT_MASK                      (IMG_UINT64_C(0x0000000000400000))
+
+#define RGX_FEATURE_GPU_VIRTUALISATION_POS                              (23U)
+#define RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK                         (IMG_UINT64_C(0x0000000000800000))
+
+#define RGX_FEATURE_GS_RTA_SUPPORT_POS                                  (24U)
+#define RGX_FEATURE_GS_RTA_SUPPORT_BIT_MASK                             (IMG_UINT64_C(0x0000000001000000))
+
+#define RGX_FEATURE_IDLE_CYCLE_STEALING_POS                             (25U)
+#define RGX_FEATURE_IDLE_CYCLE_STEALING_BIT_MASK                        (IMG_UINT64_C(0x0000000002000000))
+
+#define RGX_FEATURE_META_DMA_POS                                        (26U)
+#define RGX_FEATURE_META_DMA_BIT_MASK                                   (IMG_UINT64_C(0x0000000004000000))
+
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES_POS                 (27U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES_BIT_MASK            (IMG_UINT64_C(0x0000000008000000))
+
+#define RGX_FEATURE_MH_PARITY_POS                                       (28U)
+#define RGX_FEATURE_MH_PARITY_BIT_MASK                                  (IMG_UINT64_C(0x0000000010000000))
+
+#define RGX_FEATURE_PBE_CHECKSUM_2D_POS                                 (29U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D_BIT_MASK                            (IMG_UINT64_C(0x0000000020000000))
+
+#define RGX_FEATURE_PBVNC_COREID_REG_POS                                (30U)
+#define RGX_FEATURE_PBVNC_COREID_REG_BIT_MASK                           (IMG_UINT64_C(0x0000000040000000))
+
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE_POS           (31U)
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE_BIT_MASK      (IMG_UINT64_C(0x0000000080000000))
+
+#define RGX_FEATURE_PDS_TEMPSIZE8_POS                                   (32U)
+#define RGX_FEATURE_PDS_TEMPSIZE8_BIT_MASK                              (IMG_UINT64_C(0x0000000100000000))
+
+#define RGX_FEATURE_PERFBUS_POS                                         (33U)
+#define RGX_FEATURE_PERFBUS_BIT_MASK                                    (IMG_UINT64_C(0x0000000200000000))
+
+#define RGX_FEATURE_PERF_COUNTER_BATCH_POS                              (34U)
+#define RGX_FEATURE_PERF_COUNTER_BATCH_BIT_MASK                         (IMG_UINT64_C(0x0000000400000000))
+
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES_POS                  (35U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES_BIT_MASK             (IMG_UINT64_C(0x0000000800000000))
+
+#define RGX_FEATURE_RISCV_DUAL_LOCKSTEP_POS                             (36U)
+#define RGX_FEATURE_RISCV_DUAL_LOCKSTEP_BIT_MASK                        (IMG_UINT64_C(0x0000001000000000))
+
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_POS                              (37U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_BIT_MASK                         (IMG_UINT64_C(0x0000002000000000))
+
+#define RGX_FEATURE_RT_RAC_PER_SPU_POS                                  (38U)
+#define RGX_FEATURE_RT_RAC_PER_SPU_BIT_MASK                             (IMG_UINT64_C(0x0000004000000000))
+
+#define RGX_FEATURE_S7_CACHE_HIERARCHY_POS                              (39U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY_BIT_MASK                         (IMG_UINT64_C(0x0000008000000000))
+
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE_POS                           (40U)
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE_BIT_MASK                      (IMG_UINT64_C(0x0000010000000000))
+
+#define RGX_FEATURE_SAFETY_IRQ_POS                                      (41U)
+#define RGX_FEATURE_SAFETY_IRQ_BIT_MASK                                 (IMG_UINT64_C(0x0000020000000000))
+
+#define RGX_FEATURE_SAFETY_SELF_TEST_POS                                (42U)
+#define RGX_FEATURE_SAFETY_SELF_TEST_BIT_MASK                           (IMG_UINT64_C(0x0000040000000000))
+
+#define RGX_FEATURE_SCALABLE_VDM_GPP_POS                                (43U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP_BIT_MASK                           (IMG_UINT64_C(0x0000080000000000))
+
+#define RGX_FEATURE_SIGNAL_SNOOPING_POS                                 (44U)
+#define RGX_FEATURE_SIGNAL_SNOOPING_BIT_MASK                            (IMG_UINT64_C(0x0000100000000000))
+
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT_POS                             (45U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT_BIT_MASK                        (IMG_UINT64_C(0x0000200000000000))
+
+#define RGX_FEATURE_SLC_VIVT_POS                                        (46U)
+#define RGX_FEATURE_SLC_VIVT_BIT_MASK                                   (IMG_UINT64_C(0x0000400000000000))
+
+#define RGX_FEATURE_SOC_TIMER_POS                                       (47U)
+#define RGX_FEATURE_SOC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0000800000000000))
+
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING_POS                           (48U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING_BIT_MASK                      (IMG_UINT64_C(0x0001000000000000))
+
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_POS                            (49U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_BIT_MASK                       (IMG_UINT64_C(0x0002000000000000))
+
+#define RGX_FEATURE_SYS_PARITY_POS                                      (50U)
+#define RGX_FEATURE_SYS_PARITY_BIT_MASK                                 (IMG_UINT64_C(0x0004000000000000))
+
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_POS                                (51U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_BIT_MASK                           (IMG_UINT64_C(0x0008000000000000))
+
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS_POS                      (52U)
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK                 (IMG_UINT64_C(0x0010000000000000))
+
+#define RGX_FEATURE_TESSELLATION_POS                                    (53U)
+#define RGX_FEATURE_TESSELLATION_BIT_MASK                               (IMG_UINT64_C(0x0020000000000000))
+
+#define RGX_FEATURE_TILE_REGION_PROTECTION_POS                          (54U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION_BIT_MASK                     (IMG_UINT64_C(0x0040000000000000))
+
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_POS             (55U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_BIT_MASK        (IMG_UINT64_C(0x0080000000000000))
+
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_POS                         (56U)
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_BIT_MASK                    (IMG_UINT64_C(0x0100000000000000))
+
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE_POS           (57U)
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE_BIT_MASK      (IMG_UINT64_C(0x0200000000000000))
+
+#define RGX_FEATURE_USC_TIMER_POS                                       (58U)
+#define RGX_FEATURE_USC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0400000000000000))
+
+#define RGX_FEATURE_VDM_DRAWINDIRECT_POS                                (59U)
+#define RGX_FEATURE_VDM_DRAWINDIRECT_BIT_MASK                           (IMG_UINT64_C(0x0800000000000000))
+
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS_POS                            (60U)
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS_BIT_MASK                       (IMG_UINT64_C(0x1000000000000000))
+
+#define RGX_FEATURE_WATCHDOG_TIMER_POS                                  (61U)
+#define RGX_FEATURE_WATCHDOG_TIMER_BIT_MASK                             (IMG_UINT64_C(0x2000000000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_POS                            (62U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_BIT_MASK                       (IMG_UINT64_C(0x4000000000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP_POS                        (63U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP_BIT_MASK                   (IMG_UINT64_C(0x8000000000000000))
+
+#define RGX_FEATURE_ZLS_CHECKSUM_POS                                    (64U)
+#define RGX_FEATURE_ZLS_CHECKSUM_BIT_MASK                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/******************************************************************************
+ * Defines for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (3U)
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_MAX_VALUE_IDX               (3U)
+#define RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX                              (3U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX                  (4U)
+#define RGX_FEATURE_FBCDC_MAX_VALUE_IDX                                 (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX                       (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX                    (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX           (3U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX             (3U)
+#define RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX                 (5U)
+#define RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX                           (2U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU_MAX_VALUE_IDX                       (4U)
+#define RGX_FEATURE_META_MAX_VALUE_IDX                                  (2U)
+#define RGX_FEATURE_META_COREMEM_BANKS_MAX_VALUE_IDX                    (2U)
+#define RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX                     (2U)
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT_MAX_VALUE_IDX                (3U)
+#define RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX                          (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX                     (9U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX                       (4U)
+#define RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX                            (5U)
+#define RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX                             (2U)
+#define RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX                               (5U)
+#define RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX                        (2U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX         (4U)
+#define RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX                  (7U)
+#define RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX_MAX_VALUE_IDX                  (3U)
+#define RGX_FEATURE_SCALABLE_TE_ARCH_MAX_VALUE_IDX                      (4U)
+#define RGX_FEATURE_SCALABLE_VCE_MAX_VALUE_IDX                          (5U)
+#define RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX                             (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX              (2U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX                 (7U)
+#define RGX_FEATURE_SPU0_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX                          (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (4U)
+#define RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX                           (2U)
+#define RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX                           (2U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX            (2U)
+
+/******************************************************************************
+ * Features with values indexes
+ *****************************************************************************/
+
+typedef enum _RGX_FEATURE_WITH_VALUE_INDEX_ {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_IDX,
+	RGX_FEATURE_ECC_RAMS_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_IDX,
+	RGX_FEATURE_FBCDC_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_IDX,
+	RGX_FEATURE_HOST_SECURITY_VERSION_IDX,
+	RGX_FEATURE_LAYOUT_MARS_IDX,
+	RGX_FEATURE_MAX_TPU_PER_SPU_IDX,
+	RGX_FEATURE_META_IDX,
+	RGX_FEATURE_META_COREMEM_BANKS_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_IDX,
+	RGX_FEATURE_META_DMA_CHANNEL_COUNT_IDX,
+	RGX_FEATURE_MMU_VERSION_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX,
+	RGX_FEATURE_NUM_ISP_PER_SPU_IDX,
+	RGX_FEATURE_NUM_MEMBUS_IDX,
+	RGX_FEATURE_NUM_OSIDS_IDX,
+	RGX_FEATURE_NUM_SPU_IDX,
+	RGX_FEATURE_PBE_PER_SPU_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_IDX,
+	RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_IDX,
+	RGX_FEATURE_POWER_ISLAND_VERSION_IDX,
+	RGX_FEATURE_RAY_TRACING_ARCH_IDX,
+	RGX_FEATURE_RENDER_TARGET_XY_MAX_IDX,
+	RGX_FEATURE_SCALABLE_TE_ARCH_IDX,
+	RGX_FEATURE_SCALABLE_VCE_IDX,
+	RGX_FEATURE_SLC_BANKS_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_IDX,
+	RGX_FEATURE_SPU0_RAC_PRESENT_IDX,
+	RGX_FEATURE_SPU1_RAC_PRESENT_IDX,
+	RGX_FEATURE_SPU2_RAC_PRESENT_IDX,
+	RGX_FEATURE_SPU3_RAC_PRESENT_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_IDX,
+	RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_TILE_SIZE_X_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_IDX,
+	RGX_FEATURE_WITH_VALUES_MAX_IDX,
+} RGX_FEATURE_WITH_VALUE_INDEX;
+
+
+/******************************************************************************
+ * Mask and bit-position macros for ERNs and BRNs
+ *****************************************************************************/
+
+#define HW_ERN_65104_POS                                                (0U)
+#define HW_ERN_65104_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000001))
+
+#define HW_ERN_69700_POS                                                (1U)
+#define HW_ERN_69700_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000002))
+
+#define FIX_HW_BRN_71157_POS                                            (2U)
+#define FIX_HW_BRN_71157_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000004))
+
+#define FIX_HW_BRN_71422_POS                                            (3U)
+#define FIX_HW_BRN_71422_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000008))
+
+#define FIX_HW_BRN_71960_POS                                            (4U)
+#define FIX_HW_BRN_71960_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000010))
+
+#define FIX_HW_BRN_72143_POS                                            (5U)
+#define FIX_HW_BRN_72143_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000020))
+
+#define FIX_HW_BRN_72144_POS                                            (6U)
+#define FIX_HW_BRN_72144_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000040))
+
+#define FIX_HW_BRN_74812_POS                                            (7U)
+#define FIX_HW_BRN_74812_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000080))
+
+/* Macro used for padding the unavailable values for features with values */
+#define RGX_FEATURE_VALUE_INVALID	(0xFFFFFFFEU)
+
+/* Macro used for marking a feature with value as disabled for a specific bvnc */
+#define RGX_FEATURE_VALUE_DISABLED	(0xFFFFFFFFU)
+
+#endif /* RGX_BVNC_DEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_table_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_table_km.h
new file mode 100644
index 000000000000..38ca0805c203
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_table_km.h
@@ -0,0 +1,530 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_table_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_TABLE_KM_H
+#define RGX_BVNC_TABLE_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+
+#ifndef RGXBVNC_C
+#error "This file should only be included from rgxbvnc.c"
+#endif
+
+#if defined(RGX_BVNC_TABLE_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_table_um.h"
+#endif
+
+
+/******************************************************************************
+ * Arrays for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 5, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_values[RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_ECC_RAMS_values[RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values[RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_values[RGX_FEATURE_FBCDC_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 5, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ALGORITHM_values[RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 5, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values[RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 512, 2048, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_HOST_SECURITY_VERSION_values[RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_LAYOUT_MARS_values[RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_MAX_TPU_PER_SPU_values[RGX_FEATURE_MAX_TPU_PER_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_values[RGX_FEATURE_META_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, MTP219, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_BANKS_values[RGX_FEATURE_META_COREMEM_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_SIZE_values[RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 96, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_DMA_CHANNEL_COUNT_values[RGX_FEATURE_META_DMA_CHANNEL_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_MMU_VERSION_values[RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_CLUSTERS_values[RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 6, 8, 9, 12, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values[RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 8, 12, 16, 24, 32, 36, 48, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_PER_SPU_values[RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_MEMBUS_values[RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_OSIDS_values[RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_SPU_values[RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PBE_PER_SPU_values[RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values[RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_values[RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_POWER_ISLAND_VERSION_values[RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 5, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_RAY_TRACING_ARCH_values[RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_RENDER_TARGET_XY_MAX_values[RGX_FEATURE_RENDER_TARGET_XY_MAX_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16384, 32768, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SCALABLE_TE_ARCH_values[RGX_FEATURE_SCALABLE_TE_ARCH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SCALABLE_VCE_values[RGX_FEATURE_SCALABLE_VCE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_BANKS_values[RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, 6, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values[RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1024, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values[RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 128, 256, 512, 1024, 1536, 2048, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU0_RAC_PRESENT_values[RGX_FEATURE_SPU0_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU1_RAC_PRESENT_values[RGX_FEATURE_SPU1_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU2_RAC_PRESENT_values[RGX_FEATURE_SPU2_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU3_RAC_PRESENT_values[RGX_FEATURE_SPU3_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TB_GPU_COUNT_values[RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_X_values[RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_Y_values[RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values[RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+
+/******************************************************************************
+ * Table contains pointers to each feature value array for features that have
+ * values.
+ * Indexed using enum RGX_FEATURE_WITH_VALUE_INDEX from rgx_bvnc_defs_km.h
+ *****************************************************************************/
+
+static const void * const gaFeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX] = {
+	aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_values,
+	aui16_RGX_FEATURE_ECC_RAMS_values,
+	aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values,
+	aui16_RGX_FEATURE_FBCDC_values,
+	aui16_RGX_FEATURE_FBCDC_ALGORITHM_values,
+	aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_HOST_SECURITY_VERSION_values,
+	aui16_RGX_FEATURE_LAYOUT_MARS_values,
+	aui16_RGX_FEATURE_MAX_TPU_PER_SPU_values,
+	aui16_RGX_FEATURE_META_values,
+	aui16_RGX_FEATURE_META_COREMEM_BANKS_values,
+	aui16_RGX_FEATURE_META_COREMEM_SIZE_values,
+	aui16_RGX_FEATURE_META_DMA_CHANNEL_COUNT_values,
+	aui16_RGX_FEATURE_MMU_VERSION_values,
+	aui16_RGX_FEATURE_NUM_CLUSTERS_values,
+	aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values,
+	aui16_RGX_FEATURE_NUM_ISP_PER_SPU_values,
+	aui16_RGX_FEATURE_NUM_MEMBUS_values,
+	aui16_RGX_FEATURE_NUM_OSIDS_values,
+	aui16_RGX_FEATURE_NUM_SPU_values,
+	aui16_RGX_FEATURE_PBE_PER_SPU_values,
+	aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values,
+	aui16_RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_values,
+	aui16_RGX_FEATURE_POWER_ISLAND_VERSION_values,
+	aui16_RGX_FEATURE_RAY_TRACING_ARCH_values,
+	aui16_RGX_FEATURE_RENDER_TARGET_XY_MAX_values,
+	aui16_RGX_FEATURE_SCALABLE_TE_ARCH_values,
+	aui16_RGX_FEATURE_SCALABLE_VCE_values,
+	aui16_RGX_FEATURE_SLC_BANKS_values,
+	aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values,
+	aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values,
+	aui16_RGX_FEATURE_SPU0_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_SPU1_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_SPU2_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_SPU3_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_TB_GPU_COUNT_values,
+	aui16_RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_TILE_SIZE_X_values,
+	aui16_RGX_FEATURE_TILE_SIZE_Y_values,
+	aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values,
+};
+
+
+/******************************************************************************
+ * Array containing the lengths of the arrays containing the values.
+ * Used for indexing the aui16_<FEATURE>_values defined upwards
+ *****************************************************************************/
+
+
+static const IMG_UINT16 gaFeaturesValuesMaxIndexes[] = {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_MAX_VALUE_IDX,
+	RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX,
+	RGX_FEATURE_MAX_TPU_PER_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_META_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX,
+	RGX_FEATURE_META_DMA_CHANNEL_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX,
+	RGX_FEATURE_RENDER_TARGET_XY_MAX_MAX_VALUE_IDX,
+	RGX_FEATURE_SCALABLE_TE_ARCH_MAX_VALUE_IDX,
+	RGX_FEATURE_SCALABLE_VCE_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU0_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU1_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU2_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU3_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX,
+};
+
+#define RGX_FEATURE_VALUE_TYPE_UINT16 (0x0000U)
+#define RGX_FEATURE_VALUE_TYPE_UINT32 (0x8000U)
+#define RGX_FEATURE_TYPE_BIT_SHIFT 14
+
+/******************************************************************************
+ * Bit-positions for features with values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16FeaturesWithValuesBitPositions[] = {
+	(0U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_POS */
+	(2U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_POS */
+	(4U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_ECC_RAMS_POS */
+	(6U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FAULT_DECODE_VERSION_POS */
+	(9U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_POS */
+	(11U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ALGORITHM_POS */
+	(13U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ARCHITECTURE_POS */
+	(15U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_POS */
+	(17U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_POS */
+	(19U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_HOST_SECURITY_VERSION_POS */
+	(22U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_LAYOUT_MARS_POS */
+	(24U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_MAX_TPU_PER_SPU_POS */
+	(27U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_POS */
+	(29U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_BANKS_POS */
+	(31U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_SIZE_POS */
+	(33U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_DMA_CHANNEL_COUNT_POS */
+	(35U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_MMU_VERSION_POS */
+	(37U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_CLUSTERS_POS */
+	(41U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_IPP_PIPES_POS */
+	(45U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_PER_SPU_POS */
+	(48U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_MEMBUS_POS */
+	(51U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_OSIDS_POS */
+	(53U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_SPU_POS */
+	(56U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PBE_PER_SPU_POS */
+	(58U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PHYS_BUS_WIDTH_POS */
+	(60U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_POS */
+	(64U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_POWER_ISLAND_VERSION_POS */
+	(67U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_RAY_TRACING_ARCH_POS */
+	(69U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_RENDER_TARGET_XY_MAX_POS */
+	(71U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SCALABLE_TE_ARCH_POS */
+	(74U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SCALABLE_VCE_POS */
+	(77U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_BANKS_POS */
+	(80U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_POS */
+	(82U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_POS */
+	(85U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU0_RAC_PRESENT_POS */
+	(87U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU1_RAC_PRESENT_POS */
+	(89U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU2_RAC_PRESENT_POS */
+	(91U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU3_RAC_PRESENT_POS */
+	(93U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TB_GPU_COUNT_POS */
+	(96U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_POS */
+	(99U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_X_POS */
+	(101U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_Y_POS */
+	(103U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_POS */
+};
+
+
+/******************************************************************************
+ * Bit-masks for features with values
+ *****************************************************************************/
+
+static const IMG_UINT64 aui64FeaturesWithValuesBitMasks[] = {
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_ECC_RAMS_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000001C0)), /* RGX_FEATURE_FAULT_DECODE_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000600)), /* RGX_FEATURE_FBCDC_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000001800)), /* RGX_FEATURE_FBCDC_ALGORITHM_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000006000)), /* RGX_FEATURE_FBCDC_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000018000)), /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000060000)), /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000380000)), /* RGX_FEATURE_HOST_SECURITY_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000C00000)), /* RGX_FEATURE_LAYOUT_MARS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000007000000)), /* RGX_FEATURE_MAX_TPU_PER_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0000000018000000)), /* RGX_FEATURE_META_BIT_MASK */
+	(IMG_UINT64_C(0x0000000060000000)), /* RGX_FEATURE_META_COREMEM_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000180000000)), /* RGX_FEATURE_META_COREMEM_SIZE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000600000000)), /* RGX_FEATURE_META_DMA_CHANNEL_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_MMU_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x000001E000000000)), /* RGX_FEATURE_NUM_CLUSTERS_BIT_MASK */
+	(IMG_UINT64_C(0x00001E0000000000)), /* RGX_FEATURE_NUM_ISP_IPP_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000E00000000000)), /* RGX_FEATURE_NUM_ISP_PER_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0007000000000000)), /* RGX_FEATURE_NUM_MEMBUS_BIT_MASK */
+	(IMG_UINT64_C(0x0018000000000000)), /* RGX_FEATURE_NUM_OSIDS_BIT_MASK */
+	(IMG_UINT64_C(0x00E0000000000000)), /* RGX_FEATURE_NUM_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0300000000000000)), /* RGX_FEATURE_PBE_PER_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0C00000000000000)), /* RGX_FEATURE_PHYS_BUS_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x7000000000000000)), /* RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000007)), /* RGX_FEATURE_POWER_ISLAND_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000018)), /* RGX_FEATURE_RAY_TRACING_ARCH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000060)), /* RGX_FEATURE_RENDER_TARGET_XY_MAX_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000380)), /* RGX_FEATURE_SCALABLE_TE_ARCH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000001C00)), /* RGX_FEATURE_SCALABLE_VCE_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000E000)), /* RGX_FEATURE_SLC_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000030000)), /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x00000000001C0000)), /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000600000)), /* RGX_FEATURE_SPU0_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000001800000)), /* RGX_FEATURE_SPU1_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000006000000)), /* RGX_FEATURE_SPU2_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000018000000)), /* RGX_FEATURE_SPU3_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x00000000E0000000)), /* RGX_FEATURE_TB_GPU_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000700000000)), /* RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_TILE_SIZE_X_BIT_MASK */
+	(IMG_UINT64_C(0x0000006000000000)), /* RGX_FEATURE_TILE_SIZE_Y_BIT_MASK */
+	(IMG_UINT64_C(0x0000018000000000)), /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_BIT_MASK */
+};
+
+
+/******************************************************************************
+ * Table mapping bitmasks for features and features with values
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaFeatures[][4]=
+{
+	{ IMG_UINT64_C(0x001b000000fe0002), IMG_UINT64_C(0x1daad98f65952dca), IMG_UINT64_C(0x1629424ca94aaa55), IMG_UINT64_C(0x000000a92aa524a9) },	/* 27.0.254.2 */
+	{ IMG_UINT64_C(0x001e000001980065), IMG_UINT64_C(0x1fbaf98fedbd3dcb), IMG_UINT64_C(0x15292234a94aaa59), IMG_UINT64_C(0x000000a92aa524a9) },	/* 30.0.408.101 */
+	{ IMG_UINT64_C(0x001e000003300014), IMG_UINT64_C(0x1fbaf98fedbd3dcb), IMG_UINT64_C(0x15294454aa4aaa59), IMG_UINT64_C(0x000000a92aa944a9) },	/* 30.0.816.20 */
+	{ IMG_UINT64_C(0x0023000001980017), IMG_UINT64_C(0x1fbaf98fedfd3dcf), IMG_UINT64_C(0x15292234a94d2a59), IMG_UINT64_C(0x000000aa6aa524ca) },	/* 35.0.408.23 */
+	{ IMG_UINT64_C(0x0023000001980021), IMG_UINT64_C(0xfffaf9afedfd3dcf), IMG_UINT64_C(0x15292232a14d2aa9), IMG_UINT64_C(0x000000aa2aa524ca) },	/* 35.0.408.33 */
+	{ IMG_UINT64_C(0x0023000001980022), IMG_UINT64_C(0xfffaf9afedfd3dcf), IMG_UINT64_C(0x15292232a14d2aa9), IMG_UINT64_C(0x000000aa4aa524ca) },	/* 35.0.408.34 */
+	{ IMG_UINT64_C(0x0023000001980065), IMG_UINT64_C(0x1fbaf98fedbd3dcb), IMG_UINT64_C(0x15292234a94aaa59), IMG_UINT64_C(0x000000a92aa524a9) },	/* 35.0.408.101 */
+	{ IMG_UINT64_C(0x0023000006600015), IMG_UINT64_C(0x1fbaf98fedfd3dcf), IMG_UINT64_C(0x154a4894aa4d2a59), IMG_UINT64_C(0x000000aa4aad694a) },	/* 35.0.1632.21 */
+	{ IMG_UINT64_C(0x0023000006600017), IMG_UINT64_C(0x1fbaf98fedfd3dcf), IMG_UINT64_C(0x154a4894aa4d2a59), IMG_UINT64_C(0x000000aa6aad694a) },	/* 35.0.1632.23 */
+	{ IMG_UINT64_C(0x0023000006600022), IMG_UINT64_C(0x7ffaf9afedfd3dcf), IMG_UINT64_C(0x154a4892a24d2aa9), IMG_UINT64_C(0x000000aa4ab9694a) },	/* 35.0.1632.34 */
+	{ IMG_UINT64_C(0x0023000006600023), IMG_UINT64_C(0xfffaf9afedfd3dcf), IMG_UINT64_C(0x154a4892a24d2aa9), IMG_UINT64_C(0x000000aa6ab9694a) },	/* 35.0.1632.35 */
+	{ IMG_UINT64_C(0x0026000009900192), IMG_UINT64_C(0x1fbbf9cfedfd3ddf), IMG_UINT64_C(0x156c4ab4aa4d2a59), IMG_UINT64_C(0x000000aa2d59ad52) },	/* 38.0.2448.402 */
+	{ IMG_UINT64_C(0x00460000099001a2), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156c4ab4aa553459), IMG_UINT64_C(0x000000aa2d59ad53) },	/* 70.0.2448.418 */
+	{ IMG_UINT64_C(0x0046000009900411), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156b4ab4aa5d3459), IMG_UINT64_C(0x000000aa2ab58d4b) },	/* 70.0.2448.1041 */
+	{ IMG_UINT64_C(0x0046000009900412), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156b4ab4aa5d3459), IMG_UINT64_C(0x000000aa2ab58d4b) },	/* 70.0.2448.1042 */
+	{ IMG_UINT64_C(0x0046000009900550), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156c4ab4aa5d3459), IMG_UINT64_C(0x000000aa2d59ad53) },	/* 70.0.2448.1360 */
+	{ IMG_UINT64_C(0x00470000099004ba), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x254b6ab2a35d345a), IMG_UINT64_C(0x000000ab2b558954) },	/* 71.0.2448.1210 */
+	{ IMG_UINT64_C(0x00470000099004bb), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x254b6ab2a365345a), IMG_UINT64_C(0x000000ab2b558954) },	/* 71.0.2448.1211 */
+	{ IMG_UINT64_C(0x00470000099004bc), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x254b6ab2a365345a), IMG_UINT64_C(0x000000ab2b558954) },	/* 71.0.2448.1212 */
+	{ IMG_UINT64_C(0x004700000e580820), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x256c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4c) },	/* 71.0.3672.2080 */
+	{ IMG_UINT64_C(0x004700000e580821), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2081 */
+	{ IMG_UINT64_C(0x004700000e580822), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2082 */
+	{ IMG_UINT64_C(0x004700000e580823), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2083 */
+	{ IMG_UINT64_C(0x004700000e580824), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2084 */
+};
+
+/******************************************************************************
+ * Table mapping bitmasks for ERNs/BRNs
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaErnsBrns[][2]=
+{
+	{ IMG_UINT64_C(0x001b000500fe0002), IMG_UINT64_C(0x0000000000000009) },	/* 27.5.254.2 */
+	{ IMG_UINT64_C(0x001b000700fe0002), IMG_UINT64_C(0x0000000000000001) },	/* 27.7.254.2 */
+	{ IMG_UINT64_C(0x001b000800fe0002), IMG_UINT64_C(0x0000000000000001) },	/* 27.8.254.2 */
+	{ IMG_UINT64_C(0x001e000301980065), IMG_UINT64_C(0x0000000000000053) },	/* 30.3.408.101 */
+	{ IMG_UINT64_C(0x001e000303300014), IMG_UINT64_C(0x0000000000000053) },	/* 30.3.816.20 */
+	{ IMG_UINT64_C(0x0023000201980022), IMG_UINT64_C(0x0000000000000053) },	/* 35.2.408.34 */
+	{ IMG_UINT64_C(0x0023000206600015), IMG_UINT64_C(0x0000000000000057) },	/* 35.2.1632.21 */
+	{ IMG_UINT64_C(0x0023000206600017), IMG_UINT64_C(0x0000000000000057) },	/* 35.2.1632.23 */
+	{ IMG_UINT64_C(0x0023000206600022), IMG_UINT64_C(0x0000000000000053) },	/* 35.2.1632.34 */
+	{ IMG_UINT64_C(0x0023000206600023), IMG_UINT64_C(0x0000000000000053) },	/* 35.2.1632.35 */
+	{ IMG_UINT64_C(0x0023000301980021), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.408.33 */
+	{ IMG_UINT64_C(0x0023000301980022), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.408.34 */
+	{ IMG_UINT64_C(0x0023000301980065), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.408.101 */
+	{ IMG_UINT64_C(0x0023000306600017), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.1632.23 */
+	{ IMG_UINT64_C(0x0023000406600017), IMG_UINT64_C(0x0000000000000053) },	/* 35.4.1632.23 */
+	{ IMG_UINT64_C(0x0023000501980017), IMG_UINT64_C(0x0000000000000053) },	/* 35.5.408.23 */
+	{ IMG_UINT64_C(0x0026000409900192), IMG_UINT64_C(0x0000000000000073) },	/* 38.4.2448.402 */
+	{ IMG_UINT64_C(0x0026000609900192), IMG_UINT64_C(0x0000000000000073) },	/* 38.6.2448.402 */
+	{ IMG_UINT64_C(0x00460002099001a2), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.418 */
+	{ IMG_UINT64_C(0x0046000209900411), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.1041 */
+	{ IMG_UINT64_C(0x0046000209900412), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.1042 */
+	{ IMG_UINT64_C(0x0046000209900550), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.1360 */
+	{ IMG_UINT64_C(0x0046000309900412), IMG_UINT64_C(0x0000000000000003) },	/* 70.3.2448.1042 */
+	{ IMG_UINT64_C(0x0046000309900550), IMG_UINT64_C(0x0000000000000003) },	/* 70.3.2448.1360 */
+	{ IMG_UINT64_C(0x00470002099004ba), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.2448.1210 */
+	{ IMG_UINT64_C(0x00470002099004bb), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.2448.1211 */
+	{ IMG_UINT64_C(0x00470002099004bc), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.2448.1212 */
+	{ IMG_UINT64_C(0x004700020e580820), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2080 */
+	{ IMG_UINT64_C(0x004700020e580821), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2081 */
+	{ IMG_UINT64_C(0x004700020e580822), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2082 */
+	{ IMG_UINT64_C(0x004700020e580823), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2083 */
+	{ IMG_UINT64_C(0x004700030e580823), IMG_UINT64_C(0x0000000000000083) },	/* 71.3.3672.2083 */
+	{ IMG_UINT64_C(0x004700030e580824), IMG_UINT64_C(0x0000000000000083) },	/* 71.3.3672.2084 */
+};
+
+#if defined(DEBUG)
+
+#define FEATURE_NO_VALUES_NAMES_MAX_IDX (65U)
+
+static const IMG_CHAR * const gaszFeaturesNoValuesNames[FEATURE_NO_VALUES_NAMES_MAX_IDX] =
+{
+	"ALBIORIX_TOP_INFRASTRUCTURE",
+	"AXI_ACE",
+	"BARREX_TOP_INFRASTRUCTURE",
+	"BINDLESS_IMAGE_AND_TEXTURE_STATE",
+	"CATURIX_TOP_INFRASTRUCTURE",
+	"CATURIX_XTP_TOP_INFRASTRUCTURE",
+	"CLUSTER_GROUPING",
+	"COMPUTE",
+	"COMPUTE_MORTON_CAPABLE",
+	"COMPUTE_ONLY",
+	"COMPUTE_OVERLAP",
+	"COMPUTE_OVERLAP_WITH_BARRIERS",
+	"COMPUTE_SLC_MMU_AUTO_CACHE_OPS",
+	"COREID_PER_OS",
+	"CR_PARITY",
+	"DIVANO_TOP_INFRASTRUCTURE",
+	"DUST_POWER_ISLAND_S7",
+	"ERYX_TOP_INFRASTRUCTURE",
+	"FASTRENDER_DM",
+	"FRAG_SLC_MMU_AUTO_CACHE_OPS",
+	"GEOMETRY_BIF_ARBITER",
+	"GEOM_SLC_MMU_AUTO_CACHE_OPS",
+	"GPU_MULTICORE_SUPPORT",
+	"GPU_VIRTUALISATION",
+	"GS_RTA_SUPPORT",
+	"IDLE_CYCLE_STEALING",
+	"META_DMA",
+	"META_REGISTER_UNPACKED_ACCESSES",
+	"MH_PARITY",
+	"PBE_CHECKSUM_2D",
+	"PBVNC_COREID_REG",
+	"PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE",
+	"PDS_TEMPSIZE8",
+	"PERFBUS",
+	"PERF_COUNTER_BATCH",
+	"PM_BYTE_ALIGNED_BASE_ADDRESSES",
+	"RISCV_DUAL_LOCKSTEP",
+	"RISCV_FW_PROCESSOR",
+	"RT_RAC_PER_SPU",
+	"S7_CACHE_HIERARCHY",
+	"S7_TOP_INFRASTRUCTURE",
+	"SAFETY_IRQ",
+	"SAFETY_SELF_TEST",
+	"SCALABLE_VDM_GPP",
+	"SIGNAL_SNOOPING",
+	"SLC_SIZE_ADJUSTMENT",
+	"SLC_VIVT",
+	"SOC_TIMER",
+	"SPU_ARCH_CLOCK_GATING",
+	"SYS_BUS_SECURE_RESET",
+	"SYS_PARITY",
+	"TDM_PDS_CHECKSUM",
+	"TDM_SLC_MMU_AUTO_CACHE_OPS",
+	"TESSELLATION",
+	"TILE_REGION_PROTECTION",
+	"TPU_CEM_DATAMASTER_GLOBAL_REGISTERS",
+	"TPU_DM_GLOBAL_REGISTERS",
+	"USC_INSTRUCTION_CACHE_AUTO_INVALIDATE",
+	"USC_TIMER",
+	"VDM_DRAWINDIRECT",
+	"VDM_OBJECT_LEVEL_LLS",
+	"WATCHDOG_TIMER",
+	"WORKGROUP_PROTECTION",
+	"WORKGROUP_PROTECTION_SMP",
+	"ZLS_CHECKSUM",
+};
+
+#define ERNSBRNS_IDS_MAX_IDX (8U)
+
+static const IMG_UINT32 gaui64ErnsBrnsIDs[ERNSBRNS_IDS_MAX_IDX] =
+{
+	65104,
+	69700,
+	71157,
+	71422,
+	71960,
+	72143,
+	72144,
+	74812,
+};
+
+#endif /* defined(DEBUG) */
+#endif /* RGX_BVNC_TABLE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_cr_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_cr_defs_km.h
new file mode 100644
index 000000000000..a9edab2f32e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_cr_defs_km.h
@@ -0,0 +1,10332 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_cr_defs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGX_CR_DEFS_KM_H
+#define RGX_CR_DEFS_KM_H
+
+#if !defined(IMG_EXPLICIT_INCLUDE_HWDEFS)
+#error This file may only be included if explicitly defined
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGX_CR_DEFS_KM_REVISION 217
+
+/*
+    Register RGX_CR_USC_INDIRECT
+*/
+#define RGX_CR_USC_INDIRECT                                                         (0x8000U)
+#define RGX_CR_USC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_USC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_USC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_MERCER_INDIRECT
+*/
+#define RGX_CR_MERCER_INDIRECT                                                      (0x8238U)
+#define RGX_CR_MERCER_INDIRECT_MASKFULL                                             (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MERCER_INDIRECT_ADDRESS_SHIFT                                        (0U)
+#define RGX_CR_MERCER_INDIRECT_ADDRESS_CLRMSK                                       (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_PBE_INDIRECT
+*/
+#define RGX_CR_PBE_INDIRECT                                                         (0x83E0U)
+#define RGX_CR_PBE_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_PBE_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_PBE_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_PBE_SHARED_INDIRECT
+*/
+#define RGX_CR_PBE_SHARED_INDIRECT                                                  (0x8388U)
+#define RGX_CR_PBE_SHARED_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_PBE_SHARED_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_PBE_SHARED_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFE0U)
+
+
+/*
+    Register RGX_CR_ISP_INDIRECT
+*/
+#define RGX_CR_ISP_INDIRECT                                                         (0x83F8U)
+#define RGX_CR_ISP_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_ISP_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_ISP_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_TPU_INDIRECT
+*/
+#define RGX_CR_TPU_INDIRECT                                                         (0x83E8U)
+#define RGX_CR_TPU_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_TPU_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_TPU_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_SWIFT_INDIRECT
+*/
+#define RGX_CR_SWIFT_INDIRECT                                                       (0x8308U)
+#define RGX_CR_SWIFT_INDIRECT_MASKFULL                                              (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_SWIFT_INDIRECT_ADDRESS_SHIFT                                         (0U)
+#define RGX_CR_SWIFT_INDIRECT_ADDRESS_CLRMSK                                        (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_TEXAS_INDIRECT
+*/
+#define RGX_CR_TEXAS_INDIRECT                                                       (0x8390U)
+#define RGX_CR_TEXAS_INDIRECT_MASKFULL                                              (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TEXAS_INDIRECT_ADDRESS_SHIFT                                         (0U)
+#define RGX_CR_TEXAS_INDIRECT_ADDRESS_CLRMSK                                        (0xFFFFFFE0U)
+
+
+/*
+    Register RGX_CR_RAC_INDIRECT
+*/
+#define RGX_CR_RAC_INDIRECT                                                         (0x8398U)
+#define RGX_CR_RAC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_RAC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_RAC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_CLK_CTRL0
+*/
+#define RGX_CR_CLK_CTRL0                                                            (0x0000U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__MASKFULL                                      (IMG_UINT64_C(0xFFCF0303FF3F333F))
+#define RGX_CR_CLK_CTRL0_MASKFULL                                                   (IMG_UINT64_C(0xFFCF0303FF3F3303))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_SHIFT                                            (62U)
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_CLRMSK                                           (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_ON                                               (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_AUTO                                             (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS__PIPEDM_GT0__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS__PIPEDM_GT0__ON                                  (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS__PIPEDM_GT0__AUTO                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE_SHIFT                                              (60U)
+#define RGX_CR_CLK_CTRL0_FBCACHE_CLRMSK                                             (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBCACHE_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE_ON                                                 (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE_AUTO                                               (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE__PIPEDM_GT0__OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE__PIPEDM_GT0__ON                                    (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE__PIPEDM_GT0__AUTO                                  (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL0_FBC_SHIFT                                                  (58U)
+#define RGX_CR_CLK_CTRL0_FBC_CLRMSK                                                 (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBC_ON                                                     (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL0_FBC_AUTO                                                   (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL0_FBC__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBC__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL0_FBC__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC_SHIFT                                                 (56U)
+#define RGX_CR_CLK_CTRL0_FBDC_CLRMSK                                                (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBDC_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC_ON                                                    (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC_AUTO                                                  (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC__PIPEDM_GT0__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC__PIPEDM_GT0__ON                                       (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC__PIPEDM_GT0__AUTO                                     (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL0_FBM_SHIFT                                                  (54U)
+#define RGX_CR_CLK_CTRL0_FBM_CLRMSK                                                 (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBM_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBM_ON                                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL0_FBM_AUTO                                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL0_FBM__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBM__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL0_FBM__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL0_PBE_SHIFT                                                  (50U)
+#define RGX_CR_CLK_CTRL0_PBE_CLRMSK                                                 (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_PBE_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PBE_ON                                                     (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL0_PBE_AUTO                                                   (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL0_PBE__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PBE__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL0_PBE__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1_SHIFT                                               (48U)
+#define RGX_CR_CLK_CTRL0_MCU_L1_CLRMSK                                              (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_MCU_L1_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1_ON                                                  (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1_AUTO                                                (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1__PIPEDM_GT0__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1__PIPEDM_GT0__ON                                     (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1__PIPEDM_GT0__AUTO                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_SHIFT                                                  (40U)
+#define RGX_CR_CLK_CTRL0_BIF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_BIF_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_ON                                                     (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL0_BIF_AUTO                                                   (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL0_BIF__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL0_BIF__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL0_TAG_SHIFT                                                  (32U)
+#define RGX_CR_CLK_CTRL0_TAG_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_TAG_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TAG_ON                                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_CTRL0_TAG_AUTO                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_CTRL0_TAG__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TAG__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_CTRL0_TAG__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_CTRL0_MADD_SHIFT                                                 (30U)
+#define RGX_CR_CLK_CTRL0_MADD_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_CTRL0_MADD_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MADD_ON                                                    (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_CTRL0_MADD_AUTO                                                  (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_CTRL0_MADD__PIPEDM_GT0__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MADD__PIPEDM_GT0__ON                                       (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_CTRL0_MADD__PIPEDM_GT0__AUTO                                     (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_CTRL0_TF_SHIFT                                                   (28U)
+#define RGX_CR_CLK_CTRL0_TF_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL0_TF_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TF_ON                                                      (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL0_TF_AUTO                                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL0_TF__PIPEDM_GT0__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TF__PIPEDM_GT0__ON                                         (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL0_TF__PIPEDM_GT0__AUTO                                       (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0_SHIFT                                               (26U)
+#define RGX_CR_CLK_CTRL0_MCU_L0_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL0_MCU_L0_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0_ON                                                  (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0_AUTO                                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0__PIPEDM_GT0__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0__PIPEDM_GT0__ON                                     (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0__PIPEDM_GT0__AUTO                                   (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL0_TPU_SHIFT                                                  (24U)
+#define RGX_CR_CLK_CTRL0_TPU_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL0_TPU_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TPU_ON                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL0_TPU_AUTO                                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL0_TPU__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TPU__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL0_TPU__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL0_USC_SHIFT                                                  (20U)
+#define RGX_CR_CLK_CTRL0_USC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL0_USC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_USC_ON                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL0_USC_AUTO                                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL0_USC__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_USC__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL0_USC__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_SHIFT                                             (18U)
+#define RGX_CR_CLK_CTRL0_SLC_BANK_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_ON                                                (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_AUTO                                              (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK__PIPEDM_GT0__OFF                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK__PIPEDM_GT0__ON                                   (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK__PIPEDM_GT0__AUTO                                 (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL0_SLC_SHIFT                                                  (16U)
+#define RGX_CR_CLK_CTRL0_SLC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL0_SLC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC_ON                                                     (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL0_SLC_AUTO                                                   (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL0_SLC__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL0_SLC__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL0_PDS_SHIFT                                                  (12U)
+#define RGX_CR_CLK_CTRL0_PDS_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL0_PDS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PDS_ON                                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL0_PDS_AUTO                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL0_PDS__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PDS__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL0_PDS__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL0_PM_SHIFT                                                   (8U)
+#define RGX_CR_CLK_CTRL0_PM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL0_PM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PM_ON                                                      (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL0_PM_AUTO                                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL0_PM__PIPEDM_GT0__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PM__PIPEDM_GT0__ON                                         (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL0_PM__PIPEDM_GT0__AUTO                                       (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_SHIFT                              (4U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_OFF                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_ON                                 (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_AUTO                               (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_SHIFT                                  (2U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_ON                                     (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_AUTO                                   (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL0_ISP_SHIFT                                                  (0U)
+#define RGX_CR_CLK_CTRL0_ISP_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL0_ISP_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_ISP_ON                                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL0_ISP_AUTO                                                   (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL0_ISP__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_ISP__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL0_ISP__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_SHIFT                                  (0U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_ON                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_AUTO                                   (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS0
+*/
+#define RGX_CR_CLK_STATUS0                                                          (0x0008U)
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__MASKFULL                                    (IMG_UINT64_C(0x00000001BF107F53))
+#define RGX_CR_CLK_STATUS0_MASKFULL                                                 (IMG_UINT64_C(0x00000001BF107F51))
+#define RGX_CR_CLK_STATUS0_MCU_L0_SHIFT                                             (32U)
+#define RGX_CR_CLK_STATUS0_MCU_L0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS0_MCU_L0_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L0_RUNNING                                           (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS0_MCU_L0__PIPEDM_GT0__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L0__PIPEDM_GT0__RUNNING                              (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_SHIFT                                          (31U)
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_RUNNING                                        (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS__PIPEDM_GT0__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS__PIPEDM_GT0__RUNNING                           (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE_SHIFT                                            (29U)
+#define RGX_CR_CLK_STATUS0_FBCACHE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS0_FBCACHE_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE_RUNNING                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE__PIPEDM_GT0__GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE__PIPEDM_GT0__RUNNING                             (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS0_FBC_SHIFT                                                (28U)
+#define RGX_CR_CLK_STATUS0_FBC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS0_FBC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBC_RUNNING                                              (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS0_FBC__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBC__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS0_FBDC_SHIFT                                               (27U)
+#define RGX_CR_CLK_STATUS0_FBDC_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS0_FBDC_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBDC_RUNNING                                             (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS0_FBDC__PIPEDM_GT0__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBDC__PIPEDM_GT0__RUNNING                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS0_FBM_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS0_FBM_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS0_FBM_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBM_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS0_FBM__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBM__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS0_PBE_SHIFT                                                (25U)
+#define RGX_CR_CLK_STATUS0_PBE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS0_PBE_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PBE_RUNNING                                              (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS0_PBE__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PBE__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1_SHIFT                                             (24U)
+#define RGX_CR_CLK_STATUS0_MCU_L1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS0_MCU_L1_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1_RUNNING                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1__PIPEDM_GT0__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1__PIPEDM_GT0__RUNNING                              (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS0_BIF_SHIFT                                                (20U)
+#define RGX_CR_CLK_STATUS0_BIF_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS0_BIF_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF_RUNNING                                              (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS0_BIF__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS0_TF_SHIFT                                                 (14U)
+#define RGX_CR_CLK_STATUS0_TF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS0_TF_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TF_RUNNING                                               (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS0_TF__PIPEDM_GT0__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TF__PIPEDM_GT0__RUNNING                                  (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS0_MADD_SHIFT                                               (13U)
+#define RGX_CR_CLK_STATUS0_MADD_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS0_MADD_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MADD_RUNNING                                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS0_MADD__PIPEDM_GT0__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MADD__PIPEDM_GT0__RUNNING                                (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS0_TPU_SHIFT                                                (12U)
+#define RGX_CR_CLK_STATUS0_TPU_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS0_TPU_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TPU_RUNNING                                              (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS0_TPU__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TPU__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS0_TAG_SHIFT                                                (11U)
+#define RGX_CR_CLK_STATUS0_TAG_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_CLK_STATUS0_TAG_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TAG_RUNNING                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_STATUS0_TAG__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TAG__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_STATUS0_USC_SHIFT                                                (10U)
+#define RGX_CR_CLK_STATUS0_USC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS0_USC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_USC_RUNNING                                              (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS0_USC__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_USC__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS0_SLC_BANK_SHIFT                                           (9U)
+#define RGX_CR_CLK_STATUS0_SLC_BANK_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS0_SLC_BANK_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC_BANK_RUNNING                                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS0_SLC_BANK__PIPEDM_GT0__GATED                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC_BANK__PIPEDM_GT0__RUNNING                            (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS0_SLC_SHIFT                                                (8U)
+#define RGX_CR_CLK_STATUS0_SLC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS0_SLC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC_RUNNING                                              (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS0_SLC__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS0_PDS_SHIFT                                                (6U)
+#define RGX_CR_CLK_STATUS0_PDS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS0_PDS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PDS_RUNNING                                              (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS0_PDS__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PDS__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS0_PM_SHIFT                                                 (4U)
+#define RGX_CR_CLK_STATUS0_PM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS0_PM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PM_RUNNING                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS0_PM__PIPEDM_GT0__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PM__PIPEDM_GT0__RUNNING                                  (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_SHIFT                                (1U)
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_RUNNING                              (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS0_ISP_SHIFT                                                (0U)
+#define RGX_CR_CLK_STATUS0_ISP_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS0_ISP_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_ISP_RUNNING                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS0_ISP__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_ISP__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_SHIFT                                (0U)
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_RUNNING                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID                                                              (0x0020U)
+#define RGX_CR_CORE_ID_MASKFULL                                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CORE_ID_BRANCH_ID_SHIFT                                              (48U)
+#define RGX_CR_CORE_ID_BRANCH_ID_CLRMSK                                             (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CORE_ID_VERSION_ID_SHIFT                                             (32U)
+#define RGX_CR_CORE_ID_VERSION_ID_CLRMSK                                            (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_SHIFT                               (16U)
+#define RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_CORE_ID_CONFIG_ID_SHIFT                                              (0U)
+#define RGX_CR_CORE_ID_CONFIG_ID_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SPU_ENABLE
+*/
+#define RGX_CR_SPU_ENABLE                                                           (0x0050U)
+#define RGX_CR_SPU_ENABLE_MASKFULL                                                  (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SPU_ENABLE_ENABLE_SHIFT                                              (0U)
+#define RGX_CR_SPU_ENABLE_ENABLE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_SOC_TIMER_GRAY
+*/
+#define RGX_CR_SOC_TIMER_GRAY                                                       (0x00E0U)
+#define RGX_CR_SOC_TIMER_GRAY_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_SHIFT                                           (0U)
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_CLRMSK                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_BINARY
+*/
+#define RGX_CR_SOC_TIMER_BINARY                                                     (0x00E8U)
+#define RGX_CR_SOC_TIMER_BINARY_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_CLRMSK                                        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_CLK_CTRL1
+*/
+#define RGX_CR_CLK_CTRL1                                                            (0x0080U)
+#define RGX_CR_CLK_CTRL1_MASKFULL                                                   (IMG_UINT64_C(0xFFFC3F3FFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_BSC_SHIFT                                                  (62U)
+#define RGX_CR_CLK_CTRL1_BSC_CLRMSK                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_BSC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_BSC_ON                                                     (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL1_BSC_AUTO                                                   (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_SHIFT                                         (60U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_CLRMSK                                        (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_ON                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_AUTO                                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_SHIFT                                     (58U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_CLRMSK                                    (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_ON                                        (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_AUTO                                      (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_SHIFT                                         (56U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_CLRMSK                                        (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_ON                                            (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_AUTO                                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_SHIFT                                      (54U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_CLRMSK                                     (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_ON                                         (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_AUTO                                       (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL1_PSB_SHIFT                                                  (52U)
+#define RGX_CR_CLK_CTRL1_PSB_CLRMSK                                                 (IMG_UINT64_C(0xFFCFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_PSB_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_PSB_ON                                                     (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL1_PSB_AUTO                                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_SHIFT                                       (50U)
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_CLRMSK                                      (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_ON                                          (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_AUTO                                        (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_SHIFT                                             (44U)
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_CLRMSK                                            (IMG_UINT64_C(0xFFFFCFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_ON                                                (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_AUTO                                              (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_SHIFT                                         (42U)
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_CLRMSK                                        (IMG_UINT64_C(0xFFFFF3FFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_ON                                            (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_AUTO                                          (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL1_TCU_L1_SHIFT                                               (40U)
+#define RGX_CR_CLK_CTRL1_TCU_L1_CLRMSK                                              (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_TCU_L1_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TCU_L1_ON                                                  (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL1_TCU_L1_AUTO                                                (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL1_TDM_SHIFT                                                  (36U)
+#define RGX_CR_CLK_CTRL1_TDM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_TDM_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TDM_ON                                                     (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_CLK_CTRL1_TDM_AUTO                                                   (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_CLK_CTRL1_ASTC_SHIFT                                                 (34U)
+#define RGX_CR_CLK_CTRL1_ASTC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFF3FFFFFFFF))
+#define RGX_CR_CLK_CTRL1_ASTC_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_ASTC_ON                                                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_CTRL1_ASTC_AUTO                                                  (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_CLK_CTRL1_IPF_SHIFT                                                  (32U)
+#define RGX_CR_CLK_CTRL1_IPF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_IPF_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_IPF_ON                                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_CTRL1_IPF_AUTO                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_CTRL1_COMPUTE_SHIFT                                              (30U)
+#define RGX_CR_CLK_CTRL1_COMPUTE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_CTRL1_COMPUTE_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_COMPUTE_ON                                                 (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_CTRL1_COMPUTE_AUTO                                               (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_CTRL1_PIXEL_SHIFT                                                (28U)
+#define RGX_CR_CLK_CTRL1_PIXEL_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL1_PIXEL_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_PIXEL_ON                                                   (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL1_PIXEL_AUTO                                                 (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL1_VERTEX_SHIFT                                               (26U)
+#define RGX_CR_CLK_CTRL1_VERTEX_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL1_VERTEX_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_VERTEX_ON                                                  (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL1_VERTEX_AUTO                                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL1_TPF_SHIFT                                                  (24U)
+#define RGX_CR_CLK_CTRL1_TPF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL1_TPF_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TPF_ON                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL1_TPF_AUTO                                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_SHIFT                                           (22U)
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_ON                                              (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_AUTO                                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_SHIFT                                         (20U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_ON                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_AUTO                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_SHIFT                                           (18U)
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_ON                                              (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_AUTO                                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_SHIFT                                             (16U)
+#define RGX_CR_CLK_CTRL1_GEO_TESS_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_ON                                                (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_AUTO                                              (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_SHIFT                                         (14U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_ON                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_AUTO                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_SHIFT                                         (12U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_ON                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_AUTO                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_SHIFT                                         (10U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_ON                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_AUTO                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_SHIFT                                          (8U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_ON                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_AUTO                                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_SHIFT                                         (6U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_ON                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_AUTO                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_SHIFT                                         (4U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_ON                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_AUTO                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_SHIFT                                         (2U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_ON                                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_AUTO                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_SHIFT                                         (0U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_ON                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_AUTO                                          (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS1
+*/
+#define RGX_CR_CLK_STATUS1                                                          (0x0088U)
+#define RGX_CR_CLK_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0x00000007FFFF77FB))
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_SHIFT                                         (34U)
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_RUNNING                                       (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_STATUS1_TILING_SHIFT                                             (33U)
+#define RGX_CR_CLK_STATUS1_TILING_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_CLK_STATUS1_TILING_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TILING_RUNNING                                           (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_STATUS1_GEO_VDM_SHIFT                                            (32U)
+#define RGX_CR_CLK_STATUS1_GEO_VDM_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS1_GEO_VDM_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_VDM_RUNNING                                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_SHIFT                                       (31U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_RUNNING                                     (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_SHIFT                                   (30U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_RUNNING                                 (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_SHIFT                                       (29U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_RUNNING                                     (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_SHIFT                                    (28U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_RUNNING                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS1_BSC_SHIFT                                                (27U)
+#define RGX_CR_CLK_STATUS1_BSC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS1_BSC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_BSC_RUNNING                                              (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS1_PSB_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS1_PSB_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS1_PSB_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_PSB_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_SHIFT                                     (25U)
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_RUNNING                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_SHIFT                                       (24U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_RUNNING                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_SHIFT                                       (23U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_RUNNING                                     (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_SHIFT                                       (22U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_RUNNING                                     (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_SHIFT                                        (21U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_RUNNING                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_SHIFT                                       (20U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_RUNNING                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_SHIFT                                       (19U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_RUNNING                                     (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_SHIFT                                       (18U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_RUNNING                                     (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_SHIFT                                       (17U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_RUNNING                                     (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_SHIFT                                       (16U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_RUNNING                                     (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_SHIFT                                           (14U)
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_RUNNING                                         (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS1_TCU_L1_SHIFT                                             (13U)
+#define RGX_CR_CLK_STATUS1_TCU_L1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS1_TCU_L1_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TCU_L1_RUNNING                                           (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_SHIFT                                       (12U)
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_RUNNING                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS1_TDM_SHIFT                                                (10U)
+#define RGX_CR_CLK_STATUS1_TDM_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS1_TDM_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TDM_RUNNING                                              (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS1_IPF_SHIFT                                                (9U)
+#define RGX_CR_CLK_STATUS1_IPF_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS1_IPF_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_IPF_RUNNING                                              (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS1_COMPUTE_SHIFT                                            (8U)
+#define RGX_CR_CLK_STATUS1_COMPUTE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS1_COMPUTE_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_COMPUTE_RUNNING                                          (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS1_ASTC_SHIFT                                               (7U)
+#define RGX_CR_CLK_STATUS1_ASTC_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS1_ASTC_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_ASTC_RUNNING                                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS1_PIXEL_SHIFT                                              (6U)
+#define RGX_CR_CLK_STATUS1_PIXEL_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS1_PIXEL_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_PIXEL_RUNNING                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS1_VERTEX_SHIFT                                             (5U)
+#define RGX_CR_CLK_STATUS1_VERTEX_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS1_VERTEX_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_VERTEX_RUNNING                                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS1_TPF_SHIFT                                                (4U)
+#define RGX_CR_CLK_STATUS1_TPF_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS1_TPF_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TPF_RUNNING                                              (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_SHIFT                                         (3U)
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_RUNNING                                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_SHIFT                                         (1U)
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_RUNNING                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS1_GEO_TESS_SHIFT                                           (0U)
+#define RGX_CR_CLK_STATUS1_GEO_TESS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS1_GEO_TESS_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_TESS_RUNNING                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CLK_CTRL2
+*/
+#define RGX_CR_CLK_CTRL2                                                            (0x0090U)
+#define RGX_CR_CLK_CTRL2_MASKFULL                                                   (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CLK_CTRL2_TILING_SHIFT                                               (28U)
+#define RGX_CR_CLK_CTRL2_TILING_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL2_TILING_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_TILING_ON                                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL2_TILING_AUTO                                                (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_SHIFT                                              (26U)
+#define RGX_CR_CLK_CTRL2_GEO_VDM_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_ON                                                 (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_AUTO                                               (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL2_RTS_SHIFT                                                  (24U)
+#define RGX_CR_CLK_CTRL2_RTS_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL2_RTS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RTS_ON                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL2_RTS_AUTO                                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL2_PCG_SHIFT                                                  (22U)
+#define RGX_CR_CLK_CTRL2_PCG_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_CTRL2_PCG_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_PCG_ON                                                     (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_CTRL2_PCG_AUTO                                                   (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_CTRL2_BPS_SHIFT                                                  (20U)
+#define RGX_CR_CLK_CTRL2_BPS_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL2_BPS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_BPS_ON                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL2_BPS_AUTO                                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL2_RS_SHIFT                                                   (18U)
+#define RGX_CR_CLK_CTRL2_RS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL2_RS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RS_ON                                                      (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL2_RS_AUTO                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL2_RRC_SHIFT                                                  (16U)
+#define RGX_CR_CLK_CTRL2_RRC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL2_RRC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RRC_ON                                                     (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL2_RRC_AUTO                                                   (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL2_URI_SHIFT                                                  (14U)
+#define RGX_CR_CLK_CTRL2_URI_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL2_URI_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_URI_ON                                                     (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL2_URI_AUTO                                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL2_RCE_SHIFT                                                  (12U)
+#define RGX_CR_CLK_CTRL2_RCE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL2_RCE_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RCE_ON                                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL2_RCE_AUTO                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL2_ASC_SHIFT                                                  (10U)
+#define RGX_CR_CLK_CTRL2_ASC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL2_ASC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_ASC_ON                                                     (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL2_ASC_AUTO                                                   (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL2_FP_SHIFT                                                   (8U)
+#define RGX_CR_CLK_CTRL2_FP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL2_FP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_FP_ON                                                      (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL2_FP_AUTO                                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL2_INT_SHIFT                                                  (6U)
+#define RGX_CR_CLK_CTRL2_INT_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL2_INT_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_INT_ON                                                     (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL2_INT_AUTO                                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL2_YUV_SHIFT                                                  (4U)
+#define RGX_CR_CLK_CTRL2_YUV_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL2_YUV_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_YUV_ON                                                     (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL2_YUV_AUTO                                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_SHIFT                                    (2U)
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_ON                                       (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_AUTO                                     (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_SHIFT                                     (0U)
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_ON                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_AUTO                                      (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS2
+*/
+#define RGX_CR_CLK_STATUS2                                                          (0x0098U)
+#define RGX_CR_CLK_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_CLK_STATUS2_RTS_SHIFT                                                (12U)
+#define RGX_CR_CLK_STATUS2_RTS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS2_RTS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RTS_RUNNING                                              (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS2_PCG_SHIFT                                                (11U)
+#define RGX_CR_CLK_STATUS2_PCG_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_CLK_STATUS2_PCG_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_PCG_RUNNING                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_STATUS2_BPS_SHIFT                                                (10U)
+#define RGX_CR_CLK_STATUS2_BPS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS2_BPS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_BPS_RUNNING                                              (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS2_RS_SHIFT                                                 (9U)
+#define RGX_CR_CLK_STATUS2_RS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS2_RS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RS_RUNNING                                               (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS2_RRC_SHIFT                                                (8U)
+#define RGX_CR_CLK_STATUS2_RRC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS2_RRC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RRC_RUNNING                                              (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS2_URI_SHIFT                                                (7U)
+#define RGX_CR_CLK_STATUS2_URI_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS2_URI_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_URI_RUNNING                                              (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS2_RCE_SHIFT                                                (6U)
+#define RGX_CR_CLK_STATUS2_RCE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS2_RCE_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RCE_RUNNING                                              (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS2_ASC_SHIFT                                                (5U)
+#define RGX_CR_CLK_STATUS2_ASC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS2_ASC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_ASC_RUNNING                                              (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS2_FP_SHIFT                                                 (4U)
+#define RGX_CR_CLK_STATUS2_FP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS2_FP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_FP_RUNNING                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS2_INT_SHIFT                                                (3U)
+#define RGX_CR_CLK_STATUS2_INT_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS2_INT_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_INT_RUNNING                                              (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS2_YUV_SHIFT                                                (2U)
+#define RGX_CR_CLK_STATUS2_YUV_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS2_YUV_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_YUV_RUNNING                                              (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_SHIFT                                  (1U)
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_RUNNING                                (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_SHIFT                                   (0U)
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_RUNNING                                 (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET
+*/
+#define RGX_CR_SOFT_RESET                                                           (0x0100U)
+#define RGX_CR_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x1FFFFFE0007BDEFF))
+#define RGX_CR_SOFT_RESET_TPU2_LCTRL_SHIFT                                          (60U)
+#define RGX_CR_SOFT_RESET_TPU2_LCTRL_CLRMSK                                         (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TPU2_LCTRL_EN                                             (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SOFT_RESET_TILING_SHIFT                                              (59U)
+#define RGX_CR_SOFT_RESET_TILING_CLRMSK                                             (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TILING_EN                                                 (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SOFT_RESET_GEO_VDM_SHIFT                                             (58U)
+#define RGX_CR_SOFT_RESET_GEO_VDM_CLRMSK                                            (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GEO_VDM_EN                                                (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SOFT_RESET_RAC_SHIFT                                                 (57U)
+#define RGX_CR_SOFT_RESET_RAC_CLRMSK                                                (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_RAC_EN                                                    (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SOFT_RESET_GEO_TESS_SHIFT                                            (56U)
+#define RGX_CR_SOFT_RESET_GEO_TESS_CLRMSK                                           (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GEO_TESS_EN                                               (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SOFT_RESET_INT_SHIFT                                                 (55U)
+#define RGX_CR_SOFT_RESET_INT_CLRMSK                                                (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_INT_EN                                                    (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SOFT_RESET_FP_SHIFT                                                  (54U)
+#define RGX_CR_SOFT_RESET_FP_CLRMSK                                                 (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FP_EN                                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SOFT_RESET_YUV_SHIFT                                                 (53U)
+#define RGX_CR_SOFT_RESET_YUV_CLRMSK                                                (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_YUV_EN                                                    (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SOFT_RESET_PSB_SHIFT                                                 (52U)
+#define RGX_CR_SOFT_RESET_PSB_CLRMSK                                                (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PSB_EN                                                    (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SOFT_RESET_ASC_SHIFT                                                 (51U)
+#define RGX_CR_SOFT_RESET_ASC_CLRMSK                                                (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_ASC_EN                                                    (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SOFT_RESET_RCE_SHIFT                                                 (50U)
+#define RGX_CR_SOFT_RESET_RCE_CLRMSK                                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_RCE_EN                                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SOFT_RESET_BSC_SHIFT                                                 (49U)
+#define RGX_CR_SOFT_RESET_BSC_CLRMSK                                                (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BSC_EN                                                    (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SOFT_RESET_TPU_USC_SELECT_SHIFT                                      (48U)
+#define RGX_CR_SOFT_RESET_TPU_USC_SELECT_CLRMSK                                     (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TPU_USC_SELECT_EN                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SOFT_RESET_USC_L2ICACHE_SHIFT                                        (47U)
+#define RGX_CR_SOFT_RESET_USC_L2ICACHE_CLRMSK                                       (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_USC_L2ICACHE_EN                                           (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SOFT_RESET_TCU_L1_SHIFT                                              (46U)
+#define RGX_CR_SOFT_RESET_TCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TCU_L1_EN                                                 (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_SHIFT                                           (45U)
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_EN                                              (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SOFT_RESET_BIF_JONES_SHIFT                                           (44U)
+#define RGX_CR_SOFT_RESET_BIF_JONES_CLRMSK                                          (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_JONES_EN                                              (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SOFT_RESET_SLC_SHIFT                                                 (43U)
+#define RGX_CR_SOFT_RESET_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_SLC_EN                                                    (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SOFT_RESET_FBCACHE_SHIFT                                             (42U)
+#define RGX_CR_SOFT_RESET_FBCACHE_CLRMSK                                            (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBCACHE_EN                                                (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SOFT_RESET_FBM_SHIFT                                                 (41U)
+#define RGX_CR_SOFT_RESET_FBM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBM_EN                                                    (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SOFT_RESET_FBDC_SHIFT                                                (40U)
+#define RGX_CR_SOFT_RESET_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBDC_EN                                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SOFT_RESET_FBC_SHIFT                                                 (39U)
+#define RGX_CR_SOFT_RESET_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBC_EN                                                    (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SOFT_RESET_PM_SHIFT                                                  (38U)
+#define RGX_CR_SOFT_RESET_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PM_EN                                                     (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SOFT_RESET_GARTEN_SHIFT                                              (37U)
+#define RGX_CR_SOFT_RESET_GARTEN_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GARTEN_EN                                                 (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SOFT_RESET_TAG_SHIFT                                                 (22U)
+#define RGX_CR_SOFT_RESET_TAG_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SOFT_RESET_TAG_EN                                                    (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SOFT_RESET_TF_SHIFT                                                  (21U)
+#define RGX_CR_SOFT_RESET_TF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SOFT_RESET_TF_EN                                                     (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SOFT_RESET_MADD_SHIFT                                                (20U)
+#define RGX_CR_SOFT_RESET_MADD_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SOFT_RESET_MADD_EN                                                   (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SOFT_RESET_PBE_SHIFT                                                 (19U)
+#define RGX_CR_SOFT_RESET_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SOFT_RESET_PBE_EN                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SOFT_RESET_MCU_L1_SHIFT                                              (17U)
+#define RGX_CR_SOFT_RESET_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SOFT_RESET_MCU_L1_EN                                                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SOFT_RESET_CDM_PIPE_SHIFT                                            (16U)
+#define RGX_CR_SOFT_RESET_CDM_PIPE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SOFT_RESET_CDM_PIPE_EN                                               (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SOFT_RESET_TDM_SHIFT                                                 (15U)
+#define RGX_CR_SOFT_RESET_TDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SOFT_RESET_TDM_EN                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SOFT_RESET_ASTC_SHIFT                                                (14U)
+#define RGX_CR_SOFT_RESET_ASTC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SOFT_RESET_ASTC_EN                                                   (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SOFT_RESET_PDS_SHIFT                                                 (12U)
+#define RGX_CR_SOFT_RESET_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SOFT_RESET_PDS_EN                                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SOFT_RESET_ISP_SHIFT                                                 (11U)
+#define RGX_CR_SOFT_RESET_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SOFT_RESET_ISP_EN                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SOFT_RESET_TPF_SHIFT                                                 (10U)
+#define RGX_CR_SOFT_RESET_TPF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SOFT_RESET_TPF_EN                                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SOFT_RESET_IPF_SHIFT                                                 (9U)
+#define RGX_CR_SOFT_RESET_IPF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SOFT_RESET_IPF_EN                                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SOFT_RESET_GEO_SHARED_SHIFT                                          (7U)
+#define RGX_CR_SOFT_RESET_GEO_SHARED_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SOFT_RESET_GEO_SHARED_EN                                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SOFT_RESET_GEO_VERTEX_SHIFT                                          (6U)
+#define RGX_CR_SOFT_RESET_GEO_VERTEX_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SOFT_RESET_GEO_VERTEX_EN                                             (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SOFT_RESET_PIXEL_SHIFT                                               (5U)
+#define RGX_CR_SOFT_RESET_PIXEL_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SOFT_RESET_PIXEL_EN                                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SOFT_RESET_COMPUTE_SHIFT                                             (4U)
+#define RGX_CR_SOFT_RESET_COMPUTE_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SOFT_RESET_COMPUTE_EN                                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SOFT_RESET_MCU_L0_SHIFT                                              (3U)
+#define RGX_CR_SOFT_RESET_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SOFT_RESET_MCU_L0_EN                                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SOFT_RESET_TPU_SHIFT                                                 (2U)
+#define RGX_CR_SOFT_RESET_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SOFT_RESET_TPU_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SOFT_RESET_VERTEX_SHIFT                                              (1U)
+#define RGX_CR_SOFT_RESET_VERTEX_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SOFT_RESET_VERTEX_EN                                                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SOFT_RESET_USC_SHIFT                                                 (0U)
+#define RGX_CR_SOFT_RESET_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SOFT_RESET_USC_EN                                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET_SPU
+*/
+#define RGX_CR_SOFT_RESET_SPU                                                       (0x0108U)
+#define RGX_CR_SOFT_RESET_SPU_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SOFT_RESET_SPU_SPU31_SHIFT                                           (31U)
+#define RGX_CR_SOFT_RESET_SPU_SPU31_CLRMSK                                          (0x7FFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU31_EN                                              (0x80000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU30_SHIFT                                           (30U)
+#define RGX_CR_SOFT_RESET_SPU_SPU30_CLRMSK                                          (0xBFFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU30_EN                                              (0x40000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU29_SHIFT                                           (29U)
+#define RGX_CR_SOFT_RESET_SPU_SPU29_CLRMSK                                          (0xDFFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU29_EN                                              (0x20000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU28_SHIFT                                           (28U)
+#define RGX_CR_SOFT_RESET_SPU_SPU28_CLRMSK                                          (0xEFFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU28_EN                                              (0x10000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU27_SHIFT                                           (27U)
+#define RGX_CR_SOFT_RESET_SPU_SPU27_CLRMSK                                          (0xF7FFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU27_EN                                              (0x08000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU26_SHIFT                                           (26U)
+#define RGX_CR_SOFT_RESET_SPU_SPU26_CLRMSK                                          (0xFBFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU26_EN                                              (0x04000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU25_SHIFT                                           (25U)
+#define RGX_CR_SOFT_RESET_SPU_SPU25_CLRMSK                                          (0xFDFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU25_EN                                              (0x02000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU24_SHIFT                                           (24U)
+#define RGX_CR_SOFT_RESET_SPU_SPU24_CLRMSK                                          (0xFEFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU24_EN                                              (0x01000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU23_SHIFT                                           (23U)
+#define RGX_CR_SOFT_RESET_SPU_SPU23_CLRMSK                                          (0xFF7FFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU23_EN                                              (0x00800000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU22_SHIFT                                           (22U)
+#define RGX_CR_SOFT_RESET_SPU_SPU22_CLRMSK                                          (0xFFBFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU22_EN                                              (0x00400000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU21_SHIFT                                           (21U)
+#define RGX_CR_SOFT_RESET_SPU_SPU21_CLRMSK                                          (0xFFDFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU21_EN                                              (0x00200000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU20_SHIFT                                           (20U)
+#define RGX_CR_SOFT_RESET_SPU_SPU20_CLRMSK                                          (0xFFEFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU20_EN                                              (0x00100000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU19_SHIFT                                           (19U)
+#define RGX_CR_SOFT_RESET_SPU_SPU19_CLRMSK                                          (0xFFF7FFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU19_EN                                              (0x00080000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU18_SHIFT                                           (18U)
+#define RGX_CR_SOFT_RESET_SPU_SPU18_CLRMSK                                          (0xFFFBFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU18_EN                                              (0x00040000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU17_SHIFT                                           (17U)
+#define RGX_CR_SOFT_RESET_SPU_SPU17_CLRMSK                                          (0xFFFDFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU17_EN                                              (0x00020000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU16_SHIFT                                           (16U)
+#define RGX_CR_SOFT_RESET_SPU_SPU16_CLRMSK                                          (0xFFFEFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU16_EN                                              (0x00010000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU15_SHIFT                                           (15U)
+#define RGX_CR_SOFT_RESET_SPU_SPU15_CLRMSK                                          (0xFFFF7FFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU15_EN                                              (0x00008000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU14_SHIFT                                           (14U)
+#define RGX_CR_SOFT_RESET_SPU_SPU14_CLRMSK                                          (0xFFFFBFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU14_EN                                              (0x00004000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU13_SHIFT                                           (13U)
+#define RGX_CR_SOFT_RESET_SPU_SPU13_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU13_EN                                              (0x00002000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU12_SHIFT                                           (12U)
+#define RGX_CR_SOFT_RESET_SPU_SPU12_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU12_EN                                              (0x00001000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU11_SHIFT                                           (11U)
+#define RGX_CR_SOFT_RESET_SPU_SPU11_CLRMSK                                          (0xFFFFF7FFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU11_EN                                              (0x00000800U)
+#define RGX_CR_SOFT_RESET_SPU_SPU10_SHIFT                                           (10U)
+#define RGX_CR_SOFT_RESET_SPU_SPU10_CLRMSK                                          (0xFFFFFBFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU10_EN                                              (0x00000400U)
+#define RGX_CR_SOFT_RESET_SPU_SPU9_SHIFT                                            (9U)
+#define RGX_CR_SOFT_RESET_SPU_SPU9_CLRMSK                                           (0xFFFFFDFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU9_EN                                               (0x00000200U)
+#define RGX_CR_SOFT_RESET_SPU_SPU8_SHIFT                                            (8U)
+#define RGX_CR_SOFT_RESET_SPU_SPU8_CLRMSK                                           (0xFFFFFEFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU8_EN                                               (0x00000100U)
+#define RGX_CR_SOFT_RESET_SPU_SPU7_SHIFT                                            (7U)
+#define RGX_CR_SOFT_RESET_SPU_SPU7_CLRMSK                                           (0xFFFFFF7FU)
+#define RGX_CR_SOFT_RESET_SPU_SPU7_EN                                               (0x00000080U)
+#define RGX_CR_SOFT_RESET_SPU_SPU6_SHIFT                                            (6U)
+#define RGX_CR_SOFT_RESET_SPU_SPU6_CLRMSK                                           (0xFFFFFFBFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU6_EN                                               (0x00000040U)
+#define RGX_CR_SOFT_RESET_SPU_SPU5_SHIFT                                            (5U)
+#define RGX_CR_SOFT_RESET_SPU_SPU5_CLRMSK                                           (0xFFFFFFDFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU5_EN                                               (0x00000020U)
+#define RGX_CR_SOFT_RESET_SPU_SPU4_SHIFT                                            (4U)
+#define RGX_CR_SOFT_RESET_SPU_SPU4_CLRMSK                                           (0xFFFFFFEFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU4_EN                                               (0x00000010U)
+#define RGX_CR_SOFT_RESET_SPU_SPU3_SHIFT                                            (3U)
+#define RGX_CR_SOFT_RESET_SPU_SPU3_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_SOFT_RESET_SPU_SPU3_EN                                               (0x00000008U)
+#define RGX_CR_SOFT_RESET_SPU_SPU2_SHIFT                                            (2U)
+#define RGX_CR_SOFT_RESET_SPU_SPU2_CLRMSK                                           (0xFFFFFFFBU)
+#define RGX_CR_SOFT_RESET_SPU_SPU2_EN                                               (0x00000004U)
+#define RGX_CR_SOFT_RESET_SPU_SPU1_SHIFT                                            (1U)
+#define RGX_CR_SOFT_RESET_SPU_SPU1_CLRMSK                                           (0xFFFFFFFDU)
+#define RGX_CR_SOFT_RESET_SPU_SPU1_EN                                               (0x00000002U)
+#define RGX_CR_SOFT_RESET_SPU_SPU0_SHIFT                                            (0U)
+#define RGX_CR_SOFT_RESET_SPU_SPU0_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SOFT_RESET_SPU_SPU0_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_MULTICORE_EVENT_REDUCE
+*/
+#define RGX_CR_MULTICORE_EVENT_REDUCE                                               (0x2428U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__MASKFULL                         (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MASKFULL                                      (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_FINISHED_SHIFT                            (31U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_FINISHED_CLRMSK                           (0x7FFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_FINISHED_EN                               (0x80000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE1_SHIFT                (31U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE1_CLRMSK               (0x7FFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE1_EN                   (0x80000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_BUFFER_STALL_SHIFT                        (30U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_BUFFER_STALL_CLRMSK                       (0xBFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_BUFFER_STALL_EN                           (0x40000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE0_SHIFT                (30U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE0_CLRMSK               (0xBFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE0_EN                   (0x40000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_SIGNAL_FAILURE_SHIFT                  (29U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_SIGNAL_FAILURE_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_SIGNAL_FAILURE_EN                     (0x20000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE1_SHIFT                  (29U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE1_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE1_EN                     (0x20000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_FINISHED_SHIFT                            (28U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_FINISHED_CLRMSK                           (0xEFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_FINISHED_EN                               (0x10000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE0_SHIFT                  (28U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE0_CLRMSK                 (0xEFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE0_EN                     (0x10000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_BARRIER_HIT_SHIFT                         (27U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_BARRIER_HIT_CLRMSK                        (0xF7FFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_BARRIER_HIT_EN                            (0x08000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT            (26U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK           (0xFBFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TDM_BARRIER_HIT_EN               (0x04000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT          (24U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK         (0xFEFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_CORRECTABLE_EN             (0x01000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_LATENT_SHIFT               (22U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_LATENT_CLRMSK              (0xFFBFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_LATENT_EN                  (0x00400000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE1_SHIFT              (21U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK             (0xFFDFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE1_EN                 (0x00200000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE0_SHIFT              (20U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK             (0xFFEFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE0_EN                 (0x00100000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_WDT_TIMEOUT_SHIFT                             (19U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_WDT_TIMEOUT_CLRMSK                            (0xFFF7FFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_WDT_TIMEOUT_EN                                (0x00080000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT        (19U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK       (0xFFF7FFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__IPP_END_RENDER_SENT_EN           (0x00080000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_CDM_BARRIER_HIT_SHIFT                         (18U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_CDM_BARRIER_HIT_CLRMSK                        (0xFFFBFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_CDM_BARRIER_HIT_EN                            (0x00040000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__WDT_TIMEOUT_SHIFT                (18U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__WDT_TIMEOUT_CLRMSK               (0xFFFBFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__WDT_TIMEOUT_EN                   (0x00040000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT            (17U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK           (0xFFFDFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__CDM_BARRIER_HIT_EN               (0x00020000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_BUFFER_STALL_SHIFT                    (16U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_BUFFER_STALL_CLRMSK                   (0xFFFEFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_BUFFER_STALL_EN                       (0x00010000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT     (16U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK    (0xFFFEFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN        (0x00010000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_USC_TRIGGER_SHIFT                             (15U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_USC_TRIGGER_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_USC_TRIGGER_EN                                (0x00008000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_SHIFT                                (14U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_CLRMSK                               (0xFFFFBFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_EN                                   (0x00004000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_ACK_SHIFT                                (13U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_ACK_CLRMSK                               (0xFFFFDFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_ACK_EN                                   (0x00002000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_REQ_SHIFT                                (12U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_REQ_CLRMSK                               (0xFFFFEFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_REQ_EN                                   (0x00001000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_ABORT_SHIFT                             (11U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_ABORT_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_ABORT_EN                                (0x00000800U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_COMPLETE_SHIFT                          (10U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_COMPLETE_CLRMSK                         (0xFFFFFBFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_COMPLETE_EN                             (0x00000400U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MMU_PAGE_FAULT_SHIFT                          (9U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MMU_PAGE_FAULT_CLRMSK                         (0xFFFFFDFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MMU_PAGE_FAULT_EN                             (0x00000200U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_FRAG_DONE_SHIFT                            (8U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_FRAG_DONE_CLRMSK                           (0xFFFFFEFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_FRAG_DONE_EN                               (0x00000100U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT           (8U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK          (0xFFFFFEFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE1_EN              (0x00000100U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_OUT_OF_MEMORY_SHIFT                        (7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_OUT_OF_MEMORY_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_OUT_OF_MEMORY_EN                           (0x00000080U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT           (7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK          (0xFFFFFF7FU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE0_EN              (0x00000080U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_TERMINATE_SHIFT                            (6U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_TERMINATE_CLRMSK                           (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_TERMINATE_EN                               (0x00000040U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT           (6U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK          (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__PM_OUT_OF_MEMORY_EN              (0x00000040U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_FINISHED_SHIFT                             (5U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_FINISHED_CLRMSK                            (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_FINISHED_EN                                (0x00000020U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TA_TERMINATE_SHIFT               (5U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TA_TERMINATE_CLRMSK              (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TA_TERMINATE_EN                  (0x00000020U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_IPP_END_RENDER_SENT_SHIFT                     (4U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_IPP_END_RENDER_SENT_CLRMSK                    (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_IPP_END_RENDER_SENT_EN                        (0x00000010U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE1_SHIFT                 (4U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE1_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE1_EN                    (0x00000010U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_ISP_END_RENDER_SHIFT                          (3U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_ISP_END_RENDER_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_ISP_END_RENDER_EN                             (0x00000008U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE0_SHIFT                 (3U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE0_CLRMSK                (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE0_EN                    (0x00000008U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_FINISHED_SHIFT                        (2U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_FINISHED_CLRMSK                       (0xFFFFFFFBU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_FINISHED_EN                           (0x00000004U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE1_SHIFT              (2U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE1_CLRMSK             (0xFFFFFFFBU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE1_EN                 (0x00000004U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TE_END_SHIFT                                  (1U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TE_END_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TE_END_EN                                     (0x00000002U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE0_SHIFT              (1U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE0_CLRMSK             (0xFFFFFFFDU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE0_EN                 (0x00000002U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_SHIFT                               (0U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON                                       (0x24B0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_EXECUTE_COUNT_SHIFT                   (8U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_EXECUTE_COUNT_CLRMSK                  (0xC00000FFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_TDM_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON                                            (0x24B8U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                        (30U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                       (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_EXECUTE_COUNT_SHIFT                        (8U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_EXECUTE_COUNT_CLRMSK                       (0xC00000FFU)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_GPU_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_GPU_ENABLE_CLRMSK                          (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON                                        (0x24C0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                    (30U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                   (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_EXECUTE_COUNT_SHIFT                    (8U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_EXECUTE_COUNT_CLRMSK                   (0xC00000FFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_BROADCAST
+*/
+#define RGX_CR_MULTICORE_BROADCAST                                                  (0x24E0U)
+#define RGX_CR_MULTICORE_BROADCAST_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MULTICORE_BROADCAST_PROGRAMMER_MASK_SHIFT                            (8U)
+#define RGX_CR_MULTICORE_BROADCAST_PROGRAMMER_MASK_CLRMSK                           (0xFFFF00FFU)
+#define RGX_CR_MULTICORE_BROADCAST_FIRMWARE_MASK_SHIFT                              (0U)
+#define RGX_CR_MULTICORE_BROADCAST_FIRMWARE_MASK_CLRMSK                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE
+*/
+#define RGX_CR_MULTICORE                                                            (0x24E8U)
+#define RGX_CR_MULTICORE_MASKFULL                                                   (IMG_UINT64_C(0x00000007FFFFFFFF))
+#define RGX_CR_MULTICORE_DOMAIN_GPU_MASK_SHIFT                                      (27U)
+#define RGX_CR_MULTICORE_DOMAIN_GPU_MASK_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF807FFFFFF))
+#define RGX_CR_MULTICORE_PRIMARY_CORE_ID_SHIFT                                      (3U)
+#define RGX_CR_MULTICORE_PRIMARY_CORE_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFF8000007))
+#define RGX_CR_MULTICORE_ID_SHIFT                                                   (0U)
+#define RGX_CR_MULTICORE_ID_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF8))
+
+
+/*
+    Register RGX_CR_MULTICORE_SYSTEM
+*/
+#define RGX_CR_MULTICORE_SYSTEM                                                     (0x24F0U)
+#define RGX_CR_MULTICORE_SYSTEM_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MULTICORE_DOMAIN
+*/
+#define RGX_CR_MULTICORE_DOMAIN                                                     (0x24F8U)
+#define RGX_CR_MULTICORE_DOMAIN_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MMU_PT_PARITY_CLEAR
+*/
+#define RGX_CR_MMU_PT_PARITY_CLEAR                                                  (0x0110U)
+#define RGX_CR_MMU_PT_PARITY_CLEAR_MASKFULL                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_PT_PARITY_CLEAR_CTXTN_SHIFT                                      (0U)
+#define RGX_CR_MMU_PT_PARITY_CLEAR_CTXTN_CLRMSK                                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_MMU_PT_PARITY_STATUS
+*/
+#define RGX_CR_MMU_PT_PARITY_STATUS                                                 (0x0120U)
+#define RGX_CR_MMU_PT_PARITY_STATUS_MASKFULL                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_PT_PARITY_STATUS_CTXTN_SHIFT                                     (0U)
+#define RGX_CR_MMU_PT_PARITY_STATUS_CTXTN_CLRMSK                                    (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_EVENT_ENABLE
+*/
+#define RGX_CR_EVENT_ENABLE                                                         (0x0128U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__MASKFULL                                   (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__MASKFULL                    (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__MASKFULL                    (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_EVENT_ENABLE_MASKFULL                                                (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_SHIFT          (31U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_CLRMSK         (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_EN             (0x80000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_SHIFT           (31U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_CLRMSK          (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_EN              (0x80000000U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_SHIFT           (30U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_CLRMSK          (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_EN              (0x40000000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_SHIFT             (29U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_CLRMSK            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_EN                (0x20000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_SHIFT          (28U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_CLRMSK         (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_EN             (0x10000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_SHIFT             (28U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_CLRMSK            (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_EN                (0x10000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT       (26U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK      (0xFBFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_EN          (0x04000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT     (24U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK    (0xFEFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_EN        (0x01000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_SHIFT          (22U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_CLRMSK         (0xFFBFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_EN             (0x00400000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_SHIFT         (21U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK        (0xFFDFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_EN            (0x00200000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_SHIFT         (20U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK        (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_EN            (0x00100000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT           (19U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_CLRMSK          (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN              (0x00080000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT   (19U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_EN      (0x00080000U)
+#define RGX_CR_EVENT_ENABLE_TDM_CONTEXT_STORE_FINISHED_SHIFT                        (18U)
+#define RGX_CR_EVENT_ENABLE_TDM_CONTEXT_STORE_FINISHED_CLRMSK                       (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_CONTEXT_STORE_FINISHED_EN                           (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_SHIFT       (18U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_EN          (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT           (18U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_CLRMSK          (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_EN              (0x00040000U)
+#define RGX_CR_EVENT_ENABLE_SPFILTER_SIGNAL_UPDATE_SHIFT                            (17U)
+#define RGX_CR_EVENT_ENABLE_SPFILTER_SIGNAL_UPDATE_CLRMSK                           (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE_SPFILTER_SIGNAL_UPDATE_EN                               (0x00020000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT       (17U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_EN          (0x00020000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT (16U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN   (0x00010000U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_ENABLE_FAULT_FW_SHIFT                                          (14U)
+#define RGX_CR_EVENT_ENABLE_FAULT_FW_CLRMSK                                         (0xFFFFBFFFU)
+#define RGX_CR_EVENT_ENABLE_FAULT_FW_EN                                             (0x00004000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_ENABLE_PM_FRAG_DONE_SHIFT                                      (8U)
+#define RGX_CR_EVENT_ENABLE_PM_FRAG_DONE_CLRMSK                                     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE_PM_FRAG_DONE_EN                                         (0x00000100U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT      (8U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_EN         (0x00000100U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT      (7U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK     (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_EN         (0x00000080U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT      (6U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_EN         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_SHIFT          (5U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_CLRMSK         (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_EN             (0x00000020U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__IPP_END_RENDER_SENT_SHIFT                  (4U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__IPP_END_RENDER_SENT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__IPP_END_RENDER_SENT_EN                     (0x00000010U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_SHIFT   (4U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_CLRMSK  (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_EN      (0x00000010U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_SHIFT            (4U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_CLRMSK           (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_EN               (0x00000010U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_RENDER_SHIFT                                    (3U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_RENDER_CLRMSK                                   (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_RENDER_EN                                       (0x00000008U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_SHIFT            (3U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_CLRMSK           (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_EN               (0x00000008U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_SHIFT         (2U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_CLRMSK        (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_EN            (0x00000004U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__TE_END_SHIFT                               (1U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__TE_END_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__TE_END_EN                                  (0x00000002U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_SHIFT                (1U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_CLRMSK               (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_EN                   (0x00000002U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_SHIFT         (1U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_CLRMSK        (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_EN            (0x00000002U)
+#define RGX_CR_EVENT_ENABLE_FAULT_GPU_SHIFT                                         (0U)
+#define RGX_CR_EVENT_ENABLE_FAULT_GPU_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_EVENT_ENABLE_FAULT_GPU_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_STATUS
+*/
+#define RGX_CR_EVENT_STATUS                                                         (0x0130U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__MASKFULL                                   (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__MASKFULL                    (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__MASKFULL                    (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_EVENT_STATUS_MASKFULL                                                (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_SHIFT          (31U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_CLRMSK         (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_EN             (0x80000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_SHIFT           (31U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_CLRMSK          (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_EN              (0x80000000U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_SHIFT           (30U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_CLRMSK          (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_EN              (0x40000000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_SHIFT             (29U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_CLRMSK            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_EN                (0x20000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_SHIFT          (28U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_CLRMSK         (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_EN             (0x10000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_SHIFT             (28U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_CLRMSK            (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_EN                (0x10000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT       (26U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK      (0xFBFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_EN          (0x04000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT     (24U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK    (0xFEFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_EN        (0x01000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_SHIFT          (22U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_CLRMSK         (0xFFBFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_EN             (0x00400000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_SHIFT         (21U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK        (0xFFDFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_EN            (0x00200000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_SHIFT         (20U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK        (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_EN            (0x00100000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT           (19U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_CLRMSK          (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN              (0x00080000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT   (19U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_EN      (0x00080000U)
+#define RGX_CR_EVENT_STATUS_TDM_CONTEXT_STORE_FINISHED_SHIFT                        (18U)
+#define RGX_CR_EVENT_STATUS_TDM_CONTEXT_STORE_FINISHED_CLRMSK                       (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_CONTEXT_STORE_FINISHED_EN                           (0x00040000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_SHIFT       (18U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_EN          (0x00040000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT           (18U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_CLRMSK          (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_EN              (0x00040000U)
+#define RGX_CR_EVENT_STATUS_SPFILTER_SIGNAL_UPDATE_SHIFT                            (17U)
+#define RGX_CR_EVENT_STATUS_SPFILTER_SIGNAL_UPDATE_CLRMSK                           (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS_SPFILTER_SIGNAL_UPDATE_EN                               (0x00020000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT       (17U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_EN          (0x00020000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT (16U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN   (0x00010000U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_STATUS_FAULT_FW_SHIFT                                          (14U)
+#define RGX_CR_EVENT_STATUS_FAULT_FW_CLRMSK                                         (0xFFFFBFFFU)
+#define RGX_CR_EVENT_STATUS_FAULT_FW_EN                                             (0x00004000U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_STATUS_PM_FRAG_DONE_SHIFT                                      (8U)
+#define RGX_CR_EVENT_STATUS_PM_FRAG_DONE_CLRMSK                                     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS_PM_FRAG_DONE_EN                                         (0x00000100U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT      (8U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_EN         (0x00000100U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT      (7U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK     (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_EN         (0x00000080U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT      (6U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_EN         (0x00000040U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_SHIFT          (5U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_CLRMSK         (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_EN             (0x00000020U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__IPP_END_RENDER_SENT_SHIFT                  (4U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__IPP_END_RENDER_SENT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__IPP_END_RENDER_SENT_EN                     (0x00000010U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_SHIFT   (4U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_CLRMSK  (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_EN      (0x00000010U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_SHIFT            (4U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_CLRMSK           (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_EN               (0x00000010U)
+#define RGX_CR_EVENT_STATUS_ISP_END_RENDER_SHIFT                                    (3U)
+#define RGX_CR_EVENT_STATUS_ISP_END_RENDER_CLRMSK                                   (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS_ISP_END_RENDER_EN                                       (0x00000008U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_SHIFT            (3U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_CLRMSK           (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_EN               (0x00000008U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_SHIFT         (2U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_CLRMSK        (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_EN            (0x00000004U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__TE_END_SHIFT                               (1U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__TE_END_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__TE_END_EN                                  (0x00000002U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_SHIFT                (1U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_CLRMSK               (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_EN                   (0x00000002U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_SHIFT         (1U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_CLRMSK        (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_EN            (0x00000002U)
+#define RGX_CR_EVENT_STATUS_FAULT_GPU_SHIFT                                         (0U)
+#define RGX_CR_EVENT_STATUS_FAULT_GPU_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_EVENT_STATUS_FAULT_GPU_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_CLEAR
+*/
+#define RGX_CR_EVENT_CLEAR                                                          (0x0138U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__MASKFULL                                    (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__MASKFULL                     (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__MASKFULL                     (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_EVENT_CLEAR_MASKFULL                                                 (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_SHIFT                                 (31U)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_CLRMSK                                (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_EN                                    (0x80000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_SHIFT           (31U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_CLRMSK          (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_EN              (0x80000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_SHIFT            (31U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_CLRMSK           (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_EN               (0x80000000U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_SHIFT                                   (30U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_CLRMSK                                  (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_EN                                      (0x40000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_SHIFT            (30U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_CLRMSK           (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_EN               (0x40000000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_SHIFT                             (29U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_EN                                (0x20000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_SHIFT              (29U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_CLRMSK             (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_EN                 (0x20000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_SHIFT           (28U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_CLRMSK          (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_EN              (0x10000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_SHIFT              (28U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_CLRMSK             (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_EN                 (0x10000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_SHIFT        (27U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_CLRMSK       (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_EN           (0x08000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_SHIFT        (27U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_CLRMSK       (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_EN           (0x08000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT        (26U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK       (0xFBFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_EN           (0x04000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT      (24U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK     (0xFEFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_EN         (0x01000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_SHIFT           (22U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_CLRMSK          (0xFFBFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_EN              (0x00400000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_SHIFT          (21U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK         (0xFFDFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_EN             (0x00200000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_SHIFT          (20U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK         (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_EN             (0x00100000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT            (19U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_CLRMSK           (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN               (0x00080000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT    (19U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK   (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_EN       (0x00080000U)
+#define RGX_CR_EVENT_CLEAR_TDM_CONTEXT_STORE_FINISHED_SHIFT                         (18U)
+#define RGX_CR_EVENT_CLEAR_TDM_CONTEXT_STORE_FINISHED_CLRMSK                        (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_CONTEXT_STORE_FINISHED_EN                            (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_SHIFT        (18U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_CLRMSK       (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_EN           (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT            (18U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_CLRMSK           (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_EN               (0x00040000U)
+#define RGX_CR_EVENT_CLEAR_SPFILTER_SIGNAL_UPDATE_SHIFT                             (17U)
+#define RGX_CR_EVENT_CLEAR_SPFILTER_SIGNAL_UPDATE_CLRMSK                            (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR_SPFILTER_SIGNAL_UPDATE_EN                                (0x00020000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT        (17U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK       (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_EN           (0x00020000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_SHIFT                               (16U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_EN                                  (0x00010000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT (16U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN    (0x00010000U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_SHIFT                                        (15U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_CLRMSK                                       (0xFFFF7FFFU)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_EN                                           (0x00008000U)
+#define RGX_CR_EVENT_CLEAR_FAULT_FW_SHIFT                                           (14U)
+#define RGX_CR_EVENT_CLEAR_FAULT_FW_CLRMSK                                          (0xFFFFBFFFU)
+#define RGX_CR_EVENT_CLEAR_FAULT_FW_EN                                              (0x00004000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_SHIFT                                           (13U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_EN                                              (0x00002000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_SHIFT                                           (12U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_EN                                              (0x00001000U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_SHIFT                                        (11U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_CLRMSK                                       (0xFFFFF7FFU)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_EN                                           (0x00000800U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_SHIFT                                     (10U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_EN                                        (0x00000400U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_SHIFT                                     (9U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_EN                                        (0x00000200U)
+#define RGX_CR_EVENT_CLEAR_PM_FRAG_DONE_SHIFT                                       (8U)
+#define RGX_CR_EVENT_CLEAR_PM_FRAG_DONE_CLRMSK                                      (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR_PM_FRAG_DONE_EN                                          (0x00000100U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT       (8U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK      (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_EN          (0x00000100U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_SHIFT                                   (7U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_EN                                      (0x00000080U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT       (7U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK      (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_EN          (0x00000080U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_SHIFT                                       (6U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_CLRMSK                                      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_EN                                          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT       (6U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_EN          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_SHIFT                                        (5U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_EN                                           (0x00000020U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_SHIFT           (5U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_CLRMSK          (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_EN              (0x00000020U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_SHIFT                                  (4U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_EN                                     (0x00000010U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__IPP_END_RENDER_SENT_SHIFT                   (4U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__IPP_END_RENDER_SENT_CLRMSK                  (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__IPP_END_RENDER_SENT_EN                      (0x00000010U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_SHIFT    (4U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_CLRMSK   (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_EN       (0x00000010U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_SHIFT             (4U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_CLRMSK            (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_EN                (0x00000010U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_RENDER_SHIFT                                     (3U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_RENDER_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_RENDER_EN                                        (0x00000008U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_SHIFT             (3U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_CLRMSK            (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_EN                (0x00000008U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_SHIFT                                   (2U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_EN                                      (0x00000004U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_SHIFT          (2U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_CLRMSK         (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_EN             (0x00000004U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_SHIFT                                    (1U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_EN                                       (0x00000002U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__TE_END_SHIFT                                (1U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__TE_END_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__TE_END_EN                                   (0x00000002U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_SHIFT                 (1U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_CLRMSK                (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_EN                    (0x00000002U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_SHIFT          (1U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_CLRMSK         (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_EN             (0x00000002U)
+#define RGX_CR_EVENT_CLEAR_FAULT_GPU_SHIFT                                          (0U)
+#define RGX_CR_EVENT_CLEAR_FAULT_GPU_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_EVENT_CLEAR_FAULT_GPU_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_TIMER
+*/
+#define RGX_CR_TIMER                                                                (0x0160U)
+#define RGX_CR_TIMER_MASKFULL                                                       (IMG_UINT64_C(0x8000FFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_SHIFT                                                    (63U)
+#define RGX_CR_TIMER_BIT31_CLRMSK                                                   (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_EN                                                       (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_TIMER_VALUE_SHIFT                                                    (0U)
+#define RGX_CR_TIMER_VALUE_CLRMSK                                                   (IMG_UINT64_C(0xFFFF000000000000))
+
+
+/*
+    Register RGX_CR_FAULT_FW_STATUS
+*/
+#define RGX_CR_FAULT_FW_STATUS                                                      (0x0170U)
+#define RGX_CR_FAULT_FW_STATUS_MASKFULL                                             (IMG_UINT64_C(0x00000000000F007F))
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_CORRECT_SHIFT                              (19U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_CORRECT_CLRMSK                             (0xFFF7FFFFU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_CORRECT_EN                                 (0x00080000U)
+#define RGX_CR_FAULT_FW_STATUS_META_CORRECT_SHIFT                                   (18U)
+#define RGX_CR_FAULT_FW_STATUS_META_CORRECT_CLRMSK                                  (0xFFFBFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_META_CORRECT_EN                                      (0x00040000U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CORRECT_SHIFT                                    (17U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CORRECT_CLRMSK                                   (0xFFFDFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CORRECT_EN                                       (0x00020000U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_SHIFT                                    (16U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_CLRMSK                                   (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_EN                                       (0x00010000U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_DETECT_SHIFT                               (6U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_DETECT_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_DETECT_EN                                  (0x00000040U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_SHIFT                          (5U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_CLRMSK                         (0xFFFFFFDFU)
+#define RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_EN                             (0x00000020U)
+#define RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_SHIFT                                  (4U)
+#define RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_EN                                     (0x00000010U)
+#define RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_SHIFT                                (3U)
+#define RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_EN                                   (0x00000008U)
+#define RGX_CR_FAULT_FW_STATUS_META_DETECT_SHIFT                                    (2U)
+#define RGX_CR_FAULT_FW_STATUS_META_DETECT_CLRMSK                                   (0xFFFFFFFBU)
+#define RGX_CR_FAULT_FW_STATUS_META_DETECT_EN                                       (0x00000004U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_DETECT_SHIFT                                     (1U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_DETECT_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_DETECT_EN                                        (0x00000002U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_DETECT_SHIFT                                     (0U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_DETECT_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_STATUS_MMU_DETECT_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_CLEAR
+*/
+#define RGX_CR_FAULT_FW_CLEAR                                                       (0x0178U)
+#define RGX_CR_FAULT_FW_CLEAR_MASKFULL                                              (IMG_UINT64_C(0x00000000000F007F))
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_CORRECT_SHIFT                               (19U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_CORRECT_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_CORRECT_EN                                  (0x00080000U)
+#define RGX_CR_FAULT_FW_CLEAR_META_CORRECT_SHIFT                                    (18U)
+#define RGX_CR_FAULT_FW_CLEAR_META_CORRECT_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_META_CORRECT_EN                                       (0x00040000U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CORRECT_SHIFT                                     (17U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CORRECT_CLRMSK                                    (0xFFFDFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CORRECT_EN                                        (0x00020000U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_CORRECT_SHIFT                                     (16U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_CORRECT_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_CORRECT_EN                                        (0x00010000U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_DETECT_SHIFT                                (6U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_DETECT_CLRMSK                               (0xFFFFFFBFU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_DETECT_EN                                   (0x00000040U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_PTE_PARITY_DETECT_SHIFT                           (5U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_PTE_PARITY_DETECT_CLRMSK                          (0xFFFFFFDFU)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_PTE_PARITY_DETECT_EN                              (0x00000020U)
+#define RGX_CR_FAULT_FW_CLEAR_PARITY_DETECT_SHIFT                                   (4U)
+#define RGX_CR_FAULT_FW_CLEAR_PARITY_DETECT_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_FAULT_FW_CLEAR_PARITY_DETECT_EN                                      (0x00000010U)
+#define RGX_CR_FAULT_FW_CLEAR_LOCKSTEP_DETECT_SHIFT                                 (3U)
+#define RGX_CR_FAULT_FW_CLEAR_LOCKSTEP_DETECT_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_FAULT_FW_CLEAR_LOCKSTEP_DETECT_EN                                    (0x00000008U)
+#define RGX_CR_FAULT_FW_CLEAR_META_DETECT_SHIFT                                     (2U)
+#define RGX_CR_FAULT_FW_CLEAR_META_DETECT_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FAULT_FW_CLEAR_META_DETECT_EN                                        (0x00000004U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_DETECT_SHIFT                                      (1U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_DETECT_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_DETECT_EN                                         (0x00000002U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_DETECT_SHIFT                                      (0U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_DETECT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_DETECT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_GPU_STATUS
+*/
+#define RGX_CR_FAULT_GPU_STATUS                                                     (0x0180U)
+#define RGX_CR_FAULT_GPU_STATUS_MASKFULL                                            (IMG_UINT64_C(0x000000000001000F))
+#define RGX_CR_FAULT_GPU_STATUS_REST_CORRECT_SHIFT                                  (16U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_CORRECT_CLRMSK                                 (0xFFFEFFFFU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_CORRECT_EN                                     (0x00010000U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_FAULT_SHIFT                             (3U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_FAULT_CLRMSK                            (0xFFFFFFF7U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_FAULT_EN                                (0x00000008U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_PARITY_SHIFT                                   (2U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_PARITY_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_PARITY_EN                                      (0x00000004U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_SHIFT                                   (1U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_EN                                      (0x00000002U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_DETECT_SHIFT                                   (0U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_DETECT_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_DETECT_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_GPU_CLEAR
+*/
+#define RGX_CR_FAULT_GPU_CLEAR                                                      (0x0188U)
+#define RGX_CR_FAULT_GPU_CLEAR_MASKFULL                                             (IMG_UINT64_C(0x000000000001000F))
+#define RGX_CR_FAULT_GPU_CLEAR_REST_CORRECT_SHIFT                                   (16U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_CORRECT_CLRMSK                                  (0xFFFEFFFFU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_CORRECT_EN                                      (0x00010000U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_FAULT_SHIFT                              (3U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_FAULT_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_FAULT_EN                                 (0x00000008U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_PARITY_SHIFT                                    (2U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_PARITY_CLRMSK                                   (0xFFFFFFFBU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_PARITY_EN                                       (0x00000004U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_SHIFT                                    (1U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_EN                                       (0x00000002U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_DETECT_SHIFT                                    (0U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_DETECT_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_DETECT_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS
+*/
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS                                    (0x1220U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MASKFULL                           (IMG_UINT64_C(0x0000003FFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CDM_FAULT_DS_SHIFT                 (37U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CDM_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CDM_FAULT_DS_EN                    (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_FAULT_DS_SHIFT                 (36U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_FAULT_DS_EN                    (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_FAULT_DS_SHIFT                 (35U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_FAULT_DS_EN                    (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_IPP_FAULT_DS_SHIFT                 (34U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_IPP_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_IPP_FAULT_DS_EN                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TILING_FAULT_DS_SHIFT              (33U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TILING_FAULT_DS_CLRMSK             (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TILING_FAULT_DS_EN                 (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_RED_SHIFT                 (32U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_RED_CLRMSK                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_RED_EN                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_PARITY_SHIFT                 (31U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_PARITY_EN                    (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_BUS_PARITY_SHIFT             (30U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_BUS_PARITY_CLRMSK            (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_BUS_PARITY_EN                (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_BUS_PARITY_SHIFT                (29U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_BUS_PARITY_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_BUS_PARITY_EN                   (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GPU_IRQ_CR_PARITY_SHIFT            (28U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GPU_IRQ_CR_PARITY_CLRMSK           (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GPU_IRQ_CR_PARITY_EN               (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_FILTER_PARITY_SHIFT            (27U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_FILTER_PARITY_CLRMSK           (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_FILTER_PARITY_EN               (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_PARITY_SHIFT                 (26U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_PARITY_EN                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_EXTERNAL_AXI_MEM_BUS_PARITY_SHIFT  (25U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_EXTERNAL_AXI_MEM_BUS_PARITY_CLRMSK (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_EXTERNAL_AXI_MEM_BUS_PARITY_EN     (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_PARITY_SHIFT                   (24U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_PARITY_EN                      (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_PARITY_SHIFT                   (23U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_PARITY_EN                      (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DFU_PARITY_SHIFT                   (22U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DFU_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DFU_PARITY_EN                      (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_SHIFT           (21U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_CLRMSK          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_EN              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_AXI_BUS_PARITY_SHIFT               (20U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_AXI_BUS_PARITY_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_AXI_BUS_PARITY_EN                  (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BUS_PARITY_SHIFT                   (19U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BUS_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BUS_PARITY_EN                      (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_AXI_BUS_PARITY_SHIFT    (18U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_AXI_BUS_PARITY_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_AXI_BUS_PARITY_EN       (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_BUS_PARITY_SHIFT        (17U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_BUS_PARITY_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_BUS_PARITY_EN           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_PARITY_SHIFT                    (16U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_PARITY_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_PARITY_EN                       (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_CR_PARITY_SHIFT             (15U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_CR_PARITY_CLRMSK            (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_CR_PARITY_EN                (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_SHIFT                     (14U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_EN                        (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TEXAS_SHIFT                        (13U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TEXAS_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TEXAS_EN                           (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_DMA_SHIFT                       (12U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_DMA_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_DMA_EN                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBSC_SHIFT                         (11U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBSC_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBSC_EN                            (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBHC_SHIFT                         (10U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBHC_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBHC_EN                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_SHIFT                          (9U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_SHIFT                          (8U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_EN                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_SHIFT                          (7U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_EN                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GARTEN_SHIFT                       (6U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GARTEN_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GARTEN_EN                          (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TE3_SHIFT                          (5U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TE3_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TE3_EN                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MMU_SHIFT                          (4U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MMU_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MMU_EN                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_ACE_SHIFT                      (3U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_ACE_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_ACE_EN                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_SHIFT                          (2U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_EN                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_FBCDC_SHIFT                  (1U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_FBCDC_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_FBCDC_EN                     (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_SLC_SHIFT                    (0U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_SLC_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_SLC_EN                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FAULT_JONES_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE                                          (0x1230U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x0000003FFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CDM_FAULT_DS_SHIFT                       (37U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CDM_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CDM_FAULT_DS_EN                          (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_FAULT_DS_SHIFT                       (36U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_FAULT_DS_EN                          (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_FAULT_DS_SHIFT                       (35U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_FAULT_DS_EN                          (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_IPP_FAULT_DS_SHIFT                       (34U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_IPP_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_IPP_FAULT_DS_EN                          (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TILING_FAULT_DS_SHIFT                    (33U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TILING_FAULT_DS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TILING_FAULT_DS_EN                       (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_RED_SHIFT                       (32U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_RED_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_RED_EN                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_PARITY_SHIFT                       (31U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_PARITY_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_PARITY_EN                          (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_BUS_PARITY_SHIFT                   (30U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_BUS_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_BUS_PARITY_EN                      (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_BUS_PARITY_SHIFT                      (29U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_BUS_PARITY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_BUS_PARITY_EN                         (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GPU_IRQ_CR_PARITY_SHIFT                  (28U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GPU_IRQ_CR_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GPU_IRQ_CR_PARITY_EN                     (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_FILTER_PARITY_SHIFT                  (27U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_FILTER_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_FILTER_PARITY_EN                     (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_PARITY_SHIFT                       (26U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_PARITY_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_PARITY_EN                          (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_EXTERNAL_AXI_MEM_BUS_PARITY_SHIFT        (25U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_EXTERNAL_AXI_MEM_BUS_PARITY_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_EXTERNAL_AXI_MEM_BUS_PARITY_EN           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_PARITY_SHIFT                         (24U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_PARITY_EN                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_PARITY_SHIFT                         (23U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_PARITY_EN                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DFU_PARITY_SHIFT                         (22U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DFU_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DFU_PARITY_EN                            (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                 (21U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_AXI_BUS_PARITY_SHIFT                     (20U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_AXI_BUS_PARITY_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_AXI_BUS_PARITY_EN                        (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BUS_PARITY_SHIFT                         (19U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BUS_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BUS_PARITY_EN                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_AXI_BUS_PARITY_SHIFT          (18U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_AXI_BUS_PARITY_CLRMSK         (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_AXI_BUS_PARITY_EN             (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_BUS_PARITY_SHIFT              (17U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_BUS_PARITY_CLRMSK             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_BUS_PARITY_EN                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_PARITY_SHIFT                          (16U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_PARITY_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_PARITY_EN                             (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_CR_PARITY_SHIFT                   (15U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_CR_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_CR_PARITY_EN                      (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_SHIFT                           (14U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_EN                              (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TEXAS_SHIFT                              (13U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TEXAS_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TEXAS_EN                                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_DMA_SHIFT                             (12U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_DMA_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_DMA_EN                                (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBSC_SHIFT                               (11U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBSC_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBSC_EN                                  (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBHC_SHIFT                               (10U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBHC_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBHC_EN                                  (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_SHIFT                                (9U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_EN                                   (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_SHIFT                                (8U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_EN                                   (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_SHIFT                                (7U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_EN                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GARTEN_SHIFT                             (6U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GARTEN_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GARTEN_EN                                (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TE3_SHIFT                                (5U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TE3_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TE3_EN                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MMU_SHIFT                                (4U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MMU_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MMU_EN                                   (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_ACE_SHIFT                            (3U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_ACE_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_ACE_EN                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_SHIFT                                (2U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_FBCDC_SHIFT                        (1U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_FBCDC_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_FBCDC_EN                           (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_SLC_SHIFT                          (0U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_SLC_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_SLC_EN                             (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FAULT_TEXAS_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE                                          (0x1248U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x0000077FFFFFFC0F))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_FAULT_DS_SHIFT                       (42U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_FAULT_DS_EN                          (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CDM_PIPE_FAULT_DS_SHIFT                  (41U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CDM_PIPE_FAULT_DS_CLRMSK                 (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CDM_PIPE_FAULT_DS_EN                     (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                 (40U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                    (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_PARITY_SHIFT                 (38U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_PARITY_EN                    (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_PARITY_SHIFT                      (37U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_PARITY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_PARITY_EN                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_PARITY_SHIFT                  (36U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_PARITY_EN                     (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_PARITY_SHIFT                  (35U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_PARITY_EN                     (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_BUS_PARITY_SHIFT                   (34U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_BUS_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_BUS_PARITY_EN                      (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_BUS_PARITY_SHIFT              (33U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_BUS_PARITY_CLRMSK             (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_BUS_PARITY_EN                 (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_BUS_PARITY_SHIFT                     (32U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_BUS_PARITY_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_BUS_PARITY_EN                        (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CR_PARITY_SHIFT                      (31U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CR_PARITY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CR_PARITY_EN                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_CR_PARITY_SHIFT               (30U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_CR_PARITY_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_CR_PARITY_EN                  (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_CR_PARITY_SHIFT                    (29U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_CR_PARITY_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_CR_PARITY_EN                       (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SWIFT_SHIFT                              (28U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SWIFT_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SWIFT_EN                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MERCER_SHIFT                             (27U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MERCER_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MERCER_EN                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_SHIFT                         (26U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_EN                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_SHIFT                         (25U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_EN                            (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_SHIFT                        (24U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_EN                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_SHIFT                             (23U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_EN                                (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VDM_SHIFT                                (22U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VDM_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VDM_EN                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFA_SHIFT                                (21U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFA_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFA_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_UVB_SHIFT                                (20U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_UVB_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_UVB_EN                                   (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PPP_SHIFT                                (19U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PPP_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PPP_EN                                   (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CLIPPER_SHIFT                            (18U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CLIPPER_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CLIPPER_EN                               (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBG_SHIFT                                (17U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBG_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBG_EN                                   (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBS_SHIFT                                (16U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBS_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBS_EN                                   (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VCE_SHIFT                                (15U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VCE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VCE_EN                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PRIM_BUFFER_SHIFT                        (14U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PRIM_BUFFER_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PRIM_BUFFER_EN                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TESS_SHIFT                               (13U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TESS_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TESS_EN                                  (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IPF_SHIFT                                (12U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IPF_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IPF_EN                                   (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFPU_SHIFT                               (11U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFPU_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFPU_EN                                  (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TPF_SHIFT                                (10U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TPF_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TPF_EN                                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_SHIFT                                (3U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_EN                                   (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHIFT                                (2U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_USC_L2ICACHE_SHIFT                       (1U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_USC_L2ICACHE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_USC_L2ICACHE_EN                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PSB_SHIFT                                (0U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PSB_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PSB_EN                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FAULT_SWIFT_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE                                          (0x1260U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x00000000FFC007FF))
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BUS_PARITY_SHIFT                         (31U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BUS_PARITY_CLRMSK                        (0x7FFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BUS_PARITY_EN                            (0x80000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SWIFT_CR_PARITY_SHIFT                    (30U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SWIFT_CR_PARITY_CLRMSK                   (0xBFFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SWIFT_CR_PARITY_EN                       (0x40000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_CR_PARITY_SHIFT                      (29U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_CR_PARITY_CLRMSK                     (0xDFFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_CR_PARITY_EN                         (0x20000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CR_PARITY_SHIFT                      (28U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CR_PARITY_CLRMSK                     (0xEFFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CR_PARITY_EN                         (0x10000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_LATENT_SHIFT                             (27U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_LATENT_CLRMSK                            (0xF7FFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_LATENT_EN                                (0x08000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ICS_FAULT_SHIFT                          (26U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ICS_FAULT_CLRMSK                         (0xFBFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ICS_FAULT_EN                             (0x04000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_PARITY_SHIFT                         (25U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_PARITY_CLRMSK                        (0xFDFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_PARITY_EN                            (0x02000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_L1_PARITY_SHIFT                      (24U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_L1_PARITY_CLRMSK                     (0xFEFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_L1_PARITY_EN                         (0x01000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_DEMUX_PARITY_SHIFT                       (23U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_DEMUX_PARITY_CLRMSK                      (0xFF7FFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_DEMUX_PARITY_EN                          (0x00800000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                 (22U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK                (0xFFBFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                    (0x00400000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_SHIFT                                (10U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_CLRMSK                               (0xFFFFFBFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_EN                                   (0x00000400U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TF_SHIFT                             (9U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TF_CLRMSK                            (0xFFFFFDFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TF_EN                                (0x00000200U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TAG_SHIFT                            (8U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TAG_CLRMSK                           (0xFFFFFEFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TAG_EN                               (0x00000100U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_MADD_SHIFT                           (7U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_MADD_CLRMSK                          (0xFFFFFF7FU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_MADD_EN                              (0x00000080U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_SHIFT                                (6U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_CLRMSK                               (0xFFFFFFBFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_EN                                   (0x00000040U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_IDEC_SHIFT                           (5U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_IDEC_CLRMSK                          (0xFFFFFFDFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_IDEC_EN                              (0x00000020U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CT_SHIFT                             (4U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CT_CLRMSK                            (0xFFFFFFEFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CT_EN                                (0x00000010U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_DEPTHSORT_SHIFT                      (3U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_DEPTHSORT_CLRMSK                     (0xFFFFFFF7U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_DEPTHSORT_EN                         (0x00000008U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_LUT_SHIFT                    (2U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_LUT_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_LUT_EN                       (0x00000004U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_BUFFER_SHIFT                 (1U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_BUFFER_CLRMSK                (0xFFFFFFFDU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_BUFFER_EN                    (0x00000002U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_ZLS_SHIFT                            (0U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_ZLS_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_ZLS_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS
+*/
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS                                   (0x1268U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MASKFULL                          (IMG_UINT64_C(0x00000000F000001F))
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_BUS_PARITY_SHIFT                  (31U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_BUS_PARITY_CLRMSK                 (0x7FFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_BUS_PARITY_EN                     (0x80000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_SHIFT          (30U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_CLRMSK         (0xBFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_EN             (0x40000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_CR_PARITY_SHIFT                   (29U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_CR_PARITY_CLRMSK                  (0xDFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_CR_PARITY_EN                      (0x20000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MH_PARITY_SHIFT                   (28U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MH_PARITY_CLRMSK                  (0xEFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MH_PARITY_EN                      (0x10000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_LATENT_SHIFT              (4U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_LATENT_CLRMSK             (0xFFFFFFEFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_LATENT_EN                 (0x00000010U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_FAULT_SHIFT               (3U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_FAULT_CLRMSK              (0xFFFFFFF7U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_FAULT_EN                  (0x00000008U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ECC_SHIFT                     (2U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ECC_CLRMSK                    (0xFFFFFFFBU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ECC_EN                        (0x00000004U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_SHIFT                         (1U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_EN                            (0x00000002U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MCU_L0_SHIFT                      (0U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MCU_L0_CLRMSK                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MCU_L0_EN                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_MERCER_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE                                         (0x1278U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MASKFULL                                (IMG_UINT64_C(0x00000000F000001F))
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_BUS_PARITY_SHIFT                        (31U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_BUS_PARITY_CLRMSK                       (0x7FFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_BUS_PARITY_EN                           (0x80000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                (30U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK               (0xBFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                   (0x40000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_CR_PARITY_SHIFT                         (29U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_CR_PARITY_CLRMSK                        (0xDFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_CR_PARITY_EN                            (0x20000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MH_PARITY_SHIFT                         (28U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MH_PARITY_CLRMSK                        (0xEFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MH_PARITY_EN                            (0x10000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_LATENT_SHIFT                    (4U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_LATENT_CLRMSK                   (0xFFFFFFEFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_LATENT_EN                       (0x00000010U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_FAULT_SHIFT                     (3U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_FAULT_CLRMSK                    (0xFFFFFFF7U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_FAULT_EN                        (0x00000008U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ECC_SHIFT                           (2U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ECC_CLRMSK                          (0xFFFFFFFBU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ECC_EN                              (0x00000004U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_SHIFT                               (1U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_EN                                  (0x00000002U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MCU_L0_SHIFT                            (0U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MCU_L0_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MCU_L0_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS
+*/
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS                                      (0x1280U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MASKFULL                             (IMG_UINT64_C(0x00000000FFFC1FFF))
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_BUS_PARITY_SHIFT                     (31U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_BUS_PARITY_CLRMSK                    (0x7FFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_BUS_PARITY_EN                        (0x80000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CR_PARITY_SHIFT                      (30U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CR_PARITY_CLRMSK                     (0xBFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CR_PARITY_EN                         (0x40000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_LATENT_SHIFT                     (29U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_LATENT_CLRMSK                    (0xDFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_LATENT_EN                        (0x20000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_LATENT_SHIFT                     (28U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_LATENT_CLRMSK                    (0xEFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_LATENT_EN                        (0x10000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_LATENT_SHIFT                     (27U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_LATENT_CLRMSK                    (0xF7FFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_LATENT_EN                        (0x08000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_LATENT_SHIFT                     (26U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_LATENT_CLRMSK                    (0xFBFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_LATENT_EN                        (0x04000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_ICS_FAULT_SHIFT                  (25U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_ICS_FAULT_CLRMSK                 (0xFDFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_ICS_FAULT_EN                     (0x02000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_ICS_FAULT_SHIFT                  (24U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_ICS_FAULT_CLRMSK                 (0xFEFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_ICS_FAULT_EN                     (0x01000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_ICS_FAULT_SHIFT                  (23U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_ICS_FAULT_CLRMSK                 (0xFF7FFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_ICS_FAULT_EN                     (0x00800000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_ICS_FAULT_SHIFT                  (22U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_ICS_FAULT_CLRMSK                 (0xFFBFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_ICS_FAULT_EN                     (0x00400000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_PARITY_FAULT_SHIFT               (21U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_PARITY_FAULT_CLRMSK              (0xFFDFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_PARITY_FAULT_EN                  (0x00200000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_PARITY_FAULT_SHIFT                (20U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_PARITY_FAULT_CLRMSK               (0xFFEFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_PARITY_FAULT_EN                   (0x00100000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_FAULT_DS_SHIFT                 (19U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_FAULT_DS_CLRMSK                (0xFFF7FFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_FAULT_DS_EN                    (0x00080000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_FAULT_DS_SHIFT                 (18U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_FAULT_DS_CLRMSK                (0xFFFBFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_FAULT_DS_EN                    (0x00040000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PRED_STORE_SHIFT                     (12U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PRED_STORE_CLRMSK                    (0xFFFFEFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PRED_STORE_EN                        (0x00001000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_SHIFT                            (11U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_CLRMSK                           (0xFFFFF7FFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_EN                               (0x00000800U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_TS_SHIFT                         (10U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_TS_CLRMSK                        (0xFFFFFBFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_TS_EN                            (0x00000400U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_CRB_SHIFT                        (9U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_CRB_CLRMSK                       (0xFFFFFDFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_CRB_EN                           (0x00000200U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_IF_SHIFT                       (8U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_IF_CLRMSK                      (0xFFFFFEFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_IF_EN                          (0x00000100U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_SHIFT                             (7U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_CLRMSK                            (0xFFFFFF7FU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_EN                                (0x00000080U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CEU_SHIFT                            (6U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CEU_CLRMSK                           (0xFFFFFFBFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CEU_EN                               (0x00000040U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_TS_SHIFT                       (5U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_TS_CLRMSK                      (0xFFFFFFDFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_TS_EN                          (0x00000020U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_IRS_SHIFT                            (4U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_IRS_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_IRS_EN                               (0x00000010U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_LMS_SHIFT                            (3U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_LMS_CLRMSK                           (0xFFFFFFF7U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_LMS_EN                               (0x00000008U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SHS_SHIFT                            (2U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SHS_CLRMSK                           (0xFFFFFFFBU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SHS_EN                               (0x00000004U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_COEFF_STORE_SHIFT                    (1U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_COEFF_STORE_CLRMSK                   (0xFFFFFFFDU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_COEFF_STORE_EN                       (0x00000002U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ITR_SHIFT                            (0U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ITR_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ITR_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_USC_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE                                            (0x1290U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFC1FFF))
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_BUS_PARITY_SHIFT                           (31U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_BUS_PARITY_CLRMSK                          (0x7FFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_BUS_PARITY_EN                              (0x80000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CR_PARITY_SHIFT                            (30U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CR_PARITY_CLRMSK                           (0xBFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CR_PARITY_EN                               (0x40000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_LATENT_SHIFT                           (29U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_LATENT_CLRMSK                          (0xDFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_LATENT_EN                              (0x20000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_LATENT_SHIFT                           (28U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_LATENT_CLRMSK                          (0xEFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_LATENT_EN                              (0x10000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_LATENT_SHIFT                           (27U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_LATENT_CLRMSK                          (0xF7FFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_LATENT_EN                              (0x08000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_LATENT_SHIFT                           (26U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_LATENT_CLRMSK                          (0xFBFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_LATENT_EN                              (0x04000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_ICS_FAULT_SHIFT                        (25U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_ICS_FAULT_CLRMSK                       (0xFDFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_ICS_FAULT_EN                           (0x02000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_ICS_FAULT_SHIFT                        (24U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_ICS_FAULT_CLRMSK                       (0xFEFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_ICS_FAULT_EN                           (0x01000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_ICS_FAULT_SHIFT                        (23U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_ICS_FAULT_CLRMSK                       (0xFF7FFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_ICS_FAULT_EN                           (0x00800000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_ICS_FAULT_SHIFT                        (22U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_ICS_FAULT_CLRMSK                       (0xFFBFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_ICS_FAULT_EN                           (0x00400000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_PARITY_FAULT_SHIFT                     (21U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_PARITY_FAULT_CLRMSK                    (0xFFDFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_PARITY_FAULT_EN                        (0x00200000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_PARITY_FAULT_SHIFT                      (20U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_PARITY_FAULT_CLRMSK                     (0xFFEFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_PARITY_FAULT_EN                         (0x00100000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_FAULT_DS_SHIFT                       (19U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_FAULT_DS_CLRMSK                      (0xFFF7FFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_FAULT_DS_EN                          (0x00080000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_FAULT_DS_SHIFT                       (18U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_FAULT_DS_CLRMSK                      (0xFFFBFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_FAULT_DS_EN                          (0x00040000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PRED_STORE_SHIFT                           (12U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PRED_STORE_CLRMSK                          (0xFFFFEFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PRED_STORE_EN                              (0x00001000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_SHIFT                                  (11U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_CLRMSK                                 (0xFFFFF7FFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_EN                                     (0x00000800U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_TS_SHIFT                               (10U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_TS_CLRMSK                              (0xFFFFFBFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_TS_EN                                  (0x00000400U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_CRB_SHIFT                              (9U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_CRB_CLRMSK                             (0xFFFFFDFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_CRB_EN                                 (0x00000200U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_IF_SHIFT                             (8U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_IF_CLRMSK                            (0xFFFFFEFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_IF_EN                                (0x00000100U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_SHIFT                                   (7U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_EN                                      (0x00000080U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CEU_SHIFT                                  (6U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CEU_CLRMSK                                 (0xFFFFFFBFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CEU_EN                                     (0x00000040U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_TS_SHIFT                             (5U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_TS_CLRMSK                            (0xFFFFFFDFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_TS_EN                                (0x00000020U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_IRS_SHIFT                                  (4U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_IRS_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_IRS_EN                                     (0x00000010U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_LMS_SHIFT                                  (3U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_LMS_CLRMSK                                 (0xFFFFFFF7U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_LMS_EN                                     (0x00000008U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SHS_SHIFT                                  (2U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SHS_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SHS_EN                                     (0x00000004U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_COEFF_STORE_SHIFT                          (1U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_COEFF_STORE_CLRMSK                         (0xFFFFFFFDU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_COEFF_STORE_EN                             (0x00000002U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ITR_SHIFT                                  (0U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ITR_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ITR_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_CHEST_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE                                          (0x12A8U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_BUS_PARITY_SHIFT                      (3U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_BUS_PARITY_CLRMSK                     (0xFFFFFFF7U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_BUS_PARITY_EN                         (0x00000008U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_PARITY_SHIFT                          (2U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_PARITY_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_PARITY_EN                             (0x00000004U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_BANK_PARITY_SHIFT                        (1U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_BANK_PARITY_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_BANK_PARITY_EN                           (0x00000002U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_OWDB_PARITY_SHIFT                        (0U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_OWDB_PARITY_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_OWDB_PARITY_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_SLC_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE                                            (0x1328U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MASKFULL                                   (IMG_UINT64_C(0x00000000C00007FF))
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_BUS_PARITY_SHIFT                           (31U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_BUS_PARITY_CLRMSK                          (0x7FFFFFFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_BUS_PARITY_EN                              (0x80000000U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_CR_PARITY_SHIFT                            (30U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_CR_PARITY_CLRMSK                           (0xBFFFFFFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_CR_PARITY_EN                               (0x40000000U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPD_CACHE_SHIFT                       (10U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPD_CACHE_CLRMSK                      (0xFFFFFBFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPD_CACHE_EN                          (0x00000400U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPC_CACHE_SHIFT                       (9U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPC_CACHE_CLRMSK                      (0xFFFFFDFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPC_CACHE_EN                          (0x00000200U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPT_CACHE_SHIFT                       (8U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPT_CACHE_CLRMSK                      (0xFFFFFEFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPT_CACHE_EN                          (0x00000100U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_DATA_STORE_SHIFT                   (7U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_DATA_STORE_CLRMSK                  (0xFFFFFF7FU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_DATA_STORE_EN                      (0x00000080U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_CAM_SHIFT                          (6U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_CAM_CLRMSK                         (0xFFFFFFBFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_CAM_EN                             (0x00000040U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_INST_QUEUE_SHIFT                      (5U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_INST_QUEUE_CLRMSK                     (0xFFFFFFDFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_INST_QUEUE_EN                         (0x00000020U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_DATA_SHIFT                       (4U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_DATA_CLRMSK                      (0xFFFFFFEFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_DATA_EN                          (0x00000010U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_CAM_SHIFT                        (3U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_CAM_CLRMSK                       (0xFFFFFFF7U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_CAM_EN                           (0x00000008U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_SPFILTER_SHIFT                         (2U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_SPFILTER_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_SPFILTER_EN                            (0x00000004U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_CCM_SHIFT                              (1U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_CCM_CLRMSK                             (0xFFFFFFFDU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_CCM_EN                                 (0x00000002U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_IWDB_SHIFT                             (0U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_IWDB_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_IWDB_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_REDUCER_CLEAR
+*/
+#define RGX_CR_FAULT_REDUCER_CLEAR                                                  (0x1330U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_FAULT_REDUCER_CLEAR_SLC_SHIFT                                        (6U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SLC_CLRMSK                                       (0xFFFFFFBFU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SLC_EN                                           (0x00000040U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_CHEST_SHIFT                                      (5U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_CHEST_CLRMSK                                     (0xFFFFFFDFU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_CHEST_EN                                         (0x00000020U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_JONES_SHIFT                                      (4U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_JONES_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_JONES_EN                                         (0x00000010U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_TEXAS_SHIFT                                      (3U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_TEXAS_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_TEXAS_EN                                         (0x00000008U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SWIFT_SHIFT                                      (2U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SWIFT_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SWIFT_EN                                         (0x00000004U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MERCER_SHIFT                                     (1U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MERCER_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MERCER_EN                                        (0x00000002U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_USC_SHIFT                                        (0U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_USC_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_USC_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_FIRMWARE_PROCESSOR_LS
+*/
+#define RGX_CR_FIRMWARE_PROCESSOR_LS                                                (0x01A0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__SELF_TEST_ENABLE_SHIFT                   (2U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__SELF_TEST_ENABLE_CLRMSK                  (0xFFFFFFF3U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_ENABLE_SHIFT                                   (0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_ENABLE_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__ENABLE_SHIFT                             (0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__ENABLE_CLRMSK                            (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS
+*/
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS                                         (0x01A8U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS_MASKFULL                                (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS_SELF_TEST_DONE_SHIFT                    (0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS_SELF_TEST_DONE_CLRMSK                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_JONES_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN                                            (0x8C30U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_MASKFULL                                   (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CHEST_SHIFT                            (22U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CHEST_CLRMSK                           (0xFFBFFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CHEST_EN                               (0x00400000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_RED_SHIFT                             (21U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_RED_CLRMSK                            (0xFFDFFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_RED_EN                                (0x00200000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_PRI_SHIFT                             (20U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_PRI_CLRMSK                            (0xFFEFFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_PRI_EN                                (0x00100000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_FILTER_SHIFT                           (19U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_FILTER_CLRMSK                          (0xFFF7FFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_FILTER_EN                              (0x00080000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PT_PARITY_SHIFT                            (18U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PT_PARITY_CLRMSK                           (0xFFFBFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PT_PARITY_EN                               (0x00040000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_JONES_REGBANK_SHIFT                        (17U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_JONES_REGBANK_CLRMSK                       (0xFFFDFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_JONES_REGBANK_EN                           (0x00020000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SI_AXI2IMG_SHIFT                           (16U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SI_AXI2IMG_CLRMSK                          (0xFFFEFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SI_AXI2IMG_EN                              (0x00010000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                              (15U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                             (0xFFFF7FFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_REDUCER_EN                                 (0x00008000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI_R_SHIFT                                (14U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI_R_CLRMSK                               (0xFFFFBFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI_R_EN                                   (0x00004000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DFU_SHIFT                                  (13U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DFU_CLRMSK                                 (0xFFFFDFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DFU_EN                                     (0x00002000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SAFETY_IRQ_SHIFT                           (12U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SAFETY_IRQ_CLRMSK                          (0xFFFFEFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SAFETY_IRQ_EN                              (0x00001000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GPU_IRQ_SHIFT                              (11U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GPU_IRQ_CLRMSK                             (0xFFFFF7FFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GPU_IRQ_EN                                 (0x00000800U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI2IMG_SHIFT                              (10U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI2IMG_CLRMSK                             (0xFFFFFBFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI2IMG_EN                                 (0x00000400U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SOCIF_SHIFT                                (9U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SOCIF_CLRMSK                               (0xFFFFFDFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SOCIF_EN                                   (0x00000200U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GARTEN_SHIFT                               (8U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GARTEN_CLRMSK                              (0xFFFFFEFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GARTEN_EN                                  (0x00000100U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_TDM_SHIFT                                  (7U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_TDM_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_TDM_EN                                     (0x00000080U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_VERTEX_SHIFT                               (6U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_VERTEX_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_VERTEX_EN                                  (0x00000040U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PIXEL_SHIFT                                (5U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PIXEL_CLRMSK                               (0xFFFFFFDFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PIXEL_EN                                   (0x00000020U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_COMPUTE_SHIFT                              (4U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_COMPUTE_CLRMSK                             (0xFFFFFFEFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_COMPUTE_EN                                 (0x00000010U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_FBCDC_SHIFT                                (3U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_FBCDC_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_FBCDC_EN                                   (0x00000008U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PM_SHIFT                                   (2U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PM_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PM_EN                                      (0x00000004U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_SHIFT                                  (1U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_EN                                     (0x00000002U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_SHIFT                                  (0U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN                                            (0x8C38U)
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN_SPU_SHIFT                                  (0U)
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN_SPU_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_SWIFT_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN                                            (0x8C40U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU31_SWIFT0_SHIFT                         (31U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU31_SWIFT0_CLRMSK                        (0x7FFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU31_SWIFT0_EN                            (0x80000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU30_SWIFT0_SHIFT                         (30U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU30_SWIFT0_CLRMSK                        (0xBFFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU30_SWIFT0_EN                            (0x40000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU29_SWIFT0_SHIFT                         (29U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU29_SWIFT0_CLRMSK                        (0xDFFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU29_SWIFT0_EN                            (0x20000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU28_SWIFT0_SHIFT                         (28U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU28_SWIFT0_CLRMSK                        (0xEFFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU28_SWIFT0_EN                            (0x10000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU27_SWIFT0_SHIFT                         (27U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU27_SWIFT0_CLRMSK                        (0xF7FFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU27_SWIFT0_EN                            (0x08000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU26_SWIFT0_SHIFT                         (26U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU26_SWIFT0_CLRMSK                        (0xFBFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU26_SWIFT0_EN                            (0x04000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU25_SWIFT0_SHIFT                         (25U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU25_SWIFT0_CLRMSK                        (0xFDFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU25_SWIFT0_EN                            (0x02000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU24_SWIFT0_SHIFT                         (24U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU24_SWIFT0_CLRMSK                        (0xFEFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU24_SWIFT0_EN                            (0x01000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU23_SWIFT0_SHIFT                         (23U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU23_SWIFT0_CLRMSK                        (0xFF7FFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU23_SWIFT0_EN                            (0x00800000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU22_SWIFT0_SHIFT                         (22U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU22_SWIFT0_CLRMSK                        (0xFFBFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU22_SWIFT0_EN                            (0x00400000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU21_SWIFT0_SHIFT                         (21U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU21_SWIFT0_CLRMSK                        (0xFFDFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU21_SWIFT0_EN                            (0x00200000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU20_SWIFT0_SHIFT                         (20U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU20_SWIFT0_CLRMSK                        (0xFFEFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU20_SWIFT0_EN                            (0x00100000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU19_SWIFT0_SHIFT                         (19U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU19_SWIFT0_CLRMSK                        (0xFFF7FFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU19_SWIFT0_EN                            (0x00080000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU18_SWIFT0_SHIFT                         (18U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU18_SWIFT0_CLRMSK                        (0xFFFBFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU18_SWIFT0_EN                            (0x00040000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU17_SWIFT0_SHIFT                         (17U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU17_SWIFT0_CLRMSK                        (0xFFFDFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU17_SWIFT0_EN                            (0x00020000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU16_SWIFT0_SHIFT                         (16U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU16_SWIFT0_CLRMSK                        (0xFFFEFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU16_SWIFT0_EN                            (0x00010000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU15_SWIFT0_SHIFT                         (15U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU15_SWIFT0_CLRMSK                        (0xFFFF7FFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU15_SWIFT0_EN                            (0x00008000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU14_SWIFT0_SHIFT                         (14U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU14_SWIFT0_CLRMSK                        (0xFFFFBFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU14_SWIFT0_EN                            (0x00004000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU13_SWIFT0_SHIFT                         (13U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU13_SWIFT0_CLRMSK                        (0xFFFFDFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU13_SWIFT0_EN                            (0x00002000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU12_SWIFT0_SHIFT                         (12U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU12_SWIFT0_CLRMSK                        (0xFFFFEFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU12_SWIFT0_EN                            (0x00001000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU11_SWIFT0_SHIFT                         (11U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU11_SWIFT0_CLRMSK                        (0xFFFFF7FFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU11_SWIFT0_EN                            (0x00000800U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU10_SWIFT0_SHIFT                         (10U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU10_SWIFT0_CLRMSK                        (0xFFFFFBFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU10_SWIFT0_EN                            (0x00000400U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU9_SWIFT0_SHIFT                          (9U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU9_SWIFT0_CLRMSK                         (0xFFFFFDFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU9_SWIFT0_EN                             (0x00000200U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU8_SWIFT0_SHIFT                          (8U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU8_SWIFT0_CLRMSK                         (0xFFFFFEFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU8_SWIFT0_EN                             (0x00000100U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU7_SWIFT0_SHIFT                          (7U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU7_SWIFT0_CLRMSK                         (0xFFFFFF7FU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU7_SWIFT0_EN                             (0x00000080U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU6_SWIFT0_SHIFT                          (6U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU6_SWIFT0_CLRMSK                         (0xFFFFFFBFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU6_SWIFT0_EN                             (0x00000040U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU5_SWIFT0_SHIFT                          (5U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU5_SWIFT0_CLRMSK                         (0xFFFFFFDFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU5_SWIFT0_EN                             (0x00000020U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU4_SWIFT0_SHIFT                          (4U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU4_SWIFT0_CLRMSK                         (0xFFFFFFEFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU4_SWIFT0_EN                             (0x00000010U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU3_SWIFT0_SHIFT                          (3U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU3_SWIFT0_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU3_SWIFT0_EN                             (0x00000008U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU2_SWIFT0_SHIFT                          (2U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU2_SWIFT0_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU2_SWIFT0_EN                             (0x00000004U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU1_SWIFT0_SHIFT                          (1U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU1_SWIFT0_CLRMSK                         (0xFFFFFFFDU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU1_SWIFT0_EN                             (0x00000002U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU0_SWIFT0_SHIFT                          (0U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU0_SWIFT0_CLRMSK                         (0xFFFFFFFEU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU0_SWIFT0_EN                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MERCER_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN                                           (0x8C48U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_MASKFULL                                  (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER2_SHIFT                       (62U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER2_CLRMSK                      (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER2_EN                          (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER1_SHIFT                       (61U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER1_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER1_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER0_SHIFT                       (60U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER0_CLRMSK                      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER0_EN                          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER2_SHIFT                       (59U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER2_CLRMSK                      (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER2_EN                          (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER1_SHIFT                       (58U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER1_EN                          (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER0_SHIFT                       (57U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER0_EN                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER2_SHIFT                       (56U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER2_EN                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER1_SHIFT                       (55U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER1_EN                          (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER0_SHIFT                       (54U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER0_EN                          (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER2_SHIFT                       (53U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER2_EN                          (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER1_SHIFT                       (52U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER1_EN                          (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER0_SHIFT                       (51U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER0_EN                          (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER2_SHIFT                       (50U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER2_EN                          (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER1_SHIFT                       (49U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER1_EN                          (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER0_SHIFT                       (48U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER0_EN                          (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER2_SHIFT                       (47U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER2_EN                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER1_SHIFT                       (46U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER1_EN                          (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER0_SHIFT                       (45U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER0_EN                          (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER2_SHIFT                       (44U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER2_EN                          (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER1_SHIFT                       (43U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER1_EN                          (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER0_SHIFT                       (42U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER0_EN                          (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER2_SHIFT                       (41U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER2_EN                          (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER1_SHIFT                       (40U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER1_EN                          (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER0_SHIFT                       (39U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER0_EN                          (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER2_SHIFT                       (38U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER2_EN                          (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER1_SHIFT                       (37U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER1_EN                          (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER0_SHIFT                       (36U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER0_EN                          (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER2_SHIFT                       (35U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER2_EN                          (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER1_SHIFT                       (34U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER1_EN                          (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER0_SHIFT                       (33U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER0_EN                          (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER2_SHIFT                       (32U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER2_EN                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER1_SHIFT                       (31U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER1_EN                          (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER0_SHIFT                       (30U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER0_EN                          (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER2_SHIFT                        (29U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER2_EN                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER1_SHIFT                        (28U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER1_EN                           (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER0_SHIFT                        (27U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER0_EN                           (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER2_SHIFT                        (26U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER2_EN                           (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER1_SHIFT                        (25U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER1_EN                           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER0_SHIFT                        (24U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER0_EN                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER2_SHIFT                        (23U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER2_EN                           (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER1_SHIFT                        (22U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER1_EN                           (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER0_SHIFT                        (21U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER0_EN                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER2_SHIFT                        (20U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER2_EN                           (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER1_SHIFT                        (19U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER1_EN                           (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER0_SHIFT                        (18U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER0_EN                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER2_SHIFT                        (17U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER2_EN                           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER1_SHIFT                        (16U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER1_EN                           (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER0_SHIFT                        (15U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER0_EN                           (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER2_SHIFT                        (14U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER2_EN                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER1_SHIFT                        (13U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER1_EN                           (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER0_SHIFT                        (12U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER0_EN                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER2_SHIFT                        (11U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER1_SHIFT                        (10U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER0_SHIFT                        (9U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER2_SHIFT                        (8U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER1_SHIFT                        (7U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER0_SHIFT                        (6U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER2_SHIFT                        (5U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER1_SHIFT                        (4U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER0_SHIFT                        (3U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER2_SHIFT                        (2U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER1_SHIFT                        (1U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER0_SHIFT                        (0U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_USC3_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN                                             (0x8C60U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                               (1U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REGBANK_EN                                  (0x00000002U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                               (0U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REDUCER_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_TPU_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN                                              (0x8C68U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_MASKFULL                                     (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                                (1U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REGBANK_EN                                   (0x00000002U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                                (0U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REDUCER_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_ISP_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN                                              (0x8C70U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_MASKFULL                                     (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                                (1U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REGBANK_EN                                   (0x00000002U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                                (0U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REDUCER_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN                                              (0x8C78U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_MASKFULL                                     (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                                (1U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REGBANK_EN                                   (0x00000002U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                                (0U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REDUCER_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE
+*/
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE                                        (0x2458U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_MASKFULL                               (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REDUCER_SHIFT                    (3U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REDUCER_CLRMSK                   (0xFFFFFFF7U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REDUCER_EN                       (0x00000008U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REGBANK_SHIFT                    (2U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REGBANK_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REGBANK_EN                       (0x00000004U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_BANK_SHIFT                             (1U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_BANK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_BANK_EN                                (0x00000002U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_OWDB_SHIFT                             (0U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_OWDB_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_OWDB_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_RAM_STATUS
+*/
+#define RGX_CR_JONES_RAM_STATUS                                                     (0x1148U)
+#define RGX_CR_JONES_RAM_STATUS_MASKFULL                                            (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_JONES_RAM_STATUS_SLC_CHEST_SHIFT                                     (9U)
+#define RGX_CR_JONES_RAM_STATUS_SLC_CHEST_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_JONES_RAM_STATUS_SLC_CHEST_EN                                        (0x00000200U)
+#define RGX_CR_JONES_RAM_STATUS_GARTEN_SHIFT                                        (8U)
+#define RGX_CR_JONES_RAM_STATUS_GARTEN_CLRMSK                                       (0xFFFFFEFFU)
+#define RGX_CR_JONES_RAM_STATUS_GARTEN_EN                                           (0x00000100U)
+#define RGX_CR_JONES_RAM_STATUS_TDM_SHIFT                                           (7U)
+#define RGX_CR_JONES_RAM_STATUS_TDM_CLRMSK                                          (0xFFFFFF7FU)
+#define RGX_CR_JONES_RAM_STATUS_TDM_EN                                              (0x00000080U)
+#define RGX_CR_JONES_RAM_STATUS_VERTEX_SHIFT                                        (6U)
+#define RGX_CR_JONES_RAM_STATUS_VERTEX_CLRMSK                                       (0xFFFFFFBFU)
+#define RGX_CR_JONES_RAM_STATUS_VERTEX_EN                                           (0x00000040U)
+#define RGX_CR_JONES_RAM_STATUS_PIXEL_SHIFT                                         (5U)
+#define RGX_CR_JONES_RAM_STATUS_PIXEL_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_JONES_RAM_STATUS_PIXEL_EN                                            (0x00000020U)
+#define RGX_CR_JONES_RAM_STATUS_COMPUTE_SHIFT                                       (4U)
+#define RGX_CR_JONES_RAM_STATUS_COMPUTE_CLRMSK                                      (0xFFFFFFEFU)
+#define RGX_CR_JONES_RAM_STATUS_COMPUTE_EN                                          (0x00000010U)
+#define RGX_CR_JONES_RAM_STATUS_FBCDC_SHIFT                                         (3U)
+#define RGX_CR_JONES_RAM_STATUS_FBCDC_CLRMSK                                        (0xFFFFFFF7U)
+#define RGX_CR_JONES_RAM_STATUS_FBCDC_EN                                            (0x00000008U)
+#define RGX_CR_JONES_RAM_STATUS_PM_SHIFT                                            (2U)
+#define RGX_CR_JONES_RAM_STATUS_PM_CLRMSK                                           (0xFFFFFFFBU)
+#define RGX_CR_JONES_RAM_STATUS_PM_EN                                               (0x00000004U)
+#define RGX_CR_JONES_RAM_STATUS_BIF_SHIFT                                           (1U)
+#define RGX_CR_JONES_RAM_STATUS_BIF_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_JONES_RAM_STATUS_BIF_EN                                              (0x00000002U)
+#define RGX_CR_JONES_RAM_STATUS_SLC_SHIFT                                           (0U)
+#define RGX_CR_JONES_RAM_STATUS_SLC_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_JONES_RAM_STATUS_SLC_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_RAM_INIT_KICK
+*/
+#define RGX_CR_JONES_RAM_INIT_KICK                                                  (0x1158U)
+#define RGX_CR_JONES_RAM_INIT_KICK_MASKFULL                                         (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_SHIFT                                  (9U)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_EN                                     (0x00000200U)
+#define RGX_CR_JONES_RAM_INIT_KICK_GARTEN_SHIFT                                     (8U)
+#define RGX_CR_JONES_RAM_INIT_KICK_GARTEN_CLRMSK                                    (0xFFFFFEFFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_GARTEN_EN                                        (0x00000100U)
+#define RGX_CR_JONES_RAM_INIT_KICK_TDM_SHIFT                                        (7U)
+#define RGX_CR_JONES_RAM_INIT_KICK_TDM_CLRMSK                                       (0xFFFFFF7FU)
+#define RGX_CR_JONES_RAM_INIT_KICK_TDM_EN                                           (0x00000080U)
+#define RGX_CR_JONES_RAM_INIT_KICK_VERTEX_SHIFT                                     (6U)
+#define RGX_CR_JONES_RAM_INIT_KICK_VERTEX_CLRMSK                                    (0xFFFFFFBFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_VERTEX_EN                                        (0x00000040U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PIXEL_SHIFT                                      (5U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PIXEL_CLRMSK                                     (0xFFFFFFDFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_PIXEL_EN                                         (0x00000020U)
+#define RGX_CR_JONES_RAM_INIT_KICK_COMPUTE_SHIFT                                    (4U)
+#define RGX_CR_JONES_RAM_INIT_KICK_COMPUTE_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_COMPUTE_EN                                       (0x00000010U)
+#define RGX_CR_JONES_RAM_INIT_KICK_FBCDC_SHIFT                                      (3U)
+#define RGX_CR_JONES_RAM_INIT_KICK_FBCDC_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_JONES_RAM_INIT_KICK_FBCDC_EN                                         (0x00000008U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PM_SHIFT                                         (2U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PM_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_CR_JONES_RAM_INIT_KICK_PM_EN                                            (0x00000004U)
+#define RGX_CR_JONES_RAM_INIT_KICK_BIF_SHIFT                                        (1U)
+#define RGX_CR_JONES_RAM_INIT_KICK_BIF_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_JONES_RAM_INIT_KICK_BIF_EN                                           (0x00000002U)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_SHIFT                                        (0U)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_ARCH_CLK_GATE_ENABLE
+*/
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG                            (0x1208U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__MASKFULL                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__SACG_AND_NOT_CXTP_INFRA__MASKFULL              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_SHIFT                (14U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_ON                   (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_AUTO                 (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_SHIFT                (12U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_ON                   (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_AUTO                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_SHIFT                (10U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_ON                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_AUTO                 (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_SHIFT                (8U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_ON                   (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_AUTO                 (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_SHIFT                (6U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_ON                   (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_AUTO                 (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_SHIFT                (4U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_ON                   (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_AUTO                 (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_SHIFT                (2U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_ON                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_AUTO                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_SHIFT                (0U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_ON                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_AUTO                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__SACG_AND_NOT_CXTP_INFRA__CLUSTER_SHIFT         (0U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__SACG_AND_NOT_CXTP_INFRA__CLUSTER_CLRMSK        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_PM_PARTIAL_RENDER_ENABLE
+*/
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE                                             (0x0338U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_SHIFT                                    (0U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5                                (0x04A0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__MASKFULL                      (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__MASKFULL                      (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__NEED_RESUME_SHIFT             (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__NEED_RESUME_CLRMSK            (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__NEED_RESUME_EN                (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__NEED_RESUME_SHIFT             (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__NEED_RESUME_CLRMSK            (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__NEED_RESUME_EN                (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__COMPLETE_SHIFT                (0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__COMPLETE_CLRMSK               (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__COMPLETE_EN                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_PDS0                                                     (0x04A8U)
+#define RGX_CR_CDM_CONTEXT_PDS0_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_SHIFT                                     (36U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_CLRMSK                                    (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSIZE                                 (16U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_SHIFT                                     (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_PDS1                                                     (0x04B0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_MASKFULL                                            (IMG_UINT64_C(0x00000001FFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_SHIFT                                   (32U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_EN                                      (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_SHIFT                                   (31U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_EN                                      (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_SHIFT                                        (30U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_EN                                           (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_SHIFT                                  (23U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC07FFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_ALIGNSHIFT                             (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_ALIGNSIZE                              (2U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_TYPE_SHIFT                                (22U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_TYPE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_TYPE_EN                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_SHIFT                                (11U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFC007FF))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_SHIFT                                     (6U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_ALIGNSHIFT                                (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_ALIGNSIZE                                 (2U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_SHIFT                                     (0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFC0))
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_ALIGNSHIFT                                (2U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_ALIGNSIZE                                 (4U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0                                                (0x04D8U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_SHIFT                                (36U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_CLRMSK                               (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_SHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSIZE                            (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1                                                (0x04E0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_MASKFULL                                       (IMG_UINT64_C(0x00000001BFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_SHIFT                              (32U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_EN                                 (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_SHIFT                              (31U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_EN                                 (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_SHIFT                             (23U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFC07FFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_ALIGNSHIFT                        (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_ALIGNSIZE                         (2U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_TYPE_SHIFT                           (22U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_TYPE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_TYPE_EN                              (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_SHIFT                           (11U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFC007FF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_ALIGNSHIFT                      (6U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_ALIGNSIZE                       (64U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_SHIFT                                (6U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_ALIGNSHIFT                           (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_ALIGNSIZE                            (2U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_SHIFT                                (0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFC0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_ALIGNSHIFT                           (2U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_ALIGNSIZE                            (4U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS
+*/
+#define RGX_CR_CDM_TERMINATE_PDS                                                    (0x04B8U)
+#define RGX_CR_CDM_TERMINATE_PDS_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_SHIFT                                    (36U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_CLRMSK                                   (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSIZE                                (16U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_SHIFT                                    (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSIZE                                (16U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS1
+*/
+#define RGX_CR_CDM_TERMINATE_PDS1                                                   (0x04C0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_MASKFULL                                          (IMG_UINT64_C(0x00000001BFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_SHIFT                                 (32U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_EN                                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_SHIFT                                 (31U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_EN                                    (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_SHIFT                                (23U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFC07FFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_ALIGNSHIFT                           (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_ALIGNSIZE                            (2U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_TYPE_SHIFT                              (22U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_TYPE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_TYPE_EN                                 (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_SHIFT                              (11U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFC007FF))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_ALIGNSHIFT                         (6U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_ALIGNSIZE                          (64U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_SHIFT                                   (6U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_ALIGNSHIFT                              (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_ALIGNSIZE                               (2U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_SHIFT                                   (0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFC0))
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_ALIGNSHIFT                              (2U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_ALIGNSIZE                               (4U)
+
+
+/*
+    Register group: RGX_CR_SCRATCH, with 16 repeats
+*/
+#define RGX_CR_SCRATCH_REPEATCOUNT                                                  (16U)
+/*
+    Register RGX_CR_SCRATCH0
+*/
+#define RGX_CR_SCRATCH0                                                             (0x0800U)
+#define RGX_CR_SCRATCH0_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH0_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH0_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH1
+*/
+#define RGX_CR_SCRATCH1                                                             (0x0808U)
+#define RGX_CR_SCRATCH1_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH1_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH1_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH2
+*/
+#define RGX_CR_SCRATCH2                                                             (0x0810U)
+#define RGX_CR_SCRATCH2_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH2_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH2_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH3
+*/
+#define RGX_CR_SCRATCH3                                                             (0x0818U)
+#define RGX_CR_SCRATCH3_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH3_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH3_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH4
+*/
+#define RGX_CR_SCRATCH4                                                             (0x0820U)
+#define RGX_CR_SCRATCH4_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH4_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH4_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH5
+*/
+#define RGX_CR_SCRATCH5                                                             (0x0828U)
+#define RGX_CR_SCRATCH5_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH5_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH5_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH6
+*/
+#define RGX_CR_SCRATCH6                                                             (0x0830U)
+#define RGX_CR_SCRATCH6_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH6_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH6_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH7
+*/
+#define RGX_CR_SCRATCH7                                                             (0x0838U)
+#define RGX_CR_SCRATCH7_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH7_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH7_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH8
+*/
+#define RGX_CR_SCRATCH8                                                             (0x0840U)
+#define RGX_CR_SCRATCH8_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH8_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH8_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH9
+*/
+#define RGX_CR_SCRATCH9                                                             (0x0848U)
+#define RGX_CR_SCRATCH9_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH9_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH9_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH10
+*/
+#define RGX_CR_SCRATCH10                                                            (0x0850U)
+#define RGX_CR_SCRATCH10_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH10_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH10_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH11
+*/
+#define RGX_CR_SCRATCH11                                                            (0x0858U)
+#define RGX_CR_SCRATCH11_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH11_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH11_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH12
+*/
+#define RGX_CR_SCRATCH12                                                            (0x0860U)
+#define RGX_CR_SCRATCH12_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH12_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH12_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH13
+*/
+#define RGX_CR_SCRATCH13                                                            (0x0868U)
+#define RGX_CR_SCRATCH13_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH13_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH13_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH14
+*/
+#define RGX_CR_SCRATCH14                                                            (0x0870U)
+#define RGX_CR_SCRATCH14_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH14_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH14_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH15
+*/
+#define RGX_CR_SCRATCH15                                                            (0x0878U)
+#define RGX_CR_SCRATCH15_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH15_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH15_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_OS0_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS0_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS0_SCRATCH0
+*/
+#define RGX_CR_OS0_SCRATCH0                                                         (0x0880U)
+#define RGX_CR_OS0_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH1
+*/
+#define RGX_CR_OS0_SCRATCH1                                                         (0x0888U)
+#define RGX_CR_OS0_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH2
+*/
+#define RGX_CR_OS0_SCRATCH2                                                         (0x0890U)
+#define RGX_CR_OS0_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH3
+*/
+#define RGX_CR_OS0_SCRATCH3                                                         (0x0898U)
+#define RGX_CR_OS0_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS1_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS1_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS1_SCRATCH0
+*/
+#define RGX_CR_OS1_SCRATCH0                                                         (0x10880U)
+#define RGX_CR_OS1_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH1
+*/
+#define RGX_CR_OS1_SCRATCH1                                                         (0x10888U)
+#define RGX_CR_OS1_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH2
+*/
+#define RGX_CR_OS1_SCRATCH2                                                         (0x10890U)
+#define RGX_CR_OS1_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH3
+*/
+#define RGX_CR_OS1_SCRATCH3                                                         (0x10898U)
+#define RGX_CR_OS1_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS2_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS2_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS2_SCRATCH0
+*/
+#define RGX_CR_OS2_SCRATCH0                                                         (0x20880U)
+#define RGX_CR_OS2_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH1
+*/
+#define RGX_CR_OS2_SCRATCH1                                                         (0x20888U)
+#define RGX_CR_OS2_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH2
+*/
+#define RGX_CR_OS2_SCRATCH2                                                         (0x20890U)
+#define RGX_CR_OS2_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH3
+*/
+#define RGX_CR_OS2_SCRATCH3                                                         (0x20898U)
+#define RGX_CR_OS2_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS3_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS3_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS3_SCRATCH0
+*/
+#define RGX_CR_OS3_SCRATCH0                                                         (0x30880U)
+#define RGX_CR_OS3_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH1
+*/
+#define RGX_CR_OS3_SCRATCH1                                                         (0x30888U)
+#define RGX_CR_OS3_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH2
+*/
+#define RGX_CR_OS3_SCRATCH2                                                         (0x30890U)
+#define RGX_CR_OS3_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH3
+*/
+#define RGX_CR_OS3_SCRATCH3                                                         (0x30898U)
+#define RGX_CR_OS3_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS4_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS4_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS4_SCRATCH0
+*/
+#define RGX_CR_OS4_SCRATCH0                                                         (0x40880U)
+#define RGX_CR_OS4_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH1
+*/
+#define RGX_CR_OS4_SCRATCH1                                                         (0x40888U)
+#define RGX_CR_OS4_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH2
+*/
+#define RGX_CR_OS4_SCRATCH2                                                         (0x40890U)
+#define RGX_CR_OS4_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH3
+*/
+#define RGX_CR_OS4_SCRATCH3                                                         (0x40898U)
+#define RGX_CR_OS4_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS5_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS5_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS5_SCRATCH0
+*/
+#define RGX_CR_OS5_SCRATCH0                                                         (0x50880U)
+#define RGX_CR_OS5_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH1
+*/
+#define RGX_CR_OS5_SCRATCH1                                                         (0x50888U)
+#define RGX_CR_OS5_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH2
+*/
+#define RGX_CR_OS5_SCRATCH2                                                         (0x50890U)
+#define RGX_CR_OS5_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH3
+*/
+#define RGX_CR_OS5_SCRATCH3                                                         (0x50898U)
+#define RGX_CR_OS5_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS6_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS6_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS6_SCRATCH0
+*/
+#define RGX_CR_OS6_SCRATCH0                                                         (0x60880U)
+#define RGX_CR_OS6_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH1
+*/
+#define RGX_CR_OS6_SCRATCH1                                                         (0x60888U)
+#define RGX_CR_OS6_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH2
+*/
+#define RGX_CR_OS6_SCRATCH2                                                         (0x60890U)
+#define RGX_CR_OS6_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH3
+*/
+#define RGX_CR_OS6_SCRATCH3                                                         (0x60898U)
+#define RGX_CR_OS6_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS7_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS7_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS7_SCRATCH0
+*/
+#define RGX_CR_OS7_SCRATCH0                                                         (0x70880U)
+#define RGX_CR_OS7_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH1
+*/
+#define RGX_CR_OS7_SCRATCH1                                                         (0x70888U)
+#define RGX_CR_OS7_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH2
+*/
+#define RGX_CR_OS7_SCRATCH2                                                         (0x70890U)
+#define RGX_CR_OS7_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH3
+*/
+#define RGX_CR_OS7_SCRATCH3                                                         (0x70898U)
+#define RGX_CR_OS7_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA                        (0xF3000U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAX_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAX_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A00U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAX_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAX_CLRMSK  (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA                        (0x3000U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAX_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAX_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX                                                    (0x0A00U)
+#define RGX_CR_META_SP_MSLVDATAX_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA                        (0xF3040U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAT_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A08U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAT_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAT_CLRMSK  (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA                        (0x3040U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAT_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT                                                    (0x0A08U)
+#define RGX_CR_META_SP_MSLVDATAT_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA                        (0xF3080U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__ADDR_SHIFT            (2U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__ADDR_CLRMSK           (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__AUTOINCR_SHIFT        (1U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__AUTOINCR_CLRMSK       (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__AUTOINCR_EN           (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_SHIFT              (0U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_CLRMSK             (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__ADDR_SHIFT        (2U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__ADDR_CLRMSK       (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__AUTOINCR_SHIFT    (1U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__AUTOINCR_CLRMSK   (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__AUTOINCR_EN       (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__RD_SHIFT          (0U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__RD_CLRMSK         (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__RD_EN             (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA                        (0x3080U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__ADDR_SHIFT            (2U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__ADDR_CLRMSK           (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__AUTOINCR_SHIFT        (1U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__AUTOINCR_CLRMSK       (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__AUTOINCR_EN           (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_SHIFT              (0U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_CLRMSK             (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0                                                    (0x0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_SHIFT                                         (2U)
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_CLRMSK                                        (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_SHIFT                                     (1U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN                                        (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_SHIFT                                           (0U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA                        (0xF30C0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRTHREAD_SHIFT    (30U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRTHREAD_CLRMSK   (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__LOCK2_INTERLOCK_SHIFT (29U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__LOCK2_INTERLOCK_CLRMSK (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__LOCK2_INTERLOCK_EN    (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__ATOMIC_INTERLOCK_SHIFT (28U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__ATOMIC_INTERLOCK_CLRMSK (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__ATOMIC_INTERLOCK_EN   (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_SHIFT    (26U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_CLRMSK   (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN       (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__COREMEM_IDLE_SHIFT    (25U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__COREMEM_IDLE_CLRMSK   (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__COREMEM_IDLE_EN       (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_SHIFT           (24U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_CLRMSK          (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN              (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRID_SHIFT        (21U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRID_CLRMSK       (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERR_SHIFT          (20U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERR_CLRMSK         (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERR_EN             (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__WR_ACTIVE_SHIFT       (18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__WR_ACTIVE_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__WR_ACTIVE_EN          (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__THREAD_SHIFT          (4U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__THREAD_CLRMSK         (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__TRANS_SIZE_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__TRANS_SIZE_CLRMSK     (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__BYTE_ROUND_SHIFT      (0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__BYTE_ROUND_CLRMSK     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRTHREAD_SHIFT (30U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRTHREAD_CLRMSK (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__LOCK2_INTERLOCK_SHIFT (29U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__LOCK2_INTERLOCK_CLRMSK (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__LOCK2_INTERLOCK_EN (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__ATOMIC_INTERLOCK_SHIFT (28U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__ATOMIC_INTERLOCK_CLRMSK (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__ATOMIC_INTERLOCK_EN (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__GBLPORT_IDLE_SHIFT (26U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__GBLPORT_IDLE_CLRMSK (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__GBLPORT_IDLE_EN   (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__COREMEM_IDLE_SHIFT (25U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__COREMEM_IDLE_CLRMSK (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__COREMEM_IDLE_EN   (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__READY_SHIFT       (24U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__READY_CLRMSK      (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__READY_EN          (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRID_SHIFT    (21U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRID_CLRMSK   (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERR_SHIFT      (20U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERR_CLRMSK     (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERR_EN         (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__WR_ACTIVE_SHIFT   (18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__WR_ACTIVE_CLRMSK  (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__WR_ACTIVE_EN      (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__THREAD_SHIFT      (4U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__THREAD_CLRMSK     (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__TRANS_SIZE_SHIFT  (2U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__TRANS_SIZE_CLRMSK (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__BYTE_ROUND_SHIFT  (0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__BYTE_ROUND_CLRMSK (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA                        (0x30C0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRTHREAD_SHIFT    (30U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRTHREAD_CLRMSK   (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__LOCK2_INTERLOCK_SHIFT (29U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__LOCK2_INTERLOCK_CLRMSK (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__LOCK2_INTERLOCK_EN    (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__ATOMIC_INTERLOCK_SHIFT (28U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__ATOMIC_INTERLOCK_CLRMSK (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__ATOMIC_INTERLOCK_EN   (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_SHIFT    (26U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_CLRMSK   (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN       (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__COREMEM_IDLE_SHIFT    (25U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__COREMEM_IDLE_CLRMSK   (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__COREMEM_IDLE_EN       (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_SHIFT           (24U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_CLRMSK          (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN              (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRID_SHIFT        (21U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRID_CLRMSK       (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERR_SHIFT          (20U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERR_CLRMSK         (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERR_EN             (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__WR_ACTIVE_SHIFT       (18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__WR_ACTIVE_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__WR_ACTIVE_EN          (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__THREAD_SHIFT          (4U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__THREAD_CLRMSK         (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__TRANS_SIZE_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__TRANS_SIZE_CLRMSK     (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__BYTE_ROUND_SHIFT      (0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__BYTE_ROUND_CLRMSK     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1                                                    (0x0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1_MASKFULL                                           (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_SHIFT                                 (30U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_CLRMSK                                (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_SHIFT                              (29U)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_EN                                 (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_SHIFT                             (28U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_EN                                (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_SHIFT                                 (26U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN                                    (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_SHIFT                                 (25U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_CLRMSK                                (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_EN                                    (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_SHIFT                                        (24U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_CLRMSK                                       (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_EN                                           (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_SHIFT                                     (21U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_CLRMSK                                    (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_SHIFT                                       (20U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_CLRMSK                                      (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_EN                                          (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_SHIFT                                    (18U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_EN                                       (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_SHIFT                                       (4U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_CLRMSK                                      (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_CLRMSK                                  (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_SHIFT                                   (0U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_CLRMSK                                  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA                     (0xF3280U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__INPUT_SHIFT        (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__INPUT_CLRMSK       (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__OUTPUT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__OUTPUT_CLRMSK      (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA                 (0xF0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL       (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__INPUT_SHIFT    (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__INPUT_CLRMSK   (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__OUTPUT_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__OUTPUT_CLRMSK  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA                     (0x3280U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__INPUT_SHIFT        (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__INPUT_CLRMSK       (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__OUTPUT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__OUTPUT_CLRMSK      (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE                                                 (0x0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_SHIFT                                     (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3400U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3400U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK                                                   (0x0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF3440U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x3440U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI                                                  (0x0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3480U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3480U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK                                                   (0x0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF34C0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x34C0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI                                                  (0x0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3500U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3500U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK                                                   (0x0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF3540U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x3540U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI                                                  (0x0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3580U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3580U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK                                                   (0x0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF35C0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x35C0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI                                                  (0x0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA                          (0xF3600U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__MASKFULL                (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__SOFTRESET_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__SOFTRESET_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__SOFTRESET_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA                      (0xF0AC0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__SOFTRESET_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__SOFTRESET_CLRMSK    (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__SOFTRESET_EN        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA                          (0x3600U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL                (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__SOFTRESET_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__SOFTRESET_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__SOFTRESET_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST                                                      (0x0AC0U)
+#define RGX_CR_META_SP_MSLVRST_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA                    (0xF3640U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT3_SHIFT   (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT3_CLRMSK  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT3_EN      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT2_SHIFT   (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT2_CLRMSK  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT2_EN      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA                (0xF0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL      (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT3_SHIFT (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT3_CLRMSK (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT3_EN  (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT2_SHIFT (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT2_CLRMSK (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT2_EN  (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA                    (0x3640U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT3_SHIFT   (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT3_CLRMSK  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT3_EN      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT2_SHIFT   (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT2_CLRMSK  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT2_EN      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS                                                (0x0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_SHIFT                                (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_EN                                   (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_SHIFT                                (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN                                   (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA                    (0xF3680U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT1_SHIFT      (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT1_CLRMSK     (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT1_EN         (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT0_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT0_CLRMSK     (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT0_EN         (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA                (0xF0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL      (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT1_SHIFT  (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT1_CLRMSK (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT1_EN     (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT0_SHIFT  (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT0_CLRMSK (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT0_EN     (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA                    (0x3680U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT1_SHIFT      (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT1_CLRMSK     (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT1_EN         (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT0_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT0_CLRMSK     (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT0_EN         (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE                                                (0x0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_SHIFT                                   (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_CLRMSK                                  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_EN                                      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_EN                                      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA                     (0xF36C0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MODE_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MODE_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MODE_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA                 (0xF0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MODE_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MODE_CLRMSK    (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MODE_EN        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA                     (0x36C0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MODE_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MODE_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MODE_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL                                                 (0x0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_EN                                         (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4_REPEATCOUNT                (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4                            (0xF8FC0U)
+#define RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED0, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED0_REPEATCOUNT                                     (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00                                                (0x8FC0U)
+#define RGX_CR_FWCORE_DMI_RESERVED00_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01__HOST_SECURITY_GEQ4                            (0xF8FC8U)
+#define RGX_CR_FWCORE_DMI_RESERVED01__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01                                                (0x8FC8U)
+#define RGX_CR_FWCORE_DMI_RESERVED01_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02__HOST_SECURITY_GEQ4                            (0xF8FD0U)
+#define RGX_CR_FWCORE_DMI_RESERVED02__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02                                                (0x8FD0U)
+#define RGX_CR_FWCORE_DMI_RESERVED02_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03__HOST_SECURITY_GEQ4                            (0xF8FD8U)
+#define RGX_CR_FWCORE_DMI_RESERVED03__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03                                                (0x8FD8U)
+#define RGX_CR_FWCORE_DMI_RESERVED03_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4                                 (0xF90C0U)
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4__MASKFULL                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4__VAL_SHIFT                      (0U)
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4__VAL_CLRMSK                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0                                                     (0x90C0U)
+#define RGX_CR_FWCORE_DMI_DATA0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4_REPEATCOUNT                (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4                            (0xF90C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED1, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED1_REPEATCOUNT                                     (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10                                                (0x90C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED10_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11__HOST_SECURITY_GEQ4                            (0xF90D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED11__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11                                                (0x90D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED11_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12__HOST_SECURITY_GEQ4                            (0xF90D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED12__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12                                                (0x90D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED12_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13__HOST_SECURITY_GEQ4                            (0xF90E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED13__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13                                                (0x90E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED13_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14__HOST_SECURITY_GEQ4                            (0xF90E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED14__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14                                                (0x90E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED14_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED15
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED15__HOST_SECURITY_GEQ4                            (0xF90F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED15__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED15
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED15                                                (0x90F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED15_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED16
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED16__HOST_SECURITY_GEQ4                            (0xF90F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED16__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED16
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED16                                                (0x90F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED16_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4                                 (0xF9100U)
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4__MASKFULL                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4__VAL_SHIFT                      (0U)
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4__VAL_CLRMSK                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1                                                     (0x9100U)
+#define RGX_CR_FWCORE_DMI_DATA1_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4_REPEATCOUNT                (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4                            (0xF9108U)
+#define RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED2, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED2_REPEATCOUNT                                     (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20                                                (0x9108U)
+#define RGX_CR_FWCORE_DMI_RESERVED20_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21__HOST_SECURITY_GEQ4                            (0xF9110U)
+#define RGX_CR_FWCORE_DMI_RESERVED21__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21                                                (0x9110U)
+#define RGX_CR_FWCORE_DMI_RESERVED21_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22__HOST_SECURITY_GEQ4                            (0xF9118U)
+#define RGX_CR_FWCORE_DMI_RESERVED22__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22                                                (0x9118U)
+#define RGX_CR_FWCORE_DMI_RESERVED22_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23__HOST_SECURITY_GEQ4                            (0xF9120U)
+#define RGX_CR_FWCORE_DMI_RESERVED23__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23                                                (0x9120U)
+#define RGX_CR_FWCORE_DMI_RESERVED23_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4, with 80 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4_REPEATCOUNT                (80U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4                            (0xF9140U)
+#define RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED3, with 80 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED3_REPEATCOUNT                                     (80U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30                                                (0x9140U)
+#define RGX_CR_FWCORE_DMI_RESERVED30_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31__HOST_SECURITY_GEQ4                            (0xF9148U)
+#define RGX_CR_FWCORE_DMI_RESERVED31__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31                                                (0x9148U)
+#define RGX_CR_FWCORE_DMI_RESERVED31_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED32
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED32__HOST_SECURITY_GEQ4                            (0xF9150U)
+#define RGX_CR_FWCORE_DMI_RESERVED32__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED32
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED32                                                (0x9150U)
+#define RGX_CR_FWCORE_DMI_RESERVED32_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED33
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED33__HOST_SECURITY_GEQ4                            (0xF9158U)
+#define RGX_CR_FWCORE_DMI_RESERVED33__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED33
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED33                                                (0x9158U)
+#define RGX_CR_FWCORE_DMI_RESERVED33_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED34
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED34__HOST_SECURITY_GEQ4                            (0xF9160U)
+#define RGX_CR_FWCORE_DMI_RESERVED34__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED34
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED34                                                (0x9160U)
+#define RGX_CR_FWCORE_DMI_RESERVED34_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED35
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED35__HOST_SECURITY_GEQ4                            (0xF9168U)
+#define RGX_CR_FWCORE_DMI_RESERVED35__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED35
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED35                                                (0x9168U)
+#define RGX_CR_FWCORE_DMI_RESERVED35_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED36
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED36__HOST_SECURITY_GEQ4                            (0xF9170U)
+#define RGX_CR_FWCORE_DMI_RESERVED36__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED36
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED36                                                (0x9170U)
+#define RGX_CR_FWCORE_DMI_RESERVED36_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED37
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED37__HOST_SECURITY_GEQ4                            (0xF9178U)
+#define RGX_CR_FWCORE_DMI_RESERVED37__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED37
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED37                                                (0x9178U)
+#define RGX_CR_FWCORE_DMI_RESERVED37_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED38
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED38__HOST_SECURITY_GEQ4                            (0xF9180U)
+#define RGX_CR_FWCORE_DMI_RESERVED38__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED38
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED38                                                (0x9180U)
+#define RGX_CR_FWCORE_DMI_RESERVED38_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED39
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED39__HOST_SECURITY_GEQ4                            (0xF9188U)
+#define RGX_CR_FWCORE_DMI_RESERVED39__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED39
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED39                                                (0x9188U)
+#define RGX_CR_FWCORE_DMI_RESERVED39_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED310
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED310__HOST_SECURITY_GEQ4                           (0xF9190U)
+#define RGX_CR_FWCORE_DMI_RESERVED310__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED310
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED310                                               (0x9190U)
+#define RGX_CR_FWCORE_DMI_RESERVED310_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED311
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED311__HOST_SECURITY_GEQ4                           (0xF9198U)
+#define RGX_CR_FWCORE_DMI_RESERVED311__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED311
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED311                                               (0x9198U)
+#define RGX_CR_FWCORE_DMI_RESERVED311_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED312
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED312__HOST_SECURITY_GEQ4                           (0xF91A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED312__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED312
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED312                                               (0x91A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED312_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED313
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED313__HOST_SECURITY_GEQ4                           (0xF91A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED313__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED313
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED313                                               (0x91A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED313_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED314
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED314__HOST_SECURITY_GEQ4                           (0xF91B0U)
+#define RGX_CR_FWCORE_DMI_RESERVED314__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED314
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED314                                               (0x91B0U)
+#define RGX_CR_FWCORE_DMI_RESERVED314_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED315
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED315__HOST_SECURITY_GEQ4                           (0xF91B8U)
+#define RGX_CR_FWCORE_DMI_RESERVED315__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED315
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED315                                               (0x91B8U)
+#define RGX_CR_FWCORE_DMI_RESERVED315_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED316
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED316__HOST_SECURITY_GEQ4                           (0xF91C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED316__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED316
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED316                                               (0x91C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED316_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED317
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED317__HOST_SECURITY_GEQ4                           (0xF91C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED317__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED317
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED317                                               (0x91C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED317_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED318
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED318__HOST_SECURITY_GEQ4                           (0xF91D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED318__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED318
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED318                                               (0x91D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED318_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED319
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED319__HOST_SECURITY_GEQ4                           (0xF91D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED319__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED319
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED319                                               (0x91D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED319_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED320
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED320__HOST_SECURITY_GEQ4                           (0xF91E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED320__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED320
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED320                                               (0x91E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED320_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED321
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED321__HOST_SECURITY_GEQ4                           (0xF91E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED321__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED321
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED321                                               (0x91E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED321_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED322
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED322__HOST_SECURITY_GEQ4                           (0xF91F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED322__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED322
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED322                                               (0x91F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED322_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED323
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED323__HOST_SECURITY_GEQ4                           (0xF91F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED323__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED323
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED323                                               (0x91F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED323_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED324
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED324__HOST_SECURITY_GEQ4                           (0xF9200U)
+#define RGX_CR_FWCORE_DMI_RESERVED324__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED324
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED324                                               (0x9200U)
+#define RGX_CR_FWCORE_DMI_RESERVED324_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED325
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED325__HOST_SECURITY_GEQ4                           (0xF9208U)
+#define RGX_CR_FWCORE_DMI_RESERVED325__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED325
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED325                                               (0x9208U)
+#define RGX_CR_FWCORE_DMI_RESERVED325_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED326
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED326__HOST_SECURITY_GEQ4                           (0xF9210U)
+#define RGX_CR_FWCORE_DMI_RESERVED326__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED326
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED326                                               (0x9210U)
+#define RGX_CR_FWCORE_DMI_RESERVED326_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED327
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED327__HOST_SECURITY_GEQ4                           (0xF9218U)
+#define RGX_CR_FWCORE_DMI_RESERVED327__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED327
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED327                                               (0x9218U)
+#define RGX_CR_FWCORE_DMI_RESERVED327_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED328
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED328__HOST_SECURITY_GEQ4                           (0xF9220U)
+#define RGX_CR_FWCORE_DMI_RESERVED328__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED328
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED328                                               (0x9220U)
+#define RGX_CR_FWCORE_DMI_RESERVED328_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED329
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED329__HOST_SECURITY_GEQ4                           (0xF9228U)
+#define RGX_CR_FWCORE_DMI_RESERVED329__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED329
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED329                                               (0x9228U)
+#define RGX_CR_FWCORE_DMI_RESERVED329_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED330
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED330__HOST_SECURITY_GEQ4                           (0xF9230U)
+#define RGX_CR_FWCORE_DMI_RESERVED330__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED330
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED330                                               (0x9230U)
+#define RGX_CR_FWCORE_DMI_RESERVED330_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED331
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED331__HOST_SECURITY_GEQ4                           (0xF9238U)
+#define RGX_CR_FWCORE_DMI_RESERVED331__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED331
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED331                                               (0x9238U)
+#define RGX_CR_FWCORE_DMI_RESERVED331_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4                             (0xF93C0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__MASKFULL                   (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__HALTREQ_SHIFT              (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__HALTREQ_CLRMSK             (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__HALTREQ_EN                 (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__RESUMEREQ_SHIFT            (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__RESUMEREQ_CLRMSK           (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__RESUMEREQ_EN               (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__ACKHAVERESET_SHIFT         (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__ACKHAVERESET_CLRMSK        (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__ACKHAVERESET_EN            (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__NDMRESET_SHIFT             (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__NDMRESET_CLRMSK            (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__NDMRESET_EN                (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_SHIFT             (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_CLRMSK            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL                                                 (0x93C0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_MASKFULL                                        (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_SHIFT                                   (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_CLRMSK                                  (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN                                      (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_SHIFT                                 (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_CLRMSK                                (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN                                    (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_SHIFT                              (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_EN                                 (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_SHIFT                                  (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_EN                                     (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN                                     (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4_REPEATCOUNT                (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED40
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4                            (0xF93C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED4_REPEATCOUNT                                     (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED40
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED40                                                (0x93C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED40_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED41
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED41__HOST_SECURITY_GEQ4                            (0xF93D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED41__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED41
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED41                                                (0x93D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED41_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4                              (0xF9400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__MASKFULL                    (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__IMPEBREAK_SHIFT             (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__IMPEBREAK_CLRMSK            (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__IMPEBREAK_EN                (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHAVERESET_SHIFT          (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHAVERESET_CLRMSK         (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHAVERESET_EN             (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHAVERESET_SHIFT          (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHAVERESET_CLRMSK         (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHAVERESET_EN             (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRESUMEACK_SHIFT          (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRESUMEACK_CLRMSK         (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRESUMEACK_EN             (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRESUMEACK_SHIFT          (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRESUMEACK_CLRMSK         (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRESUMEACK_EN             (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLNONEXISTENT_SHIFT        (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLNONEXISTENT_CLRMSK       (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLNONEXISTENT_EN           (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYNONEXISTENT_SHIFT        (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYNONEXISTENT_CLRMSK       (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYNONEXISTENT_EN           (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLUNAVAIL_SHIFT            (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLUNAVAIL_CLRMSK           (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLUNAVAIL_EN               (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYUNAVAIL_SHIFT            (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYUNAVAIL_CLRMSK           (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYUNAVAIL_EN               (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRUNNING_SHIFT            (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRUNNING_CLRMSK           (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRUNNING_EN               (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRUNNING_SHIFT            (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRUNNING_CLRMSK           (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRUNNING_EN               (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHALTED_SHIFT             (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHALTED_CLRMSK            (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHALTED_EN                (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHALTED_SHIFT             (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHALTED_CLRMSK            (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHALTED_EN                (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHENTICATED_SHIFT         (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHENTICATED_CLRMSK        (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHENTICATED_EN            (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHBUSY_SHIFT              (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHBUSY_CLRMSK             (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHBUSY_EN                 (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__HASRESETHALTREQ_SHIFT       (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__HASRESETHALTREQ_CLRMSK      (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__HASRESETHALTREQ_EN          (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__CONFSTRPTRVALID_SHIFT       (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__CONFSTRPTRVALID_CLRMSK      (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__CONFSTRPTRVALID_EN          (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__VERSION_SHIFT               (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__VERSION_CLRMSK              (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS                                                  (0x9400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_SHIFT                                  (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_EN                                     (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_SHIFT                               (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_EN                                  (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_SHIFT                               (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_EN                                  (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_SHIFT                               (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_CLRMSK                              (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN                                  (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_SHIFT                               (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_EN                                  (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_SHIFT                             (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_EN                                (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_SHIFT                             (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_EN                                (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_SHIFT                                 (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_CLRMSK                                (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_EN                                    (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_SHIFT                                 (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_CLRMSK                                (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_EN                                    (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_SHIFT                                 (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_CLRMSK                                (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_EN                                    (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_SHIFT                                 (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_CLRMSK                                (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_EN                                    (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_SHIFT                                  (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN                                     (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_SHIFT                                  (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_EN                                     (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_SHIFT                              (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_EN                                 (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_SHIFT                                   (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_CLRMSK                                  (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_EN                                      (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_SHIFT                            (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_CLRMSK                           (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_EN                               (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_SHIFT                            (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_EN                               (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_SHIFT                                    (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_CLRMSK                                   (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4                            (0xF9540U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__PROGBUFSIZE_SHIFT         (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__PROGBUFSIZE_CLRMSK        (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__BUSY_SHIFT                (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__BUSY_CLRMSK               (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__BUSY_EN                   (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__CMDERR_SHIFT              (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__CMDERR_CLRMSK             (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__DATACOUNT_SHIFT           (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__DATACOUNT_CLRMSK          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS                                                (0x9540U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_MASKFULL                                       (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_SHIFT                              (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_CLRMSK                             (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_SHIFT                                     (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN                                        (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT                                   (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK                                  (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_SHIFT                                (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_CLRMSK                               (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4                               (0xF9580U)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CMDTYPE_SHIFT                (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CMDTYPE_CLRMSK               (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CONTROL_SHIFT                (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CONTROL_CLRMSK               (0xFF000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND                                                   (0x9580U)
+#define RGX_CR_FWCORE_DMI_COMMAND_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT                                     (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_CLRMSK                                    (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_CLRMSK                                    (0xFF000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4                                  (0xF9DC0U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__MASKFULL                        (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBVERSION_SHIFT                 (29U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBVERSION_CLRMSK                (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSYERROR_SHIFT               (22U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSYERROR_CLRMSK              (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSYERROR_EN                  (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSY_SHIFT                    (21U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSY_CLRMSK                   (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSY_EN                       (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONADDR_SHIFT              (20U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONADDR_CLRMSK             (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONADDR_EN                 (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS_SHIFT                  (17U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS_CLRMSK                 (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBAUTOINCREMENT_SHIFT           (16U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBAUTOINCREMENT_CLRMSK          (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBAUTOINCREMENT_EN              (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONDATA_SHIFT              (15U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONDATA_CLRMSK             (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONDATA_EN                 (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBERROR_SHIFT                   (12U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBERROR_CLRMSK                  (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBASIZE_SHIFT                   (5U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBASIZE_CLRMSK                  (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS128_SHIFT               (4U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS128_CLRMSK              (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS128_EN                  (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS64_SHIFT                (3U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS64_CLRMSK               (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS64_EN                   (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS32_SHIFT                (2U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS32_CLRMSK               (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS32_EN                   (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS16_SHIFT                (1U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS16_CLRMSK               (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS16_EN                   (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS8_SHIFT                 (0U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS8_CLRMSK                (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS8_EN                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS                                                      (0x9DC0U)
+#define RGX_CR_FWCORE_DMI_SBCS_MASKFULL                                             (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_SHIFT                                      (29U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_CLRMSK                                     (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_SHIFT                                    (22U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_CLRMSK                                   (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_EN                                       (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_SHIFT                                         (21U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN                                            (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_SHIFT                                   (20U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN                                      (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT                                       (17U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_CLRMSK                                      (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_SHIFT                                (16U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_CLRMSK                               (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_EN                                   (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_SHIFT                                   (15U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_CLRMSK                                  (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_EN                                      (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT                                        (12U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK                                       (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_SHIFT                                        (5U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_CLRMSK                                       (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_SHIFT                                    (4U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_EN                                       (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_SHIFT                                     (3U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_EN                                        (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_SHIFT                                     (2U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_EN                                        (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_SHIFT                                     (1U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_EN                                        (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_SHIFT                                      (0U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4                            (0xF9E00U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4__ADDRESS_SHIFT             (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4__ADDRESS_CLRMSK            (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0                                                (0x9E00U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0__HOST_SECURITY_GEQ4                               (0xF9EC0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0                                                   (0x9EC0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1__HOST_SECURITY_GEQ4                               (0xF9F00U)
+#define RGX_CR_FWCORE_DMI_SBDATA1__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1                                                   (0x9F00U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2__HOST_SECURITY_GEQ4                               (0xF9F40U)
+#define RGX_CR_FWCORE_DMI_SBDATA2__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2                                                   (0x9F40U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3__HOST_SECURITY_GEQ4                               (0xF9F80U)
+#define RGX_CR_FWCORE_DMI_SBDATA3__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3                                                   (0x9F80U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4                              (0xF9FC0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4__MASKFULL                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4__VAL_SHIFT                   (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4__VAL_CLRMSK                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0                                                  (0x9FC0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE
+*/
+#define RGX_CR_MTS_SCHEDULE                                                         (0x0B00U)
+#define RGX_CR_MTS_SCHEDULE_MASKFULL                                                (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE_HOST_SHIFT                                              (8U)
+#define RGX_CR_MTS_SCHEDULE_HOST_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE_HOST_BG_TIMER                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_HOST_HOST                                               (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_SHIFT                                          (6U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_CLRMSK                                         (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT0                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT1                                           (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT2                                           (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT3                                           (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_SHIFT                                           (5U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_CLRMSK                                          (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_BGCTX                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_INTCTX                                          (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE_TASK_SHIFT                                              (4U)
+#define RGX_CR_MTS_SCHEDULE_TASK_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED                                        (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_TASK_COUNTED                                            (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE_DM_SHIFT                                                (0U)
+#define RGX_CR_MTS_SCHEDULE_DM_CLRMSK                                               (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM0                                                  (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM1                                                  (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM2                                                  (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM3                                                  (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM4                                                  (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM5                                                  (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM6                                                  (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM7                                                  (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM_ALL                                               (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE1
+*/
+#define RGX_CR_MTS_SCHEDULE1                                                        (0x10B00U)
+#define RGX_CR_MTS_SCHEDULE1_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE1_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE1_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE1_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE1_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE2
+*/
+#define RGX_CR_MTS_SCHEDULE2                                                        (0x20B00U)
+#define RGX_CR_MTS_SCHEDULE2_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE2_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE2_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE2_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE2_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE3
+*/
+#define RGX_CR_MTS_SCHEDULE3                                                        (0x30B00U)
+#define RGX_CR_MTS_SCHEDULE3_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE3_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE3_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE3_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE3_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE4
+*/
+#define RGX_CR_MTS_SCHEDULE4                                                        (0x40B00U)
+#define RGX_CR_MTS_SCHEDULE4_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE4_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE4_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE4_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE4_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE5
+*/
+#define RGX_CR_MTS_SCHEDULE5                                                        (0x50B00U)
+#define RGX_CR_MTS_SCHEDULE5_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE5_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE5_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE5_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE5_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE6
+*/
+#define RGX_CR_MTS_SCHEDULE6                                                        (0x60B00U)
+#define RGX_CR_MTS_SCHEDULE6_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE6_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE6_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE6_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE6_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE7
+*/
+#define RGX_CR_MTS_SCHEDULE7                                                        (0x70B00U)
+#define RGX_CR_MTS_SCHEDULE7_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE7_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE7_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE7_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE7_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC                                           (0x0B30U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC                                           (0x0B38U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC                                          (0x0B40U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC                                          (0x0B48U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_GARTEN_WRAPPER_CONFIG
+*/
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG                                            (0x0B50U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S8CPR__MASKFULL                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_MASKFULL                                   (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_SHIFT                   (1U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_CLRMSK                  (0xFFFFFFFDU)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_EN                      (0x00000002U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_SHIFT                            (0U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META                             (0x00000000U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_MTS                              (0x00000001U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S8CPR__META                     (0x00000000U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S8CPR__MTS                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_DM0_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE                                             (0x0B58U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM1_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE                                             (0x0B60U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM2_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE                                             (0x0B68U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM3_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE                                             (0x0B70U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM4_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE                                             (0x0B78U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM5_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE                                             (0x0B80U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM6_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE                                             (0xD138U)
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM7_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE                                             (0xD140U)
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX
+*/
+#define RGX_CR_MTS_INTCTX                                                           (0x0B98U)
+#define RGX_CR_MTS_INTCTX_MASKFULL                                                  (IMG_UINT64_C(0x000000003FC0FFFF))
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_SHIFT                                    (22U)
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_CLRMSK                                   (0xC03FFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_SHIFT                                   (8U)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_CLRMSK                                  (0xFFFF00FFU)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX
+*/
+#define RGX_CR_MTS_BGCTX                                                            (0x0BA0U)
+#define RGX_CR_MTS_BGCTX_MASKFULL                                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE
+*/
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE                                           (0x0BA8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_SHIFT                                 (56U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_CLRMSK                                (IMG_UINT64_C(0x00FFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_SHIFT                                 (48U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_CLRMSK                                (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_SHIFT                                 (40U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_CLRMSK                                (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_SHIFT                                 (32U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_CLRMSK                                (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_SHIFT                                 (24U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_SHIFT                                 (16U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_SHIFT                                 (8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_SHIFT                                 (0U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MTS_GPU_INT_STATUS
+*/
+#define RGX_CR_MTS_GPU_INT_STATUS                                                   (0x0BB0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_SHIFT                                      (0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS0_EVENT_STATUS                                                 (0x0BD0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR                                                  (0x0BE0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS1_EVENT_STATUS                                                 (0x10BD0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR                                                  (0x10BE0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS2_EVENT_STATUS                                                 (0x20BD0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR                                                  (0x20BE0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS3_EVENT_STATUS                                                 (0x30BD0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR                                                  (0x30BE0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS4_EVENT_STATUS                                                 (0x40BD0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR                                                  (0x40BE0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS5_EVENT_STATUS                                                 (0x50BD0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR                                                  (0x50BE0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS6_EVENT_STATUS                                                 (0x60BD0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR                                                  (0x60BE0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS7_EVENT_STATUS                                                 (0x70BD0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR                                                  (0x70BE0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE_ENABLE
+*/
+#define RGX_CR_MTS_SCHEDULE_ENABLE                                                  (0x0BD8U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_SHIFT                                       (0U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_CLRMSK                                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_FWCORE_BOOT
+*/
+#define RGX_CR_FWCORE_BOOT                                                          (0x70C0U)
+#define RGX_CR_FWCORE_BOOT_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_BOOT_MODE_SHIFT                                               (0U)
+#define RGX_CR_FWCORE_BOOT_MODE_CLRMSK                                              (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_BOOT_MODE_EN                                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_BOOT
+*/
+#define RGX_CR_META_BOOT                                                            (0x0BF8U)
+#define RGX_CR_META_BOOT_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_BOOT_MODE_SHIFT                                                 (0U)
+#define RGX_CR_META_BOOT_MODE_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_META_BOOT_MODE_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_GARTEN_SLC
+*/
+#define RGX_CR_GARTEN_SLC                                                           (0x0BB8U)
+#define RGX_CR_GARTEN_SLC_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_SHIFT                                     (0U)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_ISP_RENDER
+*/
+#define RGX_CR_ISP_RENDER                                                           (0x0F08U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MASKFULL                             (IMG_UINT64_C(0x000000000003FFFF))
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__MASKFULL                       (IMG_UINT64_C(0x000000000007FF53))
+#define RGX_CR_ISP_RENDER_MASKFULL                                                  (IMG_UINT64_C(0x000000000003FFF0))
+#define RGX_CR_ISP_RENDER_TILES_PER_ISP_SHIFT                                       (16U)
+#define RGX_CR_ISP_RENDER_TILES_PER_ISP_CLRMSK                                      (0xFFFCFFFFU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__TILES_PER_ISP_SHIFT            (16U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__TILES_PER_ISP_CLRMSK           (0xFFF8FFFFU)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_HIGH_SHIFT                                     (12U)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_HIGH_CLRMSK                                    (0xFFFF0FFFU)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_LOW_SHIFT                                      (8U)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_LOW_CLRMSK                                     (0xFFFFF0FFU)
+#define RGX_CR_ISP_RENDER_TILE_STARVATION_SHIFT                                     (7U)
+#define RGX_CR_ISP_RENDER_TILE_STARVATION_CLRMSK                                    (0xFFFFFF7FU)
+#define RGX_CR_ISP_RENDER_TILE_STARVATION_EN                                        (0x00000080U)
+#define RGX_CR_ISP_RENDER_PROCESS_EMPTY_TILES_SHIFT                                 (6U)
+#define RGX_CR_ISP_RENDER_PROCESS_EMPTY_TILES_CLRMSK                                (0xFFFFFFBFU)
+#define RGX_CR_ISP_RENDER_PROCESS_EMPTY_TILES_EN                                    (0x00000040U)
+#define RGX_CR_ISP_RENDER_RESUME_SHIFT                                              (4U)
+#define RGX_CR_ISP_RENDER_RESUME_CLRMSK                                             (0xFFFFFFCFU)
+#define RGX_CR_ISP_RENDER_RESUME_CONTEXT_NONE                                       (0x00000000U)
+#define RGX_CR_ISP_RENDER_RESUME_CONTEXT_TILE                                       (0x00000010U)
+#define RGX_CR_ISP_RENDER_RESUME_CONTEXT_PBLK                                       (0x00000030U)
+#define RGX_CR_ISP_RENDER_RESUME__CS3DL_EQ4_AND_PIPEDM_EQ0__CONTEXT_NONE            (0x00000000U)
+#define RGX_CR_ISP_RENDER_RESUME__CS3DL_EQ4_AND_PIPEDM_EQ0__CONTEXT_RESUME          (0x00000010U)
+#define RGX_CR_ISP_RENDER_RESUME__IFR_AND_PIPEDM_EQ0__CONTEXT_NONE                  (0x00000000U)
+#define RGX_CR_ISP_RENDER_RESUME__IFR_AND_PIPEDM_EQ0__CONTEXT_TILE                  (0x00000010U)
+#define RGX_CR_ISP_RENDER_RESUME__IFR_AND_PIPEDM_EQ0__CONTEXT_PBLK                  (0x00000030U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_SHIFT                   (4U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_CLRMSK                  (0xFFFFFFEFU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_CONTEXT_NONE            (0x00000000U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_CONTEXT_RESUME          (0x00000010U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_SHIFT                            (2U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_CLRMSK                           (0xFFFFFFF3U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_TL2BR                            (0x00000000U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_TR2BL                            (0x00000004U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_BL2TR                            (0x00000008U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_BR2TL                            (0x0000000CU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_PROTECTED_TILES_SHIFT  (1U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_PROTECTED_TILES_CLRMSK (0xFFFFFFFDU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_PROTECTED_TILES_EN     (0x00000002U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_SHIFT                           (0U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_CLRMSK                          (0xFFFFFFFCU)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_NORM                            (0x00000000U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_FAST_2D                         (0x00000002U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_FAST_SCALE                      (0x00000003U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_UNPROTECTED_TILES_SHIFT (0U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_UNPROTECTED_TILES_CLRMSK (0xFFFFFFFEU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_UNPROTECTED_TILES_EN   (0x00000001U)
+
+
+/*
+    Register RGX_CR_ISP_CTL
+*/
+#define RGX_CR_ISP_CTL                                                              (0x0FB0U)
+#define RGX_CR_ISP_CTL_MASKFULL                                                     (IMG_UINT64_C(0x00000000007BF8FF))
+#define RGX_CR_ISP_CTL_DBUFFER_COUNT_SHIFT                                          (20U)
+#define RGX_CR_ISP_CTL_DBUFFER_COUNT_CLRMSK                                         (0xFF8FFFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_SHIFT                                     (19U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_EN                                        (0x00080000U)
+#define RGX_CR_ISP_CTL_UPFRONT_DEPTH_DISABLE_SHIFT                                  (17U)
+#define RGX_CR_ISP_CTL_UPFRONT_DEPTH_DISABLE_CLRMSK                                 (0xFFFDFFFFU)
+#define RGX_CR_ISP_CTL_UPFRONT_DEPTH_DISABLE_EN                                     (0x00020000U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ONE_SHIFT                                        (16U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ONE_CLRMSK                                       (0xFFFEFFFFU)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ONE_EN                                           (0x00010000U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ZERO_SHIFT                                       (15U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ZERO_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ZERO_EN                                          (0x00008000U)
+#define RGX_CR_ISP_CTL_LINE_SAMPLE_SHIFT                                            (14U)
+#define RGX_CR_ISP_CTL_LINE_SAMPLE_CLRMSK                                           (0xFFFFBFFFU)
+#define RGX_CR_ISP_CTL_LINE_SAMPLE_EN                                               (0x00004000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SHIFT                                             (13U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_CLRMSK                                            (0xFFFFDFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_EN                                                (0x00002000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SINGLE_PIXEL_SHIFT                                (12U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SINGLE_PIXEL_CLRMSK                               (0xFFFFEFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SINGLE_PIXEL_EN                                   (0x00001000U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_SHIFT                                           (11U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_CLRMSK                                          (0xFFFFF7FFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_EN                                              (0x00000800U)
+#define RGX_CR_ISP_CTL_UPASS_START_SHIFT                                            (0U)
+#define RGX_CR_ISP_CTL_UPASS_START_CLRMSK                                           (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_MEM_TILING_CFG, with 8 repeats
+*/
+#define RGX_CR_MEM_TILING_CFG_REPEATCOUNT                                           (8U)
+/*
+    Register RGX_CR_MEM_TILING_CFG0
+*/
+#define RGX_CR_MEM_TILING_CFG0                                                      (0x12D8U)
+#define RGX_CR_MEM_TILING_CFG0_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG0_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG0_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG1
+*/
+#define RGX_CR_MEM_TILING_CFG1                                                      (0x12E0U)
+#define RGX_CR_MEM_TILING_CFG1_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG1_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG1_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG2
+*/
+#define RGX_CR_MEM_TILING_CFG2                                                      (0x12E8U)
+#define RGX_CR_MEM_TILING_CFG2_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG2_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG2_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG3
+*/
+#define RGX_CR_MEM_TILING_CFG3                                                      (0x12F0U)
+#define RGX_CR_MEM_TILING_CFG3_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG3_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG3_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG4
+*/
+#define RGX_CR_MEM_TILING_CFG4                                                      (0x12F8U)
+#define RGX_CR_MEM_TILING_CFG4_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG4_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG4_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG5
+*/
+#define RGX_CR_MEM_TILING_CFG5                                                      (0x1300U)
+#define RGX_CR_MEM_TILING_CFG5_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG5_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG5_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG6
+*/
+#define RGX_CR_MEM_TILING_CFG6                                                      (0x1308U)
+#define RGX_CR_MEM_TILING_CFG6_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG6_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG6_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG7
+*/
+#define RGX_CR_MEM_TILING_CFG7                                                      (0x1310U)
+#define RGX_CR_MEM_TILING_CFG7_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG7_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG7_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_USC_TIMER
+*/
+#define RGX_CR_USC_TIMER                                                            (0x46C8U)
+#define RGX_CR_USC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_USC_TIMER_CNT_SHIFT                                                  (0U)
+#define RGX_CR_USC_TIMER_CNT_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_USC_TIMER_CNT
+*/
+#define RGX_CR_USC_TIMER_CNT                                                        (0x46D0U)
+#define RGX_CR_USC_TIMER_CNT_MASKFULL                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_USC_TIMER_CNT_RESET_SHIFT                                            (0U)
+#define RGX_CR_USC_TIMER_CNT_RESET_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_USC_TIMER_CNT_RESET_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_PAPSAG_FAULT_INJECT
+*/
+#define RGX_CR_PAPSAG_FAULT_INJECT                                                  (0x4D00U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_SHIFT                                      (0U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_CLRMSK                                     (0xFFFFFFFCU)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_DISABLE                                    (0x00000000U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_STUCK0                                     (0x00000001U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_STUCK1                                     (0x00000002U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_RESERVED                                   (0x00000003U)
+
+
+/*
+    Register RGX_CR_PAPSAG_FAULT_INJECT_SAFETY
+*/
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY                                           (0x4D08U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_MASKFULL                                  (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_SHIFT                               (0U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_CLRMSK                              (0xFFFFFFFCU)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_DISABLE                             (0x00000000U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_STUCK0                              (0x00000001U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_STUCK1                              (0x00000002U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_RESERVED                            (0x00000003U)
+
+
+/*
+    Register RGX_CR_TE_CHECKSUM
+*/
+#define RGX_CR_TE_CHECKSUM                                                          (0x5110U)
+#define RGX_CR_TE_CHECKSUM_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_CHECKSUM_VALUE_SHIFT                                              (0U)
+#define RGX_CR_TE_CHECKSUM_VALUE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVB_CHECKSUM
+*/
+#define RGX_CR_USC_UVB_CHECKSUM                                                     (0x5118U)
+#define RGX_CR_USC_UVB_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_TMA_CHECKSUM
+*/
+#define RGX_CR_TE_TMA_CHECKSUM                                                      (0x5128U)
+#define RGX_CR_TE_TMA_CHECKSUM_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_TMA_CHECKSUM_VALUE_SHIFT                                          (0U)
+#define RGX_CR_TE_TMA_CHECKSUM_VALUE_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_CDM_PDS_CHECKSUM
+*/
+#define RGX_CR_CDM_PDS_CHECKSUM                                                     (0x5130U)
+#define RGX_CR_CDM_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CDM_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_CDM_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_CHECKSUM
+*/
+#define RGX_CR_VCE_CHECKSUM                                                         (0x5030U)
+#define RGX_CR_VCE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_VCE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_PDS_CHECKSUM
+*/
+#define RGX_CR_ISP_PDS_CHECKSUM                                                     (0x5038U)
+#define RGX_CR_ISP_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_TPF_CHECKSUM
+*/
+#define RGX_CR_ISP_TPF_CHECKSUM                                                     (0x5040U)
+#define RGX_CR_ISP_TPF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_CHECKSUM
+*/
+#define RGX_CR_TFPU_CHECKSUM                                                        (0x5048U)
+#define RGX_CR_TFPU_CHECKSUM_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_CHECKSUM_VALUE_SHIFT                                            (0U)
+#define RGX_CR_TFPU_CHECKSUM_VALUE_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_ZLS_CHECKSUM
+*/
+#define RGX_CR_ZLS_CHECKSUM                                                         (0x5050U)
+#define RGX_CR_ZLS_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ZLS_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_ZLS_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM_3D
+*/
+#define RGX_CR_PBE_CHECKSUM_3D__PC2                                                 (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_3D__PC2__MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_3D__PC2__VALUE_SHIFT                                    (0U)
+#define RGX_CR_PBE_CHECKSUM_3D__PC2__VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM
+*/
+#define RGX_CR_PBE_CHECKSUM                                                         (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_PBE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PDS_DOUTM_STM_CHECKSUM
+*/
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM                                               (0x5060U)
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM_MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM_VALUE_SHIFT                                   (0U)
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM_VALUE_CLRMSK                                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_IFPU_ISP_CHECKSUM
+*/
+#define RGX_CR_IFPU_ISP_CHECKSUM                                                    (0x5068U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_CLIP_CHECKSUM
+*/
+#define RGX_CR_PPP_CLIP_CHECKSUM                                                    (0x5120U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_PRIM_CHECKSUM
+*/
+#define RGX_CR_VCE_PRIM_CHECKSUM                                                    (0x5140U)
+#define RGX_CR_VCE_PRIM_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_PRIM_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_VCE_PRIM_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_TDM_PDS_CHECKSUM
+*/
+#define RGX_CR_TDM_PDS_CHECKSUM                                                     (0x5148U)
+#define RGX_CR_TDM_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TDM_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_TDM_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM_2D
+*/
+#define RGX_CR_PBE_CHECKSUM_2D                                                      (0x5158U)
+#define RGX_CR_PBE_CHECKSUM_2D_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_2D_VALUE_SHIFT                                          (0U)
+#define RGX_CR_PBE_CHECKSUM_2D_VALUE_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_GEOM
+*/
+#define RGX_CR_PERF_PHASE_GEOM                                                      (0x6008U)
+#define RGX_CR_PERF_PHASE_GEOM__PIPEDM_EQ0__MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_GEOM_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_GEOM_COUNT_SHIFT                                          (0U)
+#define RGX_CR_PERF_PHASE_GEOM_COUNT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_FRAG
+*/
+#define RGX_CR_PERF_PHASE_FRAG                                                      (0x6010U)
+#define RGX_CR_PERF_PHASE_FRAG__PIPEDM_EQ0__MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_FRAG_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_FRAG_COUNT_SHIFT                                          (0U)
+#define RGX_CR_PERF_PHASE_FRAG_COUNT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_COMP
+*/
+#define RGX_CR_PERF_PHASE_COMP                                                      (0x6018U)
+#define RGX_CR_PERF_PHASE_COMP__PIPEDM_EQ0__MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_COMP_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_COMP_COUNT_SHIFT                                          (0U)
+#define RGX_CR_PERF_PHASE_COMP_COUNT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_GEOM_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL                                                (0x6020U)
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_FRAG_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL                                                (0x6028U)
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_COMP_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL                                                (0x6030U)
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL                                        (0x6038U)
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL_COUNT_SHIFT                            (0U)
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL_COUNT_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_2D
+*/
+#define RGX_CR_PERF_PHASE_2D                                                        (0x6050U)
+#define RGX_CR_PERF_PHASE_2D__PIPEDM_EQ0__MASKFULL                                  (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_2D_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_2D_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_PHASE_2D_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_2D_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_2D_TOTAL                                                  (0x6058U)
+#define RGX_CR_PERF_CYCLE_2D_TOTAL_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_2D_TOTAL_COUNT_SHIFT                                      (0U)
+#define RGX_CR_PERF_CYCLE_2D_TOTAL_COUNT_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_READ_STALL
+*/
+#define RGX_CR_PERF_SLC0_READ_STALL                                                 (0x60B8U)
+#define RGX_CR_PERF_SLC0_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC0_WRITE_STALL                                                (0x60C0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_READ_STALL
+*/
+#define RGX_CR_PERF_SLC1_READ_STALL                                                 (0x60E0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC1_WRITE_STALL                                                (0x60E8U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_READ_STALL
+*/
+#define RGX_CR_PERF_SLC2_READ_STALL                                                 (0x6158U)
+#define RGX_CR_PERF_SLC2_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC2_WRITE_STALL                                                (0x6160U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_READ_STALL
+*/
+#define RGX_CR_PERF_SLC3_READ_STALL                                                 (0x6180U)
+#define RGX_CR_PERF_SLC3_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC3_WRITE_STALL                                                (0x6188U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL                                       (0x6408U)
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL_COUNT_SHIFT                           (0U)
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL_COUNT_CLRMSK                          (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4                        (0xF4000U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register group: RGX_CR_FWCORE_ADDR_REMAP_CONFIG, with 16 repeats
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG_REPEATCOUNT                                 (16U)
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0                                            (0x7020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4                        (0xF4008U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1                                            (0x7028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4                        (0xF4010U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2                                            (0x7030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4                        (0xF4018U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3                                            (0x7038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4                        (0xF4020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4                                            (0x7040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4                        (0xF4028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5                                            (0x7048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4                        (0xF4030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6                                            (0x7050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4                        (0xF4038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7                                            (0x7058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4                        (0xF4040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8                                            (0x7060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4                        (0xF4048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9                                            (0x7068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4                       (0xF4050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10                                           (0x7070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4                       (0xF4058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11                                           (0x7078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4                       (0xF4060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12                                           (0x7080U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4                       (0xF4068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13                                           (0x7088U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4                       (0xF4070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14                                           (0x7090U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4                       (0xF4078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15                                           (0x7098U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_ENABLE
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE                                            (0x70A0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_MASKFULL                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_EVENT_EN_SHIFT                             (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_EVENT_EN_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_EVENT_EN_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT                                             (0x70A8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_SHIFT                            (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_RESET_VECTOR
+*/
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR                                          (0x70B0U)
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR_MASKFULL                                 (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR_ADDR_SHIFT                               (1U)
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR_ADDR_CLRMSK                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_VECTOR
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR                                            (0x70B8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR_ADDR_SHIFT                                 (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR_ADDR_CLRMSK                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_IDLE
+*/
+#define RGX_CR_JONES_IDLE                                                           (0x8328U)
+#define RGX_CR_JONES_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x00000000000FFEFF))
+#define RGX_CR_JONES_IDLE_SI_AXI2IMG_SHIFT                                          (19U)
+#define RGX_CR_JONES_IDLE_SI_AXI2IMG_CLRMSK                                         (0xFFF7FFFFU)
+#define RGX_CR_JONES_IDLE_SI_AXI2IMG_EN                                             (0x00080000U)
+#define RGX_CR_JONES_IDLE_ASC_SHIFT                                                 (18U)
+#define RGX_CR_JONES_IDLE_ASC_CLRMSK                                                (0xFFFBFFFFU)
+#define RGX_CR_JONES_IDLE_ASC_EN                                                    (0x00040000U)
+#define RGX_CR_JONES_IDLE_RCE_SHIFT                                                 (17U)
+#define RGX_CR_JONES_IDLE_RCE_CLRMSK                                                (0xFFFDFFFFU)
+#define RGX_CR_JONES_IDLE_RCE_EN                                                    (0x00020000U)
+#define RGX_CR_JONES_IDLE_AXI2IMG_SHIFT                                             (16U)
+#define RGX_CR_JONES_IDLE_AXI2IMG_CLRMSK                                            (0xFFFEFFFFU)
+#define RGX_CR_JONES_IDLE_AXI2IMG_EN                                                (0x00010000U)
+#define RGX_CR_JONES_IDLE_SLC_SHIFT                                                 (15U)
+#define RGX_CR_JONES_IDLE_SLC_CLRMSK                                                (0xFFFF7FFFU)
+#define RGX_CR_JONES_IDLE_SLC_EN                                                    (0x00008000U)
+#define RGX_CR_JONES_IDLE_TDM_SHIFT                                                 (14U)
+#define RGX_CR_JONES_IDLE_TDM_CLRMSK                                                (0xFFFFBFFFU)
+#define RGX_CR_JONES_IDLE_TDM_EN                                                    (0x00004000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_SHIFT                                          (13U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_EN                                             (0x00002000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_SHIFT                                              (12U)
+#define RGX_CR_JONES_IDLE_FB_CDC_CLRMSK                                             (0xFFFFEFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_EN                                                 (0x00001000U)
+#define RGX_CR_JONES_IDLE_MMU_SHIFT                                                 (11U)
+#define RGX_CR_JONES_IDLE_MMU_CLRMSK                                                (0xFFFFF7FFU)
+#define RGX_CR_JONES_IDLE_MMU_EN                                                    (0x00000800U)
+#define RGX_CR_JONES_IDLE_DFU_SHIFT                                                 (10U)
+#define RGX_CR_JONES_IDLE_DFU_CLRMSK                                                (0xFFFFFBFFU)
+#define RGX_CR_JONES_IDLE_DFU_EN                                                    (0x00000400U)
+#define RGX_CR_JONES_IDLE_GARTEN_SHIFT                                              (9U)
+#define RGX_CR_JONES_IDLE_GARTEN_CLRMSK                                             (0xFFFFFDFFU)
+#define RGX_CR_JONES_IDLE_GARTEN_EN                                                 (0x00000200U)
+#define RGX_CR_JONES_IDLE_SOCIF_SHIFT                                               (7U)
+#define RGX_CR_JONES_IDLE_SOCIF_CLRMSK                                              (0xFFFFFF7FU)
+#define RGX_CR_JONES_IDLE_SOCIF_EN                                                  (0x00000080U)
+#define RGX_CR_JONES_IDLE_TILING_SHIFT                                              (6U)
+#define RGX_CR_JONES_IDLE_TILING_CLRMSK                                             (0xFFFFFFBFU)
+#define RGX_CR_JONES_IDLE_TILING_EN                                                 (0x00000040U)
+#define RGX_CR_JONES_IDLE_IPP_SHIFT                                                 (5U)
+#define RGX_CR_JONES_IDLE_IPP_CLRMSK                                                (0xFFFFFFDFU)
+#define RGX_CR_JONES_IDLE_IPP_EN                                                    (0x00000020U)
+#define RGX_CR_JONES_IDLE_USC_GMUTEX_SHIFT                                          (4U)
+#define RGX_CR_JONES_IDLE_USC_GMUTEX_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_JONES_IDLE_USC_GMUTEX_EN                                             (0x00000010U)
+#define RGX_CR_JONES_IDLE_PM_SHIFT                                                  (3U)
+#define RGX_CR_JONES_IDLE_PM_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_JONES_IDLE_PM_EN                                                     (0x00000008U)
+#define RGX_CR_JONES_IDLE_CDM_SHIFT                                                 (2U)
+#define RGX_CR_JONES_IDLE_CDM_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_JONES_IDLE_CDM_EN                                                    (0x00000004U)
+#define RGX_CR_JONES_IDLE_DCE_SHIFT                                                 (1U)
+#define RGX_CR_JONES_IDLE_DCE_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_JONES_IDLE_DCE_EN                                                    (0x00000002U)
+#define RGX_CR_JONES_IDLE_BIF_SHIFT                                                 (0U)
+#define RGX_CR_JONES_IDLE_BIF_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_JONES_IDLE_BIF_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_SYS_BUS_SECURE
+*/
+#define RGX_CR_SYS_BUS_SECURE                                                       (0xA100U)
+#define RGX_CR_SYS_BUS_SECURE__SBSR__MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_MASKFULL                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_SHIFT                                          (0U)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6            (0xF0000U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__MASKFULL  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_CLRMSK (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4 (0xF0000U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__MASKFULL (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_SHIFT (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_CLRMSK (0xFFFFFFE0U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6            (0xF0000U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6__MASKFULL  (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6__ID_CLRMSK (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT                                            (0xE140U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__MASKFULL (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_EQ6__MASKFULL  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_MASKFULL                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_SHIFT                                   (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_CLRMSK                                  (0xFFFFFF00U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_SHIFT (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_CLRMSK (0xFFFFFFE0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_SHIFT (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_CLRMSK (0xFFFFFFE0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_EQ6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_EQ6__ID_CLRMSK (0xFFFFFFC0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_CLRMSK (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1                                 (0xF0008U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__MASKFULL                       (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_SHIFT                  (28U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_CLRMSK                 (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN                     (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT                (0U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK               (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT           (12U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSIZE            (4096U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU                         (0xF0008U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__MASKFULL               (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__INVALID_SHIFT          (28U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__INVALID_CLRMSK         (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__INVALID_EN             (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_SHIFT        (0U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_CLRMSK       (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_ALIGNSHIFT   (12U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_ALIGNSIZE    (4096U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING__VPU                                               (0x1E010U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__MASKFULL                                     (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__INVALID_SHIFT                                (28U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__INVALID_CLRMSK                               (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__INVALID_EN                                   (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_SHIFT                              (0U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_CLRMSK                             (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_ALIGNSHIFT                         (12U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_ALIGNSIZE                          (4096U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING                                                    (0xE148U)
+#define RGX_CR_MMU_CBASE_MAPPING_MASKFULL                                           (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING_INVALID_SHIFT                                      (28U)
+#define RGX_CR_MMU_CBASE_MAPPING_INVALID_CLRMSK                                     (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING_INVALID_EN                                         (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT                                    (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK                                   (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT                               (12U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS1
+*/
+#define RGX_CR_MMU_FAULT_STATUS1                                                    (0xE150U)
+#define RGX_CR_MMU_FAULT_STATUS1_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_LEVEL_SHIFT                                        (62U)
+#define RGX_CR_MMU_FAULT_STATUS1_LEVEL_CLRMSK                                       (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_REQ_ID_SHIFT                                       (56U)
+#define RGX_CR_MMU_FAULT_STATUS1_REQ_ID_CLRMSK                                      (IMG_UINT64_C(0xC0FFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT                                      (48U)
+#define RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK                                     (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT                                      (4U)
+#define RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK                                     (IMG_UINT64_C(0xFFFF00000000000F))
+#define RGX_CR_MMU_FAULT_STATUS1_RNW_SHIFT                                          (3U)
+#define RGX_CR_MMU_FAULT_STATUS1_RNW_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS1_RNW_EN                                             (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT                                         (1U)
+#define RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS1_FAULT_SHIFT                                        (0U)
+#define RGX_CR_MMU_FAULT_STATUS1_FAULT_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS1_FAULT_EN                                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS2
+*/
+#define RGX_CR_MMU_FAULT_STATUS2                                                    (0xE158U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__MASKFULL                               (IMG_UINT64_C(0x00000000003FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS2_MASKFULL                                           (IMG_UINT64_C(0x000000003FFF07FF))
+#define RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_SHIFT                                    (29U)
+#define RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_CLRMSK                                   (0xDFFFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_EN                                       (0x20000000U)
+#define RGX_CR_MMU_FAULT_STATUS2_CLEANUNIQUE_SHIFT                                  (28U)
+#define RGX_CR_MMU_FAULT_STATUS2_CLEANUNIQUE_CLRMSK                                 (0xEFFFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_CLEANUNIQUE_EN                                     (0x10000000U)
+#define RGX_CR_MMU_FAULT_STATUS2_BANK_SHIFT                                         (24U)
+#define RGX_CR_MMU_FAULT_STATUS2_BANK_CLRMSK                                        (0xF0FFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_SHIFT                        (21U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_CLRMSK                       (0xFFDFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_EN                           (0x00200000U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_SHIFT                        (20U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_CLRMSK                       (0xFFEFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_EN                           (0x00100000U)
+#define RGX_CR_MMU_FAULT_STATUS2_TLB_ENTRY_SHIFT                                    (16U)
+#define RGX_CR_MMU_FAULT_STATUS2_TLB_ENTRY_CLRMSK                                   (0xFF00FFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_SHIFT                           (12U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_CLRMSK                          (0xFFF00FFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_SHIFT                                    (10U)
+#define RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_EN                                       (0x00000400U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BANK_SHIFT                             (8U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BANK_CLRMSK                            (0xFFFFF0FFU)
+#define RGX_CR_MMU_FAULT_STATUS2_BIF_ID_SHIFT                                       (0U)
+#define RGX_CR_MMU_FAULT_STATUS2_BIF_ID_CLRMSK                                      (0xFFFFFC00U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__ACTIVE_ID_SHIFT                        (0U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__ACTIVE_ID_CLRMSK                       (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS_META                                                (0xE160U)
+#define RGX_CR_MMU_FAULT_STATUS_META_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT                                    (62U)
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK                                   (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT                                   (56U)
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK                                  (IMG_UINT64_C(0xC0FFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT                                  (48U)
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK                                 (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT                                  (4U)
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK                                 (IMG_UINT64_C(0xFFFF00000000000F))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_SHIFT                                      (3U)
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_EN                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT                                     (1U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_SHIFT                                    (0U)
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN                                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS2_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS2_META                                               (0xE198U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__MASKFULL                          (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_MMU_FAULT_STATUS2_META_MASKFULL                                      (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_MMU_FAULT_STATUS2_META_WRITEBACK_SHIFT                               (13U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_WRITEBACK_CLRMSK                              (0xFFFFDFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META_WRITEBACK_EN                                  (0x00002000U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_CLEANUNIQUE_SHIFT                             (12U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_CLEANUNIQUE_CLRMSK                            (0xFFFFEFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META_CLEANUNIQUE_EN                                (0x00001000U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__WRITEBACK_SHIFT                   (12U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__WRITEBACK_CLRMSK                  (0xFFFFEFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__WRITEBACK_EN                      (0x00001000U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_BANK_SHIFT                                    (8U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_BANK_CLRMSK                                   (0xFFFFF0FFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META_TLB_ENTRY_SHIFT                               (0U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_TLB_ENTRY_CLRMSK                              (0xFFFFFF00U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__ACTIVE_ID_SHIFT                   (0U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__ACTIVE_ID_CLRMSK                  (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_PM
+*/
+#define RGX_CR_MMU_FAULT_STATUS_PM                                                  (0xE130U)
+#define RGX_CR_MMU_FAULT_STATUS_PM__PM_RECYCLE__MASKFULL                            (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_MASKFULL                                         (IMG_UINT64_C(0x0000000007FFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_DM_SHIFT                                         (24U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_DM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM__PM_RECYCLE__DM_SHIFT                            (24U)
+#define RGX_CR_MMU_FAULT_STATUS_PM__PM_RECYCLE__DM_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFF0FFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_RNW_SHIFT                                        (23U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_RNW_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_RNW_EN                                           (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_MMU_FAULT_STATUS_PM_ADDRESS_SHIFT                                    (3U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFF800007))
+#define RGX_CR_MMU_FAULT_STATUS_PM_LEVEL_SHIFT                                      (1U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_LEVEL_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_PM_FAULT_SHIFT                                      (0U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_FAULT_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_PM_FAULT_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_STATUS
+*/
+#define RGX_CR_MMU_STATUS__VPU                                                      (0x10288U)
+#define RGX_CR_MMU_STATUS__VPU__MASKFULL                                            (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__MMU_STALLED_SHIFT                                   (40U)
+#define RGX_CR_MMU_STATUS__VPU__MMU_STALLED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__MMU_STALLED_EN                                      (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MMU_STATUS__VPU__PM_WRITES_SHIFT                                     (38U)
+#define RGX_CR_MMU_STATUS__VPU__PM_WRITES_CLRMSK                                    (IMG_UINT64_C(0xFFFFFF3FFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__PM_READS_SHIFT                                      (36U)
+#define RGX_CR_MMU_STATUS__VPU__PM_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__PC_READS_SHIFT                                      (24U)
+#define RGX_CR_MMU_STATUS__VPU__PC_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__PD_READS_SHIFT                                      (12U)
+#define RGX_CR_MMU_STATUS__VPU__PD_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_MMU_STATUS__VPU__PT_READS_SHIFT                                      (0U)
+#define RGX_CR_MMU_STATUS__VPU__PT_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_MMU_STATUS
+*/
+#define RGX_CR_MMU_STATUS                                                           (0xE170U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MASKFULL                                      (IMG_UINT64_C(0x000003FFFFFFFFFF))
+#define RGX_CR_MMU_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MMU_STALLED_SHIFT                             (41U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MMU_STALLED_CLRMSK                            (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MMU_STALLED_EN                                (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MMU_STATUS_MMU_STALLED_SHIFT                                         (40U)
+#define RGX_CR_MMU_STATUS_MMU_STALLED_CLRMSK                                        (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MMU_STATUS_MMU_STALLED_EN                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_WRITES_SHIFT                               (39U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_WRITES_CLRMSK                              (IMG_UINT64_C(0xFFFFFE7FFFFFFFFF))
+#define RGX_CR_MMU_STATUS_PM_WRITES_SHIFT                                           (38U)
+#define RGX_CR_MMU_STATUS_PM_WRITES_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF3FFFFFFFFF))
+#define RGX_CR_MMU_STATUS_PM_READS_SHIFT                                            (36U)
+#define RGX_CR_MMU_STATUS_PM_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_READS_SHIFT                                (36U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_READS_CLRMSK                               (IMG_UINT64_C(0xFFFFFF8FFFFFFFFF))
+#define RGX_CR_MMU_STATUS_PC_READS_SHIFT                                            (24U)
+#define RGX_CR_MMU_STATUS_PC_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_MMU_STATUS_PD_READS_SHIFT                                            (12U)
+#define RGX_CR_MMU_STATUS_PD_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_MMU_STATUS_PT_READS_SHIFT                                            (0U)
+#define RGX_CR_MMU_STATUS_PT_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_MMU_ENTRY_STATUS__VPU                                                (0x1E028U)
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__MASKFULL                                      (IMG_UINT64_C(0x000000FFFFFF80FF))
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__ADDRESS_SHIFT                                 (15U)
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__ADDRESS_CLRMSK                                (IMG_UINT64_C(0xFFFFFF0000007FFF))
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__CONTEXT_ID_SHIFT                              (0U)
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__CONTEXT_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_MMU_ENTRY_STATUS                                                     (0xE178U)
+#define RGX_CR_MMU_ENTRY_STATUS_MASKFULL                                            (IMG_UINT64_C(0x000000FFFFFF80FF))
+#define RGX_CR_MMU_ENTRY_STATUS_ADDRESS_SHIFT                                       (15U)
+#define RGX_CR_MMU_ENTRY_STATUS_ADDRESS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000007FFF))
+#define RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_SHIFT                                    (0U)
+#define RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MMU_ENTRY
+*/
+#define RGX_CR_MMU_ENTRY__VPU                                                       (0x1E030U)
+#define RGX_CR_MMU_ENTRY__VPU__MASKFULL                                             (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_MMU_ENTRY__VPU__ENABLE_SHIFT                                         (1U)
+#define RGX_CR_MMU_ENTRY__VPU__ENABLE_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_MMU_ENTRY__VPU__ENABLE_EN                                            (0x00000002U)
+#define RGX_CR_MMU_ENTRY__VPU__PENDING_SHIFT                                        (0U)
+#define RGX_CR_MMU_ENTRY__VPU__PENDING_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_MMU_ENTRY__VPU__PENDING_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_MMU_ENTRY
+*/
+#define RGX_CR_MMU_ENTRY                                                            (0xE180U)
+#define RGX_CR_MMU_ENTRY_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_MMU_ENTRY_ENABLE_SHIFT                                               (1U)
+#define RGX_CR_MMU_ENTRY_ENABLE_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_MMU_ENTRY_ENABLE_EN                                                  (0x00000002U)
+#define RGX_CR_MMU_ENTRY_PENDING_SHIFT                                              (0U)
+#define RGX_CR_MMU_ENTRY_PENDING_CLRMSK                                             (0xFFFFFFFEU)
+#define RGX_CR_MMU_ENTRY_PENDING_EN                                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_ONE
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE                                              (0xE350U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_MASKFULL                                     (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT                              (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_CLRMSK                             (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT                               (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_CLRMSK                              (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT                          (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSIZE                           (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT                              (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT                         (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSIZE                          (2097152U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_TWO
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO                                              (0xE358U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_MASKFULL                                     (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_PAGE_SIZE_SHIFT                              (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_PAGE_SIZE_CLRMSK                             (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_SHIFT                               (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_CLRMSK                              (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_ALIGNSHIFT                          (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_ALIGNSIZE                           (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_SHIFT                              (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_ALIGNSHIFT                         (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_ALIGNSIZE                          (2097152U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_THREE
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE                                            (0xE360U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_MASKFULL                                   (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_PAGE_SIZE_SHIFT                            (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_PAGE_SIZE_CLRMSK                           (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_SHIFT                             (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_ALIGNSHIFT                        (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_ALIGNSIZE                         (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_SHIFT                            (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_ALIGNSHIFT                       (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_ALIGNSIZE                        (2097152U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR                                             (0xE368U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_MASKFULL                                    (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_PAGE_SIZE_SHIFT                             (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_PAGE_SIZE_CLRMSK                            (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_SHIFT                              (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_ALIGNSHIFT                         (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_ALIGNSIZE                          (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_SHIFT                             (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_ALIGNSHIFT                        (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_ALIGNSIZE                         (2097152U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS1
+*/
+#define RGX_CR_SLC_STATUS1                                                          (0xE210U)
+#define RGX_CR_SLC_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_XBAR_CFI_TIMEOUTS_SHIFT                                  (48U)
+#define RGX_CR_SLC_STATUS1_XBAR_CFI_TIMEOUTS_CLRMSK                                 (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_WRITES_SHIFT                            (36U)
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFF000FFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_WRITES_SHIFT                            (24U)
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_READS_SHIFT                             (12U)
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_READS_SHIFT                             (0U)
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_SLC_STATUS2
+*/
+#define RGX_CR_SLC_STATUS2                                                          (0xE218U)
+#define RGX_CR_SLC_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_SHIFT                                     (48U)
+#define RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_CLRMSK                                    (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_WRITES_SHIFT                            (36U)
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFF000FFFFFFFFF))
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_WRITES_SHIFT                            (24U)
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_READS_SHIFT                             (12U)
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_READS_SHIFT                             (0U)
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_SLC_IDLE
+*/
+#define RGX_CR_SLC_IDLE                                                             (0xE230U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__MASKFULL                               (IMG_UINT64_C(0x0000000001FFFFFF))
+#define RGX_CR_SLC_IDLE_MASKFULL                                                    (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__FBCDC_ARB_SHIFT                        (24U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__FBCDC_ARB_CLRMSK                       (0xFEFFFFFFU)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__FBCDC_ARB_EN                           (0x01000000U)
+#define RGX_CR_SLC_IDLE_FBCDC_ARB_SHIFT                                             (20U)
+#define RGX_CR_SLC_IDLE_FBCDC_ARB_CLRMSK                                            (0xFFEFFFFFU)
+#define RGX_CR_SLC_IDLE_FBCDC_ARB_EN                                                (0x00100000U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__OWDB_SHIFT                             (20U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__OWDB_CLRMSK                            (0xFF0FFFFFU)
+#define RGX_CR_SLC_IDLE_OWDB_SHIFT                                                  (16U)
+#define RGX_CR_SLC_IDLE_OWDB_CLRMSK                                                 (0xFFF0FFFFU)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__ACE_CLBS_SHIFT                         (16U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__ACE_CLBS_CLRMSK                        (0xFFF0FFFFU)
+#define RGX_CR_SLC_IDLE_ACE_CONVERTERS_SHIFT                                        (12U)
+#define RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK                                       (0xFFFF0FFFU)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_SHIFT                                           (4U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_CLRMSK                                          (0xFFFFF00FU)
+#define RGX_CR_SLC_IDLE_MMU_SHIFT                                                   (3U)
+#define RGX_CR_SLC_IDLE_MMU_CLRMSK                                                  (0xFFFFFFF7U)
+#define RGX_CR_SLC_IDLE_MMU_EN                                                      (0x00000008U)
+#define RGX_CR_SLC_IDLE_CCM_SHIFT                                                   (2U)
+#define RGX_CR_SLC_IDLE_CCM_CLRMSK                                                  (0xFFFFFFFBU)
+#define RGX_CR_SLC_IDLE_CCM_EN                                                      (0x00000004U)
+#define RGX_CR_SLC_IDLE_RDI_SHIFT                                                   (1U)
+#define RGX_CR_SLC_IDLE_RDI_CLRMSK                                                  (0xFFFFFFFDU)
+#define RGX_CR_SLC_IDLE_RDI_EN                                                      (0x00000002U)
+#define RGX_CR_SLC_IDLE_XBAR_SHIFT                                                  (0U)
+#define RGX_CR_SLC_IDLE_XBAR_CLRMSK                                                 (0xFFFFFFFEU)
+#define RGX_CR_SLC_IDLE_XBAR_EN                                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS3
+*/
+#define RGX_CR_SLC_STATUS3                                                          (0xE238U)
+#define RGX_CR_SLC_STATUS3_MASKFULL                                                 (IMG_UINT64_C(0x0FFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS3_MAX_CRITICAL_QOS_READ_LATENCY_SHIFT                      (50U)
+#define RGX_CR_SLC_STATUS3_MAX_CRITICAL_QOS_READ_LATENCY_CLRMSK                     (IMG_UINT64_C(0xF003FFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS3_MAX_LOW_QOS_READ_LATENCY_SHIFT                           (40U)
+#define RGX_CR_SLC_STATUS3_MAX_LOW_QOS_READ_LATENCY_CLRMSK                          (IMG_UINT64_C(0xFFFC00FFFFFFFFFF))
+#define RGX_CR_SLC_STATUS3_AVG_CRITICAL_QOS_READ_LATENCY_SHIFT                      (30U)
+#define RGX_CR_SLC_STATUS3_AVG_CRITICAL_QOS_READ_LATENCY_CLRMSK                     (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_CR_SLC_STATUS3_AVG_LOW_QOS_READ_LATENCY_SHIFT                           (20U)
+#define RGX_CR_SLC_STATUS3_AVG_LOW_QOS_READ_LATENCY_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFC00FFFFF))
+#define RGX_CR_SLC_STATUS3_MIN_CRITICAL_QOS_READ_LATENCY_SHIFT                      (10U)
+#define RGX_CR_SLC_STATUS3_MIN_CRITICAL_QOS_READ_LATENCY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFF003FF))
+#define RGX_CR_SLC_STATUS3_MIN_LOW_QOS_READ_LATENCY_SHIFT                           (0U)
+#define RGX_CR_SLC_STATUS3_MIN_LOW_QOS_READ_LATENCY_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFC00))
+
+
+/*
+    Register RGX_CR_SLC_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU                                           (0x1E240U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU__MASKFULL                                 (IMG_UINT64_C(0x000000000001FFFF))
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU__BIF_SHIFT                                (0U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU__BIF_CLRMSK                               (0xFFFE0000U)
+
+
+/*
+    Register RGX_CR_SLC_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC_FAULT_STOP_STATUS                                                (0xE240U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000001FFFF))
+#define RGX_CR_SLC_FAULT_STOP_STATUS_BIF_SHIFT                                      (0U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS_BIF_CLRMSK                                     (0xFFFE0000U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS_DEBUG
+*/
+#define RGX_CR_SLC_STATUS_DEBUG__VPU                                                (0x1E260U)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_COH_REQ_SHIFT                             (16U)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_COH_REQ_CLRMSK                            (0x0000FFFFU)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_ADDR_ALIAS_SHIFT                          (0U)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_ADDR_ALIAS_CLRMSK                         (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS_DEBUG
+*/
+#define RGX_CR_SLC_STATUS_DEBUG                                                     (0xE260U)
+#define RGX_CR_SLC_STATUS_DEBUG_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_COH_REQ_SHIFT                                   (16U)
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_COH_REQ_CLRMSK                                  (0x0000FFFFU)
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_ADDR_ALIAS_SHIFT                                (0U)
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_ADDR_ALIAS_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_HMMU_OSID_PAGE_SIZE
+*/
+#define RGX_CR_HMMU_OSID_PAGE_SIZE                                                  (0x80000U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_MASKFULL                                         (IMG_UINT64_C(0x0000000077777777))
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_7_SHIFT                                     (28U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_7_CLRMSK                                    (0x8FFFFFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_6_SHIFT                                     (24U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_6_CLRMSK                                    (0xF8FFFFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_5_SHIFT                                     (20U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_5_CLRMSK                                    (0xFF8FFFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_4_SHIFT                                     (16U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_4_CLRMSK                                    (0xFFF8FFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_3_SHIFT                                     (12U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_3_CLRMSK                                    (0xFFFF8FFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_2_SHIFT                                     (8U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_2_CLRMSK                                    (0xFFFFF8FFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_1_SHIFT                                     (4U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_1_CLRMSK                                    (0xFFFFFF8FU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_0_SHIFT                                     (0U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_0_CLRMSK                                    (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_HMMU_BYPASS
+*/
+#define RGX_CR_HMMU_BYPASS                                                          (0x80008U)
+#define RGX_CR_HMMU_BYPASS_MASKFULL                                                 (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_HMMU_BYPASS_EN_SHIFT                                                 (0U)
+#define RGX_CR_HMMU_BYPASS_EN_CLRMSK                                                (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_HMMU_INVAL
+*/
+#define RGX_CR_HMMU_INVAL                                                           (0x80010U)
+#define RGX_CR_HMMU_INVAL_MASKFULL                                                  (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_HMMU_INVAL_OS_ID_SHIFT                                               (4U)
+#define RGX_CR_HMMU_INVAL_OS_ID_CLRMSK                                              (0xFFFFFF8FU)
+#define RGX_CR_HMMU_INVAL_ALL_OS_IDS_SHIFT                                          (3U)
+#define RGX_CR_HMMU_INVAL_ALL_OS_IDS_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_HMMU_INVAL_ALL_OS_IDS_EN                                             (0x00000008U)
+#define RGX_CR_HMMU_INVAL_HPC_SHIFT                                                 (2U)
+#define RGX_CR_HMMU_INVAL_HPC_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_HMMU_INVAL_HPC_EN                                                    (0x00000004U)
+#define RGX_CR_HMMU_INVAL_HPD_SHIFT                                                 (1U)
+#define RGX_CR_HMMU_INVAL_HPD_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_HMMU_INVAL_HPD_EN                                                    (0x00000002U)
+#define RGX_CR_HMMU_INVAL_HPT_SHIFT                                                 (0U)
+#define RGX_CR_HMMU_INVAL_HPT_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_HMMU_INVAL_HPT_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING0
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0                                               (0x80018U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR1_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR1_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID1_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID1_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID1_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR0_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR0_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID0_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID0_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID0_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING1
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1                                               (0x80020U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR3_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR3_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID3_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID3_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID3_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR2_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR2_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID2_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID2_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID2_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING2
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2                                               (0x80028U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR5_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR5_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID5_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID5_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID5_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR4_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR4_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID4_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID4_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID4_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING3
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3                                               (0x80030U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR7_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR7_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID7_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID7_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID7_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR6_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR6_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID6_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID6_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID6_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register group: RGX_CR_HMMU_PAGE_FAULT_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO_REPEATCOUNT                                     (8U)
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO0
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0                                                (0x80038U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO1
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1                                                (0x80040U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO2
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2                                                (0x80048U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO3
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3                                                (0x80050U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO4
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4                                                (0x80058U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO5
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5                                                (0x80060U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO6
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6                                                (0x80068U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO7
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7                                                (0x80070U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register group: RGX_CR_HMMU_PENDING_ENTRY_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO_REPEATCOUNT                                  (8U)
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO0
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0                                             (0x800C0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO1
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1                                             (0x800C8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO2
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2                                             (0x800D0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO3
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3                                             (0x800D8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO4
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4                                             (0x800E0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO5
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5                                             (0x800E8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO6
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6                                             (0x800F0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO7
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7                                             (0x800F8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_HOST_IRQ_ENABLE
+*/
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE                                                 (0x80100U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_SHIFT                            (3U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_EN                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_PM_SHIFT                         (2U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_PM_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_PM_EN                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PAGE_FAULT_SHIFT                                (1U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PAGE_FAULT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PAGE_FAULT_EN                                   (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PENDING_ENTRY_SHIFT                             (0U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PENDING_ENTRY_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PENDING_ENTRY_EN                                (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY                                                   (0x80108U)
+#define RGX_CR_HMMU_PENDING_ENTRY_MASKFULL                                          (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_HMMU_PENDING_ENTRY_ENABLE_SHIFT                                      (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_ENABLE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_HMMU_PENDING_ENTRY_ENABLE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_HMMU_FAULT_STATUS
+*/
+#define RGX_CR_HMMU_FAULT_STATUS                                                    (0x80120U)
+#define RGX_CR_HMMU_FAULT_STATUS_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID7_SHIFT                      (31U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID7_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID7_EN                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID6_SHIFT                      (30U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID6_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID6_EN                         (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID5_SHIFT                      (29U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID5_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID5_EN                         (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID4_SHIFT                      (28U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID4_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID4_EN                         (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID3_SHIFT                      (27U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID3_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID3_EN                         (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID2_SHIFT                      (26U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID2_EN                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID1_SHIFT                      (25U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID1_EN                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID0_SHIFT                      (24U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID0_EN                         (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID7_SHIFT                         (23U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID7_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID7_EN                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID6_SHIFT                         (22U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID6_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID6_EN                            (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID5_SHIFT                         (21U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID5_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID5_EN                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID4_SHIFT                         (20U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID4_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID4_EN                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID3_SHIFT                         (19U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID3_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID3_EN                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID2_SHIFT                         (18U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID2_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID2_EN                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID1_SHIFT                         (17U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID1_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID1_EN                            (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID0_SHIFT                         (16U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID0_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID0_EN                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID7_SHIFT                          (15U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID7_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID7_EN                             (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID6_SHIFT                          (14U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID6_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID6_EN                             (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID5_SHIFT                          (13U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID5_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID5_EN                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID4_SHIFT                          (12U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID4_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID4_EN                             (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID3_SHIFT                          (11U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID3_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID3_EN                             (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID2_SHIFT                          (10U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID2_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID2_EN                             (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID1_SHIFT                          (9U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID1_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID1_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID0_SHIFT                          (8U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID0_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID0_EN                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID7_SHIFT                             (7U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID7_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID7_EN                                (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID6_SHIFT                             (6U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID6_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID6_EN                                (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID5_SHIFT                             (5U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID5_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID5_EN                                (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID4_SHIFT                             (4U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID4_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID4_EN                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID3_SHIFT                             (3U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID3_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID3_EN                                (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID2_SHIFT                             (2U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID2_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID2_EN                                (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID1_SHIFT                             (1U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID1_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID1_EN                                (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID0_SHIFT                             (0U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID0_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID0_EN                                (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register group: RGX_CR_HMMU_READONLY_FAULT_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO_REPEATCOUNT                                 (8U)
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO0
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0                                            (0x80190U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO1
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1                                            (0x80198U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO2
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2                                            (0x801A0U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO3
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3                                            (0x801A8U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO4
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4                                            (0x801B0U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO5
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5                                            (0x801B8U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO6
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6                                            (0x801C0U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO7
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7                                            (0x801C8U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register group: RGX_CR_HMMU_READONLY_FAULT_PM_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO_REPEATCOUNT                              (8U)
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO0
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0                                         (0x801D0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO1
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1                                         (0x801D8U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO2
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2                                         (0x801E0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO3
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3                                         (0x801E8U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO4
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4                                         (0x801F0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO5
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5                                         (0x801F8U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO6
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6                                         (0x80200U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO7
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7                                         (0x80208U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_ACE_CTRL
+*/
+#define RGX_CR_ACE_CTRL__VPU                                                        (0x1E320U)
+#define RGX_CR_ACE_CTRL__VPU__MASKFULL                                              (IMG_UINT64_C(0x00000000007FCFFF))
+#define RGX_CR_ACE_CTRL__VPU__CLB_AXQOS_SHIFT                                       (19U)
+#define RGX_CR_ACE_CTRL__VPU__CLB_AXQOS_CLRMSK                                      (0xFF87FFFFU)
+#define RGX_CR_ACE_CTRL__VPU__PM_MMU_AXCACHE_SHIFT                                  (15U)
+#define RGX_CR_ACE_CTRL__VPU__PM_MMU_AXCACHE_CLRMSK                                 (0xFFF87FFFU)
+#define RGX_CR_ACE_CTRL__VPU__ENABLE_NONSECURE_PROT_MATCH_SHIFT                     (14U)
+#define RGX_CR_ACE_CTRL__VPU__ENABLE_NONSECURE_PROT_MATCH_CLRMSK                    (0xFFFFBFFFU)
+#define RGX_CR_ACE_CTRL__VPU__ENABLE_NONSECURE_PROT_MATCH_EN                        (0x00004000U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_SHIFT                                     (8U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_CLRMSK                                    (0xFFFFF0FFU)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_DEVICE_BUFFERABLE                         (0x00000100U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_NORMAL_NC_NON_BUFFERABLE                  (0x00000200U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_NORMAL_NC_BUFFERABLE                      (0x00000300U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_THROUGH_NO_ALLOCATE                 (0x00000600U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_THROUGH_WRITE_ALLOCATE              (0x00000E00U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_BACK_NO_ALLOCATE                    (0x00000700U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE                 (0x00000F00U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_SHIFT                                     (4U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_CLRMSK                                    (0xFFFFFF0FU)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_DEVICE_BUFFERABLE                         (0x00000010U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_NORMAL_NC_NON_BUFFERABLE                  (0x00000020U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_NORMAL_NC_BUFFERABLE                      (0x00000030U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_THROUGH_NO_ALLOCATE                 (0x000000A0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_THROUGH_READ_ALLOCATE               (0x000000E0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_BACK_NO_ALLOCATE                    (0x000000B0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE                  (0x000000F0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_DOMAIN_SHIFT                                      (2U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_DOMAIN_CLRMSK                                     (0xFFFFFFF3U)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_SHIFT                                      (1U)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_INNER_SHAREABLE                            (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_OUTER_SHAREABLE                            (0x00000002U)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_SHIFT                                  (0U)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_NON_SHAREABLE                          (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_SYSTEM                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_ACE_CTRL
+*/
+#define RGX_CR_ACE_CTRL                                                             (0xE320U)
+#define RGX_CR_ACE_CTRL_MASKFULL                                                    (IMG_UINT64_C(0x0000000000FFCFFF))
+#define RGX_CR_ACE_CTRL_DISABLE_EMPTY_BURST_REMOVAL_SHIFT                           (23U)
+#define RGX_CR_ACE_CTRL_DISABLE_EMPTY_BURST_REMOVAL_CLRMSK                          (0xFF7FFFFFU)
+#define RGX_CR_ACE_CTRL_DISABLE_EMPTY_BURST_REMOVAL_EN                              (0x00800000U)
+#define RGX_CR_ACE_CTRL_CLB_AXQOS_SHIFT                                             (19U)
+#define RGX_CR_ACE_CTRL_CLB_AXQOS_CLRMSK                                            (0xFF87FFFFU)
+#define RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT                                        (15U)
+#define RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_CLRMSK                                       (0xFFF87FFFU)
+#define RGX_CR_ACE_CTRL_ENABLE_NONSECURE_PROT_MATCH_SHIFT                           (14U)
+#define RGX_CR_ACE_CTRL_ENABLE_NONSECURE_PROT_MATCH_CLRMSK                          (0xFFFFBFFFU)
+#define RGX_CR_ACE_CTRL_ENABLE_NONSECURE_PROT_MATCH_EN                              (0x00004000U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_SHIFT                                           (8U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_CLRMSK                                          (0xFFFFF0FFU)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_DEVICE_NON_BUFFERABLE                           (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_DEVICE_BUFFERABLE                               (0x00000100U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_NORMAL_NC_NON_BUFFERABLE                        (0x00000200U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_NORMAL_NC_BUFFERABLE                            (0x00000300U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_THROUGH_NO_ALLOCATE                       (0x00000600U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_THROUGH_WRITE_ALLOCATE                    (0x00000E00U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_NO_ALLOCATE                          (0x00000700U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE                       (0x00000F00U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__DEVICE_BUFFERABLE                         (0x00000100U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__NORMAL_NC_NON_BUFFERABLE                  (0x00000200U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__NORMAL_NC_BUFFERABLE                      (0x00000300U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_THROUGH_NO_ALLOCATE                 (0x00000600U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_THROUGH_WRITE_ALLOCATE              (0x00000E00U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_BACK_NO_ALLOCATE                    (0x00000700U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_BACK_WRITE_ALLOCATE                 (0x00000F00U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_SHIFT                                           (4U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_CLRMSK                                          (0xFFFFFF0FU)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_DEVICE_NON_BUFFERABLE                           (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_DEVICE_BUFFERABLE                               (0x00000010U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_NORMAL_NC_NON_BUFFERABLE                        (0x00000020U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_NORMAL_NC_BUFFERABLE                            (0x00000030U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_THROUGH_NO_ALLOCATE                       (0x000000A0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_THROUGH_READ_ALLOCATE                     (0x000000E0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_NO_ALLOCATE                          (0x000000B0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE                        (0x000000F0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__DEVICE_BUFFERABLE                         (0x00000010U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__NORMAL_NC_NON_BUFFERABLE                  (0x00000020U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__NORMAL_NC_BUFFERABLE                      (0x00000030U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_THROUGH_NO_ALLOCATE                 (0x000000A0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_THROUGH_READ_ALLOCATE               (0x000000E0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_BACK_NO_ALLOCATE                    (0x000000B0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_BACK_READ_ALLOCATE                  (0x000000F0U)
+#define RGX_CR_ACE_CTRL_MMU_DOMAIN_SHIFT                                            (2U)
+#define RGX_CR_ACE_CTRL_MMU_DOMAIN_CLRMSK                                           (0xFFFFFFF3U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_SHIFT                                            (1U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_CLRMSK                                           (0xFFFFFFFDU)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_INNER_SHAREABLE                                  (0x00000000U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE                                  (0x00000002U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN__VPU__INNER_SHAREABLE                            (0x00000000U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN__VPU__OUTER_SHAREABLE                            (0x00000002U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_SHIFT                                        (0U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE                                (0x00000000U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_SYSTEM                                       (0x00000001U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN__VPU__NON_SHAREABLE                          (0x00000000U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN__VPU__SYSTEM                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_SOC_AXI
+*/
+#define RGX_CR_SOC_AXI                                                              (0xE338U)
+#define RGX_CR_SOC_AXI_MASKFULL                                                     (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_SOC_AXI_NON_COHERENT_128_BYTE_BURST_SUPPORT_SHIFT                    (3U)
+#define RGX_CR_SOC_AXI_NON_COHERENT_128_BYTE_BURST_SUPPORT_CLRMSK                   (0xFFFFFFF7U)
+#define RGX_CR_SOC_AXI_NON_COHERENT_128_BYTE_BURST_SUPPORT_EN                       (0x00000008U)
+#define RGX_CR_SOC_AXI_COHERENT_128_BYTE_BURST_SUPPORT_SHIFT                        (2U)
+#define RGX_CR_SOC_AXI_COHERENT_128_BYTE_BURST_SUPPORT_CLRMSK                       (0xFFFFFFFBU)
+#define RGX_CR_SOC_AXI_COHERENT_128_BYTE_BURST_SUPPORT_EN                           (0x00000004U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT                                      (0U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK                                     (0xFFFFFFFCU)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY                               (0x00000000U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY                         (0x00000001U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY                         (0x00000002U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING0
+*/
+#define RGX_CR_CONTEXT_MAPPING0                                                     (0xF078U)
+#define RGX_CR_CONTEXT_MAPPING0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_2D_SHIFT                                            (24U)
+#define RGX_CR_CONTEXT_MAPPING0_2D_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_CDM_SHIFT                                           (16U)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING0_3D_SHIFT                                            (8U)
+#define RGX_CR_CONTEXT_MAPPING0_3D_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING0_GEOM_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING0_GEOM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING2
+*/
+#define RGX_CR_CONTEXT_MAPPING2                                                     (0xF088U)
+#define RGX_CR_CONTEXT_MAPPING2_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING3
+*/
+#define RGX_CR_CONTEXT_MAPPING3                                                     (0xF090U)
+#define RGX_CR_CONTEXT_MAPPING3_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_OUTSTANDING_READ                                                 (0xF098U)
+#define RGX_CR_BIF_OUTSTANDING_READ_MASKFULL                                        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_OUTSTANDING_READ_COUNTER_SHIFT                                   (0U)
+#define RGX_CR_BIF_OUTSTANDING_READ_COUNTER_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS1_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ                                          (0xF0A0U)
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ_COUNTER_SHIFT                            (0U)
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ_COUNTER_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS0_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ                                          (0xF0A8U)
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ_COUNTER_SHIFT                            (0U)
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ_COUNTER_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_PFS
+*/
+#define RGX_CR_BIF_PFS                                                              (0xF0B0U)
+#define RGX_CR_BIF_PFS_MASKFULL                                                     (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_BIF_PFS_SLC_STALLING_SHIFT                                           (8U)
+#define RGX_CR_BIF_PFS_SLC_STALLING_CLRMSK                                          (0xFFFFFEFFU)
+#define RGX_CR_BIF_PFS_SLC_STALLING_EN                                              (0x00000100U)
+#define RGX_CR_BIF_PFS_TILING_IDLE_SHIFT                                            (7U)
+#define RGX_CR_BIF_PFS_TILING_IDLE_CLRMSK                                           (0xFFFFFF7FU)
+#define RGX_CR_BIF_PFS_TILING_IDLE_EN                                               (0x00000080U)
+#define RGX_CR_BIF_PFS_ARB_IDLE_SHIFT                                               (6U)
+#define RGX_CR_BIF_PFS_ARB_IDLE_CLRMSK                                              (0xFFFFFFBFU)
+#define RGX_CR_BIF_PFS_ARB_IDLE_EN                                                  (0x00000040U)
+#define RGX_CR_BIF_PFS_RTN_MEM_IF_IDLE_SHIFT                                        (5U)
+#define RGX_CR_BIF_PFS_RTN_MEM_IF_IDLE_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_BIF_PFS_RTN_MEM_IF_IDLE_EN                                           (0x00000020U)
+#define RGX_CR_BIF_PFS_NONMETA_REQ_COMPLETE_SHIFT                                   (4U)
+#define RGX_CR_BIF_PFS_NONMETA_REQ_COMPLETE_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_BIF_PFS_NONMETA_REQ_COMPLETE_EN                                      (0x00000010U)
+#define RGX_CR_BIF_PFS_NONMETA_WDATA_COMPLETE_SHIFT                                 (3U)
+#define RGX_CR_BIF_PFS_NONMETA_WDATA_COMPLETE_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_BIF_PFS_NONMETA_WDATA_COMPLETE_EN                                    (0x00000008U)
+#define RGX_CR_BIF_PFS_WDATA_IP_IDLE_SHIFT                                          (2U)
+#define RGX_CR_BIF_PFS_WDATA_IP_IDLE_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BIF_PFS_WDATA_IP_IDLE_EN                                             (0x00000004U)
+#define RGX_CR_BIF_PFS_REQUEST_IP_IDLE_SHIFT                                        (1U)
+#define RGX_CR_BIF_PFS_REQUEST_IP_IDLE_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_BIF_PFS_REQUEST_IP_IDLE_EN                                           (0x00000002U)
+#define RGX_CR_BIF_PFS_STALL_COMPLETE_SHIFT                                         (0U)
+#define RGX_CR_BIF_PFS_STALL_COMPLETE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_BIF_PFS_STALL_COMPLETE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS0_PFS
+*/
+#define RGX_CR_BIF_TEXAS0_PFS                                                       (0xF0B8U)
+#define RGX_CR_BIF_TEXAS0_PFS_MASKFULL                                              (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_INFLIGHT_SHIFT                              (6U)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_INFLIGHT_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_INFLIGHT_EN                                 (0x00000040U)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_IDLE_SHIFT                                  (5U)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_IDLE_CLRMSK                                 (0xFFFFFFDFU)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_IDLE_EN                                     (0x00000020U)
+#define RGX_CR_BIF_TEXAS0_PFS_SLC_STALLING_SHIFT                                    (4U)
+#define RGX_CR_BIF_TEXAS0_PFS_SLC_STALLING_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_BIF_TEXAS0_PFS_SLC_STALLING_EN                                       (0x00000010U)
+#define RGX_CR_BIF_TEXAS0_PFS_TILING_IDLE_SHIFT                                     (3U)
+#define RGX_CR_BIF_TEXAS0_PFS_TILING_IDLE_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_BIF_TEXAS0_PFS_TILING_IDLE_EN                                        (0x00000008U)
+#define RGX_CR_BIF_TEXAS0_PFS_ARB_IDLE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TEXAS0_PFS_ARB_IDLE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TEXAS0_PFS_ARB_IDLE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TEXAS0_PFS_RTN_MEM_IF_IDLE_SHIFT                                 (1U)
+#define RGX_CR_BIF_TEXAS0_PFS_RTN_MEM_IF_IDLE_CLRMSK                                (0xFFFFFFFDU)
+#define RGX_CR_BIF_TEXAS0_PFS_RTN_MEM_IF_IDLE_EN                                    (0x00000002U)
+#define RGX_CR_BIF_TEXAS0_PFS_STALL_COMPLETE_SHIFT                                  (0U)
+#define RGX_CR_BIF_TEXAS0_PFS_STALL_COMPLETE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_BIF_TEXAS0_PFS_STALL_COMPLETE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS1_PFS
+*/
+#define RGX_CR_BIF_TEXAS1_PFS                                                       (0xF0C8U)
+#define RGX_CR_BIF_TEXAS1_PFS_MASKFULL                                              (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_PDSRW_SHIFT                        (6U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_PDSRW_CLRMSK                       (0xFFFFFFBFU)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_PDSRW_EN                           (0x00000040U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_MCU_SHIFT                          (5U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_MCU_CLRMSK                         (0xFFFFFFDFU)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_MCU_EN                             (0x00000020U)
+#define RGX_CR_BIF_TEXAS1_PFS_SLC_STALLING_SHIFT                                    (4U)
+#define RGX_CR_BIF_TEXAS1_PFS_SLC_STALLING_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_BIF_TEXAS1_PFS_SLC_STALLING_EN                                       (0x00000010U)
+#define RGX_CR_BIF_TEXAS1_PFS_TILING_IDLE_SHIFT                                     (3U)
+#define RGX_CR_BIF_TEXAS1_PFS_TILING_IDLE_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_BIF_TEXAS1_PFS_TILING_IDLE_EN                                        (0x00000008U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_IDLE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_IDLE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_IDLE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TEXAS1_PFS_RTN_MEM_IF_IDLE_SHIFT                                 (1U)
+#define RGX_CR_BIF_TEXAS1_PFS_RTN_MEM_IF_IDLE_CLRMSK                                (0xFFFFFFFDU)
+#define RGX_CR_BIF_TEXAS1_PFS_RTN_MEM_IF_IDLE_EN                                    (0x00000002U)
+#define RGX_CR_BIF_TEXAS1_PFS_STALL_COMPLETE_SHIFT                                  (0U)
+#define RGX_CR_BIF_TEXAS1_PFS_STALL_COMPLETE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_BIF_TEXAS1_PFS_STALL_COMPLETE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_FIX
+*/
+#define RGX_CR_JONES_FIX__ROGUE3                                                    (0xF0C0U)
+#define RGX_CR_JONES_FIX__ROGUE3__MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX__ROGUE3__DISABLE_SHIFT                                     (0U)
+#define RGX_CR_JONES_FIX__ROGUE3__DISABLE_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_FBCDC_STATUS
+*/
+#define RGX_CR_FBCDC_STATUS                                                         (0xF600U)
+#define RGX_CR_FBCDC_STATUS_MASKFULL                                                (IMG_UINT64_C(0x000000000F0F0F0F))
+#define RGX_CR_FBCDC_STATUS_STATE_CHECK_FAIL_SHIFT                                  (24U)
+#define RGX_CR_FBCDC_STATUS_STATE_CHECK_FAIL_CLRMSK                                 (0xF0FFFFFFU)
+#define RGX_CR_FBCDC_STATUS_UFOD_DECODER_ERROR_SHIFT                                (16U)
+#define RGX_CR_FBCDC_STATUS_UFOD_DECODER_ERROR_CLRMSK                               (0xFFF0FFFFU)
+#define RGX_CR_FBCDC_STATUS_HEADER_CHECK_FAIL_SHIFT                                 (8U)
+#define RGX_CR_FBCDC_STATUS_HEADER_CHECK_FAIL_CLRMSK                                (0xFFFFF0FFU)
+#define RGX_CR_FBCDC_STATUS_TILE_LENGTH_CHECK_FAIL_SHIFT                            (0U)
+#define RGX_CR_FBCDC_STATUS_TILE_LENGTH_CHECK_FAIL_CLRMSK                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FBCDC_SIGNATURE_STATUS
+*/
+#define RGX_CR_FBCDC_SIGNATURE_STATUS                                               (0xF618U)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_MASKFULL                                      (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBM_ERROR_SHIFT                               (4U)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBM_ERROR_CLRMSK                              (0xFFFFFF0FU)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBDC_ERROR_SHIFT                              (0U)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBDC_ERROR_CLRMSK                             (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING4
+*/
+#define RGX_CR_CONTEXT_MAPPING4                                                     (0xF210U)
+#define RGX_CR_CONTEXT_MAPPING4_MASKFULL                                            (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_SHIFT                                  (40U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_SHIFT                                    (32U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_SHIFT                                     (24U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_SHIFT                                  (16U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_SHIFT                                    (8U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_SHIFT                                     (0U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FBCDC_IDLE
+*/
+#define RGX_CR_FBCDC_IDLE                                                           (0xF218U)
+#define RGX_CR_FBCDC_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF3_SHIFT                                          (13U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF3_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF3_EN                                             (0x00002000U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF3_SHIFT                                         (12U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF3_CLRMSK                                        (0xFFFFEFFFU)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF3_EN                                            (0x00001000U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF3_SHIFT                                          (11U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF3_CLRMSK                                         (0xFFFFF7FFU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF3_EN                                             (0x00000800U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF2_SHIFT                                          (10U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF2_CLRMSK                                         (0xFFFFFBFFU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF2_EN                                             (0x00000400U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF2_SHIFT                                         (9U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF2_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF2_EN                                            (0x00000200U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF2_SHIFT                                          (8U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF2_CLRMSK                                         (0xFFFFFEFFU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF2_EN                                             (0x00000100U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF1_SHIFT                                          (7U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF1_CLRMSK                                         (0xFFFFFF7FU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF1_EN                                             (0x00000080U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF1_SHIFT                                         (6U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF1_CLRMSK                                        (0xFFFFFFBFU)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF1_EN                                            (0x00000040U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF1_SHIFT                                          (5U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF1_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF1_EN                                             (0x00000020U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF0_SHIFT                                          (4U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF0_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF0_EN                                             (0x00000010U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF0_SHIFT                                         (3U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF0_CLRMSK                                        (0xFFFFFFF7U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF0_EN                                            (0x00000008U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF0_SHIFT                                          (2U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF0_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF0_EN                                             (0x00000004U)
+#define RGX_CR_FBCDC_IDLE_FBHC_SHIFT                                                (1U)
+#define RGX_CR_FBCDC_IDLE_FBHC_CLRMSK                                               (0xFFFFFFFDU)
+#define RGX_CR_FBCDC_IDLE_FBHC_EN                                                   (0x00000002U)
+#define RGX_CR_FBCDC_IDLE_FBSC_SHIFT                                                (0U)
+#define RGX_CR_FBCDC_IDLE_FBSC_CLRMSK                                               (0xFFFFFFFEU)
+#define RGX_CR_FBCDC_IDLE_FBSC_EN                                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_MERCER_SOFT_RESET
+*/
+#define RGX_CR_MERCER_SOFT_RESET                                                    (0x0630U)
+#define RGX_CR_MERCER_SOFT_RESET_MASKFULL                                           (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_SHIFT                                (62U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_CLRMSK                               (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_EN                                   (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_SHIFT                                (61U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_CLRMSK                               (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_EN                                   (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_SHIFT                                (60U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_CLRMSK                               (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_EN                                   (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_SHIFT                                (59U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_CLRMSK                               (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_EN                                   (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_SHIFT                                (58U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_EN                                   (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_SHIFT                                (57U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_EN                                   (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_SHIFT                                (56U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_EN                                   (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_SHIFT                                (55U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_EN                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_SHIFT                                (54U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_EN                                   (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_SHIFT                                (53U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_EN                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_SHIFT                                (52U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_EN                                   (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_SHIFT                                (51U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_EN                                   (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_SHIFT                                (50U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_EN                                   (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_SHIFT                                (49U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_EN                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_SHIFT                                (48U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_EN                                   (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_SHIFT                                (47U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_EN                                   (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_SHIFT                                (46U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_EN                                   (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_SHIFT                                (45U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_EN                                   (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_SHIFT                                (44U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_EN                                   (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_SHIFT                                (43U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_EN                                   (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_SHIFT                                (42U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_EN                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_SHIFT                                (41U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_EN                                   (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_SHIFT                                (40U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_EN                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_SHIFT                                (39U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_EN                                   (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_SHIFT                                (38U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_EN                                   (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_SHIFT                                (37U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_EN                                   (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_SHIFT                                (36U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_EN                                   (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_SHIFT                                (35U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_EN                                   (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_SHIFT                                (34U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_EN                                   (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_SHIFT                                (33U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_EN                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_SHIFT                                (32U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_EN                                   (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_SHIFT                                (31U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_EN                                   (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_SHIFT                                (30U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_EN                                   (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_SHIFT                                 (29U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_EN                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_SHIFT                                 (28U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_EN                                    (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_SHIFT                                 (27U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_EN                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_SHIFT                                 (26U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_EN                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_SHIFT                                 (25U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_EN                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_SHIFT                                 (24U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_EN                                    (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_SHIFT                                 (23U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_SHIFT                                 (22U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_SHIFT                                 (21U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_SHIFT                                 (20U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_SHIFT                                 (19U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_SHIFT                                 (18U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_SHIFT                                 (17U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_SHIFT                                 (16U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_SHIFT                                 (15U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_SHIFT                                 (14U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_SHIFT                                 (13U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_SHIFT                                 (12U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_SHIFT                                 (11U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_SHIFT                                 (10U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_SHIFT                                 (9U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_SHIFT                                 (8U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_SHIFT                                 (7U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_SHIFT                                 (6U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_SHIFT                                 (5U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_SHIFT                                 (4U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_SHIFT                                 (3U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_SHIFT                                 (2U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_SHIFT                                 (1U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_SHIFT                                 (0U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_TEXAS_SOFT_RESET
+*/
+#define RGX_CR_TEXAS_SOFT_RESET                                                     (0x0640U)
+#define RGX_CR_TEXAS_SOFT_RESET_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_SOFT_RESET_SPU31_SHIFT                                         (31U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU31_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU31_EN                                            (0x80000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU30_SHIFT                                         (30U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU30_CLRMSK                                        (0xBFFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU30_EN                                            (0x40000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU29_SHIFT                                         (29U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU29_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU29_EN                                            (0x20000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU28_SHIFT                                         (28U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU28_CLRMSK                                        (0xEFFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU28_EN                                            (0x10000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU27_SHIFT                                         (27U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU27_CLRMSK                                        (0xF7FFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU27_EN                                            (0x08000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU26_SHIFT                                         (26U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU26_CLRMSK                                        (0xFBFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU26_EN                                            (0x04000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU25_SHIFT                                         (25U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU25_CLRMSK                                        (0xFDFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU25_EN                                            (0x02000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU24_SHIFT                                         (24U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU24_CLRMSK                                        (0xFEFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU24_EN                                            (0x01000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU23_SHIFT                                         (23U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU23_CLRMSK                                        (0xFF7FFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU23_EN                                            (0x00800000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU22_SHIFT                                         (22U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU22_CLRMSK                                        (0xFFBFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU22_EN                                            (0x00400000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU21_SHIFT                                         (21U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU21_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU21_EN                                            (0x00200000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU20_SHIFT                                         (20U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU20_CLRMSK                                        (0xFFEFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU20_EN                                            (0x00100000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU19_SHIFT                                         (19U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU19_CLRMSK                                        (0xFFF7FFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU19_EN                                            (0x00080000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU18_SHIFT                                         (18U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU18_CLRMSK                                        (0xFFFBFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU18_EN                                            (0x00040000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU17_SHIFT                                         (17U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU17_CLRMSK                                        (0xFFFDFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU17_EN                                            (0x00020000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU16_SHIFT                                         (16U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU16_CLRMSK                                        (0xFFFEFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU16_EN                                            (0x00010000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU15_SHIFT                                         (15U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU15_CLRMSK                                        (0xFFFF7FFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU15_EN                                            (0x00008000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU14_SHIFT                                         (14U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU14_CLRMSK                                        (0xFFFFBFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU14_EN                                            (0x00004000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU13_SHIFT                                         (13U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU13_CLRMSK                                        (0xFFFFDFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU13_EN                                            (0x00002000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU12_SHIFT                                         (12U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU12_CLRMSK                                        (0xFFFFEFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU12_EN                                            (0x00001000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU11_SHIFT                                         (11U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU11_CLRMSK                                        (0xFFFFF7FFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU11_EN                                            (0x00000800U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU10_SHIFT                                         (10U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU10_CLRMSK                                        (0xFFFFFBFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU10_EN                                            (0x00000400U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU9_SHIFT                                          (9U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU9_CLRMSK                                         (0xFFFFFDFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU9_EN                                             (0x00000200U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU8_SHIFT                                          (8U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU8_CLRMSK                                         (0xFFFFFEFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU8_EN                                             (0x00000100U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU7_SHIFT                                          (7U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU7_CLRMSK                                         (0xFFFFFF7FU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU7_EN                                             (0x00000080U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU6_SHIFT                                          (6U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU6_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU6_EN                                             (0x00000040U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU5_SHIFT                                          (5U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU5_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU5_EN                                             (0x00000020U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU4_SHIFT                                          (4U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU4_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU4_EN                                             (0x00000010U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU3_SHIFT                                          (3U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU3_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU3_EN                                             (0x00000008U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU2_SHIFT                                          (2U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU2_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU2_EN                                             (0x00000004U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU1_SHIFT                                          (1U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU1_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU1_EN                                             (0x00000002U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU0_SHIFT                                          (0U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU0_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU0_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_SWIFT_SOFT_RESET
+*/
+#define RGX_CR_SWIFT_SOFT_RESET                                                     (0x0650U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__MASKFULL                               (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT2_SHIFT                     (62U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT2_EN                        (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT1_SHIFT                     (61U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT1_EN                        (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT0_SHIFT                     (60U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT0_EN                        (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT2_SHIFT                     (59U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT2_EN                        (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT1_SHIFT                     (58U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT1_EN                        (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT0_SHIFT                     (57U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT0_EN                        (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT2_SHIFT                     (56U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT2_EN                        (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT1_SHIFT                     (55U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT1_EN                        (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT0_SHIFT                     (54U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT0_EN                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT2_SHIFT                     (53U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT2_EN                        (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT1_SHIFT                     (52U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT1_EN                        (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT0_SHIFT                     (51U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT0_EN                        (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT2_SHIFT                     (50U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT2_EN                        (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT1_SHIFT                     (49U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT1_EN                        (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT0_SHIFT                     (48U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT0_EN                        (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT2_SHIFT                     (47U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT2_EN                        (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT1_SHIFT                     (46U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT1_EN                        (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT0_SHIFT                     (45U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT0_EN                        (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT2_SHIFT                     (44U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT2_EN                        (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT1_SHIFT                     (43U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT1_EN                        (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT0_SHIFT                     (42U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT0_EN                        (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT2_SHIFT                     (41U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT2_EN                        (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT1_SHIFT                     (40U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT1_EN                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT0_SHIFT                     (39U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT0_EN                        (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT2_SHIFT                     (38U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT2_EN                        (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT1_SHIFT                     (37U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT1_EN                        (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT0_SHIFT                     (36U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT0_EN                        (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT2_SHIFT                     (35U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT2_EN                        (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT1_SHIFT                     (34U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT1_EN                        (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT0_SHIFT                     (33U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT0_EN                        (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT2_SHIFT                     (32U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT2_EN                        (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU31_SHIFT                                         (31U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU31_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU31_EN                                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT1_SHIFT                     (31U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT1_EN                        (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU30_SHIFT                                         (30U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU30_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU30_EN                                            (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT0_SHIFT                     (30U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT0_EN                        (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU29_SHIFT                                         (29U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU29_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU29_EN                                            (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT2_SHIFT                      (29U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT2_EN                         (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU28_SHIFT                                         (28U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU28_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU28_EN                                            (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT1_SHIFT                      (28U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT1_EN                         (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU27_SHIFT                                         (27U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU27_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU27_EN                                            (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT0_SHIFT                      (27U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT0_EN                         (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU26_SHIFT                                         (26U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU26_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU26_EN                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT2_SHIFT                      (26U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT2_EN                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU25_SHIFT                                         (25U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU25_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU25_EN                                            (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT1_SHIFT                      (25U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT1_EN                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU24_SHIFT                                         (24U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU24_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU24_EN                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT0_SHIFT                      (24U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT0_EN                         (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU23_SHIFT                                         (23U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU23_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU23_EN                                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT2_SHIFT                      (23U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU22_SHIFT                                         (22U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU22_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU22_EN                                            (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT1_SHIFT                      (22U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU21_SHIFT                                         (21U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU21_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU21_EN                                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT0_SHIFT                      (21U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU20_SHIFT                                         (20U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU20_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU20_EN                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT2_SHIFT                      (20U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU19_SHIFT                                         (19U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU19_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU19_EN                                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT1_SHIFT                      (19U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU18_SHIFT                                         (18U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU18_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU18_EN                                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT0_SHIFT                      (18U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU17_SHIFT                                         (17U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU17_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU17_EN                                            (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT2_SHIFT                      (17U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU16_SHIFT                                         (16U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU16_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU16_EN                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT1_SHIFT                      (16U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU15_SHIFT                                         (15U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU15_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU15_EN                                            (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT0_SHIFT                      (15U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU14_SHIFT                                         (14U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU14_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU14_EN                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT2_SHIFT                      (14U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU13_SHIFT                                         (13U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU13_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU13_EN                                            (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT1_SHIFT                      (13U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU12_SHIFT                                         (12U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU12_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU12_EN                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT0_SHIFT                      (12U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU11_SHIFT                                         (11U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU11_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU11_EN                                            (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT2_SHIFT                      (11U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU10_SHIFT                                         (10U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU10_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU10_EN                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT1_SHIFT                      (10U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU9_SHIFT                                          (9U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU9_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU9_EN                                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT0_SHIFT                      (9U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU8_SHIFT                                          (8U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU8_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU8_EN                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT2_SHIFT                      (8U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU7_SHIFT                                          (7U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU7_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU7_EN                                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT1_SHIFT                      (7U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU6_SHIFT                                          (6U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU6_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU6_EN                                             (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT0_SHIFT                      (6U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU5_SHIFT                                          (5U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU5_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU5_EN                                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT2_SHIFT                      (5U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU4_SHIFT                                          (4U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU4_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU4_EN                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT1_SHIFT                      (4U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU3_SHIFT                                          (3U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU3_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU3_EN                                             (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT0_SHIFT                      (3U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU2_SHIFT                                          (2U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU2_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU2_EN                                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT2_SHIFT                      (2U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU1_SHIFT                                          (1U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU1_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU1_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT1_SHIFT                      (1U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU0_SHIFT                                          (0U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU0_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU0_EN                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT0_SHIFT                      (0U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_RAC_SOFT_RESET
+*/
+#define RGX_CR_RAC_SOFT_RESET                                                       (0x0660U)
+#define RGX_CR_RAC_SOFT_RESET_MASKFULL                                              (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC2_SHIFT                                      (62U)
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC2_CLRMSK                                     (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC2_EN                                         (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC1_SHIFT                                      (61U)
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC1_CLRMSK                                     (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC1_EN                                         (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC0_SHIFT                                      (60U)
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC0_CLRMSK                                     (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC0_EN                                         (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC2_SHIFT                                      (59U)
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC2_CLRMSK                                     (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC2_EN                                         (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC1_SHIFT                                      (58U)
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC1_EN                                         (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC0_SHIFT                                      (57U)
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC0_EN                                         (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC2_SHIFT                                      (56U)
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC2_EN                                         (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC1_SHIFT                                      (55U)
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC1_EN                                         (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC0_SHIFT                                      (54U)
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC0_EN                                         (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC2_SHIFT                                      (53U)
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC2_EN                                         (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC1_SHIFT                                      (52U)
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC1_EN                                         (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC0_SHIFT                                      (51U)
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC0_EN                                         (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC2_SHIFT                                      (50U)
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC2_EN                                         (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC1_SHIFT                                      (49U)
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC1_EN                                         (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC0_SHIFT                                      (48U)
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC0_EN                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC2_SHIFT                                      (47U)
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC2_EN                                         (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC1_SHIFT                                      (46U)
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC1_EN                                         (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC0_SHIFT                                      (45U)
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC0_EN                                         (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC2_SHIFT                                      (44U)
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC2_EN                                         (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC1_SHIFT                                      (43U)
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC1_EN                                         (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC0_SHIFT                                      (42U)
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC0_EN                                         (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC2_SHIFT                                      (41U)
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC2_EN                                         (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC1_SHIFT                                      (40U)
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC1_EN                                         (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC0_SHIFT                                      (39U)
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC0_EN                                         (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC2_SHIFT                                      (38U)
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC2_EN                                         (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC1_SHIFT                                      (37U)
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC1_EN                                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC0_SHIFT                                      (36U)
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC0_EN                                         (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC2_SHIFT                                      (35U)
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC2_EN                                         (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC1_SHIFT                                      (34U)
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC1_EN                                         (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC0_SHIFT                                      (33U)
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC0_EN                                         (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC2_SHIFT                                      (32U)
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC2_EN                                         (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC1_SHIFT                                      (31U)
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC1_EN                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC0_SHIFT                                      (30U)
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC0_EN                                         (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC2_SHIFT                                       (29U)
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC2_EN                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC1_SHIFT                                       (28U)
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC1_EN                                          (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC0_SHIFT                                       (27U)
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC0_EN                                          (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC2_SHIFT                                       (26U)
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC2_EN                                          (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC1_SHIFT                                       (25U)
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC1_EN                                          (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC0_SHIFT                                       (24U)
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC0_EN                                          (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC2_SHIFT                                       (23U)
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC2_EN                                          (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC1_SHIFT                                       (22U)
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC1_EN                                          (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC0_SHIFT                                       (21U)
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC0_EN                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC2_SHIFT                                       (20U)
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC2_EN                                          (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC1_SHIFT                                       (19U)
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC1_EN                                          (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC0_SHIFT                                       (18U)
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC0_EN                                          (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC2_SHIFT                                       (17U)
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC2_EN                                          (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC1_SHIFT                                       (16U)
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC1_EN                                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC0_SHIFT                                       (15U)
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC0_EN                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC2_SHIFT                                       (14U)
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC2_EN                                          (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC1_SHIFT                                       (13U)
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC1_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC0_SHIFT                                       (12U)
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC0_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC2_SHIFT                                       (11U)
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC1_SHIFT                                       (10U)
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC0_SHIFT                                       (9U)
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC2_SHIFT                                       (8U)
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC1_SHIFT                                       (7U)
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC0_SHIFT                                       (6U)
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC2_SHIFT                                       (5U)
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC1_SHIFT                                       (4U)
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC0_SHIFT                                       (3U)
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC2_SHIFT                                       (2U)
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC1_SHIFT                                       (1U)
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC0_SHIFT                                       (0U)
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_RESET
+*/
+#define RGX_CR_FWCORE_WDT_RESET                                                     (0x4500U)
+#define RGX_CR_FWCORE_WDT_RESET_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WDT_RESET_EN_SHIFT                                            (0U)
+#define RGX_CR_FWCORE_WDT_RESET_EN_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_RESET_EN_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_CTRL
+*/
+#define RGX_CR_FWCORE_WDT_CTRL                                                      (0x4508U)
+#define RGX_CR_FWCORE_WDT_CTRL_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFF1F01))
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_SHIFT                                           (16U)
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_CLRMSK                                          (0x0000FFFFU)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_SHIFT                                      (8U)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_CLRMSK                                     (0xFFFFE0FFU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_COUNT
+*/
+#define RGX_CR_FWCORE_WDT_COUNT                                                     (0x4510U)
+#define RGX_CR_FWCORE_WDT_COUNT_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_CLRMSK                                        (0x00000000U)
+
+
+#endif /* RGX_CR_DEFS_KM_H */
+/*****************************************************************************
+ End of file (rgx_cr_defs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxdefs_km.h
new file mode 100644
index 000000000000..3342fceb3a12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxdefs_km.h
@@ -0,0 +1,440 @@
+/*************************************************************************/ /*!
+@Title          Rogue hw definitions (kernel mode)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXDEFS_KM_H
+#define RGXDEFS_KM_H
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+#include RGX_BVNC_CORE_KM_HEADER
+#include RGX_BNC_CONFIG_KM_HEADER
+#endif
+
+#define IMG_EXPLICIT_INCLUDE_HWDEFS
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#include "rgx_cr_defs_km.h"
+#endif
+#undef IMG_EXPLICIT_INCLUDE_HWDEFS
+
+#include "rgx_heap_firmware.h"
+
+/* The following Macros are picked up through BVNC headers for no hardware
+ * operations to be compatible with old build infrastructure.
+ */
+#if defined(NO_HARDWARE)
+/******************************************************************************
+ * Check for valid B.X.N.C
+ *****************************************************************************/
+#if !defined(RGX_BVNC_KM_B) || !defined(RGX_BVNC_KM_V) || !defined(RGX_BVNC_KM_N) || !defined(RGX_BVNC_KM_C)
+#error "Need to specify BVNC (RGX_BVNC_KM_B, RGX_BVNC_KM_V, RGX_BVNC_KM_N and RGX_BVNC_C)"
+#endif
+
+/* Check core/config compatibility */
+#if (RGX_BVNC_KM_B != RGX_BNC_KM_B) || (RGX_BVNC_KM_N != RGX_BNC_KM_N) || (RGX_BVNC_KM_C != RGX_BNC_KM_C)
+#error "BVNC headers are mismatching (KM core/config)"
+#endif
+#endif
+
+/******************************************************************************
+ * RGX Version name
+ *****************************************************************************/
+#define RGX_BVNC_KM_ST2(S)	#S
+#define RGX_BVNC_KM_ST(S)	RGX_BVNC_KM_ST2(S)
+#define RGX_BVNC_KM			RGX_BVNC_KM_ST(RGX_BVNC_KM_B) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_V) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_N) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_C)
+#define RGX_BVNC_KM_V_ST	RGX_BVNC_KM_ST(RGX_BVNC_KM_V)
+
+/* Maximum string size is [bb.vvvp.nnnn.cccc\0], includes null char */
+#define RGX_BVNC_STR_SIZE_MAX (2+1+4+1+4+1+4+1)
+#define RGX_BVNC_STR_FMTSPEC  "%u.%u.%u.%u"
+#define RGX_BVNC_STRP_FMTSPEC "%u.%up.%u.%u"
+
+
+/******************************************************************************
+ * RGX Defines
+ *****************************************************************************/
+
+#define BVNC_FIELD_MASK     ((1 << BVNC_FIELD_WIDTH) - 1)
+#define C_POSITION          (0)
+#define N_POSITION          ((C_POSITION) + (BVNC_FIELD_WIDTH))
+#define V_POSITION          ((N_POSITION) + (BVNC_FIELD_WIDTH))
+#define B_POSITION          ((V_POSITION) + (BVNC_FIELD_WIDTH))
+
+#define B_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (B_POSITION)))
+#define V_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (V_POSITION)))
+#define N_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (N_POSITION)))
+#define C_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (C_POSITION)))
+
+#define GET_B(x)            (((x) & (B_POSTION_MASK)) >> (B_POSITION))
+#define GET_V(x)            (((x) & (V_POSTION_MASK)) >> (V_POSITION))
+#define GET_N(x)            (((x) & (N_POSTION_MASK)) >> (N_POSITION))
+#define GET_C(x)            (((x) & (C_POSTION_MASK)) >> (C_POSITION))
+
+#define BVNC_PACK(B,V,N,C)  ((((IMG_UINT64)B)) << (B_POSITION) | \
+                             (((IMG_UINT64)V)) << (V_POSITION) | \
+                             (((IMG_UINT64)N)) << (N_POSITION) | \
+                             (((IMG_UINT64)C)) << (C_POSITION) \
+                            )
+
+#define RGX_CR_CORE_ID_CONFIG_N_SHIFT    (8U)
+#define RGX_CR_CORE_ID_CONFIG_C_SHIFT    (0U)
+
+#define RGX_CR_CORE_ID_CONFIG_N_CLRMSK   (0XFFFF00FFU)
+#define RGX_CR_CORE_ID_CONFIG_C_CLRMSK   (0XFFFFFF00U)
+
+/* The default number of OSID is 1, higher number implies VZ enabled firmware */
+#if !defined(RGXFW_NATIVE) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED + 1U > 1U)
+#define RGXFW_NUM_OS RGX_NUM_DRIVERS_SUPPORTED
+#else
+#define RGXFW_NUM_OS 1U
+#endif
+
+#if defined(RGX_FEATURE_NUM_OSIDS)
+#define RGXFW_MAX_NUM_OSIDS                               (RGX_FEATURE_NUM_OSIDS)
+#else
+#define RGXFW_MAX_NUM_OSIDS                               (8U)
+#endif
+
+#define RGXFW_HOST_DRIVER_ID                              (0U)
+#define RGXFW_GUEST_DRIVER_ID_START                       (RGXFW_HOST_DRIVER_ID + 1U)
+
+#define RGXFW_THREAD_0                                    (0U)
+#define RGXFW_THREAD_1                                    (1U)
+
+/* META cores (required for the RGX_FEATURE_META) */
+#define MTP218   (1)
+#define MTP219   (2)
+#define LTP218   (3)
+#define LTP217   (4)
+
+/* META Core memory feature depending on META variants */
+#define RGX_META_COREMEM_32K      (32*1024)
+#define RGX_META_COREMEM_48K      (48*1024)
+#define RGX_META_COREMEM_64K      (64*1024)
+#define RGX_META_COREMEM_96K      (96*1024)
+#define RGX_META_COREMEM_128K     (128*1024)
+#define RGX_META_COREMEM_256K     (256*1024)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(RGX_FEATURE_META_DMA)
+#undef SUPPORT_META_COREMEM
+#undef RGX_FEATURE_META_COREMEM_SIZE
+#define RGX_FEATURE_META_COREMEM_SIZE (0)
+#define RGX_META_COREMEM_SIZE         (0)
+#elif defined(RGX_FEATURE_META_COREMEM_SIZE)
+#define RGX_META_COREMEM_SIZE         (RGX_FEATURE_META_COREMEM_SIZE*1024U)
+#else
+#define RGX_META_COREMEM_SIZE         (0)
+#endif
+
+#if RGX_META_COREMEM_SIZE != 0
+#define RGX_META_COREMEM
+#define RGX_META_COREMEM_CODE
+#define RGX_META_COREMEM_DATA
+#endif
+#endif
+
+#define GET_ROGUE_CACHE_LINE_SIZE(x)    ((((IMG_INT32)x) > 0) ? ((x)/8) : (0))
+
+#if defined(RGX_FEATURE_META_DMA)
+#define RGX_META_DMA_BLOCK_SIZE (32U)
+#else
+#define RGX_META_DMA_BLOCK_SIZE (0U)
+#endif
+
+#if defined(SUPPORT_AGP)
+#if defined(SUPPORT_AGP4)
+#define MAX_HW_TA3DCONTEXTS	5U
+#else
+#define MAX_HW_TA3DCONTEXTS	3U
+#endif
+#else
+#define MAX_HW_TA3DCONTEXTS	2U
+#endif
+
+#define RGX_CR_CLK_CTRL0_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL0_MASKFULL)
+#define RGX_CR_CLK_CTRL0_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL0_MASKFULL)
+#define RGX_CR_CLK_CTRL1_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL1_MASKFULL)
+#define RGX_CR_CLK_CTRL1_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL1_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL2_MASKFULL)
+
+#if defined(__KERNEL__)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL0__PIPEDM_GT0__MASKFULL)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL0__PIPEDM_GT0__MASKFULL)
+#endif
+
+#define RGX_CR_MERCER0_SOFT_RESET_SPU_EN (RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_EN)
+
+#define RGX_CR_MERCER1_SOFT_RESET_SPU_EN (RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_EN)
+
+#define RGX_CR_MERCER2_SOFT_RESET_SPU_EN (RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_EN)
+
+
+/* SOFT_RESET steps as defined in the TRM */
+#define RGX_SOFT_RESET_JONES             (RGX_CR_SOFT_RESET_PM_EN | \
+                                          RGX_CR_SOFT_RESET_ISP_EN)
+#define RGX_SOFT_RESET_JONES_ALL         (RGX_SOFT_RESET_JONES | \
+                                          RGX_CR_SOFT_RESET_BIF_TEXAS_EN | \
+                                          RGX_CR_SOFT_RESET_BIF_JONES_EN | \
+										  RGX_CR_SOFT_RESET_SLC_EN | \
+                                          RGX_CR_SOFT_RESET_GARTEN_EN)
+#define RGX_SOFT_RESET_EXTRA             (RGX_CR_SOFT_RESET_PIXEL_EN | \
+                                          RGX_CR_SOFT_RESET_VERTEX_EN |  \
+                                          RGX_CR_SOFT_RESET_GEO_VERTEX_EN | \
+                                          RGX_CR_SOFT_RESET_GEO_SHARED_EN | \
+                                          RGX_CR_SOFT_RESET_COMPUTE_EN | \
+                                          RGX_CR_SOFT_RESET_TDM_EN)
+#define RGX_SOFT_RESET_FROM_WITHIN_CORE  (RGX_CR_SOFT_RESET_MASKFULL ^ \
+                                          (RGX_CR_SOFT_RESET_GARTEN_EN | \
+                                           RGX_CR_SOFT_RESET_BIF_JONES_EN | \
+                                           RGX_CR_SOFT_RESET_SLC_EN))
+
+#define RGX_MLIST_ENTRY_STRIDE	(4U) /* 4 bytes */
+#define RGX_NUM_PM_ADDR_SPACES	(2U) /* VCE & TE share virtual space and Alist */
+#define RGX_PM_MAX_PB_VIRT_ADDR_SPACE  (IMG_UINT64_C(0x400000000)) /* PM Maximum addressable limit */
+
+
+#define RGX_PM_MAX_RSTATE_SIZE_DWORDS (46U)
+#define RGX_PM_MLIST_BASE_ADDR_MAX_ALIGNSIZE (32U)
+#define RGX_PM_VHEAP_BASE_ADDR_MAX_ALIGNSIZE (32U)
+#define _RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_)[2] = ((_x_) & (IMG_UINT64_C(0x00000000fffffff0)))); \
+                                                          ((_ft_)[3] = (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32)); }
+#define _RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(_ft_,_x_) { ((_ft_)[6] = ((_x_) & (IMG_UINT64_C(0x00000000fffffff0)))); \
+                                                          ((_ft_)[7] = (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32)); }
+
+#define RGX_PM_RENDERSTATE_BASE_ADDR_ALIGNSIZE (32U)
+
+#define RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT		(12U)
+#define RGX_BIF_PM_PHYSICAL_PAGE_SIZE			(1U << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT		(14U)
+#define RGX_BIF_PM_VIRTUAL_PAGE_SIZE			(1U << RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE	(32U)
+
+/* To get the number of required Bernado/Phantom(s), divide
+ * the number of clusters by 4 and round up
+ */
+#define RGX_REQ_NUM_PHANTOMS(CLUSTERS) ((CLUSTERS + 3U) / 4U)
+#define RGX_REQ_NUM_BERNADOS(CLUSTERS) ((CLUSTERS + 3U) / 4U)
+#define RGX_REQ_NUM_BLACKPEARLS(CLUSTERS) ((CLUSTERS + 3U) / 4U)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+# define RGX_NUM_PHANTOMS (RGX_REQ_NUM_PHANTOMS(RGX_FEATURE_NUM_CLUSTERS))
+#endif
+
+/* for nohw multicore, true max number of cores returned to client */
+#define RGX_MULTICORE_MAX_NOHW_CORES                (8U)
+
+/*
+ * META second thread feature depending on META variants and
+ * available CoreMem
+ */
+#if defined(RGX_FEATURE_META) && (RGX_FEATURE_META == MTP218 || RGX_FEATURE_META == MTP219) && (RGX_FEATURE_META_COREMEM_SIZE >= 96) && (RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION == 0)
+#define RGXFW_META_SUPPORT_2ND_THREAD
+#endif
+
+/*
+ * FWCORE wrapper register defines
+ */
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT   RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK  RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK
+#define FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT     (12U)
+
+
+/*
+ * FBC clear color register defaults based on HW defaults
+ * non-YUV clear colour 0: 0x00000000 (encoded as ch3,2,1,0)
+ * non-YUV clear colour 1: 0x01000000 (encoded as ch3,2,1,0)
+ * YUV clear colour 0: 0x000 000 (encoded as UV Y)
+ * YUV clear colour 1: 0x000 3FF (encoded as UV Y)
+ */
+#define RGX_FBC_CC_DEFAULT (0x0100000000000000)
+#define RGX_FBC_CC_YUV_DEFAULT (0x000003FF00000000)
+
+/*
+ * Virtualisation definitions
+ */
+
+#define RGX_VIRTUALISATION_REG_SIZE_PER_OS (RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE)
+
+/*
+ * Renaming MTS sideband bitfields to emphasize that the Register Bank number
+ * of the MTS register used identifies a specific Driver/VM rather than the OSID tag
+ * emitted on bus memory transactions.
+ */
+#define RGX_MTS_SBDATA_DRIVERID_CLRMSK RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_CLRMSK
+#define RGX_MTS_SBDATA_DRIVERID_SHIFT RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_SHIFT
+
+/* Register Bank containing registers secured against host access */
+#define RGX_HOST_SECURE_REGBANK_OFFSET				(0xF0000U)
+#define RGX_HOST_SECURE_REGBANK_SIZE				(0x10000U)
+
+/* GPU CR timer tick in GPU cycles */
+#define RGX_CRTIME_TICK_IN_CYCLES					(256U)
+#define RGX_CRTIME_TICK_IN_CYCLES_SHIFT				(8U)
+
+#if defined(FIX_HW_BRN_71840)
+#define ROGUE_RENDERSIZE_MAXX						(16384U)
+#define ROGUE_RENDERSIZE_MAXY						(16384U)
+#else
+#define ROGUE_RENDERSIZE_MAXX						(RGX_FEATURE_RENDER_TARGET_XY_MAX)
+#define ROGUE_RENDERSIZE_MAXY						(RGX_FEATURE_RENDER_TARGET_XY_MAX)
+#endif
+
+/*
+ * Register used by the FW to track the current boot stage (not used in MIPS)
+ */
+#define RGX_FW_BOOT_STAGE_REGISTER     (RGX_CR_SCRATCH14)
+
+/*
+ * Define used to determine whether or not SLC range-based flush/invalidate
+ * interface is supported.
+ */
+#define RGX_SRV_SLC_RANGEBASED_CFI_SUPPORTED 1
+
+/*
+ * Macro used to indicate which version of HWPerf is active
+ */
+#define RGX_FEATURE_HWPERF_VOLCANIC
+
+/*
+ * Maximum number of cores supported by TRP
+ */
+#define RGX_TRP_MAX_NUM_CORES                           (8U)
+
+/*
+ * Maximum number of cores supported by WGP
+ */
+#define RGX_WGP_MAX_NUM_CORES                           (8U)
+
+/*
+ * Supports command to invalidate FBCDC descriptor state cache
+ */
+#define RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED 1
+
+#if defined(FIX_HW_BRN_71422)
+/*
+ * The BRN71422 software workaround requires a target physical address on
+ * the hardware platform with a low latency response time and which will
+ * not suffer from delays of DRAM hardware operations such as refresh and
+ * recalibration. Only with that address defined will the workaround be used.
+ */
+#if !defined(PDUMP)
+//#define RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR  (IMG_UINT64_C(0x0000000000))
+#endif
+#define RGX_BRN71422_WORKAROUND_READ_SIZE           (32U)
+#endif
+
+#define SUPPORT_VOLCANIC_TB
+#define RGX_FEATURE_SECURITY_VOLCANIC
+
+/* Typically the PCI bus returns this value on error */
+#define RGX_PCI_ERROR_VALUE_BYTE (0xFFU)
+#define RGX_PCI_ERROR_VALUE_DWORD (0xFFFFFFFFU)
+
+
+/*
+	Maximum number of render targets in array.
+	This derives from the maximum value for RGX_FEATURE_RTA_INDICES across all cores.
+*/
+#define RGX_MAX_RTA_INDICES (2048U)
+
+#endif /* RGXDEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxmmudefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxmmudefs_km.h
new file mode 100644
index 000000000000..4dedb33467e6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxmmudefs_km.h
@@ -0,0 +1,278 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmmudefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_bif.def
+ */
+
+
+#ifndef RGXMMUDEFS_KM_H
+#define RGXMMUDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMMUDEFS_KM_REVISION 0
+
+#define OSID_CTXT_MAPPING_REGISTERS_CONSTANTS_OSID_CTXT_MAPPING_NUM_ENTRIES_VALUE   (0x00000010U)
+
+
+#define OSID_CTXT_MAPPING_REGISTERS_CONSTANTS_OSID_CTXT_MAPPING_PER_OS_SHIFT_VALUE  (0x00000004U)
+
+
+#define OSID_CTXT_MAPPING_REGISTERS_CONSTANTS_OSID_CTXT_MAPPING_PER_OS_MASK_VALUE   (0x00000007U)
+
+
+#define RGX_BIF_DM_ENCODING_VERTEX                                                  (0x00000000U)
+#define RGX_BIF_DM_ENCODING_PIXEL                                                   (0x00000001U)
+#define RGX_BIF_DM_ENCODING_COMPUTE                                                 (0x00000002U)
+#define RGX_BIF_DM_ENCODING_TLA                                                     (0x00000003U)
+#define RGX_BIF_DM_ENCODING_PB_VCE                                                  (0x00000004U)
+#define RGX_BIF_DM_ENCODING_PB_TE                                                   (0x00000005U)
+#define RGX_BIF_DM_ENCODING_META                                                    (0x00000007U)
+#define RGX_BIF_DM_ENCODING_HOST                                                    (0x00000008U)
+#define RGX_BIF_DM_ENCODING_PM_ALIST                                                (0x00000009U)
+
+
+#define RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT                                            (30U)
+#define RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT                                            (21U)
+#define RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFC01FFFFF))
+#define RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT                                            (12U)
+#define RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFE00FFF))
+
+
+#define RGX_MMUCTRL_ENTRIES_PC_VALUE                                                (0x00000400U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PD_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PT_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PC_VALUE                                             (0x00000020U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PD_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PT_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_PAGE_SIZE_MASK                                                  (0x00000007U)
+#define RGX_MMUCTRL_PAGE_SIZE_4KB                                                   (0x00000000U)
+#define RGX_MMUCTRL_PAGE_SIZE_16KB                                                  (0x00000001U)
+#define RGX_MMUCTRL_PAGE_SIZE_64KB                                                  (0x00000002U)
+#define RGX_MMUCTRL_PAGE_SIZE_256KB                                                 (0x00000003U)
+#define RGX_MMUCTRL_PAGE_SIZE_1MB                                                   (0x00000004U)
+#define RGX_MMUCTRL_PAGE_SIZE_2MB                                                   (0x00000005U)
+
+
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT                                            (12U)
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_SHIFT                                           (14U)
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF0000003FFF))
+
+
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_SHIFT                                           (16U)
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_SHIFT                                          (18U)
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFF000003FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_SHIFT                                            (20U)
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00000FFFFF))
+
+
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_SHIFT                                            (21U)
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00001FFFFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_SHIFT                                         (12U)
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_SHIFT                                        (10U)
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000003FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_SHIFT                                        (8U)
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000000FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_SHIFT                                       (6U)
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF000000003F))
+
+
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_AXCACHE_MASK                                                    (0x0000000FU)
+/*
+Device Non-bufferable */
+#define RGX_MMUCTRL_AXCACHE_DEVNONBUFF                                              (0x00000000U)
+/*
+Device Bufferable */
+#define RGX_MMUCTRL_AXCACHE_DEVBUFF                                                 (0x00000001U)
+/*
+Normal Non-cacheable Non-bufferable */
+#define RGX_MMUCTRL_AXCACHE_NORMNONBUFF                                             (0x00000002U)
+/*
+Normal Non-cacheable Bufferable */
+#define RGX_MMUCTRL_AXCACHE_NORMBUFF                                                (0x00000003U)
+/*
+Write-through No-allocate*/
+#define RGX_MMUCTRL_AXCACHE_WTNOALLOC                                               (0x00000006U)
+/*
+Write-back No-allocate*/
+#define RGX_MMUCTRL_AXCACHE_WBNOALLOC                                               (0x00000007U)
+/*
+Write-through Read-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WTRALLOC                                                (0x00000008U)
+/*
+Write-back Read-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WBRALLOC                                                (0x00000009U)
+/*
+Write-through Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WTWALLOC                                                (0x0000000aU)
+/*
+Write-back Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WBWALLOC                                                (0x0000000bU)
+/*
+Write-through Read/Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WTRWALLOC                                               (0x0000000eU)
+/*
+Write-back Read/Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WBRWALLOC                                               (0x0000000fU)
+
+
+#define RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT                                         (63U)
+#define RGX_MMUCTRL_PT_DATA_PT_PARITY_CLRMSK                                        (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PT_PARITY_EN                                            (IMG_UINT64_C(0x8000000000000000))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_SHIFT                                   (62U)
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_CLRMSK                                  (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_SHIFT                                           (58U)
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_CLRMSK                                          (IMG_UINT64_C(0xC3FFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_DEVNONBUFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_DEVBUFF                                         (IMG_UINT64_C(0x0400000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_NORMNONBUFF                                     (IMG_UINT64_C(0x0800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_NORMBUFF                                        (IMG_UINT64_C(0x0c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTNOALLOC                                       (IMG_UINT64_C(0x1800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBNOALLOC                                       (IMG_UINT64_C(0x1c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTRALLOC                                        (IMG_UINT64_C(0x2000000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBRALLOC                                        (IMG_UINT64_C(0x2400000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTWALLOC                                        (IMG_UINT64_C(0x2800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBWALLOC                                        (IMG_UINT64_C(0x2c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTRWALLOC                                       (IMG_UINT64_C(0x3800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC                                       (IMG_UINT64_C(0x3c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_SHIFT                                        (40U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_CLRMSK                                       (IMG_UINT64_C(0xFC0000FFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PAGE_SHIFT                                              (12U)
+#define RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_SHIFT                                        (6U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_SHIFT                                     (5U)
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000000000000020))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_SHIFT                                            (4U)
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_MMUCTRL_PT_DATA_CC_SHIFT                                                (2U)
+#define RGX_MMUCTRL_PT_DATA_CC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_MMUCTRL_PT_DATA_CC_EN                                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_EN                                            (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PT_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PT_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PT_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_SHIFT                                     (40U)
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_SHIFT                                           (5U)
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF1))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB                                         (IMG_UINT64_C(0x0000000000000006))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB                                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB                                           (IMG_UINT64_C(0x000000000000000a))
+#define RGX_MMUCTRL_PD_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PD_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PD_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT                                           (4U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK                                          (0x0000000FU)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT                                      (12U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE                                       (4096U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_SHIFT                                     (1U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN                                        (0x00000002U)
+#define RGX_MMUCTRL_PC_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PC_DATA_VALID_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_MMUCTRL_PC_DATA_VALID_EN                                                (0x00000001U)
+
+
+#endif /* RGXMMUDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmmudefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxtbdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxtbdefs_km.h
new file mode 100644
index 000000000000..c265ef0059b2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxtbdefs_km.h
@@ -0,0 +1,1899 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxtbdefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGXTBDEFS_KM_H
+#define RGXTBDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXTBDEFS_KM_REVISION 1
+
+/*
+    Register RGX_TB_SOFT_RESET
+*/
+#define RGX_TB_SOFT_RESET                                                           (0x0000U)
+#define RGX_TB_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x00000000FFFF0107))
+#define RGX_TB_SOFT_RESET_SPU_SHIFT                                                 (16U)
+#define RGX_TB_SOFT_RESET_SPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_TB_SOFT_RESET_JONES_SHIFT                                               (8U)
+#define RGX_TB_SOFT_RESET_JONES_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_TB_SOFT_RESET_JONES_EN                                                  (IMG_UINT64_C(0x0000000000000100))
+#define RGX_TB_SOFT_RESET_SYS_SHIFT                                                 (2U)
+#define RGX_TB_SOFT_RESET_SYS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_TB_SOFT_RESET_SYS_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_TB_SOFT_RESET_MEM_SHIFT                                                 (1U)
+#define RGX_TB_SOFT_RESET_MEM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_TB_SOFT_RESET_MEM_EN                                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_TB_SOFT_RESET_CORE_SHIFT                                                (0U)
+#define RGX_TB_SOFT_RESET_CORE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_SOFT_RESET_CORE_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_PCI_MASTER
+*/
+#define RGX_TB_PCI_MASTER                                                           (0x0008U)
+#define RGX_TB_PCI_MASTER_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_PCI_MASTER_MODE_SHIFT                                                (0U)
+#define RGX_TB_PCI_MASTER_MODE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_PCI_MASTER_MODE_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_MEM_ARBITER
+*/
+#define RGX_TB_MEM_ARBITER                                                          (0x0088U)
+#define RGX_TB_MEM_ARBITER_MASKFULL                                                 (IMG_UINT64_C(0x0000000000010F11))
+#define RGX_TB_MEM_ARBITER_LIMIT_BW_SHIFT                                           (16U)
+#define RGX_TB_MEM_ARBITER_LIMIT_BW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_TB_MEM_ARBITER_LIMIT_BW_EN                                              (IMG_UINT64_C(0x0000000000010000))
+#define RGX_TB_MEM_ARBITER_PRI_SKEW_SHIFT                                           (8U)
+#define RGX_TB_MEM_ARBITER_PRI_SKEW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFF0FF))
+#define RGX_TB_MEM_ARBITER_PRI_RNW_SHIFT                                            (4U)
+#define RGX_TB_MEM_ARBITER_PRI_RNW_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_TB_MEM_ARBITER_PRI_RNW_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_TB_MEM_ARBITER_ENABLE_SHIFT                                             (0U)
+#define RGX_TB_MEM_ARBITER_ENABLE_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_MEM_ARBITER_ENABLE_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_QOS_RD_LATENCY
+*/
+#define RGX_TB_QOS_RD_LATENCY                                                       (0x0090U)
+#define RGX_TB_QOS_RD_LATENCY_MASKFULL                                              (IMG_UINT64_C(0xFFFF3FFF3FFF3FFF))
+#define RGX_TB_QOS_RD_LATENCY_DIST_SHIFT                                            (62U)
+#define RGX_TB_QOS_RD_LATENCY_DIST_CLRMSK                                           (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_TB_QOS_RD_LATENCY_MAX_15_SHIFT                                          (48U)
+#define RGX_TB_QOS_RD_LATENCY_MAX_15_CLRMSK                                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_TB_QOS_RD_LATENCY_MIN_15_SHIFT                                          (32U)
+#define RGX_TB_QOS_RD_LATENCY_MIN_15_CLRMSK                                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_TB_QOS_RD_LATENCY_MAX_0_SHIFT                                           (16U)
+#define RGX_TB_QOS_RD_LATENCY_MAX_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFC000FFFF))
+#define RGX_TB_QOS_RD_LATENCY_MIN_0_SHIFT                                           (0U)
+#define RGX_TB_QOS_RD_LATENCY_MIN_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFC000))
+
+
+/*
+    Register RGX_TB_QOS_WR_LATENCY
+*/
+#define RGX_TB_QOS_WR_LATENCY                                                       (0x0098U)
+#define RGX_TB_QOS_WR_LATENCY_MASKFULL                                              (IMG_UINT64_C(0xFFFF3FFF3FFF3FFF))
+#define RGX_TB_QOS_WR_LATENCY_DIST_SHIFT                                            (62U)
+#define RGX_TB_QOS_WR_LATENCY_DIST_CLRMSK                                           (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_TB_QOS_WR_LATENCY_MAX_15_SHIFT                                          (48U)
+#define RGX_TB_QOS_WR_LATENCY_MAX_15_CLRMSK                                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_TB_QOS_WR_LATENCY_MIN_15_SHIFT                                          (32U)
+#define RGX_TB_QOS_WR_LATENCY_MIN_15_CLRMSK                                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_TB_QOS_WR_LATENCY_MAX_0_SHIFT                                           (16U)
+#define RGX_TB_QOS_WR_LATENCY_MAX_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFC000FFFF))
+#define RGX_TB_QOS_WR_LATENCY_MIN_0_SHIFT                                           (0U)
+#define RGX_TB_QOS_WR_LATENCY_MIN_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFC000))
+
+
+/*
+    Register RGX_TB_MAX_ID_OUTSTANDING
+*/
+#define RGX_TB_MAX_ID_OUTSTANDING                                                   (0x00B0U)
+#define RGX_TB_MAX_ID_OUTSTANDING_MASKFULL                                          (IMG_UINT64_C(0x000003FF03FF03FF))
+#define RGX_TB_MAX_ID_OUTSTANDING_RD_WR_SHIFT                                       (32U)
+#define RGX_TB_MAX_ID_OUTSTANDING_RD_WR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_TB_MAX_ID_OUTSTANDING_WRITE_SHIFT                                       (16U)
+#define RGX_TB_MAX_ID_OUTSTANDING_WRITE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_TB_MAX_ID_OUTSTANDING_READ_SHIFT                                        (0U)
+#define RGX_TB_MAX_ID_OUTSTANDING_READ_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFC00))
+
+
+/*
+    Register RGX_TB_COHERENT_MEM_REGION
+*/
+#define RGX_TB_COHERENT_MEM_REGION                                                  (0x00C0U)
+#define RGX_TB_COHERENT_MEM_REGION_MASKFULL                                         (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_TB_COHERENT_MEM_REGION_START_ADDR_SHIFT                                 (12U)
+#define RGX_TB_COHERENT_MEM_REGION_START_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+/*
+    Register RGX_TB_LMA_MEM_REGION
+*/
+#define RGX_TB_LMA_MEM_REGION                                                       (0x00C8U)
+#define RGX_TB_LMA_MEM_REGION_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_LMA_MEM_REGION_SIZE_SHIFT                                            (0U)
+#define RGX_TB_LMA_MEM_REGION_SIZE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_UMA_MEM_REGION
+*/
+#define RGX_TB_UMA_MEM_REGION                                                       (0x00D0U)
+#define RGX_TB_UMA_MEM_REGION_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_TB_UMA_MEM_REGION_START_ADDR_SHIFT                                      (12U)
+#define RGX_TB_UMA_MEM_REGION_START_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+/*
+    Register RGX_TB_SYSTEM_STATUS
+*/
+#define RGX_TB_SYSTEM_STATUS                                                        (0x00E0U)
+#define RGX_TB_SYSTEM_STATUS_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFF3FF700FF))
+#define RGX_TB_SYSTEM_STATUS_SPU_ISON_SHIFT                                         (48U)
+#define RGX_TB_SYSTEM_STATUS_SPU_ISON_CLRMSK                                        (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_SPU_POWER_SHIFT                                        (32U)
+#define RGX_TB_SYSTEM_STATUS_SPU_POWER_CLRMSK                                       (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_DAVY_ISON_SHIFT                                        (29U)
+#define RGX_TB_SYSTEM_STATUS_DAVY_ISON_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_DAVY_ISON_EN                                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_TB_SYSTEM_STATUS_DAVY_POWER_SHIFT                                       (28U)
+#define RGX_TB_SYSTEM_STATUS_DAVY_POWER_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_DAVY_POWER_EN                                          (IMG_UINT64_C(0x0000000010000000))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_ISON_SHIFT                                 (27U)
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_ISON_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_ISON_EN                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_POWER_SHIFT                                (26U)
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_POWER_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_POWER_EN                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_ABORT_SHIFT                           (25U)
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_ABORT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_ABORT_EN                              (IMG_UINT64_C(0x0000000002000000))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_COMPLETE_SHIFT                        (24U)
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_COMPLETE_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_COMPLETE_EN                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_TB_SYSTEM_STATUS_GPU_STATE_SHIFT                                        (20U)
+#define RGX_TB_SYSTEM_STATUS_GPU_STATE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFF0FFFFF))
+#define RGX_TB_SYSTEM_STATUS_SYSTEM_IRQ_SHIFT                                       (18U)
+#define RGX_TB_SYSTEM_STATUS_SYSTEM_IRQ_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_TB_SYSTEM_STATUS_SYSTEM_IRQ_EN                                          (IMG_UINT64_C(0x0000000000040000))
+#define RGX_TB_SYSTEM_STATUS_TRIGGER_SHIFT                                          (17U)
+#define RGX_TB_SYSTEM_STATUS_TRIGGER_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_TB_SYSTEM_STATUS_TRIGGER_EN                                             (IMG_UINT64_C(0x0000000000020000))
+#define RGX_TB_SYSTEM_STATUS_HMMU_IRQ_SHIFT                                         (16U)
+#define RGX_TB_SYSTEM_STATUS_HMMU_IRQ_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_TB_SYSTEM_STATUS_HMMU_IRQ_EN                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_TB_SYSTEM_STATUS_IRQ_SHIFT                                              (0U)
+#define RGX_TB_SYSTEM_STATUS_IRQ_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_TB_SYSTEM_CONFIG
+*/
+#define RGX_TB_SYSTEM_CONFIG                                                        (0x00F0U)
+#define RGX_TB_SYSTEM_CONFIG_MASKFULL                                               (IMG_UINT64_C(0x0000000007007737))
+#define RGX_TB_SYSTEM_CONFIG_SOC_AXI_FEATURE_SHIFT                                  (24U)
+#define RGX_TB_SYSTEM_CONFIG_SOC_AXI_FEATURE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_TB_SYSTEM_CONFIG_ICE_CABLEIF256_SHIFT                                   (14U)
+#define RGX_TB_SYSTEM_CONFIG_ICE_CABLEIF256_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_TB_SYSTEM_CONFIG_ICE_CABLEIF256_EN                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_TBX_SHIFT                                       (13U)
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_TBX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_TBX_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_ICE_SHIFT                                       (12U)
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_ICE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_ICE_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_AVIP_SHIFT                                     (10U)
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_AVIP_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_AVIP_EN                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_TBA_SHIFT                                      (9U)
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_TBA_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_TBA_EN                                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_ICE_SHIFT                                      (8U)
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_ICE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_ICE_EN                                         (IMG_UINT64_C(0x0000000000000100))
+#define RGX_TB_SYSTEM_CONFIG_EMU_UMA_SHIFT                                          (5U)
+#define RGX_TB_SYSTEM_CONFIG_EMU_UMA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_TB_SYSTEM_CONFIG_EMU_UMA_EN                                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_TB_SYSTEM_CONFIG_EMU_BUILD_SHIFT                                        (4U)
+#define RGX_TB_SYSTEM_CONFIG_EMU_BUILD_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_TB_SYSTEM_CONFIG_EMU_BUILD_EN                                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_TB_SYSTEM_CONFIG_NS_NOC_SHIFT                                           (2U)
+#define RGX_TB_SYSTEM_CONFIG_NS_NOC_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_TB_SYSTEM_CONFIG_NS_NOC_EN                                              (IMG_UINT64_C(0x0000000000000004))
+#define RGX_TB_SYSTEM_CONFIG_IMG_NOC_SHIFT                                          (1U)
+#define RGX_TB_SYSTEM_CONFIG_IMG_NOC_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_TB_SYSTEM_CONFIG_IMG_NOC_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_TB_SYSTEM_CONFIG_TB_NONCOHERENT_SHIFT                                   (0U)
+#define RGX_TB_SYSTEM_CONFIG_TB_NONCOHERENT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_SYSTEM_CONFIG_TB_NONCOHERENT_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_TB_DOMAINSPLIT_TYPE_RESERVED_SHIFT                                      (9U)
+#define RGX_TB_DOMAINSPLIT_TYPE_RESERVED_CLRMSK                                     (0xFFFF01FFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU7_SHIFT                                          (8U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU7_CLRMSK                                         (0xFFFFFEFFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU7_EN                                             (0x00000100U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU6_SHIFT                                          (7U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU6_CLRMSK                                         (0xFFFFFF7FU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU6_EN                                             (0x00000080U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU5_SHIFT                                          (6U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU5_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU5_EN                                             (0x00000040U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU4_SHIFT                                          (5U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU4_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU4_EN                                             (0x00000020U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU3_SHIFT                                          (4U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU3_EN                                             (0x00000010U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU2_SHIFT                                          (3U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU2_EN                                             (0x00000008U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU1_SHIFT                                          (2U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU1_EN                                             (0x00000004U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU0_SHIFT                                          (1U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU0_EN                                             (0x00000002U)
+#define RGX_TB_DOMAINSPLIT_TYPE_JONES_SHIFT                                         (0U)
+#define RGX_TB_DOMAINSPLIT_TYPE_JONES_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_TB_DOMAINSPLIT_TYPE_JONES_EN                                            (0x00000001U)
+
+
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_MASK                                        (0xFFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER31_SHIFT                             (31U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER31_CLRMSK                            (0x7FFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER31_EN                                (0x80000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER30_SHIFT                             (30U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER30_CLRMSK                            (0xBFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER30_EN                                (0x40000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER29_SHIFT                             (29U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER29_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER29_EN                                (0x20000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER28_SHIFT                             (28U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER28_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER28_EN                                (0x10000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER27_SHIFT                             (27U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER27_CLRMSK                            (0xF7FFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER27_EN                                (0x08000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER26_SHIFT                             (26U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER26_CLRMSK                            (0xFBFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER26_EN                                (0x04000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER25_SHIFT                             (25U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER25_CLRMSK                            (0xFDFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER25_EN                                (0x02000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER24_SHIFT                             (24U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER24_CLRMSK                            (0xFEFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER24_EN                                (0x01000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER23_SHIFT                             (23U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER23_CLRMSK                            (0xFF7FFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER23_EN                                (0x00800000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER22_SHIFT                             (22U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER22_CLRMSK                            (0xFFBFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER22_EN                                (0x00400000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER21_SHIFT                             (21U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER21_CLRMSK                            (0xFFDFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER21_EN                                (0x00200000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER20_SHIFT                             (20U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER20_CLRMSK                            (0xFFEFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER20_EN                                (0x00100000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER19_SHIFT                             (19U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER19_CLRMSK                            (0xFFF7FFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER19_EN                                (0x00080000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER18_SHIFT                             (18U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER18_CLRMSK                            (0xFFFBFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER18_EN                                (0x00040000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER17_SHIFT                             (17U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER17_CLRMSK                            (0xFFFDFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER17_EN                                (0x00020000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER16_SHIFT                             (16U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER16_CLRMSK                            (0xFFFEFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER16_EN                                (0x00010000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER15_SHIFT                             (15U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER15_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER15_EN                                (0x00008000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER14_SHIFT                             (14U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER14_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER14_EN                                (0x00004000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER13_SHIFT                             (13U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER13_CLRMSK                            (0xFFFFDFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER13_EN                                (0x00002000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER12_SHIFT                             (12U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER12_CLRMSK                            (0xFFFFEFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER12_EN                                (0x00001000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER11_SHIFT                             (11U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER11_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER11_EN                                (0x00000800U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER10_SHIFT                             (10U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER10_CLRMSK                            (0xFFFFFBFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER10_EN                                (0x00000400U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER9_SHIFT                              (9U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER9_CLRMSK                             (0xFFFFFDFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER9_EN                                 (0x00000200U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER8_SHIFT                              (8U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER8_CLRMSK                             (0xFFFFFEFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER8_EN                                 (0x00000100U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER7_SHIFT                              (7U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER7_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER7_EN                                 (0x00000080U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER6_SHIFT                              (6U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER6_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER6_EN                                 (0x00000040U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER5_SHIFT                              (5U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER5_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER5_EN                                 (0x00000020U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER4_SHIFT                              (4U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER4_CLRMSK                             (0xFFFFFFEFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER4_EN                                 (0x00000010U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER3_SHIFT                              (3U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER3_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER3_EN                                 (0x00000008U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER2_SHIFT                              (2U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER2_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER2_EN                                 (0x00000004U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER1_SHIFT                              (1U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER1_CLRMSK                             (0xFFFFFFFDU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER1_EN                                 (0x00000002U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER0_SHIFT                              (0U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER0_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER0_EN                                 (0x00000001U)
+
+
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC31_SHIFT                                     (31U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC31_CLRMSK                                    (0x7FFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC31_EN                                        (0x80000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC30_SHIFT                                     (30U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC30_CLRMSK                                    (0xBFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC30_EN                                        (0x40000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC29_SHIFT                                     (29U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC29_CLRMSK                                    (0xDFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC29_EN                                        (0x20000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC28_SHIFT                                     (28U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC28_CLRMSK                                    (0xEFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC28_EN                                        (0x10000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC27_SHIFT                                     (27U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC27_CLRMSK                                    (0xF7FFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC27_EN                                        (0x08000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC26_SHIFT                                     (26U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC26_CLRMSK                                    (0xFBFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC26_EN                                        (0x04000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC25_SHIFT                                     (25U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC25_CLRMSK                                    (0xFDFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC25_EN                                        (0x02000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC24_SHIFT                                     (24U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC24_CLRMSK                                    (0xFEFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC24_EN                                        (0x01000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC23_SHIFT                                     (23U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC23_CLRMSK                                    (0xFF7FFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC23_EN                                        (0x00800000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC22_SHIFT                                     (22U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC22_CLRMSK                                    (0xFFBFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC22_EN                                        (0x00400000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC21_SHIFT                                     (21U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC21_CLRMSK                                    (0xFFDFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC21_EN                                        (0x00200000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC20_SHIFT                                     (20U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC20_CLRMSK                                    (0xFFEFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC20_EN                                        (0x00100000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC19_SHIFT                                     (19U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC19_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC19_EN                                        (0x00080000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC18_SHIFT                                     (18U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC18_CLRMSK                                    (0xFFFBFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC18_EN                                        (0x00040000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC17_SHIFT                                     (17U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC17_CLRMSK                                    (0xFFFDFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC17_EN                                        (0x00020000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC16_SHIFT                                     (16U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC16_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC16_EN                                        (0x00010000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC15_SHIFT                                     (15U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC15_CLRMSK                                    (0xFFFF7FFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC15_EN                                        (0x00008000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC14_SHIFT                                     (14U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC14_CLRMSK                                    (0xFFFFBFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC14_EN                                        (0x00004000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC13_SHIFT                                     (13U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC13_CLRMSK                                    (0xFFFFDFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC13_EN                                        (0x00002000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC12_SHIFT                                     (12U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC12_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC12_EN                                        (0x00001000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC11_SHIFT                                     (11U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC11_CLRMSK                                    (0xFFFFF7FFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC11_EN                                        (0x00000800U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC10_SHIFT                                     (10U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC10_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC10_EN                                        (0x00000400U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC9_SHIFT                                      (9U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC9_CLRMSK                                     (0xFFFFFDFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC9_EN                                         (0x00000200U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC8_SHIFT                                      (8U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC8_CLRMSK                                     (0xFFFFFEFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC8_EN                                         (0x00000100U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC7_SHIFT                                      (7U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC7_CLRMSK                                     (0xFFFFFF7FU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC7_EN                                         (0x00000080U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC6_SHIFT                                      (6U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC6_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC6_EN                                         (0x00000040U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC5_SHIFT                                      (5U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC5_CLRMSK                                     (0xFFFFFFDFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC5_EN                                         (0x00000020U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC4_SHIFT                                      (4U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC4_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC4_EN                                         (0x00000010U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC3_SHIFT                                      (3U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC3_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC3_EN                                         (0x00000008U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC2_SHIFT                                      (2U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC2_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC2_EN                                         (0x00000004U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC1_SHIFT                                      (1U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC1_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC1_EN                                         (0x00000002U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC0_SHIFT                                      (0U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC0_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC0_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT
+*/
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3                                  (0x0100U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__MASKFULL                        (IMG_UINT64_C(0xFFFFFFFF0000FF03))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_SHIFT            (32U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLRMSK           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER31_SHIFT  (63U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER31_CLRMSK (IMG_UINT64_C(0x7fffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER30_SHIFT  (62U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER30_CLRMSK (IMG_UINT64_C(0xbfffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER29_SHIFT  (61U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER29_CLRMSK (IMG_UINT64_C(0xdfffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER28_SHIFT  (60U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER28_CLRMSK (IMG_UINT64_C(0xefffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER27_SHIFT  (59U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER27_CLRMSK (IMG_UINT64_C(0xf7ffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER26_SHIFT  (58U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER26_CLRMSK (IMG_UINT64_C(0xfbffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER25_SHIFT  (57U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER25_CLRMSK (IMG_UINT64_C(0xfdffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER24_SHIFT  (56U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER24_CLRMSK (IMG_UINT64_C(0xfeffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER23_SHIFT  (55U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER23_CLRMSK (IMG_UINT64_C(0xff7fffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER22_SHIFT  (54U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER22_CLRMSK (IMG_UINT64_C(0xffbfffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER21_SHIFT  (53U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER21_CLRMSK (IMG_UINT64_C(0xffdfffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER20_SHIFT  (52U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER20_CLRMSK (IMG_UINT64_C(0xffefffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER19_SHIFT  (51U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER19_CLRMSK (IMG_UINT64_C(0xfff7ffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER18_SHIFT  (50U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER18_CLRMSK (IMG_UINT64_C(0xfffbffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER17_SHIFT  (49U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER17_CLRMSK (IMG_UINT64_C(0xfffdffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER16_SHIFT  (48U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER16_CLRMSK (IMG_UINT64_C(0xfffeffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER15_SHIFT  (47U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER15_CLRMSK (IMG_UINT64_C(0xffff7fffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER14_SHIFT  (46U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER14_CLRMSK (IMG_UINT64_C(0xffffbfffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER13_SHIFT  (45U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER13_CLRMSK (IMG_UINT64_C(0xffffdfffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER12_SHIFT  (44U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER12_CLRMSK (IMG_UINT64_C(0xffffefffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER11_SHIFT  (43U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER11_CLRMSK (IMG_UINT64_C(0xfffff7ffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER10_SHIFT  (42U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER10_CLRMSK (IMG_UINT64_C(0xfffffbffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER9_SHIFT   (41U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER9_CLRMSK  (IMG_UINT64_C(0xfffffdffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER8_SHIFT   (40U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER8_CLRMSK  (IMG_UINT64_C(0xfffffeffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER7_SHIFT   (39U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER7_CLRMSK  (IMG_UINT64_C(0xffffff7fffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER6_SHIFT   (38U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER6_CLRMSK  (IMG_UINT64_C(0xffffffbfffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER5_SHIFT   (37U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER5_CLRMSK  (IMG_UINT64_C(0xffffffdfffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER4_SHIFT   (36U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER4_CLRMSK  (IMG_UINT64_C(0xffffffefffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER3_SHIFT   (35U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER3_CLRMSK  (IMG_UINT64_C(0xfffffff7ffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER2_SHIFT   (34U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER2_CLRMSK  (IMG_UINT64_C(0xfffffffbffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER1_SHIFT   (33U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER1_CLRMSK  (IMG_UINT64_C(0xfffffffdffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER0_SHIFT   (32U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER0_CLRMSK  (IMG_UINT64_C(0xfffffffeffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__GPU_MASK_SHIFT                  (8U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__GPU_MASK_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__REQ_SHIFT                       (1U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__REQ_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__REQ_EN                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_SHIFT                      (0U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_POWER_DOWN                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_POWER_UP                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_GPU_CONTROL
+*/
+#define RGX_TB_GPU_CONTROL                                                          (0x0110U)
+#define RGX_TB_GPU_CONTROL_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000007))
+#define RGX_TB_GPU_CONTROL_FW_LOG_DISABLE_SHIFT                                     (2U)
+#define RGX_TB_GPU_CONTROL_FW_LOG_DISABLE_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_TB_GPU_CONTROL_FW_LOG_DISABLE_EN                                        (0x00000004U)
+#define RGX_TB_GPU_CONTROL_DXT_BC_ENABLE_SHIFT                                      (1U)
+#define RGX_TB_GPU_CONTROL_DXT_BC_ENABLE_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_TB_GPU_CONTROL_DXT_BC_ENABLE_EN                                         (0x00000002U)
+#define RGX_TB_GPU_CONTROL_ASTC_ENABLE_SHIFT                                        (0U)
+#define RGX_TB_GPU_CONTROL_ASTC_ENABLE_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_CONTROL_ASTC_ENABLE_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT_JONES
+*/
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5      (0x0118U)
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__MASKFULL (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__DOMAIN_SHIFT (0U)
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__DOMAIN_CLRMSK (0xFFFFFFFEU)
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__DOMAIN_EN (0x00000001U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT_TEXAS
+*/
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5      (0x0168U)
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__MASKFULL (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_SHIFT (0U)
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA
+*/
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5 (0x0170U)
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__MASKFULL (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_SHIFT (0U)
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_CLRMSK (0xFFFFFFFEU)
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_EN (0x00000001U)
+
+
+/*
+    Register RGX_TB_TRIGGER_SHORT
+*/
+#define RGX_TB_TRIGGER_SHORT                                                        (0x1500U)
+#define RGX_TB_TRIGGER_SHORT_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_SHORT_TIMEOUT_SHIFT                                          (0U)
+#define RGX_TB_TRIGGER_SHORT_TIMEOUT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRIGGER_MEDIUM
+*/
+#define RGX_TB_TRIGGER_MEDIUM                                                       (0x1508U)
+#define RGX_TB_TRIGGER_MEDIUM_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_MEDIUM_TIMEOUT_SHIFT                                         (0U)
+#define RGX_TB_TRIGGER_MEDIUM_TIMEOUT_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRIGGER_HIGH
+*/
+#define RGX_TB_TRIGGER_HIGH                                                         (0x1510U)
+#define RGX_TB_TRIGGER_HIGH_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_HIGH_TIMEOUT_SHIFT                                           (0U)
+#define RGX_TB_TRIGGER_HIGH_TIMEOUT_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRIGGER_MAX
+*/
+#define RGX_TB_TRIGGER_MAX                                                          (0x1518U)
+#define RGX_TB_TRIGGER_MAX_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_MAX_TIMEOUT_SHIFT                                            (0U)
+#define RGX_TB_TRIGGER_MAX_TIMEOUT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_TB_RDATA_CORRUPT_ENABLE
+*/
+#define RGX_TB_RDATA_CORRUPT_ENABLE                                                 (0x1560U)
+#define RGX_TB_RDATA_CORRUPT_ENABLE_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_RDATA_CORRUPT_ENABLE_ENABLE_SHIFT                                    (0U)
+#define RGX_TB_RDATA_CORRUPT_ENABLE_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_TB_RDATA_CORRUPT_ENABLE_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_TB_RDATA_CORRUPT_MASK
+*/
+#define RGX_TB_RDATA_CORRUPT_MASK                                                   (0x1568U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_RDATA_CORRUPT_MASK_MMU_SHIFT                                         (31U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MMU_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_MMU_EN                                            (0x80000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_UPS_SHIFT                                         (30U)
+#define RGX_TB_RDATA_CORRUPT_MASK_UPS_CLRMSK                                        (0xBFFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_UPS_EN                                            (0x40000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBM_SHIFT                                         (29U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBM_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBM_EN                                            (0x20000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TUL_SHIFT                                         (28U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TUL_CLRMSK                                        (0xEFFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TUL_EN                                            (0x10000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHR_SHIFT                                         (27U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHR_CLRMSK                                        (0xF7FFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHR_EN                                            (0x08000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBA_SHIFT                                         (26U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBA_CLRMSK                                        (0xFBFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBA_EN                                            (0x04000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VDM_SHIFT                                         (25U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VDM_CLRMSK                                        (0xFDFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_VDM_EN                                            (0x02000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_USC_L2_SHIFT                                      (24U)
+#define RGX_TB_RDATA_CORRUPT_MASK_USC_L2_CLRMSK                                     (0xFEFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_USC_L2_EN                                         (0x01000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDS_SHIFT                                         (23U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDS_CLRMSK                                        (0xFF7FFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDS_EN                                            (0x00800000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDSRW_SHIFT                                       (22U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDSRW_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDSRW_EN                                          (0x00400000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TPF_SHIFT                                         (21U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TPF_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TPF_EN                                            (0x00200000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHF_SHIFT                                         (20U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHF_CLRMSK                                        (0xFFEFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHF_EN                                            (0x00100000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_AMC_SHIFT                                         (19U)
+#define RGX_TB_RDATA_CORRUPT_MASK_AMC_CLRMSK                                        (0xFFF7FFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_AMC_EN                                            (0x00080000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RAC_SHIFT                                         (18U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RAC_CLRMSK                                        (0xFFFBFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_RAC_EN                                            (0x00040000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_RTC_SHIFT                                     (17U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_RTC_CLRMSK                                    (0xFFFDFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_RTC_EN                                        (0x00020000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_ISP_SHIFT                                         (16U)
+#define RGX_TB_RDATA_CORRUPT_MASK_ISP_CLRMSK                                        (0xFFFEFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_ISP_EN                                            (0x00010000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PPP_SHIFT                                         (15U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PPP_CLRMSK                                        (0xFFFF7FFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PPP_EN                                            (0x00008000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPF_SHIFT                                         (14U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPF_CLRMSK                                        (0xFFFFBFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPF_EN                                            (0x00004000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_SHIFT                                         (13U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_CLRMSK                                        (0xFFFFDFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_EN                                            (0x00002000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PBE_SHIFT                                         (12U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PBE_CLRMSK                                        (0xFFFFEFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PBE_EN                                            (0x00001000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TCU_SHIFT                                         (11U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TCU_CLRMSK                                        (0xFFFFF7FFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TCU_EN                                            (0x00000800U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MCU_SHIFT                                         (10U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MCU_CLRMSK                                        (0xFFFFFBFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_MCU_EN                                            (0x00000400U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RPM_SHIFT                                         (9U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RPM_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_RPM_EN                                            (0x00000200U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RTU_SHIFT                                         (8U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RTU_CLRMSK                                        (0xFFFFFEFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_RTU_EN                                            (0x00000100U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TILING_SHIFT                                      (7U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TILING_CLRMSK                                     (0xFFFFFF7FU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TILING_EN                                         (0x00000080U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_DMA_SHIFT                                    (6U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_DMA_CLRMSK                                   (0xFFFFFFBFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_DMA_EN                                       (0x00000040U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_SHIFT                                        (5U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_EN                                           (0x00000020U)
+#define RGX_TB_RDATA_CORRUPT_MASK_CDM_SHIFT                                         (4U)
+#define RGX_TB_RDATA_CORRUPT_MASK_CDM_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_CDM_EN                                            (0x00000010U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PM_SHIFT                                          (3U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PM_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PM_EN                                             (0x00000008U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TDM_SHIFT                                         (2U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TDM_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TDM_EN                                            (0x00000004U)
+#define RGX_TB_RDATA_CORRUPT_MASK_DCE_SHIFT                                         (1U)
+#define RGX_TB_RDATA_CORRUPT_MASK_DCE_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_TB_RDATA_CORRUPT_MASK_DCE_EN                                            (0x00000002U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPP_SHIFT                                         (0U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPP_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPP_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_TB_RDATA_CORRUPT_FREQ
+*/
+#define RGX_TB_RDATA_CORRUPT_FREQ                                                   (0x1570U)
+#define RGX_TB_RDATA_CORRUPT_FREQ_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_RDATA_CORRUPT_FREQ_FREQ_SHIFT                                        (0U)
+#define RGX_TB_RDATA_CORRUPT_FREQ_FREQ_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE
+*/
+#define RGX_TB_TRUSTED_DEVICE                                                       (0x2000U)
+#define RGX_TB_TRUSTED_DEVICE_MASKFULL                                              (IMG_UINT64_C(0x000000000000001F))
+#define RGX_TB_TRUSTED_DEVICE_ALLOW_SECURE_READS_SHIFT                              (4U)
+#define RGX_TB_TRUSTED_DEVICE_ALLOW_SECURE_READS_CLRMSK                             (0xFFFFFFEFU)
+#define RGX_TB_TRUSTED_DEVICE_ALLOW_SECURE_READS_EN                                 (0x00000010U)
+#define RGX_TB_TRUSTED_DEVICE_HWCONFIG_SHIFT                                        (2U)
+#define RGX_TB_TRUSTED_DEVICE_HWCONFIG_CLRMSK                                       (0xFFFFFFF3U)
+#define RGX_TB_TRUSTED_DEVICE_OSID_DISABLE_SHIFT                                    (1U)
+#define RGX_TB_TRUSTED_DEVICE_OSID_DISABLE_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_TB_TRUSTED_DEVICE_OSID_DISABLE_EN                                       (0x00000002U)
+#define RGX_TB_TRUSTED_DEVICE_ENABLE_SHIFT                                          (0U)
+#define RGX_TB_TRUSTED_DEVICE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_TB_TRUSTED_DEVICE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_FAULT_STATUS
+*/
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS                                          (0x2008U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x01FFFFFFFFFF1771))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_AXPROT_SHIFT                             (56U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_AXPROT_CLRMSK                            (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_AXPROT_EN                                (IMG_UINT64_C(0x0100000000000000))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_ADDR_SHIFT                               (16U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_ADDR_CLRMSK                              (IMG_UINT64_C(0xFF0000000000FFFF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_RNW_SHIFT                                (12U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_RNW_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_RNW_EN                                   (IMG_UINT64_C(0x0000000000001000))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_OS_ID_SHIFT                              (8U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_OS_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFF8FF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_BANK_SHIFT                               (4U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_BANK_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFF8F))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_VALID_SHIFT                              (0U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_VALID_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_VALID_EN                                 (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR
+*/
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR                                           (0x2010U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_MASKFULL                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_PULSE_SHIFT                               (0U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_PULSE_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_PULSE_EN                                  (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0                                       (0x2018U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1                                       (0x2020U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2                                       (0x2028U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3                                       (0x2030U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4                                       (0x2038U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5                                       (0x2040U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6                                       (0x2048U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7                                       (0x2050U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0                                       (0x2058U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1                                       (0x2060U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2                                       (0x2068U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3                                       (0x2070U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4                                       (0x2078U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5                                       (0x2080U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6                                       (0x2088U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7                                       (0x2090U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0                                       (0x2098U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1                                       (0x20A0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2                                       (0x20A8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3                                       (0x20B0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4                                       (0x20B8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5                                       (0x20C0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6                                       (0x20C8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7                                       (0x20D0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0                                       (0x20D8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1                                       (0x20E0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2                                       (0x20E8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3                                       (0x20F0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4                                       (0x20F8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5                                       (0x2100U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6                                       (0x2108U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7                                       (0x2110U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_BW_LIMITER
+*/
+#define RGX_TB_BW_LIMITER                                                           (0x2118U)
+#define RGX_TB_BW_LIMITER_MASKFULL                                                  (IMG_UINT64_C(0x00000000007707FF))
+#define RGX_TB_BW_LIMITER_DROPN_EXT_SHIFT                                           (20U)
+#define RGX_TB_BW_LIMITER_DROPN_EXT_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFF8FFFFF))
+#define RGX_TB_BW_LIMITER_PERIOD_EXT_SHIFT                                          (16U)
+#define RGX_TB_BW_LIMITER_PERIOD_EXT_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFF8FFFF))
+#define RGX_TB_BW_LIMITER_DROPN_SHIFT                                               (6U)
+#define RGX_TB_BW_LIMITER_DROPN_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_TB_BW_LIMITER_PERIOD_SHIFT                                              (1U)
+#define RGX_TB_BW_LIMITER_PERIOD_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFC1))
+#define RGX_TB_BW_LIMITER_ENABLE_SHIFT                                              (0U)
+#define RGX_TB_BW_LIMITER_ENABLE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_BW_LIMITER_ENABLE_EN                                                 (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_ACECONFIG
+*/
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG                                             (0x2120U)
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_MASKFULL                                    (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_OSID_SECURITY_SHIFT                         (1U)
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_OSID_SECURITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFE01))
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_ENABLE_SHIFT                                (0U)
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_ENABLE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_ENABLE_EN                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_DRAM_CROSSBAR
+*/
+#define RGX_TB_DRAM_CROSSBAR                                                        (0x2128U)
+#define RGX_TB_DRAM_CROSSBAR_MASKFULL                                               (IMG_UINT64_C(0x0000000000003301))
+#define RGX_TB_DRAM_CROSSBAR_CHANNELS_SHIFT                                         (12U)
+#define RGX_TB_DRAM_CROSSBAR_CHANNELS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_TB_DRAM_CROSSBAR_SEL_MODE_SHIFT                                         (8U)
+#define RGX_TB_DRAM_CROSSBAR_SEL_MODE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_TB_DRAM_CROSSBAR_ENABLE_SHIFT                                           (0U)
+#define RGX_TB_DRAM_CROSSBAR_ENABLE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_DRAM_CROSSBAR_ENABLE_EN                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_LOCKUP
+*/
+#define RGX_TB_LOCKUP                                                               (0x2130U)
+#define RGX_TB_LOCKUP_MASKFULL                                                      (IMG_UINT64_C(0xFFFFFFFF00000001))
+#define RGX_TB_LOCKUP_CYCLES_BEFORE_SHIFT                                           (32U)
+#define RGX_TB_LOCKUP_CYCLES_BEFORE_CLRMSK                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_LOCKUP_ENABLE_DETECT_SHIFT                                           (0U)
+#define RGX_TB_LOCKUP_ENABLE_DETECT_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_LOCKUP_ENABLE_DETECT_EN                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_SOC_TIMER
+*/
+#define RGX_TB_SOC_TIMER                                                            (0x2140U)
+#define RGX_TB_SOC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_TB_SOC_TIMER_COUNT_SHIFT                                                (0U)
+#define RGX_TB_SOC_TIMER_COUNT_CLRMSK                                               (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_TB_PROGRAMABLE_CLK_DIV
+*/
+#define RGX_TB_PROGRAMABLE_CLK_DIV                                                  (0x2150U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MASKFULL                                         (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MODE_SHIFT                                       (11U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MODE_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MODE_EN                                          (0x00000800U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_PROFILE_SEL_SHIFT                                (9U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_PROFILE_SEL_CLRMSK                               (0xFFFFF9FFU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DIV_SHIFT                                        (5U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DIV_CLRMSK                                       (0xFFFFFE1FU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DELAY_SHIFT                                      (1U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DELAY_CLRMSK                                     (0xFFFFFFE1U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_EVENT_SHIFT                                      (0U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_EVENT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_EVENT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPIO_FREQ_CTRL
+*/
+#define RGX_TB_GPIO_FREQ_CTRL                                                       (0x2160U)
+#define RGX_TB_GPIO_FREQ_CTRL_MASKFULL                                              (IMG_UINT64_C(0x000000000000001F))
+#define RGX_TB_GPIO_FREQ_CTRL_COUNT_SHIFT                                           (1U)
+#define RGX_TB_GPIO_FREQ_CTRL_COUNT_CLRMSK                                          (0xFFFFFFE1U)
+#define RGX_TB_GPIO_FREQ_CTRL_ENABLE_SHIFT                                          (0U)
+#define RGX_TB_GPIO_FREQ_CTRL_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_TB_GPIO_FREQ_CTRL_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPIO_MODE
+*/
+#define RGX_TB_GPIO_MODE                                                            (0x2170U)
+#define RGX_TB_GPIO_MODE_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_GPIO_MODE_PROTOCOL_SHIFT                                             (0U)
+#define RGX_TB_GPIO_MODE_PROTOCOL_CLRMSK                                            (0xFFFFFFFCU)
+
+
+/*
+    Register group: RGX_TB_SAFETY_STATUS, with 8 repeats
+*/
+#define RGX_TB_SAFETY_STATUS_REPEATCOUNT                                            (8U)
+/*
+    Register RGX_TB_SAFETY_STATUS0
+*/
+#define RGX_TB_SAFETY_STATUS0                                                       (0x6000U)
+#define RGX_TB_SAFETY_STATUS0_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS0_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS0_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS1
+*/
+#define RGX_TB_SAFETY_STATUS1                                                       (0x6008U)
+#define RGX_TB_SAFETY_STATUS1_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS1_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS1_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS2
+*/
+#define RGX_TB_SAFETY_STATUS2                                                       (0x6010U)
+#define RGX_TB_SAFETY_STATUS2_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS2_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS2_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS3
+*/
+#define RGX_TB_SAFETY_STATUS3                                                       (0x6018U)
+#define RGX_TB_SAFETY_STATUS3_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS3_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS3_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS4
+*/
+#define RGX_TB_SAFETY_STATUS4                                                       (0x6020U)
+#define RGX_TB_SAFETY_STATUS4_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS4_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS4_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS5
+*/
+#define RGX_TB_SAFETY_STATUS5                                                       (0x6028U)
+#define RGX_TB_SAFETY_STATUS5_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS5_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS5_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS6
+*/
+#define RGX_TB_SAFETY_STATUS6                                                       (0x6030U)
+#define RGX_TB_SAFETY_STATUS6_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS6_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS6_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS7
+*/
+#define RGX_TB_SAFETY_STATUS7                                                       (0x6038U)
+#define RGX_TB_SAFETY_STATUS7_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS7_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS7_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register group: RGX_TB_SAFETY_IRQ_ENABLE, with 8 repeats
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE_REPEATCOUNT                                        (8U)
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE0
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE0                                                   (0x6040U)
+#define RGX_TB_SAFETY_IRQ_ENABLE0_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE0_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE0_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE1
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE1                                                   (0x6048U)
+#define RGX_TB_SAFETY_IRQ_ENABLE1_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE1_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE1_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE2
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE2                                                   (0x6050U)
+#define RGX_TB_SAFETY_IRQ_ENABLE2_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE2_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE2_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE3
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE3                                                   (0x6058U)
+#define RGX_TB_SAFETY_IRQ_ENABLE3_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE3_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE3_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE4
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE4                                                   (0x6060U)
+#define RGX_TB_SAFETY_IRQ_ENABLE4_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE4_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE4_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE5
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE5                                                   (0x6068U)
+#define RGX_TB_SAFETY_IRQ_ENABLE5_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE5_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE5_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE6
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE6                                                   (0x6070U)
+#define RGX_TB_SAFETY_IRQ_ENABLE6_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE6_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE6_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE7
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE7                                                   (0x6078U)
+#define RGX_TB_SAFETY_IRQ_ENABLE7_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE7_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE7_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_FAULT_MASK
+*/
+#define RGX_TB_FAULT_MASK                                                           (0x6080U)
+#define RGX_TB_FAULT_MASK_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_FAULT_MASK_MASK_FAULT_EVENT_SHIFT                                    (0U)
+#define RGX_TB_FAULT_MASK_MASK_FAULT_EVENT_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_TB_FAULT_MASK_MASK_FAULT_EVENT_EN                                       (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_GPU_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL_REPEATCOUNT                                    (8U)
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0                                               (0x60A0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1                                               (0x60A8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2                                               (0x60B0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3                                               (0x60B8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4                                               (0x60C0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5                                               (0x60C8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6                                               (0x60D0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7                                               (0x60D8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_SFTY_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL_REPEATCOUNT                                   (8U)
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0                                              (0x60E0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1                                              (0x60E8U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2                                              (0x60F0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3                                              (0x60F8U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4                                              (0x6100U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5                                              (0x6108U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6                                              (0x6110U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7                                              (0x6118U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_XPU_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL_REPEATCOUNT                                    (8U)
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0                                               (0x6120U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1                                               (0x6128U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2                                               (0x6130U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3                                               (0x6138U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4                                               (0x6140U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5                                               (0x6148U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6                                               (0x6150U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7                                               (0x6158U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_MEM_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL_REPEATCOUNT                                    (8U)
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0                                               (0x6160U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1                                               (0x6168U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2                                               (0x6170U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3                                               (0x6178U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4                                               (0x6180U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5                                               (0x6188U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6                                               (0x6190U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7                                               (0x6198U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_INSERT_EN                               (0x00000001U)
+
+
+#endif /* RGXTBDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxtbdefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/rgxpmdefs.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/rgxpmdefs.h
new file mode 100644
index 000000000000..5db7c2585356
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/rgxpmdefs.h
@@ -0,0 +1,5019 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxpmdefs.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_pm.def: #15
+ */
+
+
+#ifndef RGXPMDEFS_H
+#define RGXPMDEFS_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXPMDEFS_REVISION 15
+
+/*
+The mini PB size on a per-RT basis
+*/
+typedef struct PM_DATA_MINI_PB_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_MINI_PB;
+
+/*
+
+*/
+#define PM_DATA_MINI_PB_SIZE_WOFF                         (0U)
+#define PM_DATA_MINI_PB_SIZE_SHIFT                        (0U)
+#define PM_DATA_MINI_PB_SIZE_CLRMSK                       (0xFFFFFC00U)
+#define PM_DATA_MINI_PB_SET_SIZE(_ft_,_x_)                ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_MINI_PB_SIZE_CLRMSK ) | (((_x_) & (0x000003ffU))  <<  (PM_DATA_MINI_PB_SIZE_SHIFT))))
+#define PM_DATA_MINI_PB_GET_SIZE(_ft_)                    (((_ft_).u32_0  >>  (PM_DATA_MINI_PB_SIZE_SHIFT)) & 0x000003ffU)
+
+
+/*
+The minimum PB size for the WDDM driver only. It is consistent with the OPENGL/OPENGLES. However, it is breaking down as two parts: the pagable memory and non pagable memory.
+*/
+typedef struct PM_DATA_WDDM_MINI_PB_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_WDDM_MINI_PB;
+
+/*
+
+*/
+#define PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_WOFF        (0U)
+#define PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_SHIFT       (10U)
+#define PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_CLRMSK      (0xFFF003FFU)
+#define PM_DATA_WDDM_MINI_PB_SET_NON_PAGABLE_SIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_CLRMSK ) | (((_x_) & (0x000003ffU))  <<  (PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_SHIFT))))
+#define PM_DATA_WDDM_MINI_PB_GET_NON_PAGABLE_SIZE(_ft_)   (((_ft_).u32_0  >>  (PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_SHIFT)) & 0x000003ffU)
+/*
+
+*/
+#define PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_WOFF            (0U)
+#define PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_SHIFT           (0U)
+#define PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_CLRMSK          (0xFFFFFC00U)
+#define PM_DATA_WDDM_MINI_PB_SET_PAGABLE_SIZE(_ft_,_x_)   ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_CLRMSK ) | (((_x_) & (0x000003ffU))  <<  (PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_SHIFT))))
+#define PM_DATA_WDDM_MINI_PB_GET_PAGABLE_SIZE(_ft_)       (((_ft_).u32_0  >>  (PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_SHIFT)) & 0x000003ffU)
+
+
+/*
+the mini number of the reserve pages when only the local free list is used */
+#define PM_DATA_PM_RESERVE_PAGES_MIN_SIZE                 (0x00000007U)
+
+
+/*
+the mini number of the reserve pages when unified free list is present */
+#define PM_DATA_PM_RESERVE_PAGES_MIN_UNIFIED_SIZE         (0x0000000bU)
+
+
+/*
+This defines the format of entries in the FSTACK, UFSTACK and MMUSTACK
+*/
+typedef struct PM_DATA_FSTACK_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_FSTACK_ENTRY;
+
+/*
+Reserved for future use
+*/
+#define PM_DATA_FSTACK_ENTRY_RSV_WOFF                     (0U)
+#define PM_DATA_FSTACK_ENTRY_RSV_SHIFT                    (28U)
+#define PM_DATA_FSTACK_ENTRY_RSV_CLRMSK                   (0x0FFFFFFFU)
+#define PM_DATA_FSTACK_ENTRY_SET_RSV(_ft_,_x_)            ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_FSTACK_ENTRY_RSV_CLRMSK ) | (((_x_) & (0x0000000fU))  <<  (PM_DATA_FSTACK_ENTRY_RSV_SHIFT))))
+#define PM_DATA_FSTACK_ENTRY_GET_RSV(_ft_)                (((_ft_).u32_0  >>  (PM_DATA_FSTACK_ENTRY_RSV_SHIFT)) & 0x0000000fU)
+/*
+Address of 4 kB physical page
+*/
+#define PM_DATA_FSTACK_ENTRY_PPAGE_WOFF                   (0U)
+#define PM_DATA_FSTACK_ENTRY_PPAGE_SHIFT                  (0U)
+#define PM_DATA_FSTACK_ENTRY_PPAGE_CLRMSK                 (0xF0000000U)
+#define PM_DATA_FSTACK_ENTRY_SET_PPAGE(_ft_,_x_)          ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_FSTACK_ENTRY_PPAGE_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_FSTACK_ENTRY_PPAGE_SHIFT))))
+#define PM_DATA_FSTACK_ENTRY_GET_PPAGE(_ft_)              (((_ft_).u32_0  >>  (PM_DATA_FSTACK_ENTRY_PPAGE_SHIFT)) & 0x0fffffffU)
+
+
+/*
+This defines the format of an ALIST (Allocation List) entry
+*/
+typedef struct PM_DATA_ALIST_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+} PM_DATA_ALIST_ENTRY;
+
+/*
+Valid bit.  Indicates whether this ALIST entry is valid.
+*/
+#define PM_DATA_ALIST_ENTRY_VAL_WOFF                      (1U)
+#define PM_DATA_ALIST_ENTRY_VAL_SHIFT                     (31U)
+#define PM_DATA_ALIST_ENTRY_VAL_CLRMSK                    (0x7FFFFFFFU)
+#define PM_DATA_ALIST_ENTRY_SET_VAL(_ft_,_x_)             ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_VAL_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_ALIST_ENTRY_VAL_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_VAL(_ft_)                 (((_ft_).u32_1  >>  (PM_DATA_ALIST_ENTRY_VAL_SHIFT)) & 0x00000001U)
+/*
+The "data master" of the virtual page.  0=VCE, 1=TE, 2,3=reserved.
+*/
+#define PM_DATA_ALIST_ENTRY_DM_INDEX_WOFF                 (1U)
+#define PM_DATA_ALIST_ENTRY_DM_INDEX_SHIFT                (26U)
+#define PM_DATA_ALIST_ENTRY_DM_INDEX_CLRMSK               (0xF3FFFFFFU)
+#define PM_DATA_ALIST_ENTRY_SET_DM_INDEX(_ft_,_x_)        ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_DM_INDEX_CLRMSK ) | (((_x_) & (0x00000003U))  <<  (PM_DATA_ALIST_ENTRY_DM_INDEX_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_DM_INDEX(_ft_)            (((_ft_).u32_1  >>  (PM_DATA_ALIST_ENTRY_DM_INDEX_SHIFT)) & 0x00000003U)
+/*
+Render Target Array index.  Up to 2 k Render Target Arrays are supported.
+*/
+#define PM_DATA_ALIST_ENTRY_RTA_INDEX_WOFF                (1U)
+#define PM_DATA_ALIST_ENTRY_RTA_INDEX_SHIFT               (14U)
+#define PM_DATA_ALIST_ENTRY_RTA_INDEX_CLRMSK              (0xFE003FFFU)
+#define PM_DATA_ALIST_ENTRY_SET_RTA_INDEX(_ft_,_x_)       ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_RTA_INDEX_CLRMSK ) | (((_x_) & (0x000007ffU))  <<  (PM_DATA_ALIST_ENTRY_RTA_INDEX_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_RTA_INDEX(_ft_)           (((_ft_).u32_1  >>  (PM_DATA_ALIST_ENTRY_RTA_INDEX_SHIFT)) & 0x000007ffU)
+/*
+The virtual page number (16 kB virtual page).
+*/
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_WOFF             (0U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_WOFF             (1U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_SHIFT            (16U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_SHIFT            (0U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_CLRMSK           (0x0000FFFFU)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_CLRMSK           (0xFFFFFFF0U)
+#define PM_DATA_ALIST_ENTRY_SET_VRP_PPAGE(_ft_,_x_)       { ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000ffff)))  <<  16))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000f0000)))  >>  16))); }
+#define PM_DATA_ALIST_ENTRY_GET_VRP_PPAGE(_ft_)           (((_ft_).u32_0  >>  (16)) | ((IMG_UINT64)((_ft_).u32_1 & 0x0000000fU ) << (16)))
+/*
+The 16-bit macrotile mask.  Indicates which macrotile(s) are using this 16 kB page
+*/
+#define PM_DATA_ALIST_ENTRY_MTILE_MASK_WOFF               (0U)
+#define PM_DATA_ALIST_ENTRY_MTILE_MASK_SHIFT              (0U)
+#define PM_DATA_ALIST_ENTRY_MTILE_MASK_CLRMSK             (0xFFFF0000U)
+#define PM_DATA_ALIST_ENTRY_SET_MTILE_MASK(_ft_,_x_)      ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_ALIST_ENTRY_MTILE_MASK_CLRMSK ) | (((_x_) & (0x0000ffffU))  <<  (PM_DATA_ALIST_ENTRY_MTILE_MASK_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_MTILE_MASK(_ft_)          (((_ft_).u32_0  >>  (PM_DATA_ALIST_ENTRY_MTILE_MASK_SHIFT)) & 0x0000ffffU)
+
+
+/*
+This defines the format of entries in the MLIST
+*/
+typedef struct PM_DATA_MLIST_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_MLIST_ENTRY;
+
+/*
+Original source of the MMU page:
+0=Page was allocated from the FSTACK,
+1=Page was allocated from the UFSTACK.
+This bit is ignored when RGX_CR_PM_MMU_STACK_POLICY=1
+*/
+#define PM_DATA_MLIST_ENTRY_SRC_WOFF                      (0U)
+#define PM_DATA_MLIST_ENTRY_SRC_SHIFT                     (31U)
+#define PM_DATA_MLIST_ENTRY_SRC_CLRMSK                    (0x7FFFFFFFU)
+#define PM_DATA_MLIST_ENTRY_SET_SRC(_ft_,_x_)             ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_MLIST_ENTRY_SRC_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_MLIST_ENTRY_SRC_SHIFT))))
+#define PM_DATA_MLIST_ENTRY_GET_SRC(_ft_)                 (((_ft_).u32_0  >>  (PM_DATA_MLIST_ENTRY_SRC_SHIFT)) & 0x00000001U)
+/*
+Address of Physical Page allocated to MMU
+*/
+#define PM_DATA_MLIST_ENTRY_PPAGE_WOFF                    (0U)
+#define PM_DATA_MLIST_ENTRY_PPAGE_SHIFT                   (0U)
+#define PM_DATA_MLIST_ENTRY_PPAGE_CLRMSK                  (0xF0000000U)
+#define PM_DATA_MLIST_ENTRY_SET_PPAGE(_ft_,_x_)           ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_MLIST_ENTRY_PPAGE_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_MLIST_ENTRY_PPAGE_SHIFT))))
+#define PM_DATA_MLIST_ENTRY_GET_PPAGE(_ft_)               (((_ft_).u32_0  >>  (PM_DATA_MLIST_ENTRY_PPAGE_SHIFT)) & 0x0fffffffU)
+
+
+/*
+This defines the format of entries in the VFP Table
+*/
+typedef struct PM_DATA_VFP_TABLE_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+} PM_DATA_VFP_TABLE_ENTRY;
+
+/*
+Valid bit.  0=VFP is unmapped, 1=VFP is mapped.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_VALID_WOFF                (1U)
+#define PM_DATA_VFP_TABLE_ENTRY_VALID_SHIFT               (31U)
+#define PM_DATA_VFP_TABLE_ENTRY_VALID_CLRMSK              (0x7FFFFFFFU)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_VALID(_ft_,_x_)       ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VFP_TABLE_ENTRY_VALID_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VFP_TABLE_ENTRY_VALID_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_VALID(_ft_)           (((_ft_).u32_1  >>  (PM_DATA_VFP_TABLE_ENTRY_VALID_SHIFT)) & 0x00000001U)
+/*
+Address of MMU Page Table Entry.  8 Byte Granular.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_WOFF              (1U)
+#define PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_SHIFT             (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_CLRMSK            (0x80000000U)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_PTE_PTR(_ft_,_x_)     ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_PTE_PTR(_ft_)         (((_ft_).u32_1  >>  (PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_SHIFT)) & 0x7fffffffU)
+/*
+Reserved for future use.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_RSV_WOFF                  (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_RSV_SHIFT                 (28U)
+#define PM_DATA_VFP_TABLE_ENTRY_RSV_CLRMSK                (0x0FFFFFFFU)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_RSV(_ft_,_x_)         ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VFP_TABLE_ENTRY_RSV_CLRMSK ) | (((_x_) & (0x0000000fU))  <<  (PM_DATA_VFP_TABLE_ENTRY_RSV_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_RSV(_ft_)             (((_ft_).u32_0  >>  (PM_DATA_VFP_TABLE_ENTRY_RSV_SHIFT)) & 0x0000000fU)
+/*
+Address of 1 kB Physical Page. 1 TB addressable.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_PPAGE_WOFF                (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_PPAGE_SHIFT               (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_PPAGE_CLRMSK              (0xF0000000U)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_PPAGE(_ft_,_x_)       ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VFP_TABLE_ENTRY_PPAGE_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VFP_TABLE_ENTRY_PPAGE_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_PPAGE(_ft_)           (((_ft_).u32_0  >>  (PM_DATA_VFP_TABLE_ENTRY_PPAGE_SHIFT)) & 0x0fffffffU)
+
+
+/*
+PerPipe Segment SIZE, it has a fixed mapping as follows:
+PIPE Number - Segment Size
+1             16G
+2             8G
+4             4G
+8             2G
+*/
+typedef struct PM_DATA_PERPIPE_SEGSIZE_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_PERPIPE_SEGSIZE;
+
+/*
+PerSegment Size 2G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_SHIFT      (3U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_CLRMSK     (0xFFFFFFF7U)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE8_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE8_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_SHIFT)) & 0x00000001U)
+/*
+PerSegment Size 4G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_SHIFT      (2U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_CLRMSK     (0xFFFFFFFBU)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE4_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE4_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_SHIFT)) & 0x00000001U)
+/*
+PerSegment Size 8G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_SHIFT      (1U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_CLRMSK     (0xFFFFFFFDU)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE2_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE2_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_SHIFT)) & 0x00000001U)
+/*
+PerSegment Size 16G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_SHIFT      (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_CLRMSK     (0xFFFFFFFEU)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE1_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE1_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_SHIFT)) & 0x00000001U)
+
+
+/*
+PM Virtual Heap Buffer Offset. This buffer contains all the meta-data associated with each render target.
+size is 11904/8 = 1488 Bytes.
+
+Natively the buffer supports up to 8-VCEs and 8-TEs scaling without changing HW.
+
+In case relevant PIPE N is not present, the corresponding space is just reserved.
+*/
+typedef struct PM_DATA_VHEAP_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+     IMG_UINT32 u32_46;
+     IMG_UINT32 u32_47;
+     IMG_UINT32 u32_48;
+     IMG_UINT32 u32_49;
+     IMG_UINT32 u32_50;
+     IMG_UINT32 u32_51;
+     IMG_UINT32 u32_52;
+     IMG_UINT32 u32_53;
+     IMG_UINT32 u32_54;
+     IMG_UINT32 u32_55;
+     IMG_UINT32 u32_56;
+     IMG_UINT32 u32_57;
+     IMG_UINT32 u32_58;
+     IMG_UINT32 u32_59;
+     IMG_UINT32 u32_60;
+     IMG_UINT32 u32_61;
+     IMG_UINT32 u32_62;
+     IMG_UINT32 u32_63;
+     IMG_UINT32 u32_64;
+     IMG_UINT32 u32_65;
+     IMG_UINT32 u32_66;
+     IMG_UINT32 u32_67;
+     IMG_UINT32 u32_68;
+     IMG_UINT32 u32_69;
+     IMG_UINT32 u32_70;
+     IMG_UINT32 u32_71;
+     IMG_UINT32 u32_72;
+     IMG_UINT32 u32_73;
+     IMG_UINT32 u32_74;
+     IMG_UINT32 u32_75;
+     IMG_UINT32 u32_76;
+     IMG_UINT32 u32_77;
+     IMG_UINT32 u32_78;
+     IMG_UINT32 u32_79;
+     IMG_UINT32 u32_80;
+     IMG_UINT32 u32_81;
+     IMG_UINT32 u32_82;
+     IMG_UINT32 u32_83;
+     IMG_UINT32 u32_84;
+     IMG_UINT32 u32_85;
+     IMG_UINT32 u32_86;
+     IMG_UINT32 u32_87;
+     IMG_UINT32 u32_88;
+     IMG_UINT32 u32_89;
+     IMG_UINT32 u32_90;
+     IMG_UINT32 u32_91;
+     IMG_UINT32 u32_92;
+     IMG_UINT32 u32_93;
+     IMG_UINT32 u32_94;
+     IMG_UINT32 u32_95;
+     IMG_UINT32 u32_96;
+     IMG_UINT32 u32_97;
+     IMG_UINT32 u32_98;
+     IMG_UINT32 u32_99;
+     IMG_UINT32 u32_100;
+     IMG_UINT32 u32_101;
+     IMG_UINT32 u32_102;
+     IMG_UINT32 u32_103;
+     IMG_UINT32 u32_104;
+     IMG_UINT32 u32_105;
+     IMG_UINT32 u32_106;
+     IMG_UINT32 u32_107;
+     IMG_UINT32 u32_108;
+     IMG_UINT32 u32_109;
+     IMG_UINT32 u32_110;
+     IMG_UINT32 u32_111;
+     IMG_UINT32 u32_112;
+     IMG_UINT32 u32_113;
+     IMG_UINT32 u32_114;
+     IMG_UINT32 u32_115;
+     IMG_UINT32 u32_116;
+     IMG_UINT32 u32_117;
+     IMG_UINT32 u32_118;
+     IMG_UINT32 u32_119;
+     IMG_UINT32 u32_120;
+     IMG_UINT32 u32_121;
+     IMG_UINT32 u32_122;
+     IMG_UINT32 u32_123;
+     IMG_UINT32 u32_124;
+     IMG_UINT32 u32_125;
+     IMG_UINT32 u32_126;
+     IMG_UINT32 u32_127;
+     IMG_UINT32 u32_128;
+     IMG_UINT32 u32_129;
+     IMG_UINT32 u32_130;
+     IMG_UINT32 u32_131;
+     IMG_UINT32 u32_132;
+     IMG_UINT32 u32_133;
+     IMG_UINT32 u32_134;
+     IMG_UINT32 u32_135;
+     IMG_UINT32 u32_136;
+     IMG_UINT32 u32_137;
+     IMG_UINT32 u32_138;
+     IMG_UINT32 u32_139;
+     IMG_UINT32 u32_140;
+     IMG_UINT32 u32_141;
+     IMG_UINT32 u32_142;
+     IMG_UINT32 u32_143;
+     IMG_UINT32 u32_144;
+     IMG_UINT32 u32_145;
+     IMG_UINT32 u32_146;
+     IMG_UINT32 u32_147;
+     IMG_UINT32 u32_148;
+     IMG_UINT32 u32_149;
+     IMG_UINT32 u32_150;
+     IMG_UINT32 u32_151;
+     IMG_UINT32 u32_152;
+     IMG_UINT32 u32_153;
+     IMG_UINT32 u32_154;
+     IMG_UINT32 u32_155;
+     IMG_UINT32 u32_156;
+     IMG_UINT32 u32_157;
+     IMG_UINT32 u32_158;
+     IMG_UINT32 u32_159;
+     IMG_UINT32 u32_160;
+     IMG_UINT32 u32_161;
+     IMG_UINT32 u32_162;
+     IMG_UINT32 u32_163;
+     IMG_UINT32 u32_164;
+     IMG_UINT32 u32_165;
+     IMG_UINT32 u32_166;
+     IMG_UINT32 u32_167;
+     IMG_UINT32 u32_168;
+     IMG_UINT32 u32_169;
+     IMG_UINT32 u32_170;
+     IMG_UINT32 u32_171;
+     IMG_UINT32 u32_172;
+     IMG_UINT32 u32_173;
+     IMG_UINT32 u32_174;
+     IMG_UINT32 u32_175;
+     IMG_UINT32 u32_176;
+     IMG_UINT32 u32_177;
+     IMG_UINT32 u32_178;
+     IMG_UINT32 u32_179;
+     IMG_UINT32 u32_180;
+     IMG_UINT32 u32_181;
+     IMG_UINT32 u32_182;
+     IMG_UINT32 u32_183;
+     IMG_UINT32 u32_184;
+     IMG_UINT32 u32_185;
+     IMG_UINT32 u32_186;
+     IMG_UINT32 u32_187;
+     IMG_UINT32 u32_188;
+     IMG_UINT32 u32_189;
+     IMG_UINT32 u32_190;
+     IMG_UINT32 u32_191;
+     IMG_UINT32 u32_192;
+     IMG_UINT32 u32_193;
+     IMG_UINT32 u32_194;
+     IMG_UINT32 u32_195;
+     IMG_UINT32 u32_196;
+     IMG_UINT32 u32_197;
+     IMG_UINT32 u32_198;
+     IMG_UINT32 u32_199;
+     IMG_UINT32 u32_200;
+     IMG_UINT32 u32_201;
+     IMG_UINT32 u32_202;
+     IMG_UINT32 u32_203;
+     IMG_UINT32 u32_204;
+     IMG_UINT32 u32_205;
+     IMG_UINT32 u32_206;
+     IMG_UINT32 u32_207;
+     IMG_UINT32 u32_208;
+     IMG_UINT32 u32_209;
+     IMG_UINT32 u32_210;
+     IMG_UINT32 u32_211;
+     IMG_UINT32 u32_212;
+     IMG_UINT32 u32_213;
+     IMG_UINT32 u32_214;
+     IMG_UINT32 u32_215;
+     IMG_UINT32 u32_216;
+     IMG_UINT32 u32_217;
+     IMG_UINT32 u32_218;
+     IMG_UINT32 u32_219;
+     IMG_UINT32 u32_220;
+     IMG_UINT32 u32_221;
+     IMG_UINT32 u32_222;
+     IMG_UINT32 u32_223;
+     IMG_UINT32 u32_224;
+     IMG_UINT32 u32_225;
+     IMG_UINT32 u32_226;
+     IMG_UINT32 u32_227;
+     IMG_UINT32 u32_228;
+     IMG_UINT32 u32_229;
+     IMG_UINT32 u32_230;
+     IMG_UINT32 u32_231;
+     IMG_UINT32 u32_232;
+     IMG_UINT32 u32_233;
+     IMG_UINT32 u32_234;
+     IMG_UINT32 u32_235;
+     IMG_UINT32 u32_236;
+     IMG_UINT32 u32_237;
+     IMG_UINT32 u32_238;
+     IMG_UINT32 u32_239;
+     IMG_UINT32 u32_240;
+     IMG_UINT32 u32_241;
+     IMG_UINT32 u32_242;
+     IMG_UINT32 u32_243;
+     IMG_UINT32 u32_244;
+     IMG_UINT32 u32_245;
+     IMG_UINT32 u32_246;
+     IMG_UINT32 u32_247;
+     IMG_UINT32 u32_248;
+     IMG_UINT32 u32_249;
+     IMG_UINT32 u32_250;
+     IMG_UINT32 u32_251;
+     IMG_UINT32 u32_252;
+     IMG_UINT32 u32_253;
+     IMG_UINT32 u32_254;
+     IMG_UINT32 u32_255;
+     IMG_UINT32 u32_256;
+     IMG_UINT32 u32_257;
+     IMG_UINT32 u32_258;
+     IMG_UINT32 u32_259;
+     IMG_UINT32 u32_260;
+     IMG_UINT32 u32_261;
+     IMG_UINT32 u32_262;
+     IMG_UINT32 u32_263;
+     IMG_UINT32 u32_264;
+     IMG_UINT32 u32_265;
+     IMG_UINT32 u32_266;
+     IMG_UINT32 u32_267;
+     IMG_UINT32 u32_268;
+     IMG_UINT32 u32_269;
+     IMG_UINT32 u32_270;
+     IMG_UINT32 u32_271;
+     IMG_UINT32 u32_272;
+     IMG_UINT32 u32_273;
+     IMG_UINT32 u32_274;
+     IMG_UINT32 u32_275;
+     IMG_UINT32 u32_276;
+     IMG_UINT32 u32_277;
+     IMG_UINT32 u32_278;
+     IMG_UINT32 u32_279;
+     IMG_UINT32 u32_280;
+     IMG_UINT32 u32_281;
+     IMG_UINT32 u32_282;
+     IMG_UINT32 u32_283;
+     IMG_UINT32 u32_284;
+     IMG_UINT32 u32_285;
+     IMG_UINT32 u32_286;
+     IMG_UINT32 u32_287;
+     IMG_UINT32 u32_288;
+     IMG_UINT32 u32_289;
+     IMG_UINT32 u32_290;
+     IMG_UINT32 u32_291;
+     IMG_UINT32 u32_292;
+     IMG_UINT32 u32_293;
+     IMG_UINT32 u32_294;
+     IMG_UINT32 u32_295;
+     IMG_UINT32 u32_296;
+     IMG_UINT32 u32_297;
+     IMG_UINT32 u32_298;
+     IMG_UINT32 u32_299;
+     IMG_UINT32 u32_300;
+     IMG_UINT32 u32_301;
+     IMG_UINT32 u32_302;
+     IMG_UINT32 u32_303;
+     IMG_UINT32 u32_304;
+     IMG_UINT32 u32_305;
+     IMG_UINT32 u32_306;
+     IMG_UINT32 u32_307;
+     IMG_UINT32 u32_308;
+     IMG_UINT32 u32_309;
+     IMG_UINT32 u32_310;
+     IMG_UINT32 u32_311;
+     IMG_UINT32 u32_312;
+     IMG_UINT32 u32_313;
+     IMG_UINT32 u32_314;
+     IMG_UINT32 u32_315;
+     IMG_UINT32 u32_316;
+     IMG_UINT32 u32_317;
+     IMG_UINT32 u32_318;
+     IMG_UINT32 u32_319;
+     IMG_UINT32 u32_320;
+     IMG_UINT32 u32_321;
+     IMG_UINT32 u32_322;
+     IMG_UINT32 u32_323;
+     IMG_UINT32 u32_324;
+     IMG_UINT32 u32_325;
+     IMG_UINT32 u32_326;
+     IMG_UINT32 u32_327;
+     IMG_UINT32 u32_328;
+     IMG_UINT32 u32_329;
+     IMG_UINT32 u32_330;
+     IMG_UINT32 u32_331;
+     IMG_UINT32 u32_332;
+     IMG_UINT32 u32_333;
+     IMG_UINT32 u32_334;
+     IMG_UINT32 u32_335;
+     IMG_UINT32 u32_336;
+     IMG_UINT32 u32_337;
+     IMG_UINT32 u32_338;
+     IMG_UINT32 u32_339;
+     IMG_UINT32 u32_340;
+     IMG_UINT32 u32_341;
+     IMG_UINT32 u32_342;
+     IMG_UINT32 u32_343;
+     IMG_UINT32 u32_344;
+     IMG_UINT32 u32_345;
+     IMG_UINT32 u32_346;
+     IMG_UINT32 u32_347;
+     IMG_UINT32 u32_348;
+     IMG_UINT32 u32_349;
+     IMG_UINT32 u32_350;
+     IMG_UINT32 u32_351;
+     IMG_UINT32 u32_352;
+     IMG_UINT32 u32_353;
+     IMG_UINT32 u32_354;
+     IMG_UINT32 u32_355;
+     IMG_UINT32 u32_356;
+     IMG_UINT32 u32_357;
+     IMG_UINT32 u32_358;
+     IMG_UINT32 u32_359;
+     IMG_UINT32 u32_360;
+     IMG_UINT32 u32_361;
+     IMG_UINT32 u32_362;
+     IMG_UINT32 u32_363;
+     IMG_UINT32 u32_364;
+     IMG_UINT32 u32_365;
+     IMG_UINT32 u32_366;
+     IMG_UINT32 u32_367;
+     IMG_UINT32 u32_368;
+     IMG_UINT32 u32_369;
+     IMG_UINT32 u32_370;
+     IMG_UINT32 u32_371;
+} PM_DATA_VHEAP_BUFFER;
+
+/*
+TE7 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_WOFF   (371U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_371 = (((_ft_).u32_371 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_371  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE7 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_WOFF (370U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_WOFF (371U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_370 = (((_ft_).u32_370 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_371 = (((_ft_).u32_371 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_370  >>  (20)) | ((IMG_UINT64)((_ft_).u32_371 & 0x0000ffffU ) << (12)))
+/*
+TE7 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_WOFF (369U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_WOFF (370U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_369 = (((_ft_).u32_369 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_370 = (((_ft_).u32_370 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_369  >>  (24)) | ((IMG_UINT64)((_ft_).u32_370 & 0x000fffffU ) << (8)))
+/*
+TE7 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_WOFF (368U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_WOFF (369U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_368 = (((_ft_).u32_368 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_369 = (((_ft_).u32_369 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_368  >>  (28)) | ((IMG_UINT64)((_ft_).u32_369 & 0x00ffffffU ) << (4)))
+/*
+TE7 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_WOFF    (368U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_368 = (((_ft_).u32_368 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_368  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE7 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_WOFF      (364U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_364 = (((_ft_).u32_364 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_364  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE6 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_WOFF   (363U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_363 = (((_ft_).u32_363 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_363  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE6 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_WOFF (362U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_WOFF (363U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_362 = (((_ft_).u32_362 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_363 = (((_ft_).u32_363 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_362  >>  (20)) | ((IMG_UINT64)((_ft_).u32_363 & 0x0000ffffU ) << (12)))
+/*
+TE6 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_WOFF (361U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_WOFF (362U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_361 = (((_ft_).u32_361 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_362 = (((_ft_).u32_362 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_361  >>  (24)) | ((IMG_UINT64)((_ft_).u32_362 & 0x000fffffU ) << (8)))
+/*
+TE6 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_WOFF (360U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_WOFF (361U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_360 = (((_ft_).u32_360 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_361 = (((_ft_).u32_361 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_360  >>  (28)) | ((IMG_UINT64)((_ft_).u32_361 & 0x00ffffffU ) << (4)))
+/*
+TE6 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_WOFF    (360U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_360 = (((_ft_).u32_360 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_360  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE6 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_WOFF      (356U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_356 = (((_ft_).u32_356 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_356  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE5 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_WOFF   (355U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_355 = (((_ft_).u32_355 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_355  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE5 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_WOFF (354U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_WOFF (355U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_354 = (((_ft_).u32_354 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_355 = (((_ft_).u32_355 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_354  >>  (20)) | ((IMG_UINT64)((_ft_).u32_355 & 0x0000ffffU ) << (12)))
+/*
+TE5 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_WOFF (353U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_WOFF (354U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_353 = (((_ft_).u32_353 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_354 = (((_ft_).u32_354 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_353  >>  (24)) | ((IMG_UINT64)((_ft_).u32_354 & 0x000fffffU ) << (8)))
+/*
+TE5 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_WOFF (352U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_WOFF (353U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_352 = (((_ft_).u32_352 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_353 = (((_ft_).u32_353 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_352  >>  (28)) | ((IMG_UINT64)((_ft_).u32_353 & 0x00ffffffU ) << (4)))
+/*
+TE5 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_WOFF    (352U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_352 = (((_ft_).u32_352 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_352  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE5 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_WOFF      (348U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_348 = (((_ft_).u32_348 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_348  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE4 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_WOFF   (347U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_347 = (((_ft_).u32_347 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_347  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE4 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_WOFF (346U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_WOFF (347U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_346 = (((_ft_).u32_346 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_347 = (((_ft_).u32_347 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_346  >>  (20)) | ((IMG_UINT64)((_ft_).u32_347 & 0x0000ffffU ) << (12)))
+/*
+TE4 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_WOFF (345U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_WOFF (346U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_345 = (((_ft_).u32_345 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_346 = (((_ft_).u32_346 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_345  >>  (24)) | ((IMG_UINT64)((_ft_).u32_346 & 0x000fffffU ) << (8)))
+/*
+TE4 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_WOFF (344U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_WOFF (345U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_344 = (((_ft_).u32_344 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_345 = (((_ft_).u32_345 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_344  >>  (28)) | ((IMG_UINT64)((_ft_).u32_345 & 0x00ffffffU ) << (4)))
+/*
+TE4 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_WOFF    (344U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_344 = (((_ft_).u32_344 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_344  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE4 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_WOFF      (340U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_340 = (((_ft_).u32_340 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_340  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE3 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_WOFF   (339U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_339 = (((_ft_).u32_339 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_339  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE3 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_WOFF (338U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_WOFF (339U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_338 = (((_ft_).u32_338 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_339 = (((_ft_).u32_339 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_338  >>  (20)) | ((IMG_UINT64)((_ft_).u32_339 & 0x0000ffffU ) << (12)))
+/*
+TE3 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_WOFF (337U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_WOFF (338U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_337 = (((_ft_).u32_337 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_338 = (((_ft_).u32_338 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_337  >>  (24)) | ((IMG_UINT64)((_ft_).u32_338 & 0x000fffffU ) << (8)))
+/*
+TE3 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_WOFF (336U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_WOFF (337U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_336 = (((_ft_).u32_336 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_337 = (((_ft_).u32_337 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_336  >>  (28)) | ((IMG_UINT64)((_ft_).u32_337 & 0x00ffffffU ) << (4)))
+/*
+TE3 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_WOFF    (336U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_336 = (((_ft_).u32_336 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_336  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE3 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_WOFF      (332U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_332 = (((_ft_).u32_332 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_332  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE2 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_WOFF   (331U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_331 = (((_ft_).u32_331 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_331  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE2 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_WOFF (330U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_WOFF (331U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_330 = (((_ft_).u32_330 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_331 = (((_ft_).u32_331 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_330  >>  (20)) | ((IMG_UINT64)((_ft_).u32_331 & 0x0000ffffU ) << (12)))
+/*
+TE2 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_WOFF (329U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_WOFF (330U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_329 = (((_ft_).u32_329 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_330 = (((_ft_).u32_330 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_329  >>  (24)) | ((IMG_UINT64)((_ft_).u32_330 & 0x000fffffU ) << (8)))
+/*
+TE2 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_WOFF (328U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_WOFF (329U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_328 = (((_ft_).u32_328 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_329 = (((_ft_).u32_329 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_328  >>  (28)) | ((IMG_UINT64)((_ft_).u32_329 & 0x00ffffffU ) << (4)))
+/*
+TE2 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_WOFF    (328U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_328 = (((_ft_).u32_328 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_328  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE2 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_WOFF      (324U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_324 = (((_ft_).u32_324 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_324  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE1 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_WOFF   (323U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_323 = (((_ft_).u32_323 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_323  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE1 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_WOFF (322U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_WOFF (323U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_322 = (((_ft_).u32_322 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_323 = (((_ft_).u32_323 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_322  >>  (20)) | ((IMG_UINT64)((_ft_).u32_323 & 0x0000ffffU ) << (12)))
+/*
+TE1 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_WOFF (321U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_WOFF (322U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_321 = (((_ft_).u32_321 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_322 = (((_ft_).u32_322 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_321  >>  (24)) | ((IMG_UINT64)((_ft_).u32_322 & 0x000fffffU ) << (8)))
+/*
+TE1 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_WOFF (320U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_WOFF (321U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_320 = (((_ft_).u32_320 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_321 = (((_ft_).u32_321 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_320  >>  (28)) | ((IMG_UINT64)((_ft_).u32_321 & 0x00ffffffU ) << (4)))
+/*
+TE1 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_WOFF    (320U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_320 = (((_ft_).u32_320 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_320  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE1 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_WOFF      (316U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_316 = (((_ft_).u32_316 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_316  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE0 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_WOFF   (315U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_315 = (((_ft_).u32_315 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_315  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE0 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_WOFF (314U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_WOFF (315U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_314 = (((_ft_).u32_314 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_315 = (((_ft_).u32_315 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_314  >>  (20)) | ((IMG_UINT64)((_ft_).u32_315 & 0x0000ffffU ) << (12)))
+/*
+TE0 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_WOFF (313U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_WOFF (314U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_313 = (((_ft_).u32_313 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_314 = (((_ft_).u32_314 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_313  >>  (24)) | ((IMG_UINT64)((_ft_).u32_314 & 0x000fffffU ) << (8)))
+/*
+TE0 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_WOFF (312U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_WOFF (313U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_312 = (((_ft_).u32_312 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_313 = (((_ft_).u32_313 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_312  >>  (28)) | ((IMG_UINT64)((_ft_).u32_313 & 0x00ffffffU ) << (4)))
+/*
+TE0 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_WOFF    (312U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_312 = (((_ft_).u32_312 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_312  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE0 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_WOFF      (308U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_308 = (((_ft_).u32_308 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_308  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_WOFF  (307U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_307 = (((_ft_).u32_307 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_307  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_WOFF (306U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_WOFF (307U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_306 = (((_ft_).u32_306 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_307 = (((_ft_).u32_307 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_306  >>  (20)) | ((IMG_UINT64)((_ft_).u32_307 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_WOFF (305U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_WOFF (306U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_305 = (((_ft_).u32_305 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_306 = (((_ft_).u32_306 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_305  >>  (24)) | ((IMG_UINT64)((_ft_).u32_306 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_WOFF (304U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_WOFF (305U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_304 = (((_ft_).u32_304 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_305 = (((_ft_).u32_305 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_304  >>  (28)) | ((IMG_UINT64)((_ft_).u32_305 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_WOFF   (304U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_304 = (((_ft_).u32_304 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_304  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_WOFF     (300U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_300 = (((_ft_).u32_300 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_300  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_WOFF  (299U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_299 = (((_ft_).u32_299 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_299  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_WOFF (298U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_WOFF (299U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_298 = (((_ft_).u32_298 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_299 = (((_ft_).u32_299 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_298  >>  (20)) | ((IMG_UINT64)((_ft_).u32_299 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_WOFF (297U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_WOFF (298U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_297 = (((_ft_).u32_297 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_298 = (((_ft_).u32_298 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_297  >>  (24)) | ((IMG_UINT64)((_ft_).u32_298 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_WOFF (296U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_WOFF (297U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_296 = (((_ft_).u32_296 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_297 = (((_ft_).u32_297 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_296  >>  (28)) | ((IMG_UINT64)((_ft_).u32_297 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_WOFF   (296U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_296 = (((_ft_).u32_296 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_296  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_WOFF     (292U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_292 = (((_ft_).u32_292 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_292  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_WOFF  (291U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_291 = (((_ft_).u32_291 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_291  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_WOFF (290U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_WOFF (291U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_290 = (((_ft_).u32_290 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_291 = (((_ft_).u32_291 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_290  >>  (20)) | ((IMG_UINT64)((_ft_).u32_291 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_WOFF (289U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_WOFF (290U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_289 = (((_ft_).u32_289 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_290 = (((_ft_).u32_290 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_289  >>  (24)) | ((IMG_UINT64)((_ft_).u32_290 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_WOFF (288U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_WOFF (289U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_288 = (((_ft_).u32_288 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_289 = (((_ft_).u32_289 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_288  >>  (28)) | ((IMG_UINT64)((_ft_).u32_289 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_WOFF   (288U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_288 = (((_ft_).u32_288 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_288  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_WOFF     (284U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_284 = (((_ft_).u32_284 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_284  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_WOFF  (283U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_283 = (((_ft_).u32_283 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_283  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_WOFF (282U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_WOFF (283U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_282 = (((_ft_).u32_282 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_283 = (((_ft_).u32_283 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_282  >>  (20)) | ((IMG_UINT64)((_ft_).u32_283 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_WOFF (281U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_WOFF (282U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_281 = (((_ft_).u32_281 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_282 = (((_ft_).u32_282 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_281  >>  (24)) | ((IMG_UINT64)((_ft_).u32_282 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_WOFF (280U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_WOFF (281U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_280 = (((_ft_).u32_280 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_281 = (((_ft_).u32_281 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_280  >>  (28)) | ((IMG_UINT64)((_ft_).u32_281 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_WOFF   (280U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_280 = (((_ft_).u32_280 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_280  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_WOFF     (276U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_276 = (((_ft_).u32_276 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_276  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE6 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_WOFF  (259U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_259 = (((_ft_).u32_259 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_259  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE6 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_WOFF (258U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_WOFF (259U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_258 = (((_ft_).u32_258 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_259 = (((_ft_).u32_259 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_258  >>  (20)) | ((IMG_UINT64)((_ft_).u32_259 & 0x0000ffffU ) << (12)))
+/*
+VCE6 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_WOFF (257U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_WOFF (258U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_257 = (((_ft_).u32_257 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_258 = (((_ft_).u32_258 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_257  >>  (24)) | ((IMG_UINT64)((_ft_).u32_258 & 0x000fffffU ) << (8)))
+/*
+VCE6 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_WOFF (256U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_WOFF (257U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_256 = (((_ft_).u32_256 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_257 = (((_ft_).u32_257 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_256  >>  (28)) | ((IMG_UINT64)((_ft_).u32_257 & 0x00ffffffU ) << (4)))
+/*
+VCE6 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_WOFF   (256U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_256 = (((_ft_).u32_256 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_256  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE6 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_WOFF     (252U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_252 = (((_ft_).u32_252 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_252  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE6 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_WOFF  (251U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_251 = (((_ft_).u32_251 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_251  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE6 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_WOFF (250U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_WOFF (251U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_250 = (((_ft_).u32_250 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_251 = (((_ft_).u32_251 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_250  >>  (20)) | ((IMG_UINT64)((_ft_).u32_251 & 0x0000ffffU ) << (12)))
+/*
+VCE6 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_WOFF (249U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_WOFF (250U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_249 = (((_ft_).u32_249 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_250 = (((_ft_).u32_250 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_249  >>  (24)) | ((IMG_UINT64)((_ft_).u32_250 & 0x000fffffU ) << (8)))
+/*
+VCE6 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_WOFF (248U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_WOFF (249U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_248 = (((_ft_).u32_248 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_249 = (((_ft_).u32_249 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_248  >>  (28)) | ((IMG_UINT64)((_ft_).u32_249 & 0x00ffffffU ) << (4)))
+/*
+VCE6 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_WOFF   (248U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_248 = (((_ft_).u32_248 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_248  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE6 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_WOFF     (244U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_244 = (((_ft_).u32_244 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_244  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_WOFF  (243U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_243 = (((_ft_).u32_243 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_243  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_WOFF (242U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_WOFF (243U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_242 = (((_ft_).u32_242 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_243 = (((_ft_).u32_243 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_242  >>  (20)) | ((IMG_UINT64)((_ft_).u32_243 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_WOFF (241U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_WOFF (242U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_241 = (((_ft_).u32_241 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_242 = (((_ft_).u32_242 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_241  >>  (24)) | ((IMG_UINT64)((_ft_).u32_242 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_WOFF (240U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_WOFF (241U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_240 = (((_ft_).u32_240 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_241 = (((_ft_).u32_241 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_240  >>  (28)) | ((IMG_UINT64)((_ft_).u32_241 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_WOFF   (240U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_240 = (((_ft_).u32_240 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_240  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_WOFF     (236U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_236 = (((_ft_).u32_236 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_236  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_WOFF  (235U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_235 = (((_ft_).u32_235 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_235  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_WOFF (234U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_WOFF (235U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_234 = (((_ft_).u32_234 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_235 = (((_ft_).u32_235 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_234  >>  (20)) | ((IMG_UINT64)((_ft_).u32_235 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_WOFF (233U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_WOFF (234U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_233 = (((_ft_).u32_233 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_234 = (((_ft_).u32_234 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_233  >>  (24)) | ((IMG_UINT64)((_ft_).u32_234 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_WOFF (232U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_WOFF (233U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_232 = (((_ft_).u32_232 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_233 = (((_ft_).u32_233 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_232  >>  (28)) | ((IMG_UINT64)((_ft_).u32_233 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_WOFF   (232U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_232 = (((_ft_).u32_232 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_232  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_WOFF     (228U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_228 = (((_ft_).u32_228 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_228  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_WOFF  (227U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_227 = (((_ft_).u32_227 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_227  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_WOFF (226U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_WOFF (227U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_226 = (((_ft_).u32_226 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_227 = (((_ft_).u32_227 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_226  >>  (20)) | ((IMG_UINT64)((_ft_).u32_227 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_WOFF (225U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_WOFF (226U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_225 = (((_ft_).u32_225 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_226 = (((_ft_).u32_226 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_225  >>  (24)) | ((IMG_UINT64)((_ft_).u32_226 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_WOFF (224U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_WOFF (225U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_224 = (((_ft_).u32_224 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_225 = (((_ft_).u32_225 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_224  >>  (28)) | ((IMG_UINT64)((_ft_).u32_225 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_WOFF   (224U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_224 = (((_ft_).u32_224 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_224  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_WOFF     (220U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_220 = (((_ft_).u32_220 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_220  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_WOFF  (219U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_219 = (((_ft_).u32_219 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_219  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_WOFF (218U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_WOFF (219U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_218 = (((_ft_).u32_218 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_219 = (((_ft_).u32_219 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_218  >>  (20)) | ((IMG_UINT64)((_ft_).u32_219 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_WOFF (217U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_WOFF (218U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_217 = (((_ft_).u32_217 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_218 = (((_ft_).u32_218 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_217  >>  (24)) | ((IMG_UINT64)((_ft_).u32_218 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_WOFF (216U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_WOFF (217U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_216 = (((_ft_).u32_216 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_217 = (((_ft_).u32_217 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_216  >>  (28)) | ((IMG_UINT64)((_ft_).u32_217 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_WOFF   (216U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_216 = (((_ft_).u32_216 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_216  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_WOFF     (212U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_212 = (((_ft_).u32_212 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_212  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_WOFF  (211U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_211 = (((_ft_).u32_211 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_211  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_WOFF (210U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_WOFF (211U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_210 = (((_ft_).u32_210 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_211 = (((_ft_).u32_211 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_210  >>  (20)) | ((IMG_UINT64)((_ft_).u32_211 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_WOFF (209U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_WOFF (210U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_209 = (((_ft_).u32_209 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_210 = (((_ft_).u32_210 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_209  >>  (24)) | ((IMG_UINT64)((_ft_).u32_210 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_WOFF (208U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_WOFF (209U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_208 = (((_ft_).u32_208 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_209 = (((_ft_).u32_209 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_208  >>  (28)) | ((IMG_UINT64)((_ft_).u32_209 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_WOFF   (208U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_208 = (((_ft_).u32_208 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_208  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_WOFF     (204U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_204 = (((_ft_).u32_204 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_204  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_WOFF  (203U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_203 = (((_ft_).u32_203 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_203  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_WOFF (202U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_WOFF (203U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_202 = (((_ft_).u32_202 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_203 = (((_ft_).u32_203 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_202  >>  (20)) | ((IMG_UINT64)((_ft_).u32_203 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_WOFF (201U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_WOFF (202U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_201 = (((_ft_).u32_201 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_202 = (((_ft_).u32_202 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_201  >>  (24)) | ((IMG_UINT64)((_ft_).u32_202 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_WOFF (200U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_WOFF (201U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_200 = (((_ft_).u32_200 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_201 = (((_ft_).u32_201 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_200  >>  (28)) | ((IMG_UINT64)((_ft_).u32_201 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_WOFF   (200U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_200 = (((_ft_).u32_200 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_200  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_WOFF     (196U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_196 = (((_ft_).u32_196 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_196  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_WOFF  (195U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_195 = (((_ft_).u32_195 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_195  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_WOFF (194U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_WOFF (195U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_194 = (((_ft_).u32_194 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_195 = (((_ft_).u32_195 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_194  >>  (20)) | ((IMG_UINT64)((_ft_).u32_195 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_WOFF (193U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_WOFF (194U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_193 = (((_ft_).u32_193 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_194 = (((_ft_).u32_194 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_193  >>  (24)) | ((IMG_UINT64)((_ft_).u32_194 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_WOFF (192U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_WOFF (193U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_192 = (((_ft_).u32_192 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_193 = (((_ft_).u32_193 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_192  >>  (28)) | ((IMG_UINT64)((_ft_).u32_193 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_WOFF   (192U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_192 = (((_ft_).u32_192 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_192  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_WOFF     (188U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_188 = (((_ft_).u32_188 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_188  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_WOFF  (187U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_187 = (((_ft_).u32_187 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_187  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_WOFF (186U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_WOFF (187U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_186 = (((_ft_).u32_186 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_187 = (((_ft_).u32_187 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_186  >>  (20)) | ((IMG_UINT64)((_ft_).u32_187 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_WOFF (185U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_WOFF (186U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_185 = (((_ft_).u32_185 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_186 = (((_ft_).u32_186 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_185  >>  (24)) | ((IMG_UINT64)((_ft_).u32_186 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_WOFF (184U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_WOFF (185U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_184 = (((_ft_).u32_184 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_185 = (((_ft_).u32_185 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_184  >>  (28)) | ((IMG_UINT64)((_ft_).u32_185 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_WOFF   (184U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_184 = (((_ft_).u32_184 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_184  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_WOFF     (180U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_180 = (((_ft_).u32_180 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_180  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_WOFF  (179U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_179 = (((_ft_).u32_179 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_179  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_WOFF (178U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_WOFF (179U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_178 = (((_ft_).u32_178 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_179 = (((_ft_).u32_179 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_178  >>  (20)) | ((IMG_UINT64)((_ft_).u32_179 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_WOFF (177U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_WOFF (178U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_177 = (((_ft_).u32_177 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_178 = (((_ft_).u32_178 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_177  >>  (24)) | ((IMG_UINT64)((_ft_).u32_178 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_WOFF (176U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_WOFF (177U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_176 = (((_ft_).u32_176 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_177 = (((_ft_).u32_177 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_176  >>  (28)) | ((IMG_UINT64)((_ft_).u32_177 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_WOFF   (176U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_176 = (((_ft_).u32_176 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_176  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_WOFF     (172U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_172 = (((_ft_).u32_172 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_172  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_WOFF  (171U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_171 = (((_ft_).u32_171 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_171  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_WOFF (170U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_WOFF (171U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_170 = (((_ft_).u32_170 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_171 = (((_ft_).u32_171 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_170  >>  (20)) | ((IMG_UINT64)((_ft_).u32_171 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_WOFF (169U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_WOFF (170U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_169 = (((_ft_).u32_169 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_170 = (((_ft_).u32_170 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_169  >>  (24)) | ((IMG_UINT64)((_ft_).u32_170 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_WOFF (168U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_WOFF (169U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_168 = (((_ft_).u32_168 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_169 = (((_ft_).u32_169 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_168  >>  (28)) | ((IMG_UINT64)((_ft_).u32_169 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_WOFF   (168U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_168 = (((_ft_).u32_168 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_168  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_WOFF     (164U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_164 = (((_ft_).u32_164 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_164  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_WOFF  (163U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_163 = (((_ft_).u32_163 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_163  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_WOFF (162U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_WOFF (163U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_162 = (((_ft_).u32_162 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_163 = (((_ft_).u32_163 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_162  >>  (20)) | ((IMG_UINT64)((_ft_).u32_163 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_WOFF (161U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_WOFF (162U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_161 = (((_ft_).u32_161 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_162 = (((_ft_).u32_162 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_161  >>  (24)) | ((IMG_UINT64)((_ft_).u32_162 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_WOFF (160U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_WOFF (161U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_160 = (((_ft_).u32_160 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_161 = (((_ft_).u32_161 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_160  >>  (28)) | ((IMG_UINT64)((_ft_).u32_161 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_WOFF   (160U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_160 = (((_ft_).u32_160 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_160  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_WOFF     (156U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_156 = (((_ft_).u32_156 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_156  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_WOFF  (155U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_155 = (((_ft_).u32_155 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_155  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_WOFF (154U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_WOFF (155U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_154 = (((_ft_).u32_154 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_155 = (((_ft_).u32_155 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_154  >>  (20)) | ((IMG_UINT64)((_ft_).u32_155 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_WOFF (153U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_WOFF (154U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_153 = (((_ft_).u32_153 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_154 = (((_ft_).u32_154 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_153  >>  (24)) | ((IMG_UINT64)((_ft_).u32_154 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_WOFF (152U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_WOFF (153U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_152 = (((_ft_).u32_152 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_153 = (((_ft_).u32_153 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_152  >>  (28)) | ((IMG_UINT64)((_ft_).u32_153 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_WOFF   (152U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_152 = (((_ft_).u32_152 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_152  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_WOFF     (148U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_148 = (((_ft_).u32_148 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_148  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_WOFF  (147U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_147 = (((_ft_).u32_147 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_147  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_WOFF (146U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_WOFF (147U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_146 = (((_ft_).u32_146 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_147 = (((_ft_).u32_147 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_146  >>  (20)) | ((IMG_UINT64)((_ft_).u32_147 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_WOFF (145U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_WOFF (146U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_145 = (((_ft_).u32_145 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_146 = (((_ft_).u32_146 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_145  >>  (24)) | ((IMG_UINT64)((_ft_).u32_146 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_WOFF (144U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_WOFF (145U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_144 = (((_ft_).u32_144 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_145 = (((_ft_).u32_145 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_144  >>  (28)) | ((IMG_UINT64)((_ft_).u32_145 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_WOFF   (144U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_144 = (((_ft_).u32_144 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_144  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_WOFF     (140U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_140 = (((_ft_).u32_140 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_140  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_WOFF  (139U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_139 = (((_ft_).u32_139 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_139  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_WOFF (138U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_WOFF (139U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_138 = (((_ft_).u32_138 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_139 = (((_ft_).u32_139 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_138  >>  (20)) | ((IMG_UINT64)((_ft_).u32_139 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_WOFF (137U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_WOFF (138U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_137 = (((_ft_).u32_137 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_138 = (((_ft_).u32_138 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_137  >>  (24)) | ((IMG_UINT64)((_ft_).u32_138 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_WOFF (136U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_WOFF (137U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_136 = (((_ft_).u32_136 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_137 = (((_ft_).u32_137 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_136  >>  (28)) | ((IMG_UINT64)((_ft_).u32_137 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_WOFF   (136U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_136 = (((_ft_).u32_136 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_136  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_WOFF     (132U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_132 = (((_ft_).u32_132 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_132  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_WOFF  (131U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_131 = (((_ft_).u32_131 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_131  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_WOFF (130U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_WOFF (131U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_130 = (((_ft_).u32_130 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_131 = (((_ft_).u32_131 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_130  >>  (20)) | ((IMG_UINT64)((_ft_).u32_131 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_WOFF (129U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_WOFF (130U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_129 = (((_ft_).u32_129 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_130 = (((_ft_).u32_130 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_129  >>  (24)) | ((IMG_UINT64)((_ft_).u32_130 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_WOFF (128U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_WOFF (129U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_128 = (((_ft_).u32_128 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_129 = (((_ft_).u32_129 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_128  >>  (28)) | ((IMG_UINT64)((_ft_).u32_129 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_WOFF   (128U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_128 = (((_ft_).u32_128 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_128  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_WOFF     (124U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_124 = (((_ft_).u32_124 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_124  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_WOFF  (123U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_123 = (((_ft_).u32_123 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_123  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_WOFF (122U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_WOFF (123U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_122 = (((_ft_).u32_122 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_123 = (((_ft_).u32_123 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_122  >>  (20)) | ((IMG_UINT64)((_ft_).u32_123 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_WOFF (121U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_WOFF (122U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_121 = (((_ft_).u32_121 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_122 = (((_ft_).u32_122 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_121  >>  (24)) | ((IMG_UINT64)((_ft_).u32_122 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_WOFF (120U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_WOFF (121U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_120 = (((_ft_).u32_120 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_121 = (((_ft_).u32_121 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_120  >>  (28)) | ((IMG_UINT64)((_ft_).u32_121 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_WOFF   (120U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_120 = (((_ft_).u32_120 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_120  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_WOFF     (116U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_116 = (((_ft_).u32_116 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_116  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_WOFF  (115U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_115 = (((_ft_).u32_115 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_115  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_WOFF (114U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_WOFF (115U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_114 = (((_ft_).u32_114 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_115 = (((_ft_).u32_115 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_114  >>  (20)) | ((IMG_UINT64)((_ft_).u32_115 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_WOFF (113U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_WOFF (114U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_113 = (((_ft_).u32_113 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_114 = (((_ft_).u32_114 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_113  >>  (24)) | ((IMG_UINT64)((_ft_).u32_114 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_WOFF (112U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_WOFF (113U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_112 = (((_ft_).u32_112 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_113 = (((_ft_).u32_113 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_112  >>  (28)) | ((IMG_UINT64)((_ft_).u32_113 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_WOFF   (112U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_112 = (((_ft_).u32_112 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_112  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_WOFF     (108U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_108 = (((_ft_).u32_108 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_108  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_WOFF  (107U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_107 = (((_ft_).u32_107 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_107  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_WOFF (106U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_WOFF (107U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_106 = (((_ft_).u32_106 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_107 = (((_ft_).u32_107 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_106  >>  (20)) | ((IMG_UINT64)((_ft_).u32_107 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_WOFF (105U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_WOFF (106U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_105 = (((_ft_).u32_105 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_106 = (((_ft_).u32_106 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_105  >>  (24)) | ((IMG_UINT64)((_ft_).u32_106 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_WOFF (104U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_WOFF (105U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_104 = (((_ft_).u32_104 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_105 = (((_ft_).u32_105 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_104  >>  (28)) | ((IMG_UINT64)((_ft_).u32_105 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_WOFF   (104U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_104 = (((_ft_).u32_104 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_104  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_WOFF     (100U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_100 = (((_ft_).u32_100 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_100  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_WOFF  (99U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_99 = (((_ft_).u32_99 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_99  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_WOFF (98U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_WOFF (99U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_98 = (((_ft_).u32_98 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_99 = (((_ft_).u32_99 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_98  >>  (20)) | ((IMG_UINT64)((_ft_).u32_99 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_WOFF (97U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_WOFF (98U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_97 = (((_ft_).u32_97 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_98 = (((_ft_).u32_98 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_97  >>  (24)) | ((IMG_UINT64)((_ft_).u32_98 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_WOFF (96U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_WOFF (97U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_96 = (((_ft_).u32_96 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_97 = (((_ft_).u32_97 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_96  >>  (28)) | ((IMG_UINT64)((_ft_).u32_97 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_WOFF   (96U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_96 = (((_ft_).u32_96 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_96  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_WOFF     (92U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_92 = (((_ft_).u32_92 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_92  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_WOFF  (91U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_91 = (((_ft_).u32_91 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_91  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_WOFF (90U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_WOFF (91U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_90 = (((_ft_).u32_90 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_91 = (((_ft_).u32_91 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_90  >>  (20)) | ((IMG_UINT64)((_ft_).u32_91 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_WOFF (89U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_WOFF (90U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_89 = (((_ft_).u32_89 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_90 = (((_ft_).u32_90 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_89  >>  (24)) | ((IMG_UINT64)((_ft_).u32_90 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_WOFF (88U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_WOFF (89U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_88 = (((_ft_).u32_88 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_89 = (((_ft_).u32_89 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_88  >>  (28)) | ((IMG_UINT64)((_ft_).u32_89 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_WOFF   (88U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_88 = (((_ft_).u32_88 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_88  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_WOFF     (84U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_84 = (((_ft_).u32_84 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_84  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_WOFF  (83U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_83 = (((_ft_).u32_83 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_83  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_WOFF (82U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_WOFF (83U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_82 = (((_ft_).u32_82 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_83 = (((_ft_).u32_83 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_82  >>  (20)) | ((IMG_UINT64)((_ft_).u32_83 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_WOFF (81U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_WOFF (82U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_81 = (((_ft_).u32_81 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_82 = (((_ft_).u32_82 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_81  >>  (24)) | ((IMG_UINT64)((_ft_).u32_82 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_WOFF (80U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_WOFF (81U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_80 = (((_ft_).u32_80 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_81 = (((_ft_).u32_81 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_80  >>  (28)) | ((IMG_UINT64)((_ft_).u32_81 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_WOFF   (80U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_80 = (((_ft_).u32_80 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_80  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_WOFF     (76U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_76 = (((_ft_).u32_76 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_76  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_WOFF  (75U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_75 = (((_ft_).u32_75 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_75  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_WOFF (74U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_WOFF (75U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_74 = (((_ft_).u32_74 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_75 = (((_ft_).u32_75 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_74  >>  (20)) | ((IMG_UINT64)((_ft_).u32_75 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_WOFF (73U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_WOFF (74U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_73 = (((_ft_).u32_73 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_74 = (((_ft_).u32_74 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_73  >>  (24)) | ((IMG_UINT64)((_ft_).u32_74 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_WOFF (72U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_WOFF (73U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_72 = (((_ft_).u32_72 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_73 = (((_ft_).u32_73 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_72  >>  (28)) | ((IMG_UINT64)((_ft_).u32_73 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_WOFF   (72U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_72 = (((_ft_).u32_72 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_72  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_WOFF     (68U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_68 = (((_ft_).u32_68 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_68  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_WOFF  (67U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_67 = (((_ft_).u32_67 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_67  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_WOFF (66U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_WOFF (67U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_66 = (((_ft_).u32_66 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_67 = (((_ft_).u32_67 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_66  >>  (20)) | ((IMG_UINT64)((_ft_).u32_67 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_WOFF (65U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_WOFF (66U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_65 = (((_ft_).u32_65 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_66 = (((_ft_).u32_66 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_65  >>  (24)) | ((IMG_UINT64)((_ft_).u32_66 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_WOFF (64U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_WOFF (65U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_64 = (((_ft_).u32_64 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_65 = (((_ft_).u32_65 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_64  >>  (28)) | ((IMG_UINT64)((_ft_).u32_65 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_WOFF   (64U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_64 = (((_ft_).u32_64 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_64  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_WOFF     (60U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_60 = (((_ft_).u32_60 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_60  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_WOFF  (59U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_59 = (((_ft_).u32_59 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_59  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_WOFF (58U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_WOFF (59U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_58 = (((_ft_).u32_58 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_59 = (((_ft_).u32_59 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_58  >>  (20)) | ((IMG_UINT64)((_ft_).u32_59 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_WOFF (57U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_WOFF (58U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_57 = (((_ft_).u32_57 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_58 = (((_ft_).u32_58 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_57  >>  (24)) | ((IMG_UINT64)((_ft_).u32_58 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_WOFF (56U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_WOFF (57U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_56 = (((_ft_).u32_56 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_57 = (((_ft_).u32_57 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_56  >>  (28)) | ((IMG_UINT64)((_ft_).u32_57 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_WOFF   (56U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_56 = (((_ft_).u32_56 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_56  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_WOFF     (52U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_52 = (((_ft_).u32_52 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_52  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+Rsv2 area
+*/
+#define PM_DATA_VHEAP_BUFFER_RESV2_W0_WOFF                (42U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W0_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W1_WOFF                (43U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W1_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W2_WOFF                (44U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W2_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W3_WOFF                (45U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W3_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W4_WOFF                (46U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W4_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W5_WOFF                (47U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W5_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W6_WOFF                (48U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W6_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W7_WOFF                (49U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W7_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W8_WOFF                (50U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W8_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W9_WOFF                (51U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W9_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W0_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W1_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W2_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W3_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W4_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W5_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W6_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W7_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W8_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W9_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W0(_ft_,_x_)       ((_ft_).u32_42 = (((_ft_).u32_42 & PM_DATA_VHEAP_BUFFER_RESV2_W0_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W0_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W0(_ft_)           (((_ft_).u32_42  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W0_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W1(_ft_,_x_)       ((_ft_).u32_43 = (((_ft_).u32_43 & PM_DATA_VHEAP_BUFFER_RESV2_W1_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W1_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W1(_ft_)           (((_ft_).u32_43  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W1_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W2(_ft_,_x_)       ((_ft_).u32_44 = (((_ft_).u32_44 & PM_DATA_VHEAP_BUFFER_RESV2_W2_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W2_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W2(_ft_)           (((_ft_).u32_44  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W2_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W3(_ft_,_x_)       ((_ft_).u32_45 = (((_ft_).u32_45 & PM_DATA_VHEAP_BUFFER_RESV2_W3_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W3_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W3(_ft_)           (((_ft_).u32_45  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W3_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W4(_ft_,_x_)       ((_ft_).u32_46 = (((_ft_).u32_46 & PM_DATA_VHEAP_BUFFER_RESV2_W4_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W4_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W4(_ft_)           (((_ft_).u32_46  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W4_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W5(_ft_,_x_)       ((_ft_).u32_47 = (((_ft_).u32_47 & PM_DATA_VHEAP_BUFFER_RESV2_W5_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W5_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W5(_ft_)           (((_ft_).u32_47  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W5_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W6(_ft_,_x_)       ((_ft_).u32_48 = (((_ft_).u32_48 & PM_DATA_VHEAP_BUFFER_RESV2_W6_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W6_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W6(_ft_)           (((_ft_).u32_48  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W6_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W7(_ft_,_x_)       ((_ft_).u32_49 = (((_ft_).u32_49 & PM_DATA_VHEAP_BUFFER_RESV2_W7_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W7_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W7(_ft_)           (((_ft_).u32_49  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W7_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W8(_ft_,_x_)       ((_ft_).u32_50 = (((_ft_).u32_50 & PM_DATA_VHEAP_BUFFER_RESV2_W8_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W8_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W8(_ft_)           (((_ft_).u32_50  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W8_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W9(_ft_,_x_)       ((_ft_).u32_51 = (((_ft_).u32_51 & PM_DATA_VHEAP_BUFFER_RESV2_W9_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W9_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W9(_ft_)           (((_ft_).u32_51  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W9_SHIFT)) & 0xffffffffU)
+/*
+Number of pages allocated to TE7 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_SHIFT       (24U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_CLRMSK      (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE6 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_SHIFT       (16U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_CLRMSK      (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE5 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_SHIFT       (8U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_CLRMSK      (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE4 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_SHIFT       (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_CLRMSK      (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE3 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_SHIFT       (24U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_CLRMSK      (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE2 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_SHIFT       (16U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_CLRMSK      (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE1 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_SHIFT       (8U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_CLRMSK      (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE0 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_SHIFT       (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_CLRMSK      (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE7 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_SHIFT      (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_CLRMSK     (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE6 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_SHIFT      (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_CLRMSK     (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE5 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_SHIFT      (8U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_CLRMSK     (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE4 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_SHIFT      (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_CLRMSK     (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE3 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_SHIFT      (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_CLRMSK     (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE2 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_SHIFT      (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_CLRMSK     (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE1 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_SHIFT      (8U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_CLRMSK     (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE0 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_SHIFT      (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_CLRMSK     (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+1=The PM ran out of memory during processing
+*/
+#define PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_WOFF           (37U)
+#define PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_PM_OUTOFMEM_R(_ft_,_x_)  ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_PM_OUTOFMEM_R(_ft_)      (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_SHIFT)) & 0x00000001U)
+/*
+A copy of rgx_cr_pm_outofmem_abortall (at the point the VHEAP buffer was written)
+*/
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_WOFF          (37U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_SHIFT         (30U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_CLRMSK        (0xBFFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_OUTOFMEM_ABORT(_ft_,_x_) ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_OUTOFMEM_ABORT(_ft_)     (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_SHIFT)) & 0x00000001U)
+/*
+When running out of memory, indicates which of the free stacks have run out of memory.
+If bit 2 is set, MMUSTACK has run out of memory.
+Bit 2 is reserved.
+If bit 1 is set, UFSTACK has run out of memory.
+If bit 0 is set, FSTACK has run out of memory.
+*/
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_WOFF            (37U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_SHIFT           (2U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_CLRMSK          (0xFFFFFFE3U)
+#define PM_DATA_VHEAP_BUFFER_SET_OUTOFMEM_SRC(_ft_,_x_)   ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_CLRMSK ) | (((_x_) & (0x00000007U))  <<  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_OUTOFMEM_SRC(_ft_)       (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_SHIFT)) & 0x00000007U)
+/*
+When running out of memory, indicates the source of the request that caused the OOM event
+If bit 1 is set, TE caused the OOM.
+If bit 0 is set, VCE caused the OOM.
+*/
+#define PM_DATA_VHEAP_BUFFER_REQ_SRC_WOFF                 (37U)
+#define PM_DATA_VHEAP_BUFFER_REQ_SRC_SHIFT                (0U)
+#define PM_DATA_VHEAP_BUFFER_REQ_SRC_CLRMSK               (0xFFFFFFFCU)
+#define PM_DATA_VHEAP_BUFFER_SET_REQ_SRC(_ft_,_x_)        ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_REQ_SRC_CLRMSK ) | (((_x_) & (0x00000003U))  <<  (PM_DATA_VHEAP_BUFFER_REQ_SRC_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_REQ_SRC(_ft_)            (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_REQ_SRC_SHIFT)) & 0x00000003U)
+/*
+MAX RTA index dword in TA stream
+*/
+#define PM_DATA_VHEAP_BUFFER_MAX_RTA_WOFF                 (36U)
+#define PM_DATA_VHEAP_BUFFER_MAX_RTA_SHIFT                (0U)
+#define PM_DATA_VHEAP_BUFFER_MAX_RTA_CLRMSK               (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_MAX_RTA(_ft_,_x_)        ((_ft_).u32_36 = (((_ft_).u32_36 & PM_DATA_VHEAP_BUFFER_MAX_RTA_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (PM_DATA_VHEAP_BUFFER_MAX_RTA_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_MAX_RTA(_ft_)            (((_ft_).u32_36  >>  (PM_DATA_VHEAP_BUFFER_MAX_RTA_SHIFT)) & 0xffffffffU)
+/*
+Rsv1 area
+*/
+#define PM_DATA_VHEAP_BUFFER_RESV1_W0_WOFF                (20U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W0_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W1_WOFF                (21U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W1_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W2_WOFF                (22U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W2_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W3_WOFF                (23U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W3_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W4_WOFF                (24U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W4_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W5_WOFF                (25U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W5_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W6_WOFF                (26U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W6_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W7_WOFF                (27U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W7_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W8_WOFF                (28U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W8_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W9_WOFF                (29U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W9_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W10_WOFF               (30U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W10_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W11_WOFF               (31U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W11_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W12_WOFF               (32U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W12_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W13_WOFF               (33U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W13_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W14_WOFF               (34U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W14_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W15_WOFF               (35U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W15_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W0_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W1_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W2_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W3_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W4_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W5_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W6_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W7_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W8_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W9_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W10_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W11_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W12_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W13_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W14_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W15_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W0(_ft_,_x_)       ((_ft_).u32_20 = (((_ft_).u32_20 & PM_DATA_VHEAP_BUFFER_RESV1_W0_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W0_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W0(_ft_)           (((_ft_).u32_20  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W0_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W1(_ft_,_x_)       ((_ft_).u32_21 = (((_ft_).u32_21 & PM_DATA_VHEAP_BUFFER_RESV1_W1_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W1_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W1(_ft_)           (((_ft_).u32_21  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W1_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W2(_ft_,_x_)       ((_ft_).u32_22 = (((_ft_).u32_22 & PM_DATA_VHEAP_BUFFER_RESV1_W2_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W2_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W2(_ft_)           (((_ft_).u32_22  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W2_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W3(_ft_,_x_)       ((_ft_).u32_23 = (((_ft_).u32_23 & PM_DATA_VHEAP_BUFFER_RESV1_W3_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W3_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W3(_ft_)           (((_ft_).u32_23  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W3_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W4(_ft_,_x_)       ((_ft_).u32_24 = (((_ft_).u32_24 & PM_DATA_VHEAP_BUFFER_RESV1_W4_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W4_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W4(_ft_)           (((_ft_).u32_24  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W4_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W5(_ft_,_x_)       ((_ft_).u32_25 = (((_ft_).u32_25 & PM_DATA_VHEAP_BUFFER_RESV1_W5_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W5_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W5(_ft_)           (((_ft_).u32_25  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W5_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W6(_ft_,_x_)       ((_ft_).u32_26 = (((_ft_).u32_26 & PM_DATA_VHEAP_BUFFER_RESV1_W6_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W6_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W6(_ft_)           (((_ft_).u32_26  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W6_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W7(_ft_,_x_)       ((_ft_).u32_27 = (((_ft_).u32_27 & PM_DATA_VHEAP_BUFFER_RESV1_W7_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W7_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W7(_ft_)           (((_ft_).u32_27  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W7_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W8(_ft_,_x_)       ((_ft_).u32_28 = (((_ft_).u32_28 & PM_DATA_VHEAP_BUFFER_RESV1_W8_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W8_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W8(_ft_)           (((_ft_).u32_28  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W8_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W9(_ft_,_x_)       ((_ft_).u32_29 = (((_ft_).u32_29 & PM_DATA_VHEAP_BUFFER_RESV1_W9_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W9_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W9(_ft_)           (((_ft_).u32_29  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W9_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W10(_ft_,_x_)      ((_ft_).u32_30 = (((_ft_).u32_30 & PM_DATA_VHEAP_BUFFER_RESV1_W10_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W10_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W10(_ft_)          (((_ft_).u32_30  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W10_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W11(_ft_,_x_)      ((_ft_).u32_31 = (((_ft_).u32_31 & PM_DATA_VHEAP_BUFFER_RESV1_W11_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W11_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W11(_ft_)          (((_ft_).u32_31  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W11_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W12(_ft_,_x_)      ((_ft_).u32_32 = (((_ft_).u32_32 & PM_DATA_VHEAP_BUFFER_RESV1_W12_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W12_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W12(_ft_)          (((_ft_).u32_32  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W12_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W13(_ft_,_x_)      ((_ft_).u32_33 = (((_ft_).u32_33 & PM_DATA_VHEAP_BUFFER_RESV1_W13_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W13_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W13(_ft_)          (((_ft_).u32_33  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W13_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W14(_ft_,_x_)      ((_ft_).u32_34 = (((_ft_).u32_34 & PM_DATA_VHEAP_BUFFER_RESV1_W14_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W14_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W14(_ft_)          (((_ft_).u32_34  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W14_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W15(_ft_,_x_)      ((_ft_).u32_35 = (((_ft_).u32_35 & PM_DATA_VHEAP_BUFFER_RESV1_W15_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W15_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W15(_ft_)          (((_ft_).u32_35  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W15_SHIFT)) & 0xffffffffU)
+/*
+Rsv0 area
+*/
+#define PM_DATA_VHEAP_BUFFER_RESV0_WOFF                   (19U)
+#define PM_DATA_VHEAP_BUFFER_RESV0_SHIFT                  (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV0_CLRMSK                 (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV0(_ft_,_x_)          ((_ft_).u32_19 = (((_ft_).u32_19 & PM_DATA_VHEAP_BUFFER_RESV0_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (PM_DATA_VHEAP_BUFFER_RESV0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV0(_ft_)              (((_ft_).u32_19  >>  (PM_DATA_VHEAP_BUFFER_RESV0_SHIFT)) & 0xffffffffU)
+/*
+Init Bit Sent Flag for TE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_WOFF           (18U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE7_INIT(_ft_,_x_)  ((_ft_).u32_18 = (((_ft_).u32_18 & PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE7_INIT(_ft_)      (((_ft_).u32_18  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_WOFF                (18U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE7(_ft_,_x_)       ((_ft_).u32_18 = (((_ft_).u32_18 & PM_DATA_VHEAP_BUFFER_VPTR_TE7_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE7(_ft_)           (((_ft_).u32_18  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_WOFF          (17U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE7_INIT(_ft_,_x_) ((_ft_).u32_17 = (((_ft_).u32_17 & PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE7_INIT(_ft_)     (((_ft_).u32_17  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_WOFF               (17U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE7(_ft_,_x_)      ((_ft_).u32_17 = (((_ft_).u32_17 & PM_DATA_VHEAP_BUFFER_VPTR_VCE7_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE7(_ft_)          (((_ft_).u32_17  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_WOFF           (16U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE6_INIT(_ft_,_x_)  ((_ft_).u32_16 = (((_ft_).u32_16 & PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE6_INIT(_ft_)      (((_ft_).u32_16  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_WOFF                (16U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE6(_ft_,_x_)       ((_ft_).u32_16 = (((_ft_).u32_16 & PM_DATA_VHEAP_BUFFER_VPTR_TE6_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE6(_ft_)           (((_ft_).u32_16  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_WOFF          (15U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE6_INIT(_ft_,_x_) ((_ft_).u32_15 = (((_ft_).u32_15 & PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE6_INIT(_ft_)     (((_ft_).u32_15  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_WOFF               (15U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE6(_ft_,_x_)      ((_ft_).u32_15 = (((_ft_).u32_15 & PM_DATA_VHEAP_BUFFER_VPTR_VCE6_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE6(_ft_)          (((_ft_).u32_15  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_WOFF           (14U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE5_INIT(_ft_,_x_)  ((_ft_).u32_14 = (((_ft_).u32_14 & PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE5_INIT(_ft_)      (((_ft_).u32_14  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_WOFF                (14U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE5(_ft_,_x_)       ((_ft_).u32_14 = (((_ft_).u32_14 & PM_DATA_VHEAP_BUFFER_VPTR_TE5_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE5(_ft_)           (((_ft_).u32_14  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_WOFF          (13U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE5_INIT(_ft_,_x_) ((_ft_).u32_13 = (((_ft_).u32_13 & PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE5_INIT(_ft_)     (((_ft_).u32_13  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_WOFF               (13U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE5(_ft_,_x_)      ((_ft_).u32_13 = (((_ft_).u32_13 & PM_DATA_VHEAP_BUFFER_VPTR_VCE5_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE5(_ft_)          (((_ft_).u32_13  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_WOFF           (12U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE4_INIT(_ft_,_x_)  ((_ft_).u32_12 = (((_ft_).u32_12 & PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE4_INIT(_ft_)      (((_ft_).u32_12  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_WOFF                (12U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE4(_ft_,_x_)       ((_ft_).u32_12 = (((_ft_).u32_12 & PM_DATA_VHEAP_BUFFER_VPTR_TE4_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE4(_ft_)           (((_ft_).u32_12  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_WOFF          (11U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE4_INIT(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE4_INIT(_ft_)     (((_ft_).u32_11  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_WOFF               (11U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE4(_ft_,_x_)      ((_ft_).u32_11 = (((_ft_).u32_11 & PM_DATA_VHEAP_BUFFER_VPTR_VCE4_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE4(_ft_)          (((_ft_).u32_11  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_WOFF           (10U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE3_INIT(_ft_,_x_)  ((_ft_).u32_10 = (((_ft_).u32_10 & PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE3_INIT(_ft_)      (((_ft_).u32_10  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_WOFF                (10U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE3(_ft_,_x_)       ((_ft_).u32_10 = (((_ft_).u32_10 & PM_DATA_VHEAP_BUFFER_VPTR_TE3_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE3(_ft_)           (((_ft_).u32_10  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_WOFF          (9U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE3_INIT(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE3_INIT(_ft_)     (((_ft_).u32_9  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_WOFF               (9U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE3(_ft_,_x_)      ((_ft_).u32_9 = (((_ft_).u32_9 & PM_DATA_VHEAP_BUFFER_VPTR_VCE3_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE3(_ft_)          (((_ft_).u32_9  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_WOFF           (8U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE2_INIT(_ft_,_x_)  ((_ft_).u32_8 = (((_ft_).u32_8 & PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE2_INIT(_ft_)      (((_ft_).u32_8  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_WOFF                (8U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE2(_ft_,_x_)       ((_ft_).u32_8 = (((_ft_).u32_8 & PM_DATA_VHEAP_BUFFER_VPTR_TE2_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE2(_ft_)           (((_ft_).u32_8  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_WOFF          (7U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE2_INIT(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE2_INIT(_ft_)     (((_ft_).u32_7  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_WOFF               (7U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE2(_ft_,_x_)      ((_ft_).u32_7 = (((_ft_).u32_7 & PM_DATA_VHEAP_BUFFER_VPTR_VCE2_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE2(_ft_)          (((_ft_).u32_7  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_WOFF           (6U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE1_INIT(_ft_,_x_)  ((_ft_).u32_6 = (((_ft_).u32_6 & PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE1_INIT(_ft_)      (((_ft_).u32_6  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_WOFF                (6U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE1(_ft_,_x_)       ((_ft_).u32_6 = (((_ft_).u32_6 & PM_DATA_VHEAP_BUFFER_VPTR_TE1_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE1(_ft_)           (((_ft_).u32_6  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_WOFF          (5U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE1_INIT(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE1_INIT(_ft_)     (((_ft_).u32_5  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_WOFF               (5U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE1(_ft_,_x_)      ((_ft_).u32_5 = (((_ft_).u32_5 & PM_DATA_VHEAP_BUFFER_VPTR_VCE1_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE1(_ft_)          (((_ft_).u32_5  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_SHIFT)) & 0x000fffffU)
+/*
+4KB aligned  top pointer for MMU
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_MMU_WOFF                (4U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_MMU_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_MMU_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_MMU(_ft_,_x_)       ((_ft_).u32_4 = (((_ft_).u32_4 & PM_DATA_VHEAP_BUFFER_VPTR_MMU_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_MMU_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_MMU(_ft_)           (((_ft_).u32_4  >>  (PM_DATA_VHEAP_BUFFER_VPTR_MMU_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for ALIST
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_WOFF         (3U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_SHIFT        (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_CLRMSK       (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_ALIST_INIT(_ft_,_x_) ((_ft_).u32_3 = (((_ft_).u32_3 & PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_ALIST_INIT(_ft_)    (((_ft_).u32_3  >>  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for ALIST
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_WOFF              (3U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_SHIFT             (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_CLRMSK            (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_ALIST(_ft_,_x_)     ((_ft_).u32_3 = (((_ft_).u32_3 & PM_DATA_VHEAP_BUFFER_VPTR_ALIST_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_ALIST(_ft_)         (((_ft_).u32_3  >>  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_WOFF           (1U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE0_INIT(_ft_,_x_)  ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE0_INIT(_ft_)      (((_ft_).u32_1  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_WOFF                (1U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE0(_ft_,_x_)       ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VHEAP_BUFFER_VPTR_TE0_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE0(_ft_)           (((_ft_).u32_1  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_WOFF          (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE0_INIT(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE0_INIT(_ft_)     (((_ft_).u32_0  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_WOFF               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE0(_ft_,_x_)      ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VHEAP_BUFFER_VPTR_VCE0_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE0(_ft_)          (((_ft_).u32_0  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_SHIFT)) & 0x000fffffU)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+The PM FreeListState Buffer Layout - this will apply to 3 resources - FSTACK, UFSTACK and MMUSTACK
+*/
+typedef struct RGX_PM_FREELISTSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+} RGX_PM_FREELISTSTATE_BUFFER;
+
+/*
+Reserved field word 2
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_WOFF    (7U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_2(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_2(_ft_) (((_ft_).u32_7  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 1
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_WOFF    (6U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_1(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_1(_ft_) (((_ft_).u32_6  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 0
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_WOFF    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_0(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_0(_ft_) (((_ft_).u32_5  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT)) & 0xffffffffU)
+/*
+The number of pages consumed for the MMU Page Table. Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_WOFF   (4U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT  (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_MMUPAGE_STATUS(_ft_) (((_ft_).u32_4  >>  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The total number of pages consumed from the free stack. Must be initialised to zero.  This field is unused in the MMUSTACK.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_WOFF      (3U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK    (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(_ft_,_x_) ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_PAGE_STATUS(_ft_) (((_ft_).u32_3  >>  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Stack pointer for the free stack - the location of the next free page relative to the BaseAddr, in number of DWORDs.  Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_WOFF        (2U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT       (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK      (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(_ft_,_x_) ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_STACK_PTR(_ft_)   (((_ft_).u32_2  >>  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT)) & 0xffffffffU)
+/*
+Base address of the free stack - points to the bottom of the stack.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_WOFF     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_SHIFT    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_WOFF     (1U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_SHIFT    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK   (0x0000001FU)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK   (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_BASE_ADDR(_ft_)   (((_ft_).u32_0  >>  (5)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (27)))
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+The PM FreeListState Buffer Layout - this will apply to 3 resources - FSTACK, UFSTACK and MMUSTACK
+*/
+typedef struct RGX_PM_FREELISTSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+} RGX_PM_FREELISTSTATE_BUFFER;
+
+/*
+Reserved field word 2
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_WOFF    (7U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_2(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_2(_ft_) (((_ft_).u32_7  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 1
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_WOFF    (6U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_1(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_1(_ft_) (((_ft_).u32_6  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 0
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_WOFF    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_0(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_0(_ft_) (((_ft_).u32_5  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT)) & 0xffffffffU)
+/*
+The number of pages consumed for the MMU Page Table. Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_WOFF   (4U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT  (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_MMUPAGE_STATUS(_ft_) (((_ft_).u32_4  >>  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The total number of pages consumed from the free stack. Must be initialised to zero.  This field is unused in the MMUSTACK.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_WOFF      (3U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK    (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(_ft_,_x_) ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_PAGE_STATUS(_ft_) (((_ft_).u32_3  >>  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Stack pointer for the free stack - the location of the next free page relative to the BaseAddr, in number of DWORDs.  Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_WOFF        (2U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT       (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK      (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(_ft_,_x_) ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_STACK_PTR(_ft_)   (((_ft_).u32_2  >>  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT)) & 0xffffffffU)
+/*
+Base address of the free stack - points to the bottom of the stack.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_WOFF     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_SHIFT    (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_WOFF     (1U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_SHIFT    (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK   (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK   (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_BASE_ADDR(_ft_)   (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (32)))
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+256-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT         (5U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSIZE          (32U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR          (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_LOWER    (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_UPPER    (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT         (4U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSIZE          (16U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR          (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_LOWER    (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_UPPER    (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+/*
+Maximum range supported by hardware is 23 bits.
+*/
+#define RGX_PM_FREELISTSTATE_STACK_PTR_STACK_PTR          (0U)
+#define RGX_PM_FREELISTSTATE_STACK_PTR_STACK_PTR_LOWER    (0U)
+#define RGX_PM_FREELISTSTATE_STACK_PTR_STACK_PTR_UPPER    (16777215U)
+
+
+/*
+Maximum range supported by hardware is 23 bits.
+*/
+#define RGX_PM_FREELISTSTATE_PAGE_STATUS_PAGE_STATUS      (0U)
+#define RGX_PM_FREELISTSTATE_PAGE_STATUS_PAGE_STATUS_LOWER (0U)
+#define RGX_PM_FREELISTSTATE_PAGE_STATUS_PAGE_STATUS_UPPER (16777215U)
+
+
+/*
+Maximum range supported by hardware is 23 bits.
+*/
+#define RGX_PM_FREELISTSTATE_MMUPAGE_STATUS_MMUPAGE_STATUS (0U)
+#define RGX_PM_FREELISTSTATE_MMUPAGE_STATUS_MMUPAGE_STATUS_LOWER (0U)
+#define RGX_PM_FREELISTSTATE_MMUPAGE_STATUS_MMUPAGE_STATUS_UPPER (16777215U)
+
+
+#if defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+The base address of the Virtual-Physical Page Translation Table.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_WOFF   (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_SHIFT  (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_WOFF   (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_SHIFT  (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK (0x0000000FU)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VFP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffffff)))  <<  4))); \
+                                                          ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0ffffffff0000000)))  >>  28))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VFP_BASE_ADDR(_ft_) (((_ft_).u32_10  >>  (4)) | ((IMG_UINT64)((_ft_).u32_11 & 0xffffffffU ) << (28)))
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The base address of the VHEAP buffer.
+Must be initialised to point to the location of the VHEAP buffer in memory.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_SHIFT (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_WOFF (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_SHIFT (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK (0x0000000FU)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffffff)))  <<  4))); \
+                                                          ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0ffffffff0000000)))  >>  28))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VHEAP_BASE_ADDR(_ft_) (((_ft_).u32_6  >>  (4)) | ((IMG_UINT64)((_ft_).u32_7 & 0xffffffffU ) << (28)))
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x0000001FU)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (5)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (27)))
+/*
+The number of entries on the ALIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (32)))
+#endif /* defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE) */
+
+
+#if defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+The base address of the Virtual-Physical Page Translation Table.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_WOFF   (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_WOFF   (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VFP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VFP_BASE_ADDR(_ft_) (((_ft_).u32_10  >>  (0)) | ((IMG_UINT64)((_ft_).u32_11 & 0xffffffffU ) << (32)))
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The base address of the VHEAP buffer.
+Must be initialised to point to the location of the VHEAP buffer in memory.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_WOFF (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VHEAP_BASE_ADDR(_ft_) (((_ft_).u32_6  >>  (0)) | ((IMG_UINT64)((_ft_).u32_7 & 0xffffffffU ) << (32)))
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (0)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (32)))
+/*
+The number of entries on the ALIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (32)))
+#endif /* defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE) */
+
+
+#if defined(RGX_FEATURE_SINGLE_TE_VSPACE)
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+MMU catalogue base address for VCE pipe 3 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_WOFF (37U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_,_x_) ((_ft_).u32_37 = (((_ft_).u32_37 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_) (((_ft_).u32_37  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 3 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 3 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_WOFF (35U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_,_x_) ((_ft_).u32_35 = (((_ft_).u32_35 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_) (((_ft_).u32_35  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 2 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_WOFF (33U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_,_x_) ((_ft_).u32_33 = (((_ft_).u32_33 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_) (((_ft_).u32_33  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 1 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_WOFF (30U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_30 = (((_ft_).u32_30 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_30  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_WOFF (29U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_29 = (((_ft_).u32_29 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_) (((_ft_).u32_29  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for VCE pipe 0 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 0 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_WOFF (26U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_26 = (((_ft_).u32_26 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_26  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_WOFF (25U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_25 = (((_ft_).u32_25 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_) (((_ft_).u32_25  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for TE MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_WOFF (18U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_,_x_) ((_ft_).u32_18 = (((_ft_).u32_18 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_) (((_ft_).u32_18  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_WOFF (17U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_ADDR(_ft_,_x_) ((_ft_).u32_17 = (((_ft_).u32_17 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_ADDR(_ft_) (((_ft_).u32_17  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for ALIST MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_MAPPED(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_MAPPED(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for ALIST INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_WOFF      (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0_INIT(_ft_,_x_) ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0_INIT(_ft_) (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_WOFF           (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0(_ft_,_x_)  ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0(_ft_)      (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_WOFF     (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3_INIT(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3_INIT(_ft_) (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_WOFF          (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3(_ft_)     (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_WOFF     (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2_INIT(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2_INIT(_ft_) (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_WOFF          (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2(_ft_)     (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_WOFF     (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1_INIT(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1_INIT(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_WOFF          (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1(_ft_)     (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_WOFF     (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0_INIT(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0_INIT(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_WOFF          (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0(_ft_)     (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT)) & 0x7fffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_7  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_6  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x0000001FU)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (5)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (27)))
+/*
+Init bit sent flag for ALIST
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_WOFF         (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT        (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK       (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_INIT(_ft_,_x_) ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_INIT(_ft_)    (((_ft_).u32_1  >>  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT)) & 0x00000001U)
+/*
+The number of entries on the ALIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x7fffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0x7fffffffU ) << (32)))
+#endif /* defined(RGX_FEATURE_SINGLE_TE_VSPACE) */
+
+
+#if !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_SINGLE_TE_VSPACE))
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+MMU catalogue base address for VCE pipe 3 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_WOFF (45U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_,_x_) ((_ft_).u32_45 = (((_ft_).u32_45 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_) (((_ft_).u32_45  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 3 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_WOFF (44U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_,_x_) ((_ft_).u32_44 = (((_ft_).u32_44 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_) (((_ft_).u32_44  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 3 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_WOFF (44U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_,_x_) ((_ft_).u32_44 = (((_ft_).u32_44 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_) (((_ft_).u32_44  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_WOFF (43U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_,_x_) ((_ft_).u32_43 = (((_ft_).u32_43 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_) (((_ft_).u32_43  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_WOFF (42U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_,_x_) ((_ft_).u32_42 = (((_ft_).u32_42 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_) (((_ft_).u32_42  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 2 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_WOFF (42U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_,_x_) ((_ft_).u32_42 = (((_ft_).u32_42 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_) (((_ft_).u32_42  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_WOFF (41U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_) (((_ft_).u32_41  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_WOFF (40U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_) (((_ft_).u32_40  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 1 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_WOFF (40U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_) (((_ft_).u32_40  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 3 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_WOFF (37U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE(_ft_,_x_) ((_ft_).u32_37 = (((_ft_).u32_37 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE(_ft_) (((_ft_).u32_37  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 3 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE3_MAPPED(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE3_MAPPED(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 3 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 2 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_WOFF (35U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE(_ft_,_x_) ((_ft_).u32_35 = (((_ft_).u32_35 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE(_ft_) (((_ft_).u32_35  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 2 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE2_MAPPED(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE2_MAPPED(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 2 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 1 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_WOFF (33U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE(_ft_,_x_) ((_ft_).u32_33 = (((_ft_).u32_33 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE(_ft_) (((_ft_).u32_33  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 1 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE1_MAPPED(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE1_MAPPED(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 1 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_WOFF (30U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_30 = (((_ft_).u32_30 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_30  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_WOFF (29U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_29 = (((_ft_).u32_29 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_) (((_ft_).u32_29  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for VCE pipe 0 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 0 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 0 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_WOFF (26U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_26 = (((_ft_).u32_26 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_26  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 0 ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_WOFF (25U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_25 = (((_ft_).u32_25 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_) (((_ft_).u32_25  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for TE  pipe 0 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 0 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_WOFF (18U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_,_x_) ((_ft_).u32_18 = (((_ft_).u32_18 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_) (((_ft_).u32_18  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_WOFF (17U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_ADDR(_ft_,_x_) ((_ft_).u32_17 = (((_ft_).u32_17 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_ADDR(_ft_) (((_ft_).u32_17  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for ALIST MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_MAPPED(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_MAPPED(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for ALIST INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_WOFF      (15U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE3_INIT(_ft_,_x_) ((_ft_).u32_15 = (((_ft_).u32_15 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE3_INIT(_ft_) (((_ft_).u32_15  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_WOFF           (15U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE3(_ft_,_x_)  ((_ft_).u32_15 = (((_ft_).u32_15 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE3(_ft_)      (((_ft_).u32_15  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_WOFF     (14U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3_INIT(_ft_,_x_) ((_ft_).u32_14 = (((_ft_).u32_14 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3_INIT(_ft_) (((_ft_).u32_14  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_WOFF          (14U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3(_ft_,_x_) ((_ft_).u32_14 = (((_ft_).u32_14 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3(_ft_)     (((_ft_).u32_14  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for TE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_WOFF      (13U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE2_INIT(_ft_,_x_) ((_ft_).u32_13 = (((_ft_).u32_13 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE2_INIT(_ft_) (((_ft_).u32_13  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_WOFF           (13U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE2(_ft_,_x_)  ((_ft_).u32_13 = (((_ft_).u32_13 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE2(_ft_)      (((_ft_).u32_13  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_WOFF     (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2_INIT(_ft_,_x_) ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2_INIT(_ft_) (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_WOFF          (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2(_ft_,_x_) ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2(_ft_)     (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for TE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_WOFF      (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE1_INIT(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE1_INIT(_ft_) (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_WOFF           (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE1(_ft_,_x_)  ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE1(_ft_)      (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_WOFF     (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1_INIT(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1_INIT(_ft_) (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_WOFF          (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1(_ft_)     (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_WOFF      (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0_INIT(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0_INIT(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_WOFF           (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0(_ft_,_x_)  ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0(_ft_)      (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_WOFF     (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0_INIT(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0_INIT(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_WOFF          (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0(_ft_)     (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT)) & 0x7fffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_7  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_6  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x0000001FU)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (5)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (27)))
+/*
+Init bit sent flag for ALIST
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_WOFF         (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT        (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK       (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_INIT(_ft_,_x_) ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_INIT(_ft_)    (((_ft_).u32_1  >>  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT)) & 0x00000001U)
+/*
+The number of entries on the ALIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x7fffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0x7fffffffU ) << (32)))
+#endif /* !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_SINGLE_TE_VSPACE)) */
+
+
+/*
+Maximum range supported by hardware is 33 bits.
+*/
+#define RGX_PM_RENDERSTATE_ALIST_TAIL_ALIST_TAIL          (0U)
+#define RGX_PM_RENDERSTATE_ALIST_TAIL_ALIST_TAIL_LOWER    (0U)
+#define RGX_PM_RENDERSTATE_ALIST_TAIL_ALIST_TAIL_UPPER    (8589934591ULL)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+256-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSHIFT     (5U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSIZE      (32U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSHIFT     (4U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSIZE      (16U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+/*
+Maximum range supported by hardware is 33 bits.
+*/
+#define RGX_PM_RENDERSTATE_MLIST_TAIL_MLIST_TAIL          (0U)
+#define RGX_PM_RENDERSTATE_MLIST_TAIL_MLIST_TAIL_LOWER    (0U)
+#define RGX_PM_RENDERSTATE_MLIST_TAIL_MLIST_TAIL_UPPER    (8589934591ULL)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSHIFT     (4U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSIZE      (16U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSHIFT     (4U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSIZE      (16U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+/*
+Only the 16 least-significant bits are used
+*/
+#define RGX_PM_RENDERSTATE_DEALLOC_MASK_STATUS_DEALLOC_MASK_STATUS (0U)
+#define RGX_PM_RENDERSTATE_DEALLOC_MASK_STATUS_DEALLOC_MASK_STATUS_LOWER (0U)
+#define RGX_PM_RENDERSTATE_DEALLOC_MASK_STATUS_DEALLOC_MASK_STATUS_UPPER (65535U)
+
+
+/*
+Only the 16 least-significant bits are used
+*/
+#define RGX_PM_RENDERSTATE_MTILEFREE_STATUS_MTILEFREE_STATUS (0U)
+#define RGX_PM_RENDERSTATE_MTILEFREE_STATUS_MTILEFREE_STATUS_LOWER (0U)
+#define RGX_PM_RENDERSTATE_MTILEFREE_STATUS_MTILEFREE_STATUS_UPPER (65535U)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSHIFT       (4U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSIZE        (16U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR    (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSHIFT       (4U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSIZE        (16U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR    (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#endif /* RGXPMDEFS_H */
+/*****************************************************************************
+ End of file (rgxpmdefs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/cache_ops.h b/drivers/gpu/drm/img/img-volcanic/include/cache_ops.h
new file mode 100644
index 000000000000..a1d714519d59
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/cache_ops.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services cache management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines for cache management which are visible internally
+                and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef CACHE_OPS_H
+#define CACHE_OPS_H
+#include "img_types.h"
+/*!
+* @Defgroup CPUCacheAPIs
+* @{
+*/
+#define CACHE_BATCH_MAX (8U)
+#define MAX_DMA_OPS (34)
+typedef IMG_UINT32 PVRSRV_CACHE_OP;				/*!< Type represents cache maintenance operation */
+#define PVRSRV_CACHE_OP_NONE				0x0U	/*!< No operation */
+#define PVRSRV_CACHE_OP_CLEAN				0x1U	/*!< Flush w/o invalidate */
+#define PVRSRV_CACHE_OP_INVALIDATE			0x2U	/*!< Invalidate w/o flush */
+#define PVRSRV_CACHE_OP_FLUSH				0x3U	/*!< Flush w/ invalidate */
+/*! @} End of Defgroup CPUCacheAPIs */
+
+#endif	/* CACHE_OPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/dc_external.h b/drivers/gpu/drm/img/img-volcanic/include/dc_external.h
new file mode 100644
index 000000000000..2773ba0a833b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/dc_external.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          Display class external
+@Description    Defines DC specific structures which are externally visible
+                (i.e. visible to clients of services), but are also required
+                within services.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DC_EXTERNAL_H
+#define DC_EXTERNAL_H
+
+#include "img_types.h"
+
+/*!
+ * Maximum size of the display name in DC_DISPLAY_INFO
+ */
+#define DC_NAME_SIZE	50
+
+/*!
+ * Maximum planes supported by DC interface based display drivers.
+ */
+#define DC_MAX_PLANES		(4)
+/*!
+ * This contains information about a display.
+ * The structure can be queried by services from the display driver via a
+ * registered callback.
+ *
+ *   Structure: #DC_DISPLAY_INFO_TAG
+ *   Typedef: ::DC_DISPLAY_INFO
+ */
+typedef struct DC_DISPLAY_INFO_TAG
+{
+	IMG_CHAR		szDisplayName[DC_NAME_SIZE];	/*!< Display identifier string */
+	IMG_UINT32		ui32MinDisplayPeriod;			/*!< Minimum number of VSync periods */
+	IMG_UINT32		ui32MaxDisplayPeriod;			/*!< Maximum number of VSync periods */
+	IMG_UINT32		ui32MaxPipes;					/*!< Maximum number of pipes for this display */
+	IMG_BOOL		bUnlatchedSupported;			/*!< Can the device be unlatched? */
+} DC_DISPLAY_INFO;
+
+/*!
+ * When services imports a buffer from the display driver it has to fill
+ * this structure to inform services about the buffer properties.
+ *
+ *   Structure: #DC_BUFFER_IMPORT_INFO_TAG
+ *   Typedef: ::DC_BUFFER_IMPORT_INFO
+ */
+typedef struct DC_BUFFER_IMPORT_INFO_TAG
+{
+	IMG_UINT32		ePixFormat;			/*!< Enum value of type IMG_PIXFMT for the pixel format */
+	IMG_UINT32		ui32BPP;			/*!< Bits per pixel */
+	IMG_UINT32		ui32Width[3];		/*!< Width of the different channels (defined by ePixFormat) */
+	IMG_UINT32		ui32Height[3];		/*!< Height of the different channels (defined by ePixFormat) */
+	IMG_UINT32		ui32ByteStride[3];	/*!< Byte stride of the different channels (defined by ePixFormat) */
+	IMG_UINT32		ui32PrivData[3];	/*!< Private data of the display for each of the channels */
+} DC_BUFFER_IMPORT_INFO;
+
+/* DC_BUFFER_IMPORT_INFO is passed over the bridge so we need to ensure
+ * it has proper size */
+static_assert((sizeof(DC_BUFFER_IMPORT_INFO) % 4U) == 0U, "invalid size of DC_BUFFER_IMPORT_INFO");
+
+/*!
+ * Configuration details of the frame buffer compression module
+ *
+ *   Structure: #DC_FBC_CREATE_INFO_TAG
+ *   Typedef: ::DC_FBC_CREATE_INFO
+ */
+typedef struct DC_FBC_CREATE_INFO_TAG
+{
+	IMG_UINT32		ui32FBCWidth;	/*!< Pixel width that the FBC module is working on */
+	IMG_UINT32		ui32FBCHeight;	/*!< Pixel height that the FBC module is working on */
+	IMG_UINT32		ui32FBCStride;	/*!< Pixel stride that the FBC module is working on */
+	IMG_UINT32		ui32Size;		/*!< Size of the buffer to create */
+} DC_FBC_CREATE_INFO;
+
+/*!
+ * DC buffer details like frame buffer compression and surface properties
+ *
+ *   Structure: #DC_BUFFER_CREATE_INFO_TAG
+ *   Typedef: ::DC_BUFFER_CREATE_INFO
+ */
+typedef struct DC_BUFFER_CREATE_INFO_TAG
+{
+	PVRSRV_SURFACE_INFO		sSurface;	/*!< Surface properties, specified by user */
+	IMG_UINT32				ui32BPP;	/*!< Bits per pixel */
+	DC_FBC_CREATE_INFO		sFBC;		/*!< Frame buffer compressed specific data */
+} DC_BUFFER_CREATE_INFO;
+
+/* DC_BUFFER_CREATE_INFO is passed over the bridge so we need to ensure
+ * it has proper size */
+static_assert((sizeof(DC_BUFFER_CREATE_INFO) % 4U) == 0U, "invalid size of DC_BUFFER_CREATE_INFO");
+
+#endif /* DC_EXTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/devicemem_typedefs.h b/drivers/gpu/drm/img/img-volcanic/include/devicemem_typedefs.h
new file mode 100644
index 000000000000..d51c099eed7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/devicemem_typedefs.h
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Client side part of device memory management -- this file
+                is forked from new_devmem_allocation.h as this one has to
+                reside in the top level include so that client code is able
+                to make use of the typedefs.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_TYPEDEFS_H
+#define DEVICEMEM_TYPEDEFS_H
+
+#include <powervr/mem_types.h>
+#include "img_types.h"
+#include "pvrsrv_memallocflags.h"
+
+typedef struct DEVMEM_CONTEXT_TAG DEVMEM_CONTEXT;     /*!< Convenience typedef for struct DEVMEM_CONTEXT_TAG */
+typedef struct DEVMEM_HEAP_TAG DEVMEM_HEAP;           /*!< Convenience typedef for struct DEVMEM_HEAP_TAG */
+typedef struct DEVMEM_MEMDESC_TAG DEVMEM_MEMDESC;     /*!< Convenience typedef for struct DEVMEM_MEMDESC_TAG */
+typedef struct DEVMEM_PAGELIST_TAG DEVMEM_PAGELIST;   /*!< Convenience typedef for struct DEVMEM_PAGELIST_TAG */
+
+typedef IMG_HANDLE DEVMEM_EXPORTHANDLE;             /*!< Typedef for DeviceMem Export Handle */
+typedef IMG_UINT64 DEVMEM_EXPORTKEY;                /*!< Typedef for DeviceMem Export Key */
+typedef IMG_DEVMEM_SIZE_T DEVMEM_SIZE_T;            /*!< Typedef for DeviceMem SIZE_T */
+typedef IMG_DEVMEM_LOG2ALIGN_T DEVMEM_LOG2ALIGN_T;  /*!< Typedef for DeviceMem LOG2 Alignment */
+
+typedef struct DEVMEMX_PHYS_MEMDESC_TAG DEVMEMX_PHYSDESC;    /*!< Convenience typedef for DevmemX physical */
+typedef struct DEVMEMX_VIRT_MEMDESC_TAG DEVMEMX_VIRTDESC;    /*!< Convenience typedef for DevmemX virtual */
+
+/*! calling code needs all the info in this struct, to be able to pass it around */
+typedef struct
+{
+    /*! A handle to the PMR. */
+    IMG_HANDLE hPMRExportHandle;
+    /*! The "key" to prove we have authorisation to use this PMR */
+    IMG_UINT64 uiPMRExportPassword;
+    /*! Size and alignment properties for this PMR.  Note, these
+       numbers are not trusted in kernel, but we need to cache them
+       client-side in order to allocate from the VM arena.  The kernel
+       will know the actual alignment and size of the PMR and thus
+       would prevent client code from breaching security here.  Ditto
+       for physmem granularity (aka page size) if this is different
+       from alignment */
+    IMG_DEVMEM_SIZE_T uiSize;
+    /*! We call this "contiguity guarantee" to be more precise than
+       calling it "alignment" or "page size", terms which may seem
+       similar but have different emphasis.  The number reported here
+       is the minimum contiguity guarantee from the creator of the
+       PMR.  Now, there is no requirement to allocate that coarsely
+       from the RA.  The alignment given to the RA simply needs to be
+       at least as coarse as the device page size for the heap we
+       ultimately intend to map into.  What is important is that the
+       device MMU data page size is not greater than the minimum
+       contiguity guarantee from the PMR.  This value is reported to
+       the client in order that it can choose to make early checks and
+       perhaps decide which heap (in a variable page size scenario) it
+       would be safe to map this PMR into.  For convenience, the
+       client may choose to use this argument as the alignment of the
+       virtual range he chooses to allocate, but this is _not_
+       necessary and in many cases would be able to get away with a
+       finer alignment, should the heap into which this PMR will be
+       mapped support it. */
+    IMG_DEVMEM_LOG2ALIGN_T uiLog2ContiguityGuarantee;
+} DEVMEM_EXPORTCOOKIE;
+
+/* Enum that describes the operation associated with changing sparse memory */
+typedef IMG_UINT32 SPARSE_MEM_RESIZE_FLAGS;
+#define SPARSE_RESIZE_NONE 0U
+
+	/* This should be set to indicate the change needs allocation */
+#define SPARSE_RESIZE_ALLOC 1U
+
+	/* This should be set to indicate the change needs free */
+#define SPARSE_RESIZE_FREE 2U
+
+#define SPARSE_RESIZE_BOTH (SPARSE_RESIZE_ALLOC | SPARSE_RESIZE_FREE)
+
+/* To use with DevmemSubAllocate() as the default factor if no over-allocation
+ * is desired.
+ */
+#define DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER (1U)
+
+/* Defines the max length for PMR, MemDesc, Device memory History and RI debug
+ * annotations stored in memory, including the null terminator.
+ */
+#define DEVMEM_ANNOTATION_MAX_LEN ((IMG_UINT32)PVR_ANNOTATION_MAX_LEN + 1U)
+
+
+/* Reserved VA space of a heap must always be multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+ * Granularity has been chosen to support the max possible practically used OS page size.
+ */
+#define DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY        0x10000 /* 64KB is MAX anticipated OS page size */
+
+#endif /* #ifndef DEVICEMEM_TYPEDEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/dllist.h b/drivers/gpu/drm/img/img-volcanic/include/dllist.h
new file mode 100644
index 000000000000..f59c26356a87
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/dllist.h
@@ -0,0 +1,423 @@
+/*************************************************************************/ /*!
+@File
+@Title          Double linked list header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Double linked list interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DLLIST_H
+#define DLLIST_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+/*!
+	Pointer to a linked list node
+*/
+typedef struct DLLIST_NODE_TAG	*PDLLIST_NODE;
+
+
+/*!
+	Node in a linked list.
+	A list is comprised of a single list head and 0 to n nodes.
+	The head and nodes are all represented by `DLLIST_NODE`'s.
+	The head node is a special sentinel and should not be associated with
+	any elements in the list.
+
+	For example, the following list of fruits has 3 elements:
+	`... <--> "banana" <--> "apple" <--> head <--> "orange" <--> ...`
+	Therefore, using dllist_foreach_*(head) will iterate just the 3 fruits.
+
+	This is important because a `dllist_is_empty` is when the number of
+	elements == 0 is equivalent to when the head points to itself.
+	Proper use of the dllist_* functions requires a head.
+
+	For example, the following list is improper:
+	`... <--> "banana" <--> "apple" <--> "orange" <--> ...`
+	as one element must be treated as the head, and therefore is ignored.
+*/
+/*
+ * Note: the following structure's size is architecture-dependent and clients
+ * may need to create a mirror of the structure definition if it needs to be
+ * used in a structure shared between host and device.
+ * Consider such clients if any changes are made to this structure.
+ */
+typedef struct DLLIST_NODE_TAG
+{
+	struct DLLIST_NODE_TAG	*psPrevNode;
+	struct DLLIST_NODE_TAG	*psNextNode;
+} DLLIST_NODE;
+
+
+/*!
+	Static initialiser
+*/
+#define DECLARE_DLLIST(n) \
+DLLIST_NODE (n) = {&(n), &(n)}
+
+/*************************************************************************/ /*!
+@Function       dllist_foreach_node
+
+@Description    Walk through all the nodes on the list.
+                Safe against removal of (node).
+
+@Input          list_head              List node to start the operation
+@Input          node                   Current list node
+@Input          next                   Node after the current one
+
+*/
+/*****************************************************************************/
+#define dllist_foreach_node(list_head, node, next)						\
+	for ((node) = (list_head)->psNextNode, (next) = (node)->psNextNode;		\
+		 (node) != (list_head);											\
+		 (node) = (next), (next) = (node)->psNextNode)
+
+#define dllist_foreach_node_backwards(list_head, node, prev)			\
+	for ((node) = (list_head)->psPrevNode, (prev) = (node)->psPrevNode;		\
+		 (node) != (list_head);											\
+		 (node) = (prev), (prev) = (node)->psPrevNode)
+
+
+/*************************************************************************/ /*!
+@Function       dllist_foreach
+
+@Description    Simplification of dllist_foreach_node.
+                Walk through all the nodes on the list.
+                Safe against removal of currently-iterated node.
+
+                Adds utility-macro dllist_cur() to typecast the current node.
+
+@Input          list_head              List node to start the operation
+
+*/
+/*****************************************************************************/
+#define dllist_foreach(list_head)	\
+	for (DLLIST_NODE *DllCurNode = (list_head).psNextNode, *DllNextNode = DllCurNode->psNextNode;		\
+		 DllCurNode != &(list_head);																\
+		 DllCurNode = DllNextNode, DllNextNode = DllCurNode->psNextNode)
+
+#define dllist_foreach_backwards(list_head)	\
+	for (DLLIST_NODE *DllCurNode = (list_head).psPrevNode, *DllPrevNode = DllCurNode->psPrevNode;		\
+		 DllCurNode != &(list_head);																\
+		 DllCurNode = DllPrevNode, DllPrevNode = DllCurNode->psPrevNode)
+
+#define dllist_cur(type, member)	IMG_CONTAINER_OF(DllCurNode, type, member)
+
+/*************************************************************************/ /*!
+@Function       dllist_init
+
+@Description    Initialize a new double linked list
+
+@Input          psListHead             List head Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_init(PDLLIST_NODE psListHead)
+{
+	psListHead->psPrevNode = psListHead;
+	psListHead->psNextNode = psListHead;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_is_empty
+
+@Description    Returns whether the list is empty
+
+@Input          psListHead             List head Node
+
+*/
+/*****************************************************************************/
+static INLINE
+bool dllist_is_empty(const DLLIST_NODE *const psListHead)
+{
+	return (psListHead->psPrevNode == psListHead);
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_add_to_head
+
+@Description    Add psNewNode to head of list psListHead
+
+@Input          psListHead             Head Node
+@Input          psNewNode              New Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_add_to_head(PDLLIST_NODE psListHead, PDLLIST_NODE psNewNode)
+{
+	PDLLIST_NODE psTmp;
+
+	psTmp = psListHead->psNextNode;
+
+	psListHead->psNextNode = psNewNode;
+	psNewNode->psNextNode = psTmp;
+
+	psTmp->psPrevNode = psNewNode;
+	psNewNode->psPrevNode = psListHead;
+}
+
+
+/*************************************************************************/ /*!
+@Function       dllist_add_to_tail
+
+@Description    Add psNewNode to tail of list psListHead
+
+@Input          psListHead             Head Node
+@Input          psNewNode              New Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_add_to_tail(PDLLIST_NODE psListHead, PDLLIST_NODE psNewNode)
+{
+	PDLLIST_NODE psTmp;
+
+	psTmp = psListHead->psPrevNode;
+
+	psListHead->psPrevNode = psNewNode;
+	psNewNode->psPrevNode = psTmp;
+
+	psTmp->psNextNode = psNewNode;
+	psNewNode->psNextNode = psListHead;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_node_is_in_list
+
+@Description    Returns true if psNode is in a list
+
+@Input          psNode                 List node
+
+*/
+/*****************************************************************************/
+static INLINE
+bool dllist_node_is_in_list(const DLLIST_NODE *const psNode)
+{
+	return (psNode->psNextNode != NULL);
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_get_next_node
+
+@Description    Returns the list node after psListHead or NULL psListHead is
+                the only element in the list.
+
+@Input          psListHead             List node to start the operation
+
+*/
+/*****************************************************************************/
+static INLINE
+PDLLIST_NODE dllist_get_next_node(PDLLIST_NODE psListHead)
+{
+	if (psListHead->psNextNode == psListHead)
+	{
+		return NULL;
+	}
+	else
+	{
+		return psListHead->psNextNode;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_get_prev_node
+
+@Description    Returns the list node preceding psListHead or NULL if
+                psListHead is the only element in the list.
+
+@Input          psListHead             List node to start the operation
+
+*/
+/*****************************************************************************/
+static INLINE
+PDLLIST_NODE dllist_get_prev_node(PDLLIST_NODE psListHead)
+{
+	if (psListHead->psPrevNode == psListHead)
+	{
+		return NULL;
+	}
+	else
+	{
+		return psListHead->psPrevNode;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_remove_node
+
+@Description    Removes psListNode from the list where it currently belongs
+
+@Input          psListNode             List node to be removed
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_remove_node(PDLLIST_NODE psListNode)
+{
+	psListNode->psNextNode->psPrevNode = psListNode->psPrevNode;
+	psListNode->psPrevNode->psNextNode = psListNode->psNextNode;
+
+	/* Clear the node to show it's not in a list */
+	psListNode->psPrevNode = NULL;
+	psListNode->psNextNode = NULL;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_replace_head
+
+@Description    Moves the list from psOldHead to psNewHead
+
+@Input          psOldHead              List node to be replaced. Will become a
+                                       head node of an empty list.
+@Input          psNewHead              List node to be inserted. Must be an
+                                       empty list head.
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_replace_head(PDLLIST_NODE psOldHead, PDLLIST_NODE psNewHead)
+{
+	if (dllist_is_empty(psOldHead))
+	{
+		psNewHead->psNextNode = psNewHead;
+		psNewHead->psPrevNode = psNewHead;
+	}
+	else
+	{
+		/* Change the neighbouring nodes */
+		psOldHead->psNextNode->psPrevNode = psNewHead;
+		psOldHead->psPrevNode->psNextNode = psNewHead;
+
+		/* Copy the old data to the new node */
+		psNewHead->psNextNode = psOldHead->psNextNode;
+		psNewHead->psPrevNode = psOldHead->psPrevNode;
+
+		/* Remove links to the previous list */
+		psOldHead->psNextNode = psOldHead;
+		psOldHead->psPrevNode = psOldHead;
+	}
+}
+
+/**************************************************************************/ /*!
+@Function       dllist_insert_list_at_head
+
+@Description    Inserts psInHead list into the head of the psOutHead list.
+                After this operation psOutHead will contain psInHead at the
+                head of the list and the remaining elements that were
+                already in psOutHead will be places after the psInList (so
+                at a tail of the original list).
+
+@Input          psOutHead       List node psInHead will be inserted to.
+@Input          psInHead        List node to be inserted to psOutHead.
+                                After this operation this becomes an empty list.
+*/ /***************************************************************************/
+static INLINE
+void dllist_insert_list_at_head(PDLLIST_NODE psOutHead, PDLLIST_NODE psInHead)
+{
+	PDLLIST_NODE psInHeadNextNode = psInHead->psNextNode;
+	PDLLIST_NODE psOutHeadNextNode = psOutHead->psNextNode;
+
+	if (!dllist_is_empty(psInHead))
+	{
+		psOutHead->psNextNode = psInHeadNextNode;
+		psInHeadNextNode->psPrevNode = psOutHead;
+
+		psInHead->psPrevNode->psNextNode = psOutHeadNextNode;
+		psOutHeadNextNode->psPrevNode = psInHead->psPrevNode;
+
+		dllist_init(psInHead);
+	}
+ }
+
+/*************************************************************************/ /*!
+@Description    Pointer to a dllist comparison callback function.
+@Input          psNode  Pointer to a node in a dllist.
+@Input          psNext  Pointer to psNode's next neighbour.
+*/ /**************************************************************************/
+typedef bool (*DLLIST_CMP_CB)(const DLLIST_NODE *psNode, const DLLIST_NODE *psNext);
+
+/*************************************************************************/ /*!
+@Function       dllist_sort
+
+@Description    Insert-sorts the List in place
+                The cmpr function passes the current and next node,
+                From which the user writes the function responsible
+                for choosing to swap order or not.
+                The function returns true if a swap is required
+
+@Input          psListHead              List Head to be sorted.
+
+@Input          cmpr                    Function pointer to use for sorting
+
+*/
+/*****************************************************************************/
+static INLINE void dllist_sort(PDLLIST_NODE psListHead,
+                        DLLIST_CMP_CB cmpr)
+{
+	DLLIST_NODE *node, *next;
+	DLLIST_NODE sTempHead;
+
+	dllist_init(&sTempHead);
+
+	dllist_foreach_node(psListHead, node, next)
+	{
+		dllist_remove_node(node);
+		dllist_add_to_head(&sTempHead, node);
+	}
+
+	while (!dllist_is_empty(&sTempHead))
+	{
+		DLLIST_NODE *psSmallestNode = NULL;
+
+		dllist_foreach_node(&sTempHead, node, next)
+		{
+			if (!psSmallestNode || cmpr(psSmallestNode, node))
+			{
+				psSmallestNode = node;
+			}
+		}
+
+		dllist_remove_node(psSmallestNode);
+		dllist_add_to_tail(psListHead, psSmallestNode);
+	}
+}
+
+#endif /* DLLIST_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/netlink.h b/drivers/gpu/drm/img/img-volcanic/include/drm/netlink.h
new file mode 100644
index 000000000000..7b0a71fbaab3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/netlink.h
@@ -0,0 +1,147 @@
+/*
+ * @File
+ * @Title       Nulldisp/Netlink interface definition
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+
+#ifndef __NETLINK_H__
+#define __NETLINK_H__
+
+/* For multi-plane pixel formats */
+#define NLPVRDPY_MAX_NUM_PLANES 3
+
+enum nlpvrdpy_cmd {
+	__NLPVRDPY_CMD_INVALID,
+	NLPVRDPY_CMD_CONNECT,
+	NLPVRDPY_CMD_CONNECTED,
+	NLPVRDPY_CMD_DISCONNECT,
+	NLPVRDPY_CMD_FLIP,
+	NLPVRDPY_CMD_FLIPPED,
+	NLPVRDPY_CMD_COPY,
+	NLPVRDPY_CMD_COPIED,
+	__NLPVRDPY_CMD_MAX
+};
+#define NLPVRDPY_CMD_MAX (__NLPVRDPY_CMD_MAX - 1)
+
+enum nlpvrdpy_attr {
+	__NLPVRDPY_ATTR_INVALID,
+	NLPVRDPY_ATTR_NAME,
+	NLPVRDPY_ATTR_MINOR,
+	NLPVRDPY_ATTR_NUM_PLANES,
+	NLPVRDPY_ATTR_WIDTH,
+	NLPVRDPY_ATTR_HEIGHT,
+	NLPVRDPY_ATTR_PIXFMT,
+	NLPVRDPY_ATTR_YUV_CSC,
+	NLPVRDPY_ATTR_YUV_BPP,
+	NLPVRDPY_ATTR_PLANE0_ADDR,
+	NLPVRDPY_ATTR_PLANE0_SIZE,
+	NLPVRDPY_ATTR_PLANE0_OFFSET,
+	NLPVRDPY_ATTR_PLANE0_PITCH,
+	NLPVRDPY_ATTR_PLANE0_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_PLANE1_ADDR,
+	NLPVRDPY_ATTR_PLANE1_SIZE,
+	NLPVRDPY_ATTR_PLANE1_OFFSET,
+	NLPVRDPY_ATTR_PLANE1_PITCH,
+	NLPVRDPY_ATTR_PLANE1_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_PLANE2_ADDR,
+	NLPVRDPY_ATTR_PLANE2_SIZE,
+	NLPVRDPY_ATTR_PLANE2_OFFSET,
+	NLPVRDPY_ATTR_PLANE2_PITCH,
+	NLPVRDPY_ATTR_PLANE2_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_FB_MODIFIER,
+	NLPVRDPY_ATTR_NAMING_REQUIRED,
+	NLPVRDPY_ATTR_PAD,
+	__NLPVRDPY_ATTR_MAX
+};
+#define NLPVRDPY_ATTR_MAX  (__NLPVRDPY_ATTR_MAX - 1)
+
+static struct nla_policy __attribute__((unused))
+nlpvrdpy_policy[NLPVRDPY_ATTR_MAX + 1] = {
+	[NLPVRDPY_ATTR_NAME]                = { .type = NLA_STRING },
+	[NLPVRDPY_ATTR_MINOR]               = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_NUM_PLANES]          = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_WIDTH]               = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_HEIGHT]              = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PIXFMT]              = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_YUV_CSC]             = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_YUV_BPP]             = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_PLANE0_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PLANE1_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PLANE2_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_FB_MODIFIER]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_NAMING_REQUIRED]     = { .type = NLA_FLAG },
+};
+
+#define NLPVRDPY_ATTR_PLANE(index, type)				\
+	({								\
+		enum nlpvrdpy_attr __retval;				\
+									\
+		switch (index) {					\
+		case 0:							\
+			__retval = NLPVRDPY_ATTR_PLANE0_ ## type;	\
+			break;						\
+		case 1:							\
+			__retval = NLPVRDPY_ATTR_PLANE1_ ## type;	\
+			break;						\
+		case 2:							\
+			__retval = NLPVRDPY_ATTR_PLANE2_ ## type;	\
+			break;						\
+		default:						\
+			__retval = __NLPVRDPY_ATTR_INVALID;		\
+			break;						\
+		};							\
+									\
+		__retval;						\
+	})
+
+#endif /* __NETLINK_H__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/nulldisp_drm.h b/drivers/gpu/drm/img/img-volcanic/include/drm/nulldisp_drm.h
new file mode 100644
index 000000000000..8403fb56f1a5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/nulldisp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       Nulldisp DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__NULLDISP_DRM_H__)
+#define __NULLDISP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_nulldisp_gem_create {
+	__u64 size;   /* in */
+	__u32 flags;  /* in */
+	__u32 handle; /* out */
+};
+
+struct drm_nulldisp_gem_mmap {
+	__u32 handle; /* in */
+	__u32 pad;
+	__u64 offset; /* out */
+};
+
+#define NULLDISP_GEM_CPU_PREP_READ   (1 << 0)
+#define NULLDISP_GEM_CPU_PREP_WRITE  (1 << 1)
+#define NULLDISP_GEM_CPU_PREP_NOWAIT (1 << 2)
+
+struct drm_nulldisp_gem_cpu_prep {
+	__u32 handle; /* in */
+	__u32 flags;  /* in */
+};
+
+struct drm_nulldisp_gem_cpu_fini {
+	__u32 handle; /* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_NULLDISP_GEM_CREATE   0x00
+#define DRM_NULLDISP_GEM_MMAP     0x01
+#define DRM_NULLDISP_GEM_CPU_PREP 0x02
+#define DRM_NULLDISP_GEM_CPU_FINI 0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_NULLDISP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CREATE, \
+		 struct drm_nulldisp_gem_create)
+
+#define DRM_IOCTL_NULLDISP_GEM_MMAP \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_MMAP, \
+		 struct drm_nulldisp_gem_mmap)
+
+#define DRM_IOCTL_NULLDISP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CPU_PREP, \
+		struct drm_nulldisp_gem_cpu_prep)
+
+#define DRM_IOCTL_NULLDISP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CPU_FINI, \
+		struct drm_nulldisp_gem_cpu_fini)
+
+#endif /* defined(__NULLDISP_DRM_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/pdp_drm.h b/drivers/gpu/drm/img/img-volcanic/include/drm/pdp_drm.h
new file mode 100644
index 000000000000..f5d747d3ad10
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/pdp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       PDP DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_DRM_H__)
+#define __PDP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_pdp_gem_create {
+	__u64 size;	/* in */
+	__u32 flags;	/* in */
+	__u32 handle;	/* out */
+};
+
+struct drm_pdp_gem_mmap {
+	__u32 handle;	/* in */
+	__u32 pad;
+	__u64 offset;	/* out */
+};
+
+#define PDP_GEM_CPU_PREP_READ	(1 << 0)
+#define PDP_GEM_CPU_PREP_WRITE	(1 << 1)
+#define PDP_GEM_CPU_PREP_NOWAIT	(1 << 2)
+
+struct drm_pdp_gem_cpu_prep {
+	__u32 handle;	/* in */
+	__u32 flags;	/* in */
+};
+
+struct drm_pdp_gem_cpu_fini {
+	__u32 handle;	/* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_PDP_GEM_CREATE		0x00
+#define DRM_PDP_GEM_MMAP		0x01
+#define DRM_PDP_GEM_CPU_PREP		0x02
+#define DRM_PDP_GEM_CPU_FINI		0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_PDP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_CREATE, \
+		 struct drm_pdp_gem_create)
+
+#define DRM_IOCTL_PDP_GEM_MMAP\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_MMAP, \
+		 struct drm_pdp_gem_mmap)
+
+#define DRM_IOCTL_PDP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_PREP, \
+		struct drm_pdp_gem_cpu_prep)
+
+#define DRM_IOCTL_PDP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_FINI, \
+		struct drm_pdp_gem_cpu_fini)
+
+#endif /* defined(__PDP_DRM_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/pvr_drm.h b/drivers/gpu/drm/img/img-volcanic/include/drm/pvr_drm.h
new file mode 100644
index 000000000000..bc76ddf9b597
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/pvr_drm.h
@@ -0,0 +1,164 @@
+/*
+ * @File
+ * @Title       PVR DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_DRM_H__)
+#define __PVR_DRM_H__
+
+#include <linux/types.h>
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+/*
+ * IMPORTANT:
+ * All structures below are designed to be the same size when compiled for 32
+ * and/or 64 bit architectures, i.e. there should be no compiler inserted
+ * padding. This is achieved by sticking to the following rules:
+ * 1) only use fixed width types
+ * 2) always naturally align fields by arranging them appropriately and by using
+ *    padding fields when necessary
+ *
+ * These rules should _always_ be followed when modifying or adding new
+ * structures to this file.
+ */
+
+struct drm_pvr_srvkm_cmd {
+	__u32 bridge_id;
+	__u32 bridge_func_id;
+	__u64 in_data_ptr;
+	__u64 out_data_ptr;
+	__u32 in_data_size;
+	__u32 out_data_size;
+};
+
+struct pvr_sync_rename_ioctl_data {
+	char szName[32];
+};
+
+struct pvr_sw_sync_create_fence_data {
+	char name[32];
+	__s32 fence;
+	__u32 pad;
+	__u64 sync_pt_idx;
+};
+
+typedef struct pvr_sw_sync_create_fence_data pvr_exp_sync_create_fence_data_t;
+
+struct pvr_sw_timeline_advance_data {
+	__u64 sync_pt_idx;
+};
+
+#define PVR_SRVKM_SERVICES_INIT  1
+#define PVR_SRVKM_SYNC_INIT 2
+#define PVR_SRVKM_SYNC_EXP_FENCE_INIT 3
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#define PVR_SRVKM_SERVICES_PAGE_MIGRATE_INIT 4
+#endif
+struct drm_pvr_srvkm_init_data {
+	__u32 init_module;
+};
+
+/* Values used to configure the PVRSRV_DEVICE_INIT_MODE tunable (Linux-only) */
+#define PVRSRV_LINUX_DEV_INIT_ON_PROBE   1
+#define PVRSRV_LINUX_DEV_INIT_ON_OPEN    2
+#define PVRSRV_LINUX_DEV_INIT_ON_CONNECT 3
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+
+/* PVR Services command */
+#define DRM_PVR_SRVKM_CMD			0
+
+/* PVR Sync commands */
+#define DRM_PVR_SYNC_RENAME_CMD			1
+#define DRM_PVR_SYNC_FORCE_SW_ONLY_CMD		2
+
+/* PVR Software Sync commands */
+#define DRM_PVR_SW_SYNC_CREATE_FENCE_CMD	3
+#define DRM_PVR_SW_SYNC_INC_CMD			4
+
+/* PVR Services Render Device Init command */
+#define DRM_PVR_SRVKM_INIT		5
+
+/* PVR Services Export Fence Sync commands */
+#define DRM_PVR_EXP_FENCE_SYNC_FORCE_CMD	6
+#define DRM_PVR_SYNC_CREATE_EXPORT_FENCE_CMD	7
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define	DRM_IOCTL_PVR_SRVKM_CMD	\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SRVKM_CMD, \
+		 struct drm_pvr_srvkm_cmd)
+
+#define DRM_IOCTL_PVR_SYNC_RENAME_CMD \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PVR_SYNC_RENAME_CMD, \
+		struct pvr_sync_rename_ioctl_data)
+
+#define DRM_IOCTL_PVR_SYNC_FORCE_SW_ONLY_CMD \
+	DRM_IO(DRM_COMMAND_BASE + DRM_PVR_SYNC_FORCE_SW_ONLY_CMD)
+
+#define	DRM_IOCTL_PVR_SW_SYNC_CREATE_FENCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SW_SYNC_CREATE_FENCE_CMD, \
+		struct pvr_sw_sync_create_fence_data)
+
+#define DRM_IOCTL_PVR_SW_SYNC_INC_CMD \
+	DRM_IOR(DRM_COMMAND_BASE + DRM_PVR_SW_SYNC_INC_CMD, \
+		struct pvr_sw_timeline_advance_data)
+
+#define DRM_IOCTL_PVR_SRVKM_INIT \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PVR_SRVKM_INIT, \
+		struct drm_pvr_srvkm_init_data)
+
+#define DRM_IOCTL_PVR_EXP_FENCE_SYNC_FORCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_EXP_FENCE_SYNC_FORCE_CMD, \
+		pvr_exp_sync_create_fence_data_t)
+
+#define DRM_IOCTL_PVR_SYNC_CREATE_EXPORT_FENCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SYNC_CREATE_EXPORT_FENCE_CMD, \
+		pvr_exp_sync_create_fence_data_t)
+
+#endif /* defined(__PVR_DRM_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_3dtypes.h b/drivers/gpu/drm/img/img-volcanic/include/img_3dtypes.h
new file mode 100644
index 000000000000..102c6eeda959
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_3dtypes.h
@@ -0,0 +1,248 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global 3D types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines 3D types for use by IMG APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_3DTYPES_H
+#define IMG_3DTYPES_H
+
+#include <powervr/buffer_attribs.h>
+#include "img_types.h"
+#include "img_defs.h"
+
+/**
+ * Comparison functions
+ * This comparison function is defined as:
+ * A {CmpFunc} B
+ * A is a reference value, e.g., incoming depth etc.
+ * B is the sample value, e.g., value in depth buffer.
+ */
+typedef enum _IMG_COMPFUNC_
+{
+	IMG_COMPFUNC_NEVER,			/**< The comparison never succeeds */
+	IMG_COMPFUNC_LESS,			/**< The comparison is a less-than operation */
+	IMG_COMPFUNC_EQUAL,			/**< The comparison is an equal-to operation */
+	IMG_COMPFUNC_LESS_EQUAL,	/**< The comparison is a less-than or equal-to
+									 operation */
+	IMG_COMPFUNC_GREATER,		/**< The comparison is a greater-than operation
+								*/
+	IMG_COMPFUNC_NOT_EQUAL,		/**< The comparison is a no-equal-to operation
+								*/
+	IMG_COMPFUNC_GREATER_EQUAL,	/**< The comparison is a greater-than or
+									 equal-to operation */
+	IMG_COMPFUNC_ALWAYS,		/**< The comparison always succeeds */
+} IMG_COMPFUNC;
+
+/**
+ * Stencil op functions
+ */
+typedef enum _IMG_STENCILOP_
+{
+	IMG_STENCILOP_KEEP,		/**< Keep original value */
+	IMG_STENCILOP_ZERO,		/**< Set stencil to 0 */
+	IMG_STENCILOP_REPLACE,	/**< Replace stencil entry */
+	IMG_STENCILOP_INCR_SAT,	/**< Increment stencil entry, clamping to max */
+	IMG_STENCILOP_DECR_SAT,	/**< Decrement stencil entry, clamping to zero */
+	IMG_STENCILOP_INVERT,	/**< Invert bits in stencil entry */
+	IMG_STENCILOP_INCR,		/**< Increment stencil entry,
+								 wrapping if necessary */
+	IMG_STENCILOP_DECR,		/**< Decrement stencil entry,
+								 wrapping if necessary */
+} IMG_STENCILOP;
+
+/**
+ * Alpha blending allows colours and textures on one surface
+ * to be blended with transparency onto another surface.
+ * These definitions apply to both source and destination blending
+ * states
+ */
+typedef enum _IMG_BLEND_
+{
+	IMG_BLEND_ZERO = 0,        /**< Blend factor is (0,0,0,0) */
+	IMG_BLEND_ONE,             /**< Blend factor is (1,1,1,1) */
+	IMG_BLEND_SRC_COLOUR,      /**< Blend factor is the source colour */
+	IMG_BLEND_INV_SRC_COLOUR,  /**< Blend factor is the inverted source colour
+									(i.e. 1-src_col) */
+	IMG_BLEND_SRC_ALPHA,       /**< Blend factor is the source alpha */
+	IMG_BLEND_INV_SRC_ALPHA,   /**< Blend factor is the inverted source alpha
+									(i.e. 1-src_alpha) */
+	IMG_BLEND_DEST_ALPHA,      /**< Blend factor is the destination alpha */
+	IMG_BLEND_INV_DEST_ALPHA,  /**< Blend factor is the inverted destination
+									alpha */
+	IMG_BLEND_DEST_COLOUR,     /**< Blend factor is the destination colour */
+	IMG_BLEND_INV_DEST_COLOUR, /**< Blend factor is the inverted destination
+									colour */
+	IMG_BLEND_SRC_ALPHASAT,    /**< Blend factor is the alpha saturation (the
+									minimum of (Src alpha,
+									1 - destination alpha)) */
+	IMG_BLEND_BLEND_FACTOR,    /**< Blend factor is a constant */
+	IMG_BLEND_INVBLEND_FACTOR, /**< Blend factor is a constant (inverted)*/
+	IMG_BLEND_SRC1_COLOUR,     /**< Blend factor is the colour outputted from
+									the pixel shader */
+	IMG_BLEND_INV_SRC1_COLOUR, /**< Blend factor is the inverted colour
+									outputted from the pixel shader */
+	IMG_BLEND_SRC1_ALPHA,      /**< Blend factor is the alpha outputted from
+									the pixel shader */
+	IMG_BLEND_INV_SRC1_ALPHA   /**< Blend factor is the inverted alpha
+									outputted from the pixel shader */
+} IMG_BLEND;
+
+/**
+ * The arithmetic operation to perform when blending
+ */
+typedef enum _IMG_BLENDOP_
+{
+	IMG_BLENDOP_ADD = 0,      /**< Result = (Source + Destination) */
+	IMG_BLENDOP_SUBTRACT,     /**< Result = (Source - Destination) */
+	IMG_BLENDOP_REV_SUBTRACT, /**< Result = (Destination - Source) */
+	IMG_BLENDOP_MIN,          /**< Result = min (Source, Destination) */
+	IMG_BLENDOP_MAX           /**< Result = max (Source, Destination) */
+} IMG_BLENDOP;
+
+/**
+ * Logical operation to perform when logic ops are enabled
+ */
+typedef enum _IMG_LOGICOP_
+{
+	IMG_LOGICOP_CLEAR = 0,     /**< Result = 0 */
+	IMG_LOGICOP_SET,           /**< Result = -1 */
+	IMG_LOGICOP_COPY,          /**< Result = Source */
+	IMG_LOGICOP_COPY_INVERTED, /**< Result = ~Source */
+	IMG_LOGICOP_NOOP,          /**< Result = Destination */
+	IMG_LOGICOP_INVERT,        /**< Result = ~Destination */
+	IMG_LOGICOP_AND,           /**< Result = Source & Destination */
+	IMG_LOGICOP_NAND,          /**< Result = ~(Source & Destination) */
+	IMG_LOGICOP_OR,            /**< Result = Source | Destination */
+	IMG_LOGICOP_NOR,           /**< Result = ~(Source | Destination) */
+	IMG_LOGICOP_XOR,           /**< Result = Source ^ Destination */
+	IMG_LOGICOP_EQUIV,         /**< Result = ~(Source ^ Destination) */
+	IMG_LOGICOP_AND_REVERSE,   /**< Result = Source & ~Destination */
+	IMG_LOGICOP_AND_INVERTED,  /**< Result = ~Source & Destination */
+	IMG_LOGICOP_OR_REVERSE,    /**< Result = Source | ~Destination */
+	IMG_LOGICOP_OR_INVERTED    /**< Result = ~Source | Destination */
+} IMG_LOGICOP;
+
+/**
+ * Type of fog blending supported
+ */
+typedef enum _IMG_FOGMODE_
+{
+	IMG_FOGMODE_NONE, /**< No fog blending - fog calculations are
+					   *   based on the value output from the vertex phase */
+	IMG_FOGMODE_LINEAR, /**< Linear interpolation */
+	IMG_FOGMODE_EXP, /**< Exponential */
+	IMG_FOGMODE_EXP2, /**< Exponential squaring */
+} IMG_FOGMODE;
+
+/**
+ * Types of filtering
+ */
+typedef enum _IMG_FILTER_
+{
+	IMG_FILTER_DONTCARE,	/**< Any filtering mode is acceptable */
+	IMG_FILTER_POINT,		/**< Point filtering */
+	IMG_FILTER_LINEAR,		/**< Bi-linear filtering */
+	IMG_FILTER_BICUBIC,		/**< Bi-cubic filtering */
+} IMG_FILTER;
+
+/**
+ * Addressing modes for textures
+ */
+typedef enum _IMG_ADDRESSMODE_
+{
+	IMG_ADDRESSMODE_DONTCARE,
+	IMG_ADDRESSMODE_REPEAT,	/**< Texture repeats continuously */
+	IMG_ADDRESSMODE_FLIP, /**< Texture flips on odd integer part */
+	IMG_ADDRESSMODE_CLAMP, /**< Texture clamped at 0 or 1 */
+	IMG_ADDRESSMODE_FLIPCLAMP, /**< Flipped once, then clamp */
+	IMG_ADDRESSMODE_CLAMPBORDER,
+	IMG_ADDRESSMODE_OGL_CLAMP,
+	IMG_ADDRESSMODE_OVG_TILEFILL,
+} IMG_ADDRESSMODE;
+
+/**
+ * Culling based on winding order of triangle.
+ */
+typedef enum _IMG_CULLMODE_
+{
+	IMG_CULLMODE_NONE,			/**< Don't cull */
+	IMG_CULLMODE_FRONTFACING,	/**< Front facing triangles */
+	IMG_CULLMODE_BACKFACING,	/**< Back facing triangles */
+} IMG_CULLMODE;
+
+/**
+ * Colour for clearing surfaces.
+ *  The four elements of the 4 x 32 bit array will map to colour
+ *  R,G,B,A components, in order.
+ *  For YUV colour space the order is Y,U,V.
+ *  For Depth and Stencil formats D maps to R and S maps to G.
+ */
+typedef union IMG_CLEAR_COLOUR_TAG {
+	IMG_UINT32        aui32[4];
+	IMG_INT32         ai32[4];
+	IMG_FLOAT         af32[4];
+} IMG_CLEAR_COLOUR;
+
+static_assert(sizeof(IMG_FLOAT) == sizeof(IMG_INT32), "Size of IMG_FLOAT is not 32 bits.");
+
+/*! ************************************************************************//**
+@brief          Specifies the MSAA resolve operation.
+*/ /**************************************************************************/
+typedef enum _IMG_RESOLVE_OP_
+{
+	IMG_RESOLVE_BLEND   = 0,          /*!< box filter on the samples */
+	IMG_RESOLVE_MIN     = 1,          /*!< minimum of the samples */
+	IMG_RESOLVE_MAX     = 2,          /*!< maximum of the samples */
+	IMG_RESOLVE_SAMPLE0 = 3,          /*!< choose sample 0 */
+	IMG_RESOLVE_SAMPLE1 = 4,          /*!< choose sample 1 */
+	IMG_RESOLVE_SAMPLE2 = 5,          /*!< choose sample 2 */
+	IMG_RESOLVE_SAMPLE3 = 6,          /*!< choose sample 3 */
+	IMG_RESOLVE_SAMPLE4 = 7,          /*!< choose sample 4 */
+	IMG_RESOLVE_SAMPLE5 = 8,          /*!< choose sample 5 */
+	IMG_RESOLVE_SAMPLE6 = 9,          /*!< choose sample 6 */
+	IMG_RESOLVE_SAMPLE7 = 10,         /*!< choose sample 7 */
+} IMG_RESOLVE_OP;
+
+
+#endif /* IMG_3DTYPES_H */
+/******************************************************************************
+ End of file (img_3dtypes.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_defs.h b/drivers/gpu/drm/img/img-volcanic/include/img_defs.h
new file mode 100644
index 000000000000..4cb065edb788
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_defs.h
@@ -0,0 +1,629 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common header containing type definitions for portability
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains variable and structure definitions. Any platform
+                specific types should be defined in this file.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DEFS_H
+#define IMG_DEFS_H
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include <linux/types.h>
+#else
+#include <stddef.h>
+#endif
+#if !(defined(__linux__) && defined(__KERNEL__))
+#if defined(__riscv)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wundef"
+#endif
+#include <assert.h>
+#if defined(__riscv)
+#pragma GCC diagnostic pop
+#endif
+#endif
+
+#include "img_types.h"
+
+#if defined(NO_INLINE_FUNCS)
+	#define	INLINE
+	#define	FORCE_INLINE
+#else
+#if defined(__cplusplus) || defined(INTEGRITY_OS)
+	#if	!defined(INLINE)
+		#define INLINE				inline
+	#endif
+	#define	FORCE_INLINE			static inline
+#else
+#if	!defined(INLINE)
+	#define	INLINE					__inline
+#endif
+#if (defined(UNDER_WDDM) || defined(WINDOWS_WDF)) && defined(_X86_)
+	#define	FORCE_INLINE			__forceinline
+#else
+	#define	FORCE_INLINE			static __inline
+#endif
+#endif
+#endif
+
+/* True if the GCC version is at least the given version. False for older
+ * versions of GCC, or other compilers.
+ */
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+/* Use Clang's __has_extension and __has_builtin macros if available. */
+#if defined(__has_extension)
+#define has_clang_extension(e) __has_extension(e)
+#else
+#define has_clang_extension(e) 0
+#endif
+
+#if defined(__has_builtin)
+#define has_clang_builtin(e) __has_builtin(e)
+#else
+#define has_clang_builtin(e) 0
+#endif
+
+/* Use this in any file, or use attributes under GCC - see below */
+#ifndef PVR_UNREFERENCED_PARAMETER
+#define	PVR_UNREFERENCED_PARAMETER(param) ((void)(param))
+#endif
+
+/* static_assert(condition, "message to print if it fails");
+ *
+ * Assert something at compile time. If the assertion fails, try to print
+ * the message, otherwise do nothing. static_assert is available if:
+ *
+ * - It's already defined as a macro (e.g. by <assert.h> in C11)
+ * - We're using MSVC which exposes static_assert unconditionally
+ * - We're using a C++ compiler that supports C++11
+ * - We're using GCC 4.6 and up in C mode (in which case it's available as
+ *   _Static_assert)
+ *
+ * In all other cases, fall back to an equivalent that makes an invalid
+ * declaration.
+ */
+#if !defined(static_assert) && !defined(_MSC_VER) && \
+		(!defined(__cplusplus) || __cplusplus < 201103L) || defined(__KLOCWORK__)
+	/* static_assert isn't already available */
+	#if !defined(__cplusplus) && (GCC_VERSION_AT_LEAST(4, 6) || \
+								  (defined(__clang__) && has_clang_extension(c_static_assert)))
+		#define static_assert _Static_assert
+	#else
+		#define static_assert(expr, message) \
+			extern int static_assert_failed[(expr) ? 1 : -1] __attribute__((unused))
+	#endif
+#endif
+
+/*
+ * unreachable("explanation") can be used to indicate to the compiler that
+ * some parts of the code can never be reached, like the default branch
+ * of a switch that covers all real-world possibilities, even though there
+ * are other ints that exist for instance.
+ *
+ * The message will be printed as an assert() when debugging.
+ *
+ * Note: there is no need to add a 'return' or any error handling after
+ * calling unreachable(), as this call will never return.
+ */
+#if defined(__linux__) && defined(__KERNEL__)
+/* Kernel has its own unreachable(), which is a simple infinite loop */
+#elif GCC_VERSION_AT_LEAST(4, 5) || has_clang_builtin(__builtin_unreachable)
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			__builtin_unreachable(); \
+		} while (false)
+#elif defined(_MSC_VER)
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			__assume(0); \
+		} while (false)
+#else
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			while (1); \
+		} while (false)
+#endif
+
+/*
+ * assume(x > 2 && x <= 7) works like an assert(), except it hints to the
+ * compiler what it can assume to optimise the code, like a limited range
+ * of parameter values.
+ */
+#if has_clang_builtin(__builtin_assume)
+	#define assume(expr) \
+		do { \
+			assert(expr); \
+			__builtin_assume(expr); \
+		} while (false)
+#elif defined(_MSC_VER)
+	#define assume(expr) \
+		do { \
+			assert(expr); \
+			__assume(expr); \
+		} while (false)
+#elif defined(__linux__) && defined(__KERNEL__)
+	#define assume(expr) ((void)(expr))
+#elif GCC_VERSION_AT_LEAST(4, 5) || has_clang_builtin(__builtin_unreachable)
+	#define assume(expr) \
+		do { \
+			if (unlikely(!(expr))) \
+				unreachable("Assumption isn't true: " # expr); \
+		} while (false)
+#else
+	#define assume(expr) assert(expr)
+#endif
+
+/*! Macro to calculate the n-byte aligned value from that supplied rounding up.
+ * n must be a power of two.
+ */
+#define PVR_ALIGN(_x, _n)	((((_x)+((_n)-1U))|((_n)-1U))^((_n)-1U))
+
+#if defined(_WIN32)
+
+#if defined(WINDOWS_WDF)
+
+	/*
+	 * For WINDOWS_WDF drivers we don't want these defines to overwrite calling conventions propagated through the build system.
+	 * This 'empty' choice helps to resolve all the calling conv issues.
+	 *
+	 */
+	#define IMG_CALLCONV
+	#define C_CALLCONV
+
+	#define IMG_INTERNAL
+	#define IMG_RESTRICT __restrict
+
+	/*
+	 * The proper way of dll linking under MS compilers is made of two things:
+	 * - decorate implementation with __declspec(dllexport)
+	 *   this decoration helps compiler with making the so called
+	 *   'export library'
+	 * - decorate forward-declaration (in a source dependent on a dll) with
+	 *   __declspec(dllimport), this decoration helps the compiler to make
+	 *   faster and smaller code in terms of calling dll-imported functions
+	 *
+	 * Usually these decorations are performed by having a single macro define
+	 * making that expands to a proper __declspec() depending on the
+	 * translation unit, dllexport inside the dll source and dllimport outside
+	 * the dll source. Having IMG_EXPORT and IMG_IMPORT resolving to the same
+	 * __declspec() makes no sense, but at least works.
+	 */
+	#define IMG_IMPORT __declspec(dllexport)
+	#define IMG_EXPORT __declspec(dllexport)
+
+#else
+
+	#define IMG_CALLCONV __stdcall
+	#define IMG_INTERNAL
+	#define	IMG_EXPORT	__declspec(dllexport)
+	#define IMG_RESTRICT __restrict
+	#define C_CALLCONV	__cdecl
+
+	/*
+	 * IMG_IMPORT is defined as IMG_EXPORT so that headers and implementations
+	 * match. Some compilers require the header to be declared IMPORT, while
+	 * the implementation is declared EXPORT.
+	 */
+	#define	IMG_IMPORT	IMG_EXPORT
+
+#endif
+
+#if defined(UNDER_WDDM)
+	#ifndef	_INC_STDLIB
+		#if defined(__mips)
+			/* do nothing */
+		#elif defined(UNDER_MSBUILD)
+			/* do nothing */
+		#else
+			_CRTIMP void __cdecl abort(void);
+		#endif
+	#endif
+#endif /* UNDER_WDDM */
+#else
+	#if (defined(__linux__) || defined(__QNXNTO__)) && defined(__KERNEL__)
+		#define IMG_INTERNAL
+		#define IMG_EXPORT
+		#define IMG_CALLCONV
+	#elif defined(__linux__) || defined(__METAG) || defined(__mips) || defined(__QNXNTO__) || defined(__riscv) || defined(__APPLE__) || defined(TEE_DDK)
+		#define IMG_CALLCONV
+		#define C_CALLCONV
+
+		#if defined(__METAG)
+			#define IMG_INTERNAL
+		#else
+			#define IMG_INTERNAL    __attribute__((visibility("hidden")))
+		#endif
+
+		#define IMG_EXPORT      __attribute__((visibility("default")))
+		#define IMG_RESTRICT    __restrict__
+	#elif defined(INTEGRITY_OS)
+		#define IMG_CALLCONV
+		#define IMG_INTERNAL
+		#define IMG_EXPORT
+		#define IMG_RESTRICT
+		#define C_CALLCONV
+		#define __cdecl
+
+		#ifndef USE_CODE
+			#define IMG_ABORT() printf("IMG_ABORT was called.\n")
+		#endif
+	#else
+		#error("define an OS")
+	#endif
+
+#endif
+
+/* Use default definition if not overridden */
+#ifndef IMG_ABORT
+	#if defined(EXIT_ON_ABORT)
+		#define IMG_ABORT()	exit(1)
+	#else
+		#define IMG_ABORT()	abort()
+	#endif
+#endif
+
+/* The best way to suppress unused parameter warnings using GCC is to use a
+ * variable attribute. Place the __maybe_unused between the type and name of an
+ * unused parameter in a function parameter list e.g. 'int __maybe_unused var'.
+ * This should only be used in GCC build environments, for example, in files
+ * that compile only on Linux.
+ * Other files should use PVR_UNREFERENCED_PARAMETER
+ */
+
+/* Kernel macros for compiler attributes */
+/* Note: param positions start at 1 */
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/compiler.h>
+
+	#if !defined(__fallthrough)
+		#if (GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)) && !defined(__CHECKER__)
+			#define __fallthrough __attribute__((__fallthrough__))
+		#else
+			#define __fallthrough
+		#endif
+	#endif
+#elif defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES)
+	#define __must_check       __attribute__((warn_unused_result))
+	#define __maybe_unused     __attribute__((unused))
+	#define __malloc           __attribute__((malloc))
+
+	/* Bionic's <sys/cdefs.h> might have defined these already */
+	/* See https://android.googlesource.com/platform/bionic.git/+/master/libc/include/sys/cdefs.h */
+	#if !defined(__packed)
+		#define __packed           __attribute__((packed))
+	#endif
+	#if !defined(__aligned)
+		#define __aligned(n)       __attribute__((aligned(n)))
+	#endif
+	#if !defined(__noreturn)
+		#define __noreturn         __attribute__((noreturn))
+	#endif
+
+	/* That one compiler that supports attributes but doesn't support
+	 * the printf attribute... */
+	#if defined(__GNUC__)
+		#if defined(__MINGW32__)
+		    #define __printf(fmt, va)  __attribute__((format(gnu_printf, (fmt), (va))))
+		#else
+			#define __printf(fmt, va)  __attribute__((format(printf, (fmt), (va))))
+		#endif
+	#else
+		#define __printf(fmt, va)
+	#endif /* defined(__GNUC__) */
+
+	#if defined(__cplusplus) && (__cplusplus >= 201703L)
+		#define __fallthrough [[fallthrough]]
+	#elif GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+
+	#define __user
+	#define __force
+	#define __iomem
+#else
+	/* Silently ignore those attributes */
+	#define __printf(fmt, va)
+	#define __packed
+	#define __aligned(n)
+	#define __must_check
+	#define __maybe_unused
+	#define __malloc
+
+	#if defined(_MSC_VER) || defined(CC_ARM)
+		#define __noreturn __declspec(noreturn)
+	#else
+		#define __noreturn
+	#endif
+
+	/* This may already been defined, e.g. by SAL (Source Annotation Language) */
+	#if !defined(__fallthrough)
+		#define __fallthrough
+	#endif
+
+	#define __user
+	#define __force
+	#define __iomem
+#endif
+
+
+/* Other attributes, following the same style */
+#if defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES)
+	#define __const_function      __attribute__((const))
+#else
+	#define __const_function
+#endif
+
+
+/* GCC builtins */
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/compiler.h>
+#elif defined(__GNUC__) || defined(INTEGRITY_OS)
+
+/* Klocwork does not support __builtin_expect, which makes the actual condition
+ * expressions hidden during analysis, affecting it negatively. */
+#if !defined(__KLOCWORK__) && !defined(INTEGRITY_OS) && !defined(DEBUG)
+	#define likely(x)   __builtin_expect(!!(x), 1)
+	#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+
+	/* Compiler memory barrier to prevent reordering */
+	#define barrier() __asm__ __volatile__("": : :"memory")
+#else
+	#define barrier() static_assert(0, "barrier() isn't supported by your compiler");
+#endif
+
+/* That one OS that defines one but not the other... */
+#ifndef likely
+	#define likely(x)   (x)
+#endif
+#ifndef unlikely
+	#define unlikely(x) (x)
+#endif
+
+#if !defined(BITS_PER_BYTE)
+#define BITS_PER_BYTE (8)
+#endif /* BITS_PER_BYTE */
+
+/* These two macros are also provided by the kernel */
+#ifndef BIT
+#define BIT(b) (1UL << (b))
+#endif
+
+#ifndef BIT_ULL
+#define BIT_ULL(b) (1ULL << (b))
+#endif
+
+#define BIT_SET(f, b)     BITMASK_SET((f),    BIT(b))
+#define BIT_UNSET(f, b)   BITMASK_UNSET((f),  BIT(b))
+#define BIT_TOGGLE(f, b)  BITMASK_TOGGLE((f), BIT(b))
+#define BIT_ISSET(f, b)   BITMASK_HAS((f),    BIT(b))
+
+#define BITMASK_SET(f, m)     do { ((f) |= (m)); } while (false)
+#define BITMASK_UNSET(f, m)   do { ((f) &= ~(m)); } while (false)
+#define BITMASK_TOGGLE(f, m)  do { ((f) ^= (m)); } while (false)
+#define BITMASK_HAS(f, m)     (((f) & (m)) == (m)) /* the bits from the mask are all set */
+#define BITMASK_ANY(f, m)     (((f) & (m)) != 0U)  /* any bit from the mask is set */
+
+#ifndef MAX
+#define MAX(a ,b)	(((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef MIN
+#define MIN(a, b)	(((a) < (b)) ? (a) : (b))
+#endif
+
+#ifndef CLAMP
+#define CLAMP(min, max, n)  ((n) < (min) ? (min) : ((n) > (max) ? (max) : (n)))
+#endif
+
+#define SWAP(X, Y) (X) ^= (Y); (Y) ^= (X); (X) ^= (Y);
+
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/kernel.h>
+	#include <linux/bug.h>
+#endif
+
+/* Get a structure's address from the address of a member */
+#define IMG_CONTAINER_OF(ptr, type, member) \
+	(type *) ((uintptr_t) (ptr) - offsetof(type, member))
+
+/* Get a new pointer with an offset (in bytes) from a base address, useful
+ * when traversing byte buffers and accessing data in buffers through struct
+ * pointers.
+ * Note, this macro is not equivalent to or replacing offsetof() */
+#define IMG_OFFSET_ADDR(addr, offset_in_bytes) \
+	(void*)&(((IMG_UINT8*)(void*)(addr))[offset_in_bytes])
+
+/* Get a new pointer with an offset (in bytes) from a base address, version
+ * for volatile memory.
+ */
+#define IMG_OFFSET_ADDR_VOLATILE(addr, offset_in_bytes) \
+	(volatile void*)&(((volatile IMG_UINT8*)(volatile void*)(addr))[offset_in_bytes])
+
+/* Get a new pointer with an offset (in dwords) from a base address, useful
+ * when traversing byte buffers and accessing data in buffers through struct
+ * pointers.
+ * Note, this macro is not equivalent to or replacing offsetof() */
+#define IMG_OFFSET_ADDR_DW(addr, offset_in_dwords) \
+	(void*)(((IMG_UINT32*)(void*)(addr)) + (offset_in_dwords))
+
+/* The number of elements in a fixed-sized array */
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(ARR) (sizeof(ARR) / sizeof((ARR)[0]))
+#endif
+
+/* To guarantee that __func__ can be used, define it as a macro here if it
+   isn't already provided by the compiler. */
+#if defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus < 201103L)
+#define __func__ __FUNCTION__
+#endif
+
+#if defined(__cplusplus)
+/* C++ Specific:
+ * Disallow use of copy and assignment operator within a class.
+ * Should be placed under private. */
+#define IMG_DISALLOW_COPY_AND_ASSIGN(C) \
+	C(const C&); \
+	void operator=(const C&)
+#endif
+
+#if defined(SUPPORT_PVR_VALGRIND) && !defined(__METAG) && !defined(__mips) && !defined(__riscv)
+	#include "/usr/include/valgrind/memcheck.h"
+
+	#define VG_MARK_INITIALIZED(pvData,ui32Size) VALGRIND_MAKE_MEM_DEFINED(pvData,ui32Size)
+	#define VG_MARK_NOACCESS(pvData,ui32Size) VALGRIND_MAKE_MEM_NOACCESS(pvData,ui32Size)
+	#define VG_MARK_ACCESS(pvData,ui32Size) VALGRIND_MAKE_MEM_UNDEFINED(pvData,ui32Size)
+	#define VG_ASSERT_DEFINED(pvData,ui32Size) VALGRIND_CHECK_MEM_IS_DEFINED(pvData,ui32Size)
+#else
+	#if defined(_MSC_VER)
+	#	define PVR_MSC_SUPPRESS_4127 __pragma(warning(suppress:4127))
+	#else
+	#	define PVR_MSC_SUPPRESS_4127
+	#endif
+
+	#define VG_MARK_INITIALIZED(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_MARK_NOACCESS(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_MARK_ACCESS(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_ASSERT_DEFINED(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+#endif
+
+#define IMG_STRINGIFY_IMPL(x) # x
+#define IMG_STRINGIFY(x) IMG_STRINGIFY_IMPL(x)
+
+#define IMG_CONCATENATE_IMPL(x,y) x ## y
+#define IMG_CONCATENATE(x,y) IMG_CONCATENATE_IMPL(x,y)
+
+#if defined(DEBUG) && !defined(INTEGRITY_OS)
+#define IMG_PAGESLOG2BYTES(_tcast, _npages, _log2) \
+	({ \
+		PVR_ASSERT( ((IMG_UINT64)(1ULL) << (sizeof(_tcast)*8UL)) >= ((IMG_UINT64)(_npages) << (_log2)) ); \
+		(_tcast)(_npages) << (_log2); \
+	})
+#else
+#define IMG_PAGESLOG2BYTES(_tcast, _npages, _log2) ((_npages) << (_log2))
+#endif
+
+#define IMG_PAGE2BYTES32(logsize) IMG_PAGESLOG2BYTES(IMG_UINT32,IMG_UINT32_C(1),logsize)
+#define IMG_PAGE2BYTES64(logsize) ((IMG_UINT64)IMG_UINT64_C(1) << (logsize))
+
+#define IMG_PAGES2BYTES32(pages,logsize) IMG_PAGESLOG2BYTES(IMG_UINT32,pages,logsize)
+#define IMG_PAGES2BYTES64(pages,logsize) ((IMG_UINT64)(pages) << (logsize))
+
+#define IMG_PAGE_SHIFT_4KB   12U
+#define IMG_PAGE_SHIFT_16KB  14U
+#define IMG_PAGE_SHIFT_64KB  16U
+#define IMG_PAGE_SHIFT_256KB 18U
+#define IMG_PAGE_SHIFT_1MB   20U
+#define IMG_PAGE_SHIFT_2MB   21U
+
+#if defined(INTEGRITY_OS)
+	/* Definitions not present in INTEGRITY. */
+	#define PATH_MAX	200
+#endif
+
+#if defined(__clang__) || defined(__GNUC__)
+	/* __SIZEOF_POINTER__ is defined already by these compilers */
+#elif defined(INTEGRITY_OS)
+	#if defined(__Ptr_Is_64)
+		#define __SIZEOF_POINTER__ 8
+	#else
+		#define __SIZEOF_POINTER__ 4
+	#endif
+#elif defined(_WIN32)
+	#define __SIZEOF_POINTER__ sizeof(char *)
+#else
+	#warning Unknown OS - using default method to determine whether CPU arch is 64-bit.
+	#define __SIZEOF_POINTER__ sizeof(char *)
+#endif
+
+/* RDI8567: gcc/clang/llvm load/store optimisations may cause issues with
+ * uncached device memory allocations. Some pointers are made 'volatile'
+ * to prevent those optimisations being applied to writes through those
+ * pointers.
+ */
+#if (GCC_VERSION_AT_LEAST(7, 0) || defined(__clang__)) && (defined(__arm64__) || defined(__aarch64__))
+#define NOLDSTOPT volatile
+/* after applying 'volatile' to a pointer, we may need to cast it to 'void *'
+ * to keep it compatible with its existing uses.
+ */
+#define NOLDSTOPT_VOID (void *)
+
+#define NOLDSTOPT_REQUIRED 1
+#else
+#define NOLDSTOPT
+#define NOLDSTOPT_VOID
+#endif
+
+#define PVR_PRE_DPF (void) printf
+
+/* C STD >= C99 */
+#if !defined(INTEGRITY_OS) && !defined(__CHECKER__) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+#define IMG_FLEX_ARRAY_MEMBER
+#define IMG_FLEX_ARRAY_SIZE(size, count) ((size) * (count))
+#else
+/* In C STD prior to C99 flexible array members are an extension feature and syntax requires alternative approach */
+#define IMG_FLEX_ARRAY_MEMBER (1)
+#define IMG_FLEX_ARRAY_SIZE(size, count) ((size) * ((count) - 1))
+#endif
+
+#endif /* IMG_DEFS_H */
+/*****************************************************************************
+ End of file (img_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_drm_fourcc_internal.h b/drivers/gpu/drm/img/img-volcanic/include/img_drm_fourcc_internal.h
new file mode 100644
index 000000000000..ffde181fcb48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_drm_fourcc_internal.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          Wrapper around drm_fourcc.h
+@Description    FourCCs and the DRM framebuffer modifiers should be added here
+                unless they are used by kernel code or a known user outside of
+                the DDK. If FourCCs or DRM framebuffer modifiers are required
+                outside of the DDK, they shall be moved to the corresponding
+                public header.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DRM_FOURCC_INTERNAL_H
+#define IMG_DRM_FOURCC_INTERNAL_H
+
+#include <powervr/img_drm_fourcc.h>
+
+/*
+ * Modifier names are structured using the following convention,
+ * with underscores (_) between items:
+ * - prefix: DRM_FORMAT_MOD
+ * - identifier for our driver: PVR
+ * - category: FBCDC
+ *   - compression tile dimension: 8x8, 16x4, 32x2
+ *   - FBDC version: V0, V1, V2, V3, V7, V8, V10, V12, V13, V14
+ */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0      fourcc_mod_code(PVR, 1)
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0_FIX  fourcc_mod_code(PVR, 2) /* Fix for HW_BRN_37464 */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V2      fourcc_mod_code(PVR, 4)
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V3      fourcc_mod_code(PVR, 5)
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V8      fourcc_mod_code(PVR, 18)
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V13 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0     fourcc_mod_code(PVR, 7)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0_FIX fourcc_mod_code(PVR, 8) /* Fix for HW_BRN_37464 */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V2     fourcc_mod_code(PVR, 10)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V3     fourcc_mod_code(PVR, 11)
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V8     fourcc_mod_code(PVR, 19)
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V13 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V1     fourcc_mod_code(PVR, 13)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V3     fourcc_mod_code(PVR, 14)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V8     fourcc_mod_code(PVR, 20)
+/* DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V12    fourcc_mod_code(PVR, 17)
+
+#endif /* IMG_DRM_FOURCC_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_elf.h b/drivers/gpu/drm/img/img-volcanic/include/img_elf.h
new file mode 100644
index 000000000000..8837d9592599
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_elf.h
@@ -0,0 +1,111 @@
+/*************************************************************************/ /*!
+@File           img_elf.h
+@Title          IMG ELF file definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    Definitions for ELF file structures used in the DDK.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(IMG_ELF_H)
+#define IMG_ELF_H
+
+#include "img_types.h"
+
+/* ELF format defines */
+#define ELF_PT_LOAD     (0x1U)   /* Program header identifier as Load */
+#define ELF_SHT_SYMTAB  (0x2U)   /* Section identifier as Symbol Table */
+#define ELF_SHT_STRTAB  (0x3U)   /* Section identifier as String Table */
+#define MAX_STRTAB_NUM  (0x8U)   /* Maximum number of string table in the ELF file */
+
+/* Redefined structs of ELF format */
+typedef struct
+{
+	IMG_UINT8    ui32Eident[16];
+	IMG_UINT16   ui32Etype;
+	IMG_UINT16   ui32Emachine;
+	IMG_UINT32   ui32Eversion;
+	IMG_UINT32   ui32Eentry;
+	IMG_UINT32   ui32Ephoff;
+	IMG_UINT32   ui32Eshoff;
+	IMG_UINT32   ui32Eflags;
+	IMG_UINT16   ui32Eehsize;
+	IMG_UINT16   ui32Ephentsize;
+	IMG_UINT16   ui32Ephnum;
+	IMG_UINT16   ui32Eshentsize;
+	IMG_UINT16   ui32Eshnum;
+	IMG_UINT16   ui32Eshtrndx;
+} IMG_ELF_HDR;
+
+typedef struct
+{
+	IMG_UINT32   ui32Stname;
+	IMG_UINT32   ui32Stvalue;
+	IMG_UINT32   ui32Stsize;
+	IMG_UINT8    ui32Stinfo;
+	IMG_UINT8    ui32Stother;
+	IMG_UINT16   ui32Stshndx;
+} IMG_ELF_SYM;
+
+typedef struct
+{
+	IMG_UINT32   ui32Shname;
+	IMG_UINT32   ui32Shtype;
+	IMG_UINT32   ui32Shflags;
+	IMG_UINT32   ui32Shaddr;
+	IMG_UINT32   ui32Shoffset;
+	IMG_UINT32   ui32Shsize;
+	IMG_UINT32   ui32Shlink;
+	IMG_UINT32   ui32Shinfo;
+	IMG_UINT32   ui32Shaddralign;
+	IMG_UINT32   ui32Shentsize;
+} IMG_ELF_SHDR;
+
+typedef struct
+{
+	IMG_UINT32   ui32Ptype;
+	IMG_UINT32   ui32Poffset;
+	IMG_UINT32   ui32Pvaddr;
+	IMG_UINT32   ui32Ppaddr;
+	IMG_UINT32   ui32Pfilesz;
+	IMG_UINT32   ui32Pmemsz;
+	IMG_UINT32   ui32Pflags;
+	IMG_UINT32   ui32Palign;
+} IMG_ELF_PROGRAM_HDR;
+
+#endif /* IMG_ELF_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_types.h b/drivers/gpu/drm/img/img-volcanic/include/img_types.h
new file mode 100644
index 000000000000..c18c5da47da5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_types.h
@@ -0,0 +1,333 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines type aliases for use by IMG APIs.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_TYPES_H
+#define IMG_TYPES_H
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* To use C99 types and definitions, there are two special cases we need to
+ * cater for:
+ *
+ * - Visual Studio: in VS2010 or later, some standard headers are available,
+ *   and MSVC has its own built-in sized types. We can define the C99 types
+ *   in terms of these.
+ *
+ * - Linux kernel code: C99 sized types are defined in <linux/types.h>, but
+ *   some other features (like macros for constants or printf format
+ *   strings) are missing, so we need to fill in the gaps ourselves.
+ *
+ * For other cases (userspace code under Linux, Android or Neutrino, or
+ * firmware code), we can include the standard headers.
+ */
+#if defined(_MSC_VER)
+	#include <stdbool.h>		/* bool */
+	#include "msvc_types.h"
+#elif defined(__linux__) && defined(__KERNEL__)
+	#include <linux/version.h>
+	#include <linux/types.h>
+	#include "kernel_types.h"
+#elif defined(__linux__) || defined(__METAG) || defined(__MINGW32__) || \
+	defined(__QNXNTO__) || defined(INTEGRITY_OS) || defined(__riscv) || \
+	defined(__APPLE__) || defined(TEE_DDK)
+	#include <stddef.h>			/* NULL */
+	#include <stdint.h>
+#if defined(__riscv)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wundef"
+#endif
+	#include <inttypes.h>		/* intX_t/uintX_t, format specifiers */
+	#include <limits.h>			/* INT_MIN, etc */
+	#include <sys/types.h>		/* ssize_t */
+#if defined(__riscv)
+#pragma GCC diagnostic pop
+#endif
+	#include <stdbool.h>		/* bool */
+#elif defined(__mips)
+	#include <stddef.h>			/* NULL */
+	#include <inttypes.h>		/* intX_t/uintX_t, format specifiers */
+	#include <stdbool.h>		/* bool */
+#else
+	#error C99 support not set up for this build
+#endif
+
+/*
+ * Due to a Klocwork bug, 'true'/'false' constants are not recognized to be of
+ * boolean type. This results in large number of false-positives being reported
+ * (MISRA.ETYPE.ASSIGN.2012: "An expression value of essential type 'signed char'
+ * is assigned to an object of essential type 'bool'"). Work around this by
+ * redefining those constants with cast to bool added.
+ */
+#if defined(__KLOCWORK__) && !defined(__cplusplus)
+#undef true
+#undef false
+#define true ((bool) 1)
+#define false ((bool) 0)
+#endif
+
+typedef unsigned int	IMG_UINT;
+typedef int				IMG_INT;
+
+typedef uint8_t			IMG_UINT8,	*IMG_PUINT8;
+typedef uint8_t			IMG_BYTE,	*IMG_PBYTE;
+typedef int8_t			IMG_INT8;
+typedef char			IMG_CHAR,	*IMG_PCHAR;
+
+typedef uint16_t		IMG_UINT16,	*IMG_PUINT16;
+typedef int16_t			IMG_INT16;
+typedef uint32_t		IMG_UINT32,	*IMG_PUINT32;
+typedef int32_t			IMG_INT32,	*IMG_PINT32;
+#if defined(INTEGRITY_OS)
+#if __INT_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## U))
+#elif __LONG_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## UL))
+#elif defined(__LLONG_BIT) && __LLONG_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## ULL))
+#endif
+#else /* defined(INTEGRITY_OS) */
+#define IMG_UINT32_C(c) ((IMG_UINT32)UINT32_C(c))
+#endif /* defined(INTEGRITY_OS) */
+
+typedef uint64_t		IMG_UINT64,	*IMG_PUINT64;
+typedef int64_t			IMG_INT64;
+#define IMG_INT64_C(c)	INT64_C(c)
+#if defined(INTEGRITY_OS)
+#if __INT_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## U)
+#elif defined(__LONG_BIT) && __LONG_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## UL)
+#elif defined(__LLONG_BIT) && __LLONG_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## ULL)
+#endif
+#else /* defined(INTEGRITY_OS) */
+#define IMG_UINT64_C(c)	UINT64_C(c)
+#endif /* defined(INTEGRITY_OS) */
+#define IMG_UINT16_C(c)	UINT16_C(c)
+#define IMG_UINT64_FMTSPEC PRIu64
+#define IMG_UINT64_FMTSPECX PRIX64
+#define IMG_UINT64_FMTSPECx PRIx64
+#define IMG_UINT64_FMTSPECo PRIo64
+#define IMG_INT64_FMTSPECd PRId64
+
+#define IMG_UINT8_MAX	UINT8_MAX
+#define IMG_UINT16_MAX	UINT16_MAX
+#define IMG_UINT32_MAX	UINT32_MAX
+#define IMG_UINT64_MAX	UINT64_MAX
+
+#define IMG_INT_MIN		INT_MIN
+#define IMG_INT_MAX		INT_MAX
+#define IMG_INT8_MAX	INT8_MAX
+#define IMG_INT16_MAX	INT16_MAX
+#define IMG_INT32_MAX	INT32_MAX
+#define IMG_INT64_MAX	INT64_MAX
+
+/* Linux kernel mode does not use floating point */
+typedef float			IMG_FLOAT,	*IMG_PFLOAT;
+typedef double			IMG_DOUBLE;
+
+typedef union
+{
+	IMG_UINT32 ui32;
+	IMG_FLOAT f;
+} IMG_UINT32_FLOAT;
+
+typedef int				IMG_SECURE_TYPE;
+
+typedef bool      IMG_BOOL;
+typedef bool*     IMG_PBOOL;
+#define IMG_FALSE ((bool) 0)
+#define IMG_TRUE  ((bool) 1)
+
+typedef IMG_CHAR const* IMG_PCCHAR;
+
+/* Format specifiers for 'size_t' type */
+#if defined(_MSC_VER)
+#define IMG_SIZE_FMTSPEC  "%Iu"
+#define IMG_SIZE_FMTSPECX "%Ix"
+#else
+#define IMG_SIZE_FMTSPEC  "%zu"
+#define IMG_SIZE_FMTSPECX "%zx"
+#endif
+
+#if defined(__linux__) && defined(__KERNEL__)
+/* prints the function name when used with printk */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#define IMG_PFN_FMTSPEC "%ps"
+#else
+#define IMG_PFN_FMTSPEC "%pf"
+#endif
+#else
+#define IMG_PFN_FMTSPEC "%p"
+#endif
+
+typedef void           *IMG_HANDLE;
+
+/* Process IDs */
+typedef IMG_UINT32      IMG_PID;
+
+/* OS connection type */
+typedef int             IMG_OS_CONNECTION;
+
+
+/*
+ * Address types.
+ * All types used to refer to a block of memory are wrapped in structures
+ * to enforce some degree of type safety, i.e. a IMG_DEV_VIRTADDR cannot
+ * be assigned to a variable of type IMG_DEV_PHYADDR because they are not the
+ * same thing.
+ *
+ * There is an assumption that the system contains at most one non-cpu mmu,
+ * and a memory block is only mapped by the MMU once.
+ *
+ * Different devices could have offset views of the physical address space.
+ *
+ */
+
+
+/*
+ *
+ * +------------+    +------------+      +------------+        +------------+
+ * |    CPU     |    |    DEV     |      |    DEV     |        |    DEV     |
+ * +------------+    +------------+      +------------+        +------------+
+ *       |                 |                   |                     |
+ *       | void *          |IMG_DEV_VIRTADDR   |IMG_DEV_VIRTADDR     |
+ *       |                 \-------------------/                     |
+ *       |                          |                                |
+ * +------------+             +------------+                         |
+ * |    MMU     |             |    MMU     |                         |
+ * +------------+             +------------+                         |
+ *       |                          |                                |
+ *       |                          |                                |
+ *       |                          |                                |
+ *   +--------+                +---------+                      +--------+
+ *   | Offset |                | (Offset)|                      | Offset |
+ *   +--------+                +---------+                      +--------+
+ *       |                          |                IMG_DEV_PHYADDR |
+ *       |                          |                                |
+ *       |                          | IMG_DEV_PHYADDR                |
+ * +---------------------------------------------------------------------+
+ * |                         System Address bus                          |
+ * +---------------------------------------------------------------------+
+ *
+ */
+
+#define IMG_DEV_VIRTADDR_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_SIZE_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_ALIGN_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_OFFSET_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+
+/* cpu physical address */
+typedef struct
+{
+#if defined(UNDER_WDDM) || defined(WINDOWS_WDF)
+	uintptr_t uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(uintptr_t)(var)
+#define CPUPHYADDR_FMTARG(var)				(IMG_UINT64)(var)
+#define CPUPHYADDR_UINT_FMTSPEC "0x%016" IMG_UINT64_FMTSPECx
+#elif defined(__linux__) && defined(__KERNEL__)
+	phys_addr_t uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(phys_addr_t)(var)
+#define CPUPHYADDR_FMTARG(var)				(&var)
+#define CPUPHYADDR_UINT_FMTSPEC "%pa"
+#else
+	IMG_UINT64 uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(IMG_UINT64)(var)
+#define CPUPHYADDR_FMTARG(var)				(var)
+#define CPUPHYADDR_UINT_FMTSPEC "0x%016" IMG_UINT64_FMTSPECx
+#endif
+} IMG_CPU_PHYADDR;
+
+/* device physical address */
+typedef struct
+{
+	IMG_UINT64 uiAddr;
+} IMG_DEV_PHYADDR;
+
+/* dma address */
+typedef struct
+{
+	IMG_UINT64 uiAddr;
+} IMG_DMA_ADDR;
+
+/*
+	rectangle structure
+*/
+typedef struct
+{
+	IMG_INT32	x0;
+	IMG_INT32	y0;
+	IMG_INT32	x1;
+	IMG_INT32	y1;
+} IMG_RECT;
+
+typedef struct
+{
+	IMG_INT16	x0;
+	IMG_INT16	y0;
+	IMG_INT16	x1;
+	IMG_INT16	y1;
+} IMG_RECT_16;
+
+/*
+ * box structure
+ */
+typedef struct
+{
+	IMG_INT32	x0;
+	IMG_INT32	y0;
+	IMG_INT32	z0;
+	IMG_INT32	x1;
+	IMG_INT32	y1;
+	IMG_INT32	z1;
+} IMG_BOX;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* IMG_TYPES_H */
+/******************************************************************************
+ End of file (img_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/kernel_types.h b/drivers/gpu/drm/img/img-volcanic/include/kernel_types.h
new file mode 100644
index 000000000000..c3305102fc20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/kernel_types.h
@@ -0,0 +1,137 @@
+/*************************************************************************/ /*!
+@Title          C99-compatible types and definitions for Linux kernel code
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/kernel.h>
+
+/* Limits of specified-width integer types */
+
+/* S8_MIN, etc were added in kernel version 3.14. The other versions are for
+ * earlier kernels. They can be removed once older kernels don't need to be
+ * supported.
+ */
+#ifdef S8_MIN
+	#define INT8_MIN	S8_MIN
+#else
+	#define INT8_MIN	(-128)
+#endif
+
+#ifdef S8_MAX
+	#define INT8_MAX	S8_MAX
+#else
+	#define INT8_MAX	127
+#endif
+
+#ifdef U8_MAX
+	#define UINT8_MAX	U8_MAX
+#else
+	#define UINT8_MAX	0xFF
+#endif
+
+#ifdef S16_MIN
+	#define INT16_MIN	S16_MIN
+#else
+	#define INT16_MIN	(-32768)
+#endif
+
+#ifdef S16_MAX
+	#define INT16_MAX	S16_MAX
+#else
+	#define INT16_MAX	32767
+#endif
+
+#ifdef U16_MAX
+	#define UINT16_MAX	U16_MAX
+#else
+	#define UINT16_MAX	0xFFFF
+#endif
+
+#ifdef S32_MIN
+	#define INT32_MIN	S32_MIN
+#else
+	#define INT32_MIN	(-2147483647 - 1)
+#endif
+
+#ifdef S32_MAX
+	#define INT32_MAX	S32_MAX
+#else
+	#define INT32_MAX	2147483647
+#endif
+
+#ifdef U32_MAX
+	#define UINT32_MAX	U32_MAX
+#else
+	#define UINT32_MAX	0xFFFFFFFF
+#endif
+
+#ifdef S64_MIN
+	#define INT64_MIN	S64_MIN
+#else
+	#define INT64_MIN	(-9223372036854775807LL)
+#endif
+
+#ifdef S64_MAX
+	#define INT64_MAX	S64_MAX
+#else
+	#define INT64_MAX	9223372036854775807LL
+#endif
+
+#ifdef U64_MAX
+	#define UINT64_MAX	U64_MAX
+#else
+	#define UINT64_MAX	0xFFFFFFFFFFFFFFFFULL
+#endif
+
+/* Macros for integer constants */
+#define INT8_C			S8_C
+#define UINT8_C			U8_C
+#define INT16_C			S16_C
+#define UINT16_C		U16_C
+#define INT32_C			S32_C
+#define UINT32_C		U32_C
+#define INT64_C			S64_C
+#define UINT64_C		U64_C
+
+/* Format conversion of integer types <inttypes.h> */
+
+#define PRIX64		"llX"
+#define PRIx64		"llx"
+#define PRIu64		"llu"
+#define PRId64		"lld"
diff --git a/drivers/gpu/drm/img/img-volcanic/include/linux_sw_sync.h b/drivers/gpu/drm/img/img-volcanic/include/linux_sw_sync.h
new file mode 100644
index 000000000000..c12c650294a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/linux_sw_sync.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef _UAPI_LINUX_PVR_SW_SYNC_H
+#define _UAPI_LINUX_PVR_SW_SYNC_H
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+
+#include <linux/types.h>
+
+#include "pvrsrv_sync_km.h"
+#include "pvr_drm.h"
+
+#endif /* defined(SUPPORT_NATIVE_FENCE_SYNC) */
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/lock_types.h b/drivers/gpu/drm/img/img-volcanic/include/lock_types.h
new file mode 100644
index 000000000000..bf7167b554f6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/lock_types.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           lock_types.h
+@Title          Locking types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Locking specific enums, defines and structures
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOCK_TYPES_H
+#define LOCK_TYPES_H
+
+/* In Linux kernel mode we are using the kernel mutex implementation directly
+ * with macros. This allows us to use the kernel lockdep feature for lock
+ * debugging. */
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include <linux/types.h>
+#include <linux/mutex.h>
+/* The mutex is defined as a pointer to be compatible with the other code. This
+ * isn't ideal and usually you wouldn't do that in kernel code. */
+typedef struct mutex *POS_LOCK;
+typedef struct rw_semaphore *POSWR_LOCK;
+typedef spinlock_t *POS_SPINLOCK;
+typedef atomic_t ATOMIC_T;
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+#include "img_types.h" /* needed for IMG_INT */
+typedef struct OS_LOCK_TAG *POS_LOCK;
+
+#if defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+typedef struct OSWR_LOCK_TAG *POSWR_LOCK;
+#else /* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+typedef struct OSWR_LOCK_TAG {
+	IMG_UINT32 ui32Unused;
+} *POSWR_LOCK;
+#endif /* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+
+#if defined(__linux__) || defined(__APPLE__)
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(__QNXNTO__)
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(_WIN32)
+	/*
+	 * Placeholder definition. WDDM doesn't use Services, but some headers
+	 * still have to be shared. This is one such case.
+	 */
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(INTEGRITY_OS)
+	/* Only lower 32bits are used in OS ATOMIC APIs to have consistent behaviour across all OS */
+	typedef struct OS_ATOMIC_TAG {IMG_INT64 counter;} ATOMIC_T;
+#else
+	#error "Please type-define an atomic lock for this environment"
+#endif
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#endif /* LOCK_TYPES_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/log2.h b/drivers/gpu/drm/img/img-volcanic/include/log2.h
new file mode 100644
index 000000000000..795660400850
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/log2.h
@@ -0,0 +1,417 @@
+/*************************************************************************/ /*!
+@Title          Integer log2 and related functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOG2_H
+#define LOG2_H
+
+#include "img_defs.h"
+
+/*************************************************************************/ /*!
+@Description    Determine if a number is a power of two.
+@Input          n
+@Return         True if n is a power of 2, false otherwise. True if n == 0.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL __const_function IsPower2(uint32_t n)
+{
+	/* C++ needs this cast. */
+	return (IMG_BOOL)((n & (n - 1U)) == 0U);
+}
+
+/*************************************************************************/ /*!
+@Description    Determine if a number is a power of two.
+@Input          n
+@Return         True if n is a power of 2, false otherwise. True if n == 0.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL __const_function IsPower2_64(uint64_t n)
+{
+	/* C++ needs this cast. */
+	return (IMG_BOOL)((n & (n - 1U)) == 0U);
+}
+
+/* Code using GNU GCC intrinsics */
+#if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER))
+
+/* CHAR_BIT is typically found in <limits.h>. For all the platforms where
+ * CHAR_BIT is not available, defined it here with the assumption that there
+ * are 8 bits in a byte */
+#ifndef CHAR_BIT
+#define CHAR_BIT 8U
+#endif
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2(uint32_t n)
+{
+	if (unlikely(n == 0U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+		return uNumBits - (uint32_t)__builtin_clz(n) - 1U;
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2_64(uint64_t n)
+{
+	if (unlikely(n == 0U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+		return uNumBits - (uint32_t)__builtin_clzll(n) - 1U;
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2(uint32_t n)
+{
+	if (unlikely(n == 0U || n == 1U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+
+		n--; /* Handle powers of 2 */
+		return uNumBits - (uint32_t)__builtin_clz(n);
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2_64(uint64_t n)
+{
+	if (unlikely(n == 0U || n == 1U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+
+		n--; /* Handle powers of 2 */
+		return uNumBits - (uint32_t)__builtin_clzll(n);
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2(uint32_t n)
+{
+	return (uint32_t)CHAR_BIT * (uint32_t)sizeof(n) - (uint32_t)__builtin_clz(n) - 1U;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2_64(uint64_t n)
+{
+	return (uint32_t)CHAR_BIT * (uint32_t)sizeof(n) - (uint32_t)__builtin_clzll(n) - 1U;
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function RoundUpToNextPowerOfTwo(uint32_t n)
+{
+	/* Cases with n greater than 2^31 needs separate handling
+	 * as result of (1<<32) is undefined. */
+	if (unlikely( n == 0U || n > (uint32_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - 1U)))
+	{
+		return 0;
+	}
+
+	/* Return n if it is already a power of 2 */
+	if ((IMG_BOOL)((n & (n - 1U)) == 0U))
+	{
+		return n;
+	}
+
+	return (uint32_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - (uint32_t)__builtin_clz(n));
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint64_t __const_function RoundUpToNextPowerOfTwo_64(uint64_t n)
+{
+	/* Cases with n greater than 2^63 needs separate handling
+	 * as result of (1<<64) is undefined. */
+	if (unlikely( n == 0U || n > (uint64_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - 1U)))
+	{
+		return 0;
+	}
+
+	/* Return n if it is already a power of 2 */
+	if ((IMG_BOOL)((n & (n - 1U)) == 0U))
+	{
+		return n;
+	}
+
+	return (uint64_t)1 << ((uint64_t)CHAR_BIT * sizeof(n) - (uint64_t)__builtin_clzll(n));
+}
+
+#else /* #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER)) */
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function RoundUpToNextPowerOfTwo(uint32_t n)
+{
+	n--;
+	n |= n >> 1;  /* handle  2 bit numbers */
+	n |= n >> 2;  /* handle  4 bit numbers */
+	n |= n >> 4;  /* handle  8 bit numbers */
+	n |= n >> 8;  /* handle 16 bit numbers */
+	n |= n >> 16; /* handle 32 bit numbers */
+	n++;
+
+	return n;
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint64_t __const_function RoundUpToNextPowerOfTwo_64(uint64_t n)
+{
+	n--;
+	n |= n >> 1;  /* handle  2 bit numbers */
+	n |= n >> 2;  /* handle  4 bit numbers */
+	n |= n >> 4;  /* handle  8 bit numbers */
+	n |= n >> 8;  /* handle 16 bit numbers */
+	n |= n >> 16; /* handle 32 bit numbers */
+	n |= n >> 32; /* handle 64 bit numbers */
+	n++;
+
+	return n;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2(uint32_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	for (n >>= 1; n != 0U; n >>= 1)
+	{
+		ui32log2++;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2_64(uint64_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	for (n >>= 1; n != 0U; n >>= 1)
+	{
+		ui32log2++;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2(uint32_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	if (n == 0U)
+	{
+		return 0;
+	}
+
+	n--; /* Handle powers of 2 */
+
+	while (n != 0U)
+	{
+		ui32log2++;
+		n >>= 1;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2_64(uint64_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	if (n == 0U)
+	{
+		return 0;
+	}
+
+	n--; /* Handle powers of 2 */
+
+	while (n != 0U)
+	{
+		ui32log2++;
+		n >>= 1;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2(uint32_t n)
+{
+	static const uint32_t b[] =
+		{0xAAAAAAAAU, 0xCCCCCCCCU, 0xF0F0F0F0U, 0xFF00FF00U, 0xFFFF0000U};
+	uint32_t r = (((n & b[0]) != 0U) ? 1U : 0U);
+
+	r |= (uint32_t) (((n & b[4]) != 0U) ? (1U << 4) : 0U);
+	r |= (uint32_t) (((n & b[3]) != 0U) ? (1U << 3) : 0U);
+	r |= (uint32_t) (((n & b[2]) != 0U) ? (1U << 2) : 0U);
+	r |= (uint32_t) (((n & b[1]) != 0U) ? (1U << 1) : 0U);
+
+	return r;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2_64(uint64_t n)
+{
+	static const uint64_t b[] =
+		{0xAAAAAAAAAAAAAAAAUL, 0xCCCCCCCCCCCCCCCCUL,
+		 0xF0F0F0F0F0F0F0F0UL, 0xFF00FF00FF00FF00UL,
+		 0xFFFF0000FFFF0000UL, 0xFFFFFFFF00000000UL};
+	uint32_t r = (((n & b[0]) != 0U) ? 1U : 0U);
+
+	r |= (uint32_t) (((n & b[5]) != 0U) ? (1U << 5) : 0U);
+	r |= (uint32_t) (((n & b[4]) != 0U) ? (1U << 4) : 0U);
+	r |= (uint32_t) (((n & b[3]) != 0U) ? (1U << 3) : 0U);
+	r |= (uint32_t) (((n & b[2]) != 0U) ? (1U << 2) : 0U);
+	r |= (uint32_t) (((n & b[1]) != 0U) ? (1U << 1) : 0U);
+
+	return r;
+}
+
+#endif /* #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER)) */
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(size)) , where size is the max of 3 sizes
+				This is almost always the ONLY EVER valid use of FloorLog2.
+				Usually CeilLog2() should be used instead.
+				For a 5x5x1 texture, the 3 miplevels are:
+					0:  5x5x1
+					1:	2x2x1
+					2:	1x1x1
+
+				For an 8x8x1 texture, the 4 miplevels are:
+					0:  8x8x1
+					1:	4x4x1
+					2:  2x2x1
+					3:  1x1x1
+
+
+@Input          sizeX, sizeY, sizeZ
+@Return         Count of mipmap levels for given dimensions
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function NumMipLevels(uint32_t sizeX, uint32_t sizeY, uint32_t sizeZ)
+{
+
+	uint32_t maxSize = MAX(MAX(sizeX, sizeY), sizeZ);
+	return FloorLog2(maxSize) + 1U;
+}
+
+#endif /* LOG2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/multicore_defs.h b/drivers/gpu/drm/img/img-volcanic/include/multicore_defs.h
new file mode 100644
index 000000000000..b410f57075f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/multicore_defs.h
@@ -0,0 +1,53 @@
+/**************************************************************************/ /*!
+@File
+@Title          RGX Multicore Information flags
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_MULTICORE_DEFS_H
+#define RGX_MULTICORE_DEFS_H
+
+/* Capability bits returned to client in RGXGetMultiCoreInfo */
+#define RGX_MULTICORE_CAPABILITY_FRAGMENT_EN    (0x00000040U)
+#define RGX_MULTICORE_CAPABILITY_GEOMETRY_EN    (0x00000020U)
+#define RGX_MULTICORE_CAPABILITY_COMPUTE_EN     (0x00000010U)
+#define RGX_MULTICORE_CAPABILITY_PRIMARY_EN     (0x00000008U)
+#define RGX_MULTICORE_ID_CLRMSK                 (0xFFFFFFF8U)
+
+#endif /* RGX_MULTICORE_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/osfunc_common.h b/drivers/gpu/drm/img/img-volcanic/include/osfunc_common.h
new file mode 100644
index 000000000000..d93cb6d61aba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/osfunc_common.h
@@ -0,0 +1,274 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS functions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSFUNC_COMMON_H
+/*! @cond Doxygen_Suppress */
+#define OSFUNC_COMMON_H
+/*! @endcond */
+
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/string.h>
+#else
+#include <string.h>
+#endif
+
+#include "img_types.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/**************************************************************************/ /*!
+@Function       DeviceMemSet
+@Description    Set memory, whose mapping may be uncached, to a given value.
+                Safe implementation for all architectures for uncached mapping,
+                optimised for speed where supported by tool chains.
+                In such cases, OSDeviceMemSet() is defined as a call to this
+                function.
+@Input          pvDest     void pointer to the memory to be set
+@Input          ui8Value   byte containing the value to be set
+@Input          ui32Size   the number of bytes to be set to the given value
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemCopy
+@Description    Copy values from one area of memory. Safe implementation for
+                all architectures for uncached mapping, of either the source
+                or destination, optimised for speed where supported by tool
+                chains. In such cases, OSDeviceMemCopy() is defined as a call
+                to this function.
+@Input          pvDst      void pointer to the destination memory
+@Input          pvSrc      void pointer to the source memory
+@Input          ui32Size   the number of bytes to be copied
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemCopy(void *pvDst, const void *pvSrc, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemSetBytes
+@Description    Potentially very slow (but safe) memset fallback for non-GNU C
+                compilers for arm64/aarch64
+@Input          pvDest     void pointer to the memory to be set
+@Input          ui8Value   byte containing the value to be set
+@Input          ui32Size   the number of bytes to be set to the given value
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemSetBytes(void *pvDest, IMG_UINT8 ui8Value, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemCopyBytes
+@Description    Potentially very slow (but safe) memcpy fallback for non-GNU C
+                compilers for arm64/aarch64
+@Input          pvDst      void pointer to the destination memory
+@Input          pvSrc      void pointer to the source memory
+@Input          ui32Size   the number of bytes to be copied
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemCopyBytes(void *pvDst, const void *pvSrc, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSStringSafeCopy
+@Description    Copy at most uDataSize-1 bytes from pszSrc to pszDest. pszDest
+                is always null-terminated. If no null byte ('\0') is contained
+                within the first uDataSize-1 characters of the source string,
+                the destination will contain the truncated source string,
+                otherwise pszDest will contain the entire source string.
+@Input          pszDest    char pointer to the destination string
+@Input          pszSrc     const char pointer to the source string
+@Input          uDataSize  the maximum number of bytes to be copied; this
+                           should normally be the size of the destination
+                           buffer
+@Return         Number of bytes copied if uDataSize > strlen(pszSrc), negative
+                value otherwise
+ */ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__) && !defined(DEBUG)
+static inline ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	return strscpy(pszDest, pszSrc, uDataSize);
+}
+#else
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize);
+#endif
+
+#if defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)
+#if defined(__GNUC__)
+/* Workarounds for assumptions made that memory will not be mapped uncached
+ * in kernel or user address spaces on arm64 platforms (or other testing).
+ */
+
+#define OSDeviceMemSet(a,b,c)  DeviceMemSet((a), (b), (c))
+#define OSDeviceMemCopy(a,b,c) DeviceMemCopy((a), (b), (c))
+
+#else /* defined __GNUC__ */
+
+#define OSDeviceMemSet(a,b,c)  DeviceMemSetBytes((a), (b), (c))
+#define OSDeviceMemCopy(a,b,c) DeviceMemCopyBytes((a), (b), (c))
+
+#endif /* defined __GNUC__ */
+
+#else /* (defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)) */
+
+/* Everything else */
+
+/**************************************************************************/ /*!
+@Function       OSDeviceMemSet
+@Description    Set memory, whose mapping may be uncached, to a given value.
+                On some architectures, additional processing may be needed
+                if the mapping is uncached.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSDeviceMemSet(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memset((a), (b), (c)); \
+			(void) *(volatile IMG_UINT32*)((void*)(a)); \
+		} \
+	} while (false)
+
+/**************************************************************************/ /*!
+@Function       OSDeviceMemCopy
+@Description    Copy values from one area of memory, to another, when one
+                or both mappings may be uncached.
+                On some architectures, additional processing may be needed
+                if mappings are uncached.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSDeviceMemCopy(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			memcpy((a), (b), (c)); \
+			(void) *(volatile IMG_UINT32*)((void*)(a)); \
+		} \
+	} while (false)
+
+#endif /* (defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)) */
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemSet
+@Description    Set memory, where the mapping is known to be cached, to a
+                given value. This function exists to allow an optimal memset
+                to be performed when memory is known to be cached.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemSet(a,b,c)  (void) memset((a), (b), (c))
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemCopy
+@Description    Copy values from one area of memory, to another, when both
+                mappings are known to be cached.
+                This function exists to allow an optimal memcpy to be
+                performed when memory is known to be cached.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemCopy(a,b,c) memcpy((a), (b), (c))
+
+#if defined(__KERNEL__)
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemSetWMB
+@Description    Set memory, where the mapping is known to be cached or
+                write-combine, to a given value and issue a write memory barrier
+                after. This
+                function exists to allow an optimal memset to be performed when
+                memory is known to be cached or write-combine.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemSetWMB(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memset((a), (b), (c)); \
+			OSWriteMemoryBarrier(a); \
+		} \
+	} while (false)
+/**************************************************************************/ /*!
+@Function       OSCachedMemCopy
+@Description    Copy values from one area of memory, to another, when both
+                mappings are known to be cached or write-combine and issue
+                a write memory barrier after.
+                This function exists to allow an optimal memcpy to be
+                performed when memory is known to be cached or write-combine.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemCopyWMB(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memcpy((a), (b), (c)); \
+			OSWriteMemoryBarrier(a); \
+		} \
+	} while (false)
+#endif /* defined(__KERNEL__) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OSFUNC_COMMON_H */
+
+/******************************************************************************
+ End of file (osfunc_common.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pdumpdefs.h b/drivers/gpu/drm/img/img-volcanic/include/pdumpdefs.h
new file mode 100644
index 000000000000..7cea362a4152
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pdumpdefs.h
@@ -0,0 +1,268 @@
+/*************************************************************************/ /*!
+@File
+@Title          PDUMP definitions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    PDUMP definitions header
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PDUMPDEFS_H
+#define PDUMPDEFS_H
+
+/*! PDump Pixel Format Enumeration */
+typedef enum _PDUMP_PIXEL_FORMAT_
+{
+	PVRSRV_PDUMP_PIXEL_FORMAT_UNSUPPORTED = 0,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB8 = 1,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB332 = 2,
+	PVRSRV_PDUMP_PIXEL_FORMAT_KRGB555 = 3,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB565 = 4,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB4444 = 5,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB1555 = 6,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB888 = 7,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB8888 = 8,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV8 = 9,
+/*	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV4444 = 10, */
+	PVRSRV_PDUMP_PIXEL_FORMAT_VY0UY1_8888 = 11,
+	PVRSRV_PDUMP_PIXEL_FORMAT_UY0VY1_8888 = 12,
+	PVRSRV_PDUMP_PIXEL_FORMAT_Y0UY1V_8888 = 13,
+	PVRSRV_PDUMP_PIXEL_FORMAT_Y0VY1U_8888 = 14,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV888 = 15,
+	PVRSRV_PDUMP_PIXEL_FORMAT_UYVY10101010 = 16,
+	PVRSRV_PDUMP_PIXEL_FORMAT_VYAUYA8888 = 17,
+	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV8888 = 18,
+	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV2101010 = 19,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV101010 = 20,
+	PVRSRV_PDUMP_PIXEL_FORMAT_PL12Y8 = 21,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_IMC2 = 22,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV12 = 23,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_PL8 = 24,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_PL12 = 25,
+	PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV8 = 26,
+	PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV8 = 27,
+	PVRSRV_PDUMP_PIXEL_FORMAT_PL12Y10 = 28,
+	PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV10 = 29,
+	PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV10 = 30,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR8888 = 31,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA8888 = 32,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB8332 = 33,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB555 = 34,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16 = 35,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32 = 36,
+	PVRSRV_PDUMP_PIXEL_FORMAT_L16 = 37,
+	PVRSRV_PDUMP_PIXEL_FORMAT_L32 = 38,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA8888 = 39,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR4444 = 40,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA4444 = 41,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA4444 = 42,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR1555 = 43,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA5551 = 44,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA5551 = 45,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGR565 = 46,
+	PVRSRV_PDUMP_PIXEL_FORMAT_A8 = 47,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16F16F16 = 49,
+	PVRSRV_PDUMP_PIXEL_FORMAT_A4 = 50,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB2101010 = 51,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RSGSBS888 = 52,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32F32F32 = 53,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16 = 54,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32 = 55,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16F16 = 56,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32F32 = 57,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U8 = 58,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U8U8 = 59,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16 = 60,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16U16 = 61,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16U16U16U16 = 62,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32 = 63,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32U32 = 64,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32U32U32U32 = 65,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV32 = 66,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR10_PACK16 = 67,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB10_PACK16 = 68,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA10_PACK16 = 69,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA10_PACK16 = 70,
+
+	PVRSRV_PDUMP_PIXEL_FORMAT_FORCE_I32 = 0x7fffffff
+
+} PDUMP_PIXEL_FORMAT;
+
+typedef enum _PDUMP_FBC_SWIZZLE_
+{
+	PVRSRV_PDUMP_FBC_SWIZZLE_ARGB = 0x0,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ARBG = 0x1,
+	PVRSRV_PDUMP_FBC_SWIZZLE_AGRB = 0x2,
+	PVRSRV_PDUMP_FBC_SWIZZLE_AGBR = 0x3,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ABGR = 0x4,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ABRG = 0x5,
+	PVRSRV_PDUMP_FBC_SWIZZLE_RGBA = 0x8,
+	PVRSRV_PDUMP_FBC_SWIZZLE_RBGA = 0x9,
+	PVRSRV_PDUMP_FBC_SWIZZLE_GRBA = 0xA,
+	PVRSRV_PDUMP_FBC_SWIZZLE_GBRA = 0xB,
+	PVRSRV_PDUMP_FBC_SWIZZLE_BGRA = 0xC,
+	PVRSRV_PDUMP_FBC_SWIZZLE_BRGA = 0xD,
+} PDUMP_FBC_SWIZZLE;
+
+/*! PDump addrmode */
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT			0
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_MASK			0x000000FF
+
+#define PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_SHIFT			8
+#define PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_NEGATIVE		(1U << PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT		11
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_MASK		0x00000800
+
+#define PVRSRV_PDUMP_ADDRMODE_BIFTILE_MODE_SHIFT		12
+#define PVRSRV_PDUMP_ADDRMODE_BIFTILE_MODE_MASK			0x000FF000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT				20
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_MASK				0x00F00000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCDECOR_SHIFT			24
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_SHIFT			25
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT		26
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_MASK		0x0C000000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT			28
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_MASK			0xF0000000
+
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_STRIDE			(0U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE1 (1U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE2 (2U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE3 (3U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE4 (4U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE5 (5U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE6 (6U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE7 (7U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_TWIDDLED		(9U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_PAGETILED		(11U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_ZTWIDDLED		(12U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_25_50_75	(0U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_25_37_50	(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_NONE				(0U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_DIRECT		(1U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_DIRECT		(2U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_32X2_DIRECT		(3U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_INDIRECT		(4U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_INDIRECT		(5U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_INDIRECT_4TILE	(6U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_INDIRECT_4TILE	(7U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_NONE		(0U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_75		(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_37		(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_50		(2U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_25		(3U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBC_DECOR					(1U << PVRSRV_PDUMP_ADDRMODE_FBCDECOR_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBC_LOSSY					(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_BASE			(1U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_ENHANCED		(2U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V2				(3U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_SURFACE		(4U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_RESOURCE		(5U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_1_SURFACE	(6U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_1_RESOURCE	(7U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V4				(8U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V4PLUS			(9U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_TFBCDC			(10U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+
+/*! PDump Poll Operator */
+typedef enum _PDUMP_POLL_OPERATOR
+{
+	PDUMP_POLL_OPERATOR_EQUAL = 0,
+	PDUMP_POLL_OPERATOR_LESS = 1,
+	PDUMP_POLL_OPERATOR_LESSEQUAL = 2,
+	PDUMP_POLL_OPERATOR_GREATER = 3,
+	PDUMP_POLL_OPERATOR_GREATEREQUAL = 4,
+	PDUMP_POLL_OPERATOR_NOTEQUAL = 5,
+} PDUMP_POLL_OPERATOR;
+
+
+#define PVRSRV_PDUMP_MAX_FILENAME_SIZE			75  /*!< Max length of a pdump log file name */
+#define PVRSRV_PDUMP_MAX_COMMENT_SIZE			350 /*!< Max length of a pdump comment */
+
+/*!
+	PDump MMU type
+*/
+typedef enum
+{
+	PDUMP_MMU_TYPE_4KPAGE_32BIT_STDTILE  = 1,
+	PDUMP_MMU_TYPE_VARPAGE_32BIT_STDTILE = 2,
+	PDUMP_MMU_TYPE_4KPAGE_36BIT_EXTTILE  = 3,
+	PDUMP_MMU_TYPE_4KPAGE_32BIT_EXTTILE  = 4,
+	PDUMP_MMU_TYPE_4KPAGE_36BIT_STDTILE  = 5,
+	PDUMP_MMU_TYPE_VARPAGE_40BIT         = 6,
+	PDUMP_MMU_TYPE_VIDEO_40BIT_STDTILE   = 7,
+	PDUMP_MMU_TYPE_VIDEO_40BIT_EXTTILE   = 8,
+	PDUMP_MMU_TYPE_MIPS_MICROAPTIV       = 9,
+	PDUMP_MMU_TYPE_LAST
+} PDUMP_MMU_TYPE;
+
+/*!
+	PDump states
+	These values are used by the bridge call PVRSRVPDumpGetState
+*/
+#define PDUMP_STATE_CAPTURE_FRAME		(1U)		/*!< Flag represents the PDump being in capture range or not*/
+#define PDUMP_STATE_CONNECTED			(2U)		/*!< Flag represents the PDump Client App being connected on not */
+#define PDUMP_STATE_SUSPENDED			(4U)		/*!< Flag represents the PDump being suspended or not */
+#define PDUMP_STATE_CAPTURE_IN_INTERVAL	(8U)		/*!< Flag represents the PDump being in a capture range interval */
+#define PDUMP_STATE_APP_TERMINATED		(16U)		/*!< Flag represents the PDump captured app has been terminated */
+
+/*!
+	PDump Capture modes
+	Values used with calls to PVRSRVPDumpSetDefaultCaptureParams
+*/
+#define PDUMP_CAPMODE_UNSET                     0x00000000UL
+#define PDUMP_CAPMODE_FRAMED                    0x00000001UL
+#define PDUMP_CAPMODE_CONTINUOUS                0x00000002UL
+#define PDUMP_CAPMODE_BLOCKED                   0x00000003UL
+
+#define PDUMP_CAPMODE_MAX                       PDUMP_CAPMODE_BLOCKED
+
+#endif /* PDUMPDEFS_H */
+
+/*****************************************************************************
+ End of file (pdumpdefs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pdumpdesc.h b/drivers/gpu/drm/img/img-volcanic/include/pdumpdesc.h
new file mode 100644
index 000000000000..273cde910e7f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pdumpdesc.h
@@ -0,0 +1,230 @@
+/*************************************************************************/ /*!
+@File           pdumpdesc.h
+@Title          PDump Descriptor format
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Describes PDump descriptors that may be passed to the
+                extraction routines (SAB).
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PDUMPDESC_H)
+#define PDUMPDESC_H
+
+#include "pdumpdefs.h"
+
+/*
+ * Common fields
+ */
+#define HEADER_WORD0_TYPE_SHIFT				(0)
+#define HEADER_WORD0_TYPE_CLRMSK			(0xFFFFFFFFU)
+
+#define HEADER_WORD1_SIZE_SHIFT				(0)
+#define HEADER_WORD1_SIZE_CLRMSK			(0x0000FFFFU)
+#define HEADER_WORD1_VERSION_SHIFT			(16)
+#define HEADER_WORD1_VERSION_CLRMSK			(0xFFFF0000U)
+
+#define HEADER_WORD2_DATA_SIZE_SHIFT		(0)
+#define HEADER_WORD2_DATA_SIZE_CLRMSK		(0xFFFFFFFFU)
+
+
+/*
+ * The image type descriptor
+ */
+
+/*
+ * Header type (IMGBv3) - 'IMGB' in hex + VERSION 3
+ * Header size - 64 bytes
+ */
+#define IMAGE_HEADER_TYPE					(0x42474D49)
+#define IMAGE_HEADER_SIZE					(64)
+#define IMAGE_HEADER_VERSION				(3)
+
+/*
+ * Image type-specific fields
+ */
+#define IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT		(0)
+#define IMAGE_HEADER_WORD3_LOGICAL_WIDTH_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT		(0)
+#define IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD5_FORMAT_SHIFT				(0)
+#define IMAGE_HEADER_WORD5_FORMAT_CLRMSK			(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT		(0)
+#define IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT	(0)
+#define IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD8_TWIDDLING_SHIFT			(0)
+#define IMAGE_HEADER_WORD8_TWIDDLING_CLRMSK			(0x000000FFU)
+#define IMAGE_HEADER_WORD8_TWIDDLING_STRIDED		(0 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+#define IMAGE_HEADER_WORD8_TWIDDLING_NTWIDDLE		(9 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+#define IMAGE_HEADER_WORD8_TWIDDLING_ZTWIDDLE		(12 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+
+
+#define IMAGE_HEADER_WORD8_STRIDE_SHIFT				(8)
+#define IMAGE_HEADER_WORD8_STRIDE_CLRMSK			(0x0000FF00U)
+#define IMAGE_HEADER_WORD8_STRIDE_POSITIVE			(0 << IMAGE_HEADER_WORD8_STRIDE_SHIFT)
+#define IMAGE_HEADER_WORD8_STRIDE_NEGATIVE			(1 << IMAGE_HEADER_WORD8_STRIDE_SHIFT)
+
+#define IMAGE_HEADER_WORD8_BIFTYPE_SHIFT			(16)
+#define IMAGE_HEADER_WORD8_BIFTYPE_CLRMSK			(0x00FF0000U)
+#define IMAGE_HEADER_WORD8_BIFTYPE_NONE				(0 << IMAGE_HEADER_WORD8_BIFTYPE_SHIFT)
+
+#define IMAGE_HEADER_WORD8_FBCTYPE_SHIFT			(24)
+#define IMAGE_HEADER_WORD8_FBCTYPE_CLRMSK			(0xFF000000U)
+#define IMAGE_HEADER_WORD8_FBCTYPE_8X8				(1 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+#define IMAGE_HEADER_WORD8_FBCTYPE_16x4				(2 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+#define IMAGE_HEADER_WORD8_FBCTYPE_32x2				(3 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+
+#define IMAGE_HEADER_WORD9_FBCDECOR_SHIFT			(0)
+#define IMAGE_HEADER_WORD9_FBCDECOR_CLRMSK			(0x000000FFU)
+#define IMAGE_HEADER_WORD9_FBCDECOR_ENABLE			(1 << IMAGE_HEADER_WORD9_FBCDECOR_SHIFT)
+
+/* Align with fbcomp_export_c.h in pdump_tools branch */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT			(8)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_CLRMSK			(0x0000FF00U)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_SAME_AS_GPU	(0 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_BASE			(1 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_TWIDDLED_EN	(2 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* TWIDDLED_ENHANCED */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V2				(3 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT1   (4 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2   (5 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* V30_WITH_HEADER_REMAP */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT1   (6 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT2   (7 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* V31_WITH_HEADER_REMAP */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V4				(8 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V4_PLUS		(9 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_TFBC			(10 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+
+#define IMAGE_HEADER_WORD9_LOSSY_SHIFT				(16)
+#define IMAGE_HEADER_WORD9_LOSSY_CLRMSK				(0x00FF0000U)
+/* Non-TFBC */
+#define IMAGE_HEADER_WORD9_LOSSY_ON					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+
+/* TFBC */
+#define IMAGE_HEADER_WORD9_LOSSY_75					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_37					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_50					(2 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_25					(3 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_OFF				(0 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+
+#define IMAGE_HEADER_WORD9_SWIZZLE_SHIFT			(24)
+#define IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK			(0xFF000000U)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ARGB		(0x0 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ARBG		(0x1 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_AGRB		(0x2 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_AGBR		(0x3 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ABGR		(0x4 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ABRG		(0x5 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_RGBA		(0x8 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_RBGA		(0x9 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_GRBA		(0xA << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_GBRA		(0xB << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_BGRA		(0xC << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_BRGA		(0xD << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+
+#define IMAGE_HEADER_WORD10_FBCCLEAR_CH0_SHIFT		(0)
+#define IMAGE_HEADER_WORD10_FBCCLEAR_CH0_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD11_FBCCLEAR_CH1_SHIFT		(0)
+#define IMAGE_HEADER_WORD11_FBCCLEAR_CH1_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD12_FBCCLEAR_CH2_SHIFT		(0)
+#define IMAGE_HEADER_WORD12_FBCCLEAR_CH2_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD13_FBCCLEAR_CH3_SHIFT		(0)
+#define IMAGE_HEADER_WORD13_FBCCLEAR_CH3_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT		(0)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_CLRMSK		(0x000000FFU)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75		(0 << IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50		(1 << IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT)
+
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT		(8)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_CLRMSK		(0x0000FF00U)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_ALL			(0 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR	(1 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY	(2 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_PTC_ONLY	(3 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_SHIFT	(16)
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_CLRMSK	(0x00FF0000U)
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN		(1 << IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_SHIFT) /* Treat YUV10 optimal formats as 8 bits */
+
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_SHIFT	(24)
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_CLRMSK	(0xFF000000U)
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN		(1 << IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_SHIFT) /* Override lossy min channel setting */
+
+/* IMAGE_HEADER_WORD15_RESERVED2 */
+
+/*
+ * The data type descriptor
+ */
+
+/*
+ * Header type (IMGCv1) - 'IMGC' in hex + VERSION 0
+ * Header size - 20 bytes (5 x 32 bit WORDS)
+ */
+#define DATA_HEADER_TYPE                    (0x43474D49)
+#define DATA_HEADER_SIZE                    (20)
+#define DATA_HEADER_VERSION                 (0)
+
+/*
+ * The IBIN type descriptor
+ */
+
+/*
+ * Header type (IBIN) - 'IBIN' in hex + VERSION 0
+ * Header size - 12 bytes (3 x 32 bit WORDS)
+ */
+#define IBIN_HEADER_TYPE                    (0x4e494249)
+#define IBIN_HEADER_SIZE                    (12)
+#define IBIN_HEADER_VERSION                 (0)
+
+/*
+ * Data type-specific fields
+ */
+#define DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT        (0)
+#define DATA_HEADER_WORD3_ELEMENT_TYPE_CLRMSK       (0xFFFFFFFFU)
+
+#define DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT       (0)
+#define DATA_HEADER_WORD4_ELEMENT_COUNT_CLRMSK      (0xFFFFFFFFU)
+
+#endif /* PDUMPDESC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/buffer_attribs.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/buffer_attribs.h
new file mode 100644
index 000000000000..41eaaaecd19e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/buffer_attribs.h
@@ -0,0 +1,193 @@
+/*************************************************************************/ /*!
+@File
+@Title          3D types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef POWERVR_BUFFER_ATTRIBS_H
+#define POWERVR_BUFFER_ATTRIBS_H
+
+/*!
+ * Memory layouts
+ * Defines how pixels are laid out within a surface.
+ */
+typedef enum
+{
+	IMG_MEMLAYOUT_STRIDED,       /**< Resource is strided, one row at a time */
+	IMG_MEMLAYOUT_TWIDDLED,      /**< Resource is 2D twiddled to match HW */
+	IMG_MEMLAYOUT_3DTWIDDLED,    /**< Resource is 3D twiddled, classic style */
+	IMG_MEMLAYOUT_TILED,         /**< Resource is tiled, tiling config specified elsewhere. */
+	IMG_MEMLAYOUT_PAGETILED,     /**< Resource is pagetiled */
+	IMG_MEMLAYOUT_INVNTWIDDLED,  /**< Resource is 2D twiddled !N style */
+} IMG_MEMLAYOUT;
+
+/*!
+ * Rotation types
+ */
+typedef enum
+{
+	IMG_ROTATION_0DEG = 0,
+	IMG_ROTATION_90DEG = 1,
+	IMG_ROTATION_180DEG = 2,
+	IMG_ROTATION_270DEG = 3,
+	IMG_ROTATION_FLIP_Y = 4,
+
+	IMG_ROTATION_BAD = 255,
+} IMG_ROTATION;
+
+/*!
+ * Alpha types.
+ */
+typedef enum
+{
+	IMG_COLOURSPACE_FORMAT_UNKNOWN                 =  0x0UL << 16,
+	IMG_COLOURSPACE_FORMAT_LINEAR                  =  0x1UL << 16,
+	IMG_COLOURSPACE_FORMAT_SRGB                    =  0x2UL << 16,
+	IMG_COLOURSPACE_FORMAT_SCRGB                   =  0x3UL << 16,
+	IMG_COLOURSPACE_FORMAT_SCRGB_LINEAR            =  0x4UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3_LINEAR       =  0x5UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3              =  0x6UL << 16,
+	IMG_COLOURSPACE_FORMAT_BT2020_PQ               =  0x7UL << 16,
+	IMG_COLOURSPACE_FORMAT_BT2020_LINEAR           =  0x8UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3_PASSTHROUGH  =  0x9UL << 16,
+	IMG_COLOURSPACE_FORMAT_MASK                    =  0xFUL << 16,
+} IMG_COLOURSPACE_FORMAT;
+
+/*!
+ * Determines if FB Compression is Lossy
+ */
+#define IS_FBCDC_LOSSY(mode)			((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_TRUE : IMG_FALSE)
+
+/*!
+ * Determines if FB Compression is Packed
+ */
+#define IS_FBCDC_PACKED(mode)			((mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8) ? IMG_TRUE : IMG_FALSE)
+
+/*!
+ * Returns type of FB Compression
+ */
+#define GET_FBCDC_BLOCK_TYPE(mode)		((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8)   ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : mode)
+
+/*!
+ * Adds Packing compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_PACKING(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_PACKED_8x8 : mode)
+
+/*!
+ * Removes Packing compression setting from mode
+ */
+#define FBCDC_MODE_REMOVE_PACKING(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8 : mode)
+
+/*!
+ * Adds Lossy25 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY25(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2 : mode)
+
+/*!
+ * Adds Lossy37 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY37(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2 : mode)
+
+/*!
+ * Adds Lossy50 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY50(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2 : mode)
+
+/*!
+ * Adds Lossy75 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY75(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2 : mode)
+
+/*!
+ * Removes Lossy compression setting from mode
+ */
+#define FBCDC_MODE_REMOVE_LOSSY(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : mode)
+
+/*!
+ * Types of framebuffer compression
+ */
+typedef enum
+{
+	IMG_FB_COMPRESSION_NONE,
+	IMG_FB_COMPRESSION_DIRECT_8x8,
+	IMG_FB_COMPRESSION_DIRECT_16x4,
+	IMG_FB_COMPRESSION_DIRECT_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2,
+	IMG_FB_COMPRESSION_DIRECT_PACKED_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2,
+} IMG_FB_COMPRESSION;
+
+
+#endif /* POWERVR_BUFFER_ATTRIBS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/img_drm_fourcc.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/img_drm_fourcc.h
new file mode 100644
index 000000000000..d4717db1cccb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/img_drm_fourcc.h
@@ -0,0 +1,161 @@
+/*************************************************************************/ /*!
+@File
+@Title          Wrapper around drm_fourcc.h
+@Description    FourCCs and DRM framebuffer modifiers that are not in the
+                Kernel's and libdrm's drm_fourcc.h can be added here.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DRM_FOURCC_H
+#define IMG_DRM_FOURCC_H
+
+#if defined(__KERNEL__)
+#include <drm/drm_fourcc.h>
+#else
+/*
+ * Include types.h to workaround versions of libdrm older than 2.4.68
+ * not including the correct headers.
+ */
+#include <linux/types.h>
+
+#include <drm_fourcc.h>
+#endif
+
+/*
+ * Don't get too inspired by this example :)
+ * ADF doesn't support DRM modifiers, so the memory layout had to be
+ * included in the fourcc name, but the proper way to specify information
+ * additional to pixel formats is to use DRM modifiers.
+ *
+ * See upstream drm_fourcc.h for the proper naming convention.
+ */
+#ifndef DRM_FORMAT_BGRA8888_DIRECT_16x4
+#define DRM_FORMAT_BGRA8888_DIRECT_16x4 fourcc_code('I', 'M', 'G', '0')
+#endif
+
+#if !defined(__KERNEL__)
+/*
+ * A definition for the same format was added in Linux kernel 5.2 in commit
+ * 88ab9c76d191ad8645b483f31e2b394b0f3e280e. As such, this definition has been
+ * deprecated and the DRM_FORMAT_ABGR16161616F kernel define should be used
+ * instead of this one.
+ */
+#define DRM_FORMAT_ABGR16_IMG_DEPRECATED fourcc_code('I', 'M', 'G', '1')
+#endif
+
+/*
+ * Upstream does not have a packed 10 Bits Per Channel YVU format yet,
+ * so let`s make one up.
+ * Note: at the moment this format is not intended to be used with
+ *       a framebuffer, so the kernels core DRM doesn`t need to know
+ *       about this format. This means that the kernel doesn`t need
+ *       to be patched.
+ */
+#if !defined(__KERNEL__)
+#define DRM_FORMAT_YVU444_PACK10_IMG fourcc_code('I', 'M', 'G', '2')
+#define DRM_FORMAT_YUV422_2PLANE_PACK10_IMG fourcc_code('I', 'M', 'G', '3')
+#define DRM_FORMAT_YUV420_2PLANE_PACK10_IMG fourcc_code('I', 'M', 'G', '4')
+#endif
+
+/*
+ * Value chosen in the middle of 255 pool to minimise the chance of hitting
+ * the same value potentially defined by other vendors in the drm_fourcc.h
+ */
+#define DRM_FORMAT_MOD_VENDOR_PVR 0x92
+
+#ifndef DRM_FORMAT_MOD_VENDOR_NONE
+#define DRM_FORMAT_MOD_VENDOR_NONE 0
+#endif
+
+#ifndef DRM_FORMAT_RESERVED
+#define DRM_FORMAT_RESERVED ((1ULL << 56) - 1)
+#endif
+
+#define img_fourcc_mod_combine(uiModHi, uiModLo) \
+	((__u64) ((__u32) (uiModHi)) << 32 | (__u64) ((__u32) (uiModLo)))
+
+#define img_fourcc_mod_hi(ui64Mod) \
+	((__u32) ((__u64) (ui64Mod) >> 32))
+
+#define img_fourcc_mod_lo(ui64Mod) \
+	((__u32) ((__u64) (ui64Mod)) & 0xffffffff)
+
+#ifndef fourcc_mod_code
+#define fourcc_mod_code(vendor, val) \
+	((((__u64)DRM_FORMAT_MOD_VENDOR_## vendor) << 56) | (val & 0x00ffffffffffffffULL))
+#endif
+
+#ifndef DRM_FORMAT_MOD_INVALID
+#define DRM_FORMAT_MOD_INVALID fourcc_mod_code(NONE, DRM_FORMAT_RESERVED)
+#endif
+
+#ifndef DRM_FORMAT_MOD_LINEAR
+#define DRM_FORMAT_MOD_LINEAR fourcc_mod_code(NONE, 0)
+#endif
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1      fourcc_mod_code(PVR, 3)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1     fourcc_mod_code(PVR, 9)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7      fourcc_mod_code(PVR, 6)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7     fourcc_mod_code(PVR, 12)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10     fourcc_mod_code(PVR, 21)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10    fourcc_mod_code(PVR, 22)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10    fourcc_mod_code(PVR, 23)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12     fourcc_mod_code(PVR, 15)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12    fourcc_mod_code(PVR, 16)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13             fourcc_mod_code(PVR, 24)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V13     fourcc_mod_code(PVR, 25)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V13     fourcc_mod_code(PVR, 26)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V13     fourcc_mod_code(PVR, 27)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13            fourcc_mod_code(PVR, 28)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V13    fourcc_mod_code(PVR, 29)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V13    fourcc_mod_code(PVR, 30)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V13    fourcc_mod_code(PVR, 31)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V13     fourcc_mod_code(PVR, 32)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V13    fourcc_mod_code(PVR, 33)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V14             fourcc_mod_code(PVR, 34)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V14     fourcc_mod_code(PVR, 35)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V14     fourcc_mod_code(PVR, 36)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V14     fourcc_mod_code(PVR, 37)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V14     fourcc_mod_code(PVR, 38)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V14            fourcc_mod_code(PVR, 39)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V14    fourcc_mod_code(PVR, 40)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V14    fourcc_mod_code(PVR, 41)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V14    fourcc_mod_code(PVR, 42)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V14    fourcc_mod_code(PVR, 43)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V14            fourcc_mod_code(PVR, 44)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_32x2_V14    fourcc_mod_code(PVR, 45)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_32x2_V14    fourcc_mod_code(PVR, 46)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_32x2_V14    fourcc_mod_code(PVR, 47)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_32x2_V14    fourcc_mod_code(PVR, 48)
+
+#endif /* IMG_DRM_FOURCC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/imgpixfmts.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/imgpixfmts.h
new file mode 100644
index 000000000000..127752640e92
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/imgpixfmts.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@File           imgpixfmts.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ **
+ ** WARNING: File is autogenerated by parsesystable.py - DO NOT EDIT.
+ ** Use fmts_systable.txt to add new formats.
+ **
+ *****************************************************************************/
+
+#if !defined(IMGPIXFMTS_H)
+#define IMGPIXFMTS_H
+
+typedef enum _IMG_PIXFMT_
+{
+	IMG_PIXFMT_UNKNOWN = 0,
+	IMG_PIXFMT_RESERVED_1 = 1,
+	IMG_PIXFMT_RESERVED_2 = 2,
+	IMG_PIXFMT_RESERVED_3 = 3,
+	IMG_PIXFMT_RESERVED_4 = 4,
+	IMG_PIXFMT_RESERVED_5 = 5,
+	IMG_PIXFMT_RESERVED_6 = 6,
+	IMG_PIXFMT_RESERVED_7 = 7,
+	IMG_PIXFMT_RESERVED_8 = 8,
+	IMG_PIXFMT_RESERVED_9 = 9,
+	IMG_PIXFMT_RESERVED_10 = 10,
+	IMG_PIXFMT_RESERVED_11 = 11,
+	IMG_PIXFMT_RESERVED_12 = 12,
+	IMG_PIXFMT_RESERVED_13 = 13,
+	IMG_PIXFMT_RESERVED_14 = 14,
+	IMG_PIXFMT_RESERVED_15 = 15,
+	IMG_PIXFMT_RESERVED_16 = 16,
+	IMG_PIXFMT_RESERVED_17 = 17,
+	IMG_PIXFMT_RESERVED_18 = 18,
+	IMG_PIXFMT_RESERVED_19 = 19,
+	IMG_PIXFMT_RESERVED_20 = 20,
+	IMG_PIXFMT_RESERVED_21 = 21,
+	IMG_PIXFMT_RESERVED_22 = 22,
+	IMG_PIXFMT_RESERVED_23 = 23,
+	IMG_PIXFMT_RESERVED_24 = 24,
+	IMG_PIXFMT_R10G10B10A2_UNORM = 25,
+	IMG_PIXFMT_RESERVED_26 = 26,
+	IMG_PIXFMT_RESERVED_27 = 27,
+	IMG_PIXFMT_B10G10R10A2_UNORM = 28,
+	IMG_PIXFMT_RESERVED_29 = 29,
+	IMG_PIXFMT_RESERVED_30 = 30,
+	IMG_PIXFMT_RESERVED_31 = 31,
+	IMG_PIXFMT_R8G8B8A8_UNORM = 32,
+	IMG_PIXFMT_R8G8B8A8_UNORM_SRGB = 33,
+	IMG_PIXFMT_RESERVED_34 = 34,
+	IMG_PIXFMT_RESERVED_35 = 35,
+	IMG_PIXFMT_RESERVED_36 = 36,
+	IMG_PIXFMT_R8G8B8X8_UNORM = 37,
+	IMG_PIXFMT_RESERVED_38 = 38,
+	IMG_PIXFMT_RESERVED_39 = 39,
+	IMG_PIXFMT_RESERVED_40 = 40,
+	IMG_PIXFMT_RESERVED_41 = 41,
+	IMG_PIXFMT_RESERVED_42 = 42,
+	IMG_PIXFMT_RESERVED_43 = 43,
+	IMG_PIXFMT_RESERVED_44 = 44,
+	IMG_PIXFMT_RESERVED_45 = 45,
+	IMG_PIXFMT_RESERVED_46 = 46,
+	IMG_PIXFMT_RESERVED_47 = 47,
+	IMG_PIXFMT_RESERVED_48 = 48,
+	IMG_PIXFMT_RESERVED_49 = 49,
+	IMG_PIXFMT_RESERVED_50 = 50,
+	IMG_PIXFMT_RESERVED_51 = 51,
+	IMG_PIXFMT_D32_FLOAT = 52,
+	IMG_PIXFMT_RESERVED_53 = 53,
+	IMG_PIXFMT_RESERVED_54 = 54,
+	IMG_PIXFMT_RESERVED_55 = 55,
+	IMG_PIXFMT_RESERVED_56 = 56,
+	IMG_PIXFMT_RESERVED_57 = 57,
+	IMG_PIXFMT_RESERVED_58 = 58,
+	IMG_PIXFMT_D24_UNORM_X8_TYPELESS = 59,
+	IMG_PIXFMT_RESERVED_60 = 60,
+	IMG_PIXFMT_RESERVED_61 = 61,
+	IMG_PIXFMT_RESERVED_62 = 62,
+	IMG_PIXFMT_R8G8_UNORM = 63,
+	IMG_PIXFMT_RESERVED_64 = 64,
+	IMG_PIXFMT_RESERVED_65 = 65,
+	IMG_PIXFMT_RESERVED_66 = 66,
+	IMG_PIXFMT_RESERVED_67 = 67,
+	IMG_PIXFMT_RESERVED_68 = 68,
+	IMG_PIXFMT_RESERVED_69 = 69,
+	IMG_PIXFMT_D16_UNORM = 70,
+	IMG_PIXFMT_RESERVED_71 = 71,
+	IMG_PIXFMT_RESERVED_72 = 72,
+	IMG_PIXFMT_RESERVED_73 = 73,
+	IMG_PIXFMT_RESERVED_74 = 74,
+	IMG_PIXFMT_RESERVED_75 = 75,
+	IMG_PIXFMT_RESERVED_76 = 76,
+	IMG_PIXFMT_R8_UNORM = 77,
+	IMG_PIXFMT_RESERVED_78 = 78,
+	IMG_PIXFMT_RESERVED_79 = 79,
+	IMG_PIXFMT_RESERVED_80 = 80,
+	IMG_PIXFMT_RESERVED_81 = 81,
+	IMG_PIXFMT_S8_UINT = 82,
+	IMG_PIXFMT_RESERVED_83 = 83,
+	IMG_PIXFMT_RESERVED_84 = 84,
+	IMG_PIXFMT_RESERVED_85 = 85,
+	IMG_PIXFMT_B5G6R5_UNORM = 86,
+	IMG_PIXFMT_R5G6B5_UNORM = 87,
+	IMG_PIXFMT_B5G5R5A1_UNORM = 88,
+	IMG_PIXFMT_B5G5R5X1_UNORM = 89,
+	IMG_PIXFMT_B8G8R8A8_UNORM = 90,
+	IMG_PIXFMT_B8G8R8X8_UNORM = 91,
+	IMG_PIXFMT_B8G8R8A8_UINT = 92,
+	IMG_PIXFMT_B8G8R8A8_SNORM = 93,
+	IMG_PIXFMT_B8G8R8A8_SINT = 94,
+	IMG_PIXFMT_B8G8R8A8_UNORM_SRGB = 95,
+	IMG_PIXFMT_RESERVED_96 = 96,
+	IMG_PIXFMT_RESERVED_97 = 97,
+	IMG_PIXFMT_RESERVED_98 = 98,
+	IMG_PIXFMT_RESERVED_99 = 99,
+	IMG_PIXFMT_RESERVED_100 = 100,
+	IMG_PIXFMT_RESERVED_101 = 101,
+	IMG_PIXFMT_RESERVED_102 = 102,
+	IMG_PIXFMT_RESERVED_103 = 103,
+	IMG_PIXFMT_RESERVED_104 = 104,
+	IMG_PIXFMT_RESERVED_105 = 105,
+	IMG_PIXFMT_RESERVED_106 = 106,
+	IMG_PIXFMT_RESERVED_107 = 107,
+	IMG_PIXFMT_RESERVED_108 = 108,
+	IMG_PIXFMT_RESERVED_109 = 109,
+	IMG_PIXFMT_RESERVED_110 = 110,
+	IMG_PIXFMT_RESERVED_111 = 111,
+	IMG_PIXFMT_RESERVED_112 = 112,
+	IMG_PIXFMT_RESERVED_113 = 113,
+	IMG_PIXFMT_RESERVED_114 = 114,
+	IMG_PIXFMT_RESERVED_115 = 115,
+	IMG_PIXFMT_RESERVED_116 = 116,
+	IMG_PIXFMT_RESERVED_117 = 117,
+	IMG_PIXFMT_RESERVED_118 = 118,
+	IMG_PIXFMT_RESERVED_119 = 119,
+	IMG_PIXFMT_RESERVED_120 = 120,
+	IMG_PIXFMT_RESERVED_121 = 121,
+	IMG_PIXFMT_RESERVED_122 = 122,
+	IMG_PIXFMT_RESERVED_123 = 123,
+	IMG_PIXFMT_RESERVED_124 = 124,
+	IMG_PIXFMT_RESERVED_125 = 125,
+	IMG_PIXFMT_RESERVED_126 = 126,
+	IMG_PIXFMT_RESERVED_127 = 127,
+	IMG_PIXFMT_RESERVED_128 = 128,
+	IMG_PIXFMT_RESERVED_129 = 129,
+	IMG_PIXFMT_RESERVED_130 = 130,
+	IMG_PIXFMT_RESERVED_131 = 131,
+	IMG_PIXFMT_RESERVED_132 = 132,
+	IMG_PIXFMT_RESERVED_133 = 133,
+	IMG_PIXFMT_RESERVED_134 = 134,
+	IMG_PIXFMT_RESERVED_135 = 135,
+	IMG_PIXFMT_RESERVED_136 = 136,
+	IMG_PIXFMT_RESERVED_137 = 137,
+	IMG_PIXFMT_RESERVED_138 = 138,
+	IMG_PIXFMT_L8_UNORM = 139,
+	IMG_PIXFMT_RESERVED_140 = 140,
+	IMG_PIXFMT_L8A8_UNORM = 141,
+	IMG_PIXFMT_RESERVED_142 = 142,
+	IMG_PIXFMT_RESERVED_143 = 143,
+	IMG_PIXFMT_RESERVED_144 = 144,
+	IMG_PIXFMT_RESERVED_145 = 145,
+	IMG_PIXFMT_RESERVED_146 = 146,
+	IMG_PIXFMT_RESERVED_147 = 147,
+	IMG_PIXFMT_B4G4R4A4_UNORM = 148,
+	IMG_PIXFMT_RESERVED_149 = 149,
+	IMG_PIXFMT_RESERVED_150 = 150,
+	IMG_PIXFMT_RESERVED_151 = 151,
+	IMG_PIXFMT_RESERVED_152 = 152,
+	IMG_PIXFMT_RESERVED_153 = 153,
+	IMG_PIXFMT_RESERVED_154 = 154,
+	IMG_PIXFMT_RESERVED_155 = 155,
+	IMG_PIXFMT_RESERVED_156 = 156,
+	IMG_PIXFMT_RESERVED_157 = 157,
+	IMG_PIXFMT_RESERVED_158 = 158,
+	IMG_PIXFMT_RESERVED_159 = 159,
+	IMG_PIXFMT_RESERVED_160 = 160,
+	IMG_PIXFMT_RESERVED_161 = 161,
+	IMG_PIXFMT_RESERVED_162 = 162,
+	IMG_PIXFMT_R8G8B8_UNORM = 163,
+	IMG_PIXFMT_R8G8B8_UNORM_SRGB = 164,
+	IMG_PIXFMT_RESERVED_165 = 165,
+	IMG_PIXFMT_RESERVED_166 = 166,
+	IMG_PIXFMT_RESERVED_167 = 167,
+	IMG_PIXFMT_RESERVED_168 = 168,
+	IMG_PIXFMT_RESERVED_169 = 169,
+	IMG_PIXFMT_RESERVED_170 = 170,
+	IMG_PIXFMT_RESERVED_171 = 171,
+	IMG_PIXFMT_RESERVED_172 = 172,
+	IMG_PIXFMT_RESERVED_173 = 173,
+	IMG_PIXFMT_UYVY = 174,
+	IMG_PIXFMT_VYUY = 175,
+	IMG_PIXFMT_YUYV = 176,
+	IMG_PIXFMT_YVYU = 177,
+	IMG_PIXFMT_YVU420_2PLANE = 178,
+	IMG_PIXFMT_YUV420_2PLANE = 179,
+	IMG_PIXFMT_YVU420_2PLANE_MACRO_BLOCK = 180,
+	IMG_PIXFMT_YUV420_3PLANE = 181,
+	IMG_PIXFMT_YVU420_3PLANE = 182,
+	IMG_PIXFMT_RESERVED_183 = 183,
+	IMG_PIXFMT_RESERVED_184 = 184,
+	IMG_PIXFMT_RESERVED_185 = 185,
+	IMG_PIXFMT_RESERVED_186 = 186,
+	IMG_PIXFMT_RESERVED_187 = 187,
+	IMG_PIXFMT_RESERVED_188 = 188,
+	IMG_PIXFMT_RESERVED_189 = 189,
+	IMG_PIXFMT_RESERVED_190 = 190,
+	IMG_PIXFMT_RESERVED_191 = 191,
+	IMG_PIXFMT_RESERVED_192 = 192,
+	IMG_PIXFMT_RESERVED_193 = 193,
+	IMG_PIXFMT_RESERVED_194 = 194,
+	IMG_PIXFMT_RESERVED_195 = 195,
+	IMG_PIXFMT_RESERVED_196 = 196,
+	IMG_PIXFMT_RESERVED_197 = 197,
+	IMG_PIXFMT_RESERVED_198 = 198,
+	IMG_PIXFMT_RESERVED_199 = 199,
+	IMG_PIXFMT_RESERVED_200 = 200,
+	IMG_PIXFMT_RESERVED_201 = 201,
+	IMG_PIXFMT_RESERVED_202 = 202,
+	IMG_PIXFMT_RESERVED_203 = 203,
+	IMG_PIXFMT_YVU8_422_2PLANE_PACK8 = 204,
+	IMG_PIXFMT_RESERVED_205 = 205,
+	IMG_PIXFMT_YVU10_444_1PLANE_PACK10 = 206,
+	IMG_PIXFMT_YVU10_420_2PLANE_PACK16 = 207,
+	IMG_PIXFMT_RESERVED_208 = 208,
+	IMG_PIXFMT_RESERVED_209 = 209,
+	IMG_PIXFMT_YUV8_422_2PLANE_PACK8 = 210,
+	IMG_PIXFMT_YUV8_444_3PLANE_PACK8 = 211,
+	IMG_PIXFMT_RESERVED_212 = 212,
+	IMG_PIXFMT_RESERVED_213 = 213,
+	IMG_PIXFMT_RESERVED_214 = 214,
+	IMG_PIXFMT_RESERVED_215 = 215,
+	IMG_PIXFMT_RESERVED_216 = 216,
+	IMG_PIXFMT_RESERVED_217 = 217,
+	IMG_PIXFMT_RESERVED_218 = 218,
+	IMG_PIXFMT_RESERVED_219 = 219,
+	IMG_PIXFMT_RESERVED_220 = 220,
+	IMG_PIXFMT_RESERVED_221 = 221,
+	IMG_PIXFMT_RESERVED_222 = 222,
+	IMG_PIXFMT_RESERVED_223 = 223,
+	IMG_PIXFMT_RESERVED_224 = 224,
+	IMG_PIXFMT_RESERVED_225 = 225,
+	IMG_PIXFMT_RESERVED_226 = 226,
+	IMG_PIXFMT_RESERVED_227 = 227,
+	IMG_PIXFMT_RESERVED_228 = 228,
+	IMG_PIXFMT_RESERVED_229 = 229,
+	IMG_PIXFMT_RESERVED_230 = 230,
+	IMG_PIXFMT_RESERVED_231 = 231,
+	IMG_PIXFMT_RESERVED_232 = 232,
+	IMG_PIXFMT_RESERVED_233 = 233,
+	IMG_PIXFMT_RESERVED_234 = 234,
+	IMG_PIXFMT_RESERVED_235 = 235,
+	IMG_PIXFMT_RESERVED_236 = 236,
+	IMG_PIXFMT_RESERVED_237 = 237,
+	IMG_PIXFMT_RESERVED_238 = 238,
+	IMG_PIXFMT_RESERVED_239 = 239,
+	IMG_PIXFMT_RESERVED_240 = 240,
+	IMG_PIXFMT_RESERVED_241 = 241,
+	IMG_PIXFMT_RESERVED_242 = 242,
+	IMG_PIXFMT_RESERVED_243 = 243,
+	IMG_PIXFMT_RESERVED_244 = 244,
+	IMG_PIXFMT_RESERVED_245 = 245,
+	IMG_PIXFMT_RESERVED_246 = 246,
+	IMG_PIXFMT_RESERVED_247 = 247,
+	IMG_PIXFMT_YVU8_420_2PLANE_PACK8_P = 248,
+	IMG_PIXFMT_RESERVED_249 = 249,
+	IMG_PIXFMT_RESERVED_250 = 250,
+	IMG_PIXFMT_RESERVED_251 = 251,
+	IMG_PIXFMT_YUV8_420_2PLANE_PACK8_P = 252,
+	IMG_PIXFMT_RESERVED_253 = 253,
+	IMG_PIXFMT_RESERVED_254 = 254,
+	IMG_PIXFMT_UYVY10_422_1PLANE_PACK10_CUST1 = 255,
+	IMG_PIXFMT_RESERVED_256 = 256,
+	IMG_PIXFMT_RESERVED_257 = 257,
+	IMG_PIXFMT_RESERVED_258 = 258,
+	IMG_PIXFMT_RESERVED_259 = 259,
+	IMG_PIXFMT_RESERVED_260 = 260,
+	IMG_PIXFMT_RESERVED_261 = 261,
+	IMG_PIXFMT_RESERVED_262 = 262,
+	IMG_PIXFMT_RESERVED_263 = 263,
+	IMG_PIXFMT_RESERVED_264 = 264,
+	IMG_PIXFMT_RESERVED_265 = 265,
+	IMG_PIXFMT_RESERVED_266 = 266,
+	IMG_PIXFMT_RESERVED_267 = 267,
+	IMG_PIXFMT_R10G10B10X2_UNORM = 268,
+#define IMG_PIXFMT_ENUM_COUNT 269
+} IMG_PIXFMT;
+
+#endif /* IMGPIXFMTS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/mem_types.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/mem_types.h
new file mode 100644
index 000000000000..a6dce8fe9889
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/mem_types.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File
+@Title          Public types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef POWERVR_TYPES_H
+#define POWERVR_TYPES_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(_MSC_VER)
+	#include "msvc_types.h"
+#elif defined(__linux__) && defined(__KERNEL__)
+	#include <linux/types.h>
+	#include <linux/compiler.h>
+#else
+	#include <stdint.h>
+	#define __iomem
+#endif
+
+typedef void *IMG_CPU_VIRTADDR;
+
+/* device virtual address */
+typedef struct
+{
+	uint64_t uiAddr;
+#define IMG_CAST_TO_DEVVADDR_UINT(var)		(uint64_t)(var)
+
+} IMG_DEV_VIRTADDR;
+
+typedef uint64_t IMG_DEVMEM_SIZE_T;
+typedef uint64_t IMG_DEVMEM_ALIGN_T;
+typedef uint64_t IMG_DEVMEM_OFFSET_T;
+typedef uint32_t IMG_DEVMEM_LOG2ALIGN_T;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/pvrsrv_sync_ext.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/pvrsrv_sync_ext.h
new file mode 100644
index 000000000000..933212e03c17
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/pvrsrv_sync_ext.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services external synchronisation interface header
+@Description    Defines synchronisation structures that are visible internally
+                and externally
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef POWERVR_SYNC_EXT_H
+#define POWERVR_SYNC_EXT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*!
+ * Number of sync prims still used internally in operations
+ */
+#define PVRSRV_MAX_SYNC_PRIMS 4U
+
+/*!
+ * Maximum number of dev var updates passed in a kick call
+ */
+#define PVRSRV_MAX_DEV_VARS 13U
+
+/*!
+ * Number of UFOs in operations
+ */
+#define	PVRSRV_MAX_SYNCS (PVRSRV_MAX_SYNC_PRIMS + PVRSRV_MAX_DEV_VARS)
+
+/*! Implementation independent types for passing fence/timeline to Services.
+ */
+typedef int32_t PVRSRV_FENCE;
+typedef int32_t PVRSRV_TIMELINE;
+typedef int32_t PVRSRV_EXP_FENCE_CTX;
+
+/*! Maximum length for an annotation name string for fence sync model objects.
+ */
+#define PVRSRV_SYNC_NAME_LENGTH 32U
+
+/* Macros for API callers using the fence sync model
+ */
+#define PVRSRV_NO_TIMELINE      ((PVRSRV_TIMELINE) -1)
+#define PVRSRV_NO_FENCE         ((PVRSRV_FENCE)    -1)
+#define PVRSRV_NO_EXP_FENCE_CTX ((PVRSRV_EXP_FENCE_CTX) -1)
+#define PVRSRV_NO_FENCE_PTR     NULL
+#define PVRSRV_NO_TIMELINE_PTR  NULL
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_buffer_sync_shared.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_buffer_sync_shared.h
new file mode 100644
index 000000000000..7a110910dbd6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_buffer_sync_shared.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR buffer sync shared
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared definitions between client and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BUFFER_SYNC_SHARED_H
+#define PVR_BUFFER_SYNC_SHARED_H
+
+#define PVR_BUFFER_FLAG_READ		(1U << 0)
+#define PVR_BUFFER_FLAG_WRITE		(1U << 1)
+#define PVR_BUFFER_FLAG_MASK		(PVR_BUFFER_FLAG_READ | \
+									 PVR_BUFFER_FLAG_WRITE)
+
+/* Maximum number of PMRs passed
+ * in a kick when using buffer sync
+ */
+#define PVRSRV_MAX_BUFFERSYNC_PMRS 32
+
+#endif /* PVR_BUFFER_SYNC_SHARED_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_debug.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_debug.h
new file mode 100644
index 000000000000..46033a0da47e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_debug.h
@@ -0,0 +1,1071 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Debug Declarations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides debug functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DEBUG_H
+#define PVR_DEBUG_H
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/* If the kernel pre defined macro is present, we will use the definition of HTBLOGK
+ * in htbserver.h with the format strings specified in ht_buffer_sf.h. Otherwise,
+ * HTBLOGK needs to be defined as empty since it is not used by user-mode code.
+ */
+#if defined(__KERNEL__)
+#	include "htbserver.h"
+#	include "htbuffer_sf.h"
+#else
+#	define HTBLOGK(...) ((void) 0)
+#endif
+
+/*! @cond Doxygen_Suppress */
+#if defined(_MSC_VER)
+#	define MSC_SUPPRESS_4127 __pragma(warning(suppress:4127))
+#else
+#	define MSC_SUPPRESS_4127
+#endif
+/*! @endcond */
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ && __KERNEL__*/
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define PVR_MAX_DEBUG_MESSAGE_LEN	(512)   /*!< Max length of a Debug Message */
+
+/* These are privately used by pvr_debug, use the PVR_DBG_ defines instead */
+#define DBGPRIV_FATAL     0x001U  /*!< Debug-Fatal. Privately used by pvr_debug. */
+#define DBGPRIV_ERROR     0x002U  /*!< Debug-Error. Privately used by pvr_debug. */
+#define DBGPRIV_WARNING   0x004U  /*!< Debug-Warning. Privately used by pvr_debug. */
+#define DBGPRIV_MESSAGE   0x008U  /*!< Debug-Message. Privately used by pvr_debug. */
+#define DBGPRIV_VERBOSE   0x010U  /*!< Debug-Verbose. Privately used by pvr_debug. */
+#define DBGPRIV_CALLTRACE 0x020U  /*!< Debug-CallTrace. Privately used by pvr_debug. */
+#define DBGPRIV_ALLOC     0x040U  /*!< Debug-Alloc. Privately used by pvr_debug. */
+#define DBGPRIV_BUFFERED  0x080U  /*!< Debug-Buffered. Privately used by pvr_debug. */
+#define DBGPRIV_DEBUG     0x100U  /*!< Debug-AdHoc-Debug. Never submitted. Privately used by pvr_debug. */
+#define DBGPRIV_LAST      0x100U  /*!< Always set to highest mask value. Privately used by pvr_debug. */
+
+/* Enable DPF logging for locally from some make targets */
+#if defined(PVRSRV_NEED_PVR_DPF_LOCAL)
+#undef PVRSRV_NEED_PVR_DPF
+#define PVRSRV_NEED_PVR_DPF
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_ASSERT) && defined(DEBUG)
+#define PVRSRV_NEED_PVR_ASSERT
+#endif
+
+#if defined(PVRSRV_NEED_PVR_ASSERT) && !defined(PVRSRV_NEED_PVR_DPF)
+#define PVRSRV_NEED_PVR_DPF
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_TRACE) && (defined(DEBUG) || defined(TIMING))
+#define PVRSRV_NEED_PVR_TRACE
+#endif
+
+#if !defined(DOXYGEN)
+/*************************************************************************/ /*
+PVRSRVGetErrorString
+Returns a string describing the provided PVRSRV_ERROR code
+NB No doxygen comments provided as this function does not require porting
+   for other operating systems
+*/ /**************************************************************************/
+const IMG_CHAR *PVRSRVGetErrorString(PVRSRV_ERROR eError);
+#define PVRSRVGETERRORSTRING PVRSRVGetErrorString
+#endif
+
+/* PVR_ASSERT() and PVR_DBG_BREAK handling */
+
+#if defined(__KLOCWORK__)
+/* A dummy no-return function to be used under Klocwork to mark unreachable
+   paths instead of abort() in order to avoid MISRA.STDLIB.ABORT issues. */
+__noreturn void klocwork_abort(void);
+#endif
+
+#if defined(PVRSRV_NEED_PVR_ASSERT) || defined(DOXYGEN)
+
+/* Unfortunately the Klocwork static analysis checker doesn't understand our
+ * ASSERT macros. Thus it reports lots of false positive. Defining our Assert
+ * macros in a special way when the code is analysed by Klocwork avoids
+ * them.
+ */
+#if defined(__KLOCWORK__)
+#define PVR_ASSERT(x) do { if (!(x)) {klocwork_abort();} } while (false)
+#else /* ! __KLOCWORKS__ */
+
+#if defined(_WIN32)
+#define PVR_ASSERT(expr) do										\
+	{															\
+		MSC_SUPPRESS_4127										\
+		if (unlikely(!(expr)))								\
+		{														\
+			PVRSRVDebugPrintf(DBGPRIV_FATAL, __FILE__, __LINE__,\
+					  "*** Debug assertion failed!");			\
+			__debugbreak();										\
+		}														\
+	MSC_SUPPRESS_4127											\
+	} while (false)
+
+#else
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include <linux/kernel.h>
+#include <linux/bug.h>
+
+/* In Linux kernel mode, use WARN_ON() directly. This produces the
+ * correct filename and line number in the warning message.
+ */
+#define PVR_ASSERT(EXPR) do											\
+	{																\
+		if (unlikely(!(EXPR)))										\
+		{															\
+			PVRSRVDebugPrintf(DBGPRIV_FATAL, __FILE__, __LINE__,	\
+							  "Debug assertion failed!");			\
+			WARN_ON(1);												\
+		}															\
+	} while (false)
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugAssertFail
+@Description    Indicate to the user that a debug assertion has failed and
+                prevent the program from continuing.
+                Invoked from the macro PVR_ASSERT().
+@Input          pszFile       The name of the source file where the assertion failed
+@Input          ui32Line      The line number of the failed assertion
+@Input          pszAssertion  String describing the assertion
+@Return         NEVER!
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV __noreturn
+PVRSRVDebugAssertFail(const IMG_CHAR *pszFile,
+                      IMG_UINT32 ui32Line,
+                      const IMG_CHAR *pszAssertion);
+
+#define PVR_ASSERT(EXPR) do										\
+	{															\
+		if (unlikely(!(EXPR)))									\
+		{														\
+			PVRSRVDebugAssertFail(__FILE__, __LINE__, #EXPR);	\
+		}														\
+	} while (false)
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+#endif /* defined(_WIN32) */
+#endif /* defined(__KLOCWORK__) */
+
+#if defined(__KLOCWORK__)
+	#define PVR_DBG_BREAK do { klocwork_abort(); } while (false)
+#else
+	#if defined(WIN32)
+		#define PVR_DBG_BREAK __debugbreak()   /*!< Implementation of PVR_DBG_BREAK for (non-WinCE) Win32 */
+	#else
+		#if defined(PVR_DBG_BREAK_ASSERT_FAIL)
+		/*!< Implementation of PVR_DBG_BREAK that maps onto PVRSRVDebugAssertFail */
+			#if defined(_WIN32)
+				#define PVR_DBG_BREAK	DBG_BREAK
+			#else
+				#if defined(__linux__) && defined(__KERNEL__)
+					#define PVR_DBG_BREAK BUG()
+				#else
+					#define PVR_DBG_BREAK	PVRSRVDebugAssertFail(__FILE__, __LINE__, "PVR_DBG_BREAK")
+				#endif
+			#endif
+		#else
+			/*!< Null Implementation of PVR_DBG_BREAK (does nothing) */
+			#define PVR_DBG_BREAK
+		#endif
+	#endif
+#endif
+
+
+#else /* defined(PVRSRV_NEED_PVR_ASSERT) */
+	/* Unfortunately the Klocwork static analysis checker doesn't understand our
+	* ASSERT macros. Thus it reports lots of false positive. Defining our Assert
+	* macros in a special way when the code is analysed by Klocwork avoids
+	* them.
+	*/
+	#if defined(__KLOCWORK__)
+		#define PVR_ASSERT(EXPR) do { if (!(EXPR)) {klocwork_abort();} } while (false)
+	#else
+		#define PVR_ASSERT(EXPR) (void)(EXPR) /*!< Null Implementation of PVR_ASSERT (does nothing) */
+	#endif
+
+	#define PVR_DBG_BREAK    /*!< Null Implementation of PVR_DBG_BREAK (does nothing) */
+
+#endif /* defined(PVRSRV_NEED_PVR_ASSERT) */
+
+
+/* PVR_DPF() handling */
+
+#if defined(PVRSRV_NEED_PVR_DPF) || defined(DOXYGEN)
+
+	/* New logging mechanism */
+	#define PVR_DBG_FATAL     DBGPRIV_FATAL     /*!< Debug level passed to PVRSRVDebugPrintf() for fatal errors. */
+	#define PVR_DBG_ERROR     DBGPRIV_ERROR     /*!< Debug level passed to PVRSRVDebugPrintf() for non-fatal errors. */
+	#define PVR_DBG_WARNING   DBGPRIV_WARNING   /*!< Debug level passed to PVRSRVDebugPrintf() for warnings. */
+	#define PVR_DBG_MESSAGE   DBGPRIV_MESSAGE   /*!< Debug level passed to PVRSRVDebugPrintf() for information only. */
+	#define PVR_DBG_VERBOSE   DBGPRIV_VERBOSE   /*!< Debug level passed to PVRSRVDebugPrintf() for very low-priority debug. */
+	#define PVR_DBG_CALLTRACE DBGPRIV_CALLTRACE /*!< Debug level passed to PVRSRVDebugPrintf() for function tracing purposes. */
+	#define PVR_DBG_ALLOC     DBGPRIV_ALLOC     /*!< Debug level passed to PVRSRVDebugPrintf() for tracking some of drivers memory operations. */
+	#define PVR_DBG_BUFFERED  DBGPRIV_BUFFERED  /*!< Debug level passed to PVRSRVDebugPrintf() when debug should be written to the debug circular buffer. */
+	#define PVR_DBG_DEBUG     DBGPRIV_DEBUG     /*!< Debug level passed to PVRSRVDebugPrintf() for debug messages. */
+
+	/* These levels are always on with PVRSRV_NEED_PVR_DPF */
+	/*! @cond Doxygen_Suppress */
+	#define PVR_DPF_0x001U(...) PVRSRVDebugPrintf(DBGPRIV_FATAL, __VA_ARGS__)
+	#define PVR_DPF_0x002U(...) PVRSRVDebugPrintf(DBGPRIV_ERROR, __VA_ARGS__)
+	#define PVR_DPF_0x080U(...) PVRSRVDebugPrintf(DBGPRIV_BUFFERED, __VA_ARGS__)
+
+	/*
+	 * The AdHoc-Debug level is only supported when enabled in the local
+	 * build environment and may need to be used in both debug and release
+	 * builds. An error is generated in the formal build if it is checked in.
+	 */
+#if defined(PVR_DPF_ADHOC_DEBUG_ON)
+	#define PVR_DPF_0x100U(...) PVRSRVDebugPrintf(DBGPRIV_DEBUG, __VA_ARGS__)
+#else
+	/* Use an undefined token here to stop compilation dead in the offending module */
+	#define PVR_DPF_0x100U(...) __ERROR__PVR_DBG_DEBUG_is_in_use_but_has_not_been_enabled__Note_Debug_DPF_must_not_be_checked_in__Define_PVR_DPF_ADHOC_DEBUG_ON_for_testing
+#endif
+
+	/* Some are compiled out completely in release builds */
+#if defined(DEBUG) || defined(DOXYGEN)
+	#define PVR_DPF_0x004U(...) PVRSRVDebugPrintf(DBGPRIV_WARNING, __VA_ARGS__)
+	#define PVR_DPF_0x008U(...) PVRSRVDebugPrintf(DBGPRIV_MESSAGE, __VA_ARGS__)
+	#define PVR_DPF_0x010U(...) PVRSRVDebugPrintf(DBGPRIV_VERBOSE, __VA_ARGS__)
+	#define PVR_DPF_0x020U(...) PVRSRVDebugPrintf(DBGPRIV_CALLTRACE, __VA_ARGS__)
+	#define PVR_DPF_0x040U(...) PVRSRVDebugPrintf(DBGPRIV_ALLOC, __VA_ARGS__)
+#else
+	#define PVR_DPF_0x004U(...)
+	#define PVR_DPF_0x008U(...)
+	#define PVR_DPF_0x010U(...)
+	#define PVR_DPF_0x020U(...)
+	#define PVR_DPF_0x040U(...)
+#endif
+
+	/* Translate the different log levels to separate macros
+	 * so they can each be compiled out.
+	 */
+#if defined(DEBUG)
+	#define PVR_DPF_EX(lvl, ...) PVR_DPF_ ## lvl (__FILE__, __LINE__, __VA_ARGS__)
+#else
+	#define PVR_DPF_EX(lvl, ...) PVR_DPF_ ## lvl ("", __LINE__, __VA_ARGS__)
+#endif
+	/*! @endcond */
+
+	/* Get rid of the double bracketing */
+	#define PVR_DPF(x) PVR_DPF_EX x
+
+	#define PVR_LOG_ERROR(_rc, _call) do \
+		{ \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+		} while (false)
+
+	#define PVR_LOG_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_WARN_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_WARNING, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_NOMEM(_expr, _call) do \
+		{ \
+			if (unlikely(_expr == NULL)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_OUT_OF_MEMORY, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()", _call, __func__)); \
+				return PVRSRV_ERROR_OUT_OF_MEMORY; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_NOMEM(_expr, _err, _go) do \
+		{ \
+			if (unlikely(_expr == NULL)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_OUT_OF_MEMORY, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()", #_expr, __func__)); \
+				_err = PVRSRV_ERROR_OUT_OF_MEMORY; \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				return; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR(_rc, _call, _go) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_WITH_ERROR(_call, _err, _rc, _go) do \
+		{ \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			_err = _rc; \
+			goto _go; \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_IF_FALSE(_expr, _msg) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_FALSE(_expr, _msg, _rc) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_FALSE(_expr, _msg) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_FALSE(_expr, _msg, _go) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE(_expr, _msg, _rc) do \
+		{ \
+			if (unlikely((_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_T, PVRSRV_ERROR_UNEXPECTED_TRUE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_INVALID_PARAM(_expr, _param) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_INVALID_PARAMS, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s invalid in %s()", _param, __func__)); \
+				return PVRSRV_ERROR_INVALID_PARAMS; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do \
+		{ if (unlikely(!(_expr))) { \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_INVALID_PARAMS, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s invalid in %s()", #_expr, __func__)); \
+			_err = PVRSRV_ERROR_INVALID_PARAMS; \
+			goto _go; } \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_MSG(_lvl, _msg) do \
+		{ \
+			if (_lvl == PVR_DBG_ERROR) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+			} \
+			else if (_lvl == PVR_DBG_WARNING) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+			} \
+			PVR_DPF((_lvl, ("In %s() "_msg), __func__)); \
+		} while (false)
+
+	#define PVR_LOG_VA(_lvl, _msg, ...) do \
+		{ \
+			if (_lvl == PVR_DBG_ERROR) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+			} \
+			else if (_lvl == PVR_DBG_WARNING) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+			} \
+			PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+		} while (false)
+
+	#define PVR_LOG_IF_ERROR_VA(_lvl, _rc, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				if (_lvl == PVR_DBG_ERROR) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				} \
+				else if (_lvl == PVR_DBG_WARNING) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_WARNING, _rc, HTB_FILE_NAME, __LINE__); \
+				} \
+				PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_IF_FALSE_VA(_lvl, _expr, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				if (_lvl == PVR_DBG_ERROR) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+				} \
+				else if (_lvl == PVR_DBG_WARNING) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+				} \
+				PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR_VA(_rc, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR_VA(_rc, _go, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_FALSE_VA(_expr, _rc, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("At %s: "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_FALSE_VA(_expr, _go, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE_VA(_expr, _rc, _msg, ...) do \
+		{ \
+			if (unlikely((_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("At %s: "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+#else /* defined(PVRSRV_NEED_PVR_DPF) */
+
+	#define PVR_DPF(X)  /*!< Null Implementation of PowerVR Debug Printf (does nothing) */
+
+	#define PVR_LOG_MSG(_lvl, _msg)
+	#define PVR_LOG_VA(_lvl, _msg, ...)
+	#define PVR_LOG_ERROR(_rc, _call) (void)(_rc)
+	#define PVR_LOG_IF_ERROR(_rc, _call) (void)(_rc)
+	#define PVR_WARN_IF_ERROR(_rc, _call) (void)(_rc)
+
+	#define PVR_LOG_IF_ERROR_VA(_lvl, _rc, _msg, ...) (void)(_rc)
+	#define PVR_LOG_IF_FALSE_VA(_lvl, _expr, _msg, ...) (void)(_expr)
+
+	#define PVR_LOG_RETURN_IF_NOMEM(_expr, _call) do { if (unlikely(_expr == NULL)) { return PVRSRV_ERROR_OUT_OF_MEMORY; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_NOMEM(_expr, _err, _go) do { if (unlikely(_expr == NULL)) { _err = PVRSRV_ERROR_OUT_OF_MEMORY; goto _go; } MSC_SUPPRESS_4127	} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR(_rc, _call) do { if (unlikely(_rc != PVRSRV_OK)) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_ERROR_VA(_rc, _msg, ...) do { if (unlikely(_rc != PVRSRV_OK)) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_VOID_IF_ERROR(_rc, _call) do { if (unlikely(_rc != PVRSRV_OK)) { return; } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR(_rc, _call, _go) do { if (unlikely(_rc != PVRSRV_OK)) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_ERROR_VA(_rc, _go, _msg, ...) do { if (unlikely(_rc != PVRSRV_OK)) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_WITH_ERROR(_call, _err, _rc, _go) do { _err = _rc; goto _go; MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_IF_FALSE(_expr, _msg) (void)(_expr)
+	#define PVR_LOG_RETURN_IF_FALSE(_expr, _msg, _rc) do { if (unlikely(!(_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_FALSE_VA(_expr, _rc, _msg, ...) do { if (unlikely(!(_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE(_expr, _msg, _rc) do { if (unlikely((_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_TRUE_VA(_expr, _rc, _msg, ...) do { if (unlikely((_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_FALSE(_expr, _msg) do { if (unlikely(!(_expr))) { return; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_FALSE(_expr, _msg, _go) do { if (unlikely(!(_expr))) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_FALSE_VA(_expr, _go, _msg, ...) do { if (unlikely(!(_expr))) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_IF_INVALID_PARAM(_expr, _param) do { if (unlikely(!(_expr))) { return PVRSRV_ERROR_INVALID_PARAMS; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do { if (unlikely(!(_expr))) { _err = PVRSRV_ERROR_INVALID_PARAMS; goto _go; } MSC_SUPPRESS_4127 } while (false)
+
+	#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#endif /* defined(PVRSRV_NEED_PVR_DPF) */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintf
+@Description    Output a debug message to the user, using an OS-specific
+                method, to a log or console which can be read by developers
+                Invoked from the macro PVR_DPF().
+@Input          ui32DebugLevel   The debug level of the message. This can
+                                 be used to restrict the output of debug
+                                 messages based on their severity.
+                                 If this is PVR_DBG_BUFFERED, the message
+                                 should be written into a debug circular
+                                 buffer instead of being output immediately
+                                 (useful when performance would otherwise
+                                 be adversely affected).
+                                 The debug circular buffer shall only be
+                                 output when PVRSRVDebugPrintfDumpCCB() is
+                                 called.
+@Input          pszFileName      The source file containing the code that is
+                                 generating the message
+@Input          ui32Line         The line number in the source file
+@Input          pszFormat        The formatted message string
+@Input          ...              Zero or more arguments for use by the
+                                 formatted string
+@Return         None
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVDebugPrintf(IMG_UINT32 ui32DebugLevel,
+                                               const IMG_CHAR *pszFileName,
+                                               IMG_UINT32 ui32Line,
+                                               const IMG_CHAR *pszFormat,
+                                               ...) __printf(4, 5);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintfDumpCCB
+@Description    When PVRSRVDebugPrintf() is called with the ui32DebugLevel
+                specified as DBGPRIV_BUFFERED, the debug shall be written to
+                the debug circular buffer instead of being output immediately.
+                (This could be used to obtain debug without incurring a
+                performance hit by printing it at that moment).
+                This function shall dump the contents of that debug circular
+                buffer to be output in an OS-specific method to a log or
+                console which can be read by developers.
+@Return         None
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVDebugPrintfDumpCCB(void);
+
+#if !defined(DOXYGEN)
+#define PVR_DPF_FUNC__(lvl, message, ...) PVR_DPF((lvl, "%s: " message, __func__, ##__VA_ARGS__))
+#define PVR_DPF_FUNC(x) PVR_DPF_FUNC__ x
+#endif
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_ERROR macro.
+ */
+#define PVR_RETURN_IF_ERROR(_rc) do \
+	{ if (unlikely(_rc != PVRSRV_OK)) { \
+		return _rc; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_FALSE macro.
+ */
+#define PVR_RETURN_IF_FALSE(_expr, _rc) do \
+	{ if (unlikely(!(_expr))) { \
+		return _rc; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_INVALID_PARAM macro.
+ */
+#define PVR_RETURN_IF_INVALID_PARAM(_expr) do \
+	{ if (unlikely(!(_expr))) { \
+		return PVRSRV_ERROR_INVALID_PARAMS; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_NOMEM macro.
+ */
+#define PVR_RETURN_IF_NOMEM(_expr) do \
+	{ if (unlikely(!(_expr))) { \
+		return PVRSRV_ERROR_OUT_OF_MEMORY; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_NOMEM macro.
+ */
+#define PVR_GOTO_IF_NOMEM(_expr, _err, _go) do \
+	{ if (unlikely(_expr == NULL)) { \
+		_err = PVRSRV_ERROR_OUT_OF_MEMORY; \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_INVALID_PARAM macro.
+ */
+#define PVR_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do \
+	{ if (unlikely(!(_expr))) { \
+		_err = PVRSRV_ERROR_INVALID_PARAMS; \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_FALSE macro.
+ */
+#define PVR_GOTO_IF_FALSE(_expr, _go) do \
+	{ if (unlikely(!(_expr))) { \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_ERROR macro.
+ */
+#define PVR_GOTO_IF_ERROR(_rc, _go) do \
+	{ if (unlikely(_rc != PVRSRV_OK)) { \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_WITH_ERROR macro.
+ */
+#define PVR_GOTO_WITH_ERROR(_err, _rc, _go) do \
+	{ _err = _rc; goto _go; \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/*! @cond Doxygen_Suppress */
+#if defined(PVR_DPF_FUNCTION_TRACE_ON)
+
+	#define PVR_DPF_ENTERED \
+	PVR_DPF((PVR_DBG_CALLTRACE, "|-> %s:%d entered", __func__, __LINE__))
+
+	#define PVR_DPF_ENTERED1(p1) \
+		PVR_DPF((PVR_DBG_CALLTRACE, "|-> %s:%d entered (0x%lx)", __func__, __LINE__, ((unsigned long)p1)))
+
+	#define PVR_DPF_RETURN_RC(a) \
+	do { int _r = (a); PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned %d", __func__, __LINE__, (_r))); return (_r); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_RC1(a,p1) \
+		do { int _r = (a); PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned %d (0x%lx)", __func__, __LINE__, (_r), ((unsigned long)p1))); return (_r); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_VAL(a) \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned with value", __func__, __LINE__)); return (a); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_OK \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned ok", __func__, __LINE__)); return PVRSRV_OK; MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned", __func__, __LINE__)); return; MSC_SUPPRESS_4127 } while (false)
+
+	#if !defined(DEBUG)
+	#error PVR DPF Function trace enabled in release build, rectify
+	#endif
+
+#else /* defined(PVR_DPF_FUNCTION_TRACE_ON) */
+
+	#define PVR_DPF_ENTERED
+	#define PVR_DPF_ENTERED1(p1)
+	#define PVR_DPF_RETURN_RC(a)     return (a)
+	#define PVR_DPF_RETURN_RC1(a,p1) return (a)
+	#define PVR_DPF_RETURN_VAL(a)    return (a)
+	#define PVR_DPF_RETURN_OK        return PVRSRV_OK
+	#define PVR_DPF_RETURN           return
+
+#endif /* defined(PVR_DPF_FUNCTION_TRACE_ON) */
+/*! @endcond */
+
+#if defined(__KERNEL__) || defined(DOXYGEN) || defined(__QNXNTO__)
+/*Use PVR_DPF() unless message is necessary in release build */
+#define PVR_LOG(X) PVRSRVReleasePrintf X
+
+/*************************************************************************/ /*!
+@Function       PVRSRVReleasePrintf
+@Description    Output an important message, using an OS-specific method,
+                to the Server log or console which will always be output in
+                both release and debug builds.
+                Invoked from the macro PVR_LOG(). Used in Services Server only.
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+@Return         None
+*/ /**************************************************************************/
+void IMG_CALLCONV PVRSRVReleasePrintf(const IMG_CHAR *pszFormat, ...) __printf(1, 2);
+
+ /*************************************************************************/ /*!
+ @Function       PVRSRVReleasePrintfVArgs
+ @Description    Output an important message, using an OS-specific method,
+                 to the Server log or console which will always be output in
+                 both release and debug builds.
+                 Calls to va_start and va_end should wrap this function when
+                 passing in va_list args.
+ @Input          pszFormat   The message format string
+ @Input          vaArgs      va_list arguments to print using pszFormat.
+ @Return         None
+ */ /**************************************************************************/
+void IMG_CALLCONV PVRSRVReleasePrintfVArgs(const IMG_CHAR *pszFormat, va_list vaArgs);
+#endif
+
+/* PVR_TRACE() handling */
+
+#if defined(PVRSRV_NEED_PVR_TRACE) || defined(DOXYGEN)
+
+	#define PVR_TRACE(X)	PVRSRVTrace X    /*!< PowerVR Debug Trace Macro */
+	/* Empty string implementation that is -O0 build friendly */
+	#define PVR_TRACE_EMPTY_LINE()	PVR_TRACE(("%s", ""))
+
+/*************************************************************************/ /*!
+@Function       PVRTrace
+@Description    Output a debug message to the user
+                Invoked from the macro PVR_TRACE().
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVTrace(const IMG_CHAR* pszFormat, ... )
+	__printf(1, 2);
+
+#else /* defined(PVRSRV_NEED_PVR_TRACE) */
+	/*! Null Implementation of PowerVR Debug Trace Macro (does nothing) */
+	#define PVR_TRACE(X)
+
+#endif /* defined(PVRSRV_NEED_PVR_TRACE) */
+
+
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_64BITS_TO_32BITS)
+#endif
+	INLINE static IMG_UINT32 TRUNCATE_64BITS_TO_32BITS(IMG_UINT64 uiInput)
+	{
+		IMG_UINT32 uiTruncated;
+
+		uiTruncated = (IMG_UINT32)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_64BITS_TO_SIZE_T)
+#endif
+	INLINE static size_t TRUNCATE_64BITS_TO_SIZE_T(IMG_UINT64 uiInput)
+	{
+		size_t uiTruncated;
+
+		uiTruncated = (size_t)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_SIZE_T_TO_32BITS)
+#endif
+	INLINE static IMG_UINT32 TRUNCATE_SIZE_T_TO_32BITS(size_t uiInput)
+	{
+		IMG_UINT32 uiTruncated;
+
+		uiTruncated = (IMG_UINT32)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#else /* defined(PVRSRV_NEED_PVR_ASSERT) */
+	#define TRUNCATE_64BITS_TO_32BITS(expr) ((IMG_UINT32)(expr))
+	#define TRUNCATE_64BITS_TO_SIZE_T(expr) ((size_t)(expr))
+	#define TRUNCATE_SIZE_T_TO_32BITS(expr) ((IMG_UINT32)(expr))
+#endif /* defined(PVRSRV_NEED_PVR_ASSERT) */
+
+/*! @cond Doxygen_Suppress */
+/* Macros used to trace calls */
+#if defined(DEBUG)
+	#define PVR_DBG_FILELINE , (__FILE__), (__LINE__)
+	#define PVR_DBG_FILELINE_PARAM , const IMG_CHAR *pszaFile, IMG_UINT32 ui32Line
+	#define PVR_DBG_FILELINE_ARG , pszaFile, ui32Line
+	#define PVR_DBG_FILELINE_FMT " %s:%u"
+	#define PVR_DBG_FILELINE_UNREF() do { PVR_UNREFERENCED_PARAMETER(pszaFile); \
+				PVR_UNREFERENCED_PARAMETER(ui32Line); } while (false)
+#else
+	#define PVR_DBG_FILELINE
+	#define PVR_DBG_FILELINE_PARAM
+	#define PVR_DBG_FILELINE_ARG
+	#define PVR_DBG_FILELINE_FMT
+	#define PVR_DBG_FILELINE_UNREF()
+#endif
+/*! @endcond */
+
+#if defined(__cplusplus)
+}
+#endif
+
+/*!
+    @def PVR_ASSERT
+    @brief Aborts the program if assertion fails.
+
+    The macro will be defined only when PVRSRV_NEED_PVR_ASSERT macro is
+    enabled. It's ignored otherwise.
+
+    @def PVR_DPF
+    @brief PowerVR Debug Printf logging macro used throughout the driver.
+
+    The macro allows to print logging messages to appropriate log. The
+    destination log is based on the component (user space / kernel space) and
+    operating system (Linux, Android, etc.).
+
+    The macro also supports severity levels that allow to turn on/off messages
+    based on their importance.
+
+    This macro will print messages with severity level higher that error only
+    if PVRSRV_NEED_PVR_DPF macro is defined.
+
+    @def PVR_LOG_ERROR
+    @brief Logs error.
+
+    @def PVR_LOG_IF_ERROR
+    @brief Logs error if not PVRSRV_OK.
+
+    @def PVR_WARN_IF_ERROR
+    @brief Logs warning if not PVRSRV_OK.
+
+    @def PVR_LOG_RETURN_IF_NOMEM
+    @brief Logs error if expression is NULL and returns PVRSRV_ERROR_OUT_OF_MEMORY.
+
+    @def PVR_LOG_GOTO_IF_NOMEM
+    @brief Logs error if expression is NULL and jumps to given label.
+
+    @def PVR_LOG_RETURN_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and returns the error.
+
+    @def PVR_LOG_RETURN_VOID_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and returns (used in function that return void).
+
+    @def PVR_LOG_GOTO_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and jumps to label.
+
+    @def PVR_LOG_GOTO_WITH_ERROR
+    @brief Logs error, goes to a label and sets the error code.
+
+    @def PVR_LOG_IF_FALSE
+    @brief Prints error message if expression is false.
+
+    @def PVR_LOG_RETURN_IF_FALSE
+    @brief Prints error message if expression is false and returns given error.
+
+    @def PVR_LOG_RETURN_VOID_IF_FALSE
+    @brief Prints error message if expression is false and returns (used in function that return void).
+
+    @def PVR_LOG_GOTO_IF_FALSE
+    @brief Prints error message if expression is false and jumps to label.
+
+    @def PVR_LOG_RETURN_IF_TRUE
+    @brief Prints error message if expression is true and returns given error.
+
+    @def PVR_LOG_RETURN_IF_INVALID_PARAM
+    @brief Prints error message if expression is false and returns PVRSRV_ERROR_INVALID_PARAMS.
+
+    @def PVR_LOG_GOTO_IF_INVALID_PARAM
+    @brief Prints error message if expression is false and jumps to label.
+
+    @def PVR_RETURN_IF_ERROR
+    @brief Returns passed error code if it's different than PVRSRV_OK;
+
+    @def PVR_RETURN_IF_FALSE
+    @brief Returns passed error code if expression is false.
+
+    @def PVR_RETURN_IF_INVALID_PARAM
+    @brief Returns PVRSRV_ERROR_INVALID_PARAMS if expression is false.
+
+    @def PVR_RETURN_IF_NOMEM
+    @brief Returns PVRSRV_ERROR_OUT_OF_MEMORY if expression is NULL.
+
+    @def PVR_GOTO_IF_NOMEM
+    @brief Goes to a label if expression is NULL.
+
+    @def PVR_GOTO_IF_INVALID_PARAM
+    @brief Goes to a label if expression is false.
+
+    @def PVR_GOTO_IF_FALSE
+    @brief Goes to a label if expression is false.
+
+    @def PVR_GOTO_IF_ERROR
+    @brief Goes to a label if the error code is different than PVRSRV_OK;
+
+    @def PVR_GOTO_WITH_ERROR
+    @brief Goes to a label and sets the error code.
+
+    @def PVR_LOG
+    @brief Prints message to a log unconditionally.
+
+    This macro will print messages only if PVRSRV_NEED_PVR_LOG macro is defined.
+    @def PVR_LOG_MSG
+    @brief Prints message to a log with the given log-level.
+
+    @def PVR_LOG_VA
+    @brief Prints message with var-args to a log with the given log-level.
+
+    @def PVR_LOG_IF_ERROR_VA
+    @brief Prints message with var-args to a log if the error code is different than PVRSRV_OK.
+
+    @def PVR_LOG_IF_FALSE_VA
+    @brief Prints message with var-args if expression is false.
+
+    @def PVR_LOG_RETURN_IF_ERROR_VA
+    @brief Prints message with var-args to a log and returns the error code.
+
+    @def PVR_LOG_GOTO_IF_ERROR_VA
+    @brief Prints message with var-args to a log and goes to a label if the error code is different than PVRSRV_OK.
+
+    @def PVR_LOG_RETURN_IF_FALSE_VA
+    @brief Logs the error message with var-args if the expression is false and returns the error code.
+
+    @def PVR_LOG_GOTO_IF_FALSE_VA
+    @brief Logs the error message with var-args and goes to a label if the expression is false.
+
+    @def PVR_LOG_RETURN_IF_TRUE_VA
+    @brief Logs the error message with var-args if the expression is true and returns the error code.
+
+    @def PVR_TRACE_EMPTY_LINE
+    @brief Prints empty line to a log (PVRSRV_NEED_PVR_LOG must be defined).
+
+    @def TRUNCATE_64BITS_TO_32BITS
+    @brief Truncates 64 bit value to 32 bit value (with possible precision loss).
+
+    @def TRUNCATE_64BITS_TO_SIZE_T
+    @brief Truncates 64 bit value to size_t value (with possible precision loss).
+
+    @def TRUNCATE_SIZE_T_TO_32BITS
+    @brief Truncates size_t value to 32 bit value (with possible precision loss).
+ */
+
+#endif /* PVR_DEBUG_H */
+
+/******************************************************************************
+ End of file (pvr_debug.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_fd_sync_kernel.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_fd_sync_kernel.h
new file mode 100644
index 000000000000..3645e29079b1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_fd_sync_kernel.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File           pvr_fd_sync_kernel.h
+@Title          Kernel/userspace interface definitions to use the kernel sync
+                driver
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef _PVR_FD_SYNC_KERNEL_H_
+#define _PVR_FD_SYNC_KERNEL_H_
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+#include "pvr_drm.h"
+
+#define PVR_SYNC_MAX_QUERY_FENCE_POINTS 14
+
+struct pvr_sync_pt_info {
+	/* Output */
+	__u32 id;
+	__u32 ui32FWAddr;
+	__u32 ui32CurrOp;
+	__u32 ui32NextOp;
+	__u32 ui32TlTaken;
+} __attribute__((packed, aligned(8)));
+
+#endif /* _PVR_FD_SYNC_KERNEL_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_intrinsics.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_intrinsics.h
new file mode 100644
index 000000000000..410a2f5a50b5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_intrinsics.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File
+@Title          Intrinsics definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_INTRINSICS_H
+#define PVR_INTRINSICS_H
+
+/* PVR_CTZLL:
+ * Count the number of trailing zeroes in a long long integer
+ */
+
+#if defined(__GNUC__)
+#if defined(__x86_64__)
+
+	#define PVR_CTZLL __builtin_ctzll
+#endif
+#endif
+
+/* PVR_CLZLL:
+ * Count the number of leading zeroes in a long long integer
+ */
+
+#if defined(__GNUC__)
+#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || \
+					defined(__arm__) || defined(__mips)
+
+#define PVR_CLZLL __builtin_clzll
+
+#endif
+#endif
+
+#endif /* PVR_INTRINSICS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrmodule.h b/drivers/gpu/drm/img/img-volcanic/include/pvrmodule.h
new file mode 100644
index 000000000000..267c7b687487
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrmodule.h
@@ -0,0 +1,48 @@
+/*************************************************************************/ /*!
+@Title          Module Author and License.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef	_PVRMODULE_H_
+#define	_PVRMODULE_H_
+
+MODULE_AUTHOR("Imagination Technologies Ltd. <gpl-support@imgtec.com>");
+MODULE_LICENSE("Dual MIT/GPL");
+
+#endif	/* _PVRMODULE_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_device_types.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_device_types.h
new file mode 100644
index 000000000000..20e0d5ce3ed5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_device_types.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR device type definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVRSRV_DEVICE_TYPES_H)
+#define PVRSRV_DEVICE_TYPES_H
+
+#include "img_types.h"
+
+#if !defined(PVRSRV_MAX_DEVICES)
+#error "PVRSRV_MAX_DEVICES must be defined in the DDK build environment"
+#endif	/* !defined(PVRSRV_NUM_DEVICES) */
+#define PVRSRV_HOST_DEVICE_ID	255U /*!< Device ID used for host (non-GPU) device. */
+
+static_assert(PVRSRV_MAX_DEVICES < PVRSRV_HOST_DEVICE_ID, "Invalid host device ID.");
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#endif /* PVRSRV_DEVICE_TYPES_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_devvar.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_devvar.h
new file mode 100644
index 000000000000..a8c64e309fda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_devvar.h
@@ -0,0 +1,291 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Device Variable interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for device variables
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_DEVVAR_H
+#define PVRSRV_DEVVAR_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define DEVVAR_MAX_NAME_LEN 32
+
+typedef struct SYNC_PRIM_CONTEXT_TAG *PDEVVARCTX;
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_TAG *PDEVVAR;
+
+typedef struct PVRSRV_DEV_VAR_UPDATE_TAG
+{
+	PDEVVAR					psDevVar;			/*!< Pointer to the dev var */
+	IMG_UINT32				ui32UpdateValue;	/*!< the update value */
+} PVRSRV_DEV_VAR_UPDATE;
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarContextCreate
+
+@Description    Create a new device variable context
+
+@Input          psDevConnection         Device to create the device
+                                        variable context on
+
+@Output         phDevVarContext         Handle to the created device
+                                        variable context
+
+@Return         PVRSRV_OK if the device variable context was successfully
+                created
+*/
+/*****************************************************************************/
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVDevVarContextCreate(const PVRSRV_DEV_CONNECTION *psDevConnection,
+                          PDEVVARCTX                  *phDevVarContext);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarContextDestroy
+
+@Description    Destroy a device variable context
+
+@Input          hDevVarContext          Handle to the device variable
+                                        context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarContextDestroy(PDEVVARCTX hDevVarContext);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarAlloc
+
+@Description    Allocate a new device variable on the specified device
+                variable context. The device variable's value is initialised
+                with the value passed in ui32InitialValue.
+
+@Input          hDevVarContext          Handle to the device variable
+                                        context
+@Input          ui32InitialValue        Value to initially assign to the
+                                        new variable
+@Input          pszDevVarName           Name assigned to the device variable
+                                        (for debug purposes)
+
+@Output         ppsDevVar               Created device variable
+
+@Return         PVRSRV_OK if the device variable was successfully created
+*/
+/*****************************************************************************/
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVDevVarAllocI(PDEVVARCTX hDevVarContext,
+                   PDEVVAR *ppsDevVar,
+                   IMG_UINT32 ui32InitialValue,
+                   const IMG_CHAR *pszDevVarName
+                   PVR_DBG_FILELINE_PARAM);
+#define PVRSRVDevVarAlloc(hDevVarContext, ppsDevVar, ui32InitialValue, pszDevVarName) \
+	PVRSRVDevVarAllocI( (hDevVarContext), (ppsDevVar), (ui32InitialValue), (pszDevVarName) \
+	                    PVR_DBG_FILELINE )
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarFree
+
+@Description    Free a device variable
+
+@Input          psDevVar                The device variable to free
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarFree(PDEVVAR psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarSet
+
+@Description    Set the device variable to a value
+
+@Input          psDevVar                The device variable to set
+
+@Input          ui32Value               Value to set it to
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarSet(PDEVVAR		psDevVar,
+                IMG_UINT32	ui32Value);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarGet
+
+@Description    Get the current value of the device variable
+
+@Input          psDevVar                The device variable to get the
+                                        value of
+
+@Return         Value of the variable
+*/
+/*****************************************************************************/
+IMG_EXPORT IMG_UINT32
+PVRSRVDevVarGet(PDEVVAR		psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarGetFirmwareAddr
+
+@Description    Returns the address of the associated firmware value for a
+                specified device integer (not exposed to client)
+
+@Input          psDevVar               The device variable to resolve
+
+@Return         The firmware address of the device variable
+*/
+/*****************************************************************************/
+IMG_EXPORT IMG_UINT32
+PVRSRVDevVarGetFirmwareAddr(PDEVVAR	psDevVar);
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDump
+
+@Description    PDump the current value of the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDump(PDEVVAR psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDumpPol
+
+@Description    Do a PDump poll of the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Input          ui32Value               Value to Poll for
+
+@Input          ui32Mask                PDump mask operator
+
+@Input          ui32PDumpFlags          PDump flags
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDumpPol(PDEVVAR psDevVar,
+                     IMG_UINT32 ui32Value,
+                     IMG_UINT32 ui32Mask,
+                     PDUMP_POLL_OPERATOR eOperator,
+                     IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDumpCBP
+
+@Description    Do a PDump CB poll using the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Input          uiWriteOffset           Current write offset of buffer
+
+@Input          uiPacketSize            Size of the packet to write into CB
+
+@Input          uiBufferSize            Size of the CB
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDumpCBP(PDEVVAR psDevVar,
+                     IMG_UINT64 uiWriteOffset,
+                     IMG_UINT64 uiPacketSize,
+                     IMG_UINT64 uiBufferSize);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDump)
+#endif
+static INLINE void
+PVRSRVDevVarPDump(PDEVVAR psDevVar)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevVar);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDumpPol)
+#endif
+static INLINE void
+PVRSRVDevVarPDumpPol(PDEVVAR psDevVar,
+                     IMG_UINT32 ui32Value,
+                     IMG_UINT32 ui32Mask,
+                     PDUMP_POLL_OPERATOR eOperator,
+                     IMG_UINT32 ui32PDumpFlags)
+{
+	 PVR_UNREFERENCED_PARAMETER(psDevVar);
+	 PVR_UNREFERENCED_PARAMETER(ui32Value);
+	 PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	 PVR_UNREFERENCED_PARAMETER(eOperator);
+	 PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDumpCBP)
+#endif
+static INLINE void
+PVRSRVDevVarPDumpCBP(PDEVVAR psDevVar,
+                     IMG_UINT64 uiWriteOffset,
+                     IMG_UINT64 uiPacketSize,
+                     IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevVar);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+}
+#endif /* PDUMP */
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* PVRSRV_DEVVAR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_error.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_error.h
new file mode 100644
index 000000000000..120c42d87155
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_error.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_error.h
+@Title          services error enumerant
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines error codes used by any/all services modules
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVRSRV_ERROR_H)
+#define PVRSRV_ERROR_H
+
+/*!
+ *****************************************************************************
+ * Error values
+ *****************************************************************************/
+typedef enum PVRSRV_ERROR_TAG
+{
+	PVRSRV_OK,
+
+#define PVRE(x) x,
+#include "pvrsrv_errors.h"
+#undef PVRE
+
+	PVRSRV_ERROR_FORCE_I32 = 0x7fffffff
+
+} PVRSRV_ERROR;
+
+/*!
+ * @Function PVRSRVIsRetryError
+ * @Description Checks if error code is one of the errors that require retry
+ *              from the caller.
+ * @Input eError Error code.
+ * @Return IMG_TRUE if eError is one of the error codes that require the caller
+ *         to retry.
+ */
+#define PVRSRVIsRetryError(eError) \
+	(((eError == PVRSRV_ERROR_RETRY) || (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)) ? \
+	 IMG_TRUE : IMG_FALSE)
+
+#endif /* !defined(PVRSRV_ERROR_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_errors.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_errors.h
new file mode 100644
index 000000000000..4c88e675d0d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_errors.h
@@ -0,0 +1,429 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_errors.h
+@Title          services error codes
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines error codes used by any/all services modules
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Don't add include guards to this file! */
+
+PVRE(PVRSRV_ERROR_OUT_OF_MEMORY)
+PVRE(PVRSRV_ERROR_TOO_FEW_BUFFERS)
+PVRE(PVRSRV_ERROR_INVALID_PARAMS)
+PVRE(PVRSRV_ERROR_INIT_FAILURE)
+PVRE(PVRSRV_ERROR_CANT_REGISTER_CALLBACK)
+PVRE(PVRSRV_ERROR_INVALID_DEVICE)
+PVRE(PVRSRV_ERROR_NOT_OWNER)
+PVRE(PVRSRV_ERROR_BAD_MAPPING)
+PVRE(PVRSRV_ERROR_TIMEOUT)
+PVRE(PVRSRV_ERROR_NOT_IMPLEMENTED)
+PVRE(PVRSRV_ERROR_FLIP_CHAIN_EXISTS)
+PVRE(PVRSRV_ERROR_INVALID_SWAPINTERVAL)
+PVRE(PVRSRV_ERROR_SCENE_INVALID)
+PVRE(PVRSRV_ERROR_STREAM_ERROR)
+PVRE(PVRSRV_ERROR_FAILED_DEPENDENCIES)
+PVRE(PVRSRV_ERROR_CMD_NOT_PROCESSED)
+PVRE(PVRSRV_ERROR_CMD_TOO_BIG)
+PVRE(PVRSRV_ERROR_DEVICE_REGISTER_FAILED)
+PVRE(PVRSRV_ERROR_TOOMANYBUFFERS)
+PVRE(PVRSRV_ERROR_NOT_SUPPORTED)
+PVRE(PVRSRV_ERROR_PROCESSING_BLOCKED)
+PVRE(PVRSRV_ERROR_CANNOT_FLUSH_QUEUE)
+PVRE(PVRSRV_ERROR_CANNOT_GET_QUEUE_SPACE)
+PVRE(PVRSRV_ERROR_CANNOT_GET_RENDERDETAILS)
+PVRE(PVRSRV_ERROR_RETRY)
+PVRE(PVRSRV_ERROR_DDK_VERSION_MISMATCH)
+PVRE(PVRSRV_ERROR_DDK_BUILD_MISMATCH)
+PVRE(PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH)
+PVRE(PVRSRV_ERROR_BVNC_MISMATCH)
+PVRE(PVRSRV_ERROR_FWPROCESSOR_MISMATCH)
+PVRE(PVRSRV_ERROR_UPLOAD_TOO_BIG)
+PVRE(PVRSRV_ERROR_INVALID_FLAGS)
+PVRE(PVRSRV_ERROR_FAILED_TO_REGISTER_PROCESS)
+PVRE(PVRSRV_ERROR_UNABLE_TO_LOAD_LIBRARY)
+PVRE(PVRSRV_ERROR_UNABLE_GET_FUNC_ADDR)
+PVRE(PVRSRV_ERROR_UNLOAD_LIBRARY_FAILED)
+PVRE(PVRSRV_ERROR_BRIDGE_CALL_FAILED)
+PVRE(PVRSRV_ERROR_IOCTL_CALL_FAILED)
+PVRE(PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR)
+PVRE(PVRSRV_ERROR_MMU_CONFIG_IS_WRONG)
+PVRE(PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_CREATE_HEAP)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_UNMAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE)
+PVRE(PVRSRV_ERROR_MMU_LIVE_ALLOCATIONS_IN_HEAP)
+PVRE(PVRSRV_ERROR_MMU_RESERVATION_NOT_INSIDE_HEAP)
+PVRE(PVRSRV_ERROR_PMR_NEW_MEMORY)
+PVRE(PVRSRV_ERROR_PMR_STILL_REFERENCED)
+PVRE(PVRSRV_ERROR_PMR_CLIENT_NOT_TRUSTED)
+PVRE(PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES)
+PVRE(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY)
+PVRE(PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES)
+PVRE(PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE)
+PVRE(PVRSRV_ERROR_PMR_NOT_PERMITTED)
+PVRE(PVRSRV_ERROR_PMR_ALREADY_OCCUPIED)
+PVRE(PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR)
+PVRE(PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR)
+PVRE(PVRSRV_ERROR_PMR_WRONG_PMR_TYPE)
+PVRE(PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS)
+PVRE(PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE)
+PVRE(PVRSRV_ERROR_PMR_BAD_CHUNK_SIZE)
+PVRE(PVRSRV_ERROR_PMR_MAPPINGTABLE_MISMATCH)
+PVRE(PVRSRV_ERROR_PMR_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING)
+PVRE(PVRSRV_ERROR_PMR_EMPTY)
+PVRE(PVRSRV_ERROR_PMR_NO_CPU_MAP_FOUND)
+PVRE(PVRSRV_ERROR_PMR_CPU_PAGE_UNMAP_FAILED)
+PVRE(PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED)
+PVRE(PVRSRV_ERROR_PMR_PAGE_POISONING_FAILED)
+PVRE(PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY)
+PVRE(PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_PMR_TOO_LARGE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP)
+PVRE(PVRSRV_ERROR_DEVICEMEM_BAD_IMPORT_SIZE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX)
+PVRE(PVRSRV_ERROR_DEVICEMEM_MAP_FAILED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_NON_ZERO_USAGE_COUNT)
+PVRE(PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA)
+PVRE(PVRSRV_ERROR_DEVICEMEM_OUT_OF_DEVICE_VM)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_NO_MAPPING)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_LMA_HEAP)
+PVRE(PVRSRV_ERROR_INVALID_MMU_TYPE)
+PVRE(PVRSRV_ERROR_BUFFER_DEVICE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_BUFFER_DEVICE_ALREADY_PRESENT)
+PVRE(PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_PCI_CALL_FAILED)
+PVRE(PVRSRV_ERROR_PCI_REGION_TOO_SMALL)
+PVRE(PVRSRV_ERROR_PCI_REGION_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH)
+PVRE(PVRSRV_ERROR_REGISTER_BASE_NOT_SET)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_USER_MEM)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_VP_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_SHARED_PBDESC)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_KERNELVIRTUAL)
+PVRE(PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_VIRT_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES)
+PVRE(PVRSRV_ERROR_FAILED_TO_FREE_PAGES)
+PVRE(PVRSRV_ERROR_FAILED_TO_COPY_PAGES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_LOCK_PAGES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNLOCK_PAGES)
+PVRE(PVRSRV_ERROR_STILL_MAPPED)
+PVRE(PVRSRV_ERROR_MAPPING_NOT_FOUND)
+PVRE(PVRSRV_ERROR_PHYS_ADDRESS_EXCEEDS_32BIT)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_INVALID_SEGMENT_BLOCK)
+PVRE(PVRSRV_ERROR_INVALID_GFXDEVDEVDATA)
+PVRE(PVRSRV_ERROR_INVALID_DEVINFO)
+PVRE(PVRSRV_ERROR_INVALID_MEMINFO)
+PVRE(PVRSRV_ERROR_INVALID_MISCINFO)
+PVRE(PVRSRV_ERROR_UNKNOWN_IOCTL)
+PVRE(PVRSRV_ERROR_INVALID_CONTEXT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT)
+PVRE(PVRSRV_ERROR_INVALID_HEAP)
+PVRE(PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE)
+PVRE(PVRSRV_ERROR_INVALID_KERNELINFO)
+PVRE(PVRSRV_ERROR_UNKNOWN_POWER_STATE)
+PVRE(PVRSRV_ERROR_INVALID_HANDLE_TYPE)
+PVRE(PVRSRV_ERROR_INVALID_WRAP_TYPE)
+PVRE(PVRSRV_ERROR_INVALID_PHYS_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_CPU_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_HEAPINFO)
+PVRE(PVRSRV_ERROR_INVALID_PERPROC)
+PVRE(PVRSRV_ERROR_FAILED_TO_RETRIEVE_HEAPINFO)
+PVRE(PVRSRV_ERROR_INVALID_MAP_REQUEST)
+PVRE(PVRSRV_ERROR_INVALID_UNMAP_REQUEST)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FIND_MAPPING_HEAP)
+PVRE(PVRSRV_ERROR_MAPPING_STILL_IN_USE)
+PVRE(PVRSRV_ERROR_EXCEEDED_HW_LIMITS)
+PVRE(PVRSRV_ERROR_NO_STAGING_BUFFER_ALLOCATED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_PERPROC_AREA)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CLOSE_THREAD)
+PVRE(PVRSRV_ERROR_THREAD_READ_ERROR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNINSTALL_ISR)
+PVRE(PVRSRV_ERROR_ISR_ALREADY_INSTALLED)
+PVRE(PVRSRV_ERROR_ISR_NOT_INSTALLED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INITIALISE_INTERRUPT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RETRIEVE_INFO)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DO_BACKWARDS_BLIT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CLOSE_SERVICES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_CONTEXT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_RESOURCE)
+PVRE(PVRSRV_ERROR_INVALID_CCB_COMMAND)
+PVRE(PVRSRV_ERROR_KERNEL_CCB_FULL)
+PVRE(PVRSRV_ERROR_FLIP_FAILED)
+PVRE(PVRSRV_ERROR_UNBLANK_DISPLAY_FAILED)
+PVRE(PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE)
+PVRE(PVRSRV_ERROR_TIMEOUT_WAITING_FOR_CLIENT_CCB)
+PVRE(PVRSRV_ERROR_CREATE_RENDER_CONTEXT_FAILED)
+PVRE(PVRSRV_ERROR_UNKNOWN_PRIMARY_FRAG)
+PVRE(PVRSRV_ERROR_UNEXPECTED_SECONDARY_FRAG)
+PVRE(PVRSRV_ERROR_UNEXPECTED_PRIMARY_FRAG)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSERT_FENCE_ID)
+PVRE(PVRSRV_ERROR_BLIT_SETUP_FAILED)
+PVRE(PVRSRV_ERROR_SUBMIT_NEEDED)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_AVAILABLE)
+PVRE(PVRSRV_ERROR_PDUMP_BUFFER_FULL)
+PVRE(PVRSRV_ERROR_PDUMP_BUF_OVERFLOW)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_ACTIVE)
+PVRE(PVRSRV_ERROR_INCOMPLETE_LINE_OVERLAPS_PAGES)
+PVRE(PVRSRV_ERROR_MUTEX_DESTROY_FAILED)
+PVRE(PVRSRV_ERROR_MUTEX_INTERRUPTIBLE_ERROR)
+PVRE(PVRSRV_ERROR_INSUFFICIENT_SPACE_FOR_COMMAND)
+PVRE(PVRSRV_ERROR_PROCESS_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_PROCESS_NOT_FOUND)
+PVRE(PVRSRV_ERROR_SRV_CONNECT_FAILED)
+PVRE(PVRSRV_ERROR_SRV_DISCONNECT_FAILED)
+PVRE(PVRSRV_ERROR_DEINT_PHASE_FAILED)
+PVRE(PVRSRV_ERROR_INIT2_PHASE_FAILED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FIND_RESOURCE)
+PVRE(PVRSRV_ERROR_NO_DC_DEVICES_FOUND)
+PVRE(PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE)
+PVRE(PVRSRV_ERROR_DC_INVALID_MAXDEPTH)
+PVRE(PVRSRV_ERROR_UNABLE_TO_OPEN_DC_DEVICE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNREGISTER_DEVICE)
+PVRE(PVRSRV_ERROR_NO_DEVICEDATA_FOUND)
+PVRE(PVRSRV_ERROR_NO_DEVICENODE_FOUND)
+PVRE(PVRSRV_ERROR_NO_CLIENTNODE_FOUND)
+PVRE(PVRSRV_ERROR_FAILED_TO_PROCESS_QUEUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INIT_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SCHEDULE_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_KILL_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_TIMER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DISABLE_TIMER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REMOVE_TIMER)
+PVRE(PVRSRV_ERROR_UNKNOWN_PIXEL_FORMAT)
+PVRE(PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_ALLOCATED)
+PVRE(PVRSRV_ERROR_HANDLE_TYPE_MISMATCH)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_SHAREABLE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_INVALID_SUBHANDLE)
+PVRE(PVRSRV_ERROR_HANDLE_BATCH_IN_USE)
+PVRE(PVRSRV_ERROR_HANDLE_BATCH_COMMIT_FAILURE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_HASH_TABLE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REMOVE_HASH_VALUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSERT_HASH_VALUE)
+PVRE(PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_BACKING_STORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_BM_HEAP)
+PVRE(PVRSRV_ERROR_UNKNOWN_INIT_SERVER_STATE)
+PVRE(PVRSRV_ERROR_NO_FREE_DEVICEIDS_AVAILABLE)
+PVRE(PVRSRV_ERROR_INVALID_DEVICEID)
+PVRE(PVRSRV_ERROR_DEVICEID_NOT_FOUND)
+PVRE(PVRSRV_ERROR_MEMORY_TEST_FAILED)
+PVRE(PVRSRV_ERROR_CPUPADDR_TEST_FAILED)
+PVRE(PVRSRV_ERROR_COPY_TEST_FAILED)
+PVRE(PVRSRV_ERROR_SEMAPHORE_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RELEASE_CLOCK)
+PVRE(PVRSRV_ERROR_CLOCK_REQUEST_FAILED)
+PVRE(PVRSRV_ERROR_DISABLE_CLOCK_FAILURE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SET_CLOCK_RATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ROUND_CLOCK_RATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_PARENT_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_SYSTEM_CLOCK)
+PVRE(PVRSRV_ERROR_UNKNOWN_SGL_ERROR)
+PVRE(PVRSRV_ERROR_SYSTEM_POWER_CHANGE_FAILURE)
+PVRE(PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE)
+PVRE(PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+PVRE(PVRSRV_ERROR_BAD_SYNC_STATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_MMUCONTEXT_ID)
+PVRE(PVRSRV_ERROR_PARAMETER_BUFFER_INVALID_ALIGNMENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ACQUIRE_CONNECTION)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RELEASE_CONNECTION)
+PVRE(PVRSRV_ERROR_PHYSHEAP_ID_IN_USE)
+PVRE(PVRSRV_ERROR_PHYSHEAP_ID_INVALID)
+PVRE(PVRSRV_ERROR_PHYSHEAP_CONFIG)
+PVRE(PVRSRV_ERROR_HP_REQUEST_TOO_LONG)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_PRIM)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_PRIM_OP)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_CONTEXT)
+PVRE(PVRSRV_ERROR_BP_NOT_SET)
+PVRE(PVRSRV_ERROR_BP_ALREADY_SET)
+PVRE(PVRSRV_ERROR_FEATURE_DISABLED)
+PVRE(PVRSRV_ERROR_REG_CONFIG_ENABLED)
+PVRE(PVRSRV_ERROR_REG_CONFIG_FULL)
+PVRE(PVRSRV_ERROR_REG_CONFIG_INVALID_TYPE)
+PVRE(PVRSRV_ERROR_MEMORY_ACCESS)
+PVRE(PVRSRV_ERROR_NO_SYSTEM_BUFFER)
+PVRE(PVRSRV_ERROR_DC_INVALID_CONFIG)
+PVRE(PVRSRV_ERROR_DC_INVALID_CROP_RECT)
+PVRE(PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT)
+PVRE(PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS)
+PVRE(PVRSRV_ERROR_DC_INVALID_TRANSFORM)
+PVRE(PVRSRV_ERROR_DC_INVALID_SCALE)
+PVRE(PVRSRV_ERROR_DC_INVALID_CUSTOM)
+PVRE(PVRSRV_ERROR_DC_TOO_MANY_PIPES)
+PVRE(PVRSRV_ERROR_DC_INVALID_PLANE_ALPHA)
+PVRE(PVRSRV_ERROR_NOT_READY)
+PVRE(PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_DIMS)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ADD_TIMER)
+PVRE(PVRSRV_ERROR_NOT_FOUND)
+PVRE(PVRSRV_ERROR_ALREADY_OPEN)
+PVRE(PVRSRV_ERROR_STREAM_MISUSE)
+PVRE(PVRSRV_ERROR_STREAM_FULL)
+PVRE(PVRSRV_ERROR_STREAM_READLIMIT_REACHED)
+PVRE(PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE)
+PVRE(PVRSRV_ERROR_PHYSMEM_NOT_ALLOCATED)
+PVRE(PVRSRV_ERROR_PBSIZE_ALREADY_MAX)
+PVRE(PVRSRV_ERROR_PBSIZE_ALREADY_MIN)
+PVRE(PVRSRV_ERROR_INVALID_PB_CONFIG)
+PVRE(PVRSRV_ERROR_META_THREAD0_NOT_ENABLED)
+PVRE(PVRSRV_ERROR_NOT_AUTHENTICATED)
+PVRE(PVRSRV_ERROR_REQUEST_TDFWMEM_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_INIT_TDFWMEM_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_REQUEST_TDSECUREBUF_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_INIT_TDSECUREBUF_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_MUTEX_ALREADY_CREATED)
+PVRE(PVRSRV_ERROR_DBGTABLE_ALREADY_REGISTERED)
+PVRE(PVRSRV_ERROR_ALREADY_EXISTS)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SEND_PULSE)
+PVRE(PVRSRV_ERROR_TASK_FAILED)
+PVRE(PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+PVRE(PVRSRV_ERROR_INVALID_GPU_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_OFFSET)
+PVRE(PVRSRV_ERROR_CCCB_STALLED)
+PVRE(PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_NOT_ENABLED)
+PVRE(PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL)
+PVRE(PVRSRV_ERROR_FW_IMAGE_MISMATCH)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+PVRE(PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL)
+PVRE(PVRSRV_ERROR_RPM_PBSIZE_ALREADY_MAX)
+PVRE(PVRSRV_ERROR_NONZERO_REFCOUNT)
+PVRE(PVRSRV_ERROR_SETAFFINITY_FAILED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_COMPILE_PDS)
+PVRE(PVRSRV_ERROR_INTERNAL_ERROR)
+PVRE(PVRSRV_ERROR_BRIDGE_EFAULT)
+PVRE(PVRSRV_ERROR_BRIDGE_EINVAL)
+PVRE(PVRSRV_ERROR_BRIDGE_ENOMEM)
+PVRE(PVRSRV_ERROR_BRIDGE_ERANGE)
+PVRE(PVRSRV_ERROR_BRIDGE_EPERM)
+PVRE(PVRSRV_ERROR_BRIDGE_ENOTTY)
+PVRE(PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)
+PVRE(PVRSRV_ERROR_PROBE_DEFER)
+PVRE(PVRSRV_ERROR_INVALID_ALIGNMENT)
+PVRE(PVRSRV_ERROR_CLOSE_FAILED)
+PVRE(PVRSRV_ERROR_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_CONVERSION_FAILED)
+PVRE(PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+PVRE(PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL)
+PVRE(PVRSRV_ERROR_RA_OUT_OF_RESOURCE)
+PVRE(PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS)
+PVRE(PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED)
+PVRE(PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED)
+PVRE(PVRSRV_ERROR_RA_FREE_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_OBJECT_STILL_REFERENCED)
+PVRE(PVRSRV_ERROR_BVNC_UNSUPPORTED)
+PVRE(PVRSRV_ERROR_INVALID_BVNC_PARAMS)
+PVRE(PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT)
+PVRE(PVRSRV_ERROR_PID_ALREADY_REGISTERED)
+PVRE(PVRSRV_ERROR_PID_NOT_REGISTERED)
+PVRE(PVRSRV_ERROR_SIGNAL_FAILED)
+PVRE(PVRSRV_ERROR_INVALID_NOTIF_STREAM)
+PVRE(PVRSRV_ERROR_INVALID_SPU_MASK)
+PVRE(PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED)
+PVRE(PVRSRV_ERROR_INVALID_PVZ_CONFIG)
+PVRE(PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED)
+PVRE(PVRSRV_ERROR_NOT_SW_TIMELINE)
+PVRE(PVRSRV_ERROR_SW_TIMELINE_AT_LATEST_POINT)
+PVRE(PVRSRV_ERROR_INVALID_PVZ_OSID)
+PVRE(PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE)
+PVRE(PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG)
+PVRE(PVRSRV_ERROR_INTERRUPTED)
+PVRE(PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+PVRE(PVRSRV_ERROR_PDUMP_INVALID_BLOCKLEN)
+PVRE(PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF)
+PVRE(PVRSRV_ERROR_MULTIPLE_SECURITY_PDUMPS)
+PVRE(PVRSRV_ERROR_BAD_PARAM_SIZE)
+PVRE(PVRSRV_ERROR_INVALID_REQUEST)
+PVRE(PVRSRV_ERROR_FAILED_TO_ACQUIRE_PAGES)
+PVRE(PVRSRV_ERROR_TEST_FAILED)
+PVRE(PVRSRV_ERROR_SYNC_PRIM_OP_NOT_SUPPORTED)
+PVRE(PVRSRV_ERROR_FAILED_TO_GET_VIRT_ADDR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FREE_RESOURCE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_SEMAPHORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_SEMAPHORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_SEMAPHORE)
+PVRE(PVRSRV_ERROR_TOO_MANY_SYNCS)
+PVRE(PVRSRV_ERROR_ION_NO_CLIENT)
+PVRE(PVRSRV_ERROR_ION_FAILED_TO_ALLOC)
+PVRE(PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+PVRE(PVRSRV_ERROR_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_OUT_OF_APP_POOL_MEMORY)
+PVRE(PVRSRV_ERROR_REFCOUNT_OVERFLOW)
+PVRE(PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+PVRE(PVRSRV_ERROR_UNEXPECTED_TRUE_EXPR)
+PVRE(PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR)
+PVRE(PVRSRV_ERROR_KERNEL_CCB_OFFSET)
+PVRE(PVRSRV_ERROR_PMB_NOT_PAGE_MULTIPLE)
+PVRE(PVRSRV_ERROR_ATOMIC_OVERFLOW)
+PVRE(PVRSRV_ERROR_ATOMIC_UNDERFLOW)
+PVRE(PVRSRV_ERROR_DMABUF_LOCK)
+PVRE(PVRSRV_ERROR_DMABUF_ATTACH)
+PVRE(PVRSRV_ERROR_DMABUF_ATTACHMENT_MAPPING)
+PVRE(PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED)
+
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memalloc_physheap.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memalloc_physheap.h
new file mode 100644
index 000000000000..3c17d8b4495d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memalloc_physheap.h
@@ -0,0 +1,212 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_memalloc_physheap.h
+@Title          Services Phys Heap types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Used in creating and allocating from Physical Heaps.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVRSRV_MEMALLOC_PHYSHEAP_H
+#define PVRSRV_MEMALLOC_PHYSHEAP_H
+
+#include "img_defs.h"
+
+/*
+ * These IDs are replicated in the Device Memory allocation flags to allow
+ * allocations to be made in terms of their locality/use to ensure the correct
+ * physical heap is accessed for the given system/platform configuration.
+ * A system Phys Heap Config is linked to one or more Phys Heaps. When a heap
+ * is not present in the system configuration the allocation will fallback to
+ * the default GPU_LOCAL physical heap which all systems must define.
+ * See PVRSRV_MEMALLOCFLAGS_*_MAPPABLE_MASK.
+ *
+ * NOTE: Enum order important, table in physheap.c must change if order changed.
+ */
+#define PHYS_HEAP_LIST                                                                                                                           \
+	X(DEFAULT)       /* Client: default phys heap for device memory allocations */                                                               \
+	X(CPU_LOCAL)     /* Client: used for buffers with more CPU access than GPU */                                                                \
+	X(GPU_LOCAL)     /* Client: used for buffers with more GPU access than CPU */                                                                \
+	X(GPU_PRIVATE)   /* Client: used for buffers that only required GPU read/write access, not visible to the CPU. */                            \
+	X(FW_MAIN)       /* Internal: runtime data, e.g. CCBs, sync objects */                                                                       \
+	X(EXTERNAL)      /* Internal: used by some PMR import/export factories where the physical memory heap is not managed by the pvrsrv driver */ \
+	X(GPU_COHERENT)  /* Internal: used for a cache coherent region */                                                                            \
+	X(GPU_SECURE)    /* Internal: used by security validation */                                                                                 \
+	X(FW_CONFIG)     /* Internal: subheap of FW_MAIN, configuration data for FW init */                                                          \
+	X(FW_CODE)       /* Internal: used by security validation or dedicated fw */                                                                 \
+	X(FW_PRIV_DATA)  /* Internal: internal FW data (like the stack, FW control data structures, etc.) */                                         \
+	X(FW_PREMAP_PT)  /* Internal: page tables for premapped firmware memory */                                                                   \
+	X(FW_PREMAP0)    /* Internal: Host OS premap fw heap */                                                                                      \
+	X(FW_PREMAP1)    /* Internal: Guest OS 1 premap fw heap */                                                                                   \
+	X(FW_PREMAP2)    /* Internal: Guest OS 2 premap fw heap */                                                                                   \
+	X(FW_PREMAP3)    /* Internal: Guest OS 3 premap fw heap */                                                                                   \
+	X(FW_PREMAP4)    /* Internal: Guest OS 4 premap fw heap */                                                                                   \
+	X(FW_PREMAP5)    /* Internal: Guest OS 5 premap fw heap */                                                                                   \
+	X(FW_PREMAP6)    /* Internal: Guest OS 6 premap fw heap */                                                                                   \
+	X(FW_PREMAP7)    /* Internal: Guest OS 7 premap fw heap */                                                                                   \
+	X(WRAP)          /* External: Wrap memory */                                                                                                 \
+	X(DISPLAY)       /* External: Display memory */                                                                                              \
+	X(LAST)
+
+typedef enum _PVRSRV_PHYS_HEAP_
+{
+#define X(_name) PVRSRV_PHYS_HEAP_ ## _name,
+	PHYS_HEAP_LIST
+#undef X
+
+	PVRSRV_PHYS_HEAP_INVALID = 0x7FFFFFFF
+} PVRSRV_PHYS_HEAP;
+
+/* Defines the number of user mode physheaps. These physheaps are: DEFAULT, GPU_LOCAL,
+ * CPU_LOCAL, GPU_PRIVATE, GPU_SECURE. */
+#define MAX_USER_MODE_ALLOC_PHYS_HEAPS 5
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= (0x1FU + 1U), "Ensure enum fits in memalloc flags bitfield.");
+
+/*! Type conveys the class of physical heap to instantiate within Services
+ * for the physical pool of memory. */
+#define PHYS_HEAP_TYPE_LIST                                                                                                                                                                                                                                           \
+	X(UNKNOWN)              /* Not a valid value for any config */                                                                                                                                                                                                    \
+	X(UMA)                  /* Heap represents OS managed physical memory heap i.e. system RAM. Unified Memory Architecture physmem_osmem PMR factory */                                                                                                              \
+	X(LMA)                  /* Heap represents physical memory pool managed by Services i.e. carve out from system RAM or local card memory. Local Memory Architecture physmem_lma PMR factory */                                                                     \
+	X(DLM)                  /* Heap represents local card memory. Used in a DLM heap (Dedicated Local Memory) system. */                                                                                                                                              \
+	X(IMA)                  /* Heap represents phys heap that imports PMBs from a DLM heap.*/                                                                                                                                                                         \
+	X(DMA)                  /* Heap represents a physical memory pool managed by Services, alias of LMA and is only used on VZ non-native system configurations for a heap used for allocations tagged with PVRSRV_PHYS_HEAP_FW_MAIN or PVRSRV_PHYS_HEAP_FW_CONFIG */ \
+	X(WRAP)                 /* Heap used to group UM buffers given to Services. Integrity OS port only. */                                                                                                                                                            \
+	X(LAST)                                                                                                                                                                                                                                                           \
+
+typedef enum _PHYS_HEAP_TYPE_
+{
+#define X(_name) PHYS_HEAP_TYPE_ ## _name,
+	PHYS_HEAP_TYPE_LIST
+#undef X
+
+} PHYS_HEAP_TYPE;
+
+/* Defines used when interpreting the ui32PhysHeapFlags in PHYS_HEAP_MEM_STATS
+     0x000000000000000d
+     d = is this the default heap? (1=yes, 0=no)
+*/
+#define PVRSRV_PHYS_HEAP_FLAGS_IS_DEFAULT (0x1U)
+
+/* Force PHYS_HEAP_MEM_STATS size to be a multiple of 8 bytes
+ * (as type is a parameter in bridge calls)
+ */
+typedef struct PHYS_HEAP_MEM_STATS_V1_TAG
+{
+	IMG_UINT64 ui64TotalSize;     /*!< Total number of bytes in the heap. */
+	IMG_UINT64 ui64FreeSize;      /*!< Remaining number of bytes free for allocation. */
+	IMG_UINT32 ui32PhysHeapFlags; /*!< Flags associated within the heap. */
+	PHYS_HEAP_TYPE ePhysHeapType; /*!< The type of physheap. */
+} PHYS_HEAP_MEM_STATS_V1;
+typedef struct PHYS_HEAP_MEM_STATS_V2_TAG
+{
+	IMG_UINT64 ui64TotalSize;     /*!< Total number of bytes in the heap. */
+	IMG_UINT64 ui64FreeSize;      /*!< Remaining number of bytes free for allocation. */
+	IMG_UINT32 ui32PhysHeapFlags; /*!< Flags associated within the heap. */
+	PHYS_HEAP_TYPE ePhysHeapType; /*!< The type of physheap. */
+	IMG_UINT64 ui64DevicesInSPAS; /*!< A bitmap of devices that are linked to the heap via a SPAS.
+	                                   Where a device is encoded as a bit using the following:
+	                                   (1 << psDevNode->sDevId.ui32InternalID) */
+} PHYS_HEAP_MEM_STATS_V2;
+typedef PHYS_HEAP_MEM_STATS_V2 PHYS_HEAP_MEM_STATS, *PHYS_HEAP_MEM_STATS_PTR;
+
+#if defined(PHYSHEAP_STRINGS)
+
+static const char *const _pszPhysHeapStrings[] = {
+#define X(_name) #_name,
+	PHYS_HEAP_LIST
+#undef X
+};
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetClientPhysHeapTypeName
+@Description    Returns the phys heap type as a string.
+
+@Input          ePhysHeapType   The physheap type.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetClientPhysHeapTypeName(PHYS_HEAP_TYPE ePhysHeapType)
+{
+	switch (ePhysHeapType)
+	{
+#define X(_name) case PHYS_HEAP_TYPE_ ## _name: return "PHYS_HEAP_TYPE_" # _name;
+		PHYS_HEAP_TYPE_LIST
+#undef X
+		default:
+			return "Unknown Heap Type";
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetPhysHeapName
+@Description    Returns the name of a PhysHeap.
+
+@Input          ePhysHeap   The enum value of the physheap.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetPhysHeapName(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	if (ePhysHeap < 0 || ePhysHeap >= PVRSRV_PHYS_HEAP_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszPhysHeapStrings[ePhysHeap];
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetClientPhysHeapName
+@Description    Returns the name of a client PhysHeap.
+
+@Input          ePhysHeap   The enum value of the physheap.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetClientPhysHeapName(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+    if (ePhysHeap > PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+    {
+        return "Unknown Heap";
+    }
+
+    return PVRSRVGetPhysHeapName(ePhysHeap);
+}
+#endif /* PHYSHEAP_STRINGS */
+
+#endif /* PVRSRV_MEMALLOC_PHYSHEAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags.h
new file mode 100644
index 000000000000..d0703131e62d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags.h
@@ -0,0 +1,1121 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines flags used on memory allocations and mappings
+                These flags are relevant throughout the memory management
+                software stack and are specified by users of services and
+                understood by all levels of the memory management in both
+                client and server.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_MEMALLOCFLAGS_H
+#define PVRSRV_MEMALLOCFLAGS_H
+
+#include "img_types.h"
+#include "pvrsrv_memalloc_physheap.h"
+
+/*!
+  Type for specifying memory allocation flags.
+ */
+
+typedef IMG_UINT64 PVRSRV_MEMALLOCFLAGS_T;
+#define PVRSRV_MEMALLOCFLAGS_FMTSPEC IMG_UINT64_FMTSPECx
+
+#if defined(__KERNEL__)
+#include "pvrsrv_memallocflags_internal.h"
+#endif /* __KERNEL__ */
+
+/*
+ * --- MAPPING FLAGS      0..14 (15-bits) ---
+ * | 0-3    | 4-7    | 8-10        | 11-13       | 14          |
+ * | GPU-RW | CPU-RW | GPU-Caching | CPU-Caching | KM-Mappable |
+ *
+ * --- MISC FLAGS         15..20 (9-bits) ---
+ * | 15    | 16        | 17  | 18         | 19              | 20      |
+ * | Defer | Reserved  | SVM | Scratch-Pg | CPU-Cache-Clean | Zero-Pg |
+ *
+ * --- RI FLAGS  21..23 (3-bits) ---
+ * | 21     | 22       | 23        |
+ * | Import | Suballoc | FW alloc  |
+ *
+ * --- DEV CONTROL FLAGS  26..27 (2-bits) ---
+ * | 24-25 | 26-27        |
+ * | ...   | Device-Flags |
+ *
+ * --- MISC FLAGS         28..31 (4-bits) ---
+ * | 28             | 29             | 30          | 31            |
+ * | No-Cache-Align | Poison-On-Free | P.-On-Alloc | Zero-On-Alloc |
+ *
+ * --- VALIDATION FLAGS ---
+ * | 35             |
+ * | Shared-buffer  |
+ *
+ * --- OS SPECIFIC FLAGS ---
+ * | 36             | 37            | 38              |
+ * | Linux Pref CMA | Linux Movable | Linux Deny Move |
+ *
+ * --- IPA Policy ---
+ * | 53-55      |
+ * | IPA Policy |
+ *
+ * --- PHYS HEAP HINTS ---
+ * | 56            | 57-58 | 59-63          |
+ * | Mandate Heap  |       | PhysHeap Hints |
+ *
+ */
+
+/*
+ *  **********************************************************
+ *  *                                                        *
+ *  *                      MAPPING FLAGS                     *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*!
+ * This flag affects the device MMU protection flags, and specifies
+ * that the memory may be read by the GPU.
+ *
+ * Typically all device memory allocations would specify this flag.
+ *
+ * At the moment, memory allocations without this flag are not supported
+ *
+ * This flag will live with the PMR, thus subsequent mappings would
+ * honour this flag.
+ *
+ * This is a dual purpose flag.  It specifies that memory is permitted
+ * to be read by the GPU, and also requests that the allocation is
+ * mapped into the GPU as a readable mapping
+ *
+ * To be clear:
+ * - When used as an argument on PMR creation; it specifies
+ *       that GPU readable mappings will be _permitted_
+ * - When used as an argument to a "map" function: it specifies
+ *       that a GPU readable mapping is _desired_
+ * - When used as an argument to "AllocDeviceMem": it specifies
+ *       that the PMR will be created with permission to be mapped
+ *       with a GPU readable mapping, _and_ that this PMR will be
+ *       mapped with a GPU readable mapping.
+ * This distinction becomes important when (a) we export allocations;
+ * and (b) when we separate the creation of the PMR from the mapping.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_READABLE		(IMG_UINT64_C(1)<<0)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_READABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_READABLE(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_READABLE) != 0U)
+
+/*!
+ * This flag affects the device MMU protection flags, and specifies
+ * that the memory may be written by the GPU
+ *
+ * Using this flag on an allocation signifies that the allocation is
+ * intended to be written by the GPU.
+ *
+ * Omitting this flag causes a read-only mapping.
+ *
+ * This flag will live with the PMR, thus subsequent mappings would
+ * honour this flag.
+ *
+ * This is a dual purpose flag.  It specifies that memory is permitted
+ * to be written by the GPU, and also requests that the allocation is
+ * mapped into the GPU as a writable mapping (see note above about
+ * permission vs. mapping mode, and why this flag causes permissions
+ * to be inferred from mapping mode on first allocation)
+ *
+ * N.B.  This flag has no relevance to the CPU's MMU mapping, if any,
+ * and would therefore not enforce read-only mapping on CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE       (IMG_UINT64_C(1)<<1)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITEABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as GPU readable in another GPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED  (IMG_UINT64_C(1)<<2)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_READ_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as GPU writable in another GPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED (IMG_UINT64_C(1)<<3)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITE_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED) != 0U)
+
+/*!
+  The flag indicates that an allocation is mapped as readable to the CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_READABLE        (IMG_UINT64_C(1)<<4)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_READABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_READABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_READABLE) != 0U)
+
+/*!
+  The flag indicates that an allocation is mapped as writable to the CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE       (IMG_UINT64_C(1)<<5)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITEABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as CPU readable in another CPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED  (IMG_UINT64_C(1)<<6)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_READ_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as CPU writable in another CPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED (IMG_UINT64_C(1)<<7)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITE_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED) != 0U)
+
+
+/*
+ *  **********************************************************
+ *  *                                                        *
+ *  *                   CACHE CONTROL FLAGS                  *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*
+	GPU domain
+	==========
+
+	The following defines are used to control the GPU cache bit field.
+	The defines are mutually exclusive.
+
+	A helper macro, PVRSRV_GPU_CACHE_MODE, is provided to obtain just the GPU
+	cache bit field from the flags. This should be used whenever the GPU cache
+	mode needs to be determined.
+*/
+
+/*!
+  GPU domain. Flag indicating uncached memory. This means that any writes to memory
+  allocated with this flag are written straight to memory and thus are
+  coherent for any device in the system.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_UNCACHED				(IMG_UINT64_C(1)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_UNCACHED(uiFlags)				(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_UNCACHED)
+
+/*!
+   GPU domain. Use write combiner (if supported) to combine sequential writes
+   together to reduce memory access by doing burst writes.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC			(IMG_UINT64_C(0)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags)			(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC)
+
+/*!
+    GPU domain. This flag affects the GPU MMU protection flags.
+    The allocation will be cached.
+    Services will try to set the coherent bit in the GPU MMU tables so the
+    GPU cache is snooping the CPU cache. If coherency is not supported the
+    caller is responsible to ensure the caches are up to date.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT			(IMG_UINT64_C(2)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags)		(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+
+/*!
+   GPU domain. Request cached memory, but not coherent (i.e. no cache
+   snooping). Services will flush the GPU internal caches after every GPU
+   task so no cache maintenance requests from the users are necessary.
+
+    Note: We reserve 3 bits in the CPU/GPU cache mode to allow for future
+    expansion.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT		(IMG_UINT64_C(3)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)		(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT)
+
+/*!
+    GPU domain. This flag is for internal use only and is used to indicate
+    that the underlying allocation should be cached on the GPU after all
+    the snooping and coherent checks have been done
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHED					(IMG_UINT64_C(7)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHED(uiFlags)				(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHED)
+
+/*!
+    GPU domain. GPU cache mode mask.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK			(IMG_UINT64_C(7)<<8)
+
+/*!
+  @Description    A helper macro to obtain just the GPU	cache bit field from the flags.
+                  This should be used whenever the GPU cache mode needs to be determined.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the GPU cache bit field.
+ */
+#define PVRSRV_GPU_CACHE_MODE(uiFlags)					((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK)
+
+
+/*
+	CPU domain
+	==========
+
+	The following defines are used to control the CPU cache bit field.
+	The defines are mutually exclusive.
+
+	A helper macro, PVRSRV_CPU_CACHE_MODE, is provided to obtain just the CPU
+	cache bit field from the flags. This should be used whenever the CPU cache
+	mode needs to be determined.
+*/
+
+/*!
+   CPU domain. Use write combiner (if supported) to combine sequential writes
+   together to reduce memory access by doing burst writes.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC			(IMG_UINT64_C(0)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)			(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC)
+
+/*!
+    CPU domain. This flag affects the CPU MMU protection flags.
+    The allocation will be cached.
+    Services will try to set the coherent bit in the CPU MMU tables so the
+    CPU cache is snooping the GPU cache. If coherency is not supported the
+    caller is responsible to ensure the caches are up to date.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT			(IMG_UINT64_C(2)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags)		(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+
+/*!
+    CPU domain. Request cached memory, but not coherent (i.e. no cache
+    snooping). This means that if the allocation needs to transition from
+    one device to another services has to be informed so it can
+    flush/invalidate the appropriate caches.
+
+    Note: We reserve 3 bits in the CPU/GPU cache mode to allow for future
+    expansion.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT		(IMG_UINT64_C(3)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)		(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT)
+
+/*!
+    CPU domain. This flag is for internal use only and is used to indicate
+    that the underlying allocation should be cached on the CPU
+    after all the snooping and coherent checks have been done
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHED					(IMG_UINT64_C(7)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHED(uiFlags)				(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHED)
+
+/*!
+	CPU domain. CPU cache mode mask
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK			(IMG_UINT64_C(7)<<11)
+
+/*!
+  @Description    A helper macro to obtain just the CPU	cache bit field from the flags.
+                  This should be used whenever the CPU cache mode needs to be determined.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the CPU cache bit field.
+ */
+#define PVRSRV_CPU_CACHE_MODE(uiFlags)					((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)
+
+/* Helper flags for usual cases */
+
+/*!
+ * Memory will be write-combined on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_UNCACHED_WC		(PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_WRITE_COMBINE(uiFlags)				(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_UNCACHED_WC)
+
+/*!
+ * Memory will be cached.
+ * Services will try to set the correct flags in the MMU tables.
+ * In case there is no coherency support the caller has to ensure caches are up to date */
+#define PVRSRV_MEMALLOCFLAG_CACHE_COHERENT				(PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT | PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CACHE_COHERENT(uiFlags)			(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CACHE_COHERENT)
+
+/*!
+ * Memory will be cache-incoherent on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT			(PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CACHE_INCOHERENT(uiFlags)			(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT)
+
+/*!
+	Cache mode mask
+*/
+#define PVRSRV_CACHE_MODE(uiFlags)						(PVRSRV_GPU_CACHE_MODE(uiFlags) | PVRSRV_CPU_CACHE_MODE(uiFlags))
+
+
+/*!
+   CPU MMU Flags mask -- intended for use internal to services only
+ */
+#define PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK  (PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+												PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+												PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)
+
+/*!
+   MMU Flags mask -- intended for use internal to services only - used for
+   partitioning the flags bits and determining which flags to pass down to
+   mmu_common.c
+ */
+#define PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK  (PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK)
+
+/*!
+    Indicates that the PMR created due to this allocation will support
+    in-kernel CPU mappings.  Only privileged processes may use this flag as
+    it may cause wastage of precious kernel virtual memory on some platforms.
+ */
+#define PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE			(IMG_UINT64_C(1)<<14)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE) != 0U)
+
+
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                   ALLOC MEMORY FLAGS                   *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*!	----- Bit 15
+
+    Indicates when the allocation of physical memory pages backing the PMR
+    is carried out. When set, pages are not allocated at PMR creation but are
+    instead deferred until they are first needed, i.e. "on demand".
+    When unset, the pages may be allocated at the same time the PMR is created
+    or deferred (at the KM/Server's discretion).
+ */
+#define PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC			(IMG_UINT64_C(1)<<15)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_ON_DEMAND(uiFlags)					(((uiFlags) & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC) != 0U)
+
+/*! ----- Bit 16
+
+    This flag is unused but kept for compatibility reasons. Once not a concern
+    the flag can be removed and but reused.
+ */
+#define PVRSRV_MEMALLOCFLAG_PHYS_RESERVED				(IMG_UINT64_C(1)<<16)
+
+/*! ----- Bit 17
+
+    Indicates that the allocation will be accessed by the CPU and GPU using
+    the same virtual address, i.e. for all SVM allocs,
+    IMG_CPU_VIRTADDR == IMG_DEV_VIRTADDR
+ */
+#define PVRSRV_MEMALLOCFLAG_SVM_ALLOC					(IMG_UINT64_C(1)<<17)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_SVM_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_SVM_ALLOC(uiFlags)					(((uiFlags) & PVRSRV_MEMALLOCFLAG_SVM_ALLOC) != 0U)
+
+/*! ----- Bit 18
+
+    Indicates the particular memory that's being allocated is sparse and the
+    sparse regions should not be backed by scratch page
+ */
+#define PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING		(IMG_UINT64_C(1) << 18)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING) == 0U)
+
+/*! ----- Bit 19
+
+  Used to force Services to carry out at least one CPU cache flush and
+  invalidate on a CPU cached buffer during allocation of the memory. Applicable
+  to incoherent systems, it must be used for buffers which are CPU cached and
+  which will not be 100% written to by the CPU before the GPU accesses it. For
+  performance reasons, avoid usage if the whole buffer that is allocated is
+  written to by the CPU anyway before the next GPU kick, or if the system is
+  coherent.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN				(IMG_UINT64_C(1)<<19)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN) != 0U)
+
+/*! ----- Bit 20
+
+    Indicates the particular memory region should be backed by zero page.
+    This is different with zero on alloc flag such that only physically unbacked
+    pages are backed by zero page at the time of mapping.
+    The zero backed page is always with read only attribute irrespective of its
+    original attributes.
+ */
+#define PVRSRV_MEMALLOCFLAG_ZERO_BACKING			(IMG_UINT64_C(1) << 20)
+#define PVRSRV_IS_ZERO_BACKING_REQUIRED(uiFlags)		(((uiFlags) & \
+			PVRSRV_MEMALLOCFLAG_ZERO_BACKING) == PVRSRV_MEMALLOCFLAG_ZERO_BACKING)
+
+/*
+ ************************************************************
+ *                         RI flags                         *
+ ************************************************************
+ * The Flags are used exclusively by the RI (Reference Info)
+ * server for tracking KM allocations by various processes
+ *
+ * Import - Handle imported from other process
+ * Suballoc - Handle Suballocation of existing PMR
+ * FW Alloc - Used by FW during driver initialisation
+ *
+ * --- RI FLAGS  21..23 (3-bits) ---
+ * | 21     | 22       | 23        |
+ * | Import | Suballoc | FW alloc  |
+ *
+ */
+
+/*! ----- Bit 21
+  Used by RI server to register Allocation as an import from other process
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_IMPORT					(IMG_UINT64_C(1)<<21)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_IMPORT flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_IMPORT(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_IMPORT) != 0U)
+
+/*! ----- Bit 22
+  Used by RI server to register Allocation as a suballocation of existing PMR
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_SUBALLOC					(IMG_UINT64_C(1)<<22)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_SUBALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_SUBALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_SUBALLOC) != 0U)
+
+/*! ----- Bit 23
+  Used by RI server to register Allocation as FW/System process
+  'FW Alloc' also means the allocation is considered to belong to the SYS process
+  (ie that it will have a lifetime longer than the process which allocated it)
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC				(IMG_UINT64_C(1)<<23)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC) != 0U)
+
+/*!
+  @Description    Macro passing the PVRSRV_MEMALLOCFLAG_RI_ Flags.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the RI server bit field
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_MASK(uiFlags)			((uiFlags) & (IMG_UINT64_C(7)<<21))
+
+
+/*! ----- Bit 24
+  *
+    Not used.
+ */
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *           MEMORY ZEROING AND POISONING FLAGS           *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * Zero / Poison, on alloc/free
+ *
+ * We think the following usecases are required:
+ *
+ *  don't poison or zero on alloc or free
+ *     (normal operation, also most efficient)
+ *  poison on alloc
+ *     (for helping to highlight bugs)
+ *  poison on alloc and free
+ *     (for helping to highlight bugs)
+ *  zero on alloc
+ *     (avoid highlighting security issues in other uses of memory)
+ *  zero on alloc and poison on free
+ *     (avoid highlighting security issues in other uses of memory, while
+ *      helping to highlight a subset of bugs e.g. memory freed prematurely)
+ *
+ * Since there are more than 4, we can't encode this in just two bits,
+ * so we might as well have a separate flag for each of the three
+ * actions.
+ */
+
+/*!
+    Ensures that the memory allocated is initialised with zeroes.
+ */
+#define PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC				(IMG_UINT64_C(1)<<31)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) != 0U)
+
+/*!
+    Scribbles over the allocated memory with a poison value
+
+    Not compatible with ZERO_ON_ALLOC
+
+    Poisoning is very deliberately _not_ reflected in PDump as we want
+    a simulation to cry loudly if the initialised data propagates to a
+    result.
+ */
+#define PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC				(IMG_UINT64_C(1)<<30)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC) != 0U)
+
+#if defined(DEBUG)
+/*!
+    Causes memory to be trashed when freed, used when debugging only, not to be used
+    as a security measure.
+ */
+#define PVRSRV_MEMALLOCFLAG_POISON_ON_FREE				(IMG_UINT64_C(1)<<29)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_POISON_ON_FREE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_POISON_ON_FREE(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_POISON_ON_FREE) != 0U)
+#endif /* DEBUG */
+
+/*!
+    Avoid address alignment to a CPU or GPU cache line size.
+ */
+#define PVRSRV_MEMALLOCFLAG_NO_CACHE_LINE_ALIGN			(IMG_UINT64_C(1)<<28)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_CHECK_NO_CACHE_LINE_ALIGN flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_NO_CACHE_LINE_ALIGN(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_NO_CACHE_LINE_ALIGN) != 0U)
+
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                Device specific MMU flags               *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 26 to 27)
+ *
+ * Some services controlled devices have device specific control bits in
+ * their page table entries, we need to allow these flags to be passed down
+ * the memory management layers so the user can control these bits.
+ * For example, RGX device has the file rgx_memallocflags.h
+ */
+
+/*!
+ * Offset of device specific MMU flags.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET		26
+
+/*!
+ * Mask for retrieving device specific MMU flags.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK		(IMG_UINT64_C(3) << PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
+
+/*!
+  @Description    Helper macro for setting device specific MMU flags.
+  @Input  uiFlags Flag index.
+  @Return         Flag vector with the specified bit set.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(uiFlags)	\
+			(((PVRSRV_MEMALLOCFLAGS_T)(uiFlags) << PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET) & \
+			PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                 Secure validation flags                *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bit 35)
+ *
+ */
+
+/*!
+    PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER
+ */
+
+#define PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER           (IMG_UINT64_C(1)<<35)
+#define PVRSRV_CHECK_SHARED_BUFFER(uiFlags)             (((uiFlags) & PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER) != 0U)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                OS Specific alloc flags                 *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 36 to 38)
+ *
+ */
+#define PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_OFFSET 36
+#define PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK          (IMG_UINT64_C(7) << PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_OFFSET)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA        (IMG_UINT64_C(1)<<36)
+#define PVRSRV_CHECK_OS_LINUX_PREFER_CMA(uiFlags)      (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA) != 0U)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE           (IMG_UINT64_C(1)<<37)
+#define PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags)         (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE) != 0U)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE         (IMG_UINT64_C(1)<<38)
+#define PVRSRV_CHECK_OS_LINUX_DENY_MOVE(uiFlags)       (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE) != 0U)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                       IPA Policy                       *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 53 to 55)
+ *
+ */
+
+/*!
+ * Offset of Intermediate Physical Address (IPA) policy.
+ */
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET      53
+
+/*!
+ * Mask for retrieving IPA policy.
+ */
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK        (IMG_UINT64_C(7) << PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET)
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY(uiFlags)    (((uiFlags) & PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK) >> PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                 Phys Heap Hints                        *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 56 to 63)
+ *
+ */
+
+/*!
+ * Ensures Physheap isn't reassigned when considered favourable by driver under a OOM condition.
+ */
+#define PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP            (IMG_UINT64_C(1)<<56)
+#define PVRSRV_CHECK_MANDATED_PHYSHEAP(uiFlags)         (((uiFlags) & PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP) != 0U)
+
+/*!
+ * Value of enum PVRSRV_PHYS_HEAP stored in memalloc flags. If not set
+ * i.e. PVRSRV_PHYS_HEAP_DEFAULT (value 0) used, the system layer defined default physical heap is used.
+ */
+#define PVRSRV_PHYS_HEAP_HINT_SHIFT        (59)
+#define PVRSRV_PHYS_HEAP_HINT_MASK         (IMG_UINT64_C(0x1F) << PVRSRV_PHYS_HEAP_HINT_SHIFT)
+
+
+/*!
+  @Description    Macro extracting the Phys Heap hint from memalloc flag value.
+  @Input uiFlags  Allocation flags
+  @Return         returns the value of the PHYS_HEAP_HINT bitfield
+ */
+#define PVRSRV_GET_PHYS_HEAP_HINT(uiFlags)      ((PVRSRV_PHYS_HEAP)(((uiFlags) & PVRSRV_PHYS_HEAP_HINT_MASK) \
+                                                 >> PVRSRV_PHYS_HEAP_HINT_SHIFT))
+
+/*!
+  @Description    Macro converting a Phys Heap value into a memalloc bitfield
+  @Input uiFlags  Device Phys Heap
+  @Return         returns a shifted bitfield with the Device Phys Heap value
+ */
+#define PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(PhysHeap)      ((((PVRSRV_MEMALLOCFLAGS_T)PVRSRV_PHYS_HEAP_ ## PhysHeap) << \
+                                                            PVRSRV_PHYS_HEAP_HINT_SHIFT) \
+                                                           & PVRSRV_PHYS_HEAP_HINT_MASK)
+/*!
+  @Description    Macro to replace an existing phys heap hint value in flags.
+  @Input PhysHeap Phys Heap Macro
+  @Input uiFlags  Allocation flags
+  @Return         N/A
+ */
+#define PVRSRV_SET_PHYS_HEAP_HINT(PhysHeap, uiFlags)	  (uiFlags) = ((uiFlags) & ~PVRSRV_PHYS_HEAP_HINT_MASK) | \
+                                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(PhysHeap)
+
+/*!
+  @Description    Macro to replace an existing phys heap hint value using Phys Heap value.
+  @Input PhysHeap Phys Heap Value
+  @Input uiFlags  Allocation flags
+  @Return         N/A
+ */
+#define PVRSRV_CHANGE_PHYS_HEAP_HINT(Physheap, uiFlags)	  (uiFlags) = ((uiFlags) & ~PVRSRV_PHYS_HEAP_HINT_MASK) | \
+                                                           (((PVRSRV_MEMALLOCFLAGS_T)(Physheap) << \
+                                                            PVRSRV_PHYS_HEAP_HINT_SHIFT) \
+                                                           & PVRSRV_PHYS_HEAP_HINT_MASK)
+
+/*!
+  @Description    Macros checking if a Phys Heap hint is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the hint is set, false otherwise
+ */
+#define PVRSRV_CHECK_PHYS_HEAP(PhysHeap, uiFlags) (PVRSRV_PHYS_HEAP_ ## PhysHeap == PVRSRV_GET_PHYS_HEAP_HINT(uiFlags))
+
+#define PVRSRV_CHECK_FW_MAIN(uiFlags)            (PVRSRV_CHECK_PHYS_HEAP(FW_MAIN, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_CONFIG, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_CODE, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PRIV_DATA, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP0, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP1, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP2, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP3, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP4, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP5, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP6, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP7, uiFlags))
+
+/*!
+ * Secure buffer mask -- Flags in the mask are allowed for secure buffers
+ * because they are not related to CPU mappings.
+ */
+#define PVRSRV_MEMALLOCFLAGS_SECBUFMASK  ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                           PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED)
+
+/*!
+ * Trusted device mask -- Flags in the mask are allowed for trusted device
+ * because the driver cannot access the memory
+ */
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_TDFWMASK    ~(PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                           PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                           PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+#else
+#define PVRSRV_MEMALLOCFLAGS_TDFWMASK    ~(PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                           PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+#endif
+
+/*!
+  PMR flags mask -- for internal services use only.  This is the set of flags
+  that will be passed down and stored with the PMR, this also includes the
+  MMU flags which the PMR has to pass down to mm_common.c at PMRMap time.
+*/
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK  (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                            PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                            PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER | \
+                                            PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                            PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                            PVRSRV_PHYS_HEAP_HINT_MASK)
+#else
+#define PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK  (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                            PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER | \
+                                            PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP  | \
+                                            PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                            PVRSRV_PHYS_HEAP_HINT_MASK)
+#endif
+
+/*!
+ * CPU mappable mask -- Any flag set in the mask requires memory to be CPU mappable
+ */
+#define PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK (PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                                PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                                PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)
+/*!
+  RA differentiation mask
+
+  for use internal to services
+
+  this is the set of flags bits that are able to determine whether a pair of
+  allocations are permitted to live in the same page table. Allocations
+  whose flags differ in any of these places would be allocated from separate
+  RA Imports and therefore would never coexist in the same page.
+  Special cases are zeroing and poisoning of memory. The caller is responsible
+  to set the sub-allocations to the value he wants it to be. To differentiate
+  between zeroed and poisoned RA Imports does not make sense because the
+  memory might be reused.
+
+*/
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK (PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK \
+                                                      & \
+                                                      ~(PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC   | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_FREE))
+#else
+#define PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK (PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK \
+                                                      & \
+                                                      ~(PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC   | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
+#endif
+/*!
+  Flags that affect _allocation_
+*/
+#define PVRSRV_MEMALLOCFLAGS_PERALLOCFLAGSMASK (0xFFFFFFFFU)
+
+/*!
+  Flags that affect _mapping_
+*/
+#define PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK   (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                                    PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                                    PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+
+#if ((~(PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK) & PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK) != 0U)
+#error PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK is not a subset of PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK
+#endif
+
+
+/*!
+  Flags that affect _physical allocations_ in the DevMemX API
+ */
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_PHYSICAL_MASK (PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                                    PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                                    PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE | \
+                                                    PVRSRV_PHYS_HEAP_HINT_MASK)
+#else
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_PHYSICAL_MASK (PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                                    PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE | \
+                                                    PVRSRV_PHYS_HEAP_HINT_MASK)
+#endif
+
+/*!
+  Flags that affect _virtual allocations_ in the DevMemX API
+ */
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK  (PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED)
+
+#endif /* PVRSRV_MEMALLOCFLAGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags_internal.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags_internal.h
new file mode 100644
index 000000000000..4fee3d49b927
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags_internal.h
@@ -0,0 +1,78 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management allocation flags for internal Services
+                use only
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines flags used on memory allocations and mappings
+                These flags are relevant throughout the memory management
+                software stack and are specified by users of services and
+                understood by all levels of the memory management in the server
+                and in special cases in the client.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_MEMALLOCFLAGS_INTERNAL_H
+#define PVRSRV_MEMALLOCFLAGS_INTERNAL_H
+
+/*!
+   CPU domain. Request uncached memory. This means that any writes to memory
+   allocated with this flag are written straight to memory and thus are
+   coherent for any device in the system.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_UNCACHED (1ULL<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_UNCACHED(uiFlags) (PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_UNCACHED)
+
+/*!
+ * Memory will be uncached on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_UNCACHED (PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_UNCACHED(uiFlags) (PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_UNCACHED)
+
+#endif /* PVRSRV_MEMALLOCFLAGS_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_sync_km.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_sync_km.h
new file mode 100644
index 000000000000..04611f9f7cee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_sync_km.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File
+@Title         PVR synchronisation interface
+@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description   Types for server side code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVRSRV_SYNC_KM_H
+#define PVRSRV_SYNC_KM_H
+
+#include <powervr/pvrsrv_sync_ext.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define SYNC_FB_FILE_STRING_MAX			256
+#define SYNC_FB_MODULE_STRING_LEN_MAX	(32)
+#define	SYNC_FB_DESC_STRING_LEN_MAX		(32)
+
+/* By default, fence-sync module emits into HWPerf (of course, if enabled) and
+ * considers a process (sleepable) context */
+#define PVRSRV_FENCE_FLAG_NONE             (0U)
+#define PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT (1U << 0)
+#define PVRSRV_FENCE_FLAG_CTX_ATOMIC       (1U << 1)
+
+#if defined(__cplusplus)
+}
+#endif
+#endif	/* PVRSRV_SYNC_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlcommon.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlcommon.h
new file mode 100644
index 000000000000..9bc65fe05f98
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlcommon.h
@@ -0,0 +1,260 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Transport Layer common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common types and definitions included into
+                both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_TLCOMMON_H
+#define PVR_TLCOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+
+/*! Handle type for stream descriptor objects as created by this API */
+typedef IMG_HANDLE PVRSRVTL_SD;
+
+/*! Maximum stream name length including the null byte */
+#define PRVSRVTL_MAX_STREAM_NAME_SIZE	40U
+
+/*! Maximum number of streams expected to exist */
+#define PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER (32*PRVSRVTL_MAX_STREAM_NAME_SIZE)
+
+/*! Packet lengths are always rounded up to a multiple of 8 bytes */
+#define PVRSRVTL_PACKET_ALIGNMENT		8U
+#define PVRSRVTL_ALIGN(x)				PVR_ALIGN(x, PVRSRVTL_PACKET_ALIGNMENT)
+
+
+/*! A packet is made up of a header structure followed by the data bytes.
+ * There are 3 types of packet: normal (has data), data lost and padding,
+ * see packet flags. Header kept small to reduce data overhead.
+ *
+ * if the ORDER of the structure members is changed, please UPDATE the
+ * PVRSRVTL_PACKET_FLAG_OFFSET macro.
+ *
+ * Layout of uiTypeSize member is :
+ *
+ * |<---------------------------32-bits------------------------------>|
+ * |<----8---->|<-----1----->|<----7--->|<------------16------------->|
+ * |    Type   | Drop-Oldest |  UNUSED  |             Size            |
+ *
+ */
+typedef struct
+{
+	IMG_UINT32 uiTypeSize;	/*!< Type, Drop-Oldest flag & number of bytes following header */
+	IMG_UINT32 uiReserved;	/*!< Reserve, packets and data must be 8 byte aligned */
+
+	/* First bytes of TL packet data follow header ... */
+} PVRSRVTL_PACKETHDR, *PVRSRVTL_PPACKETHDR;
+
+/* Structure must always be a size multiple of 8 as stream buffer
+ * still an array of IMG_UINT32s.
+ */
+static_assert((sizeof(PVRSRVTL_PACKETHDR) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(PVRSRVTL_PACKETHDR) must be a multiple of 8");
+
+/*! Packet header reserved word fingerprint "TLP1" */
+#define PVRSRVTL_PACKETHDR_RESERVED 0x31504C54U
+
+/*! Packet header mask used to extract the size from the uiTypeSize member.
+ * Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_SIZE_MASK    0x0000FFFFU
+#define PVRSRVTL_MAX_PACKET_SIZE        (PVRSRVTL_PACKETHDR_SIZE_MASK & ~0xFU)
+
+
+/*! Packet header mask used to extract the type from the uiTypeSize member.
+ * Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_TYPE_MASK    0xFF000000U
+#define PVRSRVTL_PACKETHDR_TYPE_OFFSET  24U
+
+/*! Packet header mask used to check if packets before this one were dropped
+ * or not. Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_OLDEST_DROPPED_MASK    0x00800000U
+#define PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET    23U
+
+/*! Packet type enumeration.
+ */
+typedef IMG_UINT32 PVRSRVTL_PACKETTYPE;
+
+/*! Undefined packet */
+#define PVRSRVTL_PACKETTYPE_UNDEF 0U
+
+/*! Normal packet type. Indicates data follows the header.
+ */
+#define PVRSRVTL_PACKETTYPE_DATA 1U
+
+/*! When seen this packet type indicates that at this moment in the stream
+ * packet(s) were not able to be accepted due to space constraints and
+ * that recent data may be lost - depends on how the producer handles the
+ * error. Such packets have no data, data length is 0.
+ */
+#define PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED 2U
+
+/*! Packets with this type set are padding packets that contain undefined
+ * data and must be ignored/skipped by the client. They are used when the
+ * circular stream buffer wraps around and there is not enough space for
+ * the data at the end of the buffer. Such packets have a length of 0 or
+ * more.
+ */
+#define PVRSRVTL_PACKETTYPE_PADDING 3U
+
+/*! This packet type conveys to the stream consumer that the stream
+ * producer has reached the end of data for that data sequence. The
+ * TLDaemon has several options for processing these packets that can
+ * be selected on a per stream basis.
+ */
+#define PVRSRVTL_PACKETTYPE_MARKER_EOS 4U
+
+/*! This is same as PVRSRVTL_PACKETTYPE_MARKER_EOS but additionally removes
+ * old data record output file before opening new/next one
+ */
+#define PVRSRVTL_PACKETTYPE_MARKER_EOS_REMOVEOLD 5U
+
+/*! Packet emitted on first stream opened by writer. Packet carries a name
+ * of the opened stream in a form of null-terminated string.
+ */
+#define PVRSRVTL_PACKETTYPE_STREAM_OPEN_FOR_WRITE 6U
+
+/*! Packet emitted on last stream closed by writer. Packet carries a name
+ * of the closed stream in a form of null-terminated string.
+ */
+#define PVRSRVTL_PACKETTYPE_STREAM_CLOSE_FOR_WRITE 7U
+
+#define PVRSRVTL_PACKETTYPE_LAST 8U
+
+/* The SET_PACKET_* macros rely on the order the PVRSRVTL_PACKETHDR members are declared:
+ * uiFlags is the upper half of a structure consisting of 2 uint16 quantities.
+ */
+#define PVRSRVTL_SET_PACKET_DATA(len)       (len) | (PVRSRVTL_PACKETTYPE_DATA                     << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_PADDING(len)    (len) | (PVRSRVTL_PACKETTYPE_PADDING                  << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_WRITE_FAILED    (0U)   | (PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_HDR(len, type)  (len) | ((type)                                       << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+
+/*! Returns the number of bytes of data in the packet.
+ * p may be any address type.
+ */
+#define GET_PACKET_DATA_LEN(p)	\
+	((IMG_UINT32) ((PVRSRVTL_PPACKETHDR) (void *) (p))->uiTypeSize & PVRSRVTL_PACKETHDR_SIZE_MASK)
+
+
+/*! Returns a IMG_BYTE* pointer to the first byte of data in the packet */
+#define GET_PACKET_DATA_PTR(p)	\
+	(((IMG_UINT8 *) (void *) (p)) + sizeof(PVRSRVTL_PACKETHDR))
+
+/*! Turns the packet address p into a PVRSRVTL_PPACKETHDR pointer type.
+ */
+#define GET_PACKET_HDR(p)		((PVRSRVTL_PPACKETHDR) ((void *) (p)))
+
+/*! Given a PVRSRVTL_PPACKETHDR address, return the address of the next pack
+ *  It is up to the caller to determine if the new address is within the
+ *  packet buffer.
+ */
+#define GET_NEXT_PACKET_ADDR(p) \
+	GET_PACKET_HDR( \
+		GET_PACKET_DATA_PTR(p) + \
+		( \
+			(GET_PACKET_DATA_LEN(p) + (PVRSRVTL_PACKET_ALIGNMENT-1U)) & \
+			(~(PVRSRVTL_PACKET_ALIGNMENT-1U)) \
+		) \
+	)
+
+/*! Get the type of the packet. p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define GET_PACKET_TYPE(p)		(((p)->uiTypeSize & PVRSRVTL_PACKETHDR_TYPE_MASK)>>PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+
+/*! Set PACKETS_DROPPED flag in packet header as a part of uiTypeSize.
+ * p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define SET_PACKETS_DROPPED(p)		(((p)->uiTypeSize) | (1UL << PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET))
+
+/*! Check if packets were dropped before this packet.
+ * p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define CHECK_PACKETS_DROPPED(p)	((((p)->uiTypeSize & PVRSRVTL_PACKETHDR_OLDEST_DROPPED_MASK)>>PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET) != 0U)
+
+/*! Flags for use with PVRSRVTLOpenStream
+ * 0x01 - Do not block in PVRSRVTLAcquireData() when no bytes are available
+ * 0x02 - When the stream does not exist wait for a bit (2s) in
+ *        PVRSRVTLOpenStream() and then exit with a timeout error if it still
+ *        does not exist.
+ * 0x04 - Open stream for write only operations.
+ *        If flag is not used stream is opened as read-only. This flag is
+ *        required if one wants to call reserve/commit/write function on the
+ *        stream descriptor. Read from on the stream descriptor opened
+ *        with this flag will fail.
+ * 0x08 - Disable Producer Callback.
+ *        If this flag is set and the stream becomes empty, do not call any
+ *        associated producer callback to generate more data from the reader
+ *        context.
+ * 0x10 - Reset stream on open.
+ *        When this flag is used the stream will drop all of the stored data.
+ * 0x20 - Limit read position to the write position at time the stream
+ *        was opened. Hence this flag will freeze the content read to that
+ *        produced before the stream was opened for reading.
+ * 0x40 - Ignore Open Callback.
+ *        When this flag is set ignore any OnReaderOpenCallback setting for
+ *        the stream. This allows access to the stream to be made without
+ *        generating any extra packets into the stream.
+ */
+
+#define PVRSRV_STREAM_FLAG_NONE                        (0U)
+#define PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING         (1U<<0)
+#define PVRSRV_STREAM_FLAG_OPEN_WAIT                   (1U<<1)
+#define PVRSRV_STREAM_FLAG_OPEN_WO                     (1U<<2)
+#define PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK   (1U<<3)
+#define PVRSRV_STREAM_FLAG_RESET_ON_OPEN               (1U<<4)
+#define PVRSRV_STREAM_FLAG_READ_LIMIT                  (1U<<5)
+#define PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK        (1U<<6)
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_TLCOMMON_H */
+/******************************************************************************
+ End of file (pvrsrv_tlcommon.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlstreams.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlstreams.h
new file mode 100644
index 000000000000..624b8eb1f47d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlstreams.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Transport Layer stream names
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common types and definitions included into
+                both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_TLSTREAMS_H
+#define PVRSRV_TLSTREAMS_H
+
+#define PVRSRV_TL_CTRL_STREAM "tlctrl"
+
+#define PVRSRV_TL_HWPERF_RGX_FW_STREAM      "hwperf_fw_"
+#define PVRSRV_TL_HWPERF_HOST_SERVER_STREAM "hwperf_host_"
+
+#define PVRSRV_TL_FTRACE_RGX_FW_STREAM      "ftrace_fw_"
+
+/* Host HWPerf client stream names are of the form 'hwperf_client_<pid>' */
+#define PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM         "hwperf_client_"
+#define PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC "hwperf_client_%u_%u"
+
+#endif /* PVRSRV_TLSTREAMS_H */
+
+/******************************************************************************
+ End of file (pvrsrv_tlstreams.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrversion.h b/drivers/gpu/drm/img/img-volcanic/include/pvrversion.h
new file mode 100644
index 000000000000..4d17f6e55594
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrversion.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File           pvrversion.h
+@Title          PowerVR version numbers and strings.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Version numbers and strings for PowerVR components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRVERSION_H
+#define PVRVERSION_H
+
+#define PVRVERSION_MAJ               24U
+#define PVRVERSION_MIN               2U
+
+#define PVRVERSION_FAMILY           "rogueddk"
+#define PVRVERSION_BRANCHNAME       "24.2"
+#define PVRVERSION_BUILD             6643903
+#define PVRVERSION_BSCONTROL        "Rogue_DDK_Linux_WS"
+
+#define PVRVERSION_STRING           "Rogue_DDK_Linux_WS rogueddk 24.2@6643903"
+#define PVRVERSION_STRING_SHORT     "24.2@6643903"
+
+#define COPYRIGHT_TXT               "Copyright (c) Imagination Technologies Ltd. All Rights Reserved."
+
+#define PVRVERSION_BUILD_HI          664
+#define PVRVERSION_BUILD_LO          3903
+#define PVRVERSION_STRING_NUMERIC   "24.2.664.3903"
+
+#define PVRVERSION_PACK(MAJOR,MINOR) (((IMG_UINT32)((IMG_UINT32)(MAJOR) & 0xFFFFU) << 16U) | (((MINOR) & 0xFFFFU) << 0U))
+#define PVRVERSION_UNPACK_MAJ(VERSION) (((VERSION) >> 16U) & 0xFFFFU)
+#define PVRVERSION_UNPACK_MIN(VERSION) (((VERSION) >> 0U) & 0xFFFFU)
+
+#endif /* PVRVERSION_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_common.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_common.h
new file mode 100644
index 000000000000..55b0ed450e1c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_common.h
@@ -0,0 +1,262 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Common Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common types and definitions for RGX software
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_COMMON_H
+#define RGX_COMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+/* Included to get the BVNC_KM_N defined and other feature defs */
+#include "km/rgxdefs_km.h"
+
+#include "rgx_common_asserts.h"
+
+typedef struct RGXFWIF_DEV_VIRTADDR_
+{
+	IMG_UINT32	ui32Addr;
+} RGXFWIF_DEV_VIRTADDR;
+
+/* Virtualisation validation builds are meant to test the VZ-related hardware without a fully virtualised platform.
+ * As such a driver can support either the vz-validation code or real virtualisation. */
+#if defined(SUPPORT_GPUVIRT_VALIDATION) && (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+#error "Invalid build configuration: Virtualisation support (RGX_NUM_DRIVERS_SUPPORTED > 1) and virtualisation validation code (SUPPORT_GPUVIRT_VALIDATION) are mutually exclusive."
+#endif
+
+/* The RGXFWIF_DM defines assume only one of RGX_FEATURE_TLA or
+ * RGX_FEATURE_FASTRENDER_DM is present. Ensure this with a compile-time check.
+ */
+#if defined(RGX_FEATURE_TLA) && defined(RGX_FEATURE_FASTRENDER_DM)
+#error "Both RGX_FEATURE_TLA and RGX_FEATURE_FASTRENDER_DM defined. Fix code to handle this!"
+#endif
+
+#define FW_OSID							(0U)
+#define MMU_CONTEXT_MAPPING_FWPRIV		(0U) /* FW code/private data */
+
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID) || defined(RGX_FEATURE_MIPS)
+/* The Firmware accesses its private code & data and the interface
+ * memory it shares with the KM drivers using the same MMU context */
+#define MMU_CONTEXT_MAPPING_FWIF		MMU_CONTEXT_MAPPING_FWPRIV
+#else
+/* The Firmware accesses the interface memory it shares
+ * with the KM drivers using a reserved MMU context */
+#define MMU_CONTEXT_MAPPING_FWIF		(7U)
+#endif
+
+/*! The master definition for data masters known to the firmware of RGX.
+ * When a new DM is added to this list, relevant entry should be added to
+ * RGX_HWPERF_DM enum list.
+ * The DM in a V1 HWPerf packet uses this definition. */
+
+typedef IMG_UINT32 RGXFWIF_DM;
+
+#define	RGXFWIF_DM_GP			IMG_UINT32_C(0)
+/* Either TDM or 2D DM is present. The above build time error is present to verify this */
+#define	RGXFWIF_DM_2D			IMG_UINT32_C(1) /* when RGX_FEATURE_TLA defined */
+#define	RGXFWIF_DM_TDM			IMG_UINT32_C(1) /* when RGX_FEATURE_FASTRENDER_DM defined */
+
+#define	RGXFWIF_DM_GEOM			IMG_UINT32_C(2)
+#define	RGXFWIF_DM_3D			IMG_UINT32_C(3)
+#define	RGXFWIF_DM_CDM			IMG_UINT32_C(4)
+#define	RGXFWIF_DM_RAY			IMG_UINT32_C(5)
+#define	RGXFWIF_DM_GEOM2		IMG_UINT32_C(6)
+#define	RGXFWIF_DM_GEOM3		IMG_UINT32_C(7)
+#define	RGXFWIF_DM_GEOM4		IMG_UINT32_C(8)
+
+#define	RGXFWIF_DM_LAST			RGXFWIF_DM_GEOM4
+
+typedef IMG_UINT32 RGX_KICK_TYPE_DM;
+#define RGX_KICK_TYPE_DM_GP		IMG_UINT32_C(0x001)
+#define RGX_KICK_TYPE_DM_TDM_2D	IMG_UINT32_C(0x002)
+#define RGX_KICK_TYPE_DM_TA		IMG_UINT32_C(0x004)
+#define RGX_KICK_TYPE_DM_3D		IMG_UINT32_C(0x008)
+#define RGX_KICK_TYPE_DM_CDM	IMG_UINT32_C(0x010)
+#define RGX_KICK_TYPE_DM_RTU	IMG_UINT32_C(0x020)
+#define RGX_KICK_TYPE_DM_SHG	IMG_UINT32_C(0x040)
+#define RGX_KICK_TYPE_DM_TQ2D	IMG_UINT32_C(0x080)
+#define RGX_KICK_TYPE_DM_TQ3D	IMG_UINT32_C(0x100)
+#define RGX_KICK_TYPE_DM_RAY	IMG_UINT32_C(0x200)
+#define RGX_KICK_TYPE_DM_LAST	IMG_UINT32_C(0x400)
+
+/* Maximum number of DM in use: GP, 2D/TDM, GEOM, 3D, CDM, RDM, GEOM2, GEOM3, GEOM4 */
+#define RGXFWIF_DM_MAX			(RGXFWIF_DM_LAST + 1U)
+/* The set of DMs for gathering stats on GPU utilisation excludes GP */
+#define RGXFWIF_GPU_UTIL_DM_MAX (RGXFWIF_DM_MAX - 1U)
+
+/*
+ * Data Master Tags to be appended to resources created on behalf of each RGX
+ * Context.
+ */
+#define RGX_RI_DM_TAG_KS   'K'
+#define RGX_RI_DM_TAG_CDM  'C'
+#define RGX_RI_DM_TAG_RC   'R' /* To be removed once TA/3D Timelines are split */
+#define RGX_RI_DM_TAG_TA   'V'
+#define RGX_RI_DM_TAG_GEOM 'V'
+#define RGX_RI_DM_TAG_3D   'P'
+#define RGX_RI_DM_TAG_TDM  'T'
+#define RGX_RI_DM_TAG_TQ2D '2'
+#define RGX_RI_DM_TAG_TQ3D 'Q'
+#define RGX_RI_DM_TAG_RAY  'r'
+
+/*
+ * Client API Tags to be appended to resources created on behalf of each
+ * Client API.
+ */
+#define RGX_RI_CLIENT_API_GLES1    '1'
+#define RGX_RI_CLIENT_API_GLES3    '3'
+#define RGX_RI_CLIENT_API_VULKAN   'V'
+#define RGX_RI_CLIENT_API_EGL      'E'
+#define RGX_RI_CLIENT_API_OPENCL   'C'
+#define RGX_RI_CLIENT_API_OPENGL   'G'
+#define RGX_RI_CLIENT_API_SERVICES 'S'
+#define RGX_RI_CLIENT_API_WSEGL    'W'
+#define RGX_RI_CLIENT_API_ANDROID  'A'
+#define RGX_RI_CLIENT_API_LWS      'L'
+
+/*
+ * Format a RI annotation for a given RGX Data Master context
+ */
+#define RGX_RI_FORMAT_DM_ANNOTATION(annotation, dmTag, clientAPI) do         \
+	{                                                                        \
+		(annotation)[0] = (dmTag);                                           \
+		(annotation)[1] = (clientAPI);                                       \
+		(annotation)[2] = '\0';                                              \
+	} while (false)
+
+/*!
+ ******************************************************************************
+ * RGXFW Compiler alignment definitions
+ *****************************************************************************/
+#if defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES) || defined(INTEGRITY_OS)
+#define RGXFW_ALIGN			__attribute__ ((aligned (8)))
+#define	RGXFW_ALIGN_DCACHEL		__attribute__((aligned (64)))
+#elif defined(_MSC_VER)
+#define RGXFW_ALIGN			__declspec(align(8))
+#define	RGXFW_ALIGN_DCACHEL		__declspec(align(64))
+#pragma warning (disable : 4324)
+#else
+#error "Align MACROS need to be defined for this compiler"
+#endif
+
+/*!
+ ******************************************************************************
+ * Force 8-byte alignment for structures allocated uncached.
+ *****************************************************************************/
+#define UNCACHED_ALIGN      RGXFW_ALIGN
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation states
+ *****************************************************************************/
+#define RGXFWIF_GPU_UTIL_STATE_IDLE      (0U)
+#define RGXFWIF_GPU_UTIL_STATE_ACTIVE    (1U)
+#define RGXFWIF_GPU_UTIL_STATE_BLOCKED   (2U)
+#define RGXFWIF_GPU_UTIL_STATE_NUM       (3U)
+/* the state below "combines" IDLE and BLOCKED
+ * and is used when we only care about GPU being in ACTIVE or not */
+#define RGXFWIF_GPU_UTIL_STATE_INACTIVE  (0U)
+/* when we combine IDLE and BLOCKED we end up with one state less */
+#define RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM       (RGXFWIF_GPU_UTIL_STATE_NUM-1U)
+#define RGXFWIF_GPU_UTIL_STATE_MASK      IMG_UINT64_C(0x0000000000000003)
+#define RGXFWIF_GPU_UTIL_STATE_MASK32    IMG_UINT32_C(0x00000003)
+
+
+/*
+ * Maximum amount of register writes that can be done by the register
+ * programmer (FW or META DMA). This is not a HW limitation, it is only
+ * a protection against malformed inputs to the register programmer.
+ */
+#define RGX_MAX_NUM_REGISTER_PROGRAMMER_WRITES  (256U)
+
+/* FW common context priority. */
+/*!
+ * @AddToGroup WorkloadContexts
+ * @{
+ */
+#define RGX_CTX_PRIORITY_REALTIME  (INT32_MAX)
+#define RGX_CTX_PRIORITY_HIGH      (2) /*!< HIGH priority */
+#define RGX_CTX_PRIORITY_MEDIUM    (1) /*!< MEDIUM priority */
+#define RGX_CTX_PRIORITY_LOW       (0) /*!< LOW priority */
+/*!
+ * @} End of AddToGroup WorkloadContexts
+ */
+
+
+/*
+ *   Use of the 32-bit context property flags mask
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *                                   0
+ *                                   |
+ *    -------------------------------x
+ */
+/*
+ * Context creation flags
+ * (specify a context's properties at creation time)
+ */
+#define RGX_CONTEXT_FLAG_DISABLESLR					(1UL << 0) /*!< Disable SLR */
+
+/* Bitmask of context flags allowed to be modified after context create. */
+#define RGX_CONTEXT_FLAGS_WRITEABLE_MASK            (RGX_CONTEXT_FLAG_DISABLESLR)
+
+/* List of attributes that may be set for a context */
+typedef IMG_UINT32 RGX_CONTEXT_PROPERTY;
+#define RGX_CONTEXT_PROPERTY_FLAGS 0U /*!< Context flags */
+
+/* MMU4 supported number of ranges */
+#define RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES (4U)
+#define RGX_MMU_RANGE_NON4KHEAP          (0U)
+#define RGX_MMU_RANGE_GLOBAL             (RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES - 1)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_COMMON_H */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_common_asserts.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_common_asserts.h
new file mode 100644
index 000000000000..c571cc6f008e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_common_asserts.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Common Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common types and definitions for RGX software
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_COMMON_ASSERTS_H
+#define RGX_COMMON_ASSERTS_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*! This macro represents a mask of LSBs that must be zero on data structure
+ * sizes and offsets to ensure they are 8-byte granular on types shared between
+ * the FW and host driver */
+#define RGX_FW_ALIGNMENT_LSB (7U)
+
+/*! Macro to test structure size alignment */
+#define RGX_FW_STRUCT_SIZE_ASSERT(_a)	\
+	static_assert((sizeof(_a) & RGX_FW_ALIGNMENT_LSB) == 0U,	\
+				  "Size of " #_a " is not properly aligned")
+
+/*! Macro to test structure member alignment */
+#define RGX_FW_STRUCT_OFFSET_ASSERT(_a, _b)	\
+	static_assert((offsetof(_a, _b) & RGX_FW_ALIGNMENT_LSB) == 0U,	\
+				  "Offset of " #_a "." #_b " is not properly aligned")
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_COMMON_ASSERTS_H */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_compat_bvnc.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_compat_bvnc.h
new file mode 100644
index 000000000000..c3e1333cdb0f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_compat_bvnc.h
@@ -0,0 +1,140 @@
+/*************************************************************************/ /*!
+@File           rgx_compat_bvnc.h
+@Title          BVNC compatibility check utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used for packing BNC and V.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_COMPAT_BVNC_H)
+#define RGX_COMPAT_BVNC_H
+
+#include "img_types.h"
+
+#if defined(RGX_FIRMWARE)               /* Services firmware */
+# include "rgxfw_utils.h"
+# define PVR_COMPAT_ASSERT RGXFW_ASSERT
+#elif !defined(RGX_BUILD_BINARY)        /* Services host driver code */
+# include "pvr_debug.h"
+# define PVR_COMPAT_ASSERT PVR_ASSERT
+#else                                   /* FW user-mode tools */
+# include <assert.h>
+# define PVR_COMPAT_ASSERT assert
+#endif
+
+/* 64bit endian conversion macros */
+#if defined(__BIG_ENDIAN__)
+#define RGX_INT64_TO_BE(N) (N)
+#define RGX_INT64_FROM_BE(N) (N)
+#define RGX_INT32_TO_BE(N) (N)
+#define RGX_INT32_FROM_BE(N) (N)
+#else
+#define RGX_INT64_TO_BE(N)        \
+	((((N) >> 56)   & 0xff)       \
+	 | (((N) >> 40) & 0xff00)     \
+	 | (((N) >> 24) & 0xff0000)   \
+	 | (((N) >> 8)  & 0xff000000U) \
+	 | ((N)                << 56) \
+	 | (((N) & 0xff00)     << 40) \
+	 | (((N) & 0xff0000)   << 24) \
+	 | (((N) & 0xff000000U) << 8))
+#define RGX_INT64_FROM_BE(N) RGX_INT64_TO_BE(N)
+
+#define RGX_INT32_TO_BE(N)   \
+	((((N) >> 24)  & 0xff)   \
+	 | (((N) >> 8) & 0xff00) \
+	 | ((N)           << 24) \
+	 | ((((N) & 0xff00) << 8)))
+#define RGX_INT32_FROM_BE(N) RGX_INT32_TO_BE(N)
+#endif
+
+/******************************************************************************
+ * RGX Version packed into 64-bit (BVNC) to be used by Compatibility Check
+ *****************************************************************************/
+
+#define RGX_BVNC_PACK_SHIFT_B 48
+#define RGX_BVNC_PACK_SHIFT_V 32
+#define RGX_BVNC_PACK_SHIFT_N 16
+#define RGX_BVNC_PACK_SHIFT_C 0
+
+#define RGX_BVNC_PACK_MASK_B (IMG_UINT64_C(0xFFFF000000000000))
+#define RGX_BVNC_PACK_MASK_V (IMG_UINT64_C(0x0000FFFF00000000))
+#define RGX_BVNC_PACK_MASK_N (IMG_UINT64_C(0x00000000FFFF0000))
+#define RGX_BVNC_PACK_MASK_C (IMG_UINT64_C(0x000000000000FFFF))
+
+#define RGX_BVNC_PACKED_EXTR_B(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_B) >> RGX_BVNC_PACK_SHIFT_B))
+#define RGX_BVNC_PACKED_EXTR_V(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_V) >> RGX_BVNC_PACK_SHIFT_V))
+#define RGX_BVNC_PACKED_EXTR_N(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_N) >> RGX_BVNC_PACK_SHIFT_N))
+#define RGX_BVNC_PACKED_EXTR_C(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_C) >> RGX_BVNC_PACK_SHIFT_C))
+
+#define RGX_BVNC_EQUAL(L,R,all,version,bvnc) do {															\
+										(bvnc) = IMG_FALSE;													\
+										(version) = ((L).ui32LayoutVersion == (R).ui32LayoutVersion);		\
+										if (version)														\
+										{																	\
+											(bvnc) = ((L).ui64BVNC == (R).ui64BVNC);						\
+										}																	\
+										(all) = (version) && (bvnc);										\
+									} while (false)
+
+
+/**************************************************************************//**
+ * Utility function for packing BVNC
+ *****************************************************************************/
+static inline IMG_UINT64 rgx_bvnc_pack(IMG_UINT32 ui32B, IMG_UINT32 ui32V, IMG_UINT32 ui32N, IMG_UINT32 ui32C)
+{
+	/*
+	 * Test for input B, V, N and C exceeding max bit width.
+	 */
+	PVR_COMPAT_ASSERT((ui32B & (~(RGX_BVNC_PACK_MASK_B >> RGX_BVNC_PACK_SHIFT_B))) == 0U);
+	PVR_COMPAT_ASSERT((ui32V & (~(RGX_BVNC_PACK_MASK_V >> RGX_BVNC_PACK_SHIFT_V))) == 0U);
+	PVR_COMPAT_ASSERT((ui32N & (~(RGX_BVNC_PACK_MASK_N >> RGX_BVNC_PACK_SHIFT_N))) == 0U);
+	PVR_COMPAT_ASSERT((ui32C & (~(RGX_BVNC_PACK_MASK_C >> RGX_BVNC_PACK_SHIFT_C))) == 0U);
+
+	return (((IMG_UINT64)ui32B << RGX_BVNC_PACK_SHIFT_B) |
+			((IMG_UINT64)ui32V << RGX_BVNC_PACK_SHIFT_V) |
+			((IMG_UINT64)ui32N << RGX_BVNC_PACK_SHIFT_N) |
+			((IMG_UINT64)ui32C << RGX_BVNC_PACK_SHIFT_C));
+}
+
+
+#endif /* RGX_COMPAT_BVNC_H */
+
+/******************************************************************************
+ End of file (rgx_compat_bvnc.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_resetframework.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_resetframework.h
new file mode 100644
index 000000000000..e60bafd84536
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_resetframework.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_resetframework.h
+@Title          Post-reset work-around framework FW interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_RESETFRAMEWORK_H)
+#define RGX_FWIF_RESETFRAMEWORK_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+
+typedef struct
+{
+	union
+	{
+		IMG_UINT64	uCDMReg_CDM_CB_BASE;			//  defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+		IMG_UINT64	uCDMReg_CDM_CTRL_STREAM_BASE;	// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+	};
+	IMG_UINT64	uCDMReg_CDM_CB_QUEUE;				// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+	IMG_UINT64	uCDMReg_CDM_CB;						// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+} RGXFWIF_RF_REGISTERS;
+
+typedef struct
+{
+	/* THIS MUST BE THE LAST MEMBER OF THE CONTAINING STRUCTURE */
+	RGXFWIF_RF_REGISTERS RGXFW_ALIGN sFWRegisters;
+
+} RGXFWIF_RF_CMD;
+
+/* to opaquely allocate and copy in the kernel */
+#define RGXFWIF_RF_CMD_SIZE  sizeof(RGXFWIF_RF_CMD)
+
+#endif /* RGX_FWIF_RESETFRAMEWORK_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_sf.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_sf.h
new file mode 100644
index 000000000000..a190875b5e6d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_sf.h
@@ -0,0 +1,1067 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_sf.h
+@Title          RGX firmware interface string format specifiers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the rgx firmware logging messages. The following
+                list are the messages the firmware prints. Changing anything
+                but the first column or spelling mistakes in the strings will
+                break compatibility with log files created with older/newer
+                firmware versions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_SF_H
+#define RGX_FWIF_SF_H
+
+/******************************************************************************
+ * *DO*NOT* rearrange or delete lines in SFIDLIST or SFGROUPLIST or you
+ *           WILL BREAK fw tracing message compatibility with previous
+ *           fw versions. Only add new ones, if so required.
+ *****************************************************************************/
+/* Available log groups */
+#define RGXFW_LOG_SFGROUPLIST       \
+	X(RGXFW_GROUP_NULL,NULL)        \
+	X(RGXFW_GROUP_MAIN,MAIN)        \
+	X(RGXFW_GROUP_CLEANUP,CLEANUP)  \
+	X(RGXFW_GROUP_CSW,CSW)          \
+	X(RGXFW_GROUP_PM, PM)           \
+	X(RGXFW_GROUP_RTD,RTD)          \
+	X(RGXFW_GROUP_SPM,SPM)          \
+	X(RGXFW_GROUP_MTS,MTS)          \
+	X(RGXFW_GROUP_BIF,BIF)          \
+	X(RGXFW_GROUP_MISC,MISC)        \
+	X(RGXFW_GROUP_POW,POW)          \
+	X(RGXFW_GROUP_HWR,HWR)          \
+	X(RGXFW_GROUP_HWP,HWP)          \
+	X(RGXFW_GROUP_RPM,RPM)          \
+	X(RGXFW_GROUP_DMA,DMA)          \
+	X(RGXFW_GROUP_DBG,DBG)          \
+	X(RGXFW_GROUP_VZ,VZ)            \
+	X(RGXFW_GROUP_SAFETY,SAFETY)    \
+	X(RGXFW_GROUP_VERBOSE,VERBOSE)  \
+	X(RGXFW_GROUP_CUSTOMER,CUSTOMER)
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief FW Trace log groups(GID) list
+ */
+enum RGXFW_LOG_SFGROUPS {
+#define X(A,B) A,
+	RGXFW_LOG_SFGROUPLIST
+#undef X
+};
+
+#define IMG_SF_STRING_MAX_SIZE 256U
+
+typedef struct {
+	IMG_UINT32 ui32Id;
+	IMG_CHAR sName[IMG_SF_STRING_MAX_SIZE];
+} RGXFW_STID_FMT; /*  pair of string format id and string formats */
+
+typedef struct {
+	IMG_UINT32 ui32Id;
+	const IMG_CHAR *psName;
+} RGXKM_STID_FMT; /*  pair of string format id and string formats */
+
+/* Table of String Format specifiers, the group they belong and the number of
+ * arguments each expects. Xmacro styled macros are used to generate what is
+ * needed without requiring hand editing.
+ *
+ * id		: id within a group
+ * gid		: group id
+ * Sym name	: name of enumerations used to identify message strings
+ * String	: Actual string
+ * #args	: number of arguments the string format requires
+ */
+#define RGXFW_LOG_SFIDLIST \
+/*id, gid,              id name,        string,                           # arguments */ \
+X(  0, RGXFW_GROUP_NULL, RGXFW_SF_FIRST, "You should not use this string", 0) \
+\
+X(  1, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D_DEPRECATED, "Kick 3D: FWCtx 0x%08.8x @ %u, RTD 0x%08x. Partial render:%u, CSW resume:%u, prio:%d", 6) \
+X(  2, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_FINISHED, "3D finished, HWRTData0State=%x, HWRTData1State=%x", 2) \
+X(  3, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK3D_TQ_DEPRECATED, "Kick 3D TQ: FWCtx 0x%08.8x @ %u, CSW resume:%u, prio: %d", 4) \
+X(  4, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_TQ_FINISHED, "3D Transfer finished", 0) \
+X(  5, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_DEPRECATED, "Kick Compute: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X(  6, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_FINISHED, "Compute finished", 0) \
+X(  7, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA_DEPRECATED, "Kick TA: FWCtx 0x%08.8x @ %u, RTD 0x%08x. First kick:%u, Last kick:%u, CSW resume:%u, prio:%d", 7) \
+X(  8, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_FINISHED, "TA finished", 0) \
+X(  9, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_RESTART_AFTER_PRENDER, "Restart TA after partial render", 0) \
+X( 10, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_RESUME_WOUT_PRENDER, "Resume TA without partial render", 0) \
+X( 11, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OOM, "Out of memory! Context 0x%08x, HWRTData 0x%x", 2) \
+X( 12, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TLA_DEPRECATED, "Kick TLA: FWCtx 0x%08.8x @ %u, prio:%d", 3) \
+X( 13, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TLA_FINISHED, "TLA finished", 0) \
+X( 14, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CCCB_WOFF_UPDATE, "cCCB Woff update = %u, DM = %u, FWCtx = 0x%08.8x", 3) \
+X( 16, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_START, "UFO Checks for FWCtx 0x%08.8x @ %u", 2) \
+X( 17, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK, "UFO Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 18, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_SUCCEEDED, "UFO Checks succeeded", 0) \
+X( 19, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_PR_CHECK, "UFO PR-Check: [0x%08.8x] is 0x%08.8x requires >= 0x%08.8x", 3) \
+X( 20, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK_START, "UFO SPM PR-Checks for FWCtx 0x%08.8x", 1) \
+X( 21, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK_DEPRECATED, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires >= ????????, [0x%08.8x] is ???????? requires 0x%08.8x", 4) \
+X( 22, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_UPDATE_START, "UFO Updates for FWCtx 0x%08.8x @ %u", 2) \
+X( 23, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_UPDATE, "UFO Update: [0x%08.8x] = 0x%08.8x", 2) \
+X( 24, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ASSERT_FAILED, "ASSERT Failed: line %u of:", 1) \
+X( 25, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_LOCKUP_DEPRECATED, "HWR: Lockup detected on DM%u, FWCtx: 0x%08.8x", 2) \
+X( 26, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_RESET_FW_DEPRECATED, "HWR: Reset fw state for DM%u, FWCtx: 0x%08.8x, MemCtx: 0x%08.8x", 3) \
+X( 27, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_RESET_HW_DEPRECATED, "HWR: Reset HW", 0) \
+X( 28, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_TERMINATED_DEPRECATED, "HWR: Lockup recovered.", 0) \
+X( 29, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_FALSE_LOCKUP_DEPRECATED, "HWR: False lockup detected for DM%u", 1) \
+X( 30, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ALIGN_FAILED, "Alignment check %u failed: host = 0x%x, fw = 0x%x", 3) \
+X( 31, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GP_USC_TRIGGERED, "GP USC triggered", 0) \
+X( 32, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_OVERALLOC_REGS, "Overallocating %u temporary registers and %u shared registers for breakpoint handler", 2) \
+X( 33, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET_DEPRECATED, "Setting breakpoint: Addr 0x%08.8x", 1) \
+X( 34, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_STORE, "Store breakpoint state", 0) \
+X( 35, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_UNSET, "Unsetting BP Registers", 0) \
+X( 36, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NONZERO_RT, "Active RTs expected to be zero, actually %u", 1) \
+X( 37, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTC_PRESENT, "RTC present, %u active render targets", 1) \
+X( 38, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_EST_POWER_DEPRECATED, "Estimated Power 0x%x", 1) \
+X( 39, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTA_TARGET, "RTA render target %u", 1) \
+X( 40, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTA_KICK_RENDER, "Kick RTA render %u of %u", 2) \
+X( 41, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_SIZES_CHECK_DEPRECATED, "HWR sizes check %u failed: addresses = %u, sizes = %u", 3) \
+X( 42, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_ENABLE_DEPRECATED, "Pow: DUSTS_ENABLE = 0x%x", 1) \
+X( 43, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_HWREQ_DEPRECATED, "Pow: On(1)/Off(0): %u, Units: 0x%08.8x", 2) \
+X( 44, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_CHANGE_DEPRECATED, "Pow: Changing number of dusts from %u to %u", 2) \
+X( 45, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_SIDEKICK_IDLE_DEPRECATED, "Pow: Sidekick ready to be powered down", 0) \
+X( 46, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_CHANGE_REQ_DEPRECATED, "Pow: Request to change num of dusts to %u (bPowRascalDust=%u)", 2) \
+X( 47, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PARTIALRENDER_WITHOUT_ZSBUFFER_STORE, "No ZS Buffer used for partial render (store)", 0) \
+X( 48, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PARTIALRENDER_WITHOUT_ZSBUFFER_LOAD, "No Depth/Stencil Buffer used for partial render (load)", 0) \
+X( 49, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_SET_LOCKUP_DEPRECATED, "HWR: Lock-up DM%u FWCtx: 0x%08.8x", 2) \
+X( 50, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_REG_VALUE_DEPRECATED, "MLIST%u checker: CatBase TE=0x%08x (%u Pages), VCE=0x%08x (%u Pages), ALIST=0x%08x, IsTA=%u", 7) \
+X( 51, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_MLIST_VALUE, "MLIST%u checker: MList[%u] = 0x%08x", 3) \
+X( 52, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_OK, "MLIST%u OK", 1) \
+X( 53, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_EMPTY, "MLIST%u is empty", 1) \
+X( 54, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_REG_VALUE, "MLIST%u checker: CatBase TE=0x%08x%08x, VCE=0x%08x%08x, ALIST=0x%08x%08x, IsTA=%u", 8) \
+X( 55, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_40480KICK, "3D OQ flush kick", 0) \
+X( 56, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWP_UNSUPPORTED_BLOCK, "HWPerf block ID (0x%x) unsupported by device", 1) \
+X( 57, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET_DEPRECATED2, "Setting breakpoint: Addr 0x%08.8x DM%u", 2) \
+X( 58, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED, "Kick RTU: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X( 59, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTU_FINISHED_DEPRECATED, "RDM finished on context %u", 1) \
+X( 60, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_SHG_DEPRECATED, "Kick SHG: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X( 61, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SHG_FINISHED_DEPRECATED, "SHG finished", 0) \
+X( 62, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBA_FINISHED_DEPRECATED, "FBA finished on context %u", 1) \
+X( 63, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_FAILED, "UFO Checks failed", 0) \
+X( 64, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_START, "Kill DM%u start", 1) \
+X( 65, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_COMPLETE, "Kill DM%u complete", 1) \
+X( 66, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FC_CCB_UPDATE_DEPRECATED, "FC%u cCCB Woff update = %u", 2) \
+X( 67, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED2, "Kick RTU: FWCtx 0x%08.8x @ %u, prio: %d, Frame Context: %u", 4) \
+X( 68, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_INIT, "GPU init", 0) \
+X( 69, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNITS_INIT, "GPU Units init (# mask: 0x%x)", 1) \
+X( 70, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGTIMES, "Register access cycles: read: %u cycles, write: %u cycles, iterations: %u", 3) \
+X( 71, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGCONFIG_ADD, "Register configuration added. Address: 0x%x Value: 0x%x%x", 3) \
+X( 72, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGCONFIG_SET, "Register configuration applied to type %u. (0:pow on, 1:Rascal/dust init, 2-5: TA,3D,CDM,TLA, 6:All)", 1) \
+X( 73, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TPC_FLUSH, "Perform TPC flush.", 0) \
+X( 74, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP_DEPRECATED, "GPU has locked up (see HWR logs for more info)", 0) \
+X( 75, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_OUTOFTIME, "HWR has been triggered - GPU has overrun its deadline (see HWR logs)", 0) \
+X( 76, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_POLLFAILURE_DEPRECATED, "HWR has been triggered - GPU has failed a poll (see HWR logs)", 0) \
+X( 77, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DOPPLER_OOM_DEPRECATED, "Doppler out of memory event for FC %u", 1) \
+X( 78, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK1, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires >= 0x%08.8x", 3) \
+X( 79, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK2, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 80, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIMESTAMP, "TIMESTAMP -> [0x%08.8x]", 1) \
+X( 81, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_RMW_UPDATE_START, "UFO RMW Updates for FWCtx 0x%08.8x @ %u", 2) \
+X( 82, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_RMW_UPDATE, "UFO Update: [0x%08.8x] = 0x%08.8x", 2) \
+X( 83, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NULLCMD, "Kick Null cmd: FWCtx 0x%08.8x @ %u", 2) \
+X( 84, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RPM_OOM_DEPRECATED, "RPM Out of memory! Context 0x%08x, SH requestor %u", 2) \
+X( 85, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTU_ABORT_DISCARD_DEPRECATED, "Discard RTU due to RPM abort: FWCtx 0x%08.8x @ %u, prio: %d, Frame Context: %u", 4) \
+X( 86, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED, "Deferring DM%u from running context 0x%08x @ %u (deferred DMs = 0x%08x)", 4) \
+X( 87, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED_WAITING_TURN_DEPRECATED, "Deferring DM%u from running context 0x%08x @ %u to let other deferred DMs run (deferred DMs = 0x%08x)", 4) \
+X( 88, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED_NO_LONGER, "No longer deferring DM%u from running context = 0x%08x @ %u (deferred DMs = 0x%08x)", 4) \
+X( 89, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_FWCCB_DEPRECATED, "FWCCB for DM%u is full, we will have to wait for space! (Roff = %u, Woff = %u)", 3) \
+X( 90, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_FWCCB, "FWCCB for Driver ID %u is full, we will have to wait for space! (Roff = %u, Woff = %u)", 3) \
+X( 91, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SYNC_PART, "Host Sync Partition marker: %u", 1) \
+X( 92, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SYNC_PART_RPT, "Host Sync Partition repeat: %u", 1) \
+X( 93, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CLOCK_SPEED_CHANGE, "Core clock set to %u Hz", 1) \
+X( 94, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_OFFSETS, "Compute Queue: FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 7) \
+X( 95, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_DEPRECATED, "Signal check failed, Required Data: 0x%x, Address: 0x%08x%08x", 3) \
+X( 96, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE_DEPRECATED, "Signal update, Snoop Filter: %u, MMU Ctx: %u, Signal Id: %u, Signals Base: 0x%08x%08x", 5) \
+X( 97, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWCONTEXT_SIGNALED, "Signalled the previously waiting FWCtx: 0x%08.8x, OSId: %u, Signal Address: 0x%08x%08x", 4) \
+X( 98, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_STALLED_DEPRECATED, "Compute stalled", 0) \
+X( 99, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_STALLED, "Compute stalled (Roff = %u, Woff = %u, Size = %u)", 3) \
+X(100, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_RESUMED_FROM_STALL, "Compute resumed (Roff = %u, Woff = %u, Size = %u)", 3) \
+X(101, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOTIFY_SIGNAL_UPDATE, "Signal update notification from the host, PC Physical Address: 0x%08x%08x, Signal Virtual Address: 0x%08x%08x", 4) \
+X(102, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE_OSID_DM_DEPRECATED, "Signal update from DM: %u, OSId: %u, PC Physical Address: 0x%08x%08x", 4) \
+X(103, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_DM_DEPRECATED, "DM: %u signal check failed", 1) \
+X(104, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED, "Kick TDM: FWCtx 0x%08.8x @ %u, prio:%d", 3) \
+X(105, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_FINISHED, "TDM finished", 0) \
+X(106, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TE_PIPE_STATUS_DEPRECATED, "MMU_PM_CAT_BASE_TE[%u]_PIPE[%u]:  0x%08x 0x%08x)", 4) \
+X(107, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_HIT_DEPRECATED, "BRN 54141 HIT", 0) \
+X(108, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_APPLYING_DUMMY_TA_DEPRECATED, "BRN 54141 Dummy TA kicked", 0) \
+X(109, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_RESUME_TA_DEPRECATED, "BRN 54141 resume TA", 0) \
+X(110, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_DOUBLE_HIT_DEPRECATED, "BRN 54141 double hit after applying WA", 0) \
+X(111, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_DUMMY_TA_VDM_BASE_DEPRECATED, "BRN 54141 Dummy TA VDM base address: 0x%08x%08x", 2) \
+X(112, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_WITH_CURRENT, "Signal check failed, Required Data: 0x%x, Current Data: 0x%x, Address: 0x%08x%08x", 4) \
+X(113, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BUFFER_STALL_DEPRECATED, "TDM stalled (Roff = %u, Woff = %u)", 2) \
+X(114, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOTIFY_WRITE_OFFSET_UPDATE, "Write Offset update notification for stalled FWCtx 0x%08.8x", 1) \
+X(115, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED, "Changing OSid %u's priority from %u to %u", 3) \
+X(116, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_RESUMED, "Compute resumed", 0) \
+X(117, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TLA, "Kick TLA: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(118, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED3, "Kick TDM: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(119, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA, "Kick TA: FWCtx 0x%08.8x @ %u, RTD 0x%08x, First kick:%u, Last kick:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(120, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D, "Kick 3D: FWCtx 0x%08.8x @ %u, RTD 0x%08x, Partial render:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 10) \
+X(121, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3DTQ, "Kick 3D TQ: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(122, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_DEPRECATED2, "Kick Compute: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 6) \
+X(123, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED3, "Kick RTU: FWCtx 0x%08.8x @ %u, Frame Context:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(124, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_SHG_DEPRECATED2, "Kick SHG: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(125, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CSRM_RECONFIG, "Reconfigure CSRM: special coeff support enable %u.", 1) \
+X(127, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_REQ_MAX_COEFFS, "TA requires max coeff mode, deferring: %u.", 1) \
+X(128, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_REQ_MAX_COEFFS, "3D requires max coeff mode, deferring: %u.", 1) \
+X(129, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_FAILED, "Kill DM%u failed", 1) \
+X(130, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_QUEUE, "Thread Queue is full, we will have to wait for space! (Roff = %u, Woff = %u)", 2) \
+X(131, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_QUEUE_FENCE, "Thread Queue is fencing, we are waiting for Roff = %u (Roff = %u, Woff = %u)", 3) \
+X(132, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_HCS_TRIGGERED_DEPRECATED, "DM %u failed to Context Switch on time. Triggered HCS (see HWR logs).", 1) \
+X(133, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HCS_SET_DEPRECATED, "HCS changed to %u ms", 1) \
+X(134, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UPDATE_TILES_IN_FLIGHT_DEPRECATED, "Updating Tiles In Flight (Dusts=%u, PartitionMask=0x%08x, ISPCtl=0x%08x%08x)", 4) \
+X(135, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_TILES_IN_FLIGHT, "  Phantom %u: USCTiles=%u", 2) \
+X(136, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ISOLATION_CONF_OFF_DEPRECATED, "Isolation grouping is disabled", 0) \
+X(137, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ISOLATION_CONF_DEPRECATED, "Isolation group configured with a priority threshold of %u", 1) \
+X(138, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_ONLINE_DEPRECATED, "OS %u has come online", 1) \
+X(139, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_OFFLINE_DEPRECATED, "OS %u has gone offline", 1) \
+X(140, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWCONTEXT_SIGNAL_REKICK, "Signalled the previously stalled FWCtx: 0x%08.8x, OSId: %u, Signal Address: 0x%08x%08x", 4) \
+X(141, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSETS_DEPRECATED, "TDM Queue: FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 7) \
+X(142, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSET_READ_RESET, "Reset TDM Queue Read Offset: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u becomes 0, Woff = %u, Size = %u)", 6) \
+X(143, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UMQ_MISMATCHED_READ_OFFSET, "User Mode Queue mismatched stream start: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u, StreamStartOffset = %u)", 5) \
+X(144, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_DEINIT, "GPU deinit", 0) \
+X(145, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNITS_DEINIT, "GPU units deinit", 0) \
+X(146, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CONFIG_DEPRECATED, "Initialised OS %u with config flags 0x%08x", 2) \
+X(147, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_LIMIT, "UFO limit exceeded %u/%u", 2) \
+X(148, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_62850KICK, "3D Dummy stencil store", 0) \
+X(149, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CONFIG_DEPRECATED2, "Initialised OS %u with config flags 0x%08x and extended config flags 0x%08x", 3) \
+X(150, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_COMMAND_DEPRECATED, "Unknown Command (eCmdType=0x%08x)", 1) \
+X(151, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE, "UFO forced update: FWCtx 0x%08.8x @ %u [0x%08.8x] = 0x%08.8x", 4) \
+X(152, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE_NOP_DEPRECATED, "UFO forced update NOP: FWCtx 0x%08.8x @ %u [0x%08.8x] = 0x%08.8x, reason %u", 5) \
+X(153, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN66075_CHECK, "TDM context switch check: Roff %u points to 0x%08x, Match=%u", 3) \
+X(154, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CCBS_DEPRECATED, "Driver ID %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x", 6) \
+X(155, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWIRQ, "FW IRQ # %u @ %u", 2) \
+X(156, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET, "Setting breakpoint: Addr 0x%08.8x DM%u usc_breakpoint_ctrl_dm = %u", 3) \
+X(157, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KERNEL_CCB_DEPRECATED, "Invalid KCCB setup for OSid %u: KCCB 0x%08x, KCCB Ctrl 0x%08x", 3) \
+X(158, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KERNEL_CCB_CMD, "Invalid KCCB cmd (%u) for Driver ID %u @ KCCB 0x%08x", 3) \
+X(159, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_FAULT_DEPRECATED, "FW FAULT: At line %u in file 0x%08x%08x, additional data=0x%08x", 4) \
+X(160, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_INVALID, "Invalid breakpoint: MemCtx 0x%08x Addr 0x%08.8x DM%u usc_breakpoint_ctrl_dm = %u", 4) \
+X(161, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FLUSHINVAL_CMD_INVALID_DEPRECATED, "Discarding invalid SLC flushinval command for OSid %u: DM %u, FWCtx 0x%08x", 3) \
+X(162, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NOTIFY_WRITE_OFFSET_UPDATE_DEPRECATED, "Invalid Write Offset update notification from OSid %u to DM %u: FWCtx 0x%08x, MemCtx 0x%08x", 4) \
+X(163, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KCCB_KICK_CMD_DEPRECATED, "Null FWCtx in KCCB kick cmd for OSid %u: KCCB 0x%08x, ROff %u, WOff %u", 4) \
+X(164, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FULL_CHPTCCB, "Checkpoint CCB for Driver ID %u is full, signalling host for full check state (Roff = %u, Woff = %u)", 3) \
+X(165, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CCBS_DEPRECATED2, "OSid %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x chptCCBCtl@0x%x chptCCB@0x%x", 8) \
+X(166, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_STATE_CHANGE_DEPRECATED, "Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-offloading 4-cooldown). Status %u (1-ok 0-fail)", 4) \
+X(167, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_STALE_KCCB_CMDS, "Driver ID %u has %u stale commands in its KCCB", 2) \
+X(168, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_VCE_PAUSE, "Applying VCE pause", 0) \
+X(169, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KCCB_UPDATE_RTN_SLOT_DEPRECATED, "OSid %u KCCB slot %u value updated to %u", 3) \
+X(170, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_KCCB_COMMAND, "Unknown KCCB Command: KCCBCtl=0x%08x, KCCB=0x%08x, Roff=%u, Woff=%u, Wrap=%u, Cmd=0x%08x, CmdType=0x%08x", 7) \
+X(171, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_CCB_COMMAND1, "Unknown Client CCB Command processing fences: FWCtx=0x%08x, CCBCtl=0x%08x, CCB=0x%08x, Roff=%u, Doff=%u, Woff=%u, Wrap=%u, CmdHdr=0x%08x, CmdType=0x%08x, CmdSize=%u", 10) \
+X(172, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_CCB_COMMAND2, "Unknown Client CCB Command executing kick: FWCtx=0x%08x, CCBCtl=0x%08x, CCB=0x%08x, Roff=%u, Doff=%u, Woff=%u, Wrap=%u, CmdHdr=0x%08x, CmdType=0x%08x, CmdSize=%u", 10) \
+X(173, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KCCB_KICK_CMD, "Null FWCtx in KCCB kick cmd for Driver ID %u with WOff %u", 2) \
+X(174, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FLUSHINVAL_CMD_INVALID, "Discarding invalid SLC flushinval command for Driver ID %u, FWCtx 0x%08x", 2) \
+X(175, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NOTIFY_WRITE_OFFSET_UPDATE, "Invalid Write Offset update notification from Driver ID %u: FWCtx 0x%08x, MemCtx 0x%08x", 3) \
+X(176, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_INIT_CONFIG, "Initialised Firmware with config flags 0x%08x and extended config flags 0x%08x", 2) \
+X(177, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_CONFIG, "Set Periodic Hardware Reset Mode: %u", 1) \
+X(179, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_TRIG, "PHR mode %u, FW state: 0x%08x, HWR flags: 0x%08x", 3) \
+X(180, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_RESET_DEPRECATED, "PHR mode %u triggered a reset", 1) \
+X(181, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE, "Signal update, Snoop Filter: %u, Signal Id: %u", 2) \
+X(182, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEV_SERIES8_DEPRECATED, "WARNING: Skipping FW KCCB Cmd type %u which is not yet supported on Series8.", 1) \
+X(183, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INCONSISTENT_MMU_FLAGS, "MMU context cache data NULL, but cache flags=0x%x (sync counter=%u, update value=%u) OSId=%u", 4) \
+X(184, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SLC_FLUSH, "SLC range based flush: Context=%u VAddr=0x%02x%08x, Size=0x%08x, Invalidate=%u", 5) \
+X(185, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBSC_INVAL_DEPRECATED, "FBSC invalidate for Context Set [0x%08x]: Entry mask 0x%08x%08x.", 3) \
+X(186, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN66284_UPDATE, "TDM context switch check: Roff %u was not valid for kick starting at %u, moving back to %u", 3) \
+X(187, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SPFILTER_UPDATES, "Signal updates: FIFO: %u, Signals: 0x%08x", 2) \
+X(188, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_FBSC_CMD, "Invalid FBSC cmd: FWCtx 0x%08x, MemCtx 0x%08x", 2) \
+X(189, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN68497_BLIT, "Insert BRN68497 WA blit after TDM Context store.", 0) \
+X(190, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PENDING_UFO_UPDATE_START, "UFO Updates for previously finished FWCtx 0x%08.8x", 1) \
+X(191, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTC_RTA_PRESENT, "RTC with RTA present, %u active render targets", 1) \
+X(192, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NULL_RTAS, "Invalid RTA Set-up. The ValidRenderTargets array in RTACtl is Null!", 0) \
+X(193, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_COUNTER, "Block 0x%x / Counter 0x%x INVALID and ignored", 2) \
+X(194, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ECC_FAULT_DEPRECATED, "ECC fault GPU=0x%08x FW=0x%08x", 2) \
+X(195, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PROCESS_XPU_EVENT, "Processing XPU event on DM = %u", 1) \
+X(196, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_VZ_WDG_TRIGGER_DEPRECATED, "Driver ID %u failed to respond to the virtualisation watchdog in time. Timestamp of its last input = %u", 2) \
+X(197, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP, "GPU-%u has locked up (see HWR logs for more info)", 1) \
+X(198, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UPDATE_TILES_IN_FLIGHT, "Updating Tiles In Flight (Dusts=%u, PartitionMask=0x%08x, ISPCtl=0x%08x)", 3) \
+X(199, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP_DM, "GPU has locked up (see HWR logs for more info)", 0) \
+X(200, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REPROCESS_XPU_EVENTS, "Reprocessing outstanding XPU events from cores 0x%02x", 1) \
+X(201, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SECONDARY_XPU_EVENT, "Secondary XPU event on DM=%u, CoreMask=0x%02x, Raised=0x%02x", 3) \
+X(202, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSETS, "TDM Queue: Core %u, FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 8) \
+X(203, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BUFFER_STALL, "TDM stalled Core %u (Roff = %u, Woff = %u)", 3) \
+X(204, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_OFFSETS, "Compute Queue: Core %u, FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 8) \
+X(205, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_STALLED, "Compute stalled core %u (Roff = %u, Woff = %u, Size = %u)", 4) \
+X(206, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UMQ_MISMATCHED_CORE_READ_OFFSET, "User Mode Queue mismatched stream start: Core %u, FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u, StreamStartOffset = %u)", 6) \
+X(207, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_RESUMED_FROM_STALL, "TDM resumed core %u (Roff = %u, Woff = %u)", 3) \
+X(208, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_RESUMED_FROM_STALL, "Compute resumed core %u (Roff = %u, Woff = %u, Size = %u)", 4) \
+X(209, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_MTS_PERMISSION_CHANGED_DEPRECATED, " Updated permission for Driver ID %u to perform MTS kicks: %u (1 = allowed, 0 = not allowed)", 2) \
+X(210, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TEST1, "Mask = 0x%X, mask2 = 0x%X", 2) \
+X(211, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TEST2, "  core %u, reg = %u, mask = 0x%X)", 3) \
+X(212, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ECC_FAULT_SAFETY_BUS, "ECC fault received from safety bus: 0x%08x", 1) \
+X(213, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SAFETY_WDG_CONFIG, "Safety Watchdog threshold period set to 0x%x clock cycles", 1) \
+X(214, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SAFETY_WDG_TRIGGER, "MTS Safety Event triggered by the safety watchdog.", 0) \
+X(215, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_USC_TASKS_RANGE, "DM%u USC tasks range limit 0 - %u, stride %u", 3) \
+X(216, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_ECC_FAULT, "ECC fault GPU=0x%08x", 1) \
+X(217, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_SAFETY_RESET, "GPU Hardware units reset to prevent transient faults.", 0) \
+X(218, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ABORTCMD, "Kick Abort cmd: FWCtx 0x%08.8x @ %u", 2) \
+X(219, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_DEPRECATED, "Kick Ray: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7)\
+X(220, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_FINISHED_DEPRECATED, "Ray finished", 0) \
+X(221, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWDATA_INIT_STATUS, "State of firmware's private data at boot time: %u (0 = uninitialised, 1 = initialised); Fw State Flags = 0x%08X", 2) \
+X(222, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CFI_TIMEOUT, "CFI Timeout detected (%u increasing to %u)", 2) \
+X(223, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CFI_TIMEOUT_FBM, "CFI Timeout detected for FBM (%u increasing to %u)", 2) \
+X(224, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GEOM_OOM_DISALLOWED, "Geom OOM event not allowed", 0) \
+X(225, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED2, "Changing Driver ID %u's priority from %u to %u; Isolation = %u (0 = off; 1 = on)", 4) \
+X(226, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SKIP_ALREADY_RUN_GEOM, "Skipping already executed TA FWCtx 0x%08.8x @ %u", 2) \
+X(227, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ATTEMPT_TO_RUN_AHEAD_GEOM, "Attempt to execute TA FWCtx 0x%08.8x @ %u ahead of time on other GEOM", 2) \
+X(228, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED2, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(229, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA_PIPELINE, "Kick TA: Kick ID %u FWCtx 0x%08.8x @ %u, RTD 0x%08x, First kick:%u, Last kick:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 12) \
+X(230, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D_PIPELINE, "Kick 3D: Kick ID %u FWCtx 0x%08.8x @ %u, RTD 0x%08x, Partial render:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(231, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_PIPELINE_DEPRECATED, "Kick Compute: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 7) \
+X(232, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_FINISHED_PIPELINE, "TDM finished: Kick ID %u", 1) \
+X(233, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_FINISHED_PIPELINE, "TA finished: Kick ID %u", 1) \
+X(234, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_FINISHED_PIPELINE, "3D finished: Kick ID %u, HWRTData0State=%x, HWRTData1State=%x", 3) \
+X(235, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_FINISHED_PIPELINE, "Compute finished: Kick ID %u", 1) \
+X(236, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_PIPELINE_DEPRECATED, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u, Base 0x%08x%08x. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 10) \
+X(237, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_PIPELINE_DEPRECATED, "Kick Ray: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8)\
+X(238, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_FINISHED_PIPELINE, "Ray finished: Kick ID %u", 1) \
+X(239, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RT_UNITS_INIT, "GPU RT Units init (# mask: 0x%08x%08x)", 2) \
+X(240, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_PENDING_PASS, "UFO Check: [0x%08.8x] is pending update to 0x%08.8x and therefore passes", 2) \
+X(241, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_PR_CHECK_PENDING_PASS, "UFO PR-Check: [0x%08.8x] is pending update to 0x%08.8x and therefore passes", 2) \
+X(242, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DELAY_DM_TO_OVERLAP_PIPES, "Holding kick of DM %u pipe %u to encourage pipeline overlap", 2) \
+X(243, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RELEASE_DM_PIPE, "Releasing kick for DM %u pipe %u", 2) \
+X(244, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED3, "Changing Driver ID %u's priority from %u to %u", 3) \
+X(245, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_ISOLATION_GROUP_CHANGE_DEPRECATED, "Changing Driver ID %u's isolation group from %u to %u", 3) \
+X(246, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_VK_TIMESTAMP, "VK Timestamp: addr=0x%08x%08x, avail=0x%08x%08x stamp=0x%08x%08x", 6) \
+X(247, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_HCS_TRIGGERED, "DM %u failed to Context Switch on time (Current time: 0x%08x%08x, deadline: 0x%08x%08x). Triggered HCS (see HWR logs).", 5) \
+X(248, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_POLLFAILURE, "HWR has been triggered - GPU has failed a poll, RGX_CR_EVENT_STATUS=0x%08x (see HWR logs)", 1) \
+X(249, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBCDC_FAILURE_DETECTED_DEPRECATED, "FBCDC signature failure detected so block scheduling more work", 0) \
+X(250, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBCDC_FAILURE_CLEARED_DEPRECATED, "FBCDC signature cleared which unlocks scheduling more work", 0) \
+X(251, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_FAULT, "FW FAULT: At line %u in file 0x%08x%08x, additional data=0x%08x%08x", 5) \
+X(252, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE_NOP, "UFO forced update NOP: FWCtx 0x%08.8x @ %u, reason %u", 3) \
+X(253, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_CONTEXT_STORED, "TDM FWCtx:0x%08.8x stored", 1) \
+X(254, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CDM_CONTEXT_STORED, "CDM FWCtx:0x%08.8x stored, resumeMask:0x%08x", 2) \
+X(255, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GEOM_CONTEXT_STORED, "GEOM FWCtx:0x%08.8x stored", 1) \
+X(256, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_CONTEXT_STORED, "3D FWCtx:0x%08.8x stored, resumeMask:0x%08x", 2) \
+X(257, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RDM_CONTEXT_STORED, "RAY FWCtx:0x%08.8x stored", 1) \
+X(258, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM, "Kick TDM: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(259, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE, "Kick Compute: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 7) \
+X(260, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_PIPELINE, "Kick Compute: Kick ID %u FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 8) \
+X(261, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_PIPELINE, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u, Base 0x%08x%08x, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(262, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_PIPELINE, "Kick Ray: Kick ID %u FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 9)\
+X(263, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DEPTH, "3D Disable Depth. ExtJobRef = 0x%08x", 1) \
+X(264, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_STENCIL, "3D Disable Stencil. ExtJobRef = 0x%08x", 1) \
+X(265, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DS_IN_3D_RUNNING, "3D Disable DS in 3D running. RenderContext 0x%08.8x ExtJobRef 0x%08x", 2) \
+X(266, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DS_IN_KICK_3D, "3D Disable DS in kick 3D. RenderContext 0x%08.8x ExtJobRef 0x%08x", 2) \
+X(267, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ADD_DISABLE_DS_QUEUE, "Add disable DS in queue. RenderContext 0x%08.8x DisableJobRef 0x%08x Uncheck %u", 3) \
+X(268, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOT_FIND_USABLE_DS_IN_QUEUE, "Not find usable DS in queue. RenderContext 0x%08.8x ExtJobRef 0x%08x DisableJobRef 0x%08x Uncheck %u index %u", 5) \
+X(269, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_LIST_FULL, "Unable to set breakpoint for MemCtx 0x%08x as the breakpoint list is full.", 1) \
+X(270, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NUM_LOG_PARAMS, "Invalid number of log parameters passed! (Group:%u ID:%u Params:%u Passed:%u)", 4) \
+X(271, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_CANCEL_PIPELINE, "TDM cancelled: Kick ID %u", 1) \
+X(272, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_CANCEL_PIPELINE, "TA cancelled: Kick ID %u", 1) \
+X(273, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_CANCEL_PIPELINE, "3D cancelled: Kick ID %u", 1) \
+X(274, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CDM_CANCEL_PIPELINE, "Compute cancelled: Kick ID %u", 1) \
+X(275, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_CANCEL_PIPELINE, "Ray cancelled: Kick ID %u", 1) \
+X(276, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_MAX_DEPRECATED, "Time Slice: Update Failed, Max total allocations exceeded 100pc", 0)\
+X(277, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_DYNAMIC_LOW_DEPRECATED, "Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 0)\
+X(278, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_UPDATE_SUCCESS_DEPRECATED, "Time Slice: Updated Successfully", 0)\
+X(279, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISCARD_FWCCB, "FWCCB for Driver ID %u is full, discarding command! (Roff = %u, Woff = %u)", 3) \
+X(280, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_RECEIVED_DEPRECATED, "Cancelling jobs with intjobref<0x%08x, DM = %u, FWCtx = 0x%08.8x", 3) \
+X(281, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_RECEIVED, "Cancelling jobs with intjobref from 0x%08x to 0x%08x, DM = %u, FWCtx = 0x%08.8x", 4) \
+X(282, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_MAX_DEPRECATED2, "Time Slice: Update Failed, Max total allocations exceeded 100pc", 0)\
+X(283, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_DYNAMIC_LOW_DEPRECATED2, "Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 0)\
+X(284, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_UPDATE_SUCCESS_DEPRECATED2, "Time Slice: Updated Successfully", 0)\
+X(285, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_STATE_CHANGE_DEPRECATED2, "Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-graceful_offloading 4-forced_offloading 5-cooldown). Status %u (1-ok 0-fail)", 4) \
+X(286, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBSC_INVAL, "FBSC invalidate for Context Set [0x%08x]", 1) \
+X(287, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_DROPPED, "Dropped cancel request with intjobref from 0x%08x to 0x%08x, DM = %u, FWCtx = 0x%08.8x", 4) \
+X(288, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ZERO_LM, "Zeroing local memory after context storing FWCtx = 0x%08.8x", 1) \
+X(289, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FAILED_IRQ_CLEAR, "Host failed to clear IRQ DriverID: %u Reg: 0x%08x",2) \
+X(290, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_ABORTED, "Kill DM%u aborted", 1) \
+X(291, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SOC_CLOCK_SPEED_CHANGE, "SOC clock set to %u Hz", 1) \
+\
+X(  1, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK_DEPRECATED, "Bg Task DM = %u, counted = %u", 2) \
+X(  2, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_COMPLETE_DEPRECATED, "Bg Task complete DM = %u", 1) \
+X(  3, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_KICK_DEPRECATED, "Irq Task DM = %u, Breq = %u, SBIrq = 0x%x", 3) \
+X(  4, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_COMPLETE_DEPRECATED, "Irq Task complete DM = %u", 1) \
+X(  5, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KICK_MTS_BG_ALL_DEPRECATED, "Kick MTS Bg task DM=All", 0) \
+X(  6, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KICK_MTS_IRQ, "Kick MTS Irq task DM=%u", 1) \
+X(  7, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYCELLTYPE_DEPRECATED, "Ready queue debug DM = %u, celltype = %u", 2) \
+X(  8, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYTORUN_DEPRECATED, "Ready-to-run debug DM = %u, item = 0x%x", 2) \
+X(  9, RGXFW_GROUP_MTS, RGXFW_SF_MTS_CMDHEADER, "Client command header DM = %u, client CCB = 0x%x, cmd = 0x%x", 3) \
+X( 10, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYTORUN, "Ready-to-run debug Driver ID = %u, DM = %u, item = 0x%x", 3) \
+X( 11, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYCELLTYPE_DEPRECATED2, "Ready queue debug DM = %u, celltype = %u, OSid = %u", 3) \
+X( 12, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK_DEPRECATED2, "Bg Task DM = %u, counted = %u, OSid = %u", 3) \
+X( 13, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_COMPLETE, "Bg Task complete DM Bitfield: %u", 1) \
+X( 14, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_COMPLETE, "Irq Task complete.", 0) \
+X( 15, RGXFW_GROUP_MTS, RGXFW_SF_MTS_CMD_DISCARD, "Discarded Command Type: %u OS ID = %u PID = %u context = 0x%08x cccb ROff = 0x%x, due to USC breakpoint hit by OS ID = %u PID = %u.", 7) \
+X( 16, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_EXEC_DEPRECATED, "KCCB Slot %u: DM=%u, Cmd=0x%08x, OSid=%u", 4) \
+X( 17, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_RTN_VALUE, "KCCB Slot %u: Return value %u", 2) \
+X( 18, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK, "Bg Task Driver ID = %u", 1) \
+X( 19, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_EXEC, "KCCB Slot %u: Cmd=0x%08x, Driver ID=%u", 3) \
+X( 20, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_KICK, "Irq Task (EVENT_STATUS=0x%08x)", 1) \
+X( 21, RGXFW_GROUP_MTS, RGXFW_SF_MTS_VZ_SIDEBAND, "VZ sideband test, kicked with DriverID=%u from MTS, OSid for test=%u", 2) \
+\
+X(  1, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FWCTX_CLEANUP, "FwCommonContext [0x%08x] cleaned", 1) \
+X(  2, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FWCTX_BUSY, "FwCommonContext [0x%08x] is busy: ReadOffset = %u, WriteOffset = %u", 3) \
+X(  3, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANUP_DEPRECATED, "HWRTData [0x%08x] for DM=%u, received cleanup request", 2) \
+X(  4, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED_FOR_DM_DEPRECATED, "HWRTData [0x%08x] HW Context cleaned for DM%u, executed commands = %u", 3) \
+X(  5, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY_DEPRECATED, "HWRTData [0x%08x] HW Context for DM%u is busy", 2) \
+X(  6, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED_DEPRECATED, "HWRTData [0x%08x] HW Context %u cleaned", 2) \
+X(  7, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FL_CLEANED, "Freelist [0x%08x] cleaned", 1) \
+X(  8, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_ZSBUFFER_CLEANED, "ZSBuffer [0x%08x] cleaned", 1) \
+X(  9, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_ZSBUFFER_BUSY, "ZSBuffer [0x%08x] is busy: submitted = %u, executed = %u", 3) \
+X( 10, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY_DEPRECATED2, "HWRTData [0x%08x] HW Context for DM%u is busy: submitted = %u, executed = %u", 4) \
+X( 11, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANUP_DEPRECATED, "HW Ray Frame data [0x%08x] for DM=%u, received cleanup request", 2) \
+X( 12, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANED_FOR_DM_DEPRECATED, "HW Ray Frame Data [0x%08x] cleaned for DM%u, executed commands = %u", 3) \
+X( 13, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_BUSY_DEPRECATED, "HW Ray Frame Data [0x%08x] for DM%u is busy: submitted = %u, executed = %u", 4) \
+X( 14, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANED_DEPRECATED, "HW Ray Frame Data [0x%08x] HW Context %u cleaned", 2) \
+X( 15, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_INVALID_REQUEST, "Discarding invalid cleanup request of type 0x%x", 1) \
+X( 16, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANUP, "Received cleanup request for HWRTData [0x%08x]", 1) \
+X( 17, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY, "HWRTData [0x%08x] HW Context is busy: submitted = %u, executed = %u", 3) \
+X( 18, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED, "HWRTData [0x%08x] HW Context %u cleaned, executed commands = %u", 3) \
+\
+X(  1, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_NEEDS_RESUME, "CDM FWCtx 0x%08.8x needs resume", 1) \
+X(  2, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_RESUME_DEPRECATED, "*** CDM FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X(  3, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_SHARED, "CDM FWCtx shared alloc size load 0x%x", 1) \
+X(  4, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_COMPLETE, "*** CDM FWCtx store complete", 0) \
+X(  5, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_START, "*** CDM FWCtx store start", 0) \
+X(  6, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_SOFT_RESET, "CDM Soft Reset", 0) \
+X(  7, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_NEEDS_RESUME, "3D FWCtx 0x%08.8x needs resume", 1) \
+X(  8, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME, "*** 3D FWCtx 0x%08.8x resume", 1) \
+X(  9, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_COMPLETE, "*** 3D context store complete", 0) \
+X( 10, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_DEPRECATED, "3D context store pipe state: 0x%08.8x 0x%08.8x 0x%08.8x", 3) \
+X( 11, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_START, "*** 3D context store start", 0) \
+X( 12, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_TQ_RESUME, "*** 3D TQ FWCtx 0x%08.8x resume", 1) \
+X( 13, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_NEEDS_RESUME, "TA FWCtx 0x%08.8x needs resume", 1) \
+X( 14, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_RESUME, "*** TA FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X( 15, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_SHARED, "TA context shared alloc size store 0x%x, load 0x%x", 2) \
+X( 16, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_COMPLETE, "*** TA context store complete", 0) \
+X( 17, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_START, "*** TA context store start", 0) \
+X( 18, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM_DEPRECATED, "Higher priority context requests DM %u, old prio:%d, new prio:%d", 3) \
+X( 19, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SET_CONTEXT_PRIORITY, "Set FWCtx 0x%x priority to %u", 2) \
+X( 20, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_DEPRECATED2, "3D context store pipe%u state: 0x%08.8x", 2) \
+X( 21, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE_DEPRECATED, "3D context resume pipe%u state: 0x%08.8x", 2) \
+X( 22, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_NEEDS_RESUME_DEPRECATED, "SHG FWCtx 0x%08.8x needs resume", 1) \
+X( 23, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_RESUME_DEPRECATED, "*** SHG FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X( 24, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_SHARED_DEPRECATED, "SHG context shared alloc size store 0x%x, load 0x%x", 2) \
+X( 25, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_STORE_COMPLETE_DEPRECATED, "*** SHG context store complete", 0) \
+X( 26, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_STORE_START_DEPRECATED, "*** SHG context store start", 0) \
+X( 27, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_PIPE_INDIRECT, "Performing TA indirection, last used pipe %u", 1) \
+X( 28, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_CTRL_STREAM_TERMINATE, "CDM context store hit ctrl stream terminate. Skip resume.", 0) \
+X( 29, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_RESUME_AB_BUFFER, "*** CDM FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x, shader state %u", 4) \
+X( 30, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STATE_BUFFER_FLIP, "TA PDS/USC state buffer flip (%u->%u)", 2) \
+X( 31, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_52563_HIT_DEPRECATED, "TA context store hit BRN 52563: vertex store tasks outstanding", 0) \
+X( 32, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_USC_POLL_FAILED, "TA USC poll failed (USC vertex task count: %u)", 1) \
+X( 33, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_DEFERRED_DEPRECATED, "TA context store deferred due to BRN 54141.", 0) \
+X( 34, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM_DEPRECATED2, "Higher priority context requests DM %u. Prios (OSid, OSid Prio, Context Prio): Current: %u, %u, %u New: %u, %u, %u", 7) \
+X( 35, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_START, "*** TDM context store start", 0) \
+X( 36, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_COMPLETE, "*** TDM context store complete", 0) \
+X( 37, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_NEEDS_RESUME_DEPRECATED, "TDM context needs resume, header [0x%08.8x, 0x%08.8x]", 2) \
+X( 38, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM, "Higher priority context requests DM %u. Prios (Driver ID, Driver ID Prio, Context Prio): Current: %u, %u, %u New: %u, %u, %u. Hard Context Switching: %u", 8) \
+X( 39, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE, "3D context store pipe %2d (%2d) state: 0x%08.8x", 3) \
+X( 40, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE, "3D context resume pipe %2d (%2d) state: 0x%08.8x", 3) \
+X( 41, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_START_VOLCANIC, "*** 3D context store start version %u (1=IPP_TILE, 2=ISP_TILE)", 1) \
+X( 42, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_VOLCANIC, "3D context store pipe%u state: 0x%08.8x%08x", 3) \
+X( 43, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE_VOLCANIC, "3D context resume pipe%u state: 0x%08.8x%08x", 3) \
+X( 44, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_IPP_STATE,  "3D context resume IPP state: 0x%08.8x%08x", 2) \
+X( 45, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_PIPES_EMPTY, "All 3D pipes empty after ISP tile mode store! IPP_status: 0x%08x", 1) \
+X( 46, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_RESUME_PIPE_STATE_DEPRECATED, "TDM context resume pipe%u state: 0x%08.8x%08x", 3) \
+X( 47, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_LEVEL4_STORE_START, "*** 3D context store start version 4", 0) \
+X( 48, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RESUME_MULTICORE, "Multicore context resume on DM%u active core mask 0x%04.4x", 2) \
+X( 49, RGXFW_GROUP_CSW, RGXFW_SF_CSW_STORE_MULTICORE, "Multicore context store on DM%u active core mask 0x%04.4x", 2) \
+X( 50, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_RESUME_PIPE_STATE, "TDM context resume Core %u, pipe%u state: 0x%08.8x%08x%08x", 5) \
+X( 51, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_STORE_COMPLETE, "*** RDM FWCtx store complete", 0) \
+X( 52, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_STORE_START, "*** RDM FWCtx store start", 0) \
+X( 53, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_NEEDS_RESUME, "RDM FWCtx 0x%08.8x needs resume", 1) \
+X( 54, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_RESUME, "RDM FWCtx 0x%08.8x resume", 1) \
+\
+X(  1, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_BIFREQ_DEPRECATED, "Activate MemCtx=0x%08x BIFreq=%u secure=%u", 3) \
+X(  2, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE, "Deactivate MemCtx=0x%08x", 1) \
+X(  3, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCREG_ALLOC_DEPRECATED, "Alloc PC reg %u", 1) \
+X(  4, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_GRAB, "Grab reg set %u refcount now %u", 2) \
+X(  5, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_UNGRAB_DEPRECATED, "Ungrab reg set %u refcount now %u", 2) \
+X(  6, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_BIFREQ_DEPRECATED, "Setup reg=%u BIFreq=%u, expect=0x%08x%08x, actual=0x%08x%08x", 6) \
+X(  7, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TRUST_DEPRECATED, "Trust enabled:%u, for BIFreq=%u", 2) \
+X(  8, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TILECFG_DEPRECATED, "BIF Tiling Cfg %u base 0x%08x%08x len 0x%08x%08x enable %u stride %u --> 0x%08x%08x", 9) \
+X(  9, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSID0, "Wrote the Value %u to OSID0, Cat Base %u, Register's contents are now 0x%08x 0x%08x", 4) \
+X( 10, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSID1, "Wrote the Value %u to OSID1, Context  %u, Register's contents are now 0x%04x", 3) \
+X( 11, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSIDx_DEPRECATED, "ui32OSid = %u, Catbase = %u, Reg Address = 0x%x, Reg index = %u, Bitshift index = %u, Val = 0x%08x%08x", 7) \
+X( 12, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY_BIFREQ_DEPRECATED, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u, BIFREQ %u", 5) \
+X( 13, RGXFW_GROUP_BIF, RGXFW_SF_BIF_UNMAP_GPU_MEMORY, "Unmap GPU memory (event status 0x%x)", 1) \
+X( 14, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_DM, "Activate MemCtx=0x%08x DM=%u secure=%u", 3) \
+X( 15, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM_DEPRECATED, "Setup reg=%u DM=%u, expect=0x%08x%08x, actual=0x%08x%08x", 6) \
+X( 16, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u", 4) \
+X( 17, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TRUST_DM, "Trust enabled:%u, for DM=%u", 2) \
+X( 18, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY_DM, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u, DM %u", 5) \
+X( 19, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM, "Setup register set=%u DM=%u, PC address=0x%08x%08x, OSid=%u, NewPCRegRequired=%u", 6) \
+X( 20, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_ALLOC, "Alloc PC set %u as register range [%u - %u]", 3) \
+X( 21, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSIDx, "ui32CoreID = %u, ui32OSid = %u, Catbase = %u, Reg Address = 0x%x, Reg index = %u, Bitshift index = %u, Val = 0x%08x%08x", 8) \
+X( 22, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM_SECURE, "Setup secure register=%u DM=%u, PC address=0x%08x%08x, OSid=%u, NewContext=%u", 6) \
+X( 23, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_DM_DEPRECATED, "Activate MemCtx=0x%08x DM=%u secure=%u CtxFlags=0x%08x", 4) \
+X( 24, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE_DEPRECATED, "Deactivate MemCtx=0x%08x CtxFlags=0x%08x", 2) \
+X( 25, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE_AND_UNGRAB_PCSET, "Deactivate MemCtx=0x%08x, ungrab reg set %u refcount now %u", 3) \
+\
+X(  1, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_WRITE, "GPIO write 0x%02x", 1) \
+X(  2, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_READ, "GPIO read 0x%02x", 1) \
+X(  3, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_ENABLED, "GPIO enabled", 0) \
+X(  4, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_DISABLED, "GPIO disabled", 0) \
+X(  5, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_STATUS, "GPIO status=%u (0=OK, 1=Disabled)", 1) \
+X(  6, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_READ, "GPIO_AP: Read address=0x%02x (%u byte(s))", 2) \
+X(  7, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_WRITE, "GPIO_AP: Write address=0x%02x (%u byte(s))", 2) \
+X(  8, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_TIMEOUT, "GPIO_AP timeout!", 0) \
+X(  9, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_ERROR, "GPIO_AP error. GPIO status=%u (0=OK, 1=Disabled)", 1) \
+X( 10, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_ALREADY_READ, "GPIO already read 0x%02x", 1) \
+X( 11, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_CHECK_BUFFER_AVAILABLE, "SR: Check buffer %u available returned %u", 2) \
+X( 12, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_WAITING_BUFFER_AVAILABLE, "SR: Waiting for buffer %u", 1) \
+X( 13, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_WAIT_BUFFER_TIMEOUT, "SR: Timeout waiting for buffer %u (after %u ticks)", 2) \
+X( 14, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_FRAME_CHECK, "SR: Skip frame check for strip %u returned %u (0=No skip, 1=Skip frame)", 2) \
+X( 15, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_REMAINING_STRIPS, "SR: Skip remaining strip %u in frame", 1) \
+X( 16, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_FRAME_SKIP_NEW_FRAME, "SR: Inform HW that strip %u is a new frame", 1) \
+X( 17, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_FRAME_TIMEOUT, "SR: Timeout waiting for INTERRUPT_FRAME_SKIP (after %u ticks)", 1) \
+X( 18, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_STRIP_MODE, "SR: Strip mode is %u", 1) \
+X( 19, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_STRIP_INDEX, "SR: Strip Render start (strip %u)", 1) \
+X( 20, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_BUFFER_RENDERED, "SR: Strip Render complete (buffer %u)", 1) \
+X( 21, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_BUFFER_FAULT, "SR: Strip Render fault (buffer %u)", 1) \
+X( 22, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_DEPRECATED2, "TRP state: %u", 1) \
+X( 23, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_FAILURE_DEPRECATED, "TRP failure: %u", 1) \
+X( 24, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SW_TRP_STATE_DEPRECATED, "SW TRP State: %u", 1) \
+X( 25, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SW_TRP_FAILURE_DEPRECATED2, "SW TRP failure: %u", 1) \
+X( 26, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HW_KICK_DEPRECATED, "HW kick event (%u)", 1) \
+X( 27, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WGP_CHECKSUMS_DEPRECATED, "GPU core (%u/%u): checksum 0x%08x vs. 0x%08x", 4) \
+X( 28, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WGP_UNIT_CHECKSUMS_DEPRECATED, "GPU core (%u/%u), unit (%u,%u): checksum 0x%08x vs. 0x%08x", 6) \
+X( 29, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_CHECK_REG_DEPRECATED, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 30, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_USC_SLOTS_CHECK_DEPRECATED, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X( 31, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_USC_REG_CHECK_DEPRECATED, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 32, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_HWRTDATA_DEPRECATED, "TRP HWRTData: 0x%08x, state: %u", 2) \
+X( 33, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_CNTX_DEPRECATED, "TRP Context: 0x%08x, state: %u", 2) \
+X( 34, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_FAILURE_CNTX_DEPRECATED, "TRP Context: 0x%08x, failure: %u", 2) \
+X( 35, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DEPRECATED, "Memory dump: Addr=0x%02x%08x, Size=%d, ContextId=%u, DM=%u", 5) \
+X( 36, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS4_DEPRECATED, "  0x%02x%08x  %08x %08x %08x %08x", 6) \
+X( 37, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS3_DEPRECATED, "  0x%02x%08x  %08x %08x %08x", 5) \
+X( 38, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS2_DEPRECATED, "  0x%02x%08x  %08x %08x", 4) \
+X( 39, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS1_DEPRECATED, "  0x%02x%08x  %08x", 3) \
+X( 40, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DEPRECATED, "Memory dump: Addr=0x%08x, Size=%d", 2) \
+X( 41, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS4_DEPRECATED, "  0x%08x  %08x %08x %08x %08x", 5) \
+X( 42, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS3_DEPRECATED, "  0x%08x  %08x %08x %08x", 4) \
+X( 43, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS2_DEPRECATED, "  0x%08x  %08x %08x", 3) \
+X( 44, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS1_DEPRECATED, "  0x%08x  %08x", 2) \
+X( 45, RGXFW_GROUP_MISC, RGXFW_SF_MISC_FBCDC_FAILURE_STATUS_DEPRECATED, "FBCDC: Core=0x%08x, Status=0x%08x, Signature status=0x%08x", 3) \
+X( 46, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WORK_CYCLES_PIPEDM_EN, "FWCtx 0x%08.8x, PipeDM state %04x, (start) %08x, (end) %08x, elapsed %08x", 5) \
+X( 47, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WORK_CYCLES, "FWCtx 0x%08.8x, elapsed %08x", 2) \
+X( 48, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_UNEXPECTED_EVENT_DEPRECATED, "Unexpected TRP pass completion on DM: %u",1) \
+X( 49, RGXFW_GROUP_MISC, RGXFW_SF_MISC_FAILED_IRQ_CLEAR_DEPRECATED, "Host failed to clear IRQ, Reg: 0x%08x",1) \
+\
+X(  1, RGXFW_GROUP_PM, RGXFW_SF_PM_AMLIST, "ALIST%u SP = %u, MLIST%u SP = %u (VCE 0x%08x%08x, TE 0x%08x%08x, ALIST 0x%08x%08x)", 10) \
+X(  2, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED_DEPRECATED, "Is TA: %u, finished: %u on HW %u (HWRTData = 0x%08x, MemCtx = 0x%08x). FL different between TA/3D: global:%u, local:%u, mmu:%u", 8) \
+X(  3, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE_DEPRECATED, "UFL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), MFL-3D-Base: 0x%08x%08x (SP = %u, 4PT = %u)", 14) \
+X(  4, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE_DEPRECATED, "UFL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), MFL-TA-Base: 0x%08x%08x (SP = %u, 4PT = %u)", 14) \
+X(  5, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_GROW_COMPLETE_DEPRECATED, "Freelist grow completed [0x%08x]: added pages 0x%08x, total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 5) \
+X(  6, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_GROW_DENIED_DEPRECATED, "Grow for freelist ID=0x%08x denied by host", 1) \
+X(  7, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_COMPLETE, "Freelist update completed [0x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 5) \
+X(  8, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_RECONSTRUCTION_FAILED_DEPRECATED, "Reconstruction of freelist ID=0x%08x failed", 1) \
+X(  9, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_WARNING, "Ignored attempt to pause or unpause the DM while there is no relevant operation in progress (0-TA,1-3D): %u, operation(0-unpause, 1-pause): %u", 2) \
+X( 10, RGXFW_GROUP_PM, RGXFW_SF_PM_3D_TIMEOUT_STATUS, "Force free 3D Context memory, FWCtx: 0x%08x, status(1:success, 0:fail): %u", 2)\
+X( 11, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_ALLOC, "PM pause TA ALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 12, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_UNPAUSE_ALLOC, "PM unpause TA ALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 13, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_DALLOC, "PM pause 3D DALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 14, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_UNPAUSE_DALLOC, "PM unpause 3D DALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 15, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_FAILED, "PM ALLOC/DALLOC change was not actioned: PM_PAGE_MANAGEOP_STATUS=0x%x", 1) \
+X( 16, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED, "Is TA: %u, finished: %u on HW %u (HWRTData = 0x%08x, MemCtx = 0x%08x). FL different between TA/3D: global:%u, local:%u", 7) \
+X( 17, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE, "UFL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 18, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE, "UFL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 19, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_COMPLETE_VOLCANIC, "Freelist update completed [0x%08x / FL State 0x%08x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 7) \
+X( 20, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_FAILED, "Freelist update failed [0x%08x / FL State 0x%08x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 7) \
+X( 21, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE_VOLCANIC, "UFL-3D-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 22, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE_VOLCANIC, "UFL-TA-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 23, RGXFW_GROUP_PM, RGXFW_SF_PM_CHECK_FL_BASEADDR, "Freelist 0x%08x base address from HW: 0x%02x%08x (expected value: 0x%02x%08x)", 5) \
+X( 24, RGXFW_GROUP_PM, RGXFW_SF_PM_ANALYSE_FL_GROW, "Analysis of FL grow: Pause=(%u,%u) Paused+Valid(%u,%u) PMStateBuffer=0x%x", 5) \
+X( 25, RGXFW_GROUP_PM, RGXFW_SF_PM_ATTEMPT_FL_GROW, "Attempt FL grow for FL: 0x%08x, new dev address: 0x%02x%08x, new page count: %u, new ready count: %u", 5) \
+X( 26, RGXFW_GROUP_PM, RGXFW_SF_PM_DEFER_FL_GROW, "Deferring FL grow for non-loaded FL: 0x%08x, new dev address: 0x%02x%08x, new page count: %u, new ready count: %u", 5) \
+X( 27, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED_ALBIORIX, "Is GEOM: %u, finished: %u (HWRTData = 0x%08x, MemCtx = 0x%08x)", 4) \
+X( 28, RGXFW_GROUP_PM, RGXFW_SF_PM_3D_TIMEOUT, "3D Timeout Now for FWCtx 0x%08.8x", 1) \
+X( 29, RGXFW_GROUP_PM, RGXFW_SF_PM_RECYCLE, "GEOM PM Recycle for FWCtx 0x%08.8x", 1) \
+X( 30, RGXFW_GROUP_PM, RGXFW_SF_PM_PRIMARY_CONFIG, "PM running primary config (Core %u)", 1) \
+X( 31, RGXFW_GROUP_PM, RGXFW_SF_PM_SECONDARY_CONFIG, "PM running secondary config (Core %u)", 1) \
+X( 32, RGXFW_GROUP_PM, RGXFW_SF_PM_TERTIARY_CONFIG, "PM running tertiary config (Core %u)", 1) \
+X( 33, RGXFW_GROUP_PM, RGXFW_SF_PM_QUATERNARY_CONFIG, "PM running quaternary config (Core %u)", 1) \
+X( 34, RGXFW_GROUP_PM, RGXFW_SF_PM_REVERT_CONFIG, "PM reverting to previous config (Core %u)", 1) \
+\
+X(  1, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GLL_DYNAMIC_STATUS_DEPRECATED, "Global link list dynamic page count: vertex 0x%x, varying 0x%x, node 0x%x", 3) \
+X(  2, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GLL_STATIC_STATUS_DEPRECATED, "Global link list static page count: vertex 0x%x, varying 0x%x, node 0x%x", 3) \
+X(  3, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_WAIT_FOR_GROW_DEPRECATED, "RPM request failed. Waiting for freelist grow.", 0) \
+X(  4, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_ABORT_DEPRECATED, "RPM request failed. Aborting the current frame.", 0) \
+X(  5, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_WAIT_FOR_PENDING_GROW_DEPRECATED, "RPM waiting for pending grow on freelist 0x%08x", 1) \
+X(  6, RGXFW_GROUP_RPM, RGXFW_SF_RPM_REQUEST_HOST_GROW_DEPRECATED, "Request freelist grow [0x%08x] current pages %u, grow size %u", 3) \
+X(  7, RGXFW_GROUP_RPM, RGXFW_SF_RPM_FREELIST_LOAD_DEPRECATED, "Freelist load: SHF = 0x%08x, SHG = 0x%08x", 2) \
+X(  8, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_DEPRECATED, "SHF FPL register: 0x%08x.0x%08x", 2) \
+X(  9, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_DEPRECATED, "SHG FPL register: 0x%08x.0x%08x", 2) \
+X( 10, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_FREELIST_DEPRECATED, "Kernel requested RPM grow on freelist (type %u) at 0x%08x from current size %u to new size %u, RPM restart: %u (1=Yes)", 5) \
+X( 11, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_RESTART_DEPRECATED, "Restarting SHG", 0) \
+X( 12, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_ABORTED_DEPRECATED, "Grow failed, aborting the current frame.", 0) \
+X( 13, RGXFW_GROUP_RPM, RGXFW_SF_RPM_ABORT_COMPLETE_DEPRECATED, "RPM abort complete on HWFrameData [0x%08x].", 1) \
+X( 14, RGXFW_GROUP_RPM, RGXFW_SF_RPM_CLEANUP_NEEDS_ABORT_DEPRECATED, "RPM freelist cleanup [0x%08x] requires abort to proceed.", 1) \
+X( 15, RGXFW_GROUP_RPM, RGXFW_SF_RPM_RPM_PT_DEPRECATED, "RPM page table base register: 0x%08x.0x%08x", 2) \
+X( 16, RGXFW_GROUP_RPM, RGXFW_SF_RPM_OOM_ABORT_DEPRECATED, "Issuing RPM abort.", 0) \
+X( 17, RGXFW_GROUP_RPM, RGXFW_SF_RPM_OOM_TOGGLE_CHECK_FULL_DEPRECATED, "RPM OOM received but toggle bits indicate free pages available", 0) \
+X( 18, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_HW_TIMEOUT_DEPRECATED, "RPM hardware timeout. Unable to process OOM event.", 0) \
+X( 19, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_LOAD_DEPRECATED_DEPRECATED, "SHF FL (0x%08x) load, FPL: 0x%08x.0x%08x, roff: 0x%08x, woff: 0x%08x", 5) \
+X( 20, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_LOAD_DEPRECATED, "SHG FL (0x%08x) load, FPL: 0x%08x.0x%08x, roff: 0x%08x, woff: 0x%08x", 5) \
+X( 21, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_STORE_DEPRECATED, "SHF FL (0x%08x) store, roff: 0x%08x, woff: 0x%08x", 3) \
+X( 22, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_STORE_DEPRECATED, "SHG FL (0x%08x) store, roff: 0x%08x, woff: 0x%08x", 3) \
+\
+X(  1, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_FINISHED, "3D RTData 0x%08x finished on HW context %u", 2) \
+X(  2, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_READY, "3D RTData 0x%08x ready on HW context %u", 2) \
+X(  3, RGXFW_GROUP_RTD, RGXFW_SF_RTD_PB_SET_TO_DEPRECATED, "CONTEXT_PB_BASE set to 0x%x, FL different between TA/3D: local: %u, global: %u, mmu: %u", 4) \
+X(  4, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOADVFP_3D_DEPRECATED, "Loading VFP table 0x%08x%08x for 3D", 2) \
+X(  5, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOADVFP_TA_DEPRECATED, "Loading VFP table 0x%08x%08x for TA", 2) \
+X(  6, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_DEPRECATED, "Load Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: TotalPMPages = %u, FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 10) \
+X(  7, RGXFW_GROUP_RTD, RGXFW_SF_RTD_VHEAP_STORE, "Perform VHEAP table store", 0) \
+X(  8, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_MATCH_FOUND, "RTData 0x%08x: found match in Context=%u: Load=No, Store=No", 2) \
+X(  9, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_NULL_FOUND, "RTData 0x%08x: found NULL in Context=%u: Load=Yes, Store=No", 2) \
+X( 10, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_3D_FINISHED, "RTData 0x%08x: found state 3D finished (0x%08x) in Context=%u: Load=Yes, Store=Yes", 3) \
+X( 11, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_TA_FINISHED, "RTData 0x%08x: found state TA finished (0x%08x) in Context=%u: Load=Yes, Store=Yes", 3) \
+X( 12, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_STACK_POINTERS, "Loading stack-pointers for %u (0:MidTA,1:3D) on context %u, MLIST = 0x%08x, ALIST = 0x%08x%08x", 5) \
+X( 13, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_PB_DEPRECATED, "Store Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: TotalPMPages = %u, FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 10) \
+X( 14, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_FINISHED, "TA RTData 0x%08x finished on HW context %u", 2) \
+X( 15, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_LOADED, "TA RTData 0x%08x loaded on HW context %u", 2) \
+X( 16, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_PB_DEPRECATED2, "Store Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 17, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_DEPRECATED2, "Load Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 18, RGXFW_GROUP_RTD, RGXFW_SF_RTD_DEBUG_DEPRECATED, "Freelist 0x%x RESET!!!!!!!!", 1) \
+X( 19, RGXFW_GROUP_RTD, RGXFW_SF_RTD_DEBUG2_DEPRECATED, "Freelist 0x%x stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 5) \
+X( 20, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_DEPRECATED, "Request reconstruction of Freelist 0x%x type: %u (0:local,1:global,2:mmu) on HW context %u", 3) \
+X( 21, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_ACK_DEPRECATED, "Freelist reconstruction ACK from host (HWR state :%u)", 1) \
+X( 22, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_ACK_DEPRECATED2, "Freelist reconstruction completed", 0) \
+X( 23, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_LOADED_DEPRECATED, "TA RTData 0x%08x loaded on HW context %u HWRTDataNeedsLoading=%u", 3) \
+X( 24, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TE_RGNHDR_INFO, "TE Region headers base 0x%08x%08x (RGNHDR Init: %u)", 3) \
+X( 25, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_BUFFER_ADDRS_DEPRECATED, "TA Buffers: FWCtx 0x%08x, RT 0x%08x, RTData 0x%08x, VHeap 0x%08x%08x, TPC 0x%08x%08x (MemCtx 0x%08x)", 8) \
+X( 26, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_LOADED_DEPRECATED, "3D RTData 0x%08x loaded on HW context %u", 2) \
+X( 27, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS_DEPRECATED, "3D Buffers: FWCtx 0x%08x, RT 0x%08x, RTData 0x%08x (MemCtx 0x%08x)", 4) \
+X( 28, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RESTART_AFTER_PR_EXECUTED, "Restarting TA after partial render, HWRTData0State=0x%x, HWRTData1State=0x%x", 2) \
+X( 29, RGXFW_GROUP_RTD, RGXFW_SF_RTD_PB_SET_TO, "CONTEXT_PB_BASE set to 0x%x, FL different between TA/3D: local: %u, global: %u", 3) \
+X( 30, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_FL, "Store Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 31, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL, "Load Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 32, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS_DEPRECATED2, "3D Buffers: FWCtx 0x%08x, parent RT 0x%08x, RTData 0x%08x on ctx %u, (MemCtx 0x%08x)", 5) \
+X( 33, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_BUFFER_ADDRS, "TA Buffers: FWCtx 0x%08x, RTData 0x%08x, VHeap 0x%08x%08x, TPC 0x%08x%08x (MemCtx 0x%08x)", 7) \
+X( 34, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS, "3D Buffers: FWCtx 0x%08x, RTData 0x%08x on ctx %u, (MemCtx 0x%08x)", 4) \
+X( 35, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_V2, "Load Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u)", 6) \
+X( 36, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILLED_TA, "TA RTData 0x%08x marked as killed.", 1) \
+X( 37, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILLED_3D, "3D RTData 0x%08x marked as killed.", 1) \
+X( 38, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILL_TA_AFTER_RESTART, "RTData 0x%08x will be killed after TA restart.", 1) \
+X( 39, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RENDERSTATE_RESET, "RTData 0x%08x Render State Buffer 0x%02x%08x will be reset.", 3) \
+X( 40, RGXFW_GROUP_RTD, RGXFW_SF_RTD_GEOM_RENDERSTATE, "GEOM RTData 0x%08x using Render State Buffer 0x%02x%08x.", 3) \
+X( 41, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FRAG_RENDERSTATE, "FRAG RTData 0x%08x using Render State Buffer 0x%02x%08x.", 3) \
+\
+X(  1, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZLOAD_DEPRECATED, "Force Z-Load for partial render", 0) \
+X(  2, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSTORE_DEPRECATED, "Force Z-Store for partial render", 0) \
+X(  3, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_LOCAL_DEPRECATED, "3D MemFree: Local FL 0x%08x", 1) \
+X(  4, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_MMU_DEPRECATED, "3D MemFree: MMU FL 0x%08x", 1) \
+X(  5, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_GLOBAL_DEPRECATED, "3D MemFree: Global FL 0x%08x", 1) \
+X(  6, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_DEPRECATED, "OOM TA/3D PR Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x, HardwareSync Fence [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 6) \
+X(  7, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_UN_FL, "OOM TA_cmd=0x%08x, U-FL 0x%08x, N-FL 0x%08x", 3) \
+X(  8, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_UN_MMU_FL_DEPRECATED, "OOM TA_cmd=0x%08x, OOM MMU:%u, U-FL 0x%08x, N-FL 0x%08x, MMU-FL 0x%08x", 5) \
+X(  9, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_AVOIDED_DEPRECATED, "Partial render avoided", 0) \
+X( 10, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_DISCARDED_DEPRECATED, "Partial render discarded", 0) \
+X( 11, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_FINISHED, "Partial Render finished", 0) \
+X( 12, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_3DBG_DEPRECATED, "SPM Owner = 3D-BG", 0) \
+X( 13, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_3DIRQ_DEPRECATED, "SPM Owner = 3D-IRQ", 0) \
+X( 14, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_NONE_DEPRECATED, "SPM Owner = NONE", 0) \
+X( 15, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_TABG_DEPRECATED, "SPM Owner = TA-BG", 0) \
+X( 16, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_TAIRQ_DEPRECATED, "SPM Owner = TA-IRQ", 0) \
+X( 17, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSTORE_ADDRESS, "ZStore address 0x%08x%08x", 2) \
+X( 18, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SSTORE_ADDRESS, "SStore address 0x%08x%08x", 2) \
+X( 19, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZLOAD_ADDRESS, "ZLoad address 0x%08x%08x", 2) \
+X( 20, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SLOAD_ADDRESS, "SLoad address 0x%08x%08x", 2) \
+X( 21, RGXFW_GROUP_SPM, RGXFW_SF_SPM_NO_DEFERRED_ZSBUFFER_DEPRECATED, "No deferred ZS Buffer provided", 0) \
+X( 22, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_POPULATED, "ZS Buffer successfully populated (ID=0x%08x)", 1) \
+X( 23, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_POP_UNNEEDED_DEPRECATED, "No need to populate ZS Buffer (ID=0x%08x)", 1) \
+X( 24, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNPOPULATED, "ZS Buffer successfully unpopulated (ID=0x%08x)", 1) \
+X( 25, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNPOP_UNNEEDED_DEPRECATED, "No need to unpopulate ZS Buffer (ID=0x%08x)", 1) \
+X( 26, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_DEPRECATED, "Send ZS-Buffer backing request to host (ID=0x%08x)", 1) \
+X( 27, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNBACKING_REQUEST_DEPRECATED, "Send ZS-Buffer unbacking request to host (ID=0x%08x)", 1) \
+X( 28, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_PENDING_DEPRECATED, "Don't send ZS-Buffer backing request. Previous request still pending (ID=0x%08x)", 1) \
+X( 29, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNBACKING_REQUEST_PENDING_DEPRECATED, "Don't send ZS-Buffer unbacking request. Previous request still pending (ID=0x%08x)", 1) \
+X( 30, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZBUFFER_NOT_READY_DEPRECATED, "Partial Render waiting for ZBuffer to be backed (ID=0x%08x)", 1) \
+X( 31, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SBUFFER_NOT_READY_DEPRECATED, "Partial Render waiting for SBuffer to be backed (ID=0x%08x)", 1) \
+X( 32, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_NONE, "SPM State = none", 0) \
+X( 33, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_BLOCKED, "SPM State = PR blocked", 0) \
+X( 34, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_GROW, "SPM State = wait for grow", 0) \
+X( 35, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_HW, "SPM State = wait for HW", 0) \
+X( 36, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_RUNNING, "SPM State = PR running", 0) \
+X( 37, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_AVOIDED, "SPM State = PR avoided", 0) \
+X( 38, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_EXECUTED, "SPM State = PR executed", 0) \
+X( 39, RGXFW_GROUP_SPM, RGXFW_SF_SPM_FREELIST_MATCH, "3DMemFree matches freelist 0x%08x (FL type = %u)", 2) \
+X( 40, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_FLAG_SET, "Raise the 3DMemFreeDetected flag", 0) \
+X( 41, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_PENDING_GROW, "Wait for pending grow on Freelist 0x%08x", 1) \
+X( 42, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_FAILED, "ZS Buffer failed to be populated (ID=0x%08x)", 1) \
+X( 43, RGXFW_GROUP_SPM, RGXFW_SF_SPM_FL_GROW_DEBUG, "Grow update inconsistency: FL addr: 0x%02x%08x, curr pages: %u, ready: %u, new: %u", 5) \
+X( 44, RGXFW_GROUP_SPM, RGXFW_SF_SPM_RESUMED_TA_WITH_SP, "OOM: Resumed TA with ready pages, FL addr: 0x%02x%08x, current pages: %u, SP : %u", 4) \
+X( 45, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ACK_GROW_UPDATE_DEPRECATED, "Received grow update, FL addr: 0x%02x%08x, current pages: %u, ready pages: %u, threshold: %u", 5) \
+X( 46, RGXFW_GROUP_SPM, RGXFW_SF_SPM_NO_DEFERRED_PRBUFFER, "No deferred partial render FW (Type=%u) Buffer provided", 1) \
+X( 47, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_POP_UNNEEDED, "No need to populate PR Buffer (ID=0x%08x)", 1) \
+X( 48, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNPOP_UNNEEDED, "No need to unpopulate PR Buffer (ID=0x%08x)", 1) \
+X( 49, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_REQUEST, "Send PR Buffer backing request to host (ID=0x%08x)", 1) \
+X( 50, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNBACKING_REQUEST, "Send PR Buffer unbacking request to host (ID=0x%08x)", 1) \
+X( 51, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_REQUEST_PENDING, "Don't send PR Buffer backing request. Previous request still pending (ID=0x%08x)", 1) \
+X( 52, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNBACKING_REQUEST_PENDING, "Don't send PR Buffer unbacking request. Previous request still pending (ID=0x%08x)", 1) \
+X( 53, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_NOT_READY, "Partial Render waiting for Buffer %u type to be backed (ID=0x%08x)", 2) \
+X( 54, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ACK_GROW_UPDATE, "Received grow update, FL addr: 0x%02x%08x, new pages: %u, ready pages: %u", 4) \
+X( 66, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD, "OOM TA/3D PR Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 67, RGXFW_GROUP_SPM, RGXFW_SF_SPM_RESUMED_TA, "OOM: Resumed TA with ready pages, FL addr: 0x%02x%08x, current pages: %u", 3) \
+X( 68, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PR_DEADLOCK_UNBLOCKED, "OOM TA/3D PR deadlock unblocked reordering DM%u runlist head from Context 0x%08x to 0x%08x", 3) \
+X( 69, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_FORCEFREE, "SPM State = PR force free", 0) \
+X( 70, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_FAILED, "Failure to back PR Buffer", 0) \
+\
+X(  1, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK_DEPRECATED, "Check Pow state DM%u int: 0x%x, ext: 0x%x, pow flags: 0x%x", 4) \
+X(  2, RGXFW_GROUP_POW, RGXFW_SF_POW_GPU_IDLE, "GPU idle (might be powered down). Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X(  3, RGXFW_GROUP_POW, RGXFW_SF_POW_OSREQ_DEPRECATED, "OS requested pow off (forced = %u), DM%u, pow flags: 0x%x", 3) \
+X(  4, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_DEPRECATED, "Initiate powoff query. Inactive DMs: %u %u %u %u", 4) \
+X(  5, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECKOFF_DEPRECATED, "Any RD-DM pending? %u, Any RD-DM Active? %u", 2) \
+X(  6, RGXFW_GROUP_POW, RGXFW_SF_POW_GPU_OFF, "GPU ready to be powered down. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X(  7, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ, "HW Request On(1)/Off(0): %u, Units: 0x%08.8x", 2) \
+X(  8, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE_REQ, "Request to change num of dusts to %u (Power flags=%u)", 2) \
+X(  9, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE, "Changing number of dusts from %u to %u", 2) \
+X( 11, RGXFW_GROUP_POW, RGXFW_SF_POW_SIDEKICK_INIT_DEPRECATED, "Sidekick init", 0) \
+X( 12, RGXFW_GROUP_POW, RGXFW_SF_POW_RD_INIT_DEPRECATED, "Rascal+Dusts init (# dusts mask: 0x%x)", 1) \
+X( 13, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_RD, "Initiate powoff query for RD-DMs.", 0) \
+X( 14, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_TLA, "Initiate powoff query for TLA-DM.", 0) \
+X( 15, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_RD, "Any RD-DM pending? %u, Any RD-DM Active? %u", 2) \
+X( 16, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_TLA, "TLA-DM pending? %u, TLA-DM Active? %u", 2) \
+X( 17, RGXFW_GROUP_POW, RGXFW_SF_POW_BRN37270_DEPRECATED, "Request power up due to BRN37270. Pow stat int: 0x%x", 1) \
+X( 18, RGXFW_GROUP_POW, RGXFW_SF_POW_REQ_CANCEL, "Cancel power off request int: 0x%x, ext: 0x%x, pow flags: 0x%x", 3) \
+X( 19, RGXFW_GROUP_POW, RGXFW_SF_POW_FORCED_IDLE, "OS requested forced IDLE, pow flags: 0x%x", 1) \
+X( 20, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_FORCED_IDLE, "OS cancelled forced IDLE, pow flags: 0x%x", 1) \
+X( 21, RGXFW_GROUP_POW, RGXFW_SF_POW_IDLE_TIMER, "Idle timer start. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 22, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_IDLE_TIMER, "Cancel idle timer. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 23, RGXFW_GROUP_POW, RGXFW_SF_POW_APM_LATENCY_CHANGE, "Active PM latency set to %ums. Core clock: %u Hz", 2) \
+X( 24, RGXFW_GROUP_POW, RGXFW_SF_POW_CDM_CLUSTERS, "Compute cluster mask change to 0x%x, %u dusts powered.", 2) \
+X( 25, RGXFW_GROUP_POW, RGXFW_SF_POW_NULL_CMD_INIOFF_RD, "Null command executed, repeating initiate powoff query for RD-DMs.", 0) \
+X( 26, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_ENERGY, "Power monitor: Estimate of dynamic energy %u", 1) \
+X( 27, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK_DEPRECATED2, "Check Pow state: Int: 0x%x, Ext: 0x%x, Pow flags: 0x%x", 3) \
+X( 28, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_NEW_DEADLINE, "Proactive DVFS: New deadline, time = 0x%08x%08x", 2) \
+X( 29, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_NEW_WORKLOAD, "Proactive DVFS: New workload, cycles = 0x%08x%08x", 2) \
+X( 30, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_CALCULATE, "Proactive DVFS: Proactive frequency calculated = %u", 1) \
+X( 31, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UTILISATION, "Proactive DVFS: Reactive utilisation = %u percent", 1) \
+X( 32, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_REACT, "Proactive DVFS: Reactive frequency calculated = %u.%u", 2) \
+X( 33, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_SEND_DEPRECATED, "Proactive DVFS: OPP Point Sent = 0x%x", 1) \
+X( 34, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DEADLINE_REMOVED, "Proactive DVFS: Deadline removed = 0x%08x%08x", 2) \
+X( 35, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_WORKLOAD_REMOVED, "Proactive DVFS: Workload removed = 0x%08x%08x", 2) \
+X( 36, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_THROTTLE, "Proactive DVFS: Throttle to a maximum = 0x%x", 1) \
+X( 37, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_FAILURE, "Proactive DVFS: Failed to pass OPP point via GPIO.", 0) \
+X( 38, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_INVALID_NODE_DEPRECATED, "Proactive DVFS: Invalid node passed to function.", 0) \
+X( 39, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GUEST_BAD_ACCESS_DEPRECATED, "Proactive DVFS: Guest OS attempted to do a privileged action. OSid = %u", 1) \
+X( 40, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UNPROFILED_STARTED, "Proactive DVFS: Unprofiled work started. Total unprofiled work present: %u", 1) \
+X( 41, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UNPROFILED_FINISHED, "Proactive DVFS: Unprofiled work finished. Total unprofiled work present: %u", 1) \
+X( 42, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DISABLED, "Proactive DVFS: Disabled: Not enabled by host.", 0) \
+X( 43, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ_RESULT, "HW Request Completed(1)/Aborted(0): %u, Ticks: %u", 2) \
+X( 44, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE_FIX_59042_DEPRECATED, "Allowed number of dusts is %u due to BRN59042.", 1) \
+X( 45, RGXFW_GROUP_POW, RGXFW_SF_POW_HOST_TIMEOUT_NOTIFICATION, "Host timed out while waiting for a forced idle state. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 46, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK, "Check Pow state: Int: 0x%x, Ext: 0x%x, Pow flags: 0x%x, Fence Counters: Check: %u - Update: %u", 5) \
+X( 47, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_SEND, "Custom/Proactive DVFS: OPP Point Sent = 0x%x, Success = 0x%x", 2) \
+X( 48, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_TO_IDLE, "Proactive DVFS: GPU transitioned to idle", 0) \
+X( 49, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_TO_ACTIVE, "Proactive DVFS: GPU transitioned to active", 0) \
+X( 50, RGXFW_GROUP_POW, RGXFW_SF_POW_POWDUMP_BUFFER_SIZE, "Power counter dumping: Data truncated writing register %u. Buffer too small.", 1) \
+X( 51, RGXFW_GROUP_POW, RGXFW_SF_POW_POWCTRL_ABORT, "Power controller returned ABORT for last request so retrying.", 0) \
+X( 52, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_POWER_REQUEST_DEPRECATED, "Discarding invalid power request: type 0x%x, DM %u", 2) \
+X( 53, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_FORCED_IDLE_NOT_IDLE, "Detected attempt to cancel forced idle while not forced idle (pow state 0x%x, pow flags 0x%x)", 2) \
+X( 54, RGXFW_GROUP_POW, RGXFW_SF_POW_FORCED_POW_OFF_NOT_IDLE, "Detected attempt to force power off while not forced idle (pow state 0x%x, pow flags 0x%x)", 2) \
+X( 55, RGXFW_GROUP_POW, RGXFW_SF_POW_NUMDUST_CHANGE_NOT_IDLE, "Detected attempt to change dust count while not forced idle (pow state 0x%x)", 1) \
+X( 56, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_RESULT, "Power monitor: Type = %u (0 = power, 1 = energy), Estimate result = 0x%08x%08x", 3) \
+X( 57, RGXFW_GROUP_POW, RGXFW_SF_POW_MINMAX_CONFLICT, "Conflicting clock frequency range: OPP min = %u, max = %u", 2) \
+X( 58, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_FLOOR, "Proactive DVFS: Set floor to a minimum = 0x%x", 1) \
+X( 59, RGXFW_GROUP_POW, RGXFW_SF_POW_OSREQ, "OS requested pow off (forced = %u), pow flags: 0x%x", 2) \
+X( 60, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_POWER_REQUEST, "Discarding invalid power request: type 0x%x", 1) \
+X( 61, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_STATE_CHANGE_REQ, "Request to change SPU power state mask from 0x%x to 0x%x. Pow flags: 0x%x", 3) \
+X( 62, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_STATE_CHANGE, "Changing SPU power state mask from 0x%x to 0x%x", 2) \
+X( 63, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_CHANGE_NOT_IDLE, "Detected attempt to change SPU power state mask while not forced idle (pow state 0x%x)", 1) \
+X( 64, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_SPU_POWER_MASK, "Invalid SPU power mask 0x%x! Changing to 1", 1) \
+X( 65, RGXFW_GROUP_POW, RGXFW_SF_POW_CLKDIV_UPDATE, "Proactive DVFS: Send OPP %u with clock divider value %u", 2) \
+X( 66, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_PERF_MODE, "Power counters in raw/validation mode.", 0) \
+X( 67, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_RESET, "Reset PPA block state %u (1=reset, 0=recalculate).", 1) \
+X( 68, RGXFW_GROUP_POW, RGXFW_SF_POW_POWCTRL_ABORT_WITH_CORE, "Power controller returned ABORT for Core-%u last request so retrying.", 1) \
+X( 69, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ64BIT, "HW Request On(1)/Off(0): %u, Units: 0x%08x%08x", 3) \
+X( 70, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_RAC_POW_STATE_CHANGE_REQ, "Request to change SPU power state mask from 0x%x to 0x%x and RAC from 0x%x to 0x%x. Pow flags: 0x%x", 5) \
+X( 71, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_RAC_POW_STATE_CHANGE, "Changing SPU power state mask from 0x%x to 0x%x and RAC from 0x%x to 0x%x", 4) \
+X( 72, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_RAC, "RAC pending? %u, RAC Active? %u", 2) \
+X( 73, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_RAC, "Initiate powoff query for RAC.", 0) \
+X( 74, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DEFER_REACTIVE_UPDATE, "Proactive DVFS: Defer reactive update to meet next deadline 0x%08x%08x", 2) \
+\
+X(  1, RGXFW_GROUP_HWR, RGXFW_SF_HWR_LOCKUP_DEPRECATED, "Lockup detected on DM%u, FWCtx: 0x%08.8x", 2) \
+X(  2, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_FW_DEPRECATED, "Reset fw state for DM%u, FWCtx: 0x%08.8x, MemCtx: 0x%08.8x", 3) \
+X(  3, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED, "Reset HW", 0) \
+X(  4, RGXFW_GROUP_HWR, RGXFW_SF_HWR_TERMINATED_DEPRECATED, "Lockup recovered.", 0) \
+X(  5, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DEPRECATED, "Lock-up DM%u FWCtx: 0x%08.8x", 2) \
+X(  6, RGXFW_GROUP_HWR, RGXFW_SF_HWR_LOCKUP_DETECTED_DEPRECATED, "Lockup detected: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X(  7, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EARLY_FAULT_DETECTION_DEPRECATED, "Early fault detection: GLB(%u->%u), PER-DM(0x%08x)", 3) \
+X(  8, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_LOCKUP_DEPRECATED, "Hold scheduling due lockup: GLB(%u), PER-DM(0x%08x->0x%08x)", 3) \
+X(  9, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FALSE_LOCKUP_DEPRECATED, "False lockup detected: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X( 10, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BRN37729_DEPRECATED, "BRN37729: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X( 11, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FREELISTS_RECONSTRUCTED_DEPRECATED, "Freelists reconstructed: GLB(%u->%u), PER-DM(0x%08x)", 3) \
+X( 12, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RECONSTRUCTING_FREELISTS_DEPRECATED, "Reconstructing freelists: %u (0-No, 1-Yes): GLB(%u->%u), PER-DM(0x%08x)", 4) \
+X( 13, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FAILED_HW_POLL, "HW poll %u (0-Unset 1-Set) failed (reg:0x%08x val:0x%08x)", 3) \
+X( 14, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_DISCARDED_DEPRECATED, "Discarded cmd on DM%u FWCtx=0x%08x", 2) \
+X( 15, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_DISCARDED, "Discarded cmd on DM%u (reason=%u) HWRTData=0x%08x (st: %u), FWCtx 0x%08x @ %u", 6) \
+X( 16, RGXFW_GROUP_HWR, RGXFW_SF_HWR_PM_FENCE_DEPRECATED, "PM fence WA could not be applied, Valid TA Setup: %u, RD powered off: %u", 2) \
+X( 17, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_SNAPSHOT, "FL snapshot RTD 0x%08.8x - local (0x%08.8x): %u, global (0x%08.8x): %u", 5) \
+X( 18, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_CHECK, "FL check RTD 0x%08.8x, discard: %u - local (0x%08.8x): s%u?=c%u, global (0x%08.8x): s%u?=c%u", 8) \
+X( 19, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_DEPRECATED, "FL reconstruction 0x%08.8x c%u", 2) \
+X( 20, RGXFW_GROUP_HWR, RGXFW_SF_HWR_3D_CHECK, "3D check: missing TA FWCtx 0x%08.8x @ %u, RTD 0x%08x.", 3) \
+X( 21, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED2, "Reset HW (mmu:%u, extmem: %u)", 2) \
+X( 22, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ZERO_TA_CACHES, "Zero TA caches for FWCtx: 0x%08.8x (TPC addr: 0x%08x%08x, size: %u bytes)", 4) \
+X( 23, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FREELISTS_RECONSTRUCTED_DEPRECATED2, "Recovery DM%u: Freelists reconstructed. New R-Flags=0x%08x", 2) \
+X( 24, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SKIPPED_CMD, "Recovery DM%u: FWCtx 0x%08x skipped to command @ %u. PR=%u. New R-Flags=0x%08x", 5) \
+X( 25, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_RECOVERED, "Recovery DM%u: DM fully recovered", 1) \
+X( 26, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_LOCKUP, "DM%u: Hold scheduling due to R-Flag = 0x%08x", 2) \
+X( 27, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_RECONSTRUCTION, "Analysis: Need freelist reconstruction", 0) \
+X( 28, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_SKIP, "Analysis DM%u: Lockup FWCtx: 0x%08.8x. Need to skip to next command", 2) \
+X( 29, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_SKIP_OOM_TA, "Analysis DM%u: Lockup while TA is OOM FWCtx: 0x%08.8x. Need to skip to next command", 2) \
+X( 30, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_PR_CLEANUP, "Analysis DM%u: Lockup while partial render FWCtx: 0x%08.8x. Need PR cleanup", 2) \
+X( 31, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DEPRECATED2, "GPU has locked up", 0) \
+X( 32, RGXFW_GROUP_HWR, RGXFW_SF_HWR_READY, "DM%u ready for HWR", 1) \
+X( 33, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_UPDATE_RECOVERY, "Recovery DM%u: Updated Recovery counter. New R-Flags=0x%08x", 2) \
+X( 34, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BRN37729_DEPRECATED2, "Analysis: BRN37729 detected, reset TA and re-kicked 0x%08x)", 1) \
+X( 35, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_TIMED_OUT_DEPRECATED, "DM%u timed out", 1) \
+X( 36, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EVENT_STATUS_REG, "RGX_CR_EVENT_STATUS=0x%08x", 1) \
+X( 37, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_FALSE_LOCKUP, "DM%u lockup falsely detected, R-Flags=0x%08x", 2) \
+X( 38, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_OUTOFTIME, "GPU has overrun its deadline", 0) \
+X( 39, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_POLLFAILURE, "GPU has failed a poll", 0) \
+X( 40, RGXFW_GROUP_HWR, RGXFW_SF_HWR_PERF_PHASE_REG, "RGX DM%u phase count=0x%08x", 2) \
+X( 41, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED3, "Reset HW (loop:%u, poll failures: 0x%08x)", 2) \
+X( 42, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_FAULT_EVENT, "MMU fault event: 0x%08x", 1) \
+X( 43, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BIF1_FAULT, "BIF1 page fault detected (Bank1 MMU Status: 0x%08x)", 1) \
+X( 44, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK_TRUE_DEPRECATED, "Fast CRC Failed. Proceeding to full register checking (DM: %u).", 1) \
+X( 45, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_META_FAULT, "Meta MMU page fault detected (Meta MMU Status: 0x%08x%08x)", 2) \
+X( 46, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK_DEPRECATED, "Fast CRC Check result for DM%u is HWRNeeded=%u", 2) \
+X( 47, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FULL_CHECK_DEPRECATED, "Full Signature Check result for DM%u is HWRNeeded=%u", 2) \
+X( 48, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FINAL_RESULT, "Final result for DM%u is HWRNeeded=%u with HWRChecksToGo=%u", 3) \
+X( 49, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_CHECK_DEPRECATED, "USC Slots result for DM%u is HWRNeeded=%u USCSlotsUsedByDM=%u", 3) \
+X( 50, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DEADLINE_CHECK_DEPRECATED, "Deadline counter for DM%u is HWRDeadline=%u", 2) \
+X( 51, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_FREELIST_DEPRECATED, "Holding Scheduling on OSid %u due to pending freelist reconstruction", 1) \
+X( 52, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_REQUEST, "Requesting reconstruction for freelist 0x%x (ID=%u)", 2) \
+X( 53, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_PASSED, "Reconstruction of freelist ID=%u complete", 1) \
+X( 54, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global,2:mmu) on HW context %u", 4) \
+X( 55, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_FAILED, "Reconstruction of freelist ID=%u failed", 1) \
+X( 56, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESTRICTING_PDS_TASKS, "Restricting PDS Tasks to help other stalling DMs (RunningMask=0x%02x, StallingMask=0x%02x, PDS_CTRL=0x%08x%08x)", 4) \
+X( 57, RGXFW_GROUP_HWR, RGXFW_SF_HWR_UNRESTRICTING_PDS_TASKS, "Unrestricting PDS Tasks again (RunningMask=0x%02x, StallingMask=0x%02x, PDS_CTRL=0x%08x%08x)", 4) \
+X( 58, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_USED, "USC slots: %u used by DM%u", 2) \
+X( 59, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_EMPTY, "USC slots: %u empty", 1) \
+X( 60, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HCS_FIRE, "HCS DM%u's Context Switch failed to meet deadline. Current time: 0x%08x%08x, deadline: 0x%08x%08x", 5) \
+X( 61, RGXFW_GROUP_HWR, RGXFW_SF_HWR_START_HW_RESET, "Begin hardware reset (HWR Counter=%u)", 1) \
+X( 62, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FINISH_HW_RESET, "Finished hardware reset (HWR Counter=%u)", 1) \
+X( 63, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_FREELIST, "Holding Scheduling on DM %u for Driver ID %u due to pending freelist reconstruction", 2) \
+X( 64, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_UMQ_READ_OFFSET, "User Mode Queue ROff reset: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u becomes StreamStartOffset = %u)", 5) \
+X( 65, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED2, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global) on HW context %u", 4) \
+X( 66, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MIPS_FAULT, "Mips page fault detected (BadVAddr: 0x%08x, EntryLo0: 0x%08x, EntryLo1: 0x%08x)", 3) \
+X( 67, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ANOTHER_CHANCE, "At least one other DM is running okay so DM%u will get another chance", 1) \
+X( 68, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_FW, "Reconstructing in FW, FL: 0x%x (ID=%u)", 2) \
+X( 69, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ZERO_RTC, "Zero RTC for FWCtx: 0x%08.8x (RTC addr: 0x%08x%08x, size: %u bytes)", 4) \
+X( 70, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED3, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global) phase: %u (0:TA, 1:3D) on HW context %u", 5) \
+X( 71, RGXFW_GROUP_HWR, RGXFW_SF_HWR_START_LONG_HW_POLL, "Start long HW poll %u (0-Unset 1-Set) for (reg:0x%08x val:0x%08x)", 3) \
+X( 72, RGXFW_GROUP_HWR, RGXFW_SF_HWR_END_LONG_HW_POLL, "End long HW poll (result=%u)", 1) \
+X( 73, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DEADLINE_CHECK, "DM%u has taken %u ticks and deadline is %u ticks", 3) \
+X( 74, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WATCHDOG_CHECK_DEPRECATED, "USC Watchdog result for DM%u is HWRNeeded=%u Status=%u USCs={0x%x} with HWRChecksToGo=%u", 5) \
+X( 75, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED, "Reconstruction needed for freelist 0x%x (ID=%u) Driver ID: %u type: %u (0:local,1:global) phase: %u (0:TA, 1:3D) on HW context %u", 6) \
+X( 76, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP, "GPU-%u has locked up", 1) \
+X( 77, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DM, "DM%u has locked up", 1) \
+X( 78, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_EVENT_STATUS_REG, "Core %u RGX_CR_EVENT_STATUS=0x%08x", 2) \
+X( 79, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MULTICORE_EVENT_STATUS_REG, "RGX_CR_MULTICORE_EVENT_STATUS%u=0x%08x", 2) \
+X( 80, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_BIF0_FAULT, "BIF0 page fault detected (Core %u MMU Status: 0x%08x%08x Req Status: 0x%08x%08x)", 5) \
+X( 81, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_MMU_FAULT_S7, "MMU page fault detected (Core %u MMU Status: 0x%08x%08x)", 3) \
+X( 82, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_MMU_FAULT, "MMU page fault detected (Core %u MMU Status: 0x%08x%08x 0x%08x)", 4) \
+X( 83, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW, "Reset HW (core:%u of %u, loop:%u, poll failures: 0x%08x)", 4) \
+X( 84, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK, "Fast CRC Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 85, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FULL_CHECK, "Full Signature Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 86, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_CHECK, "USC Slots result for Core%u, DM%u is HWRNeeded=%u USCSlotsUsedByDM=%u", 4) \
+X( 87, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WATCHDOG_CHECK, "USC Watchdog result for Core%u DM%u is HWRNeeded=%u Status=%u USCs={0x%x} with HWRChecksToGo=%u", 6) \
+X( 88, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_RISCV_FAULT, "RISC-V MMU page fault detected (FWCORE MMU Status 0x%08x Req Status 0x%08x%08x)", 3) \
+X( 89, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_TEXAS1_PFS_DEPRECATED, "TEXAS1_PFS poll failed on core %u with value 0x%08x", 2) \
+X( 90, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_PFS, "BIF_PFS poll failed on core %u with value 0x%08x", 2) \
+X( 91, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SET_ABORT_PM_STATUS, "MMU_ABORT_PM_STATUS set poll failed on core %u with value 0x%08x", 2) \
+X( 92, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_UNSET_ABORT_PM_STATUS, "MMU_ABORT_PM_STATUS unset poll failed on core %u with value 0x%08x", 2) \
+X( 93, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SLC_INVAL, "MMU_CTRL_INVAL poll (all but fw) failed on core %u with value 0x%08x", 2) \
+X( 94, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SLCMMU_INVAL, "MMU_CTRL_INVAL poll (all) failed on core %u with value 0x%08x", 2) \
+X( 95, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_TEXAS_PFS, "TEXAS%u_PFS poll failed on core %u with value 0x%08x", 3) \
+X( 96, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EXTRA_CHECK, "Extra Registers Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 97, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WRITE_TO_GPU_READONLY_ADDR, "FW attempted to write to read-only GPU address 0x%08x", 1) \
+X( 98, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_TIMED_OUT, "DM%u timed out (phase count=0x%08x)", 2) \
+\
+X(  1, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CFGBLK, "Block 0x%x mapped to Config Idx %u", 2) \
+X(  2, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_OMTBLK, "Block 0x%x omitted from event - not enabled in HW", 1) \
+X(  3, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INCBLK, "Block 0x%x included in event - enabled in HW", 1) \
+X(  4, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_SELREG, "Select register state hi_0x%x lo_0x%x", 2) \
+X(  5, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CSBHDR, "Counter stream block header word 0x%x", 1) \
+X(  6, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTROFF, "Counter register offset 0x%x", 1) \
+X(  7, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CFGSKP, "Block 0x%x config unset, skipping", 1) \
+X(  8, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INDBLK, "Accessing Indirect block 0x%x", 1) \
+X(  9, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DIRBLK, "Accessing Direct block 0x%x", 1) \
+X( 10, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CNTPRG, "Programmed counter select register at offset 0x%x", 1) \
+X( 11, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKPRG, "Block register offset 0x%x and value 0x%x", 2) \
+X( 12, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKCG, "Reading config block from driver 0x%x", 1) \
+X( 13, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKRG, "Reading block range 0x%x to 0x%x", 2) \
+X( 14, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKREC, "Recording block 0x%x config from driver", 1) \
+X( 15, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKED, "Finished reading config block from driver", 0) \
+X( 16, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CUSTOM_COUNTER, "Custom Counter offset: 0x%x  value: 0x%x", 2) \
+X( 17, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_SELECT_CNTR, "Select counter n:%u  ID:0x%x", 2) \
+X( 18, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_SELECT_PACK, "The counter ID 0x%x is not allowed. The package [b:%u, n:%u] will be discarded", 3) \
+X( 19, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHANGE_FILTER_STATUS_CUSTOM, "Custom Counters filter status %u", 1) \
+X( 20, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_WRONG_BLOCK, "The Custom block %u is not allowed. Use only blocks lower than %u. The package will be discarded", 2) \
+X( 21, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_TOO_MANY_ID, "The package will be discarded because it contains %u counters IDs while the upper limit is %u", 2) \
+X( 22, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHECK_FILTER, "Check Filter 0x%x is 0x%x ?", 2) \
+X( 23, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_RESET_CUSTOM_BLOCK, "The custom block %u is reset", 1) \
+X( 24, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INVALID_CMD_DEPRECATED, "Encountered an invalid command (%u)", 1) \
+X( 25, RGXFW_GROUP_HWP, RGXFW_SF_HWP_WAITING_FOR_QUEUE_DEPRECATED, "HWPerf Queue is full, we will have to wait for space! (Roff = %u, Woff = %u)", 2) \
+X( 26, RGXFW_GROUP_HWP, RGXFW_SF_HWP_WAITING_FOR_QUEUE_FENCE_DEPRECATED, "HWPerf Queue is fencing, we are waiting for Roff = %u (Roff = %u, Woff = %u)", 3) \
+X( 27, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CUSTOM_BLOCK, "Custom Counter block: %u", 1) \
+X( 28, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKENA, "Block 0x%x ENABLED", 1) \
+X( 29, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKDIS, "Block 0x%x DISABLED", 1) \
+X( 30, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INDBLK_INSTANCE, "Accessing Indirect block 0x%x, instance %u", 2) \
+X( 31, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTRVAL, "Counter register 0x%x, Value 0x%x", 2) \
+X( 32, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHANGE_FILTER_STATUS, "Counters filter status %u", 1) \
+X( 33, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTLBLK, "Block 0x%x mapped to Ctl Idx %u", 2) \
+X( 34, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_WORKEST_EN, "Block(s) in use for workload estimation.", 0) \
+X( 35, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CYCCTR, "GPU %u Cycle counter 0x%x, Value 0x%x", 3) \
+X( 36, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CYCMAX, "GPU Mask 0x%x Cycle counter 0x%x, Value 0x%x", 3) \
+X( 37, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_IGNORE_BLOCKS, "Blocks IGNORED for GPU %u", 1) \
+\
+X(  1, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_REQUEST_DEPRECATED, "Transfer 0x%02x request: 0x%02x%08x -> 0x%08x, size %u", 5) \
+X(  2, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_COMPLETE, "Transfer of type 0x%02x expected on channel %u, 0x%02x found, status %u", 4) \
+X(  3, RGXFW_GROUP_DMA, RGXFW_SF_DMA_INT_REG, "DMA Interrupt register 0x%08x", 1) \
+X(  4, RGXFW_GROUP_DMA, RGXFW_SF_DMA_WAIT, "Waiting for transfer of type 0x%02x completion...", 1) \
+X(  5, RGXFW_GROUP_DMA, RGXFW_SF_DMA_CCB_LOADING_FAILED, "Loading of cCCB data from FW common context 0x%08x (offset: %u, size: %u) failed", 3) \
+X(  6, RGXFW_GROUP_DMA, RGXFW_SF_DMA_CCB_LOAD_INVALID, "Invalid load of cCCB data from FW common context 0x%08x (offset: %u, size: %u)", 3) \
+X(  7, RGXFW_GROUP_DMA, RGXFW_SF_DMA_POLL_FAILED, "Transfer 0x%02x request poll failure", 1) \
+X(  8, RGXFW_GROUP_DMA, RGXFW_SF_DMA_BOOT_TRANSFER_FAILED_DEPRECATED, "Boot transfer(s) failed (code? %u, data? %u), used slower memcpy instead", 2) \
+X(  9, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_REQUEST, "Transfer 0x%02x request on ch. %u: system 0x%02x%08x, coremem 0x%08x, flags 0x%x, size %u", 7) \
+\
+X(  1, RGXFW_GROUP_DBG, RGXFW_SF_DBG_INTPAIR, "0x%08x 0x%08x", 2) \
+X(  2, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1HEX, "0x%08x", 1) \
+X(  3, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2HEX, "0x%08x 0x%08x", 2) \
+X(  4, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3HEX, "0x%08x 0x%08x 0x%08x", 3) \
+X(  5, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4HEX, "0x%08x 0x%08x 0x%08x 0x%08x", 4) \
+X(  6, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 5) \
+X(  7, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 6) \
+X(  8, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 7) \
+X(  9, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 8) \
+X( 10, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1SIGNED, "%d", 1) \
+X( 11, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2SIGNED, "%d %d", 2) \
+X( 12, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3SIGNED, "%d %d %d", 3) \
+X( 13, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4SIGNED, "%d %d %d %d", 4) \
+X( 14, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5SIGNED, "%d %d %d %d %d", 5) \
+X( 15, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6SIGNED, "%d %d %d %d %d %d", 6) \
+X( 16, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7SIGNED, "%d %d %d %d %d %d %d", 7) \
+X( 17, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8SIGNED, "%d %d %d %d %d %d %d %d", 8) \
+X( 18, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1UNSIGNED, "%u", 1) \
+X( 19, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2UNSIGNED, "%u %u", 2) \
+X( 20, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3UNSIGNED, "%u %u %u", 3) \
+X( 21, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4UNSIGNED, "%u %u %u %u", 4) \
+X( 22, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5UNSIGNED, "%u %u %u %u %u", 5) \
+X( 23, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6UNSIGNED, "%u %u %u %u %u %u", 6) \
+X( 24, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7UNSIGNED, "%u %u %u %u %u %u %u", 7) \
+X( 25, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8UNSIGNED, "%u %u %u %u %u %u %u %u", 8) \
+\
+X(  1, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_STATE_CHANGE, "[Host time %09u%09u%09u] Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-graceful_offloading 4-forced_offloading 5-cooldown). Status %u (1-ok 0-fail)", 7) \
+X(  2, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_MTS_PERMISSION_CHANGED, "[Host time %09u%09u%09u] Updated permission for Driver ID %u to perform MTS kicks: %u (1 = allowed, 0 = not allowed)", 5) \
+X(  3, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_INIT_CCBS, "[Host time %09u%09u%09u] Driver ID %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x", 9) \
+X(  4, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_INIT_CONFIG, "[Host time %09u%09u%09u] Initialised OS %u with config flags 0x%08x", 5) \
+X(  5, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_MAX, "[Host time %09u%09u%09u] Time Slice: Update Failed, Max total allocations exceeded 100pc", 3)\
+X(  6, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_DYNAMIC_LOW, "[Host time %09u%09u%09u] Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 3)\
+X(  7, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_UPDATE_SUCCESS, "[Host time %09u%09u%09u] Time Slice: Updated Successfully", 3)\
+X(  8, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_ISOLATION_GROUP_CHANGE, "[Host time %09u%09u%09u] Changing Driver ID %u's isolation group from %u to %u", 6) \
+X(  9, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_PRIORITY_CHANGE, "[Host time %09u%09u%09u] Changing Driver ID %u's priority from %u to %u", 6) \
+X( 10, RGXFW_GROUP_VZ, RGXFW_SF_VZ_WDG_TRIGGER, "[Host time %09u%09u%09u] Driver ID %u failed to respond to the virtualisation watchdog in time. Timestamp of its last input = %u", 5) \
+\
+\
+X(  1, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_STATUS, "FBCDC: Core=0x%08x, Status=0x%08x, Signature status=0x%08x", 3) \
+X(  2, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_DETECTED, "FBCDC signature failure detected so block scheduling more work", 0) \
+X(  3, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_CLEARED, "FBCDC signature cleared which unlocks scheduling more work", 0) \
+X(  4, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_FAILURE, "TRP failure: %u", 1) \
+X(  5, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SW_TRP_STATE, "SW TRP State: %u", 1) \
+X(  6, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_HW_KICK, "HW kick event (%u)", 1) \
+X(  7, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_CHECKSUMS, "GPU core (%u/%u): checksum 0x%08x vs. 0x%08x", 4) \
+X(  8, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_UNIT_CHECKSUMS, "GPU core (%u/%u), unit (%u,%u): checksum 0x%08x vs. 0x%08x", 6) \
+X(  9, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_CHECK_REG, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 10, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_USC_SLOTS_CHECK, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X( 11, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_USC_REG_CHECK, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 12, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_STATE_HWRTDATA_DEPRECATED, "TRP HWRTData: 0x%08x, state: %u", 2) \
+X( 13, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_STATE_CNTX_DEPRECATED, "TRP Context: 0x%08x, state: %u", 2) \
+X( 14, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_FAILURE_CNTX, "TRP Context: 0x%08x, failure: %u", 2) \
+X( 15, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_UNEXPECTED_EVENT_DEPRECATED, "Unexpected TRP pass completion on DM: %u",1) \
+X( 16, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_ENABLED, "ICS enabled for DM: %u",1) \
+X( 17, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_INTERVAL,"ICS: interval: %u cycles, threshold: %u cycles",2) \
+X( 18, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_FAULT,"ICS fault detected, GPU cores mask 0x%08x",1) \
+X( 19, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_MODE,"Set ICS mode to: %u ",1) \
+X( 20, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_TESTS_COMPLETION_FAILURE,"ICS tests not completed in mode %u",1) \
+X( 21, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_TESTS_START_FAILURE,"ICS tests failed to start",0) \
+X( 22, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SELF_TEST_START,"Start Safety self-test",0) \
+X( 23, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SELF_TEST_END,"Safety self-test completed",0) \
+\
+X(  1, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_CHECK_REG, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X(  2, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_USC_SLOTS_CHECK, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X(  3, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_USC_REG_CHECK, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X(  4, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU, "Memory dump: Addr=0x%02x%08x, Size=%d, ContextId=%u, DM=%u", 5) \
+X(  5, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS4, "  0x%02x%08x  %08x %08x %08x %08x", 6) \
+X(  6, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS3, "  0x%02x%08x  %08x %08x %08x", 5) \
+X(  7, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS2, "  0x%02x%08x  %08x %08x", 4) \
+X(  8, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS1, "  0x%02x%08x  %08x", 3) \
+X(  9, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW, "Memory dump: Addr=0x%08x, Size=%d", 2) \
+X(  0, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS4, "  0x%08x  %08x %08x %08x %08x", 5) \
+X( 11, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS3, "  0x%08x  %08x %08x %08x", 4) \
+X( 12, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS2, "  0x%08x  %08x %08x", 3) \
+X( 13, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS1, "  0x%08x  %08x", 2) \
+\
+X(65535, RGXFW_GROUP_NULL, RGXFW_SF_LAST, "You should not use this string", 15)
+
+
+/*  The symbolic names found in the table above are assigned an ui32 value of
+ *  the following format:
+ *  31 30 28 26      20   19  16    15  12      11  10           0   bits
+ *  -   ---   --- ----     ----      ----        -   --- ---- ----
+ *        0-10: id number
+ *    12-15,11: group id number (bits 0-3 and bit 4)
+ *       16-19: number of parameters
+ *       20-26: unused
+ *       28-30: active: identify SF packet, otherwise regular int32
+ *          31: reserved for signed/unsigned compatibility
+ *
+ *   The following macro assigns those values to the enum generated SF ids list.
+ */
+#define RGXFW_LOG_IDMARKER			(0x70000000U)
+#define RGXFW_LOG_CREATESFID(a,b,e) ((IMG_UINT32)(a) | (((IMG_UINT32)(b)&0x10U)<<7U) | (((IMG_UINT32)(b)&0xFU)<<12U) | ((IMG_UINT32)(e)<<16U)) | RGXFW_LOG_IDMARKER
+
+#define RGXFW_LOG_IDMASK			(0xFFF00000U)
+#define RGXFW_LOG_VALIDID(I)		(((I) & RGXFW_LOG_IDMASK) == RGXFW_LOG_IDMARKER)
+
+typedef enum {
+#define X(a, b, c, d, e) c = RGXFW_LOG_CREATESFID(a,b,e),
+	RGXFW_LOG_SFIDLIST
+#undef X
+} RGXFW_LOG_SFids;
+
+/* Return the group id number that the given (enum generated) id belongs to */
+#define RGXFW_SF_GID(x) ((((IMG_UINT32)(x)>>12) & 0xfU)|(((IMG_UINT32)(x)>>7) & 0x10U))
+/* Return the id number that the given (enum generated) id belongs to */
+#define RGXFW_SF_ID(x) ((IMG_UINT32)(x) & 0x7ffU)
+/* Returns how many arguments the SF(string format) for the given (enum generated) id requires */
+#define RGXFW_SF_PARAMNUM(x) (((IMG_UINT32)(x)>>16) & 0xfU)
+
+#endif /* RGX_FWIF_SF_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_heap_firmware.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_heap_firmware.h
new file mode 100644
index 000000000000..2628cd82e38e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_heap_firmware.h
@@ -0,0 +1,135 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX FW heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAP_FIRMWARE_H)
+#define RGX_HEAP_FIRMWARE_H
+
+/* Start at 903GiB. Size of 32MB per OSID (see rgxheapconfig.h)
+ * NOTE:
+ *      The firmware heaps bases and sizes are defined here to
+ *      simplify #include dependencies, see rgxheapconfig.h
+ *      for the full RGX virtual address space layout.
+ */
+
+/*
+ * The Config heap holds initialisation data shared between the
+ * the driver and firmware (e.g. pointers to the KCCB and FWCCB).
+ * The Main Firmware heap size is adjusted accordingly but most
+ * of the map / unmap functions must take into consideration
+ * the entire range (i.e. main and config heap).
+ */
+#define RGX_FIRMWARE_NUMBER_OF_FW_HEAPS              (IMG_UINT32_C(2))
+#define RGX_FIRMWARE_HEAP_SHIFT                      RGX_FW_HEAP_SHIFT
+#define RGX_FIRMWARE_RAW_HEAP_BASE                   (IMG_UINT64_C(0xE1C0000000))
+#define RGX_FIRMWARE_RAW_HEAP_SIZE                   (IMG_UINT32_C(1) << RGX_FIRMWARE_HEAP_SHIFT)
+
+/* To enable the firmware to compute the exact address of structures allocated by the KM
+ * in the Fw Config subheap, regardless of the KM's page size (and PMR granularity),
+ * objects allocated consecutively but from different PMRs (due to differing memalloc flags)
+ * are allocated with a 64kb offset. This way, all structures will be located at the same base
+ * addresses when the KM is running with a page size of 4k, 16k or 64k.  */
+#define RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY    (IMG_UINT32_C(0x10000))
+
+/* Ensure the heap can hold 3 PMRs of maximum supported granularity (192KB):
+ * 1st PMR: RGXFWIF_CONNECTION_CTL
+ * 2nd PMR: RGXFWIF_OSINIT
+ * 3rd PMR: RGXFWIF_SYSINIT */
+#define RGX_FIRMWARE_CONFIG_HEAP_SIZE                (IMG_UINT32_C(3)*RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY)
+
+#define RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE          (RGX_FIRMWARE_RAW_HEAP_SIZE - RGX_FIRMWARE_CONFIG_HEAP_SIZE)
+/*
+ * MIPS FW needs space in the Main heap to map GPU memory.
+ * This space is taken from the MAIN heap, to avoid creating a new heap.
+ */
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL       (IMG_UINT32_C(0x100000)) /* 1MB */
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101     (IMG_UINT32_C(0x400000)) /* 4MB */
+
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL      (RGX_FIRMWARE_RAW_HEAP_SIZE -  RGX_FIRMWARE_CONFIG_HEAP_SIZE - \
+                                                           RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL)
+
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101    (RGX_FIRMWARE_RAW_HEAP_SIZE -  RGX_FIRMWARE_CONFIG_HEAP_SIZE - \
+                                                           RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101)
+
+#if !defined(__KERNEL__)
+#if defined(FIX_HW_BRN_65101)
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE      RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE        RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101
+
+#include "img_defs.h"
+static_assert((RGX_FIRMWARE_RAW_HEAP_SIZE) >= IMG_UINT32_C(0x800000), "MIPS GPU map size cannot be increased due to BRN65101 with a small FW heap");
+
+#else
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE      RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE        RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL
+#endif
+#endif /* !defined(__KERNEL__) */
+
+#define RGX_FIRMWARE_MAIN_HEAP_BASE             RGX_FIRMWARE_RAW_HEAP_BASE
+#define RGX_FIRMWARE_CONFIG_HEAP_BASE           (RGX_FIRMWARE_MAIN_HEAP_BASE + \
+                                                 RGX_FIRMWARE_RAW_HEAP_SIZE - \
+                                                 RGX_FIRMWARE_CONFIG_HEAP_SIZE)
+
+
+/* 1 Mb can hold the maximum amount of page tables for the memory shared between the firmware and all KM drivers:
+ *  MAX(RAW_HEAP_SIZE) = 32 Mb; MAX(NUMBER_OS) = 8; Total shared memory = 256 Mb;
+ *  MMU objects required: 65536 PTEs; 16 PDEs; 1 PCE; */
+#define RGX_FIRMWARE_MAX_PAGETABLE_SIZE (1 * 1024 * 1024)
+
+#define RGX_FW_CONFIG_HEAP_SIZE (1 << RGX_FW_HEAP_SHIFT)
+#define RGX_FW_MAX_HEAP_SIZE (1 << 28)
+/*
+ * The maximum configurable size via RGX_FW_HEAP_SHIFT is 256MiB (1<<28) and
+ * the minimum is 4MiB (1<<22); the firmware heap size is dependent
+ * on the number of drivers supported.
+ */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	#if defined(RGX_FW_HEAP_SHIFT) && (RGX_FW_HEAP_SHIFT < 22 || (RGX_FW_CONFIG_HEAP_SIZE > (RGX_FW_MAX_HEAP_SIZE / RGX_NUM_DRIVERS_SUPPORTED)))
+	#error "RGX_FW_HEAP_SHIFT is outside valid range"
+	#endif
+#else
+	#if defined(RGX_FW_HEAP_SHIFT) && (RGX_FW_HEAP_SHIFT < 22 || RGX_FW_HEAP_SHIFT > 28)
+	#error "RGX_FW_HEAP_SHIFT is outside valid range [22,28]"
+	#endif
+#endif
+
+
+#endif /* RGX_HEAP_FIRMWARE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_hwperf_common.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_hwperf_common.h
new file mode 100644
index 000000000000..070bfede23d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_hwperf_common.h
@@ -0,0 +1,1645 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_COMMON_H_
+#define RGX_HWPERF_COMMON_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_common_asserts.h"
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_sync_km.h"
+
+
+/******************************************************************************
+ * Packet Event Type Enumerations
+ *****************************************************************************/
+
+/*! Type used to encode the event that generated the packet.
+ * NOTE: When this type is updated the corresponding hwperfbin2json tool
+ * source needs to be updated as well. The RGX_HWPERF_EVENT_MASK_* macros will
+ * also need updating when adding new types.
+ *
+ * @par
+ * The event type values are incrementing integers for use as a shift ordinal
+ * in the event filtering process at the point events are generated.
+ * This scheme thus implies a limit of 63 event types.
+ */
+
+typedef IMG_UINT32 RGX_HWPERF_EVENT_TYPE;
+
+#define RGX_HWPERF_INVALID				0x00U /*!< Invalid. Reserved value. */
+
+/*! FW types 0x01..0x06 */
+#define RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE	0x01U
+
+#define RGX_HWPERF_FW_BGSTART			0x01U /*!< Background task processing start */
+#define RGX_HWPERF_FW_BGEND				0x02U /*!< Background task end */
+#define RGX_HWPERF_FW_IRQSTART			0x03U /*!< IRQ task processing start */
+
+#define RGX_HWPERF_FW_IRQEND			0x04U /*!< IRQ task end */
+#define RGX_HWPERF_FW_DBGSTART			0x05U /*!< Debug event start */
+#define RGX_HWPERF_FW_DBGEND			0x06U /*!< Debug event end */
+
+#define RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE		0x06U
+
+/*! HW types 0x07..0x19 */
+#define RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE	0x07U
+
+#define RGX_HWPERF_HW_PMOOM_TAPAUSE		0x07U /*!< TA Pause at PM Out of Memory */
+
+#define RGX_HWPERF_HW_TAKICK			0x08U /*!< TA task started */
+#define RGX_HWPERF_HW_TAFINISHED		0x09U /*!< TA task finished */
+#define RGX_HWPERF_HW_3DTQKICK			0x0AU /*!< 3D TQ started */
+#define RGX_HWPERF_HW_3DKICK			0x0BU /*!< 3D task started */
+#define RGX_HWPERF_HW_3DFINISHED		0x0CU /*!< 3D task finished */
+#define RGX_HWPERF_HW_CDMKICK			0x0DU /*!< CDM task started */
+#define RGX_HWPERF_HW_CDMFINISHED		0x0EU /*!< CDM task finished */
+#define RGX_HWPERF_HW_TLAKICK			0x0FU /*!< TLA task started */
+#define RGX_HWPERF_HW_TLAFINISHED		0x10U /*!< TLS task finished */
+#define RGX_HWPERF_HW_3DSPMKICK			0x11U /*!< 3D SPM task started */
+#define RGX_HWPERF_HW_PERIODIC			0x12U /*!< Periodic event with updated HW counters */
+#define RGX_HWPERF_HW_RTUKICK			0x13U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_RTUFINISHED		0x14U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_SHGKICK			0x15U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_SHGFINISHED		0x16U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_3DTQFINISHED		0x17U /*!< 3D TQ finished */
+#define RGX_HWPERF_HW_3DSPMFINISHED		0x18U /*!< 3D SPM task finished */
+
+#define RGX_HWPERF_HW_PMOOM_TARESUME	0x19U /*!< TA Resume after PM Out of Memory */
+
+/*! HW_EVENT_RANGE0 used up. Use next empty range below to add new hardware events */
+#define RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE	0x19U
+
+/*! other types 0x1A..0x1F */
+#define RGX_HWPERF_CLKS_CHG				0x1AU /*!< Clock speed change in GPU */
+#define RGX_HWPERF_GPU_STATE_CHG		0x1BU /*!< GPU work state change */
+
+/*! power types 0x20..0x27 */
+#define RGX_HWPERF_PWR_EST_RANGE_FIRST_TYPE	0x20U
+#define RGX_HWPERF_PWR_EST_REQUEST		0x20U /*!< Power estimate requested (via GPIO) */
+#define RGX_HWPERF_PWR_EST_READY		0x21U /*!< Power estimate inputs ready */
+#define RGX_HWPERF_PWR_EST_RESULT		0x22U /*!< Power estimate result calculated */
+#define RGX_HWPERF_PWR_EST_RANGE_LAST_TYPE	0x22U
+
+#define RGX_HWPERF_PWR_CHG				0x23U /*!< Power state change */
+
+/*! HW_EVENT_RANGE1 0x28..0x2F, for accommodating new hardware events */
+#define RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE	0x28U
+
+#define RGX_HWPERF_HW_TDMKICK			0x28U /*!< TDM task started */
+#define RGX_HWPERF_HW_TDMFINISHED		0x29U /*!< TDM task finished */
+#define RGX_HWPERF_HW_NULLKICK			0x2AU /*!< NULL event */
+
+#define RGX_HWPERF_HW_EVENT_RANGE1_LAST_TYPE 0x2AU
+#define RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD		0x2BU
+
+/*! context switch types 0x30..0x31 */
+#define RGX_HWPERF_CSW_START			0x30U /*!< HW context store started */
+#define RGX_HWPERF_CSW_FINISHED			0x31U /*!< HW context store finished */
+
+/*! DVFS events */
+#define RGX_HWPERF_DVFS					0x32U /*!< Dynamic voltage/frequency scaling events */
+
+/*! firmware misc 0x38..0x39 */
+#define RGX_HWPERF_UFO					0x38U /*!< FW UFO Check / Update */
+#define RGX_HWPERF_FWACT				0x39U /*!< FW Activity notification */
+
+/*! last */
+#define RGX_HWPERF_LAST_TYPE			0x3BU
+
+/*! This enumeration must have a value that is a power of two as it is
+ * used in masks and a filter bit field (currently 64 bits long).
+ */
+#define RGX_HWPERF_MAX_TYPE				0x40U
+
+static_assert(RGX_HWPERF_LAST_TYPE < RGX_HWPERF_MAX_TYPE, "Too many HWPerf event types");
+
+/*! Macro used to check if an event type ID is present in the known set of hardware type events */
+#define HWPERF_PACKET_IS_HW_TYPE(_etype)	(((_etype) >= RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE && (_etype) <= RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE) || \
+											 ((_etype) >= RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE && (_etype) <= RGX_HWPERF_HW_EVENT_RANGE1_LAST_TYPE))
+
+/*! Macro used to check if an event type ID is present in the known set of firmware type events */
+#define HWPERF_PACKET_IS_FW_TYPE(_etype)					\
+	((_etype) >= RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE &&	\
+	 (_etype) <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE)
+
+
+typedef enum {
+	RGX_HWPERF_HOST_INVALID   = 0x00,           /*!< Invalid, do not use. */
+	RGX_HWPERF_HOST_ENQ       = 0x01,           /*!< ``0x01`` Kernel driver has queued GPU work.
+	                                             See RGX_HWPERF_HOST_ENQ_DATA */
+	RGX_HWPERF_HOST_UFO       = 0x02,           /*!< ``0x02`` UFO updated by the driver.
+	                                             See RGX_HWPERF_HOST_UFO_DATA */
+	RGX_HWPERF_HOST_ALLOC     = 0x03,           /*!< ``0x03`` Resource allocated.
+	                                             See RGX_HWPERF_HOST_ALLOC_DATA */
+	RGX_HWPERF_HOST_CLK_SYNC  = 0x04,           /*!< ``0x04`` GPU / Host clocks correlation data.
+	                                             See RGX_HWPERF_HOST_CLK_SYNC_DATA */
+	RGX_HWPERF_HOST_FREE      = 0x05,           /*!< ``0x05`` Resource freed,
+	                                             See RGX_HWPERF_HOST_FREE_DATA */
+	RGX_HWPERF_HOST_MODIFY    = 0x06,           /*!< ``0x06`` Resource modified / updated.
+	                                             See RGX_HWPERF_HOST_MODIFY_DATA */
+	RGX_HWPERF_HOST_DEV_INFO  = 0x07,           /*!< ``0x07`` Device Health status.
+	                                             See RGX_HWPERF_HOST_DEV_INFO_DATA */
+	RGX_HWPERF_HOST_INFO      = 0x08,           /*!< ``0x08`` Device memory usage information.
+	                                             See RGX_HWPERF_HOST_INFO_DATA */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT = 0x09,     /*!< ``0x09`` Wait for sync event.
+	                                             See RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA */
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE  = 0x0A, /*!< ``0x0A`` Software timeline advanced.
+	                                             See RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA */
+	RGX_HWPERF_HOST_CLIENT_INFO = 0x0B,			/*!< ``0x0B`` Additional client info.
+	                                             See RGX_HWPERF_HOST_CLIENT_INFO_DATA */
+
+	/*! last */
+	RGX_HWPERF_HOST_LAST_TYPE,
+
+	/*! This enumeration must have a value that is a power of two as it is
+	 * used in masks and a filter bit field (currently 32 bits long).
+	 */
+	RGX_HWPERF_HOST_MAX_TYPE  = 0x20
+} RGX_HWPERF_HOST_EVENT_TYPE;
+
+/*!< The event type values are incrementing integers for use as a shift ordinal
+ * in the event filtering process at the point events are generated.
+ * This scheme thus implies a limit of 31 event types.
+ */
+static_assert(RGX_HWPERF_HOST_LAST_TYPE < RGX_HWPERF_HOST_MAX_TYPE, "Too many HWPerf host event types");
+
+/*! Define containing bit position for 32bit feature flags used in hwperf and api */
+typedef IMG_UINT32 RGX_HWPERF_FEATURE_FLAGS;
+#define RGX_HWPERF_FEATURE_PERFBUS_FLAG                    0x00000001U
+#define RGX_HWPERF_FEATURE_S7_TOP_INFRASTRUCTURE_FLAG      0x00000002U
+#define RGX_HWPERF_FEATURE_XT_TOP_INFRASTRUCTURE_FLAG      0x00000004U
+#define RGX_HWPERF_FEATURE_PERF_COUNTER_BATCH_FLAG         0x00000008U
+#define RGX_HWPERF_FEATURE_ROGUEXE_FLAG                    0x00000010U
+#define RGX_HWPERF_FEATURE_DUST_POWER_ISLAND_S7_FLAG       0x00000020U
+#define RGX_HWPERF_FEATURE_PBE2_IN_XE_FLAG                 0x00000040U
+#define RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION             0x00000080U
+#define RGX_HWPERF_FEATURE_MULTICORE_FLAG                  0x00000100U
+#define RGX_HWPERF_FEATURE_RAYTRACING_FLAG                 0x00000200U
+#define RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG     0x00000400U
+#define RGX_HWPERF_FEATURE_VOLCANIC_FLAG                   0x00000800U
+#define RGX_HWPERF_FEATURE_ROGUE_FLAG                      0x00001000U
+#define RGX_HWPERF_FEATURE_RESERVED1_FLAG                  0x00002000U
+#define RGX_HWPERF_FEATURE_CXT_XTP_TOP_INFRASTRUCTURE_FLAG 0x00004000U
+#define RGX_HWPERF_FEATURE_AX_TOP_INFRASTRUCTURE_FLAG      0x00008000U
+#define RGX_HWPERF_FEATURE_BX_TOP_INFRASTRUCTURE_FLAG      0x00010000U
+#define RGX_HWPERF_FEATURE_DX_TOP_INFRASTRUCTURE_FLAG      0x00020000U
+
+/* ! Define for RGX_HWPERF_DM type. The values are architecture specific */
+typedef IMG_UINT32 RGX_HWPERF_DM;
+
+/******************************************************************************
+ * Packet Header Format Version 2 Types
+ *****************************************************************************/
+
+/*! Major version number of the protocol in operation
+ */
+#define RGX_HWPERF_V2_FORMAT 2
+
+/*! Signature ASCII pattern 'HWP2' found in the first word of a HWPerfV2 packet
+ */
+#define HWPERF_PACKET_V2_SIG		0x48575032
+
+/*! Signature ASCII pattern 'HWPA' found in the first word of a HWPerfV2a packet
+ */
+#define HWPERF_PACKET_V2A_SIG		0x48575041
+
+/*! Signature ASCII pattern 'HWPB' found in the first word of a HWPerfV2b packet
+ */
+#define HWPERF_PACKET_V2B_SIG		0x48575042
+
+/*! Signature ASCII pattern 'HWPC' found in the first word of a HWPerfV2c packet
+ */
+#define HWPERF_PACKET_V2C_SIG		0x48575043
+
+#define HWPERF_PACKET_ISVALID(_val) (((_val) == HWPERF_PACKET_V2_SIG) || ((_val) == HWPERF_PACKET_V2A_SIG) || ((_val) == HWPERF_PACKET_V2B_SIG) || ((_val) == HWPERF_PACKET_V2C_SIG))
+/*!< Checks that the packet signature is one of the supported versions */
+
+/*! Type defines the HWPerf packet header common to all events. */
+typedef struct
+{
+	IMG_UINT32  ui32Sig;        /*!< Always the value HWPERF_PACKET_SIG */
+	IMG_UINT32  ui32Size;       /*!< Overall packet size in bytes */
+	IMG_UINT32  eTypeId;        /*!< Event type information field */
+	IMG_UINT32  ui32Ordinal;    /*!< Sequential number of the packet */
+	IMG_UINT64  ui64Timestamp;  /*!< Event timestamp */
+} RGX_HWPERF_V2_PACKET_HDR, *RGX_PHWPERF_V2_PACKET_HDR;
+
+RGX_FW_STRUCT_OFFSET_ASSERT(RGX_HWPERF_V2_PACKET_HDR, ui64Timestamp);
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_V2_PACKET_HDR);
+
+
+/*! Mask for use with the IMG_UINT32 ui32Size header field */
+#define RGX_HWPERF_SIZE_MASK         0xFFFFU
+
+/*! This macro defines an upper limit to which the size of the largest variable
+ * length HWPerf packet must fall within, currently 3KB. This constant may be
+ * used to allocate a buffer to hold one packet.
+ * This upper limit is policed by packet producing code.
+ */
+#define RGX_HWPERF_MAX_PACKET_SIZE   0xC00U
+
+/*! Defines an upper limit to the size of a variable length packet payload.
+ */
+#define RGX_HWPERF_MAX_PAYLOAD_SIZE	 ((IMG_UINT32)(RGX_HWPERF_MAX_PACKET_SIZE-\
+	sizeof(RGX_HWPERF_V2_PACKET_HDR)))
+
+/*! Macro which takes a structure name and provides the packet size for
+ * a fixed size payload packet, rounded up to 8 bytes to align packets
+ * for 64 bit architectures. */
+#define RGX_HWPERF_MAKE_SIZE_FIXED(_struct)       ((IMG_UINT32)(RGX_HWPERF_SIZE_MASK&(sizeof(RGX_HWPERF_V2_PACKET_HDR)+PVR_ALIGN(sizeof(_struct), PVRSRVTL_PACKET_ALIGNMENT))))
+
+/*! Macro which takes the number of bytes written in the data payload of a
+ * packet for a variable size payload packet, rounded up to 8 bytes to
+ * align packets for 64 bit architectures. */
+#define RGX_HWPERF_MAKE_SIZE_VARIABLE(_size)      ((IMG_UINT32)(RGX_HWPERF_SIZE_MASK&((IMG_UINT32)sizeof(RGX_HWPERF_V2_PACKET_HDR)+PVR_ALIGN((_size), PVRSRVTL_PACKET_ALIGNMENT))))
+
+/*! Macro to obtain the size of the packet */
+#define RGX_HWPERF_GET_SIZE(_packet_addr)         ((IMG_UINT16)(((_packet_addr)->ui32Size) & RGX_HWPERF_SIZE_MASK))
+
+/*! Macro to obtain the size of the packet data */
+#define RGX_HWPERF_GET_DATA_SIZE(_packet_addr)    (RGX_HWPERF_GET_SIZE(_packet_addr) - sizeof(RGX_HWPERF_V2_PACKET_HDR))
+
+/*! Masks for use with the IMG_UINT32 eTypeId header field */
+#define RGX_HWPERF_TYPEID_MASK			0x0007FFFFU
+#define RGX_HWPERF_TYPEID_EVENT_MASK	0x00007FFFU
+#define RGX_HWPERF_TYPEID_THREAD_MASK	0x00008000U
+#define RGX_HWPERF_TYPEID_STREAM_MASK	0x00070000U
+#define RGX_HWPERF_TYPEID_META_DMA_MASK	0x00080000U
+#define RGX_HWPERF_TYPEID_M_CORE_MASK	0x00100000U
+#define RGX_HWPERF_TYPEID_PIPEDM_MASK	0x00200000U
+#define RGX_HWPERF_TYPEID_OSID_MASK		0x1f000000U
+
+/*! Meta thread macros for encoding the ID into the type field of a packet */
+#define RGX_HWPERF_META_THREAD_SHIFT	15U
+#define RGX_HWPERF_META_THREAD_ID0		0x0U  /*!< Meta Thread 0 ID */
+#define RGX_HWPERF_META_THREAD_ID1		0x1U  /*!< Meta Thread 1 ID */
+/*! Obsolete, kept for source compatibility */
+#define RGX_HWPERF_META_THREAD_MASK		0x1U
+/*! Stream ID macros for encoding the ID into the type field of a packet */
+#define RGX_HWPERF_STREAM_SHIFT			16U
+/*! Meta DMA macro for encoding how the packet was generated into the type field of a packet */
+#define RGX_HWPERF_META_DMA_SHIFT		19U
+/*! Bit-shift macro used for encoding multi-core data into the type field of a packet */
+#define RGX_HWPERF_M_CORE_SHIFT			20U
+/*! Bit-shift macro used for encoding Pipeline DM data into the type field of a packet */
+#define RGX_HWPERF_PIPEDM_SHIFT			21U
+/*! OSID bit-shift macro used for encoding OSID into type field of a packet */
+#define RGX_HWPERF_OSID_SHIFT			24U
+
+/*! HWPerf Stream ID type definition. Maximum of 32bits. */
+typedef IMG_UINT32 RGX_HWPERF_STREAM_ID;
+/*! Events from the Firmware/GPU */
+#define RGX_HWPERF_STREAM_ID0_FW		0U
+/*! Events from the Server host driver component */
+#define RGX_HWPERF_STREAM_ID1_HOST		1U
+/*! Events from the Client host driver component */
+#define RGX_HWPERF_STREAM_ID2_CLIENT	2U
+#define RGX_HWPERF_STREAM_ID_LAST		3U
+
+/* Checks if all stream IDs can fit under RGX_HWPERF_TYPEID_STREAM_MASK. */
+static_assert(((IMG_UINT32)RGX_HWPERF_STREAM_ID_LAST - 1U) < (RGX_HWPERF_TYPEID_STREAM_MASK >> RGX_HWPERF_STREAM_SHIFT),
+		"Too many HWPerf stream IDs.");
+
+/*! Compile-time value used to seed the Multi-Core (MC) bit in the typeID field.
+ *  Only set by RGX_FIRMWARE builds.
+ */
+#if defined(RGX_FIRMWARE)
+# if defined(RGX_FEATURE_GPU_MULTICORE_SUPPORT)
+#define RGX_HWPERF_M_CORE_VALUE 1U  /*!< 1 => Multi-core supported */
+# else
+#define RGX_HWPERF_M_CORE_VALUE 0U  /*!< 0 => Multi-core not supported */
+# endif
+# if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION) && (RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION > 0)
+#define RGX_HWPERF_PIPEDM_VALUE 1U  /*!< 1 => Pipeline DM supported */
+# else
+#define RGX_HWPERF_PIPEDM_VALUE 0U  /*!< 0 => Pipeline DM not supported */
+# endif
+#else
+#define RGX_HWPERF_M_CORE_VALUE 0U  /*!< 0 => Multi-core not supported */
+#define RGX_HWPERF_PIPEDM_VALUE 0U  /*!< 0 => Pipeline DM not supported */
+#endif
+
+/*! Macros used to set the packet type and encode meta thread ID (0|1),
+ * HWPerf stream ID, multi-core capability and OSID within the typeID */
+#define RGX_HWPERF_MAKE_TYPEID(_stream, _type, _thread, _metadma, _osid)\
+		((IMG_UINT32) ((RGX_HWPERF_TYPEID_STREAM_MASK&((IMG_UINT32)(_stream) << RGX_HWPERF_STREAM_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_THREAD_MASK & ((IMG_UINT32)(_thread) << RGX_HWPERF_META_THREAD_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_EVENT_MASK & (IMG_UINT32)(_type)) | \
+		(RGX_HWPERF_TYPEID_META_DMA_MASK & ((IMG_UINT32)(_metadma) << RGX_HWPERF_META_DMA_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_OSID_MASK & ((IMG_UINT32)(_osid) << RGX_HWPERF_OSID_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_M_CORE_MASK & ((IMG_UINT32)(RGX_HWPERF_M_CORE_VALUE) << RGX_HWPERF_M_CORE_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_PIPEDM_MASK & ((IMG_UINT32)(RGX_HWPERF_PIPEDM_VALUE) << RGX_HWPERF_PIPEDM_SHIFT))))
+
+/*! Obtains the event type that generated the packet */
+#define RGX_HWPERF_GET_TYPE(_packet_addr)            (((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_EVENT_MASK)
+
+/*! Obtains the META Thread number that generated the packet */
+#define RGX_HWPERF_GET_THREAD_ID(_packet_addr)       (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_THREAD_MASK) >> RGX_HWPERF_META_THREAD_SHIFT))
+
+/*! Determines if the packet generated contains multi-core data */
+#define RGX_HWPERF_GET_M_CORE(_packet_addr)          (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_M_CORE_MASK) >> RGX_HWPERF_M_CORE_SHIFT)
+
+/*! Determines if the packet generated contains multi-core data */
+#define RGX_HWPERF_GET_PIPEDM(_packet_addr)          (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_PIPEDM_MASK) >> RGX_HWPERF_PIPEDM_SHIFT)
+
+/*! Obtains the guest OSID which resulted in packet generation */
+#define RGX_HWPERF_GET_OSID(_packet_addr)            (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_OSID_MASK) >> RGX_HWPERF_OSID_SHIFT)
+
+/*! Obtain stream id */
+#define RGX_HWPERF_GET_STREAM_ID(_packet_addr)       (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_STREAM_MASK) >> RGX_HWPERF_STREAM_SHIFT))
+
+/*! Obtain information about how the packet was generated, which might affect payload total size */
+#define RGX_HWPERF_GET_META_DMA_INFO(_packet_addr)   (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_META_DMA_MASK) >> RGX_HWPERF_META_DMA_SHIFT))
+
+/*! Obtains a typed pointer to a packet given a buffer address */
+#define RGX_HWPERF_GET_PACKET(_buffer_addr)            ((RGX_HWPERF_V2_PACKET_HDR *)(void *)  (_buffer_addr))
+/*! Obtains a typed pointer to a data structure given a packet address */
+#define RGX_HWPERF_GET_PACKET_DATA_BYTES(_packet_addr) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR)))
+/*! Obtains a typed pointer to the next packet given a packet address */
+#define RGX_HWPERF_GET_NEXT_PACKET(_packet_addr)       ((RGX_HWPERF_V2_PACKET_HDR *)  (IMG_OFFSET_ADDR((_packet_addr), RGX_HWPERF_SIZE_MASK&((_packet_addr)->ui32Size))))
+
+/*! Obtains a typed pointer to a packet header given the packet data address */
+#define RGX_HWPERF_GET_PACKET_HEADER(_packet_addr)     ((RGX_HWPERF_V2_PACKET_HDR *)  (IMG_OFFSET_ADDR((_packet_addr), -(IMG_INT32)sizeof(RGX_HWPERF_V2_PACKET_HDR))))
+
+
+/******************************************************************************
+ * Other Common Defines
+ *****************************************************************************/
+
+/*! Used to align structs using a flexible array member as the definition of the member
+ * size can change between C standards */
+#if (IMG_FLEX_ARRAY_MEMBER + 0)
+#define RGX_HWPERF_FLEX_ARRAY_ALIGN(_align_var) _align_var
+#else
+#define RGX_HWPERF_FLEX_ARRAY_ALIGN(...)
+#endif
+
+/*! Masks for use with the IMG_UINT32 ui32BlkInfo field */
+#define RGX_HWPERF_BLKINFO_BLKCOUNT_MASK	0xFFFF0000U
+#define RGX_HWPERF_BLKINFO_BLKOFFSET_MASK	0x0000FFFFU
+
+/*! Shift for the NumBlocks and counter block offset field in ui32BlkInfo */
+#define RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT	16U
+#define RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT 0U
+
+/*! Macro used to set the block info word as a combination of two 16-bit integers */
+#define RGX_HWPERF_MAKE_BLKINFO(_numblks, _blkoffset) ((IMG_UINT32) ((RGX_HWPERF_BLKINFO_BLKCOUNT_MASK&((_numblks) << RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT)) | (RGX_HWPERF_BLKINFO_BLKOFFSET_MASK&((_blkoffset) << RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT))))
+
+/*! Macro used to obtain the number of counter blocks present in the packet */
+#define RGX_HWPERF_GET_BLKCOUNT(_blkinfo)            (((_blkinfo) & RGX_HWPERF_BLKINFO_BLKCOUNT_MASK) >> RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT)
+
+/*! Obtains the offset of the counter block stream in the packet */
+#define RGX_HWPERF_GET_BLKOFFSET(_blkinfo)           (((_blkinfo) & RGX_HWPERF_BLKINFO_BLKOFFSET_MASK) >> RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT)
+
+/*! This macro gets the number of blocks depending on the packet version */
+#define RGX_HWPERF_GET_NUMBLKS(_sig, _packet_data, _numblocks) \
+	do { \
+		if (HWPERF_PACKET_V2B_SIG == (_sig) || HWPERF_PACKET_V2C_SIG == (_sig)) \
+		{ \
+			(_numblocks) = RGX_HWPERF_GET_BLKCOUNT((_packet_data)->ui32BlkInfo);\
+		} \
+		else \
+		{ \
+			IMG_UINT32 ui32VersionOffset = (((_sig) == HWPERF_PACKET_V2_SIG) ? 1 : 3);\
+			(_numblocks) = *(IMG_UINT16 *)(IMG_OFFSET_ADDR(&(_packet_data)->ui32WorkTarget, ui32VersionOffset)); \
+		} \
+	} while (0)
+
+/*! This macro gets the counter stream pointer depending on the packet version */
+#define RGX_HWPERF_GET_CNTSTRM(_sig, _hw_packet_data, _cntstream_ptr) \
+{ \
+	if (HWPERF_PACKET_V2B_SIG == (_sig) || HWPERF_PACKET_V2C_SIG == (_sig)) \
+	{ \
+		(_cntstream_ptr) = (IMG_UINT32 *)(IMG_OFFSET_ADDR((_hw_packet_data), RGX_HWPERF_GET_BLKOFFSET((_hw_packet_data)->ui32BlkInfo))); \
+	} \
+	else \
+	{ \
+		IMG_UINT32 ui32BlkStreamOffsetInWords = (((_sig) == HWPERF_PACKET_V2_SIG) ? 6 : 8); \
+		(_cntstream_ptr) = (IMG_UINT32 *)(IMG_OFFSET_ADDR_DW((_hw_packet_data), ui32BlkStreamOffsetInWords)); \
+	} \
+}
+
+/*! Masks for use with the IMG_UINT32 ui32KickInfo field */
+#define RGX_HWPERF_KICKINFO_STARTBE_MASK	0xFFFFFF00U
+#define RGX_HWPERF_KICKINFO_KICKID_MASK		0x000000FFU
+
+/*! Shift for the Kick ID field in ui32KickInfo */
+#define RGX_HWPERF_KICKINFO_STARTBE_SHIFT 8U
+#define RGX_HWPERF_KICKINFO_KICKID_SHIFT  0U
+
+/*! Macro used to set the kick info field. */
+#define RGX_HWPERF_MAKE_KICKINFO(_startbe, _kickid) \
+        ((IMG_UINT32) (RGX_HWPERF_KICKINFO_STARTBE_MASK&((_startbe) << RGX_HWPERF_KICKINFO_STARTBE_SHIFT))| \
+                      (RGX_HWPERF_KICKINFO_KICKID_MASK&((_kickid) << RGX_HWPERF_KICKINFO_KICKID_SHIFT)))
+
+/*! Macro used to obtain the lowest 24 bits of START_BE if present in the packet */
+#define RGX_HWPERF_GET_STARTBE(_kickinfo)           (((_kickinfo) & RGX_HWPERF_KICKINFO_STARTBE_MASK) >> RGX_HWPERF_KICKINFO_STARTBE_SHIFT)
+
+/*! Macro used to obtain the Kick ID if present in the packet */
+#define RGX_HWPERF_GET_KICKID(_kickinfo)            (((_kickinfo) & RGX_HWPERF_KICKINFO_KICKID_MASK) >> RGX_HWPERF_KICKINFO_KICKID_SHIFT)
+
+/*! Masks for use with the RGX_HWPERF_UFO_EV eEvType field */
+#define RGX_HWPERF_UFO_STREAMSIZE_MASK 0xFFFF0000U
+#define RGX_HWPERF_UFO_STREAMOFFSET_MASK 0x0000FFFFU
+
+/*! Shift for the UFO count and data stream fields */
+#define RGX_HWPERF_UFO_STREAMSIZE_SHIFT 16U
+#define RGX_HWPERF_UFO_STREAMOFFSET_SHIFT 0U
+
+/*! Macro used to set UFO stream info word as a combination of two 16-bit integers */
+#define RGX_HWPERF_MAKE_UFOPKTINFO(_ssize, _soff) \
+        ((IMG_UINT32) ((RGX_HWPERF_UFO_STREAMSIZE_MASK&((_ssize) << RGX_HWPERF_UFO_STREAMSIZE_SHIFT)) | \
+        (RGX_HWPERF_UFO_STREAMOFFSET_MASK&((_soff) << RGX_HWPERF_UFO_STREAMOFFSET_SHIFT))))
+
+/*! Macro used to obtain UFO count*/
+#define RGX_HWPERF_GET_UFO_STREAMSIZE(_streaminfo) \
+        (((_streaminfo) & RGX_HWPERF_UFO_STREAMSIZE_MASK) >> RGX_HWPERF_UFO_STREAMSIZE_SHIFT)
+
+/*! Obtains the offset of the UFO stream in the packet */
+#define RGX_HWPERF_GET_UFO_STREAMOFFSET(_streaminfo) \
+        (((_streaminfo) & RGX_HWPERF_UFO_STREAMOFFSET_MASK) >> RGX_HWPERF_UFO_STREAMOFFSET_SHIFT)
+
+
+/*! This structure holds the data of a firmware packet. */
+typedef struct
+{
+	RGX_HWPERF_DM eDM;             /*!< DataMaster identifier, see RGX_HWPERF_DM */
+	IMG_UINT32 ui32TxtActCyc;      /*!< Meta TXTACTCYC register value */
+	IMG_UINT32 ui32FWPerfCount0;   /*!< Meta/MIPS PERF_COUNT0 register */
+	IMG_UINT32 ui32FWPerfCount1;   /*!< Meta/MIPS PERF_COUNT1 register */
+	IMG_UINT32 ui32TimeCorrIndex;  /*!< Internal field */
+	IMG_UINT32 ui32Padding;        /*!< Reserved */
+} RGX_HWPERF_FW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FW_DATA);
+
+
+/*! This structure holds the data of a gpu work period packet. */
+typedef struct
+{
+	IMG_UINT64 ui64GPUWorkPeriodStartTime;
+	IMG_UINT32 ui32UID;               /*!< Process identifier */
+	IMG_UINT32 ui32TimeCorrIndex;     /*!< Index to the time correlation at the beginning of gpu work period */
+	IMG_UINT32 ui32StartTimeCorrIndex;     /*!< Index to the time correlation at the time the packet was generated */
+	IMG_UINT32 ui32Padding2;          /*!< Reserved. To ensure correct alignment (not written in the packet) */
+} RGX_HWPERF_GPU_WORK_PERIOD_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_GPU_WORK_PERIOD_DATA);
+
+/*! This structure holds the data of a hardware packet, including counters. */
+typedef struct
+{
+	union
+	{
+		IMG_UINT32 ui32DMCyc;         /*!< DataMaster cycle count register, 0 if none */
+		IMG_UINT32 ui32KickStartTime; /*!< Front End start time for Pipeline DMs */
+	};
+	IMG_UINT32 ui32FrameNum;          /*!< Frame number, undefined on some DataMasters */
+	IMG_UINT32 ui32PID;               /*!< Process identifier */
+	IMG_UINT32 ui32DMContext;         /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32WorkTarget;        /*!< RenderTarget for a TA,3D; Frame context for RTU, 0x0 otherwise */
+	IMG_UINT32 ui32ExtJobRef;         /*!< Client driver context job reference used for tracking/debugging */
+	IMG_UINT32 ui32IntJobRef;         /*!< RGX Data master context job reference used for tracking/debugging */
+	IMG_UINT32 ui32TimeCorrIndex;     /*!< Index to the time correlation at the time the packet was generated */
+	IMG_UINT32 ui32BlkInfo;           /*!< <31..16> NumBlocks <15..0> Counter block stream offset */
+	IMG_UINT32 ui32WorkCtx;           /*!< Work context: Render Context for TA/3D; RayTracing Context for RTU/SHG; 0x0 otherwise */
+	IMG_UINT32 ui32CtxPriority;       /*!< Context priority */
+	IMG_UINT32 ui32GPUIdMask;         /*!< GPU IDs active within this event */
+	IMG_UINT32 ui32KickInfo;          /*!< <31..8> Back End start time lowest 24 bits <7..0> GPU Pipeline DM kick ID, 0 if not using Pipeline DMs */
+	IMG_UINT32 ui32KickEndTime;       /*!< Back End finish time for Pipeline DMs */
+	IMG_UINT32 aui32CountBlksStream[IMG_FLEX_ARRAY_MEMBER]; /*!< Optional variable length Counter data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);          /*!< Reserved. To ensure correct alignment (not written in the packet) */
+} RGX_HWPERF_HW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_HW_DATA);
+RGX_FW_STRUCT_OFFSET_ASSERT(RGX_HWPERF_HW_DATA, aui32CountBlksStream);
+
+/*! Mask for use with the aui32CountBlksStream field when decoding the
+ * counter block ID and mask word. */
+#define RGX_HWPERF_CNTBLK_ID_MASK	0xFFFF0000U
+#define RGX_HWPERF_CNTBLK_ID_SHIFT	16U
+
+/*! MAX value used in server handling of counter config arrays */
+#define RGX_CNTBLK_COUNTERS_MAX PVRSRV_HWPERF_COUNTERS_PERBLK
+
+/*! Obtains the counter block ID word from an aui32CountBlksStream field.
+ * The word combines Control bits (15-12), GPU-Id (11-8), Group (7-4), Unit
+ * within group (3-0) */
+#define RGX_HWPERF_GET_CNTBLK_IDW(_word)           ((IMG_UINT16)(((_word)&RGX_HWPERF_CNTBLK_ID_MASK)>>RGX_HWPERF_CNTBLK_ID_SHIFT))
+
+/*! Obtains the counter block ID from the supplied RGX_HWPERF_HW_DATA address
+ * and stream index. May be used in decoding the counter block stream words of
+ * a RGX_HWPERF_HW_DATA structure. */
+#define RGX_HWPERF_GET_CNTBLK_ID(_data_addr, _idx) RGX_HWPERF_GET_CNTBLK_IDW((_data_addr)->aui32CountBlksStream[(_idx)])
+
+/*! Obtains the GPU ID from the supplied RGX_HWPERF_HW_DATA CNTBLK_IDW */
+#define RGX_HWPERF_GET_CNTBLK_GPUW(_word)          ((IMG_UINT16)(((_word)&RGX_CNTBLK_ID_MC_GPU_MASK)>>RGX_CNTBLK_ID_MC_GPU_SHIFT))
+
+#define RGX_HWPERF_GET_CNT_MASKW(_word)           ((IMG_UINT16)((_word)&(~RGX_HWPERF_CNTBLK_ID_MASK)))
+
+/*! Obtains the counter mask from the supplied RGX_HWPERF_HW_DATA address
+ * and stream index. May be used in decoding the counter block stream words
+ * of a RGX_HWPERF_HW_DATA structure. */
+#define RGX_HWPERF_GET_CNT_MASK(_data_addr, _idx) RGX_HWPERF_GET_CNT_MASKW((_data_addr)->aui32CountBlksStream[(_idx)])
+
+
+/*! Context switch packet event */
+typedef struct
+{
+	RGX_HWPERF_DM	eDM;					/*!< DataMaster identifier, see RGX_HWPERF_DM */
+	IMG_UINT32		ui32DMContext;			/*!< GPU Data Master (FW) Context */
+	IMG_UINT32		ui32FrameNum;			/*!< Client Frame number (TA, 3D only) */
+	IMG_UINT32		ui32TxtActCyc;			/*!< Meta TXTACTCYC register value */
+	IMG_UINT32		ui32PerfCycle;			/*!< Cycle count. Used to measure HW context store latency */
+	IMG_UINT32		ui32PerfPhase;			/*!< Phase. Used to determine geometry content */
+	IMG_UINT32		ui32GPUIdMask;			/*!< GPU IDs active within this event */
+	IMG_UINT32		ui32Padding;			/*!< Padding to 8 DWords */
+} RGX_HWPERF_CSW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CSW_DATA);
+
+/*! Enumeration of clocks supporting this event */
+typedef enum
+{
+	RGX_HWPERF_CLKS_CHG_INVALID = 0,
+
+	RGX_HWPERF_CLKS_CHG_NAME_CORE = 1,
+
+	RGX_HWPERF_CLKS_CHG_LAST,
+} RGX_HWPERF_CLKS_CHG_NAME;
+
+/*! This structure holds the data of a clocks change packet. */
+typedef struct
+{
+	IMG_UINT64                ui64NewClockSpeed;         /*!< New Clock Speed (in Hz) */
+	RGX_HWPERF_CLKS_CHG_NAME  eClockName;                /*!< Clock name */
+	IMG_UINT32                ui32CalibratedClockSpeed;  /*!< Calibrated new GPU clock speed (in Hz) */
+	IMG_UINT64                ui64OSMonoTimeStamp;       /*!< OSTimeStamp sampled by the host (monotonic) */
+	IMG_UINT64                ui64CRTimeStamp;           /*!< CRTimeStamp sampled by the host and
+	                                                          correlated to OSTimeStamp */
+	IMG_UINT64                ui64OSSecondaryTimeStamp;  /*!< OSTimeStamp sampled by the host */
+} RGX_HWPERF_CLKS_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CLKS_CHG_DATA);
+
+/*! Enumeration of GPU utilisation states supported by this event */
+typedef IMG_UINT32 RGX_HWPERF_GPU_STATE;
+
+/*! This structure holds the data of a GPU utilisation state change packet. */
+typedef struct
+{
+	RGX_HWPERF_GPU_STATE	eState;		/*!< New GPU utilisation state */
+	IMG_UINT32				uiUnused1;	/*!< Padding */
+	IMG_UINT32				uiUnused2;	/*!< Padding */
+	IMG_UINT32				uiUnused3;	/*!< Padding */
+} RGX_HWPERF_GPU_STATE_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_GPU_STATE_CHG_DATA);
+
+
+/*! Signature pattern 'HPE1' found in the first word of a PWR_EST packet data */
+#define HWPERF_PWR_EST_V1_SIG	0x48504531
+
+/*! Macros to obtain a component field from a counter ID word */
+#define RGX_HWPERF_GET_PWR_EST_HIGH_FLAG(_word) (((_word)&0x80000000)>>31)
+#define RGX_HWPERF_GET_PWR_EST_GPUID(_word)     (((_word)&0x70000000)>>28)
+/*!< Obtains the GPU ID from a counter ID word */
+#define RGX_HWPERF_GET_PWR_EST_UNIT(_word)      (((_word)&0x0F000000)>>24)
+#define RGX_HWPERF_GET_PWR_EST_NUMBER(_word)    ((_word)&0x0000FFFF)
+
+#define RGX_HWPERF_PWR_EST_HIGH_OFFSET		(31)
+#define RGX_HWPERF_PWR_EST_GPUID_OFFSET		(28)
+#define RGX_HWPERF_PWR_EST_GPUID_MASK		(0x7U)
+#define RGX_HWPERF_PWR_EST_UNIT_OFFSET		(24)
+#define RGX_HWPERF_PWR_EST_UNIT_MASK		(0xFU)
+#define RGX_HWPERF_PWR_EST_VALUE_MASK		(0xFFFFU)
+
+/*! This macro constructs a counter ID for a power estimate data stream from
+ * the component parts of: high word flag, unit id, GPU id, counter number */
+#define RGX_HWPERF_MAKE_PWR_EST_COUNTERID(_high, _unit, _core, _number) \
+			((IMG_UINT32)(((IMG_UINT32)((IMG_UINT32)(_high)&0x1U)<<RGX_HWPERF_PWR_EST_HIGH_OFFSET) | \
+			((IMG_UINT32) ((IMG_UINT32)(_unit)&RGX_HWPERF_PWR_EST_UNIT_MASK)<<RGX_HWPERF_PWR_EST_UNIT_OFFSET) | \
+			((IMG_UINT32) ((IMG_UINT32)(_core)&RGX_HWPERF_PWR_EST_GPUID_MASK)<<RGX_HWPERF_PWR_EST_GPUID_OFFSET) | \
+			                           ((_number)&RGX_HWPERF_PWR_EST_VALUE_MASK)))
+
+/*! This structure holds the data for a power estimate packet. */
+typedef struct
+{
+	IMG_UINT32  ui32StreamVersion;  /*!< Version word, HWPERF_PWR_EST_V1_SIG */
+	IMG_UINT32  ui32StreamSize;     /*!< Size of array in bytes of stream data
+	                                     held in the aui32StreamData member */
+	IMG_UINT32  aui32StreamData[IMG_FLEX_ARRAY_MEMBER]; /*!< Counter data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved); /*!< Reserved. To ensure correct alignment */
+} RGX_HWPERF_PWR_EST_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_PWR_EST_DATA);
+
+/*! Enumeration of the kinds of power change events that can occur */
+typedef enum
+{
+	RGX_HWPERF_PWR_UNDEFINED    = 0,
+	RGX_HWPERF_PWR_ON           = 1, /*!< Whole device powered on */
+	RGX_HWPERF_PWR_OFF          = 2, /*!< Whole device powered off */
+	RGX_HWPERF_PWR_UP           = 3, /*!< Power turned on to a HW domain */
+	RGX_HWPERF_PWR_DOWN         = 4, /*!< Power turned off to a HW domain */
+	RGX_HWPERF_PWR_SAFETY_RESET = 5, /*!< Resetting the GPU HW units for safety reasons */
+	RGX_HWPERF_PWR_PHR_FULL     = 6, /*!< Periodic HW full GPU Reset */
+	RGX_HWPERF_PWR_RD_UP        = 7, /*!< Power turned on to a Rascal+Dust */
+	RGX_HWPERF_PWR_RD_DOWN      = 8, /*!< Power turned off to a Rascal+Dust */
+	RGX_HWPERF_PWR_SPU_UP       = 9, /*!< Power turned on to a SPU */
+	RGX_HWPERF_PWR_SPU_DOWN     = 10, /*!< Power turned off to a SPU */
+	RGX_HWPERF_PWR_CLUSTER_UP   = 11, /*!< Power turned on to a cluster */
+	RGX_HWPERF_PWR_CLUSTER_DOWN = 12, /*!< Power turned off to a cluster */
+	RGX_HWPERF_PWR_RAC_UP       = 13, /*!< Power turned on to a RAC */
+	RGX_HWPERF_PWR_RAC_DOWN     = 14, /*!< Power turned off to a RAC */
+
+	RGX_HWPERF_PWR_LAST,
+} RGX_HWPERF_PWR;
+
+/*! This structure holds the data of a power packet. */
+typedef struct
+{
+	RGX_HWPERF_PWR eChange;                  /*!< Defines the type of power change */
+	IMG_UINT32     ui32Domains;              /*!< HW Domains affected */
+	IMG_UINT64     ui64OSMonoTimeStamp;      /*!< OSTimeStamp sampled by the host (monotonic) */
+	IMG_UINT64     ui64CRTimeStamp;          /*!< CRTimeStamp sampled by the host and
+	                                              correlated to OSTimeStamp */
+	IMG_UINT32     ui32CalibratedClockSpeed; /*!< GPU clock speed (in Hz) at the time
+	                                              the two timers were correlated */
+	IMG_UINT32     ui32Unused1;              /*!< Padding */
+	IMG_UINT64     ui64OSSecondaryTimeStamp; /*!< OSTimeStamp sampled by the host */
+} RGX_HWPERF_PWR_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_PWR_CHG_DATA);
+
+/*
+ * PDVFS, GPU clock frequency changes and workload estimation profiling
+ * data.
+ */
+/*! DVFS and work estimation events. */
+typedef enum
+{
+	RGX_HWPERF_DVFS_EV_INVALID,                 /*!< Invalid value. */
+	RGX_HWPERF_DVFS_EV_PROACTIVE_EST_START,     /*!< Proactive DVFS estimate start */
+	RGX_HWPERF_DVFS_EV_PROACTIVE_EST_FINISHED,  /*!< Proactive DVFS estimate finished */
+	RGX_HWPERF_DVFS_EV_REACTIVE_EST_START,      /*!< Reactive DVFS estimate start */
+	RGX_HWPERF_DVFS_EV_REACTIVE_EST_FINISHED,   /*!< Reactive DVFS estimate finished */
+	/* workload estimation */
+	RGX_HWPERF_DVFS_EV_WORK_EST_START,          /*!< Workload estimation start */
+	RGX_HWPERF_DVFS_EV_WORK_EST_FINISHED,       /*!< Workload estimation finished */
+	RGX_HWPERF_DVFS_EV_FREQ_CHG,                /*!< DVFS OPP/clock frequency change */
+
+	RGX_HWPERF_DVFS_EV_LAST               /*!< Number of element. */
+} RGX_HWPERF_DVFS_EV;
+
+/*! Enumeration of DVFS transitions that can occur */
+typedef enum
+{
+	RGX_HWPERF_DVFS_OPP_NONE        = 0x0,  /*!< No OPP change, already operating at required freq */
+	RGX_HWPERF_DVFS_OPP_IDLE        = 0x1,  /*!< GPU is idle, defer the OPP change */
+	/* 0x2 to 0xF reserved */
+	RGX_HWPERF_DVFS_OPP_UPDATE      = 0x10, /*!< OPP change, new point is encoded in bits [3:0] */
+	RGX_HWPERF_DVFS_OPP_LAST        = 0x20,
+} RGX_HWPERF_DVFS_OPP;
+
+typedef union
+{
+	/*! This structure holds the data of a proactive DVFS calculation packet. */
+	struct
+	{
+		IMG_UINT64     ui64DeadlineInus;         /*!< Next deadline in microseconds */
+		IMG_UINT32     ui32Frequency;            /*!< Required freq to meet deadline at 90% utilisation */
+		IMG_UINT32     ui32WorkloadCycles;       /*!< Current workload estimate in cycles */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sProDVFSCalc;
+
+	/*! This structure holds the data of a reactive DVFS calculation packet. */
+	struct
+	{
+		IMG_UINT32     ui32Frequency;            /*!< Required freq to achieve average 90% utilisation */
+		IMG_UINT32     ui32Utilisation;          /*!< GPU utilisation since last update in percentage points (0..100%) */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sDVFSCalc;
+
+	/*! This structure holds the data of a work estimation packet. */
+	struct
+	{
+		IMG_UINT32     ui32CyclesPrediction;     /*!< Predicted cycle count for this workload */
+		IMG_UINT32     ui32CyclesTaken;          /*!< Actual cycle count for this workload */
+		RGX_HWPERF_DM  eDM;                      /*!< Target DM */
+		IMG_UINT32     ui32ReturnDataIndex;      /*!< Index into workload estimation table */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sWorkEst;
+
+	/*! This structure holds the data of an Operating Performance Points (OPP) clock frequency transition packet. */
+	struct
+	{
+		IMG_UINT32     ui32OPPData;              /*!< New Operating Performance Point (OPP) */
+	} sOPP;
+
+} RGX_HWPERF_DVFS_DETAIL;
+
+/*! DVFS sub-event data structure */
+typedef struct {
+	RGX_HWPERF_DVFS_EV      eEventType;          /*!< DVFS sub-event type */
+	RGX_HWPERF_DVFS_DETAIL  uData;               /*!< DVFS sub-event data */
+} RGX_HWPERF_DVFS_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_DVFS_DATA);
+
+typedef enum {
+	RGX_HWPERF_UFO_EV_UPDATE,          /*!< Update on the UFO objects. */
+	RGX_HWPERF_UFO_EV_CHECK_SUCCESS,   /*!< Successful check on UFO objects. */
+	RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS, /*!< Successful partial render check on UFO objects. */
+	RGX_HWPERF_UFO_EV_CHECK_FAIL,      /*!< Unsuccessful check on UFO objects. */
+	RGX_HWPERF_UFO_EV_PRCHECK_FAIL,    /*!< Unsuccessful partial render check on UFO objects. */
+	RGX_HWPERF_UFO_EV_FORCE_UPDATE,    /*!< Forced erroring of the UFO objects. */
+
+	RGX_HWPERF_UFO_EV_LAST             /*!< Reserved. Do not use. */
+} RGX_HWPERF_UFO_EV;
+
+/*! Data stream tuple. */
+typedef union
+{
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32Value;         /*!< Value of the UFO object */
+	} sCheckSuccess;
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32Value;         /*!< Value of the UFO object */
+		IMG_UINT32 ui32Required;      /*!< Value of the UFO object required by the fence */
+	} sCheckFail;
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32OldValue;      /*!< Value of UFO object before update */
+		IMG_UINT32 ui32NewValue;      /*!< Value of UFO object after update */
+	} sUpdate;
+} RGX_HWPERF_UFO_DATA_ELEMENT;
+
+/*! This structure holds the packet payload data for UFO event. */
+typedef struct
+{
+	RGX_HWPERF_UFO_EV eEvType;     /*!< Subtype of the event. See RGX_HWPERF_UFO_EV */
+	IMG_UINT32 ui32TimeCorrIndex;  /*!< Index to the timer correlation data
+	                                 at the time the packet was generated.
+	                                 Used to approximate Host timestamps for
+	                                 these events. */
+	IMG_UINT32 ui32PID;            /*!< Client process identifier */
+	IMG_UINT32 ui32ExtJobRef;      /*!< Reference used by callers of the RGX
+	                                 API to track submitted work (for
+	                                 debugging/trace purposes) */
+	IMG_UINT32 ui32IntJobRef;      /*!< Internal reference used to track
+	                                 submitted work (for debugging / trace
+	                                 purposes) */
+	IMG_UINT32 ui32DMContext;      /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32StreamInfo;     /*!< Encoded number of elements in the
+	                                 stream and stream data offset in the
+	                                 payload */
+	RGX_HWPERF_DM eDM;             /*!< Data Master number, see RGX_HWPERF_DM */
+	IMG_UINT32 aui32StreamData[IMG_FLEX_ARRAY_MEMBER];  /*!< Series of tuples holding UFO objects data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);        /*!< Unused, reserved */
+} RGX_HWPERF_UFO_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_UFO_DATA);
+
+
+/*!
+ * RGX_HWPERF_KICK_TYPE describes the type of kick for events received / sent
+ * between KICK_START / KICK_END inclusively for all event types.
+ */
+typedef enum
+{
+	RGX_HWPERF_KICK_TYPE_RESERVED_0,  /*!< Replaced by separate TA and 3D types (Deprecated) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_1,  /*!< Compute Data Master Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_CDM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_2,  /*!< Ray Store Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_RS) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_3,  /*!< Scene Hierarchy Generator Kick (Deprecated) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_4,  /*!< TQ 2D Data Master Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_TQTDM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_5,  /*!< Sync Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_SYNC) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_6,  /*!< TA Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_GEOM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_7,  /*!< 3D Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_3D) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_8,
+	RGX_HWPERF_KICK_TYPE_RESERVED_9,
+
+	RGX_HWPERF_KICK_TYPE_UNIFIED = 0x10,
+
+	RGX_HWPERF_KICK_TYPE2_TQ2D,  /*!< 2D TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_TQ3D,  /*!< 3D TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_TQTDM, /*!< 2D Data Master TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_CDM,   /*!< Compute Kick */
+	RGX_HWPERF_KICK_TYPE2_GEOM,  /*!< GEOM Kick */
+	RGX_HWPERF_KICK_TYPE2_3D,    /*!< 3D Kick */
+	RGX_HWPERF_KICK_TYPE2_SYNC,  /*!< Sync Kick */
+	RGX_HWPERF_KICK_TYPE2_RS,    /*!< Ray Store Kick */
+	RGX_HWPERF_KICK_TYPE2_LAST,
+
+	RGX_HWPERF_KICK_TYPE_FORCE_32BIT = 0x7fffffff
+} RGX_HWPERF_KICK_TYPE;
+
+typedef enum
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_NONE,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_DEFAULT_FRAMEBUFFER,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_OFFSCREEN_FB_ATTACHMENTS,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_TILE_LIFETIME_DATA,
+
+	RGX_HWPERF_RESOURCE_TYPE_COUNT
+} RGX_HWPERF_RESOURCE_CAPTURE_TYPE;
+
+typedef struct
+{
+	IMG_UINT32 ui32Height;
+	IMG_UINT32 ui32Width;
+	IMG_UINT32 ui32BPP;
+	IMG_UINT32 ui32PixFormat;
+} RGX_RESOURCE_PER_SURFACE_INFO, *PRGX_RESOURCE_PER_SURFACE_INFO;
+
+typedef struct
+{
+	IMG_INT32  i32XOffset;        /*!< render surface X shift */
+	IMG_INT32  i32YOffset;        /*!< render surface Y shift */
+	IMG_UINT32 ui32WidthInTiles;  /*!< number of TLT data points in X */
+	IMG_UINT32 ui32HeightInTiles; /*!< number of TLT data points in Y */
+} RGX_RESOURCE_PER_TLT_BUFFER_INFO, *PRGX_RESOURCE_PER_TLT_BUFFER_INFO;
+
+typedef union
+{
+	struct RGX_RESOURCE_CAPTURE_RENDER_SURFACES
+	{
+		IMG_UINT32 ui32RenderSurfaceCount;
+		RGX_RESOURCE_PER_SURFACE_INFO sSurface[IMG_FLEX_ARRAY_MEMBER];
+	} sRenderSurfaces;
+
+	struct RGX_RESOURCE_CAPTURE_TILE_LIFETIME_BUFFERS
+	{
+		IMG_UINT32 uiReserved;
+		RGX_RESOURCE_PER_TLT_BUFFER_INFO sTLTBufInfo[IMG_FLEX_ARRAY_MEMBER];
+	} sTLTBuffers;
+} RGX_RESOURCE_CAPTURE_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE eType;
+	IMG_PID uPID;
+	IMG_UINT32 ui32ContextID;
+	IMG_UINT32 ui32FrameNum;
+	IMG_UINT32 ui32CapturedTaskJobRef;	/* The job ref of the HW task that emitted the data */
+	IMG_INT32 eClientModule;			/* RGX_HWPERF_CLIENT_API - ID that the capture is originating from. */
+	RGX_RESOURCE_CAPTURE_DETAIL uDetail; /* eType determines the value of the union */
+} RGX_RESOURCE_CAPTURE_INFO, *PRGX_RESOURCE_CAPTURE_INFO;
+
+#define RGX_RESOURCE_CAPTURE_INFO_BASE_SIZE() offsetof(RGX_RESOURCE_CAPTURE_INFO, uDetail)
+
+/*! Tile Lifetime Tracking header size. Only available if
+ * RGX_FEATURE_ISP_TILE_LIFETIME_TRACKING is present and enabled via
+ * SUPPORT_TLT_PERF
+ */
+#define RGX_TLT_HARDWARE_HDR_SIZE   (16U)
+
+/* PVRSRVGetHWPerfResourceCaptureResult */
+typedef enum
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_NONE = 0,
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK,					/* We got data ok, expect more packets for this request. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_NOT_READY,			/* Signals a timeout on the connection - no data available yet. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_COMPLETE_SUCCESS,	/* The request completed successfully, signals the end of packets for the request. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_COMPLETE_FAILURE		/* The request failed, signals the end of packets for the request. */
+} RGX_HWPERF_RESOURCE_CAPTURE_RESULT_STATUS;
+
+typedef struct
+{
+	IMG_PID uPID;						/* In case of a failed request pass the caller the PID and context ID. */
+	IMG_UINT32 ui32CtxID;
+	RGX_RESOURCE_CAPTURE_INFO *psInfo;	/* Various meta-data regarding the captured resource which aid the requester when,
+											unpacking the resource data, valid if RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK is returned. */
+	IMG_BYTE *pbData;					/* Buffer containing the captured resource data, valid if RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK is returned. */
+} RGX_RESOURCE_CAPTURE_RESULT;
+
+/*! Firmware Activity event. */
+typedef enum
+{
+	RGX_HWPERF_FWACT_EV_INVALID,            /*!< Invalid value. */
+	RGX_HWPERF_FWACT_EV_REGS_SET,           /*!< Registers set. */
+	RGX_HWPERF_FWACT_EV_HWR_DETECTED,       /*!< HWR detected. */
+	RGX_HWPERF_FWACT_EV_HWR_RESET_REQUIRED, /*!< Reset required. */
+	RGX_HWPERF_FWACT_EV_HWR_RECOVERED,      /*!< HWR recovered. */
+	RGX_HWPERF_FWACT_EV_HWR_FREELIST_READY, /*!< Freelist ready. */
+	RGX_HWPERF_FWACT_EV_FEATURES,           /*!< Features present */
+	RGX_HWPERF_FWACT_EV_FILTER_SET,         /*!< Event filter set. */
+	RGX_HWPERF_FWACT_EV_KICK_CANCEL,        /*!< A previous pipelined kick cancel. */
+
+	RGX_HWPERF_FWACT_EV_LAST                /*!< Number of element. */
+} RGX_HWPERF_FWACT_EV;
+
+/*! Cause of the HWR event. */
+typedef enum
+{
+	RGX_HWPERF_HWR_REASON_INVALID,              /*!< Invalid value. */
+	RGX_HWPERF_HWR_REASON_LOCKUP,               /*!< Lockup. */
+	RGX_HWPERF_HWR_REASON_PAGEFAULT,            /*!< Page fault. */
+	RGX_HWPERF_HWR_REASON_POLLFAIL,             /*!< Poll fail. */
+	RGX_HWPERF_HWR_REASON_DEADLINE_OVERRUN,     /*!< Deadline overrun. */
+	RGX_HWPERF_HWR_REASON_CSW_DEADLINE_OVERRUN, /*!< Hard Context Switch deadline overrun. */
+
+	RGX_HWPERF_HWR_REASON_LAST                  /*!< Number of elements. */
+} RGX_HWPERF_HWR_REASON;
+
+
+/* Fixed size for BVNC string so it does not alter packet data format
+ * Check it is large enough against official BVNC string length maximum
+ */
+#define RGX_HWPERF_MAX_BVNC_LEN (24)
+static_assert((RGX_HWPERF_MAX_BVNC_LEN >= RGX_BVNC_STR_SIZE_MAX),
+			  "Space inside HWPerf packet data for BVNC string insufficient");
+
+#define RGX_HWPERF_MAX_BVNC_BLOCK_LEN (20U)
+
+/*! BVNC Features */
+typedef struct
+{
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of counters in this block type */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Number of blocks of this type */
+	IMG_UINT16 ui16NumBlocks;
+
+	/*! Reserved for future use */
+	IMG_UINT16 ui16Reserved;
+} RGX_HWPERF_BVNC_BLOCK;
+
+/*! BVNC Features */
+typedef struct
+{
+	IMG_CHAR aszBvncString[RGX_HWPERF_MAX_BVNC_LEN]; /*!< BVNC string */
+	IMG_UINT32 ui32BvncKmFeatureFlags;               /*!< See RGX_HWPERF_FEATURE_FLAGS */
+	IMG_UINT16 ui16BvncBlocks;                       /*!< Number of blocks described in aBvncBlocks */
+	IMG_UINT16 ui16BvncGPUCores;                     /*!< Number of GPU cores present */
+	RGX_HWPERF_BVNC_BLOCK aBvncBlocks[RGX_HWPERF_MAX_BVNC_BLOCK_LEN];  /*!< Supported Performance Blocks for BVNC. See RGX_HWPERF_BVNC_BLOCK */
+} RGX_HWPERF_BVNC;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_BVNC);
+
+/*! Performance Counter Configuration data element. */
+typedef struct
+{
+	IMG_UINT32 ui32BlockID;               /*!< Counter Block ID. See RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT32 ui32NumCounters;           /*!< Number of counters configured */
+	IMG_UINT32 ui32CounterVals[RGX_CNTBLK_COUNTERS_MAX];  /*!< Counters configured (ui32NumCounters worth of entries) */
+} RGX_HWPERF_COUNTER_CFG_DATA_EL;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_COUNTER_CFG_DATA_EL);
+
+/*! Performance Counter Configuration data. */
+typedef struct
+{
+	IMG_UINT32 ui32EnabledBlocks;          /*!< Number of Enabled Blocks. */
+	RGX_HWPERF_COUNTER_CFG_DATA_EL uData;  /*!< Start of variable length data. See RGX_HWPERF_COUNTER_CFG_DATA_EL */
+	IMG_UINT32 ui32Padding;                /*!< reserved */
+} RGX_HWPERF_COUNTER_CFG;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_COUNTER_CFG);
+
+/*! Sub-event's data. */
+typedef union
+{
+	struct
+	{
+		RGX_HWPERF_DM eDM;				/*!< Data Master ID. */
+		RGX_HWPERF_HWR_REASON eReason;	/*!< Reason of the HWR. */
+		IMG_UINT32 ui32DMContext;		/*!< FW render context */
+	} sHWR;								/*!< HWR sub-event data. */
+
+	RGX_HWPERF_BVNC sBVNC;              /*!< BVNC Features. See RGX_HWPERF_BVNC */
+	struct
+	{
+		IMG_UINT32 ui32EvMaskLo;		/*!< Low order 32 bits of Filter Mask */
+		IMG_UINT32 ui32EvMaskHi;		/*!< High order 32 bits of Filter Mask */
+	} sEvMsk;							/*!< HW Filter Mask */
+	RGX_HWPERF_COUNTER_CFG sPCC;        /*!< Performance Counter Config. See RGX_HWPERF_COUNTER_CFG */
+
+	struct
+	{
+		RGX_HWPERF_DM eDM;				/*!< Data Master ID. */
+		IMG_UINT32 ui32DMContext;		/*!< FW context */
+		IMG_UINT32 ui32GPUIdMask;		/*!< Multicore mask. */
+		IMG_UINT32 ui32KickID;			/*!< Kick Id cancelled. */
+	} sKickCancel;						/*!< Kick cancel sub-event data. */
+} RGX_HWPERF_FWACT_DETAIL;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FWACT_DETAIL);
+
+/*! This structure holds the data of a FW activity event packet */
+typedef struct
+{
+	RGX_HWPERF_FWACT_EV eEvType;           /*!< Event type. */
+	RGX_HWPERF_FWACT_DETAIL uFwActDetail;  /*!< Data of the sub-event. */
+	IMG_UINT32 ui32Padding;                /*!< Reserved. */
+} RGX_HWPERF_FWACT_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FWACT_DATA);
+
+typedef struct
+{
+	RGX_HWPERF_KICK_TYPE ui32EnqType; /*!< Workload type sent to FW for
+	                                   scheduling on GPU hardware.
+	                                   See RGX_HWPERF_KICK_TYPE */
+	IMG_UINT32 ui32PID;               /*!< Client process identifier */
+	IMG_UINT32 ui32ExtJobRef;         /*!< Reference used by callers of the RGX API
+	                                   to track submitted work (for debugging /
+	                                   trace purposes) */
+	IMG_UINT32 ui32IntJobRef;         /*!< internal reference used to track submitted
+	                                   work (for debugging / trace purposes) */
+	IMG_UINT32 ui32DMContext;         /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32Padding;           /*!< Unused, reserved */
+	IMG_UINT64 ui64CheckFence_UID;    /*!< ID of fence gating work execution on GPU */
+	IMG_UINT64 ui64UpdateFence_UID;   /*!< ID of fence triggered after work completes on GPU */
+	IMG_UINT64 ui64DeadlineInus;      /*!< Workload deadline in system monotonic time */
+	IMG_UINT32 ui32CycleEstimate;     /*!< Estimated cycle time for the workload */
+	PVRSRV_FENCE hCheckFence;         /*!< Fence this enqueue task waits for, before starting */
+	PVRSRV_FENCE hUpdateFence;        /*!< Fence this enqueue task signals, on completion */
+	PVRSRV_TIMELINE hUpdateTimeline;  /*!< Timeline on which the above hUpdateFence is created */
+
+	/* Align structure size to 8 bytes */
+} RGX_HWPERF_HOST_ENQ_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_ENQ_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_ENQ_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef struct
+{
+	RGX_HWPERF_UFO_EV eEvType;   /*!< Subtype of the event */
+	IMG_UINT32 ui32StreamInfo;   /*!< Encoded number of elements in the stream and
+	                              stream data offset in the payload */
+#ifdef __CHECKER__
+	/* Since we're not conforming to the C99 standard by not using a flexible
+	 * array member need to add a special case for Smatch static code analyser. */
+	IMG_UINT32 aui32StreamData[];
+#else
+	IMG_UINT32 aui32StreamData[IMG_FLEX_ARRAY_MEMBER];       /*!< Series of tuples holding UFO objects data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);      /*!< Reserved, align structure size to 8 bytes */
+#endif
+} RGX_HWPERF_HOST_UFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_UFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_UFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*!
+ * RGX_HWPERF_HOST_RESOURCE_TYPE describes the type of resource which has been
+ * Allocated, Freed or Modified. The values are used to determine which event
+ * data structure to use to decode the data from the event stream
+ */
+typedef enum
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE_INVALID,   /*!< Invalid */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC,      /*!< SyncPrim */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_TIMELINE_DEPRECATED,
+	                                         /*!< Timeline resource packets are
+	                                           now emitted in client hwperf buffer */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR, /*!< Fence for use on GPU (SYNC_CP backed) */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP,   /*!< Sync Checkpoint */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW,  /*!< Fence created on SW timeline */
+
+	RGX_HWPERF_HOST_RESOURCE_TYPE_LAST       /*!< End of enumeration */
+} RGX_HWPERF_HOST_RESOURCE_TYPE;
+
+typedef union
+{
+	/*! Data for TYPE_TIMELINE (*Deprecated*). This sub-event is no longer
+     *  generated in the HOST stream. Timeline data is now provided in the
+     *  CLIENT stream instead.
+     */
+	struct
+	{
+		IMG_UINT32 uiPid;             /*!< Identifier of owning process */
+		IMG_UINT64 ui64Timeline_UID1; /*!< Unique identifier for timeline resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sTimelineAlloc;
+
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_PID uiPID;                /*!< Identifier of owning process */
+		PVRSRV_FENCE hFence;          /*!< Unique identifier for the fence resource */
+		IMG_UINT32 ui32CheckPt_FWAddr; /*!< Unique identifier of the check point
+		                                backing this fence on the GPU */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+	} sFenceAlloc;
+
+	/*! Data for TYPE_SYNC_CP */
+	struct
+	{
+		IMG_UINT32 ui32CheckPt_FWAddr; /*!< Unique identifier for the check point resource */
+		PVRSRV_TIMELINE hTimeline;     /*!< Unique identifier for the timeline resource */
+		IMG_PID uiPID;                 /*!< Identifier of owning process */
+		PVRSRV_FENCE hFence;           /*!< Unique identifier for the fence resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSyncCheckPointAlloc;
+
+	/*! Data for TYPE_FENCE_SW */
+	struct
+	{
+		IMG_PID uiPID;                 /*!< Identifier of owning process */
+		PVRSRV_FENCE hSWFence;         /*!< Unique identifier for the SWFence resource */
+		PVRSRV_TIMELINE hSWTimeline;   /*!< Unique identifier for the timeline resource */
+		IMG_UINT64 ui64SyncPtIndex;    /*!< Sync-pt index where this SW timeline has reached */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSWFenceAlloc;
+
+	/*! Data for TYPE_SYNC */
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;         /*!< Identifier of sync resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSyncAlloc;
+} RGX_HWPERF_HOST_ALLOC_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32AllocType;
+	                                 /*!< This describes the type of the resource
+	                                  allocated in the driver. See
+	                                  RGX_HWPERF_HOST_RESOURCE_TYPE */
+	RGX_HWPERF_HOST_ALLOC_DETAIL RGXFW_ALIGN uAllocDetail;
+	                                 /*!< Union of structures providing further
+	                                  data regarding the resource allocated.
+	                                  Size of data varies with union member that
+	                                  is present, check ``ui32AllocType`` value
+	                                  to decode */
+} RGX_HWPERF_HOST_ALLOC_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_ALLOC_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_ALLOC_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef union
+{
+	/*! Data for TYPE_TIMELINE (*Deprecated*) */
+	struct
+	{
+		IMG_UINT32 uiPid;             /*!< Identifier of owning process */
+		IMG_UINT64 ui64Timeline_UID1; /*!< Unique identifier for the timeline resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sTimelineDestroy;
+
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_UINT64 ui64Fence_UID;     /*!< Unique identifier for the fence resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. */
+	} sFenceDestroy;
+
+	/*! Data for TYPE_SYNC_CP */
+	struct
+	{
+		IMG_UINT32 ui32CheckPt_FWAddr;  /*!< Unique identifier for the check point resource */
+	} sSyncCheckPointFree;
+
+	/*! Data for TYPE_SYNC */
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< Unique identifier for the sync resource */
+	} sSyncFree;
+} RGX_HWPERF_HOST_FREE_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32FreeType;
+	                              /*!< This describes the type of the resource
+	                               freed or released by the driver. See
+	                               RGX_HWPERF_HOST_RESOURCE_TYPE */
+	RGX_HWPERF_HOST_FREE_DETAIL uFreeDetail;
+	                              /*!< Union of structures providing further data
+	                               regarding the resource freed. Size of data
+	                               varies with union member that is present,
+	                               check ``ui32FreeType`` value to decode */
+	IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+} RGX_HWPERF_HOST_FREE_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_FREE_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_FREE_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef struct
+{
+	IMG_UINT64 ui64CRTimestamp;          /*!< CR timer value from the latest entry of
+	                                      the time domains correlation table */
+	IMG_UINT64 ui64OSMonoTimestamp;      /*!< OS timestamp (monotonic) in nanoseconds from the
+	                                      latest entry of the time domains correlation table */
+	IMG_UINT32 ui32ClockSpeed;           /*!< GPU clock speed from the latest entry of
+	                                      the time domains correlation table */
+	IMG_UINT32 ui32Padding;              /*!< Reserved, align structure size to 8 bytes */
+	IMG_UINT64 ui64OSSecondaryTimeStamp; /*!< OS timestamp in nanoseconds from the latest entry
+	                                      of the time domains correlation table */
+} RGX_HWPERF_HOST_CLK_SYNC_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_CLK_SYNC_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_CLK_SYNC_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef union
+{
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_UINT64 ui64NewFence_UID;  /*!< Unique identifier for the new merged fence
+		                               resource that has been created */
+		IMG_UINT64 ui64InFence1_UID;  /*!< Unique identifier for the fence resource */
+		IMG_UINT64 ui64InFence2_UID;  /*!< Unique identifier of the check point backing
+		                               the fence on the GPU */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sFenceMerge;
+} RGX_HWPERF_HOST_MODIFY_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32ModifyType;
+	                                /*!< Describes the type of the resource
+	                                 modified by the driver. See
+	                                 RGX_HWPERF_HOST_RESOURCE_TYPE */
+
+	RGX_HWPERF_HOST_MODIFY_DETAIL uModifyDetail;
+	                                /*!< Union of structures providing further
+	                                 data regarding the resource modified.
+	                                 Size of data varies with union member that
+	                                 is present.
+	                                 Check ``uiModifyType`` value to decode */
+} RGX_HWPERF_HOST_MODIFY_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_MODIFY_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_MODIFY_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef enum
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED = 0, /*!< Invalid */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_OK,            /*!< Device OK */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_NOT_RESPONDING,/*!< Device not responding to requests */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_DEAD,          /*!< Device not responding */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_FAULT,         /*!< Device has faulted */
+
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_LAST
+} RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS;
+
+typedef enum
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_UNDEFINED = 0,     /*!< Invalid */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_NONE,              /*!< No underlying health reason. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_ASSERTED,          /*!< Device has asserted. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_POLL_FAILING,      /*!< Device poll has failed. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_TIMEOUTS,          /*!< Device timeout has fired. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_CORRUPT,     /*!< Queue has become corrupt. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_STALLED,     /*!< Queue has stalled. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_IDLING,            /*!< Device is idling. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_RESTARTING,        /*!< Device restarting. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS,/*!< Interrupts have been discarded. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_PCI_ERROR,         /*!< PCI error detected. */
+
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_LAST
+} RGX_HWPERF_HOST_DEVICE_HEALTH_REASON;
+
+/*! Data for device status event */
+typedef struct
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS eDeviceHealthStatus;
+	                                 /*!< Device's health status */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON eDeviceHealthReason;
+	                                 /*!< Reason for device's health status */
+} RGX_HWPERF_HOST_DEVICE_HEALTH;
+
+/*! RGX_HWPERF_DEV_INFO_EV values */
+typedef enum
+{
+	RGX_HWPERF_DEV_INFO_EV_HEALTH,      /*!< Health sub-event */
+	RGX_HWPERF_DEV_INFO_EV_FEATURES,    /*!< Features sub-event */
+
+	RGX_HWPERF_DEV_INFO_EV_LAST         /*!< Last enumeration value */
+} RGX_HWPERF_DEV_INFO_EV;
+
+/*! RGX_HWPERF_HOST_DEV_INFO_DETAIL is a union of structures providing
+ *  further data regarding the device's status
+ */
+typedef union
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH sDeviceStatus;    /*!< Device health status */
+	RGX_HWPERF_BVNC               sBVNC;            /*!< Device features */
+} RGX_HWPERF_HOST_DEV_INFO_DETAIL;
+
+/*! RGX_HWPERF_HOST_DEV_INFO_DATA contains device health status information */
+typedef struct
+{
+	IMG_UINT32                      ui32Padding;
+	                            /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_DEV_INFO_EV          eEvType;
+	                            /*!< Type of the sub-event. See
+	                              RGX_HWPERF_DEV_INFO_EV */
+	RGX_HWPERF_HOST_DEV_INFO_DETAIL uDevInfoDetail;
+	                            /*!< Union of structures providing further data
+	                              regarding the device's status. Size of data
+	                              varies with union member that is present,
+	                              check ``eEvType`` value to decode */
+} RGX_HWPERF_HOST_DEV_INFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_DEV_INFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			 "sizeof(RGX_HWPERF_HOST_DEV_INFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! RGX_HWPERF_INFO_EV event subtype for RGX_HWPERF_HOST_INFO_DATA events */
+typedef enum
+{
+	RGX_HWPERF_INFO_EV_RESERVED_0,
+	RGX_HWPERF_INFO_EV_MEM64_USAGE,     /*!< 64-bit Memory usage event */
+	RGX_HWPERF_INFO_EV_LAST             /*!< End of enumeration */
+} RGX_HWPERF_INFO_EV;
+
+/*! RGX_HWPERF_HOST_INFO_DETAIL contains the data payload for the
+ * RGX_HWPERF_HOST_INFO_DATA event.
+ */
+typedef union
+{
+	/*! Host Memory usage statistics */
+	struct
+	{
+		IMG_UINT64 ui64TotalMemoryUsage;   /*!< Total memory usage (bytes) */
+		/*! Detailed memory usage */
+		struct _RGX_HWPERF_HOST_INFO_PER_PROC_USAGE_
+		{
+			IMG_UINT32 ui32Pid;              /*!< Process ID */
+			IMG_UINT32 ui32Padding;          /*!< Padding */
+			IMG_UINT64 ui64KernelMemUsage;   /*!< Kernel memory usage (bytes) */
+			IMG_UINT64 ui64GraphicsMemUsage; /*!< GPU memory usage (bytes) */
+		} sPerProcessUsage[IMG_FLEX_ARRAY_MEMBER];
+	} sMemUsageStats;
+} RGX_HWPERF_HOST_INFO_DETAIL;
+
+/*! RGX_HWPERF_HOST_INFO_DATA. Host Info data event payload contains device
+ * memory usage information.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_INFO_EV eEvType;   /*!< Type of subevent. See RGX_HWPERF_INFO_EV */
+	RGX_HWPERF_HOST_INFO_DETAIL uInfoDetail;
+	                              /*!< Union of structures providing further data
+	                               regarding memory usage. Size varies with union
+	                               member that is present, check ``eEvType``
+	                               value to decode */
+} RGX_HWPERF_HOST_INFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_INFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_INFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! FENCE_WAIT_TYPE definitions */
+typedef enum
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN = 0,    /*!< Begin */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END,          /*!< End */
+
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_LAST,         /*!< Do not use */
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE;
+
+/*! FENCE_WAIT_RESULT definitions */
+typedef enum
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_INVALID = 0, /*!< Invalid */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_TIMEOUT,     /*!< Timed Out */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_PASSED,      /*!< Passed */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_ERROR,       /*!< Errored */
+
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_LAST,        /*!< Do not use */
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT;
+
+/*! FENCE_WAIT_DETAIL Event Payload */
+typedef union
+{
+/*! Data for SYNC_FENCE_WAIT_TYPE_BEGIN */
+	struct
+	{
+		IMG_UINT32 ui32TimeoutInMs;                     /*!< Wait timeout (ms) */
+	} sBegin;
+
+	/*! Data for SYNC_FENCE_WAIT_TYPE_END */
+	struct
+	{
+		RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT eResult; /*!< Wait result */
+	} sEnd;
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DETAIL;
+
+/*! RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA Event Payload. This data structure
+ * is received whenever the host driver handles a wait for sync event request.
+ */
+typedef struct
+{
+	IMG_PID uiPID;          /*!< Identifier of the owning process */
+	PVRSRV_FENCE hFence;    /*!< Unique identifier for the fence resource */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType;
+	                        /*!< Type of the subevent, see
+	                         RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DETAIL uDetail;
+	                        /*!< Union of structures providing further data
+	                         regarding device's status. Size of data varies with
+	                         union member that is present, check ``eType`` value
+	                         to decode */
+
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA;
+
+static_assert((sizeof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA.
+ * Software Timeline Advanced Event Payload. This data structure is received
+ * whenever the host driver processes a Software Timeline Advanced event.
+ */
+typedef struct
+{
+	IMG_PID uiPID;                /*!< Identifier of the owning process */
+	PVRSRV_TIMELINE hTimeline;    /*!< Unique identifier for the timeline resource */
+	IMG_UINT64 ui64SyncPtIndex;   /*!< Index of the sync point to which the
+	                               timeline has advanced */
+
+} RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA;
+
+static_assert((sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef enum
+{
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_INVALID = 0,  /*!< Invalid */
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME, /*!< Process Name */
+
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_LAST,         /*!< Do not use */
+} RGX_HWPERF_HOST_CLIENT_INFO_TYPE;
+
+typedef struct
+{
+	IMG_PID uiClientPID; /*!< Client process identifier */
+	IMG_UINT32 ui32Length;  /*!< Number of bytes present in ``acName`` */
+	IMG_CHAR acName[IMG_FLEX_ARRAY_MEMBER]; /*!< Process name string, null terminated */
+} RGX_HWPERF_HOST_CLIENT_PROC_NAME;
+
+#define RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen) \
+	((IMG_UINT32)(offsetof(RGX_HWPERF_HOST_CLIENT_PROC_NAME, acName) + (ui32NameLen)))
+
+typedef union
+{
+	struct
+	{
+		IMG_UINT32 ui32Count; /*!< Number of elements in ``asProcNames`` */
+		RGX_HWPERF_HOST_CLIENT_PROC_NAME asProcNames[IMG_FLEX_ARRAY_MEMBER];
+	} sProcName;
+} RGX_HWPERF_HOST_CLIENT_INFO_DETAIL;
+
+typedef struct
+{
+	IMG_UINT32 uiReserved; /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE eType;
+	                        /*!< Type of the subevent, see
+	                         RGX_HWPERF_HOST_CLIENT_INFO_TYPE */
+	RGX_HWPERF_HOST_CLIENT_INFO_DETAIL uDetail;
+	                        /*!< Union of structures. Size of data
+	                         varies with union member that is present,
+	                         check ``eType`` value to decode */
+} RGX_HWPERF_HOST_CLIENT_INFO_DATA;
+
+/*! This type is a union of packet payload data structures associated with
+ * various FW and Host events */
+typedef union
+{
+	RGX_HWPERF_FW_DATA             sFW;           /*!< Firmware event packet data,
+	                                               events ``0x01-0x06`` */
+	RGX_HWPERF_HW_DATA             sHW;           /*!< Hardware event packet data,
+	                                               events ``0x07-0x19``, ``0x28-0x29``
+	                                               See RGX_HWPERF_HW_DATA */
+	RGX_HWPERF_GPU_WORK_PERIOD_DATA sGPUWorkPeriod;
+
+	RGX_HWPERF_CLKS_CHG_DATA       sCLKSCHG;      /*!< Clock change event packet
+	                                               data, events ``0x1A`` */
+	RGX_HWPERF_GPU_STATE_CHG_DATA  sGPUSTATECHG;  /*!< GPU utilisation state
+	                                               change event packet data,
+	                                               events ``0x1B`` */
+	RGX_HWPERF_PWR_EST_DATA        sPWREST;       /*!< Power estimate event
+	                                               packet data,
+	                                               events ``0x20-0x22`` */
+	RGX_HWPERF_PWR_CHG_DATA        sPWR;          /*!< Power event packet data,
+	                                               events ``0x23`` */
+	RGX_HWPERF_CSW_DATA            sCSW;          /*!< Context switch packet data,
+	                                               events ``0x30-0x31`` */
+	RGX_HWPERF_DVFS_DATA           sDVFS;         /*!< DVFS activity data,
+	                                               events ``0x32`` */
+	RGX_HWPERF_UFO_DATA            sUFO;          /*!< UFO data, events ``0x38`` */
+	RGX_HWPERF_FWACT_DATA          sFWACT;        /*!< Firmware activity event
+	                                               packet data,
+	                                               events ``0x39`` */
+	/* */
+	RGX_HWPERF_HOST_ENQ_DATA       sENQ;          /*!< Host ENQ data,
+	                                               events ``0x01`` (Host) */
+	RGX_HWPERF_HOST_UFO_DATA       sHUFO;         /*!< Host UFO data,
+	                                               events ``0x02`` (Host) */
+	RGX_HWPERF_HOST_ALLOC_DATA     sHALLOC;       /*!< Host Alloc data,
+	                                               events ``0x03`` (Host) */
+	RGX_HWPERF_HOST_CLK_SYNC_DATA  sHCLKSYNC;     /*!< Host CLK_SYNC data,
+	                                               events ``0x04`` (Host) */
+	RGX_HWPERF_HOST_FREE_DATA      sHFREE;        /*!< Host Free data,
+	                                               events ``0x05`` (Host) */
+	RGX_HWPERF_HOST_MODIFY_DATA    sHMOD;         /*!< Host Modify data,
+	                                               events ``0x06`` (Host) */
+	RGX_HWPERF_HOST_DEV_INFO_DATA  sHDEVINFO;     /*!< Host device info data,
+	                                               events ``0x07`` (Host) */
+	RGX_HWPERF_HOST_INFO_DATA      sHINFO;        /*!< Host info data,
+	                                               events ``0x08`` (Host) */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA sWAIT;   /*!< Host fence-wait data,
+	                                               events ``0x09`` (Host) */
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA sSWTLADV; /*!< Host SW-timeline advance
+	                                               data, events ``0x0A`` (Host) */
+	RGX_HWPERF_HOST_CLIENT_INFO_DATA sHClientInfo; /*!< Host client info,
+	                                               events ``0x0B`` (Host) */
+} RGX_HWPERF_V2_PACKET_DATA, *RGX_PHWPERF_V2_PACKET_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_V2_PACKET_DATA);
+
+#define RGX_HWPERF_GET_PACKET_DATA(_packet_addr) ((RGX_PHWPERF_V2_PACKET_DATA) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR))))
+
+#define RGX_HWPERF_GET_DVFS_EVENT_TYPE_PTR(_packet_addr)	\
+	((RGX_HWPERF_DVFS_EV*) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR) + offsetof(RGX_HWPERF_DVFS_DATA,eEventType))))
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_COMMON_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_meta.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_meta.h
new file mode 100644
index 000000000000..1e6214c9ba1c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_meta.h
@@ -0,0 +1,389 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX META definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX META helper definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_META_H)
+#define RGX_META_H
+
+
+/***** The META HW register definitions in the file are updated manually *****/
+
+
+#include "img_defs.h"
+#include "km/rgxdefs_km.h"
+
+
+/******************************************************************************
+* META registers and MACROS
+******************************************************************************/
+#define META_CR_CTRLREG_BASE(T)					(0x04800000U + (0x1000U*(T)))
+
+#define META_CR_TXPRIVEXT						(0x048000E8)
+#define META_CR_TXPRIVEXT_MINIM_EN				(IMG_UINT32_C(0x1) << 7)
+
+#define META_CR_SYSC_JTAG_THREAD				(0x04830030)
+#define META_CR_SYSC_JTAG_THREAD_PRIV_EN		(0x00000004)
+
+#define META_CR_PERF_COUNT0						(0x0480FFE0)
+#define META_CR_PERF_COUNT1						(0x0480FFE8)
+#define META_CR_PERF_COUNT_CTRL_SHIFT			(28)
+#define META_CR_PERF_COUNT_CTRL_MASK			(0xF0000000)
+#define META_CR_PERF_COUNT_CTRL_DCACHEHITS		(IMG_UINT32_C(0x8) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICACHEHITS		(IMG_UINT32_C(0x9) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICACHEMISS		(IMG_UINT32_C(0xA) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICORE			(IMG_UINT32_C(0xD) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_THR_SHIFT			(24)
+#define META_CR_PERF_COUNT_THR_MASK				(0x0F000000)
+#define META_CR_PERF_COUNT_THR_0				(IMG_UINT32_C(0x1) << META_CR_PERF_COUNT_THR_SHIFT)
+#define META_CR_PERF_COUNT_THR_1				(IMG_UINT32_C(0x2) << META_CR_PERF_COUNT_THR_SHIFT)
+
+#define META_CR_TxVECINT_BHALT					(0x04820500)
+#define META_CR_PERF_ICORE0						(0x0480FFD0)
+#define META_CR_PERF_ICORE1						(0x0480FFD8)
+#define META_CR_PERF_ICORE_DCACHEMISS			(0x8)
+
+#define META_CR_PERF_COUNT(CTRL, THR)			((META_CR_PERF_COUNT_CTRL_##CTRL << META_CR_PERF_COUNT_CTRL_SHIFT) | \
+												 (THR << META_CR_PERF_COUNT_THR_SHIFT))
+
+#define META_CR_TXUXXRXDT_OFFSET				(META_CR_CTRLREG_BASE(0U) + 0x0000FFF0U)
+#define META_CR_TXUXXRXRQ_OFFSET				(META_CR_CTRLREG_BASE(0U) + 0x0000FFF8U)
+
+#define META_CR_TXUXXRXRQ_DREADY_BIT			(0x80000000U)	/* Poll for done */
+#define META_CR_TXUXXRXRQ_RDnWR_BIT				(0x00010000U)	/* Set for read  */
+#define META_CR_TXUXXRXRQ_TX_S					(12)
+#define META_CR_TXUXXRXRQ_RX_S					(4)
+#define META_CR_TXUXXRXRQ_UXX_S					(0)
+
+#define META_CR_TXUIN_ID						(0x0)			/* Internal ctrl regs */
+#define META_CR_TXUD0_ID						(0x1)			/* Data unit regs */
+#define META_CR_TXUD1_ID						(0x2)			/* Data unit regs */
+#define META_CR_TXUA0_ID						(0x3)			/* Address unit regs */
+#define META_CR_TXUA1_ID						(0x4)			/* Address unit regs */
+#define META_CR_TXUPC_ID						(0x5)			/* PC registers */
+
+/* Macros to calculate register access values */
+#define META_CR_CORE_REG(Thr, RegNum, Unit)	(((IMG_UINT32)(Thr)		<< META_CR_TXUXXRXRQ_TX_S) | \
+											 ((IMG_UINT32)(RegNum)	<< META_CR_TXUXXRXRQ_RX_S) | \
+											 ((IMG_UINT32)(Unit)	<< META_CR_TXUXXRXRQ_UXX_S))
+
+#define META_CR_THR0_PC		META_CR_CORE_REG(0, 0, META_CR_TXUPC_ID)
+#define META_CR_THR0_PCX	META_CR_CORE_REG(0, 1, META_CR_TXUPC_ID)
+#define META_CR_THR0_SP		META_CR_CORE_REG(0, 0, META_CR_TXUA0_ID)
+
+#define META_CR_THR1_PC		META_CR_CORE_REG(1, 0, META_CR_TXUPC_ID)
+#define META_CR_THR1_PCX	META_CR_CORE_REG(1, 1, META_CR_TXUPC_ID)
+#define META_CR_THR1_SP		META_CR_CORE_REG(1, 0, META_CR_TXUA0_ID)
+
+#define SP_ACCESS(Thread)	META_CR_CORE_REG(Thread, 0, META_CR_TXUA0_ID)
+#define PC_ACCESS(Thread)	META_CR_CORE_REG(Thread, 0, META_CR_TXUPC_ID)
+
+#define META_CR_COREREG_ENABLE			(0x0000000U)
+#define META_CR_COREREG_STATUS			(0x0000010U)
+#define META_CR_COREREG_DEFR			(0x00000A0U)
+#define META_CR_COREREG_PRIVEXT			(0x00000E8U)
+
+#define META_CR_T0ENABLE_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_ENABLE)
+#define META_CR_T0STATUS_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_STATUS)
+#define META_CR_T0DEFR_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_DEFR)
+#define META_CR_T0PRIVEXT_OFFSET		(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_PRIVEXT)
+
+#define META_CR_T1ENABLE_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_ENABLE)
+#define META_CR_T1STATUS_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_STATUS)
+#define META_CR_T1DEFR_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_DEFR)
+#define META_CR_T1PRIVEXT_OFFSET		(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_PRIVEXT)
+
+#define META_CR_TXENABLE_ENABLE_BIT		(0x00000001U)   /* Set if running */
+#define META_CR_TXSTATUS_PRIV			(0x00020000U)
+#define META_CR_TXPRIVEXT_MINIM			(0x00000080U)
+
+#define META_MEM_GLOBAL_RANGE_BIT		(0x80000000U)
+
+#define META_CR_TXCLKCTRL          (0x048000B0)
+#define META_CR_TXCLKCTRL_ALL_ON   (0x55111111)
+#define META_CR_TXCLKCTRL_ALL_AUTO (0xAA222222)
+
+
+/******************************************************************************
+* META LDR Format
+******************************************************************************/
+/* Block header structure */
+typedef struct
+{
+	IMG_UINT32	ui32DevID;
+	IMG_UINT32	ui32SLCode;
+	IMG_UINT32	ui32SLData;
+	IMG_UINT16	ui16PLCtrl;
+	IMG_UINT16	ui16CRC;
+
+} RGX_META_LDR_BLOCK_HDR;
+
+/* High level data stream block structure */
+typedef struct
+{
+	IMG_UINT16	ui16Cmd;
+	IMG_UINT16	ui16Length;
+	IMG_UINT32	ui32Next;
+	IMG_UINT32	aui32CmdData[4];
+
+} RGX_META_LDR_L1_DATA_BLK;
+
+/* High level data stream block structure */
+typedef struct
+{
+	IMG_UINT16	ui16Tag;
+	IMG_UINT16	ui16Length;
+	IMG_UINT32	aui32BlockData[4];
+
+} RGX_META_LDR_L2_DATA_BLK;
+
+/* Config command structure */
+typedef struct
+{
+	IMG_UINT32	ui32Type;
+	IMG_UINT32	aui32BlockData[4];
+
+} RGX_META_LDR_CFG_BLK;
+
+/* Block type definitions */
+#define RGX_META_LDR_COMMENT_TYPE_MASK			(0x0010U)
+#define RGX_META_LDR_BLK_IS_COMMENT(X)			((X & RGX_META_LDR_COMMENT_TYPE_MASK) != 0U)
+
+/* Command definitions
+ *  Value   Name            Description
+ *  0       LoadMem         Load memory with binary data.
+ *  1       LoadCore        Load a set of core registers.
+ *  2       LoadMMReg       Load a set of memory mapped registers.
+ *  3       StartThreads    Set each thread PC and SP, then enable threads.
+ *  4       ZeroMem         Zeros a memory region.
+ *  5       Config          Perform a configuration command.
+ */
+#define RGX_META_LDR_CMD_MASK				(0x000FU)
+
+#define RGX_META_LDR_CMD_LOADMEM			(0x0000U)
+#define RGX_META_LDR_CMD_LOADCORE			(0x0001U)
+#define RGX_META_LDR_CMD_LOADMMREG			(0x0002U)
+#define RGX_META_LDR_CMD_START_THREADS		(0x0003U)
+#define RGX_META_LDR_CMD_ZEROMEM			(0x0004U)
+#define RGX_META_LDR_CMD_CONFIG			(0x0005U)
+
+/* Config Command definitions
+ *  Value   Name        Description
+ *  0       Pause       Pause for x times 100 instructions
+ *  1       Read        Read a value from register - No value return needed.
+ *                      Utilises effects of issuing reads to certain registers
+ *  2       Write       Write to mem location
+ *  3       MemSet      Set mem to value
+ *  4       MemCheck    check mem for specific value.
+ */
+#define RGX_META_LDR_CFG_PAUSE			(0x0000)
+#define RGX_META_LDR_CFG_READ			(0x0001)
+#define RGX_META_LDR_CFG_WRITE			(0x0002)
+#define RGX_META_LDR_CFG_MEMSET			(0x0003)
+#define RGX_META_LDR_CFG_MEMCHECK		(0x0004)
+
+
+/******************************************************************************
+* RGX FW segmented MMU definitions
+******************************************************************************/
+/* All threads can access the segment */
+#define RGXFW_SEGMMU_ALLTHRS	(IMG_UINT32_C(0xf) << 8U)
+/* Writable */
+#define RGXFW_SEGMMU_WRITEABLE	(0x1U << 1U)
+/* All threads can access and writable */
+#define RGXFW_SEGMMU_ALLTHRS_WRITEABLE	(RGXFW_SEGMMU_ALLTHRS | RGXFW_SEGMMU_WRITEABLE)
+
+/* Direct map region 8 optionally used for custom mappings - max 8MB */
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_ID			(8U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_ADDR_START	(0x06000000U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_MAX_SIZE		(0x00800000U)
+
+/* Direct map region 9 optionally used for custom mappings - max 8MB */
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_ID			(9U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_ADDR_START	(0x06800000U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_MAX_SIZE		(0x00800000U)
+
+/* Direct map region 10 used for mapping GPU memory - max 8MB */
+#define RGXFW_SEGMMU_DMAP_GPU_ID			(10U)
+#define RGXFW_SEGMMU_DMAP_GPU_ADDR_START	(0x07000000U)
+#define RGXFW_SEGMMU_DMAP_GPU_MAX_SIZE		(0x00800000U)
+
+/* Segment IDs */
+#define RGXFW_SEGMMU_DATA_ID			(1U)
+#define RGXFW_SEGMMU_BOOTLDR_ID			(2U)
+#define RGXFW_SEGMMU_TEXT_ID			(RGXFW_SEGMMU_BOOTLDR_ID)
+
+/*
+ * SLC caching strategy in S7 and volcanic is emitted through the segment MMU.
+ * All the segments configured through the macro RGXFW_SEGMMU_OUTADDR_TOP are
+ * CACHED in the SLC.
+ * The interface has been kept the same to simplify the code changes.
+ * The bifdm argument is ignored (no longer relevant) in S7 and volcanic.
+ */
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(pers, slc_policy, mmu_ctx)      ((((IMG_UINT64) ((pers)    & 0x3U))  << 52) | \
+                                                                           (((IMG_UINT64) ((mmu_ctx) & 0xFFU)) << 44) | \
+                                                                           (((IMG_UINT64) ((slc_policy) & 0x1U))  << 40))
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(mmu_ctx)      RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(0x3U, 0x0U, mmu_ctx)
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED(mmu_ctx)    RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(0x0U, 0x1U, mmu_ctx)
+
+/* To configure the Page Catalog and BIF-DM fed into the BIF for Garten
+ * accesses through this segment
+ */
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC(pc, bifdm)              (((IMG_UINT64)((IMG_UINT64)(pc)    & 0xFU) << 44U) | \
+                                                              ((IMG_UINT64)((IMG_UINT64)(bifdm) & 0xFU) << 40U))
+
+#define RGXFW_SEGMMU_META_BIFDM_ID   (0x7U)
+#if !(defined(__KERNEL__) || defined(TEE_DDK)) && defined(RGX_FEATURE_META)
+#if defined(RGX_FEATURE_SLC_VIVT)
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED    RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_UNCACHED  RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED
+#define RGXFW_SEGMMU_OUTADDR_TOP_META          RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED
+#else
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED    RGXFW_SEGMMU_OUTADDR_TOP_SLC
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_UNCACHED  RGXFW_SEGMMU_OUTADDR_TOP_SLC
+#define RGXFW_SEGMMU_OUTADDR_TOP_META(pc)      RGXFW_SEGMMU_OUTADDR_TOP_SLC(pc, RGXFW_SEGMMU_META_BIFDM_ID)
+#endif
+#endif
+
+/* META segments have 4kB minimum size */
+#define RGXFW_SEGMMU_ALIGN			(0x1000U)
+
+/* Segmented MMU registers (n = segment id) */
+#define META_CR_MMCU_SEGMENTn_BASE(n)			(0x04850000U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_LIMIT(n)			(0x04850004U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_OUTA0(n)			(0x04850008U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_OUTA1(n)			(0x0485000CU + ((n)*0x10U))
+
+/* The following defines must be recalculated if the Meta MMU segments used
+ * to access Host-FW data are changed
+ * Current combinations are:
+ * - SLC uncached, META cached,   FW base address 0x70000000
+ * - SLC uncached, META uncached, FW base address 0xF0000000
+ * - SLC cached,   META cached,   FW base address 0x10000000
+ * - SLC cached,   META uncached, FW base address 0x90000000
+ */
+#define RGXFW_SEGMMU_DATA_BASE_ADDRESS        (0x10000000U)
+#define RGXFW_SEGMMU_DATA_META_CACHED         (0x0U)
+#define RGXFW_SEGMMU_DATA_META_UNCACHED       (META_MEM_GLOBAL_RANGE_BIT) // 0x80000000
+#define RGXFW_SEGMMU_DATA_META_CACHE_MASK     (META_MEM_GLOBAL_RANGE_BIT)
+/* For non-VIVT SLCs the cacheability of the FW data in the SLC is selected in
+ * the PTEs for the FW data, not in the Meta Segment MMU, which means these
+ * defines have no real effect in those cases
+ */
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED     (0x0U)
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED   (0x60000000U)
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK (0x60000000U)
+
+/******************************************************************************
+* RGX FW Bootloader defaults
+******************************************************************************/
+#define RGXFW_BOOTLDR_META_ADDR		(0x40000000U)
+#define RGXFW_BOOTLDR_DEVV_ADDR_0	(0xC0000000U)
+#define RGXFW_BOOTLDR_DEVV_ADDR_1	(0x000000E1)
+#define RGXFW_BOOTLDR_DEVV_ADDR		((((IMG_UINT64) RGXFW_BOOTLDR_DEVV_ADDR_1) << 32) | RGXFW_BOOTLDR_DEVV_ADDR_0)
+#define RGXFW_BOOTLDR_LIMIT		(0x1FFFF000)
+#define RGXFW_MAX_BOOTLDR_OFFSET	(0x1000)
+
+/* Bootloader configuration offset is in dwords (512 bytes) */
+#define RGXFW_BOOTLDR_CONF_OFFSET	(0x80)
+
+
+/******************************************************************************
+* RGX META Stack
+******************************************************************************/
+#define RGX_META_STACK_SIZE  (0x1000U)
+
+/******************************************************************************
+ RGX META Core memory
+******************************************************************************/
+/* code and data both map to the same physical memory */
+#define RGX_META_COREMEM_CODE_ADDR   (0x80000000U)
+#define RGX_META_COREMEM_DATA_ADDR   (0x82000000U)
+#define RGX_META_COREMEM_OFFSET_MASK (0x01ffffffU)
+
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#define RGX_META_IS_COREMEM_CODE(A, B)  (((A) >= RGX_META_COREMEM_CODE_ADDR) && ((A) < (RGX_META_COREMEM_CODE_ADDR + (B))))
+#define RGX_META_IS_COREMEM_DATA(A, B)  (((A) >= RGX_META_COREMEM_DATA_ADDR) && ((A) < (RGX_META_COREMEM_DATA_ADDR + (B))))
+#endif
+
+/******************************************************************************
+* 2nd thread
+******************************************************************************/
+#define RGXFW_THR1_PC		(0x18930000)
+#define RGXFW_THR1_SP		(0x78890000)
+
+/******************************************************************************
+* META compatibility
+******************************************************************************/
+
+#define META_CR_CORE_ID			(0x04831000)
+#define META_CR_CORE_ID_VER_SHIFT	(16U)
+#define META_CR_CORE_ID_VER_CLRMSK	(0XFF00FFFFU)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK)) && defined(RGX_FEATURE_META)
+
+	#if (RGX_FEATURE_META == MTP218)
+	#define RGX_CR_META_CORE_ID_VALUE 0x19
+	#elif (RGX_FEATURE_META == MTP219)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1E
+	#elif (RGX_FEATURE_META == LTP218)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1C
+	#elif (RGX_FEATURE_META == LTP217)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1F
+	#else
+	#error "Unknown META ID"
+	#endif
+#else
+
+	#define RGX_CR_META_MTP218_CORE_ID_VALUE 0x19
+	#define RGX_CR_META_MTP219_CORE_ID_VALUE 0x1E
+	#define RGX_CR_META_LTP218_CORE_ID_VALUE 0x1C
+	#define RGX_CR_META_LTP217_CORE_ID_VALUE 0x1F
+
+#endif
+#define RGXFW_PROCESSOR_META        "META"
+
+
+#endif /* RGX_META_H */
+
+/******************************************************************************
+ End of file (rgx_meta.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_mips.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_mips.h
new file mode 100644
index 000000000000..a80d2f66f0b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_mips.h
@@ -0,0 +1,406 @@
+/*************************************************************************/ /*!
+@File           rgx_mips.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    RGX MIPS definitions, kernel/user space
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_MIPS_H)
+#define RGX_MIPS_H
+
+/*
+ * Utility defines for memory management
+ */
+#define RGXMIPSFW_LOG2_PAGE_SIZE_4K              (12)
+#define RGXMIPSFW_PAGE_SIZE_4K                   (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+#define RGXMIPSFW_PAGE_MASK_4K                   (RGXMIPSFW_PAGE_SIZE_4K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_64K             (16)
+#define RGXMIPSFW_PAGE_SIZE_64K                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_64K)
+#define RGXMIPSFW_PAGE_MASK_64K                  (RGXMIPSFW_PAGE_SIZE_64K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_256K            (18)
+#define RGXMIPSFW_PAGE_SIZE_256K                 (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_256K)
+#define RGXMIPSFW_PAGE_MASK_256K                 (RGXMIPSFW_PAGE_SIZE_256K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_1MB             (20)
+#define RGXMIPSFW_PAGE_SIZE_1MB                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_1MB)
+#define RGXMIPSFW_PAGE_MASK_1MB                  (RGXMIPSFW_PAGE_SIZE_1MB - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_4MB             (22)
+#define RGXMIPSFW_PAGE_SIZE_4MB                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_4MB)
+#define RGXMIPSFW_PAGE_MASK_4MB                  (RGXMIPSFW_PAGE_SIZE_4MB - 1)
+#define RGXMIPSFW_LOG2_PTE_ENTRY_SIZE            (2)
+/* log2 page table sizes dependent on FW heap size and page size (for each OS) */
+#define RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K         (RGX_FIRMWARE_HEAP_SHIFT - RGXMIPSFW_LOG2_PAGE_SIZE_4K + RGXMIPSFW_LOG2_PTE_ENTRY_SIZE)
+#define RGXMIPSFW_LOG2_PAGETABLE_SIZE_64K        (RGX_FIRMWARE_HEAP_SHIFT - RGXMIPSFW_LOG2_PAGE_SIZE_64K + RGXMIPSFW_LOG2_PTE_ENTRY_SIZE)
+/* Maximum number of page table pages (both Host and MIPS pages) */
+#define RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES        (4)
+/* Total number of TLB entries */
+#define RGXMIPSFW_NUMBER_OF_TLB_ENTRIES          (16)
+/* "Uncached" caching policy */
+#define RGXMIPSFW_UNCACHED_CACHE_POLICY          (0X00000002U)
+/* "Write-back write-allocate" caching policy */
+#define RGXMIPSFW_WRITEBACK_CACHE_POLICY         (0X00000003)
+/* "Write-through no write-allocate" caching policy */
+#define RGXMIPSFW_WRITETHROUGH_CACHE_POLICY      (0X00000001)
+/* Cached policy used by MIPS in case of physical bus on 32 bit */
+#define RGXMIPSFW_CACHED_POLICY                  (RGXMIPSFW_WRITEBACK_CACHE_POLICY)
+/* Cached policy used by MIPS in case of physical bus on more than 32 bit */
+#define RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT      (RGXMIPSFW_WRITETHROUGH_CACHE_POLICY)
+/* Total number of Remap entries */
+#define RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES        (2 * RGXMIPSFW_NUMBER_OF_TLB_ENTRIES)
+
+
+/*
+ * MIPS EntryLo/PTE format
+ */
+
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_SHIFT     (31U)
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_CLRMSK    (0X7FFFFFFF)
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN        (0X80000000U)
+
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_SHIFT     (30U)
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_CLRMSK    (0XBFFFFFFF)
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN        (0X40000000U)
+
+/* Page Frame Number */
+#define RGXMIPSFW_ENTRYLO_PFN_SHIFT              (6)
+#define RGXMIPSFW_ENTRYLO_PFN_ALIGNSHIFT         (12)
+/* Mask used for the MIPS Page Table in case of physical bus on 32 bit */
+#define RGXMIPSFW_ENTRYLO_PFN_MASK               (0x03FFFFC0)
+#define RGXMIPSFW_ENTRYLO_PFN_SIZE               (20)
+/* Mask used for the MIPS Page Table in case of physical bus on more than 32 bit */
+#define RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT   (0x3FFFFFC0U)
+#define RGXMIPSFW_ENTRYLO_PFN_SIZE_ABOVE_32BIT   (24)
+#define RGXMIPSFW_ADDR_TO_ENTRYLO_PFN_RSHIFT     (RGXMIPSFW_ENTRYLO_PFN_ALIGNSHIFT - \
+                                                  RGXMIPSFW_ENTRYLO_PFN_SHIFT)
+
+#define RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT     (3U)
+#define RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK    (0XFFFFFFC7U)
+
+#define RGXMIPSFW_ENTRYLO_DIRTY_SHIFT            (2U)
+#define RGXMIPSFW_ENTRYLO_DIRTY_CLRMSK           (0XFFFFFFFB)
+#define RGXMIPSFW_ENTRYLO_DIRTY_EN               (0X00000004U)
+
+#define RGXMIPSFW_ENTRYLO_VALID_SHIFT            (1U)
+#define RGXMIPSFW_ENTRYLO_VALID_CLRMSK           (0XFFFFFFFD)
+#define RGXMIPSFW_ENTRYLO_VALID_EN               (0X00000002U)
+
+#define RGXMIPSFW_ENTRYLO_GLOBAL_SHIFT           (0U)
+#define RGXMIPSFW_ENTRYLO_GLOBAL_CLRMSK          (0XFFFFFFFE)
+#define RGXMIPSFW_ENTRYLO_GLOBAL_EN              (0X00000001U)
+
+#define RGXMIPSFW_ENTRYLO_DVG                    (RGXMIPSFW_ENTRYLO_DIRTY_EN | \
+                                                  RGXMIPSFW_ENTRYLO_VALID_EN | \
+                                                  RGXMIPSFW_ENTRYLO_GLOBAL_EN)
+#define RGXMIPSFW_ENTRYLO_UNCACHED               (RGXMIPSFW_UNCACHED_CACHE_POLICY << \
+                                                  RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT)
+#define RGXMIPSFW_ENTRYLO_DVG_UNCACHED           (RGXMIPSFW_ENTRYLO_DVG | RGXMIPSFW_ENTRYLO_UNCACHED)
+
+
+/* Remap Range Config Addr Out */
+/* These defines refer to the upper half of the Remap Range Config register */
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_MASK      (0x0FFFFFF0)
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_SHIFT     (4)  /* wrt upper half of the register */
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_ALIGNSHIFT (12)
+#define RGXMIPSFW_ADDR_TO_RR_ADDR_OUT_RSHIFT     (RGXMIPSFW_REMAP_RANGE_ADDR_OUT_ALIGNSHIFT - \
+                                                  RGXMIPSFW_REMAP_RANGE_ADDR_OUT_SHIFT)
+/*
+ * Pages to trampoline problematic physical addresses:
+ *   - RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN : 0x1FC0_0000
+ *   - RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN : 0x1FC0_1000
+ *   - RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN : 0x1FC0_2000
+ *   - (benign trampoline)               : 0x1FC0_3000
+ * that would otherwise be erroneously remapped by the MIPS wrapper
+ * (see "Firmware virtual layout and remap configuration" section below)
+ */
+
+#define RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES       (2)
+#define RGXMIPSFW_TRAMPOLINE_NUMPAGES            (1U << RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES)
+#define RGXMIPSFW_TRAMPOLINE_SIZE                (RGXMIPSFW_TRAMPOLINE_NUMPAGES << RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+#define RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE   (RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES + RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+
+#define RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR    (RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN)
+#define RGXMIPSFW_TRAMPOLINE_OFFSET(a)           (a - RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN)
+
+#define RGXMIPSFW_SENSITIVE_ADDR(a)              (RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN == (~((1UL << RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE)-1U) & a))
+
+#define RGXMIPSFW_C0_PAGEMASK_4K                 (0x00001800)
+#define RGXMIPSFW_C0_PAGEMASK_16K                (0x00007800)
+#define RGXMIPSFW_C0_PAGEMASK_64K                (0x0001F800)
+#define RGXMIPSFW_C0_PAGEMASK_256K               (0x0007F800)
+#define RGXMIPSFW_C0_PAGEMASK_1MB                (0x001FF800)
+#define RGXMIPSFW_C0_PAGEMASK_4MB                (0x007FF800)
+
+#if defined(RGX_FEATURE_GPU_MULTICORE_SUPPORT)
+/* GPU_COUNT: number of physical cores in the system
+ * NUM_OF_REGBANKS = GPU_COUNT + 1 //XPU BROADCAST BANK
+ * RGXMIPSFW_REGISTERS_PAGE_SIZE = NUM_OF_REGBANKS * REGBANK_SIZE(64KB) * NUM_OF_OSID(8)
+ * For RGXMIPSFW_REGISTERS_PAGE_SIZE = 4MB, NUM_OF_REGBANKS = 8 so supports upto GPU_COUNT = 7 cores
+ */
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_4MB)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_4MB)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4MB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED == 1)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_64K)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_64K)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64KB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED <= 4)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_256K)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_256K)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256KB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED <= 8)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_1MB)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_1MB)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_1MB)
+#else
+#error "MIPS TLB invalid params"
+#endif
+
+#define RGXMIPSFW_DECODE_REMAP_CONFIG_REGION_SIZE(r)       ((1U << (((r >> 7) + 1U) << 1U))*0x400)
+
+/*
+ * Firmware virtual layout and remap configuration
+ */
+/*
+ * For each remap region we define:
+ * - the virtual base used by the Firmware to access code/data through that region
+ * - the microAptivAP physical address correspondent to the virtual base address,
+ *   used as input address and remapped to the actual physical address
+ * - log2 of size of the region remapped by the MIPS wrapper, i.e. number of bits from
+ *   the bottom of the base input address that survive onto the output address
+ *   (this defines both the alignment and the maximum size of the remapped region)
+ * - one or more code/data segments within the remapped region
+ */
+
+/* Boot remap setup */
+#define RGXMIPSFW_BOOT_REMAP_VIRTUAL_BASE        (0xBFC00000)
+#define RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN        (0x1FC00000U)
+#define RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_BOOT_NMI_CODE_VIRTUAL_BASE     (RGXMIPSFW_BOOT_REMAP_VIRTUAL_BASE)
+
+/* Data remap setup */
+#define RGXMIPSFW_DATA_REMAP_VIRTUAL_BASE        (0xBFC01000)
+#define RGXMIPSFW_DATA_CACHED_REMAP_VIRTUAL_BASE (0x9FC01000)
+#define RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN        (0x1FC01000U)
+#define RGXMIPSFW_DATA_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_BOOT_NMI_DATA_VIRTUAL_BASE     (RGXMIPSFW_DATA_REMAP_VIRTUAL_BASE)
+
+/* Code remap setup */
+#define RGXMIPSFW_CODE_REMAP_VIRTUAL_BASE        (0x9FC02000)
+#define RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN        (0x1FC02000U)
+#define RGXMIPSFW_CODE_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_EXCEPTIONS_VIRTUAL_BASE        (RGXMIPSFW_CODE_REMAP_VIRTUAL_BASE)
+
+/* Permanent mappings setup */
+#define RGXMIPSFW_PT_VIRTUAL_BASE                (0xCF000000)
+#define RGXMIPSFW_REGISTERS_VIRTUAL_BASE         (0xCF800000)
+#define RGXMIPSFW_STACK_VIRTUAL_BASE             (0xCF600000)
+#define RGXMIPSFW_MIPS_STATE_VIRTUAL_BASE        (RGXMIPSFW_REGISTERS_VIRTUAL_BASE + RGXMIPSFW_REGISTERS_PAGE_SIZE)
+
+/* Offset inside the bootloader data page where the general_exception handler saves the error state.
+ * The error value is then copied by the NMI handler to the MipsState struct in shared memory.
+ * This is done because it's difficult to obtain the address of MipsState inside the general
+ * exception handler. */
+#define RGXMIPSFW_ERROR_STATE_BASE                        (0x100)
+
+/*
+ * Bootloader configuration data
+ */
+/* Bootloader configuration offset (where RGXMIPSFW_BOOT_DATA lives)
+ * within the bootloader/NMI data page */
+#define RGXMIPSFW_BOOTLDR_CONF_OFFSET                         (0x0U)
+
+/*
+ * MIPS boot stage
+ */
+#define RGXMIPSFW_BOOT_STAGE_OFFSET                           (0x400)
+
+/*
+ * MIPS private data in the bootloader data page.
+ * Memory below this offset is used by the FW only, no interface data allowed.
+ */
+#define RGXMIPSFW_PRIVATE_DATA_OFFSET                         (0x800)
+
+
+/* The things that follow are excluded when compiling assembly sources */
+#if !defined(RGXMIPSFW_ASSEMBLY_CODE)
+#include "img_types.h"
+#include "km/rgxdefs_km.h"
+
+typedef struct
+{
+	IMG_UINT64 ui64StackPhyAddr;
+	IMG_UINT64 ui64RegBase;
+	IMG_UINT64 aui64PTPhyAddr[RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES];
+	IMG_UINT32 ui32PTLog2PageSize;
+	IMG_UINT32 ui32PTNumPages;
+	IMG_UINT32 ui32Reserved1;
+	IMG_UINT32 ui32Reserved2;
+} RGXMIPSFW_BOOT_DATA;
+
+#define RGXMIPSFW_GET_OFFSET_IN_DWORDS(offset)                (offset / sizeof(IMG_UINT32))
+#define RGXMIPSFW_GET_OFFSET_IN_QWORDS(offset)                (offset / sizeof(IMG_UINT64))
+
+/* Used for compatibility checks */
+#define RGXMIPSFW_ARCHTYPE_VER_CLRMSK                         (0xFFFFE3FFU)
+#define RGXMIPSFW_ARCHTYPE_VER_SHIFT                          (10U)
+#define RGXMIPSFW_CORE_ID_VALUE                               (0x001U)
+#define RGXFW_PROCESSOR_MIPS                                  "MIPS"
+
+/* microAptivAP cache line size */
+#define RGXMIPSFW_MICROAPTIVEAP_CACHELINE_SIZE                (16U)
+
+/* The SOCIF transactions are identified with the top 16 bits of the physical address emitted by the MIPS */
+#define RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN           (16U)
+
+/* Values to put in the MIPS selectors for performance counters */
+#define RGXMIPSFW_PERF_COUNT_CTRL_ICACHE_ACCESSES_C0          (9U)   /* Icache accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_ICACHE_MISSES_C1            (9U)   /* Icache misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_DCACHE_ACCESSES_C0          (10U)  /* Dcache accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_DCACHE_MISSES_C1            (11U) /* Dcache misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_ITLB_INSTR_ACCESSES_C0      (5U)  /* ITLB instruction accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_JTLB_INSTR_MISSES_C1        (7U)  /* JTLB instruction accesses misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_INSTR_COMPLETED_C0          (1U)  /* Instructions completed in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_JTLB_DATA_MISSES_C1         (8U)  /* JTLB data misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_EVENT_SHIFT                 (5U)  /* Shift for the Event field in the MIPS perf ctrl registers */
+/* Additional flags for performance counters. See MIPS manual for further reference */
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_USER_MODE             (8U)
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_KERNEL_MODE           (2U)
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_EXL                   (1U)
+
+
+#define RGXMIPSFW_C0_NBHWIRQ	8
+
+/* Macros to decode C0_Cause register */
+#define RGXMIPSFW_C0_CAUSE_EXCCODE(CAUSE)       (((CAUSE) & 0x7cU) >> 2U)
+#define RGXMIPSFW_C0_CAUSE_EXCCODE_FWERROR      9
+/* Use only when Coprocessor Unusable exception */
+#define RGXMIPSFW_C0_CAUSE_UNUSABLE_UNIT(CAUSE) (((CAUSE) >> 28U) & 0x3U)
+#define RGXMIPSFW_C0_CAUSE_PENDING_HWIRQ(CAUSE) (((CAUSE) & 0x3fc00) >> 10)
+#define RGXMIPSFW_C0_CAUSE_FDCIPENDING          (1UL << 21)
+#define RGXMIPSFW_C0_CAUSE_IV                   (1UL << 23)
+#define RGXMIPSFW_C0_CAUSE_IC                   (1UL << 25)
+#define RGXMIPSFW_C0_CAUSE_PCIPENDING           (1UL << 26)
+#define RGXMIPSFW_C0_CAUSE_TIPENDING            (1UL << 30)
+#define RGXMIPSFW_C0_CAUSE_BRANCH_DELAY         (1UL << 31)
+
+/* Macros to decode C0_Debug register */
+#define RGXMIPSFW_C0_DEBUG_EXCCODE(DEBUG) (((DEBUG) >> 10U) & 0x1fU)
+#define RGXMIPSFW_C0_DEBUG_DSS            (1UL << 0)
+#define RGXMIPSFW_C0_DEBUG_DBP            (1UL << 1)
+#define RGXMIPSFW_C0_DEBUG_DDBL           (1UL << 2)
+#define RGXMIPSFW_C0_DEBUG_DDBS           (1UL << 3)
+#define RGXMIPSFW_C0_DEBUG_DIB            (1UL << 4)
+#define RGXMIPSFW_C0_DEBUG_DINT           (1UL << 5)
+#define RGXMIPSFW_C0_DEBUG_DIBIMPR        (1UL << 6)
+#define RGXMIPSFW_C0_DEBUG_DDBLIMPR       (1UL << 18)
+#define RGXMIPSFW_C0_DEBUG_DDBSIMPR       (1UL << 19)
+#define RGXMIPSFW_C0_DEBUG_IEXI           (1UL << 20)
+#define RGXMIPSFW_C0_DEBUG_DBUSEP         (1UL << 21)
+#define RGXMIPSFW_C0_DEBUG_CACHEEP        (1UL << 22)
+#define RGXMIPSFW_C0_DEBUG_MCHECKP        (1UL << 23)
+#define RGXMIPSFW_C0_DEBUG_IBUSEP         (1UL << 24)
+#define RGXMIPSFW_C0_DEBUG_DM             (1UL << 30)
+#define RGXMIPSFW_C0_DEBUG_DBD            (1UL << 31)
+
+/* Macros to decode TLB entries */
+#define RGXMIPSFW_TLB_GET_MASK(PAGE_MASK)       (((PAGE_MASK) >> 13) & 0XFFFFU)
+#define RGXMIPSFW_TLB_GET_PAGE_SIZE(PAGE_MASK)  ((((PAGE_MASK) | 0x1FFFU) + 1U) >> 11U) /* page size in KB */
+#define RGXMIPSFW_TLB_GET_PAGE_MASK(PAGE_SIZE)  ((((PAGE_SIZE) << 11) - 1) & ~0x7FF) /* page size in KB */
+#define RGXMIPSFW_TLB_GET_VPN2(ENTRY_HI)        ((ENTRY_HI) >> 13)
+#define RGXMIPSFW_TLB_GET_COHERENCY(ENTRY_LO)   (((ENTRY_LO) >> 3) & 0x7U)
+#define RGXMIPSFW_TLB_GET_PFN(ENTRY_LO)         (((ENTRY_LO) >> 6) & 0XFFFFFU)
+/* GET_PA uses a non-standard PFN mask for 36 bit addresses */
+#define RGXMIPSFW_TLB_GET_PA(ENTRY_LO)          (((IMG_UINT64)(ENTRY_LO) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) << 6)
+#define RGXMIPSFW_TLB_GET_INHIBIT(ENTRY_LO)     (((ENTRY_LO) >> 30) & 0x3U)
+#define RGXMIPSFW_TLB_GET_DGV(ENTRY_LO)         ((ENTRY_LO) & 0x7U)
+#define RGXMIPSFW_TLB_GLOBAL                    (1U)
+#define RGXMIPSFW_TLB_VALID                     (1U << 1)
+#define RGXMIPSFW_TLB_DIRTY                     (1U << 2)
+#define RGXMIPSFW_TLB_XI                        (1U << 30)
+#define RGXMIPSFW_TLB_RI                        (1U << 31)
+
+typedef struct {
+	IMG_UINT32 ui32TLBPageMask;
+	IMG_UINT32 ui32TLBHi;
+	IMG_UINT32 ui32TLBLo0;
+	IMG_UINT32 ui32TLBLo1;
+} RGX_MIPS_TLB_ENTRY;
+
+typedef struct {
+	IMG_UINT32 ui32RemapAddrIn;     /* always 4k aligned */
+	IMG_UINT32 ui32RemapAddrOut;    /* always 4k aligned */
+	IMG_UINT32 ui32RemapRegionSize;
+} RGX_MIPS_REMAP_ENTRY;
+
+typedef struct {
+	IMG_UINT32 ui32ErrorState; /* This must come first in the structure */
+	IMG_UINT32 ui32Sync;
+	IMG_UINT32 ui32ErrorEPC;
+	IMG_UINT32 ui32StatusRegister;
+	IMG_UINT32 ui32CauseRegister;
+	IMG_UINT32 ui32BadRegister;
+	IMG_UINT32 ui32EPC;
+	IMG_UINT32 ui32SP;
+	IMG_UINT32 ui32Debug;
+	IMG_UINT32 ui32DEPC;
+	IMG_UINT32 ui32BadInstr;
+	IMG_UINT32 ui32UnmappedAddress;
+	RGX_MIPS_TLB_ENTRY asTLB[RGXMIPSFW_NUMBER_OF_TLB_ENTRIES];
+	IMG_UINT64 aui64Remap[RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES];
+} RGX_MIPS_STATE;
+
+static_assert(offsetof(RGX_MIPS_STATE, ui32ErrorState) == 0,
+				"ui32ErrorState is not the first member of the RGX_MIPS_STATE struct");
+
+#if defined(SUPPORT_MIPS_64K_PAGE_SIZE)
+static_assert(RGXMIPSFW_REGISTERS_PAGE_SIZE >= RGXMIPSFW_PAGE_SIZE_64K,
+				"Register page size must be greater or equal to MIPS page size");
+#else
+static_assert(RGXMIPSFW_REGISTERS_PAGE_SIZE >= RGXMIPSFW_PAGE_SIZE_4K,
+				"Register page size must be greater or equal to MIPS page size");
+#endif
+
+
+#endif /* RGXMIPSFW_ASSEMBLY_CODE */
+
+#endif /* RGX_MIPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_riscv.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_riscv.h
new file mode 100644
index 000000000000..f98b77658d8f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_riscv.h
@@ -0,0 +1,258 @@
+/*************************************************************************/ /*!
+@File           rgx_riscv.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    RGX RISCV definitions, kernel/user space
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_RISCV_H)
+#define RGX_RISCV_H
+
+#include "km/rgxdefs_km.h"
+
+
+/* Utility defines to convert regions to virtual addresses and remaps */
+#define RGXRISCVFW_NUM_REGIONS_LOG2             IMG_UINT32_C(4)
+#define RGXRISCVFW_REGION_SIZE_LOG2             IMG_UINT32_C(28)
+#define RGXRISCVFW_NUM_REGIONS                  (IMG_UINT32_C(1) << RGXRISCVFW_NUM_REGIONS_LOG2)
+#define RGXRISCVFW_GET_REGION_BASE(r)           ((r) << RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_GET_REGION(a)                ((a) >> RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_MAX_REGION_SIZE              (IMG_UINT32_C(1) << RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_GET_REMAP(r)                 (RGX_CR_FWCORE_ADDR_REMAP_CONFIG0 + ((r) * 8U))
+
+#define RGXRISCVFW_REGION_MASK                  (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_NUM_REGIONS - IMG_UINT32_C(1)))
+
+#if !defined(RGXRISCVFW_ASSEMBLY_CODE)
+static_assert(RGXRISCVFW_NUM_REGIONS_LOG2 + RGXRISCVFW_REGION_SIZE_LOG2 == 32,
+              "Mismatch between number of remap regions and remap size");
+#endif
+
+/* RISCV remap output is aligned to 4K */
+#define RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN  (0x1000U)
+
+/*
+ * FW bootloader defines
+ */
+#define RGXRISCVFW_BOOTLDR_CODE_REGION          (IMG_UINT32_C(0xC0000000) >> RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_BOOTLDR_DATA_REGION          IMG_UINT32_C(0x5)
+#define RGXRISCVFW_BOOTLDR_CODE_BASE            (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_BASE            (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_BOOTLDR_DATA_REGION))
+#define RGXRISCVFW_BOOTLDR_CODE_REMAP           (RGXRISCVFW_GET_REMAP(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_REMAP           (RGXRISCVFW_GET_REMAP(RGXRISCVFW_BOOTLDR_DATA_REGION))
+
+/* Bootloader data offset in dwords from the beginning of the FW data allocation */
+#define RGXRISCVFW_BOOTLDR_CONF_OFFSET          (0x0)
+
+/*
+ * FW coremem region defines
+ */
+#define RGXRISCVFW_COREMEM_REGION               IMG_UINT32_C(0x8)
+#define RGXRISCVFW_COREMEM_MAX_SIZE             IMG_UINT32_C(0x10000000) /* 256 MB */
+#define RGXRISCVFW_COREMEM_BASE                 (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_COREMEM_REGION))
+#define RGXRISCVFW_COREMEM_END                  (RGXRISCVFW_COREMEM_BASE + RGXRISCVFW_COREMEM_MAX_SIZE - 1U)
+
+
+/*
+ * Host-FW shared data defines
+ */
+#define RGXRISCVFW_SHARED_CACHED_DATA_REGION    (0x0UL)
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_REGION  (0x1UL)
+#define RGXRISCVFW_SHARED_CACHED_DATA_BASE      (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SHARED_CACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_BASE    (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SHARED_UNCACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_CACHED_DATA_REMAP     (RGXRISCVFW_GET_REMAP(RGXRISCVFW_SHARED_CACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_REMAP   (RGXRISCVFW_GET_REMAP(RGXRISCVFW_SHARED_UNCACHED_DATA_REGION))
+
+
+/*
+ * GPU SOCIF access defines
+ */
+#define RGXRISCVFW_SOCIF_REGION                 (0x2U)
+#define RGXRISCVFW_SOCIF_BASE                   (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SOCIF_REGION))
+
+
+/* The things that follow are excluded when compiling assembly sources */
+#if !defined(RGXRISCVFW_ASSEMBLY_CODE)
+#include "img_types.h"
+
+#define RGXFW_PROCESSOR_RISCV       "RISCV"
+#define RGXRISCVFW_CORE_ID_VALUE    (0x00450B02U)
+#define RGXRISCVFW_MISA_ADDR        (0x301U)
+#define RGXRISCVFW_MISA_VALUE       (0x40001104U)
+#define RGXRISCVFW_MSCRATCH_ADDR    (0x340U)
+
+typedef struct
+{
+	IMG_UINT64 ui64CorememCodeDevVAddr;
+	IMG_UINT64 ui64CorememDataDevVAddr;
+	IMG_UINT32 ui32CorememCodeFWAddr;
+	IMG_UINT32 ui32CorememDataFWAddr;
+	IMG_UINT32 ui32CorememCodeSize;
+	IMG_UINT32 ui32CorememDataSize;
+	IMG_UINT32 ui32Flags;
+	IMG_UINT32 ui32Reserved;
+} RGXRISCVFW_BOOT_DATA;
+
+/*
+ * List of registers to be printed in debug dump.
+ * First column:  register names (general purpose or control/status registers)
+ * Second column: register number to be used in abstract access register command
+ * (see RISC-V debug spec v0.13)
+ */
+#define RGXRISCVFW_DEBUG_DUMP_REGISTERS \
+	X(pc,        0x7b1) /* dpc */ \
+	X(ra,       0x1001) \
+	X(sp,       0x1002) \
+	X(mepc,      0x341) \
+	X(mcause,    0x342) \
+	X(mdseac,    0xfc0) \
+	X(mstatus,   0x300) \
+	X(mie,       0x304) \
+	X(mip,       0x344) \
+	X(mscratch,  0x340) \
+	X(mbvnc0,    0xffe) \
+	X(mbvnc1,    0xfff) \
+	X(micect,    0x7f0) \
+	X(mdcect,    0x7f3) \
+	X(mdcrfct,   0x7f4) \
+
+typedef struct
+{
+#define X(name, address) \
+	IMG_UINT32 name;
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+} RGXRISCVFW_STATE;
+
+
+#define RGXRISCVFW_MCAUSE_INTERRUPT  (1U << 31)
+
+#define RGXRISCVFW_MCAUSE_TABLE \
+	X(0x00000000U, IMG_FALSE, "NMI pin assertion") /* Also reset value */ \
+	X(0x00000001U, IMG_TRUE,  "Instruction access fault") \
+	X(0x00000002U, IMG_TRUE,  "Illegal instruction") \
+	X(0x00000003U, IMG_TRUE,  "Breakpoint") \
+	X(0x00000004U, IMG_TRUE,  "Load address misaligned") \
+	X(0x00000005U, IMG_TRUE,  "Load access fault") \
+	X(0x00000006U, IMG_TRUE,  "Store/AMO address misaligned") \
+	X(0x00000007U, IMG_TRUE,  "Store/AMO access fault") \
+	X(0x0000000BU, IMG_TRUE,  "Environment call from M-mode (FW assert)") \
+	X(0x80000007U, IMG_FALSE, "Machine timer interrupt") \
+	X(0x8000000BU, IMG_FALSE, "Machine external interrupt") \
+	X(0x8000001EU, IMG_FALSE, "Machine correctable error local interrupt") \
+	X(0xF0000000U, IMG_TRUE,  "Machine D-bus store error NMI") \
+	X(0xF0000001U, IMG_TRUE,  "Machine D-bus non-blocking load error NMI") \
+	X(0xF0000002U, IMG_TRUE,  "dCache unrecoverable NMI")
+
+
+/* Debug module HW defines */
+#define RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER (0U)
+#define RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY   (2U)
+#define RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT   (2UL << 20)
+#define RGXRISCVFW_DMI_COMMAND_WRITE           (1UL << 16)
+#define RGXRISCVFW_DMI_COMMAND_READ            (0UL << 16)
+#define RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT     (2U)
+
+/* Abstract command error codes (descriptions from RISC-V debug spec v0.13) */
+typedef IMG_UINT32 RGXRISCVFW_ABSTRACT_CMD_ERR;
+
+/* No error. */
+#define RISCV_ABSTRACT_CMD_NO_ERROR 0U
+
+/*
+ * An abstract command was executing while command, abstractcs, or abstractauto
+ * was written, or when one of the data or progbuf registers was read or
+ * written. This status is only written if cmderr contains 0.
+ */
+#define RISCV_ABSTRACT_CMD_BUSY 1U
+
+/*
+ * The requested command is not supported, regardless of whether
+ * the hart is running or not.
+ */
+#define RISCV_ABSTRACT_CMD_NOT_SUPPORTED 2U
+
+/*
+ * An exception occurred while executing the command
+ * (e.g. while executing the Program Buffer).
+ */
+#define RISCV_ABSTRACT_CMD_EXCEPTION 3U
+
+/*
+ * The abstract command couldn't execute because the hart wasn't in the required
+ * state (running/halted), or unavailable.
+ */
+#define RISCV_ABSTRACT_CMD_HALT_RESUME 4U
+
+/*
+ * The abstract command failed due to a bus error
+ * (e.g. alignment, access size, or timeout).
+ */
+#define RISCV_ABSTRACT_CMD_BUS_ERROR 5U
+
+/* The command failed for another reason. */
+#define RISCV_ABSTRACT_CMD_OTHER_ERROR 7U
+
+
+/* System Bus error codes (descriptions from RISC-V debug spec v0.13) */
+typedef IMG_UINT32 RGXRISCVFW_SYSBUS_ERR;
+
+/* There was no bus error. */
+#define RISCV_SYSBUS_NO_ERROR 0U
+
+/* There was a timeout. */
+#define RISCV_SYSBUS_TIMEOUT 1U
+
+/* A bad address was accessed. */
+#define RISCV_SYSBUS_BAD_ADDRESS 2U
+
+/* There was an alignment error. */
+#define RISCV_SYSBUS_BAD_ALIGNMENT 3U
+
+/* An access of unsupported size was requested. */
+#define RISCV_SYSBUS_UNSUPPORTED_SIZE 4U
+
+/* Other. */
+#define RISCV_SYSBUS_OTHER_ERROR 7U
+
+
+#endif /* RGXRISCVFW_ASSEMBLY_CODE */
+
+#endif /* RGX_RISCV_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgxfw_log_helper.h b/drivers/gpu/drm/img/img-volcanic/include/rgxfw_log_helper.h
new file mode 100644
index 000000000000..275b63aca46b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgxfw_log_helper.h
@@ -0,0 +1,79 @@
+/*************************************************************************/ /*!
+@File           rgxfw_log_helper.h
+@Title          Firmware TBI logging helper function
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       Generic
+@Description    This file contains some helper code to make TBI logging possible
+                Specifically, it uses the SFIDLIST xmacro to trace ids back to
+                the original strings.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGXFW_LOG_HELPER_H
+#define RGXFW_LOG_HELPER_H
+
+#include "rgx_fwif_sf.h"
+
+static const IMG_CHAR *const groups[]= {
+#define X(A,B) #B,
+	RGXFW_LOG_SFGROUPLIST
+#undef X
+};
+
+/*  idToStringID : Search SFs tuples {id,string} for a matching id.
+ *   return index to array if found or RGXFW_SF_LAST if none found.
+ *   bsearch could be used as ids are in increasing order. */
+#if defined(RGX_FIRMWARE)
+static IMG_UINT32 idToStringID(IMG_UINT32 ui32CheckData, const RGXFW_STID_FMT *const psSFs)
+#else
+static IMG_UINT32 idToStringID(IMG_UINT32 ui32CheckData, const RGXKM_STID_FMT *const psSFs)
+#endif
+{
+	IMG_UINT32 i = 0, ui32Id = (IMG_UINT32)RGXFW_SF_LAST;
+
+	for ( i = 0 ; psSFs[i].ui32Id != (IMG_UINT32)RGXFW_SF_LAST ; i++)
+	{
+		if ( ui32CheckData == psSFs[i].ui32Id )
+		{
+			ui32Id = i;
+			break;
+		}
+	}
+	return ui32Id;
+}
+
+#endif /* RGXFW_LOG_HELPER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/ri_typedefs.h b/drivers/gpu/drm/img/img-volcanic/include/ri_typedefs.h
new file mode 100644
index 000000000000..77be10e2ab03
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/ri_typedefs.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Information (RI) Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Client side part of RI management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RI_TYPEDEFS_H
+#define RI_TYPEDEFS_H
+
+#include "img_types.h"
+
+typedef struct RI_SUBLIST_ENTRY RI_ENTRY;
+typedef RI_ENTRY* RI_HANDLE;
+
+#endif /* #ifndef RI_TYPEDEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/pvrsrv_surface.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/pvrsrv_surface.h
new file mode 100644
index 000000000000..ae80f3cf1793
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/pvrsrv_surface.h
@@ -0,0 +1,218 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR Services Surface
+@Description    Surface definitions and utilities that are externally visible
+                (i.e. visible to clients of services), but are also required
+                within services.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_SURFACE_H
+#define PVRSRV_SURFACE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_SURFACE_TRANSFORM_NONE      (0U << 0)		/*!< No transformation */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_H    (1U << 0)		/*!< Flip horizontally */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_V    (1U << 1)		/*!< Flip vertically */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_90    (1U << 2)                          /*!< Rotate  90 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_180   ((1U << 0) + (1U << 1))             /*!< Rotate 180 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_270   ((1U << 0) + (1U << 1) + (1U << 2))  /*!< Rotate 270 degree clockwise */
+
+#define PVRSRV_SURFACE_BLENDING_NONE       0	/*!< Use no blending */
+#define PVRSRV_SURFACE_BLENDING_PREMULT    1	/*!< Use blending with pre-multiplier */
+#define PVRSRV_SURFACE_BLENDING_COVERAGE   2	/*!< Use coverage blending */
+
+/*!
+ * Modes of memory layouts for surfaces.
+ *
+ *   Enum: #_PVRSRV_SURFACE_MEMLAYOUT_
+ *   Typedef: ::PVRSRV_SURFACE_MEMLAYOUT
+ */
+typedef enum _PVRSRV_SURFACE_MEMLAYOUT_
+{
+	PVRSRV_SURFACE_MEMLAYOUT_STRIDED = 0,		/*!< Strided memory buffer */
+	PVRSRV_SURFACE_MEMLAYOUT_FBC,				/*!< Compressed frame buffer */
+} PVRSRV_SURFACE_MEMLAYOUT;
+
+/*!
+ * Frame Buffer Compression layout.
+ * Defines the FBC mode to use.
+ *
+ *   Structure: #PVRSRV_SURFACE_FBC_LAYOUT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FBC_LAYOUT
+ */
+typedef struct PVRSRV_SURFACE_FBC_LAYOUT_TAG
+{
+	/*! The compression mode for this surface */
+	IMG_FB_COMPRESSION	eFBCompressionMode;
+} PVRSRV_SURFACE_FBC_LAYOUT;
+
+/*!
+ * Pixel and memory format of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_FORMAT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FORMAT
+ */
+typedef struct PVRSRV_SURFACE_FORMAT_TAG
+{
+	/*! Enum value of type IMG_PIXFMT for the pixel format */
+	IMG_UINT32					ePixFormat;
+
+	/*! Enum surface memory layout */
+	PVRSRV_SURFACE_MEMLAYOUT	eMemLayout;
+
+	/*! Special layout options for the surface.
+	 * Needs services support.
+	 * Depends on eMemLayout.*/
+	union {
+		PVRSRV_SURFACE_FBC_LAYOUT	sFBCLayout;
+	} u;
+} PVRSRV_SURFACE_FORMAT;
+
+/*!
+ * Width and height of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_DIMS_TAG
+ *   Typedef: ::PVRSRV_SURFACE_DIMS
+ */
+typedef struct PVRSRV_SURFACE_DIMS_TAG
+{
+	IMG_UINT32		ui32Width;		/*!< Width in pixels */
+	IMG_UINT32		ui32Height;		/*!< Height in pixels */
+} PVRSRV_SURFACE_DIMS;
+
+/*!
+ * Dimension and format details of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_INFO
+ */
+typedef struct PVRSRV_SURFACE_INFO_TAG
+{
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Width and height */
+	PVRSRV_SURFACE_FORMAT	sFormat;	/*!< Memory format */
+
+	/*! Needed for FBCv3.
+	 * Indicates offset from allocation start
+	 * to the actual data, skipping the header.
+	 */
+	IMG_UINT32				ui32FBCBaseOffset;
+} PVRSRV_SURFACE_INFO;
+
+/*!
+ * Defines a rectangle on a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_RECT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_RECT
+ */
+typedef struct PVRSRV_SURFACE_RECT_TAG
+{
+	IMG_INT32				i32XOffset;	/*!< X offset from origin in pixels */
+	IMG_INT32				i32YOffset;	/*!< Y offset from origin in pixels */
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Rectangle dimensions */
+} PVRSRV_SURFACE_RECT;
+
+/*!
+ * Surface configuration details
+ *
+ *   Structure: #PVRSRV_SURFACE_CONFIG_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_CONFIG_INFO
+ */
+typedef struct PVRSRV_SURFACE_CONFIG_INFO_TAG
+{
+	/*! Crop applied to surface (BEFORE transformation) */
+	PVRSRV_SURFACE_RECT		sCrop;
+
+	/*! Region of screen to display surface in (AFTER scaling) */
+	PVRSRV_SURFACE_RECT		sDisplay;
+
+	/*! Surface transformation none/flip/rotate.
+	 * Use PVRSRV_SURFACE_TRANSFORM_xxx macros
+	 */
+	IMG_UINT32				ui32Transform;
+
+	/*! Alpha blending mode e.g. none/premult/coverage.
+	 * Use PVRSRV_SURFACE_BLENDING_xxx macros
+	 */
+	IMG_UINT32				eBlendType;
+
+	/*! Custom data for the display engine */
+	IMG_UINT32				ui32Custom;
+
+	/*! Alpha value for this plane */
+	IMG_UINT8				ui8PlaneAlpha;
+
+	/*! Reserved for later use */
+	IMG_UINT8				ui8Reserved1[3];
+} PVRSRV_SURFACE_CONFIG_INFO;
+
+/*!
+ * Contains information about a panel
+ */
+typedef struct PVRSRV_PANEL_INFO_TAG
+{
+	PVRSRV_SURFACE_INFO sSurfaceInfo;		/*!< Panel surface details */
+	IMG_UINT32			ui32RefreshRate;	/*!< Panel refresh rate in Hz */
+	IMG_UINT32			ui32XDpi;			/*!< Panel DPI in x direction */
+	IMG_UINT32			ui32YDpi;			/*!< Panel DPI in y direction */
+} PVRSRV_PANEL_INFO;
+
+/*!
+ * Helper function to create a Config Info based on a Surface Info
+ * to do a flip with no scale, transformation etc.
+ */
+static INLINE void SurfaceConfigFromSurfInfo(const PVRSRV_SURFACE_INFO *psSurfaceInfo,
+                                             PVRSRV_SURFACE_CONFIG_INFO *psConfigInfo)
+{
+	psConfigInfo->sCrop.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sCrop.i32XOffset = 0;
+	psConfigInfo->sCrop.i32YOffset = 0;
+	psConfigInfo->sDisplay.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sDisplay.i32XOffset = 0;
+	psConfigInfo->sDisplay.i32YOffset = 0;
+	psConfigInfo->ui32Transform = PVRSRV_SURFACE_TRANSFORM_NONE;
+	psConfigInfo->eBlendType = PVRSRV_SURFACE_BLENDING_NONE;
+	psConfigInfo->ui32Custom = 0;
+	psConfigInfo->ui8PlaneAlpha = 0xff;
+}
+
+#endif /* PVRSRV_SURFACE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_alignchecks.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_alignchecks.h
new file mode 100644
index 000000000000..a33b6d541c65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_alignchecks.h
@@ -0,0 +1,192 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX fw interface alignment checks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Checks to avoid disalignment in RGX fw data structures
+                shared with the host
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_ALIGNCHECKS_H)
+#define RGX_FWIF_ALIGNCHECKS_H
+
+/* for the offsetof macro */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/*!
+ ******************************************************************************
+ * Alignment UM/FW checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_UM_MAX 128U
+
+#define RGXFW_ALIGN_CHECKS_INIT0								\
+		(IMG_UINT32)sizeof(RGXFWIF_TRACEBUF),								\
+		offsetof(RGXFWIF_TRACEBUF, ui32LogType),				\
+		offsetof(RGXFWIF_TRACEBUF, sTraceBuf),					\
+		offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),	\
+		offsetof(RGXFWIF_TRACEBUF, ui32TracebufFlags),			\
+																\
+		(IMG_UINT32)sizeof(RGXFWIF_SYSDATA),								\
+		offsetof(RGXFWIF_SYSDATA, ePowState),					\
+		offsetof(RGXFWIF_SYSDATA, ui32HWPerfDropCount),			\
+		offsetof(RGXFWIF_SYSDATA, ui32LastDropOrdinal),			\
+		offsetof(RGXFWIF_SYSDATA, ui32FWFaults),				\
+		offsetof(RGXFWIF_SYSDATA, ui32HWRStateFlags),			\
+																\
+		(IMG_UINT32)sizeof(RGXFWIF_OSDATA),									\
+		offsetof(RGXFWIF_OSDATA, ui32HostSyncCheckMark),		\
+		offsetof(RGXFWIF_OSDATA, ui32KCCBCmdsExecuted),			\
+																	\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRINFOBUF),									\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmLockedUpCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmOverranCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmRecoveredCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmFalseDetectCount),	\
+																	\
+		/* RGXFWIF_CMDTA checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTA),							\
+		offsetof(RGXFWIF_CMDTA, sGeomRegs),				\
+														\
+		/* RGXFWIF_CMD3D checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_CMD3D),							\
+		offsetof(RGXFWIF_CMD3D, s3DRegs),				\
+														\
+		/* RGXFWIF_CMDTRANSFER checks */				\
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTRANSFER),					\
+		offsetof(RGXFWIF_CMDTRANSFER, sTransRegs),		\
+														\
+														\
+		/* RGXFWIF_CMD_COMPUTE checks */				\
+		(IMG_UINT32)sizeof(RGXFWIF_CMD_COMPUTE),					\
+		offsetof(RGXFWIF_CMD_COMPUTE, sCDMRegs),		\
+														\
+		/* RGXFWIF_FREELIST checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_FREELIST),						\
+		offsetof(RGXFWIF_FREELIST, psFreeListDevVAddr),	\
+		offsetof(RGXFWIF_FREELIST, ui32MaxPages),		\
+		offsetof(RGXFWIF_FREELIST, ui32CurrentPages),	\
+														\
+		/* RGXFWIF_HWRTDATA checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRTDATA),							\
+		offsetof(RGXFWIF_HWRTDATA, psVHeapTableDevVAddr),	\
+		offsetof(RGXFWIF_HWRTDATA, psPMMListDevVAddr),		\
+		offsetof(RGXFWIF_HWRTDATA, apsFreeLists),			\
+		offsetof(RGXFWIF_HWRTDATA, ui64VCECatBase),			\
+		offsetof(RGXFWIF_HWRTDATA, eState),					\
+															\
+		/* RGXFWIF_HWRTDATA_COMMON checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRTDATA_COMMON),						\
+		offsetof(RGXFWIF_HWRTDATA_COMMON, bTACachesNeedZeroing),\
+																\
+		/* RGXFWIF_HWPERF_CTL_BLK checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_HWPERF_CTL_BLK),						\
+		offsetof(RGXFWIF_HWPERF_CTL_BLK, aui64CounterCfg),	\
+															\
+		/* RGXFWIF_HWPERF_CTL checks */			\
+		(IMG_UINT32)sizeof(RGXFWIF_HWPERF_CTL),				\
+		offsetof(RGXFWIF_HWPERF_CTL, SelCntr)
+
+#if defined(RGX_FEATURE_TLA)
+#define RGXFW_ALIGN_CHECKS_INIT1                   \
+		RGXFW_ALIGN_CHECKS_INIT0,                  \
+		/* RGXFWIF_CMD2D checks */                 \
+		(IMG_UINT32)sizeof(RGXFWIF_CMD2D),                     \
+		offsetof(RGXFWIF_CMD2D, s2DRegs)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT1		RGXFW_ALIGN_CHECKS_INIT0
+#endif	/* RGX_FEATURE_TLA */
+
+
+#if defined(RGX_FEATURE_FASTRENDER_DM)
+#define RGXFW_ALIGN_CHECKS_INIT                    \
+		RGXFW_ALIGN_CHECKS_INIT1,                  \
+		/* RGXFWIF_CMDTDM checks */                \
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTDM),                    \
+		offsetof(RGXFWIF_CMDTDM, sTDMRegs)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT		RGXFW_ALIGN_CHECKS_INIT1
+#endif /* ! RGX_FEATURE_FASTRENDER_DM */
+
+
+
+/*!
+ ******************************************************************************
+ * Alignment KM checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_INIT_KM                                   \
+		(IMG_UINT32)sizeof(RGXFWIF_SYSINIT),                                     \
+		offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr),                   \
+		offsetof(RGXFWIF_SYSINIT, sPDSExecBase),                     \
+		offsetof(RGXFWIF_SYSINIT, sUSCExecBase),                     \
+		offsetof(RGXFWIF_SYSINIT, aui32TPUTrilinearFracMask),        \
+		offsetof(RGXFWIF_SYSINIT, sTraceBufCtl),                     \
+		offsetof(RGXFWIF_SYSINIT, sFwSysData),                       \
+		(IMG_UINT32)sizeof(RGXFWIF_OSINIT),                                      \
+		offsetof(RGXFWIF_OSINIT, psKernelCCBCtl),                    \
+		offsetof(RGXFWIF_OSINIT, psKernelCCB),                       \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCBCtl),                  \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCB),                     \
+		offsetof(RGXFWIF_OSINIT, sFwOsData),                         \
+		offsetof(RGXFWIF_OSINIT, sRGXCompChecks),                    \
+		                                                             \
+		/* RGXFWIF_FWRENDERCONTEXT checks */                         \
+		(IMG_UINT32)sizeof(RGXFWIF_FWRENDERCONTEXT),                             \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),               \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),               \
+		                                                             \
+		(IMG_UINT32)sizeof(RGXFWIF_FWCOMMONCONTEXT),                             \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psFWMemContext),           \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, sRunNode),                 \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psCCB),                    \
+		                                                             \
+		(IMG_UINT32)sizeof(RGXFWIF_MMUCACHEDATA),                                \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32CacheFlags),              \
+		offsetof(RGXFWIF_MMUCACHEDATA, sMMUCacheSync),               \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32MMUCacheSyncUpdateValue)
+
+#endif /* RGX_FWIF_ALIGNCHECKS_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_alignchecks.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_hwperf.h
new file mode 100644
index 000000000000..ec2d9bb071e7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_hwperf.h
@@ -0,0 +1,253 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_hwperf.h
+@Title          RGX HWPerf support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared header between RGX firmware and Init process
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_HWPERF_H
+#define RGX_FWIF_HWPERF_H
+
+#include "rgx_fwif_shared.h"
+#include "rgx_hwperf.h"
+#include "rgxdefs_km.h"
+
+
+/*****************************************************************************/
+
+/* Structure to hold a block's parameters for passing between the BG context
+ * and the IRQ context when applying a configuration request. */
+typedef struct
+{
+	/* Few members could be booleans but padded to IMG_UINT32
+	 * to workaround pdump alignment requirements */
+	IMG_UINT32              ui32Valid;
+	IMG_UINT32              ui32Enabled;
+	IMG_UINT32              eBlockID;
+	IMG_UINT32              uiCounterMask;
+	IMG_UINT64  RGXFW_ALIGN aui64CounterCfg[RGX_CNTBLK_MUX_COUNTERS_MAX];
+}  RGXFWIF_HWPERF_CTL_BLK;
+
+/* Structure used to hold the configuration of the non-mux counters blocks */
+typedef struct
+{
+	IMG_UINT32            ui32NumSelectedCounters;
+	IMG_UINT32            aui32SelectedCountersIDs[RGX_HWPERF_MAX_CUSTOM_CNTRS];
+} RGXFW_HWPERF_SELECT;
+
+/* Structure used to hold a Direct-Addressable block's parameters for passing
+ * between the BG context and the IRQ context when applying a configuration
+ * request. HWPERF_UNIFIED use only.
+ */
+typedef struct
+{
+	IMG_UINT32               uiEnabled;
+	IMG_UINT32               uiNumCounters;
+	IMG_UINT32               eBlockID;
+	RGXFWIF_DEV_VIRTADDR     psModel;
+	IMG_UINT32               aui32Counters[RGX_CNTBLK_COUNTERS_MAX];
+} RGXFWIF_HWPERF_DA_BLK;
+
+
+/* Structure to hold the whole configuration request details for all blocks
+ * The block masks and counts are used to optimise reading of this data. */
+typedef struct
+{
+	IMG_UINT32                         ui32HWPerfCtlFlags;
+
+	IMG_UINT32                         ui32SelectedCountersBlockMask;
+	RGXFW_HWPERF_SELECT RGXFW_ALIGN    SelCntr[RGX_HWPERF_MAX_CUSTOM_BLKS];
+
+	IMG_UINT32                         ui32EnabledMUXBlksCount;
+	RGXFWIF_HWPERF_CTL_BLK RGXFW_ALIGN sBlkCfg[RGX_HWPERF_MAX_MUX_BLKS];
+} UNCACHED_ALIGN RGXFWIF_HWPERF_CTL;
+
+/* NOTE: The switch statement in this function must be kept in alignment with
+ * the enumeration RGX_HWPERF_CNTBLK_ID defined in rgx_hwperf.h. ASSERTs may
+ * result if not.
+ * The function provides a hash lookup to get a handle on the global store for
+ * a block's configuration store from it's block ID.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(rgxfw_hwperf_get_block_ctl)
+#endif
+static INLINE RGXFWIF_HWPERF_CTL_BLK *rgxfw_hwperf_get_block_ctl(
+		RGX_HWPERF_CNTBLK_ID eBlockID, RGXFWIF_HWPERF_CTL *psHWPerfInitData)
+{
+	IMG_UINT32 ui32Idx;
+
+	/* Hash the block ID into a control configuration array index */
+	switch (eBlockID)
+	{
+		case RGX_CNTBLK_ID_TA:
+		case RGX_CNTBLK_ID_RASTER:
+		case RGX_CNTBLK_ID_HUB:
+		case RGX_CNTBLK_ID_TORNADO:
+		case RGX_CNTBLK_ID_JONES:
+		{
+			ui32Idx = eBlockID;
+			break;
+		}
+		case RGX_CNTBLK_ID_TPU_MCU0:
+		case RGX_CNTBLK_ID_TPU_MCU1:
+		case RGX_CNTBLK_ID_TPU_MCU2:
+		case RGX_CNTBLK_ID_TPU_MCU3:
+		case RGX_CNTBLK_ID_TPU_MCU4:
+		case RGX_CNTBLK_ID_TPU_MCU5:
+		case RGX_CNTBLK_ID_TPU_MCU6:
+		case RGX_CNTBLK_ID_TPU_MCU7:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_USC0:
+		case RGX_CNTBLK_ID_USC1:
+		case RGX_CNTBLK_ID_USC2:
+		case RGX_CNTBLK_ID_USC3:
+		case RGX_CNTBLK_ID_USC4:
+		case RGX_CNTBLK_ID_USC5:
+		case RGX_CNTBLK_ID_USC6:
+		case RGX_CNTBLK_ID_USC7:
+		case RGX_CNTBLK_ID_USC8:
+		case RGX_CNTBLK_ID_USC9:
+		case RGX_CNTBLK_ID_USC10:
+		case RGX_CNTBLK_ID_USC11:
+		case RGX_CNTBLK_ID_USC12:
+		case RGX_CNTBLK_ID_USC13:
+		case RGX_CNTBLK_ID_USC14:
+		case RGX_CNTBLK_ID_USC15:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_TEXAS0:
+		case RGX_CNTBLK_ID_TEXAS1:
+		case RGX_CNTBLK_ID_TEXAS2:
+		case RGX_CNTBLK_ID_TEXAS3:
+		case RGX_CNTBLK_ID_TEXAS4:
+		case RGX_CNTBLK_ID_TEXAS5:
+		case RGX_CNTBLK_ID_TEXAS6:
+		case RGX_CNTBLK_ID_TEXAS7:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_RASTER0:
+		case RGX_CNTBLK_ID_RASTER1:
+		case RGX_CNTBLK_ID_RASTER2:
+		case RGX_CNTBLK_ID_RASTER3:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_BLACKPEARL0:
+		case RGX_CNTBLK_ID_BLACKPEARL1:
+		case RGX_CNTBLK_ID_BLACKPEARL2:
+		case RGX_CNTBLK_ID_BLACKPEARL3:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(RASTER, 3) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_PBE0:
+		case RGX_CNTBLK_ID_PBE1:
+		case RGX_CNTBLK_ID_PBE2:
+		case RGX_CNTBLK_ID_PBE3:
+		case RGX_CNTBLK_ID_PBE4:
+		case RGX_CNTBLK_ID_PBE5:
+		case RGX_CNTBLK_ID_PBE6:
+		case RGX_CNTBLK_ID_PBE7:
+		case RGX_CNTBLK_ID_PBE8:
+		case RGX_CNTBLK_ID_PBE9:
+		case RGX_CNTBLK_ID_PBE10:
+		case RGX_CNTBLK_ID_PBE11:
+		case RGX_CNTBLK_ID_PBE12:
+		case RGX_CNTBLK_ID_PBE13:
+		case RGX_CNTBLK_ID_PBE14:
+		case RGX_CNTBLK_ID_PBE15:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(RASTER, 3) +
+						RGX_CNTBLK_INDIRECT_COUNT(BLACKPEARL, 3) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		default:
+		{
+			ui32Idx = RGX_HWPERF_MAX_DEFINED_BLKS;
+			break;
+		}
+	}
+	if (ui32Idx >= RGX_HWPERF_MAX_DEFINED_BLKS)
+	{
+		return NULL;
+	}
+	return &psHWPerfInitData->sBlkCfg[ui32Idx];
+}
+
+/* Stub routine for rgxfw_hwperf_get_da_block_ctl(). Just return a NULL.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(rgxfw_hwperf_get_da_block_ctl)
+#endif
+static INLINE RGXFWIF_HWPERF_DA_BLK* rgxfw_hwperf_get_da_block_ctl(
+		RGX_HWPERF_CNTBLK_ID eBlockID, RGXFWIF_HWPERF_CTL *psHWPerfInitData)
+{
+	PVR_UNREFERENCED_PARAMETER(eBlockID);
+	PVR_UNREFERENCED_PARAMETER(psHWPerfInitData);
+
+	return NULL;
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_km.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_km.h
new file mode 100644
index 000000000000..22da7ec4cc9d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_km.h
@@ -0,0 +1,1888 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_KM_H)
+#define RGX_FWIF_KM_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+#include "rgxdefs_km.h"
+#include "dllist.h"
+#include "rgx_hwperf.h"
+#include "rgx_mips.h"
+#include "rgxheapconfig.h"
+
+/************************************************************************
+* RGX FW signature checks
+************************************************************************/
+#define RGXFW_SIG_BUFFER_SIZE_MIN       (8192)
+
+#define RGXFWIF_TIMEDIFF_ID			((0x1UL << 28) | RGX_CR_TIMER)
+
+#define RGXFW_POLL_TYPE_SET 0x80000000U
+
+/* Firmware per-DM HWR states */
+#define RGXFWIF_DM_STATE_WORKING					(0x00U)		/*!< DM is working if all flags are cleared */
+#define RGXFWIF_DM_STATE_READY_FOR_HWR				(IMG_UINT32_C(0x1) << 0)	/*!< DM is idle and ready for HWR */
+#define RGXFWIF_DM_STATE_NEEDS_SKIP					(IMG_UINT32_C(0x1) << 2)	/*!< DM need to skip to next cmd before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP			(IMG_UINT32_C(0x1) << 3)	/*!< DM need partial render cleanup before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR			(IMG_UINT32_C(0x1) << 4)	/*!< DM need to increment Recovery Count once fully recovered */
+#define RGXFWIF_DM_STATE_GUILTY_LOCKUP				(IMG_UINT32_C(0x1) << 5)	/*!< DM was identified as locking up and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_LOCKUP			(IMG_UINT32_C(0x1) << 6)	/*!< DM was innocently affected by another lockup which caused HWR */
+#define RGXFWIF_DM_STATE_GUILTY_OVERRUNING			(IMG_UINT32_C(0x1) << 7)	/*!< DM was identified as over-running and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_OVERRUNING		(IMG_UINT32_C(0x1) << 8)	/*!< DM was innocently affected by another DM over-running which caused HWR */
+#define RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH		(IMG_UINT32_C(0x1) << 9)	/*!< DM was forced into HWR as it delayed more important workloads */
+#define RGXFWIF_DM_STATE_GPU_ECC_HWR				(IMG_UINT32_C(0x1) << 10)	/*!< DM was forced into HWR due to an uncorrected GPU ECC error */
+#define RGXFWIF_DM_STATE_GPU_PARITY_HWR				(IMG_UINT32_C(0x1) << 11)	/*!< DM was forced into HWR due to an uncorrected GPU PARITY error */
+#define RGXFWIF_DM_STATE_GPU_LATENT_HWR				(IMG_UINT32_C(0x1) << 12)	/*!< DM was forced into HWR due to an uncorrected GPU LATENT error */
+
+
+/* Firmware's connection state */
+typedef IMG_UINT32 RGXFWIF_CONNECTION_FW_STATE;
+#define RGXFW_CONNECTION_FW_OFFLINE				0U	/*!< Firmware is offline */
+#define RGXFW_CONNECTION_FW_READY				1U	/*!< Firmware is initialised */
+#define RGXFW_CONNECTION_FW_ACTIVE				2U	/*!< Firmware connection is fully established */
+#define RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD	3U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_FORCED_OFFLOAD		4U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_COOLDOWN			5U	/*!< Firmware connection is in cooldown period */
+#define RGXFW_CONNECTION_FW_STATE_COUNT			6U
+
+#define RGXFW_FORCED_OFFLOAD_HCS_DEADLINE_MS	2U		/*!< Workloads of Guest being forcefully offloaded must be stopped quickly */
+#define RGXFW_GRACEFUL_OFFLOAD_HCS_DEADLINE_MS	1000U	/*!< Workloads of Guest being gracefully offloaded are allowed more time to finish */
+
+/* OS' connection state */
+typedef enum
+{
+	RGXFW_CONNECTION_OS_OFFLINE = 0,	/*!< OS is offline */
+	RGXFW_CONNECTION_OS_READY,			/*!< OS's KM driver is setup and waiting */
+	RGXFW_CONNECTION_OS_ACTIVE,			/*!< OS connection is fully established */
+	RGXFW_CONNECTION_OS_STATE_COUNT
+} RGXFWIF_CONNECTION_OS_STATE;
+
+#define RGXFWIF_CTXSWITCH_PROFILE_FAST_EN		(IMG_UINT32_C(0x1))
+#define RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN		(IMG_UINT32_C(0x2))
+#define RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN		(IMG_UINT32_C(0x3))
+#define RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN	(IMG_UINT32_C(0x4))
+
+#define RGXFWIF_ISP_SCHEDMODE_VER1_IPP			(IMG_UINT32_C(0x1))
+#define RGXFWIF_ISP_SCHEDMODE_VER2_ISP			(IMG_UINT32_C(0x2))
+/*!
+ ******************************************************************************
+ * RGX firmware Init Config Data
+ *****************************************************************************/
+
+/* Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_CTXSWITCH_MODE_RAND				(IMG_UINT32_C(0x1) << 0)	/*!< Randomise context switch requests */
+#define RGXFWIF_INICFG_CTXSWITCH_SRESET_EN				(IMG_UINT32_C(0x1) << 1)
+#define RGXFWIF_INICFG_HWPERF_EN						(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN				(IMG_UINT32_C(0x1) << 3)	/*!< Randomise DM-killing requests */
+#define RGXFWIF_INICFG_POW_RASCALDUST					(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_SPU_CLOCK_GATE					(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_FBCDC_V3_1_EN					(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_CHECK_MLIST_EN					(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_DISABLE_CLKGATING_EN				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN	(IMG_UINT32_C(0x1) << 9)
+#define RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN		(IMG_UINT32_C(0x1) << 10)
+/* 11 unused */
+#define RGXFWIF_INICFG_REGCONFIG_EN						(IMG_UINT32_C(0x1) << 12)
+#define RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY			(IMG_UINT32_C(0x1) << 13)
+#define RGXFWIF_INICFG_HWP_DISABLE_FILTER				(IMG_UINT32_C(0x1) << 14)
+/* 15 unused */
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST			(RGXFWIF_CTXSWITCH_PROFILE_FAST_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM			(RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW			(RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY		(RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK			(IMG_UINT32_C(0x7) << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_DISABLE_DM_OVERLAP				(IMG_UINT32_C(0x1) << 19)
+#define RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER			(IMG_UINT32_C(0x1) << 20)
+#define RGXFWIF_INICFG_INJECT_ICS_FAULT					(IMG_UINT32_C(0x1) << 21)
+#define RGXFWIF_INICFG_VALIDATE_IRQ						(IMG_UINT32_C(0x1) << 22)
+#define RGXFWIF_INICFG_DISABLE_PDP_EN					(IMG_UINT32_C(0x1) << 23)
+#define RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN	(IMG_UINT32_C(0x1) << 24)
+#define RGXFWIF_INICFG_WORKEST							(IMG_UINT32_C(0x1) << 25)
+#define RGXFWIF_INICFG_PDVFS							(IMG_UINT32_C(0x1) << 26)
+/* 27 unused */
+/* 28 unused */
+#define RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT				(29)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_NONE				(0)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP			(RGXFWIF_ISP_SCHEDMODE_VER1_IPP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP			(RGXFWIF_ISP_SCHEDMODE_VER2_ISP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_MASK				(RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP |\
+                                                         RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP)
+#define RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER			(IMG_UINT32_C(0x1) << 31)
+
+#define RGXFWIF_INICFG_ALL								(0xFFFFFFFFU)
+
+/* Extended Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT			(0)
+/* [8]   Lossy min channel override
+ * [7]   YUV10 override
+ * [6:4] Quality
+ * [3]   Quality enable
+ * [2:1] Compression scheme
+ * [0]   Lossy group */
+#define RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK			(IMG_UINT32_C(0x1FF)) /* RGX_CR_TFBC_COMPRESSION_CONTROL_MASKFULL */
+#define RGXFWIF_INICFG_EXT_ALL							(RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK)
+
+#define RGXFWIF_INICFG_SYS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_CTXSWITCH_MODE_RAND | \
+														  RGXFWIF_INICFG_CTXSWITCH_SRESET_EN)
+
+/* Flag definitions affecting only workloads submitted by a particular OS */
+
+/*!
+ * @AddToGroup ContextSwitching
+ * @{
+ * @Name Per-OS DM context switch configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN				(IMG_UINT32_C(0x1) << 0) /*!< Enables TDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN				(IMG_UINT32_C(0x1) << 1) /*!< Enables GEOM DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN				(IMG_UINT32_C(0x1) << 2) /*!< Enables FRAG DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN				(IMG_UINT32_C(0x1) << 3) /*!< Enables CDM context switch */
+
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM				(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM				(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D				(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM				(IMG_UINT32_C(0x1) << 7)
+
+#define RGXFWIF_INICFG_OS_ALL							(0xFFU)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL				(RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL)
+
+/*!
+ * @} End of Per-OS Context switch configuration flags
+ * @} End of AddToGroup ContextSwitching
+ */
+
+#define RGXFWIF_FILTCFG_TRUNCATE_HALF					(IMG_UINT32_C(0x1) << 3)
+#define RGXFWIF_FILTCFG_TRUNCATE_INT					(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_FILTCFG_NEW_FILTER_MODE					(IMG_UINT32_C(0x1) << 1)
+
+typedef IMG_UINT32 RGX_ACTIVEPM_CONF;
+#define RGX_ACTIVEPM_FORCE_OFF	0U
+#define RGX_ACTIVEPM_FORCE_ON	1U
+#define RGX_ACTIVEPM_DEFAULT	2U
+
+typedef IMG_UINT32 RGX_RD_POWER_ISLAND_CONF;
+#define RGX_RD_POWER_ISLAND_FORCE_OFF	0U
+#define RGX_RD_POWER_ISLAND_FORCE_ON	1U
+#define RGX_RD_POWER_ISLAND_DEFAULT		2U
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+/* Unused registers re-purposed for storing counters of the Firmware's
+ * interrupts for each OS
+ */
+#if (RGXFW_MAX_NUM_OSIDS == 8)
+#define IRQ_COUNTER_STORAGE_REGS                        \
+		0x2028U, /* RGX_CR_PM_TA_MMU_FSTACK         */  \
+		0x2050U, /* RGX_CR_PM_3D_MMU_FSTACK         */  \
+		0x2030U, /* RGX_CR_PM_START_OF_MMU_TACONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/
+#elif (RGXFW_MAX_NUM_OSIDS == 2)
+#define IRQ_COUNTER_STORAGE_REGS                        \
+		0x2028U, /* RGX_CR_PM_TA_MMU_FSTACK         */  \
+		0x2050U, /* RGX_CR_PM_3D_MMU_FSTACK         */
+#else
+#error Unsupported number of IRQ_COUNTER_STORAGE_REGS registers!
+#endif
+#endif
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SIGBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TRACEBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SYSDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_OSDATA;
+#if defined(SUPPORT_TBI_INTERFACE)
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TBIBUF;
+#endif
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERFBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRINFOBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RUNTIME_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_GPU_UTIL_FW;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_REG_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERF_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_MUX_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_SELECT_CUSTOM_CNTRS;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_RTN_SLOTS;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWCOMMONCONTEXT;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_ZSBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CORE_CLK_RATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COUNTERBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FIRMWAREGCOVBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRTDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TIMESTAMP_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*!
+ * @Brief Buffer to store KM active client contexts
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+} RGXFWIF_ACTIVE_CONTEXT_BUF_DATA;
+#endif
+
+/*!
+ * This number is used to represent an invalid page catalogue physical address
+ */
+#define RGXFWIF_INVALID_PC_PHYADDR 0xFFFFFFFFFFFFFFFFLLU
+
+/*!
+ * This number is used to represent an unallocated set of page catalog base registers
+ */
+#define RGXFW_BIF_INVALID_PCSET 0xFFFFFFFFU
+
+/*!
+ * This number is used to represent an invalid OS ID for the purpose of tracking PC set ownership
+ */
+#define RGXFW_BIF_INVALID_OSID 0xFFFFFFFFU
+
+/*!
+ * Firmware memory context.
+ */
+typedef struct
+{
+	IMG_DEV_PHYADDR			RGXFW_ALIGN sPCDevPAddr;	/*!< device physical address of context's page catalogue */
+	IMG_UINT32				uiPageCatBaseRegSet;	/*!< associated page catalog base register (RGXFW_BIF_INVALID_PCSET == unallocated) */
+	IMG_UINT32				uiBreakpointAddr; /*!< breakpoint address */
+	IMG_UINT32				uiBPHandlerAddr; /*!< breakpoint handler address */
+	IMG_UINT32				uiBreakpointCtl; /*!< DM and enable control for BP */
+	IMG_UINT32				ui32FwMemCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32              ui32OSid;
+	IMG_BOOL                bOSidAxiProt;
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWMEMCONTEXT;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWMEMCONTEXT) == 32,
+				"RGXFWIF_FWMEMCONTEXT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * FW context state flags
+ */
+#define RGXFWIF_CONTEXT_FLAGS_NEED_RESUME				(0x00000001U)
+#define RGXFWIF_CONTEXT_FLAGS_MC_NEED_RESUME_MASKFULL	(0x000000FFU)
+#define RGXFWIF_CONTEXT_FLAGS_TDM_HEADER_STALE			(0x00000100U)
+#define RGXFWIF_CONTEXT_FLAGS_LAST_KICK_SECURE			(0x00000200U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend state (per GEOM core)
+ */
+typedef struct
+{
+	/* FW-accessible TA state which must be written out to memory on context store */
+	IMG_UINT64	RGXFW_ALIGN uTAReg_VDM_CALL_STACK_POINTER;		 /*!< VDM control stream stack pointer, to store in mid-TA */
+	IMG_UINT64	RGXFW_ALIGN uTAReg_VDM_CALL_STACK_POINTER_Init;	 /*!< Initial value of VDM control stream stack pointer (in case is 'lost' due to a lock-up) */
+	IMG_UINT16	ui16TACurrentIdx;
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE_PER_GEOM;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_GEOM_CORE_SIZE 4
+
+static_assert(RGX_NUM_GEOM_CORES <= MAX_GEOM_CORE_SIZE,
+				"RGX_NUM_GEOM_CORES is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend states for all GEOM cores
+ */
+typedef struct
+{
+	/*! FW-accessible TA state which must be written out to memory on context store */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[MAX_GEOM_CORE_SIZE];
+#else
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[RGX_NUM_GEOM_CORES];
+#endif
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_TACTX_STATE) == 160,
+				"RGXFWIF_TACTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware FRAG/3D context suspend state
+ */
+typedef struct
+{
+	/* FW-accessible ISP state which must be written out to memory on context store */
+	IMG_UINT32	u3DReg_PM_DEALLOCATED_MASK_STATUS; /*!< PM deallocation status */
+	IMG_UINT32	u3DReg_PM_PDS_MTILEFREE_STATUS; /*!< Macro-tiles (MTs) finished status */
+	IMG_UINT32	ui32CtxStateFlags;	/*!< Compatibility and other flags */
+	/* au3DReg_ISP_STORE should be the last element of the structure
+	 * as this is an array whose size is determined at runtime
+	 * after detecting the RGX core */
+	IMG_UINT32	au3DReg_ISP_STORE[]; /*!< ISP state (per-pipe) */
+} UNCACHED_ALIGN RGXFWIF_3DCTX_STATE;
+
+static_assert(sizeof(RGXFWIF_3DCTX_STATE) <= 16U,
+              "Size of structure RGXFWIF_3DCTX_STATE exceeds maximum expected size.");
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_3DCTX_STATE) == 16,
+				"RGXFWIF_3DCTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_CTX_USING_BUFFER_A		(0)
+#define RGXFWIF_CTX_USING_BUFFER_B		(1U)
+
+typedef struct
+{
+	IMG_UINT32	ui32CtxStateFlags; /*!< Target buffer and other flags */
+} RGXFWIF_COMPUTECTX_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_COMPUTECTX_STATE) == 4,
+				"RGXFWIF_COMPUTECTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_STATS_PENDING   (1U << 0)
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT (1U << 1)
+
+typedef struct
+{
+	IMG_UINT32	ui32ExtJobRefToDisableZSStore;
+	IMG_BOOL	bDisableZStore;
+	IMG_BOOL	bDisableSStore;
+} RGXFWIF_DISABLE_ZSSTORE;
+
+#define MAX_ZSSTORE_DISABLE 8
+
+typedef struct
+{
+	bool       bSaved;
+	IMG_UINT64 ui64CheckSum[4];
+}RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY;
+
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_TQ[RGX_TRP_MAX_NUM_CORES][1];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_2D[RGX_TRP_MAX_NUM_CORES][2];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_3D[RGX_TRP_MAX_NUM_CORES][4];
+typedef RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY RGXFWIF_TRP_CHECKSUM_GEOM[RGX_TRP_MAX_NUM_CORES];
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware render context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTAContext;				/*!< Firmware context for the TA */
+	RGXFWIF_FWCOMMONCONTEXT	s3DContext;				/*!< Firmware context for the 3D */
+
+	RGXFWIF_STATIC_RENDERCONTEXT_STATE sStaticRenderContextState;
+
+	RGXFWIF_DISABLE_ZSSTORE sDisableZSStoreQueue[MAX_ZSSTORE_DISABLE];
+
+	IMG_UINT32			ui32ZSStoreQueueCount;
+	IMG_UINT32			ui32WriteOffsetOfDisableZSStore;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32			ui32FwRenderCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_3D		aui64TRPChecksums3D;	/*!< Used by Firmware to store checksums during 3D WRR */
+	RGXFWIF_TRP_CHECKSUM_GEOM	aui64TRPChecksumsGeom;	/*!< Used by Firmware to store checksums during TA WRR */
+	RGXFWIF_DM			eTRPGeomCoreAffinity; /* !< Represent the DM affinity for pending 2nd TRP pass of GEOM otherwise points RGXFWIF_DM_MAX. */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWRENDERCONTEXT;
+
+/*!
+	Firmware compute context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sCDMContext;				/*!< Firmware context for the CDM */
+
+	RGXFWIF_STATIC_COMPUTECONTEXT_STATE sStaticComputeContextState;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32 ui32ComputeCtxFlags; /*!< Compatibility and other flags */
+
+	IMG_UINT32		aui32WGPChecksum[RGX_WGP_MAX_NUM_CORES];
+} UNCACHED_ALIGN RGXFWIF_FWCOMPUTECONTEXT;
+
+/*!
+	Firmware TDM context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTDMContext;				/*!< Firmware context for the TDM */
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+} UNCACHED_ALIGN RGXFWIF_FWTDMCONTEXT;
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware transfer context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTQContext;			 /*!< Firmware context for TQ3D */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_TQ aui64TRPChecksumsTQ;/*!< Used by Firmware to store checksums during TQ WRR */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWTRANSFERCONTEXT;
+
+/*!
+ ******************************************************************************
+ * Defines for CMD_TYPE corruption detection and forward compatibility check
+ *****************************************************************************/
+
+/* CMD_TYPE 32bit contains:
+ * 31:16	Reserved for magic value to detect corruption (16 bits)
+ * 15		Reserved for RGX_CCB_TYPE_TASK (1 bit)
+ * 14:0		Bits available for CMD_TYPEs (15 bits) */
+
+
+/* Magic value to detect corruption */
+#define RGX_CMD_MAGIC_DWORD			IMG_UINT32_C(0x2ABC)
+#define RGX_CMD_MAGIC_DWORD_MASK	(0xFFFF0000U)
+#define RGX_CMD_MAGIC_DWORD_SHIFT	(16U)
+#define RGX_CMD_MAGIC_DWORD_SHIFTED	(RGX_CMD_MAGIC_DWORD << RGX_CMD_MAGIC_DWORD_SHIFT)
+
+/*!
+ * @InGroup KCCBTypes ClientCCBTypes
+ * @Brief Generic CCB control structure
+ */
+typedef struct
+{
+	volatile IMG_UINT32		ui32WriteOffset;		/*!< write offset into array of commands (MUST be aligned to 16 bytes!) */
+	volatile IMG_UINT32		ui32ReadOffset;			/*!< read offset into array of commands */
+	IMG_UINT32				ui32WrapMask;			/*!< Offset wrapping mask (Total capacity of the CCB - 1) */
+} UNCACHED_ALIGN RGXFWIF_CCB_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCB_CTL) == 16,
+				"RGXFWIF_CCB_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @Defgroup KCCBTypes Kernel CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the KCCB functionality
+ * @{
+ */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PT      (0x1U) /* MMU_CTRL_INVAL_PT_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PD      (0x2U) /* MMU_CTRL_INVAL_PD_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PC      (0x4U) /* MMU_CTRL_INVAL_PC_EN */
+
+#if !defined(__KERNEL__)
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB   (0x10U) /* can't use PM_TLB0 bit from BIFPM_CTRL reg because it collides with PT bit from BIF_CTRL reg */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB | 0x8U) /* BIF_CTRL_INVAL_TLB1_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x0U) /* not used */
+
+#else
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB   (0x10U) /* can't use PM_TLB0 bit from BIFPM_CTRL reg because it collides with PT bit from BIF_CTRL reg */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB | 0x8U) /* BIF_CTRL_INVAL_TLB1_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x800U) /* MMU_CTRL_INVAL_ALL_CONTEXTS_EN */
+#endif
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT (0x4000000U) /* indicates FW should interrupt the host */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_MMUCACHE type command
+ */
+typedef struct
+{
+	IMG_UINT32            ui32CacheFlags;
+	RGXFWIF_DEV_VIRTADDR  sMMUCacheSync;
+	IMG_UINT32            ui32MMUCacheSyncUpdateValue;
+} RGXFWIF_MMUCACHEDATA;
+
+#define RGXFWIF_BPDATA_FLAGS_ENABLE (1U << 0)
+#define RGXFWIF_BPDATA_FLAGS_WRITE  (1U << 1)
+#define RGXFWIF_BPDATA_FLAGS_CTL    (1U << 2)
+#define RGXFWIF_BPDATA_FLAGS_REGS   (1U << 3)
+
+typedef struct
+{
+	PRGXFWIF_FWMEMCONTEXT		psFWMemContext;		/*!< Memory context */
+	IMG_UINT32					ui32BPAddr;			/*!< Breakpoint address */
+	IMG_UINT32					ui32HandlerAddr;	/*!< Breakpoint handler */
+	IMG_UINT32					ui32BPDM;			/*!< Breakpoint control */
+	IMG_UINT32					ui32BPDataFlags;
+	IMG_UINT32					ui32TempRegs;		/*!< Number of temporary registers to overallocate */
+	IMG_UINT32					ui32SharedRegs;		/*!< Number of shared registers to overallocate */
+	IMG_UINT64 RGXFW_ALIGN		ui64SpillAddr;
+	RGXFWIF_DM					eDM;				/*!< DM associated with the breakpoint */
+} RGXFWIF_BPDATA;
+
+#define RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS (RGXFWIF_PRBUFFER_MAXSUPPORTED + 1U) /* +1 is RTDATASET cleanup */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_KICK type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CWoffUpdate;	/*!< Client CCB woff update */
+	IMG_UINT32					ui32CWrapMaskUpdate; /*!< Client CCB wrap mask update after CCCB growth */
+	IMG_UINT32					ui32NumCleanupCtl;		/*!< number of CleanupCtl pointers attached */
+	PRGXFWIF_CLEANUP_CTL		apsCleanupCtl[RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS]; /*!< CleanupCtl structures associated with command */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32					ui32WorkEstCmdHeaderOffset; /*!< offset to the CmdHeader which houses the workload estimation kick data. */
+#endif
+} RGXFWIF_KCCB_CMD_KICK_DATA;
+
+/*!
+ * @Brief Command data for @Ref RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK type command
+ */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_KICK_DATA	sTACmdKickData; /*!< GEOM DM kick command data */
+	RGXFWIF_KCCB_CMD_KICK_DATA	s3DCmdKickData; /*!< FRAG DM kick command data */
+} RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FORCE_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CCBFenceOffset;	/*!< Client CCB fence offset */
+} RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;
+	RGXFWIF_DISABLE_ZSSTORE		sDisableZSStore;
+} RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA;
+
+/*!
+ * @Brief Resource types supported by \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef enum
+{
+	RGXFWIF_CLEANUP_FWCOMMONCONTEXT,		/*!< FW common context cleanup */
+	RGXFWIF_CLEANUP_HWRTDATA,				/*!< FW HW RT data cleanup */
+	RGXFWIF_CLEANUP_FREELIST,				/*!< FW freelist cleanup */
+	RGXFWIF_CLEANUP_ZSBUFFER,				/*!< FW ZS Buffer cleanup */
+} RGXFWIF_CLEANUP_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef struct
+{
+	RGXFWIF_CLEANUP_TYPE			eCleanupType;			/*!< Cleanup type */
+	union {
+		PRGXFWIF_FWCOMMONCONTEXT	psContext;				/*!< FW common context to cleanup */
+		PRGXFWIF_HWRTDATA			psHWRTData;				/*!< HW RT to cleanup */
+		PRGXFWIF_FREELIST			psFreelist;				/*!< Freelist to cleanup */
+		PRGXFWIF_ZSBUFFER			psZSBuffer;				/*!< ZS Buffer to cleanup */
+	} uCleanupData;
+} RGXFWIF_CLEANUP_REQUEST;
+
+/*!
+ * @Brief Type of power requests supported in \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef enum
+{
+	RGXFWIF_POW_OFF_REQ = 1,           /*!< GPU power-off request */
+	RGXFWIF_POW_FORCED_IDLE_REQ,       /*!< Force-idle related request */
+	RGXFWIF_POW_NUM_UNITS_CHANGE,      /*!< Request to change default powered scalable units */
+	RGXFWIF_POW_APM_LATENCY_CHANGE     /*!< Request to change the APM latency period */
+} RGXFWIF_POWER_TYPE;
+
+/*!
+ * @Brief Supported force-idle related requests with \ref RGXFWIF_POW_FORCED_IDLE_REQ type request
+ */
+typedef enum
+{
+	RGXFWIF_POWER_FORCE_IDLE = 1,      /*!< Request to force-idle GPU */
+	RGXFWIF_POWER_CANCEL_FORCED_IDLE,  /*!< Request to cancel a previously successful force-idle transition */
+	RGXFWIF_POWER_HOST_TIMEOUT,        /*!< Notification that host timed-out waiting for force-idle state */
+} RGXFWIF_POWER_FORCE_IDLE_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef struct
+{
+	RGXFWIF_POWER_TYPE					ePowType;					/*!< Type of power request */
+	union
+	{
+		IMG_UINT32						ui32PowUnits;			/*!< Number of active Dusts */
+		IMG_BOOL						bForced;				/*!< If the operation is mandatory */
+		RGXFWIF_POWER_FORCE_IDLE_TYPE	ePowRequestType;		/*!< Type of Request. Consolidating Force Idle, Cancel Forced Idle, Host Timeout */
+	} uPowerReqData;
+} RGXFWIF_POWER_REQUEST;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_SLCFLUSHINVAL type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext; /*!< Context to fence on (only useful when bDMContext == TRUE) */
+	IMG_BOOL    bInval;                 /*!< Invalidate the cache as well as flushing */
+	IMG_BOOL    bDMContext;             /*!< The data to flush/invalidate belongs to a specific DM context */
+	IMG_UINT64	RGXFW_ALIGN ui64Address;	/*!< Optional address of range (only useful when bDMContext == FALSE) */
+	IMG_UINT64	RGXFW_ALIGN ui64Size;		/*!< Optional size of range (only useful when bDMContext == FALSE) */
+} RGXFWIF_SLCFLUSHINVALDATA;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CTRL_TOGGLE = 0,
+	RGXFWIF_HWPERF_CTRL_SET    = 1,
+	RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV = 2
+} RGXFWIF_HWPERF_UPDATE_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG type command
+ */
+typedef struct
+{
+	RGXFWIF_HWPERF_UPDATE_CONFIG eOpCode; /*!< Control operation code */
+	IMG_UINT64	RGXFW_ALIGN	ui64Mask;   /*!< Mask of events to toggle */
+} RGXFWIF_HWPERF_CTRL;
+
+typedef struct
+{
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_MUX_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_MUX_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_MUX_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS type command
+ */
+typedef struct
+{
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_DA_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE type command
+ */
+typedef struct
+{
+	IMG_UINT32	ui32NewClockSpeed;			/*!< New clock speed */
+} RGXFWIF_CORECLKSPEEDCHANGE_DATA;
+
+#define RGXFWIF_HWPERF_CTRL_BLKS_MAX	16U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS type command
+ */
+typedef struct
+{
+	bool		bEnable;
+	IMG_UINT32	ui32NumBlocks;                              /*!< Number of block IDs in the array */
+	IMG_UINT16	aeBlockIDs[RGXFWIF_HWPERF_CTRL_BLKS_MAX];   /*!< Array of RGX_HWPERF_CNTBLK_ID values */
+} RGXFWIF_HWPERF_CTRL_BLKS;
+
+
+typedef struct
+{
+	IMG_UINT16                      ui16CustomBlock;
+	IMG_UINT16                      ui16NumCounters;
+	PRGX_HWPERF_SELECT_CUSTOM_CNTRS sCustomCounterIDs;
+} RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE & \ref RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE type commands
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;				/*!< ZS-Buffer FW address */
+	IMG_BOOL				bDone;								/*!< action backing/unbacking succeeded */
+} RGXFWIF_ZSBUFFER_BACKING_DATA;
+
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE type command
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;				/*!< Freelist FW address */
+	IMG_UINT32				ui32DeltaPages;						/*!< Amount of the Freelist change */
+	IMG_UINT32				ui32NewPages;						/*!< New amount of pages on the freelist (including ready pages) */
+	IMG_UINT32              ui32ReadyPages;                     /*!< Number of ready pages to be held in reserve until OOM */
+} RGXFWIF_FREELIST_GS_DATA;
+
+#define RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT         (MAX_HW_TA3DCONTEXTS * RGXFW_MAX_FREELISTS * 2U)
+#define RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG 0x80000000U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE type command
+ */
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+} RGXFWIF_FREELISTS_RECONSTRUCTION_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT  psContext; /*!< Context to that may need to be resumed following write offset update */
+} UNCACHED_ALIGN RGXFWIF_WRITE_OFFSET_UPDATE_DATA;
+
+/*!
+ ******************************************************************************
+ * Proactive DVFS Structures
+ *****************************************************************************/
+#define NUM_OPP_LEVELS 16
+
+typedef struct
+{
+	IMG_UINT32			ui32Volt; /* V  */
+	IMG_UINT32			ui32Freq; /* Hz */
+} UNCACHED_ALIGN OPP_LEVEL;
+
+typedef struct
+{
+	OPP_LEVEL		asOPPValues[NUM_OPP_LEVELS];
+#if defined(DEBUG)
+	IMG_UINT32		ui32MinOPPPoint;
+#endif
+	IMG_UINT32		ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_OPP_INFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MAX_FREQ_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32MinOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MIN_FREQ_DATA;
+
+/*!
+ ******************************************************************************
+ * Register configuration structures
+ *****************************************************************************/
+
+#define RGXFWIF_REG_CFG_MAX_SIZE 512
+
+typedef enum
+{
+	RGXFWIF_REGCFG_CMD_ADD				= 101,
+	RGXFWIF_REGCFG_CMD_CLEAR			= 102,
+	RGXFWIF_REGCFG_CMD_ENABLE			= 103,
+	RGXFWIF_REGCFG_CMD_DISABLE			= 104
+} RGXFWIF_REGDATA_CMD_TYPE;
+
+typedef struct
+{
+	IMG_UINT64		ui64Addr;
+	IMG_UINT64		ui64Mask;
+	IMG_UINT64		ui64Value;
+} RGXFWIF_REG_CFG_REC;
+
+typedef struct
+{
+	RGXFWIF_REGDATA_CMD_TYPE         eCmdType;
+	RGXFWIF_REG_CFG_TYPE             eRegConfigType;
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  sRegConfig;
+
+} RGXFWIF_REGCONFIG_DATA;
+
+typedef struct
+{
+	/**
+	 * PDump WRW command write granularity is 32 bits.
+	 * Add padding to ensure array size is 32 bit granular.
+	 */
+	IMG_UINT8           RGXFW_ALIGN  aui8NumRegsType[PVR_ALIGN((IMG_UINT32)RGXFWIF_REG_CFG_TYPE_ALL,sizeof(IMG_UINT32))];
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  asRegConfigs[RGXFWIF_REG_CFG_MAX_SIZE];
+} UNCACHED_ALIGN RGXFWIF_REG_CFG;
+
+typedef enum
+{
+	RGXFWIF_OS_ONLINE = 1,
+	RGXFWIF_OS_OFFLINE
+} RGXFWIF_OS_STATE_CHANGE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE type command
+ */
+typedef struct
+{
+	IMG_UINT32 ui32DriverID;
+	RGXFWIF_OS_STATE_CHANGE eNewOSState;
+} UNCACHED_ALIGN RGXFWIF_OS_STATE_CHANGE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext;
+	IMG_UINT32               ui32FirstIntJobRefToCancel;
+	IMG_UINT32               ui32LastIntJobRefToCancel;
+} UNCACHED_ALIGN RGXFWIF_CANCEL_WORK_DATA;
+
+typedef enum
+{
+	RGXFWIF_PWR_COUNTER_DUMP_START = 1,
+	RGXFWIF_PWR_COUNTER_DUMP_STOP,
+	RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+} RGXFWIF_COUNTER_DUMP_REQUEST;
+
+typedef struct
+{
+	RGXFWIF_COUNTER_DUMP_REQUEST eCounterDumpRequest;
+}  RGXFW_ALIGN RGXFWIF_COUNTER_DUMP_DATA;
+
+/*!
+ * @Brief List of command types supported by the Kernel CCB
+ */
+typedef enum
+{
+	/* Common commands */
+	RGXFWIF_KCCB_CMD_KICK								= 101U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< DM workload kick command */
+	RGXFWIF_KCCB_CMD_MMUCACHE							= 102U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< MMU cache invalidation request */
+	RGXFWIF_KCCB_CMD_BP									= 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_SLCFLUSHINVAL						= 104U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< SLC flush and invalidation request */
+	RGXFWIF_KCCB_CMD_CLEANUP							= 105U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Requests cleanup of a FW resource (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE			= 106U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Backing for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE			= 107U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Unbacking for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE				= 108U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelist Grow done */
+	RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE	= 109U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelists Reconstruction done */
+	RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE			= 110U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the firmware that the host has added more data to a CDM2 Circular Buffer */
+	RGXFWIF_KCCB_CMD_HEALTH_CHECK						= 111U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Health check request */
+	RGXFWIF_KCCB_CMD_FORCE_UPDATE						= 112U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Forcing signalling of all unmet UFOs for a given CCB offset */
+	RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK				= 113U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< There is a TA and a 3D command in this single kick */
+	RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE			= 114U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW that a Guest OS has come online / offline. */
+	RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE					= 115U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW to disable zs store of a running 3D or add it to queue of render context. */
+	RGXFWIF_KCCB_CMD_CANCEL_WORK						= 116U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Cancel all work up to and including a given intjobref for a given context */
+	/* Commands only permitted to the native or host OS */
+	RGXFWIF_KCCB_CMD_POW								= 200U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Power request */
+	RGXFWIF_KCCB_CMD_REGCONFIG							= 201U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE					= 202U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Core clock speed change event */
+	RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE						= 203U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Ask the firmware to update its cached ui32LogType value from the (shared) tracebuf control structure */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ				= 204U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a maximum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE				= 205U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the priority/group for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL					= 206U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set or clear firmware state flags */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ				= 207U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a minimum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_PHR_CFG							= 208U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Periodic Hardware Reset behaviour */
+	RGXFWIF_KCCB_CMD_WDG_CFG							= 210U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Safety Firmware Watchdog */
+	RGXFWIF_KCCB_CMD_COUNTER_DUMP						= 211U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Controls counter dumping in the FW */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE					= 213U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL			= 214U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice interval for all drivers. It can only be serviced for the Host DDK */
+
+	/* HWPerf commands */
+	RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG				= 300U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure HWPerf events (to be generated) and HWPerf buffer address (if required) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS					= 301U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure directly addressable counters for HWPerf */
+	RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS					= 302U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Enable or disable multiple HWPerf blocks (reusing existing configuration) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS			= 303U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks */
+	RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS			= 304U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure the custom counters for HWPerf */
+
+} RGXFWIF_KCCB_CMD_TYPE;
+
+#define RGXFWIF_LAST_ALLOWED_GUEST_KCCB_CMD (RGXFWIF_KCCB_CMD_POW - 1)
+
+/*! @Brief Kernel CCB command packet */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_TYPE  eCmdType;      /*!< Command type */
+	IMG_UINT32             ui32KCCBFlags; /*!< Compatibility and other flags */
+
+	/* NOTE: Make sure that uCmdData is the last member of this struct
+	 * This is to calculate actual command size for device mem copy.
+	 * (Refer RGXGetCmdMemCopySize())
+	 * */
+	union
+	{
+		RGXFWIF_KCCB_CMD_KICK_DATA			sCmdKickData;			/*!< Data for Kick command */
+		RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA	sCombinedTA3DCmdKickData;	/*!< Data for combined TA/3D Kick command */
+		RGXFWIF_MMUCACHEDATA				sMMUCacheData;			/*!< Data for MMU cache command */
+		RGXFWIF_BPDATA						sBPData;				/*!< Data for Breakpoint Commands */
+		RGXFWIF_SLCFLUSHINVALDATA			sSLCFlushInvalData;		/*!< Data for SLC Flush/Inval commands */
+		RGXFWIF_CLEANUP_REQUEST				sCleanupData;			/*!< Data for cleanup commands */
+		RGXFWIF_POWER_REQUEST				sPowData;				/*!< Data for power request commands */
+		RGXFWIF_HWPERF_CTRL					sHWPerfCtrl;			/*!< Data for HWPerf control command */
+		RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS	sHWPerfCfgEnableBlks;	/*!< Data for HWPerf configure, clear and enable performance counter block command */
+		RGXFWIF_HWPERF_CTRL_BLKS			sHWPerfCtrlBlks;		/*!< Data for HWPerf enable or disable performance counter block commands */
+		RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS	sHWPerfSelectCstmCntrs;	/*!< Data for HWPerf configure the custom counters to read */
+		RGXFWIF_HWPERF_CONFIG_DA_BLKS		sHWPerfCfgDABlks;		/*!< Data for HWPerf configure Directly Addressable blocks */
+		RGXFWIF_CORECLKSPEEDCHANGE_DATA		sCoreClkSpeedChangeData;/*!< Data for core clock speed change */
+		RGXFWIF_ZSBUFFER_BACKING_DATA		sZSBufferBackingData;	/*!< Feedback for Z/S Buffer backing/unbacking */
+		RGXFWIF_FREELIST_GS_DATA			sFreeListGSData;		/*!< Feedback for Freelist grow/shrink */
+		RGXFWIF_FREELISTS_RECONSTRUCTION_DATA	sFreeListsReconstructionData;	/*!< Feedback for Freelists reconstruction */
+		RGXFWIF_REGCONFIG_DATA				sRegConfigData;			/*!< Data for custom register configuration */
+		RGXFWIF_WRITE_OFFSET_UPDATE_DATA    sWriteOffsetUpdateData; /*!< Data for informing the FW about the write offset update */
+		RGXFWIF_PDVFS_MAX_FREQ_DATA			sPDVFSMaxFreqData;		/*!< Data for setting the max frequency/OPP */
+		RGXFWIF_PDVFS_MIN_FREQ_DATA			sPDVFSMinFreqData;		/*!< Data for setting the min frequency/OPP */
+		RGXFWIF_OS_STATE_CHANGE_DATA        sCmdOSOnlineStateData;  /*!< Data for updating the Guest Online states */
+		RGXFWIF_DEV_VIRTADDR                sTBIBuffer;             /*!< Dev address for TBI buffer allocated on demand */
+		RGXFWIF_COUNTER_DUMP_DATA			sCounterDumpConfigData; /*!< Data for dumping of register ranges */
+		RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA	sDisableZSStoreData;	/*!< Data for disabling zs store of a 3D workload */
+		RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA  sForceUpdateData;       /*!< Data for signalling all unmet fences for a given CCB */
+		RGXFWIF_CANCEL_WORK_DATA			sCancelWorkData;		/*!< Data for cancelling work */
+	} UNCACHED_ALIGN uCmdData;
+} UNCACHED_ALIGN RGXFWIF_KCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_KCCB_CMD);
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_KCCB_CMD) == 64,
+				"RGXFWIF_KCCB_CMD is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of KCCBTypes */
+
+/*!
+ * @Defgroup FWCCBTypes Firmware CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the Firmware CCB functionality
+ * @{
+ */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING and the
+ * \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING Firmware CCB commands
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32ZSBufferID; /*!< ZS buffer ID */
+} RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELIST_GROW Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32FreelistID; /*!< Freelist ID */
+} RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;                                     /*!< Freelists count */
+	IMG_UINT32			ui32HwrCounter;                                         /*!< HWR counter */
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT]; /*!< Array of freelist IDs to reconstruct */
+} RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA;
+
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF			(1U<<0)	/*!< 1 if a page fault happened */
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS	(1U<<1)	/*!< 1 if applicable to all contexts */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32						ui32ServerCommonContextID;	/*!< Context affected by the reset */
+	RGX_CONTEXT_RESET_REASON		eResetReason;				/*!< Reason for reset */
+	RGXFWIF_DM						eDM;						/*!< Data Master affected by the reset */
+	IMG_UINT32						ui32ResetJobRef;			/*!< Job ref running at the time of reset */
+	IMG_UINT32						ui32Flags;					/*!< RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG bitfield */
+	IMG_UINT64 RGXFW_ALIGN			ui64PCAddress;				/*!< At what page catalog address */
+	IMG_DEV_VIRTADDR RGXFW_ALIGN	sFaultAddress;				/*!< Page fault address (only when applicable) */
+} RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;	/*!< Page fault address */
+} RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA;
+
+/*!
+ ******************************************************************************
+ * List of command types supported by the Firmware CCB
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING              = 101U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be backed with physical pages
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING            = 102U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be unbacked
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_FREELIST_GROW                 = 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand freelist grow
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA */
+	RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION      = 104U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests freelists reconstruction
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA */
+	RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION    = 105U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a HWR event on a context
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA */
+	RGXFWIF_FWCCB_CMD_DEBUG_DUMP                    = 106U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand debug dump
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS                  = 107U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand update on process stats
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA */
+
+	RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE          = 108U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART           = 109U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests GPU restart
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION    = 112U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a FW pagefault
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA */
+} RGXFWIF_FWCCB_CMD_TYPE;
+
+/*!
+ ******************************************************************************
+ * List of the various stats of the process to update/increment
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS=1,		/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumPartialRenders stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumOutOfMemory stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTAStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32Num3DStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumCDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTDMStores stat */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_UPDATE_STATS Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE		eElementToUpdate;			/*!< Element to update */
+	IMG_PID									pidOwner;					/*!< The pid of the process whose stats are being updated */
+	IMG_INT32								i32AdjustmentValue;			/*!< Adjustment to be made to the statistic */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32CoreClkRate;
+} UNCACHED_ALIGN RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA;
+
+
+/*!
+ ******************************************************************************
+ * @Brief Firmware CCB command structure
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_TYPE  eCmdType;       /*!< Command type */
+	IMG_UINT32              ui32FWCCBFlags; /*!< Compatibility and other flags */
+
+	union
+	{
+		RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA				sCmdZSBufferBacking;			/*!< Data for Z/S-Buffer on-demand (un)backing*/
+		RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA					sCmdFreeListGS;					/*!< Data for on-demand freelist grow/shrink */
+		RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA		sCmdFreeListsReconstruction;	/*!< Data for freelists reconstruction */
+		RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA				sCmdContextResetNotification;	/*!< Data for context reset notification */
+		RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA					sCmdUpdateStatsData;			/*!< Data for updating process stats */
+		RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA			sCmdCoreClkRateChange;
+		RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA					sCmdFWPagefault;				/*!< Data for context reset notification */
+	} RGXFW_ALIGN uCmdData;
+} RGXFW_ALIGN RGXFWIF_FWCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_FWCCB_CMD);
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWCCB_CMD) == 48,
+				"RGXFWIF_FWCCB_CMD is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of FWCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Workload estimation Firmware CCB command structure for RGX
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT16 ui16ReturnDataIndex; /*!< Index for return data array */
+	IMG_UINT16 ui16CyclesTakenHigh;	/*!< Not used on Rogue GPU cores. */
+	IMG_UINT32 ui32CyclesTaken;     /*!< The cycles the workload took on the hardware */
+} RGXFWIF_WORKEST_FWCCB_CMD;
+
+/*!
+ * @Defgroup ClientCCBTypes Client CCB data interface
+ * @Brief Types grouping data structures and defines used in realising Client CCB commands/functionality
+ * @{
+ */
+
+/* Required memory alignment for 64-bit variables accessible by Meta
+  (The gcc meta aligns 64-bit variables to 64-bit; therefore, memory shared
+   between the host and meta that contains 64-bit variables has to maintain
+   this alignment) */
+#define RGXFWIF_FWALLOC_ALIGN	sizeof(IMG_UINT64)
+
+#define RGX_CCB_TYPE_TASK			(IMG_UINT32_C(1) << 15)
+#define RGX_CCB_FWALLOC_ALIGN(size)	(PVR_ALIGN(size, RGXFWIF_FWALLOC_ALIGN))
+
+typedef IMG_UINT32 RGXFWIF_CCB_CMD_TYPE;
+
+/*!
+ * @Name Client CCB command types
+ * @{
+ */
+#define RGXFWIF_CCB_CMD_TYPE_GEOM			(201U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TA DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_3D			(202U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_3D				(203U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command */
+#define RGXFWIF_CCB_CMD_TYPE_3D_PR			(204U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for Partial render */
+#define RGXFWIF_CCB_CMD_TYPE_CDM			(205U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Compute DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_TDM			(206U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TDM command */
+#define RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE (207U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_TQ_2D			(208U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 2D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP	(209U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_NULL			(210U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_ABORT			(211U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+
+/* Leave a gap between CCB specific commands and generic commands */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE          (212U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a command */
+#define RGXFWIF_CCB_CMD_TYPE_UPDATE         (213U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates of a command */
+#define RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE     (214U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates related to workload resources */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE_PR       (215U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a PR command */
+#define RGXFWIF_CCB_CMD_TYPE_PRIORITY       (216U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Context priority update command */
+/* Pre and Post timestamp commands are supposed to sandwich the DM cmd. The
+   padding code with the CCB wrap upsets the FW if we don't have the task type
+   bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+*/
+#define RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP (217U | RGX_CMD_MAGIC_DWORD_SHIFTED)
+/* UNFENCED type is not supported but keeping defines as it is for now */
+
+
+#define RGXFWIF_CCB_CMD_TYPE_PADDING	(221U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Skip without action type command */
+#define RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP	(223U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Process a vulkan timestamp */
+/*! @} End of Client CCB command types */
+
+
+#define RGXFWIF_TRP_STATUS_UNKNOWN				0x000U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_OK			0x001U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_ERROR		0x002U
+
+#define RGXFWIF_CR_TRP_SIGNATURE_STATUS			(RGX_CR_SCRATCH10)
+
+
+typedef struct
+{
+	/* Index for the KM Workload estimation return data array */
+	IMG_UINT16 RGXFW_ALIGN         ui16ReturnDataIndex;
+	/* Predicted time taken to do the work in cycles */
+	IMG_UINT32 RGXFW_ALIGN         ui32CyclesPrediction;
+	/* Deadline for the workload (in usecs) */
+	IMG_UINT64 RGXFW_ALIGN         ui64Deadline;
+} RGXFWIF_WORKEST_KICK_DATA;
+
+/*! @Brief Command header of a command in the client CCB buffer.
+ *
+ *  Followed by this header is the command-data specific to the
+ *  command-type as specified in the header.
+ */
+typedef struct
+{
+	RGXFWIF_CCB_CMD_TYPE					eCmdType;      /*!< Command data type following this command header */
+	IMG_UINT32								ui32CmdSize;   /*!< Size of the command following this header */
+	IMG_UINT32								ui32ExtJobRef; /*!< external job reference - provided by client and used in debug for tracking submitted work */
+	IMG_UINT32								ui32IntJobRef; /*!< internal job reference - generated by services and used in debug for tracking submitted work */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_WORKEST_KICK_DATA RGXFW_ALIGN	sWorkEstKickData; /*!< Workload Estimation - Workload Estimation Data */
+#endif
+} RGXFWIF_CCB_CMD_HEADER;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCB_CMD_HEADER) == 16,
+				"RGXFWIF_CCB_CMD_HEADER is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*
+ ******************************************************************************
+ * Client CCB commands which are only required by the kernel
+ *****************************************************************************/
+
+/*! @Brief Command data for \ref RGXFWIF_CCB_CMD_TYPE_PRIORITY type client CCB command */
+typedef struct
+{
+	IMG_INT32              i32Priority; /*!< Priority level */
+} RGXFWIF_CMD_PRIORITY;
+
+/*! @} End of ClientCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Signature and Checksums Buffer
+ *****************************************************************************/
+typedef struct
+{
+	PRGXFWIF_SIGBUFFER		sBuffer;			/*!< Ptr to Signature Buffer memory */
+	IMG_UINT32				ui32LeftSizeInRegs;	/*!< Amount of space left for storing regs in the buffer */
+} UNCACHED_ALIGN RGXFWIF_SIGBUF_CTL;
+
+typedef struct
+{
+	PRGXFWIF_COUNTERBUFFER	sBuffer;			/*!< Ptr to counter dump buffer */
+	IMG_UINT32				ui32SizeInDwords;	/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_COUNTER_DUMP_CTL;
+
+typedef struct
+{
+	PRGXFWIF_FIRMWAREGCOVBUFFER	sBuffer;		/*!< Ptr to firmware gcov buffer */
+	IMG_UINT32					ui32Size;		/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_FIRMWARE_GCOV_CTL;
+
+/*!
+ *****************************************************************************
+ * RGX Compatibility checks
+ *****************************************************************************/
+
+/* WARNING: Whenever the layout of RGXFWIF_COMPCHECKS_BVNC changes, the
+	following define should be increased by 1 to indicate to the
+	compatibility logic that layout has changed. */
+#define RGXFWIF_COMPCHECKS_LAYOUT_VERSION 3
+
+typedef struct
+{
+	IMG_UINT32	ui32LayoutVersion; /* WARNING: This field must be defined as first one in this structure */
+	IMG_UINT64	RGXFW_ALIGN ui64BVNC;
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS_BVNC;
+
+typedef struct
+{
+	IMG_UINT8	ui8OsCountSupport;
+} UNCACHED_ALIGN RGXFWIF_INIT_OPTIONS;
+
+#define RGXFWIF_COMPCHECKS_BVNC_DECLARE_AND_INIT(name) \
+	RGXFWIF_COMPCHECKS_BVNC (name) = { \
+		RGXFWIF_COMPCHECKS_LAYOUT_VERSION, \
+		0, \
+	}
+#define RGXFWIF_COMPCHECKS_BVNC_INIT(name) \
+	do { \
+		(name).ui32LayoutVersion = RGXFWIF_COMPCHECKS_LAYOUT_VERSION; \
+		(name).ui64BVNC = 0; \
+	} while (false)
+
+typedef struct
+{
+	RGXFWIF_COMPCHECKS_BVNC		sHWBVNC;				/*!< hardware BVNC (from the RGX registers) */
+	RGXFWIF_COMPCHECKS_BVNC		sFWBVNC;				/*!< firmware BVNC */
+	IMG_UINT32					ui32DDKVersion;			/*!< software DDK version */
+	IMG_UINT32					ui32DDKBuild;			/*!< software DDK build no. */
+	IMG_UINT32					ui32BuildOptions;		/*!< build options bit-field */
+	RGXFWIF_INIT_OPTIONS		sInitOptions;			/*!< initialisation options bit-field */
+	IMG_BOOL					bUpdated;				/*!< Information is valid */
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS;
+
+typedef struct
+{
+	IMG_UINT32                 ui32NumCores;
+	IMG_UINT64 RGXFW_ALIGN     aui64MultiCoreCapabilities[RGX_MULTICORE_MAX_NOHW_CORES];
+} UNCACHED_ALIGN RGXFWIF_MULTICORE_INFO;
+
+/*! @Brief Firmware Runtime configuration data \ref RGXFWIF_RUNTIME_CFG
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_UINT32         ui32ActivePMLatencyms;                    /*!< APM latency in ms before signalling IDLE to the host */
+	IMG_UINT32         ui32RuntimeCfgFlags;                      /*!< Compatibility and other flags */
+	IMG_BOOL           bActivePMLatencyPersistant;               /*!< If set, APM latency does not reset to system default each GPU power transition */
+	IMG_UINT32         ui32CoreClockSpeed;                       /*!< Core clock speed, currently only used to calculate timer ticks */
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32         ui32SOCClockSpeed;                        /*!< SOC clock speed, used for deadline scheduling */
+#endif
+	IMG_UINT32         ui32PowUnitsState;                        /*!< Last number of dusts change requested by the host */
+	IMG_UINT32         ui32PHRMode;                              /*!< Periodic Hardware Reset configuration values */
+	IMG_UINT32         ui32HCSDeadlineMS;                        /*!< New number of milliseconds C/S is allowed to last */
+	IMG_UINT32         ui32WdgPeriodUs;                          /*!< The watchdog period in microseconds */
+	IMG_INT32          ai32DriverPriority[RGXFW_MAX_NUM_OSIDS];  /*!< Array of priorities per OS */
+	IMG_UINT32         aui32DriverIsolationGroup[RGXFW_MAX_NUM_OSIDS]; /*!< Array of isolation groups per OS */
+	IMG_UINT32         aui32TSPercentage[RGXFW_MAX_NUM_OSIDS];   /*!< Array of time slice per OS */
+	IMG_UINT32         ui32TSIntervalMs;                         /*!< Time slice interval */
+	IMG_UINT32         ui32VzConnectionCooldownPeriodInSec;      /*!< Vz Connection Cooldown period in secs */
+
+	PRGXFWIF_HWPERFBUF sHWPerfBuf;                               /*!< On-demand allocated HWPerf buffer address, to be passed to the FW */
+	RGXFWIF_DMA_ADDR   sHWPerfDMABuf;
+	RGXFWIF_DMA_ADDR   sHWPerfCtlDMABuf;
+} RGXFWIF_RUNTIME_CFG;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_RUNTIME_CFG) == 68,
+				"RGXFWIF_RUNTIME_CFG is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ *****************************************************************************
+ * Control data for RGX
+ *****************************************************************************/
+
+#define RGXFWIF_HWR_DEBUG_DUMP_ALL (99999U)
+
+#if defined(PDUMP)
+
+#define RGXFWIF_PID_FILTER_MAX_NUM_PIDS 32U
+
+typedef enum
+{
+	RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+	RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT
+} RGXFWIF_PID_FILTER_MODE;
+
+typedef struct
+{
+	IMG_PID uiPID;
+	IMG_UINT32 ui32DriverID;
+} RGXFW_ALIGN RGXFWIF_PID_FILTER_ITEM;
+
+typedef struct
+{
+	RGXFWIF_PID_FILTER_MODE eMode;
+	/* each process in the filter list is specified by a PID and OS ID pair.
+	 * each PID and OS pair is an item in the items array (asItems).
+	 * if the array contains less than RGXFWIF_PID_FILTER_MAX_NUM_PIDS entries
+	 * then it must be terminated by an item with pid of zero.
+	 */
+	RGXFWIF_PID_FILTER_ITEM asItems[RGXFWIF_PID_FILTER_MAX_NUM_PIDS];
+} RGXFW_ALIGN RGXFWIF_PID_FILTER;
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA  (0x1U << 0)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE  (0x1U << 1)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE  (0x1U << 2)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE     (0x1U << 3)
+#endif
+
+typedef enum
+{
+	RGXFWIF_TPU_DM_PDM = 0,
+	RGXFWIF_TPU_DM_VDM = 1,
+	RGXFWIF_TPU_DM_CDM = 2,
+	RGXFWIF_TPU_DM_TDM = 3,
+	RGXFWIF_TPU_DM_LAST
+} RGXFWIF_TPU_DM;
+
+typedef enum
+{
+	RGXFWIF_GPIO_VAL_OFF           = 0, /*!< No GPIO validation */
+	RGXFWIF_GPIO_VAL_GENERAL       = 1, /*!< Simple test case that
+	                                         initiates by sending data via the
+	                                         GPIO and then sends back any data
+	                                         received over the GPIO */
+	RGXFWIF_GPIO_VAL_AP            = 2, /*!< More complex test case that writes
+	                                         and reads data across the entire
+	                                         GPIO AP address range.*/
+#if defined(SUPPORT_STRIP_RENDERING)
+	RGXFWIF_GPIO_VAL_SR_BASIC      = 3, /*!< Strip Rendering AP based basic test.*/
+	RGXFWIF_GPIO_VAL_SR_COMPLEX    = 4, /*!< Strip Rendering AP based complex test.*/
+#endif
+	RGXFWIF_GPIO_VAL_TESTBENCH     = 5, /*!< Validates the GPIO Testbench. */
+	RGXFWIF_GPIO_VAL_LOOPBACK      = 6, /*!< Send and then receive each byte
+	                                         in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LOOPBACK_LITE = 7, /*!< Send and then receive each power-of-2
+	                                         byte in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LAST
+} RGXFWIF_GPIO_VAL_MODE;
+
+typedef IMG_UINT32 FW_PERF_CONF;
+#define FW_PERF_CONF_NONE			0U
+#define FW_PERF_CONF_ICACHE			1U
+#define FW_PERF_CONF_DCACHE			2U
+#define FW_PERF_CONF_JTLB_INSTR		5U
+#define FW_PERF_CONF_INSTRUCTIONS	6U
+
+typedef enum
+{
+	FW_BOOT_STAGE_TLB_INIT_FAILURE = -2,
+	FW_BOOT_STAGE_NOT_AVAILABLE = -1,
+	FW_BOOT_NOT_STARTED = 0,
+	FW_BOOT_BLDR_STARTED = 1,
+	FW_BOOT_CACHE_DONE,
+	FW_BOOT_TLB_DONE,
+	FW_BOOT_MAIN_STARTED,
+	FW_BOOT_ALIGNCHECKS_DONE,
+	FW_BOOT_INIT_DONE,
+} FW_BOOT_STAGE;
+
+/*!
+ * @AddToGroup KCCBTypes
+ * @{
+ * @Name Kernel CCB return slot responses
+ * @{
+ * Usage of bit-fields instead of bare integers
+ * allows FW to possibly pack-in several responses for each single kCCB command.
+ */
+
+#define RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED   (1U << 0) /*!< Command executed (return status from FW) */
+#define RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY   (1U << 1) /*!< A cleanup was requested but resource busy */
+#define RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE   (1U << 2) /*!< Poll failed in FW for a HW operation to complete */
+
+#define RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE            0x0U      /*!< Reset value of a kCCB return slot (set by host) */
+/*!
+ * @} End of Name Kernel CCB return slot responses
+ * @} End of AddToGroup KCCBTypes
+ */
+
+/*! @Brief OS connection data \ref RGXFWIF_CONNECTION_CTL allocated
+ * by services and used to track OS state in Firmware and Services
+ **/
+typedef struct
+{
+	/* Fw-Os connection states */
+	volatile RGXFWIF_CONNECTION_FW_STATE eConnectionFwState;    /*!< Firmware-OS connection state */
+	volatile RGXFWIF_CONNECTION_OS_STATE eConnectionOsState;    /*!< Services-OS connection state */
+	volatile IMG_UINT32                  ui32AliveFwToken;      /*!< OS Alive token updated by Firmware */
+	volatile IMG_UINT32                  ui32AliveOsToken;      /*!< OS Alive token updated by Services */
+} UNCACHED_ALIGN RGXFWIF_CONNECTION_CTL;
+
+/*! @Brief Firmware OS Initialization data \ref RGXFWIF_OSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	/* Kernel CCB */
+	PRGXFWIF_CCB_CTL        psKernelCCBCtl; /*!< Kernel CCB Control */
+	PRGXFWIF_CCB            psKernelCCB; /*!<  Kernel CCB */
+	PRGXFWIF_CCB_RTN_SLOTS  psKernelCCBRtnSlots; /*!<  Kernel CCB return slots */
+
+	/* Firmware CCB */
+	PRGXFWIF_CCB_CTL        psFirmwareCCBCtl; /*!<  Firmware CCB control */
+	PRGXFWIF_CCB            psFirmwareCCB; /*!<  Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	PRGXFWIF_CCB_CTL        psWorkEstFirmwareCCBCtl; /*!<  Workload estimation control */
+	PRGXFWIF_CCB            psWorkEstFirmwareCCB; /*!<  Workload estimation buffer */
+
+	PRGXFWIF_HWRINFOBUF     sRGXFWIfHWRInfoBufCtl; /*!<  HWRecoveryInfo control */
+
+	IMG_UINT32              ui32HWRDebugDumpLimit; /*!< Firmware debug dump maximum limit */
+
+	PRGXFWIF_OSDATA         sFwOsData; /*!<  Firmware per-os shared data */
+
+	RGXFWIF_COMPCHECKS      sRGXCompChecks; /*!< Compatibility checks to be populated by the Firmware */
+	RGXFWIF_MULTICORE_INFO  sRGXMulticoreInfo; /*! < Multicore capability info */
+
+} UNCACHED_ALIGN RGXFWIF_OSINIT;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_OSINIT) == 104,
+				"RGXFWIF_OSINIT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @Brief Firmware System Initialization data \ref RGXFWIF_SYSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	RGX_MIPS_STATE          sMIPSState; /*!< MIPS Debug Data; this must be the first member in the structure */
+
+	IMG_DEV_PHYADDR         RGXFW_ALIGN sFaultPhysAddr; /*!< Fault read address */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSExecBase; /*!< PDS execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sUSCExecBase; /*!< USC execution base */
+
+	IMG_UINT64              RGXFW_ALIGN ui64HWPerfFilter; /*! Event filter for Firmware events */
+
+	IMG_UINT32              RGXFW_ALIGN aui32TPUTrilinearFracMask[RGXFWIF_TPU_DM_LAST];
+
+#if defined(PDUMP)
+	RGXFWIF_SIGBUF_CTL      asSigBufCtl[RGXFWIF_DM_MAX]; /*!< Signature and Checksum Buffers for DMs */
+#endif
+
+	RGXFWIF_DMA_ADDR        sCorememDataStore; /*!< Firmware coremem data */
+
+	RGXFWIF_COUNTER_DUMP_CTL sCounterDumpCtl;
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	RGXFWIF_FIRMWARE_GCOV_CTL sFirmwareGcovCtl; /*!< Firmware gcov buffer control */
+#endif
+
+	IMG_UINT32              ui32FilterFlags;
+
+	PRGXFWIF_RUNTIME_CFG    sRuntimeCfg; /*!<  Firmware Runtime configuration */
+
+	PRGXFWIF_TRACEBUF       sTraceBufCtl; /*!<  Firmware Trace buffer control */
+	PRGXFWIF_SYSDATA        sFwSysData; /*!< Firmware System shared data */
+#if defined(SUPPORT_TBI_INTERFACE)
+	PRGXFWIF_TBIBUF         sTBIBuf; /*!< Tbi log buffer */
+#endif
+
+	PRGXFWIF_GPU_UTIL_FW    sGpuUtilFWCtl; /*!< GPU utilization buffer */
+	PRGXFWIF_REG_CFG        sRegCfg; /*!< Firmware register user configuration */
+	PRGXFWIF_HWPERF_CTL     sHWPerfCtl; /*!< HWPerf counter block configuration.*/
+
+	RGXFWIF_DEV_VIRTADDR    sAlignChecks; /*!< Array holding Server structures alignment data */
+
+	IMG_UINT32              ui32InitialCoreClockSpeed; /*!< Core clock speed at FW boot time */
+
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32              ui32InitialSOCClockSpeed; /*!< System/SOC clock speed at FW boot time */
+#endif
+
+	IMG_UINT32              ui32InitialActivePMLatencyms; /*!< APM latency in ms before signalling IDLE to the host */
+
+	IMG_BOOL                bFirmwareStarted; /*!< Flag to be set by the Firmware after successful start */
+
+	IMG_UINT32              ui32MarkerVal; /*!< Host/FW Trace synchronisation Partition Marker */
+
+	IMG_UINT32              ui32FirmwareStartedTimeStamp; /*!< Firmware initialization complete time */
+
+	FW_PERF_CONF            eFirmwarePerf; /*!< Firmware performance counter config */
+
+#if defined(SUPPORT_FW_OPP_TABLE)
+	RGXFWIF_OPP_INFO        RGXFW_ALIGN sOPPInfo;
+
+	/**
+	 * FW Pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory when running on non primary FW thread
+	 * to communicate to host driver.
+	 */
+	PRGXFWIF_CORE_CLK_RATE  RGXFW_ALIGN sCoreClockRate;
+#endif
+
+#if defined(PDUMP)
+	RGXFWIF_PID_FILTER      sPIDFilter;
+#endif
+
+	RGXFWIF_GPIO_VAL_MODE   eGPIOValidationMode;
+
+	RGX_HWPERF_BVNC         sBvncKmFeatureFlags; /*!< Used in HWPerf for decoding BVNC Features*/
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32              ui32SecurityTestFlags;
+	RGXFWIF_DEV_VIRTADDR    pbSecureBuffer;
+	RGXFWIF_DEV_VIRTADDR    pbNonSecureBuffer;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	RGXFWIF_DEV_VIRTADDR    sActiveContextBufBase; /*!< Active context buffer base */
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	/*
+	 * Used when validation is enabled to allow the host to check
+	 * that MTS sent the correct sideband in response to a kick
+	 * from a given OSes schedule register.
+	 * Testing is enabled if RGXFWIF_KICK_TEST_ENABLED_BIT is set
+	 *
+	 * Set by the host to:
+	 * (osid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT
+	 * reset to 0 by FW when kicked by the given OSid
+	 */
+	IMG_UINT32              ui32OSKickTest;
+#endif
+
+	/* Value to write into RGX_CR_TFBC_COMPRESSION_CONTROL */
+	IMG_UINT32              ui32TFBCCompressionControl;
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_UINT32              ui32VzWdgPeriod;
+#endif
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	/* notify firmware power-up on host-side recovery */
+	IMG_BOOL                bFwHostRecoveryMode;
+#endif
+} UNCACHED_ALIGN RGXFWIF_SYSINIT;
+
+static_assert(offsetof(RGXFWIF_SYSINIT, sMIPSState) == 0,
+				"sMIPSState is not the first member of the RGXFWIF_SYSINIT struct");
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+#define RGXFWIF_KICK_TEST_ENABLED_BIT  0x1
+#define RGXFWIF_KICK_TEST_OSID_SHIFT   0x1
+#endif
+
+/*!
+ *****************************************************************************
+ * Timer correlation shared data and defines
+ *****************************************************************************/
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64OSMonoTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeStamp;
+
+	/* Utility variable used to convert CR timer deltas to OS timer deltas (nS),
+	 * where the deltas are relative to the timestamps above:
+	 * deltaOS = (deltaCR * K) >> decimal_shift, see full explanation below */
+	IMG_UINT64 RGXFW_ALIGN ui64CRDeltaToOSDeltaKNs;
+
+	IMG_UINT32             ui32CoreClockSpeed;
+	IMG_UINT32             ui32Reserved;
+} UNCACHED_ALIGN RGXFWIF_TIME_CORR;
+
+
+/* The following macros are used to help converting FW timestamps to the Host
+ * time domain. On the FW the RGX_CR_TIMER counter is used to keep track of
+ * time; it increments by 1 every 256 GPU clock ticks, so the general
+ * formula to perform the conversion is:
+ *
+ * [ GPU clock speed in Hz, if (scale == 10^9) then deltaOS is in nS,
+ *   otherwise if (scale == 10^6) then deltaOS is in uS ]
+ *
+ *             deltaCR * 256                                   256 * scale
+ *  deltaOS = --------------- * scale = deltaCR * K    [ K = --------------- ]
+ *             GPUclockspeed                                  GPUclockspeed
+ *
+ * The actual K is multiplied by 2^20 (and deltaCR * K is divided by 2^20)
+ * to get some better accuracy and to avoid returning 0 in the integer
+ * division 256000000/GPUfreq if GPUfreq is greater than 256MHz.
+ * This is the same as keeping K as a decimal number.
+ *
+ * The maximum deltaOS is slightly more than 5hrs for all GPU frequencies
+ * (deltaCR * K is more or less a constant), and it's relative to the base
+ * OS timestamp sampled as a part of the timer correlation data.
+ * This base is refreshed on GPU power-on, DVFS transition and periodic
+ * frequency calibration (executed every few seconds if the FW is doing
+ * some work), so as long as the GPU is doing something and one of these
+ * events is triggered then deltaCR * K will not overflow and deltaOS will be
+ * correct.
+ */
+
+#define RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT  (20)
+
+#define RGXFWIF_GET_DELTA_OSTIME_NS(deltaCR, K) \
+	(((deltaCR) * (K)) >> RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT)
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation
+ *****************************************************************************/
+
+/* See rgx_common.h for a list of GPU states */
+#define RGXFWIF_GPU_UTIL_TIME_MASK       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_TIME_MASK32     (IMG_UINT32_C(0xFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+#define RGXFWIF_GPU_UTIL_GET_TIME(word)    ((word) & RGXFWIF_GPU_UTIL_TIME_MASK)
+#define RGXFWIF_GPU_UTIL_GET_STATE(word)   ((word) & RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_GET_TIME32(word)  ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_TIME_MASK32)
+#define RGXFWIF_GPU_UTIL_GET_STATE32(word) ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+/* The OS timestamps computed by the FW are approximations of the real time,
+ * which means they could be slightly behind or ahead the real timer on the Host.
+ * In some cases we can perform subtractions between FW approximated
+ * timestamps and real OS timestamps, so we need a form of protection against
+ * negative results if for instance the FW one is a bit ahead of time.
+ */
+#define RGXFWIF_GPU_UTIL_GET_PERIOD(newtime,oldtime) \
+	(((newtime) > (oldtime)) ? ((newtime) - (oldtime)) : 0U)
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME(time) | RGXFWIF_GPU_UTIL_GET_STATE(state))
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD32(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME32(time) | RGXFWIF_GPU_UTIL_GET_STATE32(state))
+
+
+/* The timer correlation array must be big enough to ensure old entries won't be
+ * overwritten before all the HWPerf events linked to those entries are processed
+ * by the MISR. The update frequency of this array depends on how fast the system
+ * can change state (basically how small the APM latency is) and perform DVFS transitions.
+ *
+ * The minimum size is 2 (not 1) to avoid race conditions between the FW reading
+ * an entry while the Host is updating it. With 2 entries in the worst case the FW
+ * will read old data, which is still quite ok if the Host is updating the timer
+ * correlation at that time.
+ */
+#define RGXFWIF_TIME_CORR_ARRAY_SIZE            256U
+#define RGXFWIF_TIME_CORR_CURR_INDEX(seqcount)  ((seqcount) % RGXFWIF_TIME_CORR_ARRAY_SIZE)
+
+/* Make sure the timer correlation array size is a power of 2 */
+static_assert((RGXFWIF_TIME_CORR_ARRAY_SIZE & (RGXFWIF_TIME_CORR_ARRAY_SIZE - 1U)) == 0U,
+			  "RGXFWIF_TIME_CORR_ARRAY_SIZE must be a power of two");
+
+/* The time is stored in DM state time-stamps, and as a result in DMs states counters, in "approximately microseconds",
+ * dividing the time originally obtained in nanoseconds by 2^10 for the sake of reducing coremem usage */
+#define RGXFWIF_DM_OS_TIMESTAMP_SHIFT    10U
+
+typedef struct
+{
+	/* Last GPU DM per-OS states + OS time of the last state update */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWord[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* DMs time-stamps are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWordWrap[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* Counters for the amount of time the GPU DMs were active or inactive(idle or blocked) */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSStatsCounters[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	/* DMs Counters are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSCountersWrap[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+} RGXFWIF_GPU_STATS;
+
+typedef struct
+{
+	RGXFWIF_TIME_CORR      sTimeCorr[RGXFWIF_TIME_CORR_ARRAY_SIZE];
+	IMG_UINT32             ui32TimeCorrSeqCount;
+
+	/* Compatibility and other flags */
+	IMG_UINT32             ui32GpuUtilFlags;
+
+	/* Last GPU state + OS time of the last state update */
+	IMG_UINT64 RGXFW_ALIGN ui64GpuLastWord;
+	/* Counters for the amount of time the GPU was active/idle/blocked */
+	IMG_UINT64 RGXFW_ALIGN aui64GpuStatsCounters[RGXFWIF_GPU_UTIL_STATE_NUM];
+	/* Device off period timestamp offset */
+	IMG_INT64 RGXFW_ALIGN i64DeviceTimestampOffset;
+	/* Stats per OSID/DriverID */
+	RGXFWIF_GPU_STATS sStats[RGXFW_MAX_NUM_OSIDS];
+} UNCACHED_ALIGN RGXFWIF_GPU_UTIL_FW;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_GPU_UTIL_FW) == 12584,
+				"RGXFWIF_GPU_UTIL_FW is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES      RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES      (RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      PVR_ALIGN((RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES), DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY)
+#else
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      0
+#endif
+
+#define RGXFWIF_TDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES)
+
+/*!
+ ******************************************************************************
+ * Virtualisation and Security
+ *****************************************************************************/
+#if defined(SECURE_FW_CODE_OSID)
+/* software workaround for SoCs without fw_code, fw_priv_data signals, MIPS only */
+#if defined(RGX_FEATURE_META)
+#error "SECURE_FW_CODE_OSID is not supported on META cores"
+#elif defined(RGX_FEATURE_RISCV_FW_PROCESSOR)
+#error "SECURE_FW_CODE_OSID is not supported on RISC-V cores"
+#elif (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#error "SECURE_FW_CODE_OSID is not supported on virtualization drivers"
+#elif (SECURE_FW_CODE_OSID + 1 > 2)
+#define MIPS_FW_CODE_OSID				(SECURE_FW_CODE_OSID)
+#else
+#define MIPS_FW_CODE_OSID				(1U)
+#endif
+#endif /* defined(SECURE_FW_CODE_OSID) */
+
+static_assert((RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID),
+			  " Invalid RGX_FW_HEAP_OSID_ASSIGNMENT value. Rogue cores support only the RGX_FW_HEAP_USES_FIRMWARE_OSID config");
+
+/* Firmware and Host driver share the same OSID */
+#define FW_HEAP_OSID					(FW_OSID)
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* virtualization without security support */
+#define DRIVER_ID(osid)					(osid)
+#define OSID(did)						(did)
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT	(30U)
+#define RGXFW_VZ_PRIORITY_MASK		((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX		(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN		(5U)
+#else
+#define DRIVER_ID(osid)					(0U)
+#define OSID(did)						(did)
+#define RGXFW_VZ_PRIORITY_MASK			(0U)
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) < RGX_NUM_DRIVERS_SUPPORTED; (did)++)
+
+#if defined(__KERNEL__)
+/* Driver implementation */
+#define FOREACH_ACTIVE_DRIVER(devinfo, did)        RGXFwSharedMemCacheOpValue(psFwSysData->asOsRuntimeFlagsMirror[RGXFW_HOST_DRIVER_ID], \
+                                                                              INVALIDATE); \
+                                                   FOREACH_SUPPORTED_DRIVER(did)                                \
+                                                   {                                                            \
+                                                   if (devinfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[did].bfOsState != RGXFW_CONNECTION_FW_ACTIVE) continue;
+
+#define END_FOREACH_ACTIVE_DRIVER                  }
+
+#else
+/* Firmware implementation */
+#define FOREACH_ACTIVE_DRIVER(did)                 do {                                                                      \
+                                                   unsigned int idx;                                                         \
+                                                   for (idx = 0; idx < RGXFW_NUM_ACTIVE_DRIVERS; idx++)                      \
+                                                   {                                                                         \
+                                                       (did) = gsRGXFWCtl.aui32ActiveDrivers[idx];                           \
+                                                       {
+
+#define END_FOREACH_ACTIVE_DRIVER                  }}} while (false);
+#endif /* defined(__KERNEL__) */
+
+
+#else
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) <= RGXFW_HOST_DRIVER_ID; (did)++)
+
+#define FOREACH_ACTIVE_DRIVER(did)                 FOREACH_SUPPORTED_DRIVER(did)
+#define END_FOREACH_ACTIVE_DRIVER
+
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#define FOREACH_VALIDATION_OSID(osid)              for ((osid)=0; (osid) < GPUVIRT_VALIDATION_NUM_OS; (osid)++)
+#define FOREACH_HW_OSID(osid)                      for ((osid)=0; (osid) < RGXFW_MAX_NUM_OSIDS; (osid)++)
+#define FOREACH_DRIVER_RAW_HEAP(did, _struct, dev) for ((did)=RGX_FIRST_RAW_HEAP_DRIVER_ID; (did) < ((PVRSRV_VZ_MODE_IS(NATIVE, _struct, dev) ? 1 : RGX_NUM_DRIVERS_SUPPORTED)); (did)++)
+
+#endif /* RGX_FWIF_KM_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_shared.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_shared.h
new file mode 100644
index 000000000000..ccab45ae8fc0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_shared.h
@@ -0,0 +1,1140 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures shared by both host client
+                and host server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_SHARED_H)
+#define RGX_FWIF_SHARED_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+#include "devicemem_typedefs.h"
+
+/* Indicates the number of RTDATAs per RTDATASET */
+#if defined(SUPPORT_AGP)
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  12U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (2U)
+#else
+#define RGXMKIF_NUM_RTDATAS           2U
+#define RGXMKIF_NUM_GEOMDATAS         1U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  2U  /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (1U)
+#endif
+
+/* Maximum number of UFOs in a CCB command.
+ * The number is based on having 32 sync prims (as originally), plus 32 sync
+ * checkpoints.
+ * Once the use of sync prims is no longer supported, we will retain
+ * the same total (64) as the number of sync checkpoints which may be
+ * supporting a fence is not visible to the client driver and has to
+ * allow for the number of different timelines involved in fence merges.
+ */
+#define RGXFWIF_CCB_CMD_MAX_UFOS			(32U+32U)
+
+/*
+ * This is a generic limit imposed on any DM (TA,3D,CDM,TDM,2D,TRANSFER)
+ * command passed through the bridge.
+ * Just across the bridge in the server, any incoming kick command size is
+ * checked against this maximum limit.
+ * In case the incoming command size is larger than the specified limit,
+ * the bridge call is retired with error.
+ */
+#define RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE	(1024U)
+
+typedef struct
+{
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN psDevVirtAddr;
+	RGXFWIF_DEV_VIRTADDR    pbyFWAddr;
+} UNCACHED_ALIGN RGXFWIF_DMA_ADDR;
+
+typedef IMG_UINT8	RGXFWIF_CCCB;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_UFO_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CLEANUP_CTL;
+
+
+/*!
+ * @InGroup ClientCCBTypes
+ * @Brief Command data for fence & update types Client CCB commands.
+ */
+typedef struct
+{
+	PRGXFWIF_UFO_ADDR	puiAddrUFO; /*!< Address to be checked/updated */
+	IMG_UINT32			ui32Value;  /*!< Value to check-against/update-to */
+} RGXFWIF_UFO;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Track pending and executed workloads of HWRTDATA and ZSBUFFER
+ */
+typedef struct
+{
+	IMG_UINT32			ui32SubmittedCommands;	/*!< Number of commands received by the FW */
+	IMG_UINT32			ui32ExecutedCommands;	/*!< Number of commands executed by the FW */
+} UNCACHED_ALIGN RGXFWIF_CLEANUP_CTL;
+
+#define	RGXFWIF_PRBUFFER_START        IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_ZSBUFFER     IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_MSAABUFFER   IMG_UINT32_C(1)
+#define	RGXFWIF_PRBUFFER_MAXSUPPORTED IMG_UINT32_C(2)
+
+typedef IMG_UINT32 RGXFWIF_PRBUFFER_TYPE;
+
+typedef enum
+{
+	RGXFWIF_PRBUFFER_UNBACKED = 0,
+	RGXFWIF_PRBUFFER_BACKED,
+	RGXFWIF_PRBUFFER_BACKING_PENDING,
+	RGXFWIF_PRBUFFER_UNBACKING_PENDING,
+}RGXFWIF_PRBUFFER_STATE;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief OnDemand Z/S/MSAA Buffers
+ */
+typedef struct
+{
+	IMG_UINT32		ui32BufferID;		/*!< Buffer ID*/
+	IMG_BOOL		bOnDemand;		/*!< Needs On-demand Z/S/MSAA Buffer allocation */
+	RGXFWIF_PRBUFFER_STATE	eState;			/*!< Z/S/MSAA -Buffer state */
+	RGXFWIF_CLEANUP_CTL	sCleanupState;		/*!< Cleanup state */
+	IMG_UINT32		ui32PRBufferFlags;	/*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_PRBUFFER;
+
+/*
+ * Used to share frame numbers across UM-KM-FW,
+ * frame number is set in UM,
+ * frame number is required in both KM for HTB and FW for FW trace.
+ *
+ * May be used to house Kick flags in the future.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32FrameNum; /*!< associated frame number */
+} CMD_COMMON;
+
+/*
+ * TA and 3D commands require set of firmware addresses that are stored in the
+ * Kernel. Client has handle(s) to Kernel containers storing these addresses,
+ * instead of raw addresses. We have to patch/write these addresses in KM to
+ * prevent UM from controlling FW addresses directly.
+ * Typedefs for TA and 3D commands are shared between Client and Firmware (both
+ * single-BVNC). Kernel is implemented in a multi-BVNC manner, so it can't use
+ * TA|3D CMD type definitions directly. Therefore we have a SHARED block that
+ * is shared between UM-KM-FW across all BVNC configurations.
+ */
+typedef struct
+{
+	CMD_COMMON           sCmn;      /*!< Common command attributes */
+	RGXFWIF_DEV_VIRTADDR sHWRTData; /* RTData associated with this command,
+									   this is used for context selection and for storing out HW-context,
+									   when TA is switched out for continuing later */
+
+	RGXFWIF_DEV_VIRTADDR asPRBuffer[RGXFWIF_PRBUFFER_MAXSUPPORTED];	/* Supported PR Buffers like Z/S/MSAA Scratch */
+
+} CMDTA3D_SHARED;
+
+/*!
+ * Client Circular Command Buffer (CCCB) control structure.
+ * This is shared between the Server and the Firmware and holds byte offsets
+ * into the CCCB as well as the wrapping mask to aid wrap around. A given
+ * snapshot of this queue with Cmd 1 running on the GPU might be:
+ *
+ *          Roff                           Doff                 Woff
+ * [..........|-1----------|=2===|=3===|=4===|~5~~~~|~6~~~~|~7~~~~|..........]
+ *            <      runnable commands       ><   !ready to run   >
+ *
+ * Cmd 1    : Currently executing on the GPU data master.
+ * Cmd 2,3,4: Fence dependencies met, commands runnable.
+ * Cmd 5... : Fence dependency not met yet.
+ */
+typedef struct
+{
+	IMG_UINT32  ui32WriteOffset;    /*!< Host write offset into CCB. This
+	                                 *    must be aligned to 16 bytes. */
+	IMG_UINT32  ui32ReadOffset;     /*!< Firmware read offset into CCB.
+	                                      Points to the command that is
+	                                 *    runnable on GPU, if R!=W */
+	IMG_UINT32  ui32DepOffset;      /*!< Firmware fence dependency offset.
+	                                 *    Points to commands not ready, i.e.
+	                                 *    fence dependencies are not met. */
+	IMG_UINT32  ui32WrapMask;       /*!< Offset wrapping mask, total capacity
+	                                      in bytes of the CCB-1 */
+
+	IMG_UINT32  ui32ReadOffset2;     /*!< Firmware 2nd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R2!=W */
+	IMG_UINT32  ui32ReadOffset3;     /*!< Firmware 3rd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R3!=W */
+	IMG_UINT32  ui32ReadOffset4;     /*!< Firmware 4th read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R4!=W */
+
+} UNCACHED_ALIGN RGXFWIF_CCCB_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCCB_CTL) == 32,
+				"RGXFWIF_CCCB_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef IMG_UINT32 RGXFW_FREELIST_TYPE;
+
+#define RGXFW_LOCAL_FREELIST     IMG_UINT32_C(0)
+#define RGXFW_GLOBAL_FREELIST    IMG_UINT32_C(1)
+#if defined(SUPPORT_AGP)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL2_FREELIST + 1U)
+#else
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL_FREELIST + 1U)
+#endif
+#define RGXFW_MAX_HWFREELISTS    (2U)
+
+/*!
+ * @Defgroup ContextSwitching Context switching data interface
+ * @Brief Types grouping data structures and defines used in realising the Context Switching (CSW) functionality
+ * @{
+ */
+
+/*!
+ * @Brief GEOM DM or TA register controls for context switch
+ */
+typedef struct
+{
+	IMG_UINT64	uTAReg_VDM_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the VDM's context state buffer */
+	IMG_UINT64	uTAReg_VDM_CONTEXT_STATE_RESUME_ADDR;
+	IMG_UINT64	uTAReg_TA_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the TA's context state buffer */
+
+	struct
+	{
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK0; /*!< VDM context store task 0 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK1; /*!< VDM context store task 1 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK2; /*!< VDM context store task 2 */
+
+		/* VDM resume state update controls */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK0; /*!< VDM context resume task 0 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK1; /*!< VDM context resume task 1 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK2; /*!< VDM context resume task 2 */
+	} asTAState[2];
+
+} RGXFWIF_TAREGISTERS_CSWITCH;
+/*! @} End of Defgroup ContextSwitching */
+
+#define RGXFWIF_TAREGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_TAREGISTERS_CSWITCH)
+
+typedef struct
+{
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS1;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS1;
+
+	/* CDM resume controls */
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0_B;
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0_B;
+
+} RGXFWIF_CDM_REGISTERS_CSWITCH;
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Render context static register controls for context switch
+ */
+typedef struct
+{
+	RGXFWIF_TAREGISTERS_CSWITCH	RGXFW_ALIGN asCtxSwitch_GeomRegs[RGX_NUM_GEOM_CORES];	/*!< Geom registers for ctx switch */
+} RGXFWIF_STATIC_RENDERCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RENDERCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RENDERCONTEXT_STATE)
+
+typedef struct
+{
+	RGXFWIF_CDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< CDM registers for ctx switch */
+} RGXFWIF_STATIC_COMPUTECONTEXT_STATE;
+
+#define RGXFWIF_STATIC_COMPUTECONTEXT_SIZE sizeof(RGXFWIF_STATIC_COMPUTECONTEXT_STATE)
+
+/*!
+	@Brief Context reset reason. Last reset reason for a reset context.
+*/
+typedef enum
+{
+	RGX_CONTEXT_RESET_REASON_NONE                = 0,	/*!< No reset reason recorded */
+	RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP       = 1,	/*!< Caused a reset due to locking up */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP     = 2,	/*!< Affected by another context locking up */
+	RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING   = 3,	/*!< Overran the global deadline */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING = 4,	/*!< Affected by another context overrunning */
+	RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH = 5,	/*!< Forced reset to ensure scheduling requirements */
+	RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM        = 6,	/*!< CDM Mission/safety checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM        = 7,	/*!< TRP checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_OK          = 8,	/*!< GPU ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR         = 9,	/*!< GPU ECC error (uncorrected, HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_OK           = 10,	/*!< FW ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_ERR          = 11,	/*!< FW ECC error (uncorrected, ERR) */
+	RGX_CONTEXT_RESET_REASON_FW_WATCHDOG         = 12,	/*!< FW Safety watchdog triggered */
+	RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT        = 13,	/*!< FW page fault (no HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR         = 14,	/*!< FW execution error (GPU reset requested) */
+	RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR     = 15,	/*!< Host watchdog detected FW error */
+	RGX_CONTEXT_GEOM_OOM_DISABLED                = 16,	/*!< Geometry DM OOM event is not allowed */
+	RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH         = 17,	/*!< PVRIC Signature mismatch */
+	RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR   = 18,	/*!< Parity error in MMU Page Table Entry */
+	RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR       = 19,	/*!< Parity error in MH, system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR      = 20,	/*!< Parity error in system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR      = 21,	/*!< Latent/ICS signature mismatch error */
+	RGX_CONTEXT_RESET_REASON_DCLS_ERR            = 22,	/*!< Dual Core Lock Step FW error detected */
+} RGX_CONTEXT_RESET_REASON;
+
+/*!
+	@Brief Context reset data shared with the host
+*/
+typedef struct
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reset reason */
+	IMG_UINT32 ui32ResetExtJobRef;  /*!< External Job ID */
+} RGX_CONTEXT_RESET_REASON_DATA;
+
+#define RGX_HEAP_UM_PDS_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_PDS_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_PDS_RESERVED_TOTAL_SIZE            RGX_HEAP_UM_PDS_RESERVED_SIZE
+
+#define RGX_HEAP_UM_USC_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_USC_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET      RGX_HEAP_UM_USC_RESERVED_SIZE
+
+#define RGX_HEAP_UM_GENERAL_RESERVED_SIZE           DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_GENERAL_RESERVED_REGION_OFFSET  0
+#define RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET  RGX_HEAP_UM_GENERAL_RESERVED_SIZE
+
+/*************************************************************************/ /*!
+ Logging type
+*/ /**************************************************************************/
+#define RGXFWIF_LOG_TYPE_NONE			0x00000000U
+#define RGXFWIF_LOG_TYPE_TRACE			0x00000001U
+#define RGXFWIF_LOG_TYPE_GROUP_MAIN		0x00000002U
+#define RGXFWIF_LOG_TYPE_GROUP_MTS		0x00000004U
+#define RGXFWIF_LOG_TYPE_GROUP_CLEANUP	0x00000008U
+#define RGXFWIF_LOG_TYPE_GROUP_CSW		0x00000010U
+#define RGXFWIF_LOG_TYPE_GROUP_BIF		0x00000020U
+#define RGXFWIF_LOG_TYPE_GROUP_PM		0x00000040U
+#define RGXFWIF_LOG_TYPE_GROUP_RTD		0x00000080U
+#define RGXFWIF_LOG_TYPE_GROUP_SPM		0x00000100U
+#define RGXFWIF_LOG_TYPE_GROUP_POW		0x00000200U
+#define RGXFWIF_LOG_TYPE_GROUP_HWR		0x00000400U
+#define RGXFWIF_LOG_TYPE_GROUP_HWP		0x00000800U
+#define RGXFWIF_LOG_TYPE_GROUP_RPM		0x00001000U
+#define RGXFWIF_LOG_TYPE_GROUP_DMA		0x00002000U
+#define RGXFWIF_LOG_TYPE_GROUP_MISC		0x00004000U
+#define RGXFWIF_LOG_TYPE_GROUP_VZ		0x00008000U
+#define RGXFWIF_LOG_TYPE_GROUP_SAFETY	0x00010000U
+#define RGXFWIF_LOG_TYPE_GROUP_VERBOSE	0x00020000U
+#define RGXFWIF_LOG_TYPE_GROUP_CUSTOMER	0x00040000U
+#define RGXFWIF_LOG_TYPE_GROUP_DEBUG	0x80000000U
+#define RGXFWIF_LOG_TYPE_GROUP_MASK		0x8007FFFEU
+#define RGXFWIF_LOG_TYPE_MASK			0x8007FFFFU
+
+/* String used in pvrdebug -h output */
+#define RGXFWIF_LOG_GROUPS_STRING_LIST   "main,mts,cleanup,csw,bif,pm,rtd,spm,pow,hwr,hwp,rpm,dma,misc,vz,safety,verbose,customer,debug"
+
+/* Table entry to map log group strings to log type value */
+typedef struct {
+	const IMG_CHAR* pszLogGroupName;
+	IMG_UINT32      ui32LogGroupType;
+} RGXFWIF_LOG_GROUP_MAP_ENTRY;
+
+/*
+  Macro for use with the RGXFWIF_LOG_GROUP_MAP_ENTRY type to create a lookup
+  table where needed. Keep log group names short, no more than 20 chars.
+*/
+#define RGXFWIF_LOG_GROUP_NAME_VALUE_MAP { "none",    RGXFWIF_LOG_TYPE_NONE }, \
+                                         { "main",    RGXFWIF_LOG_TYPE_GROUP_MAIN }, \
+                                         { "mts",     RGXFWIF_LOG_TYPE_GROUP_MTS }, \
+                                         { "cleanup", RGXFWIF_LOG_TYPE_GROUP_CLEANUP }, \
+                                         { "csw",     RGXFWIF_LOG_TYPE_GROUP_CSW }, \
+                                         { "bif",     RGXFWIF_LOG_TYPE_GROUP_BIF }, \
+                                         { "pm",      RGXFWIF_LOG_TYPE_GROUP_PM }, \
+                                         { "rtd",     RGXFWIF_LOG_TYPE_GROUP_RTD }, \
+                                         { "spm",     RGXFWIF_LOG_TYPE_GROUP_SPM }, \
+                                         { "pow",     RGXFWIF_LOG_TYPE_GROUP_POW }, \
+                                         { "hwr",     RGXFWIF_LOG_TYPE_GROUP_HWR }, \
+                                         { "hwp",     RGXFWIF_LOG_TYPE_GROUP_HWP }, \
+                                         { "rpm",     RGXFWIF_LOG_TYPE_GROUP_RPM }, \
+                                         { "dma",     RGXFWIF_LOG_TYPE_GROUP_DMA }, \
+                                         { "misc",    RGXFWIF_LOG_TYPE_GROUP_MISC }, \
+                                         { "vz",      RGXFWIF_LOG_TYPE_GROUP_VZ }, \
+                                         { "safety",  RGXFWIF_LOG_TYPE_GROUP_SAFETY }, \
+                                         { "verbose", RGXFWIF_LOG_TYPE_GROUP_VERBOSE }, \
+                                         { "customer",RGXFWIF_LOG_TYPE_GROUP_CUSTOMER }, \
+                                         { "debug",   RGXFWIF_LOG_TYPE_GROUP_DEBUG }
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s"
+
+/* Used in a print statement to display log group state, one per group */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST(types)  ((((types) & RGXFWIF_LOG_TYPE_GROUP_MAIN) != 0U)	?("main ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MTS) != 0U)		?("mts ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CLEANUP) != 0U)	?("cleanup ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CSW) != 0U)		?("csw ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_BIF) != 0U)		?("bif ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_PM) != 0U)		?("pm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RTD) != 0U)		?("rtd ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SPM) != 0U)		?("spm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_POW) != 0U)		?("pow ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWR) != 0U)		?("hwr ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWP) != 0U)		?("hwp ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RPM) != 0U)		?("rpm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DMA) != 0U)		?("dma ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MISC) != 0U)	?("misc ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VZ) != 0U)		?("vz ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SAFETY) != 0U)	?("safety ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VERBOSE) != 0U)	?("verbose ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CUSTOMER) != 0U)?("customer ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DEBUG) != 0U)	?("debug ")		:(""))
+
+/*!
+ ******************************************************************************
+ * Trace Buffer
+ *****************************************************************************/
+
+/*! Min, Max, and Default size of RGXFWIF_TRACEBUF_SPACE in DWords */
+#define RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS      8192U  /*  32KB */
+#define RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS 12000U  /* ~48KB */
+#define RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS     32768U  /* 128KB */
+
+#define RGXFW_TRACE_BUFFER_ASSERT_SIZE 200U
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+#define RGXFW_THREAD_NUM 2U
+#else
+#define RGXFW_THREAD_NUM 1U
+#endif
+
+typedef struct
+{
+	IMG_CHAR	szPath[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_CHAR	szInfo[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_UINT32	ui32LineNum;
+} UNCACHED_ALIGN RGXFWIF_FILE_INFO_BUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FILE_INFO_BUF) == 408,
+				"RGXFWIF_FILE_INFO_BUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @Defgroup SRVAndFWTracing Services and Firmware Tracing data interface
+ * @Brief The document groups/lists the data structures and the interfaces related to Services and Firmware Tracing
+ * @{
+ */
+
+/*!
+ * @Brief Firmware trace buffer details
+ */
+typedef struct
+{
+	IMG_UINT32                         ui32TracePointer;          /*!< Trace pointer (write index into Trace Buffer) */
+	IMG_UINT32                         ui32WrapCount;             /*!< Number of times the Trace Buffer has wrapped */
+
+	RGXFWIF_DEV_VIRTADDR               pui32RGXFWIfTraceBuffer;   /*!< Trace buffer address (FW address), to be used by firmware for writing into trace buffer */
+
+	RGXFWIF_FILE_INFO_BUF RGXFW_ALIGN  sAssertBuf;
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF_SPACE;
+
+/*! @} End of Defgroup SRVAndFWTracing */
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief Firmware trace control data
+ */
+typedef struct
+{
+	IMG_UINT32              ui32LogType;                  /*!< FW trace log group configuration */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[MAX_THREAD_NUM];  /*!< FW Trace buffer */
+#else
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[RGXFW_THREAD_NUM];  /*!< FW Trace buffer */
+#endif
+	IMG_UINT32              ui32TraceBufSizeInDWords;     /*!< FW Trace buffer size in dwords, Member initialised only when sTraceBuf is actually allocated
+															(in RGXTraceBufferInitOnDemandResources) */
+	IMG_UINT32              ui32TracebufFlags;            /*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_TRACEBUF) == 880,
+				"RGXFWIF_TRACEBUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Debug-info sub-fields */
+/* Bit 0: RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT bit from RGX_CR_EVENT_STATUS register */
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT        (0U)
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET          (1U << RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT)
+
+/* Bit 1: RGX_CR_BIF_MMU_ENTRY_PENDING bit from RGX_CR_BIF_MMU_ENTRY register */
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT     (1U)
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET       (1U << RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT)
+
+/* Bit 2: RGX_CR_SLAVE_EVENT register is non-zero */
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT          (2U)
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET            (1U << RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT)
+
+/* Bit 3-15: Unused bits */
+
+#define RGXFWT_DEBUG_INFO_STR_MAXLEN                  64
+#define RGXFWT_DEBUG_INFO_STR_PREPEND                 " (debug info: "
+#define RGXFWT_DEBUG_INFO_STR_APPEND                  ")"
+
+/* Table of debug info sub-field's masks and corresponding message strings
+ * to be appended to firmware trace
+ *
+ * Mask     : 16 bit mask to be applied to debug-info field
+ * String   : debug info message string
+ */
+
+#define RGXFWT_DEBUG_INFO_MSKSTRLIST \
+/*Mask,                                           String*/ \
+X(RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET,      "mmu pf") \
+X(RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET,   "mmu pending") \
+X(RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET,        "slave events")
+
+/* Firmware trace time-stamp field breakup */
+
+/* RGX_CR_TIMER register read (48 bits) value*/
+#define RGXFWT_TIMESTAMP_TIME_SHIFT                   (0U)
+#define RGXFWT_TIMESTAMP_TIME_CLRMSK                  (IMG_UINT64_C(0xFFFF000000000000))
+
+/* Extra debug-info (16 bits) */
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT             (48U)
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK            ~RGXFWT_TIMESTAMP_TIME_CLRMSK
+
+typedef struct
+{
+	IMG_UINT			bfOsState		: 3;
+	IMG_UINT			bfFLOk			: 1;
+	IMG_UINT			bfFLGrowPending	: 1;
+	IMG_UINT			bfReserved		: 27;
+} RGXFWIF_OS_RUNTIME_FLAGS;
+
+#define RGXFWIF_FWFAULTINFO_MAX		(8U)			/* Total number of FW fault logs stored */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64CRTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64OSTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64Data;
+	RGXFWIF_FILE_INFO_BUF	sFaultBuf;
+} UNCACHED_ALIGN RGX_FWFAULTINFO;
+
+#define RGXFWIF_POW_STATES \
+  X(RGXFWIF_POW_OFF)			/* idle and handshaked with the host (ready to full power down) */ \
+  X(RGXFWIF_POW_ON)				/* running HW commands */ \
+  X(RGXFWIF_POW_FORCED_IDLE)	/* forced idle */ \
+  X(RGXFWIF_POW_IDLE)			/* idle waiting for host handshake */
+
+typedef enum
+{
+#define X(NAME) NAME,
+	RGXFWIF_POW_STATES
+#undef X
+} RGXFWIF_POW_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_THREAD_NUM 2
+
+static_assert(RGXFW_THREAD_NUM <= MAX_THREAD_NUM,
+				"RGXFW_THREAD_NUM is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Firmware HWR states */
+#define RGXFWIF_HWR_HARDWARE_OK			(IMG_UINT32_C(0x1) << 0U)	/*!< The HW state is ok or locked up */
+#define RGXFWIF_HWR_RESET_IN_PROGRESS	(IMG_UINT32_C(0x1) << 1U)	/*!< Tells if a HWR reset is in progress */
+#define RGXFWIF_HWR_GENERAL_LOCKUP		(IMG_UINT32_C(0x1) << 3U)	/*!< A DM unrelated lockup has been detected */
+#define RGXFWIF_HWR_DM_RUNNING_OK		(IMG_UINT32_C(0x1) << 4U)	/*!< At least one DM is running without being close to a lockup */
+#define RGXFWIF_HWR_DM_STALLING			(IMG_UINT32_C(0x1) << 5U)	/*!< At least one DM is close to lockup */
+#define RGXFWIF_HWR_FW_FAULT			(IMG_UINT32_C(0x1) << 6U)	/*!< The FW has faulted and needs to restart */
+#define RGXFWIF_HWR_RESTART_REQUESTED	(IMG_UINT32_C(0x1) << 7U)	/*!< The FW has requested the host to restart it */
+
+#define RGXFWIF_PHR_MODE_OFF			(0UL)
+#define RGXFWIF_PHR_MODE_RD_RESET		(1UL)
+
+typedef IMG_UINT32 RGXFWIF_HWR_STATEFLAGS;
+
+typedef IMG_UINT32 RGXFWIF_HWR_RECOVERYFLAGS;
+
+/*! @Brief Firmware system data shared with the Host driver */
+typedef struct
+{
+	IMG_UINT32                 ui32ConfigFlags;                       /*!< Configuration flags from host */
+	IMG_UINT32                 ui32ConfigFlagsExt;                    /*!< Extended configuration flags from host */
+	volatile RGXFWIF_POW_STATE ePowState;
+	struct {
+		volatile IMG_UINT32        ui32HWPerfRIdx;
+		volatile IMG_UINT32        ui32HWPerfWIdx;
+		volatile IMG_UINT32        ui32HWPerfWrapCount;
+	} sHWPerfCtrl; /* Struct used to inval/flush HWPerfCtrl members */
+	IMG_UINT32                 ui32HWPerfSize;                        /*!< Constant after setup, needed in FW */
+	IMG_UINT32                 ui32HWPerfDropCount;                   /*!< The number of times the FW drops a packet due to buffer full */
+
+	/* ui32HWPerfUt, ui32FirstDropOrdinal, ui32LastDropOrdinal only valid when FW is built with
+	 * RGX_HWPERF_UTILIZATION & RGX_HWPERF_DROP_TRACKING defined in rgxfw_hwperf.c */
+	IMG_UINT32                 ui32HWPerfUt;                          /*!< Buffer utilisation, high watermark of bytes in use */
+	IMG_UINT32                 ui32FirstDropOrdinal;                  /*!< The ordinal of the first packet the FW dropped */
+	IMG_UINT32                 ui32LastDropOrdinal;                   /*!< The ordinal of the last packet the FW dropped */
+	RGXFWIF_OS_RUNTIME_FLAGS   asOsRuntimeFlagsMirror[RGXFW_MAX_NUM_OSIDS];/*!< State flags for each Operating System mirrored from Fw coremem */
+	RGX_FWFAULTINFO            sFaultInfo[RGXFWIF_FWFAULTINFO_MAX];   /*!< Firmware fault info */
+	IMG_UINT32                 ui32FWFaults;                          /*!< Firmware faults count */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32                 aui32CrPollAddr[MAX_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[MAX_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[MAX_THREAD_NUM];    /*!< Failed poll count */
+#else
+	IMG_UINT32                 aui32CrPollAddr[RGXFW_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[RGXFW_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[RGXFW_THREAD_NUM];    /*!< Failed poll count */
+#endif
+	IMG_UINT64 RGXFW_ALIGN     ui64StartIdleTime;
+	IMG_UINT32                 aui32TSMirror[RGXFW_MAX_NUM_OSIDS];    /*!< Array of time slice per OS Mirrored from the FW */
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define RGXFWIF_STATS_FRAMEWORK_LINESIZE    (8)
+#define RGXFWIF_STATS_FRAMEWORK_MAX         (2048*RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+	IMG_UINT32 RGXFW_ALIGN     aui32FWStatsBuf[RGXFWIF_STATS_FRAMEWORK_MAX];
+#endif
+	RGXFWIF_HWR_STATEFLAGS     ui32HWRStateFlags; /*!< Firmware's Current HWR state */
+	RGXFWIF_HWR_RECOVERYFLAGS  aui32HWRRecoveryFlags[RGXFWIF_DM_MAX]; /*!< Each DM's HWR state */
+	IMG_UINT32                 ui32FwSysDataFlags;                      /*!< Compatibility and other flags */
+	IMG_UINT32                 ui32McConfig;                            /*!< Identify whether MC config is P-P or P-S */
+	IMG_UINT32                 ui32MemFaultCheck;                       /*!< Device mem fault check on PCI systems */
+} UNCACHED_ALIGN RGXFWIF_SYSDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_SYSDATA) == 3624,
+				"RGXFWIF_SYSDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER == 3624");
+#endif
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+#define PVR_SLR_LOG_ENTRIES 10U
+#define PVR_SLR_LOG_STRLEN  30 /*!< MAX_CLIENT_CCB_NAME not visible to this header */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64Timestamp;
+	IMG_UINT32				ui32FWCtxAddr;
+	IMG_UINT32				ui32NumUFOs;
+	IMG_CHAR				aszCCBName[PVR_SLR_LOG_STRLEN];
+} UNCACHED_ALIGN RGXFWIF_SLR_ENTRY;
+#endif
+
+#define RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS (128U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware per-os data and configuration
+ */
+typedef struct
+{
+	IMG_UINT32                 ui32FwOsConfigFlags;                   /*!< Configuration flags from an OS */
+	IMG_UINT32                 ui32FWSyncCheckMark;                   /*!< Markers to signal that the host should perform a full sync check */
+	IMG_UINT32                 ui32HostSyncCheckMark;                  /*!< Markers to signal that the Firmware should perform a full sync check */
+#if defined(PVRSRV_STALLED_CCB_ACTION) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32                 ui32ForcedUpdatesRequested;
+	IMG_UINT8                  ui8SLRLogWp;
+	RGXFWIF_SLR_ENTRY          sSLRLogFirst;
+	RGXFWIF_SLR_ENTRY          sSLRLog[PVR_SLR_LOG_ENTRIES];
+	IMG_UINT64 RGXFW_ALIGN     ui64LastForcedUpdateTime;
+#endif
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	volatile IMG_UINT32        aui32InterruptCount[MAX_THREAD_NUM]; /*!< Interrupt count from Threads > */
+#else
+	volatile IMG_UINT32        aui32InterruptCount[RGXFW_THREAD_NUM]; /*!< Interrupt count from Threads > */
+#endif
+	IMG_UINT32                 ui32KCCBCmdsExecuted;                  /*!< Executed Kernel CCB command count */
+	RGXFWIF_DEV_VIRTADDR       sPowerSync;                            /*!< Sync prim used to signal the host the power off state */
+	IMG_UINT32                 ui32FwOsDataFlags;                       /*!< Compatibility and other flags */
+	IMG_UINT32                 aui32OfflineBuffer[RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS];
+} UNCACHED_ALIGN RGXFWIF_OSDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_OSDATA) == 584,
+				"RGXFWIF_OSDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+/*!
+ ******************************************************************************
+ * HWR Data
+ *****************************************************************************/
+/*!
+ * @Defgroup HWRInfo FW HWR shared data interface
+ * @Brief Types grouping data structures and defines used in realising the HWR record.
+ * @{
+ */
+
+#define RGXFW_PROCESS_NAME_LEN	(16)
+
+/*! @Brief HWR Lockup types */
+typedef enum
+{
+	RGX_HWRTYPE_UNKNOWNFAILURE = 0, /*!< Unknown failure */
+	RGX_HWRTYPE_OVERRUN        = 1, /*!< DM overrun */
+	RGX_HWRTYPE_POLLFAILURE    = 2, /*!< Poll failure */
+	RGX_HWRTYPE_BIF0FAULT      = 3, /*!< BIF0 fault */
+	RGX_HWRTYPE_BIF1FAULT      = 4, /*!< BIF1 fault */
+	RGX_HWRTYPE_TEXASBIF0FAULT = 5, /*!< TEXASBIF0 fault */
+	RGX_HWRTYPE_MMUFAULT       = 6, /*!< MMU fault */
+	RGX_HWRTYPE_MMUMETAFAULT   = 7, /*!< MMU META fault */
+	RGX_HWRTYPE_MIPSTLBFAULT   = 8, /*!< MIPS TLB fault */
+	RGX_HWRTYPE_ECCFAULT       = 9, /*!< ECC fault */
+	RGX_HWRTYPE_MMURISCVFAULT  = 10, /*!< MMU RISCV fault */
+} RGX_HWRTYPE;
+
+#define RGXFWIF_HWRTYPE_BIF_BANK_GET(eHWRType) (((eHWRType) == RGX_HWRTYPE_BIF0FAULT) ? 0 : 1)
+
+#define RGXFWIF_HWRTYPE_PAGE_FAULT_GET(eHWRType) ((((eHWRType) == RGX_HWRTYPE_BIF0FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_BIF1FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_TEXASBIF0FAULT) ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUFAULT)       ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUMETAFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MIPSTLBFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMURISCVFAULT)) ? true : false)
+
+/************************
+ *  GPU HW error codes  *
+ ************************/
+typedef enum
+{
+	RGX_HW_ERR_NA = 0x0,
+	RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL = 0x101,
+} RGX_HW_ERR;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFReqStatus; /*!< BIF request status */
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFMMUStatus; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_BIFINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32FaultGPU; /*!< ECC fault in GPU */
+} RGX_ECCINFO;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		aui64MMUStatus[2]; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_MMUINFO;
+
+typedef struct
+{
+	IMG_UINT32	ui32ThreadNum; /*!< Thread ID performing poll operation */
+	IMG_UINT32	ui32CrPollAddr; /*!< CR Poll Address */
+	IMG_UINT32	ui32CrPollMask; /*!< CR Poll mask */
+	IMG_UINT32	ui32CrPollLastValue; /*!< CR Poll last value */
+	IMG_UINT64	RGXFW_ALIGN ui64Reserved;
+} UNCACHED_ALIGN RGX_POLLINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32BadVAddr; /*!< VA address */
+	IMG_UINT32 ui32EntryLo;
+} RGX_TLBINFO;
+
+/*! @Brief Structure to keep information specific to a lockup e.g. DM, timer, lockup type etc. */
+typedef struct
+{
+	union
+	{
+		RGX_BIFINFO  sBIFInfo; /*!< BIF failure details */
+		RGX_MMUINFO  sMMUInfo; /*!< MMU failure details */
+		RGX_POLLINFO sPollInfo; /*!< Poll failure details */
+		RGX_TLBINFO  sTLBInfo; /*!< TLB failure details */
+		RGX_ECCINFO  sECCInfo; /*!< ECC failure details */
+	} uHWRData;
+
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimer; /*!< Timer value at the time of lockup */
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimer; /*!< OS timer value at the time of lockup */
+	IMG_UINT32             ui32FrameNum; /*!< Frame number of the workload */
+	IMG_UINT32             ui32PID; /*!< PID belonging to the workload */
+	IMG_UINT32             ui32ActiveHWRTData; /*!< HWRT data of the workload */
+	IMG_UINT32             ui32HWRNumber; /*!< HWR number */
+	IMG_UINT32             ui32EventStatus; /*!< Core specific event status register at the time of lockup */
+	IMG_UINT32             ui32HWRRecoveryFlags; /*!< DM state flags */
+	RGX_HWRTYPE            eHWRType; /*!< Type of lockup */
+	RGXFWIF_DM             eDM; /*!< Recovery triggered for the DM */
+	IMG_UINT32             ui32CoreID; /*!< Core ID of the GPU */
+	RGX_HW_ERR             eHWErrorCode; /*!< Error code used to determine HW fault */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeOfKick; /*!< Workload kick time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetStart; /*!< HW reset start time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetFinish; /*!< HW reset stop time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeFreelistReady; /*!< freelist ready time on the last HWR */
+	IMG_CHAR   RGXFW_ALIGN szProcName[RGXFW_PROCESS_NAME_LEN]; /*!< User process name */
+} UNCACHED_ALIGN RGX_HWRINFO;
+
+#define RGXFWIF_HWINFO_MAX_FIRST 8U							/* Number of first HWR logs recorded (never overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX_LAST 8U							/* Number of latest HWR logs (older logs are overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX (RGXFWIF_HWINFO_MAX_FIRST + RGXFWIF_HWINFO_MAX_LAST)	/* Total number of HWR logs stored in a buffer */
+#define RGXFWIF_HWINFO_LAST_INDEX (RGXFWIF_HWINFO_MAX - 1U)	/* Index of the last log in the HWR log buffer */
+
+/*! @Brief Firmware HWR information structure allocated by the Services and used by the Firmware to update recovery information. */
+typedef struct
+{
+	RGX_HWRINFO sHWRInfo[RGXFWIF_HWINFO_MAX]; /*!< Max number of recovery record */
+	IMG_UINT32  ui32HwrCounter; /*!< HWR counter used in FL reconstruction */
+	IMG_UINT32  ui32WriteIndex; /*!< Index for updating recovery information in sHWRInfo */
+	IMG_UINT32  ui32DDReqCount; /*!< Count of DebugDump requested to the host after recovery */
+	IMG_UINT32  ui32HWRInfoBufFlags; /* Compatibility and other flags */
+	IMG_UINT32  aui32HwrDmLockedUpCount[RGXFWIF_DM_MAX]; /*!< Lockup count for each DM */
+	IMG_UINT32  aui32HwrDmOverranCount[RGXFWIF_DM_MAX]; /*!< Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmRecoveredCount[RGXFWIF_DM_MAX]; /*!< Lockup + Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmFalseDetectCount[RGXFWIF_DM_MAX]; /*!< False lockup detection count for each DM */
+} UNCACHED_ALIGN RGXFWIF_HWRINFOBUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRINFOBUF) == 2336,
+				"RGXFWIF_HWRINFOBUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of HWRInfo */
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COMMONCTX_STATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RF_CMD;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWMEMCONTEXT;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Freelist holding usage state of the Parameter Buffers
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN psFreeListDevVAddr;	/*!< Freelist page table base */
+	IMG_UINT64		RGXFW_ALIGN ui64CurrentDevVAddr;/*!< Freelist page table entry for current free page */
+	IMG_UINT32		ui32CurrentStackTop;		/*!< Freelist current free page */
+	IMG_UINT32		ui32MaxPages;			/*!< Max no. of pages can be added to the freelist */
+	IMG_UINT32		ui32GrowPages;			/*!< No pages to add in each freelist grow */
+	IMG_UINT32		ui32CurrentPages;		/*!< Total no. of pages made available to the PM HW */
+	IMG_UINT32		ui32AllocatedPageCount;		/*!< No. of pages allocated by PM HW */
+	IMG_UINT32		ui32AllocatedMMUPageCount;	/*!< No. of pages allocated for GPU MMU for PM*/
+#if defined(SUPPORT_SHADOW_FREELISTS) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32		ui32HWRCounter;
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;
+#endif
+	IMG_UINT32		ui32FreeListID;			/*!< Unique Freelist ID */
+	IMG_BOOL		bGrowPending;			/*!< Freelist grow is pending */
+	IMG_UINT32		ui32ReadyPages;			/*!< Reserved pages to be used only on PM OOM event */
+	IMG_UINT32		ui32FreelistFlags;		/*!< Compatibility and other flags */
+#if defined(SUPPORT_AGP) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32		ui32PmGlobalPb;			/*!< PM Global PB on which Freelist is loaded */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FREELIST;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FREELIST) == 64,
+				"RGXFWIF_FREELIST is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct {RGXFWIF_DEV_VIRTADDR sNext;
+                RGXFWIF_DEV_VIRTADDR sPrev;}	RGXFW_DLLIST_NODE;
+
+#define RGXFWIF_MAX_NUM_CANCEL_REQUESTS		(8U)		/* Maximum number of workload cancellation requests */
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware Common Context (or FWCC)
+ */
+typedef struct RGXFWIF_FWCOMMONCONTEXT_
+{
+	/* CCB details for this firmware context */
+	PRGXFWIF_CCCB_CTL		psCCBCtl;				/*!< CCB control */
+	PRGXFWIF_CCCB			psCCB;					/*!< CCB base */
+
+	/* Context suspend state */
+	PRGXFWIF_COMMONCTX_STATE	RGXFW_ALIGN psContextState;		/*!< TA/3D context suspend state, read/written by FW */
+
+	/* Flags e.g. for context switching */
+	IMG_UINT32				ui32FWComCtxFlags;
+	IMG_INT32				i32Priority;  /*!< Priority level */
+	IMG_UINT32				ui32PrioritySeqNum;
+
+	/* Framework state */
+	PRGXFWIF_RF_CMD			RGXFW_ALIGN psRFCmd;		/*!< Register updates for Framework */
+
+	/* Misc and compatibility flags */
+	IMG_UINT32				ui32CompatFlags;
+
+	/* Statistic updates waiting to be passed back to the host... */
+	IMG_INT32				i32StatsNumStores;		/*!< Number of stores on this context since last update */
+	IMG_INT32				i32StatsNumOutOfMemory;		/*!< Number of OOMs on this context since last update */
+	IMG_INT32				i32StatsNumPartialRenders;	/*!< Number of PRs on this context since last update */
+	RGXFWIF_DM				eDM;				/*!< Data Master type */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN  sBufStalledNode;			/*!< List entry for the buffer stalled list */
+	IMG_UINT64				RGXFW_ALIGN  ui64CBufQueueCtrlAddr;	/*!< Address of the circular buffer queue pointers */
+
+	IMG_UINT64				RGXFW_ALIGN  ui64RobustnessAddress;
+	IMG_UINT32				ui32MaxDeadlineMS;			/*!< Max HWR deadline limit in ms */
+	bool					bReadOffsetNeedsReset;			/*!< Following HWR circular buffer read-offset needs resetting */
+
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sWaitingNode;		/*!< List entry for the waiting list */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sRunNode;			/*!< List entry for the run list */
+	RGXFWIF_UFO				sLastFailedUFO;			/*!< UFO that last failed (or NULL) */
+
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;					/*!< Memory context */
+
+	/* References to the host side originators */
+	IMG_UINT32				ui32ServerCommonContextID;	/*!< the Server Common Context */
+	IMG_UINT32				ui32PID;			/*!< associated process ID */
+
+	IMG_BOOL				bGeomOOMDisabled;		/*!< True when Geom DM OOM is not allowed */
+	IMG_CHAR				szProcName[RGXFW_PROCESS_NAME_LEN];	/*!< User process name */
+	IMG_UINT32				ui32DeferCount;		/*!< Number of context defers before forced scheduling of context */
+	IMG_UINT32				aui32FirstIntJobRefToCancel[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the beginning of range of IntJobRefs at and above which workloads will be discarded */
+	IMG_UINT32				aui32FirstValidIntJobRef[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the end of range of IntJobRef below which workloads will be discarded */
+	IMG_BOOL				bCancelRangesActive;	/*!< True if any active ranges in aui32FirstIntJobRefToCancel and aui32FirstValidIntJobRef arrays */
+	IMG_BOOL				bLastKickedCmdWasSafetyOnly;
+	IMG_UINT32				ui32UID;			/*!< associated process UID used in FW managed gpu work period hwperf events */
+} UNCACHED_ALIGN RGXFWIF_FWCOMMONCONTEXT;
+
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) <= 256U,
+              "Size of structure RGXFWIF_FWCOMMONCONTEXT exceeds maximum expected size.");
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) == 168,
+				"RGXFWIF_FWCOMMONCONTEXT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ ******************************************************************************
+ * HWRTData
+ *****************************************************************************/
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FREELIST;
+
+/* HWRTData flags */
+/* Deprecated flags 1:0 */
+#define HWRTDATA_HAS_LAST_TA              (IMG_UINT32_C(1) << 2)
+#define HWRTDATA_PARTIAL_RENDERED         (IMG_UINT32_C(1) << 3)
+#define HWRTDATA_DISABLE_TILE_REORDERING  (IMG_UINT32_C(1) << 4)
+#define HWRTDATA_NEED_BRN65101_BLIT       (IMG_UINT32_C(1) << 5)
+#define HWRTDATA_FIRST_BRN65101_STRIP     (IMG_UINT32_C(1) << 6)
+#define HWRTDATA_NEED_BRN67182_2ND_RENDER (IMG_UINT32_C(1) << 7)
+#if defined(SUPPORT_AGP)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT0    (IMG_UINT32_C(1) << 8)
+#if defined(SUPPORT_AGP4)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT1    (IMG_UINT32_C(1) << 9)
+#endif
+#define HWRTDATA_GEOM_NEEDS_RESUME        (IMG_UINT32_C(1) << 10)
+#endif
+#if defined(SUPPORT_TRP)
+#define HWRTDATA_GEOM_TRP_IN_PROGRESS          (IMG_UINT32_C(1) << 9)
+#endif
+
+typedef enum
+{
+	RGXFWIF_RTDATA_STATE_NONE = 0,
+	RGXFWIF_RTDATA_STATE_KICKTA,
+	RGXFWIF_RTDATA_STATE_KICKTAFIRST,
+	RGXFWIF_RTDATA_STATE_TAFINISHED,
+	RGXFWIF_RTDATA_STATE_KICK3D,
+	RGXFWIF_RTDATA_STATE_3DFINISHED,
+	RGXFWIF_RTDATA_STATE_3DCONTEXTSTORED,
+	RGXFWIF_RTDATA_STATE_TAOUTOFMEM,
+	RGXFWIF_RTDATA_STATE_PARTIALRENDERFINISHED,
+	/* In case of HWR, we can't set the RTDATA state to NONE,
+	 * as this will cause any TA to become a first TA.
+	 * To ensure all related TA's are skipped, we use the HWR state */
+	RGXFWIF_RTDATA_STATE_HWR,
+	RGXFWIF_RTDATA_STATE_UNKNOWN = 0x7FFFFFFFU
+} RGXFWIF_RTDATA_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_FREELISTS_SIZE 3
+
+static_assert(RGXFW_MAX_FREELISTS <= MAX_FREELISTS_SIZE,
+				"RGXFW_MAX_FREELISTS is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct
+{
+	IMG_BOOL							bTACachesNeedZeroing;
+
+	IMG_UINT32							ui32ScreenPixelMax;
+	IMG_UINT64							RGXFW_ALIGN ui64MultiSampleCtl;
+	IMG_UINT64							ui64FlippedMultiSampleCtl;
+	IMG_UINT32							ui32TPCStride;
+	IMG_UINT32							ui32TPCSize;
+	IMG_UINT32							ui32TEScreen;
+	IMG_UINT32							ui32MTileStride;
+	IMG_UINT32							ui32TEAA;
+	IMG_UINT32							ui32TEMTILE1;
+	IMG_UINT32							ui32TEMTILE2;
+	IMG_UINT32							ui32ISPMergeLowerX;
+	IMG_UINT32							ui32ISPMergeLowerY;
+	IMG_UINT32							ui32ISPMergeUpperX;
+	IMG_UINT32							ui32ISPMergeUpperY;
+	IMG_UINT32							ui32ISPMergeScaleX;
+	IMG_UINT32							ui32ISPMergeScaleY;
+	IMG_UINT32							uiRgnHeaderSize;
+	IMG_UINT32							ui32ISPMtileSize;
+} UNCACHED_ALIGN RGXFWIF_HWRTDATA_COMMON;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRTDATA_COMMON) == 88,
+				"RGXFWIF_HWRTDATA_COMMON is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct
+{
+	IMG_UINT32           ui32RenderTargetIndex;		//Render number
+	IMG_UINT32           ui32CurrentRenderTarget;	//index in RTA
+	IMG_UINT32           ui32ActiveRenderTargets;	//total active RTs
+	IMG_UINT32           ui32CumulActiveRenderTargets;   //total active RTs from the first TA kick, for OOM
+	RGXFWIF_DEV_VIRTADDR sValidRenderTargets;  //Array of valid RT indices
+	RGXFWIF_DEV_VIRTADDR sRTANumPartialRenders;  //Array of number of occurred partial renders per render target
+	IMG_UINT32           ui32MaxRTs;   //Number of render targets in the array
+	IMG_UINT32           ui32RTACtlFlags; /* Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_RTA_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_RTA_CTL) == 32,
+				"RGXFWIF_RTA_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Render Target data i.e. HWRTDATA used to hold the PM context
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psPMMListDevVAddr;			/*!< MList Data Store */
+
+	IMG_UINT64			RGXFW_ALIGN ui64VCECatBase[1];			/*!< VCE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64VCELastCatBase[1];
+	IMG_UINT64			RGXFW_ALIGN ui64TECatBase[1];			/*!< TE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64TELastCatBase[1];
+	IMG_UINT64			RGXFW_ALIGN ui64AlistCatBase;			/*!< Alist Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64AlistLastCatBase;
+
+	IMG_UINT64			RGXFW_ALIGN ui64PMAListStackPointer;		/*!< Freelist page table entry for current Mlist page */
+	IMG_UINT32			ui32PMMListStackPointer;			/*!< Current Mlist page */
+
+	RGXFWIF_DEV_VIRTADDR		sHWRTDataCommonFwAddr;				/*!< Render target dimension dependent data */
+
+	IMG_UINT32			ui32HWRTDataFlags;
+	RGXFWIF_RTDATA_STATE		eState;						/*!< Current workload processing state of HWRTDATA */
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[MAX_FREELISTS_SIZE];	/*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[MAX_FREELISTS_SIZE];
+#else
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[RGXFW_MAX_FREELISTS];	/*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[RGXFW_MAX_FREELISTS];
+#endif
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psVHeapTableDevVAddr;		/*!< VHeap table base */
+
+	RGXFWIF_RTA_CTL			sRTACtl;					/*!< Render target array data */
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sTailPtrsDevVAddr;			/*!< Tail pointers base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sMacrotileArrayDevVAddr;		/*!< Macrotiling array base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sRgnHeaderDevVAddr;			/*!< Region headers base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sRTCDevVAddr;			/*!< Render target cache base */
+#if defined(RGX_FIRMWARE)
+	struct RGXFWIF_FWCOMMONCONTEXT_* RGXFW_ALIGN psOwnerGeom;
+#else
+	RGXFWIF_DEV_VIRTADDR		RGXFW_ALIGN pui32OwnerGeomNotUsedByHost;
+#endif
+#if defined(SUPPORT_TRP) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32			ui32KickFlagsCopy;
+	IMG_UINT32			ui32TEPageCopy;
+	IMG_UINT32			ui32VCEPageCopy;
+#endif
+#if defined(SUPPORT_AGP) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_BOOL			bTACachesNeedZeroing;
+#endif
+
+	RGXFWIF_CLEANUP_CTL		RGXFW_ALIGN_DCACHEL sCleanupState;					/*!< Render target clean up state */
+} RGXFW_ALIGN_DCACHEL RGXFWIF_HWRTDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRTDATA) == 256,
+				"RGXFWIF_HWRTDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Sync_checkpoint firmware object.
+ * This is the FW-addressable structure used to hold the sync checkpoint's
+ * state and other information which needs to be accessed by the firmware.
+ */
+typedef struct
+{
+	IMG_UINT32	ui32State;          /*!< Holds the current state of the sync checkpoint */
+	IMG_UINT32	ui32FwRefCount;     /*!< Holds the FW reference count (num of fences/updates processed) */
+} SYNC_CHECKPOINT_FW_OBJ;
+
+/* Bit mask Firmware can use to test if a checkpoint has signalled or errored */
+#define SYNC_CHECKPOINT_SIGNALLED_MASK (0x1 << 0)
+
+typedef enum
+{
+	RGXFWIF_REG_CFG_TYPE_PWR_ON=0,      /* Sidekick power event */
+	RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,   /* Rascal / dust power event */
+	RGXFWIF_REG_CFG_TYPE_TA,            /* TA kick */
+	RGXFWIF_REG_CFG_TYPE_3D,            /* 3D kick */
+	RGXFWIF_REG_CFG_TYPE_CDM,           /* Compute kick */
+	RGXFWIF_REG_CFG_TYPE_TLA,           /* TLA kick */
+	RGXFWIF_REG_CFG_TYPE_TDM,           /* TDM kick */
+	RGXFWIF_REG_CFG_TYPE_ALL            /* Applies to all types. Keep as last element */
+} RGXFWIF_REG_CFG_TYPE;
+
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_OFFSET_BYTES                 RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES               (1U << 19)
+
+#define RGX_HEAP_KM_USC_RESERVED_SIZE                              RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES
+#define RGX_HEAP_USC_RESERVED_TOTAL_SIZE                           RGX_HEAP_UM_USC_RESERVED_SIZE + RGX_HEAP_KM_USC_RESERVED_SIZE
+
+#endif /*  RGX_FWIF_SHARED_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_shared.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_heaps.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_heaps.h
new file mode 100644
index 000000000000..f8fcfe1fe27e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_heaps.h
@@ -0,0 +1,67 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAPS_H)
+#define RGX_HEAPS_H
+
+/*
+  Identify heaps by their names
+*/
+#define RGX_GENERAL_SVM_HEAP_IDENT          "General SVM"               /*!< RGX General SVM (shared virtual memory) Heap Identifier */
+#define RGX_GENERAL_HEAP_IDENT              "General"                   /*!< RGX General Heap Identifier */
+#define RGX_GENERAL_NON4K_HEAP_IDENT        "General NON-4K"            /*!< RGX General non-4K Heap Identifier */
+#define RGX_PDSCODEDATA_HEAP_IDENT          "PDS Code and Data"         /*!< RGX PDS Code/Data Heap Identifier */
+#define RGX_USCCODE_HEAP_IDENT              "USC Code"                  /*!< RGX USC Code Heap Identifier */
+#define RGX_VK_CAPT_REPLAY_HEAP_IDENT       "Vulkan Capture Replay"     /*!< RGX Vulkan capture replay buffer Heap Identifier */
+#define RGX_FBCDC_HEAP_IDENT                "FBCDC"                     /*!< RGX FBCDC State Table Heap Identifier */
+#define RGX_FBCDC_LARGE_HEAP_IDENT          "Large FBCDC"               /*!< RGX Large FBCDC State Table Heap Identifier */
+#define RGX_CMP_MISSION_RMW_HEAP_IDENT      "Compute Mission RMW"       /*!< Compute Mission RMW Heap Identifier */
+#define RGX_CMP_SAFETY_RMW_HEAP_IDENT       "Compute Safety RMW"        /*!< Compute Safety RMW Heap Identifier */
+#define RGX_TEXTURE_STATE_HEAP_IDENT        "Texture State"             /*!< Texture State Heap Identifier */
+#define RGX_VISIBILITY_TEST_HEAP_IDENT      "Visibility Test"           /*!< Visibility Test Heap Identifier */
+
+/* Services client internal heap identification */
+#define RGX_RGNHDR_BRN_63142_HEAP_IDENT     "RgnHdr BRN63142"           /*!< RGX RgnHdr BRN63142 Heap Identifier */
+#define RGX_TQ3DPARAMETERS_HEAP_IDENT       "TQ3DParameters"            /*!< RGX TQ 3D Parameters Heap Identifier */
+#define RGX_MMU_INIA_BRN_65273_HEAP_IDENT   "MMU INIA BRN65273"         /*!< MMU BRN65273 Heap A Identifier */
+#define RGX_MMU_INIB_BRN_65273_HEAP_IDENT   "MMU INIB BRN65273"         /*!< MMU BRN65273 Heap B Identifier */
+#endif /* RGX_HEAPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_hwperf.h
new file mode 100644
index 000000000000..d164c20a2417
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_hwperf.h
@@ -0,0 +1,483 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_H_
+#define RGX_HWPERF_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_hwperf_common.h"
+
+#if !defined(__KERNEL__)
+/* User-mode and Firmware definitions only */
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+
+/* HWPerf interface assumption checks */
+static_assert(RGX_FEATURE_NUM_CLUSTERS <= 16U, "Cluster count too large for HWPerf protocol definition");
+
+/*! The number of indirectly addressable TPU_MSC blocks in the GPU */
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST MAX(((IMG_UINT32)RGX_FEATURE_NUM_CLUSTERS >> 1), 1U)
+
+/*! The number of indirectly addressable USC blocks in the GPU */
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER (RGX_FEATURE_NUM_CLUSTERS)
+
+# if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+
+ /*! Defines the number of performance counter blocks that are directly
+  * addressable in the RGX register map for S. */
+#  define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      1 /* JONES */
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       (RGX_NUM_PHANTOMS)
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      1 /* BLACKPEARL */
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         2 /* TPU, TEXAS */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 2 /* USC, PBE */
+
+# elif defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+
+  /*! Defines the number of performance counter blocks that are directly
+   * addressable in the RGX register map. */
+#   define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS    2 /* TORNADO, TA */
+
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       (RGX_NUM_PHANTOMS)
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      2 /* RASTER, TEXAS */
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         1 /* TPU */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 1 /* USC */
+
+# else /* !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && !defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) i.e. S6 */
+
+ /*! Defines the number of performance counter blocks that are
+  * addressable in the RGX register map for Series 6. */
+#  define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      3 /* TA, RASTER, HUB */
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       0 /* PHANTOM is not there in Rogue1. Just using it to keep naming same as later series (RogueXT n Rogue XT+) */
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      0
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         1 /* TPU */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 1 /* USC */
+
+# endif
+
+/*! The number of performance counters in each layout block defined for UM/FW code */
+#if defined(RGX_FEATURE_CLUSTER_GROUPING)
+  #define RGX_HWPERF_CNTRS_IN_BLK 6
+ #else
+  #define RGX_HWPERF_CNTRS_IN_BLK 4
+#endif
+
+#endif /* #if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER) */
+#else /* defined(__KERNEL__) */
+/* Kernel/server definitions - not used, hence invalid definitions */
+
+# define RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC 0xFF
+
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST    RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+
+# define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_INDIRECT_BY_PHANTOM       RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_NONDUST_BLKS      RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_DUST_BLKS         RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+
+#endif
+
+/*! The number of custom non-mux counter blocks supported */
+#define RGX_HWPERF_MAX_CUSTOM_BLKS 5U
+
+/*! The number of counters supported in each non-mux counter block */
+#define RGX_HWPERF_MAX_CUSTOM_CNTRS 8U
+
+
+/******************************************************************************
+ * Data Stream Common Types
+ *****************************************************************************/
+
+/*! All the Data Masters HWPerf is aware of. When a new DM is added to this
+ * list, it should be appended at the end to maintain backward compatibility
+ * of HWPerf data.
+ */
+#define	RGX_HWPERF_DM_GP                  0x00000000U
+#define	RGX_HWPERF_DM_2D                  0x00000001U
+#define	RGX_HWPERF_DM_TA                  0x00000002U
+#define	RGX_HWPERF_DM_3D                  0x00000003U
+#define	RGX_HWPERF_DM_CDM                 0x00000004U
+#define	RGX_HWPERF_DM_RTU                 0x00000005U
+#define	RGX_HWPERF_DM_SHG                 0x00000006U
+#define	RGX_HWPERF_DM_TDM                 0x00000007U
+
+#define	RGX_HWPERF_DM_LAST                0x00000008U
+
+#define	RGX_HWPERF_DM_INVALID             0x1FFFFFFFU
+
+/******************************************************************************
+ * API Types
+ *****************************************************************************/
+
+/*! Counter block IDs for all the hardware blocks with counters.
+ * Directly addressable blocks must have a value between 0..15 [0..0xF].
+ * Indirect groups have following encoding:
+ * First hex digit (LSB) represents a unit number within the group
+ * and the second hex digit represents the group number.
+ * Group 0 is the direct group, all others are indirect groups.
+ */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_ID;
+
+/*! Directly addressable counter blocks */
+#if defined(DOXYGEN)
+/*! _RGX_HWPERF_CNTBLK_ID */
+#endif
+#define	RGX_CNTBLK_ID_TA			 0x0000U
+#define	RGX_CNTBLK_ID_RASTER		 0x0001U /*!< Non-cluster grouping cores */
+#define	RGX_CNTBLK_ID_HUB			 0x0002U /*!< Non-cluster grouping cores */
+#define	RGX_CNTBLK_ID_TORNADO		 0x0003U /*!< XT cores */
+#define	RGX_CNTBLK_ID_JONES			 0x0004U /*!< S7 cores */
+#define	RGX_CNTBLK_ID_DIRECT_LAST	 0x0005U /*!< Indirect blocks start from here */
+#define	RGX_CNTBLK_ID_BF_DEPRECATED	 0x0005U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_BT_DEPRECATED	 0x0006U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_RT_DEPRECATED	 0x0007U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_SH_DEPRECATED	 0x0008U /*!< Ray tracing unit (DEPRECATED) */
+
+
+/*! Indirectly addressable counter blocks. DA blocks indicate counter blocks
+ *  where the counter registers are directly accessible
+ */
+#define	RGX_CNTBLK_ID_TPU_MCU0		 0x0010U /*!< Addressable by Dust */
+#define RGX_CNTBLK_ID_TPU_MCU0_DA	 0x8010U
+#define	RGX_CNTBLK_ID_TPU_MCU1		 0x0011U
+#define RGX_CNTBLK_ID_TPU_MCU1_DA	 0x8011U
+#define	RGX_CNTBLK_ID_TPU_MCU2		 0x0012U
+#define RGX_CNTBLK_ID_TPU_MCU2_DA	 0x8012U
+#define	RGX_CNTBLK_ID_TPU_MCU3		 0x0013U
+#define RGX_CNTBLK_ID_TPU_MCU3_DA	 0x8013U
+#define	RGX_CNTBLK_ID_TPU_MCU4		 0x0014U
+#define RGX_CNTBLK_ID_TPU_MCU4_DA	 0x8014U
+#define	RGX_CNTBLK_ID_TPU_MCU5		 0x0015U
+#define RGX_CNTBLK_ID_TPU_MCU5_DA	 0x8015U
+#define	RGX_CNTBLK_ID_TPU_MCU6		 0x0016U
+#define RGX_CNTBLK_ID_TPU_MCU6_DA	 0x8016U
+#define	RGX_CNTBLK_ID_TPU_MCU7		 0x0017U
+#define RGX_CNTBLK_ID_TPU_MCU7_DA	 0x8017U
+#define	RGX_CNTBLK_ID_TPU_MCU_ALL	 0x4010U
+#define	RGX_CNTBLK_ID_TPU_MCU_ALL_DA	 0xC010U
+
+#define	RGX_CNTBLK_ID_USC0			 0x0020U /*!< Addressable by Cluster */
+#define RGX_CNTBLK_ID_USC0_DA		 0x8020U
+#define	RGX_CNTBLK_ID_USC1			 0x0021U
+#define RGX_CNTBLK_ID_USC1_DA		 0x8021U
+#define	RGX_CNTBLK_ID_USC2			 0x0022U
+#define RGX_CNTBLK_ID_USC2_DA		 0x8022U
+#define	RGX_CNTBLK_ID_USC3			 0x0023U
+#define RGX_CNTBLK_ID_USC3_DA		 0x8023U
+#define	RGX_CNTBLK_ID_USC4			 0x0024U
+#define RGX_CNTBLK_ID_USC4_DA		 0x8024U
+#define	RGX_CNTBLK_ID_USC5			 0x0025U
+#define RGX_CNTBLK_ID_USC5_DA		 0x8025U
+#define	RGX_CNTBLK_ID_USC6			 0x0026U
+#define RGX_CNTBLK_ID_USC6_DA		 0x8026U
+#define	RGX_CNTBLK_ID_USC7			 0x0027U
+#define RGX_CNTBLK_ID_USC7_DA		 0x8027U
+#define	RGX_CNTBLK_ID_USC8			 0x0028U
+#define RGX_CNTBLK_ID_USC8_DA		 0x8028U
+#define	RGX_CNTBLK_ID_USC9			 0x0029U
+#define RGX_CNTBLK_ID_USC9_DA		 0x8029U
+#define	RGX_CNTBLK_ID_USC10			 0x002AU
+#define RGX_CNTBLK_ID_USC10_DA		 0x802AU
+#define	RGX_CNTBLK_ID_USC11			 0x002BU
+#define RGX_CNTBLK_ID_USC11_DA		 0x802BU
+#define	RGX_CNTBLK_ID_USC12			 0x002CU
+#define RGX_CNTBLK_ID_USC12_DA		 0x802CU
+#define	RGX_CNTBLK_ID_USC13			 0x002DU
+#define RGX_CNTBLK_ID_USC13_DA		 0x802DU
+#define	RGX_CNTBLK_ID_USC14			 0x002EU
+#define RGX_CNTBLK_ID_USC14_DA		 0x802EU
+#define	RGX_CNTBLK_ID_USC15			 0x002FU
+#define RGX_CNTBLK_ID_USC15_DA		 0x802FU
+#define	RGX_CNTBLK_ID_USC_ALL		 0x4020U
+#define	RGX_CNTBLK_ID_USC_ALL_DA	 0xC020U
+
+#define	RGX_CNTBLK_ID_TEXAS0		 0x0030U /*!< Addressable by Phantom in XT, Dust in S7 */
+#define	RGX_CNTBLK_ID_TEXAS1		 0x0031U
+#define	RGX_CNTBLK_ID_TEXAS2		 0x0032U
+#define	RGX_CNTBLK_ID_TEXAS3		 0x0033U
+#define	RGX_CNTBLK_ID_TEXAS4		 0x0034U
+#define	RGX_CNTBLK_ID_TEXAS5		 0x0035U
+#define	RGX_CNTBLK_ID_TEXAS6		 0x0036U
+#define	RGX_CNTBLK_ID_TEXAS7		 0x0037U
+#define	RGX_CNTBLK_ID_TEXAS_ALL		 0x4030U
+
+#define	RGX_CNTBLK_ID_RASTER0		 0x0040U /*!< Addressable by Phantom, XT only */
+#define	RGX_CNTBLK_ID_RASTER1		 0x0041U
+#define	RGX_CNTBLK_ID_RASTER2		 0x0042U
+#define	RGX_CNTBLK_ID_RASTER3		 0x0043U
+#define	RGX_CNTBLK_ID_RASTER_ALL	 0x4040U
+
+#define	RGX_CNTBLK_ID_BLACKPEARL0	 0x0050U /*!< Addressable by Phantom, S7, only */
+#define	RGX_CNTBLK_ID_BLACKPEARL1	 0x0051U
+#define	RGX_CNTBLK_ID_BLACKPEARL2	 0x0052U
+#define	RGX_CNTBLK_ID_BLACKPEARL3	 0x0053U
+#define	RGX_CNTBLK_ID_BLACKPEARL_ALL 0x4050U
+
+#define	RGX_CNTBLK_ID_PBE0			 0x0060U /*!< Addressable by Cluster in S7 and PBE2_IN_XE */
+#define	RGX_CNTBLK_ID_PBE1			 0x0061U
+#define	RGX_CNTBLK_ID_PBE2			 0x0062U
+#define	RGX_CNTBLK_ID_PBE3			 0x0063U
+#define	RGX_CNTBLK_ID_PBE4			 0x0064U
+#define	RGX_CNTBLK_ID_PBE5			 0x0065U
+#define	RGX_CNTBLK_ID_PBE6			 0x0066U
+#define	RGX_CNTBLK_ID_PBE7			 0x0067U
+#define	RGX_CNTBLK_ID_PBE8			 0x0068U
+#define	RGX_CNTBLK_ID_PBE9			 0x0069U
+#define	RGX_CNTBLK_ID_PBE10			 0x006AU
+#define	RGX_CNTBLK_ID_PBE11			 0x006BU
+#define	RGX_CNTBLK_ID_PBE12			 0x006CU
+#define	RGX_CNTBLK_ID_PBE13			 0x006DU
+#define	RGX_CNTBLK_ID_PBE14			 0x006EU
+#define	RGX_CNTBLK_ID_PBE15			 0x006FU
+#define	RGX_CNTBLK_ID_PBE_ALL		 0x4060U
+
+#define	RGX_CNTBLK_ID_LAST			 0x0070U /*!< End of PBE block */
+
+#define	RGX_CNTBLK_ID_BX_TU0_DEPRECATED		 0x0070U /*!< Doppler unit, DEPRECATED */
+#define	RGX_CNTBLK_ID_BX_TU1_DEPRECATED		 0x0071U
+#define	RGX_CNTBLK_ID_BX_TU2_DEPRECATED		 0x0072U
+#define	RGX_CNTBLK_ID_BX_TU3_DEPRECATED		 0x0073U
+#define	RGX_CNTBLK_ID_BX_TU_ALL_DEPRECATED	 0x4070U
+
+#define	RGX_CNTBLK_ID_CUSTOM0		 0x70F0U
+#define	RGX_CNTBLK_ID_CUSTOM1		 0x70F1U
+#define	RGX_CNTBLK_ID_CUSTOM2		 0x70F2U
+#define	RGX_CNTBLK_ID_CUSTOM3		 0x70F3U
+#define	RGX_CNTBLK_ID_CUSTOM4_FW	 0x70F4U /*!< Custom block used for getting statistics held in the FW */
+#define	RGX_CNTBLK_ID_CUSTOM_MASK	 0x70FFU
+
+
+/* Masks for the counter block ID*/
+#define	RGX_CNTBLK_ID_UNIT_MASK      (0x000FU)
+#define	RGX_CNTBLK_ID_GROUP_MASK     (0x00F0U)
+#define	RGX_CNTBLK_ID_GROUP_SHIFT    (4U)
+#define	RGX_CNTBLK_ID_MC_GPU_MASK    (0x0F00U)
+#define	RGX_CNTBLK_ID_MC_GPU_SHIFT   (8U)
+#define	RGX_CNTBLK_ID_UNIT_ALL_MASK  (0x4000U)
+#define	RGX_CNTBLK_ID_DA_MASK        (0x8000U) /*!< Block with directly accessible counter registers */
+
+#define RGX_CNTBLK_INDIRECT_COUNT(_class, _n) ((IMG_UINT32)(RGX_CNTBLK_ID_ ## _class ## _n) - (IMG_UINT32)(RGX_CNTBLK_ID_ ## _class ## 0) + 1u)
+
+/*! The number of layout blocks defined with configurable multiplexed
+ * performance counters, hence excludes custom counter blocks.
+ */
+#define RGX_HWPERF_MAX_DEFINED_BLKS  (\
+	(IMG_UINT32)RGX_CNTBLK_ID_DIRECT_LAST    +\
+	RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU,     7)+\
+	RGX_CNTBLK_INDIRECT_COUNT(USC,        15)+\
+	RGX_CNTBLK_INDIRECT_COUNT(TEXAS,       7)+\
+	RGX_CNTBLK_INDIRECT_COUNT(RASTER,      3)+\
+	RGX_CNTBLK_INDIRECT_COUNT(BLACKPEARL,  3)+\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE,        15) )
+#define RGX_HWPERF_MAX_MUX_BLKS      (\
+    RGX_HWPERF_MAX_DEFINED_BLKS      )
+
+static_assert(
+	((RGX_CNTBLK_ID_DIRECT_LAST + ((RGX_CNTBLK_ID_LAST & RGX_CNTBLK_ID_GROUP_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT)) <= RGX_HWPERF_MAX_BVNC_BLOCK_LEN),
+	"RGX_HWPERF_MAX_BVNC_BLOCK_LEN insufficient");
+
+#define RGX_HWPERF_EVENT_MASK_VALUE(e)      (IMG_UINT64_C(1) << (IMG_UINT32)(e))
+
+#define RGX_CUSTOM_FW_CNTRS	\
+                X(TA_LOCAL_FL_SIZE,    0x0, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(TA_GLOBAL_FL_SIZE,   0x1, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(3D_LOCAL_FL_SIZE,    0x2, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(3D_GLOBAL_FL_SIZE,   0x3, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(ISP_TILES_IN_FLIGHT, 0x4, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DSPMKICK))
+
+/*! Counter IDs for the firmware held statistics */
+typedef enum
+{
+#define X(ctr, id, allow_mask)	RGX_CUSTOM_FW_CNTR_##ctr = id,
+	RGX_CUSTOM_FW_CNTRS
+#undef X
+
+	/* always the last entry in the list */
+	RGX_CUSTOM_FW_CNTR_LAST
+} RGX_HWPERF_CUSTOM_FW_CNTR_ID;
+
+/*! Identifier for each counter in a performance counting module */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_COUNTER_ID;
+
+#define	RGX_CNTBLK_COUNTER0_ID 0U
+#define	RGX_CNTBLK_COUNTER1_ID 1U
+#define	RGX_CNTBLK_COUNTER2_ID 2U
+#define	RGX_CNTBLK_COUNTER3_ID 3U
+#define	RGX_CNTBLK_COUNTER4_ID 4U
+#define	RGX_CNTBLK_COUNTER5_ID 5U
+	/* MAX value used in server handling of counter config arrays */
+#define	RGX_CNTBLK_MUX_COUNTERS_MAX 6U
+
+
+/* sets all the bits from bit _b1 to _b2, in a IMG_UINT64 type */
+#define MASK_RANGE_IMPL(b1, b2)	((IMG_UINT64)((IMG_UINT64_C(1) << ((IMG_UINT32)(b2)-(IMG_UINT32)(b1) + 1U)) - 1U) << (IMG_UINT32)(b1))
+#define MASK_RANGE(R)			MASK_RANGE_IMPL(R##_FIRST_TYPE, R##_LAST_TYPE)
+#define RGX_HWPERF_HOST_EVENT_MASK_VALUE(e) (IMG_UINT32_C(1) << (e))
+
+/*! Mask macros for use with RGXCtrlHWPerf() API.
+ */
+#define RGX_HWPERF_EVENT_MASK_NONE          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_HWPERF_EVENT_MASK_DEFAULT       RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_FWACT) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG)
+#define RGX_HWPERF_EVENT_MASK_ALL           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+
+/*! HWPerf Firmware event masks
+ * @par
+ * All FW Start/End/Debug (SED) events. */
+#define RGX_HWPERF_EVENT_MASK_FW_SED    (MASK_RANGE(RGX_HWPERF_FW_EVENT_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_FW_UFO    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO))
+#define RGX_HWPERF_EVENT_MASK_FW_CSW    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_START) |\
+                                          RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_FINISHED))
+/*! All FW events. */
+#define RGX_HWPERF_EVENT_MASK_ALL_FW    (RGX_HWPERF_EVENT_MASK_FW_SED |\
+                                          RGX_HWPERF_EVENT_MASK_FW_UFO |\
+                                          RGX_HWPERF_EVENT_MASK_FW_CSW)
+
+/*! HW Periodic events (1ms interval). */
+#define RGX_HWPERF_EVENT_MASK_HW_PERIODIC   (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PERIODIC))
+/*! All HW Kick/Finish events. */
+#define RGX_HWPERF_EVENT_MASK_HW_KICKFINISH ((MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE0) |\
+                                               MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE1)) &\
+                                              ~(RGX_HWPERF_EVENT_MASK_HW_PERIODIC))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_HW        (RGX_HWPERF_EVENT_MASK_HW_KICKFINISH |\
+                                              RGX_HWPERF_EVENT_MASK_HW_PERIODIC)
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR_EST   (MASK_RANGE(RGX_HWPERF_PWR_EST_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR       (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_GPU_STATE_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG))
+
+/*! HWPerf Host event masks
+ */
+#define RGX_HWPERF_EVENT_MASK_HOST_WORK_ENQ  (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_ENQ))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_UFO   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_UFO))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_PWR   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_CLK_SYNC))
+
+
+/*! Type used in the RGX API RGXConfigMuxHWPerfCounters() */
+typedef struct
+{
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! 4 or 6 LSBs used to select counters to configure in this block. */
+	IMG_UINT8  ui8CounterSelect;
+
+	/*! 4 or 6 LSBs used as MODE bits for the counters in the group. */
+	IMG_UINT8  ui8Mode;
+
+	/*! 5 or 6 LSBs used as the GROUP_SELECT value for the counter. */
+	IMG_UINT8  aui8GroupSelect[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 16 LSBs used as the BIT_SELECT value for the counter. */
+	IMG_UINT16 aui16BitSelect[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 14 LSBs used as the BATCH_MAX value for the counter. */
+	IMG_UINT32 aui32BatchMax[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 14 LSBs used as the BATCH_MIN value for the counter. */
+	IMG_UINT32 aui32BatchMin[RGX_CNTBLK_MUX_COUNTERS_MAX];
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_MUX_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+
+/*! Type used in the RGX API RGXConfigHWPerfCounters() */
+typedef struct
+{
+	/*! Reserved for future use */
+	IMG_UINT32 ui32Reserved;
+
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of configured counters within this block */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Counter register values */
+	IMG_UINT16 ui16Counters[RGX_CNTBLK_COUNTERS_MAX];
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_CNTBLK);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_options.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_options.h
new file mode 100644
index 000000000000..df9c949f4a1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_options.h
@@ -0,0 +1,350 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX build options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Each build option listed here is packed into a dword which provides up to
+ *  log2(RGX_BUILD_OPTIONS_MASK_KM + 1) flags for KM and
+ *  (32 - log2(RGX_BUILD_OPTIONS_MASK_KM + 1)) flags for UM.
+ * The corresponding bit is set if the build option was enabled at compile
+ * time.
+ *
+ * IMPORTANT: add new options to unused bits or define a new dword
+ * (e.g. RGX_BUILD_OPTIONS_KM2 or RGX_BUILD_OPTIONS2) so that the bitfield
+ * remains backwards compatible.
+ */
+
+#ifndef RGX_OPTIONS_H
+#define RGX_OPTIONS_H
+
+#define OPTIONS_OPEN_SOURCE_EN					(0x1UL << 0)
+#define OPTIONS_PDUMP_EN						(0x1UL << 1)
+#define OPTIONS_UNUSED1_EN						(0x1UL << 2)
+#define OPTIONS_SECURE_ALLOC_KM_EN				(0x1UL << 3)
+#define OPTIONS_RGX_EN							(0x1UL << 4)
+#define OPTIONS_SECURE_EXPORT_EN				(0x1UL << 5)
+#define OPTIONS_INSECURE_EXPORT_EN				(0x1UL << 6)
+#define OPTIONS_VFP_EN							(0x1UL << 7)
+#define OPTIONS_WORKLOAD_ESTIMATION_EN			(0x1UL << 8)
+#define OPTIONS_PDVFS_EN						(0x1UL << 9)
+#define OPTIONS_DEBUG_EN						(0x1UL << 10)
+#define OPTIONS_BUFFER_SYNC_EN					(0x1UL << 11)
+#define OPTIONS_AUTOVZ_EN						(0x1UL << 12)
+#define OPTIONS_AUTOVZ_HW_REGS_EN				(0x1UL << 13)
+#define OPTIONS_FW_IRQ_REG_COUNTERS_EN			(0x1UL << 14)
+#define OPTIONS_VALIDATION_EN					(0x1UL << 15)
+#define OPTIONS_NO_HARDWARE_EN					(0x1UL << 16)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN                  (0x1UL << 17)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_MASK                      (0x1fUL << 18)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT                     (18)
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_PERCONTEXT_FREELIST_EN			(0x1UL << 31)
+
+#define RGX_BUILD_OPTIONS_MASK_KM				  \
+			(OPTIONS_OPEN_SOURCE_EN				| \
+			 OPTIONS_PDUMP_EN					| \
+			 OPTIONS_SECURE_ALLOC_KM_EN			| \
+			 OPTIONS_RGX_EN						| \
+			 OPTIONS_SECURE_EXPORT_EN			| \
+			 OPTIONS_INSECURE_EXPORT_EN			| \
+			 OPTIONS_VFP_EN						| \
+			 OPTIONS_WORKLOAD_ESTIMATION_EN		| \
+			 OPTIONS_PDVFS_EN					| \
+			 OPTIONS_DEBUG_EN					| \
+			 OPTIONS_BUFFER_SYNC_EN				| \
+			 OPTIONS_AUTOVZ_EN					| \
+			 OPTIONS_AUTOVZ_HW_REGS_EN			| \
+			 OPTIONS_FW_IRQ_REG_COUNTERS_EN		| \
+			 OPTIONS_VALIDATION_EN				| \
+			 OPTIONS_NO_HARDWARE_EN				| \
+			 OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN)
+
+#define RGX_BUILD_OPTIONS_MASK_FW \
+			(RGX_BUILD_OPTIONS_MASK_KM & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_MASK & \
+			 ~OPTIONS_BUFFER_SYNC_EN)
+
+/* Build options that the FW must have if the present on the KM */
+#define FW_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the UM must have if the present on the KM */
+#define UM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the KM must have if the present on the UM */
+#define KM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN					| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_BUFFER_SYNC_EN))
+
+#define OPEN_SOURCE_OPTION	"OPEN_SOURCE_DRIVER  "
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	#define OPTIONS_BIT0		OPTIONS_OPEN_SOURCE_EN
+	#if OPTIONS_BIT0 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT0		0x0UL
+#endif /* SUPPORT_OPEN_SOURCE_DRIVER */
+
+#define PDUMP_OPTION	"PDUMP  "
+#if defined(PDUMP)
+	#define OPTIONS_BIT1		OPTIONS_PDUMP_EN
+	#if OPTIONS_BIT1 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT1		0x0UL
+#endif /* PDUMP */
+
+/* No longer used */
+#define INTERNAL_UNUSED1_OPTION	"INTERNAL_UNUSED1  "
+#if defined(INTERNAL_UNUSED1)
+	#define OPTIONS_BIT2		OPTIONS_UNUSED1_EN
+	#if OPTIONS_BIT2 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT2		0x0UL
+#endif
+
+#define SECURE_ALLOC_KM_OPTION "SECURE_ALLOC_KM  "
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	#define OPTIONS_BIT3		OPTIONS_SECURE_ALLOC_KM_EN
+	#if OPTIONS_BIT3 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT3 0x0UL
+#endif /* SUPPORT_SECURE_ALLOC_KM */
+
+#define RGX_OPTION	" "
+#if defined(SUPPORT_RGX)
+	#define OPTIONS_BIT4		OPTIONS_RGX_EN
+	#if OPTIONS_BIT4 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT4		0x0UL
+#endif /* SUPPORT_RGX */
+
+#define SECURE_EXPORT_OPTION	"SECURE_EXPORTS  "
+#if defined(SUPPORT_SECURE_EXPORT)
+	#define OPTIONS_BIT5		OPTIONS_SECURE_EXPORT_EN
+	#if OPTIONS_BIT5 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT5		0x0UL
+#endif /* SUPPORT_SECURE_EXPORT */
+
+#define INSECURE_EXPORT_OPTION	"INSECURE_EXPORTS  "
+#if defined(SUPPORT_INSECURE_EXPORT)
+	#define OPTIONS_BIT6		OPTIONS_INSECURE_EXPORT_EN
+	#if OPTIONS_BIT6 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT6		0x0UL
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+#define VFP_OPTION	"VFP  "
+#if defined(SUPPORT_VFP)
+	#define OPTIONS_BIT7		OPTIONS_VFP_EN
+	#if OPTIONS_BIT7 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT7		0x0UL
+#endif /* SUPPORT_VFP */
+
+#define WORKLOAD_ESTIMATION_OPTION	"WORKLOAD_ESTIMATION  "
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	#define OPTIONS_BIT8		OPTIONS_WORKLOAD_ESTIMATION_EN
+	#if OPTIONS_BIT8 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT8		0x0UL
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+
+#define PDVFS_OPTION	"PDVFS  "
+#if defined(SUPPORT_PDVFS)
+	#define OPTIONS_BIT9		OPTIONS_PDVFS_EN
+	#if OPTIONS_BIT9 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT9		0x0UL
+#endif /* SUPPORT_PDVFS */
+
+#define DEBUG_OPTION	"DEBUG  "
+#if defined(DEBUG)
+	#define OPTIONS_BIT10		OPTIONS_DEBUG_EN
+	#if OPTIONS_BIT10 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT10		0x0UL
+#endif /* DEBUG */
+
+#define BUFFER_SYNC_OPTION	"BUFFER_SYNC  "
+#if defined(SUPPORT_BUFFER_SYNC)
+	#define OPTIONS_BIT11		OPTIONS_BUFFER_SYNC_EN
+	#if OPTIONS_BIT11 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT11		0x0UL
+#endif /* SUPPORT_BUFFER_SYNC */
+
+#define AUTOVZ_OPTION	"AUTOVZ  "
+#if defined(SUPPORT_AUTOVZ)
+	#define OPTIONS_BIT12		OPTIONS_AUTOVZ_EN
+	#if OPTIONS_BIT12 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT12		0x0UL
+#endif /* SUPPORT_AUTOVZ */
+
+#define AUTOVZ_HW_REGS_OPTION	"AUTOVZ_HW_REGS  "
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+	#define OPTIONS_BIT13		OPTIONS_AUTOVZ_HW_REGS_EN
+	#if OPTIONS_BIT13 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT13		0x0UL
+#endif /* SUPPORT_AUTOVZ_HW_REGS */
+
+#define RGX_FW_IRQ_OS_COUNTERS_OPTION	"FW_IRQ_OS_COUNTERS  "
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	#define OPTIONS_BIT14		OPTIONS_FW_IRQ_REG_COUNTERS_EN
+	#if OPTIONS_BIT14 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT14		0x0UL
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+#define VALIDATION_OPTION	"VALIDATION  "
+	#define OPTIONS_BIT15		0x0UL
+
+#define NO_HARDWARE_OPTION	"NO_HARDWARE  "
+#if defined(NO_HARDWARE)
+	#define OPTIONS_BIT16		OPTIONS_NO_HARDWARE_EN
+	#if OPTIONS_BIT16 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT16		0x0UL
+#endif /* NO_HARDWARE */
+
+#define NUM_DRIVERS_SUPPORTED_CHECK_OPTION	"NUM_DRIVERS_SUPPORTED_CHECK  "
+	#define OPTIONS_BIT17		OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN
+	#if OPTIONS_BIT17 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif /* NUM_DRIVERS_SUPPORTED_CHECK */
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_BIT31		OPTIONS_PERCONTEXT_FREELIST_EN
+#if OPTIONS_BIT31 <= RGX_BUILD_OPTIONS_MASK_KM
+#error "Bit exceeds reserved range"
+#endif
+
+#define RGX_BUILD_OPTIONS_KM	\
+	(OPTIONS_BIT0				|\
+	 OPTIONS_BIT1				|\
+	 OPTIONS_BIT2				|\
+	 OPTIONS_BIT3				|\
+	 OPTIONS_BIT4				|\
+	 OPTIONS_BIT5				|\
+	 OPTIONS_BIT6				|\
+	 OPTIONS_BIT7				|\
+	 OPTIONS_BIT8				|\
+	 OPTIONS_BIT9				|\
+	 OPTIONS_BIT10				|\
+	 OPTIONS_BIT11				|\
+	 OPTIONS_BIT12				|\
+	 OPTIONS_BIT13				|\
+	 OPTIONS_BIT14				|\
+	 OPTIONS_BIT15				|\
+	 OPTIONS_BIT16				|\
+	 OPTIONS_BIT17)
+
+#define RGX_BUILD_OPTIONS (RGX_BUILD_OPTIONS_KM | OPTIONS_BIT31)
+
+#define RGX_BUILD_OPTIONS_LIST			\
+	{									\
+		OPEN_SOURCE_OPTION,				\
+		PDUMP_OPTION,					\
+		INTERNAL_UNUSED1_OPTION,		\
+		SECURE_ALLOC_KM_OPTION,			\
+		RGX_OPTION,						\
+		SECURE_EXPORT_OPTION,			\
+		INSECURE_EXPORT_OPTION,			\
+		VFP_OPTION,						\
+		WORKLOAD_ESTIMATION_OPTION,		\
+		PDVFS_OPTION,					\
+		DEBUG_OPTION,					\
+		BUFFER_SYNC_OPTION,				\
+		AUTOVZ_OPTION,					\
+		AUTOVZ_HW_REGS_OPTION,			\
+		RGX_FW_IRQ_OS_COUNTERS_OPTION,	\
+		VALIDATION_OPTION,				\
+		NO_HARDWARE_OPTION,				\
+		NUM_DRIVERS_SUPPORTED_CHECK_OPTION				\
+	}
+
+#endif /* RGX_OPTIONS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig.h
new file mode 100644
index 000000000000..6825da5caa3b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig.h
@@ -0,0 +1,288 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_H
+#define RGXHEAPCONFIG_H
+
+#include "rgxdefs_km.h"
+
+#define RGX_HEAP_SIZE_32KiB      IMG_UINT64_C(0x0000008000)
+#define RGX_HEAP_SIZE_2MiB       IMG_UINT64_C(0x0000200000)
+#define RGX_HEAP_SIZE_4MiB       IMG_UINT64_C(0x0000400000)
+#define RGX_HEAP_SIZE_16MiB      IMG_UINT64_C(0x0001000000)
+#define RGX_HEAP_SIZE_32MiB      IMG_UINT64_C(0x0002000000)
+#define RGX_HEAP_SIZE_256MiB     IMG_UINT64_C(0x0010000000)
+
+#define RGX_HEAP_SIZE_1GiB       IMG_UINT64_C(0x0040000000)
+#define RGX_HEAP_SIZE_2GiB       IMG_UINT64_C(0x0080000000)
+#define RGX_HEAP_SIZE_4GiB       IMG_UINT64_C(0x0100000000)
+#define RGX_HEAP_SIZE_16GiB      IMG_UINT64_C(0x0400000000)
+#define RGX_HEAP_SIZE_32GiB      IMG_UINT64_C(0x0800000000)
+#define RGX_HEAP_SIZE_64GiB      IMG_UINT64_C(0x1000000000)
+#define RGX_HEAP_SIZE_128GiB     IMG_UINT64_C(0x2000000000)
+#define RGX_HEAP_SIZE_256GiB     IMG_UINT64_C(0x4000000000)
+#define RGX_HEAP_SIZE_512GiB     IMG_UINT64_C(0x8000000000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address heaps that are used in
+	application memory contexts. It also shows where the Firmware memory heap
+	fits into this, but the firmware heap is only ever created in the
+	Services KM/server component.
+
+	RGX_PDSCODEDATA_HEAP_BASE and RGX_USCCODE_HEAP_BASE will be programmed,
+	on a global basis, into RGX_CR_PDS_EXEC_BASE and RGX_CR_USC_CODE_BASE_*
+	respectively. Therefore if clients use multiple configs they must still
+	be consistent with their definitions for these heaps.
+
+	Shared virtual memory (GENERAL_SVM) support requires half of the address
+	space (512 GiB) be reserved for SVM allocations to mirror application CPU
+	addresses. However, if BRN_65273 WA is active in which case the SVM heap
+	is disabled. This is reflected in the device connection capability bits
+	returned to user space.
+
+	The GENERAL non-SVM region is 512 GiB to 768 GiB and is shared between the
+	general (4KiB) heap and the general non-4K heap. The first 128 GiB is used
+	for the GENERAL_HEAP (4KiB) and the last 32 GiB is used for the
+	GENERAL_NON4K_HEAP. This heap has a default page-size of 16K.
+	AppHint PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE can be used to forced it
+	to these values: 4K,64K,256K,1M,2M.
+
+	The heaps defined for BRN_65273 _replace_ the non-BRN equivalents below
+	when this BRN WA is active on affected cores. This is different to most
+	other BRNs and hence has been given its own header file for clarity,
+	see below. This	is a special case, other BRNs that need 1 or 2 additional
+	heaps should be added to this file, like BRN_63142 below.
+	NOTE: All regular heaps below greater than 1GB require a BRN_65273 WA heap.
+
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range. Often this is the reserved
+	size of the heap within the range.
+*/
+
+/* This BRN requires a different virtual memory map from the standard one
+ * defined in this file below. Hence the alternative heap definitions for this
+ * BRN are provided in a separate file for clarity. */
+#include "rgxheapconfig_65273.h"
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0000_0000 - 0x00_0020_0000 **/
+	/* 0 MiB to 2 MiB, size of 2 MiB : RESERVED (only when General SVM
+	 *                                           doesn't exist) **/
+
+	/* BRN_65273 TQ3DPARAMETERS base 0x0000010000 */
+	/* BRN_65273 GENERAL base        0x65C0000000 */
+	/* BRN_65273 GENERAL_NON4K base  0x73C0000000 */
+
+/* 0x00_0000_8000 - 0x7F_FFFF_8000 **/
+	/* MAX(32 KiB, PAGE_SIZE) to 512 GiB, size of 512 GiB less MAX(32 KiB, PAGE_SIZE) : GENERAL_SVM_HEAP **/
+
+	/* The MAX is determined at runtime (PAGE_SIZE isn't available on all platforms)
+	 * so the #define's must NOT be used directly. Use the heap config after initialisation. */
+	#define RGX_GENERAL_SVM_HEAP_BASE           IMG_UINT64_C(0x0000008000)
+	#define RGX_GENERAL_SVM_HEAP_SIZE           (RGX_HEAP_SIZE_512GiB - RGX_HEAP_SIZE_32KiB)
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0x80_0000_0000 - 0x9F_FFFF_FFFF **/
+	/* 512 GiB to 640 GiB, size of 128 GiB : GENERAL_HEAP **/
+	#define RGX_GENERAL_HEAP_BASE               IMG_UINT64_C(0x8000000000)
+	#define RGX_GENERAL_HEAP_SIZE               RGX_HEAP_SIZE_128GiB
+
+	/* BRN_65273 PDSCODEDATA base    0xA800000000 */
+
+/* 0xA0_0000_0000 - 0xAF_FFFF_FFFF **/
+	/* 640 GiB to 704 GiB, size of 64 GiB : FREE **/
+
+/* B0_0000_0000 - 0xB7_FFFF_FFFF **/
+	/* 704 GiB to 736 GiB, size of 32 GiB : FREE **/
+
+	/* BRN_65273 USCCODE base        0xBA00000000 */
+
+/* 0xB8_0000_0000 - 0xBF_FFFF_FFFF **/
+	/* 736 GiB to 768 GiB, size of 32 GiB : GENERAL_NON4K_HEAP **/
+	#define RGX_GENERAL_NON4K_HEAP_BASE         IMG_UINT64_C(0xB800000000)
+	#define RGX_GENERAL_NON4K_HEAP_SIZE         RGX_HEAP_SIZE_32GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xC0_0000_0000 - 0xD9_FFFF_FFFF **/
+	/* 768 GiB to 872 GiB, size of 104 GiB : FREE **/
+
+/* 0xDA_0000_0000 - 0xDA_FFFF_FFFF **/
+	/* 872 GiB to 876 GiB, size of 4 GiB : PDSCODEDATA_HEAP **/
+	#define RGX_PDSCODEDATA_HEAP_BASE           IMG_UINT64_C(0xDA00000000)
+	#define RGX_PDSCODEDATA_HEAP_SIZE           RGX_HEAP_SIZE_4GiB
+
+/* 0xDB_0000_0000 - 0xDB_FFFF_FFFF **/
+	/* 876 GiB to 880 GiB, size of 256 MiB (reserved 4GiB) : BRN **/
+	/* HWBRN63142 workaround requires Region Header memory to be at the top
+	   of a 16GiB aligned range. This is so when masked with 0x03FFFFFFFF the
+	   address will avoid aliasing PB addresses. Start at 879.75GiB. Size of 256MiB. */
+	#define RGX_RGNHDR_BRN_63142_HEAP_BASE      IMG_UINT64_C(0xDBF0000000)
+	#define RGX_RGNHDR_BRN_63142_HEAP_SIZE      RGX_HEAP_SIZE_256MiB
+
+/* 0xDC_0000_0000 - 0xDF_FFFF_FFFF **/
+	/* 880 GiB to 896 GiB, size of 16 GiB : FREE **/
+
+/* 0xE0_0000_0000 - 0xE0_FDFF_FFFF **/
+	/* 896 GiB to 900 GiB, size of 4 GiB less 32 MiB : USCCODE_HEAP **/
+	#define RGX_USCCODE_HEAP_BASE               IMG_UINT64_C(0xE000000000)
+	#define RGX_USCCODE_HEAP_SIZE               (RGX_HEAP_SIZE_4GiB - RGX_HEAP_SIZE_32MiB)
+
+/* 0xE0_FE00_0000 - 0xE0_FFFF_FFFF **/
+	/* 900 GiB less 32 MiB to 900 GiB, size of 32 MiB : RESERVED VOLCANIC **/
+
+/* 0xE1_0000_0000 - 0xE1_BFFF_FFFF **/
+	/* 900 GiB to 903 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xE1_C000_000 - 0xE1_FFFF_FFFF **/
+	/* 903 GiB to 904 GiB, reserved 1 GiB, : FIRMWARE_HEAP **/
+
+	/* Firmware heaps defined in rgx_heap_firmware.h as they are not present in
+	   application memory contexts, see:
+	    RGX_FIRMWARE_RAW_HEAP_BASE
+	    RGX_FIRMWARE_RAW_HEAP_SIZE
+	   See header for other sub-heaps details
+	*/
+
+/* 0xE2_0000_0000 - 0xE3_FFFF_FFFF **/
+	/* 904 GiB to 912 GiB, size of 8 GiB : FREE **/
+
+	/* BRN_65273 VISIBILITY_TEST base 0xE400000000 */
+
+/* 0xE4_0000_0000 - 0xE7_FFFF_FFFF **/
+	/* 912 GiB to 928 GiB, size 16 GiB : TQ3DPARAMETERS_HEAP **/
+	/* Aligned to match RGX_CR_ISP_PIXEL_BASE at 16 GiB */
+	#define RGX_TQ3DPARAMETERS_HEAP_BASE        IMG_UINT64_C(0xE400000000)
+	#define RGX_TQ3DPARAMETERS_HEAP_SIZE        RGX_HEAP_SIZE_16GiB
+
+/* 0xE8_0000_0000 - 0xE8_FFFF_FFFF **/
+	/* 928 GiB to 932 GiB, size of 4 GiB : FREE **/
+
+/* 0xE9_0000_0000 - 0xE9_3FFF_FFFF **/
+	/* 932 GiB to 933 GiB, size of 1 GiB : VK_CAPT_REPLAY_HEAP **/
+	#define RGX_VK_CAPT_REPLAY_HEAP_BASE        IMG_UINT64_C(0xE900000000)
+	#define RGX_VK_CAPT_REPLAY_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xE9_4000_0000 - 0xE9_FFFF_FFFF **/
+	/* 933 GiB to 936 GiB, size of 3 GiB : FREE **/
+
+/* 0xEA_0000_0000 - 0xEA_001F_FFFF **/
+	/* 936 GiB to 937 GiB, size of 1 GiB : FREE **/
+
+/* 0xEA_4000_0000 - 0xEA_FFFF_FFFF **/
+	/* 937 GiB to 940 GiB, size of 3 GiB : FREE **/
+
+/* 0xEB_0000_0000 - 0xEB_FFFF_FFFF **/
+	/* 940 GiB to 944 GiB, size of 4 GiB : RESERVED VOLCANIC  **/
+
+/* 0xEC_0000_0000 - 0xEC_001F_FFFF **/
+	/* 944 GiB to 945 GiB, size 2 MiB : FBCDC_HEAP **/
+	#define RGX_FBCDC_HEAP_BASE                 IMG_UINT64_C(0xEC00000000)
+	#define RGX_FBCDC_HEAP_SIZE                 RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_4000_0000 - 0xEC_401F_FFFF **/
+	/* 945 GiB to 946 GiB, size 2 MiB : FBCDC_LARGE_HEAP **/
+	#define RGX_FBCDC_LARGE_HEAP_BASE           IMG_UINT64_C(0xEC40000000)
+	#define RGX_FBCDC_LARGE_HEAP_SIZE           RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_8000_0000 - 0xED_FFFF_FFFF **/
+	/* 946 GiB to 952 GiB, size of 6 GiB : RESERVED VOLCANIC  **/
+
+/* 0xEE_0000_0000 - 0xEE_3FFF_FFFF **/
+	/* 952 GiB to 953 GiB, size of 1 GiB : CMP_MISSION_RMW_HEAP **/
+	#define RGX_CMP_MISSION_RMW_HEAP_BASE       IMG_UINT64_C(0xEE00000000)
+	#define RGX_CMP_MISSION_RMW_HEAP_SIZE       RGX_HEAP_SIZE_1GiB
+
+/* 0xEE_4000_0000 - 0xEE_FFFF_FFFF **/
+	/* 953 GiB to 956 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xEF_0000_0000 - 0xEF_3FFF_FFFF **/
+	/* 956 GiB to 957 GiB, size of 1 GiB : CMP_SAFETY_RMW_HEAP **/
+	#define RGX_CMP_SAFETY_RMW_HEAP_BASE        IMG_UINT64_C(0xEF00000000)
+	#define RGX_CMP_SAFETY_RMW_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xEF_4000_0000 - 0xEF_FFFF_FFFF **/
+	/* 957 GiB to 960 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xF0_0000_0000 - 0xF0_FFFF_FFFF **/
+	/* 960 GiB to 964 GiB, size of 4 GiB : TEXTURE_STATE_HEAP (36-bit aligned) */
+	#define RGX_TEXTURE_STATE_HEAP_BASE         IMG_UINT64_C(0xF000000000)
+	#define RGX_TEXTURE_STATE_HEAP_SIZE         RGX_HEAP_SIZE_4GiB
+
+/* 0xF1_0000_0000 - 0xF1_FFFF_FFFF **/
+	/* 964 GiB to 968 GiB, size of 4 GiB : FREE **/
+
+/* 0xF2_0000_0000 - 0xF2_001F_FFFF **/
+	/* 968 GiB to 969 GiB, size of 2 MiB : VISIBILITY_TEST_HEAP **/
+	#define RGX_VISIBILITY_TEST_HEAP_BASE       IMG_UINT64_C(0xF200000000)
+	#define RGX_VISIBILITY_TEST_HEAP_SIZE       RGX_HEAP_SIZE_2MiB
+
+/* 0xF2_4000_0000 - 0xF2_FFFF_FFFF **/
+	/* 969 GiB to 972 GiB, size of 3 GiB : FREE **/
+
+	/* BRN_65273 MMU_INIA base 0xF800000000 */
+	/* BRN_65273 MMU_INIB base 0xF900000000 */
+
+/* 0xF3_0000_0000 - 0xFF_FFFF_FFFF **/
+	/* 972 GiB to 1024 GiB, size of 52 GiB : FREE **/
+
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig_65273.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig_65273.h
new file mode 100644
index 000000000000..66f44dc8c18b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig_65273.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map for BRN_65273.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_65273_H
+#define RGXHEAPCONFIG_65273_H
+
+#define RGX_HEAP_SIZE_64KiB      IMG_UINT64_C(0x0000010000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address replacement heaps that are used
+	in application memory contexts when BRN_65273 is present. The addresses
+	used are specifically crafted to avoid the BRN - under no circumstances
+	can they be moved, altered in size or additional heaps added.
+
+	The heaps defined for BRN_65273 _replace_ the non-BRN equivalents when this
+	BRN WA is active on affected cores. This is different to most other BRNs
+	and hence has been given its own header file for clarity. The SVM_HEAP is
+	also disabled and unavailable when the WA is active. This is reflected
+	in the device connection capability bits returned to user space.
+
+	NOTE: All regular heaps in rgxheapconfig.h greater than 1GB require
+	      a BRN_65273 WA heap.
+
+	Base addresses must have to be a multiple of 4MiB
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range.
+*/
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0001_0000 - 0x00_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires TQ memory to start at 64 KiB and use a
+	 * unique single 0.99GiB PCE entry. */
+	#define RGX_TQ3DPARAMETERS_BRN_65273_HEAP_BASE  IMG_UINT64_C(0x0000010000)
+	#define RGX_TQ3DPARAMETERS_BRN_65273_HEAP_SIZE  (RGX_HEAP_SIZE_1GiB - RGX_HEAP_SIZE_64KiB)
+
+/* 0x65_C000_0000 - 0x66_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires General Heap to use a unique PCE entry for each GiB in range */
+	#define RGX_GENERAL_BRN_65273_HEAP_BASE         IMG_UINT64_C(0x65C0000000)
+	#define RGX_GENERAL_BRN_65273_HEAP_SIZE         RGX_HEAP_SIZE_2GiB
+
+/* 0x73_C000_0000 - 0x74_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires Non4K memory to use a unique PCE entry for each GiB in range */
+	#define RGX_GENERAL_NON4K_BRN_65273_HEAP_BASE   IMG_UINT64_C(0x73C0000000)
+	#define RGX_GENERAL_NON4K_BRN_65273_HEAP_SIZE   RGX_HEAP_SIZE_2GiB
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0xA8_0000_0000 - 0xA8_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires PDS memory to use a unique single 1GiB PCE entry. */
+	#define RGX_PDSCODEDATA_BRN_65273_HEAP_BASE     IMG_UINT64_C(0xA800000000)
+	#define RGX_PDSCODEDATA_BRN_65273_HEAP_SIZE     RGX_HEAP_SIZE_1GiB
+
+/* 0xBA_0000_0000 - 0xBA_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires USC memory to use a unique single 1GiB PCE entry. */
+	#define RGX_USCCODE_BRN_65273_HEAP_BASE         IMG_UINT64_C(0xBA00000000)
+	#define RGX_USCCODE_BRN_65273_HEAP_SIZE         RGX_HEAP_SIZE_1GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xE4_0000_0000 - 0xE4_001F_FFFF **/
+	/* HWBRN65273 workaround requires USC memory to use a unique single 1GiB PCE entry. */
+	#define RGX_VISIBILITY_TEST_BRN_65273_HEAP_BASE IMG_UINT64_C(0xE400000000)
+	#define RGX_VISIBILITY_TEST_BRN_65273_HEAP_SIZE RGX_HEAP_SIZE_2MiB
+
+/* 0xF8_0000_0000 - 0xF9_FFFF_FFFF **/
+	/* HWBRN65273 workaround requires two Region Header buffers 4GiB apart. */
+	#define RGX_MMU_INIA_BRN_65273_HEAP_BASE        IMG_UINT64_C(0xF800000000)
+	#define RGX_MMU_INIA_BRN_65273_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+	#define RGX_MMU_INIB_BRN_65273_HEAP_BASE        IMG_UINT64_C(0xF900000000)
+	#define RGX_MMU_INIB_BRN_65273_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_65273_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig_65273.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/system/rgx_tc/tc_clocks.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/system/rgx_tc/tc_clocks.h
new file mode 100644
index 000000000000..431273de0827
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/system/rgx_tc/tc_clocks.h
@@ -0,0 +1,158 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TC_CLOCKS_H)
+#define TC_CLOCKS_H
+
+/*
+ * The core clock speed is passed through a multiplier depending on the TC
+ * version.
+ *
+ * On TC_ES1: Multiplier = x3, final speed = 270MHz
+ * On TC_ES2: Multiplier = x6, final speed = 540MHz
+ * On TCF5:   Multiplier = 1x final speed = 45MHz
+ *
+ *
+ * The base (unmultiplied speed) can be adjusted using a module parameter
+ * called "sys_core_clk_speed", a number in Hz.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_core_clk_speed=60000000" /etc/init.d/rc.pvr start
+ *
+ * would result in a core speed of 60MHz xMultiplier.
+ *
+ *
+ * The memory clock is unmultiplied and can be adjusted using a module
+ * parameter called "sys_mem_clk_speed", this should be the number in Hz for
+ * the memory clock speed.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=100000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the memory clock speed set to 100MHz.
+ *
+ *
+ * Same applies to the system interface clock speed, "sys_sysif_clk_speed".
+ * Needed for TCF5 but not for TC_ES2/ES1.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_sysif_clk_speed=45000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the system clock speed set to 45MHz.
+ *
+ *
+ * All parameters can be specified at once, e.g.,
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=MEMORY_SPEED sys_core_clk_speed=CORE_SPEED sys_sysif_clk_speed=SYSIF_SPEED" /etc/init.d/rc.pvr start
+ */
+
+#define RGX_TC_SYS_CLOCK_SPEED		(25000000) /*< At the moment just used for TCF5 */
+#define RGX_TC_CLOCK_MULTIPLEX    (1)
+
+#if defined(TC_APOLLO_TCF5_22_46_54_330)
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_22_49_21_16) || \
+      defined(TC_APOLLO_TCF5_22_60_22_29) || \
+      defined(TC_APOLLO_TCF5_22_75_22_25)
+ #define RGX_TC_CORE_CLOCK_SPEED	(20000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+#elif defined(TC_APOLLO_TCF5_22_67_54_30)
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_22_89_204_18)
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_22_86_104_218)
+ #define RGX_TC_CORE_CLOCK_SPEED	(30000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_88_104_318)
+ #define RGX_TC_CORE_CLOCK_SPEED	(28000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_98_54_230)
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_102_54_38)
+ #define RGX_TC_CORE_CLOCK_SPEED	(80000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_BVNC_NOT_SUPPORTED)
+ /* TC TCF5 (22.*) fallback frequencies */
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(20000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_33_8_22_1)
+ #define RGX_TC_CORE_CLOCK_SPEED	(25000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_REFERENCE)
+ /* TC TCF5 (Reference bitfile) */
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_BONNIE)
+ /* TC Bonnie */
+ #define RGX_TC_CORE_CLOCK_SPEED	(18000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(65000000)
+#elif defined(TC_APOLLO_ES2)
+ /* TC ES2 */
+ #define RGX_TC_CORE_CLOCK_SPEED	(90000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(104000000)
+#elif defined(TC_ORION)
+ #define RGX_TC_CORE_CLOCK_SPEED	(40000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(100000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_29_19_52_202)
+ #define RGX_TC_CORE_CLOCK_SPEED	(25000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_29_18_204_508)
+ #define RGX_TC_CORE_CLOCK_SPEED	(15000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(35000000)
+#else
+ /* TC ES1 */
+ #define RGX_TC_CORE_CLOCK_SPEED	(90000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(65000000)
+#endif
+
+#endif /* if !defined(TC_CLOCKS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/services_km.h b/drivers/gpu/drm/img/img-volcanic/include/services_km.h
new file mode 100644
index 000000000000..8903b86a76a9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/services_km.h
@@ -0,0 +1,180 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services API Kernel mode Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported services API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SERVICES_KM_H
+#define SERVICES_KM_H
+
+#include "img_types.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+/*! 4k page size definition */
+#define PVRSRV_4K_PAGE_SIZE					4096U      /*!< Size of a 4K Page */
+#define PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT		12          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 16k page size definition */
+#define PVRSRV_16K_PAGE_SIZE					16384U      /*!< Size of a 16K Page */
+#define PVRSRV_16K_PAGE_SIZE_ALIGNSHIFT		14          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 64k page size definition */
+#define PVRSRV_64K_PAGE_SIZE					65536U      /*!< Size of a 64K Page */
+#define PVRSRV_64K_PAGE_SIZE_ALIGNSHIFT		16          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 256k page size definition */
+#define PVRSRV_256K_PAGE_SIZE					262144U      /*!< Size of a 256K Page */
+#define PVRSRV_256K_PAGE_SIZE_ALIGNSHIFT		18          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 1MB page size definition */
+#define PVRSRV_1M_PAGE_SIZE					1048576U      /*!< Size of a 1M Page */
+#define PVRSRV_1M_PAGE_SIZE_ALIGNSHIFT		20          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 2MB page size definition */
+#define PVRSRV_2M_PAGE_SIZE					2097152U      /*!< Size of a 2M Page */
+#define PVRSRV_2M_PAGE_SIZE_ALIGNSHIFT		21          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+
+/*!
+ * @AddToGroup SRVConnectInterfaces
+ * @{
+ */
+
+#ifndef PVRSRV_DEV_CONNECTION_TYPEDEF
+#define PVRSRV_DEV_CONNECTION_TYPEDEF
+/*!
+ * Forward declaration (look on connection.h)
+ */
+typedef struct PVRSRV_DEV_CONNECTION_TAG PVRSRV_DEV_CONNECTION;
+#endif
+
+/*!
+ * @Anchor SRV_FLAGS
+ * @Name SRV_FLAGS: Services connection flags
+ * Allows to define per-client policy for Services.
+ * @{
+ */
+
+/*
+ *   Use of the 32-bit connection flags mask
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *   31  27     20             6 4   0
+ *    |   |      |             | |   |
+ *    X---XXXXXXXX-------------XXX----
+ */
+
+#define SRV_NO_HWPERF_CLIENT_STREAM     (1UL << 4)  /*!< Don't create HWPerf for this connection */
+#define SRV_FLAGS_CLIENT_64BIT_COMPAT   (1UL << 5)  /*!< This flags gets set if the client is 64 Bit compatible. */
+#define SRV_FLAGS_CLIENT_SLR_DISABLED   (1UL << 6)  /*!< This flag is set if the client does not want Sync Lockup Recovery (SLR) enabled. */
+#define SRV_FLAGS_PDUMPCTRL             (1UL << 31) /*!< PDump Ctrl client flag */
+
+/*! @} SRV_FLAGS */
+
+/*! @} End of SRVConnectInterfaces */
+
+/*
+ * Bits 20 - 27 are used to pass information needed for validation
+ * of the GPU Virtualisation Validation mechanism. In particular:
+ *
+ * Bits:
+ * [20 - 22]: OSid of the memory region that will be used for allocations
+ * [23 - 25]: OSid that will be emitted by the Firmware for all memory accesses
+ *            regarding that memory context.
+ *      [26]: If the AXI Protection register will be set to secure for that OSid
+ *      [27]: If the Emulator Wrapper Register checking for protection violation
+ *            will be set to secure for that OSid
+ */
+
+#define VIRTVAL_FLAG_OSID_SHIFT        (20)
+#define SRV_VIRTVAL_FLAG_OSID_MASK     (7U << VIRTVAL_FLAG_OSID_SHIFT)
+
+#define VIRTVAL_FLAG_OSIDREG_SHIFT     (23)
+#define SRV_VIRTVAL_FLAG_OSIDREG_MASK  (7U << VIRTVAL_FLAG_OSIDREG_SHIFT)
+
+#define VIRTVAL_FLAG_AXIPREG_SHIFT     (26)
+#define SRV_VIRTVAL_FLAG_AXIPREG_MASK  (1U << VIRTVAL_FLAG_AXIPREG_SHIFT)
+
+#define VIRTVAL_FLAG_AXIPTD_SHIFT      (27)
+#define SRV_VIRTVAL_FLAG_AXIPTD_MASK   (1U << VIRTVAL_FLAG_AXIPTD_SHIFT)
+
+
+/* Size of pointer on a 64 bit machine */
+#define	POINTER_SIZE_64BIT	(8U)
+
+
+/*
+    Pdump flags which are accessible to Services clients
+*/
+#define PDUMP_NONE          0x00000000U /*<! No flags */
+
+#define PDUMP_BLKDATA       0x10000000U /*<! This flag indicates block-mode PDump data to be recorded in
+                                                          Block script stream in addition to Main script stream,
+                                                          if capture mode is set to BLOCKED */
+
+#define PDUMP_CONT          0x40000000U /*<! Output this entry always regardless of framed capture range,
+                                                          used by client applications being dumped. */
+#define PDUMP_PERSIST       0x80000000U /*<! Output this entry always regardless of app and range,
+                                                          used by persistent resources created after
+                                                          driver initialisation that must appear in
+                                                          all PDump captures in that session. */
+
+/* Valid range of values for pdump block length in Block mode of PDump */
+#define PDUMP_BLOCKLEN_MIN          10
+#define PDUMP_BLOCKLEN_MAX          1000
+
+#define PDUMP_FRAME_MIN             0
+#define PDUMP_FRAME_MAX             (IMG_UINT32_MAX - 1)
+#define PDUMP_FRAME_UNSET           IMG_UINT32_MAX
+
+/* Status of the device. */
+typedef IMG_UINT32 PVRSRV_DEVICE_STATUS;
+#define PVRSRV_DEVICE_STATUS_UNKNOWN        0U /* status of the device is unknown */
+#define PVRSRV_DEVICE_STATUS_OK             1U /* the device is operational */
+#define PVRSRV_DEVICE_STATUS_NOT_RESPONDING 2U /* the device is not responding */
+#define PVRSRV_DEVICE_STATUS_DEVICE_ERROR   3U /* the device is not operational */
+
+#endif /* SERVICES_KM_H */
+/**************************************************************************//**
+End of file (services_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/servicesext.h b/drivers/gpu/drm/img/img-volcanic/include/servicesext.h
new file mode 100644
index 000000000000..f2887d48aed7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/servicesext.h
@@ -0,0 +1,134 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services definitions required by external drivers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides services data structures, defines and prototypes
+                required by external drivers
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SERVICESEXT_H)
+#define SERVICESEXT_H
+
+/* include/ */
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_3dtypes.h"
+#include "pvrsrv_device_types.h"
+
+/*
+ * Lock buffer read/write flags
+ */
+#define PVRSRV_LOCKFLG_READONLY		(1)		/*!< The locking process will only read the locked surface */
+
+/*!
+ *****************************************************************************
+ *	Services State
+ *****************************************************************************/
+typedef enum _PVRSRV_SERVICES_STATE_
+{
+	PVRSRV_SERVICES_STATE_UNDEFINED = 0,
+	PVRSRV_SERVICES_STATE_OK,
+	PVRSRV_SERVICES_STATE_BAD,
+} PVRSRV_SERVICES_STATE;
+
+
+/*!
+ *****************************************************************************
+ *	States for power management
+ *****************************************************************************/
+
+/*!
+  System Power State Enum
+ */
+
+#define _PVRSRV_SYS_POWER_STATES \
+	X(Unspecified, -1) /*!< Unspecified : Uninitialised */ \
+	X(OFF,          0) /*!< Off */ \
+	X(ON,           1) /*!< On */
+
+typedef enum _PVRSRV_SYS_POWER_STATE_
+{
+#define X(name, value) PVRSRV_SYS_POWER_STATE_##name = value,
+	_PVRSRV_SYS_POWER_STATES
+#undef X
+
+	PVRSRV_SYS_POWER_STATE_FORCE_I32 = 0x7fffffff	/*!< Force enum to be at least 32-bits wide */
+
+} PVRSRV_SYS_POWER_STATE, *PPVRSRV_SYS_POWER_STATE; /*!< Typedef for ptr to PVRSRV_SYS_POWER_STATE */
+
+/*!
+  Device Power State Enum
+ */
+typedef IMG_INT32 PVRSRV_DEV_POWER_STATE;
+typedef IMG_INT32 *PPVRSRV_DEV_POWER_STATE;	/*!< Typedef for ptr to PVRSRV_DEV_POWER_STATE */ /* PRQA S 3205 */
+#define PVRSRV_DEV_POWER_STATE_DEFAULT	-1	/*!< Default state for the device */
+#define PVRSRV_DEV_POWER_STATE_OFF		 0	/*!< Unpowered */
+#define PVRSRV_DEV_POWER_STATE_ON		 1	/*!< Running */
+
+/*!
+  Power Flags Enum
+ */
+typedef IMG_UINT32 PVRSRV_POWER_FLAGS;
+#define PVRSRV_POWER_FLAGS_NONE             0U      /*!< No flags */
+#define PVRSRV_POWER_FLAGS_FORCED           (1U << 0) /*!< Power the transition should not fail */
+#define PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ (1U << 1) /*!< Indicates an OS Power Management
+                                                           transition (S3/S4) has been requested.
+                                                           Allows system modules to save VRAM */
+#define PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ  (1U << 2) /*!< Indicates an OS Power Management
+                                                           transition has been requested.
+                                                           Allows system modules to load VRAM */
+
+/*!
+ *****************************************************************************
+ * This structure is used for OS independent registry (profile) access
+ *****************************************************************************/
+
+typedef struct PVRSRV_REGISTRY_INFO_TAG
+{
+	IMG_UINT32	ui32DevCookie;
+	IMG_PCHAR	pszKey;
+	IMG_PCHAR	pszValue;
+	IMG_PCHAR	pszBuf;
+	IMG_UINT32	ui32BufSize;
+} PVRSRV_REGISTRY_INFO, *PPVRSRV_REGISTRY_INFO;
+
+#endif /* SERVICESEXT_H */
+/******************************************************************************
+ End of file (servicesext.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/sync_checkpoint_external.h b/drivers/gpu/drm/img/img-volcanic/include/sync_checkpoint_external.h
new file mode 100644
index 000000000000..bfab937a1809
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/sync_checkpoint_external.h
@@ -0,0 +1,88 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services external synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines synchronisation checkpoint structures that are visible
+				internally and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_EXTERNAL_H
+#define SYNC_CHECKPOINT_EXTERNAL_H
+
+#include "img_types.h"
+
+#ifndef CHECKPOINT_TYPES
+#define CHECKPOINT_TYPES
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG *PSYNC_CHECKPOINT_CONTEXT;
+
+typedef struct SYNC_CHECKPOINT_TAG *PSYNC_CHECKPOINT;
+#endif
+
+/* PVRSRV_SYNC_CHECKPOINT states.
+ * The OS native sync implementation should call pfnIsSignalled() to determine if a
+ * PVRSRV_SYNC_CHECKPOINT has signalled (which will return an IMG_BOOL), but can set the
+ * state for a PVRSRV_SYNC_CHECKPOINT (which is currently in the NOT_SIGNALLED state)
+ * where that PVRSRV_SYNC_CHECKPOINT is representing a foreign sync.
+ */
+typedef IMG_UINT32 PVRSRV_SYNC_CHECKPOINT_STATE;
+
+#define PVRSRV_SYNC_CHECKPOINT_UNDEF         0x000U
+#define PVRSRV_SYNC_CHECKPOINT_ACTIVE        0xac1U  /*!< checkpoint has not signalled */
+#define PVRSRV_SYNC_CHECKPOINT_SIGNALLED     0x519U  /*!< checkpoint has signalled */
+#define PVRSRV_SYNC_CHECKPOINT_ERRORED       0xeffU   /*!< checkpoint has been errored */
+
+
+#define PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(fwaddr)	(((fwaddr) & 0x1U) != 0U)
+#define PVRSRV_UFO_IS_SYNC_CHECKPOINT(ufoptr)			(PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR((ufoptr)->puiAddrUFO.ui32Addr))
+
+/* Maximum number of sync checkpoints the firmware supports in one fence */
+#define MAX_SYNC_CHECKPOINTS_PER_FENCE 32U
+
+/*!
+ * Define to be used with SyncCheckpointAlloc() to indicate a checkpoint which
+ * represents a foreign sync point or collection of foreign sync points.
+ */
+#define SYNC_CHECKPOINT_FOREIGN_CHECKPOINT ((PVRSRV_TIMELINE) - 2U)
+/*!
+ * Define to be used with SyncCheckpointAlloc() to indicate a checkpoint which
+ * represents a mirrored cross-device foreign sync point
+ */
+#define SYNC_CHECKPOINT_MIRRORED_CHECKPOINT ((PVRSRV_TIMELINE) - 3U)
+
+#endif /* SYNC_CHECKPOINT_EXTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/sync_prim_internal.h b/drivers/gpu/drm/img/img-volcanic/include/sync_prim_internal.h
new file mode 100644
index 000000000000..c4dec49e9ca4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/sync_prim_internal.h
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation typedef header
+@Description    Defines synchronisation types that are used internally
+                only
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_INTERNAL_H
+#define SYNC_INTERNAL_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include <powervr/mem_types.h>
+
+/* These are included here as the typedefs are required
+ * internally.
+ */
+
+typedef struct SYNC_PRIM_CONTEXT_TAG *PSYNC_PRIM_CONTEXT;
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_TAG
+{
+	volatile uint32_t __iomem *pui32LinAddr;	/*!< User pointer to the primitive */
+} PVRSRV_CLIENT_SYNC_PRIM;
+
+/*!
+ * Bundled information for a sync prim operation
+ *
+ *   Structure: #PVRSRV_CLIENT_SYNC_PRIM_OP
+ *   Typedef: ::PVRSRV_CLIENT_SYNC_PRIM_OP
+ */
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_OP_TAG
+{
+	#define PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK	(1U << 0)
+	#define PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE	(1U << 1)
+	uint32_t                    ui32Flags;       /*!< Operation flags: PVRSRV_CLIENT_SYNC_PRIM_OP_XXX */
+	PVRSRV_CLIENT_SYNC_PRIM    *psSync;          /*!< Pointer to the client sync primitive */
+	uint32_t                    ui32FenceValue;  /*!< The Fence value (only used if PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK is set) */
+	uint32_t                    ui32UpdateValue; /*!< The Update value (only used if PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE is set) */
+} PVRSRV_CLIENT_SYNC_PRIM_OP;
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* SYNC_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/apollo_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/apollo_regs.h
new file mode 100644
index 000000000000..4081e2123ac7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/apollo_regs.h
@@ -0,0 +1,108 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(APOLLO_REGS_H)
+#define APOLLO_REGS_H
+
+#include "tc_clocks.h"
+
+/* TC TCF5 */
+#define TC5_SYS_APOLLO_REG_PCI_BASENUM (1)
+#define TC5_SYS_APOLLO_REG_PDP2_OFFSET (0x800000)
+#define TC5_SYS_APOLLO_REG_PDP2_SIZE   (0x7C4)
+
+#define TC5_SYS_APOLLO_REG_PDP2_FBDC_OFFSET (0xA00000)
+#define TC5_SYS_APOLLO_REG_PDP2_FBDC_SIZE   (0x14)
+
+#define TC5_SYS_APOLLO_REG_HDMI_OFFSET (0xC00000)
+#define TC5_SYS_APOLLO_REG_HDMI_SIZE   (0x1C)
+
+/* TC ES2 */
+#define TCF_TEMP_SENSOR_SPI_OFFSET	0xe
+#define TCF_TEMP_SENSOR_TO_C(raw)	(((raw) * 248 / 4096) - 54)
+
+/* Number of bytes that are broken */
+#define SYS_DEV_MEM_BROKEN_BYTES	(1024 * 1024)
+#define SYS_DEV_MEM_REGION_SIZE		(0x40000000 - SYS_DEV_MEM_BROKEN_BYTES)
+
+/* Apollo reg on base register 0 */
+#define SYS_APOLLO_REG_PCI_BASENUM	(0)
+#define SYS_APOLLO_REG_REGION_SIZE	(0x00010000)
+
+#define SYS_APOLLO_REG_SYS_OFFSET	(0x0000)
+#define SYS_APOLLO_REG_SYS_SIZE		(0x0400)
+
+#define SYS_APOLLO_REG_PLL_OFFSET	(0x1000)
+#define SYS_APOLLO_REG_PLL_SIZE		(0x0400)
+
+#define SYS_APOLLO_REG_HOST_OFFSET	(0x4050)
+#define SYS_APOLLO_REG_HOST_SIZE	(0x0014)
+
+#define SYS_APOLLO_REG_PDP1_OFFSET	(0xC000)
+#define SYS_APOLLO_REG_PDP1_SIZE	(0x2000)
+
+/* Offsets for flashing Apollo PROMs from base 0 */
+#define APOLLO_FLASH_STAT_OFFSET	(0x4058)
+#define APOLLO_FLASH_DATA_WRITE_OFFSET	(0x4050)
+#define APOLLO_FLASH_RESET_OFFSET	(0x4060)
+
+#define APOLLO_FLASH_FIFO_STATUS_MASK	 (0xF)
+#define APOLLO_FLASH_FIFO_STATUS_SHIFT	 (0)
+#define APOLLO_FLASH_PROGRAM_STATUS_MASK (0xF)
+#define APOLLO_FLASH_PROGRAM_STATUS_SHIFT (16)
+
+#define APOLLO_FLASH_PROG_COMPLETE_BIT	(0x1)
+#define APOLLO_FLASH_PROG_PROGRESS_BIT	(0x2)
+#define APOLLO_FLASH_PROG_FAILED_BIT	(0x4)
+#define APOLLO_FLASH_INV_FILETYPE_BIT	(0x8)
+
+#define APOLLO_FLASH_FIFO_SIZE		(8)
+
+/* RGX reg on base register 1 */
+#define SYS_RGX_REG_PCI_BASENUM		(1)
+#define SYS_RGX_REG_REGION_SIZE		(0x7FFFF)
+
+/* Device memory (including HP mapping) on base register 2 */
+#define SYS_DEV_MEM_PCI_BASENUM		(2)
+
+#endif /* APOLLO_REGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/bonnie_tcf.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/bonnie_tcf.h
new file mode 100644
index 000000000000..fc87ec790df9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/bonnie_tcf.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* bonnie_tcf.h - Bonnie TCF register definitions */
+
+/* tab size 4 */
+
+#ifndef BONNIE_TCF_DEFS_H
+#define BONNIE_TCF_DEFS_H
+
+#define BONNIE_TCF_OFFSET_BONNIETC_REGBANK							0x00000000
+#define BONNIE_TCF_OFFSET_TC_IFACE_COUNTERS							0x00004000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_IMGV4_RTM_TOP				0x00008000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_TCF_SCRATCH_PAD_SECN		0x0000C000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_TCF_SCRATCH_PAD_DBG		0x00010000
+#define BONNIE_TCF_OFFSET_MULTI_CLK_ALIGN							0x00014000
+#define BONNIE_TCF_OFFSET_ALIGN_DATA_TX								0x00018000
+#define BONNIE_TCF_OFFSET_SAI_RX_1									0x0001C000
+#define BONNIE_TCF_OFFSET_SAI_RX_SDR								0x00040000
+#define BONNIE_TCF_OFFSET_SAI_TX_1									0x00044000
+#define BONNIE_TCF_OFFSET_SAI_TX_SDR								0x00068000
+
+#define BONNIE_TCF_OFFSET_SAI_RX_DELTA								0x00004000
+#define BONNIE_TCF_OFFSET_SAI_TX_DELTA								0x00004000
+
+#define BONNIE_TCF_OFFSET_SAI_CLK_TAPS								0x0000000C
+#define BONNIE_TCF_OFFSET_SAI_EYES									0x00000010
+#define BONNIE_TCF_OFFSET_SAI_TRAIN_ACK								0x00000018
+
+
+#endif /* BONNIE_TCF_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_defs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_defs.h
new file mode 100644
index 000000000000..10ca9b6d4382
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_defs.h
@@ -0,0 +1,338 @@
+/****************************************************************************
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin Memory Map - View from PCIe
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+****************************************************************************/
+
+#ifndef _ODIN_DEFS_H_
+#define _ODIN_DEFS_H_
+
+/* These defines have not been autogenerated */
+
+#define PCI_VENDOR_ID_ODIN                  (0x1AEE)
+#define DEVICE_ID_ODIN                      (0x1010)
+#define DEVICE_ID_VALI                      (0x2010)
+#define DEVICE_ID_TBA                       (0x1CF2)
+
+/* PCI BAR 0 contains the PDP regs and the Odin system regs */
+#define ODN_SYS_BAR                         0
+#define ODN_SYS_REGION_SIZE                 0x000800000 /* 8MB */
+
+#define ODN_SYS_REGS_OFFSET                 0
+#define ODN_SYS_REGS_SIZE                   0x000400000 /* 4MB */
+
+#define ODN_PDP_REGS_OFFSET                 0x000440000
+#define ODN_PDP_REGS_SIZE                   0x000040000 /* 256k */
+
+#define ODN_PDP2_REGS_OFFSET                0x000480000
+#define ODN_PDP2_REGS_SIZE                  0x000040000 /* 256k */
+
+#define ODN_PDP2_PFIM_OFFSET                0x000500000
+#define ODN_PDP2_PFIM_SIZE                  0x000040000 /* 256k */
+
+#define ODIN_DMA_REGS_OFFSET                0x0004C0000
+#define ODIN_DMA_REGS_SIZE                  0x000040000 /* 256k */
+
+#define ODIN_DMA_CHAN_REGS_SIZE             0x000001000 /*   4k */
+
+/* PCI BAR 2 contains the Device Under Test SOCIF 64MB region */
+#define ODN_DUT_SOCIF_BAR                   2
+#define ODN_DUT_SOCIF_OFFSET                0x000000000
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* Reserve only the strictly required register range for each OSID */
+#define ODN_DUT_SOCIF_SIZE                  0x000010000 /* 64k */
+#else
+/* Grant the driver access to the entire register IO range */
+#define ODN_DUT_SOCIF_SIZE                  0x004000000 /* 64MB */
+#endif
+
+/* PCI BAR 4 contains the on-board 1GB DDR memory */
+#define ODN_DDR_BAR                         4
+#define ODN_DDR_MEM_OFFSET                  0x000000000
+#define ODN_DDR_MEM_SIZE                    0x040000000 /* 1GB */
+
+/* Odin system register banks */
+#define ODN_REG_BANK_CORE                   0x00000
+#define ODN_REG_BANK_TCF_SPI_MASTER         0x02000
+#define ODN_REG_BANK_ODN_CLK_BLK            0x0A000
+#define ODN_REG_BANK_ODN_MCU_COMMUNICATOR   0x0C000
+#define ODN_REG_BANK_DB_TYPE_ID             0x0C200
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_TCFVUOCTA   0x000000C6U
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_MASK   0x000000C0U
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_SHIFT  0x6
+#define ODN_REG_BANK_ODN_I2C                0x0E000
+#define ODN_REG_BANK_MULTI_CLK_ALIGN        0x20000
+#define ODN_REG_BANK_ALIGN_DATA_TX          0x22000
+#define ODN_REG_BANK_SAI_RX_DDR_0           0x24000
+#define ODN_REG_BANK_SAI_RX_DDR(n)          (ODN_REG_BANK_SAI_RX_DDR_0 + (0x02000*n))
+#define ODN_REG_BANK_SAI_TX_DDR_0           0x3A000
+#define ODN_REG_BANK_SAI_TX_DDR(n)          (ODN_REG_BANK_SAI_TX_DDR_0 + (0x02000*n))
+#define ODN_REG_BANK_SAI_TX_SDR             0x4E000
+
+/* Odin SPI regs */
+#define ODN_SPI_MST_ADDR_RDNWR              0x0000
+#define ODN_SPI_MST_WDATA                   0x0004
+#define ODN_SPI_MST_RDATA                   0x0008
+#define ODN_SPI_MST_STATUS                  0x000C
+#define ODN_SPI_MST_GO                      0x0010
+
+
+/*
+   Odin CLK regs - the odn_clk_blk module defs are not auto generated
+ */
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG1           0x620
+#define ODN_PDP_PCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define ODN_PDP_PCLK_ODIV1_LO_TIME_SHIFT         0
+#define ODN_PDP_PCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define ODN_PDP_PCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG2           0x624
+#define ODN_PDP_PCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define ODN_PDP_PCLK_ODIV2_NOCOUNT_SHIFT         6
+#define ODN_PDP_PCLK_ODIV2_EDGE_MASK             0x00000080U
+#define ODN_PDP_PCLK_ODIV2_EDGE_SHIFT            7
+
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG3           0x61C
+
+#define ODN_PDP_M_CLK_OUT_DIVIDER_REG1           0x628
+#define ODN_PDP_MCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define ODN_PDP_MCLK_ODIV1_LO_TIME_SHIFT         0
+#define ODN_PDP_MCLK_ODIV1_HI_TIME_MASK	         0x00000FC0U
+#define ODN_PDP_MCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define ODN_PDP_M_CLK_OUT_DIVIDER_REG2           0x62C
+#define ODN_PDP_MCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define ODN_PDP_MCLK_ODIV2_NOCOUNT_SHIFT         6
+#define ODN_PDP_MCLK_ODIV2_EDGE_MASK             0x00000080U
+#define ODN_PDP_MCLK_ODIV2_EDGE_SHIFT            7
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG1            0x650
+#define ODN_PDP_PCLK_MUL1_LO_TIME_MASK           0x0000003FU
+#define ODN_PDP_PCLK_MUL1_LO_TIME_SHIFT          0
+#define ODN_PDP_PCLK_MUL1_HI_TIME_MASK           0x00000FC0U
+#define ODN_PDP_PCLK_MUL1_HI_TIME_SHIFT          6
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG2            0x654
+#define ODN_PDP_PCLK_MUL2_NOCOUNT_MASK           0x00000040U
+#define ODN_PDP_PCLK_MUL2_NOCOUNT_SHIFT          6
+#define ODN_PDP_PCLK_MUL2_EDGE_MASK              0x00000080U
+#define ODN_PDP_PCLK_MUL2_EDGE_SHIFT             7
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG3            0x64C
+
+#define ODN_PDP_P_CLK_IN_DIVIDER_REG             0x658
+#define ODN_PDP_PCLK_IDIV_LO_TIME_MASK           0x0000003FU
+#define ODN_PDP_PCLK_IDIV_LO_TIME_SHIFT          0
+#define ODN_PDP_PCLK_IDIV_HI_TIME_MASK           0x00000FC0U
+#define ODN_PDP_PCLK_IDIV_HI_TIME_SHIFT          6
+#define ODN_PDP_PCLK_IDIV_NOCOUNT_MASK           0x00001000U
+#define ODN_PDP_PCLK_IDIV_NOCOUNT_SHIFT          12
+#define ODN_PDP_PCLK_IDIV_EDGE_MASK              0x00002000U
+#define ODN_PDP_PCLK_IDIV_EDGE_SHIFT             13
+
+/*
+ * DUT core clock input divider, multiplier and out divider.
+ */
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1                (0x0028)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  (0)
+
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2                (0x002C)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_MASK      (0x00000080U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_SHIFT     (7)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_MASK   (0x00000040U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  (6)
+
+#define ODN_DUT_CORE_CLK_MULTIPLIER1                 (0x0050)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME_MASK    (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME_SHIFT   (6)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME_MASK    (0x0000003FU)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME_SHIFT   (0)
+
+#define ODN_DUT_CORE_CLK_MULTIPLIER2                 (0x0054)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_MASK       (0x00007000U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_SHIFT      (12)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_EN_MASK    (0x00000800U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_EN_SHIFT   (11)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE_MASK       (0x00000080U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE_SHIFT      (7)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT_MASK    (0x00000040U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT_SHIFT   (6)
+
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1                 (0x0058)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE_MASK       (0x00002000U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE_SHIFT      (13)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT_MASK    (0x00001000U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT_SHIFT   (12)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME_MASK    (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME_SHIFT   (6)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME_MASK    (0x0000003FU)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME_SHIFT   (0)
+
+/*
+ * DUT interface clock input divider, multiplier and out divider.
+ */
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1               (0x0220)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME_MASK  (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT (6)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME_MASK  (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT (0)
+
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2               (0x0224)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE_MASK     (0x00000080U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE_SHIFT    (7)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT_MASK  (0x00000040U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT (6)
+
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1                (0x0250)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME_SHIFT  (0)
+
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2                (0x0254)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_MASK      (0x00007000U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_SHIFT     (12)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_EN_MASK   (0x00000800U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_EN_SHIFT  (11)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE_MASK      (0x00000080U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE_SHIFT     (7)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT_MASK   (0x00000040U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT_SHIFT  (6)
+
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1                (0x0258)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE_MASK      (0x00002000U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE_SHIFT     (13)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT_MASK   (0x00001000U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT_SHIFT  (12)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME_SHIFT  (0)
+
+
+/*
+ * Min max values from Xilinx Virtex7 data sheet DS183, for speed grade 2
+ * All in Hz
+ */
+#define ODN_INPUT_CLOCK_SPEED                        (100000000U)
+#define ODN_INPUT_CLOCK_SPEED_MIN                    (10000000U)
+#define ODN_INPUT_CLOCK_SPEED_MAX                    (933000000U)
+#define ODN_OUTPUT_CLOCK_SPEED_MIN                   (4690000U)
+#define ODN_OUTPUT_CLOCK_SPEED_MAX                   (933000000U)
+#define ODN_VCO_MIN                                  (600000000U)
+#define ODN_VCO_MAX                                  (1440000000U)
+#define ODN_PFD_MIN                                  (10000000U)
+#define ODN_PFD_MAX                                  (500000000U)
+
+/*
+ * Max values that can be set in DRP registers
+ */
+#define ODN_OREG_VALUE_MAX                            (126.875f)
+#define ODN_MREG_VALUE_MAX                            (126.875f)
+#define ODN_DREG_VALUE_MAX                            (126U)
+
+
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE                (0x00000001U)
+#define ODN_MMCM_LOCK_STATUS_DUT_IF                  (0x00000002U)
+#define ODN_MMCM_LOCK_STATUS_PDPP                    (0x00000008U)
+
+/*
+    Odin interrupt flags
+*/
+#define ODN_INTERRUPT_ENABLE_PDP1           (1 << ODN_INTERRUPT_ENABLE_PDP1_SHIFT)
+#define ODN_INTERRUPT_ENABLE_PDP2           (1 << ODN_INTERRUPT_ENABLE_PDP2_SHIFT)
+#define ODN_INTERRUPT_ENABLE_DUT            (1 << ODN_INTERRUPT_ENABLE_DUT_SHIFT)
+#define ODN_INTERRUPT_STATUS_PDP1           (1 << ODN_INTERRUPT_STATUS_PDP1_SHIFT)
+#define ODN_INTERRUPT_STATUS_PDP2           (1 << ODN_INTERRUPT_STATUS_PDP2_SHIFT)
+#define ODN_INTERRUPT_STATUS_DUT            (1 << ODN_INTERRUPT_STATUS_DUT_SHIFT)
+#define ODN_INTERRUPT_CLEAR_PDP1            (1 << ODN_INTERRUPT_CLR_PDP1_SHIFT)
+#define ODN_INTERRUPT_CLEAR_PDP2            (1 << ODN_INTERRUPT_CLR_PDP2_SHIFT)
+#define ODN_INTERRUPT_CLEAR_DUT             (1 << ODN_INTERRUPT_CLR_DUT_SHIFT)
+
+#define ODN_INTERRUPT_ENABLE_CDMA           (1 << ODN_INTERRUPT_ENABLE_CDMA_SHIFT)
+#define ODN_INTERRUPT_STATUS_CDMA           (1 << ODN_INTERRUPT_STATUS_CDMA_SHIFT)
+#define ODN_INTERRUPT_CLEAR_CDMA            (1 << ODN_INTERRUPT_CLR_CDMA_SHIFT)
+
+#define ODN_INTERRUPT_ENABLE_CDMA2          (1 << (ODN_INTERRUPT_ENABLE_CDMA_SHIFT + 1))
+#define ODN_INTERRUPT_STATUS_CDMA2          (1 << (ODN_INTERRUPT_STATUS_CDMA_SHIFT + 1))
+#define ODN_INTERRUPT_CLEAR_CDMA2           (1 << (ODN_INTERRUPT_CLR_CDMA_SHIFT + 1))
+
+#define ODN_INTERRUPT_ENABLE_OSID(n)        (1 << (ODN_INTERRUPT_ENABLE_OS_IRQ_SHIFT + (n)))
+#define ODN_INTERRUPT_STATUS_OSID(n)        (1 << (ODN_INTERRUPT_STATUS_OS_IRQ_SHIFT + (n)))
+#define ODN_INTERRUPT_CLEAR_OSID(n)         (1 << (ODN_INTERRUPT_CLR_OS_IRQ_SHIFT + (n)))
+
+/*
+   Other defines
+*/
+#define ODN_STREAM_OFF                      0
+#define ODN_STREAM_ON                       1
+#define ODN_SYNC_GEN_DISABLE                0
+#define ODN_SYNC_GEN_ENABLE                 1
+#define ODN_INTERLACE_DISABLE               0
+#define ODN_INTERLACE_ENABLE                1
+#define ODN_PIXEL_CLOCK_INVERTED            1
+#define ODN_HSYNC_POLARITY_ACTIVE_HIGH      1
+
+#define ODN_PDP_INTCLR_ALL                  0x000FFFFFU
+#define	ODN_PDP_INTSTAT_ALL_OURUN_MASK      0x000FFFF0U
+
+/*
+   DMA defs
+*/
+#define ODN_CDMA_ADDR_WIDTH                35
+#define ODN_DMA_HW_DESC_HEAP_SIZE          0x100000
+#define ODN_DMA_CHAN_RX                    0
+#define ODN_DMA_CHAN_TX                    1
+
+#define ODIN_DMA_TX_CHAN_NAME              "tx"
+#define ODIN_DMA_RX_CHAN_NAME              "rx"
+
+/*
+   FBC defs
+*/
+#define ODIN_PFIM_RELNUM                   (005U)
+
+#endif /* _ODIN_DEFS_H_ */
+
+/*****************************************************************************
+ End of file (odn_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_pdp_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_pdp_regs.h
new file mode 100644
index 000000000000..da47a253db31
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_pdp_regs.h
@@ -0,0 +1,8540 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* tab size 4 */
+
+#ifndef ODN_PDP_REGS_H
+#define ODN_PDP_REGS_H
+
+/* Odin-PDP hardware register definitions */
+
+
+#define ODN_PDP_GRPH1SURF_OFFSET					(0x0000)
+
+/* PDP, GRPH1SURF, GRPH1PIXFMT
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USEGAMMA
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USECSC
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USELUT
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2SURF_OFFSET					(0x0004)
+
+/* PDP, GRPH2SURF, GRPH2PIXFMT
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USEGAMMA
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USECSC
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USELUT
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3SURF_OFFSET					(0x0008)
+
+/* PDP, GRPH3SURF, GRPH3PIXFMT
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USEGAMMA
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USECSC
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USELUT
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4SURF_OFFSET					(0x000C)
+
+/* PDP, GRPH4SURF, GRPH4PIXFMT
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USEGAMMA
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USECSC
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USELUT
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SURF_OFFSET						(0x0010)
+
+/* PDP, VID1SURF, VID1PIXFMT
+*/
+#define ODN_PDP_VID1SURF_VID1PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEGAMMA
+*/
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USECSC
+*/
+#define ODN_PDP_VID1SURF_VID1USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID1SURF_VID1USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USECSC_SHIFT			(25)
+#define ODN_PDP_VID1SURF_VID1USECSC_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEI2P
+*/
+#define ODN_PDP_VID1SURF_VID1USEI2P_MASK			(0x01000000)
+#define ODN_PDP_VID1SURF_VID1USEI2P_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEI2P_SHIFT			(24)
+#define ODN_PDP_VID1SURF_VID1USEI2P_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USEI2P_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1COSITED
+*/
+#define ODN_PDP_VID1SURF_VID1COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID1SURF_VID1COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1COSITED_SHIFT			(23)
+#define ODN_PDP_VID1SURF_VID1COSITED_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEHQCD
+*/
+#define ODN_PDP_VID1SURF_VID1USEHQCD_MASK			(0x00400000)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_SHIFT			(22)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEINSTREAM
+*/
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_MASK		(0x00200000)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_LSBMASK	(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_SHIFT		(21)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_LENGTH		(1)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2SURF_OFFSET						(0x0014)
+
+/* PDP, VID2SURF, VID2PIXFMT
+*/
+#define ODN_PDP_VID2SURF_VID2PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2COSITED
+*/
+#define ODN_PDP_VID2SURF_VID2COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID2SURF_VID2COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SURF_VID2COSITED_SHIFT			(23)
+#define ODN_PDP_VID2SURF_VID2COSITED_LENGTH			(1)
+#define ODN_PDP_VID2SURF_VID2COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USEGAMMA
+*/
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USECSC
+*/
+#define ODN_PDP_VID2SURF_VID2USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID2SURF_VID2USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2SURF_VID2USECSC_SHIFT			(25)
+#define ODN_PDP_VID2SURF_VID2USECSC_LENGTH			(1)
+#define ODN_PDP_VID2SURF_VID2USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SURF_OFFSET						(0x0018)
+
+/* PDP, VID3SURF, VID3PIXFMT
+*/
+#define ODN_PDP_VID3SURF_VID3PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3COSITED
+*/
+#define ODN_PDP_VID3SURF_VID3COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID3SURF_VID3COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SURF_VID3COSITED_SHIFT			(23)
+#define ODN_PDP_VID3SURF_VID3COSITED_LENGTH			(1)
+#define ODN_PDP_VID3SURF_VID3COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USEGAMMA
+*/
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USECSC
+*/
+#define ODN_PDP_VID3SURF_VID3USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID3SURF_VID3USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3SURF_VID3USECSC_SHIFT			(25)
+#define ODN_PDP_VID3SURF_VID3USECSC_LENGTH			(1)
+#define ODN_PDP_VID3SURF_VID3USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SURF_OFFSET						(0x001C)
+
+/* PDP, VID4SURF, VID4PIXFMT
+*/
+#define ODN_PDP_VID4SURF_VID4PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4COSITED
+*/
+#define ODN_PDP_VID4SURF_VID4COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID4SURF_VID4COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4SURF_VID4COSITED_SHIFT			(23)
+#define ODN_PDP_VID4SURF_VID4COSITED_LENGTH			(1)
+#define ODN_PDP_VID4SURF_VID4COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USEGAMMA
+*/
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USECSC
+*/
+#define ODN_PDP_VID4SURF_VID4USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID4SURF_VID4USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4SURF_VID4USECSC_SHIFT			(25)
+#define ODN_PDP_VID4SURF_VID4USECSC_LENGTH			(1)
+#define ODN_PDP_VID4SURF_VID4USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1CTRL_OFFSET					(0x0020)
+
+/* PDP, GRPH1CTRL, GRPH1STREN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_SHIFT			(31)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_LENGTH			(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYEN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYSRC
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLEND
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLENDPOS
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1DITHEREN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CTRL_OFFSET					(0x0024)
+
+/* PDP, GRPH2CTRL, GRPH2STREN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_SHIFT			(31)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_LENGTH			(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYEN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYSRC
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLEND
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLENDPOS
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2DITHEREN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CTRL_OFFSET					(0x0028)
+
+/* PDP, GRPH3CTRL, GRPH3STREN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_SHIFT			(31)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_LENGTH			(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYEN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYSRC
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLEND
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLENDPOS
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3DITHEREN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CTRL_OFFSET					(0x002C)
+
+/* PDP, GRPH4CTRL, GRPH4STREN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_SHIFT			(31)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_LENGTH			(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYEN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYSRC
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLEND
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLENDPOS
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4DITHEREN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1CTRL_OFFSET						(0x0030)
+
+/* PDP, VID1CTRL, VID1STREN
+*/
+#define ODN_PDP_VID1CTRL_VID1STREN_MASK				(0x80000000)
+#define ODN_PDP_VID1CTRL_VID1STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1STREN_SHIFT			(31)
+#define ODN_PDP_VID1CTRL_VID1STREN_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYEN
+*/
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYSRC
+*/
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLEND
+*/
+#define ODN_PDP_VID1CTRL_VID1BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID1CTRL_VID1BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID1CTRL_VID1BLEND_SHIFT			(27)
+#define ODN_PDP_VID1CTRL_VID1BLEND_LENGTH			(2)
+#define ODN_PDP_VID1CTRL_VID1BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLENDPOS
+*/
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1DITHEREN
+*/
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CTRL_OFFSET						(0x0034)
+
+/* PDP, VID2CTRL, VID2STREN
+*/
+#define ODN_PDP_VID2CTRL_VID2STREN_MASK				(0x80000000)
+#define ODN_PDP_VID2CTRL_VID2STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2STREN_SHIFT			(31)
+#define ODN_PDP_VID2CTRL_VID2STREN_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYEN
+*/
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYSRC
+*/
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLEND
+*/
+#define ODN_PDP_VID2CTRL_VID2BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID2CTRL_VID2BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID2CTRL_VID2BLEND_SHIFT			(27)
+#define ODN_PDP_VID2CTRL_VID2BLEND_LENGTH			(2)
+#define ODN_PDP_VID2CTRL_VID2BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLENDPOS
+*/
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2DITHEREN
+*/
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CTRL_OFFSET						(0x0038)
+
+/* PDP, VID3CTRL, VID3STREN
+*/
+#define ODN_PDP_VID3CTRL_VID3STREN_MASK				(0x80000000)
+#define ODN_PDP_VID3CTRL_VID3STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3STREN_SHIFT			(31)
+#define ODN_PDP_VID3CTRL_VID3STREN_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYEN
+*/
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYSRC
+*/
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLEND
+*/
+#define ODN_PDP_VID3CTRL_VID3BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID3CTRL_VID3BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID3CTRL_VID3BLEND_SHIFT			(27)
+#define ODN_PDP_VID3CTRL_VID3BLEND_LENGTH			(2)
+#define ODN_PDP_VID3CTRL_VID3BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLENDPOS
+*/
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3DITHEREN
+*/
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CTRL_OFFSET						(0x003C)
+
+/* PDP, VID4CTRL, VID4STREN
+*/
+#define ODN_PDP_VID4CTRL_VID4STREN_MASK				(0x80000000)
+#define ODN_PDP_VID4CTRL_VID4STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4STREN_SHIFT			(31)
+#define ODN_PDP_VID4CTRL_VID4STREN_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYEN
+*/
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYSRC
+*/
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLEND
+*/
+#define ODN_PDP_VID4CTRL_VID4BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID4CTRL_VID4BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID4CTRL_VID4BLEND_SHIFT			(27)
+#define ODN_PDP_VID4CTRL_VID4BLEND_LENGTH			(2)
+#define ODN_PDP_VID4CTRL_VID4BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLENDPOS
+*/
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4DITHEREN
+*/
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1UCTRL_OFFSET					(0x0050)
+
+/* PDP, VID1UCTRL, VID1UVHALFSTR
+*/
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2UCTRL_OFFSET					(0x0054)
+
+/* PDP, VID2UCTRL, VID2UVHALFSTR
+*/
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3UCTRL_OFFSET					(0x0058)
+
+/* PDP, VID3UCTRL, VID3UVHALFSTR
+*/
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4UCTRL_OFFSET					(0x005C)
+
+/* PDP, VID4UCTRL, VID4UVHALFSTR
+*/
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1STRIDE_OFFSET					(0x0060)
+
+/* PDP, GRPH1STRIDE, GRPH1STRIDE
+*/
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2STRIDE_OFFSET					(0x0064)
+
+/* PDP, GRPH2STRIDE, GRPH2STRIDE
+*/
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3STRIDE_OFFSET					(0x0068)
+
+/* PDP, GRPH3STRIDE, GRPH3STRIDE
+*/
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4STRIDE_OFFSET					(0x006C)
+
+/* PDP, GRPH4STRIDE, GRPH4STRIDE
+*/
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1STRIDE_OFFSET					(0x0070)
+
+/* PDP, VID1STRIDE, VID1STRIDE
+*/
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_SHIFT			(22)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_LENGTH		(10)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2STRIDE_OFFSET					(0x0074)
+
+/* PDP, VID2STRIDE, VID2STRIDE
+*/
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_SHIFT			(22)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_LENGTH		(10)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3STRIDE_OFFSET					(0x0078)
+
+/* PDP, VID3STRIDE, VID3STRIDE
+*/
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_SHIFT			(22)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_LENGTH		(10)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4STRIDE_OFFSET					(0x007C)
+
+/* PDP, VID4STRIDE, VID4STRIDE
+*/
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_SHIFT			(22)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_LENGTH		(10)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1SIZE_OFFSET					(0x0080)
+
+/* PDP, GRPH1SIZE, GRPH1WIDTH
+*/
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SIZE, GRPH1HEIGHT
+*/
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2SIZE_OFFSET					(0x0084)
+
+/* PDP, GRPH2SIZE, GRPH2WIDTH
+*/
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SIZE, GRPH2HEIGHT
+*/
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3SIZE_OFFSET					(0x0088)
+
+/* PDP, GRPH3SIZE, GRPH3WIDTH
+*/
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SIZE, GRPH3HEIGHT
+*/
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4SIZE_OFFSET					(0x008C)
+
+/* PDP, GRPH4SIZE, GRPH4WIDTH
+*/
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SIZE, GRPH4HEIGHT
+*/
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SIZE_OFFSET						(0x0090)
+
+/* PDP, VID1SIZE, VID1WIDTH
+*/
+#define ODN_PDP_VID1SIZE_VID1WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_SHIFT			(16)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_LENGTH			(12)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1SIZE, VID1HEIGHT
+*/
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SIZE_OFFSET						(0x0094)
+
+/* PDP, VID2SIZE, VID2WIDTH
+*/
+#define ODN_PDP_VID2SIZE_VID2WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_SHIFT			(16)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_LENGTH			(12)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2SIZE, VID2HEIGHT
+*/
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SIZE_OFFSET						(0x0098)
+
+/* PDP, VID3SIZE, VID3WIDTH
+*/
+#define ODN_PDP_VID3SIZE_VID3WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_SHIFT			(16)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_LENGTH			(12)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3SIZE, VID3HEIGHT
+*/
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SIZE_OFFSET						(0x009C)
+
+/* PDP, VID4SIZE, VID4WIDTH
+*/
+#define ODN_PDP_VID4SIZE_VID4WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_SHIFT			(16)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_LENGTH			(12)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4SIZE, VID4HEIGHT
+*/
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1POSN_OFFSET					(0x00A0)
+
+/* PDP, GRPH1POSN, GRPH1XSTART
+*/
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1POSN, GRPH1YSTART
+*/
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2POSN_OFFSET					(0x00A4)
+
+/* PDP, GRPH2POSN, GRPH2XSTART
+*/
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2POSN, GRPH2YSTART
+*/
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3POSN_OFFSET					(0x00A8)
+
+/* PDP, GRPH3POSN, GRPH3XSTART
+*/
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3POSN, GRPH3YSTART
+*/
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4POSN_OFFSET					(0x00AC)
+
+/* PDP, GRPH4POSN, GRPH4XSTART
+*/
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4POSN, GRPH4YSTART
+*/
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1POSN_OFFSET						(0x00B0)
+
+/* PDP, VID1POSN, VID1XSTART
+*/
+#define ODN_PDP_VID1POSN_VID1XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID1POSN_VID1XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1XSTART_SHIFT			(16)
+#define ODN_PDP_VID1POSN_VID1XSTART_LENGTH			(12)
+#define ODN_PDP_VID1POSN_VID1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSN, VID1YSTART
+*/
+#define ODN_PDP_VID1POSN_VID1YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1YSTART_SHIFT			(0)
+#define ODN_PDP_VID1POSN_VID1YSTART_LENGTH			(12)
+#define ODN_PDP_VID1POSN_VID1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2POSN_OFFSET						(0x00B4)
+
+/* PDP, VID2POSN, VID2XSTART
+*/
+#define ODN_PDP_VID2POSN_VID2XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID2POSN_VID2XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2XSTART_SHIFT			(16)
+#define ODN_PDP_VID2POSN_VID2XSTART_LENGTH			(12)
+#define ODN_PDP_VID2POSN_VID2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSN, VID2YSTART
+*/
+#define ODN_PDP_VID2POSN_VID2YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2YSTART_SHIFT			(0)
+#define ODN_PDP_VID2POSN_VID2YSTART_LENGTH			(12)
+#define ODN_PDP_VID2POSN_VID2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3POSN_OFFSET						(0x00B8)
+
+/* PDP, VID3POSN, VID3XSTART
+*/
+#define ODN_PDP_VID3POSN_VID3XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID3POSN_VID3XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3XSTART_SHIFT			(16)
+#define ODN_PDP_VID3POSN_VID3XSTART_LENGTH			(12)
+#define ODN_PDP_VID3POSN_VID3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSN, VID3YSTART
+*/
+#define ODN_PDP_VID3POSN_VID3YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3YSTART_SHIFT			(0)
+#define ODN_PDP_VID3POSN_VID3YSTART_LENGTH			(12)
+#define ODN_PDP_VID3POSN_VID3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4POSN_OFFSET						(0x00BC)
+
+/* PDP, VID4POSN, VID4XSTART
+*/
+#define ODN_PDP_VID4POSN_VID4XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID4POSN_VID4XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4XSTART_SHIFT			(16)
+#define ODN_PDP_VID4POSN_VID4XSTART_LENGTH			(12)
+#define ODN_PDP_VID4POSN_VID4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSN, VID4YSTART
+*/
+#define ODN_PDP_VID4POSN_VID4YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4YSTART_SHIFT			(0)
+#define ODN_PDP_VID4POSN_VID4YSTART_LENGTH			(12)
+#define ODN_PDP_VID4POSN_VID4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1GALPHA_OFFSET					(0x00C0)
+
+/* PDP, GRPH1GALPHA, GRPH1GALPHA
+*/
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2GALPHA_OFFSET					(0x00C4)
+
+/* PDP, GRPH2GALPHA, GRPH2GALPHA
+*/
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3GALPHA_OFFSET					(0x00C8)
+
+/* PDP, GRPH3GALPHA, GRPH3GALPHA
+*/
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4GALPHA_OFFSET					(0x00CC)
+
+/* PDP, GRPH4GALPHA, GRPH4GALPHA
+*/
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1GALPHA_OFFSET					(0x00D0)
+
+/* PDP, VID1GALPHA, VID1GALPHA
+*/
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_SHIFT			(0)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_LENGTH		(10)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2GALPHA_OFFSET					(0x00D4)
+
+/* PDP, VID2GALPHA, VID2GALPHA
+*/
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_SHIFT			(0)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_LENGTH		(10)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3GALPHA_OFFSET					(0x00D8)
+
+/* PDP, VID3GALPHA, VID3GALPHA
+*/
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_SHIFT			(0)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_LENGTH		(10)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4GALPHA_OFFSET				    (0x00DC)
+
+/* PDP, VID4GALPHA, VID4GALPHA
+*/
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_SHIFT			(0)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_LENGTH		(10)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1CKEY_R_OFFSET					(0x00E0)
+
+/* PDP, GRPH1CKEY_R, GRPH1CKEY_R
+*/
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1CKEY_GB_OFFSET			        (0x00E4)
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_G
+*/
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_B
+*/
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CKEY_R_OFFSET					(0x00E8)
+
+/* PDP, GRPH2CKEY_R, GRPH2CKEY_R
+*/
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CKEY_GB_OFFSET					(0x00EC)
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_G
+*/
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_B
+*/
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CKEY_R_OFFSET					(0x00F0)
+
+/* PDP, GRPH3CKEY_R, GRPH3CKEY_R
+*/
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CKEY_GB_OFFSET					(0x00F4)
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_G
+*/
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_B
+*/
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CKEY_R_OFFSET					(0x00F8)
+
+/* PDP, GRPH4CKEY_R, GRPH4CKEY_R
+*/
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CKEY_GB_OFFSET					(0x00FC)
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_G
+*/
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_B
+*/
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1CKEY_R_OFFSET					(0x0100)
+
+/* PDP, VID1CKEY_R, VID1CKEY_R
+*/
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1CKEY_GB_OFFSET					(0x0104)
+
+/* PDP, VID1CKEY_GB, VID1CKEY_G
+*/
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CKEY_GB, VID1CKEY_B
+*/
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CKEY_R_OFFSET					(0x0108)
+
+/* PDP, VID2CKEY_R, VID2CKEY_R
+*/
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CKEY_GB_OFFSET					(0x010C)
+
+/* PDP, VID2CKEY_GB, VID2CKEY_G
+*/
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CKEY_GB, VID2CKEY_B
+*/
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CKEY_R_OFFSET					(0x0110)
+
+/* PDP, VID3CKEY_R, VID3CKEY_R
+*/
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CKEY_GB_OFFSET					(0x0114)
+
+/* PDP, VID3CKEY_GB, VID3CKEY_G
+*/
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CKEY_GB, VID3CKEY_B
+*/
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CKEY_R_OFFSET					(0x0118)
+
+/* PDP, VID4CKEY_R, VID4CKEY_R
+*/
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CKEY_GB_OFFSET					(0x011C)
+
+/* PDP, VID4CKEY_GB, VID4CKEY_G
+*/
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CKEY_GB, VID4CKEY_B
+*/
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1BLND2_R_OFFSET					(0x0120)
+
+/* PDP, GRPH1BLND2_R, GRPH1PIXDBL
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1LINDBL
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1CKEYMASK_R
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1BLND2_GB_OFFSET				(0x0124)
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_G
+*/
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_B
+*/
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BLND2_R_OFFSET					(0x0128)
+
+/* PDP, GRPH2BLND2_R, GRPH2PIXDBL
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2LINDBL
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2CKEYMASK_R
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BLND2_GB_OFFSET				(0x012C)
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_G
+*/
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_B
+*/
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BLND2_R_OFFSET					(0x0130)
+
+/* PDP, GRPH3BLND2_R, GRPH3PIXDBL
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3LINDBL
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3CKEYMASK_R
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BLND2_GB_OFFSET				(0x0134)
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_G
+*/
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_B
+*/
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BLND2_R_OFFSET					(0x0138)
+
+/* PDP, GRPH4BLND2_R, GRPH4PIXDBL
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4LINDBL
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4CKEYMASK_R
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BLND2_GB_OFFSET				(0x013C)
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_G
+*/
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_B
+*/
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1BLND2_R_OFFSET					(0x0140)
+
+/* PDP, VID1BLND2_R, VID1CKEYMASK_R
+*/
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1BLND2_GB_OFFSET					(0x0144)
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_G
+*/
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_B
+*/
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2BLND2_R_OFFSET		            (0x0148)
+
+/* PDP, VID2BLND2_R, VID2CKEYMASK_R
+*/
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2BLND2_GB_OFFSET		            (0x014C)
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_G
+*/
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_B
+*/
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3BLND2_R_OFFSET		            (0x0150)
+
+/* PDP, VID3BLND2_R, VID3CKEYMASK_R
+*/
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3BLND2_GB_OFFSET		            (0x0154)
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_G
+*/
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_B
+*/
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4BLND2_R_OFFSET		            (0x0158)
+
+/* PDP, VID4BLND2_R, VID4CKEYMASK_R
+*/
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4BLND2_GB_OFFSET		            (0x015C)
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_G
+*/
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_B
+*/
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_OFFSET		    (0x0160)
+
+/* PDP, GRPH1INTERLEAVE_CTRL, GRPH1INTFIELD
+*/
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_MASK	(0x00000001)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_OFFSET		    (0x0164)
+
+/* PDP, GRPH2INTERLEAVE_CTRL, GRPH2INTFIELD
+*/
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_OFFSET		    (0x0168)
+
+/* PDP, GRPH3INTERLEAVE_CTRL, GRPH3INTFIELD
+*/
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_OFFSET		    (0x016C)
+
+/* PDP, GRPH4INTERLEAVE_CTRL, GRPH4INTFIELD
+*/
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1INTERLEAVE_CTRL_OFFSET		    (0x0170)
+
+/* PDP, VID1INTERLEAVE_CTRL, VID1INTFIELD
+*/
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SHIFT (0)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LENGTH (1)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2INTERLEAVE_CTRL_OFFSET		    (0x0174)
+
+/* PDP, VID2INTERLEAVE_CTRL, VID2INTFIELD
+*/
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SHIFT (0)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LENGTH (1)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3INTERLEAVE_CTRL_OFFSET		    (0x0178)
+
+/* PDP, VID3INTERLEAVE_CTRL, VID3INTFIELD
+*/
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SHIFT (0)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LENGTH (1)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4INTERLEAVE_CTRL_OFFSET		    (0x017C)
+
+/* PDP, VID4INTERLEAVE_CTRL, VID4INTFIELD
+*/
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SHIFT (0)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LENGTH (1)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1BASEADDR_OFFSET		        (0x0180)
+
+/* PDP, GRPH1BASEADDR, GRPH1BASEADDR
+*/
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BASEADDR_OFFSET		        (0x0184)
+
+/* PDP, GRPH2BASEADDR, GRPH2BASEADDR
+*/
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BASEADDR_OFFSET		        (0x0188)
+
+/* PDP, GRPH3BASEADDR, GRPH3BASEADDR
+*/
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BASEADDR_OFFSET		        (0x018C)
+
+/* PDP, GRPH4BASEADDR, GRPH4BASEADDR
+*/
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1BASEADDR_OFFSET		            (0x0190)
+
+/* PDP, VID1BASEADDR, VID1BASEADDR
+*/
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2BASEADDR_OFFSET		            (0x0194)
+
+/* PDP, VID2BASEADDR, VID2BASEADDR
+*/
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3BASEADDR_OFFSET		            (0x0198)
+
+/* PDP, VID3BASEADDR, VID3BASEADDR
+*/
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4BASEADDR_OFFSET		            (0x019C)
+
+/* PDP, VID4BASEADDR, VID4BASEADDR
+*/
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1UBASEADDR_OFFSET				(0x01B0)
+
+/* PDP, VID1UBASEADDR, VID1UBASEADDR
+*/
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_SHIFT	(5)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH	(27)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2UBASEADDR_OFFSET		        (0x01B4)
+
+/* PDP, VID2UBASEADDR, VID2UBASEADDR
+*/
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3UBASEADDR_OFFSET		        (0x01B8)
+
+/* PDP, VID3UBASEADDR, VID3UBASEADDR
+*/
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4UBASEADDR_OFFSET		        (0x01BC)
+
+/* PDP, VID4UBASEADDR, VID4UBASEADDR
+*/
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VBASEADDR_OFFSET		        (0x01D0)
+
+/* PDP, VID1VBASEADDR, VID1VBASEADDR
+*/
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VBASEADDR_OFFSET		        (0x01D4)
+
+/* PDP, VID2VBASEADDR, VID2VBASEADDR
+*/
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VBASEADDR_OFFSET		        (0x01D8)
+
+/* PDP, VID3VBASEADDR, VID3VBASEADDR
+*/
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VBASEADDR_OFFSET		(0x01DC)
+
+/* PDP, VID4VBASEADDR, VID4VBASEADDR
+*/
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1POSTSKIPCTRL_OFFSET		(0x0230)
+
+/* PDP, VID1POSTSKIPCTRL, VID1HPOSTCLIP
+*/
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSTSKIPCTRL, VID1VPOSTCLIP
+*/
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2POSTSKIPCTRL_OFFSET		(0x0234)
+
+/* PDP, VID2POSTSKIPCTRL, VID2HPOSTCLIP
+*/
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSTSKIPCTRL, VID2VPOSTCLIP
+*/
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3POSTSKIPCTRL_OFFSET		(0x0238)
+
+/* PDP, VID3POSTSKIPCTRL, VID3HPOSTCLIP
+*/
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSTSKIPCTRL, VID3VPOSTCLIP
+*/
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4POSTSKIPCTRL_OFFSET		(0x023C)
+
+/* PDP, VID4POSTSKIPCTRL, VID4HPOSTCLIP
+*/
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSTSKIPCTRL, VID4VPOSTCLIP
+*/
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1DECIMATE_CTRL_OFFSET		(0x0240)
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_EN
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2DECIMATE_CTRL_OFFSET		(0x0244)
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_EN
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3DECIMATE_CTRL_OFFSET		(0x0248)
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_EN
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4DECIMATE_CTRL_OFFSET		(0x024C)
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_EN
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1DECIMATE_CTRL_OFFSET		(0x0250)
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_EN
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2DECIMATE_CTRL_OFFSET		(0x0254)
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_EN
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3DECIMATE_CTRL_OFFSET		(0x0258)
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_EN
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4DECIMATE_CTRL_OFFSET		(0x025C)
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_EN
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SKIPCTRL_OFFSET		(0x0270)
+
+/* PDP, VID1SKIPCTRL, VID1HSKIP
+*/
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_SHIFT		(16)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_LENGTH		(12)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SKIPCTRL, VID1VSKIP
+*/
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_SHIFT		(0)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_LENGTH		(12)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SKIPCTRL_OFFSET		(0x0274)
+
+/* PDP, VID2SKIPCTRL, VID2HSKIP
+*/
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_SHIFT		(16)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_LENGTH		(12)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SKIPCTRL, VID2VSKIP
+*/
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_SHIFT		(0)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_LENGTH		(12)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SKIPCTRL_OFFSET		(0x0278)
+
+/* PDP, VID3SKIPCTRL, VID3HSKIP
+*/
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_SHIFT		(16)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_LENGTH		(12)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SKIPCTRL, VID3VSKIP
+*/
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_SHIFT		(0)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_LENGTH		(12)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SKIPCTRL_OFFSET		(0x027C)
+
+/* PDP, VID4SKIPCTRL, VID4HSKIP
+*/
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_SHIFT		(16)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_LENGTH		(12)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SKIPCTRL, VID4VSKIP
+*/
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_SHIFT		(0)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_LENGTH		(12)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SCALECTRL_OFFSET		(0x0460)
+
+/* PDP, VID1SCALECTRL, VID1HSCALEBP
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSCALEBP
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1HSBEFOREVS
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSURUNCTRL
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1PAN_EN
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VORDER
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_SHIFT		(16)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_LENGTH		(2)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VPITCH
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_SHIFT		(0)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_LENGTH		(16)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VSINIT_OFFSET		(0x0464)
+
+/* PDP, VID1VSINIT, VID1VINITIAL1
+*/
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1VSINIT, VID1VINITIAL0
+*/
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF0_OFFSET		(0x0468)
+
+/* PDP, VID1VCOEFF0, VID1VCOEFF0
+*/
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF1_OFFSET		(0x046C)
+
+/* PDP, VID1VCOEFF1, VID1VCOEFF1
+*/
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF2_OFFSET		(0x0470)
+
+/* PDP, VID1VCOEFF2, VID1VCOEFF2
+*/
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF3_OFFSET		(0x0474)
+
+/* PDP, VID1VCOEFF3, VID1VCOEFF3
+*/
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF4_OFFSET		(0x0478)
+
+/* PDP, VID1VCOEFF4, VID1VCOEFF4
+*/
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF5_OFFSET		(0x047C)
+
+/* PDP, VID1VCOEFF5, VID1VCOEFF5
+*/
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF6_OFFSET		(0x0480)
+
+/* PDP, VID1VCOEFF6, VID1VCOEFF6
+*/
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF7_OFFSET		(0x0484)
+
+/* PDP, VID1VCOEFF7, VID1VCOEFF7
+*/
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF8_OFFSET		(0x0488)
+
+/* PDP, VID1VCOEFF8, VID1VCOEFF8
+*/
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HSINIT_OFFSET		(0x048C)
+
+/* PDP, VID1HSINIT, VID1HINITIAL
+*/
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1HSINIT, VID1HPITCH
+*/
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_SHIFT		(0)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_LENGTH		(16)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF0_OFFSET		(0x0490)
+
+/* PDP, VID1HCOEFF0, VID1HCOEFF0
+*/
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF1_OFFSET		(0x0494)
+
+/* PDP, VID1HCOEFF1, VID1HCOEFF1
+*/
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF2_OFFSET		(0x0498)
+
+/* PDP, VID1HCOEFF2, VID1HCOEFF2
+*/
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF3_OFFSET		(0x049C)
+
+/* PDP, VID1HCOEFF3, VID1HCOEFF3
+*/
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF4_OFFSET		(0x04A0)
+
+/* PDP, VID1HCOEFF4, VID1HCOEFF4
+*/
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF5_OFFSET		(0x04A4)
+
+/* PDP, VID1HCOEFF5, VID1HCOEFF5
+*/
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF6_OFFSET		(0x04A8)
+
+/* PDP, VID1HCOEFF6, VID1HCOEFF6
+*/
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF7_OFFSET		(0x04AC)
+
+/* PDP, VID1HCOEFF7, VID1HCOEFF7
+*/
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF8_OFFSET		(0x04B0)
+
+/* PDP, VID1HCOEFF8, VID1HCOEFF8
+*/
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF9_OFFSET		(0x04B4)
+
+/* PDP, VID1HCOEFF9, VID1HCOEFF9
+*/
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF10_OFFSET		(0x04B8)
+
+/* PDP, VID1HCOEFF10, VID1HCOEFF10
+*/
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF11_OFFSET		(0x04BC)
+
+/* PDP, VID1HCOEFF11, VID1HCOEFF11
+*/
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF12_OFFSET		(0x04C0)
+
+/* PDP, VID1HCOEFF12, VID1HCOEFF12
+*/
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF13_OFFSET		(0x04C4)
+
+/* PDP, VID1HCOEFF13, VID1HCOEFF13
+*/
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF14_OFFSET		(0x04C8)
+
+/* PDP, VID1HCOEFF14, VID1HCOEFF14
+*/
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF15_OFFSET		(0x04CC)
+
+/* PDP, VID1HCOEFF15, VID1HCOEFF15
+*/
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF16_OFFSET		(0x04D0)
+
+/* PDP, VID1HCOEFF16, VID1HCOEFF16
+*/
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SCALESIZE_OFFSET		(0x04D4)
+
+/* PDP, VID1SCALESIZE, VID1SCALEWIDTH
+*/
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALESIZE, VID1SCALEHEIGHT
+*/
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CORE_ID_OFFSET		(0x04E0)
+
+/* PDP, PVR_ODN_PDP_CORE_ID, GROUP_ID
+*/
+#define ODN_PDP_CORE_ID_GROUP_ID_MASK		(0xFF000000)
+#define ODN_PDP_CORE_ID_GROUP_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_ID_GROUP_ID_SHIFT		(24)
+#define ODN_PDP_CORE_ID_GROUP_ID_LENGTH		(8)
+#define ODN_PDP_CORE_ID_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_ID, CORE_ID
+*/
+#define ODN_PDP_CORE_ID_CORE_ID_MASK		(0x00FF0000)
+#define ODN_PDP_CORE_ID_CORE_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_ID_CORE_ID_SHIFT		(16)
+#define ODN_PDP_CORE_ID_CORE_ID_LENGTH		(8)
+#define ODN_PDP_CORE_ID_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_ID, CONFIG_ID
+*/
+#define ODN_PDP_CORE_ID_CONFIG_ID_MASK		(0x0000FFFF)
+#define ODN_PDP_CORE_ID_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_CORE_ID_CONFIG_ID_SHIFT		(0)
+#define ODN_PDP_CORE_ID_CONFIG_ID_LENGTH		(16)
+#define ODN_PDP_CORE_ID_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CORE_REV_OFFSET		(0x04F0)
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MAJOR_REV
+*/
+#define ODN_PDP_CORE_REV_MAJOR_REV_MASK		(0x00FF0000)
+#define ODN_PDP_CORE_REV_MAJOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAJOR_REV_SHIFT		(16)
+#define ODN_PDP_CORE_REV_MAJOR_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MINOR_REV
+*/
+#define ODN_PDP_CORE_REV_MINOR_REV_MASK		(0x0000FF00)
+#define ODN_PDP_CORE_REV_MINOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MINOR_REV_SHIFT		(8)
+#define ODN_PDP_CORE_REV_MINOR_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MAINT_REV
+*/
+#define ODN_PDP_CORE_REV_MAINT_REV_MASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAINT_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAINT_REV_SHIFT		(0)
+#define ODN_PDP_CORE_REV_MAINT_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SCALECTRL_OFFSET		(0x0500)
+
+/* PDP, VID2SCALECTRL, VID2HSCALEBP
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSCALEBP
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2HSBEFOREVS
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSURUNCTRL
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2PAN_EN
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VORDER
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_SHIFT		(16)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_LENGTH		(2)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VPITCH
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_SHIFT		(0)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_LENGTH		(16)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VSINIT_OFFSET		(0x0504)
+
+/* PDP, VID2VSINIT, VID2VINITIAL1
+*/
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2VSINIT, VID2VINITIAL0
+*/
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF0_OFFSET		(0x0508)
+
+/* PDP, VID2VCOEFF0, VID2VCOEFF0
+*/
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF1_OFFSET		(0x050C)
+
+/* PDP, VID2VCOEFF1, VID2VCOEFF1
+*/
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF2_OFFSET		(0x0510)
+
+/* PDP, VID2VCOEFF2, VID2VCOEFF2
+*/
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF3_OFFSET		(0x0514)
+
+/* PDP, VID2VCOEFF3, VID2VCOEFF3
+*/
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF4_OFFSET		(0x0518)
+
+/* PDP, VID2VCOEFF4, VID2VCOEFF4
+*/
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF5_OFFSET		(0x051C)
+
+/* PDP, VID2VCOEFF5, VID2VCOEFF5
+*/
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF6_OFFSET		(0x0520)
+
+/* PDP, VID2VCOEFF6, VID2VCOEFF6
+*/
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF7_OFFSET		(0x0524)
+
+/* PDP, VID2VCOEFF7, VID2VCOEFF7
+*/
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF8_OFFSET		(0x0528)
+
+/* PDP, VID2VCOEFF8, VID2VCOEFF8
+*/
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HSINIT_OFFSET		(0x052C)
+
+/* PDP, VID2HSINIT, VID2HINITIAL
+*/
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2HSINIT, VID2HPITCH
+*/
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_SHIFT		(0)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_LENGTH		(16)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF0_OFFSET		(0x0530)
+
+/* PDP, VID2HCOEFF0, VID2HCOEFF0
+*/
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF1_OFFSET		(0x0534)
+
+/* PDP, VID2HCOEFF1, VID2HCOEFF1
+*/
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF2_OFFSET		(0x0538)
+
+/* PDP, VID2HCOEFF2, VID2HCOEFF2
+*/
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF3_OFFSET		(0x053C)
+
+/* PDP, VID2HCOEFF3, VID2HCOEFF3
+*/
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF4_OFFSET		(0x0540)
+
+/* PDP, VID2HCOEFF4, VID2HCOEFF4
+*/
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF5_OFFSET		(0x0544)
+
+/* PDP, VID2HCOEFF5, VID2HCOEFF5
+*/
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF6_OFFSET		(0x0548)
+
+/* PDP, VID2HCOEFF6, VID2HCOEFF6
+*/
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF7_OFFSET		(0x054C)
+
+/* PDP, VID2HCOEFF7, VID2HCOEFF7
+*/
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF8_OFFSET		(0x0550)
+
+/* PDP, VID2HCOEFF8, VID2HCOEFF8
+*/
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF9_OFFSET		(0x0554)
+
+/* PDP, VID2HCOEFF9, VID2HCOEFF9
+*/
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF10_OFFSET		(0x0558)
+
+/* PDP, VID2HCOEFF10, VID2HCOEFF10
+*/
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF11_OFFSET		(0x055C)
+
+/* PDP, VID2HCOEFF11, VID2HCOEFF11
+*/
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF12_OFFSET		(0x0560)
+
+/* PDP, VID2HCOEFF12, VID2HCOEFF12
+*/
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF13_OFFSET		(0x0564)
+
+/* PDP, VID2HCOEFF13, VID2HCOEFF13
+*/
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF14_OFFSET		(0x0568)
+
+/* PDP, VID2HCOEFF14, VID2HCOEFF14
+*/
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF15_OFFSET		(0x056C)
+
+/* PDP, VID2HCOEFF15, VID2HCOEFF15
+*/
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF16_OFFSET		(0x0570)
+
+/* PDP, VID2HCOEFF16, VID2HCOEFF16
+*/
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SCALESIZE_OFFSET		(0x0574)
+
+/* PDP, VID2SCALESIZE, VID2SCALEWIDTH
+*/
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALESIZE, VID2SCALEHEIGHT
+*/
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SCALECTRL_OFFSET		(0x0578)
+
+/* PDP, VID3SCALECTRL, VID3HSCALEBP
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSCALEBP
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3HSBEFOREVS
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSURUNCTRL
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3PAN_EN
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VORDER
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_SHIFT		(16)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_LENGTH		(2)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VPITCH
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_SHIFT		(0)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_LENGTH		(16)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VSINIT_OFFSET		(0x057C)
+
+/* PDP, VID3VSINIT, VID3VINITIAL1
+*/
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3VSINIT, VID3VINITIAL0
+*/
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF0_OFFSET		(0x0580)
+
+/* PDP, VID3VCOEFF0, VID3VCOEFF0
+*/
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF1_OFFSET		(0x0584)
+
+/* PDP, VID3VCOEFF1, VID3VCOEFF1
+*/
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF2_OFFSET		(0x0588)
+
+/* PDP, VID3VCOEFF2, VID3VCOEFF2
+*/
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF3_OFFSET		(0x058C)
+
+/* PDP, VID3VCOEFF3, VID3VCOEFF3
+*/
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF4_OFFSET		(0x0590)
+
+/* PDP, VID3VCOEFF4, VID3VCOEFF4
+*/
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF5_OFFSET		(0x0594)
+
+/* PDP, VID3VCOEFF5, VID3VCOEFF5
+*/
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF6_OFFSET		(0x0598)
+
+/* PDP, VID3VCOEFF6, VID3VCOEFF6
+*/
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF7_OFFSET		(0x059C)
+
+/* PDP, VID3VCOEFF7, VID3VCOEFF7
+*/
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF8_OFFSET		(0x05A0)
+
+/* PDP, VID3VCOEFF8, VID3VCOEFF8
+*/
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HSINIT_OFFSET		(0x05A4)
+
+/* PDP, VID3HSINIT, VID3HINITIAL
+*/
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3HSINIT, VID3HPITCH
+*/
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_SHIFT		(0)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_LENGTH		(16)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF0_OFFSET		(0x05A8)
+
+/* PDP, VID3HCOEFF0, VID3HCOEFF0
+*/
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF1_OFFSET		(0x05AC)
+
+/* PDP, VID3HCOEFF1, VID3HCOEFF1
+*/
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF2_OFFSET		(0x05B0)
+
+/* PDP, VID3HCOEFF2, VID3HCOEFF2
+*/
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF3_OFFSET		(0x05B4)
+
+/* PDP, VID3HCOEFF3, VID3HCOEFF3
+*/
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF4_OFFSET		(0x05B8)
+
+/* PDP, VID3HCOEFF4, VID3HCOEFF4
+*/
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF5_OFFSET		(0x05BC)
+
+/* PDP, VID3HCOEFF5, VID3HCOEFF5
+*/
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF6_OFFSET		(0x05C0)
+
+/* PDP, VID3HCOEFF6, VID3HCOEFF6
+*/
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF7_OFFSET		(0x05C4)
+
+/* PDP, VID3HCOEFF7, VID3HCOEFF7
+*/
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF8_OFFSET		(0x05C8)
+
+/* PDP, VID3HCOEFF8, VID3HCOEFF8
+*/
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF9_OFFSET		(0x05CC)
+
+/* PDP, VID3HCOEFF9, VID3HCOEFF9
+*/
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF10_OFFSET		(0x05D0)
+
+/* PDP, VID3HCOEFF10, VID3HCOEFF10
+*/
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_LENGTH	(32)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF11_OFFSET		(0x05D4)
+
+/* PDP, VID3HCOEFF11, VID3HCOEFF11
+*/
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_LENGTH	(32)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF12_OFFSET		(0x05D8)
+
+/* PDP, VID3HCOEFF12, VID3HCOEFF12
+*/
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF13_OFFSET		(0x05DC)
+
+/* PDP, VID3HCOEFF13, VID3HCOEFF13
+*/
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF14_OFFSET		(0x05E0)
+
+/* PDP, VID3HCOEFF14, VID3HCOEFF14
+*/
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF15_OFFSET		(0x05E4)
+
+/* PDP, VID3HCOEFF15, VID3HCOEFF15
+*/
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF16_OFFSET		(0x05E8)
+
+/* PDP, VID3HCOEFF16, VID3HCOEFF16
+*/
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SCALESIZE_OFFSET		(0x05EC)
+
+/* PDP, VID3SCALESIZE, VID3SCALEWIDTH
+*/
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALESIZE, VID3SCALEHEIGHT
+*/
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SCALECTRL_OFFSET		(0x05F0)
+
+/* PDP, VID4SCALECTRL, VID4HSCALEBP
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_SHIFT	(31)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSCALEBP
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_SHIFT	(30)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4HSBEFOREVS
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_MASK	(0x20000000)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_SHIFT	(29)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSURUNCTRL
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_MASK	(0x08000000)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_SHIFT	(27)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4PAN_EN
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VORDER
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_LSBMASK	(0x00000003)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_SHIFT		(16)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_LENGTH		(2)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VPITCH
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_LSBMASK	(0x0000FFFF)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_SHIFT		(0)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_LENGTH		(16)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VSINIT_OFFSET			(0x05F4)
+
+/* PDP, VID4VSINIT, VID4VINITIAL1
+*/
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4VSINIT, VID4VINITIAL0
+*/
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF0_OFFSET			(0x05F8)
+
+/* PDP, VID4VCOEFF0, VID4VCOEFF0
+*/
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF1_OFFSET			(0x05FC)
+
+/* PDP, VID4VCOEFF1, VID4VCOEFF1
+*/
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF2_OFFSET			(0x0600)
+
+/* PDP, VID4VCOEFF2, VID4VCOEFF2
+*/
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF3_OFFSET			(0x0604)
+
+/* PDP, VID4VCOEFF3, VID4VCOEFF3
+*/
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF4_OFFSET			(0x0608)
+
+/* PDP, VID4VCOEFF4, VID4VCOEFF4
+*/
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF5_OFFSET			(0x060C)
+
+/* PDP, VID4VCOEFF5, VID4VCOEFF5
+*/
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF6_OFFSET			(0x0610)
+
+/* PDP, VID4VCOEFF6, VID4VCOEFF6
+*/
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF7_OFFSET			(0x0614)
+
+/* PDP, VID4VCOEFF7, VID4VCOEFF7
+*/
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF8_OFFSET			(0x0618)
+
+/* PDP, VID4VCOEFF8, VID4VCOEFF8
+*/
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HSINIT_OFFSET			(0x061C)
+
+/* PDP, VID4HSINIT, VID4HINITIAL
+*/
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4HSINIT, VID4HPITCH
+*/
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_SHIFT		(0)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_LENGTH		(16)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF0_OFFSET			(0x0620)
+
+/* PDP, VID4HCOEFF0, VID4HCOEFF0
+*/
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF1_OFFSET			(0x0624)
+
+/* PDP, VID4HCOEFF1, VID4HCOEFF1
+*/
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF2_OFFSET			(0x0628)
+
+/* PDP, VID4HCOEFF2, VID4HCOEFF2
+*/
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF3_OFFSET			(0x062C)
+
+/* PDP, VID4HCOEFF3, VID4HCOEFF3
+*/
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF4_OFFSET			(0x0630)
+
+/* PDP, VID4HCOEFF4, VID4HCOEFF4
+*/
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF5_OFFSET			(0x0634)
+
+/* PDP, VID4HCOEFF5, VID4HCOEFF5
+*/
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF6_OFFSET			(0x0638)
+
+/* PDP, VID4HCOEFF6, VID4HCOEFF6
+*/
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF7_OFFSET			(0x063C)
+
+/* PDP, VID4HCOEFF7, VID4HCOEFF7
+*/
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF8_OFFSET			(0x0640)
+
+/* PDP, VID4HCOEFF8, VID4HCOEFF8
+*/
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF9_OFFSET			(0x0644)
+
+/* PDP, VID4HCOEFF9, VID4HCOEFF9
+*/
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF10_OFFSET			(0x0648)
+
+/* PDP, VID4HCOEFF10, VID4HCOEFF10
+*/
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF11_OFFSET			(0x064C)
+
+/* PDP, VID4HCOEFF11, VID4HCOEFF11
+*/
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF12_OFFSET			(0x0650)
+
+/* PDP, VID4HCOEFF12, VID4HCOEFF12
+*/
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF13_OFFSET			(0x0654)
+
+/* PDP, VID4HCOEFF13, VID4HCOEFF13
+*/
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF14_OFFSET			(0x0658)
+
+/* PDP, VID4HCOEFF14, VID4HCOEFF14
+*/
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF15_OFFSET			(0x065C)
+
+/* PDP, VID4HCOEFF15, VID4HCOEFF15
+*/
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF16_OFFSET			(0x0660)
+
+/* PDP, VID4HCOEFF16, VID4HCOEFF16
+*/
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_LSBMASK	(0x000000FF)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_LENGTH	(8)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SCALESIZE_OFFSET			(0x0664)
+
+/* PDP, VID4SCALESIZE, VID4SCALEWIDTH
+*/
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALESIZE, VID4SCALEHEIGHT
+*/
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND0_OFFSET				(0x0668)
+
+/* PDP, PORTER_BLND0, BLND0BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND0, BLND0PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND1_OFFSET				(0x066C)
+
+/* PDP, PORTER_BLND1, BLND1BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND1, BLND1PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND2_OFFSET				(0x0670)
+
+/* PDP, PORTER_BLND2, BLND2BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND2, BLND2PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND3_OFFSET				(0x0674)
+
+/* PDP, PORTER_BLND3, BLND3BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND3, BLND3PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND4_OFFSET				(0x0678)
+
+/* PDP, PORTER_BLND4, BLND4BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND4, BLND4PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND5_OFFSET				(0x067C)
+
+/* PDP, PORTER_BLND5, BLND5BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND5, BLND5PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND6_OFFSET				(0x0680)
+
+/* PDP, PORTER_BLND6, BLND6BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND6, BLND6PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND7_OFFSET				(0x0684)
+
+/* PDP, PORTER_BLND7, BLND7BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND7, BLND7PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06C8)
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06CC)
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMAX
+*/
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMIN
+*/
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_C_RG_OFFSET		(0x06D0)
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_R
+*/
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_G
+*/
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_C_B_OFFSET		(0x06D4)
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYEN
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYC_B
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06D8)
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_OFFSET				(0x06DC)
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMAX
+*/
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMIN
+*/
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_C_RG_OFFSET				(0x06E0)
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_R
+*/
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_G
+*/
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_C_B_OFFSET				(0x06E4)
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYEN
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYC_B
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06E8)
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_OFFSET			(0x06EC)
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMAX
+*/
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMIN
+*/
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_C_RG_OFFSET		(0x06F0)
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_R
+*/
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_G
+*/
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_C_B_OFFSET		(0x06F4)
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYEN
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYC_B
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06F8)
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06FC)
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMAX
+*/
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMIN
+*/
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_C_RG_OFFSET			(0x0700)
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_R
+*/
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_G
+*/
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_C_B_OFFSET		(0x0704)
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_MASK	(0x20000000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SHIFT	(29)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LENGTH	(1)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYEN
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYC_B
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF0_OFFSET			(0x0708)
+
+/* PDP, CSCCOEFF0, CSCCOEFFRU
+*/
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF0, CSCCOEFFRY
+*/
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF1_OFFSET			(0x070C)
+
+/* PDP, CSCCOEFF1, CSCCOEFFGY
+*/
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF1, CSCCOEFFRV
+*/
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF2_OFFSET			(0x0710)
+
+/* PDP, CSCCOEFF2, CSCCOEFFGV
+*/
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF2, CSCCOEFFGU
+*/
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF3_OFFSET			(0x0714)
+
+/* PDP, CSCCOEFF3, CSCCOEFFBU
+*/
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF3, CSCCOEFFBY
+*/
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF4_OFFSET			(0x0718)
+
+/* PDP, CSCCOEFF4, CSCCOEFFBV
+*/
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BGNDCOL_AR_OFFSET			(0x071C)
+
+/* PDP, BGNDCOL_AR, BGNDCOL_A
+*/
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_MASK		(0x03FF0000)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SHIFT		(16)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_AR, BGNDCOL_R
+*/
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_MASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SHIFT		(0)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BGNDCOL_GB_OFFSET			(0x0720)
+
+/* PDP, BGNDCOL_GB, BGNDCOL_G
+*/
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_MASK		(0x03FF0000)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SHIFT		(16)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_GB, BGNDCOL_B
+*/
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_MASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SHIFT		(0)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BORDCOL_R_OFFSET			(0x0724)
+
+/* PDP, BORDCOL_R, BORDCOL_R
+*/
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_MASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_SHIFT		(0)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_LENGTH		(10)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BORDCOL_GB_OFFSET			(0x0728)
+
+/* PDP, BORDCOL_GB, BORDCOL_G
+*/
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_MASK		(0x03FF0000)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_SHIFT		(16)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_LENGTH		(10)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BORDCOL_GB, BORDCOL_B
+*/
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_MASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_SHIFT		(0)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_LENGTH		(10)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_LINESTAT_OFFSET				(0x0734)
+
+/* PDP, LINESTAT, LINENO
+*/
+#define ODN_PDP_LINESTAT_LINENO_MASK			(0x00001FFF)
+#define ODN_PDP_LINESTAT_LINENO_LSBMASK			(0x00001FFF)
+#define ODN_PDP_LINESTAT_LINENO_SHIFT			(0)
+#define ODN_PDP_LINESTAT_LINENO_LENGTH			(13)
+#define ODN_PDP_LINESTAT_LINENO_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_OFFSET	(0x0738)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C11C12, CR_PROCAMP_C12
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C11C12, CR_PROCAMP_C11
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_OFFSET		(0x073C)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C13C21, CR_PROCAMP_C21
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C13C21, CR_PROCAMP_C13
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_OFFSET		(0x0740)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C22C23, CR_PROCAMP_C23
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C22C23, CR_PROCAMP_C22
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_OFFSET		(0x0744)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C31C32, CR_PROCAMP_C32
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C31C32, CR_PROCAMP_C31
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_OFFSET		(0x0748)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_C33
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_LSBMASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_RANGE
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_MASK		(0x00000030)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LSBMASK		(0x00000003)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SHIFT		(4)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LENGTH		(2)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_EN
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_MASK		(0x00000001)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_LENGTH		(1)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_OFFSET		(0x074C)
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_G
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_MASK		(0x0FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_B
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_MASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_OFFSET		(0x0750)
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_R, CR_PROCAMP_OUTOFF_R
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_MASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_OFFSET		(0x0754)
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_G
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_MASK		(0x03FF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_B
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_MASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_OFFSET		(0x0758)
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_R, CR_PROCAMP_INOFF_R
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_MASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SIGNAT_R_OFFSET		(0x075C)
+
+/* PDP, SIGNAT_R, SIGNATURE_R
+*/
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_MASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_SHIFT		(0)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_LENGTH		(10)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SIGNAT_GB_OFFSET		(0x0760)
+
+/* PDP, SIGNAT_GB, SIGNATURE_G
+*/
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_MASK		(0x03FF0000)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_SHIFT		(16)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_LENGTH		(10)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SIGNAT_GB, SIGNATURE_B
+*/
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_MASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_SHIFT		(0)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_LENGTH		(10)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGISTER_UPDATE_CTRL_OFFSET		(0x0764)
+
+/* PDP, REGISTER_UPDATE_CTRL, BYPASS_DOUBLE_BUFFERING
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_MASK		(0x00000004)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SHIFT		(2)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, REGISTERS_VALID
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_MASK		(0x00000002)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SHIFT		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, USE_VBLANK
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_MASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SHIFT		(0)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGISTER_UPDATE_STATUS_OFFSET		(0x0768)
+
+/* PDP, REGISTER_UPDATE_STATUS, REGISTERS_UPDATED
+*/
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_MASK		(0x00000002)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SHIFT		(1)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGCTRL_OFFSET		(0x076C)
+
+/* PDP, DBGCTRL, DBG_READ
+*/
+#define ODN_PDP_DBGCTRL_DBG_READ_MASK		(0x00000002)
+#define ODN_PDP_DBGCTRL_DBG_READ_LSBMASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_READ_SHIFT		(1)
+#define ODN_PDP_DBGCTRL_DBG_READ_LENGTH		(1)
+#define ODN_PDP_DBGCTRL_DBG_READ_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGCTRL, DBG_ENAB
+*/
+#define ODN_PDP_DBGCTRL_DBG_ENAB_MASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_LSBMASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_SHIFT		(0)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_LENGTH		(1)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGDATA_R_OFFSET		(0x0770)
+
+/* PDP, DBGDATA_R, DBG_DATA_R
+*/
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_MASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_SHIFT		(0)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_LENGTH		(10)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGDATA_GB_OFFSET		(0x0774)
+
+/* PDP, DBGDATA_GB, DBG_DATA_G
+*/
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_MASK		(0x03FF0000)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_SHIFT		(16)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_LENGTH		(10)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGDATA_GB, DBG_DATA_B
+*/
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_MASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_SHIFT		(0)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_LENGTH		(10)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGSIDE_OFFSET				(0x0778)
+
+/* PDP, DBGSIDE, DBG_VAL
+*/
+#define ODN_PDP_DBGSIDE_DBG_VAL_MASK			(0x00000008)
+#define ODN_PDP_DBGSIDE_DBG_VAL_LSBMASK			(0x00000001)
+#define ODN_PDP_DBGSIDE_DBG_VAL_SHIFT			(3)
+#define ODN_PDP_DBGSIDE_DBG_VAL_LENGTH			(1)
+#define ODN_PDP_DBGSIDE_DBG_VAL_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, DBGSIDE, DBG_SIDE
+*/
+#define ODN_PDP_DBGSIDE_DBG_SIDE_MASK			(0x00000007)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_LSBMASK		(0x00000007)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_SHIFT			(0)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_LENGTH			(3)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_OUTPUT_OFFSET				(0x077C)
+
+/* PDP, OUTPUT, EIGHT_BIT_OUTPUT
+*/
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_MASK		(0x00000002)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_LSBMASK		(0x00000001)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_SHIFT		(1)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_LENGTH		(1)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OUTPUT, OUTPUT_CONFIG
+*/
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_MASK		(0x00000001)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_LSBMASK		(0x00000001)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_SHIFT		(0)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_LENGTH		(1)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SYNCCTRL_OFFSET				(0x0780)
+
+/* PDP, SYNCCTRL, SYNCACTIVE
+*/
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_MASK		(0x80000000)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_SHIFT		(31)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, ODN_PDP_RST
+*/
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_MASK		(0x20000000)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_SHIFT		(29)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, POWERDN
+*/
+#define ODN_PDP_SYNCCTRL_POWERDN_MASK			(0x10000000)
+#define ODN_PDP_SYNCCTRL_POWERDN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_POWERDN_SHIFT			(28)
+#define ODN_PDP_SYNCCTRL_POWERDN_LENGTH			(1)
+#define ODN_PDP_SYNCCTRL_POWERDN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, LOWPWRMODE
+*/
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_MASK		(0x08000000)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_SHIFT		(27)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDSYNCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_MASK		(0x04000000)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_SHIFT		(26)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDINTCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_MASK		(0x02000000)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_SHIFT		(25)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDCTRL_MASK			(0x01000000)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_SHIFT			(24)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_LENGTH			(1)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDWAIT
+*/
+#define ODN_PDP_SYNCCTRL_UPDWAIT_MASK			(0x000F0000)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_LSBMASK		(0x0000000F)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_SHIFT			(16)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_LENGTH			(4)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, FIELD_EN
+*/
+#define ODN_PDP_SYNCCTRL_FIELD_EN_MASK			(0x00002000)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_SHIFT			(13)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, CSYNC_EN
+*/
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_MASK			(0x00001000)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_SHIFT			(12)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, CLKPOL
+*/
+#define ODN_PDP_SYNCCTRL_CLKPOL_MASK		(0x00000800)
+#define ODN_PDP_SYNCCTRL_CLKPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_CLKPOL_SHIFT		(11)
+#define ODN_PDP_SYNCCTRL_CLKPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_CLKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VS_SLAVE
+*/
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_MASK		(0x00000080)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_SHIFT		(7)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HS_SLAVE
+*/
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_MASK		(0x00000040)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_SHIFT		(6)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKPOL
+*/
+#define ODN_PDP_SYNCCTRL_BLNKPOL_MASK		(0x00000020)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_SHIFT		(5)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKDIS
+*/
+#define ODN_PDP_SYNCCTRL_BLNKDIS_MASK		(0x00000010)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_SHIFT		(4)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSPOL
+*/
+#define ODN_PDP_SYNCCTRL_VSPOL_MASK		(0x00000008)
+#define ODN_PDP_SYNCCTRL_VSPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VSPOL_SHIFT		(3)
+#define ODN_PDP_SYNCCTRL_VSPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSDIS
+*/
+#define ODN_PDP_SYNCCTRL_VSDIS_MASK		(0x00000004)
+#define ODN_PDP_SYNCCTRL_VSDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VSDIS_SHIFT		(2)
+#define ODN_PDP_SYNCCTRL_VSDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VSDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSPOL
+*/
+#define ODN_PDP_SYNCCTRL_HSPOL_MASK		(0x00000002)
+#define ODN_PDP_SYNCCTRL_HSPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSPOL_SHIFT		(1)
+#define ODN_PDP_SYNCCTRL_HSPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSDIS
+*/
+#define ODN_PDP_SYNCCTRL_HSDIS_MASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSDIS_SHIFT		(0)
+#define ODN_PDP_SYNCCTRL_HSDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HSDIS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_HSYNC1_OFFSET			(0x0784)
+
+/* PDP, HSYNC1, HBPS
+*/
+#define ODN_PDP_HSYNC1_HBPS_MASK		(0x1FFF0000)
+#define ODN_PDP_HSYNC1_HBPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC1_HBPS_SHIFT		(16)
+#define ODN_PDP_HSYNC1_HBPS_LENGTH		(13)
+#define ODN_PDP_HSYNC1_HBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC1, HT
+*/
+#define ODN_PDP_HSYNC1_HT_MASK			(0x00001FFF)
+#define ODN_PDP_HSYNC1_HT_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC1_HT_SHIFT			(0)
+#define ODN_PDP_HSYNC1_HT_LENGTH		(13)
+#define ODN_PDP_HSYNC1_HT_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_HSYNC2_OFFSET			(0x0788)
+
+/* PDP, HSYNC2, HAS
+*/
+#define ODN_PDP_HSYNC2_HAS_MASK			(0x1FFF0000)
+#define ODN_PDP_HSYNC2_HAS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HAS_SHIFT		(16)
+#define ODN_PDP_HSYNC2_HAS_LENGTH		(13)
+#define ODN_PDP_HSYNC2_HAS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC2, HLBS
+*/
+#define ODN_PDP_HSYNC2_HLBS_MASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HLBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HLBS_SHIFT		(0)
+#define ODN_PDP_HSYNC2_HLBS_LENGTH		(13)
+#define ODN_PDP_HSYNC2_HLBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_HSYNC3_OFFSET			(0x078C)
+
+/* PDP, HSYNC3, HFPS
+*/
+#define ODN_PDP_HSYNC3_HFPS_MASK		(0x1FFF0000)
+#define ODN_PDP_HSYNC3_HFPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HFPS_SHIFT		(16)
+#define ODN_PDP_HSYNC3_HFPS_LENGTH		(13)
+#define ODN_PDP_HSYNC3_HFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC3, HRBS
+*/
+#define ODN_PDP_HSYNC3_HRBS_MASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HRBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HRBS_SHIFT		(0)
+#define ODN_PDP_HSYNC3_HRBS_LENGTH		(13)
+#define ODN_PDP_HSYNC3_HRBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VSYNC1_OFFSET			(0x0790)
+
+/* PDP, VSYNC1, VBPS
+*/
+#define ODN_PDP_VSYNC1_VBPS_MASK		(0x1FFF0000)
+#define ODN_PDP_VSYNC1_VBPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC1_VBPS_SHIFT		(16)
+#define ODN_PDP_VSYNC1_VBPS_LENGTH		(13)
+#define ODN_PDP_VSYNC1_VBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC1, VT
+*/
+#define ODN_PDP_VSYNC1_VT_MASK			(0x00001FFF)
+#define ODN_PDP_VSYNC1_VT_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC1_VT_SHIFT			(0)
+#define ODN_PDP_VSYNC1_VT_LENGTH		(13)
+#define ODN_PDP_VSYNC1_VT_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_VSYNC2_OFFSET			(0x0794)
+
+/* PDP, VSYNC2, VAS
+*/
+#define ODN_PDP_VSYNC2_VAS_MASK			(0x1FFF0000)
+#define ODN_PDP_VSYNC2_VAS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VAS_SHIFT		(16)
+#define ODN_PDP_VSYNC2_VAS_LENGTH		(13)
+#define ODN_PDP_VSYNC2_VAS_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VSYNC2, VTBS
+*/
+#define ODN_PDP_VSYNC2_VTBS_MASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VTBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VTBS_SHIFT		(0)
+#define ODN_PDP_VSYNC2_VTBS_LENGTH		(13)
+#define ODN_PDP_VSYNC2_VTBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VSYNC3_OFFSET			(0x0798)
+
+/* PDP, VSYNC3, VFPS
+*/
+#define ODN_PDP_VSYNC3_VFPS_MASK		(0x1FFF0000)
+#define ODN_PDP_VSYNC3_VFPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VFPS_SHIFT		(16)
+#define ODN_PDP_VSYNC3_VFPS_LENGTH		(13)
+#define ODN_PDP_VSYNC3_VFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC3, VBBS
+*/
+#define ODN_PDP_VSYNC3_VBBS_MASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VBBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VBBS_SHIFT		(0)
+#define ODN_PDP_VSYNC3_VBBS_LENGTH		(13)
+#define ODN_PDP_VSYNC3_VBBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTSTAT_OFFSET			(0x079C)
+
+/* PDP, INTSTAT, INTS_VID4ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID4URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK1
+*/
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK0
+*/
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK1
+*/
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK0
+*/
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTENAB_OFFSET				(0x07A0)
+
+/* PDP, INTENAB, INTEN_VID4ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID4URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK1
+*/
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK0
+*/
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK1
+*/
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK0
+*/
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTCLR_OFFSET		(0x07A4)
+
+/* PDP, INTCLR, INTCLR_VID4ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID4URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK1
+*/
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK0
+*/
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK1
+*/
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK0
+*/
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_MEMCTRL_OFFSET		(0x07A8)
+
+/* PDP, MEMCTRL, MEMREFRESH
+*/
+#define ODN_PDP_MEMCTRL_MEMREFRESH_MASK		(0xC0000000)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_LSBMASK		(0x00000003)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_SHIFT		(30)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_LENGTH		(2)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEMCTRL, BURSTLEN
+*/
+#define ODN_PDP_MEMCTRL_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_MEMCTRL_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_MEMCTRL_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_MEMCTRL_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_MEMCTRL_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_MEM_THRESH_OFFSET		(0x07AC)
+
+/* PDP, MEM_THRESH, UVTHRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, YTHRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, THRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_ALTERNATE_3D_CTRL_OFFSET		(0x07B0)
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_ON
+*/
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_MASK		(0x00000010)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LSBMASK		(0x00000001)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SHIFT		(4)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LENGTH		(1)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_BLENDSEL
+*/
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_MASK		(0x00000007)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LSBMASK		(0x00000007)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SHIFT		(0)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LENGTH		(3)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA0_R_OFFSET		(0x07B4)
+
+/* PDP, GAMMA0_R, GAMMA0_R
+*/
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_SHIFT		(0)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_LENGTH		(10)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA0_GB_OFFSET		(0x07B8)
+
+/* PDP, GAMMA0_GB, GAMMA0_G
+*/
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_SHIFT		(16)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_LENGTH		(10)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA0_GB, GAMMA0_B
+*/
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_SHIFT		(0)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_LENGTH		(10)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA1_R_OFFSET		(0x07BC)
+
+/* PDP, GAMMA1_R, GAMMA1_R
+*/
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_SHIFT		(0)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_LENGTH		(10)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA1_GB_OFFSET		(0x07C0)
+
+/* PDP, GAMMA1_GB, GAMMA1_G
+*/
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_SHIFT		(16)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_LENGTH		(10)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA1_GB, GAMMA1_B
+*/
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_SHIFT		(0)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_LENGTH		(10)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA2_R_OFFSET		(0x07C4)
+
+/* PDP, GAMMA2_R, GAMMA2_R
+*/
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_SHIFT		(0)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_LENGTH		(10)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA2_GB_OFFSET		(0x07C8)
+
+/* PDP, GAMMA2_GB, GAMMA2_G
+*/
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_SHIFT		(16)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_LENGTH		(10)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA2_GB, GAMMA2_B
+*/
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_SHIFT		(0)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_LENGTH		(10)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA3_R_OFFSET		(0x07CC)
+
+/* PDP, GAMMA3_R, GAMMA3_R
+*/
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_SHIFT		(0)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_LENGTH		(10)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA3_GB_OFFSET		(0x07D0)
+
+/* PDP, GAMMA3_GB, GAMMA3_G
+*/
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_SHIFT		(16)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_LENGTH		(10)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA3_GB, GAMMA3_B
+*/
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_SHIFT		(0)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_LENGTH		(10)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA4_R_OFFSET		(0x07D4)
+
+/* PDP, GAMMA4_R, GAMMA4_R
+*/
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_SHIFT		(0)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_LENGTH		(10)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA4_GB_OFFSET		(0x07D8)
+
+/* PDP, GAMMA4_GB, GAMMA4_G
+*/
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_SHIFT		(16)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_LENGTH		(10)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA4_GB, GAMMA4_B
+*/
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_SHIFT		(0)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_LENGTH		(10)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA5_R_OFFSET		(0x07DC)
+
+/* PDP, GAMMA5_R, GAMMA5_R
+*/
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_SHIFT		(0)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_LENGTH		(10)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA5_GB_OFFSET		(0x07E0)
+
+/* PDP, GAMMA5_GB, GAMMA5_G
+*/
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_SHIFT		(16)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_LENGTH		(10)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA5_GB, GAMMA5_B
+*/
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_SHIFT		(0)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_LENGTH		(10)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA6_R_OFFSET		(0x07E4)
+
+/* PDP, GAMMA6_R, GAMMA6_R
+*/
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_SHIFT		(0)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_LENGTH		(10)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA6_GB_OFFSET		(0x07E8)
+
+/* PDP, GAMMA6_GB, GAMMA6_G
+*/
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_SHIFT		(16)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_LENGTH		(10)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA6_GB, GAMMA6_B
+*/
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_SHIFT		(0)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_LENGTH		(10)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA7_R_OFFSET		(0x07EC)
+
+/* PDP, GAMMA7_R, GAMMA7_R
+*/
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_SHIFT		(0)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_LENGTH		(10)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA7_GB_OFFSET		(0x07F0)
+
+/* PDP, GAMMA7_GB, GAMMA7_G
+*/
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_SHIFT		(16)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_LENGTH		(10)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA7_GB, GAMMA7_B
+*/
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_SHIFT		(0)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_LENGTH		(10)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA8_R_OFFSET		(0x07F4)
+
+/* PDP, GAMMA8_R, GAMMA8_R
+*/
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_SHIFT		(0)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_LENGTH		(10)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA8_GB_OFFSET		(0x07F8)
+
+/* PDP, GAMMA8_GB, GAMMA8_G
+*/
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_SHIFT		(16)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_LENGTH		(10)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA8_GB, GAMMA8_B
+*/
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_SHIFT		(0)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_LENGTH		(10)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA9_R_OFFSET		(0x07FC)
+
+/* PDP, GAMMA9_R, GAMMA9_R
+*/
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_SHIFT		(0)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_LENGTH		(10)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA9_GB_OFFSET		(0x0800)
+
+/* PDP, GAMMA9_GB, GAMMA9_G
+*/
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_SHIFT		(16)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_LENGTH		(10)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA9_GB, GAMMA9_B
+*/
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_SHIFT		(0)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_LENGTH		(10)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA10_R_OFFSET		(0x0804)
+
+/* PDP, GAMMA10_R, GAMMA10_R
+*/
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_SHIFT		(0)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_LENGTH		(10)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA10_GB_OFFSET		(0x0808)
+
+/* PDP, GAMMA10_GB, GAMMA10_G
+*/
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_SHIFT		(16)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_LENGTH		(10)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA10_GB, GAMMA10_B
+*/
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_SHIFT		(0)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_LENGTH		(10)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA11_R_OFFSET		(0x080C)
+
+/* PDP, GAMMA11_R, GAMMA11_R
+*/
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_SHIFT		(0)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_LENGTH		(10)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA11_GB_OFFSET		(0x0810)
+
+/* PDP, GAMMA11_GB, GAMMA11_G
+*/
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_SHIFT		(16)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_LENGTH		(10)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA11_GB, GAMMA11_B
+*/
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_SHIFT		(0)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_LENGTH		(10)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA12_R_OFFSET		(0x0814)
+
+/* PDP, GAMMA12_R, GAMMA12_R
+*/
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_SHIFT		(0)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_LENGTH		(10)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA12_GB_OFFSET		(0x0818)
+
+/* PDP, GAMMA12_GB, GAMMA12_G
+*/
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_SHIFT		(16)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_LENGTH		(10)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA12_GB, GAMMA12_B
+*/
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_SHIFT		(0)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_LENGTH		(10)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA13_R_OFFSET		(0x081C)
+
+/* PDP, GAMMA13_R, GAMMA13_R
+*/
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_SHIFT		(0)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_LENGTH		(10)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA13_GB_OFFSET		(0x0820)
+
+/* PDP, GAMMA13_GB, GAMMA13_G
+*/
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_SHIFT		(16)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_LENGTH		(10)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA13_GB, GAMMA13_B
+*/
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_SHIFT		(0)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_LENGTH		(10)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA14_R_OFFSET		(0x0824)
+
+/* PDP, GAMMA14_R, GAMMA14_R
+*/
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_SHIFT		(0)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_LENGTH		(10)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA14_GB_OFFSET		(0x0828)
+
+/* PDP, GAMMA14_GB, GAMMA14_G
+*/
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_SHIFT		(16)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_LENGTH		(10)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA14_GB, GAMMA14_B
+*/
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_SHIFT		(0)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_LENGTH		(10)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA15_R_OFFSET		(0x082C)
+
+/* PDP, GAMMA15_R, GAMMA15_R
+*/
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_SHIFT		(0)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_LENGTH		(10)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA15_GB_OFFSET		(0x0830)
+
+/* PDP, GAMMA15_GB, GAMMA15_G
+*/
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_SHIFT		(16)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_LENGTH		(10)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA15_GB, GAMMA15_B
+*/
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_SHIFT		(0)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_LENGTH		(10)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA16_R_OFFSET		(0x0834)
+
+/* PDP, GAMMA16_R, GAMMA16_R
+*/
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_SHIFT		(0)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_LENGTH		(10)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA16_GB_OFFSET		(0x0838)
+
+/* PDP, GAMMA16_GB, GAMMA16_G
+*/
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_SHIFT		(16)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_LENGTH		(10)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA16_GB, GAMMA16_B
+*/
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_SHIFT		(0)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_LENGTH		(10)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA17_R_OFFSET		(0x083C)
+
+/* PDP, GAMMA17_R, GAMMA17_R
+*/
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_SHIFT		(0)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_LENGTH		(10)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA17_GB_OFFSET		(0x0840)
+
+/* PDP, GAMMA17_GB, GAMMA17_G
+*/
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_SHIFT		(16)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_LENGTH		(10)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA17_GB, GAMMA17_B
+*/
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_SHIFT		(0)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_LENGTH		(10)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA18_R_OFFSET		(0x0844)
+
+/* PDP, GAMMA18_R, GAMMA18_R
+*/
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_SHIFT		(0)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_LENGTH		(10)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA18_GB_OFFSET		(0x0848)
+
+/* PDP, GAMMA18_GB, GAMMA18_G
+*/
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_SHIFT		(16)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_LENGTH		(10)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA18_GB, GAMMA18_B
+*/
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_SHIFT		(0)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_LENGTH		(10)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA19_R_OFFSET		(0x084C)
+
+/* PDP, GAMMA19_R, GAMMA19_R
+*/
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_SHIFT		(0)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_LENGTH		(10)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA19_GB_OFFSET		(0x0850)
+
+/* PDP, GAMMA19_GB, GAMMA19_G
+*/
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_SHIFT		(16)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_LENGTH		(10)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA19_GB, GAMMA19_B
+*/
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_SHIFT		(0)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_LENGTH		(10)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA20_R_OFFSET		(0x0854)
+
+/* PDP, GAMMA20_R, GAMMA20_R
+*/
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_SHIFT		(0)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_LENGTH		(10)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA20_GB_OFFSET		(0x0858)
+
+/* PDP, GAMMA20_GB, GAMMA20_G
+*/
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_SHIFT		(16)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_LENGTH		(10)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA20_GB, GAMMA20_B
+*/
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_SHIFT		(0)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_LENGTH		(10)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA21_R_OFFSET		(0x085C)
+
+/* PDP, GAMMA21_R, GAMMA21_R
+*/
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_SHIFT		(0)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_LENGTH		(10)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA21_GB_OFFSET		(0x0860)
+
+/* PDP, GAMMA21_GB, GAMMA21_G
+*/
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_SHIFT		(16)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_LENGTH		(10)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA21_GB, GAMMA21_B
+*/
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_SHIFT		(0)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_LENGTH		(10)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA22_R_OFFSET		(0x0864)
+
+/* PDP, GAMMA22_R, GAMMA22_R
+*/
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_SHIFT		(0)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_LENGTH		(10)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA22_GB_OFFSET		(0x0868)
+
+/* PDP, GAMMA22_GB, GAMMA22_G
+*/
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_SHIFT		(16)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_LENGTH		(10)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA22_GB, GAMMA22_B
+*/
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_SHIFT		(0)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_LENGTH		(10)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA23_R_OFFSET		(0x086C)
+
+/* PDP, GAMMA23_R, GAMMA23_R
+*/
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_SHIFT		(0)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_LENGTH		(10)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA23_GB_OFFSET		(0x0870)
+
+/* PDP, GAMMA23_GB, GAMMA23_G
+*/
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_SHIFT		(16)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_LENGTH		(10)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA23_GB, GAMMA23_B
+*/
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_SHIFT		(0)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_LENGTH		(10)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA24_R_OFFSET		(0x0874)
+
+/* PDP, GAMMA24_R, GAMMA24_R
+*/
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_SHIFT		(0)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_LENGTH		(10)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA24_GB_OFFSET		(0x0878)
+
+/* PDP, GAMMA24_GB, GAMMA24_G
+*/
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_SHIFT		(16)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_LENGTH		(10)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA24_GB, GAMMA24_B
+*/
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_SHIFT		(0)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_LENGTH		(10)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA25_R_OFFSET		(0x087C)
+
+/* PDP, GAMMA25_R, GAMMA25_R
+*/
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_SHIFT		(0)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_LENGTH		(10)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA25_GB_OFFSET		(0x0880)
+
+/* PDP, GAMMA25_GB, GAMMA25_G
+*/
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_SHIFT		(16)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_LENGTH		(10)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA25_GB, GAMMA25_B
+*/
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_SHIFT		(0)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_LENGTH		(10)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA26_R_OFFSET		(0x0884)
+
+/* PDP, GAMMA26_R, GAMMA26_R
+*/
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_SHIFT		(0)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_LENGTH		(10)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA26_GB_OFFSET		(0x0888)
+
+/* PDP, GAMMA26_GB, GAMMA26_G
+*/
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_SHIFT		(16)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_LENGTH		(10)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA26_GB, GAMMA26_B
+*/
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_SHIFT		(0)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_LENGTH		(10)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA27_R_OFFSET		(0x088C)
+
+/* PDP, GAMMA27_R, GAMMA27_R
+*/
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_SHIFT		(0)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_LENGTH		(10)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA27_GB_OFFSET		(0x0890)
+
+/* PDP, GAMMA27_GB, GAMMA27_G
+*/
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_SHIFT		(16)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_LENGTH		(10)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA27_GB, GAMMA27_B
+*/
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_SHIFT		(0)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_LENGTH		(10)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA28_R_OFFSET		(0x0894)
+
+/* PDP, GAMMA28_R, GAMMA28_R
+*/
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_SHIFT		(0)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_LENGTH		(10)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA28_GB_OFFSET		(0x0898)
+
+/* PDP, GAMMA28_GB, GAMMA28_G
+*/
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_SHIFT		(16)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_LENGTH		(10)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA28_GB, GAMMA28_B
+*/
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_SHIFT		(0)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_LENGTH		(10)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA29_R_OFFSET		(0x089C)
+
+/* PDP, GAMMA29_R, GAMMA29_R
+*/
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_SHIFT		(0)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_LENGTH		(10)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA29_GB_OFFSET		(0x08A0)
+
+/* PDP, GAMMA29_GB, GAMMA29_G
+*/
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_SHIFT		(16)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_LENGTH		(10)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA29_GB, GAMMA29_B
+*/
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_SHIFT		(0)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_LENGTH		(10)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA30_R_OFFSET		(0x08A4)
+
+/* PDP, GAMMA30_R, GAMMA30_R
+*/
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_SHIFT		(0)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_LENGTH		(10)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA30_GB_OFFSET		(0x08A8)
+
+/* PDP, GAMMA30_GB, GAMMA30_G
+*/
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_SHIFT		(16)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_LENGTH		(10)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA30_GB, GAMMA30_B
+*/
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_SHIFT		(0)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_LENGTH		(10)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA31_R_OFFSET		(0x08AC)
+
+/* PDP, GAMMA31_R, GAMMA31_R
+*/
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_SHIFT		(0)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_LENGTH		(10)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA31_GB_OFFSET		(0x08B0)
+
+/* PDP, GAMMA31_GB, GAMMA31_G
+*/
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_SHIFT		(16)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_LENGTH		(10)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA31_GB, GAMMA31_B
+*/
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_SHIFT		(0)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_LENGTH		(10)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA32_R_OFFSET			(0x08B4)
+
+/* PDP, GAMMA32_R, GAMMA32_R
+*/
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_SHIFT		(0)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_LENGTH		(10)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA32_GB_OFFSET			(0x08B8)
+
+/* PDP, GAMMA32_GB, GAMMA32_G
+*/
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_SHIFT		(16)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_LENGTH		(10)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA32_GB, GAMMA32_B
+*/
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_SHIFT		(0)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_LENGTH		(10)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VEVENT_OFFSET				(0x08BC)
+
+/* PDP, VEVENT, VEVENT
+*/
+#define ODN_PDP_VEVENT_VEVENT_MASK			(0x1FFF0000)
+#define ODN_PDP_VEVENT_VEVENT_LSBMASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VEVENT_SHIFT			(16)
+#define ODN_PDP_VEVENT_VEVENT_LENGTH			(13)
+#define ODN_PDP_VEVENT_VEVENT_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VEVENT, VFETCH
+*/
+#define ODN_PDP_VEVENT_VFETCH_MASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VFETCH_LSBMASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VFETCH_SHIFT			(0)
+#define ODN_PDP_VEVENT_VFETCH_LENGTH			(13)
+#define ODN_PDP_VEVENT_VFETCH_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_HDECTRL_OFFSET				(0x08C0)
+
+/* PDP, HDECTRL, HDES
+*/
+#define ODN_PDP_HDECTRL_HDES_MASK		(0x1FFF0000)
+#define ODN_PDP_HDECTRL_HDES_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDES_SHIFT		(16)
+#define ODN_PDP_HDECTRL_HDES_LENGTH		(13)
+#define ODN_PDP_HDECTRL_HDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HDECTRL, HDEF
+*/
+#define ODN_PDP_HDECTRL_HDEF_MASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDEF_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDEF_SHIFT		(0)
+#define ODN_PDP_HDECTRL_HDEF_LENGTH		(13)
+#define ODN_PDP_HDECTRL_HDEF_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VDECTRL_OFFSET			(0x08C4)
+
+/* PDP, VDECTRL, VDES
+*/
+#define ODN_PDP_VDECTRL_VDES_MASK		(0x1FFF0000)
+#define ODN_PDP_VDECTRL_VDES_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDES_SHIFT		(16)
+#define ODN_PDP_VDECTRL_VDES_LENGTH		(13)
+#define ODN_PDP_VDECTRL_VDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VDECTRL, VDEF
+*/
+#define ODN_PDP_VDECTRL_VDEF_MASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDEF_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDEF_SHIFT		(0)
+#define ODN_PDP_VDECTRL_VDEF_LENGTH		(13)
+#define ODN_PDP_VDECTRL_VDEF_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_OPMASK_R_OFFSET			(0x08C8)
+
+/* PDP, OPMASK_R, MASKLEVEL
+*/
+#define ODN_PDP_OPMASK_R_MASKLEVEL_MASK		(0x80000000)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_LSBMASK		(0x00000001)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_SHIFT		(31)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_LENGTH		(1)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, BLANKLEVEL
+*/
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_MASK		(0x40000000)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_LSBMASK		(0x00000001)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_SHIFT		(30)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_LENGTH		(1)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, MASKR
+*/
+#define ODN_PDP_OPMASK_R_MASKR_MASK		(0x000003FF)
+#define ODN_PDP_OPMASK_R_MASKR_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_R_MASKR_SHIFT		(0)
+#define ODN_PDP_OPMASK_R_MASKR_LENGTH		(10)
+#define ODN_PDP_OPMASK_R_MASKR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_OPMASK_GB_OFFSET		(0x08CC)
+
+/* PDP, OPMASK_GB, MASKG
+*/
+#define ODN_PDP_OPMASK_GB_MASKG_MASK		(0x03FF0000)
+#define ODN_PDP_OPMASK_GB_MASKG_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKG_SHIFT		(16)
+#define ODN_PDP_OPMASK_GB_MASKG_LENGTH		(10)
+#define ODN_PDP_OPMASK_GB_MASKG_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_GB, MASKB
+*/
+#define ODN_PDP_OPMASK_GB_MASKB_MASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKB_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKB_SHIFT		(0)
+#define ODN_PDP_OPMASK_GB_MASKB_LENGTH		(10)
+#define ODN_PDP_OPMASK_GB_MASKB_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_ADDR_CTRL_OFFSET		(0x08D0)
+
+/* PDP, REGLD_ADDR_CTRL, REGLD_ADDRIN
+*/
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_MASK		(0xFFFFFFF0)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LSBMASK		(0x0FFFFFFF)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SHIFT		(4)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LENGTH		(28)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_ADDR_STAT_OFFSET		(0x08D4)
+
+/* PDP, REGLD_ADDR_STAT, REGLD_ADDROUT
+*/
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_MASK		(0xFFFFFFF0)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LSBMASK		(0x0FFFFFFF)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SHIFT		(4)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LENGTH		(28)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_STAT_OFFSET		(0x08D8)
+
+/* PDP, REGLD_STAT, REGLD_ADDREN
+*/
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_MASK		(0x00800000)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_LSBMASK		(0x00000001)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_SHIFT		(23)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_LENGTH		(1)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_CTRL_OFFSET		(0x08DC)
+
+/* PDP, REGLD_CTRL, REGLD_ADDRLEN
+*/
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_MASK		(0xFF000000)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_LSBMASK	(0x000000FF)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_SHIFT		(24)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_LENGTH		(8)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGLD_CTRL, REGLD_VAL
+*/
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_MASK		(0x00800000)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_LSBMASK		(0x00000001)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_SHIFT		(23)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_LENGTH		(1)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_UPDCTRL_OFFSET			(0x08E0)
+
+/* PDP, UPDCTRL, UPDFIELD
+*/
+#define ODN_PDP_UPDCTRL_UPDFIELD_MASK		(0x00000001)
+#define ODN_PDP_UPDCTRL_UPDFIELD_LSBMASK	(0x00000001)
+#define ODN_PDP_UPDCTRL_UPDFIELD_SHIFT		(0)
+#define ODN_PDP_UPDCTRL_UPDFIELD_LENGTH		(1)
+#define ODN_PDP_UPDCTRL_UPDFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTCTRL_OFFSET			(0x08E4)
+
+/* PDP, PVR_ODN_PDP_INTCTRL, HBLNK_LINE
+*/
+#define ODN_PDP_INTCTRL_HBLNK_LINE_MASK		(0x00010000)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_LSBMASK	(0x00000001)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_SHIFT	(16)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_LENGTH	(1)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_INTCTRL, HBLNK_LINENO
+*/
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_MASK	(0x00001FFF)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_LSBMASK	(0x00001FFF)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_SHIFT	(0)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_LENGTH	(13)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_PDISETUP_OFFSET		(0x0900)
+
+/* PDP, PDISETUP, PDI_BLNKLVL
+*/
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_MASK		(0x00000040)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_SHIFT		(6)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_BLNK
+*/
+#define ODN_PDP_PDISETUP_PDI_BLNK_MASK		(0x00000020)
+#define ODN_PDP_PDISETUP_PDI_BLNK_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_BLNK_SHIFT		(5)
+#define ODN_PDP_PDISETUP_PDI_BLNK_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_BLNK_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_PWR
+*/
+#define ODN_PDP_PDISETUP_PDI_PWR_MASK		(0x00000010)
+#define ODN_PDP_PDISETUP_PDI_PWR_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_PWR_SHIFT		(4)
+#define ODN_PDP_PDISETUP_PDI_PWR_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_PWR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_EN
+*/
+#define ODN_PDP_PDISETUP_PDI_EN_MASK		(0x00000008)
+#define ODN_PDP_PDISETUP_PDI_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_EN_SHIFT		(3)
+#define ODN_PDP_PDISETUP_PDI_EN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_GDEN
+*/
+#define ODN_PDP_PDISETUP_PDI_GDEN_MASK		(0x00000004)
+#define ODN_PDP_PDISETUP_PDI_GDEN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_GDEN_SHIFT		(2)
+#define ODN_PDP_PDISETUP_PDI_GDEN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_GDEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_NFEN
+*/
+#define ODN_PDP_PDISETUP_PDI_NFEN_MASK		(0x00000002)
+#define ODN_PDP_PDISETUP_PDI_NFEN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_NFEN_SHIFT		(1)
+#define ODN_PDP_PDISETUP_PDI_NFEN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_NFEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_CR
+*/
+#define ODN_PDP_PDISETUP_PDI_CR_MASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_CR_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_CR_SHIFT		(0)
+#define ODN_PDP_PDISETUP_PDI_CR_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_CR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDITIMING0_OFFSET		(0x0904)
+
+/* PDP, PDITIMING0, PDI_PWRSVGD
+*/
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_MASK		(0x0F000000)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_LSBMASK		(0x0000000F)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_SHIFT		(24)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_LENGTH		(4)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_LSDEL
+*/
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_MASK		(0x007F0000)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_LSBMASK		(0x0000007F)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_SHIFT		(16)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_LENGTH		(7)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_PWRSV2GD2
+*/
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_MASK		(0x000003FF)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_LSBMASK		(0x000003FF)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_SHIFT		(0)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_LENGTH		(10)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDITIMING1_OFFSET		(0x0908)
+
+/* PDP, PDITIMING1, PDI_NLDEL
+*/
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_MASK		(0x000F0000)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_LSBMASK		(0x0000000F)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_SHIFT		(16)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_LENGTH		(4)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING1, PDI_ACBDEL
+*/
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_MASK		(0x000003FF)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_LSBMASK		(0x000003FF)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_SHIFT		(0)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_LENGTH		(10)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDICOREID_OFFSET		(0x090C)
+
+/* PDP, PDICOREID, PDI_GROUP_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_MASK		(0xFF000000)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_SHIFT		(24)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_LENGTH		(8)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CORE_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_MASK		(0x00FF0000)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_SHIFT		(16)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_LENGTH		(8)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CONFIG_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_MASK		(0x0000FFFF)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_SHIFT		(0)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_LENGTH		(16)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDICOREREV_OFFSET		(0x0910)
+
+/* PDP, PDICOREREV, PDI_MAJOR_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_MASK		(0x00FF0000)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_SHIFT		(16)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MINOR_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_MASK		(0x0000FF00)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_SHIFT		(8)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MAINT_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_MASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_SHIFT		(0)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX2_OFFSET		(0x0920)
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_MASK		(0x000000C0)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_MASK		(0x00000030)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_MASK		(0x0000000C)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SHIFT		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_MASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX4_0_OFFSET		(0x0924)
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_MASK		(0xF0000000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SHIFT		(28)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_MASK		(0x0F000000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_MASK		(0x00F00000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SHIFT		(20)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_MASK		(0x000F0000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SHIFT		(16)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_MASK		(0x0000F000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_MASK		(0x00000F00)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SHIFT		(8)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_MASK		(0x000000F0)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_MASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX4_1_OFFSET		(0x0928)
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_MASK		(0xF0000000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SHIFT		(28)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_MASK		(0x0F000000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_MASK		(0x00F00000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SHIFT		(20)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_MASK		(0x000F0000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SHIFT		(16)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_MASK		(0x0000F000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_MASK		(0x00000F00)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SHIFT		(8)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_MASK		(0x000000F0)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_MASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_0_OFFSET		(0x092C)
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X4Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X3Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X2Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_1_OFFSET		(0x0930)
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X7Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X6Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X5Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_2_OFFSET		(0x0934)
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X6Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X5Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X4Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X3Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X2Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_3_OFFSET		(0x0938)
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X3Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X2Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X1Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X0Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X7Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_4_OFFSET		(0x093C)
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X0Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X7Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X6Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X5Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X4Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_5_OFFSET		(0x0940)
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X5Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X4Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X3Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X2Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X1Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_6_OFFSET		(0x0944)
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X2Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X1Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X0Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X7Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X6Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_7_OFFSET		(0x0948)
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X7Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X6Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X5Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X4Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X3Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_8_OFFSET		(0x094C)
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X4Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X3Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X2Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X1Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X0Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_9_OFFSET		(0x0950)
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X1Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X0Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X7Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X6Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X5Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_10_OFFSET		(0x0954)
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X6Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X5Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X4Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X3Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X2Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_11_OFFSET		(0x0958)
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X3Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X2Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X1Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X0Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X7Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_12_OFFSET		(0x095C)
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X7Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X6Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X5Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X4Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_MEMCTRL_OFFSET		(0x0960)
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_BURSTLEN
+*/
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_MEM_THRESH_OFFSET		(0x0964)
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_THRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_MEMCTRL_OFFSET		(0x0968)
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_BURSTLEN
+*/
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_MEM_THRESH_OFFSET		(0x096C)
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_THRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_MEMCTRL_OFFSET		(0x0970)
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_BURSTLEN
+*/
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_MEM_THRESH_OFFSET		(0x0974)
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_THRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_MEMCTRL_OFFSET		(0x0978)
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_BURSTLEN
+*/
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_MEM_THRESH_OFFSET		(0x097C)
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_THRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_MEMCTRL_OFFSET		(0x0980)
+
+/* PDP, VID1_MEMCTRL, VID1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEMCTRL, VID1_BURSTLEN
+*/
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_MEM_THRESH_OFFSET		(0x0984)
+
+/* PDP, VID1_MEM_THRESH, VID1_UVTHRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_YTHRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_THRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_MEMCTRL_OFFSET		(0x0988)
+
+/* PDP, VID2_MEMCTRL, VID2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEMCTRL, VID2_BURSTLEN
+*/
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_MEM_THRESH_OFFSET		(0x098C)
+
+/* PDP, VID2_MEM_THRESH, VID2_UVTHRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_YTHRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_THRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_MEMCTRL_OFFSET		(0x0990)
+
+/* PDP, VID3_MEMCTRL, VID3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEMCTRL, VID3_BURSTLEN
+*/
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_MEM_THRESH_OFFSET		(0x0994)
+
+/* PDP, VID3_MEM_THRESH, VID3_UVTHRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_YTHRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_THRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_MEMCTRL_OFFSET		(0x0998)
+
+/* PDP, VID4_MEMCTRL, VID4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEMCTRL, VID4_BURSTLEN
+*/
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_MEM_THRESH_OFFSET		(0x099C)
+
+/* PDP, VID4_MEM_THRESH, VID4_UVTHRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_YTHRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_THRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_PANIC_THRESH_OFFSET		(0x09A0)
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_PANIC_THRESH_OFFSET		(0x09A4)
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_PANIC_THRESH_OFFSET		(0x09A8)
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_PANIC_THRESH_OFFSET		(0x09AC)
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_PANIC_THRESH_OFFSET		(0x09B0)
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_PANIC_THRESH_OFFSET		(0x09B4)
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_PANIC_THRESH_OFFSET		(0x09B8)
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_PANIC_THRESH_OFFSET		(0x09BC)
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BURST_BOUNDARY_OFFSET		(0x09C0)
+
+/* PDP, BURST_BOUNDARY, BURST_BOUNDARY
+*/
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_MASK		(0x0000003F)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_LSBMASK		(0x0000003F)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_SHIFT		(0)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_LENGTH		(6)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_SIGNED_FIELD	IMG_FALSE
+
+
+/* ---------------------- End of register definitions ---------------------- */
+
+/* NUMREG defines the extent of register address space.
+*/
+
+#define		ODN_PDP_NUMREG	   ((0x09C0 >> 2)+1)
+
+/* Info about video plane addresses */
+#define ODN_PDP_YADDR_BITS		(ODN_PDP_VID1BASEADDR_VID1BASEADDR_LENGTH)
+#define ODN_PDP_YADDR_ALIGN		5
+#define ODN_PDP_UADDR_BITS		(ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH)
+#define ODN_PDP_UADDR_ALIGN		5
+#define ODN_PDP_VADDR_BITS		(ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH)
+#define ODN_PDP_VADDR_ALIGN		5
+
+#define ODN_PDP_YSTRIDE_BITS	(ODN_PDP_VID1STRIDE_VID1STRIDE_LENGTH)
+#define ODN_PDP_YSTRIDE_ALIGN	5
+
+#define ODN_PDP_MAX_INPUT_WIDTH (ODN_PDP_VID1SIZE_VID1WIDTH_LSBMASK + 1)
+#define ODN_PDP_MAX_INPUT_HEIGHT (ODN_PDP_VID1SIZE_VID1HEIGHT_LSBMASK + 1)
+
+/* Maximum 6 bytes per pixel for RGB161616 */
+#define ODN_PDP_MAX_IMAGE_BYTES (ODN_PDP_MAX_INPUT_WIDTH * ODN_PDP_MAX_INPUT_HEIGHT * 6)
+
+/* Round up */
+#define ODN_PDP_MAX_IMAGE_PAGES ((ODN_PDP_MAX_IMAGE_BYTES+PAGE_SIZE-1)/PAGE_SIZE)
+
+#define ODN_PDP_YADDR_MAX		(((1 << ODN_PDP_YADDR_BITS) - 1) << ODN_PDP_YADDR_ALIGN)
+#define ODN_PDP_UADDR_MAX		(((1 << ODN_PDP_UADDR_BITS) - 1) << ODN_PDP_UADDR_ALIGN)
+#define ODN_PDP_VADDR_MAX		(((1 << ODN_PDP_VADDR_BITS) - 1) << ODN_PDP_VADDR_ALIGN)
+#define ODN_PDP_YSTRIDE_MAX		((1 << ODN_PDP_YSTRIDE_BITS) << ODN_PDP_YSTRIDE_ALIGN)
+#define ODN_PDP_YADDR_ALIGNMASK		((1 << ODN_PDP_YADDR_ALIGN) - 1)
+#define ODN_PDP_UADDR_ALIGNMASK		((1 << ODN_PDP_UADDR_ALIGN) - 1)
+#define ODN_PDP_VADDR_ALIGNMASK		((1 << ODN_PDP_VADDR_ALIGN) - 1)
+#define ODN_PDP_YSTRIDE_ALIGNMASK	((1 << ODN_PDP_YSTRIDE_ALIGN) - 1)
+
+/* Field Values (some are reserved for future use) */
+#define ODN_PDP_SURF_PIXFMT_RGB332					0x3
+#define ODN_PDP_SURF_PIXFMT_ARGB4444				0x4
+#define ODN_PDP_SURF_PIXFMT_ARGB1555				0x5
+#define ODN_PDP_SURF_PIXFMT_RGB888					0x6
+#define ODN_PDP_SURF_PIXFMT_RGB565					0x7
+#define ODN_PDP_SURF_PIXFMT_ARGB8888				0x8
+#define ODN_PDP_SURF_PIXFMT_420_PL8					0x9
+#define ODN_PDP_SURF_PIXFMT_420_PL8IVU				0xA
+#define ODN_PDP_SURF_PIXFMT_420_PL8IUV				0xB
+#define ODN_PDP_SURF_PIXFMT_422_UY0VY1_8888			0xC
+#define ODN_PDP_SURF_PIXFMT_422_VY0UY1_8888			0xD
+#define ODN_PDP_SURF_PIXFMT_422_Y0UY1V_8888			0xE
+#define ODN_PDP_SURF_PIXFMT_422_Y0VY1U_8888			0xF
+#define ODN_PDP_SURF_PIXFMT_AYUV8888				0x10
+#define ODN_PDP_SURF_PIXFMT_YUV101010				0x15
+#define ODN_PDP_SURF_PIXFMT_RGB101010				0x17
+#define ODN_PDP_SURF_PIXFMT_420_PL10IUV				0x18
+#define ODN_PDP_SURF_PIXFMT_420_PL10IVU				0x19
+#define ODN_PDP_SURF_PIXFMT_422_PL10IUV				0x1A
+#define ODN_PDP_SURF_PIXFMT_422_PL10IVU				0x1B
+#define ODN_PDP_SURF_PIXFMT_RGB121212				0x1E
+#define ODN_PDP_SURF_PIXFMT_RGB161616				0x1F
+
+#define ODN_PDP_CTRL_CKEYSRC_PREV					0x0
+#define ODN_PDP_CTRL_CKEYSRC_CUR					0x1
+
+#define ODN_PDP_MEMCTRL_MEMREFRESH_ALWAYS			0x0
+#define ODN_PDP_MEMCTRL_MEMREFRESH_HBLNK			0x1
+#define ODN_PDP_MEMCTRL_MEMREFRESH_VBLNK			0x2
+#define ODN_PDP_MEMCTRL_MEMREFRESH_BOTH				0x3
+
+#define ODN_PDP_3D_CTRL_BLENDSEL_BGND_WITH_POS0		0x0
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS0_WITH_POS1		0x1
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS1_WITH_POS2		0x2
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS2_WITH_POS3		0x3
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS3_WITH_POS4		0x4
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS4_WITH_POS5		0x5
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS5_WITH_POS6		0x6
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS6_WITH_POS7		0x7
+
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_Y_STRIDE	0x0
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_DOUBLE_Y_STRIDE 0x1
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_HALF_Y_STRIDE 0x2
+
+#define ODN_PDP_PROCAMP_OUTPUT_OFFSET_FRACTIONAL_BITS 1
+#define ODN_PDP_PROCAMP_COEFFICIENT_FRACTIONAL_BITS	10
+
+/*---------------------------------------------------------------------------*/
+
+#endif /* ODN_PDP_REGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_regs.h
new file mode 100644
index 000000000000..5d5821623ac7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_regs.h
@@ -0,0 +1,1026 @@
+/******************************************************************************
+@Title          Odin system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin FPGA register defs for IMG 3rd generation TCF
+
+	Auto generated headers, eg. odn_core.h:
+		regconv -d . -a 8 odn_core.def
+
+	Source files :
+		odn_core.def
+		mca_debug.def
+		sai_rx_debug.def
+		sai_tx_debug.def
+		ad_tx.def
+
+	Changes:
+		Removed obsolete copyright dates
+		Changed lower case to upper case
+			(eg. odn_core changed to ODN_CORE)
+		Changed PVR5__ to ODN_
+		Merged multiple .def files into one header
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+/* tab size 4 */
+
+#ifndef _ODIN_REGS_H_
+#define _ODIN_REGS_H_
+
+/******************************
+  Generated from: odn_core.def
+*******************************/
+
+/*
+	Register ID
+*/
+#define ODN_CORE_ID                             0x0000
+#define ODN_ID_VARIANT_MASK                     0x0000FFFFU
+#define ODN_ID_VARIANT_SHIFT                    0
+#define ODN_ID_VARIANT_SIGNED                   0
+
+#define ODN_ID_ID_MASK                          0xFFFF0000U
+#define ODN_ID_ID_SHIFT                         16
+#define ODN_ID_ID_SIGNED                        0
+
+/*
+	Register REL
+*/
+#define ODN_CORE_REL                            0x0004
+#define ODN_REL_MINOR_MASK                      0x0000FFFFU
+#define ODN_REL_MINOR_SHIFT                     0
+#define ODN_REL_MINOR_SIGNED                    0
+
+#define ODN_REL_MAJOR_MASK                      0xFFFF0000U
+#define ODN_REL_MAJOR_SHIFT                     16
+#define ODN_REL_MAJOR_SIGNED                    0
+
+/*
+	Register CHANGE_SET
+*/
+#define ODN_CORE_CHANGE_SET                     0x0008
+#define ODN_CHANGE_SET_SET_MASK                 0xFFFFFFFFU
+#define ODN_CHANGE_SET_SET_SHIFT                0
+#define ODN_CHANGE_SET_SET_SIGNED               0
+
+/*
+	Register USER_ID
+*/
+#define ODN_CORE_USER_ID                        0x000C
+#define ODN_USER_ID_ID_MASK                     0x000000FFU
+#define ODN_USER_ID_ID_SHIFT                    0
+#define ODN_USER_ID_ID_SIGNED                   0
+
+/*
+	Register USER_BUILD
+*/
+#define ODN_CORE_USER_BUILD                     0x0010
+#define ODN_USER_BUILD_BUILD_MASK               0xFFFFFFFFU
+#define ODN_USER_BUILD_BUILD_SHIFT              0
+#define ODN_USER_BUILD_BUILD_SIGNED             0
+
+/*
+	Register SW_IF_VERSION
+*/
+#define ODN_CORE_SW_IF_VERSION                  0x0014
+#define ODN_SW_IF_VERSION_VERSION_MASK          0x0000FFFFU
+#define ODN_SW_IF_VERSION_VERSION_SHIFT         0
+#define ODN_SW_IF_VERSION_VERSION_SIGNED        0
+
+/*
+	Register INTERNAL_RESETN
+*/
+#define ODN_CORE_INTERNAL_RESETN                0x0080
+#define ODN_INTERNAL_RESETN_DDR_MASK            0x00000001U
+#define ODN_INTERNAL_RESETN_DDR_SHIFT           0
+#define ODN_INTERNAL_RESETN_DDR_SIGNED          0
+
+#define ODN_INTERNAL_RESETN_MIG0_MASK           0x00000002U
+#define ODN_INTERNAL_RESETN_MIG0_SHIFT          1
+#define ODN_INTERNAL_RESETN_MIG0_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_MIG1_MASK           0x00000004U
+#define ODN_INTERNAL_RESETN_MIG1_SHIFT          2
+#define ODN_INTERNAL_RESETN_MIG1_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PDP1_MASK           0x00000008U
+#define ODN_INTERNAL_RESETN_PDP1_SHIFT          3
+#define ODN_INTERNAL_RESETN_PDP1_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PDP2_MASK           0x00000010U
+#define ODN_INTERNAL_RESETN_PDP2_SHIFT          4
+#define ODN_INTERNAL_RESETN_PDP2_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PERIP_MASK          0x00000020U
+#define ODN_INTERNAL_RESETN_PERIP_SHIFT         5
+#define ODN_INTERNAL_RESETN_PERIP_SIGNED        0
+
+#define ODN_INTERNAL_RESETN_GIST_MASK           0x00000040U
+#define ODN_INTERNAL_RESETN_GIST_SHIFT          6
+#define ODN_INTERNAL_RESETN_GIST_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PIKE_MASK           0x00000080U
+#define ODN_INTERNAL_RESETN_PIKE_SHIFT          7
+#define ODN_INTERNAL_RESETN_PIKE_SIGNED         0
+
+/*
+	Register EXTERNAL_RESETN
+*/
+#define ODN_CORE_EXTERNAL_RESETN                0x0084
+#define ODN_EXTERNAL_RESETN_DUT_MASK            0x00000001U
+#define ODN_EXTERNAL_RESETN_DUT_SHIFT           0
+#define ODN_EXTERNAL_RESETN_DUT_SIGNED          0
+
+#define ODN_EXTERNAL_RESETN_DUT_SPI_MASK        0x00000002U
+#define ODN_EXTERNAL_RESETN_DUT_SPI_SHIFT       1
+#define ODN_EXTERNAL_RESETN_DUT_SPI_SIGNED      0
+
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_MASK    0x00000004U
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_SHIFT   2
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_SIGNED  0
+
+#define ODN_EXTERNAL_RESETN_DUT_IF_MASK         0x00000008U
+#define ODN_EXTERNAL_RESETN_DUT_IF_SHIFT        3
+#define ODN_EXTERNAL_RESETN_DUT_IF_SIGNED       0
+
+#define ODN_EXTERNAL_RESETN_DUT1_MASK           0x00000010U
+#define ODN_EXTERNAL_RESETN_DUT1_SHIFT          4
+#define ODN_EXTERNAL_RESETN_DUT1_SIGNED         0
+
+#define ODN_EXTERNAL_RESETN_DUT2_MASK           0x00000020U
+#define ODN_EXTERNAL_RESETN_DUT2_SHIFT          5
+#define ODN_EXTERNAL_RESETN_DUT2_SIGNED         0
+
+/*
+	Register EXTERNAL_RESET
+*/
+#define ODN_CORE_EXTERNAL_RESET                 0x0088
+#define ODN_EXTERNAL_RESET_PVT_CAL_MASK         0x00000001U
+#define ODN_EXTERNAL_RESET_PVT_CAL_SHIFT        0
+#define ODN_EXTERNAL_RESET_PVT_CAL_SIGNED       0
+
+#define ODN_EXTERNAL_RESET_PLL_MASK             0x00000002U
+#define ODN_EXTERNAL_RESET_PLL_SHIFT            1
+#define ODN_EXTERNAL_RESET_PLL_SIGNED           0
+
+/*
+	Register INTERNAL_AUTO_RESETN
+*/
+#define ODN_CORE_INTERNAL_AUTO_RESETN           0x008C
+#define ODN_INTERNAL_AUTO_RESETN_AUX_MASK       0x00000001U
+#define ODN_INTERNAL_AUTO_RESETN_AUX_SHIFT      0
+#define ODN_INTERNAL_AUTO_RESETN_AUX_SIGNED     0
+
+/*
+	Register CLK_GEN_RESET
+*/
+#define ODN_CORE_CLK_GEN_RESET                  0x0090
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_MASK    0x00000001U
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_SHIFT   0
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_SIGNED  0
+
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_MASK      0x00000002U
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_SHIFT     1
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_SIGNED    0
+
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_MASK       0x00000004U
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_SHIFT      2
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_SIGNED     0
+
+#define ODN_CLK_GEN_RESET_PDP_MMCM_MASK         0x00000008U
+#define ODN_CLK_GEN_RESET_PDP_MMCM_SHIFT        3
+#define ODN_CLK_GEN_RESET_PDP_MMCM_SIGNED       0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define ODN_CORE_INTERRUPT_STATUS               0x0100
+#define ODN_INTERRUPT_STATUS_DUT_MASK           0x00000001U
+#define ODN_INTERRUPT_STATUS_DUT_SHIFT          0
+#define ODN_INTERRUPT_STATUS_DUT_SIGNED         0
+
+#define ODN_INTERRUPT_STATUS_PDP1_MASK          0x00000002U
+#define ODN_INTERRUPT_STATUS_PDP1_SHIFT         1
+#define ODN_INTERRUPT_STATUS_PDP1_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_PDP2_MASK          0x00000004U
+#define ODN_INTERRUPT_STATUS_PDP2_SHIFT         2
+#define ODN_INTERRUPT_STATUS_PDP2_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_PERIP_MASK         0x00000008U
+#define ODN_INTERRUPT_STATUS_PERIP_SHIFT        3
+#define ODN_INTERRUPT_STATUS_PERIP_SIGNED       0
+
+#define ODN_INTERRUPT_STATUS_UART_MASK          0x00000010U
+#define ODN_INTERRUPT_STATUS_UART_SHIFT         4
+#define ODN_INTERRUPT_STATUS_UART_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_MASK 0x00000020U
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_MASK 0x00000040U
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_SHIFT 6
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_MASK 0x00000100U
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_DUT2_MASK          0x00000200U
+#define ODN_INTERRUPT_STATUS_DUT2_SHIFT         9
+#define ODN_INTERRUPT_STATUS_DUT2_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_CDMA_MASK          0x00001800U
+#define ODN_INTERRUPT_STATUS_CDMA_SHIFT         11
+#define ODN_INTERRUPT_STATUS_CDMA_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_OS_IRQ_MASK        0x001FE000U
+#define ODN_INTERRUPT_STATUS_OS_IRQ_SHIFT       13
+#define ODN_INTERRUPT_STATUS_OS_IRQ_SIGNED      0
+
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_MASK      0x40000000U
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_SHIFT     30
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_SIGNED    0
+
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_MASK 0x80000000U
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_SHIFT 31
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_SIGNED 0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define ODN_CORE_INTERRUPT_ENABLE               0x0104
+#define ODN_INTERRUPT_ENABLE_DUT_MASK           0x00000001U
+#define ODN_INTERRUPT_ENABLE_DUT_SHIFT          0
+#define ODN_INTERRUPT_ENABLE_DUT_SIGNED         0
+
+#define ODN_INTERRUPT_ENABLE_PDP1_MASK          0x00000002U
+#define ODN_INTERRUPT_ENABLE_PDP1_SHIFT         1
+#define ODN_INTERRUPT_ENABLE_PDP1_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_PDP2_MASK          0x00000004U
+#define ODN_INTERRUPT_ENABLE_PDP2_SHIFT         2
+#define ODN_INTERRUPT_ENABLE_PDP2_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_PERIP_MASK         0x00000008U
+#define ODN_INTERRUPT_ENABLE_PERIP_SHIFT        3
+#define ODN_INTERRUPT_ENABLE_PERIP_SIGNED       0
+
+#define ODN_INTERRUPT_ENABLE_UART_MASK          0x00000010U
+#define ODN_INTERRUPT_ENABLE_UART_SHIFT         4
+#define ODN_INTERRUPT_ENABLE_UART_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_MASK 0x00000020U
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_MASK 0x00000040U
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_SHIFT 6
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_MASK 0x00000100U
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_DUT2_MASK          0x00000200U
+#define ODN_INTERRUPT_ENABLE_DUT2_SHIFT         9
+#define ODN_INTERRUPT_ENABLE_DUT2_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_CDMA_MASK          0x00001800U
+#define ODN_INTERRUPT_ENABLE_CDMA_SHIFT         11
+#define ODN_INTERRUPT_ENABLE_CDMA_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_MASK        0x001FE000U
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_SHIFT       13
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_SIGNED      0
+
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_MASK      0x40000000U
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_SHIFT     30
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_SIGNED    0
+
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_MASK 0x80000000U
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_SHIFT 31
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_SIGNED 0
+
+/*
+	Register INTERRUPT_CLR
+*/
+#define ODN_CORE_INTERRUPT_CLR                  0x010C
+#define ODN_INTERRUPT_CLR_DUT_MASK              0x00000001U
+#define ODN_INTERRUPT_CLR_DUT_SHIFT             0
+#define ODN_INTERRUPT_CLR_DUT_SIGNED            0
+
+#define ODN_INTERRUPT_CLR_PDP1_MASK             0x00000002U
+#define ODN_INTERRUPT_CLR_PDP1_SHIFT            1
+#define ODN_INTERRUPT_CLR_PDP1_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_PDP2_MASK             0x00000004U
+#define ODN_INTERRUPT_CLR_PDP2_SHIFT            2
+#define ODN_INTERRUPT_CLR_PDP2_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_PERIP_MASK            0x00000008U
+#define ODN_INTERRUPT_CLR_PERIP_SHIFT           3
+#define ODN_INTERRUPT_CLR_PERIP_SIGNED          0
+
+#define ODN_INTERRUPT_CLR_UART_MASK             0x00000010U
+#define ODN_INTERRUPT_CLR_UART_SHIFT            4
+#define ODN_INTERRUPT_CLR_UART_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_MASK  0x00000020U
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_MASK   0x00000040U
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_SHIFT  6
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_MASK  0x00000100U
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_DUT2_MASK             0x00000200U
+#define ODN_INTERRUPT_CLR_DUT2_SHIFT            9
+#define PVR5__INTERRUPT_CLR_DUT2_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_CDMA_MASK             0x00001800U
+#define ODN_INTERRUPT_CLR_CDMA_SHIFT            11
+#define ODN_INTERRUPT_CLR_CDMA_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_OS_IRQ_MASK           0x001FE000U
+#define ODN_INTERRUPT_CLR_OS_IRQ_SHIFT          13
+#define ODN_INTERRUPT_CLR_OS_IRQ_SIGNED         0
+
+#define ODN_INTERRUPT_CLR_IRQ_TEST_MASK         0x40000000U
+#define ODN_INTERRUPT_CLR_IRQ_TEST_SHIFT        30
+#define ODN_INTERRUPT_CLR_IRQ_TEST_SIGNED       0
+
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_MASK     0x80000000U
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_SHIFT    31
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_SIGNED   0
+
+/*
+	Register INTERRUPT_TEST
+*/
+#define ODN_CORE_INTERRUPT_TEST                 0x0110
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_MASK  0x00000001U
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_SHIFT 0
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT_CLR
+*/
+#define ODN_CORE_INTERRUPT_TIMEOUT_CLR      0x0114
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK 0x00000002U
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SHIFT 1
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SIGNED 0
+
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_MASK 0x00000001U
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SHIFT 0
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT
+*/
+#define ODN_CORE_INTERRUPT_TIMEOUT          0x0118
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_MASK 0xFFFFFFFFU
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SHIFT 0
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SIGNED 0
+/*
+	Register SYSTEM_ID
+*/
+#define ODN_CORE_SYSTEM_ID                      0x011C
+#define ODN_SYSTEM_ID_ID_MASK                   0x0000FFFFU
+#define ODN_SYSTEM_ID_ID_SHIFT                  0
+#define ODN_SYSTEM_ID_ID_SIGNED                 0
+
+/*
+	Register SUPPORTED_FEATURES
+*/
+#define ODN_CORE_SUPPORTED_FEATURES             0x0120
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_MASK 0xFFFFFFFEU
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_SHIFT 1
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_SIGNED 0
+
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_MASK 0x00000001U
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_SHIFT 0
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_SIGNED 0
+
+/*
+	Register NUM_GPIO
+*/
+#define ODN_CORE_NUM_GPIO                       0x0180
+#define ODN_NUM_GPIO_NUMBER_MASK                0x0000000FU
+#define ODN_NUM_GPIO_NUMBER_SHIFT               0
+#define ODN_NUM_GPIO_NUMBER_SIGNED              0
+
+/*
+	Register GPIO_EN
+*/
+#define ODN_CORE_GPIO_EN                        0x0184
+#define ODN_GPIO_EN_DIRECTION_MASK              0x000000FFU
+#define ODN_GPIO_EN_DIRECTION_SHIFT             0
+#define ODN_GPIO_EN_DIRECTION_SIGNED            0
+
+/*
+	Register GPIO
+*/
+#define ODN_CORE_GPIO                           0x0188
+#define ODN_GPIO_GPIO_MASK                      0x000000FFU
+#define ODN_GPIO_GPIO_SHIFT                     0
+#define ODN_GPIO_GPIO_SIGNED                    0
+
+/*
+	Register NUM_DUT_CTRL
+*/
+#define ODN_CORE_NUM_DUT_CTRL                   0x0190
+#define ODN_NUM_DUT_CTRL_NUM_PINS_MASK          0xFFFFFFFFU
+#define ODN_NUM_DUT_CTRL_NUM_PINS_SHIFT         0
+#define ODN_NUM_DUT_CTRL_NUM_PINS_SIGNED        0
+
+/*
+	Register DUT_CTRL1
+*/
+#define ODN_CORE_DUT_CTRL1                      0x0194
+#define ODN_DUT_CTRL1_CONTROL1_MASK             0x3FFFFFFFU
+#define ODN_DUT_CTRL1_CONTROL1_SHIFT            0
+#define ODN_DUT_CTRL1_CONTROL1_SIGNED           0
+
+#define ODN_DUT_CTRL1_FBDC_BYPASS_MASK          0x40000000U
+#define ODN_DUT_CTRL1_FBDC_BYPASS_SHIFT         30
+#define ODN_DUT_CTRL1_FBDC_BYPASS_SIGNED        0
+
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_MASK       0x80000000U
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_SHIFT      31
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_SIGNED     0
+
+/*
+	Register DUT_CTRL2
+*/
+#define ODN_CORE_DUT_CTRL2                      0x0198
+#define ODN_DUT_CTRL2_CONTROL2_MASK             0xFFFFFFFFU
+#define ODN_DUT_CTRL2_CONTROL2_SHIFT            0
+#define ODN_DUT_CTRL2_CONTROL2_SIGNED           0
+
+/*
+	Register NUM_DUT_STAT
+*/
+#define ODN_CORE_NUM_DUT_STAT                   0x019C
+#define ODN_NUM_DUT_STAT_NUM_PINS_MASK          0xFFFFFFFFU
+#define ODN_NUM_DUT_STAT_NUM_PINS_SHIFT         0
+#define ODN_NUM_DUT_STAT_NUM_PINS_SIGNED        0
+
+/*
+	Register DUT_STAT1
+*/
+#define ODN_CORE_DUT_STAT1                      0x01A0
+#define ODN_DUT_STAT1_STATUS1_MASK              0xFFFFFFFFU
+#define ODN_DUT_STAT1_STATUS1_SHIFT             0
+#define ODN_DUT_STAT1_STATUS1_SIGNED            0
+
+/*
+	Register DUT_STAT2
+*/
+#define ODN_CORE_DUT_STAT2                      0x01A4
+#define ODN_DUT_STAT2_STATUS2_MASK              0xFFFFFFFFU
+#define ODN_DUT_STAT2_STATUS2_SHIFT             0
+#define ODN_DUT_STAT2_STATUS2_SIGNED            0
+
+/*
+	Register DASH_LEDS
+*/
+#define ODN_CORE_DASH_LEDS                      0x01A8
+#define ODN_DASH_LEDS_REPA_MASK                 0xFFF00000U
+#define ODN_DASH_LEDS_REPA_SHIFT                20
+#define ODN_DASH_LEDS_REPA_SIGNED               0
+
+#define ODN_DASH_LEDS_PIKE_MASK                 0x00000FFFU
+#define ODN_DASH_LEDS_PIKE_SHIFT                0
+#define ODN_DASH_LEDS_PIKE_SIGNED               0
+
+/*
+	Register DUT_CLK_INFO
+*/
+#define ODN_CORE_DUT_CLK_INFO                   0x01B0
+#define ODN_DUT_CLK_INFO_CORE_MASK              0x0000FFFFU
+#define ODN_DUT_CLK_INFO_CORE_SHIFT             0
+#define ODN_DUT_CLK_INFO_CORE_SIGNED            0
+
+#define ODN_DUT_CLK_INFO_MEM_MASK               0xFFFF0000U
+#define ODN_DUT_CLK_INFO_MEM_SHIFT              16
+#define ODN_DUT_CLK_INFO_MEM_SIGNED             0
+
+/*
+	Register DUT_CLK_PHSE
+*/
+#define ODN_CORE_DUT_CLK_PHSE                   0x01B4
+#define ODN_DUT_CLK_PHSE_MEM_REQ_MASK           0x0000FFFFU
+#define ODN_DUT_CLK_PHSE_MEM_REQ_SHIFT          0
+#define ODN_DUT_CLK_PHSE_MEM_REQ_SIGNED         0
+
+#define ODN_DUT_CLK_PHSE_MEM_RD_MASK            0xFFFF0000U
+#define ODN_DUT_CLK_PHSE_MEM_RD_SHIFT           16
+#define ODN_DUT_CLK_PHSE_MEM_RD_SIGNED          0
+
+/*
+	Register CORE_STATUS
+*/
+#define ODN_CORE_CORE_STATUS                    0x0200
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_MASK   0x00000001U
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_SHIFT  0
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_MASK 0x00000010U
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_SHIFT 4
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_MASK 0x00000020U
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_SHIFT 5
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_MASK 0x00000040U
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_SHIFT 6
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_MASK 0x00000080U
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_SHIFT 7
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_MASK 0x00000100U
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_SHIFT 8
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_SIGNED 0
+
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_MASK 0x00000200U
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_SHIFT 9
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_SIGNED 0
+
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_MASK 0x00001000U
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SHIFT 12
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_MASK 0x00002000U
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SHIFT 13
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+/*
+	Register CORE_CONTROL
+*/
+#define ODN_CORE_CORE_CONTROL                   0x0204
+#define ODN_CORE_CONTROL_BAR4_OFFSET_MASK       0x0000001FU
+#define ODN_CORE_CONTROL_BAR4_OFFSET_SHIFT      0
+#define ODN_CORE_CONTROL_BAR4_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_MASK 0x00000300U
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SHIFT 8
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SIGNED 0
+
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_MASK    0x00001C00U
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_SHIFT   10
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_SIGNED  0
+
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_MASK 0x00002000U
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SHIFT 13
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SIGNED 0
+
+#define ODN_CORE_CONTROL_PDP1_OFFSET_MASK       0x00070000U
+#define ODN_CORE_CONTROL_PDP1_OFFSET_SHIFT      16
+#define ODN_CORE_CONTROL_PDP1_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_PDP2_OFFSET_MASK       0x00700000U
+#define ODN_CORE_CONTROL_PDP2_OFFSET_SHIFT      20
+#define ODN_CORE_CONTROL_PDP2_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_DUT_OFFSET_MASK        0x07000000U
+#define ODN_CORE_CONTROL_DUT_OFFSET_SHIFT       24
+#define ODN_CORE_CONTROL_DUT_OFFSET_SIGNED      0
+
+/*
+	Register REG_BANK_STATUS
+*/
+#define ODN_CORE_REG_BANK_STATUS                0x0208
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_MASK 0xFFFFFFFFU
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SHIFT 0
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SIGNED 0
+
+/*
+	Register MMCM_LOCK_STATUS
+*/
+#define ODN_CORE_MMCM_LOCK_STATUS               0x020C
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_MASK      0x00000001U
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_SHIFT     0
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_SIGNED    0
+
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_MASK        0x00000002U
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_SHIFT       1
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_SIGNED      0
+
+#define ODN_MMCM_LOCK_STATUS_MULTI_MASK         0x00000004U
+#define ODN_MMCM_LOCK_STATUS_MULTI_SHIFT        2
+#define ODN_MMCM_LOCK_STATUS_MULTI_SIGNED       0
+
+#define ODN_MMCM_LOCK_STATUS_PDPP_MASK          0x00000008U
+#define ODN_MMCM_LOCK_STATUS_PDPP_SHIFT         3
+#define ODN_MMCM_LOCK_STATUS_PDPP_SIGNED        0
+
+/*
+	Register GIST_STATUS
+*/
+#define ODN_CORE_GIST_STATUS                    0x0210
+#define ODN_GIST_STATUS_MST_MASK                0x000001FFU
+#define ODN_GIST_STATUS_MST_SHIFT               0
+#define ODN_GIST_STATUS_MST_SIGNED              0
+
+#define ODN_GIST_STATUS_SLV_MASK                0x001FF000U
+#define ODN_GIST_STATUS_SLV_SHIFT               12
+#define ODN_GIST_STATUS_SLV_SIGNED              0
+
+#define ODN_GIST_STATUS_SLV_OUT_MASK            0x03000000U
+#define ODN_GIST_STATUS_SLV_OUT_SHIFT           24
+#define ODN_GIST_STATUS_SLV_OUT_SIGNED          0
+
+#define ODN_GIST_STATUS_MST_OUT_MASK            0x70000000U
+#define ODN_GIST_STATUS_MST_OUT_SHIFT           28
+#define ODN_GIST_STATUS_MST_OUT_SIGNED          0
+
+/*
+	Register DUT_MST_ADD
+*/
+#define ODN_CORE_DUT_MST_ADD                    0x0214
+#define ODN_DUT_MST_ADD_SLV_OUT_MASK            0x0000003FU
+#define ODN_DUT_MST_ADD_SLV_OUT_SHIFT           0
+#define ODN_DUT_MST_ADD_SLV_OUT_SIGNED          0
+
+/*
+	Register DUT_MULTIPLX_INFO
+*/
+#define ODN_CORE_DUT_MULTIPLX_INFO              0x0218
+#define ODN_DUT_MULTIPLX_INFO_MEM_MASK          0x000000FFU
+#define ODN_DUT_MULTIPLX_INFO_MEM_SHIFT         0
+#define ODN_DUT_MULTIPLX_INFO_MEM_SIGNED        0
+
+/****************************
+  Generated from: ad_tx.def
+*****************************/
+
+/*
+	Register ADT_CONTROL
+*/
+#define ODN_AD_TX_DEBUG_ADT_CONTROL             0x0000
+#define ODN_SET_ADTX_READY_MASK                 0x00000004U
+#define ODN_SET_ADTX_READY_SHIFT                2
+#define ODN_SET_ADTX_READY_SIGNED               0
+
+#define ODN_SEND_ALIGN_DATA_MASK                0x00000002U
+#define ODN_SEND_ALIGN_DATA_SHIFT               1
+#define ODN_SEND_ALIGN_DATA_SIGNED              0
+
+#define ODN_ENABLE_FLUSHING_MASK                0x00000001U
+#define ODN_ENABLE_FLUSHING_SHIFT               0
+#define ODN_ENABLE_FLUSHING_SIGNED              0
+
+/*
+	Register ADT_STATUS
+*/
+#define ODN_AD_TX_DEBUG_ADT_STATUS              0x0004
+#define ODN_REQUEST_COMPLETE_MASK               0x00000001U
+#define ODN_REQUEST_COMPLETE_SHIFT              0
+#define ODN_REQUEST_COMPLETE_SIGNED             0
+
+
+/******************************
+ Generated from: mca_debug.def
+*******************************/
+
+/*
+	Register MCA_CONTROL
+*/
+#define ODN_MCA_DEBUG_MCA_CONTROL               0x0000
+#define ODN_ALIGN_START_MASK                    0x00000001U
+#define ODN_ALIGN_START_SHIFT                   0
+#define ODN_ALIGN_START_SIGNED                  0
+
+/*
+	Register MCA_STATUS
+*/
+#define ODN_MCA_DEBUG_MCA_STATUS                0x0004
+#define ODN_TCHECK_SDEBUG_MASK                  0x40000000U
+#define ODN_TCHECK_SDEBUG_SHIFT                 30
+#define ODN_TCHECK_SDEBUG_SIGNED                0
+
+#define ODN_CHECK_SDEBUG_MASK                   0x20000000U
+#define ODN_CHECK_SDEBUG_SHIFT                  29
+#define ODN_CHECK_SDEBUG_SIGNED                 0
+
+#define ODN_ALIGN_SDEBUG_MASK                   0x10000000U
+#define ODN_ALIGN_SDEBUG_SHIFT                  28
+#define ODN_ALIGN_SDEBUG_SIGNED                 0
+
+#define ODN_FWAIT_SDEBUG_MASK                   0x08000000U
+#define ODN_FWAIT_SDEBUG_SHIFT                  27
+#define ODN_FWAIT_SDEBUG_SIGNED                 0
+
+#define ODN_IDLE_SDEBUG_MASK                    0x04000000U
+#define ODN_IDLE_SDEBUG_SHIFT                   26
+#define ODN_IDLE_SDEBUG_SIGNED                  0
+
+#define ODN_FIFO_FULL_MASK                      0x03FF0000U
+#define ODN_FIFO_FULL_SHIFT                     16
+#define ODN_FIFO_FULL_SIGNED                    0
+
+#define ODN_FIFO_EMPTY_MASK                     0x0000FFC0U
+#define ODN_FIFO_EMPTY_SHIFT                    6
+#define ODN_FIFO_EMPTY_SIGNED                   0
+
+#define ODN_TAG_CHECK_ERROR_MASK                0x00000020U
+#define ODN_TAG_CHECK_ERROR_SHIFT               5
+#define ODN_TAG_CHECK_ERROR_SIGNED              0
+
+#define ODN_ALIGN_CHECK_ERROR_MASK              0x00000010U
+#define ODN_ALIGN_CHECK_ERROR_SHIFT             4
+#define ODN_ALIGN_CHECK_ERROR_SIGNED            0
+
+#define ODN_ALIGN_ERROR_MASK                    0x00000008U
+#define ODN_ALIGN_ERROR_SHIFT                   3
+#define ODN_ALIGN_ERROR_SIGNED                  0
+
+#define ODN_TAG_CHECKING_OK_MASK                0x00000004U
+#define ODN_TAG_CHECKING_OK_SHIFT               2
+#define ODN_TAG_CHECKING_OK_SIGNED              0
+
+#define ODN_ALIGN_CHECK_OK_MASK                 0x00000002U
+#define ODN_ALIGN_CHECK_OK_SHIFT                1
+#define ODN_ALIGN_CHECK_OK_SIGNED               0
+
+#define ODN_ALIGNMENT_FOUND_MASK                0x00000001U
+#define ODN_ALIGNMENT_FOUND_SHIFT               0
+#define ODN_ALIGNMENT_FOUND_SIGNED              0
+
+
+/*********************************
+ Generated from: sai_rx_debug.def
+**********************************/
+
+/*
+	Register SIG_RESULT
+*/
+#define ODN_SAI_RX_DEBUG_SIG_RESULT             0x0000
+#define ODN_SIG_RESULT_VALUE_MASK               0xFFFFFFFFU
+#define ODN_SIG_RESULT_VALUE_SHIFT              0
+#define ODN_SIG_RESULT_VALUE_SIGNED             0
+
+/*
+	Register INIT_SIG
+*/
+#define ODN_SAI_RX_DEBUG_INIT_SIG               0x0004
+#define ODN_INIT_SIG_VALUE_MASK                 0x00000001U
+#define ODN_INIT_SIG_VALUE_SHIFT                0
+#define ODN_INIT_SIG_VALUE_SIGNED               0
+
+/*
+	Register SAI_BYPASS
+*/
+#define ODN_SAI_RX_DEBUG_SAI_BYPASS             0x0008
+#define ODN_BYPASS_CLK_TAPS_VALUE_MASK          0x000003FFU
+#define ODN_BYPASS_CLK_TAPS_VALUE_SHIFT         0
+#define ODN_BYPASS_CLK_TAPS_VALUE_SIGNED        0
+
+#define ODN_BYPASS_SET_MASK                     0x00010000U
+#define ODN_BYPASS_SET_SHIFT                    16
+#define ODN_BYPASS_SET_SIGNED                   0
+
+#define ODN_BYPASS_EN_MASK                      0x00100000U
+#define ODN_BYPASS_EN_SHIFT                     20
+#define ODN_BYPASS_EN_SIGNED                    0
+
+#define ODN_EN_STATUS_MASK                      0x01000000U
+#define ODN_EN_STATUS_SHIFT                     24
+#define ODN_EN_STATUS_SIGNED                    0
+
+/*
+	Register SAI_CLK_TAPS
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CLK_TAPS           0x000C
+#define ODN_CLK_TAPS_VALUE_MASK                 0x000003FFU
+#define ODN_CLK_TAPS_VALUE_SHIFT                0
+#define ODN_CLK_TAPS_VALUE_SIGNED               0
+
+#define ODN_TRAINING_COMPLETE_MASK              0x00010000U
+#define ODN_TRAINING_COMPLETE_SHIFT             16
+#define ODN_TRAINING_COMPLETE_SIGNED            0
+
+/*
+	Register SAI_EYES
+*/
+#define ODN_SAI_RX_DEBUG_SAI_EYES               0x0010
+#define ODN_MIN_EYE_END_MASK                    0x0000FFFFU
+#define ODN_MIN_EYE_END_SHIFT                   0
+#define ODN_MIN_EYE_END_SIGNED                  0
+
+#define ODN_MAX_EYE_START_MASK                  0xFFFF0000U
+#define ODN_MAX_EYE_START_SHIFT                 16
+#define ODN_MAX_EYE_START_SIGNED                0
+
+/*
+	Register SAI_DDR_INVERT
+*/
+#define ODN_SAI_RX_DEBUG_SAI_DDR_INVERT         0x0014
+#define ODN_DDR_INVERT_MASK                     0x00000001U
+#define ODN_DDR_INVERT_SHIFT                    0
+#define ODN_DDR_INVERT_SIGNED                   0
+
+#define ODN_OVERIDE_VALUE_MASK                  0x00010000U
+#define ODN_OVERIDE_VALUE_SHIFT                 16
+#define ODN_OVERIDE_VALUE_SIGNED                0
+
+#define ODN_INVERT_OVERIDE_MASK                 0x00100000U
+#define ODN_INVERT_OVERIDE_SHIFT                20
+#define ODN_INVERT_OVERIDE_SIGNED               0
+
+/*
+	Register SAI_TRAIN_ACK
+*/
+#define ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK          0x0018
+#define ODN_TRAIN_ACK_FAIL_MASK                 0x00000001U
+#define ODN_TRAIN_ACK_FAIL_SHIFT                0
+#define ODN_TRAIN_ACK_FAIL_SIGNED               0
+
+#define ODN_TRAIN_ACK_FAIL_COUNT_MASK           0x000000F0U
+#define ODN_TRAIN_ACK_FAIL_COUNT_SHIFT          4
+#define ODN_TRAIN_ACK_FAIL_COUNT_SIGNED         0
+
+#define ODN_TRAIN_ACK_COMPLETE_MASK             0x00000100U
+#define ODN_TRAIN_ACK_COMPLETE_SHIFT            8
+#define ODN_TRAIN_ACK_COMPLETE_SIGNED           0
+
+#define ODN_TRAIN_ACK_OVERIDE_MASK              0x00001000U
+#define ODN_TRAIN_ACK_OVERIDE_SHIFT             12
+#define ODN_TRAIN_ACK_OVERIDE_SIGNED            0
+
+/*
+	Register SAI_TRAIN_ACK_COUNT
+*/
+#define ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK_COUNT    0x001C
+#define ODN_TRAIN_COUNT_MASK                    0xFFFFFFFFU
+#define ODN_TRAIN_COUNT_SHIFT                   0
+#define ODN_TRAIN_COUNT_SIGNED                  0
+
+/*
+	Register SAI_CHANNEL_NUMBER
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_NUMBER     0x0020
+#define ODN_CHANNEL_NUMBER_MASK                 0x0000FFFFU
+#define ODN_CHANNEL_NUMBER_SHIFT                0
+#define ODN_CHANNEL_NUMBER_SIGNED               0
+
+/*
+	Register SAI_CHANNEL_EYE_START
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_START  0x0024
+#define ODN_CHANNEL_EYE_START_MASK              0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_START_SHIFT             0
+#define ODN_CHANNEL_EYE_START_SIGNED            0
+
+/*
+	Register SAI_CHANNEL_EYE_END
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_END    0x0028
+#define ODN_CHANNEL_EYE_END_MASK                0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_END_SHIFT               0
+#define ODN_CHANNEL_EYE_END_SIGNED              0
+
+/*
+	Register SAI_CHANNEL_EYE_PATTERN
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_PATTERN 0x002C
+#define ODN_CHANNEL_EYE_PATTERN_MASK            0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_PATTERN_SHIFT           0
+#define ODN_CHANNEL_EYE_PATTERN_SIGNED          0
+
+/*
+	Register SAI_CHANNEL_EYE_DEBUG
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_DEBUG  0x0030
+#define ODN_CHANNEL_EYE_SENSE_MASK              0x00000001U
+#define ODN_CHANNEL_EYE_SENSE_SHIFT             0
+#define ODN_CHANNEL_EYE_SENSE_SIGNED            0
+
+#define ODN_CHANNEL_EYE_COMPLETE_MASK           0x00000002U
+#define ODN_CHANNEL_EYE_COMPLETE_SHIFT          1
+#define ODN_CHANNEL_EYE_COMPLETE_SIGNED         0
+
+
+/*********************************
+ Generated from: sai_tx_debug.def
+**********************************/
+
+/*
+	Register SIG_RESULT
+*/
+#define ODN_SAI_TX_DEBUG_SIG_RESULT             0x0000
+#define ODN_TX_SIG_RESULT_VALUE_MASK            0xFFFFFFFFU
+#define ODN_TX_SIG_RESULT_VALUE_SHIFT           0
+#define ODN_TX_SIG_RESULT_VALUE_SIGNED          0
+
+/*
+	Register INIT_SIG
+*/
+#define ODN_SAI_TX_DEBUG_INIT_SIG               0x0004
+#define ODN_TX_INIT_SIG_VALUE_MASK              0x00000001U
+#define ODN_TX_INIT_SIG_VALUE_SHIFT             0
+#define ODN_TX_INIT_SIG_VALUE_SIGNED            0
+
+/*
+	Register SAI_BYPASS
+*/
+#define ODN_SAI_TX_DEBUG_SAI_BYPASS             0x0008
+#define ODN_TX_BYPASS_EN_MASK                   0x00000001U
+#define ODN_TX_BYPASS_EN_SHIFT                  0
+#define ODN_TX_BYPASS_EN_SIGNED                 0
+
+#define ODN_TX_ACK_RESEND_MASK                  0x00000002U
+#define ODN_TX_ACK_RESEND_SHIFT                 1
+#define ODN_TX_ACK_RESEND_SIGNED                0
+
+#define ODN_TX_DISABLE_ACK_SEND_MASK            0x00000004U
+#define ODN_TX_DISABLE_ACK_SEND_SHIFT           2
+#define ODN_TX_DISABLE_ACK_SEND_SIGNED          0
+
+/*
+	Register SAI_STATUS
+*/
+#define ODN_SAI_TX_DEBUG_SAI_STATUS             0x000C
+#define ODN_TX_TRAINING_COMPLETE_MASK           0x00000001U
+#define ODN_TX_TRAINING_COMPLETE_SHIFT          0
+#define ODN_TX_TRAINING_COMPLETE_SIGNED         0
+
+#define ODN_TX_TRAINING_ACK_COMPLETE_MASK       0x00000002U
+#define ODN_TX_TRAINING_ACK_COMPLETE_SHIFT      1
+#define ODN_TX_TRAINING_ACK_COMPLETE_SIGNED     0
+
+
+
+#endif /* _ODIN_REGS_H_ */
+
+/******************************************************************************
+ End of file (odin_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_defs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_defs.h
new file mode 100644
index 000000000000..1691151de58d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_defs.h
@@ -0,0 +1,183 @@
+/****************************************************************************
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Orion Memory Map - View from PCIe
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+****************************************************************************/
+
+#ifndef _ORION_DEFS_H_
+#define _ORION_DEFS_H_
+
+/*
+ * These defines have not been autogenerated
+ * Only values different from Odin will be included here
+ */
+
+#define DEVICE_ID_ORION                          0x1020
+
+/* Odin system register banks */
+#define SRS_REG_BANK_ODN_CLK_BLK                 0x02000
+
+/*
+ * Orion CLK regs - the srs_clk_blk module defs are not auto generated
+ */
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG1           0x620
+#define SRS_PDP_PCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define SRS_PDP_PCLK_ODIV1_LO_TIME_SHIFT         0
+#define SRS_PDP_PCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define SRS_PDP_PCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG2           0x624
+#define SRS_PDP_PCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define SRS_PDP_PCLK_ODIV2_NOCOUNT_SHIFT         6
+#define SRS_PDP_PCLK_ODIV2_EDGE_MASK             0x00000080U
+#define SRS_PDP_PCLK_ODIV2_EDGE_SHIFT            7
+#define SRS_PDP_PCLK_ODIV2_FRAC_MASK             0x00007C00U
+#define SRS_PDP_PCLK_ODIV2_FRAC_SHIFT            10
+
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG3           0x61C
+
+#define SRS_PDP_M_CLK_OUT_DIVIDER_REG1           0x628
+#define SRS_PDP_MCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define SRS_PDP_MCLK_ODIV1_LO_TIME_SHIFT         0
+#define SRS_PDP_MCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define SRS_PDP_MCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define SRS_PDP_M_CLK_OUT_DIVIDER_REG2           0x62C
+#define SRS_PDP_MCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define SRS_PDP_MCLK_ODIV2_NOCOUNT_SHIFT         6
+#define SRS_PDP_MCLK_ODIV2_EDGE_MASK             0x00000080U
+#define SRS_PDP_MCLK_ODIV2_EDGE_SHIFT            7
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG1            0x650
+#define SRS_PDP_PCLK_MUL1_LO_TIME_MASK           0x0000003FU
+#define SRS_PDP_PCLK_MUL1_LO_TIME_SHIFT          0
+#define SRS_PDP_PCLK_MUL1_HI_TIME_MASK           0x00000FC0U
+#define SRS_PDP_PCLK_MUL1_HI_TIME_SHIFT          6
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG2            0x654
+#define SRS_PDP_PCLK_MUL2_NOCOUNT_MASK           0x00000040U
+#define SRS_PDP_PCLK_MUL2_NOCOUNT_SHIFT          6
+#define SRS_PDP_PCLK_MUL2_EDGE_MASK              0x00000080U
+#define SRS_PDP_PCLK_MUL2_EDGE_SHIFT             7
+#define SRS_PDP_PCLK_MUL2_FRAC_MASK              0x00007C00U
+#define SRS_PDP_PCLK_MUL2_FRAC_SHIFT             10
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG3            0x64C
+
+#define SRS_PDP_P_CLK_IN_DIVIDER_REG             0x658
+#define SRS_PDP_PCLK_IDIV_LO_TIME_MASK           0x0000003FU
+#define SRS_PDP_PCLK_IDIV_LO_TIME_SHIFT          0
+#define SRS_PDP_PCLK_IDIV_HI_TIME_MASK           0x00000FC0U
+#define SRS_PDP_PCLK_IDIV_HI_TIME_SHIFT          6
+#define SRS_PDP_PCLK_IDIV_NOCOUNT_MASK           0x00001000U
+#define SRS_PDP_PCLK_IDIV_NOCOUNT_SHIFT          12
+#define SRS_PDP_PCLK_IDIV_EDGE_MASK              0x00002000U
+#define SRS_PDP_PCLK_IDIV_EDGE_SHIFT             13
+
+/*
+ * DUT core clock input divider, DUT reference clock input divider
+ */
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1                0x0020
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_MASK   0x00000FC0U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  6
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_MASK   0x0000003FU
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  0
+
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2                0x0024
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_MASK      0x00000080U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_SHIFT     7
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_MASK   0x00000040U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  6
+
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1                 0x0028
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_HI_TIME_MASK    0x00000FC0U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_HI_TIME_SHIFT   6
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_LO_TIME_MASK    0x0000003FU
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_LO_TIME_SHIFT   0
+
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2                 0x002C
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE_MASK       0x00000080U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE_SHIFT      7
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT_MASK    0x00000040U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT   6
+
+/*
+ * DUT interface reference clock input divider
+ */
+
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1                0x0228
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME_MASK   0x00000FC0U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  6
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME_MASK   0x0000003FU
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  0
+
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2                0x022C
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE_MASK      0x00000080U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE_SHIFT     7
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT_MASK   0x00000040U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  6
+
+/*
+ * Min max values from Xilinx Virtex Ultrascale data sheet DS893,
+ * for speed grade 1. All in Hz.
+ */
+#define SRS_INPUT_CLOCK_SPEED                       100000000U
+#define SRS_INPUT_CLOCK_SPEED_MIN                   10000000U
+#define SRS_INPUT_CLOCK_SPEED_MAX                   800000000U
+#define SRS_OUTPUT_CLOCK_SPEED_MIN                  4690000U
+#define SRS_OUTPUT_CLOCK_SPEED_MAX                  630000000U
+#define SRS_VCO_MIN                                 600000000U
+#define SRS_VCO_MAX                                 1200000000U
+#define SRS_PFD_MIN                                 10000000U
+#define SRS_PFD_MAX                                 450000000U
+
+/*
+ * Orion interrupt flags
+ */
+#define SRS_INTERRUPT_ENABLE_PDP1          (1 << SRS_INTERRUPT_ENABLE_PDP_SHIFT)
+#define SRS_INTERRUPT_ENABLE_DUT           (1 << SRS_INTERRUPT_ENABLE_DUT_SHIFT)
+#define SRS_INTERRUPT_STATUS_PDP1          (1 << SRS_INTERRUPT_STATUS_PDP_SHIFT)
+#define SRS_INTERRUPT_STATUS_DUT           (1 << SRS_INTERRUPT_STATUS_DUT_SHIFT)
+#define SRS_INTERRUPT_CLEAR_PDP1           (1 << SRS_INTERRUPT_CLR_PDP_SHIFT)
+#define SRS_INTERRUPT_CLEAR_DUT            (1 << SRS_INTERRUPT_CLR_DUT_SHIFT)
+
+#endif /* _ORION_DEFS_H_ */
+
+/*****************************************************************************
+ End of file (orion_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_regs.h
new file mode 100644
index 000000000000..2a626bd2b70a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_regs.h
@@ -0,0 +1,439 @@
+/******************************************************************************
+@Title          Orion system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Orion FPGA register defs for Sirius RTL
+@Author         Autogenerated
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef _OUT_DRV_H_
+#define _OUT_DRV_H_
+
+/*
+	Register ID
+*/
+#define SRS_CORE_ID                         0x0000
+#define SRS_ID_VARIANT_MASK                     0x0000FFFFU
+#define SRS_ID_VARIANT_SHIFT                    0
+#define SRS_ID_VARIANT_SIGNED                   0
+
+#define SRS_ID_ID_MASK                          0xFFFF0000U
+#define SRS_ID_ID_SHIFT                         16
+#define SRS_ID_ID_SIGNED                        0
+
+/*
+	Register REVISION
+*/
+#define SRS_CORE_REVISION                   0x0004
+#define SRS_REVISION_MINOR_MASK                 0x000000FFU
+#define SRS_REVISION_MINOR_SHIFT                0
+#define SRS_REVISION_MINOR_SIGNED               0
+
+#define SRS_REVISION_MAJOR_MASK                 0x00000F00U
+#define SRS_REVISION_MAJOR_SHIFT                8
+#define SRS_REVISION_MAJOR_SIGNED               0
+
+/*
+	Register CHANGE_SET
+*/
+#define SRS_CORE_CHANGE_SET                 0x0008
+#define SRS_CHANGE_SET_SET_MASK                 0xFFFFFFFFU
+#define SRS_CHANGE_SET_SET_SHIFT                0
+#define SRS_CHANGE_SET_SET_SIGNED               0
+
+/*
+	Register USER_ID
+*/
+#define SRS_CORE_USER_ID                    0x000C
+#define SRS_USER_ID_ID_MASK                     0x0000000FU
+#define SRS_USER_ID_ID_SHIFT                    0
+#define SRS_USER_ID_ID_SIGNED                   0
+
+/*
+	Register USER_BUILD
+*/
+#define SRS_CORE_USER_BUILD                 0x0010
+#define SRS_USER_BUILD_BUILD_MASK               0xFFFFFFFFU
+#define SRS_USER_BUILD_BUILD_SHIFT              0
+#define SRS_USER_BUILD_BUILD_SIGNED             0
+
+/*
+	Register SOFT_RESETN
+*/
+#define SRS_CORE_SOFT_RESETN                0x0080
+#define SRS_SOFT_RESETN_DDR_MASK                0x00000001U
+#define SRS_SOFT_RESETN_DDR_SHIFT               0
+#define SRS_SOFT_RESETN_DDR_SIGNED              0
+
+#define SRS_SOFT_RESETN_USB_MASK                0x00000002U
+#define SRS_SOFT_RESETN_USB_SHIFT               1
+#define SRS_SOFT_RESETN_USB_SIGNED              0
+
+#define SRS_SOFT_RESETN_PDP_MASK                0x00000004U
+#define SRS_SOFT_RESETN_PDP_SHIFT               2
+#define SRS_SOFT_RESETN_PDP_SIGNED              0
+
+#define SRS_SOFT_RESETN_GIST_MASK               0x00000008U
+#define SRS_SOFT_RESETN_GIST_SHIFT              3
+#define SRS_SOFT_RESETN_GIST_SIGNED             0
+
+/*
+	Register DUT_SOFT_RESETN
+*/
+#define SRS_CORE_DUT_SOFT_RESETN            0x0084
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_MASK       0x00000001U
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_SHIFT      0
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_SIGNED     0
+
+/*
+	Register SOFT_AUTO_RESETN
+*/
+#define SRS_CORE_SOFT_AUTO_RESETN           0x0088
+#define SRS_SOFT_AUTO_RESETN_CFG_MASK           0x00000001U
+#define SRS_SOFT_AUTO_RESETN_CFG_SHIFT          0
+#define SRS_SOFT_AUTO_RESETN_CFG_SIGNED         0
+
+/*
+	Register CLK_GEN_RESET
+*/
+#define SRS_CORE_CLK_GEN_RESET              0x0090
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_MASK    0x00000001U
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_SHIFT   0
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_SIGNED  0
+
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_MASK      0x00000002U
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_SHIFT     1
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_SIGNED    0
+
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_MASK       0x00000004U
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_SHIFT      2
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_SIGNED     0
+
+#define SRS_CLK_GEN_RESET_PDP_MMCM_MASK         0x00000008U
+#define SRS_CLK_GEN_RESET_PDP_MMCM_SHIFT        3
+#define SRS_CLK_GEN_RESET_PDP_MMCM_SIGNED       0
+
+/*
+	Register DUT_MEM
+*/
+#define SRS_CORE_DUT_MEM                    0x0120
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_MASK  0x0000FFFFU
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_SHIFT 0
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_SIGNED 0
+
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_MASK 0xFFFF0000U
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_SHIFT 16
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_SIGNED 0
+
+/*
+	Register APM
+*/
+#define SRS_CORE_APM                        0x0150
+#define SRS_APM_RESET_EVENT_MASK                0x00000001U
+#define SRS_APM_RESET_EVENT_SHIFT               0
+#define SRS_APM_RESET_EVENT_SIGNED              0
+
+#define SRS_APM_CAPTURE_EVENT_MASK              0x00000002U
+#define SRS_APM_CAPTURE_EVENT_SHIFT             1
+#define SRS_APM_CAPTURE_EVENT_SIGNED            0
+
+/*
+	Register NUM_GPIO
+*/
+#define SRS_CORE_NUM_GPIO                   0x0180
+#define SRS_NUM_GPIO_NUMBER_MASK                0x0000000FU
+#define SRS_NUM_GPIO_NUMBER_SHIFT               0
+#define SRS_NUM_GPIO_NUMBER_SIGNED              0
+
+/*
+	Register GPIO_EN
+*/
+#define SRS_CORE_GPIO_EN                    0x0184
+#define SRS_GPIO_EN_DIRECTION_MASK              0x000000FFU
+#define SRS_GPIO_EN_DIRECTION_SHIFT             0
+#define SRS_GPIO_EN_DIRECTION_SIGNED            0
+
+/*
+	Register GPIO
+*/
+#define SRS_CORE_GPIO                       0x0188
+#define SRS_GPIO_GPIO_MASK                      0x000000FFU
+#define SRS_GPIO_GPIO_SHIFT                     0
+#define SRS_GPIO_GPIO_SIGNED                    0
+
+/*
+	Register SPI_MASTER_IFACE
+*/
+#define SRS_CORE_SPI_MASTER_IFACE           0x018C
+#define SRS_SPI_MASTER_IFACE_ENABLE_MASK        0x00000001U
+#define SRS_SPI_MASTER_IFACE_ENABLE_SHIFT       0
+#define SRS_SPI_MASTER_IFACE_ENABLE_SIGNED      0
+
+/*
+	Register SRS_IP_STATUS
+*/
+#define SRS_CORE_SRS_IP_STATUS              0x0200
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_MASK 0x00000001U
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_SHIFT 0
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_MASK 0x00000002U
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_SHIFT 1
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_MASK 0x00000004U
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SHIFT 2
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_MASK 0x00000008U
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SHIFT 3
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+/*
+	Register CORE_CONTROL
+*/
+#define SRS_CORE_CORE_CONTROL               0x0204
+#define SRS_CORE_CONTROL_BAR4_OFFSET_MASK       0x0000001FU
+#define SRS_CORE_CONTROL_BAR4_OFFSET_SHIFT      0
+#define SRS_CORE_CONTROL_BAR4_OFFSET_SIGNED     0
+
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_MASK 0x00000300U
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SHIFT 8
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SIGNED 0
+
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_MASK    0x00001C00U
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_SHIFT   10
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_SIGNED  0
+
+/*
+	Register REG_BANK_STATUS
+*/
+#define SRS_CORE_REG_BANK_STATUS            0x0208
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_MASK 0xFFFFFFFFU
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SHIFT 0
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SIGNED 0
+
+/*
+	Register MMCM_LOCK_STATUS
+*/
+#define SRS_CORE_MMCM_LOCK_STATUS           0x020C
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_MASK      0x00000001U
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_SHIFT     0
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_SIGNED    0
+
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_MASK        0x00000002U
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_SHIFT       1
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_SIGNED      0
+
+#define SRS_MMCM_LOCK_STATUS_MULTI_MASK         0x00000004U
+#define SRS_MMCM_LOCK_STATUS_MULTI_SHIFT        2
+#define SRS_MMCM_LOCK_STATUS_MULTI_SIGNED       0
+
+#define SRS_MMCM_LOCK_STATUS_PDP_MASK           0x00000008U
+#define SRS_MMCM_LOCK_STATUS_PDP_SHIFT          3
+#define SRS_MMCM_LOCK_STATUS_PDP_SIGNED         0
+
+/*
+	Register GIST_STATUS
+*/
+#define SRS_CORE_GIST_STATUS                0x0210
+#define SRS_GIST_STATUS_MST_MASK                0x000001FFU
+#define SRS_GIST_STATUS_MST_SHIFT               0
+#define SRS_GIST_STATUS_MST_SIGNED              0
+
+#define SRS_GIST_STATUS_SLV_MASK                0x001FF000U
+#define SRS_GIST_STATUS_SLV_SHIFT               12
+#define SRS_GIST_STATUS_SLV_SIGNED              0
+
+#define SRS_GIST_STATUS_SLV_OUT_MASK            0x03000000U
+#define SRS_GIST_STATUS_SLV_OUT_SHIFT           24
+#define SRS_GIST_STATUS_SLV_OUT_SIGNED          0
+
+#define SRS_GIST_STATUS_MST_OUT_MASK            0x70000000U
+#define SRS_GIST_STATUS_MST_OUT_SHIFT           28
+#define SRS_GIST_STATUS_MST_OUT_SIGNED          0
+
+/*
+	Register SENSOR_BOARD
+*/
+#define SRS_CORE_SENSOR_BOARD               0x0214
+#define SRS_SENSOR_BOARD_ID_MASK                0x00000003U
+#define SRS_SENSOR_BOARD_ID_SHIFT               0
+#define SRS_SENSOR_BOARD_ID_SIGNED              0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define SRS_CORE_INTERRUPT_STATUS           0x0218
+#define SRS_INTERRUPT_STATUS_DUT_MASK           0x00000001U
+#define SRS_INTERRUPT_STATUS_DUT_SHIFT          0
+#define SRS_INTERRUPT_STATUS_DUT_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_PDP_MASK           0x00000002U
+#define SRS_INTERRUPT_STATUS_PDP_SHIFT          1
+#define SRS_INTERRUPT_STATUS_PDP_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_I2C_MASK           0x00000004U
+#define SRS_INTERRUPT_STATUS_I2C_SHIFT          2
+#define SRS_INTERRUPT_STATUS_I2C_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_SPI_MASK           0x00000008U
+#define SRS_INTERRUPT_STATUS_SPI_SHIFT          3
+#define SRS_INTERRUPT_STATUS_SPI_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_APM_MASK           0x00000010U
+#define SRS_INTERRUPT_STATUS_APM_SHIFT          4
+#define SRS_INTERRUPT_STATUS_APM_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_OS_IRQ_MASK        0x00001FE0U
+#define SRS_INTERRUPT_STATUS_OS_IRQ_SHIFT       5
+#define SRS_INTERRUPT_STATUS_OS_IRQ_SIGNED      0
+
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_MASK      0x40000000U
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_SHIFT     30
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_SIGNED    0
+
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_MASK 0x80000000U
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_SHIFT 31
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_SIGNED 0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define SRS_CORE_INTERRUPT_ENABLE           0x021C
+#define SRS_INTERRUPT_ENABLE_DUT_MASK           0x00000001U
+#define SRS_INTERRUPT_ENABLE_DUT_SHIFT          0
+#define SRS_INTERRUPT_ENABLE_DUT_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_PDP_MASK           0x00000002U
+#define SRS_INTERRUPT_ENABLE_PDP_SHIFT          1
+#define SRS_INTERRUPT_ENABLE_PDP_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_I2C_MASK           0x00000004U
+#define SRS_INTERRUPT_ENABLE_I2C_SHIFT          2
+#define SRS_INTERRUPT_ENABLE_I2C_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_SPI_MASK           0x00000008U
+#define SRS_INTERRUPT_ENABLE_SPI_SHIFT          3
+#define SRS_INTERRUPT_ENABLE_SPI_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_APM_MASK           0x00000010U
+#define SRS_INTERRUPT_ENABLE_APM_SHIFT          4
+#define SRS_INTERRUPT_ENABLE_APM_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_MASK        0x00001FE0U
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_SHIFT       5
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_SIGNED      0
+
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_MASK      0x40000000U
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_SHIFT     30
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_SIGNED    0
+
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_MASK 0x80000000U
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_SHIFT 31
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_SIGNED 0
+
+/*
+	Register INTERRUPT_CLR
+*/
+#define SRS_CORE_INTERRUPT_CLR              0x0220
+#define SRS_INTERRUPT_CLR_DUT_MASK              0x00000001U
+#define SRS_INTERRUPT_CLR_DUT_SHIFT             0
+#define SRS_INTERRUPT_CLR_DUT_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_PDP_MASK              0x00000002U
+#define SRS_INTERRUPT_CLR_PDP_SHIFT             1
+#define SRS_INTERRUPT_CLR_PDP_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_I2C_MASK              0x00000004U
+#define SRS_INTERRUPT_CLR_I2C_SHIFT             2
+#define SRS_INTERRUPT_CLR_I2C_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_SPI_MASK              0x00000008U
+#define SRS_INTERRUPT_CLR_SPI_SHIFT             3
+#define SRS_INTERRUPT_CLR_SPI_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_APM_MASK              0x00000010U
+#define SRS_INTERRUPT_CLR_APM_SHIFT             4
+#define SRS_INTERRUPT_CLR_APM_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_OS_IRQ_MASK           0x00001FE0U
+#define SRS_INTERRUPT_CLR_OS_IRQ_SHIFT          5
+#define SRS_INTERRUPT_CLR_OS_IRQ_SIGNED         0
+
+#define SRS_INTERRUPT_CLR_IRQ_TEST_MASK         0x40000000U
+#define SRS_INTERRUPT_CLR_IRQ_TEST_SHIFT        30
+#define SRS_INTERRUPT_CLR_IRQ_TEST_SIGNED       0
+
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_MASK     0x80000000U
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_SHIFT    31
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_SIGNED   0
+
+/*
+	Register INTERRUPT_TEST
+*/
+#define SRS_CORE_INTERRUPT_TEST             0x0224
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_MASK  0x00000001U
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_SHIFT 0
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT_CLR
+*/
+#define SRS_CORE_INTERRUPT_TIMEOUT_CLR      0x0228
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK 0x00000002U
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SHIFT 1
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SIGNED 0
+
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_MASK 0x00000001U
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SHIFT 0
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT
+*/
+#define SRS_CORE_INTERRUPT_TIMEOUT          0x022C
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_MASK 0xFFFFFFFFU
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SHIFT 0
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SIGNED 0
+
+#endif /* _OUT_DRV_H_ */
+
+/******************************************************************************
+ End of file (orion_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pdp_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pdp_regs.h
new file mode 100644
index 000000000000..bd26b0617f95
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pdp_regs.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__PDP_REGS_H__)
+#define __PDP_REGS_H__
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+#define DCPDP_VENDOR_ID_POWERVR			(0x1010)
+
+#define DCPDP_DEVICE_ID_PCI_APOLLO_FPGA		(0x1CF1)
+#define DCPDP_DEVICE_ID_PCIE_APOLLO_FPGA	(0x1CF2)
+
+/*************************************************************************/ /*!
+ PCI Device Base Address Information
+*/ /**************************************************************************/
+
+/* PLL and PDP registers on base address register 0 */
+#define DCPDP_REG_PCI_BASENUM			(0)
+
+#define DCPDP_PCI_PLL_REG_OFFSET		(0x1000)
+#define DCPDP_PCI_PLL_REG_SIZE			(0x0400)
+
+#define DCPDP_PCI_PDP_REG_OFFSET		(0xC000)
+#define DCPDP_PCI_PDP_REG_SIZE			(0x2000)
+
+/*************************************************************************/ /*!
+ Misc register information
+*/ /**************************************************************************/
+
+/* This information isn't captured in tcf_rgbpdp_regs.h so define it here */
+#define DCPDP_STR1SURF_FORMAT_ARGB8888		(0xE)
+#define DCPDP_STR1ADDRCTRL_BASE_ADDR_SHIFT	(4)
+#define DCPDP_STR1POSN_STRIDE_SHIFT		(4)
+
+#endif /* !defined(__PDP_REGS_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_defs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_defs.h
new file mode 100644
index 000000000000..d39c06f703bd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_defs.h
@@ -0,0 +1,69 @@
+/******************************************************************************
+@Title          Odin PFIM definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin register defs for PDP-FBDC Interface Module
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef _PFIM_DEFS_H_
+#define _PFIM_DEFS_H_
+
+/* Supported FBC modes */
+#define ODIN_PFIM_MOD_LINEAR       (0x00)
+#define ODIN_PFIM_FBCDC_8X8_V12    (0x01)
+#define ODIN_PFIM_FBCDC_16X4_V12   (0x02)
+#define ODIN_PFIM_FBCDC_MAX        (0x03)
+
+/* Supported pixel formats */
+#define ODN_PFIM_PIXFMT_NONE       (0x00)
+#define ODN_PFIM_PIXFMT_ARGB8888   (0x0C)
+#define ODN_PFIM_PIXFMT_RGB565     (0x05)
+
+/* Tile types */
+#define ODN_PFIM_TILETYPE_8X8      (0x01)
+#define ODN_PFIM_TILETYPE_16X4     (0x02)
+#define ODN_PFIM_TILETYPE_32x2     (0x03)
+
+#define PFIM_ROUNDUP(X, Y)         (((X) + ((Y) - 1U)) & ~((Y) - 1U))
+#define PFIM_RND_TAG               (0x10)
+
+#endif /* _PFIM_DEFS_H_ */
+
+/******************************************************************************
+ End of file (pfim_defs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_regs.h
new file mode 100644
index 000000000000..4b8ff82138b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_regs.h
@@ -0,0 +1,265 @@
+/******************************************************************************
+@Title          Odin PFIM control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin register defs for PDP-FBDC Interface Module
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+#ifndef _PFIM_REGS_H_
+#define _PFIM_REGS_H_
+
+/*
+	Register CR_PFIM_NUM_TILES
+*/
+#define CR_PFIM_NUM_TILES 0x0000
+#define CR_PFIM_NUM_TILES_MASK 0x007FFFFFU
+#define CR_PFIM_NUM_TILES_SHIFT 0
+#define CR_PFIM_NUM_TILES_SIGNED 0
+
+/*
+	Register CR_PFIM_TILES_PER_LINE
+*/
+#define CR_PFIM_TILES_PER_LINE 0x0004
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_MASK 0x000000FFU
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_SHIFT 0
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB
+*/
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB 0x0008
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB
+*/
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB 0x000C
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_MASK 0x00000003U
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_UV_BASE_ADDR_LSB
+*/
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB 0x0010
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_UV_BASE_ADDR_MSB
+*/
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB 0x0014
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_MASK 0x00000003U
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_PDP_Y_BASE_ADDR
+*/
+#define CR_PFIM_PDP_Y_BASE_ADDR 0x0018
+#define CR_PFIM_PDP_Y_BASE_ADDR_MASK 0xFFFFFFFFU
+#define CR_PFIM_PDP_Y_BASE_ADDR_SHIFT 0
+#define CR_PFIM_PDP_Y_BASE_ADDR_SIGNED 0
+
+/*
+	Register CR_PFIM_PDP_UV_BASE_ADDR
+*/
+#define CR_PFIM_PDP_UV_BASE_ADDR 0x001C
+#define CR_PFIM_PDP_UV_BASE_ADDR_MASK 0xFFFFFFFFU
+#define CR_PFIM_PDP_UV_BASE_ADDR_SHIFT 0
+#define CR_PFIM_PDP_UV_BASE_ADDR_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_CONTEXT
+*/
+#define CR_PFIM_FBDC_REQ_CONTEXT 0x0020
+#define CR_PFIM_FBDC_REQ_CONTEXT_MASK 0x00000007U
+#define CR_PFIM_FBDC_REQ_CONTEXT_SHIFT 0
+#define CR_PFIM_FBDC_REQ_CONTEXT_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_TAG
+*/
+#define CR_PFIM_FBDC_REQ_TAG 0x0024
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_MASK     0x00000003U
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_SHIFT    0
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_SIGNED   0
+
+#define CR_PFIM_FBDC_REQ_TAG_UV_MASK        0x00000030U
+#define CR_PFIM_FBDC_REQ_TAG_UV_SHIFT       4
+#define CR_PFIM_FBDC_REQ_TAG_UV_SIGNED      0
+
+/*
+	Register CR_PFIM_FBDC_REQ_SB_TAG
+*/
+#define CR_PFIM_FBDC_REQ_SB_TAG 0x0028
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_MASK  0x00000003U
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_SHIFT 0
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_SIGNED 0
+
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_MASK     0x00000030U
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_SHIFT    4
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_SIGNED   0
+
+/*
+	Register CR_PFIM_FBDC_HDR_INVAL_REQ
+*/
+#define CR_PFIM_FBDC_HDR_INVAL_REQ 0x002C
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_MASK 0x00000001U
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_SHIFT 0
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_PIX_FORMAT
+*/
+#define CR_PFIM_FBDC_PIX_FORMAT 0x0030
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_MASK 0x0000007FU
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_SHIFT 0
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_CH0123_VAL0
+*/
+#define CR_PFIM_FBDC_CR_CH0123_VAL0 0x0034
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_CH0123_VAL1
+*/
+#define CR_PFIM_FBDC_CR_CH0123_VAL1 0x0038
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_Y_VAL0
+*/
+#define CR_PFIM_FBDC_CR_Y_VAL0 0x003C
+#define CR_PFIM_FBDC_CR_Y_VAL0_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_Y_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_Y_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_UV_VAL0
+*/
+#define CR_PFIM_FBDC_CR_UV_VAL0 0x0040
+#define CR_PFIM_FBDC_CR_UV_VAL0_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_UV_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_UV_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_Y_VAL1
+*/
+#define CR_PFIM_FBDC_CR_Y_VAL1 0x0044
+#define CR_PFIM_FBDC_CR_Y_VAL1_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_Y_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_Y_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_UV_VAL1
+*/
+#define CR_PFIM_FBDC_CR_UV_VAL1 0x0048
+#define CR_PFIM_FBDC_CR_UV_VAL1_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_UV_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_UV_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_ENABLE
+*/
+#define CR_PFIM_FBDC_FILTER_ENABLE 0x004C
+#define CR_PFIM_FBDC_FILTER_ENABLE_MASK 0x00000001U
+#define CR_PFIM_FBDC_FILTER_ENABLE_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_ENABLE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_STATUS
+*/
+#define CR_PFIM_FBDC_FILTER_STATUS 0x0050
+#define CR_PFIM_FBDC_FILTER_STATUS_MASK 0x0000000FU
+#define CR_PFIM_FBDC_FILTER_STATUS_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_STATUS_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_CLEAR
+*/
+#define CR_PFIM_FBDC_FILTER_CLEAR 0x0054
+#define CR_PFIM_FBDC_FILTER_CLEAR_MASK 0x0000000FU
+#define CR_PFIM_FBDC_FILTER_CLEAR_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_CLEAR_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_TILE_TYPE
+*/
+#define CR_PFIM_FBDC_TILE_TYPE 0x0058
+#define CR_PFIM_FBDC_TILE_TYPE_MASK 0x00000003U
+#define CR_PFIM_FBDC_TILE_TYPE_SHIFT 0
+#define CR_PFIM_FBDC_TILE_TYPE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CLEAR_COLOUR_LSB
+*/
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB 0x005C
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CLEAR_COLOUR_MSB
+*/
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB 0x0060
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_LOSSY
+*/
+#define CR_PFIM_FBDC_REQ_LOSSY 0x0064
+#define CR_PFIM_FBDC_REQ_LOSSY_MASK 0x00000001U
+#define CR_PFIM_FBDC_REQ_LOSSY_SHIFT 0
+#define CR_PFIM_FBDC_REQ_LOSSY_SIGNED 0
+
+#endif /* _PFIM_REGS_H_ */
+
+/******************************************************************************
+ End of file (pfim_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_clk_ctrl.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_clk_ctrl.h
new file mode 100644
index 000000000000..cc7b10fd8116
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_clk_ctrl.h
@@ -0,0 +1,1018 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from: tcf_clk_ctrl.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_CLK_CTRL_H_)
+#define _TCF_CLK_CTRL_H_
+
+/*
+ * The following register definitions are valid if register 0x28 has value 0.
+ */
+
+/*
+	Register FPGA_ID_REG
+*/
+#define TCF_CLK_CTRL_FPGA_ID_REG            0x0000
+#define FPGA_ID_REG_CORE_CFG_MASK           0x0000FFFFU
+#define FPGA_ID_REG_CORE_CFG_SHIFT          0
+#define FPGA_ID_REG_CORE_CFG_SIGNED         0
+
+#define FPGA_ID_REG_CORE_ID_MASK            0xFFFF0000U
+#define FPGA_ID_REG_CORE_ID_SHIFT           16
+#define FPGA_ID_REG_CORE_ID_SIGNED          0
+
+/*
+	Register FPGA_REV_REG
+*/
+#define TCF_CLK_CTRL_FPGA_REV_REG           0x0008
+#define FPGA_REV_REG_MAINT_MASK             0x000000FFU
+#define FPGA_REV_REG_MAINT_SHIFT            0
+#define FPGA_REV_REG_MAINT_SIGNED           0
+
+#define FPGA_REV_REG_MINOR_MASK             0x0000FF00U
+#define FPGA_REV_REG_MINOR_SHIFT            8
+#define FPGA_REV_REG_MINOR_SIGNED           0
+
+#define FPGA_REV_REG_MAJOR_MASK             0x00FF0000U
+#define FPGA_REV_REG_MAJOR_SHIFT            16
+#define FPGA_REV_REG_MAJOR_SIGNED           0
+
+#define FPGA_REV_REG_DESIGNER_MASK          0xFF000000U
+#define FPGA_REV_REG_DESIGNER_SHIFT         24
+#define FPGA_REV_REG_DESIGNER_SIGNED        0
+
+/*
+	Register FPGA_DES_REV_1
+*/
+#define TCF_CLK_CTRL_FPGA_DES_REV_1         0x0010
+#define FPGA_DES_REV_1_MASK                 0xFFFFFFFFU
+#define FPGA_DES_REV_1_SHIFT                0
+#define FPGA_DES_REV_1_SIGNED               0
+
+/*
+	Register FPGA_DES_REV_2
+*/
+#define TCF_CLK_CTRL_FPGA_DES_REV_2         0x0018
+#define FPGA_DES_REV_2_MASK                 0xFFFFFFFFU
+#define FPGA_DES_REV_2_SHIFT                0
+#define FPGA_DES_REV_2_SIGNED               0
+
+/*
+	Register TCF_CORE_ID_REG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_ID_REG        0x0020
+#define TCF_CORE_ID_REG_CORE_CFG_MASK       0x0000FFFFU
+#define TCF_CORE_ID_REG_CORE_CFG_SHIFT      0
+#define TCF_CORE_ID_REG_CORE_CFG_SIGNED     0
+
+#define TCF_CORE_ID_REG_CORE_ID_MASK        0xFFFF0000U
+#define TCF_CORE_ID_REG_CORE_ID_SHIFT       16
+#define TCF_CORE_ID_REG_CORE_ID_SIGNED      0
+
+/*
+	Register TCF_CORE_REV_REG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_REV_REG       0x0028
+#define TCF_CORE_REV_REG_MAINT_MASK         0x000000FFU
+#define TCF_CORE_REV_REG_MAINT_SHIFT        0
+#define TCF_CORE_REV_REG_MAINT_SIGNED       0
+
+#define TCF_CORE_REV_REG_MINOR_MASK         0x0000FF00U
+#define TCF_CORE_REV_REG_MINOR_SHIFT        8
+#define TCF_CORE_REV_REG_MINOR_SIGNED       0
+
+#define TCF_CORE_REV_REG_MAJOR_MASK         0x00FF0000U
+#define TCF_CORE_REV_REG_MAJOR_SHIFT        16
+#define TCF_CORE_REV_REG_MAJOR_SIGNED       0
+
+#define TCF_CORE_REV_REG_DESIGNER_MASK      0xFF000000U
+#define TCF_CORE_REV_REG_DESIGNER_SHIFT     24
+#define TCF_CORE_REV_REG_DESIGNER_SIGNED    0
+
+/*
+	Register TCF_CORE_DES_REV_1
+*/
+#define TCF_CLK_CTRL_TCF_CORE_DES_REV_1     0x0030
+#define TCF_CORE_DES_REV_1_MASK             0xFFFFFFFFU
+#define TCF_CORE_DES_REV_1_SHIFT            0
+#define TCF_CORE_DES_REV_1_SIGNED           0
+
+/*
+	Register TCF_CORE_DES_REV_2
+*/
+#define TCF_CLK_CTRL_TCF_CORE_DES_REV_2     0x0038
+#define TCF_CORE_DES_REV_2_MASK             0xFFFFFFFFU
+#define TCF_CORE_DES_REV_2_SHIFT            0
+#define TCF_CORE_DES_REV_2_SIGNED           0
+
+
+/*
+ * The following register definitions are valid if register 0x28 has value 1.
+ */
+
+/*
+	Register ID
+*/
+#define TCF_CLK_CTRL_ID                     0x0000
+#define VARIANT_MASK                        0x0000FFFFU
+#define VARIANT_SHIFT                       0
+#define VARIANT_SIGNED                      0
+
+#define ID_MASK                             0xFFFF0000U
+#define ID_SHIFT                            16
+#define ID_SIGNED                           0
+
+/*
+	Register REL
+*/
+#define TCF_CLK_CTRL_REL                    0x0008
+#define MINOR_MASK                          0x0000FFFFU
+#define MINOR_SHIFT                         0
+#define MINOR_SIGNED                        0
+
+#define MAJOR_MASK                          0xFFFF0000U
+#define MAJOR_SHIFT                         16
+#define MAJOR_SIGNED                        0
+
+/*
+	Register CHANGE_SET
+*/
+#define TCF_CLK_CTRL_CHANGE_SET             0x0010
+#define SET_MASK                            0xFFFFFFFFU
+#define SET_SHIFT                           0
+#define SET_SIGNED                          0
+
+/*
+	Register USER_ID
+*/
+#define TCF_CLK_CTRL_USER_ID                0x0018
+#define USER_ID_MASK                        0x0000000FU
+#define USER_ID_SHIFT                       0
+#define USER_ID_SIGNED                      0
+
+/*
+	Register USER_BUILD
+*/
+#define TCF_CLK_CTRL_USER_BUILD             0x0020
+#define BUILD_MASK                          0xFFFFFFFFU
+#define BUILD_SHIFT                         0
+#define BUILD_SIGNED                        0
+
+/*
+	Register SW_IF_VERSION
+*/
+#define TCF_CLK_CTRL_SW_IF_VERSION          0x0028
+#define VERSION_MASK                        0x0000FFFFU
+#define VERSION_SHIFT                       0
+#define VERSION_SIGNED                      0
+
+/*
+ * The following register definitions are valid for all Apollo builds,
+ * even if some of the registers are not available for certain cores.
+ */
+
+/*
+	Register SCB_GENERAL_CONTROL
+*/
+#define TCF_CLK_CTRL_SCB_GENERAL_CONTROL    0x0040
+#define SCB_GC_TRANS_HALT_MASK              0x00000200U
+#define SCB_GC_TRANS_HALT_SHIFT             9
+#define SCB_GC_TRANS_HALT_SIGNED            0
+
+#define SCB_GC_CKD_REGS_MASK                0x00000100U
+#define SCB_GC_CKD_REGS_SHIFT               8
+#define SCB_GC_CKD_REGS_SIGNED              0
+
+#define SCB_GC_CKD_SLAVE_MASK               0x00000080U
+#define SCB_GC_CKD_SLAVE_SHIFT              7
+#define SCB_GC_CKD_SLAVE_SIGNED             0
+
+#define SCB_GC_CKD_MASTER_MASK              0x00000040U
+#define SCB_GC_CKD_MASTER_SHIFT             6
+#define SCB_GC_CKD_MASTER_SIGNED            0
+
+#define SCB_GC_CKD_XDATA_MASK               0x00000020U
+#define SCB_GC_CKD_XDATA_SHIFT              5
+#define SCB_GC_CKD_XDATA_SIGNED             0
+
+#define SCB_GC_SFR_REG_MASK                 0x00000010U
+#define SCB_GC_SFR_REG_SHIFT                4
+#define SCB_GC_SFR_REG_SIGNED               0
+
+#define SCB_GC_SFR_SLAVE_MASK               0x00000008U
+#define SCB_GC_SFR_SLAVE_SHIFT              3
+#define SCB_GC_SFR_SLAVE_SIGNED             0
+
+#define SCB_GC_SFR_MASTER_MASK              0x00000004U
+#define SCB_GC_SFR_MASTER_SHIFT             2
+#define SCB_GC_SFR_MASTER_SIGNED            0
+
+#define SCB_GC_SFR_DET_DATA_MASK            0x00000002U
+#define SCB_GC_SFR_DET_DATA_SHIFT           1
+#define SCB_GC_SFR_DET_DATA_SIGNED          0
+
+#define SCB_GC_SFR_GEN_DATA_MASK            0x00000001U
+#define SCB_GC_SFR_GEN_DATA_SHIFT           0
+#define SCB_GC_SFR_GEN_DATA_SIGNED          0
+
+/*
+	Register SCB_MASTER_READ_COUNT
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_READ_COUNT  0x0048
+#define MASTER_READ_COUNT_MASK              0x0000FFFFU
+#define MASTER_READ_COUNT_SHIFT             0
+#define MASTER_READ_COUNT_SIGNED            0
+
+/*
+	Register SCB_MASTER_READ_DATA
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_READ_DATA   0x0050
+#define MASTER_READ_DATA_MASK               0x000000FFU
+#define MASTER_READ_DATA_SHIFT              0
+#define MASTER_READ_DATA_SIGNED             0
+
+/*
+	Register SCB_MASTER_ADDRESS
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_ADDRESS     0x0058
+#define SCB_MASTER_ADDRESS_MASK             0x000003FFU
+#define SCB_MASTER_ADDRESS_SHIFT            0
+#define SCB_MASTER_ADDRESS_SIGNED           0
+
+/*
+	Register SCB_MASTER_WRITE_DATA
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_WRITE_DATA  0x0060
+#define MASTER_WRITE_DATA_MASK              0x000000FFU
+#define MASTER_WRITE_DATA_SHIFT             0
+#define MASTER_WRITE_DATA_SIGNED            0
+
+/*
+	Register SCB_MASTER_WRITE_COUNT
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_WRITE_COUNT 0x0068
+#define MASTER_WRITE_COUNT_MASK             0x0000FFFFU
+#define MASTER_WRITE_COUNT_SHIFT            0
+#define MASTER_WRITE_COUNT_SIGNED           0
+
+/*
+	Register SCB_BUS_SELECT
+*/
+#define TCF_CLK_CTRL_SCB_BUS_SELECT         0x0070
+#define BUS_SELECT_MASK                     0x00000003U
+#define BUS_SELECT_SHIFT                    0
+#define BUS_SELECT_SIGNED                   0
+
+/*
+	Register SCB_MASTER_FILL_STATUS
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_FILL_STATUS 0x0078
+#define MASTER_WRITE_FIFO_EMPTY_MASK        0x00000008U
+#define MASTER_WRITE_FIFO_EMPTY_SHIFT       3
+#define MASTER_WRITE_FIFO_EMPTY_SIGNED      0
+
+#define MASTER_WRITE_FIFO_FULL_MASK         0x00000004U
+#define MASTER_WRITE_FIFO_FULL_SHIFT        2
+#define MASTER_WRITE_FIFO_FULL_SIGNED       0
+
+#define MASTER_READ_FIFO_EMPTY_MASK         0x00000002U
+#define MASTER_READ_FIFO_EMPTY_SHIFT        1
+#define MASTER_READ_FIFO_EMPTY_SIGNED       0
+
+#define MASTER_READ_FIFO_FULL_MASK          0x00000001U
+#define MASTER_READ_FIFO_FULL_SHIFT         0
+#define MASTER_READ_FIFO_FULL_SIGNED        0
+
+/*
+	Register CLK_AND_RST_CTRL
+*/
+#define TCF_CLK_CTRL_CLK_AND_RST_CTRL       0x0080
+#define GLB_CLKG_EN_MASK                    0x00020000U
+#define GLB_CLKG_EN_SHIFT                   17
+#define GLB_CLKG_EN_SIGNED                  0
+
+#define CLK_GATE_CNTL_MASK                  0x00010000U
+#define CLK_GATE_CNTL_SHIFT                 16
+#define CLK_GATE_CNTL_SIGNED                0
+
+#define DUT_DCM_RESETN_MASK                 0x00000400U
+#define DUT_DCM_RESETN_SHIFT                10
+#define DUT_DCM_RESETN_SIGNED               0
+
+#define MEM_RESYNC_BYPASS_MASK              0x00000200U
+#define MEM_RESYNC_BYPASS_SHIFT             9
+#define MEM_RESYNC_BYPASS_SIGNED            0
+
+#define SYS_RESYNC_BYPASS_MASK              0x00000100U
+#define SYS_RESYNC_BYPASS_SHIFT             8
+#define SYS_RESYNC_BYPASS_SIGNED            0
+
+#define SCB_RESETN_MASK                     0x00000010U
+#define SCB_RESETN_SHIFT                    4
+#define SCB_RESETN_SIGNED                   0
+
+#define PDP2_RESETN_MASK                    0x00000008U
+#define PDP2_RESETN_SHIFT                   3
+#define PDP2_RESETN_SIGNED                  0
+
+#define PDP1_RESETN_MASK                    0x00000004U
+#define PDP1_RESETN_SHIFT                   2
+#define PDP1_RESETN_SIGNED                  0
+
+#define DDR_RESETN_MASK                     0x00000002U
+#define DDR_RESETN_SHIFT                    1
+#define DDR_RESETN_SIGNED                   0
+
+#define DUT_RESETN_MASK                     0x00000001U
+#define DUT_RESETN_SHIFT                    0
+#define DUT_RESETN_SIGNED                   0
+
+/*
+	Register TEST_REG_OUT
+*/
+#define TCF_CLK_CTRL_TEST_REG_OUT           0x0088
+#define TEST_REG_OUT_MASK                   0xFFFFFFFFU
+#define TEST_REG_OUT_SHIFT                  0
+#define TEST_REG_OUT_SIGNED                 0
+
+/*
+	Register TEST_REG_IN
+*/
+#define TCF_CLK_CTRL_TEST_REG_IN            0x0090
+#define TEST_REG_IN_MASK                    0xFFFFFFFFU
+#define TEST_REG_IN_SHIFT                   0
+#define TEST_REG_IN_SIGNED                  0
+
+/*
+	Register TEST_CTRL
+*/
+#define TCF_CLK_CTRL_TEST_CTRL              0x0098
+#define PCI_TEST_OFFSET_MASK                0xF8000000U
+#define PCI_TEST_OFFSET_SHIFT               27
+#define PCI_TEST_OFFSET_SIGNED              0
+
+#define PDP1_HOST_MEM_SELECT_MASK           0x00000200U
+#define PDP1_HOST_MEM_SELECT_SHIFT          9
+#define PDP1_HOST_MEM_SELECT_SIGNED         0
+
+#define HOST_PHY_MODE_MASK                  0x00000100U
+#define HOST_PHY_MODE_SHIFT                 8
+#define HOST_PHY_MODE_SIGNED                0
+
+#define HOST_ONLY_MODE_MASK                 0x00000080U
+#define HOST_ONLY_MODE_SHIFT                7
+#define HOST_ONLY_MODE_SIGNED               0
+
+#define PCI_TEST_MODE_MASK                  0x00000040U
+#define PCI_TEST_MODE_SHIFT                 6
+#define PCI_TEST_MODE_SIGNED                0
+
+#define TURN_OFF_DDR_MASK                   0x00000020U
+#define TURN_OFF_DDR_SHIFT                  5
+#define TURN_OFF_DDR_SIGNED                 0
+
+#define SYS_RD_CLK_INV_MASK                 0x00000010U
+#define SYS_RD_CLK_INV_SHIFT                4
+#define SYS_RD_CLK_INV_SIGNED               0
+
+#define MEM_REQ_CLK_INV_MASK                0x00000008U
+#define MEM_REQ_CLK_INV_SHIFT               3
+#define MEM_REQ_CLK_INV_SIGNED              0
+
+#define BURST_SPLIT_MASK                    0x00000004U
+#define BURST_SPLIT_SHIFT                   2
+#define BURST_SPLIT_SIGNED                  0
+
+#define CLK_INVERSION_MASK                  0x00000002U
+#define CLK_INVERSION_SHIFT                 1
+#define CLK_INVERSION_SIGNED                0
+
+#define ADDRESS_FORCE_MASK                  0x00000001U
+#define ADDRESS_FORCE_SHIFT                 0
+#define ADDRESS_FORCE_SIGNED                0
+
+/*
+	Register CLEAR_HOST_MEM_SIG
+*/
+#define TCF_CLK_CTRL_CLEAR_HOST_MEM_SIG     0x00A0
+#define SIGNATURE_TAG_ID_MASK               0x00000F00U
+#define SIGNATURE_TAG_ID_SHIFT              8
+#define SIGNATURE_TAG_ID_SIGNED             0
+
+#define CLEAR_HOST_MEM_SIGNATURE_MASK       0x00000001U
+#define CLEAR_HOST_MEM_SIGNATURE_SHIFT      0
+#define CLEAR_HOST_MEM_SIGNATURE_SIGNED     0
+
+/*
+	Register HOST_MEM_SIGNATURE
+*/
+#define TCF_CLK_CTRL_HOST_MEM_SIGNATURE     0x00A8
+#define HOST_MEM_SIGNATURE_MASK             0xFFFFFFFFU
+#define HOST_MEM_SIGNATURE_SHIFT            0
+#define HOST_MEM_SIGNATURE_SIGNED           0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define TCF_CLK_CTRL_INTERRUPT_STATUS       0x00C8
+#define INTERRUPT_MASTER_STATUS_MASK        0x80000000U
+#define INTERRUPT_MASTER_STATUS_SHIFT       31
+#define INTERRUPT_MASTER_STATUS_SIGNED      0
+
+#define OTHER_INTS_MASK                     0x7FFE0000U
+#define OTHER_INTS_SHIFT                    17
+#define OTHER_INTS_SIGNED                   0
+
+#define HOST_MST_NORESPONSE_MASK            0x00010000U
+#define HOST_MST_NORESPONSE_SHIFT           16
+#define HOST_MST_NORESPONSE_SIGNED          0
+
+#define PDP2_INT_MASK                       0x00008000U
+#define PDP2_INT_SHIFT                      15
+#define PDP2_INT_SIGNED                     0
+
+#define PDP1_INT_MASK                       0x00004000U
+#define PDP1_INT_SHIFT                      14
+#define PDP1_INT_SIGNED                     0
+
+#define EXT_INT_MASK                        0x00002000U
+#define EXT_INT_SHIFT                       13
+#define EXT_INT_SIGNED                      0
+
+#define SCB_MST_HLT_BIT_MASK                0x00001000U
+#define SCB_MST_HLT_BIT_SHIFT               12
+#define SCB_MST_HLT_BIT_SIGNED              0
+
+#define SCB_SLV_EVENT_MASK                  0x00000800U
+#define SCB_SLV_EVENT_SHIFT                 11
+#define SCB_SLV_EVENT_SIGNED                0
+
+#define SCB_TDONE_RX_MASK                   0x00000400U
+#define SCB_TDONE_RX_SHIFT                  10
+#define SCB_TDONE_RX_SIGNED                 0
+
+#define SCB_SLV_WT_RD_DAT_MASK              0x00000200U
+#define SCB_SLV_WT_RD_DAT_SHIFT             9
+#define SCB_SLV_WT_RD_DAT_SIGNED            0
+
+#define SCB_SLV_WT_PRV_RD_MASK              0x00000100U
+#define SCB_SLV_WT_PRV_RD_SHIFT             8
+#define SCB_SLV_WT_PRV_RD_SIGNED            0
+
+#define SCB_SLV_WT_WR_DAT_MASK              0x00000080U
+#define SCB_SLV_WT_WR_DAT_SHIFT             7
+#define SCB_SLV_WT_WR_DAT_SIGNED            0
+
+#define SCB_MST_WT_RD_DAT_MASK              0x00000040U
+#define SCB_MST_WT_RD_DAT_SHIFT             6
+#define SCB_MST_WT_RD_DAT_SIGNED            0
+
+#define SCB_ADD_ACK_ERR_MASK                0x00000020U
+#define SCB_ADD_ACK_ERR_SHIFT               5
+#define SCB_ADD_ACK_ERR_SIGNED              0
+
+#define SCB_WR_ACK_ERR_MASK                 0x00000010U
+#define SCB_WR_ACK_ERR_SHIFT                4
+#define SCB_WR_ACK_ERR_SIGNED               0
+
+#define SCB_SDAT_LO_TIM_MASK                0x00000008U
+#define SCB_SDAT_LO_TIM_SHIFT               3
+#define SCB_SDAT_LO_TIM_SIGNED              0
+
+#define SCB_SCLK_LO_TIM_MASK                0x00000004U
+#define SCB_SCLK_LO_TIM_SHIFT               2
+#define SCB_SCLK_LO_TIM_SIGNED              0
+
+#define SCB_UNEX_START_BIT_MASK             0x00000002U
+#define SCB_UNEX_START_BIT_SHIFT            1
+#define SCB_UNEX_START_BIT_SIGNED           0
+
+#define SCB_BUS_INACTIVE_MASK               0x00000001U
+#define SCB_BUS_INACTIVE_SHIFT              0
+#define SCB_BUS_INACTIVE_SIGNED             0
+
+/*
+	Register INTERRUPT_OP_CFG
+*/
+#define TCF_CLK_CTRL_INTERRUPT_OP_CFG       0x00D0
+#define PULSE_NLEVEL_MASK                   0x80000000U
+#define PULSE_NLEVEL_SHIFT                  31
+#define PULSE_NLEVEL_SIGNED                 0
+
+#define INT_SENSE_MASK                      0x40000000U
+#define INT_SENSE_SHIFT                     30
+#define INT_SENSE_SIGNED                    0
+
+#define INTERRUPT_DEST_MASK                 0x0000000FU
+#define INTERRUPT_DEST_SHIFT                0
+#define INTERRUPT_DEST_SIGNED               0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define TCF_CLK_CTRL_INTERRUPT_ENABLE       0x00D8
+#define INTERRUPT_MASTER_ENABLE_MASK        0x80000000U
+#define INTERRUPT_MASTER_ENABLE_SHIFT       31
+#define INTERRUPT_MASTER_ENABLE_SIGNED      0
+
+#define INTERRUPT_ENABLE_MASK               0x7FFFFFFFU
+#define INTERRUPT_ENABLE_SHIFT              0
+#define INTERRUPT_ENABLE_SIGNED             0
+
+/*
+	Register INTERRUPT_CLEAR
+*/
+#define TCF_CLK_CTRL_INTERRUPT_CLEAR        0x00E0
+#define INTERRUPT_MASTER_CLEAR_MASK         0x80000000U
+#define INTERRUPT_MASTER_CLEAR_SHIFT        31
+#define INTERRUPT_MASTER_CLEAR_SIGNED       0
+
+#define INTERRUPT_CLEAR_MASK                0x7FFFFFFFU
+#define INTERRUPT_CLEAR_SHIFT               0
+#define INTERRUPT_CLEAR_SIGNED              0
+
+/*
+	Register YCC_RGB_CTRL
+*/
+#define TCF_CLK_CTRL_YCC_RGB_CTRL           0x00E8
+#define RGB_CTRL1_MASK                      0x000001FFU
+#define RGB_CTRL1_SHIFT                     0
+#define RGB_CTRL1_SIGNED                    0
+
+#define RGB_CTRL2_MASK                      0x01FF0000U
+#define RGB_CTRL2_SHIFT                     16
+#define RGB_CTRL2_SIGNED                    0
+
+/*
+	Register EXP_BRD_CTRL
+*/
+#define TCF_CLK_CTRL_EXP_BRD_CTRL           0x00F8
+#define PDP1_DATA_EN_MASK                   0x00000003U
+#define PDP1_DATA_EN_SHIFT                  0
+#define PDP1_DATA_EN_SIGNED                 0
+
+#define PDP2_DATA_EN_MASK                   0x00000030U
+#define PDP2_DATA_EN_SHIFT                  4
+#define PDP2_DATA_EN_SIGNED                 0
+
+#define EXP_BRD_OUTPUT_MASK                 0xFFFFFF00U
+#define EXP_BRD_OUTPUT_SHIFT                8
+#define EXP_BRD_OUTPUT_SIGNED               0
+
+/*
+	Register HOSTIF_CONTROL
+*/
+#define TCF_CLK_CTRL_HOSTIF_CONTROL         0x0100
+#define HOSTIF_CTRL_MASK                    0x000000FFU
+#define HOSTIF_CTRL_SHIFT                   0
+#define HOSTIF_CTRL_SIGNED                  0
+
+/*
+	Register DUT_CONTROL_1
+*/
+#define TCF_CLK_CTRL_DUT_CONTROL_1          0x0108
+#define DUT_CTRL_1_MASK                     0xFFFFFFFFU
+#define DUT_CTRL_1_SHIFT                    0
+#define DUT_CTRL_1_SIGNED                   0
+
+/* TC ES2 additional needs those: */
+#define DUT_CTRL_TEST_MODE_SHIFT            0
+#define DUT_CTRL_TEST_MODE_MASK             0x3
+
+#define DUT_CTRL_VCC_0V9EN                  (1<<12)
+#define DUT_CTRL_VCC_1V8EN                  (1<<13)
+#define DUT_CTRL_VCC_IO_INH                 (1<<14)
+#define DUT_CTRL_VCC_CORE_INH               (1<<15)
+
+/*
+	Register DUT_STATUS_1
+*/
+#define TCF_CLK_CTRL_DUT_STATUS_1           0x0110
+#define DUT_STATUS_1_MASK                   0xFFFFFFFFU
+#define DUT_STATUS_1_SHIFT                  0
+#define DUT_STATUS_1_SIGNED                 0
+
+/*
+	Register DUT_CTRL_NOT_STAT_1
+*/
+#define TCF_CLK_CTRL_DUT_CTRL_NOT_STAT_1    0x0118
+#define DUT_STAT_NOT_CTRL_1_MASK            0xFFFFFFFFU
+#define DUT_STAT_NOT_CTRL_1_SHIFT           0
+#define DUT_STAT_NOT_CTRL_1_SIGNED          0
+
+/*
+	Register DUT_CONTROL_2
+*/
+#define TCF_CLK_CTRL_DUT_CONTROL_2          0x0120
+#define DUT_CTRL_2_MASK                     0xFFFFFFFFU
+#define DUT_CTRL_2_SHIFT                    0
+#define DUT_CTRL_2_SIGNED                   0
+
+/*
+	Register DUT_STATUS_2
+*/
+#define TCF_CLK_CTRL_DUT_STATUS_2           0x0128
+#define DUT_STATUS_2_MASK                   0xFFFFFFFFU
+#define DUT_STATUS_2_SHIFT                  0
+#define DUT_STATUS_2_SIGNED                 0
+
+/*
+	Register DUT_CTRL_NOT_STAT_2
+*/
+#define TCF_CLK_CTRL_DUT_CTRL_NOT_STAT_2    0x0130
+#define DUT_CTRL_NOT_STAT_2_MASK            0xFFFFFFFFU
+#define DUT_CTRL_NOT_STAT_2_SHIFT           0
+#define DUT_CTRL_NOT_STAT_2_SIGNED          0
+
+/*
+	Register BUS_CAP_BASE_ADDR
+*/
+#define TCF_CLK_CTRL_BUS_CAP_BASE_ADDR      0x0138
+#define BUS_CAP_BASE_ADDR_MASK              0xFFFFFFFFU
+#define BUS_CAP_BASE_ADDR_SHIFT             0
+#define BUS_CAP_BASE_ADDR_SIGNED            0
+
+/*
+	Register BUS_CAP_ENABLE
+*/
+#define TCF_CLK_CTRL_BUS_CAP_ENABLE         0x0140
+#define BUS_CAP_ENABLE_MASK                 0x00000001U
+#define BUS_CAP_ENABLE_SHIFT                0
+#define BUS_CAP_ENABLE_SIGNED               0
+
+/*
+	Register BUS_CAP_COUNT
+*/
+#define TCF_CLK_CTRL_BUS_CAP_COUNT          0x0148
+#define BUS_CAP_COUNT_MASK                  0xFFFFFFFFU
+#define BUS_CAP_COUNT_SHIFT                 0
+#define BUS_CAP_COUNT_SIGNED                0
+
+/*
+	Register DCM_LOCK_STATUS
+*/
+#define TCF_CLK_CTRL_DCM_LOCK_STATUS        0x0150
+#define DCM_LOCK_STATUS_MASK                0x00000007U
+#define DCM_LOCK_STATUS_SHIFT               0
+#define DCM_LOCK_STATUS_SIGNED              0
+
+/*
+	Register AUX_DUT_RESETNS
+*/
+#define TCF_CLK_CTRL_AUX_DUT_RESETNS        0x0158
+#define AUX_DUT_RESETNS_MASK                0x0000000FU
+#define AUX_DUT_RESETNS_SHIFT               0
+#define AUX_DUT_RESETNS_SIGNED              0
+
+/*
+	Register TCF_SPI_MST_ADDR_RDNWR
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR 0x0160
+#define TCF_SPI_MST_ADDR_MASK               0x0003FFFFU
+#define TCF_SPI_MST_ADDR_SHIFT              0
+#define TCF_SPI_MST_ADDR_SIGNED             0
+
+#define TCF_SPI_MST_RDNWR_MASK              0x00040000U
+#define TCF_SPI_MST_RDNWR_SHIFT             18
+#define TCF_SPI_MST_RDNWR_SIGNED            0
+
+#define TCF_SPI_MST_SLAVE_ID_MASK           0x00080000U
+#define TCF_SPI_MST_SLAVE_ID_SHIFT          19
+#define TCF_SPI_MST_SLAVE_ID_SIGNED         0
+
+#define TCF_SPI_MST_MASTER_ID_MASK          0x00300000U
+#define TCF_SPI_MST_MASTER_ID_SHIFT         20
+#define TCF_SPI_MST_MASTER_ID_SIGNED        0
+
+/*
+	Register TCF_SPI_MST_WDATA
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_WDATA      0x0168
+#define TCF_SPI_MST_WDATA_MASK              0xFFFFFFFFU
+#define TCF_SPI_MST_WDATA_SHIFT             0
+#define TCF_SPI_MST_WDATA_SIGNED            0
+
+/*
+	Register TCF_SPI_MST_RDATA
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_RDATA      0x0170
+#define TCF_SPI_MST_RDATA_MASK              0xFFFFFFFFU
+#define TCF_SPI_MST_RDATA_SHIFT             0
+#define TCF_SPI_MST_RDATA_SIGNED            0
+
+/*
+	Register TCF_SPI_MST_STATUS
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_STATUS     0x0178
+#define TCF_SPI_MST_STATUS_MASK             0x0000000FU
+#define TCF_SPI_MST_STATUS_SHIFT            0
+#define TCF_SPI_MST_STATUS_SIGNED           0
+
+/*
+	Register TCF_SPI_MST_GO
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_GO         0x0180
+#define TCF_SPI_MST_GO_MASK                 0x00000001U
+#define TCF_SPI_MST_GO_SHIFT                0
+#define TCF_SPI_MST_GO_SIGNED               0
+
+/*
+	Register EXT_SIG_CTRL
+*/
+#define TCF_CLK_CTRL_EXT_SIG_CTRL           0x0188
+#define EXT_SYS_REQ_SIG_START_MASK          0x00000001U
+#define EXT_SYS_REQ_SIG_START_SHIFT         0
+#define EXT_SYS_REQ_SIG_START_SIGNED        0
+
+#define EXT_SYS_RD_SIG_START_MASK           0x00000002U
+#define EXT_SYS_RD_SIG_START_SHIFT          1
+#define EXT_SYS_RD_SIG_START_SIGNED         0
+
+#define EXT_MEM_REQ_SIG_START_MASK          0x00000004U
+#define EXT_MEM_REQ_SIG_START_SHIFT         2
+#define EXT_MEM_REQ_SIG_START_SIGNED        0
+
+#define EXT_MEM_RD_SIG_START_MASK           0x00000008U
+#define EXT_MEM_RD_SIG_START_SHIFT          3
+#define EXT_MEM_RD_SIG_START_SIGNED         0
+
+/*
+	Register EXT_SYS_REQ_SIG
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_SIG        0x0190
+#define EXT_SYS_REQ_SIG_MASK                0xFFFFFFFFU
+#define EXT_SYS_REQ_SIG_SHIFT               0
+#define EXT_SYS_REQ_SIG_SIGNED              0
+
+/*
+	Register EXT_SYS_RD_SIG
+*/
+#define TCF_CLK_CTRL_EXT_SYS_RD_SIG         0x0198
+#define EXT_SYS_RD_SIG_MASK                 0xFFFFFFFFU
+#define EXT_SYS_RD_SIG_SHIFT                0
+#define EXT_SYS_RD_SIG_SIGNED               0
+
+/*
+	Register EXT_MEM_REQ_SIG
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_SIG        0x01A0
+#define EXT_MEM_REQ_SIG_MASK                0xFFFFFFFFU
+#define EXT_MEM_REQ_SIG_SHIFT               0
+#define EXT_MEM_REQ_SIG_SIGNED              0
+
+/*
+	Register EXT_MEM_RD_SIG
+*/
+#define TCF_CLK_CTRL_EXT_MEM_RD_SIG         0x01A8
+#define EXT_MEM_RD_SIG_MASK                 0xFFFFFFFFU
+#define EXT_MEM_RD_SIG_SHIFT                0
+#define EXT_MEM_RD_SIG_SIGNED               0
+
+/*
+	Register EXT_SYS_REQ_WR_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_WR_CNT     0x01B0
+#define EXT_SYS_REQ_WR_CNT_MASK             0xFFFFFFFFU
+#define EXT_SYS_REQ_WR_CNT_SHIFT            0
+#define EXT_SYS_REQ_WR_CNT_SIGNED           0
+
+/*
+	Register EXT_SYS_REQ_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_RD_CNT     0x01B8
+#define EXT_SYS_REQ_RD_CNT_MASK             0xFFFFFFFFU
+#define EXT_SYS_REQ_RD_CNT_SHIFT            0
+#define EXT_SYS_REQ_RD_CNT_SIGNED           0
+
+/*
+	Register EXT_SYS_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_RD_CNT         0x01C0
+#define EXT_SYS_RD_CNT_MASK                 0xFFFFFFFFU
+#define EXT_SYS_RD_CNT_SHIFT                0
+#define EXT_SYS_RD_CNT_SIGNED               0
+
+/*
+	Register EXT_MEM_REQ_WR_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_WR_CNT     0x01C8
+#define EXT_MEM_REQ_WR_CNT_MASK             0xFFFFFFFFU
+#define EXT_MEM_REQ_WR_CNT_SHIFT            0
+#define EXT_MEM_REQ_WR_CNT_SIGNED           0
+
+/*
+	Register EXT_MEM_REQ_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_RD_CNT     0x01D0
+#define EXT_MEM_REQ_RD_CNT_MASK             0xFFFFFFFFU
+#define EXT_MEM_REQ_RD_CNT_SHIFT            0
+#define EXT_MEM_REQ_RD_CNT_SIGNED           0
+
+/*
+	Register EXT_MEM_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_RD_CNT         0x01D8
+#define EXT_MEM_RD_CNT_MASK                 0xFFFFFFFFU
+#define EXT_MEM_RD_CNT_SHIFT                0
+#define EXT_MEM_RD_CNT_SIGNED               0
+
+/*
+	Register TCF_CORE_TARGET_BUILD_CFG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG 0x01E0
+#define TCF_CORE_TARGET_BUILD_ID_MASK       0x000000FFU
+#define TCF_CORE_TARGET_BUILD_ID_SHIFT      0
+#define TCF_CORE_TARGET_BUILD_ID_SIGNED     0
+
+/*
+	Register MEM_THROUGH_SYS
+*/
+#define TCF_CLK_CTRL_MEM_THROUGH_SYS        0x01E8
+#define MEM_THROUGH_SYS_MASK                0x00000001U
+#define MEM_THROUGH_SYS_SHIFT               0
+#define MEM_THROUGH_SYS_SIGNED              0
+
+/*
+	Register HOST_PHY_OFFSET
+*/
+#define TCF_CLK_CTRL_HOST_PHY_OFFSET        0x01F0
+#define HOST_PHY_OFFSET_MASK                0xFFFFFFFFU
+#define HOST_PHY_OFFSET_SHIFT               0
+#define HOST_PHY_OFFSET_SIGNED              0
+
+/*
+	Register DEBUG_REG_SEL
+*/
+#define TCF_CLK_CTRL_DEBUG_REG_SEL          0x01F8
+#define DEBUG_REG_SELECT_MASK               0xFFFFFFFFU
+#define DEBUG_REG_SELECT_SHIFT              0
+#define DEBUG_REG_SELECT_SIGNED             0
+
+/*
+	Register DEBUG_REG
+*/
+#define TCF_CLK_CTRL_DEBUG_REG              0x0200
+#define DEBUG_REG_VALUE_MASK                0xFFFFFFFFU
+#define DEBUG_REG_VALUE_SHIFT               0
+#define DEBUG_REG_VALUE_SIGNED              0
+
+/*
+	Register JTAG_CTRL
+*/
+#define TCF_CLK_CTRL_JTAG_CTRL              0x0208
+#define JTAG_TRST_MASK                      0x00000001U
+#define JTAG_TRST_SHIFT                     0
+#define JTAG_TRST_SIGNED                    0
+
+#define JTAG_TMS_MASK                       0x00000002U
+#define JTAG_TMS_SHIFT                      1
+#define JTAG_TMS_SIGNED                     0
+
+#define JTAG_TCK_MASK                       0x00000004U
+#define JTAG_TCK_SHIFT                      2
+#define JTAG_TCK_SIGNED                     0
+
+#define JTAG_TDO_MASK                       0x00000008U
+#define JTAG_TDO_SHIFT                      3
+#define JTAG_TDO_SIGNED                     0
+
+#define JTAG_TDI_MASK                       0x00000010U
+#define JTAG_TDI_SHIFT                      4
+#define JTAG_TDI_SIGNED                     0
+
+#define JTAG_DASH_N_REG_MASK                0x40000000U
+#define JTAG_DASH_N_REG_SHIFT               30
+#define JTAG_DASH_N_REG_SIGNED              0
+
+#define JTAG_DISABLE_MASK                   0x80000000U
+#define JTAG_DISABLE_SHIFT                  31
+#define JTAG_DISABLE_SIGNED                 0
+
+/*
+	Register SAI_DEBUG_RDNWR
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_RDNWR        0x0300
+#define SAI_DEBUG_REG_ADDR_MASK             0x000001FFU
+#define SAI_DEBUG_REG_ADDR_SHIFT            0
+#define SAI_DEBUG_REG_ADDR_SIGNED           0
+
+#define SAI_DEBUG_REG_RDNWR_MASK            0x00000200U
+#define SAI_DEBUG_REG_RDNWR_SHIFT           9
+#define SAI_DEBUG_REG_RDNWR_SIGNED          0
+
+/*
+	Register SAI_DEBUG_WDATA
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_WDATA        0x0308
+#define SAI_DEBUG_REG_WDATA_MASK            0xFFFFFFFFU
+#define SAI_DEBUG_REG_WDATA_SHIFT           0
+#define SAI_DEBUG_REG_WDATA_SIGNED          0
+
+/*
+	Register SAI_DEBUG_RDATA
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_RDATA        0x0310
+#define SAI_DEBUG_REG_RDATA_MASK            0xFFFFFFFFU
+#define SAI_DEBUG_REG_RDATA_SHIFT           0
+#define SAI_DEBUG_REG_RDATA_SIGNED          0
+
+/*
+	Register SAI_DEBUG_GO
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_GO           0x0318
+#define SAI_DEBUG_REG_GO_MASK               0x00000001U
+#define SAI_DEBUG_REG_GO_SHIFT              0
+#define SAI_DEBUG_REG_GO_SIGNED             0
+
+/*
+	Register AUX_DUT_RESETS
+*/
+#define TCF_CLK_CTRL_AUX_DUT_RESETS         0x0320
+#define AUX_DUT_RESETS_MASK                 0x0000000FU
+#define AUX_DUT_RESETS_SHIFT                0
+#define AUX_DUT_RESETS_SIGNED               0
+
+/*
+	Register DUT_CLK_CTRL
+*/
+#define TCF_CLK_CTRL_DUT_CLK_CTRL           0x0328
+#define MEM_REQ_PHSE_MASK                   0x0000FFFFU
+#define MEM_REQ_PHSE_SHIFT                  0
+#define MEM_REQ_PHSE_SIGNED                 0
+
+/*
+	Register DUT_CLK_STATUS
+*/
+#define TCF_CLK_CTRL_DUT_CLK_STATUS         0x0330
+#define MEM_REQ_PHSE_SET_MASK               0x00000003U
+#define MEM_REQ_PHSE_SET_SHIFT              0
+#define MEM_REQ_PHSE_SET_SIGNED             0
+
+/*
+	Register DUT_CLK_INFO
+*/
+#define TCF_CLK_CTRL_DUT_CLK_INFO           0x0340
+#define CORE_MASK                           0x0000FFFFU
+#define CORE_SHIFT                          0
+#define CORE_SIGNED                         0
+
+#define MEM_MASK                            0xFFFF0000U
+#define MEM_SHIFT                           16
+#define MEM_SIGNED                          0
+
+/*
+	Register DUT_CLK_PHSE
+*/
+#define TCF_CLK_CTRL_DUT_CLK_PHSE           0x0348
+#define MEM_REQ_MASK                        0x0000FFFFU
+#define MEM_REQ_SHIFT                       0
+#define MEM_REQ_SIGNED                      0
+
+#define MEM_RD_MASK                         0xFFFF0000U
+#define MEM_RD_SHIFT                        16
+#define MEM_RD_SIGNED                       0
+
+#endif /* !defined(_TCF_CLK_CTRL_H_) */
+
+/*****************************************************************************
+ End of file (tcf_clk_ctrl.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_pll.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_pll.h
new file mode 100644
index 000000000000..71eaf924bbd6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_pll.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework PDP register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from tcf_pll.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_PLL_H_)
+#define _TCF_PLL_H_
+
+/*
+	Register PLL_DDR2_CLK0
+*/
+#define TCF_PLL_PLL_DDR2_CLK0               0x0000
+#define DDR2_PLL_CLK0_PHS_MASK              0x00300000U
+#define DDR2_PLL_CLK0_PHS_SHIFT             20
+#define DDR2_PLL_CLK0_PHS_SIGNED            0
+
+#define DDR2_PLL_CLK0_MS_MASK               0x00030000U
+#define DDR2_PLL_CLK0_MS_SHIFT              16
+#define DDR2_PLL_CLK0_MS_SIGNED             0
+
+#define DDR2_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define DDR2_PLL_CLK0_FREQ_SHIFT            0
+#define DDR2_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_DDR2_CLK1TO5
+*/
+#define TCF_PLL_PLL_DDR2_CLK1TO5            0x0008
+#define DDR2_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define DDR2_PLL_CLK1TO5_PHS_SHIFT          20
+#define DDR2_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define DDR2_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define DDR2_PLL_CLK1TO5_MS_SHIFT           10
+#define DDR2_PLL_CLK1TO5_MS_SIGNED          0
+
+#define DDR2_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define DDR2_PLL_CLK1TO5_FREQ_SHIFT         0
+#define DDR2_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_DDR2_DRP_GO
+*/
+#define TCF_PLL_PLL_DDR2_DRP_GO             0x0010
+#define PLL_DDR2_DRP_GO_MASK                0x00000001U
+#define PLL_DDR2_DRP_GO_SHIFT               0
+#define PLL_DDR2_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_PDP_CLK0
+*/
+#define TCF_PLL_PLL_PDP_CLK0                0x0018
+#define PDP_PLL_CLK0_PHS_MASK               0x00300000U
+#define PDP_PLL_CLK0_PHS_SHIFT              20
+#define PDP_PLL_CLK0_PHS_SIGNED             0
+
+#define PDP_PLL_CLK0_MS_MASK                0x00030000U
+#define PDP_PLL_CLK0_MS_SHIFT               16
+#define PDP_PLL_CLK0_MS_SIGNED              0
+
+#define PDP_PLL_CLK0_FREQ_MASK              0x000001FFU
+#define PDP_PLL_CLK0_FREQ_SHIFT             0
+#define PDP_PLL_CLK0_FREQ_SIGNED            0
+
+/*
+	Register PLL_PDP_CLK1TO5
+*/
+#define TCF_PLL_PLL_PDP_CLK1TO5             0x0020
+#define PDP_PLL_CLK1TO5_PHS_MASK            0x3FF00000U
+#define PDP_PLL_CLK1TO5_PHS_SHIFT           20
+#define PDP_PLL_CLK1TO5_PHS_SIGNED          0
+
+#define PDP_PLL_CLK1TO5_MS_MASK             0x000FFC00U
+#define PDP_PLL_CLK1TO5_MS_SHIFT            10
+#define PDP_PLL_CLK1TO5_MS_SIGNED           0
+
+#define PDP_PLL_CLK1TO5_FREQ_MASK           0x000003FFU
+#define PDP_PLL_CLK1TO5_FREQ_SHIFT          0
+#define PDP_PLL_CLK1TO5_FREQ_SIGNED         0
+
+/*
+	Register PLL_PDP_DRP_GO
+*/
+#define TCF_PLL_PLL_PDP_DRP_GO              0x0028
+#define PLL_PDP_DRP_GO_MASK                 0x00000001U
+#define PLL_PDP_DRP_GO_SHIFT                0
+#define PLL_PDP_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_PDP2_CLK0
+*/
+#define TCF_PLL_PLL_PDP2_CLK0               0x0030
+#define PDP2_PLL_CLK0_PHS_MASK              0x00300000U
+#define PDP2_PLL_CLK0_PHS_SHIFT             20
+#define PDP2_PLL_CLK0_PHS_SIGNED            0
+
+#define PDP2_PLL_CLK0_MS_MASK               0x00030000U
+#define PDP2_PLL_CLK0_MS_SHIFT              16
+#define PDP2_PLL_CLK0_MS_SIGNED             0
+
+#define PDP2_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define PDP2_PLL_CLK0_FREQ_SHIFT            0
+#define PDP2_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_PDP2_CLK1TO5
+*/
+#define TCF_PLL_PLL_PDP2_CLK1TO5            0x0038
+#define PDP2_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define PDP2_PLL_CLK1TO5_PHS_SHIFT          20
+#define PDP2_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define PDP2_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define PDP2_PLL_CLK1TO5_MS_SHIFT           10
+#define PDP2_PLL_CLK1TO5_MS_SIGNED          0
+
+#define PDP2_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define PDP2_PLL_CLK1TO5_FREQ_SHIFT         0
+#define PDP2_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_PDP2_DRP_GO
+*/
+#define TCF_PLL_PLL_PDP2_DRP_GO             0x0040
+#define PLL_PDP2_DRP_GO_MASK                0x00000001U
+#define PLL_PDP2_DRP_GO_SHIFT               0
+#define PLL_PDP2_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_CORE_CLK0
+*/
+#define TCF_PLL_PLL_CORE_CLK0               0x0048
+#define CORE_PLL_CLK0_PHS_MASK              0x00300000U
+#define CORE_PLL_CLK0_PHS_SHIFT             20
+#define CORE_PLL_CLK0_PHS_SIGNED            0
+
+#define CORE_PLL_CLK0_MS_MASK               0x00030000U
+#define CORE_PLL_CLK0_MS_SHIFT              16
+#define CORE_PLL_CLK0_MS_SIGNED             0
+
+#define CORE_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define CORE_PLL_CLK0_FREQ_SHIFT            0
+#define CORE_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_CORE_CLK1TO5
+*/
+#define TCF_PLL_PLL_CORE_CLK1TO5            0x0050
+#define CORE_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define CORE_PLL_CLK1TO5_PHS_SHIFT          20
+#define CORE_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define CORE_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define CORE_PLL_CLK1TO5_MS_SHIFT           10
+#define CORE_PLL_CLK1TO5_MS_SIGNED          0
+
+#define CORE_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define CORE_PLL_CLK1TO5_FREQ_SHIFT         0
+#define CORE_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_CORE_DRP_GO
+*/
+#define TCF_PLL_PLL_CORE_DRP_GO             0x0058
+#define PLL_CORE_DRP_GO_MASK                0x00000001U
+#define PLL_CORE_DRP_GO_SHIFT               0
+#define PLL_CORE_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_SYSIF_CLK0
+*/
+#define TCF_PLL_PLL_SYSIF_CLK0              0x0060
+#define SYSIF_PLL_CLK0_PHS_MASK             0x00300000U
+#define SYSIF_PLL_CLK0_PHS_SHIFT            20
+#define SYSIF_PLL_CLK0_PHS_SIGNED           0
+
+#define SYSIF_PLL_CLK0_MS_MASK              0x00030000U
+#define SYSIF_PLL_CLK0_MS_SHIFT             16
+#define SYSIF_PLL_CLK0_MS_SIGNED            0
+
+#define SYSIF_PLL_CLK0_FREQ_MASK            0x000001FFU
+#define SYSIF_PLL_CLK0_FREQ_SHIFT           0
+#define SYSIF_PLL_CLK0_FREQ_SIGNED          0
+
+/*
+	Register PLL_SYSIF_CLK1TO5
+*/
+#define TCF_PLL_PLL_SYSIF_CLK1TO5           0x0068
+#define SYSIF_PLL_CLK1TO5_PHS_MASK          0x3FF00000U
+#define SYSIF_PLL_CLK1TO5_PHS_SHIFT         20
+#define SYSIF_PLL_CLK1TO5_PHS_SIGNED        0
+
+#define SYSIF_PLL_CLK1TO5_MS_MASK           0x000FFC00U
+#define SYSIF_PLL_CLK1TO5_MS_SHIFT          10
+#define SYSIF_PLL_CLK1TO5_MS_SIGNED         0
+
+#define SYSIF_PLL_CLK1TO5_FREQ_MASK         0x000003FFU
+#define SYSIF_PLL_CLK1TO5_FREQ_SHIFT        0
+#define SYSIF_PLL_CLK1TO5_FREQ_SIGNED       0
+
+/*
+	Register PLL_SYS_DRP_GO
+*/
+#define TCF_PLL_PLL_SYS_DRP_GO              0x0070
+#define PLL_SYS_DRP_GO_MASK                 0x00000001U
+#define PLL_SYS_DRP_GO_SHIFT                0
+#define PLL_SYS_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_MEMIF_CLK0
+*/
+#define TCF_PLL_PLL_MEMIF_CLK0              0x0078
+#define MEMIF_PLL_CLK0_PHS_MASK             0x00300000U
+#define MEMIF_PLL_CLK0_PHS_SHIFT            20
+#define MEMIF_PLL_CLK0_PHS_SIGNED           0
+
+#define MEMIF_PLL_CLK0_MS_MASK              0x00030000U
+#define MEMIF_PLL_CLK0_MS_SHIFT             16
+#define MEMIF_PLL_CLK0_MS_SIGNED            0
+
+#define MEMIF_PLL_CLK0_FREQ_MASK            0x000001FFU
+#define MEMIF_PLL_CLK0_FREQ_SHIFT           0
+#define MEMIF_PLL_CLK0_FREQ_SIGNED          0
+
+/*
+	Register PLL_MEMIF_CLK1TO5
+*/
+#define TCF_PLL_PLL_MEMIF_CLK1TO5           0x0080
+#define MEMIF_PLL_CLK1TO5_PHS_MASK          0x3FF00000U
+#define MEMIF_PLL_CLK1TO5_PHS_SHIFT         20
+#define MEMIF_PLL_CLK1TO5_PHS_SIGNED        0
+
+#define MEMIF_PLL_CLK1TO5_MS_MASK           0x000FFC00U
+#define MEMIF_PLL_CLK1TO5_MS_SHIFT          10
+#define MEMIF_PLL_CLK1TO5_MS_SIGNED         0
+
+#define MEMIF_PLL_CLK1TO5_FREQ_MASK         0x000003FFU
+#define MEMIF_PLL_CLK1TO5_FREQ_SHIFT        0
+#define MEMIF_PLL_CLK1TO5_FREQ_SIGNED       0
+
+/*
+	Register PLL_MEM_DRP_GO
+*/
+#define TCF_PLL_PLL_MEM_DRP_GO              0x0088
+#define PLL_MEM_DRP_GO_MASK                 0x00000001U
+#define PLL_MEM_DRP_GO_SHIFT                0
+#define PLL_MEM_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_ALL_DRP_GO
+*/
+#define TCF_PLL_PLL_ALL_DRP_GO              0x0090
+#define PLL_ALL_DRP_GO_MASK                 0x00000001U
+#define PLL_ALL_DRP_GO_SHIFT                0
+#define PLL_ALL_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_DRP_STATUS
+*/
+#define TCF_PLL_PLL_DRP_STATUS              0x0098
+#define PLL_LOCKS_MASK                      0x00003F00U
+#define PLL_LOCKS_SHIFT                     8
+#define PLL_LOCKS_SIGNED                    0
+
+#define PLL_DRP_GOOD_MASK                   0x0000003FU
+#define PLL_DRP_GOOD_SHIFT                  0
+#define PLL_DRP_GOOD_SIGNED                 0
+
+#endif /* !defined(_TCF_PLL_H_) */
+
+/*****************************************************************************
+ End of file (tcf_pll.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_rgbpdp_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_rgbpdp_regs.h
new file mode 100644
index 000000000000..e87ba6152411
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_rgbpdp_regs.h
@@ -0,0 +1,559 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework PDP register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from: tcf_rgbpdp_regs.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_RGBPDP_REGS_H_)
+#define _TCF_RGBPDP_REGS_H_
+
+/*
+	Register PVR_TCF_RGBPDP_STR1SURF
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF  0x0000
+#define STR1HEIGHT_MASK                     0x000007FFU
+#define STR1HEIGHT_SHIFT                    0
+#define STR1HEIGHT_SIGNED                   0
+
+#define STR1WIDTH_MASK                      0x003FF800U
+#define STR1WIDTH_SHIFT                     11
+#define STR1WIDTH_SIGNED                    0
+
+#define STR1PIXFMT_MASK                     0x0F000000U
+#define STR1PIXFMT_SHIFT                    24
+#define STR1PIXFMT_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_STR1ADDRCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL 0x0004
+#define STR1BASE_MASK                       0x03FFFFFFU
+#define STR1BASE_SHIFT                      0
+#define STR1BASE_SIGNED                     0
+
+#define STR1INTFIELD_MASK                   0x40000000U
+#define STR1INTFIELD_SHIFT                  30
+#define STR1INTFIELD_SIGNED                 0
+
+#define STR1STREN_MASK                      0x80000000U
+#define STR1STREN_SHIFT                     31
+#define STR1STREN_SIGNED                    0
+
+/*
+	Register PVR_PDP_STR1POSN
+*/
+#define TCF_RGBPDP_PVR_PDP_STR1POSN         0x0008
+#define STR1STRIDE_MASK                     0x000003FFU
+#define STR1STRIDE_SHIFT                    0
+#define STR1STRIDE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_MEMCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_MEMCTRL   0x000C
+#define MEMREFRESH_MASK                     0xC0000000U
+#define MEMREFRESH_SHIFT                    30
+#define MEMREFRESH_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_STRCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL   0x0010
+#define BURSTLEN_GFX_MASK                   0x000000FFU
+#define BURSTLEN_GFX_SHIFT                  0
+#define BURSTLEN_GFX_SIGNED                 0
+
+#define THRESHOLD_GFX_MASK                  0x0000FF00U
+#define THRESHOLD_GFX_SHIFT                 8
+#define THRESHOLD_GFX_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_SYNCCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL  0x0014
+#define HSDIS_MASK                          0x00000001U
+#define HSDIS_SHIFT                         0
+#define HSDIS_SIGNED                        0
+
+#define HSPOL_MASK                          0x00000002U
+#define HSPOL_SHIFT                         1
+#define HSPOL_SIGNED                        0
+
+#define VSDIS_MASK                          0x00000004U
+#define VSDIS_SHIFT                         2
+#define VSDIS_SIGNED                        0
+
+#define VSPOL_MASK                          0x00000008U
+#define VSPOL_SHIFT                         3
+#define VSPOL_SIGNED                        0
+
+#define BLNKDIS_MASK                        0x00000010U
+#define BLNKDIS_SHIFT                       4
+#define BLNKDIS_SIGNED                      0
+
+#define BLNKPOL_MASK                        0x00000020U
+#define BLNKPOL_SHIFT                       5
+#define BLNKPOL_SIGNED                      0
+
+#define HS_SLAVE_MASK                       0x00000040U
+#define HS_SLAVE_SHIFT                      6
+#define HS_SLAVE_SIGNED                     0
+
+#define VS_SLAVE_MASK                       0x00000080U
+#define VS_SLAVE_SHIFT                      7
+#define VS_SLAVE_SIGNED                     0
+
+#define INTERLACE_MASK                      0x00000100U
+#define INTERLACE_SHIFT                     8
+#define INTERLACE_SIGNED                    0
+
+#define FIELDPOL_MASK                       0x00000200U
+#define FIELDPOL_SHIFT                      9
+#define FIELDPOL_SIGNED                     0
+
+#define CLKPOL_MASK                         0x00000800U
+#define CLKPOL_SHIFT                        11
+#define CLKPOL_SIGNED                       0
+
+#define CSYNC_EN_MASK                       0x00001000U
+#define CSYNC_EN_SHIFT                      12
+#define CSYNC_EN_SIGNED                     0
+
+#define FIELD_EN_MASK                       0x00002000U
+#define FIELD_EN_SHIFT                      13
+#define FIELD_EN_SIGNED                     0
+
+#define UPDWAIT_MASK                        0x000F0000U
+#define UPDWAIT_SHIFT                       16
+#define UPDWAIT_SIGNED                      0
+
+#define UPDCTRL_MASK                        0x01000000U
+#define UPDCTRL_SHIFT                       24
+#define UPDCTRL_SIGNED                      0
+
+#define UPDINTCTRL_MASK                     0x02000000U
+#define UPDINTCTRL_SHIFT                    25
+#define UPDINTCTRL_SIGNED                   0
+
+#define UPDSYNCTRL_MASK                     0x04000000U
+#define UPDSYNCTRL_SHIFT                    26
+#define UPDSYNCTRL_SIGNED                   0
+
+#define POWERDN_MASK                        0x10000000U
+#define POWERDN_SHIFT                       28
+#define POWERDN_SIGNED                      0
+
+#define DISP_RST_MASK                       0x20000000U
+#define DISP_RST_SHIFT                      29
+#define DISP_RST_SIGNED                     0
+
+#define SYNCACTIVE_MASK                     0x80000000U
+#define SYNCACTIVE_SHIFT                    31
+#define SYNCACTIVE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_BORDCOL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL   0x0018
+#define BORDCOL_MASK                        0x00FFFFFFU
+#define BORDCOL_SHIFT                       0
+#define BORDCOL_SIGNED                      0
+
+/*
+	Register PVR_TCF_RGBPDP_UPDCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL   0x001C
+#define UPDFIELD_MASK                       0x00000001U
+#define UPDFIELD_SHIFT                      0
+#define UPDFIELD_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC1
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1    0x0020
+#define HT_MASK                             0x00000FFFU
+#define HT_SHIFT                            0
+#define HT_SIGNED                           0
+
+#define HBPS_MASK                           0x0FFF0000U
+#define HBPS_SHIFT                          16
+#define HBPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC2
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2    0x0024
+#define HLBS_MASK                           0x00000FFFU
+#define HLBS_SHIFT                          0
+#define HLBS_SIGNED                         0
+
+#define HAS_MASK                            0x0FFF0000U
+#define HAS_SHIFT                           16
+#define HAS_SIGNED                          0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC3
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3    0x0028
+#define HRBS_MASK                           0x00000FFFU
+#define HRBS_SHIFT                          0
+#define HRBS_SIGNED                         0
+
+#define HFPS_MASK                           0x0FFF0000U
+#define HFPS_SHIFT                          16
+#define HFPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC1
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1    0x002C
+#define VT_MASK                             0x00000FFFU
+#define VT_SHIFT                            0
+#define VT_SIGNED                           0
+
+#define VBPS_MASK                           0x0FFF0000U
+#define VBPS_SHIFT                          16
+#define VBPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC2
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2    0x0030
+#define VTBS_MASK                           0x00000FFFU
+#define VTBS_SHIFT                          0
+#define VTBS_SIGNED                         0
+
+#define VAS_MASK                            0x0FFF0000U
+#define VAS_SHIFT                           16
+#define VAS_SIGNED                          0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC3
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3    0x0034
+#define VBBS_MASK                           0x00000FFFU
+#define VBBS_SHIFT                          0
+#define VBBS_SIGNED                         0
+
+#define VFPS_MASK                           0x0FFF0000U
+#define VFPS_SHIFT                          16
+#define VFPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_HDECTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL   0x0038
+#define HDEF_MASK                           0x00000FFFU
+#define HDEF_SHIFT                          0
+#define HDEF_SIGNED                         0
+
+#define HDES_MASK                           0x0FFF0000U
+#define HDES_SHIFT                          16
+#define HDES_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VDECTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL   0x003C
+#define VDEF_MASK                           0x00000FFFU
+#define VDEF_SHIFT                          0
+#define VDEF_SIGNED                         0
+
+#define VDES_MASK                           0x0FFF0000U
+#define VDES_SHIFT                          16
+#define VDES_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VEVENT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT    0x0040
+#define VFETCH_MASK                         0x00000FFFU
+#define VFETCH_SHIFT                        0
+#define VFETCH_SIGNED                       0
+
+#define VEVENT_MASK                         0x0FFF0000U
+#define VEVENT_SHIFT                        16
+#define VEVENT_SIGNED                       0
+
+/*
+	Register PVR_TCF_RGBPDP_OPMASK
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_OPMASK    0x0044
+#define MASKR_MASK                          0x000000FFU
+#define MASKR_SHIFT                         0
+#define MASKR_SIGNED                        0
+
+#define MASKG_MASK                          0x0000FF00U
+#define MASKG_SHIFT                         8
+#define MASKG_SIGNED                        0
+
+#define MASKB_MASK                          0x00FF0000U
+#define MASKB_SHIFT                         16
+#define MASKB_SIGNED                        0
+
+#define BLANKLEVEL_MASK                     0x40000000U
+#define BLANKLEVEL_SHIFT                    30
+#define BLANKLEVEL_SIGNED                   0
+
+#define MASKLEVEL_MASK                      0x80000000U
+#define MASKLEVEL_SHIFT                     31
+#define MASKLEVEL_SIGNED                    0
+
+/*
+	Register PVR_TCF_RGBPDP_INTSTAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTSTAT   0x0048
+#define INTS_HBLNK0_MASK                    0x00000001U
+#define INTS_HBLNK0_SHIFT                   0
+#define INTS_HBLNK0_SIGNED                  0
+
+#define INTS_HBLNK1_MASK                    0x00000002U
+#define INTS_HBLNK1_SHIFT                   1
+#define INTS_HBLNK1_SIGNED                  0
+
+#define INTS_VBLNK0_MASK                    0x00000004U
+#define INTS_VBLNK0_SHIFT                   2
+#define INTS_VBLNK0_SIGNED                  0
+
+#define INTS_VBLNK1_MASK                    0x00000008U
+#define INTS_VBLNK1_SHIFT                   3
+#define INTS_VBLNK1_SIGNED                  0
+
+#define INTS_STR1URUN_MASK                  0x00000010U
+#define INTS_STR1URUN_SHIFT                 4
+#define INTS_STR1URUN_SIGNED                0
+
+#define INTS_STR1ORUN_MASK                  0x00000020U
+#define INTS_STR1ORUN_SHIFT                 5
+#define INTS_STR1ORUN_SIGNED                0
+
+#define INTS_DISPURUN_MASK                  0x00000040U
+#define INTS_DISPURUN_SHIFT                 6
+#define INTS_DISPURUN_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_INTENAB
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB   0x004C
+#define INTEN_HBLNK0_MASK                   0x00000001U
+#define INTEN_HBLNK0_SHIFT                  0
+#define INTEN_HBLNK0_SIGNED                 0
+
+#define INTEN_HBLNK1_MASK                   0x00000002U
+#define INTEN_HBLNK1_SHIFT                  1
+#define INTEN_HBLNK1_SIGNED                 0
+
+#define INTEN_VBLNK0_MASK                   0x00000004U
+#define INTEN_VBLNK0_SHIFT                  2
+#define INTEN_VBLNK0_SIGNED                 0
+
+#define INTEN_VBLNK1_MASK                   0x00000008U
+#define INTEN_VBLNK1_SHIFT                  3
+#define INTEN_VBLNK1_SIGNED                 0
+
+#define INTEN_STR1URUN_MASK                 0x00000010U
+#define INTEN_STR1URUN_SHIFT                4
+#define INTEN_STR1URUN_SIGNED               0
+
+#define INTEN_STR1ORUN_MASK                 0x00000020U
+#define INTEN_STR1ORUN_SHIFT                5
+#define INTEN_STR1ORUN_SIGNED               0
+
+#define INTEN_DISPURUN_MASK                 0x00000040U
+#define INTEN_DISPURUN_SHIFT                6
+#define INTEN_DISPURUN_SIGNED               0
+
+/*
+	Register PVR_TCF_RGBPDP_INTCLEAR
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTCLEAR  0x0050
+#define INTCLR_HBLNK0_MASK                  0x00000001U
+#define INTCLR_HBLNK0_SHIFT                 0
+#define INTCLR_HBLNK0_SIGNED                0
+
+#define INTCLR_HBLNK1_MASK                  0x00000002U
+#define INTCLR_HBLNK1_SHIFT                 1
+#define INTCLR_HBLNK1_SIGNED                0
+
+#define INTCLR_VBLNK0_MASK                  0x00000004U
+#define INTCLR_VBLNK0_SHIFT                 2
+#define INTCLR_VBLNK0_SIGNED                0
+
+#define INTCLR_VBLNK1_MASK                  0x00000008U
+#define INTCLR_VBLNK1_SHIFT                 3
+#define INTCLR_VBLNK1_SIGNED                0
+
+#define INTCLR_STR1URUN_MASK                0x00000010U
+#define INTCLR_STR1URUN_SHIFT               4
+#define INTCLR_STR1URUN_SIGNED              0
+
+#define INTCLR_STR1ORUN_MASK                0x00000020U
+#define INTCLR_STR1ORUN_SHIFT               5
+#define INTCLR_STR1ORUN_SIGNED              0
+
+#define INTCLR_DISPURUN_MASK                0x00000040U
+#define INTCLR_DISPURUN_SHIFT               6
+#define INTCLR_DISPURUN_SIGNED              0
+
+/*
+	Register PVR_TCF_RGBPDP_INTCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTCTRL   0x0054
+#define HBLNK_LINENO_MASK                   0x00000FFFU
+#define HBLNK_LINENO_SHIFT                  0
+#define HBLNK_LINENO_SIGNED                 0
+
+#define HBLNK_LINE_MASK                     0x00010000U
+#define HBLNK_LINE_SHIFT                    16
+#define HBLNK_LINE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_SIGNAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_SIGNAT    0x0058
+#define SIGNATURE_MASK                      0xFFFFFFFFU
+#define SIGNATURE_SHIFT                     0
+#define SIGNATURE_SIGNED                    0
+
+/*
+	Register PVR_TCF_RGBPDP_LINESTAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_LINESTAT  0x005C
+#define LINENO_MASK                         0x00000FFFU
+#define LINENO_SHIFT                        0
+#define LINENO_SIGNED                       0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGCTRL   0x0060
+#define DBG_ENAB_MASK                       0x00000001U
+#define DBG_ENAB_SHIFT                      0
+#define DBG_ENAB_SIGNED                     0
+
+#define DBG_READ_MASK                       0x00000002U
+#define DBG_READ_SHIFT                      1
+#define DBG_READ_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGDATA
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGDATA   0x0064
+#define DBG_DATA_MASK                       0x00FFFFFFU
+#define DBG_DATA_SHIFT                      0
+#define DBG_DATA_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGSIDE
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGSIDE   0x0068
+#define DBG_SIDE_MASK                       0x00000007U
+#define DBG_SIDE_SHIFT                      0
+#define DBG_SIDE_SIGNED                     0
+
+#define DBG_VAL_MASK                        0x00000008U
+#define DBG_VAL_SHIFT                       3
+#define DBG_VAL_SIGNED                      0
+
+/*
+	Register PVR_TCF_RGBPDP_REGLD_STAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_REGLD_STAT 0x0070
+#define REGLD_ADDROUT_MASK                  0x00FFFFFFU
+#define REGLD_ADDROUT_SHIFT                 0
+#define REGLD_ADDROUT_SIGNED                0
+
+#define REGLD_ADDREN_MASK                   0x80000000U
+#define REGLD_ADDREN_SHIFT                  31
+#define REGLD_ADDREN_SIGNED                 0
+
+/*
+	Register PVR_TCF_RGBPDP_REGLD_CTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_REGLD_CTRL 0x0074
+#define REGLD_ADDRIN_MASK                   0x00FFFFFFU
+#define REGLD_ADDRIN_SHIFT                  0
+#define REGLD_ADDRIN_SIGNED                 0
+
+#define REGLD_VAL_MASK                      0x01000000U
+#define REGLD_VAL_SHIFT                     24
+#define REGLD_VAL_SIGNED                    0
+
+#define REGLD_ADDRLEN_MASK                  0xFE000000U
+#define REGLD_ADDRLEN_SHIFT                 25
+#define REGLD_ADDRLEN_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_CORE_ID
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_CORE_ID   0x0078
+#define CONFIG_ID_MASK                      0x0000FFFFU
+#define CONFIG_ID_SHIFT                     0
+#define CONFIG_ID_SIGNED                    0
+
+#define CORE_ID_MASK                        0x00FF0000U
+#define CORE_ID_SHIFT                       16
+#define CORE_ID_SIGNED                      0
+
+#define GROUP_ID_MASK                       0xFF000000U
+#define GROUP_ID_SHIFT                      24
+#define GROUP_ID_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_CORE_REV
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_CORE_REV  0x007C
+#define MAINT_REV_MASK                      0x000000FFU
+#define MAINT_REV_SHIFT                     0
+#define MAINT_REV_SIGNED                    0
+
+#define MINOR_REV_MASK                      0x0000FF00U
+#define MINOR_REV_SHIFT                     8
+#define MINOR_REV_SIGNED                    0
+
+#define MAJOR_REV_MASK                      0x00FF0000U
+#define MAJOR_REV_SHIFT                     16
+#define MAJOR_REV_SIGNED                    0
+
+#endif /* !defined(_TCF_RGBPDP_REGS_H_) */
+
+/*****************************************************************************
+ End of file (tcf_rgbpdp_regs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/virt_validation_defs.h b/drivers/gpu/drm/img/img-volcanic/include/virt_validation_defs.h
new file mode 100644
index 000000000000..5b8908f712c0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/virt_validation_defs.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Definitions for virtualization
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services shared header for virtualization definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRV_VIRT_DEFS_H
+#define SRV_VIRT_DEFS_H
+
+#if !defined(GPUVIRT_VALIDATION_MAX_STRING_LENGTH)
+	#define GPUVIRT_VALIDATION_MAX_STRING_LENGTH 512
+#endif
+
+#define GPUVIRT_VALIDATION_MAX_OS 8
+
+#define GPUVIRT_VALIDATION_NUM_REGIONS 2
+#define GPUVIRT_VAL_REGION_SECURE 0
+#define GPUVIRT_VAL_REGION_SHARED 1
+
+/* Shared region 1MB */
+#define GPUVIRT_SIZEOF_SHARED 0x100000
+
+/* Min region size 64MB */
+#define GPUVIRT_MIN_SIZE 0x4000000
+
+#endif /* SRV_VIRT_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/pvrsrv_surface.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/pvrsrv_surface.h
new file mode 100644
index 000000000000..4bb87e0dfff8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/pvrsrv_surface.h
@@ -0,0 +1,212 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR Services Surface
+@Description    Surface definitions and utilities that are externally visible
+                (i.e. visible to clients of services), but are also required
+                within services.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_SURFACE_H
+#define PVRSRV_SURFACE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_SURFACE_TRANSFORM_NONE      (0 << 0)		/*!< No transformation */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_H    (1 << 0)		/*!< Flip horizontally */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_V    (1 << 1)		/*!< Flip vertically */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_90    (1 << 2)                          /*!< Rotate  90 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_180   ((1 << 0) + (1 << 1))             /*!< Rotate 180 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_270   ((1 << 0) + (1 << 1) + (1 << 2))  /*!< Rotate 270 degree clockwise */
+
+#define PVRSRV_SURFACE_BLENDING_NONE       0	/*!< Use no blending */
+#define PVRSRV_SURFACE_BLENDING_PREMULT    1	/*!< Use blending with pre-multiplier */
+#define PVRSRV_SURFACE_BLENDING_COVERAGE   2	/*!< Use coverage blending */
+
+/*!
+ * Modes of memory layouts for surfaces.
+ *
+ *   Enum: #_PVRSRV_SURFACE_MEMLAYOUT_
+ *   Typedef: ::PVRSRV_SURFACE_MEMLAYOUT
+ */
+typedef enum _PVRSRV_SURFACE_MEMLAYOUT_
+{
+	PVRSRV_SURFACE_MEMLAYOUT_STRIDED = 0,		/*!< Strided memory buffer */
+	PVRSRV_SURFACE_MEMLAYOUT_FBC,				/*!< Compressed frame buffer */
+} PVRSRV_SURFACE_MEMLAYOUT;
+
+/*!
+ * Frame Buffer Compression layout.
+ * Defines the FBC mode to use.
+ *
+ *   Structure: #PVRSRV_SURFACE_FBC_LAYOUT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FBC_LAYOUT
+ */
+typedef struct PVRSRV_SURFACE_FBC_LAYOUT_TAG
+{
+	/*! The compression mode for this surface */
+	IMG_FB_COMPRESSION	eFBCompressionMode;
+} PVRSRV_SURFACE_FBC_LAYOUT;
+
+/*!
+ * Pixel and memory format of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_FORMAT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FORMAT
+ */
+typedef struct PVRSRV_SURFACE_FORMAT_TAG
+{
+	/*! Enum value of type IMG_PIXFMT for the pixel format */
+	IMG_UINT32					ePixFormat;
+
+	/*! Enum surface memory layout */
+	PVRSRV_SURFACE_MEMLAYOUT	eMemLayout;
+
+	/*! Special layout options for the surface.
+	 * Needs services support.
+	 * Depends on eMemLayout.*/
+	union {
+		PVRSRV_SURFACE_FBC_LAYOUT	sFBCLayout;
+	} u;
+} PVRSRV_SURFACE_FORMAT;
+
+/*!
+ * Width and height of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_DIMS_TAG
+ *   Typedef: ::PVRSRV_SURFACE_DIMS
+ */
+typedef struct PVRSRV_SURFACE_DIMS_TAG
+{
+	IMG_UINT32		ui32Width;		/*!< Width in pixels */
+	IMG_UINT32		ui32Height;		/*!< Height in pixels */
+} PVRSRV_SURFACE_DIMS;
+
+/*!
+ * Dimension and format details of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_INFO
+ */
+typedef struct PVRSRV_SURFACE_INFO_TAG
+{
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Width and height */
+	PVRSRV_SURFACE_FORMAT	sFormat;	/*!< Memory format */
+} PVRSRV_SURFACE_INFO;
+
+/*!
+ * Defines a rectangle on a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_RECT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_RECT
+ */
+typedef struct PVRSRV_SURFACE_RECT_TAG
+{
+	IMG_INT32				i32XOffset;	/*!< X offset from origin in pixels */
+	IMG_INT32				i32YOffset;	/*!< Y offset from origin in pixels */
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Rectangle dimensions */
+} PVRSRV_SURFACE_RECT;
+
+/*!
+ * Surface configuration details
+ *
+ *   Structure: #PVRSRV_SURFACE_CONFIG_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_CONFIG_INFO
+ */
+typedef struct PVRSRV_SURFACE_CONFIG_INFO_TAG
+{
+	/*! Crop applied to surface (BEFORE transformation) */
+	PVRSRV_SURFACE_RECT		sCrop;
+
+	/*! Region of screen to display surface in (AFTER scaling) */
+	PVRSRV_SURFACE_RECT		sDisplay;
+
+	/*! Surface transformation none/flip/rotate.
+	 * Use PVRSRV_SURFACE_TRANSFORM_xxx macros
+	 */
+	IMG_UINT32				ui32Transform;
+
+	/*! Alpha blending mode e.g. none/premult/coverage.
+	 * Use PVRSRV_SURFACE_BLENDING_xxx macros
+	 */
+	IMG_UINT32				eBlendType;
+
+	/*! Custom data for the display engine */
+	IMG_UINT32				ui32Custom;
+
+	/*! Alpha value for this plane */
+	IMG_UINT8				ui8PlaneAlpha;
+
+	/*! Reserved for later use */
+	IMG_UINT8				ui8Reserved1[3];
+} PVRSRV_SURFACE_CONFIG_INFO;
+
+/*!
+ * Contains information about a panel
+ */
+typedef struct PVRSRV_PANEL_INFO_TAG
+{
+	PVRSRV_SURFACE_INFO sSurfaceInfo;		/*!< Panel surface details */
+	IMG_UINT32			ui32RefreshRate;	/*!< Panel refresh rate in Hz */
+	IMG_UINT32			ui32XDpi;			/*!< Panel DPI in x direction */
+	IMG_UINT32			ui32YDpi;			/*!< Panel DPI in y direction */
+} PVRSRV_PANEL_INFO;
+
+/*!
+ * Helper function to create a Config Info based on a Surface Info
+ * to do a flip with no scale, transformation etc.
+ */
+static INLINE void SurfaceConfigFromSurfInfo(const PVRSRV_SURFACE_INFO *psSurfaceInfo,
+                                             PVRSRV_SURFACE_CONFIG_INFO *psConfigInfo)
+{
+	psConfigInfo->sCrop.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sCrop.i32XOffset = 0;
+	psConfigInfo->sCrop.i32YOffset = 0;
+	psConfigInfo->sDisplay.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sDisplay.i32XOffset = 0;
+	psConfigInfo->sDisplay.i32YOffset = 0;
+	psConfigInfo->ui32Transform = PVRSRV_SURFACE_TRANSFORM_NONE;
+	psConfigInfo->eBlendType = PVRSRV_SURFACE_BLENDING_NONE;
+	psConfigInfo->ui32Custom = 0;
+	psConfigInfo->ui8PlaneAlpha = 0xff;
+}
+
+#endif /* PVRSRV_SURFACE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_alignchecks.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_alignchecks.h
new file mode 100644
index 000000000000..411e984aaa9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_alignchecks.h
@@ -0,0 +1,190 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX fw interface alignment checks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Checks to avoid disalignment in RGX fw data structures
+                shared with the host
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_ALIGNCHECKS_H)
+#define RGX_FWIF_ALIGNCHECKS_H
+
+/* for the offsetof macro */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/*!
+ ******************************************************************************
+ * Alignment UM/FW checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_UM_MAX 128U
+
+#if defined(PM_INTERACTIVE_MODE)
+#define HWRTDATA_PM_OFFSET  offsetof(RGXFWIF_HWRTDATA, sPMMListDevVAddr),
+#define HWRTDATA_HEAPTABLE_OFFSET offsetof(RGXFWIF_HWRTDATA, psVHeapTableDevVAddr),
+#else
+#define HWRTDATA_PM_OFFSET  offsetof(RGXFWIF_HWRTDATA, sPMRenderStateDevVAddr),
+#define HWRTDATA_HEAPTABLE_OFFSET
+#endif
+
+#define RGXFW_ALIGN_CHECKS_INIT0									\
+		sizeof(RGXFWIF_TRACEBUF),									\
+		offsetof(RGXFWIF_TRACEBUF, ui32LogType),					\
+		offsetof(RGXFWIF_TRACEBUF, sTraceBuf),						\
+		offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),		\
+		offsetof(RGXFWIF_TRACEBUF, ui32TracebufFlags),				\
+																	\
+		sizeof(RGXFWIF_SYSDATA),									\
+		offsetof(RGXFWIF_SYSDATA, ePowState),						\
+		offsetof(RGXFWIF_SYSDATA, ui32HWPerfDropCount),				\
+		offsetof(RGXFWIF_SYSDATA, ui32LastDropOrdinal),				\
+		offsetof(RGXFWIF_SYSDATA, ui32FWFaults),					\
+		offsetof(RGXFWIF_SYSDATA, ui32HWRStateFlags),				\
+																	\
+		sizeof(RGXFWIF_OSDATA),										\
+		offsetof(RGXFWIF_OSDATA, ui32HostSyncCheckMark),			\
+		offsetof(RGXFWIF_OSDATA, ui32KCCBCmdsExecuted),				\
+																	\
+		sizeof(RGXFWIF_HWRINFOBUF),									\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmLockedUpCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmOverranCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmRecoveredCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmFalseDetectCount),	\
+																	\
+		/* RGXFWIF_CMDTA checks */									\
+		sizeof(RGXFWIF_CMDTA),										\
+		offsetof(RGXFWIF_CMDTA, sGeomRegs),							\
+																	\
+		/* RGXFWIF_CMD3D checks */									\
+		sizeof(RGXFWIF_CMD3D),										\
+		offsetof(RGXFWIF_CMD3D, s3DRegs),							\
+																	\
+		/* RGXFWIF_CMD_COMPUTE checks */							\
+		sizeof(RGXFWIF_CMD_COMPUTE),								\
+		offsetof(RGXFWIF_CMD_COMPUTE, sCDMRegs),					\
+																	\
+		/* RGXFWIF_FREELIST checks */								\
+		sizeof(RGXFWIF_FREELIST), \
+		offsetof(RGXFWIF_FREELIST, sFreeListBaseDevVAddr),\
+		offsetof(RGXFWIF_FREELIST, sFreeListStateDevVAddr),\
+		offsetof(RGXFWIF_FREELIST, sFreeListLastGrowDevVAddr),\
+		offsetof(RGXFWIF_FREELIST, ui32MaxPages),\
+		offsetof(RGXFWIF_FREELIST, ui32CurrentPages),\
+									\
+		/* RGXFWIF_HWRTDATA checks */					\
+		sizeof(RGXFWIF_HWRTDATA), \
+		HWRTDATA_PM_OFFSET \
+		HWRTDATA_HEAPTABLE_OFFSET \
+		offsetof(RGXFWIF_HWRTDATA, apsFreeLists),\
+		/*offsetof(RGXFWIF_HWRTDATA, ui64VCECatBase),*/ \
+		offsetof(RGXFWIF_HWRTDATA, eState), \
+							\
+\
+		sizeof(RGXFWIF_HWPERF_CTL), \
+		offsetof(RGXFWIF_HWPERF_CTL, sBlkCfg), \
+		sizeof(RGXFWIF_CMDTDM), \
+		offsetof(RGXFWIF_CMDTDM, sTDMRegs)
+
+#define RGXFW_ALIGN_CHECKS_INIT		RGXFW_ALIGN_CHECKS_INIT0
+
+
+
+/*!
+ ******************************************************************************
+ * Alignment KM checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_INIT_KM0                         \
+		sizeof(RGXFWIF_SYSINIT),                            \
+		offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr),          \
+		offsetof(RGXFWIF_SYSINIT, sPDSExecBase),            \
+		offsetof(RGXFWIF_SYSINIT, sUSCExecBase),            \
+		offsetof(RGXFWIF_SYSINIT, aui32TPUTrilinearFracMask),\
+		offsetof(RGXFWIF_SYSINIT, sTraceBufCtl),            \
+		offsetof(RGXFWIF_SYSINIT, sFwSysData),              \
+		                                                    \
+		sizeof(RGXFWIF_OSINIT),                             \
+		offsetof(RGXFWIF_OSINIT, psKernelCCBCtl),           \
+		offsetof(RGXFWIF_OSINIT, psKernelCCB),              \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCBCtl),         \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCB),            \
+		offsetof(RGXFWIF_OSINIT, sFwOsData),                \
+		offsetof(RGXFWIF_OSINIT, sRGXCompChecks),           \
+		                                                    \
+		/* RGXFWIF_FWRENDERCONTEXT checks */                \
+		sizeof(RGXFWIF_FWRENDERCONTEXT),                    \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),      \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),      \
+		                                                    \
+		sizeof(RGXFWIF_FWCOMPUTECONTEXT),                   \
+		offsetof(RGXFWIF_FWCOMPUTECONTEXT, sCDMContext),    \
+		offsetof(RGXFWIF_FWCOMPUTECONTEXT, sStaticComputeContextState),\
+		offsetof(RGXFWIF_FWCOMPUTECONTEXT, ui32WorkEstCCBSubmitted),\
+															\
+		sizeof(RGXFWIF_FWTDMCONTEXT),                       \
+		offsetof(RGXFWIF_FWTDMCONTEXT, sTDMContext),        \
+		offsetof(RGXFWIF_FWTDMCONTEXT, ui32WorkEstCCBSubmitted),\
+															\
+		sizeof(RGXFWIF_FWCOMMONCONTEXT),                    \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psFWMemContext),  \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, sRunNode),        \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psCCB),           \
+		                                                    \
+		sizeof(RGXFWIF_MMUCACHEDATA),                       \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32CacheFlags),     \
+		offsetof(RGXFWIF_MMUCACHEDATA, sMMUCacheSync),      \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32MMUCacheSyncUpdateValue)
+
+#if defined(SUPPORT_TRP)
+#define RGXFW_ALIGN_CHECKS_INIT_KM                          \
+		RGXFW_ALIGN_CHECKS_INIT_KM0,                        \
+		offsetof(RGXFWIF_FWTDMCONTEXT, aui64TRPChecksums2D)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT_KM RGXFW_ALIGN_CHECKS_INIT_KM0
+#endif
+
+#endif /* RGX_FWIF_ALIGNCHECKS_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_alignchecks.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_customer.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_customer.h
new file mode 100644
index 000000000000..60a9273570fe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_customer.h
@@ -0,0 +1,51 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_CUSTOMER_H)
+#define RGX_FWIF_CUSTOMER_H
+
+/*
+ * Custom firmware data structures here
+ */
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_hwperf.h
new file mode 100644
index 000000000000..e2287375fdc2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_hwperf.h
@@ -0,0 +1,125 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_hwperf.h
+@Title          RGX HWPerf support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared header between RGX firmware and Init process
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_HWPERF_H
+#define RGX_FWIF_HWPERF_H
+
+#include "rgx_fwif_shared.h"
+#include "rgx_hwperf.h"
+#include "rgxdefs_km.h"
+
+/* Server and Firmware definitions only */
+
+/*! The number of HWPerf blocks in the GPU */
+
+#if defined(RGX_FIRMWARE)
+#define RGX_HWPERF_NUM_SPU ((RGX_FEATURE_NUM_SPU))
+#define RGX_HWPERF_NUM_USC ((RGX_FEATURE_NUM_CLUSTERS))
+#define RGX_HWPERF_NUM_ISP_PER_SPU ((RGX_FEATURE_NUM_ISP_PER_SPU))
+#define RGX_HWPERF_NUM_PBE ((RGX_FEATURE_PBE_PER_SPU) * (RGX_FEATURE_NUM_SPU))
+#define RGX_HWPERF_NUM_MERCER ((RGX_FEATURE_NUM_CLUSTERS))
+#define RGX_HWPERF_NUM_PBE_SHARED ((RGX_FEATURE_NUM_SPU))
+#define RGX_HWPERF_NUM_SWIFT ((RGX_FEATURE_NUM_SPU * RGX_FEATURE_MAX_TPU_PER_SPU))
+#define RGX_HWPERF_NUM_TEXAS ((RGX_FEATURE_NUM_SPU))
+#if (RGX_FEATURE_RAY_TRACING_ARCH > 2) && (RGX_FEATURE_SPU0_RAC_PRESENT > 0)
+#define RGX_HWPERF_NUM_RAC   ((RGX_NUM_RAC))
+#else
+#define RGX_HWPERF_NUM_RAC   ((0))
+#endif
+#define RGX_HWPERF_NUM_TPU   ((RGX_FEATURE_NUM_SPU * RGX_FEATURE_MAX_TPU_PER_SPU))
+#define RGX_HWPERF_NUM_ISP   ((RGX_FEATURE_NUM_CLUSTERS))
+
+#define RGX_CNTBLK_INDIRECT_COUNT(_class) ((RGX_HWPERF_NUM_ ## _class))
+
+/*! The number of layout blocks defined with configurable
+ * performance counters. Compile time constants.
+ * This is for the Series 8XT+ layout.
+ */
+#define RGX_HWPERF_MAX_DEFINED_BLKS (\
+	(IMG_UINT32)RGX_CNTBLK_ID_DIRECT_LAST  +\
+	RGX_CNTBLK_INDIRECT_COUNT(ISP)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(MERCER)      +\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE_SHARED)  +\
+	RGX_CNTBLK_INDIRECT_COUNT(USC)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(TPU)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(SWIFT)       +\
+	RGX_CNTBLK_INDIRECT_COUNT(TEXAS)       +\
+	RGX_CNTBLK_INDIRECT_COUNT(RAC))
+
+#endif	/* RGX_FIRMWARE */
+
+/*****************************************************************************/
+
+/* Structure used in the FW's global control data to hold the performance
+ * counters provisioned for a given block. */
+typedef struct
+{
+	IMG_UINT32           uiBlockID;
+	IMG_UINT32           uiNumCounters;    // Number of counters held
+	                                       // in aui32CounterCfg
+	                                       // [0..RGX_CNTBLK_COUNTERS_MAX)
+	IMG_UINT32           uiEnabled;        // 1 => enabled, 0=> disabled
+	RGXFWIF_DEV_VIRTADDR psModel;          // link to model table for uiBlockID
+	IMG_UINT32           aui32CounterCfg[RGX_CNTBLK_COUNTERS_MAX];
+} RGXFWIF_HWPERF_CTL_BLK;
+
+
+/*!
+ *****************************************************************************
+ * Structure used in the FW's global RGXFW_CTL store, holding HWPerf counter
+ * block configuration. It is written to by the Server on FW initialisation
+ * (PDUMP=1) and by the FW BG kCCB command processing code. It is read by
+ * the FW IRQ register programming and HWPerf event generation routines.
+ * Size of the sBlkCfg[] array must be consistent between KM/UM and FW.
+ * FW will ASSERT if the sizes are different
+ * (ui32NumBlocks != RGX_HWPERF_MAX_DEFINED_BLKS)
+ ****************************************************************************/
+typedef struct
+{
+	IMG_UINT32                         ui32Reserved;
+	IMG_UINT32                         ui32CtrlWord;
+	IMG_UINT32                         ui32EnabledBlksCount;
+	IMG_UINT32                         ui32NumBlocks;
+	RGXFWIF_HWPERF_CTL_BLK RGXFW_ALIGN sBlkCfg[1];	// First array entry
+} UNCACHED_ALIGN RGXFWIF_HWPERF_CTL;
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_km.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_km.h
new file mode 100644
index 000000000000..45cdb2d4ca88
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_km.h
@@ -0,0 +1,1949 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_KM_H)
+#define RGX_FWIF_KM_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+#include "rgxdefs_km.h"
+#include "dllist.h"
+#include "rgx_hwperf.h"
+#include "rgxheapconfig.h"
+#include "rgx_fwif_customer.h"
+
+/************************************************************************
+* RGX FW signature checks
+************************************************************************/
+#define RGXFW_SIG_BUFFER_SIZE_MIN       (8192)
+
+#define RGXFWIF_TIMEDIFF_ID			((0x1UL << 28) | RGX_CR_TIMER)
+
+#define RGXFW_POLL_TYPE_SET 0x80000000U
+
+/* Firmware per-DM HWR states */
+#define RGXFWIF_DM_STATE_WORKING					(0x00U)		/*!< DM is working if all flags are cleared */
+#define RGXFWIF_DM_STATE_READY_FOR_HWR				(IMG_UINT32_C(0x1) << 0)	/*!< DM is idle and ready for HWR */
+#define RGXFWIF_DM_STATE_NEEDS_SKIP					(IMG_UINT32_C(0x1) << 2)	/*!< DM need to skip to next cmd before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP			(IMG_UINT32_C(0x1) << 3)	/*!< DM need partial render cleanup before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR			(IMG_UINT32_C(0x1) << 4)	/*!< DM need to increment Recovery Count once fully recovered */
+#define RGXFWIF_DM_STATE_GUILTY_LOCKUP				(IMG_UINT32_C(0x1) << 5)	/*!< DM was identified as locking up and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_LOCKUP			(IMG_UINT32_C(0x1) << 6)	/*!< DM was innocently affected by another lockup which caused HWR */
+#define RGXFWIF_DM_STATE_GUILTY_OVERRUNING			(IMG_UINT32_C(0x1) << 7)	/*!< DM was identified as over-running and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_OVERRUNING		(IMG_UINT32_C(0x1) << 8)	/*!< DM was innocently affected by another DM over-running which caused HWR */
+#define RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH		(IMG_UINT32_C(0x1) << 9)	/*!< DM was forced into HWR as it delayed more important workloads */
+#define RGXFWIF_DM_STATE_GPU_ECC_HWR				(IMG_UINT32_C(0x1) << 10)	/*!< DM was forced into HWR due to an uncorrected GPU ECC error */
+#define RGXFWIF_DM_STATE_GPU_PARITY_HWR				(IMG_UINT32_C(0x1) << 11)	/*!< DM was forced into HWR due to an uncorrected GPU PARITY error */
+#define RGXFWIF_DM_STATE_GPU_LATENT_HWR				(IMG_UINT32_C(0x1) << 12)	/*!< DM was forced into HWR due to an uncorrected GPU LATENT error */
+#define RGXFWIF_DM_STATE_ICS_HWR					(IMG_UINT32_C(0x1) << 13)	/*!< DM was forced into HWR due to a fault detected by Idle Cycle Stealing */
+
+/* Firmware's connection state */
+typedef IMG_UINT32 RGXFWIF_CONNECTION_FW_STATE;
+#define RGXFW_CONNECTION_FW_OFFLINE				0U	/*!< Firmware is offline */
+#define RGXFW_CONNECTION_FW_READY				1U	/*!< Firmware is initialised */
+#define RGXFW_CONNECTION_FW_ACTIVE				2U	/*!< Firmware connection is fully established */
+#define RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD	3U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_FORCED_OFFLOAD		4U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_COOLDOWN			5U	/*!< Firmware connection is in cooldown period */
+#define RGXFW_CONNECTION_FW_STATE_COUNT			6U
+
+#define RGXFW_FORCED_OFFLOAD_HCS_DEADLINE_MS	2U		/*!< Workloads of Guest being forcefully offloaded must be stopped quickly */
+#define RGXFW_GRACEFUL_OFFLOAD_HCS_DEADLINE_MS	1000U	/*!< Workloads of Guest being gracefully offloaded are allowed more time to finish */
+
+/* OS' connection state */
+typedef enum
+{
+	RGXFW_CONNECTION_OS_OFFLINE = 0,	/*!< OS is offline */
+	RGXFW_CONNECTION_OS_READY,			/*!< OS's KM driver is setup and waiting */
+	RGXFW_CONNECTION_OS_ACTIVE,			/*!< OS connection is fully established */
+	RGXFW_CONNECTION_OS_STATE_COUNT
+} RGXFWIF_CONNECTION_OS_STATE;
+
+#define RGXFWIF_CTXSWITCH_PROFILE_FAST_EN		(IMG_UINT32_C(0x1))
+#define RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN		(IMG_UINT32_C(0x2))
+#define RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN		(IMG_UINT32_C(0x3))
+#define RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN	(IMG_UINT32_C(0x4))
+
+#define RGXFWIF_ISP_SCHEDMODE_VER1_IPP			(IMG_UINT32_C(0x1))
+#define RGXFWIF_ISP_SCHEDMODE_VER2_ISP			(IMG_UINT32_C(0x2))
+/*!
+ ******************************************************************************
+ * RGX firmware Init Config Data
+ *****************************************************************************/
+
+/* Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_CTXSWITCH_MODE_RAND				(IMG_UINT32_C(0x1) << 0)	/*!< Randomise context switch requests */
+#define RGXFWIF_INICFG_CTXSWITCH_SRESET_EN				(IMG_UINT32_C(0x1) << 1)
+#define RGXFWIF_INICFG_HWPERF_EN						(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN				(IMG_UINT32_C(0x1) << 3)	/*!< Randomise DM-killing requests */
+#define RGXFWIF_INICFG_POW_RASCALDUST					(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_SPU_CLOCK_GATE					(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_FBCDC_V3_1_EN					(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_CHECK_MLIST_EN					(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_DISABLE_CLKGATING_EN				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN	(IMG_UINT32_C(0x1) << 9)
+#define RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN		(IMG_UINT32_C(0x1) << 10)
+/* 11 unused */
+#define RGXFWIF_INICFG_REGCONFIG_EN						(IMG_UINT32_C(0x1) << 12)
+#define RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY			(IMG_UINT32_C(0x1) << 13)
+#define RGXFWIF_INICFG_HWP_DISABLE_FILTER				(IMG_UINT32_C(0x1) << 14)
+/* 15 unused */
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST			(RGXFWIF_CTXSWITCH_PROFILE_FAST_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM			(RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW			(RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY		(RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK			(IMG_UINT32_C(0x7) << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_DISABLE_DM_OVERLAP				(IMG_UINT32_C(0x1) << 19)
+#define RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER			(IMG_UINT32_C(0x1) << 20)
+#define RGXFWIF_INICFG_INJECT_ICS_FAULT					(IMG_UINT32_C(0x1) << 21)
+#define RGXFWIF_INICFG_VALIDATE_IRQ						(IMG_UINT32_C(0x1) << 22)
+#define RGXFWIF_INICFG_DISABLE_PDP_EN					(IMG_UINT32_C(0x1) << 23)
+#define RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN	(IMG_UINT32_C(0x1) << 24)
+#define RGXFWIF_INICFG_WORKEST							(IMG_UINT32_C(0x1) << 25)
+#define RGXFWIF_INICFG_PDVFS							(IMG_UINT32_C(0x1) << 26)
+/* 27 unused */
+/* 28 unused */
+#define RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT				(29)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_NONE				(0)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP			(RGXFWIF_ISP_SCHEDMODE_VER1_IPP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP			(RGXFWIF_ISP_SCHEDMODE_VER2_ISP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_MASK				(RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP |\
+                                                         RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP)
+#define RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER			(IMG_UINT32_C(0x1) << 31)
+#define RGXFWIF_INICFG_ALL								(0xFFFFF7FFU)
+
+/* Extended Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_EXT_ALL							(0x0U)
+
+#define RGXFWIF_INICFG_SYS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_CTXSWITCH_MODE_RAND | \
+														  RGXFWIF_INICFG_CTXSWITCH_SRESET_EN)
+
+/* Flag definitions affecting only workloads submitted by a particular OS */
+
+/*!
+ * @AddToGroup ContextSwitching
+ * @{
+ * @Name Per-OS DM context switch configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN				(IMG_UINT32_C(0x1) << 0) /*!< Enables TDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN				(IMG_UINT32_C(0x1) << 1) /*!< Enables GEOM-TA and GEOM-SHG context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN				(IMG_UINT32_C(0x1) << 2) /*!< Enables FRAG DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN				(IMG_UINT32_C(0x1) << 3) /*!< Enables CDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN				(IMG_UINT32_C(0x1) << 4)
+
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM				(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM				(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D				(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM				(IMG_UINT32_C(0x1) << 9)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL				(RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL)
+
+/*!
+ * @} End of Per-OS Context switch configuration flags
+ * @} End of AddToGroup ContextSwitching
+ */
+/*!
+ * @AddToGroup IdleCycleStealing
+ * @{
+ * @Name Per-OS DM Idle Cycle Stealing configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_ICS_TDM_EN				(IMG_UINT32_C(0x1) << 10) /*!< Enables ICS for TDM */
+#define RGXFWIF_INICFG_OS_ICS_GEOM_EN				(IMG_UINT32_C(0x1) << 11) /*!< Enables ICS for GEOM */
+#define RGXFWIF_INICFG_OS_ICS_3D_EN					(IMG_UINT32_C(0x1) << 12) /*!< Enables ICS for FRAG */
+#define RGXFWIF_INICFG_OS_ICS_CDM_EN				(IMG_UINT32_C(0x1) << 13) /*!< Enables ICS for CDM */
+#define RGXFWIF_INICFG_OS_ICS_RDM_EN				(IMG_UINT32_C(0x1) << 14) /*!< Enables ICS for RDM */
+
+#define RGXFWIF_INICFG_OS_ICS_DM_ALL				(RGXFWIF_INICFG_OS_ICS_TDM_EN | \
+													 RGXFWIF_INICFG_OS_ICS_GEOM_EN | \
+													 RGXFWIF_INICFG_OS_ICS_3D_EN | \
+													 RGXFWIF_INICFG_OS_ICS_CDM_EN | \
+													 RGXFWIF_INICFG_OS_ICS_RDM_EN)
+
+#define RGXFWIF_INICFG_OS_ICS_DM_ALL_SHIFT			(10U)
+#define RGXFWIF_INICFG_OS_ICS_DM_APPHINT			(RGXFWIF_INICFG_OS_ICS_CDM_EN >> RGXFWIF_INICFG_OS_ICS_DM_ALL_SHIFT)
+
+#define RGXFWIF_INICFG_OS_ICS_CLRMSK				~(RGXFWIF_INICFG_OS_ICS_DM_ALL)
+/*!
+ * @} End of Per-OS Idle Cycle Stealing configuration flags
+ * @} End of AddToGroup IdleCycleStealing
+ */
+#define RGXFWIF_INICFG_OS_FDTI_PROFILE_LONG			(IMG_UINT32_C(0x1))
+#define RGXFWIF_INICFG_OS_FDTI_PROFILE_MEDIUM		(IMG_UINT32_C(0x2))
+#define RGXFWIF_INICFG_OS_FDTI_PROFILE_SHORT		(IMG_UINT32_C(0x3))
+
+#define RGXFWIF_INICFG_FDTI_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_FDTI_PROFILE_MASK			(IMG_UINT32_C(0x3) << RGXFWIF_INICFG_FDTI_PROFILE_SHIFT)
+
+#define RGXFWIF_INICFG_OS_ALL						(0x3FFFFU)
+
+#define RGXFWIF_FILTCFG_TRUNCATE_HALF					(IMG_UINT32_C(0x1) << 3)
+#define RGXFWIF_FILTCFG_TRUNCATE_INT					(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_FILTCFG_NEW_FILTER_MODE					(IMG_UINT32_C(0x1) << 1)
+
+typedef IMG_UINT32 RGX_ACTIVEPM_CONF;
+#define RGX_ACTIVEPM_FORCE_OFF	0U
+#define RGX_ACTIVEPM_FORCE_ON	1U
+#define RGX_ACTIVEPM_DEFAULT	2U
+
+typedef IMG_UINT32 RGX_RD_POWER_ISLAND_CONF;
+#define RGX_RD_POWER_ISLAND_FORCE_OFF	0U
+#define RGX_RD_POWER_ISLAND_FORCE_ON	1U
+#define RGX_RD_POWER_ISLAND_DEFAULT		2U
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SIGBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TRACEBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SYSDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_OSDATA;
+#if defined(SUPPORT_TBI_INTERFACE)
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TBIBUF;
+#endif
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERFBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRINFOBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RUNTIME_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_GPU_UTIL_FW;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_REG_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERF_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_RTN_SLOTS;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWCOMMONCONTEXT;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_ZSBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CORE_CLK_RATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COUNTERBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FIRMWAREGCOVBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRTDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TIMESTAMP_ADDR;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*!
+ * @Brief Buffer to store KM active client contexts
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+} RGXFWIF_ACTIVE_CONTEXT_BUF_DATA;
+#endif
+
+/*!
+ * This number is used to represent an invalid page catalogue physical address
+ */
+#define RGXFWIF_INVALID_PC_PHYADDR 0xFFFFFFFFFFFFFFFFLLU
+
+/*!
+ * This number is used to represent an unallocated set of page catalog base registers
+ */
+#define RGXFW_BIF_INVALID_PCSET 0xFFFFFFFFU
+
+/*!
+ * This number is used to represent an invalid OS ID for the purpose of tracking PC set ownership
+ */
+#define RGXFW_BIF_INVALID_OSID 0xFFFFFFFFU
+
+#define RGXFWIF_FWMEMCONTEXT_FLAGS_USED_TESS		(0x00000001U)
+
+/*!
+ * Firmware memory context.
+ */
+typedef struct
+{
+	IMG_DEV_PHYADDR			RGXFW_ALIGN sPCDevPAddr;	/*!< device physical address of context's page catalogue */
+	IMG_UINT32				uiPageCatBaseRegSet;		/*!< index of the associated set of page catalog base registers (RGXFW_BIF_INVALID_PCSET == unallocated) */
+	IMG_UINT32				uiBreakpointAddr; /*!< breakpoint address */
+	IMG_UINT32				uiBPHandlerAddr; /*!< breakpoint handler address */
+	IMG_UINT32				uiBreakpointCtl; /*!< DM and enable control for BP */
+	IMG_UINT64				RGXFW_ALIGN ui64FBCStateIDMask;	/*!< FBCDC state descriptor IDs (non-zero means defer on mem context activation) */
+	IMG_UINT64				RGXFW_ALIGN ui64SpillAddr;
+	IMG_UINT32				ui32FwMemCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	IMG_UINT32              ui32OSid;
+	IMG_BOOL                bOSidAxiProt;
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWMEMCONTEXT;
+
+/*!
+ * FW context state flags
+ */
+#define RGXFWIF_CONTEXT_FLAGS_NEED_RESUME			(0x00000001U)
+#define RGXFWIF_CONTEXT_FLAGS_TDM_HEADER_STALE		(0x00000002U)
+#define RGXFWIF_CONTEXT_FLAGS_LAST_KICK_SECURE		(0x00000200U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend state (per GEOM core)
+ */
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN uTAReg_DCE_CMD0;
+	IMG_UINT32				uTAReg_DCE_CMD1;
+	IMG_UINT32				uTAReg_DCE_WRITE;
+	IMG_UINT64	RGXFW_ALIGN uTAReg_DCE_DRAW0;
+	IMG_UINT64	RGXFW_ALIGN uTAReg_DCE_DRAW1;
+	IMG_UINT64				ui64EnabledUnitsMask;
+	IMG_UINT32				uTAReg_GTA_SO_PRIM[4];
+	IMG_UINT16	ui16TACurrentIdx;
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE_PER_GEOM;
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend states for all GEOM cores
+ */
+typedef struct
+{
+	/*! FW-accessible TA state which must be written out to memory on context store */
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[RGX_NUM_GEOM_CORES];
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE;
+
+/* The following defines need to be auto generated using the HW defines
+ * rather than hard coding it */
+#define RGXFWIF_ISP_PIPE_COUNT_MAX		(48)
+#define RGXFWIF_PIPE_COUNT_PER_ISP		(2)
+#define RGXFWIF_IPP_RESUME_REG_COUNT	(1)
+
+#if !defined(__KERNEL__)
+#define RGXFWIF_ISP_COUNT		(RGX_FEATURE_NUM_SPU * RGX_FEATURE_NUM_ISP_PER_SPU)
+#define RGXFWIF_ISP_PIPE_COUNT	(RGXFWIF_ISP_COUNT * RGXFWIF_PIPE_COUNT_PER_ISP)
+#if RGXFWIF_ISP_PIPE_COUNT > RGXFWIF_ISP_PIPE_COUNT_MAX
+#error RGXFWIF_ISP_PIPE_COUNT should not be greater than RGXFWIF_ISP_PIPE_COUNT_MAX
+#endif
+#endif /* !defined(__KERNEL__) */
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware FRAG/3D context suspend state
+ */
+typedef struct
+{
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT32	RGXFW_ALIGN u3DReg_PM_DEALLOCATED_MASK_STATUS;	/*!< Managed by PM HW in the non-interactive mode */
+#endif
+	IMG_UINT32	ui32CtxStateFlags;	/*!< Compatibility and other flags */
+
+	/* FW-accessible ISP state which must be written out to memory on context store */
+	/* au3DReg_ISP_STORE should be the last element of the structure
+	 * as this is an array whose size is determined at runtime
+	 * after detecting the RGX core */
+	IMG_UINT64	RGXFW_ALIGN au3DReg_ISP_STORE[]; /*!< ISP state (per-pipe) */
+} UNCACHED_ALIGN RGXFWIF_3DCTX_STATE;
+
+#define RGXFWIF_CTX_USING_BUFFER_A		(0)
+#define RGXFWIF_CTX_USING_BUFFER_B		(1U)
+
+typedef struct
+{
+	IMG_UINT32  ui32CtxStateFlags;		/*!< Target buffer and other flags */
+	IMG_UINT64  ui64EnabledUnitsMask;
+} RGXFWIF_COMPUTECTX_STATE;
+
+typedef struct
+{
+	bool       bSaved;
+	IMG_UINT64 ui64CheckSum[4];
+}RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY;
+
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_STATS_PENDING   (1U << 0)
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT (1U << 1)
+
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_2D[RGX_TRP_MAX_NUM_CORES][2];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_3D[RGX_TRP_MAX_NUM_CORES][4];
+typedef RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY RGXFWIF_TRP_CHECKSUM_GEOM[RGX_TRP_MAX_NUM_CORES];
+
+typedef struct
+{
+	IMG_UINT32	ui32ExtJobRefToDisableZSStore;
+	IMG_BOOL	bDisableZStore;
+	IMG_BOOL	bDisableSStore;
+} RGXFWIF_DISABLE_ZSSTORE;
+
+#define MAX_ZSSTORE_DISABLE 8
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware render context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTAContext;				/*!< Firmware context for the TA */
+	RGXFWIF_FWCOMMONCONTEXT	s3DContext;				/*!< Firmware context for the 3D */
+
+	RGXFWIF_STATIC_RENDERCONTEXT_STATE sStaticRenderContextState;
+
+	RGXFWIF_DISABLE_ZSSTORE sDisableZSStoreQueue[MAX_ZSSTORE_DISABLE];
+
+	IMG_UINT32			ui32ZSStoreQueueCount;
+	IMG_UINT32			ui32WriteOffsetOfDisableZSStore;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32			ui32FwRenderCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_3D		aui64TRPChecksums3D;	/*!< Used by Firmware to store checksums during 3D WRR */
+	RGXFWIF_TRP_CHECKSUM_GEOM	aui64TRPChecksumsGeom;	/*!< Used by Firmware to store checksums during TA WRR */
+	RGXFWIF_DM			eTRPGeomCoreAffinity; /* !< Represent the DM affinity for pending 2nd TRP pass of GEOM otherwise points RGXFWIF_DM_MAX. */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWRENDERCONTEXT;
+
+/*!
+	Firmware compute context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sCDMContext;				/*!< Firmware context for the CDM */
+
+	RGXFWIF_STATIC_COMPUTECONTEXT_STATE sStaticComputeContextState;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32 ui32ComputeCtxFlags; /*!< Compatibility and other flags */
+
+	IMG_UINT32		aui32WGPChecksum[RGX_WGP_MAX_NUM_CORES];
+} UNCACHED_ALIGN RGXFWIF_FWCOMPUTECONTEXT;
+
+/*!
+	Firmware ray context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sRDMContext;				/*!< Firmware context for the RDM */
+	RGXFWIF_STATIC_RAYCONTEXT_STATE sStaticRayContextState;
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+} UNCACHED_ALIGN RGXFWIF_FWRAYCONTEXT;
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware TDM context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTDMContext;				/*!< Firmware context for the TDM */
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_2D	RGXFW_ALIGN	aui64TRPChecksums2D; /*!< Used by Firmware to store checksums during TDM WRR */
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWTDMCONTEXT;
+
+/*!
+ ******************************************************************************
+ * Defines for CMD_TYPE corruption detection and forward compatibility check
+ *****************************************************************************/
+
+/* CMD_TYPE 32bit contains:
+ * 31:16	Reserved for magic value to detect corruption (16 bits)
+ * 15		Reserved for RGX_CCB_TYPE_TASK (1 bit)
+ * 14:0		Bits available for CMD_TYPEs (15 bits) */
+
+
+/* Magic value to detect corruption */
+#define RGX_CMD_MAGIC_DWORD			IMG_UINT32_C(0x2ABC)
+#define RGX_CMD_MAGIC_DWORD_MASK	(0xFFFF0000U)
+#define RGX_CMD_MAGIC_DWORD_SHIFT	(16U)
+#define RGX_CMD_MAGIC_DWORD_SHIFTED	(RGX_CMD_MAGIC_DWORD << RGX_CMD_MAGIC_DWORD_SHIFT)
+
+/*!
+ * @InGroup KCCBTypes ClientCCBTypes
+ * @Brief Generic CCB control structure
+ */
+typedef struct
+{
+	volatile IMG_UINT32		ui32WriteOffset;		/*!< write offset into array of commands (MUST be aligned to 16 bytes!) */
+	volatile IMG_UINT32		ui32ReadOffset;			/*!< read offset into array of commands */
+	IMG_UINT32				ui32WrapMask;			/*!< Offset wrapping mask (Total capacity of the CCB - 1) */
+} UNCACHED_ALIGN RGXFWIF_CCB_CTL;
+
+/*!
+ * @Defgroup KCCBTypes Kernel CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the KCCB functionality
+ * @{
+ */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PT      (0x1U) /* MMU_CTRL_INVAL_PT_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PD      (0x2U) /* MMU_CTRL_INVAL_PD_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PC      (0x4U) /* MMU_CTRL_INVAL_PC_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x800U) /* MMU_CTRL_INVAL_ALL_CONTEXTS_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (0x0) /* not used */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT (0x4000000U) /* indicates FW should interrupt the host */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_MMUCACHE type command
+ */
+typedef struct
+{
+	IMG_UINT32            ui32CacheFlags;
+	RGXFWIF_DEV_VIRTADDR  sMMUCacheSync;
+	IMG_UINT32            ui32MMUCacheSyncUpdateValue;
+} RGXFWIF_MMUCACHEDATA;
+
+#define RGXFWIF_BPDATA_FLAGS_ENABLE (1U << 0)
+#define RGXFWIF_BPDATA_FLAGS_WRITE  (1U << 1)
+#define RGXFWIF_BPDATA_FLAGS_CTL    (1U << 2)
+#define RGXFWIF_BPDATA_FLAGS_REGS   (1U << 3)
+
+typedef struct
+{
+	PRGXFWIF_FWMEMCONTEXT		psFWMemContext;		/*!< Memory context */
+	IMG_UINT32					ui32BPAddr;			/*!< Breakpoint address */
+	IMG_UINT32					ui32HandlerAddr;	/*!< Breakpoint handler */
+	IMG_UINT32					ui32BPDM;			/*!< Breakpoint control */
+	IMG_UINT32					ui32BPDataFlags;
+	IMG_UINT32					ui32TempRegs;		/*!< Number of temporary registers to overallocate */
+	IMG_UINT32					ui32SharedRegs;		/*!< Number of shared registers to overallocate */
+	IMG_UINT64 RGXFW_ALIGN		ui64SpillAddr;
+	RGXFWIF_DM					eDM;				/*!< DM associated with the breakpoint */
+} RGXFWIF_BPDATA;
+
+#define RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS (RGXFWIF_PRBUFFER_MAXSUPPORTED + 1U) /* +1 is RTDATASET cleanup */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_KICK type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CWoffUpdate;	/*!< Client CCB woff update */
+	IMG_UINT32					ui32CWrapMaskUpdate; /*!< Client CCB wrap mask update after CCCB growth */
+	IMG_UINT32					ui32NumCleanupCtl;		/*!< number of CleanupCtl pointers attached */
+	PRGXFWIF_CLEANUP_CTL		apsCleanupCtl[RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS]; /*!< CleanupCtl structures associated with command */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32					ui32WorkEstCmdHeaderOffset; /*!< offset to the CmdHeader which houses the workload estimation kick data. */
+#endif
+} RGXFWIF_KCCB_CMD_KICK_DATA;
+
+/*!
+ * @Brief Command data for @Ref RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK type command
+ */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_KICK_DATA	sTACmdKickData; /*!< GEOM DM kick command data */
+	RGXFWIF_KCCB_CMD_KICK_DATA	s3DCmdKickData; /*!< FRAG DM kick command data */
+} RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FORCE_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CCBFenceOffset;	/*!< Client CCB fence offset */
+} RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;
+	RGXFWIF_DISABLE_ZSSTORE		sDisableZSStore;
+} RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA;
+
+/*!
+ * @Brief Resource types supported by \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef enum
+{
+	RGXFWIF_CLEANUP_FWCOMMONCONTEXT,		/*!< FW common context cleanup */
+	RGXFWIF_CLEANUP_HWRTDATA,				/*!< FW HW RT data cleanup */
+	RGXFWIF_CLEANUP_FREELIST,				/*!< FW freelist cleanup */
+	RGXFWIF_CLEANUP_ZSBUFFER,				/*!< FW ZS Buffer cleanup */
+} RGXFWIF_CLEANUP_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef struct
+{
+	RGXFWIF_CLEANUP_TYPE			eCleanupType;			/*!< Cleanup type */
+	union {
+		PRGXFWIF_FWCOMMONCONTEXT	psContext;				/*!< FW common context to cleanup */
+		PRGXFWIF_HWRTDATA			psHWRTData;				/*!< HW RT to cleanup */
+		PRGXFWIF_FREELIST			psFreelist;				/*!< Freelist to cleanup */
+		PRGXFWIF_ZSBUFFER			psZSBuffer;				/*!< ZS Buffer to cleanup */
+	} uCleanupData;
+} RGXFWIF_CLEANUP_REQUEST;
+
+/*!
+ * @Brief Type of power requests supported in \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef enum
+{
+	RGXFWIF_POW_OFF_REQ = 1,           /*!< GPU power-off request */
+	RGXFWIF_POW_FORCED_IDLE_REQ,       /*!< Force-idle related request */
+	RGXFWIF_POW_NUM_UNITS_CHANGE,      /*!< Request to change default powered scalable units */
+	RGXFWIF_POW_APM_LATENCY_CHANGE     /*!< Request to change the APM latency period */
+} RGXFWIF_POWER_TYPE;
+
+/*!
+ * @Brief Supported force-idle related requests with \ref RGXFWIF_POW_FORCED_IDLE_REQ type request
+ */
+typedef enum
+{
+	RGXFWIF_POWER_FORCE_IDLE = 1,      /*!< Request to force-idle GPU */
+	RGXFWIF_POWER_CANCEL_FORCED_IDLE,  /*!< Request to cancel a previously successful force-idle transition */
+	RGXFWIF_POWER_HOST_TIMEOUT,        /*!< Notification that host timed-out waiting for force-idle state */
+} RGXFWIF_POWER_FORCE_IDLE_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef struct
+{
+	RGXFWIF_POWER_TYPE					ePowType;					/*!< Type of power request */
+	union
+	{
+		struct
+		{
+			IMG_UINT32					ui32PowUnits;	/*!< New power units state mask */
+			IMG_UINT32					ui32RACUnits;	/*!< New RAC state mask */
+		};
+		IMG_BOOL						bForced;				/*!< If the operation is mandatory */
+		RGXFWIF_POWER_FORCE_IDLE_TYPE	ePowRequestType;		/*!< Type of Request. Consolidating Force Idle, Cancel Forced Idle, Host Timeout */
+	} uPowerReqData;
+} RGXFWIF_POWER_REQUEST;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_SLCFLUSHINVAL type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext; /*!< Context to fence on (only useful when bDMContext == TRUE) */
+	IMG_BOOL    bInval;                 /*!< Invalidate the cache as well as flushing */
+	IMG_BOOL    bDMContext;             /*!< The data to flush/invalidate belongs to a specific DM context */
+	IMG_UINT64	RGXFW_ALIGN ui64Address;	/*!< Optional address of range (only useful when bDMContext == FALSE) */
+	IMG_UINT64	RGXFW_ALIGN ui64Size;		/*!< Optional size of range (only useful when bDMContext == FALSE) */
+} RGXFWIF_SLCFLUSHINVALDATA;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CTRL_TOGGLE = 0,
+	RGXFWIF_HWPERF_CTRL_SET    = 1,
+	RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV = 2
+} RGXFWIF_HWPERF_UPDATE_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG type command
+ */
+typedef struct
+{
+	RGXFWIF_HWPERF_UPDATE_CONFIG eOpCode; /*!< Control operation code */
+	IMG_UINT64	RGXFW_ALIGN	ui64Mask;   /*!< Mask of events to toggle */
+} RGXFWIF_HWPERF_CTRL;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CNTR_NOOP    = 0,   /* No-Op */
+	RGXFWIF_HWPERF_CNTR_ENABLE  = 1,   /* Enable Counters */
+	RGXFWIF_HWPERF_CNTR_DISABLE = 2    /* Disable Counters */
+} RGXFWIF_HWPERF_CNTR_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS type command
+ */
+typedef struct
+{
+	IMG_UINT32                ui32CtrlWord;
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE type command
+ */
+typedef struct
+{
+	IMG_UINT32	ui32NewClockSpeed;			/*!< New clock speed */
+} RGXFWIF_CORECLKSPEEDCHANGE_DATA;
+
+#define RGXFWIF_HWPERF_CTRL_BLKS_MAX	16U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS type command
+ */
+typedef struct
+{
+	bool		bEnable;
+	IMG_UINT32	ui32NumBlocks;                              /*!< Number of block IDs in the array */
+	IMG_UINT16	aeBlockIDs[RGXFWIF_HWPERF_CTRL_BLKS_MAX];   /*!< Array of RGX_HWPERF_CNTBLK_ID values */
+} RGXFWIF_HWPERF_CTRL_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE & \ref RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE type commands
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;				/*!< ZS-Buffer FW address */
+	IMG_BOOL				bDone;								/*!< action backing/unbacking succeeded */
+} RGXFWIF_ZSBUFFER_BACKING_DATA;
+
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE type command
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;				/*!< Freelist FW address */
+	IMG_UINT32				ui32DeltaPages;						/*!< Amount of the Freelist change */
+	IMG_UINT32				ui32NewPages;						/*!< New amount of pages on the freelist (including ready pages) */
+	IMG_UINT32              ui32ReadyPages;                     /*!< Number of ready pages to be held in reserve until OOM */
+} RGXFWIF_FREELIST_GS_DATA;
+
+/* Max freelists must include freelists loaded (for all kick IDs) and freelists being setup. */
+#define RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT         (MAX_HW_TA3DCONTEXTS * RGXFW_MAX_FREELISTS * 3U)
+#define RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG 0x80000000U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE type command
+ */
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+} RGXFWIF_FREELISTS_RECONSTRUCTION_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT  psContext; /*!< Context to that may need to be resumed following write offset update */
+} UNCACHED_ALIGN RGXFWIF_WRITE_OFFSET_UPDATE_DATA;
+
+/*!
+ ******************************************************************************
+ * Proactive DVFS Structures
+ *****************************************************************************/
+#define NUM_OPP_LEVELS 16
+
+typedef struct
+{
+	IMG_UINT32			ui32Volt; /* V  */
+	IMG_UINT32			ui32Freq; /* Hz */
+} UNCACHED_ALIGN OPP_LEVEL;
+
+typedef struct
+{
+	OPP_LEVEL		asOPPValues[NUM_OPP_LEVELS];
+#if defined(DEBUG)
+	IMG_UINT32		ui32MinOPPPoint;
+#endif
+	IMG_UINT32		ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_OPP_INFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MAX_FREQ_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32MinOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MIN_FREQ_DATA;
+
+/*!
+ ******************************************************************************
+ * Register configuration structures
+ *****************************************************************************/
+
+#define RGXFWIF_REG_CFG_MAX_SIZE 512
+
+typedef enum
+{
+	RGXFWIF_REGCFG_CMD_ADD				= 101,
+	RGXFWIF_REGCFG_CMD_CLEAR			= 102,
+	RGXFWIF_REGCFG_CMD_ENABLE			= 103,
+	RGXFWIF_REGCFG_CMD_DISABLE			= 104
+} RGXFWIF_REGDATA_CMD_TYPE;
+
+typedef struct
+{
+	IMG_UINT64		ui64Addr;
+	IMG_UINT64		ui64Mask;
+	IMG_UINT64		ui64Value;
+} RGXFWIF_REG_CFG_REC;
+
+typedef struct
+{
+	RGXFWIF_REGDATA_CMD_TYPE         eCmdType;
+	RGXFWIF_REG_CFG_TYPE             eRegConfigType;
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  sRegConfig;
+
+} RGXFWIF_REGCONFIG_DATA;
+
+typedef struct
+{
+	/**
+	 * PDump WRW command write granularity is 32 bits.
+	 * Add padding to ensure array size is 32 bit granular.
+	 */
+	IMG_UINT8           RGXFW_ALIGN  aui8NumRegsType[PVR_ALIGN((IMG_UINT32)RGXFWIF_REG_CFG_TYPE_ALL,sizeof(IMG_UINT32))];
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  asRegConfigs[RGXFWIF_REG_CFG_MAX_SIZE];
+} UNCACHED_ALIGN RGXFWIF_REG_CFG;
+
+typedef enum
+{
+	RGXFWIF_OS_ONLINE = 1,
+	RGXFWIF_OS_OFFLINE
+} RGXFWIF_OS_STATE_CHANGE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE type command
+ */
+typedef struct
+{
+	IMG_UINT32 ui32DriverID;
+	RGXFWIF_OS_STATE_CHANGE eNewOSState;
+} UNCACHED_ALIGN RGXFWIF_OS_STATE_CHANGE_DATA;
+
+typedef enum
+{
+	RGXFWIF_PWR_COUNTER_DUMP_START = 1,
+	RGXFWIF_PWR_COUNTER_DUMP_STOP,
+	RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+} RGXFWIF_COUNTER_DUMP_REQUEST;
+
+typedef struct
+{
+	RGXFWIF_COUNTER_DUMP_REQUEST eCounterDumpRequest;
+}  RGXFW_ALIGN RGXFWIF_COUNTER_DUMP_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext;
+	IMG_UINT32               ui32FirstIntJobRefToCancel;
+	IMG_UINT32               ui32LastIntJobRefToCancel;
+} UNCACHED_ALIGN RGXFWIF_CANCEL_WORK_DATA;
+
+/*!
+ ******************************************************************************
+ * Platform configuration structures
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32 ui32PlatformCmd;
+
+	/* !! Customisable properties/values follow !! */
+	IMG_UINT32 ui32ClockFrequencyExample;
+} UNCACHED_ALIGN RGXFWIF_PLATFORM_DATA;
+
+
+/*!
+ * @Brief List of command types supported by the Kernel CCB
+ */
+typedef enum
+{
+	/* Common commands */
+	RGXFWIF_KCCB_CMD_KICK								= 101U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< DM workload kick command */
+	RGXFWIF_KCCB_CMD_MMUCACHE							= 102U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< MMU cache invalidation request */
+	RGXFWIF_KCCB_CMD_BP									= 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_SLCFLUSHINVAL						= 104U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< SLC flush and invalidation request */
+	RGXFWIF_KCCB_CMD_CLEANUP							= 105U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Requests cleanup of a FW resource (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE			= 106U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Backing for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE			= 107U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Unbacking for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE				= 108U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelist Grow done */
+	RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE	= 109U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelists Reconstruction done */
+	RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE			= 110U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the firmware that the host has added more data to a CDM2 Circular Buffer */
+	RGXFWIF_KCCB_CMD_HEALTH_CHECK						= 111U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Health check request */
+	RGXFWIF_KCCB_CMD_FORCE_UPDATE						= 112U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Forcing signalling of all unmet UFOs for a given CCB offset */
+	RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK				= 113U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< There is a TA and a 3D command in this single kick */
+	RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE			= 114U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW that a Guest OS has come online / offline. */
+	RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE					= 115U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW to disable zs store of a running 3D or add it to queue of render context. */
+	RGXFWIF_KCCB_CMD_CANCEL_WORK						= 116U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Cancel all work up to and including a given intjobref for a given context */
+	/* Commands only permitted to the native or host OS */
+	RGXFWIF_KCCB_CMD_POW								= 200U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Power request (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_REGCONFIG							= 201U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE					= 202U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Core clock speed change event */
+	RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE						= 203U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Ask the firmware to update its cached ui32LogType value from the (shared) tracebuf control structure */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ				= 204U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a maximum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE				= 205U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the priority/group for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL					= 206U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set or clear firmware state flags */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ				= 207U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a minimum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_PHR_CFG							= 208U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Periodic Hardware Reset behaviour */
+	RGXFWIF_KCCB_CMD_WDG_CFG							= 210U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Safety Firmware Watchdog */
+	RGXFWIF_KCCB_CMD_COUNTER_DUMP						= 211U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Controls counter dumping in the FW */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE					= 213U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL			= 214U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice interval for all drivers. It can only be serviced for the Host DDK */
+
+	/* HWPerf commands */
+	RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG				= 300U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure HWPerf events (to be generated) and HWPerf buffer address (if required) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS					= 301U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks */
+	RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS					= 302U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Enable or disable multiple HWPerf blocks (reusing existing configuration) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS_DIRECT	= 303U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks during the init process */
+
+	/* Indirect platform command */
+	/* !! Do not add internal commands to the 4xx range !! */
+	RGXFWIF_KCCB_CMD_PLATFORM_CMD						= 400U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Send a platform specific command */
+
+} RGXFWIF_KCCB_CMD_TYPE;
+
+#define RGXFWIF_LAST_ALLOWED_GUEST_KCCB_CMD (RGXFWIF_KCCB_CMD_POW - 1)
+
+/*! @Brief Kernel CCB command packet */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_TYPE  eCmdType;      /*!< Command type */
+	IMG_UINT32             ui32KCCBFlags; /*!< Compatibility and other flags */
+
+	/* NOTE: Make sure that uCmdData is the last member of this struct
+	 * This is to calculate actual command size for device mem copy.
+	 * (Refer RGXGetCmdMemCopySize())
+	 * */
+	union
+	{
+		RGXFWIF_KCCB_CMD_KICK_DATA			sCmdKickData;			/*!< Data for Kick command */
+		RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA	sCombinedTA3DCmdKickData;	/*!< Data for combined TA/3D Kick command */
+		RGXFWIF_MMUCACHEDATA				sMMUCacheData;			/*!< Data for MMU cache command */
+		RGXFWIF_BPDATA						sBPData;				/*!< Data for Breakpoint Commands */
+		RGXFWIF_SLCFLUSHINVALDATA			sSLCFlushInvalData;		/*!< Data for SLC Flush/Inval commands */
+		RGXFWIF_CLEANUP_REQUEST				sCleanupData;			/*!< Data for cleanup commands */
+		RGXFWIF_POWER_REQUEST				sPowData;				/*!< Data for power request commands */
+		RGXFWIF_HWPERF_CTRL					sHWPerfCtrl;			/*!< Data for HWPerf control command */
+		RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS	sHWPerfCfgEnableBlks;	/*!< Data for HWPerf configure, clear and enable performance counter block command */
+		RGXFWIF_HWPERF_CTRL_BLKS			sHWPerfCtrlBlks;		/*!< Data for HWPerf enable or disable performance counter block commands */
+		RGXFWIF_CORECLKSPEEDCHANGE_DATA		sCoreClkSpeedChangeData;/*!< Data for core clock speed change */
+		RGXFWIF_ZSBUFFER_BACKING_DATA		sZSBufferBackingData;	/*!< Feedback for Z/S Buffer backing/unbacking */
+		RGXFWIF_FREELIST_GS_DATA			sFreeListGSData;		/*!< Feedback for Freelist grow/shrink */
+		RGXFWIF_FREELISTS_RECONSTRUCTION_DATA	sFreeListsReconstructionData;	/*!< Feedback for Freelists reconstruction */
+		RGXFWIF_REGCONFIG_DATA				sRegConfigData;			/*!< Data for custom register configuration */
+		RGXFWIF_WRITE_OFFSET_UPDATE_DATA    sWriteOffsetUpdateData; /*!< Data for informing the FW about the write offset update */
+#if defined(SUPPORT_PDVFS)
+		RGXFWIF_PDVFS_MAX_FREQ_DATA			sPDVFSMaxFreqData;
+		RGXFWIF_PDVFS_MIN_FREQ_DATA			sPDVFSMinFreqData;		/*!< Data for setting the min frequency/OPP */
+#endif
+		RGXFWIF_OS_STATE_CHANGE_DATA        sCmdOSOnlineStateData;  /*!< Data for updating the Guest Online states */
+		RGXFWIF_DEV_VIRTADDR                sTBIBuffer;             /*!< Dev address for TBI buffer allocated on demand */
+		RGXFWIF_COUNTER_DUMP_DATA			sCounterDumpConfigData; /*!< Data for dumping of register ranges */
+		RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA	sDisableZSStoreData;	/*!< Data for disabling zs store of a 3D workload */
+		RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA  sForceUpdateData;       /*!< Data for signalling all unmet fences for a given CCB */
+		RGXFWIF_CANCEL_WORK_DATA			sCancelWorkData;		/*!< Data for cancelling work */
+		RGXFWIF_PLATFORM_DATA               sPlatformData;          /*!< Data for sending platform specific parameters */
+	} UNCACHED_ALIGN uCmdData;
+} UNCACHED_ALIGN RGXFWIF_KCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_KCCB_CMD);
+
+/*! @} End of KCCBTypes */
+
+/*!
+ * @Defgroup FWCCBTypes Firmware CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the Firmware CCB functionality
+ * @{
+ */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING and the
+ * \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING Firmware CCB commands
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32ZSBufferID; /*!< ZS buffer ID */
+} RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELIST_GROW Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32FreelistID; /*!< Freelist ID */
+} RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;                                     /*!< Freelists count */
+	IMG_UINT32			ui32HwrCounter;                                         /*!< HWR counter */
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT]; /*!< Array of freelist IDs to reconstruct */
+} RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA;
+
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF			(1U<<0)	/*!< 1 if a page fault happened */
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS	(1U<<1)	/*!< 1 if applicable to all contexts */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32						ui32ServerCommonContextID;	/*!< Context affected by the reset */
+	RGX_CONTEXT_RESET_REASON		eResetReason;				/*!< Reason for reset */
+	RGXFWIF_DM						eDM;						/*!< Data Master affected by the reset */
+	IMG_UINT32						ui32ResetJobRef;			/*!< Job ref running at the time of reset */
+	IMG_UINT32						ui32Flags;					/*!< RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG bitfield */
+	IMG_UINT64 RGXFW_ALIGN			ui64PCAddress;				/*!< At what page catalog address */
+	IMG_DEV_VIRTADDR RGXFW_ALIGN	sFaultAddress;				/*!< Page fault address (only when applicable) */
+} RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;	/*!< Page fault address */
+} RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA;
+
+/*!
+ ******************************************************************************
+ * List of command types supported by the Firmware CCB
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING              = 101U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be backed with physical pages
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING            = 102U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be unbacked
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_FREELIST_GROW                 = 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand freelist grow
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA */
+	RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION      = 104U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests freelists reconstruction
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA */
+	RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION    = 105U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a HWR event on a context
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA */
+	RGXFWIF_FWCCB_CMD_DEBUG_DUMP                    = 106U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand debug dump
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS                  = 107U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand update on process stats
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA */
+	RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE          = 108U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART           = 109U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests GPU restart
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION    = 112U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a FW pagefault
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA */
+} RGXFWIF_FWCCB_CMD_TYPE;
+
+/*!
+ ******************************************************************************
+ * List of the various stats of the process to update/increment
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS=1,		/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumPartialRenders stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumOutOfMemory stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTAStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32Num3DStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumCDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumRayStores stat */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_UPDATE_STATS Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE		eElementToUpdate;			/*!< Element to update */
+	IMG_PID									pidOwner;					/*!< The pid of the process whose stats are being updated */
+	IMG_INT32								i32AdjustmentValue;			/*!< Adjustment to be made to the statistic */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32CoreClkRate;
+} UNCACHED_ALIGN RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA;
+
+
+/*!
+ ******************************************************************************
+ * @Brief Firmware CCB command structure
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_TYPE  eCmdType;       /*!< Command type */
+	IMG_UINT32              ui32FWCCBFlags; /*!< Compatibility and other flags */
+
+	union
+	{
+		RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA				sCmdZSBufferBacking;			/*!< Data for Z/S-Buffer on-demand (un)backing*/
+		RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA					sCmdFreeListGS;					/*!< Data for on-demand freelist grow/shrink */
+		RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA		sCmdFreeListsReconstruction;	/*!< Data for freelists reconstruction */
+		RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA				sCmdContextResetNotification;	/*!< Data for context reset notification */
+		RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA					sCmdUpdateStatsData;			/*!< Data for updating process stats */
+		RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA			sCmdCoreClkRateChange;
+		RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA					sCmdFWPagefault;				/*!< Data for context reset notification */
+	} RGXFW_ALIGN uCmdData;
+} RGXFW_ALIGN RGXFWIF_FWCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_FWCCB_CMD);
+
+/*! @} End of FWCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Workload estimation Firmware CCB command structure for RGX
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT16 ui16ReturnDataIndex; /*!< Index for return data array */
+	IMG_UINT16 ui16CyclesTakenHigh;	/*!< [39:32] from the cycle counter, which has up to 40-bit range. */
+	IMG_UINT32 ui32CyclesTaken;     /*!< The cycles the workload took on the hardware */
+} RGXFWIF_WORKEST_FWCCB_CMD;
+
+/*!
+ * @Defgroup ClientCCBTypes Client CCB data interface
+ * @Brief Types grouping data structures and defines used in realising Client CCB commands/functionality
+ * @{
+ */
+
+/* Required memory alignment for 64-bit variables accessible by Meta
+  (The gcc meta aligns 64-bit variables to 64-bit; therefore, memory shared
+   between the host and meta that contains 64-bit variables has to maintain
+   this alignment) */
+#define RGXFWIF_FWALLOC_ALIGN	sizeof(IMG_UINT64)
+
+#define RGX_CCB_TYPE_TASK			(IMG_UINT32_C(1) << 15)
+#define RGX_CCB_FWALLOC_ALIGN(size)	(PVR_ALIGN(size, RGXFWIF_FWALLOC_ALIGN))
+
+typedef IMG_UINT32 RGXFWIF_CCB_CMD_TYPE;
+
+/*!
+ * @Name Client CCB command types
+ * @{
+ */
+#define RGXFWIF_CCB_CMD_TYPE_GEOM			(201U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TA DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_3D			(202U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_3D				(203U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command */
+#define RGXFWIF_CCB_CMD_TYPE_3D_PR			(204U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for Partial render */
+#define RGXFWIF_CCB_CMD_TYPE_CDM			(205U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Compute DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_TDM			(206U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TDM command */
+#define RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE (207U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_TQ_2D			(208U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 2D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP	(209U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_NULL			(210U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_ABORT			(211U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+
+/* Leave a gap between CCB specific commands and generic commands */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE          (212U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a command */
+#define RGXFWIF_CCB_CMD_TYPE_UPDATE         (213U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates of a command */
+#define RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE     (214U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates related to workload resources */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE_PR       (215U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a PR command */
+#define RGXFWIF_CCB_CMD_TYPE_PRIORITY       (216U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Context priority update command */
+/* Pre and Post timestamp commands are supposed to sandwich the DM cmd. The
+   padding code with the CCB wrap upsets the FW if we don't have the task type
+   bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+*/
+#define RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP (217U | RGX_CMD_MAGIC_DWORD_SHIFTED)
+/* UNFENCED type is not supported but keeping defines as it is for now */
+
+
+#define RGXFWIF_CCB_CMD_TYPE_PADDING	(221U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Skip without action type command */
+#define RGXFWIF_CCB_CMD_TYPE_RAY		(222U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP	(223U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Process a vulkan timestamp */
+/*! @} End of Client CCB command types */
+
+
+#define RGXFWIF_TRP_STATUS_UNKNOWN				0x000U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_OK			0x001U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_ERROR		0x002U
+
+#define RGXFWIF_CR_TRP_SIGNATURE_STATUS			(RGX_CR_SCRATCH10)
+
+
+typedef struct
+{
+	/* Index for the KM Workload estimation return data array */
+	IMG_UINT16 RGXFW_ALIGN         ui16ReturnDataIndex;
+	/* Predicted time taken to do the work in cycles */
+	IMG_UINT32 RGXFW_ALIGN         ui32CyclesPrediction;
+	/* Deadline for the workload (in usecs) */
+	IMG_UINT64 RGXFW_ALIGN         ui64Deadline;
+} RGXFWIF_WORKEST_KICK_DATA;
+
+/*! @Brief Command header of a command in the client CCB buffer.
+ *
+ *  Followed by this header is the command-data specific to the
+ *  command-type as specified in the header.
+ */
+typedef struct
+{
+	RGXFWIF_CCB_CMD_TYPE					eCmdType;      /*!< Command data type following this command header */
+	IMG_UINT32								ui32CmdSize;   /*!< Size of the command following this header */
+	IMG_UINT32								ui32ExtJobRef; /*!< external job reference - provided by client and used in debug for tracking submitted work */
+	IMG_UINT32								ui32IntJobRef; /*!< internal job reference - generated by services and used in debug for tracking submitted work */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA RGXFW_ALIGN	sWorkEstKickData; /*!< Workload Estimation - Workload Estimation Data */
+#endif
+} RGXFWIF_CCB_CMD_HEADER;
+
+/*
+ ******************************************************************************
+ * Client CCB commands which are only required by the kernel
+ *****************************************************************************/
+
+/*! @Brief Command data for \ref RGXFWIF_CCB_CMD_TYPE_PRIORITY type client CCB command */
+typedef struct
+{
+	IMG_INT32              i32Priority; /*!< Priority level */
+} RGXFWIF_CMD_PRIORITY;
+
+/*! @} End of ClientCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Signature and Checksums Buffer
+ *****************************************************************************/
+typedef struct
+{
+	PRGXFWIF_SIGBUFFER		sBuffer;			/*!< Ptr to Signature Buffer memory */
+	IMG_UINT32				ui32LeftSizeInRegs;	/*!< Amount of space left for storing regs in the buffer */
+} UNCACHED_ALIGN RGXFWIF_SIGBUF_CTL;
+
+typedef struct
+{
+	PRGXFWIF_COUNTERBUFFER	sBuffer;			/*!< Ptr to counter dump buffer */
+	IMG_UINT32				ui32SizeInDwords;	/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_COUNTER_DUMP_CTL;
+
+typedef struct
+{
+	PRGXFWIF_FIRMWAREGCOVBUFFER	sBuffer;		/*!< Ptr to firmware gcov buffer */
+	IMG_UINT32					ui32Size;		/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_FIRMWARE_GCOV_CTL;
+
+/*!
+ *****************************************************************************
+ * RGX Compatibility checks
+ *****************************************************************************/
+
+/* WARNING: Whenever the layout of RGXFWIF_COMPCHECKS_BVNC changes, the
+	following define should be increased by 1 to indicate to the
+	compatibility logic that layout has changed. */
+#define RGXFWIF_COMPCHECKS_LAYOUT_VERSION 3
+
+typedef struct
+{
+	IMG_UINT32	ui32LayoutVersion; /* WARNING: This field must be defined as first one in this structure */
+	IMG_UINT64	RGXFW_ALIGN ui64BVNC;
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS_BVNC;
+
+typedef struct
+{
+	IMG_UINT8	ui8OsCountSupport;
+} UNCACHED_ALIGN RGXFWIF_INIT_OPTIONS;
+
+#define RGXFWIF_COMPCHECKS_BVNC_DECLARE_AND_INIT(name) \
+	RGXFWIF_COMPCHECKS_BVNC (name) = { \
+		RGXFWIF_COMPCHECKS_LAYOUT_VERSION, \
+		0, \
+	}
+#define RGXFWIF_COMPCHECKS_BVNC_INIT(name) \
+	do { \
+		(name).ui32LayoutVersion = RGXFWIF_COMPCHECKS_LAYOUT_VERSION; \
+		(name).ui64BVNC = 0; \
+	} while (false)
+
+typedef struct
+{
+	RGXFWIF_COMPCHECKS_BVNC		sHWBVNC;				/*!< hardware BVNC (from the RGX registers) */
+	RGXFWIF_COMPCHECKS_BVNC		sFWBVNC;				/*!< firmware BVNC */
+	IMG_UINT32					ui32DDKVersion;			/*!< software DDK version */
+	IMG_UINT32					ui32DDKBuild;			/*!< software DDK build no. */
+	IMG_UINT32					ui32BuildOptions;		/*!< build options bit-field */
+	RGXFWIF_INIT_OPTIONS		sInitOptions;			/*!< initialisation options bit-field */
+	IMG_BOOL					bUpdated;				/*!< Information is valid */
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS;
+
+typedef struct
+{
+	IMG_UINT32                 ui32NumCores;
+	IMG_UINT32                 ui32MulticoreInfo;
+} UNCACHED_ALIGN RGXFWIF_MULTICORE_INFO;
+
+/*! @Brief Firmware Runtime configuration data \ref RGXFWIF_RUNTIME_CFG
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_UINT32         ui32ActivePMLatencyms;                    /*!< APM latency in ms before signalling IDLE to the host */
+	IMG_UINT32         ui32RuntimeCfgFlags;                      /*!< Compatibility and other flags */
+	IMG_BOOL           bActivePMLatencyPersistant;               /*!< If set, APM latency does not reset to system default each GPU power transition */
+	IMG_UINT32         ui32CoreClockSpeed;                       /*!< Core clock speed, currently only used to calculate timer ticks */
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32         ui32SOCClockSpeed;                        /*!< SOC clock speed, used for deadline scheduling */
+#endif
+	IMG_UINT32         ui32PowUnitsState;                        /*!< Power Unit state mask set by the host */
+	IMG_UINT32         ui32RACUnitsState;                        /*!< RAC state mask set by the host */
+	IMG_UINT32         ui32PHRMode;                              /*!< Periodic Hardware Reset configuration values */
+	IMG_UINT32         ui32HCSDeadlineMS;                        /*!< New number of milliseconds C/S is allowed to last */
+	IMG_UINT32         ui32WdgPeriodUs;                          /*!< The watchdog period in microseconds */
+	IMG_INT32          ai32DriverPriority[RGXFW_MAX_NUM_OSIDS];  /*!< Array of priorities per OS */
+	IMG_UINT32         aui32DriverIsolationGroup[RGXFW_MAX_NUM_OSIDS]; /*!< Array of isolation groups per OS */
+	IMG_UINT32         aui32TSPercentage[RGXFW_MAX_NUM_OSIDS];   /*!< Array of time slice per OS */
+	IMG_UINT32         ui32TSIntervalMs;                         /*!< Time slice interval */
+	IMG_UINT32         ui32VzConnectionCooldownPeriodInSec;      /*!< Vz Connection Cooldown period in secs */
+
+	PRGXFWIF_HWPERFBUF sHWPerfBuf;                               /*!< On-demand allocated HWPerf buffer address, to be passed to the FW */
+	RGXFWIF_DMA_ADDR   sHWPerfDMABuf;
+	RGXFWIF_DMA_ADDR   sHWPerfCtlDMABuf;
+} RGXFWIF_RUNTIME_CFG;
+
+/*!
+ *****************************************************************************
+ * Control data for RGX
+ *****************************************************************************/
+
+#define RGXFWIF_HWR_DEBUG_DUMP_ALL (99999U)
+
+#if defined(PDUMP)
+
+#define RGXFWIF_PID_FILTER_MAX_NUM_PIDS 32U
+
+typedef enum
+{
+	RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+	RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT
+} RGXFWIF_PID_FILTER_MODE;
+
+typedef struct
+{
+	IMG_PID uiPID;
+	IMG_UINT32 ui32DriverID;
+} RGXFW_ALIGN RGXFWIF_PID_FILTER_ITEM;
+
+typedef struct
+{
+	RGXFWIF_PID_FILTER_MODE eMode;
+	/* each process in the filter list is specified by a PID and OS ID pair.
+	 * each PID and OS pair is an item in the items array (asItems).
+	 * if the array contains less than RGXFWIF_PID_FILTER_MAX_NUM_PIDS entries
+	 * then it must be terminated by an item with pid of zero.
+	 */
+	RGXFWIF_PID_FILTER_ITEM asItems[RGXFWIF_PID_FILTER_MAX_NUM_PIDS];
+} RGXFW_ALIGN RGXFWIF_PID_FILTER;
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA  (0x1U << 0)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE  (0x1U << 1)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE  (0x1U << 2)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE     (0x1U << 3)
+#endif
+
+typedef enum
+{
+	RGXFWIF_USRM_DM_VDM = 0,
+	RGXFWIF_USRM_DM_DDM = 1,
+	RGXFWIF_USRM_DM_CDM = 2,
+	RGXFWIF_USRM_DM_PDM = 3,
+	RGXFWIF_USRM_DM_TDM = 4,
+	RGXFWIF_USRM_DM_LAST
+} RGXFWIF_USRM_DM;
+
+typedef enum
+{
+	RGXFWIF_UVBRM_DM_VDM = 0,
+	RGXFWIF_UVBRM_DM_DDM = 1,
+	RGXFWIF_UVBRM_DM_LAST
+} RGXFWIF_UVBRM_DM;
+
+typedef enum
+{
+	RGXFWIF_TPU_DM_PDM = 0,
+	RGXFWIF_TPU_DM_VDM = 1,
+	RGXFWIF_TPU_DM_CDM = 2,
+	RGXFWIF_TPU_DM_TDM = 3,
+	RGXFWIF_TPU_DM_RDM = 4,
+	RGXFWIF_TPU_DM_LAST
+} RGXFWIF_TPU_DM;
+
+typedef enum
+{
+	RGXFWIF_GPIO_VAL_OFF           = 0, /*!< No GPIO validation */
+	RGXFWIF_GPIO_VAL_GENERAL       = 1, /*!< Simple test case that
+	                                         initiates by sending data via the
+	                                         GPIO and then sends back any data
+	                                         received over the GPIO */
+	RGXFWIF_GPIO_VAL_AP            = 2, /*!< More complex test case that writes
+	                                         and reads data across the entire
+	                                         GPIO AP address range.*/
+#if defined(SUPPORT_STRIP_RENDERING)
+	RGXFWIF_GPIO_VAL_SR_BASIC      = 3, /*!< Strip Rendering AP based basic test.*/
+	RGXFWIF_GPIO_VAL_SR_COMPLEX    = 4, /*!< Strip Rendering AP based complex test.*/
+#endif
+	RGXFWIF_GPIO_VAL_TESTBENCH     = 5, /*!< Validates the GPIO Testbench. */
+	RGXFWIF_GPIO_VAL_LOOPBACK      = 6, /*!< Send and then receive each byte
+	                                         in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LOOPBACK_LITE = 7, /*!< Send and then receive each power-of-2
+	                                         byte in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LAST
+} RGXFWIF_GPIO_VAL_MODE;
+
+typedef IMG_UINT32 FW_PERF_CONF;
+#define FW_PERF_CONF_NONE			0U
+#define FW_PERF_CONF_ICACHE			1U
+#define FW_PERF_CONF_DCACHE			2U
+#define FW_PERF_CONF_JTLB_INSTR		5U
+#define FW_PERF_CONF_INSTRUCTIONS	6U
+
+typedef enum
+{
+	FW_BOOT_STAGE_TLB_INIT_FAILURE = -2,
+	FW_BOOT_STAGE_NOT_AVAILABLE = -1,
+	FW_BOOT_NOT_STARTED = 0,
+	FW_BOOT_BLDR_STARTED = 1,
+	FW_BOOT_CACHE_DONE,
+	FW_BOOT_TLB_DONE,
+	FW_BOOT_MAIN_STARTED,
+	FW_BOOT_ALIGNCHECKS_DONE,
+	FW_BOOT_INIT_DONE,
+} FW_BOOT_STAGE;
+
+/*!
+ * @AddToGroup KCCBTypes
+ * @{
+ * @Name Kernel CCB return slot responses
+ * @{
+ * Usage of bit-fields instead of bare integers
+ * allows FW to possibly pack-in several responses for each single kCCB command.
+ */
+
+#define RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED   (1U << 0) /*!< Command executed (return status from FW) */
+#define RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY   (1U << 1) /*!< A cleanup was requested but resource busy */
+#define RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE   (1U << 2) /*!< Poll failed in FW for a HW operation to complete */
+
+#define RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE            0x0U      /*!< Reset value of a kCCB return slot (set by host) */
+/*!
+ * @} End of Name Kernel CCB return slot responses
+ * @} End of AddToGroup KCCBTypes
+ */
+
+/*! @Brief OS connection data \ref RGXFWIF_CONNECTION_CTL allocated
+ * by services and used to track OS state in Firmware and Services
+ **/
+typedef struct
+{
+	/* Fw-Os connection states */
+	volatile RGXFWIF_CONNECTION_FW_STATE eConnectionFwState;    /*!< Firmware-OS connection state */
+	volatile RGXFWIF_CONNECTION_OS_STATE eConnectionOsState;    /*!< Services-OS connection state */
+	volatile IMG_UINT32                  ui32AliveFwToken;      /*!< OS Alive token updated by Firmware */
+	volatile IMG_UINT32                  ui32AliveOsToken;      /*!< OS Alive token updated by Services */
+} UNCACHED_ALIGN RGXFWIF_CONNECTION_CTL;
+
+/*! @Brief Firmware OS Initialization data \ref RGXFWIF_OSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	/* Kernel CCB */
+	PRGXFWIF_CCB_CTL        psKernelCCBCtl; /*!< Kernel CCB Control */
+	PRGXFWIF_CCB            psKernelCCB; /*!<  Kernel CCB */
+	PRGXFWIF_CCB_RTN_SLOTS  psKernelCCBRtnSlots; /*!<  Kernel CCB return slots */
+
+	/* Firmware CCB */
+	PRGXFWIF_CCB_CTL        psFirmwareCCBCtl; /*!<  Firmware CCB control */
+	PRGXFWIF_CCB            psFirmwareCCB; /*!<  Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	PRGXFWIF_CCB_CTL        psWorkEstFirmwareCCBCtl; /*!<  Workload estimation control */
+	PRGXFWIF_CCB            psWorkEstFirmwareCCB; /*!<  Workload estimation buffer */
+
+	PRGXFWIF_HWRINFOBUF     sRGXFWIfHWRInfoBufCtl; /*!<  HWRecoveryInfo control */
+
+	IMG_UINT32              ui32HWRDebugDumpLimit; /*!< Firmware debug dump maximum limit */
+
+	PRGXFWIF_OSDATA         sFwOsData; /*!<  Firmware per-os shared data */
+
+	RGXFWIF_COMPCHECKS      sRGXCompChecks; /*!< Compatibility checks to be populated by the Firmware */
+	RGXFWIF_MULTICORE_INFO  sRGXMulticoreInfo; /*! < Multicore capability info */
+
+} UNCACHED_ALIGN RGXFWIF_OSINIT;
+
+/*! @Brief Firmware System Initialization data \ref RGXFWIF_SYSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_DEV_PHYADDR         RGXFW_ALIGN sFaultPhysAddr; /*!< Fault read address */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSExecBase; /*!< PDS execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sUSCExecBase; /*!< USC execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sFBCDCStateTableBase; /*!< FBCDC bindless texture state table base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sFBCDCLargeStateTableBase;
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sTextureHeapBase; /*!< Texture state base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSIndirectHeapBase; /* Pixel Indirect State base */
+
+	IMG_UINT64              RGXFW_ALIGN ui64HWPerfFilter; /*! Event filter for Firmware events */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sSLC3FenceDevVAddr; /*!< Address to use as a fence when issuing SLC3_CFI */
+
+	IMG_UINT64				RGXFW_ALIGN aui64UVBRMNumRegions[RGXFWIF_UVBRM_DM_LAST];
+	IMG_UINT32              RGXFW_ALIGN aui32TPUTrilinearFracMask[RGXFWIF_TPU_DM_LAST];
+	IMG_UINT32				RGXFW_ALIGN aui32USRMNumRegions[RGXFWIF_USRM_DM_LAST];
+
+	IMG_UINT64				ui64ClkCtrl0;
+	IMG_UINT64				ui64ClkCtrl1;
+	IMG_UINT32				ui32ClkCtrl2;
+
+	IMG_UINT32              ui32FilterFlags;
+
+#if defined(PDUMP)
+	RGXFWIF_SIGBUF_CTL      asSigBufCtl[RGXFWIF_DM_MAX]; /*!< Signature and Checksum Buffers for DMs */
+#endif
+
+	PRGXFWIF_RUNTIME_CFG    sRuntimeCfg; /*!<  Firmware Runtime configuration */
+
+	PRGXFWIF_TRACEBUF       sTraceBufCtl; /*!<  Firmware Trace buffer control */
+	PRGXFWIF_SYSDATA        sFwSysData; /*!< Firmware System shared data */
+#if defined(SUPPORT_TBI_INTERFACE)
+	PRGXFWIF_TBIBUF         sTBIBuf; /*!< Tbi log buffer */
+#endif
+
+	PRGXFWIF_GPU_UTIL_FW    sGpuUtilFWCtl; /*!< GPU utilization buffer */
+	PRGXFWIF_REG_CFG        sRegCfg; /*!< Firmware register user configuration */
+	PRGXFWIF_HWPERF_CTL     sHWPerfCtl; /*!< HWPerf counter block configuration.*/
+
+	RGXFWIF_COUNTER_DUMP_CTL sCounterDumpCtl;
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	RGXFWIF_FIRMWARE_GCOV_CTL sFirmwareGcovCtl; /*!< Firmware gcov buffer control */
+#endif
+
+	RGXFWIF_DEV_VIRTADDR    sAlignChecks; /*!< Array holding Server structures alignment data */
+
+	IMG_UINT32              ui32InitialCoreClockSpeed; /*!< Core clock speed at FW boot time */
+
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32              ui32InitialSOCClockSpeed; /*!< System/SOC clock speed at FW boot time */
+#endif
+
+	IMG_UINT32              ui32InitialActivePMLatencyms; /*!< APM latency in ms before signalling IDLE to the host */
+
+	IMG_BOOL                bFirmwareStarted; /*!< Flag to be set by the Firmware after successful start */
+
+	IMG_UINT32              ui32MarkerVal; /*!< Host/FW Trace synchronisation Partition Marker */
+
+	IMG_UINT32              ui32FirmwareStartedTimeStamp; /*!< Firmware initialization complete time */
+
+	RGXFWIF_DMA_ADDR        sCorememDataStore; /*!< Firmware coremem data */
+
+	FW_PERF_CONF            eFirmwarePerf; /*!< Firmware performance counter config */
+
+#if defined(SUPPORT_FW_OPP_TABLE)
+	RGXFWIF_OPP_INFO        RGXFW_ALIGN sOPPInfo;
+
+	/**
+	 * FW Pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory when running on non primary FW thread
+	 * to communicate to host driver.
+	 */
+	PRGXFWIF_CORE_CLK_RATE  RGXFW_ALIGN sCoreClockRate;
+#endif
+
+#if defined(PDUMP)
+	RGXFWIF_PID_FILTER      sPIDFilter;
+#endif
+
+	RGXFWIF_GPIO_VAL_MODE   eGPIOValidationMode;
+
+	RGX_HWPERF_BVNC         sBvncKmFeatureFlags; /*!< Used in HWPerf for decoding BVNC Features*/
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32              ui32SecurityTestFlags;
+	RGXFWIF_DEV_VIRTADDR    pbSecureBuffer;
+	RGXFWIF_DEV_VIRTADDR    pbNonSecureBuffer;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	RGXFWIF_DEV_VIRTADDR    sActiveContextBufBase; /*!< Active context buffer base */
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	/*
+	 * Used when validation is enabled to allow the host to check
+	 * that MTS sent the correct sideband in response to a kick
+	 * from a given OSes schedule register.
+	 * Testing is enabled if RGXFWIF_KICK_TEST_ENABLED_BIT is set
+	 *
+	 * Set by the host to:
+	 * (osid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT
+	 * reset to 0 by FW when kicked by the given OSid
+	 */
+	IMG_UINT32              ui32OSKickTest;
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_UINT32              ui32VzWdgPeriod;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	/* notify firmware power-up on host-side recovery */
+	IMG_BOOL                bFwHostRecoveryMode;
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	RGXFWIF_DEV_VIRTADDR    pbFwScratchBuf;
+#endif
+#if defined(SUPPORT_ICS)
+	IMG_UINT32 ui32FDTI;
+	IMG_UINT32 ui32ICSThreshold;
+	IMG_BOOL   bTestModeOn;
+#endif
+} UNCACHED_ALIGN RGXFWIF_SYSINIT;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+#define RGXFWIF_KICK_TEST_ENABLED_BIT  0x1
+#define RGXFWIF_KICK_TEST_OSID_SHIFT   0x1
+#endif
+
+/*!
+ *****************************************************************************
+ * Timer correlation shared data and defines
+ *****************************************************************************/
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64OSMonoTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeStamp;
+
+	/* Utility variable used to convert CR timer deltas to OS timer deltas (nS),
+	 * where the deltas are relative to the timestamps above:
+	 * deltaOS = (deltaCR * K) >> decimal_shift, see full explanation below */
+	IMG_UINT64 RGXFW_ALIGN ui64CRDeltaToOSDeltaKNs;
+
+	IMG_UINT32             ui32CoreClockSpeed;
+	IMG_UINT32             ui32Reserved;
+} UNCACHED_ALIGN RGXFWIF_TIME_CORR;
+
+
+/* The following macros are used to help converting FW timestamps to the Host
+ * time domain. On the FW the RGX_CR_TIMER counter is used to keep track of
+ * time; it increments by 1 every 256 GPU clock ticks, so the general
+ * formula to perform the conversion is:
+ *
+ * [ GPU clock speed in Hz, if (scale == 10^9) then deltaOS is in nS,
+ *   otherwise if (scale == 10^6) then deltaOS is in uS ]
+ *
+ *             deltaCR * 256                                   256 * scale
+ *  deltaOS = --------------- * scale = deltaCR * K    [ K = --------------- ]
+ *             GPUclockspeed                                  GPUclockspeed
+ *
+ * The actual K is multiplied by 2^20 (and deltaCR * K is divided by 2^20)
+ * to get some better accuracy and to avoid returning 0 in the integer
+ * division 256000000/GPUfreq if GPUfreq is greater than 256MHz.
+ * This is the same as keeping K as a decimal number.
+ *
+ * The maximum deltaOS is slightly more than 5hrs for all GPU frequencies
+ * (deltaCR * K is more or less a constant), and it's relative to the base
+ * OS timestamp sampled as a part of the timer correlation data.
+ * This base is refreshed on GPU power-on, DVFS transition and periodic
+ * frequency calibration (executed every few seconds if the FW is doing
+ * some work), so as long as the GPU is doing something and one of these
+ * events is triggered then deltaCR * K will not overflow and deltaOS will be
+ * correct.
+ */
+
+#define RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT  (20)
+
+#define RGXFWIF_GET_DELTA_OSTIME_NS(deltaCR, K) \
+	(((deltaCR) * (K)) >> RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT)
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation
+ *****************************************************************************/
+
+/* See rgx_common.h for a list of GPU states */
+#define RGXFWIF_GPU_UTIL_TIME_MASK       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_TIME_MASK32     (IMG_UINT32_C(0xFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+#define RGXFWIF_GPU_UTIL_GET_TIME(word)    ((word) & RGXFWIF_GPU_UTIL_TIME_MASK)
+#define RGXFWIF_GPU_UTIL_GET_STATE(word)   ((word) & RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_GET_TIME32(word)  ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_TIME_MASK32)
+#define RGXFWIF_GPU_UTIL_GET_STATE32(word) ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+/* The OS timestamps computed by the FW are approximations of the real time,
+ * which means they could be slightly behind or ahead the real timer on the Host.
+ * In some cases we can perform subtractions between FW approximated
+ * timestamps and real OS timestamps, so we need a form of protection against
+ * negative results if for instance the FW one is a bit ahead of time.
+ */
+#define RGXFWIF_GPU_UTIL_GET_PERIOD(newtime,oldtime) \
+	(((newtime) > (oldtime)) ? ((newtime) - (oldtime)) : 0U)
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME(time) | RGXFWIF_GPU_UTIL_GET_STATE(state))
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD32(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME32(time) | RGXFWIF_GPU_UTIL_GET_STATE32(state))
+
+
+/* The timer correlation array must be big enough to ensure old entries won't be
+ * overwritten before all the HWPerf events linked to those entries are processed
+ * by the MISR. The update frequency of this array depends on how fast the system
+ * can change state (basically how small the APM latency is) and perform DVFS transitions.
+ *
+ * The minimum size is 2 (not 1) to avoid race conditions between the FW reading
+ * an entry while the Host is updating it. With 2 entries in the worst case the FW
+ * will read old data, which is still quite ok if the Host is updating the timer
+ * correlation at that time.
+ */
+#define RGXFWIF_TIME_CORR_ARRAY_SIZE            256U
+#define RGXFWIF_TIME_CORR_CURR_INDEX(seqcount)  ((seqcount) % RGXFWIF_TIME_CORR_ARRAY_SIZE)
+
+/* Make sure the timer correlation array size is a power of 2 */
+static_assert((RGXFWIF_TIME_CORR_ARRAY_SIZE & (RGXFWIF_TIME_CORR_ARRAY_SIZE - 1U)) == 0U,
+			  "RGXFWIF_TIME_CORR_ARRAY_SIZE must be a power of two");
+
+/* The time is stored in DM state counters in "approximately microseconds",
+ * dividing the time originally obtained in nanoseconds by 2^10 for the sake of reducing coremem usage */
+#define RGXFWIF_DM_OS_TIMESTAMP_SHIFT    10U
+
+typedef struct
+{
+	/* Last GPU DM per-OS states + OS time of the last state update */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWord[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* DMs time-stamps are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWordWrap[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* Counters for the amount of time the GPU DMs were active or inactive(idle or blocked) */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSStatsCounters[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	/* DMs Counters are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSCountersWrap[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+} RGXFWIF_GPU_STATS;
+
+typedef struct
+{
+	RGXFWIF_TIME_CORR      sTimeCorr[RGXFWIF_TIME_CORR_ARRAY_SIZE];
+	IMG_UINT32             ui32TimeCorrSeqCount;
+
+	/* Compatibility and other flags */
+	IMG_UINT32             ui32GpuUtilFlags;
+
+	/* Last GPU state + OS time of the last state update */
+	IMG_UINT64 RGXFW_ALIGN ui64GpuLastWord;
+	/* Counters for the amount of time the GPU was active/idle/blocked */
+	IMG_UINT64 RGXFW_ALIGN aui64GpuStatsCounters[RGXFWIF_GPU_UTIL_STATE_NUM];
+	/* Device off period timestamp offset */
+	IMG_INT64 RGXFW_ALIGN i64DeviceTimestampOffset;
+	/* Stats per OSID/DriverID */
+	RGXFWIF_GPU_STATS sStats[RGXFW_MAX_NUM_OSIDS];
+} UNCACHED_ALIGN RGXFWIF_GPU_UTIL_FW;
+
+/*!
+ ******************************************************************************
+ * Virtualisation and Security
+ *****************************************************************************/
+
+#define RGX_GP_SUPPORTS_SECURITY		(0U)
+#define RGX_TDM_SUPPORTS_SECURITY		(1U)
+#define RGX_GEOM_SUPPORTS_SECURITY		(0U)
+#define RGX_FRAG_SUPPORTS_SECURITY		(1U)
+#define RGX_COMPUTE_SUPPORTS_SECURITY	(1U)
+#define RGX_RAY_SUPPORTS_SECURITY		(0U)
+
+#define RGX_DMS_WITH_SECURITY_COUNT		(RGX_GP_SUPPORTS_SECURITY		+ \
+										 RGX_TDM_SUPPORTS_SECURITY		+ \
+										 RGX_GEOM_SUPPORTS_SECURITY		+ \
+										 RGX_FRAG_SUPPORTS_SECURITY		+ \
+										 RGX_COMPUTE_SUPPORTS_SECURITY	+ \
+										 RGX_RAY_SUPPORTS_SECURITY)
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_DEDICATED_OSID) || (defined(RGX_LEGACY_SECURE_OSID_SCHEME))
+/* OSIDs 0 and 1 reserved for Firmware */
+#define DRIVER_OSID_START_OFFSET		(FW_OSID+2)
+#else
+/* OSIDs 1 reserved for Firmware */
+#define DRIVER_OSID_START_OFFSET		(FW_OSID+1)
+#endif
+
+#else
+/* Firmware and Host driver share the same OSID */
+#define DRIVER_OSID_START_OFFSET		(FW_OSID)
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_DEDICATED_OSID)
+/* Firmware heaps reside in a dedicated non-secure IPA space. */
+#define FW_HEAP_OSID					(FW_OSID+1)
+#elif (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_HOST_OSID)
+/* Firmware heaps reside in the Host driver's non-secure IPA space. */
+#define FW_HEAP_OSID					(DRIVER_OSID_START_OFFSET)
+#elif (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID)
+/* Firmware heaps reside in the IPA space as the Firmware. */
+#define FW_HEAP_OSID					(FW_OSID)
+#else
+#error "RGX_FW_HEAP_OSID_ASSIGNMENT not configured correctly."
+#endif
+
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) && !defined(__KERNEL__)
+/* placeholder define for UM tools including this header */
+#define RGX_NUM_DRIVERS_SUPPORTED (1U)
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+
+#if defined(RGX_LEGACY_SECURE_OSID_SCHEME)
+/* keep reverse compatibility with original scheme */
+static_assert((RGX_NUM_DRIVERS_SUPPORTED == 1),
+			  "RGX_LEGACY_SECURE_OSID_SCHEME only applicable on native drivers (no VZ support)");
+
+#define DRIVER_ID(osid)					(0U)
+#define OSID_SECURE(did)				(1U)
+#define OSID(did)						(2U)
+
+#else /* defined(RGX_LEGACY_SECURE_OSID_SCHEME) */
+/* native and virtualized security support */
+#define LAST_OSID						(RGXFW_MAX_NUM_OSIDS - 1)
+#define DRIVER_ID(osid)					(osid - DRIVER_OSID_START_OFFSET)
+#define OSID(did)						(did + DRIVER_OSID_START_OFFSET)
+#define OSID_SECURE(did) \
+			(did==0 ? (DRIVER0_SECURITY_SUPPORT ? (LAST_OSID) : OSID(did)) : \
+			(did==1 ? (DRIVER1_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==2 ? (DRIVER2_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==3 ? (DRIVER3_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==4 ? (DRIVER4_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==5 ? (DRIVER5_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT - DRIVER4_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==6 ? (DRIVER6_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT - DRIVER4_SECURITY_SUPPORT - DRIVER5_SECURITY_SUPPORT) : OSID(did)) : \
+			          (DRIVER7_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT - DRIVER4_SECURITY_SUPPORT - DRIVER5_SECURITY_SUPPORT - DRIVER6_SECURITY_SUPPORT) : OSID(did)))))))))
+
+static_assert((RGX_NUM_DRIVERS_SUPPORTED + DRIVER_OSID_START_OFFSET +
+				DRIVER0_SECURITY_SUPPORT + DRIVER1_SECURITY_SUPPORT + DRIVER2_SECURITY_SUPPORT + DRIVER3_SECURITY_SUPPORT +
+				DRIVER4_SECURITY_SUPPORT + DRIVER5_SECURITY_SUPPORT + DRIVER6_SECURITY_SUPPORT + DRIVER7_SECURITY_SUPPORT) <= RGXFW_MAX_NUM_OSIDS,
+				"The GPU hardware is not equipped with enough hardware OSIDs to satisfy the requirements.");
+#endif /* defined(RGX_LEGACY_SECURE_OSID_SCHEME) */
+
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT		(30U)
+#define RGXFW_VZ_PRIORITY_MASK			((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX			(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN			(5U)
+
+#elif defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* virtualization without security support */
+#define DRIVER_ID(osid)					(osid - DRIVER_OSID_START_OFFSET)
+#define OSID(did)						(did + DRIVER_OSID_START_OFFSET)
+
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT		(30U)
+#define RGXFW_VZ_PRIORITY_MASK			((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX			(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN			(5U)
+#else
+/* native without security support */
+#define DRIVER_ID(osid)					(0U)
+#define OSID(did)						(did)
+#define RGXFW_VZ_PRIORITY_MASK			(0U)
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) < RGX_NUM_DRIVERS_SUPPORTED; (did)++)
+
+#if defined(__KERNEL__)
+/* Driver implementation */
+#define FOREACH_ACTIVE_DRIVER(devinfo, did)        FOREACH_SUPPORTED_DRIVER(did)                                \
+                                                   {                                                            \
+                                                   if (devinfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[did].bfOsState != RGXFW_CONNECTION_FW_ACTIVE) continue;
+
+#define END_FOREACH_ACTIVE_DRIVER                  }
+
+#else
+/* Firmware implementation */
+#define FOREACH_ACTIVE_DRIVER(did)                 do {                                                                      \
+                                                   unsigned int idx;                                                         \
+                                                   for (idx = 0; idx < RGXFW_NUM_ACTIVE_DRIVERS; idx++)                      \
+                                                   {                                                                         \
+                                                       (did) = gsRGXFWCtl.aui32ActiveDrivers[idx];                           \
+                                                       {
+
+#define END_FOREACH_ACTIVE_DRIVER                  }}} while (false);
+#endif /* defined(__KERNEL__) */
+
+
+#else
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) <= RGXFW_HOST_DRIVER_ID; (did)++)
+
+#define FOREACH_ACTIVE_DRIVER(did)                 FOREACH_SUPPORTED_DRIVER(did)
+#define END_FOREACH_ACTIVE_DRIVER
+
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#define FOREACH_VALIDATION_OSID(osid)              for ((osid)=0; (osid) < GPUVIRT_VALIDATION_NUM_OS; (osid)++)
+#define FOREACH_HW_OSID(osid)                      for ((osid)=0; (osid) < RGXFW_MAX_NUM_OSIDS; (osid)++)
+#define FOREACH_DRIVER_RAW_HEAP(did, _struct, dev) for ((did)=RGX_FIRST_RAW_HEAP_DRIVER_ID; (did) < ((PVRSRV_VZ_MODE_IS(NATIVE, _struct, dev) ? 1 : RGX_NUM_DRIVERS_SUPPORTED)); (did)++)
+
+#define RGXFWIF_TDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_3D_SECURE_IPP_BUF_VADDR                            (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_SR_BUF_VADDR                            PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_OFFSET_BYTES), 1024ULL)
+#define RGXFWIF_CDM_SECURE_SR_B_BUF_VADDR                          PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_OFFSET_BYTES), 1024ULL)
+#define RGXFWIF_CDM_SECURE_CONTEXT_STATE_VADDR                     PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_OFFSET_BYTES), 128ULL)
+#define RGXFWIF_TDM_SECURE_CONTEXT_STATE_VADDR                     PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_OFFSET_BYTES), 128ULL)
+
+#define SAFETY_SELF_TEST_STEP_CLEAR     1
+#define SAFETY_SELF_TEST_STEP_ENABLE    2
+#define SAFETY_SELF_TEST_STEP_TEST      3
+#define SAFETY_SELF_TEST_STEP_DISABLE   4
+
+#endif /* RGX_FWIF_KM_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_shared.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_shared.h
new file mode 100644
index 000000000000..061f718cec62
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_shared.h
@@ -0,0 +1,1159 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures shared by both host client
+                and host server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_SHARED_H)
+#define RGX_FWIF_SHARED_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+#include "devicemem_typedefs.h"
+
+/* Indicates the number of RTDATAs per RTDATASET */
+#if defined(SUPPORT_AGP)
+#if defined(SUPPORT_AGP4)
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  20U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (4U)
+#else
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  12U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (2U)
+#endif
+#else
+#define RGXMKIF_NUM_RTDATAS           2U
+#define RGXMKIF_NUM_GEOMDATAS         1U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  2U  /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (1U)
+#endif
+
+/* Maximum number of UFOs in a CCB command.
+ * The number is based on having 32 sync prims (as originally), plus 32 sync
+ * checkpoints.
+ * Once the use of sync prims is no longer supported, we will retain
+ * the same total (64) as the number of sync checkpoints which may be
+ * supporting a fence is not visible to the client driver and has to
+ * allow for the number of different timelines involved in fence merges.
+ */
+#define RGXFWIF_CCB_CMD_MAX_UFOS			(32U+32U)
+
+/*
+ * This is a generic limit imposed on any DM (TA,3D,CDM,TDM,2D,TRANSFER)
+ * command passed through the bridge.
+ * Just across the bridge in the server, any incoming kick command size is
+ * checked against this maximum limit.
+ * In case the incoming command size is larger than the specified limit,
+ * the bridge call is retired with error.
+ */
+#define RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE	(1024U)
+
+typedef struct
+{
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN psDevVirtAddr;
+	RGXFWIF_DEV_VIRTADDR    pbyFWAddr;
+} UNCACHED_ALIGN RGXFWIF_DMA_ADDR;
+
+typedef IMG_UINT8	RGXFWIF_CCCB;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_UFO_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CLEANUP_CTL;
+
+
+/*!
+ * @InGroup ClientCCBTypes
+ * @Brief Command data for fence & update types Client CCB commands.
+ */
+typedef struct
+{
+	PRGXFWIF_UFO_ADDR	puiAddrUFO; /*!< Address to be checked/updated */
+	IMG_UINT32			ui32Value;  /*!< Value to check-against/update-to */
+} RGXFWIF_UFO;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Track pending and completed workloads of HWRTDATA and ZSBUFFER
+ */
+typedef struct
+{
+	IMG_UINT32			ui32SubmittedCommands;	/*!< Number of commands received by the FW */
+	IMG_UINT32			ui32ExecutedCommands;	/*!< Number of commands executed by the FW */
+} UNCACHED_ALIGN RGXFWIF_CLEANUP_CTL;
+
+#define	RGXFWIF_PRBUFFER_START        IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_ZSBUFFER     IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_MSAABUFFER   IMG_UINT32_C(1)
+#define	RGXFWIF_PRBUFFER_MAXSUPPORTED IMG_UINT32_C(2)
+
+typedef IMG_UINT32 RGXFWIF_PRBUFFER_TYPE;
+
+typedef enum
+{
+	RGXFWIF_PRBUFFER_UNBACKED = 0,
+	RGXFWIF_PRBUFFER_BACKED,
+	RGXFWIF_PRBUFFER_BACKING_PENDING,
+	RGXFWIF_PRBUFFER_UNBACKING_PENDING,
+}RGXFWIF_PRBUFFER_STATE;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief OnDemand Z/S/MSAA Buffers
+ */
+typedef struct
+{
+	IMG_UINT32		ui32BufferID;		/*!< Buffer ID*/
+	IMG_BOOL		bOnDemand;		/*!< Needs On-demand Z/S/MSAA Buffer allocation */
+	RGXFWIF_PRBUFFER_STATE	eState;			/*!< Z/S/MSAA -Buffer state */
+	RGXFWIF_CLEANUP_CTL	sCleanupState;		/*!< Cleanup state */
+	IMG_UINT32		ui32PRBufferFlags;	/*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_PRBUFFER;
+
+/*
+ * Used to share frame numbers across UM-KM-FW,
+ * frame number is set in UM,
+ * frame number is required in both KM for HTB and FW for FW trace.
+ *
+ * May be used to house Kick flags in the future.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32FrameNum; /*!< associated frame number */
+} CMD_COMMON;
+
+/*
+ * TA and 3D commands require set of firmware addresses that are stored in the
+ * Kernel. Client has handle(s) to Kernel containers storing these addresses,
+ * instead of raw addresses. We have to patch/write these addresses in KM to
+ * prevent UM from controlling FW addresses directly.
+ * Typedefs for TA and 3D commands are shared between Client and Firmware (both
+ * single-BVNC). Kernel is implemented in a multi-BVNC manner, so it can't use
+ * TA|3D CMD type definitions directly. Therefore we have a SHARED block that
+ * is shared between UM-KM-FW across all BVNC configurations.
+ */
+typedef struct
+{
+	CMD_COMMON           sCmn;      /*!< Common command attributes */
+	RGXFWIF_DEV_VIRTADDR sHWRTData; /* RTData associated with this command,
+									   this is used for context selection and for storing out HW-context,
+									   when TA is switched out for continuing later */
+
+	RGXFWIF_DEV_VIRTADDR asPRBuffer[RGXFWIF_PRBUFFER_MAXSUPPORTED];	/* Supported PR Buffers like Z/S/MSAA Scratch */
+
+} CMDTA3D_SHARED;
+
+/*!
+ * Client Circular Command Buffer (CCCB) control structure.
+ * This is shared between the Server and the Firmware and holds byte offsets
+ * into the CCCB as well as the wrapping mask to aid wrap around. A given
+ * snapshot of this queue with Cmd 1 running on the GPU might be:
+ *
+ *          Roff                           Doff                 Woff
+ * [..........|-1----------|=2===|=3===|=4===|~5~~~~|~6~~~~|~7~~~~|..........]
+ *            <      runnable commands       ><   !ready to run   >
+ *
+ * Cmd 1    : Currently executing on the GPU data master.
+ * Cmd 2,3,4: Fence dependencies met, commands runnable.
+ * Cmd 5... : Fence dependency not met yet.
+ */
+typedef struct
+{
+	IMG_UINT32  ui32WriteOffset;    /*!< Host write offset into CCB. This
+	                                 *    must be aligned to 16 bytes. */
+	IMG_UINT32  ui32ReadOffset;     /*!< Firmware read offset into CCB.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R!=W */
+	IMG_UINT32  ui32DepOffset;      /*!< Firmware fence dependency offset.
+	                                      Points to commands not ready, i.e.
+	                                      fence dependencies are not met. */
+	IMG_UINT32  ui32WrapMask;       /*!< Offset wrapping mask, total capacity
+	                                      in bytes of the CCB-1 */
+
+	IMG_UINT32  ui32ReadOffset2;     /*!< Firmware 2nd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R2!=W */
+	IMG_UINT32  ui32ReadOffset3;     /*!< Firmware 3rd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R3!=W */
+	IMG_UINT32  ui32ReadOffset4;     /*!< Firmware 4th read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R4!=W */
+
+} UNCACHED_ALIGN RGXFWIF_CCCB_CTL;
+
+
+typedef IMG_UINT32 RGXFW_FREELIST_TYPE;
+
+#define RGXFW_LOCAL_FREELIST     IMG_UINT32_C(0)
+#define RGXFW_GLOBAL_FREELIST    IMG_UINT32_C(1)
+#if defined(SUPPORT_AGP4)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_GLOBAL3_FREELIST   IMG_UINT32_C(3)
+#define RGXFW_GLOBAL4_FREELIST   IMG_UINT32_C(4)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL4_FREELIST + 1U)
+#elif defined(SUPPORT_AGP)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL2_FREELIST + 1U)
+#else
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL_FREELIST + 1U)
+#endif
+#define RGXFW_MAX_HWFREELISTS    (2U)
+
+/*!
+ * @Defgroup ContextSwitching Context switching data interface
+ * @Brief Types grouping data structures and defines used in realising the Context Switching (CSW) functionality
+ * @{
+ */
+
+/*!
+ * @Brief GEOM DM or TA register controls for context switch
+ */
+typedef struct
+{
+	IMG_UINT64	uTAReg_DCE_CONTEXT_STATE_BASE_ADDR;
+	IMG_UINT64	uTAReg_TA_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the TA's context state buffer */
+
+	struct
+	{
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_VDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_VDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_VDM2;
+
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_DDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_DDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_DDM2;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_XFB;
+
+		/* VDM resume state update controls */
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_VDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_VDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_VDM2;
+
+
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_DDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_DDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_DDM2;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_XFB;
+	} asTAState[2];
+
+} RGXFWIF_TAREGISTERS_CSWITCH;
+/*! @} End of Defgroup ContextSwitching */
+
+typedef struct
+{
+	IMG_UINT64	u3DReg_IPP_CONTEXT_ADDR;
+} RGXFWIF_3DREGISTERS_CSWITCH;
+
+typedef struct
+{
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS1;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS1;
+
+	/* CDM resume controls */
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0_B;
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0_B;
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_SECURE_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_SECURE_PDS0_B;
+
+	IMG_UINT64	uCDMReg_CDM_RESUME_SECURE_PDS0;
+	IMG_UINT64	uCDMReg_CDM_RESUME_SECURE_PDS0_B;
+
+	IMG_DEV_VIRTADDR	uCDMReg_CDM_CONTEXT_sSRBuffer;
+	IMG_DEV_VIRTADDR	uCDMReg_CDM_CONTEXT_sSRBuffer_B;
+
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_STATE_BASE_ADDR;
+#endif
+} RGXFWIF_CDM_REGISTERS_CSWITCH;
+
+static_assert((sizeof(RGXFWIF_CDM_REGISTERS_CSWITCH) % 8U) == 0U,
+			"the size of the structure must be multiple of 8");
+
+#define RGXFWIF_CDM_REGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_CDM_REGISTERS_CSWITCH)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Render context static register controls for context switch
+ */
+typedef struct
+{
+	RGXFWIF_TAREGISTERS_CSWITCH	RGXFW_ALIGN asCtxSwitch_GeomRegs[RGX_NUM_GEOM_CORES];
+	RGXFWIF_3DREGISTERS_CSWITCH RGXFW_ALIGN sCtxSwitch_3DRegs;  /*!< 3D registers for ctx switch */
+} RGXFWIF_STATIC_RENDERCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RENDERCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RENDERCONTEXT_STATE)
+
+typedef struct
+{
+	RGXFWIF_CDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< CDM registers for ctx switch */
+} RGXFWIF_STATIC_COMPUTECONTEXT_STATE;
+
+#define RGXFWIF_STATIC_COMPUTECONTEXT_SIZE sizeof(RGXFWIF_STATIC_COMPUTECONTEXT_STATE)
+
+typedef struct
+{
+	IMG_UINT64	uRDMReg_RDM_CONTEXT_STATE_BASE_ADDR;
+} RGXFWIF_RDM_REGISTERS_CSWITCH;
+
+static_assert((sizeof(RGXFWIF_RDM_REGISTERS_CSWITCH) % 8U) == 0U,
+			"the size of the structure must be multiple of 8");
+
+#define RGXFWIF_RDM_REGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_RDM_REGISTERS_CSWITCH)
+
+typedef struct
+{
+	RGXFWIF_RDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< RDM registers for ctx switch */
+} RGXFWIF_STATIC_RAYCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RAYCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RAYCONTEXT_STATE)
+
+/*!
+	@Brief Context reset reason. Last reset reason for a reset context.
+*/
+typedef enum
+{
+	RGX_CONTEXT_RESET_REASON_NONE                = 0,	/*!< No reset reason recorded */
+	RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP       = 1,	/*!< Caused a reset due to locking up */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP     = 2,	/*!< Affected by another context locking up */
+	RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING   = 3,	/*!< Overran the global deadline */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING = 4,	/*!< Affected by another context overrunning */
+	RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH = 5,	/*!< Forced reset to ensure scheduling requirements */
+	RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM        = 6,	/*!< CDM Mission/safety checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM        = 7,	/*!< TRP checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_OK          = 8,	/*!< GPU ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR         = 9,	/*!< GPU ECC error (uncorrected, HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_OK           = 10,	/*!< FW ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_ERR          = 11,	/*!< FW ECC error (uncorrected, ERR) */
+	RGX_CONTEXT_RESET_REASON_FW_WATCHDOG         = 12,	/*!< FW Safety watchdog triggered */
+	RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT        = 13,	/*!< FW page fault (no HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR         = 14,	/*!< FW execution error (GPU reset requested) */
+	RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR     = 15,	/*!< Host watchdog detected FW error */
+	RGX_CONTEXT_GEOM_OOM_DISABLED                = 16,	/*!< Geometry DM OOM event is not allowed */
+	RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH         = 17,	/*!< PVRIC Signature mismatch */
+	RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR   = 18,	/*!< Parity error in MMU Page Table Entry */
+	RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR       = 19,	/*!< Parity error in MH, system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR      = 20,	/*!< Parity error in system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR      = 21,	/*!< Latent/ICS signature mismatch error */
+	RGX_CONTEXT_RESET_REASON_DCLS_ERR            = 22,	/*!< Dual Core Lock Step FW error detected */
+	RGX_CONTEXT_RESET_REASON_ICS_HWR             = 23,	/*!< ICS fault */
+} RGX_CONTEXT_RESET_REASON;
+
+/*!
+	@Brief Context reset data shared with the host
+*/
+typedef struct
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reset reason */
+	IMG_UINT32 ui32ResetExtJobRef;  /*!< External Job ID */
+} RGX_CONTEXT_RESET_REASON_DATA;
+
+#define RGX_HEAP_UM_PDS_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_PDS_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_PDS_RESERVED_TOTAL_SIZE            RGX_HEAP_UM_PDS_RESERVED_SIZE
+
+#define RGX_HEAP_UM_USC_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_USC_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET      RGX_HEAP_UM_USC_RESERVED_SIZE
+
+#define RGX_HEAP_UM_GENERAL_RESERVED_SIZE           DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_GENERAL_RESERVED_REGION_OFFSET  0
+#define RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET  RGX_HEAP_UM_GENERAL_RESERVED_SIZE
+
+/*
+ * 4 dwords reserved for shared register management.
+ * The first dword is the number of shared register blocks to reload.
+ * Should be a multiple of 4 dwords, size in bytes.
+ */
+#define		RGX_LLS_SHARED_REGS_RESERVE_SIZE	(16U)
+
+/*************************************************************************/ /*!
+ Logging type
+*/ /**************************************************************************/
+#define RGXFWIF_LOG_TYPE_NONE			0x00000000U
+#define RGXFWIF_LOG_TYPE_TRACE			0x00000001U
+#define RGXFWIF_LOG_TYPE_GROUP_MAIN		0x00000002U
+#define RGXFWIF_LOG_TYPE_GROUP_MTS		0x00000004U
+#define RGXFWIF_LOG_TYPE_GROUP_CLEANUP	0x00000008U
+#define RGXFWIF_LOG_TYPE_GROUP_CSW		0x00000010U
+#define RGXFWIF_LOG_TYPE_GROUP_BIF		0x00000020U
+#define RGXFWIF_LOG_TYPE_GROUP_PM		0x00000040U
+#define RGXFWIF_LOG_TYPE_GROUP_RTD		0x00000080U
+#define RGXFWIF_LOG_TYPE_GROUP_SPM		0x00000100U
+#define RGXFWIF_LOG_TYPE_GROUP_POW		0x00000200U
+#define RGXFWIF_LOG_TYPE_GROUP_HWR		0x00000400U
+#define RGXFWIF_LOG_TYPE_GROUP_HWP		0x00000800U
+#define RGXFWIF_LOG_TYPE_GROUP_RPM		0x00001000U
+#define RGXFWIF_LOG_TYPE_GROUP_DMA		0x00002000U
+#define RGXFWIF_LOG_TYPE_GROUP_MISC		0x00004000U
+#define RGXFWIF_LOG_TYPE_GROUP_VZ		0x00008000U
+#define RGXFWIF_LOG_TYPE_GROUP_SAFETY	0x00010000U
+#define RGXFWIF_LOG_TYPE_GROUP_VERBOSE	0x00020000U
+#define RGXFWIF_LOG_TYPE_GROUP_CUSTOMER	0x00040000U
+#define RGXFWIF_LOG_TYPE_GROUP_DEBUG	0x80000000U
+#define RGXFWIF_LOG_TYPE_GROUP_MASK		0x8007FFFEU
+#define RGXFWIF_LOG_TYPE_MASK			0x8007FFFFU
+
+/* String used in pvrdebug -h output */
+#define RGXFWIF_LOG_GROUPS_STRING_LIST   "main,mts,cleanup,csw,bif,pm,rtd,spm,pow,hwr,hwp,rpm,dma,misc,vz,safety,verbose,customer,debug"
+
+/* Table entry to map log group strings to log type value */
+typedef struct {
+	const IMG_CHAR* pszLogGroupName;
+	IMG_UINT32      ui32LogGroupType;
+} RGXFWIF_LOG_GROUP_MAP_ENTRY;
+
+/*
+  Macro for use with the RGXFWIF_LOG_GROUP_MAP_ENTRY type to create a lookup
+  table where needed. Keep log group names short, no more than 20 chars.
+*/
+#define RGXFWIF_LOG_GROUP_NAME_VALUE_MAP { "none",    RGXFWIF_LOG_TYPE_NONE }, \
+                                         { "main",    RGXFWIF_LOG_TYPE_GROUP_MAIN }, \
+                                         { "mts",     RGXFWIF_LOG_TYPE_GROUP_MTS }, \
+                                         { "cleanup", RGXFWIF_LOG_TYPE_GROUP_CLEANUP }, \
+                                         { "csw",     RGXFWIF_LOG_TYPE_GROUP_CSW }, \
+                                         { "bif",     RGXFWIF_LOG_TYPE_GROUP_BIF }, \
+                                         { "pm",      RGXFWIF_LOG_TYPE_GROUP_PM }, \
+                                         { "rtd",     RGXFWIF_LOG_TYPE_GROUP_RTD }, \
+                                         { "spm",     RGXFWIF_LOG_TYPE_GROUP_SPM }, \
+                                         { "pow",     RGXFWIF_LOG_TYPE_GROUP_POW }, \
+                                         { "hwr",     RGXFWIF_LOG_TYPE_GROUP_HWR }, \
+                                         { "hwp",     RGXFWIF_LOG_TYPE_GROUP_HWP }, \
+                                         { "rpm",     RGXFWIF_LOG_TYPE_GROUP_RPM }, \
+                                         { "dma",     RGXFWIF_LOG_TYPE_GROUP_DMA }, \
+                                         { "misc",    RGXFWIF_LOG_TYPE_GROUP_MISC }, \
+                                         { "vz",      RGXFWIF_LOG_TYPE_GROUP_VZ }, \
+                                         { "safety",  RGXFWIF_LOG_TYPE_GROUP_SAFETY }, \
+                                         { "verbose", RGXFWIF_LOG_TYPE_GROUP_VERBOSE }, \
+                                         { "customer",RGXFWIF_LOG_TYPE_GROUP_CUSTOMER }, \
+                                         { "debug",   RGXFWIF_LOG_TYPE_GROUP_DEBUG }
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s"
+
+/* Used in a print statement to display log group state, one per group */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST(types)  ((((types) & RGXFWIF_LOG_TYPE_GROUP_MAIN) != 0U)	?("main ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MTS) != 0U)		?("mts ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CLEANUP) != 0U)	?("cleanup ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CSW) != 0U)		?("csw ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_BIF) != 0U)		?("bif ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_PM) != 0U)		?("pm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RTD) != 0U)		?("rtd ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SPM) != 0U)		?("spm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_POW) != 0U)		?("pow ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWR) != 0U)		?("hwr ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWP) != 0U)		?("hwp ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RPM) != 0U)		?("rpm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DMA) != 0U)		?("dma ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MISC) != 0U)	?("misc ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VZ) != 0U)		?("vz ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SAFETY) != 0U)	?("safety ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VERBOSE) != 0U)	?("verbose ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CUSTOMER) != 0U)?("customer ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DEBUG) != 0U)	?("debug ")		:(""))
+
+/*!
+ ******************************************************************************
+ * Trace Buffer
+ *****************************************************************************/
+
+/*! Min, Max, and Default size of RGXFWIF_TRACEBUF_SPACE in DWords */
+#define RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS      8192U  /*  32KB */
+#define RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS 12000U  /* ~48KB */
+#define RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS     32768U  /* 128KB */
+
+#define RGXFW_TRACE_BUFFER_ASSERT_SIZE 200U
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+#define RGXFW_THREAD_NUM 2U
+#else
+#define RGXFW_THREAD_NUM 1U
+#endif
+
+typedef struct
+{
+	IMG_CHAR	szPath[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_CHAR	szInfo[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_UINT32	ui32LineNum;
+} UNCACHED_ALIGN RGXFWIF_FILE_INFO_BUF;
+
+/*!
+ * @Defgroup SRVAndFWTracing Services and Firmware Tracing data interface
+ * @Brief The document groups/lists the data structures and the interfaces related to Services and Firmware Tracing
+ * @{
+ */
+
+/*!
+ * @Brief Firmware trace buffer details
+ */
+typedef struct
+{
+	IMG_UINT32                         ui32TracePointer;          /*!< Trace pointer (write index into Trace Buffer) */
+	IMG_UINT32                         ui32WrapCount;             /*!< Number of times the Trace Buffer has wrapped */
+
+	RGXFWIF_DEV_VIRTADDR               pui32RGXFWIfTraceBuffer;   /*!< Trace buffer address (FW address), to be used by firmware for writing into trace buffer */
+
+	RGXFWIF_FILE_INFO_BUF RGXFW_ALIGN  sAssertBuf;
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF_SPACE;
+
+/*! @} End of Defgroup SRVAndFWTracing */
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief Firmware trace control data
+ */
+typedef struct
+{
+	IMG_UINT32              ui32LogType;                  /*!< FW trace log group configuration */
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[RGXFW_THREAD_NUM];  /*!< FW Trace buffer */
+	IMG_UINT32              ui32TraceBufSizeInDWords;     /*!< FW Trace buffer size in dwords, Member initialised only when sTraceBuf is actually allocated
+															(in RGXTraceBufferInitOnDemandResources) */
+	IMG_UINT32              ui32TracebufFlags;            /*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF;
+
+/* Debug-info sub-fields */
+/* Bit 0: RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT bit from RGX_CR_EVENT_STATUS register */
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT        (0U)
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET          (1U << RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT)
+
+/* Bit 1: RGX_CR_BIF_MMU_ENTRY_PENDING bit from RGX_CR_BIF_MMU_ENTRY register */
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT     (1U)
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET       (1U << RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT)
+
+/* Bit 2: RGX_CR_SLAVE_EVENT register is non-zero */
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT          (2U)
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET            (1U << RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT)
+
+/* Bit 3-15: Unused bits */
+
+#define RGXFWT_DEBUG_INFO_STR_MAXLEN                  64
+#define RGXFWT_DEBUG_INFO_STR_PREPEND                 " (debug info: "
+#define RGXFWT_DEBUG_INFO_STR_APPEND                  ")"
+
+/* Table of debug info sub-field's masks and corresponding message strings
+ * to be appended to firmware trace
+ *
+ * Mask     : 16 bit mask to be applied to debug-info field
+ * String   : debug info message string
+ */
+
+#define RGXFWT_DEBUG_INFO_MSKSTRLIST \
+/*Mask,                                           String*/ \
+X(RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET,      "mmu pf") \
+X(RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET,   "mmu pending") \
+X(RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET,        "slave events")
+
+/* Firmware trace time-stamp field breakup */
+
+/* RGX_CR_TIMER register read (48 bits) value*/
+#define RGXFWT_TIMESTAMP_TIME_SHIFT                   (0U)
+#define RGXFWT_TIMESTAMP_TIME_CLRMSK                  (IMG_UINT64_C(0xFFFF000000000000))
+
+/* Extra debug-info (16 bits) */
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT             (48U)
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK            ~RGXFWT_TIMESTAMP_TIME_CLRMSK
+
+typedef struct
+{
+	IMG_UINT			bfOsState		: 3;
+	IMG_UINT			bfFLOk			: 1;
+	IMG_UINT			bfFLGrowPending	: 1;
+	IMG_UINT			bfReserved		: 27;
+} RGXFWIF_OS_RUNTIME_FLAGS;
+
+#define RGXFWIF_FWFAULTINFO_MAX		(8U)			/* Total number of FW fault logs stored */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64CRTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64OSTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64Data;
+	RGXFWIF_FILE_INFO_BUF	sFaultBuf;
+} UNCACHED_ALIGN RGX_FWFAULTINFO;
+
+#define RGXFWIF_POW_STATES \
+  X(RGXFWIF_POW_OFF)			/* idle and handshaked with the host (ready to full power down) */ \
+  X(RGXFWIF_POW_ON)				/* running HW commands */ \
+  X(RGXFWIF_POW_FORCED_IDLE)	/* forced idle */ \
+  X(RGXFWIF_POW_IDLE)			/* idle waiting for host handshake */
+
+typedef enum
+{
+#define X(NAME) NAME,
+	RGXFWIF_POW_STATES
+#undef X
+} RGXFWIF_POW_STATE;
+
+/* Firmware HWR states */
+#define RGXFWIF_HWR_HARDWARE_OK			(IMG_UINT32_C(0x1) << 0U)	/*!< The HW state is ok or locked up */
+#define RGXFWIF_HWR_RESET_IN_PROGRESS	(IMG_UINT32_C(0x1) << 1U)	/*!< Tells if a HWR reset is in progress */
+#define RGXFWIF_HWR_GENERAL_LOCKUP		(IMG_UINT32_C(0x1) << 3U)	/*!< A DM unrelated lockup has been detected */
+#define RGXFWIF_HWR_DM_RUNNING_OK		(IMG_UINT32_C(0x1) << 4U)	/*!< At least one DM is running without being close to a lockup */
+#define RGXFWIF_HWR_DM_STALLING			(IMG_UINT32_C(0x1) << 5U)	/*!< At least one DM is close to lockup */
+#define RGXFWIF_HWR_FW_FAULT			(IMG_UINT32_C(0x1) << 6U)	/*!< The FW has faulted and needs to restart */
+#define RGXFWIF_HWR_RESTART_REQUESTED	(IMG_UINT32_C(0x1) << 7U)	/*!< The FW has requested the host to restart it */
+
+#define RGXFWIF_PHR_MODE_OFF			(0UL)
+#define RGXFWIF_PHR_MODE_RD_RESET		(1UL)
+
+typedef IMG_UINT32 RGXFWIF_HWR_STATEFLAGS;
+
+typedef IMG_UINT32 RGXFWIF_HWR_RECOVERYFLAGS;
+
+/*! @Brief Firmware system data shared with the Host driver */
+typedef struct
+{
+	IMG_UINT32                 ui32ConfigFlags;                       /*!< Configuration flags from host */
+	IMG_UINT32                 ui32ConfigFlagsExt;                    /*!< Extended configuration flags from host */
+	volatile RGXFWIF_POW_STATE ePowState;
+	struct {
+		volatile IMG_UINT32        ui32HWPerfRIdx;
+		volatile IMG_UINT32        ui32HWPerfWIdx;
+		volatile IMG_UINT32        ui32HWPerfWrapCount;
+	} sHWPerfCtrl; /* Struct used to inval/flush HWPerfCtrl members */
+	IMG_UINT32                 ui32HWPerfSize;                        /*!< Constant after setup, needed in FW */
+	IMG_UINT32                 ui32HWPerfDropCount;                   /*!< The number of times the FW drops a packet due to buffer full */
+
+	/* ui32HWPerfUt, ui32FirstDropOrdinal, ui32LastDropOrdinal only valid when FW is built with
+	 * RGX_HWPERF_UTILIZATION & RGX_HWPERF_DROP_TRACKING defined in rgxfw_hwperf.c */
+	IMG_UINT32                 ui32HWPerfUt;                          /*!< Buffer utilisation, high watermark of bytes in use */
+	IMG_UINT32                 ui32FirstDropOrdinal;                  /*!< The ordinal of the first packet the FW dropped */
+	IMG_UINT32                 ui32LastDropOrdinal;                   /*!< The ordinal of the last packet the FW dropped */
+	RGXFWIF_OS_RUNTIME_FLAGS   asOsRuntimeFlagsMirror[RGXFW_MAX_NUM_OSIDS];/*!< State flags for each Operating System mirrored from Fw coremem */
+	RGX_FWFAULTINFO            sFaultInfo[RGXFWIF_FWFAULTINFO_MAX];   /*!< Firmware fault info */
+	IMG_UINT32                 ui32FWFaults;                          /*!< Firmware faults count */
+	IMG_UINT32                 aui32CrPollAddr[RGXFW_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[RGXFW_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[RGXFW_THREAD_NUM];    /*!< Failed poll count */
+	IMG_UINT64 RGXFW_ALIGN     ui64StartIdleTime;
+	IMG_UINT32                 aui32TSMirror[RGXFW_MAX_NUM_OSIDS];    /*!< Array of time slice per OS Mirrored from the FW */
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+#define RGXFWIF_STATS_FRAMEWORK_LINESIZE    (8)
+#define RGXFWIF_STATS_FRAMEWORK_MAX         (2048*RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+	IMG_UINT32 RGXFW_ALIGN     aui32FWStatsBuf[RGXFWIF_STATS_FRAMEWORK_MAX];
+#endif
+	RGXFWIF_HWR_STATEFLAGS     ui32HWRStateFlags; /*!< Firmware's Current HWR state */
+	RGXFWIF_HWR_RECOVERYFLAGS  aui32HWRRecoveryFlags[RGXFWIF_DM_MAX]; /*!< Each DM's HWR state */
+	IMG_UINT32                 ui32FwSysDataFlags;                      /*!< Compatibility and other flags */
+	IMG_UINT32                 ui32MemFaultCheck;                       /*!< Device mem fault check on PCI systems */
+} UNCACHED_ALIGN RGXFWIF_SYSDATA;
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+#define PVR_SLR_LOG_ENTRIES 10U
+#define PVR_SLR_LOG_STRLEN  30 /*!< MAX_CLIENT_CCB_NAME not visible to this header */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64Timestamp;
+	IMG_UINT32				ui32FWCtxAddr;
+	IMG_UINT32				ui32NumUFOs;
+	IMG_CHAR				aszCCBName[PVR_SLR_LOG_STRLEN];
+} UNCACHED_ALIGN RGXFWIF_SLR_ENTRY;
+#endif
+
+#define RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS (128U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware per-os data and configuration
+ */
+typedef struct
+{
+	IMG_UINT32                 ui32FwOsConfigFlags;                   /*!< Configuration flags from an OS */
+	IMG_UINT32                 ui32FWSyncCheckMark;                   /*!< Markers to signal that the host should perform a full sync check */
+	IMG_UINT32                 ui32HostSyncCheckMark;                  /*!< Markers to signal that the Firmware should perform a full sync check */
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+	IMG_UINT32                 ui32ForcedUpdatesRequested;
+	IMG_UINT8                  ui8SLRLogWp;
+	RGXFWIF_SLR_ENTRY          sSLRLogFirst;
+	RGXFWIF_SLR_ENTRY          sSLRLog[PVR_SLR_LOG_ENTRIES];
+	IMG_UINT64 RGXFW_ALIGN     ui64LastForcedUpdateTime;
+#endif
+	volatile IMG_UINT32        aui32InterruptCount[RGXFW_THREAD_NUM]; /*!< Interrupt count from Threads > */
+	IMG_UINT32                 ui32KCCBCmdsExecuted;                  /*!< Executed Kernel CCB command count */
+	RGXFWIF_DEV_VIRTADDR       sPowerSync;                            /*!< Sync prim used to signal the host the power off state */
+	IMG_UINT32                 ui32FwOsDataFlags;                       /*!< Compatibility and other flags */
+	IMG_UINT32                 aui32OfflineBuffer[RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS];
+} UNCACHED_ALIGN RGXFWIF_OSDATA;
+
+/*!
+ ******************************************************************************
+ * HWR Data
+ *****************************************************************************/
+/*!
+ * @Defgroup HWRInfo FW HWR shared data interface
+ * @Brief Types grouping data structures and defines used in realising the HWR record.
+ * @{
+ */
+
+#define RGXFW_PROCESS_NAME_LEN	(16)
+
+/*! @Brief HWR Lockup types */
+typedef enum
+{
+	RGX_HWRTYPE_UNKNOWNFAILURE = 0, /*!< Unknown failure */
+	RGX_HWRTYPE_OVERRUN        = 1, /*!< DM overrun */
+	RGX_HWRTYPE_POLLFAILURE    = 2, /*!< Poll failure */
+	RGX_HWRTYPE_BIF0FAULT      = 3, /*!< BIF0 fault */
+	RGX_HWRTYPE_BIF1FAULT      = 4, /*!< BIF1 fault */
+	RGX_HWRTYPE_TEXASBIF0FAULT = 5, /*!< TEXASBIF0 fault */
+	RGX_HWRTYPE_MMUFAULT       = 6, /*!< MMU fault */
+	RGX_HWRTYPE_MMUMETAFAULT   = 7, /*!< MMU META fault */
+	RGX_HWRTYPE_MIPSTLBFAULT   = 8, /*!< MIPS TLB fault */
+	RGX_HWRTYPE_ECCFAULT       = 9, /*!< ECC fault */
+	RGX_HWRTYPE_MMURISCVFAULT  = 10, /*!< MMU RISCV fault */
+	RGX_HWRTYPE_ICSFAULT       = 11, /*!< Idle Cycle Stealing fault */
+} RGX_HWRTYPE;
+
+#define RGXFWIF_HWRTYPE_BIF_BANK_GET(eHWRType) (((eHWRType) == RGX_HWRTYPE_BIF0FAULT) ? 0 : 1)
+
+#define RGXFWIF_HWRTYPE_PAGE_FAULT_GET(eHWRType) ((((eHWRType) == RGX_HWRTYPE_BIF0FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_BIF1FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_TEXASBIF0FAULT) ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUFAULT)       ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUMETAFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MIPSTLBFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMURISCVFAULT)) ? true : false)
+
+/************************
+ *  GPU HW error codes  *
+ ************************/
+typedef enum
+{
+	RGX_HW_ERR_NA = 0x0,
+	RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL = 0x101,
+} RGX_HW_ERR;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFReqStatus; /*!< BIF request status */
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFMMUStatus; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_BIFINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32FaultGPU; /*!< ECC fault in GPU */
+} RGX_ECCINFO;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		aui64MMUStatus[2]; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_MMUINFO;
+
+typedef struct
+{
+	IMG_UINT32	ui32ThreadNum; /*!< Thread ID performing poll operation */
+	IMG_UINT32	ui32CrPollAddr; /*!< CR Poll Address */
+	IMG_UINT32	ui32CrPollMask; /*!< CR Poll mask */
+	IMG_UINT32	ui32CrPollLastValue; /*!< CR Poll last value */
+	IMG_UINT64	RGXFW_ALIGN ui64Reserved;
+} UNCACHED_ALIGN RGX_POLLINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32BadVAddr; /*!< VA address */
+	IMG_UINT32 ui32EntryLo;
+} RGX_TLBINFO;
+
+/*! @Brief Structure to keep information specific to a lockup e.g. DM, timer, lockup type etc. */
+typedef struct
+{
+	union
+	{
+		RGX_BIFINFO  sBIFInfo; /*!< BIF failure details */
+		RGX_MMUINFO  sMMUInfo; /*!< MMU failure details */
+		RGX_POLLINFO sPollInfo; /*!< Poll failure details */
+		RGX_TLBINFO  sTLBInfo; /*!< TLB failure details */
+		RGX_ECCINFO  sECCInfo; /*!< ECC failure details */
+	} uHWRData;
+
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimer; /*!< Timer value at the time of lockup */
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimer; /*!< OS timer value at the time of lockup */
+	IMG_UINT32             ui32FrameNum; /*!< Frame number of the workload */
+	IMG_UINT32             ui32PID; /*!< PID belonging to the workload */
+	IMG_UINT32             ui32ActiveHWRTData; /*!< HWRT data of the workload */
+	IMG_UINT32             ui32HWRNumber; /*!< HWR number */
+	IMG_UINT32             ui32EventStatus; /*!< Core specific event status register at the time of lockup */
+	IMG_UINT32             ui32HWRRecoveryFlags; /*!< DM state flags */
+	RGX_HWRTYPE            eHWRType; /*!< Type of lockup */
+	RGXFWIF_DM             eDM; /*!< Recovery triggered for the DM */
+	IMG_UINT32             ui32CoreID; /*!< Core ID of the GPU */
+	RGX_HW_ERR             eHWErrorCode; /*!< Error code used to determine HW fault */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeOfKick; /*!< Workload kick time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetStart; /*!< HW reset start time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetFinish; /*!< HW reset stop time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeFreelistReady; /*!< freelist ready time on the last HWR */
+	IMG_CHAR   RGXFW_ALIGN szProcName[RGXFW_PROCESS_NAME_LEN]; /*!< User process name */
+	IMG_UINT32 RGXFW_ALIGN ui32PDSStalledDMs; /*!< DMs stalled on PDS Store space */
+	IMG_UINT32             ui32PDSActiveDMUSCs; /*!< Per-DM USC PDS activity */
+} UNCACHED_ALIGN RGX_HWRINFO;
+
+#define RGXFWIF_HWINFO_MAX_FIRST 8U							/* Number of first HWR logs recorded (never overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX_LAST 8U							/* Number of latest HWR logs (older logs are overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX (RGXFWIF_HWINFO_MAX_FIRST + RGXFWIF_HWINFO_MAX_LAST)	/* Total number of HWR logs stored in a buffer */
+#define RGXFWIF_HWINFO_LAST_INDEX (RGXFWIF_HWINFO_MAX - 1U)	/* Index of the last log in the HWR log buffer */
+
+/*! @Brief Firmware HWR information structure allocated by the Services and used by the Firmware to update recovery information. */
+typedef struct
+{
+	RGX_HWRINFO sHWRInfo[RGXFWIF_HWINFO_MAX]; /*!< Max number of recovery record */
+	IMG_UINT32  ui32HwrCounter; /*!< HWR counter used in FL reconstruction */
+	IMG_UINT32  ui32WriteIndex; /*!< Index for updating recovery information in sHWRInfo */
+	IMG_UINT32  ui32DDReqCount; /*!< Count of DebugDump requested to the host after recovery */
+	IMG_UINT32  ui32HWRInfoBufFlags; /* Compatibility and other flags */
+	IMG_UINT32  aui32HwrDmLockedUpCount[RGXFWIF_DM_MAX]; /*!< Lockup count for each DM */
+	IMG_UINT32  aui32HwrDmOverranCount[RGXFWIF_DM_MAX]; /*!< Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmRecoveredCount[RGXFWIF_DM_MAX]; /*!< Lockup + Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmFalseDetectCount[RGXFWIF_DM_MAX]; /*!< False lockup detection count for each DM */
+} UNCACHED_ALIGN RGXFWIF_HWRINFOBUF;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COMMONCTX_STATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RF_CMD;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWMEMCONTEXT;
+
+/*! @} End of HWRInfo */
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Freelist holding usage state of the Parameter Buffers
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN sFreeListBaseDevVAddr;	/*!< Freelist page table base address */
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN sFreeListStateDevVAddr;	/*!< Freelist state buffer base address */
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN sFreeListLastGrowDevVAddr;	/*!< Freelist base address at last grow */
+
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT64		RGXFW_ALIGN ui64CurrentDevVAddr;/*!< Freelist page table entry for current free page */
+	IMG_UINT32		ui32CurrentStackTop;		/*!< Freelist current free page */
+#endif
+
+	IMG_UINT32		ui32MaxPages;			/*!< Max no. of pages can be added to the freelist */
+	IMG_UINT32		ui32GrowPages;			/*!< No pages to add in each freelist grow */
+	IMG_UINT32		ui32CurrentPages;		/*!< Total no. of pages made available to the PM HW */
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT32		ui32AllocatedPageCount;		/*!< No. of pages allocated by PM HW */
+	IMG_UINT32		ui32AllocatedMMUPageCount;	/*!< No. of pages allocated for GPU MMU for PM*/
+#endif
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	IMG_UINT32		ui32HWRCounter;
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;
+#endif
+	IMG_UINT32		ui32FreeListID;			/*!< Unique Freelist ID */
+	IMG_BOOL		bGrowPending;			/*!< Freelist grow is pending */
+	IMG_UINT32		ui32ReadyPages;			/*!< Reserved pages to be used only on PM OOM event */
+	IMG_UINT32		ui32FreelistFlags;		/*!< Compatibility and other flags */
+
+	IMG_BOOL		bUpdatePending;
+	IMG_UINT32		ui32UpdateNewPages;
+	IMG_UINT32		ui32UpdateNewReadyPages;
+} UNCACHED_ALIGN RGXFWIF_FREELIST;
+
+typedef struct {RGXFWIF_DEV_VIRTADDR sNext;
+                RGXFWIF_DEV_VIRTADDR sPrev;}	RGXFW_DLLIST_NODE;
+
+#define RGXFWIF_MAX_NUM_CANCEL_REQUESTS		(8U)		/* Maximum number of workload cancellation requests */
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware Common Context (or FWCC)
+ */
+typedef struct RGXFWIF_FWCOMMONCONTEXT_
+{
+	/* CCB details for this firmware context */
+	PRGXFWIF_CCCB_CTL		psCCBCtl;				/*!< CCB control */
+	PRGXFWIF_CCCB			psCCB;					/*!< CCB base */
+	RGXFWIF_DMA_ADDR		sCCBMetaDMAAddr;
+
+	/* Context suspend state */
+	PRGXFWIF_COMMONCTX_STATE	RGXFW_ALIGN psContextState;		/*!< TA/3D context suspend state, read/written by FW */
+
+	/* Flags e.g. for context switching */
+	IMG_UINT32				ui32FWComCtxFlags;
+	IMG_INT32				i32Priority;  /*!< Priority level */
+	IMG_UINT32				ui32PrioritySeqNum;
+
+	/* Framework state */
+	PRGXFWIF_RF_CMD			RGXFW_ALIGN psRFCmd;		/*!< Register updates for Framework */
+
+	/* Misc and compatibility flags */
+	IMG_UINT32				ui32CompatFlags;
+
+	/* Statistic updates waiting to be passed back to the host... */
+	IMG_INT32				i32StatsNumStores;		/*!< Number of stores on this context since last update */
+	IMG_INT32				i32StatsNumOutOfMemory;		/*!< Number of OOMs on this context since last update */
+	IMG_INT32				i32StatsNumPartialRenders;	/*!< Number of PRs on this context since last update */
+	RGXFWIF_DM				eDM;				/*!< Data Master type */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN  sBufStalledNode;			/*!< List entry for the buffer stalled list */
+	IMG_UINT64				RGXFW_ALIGN  ui64CBufQueueCtrlAddr;	/*!< Address of the circular buffer queue pointers */
+
+	IMG_UINT64				RGXFW_ALIGN  ui64RobustnessAddress;
+	IMG_UINT32				ui32MaxDeadlineMS;			/*!< Max HWR deadline limit in ms */
+	bool					bReadOffsetNeedsReset;			/*!< Following HWR circular buffer read-offset needs resetting */
+	IMG_BOOL				bIPPContextNeedsReset;			/*!< Following HWR during 3D, IPP context needs resetting */
+
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sWaitingNode;		/*!< List entry for the waiting list */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sRunNode;			/*!< List entry for the run list */
+	RGXFWIF_UFO				sLastFailedUFO;			/*!< UFO that last failed (or NULL) */
+
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;					/*!< Memory context */
+
+	/* References to the host side originators */
+	IMG_UINT32				ui32ServerCommonContextID;	/*!< the Server Common Context */
+	IMG_UINT32				ui32PID;			/*!< associated process ID */
+
+	IMG_BOOL				bGeomOOMDisabled;		/*!< True when Geom DM OOM is not allowed */
+
+	IMG_UINT32				ui32PipelinedKicks;		/*!< Number of kick from this CCB currently submitted to the DM pipeline */
+	IMG_CHAR				szProcName[RGXFW_PROCESS_NAME_LEN];	/*!< User process name */
+
+	IMG_UINT32				ui32DeferCount;		/*!< Number of context defers before forced scheduling of context */
+	IMG_UINT32				aui32FirstIntJobRefToCancel[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the beginning of range of IntJobRefs at and above which workloads will be discarded */
+	IMG_UINT32				aui32FirstValidIntJobRef[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the end of range of IntJobRef below which workloads will be discarded */
+	IMG_BOOL				bCancelRangesActive;	/*!< True if any active ranges in aui32FirstIntJobRefToCancel and aui32FirstValidIntJobRef arrays */
+	IMG_BOOL				bLastKickedCmdWasSafetyOnly;
+	IMG_UINT32				ui32UID;			/*!< associated process UID used in FW managed gpu work period hwperf events */
+} UNCACHED_ALIGN RGXFWIF_FWCOMMONCONTEXT;
+
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) <= 256U,
+              "Size of structure RGXFWIF_FWCOMMONCONTEXT exceeds maximum expected size.");
+
+/*!
+ ******************************************************************************
+ * HWRTData
+ *****************************************************************************/
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FREELIST;
+
+/* HWRTData flags */
+/* Deprecated flags 1:0 */
+#define HWRTDATA_HAS_LAST_TA              (IMG_UINT32_C(1) << 2)
+#define HWRTDATA_PARTIAL_RENDERED         (IMG_UINT32_C(1) << 3)
+#define HWRTDATA_KILLED                   (IMG_UINT32_C(1) << 4)
+#define HWRTDATA_KILL_AFTER_TARESTART     (IMG_UINT32_C(1) << 5)
+#if defined(SUPPORT_AGP)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT0    (IMG_UINT32_C(1) << 6)
+#if defined(SUPPORT_AGP4)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT1    (IMG_UINT32_C(1) << 7)
+#endif
+#define HWRTDATA_GEOM_NEEDS_RESUME        (IMG_UINT32_C(1) << 8)
+#endif
+#if defined(SUPPORT_TRP)
+#define HWRTDATA_GEOM_TRP_IN_PROGRESS          (IMG_UINT32_C(1) << 9)
+#endif
+
+typedef enum
+{
+	RGXFWIF_RTDATA_STATE_NONE = 0,
+	RGXFWIF_RTDATA_STATE_KICKTA,
+	RGXFWIF_RTDATA_STATE_KICKTAFIRST,
+	RGXFWIF_RTDATA_STATE_TAFINISHED,
+	RGXFWIF_RTDATA_STATE_KICK3D,
+	RGXFWIF_RTDATA_STATE_3DFINISHED,
+	RGXFWIF_RTDATA_STATE_3DCONTEXTSTORED,
+	RGXFWIF_RTDATA_STATE_TAOUTOFMEM,
+	RGXFWIF_RTDATA_STATE_PARTIALRENDERFINISHED,
+	/* In case of HWR, we can't set the RTDATA state to NONE,
+	 * as this will cause any TA to become a first TA.
+	 * To ensure all related TA's are skipped, we use the HWR state */
+	RGXFWIF_RTDATA_STATE_HWR,
+	RGXFWIF_RTDATA_STATE_UNKNOWN = 0x7FFFFFFFU
+} RGXFWIF_RTDATA_STATE;
+
+typedef struct
+{
+    IMG_UINT32							ui32ScreenPixelMax;
+	IMG_UINT64							RGXFW_ALIGN ui64PPPMultiSampleCtl;
+	IMG_UINT32							ui32TEStride;
+	IMG_UINT32							ui32TPCSize;
+	IMG_UINT32							ui32TEScreen;
+	IMG_UINT32							ui32TEAA;
+	IMG_UINT32							ui32TEMTILE1;
+	IMG_UINT32							ui32TEMTILE2;
+	IMG_UINT32							ui32RgnStride;
+	IMG_UINT32							ui32ISPMergeLowerX;
+	IMG_UINT32							ui32ISPMergeLowerY;
+	IMG_UINT32							ui32ISPMergeUpperX;
+	IMG_UINT32							ui32ISPMergeUpperY;
+	IMG_UINT32							ui32ISPMergeScaleX;
+	IMG_UINT32							ui32ISPMergeScaleY;
+} UNCACHED_ALIGN RGXFWIF_HWRTDATA_COMMON;
+
+typedef struct
+{
+	IMG_UINT32           ui32RenderTargetIndex;		//Render number
+	IMG_UINT32           ui32CurrentRenderTarget;	//index in RTA
+	IMG_UINT32           ui32ActiveRenderTargets;	//total active RTs
+	RGXFWIF_DEV_VIRTADDR sValidRenderTargets;  //Array of valid RT indices
+	RGXFWIF_DEV_VIRTADDR sRTANumPartialRenders;  //Array of number of occurred partial renders per render target
+	IMG_UINT32           ui32MaxRTs;   //Number of render targets in the array
+	IMG_UINT32           ui32RTACtlFlags; /* Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_RTA_CTL;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Render Target data i.e. HWRTDATA used to hold the PM context
+ */
+typedef struct
+{
+	IMG_UINT64			RGXFW_ALIGN ui64VCECatBase[2];			/*!< VCE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64VCELastCatBase[2];
+	IMG_UINT64			RGXFW_ALIGN ui64TECatBase[2];			/*!< TE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64TELastCatBase[2];
+	IMG_UINT64			RGXFW_ALIGN ui64AlistCatBase;			/*!< Alist Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64AlistLastCatBase;
+
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psVHeapTableDevVAddr;		/*!< VHeap table base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sPMMListDevVAddr;			/*!< Mlist table base */
+#else
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sPMRenderStateDevVAddr;		/*!< Series8 PM State buffers */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sPMSecureRenderStateDevVAddr;
+#endif
+
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[RGXFW_MAX_FREELISTS]; /*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[RGXFW_MAX_FREELISTS];
+	IMG_BOOL			bRenderStateNeedsReset;
+
+	IMG_UINT32			ui32HWRTDataFlags;
+	RGXFWIF_RTDATA_STATE		eState;						/*!< Current workload processing state of HWRTDATA */
+
+	RGXFWIF_RTA_CTL			sRTACtl;					/*!< Render target array data */
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sTailPtrsDevVAddr;			/*!< Tail pointers base */
+#if defined(RGX_FIRMWARE)
+	struct RGXFWIF_FWCOMMONCONTEXT_* psOwnerGeom;
+#else
+	RGXFWIF_DEV_VIRTADDR		pui32OwnerGeomNotUsedByHost;
+#endif
+
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT64			RGXFW_ALIGN ui64PMAListStackPointer;		/*!< Freelist page table entry for current Mlist page */
+	IMG_UINT32			ui32PMMListStackPointer;			/*!< Current Mlist page */
+#endif
+#if defined(SUPPORT_TRP)
+	IMG_UINT32			ui32KickFlagsCopy;
+#endif
+
+	RGXFWIF_CLEANUP_CTL		RGXFW_ALIGN_DCACHEL sCleanupState;					/*!< Render target clean up state */
+} RGXFW_ALIGN_DCACHEL RGXFWIF_HWRTDATA;
+
+/* Sync_checkpoint firmware object.
+ * This is the FW-addressable structure used to hold the sync checkpoint's
+ * state and other information which needs to be accessed by the firmware.
+ */
+typedef struct
+{
+	IMG_UINT32	ui32State;          /*!< Holds the current state of the sync checkpoint */
+	IMG_UINT32	ui32FwRefCount;     /*!< Holds the FW reference count (num of fences/updates processed) */
+} SYNC_CHECKPOINT_FW_OBJ;
+
+/* Bit mask Firmware can use to test if a checkpoint has signalled or errored */
+#define SYNC_CHECKPOINT_SIGNALLED_MASK (0x1 << 0)
+
+typedef enum
+{
+	RGXFWIF_REG_CFG_TYPE_PWR_ON=0,      /* Sidekick power event */
+	RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,   /* Rascal / dust power event */
+	RGXFWIF_REG_CFG_TYPE_TA,            /* TA kick */
+	RGXFWIF_REG_CFG_TYPE_3D,            /* 3D kick */
+	RGXFWIF_REG_CFG_TYPE_CDM,           /* Compute kick */
+	RGXFWIF_REG_CFG_TYPE_TDM,           /* TDM kick */
+	RGXFWIF_REG_CFG_TYPE_ALL            /* Applies to all types. Keep as last element */
+} RGXFWIF_REG_CFG_TYPE;
+
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_OFFSET_BYTES                 RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES               (1U << 21)
+
+#define RGX_HEAP_KM_USC_RESERVED_SIZE                              RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES
+#define RGX_HEAP_USC_RESERVED_TOTAL_SIZE                           RGX_HEAP_UM_USC_RESERVED_SIZE + RGX_HEAP_KM_USC_RESERVED_SIZE
+
+#define RGXFW_SCRATCH_BUF_SIZE             (1048576U)
+
+#define RGX_MAX_SECURE_ALLOC_NUM_CORES     (8U)
+
+#define RGX_MAX_SECURE_ALLOC_NUM_CLUSTERS  (12U)
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES      RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES      (RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_OFFSET_BYTES     (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_MAX_SIZE_BYTES   8192U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_OFFSET_BYTES     (RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES   (0x00004000U * RGX_MAX_SECURE_ALLOC_NUM_CORES *  RGX_MAX_SECURE_ALLOC_NUM_CLUSTERS + 48U + 1023U)
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_OFFSET_BYTES   (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_MAX_SIZE_BYTES RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_OFFSET_BYTES    (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_MAX_SIZE_BYTES  (((0x00000080U + 127U) * RGX_MAX_SECURE_ALLOC_NUM_CORES) + 127U)
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_OFFSET_BYTES    (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_MAX_SIZE_BYTES  (((0x00000180U + 127U) * RGX_MAX_SECURE_ALLOC_NUM_CORES) + 127U)
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      PVR_ALIGN((RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_MAX_SIZE_BYTES), DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY)
+#else
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      0
+#endif
+
+#endif /*  RGX_FWIF_SHARED_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_shared.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_heaps.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_heaps.h
new file mode 100644
index 000000000000..4e72d8d1a7bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_heaps.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAPS_H)
+#define RGX_HEAPS_H
+
+/*
+  Identify heaps by their names
+*/
+#define RGX_GENERAL_SVM_HEAP_IDENT          "General SVM"               /*!< SVM (shared virtual memory) Heap Identifier */
+#define RGX_GENERAL_HEAP_IDENT              "General"                   /*!< RGX General Heap Identifier */
+#define RGX_GENERAL_NON4K_HEAP_IDENT        "General NON-4K"            /*!< RGX General non-4K Heap Identifier */
+#define RGX_PDSCODEDATA_HEAP_IDENT          "PDS Code and Data"         /*!< RGX PDS Code/Data Heap Identifier */
+#define RGX_USCCODE_HEAP_IDENT              "USC Code"                  /*!< RGX USC Code Heap Identifier */
+#define RGX_USCCODE_BPH_HEAP_IDENT          "BP Handler USC Code"       /*!< RGX USC Code for breakpoint handlers Heap Identifier */
+#define RGX_VK_CAPT_REPLAY_HEAP_IDENT       "Vulkan Capture Replay"     /*!< RGX vulkan capture replay buffer Heap Identifier */
+#define RGX_COMPONENT_CTRL_HEAP_IDENT       "Component Control"         /*!< RGX DCE Component Control Heap Identifier */
+#define RGX_FBCDC_HEAP_IDENT                "FBCDC"                     /*!< RGX FBCDC State Table Heap Identifier */
+#define RGX_FBCDC_LARGE_HEAP_IDENT          "Large FBCDC"               /*!< RGX Large FBCDC State Table Heap Identifier */
+#define RGX_PDS_INDIRECT_STATE_HEAP_IDENT   "PDS Indirect State"        /*!< PDS Indirect State Table Heap Identifier */
+#define RGX_CMP_MISSION_RMW_HEAP_IDENT      "Compute Mission RMW"       /*!< Compute Mission RMW Heap Identifier */
+#define RGX_CMP_SAFETY_RMW_HEAP_IDENT       "Compute Safety RMW"        /*!< Compute Safety RMW Heap Identifier */
+#define RGX_TEXTURE_STATE_HEAP_IDENT        "Texture State"             /*!< Texture State Heap Identifier */
+#define RGX_VISIBILITY_TEST_HEAP_IDENT      "Visibility Test"           /*!< Visibility Test Heap Identifier */
+
+#endif /* RGX_HEAPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf.h
new file mode 100644
index 000000000000..d0d4014a117a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf.h
@@ -0,0 +1,343 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_H_
+#define RGX_HWPERF_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_hwperf_common.h"
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+/* HWPerf interface assumption checks */
+static_assert(RGX_FEATURE_NUM_CLUSTERS <= 16U,
+			  "Cluster count too large for HWPerf protocol definition");
+#endif
+
+/*! Perf counter control words */
+#define RGX_HWPERF_CTRL_NOP						(0)			/*!< only update HW counters */
+#define RGX_HWPERF_CTRL_STATE_UPDATE_EN			(1U << 31)	/*!< persistent state update; see other flags below */
+#define RGX_HWPERF_CTRL_GEOM_FULLRANGE			(1U)		/*!< selectable geom and 3D counters are full range */
+#define RGX_HWPERF_CTRL_COMP_FULLRANGE			(2U)		/*!< selectable compute counters are full range */
+#define RGX_HWPERF_CTRL_TDM_FULLRANGE			(4U)		/*!< selectable TDM counters are full range */
+
+
+/******************************************************************************
+ * Data Stream Common Types
+ *****************************************************************************/
+
+/*! All the Data Masters HWPerf is aware of. When a new DM is added to this
+ * list, it should be appended at the end to maintain backward compatibility
+ * of HWPerf data.
+ */
+#define RGX_HWPERF_DM_GP                  0x00000000U
+#define	RGX_HWPERF_DM_TDM                 0x00000001U
+#define	RGX_HWPERF_DM_GEOM                0x00000002U
+#define	RGX_HWPERF_DM_3D                  0x00000003U
+#define	RGX_HWPERF_DM_CDM                 0x00000004U
+#define	RGX_HWPERF_DM_RTU                 0x00000005U
+
+#define	RGX_HWPERF_DM_LAST                0x00000006U
+
+#define	RGX_HWPERF_DM_INVALID             0x1FFFFFFFU
+
+/******************************************************************************
+ * API Types
+ *****************************************************************************/
+
+/*! Counter block IDs for all the hardware blocks with counters.
+ * Directly addressable blocks must have a value between 0..15 [0..0xF].
+ * Indirect groups have following encoding:
+ * First hex digit (LSB) represents a unit number within the group
+ * and the second hex digit represents the group number.
+ * Group 0 is the direct group, all others are indirect groups.
+ */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_ID;
+
+/*! Directly addressable non bank-switched counter blocks */
+#define RGX_CNTBLK_ID_JONES          0x0000U
+#define RGX_CNTBLK_ID_SLC            0x0001U	/*!< SLC-specific counter control */
+#define RGX_CNTBLK_ID_FBCDC          0x0002U
+#define RGX_CNTBLK_ID_FW_CUSTOM      0x0003U	/*!< Custom FW provided counters */
+
+/*! Directly addressable SLC counter blocks - presence depends on GPU. */
+#define RGX_CNTBLK_ID_SLCBANK0       0x0004U	/*!< SLCBANK0 counter control */
+#define RGX_CNTBLK_ID_SLCBANK1       0x0005U	/*!< SLCBANK1 counter control */
+#define RGX_CNTBLK_ID_SLCBANK2       0x0006U	/*!< SLCBANK2 counter control */
+#define RGX_CNTBLK_ID_SLCBANK3       0x0007U	/*!< SLCBANK3 counter control */
+#define RGX_CNTBLK_ID_SLCBANK_ALL    0x4004U	/*!< SLC ALL block ID */
+
+#define RGX_CNTBLK_ID_DIRECT_LAST    0x0008U	/*!< Indirect blocks start from here */
+
+/*! Indirectly addressable counter blocks */
+#define RGX_CNTBLK_ID_ISP0           0x0010U	/*!< ISP 1..N ISP */
+#define RGX_CNTBLK_ID_ISP1           0x0011U
+#define RGX_CNTBLK_ID_ISP2           0x0012U
+#define RGX_CNTBLK_ID_ISP3           0x0013U
+#define RGX_CNTBLK_ID_ISP4           0x0014U
+#define RGX_CNTBLK_ID_ISP5           0x0015U
+#define RGX_CNTBLK_ID_ISP6           0x0016U
+#define RGX_CNTBLK_ID_ISP7           0x0017U
+#define RGX_CNTBLK_ID_ISP8           0x0018U
+#define RGX_CNTBLK_ID_ISP9           0x0019U
+#define RGX_CNTBLK_ID_ISP_ALL        0x4010U
+
+#define RGX_CNTBLK_ID_MERCER0        0x0020U	/*!< MERCER 1..N MERCER */
+#define RGX_CNTBLK_ID_MERCER1        0x0021U
+#define RGX_CNTBLK_ID_MERCER2        0x0022U
+#define RGX_CNTBLK_ID_MERCER3        0x0023U
+#define RGX_CNTBLK_ID_MERCER4        0x0024U
+#define RGX_CNTBLK_ID_MERCER5        0x0025U
+#define RGX_CNTBLK_ID_MERCER6        0x0026U
+#define RGX_CNTBLK_ID_MERCER7        0x0027U
+#define RGX_CNTBLK_ID_MERCER8        0x0028U
+#define RGX_CNTBLK_ID_MERCER9        0x0029U
+#define RGX_CNTBLK_ID_MERCER_ALL     0x4020U
+
+#define RGX_CNTBLK_ID_PBE0           0x0030U	/*!< PBE 1..N PBE_PER_SPU x N SPU */
+#define RGX_CNTBLK_ID_PBE1           0x0031U
+#define RGX_CNTBLK_ID_PBE2           0x0032U
+#define RGX_CNTBLK_ID_PBE3           0x0033U
+#define RGX_CNTBLK_ID_PBE4           0x0034U
+#define RGX_CNTBLK_ID_PBE5           0x0035U
+#define RGX_CNTBLK_ID_PBE6           0x0036U
+#define RGX_CNTBLK_ID_PBE7           0x0037U
+#define RGX_CNTBLK_ID_PBE_ALL        0x4030U
+
+#define RGX_CNTBLK_ID_PBE_SHARED0    0x0040U	/*!< PBE_SHARED 1..N SPU */
+#define RGX_CNTBLK_ID_PBE_SHARED1    0x0041U
+#define RGX_CNTBLK_ID_PBE_SHARED2    0x0042U
+#define RGX_CNTBLK_ID_PBE_SHARED3    0x0043U
+#define RGX_CNTBLK_ID_PBE_SHARED_ALL 0x4040U
+
+#define RGX_CNTBLK_ID_USC0           0x0050U	/*!< USC 1..N USC */
+#define RGX_CNTBLK_ID_USC1           0x0051U
+#define RGX_CNTBLK_ID_USC2           0x0052U
+#define RGX_CNTBLK_ID_USC3           0x0053U
+#define RGX_CNTBLK_ID_USC4           0x0054U
+#define RGX_CNTBLK_ID_USC5           0x0055U
+#define RGX_CNTBLK_ID_USC6           0x0056U
+#define RGX_CNTBLK_ID_USC7           0x0057U
+#define RGX_CNTBLK_ID_USC8           0x0058U
+#define RGX_CNTBLK_ID_USC9           0x0059U
+#define RGX_CNTBLK_ID_USC_ALL        0x4050U
+
+#define RGX_CNTBLK_ID_TPU0           0x0060U	/*!< TPU 1..N TPU */
+#define RGX_CNTBLK_ID_TPU1           0x0061U
+#define RGX_CNTBLK_ID_TPU2           0x0062U
+#define RGX_CNTBLK_ID_TPU3           0x0063U
+#define RGX_CNTBLK_ID_TPU4           0x0064U
+#define RGX_CNTBLK_ID_TPU5           0x0065U
+#define RGX_CNTBLK_ID_TPU6           0x0066U
+#define RGX_CNTBLK_ID_TPU7           0x0067U
+#define RGX_CNTBLK_ID_TPU8           0x0068U
+#define RGX_CNTBLK_ID_TPU9           0x0069U
+#define RGX_CNTBLK_ID_TPU_ALL        0x4060U
+
+#define RGX_CNTBLK_ID_SWIFT0         0x0070U	/*!< SWIFT 1..N SWIFT */
+#define RGX_CNTBLK_ID_SWIFT1         0x0071U
+#define RGX_CNTBLK_ID_SWIFT2         0x0072U
+#define RGX_CNTBLK_ID_SWIFT3         0x0073U
+#define RGX_CNTBLK_ID_SWIFT4         0x0074U
+#define RGX_CNTBLK_ID_SWIFT5         0x0075U
+#define RGX_CNTBLK_ID_SWIFT6         0x0076U
+#define RGX_CNTBLK_ID_SWIFT7         0x0077U
+#define RGX_CNTBLK_ID_SWIFT8         0x0078U
+#define RGX_CNTBLK_ID_SWIFT9         0x0079U
+#define RGX_CNTBLK_ID_SWIFT_ALL      0x4070U
+
+#define RGX_CNTBLK_ID_TEXAS0         0x0080U	/*!< TEXAS 1..N TEXAS */
+#define RGX_CNTBLK_ID_TEXAS1         0x0081U
+#define RGX_CNTBLK_ID_TEXAS2         0x0082U
+#define RGX_CNTBLK_ID_TEXAS3         0x0083U
+#define RGX_CNTBLK_ID_TEXAS_ALL      0x4080U
+
+#define RGX_CNTBLK_ID_RAC0           0x0090U	/*!< RAC 1..N RAC */
+#define RGX_CNTBLK_ID_RAC1           0x0091U
+#define RGX_CNTBLK_ID_RAC2           0x0092U
+#define RGX_CNTBLK_ID_RAC3           0x0093U
+#define RGX_CNTBLK_ID_RAC_ALL        0x4090U
+
+#define RGX_CNTBLK_ID_LAST           0x0094U	/*!< End of RAC block */
+
+/*! Masks for the counter block ID*/
+#define RGX_CNTBLK_ID_UNIT_MASK      (0x000FU)	/*!< Unit within group */
+#define RGX_CNTBLK_ID_GROUP_MASK     (0x00F0U)	/*!< Group value */
+#define RGX_CNTBLK_ID_GROUP_SHIFT    (4U)
+#define RGX_CNTBLK_ID_MC_GPU_MASK    (0x0F00U)	/*!< GPU ID for MC use */
+#define RGX_CNTBLK_ID_MC_GPU_SHIFT   (8U)
+#define RGX_CNTBLK_ID_UNIT_ALL_MASK  (0x4000U)	/*!< Program all units within a group */
+
+static_assert(
+	((RGX_CNTBLK_ID_DIRECT_LAST + ((RGX_CNTBLK_ID_LAST & RGX_CNTBLK_ID_GROUP_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT)) <= RGX_HWPERF_MAX_BVNC_BLOCK_LEN),
+	"RGX_HWPERF_MAX_BVNC_BLOCK_LEN insufficient");
+
+#define RGX_HWPERF_EVENT_MASK_VALUE(e)      (IMG_UINT64_C(1) << (e))
+
+/* When adding new counters here, make sure changes are made to rgxfw_hwperf_fwblk_valid() as well */
+#define RGX_CUSTOM_FW_CNTRS	\
+                X(TA_LOCAL_FL_SIZE,    0x0, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(TA_GLOBAL_FL_SIZE,   0x1, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(3D_LOCAL_FL_SIZE,    0x2, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(3D_GLOBAL_FL_SIZE,   0x3, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(ISP_TILES_IN_FLIGHT, 0x4, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DSPMKICK))
+
+/*! Counter IDs for the firmware held statistics */
+typedef enum
+{
+#define X(ctr, id, allow_mask)	RGX_CUSTOM_FW_CNTR_##ctr = id,
+	RGX_CUSTOM_FW_CNTRS
+#undef X
+
+	/* always the last entry in the list */
+	RGX_CUSTOM_FW_CNTR_LAST
+} RGX_HWPERF_CUSTOM_FW_CNTR_ID;
+
+/*! Identifier for each counter in a performance counting module */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_COUNTER_ID;
+
+/* sets all the bits from bit _b1 to _b2, in a IMG_UINT64 type */
+#define MASK_RANGE_IMPL(b1, b2)	((IMG_UINT64)((IMG_UINT64_C(1) << ((IMG_UINT32)(b2)-(IMG_UINT32)(b1) + 1U)) - 1U) << (IMG_UINT32)b1)
+#define MASK_RANGE(R)			MASK_RANGE_IMPL(R##_FIRST_TYPE, R##_LAST_TYPE)
+#define RGX_HWPERF_HOST_EVENT_MASK_VALUE(e) (IMG_UINT32_C(1) << (e))
+
+/*! Mask macros for use with RGXCtrlHWPerf() API.
+ */
+#define RGX_HWPERF_EVENT_MASK_NONE          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_HWPERF_EVENT_MASK_DEFAULT       RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_FWACT) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG)
+#define RGX_HWPERF_EVENT_MASK_ALL           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+
+/*! HWPerf Firmware event masks
+ * @par
+ * All FW Start/End/Debug (SED) events. */
+#define RGX_HWPERF_EVENT_MASK_FW_SED    (MASK_RANGE(RGX_HWPERF_FW_EVENT_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_FW_UFO    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO))
+#define RGX_HWPERF_EVENT_MASK_FW_CSW    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_START) |\
+                                          RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_FINISHED))
+/*! All FW events. */
+#define RGX_HWPERF_EVENT_MASK_ALL_FW    (RGX_HWPERF_EVENT_MASK_FW_SED |\
+                                          RGX_HWPERF_EVENT_MASK_FW_UFO |\
+                                          RGX_HWPERF_EVENT_MASK_FW_CSW)
+
+/*! HW Periodic events (1ms interval). */
+#define RGX_HWPERF_EVENT_MASK_HW_PERIODIC   (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PERIODIC))
+/*! All HW Kick/Finish events. */
+#define RGX_HWPERF_EVENT_MASK_HW_KICKFINISH ((MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE0) |\
+                                               MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE1)) &\
+                                              ~(RGX_HWPERF_EVENT_MASK_HW_PERIODIC))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_HW        (RGX_HWPERF_EVENT_MASK_HW_KICKFINISH |\
+                                              RGX_HWPERF_EVENT_MASK_HW_PERIODIC)
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR_EST   (MASK_RANGE(RGX_HWPERF_PWR_EST_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR       (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_GPU_STATE_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG))
+
+/*! HWPerf Host event masks
+ */
+#define RGX_HWPERF_EVENT_MASK_HOST_WORK_ENQ  (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_ENQ))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_UFO   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_UFO))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_PWR   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_CLK_SYNC))
+
+
+/*! Type used in the RGX API RGXConfigHWPerfCounters() */
+typedef struct
+{
+	/*! Reserved for future use */
+	IMG_UINT32 ui32Reserved;
+
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of configured counters within this block */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Counter register values */
+	IMG_UINT16 ui16Counters[RGX_CNTBLK_COUNTERS_MAX];
+
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_CNTBLK);
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf_table.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf_table.h
new file mode 100644
index 000000000000..9364d500e011
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf_table.h
@@ -0,0 +1,513 @@
+/*************************************************************************/ /*!
+@File
+@Title          HWPerf counter table header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_HWPERF_TABLE_H
+#define RGX_HWPERF_TABLE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+
+/*****************************************************************************/
+
+/* Forward declaration */
+typedef struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_ RGXFW_HWPERF_CNTBLK_TYPE_MODEL;
+
+/* Function pointer type for functions to check dynamic power state of
+ * counter block instance. Used only in firmware. */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_POWERED)(
+		RGX_HWPERF_CNTBLK_ID eBlkType,
+		IMG_UINT8 ui8UnitId);
+
+/* Counter block run-time info */
+typedef struct
+{
+	IMG_UINT32 uiNumUnits;             /* Number of instances of this block type in the core */
+} RGX_HWPERF_CNTBLK_RT_INFO;
+
+/* Function pointer type for functions to check block is valid and present
+ * on that RGX Device at runtime. It may have compile logic or run-time
+ * logic depending on where the code executes: server, srvinit or firmware.
+ * Values in the psRtInfo output parameter are only valid if true returned.
+ */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_PRESENT)(
+		const struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_* psBlkTypeDesc,
+		const void *pvDev_km,
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo);
+
+/* This structure encodes properties of a type of performance counter block.
+ * The structure is sometimes referred to as a block type descriptor. These
+ * properties contained in this structure represent the columns in the block
+ * type model table variable below. These values vary depending on the build
+ * BVNC and core type.
+ * Each direct block has a unique type descriptor and each indirect group has
+ * a type descriptor.
+ */
+struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_
+{
+	IMG_UINT32 uiCntBlkIdBase;         /* The starting block id for this block type */
+	IMG_UINT32 uiIndirectReg;          /* 0 if direct type otherwise the indirect register value to select indirect unit */
+	IMG_UINT32 uiNumUnits;             /* Number of instances of this block type in the core (compile time use) */
+	const IMG_CHAR *pszBlockNameComment;             /* Name of the PERF register. Used while dumping the perf counters to pdumps */
+	PFN_RGXFW_HWPERF_CNTBLK_POWERED pfnIsBlkPowered; /* A function to determine dynamic power state for the block type */
+	PFN_RGXFW_HWPERF_CNTBLK_PRESENT pfnIsBlkPresent; /* A function to determine presence on RGX Device at run-time */
+	IMG_UINT16 *pszBlkCfgValid;        /* Array of supported counters per block type */
+};
+
+/*****************************************************************************/
+
+/* Shared compile-time context ASSERT macro */
+#if defined(RGX_FIRMWARE)
+/*  firmware context */
+#	define DBG_ASSERT(_c) RGXFW_ASSERT((_c))
+#else
+/*  host client/server context */
+#	define DBG_ASSERT(_c) PVR_ASSERT((_c))
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered()
+
+ Referenced in gasCntBlkTypeModel[] table below and only called from
+ RGX_FIRMWARE run-time context. Therefore compile time configuration is used.
+ *****************************************************************************/
+
+#if defined(RGX_FIRMWARE) && defined(RGX_FEATURE_PERFBUS)
+#	include "rgxfw_pow.h"
+#	include "rgxfw_utils.h"
+
+static inline IMG_BOOL rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId);
+static inline IMG_BOOL rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+	switch (eBlkType)
+	{
+		case RGX_CNTBLK_ID_JONES:
+		case RGX_CNTBLK_ID_SLC:
+		case RGX_CNTBLK_ID_SLCBANK0:
+		case RGX_CNTBLK_ID_FBCDC:
+		case RGX_CNTBLK_ID_FW_CUSTOM:
+			return IMG_TRUE;
+
+#if !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_SLCBANK1:
+			if (RGX_FEATURE_NUM_MEMBUS > 1U)
+			{
+				return IMG_TRUE;
+			}
+			else
+			{
+				return IMG_FALSE;
+			}
+
+		case RGX_CNTBLK_ID_SLCBANK2:
+		case RGX_CNTBLK_ID_SLCBANK3:
+			if (RGX_FEATURE_NUM_MEMBUS > 2U)
+			{
+				return IMG_TRUE;
+			}
+			else
+			{
+				return IMG_FALSE;
+			}
+#endif /* !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE) */
+
+		default:
+			return IMG_FALSE;
+	}
+}
+
+/* Only use conditional compilation when counter blocks appear in different
+ * islands for different Rogue families.
+ */
+static inline IMG_BOOL rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId);
+static inline IMG_BOOL rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+	IMG_UINT32 ui32NumDustsEnabled = rgxfw_pow_get_enabled_units();
+
+	// We don't have any Dusts Enabled until first DC opens the GPU. This makes
+	// setting the PDump HWPerf trace buffers very difficult.
+	// To work around this we special-case some of the 'have to be there'
+	// indirect registers (e.g., TPU0)
+
+	switch (eBlkType)
+	{
+		case RGX_CNTBLK_ID_TPU0:
+			return IMG_TRUE;
+			/*NOTREACHED*/
+			break;
+		default:
+			if (rgxfw_pow_is_rd_on() && (ui32NumDustsEnabled > 0U))
+			{
+				return IMG_TRUE;
+			}
+			else
+			{
+				return IMG_FALSE;
+			}
+			/*NOTREACHED*/
+			break;
+	}
+	return IMG_TRUE;
+}
+
+#else /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+# define rgxfw_hwperf_pow_st_direct   ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+# define rgxfw_hwperf_pow_st_indirect ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+
+#endif /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() start
+
+ Referenced in gasCntBlkTypeModel[] table below and called from all build
+ contexts:
+ RGX_FIRMWARE, PVRSRVCTL (UM) and PVRSRVKM (Server).
+
+ Therefore each function has two implementations, one for compile time and one
+ run time configuration depending on the context. The functions will inform the
+ caller whether this block is valid for this particular RGX device. Other
+ run-time dependent data is returned in psRtInfo for the caller to use.
+ *****************************************************************************/
+
+
+/* Used for all block types: Direct and Indirect */
+static inline IMG_BOOL rgx_hwperf_blk_present(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo)
+{
+#if defined(__KERNEL__)	/* Server context -- Run-time Only */
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)pvDev_km;
+	PVRSRV_DEVICE_NODE *psNode;
+	IMG_UINT32	ui32MaxTPUPerSPU;
+	IMG_UINT32	ui32NumMemBus;
+	IMG_UINT32	ui32RTArchVal;
+
+	DBG_ASSERT(psDevInfo != NULL);
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT(psRtInfo != NULL);
+
+	if (((psDevInfo == NULL) || (psBlkTypeDesc == NULL)) || (psRtInfo == NULL))
+	{
+		return IMG_FALSE;
+	}
+
+	psNode = psDevInfo->psDeviceNode;
+	DBG_ASSERT(psNode != NULL);
+
+	if (psNode == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	ui32MaxTPUPerSPU =
+		PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, MAX_TPU_PER_SPU);
+
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psNode, CATURIX_TOP_INFRASTRUCTURE))
+	{
+		ui32NumMemBus = 1U;
+	}
+	else
+	{
+		ui32NumMemBus =
+			PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_MEMBUS);
+	}
+
+	ui32RTArchVal =
+		PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, RAY_TRACING_ARCH);
+
+	switch (psBlkTypeDesc->uiCntBlkIdBase)
+	{
+		case RGX_CNTBLK_ID_JONES:
+		case RGX_CNTBLK_ID_SLC:
+		case RGX_CNTBLK_ID_SLCBANK0:
+		case RGX_CNTBLK_ID_FBCDC:
+		case RGX_CNTBLK_ID_FW_CUSTOM:
+			psRtInfo->uiNumUnits = 1;
+			break;
+
+#if !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_SLCBANK1:
+			if (ui32NumMemBus >= 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		case RGX_CNTBLK_ID_SLCBANK2:
+		case RGX_CNTBLK_ID_SLCBANK3:
+			if (ui32NumMemBus > 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+#endif /* !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE) */
+
+		case RGX_CNTBLK_ID_TPU0:
+		case RGX_CNTBLK_ID_SWIFT0:
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			psRtInfo->uiNumUnits *= ui32MaxTPUPerSPU;
+			break;
+
+		case RGX_CNTBLK_ID_TEXAS0:
+		case RGX_CNTBLK_ID_PBE_SHARED0:
+
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			break;
+
+		case RGX_CNTBLK_ID_RAC0:
+			if (ui32RTArchVal > 2U)
+			{
+				psRtInfo->uiNumUnits =
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU0_RAC_PRESENT) +
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU1_RAC_PRESENT) +
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU2_RAC_PRESENT) +
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU3_RAC_PRESENT);
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		case RGX_CNTBLK_ID_USC0:
+		case RGX_CNTBLK_ID_MERCER0:
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_CLUSTERS);
+			break;
+
+		case RGX_CNTBLK_ID_PBE0:
+
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, PBE_PER_SPU);
+			psRtInfo->uiNumUnits *=
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			break;
+
+		case RGX_CNTBLK_ID_ISP0:
+
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_ISP_PER_SPU);
+			/* Adjust by NUM_SPU */
+
+			psRtInfo->uiNumUnits *=
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			break;
+
+		default:
+			return IMG_FALSE;
+	}
+	/* Verify that we have at least one unit present */
+	if (psRtInfo->uiNumUnits > 0U)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#else	/* FW context -- Compile-time only */
+	IMG_UINT32	ui32NumMemBus;
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+
+	if (unlikely(psBlkTypeDesc == NULL))
+	{
+		return IMG_FALSE;
+	}
+
+#if !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE)
+	ui32NumMemBus = RGX_FEATURE_NUM_MEMBUS;
+#else
+	ui32NumMemBus = 1U;
+#endif /* !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE) */
+
+	switch (psBlkTypeDesc->uiCntBlkIdBase)
+	{
+		/* Handle the dynamic-sized SLC blocks which are only present if
+		 * RGX_FEATURE_NUM_MEMBUS is appropriately set.
+		 */
+		case RGX_CNTBLK_ID_SLCBANK1:
+			if (ui32NumMemBus >= 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		case RGX_CNTBLK_ID_SLCBANK2:
+		case RGX_CNTBLK_ID_SLCBANK3:
+			if (ui32NumMemBus > 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		default:
+			psRtInfo->uiNumUnits = psBlkTypeDesc->uiNumUnits;
+			break;
+	}
+	if (psRtInfo->uiNumUnits > 0U)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif	/* defined(__KERNEL__) */
+}
+
+#if !defined(__KERNEL__) /* Firmware or User-mode context */
+
+/* Used to instantiate a null row in the block type model table below where the
+ * block is not supported for a given build BVNC in firmware/user mode context.
+ * This is needed as the blockid to block type lookup uses the table as well
+ * and clients may try to access blocks not in the hardware. */
+#define RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(_blkid) X(_blkid, 0, 0, #_blkid, NULL, NULL, NULL)
+
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] table
+
+ This table holds the entries for the performance counter block type model.
+ Where the block is not present on an RGX device in question the
+ pfnIsBlkPresent() returns false, if valid and present it returns true.
+ Columns in the table with a ** indicate the value is a default and the value
+ returned in RGX_HWPERF_CNTBLK_RT_INFO when calling pfnIsBlkPresent()should
+ be used at runtime by the caller. These columns are only valid for compile
+ time BVNC configured contexts.
+
+ Order of table rows must match order of counter block IDs in the enumeration
+ RGX_HWPERF_CNTBLK_ID.
+
+ Table contains Xmacro styled entries. Each includer of this file must define
+ a gasCntBlkTypeModel[] structure which is local to itself. Only the layout is
+ defined here.
+
+ uiCntBlkIdBase      : Block-ID
+ uiIndirectReg       : 0 => Direct, non-zero => INDIRECT register address
+ uiNumUnits          : Number of units present on the GPU
+ pszBlockNameComment : Name of the Performance Block
+ pfnIsBlkPowered     : Function to determine power state of block
+ pfnIsBlkPresent     : Function to determine block presence on the core
+ pszBlkCfgValid      : Array of counters valid within this block type
+ *****************************************************************************/
+
+	// Furian 8XT V2 layout:
+
+	/*   uiCntBlkIdBase,  uiIndirectReg,   uiNumUnits**,   pszBlockNameComment, pfnIsBlkPowered,    pfnIsBlkPresent */
+
+	/* RGX_CNTBLK_ID_JONES */
+#if defined(RGX_FIRMWARE) || defined(__KERNEL__)
+
+/* Furian 8XT Direct Performance counter blocks */
+
+#define	RGX_CNT_BLK_TYPE_MODEL_DIRECT_LIST \
+	/* uiCntBlkIdBase,  uiIndirectReg,  uiNumUnits**, pszBlockNameComment, pfnIsBlkPowered, pfnIsBlkPresent */ \
+X(RGX_CNTBLK_ID_JONES, 0, 1, "PERF_BLK_JONES", rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiJONES), \
+X(RGX_CNTBLK_ID_SLC,   0, 1, "PERF_BLK_SLC",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC), \
+X(RGX_CNTBLK_ID_FBCDC, 0, 1, "PERF_BLK_FBCDC", rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiFBCDC), \
+X(RGX_CNTBLK_ID_FW_CUSTOM, 0, 1, "PERF_BLK_FW_CUSTOM", rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiFWCUSTOM), \
+X(RGX_CNTBLK_ID_SLCBANK0,   0, 1, "PERF_BLK_SLC0",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC0), \
+X(RGX_CNTBLK_ID_SLCBANK1,   0, 1, "PERF_BLK_SLC1",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC1), \
+X(RGX_CNTBLK_ID_SLCBANK2,   0, 1, "PERF_BLK_SLC2",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC2), \
+X(RGX_CNTBLK_ID_SLCBANK3,   0, 1, "PERF_BLK_SLC3",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC3)
+
+/* Furian 8XT Indirect Performance counter blocks */
+
+#if !defined(RGX_CR_RAC_INDIRECT)
+#define RGX_CR_RAC_INDIRECT (0x8398U)
+#endif
+
+#define RGX_CNT_BLK_TYPE_MODEL_INDIRECT_LIST \
+	/* uiCntBlkIdBase,  uiIndirectReg,  uiNumUnits**, pszBlockNameComment, pfnIsBlkPowered, pfnIsBlkPresent */ \
+X(RGX_CNTBLK_ID_ISP0, RGX_CR_ISP_INDIRECT, RGX_HWPERF_NUM_SPU * RGX_HWPERF_NUM_ISP_PER_SPU, "PERF_BLK_ISP", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiISP), \
+X(RGX_CNTBLK_ID_MERCER0, RGX_CR_MERCER_INDIRECT, RGX_HWPERF_NUM_MERCER, "PERF_BLK_MERCER", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiMERCER), \
+X(RGX_CNTBLK_ID_PBE0, RGX_CR_PBE_INDIRECT, RGX_HWPERF_NUM_PBE, "PERF_BLK_PBE", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiPBE), \
+X(RGX_CNTBLK_ID_PBE_SHARED0, RGX_CR_PBE_SHARED_INDIRECT, RGX_HWPERF_NUM_PBE_SHARED, "PERF_BLK_PBE_SHARED", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiPBE_SHARED), \
+X(RGX_CNTBLK_ID_USC0, RGX_CR_USC_INDIRECT, RGX_HWPERF_NUM_USC, "PERF_BLK_USC", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiUSC), \
+X(RGX_CNTBLK_ID_TPU0, RGX_CR_TPU_INDIRECT, RGX_HWPERF_NUM_TPU, "PERF_BLK_TPU", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiTPU), \
+X(RGX_CNTBLK_ID_SWIFT0, RGX_CR_SWIFT_INDIRECT, RGX_HWPERF_NUM_SWIFT, "PERF_BLK_SWIFT", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiSWIFT), \
+X(RGX_CNTBLK_ID_TEXAS0, RGX_CR_TEXAS_INDIRECT, RGX_HWPERF_NUM_TEXAS, "PERF_BLK_TEXAS", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiTEXAS), \
+X(RGX_CNTBLK_ID_RAC0, RGX_CR_RAC_INDIRECT, RGX_HWPERF_NUM_RAC, "PERF_BLK_RAC", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiRAC)
+
+#else /* !defined(RGX_FIRMWARE) && !defined(__KERNEL__) */
+
+#error "RGX_FIRMWARE or __KERNEL__ *MUST* be defined"
+
+#endif	/* defined(RGX_FIRMWARE) || defined(__KERNEL__) */
+
+#endif /* RGX_HWPERF_TABLE_H */
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_options.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_options.h
new file mode 100644
index 000000000000..3f489fc7fdee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_options.h
@@ -0,0 +1,350 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX build options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Each build option listed here is packed into a dword which provides up to
+ *  log2(RGX_BUILD_OPTIONS_MASK_KM + 1) flags for KM and
+ *  (32 - log2(RGX_BUILD_OPTIONS_MASK_KM + 1)) flags for UM.
+ * The corresponding bit is set if the build option was enabled at compile
+ * time.
+ *
+ * IMPORTANT: add new options to unused bits or define a new dword
+ * (e.g. RGX_BUILD_OPTIONS_KM2 or RGX_BUILD_OPTIONS2) so that the bitfield
+ * remains backwards compatible.
+ */
+
+#ifndef RGX_OPTIONS_H
+#define RGX_OPTIONS_H
+
+#define OPTIONS_OPEN_SOURCE_EN					(0x1UL << 0)
+#define OPTIONS_PDUMP_EN						(0x1UL << 1)
+#define OPTIONS_SECURE_CONTEXT_SWITCH_EN		(0x1UL << 2)
+#define OPTIONS_SECURE_ALLOC_KM_EN				(0x1UL << 3)
+#define OPTIONS_RGX_EN							(0x1UL << 4)
+#define OPTIONS_SECURE_EXPORT_EN				(0x1UL << 5)
+#define OPTIONS_INSECURE_EXPORT_EN				(0x1UL << 6)
+#define OPTIONS_VFP_EN							(0x1UL << 7)
+#define OPTIONS_WORKLOAD_ESTIMATION_EN			(0x1UL << 8)
+#define OPTIONS_PDVFS_EN						(0x1UL << 9)
+#define OPTIONS_DEBUG_EN						(0x1UL << 10)
+#define OPTIONS_BUFFER_SYNC_EN					(0x1UL << 11)
+#define OPTIONS_AUTOVZ_EN						(0x1UL << 12)
+#define OPTIONS_AUTOVZ_HW_REGS_EN				(0x1UL << 13)
+#define OPTIONS_UNUSED2_EN						(0x1UL << 14)
+#define OPTIONS_VALIDATION_EN					(0x1UL << 15)
+#define OPTIONS_NO_HARDWARE_EN					(0x1UL << 16)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN                  (0x1UL << 17)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_MASK                      (0x1fUL << 18)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT                     (18)
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_PERCONTEXT_FREELIST_EN			(0x1UL << 31)
+
+#define RGX_BUILD_OPTIONS_MASK_KM				  \
+			(OPTIONS_OPEN_SOURCE_EN				| \
+			 OPTIONS_PDUMP_EN					| \
+			 OPTIONS_SECURE_CONTEXT_SWITCH_EN	| \
+			 OPTIONS_SECURE_ALLOC_KM_EN			| \
+			 OPTIONS_RGX_EN						| \
+			 OPTIONS_SECURE_EXPORT_EN			| \
+			 OPTIONS_INSECURE_EXPORT_EN			| \
+			 OPTIONS_VFP_EN						| \
+			 OPTIONS_WORKLOAD_ESTIMATION_EN		| \
+			 OPTIONS_PDVFS_EN					| \
+			 OPTIONS_DEBUG_EN					| \
+			 OPTIONS_BUFFER_SYNC_EN				| \
+			 OPTIONS_AUTOVZ_EN					| \
+			 OPTIONS_AUTOVZ_HW_REGS_EN			| \
+			 OPTIONS_VALIDATION_EN				| \
+			 OPTIONS_NO_HARDWARE_EN				| \
+			 OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN)
+
+#define RGX_BUILD_OPTIONS_MASK_FW \
+			(RGX_BUILD_OPTIONS_MASK_KM & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_MASK & \
+			 ~OPTIONS_BUFFER_SYNC_EN)
+
+/* Build options that the FW must have if the present on the KM */
+#define FW_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the UM must have if the present on the KM */
+#define UM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the KM must have if the present on the UM */
+#define KM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN					| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_BUFFER_SYNC_EN))
+
+#define OPEN_SOURCE_OPTION	"OPEN_SOURCE_DRIVER  "
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	#define OPTIONS_BIT0		OPTIONS_OPEN_SOURCE_EN
+	#if OPTIONS_BIT0 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT0		0x0UL
+#endif /* SUPPORT_OPEN_SOURCE_DRIVER */
+
+#define PDUMP_OPTION	"PDUMP  "
+#if defined(PDUMP)
+	#define OPTIONS_BIT1		OPTIONS_PDUMP_EN
+	#if OPTIONS_BIT1 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT1		0x0UL
+#endif /* PDUMP */
+
+#define SECURE_CONTEXT_SWITCH_OPTION "SECURE_CONTEXT_SWITCH "
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	#define OPTIONS_BIT2		OPTIONS_SECURE_CONTEXT_SWITCH_EN
+	#if OPTIONS_BIT2 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT2		0x0UL
+#endif /* SUPPORT_SECURE_CONTEXT_SWITCH */
+
+#define SECURE_ALLOC_KM_OPTION "SECURE_ALLOC_KM  "
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	#define OPTIONS_BIT3		OPTIONS_SECURE_ALLOC_KM_EN
+	#if OPTIONS_BIT3 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT3 0x0UL
+#endif /* SUPPORT_SECURE_ALLOC_KM */
+
+#define RGX_OPTION	" "
+#if defined(SUPPORT_RGX)
+	#define OPTIONS_BIT4		OPTIONS_RGX_EN
+	#if OPTIONS_BIT4 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT4		0x0UL
+#endif /* SUPPORT_RGX */
+
+#define SECURE_EXPORT_OPTION	"SECURE_EXPORTS  "
+#if defined(SUPPORT_SECURE_EXPORT)
+	#define OPTIONS_BIT5		OPTIONS_SECURE_EXPORT_EN
+	#if OPTIONS_BIT5 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT5		0x0UL
+#endif /* SUPPORT_SECURE_EXPORT */
+
+#define INSECURE_EXPORT_OPTION	"INSECURE_EXPORTS  "
+#if defined(SUPPORT_INSECURE_EXPORT)
+	#define OPTIONS_BIT6		OPTIONS_INSECURE_EXPORT_EN
+	#if OPTIONS_BIT6 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT6		0x0UL
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+#define VFP_OPTION	"VFP  "
+#if defined(SUPPORT_VFP)
+	#define OPTIONS_BIT7		OPTIONS_VFP_EN
+	#if OPTIONS_BIT7 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT7		0x0UL
+#endif /* SUPPORT_VFP */
+
+#define WORKLOAD_ESTIMATION_OPTION	"WORKLOAD_ESTIMATION  "
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	#define OPTIONS_BIT8		OPTIONS_WORKLOAD_ESTIMATION_EN
+	#if OPTIONS_BIT8 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT8		0x0UL
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+
+#define PDVFS_OPTION	"PDVFS  "
+#if defined(SUPPORT_PDVFS)
+	#define OPTIONS_BIT9		OPTIONS_PDVFS_EN
+	#if OPTIONS_BIT9 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT9		0x0UL
+#endif /* SUPPORT_PDVFS */
+
+#define DEBUG_OPTION	"DEBUG  "
+#if defined(DEBUG)
+	#define OPTIONS_BIT10		OPTIONS_DEBUG_EN
+	#if OPTIONS_BIT10 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT10		0x0UL
+#endif /* DEBUG */
+
+#define BUFFER_SYNC_OPTION	"BUFFER_SYNC  "
+#if defined(SUPPORT_BUFFER_SYNC)
+	#define OPTIONS_BIT11		OPTIONS_BUFFER_SYNC_EN
+	#if OPTIONS_BIT11 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT11		0x0UL
+#endif /* SUPPORT_BUFFER_SYNC */
+
+#define AUTOVZ_OPTION	"AUTOVZ  "
+#if defined(SUPPORT_AUTOVZ)
+	#define OPTIONS_BIT12		OPTIONS_AUTOVZ_EN
+	#if OPTIONS_BIT12 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT12		0x0UL
+#endif /* SUPPORT_AUTOVZ */
+
+#define AUTOVZ_HW_REGS_OPTION	"AUTOVZ_HW_REGS  "
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+	#define OPTIONS_BIT13		OPTIONS_AUTOVZ_HW_REGS_EN
+	#if OPTIONS_BIT13 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT13		0x0UL
+#endif /* SUPPORT_AUTOVZ_HW_REGS */
+
+/* UNUSED bit, kept for rogue/volcanic compatibility */
+#define INTERNAL_UNUSED2_OPTION	"INTERNAL_UNUSED2  "
+#if defined(INTERNAL_UNUSED2)
+	#define OPTIONS_BIT14		OPTIONS_UNUSED2_EN
+	#if OPTIONS_BIT14 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT14		0x0UL
+#endif
+
+#define VALIDATION_OPTION	"VALIDATION  "
+	#define OPTIONS_BIT15		0x0UL
+
+#define NO_HARDWARE_OPTION	"NO_HARDWARE  "
+#if defined(NO_HARDWARE)
+	#define OPTIONS_BIT16		OPTIONS_NO_HARDWARE_EN
+	#if OPTIONS_BIT16 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT16		0x0UL
+#endif /* NO_HARDWARE */
+
+#define NUM_DRIVERS_SUPPORTED_CHECK_OPTION	"NUM_DRIVERS_SUPPORTED_CHECK  "
+	#define OPTIONS_BIT17		OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN
+	#if OPTIONS_BIT17 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif /* NUM_DRIVERS_SUPPORTED_CHECK */
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_BIT31		OPTIONS_PERCONTEXT_FREELIST_EN
+#if OPTIONS_BIT31 <= RGX_BUILD_OPTIONS_MASK_KM
+#error "Bit exceeds reserved range"
+#endif
+
+#define RGX_BUILD_OPTIONS_KM	\
+	(OPTIONS_BIT0				|\
+	 OPTIONS_BIT1				|\
+	 OPTIONS_BIT2				|\
+	 OPTIONS_BIT3				|\
+	 OPTIONS_BIT4				|\
+	 OPTIONS_BIT5				|\
+	 OPTIONS_BIT6				|\
+	 OPTIONS_BIT7				|\
+	 OPTIONS_BIT8				|\
+	 OPTIONS_BIT9				|\
+	 OPTIONS_BIT10				|\
+	 OPTIONS_BIT11				|\
+	 OPTIONS_BIT12				|\
+	 OPTIONS_BIT13				|\
+	 OPTIONS_BIT14				|\
+	 OPTIONS_BIT15				|\
+	 OPTIONS_BIT16				|\
+	 OPTIONS_BIT17)
+
+#define RGX_BUILD_OPTIONS (RGX_BUILD_OPTIONS_KM | OPTIONS_BIT31)
+
+#define RGX_BUILD_OPTIONS_LIST			\
+	{									\
+		OPEN_SOURCE_OPTION,				\
+		PDUMP_OPTION,					\
+		SECURE_CONTEXT_SWITCH_OPTION,	\
+		SECURE_ALLOC_KM_OPTION,			\
+		RGX_OPTION,						\
+		SECURE_EXPORT_OPTION,			\
+		INSECURE_EXPORT_OPTION,			\
+		VFP_OPTION,						\
+		WORKLOAD_ESTIMATION_OPTION,		\
+		PDVFS_OPTION,					\
+		DEBUG_OPTION,					\
+		BUFFER_SYNC_OPTION,				\
+		AUTOVZ_OPTION,					\
+		AUTOVZ_HW_REGS_OPTION,			\
+		INTERNAL_UNUSED2_OPTION,		\
+		VALIDATION_OPTION,				\
+		NO_HARDWARE_OPTION,				\
+		NUM_DRIVERS_SUPPORTED_CHECK_OPTION				\
+	}
+
+#endif /* RGX_OPTIONS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgxheapconfig.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgxheapconfig.h
new file mode 100644
index 000000000000..00801fcaae84
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgxheapconfig.h
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_H
+#define RGXHEAPCONFIG_H
+
+#include "rgxdefs_km.h"
+
+#define RGX_HEAP_SIZE_32KiB      IMG_UINT64_C(0x0000008000)
+#define RGX_HEAP_SIZE_2MiB       IMG_UINT64_C(0x0000200000)
+#define RGX_HEAP_SIZE_4MiB       IMG_UINT64_C(0x0000400000)
+#define RGX_HEAP_SIZE_16MiB      IMG_UINT64_C(0x0001000000)
+#define RGX_HEAP_SIZE_32MiB      IMG_UINT64_C(0x0002000000)
+#define RGX_HEAP_SIZE_256MiB     IMG_UINT64_C(0x0010000000)
+
+#define RGX_HEAP_SIZE_1GiB       IMG_UINT64_C(0x0040000000)
+#define RGX_HEAP_SIZE_2GiB       IMG_UINT64_C(0x0080000000)
+#define RGX_HEAP_SIZE_4GiB       IMG_UINT64_C(0x0100000000)
+#define RGX_HEAP_SIZE_16GiB      IMG_UINT64_C(0x0400000000)
+#define RGX_HEAP_SIZE_32GiB      IMG_UINT64_C(0x0800000000)
+#define RGX_HEAP_SIZE_64GiB      IMG_UINT64_C(0x1000000000)
+#define RGX_HEAP_SIZE_128GiB     IMG_UINT64_C(0x2000000000)
+#define RGX_HEAP_SIZE_256GiB     IMG_UINT64_C(0x4000000000)
+#define RGX_HEAP_SIZE_512GiB     IMG_UINT64_C(0x8000000000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address heaps that are used in
+	application memory contexts. It also shows where the Firmware memory heap
+	fits into this, but the firmware heap is only ever created in the
+	Services KM/server component.
+
+	RGX_PDSCODEDATA_HEAP_BASE and RGX_USCCODE_HEAP_BASE will be programmed,
+	on a global basis, into RGX_CR_PDS_EXEC_BASE and RGX_CR_USC_CODE_BASE_*
+	respectively. Therefore if clients use multiple configs they must still
+	be consistent with their definitions for these heaps.
+
+	Shared virtual memory (GENERAL_SVM) support requires half of the address
+	space (512 GiB) be reserved for SVM allocations to mirror application CPU
+	addresses.
+
+	The GENERAL non-SVM region is 512 GiB to 768 GiB and is shared between the
+	general (4KiB) heap and the general non-4K heap. The first 128 GiB is used
+	for the GENERAL_HEAP (4KiB) and the last 32 GiB is used for the
+	GENERAL_NON4K_HEAP. This heap has a default page-size of 16K.
+	AppHint PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE can be used to forced it
+	to these values: 4K,64K,256K,1M,2M.
+
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range. Often this is the reserved
+	size of the heap within the range.
+*/
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0000_0000 - 0x00_0020_0000 **/
+	/* 0 MiB to 2 MiB, size of 2 MiB : RESERVED (only when General SVM
+	 *                                           doesn't exist) **/
+
+/* 0x00_0000_8000 - 0x7F_FFFF_8000 **/
+	/* MAX(32 KiB, PAGE_SIZE) to 512 GiB, size of 512 GiB less MAX(32 KiB, PAGE_SIZE) : GENERAL_SVM_HEAP **/
+
+	/* The MAX is determined at runtime (PAGE_SIZE isn't available on all platforms)
+	 * so the #define's must NOT be used directly. Use the heap config after initialisation. */
+	#define RGX_GENERAL_SVM_HEAP_BASE           IMG_UINT64_C(0x0000008000)
+	#define RGX_GENERAL_SVM_HEAP_SIZE           (RGX_HEAP_SIZE_512GiB - RGX_HEAP_SIZE_32KiB)
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0x80_0000_0000 - 0x9F_FFFF_FFFF **/
+	/* 512 GiB to 640 GiB, size of 128 GiB : GENERAL_HEAP **/
+	#define RGX_GENERAL_HEAP_BASE               IMG_UINT64_C(0x8000000000)
+	#define RGX_GENERAL_HEAP_SIZE               RGX_HEAP_SIZE_128GiB
+
+/* 0xA0_0000_0000 - 0xAF_FFFF_FFFF **/
+	/* 640 GiB to 704 GiB, size of 64 GiB : FREE **/
+
+/* 0xB0_0000_0000 - 0xB1_FFFF_FFFF **/
+	/* 704 GiB to 720 GiB, size of 16 GiB : RESERVED ROGUE **/
+
+/* B4_0000_0000 - 0xB7_FFFF_FFFF **/
+	/* 720 GiB to 736 GiB, size of 16 GiB : FREE **/
+
+/* 0xB8_0000_0000 - 0xBF_FFFF_FFFF **/
+	/* 736 GiB to 768 GiB, size of 32 GiB : GENERAL_NON4K_HEAP **/
+	#define RGX_GENERAL_NON4K_HEAP_BASE         IMG_UINT64_C(0xB800000000)
+	#define RGX_GENERAL_NON4K_HEAP_SIZE         RGX_HEAP_SIZE_32GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xC0_0000_0000 - 0xD9_FFFF_FFFF **/
+	/* 768 GiB to 872 GiB, size of 104 GiB : FREE **/
+
+/* 0xDA_0000_0000 - 0xDA_FFFF_FFFF **/
+	/* 872 GiB to 876 GiB, size of 4 GiB : PDSCODEDATA_HEAP **/
+	#define RGX_PDSCODEDATA_HEAP_BASE           IMG_UINT64_C(0xDA00000000)
+	#define RGX_PDSCODEDATA_HEAP_SIZE           RGX_HEAP_SIZE_4GiB
+
+/* 0xDB_0000_0000 - 0xDC_FFFF_FFFF **/
+	/* 876 GiB to 884 GiB, size of 8 GiB : RESERVED ROGUE **/
+
+/* 0xDD_0000_0000 - 0xDF_FFFF_FFFF **/
+	/* 884 GiB to 896 GiB, size of 12 GiB : FREE **/
+
+	/*
+	 * The breakpoint handler code heap must share the same 4GiB address
+	 * range as the USC shader code heap. The address space split is
+	 * configurable.
+	 *
+	 * The breakpoint handler register is fixed, so the following parts
+	 * of the BP handler address are static:
+	 * [31:24] = 0xFE for 4064MiB offset after the USC code heap.
+	 *   [5:0] = 0x00 i.e. aligned to 64 Byte boundary.
+	 *
+	 * The remaining part of the BP handler is dynamic and encoded in
+	 * the USC instruction BABP_target_addr. The BP handler thus
+	 * allows a range of 16 MiB with granularity of 64 Bytes.
+	 */
+
+/* 0xE0_0000_0000 - 0xE0_FDFF_FFFF **/
+	/* 896 GiB to 900 GiB, size of 4 GiB less 32 MiB : USCCODE_HEAP **/
+	#define RGX_USCCODE_HEAP_BASE               IMG_UINT64_C(0xE000000000)
+	#define RGX_USCCODE_HEAP_SIZE               (RGX_HEAP_SIZE_4GiB - RGX_HEAP_SIZE_32MiB)
+
+/* 0xE0_FE00_0000 - 0xE0_FEFF_FFFF **/
+	/* 900 GiB less 32 MiB to 900 GiB less 16 MiB, size of 16 MiB : USCCODE_BPH_HEAP **/
+	#define RGX_USCCODE_BPH_HEAP_BASE           (IMG_UINT64_C(0xE100000000) - RGX_HEAP_SIZE_32MiB)
+	#define RGX_USCCODE_BPH_HEAP_SIZE           RGX_HEAP_SIZE_16MiB
+
+/* 0xE0_FF00_0000 - 0xE0_FFFF_FFFF **/
+	/* 900 GiB less 16 MiB to 900 GiB, size of 16 MiB : RESERVED **/
+
+/* 0xE1_0000_0000 - 0xE1_BFFF_FFFF **/
+	/* 900 GiB to 903 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xE1_C000_0000 - 0xE1_FFFF_FFFF **/
+	/* 903 GiB to 904 GiB, reserved 1 GiB, : FIRMWARE_HEAP **/
+
+	/* Firmware heaps defined in rgx_heap_firmware.h as they are not present in
+	   application memory contexts, see:
+	    RGX_FIRMWARE_RAW_HEAP_BASE
+	    RGX_FIRMWARE_RAW_HEAP_SIZE
+	   See header for other sub-heaps details
+	*/
+
+/* 0xE2_0000_0000 - 0xE2_FFFF_FFFF **/
+	/* 904 GiB to 908 GiB, size of 4GiB : RESERVED ROGUE **/
+
+/* 0xE3_0000_0000 - 0xE3_FFFF_FFFF **/
+	/* 908 GiB to 912 GiB, size of 4 GiB : FREE **/
+
+/* 0xE4_0000_0000 - 0xE7_FFFF_FFFF **/
+	/* 912 GiB to 928 GiB, size 16 GiB : RESERVED_ROGUE **/
+
+/* 0xE8_0000_0000 - 0xE8_FFFF_FFFF **/
+	/* 928 GiB to 932 GiB, size of 4 GiB : FREE **/
+
+/* 0xE9_0000_0000 - 0xE9_3FFF_FFFF **/
+	/* 932 GiB to 933 GiB, size of 1 GiB : VK_CAPT_REPLAY_HEAP **/
+	#define RGX_VK_CAPT_REPLAY_HEAP_BASE    IMG_UINT64_C(0xE900000000)
+	#define RGX_VK_CAPT_REPLAY_HEAP_SIZE    RGX_HEAP_SIZE_1GiB
+
+/* 0xE9_4000_0000 - 0xE9_FFFF_FFFF **/
+	/* 933 GiB to 936 GiB, size of 3 GiB : FREE **/
+
+/* 0xEA_0000_0000 - 0xEA_001F_FFFF **/
+	/* 936 GiB to 937 GiB, size of 1 GiB : FREE **/
+
+/* 0xEA_4000_0000 - 0xEA_FFFF_FFFF **/
+	/* 937 GiB to 940 GiB, size of 3 GiB : FREE **/
+
+/* 0xEB_0000_0000 - 0xEB_FFFF_FFFF **/
+	/* 940 GiB to 944 GiB, size 4 GiB : COMPONENT_CTRL_HEAP **/
+	#define RGX_COMPONENT_CTRL_HEAP_BASE        IMG_UINT64_C(0xEB00000000)
+	#define RGX_COMPONENT_CTRL_HEAP_SIZE        RGX_HEAP_SIZE_4GiB
+
+/* 0xEC_0000_0000 - 0xEC_001F_FFFF **/
+	/* 944 GiB to 945 GiB, size 2 MiB : FBCDC_HEAP **/
+	#define RGX_FBCDC_HEAP_BASE                 IMG_UINT64_C(0xEC00000000)
+	#define RGX_FBCDC_HEAP_SIZE                 RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_4000_0000 - 0xEC_401F_FFFF **/
+	/* 945 GiB to 946 GiB, size 2 MiB : FBCDC_LARGE_HEAP **/
+	#define RGX_FBCDC_LARGE_HEAP_BASE           IMG_UINT64_C(0xEC40000000)
+	#define RGX_FBCDC_LARGE_HEAP_SIZE           RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_8000_0000 - 0xEC_FFFF_FFFF **/
+	/* 946 GiB to 948 GiB, size of 3 GiB : FREE **/
+
+/* 0xED_0000_0000 - 0xED_00FF_FFFF */
+	/* 948 GiB to 949 GiB, size 16 MiB : PDS_INDIRECT_STATE_HEAP */
+	#define RGX_PDS_INDIRECT_STATE_HEAP_BASE    IMG_UINT64_C(0xED00000000)
+	#define RGX_PDS_INDIRECT_STATE_HEAP_SIZE    RGX_HEAP_SIZE_16MiB
+
+/* 0xED_4000_0000 - 0xED_FFFF_FFFF **/
+	/* 949 GiB to 952 GiB, size of 3 GiB : FREE **/
+
+/* 0xEE_0000_0000 - 0xEE_3FFF_FFFF **/
+	/* 952 GiB to 953 GiB, size of 1 GiB : CMP_MISSION_RMW_HEAP **/
+	#define RGX_CMP_MISSION_RMW_HEAP_BASE       IMG_UINT64_C(0xEE00000000)
+	#define RGX_CMP_MISSION_RMW_HEAP_SIZE       RGX_HEAP_SIZE_1GiB
+
+/* 0xEE_4000_0000 - 0xEE_FFFF_FFFF **/
+	/* 953 GiB to 956 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xEF_0000_0000 - 0xEF_3FFF_FFFF **/
+	/* 956 GiB to 957 GiB, size of 1 GiB : CMP_SAFETY_RMW_HEAP **/
+	#define RGX_CMP_SAFETY_RMW_HEAP_BASE        IMG_UINT64_C(0xEF00000000)
+	#define RGX_CMP_SAFETY_RMW_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xEF_4000_0000 - 0xEF_FFFF_FFFF **/
+	/* 957 GiB to 960 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xF0_0000_0000 - 0xF0_FFFF_FFFF **/
+	/* 960 GiB to 964 GiB, size of 4 GiB : TEXTURE_STATE_HEAP (36-bit aligned) */
+	#define RGX_TEXTURE_STATE_HEAP_BASE         IMG_UINT64_C(0xF000000000)
+	#define RGX_TEXTURE_STATE_HEAP_SIZE         RGX_HEAP_SIZE_4GiB
+
+/* 0xF1_0000_0000 - 0xF1_FFFF_FFFF **/
+	/* 964 GiB to 968 GiB, size of 4 GiB : FREE **/
+
+/* 0xF2_0000_0000 - 0xF2_001F_FFFF **/
+	/* 968 GiB to 969 GiB, size of 2 MiB : VISIBILITY_TEST_HEAP **/
+	#define RGX_VISIBILITY_TEST_HEAP_BASE       IMG_UINT64_C(0xF200000000)
+	#define RGX_VISIBILITY_TEST_HEAP_SIZE       RGX_HEAP_SIZE_2MiB
+
+/* 0xF2_4000_0000 - 0xF2_FFFF_FFFF **/
+	/* 969 GiB to 972 GiB, size of 3 GiB : FREE **/
+
+/* 0xF3_0000_0000 - 0xF7_FFFF_FFFF **/
+	/* 972 GiB to 992 GiB, size of 20 GiB : FREE **/
+
+/* 0xF8_0000_0000 - 0xF9_FFFF_FFFF **/
+	/* 992 GiB to 1000 GiB, size 8 GiB : RESERVED ROGUE **/
+
+/* 0xFA_0000_0000 - 0xFF_FFFF_FFFF **/
+	/* 1000 GiB to 1024 GiB, size of 24 GiB : FREE **/
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/system/rgx_tc/tc_clocks.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/system/rgx_tc/tc_clocks.h
new file mode 100644
index 000000000000..9463830fa1c4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/system/rgx_tc/tc_clocks.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TC_CLOCKS_H)
+#define TC_CLOCKS_H
+
+/*
+ * The core clock speed is passed through a multiplier depending on the TC
+ * version.
+ *
+ * On TC_ES1: Multiplier = x3, final speed = 270MHz
+ * On TC_ES2: Multiplier = x6, final speed = 540MHz
+ * On TCF5:   Multiplier = 1x final speed = 45MHz
+ *
+ *
+ * The base (unmultiplied speed) can be adjusted using a module parameter
+ * called "sys_core_clk_speed", a number in Hz.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_core_clk_speed=60000000" /etc/init.d/rc.pvr start
+ *
+ * would result in a core speed of 60MHz xMultiplier.
+ *
+ *
+ * The memory clock is unmultiplied and can be adjusted using a module
+ * parameter called "sys_mem_clk_speed", this should be the number in Hz for
+ * the memory clock speed.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=100000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the memory clock speed set to 100MHz.
+ *
+ *
+ * Same applies to the system interface clock speed, "sys_sysif_clk_speed".
+ * Needed for TCF5 but not for TC_ES2/ES1.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_sysif_clk_speed=45000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the system clock speed set to 45MHz.
+ *
+ *
+ * All parameters can be specified at once, e.g.,
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=MEMORY_SPEED sys_core_clk_speed=CORE_SPEED sys_sysif_clk_speed=SYSIF_SPEED" /etc/init.d/rc.pvr start
+ */
+
+#define RGX_TC_SYS_CLOCK_SPEED		(45000000) /*< Unused */
+
+#if defined(TC_ODIN_27_5_254_2)
+ #define RGX_TC_CORE_CLOCK_SPEED	(94000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+ #define RGX_TC_CLOCK_MULTIPLEX    (16)
+#else
+ /* FPGA tcfvuquad with Odin */
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000) /* 3.125MHz */
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000) /* 3.75MHz */
+ #define RGX_TC_CLOCK_MULTIPLEX    (1)
+#endif
+
+#endif /* if !defined(TC_CLOCKS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/common/env/linux/dc_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/common/env/linux/dc_osfuncs.c
new file mode 100644
index 000000000000..3d6ce091d727
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/common/env/linux/dc_osfuncs.c
@@ -0,0 +1,464 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include "pvr_notifier.h"
+#include "srvkm.h"
+#include "dc_osfuncs.h"
+#include <linux/version.h>
+#include <linux/bug.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/workqueue.h>
+#include <asm/page.h>
+
+#if defined(CONFIG_X86)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,2,0))
+#include <asm/fpu/internal.h>
+#else
+#include <asm/i387.h>
+#endif
+#endif
+
+#define MAX_DRVNAME 128
+static IMG_CHAR g_szDrvName[MAX_DRVNAME + 1];
+
+typedef struct WORK_ITEM_TAG
+{
+	struct work_struct	sWork;
+	PFN_WORK_PROCESSOR	pfnProcessor;
+	void			*pvData;
+} WORK_ITEM;
+
+void DC_OSSetDrvName(const IMG_CHAR *pszDrvName)
+{
+	strncpy(g_szDrvName, pszDrvName, MAX_DRVNAME);
+	g_szDrvName[MAX_DRVNAME] = '\0'; /* ensure null term (not guaranteed by strncpy) */
+}
+
+void DC_OSAbort(const IMG_CHAR *pszFile, IMG_UINT32 ui32Line)
+{
+	printk(KERN_EMERG "%s - Assertion failed at %s:%d\n", g_szDrvName, pszFile, ui32Line);
+	BUG();
+}
+
+void DC_OSDebugPrintf(DC_OS_DEBUG_LEVEL eDebugLevel, const IMG_CHAR *pszFormat, ...)
+{
+	va_list args;
+
+	switch (eDebugLevel)
+	{
+		case DBGLVL_FATAL:
+		{
+			printk(KERN_EMERG " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_ALERT:
+		{
+			printk(KERN_ALERT " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_ERROR:
+		{
+			printk(KERN_ERR " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_WARNING:
+		{
+			printk(KERN_WARNING " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_NOTICE:
+		{
+			printk(KERN_NOTICE " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_INFO:
+		{
+			printk(KERN_INFO " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_DEBUG:
+		{
+			printk(KERN_DEBUG " %s", g_szDrvName);
+			break;
+		}
+	}
+
+	va_start(args, pszFormat);
+	vprintk(pszFormat, args);
+	va_end(args);
+}
+
+IMG_CHAR *DC_OSStringNCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uiLength)
+{
+	return strncpy(pszDest, pszSrc, uiLength);
+}
+
+IMG_INT DC_OSSNPrintf(IMG_CHAR *pszDest, size_t uiLength, const IMG_CHAR *pszFormat, ...)
+{
+	va_list args;
+	IMG_INT iRetVal;
+
+	va_start(args, pszFormat);
+	iRetVal = vsnprintf(pszDest, uiLength, pszFormat, args);
+	va_end(args);
+
+	return iRetVal;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0))
+IMG_INT64 DC_OSClockns(void)
+{
+	struct timespec64 now;
+
+	ktime_get_raw_ts64(&now);
+	return now.tv_sec * 1000000000 + now.tv_nsec;
+}
+#else
+IMG_INT64 DC_OSClockns(void)
+{
+	struct timespec now;
+	ktime_get_ts(&now);
+	return now.tv_sec * 1000000000ULL + now.tv_nsec;
+}
+#endif
+
+IMG_UINT32 DC_OSGetPageSize(void)
+{
+	return PAGE_SIZE;
+}
+
+IMG_UINT32 DC_OSGetPageShift(void)
+{
+	return PAGE_SHIFT;
+}
+
+IMG_UINT32 DC_OSGetPageMask(void)
+{
+	return (IMG_UINT32)PAGE_MASK;
+}
+
+void *DC_OSAllocMem(size_t uiSize)
+{
+	return kmalloc(uiSize, GFP_KERNEL);
+}
+
+void *DC_OSCallocMem(size_t uiSize)
+{
+	return kzalloc(uiSize, GFP_KERNEL);
+}
+
+void DC_OSFreeMem(void *pvMem)
+{
+	kfree(pvMem);
+}
+
+void DC_OSMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t uiSize)
+{
+	memset(pvDest, ui8Value, uiSize);
+}
+
+IMG_UINT32 DC_OSAddrRangeStart(void *pvDevice, IMG_UINT8 ui8BaseNum)
+{
+	struct device *psDev = pvDevice;
+
+	DC_ASSERT(dev_is_pci(psDev));
+
+	return pci_resource_start(to_pci_dev(psDev), ui8BaseNum);
+}
+
+void *DC_OSRequestAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size, IMG_CHAR *pszRequestorName)
+{
+	return request_mem_region(sCpuPAddr.uiAddr, ui32Size, pszRequestorName);
+}
+
+void DC_OSReleaseAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size)
+{
+	release_mem_region(sCpuPAddr.uiAddr, ui32Size);
+}
+
+IMG_CPU_VIRTADDR DC_OSMapPhysAddr(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size)
+{
+	return ioremap(sCpuPAddr.uiAddr, ui32Size);
+}
+
+void DC_OSUnmapPhysAddr(IMG_CPU_VIRTADDR pvCpuVAddr, IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	iounmap(pvCpuVAddr);
+}
+
+IMG_UINT32 DC_OSReadReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset)
+{
+	return ioread32(pvRegCpuVBase + ui32Offset);
+}
+
+void DC_OSWriteReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	iowrite32(ui32Value, (pvRegCpuVBase + ui32Offset));
+}
+
+IMG_UINT32 DC_OSDiv64(IMG_UINT64 ui64Numerator, IMG_UINT32 ui32Denominator)
+{
+	(void)do_div(ui64Numerator, ui32Denominator);
+
+	return (IMG_UINT32)ui64Numerator;
+}
+
+void DC_OSFloatingPointBegin(void)
+{
+#if defined(CONFIG_X86)
+	kernel_fpu_begin();
+#else
+	DC_OSDebugPrintf(DBGLVL_WARNING, " - %s: Unsupported architecture\n", __func__);
+#endif
+}
+
+void DC_OSFloatingPointEnd(void)
+{
+#if defined(CONFIG_X86)
+	kernel_fpu_end();
+#else
+	DC_OSDebugPrintf(DBGLVL_WARNING, " - %s: Unsupported architecture\n", __func__);
+#endif
+}
+
+PVRSRV_ERROR DC_OSMutexCreate(void **ppvMutex)
+{
+	struct mutex *psLock;
+
+	psLock = DC_OSAllocMem(sizeof(*psLock));
+	if (psLock == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	mutex_init(psLock);
+
+	*ppvMutex = (void *)psLock;
+
+	return PVRSRV_OK;
+}
+
+void DC_OSMutexDestroy(void *pvMutex)
+{
+	DC_OSFreeMem(pvMutex);
+}
+
+void DC_OSMutexLock(void *pvMutex)
+{
+	mutex_lock((struct mutex *)pvMutex);
+}
+
+void DC_OSMutexUnlock(void *pvMutex)
+{
+	mutex_unlock((struct mutex *)pvMutex);
+}
+
+void DC_OSDelayus(IMG_UINT32 ui32Timeus)
+{
+	udelay(ui32Timeus);
+}
+
+PVRSRV_ERROR DC_OSPVRServicesConnectionOpen(IMG_HANDLE *phPVRServicesConnection)
+{
+	*phPVRServicesConnection = (IMG_HANDLE)NULL;
+
+	return PVRSRV_OK;
+}
+
+void DC_OSPVRServicesConnectionClose(IMG_HANDLE hPVRServicesConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRServicesConnection);
+
+	DC_ASSERT(hPVRServicesConnection == (IMG_HANDLE)NULL);
+}
+
+PVRSRV_ERROR DC_OSPVRServicesSetupFuncs(IMG_HANDLE hPVRServicesConnection, DC_SERVICES_FUNCS *psServicesFuncs)
+{
+	DC_ASSERT(hPVRServicesConnection == (IMG_HANDLE)NULL);
+
+	if (psServicesFuncs == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psServicesFuncs->pfnDCRegisterDevice = DCRegisterDevice;
+	psServicesFuncs->pfnDCUnregisterDevice = DCUnregisterDevice;
+	psServicesFuncs->pfnDCDisplayConfigurationRetired = DCDisplayConfigurationRetired;
+	psServicesFuncs->pfnDCImportBufferAcquire = DCImportBufferAcquire;
+	psServicesFuncs->pfnDCImportBufferRelease = DCImportBufferRelease;
+
+	psServicesFuncs->pfnPhysHeapAcquireByUsage = PhysHeapAcquireByUsage;
+	psServicesFuncs->pfnPhysHeapRelease = PhysHeapRelease;
+	psServicesFuncs->pfnPhysHeapGetType = PhysHeapGetType;
+	psServicesFuncs->pfnPhysHeapGetSize = PhysHeapGetSize;
+	psServicesFuncs->pfnPhysHeapGetCpuPAddr = PhysHeapGetCpuPAddr;
+	psServicesFuncs->pfnPhysHeapCpuPAddrToDevPAddr = PhysHeapCpuPAddrToDevPAddr;
+
+	psServicesFuncs->pfnSysInstallDeviceLISR = PVRSRVSystemInstallDeviceLISR;
+	psServicesFuncs->pfnSysUninstallDeviceLISR = PVRSRVSystemUninstallDeviceLISR;
+
+	psServicesFuncs->pfnCheckStatus = PVRSRVCheckStatus;
+	psServicesFuncs->pfnGetErrorString = PVRSRVGetErrorString;
+	psServicesFuncs->pfnGetDeviceInstance = PVRSRVGetDeviceInstance;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueCreate(IMG_HANDLE *phQueue, IMG_UINT32 ui32Length)
+{
+	struct workqueue_struct *psQueue;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Length);
+
+	if (phQueue == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psQueue = alloc_ordered_workqueue("%s", WQ_FREEZABLE | WQ_MEM_RECLAIM, g_szDrvName);
+	if (psQueue == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phQueue = (IMG_HANDLE)psQueue;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueDestroy(IMG_HANDLE hQueue)
+{
+	struct workqueue_struct *psQueue = (struct workqueue_struct *)hQueue;
+
+	DC_ASSERT(psQueue);
+
+	destroy_workqueue(psQueue);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueFlush(IMG_HANDLE hQueue)
+{
+	struct workqueue_struct *psQueue = (struct workqueue_struct *)hQueue;
+
+	DC_ASSERT(psQueue);
+
+	flush_workqueue(psQueue);
+
+	return PVRSRV_OK;
+}
+
+static void WorkQueueProcessorWrapper(struct work_struct *psWork)
+{
+	WORK_ITEM *psWorkItem = container_of(psWork, WORK_ITEM, sWork);
+
+#if defined(DEBUG)
+	DC_ASSERT(psWorkItem && psWorkItem->pfnProcessor);
+#else
+	// klocwork workaround
+	if (psWorkItem == NULL || psWorkItem->pfnProcessor == NULL)
+	{
+		DC_OSAbort(__FILE__, __LINE__);
+		return;
+	}
+#endif
+
+	(void)psWorkItem->pfnProcessor(psWorkItem->pvData);
+}
+
+PVRSRV_ERROR DC_OSWorkQueueCreateWorkItem(IMG_HANDLE *phWorkItem, PFN_WORK_PROCESSOR pfnProcessor, void *pvProcessorData)
+{
+	WORK_ITEM *psWorkItem;
+
+	if (phWorkItem == NULL || pfnProcessor == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psWorkItem = DC_OSAllocMem(sizeof(*psWorkItem));
+	if (psWorkItem == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_WORK(&psWorkItem->sWork, WorkQueueProcessorWrapper);
+
+	psWorkItem->pfnProcessor = pfnProcessor;
+	psWorkItem->pvData = pvProcessorData;
+
+	*phWorkItem = (IMG_HANDLE)psWorkItem;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueDestroyWorkItem(IMG_HANDLE phWorkItem)
+{
+	WORK_ITEM *psWorkItem = (WORK_ITEM *)phWorkItem;
+
+	DC_ASSERT(psWorkItem);
+
+	DC_OSFreeMem(psWorkItem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueAddWorkItem(IMG_HANDLE hQueue, IMG_HANDLE hWorkItem)
+{
+	struct workqueue_struct *psQueue = (struct workqueue_struct *)hQueue;
+	WORK_ITEM *psWorkItem = (WORK_ITEM *)hWorkItem;
+
+	DC_ASSERT(psQueue && psWorkItem);
+
+	if (!queue_work(psQueue, &psWorkItem->sWork))
+	{
+		printk(KERN_WARNING " %s - %s: Cannot queue work that's already queued\n",
+		       g_szDrvName, __func__);
+		return PVRSRV_ERROR_UNABLE_TO_SCHEDULE_TASK;
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/common/dc_example.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/common/dc_example.c
new file mode 100644
index 000000000000..893b5f0fef36
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/common/dc_example.c
@@ -0,0 +1,1675 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Services headers */
+#include <powervr/imgpixfmts.h>
+#include <powervr/buffer_attribs.h>
+#include "dc_osfuncs.h"
+#include "dc_example.h"
+#include "img_defs.h"
+#include "kerneldisplay.h"
+
+/*
+	Enable to track contexts and buffers
+*/
+/* #define DCEX_DEBUG 1 */
+
+/*
+	Enable to get more debug. Only supported on UMA
+*/
+/* #define DCEX_VERBOSE 1*/
+
+#if defined(DCEX_DEBUG)
+	#define DCEX_DEBUG_PRINT(fmt, ...) \
+		DC_OSDebugPrintf(DBGLVL_WARNING, fmt, __VA_ARGS__)
+#else
+	#define DCEX_DEBUG_PRINT(fmt, ...)
+#endif
+
+/*
+	The number of inflight commands this display driver can handle
+*/
+#define MAX_COMMANDS_INFLIGHT 2
+
+#ifdef NO_HARDWARE
+#define MAX_PIPES 1
+#else
+#define MAX_PIPES 8
+#endif
+
+static IMG_UINT32 ui32ByteStride;
+static IMG_UINT32 ePixelFormat;
+
+static IMG_BOOL CheckBufferDimensions(void)
+{
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams;
+	IMG_UINT32 ui32BytesPP = 0;
+
+	psModuleParams = DCExampleGetModuleParameters();
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return IMG_FALSE;
+	}
+
+	if (psModuleParams->ui32Width == 0 ||
+	    psModuleParams->ui32Height == 0 ||
+	    psModuleParams->ui32Depth == 0)
+	{
+		DC_OSDebugPrintf(
+			DBGLVL_WARNING,
+			": Illegal module parameters (width %u, height %u, depth %u)\n",
+			psModuleParams->ui32Width,
+			psModuleParams->ui32Height,
+			psModuleParams->ui32Depth);
+
+		return IMG_FALSE;
+	}
+
+	switch (psModuleParams->ui32Depth)
+	{
+		case 32:
+			switch (psModuleParams->ui32Format)
+			{
+				case 0:
+					ePixelFormat = IMG_PIXFMT_B8G8R8A8_UNORM;
+					break;
+				case 1:
+					ePixelFormat = IMG_PIXFMT_B10G10R10A2_UNORM;
+					break;
+				default:
+					ePixelFormat = IMG_PIXFMT_UNKNOWN;
+					break;
+			}
+
+			ui32BytesPP = 4;
+			break;
+		case 16:
+			switch (psModuleParams->ui32Format)
+			{
+				case 0:
+					ePixelFormat = IMG_PIXFMT_B5G6R5_UNORM;
+					break;
+				default:
+					ePixelFormat = IMG_PIXFMT_UNKNOWN;
+					break;
+			}
+
+			ui32BytesPP = 2;
+			break;
+		default:
+			DC_OSDebugPrintf(DBGLVL_WARNING, ": Display depth %lu not supported\n", psModuleParams->ui32Depth);
+
+			ePixelFormat = IMG_PIXFMT_UNKNOWN;
+			break;
+	}
+
+	if (ePixelFormat == IMG_PIXFMT_UNKNOWN)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING,
+						 ": Display format %lu not supported for depth %lu\n",
+						 psModuleParams->ui32Format, psModuleParams->ui32Depth);
+		return IMG_FALSE;
+	}
+
+	if ((psModuleParams->ui32MemLayout == 1 && psModuleParams->ui32FBCFormat == 0) ||
+	    (psModuleParams->ui32MemLayout != 1 && psModuleParams->ui32FBCFormat != 0))
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR,
+				 ": Invalid memory layout/FBC mode combination (memory layout %u, FBC format %u)\n",
+				 psModuleParams->ui32MemLayout,
+				 psModuleParams->ui32FBCFormat);
+
+		return IMG_FALSE;
+	}
+
+	ui32ByteStride = psModuleParams->ui32Width * ui32BytesPP;
+
+	DC_OSDebugPrintf(DBGLVL_INFO, " Width: %u pixels\n", psModuleParams->ui32Width);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Height: %u pixels\n", psModuleParams->ui32Height);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Stride: %u bytes\n", ui32ByteStride);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Depth: %u bits\n", psModuleParams->ui32Depth);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Format: %u\n", psModuleParams->ui32Format);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Memory layout: %u\n", psModuleParams->ui32MemLayout);
+	DC_OSDebugPrintf(DBGLVL_INFO, " FBC format: %u\n", psModuleParams->ui32FBCFormat);
+	DC_OSDebugPrintf(DBGLVL_INFO, " X-Dpi: %u\n", psModuleParams->ui32XDpi);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Y-Dpi: %u\n", psModuleParams->ui32YDpi);
+
+	return IMG_TRUE;
+}
+
+
+typedef struct _DCEX_DEVICE_
+{
+	IMG_HANDLE		hPVRServicesConnection;
+	IMG_HANDLE		hPVRServicesDevice;
+	DC_SERVICES_FUNCS	sPVRServicesFuncs;
+
+	IMG_HANDLE		hSrvHandle;
+#if defined(LMA)
+	PHYS_HEAP		*psPhysHeap;
+	IMG_CPU_PHYADDR		sDispStartAddr;
+	IMG_UINT64		uiDispMemSize;
+	IMG_UINT32		ui32BufferSize;
+	IMG_UINT32		ui32BufferCount;
+	IMG_UINT32		ui32BufferUseMask;
+#endif
+	DLLIST_NODE		sListNode;      /* List of all device nodes */
+	void			*hBufListLock;  /* Lock for accessing sBufListNode */
+	DLLIST_NODE		sBufListNode;   /* List of Buffers for this device */
+	/* Per-device context information */
+	IMG_HANDLE		hConfigData[MAX_COMMANDS_INFLIGHT];
+	IMG_UINT32		ui32Head;
+	IMG_UINT32		ui32Tail;
+	IMG_BOOL		b_DisplayContextActive;
+} DCEX_DEVICE;
+
+typedef enum
+{
+	DCEX_BUFFER_SOURCE_ALLOC,
+	DCEX_BUFFER_SOURCE_IMPORT,
+} DCEX_BUFFER_SOURCE;
+
+typedef struct _DCEX_BUFFER_
+{
+	ATOMIC_T		i32RefCount;	/* Only required for system buffer */
+	IMG_BOOL		bIsSysBuffer;	/* Set for system buffer */
+	IMG_UINT32		ePixFormat;
+	IMG_UINT32		ui32Width;
+	IMG_UINT32		ui32Height;
+	IMG_UINT32		ui32ByteStride;
+	IMG_UINT32		ui32Size;
+	IMG_UINT32		ui32PageCount;
+	IMG_HANDLE		hImport;
+	IMG_DEV_PHYADDR		*pasDevPAddr;
+	DCEX_DEVICE		*psDevice;
+#if defined(LMA)
+	IMG_UINT64		uiAllocHandle;
+#else
+	void			*pvAllocHandle;
+#endif
+	DCEX_BUFFER_SOURCE eSource;
+	DLLIST_NODE		sListNode;     /* List of all buffers one per device */
+} DCEX_BUFFER;
+
+static void *g_hDevDataListLock;
+
+static DLLIST_NODE g_sDeviceDataListHead;
+
+static INLINE void DCExampleConfigPush(DCEX_DEVICE *psDevice,
+                                       IMG_HANDLE hConfigData)
+{
+	IMG_UINT32 ui32Head;
+
+	DC_ASSERT(psDevice != NULL);
+	ui32Head = psDevice->ui32Head;
+	psDevice->hConfigData[ui32Head] = hConfigData;
+	ui32Head++;
+
+	if (ui32Head >= MAX_COMMANDS_INFLIGHT)
+	{
+		ui32Head = 0;
+	}
+
+	psDevice->ui32Head = ui32Head;
+}
+
+static INLINE IMG_HANDLE DCExampleConfigPop(DCEX_DEVICE *psDevice)
+{
+	IMG_UINT32 ui32Tail;
+	IMG_HANDLE hConfigData;
+
+	DC_ASSERT(psDevice != NULL);
+	ui32Tail = psDevice->ui32Tail;
+	hConfigData = psDevice->hConfigData[ui32Tail];
+	ui32Tail++;
+
+	if (ui32Tail >= MAX_COMMANDS_INFLIGHT)
+	{
+		ui32Tail = 0;
+	}
+	psDevice->ui32Tail = ui32Tail;
+
+	return hConfigData;
+}
+
+static INLINE IMG_BOOL DCExampleConfigIsEmpty(DCEX_DEVICE *psDevice)
+{
+	DC_ASSERT(psDevice != NULL);
+	return ((psDevice->ui32Tail == psDevice->ui32Head) ? IMG_TRUE : IMG_FALSE);
+}
+
+#if defined(LMA)
+/*
+	Simple unit size allocator
+*/
+static
+IMG_UINT64 _DCExampleAllocLMABuffer(DCEX_DEVICE *psDevice)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 pvRet = 0;
+
+	for (i = 0; i < psDevice->ui32BufferCount; i++)
+	{
+		if ((psDevice->ui32BufferUseMask & (1UL << i)) == 0)
+		{
+			pvRet = psDevice->sDispStartAddr.uiAddr +
+					(i * psDevice->ui32BufferSize);
+			psDevice->ui32BufferUseMask |= (1UL << i);
+			break;
+		}
+	}
+
+	return pvRet;
+}
+
+static
+void _DCExampleFreeLMABuffer(DCEX_DEVICE *psDevice, IMG_UINT64 uiAddr)
+{
+	IMG_UINT64 ui64Offset;
+
+	DC_ASSERT(uiAddr >= psDevice->sDispStartAddr.uiAddr);
+
+	DCEX_DEBUG_PRINT("Freeing %p, Size %x, UseMask %x", (void *)uiAddr,
+	                  psDevice->ui32BufferSize, psDevice->ui32BufferUseMask);
+
+	ui64Offset = uiAddr - psDevice->sDispStartAddr.uiAddr;
+	ui64Offset = DC_OSDiv64(ui64Offset, psDevice->ui32BufferSize);
+	DC_ASSERT(ui64Offset <= psDevice->ui32BufferCount);
+	psDevice->ui32BufferUseMask &= ~(1UL << ui64Offset);
+}
+#endif /* LMA */
+
+static
+void DCExampleGetInfo(IMG_HANDLE hDeviceData,
+						  DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/*
+		Copy our device name
+	*/
+	DC_OSStringNCopy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	/*
+		Report what our minimum and maximum display period is.
+	*/
+	psDisplayInfo->ui32MinDisplayPeriod	= 0;
+	psDisplayInfo->ui32MaxDisplayPeriod	= 1;
+	psDisplayInfo->ui32MaxPipes			= MAX_PIPES;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static
+PVRSRV_ERROR DCExamplePanelQueryCount(IMG_HANDLE hDeviceData,
+										 IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+	/*
+		If you know the panel count at compile time just hardcode it, if it's
+		dynamic you should probe it here
+	*/
+	*pui32NumPanels = 1;
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExamplePanelQuery(IMG_HANDLE hDeviceData,
+									IMG_UINT32 ui32PanelsArraySize,
+									IMG_UINT32 *pui32NumPanels,
+									PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/*
+		If we have hotplug displays then there is a chance a display could
+		have been removed so return the number of panels we have queried
+	*/
+	*pui32NumPanels = 1;
+
+	/*
+		Either hard code the values here or probe each panel here. If a new
+		panel has been hotplugged then ignore it as we've not been given
+		room to store its data
+	*/
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = ePixelFormat;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width = psModuleParams->ui32Width;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height = psModuleParams->ui32Height;
+
+	psPanelInfo[0].ui32RefreshRate = psModuleParams->ui32RefreshRate;
+	psPanelInfo[0].ui32XDpi = psModuleParams->ui32XDpi;
+	psPanelInfo[0].ui32YDpi = psModuleParams->ui32YDpi;
+
+	switch (psModuleParams->ui32MemLayout)
+	{
+		case 0:
+			psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+			break;
+		case 1:
+			psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_FBC;
+			break;
+		default:
+			DC_OSDebugPrintf(DBGLVL_ERROR, ": Unknown memory layout %ld\n", psModuleParams->ui32MemLayout);
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (psModuleParams->ui32FBCFormat)
+	{
+		case 0:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+			break;
+		case 1:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_8x8;
+			break;
+		case 2:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_16x4;
+			break;
+		case 5:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_32x2;
+			break;
+		case 6:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8;
+			break;
+		case 7:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4;
+			break;
+		case 8:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2;
+			break;
+		case 9:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8;
+			break;
+		case 10:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4;
+			break;
+		case 11:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2;
+			break;
+		case 12:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8;
+			break;
+		case 13:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4;
+			break;
+		case 14:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2;
+			break;
+		case 15:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8;
+			break;
+		case 16:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4;
+			break;
+		case 17:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2;
+			break;
+		default:
+			DC_OSDebugPrintf(DBGLVL_ERROR, ": Unknown FBC format %ld\n", psModuleParams->ui32FBCFormat);
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DC_ASSERT((psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout == PVRSRV_SURFACE_MEMLAYOUT_FBC &&
+		   psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode != IMG_FB_COMPRESSION_NONE) ||
+		  (psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout != PVRSRV_SURFACE_MEMLAYOUT_FBC &&
+		   psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode == IMG_FB_COMPRESSION_NONE));
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExampleFormatQuery(IMG_HANDLE hDeviceData,
+									IMG_UINT32 ui32NumFormats,
+									PVRSRV_SURFACE_FORMAT *pasFormat,
+									IMG_UINT32 *pui32Supported)
+{
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		/*
+			If the display controller has multiple display pipes (DMA engines)
+			each one should be checked to see if it supports the specified
+			format.
+		*/
+		if ((pasFormat[i].ePixFormat == IMG_PIXFMT_B10G10R10A2_UNORM) ||
+		    (pasFormat[i].ePixFormat == IMG_PIXFMT_B8G8R8A8_UNORM) ||
+		    (pasFormat[i].ePixFormat == IMG_PIXFMT_B5G6R5_UNORM))
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExampleDimQuery(IMG_HANDLE hDeviceData,
+								 IMG_UINT32 ui32NumDims,
+								 PVRSRV_SURFACE_DIMS *psDim,
+								 IMG_UINT32 *pui32Supported)
+{
+	IMG_UINT32 i;
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		/*
+			If the display controller has multiple display pipes (DMA engines)
+			each one should be checked to see if it supports the specified
+			dimension.
+		*/
+		if ((psDim[i].ui32Width == psModuleParams->ui32Width)
+		&&  (psDim[i].ui32Height == psModuleParams->ui32Height))
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExampleBufferSystemAcquire(IMG_HANDLE hDeviceData,
+											IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+											IMG_UINT32 *pui32PageCount,
+											IMG_UINT32 *pui32ByteStride,
+											IMG_HANDLE *phSystemBuffer)
+{
+	DCEX_BUFFER *psBuffer;
+	DCEX_DEVICE *psDevice = (DCEX_DEVICE *)hDeviceData;
+	DLLIST_NODE *psNode, *psNext;
+
+	/*
+		This function is optional. It provides a method for services
+		to acquire a display buffer which it didn't setup but was created
+		by the OS (e.g. Linux frame buffer).
+		If the OS should trigger a mode change then it's not allowed to free
+		the previous buffer until services has released it via BufferSystemRelease
+	*/
+
+	/*
+		Protect the access to the system buffer list
+	 */
+	DC_OSMutexLock(psDevice->hBufListLock);
+
+	/*
+		Take a reference to the system buffer 1st to make sure it isn't freed
+	*/
+	dllist_foreach_node(&psDevice->sBufListNode, psNode, psNext)
+	{
+		psBuffer = IMG_CONTAINER_OF(psNode, DCEX_BUFFER, sListNode);
+
+		if ((psBuffer->psDevice == psDevice) && psBuffer->bIsSysBuffer)
+		{
+
+			DCEX_DEBUG_PRINT("%s: psBuffer %p Ref Count %d\n", __func__,
+			                 psBuffer, DC_OSAtomicRead(&psBuffer->i32RefCount));
+
+			DC_OSAtomicIncrement(&psBuffer->i32RefCount);
+
+			*puiLog2PageSize = DC_OSGetPageShift();
+			*pui32PageCount = psBuffer->ui32Size >> DC_OSGetPageShift();
+			*pui32ByteStride = psBuffer->ui32ByteStride;
+			*phSystemBuffer = psBuffer;
+
+			DC_OSMutexUnlock(psDevice->hBufListLock);
+			return PVRSRV_OK;
+		}
+	}
+
+	DC_OSMutexUnlock(psDevice->hBufListLock);
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+static
+void DCExampleBufferSystemRelease(IMG_HANDLE hSystemBuffer)
+{
+	DCEX_BUFFER *psBuffer = hSystemBuffer;
+
+	/*
+		Acquire the buffer mutex to prevent alteration while we are
+		releasing the system buffer.
+	*/
+	DC_OSMutexLock(psBuffer->psDevice->hBufListLock);
+
+	if (!psBuffer->bIsSysBuffer)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, "%s: Unexpected non system-buffer found\n",
+		                __func__);
+		DC_OSMutexUnlock(psBuffer->psDevice->hBufListLock);
+		return;
+	}
+
+	DC_OSAtomicDecrement(&psBuffer->i32RefCount);
+
+	/*
+		If the system buffer has changed and we've just dropped the last
+		refcount then free the buffer
+	*/
+	if (!dllist_node_is_in_list(&psBuffer->sListNode) && (DC_OSAtomicRead(&psBuffer->i32RefCount) == 0))
+	{
+		/* Free the buffer and it's memory (if the memory was allocated) */
+		DCEX_DEBUG_PRINT("%s: Dropping %p", __func__, psBuffer);
+	}
+
+	/*
+		Release buffer mutex now that we've released it.
+	*/
+	DC_OSMutexUnlock(psBuffer->psDevice->hBufListLock);
+}
+
+static
+PVRSRV_ERROR DCExampleContextCreate(IMG_HANDLE hDeviceData,
+									  IMG_HANDLE *hDisplayContext)
+{
+	DCEX_DEVICE *psDevice = hDeviceData;
+
+	if (psDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/*
+		The creation of a display context is a software concept and
+		it's an "agreement" between the services client and the DC driver
+		as to what this means (if anything)
+	*/
+	if (psDevice->b_DisplayContextActive != IMG_TRUE)
+	{
+		*hDisplayContext = hDeviceData;
+
+		psDevice->b_DisplayContextActive = IMG_TRUE;
+		DCEX_DEBUG_PRINT("Create context (%p)\n", *hDisplayContext);
+		return PVRSRV_OK;
+	}
+
+	return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+}
+
+static
+PVRSRV_ERROR DCExampleContextConfigureCheck(IMG_HANDLE hDisplayContext,
+											  IMG_UINT32 ui32PipeCount,
+											  PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+											  IMG_HANDLE *ahBuffers)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+
+	PVR_UNREFERENCED_PARAMETER(hDisplayContext);
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/*
+		The display engine might have a limit on the number of discretely
+		configurable pipes. In such a case an error should be returned.
+	*/
+	if (ui32PipeCount > MAX_PIPES)
+	{
+		eError = PVRSRV_ERROR_DC_TOO_MANY_PIPES;
+		goto fail_max_pipes;
+	}
+
+	/*
+		This optional function allows the display driver to check if the display
+		configuration passed in is valid.
+		It's possible that due to HW constraints that although the client has
+		honoured the DimQuery and FormatQuery results the configuration it
+		has requested is still not possible (e.g. there isn't enough space in
+		the display controller's MMU, or due restrictions on display pipes.
+	*/
+
+	for (i = 0; i < ui32PipeCount; i++)
+	{
+		DCEX_BUFFER *psBuffer = ahBuffers[i];
+
+		if (pasSurfAttrib[i].sCrop.sDims.ui32Width  != psModuleParams->ui32Width  ||
+		    pasSurfAttrib[i].sCrop.sDims.ui32Height != psModuleParams->ui32Height ||
+		    pasSurfAttrib[i].sCrop.i32XOffset != 0 ||
+		    pasSurfAttrib[i].sCrop.i32YOffset != 0)
+		{
+			eError = PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+			break;
+		}
+
+		if (pasSurfAttrib[i].sDisplay.sDims.ui32Width !=
+			pasSurfAttrib[i].sCrop.sDims.ui32Width ||
+			pasSurfAttrib[i].sDisplay.sDims.ui32Height !=
+			pasSurfAttrib[i].sCrop.sDims.ui32Height ||
+			pasSurfAttrib[i].sDisplay.i32XOffset !=
+			pasSurfAttrib[i].sCrop.i32XOffset ||
+			pasSurfAttrib[i].sDisplay.i32YOffset !=
+			pasSurfAttrib[i].sCrop.i32YOffset)
+		{
+			eError = PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+			break;
+		}
+
+		if (psBuffer->ui32Width != psModuleParams->ui32Width
+		||  psBuffer->ui32Height != psModuleParams->ui32Height)
+		{
+			eError = PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+			break;
+		}
+	}
+
+fail_max_pipes:
+	return eError;
+}
+
+static
+void DCExampleContextConfigure(IMG_HANDLE hDisplayContext,
+									 IMG_UINT32 ui32PipeCount,
+									 PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+									 IMG_HANDLE *ahBuffers,
+									 IMG_UINT32 ui32DisplayPeriod,
+									 IMG_HANDLE hConfigData)
+{
+	DCEX_DEVICE *psDeviceData = hDisplayContext;
+	IMG_UINT32 i;
+
+	/*
+		As we have no HW and thus no VSync IRQ we just activate the
+		new config here
+	*/
+	for (i = 0; i < ui32PipeCount; i++)
+	{
+#if defined(DCEX_DEBUG)
+		DCEX_BUFFER *psBuffer = ahBuffers[i];
+		/*
+			There is no checking to be done here as we can't fail,
+			any checking should have been DCExampleContextConfigureCheck.
+		*/
+
+		/*
+			If the display controller supports scaling it should set it up
+			here.
+		*/
+
+		/*
+			Setup the DMA from the display buffer
+		*/
+
+		/*
+			Save the config data as we need to pass it back once this
+			configuration gets retired
+		*/
+		DCEX_DEBUG_PRINT("Display buffer (%p), Ref Count %d\n", psBuffer,
+		                 DC_OSAtomicRead(&psBuffer->i32RefCount));
+#endif /* DCEX_DEBUG */
+	}
+
+	/*
+		As we have no HW and thus no VSync IRQ we just retire the
+		previous config as soon as we get a new one
+	*/
+	if (DCExampleConfigIsEmpty(psDeviceData) == IMG_FALSE)
+	{
+		/* Retire the current config */
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(DCExampleConfigPop(psDeviceData));
+	}
+
+	if (ui32PipeCount != 0)
+	{
+		/* Save our new config data */
+		DCExampleConfigPush(psDeviceData, hConfigData);
+	}
+	else
+	{
+		/*
+			When the client requests the display context to be destroyed
+			services will issue a "NULL" flip to us so we can retire
+			the current configuration.
+
+			We need to pop the current (and last) configuration off the our
+			stack and retire it which we're already done above as it's our
+			default behaviour to retire the previous config immediately when we
+			get a new one. We also need to "retire" the NULL flip so the DC core
+			knows that we've done whatever we need to do so it can start to
+			tear down the display context
+
+			In real devices we could have a number of configurations in flight,
+			all these configurations _and the NULL flip_ need to be processed
+			in order with the NULL flip being completed when it's active rather
+			then when it will be retired (as it will never be retired)
+
+			At this point there is nothing that the display is being asked to
+			display by services and it's the DC driver implementation decision
+			as to what it should then do. Typically, for systems that have a
+			system surface the DC would switch back to displaying that.
+		*/
+		DCEX_DEBUG_PRINT("Display flushed (%p)\n", hDisplayContext);
+
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(hConfigData);
+	}
+}
+
+static
+void DCExampleContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DCEX_DEVICE *psDevice = hDisplayContext;
+
+	DC_ASSERT(DCExampleConfigIsEmpty(psDevice) == IMG_TRUE);
+
+	/*
+		Counterpart to ContextCreate. Any buffers created/imported on
+		this display context will have been freed before this call so
+		all the display driver needs to do is release any resources
+		allocated at ContextCreate time.
+	*/
+	psDevice->b_DisplayContextActive = IMG_FALSE;
+	DCEX_DEBUG_PRINT("Destroy display context (%p)\n", hDisplayContext);
+}
+
+static
+PVRSRV_ERROR DCExampleBufferAlloc(IMG_HANDLE hDisplayContext,
+									DC_BUFFER_CREATE_INFO *psCreateInfo,
+									IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+									IMG_UINT32 *pui32PageCount,
+									IMG_UINT32 *pui32ByteStride,
+									IMG_HANDLE *phBuffer)
+{
+	DCEX_BUFFER *psBuffer;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	PVRSRV_ERROR eError;
+	DCEX_DEVICE *psDevice = hDisplayContext;
+
+	/*
+		Allocate the buffer control structure
+	*/
+
+	psBuffer = DC_OSCallocMem(sizeof(DCEX_BUFFER));
+
+	if (psBuffer == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_bufferalloc;
+	}
+
+	dllist_init(&psBuffer->sListNode);
+
+	/* Common setup */
+	psBuffer->eSource = DCEX_BUFFER_SOURCE_ALLOC;
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 1);
+	psBuffer->ePixFormat = psSurfInfo->sFormat.ePixFormat;
+	psBuffer->bIsSysBuffer = IMG_FALSE;
+
+	/*
+		Store the display buffer size, that is the pixel width and height that
+		will be displayed which might be less than the pixel width and height
+		of the buffer
+	*/
+	psBuffer->ui32Width = psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height = psSurfInfo->sDims.ui32Height;
+
+	switch (psSurfInfo->sFormat.eMemLayout)
+	{
+		case PVRSRV_SURFACE_MEMLAYOUT_STRIDED:
+			/*
+				As we're been asked to allocate this buffer we decide what it's
+				stride should be.
+			*/
+			psBuffer->ui32ByteStride = psSurfInfo->sDims.ui32Width * psCreateInfo->ui32BPP;
+			psBuffer->ui32Size = psBuffer->ui32Height * psBuffer->ui32ByteStride;
+			break;
+		case PVRSRV_SURFACE_MEMLAYOUT_FBC:
+			psBuffer->ui32ByteStride = psCreateInfo->sFBC.ui32FBCStride * psCreateInfo->ui32BPP;
+			psBuffer->ui32Size = psCreateInfo->sFBC.ui32Size;
+
+			/*
+				Here we should program the FBC registers in the display
+				controller according to the information we have in
+				psSurfInfo->sFBC
+			*/
+			break;
+
+		default:
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+			goto fail_memlayout;
+			break;
+	}
+
+	psBuffer->psDevice = psDevice;
+
+#if defined(LMA)
+	/*
+		Check that the buffer size calculated is compatible with what was allocated during initialisation.
+	*/
+	if (psBuffer->ui32Size > psDevice->ui32BufferSize)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, "%s: Buffer size of 0x%0x is too large for allocation of 0x%0x!\n",
+		                 __func__, psBuffer->ui32Size, psDevice->ui32BufferSize);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_buffersize;
+	}
+	else if (psBuffer->ui32Size != psDevice->ui32BufferSize)
+	{
+		DCEX_DEBUG_PRINT("%s: Buffer size of 0x%0x is not same as allocated 0x%0x\n",
+		                 __func__, psBuffer->ui32Size, psDevice->ui32BufferSize);
+		psBuffer->ui32Size = psDevice->ui32BufferSize;
+	}
+
+	/*
+		Allocate display addressable memory. We only need physical addresses
+		at this stage.
+
+		Note: This could be deferred until the 1st map or acquire call.
+	*/
+	psBuffer->uiAllocHandle = _DCExampleAllocLMABuffer(psDevice);
+	if (psBuffer->uiAllocHandle == 0)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_buffermemalloc;
+	}
+#else
+	psBuffer->pvAllocHandle = DCExampleVirtualAllocUncached(psBuffer->ui32Size);
+	if (psBuffer->pvAllocHandle == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_buffermemalloc;
+	}
+#endif
+	*pui32ByteStride = psBuffer->ui32ByteStride;
+	*puiLog2PageSize = DC_OSGetPageShift();
+	*pui32PageCount = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+	*phBuffer = psBuffer;
+
+	DCEX_DEBUG_PRINT("Allocate buffer (%p)\n", psBuffer);
+
+	DC_OSMutexLock(psDevice->hBufListLock);
+	dllist_add_to_tail(&psDevice->sBufListNode, &psBuffer->sListNode);
+	DC_OSMutexUnlock(psDevice->hBufListLock);
+
+	return PVRSRV_OK;
+
+#if defined(LMA)
+fail_buffersize:
+#endif
+fail_memlayout:
+fail_buffermemalloc:
+	DC_OSFreeMem(psBuffer);
+
+fail_bufferalloc:
+	return eError;
+}
+
+#if !defined(LMA)
+static
+PVRSRV_ERROR DCExampleBufferImport(IMG_HANDLE hDisplayContext,
+									 IMG_UINT32 ui32NumPlanes,
+									 IMG_HANDLE **paphImport,
+									 DC_BUFFER_IMPORT_INFO *psSurfAttrib,
+									 IMG_HANDLE *phBuffer)
+{
+	/*
+		This it optional and should only be provided if the display controller
+		can access "general" memory (e.g. the memory doesn't have to contiguous)
+	*/
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+	DCEX_BUFFER *psBuffer;
+	DCEX_DEVICE *psDevice = hDisplayContext;
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/*
+		Check to see if our display hardware supports this buffer
+	*/
+	if ((psSurfAttrib->ePixFormat != IMG_PIXFMT_B8G8R8A8_UNORM) ||
+		(psSurfAttrib->ui32Width[0] != psModuleParams->ui32Width) ||
+		(psSurfAttrib->ui32Height[0] != psModuleParams->ui32Height) ||
+		(psSurfAttrib->ui32ByteStride[0] != ui32ByteStride))
+	{
+		return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	psBuffer = DC_OSCallocMem(sizeof(DCEX_BUFFER));
+
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psBuffer->eSource = DCEX_BUFFER_SOURCE_IMPORT;
+	psBuffer->ui32Width = psSurfAttrib->ui32Width[0];
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 1);
+	psBuffer->ePixFormat = psSurfAttrib->ePixFormat;
+	psBuffer->ui32ByteStride = psSurfAttrib->ui32ByteStride[0];
+	psBuffer->ui32Width = psSurfAttrib->ui32Width[0];
+	psBuffer->ui32Height = psSurfAttrib->ui32Height[0];
+	psBuffer->psDevice = psDevice;
+
+	/*
+		If the display controller supports mapping "general" memory, but has
+		limitations (e.g. if it doesn't have full range addressing) these
+		should be checked here by calling DCImportBufferAcquire. In this case
+		it lock down the physical address of the buffer at this strange rather
+		then being able to defer it to map time.
+	*/
+	psBuffer->hImport = paphImport[0];
+
+	*phBuffer = psBuffer;
+	DCEX_DEBUG_PRINT("Import buffer (%p)\n", psBuffer);
+	return PVRSRV_OK;
+}
+#endif
+
+static
+PVRSRV_ERROR DCExampleBufferAcquire(IMG_HANDLE hBuffer,
+									IMG_DEV_PHYADDR *pasDevPAddr,
+									void **ppvLinAddr)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+	PVRSRV_ERROR eError;
+#if defined(LMA)
+	IMG_UINT32 i;
+	unsigned long ulPages = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+	PHYS_HEAP *psPhysHeap = psBuffer->psDevice->psPhysHeap;
+	IMG_CPU_PHYADDR sCpuPAddr;
+#endif
+	/*
+		If we didn't allocate the display memory at buffer alloc time
+		we would have to do it here.
+	*/
+
+	/*
+		Fill in the array of addresses we were passed
+	*/
+#if defined(LMA)
+	sCpuPAddr.uiAddr = psBuffer->uiAllocHandle;
+	DCEX_DEBUG_PRINT("Acquire buffer (%p) memory, npages = %d, Start = %p, size = 0x%x\n",
+	                 psBuffer, ulPages, sCpuPAddr.uiAddr, psBuffer->ui32Size);
+	for (i = 0; i < ulPages; i++)
+	{
+		psBuffer->psDevice->sPVRServicesFuncs.pfnPhysHeapCpuPAddrToDevPAddr(
+				psPhysHeap,
+				1,
+				&pasDevPAddr[i],
+				&sCpuPAddr);
+
+		/* Only display first and last mappings */
+		if ((i == 0) || i == (ulPages - 1))
+		{
+			DCEX_DEBUG_PRINT("Acquire buffer (%p) page %d, DevPAddr %p, CPUPAddr %p\n",
+			                 psBuffer, i, pasDevPAddr[i].uiAddr, sCpuPAddr.uiAddr);
+		}
+		sCpuPAddr.uiAddr += DC_OSGetPageSize();
+	}
+	*ppvLinAddr = NULL;
+	eError      = PVRSRV_OK;
+#else
+	eError = DCExampleGetDevPAddrs(psBuffer->pvAllocHandle, pasDevPAddr, 0, psBuffer->ui32Size);
+	if (eError == PVRSRV_OK)
+	{
+		eError = DCExampleGetLinAddr(psBuffer->pvAllocHandle, ppvLinAddr);
+	}
+#endif
+
+	DCEX_DEBUG_PRINT("Acquire buffer (%p) memory\n", psBuffer);
+	return eError;
+}
+
+static
+void DCExampleBufferRelease(IMG_HANDLE hBuffer)
+{
+#if defined(DCEX_DEBUG)
+	DCEX_BUFFER *psBuffer = hBuffer;
+#endif
+	/*
+		We could release the display memory here (assuming it wasn't
+		still mapped into the display controller).
+
+		As the buffer hasn't been freed the contents must be preserved, i.e.
+		in the next call to Acquire different physical pages can be returned,
+		but they must have the same contents as the old pages had at Release
+		time.
+	*/
+	DCEX_DEBUG_PRINT("Release buffer (%p) memory, Ref Count = %d\n",
+	                 psBuffer, DC_OSAtomicRead(&psBuffer->i32RefCount));
+}
+
+static
+void DCExampleBufferFree(IMG_HANDLE hBuffer)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+
+	DCEX_DEBUG_PRINT("Free buffer (%p)\n", psBuffer);
+	if (psBuffer->eSource == DCEX_BUFFER_SOURCE_ALLOC)
+	{
+#if defined(LMA)
+		DCEX_DEBUG_PRINT("Free buffer memory (%llu/[0x%llx])\n",
+		                 psBuffer->uiAllocHandle, psBuffer->uiAllocHandle);
+		_DCExampleFreeLMABuffer(psBuffer->psDevice, psBuffer->uiAllocHandle);
+#else
+		DCEX_DEBUG_PRINT("Free buffer memory (%p)\n", psBuffer->pvAllocHandle);
+		DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+	}
+
+	DC_OSMutexLock(psBuffer->psDevice->hBufListLock);
+	dllist_remove_node(&psBuffer->sListNode);
+	DC_OSMutexUnlock(psBuffer->psDevice->hBufListLock);
+	DC_OSFreeMem(psBuffer);
+}
+
+static
+PVRSRV_ERROR DCExampleBufferMap(IMG_HANDLE hBuffer)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+	IMG_UINT32 ui32PageCount;
+	PVRSRV_ERROR eError;
+#if defined(DCEX_VERBOSE)
+	IMG_UINT32 i;
+#endif
+	/*
+		If the display controller needs memory to be mapped into it
+		(e.g. it has an MMU) and didn't do it in the alloc and import then it
+		should provide this function.
+	*/
+
+	if (psBuffer->hImport)
+	{
+		IMG_DEV_PHYADDR *pasDevPAddr;
+		/*
+			In the case of an import buffer we didn't allocate the buffer and
+			so need to ask for it's pages
+		*/
+		eError = psBuffer->psDevice->sPVRServicesFuncs.pfnDCImportBufferAcquire(
+					psBuffer->hImport,
+					DC_OSGetPageShift(),
+					&ui32PageCount,
+					&pasDevPAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_import;
+		}
+#if defined(DCEX_VERBOSE)
+		for (i = 0; i < ui32PageCount; i++)
+		{
+			DCEX_DEBUG_PRINT(": DCExampleBufferMap: DCExample map address 0x%016llx\n", pasDevPAddr[i].uiAddr);
+		}
+#endif
+		psBuffer->pasDevPAddr = pasDevPAddr;
+		psBuffer->ui32PageCount = ui32PageCount;
+	}
+#if defined(DCEX_VERBOSE)
+	else
+	{
+		unsigned long ulPages = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+		IMG_DEV_PHYADDR sDevPAddr;
+
+		for (i = 0; i < ulPages; i++)
+		{
+			eError = DCExampleGetDevPAddrs(psBuffer->pvAllocHandle, &sDevPAddr, i, DC_OSGetPageSize());
+			if (PVRSRV_OK == eError)
+			{
+				DC_OSDebugPrintf(DBGLVL_INFO, ": DCExampleBufferMap: DCExample map address 0x%016llx\n", sDevPAddr.uiAddr);
+			}
+		}
+	}
+#endif
+	DCEX_DEBUG_PRINT("Map buffer (%p) into display\n", psBuffer);
+	return PVRSRV_OK;
+
+fail_import:
+	return eError;
+}
+
+static
+void DCExampleBufferUnmap(IMG_HANDLE hBuffer)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+#if defined(DCEX_VERBOSE)
+	IMG_UINT32 i;
+#endif
+	/*
+		If the display controller provided buffer map then it must provide
+		this function
+	*/
+
+	/*
+		Unmap the memory from the display controller's MMU
+	*/
+	if (psBuffer->hImport)
+	{
+#if defined(DCEX_VERBOSE)
+		for (i = 0; i < psBuffer->ui32PageCount; i++)
+		{
+			DC_OSDebugPrintf(DBGLVL_INFO, ": DCExampleBufferUnmap: DCExample unmap address 0x%016llx\n", psBuffer->pasDevPAddr[i].uiAddr);
+		}
+#endif
+		/*
+			As this was an imported buffer we need to release it
+		*/
+		psBuffer->psDevice->sPVRServicesFuncs.pfnDCImportBufferRelease(
+				psBuffer->hImport,
+				psBuffer->pasDevPAddr);
+	}
+#if defined(DCEX_VERBOSE)
+	else
+	{
+		unsigned long ulPages = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+		IMG_DEV_PHYADDR sDevPAddr;
+
+		for (i = 0; i < ulPages; i++)
+		{
+			PVRSRV_ERROR eError = DCExampleGetDevPAddrs(psBuffer->pvAllocHandle, &sDevPAddr, i, DC_OSGetPageSize());
+			if (PVRSRV_OK == eError)
+			{
+				DC_OSDebugPrintf(DBGLVL_INFO, ": DCExampleBufferUnmap: DCExample unmap address 0x%016llx\n", sDevPAddr.uiAddr);
+			}
+		}
+	}
+#endif
+	DCEX_DEBUG_PRINT("Unmap buffer (%p) from display\n", psBuffer);
+}
+
+/*
+	In this example driver we provide (almost) the full range of functions
+*/
+
+static DC_DEVICE_FUNCTIONS sDCFunctions;
+
+/*
+    Release all driver data allocated during DCExampleInit()
+*/
+static void DCExampleReleaseDriverData(const IMG_BOOL bFromDeInit)
+{
+	DLLIST_NODE *psNode, *psNext;
+	DCEX_DEVICE *psDeviceData;
+	DCEX_BUFFER *psBuffer;
+
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		DLLIST_NODE *psBufNode, *psBufNext;
+
+		psDeviceData = IMG_CONTAINER_OF(psNode, DCEX_DEVICE, sListNode);
+
+		if (bFromDeInit)
+		{
+			/* Unregister the device before freeing */
+			psDeviceData->sPVRServicesFuncs.pfnDCUnregisterDevice(psDeviceData->hSrvHandle);
+		}
+
+		/* Release all Heaps */
+#if defined(LMA)
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+#endif
+
+		/* Close all connection handles */
+		DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+
+		/* Release all associated buffers for this device */
+		DC_OSMutexLock(psDeviceData->hBufListLock);
+		dllist_foreach_node(&psDeviceData->sBufListNode, psBufNode, psBufNext)
+		{
+			psBuffer = IMG_CONTAINER_OF(psBufNode, DCEX_BUFFER, sListNode);
+
+			/* Free all LMA / Virtual Buffer allocations */
+#if defined(LMA)
+			_DCExampleFreeLMABuffer(psBuffer->psDevice, psBuffer->uiAllocHandle);
+#else
+			DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+			dllist_remove_node(psBufNode);
+
+			DC_OSFreeMem(psBuffer);
+		}
+		DC_OSMutexUnlock(psDeviceData->hBufListLock);
+
+		/* Release all OS memory */
+		DC_OSMutexLock(g_hDevDataListLock);
+
+		dllist_remove_node(psNode);
+
+		DC_OSMutexUnlock(g_hDevDataListLock);
+
+		DC_ASSERT(dllist_is_empty(&psDeviceData->sBufListNode));
+
+		DC_OSMutexDestroy(psDeviceData->hBufListLock);
+		DC_OSFreeMem(psDeviceData);
+	}
+
+	DC_ASSERT(dllist_is_empty(&g_sDeviceDataListHead));
+}
+
+#if defined(INTEGRITY_OS)
+static PVRSRV_ERROR DCExampleAcquireKernelMappingData(IMG_HANDLE hBuffer, IMG_HANDLE *phMapping, void **ppPhysAddr)
+{
+	DCEX_BUFFER *psBuffer = (DCEX_BUFFER *)hBuffer;
+	if (!psBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return DCExampleOSAcquireKernelMappingData(psBuffer->pvAllocHandle, phMapping, ppPhysAddr);
+}
+#endif
+
+/*
+	functions exported by kernel services for use by 3rd party kernel display
+	class device driver
+*/
+PVRSRV_ERROR DCExampleInit(IMG_UINT32 *puiNumDevices)
+{
+	DCEX_BUFFER *psBuffer;
+	DCEX_DEVICE *psDeviceData;
+	PVRSRV_ERROR eError;
+#if defined(LMA)
+	PPVRSRV_DEVICE_NODE psDevNode;
+	IMG_UINT64 ui64BufferCount;
+#endif
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams;
+	IMG_UINT32 uiCurDev;
+	IMG_UINT32 uiNumDevicesConfigured;
+
+	DC_OSSetDrvName(DRVNAME);
+	dllist_init(&g_sDeviceDataListHead);
+
+	eError = DC_OSMutexCreate(&g_hDevDataListLock);
+	if (PVRSRV_OK != eError)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create g_hDevDataListLock (%d)\n", __func__, eError);
+		goto fail_nolocks;
+	}
+
+	/* Check the module params and setup global buffer size state */
+	if (!CheckBufferDimensions())
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psModuleParams = DCExampleGetModuleParameters();
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/* Get the number of physically registered devices in the system. */
+	uiNumDevicesConfigured = psModuleParams->ui32NumDevices;
+
+	DCEX_DEBUG_PRINT("%s: Num Registered = %u\n", __func__, uiNumDevicesConfigured);
+
+	for (uiCurDev = 0; uiCurDev < uiNumDevicesConfigured; uiCurDev++)
+	{
+
+		/*
+			If the display controller hasn't already been initialised elsewhere
+			in the system then it should be initialised here.
+
+			Create the private data structure (psDeviceData) and store all the
+			device specific data we will need later (e.g. pointer to mapped
+			registered)
+			This device specific private data will be passed into our callbacks
+			so we don't need global data and can have more than one display
+			controller driven by the same driver (we would just create an
+			"instance" of a device
+			by registering the same callbacks with different private data)
+		*/
+
+		DCEX_DEBUG_PRINT("%s: Initialising device %u\n", __func__, uiCurDev);
+
+		psDeviceData = DC_OSCallocMem(sizeof(DCEX_DEVICE));
+		if (psDeviceData == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_devicealloc;
+		}
+
+		dllist_init(&psDeviceData->sListNode);
+		dllist_init(&psDeviceData->sBufListNode);
+
+		eError = DC_OSMutexCreate(&psDeviceData->hBufListLock);
+		if (PVRSRV_OK != eError)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create hBufListLock (%d)\n",  __func__, eError);
+			goto fail_servicesconnectionclose;
+		}
+
+		eError = DC_OSPVRServicesConnectionOpen(&psDeviceData->hPVRServicesConnection);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to open connection to PVR Services (%d)\n", __func__, eError);
+			goto fail_servicesconnectionclose;
+		}
+
+		eError = DC_OSPVRServicesSetupFuncs(psDeviceData->hPVRServicesConnection, &psDeviceData->sPVRServicesFuncs);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to setup PVR Services function table (%d)\n", __func__, eError);
+			goto fail_servicesconnectionclose;
+		}
+
+#if defined(LMA)
+		psDevNode = psDeviceData->sPVRServicesFuncs.pfnGetDeviceInstance(uiCurDev);
+		PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, fail_devicealloc);
+		/*
+			If the display is using card memory then we need to know
+			where that memory is so we have to acquire the heap we want
+			to use (a carveout of the card memory) so we can get it's address
+		*/
+		eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapAcquireByUsage(
+					PHYS_HEAP_USAGE_DISPLAY,
+					psDevNode,
+					&psDeviceData->psPhysHeap);
+
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_heapacquire;
+		}
+
+		/* Verify we're operating on a LMA heap */
+		DC_ASSERT(psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetType(psDeviceData->psPhysHeap) == PHYS_HEAP_TYPE_LMA);
+
+		eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetCpuPAddr(
+					psDeviceData->psPhysHeap,
+					&psDeviceData->sDispStartAddr);
+
+		DC_ASSERT(eError == PVRSRV_OK);
+
+		eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetSize(
+					psDeviceData->psPhysHeap,
+					&psDeviceData->uiDispMemSize);
+
+		DC_ASSERT(eError == PVRSRV_OK);
+#endif
+		/*
+			If the display driver has a system surface create the buffer structure
+			that describes it here.
+
+			Note:
+			All this data and the buffer should be queried from the OS, but in this
+			example we don't have the ability to fetch this data from an OS driver,
+			so we create the data.
+		*/
+		psBuffer = DC_OSCallocMem(sizeof(DCEX_BUFFER));
+
+		if (psBuffer == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_bufferalloc;
+		}
+
+		dllist_init(&psBuffer->sListNode);
+
+		DC_OSAtomicWrite(&psBuffer->i32RefCount, 1);
+		psBuffer->ePixFormat = ePixelFormat;
+		psBuffer->ui32Width = psModuleParams->ui32Width;
+		psBuffer->ui32Height = psModuleParams->ui32Height;
+		psBuffer->ui32ByteStride = ui32ByteStride;
+		psBuffer->ui32Size = psBuffer->ui32Height * psBuffer->ui32ByteStride;
+		psBuffer->ui32Size += (psBuffer->ui32Size >> 4); /* Speculative extra 6.25% for FBC overhead. */
+		psBuffer->ui32Size = (psBuffer->ui32Size + DC_OSGetPageSize() - 1) & DC_OSGetPageMask();
+		psBuffer->psDevice = psDeviceData;
+		psBuffer->bIsSysBuffer = IMG_TRUE;
+
+#if defined(LMA)
+		/* Simple allocator, assume all buffers are going to be the same size. */
+		ui64BufferCount = psDeviceData->uiDispMemSize;
+		ui64BufferCount = DC_OSDiv64(ui64BufferCount, psBuffer->ui32Size);
+
+		psDeviceData->ui32BufferCount = (IMG_UINT32) ui64BufferCount;
+		DC_ASSERT((IMG_UINT32) ui64BufferCount == psDeviceData->ui32BufferCount);
+
+		if (psDeviceData->ui32BufferCount > 32)
+		{
+			psDeviceData->ui32BufferCount = 32;
+		}
+
+		psDeviceData->ui32BufferSize = psBuffer->ui32Size;
+		psDeviceData->ui32BufferUseMask = 0;
+
+		DC_OSDebugPrintf(DBGLVL_INFO, " Buffers: %u (%u bytes each)\n",
+		                 psDeviceData->ui32BufferCount, psDeviceData->ui32BufferSize);
+
+		psBuffer->uiAllocHandle = _DCExampleAllocLMABuffer(psDeviceData);
+		if (psBuffer->uiAllocHandle == 0)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_buffermemalloc;
+		}
+		DCEX_DEBUG_PRINT("Allocate system buffer handle = %p\n", psBuffer->uiAllocHandle);
+#else
+		psBuffer->pvAllocHandle = DCExampleVirtualAllocUncached(psBuffer->ui32Size);
+		if (psBuffer->pvAllocHandle == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_buffermemalloc;
+		}
+#endif
+		DCEX_DEBUG_PRINT("Allocate system buffer = %p\n", psBuffer);
+
+		/* Initialise DC Function Table */
+		DC_OSMemSet(&sDCFunctions, 0, sizeof(sDCFunctions));
+
+		sDCFunctions.pfnGetInfo					= DCExampleGetInfo;
+		sDCFunctions.pfnPanelQueryCount			= DCExamplePanelQueryCount;
+		sDCFunctions.pfnPanelQuery				= DCExamplePanelQuery;
+		sDCFunctions.pfnFormatQuery				= DCExampleFormatQuery;
+		sDCFunctions.pfnDimQuery				= DCExampleDimQuery;
+		sDCFunctions.pfnSetBlank				= NULL;
+		sDCFunctions.pfnSetVSyncReporting		= NULL;
+		sDCFunctions.pfnLastVSyncQuery			= NULL;
+		sDCFunctions.pfnContextCreate			= DCExampleContextCreate;
+		sDCFunctions.pfnContextDestroy			= DCExampleContextDestroy;
+		sDCFunctions.pfnContextConfigure		= DCExampleContextConfigure;
+		sDCFunctions.pfnContextConfigureCheck	= DCExampleContextConfigureCheck;
+		sDCFunctions.pfnBufferAlloc				= DCExampleBufferAlloc;
+		sDCFunctions.pfnBufferAcquire			= DCExampleBufferAcquire;
+		sDCFunctions.pfnBufferRelease			= DCExampleBufferRelease;
+		sDCFunctions.pfnBufferFree				= DCExampleBufferFree;
+#if !defined(LMA)
+		sDCFunctions.pfnBufferImport			= DCExampleBufferImport;
+#endif
+		sDCFunctions.pfnBufferMap				= DCExampleBufferMap;
+		sDCFunctions.pfnBufferUnmap				= DCExampleBufferUnmap;
+		sDCFunctions.pfnBufferSystemAcquire		= DCExampleBufferSystemAcquire;
+		sDCFunctions.pfnBufferSystemRelease		= DCExampleBufferSystemRelease;
+#if defined(INTEGRITY_OS)
+		sDCFunctions.pfnAcquireKernelMappingData = DCExampleAcquireKernelMappingData;
+#endif
+
+		/*
+			Register our DC driver with services
+		*/
+		eError = psDeviceData->sPVRServicesFuncs.pfnDCRegisterDevice(
+					&sDCFunctions,
+					MAX_COMMANDS_INFLIGHT,
+					psDeviceData,
+					&psDeviceData->hSrvHandle);
+
+		/*
+		    We may fail to register if we've had more instances enabled than
+		    there are physical devices available. If this is not the first
+		    device we should allow access to all of the devices currently
+		    registered.
+		 */
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_WARNING,
+			                 "pfnDCRegisterDevice failed (%d). %u devices registered\n",
+			                  eError, uiCurDev);
+			if (uiCurDev > 0)
+			{
+				/* Free psSystemBuffer and psDeviceData */
+#if defined(LMA)
+				_DCExampleFreeLMABuffer(psBuffer->psDevice,
+				                        psBuffer->uiAllocHandle);
+				psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+#else
+				DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+				DC_OSFreeMem(psBuffer);
+				DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+				DC_OSFreeMem(psDeviceData);
+				/* Allow access to successfully discovered devices */
+				if (puiNumDevices)
+				{
+					*puiNumDevices = uiCurDev;
+				}
+				return PVRSRV_OK;
+			}
+			else
+			{
+				goto fail_register;
+			}
+		}
+
+		/* Save the device data somewhere we can retrieve it */
+		DC_OSMutexLock(g_hDevDataListLock);
+		dllist_add_to_tail(&g_sDeviceDataListHead, &psDeviceData->sListNode);
+		DC_OSMutexUnlock(g_hDevDataListLock);
+
+		/* Store the system buffer on the device hook */
+		DC_OSMutexLock(psDeviceData->hBufListLock);
+		dllist_add_to_tail(&psDeviceData->sBufListNode, &psBuffer->sListNode);
+		DC_OSMutexUnlock(psDeviceData->hBufListLock);
+	}	/* End of loop over # devices to create */
+
+	if (puiNumDevices)
+	{
+		*puiNumDevices = uiNumDevicesConfigured;
+	}
+	return PVRSRV_OK;
+
+fail_register:
+	/*
+	 * Warning: Here we are aborting the initialisation and we may have
+	 * partially allocated some data. On entry psDeviceData refers to the
+	 * most recent 'in-progress' device allocation as does 'psBuffer' refer
+	 * to the in-progress buffer allocation.
+	 * We need to free these first, and then clean up any already established
+	 * contents which will be hanging from the DLLIST_NODEs
+	 */
+	if (!dllist_node_is_in_list(&psBuffer->sListNode))
+	{
+#if defined(LMA)
+		_DCExampleFreeLMABuffer(psBuffer->psDevice, psBuffer->uiAllocHandle);
+#else
+		DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+	}
+
+fail_buffermemalloc:
+	/* Free all buffer allocations */
+	if (!dllist_node_is_in_list(&psBuffer->sListNode))
+	{
+		DC_OSFreeMem(psBuffer);
+	}
+
+fail_bufferalloc:
+	/* Release all Heaps */
+#if defined(LMA)
+	if (!dllist_node_is_in_list(&psDeviceData->sListNode))
+	{
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+	}
+fail_heapacquire:
+#endif
+
+	/* Close all connection handles */
+	if (!dllist_node_is_in_list(&psDeviceData->sListNode))
+	{
+		DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+	}
+
+fail_servicesconnectionclose:
+	/* Release all OS memory */
+	if (!dllist_node_is_in_list(&psDeviceData->sListNode))
+	{
+		DC_OSFreeMem(psDeviceData);
+	}
+
+	/*
+	 * Now we must clean up any previously completed allocations. These will,
+	 * if present, be on the DLLIST_NODE lists for g_sDeviceDataListHead and
+	 * the incorporated sBufListNode entries within.
+	 */
+	DCExampleReleaseDriverData(IMG_FALSE);
+
+fail_devicealloc:
+	DC_OSMutexDestroy(g_hDevDataListLock);
+
+fail_nolocks:
+	return eError;
+}
+
+void DCExampleDeinit(void)
+{
+
+	DCExampleReleaseDriverData(IMG_TRUE);
+
+	/* Free the global list locks. */
+	DC_OSMutexDestroy(g_hDevDataListLock);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..22ad43c75345
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Kbuild.mk
@@ -0,0 +1,57 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-y += \
+ -I$(TOP)/services/display/include \
+ -I$(TOP)/services/display/dc_example/include
+ifeq ($(PVR_ARCH),rogue)
+ccflags-y += \
+ -I$(TOP)/include/public \
+ -I$(TOP)/include/$(PVR_ARCH)/public
+endif
+
+dc_example-y += \
+ services/display/dc_example/common/dc_example.o \
+ services/display/dc_example/env/linux/ex_osfuncs.o \
+ services/display/common/env/linux/dc_osfuncs.o
+
+ifneq ($(W),1)
+CFLAGS_services/display/dc_example/common/dc_example.o := -Werror
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Linux.mk
new file mode 100644
index 000000000000..ad36f6c8f267
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := dc_example
+
+dc_example_type := kernel_module
+dc_example_target := dc_example.ko
+dc_example_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/ex_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/ex_osfuncs.c
new file mode 100644
index 000000000000..fdc77551feee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/ex_osfuncs.c
@@ -0,0 +1,407 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include <linux/version.h>
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#include "dc_osfuncs.h"
+#include "dc_example.h"
+
+#ifndef DC_EXAMPLE_WIDTH
+#define DC_EXAMPLE_WIDTH 640
+#endif
+
+#ifndef DC_EXAMPLE_HEIGHT
+#define DC_EXAMPLE_HEIGHT 480
+#endif
+
+#ifndef DC_EXAMPLE_BIT_DEPTH
+#define DC_EXAMPLE_BIT_DEPTH 32
+#endif
+
+#ifndef DC_EXAMPLE_FORMAT
+#define DC_EXAMPLE_FORMAT 0
+#endif
+
+#ifndef DC_EXAMPLE_MEMORY_LAYOUT
+#define DC_EXAMPLE_MEMORY_LAYOUT 0
+#endif
+
+#ifndef DC_EXAMPLE_FBC_FORMAT
+#define DC_EXAMPLE_FBC_FORMAT 0
+#endif
+
+#ifndef DC_EXAMPLE_REFRESH_RATE
+#define DC_EXAMPLE_REFRESH_RATE 60
+#endif
+
+#ifndef DC_EXAMPLE_DPI
+#define DC_EXAMPLE_DPI 160
+#endif
+
+#ifndef DC_EXAMPLE_NUM_DEVICES
+#define DC_EXAMPLE_NUM_DEVICES 1
+#endif
+
+#define DC_EXAMPLE_DEBUGFS_NUM_DEVICES "num_devices"
+
+static struct dentry	*g_psDebugFSEntryDir;	/* Top-level handle */
+static struct dentry	*g_psNumDevicesEntry;	/* 'num_devices' entry */
+
+static IMG_UINT32 g_uiNumDCEXDevs;			/* Number of valid devices */
+
+DC_EXAMPLE_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32Width = DC_EXAMPLE_WIDTH,
+	.ui32Height = DC_EXAMPLE_HEIGHT,
+	.ui32Depth = DC_EXAMPLE_BIT_DEPTH,
+	.ui32Format = DC_EXAMPLE_FORMAT,
+	.ui32MemLayout = DC_EXAMPLE_MEMORY_LAYOUT,
+	.ui32FBCFormat = DC_EXAMPLE_FBC_FORMAT,
+	.ui32RefreshRate = DC_EXAMPLE_REFRESH_RATE,
+	.ui32XDpi = DC_EXAMPLE_DPI,
+	.ui32YDpi = DC_EXAMPLE_DPI,
+	.ui32NumDevices = DC_EXAMPLE_NUM_DEVICES
+};
+
+module_param_named(width,	sModuleParams.ui32Width, uint, S_IRUGO);
+module_param_named(height,	sModuleParams.ui32Height, uint, S_IRUGO);
+module_param_named(depth,	sModuleParams.ui32Depth, uint, S_IRUGO);
+module_param_named(format,	sModuleParams.ui32Format, uint, S_IRUGO);
+module_param_named(memlayout,	sModuleParams.ui32MemLayout, uint, S_IRUGO);
+module_param_named(fbcformat,	sModuleParams.ui32FBCFormat, uint, S_IRUGO);
+module_param_named(refreshrate,	sModuleParams.ui32RefreshRate, uint, S_IRUGO);
+module_param_named(xdpi,	sModuleParams.ui32XDpi, uint, S_IRUGO);
+module_param_named(ydpi,	sModuleParams.ui32YDpi, uint, S_IRUGO);
+module_param_named(num_devices,	sModuleParams.ui32NumDevices, uint, S_IRUGO);
+MODULE_PARM_DESC(num_devices,
+        "Number of display devices to register (default: 1 - max: 16)");
+
+const DC_EXAMPLE_MODULE_PARAMETERS *DCExampleGetModuleParameters(void)
+{
+	return &sModuleParams;
+}
+
+static int NumDevicesOpen(struct inode *psInode, struct file *psFile)
+{
+	psFile->private_data = psInode->i_private;
+
+	return 0;
+}
+
+static ssize_t NumDevicesRead(struct file *psFile,
+                              char __user *psUserBuffer,
+                              size_t uiCount, loff_t *puiPosition)
+{
+	char aszBuffer[]="XX\n";
+	loff_t uiPosition = *puiPosition;
+	size_t uiBufferSize = ARRAY_SIZE(aszBuffer);
+	size_t uiLocCount;
+
+	PVR_UNREFERENCED_PARAMETER(psFile);
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	(void) DC_OSSNPrintf(aszBuffer, sizeof(aszBuffer), "%u", g_uiNumDCEXDevs);
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+
+	if (copy_to_user(psUserBuffer, &aszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static const struct file_operations gsDCEXNumDevicesFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = NumDevicesOpen,
+	.read = NumDevicesRead,
+	.llseek = default_llseek
+};
+
+typedef struct _DC_EXAMPLE_PAGE_MAPPING
+{
+	void         *pvVirtAddr;
+	unsigned int  uiPageCnt;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	struct page **ppsPageArray;
+#endif
+} DC_EXAMPLE_PAGE_MAPPING;
+
+static void pvr_vfree(DC_EXAMPLE_PAGE_MAPPING *psMap)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	unsigned int i;
+#endif
+
+	if (!psMap)
+	{
+		return;
+	}
+
+	vfree(psMap->pvVirtAddr);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	/* Free up all the pages. */
+	for (i = 0; i < psMap->uiPageCnt; i++)
+	{
+		if (psMap->ppsPageArray[i])
+		{
+			__free_page(psMap->ppsPageArray[i]);
+		}
+	}
+
+	kfree(psMap->ppsPageArray);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */
+
+	kfree(psMap);
+}
+
+static void *pvr_vmalloc(size_t uiSize, gfp_t gfp_mask, pgprot_t prot)
+{
+	DC_EXAMPLE_PAGE_MAPPING *psMap;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	unsigned int i;
+#endif
+
+	psMap = kzalloc(sizeof(*psMap), GFP_KERNEL);
+	if (!psMap)
+	{
+		return NULL;
+	}
+
+	psMap->uiPageCnt = DIV_ROUND_UP(uiSize, PAGE_SIZE);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	psMap->ppsPageArray = kcalloc(psMap->uiPageCnt, sizeof(*psMap->ppsPageArray), GFP_KERNEL);
+	if (!psMap->ppsPageArray)
+	{
+		goto err_pvr_vfree;
+	}
+
+	/* Allocate kernel pages 1 by 1 - Keep reference in the "pages" struct. */
+	for (i = 0; i < psMap->uiPageCnt; i++)
+	{
+		psMap->ppsPageArray[i] = alloc_page(gfp_mask);
+		if (!psMap->ppsPageArray[i])
+		{
+			goto err_pvr_vfree;
+		}
+	}
+
+	/* Map the pages into contiguous VM space. */
+	psMap->pvVirtAddr = vmap(psMap->ppsPageArray, (uiSize >> PAGE_SHIFT), VM_MAP, prot);
+#else
+	psMap->pvVirtAddr = __vmalloc(uiSize, gfp_mask, prot);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */
+
+	if (!psMap->pvVirtAddr)
+	{
+		goto err_pvr_vfree;
+	}
+
+	return (void *) psMap;
+
+err_pvr_vfree:
+	pvr_vfree(psMap);
+	return NULL;
+}
+
+void *DCExampleVirtualAllocUncached(size_t uiSize)
+{
+	unsigned int gfp_flags = GFP_USER | __GFP_NOWARN | __GFP_NOMEMALLOC;
+
+#if defined(CONFIG_X86_64)
+	gfp_flags |= __GFP_DMA32;
+#else
+#if !defined(CONFIG_X86_PAE)
+	gfp_flags |= __GFP_HIGHMEM;
+#endif
+#endif
+
+	return pvr_vmalloc(uiSize, gfp_flags, pgprot_noncached(PAGE_KERNEL));
+}
+
+IMG_BOOL DCExampleVirtualFree(void *pvAllocHandle)
+{
+	pvr_vfree((DC_EXAMPLE_PAGE_MAPPING *) pvAllocHandle);
+
+	/* vfree does not return a value, so all we can do is hard code IMG_TRUE */
+	return IMG_TRUE;
+}
+
+#define	VMALLOC_TO_PAGE_PHYS(vAddr) page_to_phys(vmalloc_to_page(vAddr))
+
+PVRSRV_ERROR DCExampleGetLinAddr(void *pvAllocHandle, IMG_CPU_VIRTADDR *ppvLinAddr)
+{
+	DC_EXAMPLE_PAGE_MAPPING *psMap     = (DC_EXAMPLE_PAGE_MAPPING *) pvAllocHandle;
+	IMG_CPU_VIRTADDR         pvLinAddr = psMap->pvVirtAddr;
+
+	if (ppvLinAddr != NULL)
+	{
+		*ppvLinAddr = pvLinAddr;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DCExampleGetDevPAddrs(void *pvAllocHandle, IMG_DEV_PHYADDR *pasDevPAddr,
+				   uint32_t uiPageNo, size_t uiSize)
+{
+	DC_EXAMPLE_PAGE_MAPPING *psMap = (DC_EXAMPLE_PAGE_MAPPING *) pvAllocHandle;
+	unsigned long ulPages      = DC_OS_BYTES_TO_PAGES(uiSize);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	IMG_CPU_VIRTADDR pvLinAddr = (IMG_CPU_VIRTADDR) IMG_OFFSET_ADDR(psMap->pvVirtAddr, uiPageNo * PAGE_SIZE);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)) */
+	int           i;
+
+	if (ulPages > psMap->uiPageCnt)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	for (i = 0; i < ulPages; i++)
+	{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+		pasDevPAddr[i].uiAddr = VMALLOC_TO_PAGE_PHYS(pvLinAddr);
+		pvLinAddr += PAGE_SIZE;
+#else
+		pasDevPAddr[i].uiAddr = page_to_phys(psMap->ppsPageArray[uiPageNo + i]);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)) */
+	}
+
+	return PVRSRV_OK;
+}
+
+static int __init dc_example_init(void)
+{
+	if (DCExampleInit(&g_uiNumDCEXDevs) != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+	/* Create a debugfs entry to return the number of devices discovered */
+	g_psDebugFSEntryDir = debugfs_create_dir(MODNAME, NULL);
+	if (IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs root directory "
+		       "(debugfs entries won't be available)\n", __func__, MODNAME);
+		g_psDebugFSEntryDir = NULL;
+	}
+
+	g_psNumDevicesEntry = debugfs_create_file(DC_EXAMPLE_DEBUGFS_NUM_DEVICES,
+	                                          S_IFREG | S_IRUGO,
+	                                          g_psDebugFSEntryDir,
+	                                          &sModuleParams.ui32NumDevices,
+	                                          &gsDCEXNumDevicesFileOps);
+
+	if (IS_ERR_OR_NULL(g_psNumDevicesEntry))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs file "
+		       "(debugfs entries won't be available)\n", __func__,
+		       DC_EXAMPLE_DEBUGFS_NUM_DEVICES);
+		g_psNumDevicesEntry = NULL;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ Function Name: dc_example_deinit
+ Description  : Remove the driver from the kernel.
+
+                __exit places the function in a special memory section that
+                the kernel frees once the function has been run.  Refer also
+                to module_exit() macro call below.
+******************************************************************************/
+static void __exit dc_example_deinit(void)
+{
+	DCExampleDeinit();
+
+	/* Remove the debugfs entry for the number of devices */
+	if (g_psNumDevicesEntry != NULL)
+	{
+		debugfs_remove(g_psNumDevicesEntry);
+		g_psNumDevicesEntry = NULL;
+	}
+
+	if (g_psDebugFSEntryDir != NULL)
+	{
+		debugfs_remove(g_psDebugFSEntryDir);
+		g_psDebugFSEntryDir = NULL;
+	}
+}
+
+module_init(dc_example_init);
+module_exit(dc_example_deinit);
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/include/dc_example.h b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/include/dc_example.h
new file mode 100644
index 000000000000..e74362249791
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/include/dc_example.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_EXAMPLE_H)
+#define DC_EXAMPLE_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(LMA)
+	#define DRVNAME	"dc_example_LMA"
+#else
+	#define DRVNAME	"dc_example_UMA"
+#endif
+
+#define MODNAME "dc_example"
+
+/******************************************************************************
+ * dc_example OS functions
+ *****************************************************************************/
+
+/* Module parameters */
+typedef struct _DC_EXAMPLE_MODULE_PARAMETERS_
+{
+	IMG_UINT32 ui32Width;
+	IMG_UINT32 ui32Height;
+	IMG_UINT32 ui32Depth;
+	IMG_UINT32 ui32Format;
+	IMG_UINT32 ui32MemLayout;
+	IMG_UINT32 ui32FBCFormat;
+	IMG_UINT32 ui32RefreshRate;
+	IMG_UINT32 ui32XDpi;
+	IMG_UINT32 ui32YDpi;
+	IMG_UINT32 ui32NumDevices;
+} DC_EXAMPLE_MODULE_PARAMETERS;
+
+const DC_EXAMPLE_MODULE_PARAMETERS *DCExampleGetModuleParameters(void);
+
+void        *DCExampleVirtualAllocUncached(size_t uiSize);
+IMG_BOOL     DCExampleVirtualFree(void *pvAllocHandle);
+PVRSRV_ERROR DCExampleGetLinAddr(void *pvAllocHandle, IMG_CPU_VIRTADDR *ppvLinAddr);
+PVRSRV_ERROR DCExampleGetDevPAddrs(void *pvAllocHandle, IMG_DEV_PHYADDR *pasDevPAddr,
+				   uint32_t uiPageNo, size_t uiSize);
+#if defined(INTEGRITY_OS)
+PVRSRV_ERROR DCExampleOSAcquireKernelMappingData(void *pvAllocHandle, IMG_HANDLE *phMapping, void **ppPhysAddr);
+#endif
+
+/******************************************************************************
+ * dc_example functions
+ *****************************************************************************/
+
+PVRSRV_ERROR DCExampleInit(IMG_UINT32 *puiNumDevices);
+void DCExampleDeinit(void);
+
+#endif /* !defined(DC_EXAMPLE_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Kbuild.mk
new file mode 100644
index 000000000000..10b32e2870d1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Kbuild.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-y += \
+ -I$(TOP)/services/display/dc_fbdev
+
+dc_fbdev-y += \
+	services/display/dc_fbdev/dc_fbdev.o
+
+ifneq ($(W),1)
+#CFLAGS_services/display/dc_fbdev/dc_fbdev.o := -Werror
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Linux.mk
new file mode 100644
index 000000000000..a26c521e050b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := dc_fbdev
+
+dc_fbdev_type := kernel_module
+dc_fbdev_target := dc_fbdev.ko
+dc_fbdev_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/dc_fbdev.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/dc_fbdev.c
new file mode 100644
index 000000000000..5bc2a8eb35b8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/dc_fbdev.c
@@ -0,0 +1,941 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/fb.h>
+
+#include "kerneldisplay.h"
+#include "powervr/imgpixfmts.h"
+#include "pvrmodule.h" /* for MODULE_LICENSE() */
+
+#if !defined(CONFIG_FB)
+#error dc_fbdev needs Linux framebuffer support. Enable it in your kernel.
+#endif
+
+#define DRVNAME					"dc_fbdev"
+#define MAX_COMMANDS_IN_FLIGHT	2
+
+#if defined(DC_FBDEV_NUM_PREFERRED_BUFFERS)
+#define NUM_PREFERRED_BUFFERS	DC_FBDEV_NUM_PREFERRED_BUFFERS
+#else
+#define NUM_PREFERRED_BUFFERS	2
+#endif
+
+#define FALLBACK_REFRESH_RATE		60
+#define FALLBACK_DPI				160
+
+typedef struct
+{
+	IMG_HANDLE			hSrvHandle;
+	IMG_UINT32			ePixFormat;
+	struct fb_info		*psLINFBInfo;
+	bool				bCanFlip;
+}
+DC_FBDEV_DEVICE;
+
+typedef struct
+{
+	DC_FBDEV_DEVICE		*psDeviceData;
+	IMG_HANDLE			hLastConfigData;
+	IMG_UINT32		ui32AllocUseMask;
+}
+DC_FBDEV_CONTEXT;
+
+typedef struct
+{
+	DC_FBDEV_CONTEXT	*psDeviceContext;
+	IMG_UINT32			ui32Width;
+	IMG_UINT32			ui32Height;
+	IMG_UINT32			ui32ByteStride;
+	IMG_UINT32			ui32BufferID;
+}
+DC_FBDEV_BUFFER;
+
+static IMG_UINT32 gui32fb_devminor = 0;
+module_param_named(fb_devminor, gui32fb_devminor, uint, S_IRUGO | S_IWUSR);
+
+static DC_FBDEV_DEVICE *gpsDeviceData;
+
+#if defined(DC_FBDEV_FORCE_CONTEXT_CLEAN)
+static
+void DC_FBDEV_Clean(IMG_HANDLE hDisplayContext, IMG_HANDLE *ahBuffers)
+{
+	DC_FBDEV_DEVICE *psDeviceData;
+	DC_FBDEV_BUFFER *psBuffer;
+	IMG_CPU_PHYADDR sCPUPhysStart;
+	void *pvOSDevice, *pvVirtStart;
+	IMG_UINT32 ui32ByteSize, *pbPhysEnd;
+	uintptr_t uiStartAddr, uiEndAddr, uiMaxLen;
+
+	/* validate incoming parameters */
+	if (!hDisplayContext ||
+		!ahBuffers       ||
+		!((DC_FBDEV_CONTEXT *)hDisplayContext)->psDeviceData ||
+		!((DC_FBDEV_CONTEXT *)hDisplayContext)->psDeviceData->psLINFBInfo)
+	{
+		return;
+	}
+
+	/* lookup display context data, buffer and kernel device handle */
+	psDeviceData = ((DC_FBDEV_CONTEXT *)hDisplayContext)->psDeviceData;
+	pvOSDevice = gpsDeviceData->psLINFBInfo->dev;
+	PVR_UNREFERENCED_PARAMETER(pvOSDevice);
+	psBuffer = ahBuffers[0];
+
+	/* calc. framebuffer start/end addresses */
+	ui32ByteSize = psBuffer->ui32ByteStride * psBuffer->ui32Height;
+	uiStartAddr  = (uintptr_t)psDeviceData->psLINFBInfo->screen_base;
+	uiStartAddr += psBuffer->ui32BufferID * ui32ByteSize;
+	uiMaxLen     = psDeviceData->psLINFBInfo->fix.smem_len;
+	uiMaxLen    -= psBuffer->ui32BufferID * ui32ByteSize;
+	uiEndAddr    = uiStartAddr + ui32MaxLen;
+
+	/* outer framebuffer inter-page loop */
+	pvVirtStart  = (IMG_BYTE *)((uintptr_t)uiStartAddr & (~(PAGE_SIZE-1)));
+	while ((uintptr_t)pvVirtStart < uiEndAddr)
+	{
+#if !(defined(ARM64) && defined(X86))
+		/* crossing (possibly discontiguous) page boundary, translate virt -> phys page */
+		sCPUPhysStart.uiAddr =
+		page_to_phys(vmalloc_to_page(pvVirtStart));
+		pbPhysEnd = (IMG_UINT32 *)((uintptr_t)sCpuPAddr.uiAddr & (~(PAGE_SIZE-1)));
+#endif
+
+		/* inner intra-page loop */
+		while (pvVirtStart < (pvVirtStart + PAGE_SIZE))
+		{
+			/* per-arch d-cache flush mechanism */
+#if defined(CONFIG_X86)
+			asm volatile("clflush %0" : "+m" (*pvVirtStart));
+#elif defined(CONFIG_ARM64)
+			asm volatile ("dc civac, %0" :: "r" (pvVirtStart));
+#elif defined(CONFIG_ARM)
+			unsigned long uiLockFlags = {0};
+			spinlock_t spinlock = {0};
+			flush_cache_all();
+#if defined(CONFIG_OUTER_CACHE)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0))
+			spin_lock_init(&spinlock);
+			spin_lock_irqsave(&spinlock, uiLockFlags);
+			outer_clean_range(0, ULONG_MAX);
+			spin_unlock_irqrestore(&spinlock, uiLockFlags);
+#else
+			outer_clean_range(0, ULONG_MAX)
+#endif
+#endif
+			PVR_UNREFERENCED_PARAMETER(uiLockFlags);
+			PVR_UNREFERENCED_PARAMETER(spinlock);
+			return;
+#elif defined(CONFIG_MIPS)
+			dma_cache_sync(pvOSDevice, (void *)pvVirtStart, PAGE_SIZE, DMA_TO_DEVICE);
+#elif defined(CONFIG_META)
+			writeback_dcache_region((void *)pbPhysEnd, PAGE_SIZE);
+#else
+			PVR_ASSERT(0);
+#endif
+
+			/* move to next d-cache line/page */
+#if defined(ARM64) || defined(X86)
+			pvVirtStart += cache_line_size();
+#else
+			pvVirtStart += PAGE_SIZE;
+#endif
+		}
+	}
+}
+#endif
+
+static
+void DC_FBDEV_GetInfo(IMG_HANDLE hDeviceData,
+                      DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	strncpy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	psDisplayInfo->ui32MinDisplayPeriod	= 0;
+	psDisplayInfo->ui32MaxDisplayPeriod	= 1;
+	psDisplayInfo->ui32MaxPipes			= 1;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_PanelQueryCount(IMG_HANDLE hDeviceData,
+									  IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+	*pui32NumPanels = 1;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_PanelQuery(IMG_HANDLE hDeviceData,
+								 IMG_UINT32 ui32PanelsArraySize,
+								 IMG_UINT32 *pui32NumPanels,
+								 PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	DC_FBDEV_DEVICE *psDeviceData = hDeviceData;
+	struct fb_var_screeninfo *psVar = &psDeviceData->psLINFBInfo->var;
+	struct fb_var_screeninfo sVar = { .pixclock = 0 };
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+		return PVRSRV_ERROR_RETRY;
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	*pui32NumPanels = 1;
+
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = psDeviceData->ePixFormat;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width    = psVar->xres;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height   = psVar->yres;
+	psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+	psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+
+	/* Conformant fbdev drivers should have 'var' and mode in sync by now,
+	 * but some don't (like drmfb), so try a couple of different ways to
+	 * get the info before falling back to the default.
+	 */
+	if (psVar->xres > 0 && psVar->yres > 0 && psVar->pixclock > 0)
+		sVar = *psVar;
+	else if (psDeviceData->psLINFBInfo->mode)
+		fb_videomode_to_var(&sVar, psDeviceData->psLINFBInfo->mode);
+
+	/* Override the refresh rate when defined. */
+#ifdef DC_FBDEV_REFRESH
+	psPanelInfo[0].ui32RefreshRate = DC_FBDEV_REFRESH;
+#else
+	if (sVar.xres > 0 && sVar.yres > 0 && sVar.pixclock > 0)
+	{
+		psPanelInfo[0].ui32RefreshRate = 1000000000LU /
+			((sVar.upper_margin + sVar.lower_margin +
+			  sVar.yres + sVar.vsync_len) *
+			 (sVar.left_margin  + sVar.right_margin +
+			  sVar.xres + sVar.hsync_len) *
+			 (sVar.pixclock / 1000));
+	}
+	else
+		psPanelInfo[0].ui32RefreshRate = FALLBACK_REFRESH_RATE;
+#endif
+
+	psPanelInfo[0].ui32XDpi =
+		((int)sVar.width > 0) ? (254000 / sVar.width * psVar->xres / 10000) : FALLBACK_DPI;
+
+	psPanelInfo[0].ui32YDpi	=
+		((int)sVar.height > 0) ? 254000 / sVar.height * psVar->yres / 10000 : FALLBACK_DPI;
+
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_FormatQuery(IMG_HANDLE hDeviceData,
+								  IMG_UINT32 ui32NumFormats,
+								  PVRSRV_SURFACE_FORMAT *pasFormat,
+								  IMG_UINT32 *pui32Supported)
+{
+	DC_FBDEV_DEVICE *psDeviceData = hDeviceData;
+	int i;
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (pasFormat[i].ePixFormat == psDeviceData->ePixFormat)
+			pui32Supported[i]++;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_DimQuery(IMG_HANDLE hDeviceData,
+							   IMG_UINT32 ui32NumDims,
+							   PVRSRV_SURFACE_DIMS *psDim,
+							   IMG_UINT32 *pui32Supported)
+{
+	DC_FBDEV_DEVICE *psDeviceData = hDeviceData;
+	struct fb_var_screeninfo *psVar = &psDeviceData->psLINFBInfo->var;
+	int i;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+		return PVRSRV_ERROR_RETRY;
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (psDim[i].ui32Width  == psVar->xres &&
+		    psDim[i].ui32Height == psVar->yres)
+		    pui32Supported[i]++;
+	}
+
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_ContextCreate(IMG_HANDLE hDeviceData,
+									IMG_HANDLE *hDisplayContext)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	psDeviceContext = kzalloc(sizeof(DC_FBDEV_CONTEXT), GFP_KERNEL);
+	if (!psDeviceContext)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psDeviceContext->psDeviceData = hDeviceData;
+	*hDisplayContext = psDeviceContext;
+
+err_out:
+	return eError;
+}
+
+static PVRSRV_ERROR
+DC_FBDEV_ContextConfigureCheck(IMG_HANDLE hDisplayContext,
+							   IMG_UINT32 ui32PipeCount,
+							   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+							   IMG_HANDLE *ahBuffers)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+	DC_FBDEV_DEVICE *psDeviceData = psDeviceContext->psDeviceData;
+	struct fb_var_screeninfo *psVar = &psDeviceData->psLINFBInfo->var;
+	DC_FBDEV_BUFFER *psBuffer;
+	PVRSRV_ERROR eError;
+
+	if (ui32PipeCount != 1)
+	{
+		eError = PVRSRV_ERROR_DC_TOO_MANY_PIPES;
+		goto err_out;
+	}
+
+	if (!ahBuffers)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_CONFIG;
+		goto err_out;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+	{
+		eError = PVRSRV_ERROR_RETRY;
+		goto err_out;
+	}
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	psBuffer = ahBuffers[0];
+
+	if (pasSurfAttrib[0].sCrop.sDims.ui32Width  != psVar->xres ||
+		pasSurfAttrib[0].sCrop.sDims.ui32Height != psVar->yres ||
+		pasSurfAttrib[0].sCrop.i32XOffset != 0 ||
+		pasSurfAttrib[0].sCrop.i32YOffset != 0)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+		goto err_unlock;
+	}
+
+	if (pasSurfAttrib[0].sDisplay.sDims.ui32Width !=
+		pasSurfAttrib[0].sCrop.sDims.ui32Width ||
+		pasSurfAttrib[0].sDisplay.sDims.ui32Height !=
+		pasSurfAttrib[0].sCrop.sDims.ui32Height ||
+		pasSurfAttrib[0].sDisplay.i32XOffset !=
+		pasSurfAttrib[0].sCrop.i32XOffset ||
+		pasSurfAttrib[0].sDisplay.i32YOffset !=
+		pasSurfAttrib[0].sCrop.i32YOffset)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+		goto err_unlock;
+	}
+
+	if (psBuffer->ui32Width  != psVar->xres &&
+		psBuffer->ui32Height != psVar->yres)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+		goto err_unlock;
+	}
+
+	eError = PVRSRV_OK;
+err_unlock:
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+err_out:
+	return eError;
+}
+
+static
+void DC_FBDEV_ContextConfigure(IMG_HANDLE hDisplayContext,
+								   IMG_UINT32 ui32PipeCount,
+								   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+								   IMG_HANDLE *ahBuffers,
+								   IMG_UINT32 ui32DisplayPeriod,
+								   IMG_HANDLE hConfigData)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+	DC_FBDEV_DEVICE *psDeviceData = psDeviceContext->psDeviceData;
+	struct fb_var_screeninfo sVar = psDeviceData->psLINFBInfo->var;
+	int err;
+
+	PVR_UNREFERENCED_PARAMETER(ui32PipeCount);
+	PVR_UNREFERENCED_PARAMETER(pasSurfAttrib);
+	PVR_UNREFERENCED_PARAMETER(ui32DisplayPeriod);
+
+	sVar.yoffset = 0;
+
+	if (ui32PipeCount != 0)
+	{
+		BUG_ON(ahBuffers == NULL);
+
+		if (psDeviceData->bCanFlip)
+		{
+			DC_FBDEV_BUFFER *psBuffer = ahBuffers[0];
+			sVar.yoffset = sVar.yres * psBuffer->ui32BufferID;
+		}
+	}
+
+#if defined(DC_FBDEV_FORCE_CONTEXT_CLEAN)
+	DC_FBDEV_Clean(hDisplayContext, ahBuffers);
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+	{
+		return;
+	}
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	console_lock();
+
+	/* If we're supposed to be able to flip, but the yres_virtual has been
+	 * changed to an unsupported (smaller) value, we need to change it back
+	 * (this is a workaround for some Linux fbdev drivers that seem to lose any
+	 * modifications to yres_virtual after a blank.)
+	 */
+	if (psDeviceData->bCanFlip &&
+		sVar.yres_virtual < sVar.yres * NUM_PREFERRED_BUFFERS)
+	{
+		sVar.activate = FB_ACTIVATE_NOW;
+		sVar.yres_virtual = sVar.yres * NUM_PREFERRED_BUFFERS;
+
+		err = fb_set_var(psDeviceData->psLINFBInfo, &sVar);
+		if (err)
+			pr_err("fb_set_var failed (err=%d)\n", err);
+	}
+	else
+	{
+		err = fb_pan_display(psDeviceData->psLINFBInfo, &sVar);
+		if (err)
+			pr_err("fb_pan_display failed (err=%d)\n", err);
+	}
+
+	console_unlock();
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+
+	if (psDeviceContext->hLastConfigData)
+	{
+		DCDisplayConfigurationRetired(psDeviceContext->hLastConfigData);
+	}
+
+	if (ui32PipeCount != 0)
+	{
+		psDeviceContext->hLastConfigData = hConfigData;
+	}
+	else
+	{
+		/* If the pipe count is zero, we're tearing down. Don't record
+		 * any new configurations.
+		 */
+		psDeviceContext->hLastConfigData = NULL;
+
+		/* We still need to "retire" this NULL flip as that signals back to
+		 * the DC core that we've finished doing what we need to do and it
+		 * can destroy the display context
+		 */
+		DCDisplayConfigurationRetired(hConfigData);
+	}
+}
+
+static
+void DC_FBDEV_ContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+
+	BUG_ON(psDeviceContext->hLastConfigData != NULL);
+	kfree(psDeviceContext);
+}
+
+static
+IMG_BOOL DC_FBDEV_GetBufferID(DC_FBDEV_CONTEXT *psDeviceContext, IMG_UINT32 *pui32BufferID)
+{
+	IMG_UINT32 ui32IDLimit;
+	IMG_UINT32 ui32BufferID;
+
+	if (psDeviceContext->psDeviceData->bCanFlip)
+	{
+		ui32IDLimit = NUM_PREFERRED_BUFFERS;
+	}
+	else
+	{
+		ui32IDLimit = 1;
+	}
+
+	for (ui32BufferID = 0; ui32BufferID < ui32IDLimit; ++ui32BufferID)
+	{
+		if ((psDeviceContext->ui32AllocUseMask & (1UL << ui32BufferID)) == 0)
+		{
+			psDeviceContext->ui32AllocUseMask |= (1UL << ui32BufferID);
+
+			*pui32BufferID = ui32BufferID;
+
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+
+static
+void DC_FBDEV_PutBufferID(DC_FBDEV_CONTEXT *psDeviceContext, IMG_UINT32 ui32BufferID)
+{
+	psDeviceContext->ui32AllocUseMask &= ~(1UL << ui32BufferID);
+}
+
+#define BYTE_TO_PAGES(range) (((range) + (PAGE_SIZE - 1)) >> PAGE_SHIFT)
+
+static
+PVRSRV_ERROR DC_FBDEV_BufferAlloc(IMG_HANDLE hDisplayContext,
+								  DC_BUFFER_CREATE_INFO *psCreateInfo,
+								  IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+								  IMG_UINT32 *pui32PageCount,
+								  IMG_UINT32 *pui32ByteStride,
+								  IMG_HANDLE *phBuffer)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+	DC_FBDEV_DEVICE *psDeviceData = psDeviceContext->psDeviceData;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	PVRSRV_ERROR eError;
+	DC_FBDEV_BUFFER *psBuffer;
+	IMG_UINT32 ui32ByteSize;
+
+	if (psSurfInfo->sFormat.ePixFormat != psDeviceData->ePixFormat)
+	{
+		eError = PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+		goto err_out;
+	}
+
+	psBuffer = kmalloc(sizeof(DC_FBDEV_BUFFER), GFP_KERNEL);
+	if (!psBuffer)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psBuffer->psDeviceContext = psDeviceContext;
+	psBuffer->ui32ByteStride =
+		psSurfInfo->sDims.ui32Width * psCreateInfo->ui32BPP;
+
+	psBuffer->ui32Width = psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height = psSurfInfo->sDims.ui32Height;
+
+	if (!DC_FBDEV_GetBufferID(psDeviceContext, &psBuffer->ui32BufferID))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_free;
+	}
+
+	ui32ByteSize = psBuffer->ui32ByteStride * psBuffer->ui32Height;
+
+	*puiLog2PageSize = PAGE_SHIFT;
+	*pui32PageCount	 = BYTE_TO_PAGES(ui32ByteSize);
+	*pui32ByteStride = psBuffer->ui32ByteStride;
+	*phBuffer	 = psBuffer;
+
+	return PVRSRV_OK;
+
+err_free:
+	kfree(psBuffer);
+
+err_out:
+	return eError;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_BufferAcquire(IMG_HANDLE hBuffer,
+									IMG_DEV_PHYADDR *pasDevPAddr,
+									void **ppvLinAddr)
+{
+	DC_FBDEV_BUFFER *psBuffer = hBuffer;
+	DC_FBDEV_DEVICE *psDeviceData = psBuffer->psDeviceContext->psDeviceData;
+	IMG_UINT32 ui32ByteSize = psBuffer->ui32ByteStride * psBuffer->ui32Height;
+	uintptr_t uiStartAddr;
+	IMG_UINT32 i, ui32MaxLen;
+
+#if defined(DC_FBDEV_USE_SCREEN_BASE)
+	uiStartAddr = (uintptr_t)psDeviceData->psLINFBInfo->screen_base;
+#else
+	uiStartAddr = psDeviceData->psLINFBInfo->fix.smem_start;
+#endif
+
+	uiStartAddr += psBuffer->ui32BufferID * ui32ByteSize;
+	ui32MaxLen = psDeviceData->psLINFBInfo->fix.smem_len -
+				 psBuffer->ui32BufferID * ui32ByteSize;
+
+	for (i = 0; i < BYTE_TO_PAGES(ui32ByteSize); i++)
+	{
+		BUG_ON(i * PAGE_SIZE >= ui32MaxLen);
+		pasDevPAddr[i].uiAddr = uiStartAddr + (i * PAGE_SIZE);
+#if defined(DC_FBDEV_USE_SCREEN_BASE)
+		pasDevPAddr[i].uiAddr =
+		page_to_phys(vmalloc_to_page((void *)(uintptr_t)pasDevPAddr[i].uiAddr));
+#endif
+	}
+
+	/* We're UMA, so services will do the right thing and make
+	 * its own CPU virtual address mapping for the buffer.
+	 */
+	*ppvLinAddr = NULL;
+
+	return PVRSRV_OK;
+}
+
+static void DC_FBDEV_BufferRelease(IMG_HANDLE hBuffer)
+{
+	PVR_UNREFERENCED_PARAMETER(hBuffer);
+}
+
+static void DC_FBDEV_BufferFree(IMG_HANDLE hBuffer)
+{
+	DC_FBDEV_BUFFER *psBuffer = hBuffer;
+
+	DC_FBDEV_PutBufferID(psBuffer->psDeviceContext, psBuffer->ui32BufferID);
+
+	kfree(psBuffer);
+}
+
+/* If we can flip, we need to make sure we have the memory to do so.
+ *
+ * We'll assume that the fbdev device provides extra space in
+ * yres_virtual for panning; xres_virtual is theoretically supported,
+ * but it involves more work.
+ *
+ * If the fbdev device doesn't have yres_virtual > yres, we'll try
+ * requesting it before bailing. Userspace applications commonly do
+ * this with an FBIOPUT_VSCREENINFO ioctl().
+ *
+ * Another problem is with a limitation in the services DC -- it
+ * needs framebuffers to be page aligned (this is a SW limitation,
+ * the HW can support non-page-aligned buffers). So we have to
+ * check that stride * height for a single buffer is page aligned.
+ */
+
+static bool DC_FBDEV_FlipPossible(struct fb_info *psLINFBInfo)
+{
+	struct fb_var_screeninfo sVar = psLINFBInfo->var;
+	int err;
+
+	if (!psLINFBInfo->fix.xpanstep && !psLINFBInfo->fix.ypanstep &&
+		!psLINFBInfo->fix.ywrapstep)
+	{
+		pr_err("The fbdev device detected does not support ypan/ywrap. "
+			   "Flipping disabled.\n");
+		return false;
+	}
+
+	if ((psLINFBInfo->fix.line_length * sVar.yres) % PAGE_SIZE != 0)
+	{
+		pr_err("Line length (in bytes) x yres is not a multiple of "
+			   "page size. Flipping disabled.\n");
+		return false;
+	}
+
+	/* We might already have enough space */
+	if (sVar.yres * NUM_PREFERRED_BUFFERS <= sVar.yres_virtual)
+		return true;
+
+	pr_err("No buffer space for flipping; asking for more.\n");
+
+	sVar.activate = FB_ACTIVATE_NOW;
+	sVar.yres_virtual = sVar.yres * NUM_PREFERRED_BUFFERS;
+
+	err = fb_set_var(psLINFBInfo, &sVar);
+	if (err)
+	{
+		pr_err("fb_set_var failed (err=%d). Flipping disabled.\n", err);
+		return false;
+	}
+
+	if (sVar.yres * NUM_PREFERRED_BUFFERS > sVar.yres_virtual)
+	{
+		pr_err("Failed to obtain additional buffer space. "
+			   "Flipping disabled.\n");
+		return false;
+	}
+
+	/* Some fbdev drivers allow the yres_virtual modification through,
+	 * but don't actually update the fix. We need the fix to be updated
+	 * and more memory allocated, so we can actually take advantage of
+	 * the increased yres_virtual.
+	 */
+	if (psLINFBInfo->fix.smem_len < psLINFBInfo->fix.line_length * sVar.yres_virtual)
+	{
+		pr_err("'fix' not re-allocated with sufficient buffer space. "
+			   "Flipping disabled.\n");
+		return false;
+	}
+
+	return true;
+}
+
+static int __init DC_FBDEV_init(void)
+{
+	static DC_DEVICE_FUNCTIONS sDCFunctions =
+	{
+		.pfnGetInfo					= DC_FBDEV_GetInfo,
+		.pfnPanelQueryCount			= DC_FBDEV_PanelQueryCount,
+		.pfnPanelQuery				= DC_FBDEV_PanelQuery,
+		.pfnFormatQuery				= DC_FBDEV_FormatQuery,
+		.pfnDimQuery				= DC_FBDEV_DimQuery,
+		.pfnSetBlank				= NULL,
+		.pfnSetVSyncReporting		= NULL,
+		.pfnLastVSyncQuery			= NULL,
+		.pfnContextCreate			= DC_FBDEV_ContextCreate,
+		.pfnContextDestroy			= DC_FBDEV_ContextDestroy,
+		.pfnContextConfigure		= DC_FBDEV_ContextConfigure,
+		.pfnContextConfigureCheck	= DC_FBDEV_ContextConfigureCheck,
+		.pfnBufferAlloc				= DC_FBDEV_BufferAlloc,
+		.pfnBufferAcquire			= DC_FBDEV_BufferAcquire,
+		.pfnBufferRelease			= DC_FBDEV_BufferRelease,
+		.pfnBufferFree				= DC_FBDEV_BufferFree,
+	};
+
+	struct fb_info *psLINFBInfo;
+	IMG_PIXFMT ePixFormat;
+	int err = -ENODEV;
+
+	if (gui32fb_devminor >= FB_MAX)
+	{
+		pr_err("Invalid Linux framebuffer device minor number!\n"
+			   "fb_devminor (%u) >= FB_MAX (%u)",
+				gui32fb_devminor,
+				FB_MAX);
+		goto err_out;
+	}
+
+	psLINFBInfo = registered_fb[gui32fb_devminor];
+	if (!psLINFBInfo)
+	{
+		pr_err("No Linux framebuffer (/dev/fbdev%u) device is registered!\n"
+			   "Check you have a framebuffer driver compiled into your "
+			   "kernel\nand that it is enabled on the cmdline.\n",
+			   gui32fb_devminor);
+		goto err_out;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psLINFBInfo))
+		goto err_out;
+#else
+	lock_fb_info(psLINFBInfo);
+#endif
+
+	console_lock();
+
+	/* Filter out broken FB devices */
+	if (!psLINFBInfo->fix.smem_len || !psLINFBInfo->fix.line_length)
+	{
+		pr_err("The fbdev device detected had a zero smem_len or "
+			   "line_length,\nwhich suggests it is a broken driver.\n");
+		goto err_unlock;
+	}
+
+	if (psLINFBInfo->fix.type != FB_TYPE_PACKED_PIXELS ||
+		psLINFBInfo->fix.visual != FB_VISUAL_TRUECOLOR)
+	{
+		pr_err("The fbdev device detected is not truecolor with packed "
+			   "pixels.\n");
+		goto err_unlock;
+	}
+
+	if (psLINFBInfo->var.bits_per_pixel == 32)
+	{
+		if (psLINFBInfo->var.red.length   != 8  ||
+			psLINFBInfo->var.green.length != 8  ||
+			psLINFBInfo->var.blue.length  != 8  ||
+			psLINFBInfo->var.red.offset   != 16 ||
+			psLINFBInfo->var.green.offset != 8  ||
+			psLINFBInfo->var.blue.offset  != 0)
+		{
+			pr_err("The fbdev device detected uses an unrecognized "
+				   "32bit pixel format (%u/%u/%u, %u/%u/%u)\n",
+				   psLINFBInfo->var.red.length,
+				   psLINFBInfo->var.green.length,
+				   psLINFBInfo->var.blue.length,
+				   psLINFBInfo->var.red.offset,
+				   psLINFBInfo->var.green.offset,
+				   psLINFBInfo->var.blue.offset);
+			goto err_unlock;
+		}
+#if defined(DC_FBDEV_FORCE_XRGB8888)
+		ePixFormat = IMG_PIXFMT_B8G8R8X8_UNORM;
+#else
+		ePixFormat = IMG_PIXFMT_B8G8R8A8_UNORM;
+#endif
+	}
+	else if (psLINFBInfo->var.bits_per_pixel == 16)
+	{
+		if (psLINFBInfo->var.red.length   != 5  ||
+			psLINFBInfo->var.green.length != 6  ||
+			psLINFBInfo->var.blue.length  != 5  ||
+			psLINFBInfo->var.red.offset   != 11 ||
+			psLINFBInfo->var.green.offset != 5  ||
+			psLINFBInfo->var.blue.offset  != 0)
+		{
+			pr_err("The fbdev device detected uses an unrecognized "
+				   "16bit pixel format (%u/%u/%u, %u/%u/%u)\n",
+				   psLINFBInfo->var.red.length,
+				   psLINFBInfo->var.green.length,
+				   psLINFBInfo->var.blue.length,
+				   psLINFBInfo->var.red.offset,
+				   psLINFBInfo->var.green.offset,
+				   psLINFBInfo->var.blue.offset);
+			goto err_unlock;
+		}
+		ePixFormat = IMG_PIXFMT_B5G6R5_UNORM;
+	}
+	else
+	{
+		pr_err("The fbdev device detected uses an unsupported "
+			   "bpp (%u).\n", psLINFBInfo->var.bits_per_pixel);
+		goto err_unlock;
+	}
+
+	if (!try_module_get(psLINFBInfo->fbops->owner))
+	{
+		pr_err("try_module_get() failed");
+		goto err_unlock;
+	}
+
+	if (psLINFBInfo->fbops->fb_open &&
+		psLINFBInfo->fbops->fb_open(psLINFBInfo, 0) != 0)
+	{
+		pr_err("fb_open() failed");
+		goto err_module_put;
+	}
+
+	gpsDeviceData = kmalloc(sizeof(DC_FBDEV_DEVICE), GFP_KERNEL);
+	if (!gpsDeviceData)
+		goto err_module_put;
+
+	gpsDeviceData->psLINFBInfo = psLINFBInfo;
+	gpsDeviceData->ePixFormat = ePixFormat;
+
+	if (DCRegisterDevice(&sDCFunctions,
+						 MAX_COMMANDS_IN_FLIGHT,
+						 gpsDeviceData,
+						 &gpsDeviceData->hSrvHandle) != PVRSRV_OK)
+		goto err_kfree;
+
+	gpsDeviceData->bCanFlip = DC_FBDEV_FlipPossible(psLINFBInfo);
+
+	pr_info("Found usable fbdev device (%s):\n"
+#if defined(DC_FBDEV_USE_SCREEN_BASE)
+			"range (virtual)  = 0x%lx-0x%lx\n"
+#else
+			"range (physical) = 0x%lx-0x%lx\n"
+#endif
+			"size (bytes)     = 0x%x\n"
+			"xres x yres      = %ux%u\n"
+			"xres x yres (v)  = %ux%u\n"
+			"img pix fmt      = %u\n"
+			"flipping?        = %d\n",
+			psLINFBInfo->fix.id,
+			psLINFBInfo->fix.smem_start,
+			psLINFBInfo->fix.smem_start + psLINFBInfo->fix.smem_len,
+			psLINFBInfo->fix.smem_len,
+			psLINFBInfo->var.xres, psLINFBInfo->var.yres,
+			psLINFBInfo->var.xres_virtual, psLINFBInfo->var.yres_virtual,
+			ePixFormat, gpsDeviceData->bCanFlip);
+	err = 0;
+err_unlock:
+	console_unlock();
+	unlock_fb_info(psLINFBInfo);
+err_out:
+	return err;
+err_kfree:
+	kfree(gpsDeviceData);
+err_module_put:
+	module_put(psLINFBInfo->fbops->owner);
+	goto err_unlock;
+}
+
+static void __exit DC_FBDEV_exit(void)
+{
+	DC_FBDEV_DEVICE *psDeviceData = gpsDeviceData;
+	struct fb_info *psLINFBInfo = psDeviceData->psLINFBInfo;
+
+	lock_fb_info(psLINFBInfo);
+	console_lock();
+
+	if (psLINFBInfo->fbops->fb_release)
+	   psLINFBInfo->fbops->fb_release(psLINFBInfo, 0);
+
+	module_put(psLINFBInfo->fbops->owner);
+
+	console_unlock();
+	unlock_fb_info(psLINFBInfo);
+
+	DCUnregisterDevice(psDeviceData->hSrvHandle);
+	kfree(psDeviceData);
+}
+
+module_init(DC_FBDEV_init);
+module_exit(DC_FBDEV_exit);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/common/dc_pdp_common.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/common/dc_pdp_common.c
new file mode 100644
index 000000000000..eac70858205a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/common/dc_pdp_common.c
@@ -0,0 +1,1847 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dc_pdp.h"
+#include "img_defs.h"
+#include "pdp2_regs.h"
+#include "pdp2_mmu_regs.h"
+
+#if defined(__linux__)
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#endif
+
+/*******************************************************************************
+ * Debug/print macros
+ ******************************************************************************/
+#if defined(PDP_DEBUG)
+	#define PDP_CHECKPOINT PDP_DEBUG_PRINT(" CP: - %s\n", __func__);
+	#define PDP_DEBUG_PRINT(fmt, ...) \
+		DC_OSDebugPrintf(DBGLVL_WARNING, fmt, __VA_ARGS__)
+#else
+	#define PDP_CHECKPOINT  do {} while (0)
+	#define PDP_DEBUG_PRINT(fmt, ...) do {} while (0)
+#endif
+
+#define PDP_ERROR_PRINT(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_ERROR, fmt, __VA_ARGS__)
+
+#if defined(PLATO_DISPLAY_PDUMP)
+
+#undef DC_OSWriteReg32
+#define DC_OSWriteReg32 PDP_OSWriteHWReg32
+
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	plato_pdump_pol(base,reg,val,msk, DRVNAME); \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			PDP_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#else
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			PDP_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#endif // PDUMP
+
+#define pol(base,reg,val,msk) polpr(base,reg,val,msk,10,10)
+
+#if defined(PDP_DEBUG)
+static DLLIST_NODE gsPdpDeviceListHead;  /* List of discovered / configured PDP */
+static IMG_UINT32 gNumPdpDevices;        /* Number of discovered / configured PDP */
+void PDPDebugCtrl(void)
+{
+	int k = 0;
+	int i = 0;
+	DCPDP_DEVICE *psDevice;
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsPdpDeviceListHead, psNode, psNext)
+	{
+		psDevice = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sDbgListNode);
+
+		if (gNumPdpDevices > 1)
+			printk("\n-----PDP Register dump [device %d] ------\n", i+1);
+		else
+			printk("\n------PDP Register dump -------\n");
+
+		for (k = 0; k < PDP_NUMREG; k++)
+		{
+			PDP_DEBUG_PRINT(" Offset %x: %x\n", k*4, DC_OSReadReg32(psDevice->pvPDPRegCpuVAddr, k * 4));
+		}
+
+		if (gNumPdpDevices > 1)
+			printk("\n-----PDP MMU Reg dump [device %d] ------\n", i+1);
+		else
+			printk("\n------PDP MMU Reg dump -------\n");
+
+		for (k = 0; k < PDP_BIF_VERSION_OFFSET; k += 4)
+		{
+			PDP_DEBUG_PRINT(" BIF Offset %x: %x\n", k, DC_OSReadReg32(psDevice->pvPDPBifRegCpuVAddr, k));
+		}
+		++i;
+	}
+	/*
+
+	static int i = 0;
+	IMG_CPU_VIRTADDR pvBufferCpuVAddr;
+	IMG_UINT32 *pui32Pixel;
+	IMG_UINT32 ui32SizeInPixels;
+	IMG_UINT32 j;
+
+	gpsBuffer[2] = gPdpDevice->psSystemBuffer;
+
+	for (b = 0; b < 3; b++)
+	{
+		pvBufferCpuVAddr = DC_OSMapPhysAddr(gpsBuffer[b]->sCpuPAddr, gpsBuffer[b]->ui32SizeInBytes);
+		if (pvBufferCpuVAddr != NULL)
+		{
+			pui32Pixel = (IMG_UINT32 *)pvBufferCpuVAddr;
+			ui32SizeInPixels = 10; //gpsBuffer[i]->ui32Width;// * gpsBuffer[i]->ui32Height;
+
+			for (j = 0; j < ui32SizeInPixels; j++)
+			{
+				DC_OSDebugPrintf(DBGLVL_WARNING, " - %s: Buffer %d Pixel[%u]: 0x%08X\n", __func__, b, j, pui32Pixel[j]);
+			}
+
+			DC_OSUnmapPhysAddr(pvBufferCpuVAddr, gpsBuffer[b]->ui32SizeInBytes);
+		}
+	}
+	i = (i + 1) % 2;
+	*/
+}
+#endif /* PDP_DEBUG */
+
+static IMG_BOOL BufferAcquireMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDeviceData->ui32BufferCount; i++)
+	{
+		if ((psDeviceData->ui32BufferUseMask & (1UL << i)) == 0)
+		{
+			psDeviceData->ui32BufferUseMask |= (1UL << i);
+
+			psBuffer->sCpuPAddr.uiAddr = psDeviceData->sDispMemCpuPAddr.uiAddr + (i * psDeviceData->ui32BufferSize);
+
+			psDeviceData->sPVRServicesFuncs.pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap,
+										      1,
+										      &psBuffer->sDevPAddr,
+										      &psBuffer->sCpuPAddr);
+			if (psBuffer->sDevPAddr.uiAddr == psBuffer->sCpuPAddr.uiAddr)
+			{
+				/* Because the device and CPU addresses are the same we assume that we have an
+				   incompatible PDP device address. Calculate the correct PDP device address. */
+				psBuffer->sDevPAddr.uiAddr -= psDeviceData->sDispMemCpuPAddr.uiAddr;
+			}
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+
+static void BufferReleaseMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT64 uiOffset;
+
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr >= psDeviceData->sDispMemCpuPAddr.uiAddr);
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr < (psDeviceData->sDispMemCpuPAddr.uiAddr + psDeviceData->uiDispMemSize));
+
+	uiOffset = psBuffer->sCpuPAddr.uiAddr - psDeviceData->sDispMemCpuPAddr.uiAddr;
+	uiOffset = DC_OSDiv64(uiOffset, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferUseMask &= ~(1UL << uiOffset);
+
+	psBuffer->sDevPAddr.uiAddr = 0;
+	psBuffer->sCpuPAddr.uiAddr = 0;
+}
+
+static void EnableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET);
+	SET_FIELD(ui32InterruptEnable, PDP_INTENAB_INTEN_VBLNK0, 1);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET, ui32InterruptEnable);
+}
+
+static void DisableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET);
+	SET_FIELD(ui32InterruptEnable, PDP_INTENAB_INTEN_VBLNK0, 0);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET, ui32InterruptEnable);
+}
+
+static void PDPUpdateRegisters(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32Value = 0;
+	DC_ASSERT(psDeviceData != NULL);
+	/* To be set when register settings should be adopted by the PDP */
+	SET_FIELD(ui32Value, PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID, 0x01);
+	/* To indicate whether or not the PDP should adopt the updated register settings during */
+	SET_FIELD(ui32Value, PDP_REGISTER_UPDATE_CTRL_USE_VBLANK, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_REGISTER_UPDATE_CTRL_OFFSET, ui32Value); // 0x00000003
+}
+
+static void PDPResetMMU(DCPDP_DEVICE * psDeviceData)
+{
+	IMG_UINT32 ui32Value = 0;
+
+	/* Display pipeline software reset: Software reset. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_PDP_RST, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x20000000
+
+	ui32Value = 0;
+	/* Display pipeline software reset: Normal operation. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_PDP_RST, 0x00);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x00000000
+
+}
+
+static INLINE void Flip(DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 ui32Value = 0;
+	IMG_UINT32 ui32Width = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	IMG_UINT32 ui32Height = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	ui32Value = 0;
+	SET_FIELD(ui32Value, PDP_GRPH1SIZE_GRPH1WIDTH, ui32Width - 1);
+	SET_FIELD(ui32Value, PDP_GRPH1SIZE_GRPH1HEIGHT, ui32Height - 1);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1SIZE_OFFSET, ui32Value);
+
+	ui32Value = 0;
+	SET_FIELD(ui32Value, PDP_GRPH1POSN_GRPH1XSTART, 0);
+	SET_FIELD(ui32Value, PDP_GRPH1POSN_GRPH1YSTART, 0);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1POSN_OFFSET, ui32Value);
+
+	ui32Value = 0;
+	switch (psBuffer->ePixelFormat)
+	{
+		//case IMG_PIXFMT_B5G6R5_UNORM: // XXX Not supported technically we only have R5G6B5
+		//{
+		//	SET_FIELD(ui32Value, PDP_GRPH1SURF_GRPH1PIXFMT, DCPDP_PIXEL_FORMAT_RGB565);
+		//	break;
+		//}
+		//case IMG_PIXFMT_B8G8R8X8_UNORM: // XXX Not supported technically we only have RGBA8
+		case IMG_PIXFMT_B8G8R8A8_UNORM:
+		//case IMG_PIXFMT_R8G8B8A8_UNORM:
+		//case IMG_PIXFMT_R8G8B8X8_UNORM:
+		{
+			SET_FIELD(ui32Value, PDP_GRPH1SURF_GRPH1PIXFMT, DCPDP_PIXEL_FORMAT_ARGB8);
+			break;
+		}
+		default:
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Unrecognised pixel format\n");
+			DC_ASSERT(0);
+		}
+	}
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1SURF_OFFSET, ui32Value);
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1STRIDE_OFFSET,
+			PLACE_FIELD(PDP_GRPH1STRIDE_GRPH1STRIDE, (psBuffer->ui32ByteStride >> PDP_GRPH1STRIDE_DIVIDE) - 1));
+
+	DC_ASSERT(psBuffer);
+	/* Check if the address is a correct 40-bit address */
+	DC_ASSERT((psBuffer->sDevPAddr.uiAddr & 0xffffff0000000000) == 0);
+
+	/* Write 32 lsb of the address to stream address register */
+	ui32Value = 0;
+	ui32Value = DC_OSReadReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1BASEADDR_OFFSET);
+	ui32Value = (psBuffer->sDevPAddr.uiAddr & PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1BASEADDR_OFFSET, ui32Value);
+
+	/* Write 8 msb of the address to address extension bits in the PDP MMU control register */
+	ui32Value = DC_OSReadReg32(psBuffer->psDeviceData->pvPDPBifRegCpuVAddr, PDP_BIF_ADDRESS_CONTROL_OFFSET);
+	SET_FIELD(ui32Value, PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED, (psBuffer->sDevPAddr.uiAddr >> 32));
+	SET_FIELD(ui32Value, PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING, 0x00);
+	SET_FIELD(ui32Value, PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS, 0x01);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPBifRegCpuVAddr, PDP_BIF_ADDRESS_CONTROL_OFFSET, ui32Value);
+
+	PDPUpdateRegisters(psBuffer->psDeviceData);
+
+#if defined(VIRTUAL_PLATFORM)
+	/* Kick pdp sim render */
+	DC_OSDebugPrintf(DBGLVL_DEBUG, "%s:%d: Kicking PDP render\n", __func__, __LINE__);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_INTCLR_OFFSET, 0x11111111);
+#endif
+}
+
+static INLINE void FlipConfigQueueAdvanceIndex(DCPDP_FLIP_CONTEXT *psContext, IMG_UINT32 *pui32Index)
+{
+	(*pui32Index)++;
+
+	if (*pui32Index >= DCPDP_MAX_COMMANDS_INFLIGHT)
+	{
+		*pui32Index = 0;
+	}
+}
+
+static INLINE IMG_BOOL FlipConfigQueueEmptyNoLock(DCPDP_FLIP_CONTEXT *psContext)
+{
+	return (IMG_BOOL)(psContext->ui32FlipConfigInsertIndex == psContext->ui32FlipConfigRemoveIndex);
+}
+
+static IMG_BOOL FlipConfigQueueEmpty(DCPDP_FLIP_CONTEXT *psContext)
+{
+	unsigned long ulSpinlockFlags;
+	IMG_BOOL bFlipConfigQueueEmpty;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+	bFlipConfigQueueEmpty = FlipConfigQueueEmptyNoLock(psContext);
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	return bFlipConfigQueueEmpty;
+}
+
+static void FlipConfigQueueAdd(DCPDP_FLIP_CONTEXT *psContext,
+				   IMG_HANDLE hConfigData,
+				   DCPDP_BUFFER *psBuffer,
+				   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				   IMG_UINT32 ui32DisplayPeriod)
+{
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Get the next inactive flip config and update the insert index */
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigInsertIndex];
+	FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigInsertIndex);
+
+	DC_ASSERT(psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_INACTIVE);
+
+	/* Fill out the flip config */
+	psFlipConfig->psContext		= psContext;
+	psFlipConfig->hConfigData	= hConfigData;
+	psFlipConfig->psBuffer		= psBuffer;
+	psFlipConfig->ui32DisplayPeriod	= ui32DisplayPeriod;
+	psFlipConfig->pasSurfAttrib	= pasSurfAttrib;
+
+	/* Should be updated last */
+	psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_PENDING;
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+}
+
+static void SetModeRegisters(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_TIMING_DATA *psTimingData = &psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex];
+	IMG_UINT32 ui32Value;
+
+	DC_ASSERT(psDeviceData->psSystemBuffer);
+
+	PDP_CHECKPOINT;
+
+	ui32Value = 0;
+	/* Memory refreshes enabled during horizontal and vertical blanking. */
+	SET_FIELD(ui32Value, PDP_MEMCTRL_MEMREFRESH, 0x02);
+	/* Specifies number of consecutive words issued from the current base address before moving onto the next. */
+	/* Issues with NoC sending interleaved read responses to PDP require burst to be 1 */
+	SET_FIELD(ui32Value, PDP_MEMCTRL_BURSTLEN, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_MEMCTRL_OFFSET, ui32Value);
+
+	PDPResetMMU(psDeviceData);
+
+	ui32Value = 0;
+	/* XXX Test these inputs, this may be configured incorrectly */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_OPMASK_R_OFFSET , 0x00000000);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_OPMASK_GB_OFFSET, 0x00000000);
+
+	ui32Value = 0;
+	/* Power down mode: Output valid sync signals, and substitute active data with black pixels. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_POWERDN, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x10000000
+
+	/* Clear all interrupt enable bits, so that none of them are enabled for any plane */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET  , 0x00000000);
+
+	ui32Value = 0;
+	/* Power down mode: Output valid sync signals, and substitute active data with black pixels. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_POWERDN, 0x01);
+	/* Controls Blanking signal polarity: Blanking signal is active low */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_BLNKPOL, 0x01);
+	/* Controls Vertical Sync polarity: Vertical Sync is active low */
+	/* VSync polarity is active high (0), should come from hdmi driver */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_VSPOL, psTimingData->ui8VSyncPolarity);
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_HSPOL, psTimingData->ui8HSyncPolarity);
+	//SET_FIELD(ui32Value, PDP_SYNCCTRL_HSDIS, 1);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x10000028
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VSYNC1_OFFSET,
+			PLACE_FIELD(PDP_VSYNC1_VBPS, psTimingData->ui32VBackPorch) | PLACE_FIELD(PDP_VSYNC1_VT, psTimingData->ui32VTotal));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VSYNC2_OFFSET,
+			PLACE_FIELD(PDP_VSYNC2_VAS, psTimingData->ui32VActiveStart) | PLACE_FIELD(PDP_VSYNC2_VTBS, psTimingData->ui32VTopBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VSYNC3_OFFSET,
+			PLACE_FIELD(PDP_VSYNC3_VFPS, psTimingData->ui32VFrontPorch) | PLACE_FIELD(PDP_VSYNC3_VBBS, psTimingData->ui32VBottomBorder));
+
+	if (psTimingData->bReducedBlanking)
+	{
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VEVENT_OFFSET,
+			PLACE_FIELD(PDP_VEVENT_VEVENT, psTimingData->ui32VBottomBorder + DCPDP_REDUCED_BLANKING_VEVENT) |
+			PLACE_FIELD(PDP_VEVENT_VFETCH, psTimingData->ui32VBackPorch / 2));
+	}
+	else
+	{
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VEVENT_OFFSET,
+			PLACE_FIELD(PDP_VEVENT_VEVENT, 0) | PLACE_FIELD(PDP_VEVENT_VFETCH, psTimingData->ui32VBackPorch));
+	}
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VDECTRL_OFFSET,
+			PLACE_FIELD(PDP_VDECTRL_VDES, psTimingData->ui32VActiveStart) | PLACE_FIELD(PDP_VDECTRL_VDEF, psTimingData->ui32VBottomBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HSYNC1_OFFSET,
+			PLACE_FIELD(PDP_HSYNC1_HBPS, psTimingData->ui32HBackPorch) | PLACE_FIELD(PDP_HSYNC1_HT, psTimingData->ui32HTotal));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HSYNC2_OFFSET,
+			PLACE_FIELD(PDP_HSYNC2_HAS, psTimingData->ui32HActiveStart) | PLACE_FIELD(PDP_HSYNC2_HLBS, psTimingData->ui32HLeftBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HSYNC3_OFFSET,
+			PLACE_FIELD(PDP_HSYNC3_HFPS, psTimingData->ui32HFrontPorch) | PLACE_FIELD(PDP_HSYNC3_HRBS, psTimingData->ui32HRightBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HDECTRL_OFFSET,
+			PLACE_FIELD(PDP_HDECTRL_HDES, psTimingData->ui32HActiveStart) | PLACE_FIELD(PDP_HDECTRL_HDEF, psTimingData->ui32HRightBorder));
+
+	/* Enable GRPH1 */
+	DCPDPEnableMemoryRequest(psDeviceData, IMG_TRUE);
+
+	ui32Value = 0;
+	/* Set the global alpha to 1023, i.e. fully saturated */
+	SET_FIELD(ui32Value, PDP_GRPH1GALPHA_GRPH1GALPHA, 0x03FF);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1GALPHA_OFFSET, ui32Value); // 0x000003FF
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1_MEM_THRESH_OFFSET, 0x780f80f8);
+
+	ui32Value = 0;
+	/* Starts Sync Generator. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_SYNCACTIVE, 0x01);
+	/* Controls polarity of pixel clock: Pixel clock is inverted */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_CLKPOL, 0x01);
+	/* Controls Blanking signal polarity: Blanking signal is active low */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_BLNKPOL, 0x01);
+	/* Controls Vertical Sync polarity: Vertical Sync is active low */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_VSPOL, psTimingData->ui8VSyncPolarity);
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_HSPOL, psTimingData->ui8HSyncPolarity);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET, ui32Value); // 0x80000828
+}
+
+static IMG_UINT32 GetTimingIndex(DCPDP_DEVICE *psDeviceData, IMG_UINT32 width, IMG_UINT32 height)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 index = -1;
+	for (i = 0; i < psDeviceData->uiTimingDataSize; i++)
+	{
+		if (psDeviceData->pasTimingData[i].ui32HDisplay == width &&
+		    psDeviceData->pasTimingData[i].ui32VDisplay == height)
+		{
+			index = i;
+		}
+	}
+
+	return index;
+}
+
+static void FlipConfigQueueProcess(DCPDP_FLIP_CONTEXT *psContext, IMG_BOOL bFlipped, IMG_BOOL bFlush)
+{
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+	IMG_HANDLE ahRetiredConfigs[DCPDP_MAX_COMMANDS_INFLIGHT] = {0};
+	IMG_UINT32 i, ui32RetireConfigIndex = 0;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSMutexLock(psContext->pvMutex);
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+	pasSurfAttrib = psFlipConfig->pasSurfAttrib;
+
+	do
+	{
+		switch (psFlipConfig->eStatus)
+		{
+			case DCPDP_FLIP_CONFIG_PENDING:
+				if (!bFlipped)
+				{
+					Flip(psFlipConfig->psBuffer);
+
+					bFlipped = IMG_TRUE;
+				}
+
+				/* If required, set the mode after flipping */
+				if (pasSurfAttrib != NULL)
+				{
+					if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay ||
+					    pasSurfAttrib[0].sDisplay.sDims.ui32Height != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay)
+					{
+						psDeviceData->uiTimingDataIndex = GetTimingIndex(psDeviceData,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Width,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+						SetModeRegisters(psDeviceData);
+						DCPDPEnableMemoryRequest(psDeviceData, IMG_TRUE);
+					}
+				}
+
+				psFlipConfig->eStatus = DCPDP_FLIP_CONFIG_ACTIVE;
+
+				/* Now that a new flip config is active we can retire the previous one (if one exists) */
+				if (psContext->hConfigDataToRetire != NULL)
+				{
+					ahRetiredConfigs[ui32RetireConfigIndex++] = psContext->hConfigDataToRetire;
+					psContext->hConfigDataToRetire = NULL;
+				}
+
+				__fallthrough;
+			case DCPDP_FLIP_CONFIG_ACTIVE:
+				/* We can retire this config only when it's buffer has been
+				   displayed for at least the same number of frames as the
+				   display period or if we're flushing the queue */
+				if (psFlipConfig->ui32DisplayPeriod != 0)
+				{
+					psFlipConfig->ui32DisplayPeriod--;
+				}
+
+				if (psFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+				{
+					DCPDP_FLIP_CONFIG *psNextFlipConfig;
+
+					/* There should never be an outstanding config to retire at this point */
+					DC_ASSERT(psContext->hConfigDataToRetire == NULL);
+
+					/* Now that this config has been active for the minimum number of frames, i.e. the
+					   display period, we can mark the flip config queue item as inactive. However, we
+					   can only retire the config once a new config has been made active. Therefore,
+					   store the Services config data handle so that we can retire it later. */
+					psContext->hConfigDataToRetire = psFlipConfig->hConfigData;
+
+					/* Reset the flip config data */
+					psFlipConfig->hConfigData	= 0;
+					psFlipConfig->psBuffer		= NULL;
+					psFlipConfig->ui32DisplayPeriod	= 0;
+					psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_INACTIVE;
+
+					/* Move on the queue remove index */
+					FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigRemoveIndex);
+
+					psNextFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+					if (psNextFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+					{
+						psFlipConfig = psNextFlipConfig;
+						bFlipped = IMG_FALSE;
+					}
+				}
+				break;
+			case DCPDP_FLIP_CONFIG_INACTIVE:
+				if (FlipConfigQueueEmptyNoLock(psContext))
+				{
+					break;
+				}
+				DC_OSDebugPrintf(DBGLVL_ERROR, " - Tried to process an inactive config when one wasn't expected\n");
+
+				__fallthrough;
+			default:
+				DC_ASSERT(0);
+		}
+	} while (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING);
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Call into Services to retire any configs that were ready to be
+	 * retired. */
+	for (i = 0; i < ui32RetireConfigIndex; i++)
+	{
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(ahRetiredConfigs[i]);
+	}
+
+	DC_OSMutexUnlock(psContext->pvMutex);
+}
+
+static PVRSRV_ERROR VSyncProcessor(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = pvData;
+
+	psDeviceData->sPVRServicesFuncs.pfnCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FlipConfigProcessor(void *pvData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)pvData;
+
+	DC_ASSERT(psContext);
+
+	FlipConfigQueueProcess(psContext, IMG_TRUE, IMG_FALSE);
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPGetInfo(IMG_HANDLE hDeviceData, DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Copy our device name */
+	DC_OSStringNCopy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	/* Report what our minimum and maximum display period is. */
+	psDisplayInfo->ui32MinDisplayPeriod	= DCPDP_MIN_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxDisplayPeriod	= DCPDP_MAX_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxPipes		= 1;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static PVRSRV_ERROR DCPDPPanelQueryCount(IMG_HANDLE hDeviceData, IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPPanelQuery(IMG_HANDLE hDeviceData,
+				    IMG_UINT32 ui32PanelsArraySize,
+				    IMG_UINT32 *pui32NumPanels,
+				    PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	PDP_CHECKPOINT;
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	DC_OSMemSet(psPanelInfo, 0, sizeof(*psPanelInfo));
+
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = psDeviceData->ePixelFormat;
+	psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+	psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	psPanelInfo[0].ui32RefreshRate = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh;
+	psPanelInfo[0].ui32XDpi = DCPDP_DPI;
+	psPanelInfo[0].ui32YDpi = DCPDP_DPI;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPFormatQuery(IMG_HANDLE hDeviceData,
+				     IMG_UINT32 ui32NumFormats,
+				     PVRSRV_SURFACE_FORMAT *pasFormat,
+				     IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (pasFormat[i].ePixFormat == psDeviceData->ePixelFormat)
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPDimQuery(IMG_HANDLE hDeviceData,
+				  IMG_UINT32 ui32NumDims,
+				  PVRSRV_SURFACE_DIMS *psDim,
+				  IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+	IMG_UINT32 j;
+
+	PDP_CHECKPOINT;
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		for (j = 0; j < psDeviceData->uiTimingDataSize; j++)
+		{
+			if ((psDim[i].ui32Width == psDeviceData->pasTimingData[j].ui32HDisplay) &&
+			    (psDim[i].ui32Height == psDeviceData->pasTimingData[j].ui32VDisplay))
+			{
+				pui32Supported[i]++;
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetBlank(IMG_HANDLE hDeviceData,
+								  IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	IMG_UINT32 ui32Value;
+
+	PDP_CHECKPOINT;
+
+	ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET);
+
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_POWERDN, bEnabled ? 1 : 0);
+	psDeviceData->bVSyncEnabled = !bEnabled;
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET, ui32Value);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetVSyncReporting(IMG_HANDLE hDeviceData,
+										   IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	psDeviceData->bVSyncReporting = bEnabled;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPLastVSyncQuery(IMG_HANDLE hDeviceData,
+										IMG_INT64 *pi64Timestamp)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	*pi64Timestamp = psDeviceData->i64LastVSyncTimeStamp;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPContextCreate(IMG_HANDLE hDeviceData,
+				       IMG_HANDLE *hDisplayContext)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	DCPDP_FLIP_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	if (psDeviceData->psFlipContext)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Cannot create a context when one is already active\n");
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psContext = DC_OSCallocMem(sizeof(*psContext));
+	if (psContext == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Not enough memory to allocate a display context\n");
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* We don't want to process the flip config work queue in interrupt context so
+	   create a work queue which we can use to defer the work */
+	eError = DC_OSWorkQueueCreate(&psContext->hFlipConfigWorkQueue, DCPDP_MAX_COMMANDS_INFLIGHT);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Couldn't create a work queue (%s)\n",
+				 psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPDPOSFreeMemContext;
+	}
+
+	/* Create work items to place on the flip config work queue when we service a vsync interrupt */
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		eError = DC_OSWorkQueueCreateWorkItem(&psContext->asFlipConfigQueue[i].hWorkItem, FlipConfigProcessor, psContext);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create a flip config work item (%s)\n",
+					 psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+			goto ErrorDC_OSWorkQueueDestroyWorkItems;
+		}
+	}
+
+	eError = DC_OSMutexCreate(&psContext->pvMutex);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create flip config mutex (%s)\n",
+				 psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDC_OSWorkQueueDestroyWorkItems;
+	}
+
+	DC_OSSpinLockCreate(&psContext->sSpinLock);
+
+	psContext->psDeviceData		= psDeviceData;
+	psDeviceData->psFlipContext	= psContext;
+	*hDisplayContext		= (IMG_HANDLE)psContext;
+
+	return PVRSRV_OK;
+
+ErrorDC_OSWorkQueueDestroyWorkItems:
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+
+ErrorPDPOSFreeMemContext:
+	DC_OSFreeMem(psContext);
+
+	return eError;
+}
+
+static void DCPDPContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	/* Make sure there are no work items on the queue before it's destroyed */
+	DC_OSWorkQueueFlush(psContext->hFlipConfigWorkQueue);
+
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+	DC_OSMutexDestroy(psContext->pvMutex);
+	DC_OSSpinLockDestroy(&psContext->sSpinLock);
+
+	psDeviceData->psFlipContext = NULL;
+	DC_OSFreeMem(psContext);
+
+}
+
+static void DCPDPContextConfigure(IMG_HANDLE hDisplayContext,
+				  IMG_UINT32 ui32PipeCount,
+				  PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				  IMG_HANDLE *ahBuffers,
+				  IMG_UINT32 ui32DisplayPeriod,
+				  IMG_HANDLE hConfigData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_BOOL bQueueWasEmpty;
+
+	if (ui32DisplayPeriod > DCPDP_MAX_DISPLAY_PERIOD)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Requested display period of %u is larger than maximum supported display period of %u (clamping)\n",
+				 ui32DisplayPeriod, DCPDP_MAX_DISPLAY_PERIOD);
+		DC_ASSERT(ui32DisplayPeriod <= DCPDP_MAX_DISPLAY_PERIOD);
+
+		/* Clamp to the maximum so that we can continue */
+		ui32DisplayPeriod = DCPDP_MAX_DISPLAY_PERIOD;
+	}
+
+	/* Add new config to the queue */
+	if (ui32PipeCount != 0)
+	{
+		/* Check if the queue is empty before adding a new flip config */
+		bQueueWasEmpty = FlipConfigQueueEmpty(psContext);
+
+		/* We have a regular config so add it as normal */
+		FlipConfigQueueAdd(psContext, hConfigData, (DCPDP_BUFFER *)ahBuffers[0], pasSurfAttrib, ui32DisplayPeriod);
+
+		/* Check to see if vsync unlocked flipping should be done. This is determined by the
+		   display period being 0 and the flip config queue being empty before we added the
+		   new flip config. If it wasn't empty then we need to allow the remaining vsync locked
+		   flip configs to be processed before we can start processing the queue here.
+		   Also immediately flip if the current vsync interrupt is disabled.
+		   This makes sure that even when the PDP is turned off we progress. */
+		if ((ui32DisplayPeriod == 0 && bQueueWasEmpty) ||
+			 psDeviceData->bVSyncEnabled == IMG_FALSE)
+		{
+			FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_FALSE);
+		}
+	}
+	else
+	{
+		/* We have a 'NULL' config, which indicates the display context is
+		   about to be destroyed. Queue a flip back to the system buffer */
+		FlipConfigQueueAdd(psContext, hConfigData, psDeviceData->psSystemBuffer, pasSurfAttrib, 0);
+
+		/* Flush the remaining configs (this will result in the system buffer being displayed) */
+		FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_TRUE);
+
+		DC_OSMutexLock(psContext->pvMutex);
+
+		/* Immediately retire the system buffer config data */
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(hConfigData);
+		psContext->hConfigDataToRetire = NULL;
+
+		DC_OSMutexUnlock(psContext->pvMutex);
+	}
+}
+
+static PVRSRV_ERROR DCPDPContextConfigureCheck(IMG_HANDLE hDisplayContext,
+					       IMG_UINT32 ui32PipeCount,
+					       PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+					       IMG_HANDLE *ahBuffers)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_BUFFER *psBuffer;
+	IMG_UINT32 ui32NewVESATimingIndex;
+	ui32NewVESATimingIndex = GetTimingIndex(psDeviceData, pasSurfAttrib[0].sDisplay.sDims.ui32Width, pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+
+	if (ui32NewVESATimingIndex == -1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+
+	if (ui32PipeCount != 1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+	/* new crop dimensions, new mode dimensions */
+	else if (pasSurfAttrib[0].sCrop.sDims.ui32Width != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32HDisplay ||
+		 pasSurfAttrib[0].sCrop.sDims.ui32Height != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32VDisplay ||
+		 pasSurfAttrib[0].sCrop.i32XOffset != 0 ||
+		 pasSurfAttrib[0].sCrop.i32YOffset != 0)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+	}
+	/* new mode dimensions, new crop dimensions */
+	else if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != pasSurfAttrib[0].sCrop.sDims.ui32Width ||
+		 pasSurfAttrib[0].sDisplay.sDims.ui32Height != pasSurfAttrib[0].sCrop.sDims.ui32Height ||
+		 pasSurfAttrib[0].sDisplay.i32XOffset != pasSurfAttrib[0].sCrop.i32XOffset ||
+		 pasSurfAttrib[0].sDisplay.i32YOffset != pasSurfAttrib[0].sCrop.i32YOffset)
+	{
+		return PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+	}
+
+	psBuffer = ahBuffers[0];
+
+	/* new buffer dimensions, current mode dimensions */
+	if (psBuffer->ui32Width  < pasSurfAttrib[0].sDisplay.sDims.ui32Width ||
+	    psBuffer->ui32Height < pasSurfAttrib[0].sDisplay.sDims.ui32Height)
+	{
+		return PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferSystemAcquire(IMG_HANDLE hDeviceData,
+					     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+					     IMG_UINT32 *pui32PageCount,
+					     IMG_UINT32 *pui32ByteStride,
+					     IMG_HANDLE *phSystemBuffer)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	PDP_CHECKPOINT;
+
+	OSAtomicIncrement(&psDeviceData->psSystemBuffer->i32RefCount);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psDeviceData->psSystemBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psDeviceData->psSystemBuffer->ui32ByteStride;
+	*phSystemBuffer		= psDeviceData->psSystemBuffer;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferSystemRelease(IMG_HANDLE hSystemBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hSystemBuffer;
+
+	PDP_CHECKPOINT;
+
+	OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static PVRSRV_ERROR DCPDPBufferAlloc(IMG_HANDLE hDisplayContext,
+				     DC_BUFFER_CREATE_INFO *psCreateInfo,
+				     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+				     IMG_UINT32 *pui32PageCount,
+				     IMG_UINT32 *pui32ByteStride,
+				     IMG_HANDLE *phBuffer)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	DCPDP_BUFFER *psBuffer;
+
+	PDP_CHECKPOINT;
+
+	/* XXX: This need to change because we support more formats */
+	DC_ASSERT(psCreateInfo->ui32BPP == DCPDP_PIXEL_FORMAT_BPP);
+
+	if (psSurfInfo->sFormat.eMemLayout != PVRSRV_SURFACE_MEMLAYOUT_STRIDED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (psSurfInfo->sFormat.ePixFormat)
+	{
+		case DCPDP_PIXEL_FORMAT: // hard coded one supported format
+			break;
+		default:
+			return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	if (psSurfInfo->sFormat.ePixFormat != psDeviceData->ePixelFormat)
+	{
+		return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psSurfInfo->sFormat.ePixFormat;
+	psBuffer->ui32Width			= psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height		= psSurfInfo->sDims.ui32Height;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * psCreateInfo->ui32BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32)DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height, DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psBuffer->ui32ByteStride;
+	*phBuffer		= psBuffer;
+
+	PDP_DEBUG_PRINT("- %s: Acquired buffer at address %lx\n", __func__, psBuffer->sCpuPAddr.uiAddr);
+
+#if defined(PDP_DEBUG)
+	dllist_init(&psBuffer->sDbgListNode);
+	dllist_add_to_tail(&psDeviceData->sDbgBufListHead, &psBuffer->sDbgListNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferAcquire(IMG_HANDLE hBuffer, IMG_DEV_PHYADDR *pasDevPAddr, void **ppvLinAddr)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+	DCPDP_DEVICE *psDeviceData = psBuffer->psDeviceData;
+	IMG_CPU_PHYADDR sCpuPAddr = psBuffer->sCpuPAddr;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	for (i = 0; i < psBuffer->ui32SizeInPages; i++)
+	{
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap, 1,
+									      &pasDevPAddr[i], &sCpuPAddr);
+		sCpuPAddr.uiAddr += DC_OSGetPageSize();
+	}
+
+	OSAtomicIncrement(&psBuffer->i32RefCount);
+
+	*ppvLinAddr = NULL;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferRelease(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+	OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static void DCPDPBufferFree(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+	PDP_CHECKPOINT;
+
+#if defined(DEBUG)
+	if (OSAtomicRead(&psBuffer->i32RefCount) != 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING, " - Freeing buffer that still has %d references\n", OSAtomicRead(&psBuffer->i32RefCount));
+	}
+#endif
+
+	BufferReleaseMemory(psBuffer->psDeviceData, psBuffer);
+
+#if defined(PDP_DEBUG)
+	dllist_remove_node(&psBuffer->sDbgListNode);
+#endif
+
+	DC_OSFreeMem(psBuffer);
+}
+
+static DC_DEVICE_FUNCTIONS g_sDCFunctions =
+{
+	DCPDPGetInfo,				/* pfnGetInfo */
+	DCPDPPanelQueryCount,		/* pfnPanelQueryCount */
+	DCPDPPanelQuery,			/* pfnPanelQuery */
+	DCPDPFormatQuery,			/* pfnFormatQuery */
+	DCPDPDimQuery,				/* pfnDimQuery */
+	DCPDPSetBlank,				/* pfnSetBlank */
+	DCPDPSetVSyncReporting,		/* pfnSetVSyncReporting */
+	DCPDPLastVSyncQuery,		/* pfnLastVSyncQuery */
+	DCPDPContextCreate,			/* pfnContextCreate */
+	DCPDPContextDestroy,		/* pfnContextDestroy */
+	DCPDPContextConfigure,		/* pfnContextConfigure */
+	DCPDPContextConfigureCheck,	/* pfnContextConfigureCheck */
+	DCPDPBufferAlloc,			/* pfnBufferAlloc */
+	DCPDPBufferAcquire,			/* pfnBufferAcquire */
+	DCPDPBufferRelease,			/* pfnBufferRelease */
+	DCPDPBufferFree,			/* pfnBufferFree */
+	NULL,					/* pfnBufferImport */
+	NULL,					/* pfnBufferMap */
+	NULL,					/* pfnBufferUnmap */
+	DCPDPBufferSystemAcquire,	/* pfnBufferSystemAcquire */
+	DCPDPBufferSystemRelease,	/* pfnBufferSystemRelease */
+};
+
+static IMG_BOOL DisplayLISRHandler(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)pvData;
+	IMG_UINT32 ui32InterruptStatus;
+	PVRSRV_ERROR eError;
+
+	ui32InterruptStatus = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTSTAT_OFFSET);
+	if (ui32InterruptStatus & PDP_INTSTAT_INTS_VBLNK0_MASK)
+	{
+		/* If a previous register update has completed, clear the status bit */
+		if (DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_REGISTER_UPDATE_STATUS_OFFSET) & PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_MASK)
+		{
+			DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_REGISTER_UPDATE_CTRL_OFFSET, 0);
+		}
+
+		/* Get the timestamp for this interrupt. */
+		psDeviceData->i64LastVSyncTimeStamp = DC_OSClockns();
+		if (psDeviceData->bVSyncReporting)
+		{
+			eError = DC_OSWorkQueueAddWorkItem(psDeviceData->hVSyncWorkQueue, psDeviceData->hVSyncWorkItem);
+			if (eError != PVRSRV_OK)
+			{
+				DC_OSDebugPrintf(DBGLVL_WARNING,
+						 " - %s: Couldn't queue work item (%s)\n",
+						 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+
+			}
+		}
+
+		if (psDeviceData->psFlipContext)
+		{
+			DCPDP_FLIP_CONTEXT *psContext = psDeviceData->psFlipContext;
+			DCPDP_FLIP_CONFIG *psFlipConfig;
+
+			DC_OSSpinLock(&psContext->sSpinLock);
+
+			psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+
+			if (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING)
+			{
+				Flip(psFlipConfig->psBuffer);
+			}
+
+			if (psFlipConfig->eStatus != DCPDP_FLIP_CONFIG_INACTIVE)
+			{
+				eError = DC_OSWorkQueueAddWorkItem(psContext->hFlipConfigWorkQueue, psFlipConfig->hWorkItem);
+				if (eError != PVRSRV_OK)
+				{
+					DC_OSDebugPrintf(DBGLVL_WARNING,
+							 " - %s: Couldn't queue work item (%s)\n",
+							 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+				}
+			}
+
+			DC_OSSpinUnlock(&psContext->sSpinLock);
+		}
+		/* Clear the vsync status bit to show that the vsync interrupt has been handled */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTCLR_OFFSET, (0x1UL << PDP_INTSTAT_INTS_VBLNK0_SHIFT));
+		/* For some reason, first interrupt always contains HBLNK, so clear all */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTCLR_OFFSET, 0xFFFFFFFF);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static void DeInitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+	{
+		if (OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) != 0)
+		{
+			printk(KERN_INFO "%s: References to psSystemBuffer still exist"
+		   " (%d)", __func__, OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount));
+		}
+		OSAtomicWrite(&psDeviceData->psSystemBuffer->i32RefCount, 0);
+	}
+	else
+#else
+	{
+		DC_ASSERT(OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) == 0);
+	}
+#endif
+
+	PDP_CHECKPOINT;
+
+	if (psDeviceData->psSystemBuffer != NULL)
+	{
+		BufferReleaseMemory(psDeviceData, psDeviceData->psSystemBuffer);
+
+		DC_OSFreeMem(psDeviceData->psSystemBuffer);
+		psDeviceData->psSystemBuffer = NULL;
+	}
+}
+
+static PVRSRV_ERROR InitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_BUFFER *psBuffer;
+	IMG_CPU_VIRTADDR pvBufferCpuVAddr;
+	IMG_UINT32 *pui32Pixel;
+	IMG_UINT8 ui8Red, ui8Green[4], ui8Blue;
+	IMG_UINT32 i, j;
+	IMG_UINT64 ui64BufferCount;
+
+	/* On a reconfigure, system buffer will not be NULL */
+	if (psDeviceData->psSystemBuffer != NULL)
+	{
+		DeInitSystemBuffer(psDeviceData);
+		psDeviceData->psSystemBuffer = NULL;
+	}
+
+	/* Setup simple buffer allocator */
+	psDeviceData->ui32BufferSize = (IMG_UINT32)DC_ALIGN(psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay *
+							    psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay *
+							    DCPDP_PIXEL_FORMAT_BPP,
+							    DC_OSGetPageSize());
+
+	ui64BufferCount = psDeviceData->uiDispMemSize;
+	ui64BufferCount = DC_OSDiv64(ui64BufferCount, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferCount = (IMG_UINT32)ui64BufferCount;
+	if (psDeviceData->ui32BufferCount == 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Not enough space for the framebuffer\n", __func__);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	else if (psDeviceData->ui32BufferCount > DCPDP_MAX_BUFFERS)
+	{
+		psDeviceData->ui32BufferCount = DCPDP_MAX_BUFFERS;
+	}
+
+	psDeviceData->ui32BufferUseMask = 0;
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psDeviceData->ePixelFormat;
+	psBuffer->ui32Width			= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psBuffer->ui32Height		= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * DCPDP_PIXEL_FORMAT_BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	PDP_DEBUG_PRINT("- %s: System buffer width: %d, height: %d, starting addr: %lx\n", __func__, psBuffer->ui32Width, psBuffer->ui32Height, psBuffer->sCpuPAddr.uiAddr);
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32)DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height, DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	/* Initialise the system buffer to a nice rainbow. */
+	pvBufferCpuVAddr = DC_OSMapPhysAddr(psBuffer->sCpuPAddr, psBuffer->ui32SizeInBytes);
+	if (pvBufferCpuVAddr != NULL)
+	{
+		pui32Pixel = (IMG_UINT32 *)pvBufferCpuVAddr;
+
+#if (DCPDP_PIXEL_FORMAT_BPP == 4)
+		for (i = 0; i < psBuffer->ui32Height; ++i)
+		{
+			for (j = 0; j < psBuffer->ui32Width; ++j)
+			{
+				IMG_UINT32 index = i * psBuffer->ui32Width + j;
+				ui8Red   = 0xFF - (((i + 1) * 0xFF) / psBuffer->ui32Height);
+				ui8Green[0] = ((j + 1) * 0xFF) / psBuffer->ui32Width;
+				ui8Blue  = ((i + 1) * 0xFF) / psBuffer->ui32Height;
+				pui32Pixel[index] = 0xFF000000 | (ui8Red << 16) | (ui8Green[0] << 8) | (ui8Blue << 0);
+			}
+		}
+#endif
+
+#if (DCPDP_PIXEL_FORMAT_BPP == 3)
+		/* Optimisation for faster upload on emu, i varies over the height, j the width, and k the pixel buffer */
+		/* Relies on the width being a factor of 4 */
+		for (i = 0; i < psBuffer->ui32Height; ++i)
+		{
+			for (j = k = 0; j < psBuffer->ui32Width; j += 4, k += 3)
+			{
+				index = i * ((psBuffer->ui32Width * 3) / 4) + k;
+				ui8Red      = 0xFF - (((i + 1) * 0xFF) / psBuffer->ui32Height);
+				ui8Green[0] = ((j + 1) * 0xFF) / psBuffer->ui32Width;
+				ui8Green[1] = ((j + 2) * 0xFF) / psBuffer->ui32Width;
+				ui8Green[2] = ((j + 3) * 0xFF) / psBuffer->ui32Width;
+				ui8Green[3] = ((j + 4) * 0xFF) / psBuffer->ui32Width;
+				ui8Blue     = ((i + 1) * 0xFF) / psBuffer->ui32Height;
+				pui32Pixel[index + 0] = (ui8Blue << 24)     | (ui8Red << 16)      | (ui8Green[0] << 8) | (ui8Blue);
+				pui32Pixel[index + 1] = (ui8Green[1] << 24) | (ui8Blue << 16)     | (ui8Red << 8)      | (ui8Green[2]);
+				pui32Pixel[index + 2] = (ui8Red << 24)      | (ui8Green[3] << 16) | (ui8Blue << 8)     | (ui8Red);
+			}
+		}
+#endif
+		DC_OSUnmapPhysAddr(pvBufferCpuVAddr, psBuffer->ui32SizeInBytes);
+	}
+
+	psDeviceData->psSystemBuffer = psBuffer;
+
+	return PVRSRV_OK;
+}
+
+
+static PVRSRV_ERROR InitTimingData(DCPDP_DEVICE * pvDevice)
+{
+	const DCPDP_MODULE_PARAMETERS * pvModuleParameters;
+	DCPDP_TIMING_DATA * psTimingData;
+
+	pvDevice->uiTimingDataSize = 1;
+	pvDevice->pasTimingData = DC_OSAllocMem(sizeof(DCPDP_TIMING_DATA) * pvDevice->uiTimingDataSize);
+	if (pvDevice->pasTimingData	== NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	psTimingData = &pvDevice->pasTimingData[pvDevice->uiTimingDataIndex];
+
+	pvModuleParameters = DCPDPGetModuleParameters();
+
+	psTimingData->ui32HDisplay      = pvModuleParameters->ui32PDPWidth;
+	psTimingData->ui32HTotal        = psTimingData->ui32HDisplay;
+	psTimingData->ui32HBackPorch    = 0;
+	psTimingData->ui32HFrontPorch   = psTimingData->ui32HTotal;
+	psTimingData->ui32HLeftBorder   = psTimingData->ui32HBackPorch;
+	psTimingData->ui32HActiveStart  = psTimingData->ui32HLeftBorder;
+	psTimingData->ui32HRightBorder  = psTimingData->ui32HActiveStart + psTimingData->ui32HDisplay;
+	psTimingData->ui8HSyncPolarity  = 0;
+
+	psTimingData->ui32VDisplay      = pvModuleParameters->ui32PDPHeight;
+	psTimingData->ui32VTotal        = psTimingData->ui32VDisplay;
+	psTimingData->ui32VBackPorch    = 0;
+	psTimingData->ui32VFrontPorch   = psTimingData->ui32VTotal;
+	psTimingData->ui32VTopBorder    = psTimingData->ui32VBackPorch;
+	psTimingData->ui32VActiveStart  = psTimingData->ui32VTopBorder;
+	psTimingData->ui32VBottomBorder = psTimingData->ui32VActiveStart + psTimingData->ui32VDisplay;
+	psTimingData->ui8VSyncPolarity  = 0;
+	psTimingData->ui32VRefresh   = 60;
+
+	PDP_DEBUG_PRINT("- %s: HDisplay: %d, HActive: %d, HLeftBorder: %d, HTotal: %d, HFrontPorch: %d, HBackPorch: %d, HRightBorder: %d\n", __func__, psTimingData->ui32HDisplay, psTimingData->ui32HActiveStart,
+	psTimingData->ui32HLeftBorder, psTimingData->ui32HTotal, psTimingData->ui32HFrontPorch, psTimingData->ui32HBackPorch, psTimingData->ui32HRightBorder);
+
+	PDP_DEBUG_PRINT("- %s: VDisplay: %d, VActive: %d, VTopBorder: %d, VTotal: %d, VFrontPorch: %d, VBackPorch: %d, VBottomBorder: %d\n", __func__, psTimingData->ui32VDisplay, psTimingData->ui32VActiveStart,
+	psTimingData->ui32VTopBorder, psTimingData->ui32VTotal, psTimingData->ui32VFrontPorch, psTimingData->ui32VBackPorch, psTimingData->ui32VBottomBorder);
+
+	return PVRSRV_OK;
+}
+
+static void UpdateTimingData(DCPDP_DEVICE * pvDevice)
+{
+	DCPDP_TIMING_DATA * psTimingData = &pvDevice->pasTimingData[pvDevice->uiTimingDataIndex];
+	DTD * dtd;
+
+	if (pvDevice->videoParams.mDtdActiveIndex > HDMI_DTD_MAX)
+	{
+		PDP_ERROR_PRINT(" - %s: DTD Active index is greater than max (%d), resetting to 0\n", __func__, HDMI_DTD_MAX);
+		pvDevice->videoParams.mDtdActiveIndex = 0;
+	}
+
+	dtd = &pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex];
+
+	dtd->mHBackPorchWidth = dtd->mHFrontPorchWidth;
+	dtd->mVBackPorchWidth = dtd->mVFrontPorchWidth;
+
+	psTimingData->ui32HDisplay      = dtd->mHActive;
+	psTimingData->ui32HTotal        = dtd->mHActive + dtd->mHBlanking + 2 * dtd->mHBorder;
+	psTimingData->ui32HBackPorch    = dtd->mHSyncPulseWidth;
+	psTimingData->ui32HFrontPorch   = psTimingData->ui32HTotal - dtd->mHFrontPorchWidth;
+	psTimingData->ui32HLeftBorder   = psTimingData->ui32HBackPorch + dtd->mHBackPorchWidth;
+	psTimingData->ui32HActiveStart  = psTimingData->ui32HLeftBorder + dtd->mHBorder;
+	psTimingData->ui32HRightBorder  = psTimingData->ui32HActiveStart + psTimingData->ui32HDisplay;
+	psTimingData->ui8HSyncPolarity  = !dtd->mHSyncPolarity; // For PDP, 0 means active high
+
+	psTimingData->ui32VDisplay      = dtd->mVActive;
+	psTimingData->ui32VTotal        = dtd->mVActive + dtd->mVBlanking + 2 * dtd->mVBorder;
+	psTimingData->ui32VBackPorch    = dtd->mVSyncPulseWidth;
+	psTimingData->ui32VFrontPorch   = psTimingData->ui32VTotal - dtd->mVFrontPorchWidth;
+	psTimingData->ui32VTopBorder    = psTimingData->ui32VBackPorch + dtd->mVBackPorchWidth; //dtd->mVBlanking - dtd->mVFrontPorchWidth;
+	psTimingData->ui32VActiveStart  = psTimingData->ui32VTopBorder + dtd->mVBorder;
+	psTimingData->ui32VBottomBorder = psTimingData->ui32VActiveStart + psTimingData->ui32VDisplay;
+	psTimingData->ui8VSyncPolarity  = !dtd->mVSyncPolarity; // For PDP, 0 means active high
+
+	if (dtd->mHSyncPolarity == 1 && dtd->mVSyncPolarity == 0)
+	{
+		psTimingData->bReducedBlanking = IMG_TRUE;
+	}
+
+	PDP_DEBUG_PRINT("- %s: HDisplay: %d, HActive: %d, HLeftBorder: %d, HTotal: %d, HFrontPorch: %d, HBackPorch: %d, HRightBorder: %d\n", __func__, psTimingData->ui32HDisplay, psTimingData->ui32HActiveStart,
+	psTimingData->ui32HLeftBorder, psTimingData->ui32HTotal, psTimingData->ui32HFrontPorch, psTimingData->ui32HBackPorch, psTimingData->ui32HRightBorder);
+
+	PDP_DEBUG_PRINT("- %s: VDisplay: %d, VActive: %d, VTopBorder: %d, VTotal: %d, VFrontPorch: %d, VBackPorch: %d, VBottomBorder: %d\n", __func__, psTimingData->ui32VDisplay, psTimingData->ui32VActiveStart,
+	psTimingData->ui32VTopBorder, psTimingData->ui32VTotal, psTimingData->ui32VFrontPorch, psTimingData->ui32VBackPorch, psTimingData->ui32VBottomBorder);
+
+	psTimingData->ui32VRefresh   = 60;
+
+}
+
+PVRSRV_ERROR DCPDPInit(void *pvDevice, DCPDP_DEVICE **ppsDeviceData, IMG_UINT32 ui32Instance)
+{
+	DCPDP_DEVICE *psDeviceData;
+	PVRSRV_ERROR eError;
+	PPVRSRV_DEVICE_NODE psDevNode;
+#if defined(PDP_DEBUG)
+	static IMG_BOOL bFirstTime = IMG_TRUE;
+#endif
+
+	if (ppsDeviceData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DC_OSSetDrvName(DRVNAME);
+	PDP_CHECKPOINT;
+
+	psDeviceData = DC_OSCallocMem(sizeof(*psDeviceData));
+	if (psDeviceData == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to allocate device data\n", __func__);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psDeviceData->pvDevice = pvDevice;
+	psDeviceData->ePixelFormat = DCPDP_PIXEL_FORMAT;
+#if defined(PDP_DEBUG)
+	/* Initialise our debug hooks for tracking each device and the buffers
+	 * allocated for that device. Device tracking is hooked from our local
+	 * gsPdpDeviceListHead and the buffers are attached to the per-device
+	 * sDbgBufListHead DLLIST. These are used by PDPDebugCtrl().
+	 */
+	dllist_init(&psDeviceData->sDbgListNode);
+	dllist_init(&psDeviceData->sDbgBufListHead);
+	/* Initialise the tracking list if this is our first call. We assume that
+	 * we are called from a single-threaded __init() routine from the OS
+	 * driver layer so we do not need to implement mutex locking to handle
+	 * multiple simultaneous calls. */
+	if (bFirstTime)
+	{
+		dllist_init(&gsPdpDeviceListHead);
+		bFirstTime = IMG_FALSE;
+	}
+	dllist_add_to_tail(&gsPdpDeviceListHead, &psDeviceData->sDbgListNode);
+	gNumPdpDevices++;
+#endif
+
+	eError = DC_OSPVRServicesConnectionOpen(&psDeviceData->hPVRServicesConnection);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to open connection to PVR Services (%d)\n",
+				 __func__, eError);
+		goto ErrorFreeDeviceData;
+	}
+
+	eError = DC_OSPVRServicesSetupFuncs(psDeviceData->hPVRServicesConnection, &psDeviceData->sPVRServicesFuncs);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to setup PVR Services function table (%d)\n",
+				 __func__, eError);
+		goto ErrorPDPOSPVRServicesConnectionClose;
+	}
+
+	psDevNode = psDeviceData->sPVRServicesFuncs.pfnGetDeviceInstance(ui32Instance);
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, ErrorPDPOSPVRServicesConnectionClose);
+
+
+	/* Services manages the card memory so we need to acquire the display controller
+	   heap (which is a carve out of the card memory) so we can allocate our own buffers. */
+	eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapAcquireByUsage(PHYS_HEAP_USAGE_DISPLAY,
+								    psDevNode,
+								    &psDeviceData->psPhysHeap);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to acquire heap (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPDPOSPVRServicesConnectionClose;
+	}
+
+	/* The PDP can only access local memory so make sure the heap is of the correct type */
+	if (psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetType(psDeviceData->psPhysHeap) != PHYS_HEAP_TYPE_LMA)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Heap is of the wrong type\n", __func__);
+		eError = PVRSRV_ERROR_INVALID_HEAP;
+		goto ErrorPhysHeapRelease;
+	}
+
+	/* Get the CPU base address for the display heap */
+	eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetCpuPAddr(psDeviceData->psPhysHeap,
+								       &psDeviceData->sDispMemCpuPAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get display memory base address (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetSize(psDeviceData->psPhysHeap,
+								    &psDeviceData->uiDispMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get the display memory size (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	DC_OSDebugPrintf(DBGLVL_DEBUG, " - %s: Display memory size (0x%016lX)\n",
+			 __func__, psDeviceData->uiDispMemSize);
+
+	/* Reserve and map the PDP registers */
+	psDeviceData->sPDPRegCpuPAddr.uiAddr =
+		DC_OSAddrRangeStart(psDeviceData->pvDevice, DCPDP2_REG_PCI_BASENUM) +
+		DCPDP2_PCI_PDP_REG_OFFSET;
+
+	if (DC_OSRequestAddrRegion(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE, DRVNAME) == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to reserve the PDP registers\n", __func__);
+
+		psDeviceData->sPDPRegCpuPAddr.uiAddr = 0;
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+
+		goto ErrorPhysHeapRelease;
+	}
+
+	psDeviceData->pvPDPRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE);
+	if (psDeviceData->pvPDPRegCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to map PDP registers\n", __func__);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+		goto ErrorReleaseMemRegion;
+	}
+
+	{
+		/*
+		 * 1GB Plato boards have issue with PDP initialisation. When PDP fails to
+		 * properly initialize, PDP registers are inaccessible.
+		 *
+		 * We can detect this by reading a register that has a known
+		 * and unchanging value.
+		 */
+		#define PLATO_PDP_CORE_ID (0x07010003)
+
+		IMG_UINT32 ui32CoreId;
+
+		ui32CoreId =
+			DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_CORE_ID_OFFSET);
+
+		if (ui32CoreId == PLATO_PDP_CORE_ID)
+		{
+
+			DC_OSDebugPrintf(DBGLVL_INFO, "%s: PDP initialized! (CoreId = 0x%x)",
+					__func__, ui32CoreId);
+		}
+		else
+		{
+			DC_OSDebugPrintf(DBGLVL_INFO, "%s: PDP failed to initialize! (CoreId = 0x%x)"
+					, __func__, ui32CoreId);
+
+			goto ErrorPDPOSUnmapPDPRegisters;
+		}
+	}
+
+	psDeviceData->pvPDPBifRegCpuVAddr = psDeviceData->pvPDPRegCpuVAddr + DCPDP2_BIF_REGS_OFFSET;
+
+	PDP_DEBUG_PRINT(" - %s: Register base CpuPAddr: %lx\n", __func__, psDeviceData->sPDPRegCpuPAddr.uiAddr);
+	PDP_DEBUG_PRINT(" - %s: Register base CpuVAddr: %p\n", __func__, psDeviceData->pvPDPRegCpuVAddr);
+
+	/* Initialize timing data to defaults */
+	eError = InitTimingData(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialize timing data\n", __func__);
+		goto ErrorPDPOSUnmapPDPRegisters;
+	}
+
+	/* Create a system buffer */
+	eError = InitSystemBuffer(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialise the system buffer (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorFreeTimingData;
+	}
+
+	eError = psDeviceData->sPVRServicesFuncs.pfnDCRegisterDevice(&g_sDCFunctions,
+								     DCPDP_MAX_COMMANDS_INFLIGHT,
+								     psDeviceData,
+								     &psDeviceData->hPVRServicesDevice);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to register the display device (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDeinitSysBuffer;
+	}
+
+	/* Register our handler */
+	eError = psDeviceData->sPVRServicesFuncs.pfnSysInstallDeviceLISR(pvDevice,
+									 PLATO_IRQ_PDP,
+									 DRVNAME,
+									 DisplayLISRHandler,
+									 psDeviceData,
+									 &psDeviceData->hLISRData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to install the display device interrupt handler (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDCUnregisterDevice;
+	}
+
+	eError = DC_OSWorkQueueCreate(&psDeviceData->hVSyncWorkQueue, 1);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work queue (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDCUninstallLISR;
+	}
+
+	eError = DC_OSWorkQueueCreateWorkItem(&psDeviceData->hVSyncWorkItem, VSyncProcessor, psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work item (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDCDestroyWorkQueue;
+	}
+
+	*ppsDeviceData = psDeviceData;
+
+	return PVRSRV_OK;
+
+ErrorDCDestroyWorkQueue:
+	DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+ErrorDCUninstallLISR:
+	psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+
+ErrorDCUnregisterDevice:
+	psDeviceData->sPVRServicesFuncs.pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+
+ErrorDeinitSysBuffer:
+	DeInitSystemBuffer(psDeviceData);
+
+ErrorFreeTimingData:
+	DC_OSFreeMem(psDeviceData->pasTimingData);
+
+ErrorPDPOSUnmapPDPRegisters:
+	DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+ErrorReleaseMemRegion:
+	DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+ErrorPhysHeapRelease:
+	psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+ErrorPDPOSPVRServicesConnectionClose:
+	DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+
+ErrorFreeDeviceData:
+	DC_OSFreeMem(psDeviceData);
+
+	return eError;
+
+}
+
+PVRSRV_ERROR DCPDPStart(DCPDP_DEVICE *psDeviceData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	UpdateTimingData(psDeviceData);
+
+	/* Create a system buffer */
+	eError = InitSystemBuffer(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialise the system buffer (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		return eError;
+	}
+
+	SetModeRegisters(psDeviceData);
+	psDeviceData->bVSyncEnabled = IMG_TRUE;
+
+	Flip(psDeviceData->psSystemBuffer);
+
+	EnableVSyncInterrupt(psDeviceData);
+
+	/* Print some useful information */
+	DC_OSDebugPrintf(DBGLVL_INFO, " using mode %ux%u@%uHz (phys: %ux%u)\n",
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh,
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay * 1000) / DCPDP_DPI * 254) / 10000),
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay * 1000) / DCPDP_DPI * 254) / 10000));
+	DC_OSDebugPrintf(DBGLVL_INFO, " register base: 0x%llxK\n", psDeviceData->sPDPRegCpuPAddr.uiAddr);
+	DC_OSDebugPrintf(DBGLVL_INFO, " memory base: 0x%llxK\n", psDeviceData->sDispMemCpuPAddr.uiAddr);
+
+	return PVRSRV_OK;
+
+}
+
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, void **ppvDevice)
+{
+	PDP_CHECKPOINT;
+
+	if (psDeviceData)
+	{
+		DC_OSWorkQueueDestroyWorkItem(psDeviceData->hVSyncWorkItem);
+		DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+		DisableVSyncInterrupt(psDeviceData);
+
+		psDeviceData->sPVRServicesFuncs.pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+		{
+			psDeviceData->psFlipContext = NULL;
+		}
+		else
+#else
+		{
+			DC_ASSERT(psDeviceData->psFlipContext == NULL);
+		}
+#endif
+		(void)psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+
+		DeInitSystemBuffer(psDeviceData);
+
+		/* Disable stream 1 to ensure PDP2 doesn't try to access on card memory */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1CTRL_OFFSET, PLACE_FIELD(PDP_GRPH1CTRL_GRPH1STREN, 0));
+
+		DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+		DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+		DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+
+		if (ppvDevice != NULL)
+		{
+			*ppvDevice = psDeviceData->pvDevice;
+		}
+
+		PDP_CHECKPOINT;
+
+		DC_OSFreeMem(psDeviceData->pasTimingData);
+#if defined(PDP_DEBUG)
+		/* Remove this device from the list of known PDP devices */
+		dllist_remove_node(&psDeviceData->sDbgListNode);
+		gNumPdpDevices--;
+#endif
+		DC_OSFreeMem(psDeviceData);
+	}
+}
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable)
+{
+	if (psDeviceData)
+	{
+		IMG_UINT32 ui32Value;
+		ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1CTRL_OFFSET);
+		SET_FIELD(ui32Value, PDP_GRPH1CTRL_GRPH1STREN, bEnable ? 1 : 0);
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1CTRL_OFFSET, ui32Value);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..12a9dfb5780b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Kbuild.mk
@@ -0,0 +1,55 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-$(CONFIG_X86) += -mno-soft-float
+
+ccflags-y += \
+ -I$(TOP)/services/display/include \
+ -I$(TOP)/services/display/dc_pdp2/include \
+ -I$(TOP)/services/shared/include \
+ -I$(TOP)/services/system/$(PVR_SYSTEM) \
+ -I$(TOP)/services/system/$(PVR_ARCH)/$(PVR_SYSTEM) \
+ -I$(TOP)/include/public \
+ -I$(TOP)/include/$(PVR_ARCH)/public
+
+dc_pdp2-y += \
+ services/display/dc_pdp2/common/dc_pdp_common.o \
+ services/display/dc_pdp2/env/linux/pdp_osfuncs.o \
+ services/display/common/env/linux/dc_osfuncs.o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Linux.mk
new file mode 100644
index 000000000000..4ff6a1dcd739
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := dc_pdp2
+
+dc_pdp2_type := kernel_module
+dc_pdp2_target := dc_pdp2.ko
+dc_pdp2_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/pdp_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/pdp_osfuncs.c
new file mode 100644
index 000000000000..5a3d1ab5931f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/pdp_osfuncs.c
@@ -0,0 +1,620 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#include "dc_osfuncs.h"
+#include "dc_pdp.h"
+
+#if defined(ENABLE_PLATO_HDMI)
+/* APIs needed from plato_hdmi */
+EXPORT_SYMBOL(PDPInitialize);
+#endif
+
+#define DCPDP_WIDTH_MIN			(640)
+#define DCPDP_WIDTH_MAX			(1920)
+#define DCPDP_HEIGHT_MIN		(480)
+#define DCPDP_HEIGHT_MAX		(1080)
+#define DCPDP_WIDTH_DEFAULT		(640)
+#define DCPDP_HEIGHT_DEFAULT	(480)
+
+#define DCPDP_DEBUGFS_DISPLAY_ENABLED   "display_enabled"
+#define DCPDP_DEBUGFS_NUM_DEVICES       "num_devices"
+
+#if defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_WIDTH defined but DCPDP_HEIGHT not defined
+#elif !defined(DCPDP_WIDTH) && defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_HEIGHT defined but DCPDP_WIDTH not defined
+#elif !defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#define DCPDP_WIDTH			DCPDP_WIDTH_DEFAULT
+#define DCPDP_HEIGHT		DCPDP_HEIGHT_DEFAULT
+#elif (DCPDP_WIDTH > DCPDP_WIDTH_MAX)
+#error ERROR: DCPDP_WIDTH too large (max: 1920)
+#elif (DCPDP_WIDTH < DCPDP_WIDTH_MIN)
+#error ERROR: DCPDP_WIDTH too small (min: 640)
+#elif (DCPDP_HEIGHT > DCPDP_HEIGHT_MAX)
+#error ERROR: DCPDP_HEIGHT too large (max: 1080)
+#elif (DCPDP_HEIGHT < DCPDP_HEIGHT_MIN)
+#error ERROR: DCPDP_HEIGHT too small (max: 480)
+#endif
+
+static DLLIST_NODE g_sDeviceDataListHead;    /* List head for devices supported
+                                                by this driver. */
+static IMG_UINT32 g_uiNumPDPDevs;            /* Number of devices found. */
+
+static struct dentry    *g_psDebugFSEntryDir; /* Top-level 'dc_pdp2' entry */
+static struct dentry    *g_psNumDevicesEntry; /* 'num_devices' entry */
+static struct dentry    *g_psDispEnabledEntry; /* 'display_enabled' entry */
+
+/* PDP module parameters */
+DCPDP_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32PDPEnabled = 1,
+	.ui32PDPWidth   = DCPDP_WIDTH,
+	.ui32PDPHeight  = DCPDP_HEIGHT
+};
+module_param_named(mem_en, sModuleParams.ui32PDPEnabled, uint, S_IRUGO | S_IWUSR);
+module_param_named(width,  sModuleParams.ui32PDPWidth,   uint, S_IRUGO | S_IWUSR);
+module_param_named(height, sModuleParams.ui32PDPHeight,  uint, S_IRUGO | S_IWUSR);
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void)
+{
+	return &sModuleParams;
+}
+
+static int DisplayEnabledOpen(struct inode *psINode, struct file *psFile)
+{
+	psFile->private_data = psINode->i_private;
+
+	return 0;
+}
+
+static ssize_t NumDevicesRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	char aszBuffer[] = "XX\n";
+	loff_t uiPosition = *puiPosition;
+	size_t uiBufferSize = ARRAY_SIZE(aszBuffer);
+	size_t uiLocCount;
+
+	PVR_UNREFERENCED_PARAMETER(psFile);
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	(void) DC_OSSNPrintf(aszBuffer, sizeof(aszBuffer), "%u", g_uiNumPDPDevs);
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+	if (copy_to_user(psUserBuffer, &aszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static ssize_t DisplayEnabledRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)psFile->private_data;
+	IMG_BOOL bPDPEnabled;
+	loff_t uiPosition = *puiPosition;
+	char pszBuffer[] = "N\n";
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	/* Handle the top-level 'meta' entry which reports the values of all of
+	 * the underlying unit(s) using logical AND of all values.
+	 */
+	if (psDeviceData == NULL)
+	{
+		/* Iterate across all present units and construct the top-level value.
+		 * IMG_TRUE => all devices are enabled.
+		 * IMG_FALSE => at least one device is disabled.
+		 */
+		DLLIST_NODE *psNode, *psNext;
+
+		bPDPEnabled = IMG_TRUE;
+		dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+		{
+			psDeviceData = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+
+			bPDPEnabled &= psDeviceData->bPDPEnabled;
+		}
+	}
+	else
+	{
+		bPDPEnabled = psDeviceData->bPDPEnabled;
+	}
+
+	if (bPDPEnabled == IMG_TRUE)
+	{
+		pszBuffer[0] = 'Y';
+	}
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+	if (copy_to_user(psUserBuffer, &pszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static ssize_t DisplayEnabledWrite(struct file *psFile,
+				   const char __user *psUserBuffer,
+				   size_t uiCount,
+				   loff_t *puiPosition)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)(psFile->private_data);
+	char pszBuffer[3];
+	bool bPDPEnabled;
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	uiLocCount = min(uiCount, (uiBufferSize - 1U));
+
+	if (copy_from_user(pszBuffer, psUserBuffer, uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	pszBuffer[uiLocCount] = '\0';
+
+	if (strtobool(pszBuffer, &bPDPEnabled) == 0)
+	{
+		/* If we have no device-specific data we iterate over all units */
+		if (psDeviceData == NULL)
+		{
+			DLLIST_NODE *psNode, *psNext;
+			dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+			{
+				psDeviceData = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+				psDeviceData->bPDPEnabled = bPDPEnabled ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDP_DEBUG)
+				printk(KERN_INFO DRVNAME " - %s: Setting display %u %s\n", __func__,
+				       psDeviceData->ui32Instance, bPDPEnabled ? "Enabled" : "Disabled");
+#endif
+
+				DCPDPEnableMemoryRequest(psDeviceData, psDeviceData->bPDPEnabled);
+			}
+		}
+		else
+		{
+			psDeviceData->bPDPEnabled = bPDPEnabled ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDP_DEBUG)
+			printk(KERN_INFO DRVNAME " - %s: Setting display %u %s\n", __func__,
+			       psDeviceData->ui32Instance, bPDPEnabled ? "Enabled" : "Disabled");
+#endif
+
+			DCPDPEnableMemoryRequest(psDeviceData, psDeviceData->bPDPEnabled);
+		}
+	}
+
+	return (ssize_t)uiLocCount;
+}
+
+#if defined(PDP_DEBUG)
+static long DisplayEnabledIoctl(struct file *file, unsigned int ioctl, unsigned long argp)
+{
+	PDPDebugCtrl();
+	return 0;
+}
+#endif /* PDP_DEBUG */
+
+static const struct file_operations gsDisplayEnabledFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = DisplayEnabledOpen,
+	.read = DisplayEnabledRead,
+	.write = DisplayEnabledWrite,
+	.llseek = default_llseek,
+#if defined(PDP_DEBUG)
+	.unlocked_ioctl = DisplayEnabledIoctl
+#endif /* PDP_DEBUG */
+};
+
+static const struct file_operations gsNumDevicesFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = DisplayEnabledOpen,
+	.read = NumDevicesRead,
+	.llseek = default_llseek
+};
+
+PVRSRV_ERROR PDPInitialize(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+
+	/* Iterate over all devices and initialise them. May need to be changed to
+	 * support different video params per device ... TBD
+	 */
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		DCPDP_DEVICE *psDeviceData = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+		if (psDeviceData->ui32Instance == uiInstance)
+		{
+			(void) printk(KERN_INFO DRVNAME " - %s: Matched Device %u\n",
+			              __func__, psDeviceData->ui32Instance);
+
+			if (psDeviceData->bPreInitDone == IMG_FALSE)
+			{
+				(void) printk(KERN_ERR DRVNAME
+				              "-%s: Called before PreInit is complete\n",
+				              __func__);
+				return -ENODEV;
+			}
+
+			psDeviceData->videoParams = *pVideoParams;
+
+			eError = DCPDPStart(psDeviceData);
+			if (eError != PVRSRV_OK)
+			{
+				(void) printk(KERN_ERR DRVNAME
+				              " - %s: Failed to initialise device (%d)\n",
+				              __func__, eError);
+				return -ENODEV;
+			}
+
+			/* Enable / Disable display according to default modparam value */
+			if (sModuleParams.ui32PDPEnabled == 1)
+			{
+				if (!psDeviceData->bPDPEnabled)
+				{
+#if defined(PDP_DEBUG)
+					printk(KERN_WARNING DRVNAME " - %s: Enabling display %u\n",
+					       __func__, psDeviceData->ui32Instance);
+#endif
+					psDeviceData->bPDPEnabled = IMG_TRUE;
+					DCPDPEnableMemoryRequest(psDeviceData, IMG_TRUE);
+				}
+			}
+			else
+			{
+				if (psDeviceData->bPDPEnabled)
+				{
+#if defined(PDP_DEBUG)
+					printk(KERN_WARNING DRVNAME " - %s: Disabling display %u\n",
+					       __func__, psDeviceData->ui32Instance);
+#endif
+					psDeviceData->bPDPEnabled = IMG_FALSE;
+					DCPDPEnableMemoryRequest(psDeviceData, IMG_FALSE);
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+static struct pci_dev *dc_pdp_getdevice(struct pci_dev *fromDev)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(DCPDP2_VENDOR_ID_PLATO,
+	                          DCPDP2_DEVICE_ID_PCI_PLATO,
+	                          fromDev);
+
+	return psPCIDev;
+}
+
+static int __init dc_pdp_init(void)
+{
+	DCPDP_DEVICE *psDevicePriv;
+	struct pci_dev *psPCIDev, *fromDev = NULL;
+	PVRSRV_ERROR eError;
+	int error;
+	IMG_CHAR aszDevName[4];	/* Unit number of device "0" .. "N" */
+
+	/* Iterate over all PCI instances */
+	dllist_init(&g_sDeviceDataListHead);
+	while ((psPCIDev = dc_pdp_getdevice(fromDev)) != NULL)
+	{
+		fromDev = psPCIDev;
+
+		error = pci_enable_device(psPCIDev);
+		if (error != 0)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to enable PCI device (%d)\n", __func__, error);
+			continue;  /* Go to next potential instance. Drop reference */
+		}
+
+		eError = DCPDPInit(&psPCIDev->dev, &psDevicePriv, g_uiNumPDPDevs);
+		if (eError != PVRSRV_OK)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to initialise device (%d)\n", __func__, eError);
+			continue;  /* Go to next potential instance. Drop reference */
+		}
+
+		dllist_init(&psDevicePriv->sListNode);
+
+		/* To prevent possible problems with system suspend/resume, we don't
+		   keep the device enabled, but rely on the fact that the Rogue driver
+		   will have done a pci_enable_device. */
+		pci_disable_device(psPCIDev);
+
+		/* We need to create a debugFS top-level entry for the dc_pdp2 device
+		 * once we've found the first PCI device. This is used as the root
+		 * debugFS directory for device-specific entries and holds the number
+		 * of devices found entry in 'num_devices'. Only do this on the first
+		 * iteration of the discovery loop.
+		 * Note: The creation of debugFS style entries may be disabled due to
+		 *       OS configuration. If so, we will continue with the driver
+		 *       being initialised but there will not be a way of observing
+		 *       the values provided by the debugFS entries.
+		 */
+		if (g_uiNumPDPDevs == 0U)
+		{
+			/* Top-level directory entry. All devices have a sub-directory */
+			g_psDebugFSEntryDir = debugfs_create_dir(DRVNAME, NULL);
+			if (IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME
+				       " - %s: Failed to create '%s' debugfs root directory "
+				       "(debugfs entries won't be available)\n", __func__,
+				       DRVNAME);
+				g_psDebugFSEntryDir = NULL;
+			}
+			else
+			{
+				/* Create the 'num_devices' top-level file. This will report the
+				 * current value of g_uiNumPDPDevs. Varies from 1 .. N.
+				 */
+				g_psNumDevicesEntry = debugfs_create_file(DCPDP_DEBUGFS_NUM_DEVICES,
+				                                          S_IFREG | S_IRUGO,
+				                                          g_psDebugFSEntryDir,
+				                                          NULL,	/* Top device */
+				                                          &gsNumDevicesFileOps);
+
+				if (IS_ERR_OR_NULL(g_psNumDevicesEntry))
+				{
+					printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s'"
+					       " debugfs file "
+					       "(debugfs entries won't be available)\n", __func__,
+					       DCPDP_DEBUGFS_NUM_DEVICES);
+					g_psNumDevicesEntry = NULL;
+				}
+			}
+
+			/* Create top-level 'display_enabled' file. This reports the
+			 * combined status of all device display_enabled settings and sets
+			 * all devices to the same status.
+			 */
+			g_psDispEnabledEntry = debugfs_create_file(DCPDP_DEBUGFS_DISPLAY_ENABLED,
+			                                              S_IFREG | S_IRUGO | S_IWUSR,
+			                                              g_psDebugFSEntryDir,
+			                                              NULL,	/* Top device */
+			                                              &gsDisplayEnabledFileOps);
+			if (IS_ERR_OR_NULL(g_psDispEnabledEntry))
+			{
+				printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s'"
+				       " debugfs file "
+				       "(debugfs entries won't be available)\n", __func__,
+				       DCPDP_DEBUGFS_DISPLAY_ENABLED);
+				g_psDispEnabledEntry = NULL;
+			}
+		}
+
+		/* Create device-specific directory name. This is the unit / instance
+		 * number of this device (i.e., the current value of g_uiNumPDPDevs as
+		 * this is incremented for each device we enumerate). */
+		if (DC_OSSNPrintf(aszDevName, sizeof(aszDevName), "%u", g_uiNumPDPDevs) >= sizeof(aszDevName))
+		{
+			printk(KERN_WARNING DRVNAME " - %s: DC_OSSNPrintf of '%u' failed"
+			       " Returned '%s'\n",
+			       __func__, g_uiNumPDPDevs, aszDevName);
+		};
+
+		if (!IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+		{
+			psDevicePriv->psDebugFSEntryDir = debugfs_create_dir(aszDevName, g_psDebugFSEntryDir);
+			if (IS_ERR_OR_NULL(psDevicePriv->psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME
+				       " - %s: Failed to create '%s/%s' debugfs device directory "
+				       "(debugfs entries won't be available)\n", __func__, DRVNAME,
+				       aszDevName);
+				psDevicePriv->psDebugFSEntryDir = NULL;
+			}
+			else
+			{
+
+				psDevicePriv->psDisplayEnabledEntry = debugfs_create_file(DCPDP_DEBUGFS_DISPLAY_ENABLED,
+				                                                          S_IFREG | S_IRUGO | S_IWUSR,
+				                                                          psDevicePriv->psDebugFSEntryDir,
+				                                                          psDevicePriv,	/* This device */
+				                                                          &gsDisplayEnabledFileOps);
+				if (IS_ERR_OR_NULL(psDevicePriv->psDisplayEnabledEntry))
+				{
+					printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs entry\n",
+					         __func__, DCPDP_DEBUGFS_DISPLAY_ENABLED);
+					         psDevicePriv->psDisplayEnabledEntry = NULL;
+				}
+			}
+		}
+
+#if defined(PDP_DEBUG)
+		printk(KERN_INFO DRVNAME " - %s: Initialised instance %u\n",
+		       __func__, g_uiNumPDPDevs);
+#endif
+
+		psDevicePriv->ui32Instance = g_uiNumPDPDevs++;
+		psDevicePriv->bPreInitDone = IMG_TRUE;
+		/* Add to the list of known devices */
+		dllist_add_to_tail(&g_sDeviceDataListHead, &psDevicePriv->sListNode);
+	}
+
+#if defined(PDP_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: Found %u devices\n", __func__, g_uiNumPDPDevs);
+#endif
+	return (g_uiNumPDPDevs > 0) ? 0 : -ENODEV;
+}
+
+static void __exit dc_pdp_deinit(void)
+{
+	DCPDP_DEVICE *psDevicePriv;
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		struct pci_dev *psPCIDev;
+		psDevicePriv = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+
+		if (psDevicePriv->psDebugFSEntryDir)
+		{
+			if (psDevicePriv->psDisplayEnabledEntry)
+			{
+				debugfs_remove(psDevicePriv->psDisplayEnabledEntry);
+				psDevicePriv->psDisplayEnabledEntry = NULL;
+			}
+
+			debugfs_remove(psDevicePriv->psDebugFSEntryDir);
+			psDevicePriv->psDebugFSEntryDir = NULL;
+		}
+
+#if defined(PDP_DEBUG)
+		printk(KERN_WARNING DRVNAME " - %s: DeInit'ing instance %u\n",
+		       __func__, psDevicePriv->ui32Instance);
+#endif
+		dllist_remove_node(&psDevicePriv->sListNode);
+		DCPDPDeInit(psDevicePriv, (void *)&psPCIDev);
+
+		g_uiNumPDPDevs--;
+	}
+
+	/* Now clean up the top-level debugfs entries */
+	if (g_psNumDevicesEntry)
+	{
+		debugfs_remove(g_psNumDevicesEntry);
+		g_psNumDevicesEntry = NULL;
+	}
+
+	if (g_psDispEnabledEntry)
+	{
+		debugfs_remove(g_psDispEnabledEntry);
+		g_psDispEnabledEntry = NULL;
+	}
+
+	if (g_psDebugFSEntryDir)
+	{
+		debugfs_remove(g_psDebugFSEntryDir);
+		g_psDebugFSEntryDir = NULL;
+	}
+
+	if (!dllist_is_empty(&g_sDeviceDataListHead))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Still have devices initialised\n",
+		       __func__);
+	}
+}
+
+#if defined(PLATO_DISPLAY_PDUMP)
+
+void PDP_OSWriteHWReg32(void *pvLinRegBaseAddr, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	OSWriteHWReg32(pvLinRegBaseAddr, ui32Offset, ui32Value);
+	if (pvLinRegBaseAddr == g_psDeviceData->pvPDPRegCpuVAddr)
+	{
+		plato_pdump_reg32(pvLinRegBaseAddr, ui32Offset, ui32Value, DRVNAME);
+	}
+	else if (pvLinRegBaseAddr == g_psDeviceData->pvPDPBifRegCpuVAddr)
+	{
+		plato_pdump_reg32(pvLinRegBaseAddr, ui32Offset, ui32Value, "pdp_bif");
+	}
+}
+#endif
+
+module_init(dc_pdp_init);
+module_exit(dc_pdp_deinit);
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/include/dc_pdp.h b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/include/dc_pdp.h
new file mode 100644
index 000000000000..aee6b96ca92a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/include/dc_pdp.h
@@ -0,0 +1,322 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_PDP_H)
+#define DC_PDP_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "powervr/imgpixfmts.h"
+#include "pvrsrv.h"
+#include "dc_osfuncs.h"
+#include "plato.h"
+
+#if defined(PLATO_DISPLAY_PDUMP)
+#include "plato_pdump.h"
+#endif
+
+#define DRVNAME	"dc_pdp2"
+
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+#define DCPDP2_VENDOR_ID_PLATO			(0x1AEE)
+#define DCPDP2_DEVICE_ID_PCI_PLATO		(0x0003)
+
+/*************************************************************************/ /*!
+ PCI Device Base Address Information
+*/ /**************************************************************************/
+
+/* PDP registers on base address register 1 */
+#define DCPDP2_REG_PCI_BASENUM			(1)
+
+#define DCPDP2_PCI_PDP_REG_OFFSET		(0x200000)
+#define DCPDP2_PCI_PDP_REG_SIZE			(0x20000)
+
+#define DCPDP2_BIF_REGS_OFFSET			(0x1000)
+/*************************************************************************/ /*!
+ dc_pdp OS functions
+*/ /**************************************************************************/
+
+typedef struct DCPDP_MODULE_PARAMETERS_TAG
+{
+	IMG_UINT32  ui32PDPEnabled;
+	IMG_UINT32  ui32PDPWidth;
+	IMG_UINT32  ui32PDPHeight;
+} DCPDP_MODULE_PARAMETERS;
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void);
+
+/*************************************************************************/ /*!
+ dc_pdp Timing definitions
+*/ /**************************************************************************/
+#define DCPDP_MAX_COMMANDS_INFLIGHT	(2)
+
+/* This has to be less than or equal to the size of the type
+   used for ui32BufferUseMask in the DCPDP_DEVICE structure */
+#define DCPDP_MAX_BUFFERS		(32)
+
+#define DCPDP_MIN_DISPLAY_PERIOD	(0)
+#define DCPDP_MAX_DISPLAY_PERIOD	(2)
+
+#define DCPDP_PIXEL_FORMAT		IMG_PIXFMT_B8G8R8A8_UNORM
+#define DCPDP_PIXEL_FORMAT_BPP		(4)
+
+// Graphics/video pixel format:
+// 00000b 8-bit indexed (greyscale if LUT turned off) [RGB]
+// 00100b 4-bit alpha + 12-bit rgb444 [ARGB]
+// 00101b 1-bit alpha + 15-bit rgb555 [ARGB]
+// 00110b 24-bit rgb888 [RGB]
+// 00111b 16-bit rgb565 [RGB]
+// 01000b 8-bit alpha + 24-bit rgb888 [RGBA]
+// 10000b 8-bit alpha + 24-bit yuv888 [AYUV]
+// 10101b 30-bit YUV 4:4:4 YUV101010 [YUV]
+// 10110b 32-bit RGBX [RGBA]
+#define DCPDP_PIXEL_FORMAT_G		(0x00)
+#define DCPDP_PIXEL_FORMAT_ARGB4	(0x04)
+#define DCPDP_PIXEL_FORMAT_ARGB1555	(0x05)
+#define DCPDP_PIXEL_FORMAT_RGB8		(0x06)
+#define DCPDP_PIXEL_FORMAT_RGB565	(0x07)
+#define DCPDP_PIXEL_FORMAT_ARGB8	(0x08)
+#define DCPDP_PIXEL_FORMAT_AYUV8	(0x10)
+#define DCPDP_PIXEL_FORMAT_YUV10	(0x15)
+#define DCPDP_PIXEL_FORMAT_RGBA8	(0x16)
+
+/* Divide by 32, even though spec says to divide by 16 */
+#define PDP_GRPH1STRIDE_DIVIDE 5
+
+#ifndef DCPDP_DPI
+#define DCPDP_DPI			(160)
+#endif
+
+/* For reduced blanking displays, VEVENT needs to be right after VBBS */
+#define DCPDP_REDUCED_BLANKING_VEVENT	(1)
+
+/*************************************************************************/ /*!
+ dc_pdp Data structures
+*/ /**************************************************************************/
+typedef struct DCPDP_BUFFER_TAG DCPDP_BUFFER;
+typedef struct DCPDP_FLIP_CONFIG_TAG DCPDP_FLIP_CONFIG;
+typedef struct DCPDP_FLIP_CONTEXT_TAG DCPDP_FLIP_CONTEXT;
+typedef struct DCPDP_DEVICE_TAG DCPDP_DEVICE;
+
+typedef struct DCPDP_TIMING_DATA_TAG
+{
+	IMG_UINT32		ui32HDisplay;
+	IMG_UINT32		ui32HBackPorch;
+	IMG_UINT32		ui32HTotal;
+	IMG_UINT32		ui32HActiveStart;
+	IMG_UINT32		ui32HLeftBorder;
+	IMG_UINT32		ui32HRightBorder;
+	IMG_UINT32		ui32HFrontPorch;
+
+	IMG_UINT32		ui32VDisplay;
+	IMG_UINT32		ui32VBackPorch;
+	IMG_UINT32		ui32VTotal;
+	IMG_UINT32		ui32VActiveStart;
+	IMG_UINT32		ui32VTopBorder;
+	IMG_UINT32		ui32VBottomBorder;
+	IMG_UINT32		ui32VFrontPorch;
+
+	IMG_UINT8		ui8VSyncPolarity;
+	IMG_UINT8		ui8HSyncPolarity;
+
+	IMG_UINT32		ui32VRefresh;
+	IMG_UINT32		ui32ClockFreq;
+
+	IMG_BOOL		bReducedBlanking;
+} DCPDP_TIMING_DATA;
+
+struct DCPDP_BUFFER_TAG
+{
+	ATOMIC_T		i32RefCount;
+
+	DCPDP_DEVICE		*psDeviceData;
+
+	IMG_UINT32		ui32Width;
+	IMG_UINT32		ui32Height;
+	IMG_UINT32		ui32ByteStride;
+	IMG_UINT32		ui32SizeInBytes;
+	IMG_UINT32		ui32SizeInPages;
+	IMG_PIXFMT		ePixelFormat;
+
+	IMG_CPU_PHYADDR		sCpuPAddr;
+	IMG_DEV_PHYADDR		sDevPAddr;
+
+#if defined(PDP_DEBUG)
+	DLLIST_NODE		sDbgListNode;		/* Linked list of buffers per-device */
+#endif
+};
+
+typedef enum DCPDP_FLIP_CONFIG_STATUS_TAG
+{
+	DCPDP_FLIP_CONFIG_INACTIVE = 0,
+	DCPDP_FLIP_CONFIG_PENDING,
+	DCPDP_FLIP_CONFIG_ACTIVE,
+} DCPDP_FLIP_CONFIG_STATUS;
+
+
+/* Flip config structure used for queuing of flips */
+struct DCPDP_FLIP_CONFIG_TAG
+{
+	/* Context with which this config is associated */
+	DCPDP_FLIP_CONTEXT	*psContext;
+
+	/* Services config data */
+	IMG_HANDLE		hConfigData;
+
+	/* Current status of the config */
+	DCPDP_FLIP_CONFIG_STATUS eStatus;
+
+	/* Buffer to be displayed when the config is made active */
+	DCPDP_BUFFER		*psBuffer;
+
+	/* Number of frames for which this config should remain active */
+	IMG_UINT32		ui32DisplayPeriod;
+
+	/* New mode to possibly switch to after flip */
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+
+	/* Handle to a work item that processes the flip config queue */
+	IMG_HANDLE		hWorkItem;
+};
+
+struct DCPDP_FLIP_CONTEXT_TAG
+{
+	/* Device on which the context has been created */
+	DCPDP_DEVICE		*psDeviceData;
+
+	/* Lock protecting the queue of flip configs */
+	DC_SPINLOCK		sSpinLock;
+
+	/* Mutex to serialise calls back into Services */
+	void			*pvMutex;
+
+	/* Queue of configs that need processing */
+	DCPDP_FLIP_CONFIG	asFlipConfigQueue[DCPDP_MAX_COMMANDS_INFLIGHT];
+	IMG_UINT32		ui32FlipConfigInsertIndex;
+	IMG_UINT32		ui32FlipConfigRemoveIndex;
+
+	/* Handle to a work queue used to defer the processing of the
+	   flip config queue when a vsync interrupt is serviced */
+	IMG_HANDLE		hFlipConfigWorkQueue;
+
+	/* Services config data to retire */
+	IMG_HANDLE		hConfigDataToRetire;
+};
+
+struct DCPDP_DEVICE_TAG
+{
+	IMG_HANDLE              hPVRServicesConnection;
+	IMG_HANDLE              hPVRServicesDevice;
+	DC_SERVICES_FUNCS       sPVRServicesFuncs;
+
+	void                    *pvDevice;
+
+	IMG_HANDLE              hLISRData;
+
+	PHYS_HEAP               *psPhysHeap;
+
+	IMG_CPU_PHYADDR         sPDPRegCpuPAddr;
+	IMG_CPU_VIRTADDR        pvPDPRegCpuVAddr;
+
+	IMG_CPU_VIRTADDR        pvPDPBifRegCpuVAddr;
+
+	IMG_CPU_PHYADDR         sDispMemCpuPAddr;
+	IMG_UINT64              uiDispMemSize;
+
+	IMG_UINT32              uiTimingDataIndex;
+	DCPDP_TIMING_DATA       *pasTimingData;
+	IMG_UINT32              uiTimingDataSize;
+	IMG_PIXFMT              ePixelFormat;
+
+	IMG_UINT32              ui32BufferSize;
+	IMG_UINT32              ui32BufferCount;
+	IMG_UINT32              ui32BufferUseMask;
+
+	DCPDP_BUFFER            *psSystemBuffer;
+
+	DCPDP_FLIP_CONTEXT      *psFlipContext;
+
+	IMG_BOOL                bVSyncEnabled;
+	IMG_BOOL                bVSyncReporting;
+	IMG_INT64               i64LastVSyncTimeStamp;
+	IMG_HANDLE              hVSyncWorkQueue;
+	IMG_HANDLE              hVSyncWorkItem;
+
+	IMG_BOOL                bPreInitDone;
+
+#if defined(ENABLE_PLATO_HDMI)
+	VIDEO_PARAMS            videoParams;
+#endif
+#if defined(PDP_DEBUG)
+	DLLIST_NODE             sDbgListNode;            /* Linked list of debug devices
+	                                                    used by common layer */
+	DLLIST_NODE             sDbgBufListHead;         /* Head of linked buffers
+	                                                    for this device */
+#endif
+	DLLIST_NODE             sListNode;               /* Linked list of devices
+	                                                    used by OS driver layer */
+	IMG_UINT32              ui32Instance;            /* Device instance # */
+	struct dentry           *psDebugFSEntryDir;      /* Device-specific debugFS directory */
+	struct dentry           *psDisplayEnabledEntry;  /* 'display_enabled' debugFSentry for device */
+	IMG_BOOL                bPDPEnabled;             /* State of 'display_enabled' setting */
+};
+
+/*******************************************************************************
+ * dc_pdp common functions
+ ******************************************************************************/
+
+PVRSRV_ERROR DCPDPInit(void *pvDevice, DCPDP_DEVICE **ppsDeviceData, IMG_UINT32 ui32Instance);
+PVRSRV_ERROR DCPDPStart(DCPDP_DEVICE *psDeviceData);
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, void **ppvDevice);
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable);
+#if defined(PDP_DEBUG)
+void PDPDebugCtrl(void);
+#endif /* PDP_DEBUG */
+
+#if defined(PLATO_DISPLAY_PDUMP)
+void PDP_OSWriteHWReg32(void *pvLinRegBaseAddr, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+#endif
+
+#endif /* !defined(DC_PDP_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/include/dc_osfuncs.h b/drivers/gpu/drm/img/img-volcanic/services/display/include/dc_osfuncs.h
new file mode 100644
index 000000000000..ce41da1dce38
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/include/dc_osfuncs.h
@@ -0,0 +1,258 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_OSFUNCS_H)
+#define DC_OSFUNCS_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "physheap.h"
+#include "kerneldisplay.h"
+#include "pvrsrv.h"
+#include "syscommon.h"
+
+#if defined(DEBUG)
+#define DC_ASSERT(EXPR)							\
+	do								\
+	{								\
+		if (!(EXPR))						\
+		{							\
+			DC_OSAbort(__FILE__, __LINE__);			\
+		}							\
+	} while (0)
+#else
+#define DC_ASSERT(EXPR) (void)(EXPR) /* Null implementation of ASSERT (does nothing) */
+#endif
+
+#define DC_ALIGN(value, alignment) (((value) + ((alignment) - 1)) & ~((alignment) - 1))
+
+
+/******************************************************************************
+ * DC OS functions
+ *****************************************************************************/
+
+/* Services display class function pointers */
+typedef PVRSRV_ERROR (*PFN_DC_REGISTER_DEVICE)(DC_DEVICE_FUNCTIONS *psFuncTable, IMG_UINT32 ui32MaxConfigsInFlight, IMG_HANDLE hDeviceData, IMG_HANDLE *phSrvHandle);
+typedef void (*PFN_DC_UNREGISTER_DEVICE)(IMG_HANDLE hSrvHandle);
+typedef void (*PFN_DC_DISPLAY_CONFIGURATION_RETIRED)(IMG_HANDLE hConfigData);
+typedef PVRSRV_ERROR (*PFN_DC_IMPORT_BUFFER_ACQUIRE)(IMG_HANDLE hImport, IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize, IMG_UINT32 *pui32PageCount, IMG_DEV_PHYADDR **ppasDevPAddr);
+typedef void (*PFN_DC_IMPORT_BUFFER_RELEASE)(IMG_HANDLE hImport, IMG_DEV_PHYADDR *pasDevPAddr);
+
+/* Services physical heap function pointers */
+typedef PVRSRV_ERROR (*PFN_PHYS_HEAP_ACQUIRE_BY_USAGE)(PHYS_HEAP_USAGE_FLAGS ui32UsageFlag, PPVRSRV_DEVICE_NODE psDevNode, PHYS_HEAP **ppsPhysHeap);
+typedef void (*PFN_PHYS_HEAP_RELEASE)(PHYS_HEAP *psPhysHeap);
+typedef PHYS_HEAP_TYPE (*PFN_PHYS_HEAP_GET_TYPE)(PHYS_HEAP *psPhysHeap);
+typedef PVRSRV_ERROR (*PFN_PHYS_HEAP_GET_SIZE)(PHYS_HEAP *psPhysHeap, IMG_UINT64 *puiSize);
+typedef PVRSRV_ERROR (*PFN_PHYS_HEAP_GET_ADDRESS)(PHYS_HEAP *psPhysHeap, IMG_CPU_PHYADDR *psCpuPAddr);
+typedef void (*PFN_PHYS_HEAP_CPU_PADDR_TO_DEV_PADDR)(PHYS_HEAP *psPhysHeap, IMG_UINT32 ui32NumOfAddr, IMG_DEV_PHYADDR *psDevPAddr, IMG_CPU_PHYADDR *psCpuPAddr);
+
+/* Services system functions */
+typedef PVRSRV_ERROR (*PFN_SYS_INSTALL_DEVICE_LISR)(void *pvOSDevice, IMG_UINT32 ui32IRQ, const IMG_CHAR *pszName, PFN_LISR pfnLISR, void *pvData, IMG_HANDLE *phLISRData);
+typedef PVRSRV_ERROR (*PFN_SYS_UNINSTALL_DEVICE_LISR)(IMG_HANDLE hLISRData);
+
+/* Other service related functions */
+typedef void (*PFN_CHECK_STATUS)(void *hCmdCompCallerHandle);
+typedef const IMG_CHAR *(*PFN_GET_ERROR_STRING)(PVRSRV_ERROR eError);
+typedef PVRSRV_DEVICE_NODE *(*PFN_GET_DEVICE_INST)(IMG_UINT32 ui32Instance);
+
+#define DC_OS_BYTES_TO_PAGES(range)	(((range) + (DC_OSGetPageSize() - 1)) >> DC_OSGetPageShift())
+
+typedef struct DC_SERVICES_FUNCS_TAG
+{
+	/* Display class functions */
+	PFN_DC_REGISTER_DEVICE			pfnDCRegisterDevice;
+	PFN_DC_UNREGISTER_DEVICE		pfnDCUnregisterDevice;
+	PFN_DC_DISPLAY_CONFIGURATION_RETIRED	pfnDCDisplayConfigurationRetired;
+	PFN_DC_IMPORT_BUFFER_ACQUIRE		pfnDCImportBufferAcquire;
+	PFN_DC_IMPORT_BUFFER_RELEASE		pfnDCImportBufferRelease;
+
+	/* Physical heap functions */
+	PFN_PHYS_HEAP_ACQUIRE_BY_USAGE	pfnPhysHeapAcquireByUsage;
+	PFN_PHYS_HEAP_RELEASE			pfnPhysHeapRelease;
+	PFN_PHYS_HEAP_GET_TYPE			pfnPhysHeapGetType;
+	PFN_PHYS_HEAP_GET_SIZE			pfnPhysHeapGetSize;
+	PFN_PHYS_HEAP_GET_ADDRESS		pfnPhysHeapGetCpuPAddr;
+	PFN_PHYS_HEAP_CPU_PADDR_TO_DEV_PADDR	pfnPhysHeapCpuPAddrToDevPAddr;
+
+	/* System functions */
+	PFN_SYS_INSTALL_DEVICE_LISR		pfnSysInstallDeviceLISR;
+	PFN_SYS_UNINSTALL_DEVICE_LISR		pfnSysUninstallDeviceLISR;
+
+	/* Other service related functions */
+	PFN_CHECK_STATUS			pfnCheckStatus;
+	PFN_GET_ERROR_STRING			pfnGetErrorString;
+	PFN_GET_DEVICE_INST				pfnGetDeviceInstance;
+
+} DC_SERVICES_FUNCS;
+
+typedef enum
+{
+	DBGLVL_FATAL = 0,
+	DBGLVL_ALERT,
+	DBGLVL_ERROR,
+	DBGLVL_WARNING,
+	DBGLVL_NOTICE,
+	DBGLVL_INFO,
+	DBGLVL_DEBUG,
+} DC_OS_DEBUG_LEVEL;
+
+/* Logging & other misc stuff */
+void DC_OSSetDrvName(const IMG_CHAR *pszDrvName);
+void DC_OSAbort(const IMG_CHAR *pszFile, IMG_UINT32 ui32Line);
+
+void DC_OSDebugPrintf(DC_OS_DEBUG_LEVEL eDebugLevel, const IMG_CHAR *pszFormat, ...);
+
+IMG_CHAR *DC_OSStringNCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uiLength);
+IMG_INT DC_OSSNPrintf(IMG_CHAR *pszDest, size_t uiLength, const IMG_CHAR *pszFormat, ...);
+
+IMG_INT64 DC_OSClockns(void);
+
+/* Memory management */
+IMG_UINT32 DC_OSGetPageSize(void);
+IMG_UINT32 DC_OSGetPageShift(void);
+IMG_UINT32 DC_OSGetPageMask(void);
+
+void *DC_OSAllocMem(size_t uiSize);
+void *DC_OSCallocMem(size_t uiSize);
+void DC_OSFreeMem(void *pvMem);
+void DC_OSMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t uiSize);
+
+IMG_UINT32 DC_OSAddrRangeStart(void *pvDevice, IMG_UINT8 ui8BaseNum);
+void *DC_OSRequestAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size, IMG_CHAR *pszRequestorName);
+void DC_OSReleaseAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size);
+
+IMG_CPU_VIRTADDR DC_OSMapPhysAddr(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size);
+void DC_OSUnmapPhysAddr(IMG_CPU_VIRTADDR pvCpuVAddr, IMG_UINT32 ui32Size);
+
+/* Register access */
+IMG_UINT32 DC_OSReadReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset);
+void DC_OSWriteReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+/* Floating-point support */
+IMG_UINT32 DC_OSDiv64(IMG_UINT64 ui64Numerator, IMG_UINT32 ui32Denominator);
+void DC_OSFloatingPointBegin(void);
+void DC_OSFloatingPointEnd(void);
+
+/* Inter-thread synchronisation */
+PVRSRV_ERROR DC_OSMutexCreate(void **ppvMutex);
+void DC_OSMutexDestroy(void *pvMutex);
+void DC_OSMutexLock(void *pvMutex);
+void DC_OSMutexUnlock(void *pvMutex);
+
+/* Locks that can be held in interrupt handlers */
+#if defined(__linux__)
+#include <linux/spinlock.h>
+typedef spinlock_t DC_SPINLOCK;
+# define DC_OSSpinLockCreate(psLock) spin_lock_init(psLock)
+# define DC_OSSpinLockDestroy(psLock) /* no need to destroy a spinlock */
+/* used outside hardirqs */
+# define DC_OSSpinLockIRQSave(psLock, ulIRQFlags) spin_lock_irqsave(psLock, ulIRQFlags)
+# define DC_OSSpinUnlockIRQRestore(psLock, ulIRQFlags) spin_unlock_irqrestore(psLock, ulIRQFlags)
+/* used inside hardirqs */
+# define DC_OSSpinLock(psLock) spin_lock(psLock)
+# define DC_OSSpinUnlock(psLock) spin_unlock(psLock)
+
+#elif defined(__QNXNTO__)
+#include <pthread.h>
+typedef pthread_spinlock_t DC_SPINLOCK;
+# define DC_OSSpinLockCreate(psLock) pthread_spin_init(psLock, PTHREAD_PROCESS_PRIVATE)
+# define DC_OSSpinLockDestroy(psLock) pthread_spin_destroy(psLock)
+# define DC_OSSpinLockIRQSave(psLock, ulIRQFlags) PVR_UNREFERENCED_PARAMETER(ulIRQFlags); pthread_spin_lock(psLock)
+# define DC_OSSpinUnlockIRQRestore(psLock, ulIRQFlags) PVR_UNREFERENCED_PARAMETER(ulIRQFlags); pthread_spin_unlock(psLock)
+
+# define DC_OSSpinLock(psLock) pthread_spin_lock(psLock)
+# define DC_OSSpinUnlock(psLock) pthread_spin_unlock(psLock)
+
+#elif defined(INTEGRITY_OS)
+typedef struct DC_SPINLOCK_TAG
+{
+	IMG_UINT64 ui64Lock;
+} DC_SPINLOCK;
+IMG_BOOL DC_OSSpinLockCreate(void **pplMutex);
+IMG_BOOL DC_OSSpinLockDestroy(void *plMutex);
+IMG_BOOL DC_OSSpinLock(void *plMutex);
+IMG_BOOL DC_OSSpinUnlock(void *plMutex);
+#define DC_OSSpinLockIRQSave(plMutex) DC_OSSpinLock(plMutex)
+#define DC_OSSpinUnlockIRQRestore(plMutex) DC_OSSpinUnlock(plMutex)
+#else
+#error "define an OS"
+#endif
+
+void DC_OSDelayus(IMG_UINT32 ui32Timeus);
+
+/* Services access */
+PVRSRV_ERROR DC_OSPVRServicesConnectionOpen(IMG_HANDLE *phPVRServicesConnection);
+void DC_OSPVRServicesConnectionClose(IMG_HANDLE hPVRServicesConnection);
+PVRSRV_ERROR DC_OSPVRServicesSetupFuncs(IMG_HANDLE hPVRServicesConnection, DC_SERVICES_FUNCS *psServicesFuncs);
+
+/* Workqueue functionality */
+/* OS work queue interface function pointer */
+typedef PVRSRV_ERROR (*PFN_WORK_PROCESSOR)(void *pvData);
+
+PVRSRV_ERROR DC_OSWorkQueueCreate(IMG_HANDLE *phQueue, IMG_UINT32 ui32Length);
+PVRSRV_ERROR DC_OSWorkQueueDestroy(IMG_HANDLE hQueue);
+PVRSRV_ERROR DC_OSWorkQueueFlush(IMG_HANDLE hQueue);
+
+PVRSRV_ERROR DC_OSWorkQueueCreateWorkItem(IMG_HANDLE *phWorkItem, PFN_WORK_PROCESSOR pfnProcessor, void *pvProcessorData);
+
+PVRSRV_ERROR DC_OSWorkQueueDestroyWorkItem(IMG_HANDLE phWorkItem);
+PVRSRV_ERROR DC_OSWorkQueueAddWorkItem(IMG_HANDLE hQueue, IMG_HANDLE hWorkItem);
+
+/* Atomic ops */
+
+#if defined(__linux__)
+
+#include <linux/atomic.h>
+
+#define DC_OSAtomicRead(pCounter) atomic_read(pCounter)
+#define DC_OSAtomicWrite(pCounter, i) atomic_set(pCounter, i)
+#define DC_OSAtomicIncrement(pCounter) atomic_inc_return(pCounter)
+#define DC_OSAtomicDecrement(pCounter) atomic_dec_return(pCounter)
+
+#else
+
+IMG_INT DC_OSAtomicRead(const ATOMIC_T *pCounter);
+void DC_OSAtomicWrite(ATOMIC_T *pCounter, IMG_INT v);
+IMG_INT DC_OSAtomicIncrement(ATOMIC_T *pCounter);
+IMG_INT DC_OSAtomicDecrement(ATOMIC_T *pCounter);
+
+#endif
+
+#endif /* !defined(DC_OSFUNCS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/common/dc_pdp_common.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/common/dc_pdp_common.c
new file mode 100644
index 000000000000..3747e0e9a26b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/common/dc_pdp_common.c
@@ -0,0 +1,1855 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dc_pdp.h"
+#include "dc_osfuncs.h"
+#include "img_defs.h"
+#include "tcf_pll.h"
+#include "tcf_rgbpdp_regs.h"
+#include "powervr/imgpixfmts.h"
+
+#include "pvrsrv.h"
+
+#if defined(__linux__)
+#include <linux/spinlock.h>
+#endif
+
+#define DCPDP_MAX_COMMANDS_INFLIGHT	(2)
+
+/* This has to be less than or equal to the size of the type
+   used for ui32BufferUseMask in the DCPDP_DEVICE structure */
+#define DCPDP_MAX_BUFFERS		(32)
+
+#define DCPDP_MIN_DISPLAY_PERIOD	(0)
+#define DCPDP_MAX_DISPLAY_PERIOD	(2)
+
+#define DCPDP_PIXEL_FORMAT		IMG_PIXFMT_B8G8R8A8_UNORM
+#define DCPDP_PIXEL_FORMAT_BPP		(4)
+
+#ifndef DCPDP_DPI
+#define DCPDP_DPI			(160)
+#endif
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+#define	fGTF_MARGIN			1.8f	/* Top & bottom margin % */
+#define fGTF_HSYNC_MARGIN		8.0f	/* HSync margin % */
+#define fGTF_MIN_VSYNCBP		550.0f	/* Min VSync + Back Porch (us) */
+#define	fGTF_MIN_PORCH			1.0f	/* Minimum porch line / char cell */
+#define fGTF_CELL_GRAN			8.0f	/* assumed pixel granularity (cell) */
+#define fGTF_CELL_GRAN2			16.0f	/* assumed pixel granularity * 2 (cell) */
+
+#define fGTF_M				600.0f	/* Blanking Formula Gradient %/KHz */
+#define	fGTF_C				40.0f	/* Blanking Formula Offset % */
+#define fGTF_K				128.0f	/* Blanking Formula scale factor */
+#define fGTF_J				20.0f	/* Blanking Formula weighting % */
+
+#define GTF_VSYNC_WIDTH			7	/* Width of VSync lines */
+#define	GTF_MIN_PORCH			1	/* Minimum porch line / char cell */
+#define	GTF_MIN_PORCH_PIXELS		8	/* Minimum porch Pixels cell */
+#define GTF_CELL_GRAN			8	/* assumed pixel granularity (cell) */
+#define GTF_CELL_GRAN2			16	/* assumed pixel granularity * 2 (cell) */
+#endif
+
+typedef struct DCPDP_BUFFER_TAG DCPDP_BUFFER;
+typedef struct DCPDP_FLIP_CONFIG_TAG DCPDP_FLIP_CONFIG;
+typedef struct DCPDP_FLIP_CONTEXT_TAG DCPDP_FLIP_CONTEXT;
+
+typedef struct DCPDP_TIMING_DATA_TAG
+{
+	IMG_UINT32		ui32HDisplay;
+	IMG_UINT32		ui32HBackPorch;
+	IMG_UINT32		ui32HTotal;
+	IMG_UINT32		ui32HActiveStart;
+	IMG_UINT32		ui32HLeftBorder;
+	IMG_UINT32		ui32HRightBorder;
+	IMG_UINT32		ui32HFrontPorch;
+
+	IMG_UINT32		ui32VDisplay;
+	IMG_UINT32		ui32VBackPorch;
+	IMG_UINT32		ui32VTotal;
+	IMG_UINT32		ui32VActiveStart;
+	IMG_UINT32		ui32VTopBorder;
+	IMG_UINT32		ui32VBottomBorder;
+	IMG_UINT32		ui32VFrontPorch;
+
+	IMG_UINT32		ui32VRefresh;
+	IMG_UINT32		ui32ClockFreq;
+} DCPDP_TIMING_DATA;
+
+struct DCPDP_BUFFER_TAG
+{
+	ATOMIC_T		i32RefCount;
+
+	DCPDP_DEVICE		*psDeviceData;
+
+	IMG_UINT32		ui32Width;
+	IMG_UINT32		ui32Height;
+	IMG_UINT32		ui32ByteStride;
+	IMG_UINT32		ui32SizeInBytes;
+	IMG_UINT32		ui32SizeInPages;
+	IMG_PIXFMT		ePixelFormat;
+
+	IMG_CPU_PHYADDR		sCpuPAddr;
+	IMG_DEV_PHYADDR		sDevPAddr;
+	IMG_CPU_PHYADDR		sPDPPAddr;
+};
+
+typedef enum DCPDP_FLIP_CONFIG_STATUS_TAG
+{
+	DCPDP_FLIP_CONFIG_INACTIVE = 0,
+	DCPDP_FLIP_CONFIG_PENDING,
+	DCPDP_FLIP_CONFIG_ACTIVE,
+} DCPDP_FLIP_CONFIG_STATUS;
+
+
+/* Flip config structure used for queuing of flips */
+struct DCPDP_FLIP_CONFIG_TAG
+{
+	/* Context with which this config is associated */
+	DCPDP_FLIP_CONTEXT	*psContext;
+
+	/* Services config data */
+	IMG_HANDLE		hConfigData;
+
+	/* Current status of the config */
+	DCPDP_FLIP_CONFIG_STATUS eStatus;
+
+	/* Buffer to be displayed when the config is made active */
+	DCPDP_BUFFER		*psBuffer;
+
+	/* Number of frames for which this config should remain active */
+	IMG_UINT32		ui32DisplayPeriod;
+
+	/* New mode to possibly switch to after flip */
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+
+	/* Handle to a work item that processes the flip config queue */
+	IMG_HANDLE		hWorkItem;
+};
+
+struct DCPDP_FLIP_CONTEXT_TAG
+{
+	/* Device on which the context has been created */
+	DCPDP_DEVICE		*psDeviceData;
+
+	/* Lock protecting the queue of flip configs */
+	DC_SPINLOCK		sSpinLock;
+
+	/* Mutex to serialise calls back into Services */
+	void			*pvMutex;
+
+	/* Queue of configs that need processing */
+	DCPDP_FLIP_CONFIG	asFlipConfigQueue[DCPDP_MAX_COMMANDS_INFLIGHT];
+	IMG_UINT32		ui32FlipConfigInsertIndex;
+	IMG_UINT32		ui32FlipConfigRemoveIndex;
+
+	/* Handle to a work queue used to defer the processing of the
+	   flip config queue when a vsync interrupt is serviced */
+	IMG_HANDLE		hFlipConfigWorkQueue;
+
+	/* Services config data to retire */
+	IMG_HANDLE		hConfigDataToRetire;
+};
+
+struct DCPDP_DEVICE_TAG
+{
+	IMG_HANDLE		hPVRServicesDevice;
+	const DC_SERVICES_FUNCS	*psPVRServicesFuncs;
+
+	DCPDP_DEVICE_PRIV	*psDevicePriv;
+
+	PHYS_HEAP		*psPhysHeap;
+
+	IMG_CPU_PHYADDR		sPDPRegCpuPAddr;
+	IMG_CPU_VIRTADDR	pvPDPRegCpuVAddr;
+	IMG_UINT32		ui32PDPRegSize;
+
+	IMG_CPU_PHYADDR		sLMACPUPhysBaseAddr;
+	IMG_CPU_PHYADDR		sDispMemCpuPAddr;
+	IMG_UINT64		uiDispMemSize;
+
+	IMG_UINT32		uiTimingDataIndex;
+	DCPDP_TIMING_DATA      *pasTimingData;
+	IMG_UINT32		uiTimingDataSize;
+	IMG_PIXFMT		ePixelFormat;
+
+	IMG_UINT32		ui32BufferSize;
+	IMG_UINT32		ui32BufferCount;
+	IMG_UINT32		ui32BufferUseMask;
+
+	DCPDP_BUFFER		*psSystemBuffer;
+
+	DCPDP_FLIP_CONTEXT	*psFlipContext;
+
+	IMG_BOOL		bVSyncEnabled;
+	IMG_BOOL		bVSyncReporting;
+	IMG_INT64		i64LastVSyncTimeStamp;
+	IMG_HANDLE		hVSyncWorkQueue;
+	IMG_HANDLE		hVSyncWorkItem;
+};
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+static struct VESA_STANDARD_MODES
+{
+        IMG_UINT32 uiWidth;
+        IMG_UINT32 uiHeight;
+} gasVesaModes[] =
+{
+        { 640, 480},
+        { 800, 600},
+        { 1024, 768},
+        { 1280, 720},
+        { 1280, 768},
+        { 1280, 800},
+        { 1280, 1024},
+};
+#endif
+
+static IMG_BOOL BufferAcquireMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDeviceData->ui32BufferCount; i++)
+	{
+		if ((psDeviceData->ui32BufferUseMask & (1UL << i)) == 0)
+		{
+			psDeviceData->ui32BufferUseMask |= (1UL << i);
+
+			psBuffer->sCpuPAddr.uiAddr = psDeviceData->sDispMemCpuPAddr.uiAddr + (i * psDeviceData->ui32BufferSize);
+
+			psDeviceData->psPVRServicesFuncs->pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap,
+											1,
+											&psBuffer->sDevPAddr,
+											&psBuffer->sCpuPAddr);
+
+			/* Calculate the correct PDP device address.
+			 * PDP device can only access local memory but not host memory.
+			 * The PDP device address starts at 0 and extends to the size of
+			 * local memory.
+			 */
+
+			/* psBuffer->psDeviceData->sLMACPUPhysBaseAddr.uiAddr always points
+			 * to the start of CPU mapped physical address of the local memory.
+			 *
+			 * Display heap is a carve out from the local memory after LMA local heap.
+			 * Any buffers used for GPU/PDP are offsets from start of LMA heap
+			 */
+			psBuffer->sPDPPAddr.uiAddr = psBuffer->sCpuPAddr.uiAddr - psBuffer->psDeviceData->sLMACPUPhysBaseAddr.uiAddr;
+
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+
+static void BufferReleaseMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT64 uiOffset;
+
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr >= psDeviceData->sDispMemCpuPAddr.uiAddr);
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr < (psDeviceData->sDispMemCpuPAddr.uiAddr + psDeviceData->uiDispMemSize));
+
+	uiOffset = psBuffer->sCpuPAddr.uiAddr - psDeviceData->sDispMemCpuPAddr.uiAddr;
+	uiOffset = DC_OSDiv64(uiOffset, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferUseMask &= ~(1UL << uiOffset);
+
+	psBuffer->sDevPAddr.uiAddr = 0;
+	psBuffer->sCpuPAddr.uiAddr = 0;
+}
+
+static void ClearSystemBuffer(DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 *puiPixel, uiSizeInPixels, uiGray, i;
+
+	IMG_CPU_VIRTADDR pvBufferCpuVAddr = DC_OSMapPhysAddr(psBuffer->sCpuPAddr,
+	                                                     psBuffer->ui32SizeInBytes);
+	if (pvBufferCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING, "Request for clearing system buffer but no "
+		                 "buffer provided");
+		return;
+	}
+
+	puiPixel = (IMG_UINT32 *) pvBufferCpuVAddr;
+	uiSizeInPixels = psBuffer->ui32Width * psBuffer->ui32Height;
+
+	for (i = 0; i < uiSizeInPixels; i++)
+	{
+		uiGray = (IMG_UINT32) (0x5A * i / uiSizeInPixels);
+		puiPixel[i] = 0xFF000000 + uiGray + (uiGray << 8) + (uiGray << 16);
+	}
+
+	DC_OSUnmapPhysAddr(pvBufferCpuVAddr, psBuffer->ui32SizeInBytes);
+}
+
+static void EnableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB);
+	ui32InterruptEnable |= (0x1 << INTEN_VBLNK1_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB, ui32InterruptEnable);
+}
+
+static void DisableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB);
+	ui32InterruptEnable &= ~(INTEN_VBLNK1_MASK);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB, ui32InterruptEnable);
+}
+
+static void SetModeRegisters(DCPDP_DEVICE *psDeviceData);
+
+static INLINE void Flip(DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 ui32Str1AddrCtrl;
+	IMG_UINT32 ui32Stride;
+	IMG_UINT32 ui32Value;
+	IMG_UINT32 ui32Width = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	IMG_UINT32 ui32Height = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	ui32Value = (ui32Width - 1) << STR1WIDTH_SHIFT;
+	ui32Value |= (ui32Height - 1) << STR1HEIGHT_SHIFT;
+
+	switch (psBuffer->psDeviceData->ePixelFormat)
+	{
+		case IMG_PIXFMT_B8G8R8A8_UNORM:
+		{
+			ui32Value |= DCPDP_STR1SURF_FORMAT_ARGB8888 << STR1PIXFMT_SHIFT;
+			break;
+		}
+		default:
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Unrecognised pixel format\n");
+			DC_ASSERT(0);
+		}
+	}
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF, ui32Value);
+
+	ui32Stride = ui32Width * DCPDP_PIXEL_FORMAT_BPP;
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_PDP_STR1POSN, (ui32Stride >> DCPDP_STR1POSN_STRIDE_SHIFT) - 1);
+
+	DC_ASSERT(psBuffer);
+
+	ui32Str1AddrCtrl = DC_OSReadReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL);
+	ui32Str1AddrCtrl &= ~(STR1BASE_MASK);
+	ui32Str1AddrCtrl |= ((psBuffer->sPDPPAddr.uiAddr >> DCPDP_STR1ADDRCTRL_BASE_ADDR_SHIFT) & STR1BASE_MASK);
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, ui32Str1AddrCtrl);
+}
+
+static INLINE void FlipConfigQueueAdvanceIndex(DCPDP_FLIP_CONTEXT *psContext, IMG_UINT32 *pui32Index)
+{
+	(*pui32Index)++;
+
+	if (*pui32Index >= DCPDP_MAX_COMMANDS_INFLIGHT)
+	{
+		*pui32Index = 0;
+	}
+}
+
+static INLINE IMG_BOOL FlipConfigQueueEmptyNoLock(DCPDP_FLIP_CONTEXT *psContext)
+{
+	return (IMG_BOOL)(psContext->ui32FlipConfigInsertIndex == psContext->ui32FlipConfigRemoveIndex);
+}
+
+static IMG_BOOL FlipConfigQueueEmpty(DCPDP_FLIP_CONTEXT *psContext)
+{
+	unsigned long ulSpinlockFlags;
+	IMG_BOOL bFlipConfigQueueEmpty;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+	bFlipConfigQueueEmpty = FlipConfigQueueEmptyNoLock(psContext);
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	return bFlipConfigQueueEmpty;
+}
+
+static void FlipConfigQueueAdd(DCPDP_FLIP_CONTEXT *psContext,
+				   IMG_HANDLE hConfigData,
+				   DCPDP_BUFFER *psBuffer,
+				   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				   IMG_UINT32 ui32DisplayPeriod)
+{
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Get the next inactive flip config and update the insert index */
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigInsertIndex];
+	FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigInsertIndex);
+
+	DC_ASSERT(psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_INACTIVE);
+
+	/* Fill out the flip config */
+	psFlipConfig->psContext		= psContext;
+	psFlipConfig->hConfigData	= hConfigData;
+	psFlipConfig->psBuffer		= psBuffer;
+	psFlipConfig->ui32DisplayPeriod	= ui32DisplayPeriod;
+	psFlipConfig->pasSurfAttrib	= pasSurfAttrib;
+
+	/* Should be updated last */
+	psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_PENDING;
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+}
+
+static IMG_UINT32 GetTimingIndex(DCPDP_DEVICE *psDeviceData, IMG_UINT32 width, IMG_UINT32 height);
+
+static void FlipConfigQueueProcess(DCPDP_FLIP_CONTEXT *psContext, IMG_BOOL bFlipped, IMG_BOOL bFlush)
+{
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+	IMG_HANDLE ahRetiredConfigs[DCPDP_MAX_COMMANDS_INFLIGHT] = {0};
+	IMG_UINT32 i, ui32RetireConfigIndex = 0;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSMutexLock(psContext->pvMutex);
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+	pasSurfAttrib = psFlipConfig->pasSurfAttrib;
+
+	do
+	{
+		switch (psFlipConfig->eStatus)
+		{
+			case DCPDP_FLIP_CONFIG_PENDING:
+				if (!bFlipped)
+				{
+					Flip(psFlipConfig->psBuffer);
+
+					bFlipped = IMG_TRUE;
+				}
+
+				/* If required, set the mode after flipping */
+				if (pasSurfAttrib != NULL)
+				{
+					if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay ||
+					    pasSurfAttrib[0].sDisplay.sDims.ui32Height != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay)
+					{
+						psDeviceData->uiTimingDataIndex = GetTimingIndex(psDeviceData,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Width,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+						SetModeRegisters(psDeviceData);
+					}
+				}
+
+				psFlipConfig->eStatus = DCPDP_FLIP_CONFIG_ACTIVE;
+
+				/* Now that a new flip config is active we can retire the previous one (if one exists) */
+				if (psContext->hConfigDataToRetire != NULL)
+				{
+					ahRetiredConfigs[ui32RetireConfigIndex++] = psContext->hConfigDataToRetire;
+					psContext->hConfigDataToRetire = NULL;
+				}
+
+				__fallthrough;
+			case DCPDP_FLIP_CONFIG_ACTIVE:
+				/* We can retire this config only when it's buffer has been
+				   displayed for at least the same number of frames as the
+				   display period or if we're flushing the queue */
+				if (psFlipConfig->ui32DisplayPeriod != 0)
+				{
+					psFlipConfig->ui32DisplayPeriod--;
+				}
+
+				if (psFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+				{
+					DCPDP_FLIP_CONFIG *psNextFlipConfig;
+
+					/* There should never be an outstanding config to retire at this point */
+					DC_ASSERT(psContext->hConfigDataToRetire == NULL);
+
+					/* Now that this config has been active for the minimum number of frames, i.e. the
+					   display period, we can mark the flip config queue item as inactive. However, we
+					   can only retire the config once a new config has been made active. Therefore,
+					   store the Services config data handle so that we can retire it later. */
+					psContext->hConfigDataToRetire = psFlipConfig->hConfigData;
+
+					/* Reset the flip config data */
+					psFlipConfig->hConfigData	= 0;
+					psFlipConfig->psBuffer		= NULL;
+					psFlipConfig->ui32DisplayPeriod	= 0;
+					psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_INACTIVE;
+
+					/* Move on the queue remove index */
+					FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigRemoveIndex);
+
+					psNextFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+					if (psNextFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+					{
+						psFlipConfig = psNextFlipConfig;
+						bFlipped = IMG_FALSE;
+					}
+				}
+				break;
+			case DCPDP_FLIP_CONFIG_INACTIVE:
+				if (FlipConfigQueueEmptyNoLock(psContext))
+				{
+					break;
+				}
+				DC_OSDebugPrintf(DBGLVL_ERROR, " - Tried to process an inactive config when one wasn't expected\n");
+
+				__fallthrough;
+			default:
+				DC_ASSERT(0);
+		}
+	} while (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING);
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Call into Services to retire any configs that were ready to be
+	 * retired. */
+	for (i = 0; i < ui32RetireConfigIndex; i++)
+	{
+		psDeviceData->psPVRServicesFuncs->pfnDCDisplayConfigurationRetired(ahRetiredConfigs[i]);
+	}
+
+	DC_OSMutexUnlock(psContext->pvMutex);
+}
+
+static PVRSRV_ERROR VSyncProcessor(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = pvData;
+
+	psDeviceData->psPVRServicesFuncs->pfnCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FlipConfigProcessor(void *pvData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)pvData;
+
+	DC_ASSERT(psContext);
+
+	FlipConfigQueueProcess(psContext, IMG_TRUE, IMG_FALSE);
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPGetInfo(IMG_HANDLE hDeviceData, DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Copy our device name */
+	DC_OSStringNCopy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	/* Report what our minimum and maximum display period is. */
+	psDisplayInfo->ui32MinDisplayPeriod	= DCPDP_MIN_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxDisplayPeriod	= DCPDP_MAX_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxPipes		= 1;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static PVRSRV_ERROR DCPDPPanelQueryCount(IMG_HANDLE hDeviceData, IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPPanelQuery(IMG_HANDLE hDeviceData,
+				    IMG_UINT32 ui32PanelsArraySize,
+				    IMG_UINT32 *pui32NumPanels,
+				    PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	DC_OSMemSet(psPanelInfo, 0, sizeof(*psPanelInfo));
+
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = psDeviceData->ePixelFormat;
+	psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+	psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	psPanelInfo[0].ui32RefreshRate = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh;
+	psPanelInfo[0].ui32XDpi = DCPDP_DPI;
+	psPanelInfo[0].ui32YDpi = DCPDP_DPI;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPFormatQuery(IMG_HANDLE hDeviceData,
+				     IMG_UINT32 ui32NumFormats,
+				     PVRSRV_SURFACE_FORMAT *pasFormat,
+				     IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (pasFormat[i].ePixFormat == psDeviceData->ePixelFormat)
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPDimQuery(IMG_HANDLE hDeviceData,
+				  IMG_UINT32 ui32NumDims,
+				  PVRSRV_SURFACE_DIMS *psDim,
+				  IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+	IMG_UINT32 j;
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		for (j = 0; j < psDeviceData->uiTimingDataSize; j++)
+		{
+			if ((psDim[i].ui32Width == psDeviceData->pasTimingData[j].ui32HDisplay) &&
+			    (psDim[i].ui32Height == psDeviceData->pasTimingData[j].ui32VDisplay))
+			{
+				pui32Supported[i]++;
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetBlank(IMG_HANDLE hDeviceData,
+								  IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	IMG_UINT32 ui32Value;
+
+	ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL);
+	if (bEnabled)
+	{
+		ui32Value |= 0x1 << POWERDN_SHIFT;
+		psDeviceData->bVSyncEnabled = IMG_FALSE;
+	}
+	else
+	{
+		ui32Value &= ~(POWERDN_MASK);
+		psDeviceData->bVSyncEnabled = IMG_TRUE;
+	}
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, ui32Value);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetVSyncReporting(IMG_HANDLE hDeviceData,
+										   IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	psDeviceData->bVSyncReporting = bEnabled;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPLastVSyncQuery(IMG_HANDLE hDeviceData,
+										IMG_INT64 *pi64Timestamp)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	*pi64Timestamp = psDeviceData->i64LastVSyncTimeStamp;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPContextCreate(IMG_HANDLE hDeviceData,
+				       IMG_HANDLE *hDisplayContext)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	DCPDP_FLIP_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	if (psDeviceData->psFlipContext)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Cannot create a context when one is already active\n");
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psContext = DC_OSCallocMem(sizeof(*psContext));
+	if (psContext == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Not enough memory to allocate a display context\n");
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* We don't want to process the flip config work queue in interrupt context so
+	   create a work queue which we can use to defer the work */
+	eError = DC_OSWorkQueueCreate(&psContext->hFlipConfigWorkQueue, DCPDP_MAX_COMMANDS_INFLIGHT);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Couldn't create a work queue (%s)\n",
+				 psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorPDPOSFreeMemContext;
+	}
+
+	/* Create work items to place on the flip config work queue when we service a vsync interrupt */
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		eError = DC_OSWorkQueueCreateWorkItem(&psContext->asFlipConfigQueue[i].hWorkItem, FlipConfigProcessor, psContext);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create a flip config work item (%s)\n",
+					 psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+			goto ErrorDC_OSWorkQueueDestroyWorkItems;
+		}
+	}
+
+	eError = DC_OSMutexCreate(&psContext->pvMutex);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create flip config mutex (%s)\n",
+				 psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDC_OSWorkQueueDestroyWorkItems;
+	}
+
+	DC_OSSpinLockCreate(&psContext->sSpinLock);
+
+	psContext->psDeviceData		= psDeviceData;
+	psDeviceData->psFlipContext	= psContext;
+	*hDisplayContext		= (IMG_HANDLE)psContext;
+
+	EnableVSyncInterrupt(psDeviceData);
+
+	return PVRSRV_OK;
+
+ErrorDC_OSWorkQueueDestroyWorkItems:
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+
+ErrorPDPOSFreeMemContext:
+	DC_OSFreeMem(psContext);
+
+	return eError;
+}
+
+static void DCPDPContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_UINT32 i;
+
+	DisableVSyncInterrupt(psDeviceData);
+
+	/* Make sure there are no work items on the queue before it's destroyed */
+	DC_OSWorkQueueFlush(psContext->hFlipConfigWorkQueue);
+
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+	DC_OSMutexDestroy(psContext->pvMutex);
+	DC_OSSpinLockDestroy(&psContext->sSpinLock);
+
+	psDeviceData->psFlipContext = NULL;
+	DC_OSFreeMem(psContext);
+}
+
+static IMG_UINT32 GetTimingIndex(DCPDP_DEVICE *psDeviceData, IMG_UINT32 width, IMG_UINT32 height)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 index = -1;
+	for (i = 0; i < psDeviceData->uiTimingDataSize; i++)
+	{
+		if (psDeviceData->pasTimingData[i].ui32HDisplay == width &&
+		   psDeviceData->pasTimingData[i].ui32VDisplay == height)
+		{
+			index = i;
+		}
+	}
+
+	return index;
+}
+
+static void DCPDPContextConfigure(IMG_HANDLE hDisplayContext,
+				  IMG_UINT32 ui32PipeCount,
+				  PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				  IMG_HANDLE *ahBuffers,
+				  IMG_UINT32 ui32DisplayPeriod,
+				  IMG_HANDLE hConfigData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_BOOL bQueueWasEmpty;
+
+	if (ui32DisplayPeriod > DCPDP_MAX_DISPLAY_PERIOD)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Requested display period of %u is larger than maximum supported display period of %u (clamping)\n",
+				 ui32DisplayPeriod, DCPDP_MAX_DISPLAY_PERIOD);
+		DC_ASSERT(ui32DisplayPeriod <= DCPDP_MAX_DISPLAY_PERIOD);
+
+		/* Clamp to the maximum so that we can continue */
+		ui32DisplayPeriod = DCPDP_MAX_DISPLAY_PERIOD;
+	}
+
+	/* Add new config to the queue */
+	if (ui32PipeCount != 0)
+	{
+		/* Check if the queue is empty before adding a new flip config */
+		bQueueWasEmpty = FlipConfigQueueEmpty(psContext);
+
+		/* We have a regular config so add it as normal */
+		FlipConfigQueueAdd(psContext, hConfigData, (DCPDP_BUFFER *)ahBuffers[0], pasSurfAttrib, ui32DisplayPeriod);
+
+		/* Check to see if vsync unlocked flipping should be done. This is determined by the
+		   display period being 0 and the flip config queue being empty before we added the
+		   new flip config. If it wasn't empty then we need to allow the remaining vsync locked
+		   flip configs to be processed before we can start processing the queue here.
+		   Also immediately flip if the current vsync interrupt is disabled.
+		   This makes sure that even when the PDP is turned off we progress. */
+		if ((!ui32DisplayPeriod && bQueueWasEmpty) || !psDeviceData->bVSyncEnabled)
+		{
+			FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_FALSE);
+		}
+	}
+	else
+	{
+		/* We have a 'NULL' config, which indicates the display context is
+		   about to be destroyed. Queue a flip back to the system buffer */
+		FlipConfigQueueAdd(psContext, hConfigData, psDeviceData->psSystemBuffer, pasSurfAttrib, 0);
+
+		/* Flush the remaining configs (this will result in the system buffer being displayed) */
+		FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_TRUE);
+
+		DC_OSMutexLock(psContext->pvMutex);
+
+		/* Immediately retire the system buffer config data */
+		psDeviceData->psPVRServicesFuncs->pfnDCDisplayConfigurationRetired(hConfigData);
+		psContext->hConfigDataToRetire = NULL;
+
+		DC_OSMutexUnlock(psContext->pvMutex);
+	}
+}
+
+static PVRSRV_ERROR DCPDPContextConfigureCheck(IMG_HANDLE hDisplayContext,
+					       IMG_UINT32 ui32PipeCount,
+					       PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+					       IMG_HANDLE *ahBuffers)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_BUFFER *psBuffer;
+	IMG_UINT32 ui32NewVESATimingIndex;
+	ui32NewVESATimingIndex = GetTimingIndex(psDeviceData,pasSurfAttrib[0].sDisplay.sDims.ui32Width, pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+
+	if (ui32NewVESATimingIndex == -1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+
+	if (ui32PipeCount != 1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+	/* new crop dimensions, new mode dimensions */
+	else if (pasSurfAttrib[0].sCrop.sDims.ui32Width != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32HDisplay ||
+		 pasSurfAttrib[0].sCrop.sDims.ui32Height != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32VDisplay ||
+		 pasSurfAttrib[0].sCrop.i32XOffset != 0 ||
+		 pasSurfAttrib[0].sCrop.i32YOffset != 0)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+	}
+	/* new mode dimensions, new crop dimensions */
+	else if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != pasSurfAttrib[0].sCrop.sDims.ui32Width ||
+		 pasSurfAttrib[0].sDisplay.sDims.ui32Height != pasSurfAttrib[0].sCrop.sDims.ui32Height ||
+		 pasSurfAttrib[0].sDisplay.i32XOffset != pasSurfAttrib[0].sCrop.i32XOffset ||
+		 pasSurfAttrib[0].sDisplay.i32YOffset != pasSurfAttrib[0].sCrop.i32YOffset)
+	{
+		return PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+	}
+
+	psBuffer = ahBuffers[0];
+
+	/* new buffer dimensions, current mode dimensions */
+	if (psBuffer->ui32Width  < pasSurfAttrib[0].sDisplay.sDims.ui32Width ||
+	    psBuffer->ui32Height < pasSurfAttrib[0].sDisplay.sDims.ui32Height)
+	{
+		return PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferSystemAcquire(IMG_HANDLE hDeviceData,
+					     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+					     IMG_UINT32 *pui32PageCount,
+					     IMG_UINT32 *pui32ByteStride,
+					     IMG_HANDLE *phSystemBuffer)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	DC_OSAtomicIncrement(&psDeviceData->psSystemBuffer->i32RefCount);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psDeviceData->psSystemBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psDeviceData->psSystemBuffer->ui32ByteStride;
+	*phSystemBuffer		= psDeviceData->psSystemBuffer;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferSystemRelease(IMG_HANDLE hSystemBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hSystemBuffer;
+
+	DC_OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static PVRSRV_ERROR DCPDPBufferAlloc(IMG_HANDLE hDisplayContext,
+				     DC_BUFFER_CREATE_INFO *psCreateInfo,
+				     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+				     IMG_UINT32 *pui32PageCount,
+				     IMG_UINT32 *pui32ByteStride,
+				     IMG_HANDLE *phBuffer)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	DCPDP_BUFFER *psBuffer;
+
+	DC_ASSERT(psCreateInfo->ui32BPP == DCPDP_PIXEL_FORMAT_BPP);
+
+	if (psSurfInfo->sFormat.eMemLayout != PVRSRV_SURFACE_MEMLAYOUT_STRIDED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psSurfInfo->sFormat.ePixFormat != psDeviceData->ePixelFormat)
+	{
+		return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psSurfInfo->sFormat.ePixFormat;
+	psBuffer->ui32Width		= psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height		= psSurfInfo->sDims.ui32Height;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * psCreateInfo->ui32BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32)DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height, DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psBuffer->ui32ByteStride;
+	*phBuffer		= psBuffer;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferAcquire(IMG_HANDLE hBuffer, IMG_DEV_PHYADDR *pasDevPAddr, void **ppvLinAddr)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+	DCPDP_DEVICE *psDeviceData = psBuffer->psDeviceData;
+	IMG_CPU_PHYADDR sCpuPAddr = psBuffer->sCpuPAddr;
+	IMG_UINT32 i;
+
+	for (i = 0; i < psBuffer->ui32SizeInPages; i++)
+	{
+		psDeviceData->psPVRServicesFuncs->pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap,
+										1,
+										&pasDevPAddr[i], &sCpuPAddr);
+		sCpuPAddr.uiAddr += DC_OSGetPageSize();
+	}
+
+	DC_OSAtomicIncrement(&psBuffer->i32RefCount);
+
+	*ppvLinAddr = NULL;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferRelease(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+	DC_OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static void DCPDPBufferFree(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+#if defined(DEBUG)
+	if (DC_OSAtomicRead(&psBuffer->i32RefCount) != 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING, " - Freeing buffer that still has %d references\n", DC_OSAtomicRead(&psBuffer->i32RefCount));
+	}
+#endif
+
+	BufferReleaseMemory(psBuffer->psDeviceData, psBuffer);
+
+	DC_OSFreeMem(psBuffer);
+}
+
+static PVRSRV_ERROR DCPDPResetDevice(IMG_HANDLE hDeviceData)
+{
+	DCPDP_DEVICE *psDeviceData = hDeviceData;
+
+	ClearSystemBuffer(psDeviceData->psSystemBuffer);
+
+	SetModeRegisters(psDeviceData);
+	Flip(psDeviceData->psSystemBuffer);
+
+	return PVRSRV_OK;
+}
+
+static DC_DEVICE_FUNCTIONS g_sDCFunctions =
+{
+	DCPDPGetInfo,				/* pfnGetInfo */
+	DCPDPPanelQueryCount,		/* pfnPanelQueryCount */
+	DCPDPPanelQuery,			/* pfnPanelQuery */
+	DCPDPFormatQuery,			/* pfnFormatQuery */
+	DCPDPDimQuery,				/* pfnDimQuery */
+	DCPDPSetBlank,				/* pfnSetBlank */
+	DCPDPSetVSyncReporting,		/* pfnSetVSyncReporting */
+	DCPDPLastVSyncQuery,		/* pfnLastVSyncQuery */
+	DCPDPContextCreate,			/* pfnContextCreate */
+	DCPDPContextDestroy,		/* pfnContextDestroy */
+	DCPDPContextConfigure,		/* pfnContextConfigure */
+	DCPDPContextConfigureCheck,	/* pfnContextConfigureCheck */
+	DCPDPBufferAlloc,			/* pfnBufferAlloc */
+	DCPDPBufferAcquire,			/* pfnBufferAcquire */
+	DCPDPBufferRelease,			/* pfnBufferRelease */
+	DCPDPBufferFree,			/* pfnBufferFree */
+	NULL,					/* pfnBufferImport */
+	NULL,					/* pfnBufferMap */
+	NULL,					/* pfnBufferUnmap */
+	DCPDPBufferSystemAcquire,	/* pfnBufferSystemAcquire */
+	DCPDPBufferSystemRelease,	/* pfnBufferSystemRelease */
+
+	DCPDPResetDevice,			/* pfnResetDevice */
+};
+
+static IMG_BOOL DisplayLISRHandler(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)pvData;
+	IMG_UINT32 ui32InterruptStatus;
+	PVRSRV_ERROR eError;
+
+	ui32InterruptStatus = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTSTAT);
+	if (ui32InterruptStatus & INTS_VBLNK1_MASK)
+	{
+		/* Get the timestamp for this interrupt. */
+		psDeviceData->i64LastVSyncTimeStamp = DC_OSClockns();
+		if (psDeviceData->bVSyncReporting)
+		{
+			eError = DC_OSWorkQueueAddWorkItem(psDeviceData->hVSyncWorkQueue, psDeviceData->hVSyncWorkItem);
+			if (eError != PVRSRV_OK)
+			{
+				DC_OSDebugPrintf(DBGLVL_WARNING,
+						 " - %s: Couldn't queue work item (%s)\n",
+						 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+
+			}
+		}
+
+		if (psDeviceData->psFlipContext)
+		{
+			DCPDP_FLIP_CONTEXT *psContext = psDeviceData->psFlipContext;
+			DCPDP_FLIP_CONFIG *psFlipConfig;
+
+			DC_OSSpinLock(&psContext->sSpinLock);
+
+			psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+
+			if (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING)
+			{
+				Flip(psFlipConfig->psBuffer);
+			}
+
+			if (psFlipConfig->eStatus != DCPDP_FLIP_CONFIG_INACTIVE)
+			{
+				eError = DC_OSWorkQueueAddWorkItem(psContext->hFlipConfigWorkQueue, psFlipConfig->hWorkItem);
+				if (eError != PVRSRV_OK)
+				{
+					DC_OSDebugPrintf(DBGLVL_WARNING,
+							 " - %s: Couldn't queue work item (%s)\n",
+							 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+				}
+			}
+
+			DC_OSSpinUnlock(&psContext->sSpinLock);
+		}
+		/* Clear the vsync status bit to show that the vsync interrupt has been handled */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTCLEAR, (0x1UL << INTCLR_VBLNK1_SHIFT));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+static PVRSRV_ERROR SetPDPClocks(DCPDP_DEVICE *psDeviceData, IMG_UINT32 uiClkInHz)
+{
+	IMG_CPU_PHYADDR sPLLRegCpuPAddr;
+	IMG_CPU_VIRTADDR pvPLLRegCpuVAddr;
+	IMG_UINT32 ui32PLLRegSize;
+	IMG_UINT32 uiClkInMHz;
+
+	/* Reserve the PLL register region and map the registers in */
+	sPLLRegCpuPAddr = DCPDPGetAddrRangeStart(psDeviceData->psDevicePriv,
+						 DCPDP_ADDRESS_RANGE_PLL);
+	ui32PLLRegSize = DCPDPGetAddrRangeSize(psDeviceData->psDevicePriv,
+					       DCPDP_ADDRESS_RANGE_PLL);
+
+	if (DC_OSRequestAddrRegion(sPLLRegCpuPAddr, ui32PLLRegSize, DRVNAME) == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to reserve PLL registers\n", __func__);
+		return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+	}
+
+	pvPLLRegCpuVAddr = DC_OSMapPhysAddr(sPLLRegCpuPAddr, ui32PLLRegSize);
+	if (pvPLLRegCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to map PLL registers\n", __func__);
+
+		DC_OSReleaseAddrRegion(sPLLRegCpuPAddr, ui32PLLRegSize);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Calculate the clock in MHz */
+	uiClkInMHz = (uiClkInHz + 500000) / 1000000;
+
+	/* Set phase 0, ratio 50:50 and frequency in MHz */
+	DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_CLK0, uiClkInMHz);
+
+	/* setup TCF_CR_PLL_PDP_CLK1TO5 based on the main clock speed */
+	if (uiClkInMHz >= 50)
+	{
+		/* Set clocks 1-5 to be the same frequency as clock 0 */
+		DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_CLK1TO5, 0x0);
+	}
+	else
+	{
+		/* Set clocks 1-5 to be double the frequency of clock 0 */
+		DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_CLK1TO5, 0x3);
+	}
+
+	/* Now initiate reprogramming of the PLLs */
+	DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_DRP_GO, 1);
+	DC_OSDelayus(1000);
+	DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_DRP_GO, 0);
+
+	/* Unmap and release the PLL address region */
+	DC_OSUnmapPhysAddr(pvPLLRegCpuVAddr, ui32PLLRegSize);
+	DC_OSReleaseAddrRegion(sPLLRegCpuPAddr, ui32PLLRegSize);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+
+static void SetModeRegisters(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_TIMING_DATA *psTimingData = &psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex];
+	IMG_UINT32 ui32Value;
+
+	DC_ASSERT(psDeviceData->psSystemBuffer);
+
+	/* Turn off memory requests */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, 0x00000000);
+
+	/* Disable sync gen */
+	ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL);
+	ui32Value &= ~(SYNCACTIVE_MASK);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, ui32Value);
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+	SetPDPClocks(psDeviceData, psTimingData->ui32ClockFreq);
+#endif
+
+	if (DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL) != 0x0000C010)
+	{
+		/* Buffer request threshold */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL, 0x00001C10);
+	}
+
+	/* Border colour */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL, 0x00005544);
+
+	/* Update control */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL, 0x00000000);
+
+	/* Set hsync timings */
+	ui32Value = (psTimingData->ui32HBackPorch << HBPS_SHIFT) | (psTimingData->ui32HTotal << HT_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1, ui32Value);
+
+	ui32Value = (psTimingData->ui32HActiveStart << HAS_SHIFT) | (psTimingData->ui32HLeftBorder << HLBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2, ui32Value);
+
+	ui32Value = (psTimingData->ui32HFrontPorch << HFPS_SHIFT) | (psTimingData->ui32HRightBorder << HRBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3, psTimingData->ui32HFrontPorch << 16 | ui32Value);
+
+	/* Set vsync timings */
+	ui32Value = (psTimingData->ui32VBackPorch << VBPS_SHIFT) | (psTimingData->ui32VTotal << VT_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1, ui32Value);
+
+	ui32Value = (psTimingData->ui32VActiveStart << VAS_SHIFT) | (psTimingData->ui32VTopBorder << VTBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2, ui32Value);
+
+	ui32Value = (psTimingData->ui32VFrontPorch << VFPS_SHIFT) | (psTimingData->ui32VBottomBorder << VBBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3, ui32Value);
+
+	/* Horizontal data enable */
+	ui32Value = (psTimingData->ui32HActiveStart << HDES_SHIFT) | (psTimingData->ui32HFrontPorch << HDEF_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL, ui32Value);
+
+	/* Vertical data enable */
+	ui32Value = (psTimingData->ui32VActiveStart << VDES_SHIFT) | (psTimingData->ui32VFrontPorch << VDEF_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL, ui32Value);
+
+	/* Vertical event start and vertical fetch start */
+	ui32Value = (psTimingData->ui32VBackPorch << VFETCH_SHIFT) | (psTimingData->ui32VFrontPorch << VEVENT_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT, ui32Value);
+
+	/* Enable sync gen last and set up polarities of sync/blank */
+	ui32Value = 0x1 << SYNCACTIVE_SHIFT |
+		0x1 << FIELDPOL_SHIFT |
+		0x1 << BLNKPOL_SHIFT |
+		0x1 << VSPOL_SHIFT |
+		0x1 << HSPOL_SHIFT;
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, ui32Value);
+
+	/* Turn on memory requests */
+	DCPDPEnableMemoryRequest(psDeviceData, DCPDPGetModuleParameters()->ui32PDPEnabled);
+}
+
+static PVRSRV_ERROR InitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_BUFFER *psBuffer;
+
+	DC_ASSERT(psDeviceData->psSystemBuffer == NULL);
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psDeviceData->ePixelFormat;
+	psBuffer->ui32Width			= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psBuffer->ui32Height		= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * DCPDP_PIXEL_FORMAT_BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32) DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height,
+	          DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	/* Initialise the system buffer to a nice gradient. */
+	ClearSystemBuffer(psBuffer);
+
+	psDeviceData->psSystemBuffer = psBuffer;
+
+	return PVRSRV_OK;
+}
+
+static void DeInitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+	{
+		if (DC_OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) != 0)
+		{
+			printk(KERN_INFO "%s: References to psSystemBuffer still exist"
+				   " (%d)", __func__, DC_OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount));
+		}
+		DC_OSAtomicWrite(&psDeviceData->psSystemBuffer->i32RefCount, 0);
+	}
+#else
+	{
+		DC_ASSERT(DC_OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) == 0);
+	}
+#endif
+
+	BufferReleaseMemory(psDeviceData, psDeviceData->psSystemBuffer);
+
+	DC_OSFreeMem(psDeviceData->psSystemBuffer);
+	psDeviceData->psSystemBuffer = NULL;
+}
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+static void GTFCalcFromRefresh(IMG_UINT32 ui32Width, IMG_UINT32 ui32Height,
+			       IMG_UINT32 ui32VRefresh, DCPDP_TIMING_DATA *psTimingData)
+{
+	double fVtFreq, fHzFreq;
+	double fFieldPeriod, fHzPeriod, fVtFreqEst, fDutyCycle, fClockFreq;
+	double fGTF_Mdash, fGTF_Cdash;
+	unsigned long ulVTotal, ulHTotal;
+	unsigned long ulVSyncBP, ulVBP;
+	unsigned long ulHSyncWidth, ulHFrontPorch, ulHBlank;
+	unsigned long ulVSyncMin;
+
+	DC_OSFloatingPointBegin();
+
+	if ((ui32Width * 3UL) == (ui32Height * 4UL))
+	{
+		ulVSyncMin = 4;
+	}
+	else if ((ui32Width * 9UL) == (ui32Height * 16UL))
+	{
+		ulVSyncMin = 5;
+	}
+	else if ((ui32Width * 10UL) == (ui32Height * 16UL))
+	{
+		ulVSyncMin = 6;
+	}
+	else if ((ui32Width * 4UL) == (ui32Height * 5UL))
+	{
+		ulVSyncMin = 7;
+	}
+	else if ((ui32Width * 9UL) == (ui32Height * 15UL))
+	{
+		ulVSyncMin = 7;
+	}
+	else
+	{
+		ulVSyncMin = GTF_VSYNC_WIDTH;
+	}
+
+	fGTF_Mdash = (fGTF_K * fGTF_M) / 256.0;
+	fGTF_Cdash = (((fGTF_C - fGTF_J) * fGTF_K) / 256) + fGTF_J;
+
+	/* Comes in as Hz */
+	fVtFreq = (double)ui32VRefresh;
+
+	/* Refresh rates lower than 0 and higher than 250Hz are nonsense */
+	if (fVtFreq > 250.0)
+	{
+		fVtFreq = 250.0;
+	}
+	else if (fVtFreq <= 0)
+	{
+		fVtFreq = 60.0;
+	}
+
+	/* Calculate Field Vertical Period in uS */
+	fFieldPeriod = 1000000.0f / fVtFreq;
+
+	/* Estimate Hz Period (in uS) & hence lines in (VSync + BP) & lines in Vt BP */
+	fHzPeriod = (fFieldPeriod - fGTF_MIN_VSYNCBP) / (ui32Height + fGTF_MIN_PORCH);
+
+	ulVSyncBP = (unsigned long)(fGTF_MIN_VSYNCBP / fHzPeriod);
+
+	ulVBP = ulVSyncBP - ulVSyncMin;
+	ulVTotal = ui32Height + ulVSyncBP + GTF_MIN_PORCH;
+
+	/* Estimate Vt frequency (in Hz) */
+	fVtFreqEst = 1000000.0f / (fHzPeriod * ulVTotal);
+
+	/* Find Actual Hz Period (in uS) */
+	fHzPeriod = (fHzPeriod * fVtFreqEst) / fVtFreq;
+
+	/* Find actual Vt frequency (in Hz) */
+	fVtFreq = 1000000.0f / (fHzPeriod * ulVTotal);
+
+	/* Find ideal blanking duty cycle from duty cycle equation and therefore
+	   number of pixels in blanking period (to nearest dbl char cell) */
+	fDutyCycle = fGTF_Cdash - ((fGTF_Mdash * fHzPeriod) / 1000.0f);
+
+	ulHBlank = (unsigned long)(((ui32Width * fDutyCycle) / (100.0f - fDutyCycle) / fGTF_CELL_GRAN2) + 0.5);
+	ulHBlank = ulHBlank * GTF_CELL_GRAN2;
+
+	/* Find HTotal & pixel clock(MHz) & Hz Frequency (KHz) */
+	ulHTotal = ui32Width + ulHBlank;
+	fClockFreq = (double)ulHTotal / fHzPeriod;
+	fHzFreq = 1000.0f / fHzPeriod;
+
+	ulHSyncWidth = (unsigned long)(((ulHTotal * fGTF_HSYNC_MARGIN) / (100.0 * fGTF_CELL_GRAN)) + 0.5);
+	ulHSyncWidth *= GTF_CELL_GRAN;
+
+	ulHFrontPorch = (ulHBlank >> 1) - ulHSyncWidth;
+	if (ulHFrontPorch < GTF_MIN_PORCH_PIXELS)
+	{
+		ulHFrontPorch = GTF_MIN_PORCH_PIXELS;
+	}
+
+	psTimingData->ui32HDisplay	= ui32Width;
+	psTimingData->ui32HBackPorch	= ulHSyncWidth;
+	psTimingData->ui32HTotal	= ulHTotal;
+	psTimingData->ui32HActiveStart	= ulHBlank - ulHFrontPorch;
+	psTimingData->ui32HLeftBorder	= psTimingData->ui32HActiveStart;
+	psTimingData->ui32HRightBorder	= psTimingData->ui32HLeftBorder + ui32Width;
+	psTimingData->ui32HFrontPorch	= psTimingData->ui32HRightBorder;
+
+	psTimingData->ui32VDisplay	= ui32Height;
+	psTimingData->ui32VBackPorch	= GTF_VSYNC_WIDTH;
+	psTimingData->ui32VTotal	= ulVTotal;
+	psTimingData->ui32VActiveStart	= ulVSyncBP;
+	psTimingData->ui32VTopBorder	= psTimingData->ui32VActiveStart;
+	psTimingData->ui32VBottomBorder = psTimingData->ui32VTopBorder + ui32Height;
+	psTimingData->ui32VFrontPorch	= psTimingData->ui32VBottomBorder;
+
+	psTimingData->ui32VRefresh	= (IMG_UINT32)fVtFreq;
+
+	/* scale, round and then scale */
+	psTimingData->ui32ClockFreq = ((unsigned long)(fClockFreq * 1000.0)) * 1000UL;
+
+	DC_OSFloatingPointEnd();
+}
+#endif /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+
+PVRSRV_ERROR DCPDPInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+		       const DC_SERVICES_FUNCS *psPVRServicesFuncs,
+		       DCPDP_DEVICE **ppsDeviceData)
+{
+	DCPDP_DEVICE *psDeviceData;
+	IMG_UINT64 ui64BufferCount;
+	PVRSRV_ERROR eError;
+	PHYS_HEAP *psLMAPhysHeap;
+	PPVRSRV_DEVICE_NODE psDevNode;
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+	IMG_UINT32 i;
+#endif
+
+	if (psPVRServicesFuncs == NULL || ppsDeviceData == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Invalid arguments %p %p\n", __func__, psPVRServicesFuncs, ppsDeviceData);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DC_OSSetDrvName(DRVNAME);
+
+	/* Request 0th device node as this module only supports systems with 1 GPU device  */
+	psDevNode = psPVRServicesFuncs->pfnGetDeviceInstance(0);
+	PVR_RETURN_IF_INVALID_PARAM(psDevNode != NULL);
+
+	psDeviceData = DC_OSCallocMem(sizeof(*psDeviceData));
+	if (psDeviceData == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to allocate device data\n", __func__);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psDeviceData->psDevicePriv = psDevicePriv;
+	psDeviceData->psPVRServicesFuncs = psPVRServicesFuncs;
+	psDeviceData->ePixelFormat = DCPDP_PIXEL_FORMAT;
+
+	/* Services manages the card memory so we need to acquire the Local Memory heap and display controller
+	   heap (which is a carve out of the card memory) so we can allocate our own buffers.
+	 */
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapAcquireByUsage((1 << psDevNode->psDevConfig->eDefaultHeap),
+																		 psDevNode,
+																		 &psLMAPhysHeap);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to acquire Local memory heap (%s)\n",
+		                 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeDeviceData;
+	}
+
+	/* Display controller uses local memory offsets instead of CPU Physical mapped Address.
+	 * The local memory heap CPU physical Address is required to calculate the address offset
+	 * used by the display controller.
+	 * Get the CPU base address for the Local memory heap
+	 */
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetCpuPAddr(psLMAPhysHeap,
+										&psDeviceData->sLMACPUPhysBaseAddr);
+	/* Release the acquired heap once the base address is known */
+	psDeviceData->psPVRServicesFuncs->pfnPhysHeapRelease(psLMAPhysHeap);
+
+	if (eError != PVRSRV_OK)
+	{
+
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get Local memory base address (%s)\n",
+		                 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeDeviceData;
+
+	}
+
+	DC_OSDebugPrintf(DBGLVL_DEBUG, " LMA Phys Heap Base: 0x%pX\n",
+	                 psDeviceData->sLMACPUPhysBaseAddr.uiAddr);
+
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapAcquireByUsage(PHYS_HEAP_USAGE_DISPLAY,
+																		 psDevNode,
+																		 &psDeviceData->psPhysHeap);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to acquire heap (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeDeviceData;
+	}
+
+	/* The PDP can only access local memory so make sure the heap is of the correct type */
+	if (psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetType(psDeviceData->psPhysHeap) != PHYS_HEAP_TYPE_LMA)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Heap is of the wrong type\n", __func__);
+		eError = PVRSRV_ERROR_INVALID_HEAP;
+		goto ErrorPhysHeapRelease;
+	}
+
+	/* Get the CPU base address for the display heap */
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetCpuPAddr(psDeviceData->psPhysHeap,
+										&psDeviceData->sDispMemCpuPAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get display memory base address (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetSize(psDeviceData->psPhysHeap,
+									    &psDeviceData->uiDispMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get the display memory size (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	/* Reserve and map the PDP registers */
+	psDeviceData->sPDPRegCpuPAddr = DCPDPGetAddrRangeStart(psDeviceData->psDevicePriv,
+							       DCPDP_ADDRESS_RANGE_PDP);
+	psDeviceData->ui32PDPRegSize = DCPDPGetAddrRangeSize(psDeviceData->psDevicePriv,
+							     DCPDP_ADDRESS_RANGE_PDP);
+
+	if (DC_OSRequestAddrRegion(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize, DRVNAME) == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to reserve the PDP registers\n", __func__);
+
+		psDeviceData->sPDPRegCpuPAddr.uiAddr = 0;
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+
+		goto ErrorPhysHeapRelease;
+	}
+
+	psDeviceData->pvPDPRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize);
+	if (psDeviceData->pvPDPRegCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to map PDP registers\n", __func__);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+	psDeviceData->uiTimingDataSize = ARRAY_SIZE(gasVesaModes);
+	psDeviceData->pasTimingData = DC_OSAllocMem(sizeof(DCPDP_TIMING_DATA) * psDeviceData->uiTimingDataSize);
+
+	if (psDeviceData->pasTimingData	== NULL)
+	{
+		goto ErrorPDPOSUnmapPDPRegisters;
+	}
+
+	/* fill array with timing data for VESA standard modes */
+	for (i = 0; i < psDeviceData->uiTimingDataSize; i++)
+	{
+		if (gasVesaModes[i].uiWidth == DCPDPGetModuleParameters()->ui32PDPWidth &&
+		    gasVesaModes[i].uiHeight == DCPDPGetModuleParameters()->ui32PDPHeight)
+		{
+			psDeviceData->uiTimingDataIndex = i;
+		}
+
+		/* Setup timing data */
+		GTFCalcFromRefresh(gasVesaModes[i].uiWidth,
+				   gasVesaModes[i].uiHeight,
+				   60,
+				   &psDeviceData->pasTimingData[i]);
+	}
+
+#else /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+	/* The default PLL setup is for 1280 x 1024 at 60Hz. The default PLL clock
+	   is in flash memory and will be loaded from flash on reset */
+
+	psDeviceData->uiTimingDataSize = 1;
+	psDeviceData->pasTimingData = DC_OSAllocMem(sizeof(DCPDP_TIMING_DATA) * psDeviceData->uiTimingDataSize);
+
+	if (psDeviceData->pasTimingData	== NULL)
+	{
+		goto ErrorPDPOSUnmapPDPRegisters;
+	}
+
+	psDeviceData->uiTimingDataIndex = 0;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay	= 1280;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HTotal		= 1712;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HBackPorch	= 136;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HActiveStart	= 352;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HLeftBorder	= 352;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HFrontPorch	= 1632;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HRightBorder	= 1632;
+
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay	= 1024;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VTotal		= 1059;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VBackPorch	= 7;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VActiveStart	= 34;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VTopBorder	= 34;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VFrontPorch	= 1058;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VBottomBorder	= 1058;
+
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh	= 60;
+#endif /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+
+	/* Setup simple buffer allocator */
+	psDeviceData->ui32BufferSize = (IMG_UINT32)DC_ALIGN(psDeviceData->pasTimingData[psDeviceData->uiTimingDataSize - 1].ui32HDisplay *
+							    psDeviceData->pasTimingData[psDeviceData->uiTimingDataSize - 1].ui32VDisplay *
+							    DCPDP_PIXEL_FORMAT_BPP,
+							    DC_OSGetPageSize());
+
+	ui64BufferCount = psDeviceData->uiDispMemSize;
+	ui64BufferCount = DC_OSDiv64(ui64BufferCount, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferCount = (IMG_UINT32)ui64BufferCount;
+	if (psDeviceData->ui32BufferCount == 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Not enough space for the framebuffer\n", __func__);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorFreeTimingData;
+	}
+	else if (psDeviceData->ui32BufferCount > DCPDP_MAX_BUFFERS)
+	{
+		psDeviceData->ui32BufferCount = DCPDP_MAX_BUFFERS;
+	}
+
+	psDeviceData->ui32BufferUseMask = 0;
+
+	/* Create a system buffer */
+	eError = InitSystemBuffer(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialise the system buffer (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeTimingData;
+	}
+
+	SetModeRegisters(psDeviceData);
+	psDeviceData->bVSyncEnabled = IMG_TRUE;
+	Flip(psDeviceData->psSystemBuffer);
+
+	eError = psDeviceData->psPVRServicesFuncs->pfnDCRegisterDevice(&g_sDCFunctions,
+								       DCPDP_MAX_COMMANDS_INFLIGHT,
+								       psDeviceData,
+								       &psDeviceData->hPVRServicesDevice);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to register the display device (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDestroySystemBuffer;
+	}
+
+	/* Register our handler */
+	eError = DCPDPInstallDeviceLISR(psDeviceData->psDevicePriv,
+					DisplayLISRHandler,
+					psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to install the display device interrupt handler (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDCUnregisterDevice;
+	}
+
+	eError = DC_OSWorkQueueCreate(&psDeviceData->hVSyncWorkQueue, 1);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work queue (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDCUninstallLISR;
+	}
+
+	eError = DC_OSWorkQueueCreateWorkItem(&psDeviceData->hVSyncWorkItem, VSyncProcessor, psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work item (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDCDestroyWorkQueue;
+	}
+
+	/* Print some useful information */
+	DC_OSDebugPrintf(DBGLVL_INFO, " using mode %ux%u@%uHz (phys: %ux%u)\n",
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh,
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay * 1000) / DCPDP_DPI * 254) / 10000),
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay * 1000) / DCPDP_DPI * 254) / 10000));
+	DC_OSDebugPrintf(DBGLVL_INFO, " register base: 0x%px\n", psDeviceData->sPDPRegCpuPAddr.uiAddr);
+	DC_OSDebugPrintf(DBGLVL_INFO, " memory base: 0x%px\n", psDeviceData->sDispMemCpuPAddr.uiAddr);
+
+	*ppsDeviceData = psDeviceData;
+
+	return PVRSRV_OK;
+
+ErrorDCDestroyWorkQueue:
+	DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+ErrorDCUninstallLISR:
+	DCPDPUninstallDeviceLISR(psDevicePriv);
+
+ErrorDCUnregisterDevice:
+	psDeviceData->psPVRServicesFuncs->pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+
+ErrorDestroySystemBuffer:
+	DeInitSystemBuffer(psDeviceData);
+
+ErrorFreeTimingData:
+	DC_OSFreeMem(psDeviceData->pasTimingData);
+
+ErrorPDPOSUnmapPDPRegisters:
+	DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, psDeviceData->ui32PDPRegSize);
+
+ErrorReleaseMemRegion:
+	DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize);
+
+ErrorPhysHeapRelease:
+	psDeviceData->psPVRServicesFuncs->pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+ErrorFreeDeviceData:
+	DC_OSFreeMem(psDeviceData);
+
+	return eError;
+}
+
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, DCPDP_DEVICE_PRIV **ppsDevicePriv)
+{
+	if (psDeviceData)
+	{
+		DC_OSWorkQueueDestroyWorkItem(psDeviceData->hVSyncWorkItem);
+		DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+		psDeviceData->psPVRServicesFuncs->pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+		{
+			psDeviceData->psFlipContext = NULL;
+		}
+		else
+#else
+		{
+			DC_ASSERT(psDeviceData->psFlipContext == NULL);
+		}
+#endif
+		(void)DCPDPUninstallDeviceLISR(psDeviceData->psDevicePriv);
+
+		DeInitSystemBuffer(psDeviceData);
+
+		/* Reset the register to make sure the PDP doesn't try to access on card memory */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, 0x00000000);
+
+		DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, psDeviceData->ui32PDPRegSize);
+
+		DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize);
+
+		psDeviceData->psPVRServicesFuncs->pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+		if (ppsDevicePriv != NULL)
+		{
+			*ppsDevicePriv = psDeviceData->psDevicePriv;
+		}
+
+		DC_OSFreeMem(psDeviceData->pasTimingData);
+		DC_OSFreeMem(psDeviceData);
+	}
+}
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable)
+{
+	if (psDeviceData)
+	{
+		IMG_UINT32 ui32Value;
+
+		ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL);
+		ui32Value &= ~(STR1STREN_MASK);
+
+		if (bEnable)
+		{
+			ui32Value |= 0x1 << STR1STREN_SHIFT;
+		}
+
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, ui32Value);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..3af6db74f779
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Kbuild.mk
@@ -0,0 +1,59 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-$(CONFIG_X86) += -mno-soft-float
+
+ccflags-y += \
+	-I$(TOP)/services/display/rogue/dc_pdp/include \
+	-I$(TOP)/services/display/include \
+	-I$(TOP)/include/rogue/system/rgx_tc -I$(TOP)/include/system/rgx_tc	\
+	-I$(TOP)/include/rogue/public -I$(TOP)/include/public
+
+dc_pdp-y += \
+	services/display/rogue/dc_pdp/common/dc_pdp_common.o \
+	services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.o	\
+	services/display/common/env/linux/dc_osfuncs.o
+
+ifeq ($(PVR_SYSTEM),rgx_linux_tc)
+ccflags-y += \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc
+
+CFLAGS_services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.o := -DDCPDP_REGISTER_DRIVER
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Linux.mk
new file mode 100644
index 000000000000..5c5a1f234723
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Linux.mk
@@ -0,0 +1,51 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+ifeq ($(PVR_ARCH),rogue)
+modules := dc_pdp
+
+dc_pdp_type := kernel_module
+dc_pdp_target := dc_pdp.ko
+dc_pdp_makefile := $(THIS_DIR)/Kbuild.mk
+endif
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.c
new file mode 100644
index 000000000000..aaeda41c9ed6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.c
@@ -0,0 +1,682 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+#if defined(DCPDP_REGISTER_DRIVER)
+#include "pdp_common.h"
+#include "pdp_regs.h"
+#include "tc_drv.h"
+#endif
+
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#include "dc_pdp.h"
+#include "img_defs.h"
+
+#define DCPDP_WIDTH_MIN			(640)
+#define DCPDP_WIDTH_MAX			(1280)
+#define DCPDP_HEIGHT_MIN		(480)
+#define DCPDP_HEIGHT_MAX		(1024)
+
+#define DCPDP_DEBUGFS_DISPLAY_ENABLED	"display_enabled"
+
+#if defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_WIDTH defined but DCPDP_HEIGHT not defined
+#elif !defined(DCPDP_WIDTH) && defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_HEIGHT defined but DCPDP_WIDTH not defined
+#elif !defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#define DCPDP_WIDTH			DCPDP_WIDTH_MAX
+#define DCPDP_HEIGHT			DCPDP_HEIGHT_MAX
+#elif (DCPDP_WIDTH > DCPDP_WIDTH_MAX)
+#error ERROR: DCPDP_WIDTH too large (max: 1280)
+#elif (DCPDP_WIDTH < DCPDP_WIDTH_MIN)
+#error ERROR: DCPDP_WIDTH too small (min: 640)
+#elif (DCPDP_HEIGHT > DCPDP_HEIGHT_MAX)
+#error ERROR: DCPDP_HEIGHT too large (max: 1024)
+#elif (DCPDP_HEIGHT < DCPDP_HEIGHT_MIN)
+#error ERROR: DCPDP_HEIGHT too small (max: 480)
+#endif
+
+struct DCPDP_DEVICE_PRIV_TAG
+{
+	struct device		*psDev;
+	DCPDP_DEVICE		*psPDPDevice;
+
+	IMG_HANDLE		hServicesConnection;
+	DC_SERVICES_FUNCS	sServicesFuncs;
+
+	IMG_CPU_PHYADDR		sPDPRegCpuPAddr;
+	IMG_UINT32		ui32PDPRegSize;
+	IMG_CPU_PHYADDR		sPLLRegCpuPAddr;
+	IMG_UINT32		ui32PLLRegSize;
+
+	struct dentry		*psDebugFSEntryDir;
+	struct dentry		*psDisplayEnabledEntry;
+
+#if defined(DCPDP_REGISTER_DRIVER)
+	PFN_LISR		pfnLISR;
+	void			*pvLISRData;
+#else
+	IMG_HANDLE		hLISRData;
+#endif
+};
+
+#if !defined(DCPDP_REGISTER_DRIVER)
+static DCPDP_DEVICE_PRIV *g_psDevicePriv;
+#endif
+
+/* PDP module parameters */
+DCPDP_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32PDPEnabled = 1,
+	.ui32PDPWidth   = DCPDP_WIDTH,
+	.ui32PDPHeight  = DCPDP_HEIGHT
+};
+module_param_named(mem_en, sModuleParams.ui32PDPEnabled, uint, S_IRUGO | S_IWUSR);
+module_param_named(width,  sModuleParams.ui32PDPWidth,   uint, S_IRUGO | S_IWUSR);
+module_param_named(height, sModuleParams.ui32PDPHeight,  uint, S_IRUGO | S_IWUSR);
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void)
+{
+	return &sModuleParams;
+}
+
+IMG_CPU_PHYADDR DCPDPGetAddrRangeStart(DCPDP_DEVICE_PRIV *psDevicePriv,
+				       DCPDP_ADDRESS_RANGE eRange)
+{
+	switch (eRange)
+	{
+		default:
+			DC_ASSERT(!"Unsupported address range");
+			__fallthrough;
+		case DCPDP_ADDRESS_RANGE_PDP:
+			return psDevicePriv->sPDPRegCpuPAddr;
+		case DCPDP_ADDRESS_RANGE_PLL:
+			return psDevicePriv->sPLLRegCpuPAddr;
+	}
+}
+
+IMG_UINT32 DCPDPGetAddrRangeSize(DCPDP_DEVICE_PRIV *psDevicePriv,
+				 DCPDP_ADDRESS_RANGE eRange)
+{
+	switch (eRange)
+	{
+		default:
+			DC_ASSERT(!"Unsupported address range");
+			__fallthrough;
+		case DCPDP_ADDRESS_RANGE_PDP:
+			return psDevicePriv->ui32PDPRegSize;
+		case DCPDP_ADDRESS_RANGE_PLL:
+			return psDevicePriv->ui32PLLRegSize;
+	}
+}
+
+#if defined(DCPDP_REGISTER_DRIVER)
+static void DCPDPLISRHandlerWrapper(void *pvData)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv = pvData;
+
+	if (psDevicePriv->pfnLISR != NULL)
+	{
+		(void) psDevicePriv->pfnLISR(psDevicePriv->pvLISRData);
+	}
+}
+#endif
+
+PVRSRV_ERROR DCPDPInstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv,
+				    PFN_LISR pfnLISR, void *pvData)
+{
+#if defined(DCPDP_REGISTER_DRIVER)
+	struct device *psParentDev = psDevicePriv->psDev->parent;
+	int iErr;
+
+	if (psDevicePriv->pfnLISR != NULL)
+	{
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+
+	psDevicePriv->pfnLISR = pfnLISR;
+	psDevicePriv->pvLISRData = pvData;
+
+	iErr = tc_set_interrupt_handler(psParentDev,
+					TC_INTERRUPT_PDP,
+					DCPDPLISRHandlerWrapper,
+					psDevicePriv);
+	if (iErr)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to install interrupt handler (err=%d)\n",
+		       __func__, iErr);
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	iErr = tc_enable_interrupt(psParentDev, TC_INTERRUPT_PDP);
+	if (iErr)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to enable interrupts (err=%d)\n",
+		       __func__, iErr);
+		tc_set_interrupt_handler(psParentDev, TC_INTERRUPT_PDP, NULL, NULL);
+		psDevicePriv->pfnLISR = NULL;
+		psDevicePriv->pvLISRData = NULL;
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	return PVRSRV_OK;
+#else
+	if (psDevicePriv->hLISRData != NULL)
+	{
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+
+	return psDevicePriv->sServicesFuncs.pfnSysInstallDeviceLISR(psDevicePriv->psDev,
+								    DCPDP_INTERRUPT_ID,
+								    DRVNAME,
+								    pfnLISR,
+								    pvData,
+								    &psDevicePriv->hLISRData);
+#endif
+}
+
+PVRSRV_ERROR DCPDPUninstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+#if defined(DCPDP_REGISTER_DRIVER)
+	struct device *psParentDev = psDevicePriv->psDev->parent;
+	int iErr;
+
+	if (psDevicePriv->pfnLISR == NULL)
+	{
+		return PVRSRV_ERROR_ISR_NOT_INSTALLED;
+	}
+
+	tc_disable_interrupt(psParentDev, TC_INTERRUPT_PDP);
+
+	iErr = tc_set_interrupt_handler(psParentDev, TC_INTERRUPT_PDP, NULL, NULL);
+	if (iErr)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to uninstall interrupt handler (err=%d)\n",
+		       __func__, iErr);
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psDevicePriv->pfnLISR = NULL;
+	psDevicePriv->pvLISRData = NULL;
+
+	return PVRSRV_OK;
+#else
+	PVRSRV_ERROR eError;
+
+	if (psDevicePriv->hLISRData == NULL)
+	{
+		return PVRSRV_ERROR_ISR_NOT_INSTALLED;
+	}
+
+	eError = PVRSRVSystemUninstallDeviceLISR(psDevicePriv->hLISRData);
+	if (eError == PVRSRV_OK)
+	{
+		psDevicePriv->hLISRData = NULL;
+	}
+
+	return eError;
+#endif
+}
+
+static int DisplayEnabledOpen(struct inode *psINode, struct file *psFile)
+{
+	psFile->private_data = psINode->i_private;
+
+	return 0;
+}
+
+static ssize_t DisplayEnabledRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	loff_t uiPosition = *puiPosition;
+	char pszBuffer[] = "N\n";
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	int iErr;
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if (uiPosition >= uiBufferSize || uiCount == 0)
+	{
+		return 0;
+	}
+
+	if (sModuleParams.ui32PDPEnabled)
+	{
+		pszBuffer[0] = 'Y';
+	}
+
+	if (uiCount > uiBufferSize - uiPosition)
+	{
+		uiCount = uiBufferSize - uiPosition;
+	}
+
+	iErr = copy_to_user(psUserBuffer, &pszBuffer[uiPosition], uiCount);
+	if (iErr)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + uiCount;
+
+	return uiCount;
+}
+
+static ssize_t DisplayEnabledWrite(struct file *psFile,
+				   const char __user *psUserBuffer,
+				   size_t uiCount,
+				   loff_t *puiPosition)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv = psFile->private_data;
+	char pszBuffer[3];
+	bool bPDPEnabled;
+	int iErr;
+
+	uiCount = min(uiCount, ARRAY_SIZE(pszBuffer) - 1);
+
+	iErr = copy_from_user(pszBuffer, psUserBuffer, uiCount);
+	if (iErr)
+	{
+		return -EFAULT;
+	}
+
+	pszBuffer[uiCount] = '\0';
+
+	if (strtobool(pszBuffer, &bPDPEnabled) == 0)
+	{
+		sModuleParams.ui32PDPEnabled = bPDPEnabled ? 1 : 0;
+
+		DCPDPEnableMemoryRequest(psDevicePriv->psPDPDevice, bPDPEnabled);
+	}
+
+	return uiCount;
+}
+
+static const struct file_operations gsDisplayEnabledFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = DisplayEnabledOpen,
+	.read = DisplayEnabledRead,
+	.write = DisplayEnabledWrite,
+	.llseek = default_llseek,
+};
+
+static void DCPDPDebugFSInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	psDevicePriv->psDebugFSEntryDir = debugfs_create_dir(DRVNAME, NULL);
+	if (IS_ERR_OR_NULL(psDevicePriv->psDebugFSEntryDir))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs root directory "
+		       "(debugfs entries won't be available)\n", __func__, DRVNAME);
+		psDevicePriv->psDebugFSEntryDir = NULL;
+		return;
+	}
+
+	psDevicePriv->psDisplayEnabledEntry =
+		debugfs_create_file(DCPDP_DEBUGFS_DISPLAY_ENABLED,
+				    S_IFREG | S_IRUGO | S_IWUSR,
+				    psDevicePriv->psDebugFSEntryDir,
+				    psDevicePriv,
+				    &gsDisplayEnabledFileOps);
+	if (IS_ERR_OR_NULL(psDevicePriv->psDisplayEnabledEntry))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs entry\n",
+		       __func__, DCPDP_DEBUGFS_DISPLAY_ENABLED);
+		psDevicePriv->psDisplayEnabledEntry = NULL;
+		return;
+	}
+}
+
+static void DCPDPDebugFSDeInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	if (psDevicePriv->psDisplayEnabledEntry != NULL)
+	{
+		debugfs_remove(psDevicePriv->psDisplayEnabledEntry);
+		psDevicePriv->psDisplayEnabledEntry = NULL;
+	}
+
+	if (psDevicePriv->psDebugFSEntryDir != NULL)
+	{
+		debugfs_remove(psDevicePriv->psDebugFSEntryDir);
+		psDevicePriv->psDebugFSEntryDir = NULL;
+	}
+}
+
+
+static PVRSRV_ERROR DCPDPServicesInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DC_OSPVRServicesConnectionOpen(&psDevicePriv->hServicesConnection);
+	if (eError != PVRSRV_OK)
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to open connection to PVR Services (%d)\n",
+		       __func__, eError);
+		return eError;
+	}
+
+	eError = DC_OSPVRServicesSetupFuncs(psDevicePriv->hServicesConnection,
+					    &psDevicePriv->sServicesFuncs);
+	if (eError != PVRSRV_OK)
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to setup PVR Services function table (%d)\n",
+		       __func__, eError);
+		goto ErrorServicesConnectionClose;
+	}
+
+	return PVRSRV_OK;
+
+ErrorServicesConnectionClose:
+	DC_OSPVRServicesConnectionClose(psDevicePriv->hServicesConnection);
+	psDevicePriv->hServicesConnection = NULL;
+
+	return eError;
+}
+
+static void DCPDPServicesDeInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	DC_OSMemSet(&psDevicePriv->sServicesFuncs, 0,
+		    sizeof(psDevicePriv->sServicesFuncs));
+
+	DC_OSPVRServicesConnectionClose(psDevicePriv->hServicesConnection);
+	psDevicePriv->hServicesConnection = NULL;
+}
+
+static int DCPDPPCIDevicePrivInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+				  struct pci_dev *psPCIDev)
+{
+	IMG_UINT32 ui32RegBaseAddr = DC_OSAddrRangeStart(&psPCIDev->dev,
+							 DCPDP_REG_PCI_BASENUM);
+
+	psDevicePriv->sPDPRegCpuPAddr.uiAddr = ui32RegBaseAddr + DCPDP_PCI_PDP_REG_OFFSET;
+	psDevicePriv->ui32PDPRegSize = DCPDP_PCI_PDP_REG_SIZE;
+
+	psDevicePriv->sPLLRegCpuPAddr.uiAddr = ui32RegBaseAddr + DCPDP_PCI_PLL_REG_OFFSET;
+	psDevicePriv->ui32PLLRegSize = DCPDP_PCI_PLL_REG_SIZE;
+
+	return 0;
+}
+
+static int DCPDPPlatformDevicePrivInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+				       struct platform_device *psPlatDev)
+{
+	struct resource *psRegs;
+
+	psRegs = platform_get_resource_byname(psPlatDev, IORESOURCE_MEM, "pdp-regs");
+	if (psRegs == NULL)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to get PDP registers\n",
+			   __func__);
+		return -ENXIO;
+	}
+
+	psDevicePriv->sPDPRegCpuPAddr.uiAddr =
+		IMG_CAST_TO_CPUPHYADDR_UINT(psRegs->start);
+	psDevicePriv->ui32PDPRegSize = (IMG_UINT32) resource_size(psRegs);
+
+	psRegs = platform_get_resource_byname(psPlatDev, IORESOURCE_MEM, "pll-regs");
+	if (psRegs == NULL)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to get PLL registers\n",
+			   __func__);
+		return -ENXIO;
+	}
+
+	psDevicePriv->sPLLRegCpuPAddr.uiAddr =
+		IMG_CAST_TO_CPUPHYADDR_UINT(psRegs->start);
+	psDevicePriv->ui32PLLRegSize = (IMG_UINT32) resource_size(psRegs);
+
+	return 0;
+}
+
+static DCPDP_DEVICE_PRIV *DCPDPDevicePrivCreate(struct device *psDev)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+	PVRSRV_ERROR eError;
+	int iRet;
+
+	psDevicePriv = DC_OSCallocMem(sizeof(*psDevicePriv));
+	if (psDevicePriv == NULL)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to allocate device private data\n",
+			   __func__);
+		iRet = -ENOMEM;
+		goto ErrorReturn;
+	}
+
+	psDevicePriv->psDev = psDev;
+
+	eError = DCPDPServicesInit(psDevicePriv);
+	if (eError != PVRSRV_OK)
+	{
+		iRet = -ENODEV;
+		goto ErrorFreeDevicePriv;
+	}
+
+	if (dev_is_pci(psDev))
+	{
+		iRet = DCPDPPCIDevicePrivInit(psDevicePriv, to_pci_dev(psDev));
+	}
+	else
+	{
+		iRet = DCPDPPlatformDevicePrivInit(psDevicePriv,
+						   to_platform_device(psDev));
+	}
+
+	if (iRet != 0)
+	{
+		goto ErrorServicesDeInit;
+	}
+
+	eError = DCPDPInit(psDevicePriv,
+			   &psDevicePriv->sServicesFuncs,
+			   &psDevicePriv->psPDPDevice);
+	if (eError != PVRSRV_OK)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to initialise device (%d)\n",
+			   __func__, eError);
+		iRet = -ENODEV;
+		goto ErrorServicesDeInit;
+	}
+
+	DCPDPDebugFSInit(psDevicePriv);
+
+	return psDevicePriv;
+
+ErrorServicesDeInit:
+	DCPDPServicesDeInit(psDevicePriv);
+ErrorFreeDevicePriv:
+	DC_OSFreeMem(psDevicePriv);
+ErrorReturn:
+	return ERR_PTR(iRet);
+}
+
+static void DCPDPDevicePrivDestroy(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	DCPDPDebugFSDeInit(psDevicePriv);
+	DCPDPDeInit(psDevicePriv->psPDPDevice, NULL);
+	DCPDPServicesDeInit(psDevicePriv);
+	DC_OSFreeMem(psDevicePriv);
+}
+
+#if defined(DCPDP_REGISTER_DRIVER)
+
+static int DCPDPProbe(struct platform_device *psPlatDev)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+	int iRet;
+
+	iRet = tc_enable(psPlatDev->dev.parent);
+	if (iRet)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to enable device (err=%d)\n",
+			   __func__, iRet);
+		return iRet;
+	}
+
+	psDevicePriv = DCPDPDevicePrivCreate(&psPlatDev->dev);
+	if (IS_ERR(psDevicePriv))
+	{
+		tc_disable(psPlatDev->dev.parent);
+		return PTR_ERR(psDevicePriv);
+	}
+
+	platform_set_drvdata(psPlatDev, psDevicePriv);
+
+	return 0;
+}
+
+static int DCPDPRemove(struct platform_device *psPlatDev)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+
+	psDevicePriv = platform_get_drvdata(psPlatDev);
+	if (WARN_ON(psDevicePriv == NULL))
+	{
+		return -ENODEV;
+	}
+
+	platform_set_drvdata(psPlatDev, NULL);
+
+	DCPDPDevicePrivDestroy(psDevicePriv);
+	tc_disable(psPlatDev->dev.parent);
+
+	return 0;
+}
+
+static void DCPDPShutdown(struct platform_device *psPlatDev)
+{
+	PVR_UNREFERENCED_PARAMETER(psPlatDev);
+}
+
+static struct platform_device_id DCPDPPlatformDeviceIDTable[] =
+{
+	{ .name = APOLLO_DEVICE_NAME_PDP, .driver_data = PDP_VERSION_APOLLO },
+	{ },
+};
+
+static struct platform_driver DCPDPPlatformDriver =
+{
+	.probe		= DCPDPProbe,
+	.remove		= DCPDPRemove,
+	.shutdown	= DCPDPShutdown,
+	.driver		=
+	{
+		.owner  = THIS_MODULE,
+		.name	= DRVNAME,
+	},
+	.id_table	= DCPDPPlatformDeviceIDTable,
+};
+
+module_platform_driver(DCPDPPlatformDriver);
+
+#else /* !defined(DCPDP_REGISTER_DRIVER) */
+
+static int __init dc_pdp_init(void)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+	struct pci_dev *psPCIDev;
+	int error;
+
+	psPCIDev = pci_get_device(DCPDP_VENDOR_ID_POWERVR, DCPDP_DEVICE_ID_PCI_APOLLO_FPGA, NULL);
+	if (psPCIDev == NULL)
+	{
+		psPCIDev = pci_get_device(DCPDP_VENDOR_ID_POWERVR, DCPDP_DEVICE_ID_PCIE_APOLLO_FPGA, NULL);
+		if (psPCIDev == NULL)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to get PCI device\n", __func__);
+			return -ENODEV;
+		}
+	}
+
+	error = pci_enable_device(psPCIDev);
+	if (error != 0)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to enable PCI device (%d)\n", __func__, error);
+		return -ENODEV;
+	}
+
+	psDevicePriv = DCPDPDevicePrivCreate(&psPCIDev->dev);
+
+	/* To prevent possible problems with system suspend/resume, we don't
+	   keep the device enabled, but rely on the fact that the Rogue driver
+	   will have done a pci_enable_device. */
+	pci_disable_device(psPCIDev);
+
+	if (IS_ERR(psDevicePriv))
+	{
+		return PTR_ERR(psDevicePriv);
+	}
+
+	g_psDevicePriv = psDevicePriv;
+
+	return 0;
+}
+
+static void __exit dc_pdp_deinit(void)
+{
+	if (g_psDevicePriv)
+	{
+		DCPDPDevicePrivDestroy(g_psDevicePriv);
+		g_psDevicePriv = NULL;
+	}
+}
+
+module_init(dc_pdp_init);
+module_exit(dc_pdp_deinit);
+
+#endif /* defined(DCPDP_REGISTER_DRIVER) */
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/include/dc_pdp.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/include/dc_pdp.h
new file mode 100644
index 000000000000..ccfeadafc1d7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/include/dc_pdp.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_PDP_H)
+#define DC_PDP_H
+
+#include "dc_osfuncs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define DRVNAME	"dc_pdp"
+
+#define DCPDP_INTERRUPT_ID		(1)
+
+#include "pdp_regs.h"
+
+/*************************************************************************/ /*!
+ dc_pdp OS functions
+*/ /**************************************************************************/
+
+typedef struct DCPDP_MODULE_PARAMETERS_TAG
+{
+	IMG_UINT32  ui32PDPEnabled;
+	IMG_UINT32  ui32PDPWidth;
+	IMG_UINT32  ui32PDPHeight;
+} DCPDP_MODULE_PARAMETERS;
+
+typedef enum DCPDP_ADDRESS_RANGE_TAG
+{
+	DCPDP_ADDRESS_RANGE_PDP = 0,
+	DCPDP_ADDRESS_RANGE_PLL,
+} DCPDP_ADDRESS_RANGE;
+
+typedef struct DCPDP_DEVICE_PRIV_TAG DCPDP_DEVICE_PRIV;
+
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void);
+
+IMG_CPU_PHYADDR DCPDPGetAddrRangeStart(DCPDP_DEVICE_PRIV *psDevicePriv,
+				       DCPDP_ADDRESS_RANGE eRange);
+IMG_UINT32 DCPDPGetAddrRangeSize(DCPDP_DEVICE_PRIV *psDevicePriv,
+				 DCPDP_ADDRESS_RANGE eRange);
+
+PVRSRV_ERROR DCPDPInstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv,
+				    PFN_LISR pfnLISR, void *pvData);
+PVRSRV_ERROR DCPDPUninstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv);
+
+/*******************************************************************************
+ * dc_pdp common functions
+ ******************************************************************************/
+
+typedef struct DCPDP_DEVICE_TAG DCPDP_DEVICE;
+
+PVRSRV_ERROR DCPDPInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+		       const DC_SERVICES_FUNCS *psServicesFuncs,
+		       DCPDP_DEVICE **ppsDeviceData);
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, DCPDP_DEVICE_PRIV **ppsDevicePriv);
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable);
+
+#endif /* !defined(DC_PDP_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/edid.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/edid.c
new file mode 100644
index 000000000000..4cdc79de0a9f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/edid.c
@@ -0,0 +1,1673 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+
+#include "edid.h"
+#include "phy.h"
+#include "img_defs.h"
+
+/* Section 2.12.4 of HDMI TX spec */
+const IMG_UINT32 I2C_DIV_FACTOR = 100000;
+const IMG_UINT16 EDID_I2C_MIN_FS_SCL_HIGH_TIME = 600;
+const IMG_UINT16 EDID_I2C_MIN_FS_SCL_LOW_TIME = 1300;
+const IMG_UINT16 EDID_I2C_MIN_SS_SCL_HIGH_TIME = 40000;
+const IMG_UINT16 EDID_I2C_MIN_SS_SCL_LOW_TIME = 47000;
+const IMG_UINT16 EDID_SFR_CLOCK = 2700;
+
+#ifdef HDMI_DEBUG
+
+//#define HDMI_TEST_EDID 1
+
+#ifdef HDMI_TEST_EDID
+IMG_UINT8 gSampleEdid[128] = {0x00,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0x00,
+0x10,0xAC,0x26,0xA0,0x53,0x34,0x43,0x32, // 8
+0x31,0x11,0x01,0x03,0x80,0x34,0x21,0x78, // 16
+0xEE,0x9B,0xB5,0xA9,0x54,0x34,0xAE,0x25, // 24
+0x13,0x50,0x54,0xA5,0x4B,0x00,0x81,0x80, // 32
+0xA9,0x40,0x71,0x4F,0xB3,0x00,0x01,0x01, // 40
+0x01,0x01,0x01,0x01,0x01,0x01,0x28,0x3C, // 48
+0x80,0xA0,0x70,0xB0,0x23,0x40,0x30,0x20, // 56
+0x36,0x00,0x07,0x44,0x21,0x00,0x00,0x1A, // 64
+0x00,0x00,0x00,0xFF,0x00,0x47,0x4D,0x35, // 72
+0x30,0x34,0x37,0x43,0x34,0x32,0x43,0x34, // 80
+0x53,0x20,0x00,0x00,0x00,0xFC,0x00,0x44, // 88
+0x45,0x4C,0x4C,0x32,0x34,0x30,0x37,0x57, // 96
+0x46,0x50,0x48,0x43,0x00,0x00,0x00,0xFD, // 104
+0x00,0x38,0x4C,0x1E,0x53,0x11,0x00,0x0A, // 112
+0x20,0x20,0x20,0x20,0x20,0x20,0x00,0x6F}; // 120
+#endif
+
+#endif
+
+
+#if defined(HDMI_ENABLE_FAST_COUNT)
+/* calculate the fast speed high time counter - round up */
+static IMG_UINT16 EdidCalcCount(IMG_UINT16 sclMinTime)
+{
+	unsigned long tmpSclPeriod = 0;
+	if (((EDID_SFR_CLOCK * sclMinTime) % I2C_DIV_FACTOR) != 0)
+	{
+		tmpSclPeriod = (unsigned long)((EDID_SFR_CLOCK * sclMinTime) + (I2C_DIV_FACTOR - ((EDID_SFR_CLOCK * sclMinTime) % I2C_DIV_FACTOR))) / I2C_DIV_FACTOR;
+	}
+	else
+	{
+		tmpSclPeriod = (unsigned long)(EDID_SFR_CLOCK * sclMinTime) / I2C_DIV_FACTOR;
+	}
+	return (IMG_UINT16)(tmpSclPeriod);
+}
+#endif
+
+static PVRSRV_ERROR EdidFindBaseBlock(IMG_UINT8 * buffer, IMG_UINT32 size, EDID_BASE_BLOCK * pBase)
+{
+    IMG_UINT32 index = 0;
+
+    HDMI_CHECKPOINT;
+
+    /* First find the EDID signature and fill in global structure */
+    for (index = 0; index < size; index++)
+    {
+        if (*((IMG_UINT64*)&buffer[index]) == EDID_SIGNATURE)
+        {
+            HDMI_DEBUG_PRINT("- %s: Found EDID signature at offset %d\n", __func__, index);
+            break;
+        }
+    }
+
+    if (index == size)
+    {
+        HDMI_ERROR_PRINT("- %s: Could not find EDID signature\n", __func__);
+        return PVRSRV_ERROR_NOT_FOUND;
+    }
+
+    /* Copy to global EDID buffer */
+    if (index + EDID_BLOCK_SIZE > size)
+    {
+        HDMI_ERROR_PRINT("- %s: Found signature but it spills over into next block?!?!\n", __func__);
+        return PVRSRV_ERROR_OUT_OF_MEMORY;
+    }
+    memcpy(pBase, &buffer[index], EDID_BLOCK_SIZE);
+
+    HDMI_DEBUG_PRINT("- %s EDID Version %d, Revision ID: %d", __func__, pBase->EdidVersion, pBase->EdidRevision);
+
+#ifdef HDMI_DEBUG
+    {
+        IMG_UINT8* pData = (IMG_UINT8*)pBase;
+        IMG_UINT32 i = 0;
+        HDMI_DEBUG_PRINT("- %s: EDID Data:\n\n", __func__);
+        for (i = 0; i < EDID_BLOCK_SIZE; i+=8)
+        {
+            HDMI_DEBUG_PRINT(" Offset %3d: %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
+                       pData[i], pData[i+1], pData[i+2], pData[i+3], pData[i+4], pData[i+5], pData[i+6], pData[i+7]);
+        }
+    }
+#endif
+
+    return PVRSRV_OK;
+
+}
+
+static PVRSRV_ERROR EdidParseDtd(IMG_UINT8 * pDesc, VIDEO_PARAMS * pVideoParams)
+{
+    IMG_UINT8 index = pVideoParams->mDtdIndex;
+    DTD_RAW * dtd = (DTD_RAW*)pDesc;
+    DTD decodedDtd = {0};
+
+    HDMI_CHECKPOINT;
+
+    if (index >= HDMI_DTD_MAX)
+    {
+        HDMI_ERROR_PRINT("- %s: Ran out of space for DTD\n", __func__);
+        return PVRSRV_ERROR_OUT_OF_MEMORY;
+    }
+
+
+    decodedDtd.mCode = -1;
+    decodedDtd.mPixelRepetitionInput = 0;
+
+    decodedDtd.mPixelClock = dtd->pixelClock; /*  [10000Hz] */
+    if (decodedDtd.mPixelClock < 0x01)
+    { /* 0x0000 is defined as reserved */
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    decodedDtd.mHActive = dtd->hActiveLower | (dtd->hActiveUpper << 8);
+    decodedDtd.mHBlanking = dtd->hBlankingLower | (dtd->hBlankingUpper << 8);
+    decodedDtd.mHFrontPorchWidth = dtd->hFrontPorchLower | (dtd->hFrontPorchUpper << 8);
+    decodedDtd.mHSyncPulseWidth = dtd->hSyncPulseWidthLower | (dtd->hSyncPulseWidthUpper << 8);
+    decodedDtd.mHImageSize = dtd->hImageSizeLower | (dtd->hImageSizeUpper << 8);
+    decodedDtd.mHBorder = dtd->hBorder;
+    decodedDtd.mHBackPorchWidth = decodedDtd.mHBlanking - decodedDtd.mHSyncPulseWidth - decodedDtd.mHFrontPorchWidth;
+
+    decodedDtd.mVActive = dtd->vActiveLower | (dtd->vActiveUpper << 8);
+    decodedDtd.mVBlanking = dtd->vBlankingLower | (dtd->vBlankingUpper << 8);
+    decodedDtd.mVFrontPorchWidth = dtd->vFrontPorchLower | (dtd->vFrontPorchUpper << 8);
+    decodedDtd.mVSyncPulseWidth = dtd->vSyncPulseWidthLower | (dtd->vSyncPulseWidthUpper << 8);
+    decodedDtd.mVImageSize = dtd->vImageSizeLower | (dtd->vImageSizeUpper << 8);
+    decodedDtd.mVBorder = dtd->vBorder;
+    decodedDtd.mVBackPorchWidth = decodedDtd.mVBlanking - decodedDtd.mVSyncPulseWidth - decodedDtd.mVFrontPorchWidth;
+
+    /* no stereo viewing support in HDMI */
+    decodedDtd.mInterlaced = dtd->interlaced;
+    HDMI_DEBUG_PRINT("- %s: sync signal defs: %d", __func__, dtd->syncSignalDefs);
+    if ((dtd->syncSignalDefs >> 3) & 0x01)
+    {
+        decodedDtd.mVSyncPolarity = (dtd->syncSignalDefs >> 1) & 0x01; // BIT 1
+        decodedDtd.mHSyncPolarity = (dtd->syncSignalDefs >> 0) & 0x01; // BIT 0
+    }
+
+    HDMI_DEBUG_PRINT("Decoded DTD %d with pixel clock %dHz, Horz (%d, %d, %d, %d, %d, %d, %d), Vert (%d, %d, %d, %d, %d, %d, %d)\n", index, decodedDtd.mPixelClock*10000,
+        decodedDtd.mHActive, decodedDtd.mHBlanking, decodedDtd.mHBorder, decodedDtd.mHImageSize, decodedDtd.mHFrontPorchWidth, decodedDtd.mHBackPorchWidth, decodedDtd.mHSyncPulseWidth,
+        decodedDtd.mVActive, decodedDtd.mVBlanking, decodedDtd.mVBorder, decodedDtd.mVImageSize, decodedDtd.mVFrontPorchWidth, decodedDtd.mVBackPorchWidth, decodedDtd.mVSyncPulseWidth);
+
+#if defined(HDMI_EXPERIMENTAL_CLOCKS)
+    pVideoParams->mDtdList[index] = decodedDtd;
+    pVideoParams->mDtdIndex++;
+#else
+    if (PhyIsPclkSupported(dtd->pixelClock))
+    {
+        pVideoParams->mDtdList[index] = decodedDtd;
+        pVideoParams->mDtdIndex++;
+    }
+    else
+    {
+        HDMI_DEBUG_PRINT("- %s: Found unsupported pixel clock (%d) in DTD, not adding", __func__, dtd->pixelClock);
+    }
+#endif
+
+    return PVRSRV_OK;
+}
+
+static inline IMG_UINT8 GetDescriptorType(IMG_UINT8 * pDesc)
+{
+    if (pDesc[0] == 0 && pDesc[1] == 0 && pDesc[2] == 0)
+    {
+        return EDID_DESC_DISPLAY;
+    }
+    else
+    {
+        return EDID_DESC_DETAILED_TIMING;
+    }
+}
+
+static PVRSRV_ERROR EdidParseDispDesc(IMG_UINT8 * pDesc, VIDEO_PARAMS * pVideoParams)
+{
+    DISPLAY_DESCRIPTOR * pDispDesc = (DISPLAY_DESCRIPTOR*)pDesc;
+
+    /* Confirm this is a display descriptor */
+    if (pDispDesc->DescTag[0] != 0 || pDispDesc->DescTag[1] != 0 || pDispDesc->DescTag[2] != 0)
+    {
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT(" Display descriptor tag: %x\n", pDispDesc->DispDescTag);
+
+    switch (pDispDesc->DispDescTag)
+    {
+        case DISP_DESC_TAG_PRODUCT_NAME:
+        {
+            IMG_UINT8 monitorName[14] = { 0 }; // Up to 13 characters
+            IMG_UINT8 i = 0;
+
+            for (i = 0; i < 13; i++)
+            {
+                if (pDispDesc->Data[i] != 0x0A) // 0A is terminator
+                {
+                    monitorName[i] = pDispDesc->Data[i];
+                }
+            }
+
+            /* Just print for now */
+            HDMI_DEBUG_PRINT(" Found monitor name descriptor: %s", monitorName);
+            break;
+        }
+        case DISP_DESC_TAG_RANGE_LIMITS:
+        {
+            /* Only fill range limits if offsets are 0 */
+            MONITOR_RANGE_DATA * pData = (MONITOR_RANGE_DATA*)&pDispDesc->Reserved; // This one is unique, no reserved field
+            if (pData->offsetFlags == 0)
+            {
+                HDMI_DEBUG_PRINT("- %s: Found valid monitor range limits, vMin: %x, vMax: %x, hMin: %x, hMax: %x, maxPClock: %x, support flags: %x\n",
+								 __func__, pData->vMinRate, pData->vMaxRate, pData->hMinRate, pData->hMaxRate, pData->maxPixelClock, pData->supportFlags);
+                pVideoParams->mRangeLimits.vMinRate = pData->vMinRate;
+                pVideoParams->mRangeLimits.vMaxRate = pData->vMaxRate;
+                pVideoParams->mRangeLimits.hMinRate = pData->hMinRate;
+                pVideoParams->mRangeLimits.hMaxRate = pData->hMaxRate;
+                pVideoParams->mRangeLimits.maxPixelClock = pData->maxPixelClock;
+                pVideoParams->mRangeLimits.valid = true;
+            }
+            break;
+        }
+        case DISP_DESC_TAG_EST_TIMING_3:
+        case DISP_DESC_TAG_CVT_3:
+        case DISP_DESC_TAG_DCM:
+        case DISP_DESC_TAG_STD_TIMING:
+        case DISP_DESC_TAG_COLOR_POINT_DATA:
+        case DISP_DESC_TAG_DATA_STRING:
+        case DISP_DESC_TAG_SERIAL_NUMBER:
+            /* Don't care */
+            break;
+        default:
+            HDMI_ERROR_PRINT("- %s: Invalid display descriptor tag\n", __func__);
+            return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR EdidParseDescriptor(IMG_UINT8 * pDesc, VIDEO_PARAMS * pVideoParams)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    IMG_UINT8 dtdDesc = 0;
+
+    HDMI_CHECKPOINT;
+
+    dtdDesc = GetDescriptorType(pDesc);
+
+    if (dtdDesc == EDID_DESC_DETAILED_TIMING)
+    {
+        status = EdidParseDtd(pDesc, pVideoParams);
+        if (status != PVRSRV_OK)
+        {
+            HDMI_ERROR_PRINT("- %s: EDID failed to parse descriptor. Status: %d\n", __func__, status);
+            return status;
+        }
+    }
+    else if (dtdDesc == EDID_DESC_DISPLAY)
+    {
+        status = EdidParseDispDesc(pDesc, pVideoParams);
+        if (status != PVRSRV_OK)
+        {
+            HDMI_ERROR_PRINT("- %s: EDID failed to parse descriptor. Status: %d\n", __func__, status);
+            return status;
+        }
+    }
+
+    return status;
+
+}
+
+static IMG_UINT8 EdidGetColorResolution(VIDEO_INPUT_DEFINITION videoInput)
+{
+	IMG_UINT8 colorRes = 0;
+
+	switch (videoInput.digital.colorBitDepth)
+    {
+		case 1:
+            colorRes = 6;
+            break;
+        case 2:
+            colorRes = 8;
+            break;
+        case 3:
+            colorRes = 10;
+            break;
+        case 4:
+            colorRes = 12;
+            break;
+        case 5:
+            colorRes = 14;
+            break;
+        case 6:
+            colorRes = 16;
+            break;
+        default:
+            HDMI_DEBUG_PRINT("- %s: color bit depth undefined!", __func__);
+            break;
+    }
+
+    return colorRes;
+}
+
+static PVRSRV_ERROR EdidParseBaseBlock(EDID_BASE_BLOCK * edidBlock, VIDEO_PARAMS * pVideoParams)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    IMG_UINT8 index = 0;
+    IMG_UINT8 checkSum = 0;
+    IMG_UINT8 * buffer = (IMG_UINT8*)edidBlock;
+
+    HDMI_CHECKPOINT;
+
+    /* Checksum */
+    for (index = 0; index < EDID_BLOCK_SIZE; index++)
+    {
+        checkSum = (IMG_UINT8) (checkSum + buffer[index]);
+    }
+
+    if (checkSum != EDID_CHECKSUM)
+    {
+        HDMI_ERROR_PRINT("- %s: EDID checksum failed, computed checksum: %d\n", __func__, checkSum);
+        status = PVRSRV_ERROR_DC_INVALID_CONFIG;
+        goto EXIT;
+    }
+
+    /* Extract color resolution */
+    HDMI_DEBUG_PRINT("- %s: Video Input definition 0x%x, Feature Support 0x%x", __func__, edidBlock->VideoInputDefinition.value, edidBlock->FeatureSupport.value);
+    if (edidBlock->VideoInputDefinition.digital.signalInterface == 1)
+    {
+        if (edidBlock->VideoInputDefinition.digital.colorBitDepth != 0)
+        {
+            HDMI_DEBUG_PRINT("- %s: Setting color resolution to %d", __func__, EdidGetColorResolution(edidBlock->VideoInputDefinition));
+            pVideoParams->mColorResolution = EdidGetColorResolution(edidBlock->VideoInputDefinition);
+        }
+        /* Otherwise grab from feature support flags */
+        else
+        {
+            /* RGB 4:4:4 must be supported */
+            pVideoParams->mColorResolution = 8;
+        }
+
+#if 0
+        if (edidBlock->VideoInputDefinition.digital.signalInterface == 1)
+#else
+		if (edidBlock->VideoInputDefinition.digital.vidInterfaceSupported == DIGITAL_VIDEO_INTERFACE_HDMI_A ||
+			edidBlock->VideoInputDefinition.digital.vidInterfaceSupported == DIGITAL_VIDEO_INTERFACE_HDMI_B)
+#endif
+		{
+			pVideoParams->mHdmi = 1;
+		}
+    }
+
+    /* First DTD is preferred timing (VESA 1.4) */
+    if (edidBlock->FeatureSupport.fields.preferredTimingIncluded)
+    {
+        HDMI_DEBUG_PRINT("- %s: Found preferred timing descriptor", __func__);
+        status = EdidParseDtd((IMG_UINT8*)&edidBlock->PreferredTimingMode, pVideoParams);
+        if (status != PVRSRV_OK)
+        {
+            HDMI_ERROR_PRINT("- %s: EDID failed to parse preferred timing DTD. Status: %d\n", __func__, status);
+            goto EXIT;
+        }
+
+        pVideoParams->mPreferredTimingIncluded = true;
+    }
+
+    HDMI_CHECKPOINT;
+
+    /* Parse remaining descriptors, could be DTD or display descriptor (first 3 bytes = 0) */
+    status = EdidParseDescriptor(edidBlock->Descriptor2, pVideoParams);
+    CHECK_AND_EXIT(status, EXIT);
+
+    status = EdidParseDescriptor(edidBlock->Descriptor3, pVideoParams);
+    CHECK_AND_EXIT(status, EXIT);
+
+    status = EdidParseDescriptor(edidBlock->Descriptor4, pVideoParams);
+    CHECK_AND_EXIT(status, EXIT);
+
+    HDMI_CHECKPOINT;
+
+EXIT:
+
+    return PVRSRV_OK;
+}
+
+static void EdidParseCeaDataBlock(VIDEO_PARAMS * pVideoParams, IMG_UINT8 * data, IMG_UINT32 * length)
+{
+    IMG_UINT8 tag;
+    IMG_UINT8 nBlocks;
+    IMG_UINT8 i;
+
+    HDMI_CHECKPOINT;
+
+    // Upper 3 bits of first byte are the tag
+    tag = data[0] >> 5;
+    // Bottom 5 bits of first byte are the number of blocks
+    nBlocks = data[0] & 0x1F;
+
+    HDMI_DEBUG_PRINT("- %s: Parsing CEA data block, tag: %d, nBlocks: %d\n", __func__, tag, nBlocks);
+
+    switch (tag)
+    {
+        case EDID_CEA_DATA_BLOCK_VIDEO_TAG:
+            for (i = 1; i < (nBlocks + 1); i++)
+            {
+                HDMI_DEBUG_PRINT("- %s: Native: %d, VIC: %d\n", __func__, data[i] >> 7, data[i] & 0x7F);
+                EdidAddDtd(pVideoParams, data[i] & 0x7F, EDID_REFRESH_RATE);
+                /* If its a native setup, then make it the active */
+                if (data[i] >> 7 == 1 && pVideoParams->mPreferredTimingIncluded == false)
+                {
+                    pVideoParams->mDtdActiveIndex = pVideoParams->mDtdIndex - 1;
+                }
+            }
+            break;
+        case EDID_CEA_DATA_BLOCK_VENDOR_TAG:
+        case EDID_CEA_DATA_BLOCK_AUDIO_TAG:
+        case EDID_CEA_DATA_BLOCK_SPEAKER_TAG:
+        default:
+            // don't care
+            break;
+    }
+
+    *length = nBlocks + 1;
+
+}
+
+static PVRSRV_ERROR EdidParseCeaBlock(IMG_UINT8 * data, VIDEO_PARAMS * pVideoParams)
+{
+    EDID_EXT_CEA * pCeaBlock = (EDID_EXT_CEA*)data;
+    IMG_UINT8 i = 0, j = 0;
+    IMG_UINT32 length; // of each CEA data block
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT("- %s: Parsing CEA extension block, DTD Offset: %d, total size: %d", __func__, pCeaBlock->DtdOffset, sizeof(pCeaBlock->Data));
+
+    /* Only support revision 3 */
+    if (pCeaBlock->Revision != EDID_CEA_REVISION)
+    {
+        HDMI_ERROR_PRINT("- %s: Found unsupported CEA block revision number (%d)\n", __func__, pCeaBlock->Revision);
+        return PVRSRV_ERROR_NOT_FOUND;
+    }
+
+    /* First parse data blocks */
+    for (i = 0; i < (pCeaBlock->DtdOffset + 4); i+= length)
+    {
+        EdidParseCeaDataBlock(pVideoParams, &pCeaBlock->Data[i], &length);
+    }
+
+    /* Now parse DTDs (max = 6)*/
+    for (i = pCeaBlock->DtdOffset - 4, j = 0; i < sizeof(pCeaBlock->Data) && j < 6; i += sizeof(DTD_RAW), j++)
+    {
+        EdidParseDtd(&pCeaBlock->Data[i], pVideoParams);
+    }
+
+    return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR EdidParseExtBlock(IMG_UINT8 * edidBlock, VIDEO_PARAMS * pVideoParams)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    EDID_EXT_BLOCK * pExtBlock = (EDID_EXT_BLOCK*) edidBlock;
+
+    HDMI_CHECKPOINT;
+
+#ifdef HDMI_DEBUG
+    HDMI_DEBUG_PRINT("- %s: Parsing extension block tag %d", __func__, pExtBlock->Tag);
+    {
+        IMG_UINT8* pData = (IMG_UINT8*)edidBlock;
+        IMG_UINT32 i = 0;
+        for (i = 0; i < EDID_BLOCK_SIZE; i+=8)
+        {
+            HDMI_DEBUG_PRINT(" Offset 0x%02x: %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
+                       pData[i], pData[i+1], pData[i+2], pData[i+3], pData[i+4], pData[i+5], pData[i+6], pData[i+7]);
+        }
+    }
+#endif
+
+    /* First byte is tag */
+    switch (pExtBlock->Tag)
+    {
+        case EDID_EXT_TAG_CEA:
+            status = EdidParseCeaBlock(edidBlock, pVideoParams);
+            break;
+        case EDID_EXT_TAG_VTB:
+        case EDID_EXT_TAG_DI:
+        case EDID_EXT_TAG_LS:
+        case EDID_EXT_TAG_DPVL:
+        case EDID_EXT_TAG_BLOCK_MAP:
+        case EDID_EXT_TAG_MFG_DEFINED:
+            status = PVRSRV_ERROR_NOT_SUPPORTED;
+            break;
+        default:
+            HDMI_ERROR_PRINT("- %s: Unknown ext block type %d\n", __func__, pExtBlock->Tag);
+            status = PVRSRV_ERROR_NOT_SUPPORTED;
+            break;
+    }
+
+    return status;
+}
+
+
+static PVRSRV_ERROR EdidI2cReadByte(IMG_CPU_VIRTADDR base, IMG_UINT8 offset, IMG_UINT8 * value)
+{
+    IMG_UINT32 intReg = 0;
+    IMG_UINT32 pollTime = 0;
+
+    #ifdef HDMI_TEST_EDID
+    *value = ((IMG_UINT8*)&gSampleEdid[0])[offset];
+    return PVRSRV_OK;
+    #endif
+
+    /* Write 1 to clear interrupt status */
+    HDMI_WRITE_CORE_REG(base, HDMI_IH_I2CM_STAT0_OFFSET,
+        SET_FIELD(HDMI_IH_I2CM_STAT0_I2CMASTERDONE_START, HDMI_IH_I2CM_STAT0_I2CMASTERDONE_MASK, 1));
+
+    /* Setup EDID base address */
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_SLAVE_OFFSET, EDID_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_SEGADDR_OFFSET, EDID_SEGMENT_ADDRESS);
+
+    // address
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_ADDRESS_OFFSET, offset);
+
+    /* Set operation to normal read */
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_OPERATION_OFFSET,
+        SET_FIELD(HDMI_I2CM_OPERATION_RD_START, HDMI_I2CM_OPERATION_RD_MASK, 1));
+
+    // poll on done
+
+    intReg = HDMI_READ_CORE_REG(base, HDMI_IH_I2CM_STAT0_OFFSET);
+    while (!IS_BIT_SET(intReg, HDMI_IH_I2CM_STAT0_I2CMASTERDONE_START)
+        && !IS_BIT_SET(intReg, HDMI_IH_I2CM_STAT0_I2CMASTERERROR_START))
+    {
+        intReg = HDMI_READ_CORE_REG(base, HDMI_IH_I2CM_STAT0_OFFSET);
+        DC_OSDelayus(1 * 100);
+        pollTime += 1;
+        if (pollTime > EDID_I2C_TIMEOUT_MS)
+        {
+            HDMI_ERROR_PRINT("- %s: Timeout polling on I2CMasterDoneBit\n", __func__);
+            return PVRSRV_ERROR_TIMEOUT;
+        }
+    }
+
+    if (IS_BIT_SET(intReg, HDMI_IH_I2CM_STAT0_I2CMASTERERROR_START))
+    {
+        HDMI_ERROR_PRINT("- %s: I2C EDID read failed, Master error bit is set\n", __func__);
+        return PVRSRV_ERROR_TIMEOUT;
+    }
+
+    *value = HDMI_READ_CORE_REG(base, HDMI_I2CM_DATAI_OFFSET) & 0xFF;
+
+    return PVRSRV_OK;
+}
+
+
+/*
+
+Steps for reading EDID information:
+
+1. Initialize I2C DDC interface (clocks, interrupts, etc)
+2. Read EDID block byte-by-byte or with sequential
+read (up to 8 bytes) to limit interrupts
+3. Parse base block:
+    a. Mfg/vendor info
+    b. Preferred timing
+    c. Other descriptors
+4. Parse Extension blocks
+
+*/
+PVRSRV_ERROR EdidRead(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    IMG_UINT8 bufferOffset = 0;
+    IMG_UINT8 i2cOffset = 0;
+    IMG_UINT8 localBuffer[EDID_BLOCK_SIZE];
+    IMG_UINT8 i = 0;
+    EDID_BASE_BLOCK edidBase;
+
+    HDMI_CHECKPOINT;
+
+    #if defined(HDMI_PDUMP)
+    /* Setup for VIC 16, color res 10 */
+    pvDevice->videoParams.mDtdActiveIndex = pvDevice->videoParams.mDtdIndex;
+    EdidAddDtd(&pvDevice->videoParams, 16, 60000);
+    return PVRSRV_OK;
+    #endif
+
+	/*to incorporate extensions we have to include the following - see VESA E-DDC spec. P 11 */
+
+    /*
+    Each segment pointer contains one byte of addressable space and
+    2 EDID blocks (128 bytes each)
+
+    SegPtr/Address range        Block type
+
+    00h/00h->7Fh                Base EDID structure
+    00h/80h->FFh                Extension block 1 OR block map 1
+    01h/00h->7Fh                Extension block 2
+    01h/80h->FFh                Extension block 3
+    ...
+    40h/00h->7Fh                Block map 2 if 129<=N<=254
+    ...
+    */
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SEGPTR_OFFSET, 0);
+
+    /*
+    Loop through and read byte by byte, could optimize and use ext read
+    */
+    for (bufferOffset = 0; bufferOffset < sizeof(localBuffer); bufferOffset++, i2cOffset++)
+    {
+        // read data
+        status = EdidI2cReadByte(pvDevice->pvHDMIRegCpuVAddr, i2cOffset, &localBuffer[bufferOffset]);
+        if (status != PVRSRV_OK)
+        {
+            return status;
+        }
+    }
+
+    /* Find base block and populate global */
+    status = EdidFindBaseBlock(localBuffer, EDID_BLOCK_SIZE, &edidBase);
+    if (status != PVRSRV_OK)
+    {
+        return status;
+    }
+
+    /* Extract video parameter data from base block (includes DTD parsing) */
+    status = EdidParseBaseBlock(&edidBase, &pvDevice->videoParams);
+    if (status != PVRSRV_OK)
+    {
+        return status;
+    }
+
+    HDMI_DEBUG_PRINT("- %s: Finished parsing base block, extension block count: %d\n", __func__, edidBase.ExtensionBlockCount);
+
+    /* Loop through each extension block */
+    for (i = 0; i < edidBase.ExtensionBlockCount; i++)
+    {
+        PVRSRV_ERROR extstatus = PVRSRV_OK;
+        for (bufferOffset = 0; bufferOffset < sizeof(localBuffer); bufferOffset++)
+        {
+            // read data
+            status = EdidI2cReadByte(pvDevice->pvHDMIRegCpuVAddr, i2cOffset++, &localBuffer[bufferOffset]);
+            if (status != PVRSRV_OK)
+            {
+                return status;
+            }
+        }
+
+        // don't treat error in extension block as fatal, so use separate status var.
+        extstatus = EdidParseExtBlock(localBuffer, &pvDevice->videoParams);
+        if (extstatus != PVRSRV_OK && extstatus != PVRSRV_ERROR_NOT_SUPPORTED)
+        {
+            HDMI_ERROR_PRINT("- %s: Failed to parse extension block %d\n", __func__, i);
+            break;
+        }
+    }
+
+    #if defined(HDMI_FORCE_MODE) && (HDMI_FORCE_MODE != 0)
+    HDMI_DEBUG_PRINT("- %s: Forcing HDMI mode %d", __func__, HDMI_FORCE_MODE);
+    EdidAddDtd(&pvDevice->videoParams, HDMI_FORCE_MODE, EDID_REFRESH_RATE);
+    pvDevice->videoParams.mDtdActiveIndex = pvDevice->videoParams.mDtdIndex - 1;
+    #endif
+
+    return status;
+}
+
+PVRSRV_ERROR EdidInitialize(HDMI_DEVICE * pvDevice)
+{
+    HDMI_CHECKPOINT;
+
+    /* Mask interrupts */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_INT_OFFSET,
+        SET_FIELD(HDMI_I2CM_INT_DONE_MASK_START, HDMI_I2CM_INT_DONE_MASK_MASK, 1) |
+        SET_FIELD(HDMI_I2CM_INT_READ_REQ_MASK_START, HDMI_I2CM_INT_READ_REQ_MASK_MASK, 1));
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_CTLINT_OFFSET,
+        SET_FIELD(HDMI_I2CM_CTLINT_ARB_MASK_START, HDMI_I2CM_CTLINT_ARB_MASK_START, 1) |
+        SET_FIELD(HDMI_I2CM_CTLINT_NACK_MASK_START, HDMI_I2CM_CTLINT_NACK_MASK_START, 1));
+
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_DIV_OFFSET,
+        SET_FIELD(HDMI_I2CM_DIV_FAST_STD_MODE_START, HDMI_I2CM_DIV_FAST_STD_MODE_START, 0));
+
+/* Set clock division and counts */
+
+/* Fast mode setup */
+/*
+    count = EdidCalcCount(EDID_I2C_MIN_FS_SCL_HIGH_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_HCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_HCNT_0_OFFSET, count & 0xFF);
+    count = EdidCalcCount(EDID_I2C_MIN_FS_SCL_LOW_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_LCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_LCNT_0_OFFSET, count & 0xFF);
+    count = EdidCalcCount(EDID_I2C_MIN_SS_SCL_HIGH_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_HCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_HCNT_0_OFFSET, count & 0xFF);
+    count = EdidCalcCount(EDID_I2C_MIN_SS_SCL_LOW_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_LCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_LCNT_0_OFFSET, count & 0xFF);
+*/
+    /* Re-enable interrupts */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_INT_OFFSET,
+        SET_FIELD(HDMI_I2CM_INT_DONE_MASK_START, HDMI_I2CM_INT_DONE_MASK_MASK, 0));
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_CTLINT_OFFSET,
+        SET_FIELD(HDMI_I2CM_CTLINT_ARB_MASK_START, HDMI_I2CM_CTLINT_ARB_MASK_START, 0) |
+        SET_FIELD(HDMI_I2CM_CTLINT_NACK_MASK_START, HDMI_I2CM_CTLINT_NACK_MASK_START, 0));
+
+    return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR EdidAddDtd(VIDEO_PARAMS * pVideoParams, IMG_UINT8 code, IMG_UINT32 refreshRate)
+{
+    DTD * dtd;
+
+    if (pVideoParams->mDtdIndex >= HDMI_DTD_MAX)
+    {
+        HDMI_DEBUG_PRINT("- %s: Hit max DTD entries (%d)", __func__, pVideoParams->mDtdIndex);
+        return PVRSRV_ERROR_OUT_OF_MEMORY;
+    }
+
+    dtd = &pVideoParams->mDtdList[pVideoParams->mDtdIndex];
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT("- %s: Adding DTD with VIC %d and refresh rate %d\n", __func__, code, refreshRate);
+
+	dtd->mCode = code;
+	dtd->mHBorder = DEFAULT_BORDER_WIDTH_PIXELS;
+	dtd->mVBorder = DEFAULT_BORDER_WIDTH_PIXELS;
+	dtd->mPixelRepetitionInput = 0;
+	dtd->mHImageSize = 16;
+	dtd->mVImageSize = 9;
+	dtd->mYcc420 = 0;
+	dtd->mLimitedToYcc420 = 0;
+
+	switch (code)
+	{
+	case 1: /* 640x480p @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		dtd->mHActive = 640;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 160;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 10;
+		dtd->mHSyncPulseWidth = 96;
+		dtd->mVSyncPulseWidth = 2;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0; /* Active low */
+		dtd->mInterlaced = 0; /* not(progressive_nI) */
+		dtd->mPixelClock = (refreshRate == 59940) ? 2517 : 2520; /* pixel clock depends on refresh rate */
+		break;
+	case 2: /* 720x480p @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 3: /* 720x480p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 138;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 62;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 2700 : 2702;
+		break;
+	case 4: /* 1280x720p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 370;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 110;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 7417 : 7425;
+		break;
+	case 5: /* 1920x1080i @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 59940) ? 7417 : 7425;
+		break;
+	case 6: /* 720(1440)x480i @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 7: /* 720(1440)x480i @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 59940) ? 2700 : 2702;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 8: /* 720(1440)x240p @ 59.826/60.054/59.886/60.115Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 9: /* 720(1440)x240p @ 59.826/60.054/59.886/60.115Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = (refreshRate > 60000) ? 22 : 23;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = (refreshRate > 60000) ? 4 : 5;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock
+		= ((refreshRate == 60054) || refreshRate == 59826) ? 2700
+				: 2702; /*  else 60.115/59.886 Hz */
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 10: /* 2880x480i @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 11: /* 2880x480i @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 552;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 76;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 248;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 59940) ? 5400 : 5405;
+		break;
+	case 12: /* 2880x240p @ 59.826/60.054/59.886/60.115Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 13: /* 2880x240p @ 59.826/60.054/59.886/60.115Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 552;
+		dtd->mVBlanking = (refreshRate > 60000) ? 22 : 23;
+		dtd->mHFrontPorchWidth = 76;
+		dtd->mVFrontPorchWidth = (refreshRate > 60000) ? 4 : 5;
+		dtd->mHSyncPulseWidth = 248;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock
+		= ((refreshRate == 60054) || refreshRate == 59826) ? 5400
+				: 5405; /*  else 60.115/59.886 Hz */
+		break;
+	case 14: /* 1440x480p @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 15: /* 1440x480p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 32;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 5400 : 5405;
+		break;
+	case 16: /* 1920x1080p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 17: /* 720x576p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 18: /* 720x576p @ 50Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 144;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 12;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 64;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 2700;
+		break;
+	case 19: /* 1280x720p @ 50Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 700;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 440;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 20: /* 1920x1080i @ 50Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 7425;
+		break;
+	case 21: /* 720(1440)x576i @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 22: /* 720(1440)x576i @ 50Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 2700;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 23: /* 720(1440)x288p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 24: /* 720(1440)x288p @ 50Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = (refreshRate == 50000) ? 24
+				: ((refreshRate == 49920) ? 25 : 26);
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = (refreshRate == 50000) ? 2
+				: ((refreshRate == 49920) ? 3 : 4);
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 2700;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 25: /* 2880x576i @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 26: /* 2880x576i @ 50Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 576;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 48;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 252;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 5400;
+		break;
+	case 27: /* 2880x288p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 28: /* 2880x288p @ 50Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 576;
+		dtd->mVBlanking = (refreshRate == 50000) ? 24
+				: ((refreshRate == 49920) ? 25 : 26);
+		dtd->mHFrontPorchWidth = 48;
+		dtd->mVFrontPorchWidth = (refreshRate == 50000) ? 2
+				: ((refreshRate == 49920) ? 3 : 4);
+		dtd->mHSyncPulseWidth = 252;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 5400;
+		break;
+	case 29: /* 1440x576p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 30: /* 1440x576p @ 50Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 128;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 5400;
+		break;
+	case 31: /* 1920x1080p @ 50Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 32: /* 1920x1080p @ 23.976/24Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 830;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 638;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 23976) ? 7417 : 7425;
+		break;
+	case 33: /* 1920x1080p @ 25Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 34: /* 1920x1080p @ 29.97/30Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 29970) ? 7417 : 7425;
+		break;
+	case 35: /* 2880x480p @ 60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 36: /* 2880x480p @ 60Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 552;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 64;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 248;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 10800 : 10810;
+		break;
+	case 37: /* 2880x576p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 38: /* 2880x576p @ 50Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 576;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 48;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 256;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 10800;
+		break;
+	case 39: /* 1920x1080i (1250 total) @ 50Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 384;
+		dtd->mVBlanking = 85;
+		dtd->mHFrontPorchWidth = 32;
+		dtd->mVFrontPorchWidth = 23;
+		dtd->mHSyncPulseWidth = 168;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = 1;
+		dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 7200;
+		break;
+	case 40: /* 1920x1080i @ 100Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 14850;
+		break;
+	case 41: /* 1280x720p @ 100Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 700;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 440;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 42: /* 720x576p @ 100Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 43: /* 720x576p @ 100Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 144;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 12;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 64;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 5400;
+		break;
+	case 44: /* 720(1440)x576i @ 100Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 45: /* 720(1440)x576i @ 100Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 5400;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 46: /* 1920x1080i @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 119880) ? 14835 : 14850;
+		break;
+	case 47: /* 1280x720p @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 370;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 110;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 119880) ? 14835 : 14850;
+		break;
+	case 48: /* 720x480p @ 119.88/120Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 49: /* 720x480p @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 138;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 62;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 119880) ? 5400 : 5405;
+		break;
+	case 50: /* 720(1440)x480i @ 119.88/120Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 51: /* 720(1440)x480i @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 119880) ? 5400 : 5405;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 52: /* 720X576p @ 200Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 53: /* 720X576p @ 200Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 144;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 12;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 64;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 10800;
+		break;
+	case 54: /* 720(1440)x576i @ 200Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 55: /* 720(1440)x576i @ 200Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 10800;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 56: /* 720x480p @ 239.76/240Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 57: /* 720x480p @ 239.76/240Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 138;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 62;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 239760) ? 10800 : 10810;
+		break;
+	case 58: /* 720(1440)x480i @ 239.76/240Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 59: /* 720(1440)x480i @ 239.76/240Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 239760) ? 10800 : 10810;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 60: /* 1280x720p @ 23.97/24Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 2020;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 1760;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 23970) ? 5934 : 5940;
+		break;
+	case 61: /* 1280x720p @ 25Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 2680;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 2420;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 62: /* 1280x720p @ 29.97/30Hz  16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 2020;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 1760;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 29970) ? 7417: 7425;
+		break;
+	case 63: /* 1920x1080p @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 119880) ?  29670: 29700;
+		break;
+	case 64: /* 1920x1080p @ 100Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700;
+		break;
+	case 68:
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 700;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 440;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 69: /* 1280x720p @ 60Hz 21:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 370;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 110;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 75: //1080p
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 76: //1080p
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 93: /* 4k x 2k, 30Hz, HDMI_VIC 3 */
+		dtd->mCode = 93;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1660;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1276;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 94: /* 4k x 2k, 30Hz, HDMI_VIC 2 */
+		dtd->mCode = 94;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1440;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1056;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+
+	case 95: /* 4k x 2k, HDMI_VIC 1 */
+		dtd->mCode = 95;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 560;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 176;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 96: /* 4k x 2k, HDMI_VIC 2 */
+		dtd->mCode = 96;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1440;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1056;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 97: /* 4k x 2k, HDMI_VIC 1 */
+		dtd->mCode = 97;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 560;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 176;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 98: /* 4k x 2k, HDMI_VIC 1 */
+		dtd->mCode = 98;
+		dtd->mHActive = 4096;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1404;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1020;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 103:
+		dtd->mCode = 103;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1660;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1276;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 104:
+		dtd->mCode = 104;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1440;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1056;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+    case 105:
+        dtd->mCode = 105;
+        dtd->mHActive = 3840;
+        dtd->mVActive = 2160;
+        dtd->mHBlanking = 560;
+        dtd->mVBlanking = 90;
+        dtd->mHFrontPorchWidth = 176;
+        dtd->mVFrontPorchWidth = 8;
+        dtd->mHSyncPulseWidth = 88;
+        dtd->mVSyncPulseWidth = 10;
+        dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+        dtd->mInterlaced = 0;
+        dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+        break;
+    default:
+        dtd->mCode = 0;
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+#if !defined(HDMI_EXPERIMENTAL_CLOCKS)
+    if (PhyIsPclkSupported(dtd->mPixelClock) == false)
+    {
+        /* Return success but don't add to list */
+        return PVRSRV_OK;
+    }
+#endif
+    dtd->mValid = true;
+
+    pVideoParams->mDtdIndex = (pVideoParams->mDtdIndex + 1) % HDMI_DTD_MAX;
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdcp.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdcp.c
new file mode 100644
index 000000000000..d8702c478949
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdcp.c
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+#include "hdmi.h"
+#include "hdcp.h"
+
+
+/*
+
+Steps for configuring HDCP:
+
+1. Read product ID1 to determine if HDCP is present
+2. Select DVI or HDMI mode
+3. Set the data enable, Hsync, and Vsync polarity
+4. Set encryption on
+5. Reset HDCP engine
+6. Configure device private keys
+7. Enable the encryption
+8. SHA-1 calculation if connected to a repeater
+
+*/
+PVRSRV_ERROR HdcpConfigure(HDMI_DEVICE * pvDevice)
+{
+
+    HDMI_CHECKPOINT;
+
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdmi.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdmi.c
new file mode 100644
index 000000000000..a09ba0448536
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdmi.c
@@ -0,0 +1,622 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#include "hdmi.h"
+#include "edid.h"
+#include "phy.h"
+#include "video.h"
+#include "hdcp.h"
+#include "dc_external.h"
+#include <linux/delay.h>
+#include "plato.h"
+
+static IMG_UINT32 g_uiNumHDMIInstances;   /* Number of instances */
+
+static void PrintDebugInformation(HDMI_DEVICE * pvDevice)
+{
+    HDMI_DEBUG_PRINT("\nDesign ID: %d\nRev ID: %d\nProduct ID: %d\nProduct ID HDCP: %d\n",
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_DESIGN_ID_OFFSET),
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_REVISION_ID_OFFSET),
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_PRODUCT_ID0_OFFSET),
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_PRODUCT_ID1_OFFSET),
+            HDMI_ID_PRODUCT_ID1_HDCP_START, HDMI_ID_PRODUCT_ID1_HDCP_MASK));
+
+    HDMI_DEBUG_PRINT("\nHDCP Present: %d\nHDMI 1.4: %d\nHDMI 2.0: %d\nPHY Type: %d\n",
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG0_ID_OFFSET),
+            HDMI_ID_CONFIG0_ID_HDCP_START, HDMI_ID_CONFIG0_ID_HDCP_MASK),
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG0_ID_OFFSET),
+            HDMI_ID_CONFIG0_ID_HDMI14_START, HDMI_ID_CONFIG0_ID_HDMI14_MASK),
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG1_ID_OFFSET),
+            HDMI_ID_CONFIG1_ID_HDMI20_START, HDMI_ID_CONFIG1_ID_HDMI20_MASK),
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG2_ID_OFFSET));
+}
+
+#if defined(HDMI_PDUMP)
+
+static bool HdmiPdumpSetup(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+
+    HDMI_CHECKPOINT;
+
+    /* Set to VIC 16 (1920x1080) */
+    #if DC_DEFAULT_VIDEO_FORMAT == VIDEO_FORMAT_1920_1080p
+    EdidAddDtd(&pvDevice->videoParams, 16, 60000);
+    #elif DC_DEFAULT_VIDEO_FORMAT == VIDEO_FORMAT_1280_720p
+    EdidAddDtd(&pvDevice->videoParams, 4, 60000);
+    #endif
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_INVID0_OFFSET, 0x03);
+
+    /* This maps 1:1 to pdump */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVIDCONF_OFFSET, 0x78);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV0_OFFSET, 0x10);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV0_OFFSET, 0x80);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV1_OFFSET, 0x07);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV0_OFFSET, 0x38);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV1_OFFSET, 0x04);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK0_OFFSET, 0x18);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK1_OFFSET, 0x01);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVBLANK_OFFSET, 0x2D);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY0_OFFSET, 0x58);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY1_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINDELAY_OFFSET, 0x04);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH0_OFFSET, 0x2C);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH1_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINWIDTH_OFFSET, 0x05);
+
+    PhyPowerDown(pvDevice);
+
+    /* pixel packing and 10bit color depth */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_PR_CD_OFFSET, 0x50);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_CONF_OFFSET, 0x64);
+
+    /* Enable HPD sense...user guide says this should be the first step */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_PDDQ_START, HDMI_PHY_CONF0_PDDQ_MASK, 1));
+
+    HDMI_CHECKPOINT;
+
+    /* Audio N and CTS value */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N1_OFFSET, 0xE5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N2_OFFSET, 0x0F);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N3_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS1_OFFSET, 0x19);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS2_OFFSET, 0xD5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS3_OFFSET, 0x02);
+
+    /* Write Phy values through I2C, based on MPLL settings in PHY spec (p 193):
+
+    Pixel Clock: 148.5MHz
+    Pixel repetition: no
+    Color depth: 10-bit
+    TMDS clk src: 185.625MHz
+
+    NOTE: THIS IS BASED ON PHY MODEL E302 (see HDMI reference driver: phy.c:3187)
+
+    */
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x214c);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0211);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_TXTERM_OFFSET, 0x0004);
+    CHECK_AND_EXIT(status, EXIT);
+
+    /* Similar to step 3.2.2.g (except we're setting ENHPDRXSENSE as well) */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_TXPWRON_START, HDMI_PHY_CONF0_TXPWRON_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_SVSRET_START, HDMI_PHY_CONF0_SVSRET_MASK, 1));
+
+    /* Wait for PHY to lock */
+    status = PhyWaitLock(pvDevice);
+    #if defined(EMULATOR)
+    status = PVRSRV_OK;
+    #endif
+    CHECK_AND_EXIT(status, EXIT);
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CTRLDUR_OFFSET, 0x0D);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLDUR_OFFSET, 0x20);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLSPAC_OFFSET, 0x01);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH0PREAM_OFFSET, 0x0B);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH1PREAM_OFFSET, 0x16);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH2PREAM_OFFSET, 0x21);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_MC_CLKDIS_OFFSET,
+        SET_FIELD(HDMI_MC_CLKDIS_HDCPCLK_DIS_START, HDMI_MC_CLKDIS_HDCPCLK_DIS_MASK, 1));
+    // This should already be set??
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINWIDTH_OFFSET, 0x05);
+
+EXIT:
+    return (status == PVRSRV_OK) ? true : false;
+}
+
+#endif /* HDMI_PDUMP */
+
+static void HdmiInitInterrupts(HDMI_DEVICE * psDeviceData)
+{
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_PHY_STAT0_OFFSET , 0x3e);
+
+    /* Mute all other interrupts */
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_FC_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_FC_STAT1_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_FC_STAT2_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_AS_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_I2CM_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_CEC_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_VP_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_I2CMPHY_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_AHBDMAAUD_STAT0_OFFSET , 0xff);
+}
+static void HdmiEnableHPDInterrupt(HDMI_DEVICE * psDeviceData)
+{
+    /* Enable hot plug interrupts */
+    HDMI_CHECKPOINT;
+
+    /* Clear HPD interrupt by writing 1*/
+    if (IS_BIT_SET(HDMI_READ_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET), HDMI_IH_PHY_STAT0_HPD_START))
+    {
+        HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET,
+            SET_FIELD(HDMI_IH_PHY_STAT0_HPD_START, HDMI_IH_PHY_STAT0_HPD_MASK, 1));
+    }
+
+    HdmiInitInterrupts(psDeviceData);
+
+    HDMI_CHECKPOINT;
+
+    /* Power off */
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_TXPWRON_START, HDMI_PHY_CONF0_TXPWRON_MASK, 0) |
+        SET_FIELD(HDMI_PHY_CONF0_PDDQ_START, HDMI_PHY_CONF0_PDDQ_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_SVSRET_START, HDMI_PHY_CONF0_SVSRET_MASK, 1));
+
+    /* Enable hot plug detection */
+    HDMI_READ_MOD_WRITE(psDeviceData->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1));
+
+    /* Now flip the master switch to unmute */
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_OFFSET , SET_FIELD(HDMI_IH_MUTE_ALL_START, HDMI_IH_MUTE_ALL_MASK, 0));
+
+}
+
+static IMG_BOOL HdmiLISRHandler(void *pvData)
+{
+    HDMI_DEVICE* psDevice = (HDMI_DEVICE*)pvData;
+
+    IMG_UINT32 ui32PhyStat0 = 0;
+    IMG_UINT32 ui32IHPhyStat0 = 0;
+    IMG_UINT32 ui32HPDPolarity = 0;
+    IMG_UINT32 ui32Decode = 0;
+    unsigned long ulFlags = 0;
+
+    HDMI_CHECKPOINT;
+
+    DC_OSSpinLockIRQSave(&psDevice->irqLock, ulFlags);
+
+    /* Mute all interrupts */
+    HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_OFFSET, 0x03);
+
+    ui32Decode = HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_DECODE_OFFSET);
+
+    /* Only support interrupts on PHY (eg HPD) */
+    if (!IS_BIT_SET(ui32Decode, HDMI_IH_DECODE_PHY_START))
+    {
+        HDMI_DEBUG_PRINT("- %s: Unknown interrupt generated, decode: %x\n", __func__,
+            HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_DECODE_OFFSET));
+        goto IRQRestore;
+    }
+
+    ui32PhyStat0 = HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_STAT0_OFFSET);
+    ui32HPDPolarity = GET_FIELD(HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_POL0_OFFSET), HDMI_PHY_POL0_HPD_START, HDMI_PHY_POL0_HPD_MASK);
+    ui32IHPhyStat0 = HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET);
+
+    HDMI_DEBUG_PRINT("- %s: Hdmi interrupt detected PHYStat0: 0x%x, HPD Polarity: 0x%x, IH Decode: 0x%x, IH PhyStat0: 0x%x",
+					 __func__, ui32PhyStat0, ui32HPDPolarity, ui32Decode, ui32IHPhyStat0);
+
+    /* Check if hot-plugging occurred */
+    if (GET_FIELD(ui32PhyStat0, HDMI_PHY_STAT0_HPD_START, HDMI_PHY_STAT0_HPD_MASK) == IMG_TRUE &&
+        GET_FIELD(ui32IHPhyStat0, HDMI_IH_PHY_STAT0_HPD_START, HDMI_IH_PHY_STAT0_HPD_MASK) == IMG_TRUE)
+    {
+        HDMI_DEBUG_PRINT("- %s: Hot plug detected", __func__);
+
+        /* Flip polarity */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_POL0_OFFSET,
+            SET_FIELD(HDMI_PHY_POL0_HPD_START, HDMI_PHY_POL0_HPD_MASK, !ui32HPDPolarity));
+
+        /* Write 1 to clear interrupt */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET, ui32IHPhyStat0);
+
+        /* schedule work for HDMI/PDP init*/
+        if (psDevice->bHPD == IMG_FALSE)
+        {
+            psDevice->bHPD = IMG_TRUE;
+            schedule_delayed_work(&(psDevice->delayedWork), msecs_to_jiffies(10));
+        }
+
+        /* Mute non-HPD interrupts */
+        HdmiInitInterrupts(psDevice);
+
+        /* Now flip the master switch to unmute */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_OFFSET, 0);
+
+    }
+    else
+    {
+        HDMI_DEBUG_PRINT("- %s: Cable unplugged\n", __func__);
+
+        /* Flip polarity */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_POL0_OFFSET,
+            SET_FIELD(HDMI_PHY_POL0_HPD_START, HDMI_PHY_POL0_HPD_MASK, !ui32HPDPolarity));
+
+        /* Write 1 to clear interrupts */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET, ui32IHPhyStat0);
+
+        /* Unmute and enable HPD interrupt */
+        HdmiEnableHPDInterrupt(psDevice);
+
+        /* Cable was unplugged */
+        psDevice->bHPD = IMG_FALSE;
+    }
+
+IRQRestore:
+
+    DC_OSSpinUnlockIRQRestore(&psDevice->irqLock, ulFlags);
+
+    return IMG_TRUE;
+}
+
+static void HdmiMISRHandler(struct work_struct *work)
+{
+    PVRSRV_ERROR eError;
+    struct delayed_work *delay_work = to_delayed_work(work);
+    HDMI_DEVICE* psDeviceData = container_of(delay_work, HDMI_DEVICE, delayedWork);
+
+    HDMI_CHECKPOINT;
+
+    if (!psDeviceData->bHPD || !psDeviceData->mInitialized)
+    {
+        return;
+    }
+
+#if defined(HDMI_PDUMP)
+    HdmiPdumpSetup(psDeviceData)
+#endif /* HDMI_PDUMP */
+
+    /* Set VGA (640x480p, 60kHz refresh) in DVI mode */
+    psDeviceData->videoParams.mDtdIndex = 0;
+    EdidAddDtd(&psDeviceData->videoParams, 1, 60000);
+
+    /* Step A: Initialize PHY */
+    if ((eError = PhyInitialize(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize PHY\n", __func__);
+        return;
+    }
+
+    /* Step B: Initialize video/frame composer and DTD for VGA/DVI mode */
+    if ((eError = VideoInitialize(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize Video\n", __func__);
+        return;
+    }
+
+    if ((eError = EdidInitialize(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize EDID\n", __func__);
+        return;
+    }
+
+    /* Step C: Read Sinks E-EDID information */
+    if ((eError = EdidRead(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to read EDID\n", __func__);
+        return;
+    }
+
+    /* Step D: Configure video mode */
+    if ((eError = VideoConfigureMode(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to configure video mode\n", __func__);
+        return;
+    }
+
+    /* No need to configure audio on Plato, skip to step F: Configure InfoFrames */
+    if ((eError = VideoConfigureInfoframes(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize PHY\n", __func__);
+        return;
+    }
+
+    /* Step G: Configure HDCP */
+    if ((eError = HdcpConfigure(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize PHY\n", __func__);
+        return;
+    }
+
+#if defined(HDMI_DEBUG)
+    {
+        DTD * dtd = &psDeviceData->videoParams.mDtdList[psDeviceData->videoParams.mDtdActiveIndex];
+        HDMI_DEBUG_PRINT("- %s: Final HDMI timing configuration: \n\nmHdmi: %d\nmEncodingOut: %d\nmColorRes: %d\nmPixelRep: %d\n \
+\nDTD Info: \n\tmCode: %d\n\tmPixelRep: %d\n\tmPixelClock: %d\n\tmInterlaced: %d\n\tmHActive: %d\n\tmHBlanking: %d\n\tmHBorder: %d\n\t\
+mHImageSize: %d\n\tmHFrontPorchWidth: %d\n\tmHSyncPulseWidth: %d\n\tmHsyncPolarity: %d\n\tmVActive: %d\n\tmVBlanking: %d\n\tmVBorder: %d\n\t\
+mVImageSize: %d\n\tmVFrontPorchWidth: %d\n\tmVSyncPulseWidth: %d\n\tmVsyncPolarity: %d\n\n", __func__, psDeviceData->videoParams.mHdmi,
+psDeviceData->videoParams.mEncodingOut, psDeviceData->videoParams.mColorResolution, psDeviceData->videoParams.mPixelRepetitionFactor,
+dtd->mCode, dtd->mPixelRepetitionInput, dtd->mPixelClock, dtd->mInterlaced, dtd->mHActive, dtd->mHBlanking, dtd->mHBorder,
+dtd->mHImageSize, dtd->mHFrontPorchWidth, dtd->mHSyncPulseWidth, dtd->mHSyncPolarity, dtd->mVActive, dtd->mVBlanking, dtd->mVBorder,
+dtd->mVImageSize, dtd->mVFrontPorchWidth, dtd->mVSyncPulseWidth, dtd->mVSyncPolarity);
+    }
+#endif
+
+    if (psDeviceData->pfnPDPInitialize)
+    {
+        eError = psDeviceData->pfnPDPInitialize(&(psDeviceData->videoParams),
+                                                  psDeviceData->ui32Instance);
+    }
+    else
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to get PDP<->Hdmi interface\n", __func__);
+        eError = PVRSRV_ERROR_INIT_FAILURE;
+    }
+
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: PDP initialization failed \n", __func__);
+    }
+
+}
+
+PVRSRV_ERROR HDMIDrvInit(void *pvDevice, HDMI_DEVICE **ppsDeviceData)
+{
+    HDMI_DEVICE *psDeviceData;
+    PVRSRV_ERROR eError;
+
+    if (ppsDeviceData == NULL)
+    {
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    DC_OSSetDrvName(DRVNAME);
+
+    HDMI_CHECKPOINT;
+
+    #if defined(VIRTUAL_PLATFORM)
+    HDMI_ERROR_PRINT("-%s: HDMI not supported on VP\n", __func__);
+    #endif
+
+    psDeviceData = DC_OSCallocMem(sizeof(*psDeviceData));
+    if (psDeviceData == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to allocate device data (size %d)\n", __func__, sizeof(*psDeviceData));
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto FAIL_DEV_ALLOC;
+    }
+
+    psDeviceData->pvDevice = pvDevice;
+    psDeviceData->bHPD = IMG_FALSE;
+    psDeviceData->ui32Instance = g_uiNumHDMIInstances++;
+
+    DC_OSSpinLockCreate(&psDeviceData->irqLock);
+
+    eError = DC_OSPVRServicesConnectionOpen(&psDeviceData->hPVRServicesConnection);
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to open connection to PVR Services (%d)\n", __func__, eError);
+        goto FAIL_CONN_CLOSE;
+    }
+
+    eError = DC_OSPVRServicesSetupFuncs(psDeviceData->hPVRServicesConnection, &psDeviceData->sPVRServicesFuncs);
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to setup PVR Services function table (%d)\n", __func__, eError);
+        goto FAIL_CONN_CLOSE;
+    }
+
+    HDMI_DEBUG_PRINT("- %s: Installing HDMI interrupt handler in services\n", __func__);
+
+    eError = psDeviceData->sPVRServicesFuncs.pfnSysInstallDeviceLISR(pvDevice,
+        PLATO_IRQ_HDMI,
+        DRVNAME,
+        HdmiLISRHandler,
+        psDeviceData,
+        &psDeviceData->hLISRData);
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to install system Device LISR (%s) unit %u\n",
+        __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError),
+        psDeviceData->ui32Instance);
+        goto FAIL_DETACH_ISR;
+    }
+
+    INIT_DELAYED_WORK(&psDeviceData->delayedWork, HdmiMISRHandler);
+
+    HDMI_DEBUG_PRINT("- %s: Reserving HDMI Core registers\n", __func__);
+
+    /* Reserve and map the HDMI registers */
+    psDeviceData->sHDMIRegCpuPAddr.uiAddr =
+        DC_OSAddrRangeStart(psDeviceData->pvDevice, SYS_PLATO_REG_PCI_BASENUM) + SYS_PLATO_REG_HDMI_OFFSET;
+
+    if (DC_OSRequestAddrRegion(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE, DRVNAME) == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to reserve the HDMI registers\n", __func__);
+        psDeviceData->sHDMIRegCpuPAddr.uiAddr = 0;
+        eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+        goto FAIL_CONN_CLOSE;
+    }
+
+    psDeviceData->pvHDMIRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE);
+    if (psDeviceData->pvHDMIRegCpuVAddr == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to map HDMI registers\n", __func__);
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto RELEASE_MEM_REGION;
+    }
+
+    /* Check for invalid register space */
+    if (HDMI_READ_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_ID_DESIGN_ID_OFFSET) == 0xFFFFFFFF)
+    {
+        HDMI_DEBUG_PRINT("- %s: Invalid HDMI register space", __func__);
+        eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+        goto RELEASE_MEM_REGION;
+    }
+
+    /*
+     * Map Plato TOP registers - needed for pixel clock setup.
+     */
+    psDeviceData->sTopRegCpuPAddr.uiAddr =
+        DC_OSAddrRangeStart(psDeviceData->pvDevice, SYS_PLATO_REG_PCI_BASENUM) + SYS_PLATO_REG_CHIP_LEVEL_OFFSET;
+
+    psDeviceData->pvTopRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sTopRegCpuPAddr, SYS_PLATO_REG_CHIP_LEVEL_SIZE);
+    if (psDeviceData->pvTopRegCpuVAddr == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to map TOP registers\n", __func__);
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto RELEASE_MEM_REGION;
+    }
+
+    PrintDebugInformation(psDeviceData);
+
+    HdmiEnableHPDInterrupt(psDeviceData);
+
+    *ppsDeviceData = psDeviceData;
+
+    HDMI_CHECKPOINT;
+
+    psDeviceData->mInitialized = IMG_TRUE;
+
+    return PVRSRV_OK;
+
+RELEASE_MEM_REGION:
+    DC_OSReleaseAddrRegion(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE);
+FAIL_DETACH_ISR:
+    /* Uninstall the ISR as we've failed the later stage of initialisation */
+    HDMI_ERROR_PRINT("- %s: Uninstalling Services ISR, Unit %u\n", __func__,
+                     psDeviceData->ui32Instance);
+    g_uiNumHDMIInstances--;
+    if (psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR != NULL)
+    {
+        psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+        DC_OSSpinLockDestroy(&psDeviceData->irqLock);
+    }
+    else
+    {
+        HDMI_ERROR_PRINT("- %s: SysUninstallDeviceLISR handle not present. Power-cycle system to reset\n", __func__);
+    }
+FAIL_CONN_CLOSE:
+    DC_OSFreeMem(psDeviceData);
+
+FAIL_DEV_ALLOC:
+
+    return eError;
+}
+
+void HDMIDrvDeInit(HDMI_DEVICE *psDeviceData)
+{
+    HDMI_CHECKPOINT;
+    if (psDeviceData)
+    {
+
+        HDMI_DEBUG_PRINT(" - %s: DeInit unit %u\n", __func__,
+                         psDeviceData->ui32Instance);
+
+        /* Unregister the LISR - we have to disable and flush the deferred
+         * worker threads here too
+         */
+        if (psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR != NULL)
+        {
+            HDMI_DEBUG_PRINT(" - %s: Uninstall Services LISR unit %u\n",
+                             __func__, psDeviceData->ui32Instance);
+            psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+            HDMI_DEBUG_PRINT(" - %s: Cleanup delayed work queues, unit %u\n",
+                             __func__, psDeviceData->ui32Instance);
+
+            (void) flush_delayed_work(&psDeviceData->delayedWork);
+            (void) cancel_delayed_work_sync(&psDeviceData->delayedWork);
+
+            DC_OSSpinLockDestroy(&psDeviceData->irqLock);
+        }
+
+        DC_OSUnmapPhysAddr(psDeviceData->pvHDMIRegCpuVAddr, SYS_PLATO_REG_HDMI_SIZE);
+        DC_OSReleaseAddrRegion(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE);
+        DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+        HDMI_CHECKPOINT;
+        DC_OSFreeMem(psDeviceData);
+        g_uiNumHDMIInstances--;
+    }
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/phy.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/phy.c
new file mode 100644
index 000000000000..84988b718451
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/phy.c
@@ -0,0 +1,761 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "phy.h"
+#include <linux/delay.h>
+
+/*
+ * See tables A.1 and B.1 in PHY datasheet
+ */
+#define MinPhyPixelClock    2500
+#define MaxPhyPixelClock    59400
+
+static const struct PhyGenericConfig
+{
+    IMG_UINT32  PixClock;
+    IMG_UINT32  OPMODE_PLLCFG;
+    IMG_UINT32  PLLCURRCTRL;
+    IMG_UINT32  PLLGMPCTRL;
+} gsPhyConfig_8bit[] =
+{
+    // PixelClock.    Reg: 0x06    0x10a   0x15
+    { MinPhyPixelClock,   0x00B3, 0x0018, 0x0000 },
+    {  4950,              0x0072, 0x0028, 0x0001 },
+    {  9450,              0x0051, 0x0038, 0x0002 },
+    { 18562,              0x0040, 0x0038, 0x0003 },
+    { 34850,              0x1A40, 0x0018, 0x0003 },
+    { MaxPhyPixelClock+1, 0x1A40, 0x0018, 0x0003 },    // last entry is invalid and must be > Max allowed
+};
+static const IMG_UINT32 NumPhyConfigs = ARRAY_SIZE(gsPhyConfig_8bit);
+
+#define PHY_POLL(base, offset, mask, error) \
+    { \
+        IMG_UINT16 timeout = 0; \
+        while (!(HDMI_READ_CORE_REG(base, offset) & mask) && \
+            timeout < HDMI_PHY_TIMEOUT_MS) \
+        { \
+            DC_OSDelayus(HDMI_PHY_POLL_INTERVAL_MS * 1000); \
+            timeout += HDMI_PHY_POLL_INTERVAL_MS; \
+        } \
+        if (timeout == HDMI_PHY_TIMEOUT_MS) \
+        { \
+            HDMI_ERROR_PRINT("- %s: PHY timed out polling on 0x%x in register offset 0x%x\n", __func__, mask, offset); \
+            goto error; \
+        } \
+    }
+
+bool PhyIsPclkSupported(IMG_UINT16 pixelClock)
+{
+    return ((pixelClock >= MinPhyPixelClock) && (pixelClock <= MaxPhyPixelClock));
+}
+
+#if defined(READ_HDMI_PHY_REGISTERS)
+static IMG_UINT16 PhyI2CRead(IMG_CPU_VIRTADDR base, IMG_UINT8 addr)
+{
+    IMG_UINT16 data;
+
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_ADDRESS_OFFSET, addr);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_OPERATION_OFFSET,
+        SET_FIELD(HDMI_PHY_I2CM_OPERATION_RD_START, HDMI_PHY_I2CM_OPERATION_RD_MASK, 1));
+    /* Wait for done indication */
+    //PHY_POLL(base, HDMI_IH_I2CMPHY_STAT0_OFFSET, HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_MASK, ERROR);
+    mdelay(10);
+    /* read the data registers */
+    data  = (HDMI_READ_CORE_REG(base, HDMI_PHY_I2CM_DATAI_1_OFFSET) & 0xFF) << 8;
+    data |= (HDMI_READ_CORE_REG(base, HDMI_PHY_I2CM_DATAI_0_OFFSET) & 0xFF);
+
+    return data;
+}
+#endif // READ_HDMI_PHY_REGISTERS
+
+PVRSRV_ERROR PhyI2CWrite(IMG_CPU_VIRTADDR base, IMG_UINT8 addr, IMG_UINT16 data)
+{
+    HDMI_CHECKPOINT;
+
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_ADDRESS_OFFSET, addr);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_DATAO_1_OFFSET, (data >> 8));
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_DATAO_0_OFFSET, data & 0xFF);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_OPERATION_OFFSET,
+        SET_FIELD(HDMI_PHY_I2CM_OPERATION_WR_START, HDMI_PHY_I2CM_OPERATION_WR_MASK, 1));
+
+    /* Wait for done interrupt */
+    PHY_POLL(base, HDMI_IH_I2CMPHY_STAT0_OFFSET, HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_MASK, ERROR);
+
+    HDMI_WRITE_CORE_REG(base, HDMI_IH_I2CMPHY_STAT0_OFFSET, 0x3);
+
+    return PVRSRV_OK;
+
+ERROR:
+    return PVRSRV_ERROR_TIMEOUT;
+}
+
+/*
+Configure PHY based on pixel clock, color resolution, pixel repetition, and PHY model
+
+NOTE: This assumes PHY model TSMC 28-nm HPM/ 1.8V
+*/
+void PhyConfigureMode(HDMI_DEVICE * pvDevice)
+{
+    IMG_UINT16 pixelClock;
+    IMG_UINT8 colorRes;
+
+    HDMI_CHECKPOINT;
+
+    /* Init slave address */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+
+    pixelClock = pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex].mPixelClock;
+    colorRes = pvDevice->videoParams.mColorResolution;
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_PDDQ_START, HDMI_PHY_CONF0_PDDQ_MASK, 1));
+
+    //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0000, 0x13); /* PLLPHBYCTRL */
+    //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCLKBISTPHASE_OFFSET, 0x0006);
+    /* RESISTANCE TERM 133Ohm Cfg  */
+    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_TXTERM_OFFSET, 0x0004); /* TXTERM */
+    /* REMOVE CLK TERM */
+    //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKCALCTRL_OFFSET, 0x8000); /* CKCALCTRL */
+
+    HDMI_DEBUG_PRINT(" - %s: pixel Clock: %d, colorResolution: %d\n", __func__, pixelClock, colorRes);
+
+    if ((pixelClock < MinPhyPixelClock) || (pixelClock > MaxPhyPixelClock))
+    {
+        HDMI_ERROR_PRINT("- %s: pixel clock %d out of range [%d,%d]", __func__, pixelClock, MinPhyPixelClock, MaxPhyPixelClock);
+        return;
+    }
+
+    // For 8-bit color formats look up the register values in table based on PHY datasheet.
+    if (colorRes == 8)
+    {
+        IMG_UINT32 CKSYMTXCTRL;
+        IMG_UINT32 VLEVCTRL;
+        IMG_UINT32 ipix;
+        for (ipix  = 0; ipix < NumPhyConfigs-1; ++ipix)
+        {
+            if (gsPhyConfig_8bit[ipix+1].PixClock > pixelClock)
+                break;
+        }
+        if (ipix == NumPhyConfigs-1)
+        {
+            HDMI_ERROR_PRINT("- %s: pixel clock %d > MaxPhyPixelClock %d", __func__, pixelClock, MaxPhyPixelClock);
+            return;
+        }
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, gsPhyConfig_8bit[ipix].OPMODE_PLLCFG);
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   gsPhyConfig_8bit[ipix].PLLCURRCTRL);
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    gsPhyConfig_8bit[ipix].PLLGMPCTRL);
+
+        // Clock Symbol and Transmitter Control Register 0x09
+        if (pixelClock*10 < 29700000)       // 2.97GHz
+            CKSYMTXCTRL = 0x8009;
+        else if (pixelClock*10 < 34000000)  // 3.4GHz
+            CKSYMTXCTRL = 0x8019;
+        else
+            CKSYMTXCTRL = 0x8029;
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, CKSYMTXCTRL);
+
+        // Voltage Level Control Register 0x0E
+        if (pixelClock*10 < 14850000)       // 1.485GHz
+            VLEVCTRL = 0x0232;
+        else if (pixelClock*10 < 22200000)  // 2.22GHz
+            VLEVCTRL = 0x0230;
+        else if (pixelClock*10 < 34000000)  // 3.4GHz
+            VLEVCTRL = 0x0273;
+        else
+            VLEVCTRL = 0x014A;
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, VLEVCTRL);
+    }
+    else    // old code for > 8-bit. Needs to be modified as for 8-bit.
+    switch (pixelClock)
+    {
+        // See dwc_hdmi20_tx_ns_6gbps_tsmc28hpm18_databook.pdf
+		case 2520:
+			switch (colorRes)
+			{
+				case 8:
+					/* PLL/MPLL Cfg */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x00b3);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0018); /* CURRCTRL */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0000); /* GMPCTRL */
+					break;
+				case 10: /* TMDS = 31.50MHz */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2153);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0018);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0000);
+					break;
+				case 12: /* TMDS = 37.80MHz */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x40F3);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0018);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0000);
+					break;
+				case 16: /* TMDS = 50.40MHz */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x60B2);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0028);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0001);
+					break;
+				default:
+					HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+					break;
+			}
+			/* PREEMP Cgf 0.00 */
+			//PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, 0x8009); /* CKSYMTXCTRL */
+			/* TX/CK LVL 10 */
+			//PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0251); /* VLEVCTRL */
+			break;
+        case 2700:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x00B3);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0018);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0000);
+	                break;
+	            case 10: /* TMDS = 33.75MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2153);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0018);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0000);
+	                break;
+	            case 12: /* TMDS = 40.50MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40F3);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0018);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0000);
+	                break;
+	            case 16: /* TMDS = 54MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x60B2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            default:
+	                HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+	                break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 5040:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 63MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 75.60MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40A2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 16: /* TMDS = 100.80MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+	            default:
+	                HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+	                break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 5400:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 67.50MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 81MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40A2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 16: /* TMDS = 108MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+	            default:
+	                HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+	                break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 5940:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 74.25MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 89.10MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40A2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 16: /* TMDS = 118.80MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 7200:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 90MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 108MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+	            case 16: /* TMDS = 144MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        /* 74.25 MHz pixel clock (720p)*/
+        case 7425:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0072);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0028);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0001);
+                    break;
+                case 10: /* TMDS = 92.812MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0002);
+                    break;
+                case 12: /* TMDS = 111.375MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0002);
+                    break;
+                case 16: /* TMDS = 148.5MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6071);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0002);
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, 0x8009);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0251);
+            break;
+
+        case 10080:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 126MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 12: /* TMDS = 151.20MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 16: /* TMDS = 201.60MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 10100:
+        case 10225:
+        case 10650:
+        case 10800:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 135MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 12: /* TMDS = 162MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 16: /* TMDS = 216MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 11880:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 148.50MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 12: /* TMDS = 178.20MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 16: /* TMDS = 237.60MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+
+        case 14400:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 180MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 12: /* TMDS = 216MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4064);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 16: /* TMDS = 288MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 14625:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 14850:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+                    break;
+                case 10: /* TMDS = 185.62MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x214C);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0211);
+                    break;
+                case 12: /* TMDS = 222.75MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4064);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0211);
+                    break;
+                case 16: /* TMDS = 297MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0273);
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, 0x8009);
+            break;
+        case 15400:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 24150:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0040); // section 11.2.7 and table B.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038); // section 11.2.17 and table B.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003); // section 11.2.20 and table B.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251); // section 11.2.15 and table A.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009); // section 11.2.10
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        default:
+            HDMI_ERROR_PRINT("- %s: \n\n\n*****Unsupported pixel clock ******\n\n\n!", __func__);
+            break;
+    }
+
+    {
+        IMG_UINT32 value = HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET);
+        value = SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_TXPWRON_START, HDMI_PHY_CONF0_TXPWRON_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_SVSRET_START, HDMI_PHY_CONF0_SVSRET_MASK, 1);
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET, value);
+    }
+
+#if defined(READ_HDMI_PHY_REGISTERS)
+    {
+        static IMG_UINT8 addresses[] =
+        {
+            PHY_OPMODE_PLLCFG_OFFSET,
+            PHY_PLLCURRCTRL_OFFSET,
+            PHY_PLLGMPCTRL_OFFSET,
+            PHY_VLEVCTRL_OFFSET,
+            PHY_CKSYMTXCTRL_OFFSET,
+        };
+        IMG_UINT16 data;
+        IMG_UINT32 n;
+        for (n = 0; n < sizeof(addresses); ++n)
+        {
+            data = PhyI2CRead(pvDevice->pvHDMIRegCpuVAddr, addresses[n]);
+            printk(KERN_INFO "HDMI I2 addr 0x%02x = 0x%04x\n", addresses[n], data);
+        }
+    }
+#endif // READ_HDMI_PHY_REGISTERS
+}
+
+void PhyReset(HDMI_DEVICE * pvDevice, IMG_UINT8 value)
+{
+    /* Handle different types of PHY here... */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_MC_PHYRSTZ_OFFSET, value & 0x1);
+    msleep(HDMI_PHY_RESET_TIME_MS);
+}
+
+PVRSRV_ERROR PhyPowerDown(HDMI_DEVICE * pvDevice)
+{
+    /* For HDMI TX 1.4 PHY, power down by placing PHY in reset */
+    PhyReset(pvDevice, 0);
+    PhyReset(pvDevice, 1);
+    PhyReset(pvDevice, 0);
+    PhyReset(pvDevice, 1);
+    return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PhyWaitLock(HDMI_DEVICE * pvDevice)
+{
+    PHY_POLL(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_STAT0_OFFSET, HDMI_PHY_STAT0_TX_PHY_LOCK_MASK, ERROR);
+    return PVRSRV_OK;
+
+ERROR:
+    return PVRSRV_ERROR_TIMEOUT;
+}
+
+PVRSRV_ERROR PhyInitialize(HDMI_DEVICE * pvDevice)
+{
+
+    /* Init slave address */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/video.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/video.c
new file mode 100644
index 000000000000..23fb7f36bed2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/video.c
@@ -0,0 +1,868 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+
+#include "video.h"
+#include "phy.h"
+#include "plato_top_regs.h"
+#include <linux/delay.h>
+
+//
+// Pixel clock and core clock are driven by same PLL.
+// For a given pixel clock find a list of core clocks that match.
+// Print these out for the user's information.
+static void FindExactCoreClocks(HDMI_DEVICE* pvDevice, IMG_UINT32 ui32PixelClock)
+{
+    // divide by 3 up to 48, or 4 up to 64 are also valid, but VideoEnablePixelClock doesn't use this.
+    #define VALID_PIXDIV(d) (((d) <= 16) || (((d) <= 32) && !((d) & 1)))
+    const IMG_UINT32 MinPLL =  700000000;  // 700MHz
+    const IMG_UINT32 MaxPLL = 1650000000;  // 1.65GHz
+    const IMG_UINT32 MinCoreClk = 396000000;  // 396MHz
+    const IMG_UINT32 MaxCoreClk = 600000000;  // 600MHz
+    const IMG_UINT32 mindiv = MinPLL / ui32PixelClock;
+    const IMG_UINT32 maxdiv = MaxPLL / ui32PixelClock;
+    #define MaxStoredClocks 10
+    IMG_UINT32 ValidPixelClocks[MaxStoredClocks+1] = { 0 };
+    IMG_UINT32 vpi = 0;
+    IMG_UINT32 pixdiv;
+
+    for (pixdiv = mindiv; pixdiv <= maxdiv; ++pixdiv)
+    {
+        IMG_UINT32 pll;
+        IMG_UINT32 d;
+
+        if (!VALID_PIXDIV(pixdiv))
+            continue;
+        pll = ui32PixelClock * pixdiv;
+        if (pll < MinPLL)
+            continue;
+        if (pll > MaxPLL)
+            break;
+        // find nearest GPU clock to current one for this PLL speed
+        for (d = 1; d <= 4; ++d)
+        {
+            IMG_UINT32 gpuclk = pll / d;
+            if (gpuclk < MinCoreClk)
+                break;
+            if (gpuclk > MaxCoreClk)
+                continue;
+            if (vpi < MaxStoredClocks)
+            {
+                // simple insertion sort
+                IMG_UINT32 i, j;
+                for (i = 0; i < vpi; ++i)
+                {
+                    if (ValidPixelClocks[i] == gpuclk)
+                        break;      // already present
+                    if (gpuclk < ValidPixelClocks[i])
+                    {
+                        for (j = vpi++; j > i; --j)
+                            ValidPixelClocks[j] = ValidPixelClocks[j-1];
+                        ValidPixelClocks[i] = gpuclk;
+                        break;
+                    }
+                }
+                if (i == vpi)
+                    ValidPixelClocks[vpi++] = gpuclk;
+            }
+        }
+    }
+    if (vpi > 0)
+    {
+        IMG_UINT32 i;
+        HDMI_LOG_ALWAYS(" Current core clock %u.%03uMHz: core clocks to achieve exact pixel clock %ukHz:\n",
+                                pvDevice->ui32CoreClockSpeed/1000000, (pvDevice->ui32CoreClockSpeed%1000000)/1000, ui32PixelClock/1000);
+        for (i = 0; i < vpi; ++i)
+            HDMI_LOG_ALWAYS("\t%u\n", ValidPixelClocks[i]);
+    }
+}
+
+
+static int SetPlatoPDPClockDivider(HDMI_DEVICE* pvDevice, IMG_UINT32 pll_clock, IMG_UINT32 pixel_clock)
+{
+    // max can probably go higher if we need to in future. We need about 300MHz for 4Kx2K@30 monitors
+    const IMG_UINT32 MaxPDPClk = (450*1000*1000)+1;
+    IMG_UINT32 best_clk = MaxPDPClk;
+    IMG_UINT32 reg_value = 0;
+    IMG_UINT32 min_clk;
+    IMG_UINT32 d0, d1;
+
+    // we always want PDP core to be >= PLATO_MIN_PDP_CLOCK_SPEED
+    min_clk = (pixel_clock > PLATO_MIN_PDP_CLOCK_SPEED) ? pixel_clock : PLATO_MIN_PDP_CLOCK_SPEED;
+
+    // we have 2x 2-bit dividers for Plato PDP core clock.
+    // start from largest dividers to get to lowest PDP core clock >= pixel clock.
+    for (d0 = 4; d0 >= 1; --d0)
+    {
+        for (d1 = d0; d1 >= 1; --d1)
+        {
+            IMG_UINT32 pdp_clk = pll_clock / (d0*d1);
+            if (pdp_clk >= best_clk)
+                break;	// next divider in loop only makes it larger again
+            if (pdp_clk >= min_clk)
+            {
+                reg_value = ((d0-1) << PLATO_CR_PDPV0_DIV_0_SHIFT) | ((d1-1) << PLATO_CR_PDPV1_DIV_0_SHIFT);
+                best_clk = pdp_clk;
+            }
+        }
+    }
+
+    if (best_clk < MaxPDPClk)
+    {
+        // turn off the clock first
+        OSWriteHWReg32(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_PDP_CLK_CTRL, 0);
+        TOP_REG_POLL(pvDevice->pvTopRegCpuVAddr,   PLATO_TOP_CR_PDP_CLK_CTRL, 0, 0);
+
+        reg_value |= (1 << PLATO_CR_PDPG_GATE_EN_SHIFT);
+        OSWriteHWReg32(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_PDP_CLK_CTRL, reg_value);
+        TOP_REG_POLL(pvDevice->pvTopRegCpuVAddr,   PLATO_TOP_CR_PDP_CLK_CTRL, reg_value, reg_value);
+
+        HDMI_LOG_ALWAYS(" PDP clock speed set to %u\n", best_clk);
+    }
+    else
+        HDMI_LOG_ALWAYS(" ERROR: Cannot set PDP core clock for PLL %u, pixel clock %u\n", pll_clock, pixel_clock);
+
+    return (best_clk < MaxPDPClk);
+}
+
+/*
+ * Pixel Clock setup
+ */
+static void VideoEnablePixelClock(HDMI_DEVICE * pvDevice, IMG_UINT32 ui32PixelClock)
+{
+    IMG_UINT32 ui32Div0;
+    IMG_UINT32 ui32Div1;
+    IMG_UINT32 ui32RegVal;
+    IMG_UINT32 ui32PLLClock = pvDevice->ui32PLLClockSpeed;
+    ui32PixelClock *= 10000;
+
+    /*
+     * Obtain divisor, round to nearest.
+     */
+    ui32Div1 = (ui32PLLClock + ui32PixelClock / 2) / ui32PixelClock;
+
+    if (ui32Div1 <= 32)
+    {
+        if (ui32Div1 < 17)
+        {
+            ui32Div0 = 0;
+            ui32Div1--;
+        }
+        else
+        {
+            ui32Div0 = 1;
+            ui32Div1 /= 2;
+            ui32Div1--;
+        }
+    }
+    else
+    {
+        HDMI_DEBUG_PRINT("- %s: Cannot correctly set up dividers to set pixel clock to requested value! Setting dividers to maximum values ", __func__);
+        ui32Div0 = 1;
+        ui32Div1 = 15;
+    }
+
+    // now we know the actual pixel clock ensure the PDP core clock is fast enough.
+    SetPlatoPDPClockDivider(pvDevice, ui32PLLClock, ui32PLLClock / ((ui32Div1+1) * (ui32Div0+1)));
+
+    // and set the pixel clock after the PDP core clock.
+    ui32RegVal = (PLATO_CR_HDMIG_GATE_EN_MASK | (ui32Div1 << PLATO_CR_HDMIV1_DIV_0_SHIFT) | (ui32Div0 << PLATO_CR_HDMIV0_DIV_0_SHIFT));
+
+    OSWriteHWReg32(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32RegVal);
+    TOP_REG_POLL(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32RegVal, ui32RegVal);
+
+    HDMI_DEBUG_PRINT("- %s: Setting dividers %d,%d for requested PIXEL clock %ukHz\n", __func__,
+                                            ui32Div0+1, ui32Div1+1, ui32PixelClock/1000);
+    {
+        // Always print out some log information about clock speeds.
+        // Consider match exact if within 10000Hz (EDID is in units of 10000Hz)
+        IMG_UINT32 diff = ui32PixelClock - ui32PLLClock/((ui32Div0+1)*(ui32Div1+1));
+        if ((diff < 10000) || (diff > 0xFFFFD8F0))
+            HDMI_LOG_ALWAYS(" Exact pixel clock of %ukHz achieved.\n", ui32PixelClock/1000);
+        else
+            HDMI_LOG_ALWAYS(" Requested pixel clock %ukHz, actual pixel clock %ukHz.",
+                                                ui32PixelClock/1000, ui32PLLClock/(1000*(ui32Div0+1)*(ui32Div1+1)));
+    }
+
+    FindExactCoreClocks(pvDevice, ui32PixelClock);
+}
+
+/*
+
+Sets up Video Packetizer registers
+
+*/
+static bool VideoConfigureVP(HDMI_DEVICE * pvDevice)
+{
+    VIDEO_PARAMS * pParams = &pvDevice->videoParams;
+    IMG_UINT8 regValue = 0;
+    IMG_UINT8 outputSelect = 0;
+    IMG_UINT8 remapSize = 0;
+    IMG_UINT8 colorDepth = 0;
+
+    HDMI_CHECKPOINT;
+
+    if (pParams->mEncodingOut == ENCODING_RGB ||
+        pParams->mEncodingOut == ENCODING_YCC444 ||
+        pParams->mEncodingOut == ENCODING_YCC420)
+    {
+        if (pParams->mColorResolution == 0)
+            outputSelect = 3;
+        else if (pParams->mColorResolution == 8)
+        {
+            colorDepth = 4;
+            outputSelect = 3;
+        }
+        else if (pParams->mColorResolution == 10)
+            colorDepth = 5;
+        else if (pParams->mColorResolution == 12)
+            colorDepth = 6;
+        else if (pParams->mColorResolution == 16)
+            colorDepth = 7;
+        else
+        {
+            HDMI_ERROR_PRINT("- %s: Found unsupported color resolution (%d)", __func__, pParams->mColorResolution);
+            return false;
+        }
+    }
+    else if (pParams->mEncodingOut == ENCODING_YCC422)
+    {
+        if ((pParams->mColorResolution == 8)
+                || (pParams->mColorResolution == 0))
+            remapSize = 0;
+        else if (pParams->mColorResolution == 10)
+            remapSize = 1;
+        else if (pParams->mColorResolution == 12)
+            remapSize = 2;
+        else
+        {
+            HDMI_ERROR_PRINT("- %s: Invalid color remap size: %d", __func__, pParams->mColorResolution);
+            return false;
+        }
+        outputSelect = 1;
+    }
+    else
+    {
+        HDMI_ERROR_PRINT("- %s: Invalid output encoding type: %d", __func__, pParams->mEncodingOut);
+        return false;
+    }
+
+    HDMI_DEBUG_PRINT(" - %s: outputSelect: %d, colorDepth: %d\n", __func__, outputSelect, colorDepth);
+
+    regValue = SET_FIELD(HDMI_VP_PR_CD_COLOR_DEPTH_START,
+        HDMI_VP_PR_CD_COLOR_DEPTH_MASK,
+        colorDepth) |
+    SET_FIELD(HDMI_VP_PR_CD_DESIRED_PR_FACTOR_START,
+        HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK,
+        pParams->mPixelRepetitionFactor);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_PR_CD_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_VP_STUFF_IDEFAULT_PHASE_START,
+        HDMI_VP_STUFF_IDEFAULT_PHASE_MASK,
+        pParams->mPixelPackingDefaultPhase);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_STUFF_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_VP_REMAP_YCC422_SIZE_START,
+        HDMI_VP_REMAP_YCC422_SIZE_MASK,
+        remapSize);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_REMAP_OFFSET, regValue);
+
+    /* Grabbed from reference driver */
+    regValue = 0;
+    if (outputSelect == 0)
+    {
+        /* pixel packing */
+        regValue = SET_FIELD(HDMI_VP_CONF_PP_EN_START,
+            HDMI_VP_CONF_PP_EN_MASK,
+            1);
+    }
+    else if (outputSelect == 1)
+    {
+        /* YCC422 */
+        regValue = SET_FIELD(HDMI_VP_CONF_YCC422_EN_START,
+            HDMI_VP_CONF_YCC422_EN_MASK,
+            1);
+    }
+    else if (outputSelect == 2 || outputSelect == 3)
+    {
+        /* Enable bypass */
+        regValue = SET_FIELD(HDMI_VP_CONF_BYPASS_EN_START,
+            HDMI_VP_CONF_BYPASS_EN_MASK,
+            1) |
+	    SET_FIELD(HDMI_VP_CONF_BYPASS_SELECT_START,
+	        HDMI_VP_CONF_BYPASS_SELECT_MASK,
+	        1);
+
+    }
+
+    regValue |= SET_FIELD(HDMI_VP_CONF_OUTPUT_SELECTOR_START,
+        HDMI_VP_CONF_OUTPUT_SELECTOR_MASK,
+        outputSelect);
+
+    #if defined(EMULATOR)
+    regValue |= SET_FIELD(HDMI_VP_CONF_BYPASS_EN_START,
+        HDMI_VP_CONF_BYPASS_EN_MASK,
+        1) |
+    SET_FIELD(HDMI_VP_CONF_BYPASS_SELECT_START,
+        HDMI_VP_CONF_BYPASS_SELECT_MASK,
+        1);
+    #endif
+    /* pixel packing */
+    regValue |= SET_FIELD(HDMI_VP_CONF_PP_EN_START,
+        HDMI_VP_CONF_PP_EN_MASK,
+        1);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_CONF_OFFSET, regValue);
+
+    /* YCC422 and pixel packing stuffing */
+    regValue = SET_FIELD(HDMI_VP_STUFF_IDEFAULT_PHASE_START,
+        HDMI_VP_STUFF_IDEFAULT_PHASE_MASK,
+        pParams->mPixelPackingDefaultPhase) |
+    SET_FIELD(HDMI_VP_STUFF_YCC422_STUFFING_START,
+        HDMI_VP_STUFF_YCC422_STUFFING_MASK,
+        1) |
+    SET_FIELD(HDMI_VP_STUFF_PP_STUFFING_START,
+        HDMI_VP_STUFF_PP_STUFFING_MASK,
+        1);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_STUFF_OFFSET, regValue);
+
+    return true;
+}
+
+/*
+
+Sets up Video Sampler registers
+
+*/
+static bool VideoConfigureVS(HDMI_DEVICE * pvDevice)
+{
+    VIDEO_PARAMS * pParams = &pvDevice->videoParams;
+    IMG_UINT8 mapCode = 0;
+    IMG_UINT8 regValue = 0;
+
+    HDMI_CHECKPOINT;
+
+	if (pParams->mEncodingIn == ENCODING_RGB ||
+        pParams->mEncodingIn == ENCODING_YCC444 ||
+        pParams->mEncodingIn == ENCODING_YCC420)
+	{
+		if (pParams->mColorResolution == 8 || pParams->mColorResolution == 0)
+			mapCode = 1;
+		else if (pParams->mColorResolution == 10)
+			mapCode = 3;
+		else if (pParams->mColorResolution == 12)
+			mapCode = 5;
+		else if (pParams->mColorResolution == 16)
+			mapCode = 7;
+		else
+		{
+			HDMI_ERROR_PRINT("- %s: Invalid color depth\n", __func__);
+			return false;
+		}
+		mapCode += (pParams->mEncodingIn  == ENCODING_YCC444) ? 8 : 0;
+	}
+	else if (pParams->mEncodingIn == ENCODING_YCC422)
+	{
+		/* YCC422 mapping is discontinued - only map 1 is supported */
+		if (pParams->mColorResolution == 12)
+			mapCode = 18;
+		else if (pParams->mColorResolution == 10)
+			mapCode = 20;
+		else if ((pParams->mColorResolution == 8)
+				|| (pParams->mColorResolution == 0))
+			mapCode = 22;
+		else
+		{
+			HDMI_ERROR_PRINT("- %s: Invalid color remap size: %d", __func__, pParams->mColorResolution);
+			return false;
+		}
+	}
+	else
+	{
+		HDMI_ERROR_PRINT("- %s: Invalid input encoding type: %d", __func__, pParams->mEncodingOut);
+		return false;
+    }
+
+    regValue = SET_FIELD(HDMI_TX_INVID0_INTERNAL_DE_GEN_START, HDMI_TX_INVID0_INTERNAL_DE_GEN_MASK, 0);
+    regValue |= SET_FIELD(HDMI_TX_INVID0_VIDEO_MAPPING_START, HDMI_TX_INVID0_VIDEO_MAPPING_MASK, mapCode);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_INVID0_OFFSET, regValue);
+
+    #if !defined(EMULATOR)
+    /* The following is done in the ref driver */
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_GYDATA0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_GYDATA1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_RCRDATA0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_RCRDATA1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_BCBDATA0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_BCBDATA1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_INSTUFFING_OFFSET, 0x7);
+
+    #endif
+
+    return true;
+}
+
+static bool VideoConfigureFC(HDMI_DEVICE * pvDevice)
+{
+    IMG_UINT32 regValue = 0;
+    VIDEO_PARAMS * pParams = &pvDevice->videoParams;
+    DTD * pDtd = &pParams->mDtdList[pParams->mDtdActiveIndex];
+
+    /* Input video configuration */
+    regValue = SET_FIELD(HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_START,
+            HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_MASK,
+            pDtd->mVSyncPolarity) |
+        SET_FIELD(HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_START,
+            HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_MASK,
+            pDtd->mHSyncPolarity) |
+        SET_FIELD(HDMI_FC_INVIDCONF_DE_IN_POLARITY_START,
+            HDMI_FC_INVIDCONF_DE_IN_POLARITY_MASK,
+            pParams->mDataEnablePolarity) |
+        SET_FIELD(HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_START,
+            HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_MASK,
+            pDtd->mInterlaced) | // Need to check this out, ref driver sets interlaced for both
+        SET_FIELD(HDMI_FC_INVIDCONF_IN_I_P_START,
+            HDMI_FC_INVIDCONF_IN_I_P_MASK,
+            pDtd->mInterlaced) |
+        SET_FIELD(HDMI_FC_INVIDCONF_DVI_MODEZ_START,
+            HDMI_FC_INVIDCONF_DVI_MODEZ_MASK,
+            pParams->mHdmi);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr,
+        HDMI_FC_INVIDCONF_OFFSET,
+        regValue);
+
+    /*
+    Input HActive Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INHACTIV0_H_IN_ACTIV_START,
+        HDMI_FC_INHACTIV0_H_IN_ACTIV_MASK,
+        pDtd->mHActive & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_START,
+        HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_MASK,
+        pDtd->mHActive >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV1_OFFSET, regValue);
+
+    /*
+    Input VActive Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INVACTIV0_V_IN_ACTIV_START,
+        HDMI_FC_INVACTIV0_V_IN_ACTIV_MASK,
+        pDtd->mVActive & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_START,
+        HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_MASK,
+        pDtd->mVActive >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV1_OFFSET, regValue);
+
+    /*
+    Input HBlank Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INHBLANK0_H_IN_BLANK_START,
+        HDMI_FC_INHBLANK0_H_IN_BLANK_MASK,
+        pDtd->mHBlanking & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_START,
+        HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_MASK,
+        pDtd->mHBlanking >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK1_OFFSET, regValue);
+
+    /*
+    Input VBlank Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INVBLANK_V_IN_BLANK_START,
+        HDMI_FC_INVBLANK_V_IN_BLANK_MASK,
+        pDtd->mVBlanking & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVBLANK_OFFSET, regValue);
+
+    /*
+    Input HSync Front Porch (pixel clock cycles from "de" non active edge of the last "de" valid period)
+    */
+    regValue = SET_FIELD(HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_START,
+        HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_MASK,
+        pDtd->mHFrontPorchWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_START,
+        HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_MASK,
+        pDtd->mHFrontPorchWidth >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY1_OFFSET, regValue);
+
+    /*
+    Input VSync Front porch
+    */
+    regValue = SET_FIELD(HDMI_FC_VSYNCINDELAY_V_IN_DELAY_START,
+        HDMI_FC_VSYNCINDELAY_V_IN_DELAY_MASK,
+        pDtd->mVFrontPorchWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINDELAY_OFFSET, regValue);
+
+    /*
+    Input HSync pulse width
+    */
+    regValue = SET_FIELD(HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_START,
+        HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_MASK,
+        pDtd->mHSyncPulseWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_START,
+        HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_MASK,
+        pDtd->mHSyncPulseWidth >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH1_OFFSET, regValue);
+
+    /*
+    Input Vsync pulse width
+    */
+    regValue = SET_FIELD(HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_START,
+        HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_MASK,
+        pDtd->mVSyncPulseWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINWIDTH_OFFSET, regValue);
+
+    return true;
+
+}
+
+
+static void VideoParamsInit(VIDEO_PARAMS * pParams)
+{
+    #if defined(EMULATOR)
+    pParams->mHdmi = 1;
+    pParams->mEncodingOut = ENCODING_RGB;
+    pParams->mEncodingIn = ENCODING_RGB;
+    pParams->mColorResolution = 10;
+    pParams->mPixelRepetitionFactor = 0;
+    pParams->mRgbQuantizationRange = 0;
+    pParams->mPixelPackingDefaultPhase = 0;
+    pParams->mColorimetry = 0;
+    pParams->mScanInfo = 0;
+    pParams->mActiveFormatAspectRatio = 8;
+    pParams->mNonUniformScaling = 0;
+    pParams->mExtColorimetry = ~0;
+    pParams->mItContent = 0;
+    pParams->mEndTopBar = ~0;
+    pParams->mStartBottomBar = ~0;
+    pParams->mEndLeftBar = ~0;
+    pParams->mStartRightBar = ~0;
+    pParams->mCscFilter = 0;
+    pParams->mHdmiVideoFormat = 0;
+    pParams->m3dStructure = 0;
+    pParams->m3dExtData = 0;
+    pParams->mHdmiVic = 0;
+    pParams->mDtdIndex = 0;
+    pParams->mDtdActiveIndex = 0;
+    pParams->mDataEnablePolarity = 1;
+    #else
+    pParams->mEncodingOut = ENCODING_RGB;
+    pParams->mEncodingIn = ENCODING_RGB;
+    #if 1
+    pParams->mHdmi = 0;
+    pParams->mColorResolution = 8;
+    #else
+    pParams->mColorResolution = 10;
+    pParams->mHdmi = 1;
+    #endif
+    pParams->mPixelRepetitionFactor = 0;
+    pParams->mRgbQuantizationRange = 0;
+    pParams->mPixelPackingDefaultPhase = 0;
+    pParams->mColorimetry = 0;
+    pParams->mScanInfo = 0;
+    pParams->mActiveFormatAspectRatio = 8;
+    pParams->mNonUniformScaling = 0;
+    pParams->mExtColorimetry = ~0;
+    pParams->mItContent = 0;
+    pParams->mEndTopBar = ~0;
+    pParams->mStartBottomBar = ~0;
+    pParams->mEndLeftBar = ~0;
+    pParams->mStartRightBar = ~0;
+    pParams->mCscFilter = 0;
+    pParams->mHdmiVideoFormat = 0;
+    pParams->m3dStructure = 0;
+    pParams->m3dExtData = 0;
+    pParams->mHdmiVic = 0;
+    pParams->mDtdIndex = 0;
+    pParams->mDtdActiveIndex = 0;
+    pParams->mDataEnablePolarity = 1;
+    #endif
+}
+
+/*
+
+Steps for Configuring video mode:
+
+1. Power down Tx PHY
+2. Set desired video mode (based on EDID sink information)
+3. Power-on, PLL configuration, and reset Tx PHY
+4. Wait for PHY lock to assert
+
+*/
+PVRSRV_ERROR VideoConfigureMode(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT(" - %s: Configuring video mode for VIC %d\n", __func__,
+        pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex].mCode);
+
+    VideoEnablePixelClock(pvDevice, pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex].mPixelClock);
+
+    /*
+
+    Step D.2: Set desired video mode based on current VideoParams structure
+
+    */
+    if (!VideoConfigureVP(pvDevice))
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    if (!VideoConfigureVS(pvDevice))
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    if (!VideoConfigureFC(pvDevice))
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    /* Place phy in reset */
+    PhyPowerDown(pvDevice);
+
+    /* Audio N and CTS value */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N1_OFFSET, 0xE5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N2_OFFSET, 0x0F);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N3_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS1_OFFSET, 0x19);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS2_OFFSET, 0xD5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS3_OFFSET, 0x02);
+
+    /* Setup PHY for intended video mode */
+    PhyConfigureMode(pvDevice);
+
+    /* Wait for PHY lock */
+    status = PhyWaitLock(pvDevice);
+    if (status != PVRSRV_OK)
+    {
+		goto EXIT;
+    }
+
+    /* From pdump */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CTRLDUR_OFFSET, 0x0C);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLDUR_OFFSET, 0x20);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLSPAC_OFFSET, 0x01);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH0PREAM_OFFSET, 0x0B);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH1PREAM_OFFSET, 0x16);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH2PREAM_OFFSET, 0x21);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_MC_CLKDIS_OFFSET,
+        SET_FIELD(HDMI_MC_CLKDIS_HDCPCLK_DIS_START, HDMI_MC_CLKDIS_HDCPCLK_DIS_MASK, 1));
+
+EXIT:
+    return status;
+}
+
+/*
+
+See HDMI ref driver packets.c
+
+*/
+PVRSRV_ERROR VideoConfigureInfoframes(HDMI_DEVICE * pvDevice)
+{
+    VIDEO_PARAMS * pVideoParams = &pvDevice->videoParams;
+    DTD * pDtd = &pVideoParams->mDtdList[pVideoParams->mDtdActiveIndex];
+    IMG_UINT32 regValue;
+
+    HDMI_CHECKPOINT;
+
+    #if defined(EMULATOR)
+    return PVRSRV_OK; // This breaks on emu
+    #endif
+
+    /* Only relevant for HDMI */
+    if (pVideoParams->mHdmi == 0)
+    {
+        HDMI_DEBUG_PRINT("- %s: mHdmi not set, not configuring infoframes\n", __func__);
+        return PVRSRV_OK;
+    }
+
+    /* AVI CONF0 setup */
+    regValue = SET_FIELD(HDMI_FC_AVICONF0_RGBYCC_START, HDMI_FC_AVICONF0_RGBYCC_MASK, 0);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF0_SCAN_INFO_START,
+        HDMI_FC_AVICONF0_SCAN_INFO_MASK,
+        pVideoParams->mScanInfo);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET, regValue);
+
+    /* AVI CONF1 setup */
+    regValue = SET_FIELD(HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_START,
+        HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_START,
+        (((pDtd->mHImageSize * 10) % pDtd->mVImageSize) > 5) ? 2 : 1); /* 16:9 or 4:3 */
+    regValue |= SET_FIELD(HDMI_FC_AVICONF1_COLORIMETRY_START,
+        HDMI_FC_AVICONF1_COLORIMETRY_MASK,
+        pVideoParams->mColorimetry);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_START,
+        HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_MASK,
+        pVideoParams->mActiveFormatAspectRatio);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF1_OFFSET, regValue);
+
+    /* Go back and set active format valid bit */
+    regValue = SET_FIELD(HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_START,
+        HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_MASK,
+        1);
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET, regValue);
+
+    /* AVI CONF2 setup */
+    regValue = SET_FIELD(HDMI_FC_AVICONF2_IT_CONTENT_START,
+        HDMI_FC_AVICONF2_IT_CONTENT_MASK,
+        pVideoParams->mItContent);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF2_QUANTIZATION_RANGE_START,
+        HDMI_FC_AVICONF2_QUANTIZATION_RANGE_MASK,
+        pVideoParams->mRgbQuantizationRange);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_START,
+        HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_MASK,
+        pVideoParams->mNonUniformScaling);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF2_OFFSET, regValue);
+
+    /* AVI VID setup */
+    if (pDtd->mCode != -1)
+    {
+        regValue = SET_FIELD(HDMI_FC_AVIVID_START, HDMI_FC_AVIVID_MASK, pDtd->mCode);
+    }
+    else
+    {
+        regValue = 0;
+    }
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIVID_OFFSET, regValue);
+
+    /* Set horizontal bars to 0 */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIETB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIETB1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISBB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISBB1_OFFSET, 0);
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET,
+        SET_FIELD(HDMI_FC_AVICONF0_HBAR_VALID_START, HDMI_FC_AVICONF0_HBAR_VALID_MASK, 1));
+
+    /* Set vertical bars to 0 */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIELB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIELB1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISRB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISRB1_OFFSET, 0);
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET,
+        SET_FIELD(HDMI_FC_AVICONF0_VBAR_VALID_START, HDMI_FC_AVICONF0_VBAR_VALID_MASK, 1));
+
+    /* Set out pixel repetition factor */
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_PRCONF_OFFSET,
+        SET_FIELD(HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_START,
+            HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK,
+            ((pDtd->mPixelRepetitionInput + 1) * (pVideoParams->mPixelRepetitionFactor + 1) - 1)));
+
+    return PVRSRV_OK;
+}
+
+/*
+
+Initialize routine will setup video mode for VGA DVI mode to
+start EDID communication
+
+*/
+PVRSRV_ERROR VideoInitialize(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+
+    HDMI_CHECKPOINT;
+
+    VideoParamsInit(&pvDevice->videoParams);
+
+    /* Configure based on video parameters in pvDevice (VGA/DVI on init) */
+    if (VideoConfigureMode(pvDevice) != PVRSRV_OK)
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    /* Force video output on init */
+    {
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGTMDS_2_OFFSET, 0x00); /* R */
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGTMDS_1_OFFSET, 0xFF); /* G */
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGTMDS_0_OFFSET, 0x00); /* B */
+
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGFORCE_OFFSET,
+            SET_FIELD(HDMI_FC_DBGFORCE_FORCE_VIDEO_START, HDMI_FC_DBGFORCE_FORCE_VIDEO_MASK, 0));
+    }
+
+EXIT:
+
+    return status;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..c0e9b009c62d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Kbuild.mk
@@ -0,0 +1,56 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-$(CONFIG_X86) += -mno-soft-float
+
+ccflags-y += \
+ -I$(TOP)/services/display/rogue/plato_hdmi/include \
+ -I$(TOP)/services/display/include \
+ -I$(TOP)/kernel/drivers/staging/imgtec/plato	\
+ -I$(TOP)/include/rogue/public -I$(TOP)/include/public
+
+plato_hdmi-y += \
+	services/display/rogue/plato_hdmi/common/hdmi.o \
+	services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.o \
+	services/display/common/env/linux/dc_osfuncs.o \
+	services/display/rogue/plato_hdmi/common/video.o \
+	services/display/rogue/plato_hdmi/common/edid.o \
+	services/display/rogue/plato_hdmi/common/hdcp.o \
+	services/display/rogue/plato_hdmi/common/phy.o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Linux.mk
new file mode 100644
index 000000000000..29f7568c6d87
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := plato_hdmi
+
+plato_hdmi_type := kernel_module
+plato_hdmi_target := plato_hdmi.ko
+plato_hdmi_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.c
new file mode 100644
index 000000000000..af25b8fde891
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.c
@@ -0,0 +1,354 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include "dc_osfuncs.h"
+#include "hdmi.h"
+
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#define HDMI_DEBUGFS_DISPLAY_ENABLED "display_enabled"
+
+/* Callbacks for PDP<->HDMI interface */
+extern PVRSRV_ERROR PDPInitialize(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance);
+
+/* Helpers for getting DDR/GPU/PLL clock speed */
+extern IMG_UINT32 SysGetPlatoMemClockSpeed(void);
+extern IMG_UINT32 SysGetPlatoCoreClockSpeed(void);
+extern IMG_UINT32 SysGetPlatoPLLClockSpeed(IMG_UINT32 ui32ClockSpeed);
+
+static DLLIST_NODE g_sDeviceDataListHead;    /* List of discovered devices */
+static IMG_UINT32  g_uiNumHDMIDevs;          /* Number of discovered devices */
+
+static struct dentry	*g_psDebugFSEntryDir;/* debugFS top-level directory */
+
+/* HDMI module parameters */
+HDMI_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32HDMIEnabled = 1,
+};
+
+module_param_named(mem_en, sModuleParams.ui32HDMIEnabled, uint, S_IRUGO | S_IWUSR);
+
+static int HdmiDebugFsOpen(struct inode *psINode, struct file *psFile)
+{
+	psFile->private_data = psINode->i_private;
+
+	return 0;
+}
+
+static ssize_t HdmiDebugFsRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	HDMI_DEVICE *psDevice = (HDMI_DEVICE *)psFile->private_data;
+	IMG_BOOL bHDMIEnabled = psDevice->bHDMIEnabled;
+	loff_t uiPosition = *puiPosition;
+	char pszBuffer[] = "N\n";
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	if (bHDMIEnabled == IMG_TRUE)
+	{
+		pszBuffer[0] = 'Y';
+	}
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+	if (copy_to_user(psUserBuffer, &pszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static ssize_t HdmiDebugFsWrite(struct file *psFile,
+				   const char __user *psUserBuffer,
+				   size_t uiCount,
+				   loff_t *puiPosition)
+{
+	HDMI_DEVICE *psDevice = (HDMI_DEVICE *)psFile->private_data;
+	char pszBuffer[3];
+	bool bHDMIEnabled;
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	uiLocCount = min(uiCount, (uiBufferSize - 1U));
+
+	if (copy_from_user(pszBuffer, psUserBuffer, uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	pszBuffer[uiLocCount] = '\0';
+
+	if (strtobool(pszBuffer, &bHDMIEnabled) == 0)
+	{
+		psDevice->bHDMIEnabled = bHDMIEnabled ? IMG_TRUE : IMG_FALSE;
+	}
+
+	return (ssize_t)uiLocCount;
+}
+
+static const struct file_operations gsDisplayEnabledFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = HdmiDebugFsOpen,
+	.read = HdmiDebugFsRead,
+	.write = HdmiDebugFsWrite,
+	.llseek = default_llseek,
+};
+
+static struct pci_dev * hdmi_getdevice(struct pci_dev *psFromDev)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(SYS_RGX_DEV_VENDOR_ID,
+	                          SYS_RGX_DEV_DEVICE_ID,
+	                          psFromDev);
+#if defined(HDMI_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: %x, %x, device %p fromDev %p\n",
+	       __func__,
+	       SYS_RGX_DEV_VENDOR_ID, SYS_RGX_DEV_DEVICE_ID, psPCIDev, psFromDev);
+#endif
+
+	if (psPCIDev != psFromDev)
+	{
+		return psPCIDev;
+	}
+	else
+	{
+#if defined(HDMI_DEBUG)
+		if (psPCIDev != NULL)
+		{
+			printk(KERN_WARNING DRVNAME " - %s: Duplicate entry - terminating scan.\n",
+		         __func__);
+		}
+#endif
+		return NULL;
+	}
+}
+
+static int __init hdmi_init(void)
+{
+	struct pci_dev *psPCIDev, *fromDev = NULL;
+	HDMI_DEVICE *psDeviceData;
+	PVRSRV_ERROR eError;
+	int error;
+	IMG_CHAR aszDevName[4];    /* Location for unit name ("0" .. "N") */
+	static IMG_BOOL bFirstTime = IMG_TRUE;
+
+	#if !defined(HDMI_CONTROLLER)
+	return 0;
+	#endif
+
+	/* Iterate over all PCI instances */
+	dllist_init(&g_sDeviceDataListHead);
+	while ((psPCIDev = hdmi_getdevice(fromDev)) != NULL)
+	{
+		fromDev = psPCIDev;
+#if defined(HDMI_DEBUG)
+		printk(KERN_INFO DRVNAME " - %s: Initializing unit %u, parent %p, dev_data %p\n",
+		        __func__,
+		       g_uiNumHDMIDevs, psPCIDev->dev.parent,
+		       psPCIDev->dev.driver_data);
+#endif
+
+		error = pci_enable_device(psPCIDev);
+		if (error != 0)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to enable PCI device (%d)\n", __func__, error);
+			continue;	/* Continue scan and drop device reference */
+		}
+
+		eError = HDMIDrvInit(&psPCIDev->dev, &psDeviceData);
+		if (eError != PVRSRV_OK)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to initialise device (%d)\n", __func__, eError);
+			continue;	/* Continue scan and drop device reference */
+		}
+		dllist_init(&psDeviceData->sListNode);
+
+		/* To prevent possible problems with system suspend/resume, we don't
+		   keep the device enabled, but rely on the fact that the Rogue driver
+		   will have done a pci_enable_device. */
+		pci_disable_device(psPCIDev);
+
+		/* Create top-level debugFS entries on first iteration only */
+		if (bFirstTime == IMG_TRUE)
+		{
+			bFirstTime = IMG_FALSE;
+
+			g_psDebugFSEntryDir = debugfs_create_dir(DRVNAME, NULL);
+			if (IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs root directory "
+				       "(debugfs entries won't be available)\n", __func__, DRVNAME);
+				g_psDebugFSEntryDir = NULL;
+			}
+
+		}
+
+		if (!IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+		{
+			(void) DC_OSSNPrintf(aszDevName, sizeof(aszDevName), "%d", g_uiNumHDMIDevs);
+
+			psDeviceData->psDebugFSEntryDir = debugfs_create_dir(aszDevName, g_psDebugFSEntryDir);
+			if (IS_ERR_OR_NULL(psDeviceData->psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME
+				       " - %s: Failed to create '%s/%s' debugfs root directory "
+				       "(debugfs entries won't be available)\n",
+				       __func__, DRVNAME, aszDevName);
+				psDeviceData->psDebugFSEntryDir = NULL;
+			}
+			psDeviceData->psDisplayEnabledEntry = debugfs_create_file(
+			                         HDMI_DEBUGFS_DISPLAY_ENABLED,
+			                         S_IFREG | S_IRUGO | S_IWUSR,
+			                         psDeviceData->psDebugFSEntryDir,
+			                         psDeviceData,
+			                         &gsDisplayEnabledFileOps);
+			if (IS_ERR_OR_NULL(psDeviceData->psDisplayEnabledEntry))
+			{
+				printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s/%s' debugfs entry\n",
+				       __func__, aszDevName, HDMI_DEBUGFS_DISPLAY_ENABLED);
+				psDeviceData->psDisplayEnabledEntry = NULL;
+			}
+		}
+
+		/* Initialise the bHDMIEnabled value from the modparam setting */
+		psDeviceData->bHDMIEnabled = (sModuleParams.ui32HDMIEnabled == 1U) ? IMG_TRUE : IMG_FALSE;
+
+		psDeviceData->pfnPDPInitialize = PDPInitialize;
+
+		psDeviceData->ui32CoreClockSpeed = SysGetPlatoCoreClockSpeed();
+		psDeviceData->ui32PLLClockSpeed  = SysGetPlatoPLLClockSpeed(psDeviceData->ui32CoreClockSpeed);
+
+		psDeviceData->ui32Instance = g_uiNumHDMIDevs++;
+		dllist_add_to_tail(&g_sDeviceDataListHead, &psDeviceData->sListNode);
+	}
+
+#if defined(HDMI_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: Discovered %u HDMI instances\n",
+	       __func__, g_uiNumHDMIDevs);
+#endif
+
+	return (g_uiNumHDMIDevs > 0U) ? 0 : -ENODEV;
+}
+
+/*
+ Clear all registered devices from the system
+ */
+static void __exit hdmi_deinit(void)
+{
+
+	DLLIST_NODE *psNode, *psNext;
+	HDMI_DEVICE *psDeviceData;
+
+#if defined(HDMI_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: De-Init Entry %u Devices\n", __func__,
+	       g_uiNumHDMIDevs);
+#endif
+
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		psDeviceData = IMG_CONTAINER_OF(psNode, HDMI_DEVICE, sListNode);
+
+		if (psDeviceData->psDisplayEnabledEntry)
+		{
+			debugfs_remove(psDeviceData->psDisplayEnabledEntry);
+		}
+
+#if defined(HDMI_DEBUG)
+		printk(KERN_INFO DRVNAME " - %s: De-init unit %u\n", __func__,
+		       psDeviceData->ui32Instance);
+#endif
+
+		if (psDeviceData->psDebugFSEntryDir)
+		{
+			debugfs_remove(psDeviceData->psDebugFSEntryDir);
+			psDeviceData->psDebugFSEntryDir = NULL;
+		}
+
+		dllist_remove_node(&psDeviceData->sListNode);
+		HDMIDrvDeInit(psDeviceData);
+		g_uiNumHDMIDevs--;
+	}
+
+	if (g_psDebugFSEntryDir)
+	{
+		debugfs_remove(g_psDebugFSEntryDir);
+		g_psDebugFSEntryDir = NULL;
+	}
+}
+
+module_init(hdmi_init);
+module_exit(hdmi_deinit);
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/edid.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/edid.h
new file mode 100644
index 000000000000..53072673f2bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/edid.h
@@ -0,0 +1,353 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#ifndef DC_EDID_H
+#define DC_EDID_H
+
+#include "hdmi.h"
+
+/*************************************************************************/ /*!
+ EDID Information
+*/ /**************************************************************************/
+#define EDID_SIGNATURE                      0x00ffffffffffff00ull
+#define EDID_BLOCK_SIZE                     128
+#define EDID_CHECKSUM                       0 // per VESA spec, 8 bit checksum of base block
+#define EDID_SLAVE_ADDRESS                  0x50
+#define EDID_SEGMENT_ADDRESS                0x00
+#define EDID_REFRESH_RATE                   60000
+#define EDID_I2C_TIMEOUT_MS					100
+
+/* Extension block tag numbers (sec 2.2.4 of EDID spec) */
+#define EDID_EXT_TAG_CEA                    (0x02)
+#define EDID_EXT_TAG_VTB                    (0x10)
+#define EDID_EXT_TAG_DI                     (0x40)
+#define EDID_EXT_TAG_LS                     (0x50)
+#define EDID_EXT_TAG_DPVL                   (0x60)
+#define EDID_EXT_TAG_BLOCK_MAP              (0xF0)
+#define EDID_EXT_TAG_MFG_DEFINED            (0xFF)
+
+/* CEA Extension defines */
+#define EDID_CEA_REVISION                   (3)
+#define EDID_CEA_DATA_BLOCK_AUDIO_TAG       (1)
+#define EDID_CEA_DATA_BLOCK_VIDEO_TAG       (2)
+#define EDID_CEA_DATA_BLOCK_VENDOR_TAG      (3)
+#define EDID_CEA_DATA_BLOCK_SPEAKER_TAG     (4)
+
+/* Descriptor types */
+#define EDID_DESC_DETAILED_TIMING           0
+#define EDID_DESC_DISPLAY                   1
+
+/* Display descriptor tags */
+enum
+{
+    DISP_DESC_TAG_MFG_START = 0x00,
+    DISP_DESC_TAG_MFG_END = 0x0F,
+    DISP_DESC_TAG_DUMMY,
+    DISP_DESC_TAG_RESERVED_START,
+    DISP_DESC_TAG_RESERVED_END = 0xF6,
+    DISP_DESC_TAG_EST_TIMING_3,
+    DISP_DESC_TAG_CVT_3,
+    DISP_DESC_TAG_DCM,
+    DISP_DESC_TAG_STD_TIMING,
+    DISP_DESC_TAG_COLOR_POINT_DATA,
+    DISP_DESC_TAG_PRODUCT_NAME,
+    DISP_DESC_TAG_RANGE_LIMITS,
+    DISP_DESC_TAG_DATA_STRING,
+    DISP_DESC_TAG_SERIAL_NUMBER,
+};
+
+/* Digital video interface standards.
+ * See chapter 3.6.1 of VESA spec */
+enum
+{
+	DIGITAL_VIDEO_INTERFACE_NULL = 0x0,
+	DIGITAL_VIDEO_INTERFACE_DVI,
+	DIGITAL_VIDEO_INTERFACE_HDMI_A,
+	DIGITAL_VIDEO_INTERFACE_HDMI_B,
+	DIGITAL_VIDEO_INTERFACE_MDDI,
+	DIGITAL_VIDEO_INTERFACE_DISPLAY_PORT,
+};
+
+/* See chapter 3.10.2 of VESA spec */
+typedef struct _DTD_RAW
+{
+    IMG_UINT16                  pixelClock; // 0x0001 - 0xFFFF stored value = pixel clock / 10,000
+
+    IMG_UINT8                   hActiveLower;
+    IMG_UINT8                   hBlankingLower;
+
+    IMG_UINT8                   hBlankingUpper          : 4;
+    IMG_UINT8                   hActiveUpper            : 4;
+
+    IMG_UINT8                   vActiveLower;
+    IMG_UINT8                   vBlankingLower;
+
+    IMG_UINT8                   vBlankingUpper          : 4;
+    IMG_UINT8                   vActiveUpper            : 4;
+
+    IMG_UINT8                   hFrontPorchLower;
+    IMG_UINT8                   hSyncPulseWidthLower;
+
+    // Byte 10
+    IMG_UINT8                   vSyncPulseWidthLower    : 4;
+    IMG_UINT8                   vFrontPorchLower        : 4;
+
+    // Byte 11
+    IMG_UINT8                   vSyncPulseWidthUpper    : 2;
+    IMG_UINT8                   vFrontPorchUpper        : 2;
+    IMG_UINT8                   hSyncPulseWidthUpper    : 2;
+    IMG_UINT8                   hFrontPorchUpper        : 2;
+
+    IMG_UINT8                   hImageSizeLower;
+    IMG_UINT8                   vImageSizeLower;
+    IMG_UINT8                   vImageSizeUpper			: 4;
+    IMG_UINT8                   hImageSizeUpper         : 4;
+    IMG_UINT8                   hBorder;
+    IMG_UINT8                   vBorder;
+
+    // Byte 17
+    IMG_UINT8                   stereoViewing2          : 1;
+    IMG_UINT8                   syncSignalDefs          : 4;
+    IMG_UINT8                   stereoViewing1          : 2;
+    IMG_UINT8                   interlaced              : 1;
+
+} DTD_RAW;
+
+typedef union _ESTABILISHED_TIMING_I
+{
+    IMG_UINT8 value;
+    struct {
+        IMG_UINT8 _800x600_60Hz : 1;
+        IMG_UINT8 _800x600_56Hz : 1;
+        IMG_UINT8 _640x480_75Hz : 1;
+        IMG_UINT8 _640x480_72Hz : 1;
+        IMG_UINT8 _640x480_67Hz : 1;
+        IMG_UINT8 _640x480_60Hz : 1;
+        IMG_UINT8 _720x400_88Hz : 1;
+        IMG_UINT8 _720x400_70Hz : 1;
+    } fields;
+}ESTABILISHED_TIMING_I;
+
+typedef union _STANDARD_TIMING
+{
+    IMG_UINT16 value;
+    struct
+    {
+	    IMG_UINT16 horizontalAddressablePixels : 8;
+        IMG_UINT16 refreshRate : 6;
+        IMG_UINT16 aspectRatio : 2;
+    } fields;
+} STANDARD_TIMING;
+
+typedef union _VIDEO_INPUT_DEFINITION
+{
+    IMG_UINT8 value;
+    struct
+    {
+        IMG_UINT8 serrations : 1; // BIT 0
+        IMG_UINT8 syncTypes : 3; // BIT 3-1
+        IMG_UINT8 videoSetup : 1; // BIT 4
+        IMG_UINT8 signalLevelStandard : 2; // BIT 6-5
+        IMG_UINT8 signalInterface : 1; // BIT 7
+    } analog;
+
+    struct
+    {
+        IMG_UINT8 vidInterfaceSupported : 4; // BIT 3-0
+        IMG_UINT8 colorBitDepth : 3; // BIT 6-4
+        IMG_UINT8 signalInterface : 1; // BIT 7
+    } digital;
+} VIDEO_INPUT_DEFINITION;
+
+typedef union _FEATURE_SUPPORT
+{
+    IMG_UINT8 value;
+    struct
+    {
+	        // Indicates whether Preferred Timing Mode includes native pixel format
+	        // and preferred refresh rate of the display
+        IMG_UINT8 continuousFrequency : 1; // BIT 0
+        IMG_UINT8 preferredTimingIncluded : 1; // BIT 1
+            // BIT 7 at address 14h must be 1 for this layout (signalInterface = digital)
+            // 00 : RBG 4:4:4
+            // 01 : RGB 4:4:4 & YCrCb 4:4:4
+            // 10 : RGB 4:4:4 & YCrCb 4:2:2
+            // 11 : RGB 4:4:4 & YCrCb 4:4:4 & YCrCb 4:2:2
+        IMG_UINT8 sRGBstandard : 1; // BIT 2
+        IMG_UINT8 colorEncodingFormat : 2; // BIT 4 - 3
+        IMG_UINT8 veryLowPowerSupport : 1; // BIT 5
+        IMG_UINT8 suspendSupport : 1; // BIT 6
+        IMG_UINT8 standbySupport : 1; // BIT 7
+    } fields;
+} FEATURE_SUPPORT;
+
+typedef struct _DISPLAY_DESCRIPTOR
+{
+    IMG_UINT8                   DescTag[3];                         // all 0 for display descriptor
+    IMG_UINT8                   DispDescTag;                        // See enum above
+    IMG_UINT8                   Reserved;                           // 00h except for Display Range Limits
+    IMG_UINT8                   Data[13];
+} DISPLAY_DESCRIPTOR;
+
+/* Structures for each type of display descriptor */
+typedef struct _MONITOR_RANGE_DATA
+{
+    IMG_UINT8                   offsetFlags;
+    IMG_UINT8                   vMinRate;
+    IMG_UINT8                   vMaxRate;
+    IMG_UINT8                   hMinRate;
+    IMG_UINT8                   hMaxRate;
+    IMG_UINT8                   maxPixelClock;
+    IMG_UINT8                   supportFlags;
+        // Determines layout of next 7 bytes
+        // 00: Default GTF supported (if BIT 0 in feature support = 1)
+        // 01: Range limits only
+        // 02: Secondary GTF supported
+        // 04: CVT supported
+    union
+    {
+        // Byte 10 = 00h or 01h
+        struct
+        {
+            IMG_UINT8           lineFeed;
+            IMG_UINT8           space[6];
+        };
+        // Byte 10 = 02h or 04h
+        struct
+        {
+            IMG_UINT8           timingData[7];
+        };
+    };
+} MONITOR_RANGE_DATA;
+
+typedef struct _EDID_EXT_BLOCK
+{
+    IMG_UINT8                   Tag;
+    IMG_UINT8                   Revision;
+    IMG_UINT8                   Data[125];
+    IMG_UINT8                   Checksum;
+} EDID_EXT_BLOCK;
+
+typedef struct _EDID_EXT_CEA
+{
+    IMG_UINT8                   Tag;
+    IMG_UINT8                   Revision;
+    IMG_UINT8                   DtdOffset; // Offset within block that has first 18-byte DTD
+    IMG_UINT8                   Flags; // Contains info about YC222, YC444, number of DTDs, etc
+    IMG_UINT8                   Data[123];
+    IMG_UINT8                   Checksum;
+} EDID_EXT_CEA;
+
+
+typedef struct
+{
+    IMG_UINT8                   Header[8];                          //EDID header "00 FF FF FF FF FF FF 00"
+    IMG_UINT16                  ManufactureName;                    //EISA 3-character ID (5 bit compressed ASCII)
+    IMG_UINT16                  ProductCode;                        //Vendor assigned code
+    IMG_UINT32                  SerialNumber;                       //32-bit serial number
+    IMG_UINT8                   WeekOfManufacture;                  //Week number
+    IMG_UINT8                   YearOfManufacture;                  //Year
+    IMG_UINT8                   EdidVersion;                        //EDID Structure Version
+    IMG_UINT8                   EdidRevision;                       //EDID Structure Revision
+    VIDEO_INPUT_DEFINITION      VideoInputDefinition;
+    IMG_UINT8                   MaxHorizontalImageSize;             //cm
+    IMG_UINT8                   MaxVerticalImageSize;               //cm
+    IMG_UINT8                   Gamma;
+    FEATURE_SUPPORT             FeatureSupport;
+    IMG_UINT8                   RedGreenLowBits;                    //Rx1 Rx0 Ry1 Ry0 Gx1 Gx0 Gy1Gy0
+    IMG_UINT8                   BlueWhiteLowBits;                   //Bx1 Bx0 By1 By0 Wx1 Wx0 Wy1 Wy0
+    IMG_UINT8                   RedX;                               //Red-x Bits 9 - 2
+    IMG_UINT8                   RedY;                               //Red-y Bits 9 - 2
+    IMG_UINT8                   GreenX;                             //Green-x Bits 9 - 2
+    IMG_UINT8                   GreenY;                             //Green-y Bits 9 - 2
+    IMG_UINT8                   BlueX;                              //Blue-x Bits 9 - 2
+    IMG_UINT8                   BlueY;                              //Blue-y Bits 9 - 2
+    IMG_UINT8                   WhiteX;                             //White-x Bits 9 - 2
+    IMG_UINT8                   WhiteY;                             //White-x Bits 9 - 2
+    IMG_UINT8                   EstablishedTimings[3];              // Optional (see above)
+    STANDARD_TIMING             StandardTimings[8];                 // Optional
+    /* The next 72 bytes vary per E-EDID Standard (1.4 is below) */
+    DTD_RAW                     PreferredTimingMode;                // Check FEATURE_SUPPORT BIT 1 before using this
+    IMG_UINT8                   Descriptor2[18];                    // Or display descriptor
+    IMG_UINT8                   Descriptor3[18];                    // Or display descriptor
+    IMG_UINT8                   Descriptor4[18];                    // Or display descriptor
+    IMG_UINT8                   ExtensionBlockCount;                //Number of (optional) 128-byte EDID extension blocks to follow
+    IMG_UINT8                   Checksum;
+} EDID_BASE_BLOCK;
+
+/* Function prototypes */
+PVRSRV_ERROR EdidAddDtd(VIDEO_PARAMS * pVideoParams, IMG_UINT8 code, IMG_UINT32 refreshRate);
+PVRSRV_ERROR EdidInitialize(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR EdidRead(HDMI_DEVICE * pvDevice);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdcp.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdcp.h
new file mode 100644
index 000000000000..5770f541ef49
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdcp.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+
+
+PVRSRV_ERROR HdcpConfigure(HDMI_DEVICE * pvDevice);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi.h
new file mode 100644
index 000000000000..cac41483bef2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi.h
@@ -0,0 +1,250 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#ifndef HDMI_H
+#define HDMI_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "hdmi_regs.h"
+#include "pvrsrv.h"
+#include "powervr/imgpixfmts.h"
+#include "dc_osfuncs.h"
+#include "plato.h"
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+
+
+#define DRVNAME	"plato_hdmi"
+#define HDMI_MAX_COMMANDS_INFLIGHT    (2)
+
+typedef PVRSRV_ERROR (*PFN_PDP_INIT)(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance);
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+// See sysinfo.h
+
+/*************************************************************************/ /*!
+ HDMI data structures
+*/ /**************************************************************************/
+
+typedef struct HDMI_MODULE_PARAMETERS_TAG
+{
+	IMG_UINT32	ui32HDMIEnabled;
+} HDMI_MODULE_PARAMETERS;
+
+const HDMI_MODULE_PARAMETERS *HDMIGetModuleParameters(void);
+
+typedef struct HDMI_DEVICE
+{
+	IMG_HANDLE                  hPVRServicesConnection;
+	IMG_HANDLE                  hPVRServicesDevice;
+	IMG_HANDLE                  hLISRData;
+	DC_SERVICES_FUNCS           sPVRServicesFuncs;
+	DC_SPINLOCK                 irqLock;
+	struct delayed_work         delayedWork;
+	void                        *pvDevice;
+
+	IMG_CPU_PHYADDR             sHDMIRegCpuPAddr;
+	IMG_CPU_VIRTADDR            pvHDMIRegCpuVAddr;
+
+	IMG_CPU_PHYADDR             sTopRegCpuPAddr;
+	IMG_CPU_VIRTADDR            pvTopRegCpuVAddr;
+
+	VIDEO_PARAMS                videoParams;
+
+	IMG_BOOL                    mInitialized;
+	IMG_BOOL                    bHPD;
+
+	/* Callback for PDP */
+	PFN_PDP_INIT                pfnPDPInitialize;
+
+	/* Plato core and PLL clock speeds */
+	IMG_UINT32                  ui32CoreClockSpeed;
+	IMG_UINT32                  ui32PLLClockSpeed;
+
+	/* Doubly linked list of device units */
+	DLLIST_NODE                 sListNode;
+	struct dentry               *psDebugFSEntryDir;     /* 'plato_hdmi' entry */
+	struct dentry               *psDisplayEnabledEntry; /* 'display_enabled' entry */
+	IMG_UINT32                  ui32Instance;           /* device instance */
+	IMG_BOOL                    bHDMIEnabled;           /* state of 'display_enabled' */
+} HDMI_DEVICE;
+
+
+/*******************************************************************************
+ * HDMI common functions
+ ******************************************************************************/
+
+PVRSRV_ERROR HDMIDrvInit(void *pvDevice, HDMI_DEVICE **ppsDeviceData);
+void HDMIDrvDeInit(HDMI_DEVICE *psDeviceData);
+
+/*******************************************************************************
+ * Debug/print macros
+ ******************************************************************************/
+#if defined(PLATO_DISPLAY_PDUMP)
+#include "plato_pdump.h"
+
+static void HDMIPdumpReg32(void *pvLinRegBaseAddr, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	OSWriteHWReg32(pvLinRegBaseAddr,  ui32Offset,  ui32Value);
+	plato_pdump_reg32(pvLinRegBaseAddr, ui32Offset, ui32Value, DRVNAME);
+}
+
+#undef DC_OSWriteReg32
+
+#define DC_OSWriteReg32 HDMIPdumpReg32
+
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	plato_pdump_pol(base,reg,val,msk, DRVNAME); \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			HDMI_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#else
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			HDMI_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#endif // PLATO_DISPLAY_PDUMP
+
+
+#define HDMI_REG_POLL(base,reg,val,msk) polpr(base,reg*4,val,msk,10,10)
+#define TOP_REG_POLL(base,reg,val,msk) polpr(base,reg,val,msk,10,10)
+
+
+#define CHECK_AND_EXIT(status, label) if (status != PVRSRV_OK) goto label;
+#define IS_BIT_SET(value, bit) (value & (1 << bit))
+
+#if !defined(VIRTUAL_PLATFORM)
+#if defined(HDMI_DEBUG)
+	#define HDMI_CHECKPOINT HDMI_DEBUG_PRINT(" CP: - %s, line %d\n", __func__, __LINE__);
+	#define HDMI_DEBUG_PRINT(fmt, ...) \
+		DC_OSDebugPrintf(DBGLVL_INFO, fmt, __VA_ARGS__)
+
+	#define HDMI_WRITE_CORE_REG(base, offset, value) \
+		DC_OSWriteReg32(base, offset * 4, value);
+
+#else
+	#define HDMI_CHECKPOINT
+	#define HDMI_DEBUG_PRINT(fmt, ...)
+	#define HDMI_WRITE_CORE_REG(base, offset, value) DC_OSWriteReg32(base, offset * 4, value);
+#endif
+
+#define HDMI_READ_CORE_REG(base, offset) DC_OSReadReg32(base, offset * 4)
+#define HDMI_READ_MOD_WRITE(base, offset, value) HDMI_WRITE_CORE_REG(base, offset, (HDMI_READ_CORE_REG(base,offset) | value))
+#else /* VIRTUAL_PLATFORM */
+
+#define HDMI_CHECKPOINT HDMI_DEBUG_PRINT(" CP: - %s, line %d\n", __func__, __LINE__);
+#define HDMI_DEBUG_PRINT(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_INFO, fmt, __VA_ARGS__)
+#define HDMI_WRITE_CORE_REG(base, offset, value)
+#define HDMI_READ_CORE_REG(base, offset) (0)
+#define HDMI_READ_MOD_WRITE(base, offset, value)
+
+#endif
+
+#define HDMI_ERROR_PRINT(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_ERROR, fmt, __VA_ARGS__)
+
+#define HDMI_LOG_ALWAYS(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_INFO, fmt, __VA_ARGS__)
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi_regs.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi_regs.h
new file mode 100644
index 000000000000..1e9ff9c0e9e7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi_regs.h
@@ -0,0 +1,587 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HDMI_REGS_H
+#define HDMI_REGS_H
+
+#define SET_FIELD(start, mask, value) \
+    ((value << start) & mask)
+
+#define GET_FIELD(value, start, mask) \
+    ((value & mask) >> start)
+/* Hardware Register definitions
+
+NOTE: These are HDMI core address offsets, SoC level offset will be * 4
+
+*/
+
+/* Identification registers */
+#define HDMI_ID_DESIGN_ID_OFFSET                            (0x0000)
+
+#define HDMI_ID_REVISION_ID_OFFSET                          (0x0001)
+
+#define HDMI_ID_PRODUCT_ID0_OFFSET                          (0x0002)
+
+#define HDMI_ID_PRODUCT_ID1_OFFSET                          (0x0003)
+#define HDMI_ID_PRODUCT_ID1_TX_START                        (0)
+#define HDMI_ID_PRODUCT_ID1_TX_MASK                         (0x1)
+#define HDMI_ID_PRODUCT_ID1_RX_START                        (1)
+#define HDMI_ID_PRODUCT_ID1_RX_MASK                         (0x2)
+#define HDMI_ID_PRODUCT_ID1_HDCP_START                      (6)
+#define HDMI_ID_PRODUCT_ID1_HDCP_MASK                       (0xC0)
+
+#define HDMI_ID_CONFIG0_ID_OFFSET                           (0x0004)
+#define HDMI_ID_CONFIG0_ID_HDCP_START                       (0)
+#define HDMI_ID_CONFIG0_ID_HDCP_MASK                        (0x1)
+#define HDMI_ID_CONFIG0_ID_CEC_START                        (1)
+#define HDMI_ID_CONFIG0_ID_CEC_MASK                         (0x2)
+#define HDMI_ID_CONFIG0_ID_CSC_START                        (2)
+#define HDMI_ID_CONFIG0_ID_CSC_MASK                         (0x4)
+#define HDMI_ID_CONFIG0_ID_HDMI14_START                     (3)
+#define HDMI_ID_CONFIG0_ID_HDMI14_MASK                      (0x8)
+#define HDMI_ID_CONFIG0_ID_AUDI2S_START                     (4)
+#define HDMI_ID_CONFIG0_ID_AUDI2S_MASK                      (0x10)
+#define HDMI_ID_CONFIG0_ID_AUDSPDIF_START                   (5)
+#define HDMI_ID_CONFIG0_ID_AUDSPDIF_MASK                    (0x20)
+#define HDMI_ID_CONFIG0_ID_PREPEN_START                     (7)
+#define HDMI_ID_CONFIG0_ID_PREPEN_MASK                      (0x80)
+
+#define HDMI_ID_CONFIG1_ID_OFFSET                           (0x0005)
+#define HDMI_ID_CONFIG1_ID_CONFAPB_START                    (1)
+#define HDMI_ID_CONFIG1_ID_CONFAPB_MASK                     (0x2)
+#define HDMI_ID_CONFIG1_ID_HDMI20_START                     (5)
+#define HDMI_ID_CONFIG1_ID_HDMI20_MASK                      (0x20)
+#define HDMI_ID_CONFIG1_ID_HDCP22_START                     (6)
+#define HDMI_ID_CONFIG1_ID_HDCP22_MASK                      (0x40)
+
+#define HDMI_ID_CONFIG2_ID_OFFSET                           (0x0006)
+
+#define HDMI_ID_CONFIG3_ID_OFFSET                           (0x0007)
+
+/* Interrupt Registers */
+#define HDMI_IH_FC_STAT0_OFFSET                             (0x0100)
+
+#define HDMI_IH_FC_STAT1_OFFSET                             (0x0101)
+
+#define HDMI_IH_FC_STAT2_OFFSET                             (0x0102)
+
+#define HDMI_IH_AS_STAT0_OFFSET                             (0x0103)
+
+#define HDMI_IH_PHY_STAT0_OFFSET                            (0x0104)
+#define HDMI_IH_PHY_STAT0_HPD_START                         (0)
+#define HDMI_IH_PHY_STAT0_HPD_MASK                          (0x1)
+
+#define HDMI_IH_I2CM_STAT0_OFFSET                           (0x0105)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERERROR_START             (0)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERERROR_MASK              (0x1)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERDONE_START              (1)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERDONE_MASK               (0x2)
+#define HDMI_IH_I2CM_STAT0_SCDC_READREQ_START               (2)
+#define HDMI_IH_I2CM_STAT0_SCDC_READREQ_MASK                (0x4)
+
+#define HDMI_IH_CEC_STAT0_OFFSET                            (0x0106)
+
+#define HDMI_IH_VP_STAT0_OFFSET                             (0x0107)
+
+#define HDMI_IH_I2CMPHY_STAT0_OFFSET                        (0x0108)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYERROR_START            (0)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYERROR_MASK             (0x1)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_START             (1)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_MASK              (0x2)
+
+#define HDMI_IH_AHBDMAAUD_STAT0_OFFSET                      (0x0109)
+
+#define HDMI_IH_DECODE_OFFSET                               (0x0170)
+#define HDMI_IH_DECODE_PHY_START                            (3)
+#define HDMI_IH_DECODE_PHY_MASK                             (0x8)
+
+#define HDMI_IH_MUTE_FC_STAT0_OFFSET                        (0x0180)
+
+#define HDMI_IH_MUTE_FC_STAT1_OFFSET                        (0x0181)
+
+#define HDMI_IH_MUTE_FC_STAT2_OFFSET                        (0x0182)
+
+#define HDMI_IH_MUTE_AS_STAT0_OFFSET                        (0x0183)
+
+#define HDMI_IH_MUTE_PHY_STAT0_OFFSET                       (0x0184)
+
+#define HDMI_IH_MUTE_I2CM_STAT0_OFFSET                      (0x0185)
+
+#define HDMI_IH_MUTE_CEC_STAT0_OFFSET                       (0x0186)
+
+#define HDMI_IH_MUTE_VP_STAT0_OFFSET                        (0x0187)
+
+#define HDMI_IH_MUTE_I2CMPHY_STAT0_OFFSET                   (0x0188)
+
+#define HDMI_IH_MUTE_AHBDMAAUD_STAT0_OFFSET                 (0x0189)
+
+#define HDMI_IH_MUTE_OFFSET                                 (0x01ff)
+#define HDMI_IH_MUTE_ALL_START                              (0)
+#define HDMI_IH_MUTE_ALL_MASK                               (0x3)
+
+/* Video Sampler registers */
+#define HDMI_TX_INVID0_OFFSET                               (0x0200)
+#define HDMI_TX_INVID0_VIDEO_MAPPING_START                  (0)
+#define HDMI_TX_INVID0_VIDEO_MAPPING_MASK                   (0x1F)
+#define HDMI_TX_INVID0_INTERNAL_DE_GEN_START                (7)
+#define HDMI_TX_INVID0_INTERNAL_DE_GEN_MASK                 (0x80)
+
+#define HDMI_TX_INSTUFFING_OFFSET                           (0x201)
+
+#define HDMI_TX_GYDATA0_OFFSET                              (0x202)
+
+#define HDMI_TX_GYDATA1_OFFSET                              (0x203)
+
+#define HDMI_TX_RCRDATA0_OFFSET                             (0x204)
+
+#define HDMI_TX_RCRDATA1_OFFSET                             (0x205)
+
+#define HDMI_TX_BCBDATA0_OFFSET                             (0x206)
+
+#define HDMI_TX_BCBDATA1_OFFSET                             (0x207)
+
+/* Video Packetizer */
+#define HDMI_VP_STATUS_OFFSET                               (0x0800)
+
+#define HDMI_VP_PR_CD_OFFSET                                (0x0801)
+#define HDMI_VP_PR_CD_DESIRED_PR_FACTOR_START               (0)
+#define HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK                (0xF)
+#define HDMI_VP_PR_CD_COLOR_DEPTH_START                     (4)
+#define HDMI_VP_PR_CD_COLOR_DEPTH_MASK                      (0xF0)
+
+#define HDMI_VP_STUFF_OFFSET                                (0x0802)
+#define HDMI_VP_STUFF_PR_STUFFING_START                     (0)
+#define HDMI_VP_STUFF_PR_STUFFING_MASK                      (0x1)
+#define HDMI_VP_STUFF_PP_STUFFING_START                     (1)
+#define HDMI_VP_STUFF_PP_STUFFING_MASK                      (0x2)
+#define HDMI_VP_STUFF_YCC422_STUFFING_START                 (2)
+#define HDMI_VP_STUFF_YCC422_STUFFING_MASK                  (0x4)
+#define HDMI_VP_STUFF_ICX_GOTO_P0_ST_START                  (3)
+#define HDMI_VP_STUFF_ICX_GOTO_P0_ST_MASK                   (0x8)
+#define HDMI_VP_STUFF_IFIX_PP_TO_LAST_START                 (4)
+#define HDMI_VP_STUFF_IFIX_PP_TO_LAST_MASK                  (0x10)
+#define HDMI_VP_STUFF_IDEFAULT_PHASE_START                  (5)
+#define HDMI_VP_STUFF_IDEFAULT_PHASE_MASK                   (0x20)
+
+#define HDMI_VP_REMAP_OFFSET                                (0x0803)
+#define HDMI_VP_REMAP_YCC422_SIZE_START                     (0)
+#define HDMI_VP_REMAP_YCC422_SIZE_MASK                      (0x3)
+
+#define HDMI_VP_CONF_OFFSET                                 (0x0804)
+#define HDMI_VP_CONF_OUTPUT_SELECTOR_START                  (0)
+#define HDMI_VP_CONF_OUTPUT_SELECTOR_MASK                   (0x3)
+#define HDMI_VP_CONF_BYPASS_SELECT_START                    (2)
+#define HDMI_VP_CONF_BYPASS_SELECT_MASK                     (0x4)
+#define HDMI_VP_CONF_YCC422_EN_START                        (3)
+#define HDMI_VP_CONF_YCC422_EN_MASK                         (0x8)
+#define HDMI_VP_CONF_PR_EN_START                            (4)
+#define HDMI_VP_CONF_PR_EN_MASK                             (0x10)
+#define HDMI_VP_CONF_PP_EN_START                            (5)
+#define HDMI_VP_CONF_PP_EN_MASK                             (0x20)
+#define HDMI_VP_CONF_BYPASS_EN_START                        (6)
+#define HDMI_VP_CONF_BYPASS_EN_MASK                         (0x40)
+
+#define HDMI_VP_MASK_OFFSET                                 (0x0807)
+
+/* Frame Composer */
+#define HDMI_FC_INVIDCONF_OFFSET                            (0x1000)
+#define HDMI_FC_INVIDCONF_IN_I_P_START                      (0)
+#define HDMI_FC_INVIDCONF_IN_I_P_MASK                       (0x1)
+#define HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_START            (1)
+#define HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_MASK             (0x2)
+#define HDMI_FC_INVIDCONF_DVI_MODEZ_START                   (3)
+#define HDMI_FC_INVIDCONF_DVI_MODEZ_MASK                    (0x8)
+#define HDMI_FC_INVIDCONF_DE_IN_POLARITY_START              (4)
+#define HDMI_FC_INVIDCONF_DE_IN_POLARITY_MASK               (0x10)
+#define HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_START           (5)
+#define HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_MASK            (0x20)
+#define HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_START           (6)
+#define HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_MASK            (0x40)
+#define HDMI_FC_INVIDCONF_HDCP_KEEPOUT_START                (7)
+#define HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK                 (0x80)
+
+#define HDMI_FC_INHACTIV0_OFFSET                            (0x1001)
+#define HDMI_FC_INHACTIV0_H_IN_ACTIV_START                  (0)
+#define HDMI_FC_INHACTIV0_H_IN_ACTIV_MASK                   (0xFF)
+
+#define HDMI_FC_INHACTIV1_OFFSET                            (0x1002)
+#define HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_START            (0)
+#define HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_MASK             (0x3F)
+
+#define HDMI_FC_INHBLANK0_OFFSET                            (0x1003)
+#define HDMI_FC_INHBLANK0_H_IN_BLANK_START                  (0)
+#define HDMI_FC_INHBLANK0_H_IN_BLANK_MASK                   (0xFF)
+
+#define HDMI_FC_INHBLANK1_OFFSET                            (0x1004)
+#define HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_START            (0)
+#define HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_MASK             (0x3F)
+
+#define HDMI_FC_INVACTIV0_OFFSET                            (0x1005)
+#define HDMI_FC_INVACTIV0_V_IN_ACTIV_START                  (0)
+#define HDMI_FC_INVACTIV0_V_IN_ACTIV_MASK                   (0xFF)
+
+#define HDMI_FC_INVACTIV1_OFFSET                            (0x1006)
+#define HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_START            (0)
+#define HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_MASK             (0x3F)
+
+#define HDMI_FC_INVBLANK_OFFSET                             (0x1007)
+#define HDMI_FC_INVBLANK_V_IN_BLANK_START                   (0)
+#define HDMI_FC_INVBLANK_V_IN_BLANK_MASK                    (0xFF)
+
+#define HDMI_FC_HSYNCINDELAY0_OFFSET                        (0x1008)
+#define HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_START              (0)
+#define HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_MASK               (0xFF)
+
+#define HDMI_FC_HSYNCINDELAY1_OFFSET                        (0x1009)
+#define HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_START        (0)
+#define HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_MASK         (0x3F)
+
+#define HDMI_FC_HSYNCINWIDTH0_OFFSET                        (0x100a)
+#define HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_START              (0)
+#define HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_MASK               (0xFF)
+
+#define HDMI_FC_HSYNCINWIDTH1_OFFSET                        (0x100b)
+#define HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_START        (0)
+#define HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_MASK         (0x3)
+
+#define HDMI_FC_VSYNCINDELAY_OFFSET                         (0x100c)
+#define HDMI_FC_VSYNCINDELAY_V_IN_DELAY_START               (0)
+#define HDMI_FC_VSYNCINDELAY_V_IN_DELAY_MASK                (0xFF)
+
+#define HDMI_FC_VSYNCINWIDTH_OFFSET                         (0x100d)
+#define HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_START               (0)
+#define HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_MASK                (0x3F)
+
+#define HDMI_FC_INFREQ0_OFFSET                              (0x100e)
+
+#define HDMI_FC_INFREQ1_OFFSET                              (0x100f)
+
+#define HDMI_FC_INFREQ2_OFFSET                              (0x1010)
+
+#define HDMI_FC_CTRLDUR_OFFSET                              (0x1011)
+
+#define HDMI_FC_EXCTRLDUR_OFFSET                            (0x1012)
+
+#define HDMI_FC_EXCTRLSPAC_OFFSET                           (0x1013)
+
+#define HDMI_FC_CH0PREAM_OFFSET                             (0x1014)
+
+#define HDMI_FC_CH1PREAM_OFFSET                             (0x1015)
+
+#define HDMI_FC_CH2PREAM_OFFSET                             (0x1016)
+
+#define HDMI_FC_AVICONF3_OFFSET                             (0x1017)
+
+#define HDMI_FC_GCP_OFFSET                                  (0x1018)
+
+#define HDMI_FC_AVICONF0_OFFSET                             (0x1019)
+#define HDMI_FC_AVICONF0_RGBYCC_START                       (0)
+#define HDMI_FC_AVICONF0_RGBYCC_MASK                        (0x3)
+#define HDMI_FC_AVICONF0_VBAR_VALID_START                   (2)
+#define HDMI_FC_AVICONF0_VBAR_VALID_MASK                    (0x4)
+#define HDMI_FC_AVICONF0_HBAR_VALID_START                   (3)
+#define HDMI_FC_AVICONF0_HBAR_VALID_MASK                    (0x8)
+#define HDMI_FC_AVICONF0_SCAN_INFO_START                    (4)
+#define HDMI_FC_AVICONF0_SCAN_INFO_MASK                     (0x30)
+#define HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_START        (6)
+#define HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_MASK         (0x40)
+#define HDMI_FC_AVICONF0_RGBYCC_2_START                     (7)
+#define HDMI_FC_AVICONF0_RGBYCC_2_MASK                      (0x80)
+
+#define HDMI_FC_AVICONF1_OFFSET                             (0x101a)
+#define HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_START          (0)
+#define HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_MASK           (0xF)
+#define HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_START             (4)
+#define HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_MASK              (0x30)
+#define HDMI_FC_AVICONF1_COLORIMETRY_START                  (6)
+#define HDMI_FC_AVICONF1_COLORIMETRY_MASK                   (0xC0)
+
+#define HDMI_FC_AVICONF2_OFFSET                             (0x101b)
+#define HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_START      (0)
+#define HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_MASK       (0x3)
+#define HDMI_FC_AVICONF2_QUANTIZATION_RANGE_START           (2)
+#define HDMI_FC_AVICONF2_QUANTIZATION_RANGE_MASK            (0xC)
+#define HDMI_FC_AVICONF2_EXT_COLORIMETRY_START              (4)
+#define HDMI_FC_AVICONF2_EXT_COLORIMETRY_MASK               (0x70)
+#define HDMI_FC_AVICONF2_IT_CONTENT_START                   (7)
+#define HDMI_FC_AVICONF2_IT_CONTENT_MASK                    (0x80)
+
+#define HDMI_FC_AVIVID_OFFSET                               (0x101c)
+#define HDMI_FC_AVIVID_START                                (0)
+#define HDMI_FC_AVIVID_MASK                                 (0xFF)
+
+#define HDMI_FC_AVIETB0_OFFSET                              (0x101d)
+
+#define HDMI_FC_AVIETB1_OFFSET                              (0x101e)
+
+#define HDMI_FC_AVISBB0_OFFSET                              (0x101f)
+
+#define HDMI_FC_AVISBB1_OFFSET                              (0x1020)
+
+#define HDMI_FC_AVIELB0_OFFSET                              (0x1021)
+
+#define HDMI_FC_AVIELB1_OFFSET                              (0x1022)
+
+#define HDMI_FC_AVISRB0_OFFSET                              (0x1023)
+
+#define HDMI_FC_AVISRB1_OFFSET                              (0x1024)
+
+/* A lot of registers related to audio, ACP, ISRC, etc */
+
+#define HDMI_FC_PRCONF_OFFSET                               (0x10e0)
+#define HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_START               (0)
+#define HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK                (0xF)
+#define HDMI_FC_PRCONF_INCOMING_PR_FACTOR_START             (4)
+#define HDMI_FC_PRCONF_INCOMING_PR_FACTOR_MASK              (0xF0)
+
+#define HDMI_FC_SCRAMBLER_CTRL_OFFSET                       (0x10e1)
+
+#define HDMI_FC_MULTISTREAM_CTRL_OFFSET                     (0x10e2)
+
+#define HDMI_FC_PACKET_TX_EN_OFFSET                         (0x10e3)
+
+#define HDMI_FC_DBGFORCE_OFFSET                             (0x1200)
+#define HDMI_FC_DBGFORCE_FORCE_VIDEO_START                  (0)
+#define HDMI_FC_DBGFORCE_FORCE_VIDEO_MASK                   (0x1)
+#define HDMI_FC_DBGFORCE_FORCE_AUDIO_START                  (4)
+#define HDMI_FC_DBGFORCE_FORCE_AUDIO_MASK                   (0x10)
+
+#define HDMI_FC_DBGTMDS_0_OFFSET                            (0x1219)
+
+#define HDMI_FC_DBGTMDS_1_OFFSET                            (0x121A)
+
+#define HDMI_FC_DBGTMDS_2_OFFSET                            (0x121B)
+
+
+/* HDMI Source PHY */
+#define HDMI_PHY_CONF0_OFFSET                               (0x3000)
+#define HDMI_PHY_CONF0_SELDIPIF_START                       (0)
+#define HDMI_PHY_CONF0_SELDIPIF_MASK                        (0x1)
+#define HDMI_PHY_CONF0_SELDATAENPOL_START                   (1)
+#define HDMI_PHY_CONF0_SELDATAENPOL_MASK                    (0x2)
+#define HDMI_PHY_CONF0_ENHPDRXSENSE_START                   (2)
+#define HDMI_PHY_CONF0_ENHPDRXSENSE_MASK                    (0x4)
+#define HDMI_PHY_CONF0_TXPWRON_START                        (3)
+#define HDMI_PHY_CONF0_TXPWRON_MASK                         (0x8)
+#define HDMI_PHY_CONF0_PDDQ_START                           (4)
+#define HDMI_PHY_CONF0_PDDQ_MASK                            (0x10)
+#define HDMI_PHY_CONF0_SVSRET_START                         (5)
+#define HDMI_PHY_CONF0_SVSRET_MASK                          (0x20)
+#define HDMI_PHY_CONF0_ENTMDS_START                         (6)
+#define HDMI_PHY_CONF0_ENTMDS_MASK                          (0x40)
+#define HDMI_PHY_CONF0_PDZ_START                            (7)
+#define HDMI_PHY_CONF0_PDZ_MASK                             (0x80)
+
+#define HDMI_PHY_TST0_OFFSET                                (0x3001)
+
+#define HDMI_PHY_TST1_OFFSET                                (0x3002)
+
+#define HDMI_PHY_TST2_OFFSET                                (0x3003)
+
+#define HDMI_PHY_STAT0_OFFSET                               (0x3004)
+#define HDMI_PHY_STAT0_TX_PHY_LOCK_START                    (0)
+#define HDMI_PHY_STAT0_TX_PHY_LOCK_MASK                     (0x1)
+#define HDMI_PHY_STAT0_HPD_START                            (1)
+#define HDMI_PHY_STAT0_HPD_MASK                             (0x2)
+
+#define HDMI_PHY_INT0_OFFSET                                (0x3005)
+
+#define HDMI_PHY_MASK0_OFFSET                               (0x3006)
+
+#define HDMI_PHY_POL0_OFFSET                                (0x3007)
+#define HDMI_PHY_POL0_HPD_START                             (1)
+#define HDMI_PHY_POL0_HPD_MASK                              (0x2)
+
+/* I2C Master PHY */
+#define HDMI_PHY_I2CM_SLAVE_OFFSET                          (0x3020)
+#define HDMI_PHY_I2CM_SLAVE_SLAVEADDR_START                 (0)
+#define HDMI_PHY_I2CM_SLAVE_SLAVEADDR_MASK                  (0x7F)
+
+#define HDMI_PHY_I2CM_ADDRESS_OFFSET                        (0x3021)
+#define HDMI_PHY_I2CM_ADDRESS_START                         (0)
+#define HDMI_PHY_I2CM_ADDRESS_MASK                          (0xFF)
+
+#define HDMI_PHY_I2CM_DATAO_1_OFFSET                        (0x3022)
+#define HDMI_PHY_I2CM_DATAO_1_START                         (0)
+#define HDMI_PHY_I2CM_DATAO_1_MASK                          (0xFF)
+
+#define HDMI_PHY_I2CM_DATAO_0_OFFSET                        (0x3023)
+#define HDMI_PHY_I2CM_DATAO_0_START                         (0)
+#define HDMI_PHY_I2CM_DATAO_0_MASK                          (0xFF)
+
+#define HDMI_PHY_I2CM_DATAI_1_OFFSET                        (0x3024)
+#define HDMI_PHY_I2CM_DATAI_1_DATAI_START                   (0)
+#define HDMI_PHY_I2CM_DATAI_1_DATAI_MASK                    (0xFF)
+
+#define HDMI_PHY_I2CM_DATAI_0_OFFSET                        (0x3025)
+#define HDMI_PHY_I2CM_DATAI_0_DATAI_START                   (0)
+#define HDMI_PHY_I2CM_DATAI_0_DATAI_MASK                    (0xFF)
+
+#define HDMI_PHY_I2CM_OPERATION_OFFSET                      (0x3026)
+#define HDMI_PHY_I2CM_OPERATION_RD_START                    (0)
+#define HDMI_PHY_I2CM_OPERATION_RD_MASK                     (0x1)
+#define HDMI_PHY_I2CM_OPERATION_WR_START                    (4)
+#define HDMI_PHY_I2CM_OPERATION_WR_MASK                     (0x10)
+// Many more...
+
+/* Audio Sampler */
+#define HDMI_AUDIO_SAMPLER_OFFSET                           (0x3100)
+
+/* Audio packetizer */
+#define HDMI_AUD_N1_OFFSET                                  (0x3200)
+
+#define HDMI_AUD_N2_OFFSET                                  (0x3201)
+
+#define HDMI_AUD_N3_OFFSET                                  (0x3202)
+
+#define HDMI_AUD_CTS1_OFFSET                                (0x3203)
+
+#define HDMI_AUD_CTS2_OFFSET                                (0x3204)
+
+#define HDMI_AUD_CTS3_OFFSET                                (0x3205)
+
+#define HDMI_AUD_INPUTCLKFS_OFFSET                          (0x3206)
+
+/* Generic parallel audio interface */
+#define HDMI_PARALLEL_AUDIO_INTF_OFFFSET                    (0x3500)
+
+/* Audio DMA */
+#define HDMI_AUDIO_DMA_OFFSET                               (0x3600)
+
+/* Main Controller Registers */
+#define HDMI_MC_CLKDIS_OFFSET                               (0x4001)
+#define HDMI_MC_CLKDIS_PIXELCLK_DIS_START                   (0)
+#define HDMI_MC_CLKDIS_PIXELCLK_DIS_MASK                    (0x1)
+#define HDMI_MC_CLKDIS_TMDSCLK_DIS_START                    (1)
+#define HDMI_MC_CLKDIS_TMDSCLK_DIS_MASK                     (0x2)
+#define HDMI_MC_CLKDIS_PREPCLK_DIS_START                    (2)
+#define HDMI_MC_CLKDIS_PREPCLK_DIS_MASK                     (0x4)
+#define HDMI_MC_CLKDIS_AUDCLK_DIS_START                     (3)
+#define HDMI_MC_CLKDIS_AUDCLK_DIS_MASK                      (0x8)
+#define HDMI_MC_CLKDIS_CSCCLK_DIS_START                     (4)
+#define HDMI_MC_CLKDIS_CSCCLK_DIS_MASK                      (0x10)
+#define HDMI_MC_CLKDIS_CECCLK_DIS_START                     (5)
+#define HDMI_MC_CLKDIS_CECCLK_DIS_MASK                      (0x20)
+#define HDMI_MC_CLKDIS_HDCPCLK_DIS_START                    (6)
+#define HDMI_MC_CLKDIS_HDCPCLK_DIS_MASK                     (0x40)
+
+#define HDMI_MC_SWRSTZREQ_OFFSET                            (0x4002)
+#define HDMI_MC_OPCTRL_OFFSET                               (0x4003)
+#define HDMI_MC_FLOWCTRL_OFFSET                             (0x4004)
+
+#define HDMI_MC_PHYRSTZ_OFFSET                              (0x4005)
+#define HDMI_MC_PHYRSTZ_PHYRSTZ_START                       (0)
+#define HDMI_MC_PHYRSTZ_PHYRSTZ_MASK                        (0x1)
+
+#define HDMI_MC_LOCKONCLOCK_OFFSET                          (0x4006)
+#define HDMI_MC_HEACPHY_RST_OFFSET                          (0x4007)
+#define HDMI_MC_LOCKONCLOCK_2_OFFSET                        (0x4008)
+#define HDMI_MC_SWRSTZREQ_2_OFFSET                          (0x4009)
+
+/* Color Space Converter */
+#define HDMI_COLOR_SPACE_CONVERTER_OFFSET                   (0x4100)
+
+/* HDCP Encryption */
+#define HDMI_HDCP_ENCRYPTION_OFFSET                         (0x5000)
+
+/* I2C Master (E-EDID/EDDC) */
+#define HDMI_I2CM_SLAVE_OFFSET                              (0x7e00)
+
+#define HDMI_I2CM_ADDRESS_OFFSET                            (0x7e01)
+
+#define HDMI_I2CM_DATAO_OFFSET                              (0x7e02)
+
+#define HDMI_I2CM_DATAI_OFFSET                              (0x7e03)
+
+#define HDMI_I2CM_OPERATION_OFFSET                          (0x7e04)
+#define HDMI_I2CM_OPERATION_RD_START                        (0)
+#define HDMI_I2CM_OPERATION_RD_MASK                         (0x1)
+#define HDMI_I2CM_OPERATION_RD_EXT_START                    (1)
+#define HDMI_I2CM_OPERATION_RD_EXT_MASK                     (0x2)
+#define HDMI_I2CM_OPERATION_RD_8_START                      (2)
+#define HDMI_I2CM_OPERATION_RD_8_MASK                       (0x4)
+#define HDMI_I2CM_OPERATION_RD_8_EXT_START                  (3)
+#define HDMI_I2CM_OPERATION_RD_8_EXT_MASK                   (0x8)
+#define HDMI_I2CM_OPERATION_WR_START                        (4)
+#define HDMI_I2CM_OPERATION_WR_MASK                         (0x10)
+
+#define HDMI_I2CM_INT_OFFSET                                (0x7e05)
+#define HDMI_I2CM_INT_DONE_MASK_START                       (2)
+#define HDMI_I2CM_INT_DONE_MASK_MASK                        (0x4)
+#define HDMI_I2CM_INT_READ_REQ_MASK_START                   (6)
+#define HDMI_I2CM_INT_READ_REQ_MASK_MASK                    (0x40)
+
+#define HDMI_I2CM_CTLINT_OFFSET                             (0x7e06)
+#define HDMI_I2CM_CTLINT_ARB_MASK_START                     (2)
+#define HDMI_I2CM_CTLINT_ARB_MASK_MASK                      (0x4)
+#define HDMI_I2CM_CTLINT_NACK_MASK_START                    (6)
+#define HDMI_I2CM_CTLINT_NACK_MASK_MASK                     (0x40)
+
+#define HDMI_I2CM_DIV_OFFSET                                (0x7e07)
+#define HDMI_I2CM_DIV_FAST_STD_MODE_START                   (3)
+#define HDMI_I2CM_DIV_FAST_STD_MODE_MASK                    (0x8)
+
+#define HDMI_I2CM_SEGADDR_OFFSET                            (0x7e08)
+
+#define HDMI_I2CM_SOFTRSTZ_OFFSET                           (0x7e09)
+
+#define HDMI_I2CM_SEGPTR_OFFSET                             (0x7e0a)
+
+#define HDMI_I2CM_SS_SCL_HCNT_1_OFFSET                      (0x7e0b)
+
+#define HDMI_I2CM_SS_SCL_HCNT_0_OFFSET                      (0x7e0c)
+
+#define HDMI_I2CM_SS_SCL_LCNT_1_OFFSET                      (0x7e0d)
+
+#define HDMI_I2CM_SS_SCL_LCNT_0_OFFSET                      (0x7e0e)
+
+#define HDMI_I2CM_FS_SCL_HCNT_1_OFFSET                      (0x7e0f)
+
+#define HDMI_I2CM_FS_SCL_HCNT_0_OFFSET                      (0x7e10)
+
+#define HDMI_I2CM_FS_SCL_LCNT_1_OFFSET                      (0x7e11)
+
+#define HDMI_I2CM_FS_SCL_LCNT_0_OFFSET                      (0x7e12)
+
+#define HDMI_I2CM_SDA_HOLD_OFFSET                           (0x7e13)
+
+#define HDMI_I2CM_SCDC_READ_UPDATE_OFFSET                   (0x7e14)
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/phy.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/phy.h
new file mode 100644
index 000000000000..ffd5cf33ced8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/phy.h
@@ -0,0 +1,134 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+#ifndef DC_PHY_H
+#define DC_PHY_H
+
+#include "hdmi.h"
+
+/* PHY Gen2 slave addr */
+#if defined(EMULATOR)
+#define HDMI_PHY_SLAVE_ADDRESS                  0xA0
+#else
+#define HDMI_PHY_SLAVE_ADDRESS                  0x00
+#endif
+
+#if defined(VIRTUAL_PLATFORM)
+#define HDMI_PHY_TIMEOUT_MS                     1
+#define HDMI_PHY_POLL_INTERVAL_MS               1
+/* PHY dependent time for reset */
+#define HDMI_PHY_RESET_TIME_MS                  1
+#else
+/* Timeout for waiting on PHY operations */
+#define HDMI_PHY_TIMEOUT_MS                     200
+#define HDMI_PHY_POLL_INTERVAL_MS               100
+/* PHY dependent time for reset */
+#define HDMI_PHY_RESET_TIME_MS                  10
+#endif
+
+/* PHY I2C register definitions */
+#define PHY_PWRCTRL_OFFSET                      (0x00)
+#define PHY_SERDIVCTRL_OFFSET                   (0x01)
+#define PHY_SERCKCTRL_OFFSET                    (0x02)
+#define PHY_SERCKKILLCTRL_OFFSET                (0x03)
+#define PHY_TXRESCTRL_OFFSET                    (0x04)
+#define PHY_CKCALCTRL_OFFSET                    (0x05)
+#define PHY_OPMODE_PLLCFG_OFFSET                (0x06)
+#define PHY_CLKMEASCTRL_OFFSET                  (0x07)
+#define PHY_TXMEASCTRL_OFFSET                   (0x08)
+#define PHY_CKSYMTXCTRL_OFFSET                  (0x09)
+#define PHY_CMPSEQCTRL_OFFSET                   (0x0A)
+#define PHY_CMPPWRCTRL_OFFSET                   (0x0B)
+#define PHY_CMPMODECTRL_OFFSET                  (0x0C)
+#define PHY_MEASCTRL_OFFSET                     (0x0D)
+#define PHY_VLEVCTRL_OFFSET                     (0x0E)
+#define PHY_D2ACTRL_OFFSET                      (0x0F)
+#define PHY_PLLCURRCTRL_OFFSET                  (0x10)
+#define PHY_PLLDRVANACTRL_OFFSET                (0x11)
+#define PHY_PLLCTRL_OFFSET                      (0x14)
+#define PHY_PLLGMPCTRL_OFFSET                   (0x15)
+#define PHY_PLLMEASCTRL_OFFSET                  (0x16)
+#define PHY_PLLCLKBISTPHASE_OFFSET              (0x17)
+#define PHY_COMPRCAL_OFFSET                     (0x18)
+#define PHY_TXTERM_OFFSET                       (0x19)
+#define PHY_PWRSEQ_PATGENSKIP_OFFSET            (0x1A)
+/* Leaving out BIST regs */
+
+
+bool PhyIsPclkSupported(IMG_UINT16 pixelClock);
+PVRSRV_ERROR PhyI2CWrite(IMG_CPU_VIRTADDR base, IMG_UINT8 addr, IMG_UINT16 data);
+PVRSRV_ERROR PhyInitialize(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR PhyPowerDown(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR PhyWaitLock(HDMI_DEVICE * pvDevice);
+void PhyReset(HDMI_DEVICE * pvDevice, IMG_UINT8 value);
+void PhyConfigureMode(HDMI_DEVICE * pvDevice);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/video.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/video.h
new file mode 100644
index 000000000000..7670a92ce3bd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/video.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+#ifndef DC_VIDEO_H
+#define DC_VIDEO_H
+
+#include "hdmi.h"
+
+PVRSRV_ERROR VideoInitialize(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR VideoConfigureMode(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR VideoConfigureInfoframes(HDMI_DEVICE * pvDevice);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/dma_flags.h b/drivers/gpu/drm/img/img-volcanic/services/include/dma_flags.h
new file mode 100644
index 000000000000..fa6e77fa28c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/dma_flags.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef DMA_FLAGS_H
+#define DMA_FLAGS_H
+
+/* These should match flags in pvrsrv_dma.h */
+#define DMA_FLAG_MEM_TO_DEV  (1U<<0)
+#define DMA_FLAG_DEV_TO_MEM  (0U<<0)
+
+#define DMA_FLAG_SYNCHRONOUS (1U<<1)
+
+#endif /* DMA_FLAGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_sf.h b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_sf.h
new file mode 100644
index 000000000000..83699a2d9547
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_sf.h
@@ -0,0 +1,272 @@
+/*************************************************************************/ /*!
+@File           htbuffer_sf.h
+@Title          Host Trace Buffer interface string format specifiers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the Host Trace Buffer logging messages. The following
+                list are the messages the host driver prints. Changing anything
+                but the first column or spelling mistakes in the strings will
+                break compatibility with log files created with older/newer
+                driver versions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_SF_H
+#define HTBUFFER_SF_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/******************************************************************************
+ * *DO*NOT* rearrange or delete lines in SFIDLIST or SFGROUPLIST or you
+ *          WILL BREAK host tracing message compatibility with previous
+ *          driver versions. Only add new ones, if so required.
+ *****************************************************************************/
+
+
+/* String used in pvrdebug -h output */
+#define HTB_LOG_GROUPS_STRING_LIST   "ctrl,mmu,sync,main,brg"
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define HTB_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s"
+
+/* Available log groups - Master template
+ *
+ * Group usage is as follows:
+ *    CTRL  - Internal Host Trace information and synchronisation data
+ *    MMU   - MMU page mapping information
+ *    SYNC  - Synchronisation debug
+ *    MAIN  - Data master kicks, etc. tying in with the MAIN group in FWTrace
+ *    DBG   - Temporary debugging group, logs not to be left in the driver
+ *
+ */
+#define HTB_LOG_SFGROUPLIST                               \
+	X( HTB_GROUP_NONE,     NONE  )                        \
+/*     gid,                group flag / apphint name */   \
+	X( HTB_GROUP_CTRL,     CTRL  )                        \
+	X( HTB_GROUP_MMU,      MMU   )                        \
+	X( HTB_GROUP_SYNC,     SYNC  )                        \
+	X( HTB_GROUP_MAIN,     MAIN  )                        \
+	X( HTB_GROUP_BRG,      BRG  )                         \
+/* Debug group HTB_GROUP_DBG must always be last */       \
+	X( HTB_GROUP_DBG,      DBG   )
+
+
+/* Table of String Format specifiers, the group they belong and the number of
+ * arguments each expects. Xmacro styled macros are used to generate what is
+ * needed without requiring hand editing.
+ *
+ * id		: unique id within a group
+ * gid		: group id as defined above
+ * sym name	: symbolic name of enumerations used to identify message strings
+ * string	: Actual string
+ * #args	: number of arguments the string format requires
+ */
+#define HTB_LOG_SFIDLIST \
+/*id,  gid,             sym name,                       string,                           # arguments */ \
+X( 0,  HTB_GROUP_NONE,  HTB_SF_FIRST,                   "You should not use this string", 0, 0) \
+\
+X( 1,  HTB_GROUP_CTRL,  HTB_SF_CTRL_LOGMODE,            "HTB log mode set to %d (1- all PID, 2 - restricted PID)\n", 1, 0) \
+X( 2,  HTB_GROUP_CTRL,  HTB_SF_CTRL_ENABLE_PID,         "HTB enable logging for PID %d\n", 1, 0) \
+X( 3,  HTB_GROUP_CTRL,  HTB_SF_CTRL_ENABLE_GROUP,       "HTB enable logging groups 0x%08x\n", 1, 0) \
+X( 4,  HTB_GROUP_CTRL,  HTB_SF_CTRL_LOG_LEVEL,          "HTB log level set to %d\n", 1, 0) \
+X( 5,  HTB_GROUP_CTRL,  HTB_SF_CTRL_OPMODE,             "HTB operating mode set to %d (1 - droplatest, 2 - drop oldest, 3 - block)\n", 1, 0) \
+X( 6,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_SCALE,       "HTBFWSync OSTS=%08x%08x CRTS=%08x%08x CalcClkSpd=%d\n", 5, 0) \
+X( 7,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_SCALE_RPT,   "FW Sync scale info OSTS=%08x%08x CRTS=%08x%08x CalcClkSpd=%d\n", 5, 0) \
+X( 8,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK,        "FW Sync Partition marker: %d\n", 1, 0) \
+X( 9,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK_RPT,    "FW Sync Partition repeat: %d\n", 1, 0) \
+X( 10, HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK_SCALE,  "Text not used", 6, 0)\
+\
+X( 1,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_TABLE,       "MMU page op table entry page_id=%08x%08x index=%d level=%d val=%08x%08x map=%d\n", 7, 0) \
+X( 2,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_ALLOC,       "MMU allocating DevVAddr from %08x%08x to %08x%08x\n", 4, 0) \
+X( 3,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_FREE,        "MMU freeing DevVAddr from %08x%08x to %08x%08x\n", 4, 0) \
+X( 4,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_MAP,         "MMU mapping DevVAddr %08x%08x to DevPAddr %08x%08x\n", 4, 0) \
+X( 5,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_PMRMAP,      "MMU mapping PMR DevVAddr %08x%08x to DevPAddr %08x%08x\n", 4, 0)  \
+X( 6,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_UNMAP,       "MMU unmapping DevVAddr %08x%08x\n", 2, 0) \
+\
+X( 1,  HTB_GROUP_SYNC,  HTB_SF_SYNC_SERVER_ALLOC,       "Server sync allocation [%08X]\n", 1, 0) \
+X( 2,  HTB_GROUP_SYNC,  HTB_SF_SYNC_SERVER_UNREF,       "Server sync unreferenced [%08X]\n", 1, 0) \
+X( 3,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_CREATE,     "Sync OP create 0x%08x, block count=%d, server syncs=%d, client syncs=%d\n", 4, 0) \
+X( 4,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_TAKE,       "Sync OP take 0x%08x server syncs=%d, client syncs=%d\n", 3, 0) \
+X( 5,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_COMPLETE,   "Sync OP complete 0x%08x\n", 1, 0) \
+X( 6,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_DESTROY,    "Sync OP destroy 0x%08x\n", 1, 0) \
+\
+X( 1,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_TA_DEPRECATED, "Kick TA: FWCtx %08X @ %d\n", 2, 0) \
+X( 2,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_3D_DEPRECATED, "Kick 3D: FWCtx %08X @ %d\n", 2, 0) \
+X( 3,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_CDM_DEPRECATED,"Kick CDM: FWCtx %08X @ %d\n", 2, 0) \
+X( 4,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_RTU,           "Kick RTU: FWCtx %08X @ %d\n", 2, 0) \
+X( 5,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_SHG,           "Kick SHG: FWCtx %08X @ %d\n", 2, 0) \
+X( 6,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_2D_DEPRECATED, "Kick 2D: FWCtx %08X @ %d\n", 2, 0) \
+X( 7,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_UNCOUNTED,     "Kick (uncounted) for all DMs\n", 0, 0) \
+X( 8,  HTB_GROUP_MAIN,  HTB_SF_MAIN_FWCCB_CMD,          "FW CCB Cmd: %d\n", 1, 0) \
+X( 9,  HTB_GROUP_MAIN,  HTB_SF_MAIN_PRE_POWER,          "Pre-power duration @ phase [%d] (0-shutdown,1-startup) RGX: %llu ns SYS: %llu ns\n", 3, 0) \
+X(10,  HTB_GROUP_MAIN,  HTB_SF_MAIN_POST_POWER,         "Post-power duration @ phase [%d] (0-shutdown,1-startup) SYS: %llu ns RGX: %llu ns\n", 3, 0) \
+X(11,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_TA,            "Kick TA: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(12,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_3D,            "Kick 3D: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(13,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_CDM,           "Kick CDM: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(14,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_2D,            "Kick 2D: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(15,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_ERROR,          "Error: (%u) in file: %s:%u\n", 3, 2) \
+X(16,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_MSGLVL_ERROR,   "Log error in file: %s:%u\n", 2, 1) \
+X(17,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_COND_ERROR_T,   "Error: (%u) Conditional is unexpectedly true in file: %s:%u\n", 3, 2) \
+X(18,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_COND_ERROR_F,   "Error: (%u) Conditional is unexpectedly false in file: %s:%u\n", 3, 2) \
+X(19,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_WARNING,        "Warning: (%u) in file: %s @ line: %u\n", 3, 2) \
+X(20,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_MSGLVL_WARN,    "Log warning in file: %s:%u\n", 2, 1) \
+\
+X( 1,  HTB_GROUP_BRG,   HTB_SF_BRG_BRIDGE_CALL,         "Bridge call: start: %010u: bid %03d fid %d\n", 3, 0) \
+X( 2,  HTB_GROUP_BRG,   HTB_SF_BRG_BRIDGE_CALL_ERR,     "Bridge call: start: %010u: bid %03d fid %d error %d\n", 4, 0) \
+\
+X( 1,  HTB_GROUP_DBG,   HTB_SF_DBG_INTPAIR,             "0x%8.8x 0x%8.8x\n", 2, 0) \
+\
+X( 65535, HTB_GROUP_NONE, HTB_SF_LAST,                  "You should not use this string\n", 15, 0)
+
+
+
+/* gid - Group numbers */
+typedef enum _HTB_LOG_SFGROUPS {
+#define X(A,B) A,
+	HTB_LOG_SFGROUPLIST
+#undef X
+} HTB_LOG_SFGROUPS;
+
+
+/* Group flags are stored in an array of elements.
+ * Each of which have a certain number of bits.
+ */
+#define HTB_FLAG_EL_T                   IMG_UINT32
+#define HTB_FLAG_NUM_BITS_IN_EL         (sizeof(HTB_FLAG_EL_T) * 8)
+
+#define HTB_LOG_GROUP_FLAG_GROUP(gid)   ((gid-1) / HTB_FLAG_NUM_BITS_IN_EL)
+#define HTB_LOG_GROUP_FLAG(gid)         (gid ? (0x1 << ((gid-1)%HTB_FLAG_NUM_BITS_IN_EL)) : 0)
+#define HTB_LOG_GROUP_FLAG_NAME(gid)    HTB_LOG_TYPE_ ## gid
+
+/* Group enable flags */
+typedef enum _HTB_LOG_TYPE {
+#define X(a, b) HTB_LOG_GROUP_FLAG_NAME(b) = HTB_LOG_GROUP_FLAG(a),
+	HTB_LOG_SFGROUPLIST
+#undef X
+} HTB_LOG_TYPE;
+
+
+
+/* The symbolic names found in the table above are assigned an ui32 value of
+ * the following format:
+ * 31 30 28 27       20   19  16    15  12      11            0   bits
+ * -   ---   ---- ----     ----      ----        ---- ---- ----
+ *    0-11: id number
+ *   12-15: group id number
+ *   16-19: number of parameters
+ *   20-24: If the format contains a string argument, argument
+ *          index is passed. If there isn't a string argument, 0 is passed
+ *   25-27: unused
+ *   28-30: active: identify SF packet, otherwise regular int32
+ *      31: reserved for signed/unsigned compatibility
+ *
+ * The following macro assigns those values to the enum generated SF ids list.
+ */
+#define HTB_LOG_IDMARKER            (0x70000000)
+#define HTB_LOG_CREATESFID(a,b,e,f)   (((a) | (b << 12) | (e << 16)) | (f << 20) | HTB_LOG_IDMARKER)
+
+#define HTB_LOG_IDMASK              (0xFFF00000)
+#define HTB_LOG_VALIDID(I)          ( ((I) & HTB_LOG_IDMASK) == HTB_LOG_IDMARKER )
+
+typedef enum HTB_LOG_SFids {
+#define X(a, b, c, d, e, f) c = HTB_LOG_CREATESFID(a,b,e,f),
+	HTB_LOG_SFIDLIST
+#undef X
+} HTB_LOG_SFids;
+
+/* Return the group id that the given (enum generated) id belongs to */
+#define HTB_SF_GID(x) (((x)>>12) & 0xf)
+/* Future improvement to support log levels */
+#define HTB_SF_LVL(x) (0)
+/* Returns how many arguments the SF(string format) for the given
+ * (enum generated) id requires.
+ */
+#define HTB_SF_PARAMNUM(x) (((x)>>16) & 0xf)
+/* Returns the index of the argument that contains a string, 1 indexed so 0 means no string argument.
+ */
+#define HTB_SF_STRNUM(x) (((x)>>20) & 0xf)
+/* Returns the id of given enum */
+#define HTB_SF_ID(x) (x & 0xfff)
+
+/* Format of messages is: SF:PID:TID:TIMEPT1:TIMEPT2:[PARn]*
+ */
+#define HTB_LOG_HEADER_SIZE         5
+#define HTB_LOG_MAX_PARAMS          15
+
+/* HTB supports string lengths of up to 23 characters + \0 */
+#define HTB_LOG_STR_ARG_NUM_WORDS         (6)
+#define HTB_LOG_STR_ARG_SIZE              (HTB_LOG_STR_ARG_NUM_WORDS * sizeof(IMG_UINT32))
+
+#if defined(__cplusplus)
+}
+#endif
+
+#if defined(__GNUC__)
+#if GCC_VERSION_AT_LEAST(12, 1)
+#define HTB_FILE_NAME __FILE_NAME__
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+#elif defined(__clang__)
+#if CLANG_VERSION_AT_LEAST(10)
+#define HTB_FILE_NAME __FILE_NAME__
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+
+/* Defines for handling MARK_SCALE special case */
+#define HTB_GID_CTRL 1
+#define HTB_ID_MARK_SCALE 10
+#define HTB_MARK_SCALE_ARG_ARRAY_SIZE 6
+
+/* Defines for extracting args from array for special case MARK_SCALE */
+#define HTB_ARG_SYNCMARK 0
+#define HTB_ARG_OSTS_PT1 1
+#define HTB_ARG_OSTS_PT2 2
+#define HTB_ARG_CRTS_PT1 3
+#define HTB_ARG_CRTS_PT2 4
+#define HTB_ARG_CLKSPD   5
+
+#endif /* HTBUFFER_SF_H */
+/*****************************************************************************
+ End of file (htbuffer_sf.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_types.h b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_types.h
new file mode 100644
index 000000000000..fd061eb428d7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_types.h
@@ -0,0 +1,122 @@
+/*************************************************************************/ /*!
+@File           htbuffer_types.h
+@Title          Host Trace Buffer types.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_TYPES_H
+#define HTBUFFER_TYPES_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "htbuffer_sf.h"
+
+/* The group flags array of ints large enough to store all the group flags */
+#if defined(PVRSRV_ENABLE_HTB)
+#define HTB_FLAG_NUM_EL (((HTB_GROUP_DBG-1) / HTB_FLAG_NUM_BITS_IN_EL) + 1)
+extern IMG_INTERNAL HTB_FLAG_EL_T g_auiHTBGroupEnable[HTB_FLAG_NUM_EL];
+
+#define HTB_GROUP_ENABLED(SF) (g_auiHTBGroupEnable[HTB_LOG_GROUP_FLAG_GROUP(HTB_SF_GID(SF))] & HTB_LOG_GROUP_FLAG(HTB_SF_GID(SF)))
+#else
+#define HTB_GROUP_ENABLED(SF) IMG_FALSE
+#endif
+
+/*************************************************************************/ /*!
+ Host Trace Buffer operation mode
+ Care must be taken if changing this enum to ensure the MapFlags[] array
+ in htbserver.c is kept in-step.
+*/ /**************************************************************************/
+typedef enum
+{
+	/*! Undefined operation mode */
+	HTB_OPMODE_UNDEF = 0,
+
+	/*! Drop latest, intended for continuous logging to a UM daemon.
+	 *  If the daemon does not keep up, the most recent log data
+	 *  will be dropped
+	 */
+	HTB_OPMODE_DROPLATEST,
+
+	/*! Drop oldest, intended for crash logging.
+	 *  Data will be continuously written to a circular buffer.
+	 *  After a crash the buffer will contain events leading up to the crash
+	 */
+	HTB_OPMODE_DROPOLDEST,
+
+	/*! Block write if buffer is full */
+	HTB_OPMODE_BLOCK,
+
+	HTB_OPMODE_LAST = HTB_OPMODE_BLOCK
+} HTB_OPMODE_CTRL;
+
+
+/*************************************************************************/ /*!
+ Host Trace Buffer log mode control
+*/ /**************************************************************************/
+typedef enum
+{
+	/*! Undefined log mode, used if update is not applied */
+	HTB_LOGMODE_UNDEF = 0,
+
+	/*! Log trace messages for all PIDs. */
+	HTB_LOGMODE_ALLPID,
+
+	/*! Log trace messages for specific PIDs only. */
+	HTB_LOGMODE_RESTRICTEDPID,
+
+	HTB_LOGMODE_LAST = HTB_LOGMODE_RESTRICTEDPID
+} HTB_LOGMODE_CTRL;
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_TYPES_H */
+
+/******************************************************************************
+ End of file (htbuffer_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/img_types_check.h b/drivers/gpu/drm/img/img-volcanic/services/include/img_types_check.h
new file mode 100644
index 000000000000..ce73789877f6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/img_types_check.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Performs size checks on some of the IMG types.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_TYPES_CHECK_H
+#define IMG_TYPES_CHECK_H
+
+#ifndef __KERNEL__
+#include <assert.h>
+#endif /* __KERNEL__ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+static_assert(sizeof(IMG_BOOL) == 1, "invalid size of IMG_BOOL");
+static_assert(sizeof(IMG_INT) == 4, "invalid size of IMG_INT");
+static_assert(sizeof(IMG_UINT) == 4, "invalid size of IMG_UINT");
+static_assert(sizeof(PVRSRV_ERROR) == 4, "invalid size of PVRSRV_ERROR");
+
+#endif /* IMG_TYPES_CHECK_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/info_page_client.h b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_client.h
new file mode 100644
index 000000000000..56d394dd99a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_client.h
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose shared memory (i.e. information page) mapped by
+                kernel space driver and user space clients. All info page
+                entries are sizeof(IMG_UINT32) on both 32/64-bit environments.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_CLIENT_H
+#define INFO_PAGE_CLIENT_H
+
+#include "device_connection.h"
+#include "info_page_defs.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#endif
+
+/*************************************************************************/ /*!
+@Function      GetInfoPage
+
+@Description   Return Info Page address
+
+@Input         hDevConnection - Services device connection
+
+@Return        Info Page address
+*/
+/*****************************************************************************/
+static INLINE IMG_PUINT32 GetInfoPage(SHARED_DEV_CONNECTION hDevConnection)
+{
+#if defined(__KERNEL__)
+	PVR_UNREFERENCED_PARAMETER(hDevConnection);
+
+	return (PVRSRVGetPVRSRVData())->pui32InfoPage;
+#else
+	return hDevConnection->pui32InfoPage;
+#endif
+}
+
+/*************************************************************************/ /*!
+@Function      GetInfoPageDebugFlags
+
+@Description   Return Info Page debug flags
+
+@Input         hDevConnection - Services device connection
+
+@Return        Info Page debug flags
+*/
+/*****************************************************************************/
+static INLINE IMG_UINT32 GetInfoPageDebugFlags(SHARED_DEV_CONNECTION hDevConnection)
+{
+	return GetInfoPage(hDevConnection)[DEBUG_FEATURE_FLAGS];
+}
+
+#endif /* INFO_PAGE_CLIENT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/info_page_defs.h b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_defs.h
new file mode 100644
index 000000000000..949afcfd869d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_defs.h
@@ -0,0 +1,133 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose shared memory (i.e. information page) mapped by
+                kernel space driver and user space clients. All information page
+                entries are sizeof(IMG_UINT32) on both 32/64-bit environments.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_DEFS_H
+#define INFO_PAGE_DEFS_H
+
+/* Info page is divided in "blocks" of size INFO_PAGE_CHUNK_SIZE. Each block
+ * should start with the INFO_PAGE_[NAME]_BLOCK_START macro which takes the
+ * value of previous block (except for the first block which starts from 0).
+ *
+ * Each last value of the block (INFO_PAGE_[NAME]_BLOCK_END) should be unused
+ * within that block since it's a first value of the next block. This value
+ * should be a multiple of INFO_PAGE_CHUNK_SIZE.
+ *
+ * Care must be taken to not go over allowed number of elements in each block
+ * which is marked with the INFO_PAGE_[NAME]_BLOCK_END macro.
+ *
+ * Blocks consist of entries that are defined with the INFO_PAGE_ENTRY() macro.
+ * Each entry must define a unique index within the block and as mentioned
+ * can't go over the INFO_PAGE_[NAME]_BLOCK_END limit.
+ *
+ * Always add blocks to the end of the existing list and update
+ * INFO_PAGE_TOTAL_SIZE after.
+ *
+ * See current usage of the Info Page below for examples.
+ */
+
+#define INFO_PAGE_CHUNK_SIZE                        8
+#define INFO_PAGE_BLOCK_END(start,size)             ((start) + (size) * INFO_PAGE_CHUNK_SIZE)
+#define INFO_PAGE_ENTRY(start,index)                ((start) + (index))
+#define INFO_PAGE_SIZE_IN_BYTES(end)                ((end) * sizeof(IMG_UINT32))
+
+#define INFO_PAGE_CACHEOP_BLOCK_START               0
+#define INFO_PAGE_CACHEOP_BLOCK_END                 INFO_PAGE_BLOCK_END(INFO_PAGE_CACHEOP_BLOCK_START, 1)
+#define INFO_PAGE_HWPERF_BLOCK_START                INFO_PAGE_CACHEOP_BLOCK_END
+#define INFO_PAGE_HWPERF_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_HWPERF_BLOCK_START, 1)
+#define INFO_PAGE_TIMEOUT_BLOCK_START               INFO_PAGE_HWPERF_BLOCK_END
+#define INFO_PAGE_TIMEOUT_BLOCK_END                 INFO_PAGE_BLOCK_END(INFO_PAGE_TIMEOUT_BLOCK_START, 2)
+#define INFO_PAGE_BRIDGE_BLOCK_START                INFO_PAGE_TIMEOUT_BLOCK_END
+#define INFO_PAGE_BRIDGE_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_BRIDGE_BLOCK_START, 1)
+#define INFO_PAGE_DEBUG_BLOCK_START                 INFO_PAGE_BRIDGE_BLOCK_END
+#define INFO_PAGE_DEBUG_BLOCK_END                   INFO_PAGE_BLOCK_END(INFO_PAGE_DEBUG_BLOCK_START, 1)
+#define INFO_PAGE_DEVMEM_BLOCK_START                INFO_PAGE_DEBUG_BLOCK_END
+#define INFO_PAGE_DEVMEM_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_DEVMEM_BLOCK_START, 1)
+
+/* IMPORTANT: Make sure this always uses the last INFO_PAGE_[NAME]_BLOCK_END definition.*/
+#define INFO_PAGE_TOTAL_SIZE                        INFO_PAGE_SIZE_IN_BYTES(INFO_PAGE_DEVMEM_BLOCK_END)
+
+/* CacheOp information page entries */
+
+#define CACHEOP_INFO_UMKMTHRESHLD                   INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 0) /*!< UM=>KM routing threshold in bytes */
+#define CACHEOP_INFO_KMDFTHRESHLD                   INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 1) /*!< KM/DF threshold in bytes */
+#define CACHEOP_INFO_LINESIZE                       INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 2) /*!< CPU data cache line size */
+#define CACHEOP_INFO_PGSIZE                         INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 3) /*!< CPU MMU page size */
+
+/* HWPerf information page entries */
+
+#define HWPERF_FILTER_SERVICES_IDX                  INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 0)
+#define HWPERF_FILTER_EGL_IDX                       INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 1)
+#define HWPERF_FILTER_OPENGLES_IDX                  INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 2)
+#define HWPERF_FILTER_OPENCL_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 3)
+#define HWPERF_FILTER_VULKAN_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 4)
+#define HWPERF_FILTER_OPENGL_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 5)
+
+/* Timeout values */
+
+#define TIMEOUT_INFO_VALUE_RETRIES                  INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 0)
+#define TIMEOUT_INFO_VALUE_TIMEOUT_MS               INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 1)
+#define TIMEOUT_INFO_CONDITION_RETRIES              INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 2)
+#define TIMEOUT_INFO_CONDITION_TIMEOUT_MS           INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 3)
+#define TIMEOUT_INFO_TASK_QUEUE_RETRIES             INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 4)
+#define TIMEOUT_INFO_TASK_QUEUE_FLUSH_TIMEOUT_MS    INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 5)
+
+/* Bridge Info */
+
+#define BRIDGE_INFO_RGX_BRIDGES                     INFO_PAGE_ENTRY(INFO_PAGE_BRIDGE_BLOCK_START, 0)
+#define BRIDGE_INFO_PVR_BRIDGES                     INFO_PAGE_ENTRY(INFO_PAGE_BRIDGE_BLOCK_START, 1)
+
+/* Debug features */
+
+#define DEBUG_FEATURE_FLAGS                         INFO_PAGE_ENTRY(INFO_PAGE_DEBUG_BLOCK_START, 0)
+
+#define DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED    0x1 /* flag - not part of info page */
+#define DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED      0x2 /* flag - not part of info page */
+
+/* Device memory related information */
+
+/* This value is 64-bits wide, next value should have index larger by 2 */
+#define DEVMEM_INFO_PHYS_BUF_MAX_SIZE               INFO_PAGE_ENTRY(INFO_PAGE_DEVMEM_BLOCK_START, 0)
+
+#endif /* INFO_PAGE_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/kerneldisplay.h b/drivers/gpu/drm/img/img-volcanic/services/include/kerneldisplay.h
new file mode 100644
index 000000000000..df6fc23cb676
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/kerneldisplay.h
@@ -0,0 +1,832 @@
+/*************************************************************************/ /*!
+@File
+@Title          Interface between 3rd party display controller (DC) drivers
+                and the Services server module.
+@Description    API between Services and the 3rd party DC driver and vice versa.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(KERNELDISPLAY_H)
+#define KERNELDISPLAY_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include <powervr/mem_types.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "pvrsrv_surface.h"
+#include "dc_external.h"
+
+/*!
+ ******************************************************************************
+ @mainpage Introduction and Document Scope
+This document is a function reference for the 3rd party display
+controller interface. The interface ensures that the 3rd party display driver and
+Services can exchange configurations and buffers and can inform each
+other about occurring events like render completion or vsyncs.
+
+This interface should be used and implemented if the 3rd party display driver
+is not using a native OS interface such as the Linux DRM/KMS API.
+
+To fully understand the document the reader should be aware that
+the Services driver - offering PowerVR core and platform functionality to the
+driver layers (OGLES etc.) above - is split into the "Services Client"
+user mode library and the "Services Server" kernel mode driver that communicate
+via the "Services Bridge". The terms "User-Mode" and "Client" are used as
+synonyms as well as "Kernel-Mode" and "Server".
+
+Please refer to the more comprehensive '3rd Party Display Integration Guide'
+for an architecture overview and an explanation of the data flow between a
+client process, Services and the 3rd party display driver. It also contains
+descriptions about how to make use of the client side interface that is supposed
+to be integrated in some kind of display manager like e.g. PowerVR Graphics DDK WSEGL
+window manager.
+
+The documented functions are split into different parts:
+- Callbacks that need an implementation by the 3rd party display driver and
+that are used by the Services server, some of them optional (kerneldisplay.h)
+- Functions that the Services server module exports and can be used by the
+3rd party display driver. Mainly to register/deregister a new display device
+and to query state information (kerneldisplay.h)
+- Functions that are called by the client process (a display manager e.g. WSEGL)
+to control the DC interaction (dc_client.h)
+- Commonly used structure definitions to exchange data between the modules
+(dc_external.h, pvrsrv_surface.h)
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       GetInfo
+
+@Description    Query the display controller for its information structure.
+
+   Called by client function: #PVRSRVDCGetInfo()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Output         psDisplayInfo           Display info structure
+*/
+/*****************************************************************************/
+typedef void (*GetInfo)(IMG_HANDLE hDeviceData,
+                        DC_DISPLAY_INFO *psDisplayInfo);
+
+/*************************************************************************/ /*!
+@Function       PanelQueryCount
+
+@Description    Query the display controller for how many panels are
+                connected to it.
+
+   Called by client function: #PVRSRVDCPanelQueryCount()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData            Device private data
+
+@Output         pui32NumPanels         Number of panels
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*PanelQueryCount)(IMG_HANDLE hDeviceData,
+                                        IMG_UINT32 *pui32NumPanels);
+
+/*************************************************************************/ /*!
+@Function       PanelQuery
+
+@Description    Query the display controller for information on what panel(s)
+                are connected to it and their properties.
+
+   Called by client function: #PVRSRVDCPanelQuery()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Input          ui32PanelsArraySize     Size of the PanelInfo array
+                                        (i.e. number of panels that
+                                        can be returned)
+
+@Output         pui32NumPanels          Number of panels returned
+
+@Output         pasPanelInfo            Array of formats, allocated beforehand
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*PanelQuery)(IMG_HANDLE hDeviceData,
+                                   IMG_UINT32 ui32PanelsArraySize,
+                                   IMG_UINT32 *pui32NumPanels,
+                                   PVRSRV_PANEL_INFO *pasPanelInfo);
+
+/*************************************************************************/ /*!
+@Function       FormatQuery
+
+@Description    Query the display controller to check if it supports the specified
+                format(s).
+
+   Called by client function: #PVRSRVDCFormatQuery()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Input          ui32NumFormats          Number of formats to check
+                                        (i.e. length of the Format array)
+
+@Input          pasFormat               Array of formats to check
+
+@Output         pui32Supported          For each format, the number of display
+                                        pipes that support that format
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*FormatQuery)(IMG_HANDLE hDeviceData,
+                                    IMG_UINT32 ui32NumFormats,
+                                    PVRSRV_SURFACE_FORMAT *pasFormat,
+                                    IMG_UINT32 *pui32Supported);
+
+/*************************************************************************/ /*!
+@Function       DimQuery
+
+@Description    Query the specified display plane for the display dimensions
+                it supports.
+
+   Called by client function: #PVRSRVDCDimQuery()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Input          ui32NumDims             Number of dimensions to check
+                                        (i.e. length of the Dim array)
+
+@Input          pasDim                  Array of dimensions to check
+
+@Output         pui32Supported          For each dimension, the number of
+                                        display pipes that support that
+                                        dimension
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*DimQuery)(IMG_HANDLE hDeviceData,
+                                 IMG_UINT32 ui32NumDims,
+                                 PVRSRV_SURFACE_DIMS *pasDim,
+                                 IMG_UINT32 *pui32Supported);
+
+/*************************************************************************/ /*!
+@Function       SetBlank
+
+@Description    Enable/disable blanking of the screen.
+
+   Called by client function: #PVRSRVDCSetBlank()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Input          bEnable                 Enable/Disable the blanking
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*SetBlank)(IMG_HANDLE hDeviceData,
+                                 IMG_BOOL bEnable);
+
+/*************************************************************************/ /*!
+@Function       SetVSyncReporting
+
+@Description    Enable VSync reporting. If enabled, the 3rd party display
+                driver is expected to call PVRSRVCheckStatus() after a VSync
+                event occurred. This will signal the Services driver global
+                event object.
+
+   Called by client function: #PVRSRVDCSetVSyncReporting()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Input          bEnable                 Enable/Disable the reporting
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*SetVSyncReporting)(IMG_HANDLE hDeviceData,
+                                          IMG_BOOL bEnable);
+
+/*************************************************************************/ /*!
+@Function       LastVSyncQuery
+
+@Description    Query the time the last vsync happened.
+
+   Called by client function: #PVRSRVDCLastVSyncQuery()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Output         pi64Timestamp           The requested timestamp
+                                        of the system time in ns
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*LastVSyncQuery)(IMG_HANDLE hDeviceData,
+                                       IMG_INT64 *pi64Timestamp);
+
+/*************************************************************************/ /*!
+@Function       ContextCreate
+
+@Description    Create display context.
+                The client application and the display driver have to agree
+                as to what exactly a context represents, Services will just
+                pass it through and tie it to its own concept of a
+                display context.
+                It might contain additional locks, work-queues or other
+                important data that is necessary to drive the interaction
+                with Services. A context is usually associated with one device
+                but that is not a hard requirement.
+
+   Called by client function: #PVRSRVDCDisplayContextCreate()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Output         hDisplayContext         Created display context
+
+@Return         PVRSRV_OK if the context was created
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*ContextCreate)(IMG_HANDLE hDeviceData,
+                                      IMG_HANDLE *hDisplayContext);
+
+/*************************************************************************/ /*!
+@Function       ContextConfigureCheck
+
+@Description    Check to see if a configuration is valid for the display
+                controller. Because of runtime changes the display context
+                might not be able to work with a certain configuration anymore.
+                This function is intended to be called before ContextConfigure
+                so that the application can query details about the current
+                surface config and then do the ContextConfigure call with
+                the updated data.
+                The arrays should be z-sorted, with the farthest plane
+                first and the nearest plane last.
+
+   Called by client function: #PVRSRVDCContextConfigureCheck()
+
+   Implementation of this callback is optional.
+
+@Input          hDisplayContext         Display context
+
+@Input          ui32PipeCount           Number of display pipes to configure
+                                        (length of the input arrays)
+
+@Input          pasSurfAttrib           Array of surface attributes (one for
+                                        each display plane)
+
+@Input          ahBuffers               Array of buffers (one for
+                                        each display plane)
+
+@Return         PVRSRV_OK if the configuration is valid
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*ContextConfigureCheck)(IMG_HANDLE hDisplayContext,
+                                              IMG_UINT32 ui32PipeCount,
+                                              PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+                                              IMG_HANDLE *ahBuffers);
+
+/*************************************************************************/ /*!
+@Function       ContextConfigure
+
+@Description    Configure the display pipeline to display a given buffer.
+                The arrays should be z-sorted, with the farthest plane first
+                and the nearest plane last.
+
+   Called by client function: #PVRSRVDCContextConfigureWithFence()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDisplayContext         Display context
+
+@Input          ui32PipeCount           Number of display pipes to configure
+                                        (i.e. length of the input arrays)
+
+@Input          pasSurfAttrib           Array of surface attributes (one for
+                                        each display plane)
+
+@Input          ahBuffers               Array of buffers (one for
+                                        each display plane)
+
+@Input          ui32DisplayPeriod       The number of VSync periods this
+                                        configuration should be displayed for
+
+@Input          hConfigData             Config handle which gets passed to
+                                        DisplayConfigurationRetired when this
+                                        configuration is retired
+*/
+/*****************************************************************************/
+typedef void (*ContextConfigure)(IMG_HANDLE hDisplayContext,
+                                 IMG_UINT32 ui32PipeCount,
+                                 PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+                                 IMG_HANDLE *ahBuffers,
+                                 IMG_UINT32 ui32DisplayPeriod,
+                                 IMG_HANDLE hConfigData);
+
+/*************************************************************************/ /*!
+@Function       ContextDestroy
+
+@Description    Destroy a display context.
+
+   Called by client function: #PVRSRVDCDisplayContextDestroy()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDisplayContext         Display context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+typedef void (*ContextDestroy)(IMG_HANDLE hDisplayContext);
+
+/*************************************************************************/ /*!
+@Function       BufferAlloc
+
+@Description    Allocate a display buffer. This is a request to the display
+                controller to allocate a buffer from memory that is addressable
+                by the display controller.
+
+                Note: The actual allocation of display memory can be deferred
+                until the first call to acquire, but the handle for the buffer
+                still needs to be created and returned to the caller as well
+                as some information about the buffer that's required upfront.
+
+   Called by client function: #PVRSRVDCBufferAlloc()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDisplayContext         Display context this buffer will be
+                                        used on
+
+@Input          psSurfInfo              Attributes of the buffer
+
+@Output         puiLog2PageSize         Pagesize in log2(bytes) of one page
+
+@Output         pui32PageCount          Number of pages in the buffer
+
+@Output         pui32ByteStride         Stride (in bytes) of allocated buffer
+
+@Output         phBuffer                Handle to allocated buffer
+
+@Return         PVRSRV_OK if the buffer was successfully allocated
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferAlloc)(IMG_HANDLE hDisplayContext,
+                                    DC_BUFFER_CREATE_INFO *psSurfInfo,
+                                    IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+                                    IMG_UINT32 *pui32PageCount,
+                                    IMG_UINT32 *pui32ByteStride,
+                                    IMG_HANDLE *phBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferImport
+
+@Description    Import memory allocated from an external source (e.g. Services)
+                to the display controller. The DC checks to see if the import
+                is compatible and potentially sets up HW to map the imported
+                buffer, although this isn't required to happen until the first
+                call to BufferMap.
+
+                Note: Provide this function if the controller
+                can scan out arbitrary memory, allocated for another purpose by
+                Services. To be able to use this buffer (depending on its
+                origin) the display controller probably will need a MMU.
+
+   Called by client function: #PVRSRVDCBufferImport()
+
+   Implementation of this callback is optional.
+
+@Input          hDisplayContext         Display context this buffer will be
+                                        used on
+
+@Input          ui32NumPlanes           Number of planes
+
+@Input          pahImport               Array of handles (one per colour channel)
+
+@Input          psSurfAttrib            Surface attributes of the buffer
+
+@Output         phBuffer                Handle to imported buffer
+
+@Return         PVRSRV_OK if the buffer was successfully imported
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferImport)(IMG_HANDLE hDisplayContext,
+                                     IMG_UINT32 ui32NumPlanes,
+                                     IMG_HANDLE **pahImport,
+                                     DC_BUFFER_IMPORT_INFO *psSurfAttrib,
+                                     IMG_HANDLE *phBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferAcquire
+
+@Description    Acquire the buffer's physical memory pages. If the buffer doesn't
+                have any memory backing yet then this will trigger the 3rd
+                party driver to allocate it.
+
+                Note: The page count isn't passed back in this function as
+                Services has already obtained it during BufferAlloc.
+
+   Called when Services requests buffer access for the first time. Usually part
+   of non display class functions.
+
+   Implementation of this callback is mandatory.
+
+@Input          hBuffer                 Handle to the buffer
+
+@Output         pasDevPAddr             Array of device physical page address
+                                        of this buffer
+
+@Output         ppvLinAddr              CPU virtual address of buffer. This is
+                                        optional but if you have one you must
+                                        return it otherwise return NULL.
+
+@Return         PVRSRV_OK if the buffer was successfully acquired
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferAcquire)(IMG_HANDLE hBuffer,
+                                      IMG_DEV_PHYADDR *pasDevPAddr,
+                                      void **ppvLinAddr);
+
+/*************************************************************************/ /*!
+@Function       BufferRelease
+
+@Description    Undo everything done by BufferAcquire.
+                Will release the buffer's physical memory pages if
+                BufferAcquire allocated them.
+
+   Called when Services finished buffer access.
+
+   Implementation of this callback is mandatory.
+
+@Input          hBuffer                 Handle to the buffer
+*/
+/*****************************************************************************/
+typedef void (*BufferRelease)(IMG_HANDLE hBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferFree
+
+@Description    Release a reference to the device buffer. If this was the last
+                reference the 3rd party driver is entitled to free the backing
+                memory and other related resources.
+
+   Called by client function: #PVRSRVDCBufferFree()
+
+   Implementation of this callback is mandatory.
+
+@Input          hBuffer                 Buffer handle we're releasing
+*/
+/*****************************************************************************/
+typedef void (*BufferFree)(IMG_HANDLE hBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferMap
+
+@Description    Map the buffer into the display controller
+                Note: This function depends on the behaviour of
+                BufferAlloc/BufferAcquire/BufferImport/BufferSystemAcquire
+                and the controller's ability to map in memory.
+                If the controller has no MMU or the above functions already
+                map the buffer this callback does not need an implementation.
+
+   Called by client function: #PVRSRVDCBufferPin()
+
+   Implementation of this callback is optional.
+
+@Input          hBuffer                 Buffer to map
+
+@Return         PVRSRV_OK if the buffer was successfully mapped
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferMap)(IMG_HANDLE hBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferUnmap
+
+@Description    Undo everything done by BufferMap.
+                Usually that means to unmap a buffer from the display controller.
+
+   Called by client function: #PVRSRVDCBufferUnpin()
+
+   Implementation of this callback is optional.
+
+@Input          hBuffer                 Buffer to unmap
+*/
+/*****************************************************************************/
+typedef void (*BufferUnmap)(IMG_HANDLE hBuffer);
+
+
+/*************************************************************************/ /*!
+@Function       BufferSystemAcquire
+
+@Description    DEPRICATED, please use BufferAlloc
+                Acquire the system buffer from the display driver.
+                If the OS should trigger a mode change then it's not allowed to
+                free the previous buffer until Services has released it
+                via BufferSystemRelease.
+
+   Called by client function: #PVRSRVDCSystemBufferAcquire()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Output         puiLog2PageSize         The physical pagesize in log2(bytes)
+                                        of one page that the buffer is composed of
+
+@Output         pui32PageCount          The number of pages the buffer contains
+
+@Output         pui32ByteStride         Byte stride of the buffer
+
+@Output         phSystemBuffer          Handle to the buffer object
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferSystemAcquire)(IMG_HANDLE hDeviceData,
+                                            IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+                                            IMG_UINT32 *pui32PageCount,
+                                            IMG_UINT32 *pui32ByteStride,
+                                            IMG_HANDLE *phSystemBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferSystemRelease
+
+@Description    DEPRICATED, please use BufferFree
+                Release a display buffer acquired with BufferSystemAcquire.
+                Services calls this after it has no use for the buffer anymore.
+                The buffer must not be destroyed before Services releases it
+                with this call.
+
+   Called by client function: #PVRSRVDCSystemBufferRelease()
+
+   Implementation of this callback is optional.
+
+@Input          hSystemBuffer          Handle to the buffer object
+*/ /**************************************************************************/
+typedef	void (*BufferSystemRelease)(IMG_HANDLE hSystemBuffer);
+
+/*************************************************************************/ /*!
+@Function       ResetDevice
+
+@Description    Reset the device state. This function should perform actions
+                required for getting the device to the working state. This can
+                be for example used during resume operation from a hibernation.
+
+                Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*ResetDevice)(IMG_HANDLE hDeviceData);
+
+#if defined(INTEGRITY_OS)
+typedef PVRSRV_ERROR (*AcquireKernelMappingData)(IMG_HANDLE hBuffer, IMG_HANDLE *phMapping, void **ppPhysAddr);
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+typedef IMG_HANDLE (*GetPmr)(IMG_HANDLE hBuffer, size_t ulOffset);
+#endif
+#endif
+/*!
+ * Function table for functions to be implemented by the display controller
+ * that will be called from within Services.
+ * The table will be provided to Services with the call to DCRegisterDevice.
+ *
+ * Typedef: ::DC_DEVICE_FUNCTIONS
+ */
+typedef struct _DC_DEVICE_FUNCTIONS_
+{
+	/* Mandatory query functions */
+	GetInfo                 pfnGetInfo; /*!< See #GetInfo */
+	PanelQueryCount         pfnPanelQueryCount; /*!< See #PanelQueryCount */
+	PanelQuery              pfnPanelQuery; /*!< See #PanelQuery */
+	FormatQuery             pfnFormatQuery; /*!< See #FormatQuery */
+	DimQuery                pfnDimQuery; /*!< See #DimQuery */
+
+	/* Optional blank/vsync functions */
+	SetBlank                pfnSetBlank; /*!< See #SetBlank */
+	SetVSyncReporting       pfnSetVSyncReporting; /*!< See #SetVSyncReporting */
+	LastVSyncQuery          pfnLastVSyncQuery; /*!< See #LastVSyncQuery */
+
+	/* Mandatory configure functions */
+	ContextCreate           pfnContextCreate; /*!< See #ContextCreate */
+	ContextDestroy          pfnContextDestroy; /*!< See #ContextDestroy */
+	ContextConfigure        pfnContextConfigure; /*!< See #ContextConfigure */
+
+	/* Optional context functions */
+	ContextConfigureCheck   pfnContextConfigureCheck; /*!< See #ContextConfigureCheck */
+
+	/* Mandatory buffer functions */
+	BufferAlloc             pfnBufferAlloc; /*!< See #BufferAlloc */
+	BufferAcquire           pfnBufferAcquire; /*!< See #BufferAcquire */
+	BufferRelease           pfnBufferRelease; /*!< See #BufferRelease */
+	BufferFree              pfnBufferFree; /*!< See #BufferFree */
+
+	/* Optional - Provide this function if your controller can
+	 * scan out arbitrary memory, allocated for another purpose
+	 * by Services. */
+	BufferImport            pfnBufferImport; /*!< See #BufferImport */
+
+	/* Optional - Provide these functions if your controller
+	 * has an MMU and does not (or cannot) map/unmap buffers at
+	 * alloc/free time */
+	BufferMap               pfnBufferMap; /*!< See #BufferMap */
+	BufferUnmap             pfnBufferUnmap; /*!< See #BufferUnmap */
+
+	/* Optional - DEPRICATED */
+	BufferSystemAcquire     pfnBufferSystemAcquire; /*!< See #BufferSystemAcquire */
+	BufferSystemRelease     pfnBufferSystemRelease; /*!< See #BufferSystemRelease */
+
+#if defined(INTEGRITY_OS)
+	/* The addition of these functions allow dc_server to delegate calls to
+	 * the respective functions on its PMRs towards the DC module
+	 */
+	AcquireKernelMappingData    pfnAcquireKernelMappingData;
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	GetPmr                      pfnGetPmr;
+#endif
+#endif
+
+   /* Optional - Provide this function if your controller should be allowed
+    * to be reset by Services.
+    * This can be used for example during suspend-to-RAM or suspend-to-disk
+    * procedures. */
+	ResetDevice                 pfnResetDevice; /*!< See #ResetDevice */
+} DC_DEVICE_FUNCTIONS;
+
+
+/*
+ * Functions exported by kernel Services for use by 3rd party kernel display
+ * controller device driver
+*/
+
+/*************************************************************************/ /*!
+@Function       DCRegisterDevice
+
+@Description    This needs to be called by the display driver before any further
+                communication with Services.
+                It registers a display controller device with Services. After this
+                registration Services is able to use the display controller
+                and will make use of the callbacks. Services will provide the
+                hDeviceData in the callbacks whenever necessary.
+
+@Input          psFuncTable             Callback function table
+
+@Input          ui32MaxConfigsInFlight  The maximum number of configs that this
+                                        display device can have in-flight. This
+                                        determines the number of possible calls
+                                        to ContextConfigure before Services has to
+                                        wait for DCDisplayConfigurationRetired
+                                        calls.
+
+@Input          hDeviceData             Device private data passed into callbacks
+
+@Output         phSrvHandle             Services handle to pass back into
+                                        UnregisterDCDevice
+
+@Return         PVRSRV_OK if the display controller driver was successfully registered
+*/
+/*****************************************************************************/
+PVRSRV_ERROR DCRegisterDevice(DC_DEVICE_FUNCTIONS *psFuncTable,
+                              IMG_UINT32 ui32MaxConfigsInFlight,
+                              IMG_HANDLE hDeviceData,
+                              IMG_HANDLE *phSrvHandle);
+
+/*************************************************************************/ /*!
+@Function       DCUnregisterDevice
+
+@Description    Unregister a display controller device. Undo everything done with
+                DCRegisterDevice. Services will stop using this device.
+
+@Input          hSrvHandle              Services device handle
+*/
+/*****************************************************************************/
+void DCUnregisterDevice(IMG_HANDLE hSrvHandle);
+
+/*************************************************************************/ /*!
+@Function       DCDisplayConfigurationRetired
+
+@Description    Called when a configuration as been retired due to a new
+                configuration now being active.
+
+@Input          hConfigData             ConfigData that is being retired
+*/
+/*****************************************************************************/
+void DCDisplayConfigurationRetired(IMG_HANDLE hConfigData);
+
+/*************************************************************************/ /*!
+@Function       DCDisplayHasPendingCommand
+
+@Description    Called to check if there are still pending commands in
+                the Software Command Processor queue.
+
+@Input          hConfigData             ConfigData to check for pending
+                                        commands
+
+@Return         IMG_TRUE if there is at least one pending command
+*/
+/*****************************************************************************/
+IMG_BOOL DCDisplayHasPendingCommand(IMG_HANDLE hConfigData);
+
+/*************************************************************************/ /*!
+@Function       DCImportBufferAcquire
+
+@Description    Acquire information about a buffer that was imported with
+                BufferImport. DCImportBufferRelease has to be called after
+                the buffer will not be used anymore.
+
+@Input          hImport                 Import buffer
+
+@Input          uiLog2PageSize          Pagesize in log2(bytes) of the buffer
+
+@Output         pui32PageCount          Size of the buffer in pages
+
+@Output         pasDevPAddr             Array of device physical page address
+                                        of this buffer
+
+@Return         PVRSRV_OK if the import buffer was successfully acquired
+*/
+/*****************************************************************************/
+PVRSRV_ERROR DCImportBufferAcquire(IMG_HANDLE hImport,
+                                   IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+                                   IMG_UINT32 *pui32PageCount,
+                                   IMG_DEV_PHYADDR **pasDevPAddr);
+
+/*************************************************************************/ /*!
+@Function       DCImportBufferRelease
+
+@Description    Release an imported buffer.
+
+@Input          hImport                 Import handle we are releasing
+
+@Input          pasDevPAddr             Import data which was returned from
+                                        DCImportBufferAcquire
+*/
+/*****************************************************************************/
+void DCImportBufferRelease(IMG_HANDLE hImport,
+                           IMG_DEV_PHYADDR *pasDevPAddr);
+
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* KERNELDISPLAY_H */
+
+/******************************************************************************
+ End of file (kerneldisplay.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/km_apphint_defs_common.h b/drivers/gpu/drm/img/img-volcanic/services/include/km_apphint_defs_common.h
new file mode 100644
index 000000000000..ad38c76793d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/km_apphint_defs_common.h
@@ -0,0 +1,308 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef KM_APPHINT_DEFS_COMMON_H
+#define KM_APPHINT_DEFS_COMMON_H
+
+/*
+*******************************************************************************
+ Build variables
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(EnableTrustedDeviceAceConfig,     BOOL,           GPUVIRT_VAL, PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG,  NO_PARAM_TABLE,   ALWAYS   ) \
+X(CleanupThreadPriority,            UINT32,         NEVER,       PVRSRV_APPHINT_CLEANUPTHREADPRIORITY,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(WatchdogThreadPriority,           UINT32,         NEVER,       PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientBufferSize,           UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevmemHistoryBufSizeLog2,         UINT32,         ALWAYS,      PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevmemHistoryMaxEntries,          UINT32,         ALWAYS,      PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES,    NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Module parameters
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(GeneralNon4KHeapPageSize,         UINT32,         ALWAYS,      PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE,      NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableSignatureChecks,            BOOL,           PDUMP,       PVRSRV_APPHINT_ENABLESIGNATURECHECKS,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(SignatureChecksBufSize,           UINT32,         PDUMP,       PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DisableClockGating,               BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLECLOCKGATING,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisableDMOverlap,                 BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLEDMOVERLAP,              NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableRandomContextSwitch,        BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableSoftResetContextSwitch,     BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH,  NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFWContextSwitch,            UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(FWContextSwitchProfile,           UINT32,         VALIDATION,  PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableRDPowerIsland,              UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLERDPOWERISLAND,           NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DriverMode,                       STRING,         ALWAYS,      PVRSRV_APPHINT_DRIVERMODE,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(AutoVzGPUPowerdown,               BOOL,           ALWAYS,      PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(GuestFWHeapStride,                UINT64,         ALWAYS,      PVRSRV_APPHINT_GUESTFWHEAPSTRIDE,             NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FirmwarePerf,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_FIRMWAREPERF,                  NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(HWPerfFWBufSizeInKB,              UINT32,         PDUMP,       PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB,           NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostBufSizeInKB,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostThreadTimeoutInMS,      UINT32,         VALIDATION,  PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS,   NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(NewFilteringMode,                 BOOL,           VALIDATION,  PVRSRV_APPHINT_NEWFILTERINGMODE,              NO_PARAM_TABLE,   ALWAYS   ) \
+X(TruncateMode,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_TRUNCATEMODE,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(EmuMaxFreq,                       UINT32,         ALWAYS,      PVRSRV_APPHINT_EMUMAXFREQ,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(GPIOValidationMode,               UINT32,         VALIDATION,  PVRSRV_APPHINT_GPIOVALIDATIONMODE,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(RGXBVNC,                          STRING,         ALWAYS,      PVRSRV_APPHINT_RGXBVNC,                       NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FWContextSwitchCrossDM,           UINT32,         ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(ValidateIrq,                      BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATEIRQ,                   NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(TPUTrilinearFracMaskPDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskVDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskCDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskTDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskRDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(HTBufferSizeInKB,                 UINT32,         ALWAYS,      PVRSRV_APPHINT_HTBUFFERSIZE,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(FWTraceBufSizeInDWords,           UINT32,         ALWAYS,      PVRSRV_APPHINT_FWTRACEBUFSIZEINDWORDS,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnablePageFaultDebug,             BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFullSyncTracking,           BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(IgnoreHWReportedBVNC,             BOOL,           ALWAYS,      PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC,          NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(PhysMemTestPasses,                UINT32,         ALWAYS,      PVRSRV_APPHINT_PHYSMEMTESTPASSES,             NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FBCDCVersionOverride,             UINT32,         VALIDATION,  PVRSRV_APPHINT_FBCDCVERSIONOVERRIDE,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TestSLRInterval,                  UINT32,         VALIDATION,  PVRSRV_APPHINT_TESTSLRINTERVAL,               NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnablePollOnChecksumErrorStatus,  UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(RiscvDmiTest,                     BOOL,           VALIDATION,  PVRSRV_APPHINT_RISCVDMITEST,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevMemFWHeapPolicy,               UINT32,         ALWAYS,      PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY,          NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableAPMAll,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_ENABLEAPM,                     NO_PARAM_TABLE,   ALWAYS   ) \
+X(KernelCCBSizeLog2,                UINT32,         VALIDATION,  PVRSRV_APPHINT_KCCB_SIZE_LOG2,                NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(SyncCheckpointPoolMaxLog2,        UINT32,         ALWAYS,      PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(SyncCheckpointPoolInitLog2,       UINT32,         ALWAYS,      PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(PhysHeapMinMemOnConnection,       UINT32,         ALWAYS,      PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION,    NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(RestrictGpuLocalPhysHeapSizeMB,   UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(PhysHeapHybridDefault2CpuLocal,   BOOL,           ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DebugDumpFWTLogType,              UINT32,         ALWAYS,      PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE,           NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableIdleCycleStealing,          UINT32,         VALIDATION,  PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING,       NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FaultDetectionTimeInterval,       UINT32,         VALIDATION,  PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC, NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(ICSTimeIntervalThreshold,         UINT32,         VALIDATION,  PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD,     NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(ICSTestModeOn,                    BOOL,           VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FaultInjection,                   UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Debugfs parameters - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(EnableHTBLogGroup,                UINT32Bitfield, ALWAYS,      PVRSRV_APPHINT_ENABLEHTBLOGGROUP,             htb_loggroup_tbl, ALWAYS   ) \
+X(HTBOperationMode,                 UINT32List,     ALWAYS,      PVRSRV_APPHINT_HTBOPERATIONMODE,              htb_opmode_tbl,   ALWAYS   ) \
+X(EnableFTraceGPU,                  BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEFTRACEGPU,               NO_PARAM_TABLE,   NEVER    ) \
+X(HWPerfClientFilter_Services,      UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_SERVICES,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_EGL,           UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_EGL,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenGLES,      UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGLES,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenCL,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENCL,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_Vulkan,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_VULKAN,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenGL,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGL,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(CacheOpConfig,                    UINT32,         ALWAYS,      PVRSRV_APPHINT_CACHEOPCONFIG,                 NO_PARAM_TABLE,   ALWAYS   ) \
+X(CacheOpUMKMThresholdSize,         UINT32,         ALWAYS,      PVRSRV_APPHINT_CACHEOPUMKMHRESHOLDSIZE,       NO_PARAM_TABLE,   ALWAYS   ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+/* Device Firmware config */\
+X(AssertOnHWRTrigger,               BOOL,           ALWAYS,      APPHNT_BLDVAR_ASSERTONHWRTRIGGER,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(AssertOutOfMemory,                BOOL,           ALWAYS,      PVRSRV_APPHINT_ASSERTOUTOFMEMORY,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(CheckMList,                       BOOL,           ALWAYS,      PVRSRV_APPHINT_CHECKMLIST,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableLogGroup,                   UINT32Bitfield, ALWAYS,      PVRSRV_APPHINT_ENABLELOGGROUP,                fwt_loggroup_tbl, ALWAYS   ) \
+X(FirmwareLogType,                  UINT32List,     ALWAYS,      PVRSRV_APPHINT_FIRMWARELOGTYPE,               fwt_logtype_tbl,  ALWAYS   ) \
+X(HWRDebugDumpLimit,                UINT32,         ALWAYS,      PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(SecondaryOSClockSource,           UINT32List,     ALWAYS,      PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE,        timecorr_clk_tbl, ALWAYS   ) \
+X(HWPerfFWFilter,                   UINT64,         ALWAYS,      PVRSRV_APPHINT_HWPERFFWFILTER,                NO_PARAM_TABLE,   ALWAYS   ) \
+/* Device host config */ \
+X(EnableAPM,                        UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEAPM,                     NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisableFEDLogging,                BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLEFEDLOGGING,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(ZeroFreelist,                     BOOL,           ALWAYS,      PVRSRV_APPHINT_ZEROFREELIST,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisablePDumpPanic,                BOOL,           PDUMP,       PVRSRV_APPHINT_DISABLEPDUMPPANIC,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFWPoisonOnFree,             BOOL,           DEBUG,       PVRSRV_APPHINT_ENABLEFWPOISONONFREE,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(GPUUnitsPowerChange,              BOOL,           VALIDATION,  PVRSRV_APPHINT_GPUUNITSPOWERCHANGE,           NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostFilter,                 UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFHOSTFILTER,              NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters.
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT_COMMON \
+/* debuginfo device apphint name  modparam name */ \
+X(EnableAPM,                      EnableAPMAll)
+
+/*
+*******************************************************************************
+ * Types used in the APPHINT_LIST_<GROUP> lists must be defined here.
+ * New types require specific handling code to be added
+******************************************************************************/
+#define APPHINT_DATA_TYPE_LIST \
+X(BOOL) \
+X(UINT64) \
+X(UINT32) \
+X(UINT32Bitfield) \
+X(UINT32List) \
+X(STRING)
+
+#define APPHINT_CLASS_LIST \
+X(ALWAYS) \
+X(NEVER) \
+X(DEBUG) \
+X(PDUMP) \
+X(VALIDATION) \
+X(GPUVIRT_VAL)
+
+#define APPHINT_RT_CLASS_LIST \
+X(ALWAYS) \
+X(NEVER)
+
+/*
+*******************************************************************************
+ Visibility control for module parameters
+ These bind build variables to AppHint Visibility Groups.
+******************************************************************************/
+#define APPHINT_ENABLED_CLASS_ALWAYS IMG_TRUE
+#define APPHINT_ENABLED_CLASS_NEVER IMG_FALSE
+#define apphint_modparam_class_ALWAYS(a, b, c) apphint_modparam_enable(a, b, c)
+#if defined(DEBUG)
+	#define APPHINT_ENABLED_CLASS_DEBUG IMG_TRUE
+	#define apphint_modparam_class_DEBUG(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_DEBUG IMG_FALSE
+	#define apphint_modparam_class_DEBUG(a, b, c)
+#endif
+#if defined(PDUMP)
+	#define APPHINT_ENABLED_CLASS_PDUMP IMG_TRUE
+	#define apphint_modparam_class_PDUMP(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_PDUMP IMG_FALSE
+	#define apphint_modparam_class_PDUMP(a, b, c)
+#endif
+	#define APPHINT_ENABLED_CLASS_VALIDATION IMG_FALSE
+	#define apphint_modparam_class_VALIDATION(a, b, c)
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	#define APPHINT_ENABLED_CLASS_GPUVIRT_VAL IMG_TRUE
+	#define apphint_modparam_class_GPUVIRT_VAL(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_GPUVIRT_VAL IMG_FALSE
+	#define apphint_modparam_class_GPUVIRT_VAL(a, b, c)
+#endif
+
+/*
+*******************************************************************************
+ AppHint defaults based on other build parameters
+******************************************************************************/
+#if defined(ASSERTONHWRTRIGGER_DEFAULT_ENABLED)
+	#define APPHNT_BLDVAR_ASSERTONHWRTRIGGER      1
+#else
+	#define APPHNT_BLDVAR_ASSERTONHWRTRIGGER      0
+#endif
+#if defined(DEBUG)
+	#define APPHNT_BLDVAR_DEBUG             1
+	#define APPHNT_BLDVAR_DBGDUMPLIMIT      RGXFWIF_HWR_DEBUG_DUMP_ALL
+#else
+	#define APPHNT_BLDVAR_DEBUG             0
+	#define APPHNT_BLDVAR_DBGDUMPLIMIT      1
+#endif
+#if defined(PDUMP)
+#define APPHNT_BLDVAR_ENABLESIGNATURECHECKS     IMG_TRUE
+#else
+#define APPHNT_BLDVAR_ENABLESIGNATURECHECKS     IMG_FALSE
+#endif
+#if defined(DEBUG)
+#define APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG      IMG_TRUE
+#else
+#define APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG      IMG_FALSE
+#endif
+
+#if defined(DEBUG)
+	#define APPHNT_PHYSMEMTEST_ENABLE             1
+#else
+	#define APPHNT_PHYSMEMTEST_ENABLE             0
+#endif
+
+/* Data types and actions */
+typedef enum {
+	APPHINT_DATA_TYPE_INVALID = 0,
+#define X(a) APPHINT_DATA_TYPE_ ## a,
+	APPHINT_DATA_TYPE_LIST
+#undef X
+	APPHINT_DATA_TYPE_MAX
+} APPHINT_DATA_TYPE;
+
+typedef enum {
+#define X(a) APPHINT_CLASS_ ## a,
+	APPHINT_CLASS_LIST
+#undef X
+	APPHINT_CLASS_MAX
+} APPHINT_CLASS;
+
+typedef enum {
+#define X(a) APPHINT_RT_CLASS_ ## a,
+    APPHINT_RT_CLASS_LIST
+#undef X
+    APPHINT_RT_CLASS_MAX
+} APPHINT_RT_CLASS;
+#endif /* KM_APPHINT_DEFS_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/os_cpu_cache.h b/drivers/gpu/drm/img/img-volcanic/services/include/os_cpu_cache.h
new file mode 100644
index 000000000000..56f92036ff49
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/os_cpu_cache.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS and CPU d-cache maintenance mechanisms
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines for cache management which are visible internally only
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OS_CPU_CACHE_H
+#define OS_CPU_CACHE_H
+
+#include "info_page_defs.h"
+
+#define PVRSRV_CACHE_OP_TIMELINE			0x8 /*!< Request SW_SYNC timeline notification when executed */
+#define PVRSRV_CACHE_OP_FORCE_SYNCHRONOUS	0x10 /*!< Force all batch members to be executed synchronously */
+
+#define CACHEFLUSH_ISA_X86					0x1	/*!< x86/x64 specific UM range-based cache flush */
+#define CACHEFLUSH_ISA_ARM64				0x2	/*!< Aarch64 specific UM range-based cache flush */
+#define CACHEFLUSH_ISA_GENERIC				0x3	/*!< Other ISA's without UM range-based cache flush */
+#ifndef CACHEFLUSH_ISA_TYPE
+	#if defined(__i386__) || defined(__x86_64__)
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_X86
+	#elif defined(__arm64__) || defined(__aarch64__)
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_ARM64
+	#else
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_GENERIC
+	#endif
+#endif
+
+#if (CACHEFLUSH_ISA_TYPE == CACHEFLUSH_ISA_X86) || (CACHEFLUSH_ISA_TYPE == CACHEFLUSH_ISA_ARM64)
+#define CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH		/*!< x86/x86_64/ARM64 supports user-mode d-cache flush */
+#endif
+
+#endif	/* OS_CPU_CACHE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pdump.h b/drivers/gpu/drm/img/img-volcanic/services/include/pdump.h
new file mode 100644
index 000000000000..3ef71846446e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pdump.h
@@ -0,0 +1,238 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef SERVICES_PDUMP_H
+#define SERVICES_PDUMP_H
+
+#include "img_types.h"
+#include "services_km.h"
+
+
+/* A PDump out2.txt script is made up of 3 sections from three buffers:
+ * *
+ *  - Init phase buffer    - holds PDump data written during driver
+ *                            initialisation, non-volatile.
+ *  - Main phase buffer   - holds PDump data written after driver init,
+ *                            volatile.
+ *  - Deinit phase buffer - holds PDump data  needed to shutdown HW/play back,
+ *                            written only during driver initialisation using
+ *                            the DEINIT flag.
+ *
+ * Volatile in this sense means that the buffer is drained and cleared when
+ * the pdump capture application connects and transfers the data to file.
+ *
+ * The PDump sub-system uses the driver state (init/post-init), whether
+ * the pdump capture application is connected or not (capture range set/unset)
+ * and, if pdump connected whether the frame is in the range set, to decide
+ * which of the 3 buffers to write the PDump data. Hence there are several
+ * key time periods in the lifetime of the kernel driver that is enabled
+ * with PDUMP=1 (flag XX labels below time line):
+ *
+ * Events:load              init        pdump       enter          exit         pdump
+ *       driver             done       connects     range          range     disconnects
+ *         |__________________|____________|__________|______________|____________|______ . . .
+ * State:  |   init phase     | no capture | <- capture client connected ->       | no capture
+ *         |                  |            |                                      |
+ *         |__________________|____________|______________________________________|_____ . . .
+ * Flag:   | CT,DI            | NONE,CT,PR | NONE,CT,PR                           | See no
+ *         | Never NONE or PR | Never DI   | Never DI                             |   capture
+ *         |__________________|____________|______________________________________|_____ . . .
+ * Write   | NONE -undef      | -No write  | -No write | -Main buf    | -No write | See no
+ * buffer  | CT -Init buf     | -Main buf  | -Main buf | -Main buf    | -Main buf |   capture
+ *         | PR -undef        | -Init buf  | -undef    | -Init & Main | -undef    |
+ *         | DI -Deinit buf   | -undef     | -undef    | -undef       | -undef    |
+ *         |__________________|____________|___________|______________|___________|_____ . . .
+ *
+ * Note: The time line could repeat if the pdump capture application is
+ * disconnected and reconnected without unloading the driver module.
+ *
+ * The DEINIT (DI) | CONTINUOUS (CT) | PERSISTENT (PR) flags must never
+ * be OR'd together and given to a PDump call since undefined behaviour may
+ * result and produce an invalid PDump which does not play back cleanly.
+ *
+ * The decision on which flag to use comes down to which time period the
+ * client or server driver makes the PDump write call AND the nature/purpose
+ * of the data.
+ *
+ * Note: This is a simplified time line, not all conditions represented.
+ *
+ */
+
+typedef IMG_UINT32 PDUMP_FLAGS_T;
+
+#define PDUMP_FLAGS_NONE            PDUMP_NONE     /*<! Output this entry with no special treatment i.e. output
+                                                          only if in frame range. */
+#define PDUMP_FLAGS_BLKDATA         PDUMP_BLKDATA  /*<! This flag indicates block-mode PDump data to be recorded
+                                                          in Block script stream in addition to Main script stream,
+                                                          if capture mode is set to BLOCKED */
+
+#define PDUMP_FLAGS_DEINIT          0x20000000U    /*<! Output this entry to the de-initialisation section, must
+                                                          only be used by the initialisation code in the Server. */
+
+#define PDUMP_FLAGS_POWER           0x08000000U    /*<! Output this entry even when a power transition is ongoing,
+                                                          as directed by other PDUMP flags. */
+
+#define PDUMP_FLAGS_CONTINUOUS      PDUMP_CONT     /*<! Output this entry always regardless of framed capture range,
+                                                          used by client applications being dumped.
+                                                          During init phase of driver such data carrying this flag
+                                                          will be recorded and present for all PDump client
+                                                          connections.
+                                                          Never combine with the PERSIST flag. */
+
+#define PDUMP_FLAGS_PERSISTENT      PDUMP_PERSIST  /*<! Output this entry always regardless of app and range,
+                                                          used by persistent resources created *after* driver
+                                                          initialisation that must appear in all PDump captures
+                                                          (i.e. current capture regardless of frame range (CONT)
+                                                          and all future PDump captures) for that driver
+                                                          instantiation/session.
+                                                          Effectively this is data that is not forgotten
+                                                          for the second and subsequent PDump client connections.
+                                                          Never combine with the CONTINUOUS flag. */
+
+#define PDUMP_FLAGS_INTERVAL        0x04000000U    /*<! Output this entry even when the capture is on a
+                                                          "no capture interval frame" (see pdump -sr option).
+                                                          Useful for commands that have a resource that was written
+                                                          out in a frame that was captured. For example,
+                                                          used by RGXScheduleCleanupCommand. */
+
+#define PDUMP_FLAGS_DEBUG           0x00010000U    /*<! For internal debugging use */
+
+#define PDUMP_FLAGS_NOHW            0x00000001U    /* For internal use: Skip sending instructions to the hardware
+                                                        when NO_HARDWARE=0 AND PDUMP=1 */
+
+#define PDUMP_FLAGS_FORCESPLIT      0x00000002U	   /* Forces Main and Block script streams to split - Internal
+                                                        flag used in Block mode of PDump */
+
+#define PDUMP_FLAGS_PDUMP_LOCK_HELD 0x00000004U    /* This flags denotes that PDUMP_LOCK is held already so
+                                                        further calls to PDUMP_LOCK  with this flag set will not
+                                                        try to take pdump lock. If PDUMP_LOCK is called without
+                                                        this flag by some other thread then it will try to take
+                                                        PDUMP_LOCK which would make that thread sleep. This flag
+                                                        introduced to enforce the order of pdumping after bridge
+                                                        lock removal */
+
+#define PDUMP_FILEOFFSET_FMTSPEC    "0x%08X"
+typedef IMG_UINT32 PDUMP_FILEOFFSET_T;
+
+/* PDump stream macros*/
+
+/* Parameter stream */
+#define PDUMP_PARAM_INIT_STREAM_NAME       "paramInit"
+#define PDUMP_PARAM_MAIN_STREAM_NAME       "paramMain"
+#define PDUMP_PARAM_DEINIT_STREAM_NAME     "paramDeinit"
+#define PDUMP_PARAM_BLOCK_STREAM_NAME      "paramBlock"
+
+/*** Parameter stream sizes ***/
+
+/* Parameter Init Stream */
+
+/* The streams are made tunable in core.mk, wherein the size of these buffers can be specified.
+ * In case nothing specified in core.mk, then the default size is taken
+ */
+#if defined(PDUMP_PARAM_INIT_STREAM_SIZE)
+	#if (PDUMP_PARAM_INIT_STREAM_SIZE < (700 * 1024))
+		#error PDUMP_PARAM_INIT_STREAM_SIZE must be at least 700 KB
+	#endif
+#endif
+
+/* Parameter Main Stream */
+#if defined(PDUMP_PARAM_MAIN_STREAM_SIZE)
+	#if (PDUMP_PARAM_MAIN_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_PARAM_MAIN_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+/* Parameter Deinit Stream */
+#if defined(PDUMP_PARAM_DEINIT_STREAM_SIZE)
+	#if (PDUMP_PARAM_DEINIT_STREAM_SIZE < (64 * 1024))
+		#error PDUMP_PARAM_DEINIT_STREAM_SIZE must be at least 64 KB
+	#endif
+#endif
+
+/* Parameter Block Stream */
+/* There is no separate parameter Block stream as the Block script stream is
+ * just a filtered Main script stream. Hence it will refer to the Main stream
+ * parameters themselves.
+ */
+
+/*Script stream */
+#define PDUMP_SCRIPT_INIT_STREAM_NAME      "scriptInit"
+#define PDUMP_SCRIPT_MAIN_STREAM_NAME      "scriptMain"
+#define PDUMP_SCRIPT_DEINIT_STREAM_NAME    "scriptDeinit"
+#define PDUMP_SCRIPT_BLOCK_STREAM_NAME     "scriptBlock"
+
+/*** Script stream sizes ***/
+
+/* Script Init Stream */
+#if defined(PDUMP_SCRIPT_INIT_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_INIT_STREAM_SIZE < (256 * 1024))
+		#error PDUMP_SCRIPT_INIT_STREAM_SIZE must be at least 256 KB
+	#endif
+#endif
+
+/* Script Main Stream */
+#if defined(PDUMP_SCRIPT_MAIN_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_MAIN_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_SCRIPT_MAIN_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+/* Script Deinit Stream */
+#if defined(PDUMP_SCRIPT_DEINIT_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_DEINIT_STREAM_SIZE < (64 * 1024))
+		#error PDUMP_SCRIPT_DEINIT_STREAM_SIZE must be at least 64 KB
+	#endif
+#endif
+
+/* Script Block Stream */
+#if defined(PDUMP_SCRIPT_BLOCK_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_BLOCK_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_SCRIPT_BLOCK_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+
+#define PDUMP_PARAM_0_FILE_NAME     "%%0%%.prm"      /*!< Initial Param filename used in PDump capture */
+#define PDUMP_PARAM_N_FILE_NAME     "%%0%%_%02u.prm" /*!< Param filename used when PRM file split */
+#define PDUMP_PARAM_MAX_FILE_NAME   32               /*!< Max Size of parameter name used in out2.txt */
+
+#define PDUMP_IS_CONTINUOUS(flags) ((flags & PDUMP_FLAGS_CONTINUOUS) != 0)
+
+#endif /* SERVICES_PDUMP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/physheap.h b/drivers/gpu/drm/img/img-volcanic/services/include/physheap.h
new file mode 100644
index 000000000000..9cca0dd25286
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/physheap.h
@@ -0,0 +1,564 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physical heap management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the interface for the physical heap management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+#include "opaque_types.h"
+#include "pmr_impl.h"
+#include "physheap_config.h"
+#include "pvrsrv_device.h"
+#include "ra.h"
+
+#ifndef PHYSHEAP_H
+#define PHYSHEAP_H
+
+#define B2KB(x) ((x) >> 10)
+#define B2MB(x) ((x) >> 20)
+
+static inline IMG_UINT64 KB2B(IMG_UINT64 ui64Kilobytes) { return ui64Kilobytes << 10; }
+static inline IMG_UINT64 MB2B(IMG_UINT64 ui64Megabytes) { return ui64Megabytes << 20; }
+
+typedef struct _PHYS_HEAP_ PHYS_HEAP;
+#define INVALID_PHYS_HEAP 0xDEADDEAD
+
+/* PMB (Physical Memory Block) */
+typedef struct _PMB_ PMB;
+
+typedef IMG_UINT32 PHYS_HEAP_POLICY;
+
+/* Heap has default allocation policy and does not require
+ * any additional OS Functionality. Physically contiguous
+ * allocations are required for this physheap.
+ */
+#define PHYS_HEAP_POLICY_DEFAULT (0U)
+
+/*
+ * Heap has allocation strategy that may produce non
+ * physically contiguous allocations, additional OS functionality
+ * is required to map these allocations into the kernel.
+ */
+#define PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG      (1U)
+#define PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK (1U)
+
+struct _CONNECTION_DATA_;
+
+typedef struct _PG_HANDLE_
+{
+	union
+	{
+		void *pvHandle;
+		IMG_UINT64 ui64Handle;
+	}u;
+	/* The allocation order is log2 value of the number of pages to allocate.
+	 * As such this is a correspondingly small value. E.g, for order 4 we
+	 * are talking 2^4 * PAGE_SIZE contiguous allocation.
+	 * DevPxAlloc API does not need to support orders higher than 4.
+	 */
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	IMG_BYTE    uiOrder;    /* Order of the corresponding allocation */
+	IMG_BYTE    uiOSid;     /* OSid to use for allocation arena.
+	                         * Connection-specific. */
+	IMG_BYTE    uiPad1,
+	            uiPad2;     /* Spare */
+#else
+	IMG_BYTE    uiOrder;    /* Order of the corresponding allocation */
+	IMG_BYTE    uiPad1,
+	            uiPad2,
+	            uiPad3;     /* Spare */
+#endif
+} PG_HANDLE;
+
+/*! Pointer to private implementation specific data */
+typedef void *PHEAP_IMPL_DATA;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_DESTROY_DATA
+@Description    Destroy private implementation specific data.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+*/ /**************************************************************************/
+typedef void (*PFN_DESTROY_DATA)(PHEAP_IMPL_DATA);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_DEV_PADDR
+@Description    Get heap device physical address.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_DEV_PHYADDR    Device physical address.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_DEV_PADDR)(PHEAP_IMPL_DATA, IMG_DEV_PHYADDR*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_CPU_PADDR
+@Description    Get heap CPU physical address.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_CPU_PHYADDR    CPU physical address.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_CPU_PADDR)(PHEAP_IMPL_DATA, IMG_CPU_PHYADDR*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_SIZE
+@Description    Get size of heap.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_UINT64         Size of heap.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_SIZE)(PHEAP_IMPL_DATA, IMG_UINT64*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_PAGE_SHIFT
+@Description    Get heap log2 page shift.
+@Return         IMG_UINT32         Log2 page shift
+*/ /**************************************************************************/
+typedef IMG_UINT32 (*PFN_GET_PAGE_SHIFT)(void);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_MEM_STATS
+@Description    Get total and free memory size of the physical heap managed by
+                the PMR Factory.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_UINT64         total Size of heap.
+@Output         IMG_UINT64         free Size available in a heap.
+@Return         none
+*/ /**************************************************************************/
+typedef void (*PFN_GET_MEM_STATS)(PHEAP_IMPL_DATA, IMG_UINT64 *, IMG_UINT64 *);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_HEAP_STATS_STR_ITER
+@Description    Get string of heap memory spans constituting the heap. This
+                function can be iterated on to print sequential lines of the
+                heap data. Iterate until IMG_FALSE is returned
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@InOut          IMG_CHAR           Pointer to string buffer to be populated
+                                   with sequential heap data.
+@Input          IMG_UINT32         Size of the string buffer.
+@InOut          void**             Iter handle.
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_GET_HEAP_SPANS_STR_ITER)(PHEAP_IMPL_DATA, IMG_CHAR *, IMG_UINT32, void **);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_HEAP_DLM_BACKING
+@Description    Extract reference to DLM heap backing the current IMA heap.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@InOut          PHYS_HEAP**        Pointer to DLM backing heap.
+@Return         void
+*/ /**************************************************************************/
+typedef void (*PFN_GET_HEAP_DLM_BACKING)(PHEAP_IMPL_DATA, PHYS_HEAP **);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef PVRSRV_ERROR (*PFN_PAGES_ALLOC_GPV)(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                            PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+                                            IMG_UINT32 ui32OSid, IMG_PID uiPid);
+#endif
+typedef PVRSRV_ERROR (*PFN_PAGES_ALLOC)(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                        PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+                                        IMG_PID uiPid);
+
+typedef void (*PFN_PAGES_FREE)(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle);
+
+typedef PVRSRV_ERROR (*PFN_PAGES_MAP)(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle,
+                                      size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                                      void **pvPtr);
+
+typedef void (*PFN_PAGES_UNMAP)(PHYS_HEAP *psPhysHeap,
+                                PG_HANDLE *psMemHandle, void *pvPtr);
+
+typedef PVRSRV_ERROR (*PFN_PAGES_CLEAN)(PHYS_HEAP *psPhysHeap,
+                                        PG_HANDLE *pshMemHandle,
+                                        IMG_UINT32 uiOffset,
+                                        IMG_UINT32 uiLength);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_CREATE_PMR
+@Description    Create a PMR physical allocation and back with RAM on creation,
+                if required. The RAM page comes either directly from
+                the Phys Heap's associated pool of memory or from an OS API.
+@Input          psPhysHeap         Pointer to Phys Heap.
+@Input          psConnection       Pointer to device connection.
+@Input          uiSize             Allocation size.
+@Input          ui32NumPhysChunks  Physical chunk count.
+@Input          ui32NumVirtChunks  Virtual chunk count.
+@Input          pui32MappingTable  Mapping Table.
+@Input          uiLog2PageSize     Page size.
+@Input          uiFlags            Memalloc flags.
+@Input          pszAnnotation      Annotation.
+@Input          uiPid              Process ID.
+@Output         ppsPMRPtr          Pointer to PMR.
+@Input          ui32PDumpFlag      PDump flags.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CREATE_PMR)(PHYS_HEAP *psPhysHeap,
+									   struct _CONNECTION_DATA_ *psConnection,
+									   IMG_DEVMEM_SIZE_T uiSize,
+									   IMG_UINT32 ui32NumPhysChunks,
+									   IMG_UINT32 ui32NumVirtChunks,
+									   IMG_UINT32 *pui32MappingTable,
+									   IMG_UINT32 uiLog2PageSize,
+									   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+									   const IMG_CHAR *pszAnnotation,
+									   IMG_PID uiPid,
+									   PMR **ppsPMRPtr,
+									   IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_CREATE_PMB
+@Description    Create a PMB physical allocation and back with Card memory
+                on creation, if required. The card memory comes
+                directly from the DLM Phys Heap's associated pool of memory.
+@Input          psPhysHeap         Pointer to Phys Heap to create the PMB on,
+                                   physheap should be DLM type.
+@Input          uiSize             Allocation size.
+@Input          pszAnnotation      Annotation.
+@Output         ppsPMBPtr          Pointer to PMB created.
+@Output         puiBase            Out pointer to RA Base of PMB.
+@Output         puiSize            Out pointer to size of PMB
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CREATE_PMB)(PHYS_HEAP *psPhysHeap,
+                                       IMG_DEVMEM_SIZE_T uiSize,
+                                       const IMG_CHAR *pszAnnotation,
+                                       PMB **ppsPMBPtr,
+                                       RA_BASE_T *puiBase,
+                                       RA_LENGTH_T *puiSize);
+
+/*! Implementation specific function table */
+typedef struct PHEAP_IMPL_FUNCS_TAG
+{
+	PFN_DESTROY_DATA pfnDestroyData;
+	PFN_GET_DEV_PADDR pfnGetDevPAddr;
+	PFN_GET_CPU_PADDR pfnGetCPUPAddr;
+	PFN_GET_SIZE pfnGetSize;
+	PFN_GET_PAGE_SHIFT pfnGetPageShift;
+	PFN_GET_MEM_STATS pfnGetFactoryMemStats;
+	PFN_GET_HEAP_SPANS_STR_ITER pfnGetHeapSpansStringIter;
+	PFN_GET_HEAP_DLM_BACKING pfnGetHeapDLMBacking;
+	PFN_CREATE_PMR pfnCreatePMR;
+	PFN_CREATE_PMB pfnCreatePMB;
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	PFN_PAGES_ALLOC_GPV pfnPagesAllocGPV;
+#endif
+	PFN_PAGES_ALLOC pfnPagesAlloc;
+	PFN_PAGES_FREE pfnPagesFree;
+	PFN_PAGES_MAP pfnPagesMap;
+	PFN_PAGES_UNMAP pfnPagesUnMap;
+	PFN_PAGES_CLEAN pfnPagesClean;
+} PHEAP_IMPL_FUNCS;
+
+/*************************************************************************/ /*!
+@Function       PhysHeapInitDeviceHeaps
+@Description    Registers and acquires physical memory heaps
+@Input          psDeviceNode pointer to device node
+@Input          psDevConfig  pointer to device config
+@Return         PVRSRV_ERROR PVRSRV_OK on success, or a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapInitDeviceHeaps(PPVRSRV_DEVICE_NODE psDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDeInitDeviceHeaps
+@Description    Releases and unregisters physical memory heaps
+@Input          psDeviceNode pointer to device node
+@Return         PVRSRV_ERROR PVRSRV_OK on success, or a PVRSRV_ error code
+*/ /**************************************************************************/
+void PhysHeapDeInitDeviceHeaps(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateHeapFromConfig
+@Description    Create a new heap. Calls specific heap API depending
+                on heap type.
+@Input          psDevNode    Pointer to device node struct.
+@Input          psConfig     Heap configuration.
+@Output         ppsPhysHeap  Optional pointer to the created heap. Can be NULL
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysHeapCreateHeapFromConfig(PPVRSRV_DEVICE_NODE psDevNode,
+							 PHYS_HEAP_CONFIG *psConfig,
+							 PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreate
+@Description    Create a new heap. Allocated and stored internally.
+                Destroy with PhysHeapDestroy when no longer required.
+@Input          psDevNode    Pointer to device node struct
+@Input          psConfig     Heap configuration.
+@Input          uiPolicy     Phys heap allocation policy.
+@Input          pvImplData   Implementation specific data. Can be NULL.
+@Input          psImplFuncs  Implementation specific function table. Must be
+                             a valid pointer.
+@Output         ppsPhysHeap  Optional pointer to the created heap. Can be NULL
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreate(PPVRSRV_DEVICE_NODE psDevNode,
+							PHYS_HEAP_CONFIG *psConfig,
+							PHYS_HEAP_POLICY uiPolicy,
+							PHEAP_IMPL_DATA pvImplData,
+							PHEAP_IMPL_FUNCS *psImplFuncs,
+							PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDestroyDeviceHeaps
+@Description    Destroys all heaps referenced by a device.
+@Input          psDevNode Pointer to a device node struct.
+@Return         void
+*/ /**************************************************************************/
+void PhysHeapDestroyDeviceHeaps(PPVRSRV_DEVICE_NODE psDevNode);
+
+void PhysHeapDestroy(PHYS_HEAP *psPhysHeap);
+
+PVRSRV_ERROR PhysHeapAcquire(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapAcquireByID
+@Description    Acquire PhysHeap by DevPhysHeap.
+@Input          eDevPhysHeap Device Phys Heap.
+@Input          psDevNode    Pointer to device node struct
+@Output         ppsPhysHeap  PhysHeap if found.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapAcquireByID(PVRSRV_PHYS_HEAP eDevPhysHeap,
+								 PPVRSRV_DEVICE_NODE psDevNode,
+								 PHYS_HEAP **ppsPhysHeap);
+
+void PhysHeapRelease(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetImplData
+@Description    Get physical heap implementation specific data.
+@Input          psPhysHeap   Pointer to physical heap.
+@Input          psConfig     Heap configuration.
+@Return         pvImplData   Implementation specific data. Can be NULL.
+*/ /**************************************************************************/
+PHEAP_IMPL_DATA PhysHeapGetImplData(PHYS_HEAP *psPhysHeap);
+
+PHYS_HEAP_TYPE PhysHeapGetType(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetPolicy
+@Description    Get phys heap allocation policy flags.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         PHYS_HEAP_POLICY Phys heap policy flags.
+*/ /**************************************************************************/
+PHYS_HEAP_POLICY PhysHeapGetPolicy(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetFlags
+@Description    Get phys heap usage flags.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         PHYS_HEAP_USAGE_FLAGS Phys heap usage flags.
+*/ /**************************************************************************/
+PHYS_HEAP_USAGE_FLAGS PhysHeapGetFlags(PHYS_HEAP *psPhysHeap);
+
+IMG_BOOL PhysHeapValidateDefaultHeapExists(PPVRSRV_DEVICE_NODE psDevNode);
+
+#if defined(SUPPORT_STATIC_IPA)
+IMG_UINT32 PhysHeapGetIPAValue(PHYS_HEAP *psPhysHeap);
+
+IMG_UINT32 PhysHeapGetIPAMask(PHYS_HEAP *psPhysHeap);
+
+IMG_UINT32 PhysHeapGetIPAShift(PHYS_HEAP *psPhysHeap);
+#endif
+
+PVRSRV_ERROR PhysHeapGetCpuPAddr(PHYS_HEAP *psPhysHeap,
+									   IMG_CPU_PHYADDR *psCpuPAddr);
+
+
+PVRSRV_ERROR PhysHeapGetSize(PHYS_HEAP *psPhysHeap,
+								   IMG_UINT64 *puiSize);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetMemInfo
+@Description    Get phys heap memory statistics for a given physical heap ID.
+@Input          psDevNode          Pointer to device node struct
+@Input          ui32PhysHeapCount  Physical heap count
+@Input          paePhysHeapID      Physical heap ID
+@Output         paPhysHeapMemStats Buffer that holds the memory statistics
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysHeapGetMemInfo(PPVRSRV_DEVICE_NODE psDevNode,
+                   IMG_UINT32 ui32PhysHeapCount,
+                   PVRSRV_PHYS_HEAP *paePhysHeapID,
+                   PHYS_HEAP_MEM_STATS_PTR paPhysHeapMemStats);
+
+PVRSRV_ERROR PhysHeapGetDevPAddr(PHYS_HEAP *psPhysHeap,
+								 IMG_DEV_PHYADDR *psDevPAddr);
+
+void PhysHeapCpuPAddrToDevPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PhysHeapDevPAddrToCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr,
+								IMG_DEV_PHYADDR *psDevPAddr);
+
+IMG_CHAR *PhysHeapPDumpMemspaceName(PHYS_HEAP *psPhysHeap);
+
+const IMG_CHAR *PhysHeapName(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreatePMR
+@Description    Function calls an implementation-specific function pointer.
+                See function pointer for details.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreatePMR(PHYS_HEAP *psPhysHeap,
+							   struct _CONNECTION_DATA_ *psConnection,
+							   IMG_DEVMEM_SIZE_T uiSize,
+							   IMG_UINT32 ui32NumPhysChunks,
+							   IMG_UINT32 ui32NumVirtChunks,
+							   IMG_UINT32 *pui32MappingTable,
+							   IMG_UINT32 uiLog2PageSize,
+							   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							   const IMG_CHAR *pszAnnotation,
+							   IMG_PID uiPid,
+							   PMR **ppsPMRPtr,
+							   IMG_UINT32 ui32PDumpFlags,
+							   PVRSRV_MEMALLOCFLAGS_T *uiOutFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreatePMB
+@Description    Function calls an implementation-specific function pointer.
+                See @Ref PFN_CREATE_PMB "PFN_CREATE_PMB" for details.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreatePMB(PHYS_HEAP *psPhysHeap,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               const IMG_CHAR *pszAnnotation,
+                               PMB **ppsPMRPtr,
+                               RA_BASE_T *puiBase,
+                               RA_LENGTH_T *puiSize);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDeviceNode
+@Description    Get pointer to the device node this heap belongs to.
+@Input          psPhysHeap          Pointer to physical heap.
+@Return         PPVRSRV_DEVICE_NODE Pointer to device node.
+*/ /**************************************************************************/
+PPVRSRV_DEVICE_NODE PhysHeapDeviceNode(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapInitByPVRLayer
+@Description    Is phys heap to be initialised in PVR layer?
+@Input          ePhysHeap           phys heap
+@Return         IMG_BOOL            return IMG_TRUE if yes
+*/ /**************************************************************************/
+IMG_BOOL PhysHeapInitByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapUserModeAlloc
+@Description    Is allocation from UM allowed?
+@Input          ePhysHeap           phys heap
+@Return         IMG_BOOL            return IMG_TRUE if yes
+*/ /**************************************************************************/
+IMG_BOOL PhysHeapUserModeAlloc(PVRSRV_PHYS_HEAP ePhysHeap);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+PVRSRV_ERROR PhysHeapPagesAllocGPV(PHYS_HEAP *psPhysHeap,
+                                   size_t uiSize,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_UINT32 ui32OSid, IMG_PID uiPid);
+#endif
+
+PVRSRV_ERROR PhysHeapPagesAlloc(PHYS_HEAP *psPhysHeap,
+                                size_t uiSize,
+                                PG_HANDLE *psMemHandle,
+                                IMG_DEV_PHYADDR *psDevPAddr,
+                                IMG_PID uiPid);
+
+void PhysHeapPagesFree(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle);
+
+PVRSRV_ERROR PhysHeapPagesMap(PHYS_HEAP *psPhysHeap,
+                              PG_HANDLE *pshMemHandle,
+                              size_t uiSize,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              void **pvPtr);
+
+void PhysHeapPagesUnMap(PHYS_HEAP *psPhysHeap,
+                        PG_HANDLE *psMemHandle,
+                        void *pvPtr);
+
+PVRSRV_ERROR PhysHeapPagesClean(PHYS_HEAP *psPhysHeap,
+                                PG_HANDLE *pshMemHandle,
+                                IMG_UINT32 uiOffset,
+                                IMG_UINT32 uiLength);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetPageShift
+@Description    Get phys heap page shift.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         IMG_UINT32   Log2 page shift
+*/ /**************************************************************************/
+IMG_UINT32 PhysHeapGetPageShift(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapFreeMemCheck
+@Description    Check a physheap has the required amount of free memory.
+
+@Input          psPhysHeap          Pointer to physical heap.
+@Input          ui64MinRequiredMem  The minimum free memory for success (bytes).
+@Output         pui64FreeMem        The free memory in the physical heap (bytes).
+
+@Return         PVRSRV_ERROR    If successful PVRSRV_OK else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapFreeMemCheck(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 ui64MinRequiredMem,
+                                  IMG_UINT64 *pui64FreeMem);
+
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+/*************************************************************************/ /*!
+@Function       PhysHeapSpasWithDevice
+@Description    Check to see if the device can "see" the SPAS region of the heap.
+                Or in other words, check if any physheap exists
+                in the same SPAS region as `psFromPhysHeap` and is tied to `psToDevNode`.
+@Input          psFromPhysHeap  The physheap whose SPAS region to match.
+@Input          psToDevNode     The device node the other physheap must be tied to.
+@Return         PVRSRV_ERROR    PVRSRV_OK on success, error code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapSpasWithDevice(PHYS_HEAP* psFromPhysHeap, PPVRSRV_DEVICE_NODE psToDevNode);
+#endif
+
+#endif /* PHYSHEAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/physheap_config.h b/drivers/gpu/drm/img/img-volcanic/services/include/physheap_config.h
new file mode 100644
index 000000000000..53d4c909331f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/physheap_config.h
@@ -0,0 +1,331 @@
+/*************************************************************************/ /*!
+@File           physheap_config.h
+@Title          Physical heap Config API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Physical heap configs are created in the system layer and
+                stored against each device node for use in the Services Server
+                common layer.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSHEAP_CONFIG_H
+#define PHYSHEAP_CONFIG_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "dllist.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+
+typedef IMG_UINT32 PHYS_HEAP_USAGE_FLAGS;
+
+/**
+ * ! IMPORTANT !
+ * If you update the PHYS_HEAP_USAGE_FLAGS definitions, you must update the
+ * g_asPhysHeapUsageFlagStrings structure within physheap.c and the
+ * PHYS_HEAP_USAGE_MASK.
+ */
+#define PHYS_HEAP_USAGE_CPU_LOCAL      (1U <<PVRSRV_PHYS_HEAP_CPU_LOCAL)
+#define PHYS_HEAP_USAGE_GPU_LOCAL      (1U <<PVRSRV_PHYS_HEAP_GPU_LOCAL)
+#define PHYS_HEAP_USAGE_GPU_PRIVATE    (1U <<PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+#define PHYS_HEAP_USAGE_EXTERNAL       (1U <<PVRSRV_PHYS_HEAP_EXTERNAL)
+#define PHYS_HEAP_USAGE_GPU_COHERENT   (1U <<PVRSRV_PHYS_HEAP_GPU_COHERENT)
+#define PHYS_HEAP_USAGE_GPU_SECURE     (1U <<PVRSRV_PHYS_HEAP_GPU_SECURE)
+#define PHYS_HEAP_USAGE_FW_CODE        (1U <<PVRSRV_PHYS_HEAP_FW_CODE)
+#define PHYS_HEAP_USAGE_FW_PRIV_DATA   (1U <<PVRSRV_PHYS_HEAP_FW_PRIV_DATA)
+#define PHYS_HEAP_USAGE_FW_PREMAP_PT   (1U <<PVRSRV_PHYS_HEAP_FW_PREMAP_PT)
+#define PHYS_HEAP_USAGE_FW_PREMAP      (1U <<PVRSRV_PHYS_HEAP_FW_PREMAP0)
+#define PHYS_HEAP_USAGE_WRAP           (1U <<PVRSRV_PHYS_HEAP_WRAP)
+#define PHYS_HEAP_USAGE_DISPLAY        (1U <<PVRSRV_PHYS_HEAP_DISPLAY)
+#define PHYS_HEAP_USAGE_DLM            (1U <<29)
+#define PHYS_HEAP_USAGE_FW_SHARED      (1U <<30)
+#define PHYS_HEAP_USAGE_FW_PRIVATE     (1U <<31)
+
+#define PHYS_HEAP_USAGE_MASK (PHYS_HEAP_USAGE_CPU_LOCAL    | \
+                              PHYS_HEAP_USAGE_GPU_LOCAL    | \
+                              PHYS_HEAP_USAGE_GPU_PRIVATE  | \
+                              PHYS_HEAP_USAGE_EXTERNAL     | \
+                              PHYS_HEAP_USAGE_GPU_COHERENT | \
+                              PHYS_HEAP_USAGE_GPU_SECURE   | \
+                              PHYS_HEAP_USAGE_FW_CODE      | \
+                              PHYS_HEAP_USAGE_FW_PRIV_DATA | \
+                              PHYS_HEAP_USAGE_FW_SHARED    | \
+                              PHYS_HEAP_USAGE_FW_PRIVATE   | \
+                              PHYS_HEAP_USAGE_FW_PREMAP    | \
+                              PHYS_HEAP_USAGE_FW_PREMAP_PT | \
+                              PHYS_HEAP_USAGE_WRAP         | \
+                              PHYS_HEAP_USAGE_DISPLAY      | \
+                              PHYS_HEAP_USAGE_DLM)
+
+#define FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM    PVRSRV_PHYS_HEAP_FW_CODE
+
+typedef void (*CpuPAddrToDevPAddr)(IMG_HANDLE hPrivData,
+                                   IMG_UINT32 ui32NumOfAddr,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_CPU_PHYADDR *psCpuPAddr);
+
+typedef void (*DevPAddrToCpuPAddr)(IMG_HANDLE hPrivData,
+                                   IMG_UINT32 ui32NumOfAddr,
+                                   IMG_CPU_PHYADDR *psCpuPAddr,
+                                   IMG_DEV_PHYADDR *psDevPAddr);
+
+/*! Structure used to hold function pointers used for run-time physical address
+ * translation by Services. Gives flexibility to allow the CPU and GPU to see
+ * the same pool of physical RAM and different physical bus addresses.
+ * Both fields must be valid functions even if the conversion is simple.
+ */
+typedef struct _PHYS_HEAP_FUNCTIONS_
+{
+	/*! Translate CPU physical address to device physical address */
+	CpuPAddrToDevPAddr	pfnCpuPAddrToDevPAddr;
+	/*! Translate device physical address to CPU physical address */
+	DevPAddrToCpuPAddr	pfnDevPAddrToCpuPAddr;
+} PHYS_HEAP_FUNCTIONS;
+
+/*! Structure used to describe the IPA settings for a physical heap.
+ */
+typedef struct _IPA_CONFIG_
+{
+	/*! IPA Policy Default (0-7) */
+	IMG_UINT8             ui8IPAPolicyDefault;
+	/*! IPA Policy Mask (3-bits for this use-case - 0x7) */
+	IMG_UINT8             ui8IPAPolicyMask;
+	/*! IPA Policy mask shift (LSB position - policy value will be << by this amount */
+	IMG_UINT8             ui8IPAPolicyShift;
+	/*! IPA Policy reserved byte for alignment and slop reduction */
+	IMG_UINT8             ui8IPAPolicyReserved;
+} IPA_CONFIG;
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+/* A Shared Physical Address Space (SPAS) region represents a group of physheaps
+ * that have this SPAS property. If 2 physheaps on separate devices are linked with
+ * the same SPAS region, the devices would be able to share one another's memory.
+ *
+ * The region is a dllist of physheaps.
+ * If a heap is in the list, it's in the region. */
+typedef struct _PHYS_HEAP_SPAS_REGION_ {
+	/* The head of the physheap list.
+	 * May be empty. */
+	DLLIST_NODE sListHead;
+	/* Protects the list. */
+	ATOMIC_T ui32Lock;
+} PHYS_HEAP_SPAS_REGION;
+
+#define DECLARE_PHYS_HEAP_SPAS_REGION(name)            \
+PHYS_HEAP_SPAS_REGION name = {                         \
+	.ui32Lock = { 0 },                                 \
+	.sListHead = { &name.sListHead, &name.sListHead }  \
+}
+#endif
+
+typedef struct _PHYS_HEAP_CONFIG_LMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+													 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_CPU_PHYADDR       sStartAddr;           /*!< CPU Physical base address of memory region */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Device physical base address of memory
+													 region as seen from the PoV of the GPU */
+	IMG_UINT64            uiSize;               /*!< Size of memory region in bytes */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+													 psMemFuncs */
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	PHYS_HEAP_SPAS_REGION *psSpasRegion;        /*!< Physheaps with the same SpasRegion are seen
+	                                                 to share the physical address space. */
+#endif
+} PHYS_HEAP_CONFIG_LMA;
+
+typedef struct _PHYS_HEAP_CONFIG_UMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+													 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Optional Device physical base address of memory
+													 region as seen from the PoV of the GPU */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+													 psMemFuncs */
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	PHYS_HEAP_SPAS_REGION *psSpasRegion;        /*!< Physheaps with the same SpasRegion are seen
+	                                                 to share the physical address space. */
+#endif
+} PHYS_HEAP_CONFIG_UMA;
+
+typedef struct _PHYS_HEAP_CONFIG_DLM_
+{
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_UINT32            ui32Log2PMBSize;      /*!< PMB (Physical Memory Block) Log 2 size in bytes
+	                                                 for DLM heap. */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CPU_PHYADDR       sStartAddr;           /*!< CPU Physical base address of memory region */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Device physical base address of memory
+	                                                 region as seen from the PoV of the GPU */
+	IMG_UINT64            uiSize;               /*!< Size of memory region in bytes */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+                                                     psMemFuncs */
+} PHYS_HEAP_CONFIG_DLM;
+
+typedef struct _PHYS_HEAP_CONFIG_IMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+	                                                 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+	                                                 psMemFuncs */
+	IMG_UINT32            ui32PMBStartingMultiple; /*!< Multiple of PMB size defined in DLM heap
+	                                                    to be allocated on creation */
+	IMG_UINT32            uiDLMHeapIdx;         /*!< The index in the array of physheaps to a
+	                                                 DLM heap to import PMBs from. */
+} PHYS_HEAP_CONFIG_IMA;
+
+/*! Structure used to describe a physical Heap supported by a system. A
+ * system layer module can declare multiple physical heaps for different
+ * purposes. At a minimum a system must provide one physical heap tagged for
+ * PHYS_HEAP_USAGE_GPU_LOCAL use.
+ * A heap represents a discrete pool of physical memory and how it is managed,
+ * as well as associating other properties and address translation logic.
+ * The structure fields sStartAddr, sCardBase and uiSize must be given valid
+ * values for LMA and DMA physical heaps types.
+ */
+typedef struct _PHYS_HEAP_CONFIG_
+{
+	PHYS_HEAP_TYPE        eType;                /*!< Class of heap and PMR factory used */
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;       /*!< Supported uses flags, conveys the type of
+	                                                 buffers the physical heap can be used for */
+	IPA_CONFIG            sIPAConfig;           /*!< IPA configuration to be applied to all
+	                                                 requested physical addresses when physically
+	                                                 backed */
+	union {
+		PHYS_HEAP_CONFIG_LMA sLMA;
+		PHYS_HEAP_CONFIG_LMA sDMA;
+		PHYS_HEAP_CONFIG_UMA sUMA;
+		PHYS_HEAP_CONFIG_DLM sDLM;
+		PHYS_HEAP_CONFIG_IMA sIMA;
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+		PHYS_HEAP_CONFIG_UMA sWRAP;
+#endif
+	} uConfig;
+} PHYS_HEAP_CONFIG;
+
+/* Code throughout the driver sometimes relies on the fact that the backing for
+ * LMA and DMA heaps are the same struct, this check is not comprehensive as the structs
+ * could have changed but kept the same overall size, although this is unlikely.
+ */
+static_assert(sizeof(((PHYS_HEAP_CONFIG*)0)->uConfig.sLMA) == sizeof(((PHYS_HEAP_CONFIG*)0)->uConfig.sDMA),
+              "PHYS_HEAP_CONFIG sizeof sLMA != sDMA, Type has potentially changed");
+
+static INLINE IMG_UINT64 PhysHeapConfigGetSize(PHYS_HEAP_CONFIG *psConfig)
+{
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.uiSize;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no size member");
+		return 0;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.uiSize;
+#endif
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.uiSize;
+	case PHYS_HEAP_TYPE_UMA:
+		PVR_ASSERT(!"UMA Config has no size member");
+		return 0;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return 0;
+	}
+}
+
+static INLINE IMG_CPU_PHYADDR PhysHeapConfigGetStartAddr(PHYS_HEAP_CONFIG *psConfig)
+{
+	IMG_CPU_PHYADDR sUnsupportedPhyAddr = {0};
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.sStartAddr;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no StartAddr member");
+		return sUnsupportedPhyAddr;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.sStartAddr;
+#endif
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.sStartAddr;
+	case PHYS_HEAP_TYPE_UMA:
+		PVR_ASSERT(!"UMA Config has no StartAddr member");
+		return sUnsupportedPhyAddr;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return sUnsupportedPhyAddr;
+	}
+}
+
+static INLINE IMG_DEV_PHYADDR PhysHeapConfigGetCardBase(PHYS_HEAP_CONFIG *psConfig)
+{
+	IMG_DEV_PHYADDR sUnsupportedPhyAddr = {0};
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.sCardBase;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no CardBase member");
+		return sUnsupportedPhyAddr;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.sCardBase;
+#endif
+	case PHYS_HEAP_TYPE_UMA:
+		return psConfig->uConfig.sUMA.sCardBase;
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.sCardBase;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return sUnsupportedPhyAddr;
+	}
+}
+
+#define PHYS_HEAP_NAME_SIZE 24
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pvr_bridge.h b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_bridge.h
new file mode 100644
index 000000000000..dafc51dd57f0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_bridge.h
@@ -0,0 +1,443 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the PVR Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BRIDGE_H
+#define PVR_BRIDGE_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "pvrsrv_error.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "common_dc_bridge.h"
+#if defined(SUPPORT_DCPLAT_BRIDGE)
+#include "common_dcplat_bridge.h"
+#endif
+#endif
+#include "common_mm_bridge.h"
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+#include "common_mmplat_bridge.h"
+#endif
+#if defined(SUPPORT_WRAP_EXTMEM)
+#include "common_mmextmem_bridge.h"
+#endif
+#if !defined(EXCLUDE_CMM_BRIDGE)
+#include "common_cmm_bridge.h"
+#endif
+#if defined(__linux__)
+#include "common_dmabuf_bridge.h"
+#endif
+#if defined(PDUMP)
+#include "common_pdump_bridge.h"
+#include "common_pdumpctrl_bridge.h"
+#include "common_pdumpmm_bridge.h"
+#endif
+#include "common_cache_bridge.h"
+#if defined(SUPPORT_DMA_TRANSFER)
+#include "common_dma_bridge.h"
+#endif
+#include "common_srvcore_bridge.h"
+#include "common_sync_bridge.h"
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "common_smm_bridge.h"
+#endif
+#if defined(PVRSRV_ENABLE_HTB)
+#include "common_htbuffer_bridge.h"
+#endif
+#include "common_pvrtl_bridge.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "common_ri_bridge.h"
+#endif
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+#include "common_validation_bridge.h"
+#endif
+
+
+#include "common_devicememhistory_bridge.h"
+#include "common_synctracking_bridge.h"
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#include "common_syncfallback_bridge.h"
+#endif
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+#include "common_di_bridge.h"
+#endif
+
+/*
+ * Bridge Cmd Ids
+ */
+
+
+/* Note: The pattern
+ *   #define PVRSRV_BRIDGE_FEATURE (PVRSRV_BRIDGE_PREVFEATURE + 1)
+ *   #if defined(SUPPORT_FEATURE)
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST (PVRSRV_BRIDGE_PREVFEATURE_DISPATCH_LAST + 1)
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST  (PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST + PVRSRV_BRIDGE_FEATURE_CMD_LAST)
+ *   #else
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST 0
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST  (PVRSRV_BRIDGE_PREVFEATURE_DISPATCH_LAST)
+ *   #endif
+ * is used in the macro definitions below to make PVRSRV_BRIDGE_FEATURE_*
+ * take up no space in the dispatch table if SUPPORT_FEATURE is disabled.
+ *
+ * Note however that a bridge always defines PVRSRV_BRIDGE_FEATURE, even where
+ * the feature is not enabled (each bridge group retains its own ioctl number).
+ */
+
+#define PVRSRV_BRIDGE_FIRST					0UL
+
+/*   0: Default handler */
+#define PVRSRV_BRIDGE_DEFAULT				0UL
+#define PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST 0UL
+#define PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST  (PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST)
+/*   1: CORE functions */
+#define PVRSRV_BRIDGE_SRVCORE				1UL
+#define PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST (PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST+1)
+#define PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST  (PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST + PVRSRV_BRIDGE_SRVCORE_CMD_LAST)
+
+/*   2: SYNC functions */
+#define PVRSRV_BRIDGE_SYNC					2UL
+#define PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST (PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNC_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNC_CMD_LAST)
+
+/* 3,4: Reserved */
+#define PVRSRV_BRIDGE_RESERVED1				3UL
+#define PVRSRV_BRIDGE_RESERVED1_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST)
+
+#define PVRSRV_BRIDGE_RESERVED2				4UL
+#define PVRSRV_BRIDGE_RESERVED2_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RESERVED2_DISPATCH_LAST  (PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST)
+
+/*   5: PDUMP CTRL layer functions */
+#define PVRSRV_BRIDGE_PDUMPCTRL				5UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST)
+#endif
+
+/*   6: Memory Management functions */
+#define PVRSRV_BRIDGE_MM					6UL
+#define PVRSRV_BRIDGE_MM_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MM_DISPATCH_LAST  (PVRSRV_BRIDGE_MM_DISPATCH_FIRST + PVRSRV_BRIDGE_MM_CMD_LAST)
+
+/*   7: Non-Linux Memory Management functions */
+#define PVRSRV_BRIDGE_MMPLAT				7UL
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST (PVRSRV_BRIDGE_MM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST + PVRSRV_BRIDGE_MMPLAT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_MM_DISPATCH_LAST)
+#endif
+
+/*   8: Context Memory Management functions */
+#define PVRSRV_BRIDGE_CMM					8UL
+#if !defined(EXCLUDE_CMM_BRIDGE)
+#define PVRSRV_BRIDGE_CMM_DISPATCH_FIRST (PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_CMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CMM_DISPATCH_FIRST + PVRSRV_BRIDGE_CMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_CMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_CMM_DISPATCH_LAST  (PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST)
+#endif
+
+/*   9: PDUMP Memory Management functions */
+#define PVRSRV_BRIDGE_PDUMPMM				9UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST (PVRSRV_BRIDGE_CMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMPMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CMM_DISPATCH_LAST)
+#endif
+
+/*   10: PDUMP functions */
+#define PVRSRV_BRIDGE_PDUMP					10UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMP_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST)
+#endif
+
+/*  11: DMABUF functions */
+#define PVRSRV_BRIDGE_DMABUF				11UL
+#if defined(__linux__)
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST  (PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST + PVRSRV_BRIDGE_DMABUF_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST)
+#endif
+
+/*  12: Display Class functions */
+#define PVRSRV_BRIDGE_DC					12UL
+#if defined(SUPPORT_DISPLAY_CLASS)
+#define PVRSRV_BRIDGE_DC_DISPATCH_FIRST (PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DC_DISPATCH_LAST  (PVRSRV_BRIDGE_DC_DISPATCH_FIRST + PVRSRV_BRIDGE_DC_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DC_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DC_DISPATCH_LAST  (PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST)
+#endif
+
+/*  13: Cache interface functions */
+#define PVRSRV_BRIDGE_CACHE					13UL
+#define PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST (PVRSRV_BRIDGE_DC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_CACHE_DISPATCH_LAST  (PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST + PVRSRV_BRIDGE_CACHE_CMD_LAST)
+
+/*  14: Secure Memory Management functions */
+#define PVRSRV_BRIDGE_SMM					14UL
+#if defined(SUPPORT_SECURE_EXPORT)
+#define PVRSRV_BRIDGE_SMM_DISPATCH_FIRST (PVRSRV_BRIDGE_CACHE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SMM_DISPATCH_LAST  (PVRSRV_BRIDGE_SMM_DISPATCH_FIRST + PVRSRV_BRIDGE_SMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_SMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_SMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CACHE_DISPATCH_LAST)
+#endif
+
+/*  15: Transport Layer interface functions */
+#define PVRSRV_BRIDGE_PVRTL					15UL
+#define PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST (PVRSRV_BRIDGE_SMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST  (PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST + PVRSRV_BRIDGE_PVRTL_CMD_LAST)
+
+/*  16: Resource Information (RI) interface functions */
+#define PVRSRV_BRIDGE_RI					16UL
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#define PVRSRV_BRIDGE_RI_DISPATCH_FIRST (PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RI_DISPATCH_LAST  (PVRSRV_BRIDGE_RI_DISPATCH_FIRST + PVRSRV_BRIDGE_RI_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RI_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RI_DISPATCH_LAST  (PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST)
+#endif
+
+/*  17: Validation interface functions */
+#define PVRSRV_BRIDGE_VALIDATION			17UL
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST (PVRSRV_BRIDGE_RI_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST  (PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST + PVRSRV_BRIDGE_VALIDATION_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST  (PVRSRV_BRIDGE_RI_DISPATCH_LAST)
+#endif
+
+/*  18: TUTILS interface functions */
+#define PVRSRV_BRIDGE_TUTILS				18UL
+#define PVRSRV_BRIDGE_TUTILS_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST  (PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST)
+
+/*  19: DevMem history interface functions */
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY		19UL
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST (PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST  (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST + PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST)
+
+/*  20: Host Trace Buffer interface functions */
+#define PVRSRV_BRIDGE_HTBUFFER				20UL
+#if defined(PVRSRV_ENABLE_HTB)
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST  (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST + PVRSRV_BRIDGE_HTBUFFER_CMD_LAST)
+#else	/* !PVRSRV_ENABLE_HTB */
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST  (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST)
+#endif	/* PVRSRV_ENABLE_HTB */
+
+/*  21: Non-Linux Display functions */
+#define PVRSRV_BRIDGE_DCPLAT				21UL
+#if defined(SUPPORT_DISPLAY_CLASS) && defined(SUPPORT_DCPLAT_BRIDGE)
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST + PVRSRV_BRIDGE_DCPLAT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST)
+#endif
+
+/*  22: Extmem functions */
+#define PVRSRV_BRIDGE_MMEXTMEM				22UL
+#if defined(SUPPORT_WRAP_EXTMEM)
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST (PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST  (PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST + PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST (PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST)
+#endif
+
+/*  23: Sync tracking functions */
+#define PVRSRV_BRIDGE_SYNCTRACKING			23UL
+#define PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST (PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST)
+
+/*  24: Sync fallback functions */
+#define PVRSRV_BRIDGE_SYNCFALLBACK			24UL
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST)
+#endif
+
+/*  25: Debug Information (DI) interface functions */
+#define PVRSRV_BRIDGE_DI					25UL
+#if defined(SUPPORT_DI_BRG_IMPL)
+#define PVRSRV_BRIDGE_DI_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DI_DISPATCH_LAST  (PVRSRV_BRIDGE_DI_DISPATCH_FIRST + PVRSRV_BRIDGE_DI_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DI_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DI_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST)
+#endif
+
+/*  26: DMA transfer functions */
+
+#define PVRSRV_BRIDGE_DMA			26UL
+#if defined(SUPPORT_DMA_TRANSFER)
+#define PVRSRV_BRIDGE_DMA_DISPATCH_FIRST (PVRSRV_BRIDGE_DI_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DMA_DISPATCH_LAST  (PVRSRV_BRIDGE_DMA_DISPATCH_FIRST + PVRSRV_BRIDGE_DMA_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DMA_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DMA_DISPATCH_LAST  (PVRSRV_BRIDGE_DI_DISPATCH_LAST)
+#endif
+
+/* NB PVRSRV_BRIDGE_LAST below must be the last bridge group defined above (PVRSRV_BRIDGE_FEATURE) */
+#define PVRSRV_BRIDGE_LAST					(PVRSRV_BRIDGE_DMA)
+/* NB PVRSRV_BRIDGE_DISPATCH LAST below must be the last dispatch entry defined above (PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST) */
+#define PVRSRV_BRIDGE_DISPATCH_LAST			(PVRSRV_BRIDGE_DMA_DISPATCH_LAST)
+
+/* bit mask representing the enabled PVR bridges */
+
+static const IMG_UINT32 gui32PVRBridges =
+	  (1U << (PVRSRV_BRIDGE_DEFAULT - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_SRVCORE - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_SYNC - PVRSRV_BRIDGE_FIRST))
+
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_PDUMPCTRL - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_MM - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_MMPLAT - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_CMM)
+	| (1U << (PVRSRV_BRIDGE_CMM - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_PDUMPMM - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_PDUMP - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(__linux__)
+	| (1U << (PVRSRV_BRIDGE_DMABUF - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+	| (1U << (PVRSRV_BRIDGE_DC - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_CACHE - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_SECURE_EXPORT)
+	| (1U << (PVRSRV_BRIDGE_SMM - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_PVRTL - PVRSRV_BRIDGE_FIRST))
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	| (1U << (PVRSRV_BRIDGE_RI - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_DEVICEMEMHISTORY - PVRSRV_BRIDGE_FIRST))
+#if defined(PVRSRV_ENABLE_HTB)
+	| (1U << (PVRSRV_BRIDGE_HTBUFFER - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS) && defined(SUPPORT_DCPLAT_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_DCPLAT - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_WRAP_EXTMEM)
+	| (1U << (PVRSRV_BRIDGE_MMEXTMEM - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_SYNCTRACKING - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	| (1U << (PVRSRV_BRIDGE_SYNCFALLBACK - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DI_BRG_IMPL)
+	| (1U << (PVRSRV_BRIDGE_DI - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DMA_TRANSFER)
+	| (1U << (PVRSRV_BRIDGE_DMA - PVRSRV_BRIDGE_FIRST))
+#endif
+	;
+
+/* bit field representing which PVR bridge groups may optionally not
+ * be present in the server
+ */
+#define PVR_BRIDGES_OPTIONAL \
+	( \
+		(1U << (PVRSRV_BRIDGE_RI - PVRSRV_BRIDGE_FIRST)) \
+	)
+
+/******************************************************************************
+ * Generic bridge structures
+ *****************************************************************************/
+
+
+/******************************************************************************
+ * bridge packaging structure
+ *****************************************************************************/
+typedef struct PVRSRV_BRIDGE_PACKAGE_TAG
+{
+	IMG_UINT32		ui32BridgeID;			/*!< ioctl bridge group */
+	IMG_UINT32		ui32FunctionID;			/*!< ioctl function index */
+	IMG_UINT32		ui32Size;				/*!< size of structure */
+	void __user		*pvParamIn;				/*!< input data buffer */
+	IMG_UINT32		ui32InBufferSize;		/*!< size of input data buffer */
+	void __user		*pvParamOut;			/*!< output data buffer */
+	IMG_UINT32		ui32OutBufferSize;		/*!< size of output data buffer */
+}PVRSRV_BRIDGE_PACKAGE;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_BRIDGE_H */
+
+/******************************************************************************
+ End of file (pvr_bridge.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pvr_dicommon.h b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_dicommon.h
new file mode 100644
index 000000000000..c729dde2fab6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_dicommon.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Debug Information (DI) common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Debug Information (DI) common types and definitions included
+                in both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DICOMMON_H
+#define PVR_DICOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*! Maximum DI entry path length including the null byte. */
+#define DI_IMPL_BRG_PATH_LEN 64
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_DICOMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pvr_ricommon.h b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_ricommon.h
new file mode 100644
index 000000000000..0521aa151794
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_ricommon.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Resource Information (RI) common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) common types and definitions included
+                in both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_RICOMMON_H
+#define PVR_RICOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+/*! Maximum text string length including the null byte */
+#define PRVSRVRI_MAX_TEXT_LENGTH	20U
+
+/* PID used to hold PMR allocations which are driver-wide (i.e. have a lifetime
+ * longer than an application process)
+ */
+#define PVR_SYS_ALLOC_PID 1
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_RICOMMON_H */
+/******************************************************************************
+ End of file (pvr_ricommon.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_bridge.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_bridge.h
new file mode 100644
index 000000000000..5168f6f23ea5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_bridge.h
@@ -0,0 +1,250 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the Rogue Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_BRIDGE_H
+#define RGX_BRIDGE_H
+
+#include "pvr_bridge.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define RGXFWINITPARAMS_VERSION   1
+#define RGXFWINITPARAMS_EXTENSION 128
+
+#include "common_rgxta3d_bridge.h"
+#include "common_rgxcmp_bridge.h"
+#if defined(SUPPORT_FASTRENDER_DM)
+#include "common_rgxtq2_bridge.h"
+#endif
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+#include "common_rgxtq_bridge.h"
+#endif
+#if defined(SUPPORT_USC_BREAKPOINT)
+#include "common_rgxbreakpoint_bridge.h"
+#endif
+#include "common_rgxfwdbg_bridge.h"
+#if defined(PDUMP)
+#include "common_rgxpdump_bridge.h"
+#endif
+#include "common_rgxhwperf_bridge.h"
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+#include "common_rgxregconfig_bridge.h"
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "common_rgxkicksync_bridge.h"
+#endif
+#include "common_rgxtimerquery_bridge.h"
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+#include "common_rgxray_bridge.h"
+#endif
+/*
+ * Bridge Cmd Ids
+ */
+
+/* *REMEMBER* to update PVRSRV_BRIDGE_RGX_LAST if you add/remove a bridge
+ * group!
+ * Also you need to ensure all PVRSRV_BRIDGE_RGX_xxx_DISPATCH_FIRST offsets
+ * follow on from the previous bridge group's commands!
+ *
+ * If a bridge group is optional, ensure you *ALWAYS* define its index
+ * (e.g. PVRSRV_BRIDGE_RGXCMP is always 151, even is the feature is not
+ * defined). If an optional bridge group is not defined you must still
+ * define PVRSRV_BRIDGE_RGX_xxx_DISPATCH_FIRST for it with an assigned
+ * value of 0.
+ */
+
+/* The RGX bridge groups start at 128 (PVRSRV_BRIDGE_RGX_FIRST) rather than
+ * follow-on from the other non-device bridge groups (meaning that they then
+ * won't be displaced if other non-device bridge groups are added).
+ */
+
+#define PVRSRV_BRIDGE_RGX_FIRST                  128UL
+
+/* 128: RGX TQ interface functions */
+#define PVRSRV_BRIDGE_RGXTQ                      128UL
+/* The RGXTQ bridge is conditional since the definitions in this header file
+ * support both the rogue and volcanic servers, but the RGXTQ bridge is not
+ * required at all on the Volcanic architecture.
+ */
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST       (PVRSRV_BRIDGE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST        (PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTQ_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST       0
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST        (PVRSRV_BRIDGE_DISPATCH_LAST)
+#endif
+
+/* 129: RGX Compute interface functions */
+#define PVRSRV_BRIDGE_RGXCMP                     129UL
+#define PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST      (PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXCMP_CMD_LAST)
+
+/* 130: RGX TA/3D interface functions */
+#define PVRSRV_BRIDGE_RGXTA3D                    130UL
+#define PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST     (PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST      (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTA3D_CMD_LAST)
+
+/* 131: RGX Breakpoint interface functions */
+#define PVRSRV_BRIDGE_RGXBREAKPOINT                 131UL
+#if defined(SUPPORT_USC_BREAKPOINT)
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST  0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST)
+#endif
+
+/* 132: RGX Debug/Misc interface functions */
+#define PVRSRV_BRIDGE_RGXFWDBG                   132UL
+#define PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST    (PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST)
+
+/* 133: RGX PDump interface functions */
+#define PVRSRV_BRIDGE_RGXPDUMP                   133UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST    (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST    0
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST)
+#endif
+
+/* 134: RGX HWPerf interface functions */
+#define PVRSRV_BRIDGE_RGXHWPERF                  134UL
+#define PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST   (PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST)
+
+/* 135: RGX Register Configuration interface functions */
+#define PVRSRV_BRIDGE_RGXREGCONFIG                  135UL
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST   (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST   0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST)
+#endif
+
+/* 136: RGX kicksync interface */
+#define PVRSRV_BRIDGE_RGXKICKSYNC                136UL
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST  (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST   0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST)
+#endif
+/* 137: RGX TQ2 interface */
+#define PVRSRV_BRIDGE_RGXTQ2                     137UL
+#if defined(SUPPORT_FASTRENDER_DM)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST      (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTQ2_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST      (0)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST)
+#endif
+
+/* 138: RGX timer query interface */
+#define PVRSRV_BRIDGE_RGXTIMERQUERY                 138UL
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST)
+
+/* 139: RGX Ray tracing interface */
+#define PVRSRV_BRIDGE_RGXRAY                 139UL
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXRAY_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST  0
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST)
+#endif
+
+#define PVRSRV_BRIDGE_RGX_LAST                   (PVRSRV_BRIDGE_RGXRAY)
+#define PVRSRV_BRIDGE_RGX_DISPATCH_LAST          (PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST)
+
+/* bit mask representing the enabled RGX bridges */
+
+static const IMG_UINT32 gui32RGXBridges =
+	  (1U << (PVRSRV_BRIDGE_RGXTQ - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(RGX_FEATURE_COMPUTE) || defined(__KERNEL__)
+	| (1U << (PVRSRV_BRIDGE_RGXCMP - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXTA3D - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_BREAKPOINT)
+	| (1U << (PVRSRV_BRIDGE_BREAKPOINT - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXFWDBG - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_RGXPDUMP - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_RGXKICKSYNC - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXHWPERF - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_REGCONFIG)
+	| (1U << (PVRSRV_BRIDGE_RGXREGCONFIG - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXKICKSYNC - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_FASTRENDER_DM) || defined(__KERNEL__)
+	| (1U << (PVRSRV_BRIDGE_RGXTQ2 - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+#if defined(SUPPORT_TIMERQUERY)
+	| (1U << (PVRSRV_BRIDGE_RGXTIMERQUERY - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXRAY - PVRSRV_BRIDGE_RGX_FIRST))
+	;
+/* bit field representing which RGX bridge groups may optionally not
+ * be present in the server
+ */
+
+#define RGX_BRIDGES_OPTIONAL \
+	( \
+		0 /* no RGX bridges are currently optional */ \
+	)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_fw_info.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_fw_info.h
new file mode 100644
index 000000000000..fb31ac440024
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_fw_info.h
@@ -0,0 +1,144 @@
+/*************************************************************************/ /*!
+@File
+@Title          FW image information
+
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FW_INFO_H)
+#define RGX_FW_INFO_H
+
+#include "img_types.h"
+#include "rgx_common.h"
+
+/*
+ * Firmware binary block unit in bytes.
+ * Raw data stored in FW binary will be aligned to this size.
+ */
+#define FW_BLOCK_SIZE 4096L
+
+typedef enum
+{
+	META_CODE = 0,
+	META_PRIVATE_DATA,
+	META_COREMEM_CODE,
+	META_COREMEM_DATA,
+	MIPS_CODE,
+	MIPS_EXCEPTIONS_CODE,
+	MIPS_BOOT_CODE,
+	MIPS_PRIVATE_DATA,
+	MIPS_BOOT_DATA,
+	MIPS_STACK,
+	RISCV_UNCACHED_CODE,
+	RISCV_CACHED_CODE,
+	RISCV_PRIVATE_DATA,
+	RISCV_COREMEM_CODE,
+	RISCV_COREMEM_DATA,
+} RGX_FW_SECTION_ID;
+
+typedef enum
+{
+	NONE = 0,
+	FW_CODE,
+	FW_DATA,
+	FW_COREMEM_CODE,
+	FW_COREMEM_DATA
+} RGX_FW_SECTION_TYPE;
+
+
+/*
+ * FW binary format with FW info attached:
+ *
+ *          Contents        Offset
+ *     +-----------------+
+ *     |                 |    0
+ *     |                 |
+ *     | Original binary |
+ *     |      file       |
+ *     |   (.ldr/.elf)   |
+ *     |                 |
+ *     |                 |
+ *     +-----------------+
+ *     | FW info header  |  FILE_SIZE - 4K
+ *     +-----------------+
+ *     |                 |
+ *     | FW layout table |
+ *     |                 |
+ *     +-----------------+
+ *                          FILE_SIZE
+ */
+
+#define FW_INFO_VERSION  (2)
+
+/* Firmware is built for open source driver and uses open source version numbering */
+#define FW_INFO_FLAGS_OPEN_SOURCE (1U)
+
+typedef struct
+{
+	/* FW_INFO_VERSION 1 */
+	IMG_UINT32 ui32InfoVersion;      /* FW info version */
+	IMG_UINT32 ui32HeaderLen;        /* Header length */
+	IMG_UINT32 ui32LayoutEntryNum;   /* Number of entries in the layout table */
+	IMG_UINT32 ui32LayoutEntrySize;  /* Size of an entry in the layout table */
+	IMG_UINT64 RGXFW_ALIGN ui64BVNC; /* BVNC */
+	IMG_UINT32 ui32FwPageSize;       /* Page size of processor on which firmware executes */
+	IMG_UINT32 ui32Flags;            /* Compatibility flags */
+
+	/* FW_INFO_VERSION 2 */
+	IMG_UINT16 ui16PVRVersionMajor;  /* DDK major version number */
+	IMG_UINT16 ui16PVRVersionMinor;  /* DDK minor version number */
+	IMG_UINT32 ui32PVRVersionBuild;  /* DDK build number */
+} RGX_FW_INFO_HEADER;
+
+typedef struct
+{
+	RGX_FW_SECTION_ID eId;
+	RGX_FW_SECTION_TYPE eType;
+	IMG_UINT32 ui32BaseAddr;
+	IMG_UINT32 ui32MaxSize;
+	IMG_UINT32 ui32AllocSize;
+	IMG_UINT32 ui32AllocOffset;
+} RGX_FW_LAYOUT_ENTRY;
+
+#endif /* RGX_FW_INFO_H */
+
+/******************************************************************************
+ End of file (rgx_fw_info.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_memallocflags.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_memallocflags.h
new file mode 100644
index 000000000000..e26f42c4f935
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_memallocflags.h
@@ -0,0 +1,58 @@
+/**************************************************************************/ /*!
+@File
+@Title          RGX device specific memory allocation flags
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_MEMALLOCFLAGS_H
+#define RGX_MEMALLOCFLAGS_H
+
+
+/* Include pvrsrv layer header as the flags below are used in the device
+ * field defined in this header inside Services code.
+ * See PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK */
+#include "pvrsrv_memallocflags.h"
+
+
+/* Device specific MMU flags */
+#define PMMETA_PROTECT          (1U << 0)      /*!< Memory that only the PM and Meta can access */
+#define FIRMWARE_CACHED         (1U << 1)      /*!< Memory that is cached in META/MIPS */
+
+
+#endif /* RGX_MEMALLOCFLAGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_pdump_panics.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_pdump_panics.h
new file mode 100644
index 000000000000..fce2b3efab69
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_pdump_panics.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump panic definitions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX PDump panic definitions header
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_PDUMP_PANICS_H_)
+#define RGX_PDUMP_PANICS_H_
+
+/*! Unique device specific IMG_UINT16 panic IDs to identify the cause of an
+ * RGX PDump panic in a PDump script. */
+typedef enum
+{
+	RGX_PDUMP_PANIC_UNDEFINED = 0,
+
+	/* These panics occur when test parameters and driver configuration
+	 * enable features that require the firmware and host driver to
+	 * communicate. Such features are not supported with off-line playback.
+	 */
+	RGX_PDUMP_PANIC_ZSBUFFER_BACKING         = 101, /*!< Requests ZSBuffer to be backed with physical pages */
+	RGX_PDUMP_PANIC_ZSBUFFER_UNBACKING       = 102, /*!< Requests ZSBuffer to be unbacked */
+	RGX_PDUMP_PANIC_FREELIST_GROW            = 103, /*!< Requests an on-demand freelist grow/shrink */
+	RGX_PDUMP_PANIC_FREELISTS_RECONSTRUCTION = 104, /*!< Requests freelists reconstruction */
+	RGX_PDUMP_PANIC_SPARSEMEM_SWAP           = 105, /*!< Requests sparse remap memory swap feature */
+} RGX_PDUMP_PANIC;
+
+#endif /* RGX_PDUMP_PANICS_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_tq_shared.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_tq_shared.h
new file mode 100644
index 000000000000..2db19224c441
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_tq_shared.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX transfer queue shared
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared definitions between client and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_TQ_SHARED_H
+#define RGX_TQ_SHARED_H
+
+#define TQ_MAX_PREPARES_PER_SUBMIT		16U
+
+#define TQ_PREP_FLAGS_COMMAND_3D		0x0U
+#define TQ_PREP_FLAGS_COMMAND_2D		0x1U
+#define TQ_PREP_FLAGS_COMMAND_MASK		(0xfU)
+#define TQ_PREP_FLAGS_COMMAND_SHIFT		0
+#define TQ_PREP_FLAGS_PDUMPCONTINUOUS	(1U << 4)
+
+#define TQ_PREP_FLAGS_COMMAND_SET(m) \
+	((TQ_PREP_FLAGS_COMMAND_##m << TQ_PREP_FLAGS_COMMAND_SHIFT) & TQ_PREP_FLAGS_COMMAND_MASK)
+
+#define TQ_PREP_FLAGS_COMMAND_IS(m,n) \
+	(((m & TQ_PREP_FLAGS_COMMAND_MASK) >> TQ_PREP_FLAGS_COMMAND_SHIFT)  == TQ_PREP_FLAGS_COMMAND_##n)
+
+#endif /* RGX_TQ_SHARED_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgxtransfer_shader.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgxtransfer_shader.h
new file mode 100644
index 000000000000..f261d2d04075
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgxtransfer_shader.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File           rgxtransfer_shader.h
+@Title          TQ binary shader file info
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header holds info about TQ binary shader file generated
+                by the TQ shader factory. This header is need by shader factory
+                when generating the file; by services KM when reading and
+                loading the file into memory; and by services UM when
+                constructing blits using the shaders.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSHADERHEADER_H)
+#define RGXSHADERHEADER_H
+
+#include "pvrversion.h"
+
+typedef struct _RGX_SHADER_HEADER_
+{
+	IMG_UINT32 ui32Version;
+	IMG_UINT32 ui32NumFragment;
+	IMG_UINT32 ui32SizeFragment;
+	IMG_UINT32 ui32SizeClientMem;
+} RGX_SHADER_HEADER;
+
+/* TQ shaders version is used to check compatibility between the
+   binary TQ shaders file and the DDK. This number should be incremented
+   if a change to the TQ shader factory breaks compatibility. */
+#define RGX_TQ_SHADERS_VERSION 2U
+
+#define RGX_TQ_SHADERS_VERSION_PACK \
+	(((RGX_TQ_SHADERS_VERSION & 0xFFU) << 16) | ((PVRVERSION_MAJ & 0xFFU) << 8) | ((PVRVERSION_MIN & 0xFFU) << 0))
+
+#endif /* RGXSHADERHEADER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rogue/km_apphint_defs.h b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/km_apphint_defs.h
new file mode 100644
index 000000000000..f2ffad0030cd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/km_apphint_defs.h
@@ -0,0 +1,162 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "km_apphint_defs_common.h"
+
+#ifndef KM_APPHINT_DEFS_H
+#define KM_APPHINT_DEFS_H
+
+/* NB: The 'DEVICE' AppHints must be last in this list as they will be
+ * duplicated in the case of a driver supporting multiple devices
+ */
+#define APPHINT_LIST_ALL \
+	APPHINT_LIST_BUILDVAR_COMMON \
+	APPHINT_LIST_BUILDVAR \
+	APPHINT_LIST_MODPARAM_COMMON \
+	APPHINT_LIST_MODPARAM \
+	APPHINT_LIST_DEBUGINFO_COMMON \
+	APPHINT_LIST_DEBUGINFO \
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+	APPHINT_LIST_DEBUGINFO_DEVICE
+
+
+/*
+*******************************************************************************
+ Build variables (rogue-specific)
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR
+
+/*
+*******************************************************************************
+ Module parameters (rogue-specific)
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+X(EnableCDMKillingRandMode,         BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE,         NO_PARAM_TABLE,  ALWAYS   ) \
+\
+X(HWPerfDisableCustomCounterFilter, BOOL,           VALIDATION,  PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER, NO_PARAM_TABLE,  ALWAYS   ) \
+X(ValidateSOCUSCTimer,              BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATESOCUSCTIMERS,             NO_PARAM_TABLE,  ALWAYS   ) \
+X(ECCRAMErrInj,                     UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+\
+X(TFBCCompressionControlGroup,      UINT32,         VALIDATION,  PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP,      NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlScheme,     UINT32,         VALIDATION,  PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME,     NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlYUVFormat,  BOOL,           VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlLossyMinChannel, BOOL,      VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCVersionDowngrade,             UINT32,         ALWAYS,      PVRSRV_APPHINT_TFBCVERSION,                      NO_PARAM_TABLE,  ALWAYS   ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (rogue-specific) - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+
+/*
+*******************************************************************************
+ Debugfs parameters (rogue-specific) - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters. Each entry in this table will provide a default value to all
+ devices (i.e. if there is more than one device each device's value will
+ be initialised).
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT \
+/* debuginfo device apphint name  modparam name */
+
+/*
+*******************************************************************************
+
+ Table generated enums
+
+******************************************************************************/
+/* Unique ID for all AppHints */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_ID_ ## a,
+	APPHINT_LIST_ALL
+#undef X
+	APPHINT_ID_MAX
+} APPHINT_ID;
+
+/* ID for build variable Apphints - used for build variable only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_BUILDVAR_ID_ ## a,
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+	APPHINT_BUILDVAR_ID_MAX
+} APPHINT_BUILDVAR_ID;
+
+/* ID for Modparam Apphints - used for modparam only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_MODPARAM_ID_ ## a,
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+	APPHINT_MODPARAM_ID_MAX
+} APPHINT_MODPARAM_ID;
+
+/* ID for Debugfs Apphints - used for debugfs only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+	APPHINT_DEBUGINFO_ID_MAX
+} APPHINT_DEBUGINFO_ID;
+
+/* ID for Debugfs Device Apphints - used for debugfs device only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_DEVICE_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+	APPHINT_DEBUGINFO_DEVICE_ID_MAX
+} APPHINT_DEBUGINFO_DEVICE_ID;
+
+#endif /* KM_APPHINT_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rogue/rgxapi_km.h b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/rgxapi_km.h
new file mode 100644
index 000000000000..7d7efd0afb5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/rgxapi_km.h
@@ -0,0 +1,322 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX API Header kernel mode
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported RGX API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXAPI_KM_H
+#define RGXAPI_KM_H
+
+#include "rgx_hwperf.h"
+
+/******************************************************************************
+ * RGX HW Performance Profiling Control API(s)
+ *****************************************************************************/
+
+/*! HWPerf device identification structure */
+typedef struct _RGX_HWPERF_DEVICE_
+{
+	IMG_CHAR pszName[20];	/*!< Helps identify this device uniquely */
+	IMG_HANDLE hDevData;	/*!< Handle for the server */
+
+	struct _RGX_HWPERF_DEVICE_ *psNext;	/*!< Next device if any */
+} RGX_HWPERF_DEVICE;
+
+/*! HWPerf connection structure */
+typedef struct
+{
+	RGX_HWPERF_DEVICE *psHWPerfDevList;	/*!< Pointer to list of devices */
+} RGX_HWPERF_CONNECTION;
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfLazyConnect
+@Description    Obtain a HWPerf connection object to the RGX device(s). The
+                 connections to devices are not actually opened until
+                 HWPerfOpen() is called.
+
+@Output         ppsHWPerfConnection Address of a HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfOpen
+@Description    Opens connection(s) to the RGX device(s). Valid handle to the
+                 connection object has to be provided which means the this
+                 function needs to be preceded by the call to
+                 RGXHWPerfLazyConnect() function.
+
+@Input          psHWPerfConnection HWPerf connection object
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION* psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConnect
+@Description    Obtain a connection object to the RGX HWPerf module. Allocated
+                 connection object(s) reference opened connection(s). Calling
+                 this function is an equivalent of calling RGXHWPerfLazyConnect
+                 and RGXHWPerfOpen. This connect should be used when the caller
+                 will be retrieving event data.
+
+@Output         ppsHWPerfConnection Address of HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfFreeConnection
+@Description    Frees the HWPerf connection object
+
+@Input          psHWPerfConnection Pointer to connection object as returned
+                                    from RGXHWPerfLazyConnect()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfClose
+@Description    Closes all the opened connection(s) to RGX device(s)
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object as
+                                    returned from RGXHWPerfConnect() or
+                                    RGXHWPerfOpen()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisconnect
+@Description    Disconnect from the RGX device
+
+@Input          ppsHWPerfConnection Pointer to HWPerf connection object as
+                                     returned from RGXHWPerfConnect() or
+                                     RGXHWPerfOpen(). Calling this function is
+                                     an equivalent of calling RGXHWPerfClose()
+                                     and RGXHWPerfFreeConnection().
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfControl
+@Description    Enable or disable the generation of RGX HWPerf event packets.
+                 See RGXCtrlHWPerf().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          eStreamId          ID of the HWPerf stream
+@Input          bToggle            Switch to toggle or apply mask.
+@Input          ui64Mask           Mask of events to control.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfGetFilter
+@Description    Reads HWPerf stream filter where stream is identified by the
+                 given stream ID.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ui64Filter   HWPerf filter value
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter
+);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigMuxCounters
+@Description    Enable and configure the performance counter block for one or
+                 more device layout modules.
+                 See RGXHWPerfConfigureAndEnableCustomCounters().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          asBlockConfigs     Address of the array of configuration blocks
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigMuxCounters(
+		RGX_HWPERF_CONNECTION         *psHWPerfConnection,
+		IMG_UINT32                     ui32NumBlocks,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK  *asBlockConfigs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigureAndEnableCustomCounters
+@Description    Enable and configure custom performance counters
+
+@Input          psHWPerfConnection    Pointer to HWPerf connection object
+@Input          ui16CustomBlockID     ID of the custom block to configure
+@Input          ui16NumCustomCounters Number of custom counters
+@Input          pui32CustomCounterIDs Pointer to array containing custom
+                                       counter IDs
+@Return         PVRSRV_ERROR          System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigureAndEnableCustomCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT16             ui16CustomBlockID,
+		IMG_UINT16             ui16NumCustomCounters,
+		IMG_UINT32            *pui32CustomCounterIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisableCounters
+@Description    Disable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from
+                                    the <tt>RGX_HWPERF_CNTBLK_ID</tt>
+                                    enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfEnableCounters
+@Description    Enable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from the
+                                    <tt>RGX_HWPERF_CNTBLK_ID</tt> enumeration.
+@Return         PVRSRV_ERROR  System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling Retrieval API(s)
+ *
+ * The client must ensure their use of this acquire/release API for a single
+ * connection/stream must not be shared with multiple execution contexts e.g.
+ * between a kernel thread and an ISR handler. It is the client's
+ * responsibility to ensure this API is not interrupted by a high priority
+ * thread/ISR
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfAcquireEvents
+@Description    When there is data available to read this call returns with OK
+                 and the address and length of the data buffer the client can
+                 safely read. This buffer may contain one or more event packets.
+                 When there is no data to read, this call returns with OK and
+                 sets *puiBufLen to 0 on exit.
+                 Clients must pair this call with a RGXHWPerfReleaseEvents()
+                 call.
+                 Data returned in ppBuf will be in the form of a sequence of
+                 HWPerf packets which should be traversed using the pointers,
+                 structures and macros provided by rgx_hwperf.h.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ppBuf        Address of a pointer to a byte buffer. On exit it
+                              contains the address of buffer to read from
+@Output         pui32BufLen  Pointer to an integer. On exit it is the size of
+                              the data to read from the buffer
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfReleaseEvents
+@Description    Called after client has read the event data out of the buffer
+                 retrieved from the Acquire Events call to release resources.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConvertCRTimeStamp
+@Description    Converts the timestamp given by FW events to the common OS
+                 timestamp. The first three inputs are obtained via a CLK_SYNC
+                 event, ui64CRTimeStamp is the CR timestamp from the FW event
+                 to be converted.
+
+@Input          ui32ClkSpeed        Clock speed given by sync event
+@Input          ui64CorrCRTimeStamp CR Timestamp given by sync event
+@Input          ui64CorrOSTimeStamp Correlating OS Timestamp given by sync
+                                     event
+@Input          ui64CRTimeStamp     CR Timestamp to convert
+@Return         IMG_UINT64          Calculated OS Timestamp
+*/ /**************************************************************************/
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp);
+
+#endif /* RGXAPI_KM_H */
+
+/******************************************************************************
+ End of file (rgxapi_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/sync_checkpoint_internal.h b/drivers/gpu/drm/img/img-volcanic/services/include/sync_checkpoint_internal.h
new file mode 100644
index 000000000000..2514ff3b6973
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/sync_checkpoint_internal.h
@@ -0,0 +1,273 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the internal server interface for services
+                synchronisation checkpoints.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_INTERNAL_H
+#define SYNC_CHECKPOINT_INTERNAL_H
+
+#include "img_types.h"
+#include "opaque_types.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "ra.h"
+#include "dllist.h"
+#include "lock.h"
+#include "devicemem.h"
+#include "rgx_fwif_shared.h"
+
+struct SYNC_CHECKPOINT_RECORD;
+
+/*
+	Private structures
+*/
+
+typedef struct _SYNC_CHECKPOINT_CONTEXT_CTL_ _SYNC_CHECKPOINT_CONTEXT_CTL, *_PSYNC_CHECKPOINT_CONTEXT_CTL;
+
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG _SYNC_CHECKPOINT_CONTEXT;
+
+typedef struct _SYNC_CHECKPOINT_BLOCK_
+{
+	ATOMIC_T                  hRefCount;                  /*!< Ref count for this sync block */
+	POS_LOCK                  hLock;
+	_SYNC_CHECKPOINT_CONTEXT  *psContext;                 /*!< Our copy of the services connection */
+	IMG_UINT32                ui32SyncBlockSize;          /*!< Size of the sync checkpoint block */
+	IMG_UINT32                ui32FirmwareAddr;           /*!< Firmware address */
+	DEVMEM_MEMDESC            *hMemDesc;                  /*!< DevMem allocation for block */
+	volatile IMG_UINT32       *pui32LinAddr;              /*!< Server-code CPU mapping */
+	RA_BASE_T                 uiSpanBase;                 /*!< Base of this import (FW DevMem) in the span RA */
+#if defined(PDUMP)
+	DLLIST_NODE               sListNode;                  /*!< List node for the sync chkpt blocks */
+#endif
+} SYNC_CHECKPOINT_BLOCK;
+
+typedef struct SYNC_CHECKPOINT_RECORD* PSYNC_CHECKPOINT_RECORD_HANDLE;
+
+typedef struct SYNC_CHECKPOINT_TAG
+{
+	/* A sync checkpoint is assigned a unique ID, to avoid any confusion should
+	 * the same memory be re-used later for a different checkpoint
+	 */
+	IMG_UINT32                      ui32UID;                /*!< Unique ID assigned to sync checkpoint (to distinguish checkpoints if memory is re-used)*/
+	ATOMIC_T                        hRefCount;              /*!< Ref count for this sync */
+	ATOMIC_T                        hEnqueuedCCBCount;      /*!< Num times sync has been put in CCBs */
+	SYNC_CHECKPOINT_BLOCK           *psSyncCheckpointBlock; /*!< Synchronisation block this checkpoint is allocated on */
+	RA_BASE_T                       uiAllocatedAddr;        /*!< Allocated address of the sync */
+	volatile SYNC_CHECKPOINT_FW_OBJ *psSyncCheckpointFwObj; /*!< CPU view of the data held in the sync block */
+	PRGXFWIF_UFO_ADDR               sCheckpointUFOAddr;     /*!< PRGXFWIF_UFO_ADDR struct used to pass update address to FW */
+	IMG_CHAR                        azName[PVRSRV_SYNC_NAME_LENGTH]; /*!< Name of the checkpoint */
+	PVRSRV_TIMELINE                 hTimeline;              /*!< Timeline on which this sync checkpoint was created */
+	IMG_PID                         uiProcess;              /*!< The Process ID of the process which created this sync checkpoint */
+	PSYNC_CHECKPOINT_RECORD_HANDLE  hRecord;                /*!< Sync record handle */
+	DLLIST_NODE                     sListNode;              /*!< List node for the global sync chkpt list */
+	DLLIST_NODE                     sDeferredFreeListNode;  /*!< List node for the deferred free sync chkpt list */
+	IMG_UINT32                      ui32FWAddr;             /*!< FWAddr stored at sync checkpoint alloc time */
+#if defined(PDUMP)
+	PDUMP_FLAGS_T                   ui32PDumpFlags;         /*!< Pdump Capture mode to be used for POL*/
+#endif
+#if defined(DEBUG)
+    IMG_UINT32                      ui32ValidationCheck;    /*!< Structure validity pattern */
+#endif
+} SYNC_CHECKPOINT;
+
+
+typedef struct _SYNC_CHECKPOINT_SIGNAL_
+{
+	SYNC_CHECKPOINT                asSyncCheckpoint;       /*!< Store sync checkpt for deferred signal */
+	IMG_UINT32                      ui32Status;             /*!< sync checkpt status signal/errored */
+} _SYNC_CHECKPOINT_DEFERRED_SIGNAL;
+
+#define GET_CP_CB_NEXT_IDX(_curridx) (((_curridx) + 1) % SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL)
+#define GET_CP_CB_BASE(_idx)   (IMG_OFFSET_ADDR(psDevNode->pui8DeferredSyncCPSignal, \
+                                                ((_idx) * sizeof(_SYNC_CHECKPOINT_DEFERRED_SIGNAL))))
+
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetFirmwareAddr
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the firmware address of
+
+@Return         The firmware address of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCCBEnqueued
+
+@Description    Increment the CCB enqueued reference count for a
+                synchronisation checkpoint. This indicates how many FW
+                operations (checks/update) have been placed into CCBs for the
+                sync checkpoint.
+                When the FW services these operation, it increments its own
+                reference count. When these two values are equal, we know
+                there are not outstanding FW operating for the checkpoint
+                in any CCB.
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint for which
+                                        to increment the enqueued reference
+                                        count
+
+@Return         None
+
+*/
+/*****************************************************************************/
+void
+SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetEnqueuedCount
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the enqueued count of
+
+@Return         The enqueued count of the sync checkpoint
+                (i.e. the number of FW operations (checks or updates)
+                 currently enqueued in CCBs for the sync checkpoint)
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetReferenceCount
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the reference count of
+
+@Return         The host reference count of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetCreator
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the creating process of
+
+@Return         The process id of the process which created this sync checkpoint.
+
+*/
+/*****************************************************************************/
+IMG_PID
+SyncCheckpointGetCreator(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetId
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the unique Id of
+
+@Return         The unique Id of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetTimeline
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the parent timeline of
+
+@Return         The parent timeline of the sync checkpoint
+
+*/
+/*****************************************************************************/
+PVRSRV_TIMELINE
+SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetRGXFWIFUFOAddr
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the PRGXFWIF_UFO_ADDR of
+
+@Return         The PRGXFWIF_UFO_ADDR of the sync checkpoint, used when
+                providing the update in server kick code.
+
+*/
+/*****************************************************************************/
+PRGXFWIF_UFO_ADDR*
+SyncCheckpointGetRGXFWIFUFOAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+#if !defined(SUPPORT_NATIVE_FENCE_SYNC)
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetAssociatedDevice
+
+@Description    .
+
+@Input          psSyncCheckpointContext Synchronisation Checkpoint context
+                                        to get the device node of
+
+@Return         The PVRSRV_DEVICE_NODE of the device on which the sync
+                checkpoint context was created.
+
+*/
+/*****************************************************************************/
+PPVRSRV_DEVICE_NODE
+SyncCheckpointGetAssociatedDevice(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+#endif /* !defined(SUPPORT_NATIVE_FENCE_SYNC) */
+
+#endif /* SYNC_CHECKPOINT_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/km_apphint_defs.h b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/km_apphint_defs.h
new file mode 100644
index 000000000000..81397088bddb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/km_apphint_defs.h
@@ -0,0 +1,187 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "km_apphint_defs_common.h"
+
+#ifndef KM_APPHINT_DEFS_H
+#define KM_APPHINT_DEFS_H
+
+/* NB: The 'DEVICE' AppHints must be last in this list as they will be
+ * duplicated in the case of a driver supporting multiple devices
+ */
+#define APPHINT_LIST_ALL \
+	APPHINT_LIST_BUILDVAR_COMMON \
+	APPHINT_LIST_BUILDVAR \
+	APPHINT_LIST_MODPARAM_COMMON \
+	APPHINT_LIST_MODPARAM \
+	APPHINT_LIST_DEBUGINFO_COMMON \
+	APPHINT_LIST_DEBUGINFO \
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+	APPHINT_LIST_DEBUGINFO_DEVICE
+
+
+/*
+*******************************************************************************
+ Build variables (volcanic-specific)
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR
+
+/*
+*******************************************************************************
+ Module parameters (volcanic-specific)
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM \
+/* name,                            type,           class,       default,                                         helper,          guest,         */ \
+X(FabricCoherencyOverride,          UINT32,         ALWAYS,      PVRSRV_APPHINT_FABRICCOHERENCYOVERRIDE,          NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(EnableGenericDMKillingRandMode,   BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLEDMKILLINGRANDMODE,          NO_PARAM_TABLE,  ALWAYS          ) \
+X(KillingCtl,                       UINT32,         VALIDATION,  PVRSRV_APPHINT_KILLINGCTL,                       NO_PARAM_TABLE,  ALWAYS          ) \
+X(CDMTDMKillingCtl,                 UINT32,         VALIDATION,  PVRSRV_APPHINT_CDMTDM_KILLINGCTL,                NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValEnableSPUPowerMaskChange,    BOOL,           VALIDATION,  PVRSRV_APPHINT_HWVALENABLESPUPOWERMASKCHANGE,    NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValAvailableSPUMask,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWVALAVAILABLESPUMASK,            NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValAvailableRACMask,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWVALAVAILABLERACMASK,            NO_PARAM_TABLE,  ALWAYS          ) \
+X(EnableSPUClockGating,             BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLESPUCLOCKGATING,             NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValTpuF20BilPrecision,          UINT32,         VALIDATION,  PVRSRV_APPHINT_HWVALTPUF20BILPRECISION,          NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(HWPerfDisableCounterFilter,       BOOL,           VALIDATION,  PVRSRV_APPHINT_HWPERFDISABLECOUNTERFILTER,       NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(ISPSchedulingLatencyMode,         UINT32,         ALWAYS,      PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE,         NO_PARAM_TABLE,  ALWAYS          ) \
+X(ValidateSOCUSCTimer,              BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATESOCUSCTIMERS,             NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(USRMNumRegionsVDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsCDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsDDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsPDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsTDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(UVBRMNumRegionsVDM,               UINT64,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(UVBRMNumRegionsDDM,               UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(GPUStatePin,                      BOOL,           VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(PowerDomainKickInterval,          UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(RCEDisableMask,                   UINT64,         VALIDATION,  PVRSRV_APPHINT_RCEDISABLEMASK,                   NO_PARAM_TABLE,  ALWAYS          ) \
+X(PCGPktDropThresh,                 UINT32,         VALIDATION,  PVRSRV_APPHINT_PCGPKTDROPTHRESH,                 NO_PARAM_TABLE,  ALWAYS          ) \
+X(RaySLCMMUAutoCacheOps,            UINT32,         VALIDATION,  PVRSRV_APPHINT_RAYSLCMMUAUTOCACHEOPS,            NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(ClkCtrl0,                         UINT64,         VALIDATION,  PVRSRV_APPHINT_CLKCTRL0,                         NO_PARAM_TABLE,  ALWAYS          ) \
+X(ClkCtrl1,                         UINT64,         VALIDATION,  PVRSRV_APPHINT_CLKCTRL1,                         NO_PARAM_TABLE,  ALWAYS          ) \
+X(ClkCtrl2,                         UINT32,         VALIDATION,  PVRSRV_APPHINT_CLKCTRL2,                         NO_PARAM_TABLE,  ALWAYS          ) \
+X(TPUTAGCtrl,                       UINT32,         VALIDATION,  0x10,                                            NO_PARAM_TABLE,  ALWAYS          ) \
+X(TPUMADDCtrl,                      UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (volcanic-specific) - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO \
+/* name,                            type,           class,       default,                                         helper,          guest,         */ \
+X(MMUVal_ParityFlipPMR,             STRING,         VALIDATION,  PVRSRV_APPHINT_MMUVAL_PARITYFLIPPMR,             NO_PARAM_TABLE,  ALWAYS          ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (volcanic-specific) - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE \
+/* name,                            type,           class,       default,                                         helper,          guest,         */ \
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters.
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT \
+/* debuginfo device apphint name  modparam name */
+
+/*
+*******************************************************************************
+
+ Table generated enums
+
+******************************************************************************/
+/* Unique ID for all AppHints */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_ID_ ## a,
+	APPHINT_LIST_ALL
+#undef X
+	APPHINT_ID_MAX
+} APPHINT_ID;
+
+/* ID for build variable Apphints - used for build variable only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_BUILDVAR_ID_ ## a,
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+	APPHINT_BUILDVAR_ID_MAX
+} APPHINT_BUILDVAR_ID;
+
+/* ID for Modparam Apphints - used for modparam only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_MODPARAM_ID_ ## a,
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+	APPHINT_MODPARAM_ID_MAX
+} APPHINT_MODPARAM_ID;
+
+/* ID for Debugfs Apphints - used for debugfs only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+	APPHINT_DEBUGINFO_ID_MAX
+} APPHINT_DEBUGINFO_ID;
+
+/* ID for Debugfs Device Apphints - used for debugfs device only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_DEVICE_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+	APPHINT_DEBUGINFO_DEVICE_ID_MAX
+} APPHINT_DEBUGINFO_DEVICE_ID;
+
+#endif /* KM_APPHINT_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/rgxapi_km.h b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/rgxapi_km.h
new file mode 100644
index 000000000000..487b3796a6a9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/rgxapi_km.h
@@ -0,0 +1,309 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX API Header kernel mode
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported RGX API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXAPI_KM_H
+#define RGXAPI_KM_H
+
+#include "rgx_hwperf.h"
+
+
+/******************************************************************************
+ * RGX HW Performance Profiling Control API(s)
+ *****************************************************************************/
+
+/*! HWPerf device identification structure */
+typedef struct _RGX_HWPERF_DEVICE_
+{
+	IMG_CHAR pszName[20];	/*!< Helps identify this device uniquely */
+	IMG_HANDLE hDevData;	/*!< Handle for the server */
+
+	struct _RGX_HWPERF_DEVICE_ *psNext;  /*!< Next device if any */
+} RGX_HWPERF_DEVICE;
+
+/*! HWPerf connection structure */
+typedef struct
+{
+	RGX_HWPERF_DEVICE *psHWPerfDevList;  /*!< pointer to list of devices */
+} RGX_HWPERF_CONNECTION;
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfLazyConnect
+@Description    Obtain a HWPerf connection object to the RGX device(s). The
+                connections to devices are not actually opened until
+                HWPerfOpen() is called.
+
+@Output         ppsHWPerfConnection Address of a HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfOpen
+@Description    Opens connection(s) to the RGX device(s). Valid handle to the
+                connection object has to be provided which means the this
+                function needs to be preceded by the call to
+                RGXHWPerfLazyConnect() function.
+
+@Input          psHWPerfConnection HWPerf connection object
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION* psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConnect
+@Description    Obtain a connection object to the RGX HWPerf module. Allocated
+                connection object(s) reference opened connection(s). Calling
+                this function is an equivalent of calling RGXHWPerfLazyConnect
+                and RGXHWPerfOpen. This connect should be used when the caller
+                will be retrieving event data.
+
+@Output         ppsHWPerfConnection Address of HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfFreeConnection
+@Description    Frees the HWPerf connection object
+
+@Input          psHWPerfConnection Pointer to connection object as returned
+                                    from RGXHWPerfLazyConnect()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfClose
+@Description    Closes all the opened connection(s) to RGX device(s)
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object as
+                                    returned from RGXHWPerfConnect() or
+                                    RGXHWPerfOpen()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisconnect
+@Description    Disconnect from the RGX device
+
+@Input          ppsHWPerfConnection Pointer to HWPerf connection object as
+                                     returned from RGXHWPerfConnect() or
+                                     RGXHWPerfOpen(). Calling this function is
+                                     an equivalent of calling RGXHWPerfClose()
+                                     and RGXHWPerfFreeConnection().
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfControl
+@Description    Enable or disable the generation of RGX HWPerf event packets.
+                 See RGXCtrlHWPerf().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          eStreamId          ID of the HWPerf stream
+@Input          bToggle            Switch to toggle or apply mask.
+@Input          ui64Mask           Mask of events to control.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfGetFilter
+@Description    Reads HWPerf stream filter where stream is identified by the
+                 given stream ID.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ui64Filter   HWPerf filter value
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter
+);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigureCounters
+@Description    Enable and configure the performance counter block for one or
+                 more device layout modules.
+                 See RGXConfigHWPerfCounters().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32CtrlWord       One of <tt>RGX_HWPERF_CTRL_NOP</tt>,
+                                    <tt>RGX_HWPERF_CTRL_GEOM_FULLrange</tt>,
+                                    <tt>RGX_HWPERF_CTRL_COMP_FULLRANGE</tt>,
+                                    <tt>RGX_HWPERF_CTRL_TDM_FULLRANGE</tt>
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          asBlockConfigs     Address of the array of configuration blocks
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigureCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32NumBlocks,
+		RGX_HWPERF_CONFIG_CNTBLK*  asBlockConfigs);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisableCounters
+@Description    Disable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from
+                                    the RGX_HWPERF_CNTBLK_ID enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfEnableCounters
+@Description    Enable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from the
+                                    <tt>RGX_HWPERF_CNTBLK_ID</tt> enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling Retrieval API(s)
+ *
+ * The client must ensure their use of this acquire/release API for a single
+ * connection/stream must not be shared with multiple execution contexts e.g.
+ * between a kernel thread and an ISR handler. It is the client's
+ * responsibility to ensure this API is not interrupted by a high priority
+ * thread/ISR
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfAcquireEvents
+@Description    When there is data available to read this call returns with OK
+                 and the address and length of the data buffer the client can
+                 safely read. This buffer may contain one or more event packets.
+                 When there is no data to read, this call returns with OK and
+                 sets *puiBufLen to 0 on exit.
+                 Clients must pair this call with a RGXHWPerfReleaseEvents()
+                 call.
+                 Data returned in ppBuf will be in the form of a sequence of
+                 HWPerf packets which should be traversed using the pointers,
+                 structures and macros provided in rgx_hwperf.h
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ppBuf        Address of a pointer to a byte buffer. On exit it
+                              contains the address of buffer to read from
+@Output         pui32BufLen  Pointer to an integer. On exit it is the size of
+                              the data to read from the buffer
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfReleaseEvents
+@Description    Called after client has read the event data out of the buffer
+                 retrieved from the Acquire Events call to release resources.
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConvertCRTimeStamp
+@Description    Converts the timestamp given by FW events to the common OS
+                 timestamp. The first three inputs are obtained via a CLK_SYNC
+                 event, ui64CRTimeStamp is the CR timestamp from the FW event
+                 to be converted.
+@Input          ui32ClkSpeed        Clock speed given by sync event
+@Input          ui64CorrCRTimeStamp CR Timestamp given by sync event
+@Input          ui64CorrOSTimeStamp Correlating OS Timestamp given by sync
+                                     event
+@Input          ui64CRTimeStamp     CR Timestamp to convert
+@Return         IMG_UINT64          Calculated OS Timestamp
+*/ /**************************************************************************/
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp);
+
+#endif /* RGXAPI_KM_H */
+
+/******************************************************************************
+ End of file (rgxapi_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/cache_km.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/cache_km.c
new file mode 100644
index 000000000000..f9c4ce986e18
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/cache_km.c
@@ -0,0 +1,1646 @@
+/*************************************************************************/ /*!
+@File           cache_km.c
+@Title          CPU d-cache maintenance operations framework
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements server side code for CPU d-cache maintenance taking
+                into account the idiosyncrasies of the various types of CPU
+                d-cache instruction-set architecture (ISA) maintenance
+                mechanisms.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <asm/current.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#endif
+
+#include "pmr.h"
+#include "log2.h"
+#include "device.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "cache_km.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "allocmem.h"
+#include "process_stats.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+#include "ri_server.h"
+#endif
+#include "devicemem.h"
+#include "pvrsrv_apphint.h"
+#include "pvrsrv_sync_server.h"
+#include "km_apphint_defs.h"
+#include "km_apphint_defs_common.h"
+#include "os_apphint.h"
+#include "di_server.h"
+
+/* This header must always be included last */
+#if defined(__linux__)
+#include "kernel_compatibility.h"
+#endif
+
+/* Top-level file-local build definitions */
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS) && defined(__linux__)
+#define CACHEOP_DEBUG
+#define CACHEOP_STATS_ITEMS_MAX				32
+#define INCR_WRAP(x)						((x+1) >= CACHEOP_STATS_ITEMS_MAX ? 0 : (x+1))
+#define DECR_WRAP(x)						((x-1) < 0 ? (CACHEOP_STATS_ITEMS_MAX-1) : (x-1))
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+/* Refer to CacheOpStatsExecLogHeader() for header item names */
+#define CACHEOP_RI_PRINTF_HEADER			"%-8s %-8s %-10s %-10s %-5s %-16s %-16s %-10s %-10s %-18s"
+#define CACHEOP_RI_PRINTF					"%-8d %-8d %-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-18llu\n"
+#else
+#define CACHEOP_PRINTF_HEADER				"%-8s %-8s %-10s %-10s %-5s %-10s %-10s %-18s"
+#define CACHEOP_PRINTF						"%-8d %-8d %-10s %-10s %-5s 0x%-8llx 0x%-8llx %-18llu\n"
+#endif
+#endif
+
+//#define CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING		/* Force OS page (not cache line) flush granularity */
+#define CACHEOP_PVR_ASSERT(x)							/* Define as PVR_ASSERT(x), enable for swdev & testing */
+#define CACHEOP_DEVMEM_OOR_ERROR_STRING		"cacheop device memory request is out of range"
+#define CACHEOP_MAX_DEBUG_MESSAGE_LEN		160
+
+typedef struct _CACHEOP_WORK_ITEM_
+{
+	PMR *psPMR;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_CACHE_OP uiCacheOp;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	PVRSRV_TIMELINE iTimeline;
+	SYNC_TIMELINE_OBJ sSWTimelineObj;
+	PVRSRV_DEVICE_NODE *psDevNode;
+#if defined(CACHEOP_DEBUG)
+	IMG_UINT64 ui64StartTime;
+	IMG_UINT64 ui64EndTime;
+	IMG_BOOL bKMReq;
+	IMG_PID pid;
+#endif
+} CACHEOP_WORK_ITEM;
+
+typedef struct _CACHEOP_STATS_EXEC_ITEM_
+{
+	IMG_UINT32 ui32DeviceID;
+	IMG_PID pid;
+	PVRSRV_CACHE_OP uiCacheOp;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT64 ui64StartTime;
+	IMG_UINT64 ui64EndTime;
+	IMG_BOOL bKMReq;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+#endif
+} CACHEOP_STATS_EXEC_ITEM;
+
+typedef enum _CACHEOP_CONFIG_
+{
+	CACHEOP_CONFIG_DEFAULT = 0,
+	/* cache flush mechanism types */
+	CACHEOP_CONFIG_URBF    = 4,
+	/* sw-emulated deferred flush mechanism */
+	CACHEOP_CONFIG_KDF     = 8,
+	/* pseudo configuration items */
+	CACHEOP_CONFIG_LAST    = 16,
+	CACHEOP_CONFIG_KLOG    = 16,
+	CACHEOP_CONFIG_ALL     = 31
+} CACHEOP_CONFIG;
+
+typedef struct _CACHEOP_WORK_QUEUE_
+{
+/*
+ * Init. state & primary device node framework
+ * is anchored on.
+ */
+	IMG_BOOL bInit;
+/*
+  MMU page size/shift & d-cache line size
+ */
+	size_t uiPageSize;
+	IMG_UINT32 uiLineSize;
+	IMG_UINT32 uiLineShift;
+	IMG_UINT32 uiPageShift;
+	PMR *psInfoPagePMR;
+	IMG_UINT32 *pui32InfoPage;
+
+#if defined(CACHEOP_DEBUG)
+/*
+  CacheOp statistics
+ */
+	DI_ENTRY *psDIEntry;
+	IMG_HANDLE hStatsExecLock;
+
+	IMG_UINT32 ui32ServerOps;
+	IMG_UINT32 ui32ClientOps;
+	IMG_UINT32 ui32TotalOps;
+	IMG_UINT32 ui32ServerOpUsedUMVA;
+	IMG_UINT32 ui32AvgExecTime;
+	IMG_UINT32 ui32AvgExecTimeRemainder;
+
+	IMG_INT32 i32StatsExecWriteIdx;
+	CACHEOP_STATS_EXEC_ITEM asStatsExecuted[CACHEOP_STATS_ITEMS_MAX];
+#endif
+
+	DI_ENTRY *psConfigTune;
+	IMG_HANDLE hConfigLock;
+	CACHEOP_CONFIG	eConfig;
+	IMG_UINT32		ui32Config;
+	IMG_BOOL		bSupportsUMFlush;
+} CACHEOP_WORK_QUEUE;
+
+/* Top-level CacheOp framework object */
+static CACHEOP_WORK_QUEUE gsCwq;
+
+#define CacheOpConfigSupports(e) ((gsCwq.eConfig & (e)) ? IMG_TRUE : IMG_FALSE)
+
+#if defined(CACHEOP_DEBUG)
+static INLINE void CacheOpStatsExecLogHeader(IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(szBuffer, CACHEOP_MAX_DEBUG_MESSAGE_LEN,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+				CACHEOP_RI_PRINTF_HEADER,
+#else
+				CACHEOP_PRINTF_HEADER,
+#endif
+				"DevID",
+				"Pid",
+				"CacheOp",
+				"Type",
+				"Origin",
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+				"DevVAddr",
+				"DevPAddr",
+#endif
+				"Offset",
+				"Size",
+				"xTime (us)");
+}
+
+static void CacheOpStatsExecLogWrite(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
+{
+	IMG_INT32 i32WriteOffset;
+	IMG_UINT32 ui32ExecTime;
+	printk("log write\n");
+	if (!psCacheOpWorkItem->uiCacheOp)
+	{
+		return;
+	}
+	else if (psCacheOpWorkItem->bKMReq && !CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
+	{
+		/* KM logs spams the history due to frequency, this removes it completely */
+		return;
+	}
+
+	OSLockAcquire(gsCwq.hStatsExecLock);
+
+	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
+	gsCwq.i32StatsExecWriteIdx = INCR_WRAP(gsCwq.i32StatsExecWriteIdx);
+	gsCwq.asStatsExecuted[i32WriteOffset].ui32DeviceID = psCacheOpWorkItem->psDevNode ? psCacheOpWorkItem->psDevNode->sDevId.ui32InternalID : -1;
+	gsCwq.asStatsExecuted[i32WriteOffset].pid = psCacheOpWorkItem->pid;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiSize = psCacheOpWorkItem->uiSize;
+	gsCwq.asStatsExecuted[i32WriteOffset].bKMReq = psCacheOpWorkItem->bKMReq;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiOffset	= psCacheOpWorkItem->uiOffset;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp = psCacheOpWorkItem->uiCacheOp;
+	gsCwq.asStatsExecuted[i32WriteOffset].ui64StartTime = psCacheOpWorkItem->ui64StartTime;
+	gsCwq.asStatsExecuted[i32WriteOffset].ui64EndTime = psCacheOpWorkItem->ui64EndTime;
+
+	CACHEOP_PVR_ASSERT(gsCwq.asStatsExecuted[i32WriteOffset].pid);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	if (gsCwq.bInit && psCacheOpWorkItem->psPMR)
+	{
+		IMG_CPU_PHYADDR sDevPAddr;
+		PVRSRV_ERROR eError, eLockError;
+		IMG_BOOL bValid;
+
+		/* Get more detailed information regarding the sub allocations that
+		   PMR has from RI manager for process that requested the CacheOp */
+		eError = RIDumpProcessListKM(psCacheOpWorkItem->psPMR,
+									 gsCwq.asStatsExecuted[i32WriteOffset].pid,
+									 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+									 &gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* (Re)lock here as some PMR might have not been locked */
+		eLockError = PMRLockSysPhysAddresses(psCacheOpWorkItem->psPMR);
+		PVR_GOTO_IF_ERROR(eLockError, e0);
+
+		eError = PMR_CpuPhysAddr(psCacheOpWorkItem->psPMR,
+								 gsCwq.uiPageShift,
+								 1,
+								 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+								 &sDevPAddr,
+								 &bValid,
+								 CPU_USE);
+
+		eLockError = PMRUnlockSysPhysAddresses(psCacheOpWorkItem->psPMR);
+		PVR_LOG_IF_ERROR(eLockError, "PMRUnlockSysPhysAddresses");
+
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+
+
+		gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr.uiAddr = sDevPAddr.uiAddr;
+	}
+#endif
+
+	/* Calculate the approximate cumulative moving average execution time.
+	 * This calculation is based on standard equation:
+	 *
+	 * CMAnext = (new + count * CMAprev) / (count + 1)
+	 *
+	 * but in simplified form:
+	 *
+	 * CMAnext = CMAprev + (new - CMAprev) / (count + 1)
+	 *
+	 * this gets rid of multiplication and prevents overflow.
+	 *
+	 * Also to increase accuracy that we lose with integer division,
+	 * we hold the moving remainder of the division and add it.
+	 *
+	 * CMAnext = CMAprev + (new - CMAprev + CMRprev) / (count + 1)
+	 *
+	 * Multiple tests proved it to be the best solution for approximating
+	 * CMA using integers.
+	 *
+	 */
+
+	ui32ExecTime =
+		gsCwq.asStatsExecuted[i32WriteOffset].ui64EndTime -
+		gsCwq.asStatsExecuted[i32WriteOffset].ui64StartTime;
+
+	{
+
+	IMG_INT32 i32Div =
+		(IMG_INT32) ui32ExecTime -
+		(IMG_INT32) gsCwq.ui32AvgExecTime +
+		(IMG_INT32) gsCwq.ui32AvgExecTimeRemainder;
+
+	gsCwq.ui32AvgExecTime += i32Div / (IMG_INT32)(gsCwq.ui32TotalOps + 1);
+	gsCwq.ui32AvgExecTimeRemainder = i32Div % (IMG_INT32)(gsCwq.ui32TotalOps + 1);
+
+	gsCwq.ui32TotalOps++;
+
+	}
+
+	if (!gsCwq.asStatsExecuted[i32WriteOffset].bKMReq)
+	{
+		/* This operation queues only UM CacheOp in per-PID process statistics database */
+		PVRSRVStatsUpdateCacheOpStats(
+						gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr,
+						gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr,
+#endif
+						gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+						gsCwq.asStatsExecuted[i32WriteOffset].uiSize,
+						ui32ExecTime,
+						!gsCwq.asStatsExecuted[i32WriteOffset].bKMReq,
+						psCacheOpWorkItem->pid);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+e0:
+#endif
+	OSLockRelease(gsCwq.hStatsExecLock);
+}
+
+static int CacheOpStatsExecLogRead(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	IMG_CHAR *pszFlushType;
+	IMG_CHAR *pszCacheOpType;
+	IMG_CHAR *pszFlushSource;
+	IMG_INT32 i32ReadOffset;
+	IMG_INT32 i32WriteOffset;
+
+	IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN] = {0};
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	OSLockAcquire(gsCwq.hStatsExecLock);
+
+	DIPrintf(psEntry,
+			"Primary CPU d-cache architecture: LSZ: 0x%x, URBF: %s\n",
+			gsCwq.uiLineSize,
+			gsCwq.bSupportsUMFlush ? "Yes" : "No");
+
+	DIPrintf(psEntry,
+			"Configuration: UKT: %d, URBF: %s\n",
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD],
+			gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No");
+
+	DIPrintf(psEntry,
+			"Summary: Total Ops [%d] - Server(using UMVA)/Client [%d(%d)/%d]. Avg execution time [%d]\n",
+			gsCwq.ui32TotalOps, gsCwq.ui32ServerOps, gsCwq.ui32ServerOpUsedUMVA, gsCwq.ui32ClientOps, gsCwq.ui32AvgExecTime);
+
+
+	CacheOpStatsExecLogHeader(szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
+	for (i32ReadOffset = DECR_WRAP(i32WriteOffset);
+		 i32ReadOffset != i32WriteOffset;
+		 i32ReadOffset = DECR_WRAP(i32ReadOffset))
+	{
+		IMG_UINT64 ui64ExecTime =
+			gsCwq.asStatsExecuted[i32ReadOffset].ui64EndTime -
+			gsCwq.asStatsExecuted[i32ReadOffset].ui64StartTime;
+
+		IMG_DEVMEM_SIZE_T ui64NumOfPages =
+			gsCwq.asStatsExecuted[i32ReadOffset].uiSize >> gsCwq.uiPageShift;
+
+
+		if (!gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
+		{
+			break;
+		}
+		if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			pszFlushType = "RBF.Fast";
+		}
+		else
+		{
+			pszFlushType = "RBF.Slow";
+		}
+
+		pszFlushSource = gsCwq.asStatsExecuted[i32ReadOffset].bKMReq ? " KM" : " UM";
+
+		switch (gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
+		{
+			case PVRSRV_CACHE_OP_NONE:
+				pszCacheOpType = "None";
+				break;
+			case PVRSRV_CACHE_OP_CLEAN:
+				pszCacheOpType = "Clean";
+				break;
+			case PVRSRV_CACHE_OP_INVALIDATE:
+				pszCacheOpType = "Invalidate";
+				break;
+			case PVRSRV_CACHE_OP_FLUSH:
+				pszCacheOpType = "Flush";
+				break;
+			case PVRSRV_CACHE_OP_TIMELINE:
+				pszCacheOpType = "Timeline";
+				pszFlushType = "      ";
+				break;
+			default:
+				pszCacheOpType = "Unknown";
+				break;
+		}
+
+		DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						CACHEOP_RI_PRINTF,
+#else
+						CACHEOP_PRINTF,
+#endif
+						gsCwq.asStatsExecuted[i32ReadOffset].ui32DeviceID,
+						gsCwq.asStatsExecuted[i32ReadOffset].pid,
+						pszCacheOpType,
+						pszFlushType,
+						pszFlushSource,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						gsCwq.asStatsExecuted[i32ReadOffset].sDevVAddr.uiAddr,
+						gsCwq.asStatsExecuted[i32ReadOffset].sDevPAddr.uiAddr,
+#endif
+						gsCwq.asStatsExecuted[i32ReadOffset].uiOffset,
+						gsCwq.asStatsExecuted[i32ReadOffset].uiSize,
+						ui64ExecTime);
+
+	}
+
+	OSLockRelease(gsCwq.hStatsExecLock);
+
+	return 0;
+}
+#endif /* defined(CACHEOP_DEBUG) */
+
+static INLINE void CacheOpStatsReset(void)
+{
+#if defined(CACHEOP_DEBUG)
+	gsCwq.ui32ServerOps = 0;
+	gsCwq.ui32ClientOps = 0;
+	gsCwq.ui32TotalOps = 0;
+	gsCwq.ui32ServerOpUsedUMVA = 0;
+	gsCwq.ui32AvgExecTime = 0;
+	gsCwq.ui32AvgExecTimeRemainder = 0;
+
+	gsCwq.i32StatsExecWriteIdx = 0;
+
+	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
+#endif
+}
+
+static void CacheOpConfigUpdate(IMG_UINT32 ui32Config)
+{
+	OSLockAcquire(gsCwq.hConfigLock);
+
+	/* Step 0, set the gsCwq.eConfig bits */
+	if (!(ui32Config & (CACHEOP_CONFIG_LAST - 1)))
+	{
+		gsCwq.eConfig = CACHEOP_CONFIG_KDF;
+		if (gsCwq.bSupportsUMFlush)
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
+		}
+	}
+	else
+	{
+		if (ui32Config & CACHEOP_CONFIG_KDF)
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_KDF;
+		}
+		else
+		{
+			gsCwq.eConfig &= ~CACHEOP_CONFIG_KDF;
+		}
+
+		if (gsCwq.bSupportsUMFlush && (ui32Config & CACHEOP_CONFIG_URBF))
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
+		}
+		else
+		{
+			gsCwq.eConfig &= ~CACHEOP_CONFIG_URBF;
+		}
+	}
+
+	if (ui32Config & CACHEOP_CONFIG_KLOG)
+	{
+		/* Suppress logs from KM caller */
+		gsCwq.eConfig |= CACHEOP_CONFIG_KLOG;
+	}
+	else
+	{
+		gsCwq.eConfig &= ~CACHEOP_CONFIG_KLOG;
+	}
+
+	/* Step 1, set gsCwq.ui32Config based on gsCwq.eConfig */
+	ui32Config = 0;
+
+	if (gsCwq.eConfig & CACHEOP_CONFIG_KDF)
+	{
+		ui32Config |= CACHEOP_CONFIG_KDF;
+	}
+	if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
+	{
+		ui32Config |= CACHEOP_CONFIG_URBF;
+	}
+	if (gsCwq.eConfig & CACHEOP_CONFIG_KLOG)
+	{
+		ui32Config |= CACHEOP_CONFIG_KLOG;
+	}
+	gsCwq.ui32Config = ui32Config;
+
+
+	/* Step 3, in certain cases where a CacheOp/VA is provided, this threshold determines at what point
+	   the optimisation due to the presence of said VA (i.e. us not having to remap the PMR pages in KM)
+	   is clawed-back because of the overhead of maintaining such large request which might stalls the
+	   user thread; so to hide this latency have these CacheOps executed on deferred CacheOp thread */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] = (IMG_UINT32)(PVR_DIRTY_BYTES_FLUSH_THRESHOLD >> 2);
+
+	/* Step 4, if no UM support, all requests are done in KM so zero these forcing all client requests
+	   to come down into the KM for maintenance */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = 0;
+
+	if (gsCwq.bSupportsUMFlush)
+	{
+		/* With URBF enabled we never go to the kernel */
+		if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
+		{
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = (IMG_UINT32)~0;
+		}
+	}
+
+	/* Step 5, reset stats. */
+	CacheOpStatsReset();
+
+	OSLockRelease(gsCwq.hConfigLock);
+}
+
+static int CacheOpConfigRead(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	DIPrintf(psEntry, "URBF: %s\n",
+		gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No");
+
+	return 0;
+}
+
+static INLINE PVRSRV_ERROR CacheOpConfigQuery(const PVRSRV_DEVICE_NODE *psDevNode,
+											const void *psPrivate,
+											IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	switch (ui32ID)
+	{
+		case APPHINT_ID_CacheOpConfig:
+			*pui32Value = gsCwq.ui32Config;
+			break;
+
+		case APPHINT_ID_CacheOpUMKMThresholdSize:
+			*pui32Value = gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD];
+			break;
+
+		default:
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR CacheOpConfigSet(const PVRSRV_DEVICE_NODE *psDevNode,
+											const void *psPrivate,
+											IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	switch (ui32ID)
+	{
+		case APPHINT_ID_CacheOpConfig:
+			CacheOpConfigUpdate(ui32Value & CACHEOP_CONFIG_ALL);
+			break;
+
+
+		case APPHINT_ID_CacheOpUMKMThresholdSize:
+		{
+			if (!ui32Value || !gsCwq.bSupportsUMFlush)
+			{
+				/* CPU ISA does not support UM flush, therefore every request goes down into
+				   the KM, silently ignore request to adjust threshold */
+				PVR_ASSERT(! gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD]);
+				break;
+			}
+			else if (ui32Value < gsCwq.uiPageSize)
+			{
+				/* Silently round-up to OS page size */
+				ui32Value = gsCwq.uiPageSize;
+			}
+
+			/* Align to OS page size */
+			ui32Value &= ~(gsCwq.uiPageSize - 1);
+
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = ui32Value;
+
+			break;
+		}
+
+		default:
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR CacheOpTimelineBind(PVRSRV_DEVICE_NODE *psDevNode,
+											   CACHEOP_WORK_ITEM *psCacheOpWorkItem,
+											   PVRSRV_TIMELINE iTimeline)
+{
+	PVRSRV_ERROR eError;
+
+	/* Always default the incoming CacheOp work-item to safe values */
+	SyncClearTimelineObj(&psCacheOpWorkItem->sSWTimelineObj);
+	psCacheOpWorkItem->iTimeline = PVRSRV_NO_TIMELINE;
+	psCacheOpWorkItem->psDevNode = psDevNode;
+	if (iTimeline == PVRSRV_NO_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+
+	psCacheOpWorkItem->iTimeline = iTimeline;
+	eError = SyncSWGetTimelineObj(iTimeline, &psCacheOpWorkItem->sSWTimelineObj);
+	PVR_LOG_IF_ERROR(eError, "SyncSWGetTimelineObj");
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR CacheOpTimelineExec(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
+{
+	PVRSRV_ERROR eError;
+
+	if (psCacheOpWorkItem->iTimeline == PVRSRV_NO_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->sSWTimelineObj.pvTlObj);
+
+	eError = SyncSWTimelineAdvanceKM(psCacheOpWorkItem->psDevNode,
+	                                 &psCacheOpWorkItem->sSWTimelineObj);
+	(void) SyncSWTimelineReleaseKM(&psCacheOpWorkItem->sSWTimelineObj);
+
+	return eError;
+}
+
+static INLINE void CacheOpExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
+										PVRSRV_CACHE_OP uiCacheOp,
+										IMG_BYTE *pbCpuVirtAddr,
+										IMG_CPU_PHYADDR sCpuPhyAddr,
+										IMG_DEVMEM_OFFSET_T uiPgAlignedOffset,
+										IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset,
+										IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset)
+{
+	IMG_BYTE *pbCpuVirtAddrEnd;
+	IMG_BYTE *pbCpuVirtAddrStart;
+	IMG_CPU_PHYADDR sCpuPhyAddrEnd;
+	IMG_CPU_PHYADDR sCpuPhyAddrStart;
+	IMG_DEVMEM_SIZE_T uiRelFlushSize;
+	IMG_DEVMEM_OFFSET_T uiRelFlushOffset;
+	IMG_DEVMEM_SIZE_T uiNextPgAlignedOffset;
+
+	/* These quantities allows us to perform cache operations
+	   at cache-line granularity thereby ensuring we do not
+	   perform more than is necessary */
+	CACHEOP_PVR_ASSERT(uiPgAlignedOffset < uiCLAlignedEndOffset);
+	uiRelFlushSize = (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
+	uiRelFlushOffset = 0;
+
+	if (uiCLAlignedStartOffset > uiPgAlignedOffset)
+	{
+		/* Zero unless initially starting at an in-page offset */
+		uiRelFlushOffset = uiCLAlignedStartOffset - uiPgAlignedOffset;
+		uiRelFlushSize -= uiRelFlushOffset;
+	}
+
+	/* uiRelFlushSize is gsCwq.uiPageSize unless current outstanding CacheOp
+	   size is smaller. The 1st case handles in-page CacheOp range and
+	   the 2nd case handles multiple-page CacheOp range with a last
+	   CacheOp size that is less than gsCwq.uiPageSize */
+	uiNextPgAlignedOffset = uiPgAlignedOffset + (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
+	if (uiNextPgAlignedOffset < uiPgAlignedOffset)
+	{
+		/* uiNextPgAlignedOffset is greater than uiCLAlignedEndOffset
+		   by implication of this wrap-round; this only happens when
+		   uiPgAlignedOffset is the last page aligned offset */
+		uiRelFlushSize = uiRelFlushOffset ?
+				uiCLAlignedEndOffset - uiCLAlignedStartOffset :
+				uiCLAlignedEndOffset - uiPgAlignedOffset;
+	}
+	else
+	{
+		if (uiNextPgAlignedOffset > uiCLAlignedEndOffset)
+		{
+			uiRelFlushSize = uiRelFlushOffset ?
+					uiCLAlignedEndOffset - uiCLAlignedStartOffset :
+					uiCLAlignedEndOffset - uiPgAlignedOffset;
+		}
+	}
+
+	/* More efficient to request cache maintenance operation for full
+	   relative range as opposed to multiple cache-aligned ranges */
+	sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + uiRelFlushOffset;
+	sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr + uiRelFlushSize;
+	if (pbCpuVirtAddr)
+	{
+		pbCpuVirtAddrStart = pbCpuVirtAddr + uiRelFlushOffset;
+		pbCpuVirtAddrEnd = pbCpuVirtAddrStart + uiRelFlushSize;
+	}
+	else
+	{
+		/* Some OS/Env layer support functions expect NULL(s) */
+		pbCpuVirtAddrStart = NULL;
+		pbCpuVirtAddrEnd = NULL;
+	}
+
+	/* Perform requested CacheOp on the CPU data cache for successive cache
+	   line worth of bytes up to page or in-page cache-line boundary */
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					__func__, uiCacheOp));
+			break;
+	}
+
+}
+
+static INLINE void CacheOpExecRangeBasedVA(PVRSRV_DEVICE_NODE *psDevNode,
+										 IMG_CPU_VIRTADDR pvAddress,
+										 IMG_DEVMEM_SIZE_T uiSize,
+										 PVRSRV_CACHE_OP uiCacheOp)
+{
+	IMG_CPU_PHYADDR sCpuPhyAddrUnused =
+		{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
+	IMG_BYTE *pbEnd = (IMG_BYTE*)((uintptr_t)pvAddress + (uintptr_t)uiSize);
+	IMG_BYTE *pbStart = (IMG_BYTE*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiLineSize-1));
+
+	/*
+	  If the start/end address isn't aligned to cache line size, round it up to the
+	  nearest multiple; this ensures that we flush all the cache lines affected by
+	  unaligned start/end addresses.
+	 */
+	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)gsCwq.uiLineSize);
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					 __func__, uiCacheOp));
+			break;
+	}
+
+}
+
+static INLINE PVRSRV_ERROR CacheOpValidateUMVA(PMR *psPMR,
+											   IMG_CPU_VIRTADDR pvAddress,
+											   IMG_DEVMEM_OFFSET_T uiOffset,
+											   IMG_DEVMEM_SIZE_T uiSize,
+											   PVRSRV_CACHE_OP uiCacheOp,
+											   void **ppvOutAddress)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(__linux__) && !defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma;
+#endif
+	void __user *pvAddr;
+
+	IMG_BOOL bReadOnlyInvalidate =
+		(uiCacheOp == PVRSRV_CACHE_OP_INVALIDATE) &&
+		!PVRSRV_CHECK_CPU_WRITEABLE(PMR_Flags(psPMR));
+
+	if (!pvAddress || bReadOnlyInvalidate)
+	{
+		/* As pvAddress is optional, NULL is expected from UM/KM requests */
+		/* Also don't allow invalidates for UMVA of read-only memory */
+		pvAddr = NULL;
+		goto e0;
+	}
+
+
+
+#if !defined(__linux__) || defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	pvAddr = NULL;
+#else
+	/* Validate VA, assume most basic address limit access_ok() check */
+	pvAddr = (void __user *)(uintptr_t)((uintptr_t)pvAddress + uiOffset);
+	if (!access_ok(pvAddr, uiSize))
+	{
+		pvAddr = NULL;
+		if (! mm)
+		{
+			/* Bad KM request, don't silently ignore */
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_CPU_ADDR, e0);
+		}
+	}
+	else if (mm)
+	{
+		mmap_read_lock(mm);
+		vma = find_vma(mm, (unsigned long)(uintptr_t)pvAddr);
+
+		if (!vma ||
+			vma->vm_start > (unsigned long)(uintptr_t)pvAddr ||
+			vma->vm_end < (unsigned long)(uintptr_t)pvAddr + uiSize ||
+			vma->vm_private_data != psPMR)
+		{
+			/*
+			 * Request range is not fully mapped or is not matching the PMR
+			 * Ignore request's VA.
+			 */
+			pvAddr = NULL;
+		}
+		mmap_read_unlock(mm);
+	}
+#endif
+
+e0:
+	*ppvOutAddress = (IMG_CPU_VIRTADDR __force) pvAddr;
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpPMRExec (PMR *psPMR,
+									IMG_CPU_VIRTADDR pvAddress,
+									IMG_DEVMEM_OFFSET_T uiOffset,
+									IMG_DEVMEM_SIZE_T uiSize,
+									PVRSRV_CACHE_OP uiCacheOp,
+									IMG_BOOL bIsRequestValidated)
+
+{
+	IMG_HANDLE hPrivOut = NULL;
+	IMG_BOOL bPMRIsSparse;
+	IMG_UINT32 ui32PageIndex;
+	IMG_UINT32 ui32NumOfPages;
+	size_t uiOutSize;	/* Effectively unused */
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_DEVMEM_SIZE_T uiPgAlignedSize;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedOffset;
+	IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedEndOffset;
+	IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedStartOffset;
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_CPU_PHYADDR asCpuPhyAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_CPU_PHYADDR *psCpuPhyAddr = asCpuPhyAddr;
+	IMG_BOOL bIsPMRInfoValid = IMG_FALSE;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BYTE *pbCpuVirtAddr = NULL;
+	IMG_BOOL *pbValid = abValid;
+	OS_CACHE_OP_ADDR_TYPE eCacheOpAddrType;
+
+	psDevNode = PMR_DeviceNode(psPMR);
+	eCacheOpAddrType = OSCPUCacheOpAddressType(psDevNode);
+
+	if (uiCacheOp == PVRSRV_CACHE_OP_NONE || uiCacheOp == PVRSRV_CACHE_OP_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (! bIsRequestValidated)
+	{
+		/* Need to validate parameters before proceeding */
+		/* Check for size + offset overflow */
+		PVR_LOG_RETURN_IF_FALSE(((uiOffset + uiSize) >= uiSize),
+		                        "Overflow detected on offset + size parameters",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		/* Since size + offset is later aligned to page size check for overflow with alignment */
+		PVR_LOG_RETURN_IF_FALSE((((uiOffset + uiSize) + gsCwq.uiPageSize - 1) >= (uiOffset + uiSize)),
+		                        "Overflow detected on offset + size parameters with applied alignment",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		PVR_LOG_RETURN_IF_FALSE(((uiOffset+uiSize) <= PMR_PhysicalSize(psPMR)),
+		                        CACHEOP_DEVMEM_OOR_ERROR_STRING,
+		                        PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PMRLockSysPhysAddresses");
+	}
+
+	/* Fast track the request if a CPU VA is provided and CPU ISA supports VA only maintenance */
+	eError = CacheOpValidateUMVA(psPMR, pvAddress, uiOffset, uiSize, uiCacheOp, (void**)&pbCpuVirtAddr);
+	if (eError == PVRSRV_OK)
+	{
+		pvAddress = pbCpuVirtAddr;
+
+		if (pvAddress && eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pvAddress, uiSize, uiCacheOp);
+
+			if (!bIsRequestValidated)
+			{
+				eError = PMRUnlockSysPhysAddresses(psPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+			}
+#if defined(CACHEOP_DEBUG)
+			gsCwq.ui32ServerOpUsedUMVA += 1;
+#endif
+			return PVRSRV_OK;
+		}
+		else if (pvAddress)
+		{
+			/* Round down the incoming VA (if any) down to the nearest page aligned VA */
+			pvAddress = (void*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1));
+#if defined(CACHEOP_DEBUG)
+			gsCwq.ui32ServerOpUsedUMVA += 1;
+#endif
+		}
+	}
+	else
+	{
+		/*
+		 * This validation pathway has been added to accommodate any/all requests that might
+		 * cause the kernel to Oops; essentially, KM requests should prevalidate cache maint.
+		 * parameters but if this fails then we would rather fail gracefully than cause the
+		 * kernel to Oops so instead we log the fact that an invalid KM virtual address was
+		 * supplied and what action was taken to mitigate against kernel Oops(ing) if any.
+		 */
+		CACHEOP_PVR_ASSERT(pbCpuVirtAddr == NULL);
+
+		if (eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, using paddress",
+					__func__,
+					pvAddress));
+
+			/* We can still proceed as kernel/cpu uses CPU PA for d-cache maintenance */
+			pvAddress = NULL;
+		}
+		else
+		{
+			/*
+			 * The approach here is to attempt a reacquisition of the PMR kernel VA and see if
+			 * said VA corresponds to the parameter VA, if so fail requested cache maint. op.
+			 * cause this indicates some kind of internal, memory and/or meta-data corruption
+			 * else we reissue the request using this (re)acquired alias PMR kernel VA.
+			 */
+			if (PMR_IsSparse(psPMR))
+			{
+				eError = PMRAcquireSparseKernelMappingData(psPMR,
+														   0,
+														   gsCwq.uiPageSize,
+														   (void **)&pbCpuVirtAddr,
+														   &uiOutSize,
+														   &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", e0);
+			}
+			else
+			{
+				eError = PMRAcquireKernelMappingData(psPMR,
+													 0,
+													 gsCwq.uiPageSize,
+													 (void **)&pbCpuVirtAddr,
+													 &uiOutSize,
+													 &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", e0);
+			}
+
+			/* Here, we only compare these CPU virtual addresses at granularity of the OS page size */
+			if ((uintptr_t)pbCpuVirtAddr == ((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, no alt. so failing request",
+						__func__,
+						pvAddress));
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_GOTO_WITH_ERROR("PMRReleaseKernelMappingData", eError, PVRSRV_ERROR_INVALID_CPU_ADDR, e0);
+			}
+			else if (eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s: Bad vaddress 0x%p in CPU d-cache maint. op, using reacquired vaddress 0x%p",
+						__func__,
+						pvAddress,
+						pbCpuVirtAddr));
+
+				/* Note that this might still fail if there is kernel memory/meta-data corruption;
+				   there is not much we can do here but at the least we will be informed of this
+				   before the kernel Oops(ing) */
+				CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pbCpuVirtAddr, uiSize, uiCacheOp);
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+				eError = PVRSRV_OK;
+				goto e0;
+			}
+			else
+			{
+				/* At this junction, we have exhausted every possible work-around possible but we do
+				   know that VA reacquisition returned another/alias page-aligned VA; so with this
+				   future expectation of PMRAcquireKernelMappingData(), we proceed */
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s: Bad vaddress %p in CPU d-cache maint. op, will use reacquired vaddress",
+						__func__,
+						pvAddress));
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+				/* NULL this to force per-page reacquisition down-stream */
+				pvAddress = NULL;
+			}
+		}
+	}
+
+	/* NULL clobbered var., OK to proceed */
+	pbCpuVirtAddr = NULL;
+	eError = PVRSRV_OK;
+
+	/* Need this for kernel mapping */
+	bPMRIsSparse = PMR_IsSparse(psPMR);
+
+	/* Round the incoming offset down to the nearest cache-line / page aligned-address */
+	uiCLAlignedEndOffset = uiOffset + uiSize;
+	uiCLAlignedEndOffset = PVR_ALIGN(uiCLAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiLineSize);
+	uiCLAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiLineSize-1));
+
+	uiPgAlignedEndOffset = uiCLAlignedEndOffset;
+	uiPgAlignedEndOffset = PVR_ALIGN(uiPgAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize);
+	uiPgAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiPageSize-1));
+	uiPgAlignedSize = uiPgAlignedEndOffset - uiPgAlignedStartOffset;
+
+#if defined(CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING)
+	/* For internal debug if cache-line optimised
+	   flushing is suspected of causing data corruption */
+	uiCLAlignedStartOffset = uiPgAlignedStartOffset;
+	uiCLAlignedEndOffset = uiPgAlignedEndOffset;
+#endif
+
+	/* Type of allocation backing the PMR data */
+	ui32NumOfPages = uiPgAlignedSize >> gsCwq.uiPageShift;
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		/* The pbValid array is allocated first as it is needed in
+		   both physical/virtual cache maintenance methods */
+		pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+		if (! pbValid)
+		{
+			pbValid = abValid;
+		}
+		else if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			psCpuPhyAddr = OSAllocZMem(ui32NumOfPages * sizeof(IMG_CPU_PHYADDR));
+			if (! psCpuPhyAddr)
+			{
+				psCpuPhyAddr = asCpuPhyAddr;
+				OSFreeMem(pbValid);
+				pbValid = abValid;
+			}
+		}
+	}
+
+	/* Lock the PMR while we are obtaining and using phys addrs */
+	PMRLockPMR(psPMR);
+
+	/* We always retrieve PMR data in bulk, up-front if number of pages is within
+	   PMR_MAX_TRANSLATION_STACK_ALLOC limits else we check to ensure that a
+	   dynamic buffer has been allocated to satisfy requests outside limits */
+	if (ui32NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC || pbValid != abValid)
+	{
+		if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			/* Look-up PMR CpuPhyAddr once, if possible */
+			eError = PMR_CpuPhysAddr(psPMR,
+									 gsCwq.uiPageShift,
+									 ui32NumOfPages,
+									 uiPgAlignedStartOffset,
+									 psCpuPhyAddr,
+									 pbValid,
+									 CPU_USE | MAPPING_USE);
+			if (eError == PVRSRV_OK)
+			{
+				bIsPMRInfoValid = IMG_TRUE;
+			}
+		}
+		else
+		{
+			/* Look-up PMR per-page validity once, if possible */
+			eError = PMR_IsOffsetValid(psPMR,
+									   gsCwq.uiPageShift,
+									   ui32NumOfPages,
+									   uiPgAlignedStartOffset,
+									   pbValid);
+			bIsPMRInfoValid = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
+		}
+	}
+
+	/* For each (possibly non-contiguous) PMR page(s), carry out the requested cache maint. op. */
+	for (uiPgAlignedOffset = uiPgAlignedStartOffset, ui32PageIndex = 0;
+		 uiPgAlignedOffset < uiPgAlignedEndOffset;
+		 uiPgAlignedOffset += (IMG_DEVMEM_OFFSET_T) gsCwq.uiPageSize, ui32PageIndex += 1)
+	{
+
+		if (! bIsPMRInfoValid)
+		{
+			/* Never cross page boundary without looking up corresponding PMR page physical
+			   address and/or page validity if these were not looked-up, in bulk, up-front */
+			ui32PageIndex = 0;
+			if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+			{
+				eError = PMR_CpuPhysAddr(psPMR,
+										 gsCwq.uiPageShift,
+										 1,
+										 uiPgAlignedOffset,
+										 psCpuPhyAddr,
+										 pbValid,
+										 CPU_USE | MAPPING_USE);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMR_CpuPhysAddr", ErrUnlockPMR);
+			}
+			else
+			{
+				eError = PMR_IsOffsetValid(psPMR,
+										  gsCwq.uiPageShift,
+										  1,
+										  uiPgAlignedOffset,
+										  pbValid);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMR_IsOffsetValid", ErrUnlockPMR);
+			}
+		}
+
+		/* Skip invalid PMR pages (i.e. sparse) */
+		if (pbValid[ui32PageIndex] == IMG_FALSE)
+		{
+			CACHEOP_PVR_ASSERT(bPMRIsSparse);
+			continue;
+		}
+
+		if (pvAddress)
+		{
+			/* The caller has supplied either a KM/UM CpuVA, so use it unconditionally */
+			pbCpuVirtAddr =
+				(void *)(uintptr_t)((uintptr_t)pvAddress + (uintptr_t)(uiPgAlignedOffset-uiPgAlignedStartOffset));
+		}
+		/* Skip CpuVA acquire if CacheOp can be maintained entirely using CpuPA */
+		else if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			if (bPMRIsSparse)
+			{
+				eError =
+					PMRAcquireSparseKernelMappingData(psPMR,
+													  uiPgAlignedOffset,
+													  gsCwq.uiPageSize,
+													  (void **)&pbCpuVirtAddr,
+													  &uiOutSize,
+													  &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", ErrUnlockPMR);
+			}
+			else
+			{
+				eError =
+					PMRAcquireKernelMappingData(psPMR,
+												uiPgAlignedOffset,
+												gsCwq.uiPageSize,
+												(void **)&pbCpuVirtAddr,
+												&uiOutSize,
+												&hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", ErrUnlockPMR);
+			}
+		}
+
+		/* Issue actual cache maintenance for PMR */
+		CacheOpExecRangeBased(psDevNode,
+							uiCacheOp,
+							pbCpuVirtAddr,
+							(eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL) ?
+								psCpuPhyAddr[ui32PageIndex] : psCpuPhyAddr[0],
+							uiPgAlignedOffset,
+							uiCLAlignedStartOffset,
+							uiCLAlignedEndOffset);
+
+		if (! pvAddress)
+		{
+			/* The caller has not supplied either a KM/UM CpuVA, release mapping */
+			if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+			{
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+			}
+		}
+	}
+
+ErrUnlockPMR:
+	PMRUnlockPMR(psPMR);
+e0:
+	if (psCpuPhyAddr != asCpuPhyAddr)
+	{
+		OSFreeMem(psCpuPhyAddr);
+	}
+
+	if (pbValid != abValid)
+	{
+		OSFreeMem(pbValid);
+	}
+
+	if (! bIsRequestValidated)
+	{
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpBatchExecTimeline(PVRSRV_DEVICE_NODE *psDevNode,
+											 PVRSRV_TIMELINE iTimeline)
+{
+	PVRSRV_ERROR eError;
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {NULL};
+
+	eError = CacheOpTimelineBind(psDevNode, &sCacheOpWorkItem, iTimeline);
+	PVR_LOG_RETURN_IF_ERROR(eError, "CacheOpTimelineBind");
+
+	eError = CacheOpTimelineExec(&sCacheOpWorkItem);
+	PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");
+
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpBatchExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
+											PMR **ppsPMR,
+											IMG_CPU_VIRTADDR *pvAddress,
+											IMG_DEVMEM_OFFSET_T *puiOffset,
+											IMG_DEVMEM_SIZE_T *puiSize,
+											PVRSRV_CACHE_OP *puiCacheOp,
+											IMG_UINT32 ui32NumCacheOps,
+											PVRSRV_TIMELINE uiTimeline)
+{
+	IMG_UINT32 ui32Idx;
+	IMG_BOOL bBatchHasTimeline;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+#endif
+
+	/* Check if batch has an associated timeline update */
+	bBatchHasTimeline = puiCacheOp[ui32NumCacheOps-1] & PVRSRV_CACHE_OP_TIMELINE;
+	puiCacheOp[ui32NumCacheOps-1] &= ~(PVRSRV_CACHE_OP_TIMELINE);
+
+	for (ui32Idx = 0; ui32Idx < ui32NumCacheOps; ui32Idx++)
+	{
+		/* Fail UM request, don't silently ignore */
+		PVR_GOTO_IF_INVALID_PARAM(puiSize[ui32Idx], eError, e0);
+
+#if defined(CACHEOP_DEBUG)
+		sCacheOpWorkItem.ui64StartTime = OSClockus64();
+#endif
+
+		eError = CacheOpPMRExec(ppsPMR[ui32Idx],
+								pvAddress[ui32Idx],
+								puiOffset[ui32Idx],
+								puiSize[ui32Idx],
+								puiCacheOp[ui32Idx],
+								IMG_FALSE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpExecPMR", e0);
+
+#if defined(CACHEOP_DEBUG)
+		sCacheOpWorkItem.ui64EndTime = OSClockus64();
+
+		sCacheOpWorkItem.psDevNode = psDevNode;
+		sCacheOpWorkItem.psPMR = ppsPMR[ui32Idx];
+		sCacheOpWorkItem.uiSize = puiSize[ui32Idx];
+		sCacheOpWorkItem.uiOffset = puiOffset[ui32Idx];
+		sCacheOpWorkItem.uiCacheOp = puiCacheOp[ui32Idx];
+		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+
+		gsCwq.ui32ServerOps += 1;
+#endif
+	}
+
+e0:
+	if (bBatchHasTimeline)
+	{
+		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline);
+	}
+
+	return eError;
+}
+
+
+PVRSRV_ERROR CacheOpExec (PPVRSRV_DEVICE_NODE psDevNode,
+						  void *pvVirtStart,
+						  void *pvVirtEnd,
+						  IMG_CPU_PHYADDR sCPUPhysStart,
+						  IMG_CPU_PHYADDR sCPUPhysEnd,
+						  PVRSRV_CACHE_OP uiCacheOp)
+{
+#if defined(CACHEOP_DEBUG)
+	IMG_UINT64 ui64StartTime = OSClockus64();
+#endif
+
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					 __func__, uiCacheOp));
+			break;
+	}
+
+#if defined(CACHEOP_DEBUG)
+	if (CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
+	{
+		CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+
+		gsCwq.ui32ServerOps += 1;
+
+		sCacheOpWorkItem.uiOffset = 0;
+		sCacheOpWorkItem.bKMReq = IMG_TRUE;
+		sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+		/* Use information page PMR for logging KM request */
+		sCacheOpWorkItem.psPMR = gsCwq.psInfoPagePMR;
+		sCacheOpWorkItem.psDevNode = psDevNode;
+		sCacheOpWorkItem.ui64StartTime = ui64StartTime;
+		sCacheOpWorkItem.ui64EndTime = OSClockus64();
+		sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+		sCacheOpWorkItem.uiSize = (sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr);
+
+		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
+						    IMG_UINT64 uiAddress,
+						    IMG_DEVMEM_OFFSET_T uiOffset,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    PVRSRV_CACHE_OP uiCacheOp)
+{
+	PVRSRV_ERROR eError;
+	IMG_CPU_VIRTADDR pvAddress = (IMG_CPU_VIRTADDR)(uintptr_t)uiAddress;
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+
+	sCacheOpWorkItem.ui64StartTime = OSClockus64();
+#endif
+
+	eError = CacheOpPMRExec(psPMR,
+							pvAddress,
+							uiOffset,
+							uiSize,
+							uiCacheOp,
+							IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpPMRExec", e0);
+
+#if defined(CACHEOP_DEBUG)
+	sCacheOpWorkItem.ui64EndTime = OSClockus64();
+
+	sCacheOpWorkItem.psDevNode = PMR_DeviceNode(psPMR);
+	sCacheOpWorkItem.psPMR = psPMR;
+	sCacheOpWorkItem.uiSize = uiSize;
+	sCacheOpWorkItem.uiOffset = uiOffset;
+	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+
+	gsCwq.ui32ServerOps += 1;
+#endif
+
+e0:
+	return eError;
+}
+
+PVRSRV_ERROR CacheOpQueue (CONNECTION_DATA *psConnection,
+						   PVRSRV_DEVICE_NODE *psDevNode,
+						   IMG_UINT32 ui32NumCacheOps,
+						   PMR **ppsPMR,
+						   IMG_UINT64 *puiAddress,
+						   IMG_DEVMEM_OFFSET_T *puiOffset,
+						   IMG_DEVMEM_SIZE_T *puiSize,
+						   PVRSRV_CACHE_OP *puiCacheOp,
+						   IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_TIMELINE uiTimeline = (PVRSRV_TIMELINE)ui32OpTimeline;
+	IMG_CPU_VIRTADDR *pvAddress = (IMG_CPU_VIRTADDR*)(uintptr_t)puiAddress;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!gsCwq.bInit)
+	{
+		PVR_LOG(("CacheOp framework not initialised, failing request"));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+	else if (! ui32NumCacheOps)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	/* Ensure any single timeline CacheOp request is processed immediately */
+	else if (ui32NumCacheOps == 1 && puiCacheOp[0] == PVRSRV_CACHE_OP_TIMELINE)
+	{
+		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline);
+	}
+	/* This is the default entry for all client requests */
+	else
+	{
+		if (!(gsCwq.eConfig & (CACHEOP_CONFIG_LAST-1)))
+		{
+			/* default the configuration before execution */
+			CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
+		}
+
+		eError =
+			CacheOpBatchExecRangeBased(psDevNode,
+									   ppsPMR,
+									   pvAddress,
+									   puiOffset,
+									   puiSize,
+									   puiCacheOp,
+									   ui32NumCacheOps,
+									   uiTimeline);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR CacheOpLog (PMR *psPMR,
+						 IMG_UINT64 puiAddress,
+						 IMG_DEVMEM_OFFSET_T uiOffset,
+						 IMG_DEVMEM_SIZE_T uiSize,
+						 IMG_UINT64 ui64StartTime,
+						 IMG_UINT64 ui64EndTime,
+						 PVRSRV_CACHE_OP uiCacheOp)
+{
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+	PVR_UNREFERENCED_PARAMETER(puiAddress);
+
+	sCacheOpWorkItem.psDevNode = PMR_DeviceNode(psPMR);
+	sCacheOpWorkItem.psPMR = psPMR;
+	sCacheOpWorkItem.uiSize = uiSize;
+	sCacheOpWorkItem.uiOffset = uiOffset;
+	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+
+	sCacheOpWorkItem.ui64StartTime = ui64StartTime;
+	sCacheOpWorkItem.ui64EndTime = ui64EndTime;
+
+	gsCwq.ui32ClientOps += 1;
+
+	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+#else
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiCacheOp);
+	PVR_UNREFERENCED_PARAMETER(puiAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64EndTime);
+#endif
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR CacheOpInit2 (void)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	/* Apphint read/write is not concurrent, so lock protects against this */
+	eError = OSLockCreate((POS_LOCK*)&gsCwq.hConfigLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+
+#if defined(CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH)
+	gsCwq.bSupportsUMFlush = IMG_TRUE;
+#else
+	gsCwq.bSupportsUMFlush = IMG_FALSE;
+#endif
+
+	gsCwq.pui32InfoPage = psPVRSRVData->pui32InfoPage;
+	gsCwq.psInfoPagePMR = psPVRSRVData->psInfoPagePMR;
+
+	/* Normally, platforms should use their default configurations, put exceptions here */
+#if defined(__i386__) || defined(__x86_64__)
+#if !defined(TC_MEMORY_CONFIG)
+	CacheOpConfigUpdate(CACHEOP_CONFIG_URBF | CACHEOP_CONFIG_KDF);
+#else
+	CacheOpConfigUpdate(CACHEOP_CONFIG_KDF);
+#endif
+#else /* defined(__x86__) */
+	CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
+#endif
+
+	/* Initialise the remaining occupants of the CacheOp information page */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_PGSIZE]   = (IMG_UINT32)gsCwq.uiPageSize;
+	gsCwq.pui32InfoPage[CACHEOP_INFO_LINESIZE] = (IMG_UINT32)gsCwq.uiLineSize;
+
+	/* Set before spawning thread */
+	gsCwq.bInit = IMG_TRUE;
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = CacheOpConfigRead};
+		/* Writing the unsigned integer binary encoding of CACHEOP_CONFIG
+		   into this file cycles through avail. configuration(s) */
+		eError = DICreateEntry("cacheop_config", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gsCwq.psConfigTune);
+		PVR_LOG_GOTO_IF_FALSE(gsCwq.psConfigTune, "DICreateEntry", e0);
+	}
+
+	/* Register the CacheOp framework (re)configuration handlers */
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpConfig,
+										CacheOpConfigQuery,
+										CacheOpConfigSet,
+										APPHINT_OF_DRIVER_NO_DEVICE,
+										(void *) APPHINT_ID_CacheOpConfig);
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpUMKMThresholdSize,
+										CacheOpConfigQuery,
+										CacheOpConfigSet,
+										APPHINT_OF_DRIVER_NO_DEVICE,
+										(void *) APPHINT_ID_CacheOpUMKMThresholdSize);
+
+	return PVRSRV_OK;
+e0:
+	CacheOpDeInit2();
+	return eError;
+}
+
+void CacheOpDeInit2 (void)
+{
+	gsCwq.bInit = IMG_FALSE;
+
+	if (gsCwq.hConfigLock)
+	{
+		OSLockDestroy(gsCwq.hConfigLock);
+		gsCwq.hConfigLock = NULL;
+	}
+
+	if (gsCwq.psConfigTune)
+	{
+		DIDestroyEntry(gsCwq.psConfigTune);
+		gsCwq.psConfigTune = NULL;
+	}
+
+	gsCwq.pui32InfoPage = NULL;
+	gsCwq.psInfoPagePMR = NULL;
+}
+
+PVRSRV_ERROR CacheOpInit (void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	gsCwq.uiPageSize = OSGetPageSize();
+	gsCwq.uiPageShift = OSGetPageShift();
+	gsCwq.uiLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+	gsCwq.uiLineShift = ExactLog2(gsCwq.uiLineSize);
+	PVR_LOG_RETURN_IF_FALSE((gsCwq.uiLineSize && gsCwq.uiPageSize && gsCwq.uiPageShift), "", PVRSRV_ERROR_INIT_FAILURE);
+
+#if defined(CACHEOP_DEBUG)
+	/* debugfs file read-out is not concurrent, so lock protects against this */
+	eError = OSLockCreate((POS_LOCK*)&gsCwq.hStatsExecLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	gsCwq.i32StatsExecWriteIdx = 0;
+	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = CacheOpStatsExecLogRead};
+		/* File captures the most recent subset of CacheOp(s) executed */
+		eError = DICreateEntry("cacheop_history", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gsCwq.psDIEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", e0);
+	}
+e0:
+#endif
+	return eError;
+}
+
+void CacheOpDeInit (void)
+{
+#if defined(CACHEOP_DEBUG)
+	if (gsCwq.hStatsExecLock)
+	{
+		OSLockDestroy(gsCwq.hStatsExecLock);
+		gsCwq.hStatsExecLock = NULL;
+	}
+
+	if (gsCwq.psDIEntry)
+	{
+		DIDestroyEntry(gsCwq.psDIEntry);
+		gsCwq.psDIEntry = NULL;
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/connection_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/connection_server.c
new file mode 100644
index 000000000000..b8d3994bcce1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/connection_server.c
@@ -0,0 +1,515 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Handles connections coming from the client and the management
+                connection based information
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "handle.h"
+#include "pvrsrv.h"
+#include "connection_server.h"
+#include "osconnection_server.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "sync_server.h"
+#include "process_stats.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "tlstream.h"
+#include "rgxhwperf_common.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+/* PID associated with Connection currently being purged by Cleanup thread */
+static IMG_PID gCurrentPurgeConnectionPid;
+
+static PVRSRV_ERROR ConnectionDataDestroy(CONNECTION_DATA *psConnection)
+{
+	PVRSRV_ERROR eError;
+	PROCESS_HANDLE_BASE *psProcessHandleBase;
+	IMG_UINT64 ui64MaxBridgeTime;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+
+	if (psPVRSRVData->bUnload)
+	{
+		/* driver is unloading so do not allow the bridge lock to be released */
+		ui64MaxBridgeTime = 0;
+	}
+	else
+	{
+		ui64MaxBridgeTime = CONNECTION_DEFERRED_CLEANUP_TIMESLICE_NS;
+	}
+
+	PVR_ASSERT(psConnection != NULL);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConnection, "psConnection");
+
+	/* Close HWPerfClient stream here even though we created it in
+	 * PVRSRVConnectKM(). */
+	if (psConnection->hClientTLStream)
+	{
+		TLStreamClose(psConnection->hClientTLStream);
+		psConnection->hClientTLStream = NULL;
+		PVR_DPF((PVR_DBG_MESSAGE, "Destroyed private stream."));
+	}
+
+	/* Get process handle base to decrement the refcount */
+	psProcessHandleBase = psConnection->psProcessHandleBase;
+
+	if (psProcessHandleBase != NULL)
+	{
+		/* PVRSRVReleaseProcessHandleBase() calls PVRSRVFreeKernelHandles()
+		 * and PVRSRVFreeHandleBase() for the process handle base.
+		 * Releasing kernel handles can never return RETRY error because
+		 * release function for those handles are NOPs and PVRSRVFreeKernelHandles()
+		 * doesn't even call pfnReleaseData() callback.
+		 * Process handles can potentially return RETRY hence additional check
+		 * below. */
+		eError = PVRSRVReleaseProcessHandleBase(psProcessHandleBase,
+		                                        ui64MaxBridgeTime);
+		if (PVRSRVIsRetryError(eError))
+		{
+			return eError;
+		}
+		else
+		{
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVReleaseProcessHandleBase");
+		}
+
+		psConnection->psProcessHandleBase = NULL;
+	}
+
+	/* Free handle base for this connection */
+	if (psConnection->psHandleBase != NULL)
+	{
+		eError = PVRSRVFreeHandleBase(psConnection->psHandleBase, ui64MaxBridgeTime);
+		/*
+		 * If we get PVRSRV_ERROR_RETRY we need to pass this back to the caller
+		 * who will schedule a retry.
+		 * Do not log this as it is an expected exception.
+		 * This can occur if the Firmware is still processing a workload from
+		 * the client when a tear-down request is received.
+		 * Retrying will allow the in-flight work to be completed and the
+		 * tear-down request can be completed when the FW is no longer busy.
+		 */
+		if (PVRSRVIsRetryError(eError))
+		{
+			return eError;
+		}
+		else
+		{
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeHandleBase");
+		}
+
+		psConnection->psHandleBase = NULL;
+	}
+
+	if (psConnection->psSyncConnectionData != NULL)
+	{
+		SyncUnregisterConnection(psConnection->psSyncConnectionData);
+		psConnection->psSyncConnectionData = NULL;
+	}
+
+	if (psConnection->psPDumpConnectionData != NULL)
+	{
+		PDumpUnregisterConnection(psDevNode,
+		                          psConnection->psPDumpConnectionData);
+		psConnection->psPDumpConnectionData = NULL;
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDeviceDisconnect(psDevNode);
+#endif
+
+	/* Call environment specific connection data deinit function */
+	if (psConnection->hOsPrivateData != NULL)
+	{
+		eError = OSConnectionPrivateDataDeInit(psConnection->hOsPrivateData);
+		PVR_LOG_RETURN_IF_ERROR(eError, "OSConnectionPrivateDataDeInit");
+
+		psConnection->hOsPrivateData = NULL;
+	}
+
+	/* Close the PID stats entry as late as possible to catch all frees */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+	if (psConnection->hProcessStats != NULL)
+	{
+		PVRSRVStatsDeregisterProcess(psConnection->hProcessStats);
+		psConnection->hProcessStats = NULL;
+	}
+#endif
+
+	OSFreeMemNoStats(psConnection);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Kick the Firmware to invalidate caches to clear all the zombie PMRs.
+	 * If there are not zombie PMRs or no mappings were freed the kick will not
+	 * be executed.
+	 *
+	 * This is needed:
+	 * - when the process is killed and the connection cleanup has to clean up
+	 *   all dangling handles.
+	 * - there are any outstanding PMRs in the zombie list due to no
+	 *   invalidation being executed before connection destruction
+	 */
+	eError = MMU_CacheInvalidateKick(psDevNode, NULL);
+	PVR_LOG_IF_ERROR(eError, "MMU_CacheInvalidateKick");
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVCommonConnectionConnect(void **ppvPrivData, void *pvOSData)
+{
+	CONNECTION_DATA *psConnection;
+	PVRSRV_ERROR eError;
+	PROCESS_HANDLE_BASE *psProcessHandleBase;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* Allocate connection data area, no stats since process not registered yet */
+	psConnection = OSAllocZMemNoStats(sizeof(*psConnection));
+	PVR_LOG_RETURN_IF_NOMEM(psConnection, "psConnection");
+
+	/* Allocate process statistics as early as possible to catch all allocs */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+	eError = PVRSRVStatsRegisterProcess(&psConnection->hProcessStats);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsRegisterProcess", failure);
+#endif
+
+	/* Call environment specific connection data init function */
+	eError = OSConnectionPrivateDataInit(&psConnection->hOsPrivateData, pvOSData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSConnectionPrivateDataInit", failure);
+
+	/* Must come after OSConnectionPrivateDataInit */
+	psDevNode = OSGetDevNode(psConnection);
+	PVR_LOG_GOTO_IF_NOMEM(psDevNode, eError, failure);
+
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cannot connect to the device during deinit"));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_DEVICE, failure);
+	}
+
+	psConnection->pid = OSGetCurrentClientProcessIDKM();
+	psConnection->vpid = OSGetCurrentVirtualProcessID();
+	psConnection->tid = (IMG_UINT32)OSGetCurrentClientThreadIDKM();
+	OSStringSafeCopy(psConnection->pszProcName, OSGetCurrentClientProcessNameKM(), PVRSRV_CONNECTION_PROCESS_NAME_LEN);
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	eError = PVRSRVInitialiseDMA(psDevNode, psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVInitialiseDMA", failure);
+#endif
+
+	/* Register this connection with the sync core */
+	eError = SyncRegisterConnection(&psConnection->psSyncConnectionData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncRegisterConnection", failure);
+
+	/*
+	 * Register this connection and Sync PDump callback with
+	 * the pdump core. Pass in the Sync connection data.
+	 */
+	eError = PDumpRegisterConnection(psDevNode,
+	                                 psConnection->psSyncConnectionData,
+	                                 SyncConnectionPDumpSyncBlocks,
+	                                 &psConnection->psPDumpConnectionData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpRegisterConnection", failure);
+
+	/* Allocate handle base for this connection */
+	eError = PVRSRVAllocHandleBase(&psConnection->psHandleBase,
+	                               PVRSRV_HANDLE_BASE_TYPE_CONNECTION);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", failure);
+
+	/* get process handle base for the current process (if it doesn't exist it will be allocated) */
+	eError = PVRSRVAcquireProcessHandleBase(&psProcessHandleBase);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireProcessHandleBase", failure);
+
+	/* hConnectionsLock now resides in PVRSRV_DEVICE_NODE */
+	{
+		IMG_BOOL bHostStreamIsNull;
+		PVRSRV_RGXDEV_INFO  *psRgxDevInfo;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		eError = PVRSRVStatsDeviceConnect(psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsDeviceConnect", failure);
+#endif
+
+		OSLockAcquire(psDevNode->hConnectionsLock);
+		dllist_add_to_tail(&psDevNode->sConnections, &psConnection->sConnectionListNode);
+#if defined(DEBUG) || defined(PDUMP)
+		PVR_LOG(("%s connected - (devID = %u)", psConnection->pszProcName,
+		        psDevNode->sDevId.ui32InternalID));
+#endif
+		OSLockRelease(psDevNode->hConnectionsLock);
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+		{
+			psRgxDevInfo = _RGX_DEVICE_INFO_FROM_NODE(psDevNode);
+
+			OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+			bHostStreamIsNull = (IMG_BOOL)(psRgxDevInfo->hHWPerfHostStream == NULL);
+			OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+			if (!bHostStreamIsNull)
+			{
+				if (TLStreamIsOpenForReading(psRgxDevInfo->hHWPerfHostStream))
+				{
+					/* Announce this client connection in the host stream, if event mask is set */
+					RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(psDevNode, psConnection->pid, psConnection->pszProcName);
+				}
+			}
+		}
+	}
+
+	psConnection->psProcessHandleBase = psProcessHandleBase;
+
+	*ppvPrivData = psConnection;
+
+	return PVRSRV_OK;
+
+failure:
+	ConnectionDataDestroy(psConnection);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _CleanupThreadPurgeConnectionData(void *pvConnectionData)
+{
+	PVRSRV_ERROR eErrorConnection, eErrorKernel;
+	CONNECTION_DATA *psConnectionData = pvConnectionData;
+
+	gCurrentPurgeConnectionPid = psConnectionData->pid;
+
+	eErrorConnection = ConnectionDataDestroy(psConnectionData);
+	if (eErrorConnection != PVRSRV_OK)
+	{
+		if (PVRSRVIsRetryError(eErrorConnection))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Failed to purge connection data %p "
+			        "(deferring destruction)", __func__, psConnectionData));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Connection data %p deferred destruction "
+		        "finished", __func__, psConnectionData));
+	}
+
+	/* Check if possible resize the global handle base */
+	eErrorKernel = PVRSRVPurgeHandles(KERNEL_HANDLE_BASE);
+	PVR_LOG_IF_ERROR(eErrorKernel, "PVRSRVPurgeHandles");
+
+	gCurrentPurgeConnectionPid = 0;
+
+	return eErrorConnection;
+}
+
+void PVRSRVCommonConnectionDisconnect(void *pvDataPtr)
+{
+	CONNECTION_DATA *psConnectionData = pvDataPtr;
+	PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnectionData);
+
+	OSLockAcquire(psDevNode->hConnectionsLock);
+	dllist_remove_node(&psConnectionData->sConnectionListNode);
+	OSLockRelease(psDevNode->hConnectionsLock);
+
+	/* Notify the PDump core if the pdump control client is disconnecting */
+	if (psConnectionData->ui32ClientFlags & SRV_FLAGS_PDUMPCTRL)
+	{
+		PDumpDisconnectionNotify(psDevNode);
+	}
+
+	/* Add a HOST_CLIENT_INFO event to match the one on connection */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		IMG_BOOL bHostStreamIsNull;
+		PVRSRV_RGXDEV_INFO  *psRgxDevInfo;
+
+		psRgxDevInfo = _RGX_DEVICE_INFO_FROM_NODE(psDevNode);
+
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+		bHostStreamIsNull = (IMG_BOOL)(psRgxDevInfo->hHWPerfHostStream == NULL);
+		OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+		if (!bHostStreamIsNull)
+		{
+			if (TLStreamIsOpenForReading(psRgxDevInfo->hHWPerfHostStream))
+			{
+				/* Announce this client connection in the host stream, if event mask is set */
+				RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(psDevNode, psConnectionData->pid, psConnectionData->pszProcName);
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Mark remaining resources for driver to free */
+	RIConnectionClosed(psConnectionData);
+#endif
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	PVRSRVDeInitialiseDMA(psDevNode, psConnectionData);
+#endif
+
+#if defined(DEBUG) || defined(PDUMP)
+	PVR_LOG(("%s disconnected - (devID = %u)", psConnectionData->pszProcName,
+	        psDevNode->sDevId.ui32InternalID));
+#endif
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK)
+#endif
+	{
+		/* Defer the release of the connection data */
+		psConnectionData->sCleanupThreadFn.pfnFree = _CleanupThreadPurgeConnectionData;
+		psConnectionData->sCleanupThreadFn.pvData = psConnectionData;
+		/* Some resources in HANDLE_BASE may need FW idle confirmation
+		 * hence setting to TRUE to use the global EO for retries which is
+		 * signalled by the device MISR */
+		psConnectionData->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+		psConnectionData->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_CONNECTION;
+		CLEANUP_THREAD_SET_RETRY_COUNT(&psConnectionData->sCleanupThreadFn,
+		                               CLEANUP_THREAD_RETRY_COUNT_DEFAULT);
+		PVRSRVCleanupThreadAddWork(psDevNode, &psConnectionData->sCleanupThreadFn);
+	}
+}
+
+IMG_PID PVRSRVGetPurgeConnectionPid(void)
+{
+	return gCurrentPurgeConnectionPid;
+}
+
+/* Prefix for debug messages about Active Connections */
+#define DEBUG_DUMP_CONNECTION_FORMAT_STR " P%d-V%d-T%d-%s,"
+#define CONNECTIONS_PREFIX               "Connections Device ID:%u(%d)"
+#define MAX_CONNECTIONS_PREFIX            (29)
+#define MAX_DEBUG_DUMP_CONNECTION_STR_LEN (1+10+10+10+7+PVRSRV_CONNECTION_PROCESS_NAME_LEN)
+#define MAX_DEBUG_DUMP_STRING_LEN         (1+MAX_CONNECTIONS_PREFIX+(3*MAX_DEBUG_DUMP_CONNECTION_STR_LEN))
+
+void PVRSRVConnectionDebugNotify(PVRSRV_DEVICE_NODE *psDevNode,
+								 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile)
+{
+	PDLLIST_NODE pNext, pNode;
+
+	/* We must check for an initialised device before accessing its mutex.
+	 * The mutex is initialised as part of DeviceInitialize() which occurs
+	 * on first access to the device node.
+	 */
+	if ((psDevNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE) &&
+	    (psDevNode->eDevState != PVRSRV_DEVICE_STATE_FROZEN))
+	{
+		PVR_DUMPDEBUG_LOG("Connections: No Devices: No active connections");
+		return;
+	}
+
+	OSLockAcquire(psDevNode->hConnectionsLock);
+	if (dllist_is_empty(&psDevNode->sConnections))
+	{
+		PVR_DUMPDEBUG_LOG(CONNECTIONS_PREFIX " No active connections",
+						  (unsigned char)psDevNode->sDevId.ui32InternalID,
+						  (unsigned char)psDevNode->sDevId.i32KernelDeviceID);
+	}
+	else
+	{
+		IMG_CHAR sActiveConnections[MAX_DEBUG_DUMP_STRING_LEN];
+		IMG_UINT16 i, uiPos = 0;
+		IMG_BOOL bPrinted = IMG_FALSE;
+		size_t uiSize = sizeof(sActiveConnections);
+
+		IMG_CHAR szTmpConBuff[MAX_CONNECTIONS_PREFIX + 1];
+		i = OSSNPrintf(szTmpConBuff,
+					   MAX_CONNECTIONS_PREFIX,
+					   CONNECTIONS_PREFIX,
+					   (unsigned char)psDevNode->sDevId.ui32InternalID,
+					   (unsigned char)psDevNode->sDevId.i32KernelDeviceID);
+		OSStringSafeCopy(sActiveConnections+uiPos, szTmpConBuff, uiSize);
+
+		/* Move the write offset to the end of the current string */
+		uiPos += i;
+		/* Update the amount of remaining space available to copy into */
+		uiSize -= i;
+
+		dllist_foreach_node(&psDevNode->sConnections, pNode, pNext)
+		{
+			CONNECTION_DATA *sData = IMG_CONTAINER_OF(pNode, CONNECTION_DATA, sConnectionListNode);
+
+			IMG_CHAR sTmpBuff[MAX_DEBUG_DUMP_CONNECTION_STR_LEN];
+			i = OSSNPrintf(sTmpBuff, MAX_DEBUG_DUMP_CONNECTION_STR_LEN,
+				DEBUG_DUMP_CONNECTION_FORMAT_STR, sData->pid, sData->vpid, sData->tid, sData->pszProcName);
+			i = MIN(MAX_DEBUG_DUMP_CONNECTION_STR_LEN, i);
+			bPrinted = IMG_FALSE;
+
+			OSStringSafeCopy(sActiveConnections+uiPos, sTmpBuff, uiSize);
+
+			/* Move the write offset to the end of the current string */
+			uiPos += i;
+			/* Update the amount of remaining space available to copy into */
+			uiSize -= i;
+
+			/* If there is not enough space to add another connection to this line, output the line */
+			if (uiSize <= MAX_DEBUG_DUMP_CONNECTION_STR_LEN)
+			{
+				PVR_DUMPDEBUG_LOG("%s", sActiveConnections);
+
+				/*
+				 * Remove the "Connections:" prefix from the buffer.
+				 * Leave the subsequent buffer contents indented by the same
+				 * amount to aid in interpreting the debug output.
+				 */
+				uiPos = sizeof(CONNECTIONS_PREFIX) - 1;
+				/* Reset the amount of space available to copy into */
+				uiSize = MAX_DEBUG_DUMP_STRING_LEN - uiPos;
+				bPrinted = IMG_TRUE;
+			}
+		}
+
+		/* Only print the current line if it hasn't already been printed */
+		if (!bPrinted)
+		{
+			/* Strip off the final comma */
+			sActiveConnections[OSStringNLength(sActiveConnections, MAX_DEBUG_DUMP_STRING_LEN) - 1] = '\0';
+			PVR_DUMPDEBUG_LOG("%s", sActiveConnections);
+		}
+#undef MAX_DEBUG_DUMP_STRING_LEN
+#undef MAX_DEBUG_DUMP_CONNECTIONS_PER_LINE
+	}
+	OSLockRelease(psDevNode->hConnectionsLock);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/debug_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/debug_common.c
new file mode 100644
index 000000000000..4499ed6de993
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/debug_common.c
@@ -0,0 +1,1995 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Creates common debug info entries.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__linux__)
+#include <errno.h>
+#endif /* #if !defined(__linux__) */
+
+#include "debug_common.h"
+#include "pvrsrv.h"
+#include "di_server.h"
+#include "lists.h"
+#include "pvrversion.h"
+#include "rgx_options.h"
+#include "allocmem.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "osfunc.h"
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#include "rgxfwdbg.h"
+#endif
+
+#ifdef SUPPORT_RGX
+#include "rgxdevice.h"
+#include "rgxdebug_common.h"
+#include "rgxinit.h"
+#include "rgxmmudefs_km.h"
+#endif
+
+static DI_ENTRY *gpsVersionDIEntry;
+static DI_ENTRY *gpsStatusDIEntry;
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+static DI_ENTRY *gpsDebugLevelDIEntry;
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+struct DI_VZ_DATA {
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_UINT32 ui32DriverID;
+};
+#endif
+
+static void _DumpDebugDIPrintfWrapper(void *pvDumpDebugFile, const IMG_CHAR *pszFormat, ...)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list ArgList;
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN, "%s\n", pszFormat);
+
+	va_start(ArgList, pszFormat);
+	DIVPrintf(pvDumpDebugFile, szBuffer, ArgList);
+	va_end(ArgList);
+}
+
+/*************************************************************************/ /*!
+ Version DebugFS entry
+*/ /**************************************************************************/
+
+static void *_DebugVersionCompare_AnyVaCb(PVRSRV_DEVICE_NODE *psDevNode,
+                                          va_list va)
+{
+	IMG_UINT64 *pui64CurrentPosition = va_arg(va, IMG_UINT64 *);
+	IMG_UINT64 ui64Position = va_arg(va, IMG_UINT64);
+	IMG_UINT64 ui64CurrentPosition = *pui64CurrentPosition;
+
+	(*pui64CurrentPosition)++;
+
+	return (ui64CurrentPosition == ui64Position) ? psDevNode : NULL;
+}
+
+static void *_VersionDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+
+	if (psPVRSRVData == NULL) {
+		PVR_DPF((PVR_DBG_ERROR, "psPVRSRVData = NULL"));
+		return NULL;
+	}
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+	                                      _DebugVersionCompare_AnyVaCb,
+	                                      &uiCurrentPosition,
+	                                      *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+static void _VersionDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvPriv)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+}
+
+static void *_VersionDINext(OSDI_IMPL_ENTRY *psEntry,void *pvPriv,
+                            IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	(*pui64Pos)++;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+	                                      _DebugVersionCompare_AnyVaCb,
+	                                      &uiCurrentPosition,
+	                                      *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+#define DI_PRINT_VERSION_FMTSPEC \
+		"%s Version: %u.%u @ %u (%s) build options: 0x%08x %s\n"
+#define STR_DEBUG   "debug"
+#define STR_RELEASE "release"
+
+#if defined(DEBUG)
+#define BUILD_OPT_LEN 80
+
+static inline void _AppendOptionStr(IMG_CHAR pszBuildOptions[], const IMG_CHAR* str, OSDI_IMPL_ENTRY *psEntry, IMG_UINT32* pui32BuildOptionLen)
+{
+	IMG_UINT32 ui32BuildOptionLen = *pui32BuildOptionLen;
+	const IMG_UINT32 strLen = OSStringLength(str);
+	const IMG_UINT32 optStrLen = sizeof(IMG_CHAR) * (BUILD_OPT_LEN-1);
+
+	if ((ui32BuildOptionLen + strLen) > optStrLen)
+	{
+		pszBuildOptions[ui32BuildOptionLen] = '\0';
+		DIPrintf(psEntry, "%s\n", pszBuildOptions);
+		ui32BuildOptionLen = 0;
+	}
+	if (strLen < optStrLen)
+	{
+		OSStringSafeCopy(pszBuildOptions+ui32BuildOptionLen, str, strLen);
+		ui32BuildOptionLen += strLen - 1;
+	}
+	*pui32BuildOptionLen = ui32BuildOptionLen;
+}
+#endif /* DEBUG || SUPPORT_VALIDATION */
+
+static int _VersionDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvPriv)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+
+	if (pvPriv == DI_START_TOKEN)
+	{
+		if (psPVRSRVData->sDriverInfo.bIsNoMatch)
+		{
+			const BUILD_INFO *psBuildInfo;
+
+			psBuildInfo = &psPVRSRVData->sDriverInfo.sUMBuildInfo;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+			         "UM Driver",
+			         PVRVERSION_UNPACK_MAJ(psBuildInfo->ui32BuildVersion),
+			         PVRVERSION_UNPACK_MIN(psBuildInfo->ui32BuildVersion),
+			         psBuildInfo->ui32BuildRevision,
+			         (psBuildInfo->ui32BuildType == BUILD_TYPE_DEBUG) ?
+			                 STR_DEBUG : STR_RELEASE,
+			         psBuildInfo->ui32BuildOptions,
+			         PVR_BUILD_DIR);
+
+			psBuildInfo = &psPVRSRVData->sDriverInfo.sKMBuildInfo;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+			         "KM Driver (" PVR_ARCH_NAME ")",
+			         PVRVERSION_UNPACK_MAJ(psBuildInfo->ui32BuildVersion),
+			         PVRVERSION_UNPACK_MIN(psBuildInfo->ui32BuildVersion),
+			         psBuildInfo->ui32BuildRevision,
+			         (psBuildInfo->ui32BuildType == BUILD_TYPE_DEBUG) ?
+			                 STR_DEBUG : STR_RELEASE,
+			         psBuildInfo->ui32BuildOptions,
+			         PVR_BUILD_DIR);
+		}
+		else
+		{
+			/* bIsNoMatch is `false` in one of the following cases:
+			 * - UM & KM version parameters actually match.
+			 * - A comparison between UM & KM has not been made yet, because no
+			 *   client ever connected.
+			 *
+			 * In both cases, available (KM) version info is the best output we
+			 * can provide.
+			 */
+			DIPrintf(psEntry, "Driver Version: %s (%s) (%s) build options: "
+			         "0x%08lx %s\n", PVRVERSION_STRING, PVR_ARCH_NAME,
+			         PVR_BUILD_TYPE, RGX_BUILD_OPTIONS_KM, PVR_BUILD_DIR);
+		}
+	}
+	else if (pvPriv != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *) pvPriv;
+		PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+#ifdef SUPPORT_RGX
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+#if defined(DEBUG)
+		IMG_CHAR pszBuildOptions[BUILD_OPT_LEN];
+		IMG_UINT32 ui32BuildOptionLen = 0;
+		static const char* aszOptions[] = RGX_BUILD_OPTIONS_LIST;
+		int i = 0;
+#endif
+#endif /* SUPPORT_RGX */
+		IMG_BOOL bFwVersionInfoPrinted = IMG_FALSE;
+
+		DIPrintf(psEntry, "\nDevice Name: %s\n", psDevConfig->pszName);
+		DIPrintf(psEntry, "Device ID: %u:%d\n", psDevNode->sDevId.ui32InternalID,
+		                                        psDevNode->sDevId.i32KernelDeviceID);
+
+		if (psDevConfig->pszVersion)
+		{
+			DIPrintf(psEntry, "Device Version: %s\n",
+			          psDevConfig->pszVersion);
+		}
+
+		if (psDevNode->pfnDeviceVersionString)
+		{
+			IMG_CHAR *pszVerStr;
+
+			if (psDevNode->pfnDeviceVersionString(psDevNode,
+			                                      &pszVerStr) == PVRSRV_OK)
+			{
+				DIPrintf(psEntry, "%s\n", pszVerStr);
+
+				OSFreeMem(pszVerStr);
+			}
+		}
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+		{
+#ifdef SUPPORT_RGX
+			/* print device's firmware version info */
+			if (psDevInfo->psRGXFWIfOsInitMemDesc != NULL)
+			{
+				/* psDevInfo->psRGXFWIfOsInitMemDesc should be permanently mapped */
+				if (psDevInfo->psRGXFWIfOsInit != NULL)
+				{
+					RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+				                               INVALIDATE);
+					if (psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated)
+					{
+						const RGXFWIF_COMPCHECKS *psRGXCompChecks =
+						        &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks;
+						IMG_UINT32 ui32DDKVer = psRGXCompChecks->ui32DDKVersion;
+
+						DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+						         "Firmware",
+						         PVRVERSION_UNPACK_MAJ(ui32DDKVer),
+						         PVRVERSION_UNPACK_MIN(ui32DDKVer),
+						         psRGXCompChecks->ui32DDKBuild,
+						         ((psRGXCompChecks->ui32BuildOptions &
+						          OPTIONS_DEBUG_EN) ? STR_DEBUG : STR_RELEASE),
+						         psRGXCompChecks->ui32BuildOptions,
+						         PVR_BUILD_DIR);
+						bFwVersionInfoPrinted = IMG_TRUE;
+
+#if defined(DEBUG)
+						DIPrintf(psEntry, "Firmware Build Options:\n");
+
+						for (i = 0; i < ARRAY_SIZE(aszOptions); i++)
+						{
+							if ((psRGXCompChecks->ui32BuildOptions & 1<<i))
+							{
+								_AppendOptionStr(pszBuildOptions, aszOptions[i], psEntry, &ui32BuildOptionLen);
+							}
+						}
+
+						if (ui32BuildOptionLen != 0)
+						{
+							DIPrintf(psEntry, "%s", pszBuildOptions);
+						}
+						DIPrintf(psEntry, "\n");
+#endif
+					}
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Error acquiring CPU virtual "
+					        "address of FWInitMemDesc", __func__));
+				}
+			}
+#endif /* SUPPORT_RGX */
+		}
+		else
+		{
+#ifdef SUPPORT_RGX
+			RGX_FW_INFO_HEADER	*psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+					 "Firmware",
+					 psFWInfoHeader->ui16PVRVersionMajor,
+					 psFWInfoHeader->ui16PVRVersionMinor,
+					 psFWInfoHeader->ui32PVRVersionBuild,
+					 ((psFWInfoHeader->ui32Flags &
+					 OPTIONS_DEBUG_EN) ? STR_DEBUG : STR_RELEASE),
+					 psFWInfoHeader->ui32Flags,
+					 PVR_BUILD_DIR);
+
+			bFwVersionInfoPrinted = IMG_TRUE;
+#if defined(DEBUG)
+			DIPrintf(psEntry, "Firmware Build Options:\n");
+
+			for (i = 0; i < ARRAY_SIZE(aszOptions); i++)
+			{
+				if ((psFWInfoHeader->ui32Flags & 1<<i))
+				{
+					_AppendOptionStr(pszBuildOptions, aszOptions[i], psEntry, &ui32BuildOptionLen);
+				}
+			}
+
+			if (ui32BuildOptionLen != 0)
+			{
+				DIPrintf(psEntry, "%s", pszBuildOptions);
+			}
+			DIPrintf(psEntry, "\n");
+#endif
+
+#endif /* SUPPORT_RGX */
+		}
+
+		if (!bFwVersionInfoPrinted)
+		{
+			DIPrintf(psEntry, "Firmware Version: Info unavailable %s\n",
+#ifdef NO_HARDWARE
+			         "on NoHW driver"
+#else /* NO_HARDWARE */
+			         "(Is INIT complete?)"
+#endif /* NO_HARDWARE */
+			         );
+		}
+	}
+
+	return 0;
+}
+
+#if defined(SUPPORT_RGX) && defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+
+/*************************************************************************/ /*!
+ Power data DebugFS entry
+*/ /**************************************************************************/
+
+static PVRSRV_ERROR SendPowerCounterCommand(PVRSRV_DEVICE_NODE* psDeviceNode,
+                                            RGXFWIF_COUNTER_DUMP_REQUEST eRequestType,
+                                            IMG_UINT32 *pui32kCCBCommandSlot)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFWIF_KCCB_CMD sCounterDumpCmd;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCounterDumpCmd.eCmdType = RGXFWIF_KCCB_CMD_COUNTER_DUMP;
+	sCounterDumpCmd.uCmdData.sCounterDumpConfigData.eCounterDumpRequest = eRequestType;
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sCounterDumpCmd,
+				PDUMP_FLAGS_CONTINUOUS,
+				pui32kCCBCommandSlot);
+	PVR_LOG_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot");
+
+	return eError;
+}
+
+static int _DebugPowerDataDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	int eError = 0;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "power counter data was requested!"));
+		return -EIO;
+	}
+
+	OSLockAcquire(psDevInfo->hCounterDumpingLock);
+
+	eError = SendPowerCounterCommand(psDeviceNode,
+									 RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+									 &ui32kCCBCommandSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psDevInfo->hCounterDumpingLock);
+		return -EIO;
+	}
+
+	/* Wait for FW complete completion */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo,
+									  ui32kCCBCommandSlot,
+									  PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+		OSLockRelease(psDevInfo->hCounterDumpingLock);
+		return -EIO;
+	}
+
+	/* Read back the buffer */
+	{
+		IMG_UINT32* pui32PowerBuffer;
+		IMG_UINT32 ui32NumOfRegs, ui32SamplePeriod, ui32NumOfCores;
+		IMG_UINT32 i, j;
+
+		if (!psDevInfo->psCounterBufferMemDesc)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Counter buffer not allocated!"));
+			OSLockRelease(psDevInfo->hCounterDumpingLock);
+			return -EINVAL;
+		}
+
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCounterBufferMemDesc,
+										  (void**)&pui32PowerBuffer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "DevmemAcquireCpuVirtAddr");
+			OSLockRelease(psDevInfo->hCounterDumpingLock);
+			return -EIO;
+		}
+
+		RGXFwSharedMemCacheOpExec(pui32PowerBuffer, PAGE_SIZE, PVRSRV_CACHE_OP_INVALIDATE);
+
+		ui32NumOfRegs = *pui32PowerBuffer++;
+		ui32SamplePeriod = *pui32PowerBuffer++;
+		ui32NumOfCores = *pui32PowerBuffer++;
+		PVR_DPF((PVR_DBG_MESSAGE, "Number of power counters: %u.", ui32NumOfRegs));
+
+		if (ui32NumOfCores == 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "No GPU cores enabled!"));
+			eError = -EINVAL;
+		}
+
+		if (ui32NumOfRegs && ui32NumOfCores)
+		{
+			DIPrintf(psEntry, "Power counter data for device\n");
+			DIPrintf(psEntry, "Sample period: 0x%08x\n", ui32SamplePeriod);
+
+			for (i = 0; i < ui32NumOfRegs; i++)
+			{
+				IMG_UINT32 ui32High, ui32Low;
+				IMG_UINT32 ui32RegOffset = *pui32PowerBuffer++;
+				IMG_UINT32 ui32NumOfInstances = *pui32PowerBuffer++;
+
+				PVR_ASSERT(ui32NumOfInstances);
+
+				DIPrintf(psEntry, "0x%08x:", ui32RegOffset);
+
+				for (j = 0; j < ui32NumOfInstances * ui32NumOfCores; j++)
+				{
+					ui32Low = *pui32PowerBuffer++;
+#if defined(RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_BIT_MASK)
+					if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CATURIX_XTP_TOP_INFRASTRUCTURE))
+					{
+						/* Power counters have 32-bit range */
+						DIPrintf(psEntry, " 0x%08x", ui32Low);
+					}
+					else
+#endif
+					{
+						/* Power counters have 64-bit range */
+						ui32High = *pui32PowerBuffer++;
+
+						DIPrintf(psEntry, " 0x%016" IMG_UINT64_FMTSPECx,
+								 (IMG_UINT64) ui32Low | (IMG_UINT64) ui32High << 32);
+					}
+				}
+
+				DIPrintf(psEntry, "\n");
+			}
+		}
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psCounterBufferMemDesc);
+	}
+
+	OSLockRelease(psDevInfo->hCounterDumpingLock);
+
+	return eError;
+}
+
+static IMG_INT64 PowerDataSet(const IMG_CHAR __user *pcBuffer,
+                              IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                              void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*)pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_COUNTER_DUMP_REQUEST eRequest;
+	IMG_UINT32 ui32kCCBCommandSlot;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "power counter data was requested!"));
+		return -EIO;
+	}
+
+	if (pcBuffer[0] == '1')
+	{
+		eRequest = RGXFWIF_PWR_COUNTER_DUMP_START;
+	}
+	else if (pcBuffer[0] == '0')
+	{
+		eRequest = RGXFWIF_PWR_COUNTER_DUMP_STOP;
+	}
+	else
+	{
+		return -EINVAL;
+	}
+
+	OSLockAcquire(psDevInfo->hCounterDumpingLock);
+
+	SendPowerCounterCommand(psDeviceNode,
+	                        eRequest,
+	                        &ui32kCCBCommandSlot);
+
+	OSLockRelease(psDevInfo->hCounterDumpingLock);
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+#endif /* defined(SUPPORT_RGX) && defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+/*************************************************************************/ /*!
+ Status DebugFS entry
+*/ /**************************************************************************/
+
+static void *_DebugStatusCompare_AnyVaCb(PVRSRV_DEVICE_NODE *psDevNode,
+										 va_list va)
+{
+	IMG_UINT64 *pui64CurrentPosition = va_arg(va, IMG_UINT64 *);
+	IMG_UINT64 ui64Position = va_arg(va, IMG_UINT64);
+	IMG_UINT64 ui64CurrentPosition = *pui64CurrentPosition;
+
+	(*pui64CurrentPosition)++;
+
+	return (ui64CurrentPosition == ui64Position) ? psDevNode : NULL;
+}
+
+static void *_DebugStatusDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =  List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+										  _DebugStatusCompare_AnyVaCb,
+										  &uiCurrentPosition,
+										  *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+static void _DebugStatusDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+}
+
+static void *_DebugStatusDINext(OSDI_IMPL_ENTRY *psEntry,
+								 void *pvData,
+								 IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	(*pui64Pos)++;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =  List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+										  _DebugStatusCompare_AnyVaCb,
+										  &uiCurrentPosition,
+										  *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+#ifdef SUPPORT_RGX
+	if (psDeviceNode && !PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+		if (psDevInfo && psDevInfo->pfnGetGpuUtilStats)
+		{
+			PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+			PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+			if (eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+			{
+				PVRSRV_ERROR eError;
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+				static IMG_BOOL bFirstTime = IMG_TRUE;
+#endif
+
+				OSLockAcquire(psDevInfo->hGpuUtilStatsLock);
+
+				eError = psDevInfo->pfnGetGpuUtilStats(psDeviceNode,
+													   psDebugInfo->hGpuUtilUserDebugFS,
+													   &psDevInfo->sGpuUtilStats);
+
+				OSLockRelease(psDevInfo->hGpuUtilStatsLock);
+
+				if (eError != PVRSRV_OK)
+				{
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+					if (bFirstTime)
+					{
+						bFirstTime = IMG_FALSE;
+#endif	/* defined(EMULATOR) || defined(VIRTUAL_PLATFORM) */
+					PVR_DPF((PVR_DBG_ERROR,
+					        "%s: Failed to get GPU statistics (%s)",
+					        __func__, PVRSRVGetErrorString(eError)));
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+					}
+#endif	/* defined(EMULATOR) || defined(VIRTUAL_PLATFORM) */
+				}
+			}
+		}
+	}
+#endif
+
+	return psDeviceNode;
+}
+
+static int _DebugStatusDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	if (pvData == DI_START_TOKEN)
+	{
+		PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+
+		if (psPVRSRVData != NULL)
+		{
+			switch (psPVRSRVData->eServicesState)
+			{
+				case PVRSRV_SERVICES_STATE_OK:
+					DIPrintf(psEntry, "Driver Status:   OK\n");
+					break;
+				case PVRSRV_SERVICES_STATE_BAD:
+					DIPrintf(psEntry, "Driver Status:   BAD\n");
+					break;
+				case PVRSRV_SERVICES_STATE_UNDEFINED:
+					DIPrintf(psEntry, "Driver Status:   UNDEFINED\n");
+					break;
+				default:
+					DIPrintf(psEntry, "Driver Status:   UNKNOWN (%d)\n",
+					         psPVRSRVData->eServicesState);
+					break;
+			}
+		}
+	}
+	else if (pvData != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+		IMG_CHAR           *pszStatus = "";
+		IMG_CHAR           *pszReason = "";
+		PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus;
+		PVRSRV_DEVICE_HEALTH_REASON eHealthReason;
+
+		DIPrintf(psEntry, "\nDevice ID: %u:%d\n", psDeviceNode->sDevId.ui32InternalID,
+		                                          psDeviceNode->sDevId.i32KernelDeviceID);
+
+		/* Update the health status now if possible... */
+		if (psDeviceNode->pfnUpdateHealthStatus)
+		{
+			psDeviceNode->pfnUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+		}
+		eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+		eHealthReason = OSAtomicRead(&psDeviceNode->eHealthReason);
+
+		switch (eHealthStatus)
+		{
+			case PVRSRV_DEVICE_HEALTH_STATUS_OK:  pszStatus = "OK";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:  pszStatus = "NOT RESPONDING";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:  pszStatus = "DEAD";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:  pszStatus = "FAULT";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:  pszStatus = "UNDEFINED";  break;
+			default:  pszStatus = "UNKNOWN";  break;
+		}
+
+		switch (eHealthReason)
+		{
+			case PVRSRV_DEVICE_HEALTH_REASON_NONE:  pszReason = "";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_ASSERTED:  pszReason = " (Asserted)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING:  pszReason = " (Poll failing)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS:  pszReason = " (Global Event Object timeouts rising)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT:  pszReason = " (KCCB offset invalid)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED:  pszReason = " (KCCB stalled)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_IDLING:  pszReason = " (Idling)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_RESTARTING:  pszReason = " (Restarting)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS:  pszReason = " (Missing interrupts)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR:  pszReason = " (PCI error)";  break;
+			default:  pszReason = " (Unknown reason)";  break;
+		}
+
+		DIPrintf(psEntry, "Firmware Status: %s%s\n", pszStatus, pszReason);
+		if (PVRSRV_ERROR_LIMIT_REACHED)
+		{
+			DIPrintf(psEntry, "Server Errors:   %d+\n", IMG_UINT32_MAX);
+		}
+		else
+		{
+			DIPrintf(psEntry, "Server Errors:   %d\n", PVRSRV_KM_ERRORS);
+		}
+
+
+		/* Write other useful stats to aid the test cycle... */
+		if (psDeviceNode->pvDevice != NULL)
+		{
+#ifdef SUPPORT_RGX
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+			const RGXFWIF_HWRINFOBUF *psHWRInfoBuf;
+			const RGXFWIF_SYSDATA *psFwSysData;
+
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+			psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+			psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+#ifdef PVRSRV_DEBUG_LISR_EXECUTION
+			/* Show the detected #LISR, #MISR scheduled calls */
+			DIPrintf(psEntry, "RGX #LISR: %" IMG_UINT64_FMTSPEC "\n", psDeviceNode->ui64nLISR);
+			DIPrintf(psEntry, "RGX #MISR: %" IMG_UINT64_FMTSPEC "\n", psDeviceNode->ui64nMISR);
+#endif /* PVRSRV_DEBUG_LISR_EXECUTION */
+
+			/* Calculate the number of HWR events in total across all the DMs... */
+			if (psHWRInfoBuf != NULL)
+			{
+				IMG_UINT32 ui32HWREventCount = 0;
+				IMG_UINT32 ui32CRREventCount = 0;
+				IMG_UINT32 ui32DMIndex;
+
+				for (ui32DMIndex = 0; ui32DMIndex < RGXFWIF_DM_MAX; ui32DMIndex++)
+				{
+					ui32HWREventCount += psHWRInfoBuf->aui32HwrDmLockedUpCount[ui32DMIndex];
+					ui32CRREventCount += psHWRInfoBuf->aui32HwrDmOverranCount[ui32DMIndex];
+				}
+
+				DIPrintf(psEntry, "HWR Event Count: %d\n", ui32HWREventCount);
+				DIPrintf(psEntry, "CRR Event Count: %d\n", ui32CRREventCount);
+#ifdef PVRSRV_STALLED_CCB_ACTION
+				/* Write the number of Sync Lockup Recovery (SLR) events... */
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested,
+				                           INVALIDATE);
+				DIPrintf(psEntry, "SLR Event Count: %d\n", psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested);
+#endif /* PVRSRV_STALLED_CCB_ACTION */
+			}
+
+			/* Show error counts */
+			DIPrintf(psEntry, "WGP Error Count: %d\n", psDevInfo->sErrorCounts.ui32WGPErrorCount);
+			DIPrintf(psEntry, "TRP Error Count: %d\n", psDevInfo->sErrorCounts.ui32TRPErrorCount);
+
+			/*
+			 * Guest drivers do not support the following functionality:
+			 *	- Perform actual on-chip fw tracing.
+			 *	- Collect actual on-chip GPU utilization stats.
+			 *	- Perform actual on-chip GPU power/dvfs management.
+			 *	- As a result no more information can be provided.
+			 */
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				if (psFwSysData != NULL)
+				{
+					DIPrintf(psEntry, "FWF Event Count: %d\n", psFwSysData->ui32FWFaults);
+				}
+
+				/* Write the number of APM events... */
+				DIPrintf(psEntry, "APM Event Count: %d\n", psDevInfo->ui32ActivePMReqTotal);
+
+				/* Write the current GPU Utilisation values... */
+				if (psDevInfo->pfnGetGpuUtilStats &&
+					eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+				{
+					PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+					RGXFWIF_GPU_UTIL_STATS *psGpuUtilStats = &psDevInfo->sGpuUtilStats;
+
+					OSLockAcquire(psDevInfo->hGpuUtilStatsLock);
+
+					if ((IMG_UINT32)psGpuUtilStats->ui64GpuStatCumulative)
+					{
+						const IMG_CHAR *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+						IMG_UINT64 util;
+						IMG_UINT32 rem;
+						IMG_UINT32 ui32DriverID;
+						RGXFWIF_DM eDM;
+						IMG_INT    iDM_Util = 0;
+
+						if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+						{
+							apszDmNames[RGXFWIF_DM_TDM] = "2D";
+						}
+
+						util = 100 * psGpuUtilStats->ui64GpuStatActive;
+						util = OSDivide64(util, (IMG_UINT32)psGpuUtilStats->ui64GpuStatCumulative, &rem);
+
+						DIPrintf(psEntry, "GPU Utilisation: %u%%\n", (IMG_UINT32)util);
+
+						DIPrintf(psEntry, "DM Utilisation:");
+
+						FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+						{
+							DIPrintf(psEntry, "  VM%u", ui32DriverID);
+						}
+
+						DIPrintf(psEntry, "\n");
+
+						for (eDM = RGXFWIF_DM_TDM; eDM < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; eDM++,iDM_Util++)
+						{
+							DIPrintf(psEntry, "        %5s: ", apszDmNames[eDM]);
+
+							FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+							{
+								IMG_UINT32 uiDivisor = (IMG_UINT32)psGpuUtilStats->aaui64DMOSStatCumulative[iDM_Util][ui32DriverID];
+
+								if (uiDivisor == 0U)
+								{
+									DIPrintf(psEntry, "   - ");
+									continue;
+								}
+
+								util = 100 * psGpuUtilStats->aaui64DMOSStatActive[iDM_Util][ui32DriverID];
+								util = OSDivide64(util, uiDivisor, &rem);
+
+								DIPrintf(psEntry, "%3u%% ", (IMG_UINT32)util);
+							}
+
+
+							DIPrintf(psEntry, "\n");
+						}
+					}
+					else
+					{
+						DIPrintf(psEntry, "GPU Utilisation: -\n");
+					}
+
+					OSLockRelease(psDevInfo->hGpuUtilStatsLock);
+
+				}
+			}
+#endif /* SUPPORT_RGX */
+		}
+	}
+
+	return 0;
+}
+
+#if defined(DEBUG)
+static IMG_INT64 DebugStatusSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[0] == 'k' || pcBuffer[0] == 'K', -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	psPVRSRVData->eServicesState = PVRSRV_SERVICES_STATE_BAD;
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+#endif
+
+const IMG_CHAR *PVRSRVGetDebugDevStateString(PVRSRV_DEVICE_STATE eDevState)
+{
+	static const char *const _pszDeviceStateStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_STATE_LIST
+	#undef X
+	};
+
+	if (eDevState < 0 || eDevState >= PVRSRV_DEVICE_STATE_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceStateStrings[eDevState];
+}
+
+
+const IMG_CHAR *PVRSRVGetDebugHealthStatusString(PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus)
+{
+	static const char *const _pszDeviceHealthStatusStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_HEALTH_STATUS_LIST
+	#undef X
+	};
+
+	if (eHealthStatus < 0 || eHealthStatus >= PVRSRV_DEVICE_HEALTH_STATUS_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceHealthStatusStrings[eHealthStatus];
+}
+
+
+const IMG_CHAR *PVRSRVGetDebugHealthReasonString(PVRSRV_DEVICE_HEALTH_REASON eHealthReason)
+{
+	static const char *const _pszDeviceHealthReasonStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_HEALTH_REASON_LIST
+	#undef X
+	};
+
+	if (eHealthReason < 0 || eHealthReason >= PVRSRV_DEVICE_HEALTH_REASON_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceHealthReasonStrings[eHealthReason];
+}
+
+/*************************************************************************/ /*!
+ Dump Debug DebugFS entry
+*/ /**************************************************************************/
+
+static int _DebugDumpDebugDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDeviceNode->pvDevice != NULL)
+	{
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX,
+		                   _DumpDebugDIPrintfWrapper, psEntry);
+	}
+
+	return 0;
+}
+
+#ifdef SUPPORT_RGX
+
+/*************************************************************************/ /*!
+ Firmware Trace DebugFS entry
+*/ /**************************************************************************/
+
+static int _DebugFWTraceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDevInfo != NULL)
+	{
+		RGXDumpFirmwareTrace(_DumpDebugDIPrintfWrapper, psEntry, psDevInfo);
+	}
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+ Firmware Translated Page Tables DebugFS entry
+*/ /**************************************************************************/
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+static int _FirmwareMappingsDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32FwVA;
+	IMG_UINT32 ui32FwPageSize;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	psDeviceNode = DIGetPrivData(psEntry);
+
+	if ((psDeviceNode == NULL) ||
+	    (psDeviceNode->pvDevice == NULL) ||
+	    (((PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice)->psKernelMMUCtx == NULL))
+	{
+		/* The Kernel MMU context containing the Firmware mappings is not initialised */
+		return 0;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	DIPrintf(psEntry, "+-----------------+------------------------+------------------------+--------------+\n"
+					  "|    Firmware     |           CPU          |         Device         |      PTE     |\n"
+					  "| Virtual Address |    Physical Address    |    Physical Address    |     Flags    |\n"
+					  "+-----------------+------------------------+------------------------+              +\n");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		DIPrintf(psEntry, "|                                               RI/XI = Read / Execution Inhibit   |\n"
+		                  "|                                               C     = Cache Coherent             |\n"
+		                  "|                                               D     = Dirty Page Table Entry     |\n"
+		                  "|                                               V     = Valid Page Table Entry     |\n"
+		                  "|                                               G     = Global Page Table Entry    |\n"
+		                  "+-----------------+------------------------+------------------------+--------------+\n");
+
+		/* MIPS uses the same page size as the OS */
+		ui32FwPageSize = OSGetPageSize();
+	}
+	else
+#endif
+	{
+		DIPrintf(psEntry, "|                                               P     = Pending Page Table Entry   |\n"
+		                  "|                                               PM    = Parameter Manager Source   |\n"
+		                  "|                                               B     = Bypass SLC                 |\n"
+		                  "|                                               C     = Cache Coherent             |\n"
+		                  "|                                               RW/RO = Device Access Rights       |\n"
+		                  "|                                               V     = Valid Page Table Entry     |\n"
+		                  "+-----------------+------------------------+------------------------+--------------+\n");
+
+		ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+	}
+
+	FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+	{
+		IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) ((RGX_FIRMWARE_RAW_HEAP_BASE +
+		                             (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE)) & UINT_MAX);
+		IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_SIZE & UINT_MAX);
+
+		DIPrintf(psEntry, "|                                       OS ID %u                                    |\n"
+						  "+-----------------+------------------------+------------------------+--------------+\n", ui32DriverID);
+
+		for (ui32FwVA = ui32FwHeapBase;
+		     ui32FwVA < ui32FwHeapEnd;
+		     ui32FwVA += ui32FwPageSize)
+		{
+			PVRSRV_ERROR eError;
+			IMG_UINT64 ui64PTE = 0U;
+			IMG_CPU_PHYADDR sCpuPA = {0U};
+			IMG_DEV_PHYADDR sDevPA = {0U};
+
+			eError = RGXGetFwMapping(psDevInfo, ui32FwVA, &sCpuPA, &sDevPA, &ui64PTE);
+
+			if (eError == PVRSRV_OK)
+			{
+				RGXDocumentFwMapping(psDevInfo, _DumpDebugDIPrintfWrapper, psEntry,
+				                     ui32FwVA, sCpuPA, sDevPA, ui64PTE);
+			}
+			else if (eError != PVRSRV_ERROR_DEVICEMEM_NO_MAPPING)
+			{
+				PVR_LOG_ERROR(eError, "RGXGetFwMapping");
+				return -EIO;
+			}
+		}
+
+		DIPrintf(psEntry, "+-----------------+------------------------+------------------------+--------------+\n");
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+		{
+			break;
+		}
+#endif
+	}
+
+	return 0;
+}
+#endif
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+
+static void *_FirmwareGcovDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo != NULL)
+	{
+		if (psDevInfo->psFirmwareGcovBufferMemDesc != NULL)
+		{
+			void *pvCpuVirtAddr;
+			DevmemAcquireCpuVirtAddr(psDevInfo->psFirmwareGcovBufferMemDesc, &pvCpuVirtAddr);
+			return *pui64Pos ? NULL : pvCpuVirtAddr;
+		}
+	}
+
+	return NULL;
+}
+
+static void _FirmwareGcovDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDevInfo != NULL)
+	{
+		if (psDevInfo->psFirmwareGcovBufferMemDesc != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psFirmwareGcovBufferMemDesc);
+		}
+	}
+}
+
+static void *_FirmwareGcovDINext(OSDI_IMPL_ENTRY *psEntry,
+								  void *pvData,
+								  IMG_UINT64 *pui64Pos)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(pui64Pos);
+	return NULL;
+}
+
+static int _FirmwareGcovDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo != NULL)
+	{
+		DIWrite(psEntry, pvData, psDevInfo->ui32FirmwareGcovSize);
+	}
+	return 0;
+}
+
+#endif /* SUPPORT_FIRMWARE_GCOV */
+
+
+#if  defined(SUPPORT_RISCV_GDB)
+#define RISCV_DMI_SIZE  (8U)
+
+static IMG_INT64 _RiscvDmiRead(IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+
+	ui64Count = MIN(RISCV_DMI_SIZE, ui64Count);
+	memcpy(pcBuffer, &psDebugInfo->ui64RiscvDmi, ui64Count);
+
+	return ui64Count;
+}
+
+static IMG_INT64 _RiscvDmiWrite(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: devinfo is NULL", __func__));
+		return 0;
+	}
+
+	ui64Count -= 1; /* Drop `\0` */
+	ui64Count = MIN(RISCV_DMI_SIZE, ui64Count);
+
+	memcpy(&psDebugInfo->ui64RiscvDmi, pcBuffer, ui64Count);
+
+	RGXRiscvDmiOp(psDevInfo, &psDebugInfo->ui64RiscvDmi);
+
+	return ui64Count;
+}
+#endif
+
+#endif /* SUPPORT_RGX */
+
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+
+/*************************************************************************/ /*!
+ Debug level DebugFS entry
+*/ /**************************************************************************/
+
+static int DebugLevelDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DIPrintf(psEntry, "%u\n", OSDebugLevel());
+
+	return 0;
+}
+
+#ifndef __GNUC__
+static int __builtin_ffsl(long int x)
+{
+	for (size_t i = 0; i < sizeof(x) * 8; i++)
+	{
+		if (x & (1 << i))
+		{
+			return i + 1;
+		}
+	}
+	return 0;
+}
+#endif /* __GNUC__ */
+
+static IMG_INT64 DebugLevelSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const IMG_UINT uiMaxBufferSize = 6;
+	IMG_UINT32 ui32Level;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	if (sscanf(pcBuffer, "%u", &ui32Level) == 0)
+	{
+		return -EINVAL;
+	}
+
+	OSSetDebugLevel(ui32Level & ((1 << __builtin_ffsl(DBGPRIV_LAST)) - 1));
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static int VZPriorityDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+	                    -EINVAL);
+
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ai32DriverPriority[ui32DriverID], INVALIDATE);
+	DIPrintf(psEntry, "%u\n", psRuntimeCfg->ai32DriverPriority[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZPrioritySet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32Priority;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32Priority) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverPriorityKM(NULL, psVZDriverData->psDevNode,
+	                                             psVZDriverData->ui32DriverID, ui32Priority);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZTimeSliceIntervalDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	DIPrintf(psEntry, "%u ms (0: disable)\n", psRuntimeCfg->ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT64 VZTimeSliceIntervalSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+										IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32TSIntervalMs;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32TSIntervalMs) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(NULL, psVZDriverData->psDevNode,
+														  ui32TSIntervalMs);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZTimeSliceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+						-EINVAL);
+
+	DIPrintf(psEntry, "%u (0: auto; 1%% to 100%%)\n", psRuntimeCfg->aui32TSPercentage[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZTimeSliceSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+								IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32TSPercentage;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32TSPercentage) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverTimeSliceKM(NULL, psVZDriverData->psDevNode,
+												  psVZDriverData->ui32DriverID, ui32TSPercentage);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZIsolationGroupDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+	                    -EINVAL);
+
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID], INVALIDATE);
+	DIPrintf(psEntry, "%u\n", psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZIsolationGroupSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                     IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32IsolationGroup;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32IsolationGroup) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverIsolationGroupKM(NULL, psVZDriverData->psDevNode,
+	                                                   psVZDriverData->ui32DriverID, ui32IsolationGroup);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZConnectionCooldownPeriodDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	DIPrintf(psEntry, "%u sec\n", psRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+	return 0;
+}
+
+static IMG_INT64 VZConnectionCooldownPeriodSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+										IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32VzConnectionCooldownPeriodInSec;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32VzConnectionCooldownPeriodInSec) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(NULL, psVZDriverData->psDevNode,
+														  ui32VzConnectionCooldownPeriodInSec);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+#endif
+
+PVRSRV_ERROR DebugCommonInitDriver(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPVRSRVData != NULL);
+
+	/*
+	 * The DebugFS entries are designed to work in a single device system but
+	 * this function will be called multiple times in a multi-device system.
+	 * Return an error in this case.
+	 */
+	if (gpsVersionDIEntry)
+	{
+		return -EEXIST;
+	}
+
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnStart = _VersionDIStart,
+			.pfnStop = _VersionDIStop,
+			.pfnNext = _VersionDINext,
+			.pfnShow = _VersionDIShow
+		};
+
+		eError = DICreateEntry("version", NULL, &sIterator, psPVRSRVData,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsVersionDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnStart = _DebugStatusDIStart,
+			.pfnStop = _DebugStatusDIStop,
+			.pfnNext = _DebugStatusDINext,
+			.pfnShow = _DebugStatusDIShow,
+#if defined(DEBUG)
+			.pfnWrite = DebugStatusSet,
+			//'K' expected + Null terminator
+#endif
+			.ui32WriteLenMax= ((1U)+1U)
+		};
+		eError = DICreateEntry("status", NULL, &sIterator, psPVRSRVData,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsStatusDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = DebugLevelDIShow,
+			.pfnWrite = DebugLevelSet,
+			//Max value of 255(3 char) + Null terminator
+			.ui32WriteLenMax =((3U)+1U)
+		};
+		eError = DICreateEntry("debug_level", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsDebugLevelDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+	return PVRSRV_OK;
+
+return_error_:
+	DebugCommonDeInitDriver();
+
+	return eError;
+}
+
+void DebugCommonDeInitDriver(void)
+{
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+	if (gpsDebugLevelDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsDebugLevelDIEntry);
+	}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+
+	if (gpsStatusDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsStatusDIEntry);
+	}
+
+	if (gpsVersionDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsVersionDIEntry);
+	}
+}
+
+PVRSRV_ERROR DebugCommonInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszDeviceId[sizeof("gpu4294967296")];
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSSNPrintf(pszDeviceId, sizeof(pszDeviceId), "gpu%02d",
+	           psDeviceNode->sDevId.ui32InternalID);
+	eError = DICreateGroup(pszDeviceId, NULL, &psDebugInfo->psGroup);
+	PVR_GOTO_IF_ERROR(eError, return_error_);
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	eError = SORgxGpuUtilStatsRegister(&psDebugInfo->hGpuUtilUserDebugFS);
+	PVR_GOTO_IF_ERROR(eError, return_error_);
+#endif
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = _DebugDumpDebugDIShow};
+		eError = DICreateEntry("debug_dump", psDebugInfo->psGroup, &sIterator,
+		                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+		                       &psDebugInfo->psDumpDebugEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+#ifdef SUPPORT_RGX
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		{
+			DI_ITERATOR_CB sIterator = {.pfnShow = _DebugFWTraceDIShow};
+			eError = DICreateEntry("firmware_trace", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWTraceEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+		{
+			DI_ITERATOR_CB sIterator = {
+				.pfnStart = _FirmwareGcovDIStart,
+				.pfnStop = _FirmwareGcovDIStop,
+				.pfnNext = _FirmwareGcovDINext,
+				.pfnShow = _FirmwareGcovDIShow
+			};
+
+			eError = DICreateEntry("firmware_gcov", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWGCOVEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+#endif /* SUPPORT_FIRMWARE_GCOV */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+		{
+			DI_ITERATOR_CB sIterator = {.pfnShow = _FirmwareMappingsDIShow};
+			eError = DICreateEntry("firmware_mappings", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWMappingsEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+#endif
+
+#if  defined(SUPPORT_RISCV_GDB)
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			DI_ITERATOR_CB sIterator = {
+				.pfnRead = _RiscvDmiRead,
+				.pfnWrite = _RiscvDmiWrite,
+				.ui32WriteLenMax = ((RISCV_DMI_SIZE)+1U)
+			};
+			eError = DICreateEntry("riscv_dmi", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+			                       DI_ENTRY_TYPE_RANDOM_ACCESS, &psDebugInfo->psRiscvDmiDIEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+			psDebugInfo->ui64RiscvDmi = 0ULL;
+		}
+#endif /* SUPPORT_VALIDATION || SUPPORT_RISCV_GDB */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+		{
+			eError = DICreateGroup("vz", psDebugInfo->psGroup, &psDebugInfo->psVZGroup);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+
+			{
+				IMG_UINT32 ui32DriverID;
+
+				DI_ITERATOR_CB sPriorityIterator = {
+					.pfnShow = VZPriorityDIShow,
+					.pfnWrite = VZPrioritySet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sTimeSliceIntervalIterator = {
+					.pfnShow = VZTimeSliceIntervalDIShow,
+					.pfnWrite = VZTimeSliceIntervalSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sTimeSliceIterator = {
+					.pfnShow = VZTimeSliceDIShow,
+					.pfnWrite = VZTimeSliceSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sIsolationGroupIterator = {
+					.pfnShow = VZIsolationGroupDIShow,
+					.pfnWrite = VZIsolationGroupSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sVzConnectionCooldownPeriodIterator = {
+					.pfnShow = VZConnectionCooldownPeriodDIShow,
+					.pfnWrite = VZConnectionCooldownPeriodSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+				{
+					IMG_CHAR szDriverID[4];
+					OSSNPrintf(szDriverID, 4, "%u", ui32DriverID);
+
+					eError = DICreateGroup(szDriverID, psDebugInfo->psVZGroup, &psDebugInfo->apsVZDriverGroups[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					psDebugInfo->apsVZDriverData[ui32DriverID] = OSAllocMem(sizeof(PVRSRV_DEVICE_DEBUG_INFO));
+					PVR_GOTO_IF_NOMEM(psDebugInfo->apsVZDriverData[ui32DriverID], eError, return_error_);
+
+					psDebugInfo->apsVZDriverData[ui32DriverID]->psDevNode = psDeviceNode;
+					psDebugInfo->apsVZDriverData[ui32DriverID]->ui32DriverID = ui32DriverID;
+
+					eError = DICreateEntry("priority", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sPriorityIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					eError = DICreateEntry("time_slice", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sTimeSliceIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+					{
+						eError = DICreateEntry("time_slice_interval", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+											   &sTimeSliceIntervalIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+											   &psDebugInfo->psVZDriverTimeSliceIntervalDIEntry);
+						PVR_GOTO_IF_ERROR(eError, return_error_);
+
+						eError = DICreateEntry("vz_connection_cooldown_period", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+											   &sVzConnectionCooldownPeriodIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+											   &psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry);
+						PVR_GOTO_IF_ERROR(eError, return_error_);
+					}
+
+					eError = DICreateEntry("isolation_group", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sIsolationGroupIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+				}
+			}
+		}
+#endif /* defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1 */
+	}
+
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = _DebugPowerDataDIShow,
+			.pfnWrite = PowerDataSet,
+			//Expects '0' or '1' plus Null terminator
+			.ui32WriteLenMax = ((1U)+1U)
+		};
+		eError = DICreateEntry("power_data", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+		                       DI_ENTRY_TYPE_GENERIC, &psDebugInfo->psPowerDataEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif /* SUPPORT_POWER_SAMPLING_VIA_DEBUGFS */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = PVRSRVPowerStatsPrintElements,
+		};
+		eError = DICreateEntry("power_timing_stats", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+		                       DI_ENTRY_TYPE_GENERIC, &psDebugInfo->psPowerTimingStatsEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif
+#endif /* SUPPORT_RGX */
+
+	return PVRSRV_OK;
+
+return_error_:
+	DebugCommonDeInitDevice(psDeviceNode);
+
+	return eError;
+}
+
+void DebugCommonDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	if (psDebugInfo->psPowerTimingStatsEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psPowerTimingStatsEntry);
+		psDebugInfo->psPowerTimingStatsEntry = NULL;
+	}
+#endif
+
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	if (psDebugInfo->psPowerDataEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psPowerDataEntry);
+		psDebugInfo->psPowerDataEntry = NULL;
+	}
+#endif /* SUPPORT_POWER_SAMPLING_VIA_DEBUGFS */
+
+
+#ifdef SUPPORT_RGX
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			if (psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				if (psDebugInfo->psVZDriverTimeSliceIntervalDIEntry != NULL)
+				{
+					DIDestroyEntry(psDebugInfo->psVZDriverTimeSliceIntervalDIEntry);
+					psDebugInfo->psVZDriverTimeSliceIntervalDIEntry = NULL;
+				}
+
+				if (psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry != NULL)
+				{
+					DIDestroyEntry(psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry);
+					psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry = NULL;
+				}
+			}
+
+			if (psDebugInfo->apsVZDriverData[ui32DriverID] != NULL)
+			{
+				OSFreeMem(psDebugInfo->apsVZDriverData[ui32DriverID]);
+				psDebugInfo->apsVZDriverData[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverGroups[ui32DriverID] != NULL)
+			{
+				DIDestroyGroup(psDebugInfo->apsVZDriverGroups[ui32DriverID]);
+				psDebugInfo->apsVZDriverGroups[ui32DriverID] = NULL;
+			}
+		}
+
+		if (psDebugInfo->psVZGroup != NULL)
+		{
+			DIDestroyGroup(psDebugInfo->psVZGroup);
+			psDebugInfo->psVZGroup = NULL;
+		}
+	}
+#endif
+
+	if (psDebugInfo->psFWTraceEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWTraceEntry);
+		psDebugInfo->psFWTraceEntry = NULL;
+	}
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+	if (psDebugInfo->psFWGCOVEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWGCOVEntry);
+		psDebugInfo->psFWGCOVEntry = NULL;
+	}
+#endif
+
+	if (psDebugInfo->psFWMappingsEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWMappingsEntry);
+		psDebugInfo->psFWMappingsEntry = NULL;
+	}
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR) &&
+		(psDebugInfo->psRiscvDmiDIEntry != NULL))
+	{
+		DIDestroyEntry(psDebugInfo->psRiscvDmiDIEntry);
+		psDebugInfo->psRiscvDmiDIEntry = NULL;
+	}
+#endif
+#endif /* SUPPORT_RGX */
+
+	if (psDebugInfo->psDumpDebugEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psDumpDebugEntry);
+		psDebugInfo->psDumpDebugEntry = NULL;
+	}
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	if (psDebugInfo->hGpuUtilUserDebugFS != NULL)
+	{
+		SORgxGpuUtilStatsUnregister(psDebugInfo->hGpuUtilUserDebugFS);
+		psDebugInfo->hGpuUtilUserDebugFS = NULL;
+	}
+#endif /* defined(SUPPORT_RGX) && !defined(NO_HARDWARE) */
+
+	if (psDebugInfo->psGroup != NULL)
+	{
+		DIDestroyGroup(psDebugInfo->psGroup);
+		psDebugInfo->psGroup = NULL;
+	}
+}
+
+/*
+	Appends flags strings to a null-terminated string buffer
+*/
+void DebugCommonFlagStrings(IMG_CHAR *psDesc,
+							IMG_UINT32 ui32DescSize,
+							const IMG_FLAGS2DESC *psConvTable,
+							IMG_UINT32 ui32TableSize,
+							IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (BITMASK_HAS(ui32Flags, psConvTable[ui32Idx].uiFlag))
+		{
+			OSStringLCat(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_heapcfg.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_heapcfg.c
new file mode 100644
index 000000000000..9b53d026bb5d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_heapcfg.c
@@ -0,0 +1,225 @@
+/*************************************************************************/ /*!
+@File           devicemem_heapcfg.c
+@Title          Device Heap Configuration Helper Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+/* our exported API */
+#include "devicemem_heapcfg.h"
+#include "devicemem_utils.h"
+
+#include "device.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+
+#include "connection_server.h"
+
+static INLINE void _CheckBlueprintHeapAlignment(DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint)
+{
+	IMG_UINT32 ui32OSPageSize = OSGetPageSize();
+
+	/* Any heap length should at least match OS page size at the minimum or
+	 * a multiple of OS page size */
+	if ((psHeapBlueprint->uiHeapLength < DEVMEM_HEAP_MINIMUM_SIZE) ||
+		(psHeapBlueprint->uiHeapLength & (ui32OSPageSize - 1)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC
+		         "("IMG_DEVMEM_SIZE_FMTSPEC")",
+		         __func__,
+		         psHeapBlueprint->pszName,
+		         psHeapBlueprint->uiHeapLength,
+		         psHeapBlueprint->uiHeapLength));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Heap Size should always be at least the DevMem minimum size and a "
+		         "multiple of OS Page Size:%u(0x%x)",
+		         ui32OSPageSize, ui32OSPageSize));
+		PVR_ASSERT(psHeapBlueprint->uiHeapLength >= ui32OSPageSize);
+	}
+
+
+	PVR_ASSERT(psHeapBlueprint->uiReservedRegionLength % DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY == 0);
+}
+
+void HeapCfgBlueprintInit(const IMG_CHAR        *pszName,
+	                      IMG_UINT64             ui64HeapBaseAddr,
+	                      IMG_DEVMEM_SIZE_T      uiHeapLength,
+	                      IMG_DEVMEM_SIZE_T      uiReservedRegionLength,
+	                      IMG_UINT32             ui32Log2DataPageSize,
+	                      IMG_UINT32             uiLog2ImportAlignment,
+	                      PFN_HEAP_INIT          pfnInit,
+	                      PFN_HEAP_DEINIT        pfnDeInit,
+	                      DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint)
+{
+	psHeapBlueprint->pszName                = pszName;
+	psHeapBlueprint->sHeapBaseAddr.uiAddr   = ui64HeapBaseAddr;
+	psHeapBlueprint->uiHeapLength           = uiHeapLength;
+	psHeapBlueprint->uiReservedRegionLength = uiReservedRegionLength;
+	psHeapBlueprint->uiLog2DataPageSize     = ui32Log2DataPageSize;
+	psHeapBlueprint->uiLog2ImportAlignment  = uiLog2ImportAlignment;
+	psHeapBlueprint->pfnInit                = pfnInit;
+	psHeapBlueprint->pfnDeInit              = pfnDeInit;
+
+	_CheckBlueprintHeapAlignment(psHeapBlueprint);
+}
+
+PVRSRV_ERROR
+HeapCfgHeapConfigCount(CONNECTION_DATA * psConnection,
+					   const PVRSRV_DEVICE_NODE *psDeviceNode,
+					   IMG_UINT32 *puiNumHeapConfigsOut)
+{
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	*puiNumHeapConfigsOut = psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapCount(CONNECTION_DATA * psConnection,
+				 const PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_UINT32 uiHeapConfigIndex,
+				 IMG_UINT32 *puiNumHeapsOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	*puiNumHeapsOut = psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapConfigName(CONNECTION_DATA * psConnection,
+					  const PVRSRV_DEVICE_NODE *psDeviceNode,
+					  IMG_UINT32 uiHeapConfigIndex,
+					  IMG_UINT32 uiHeapConfigNameBufSz,
+					  IMG_CHAR *pszHeapConfigNameOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	OSSNPrintf(pszHeapConfigNameOut, uiHeapConfigNameBufSz, "%s", psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].pszName);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgGetCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    PFN_HEAP_INIT *ppfnInit,
+                    PFN_HEAP_DEINIT *ppfnDeinit)
+{
+	DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppfnInit, "ppfnInit");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppfnDeinit, "ppfnDeinit");
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	if (uiHeapIndex >= psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+	}
+
+	psHeapBlueprint = &psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].psHeapBlueprintArray[uiHeapIndex];
+
+	*ppfnInit = psHeapBlueprint->pfnInit;
+	*ppfnDeinit = psHeapBlueprint->pfnDeInit;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapDetails(CONNECTION_DATA * psConnection,
+				   const PVRSRV_DEVICE_NODE *psDeviceNode,
+				   IMG_UINT32 uiHeapConfigIndex,
+				   IMG_UINT32 uiHeapIndex,
+				   IMG_UINT32 uiHeapNameBufSz,
+				   IMG_CHAR *pszHeapNameOut,
+				   IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+				   IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+				   IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+				   IMG_UINT32 *puiLog2DataPageSizeOut,
+				   IMG_UINT32 *puiLog2ImportAlignmentOut)
+{
+	DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	if (uiHeapIndex >= psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+	}
+
+	psHeapBlueprint = &psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].psHeapBlueprintArray[uiHeapIndex];
+
+	OSSNPrintf(pszHeapNameOut, uiHeapNameBufSz, "%s", psHeapBlueprint->pszName);
+	*psDevVAddrBaseOut = psHeapBlueprint->sHeapBaseAddr;
+	*puiHeapLengthOut = psHeapBlueprint->uiHeapLength;
+	*puiReservedRegionLengthOut = psHeapBlueprint->uiReservedRegionLength;
+	*puiLog2DataPageSizeOut = psHeapBlueprint->uiLog2DataPageSize;
+	*puiLog2ImportAlignmentOut = psHeapBlueprint->uiLog2ImportAlignment;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_history_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_history_server.c
new file mode 100644
index 000000000000..f28e96fa55e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_history_server.c
@@ -0,0 +1,2305 @@
+/*************************************************************************/ /*!
+@File
+@Title          Devicemem history functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Devicemem history functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "allocmem.h"
+#include "img_defs.h"
+#include "pmr.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "pvr_debug.h"
+#include "devicemem_server.h"
+#include "lock.h"
+#include "di_server.h"
+#include "devicemem_history_server.h"
+#include "pdump_km.h"
+
+
+#if (PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES < 5000)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES is too low.
+#elif (PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES > 250000)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES is too high.
+#else
+#define ALLOCATION_LIST_NUM_ENTRIES PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES
+#endif
+
+
+/* data type to hold an allocation index.
+ * we make it 16 bits wide if possible
+ */
+#if ALLOCATION_LIST_NUM_ENTRIES <= 0xFFFF
+typedef uint16_t ALLOC_INDEX_T;
+#else
+typedef uint32_t ALLOC_INDEX_T;
+#endif
+
+/* a record describing a single allocation known to DeviceMemHistory.
+ * this is an element in a doubly linked list of allocations
+ */
+typedef struct _RECORD_ALLOCATION_
+{
+	/* time when this RECORD_ALLOCATION was created/initialised */
+	IMG_UINT64 ui64CreationTime;
+	/* serial number of the PMR relating to this allocation */
+	IMG_UINT64 ui64Serial;
+	/* base DevVAddr of this allocation */
+	IMG_DEV_VIRTADDR sDevVAddr;
+	/* size in bytes of this allocation */
+	IMG_DEVMEM_SIZE_T uiSize;
+	/* Log2 page size of this allocation's GPU pages */
+	IMG_UINT32 ui32Log2PageSize;
+	/* Process ID (PID) this allocation belongs to */
+	IMG_PID uiPID;
+	/* index of previous allocation in the list */
+	ALLOC_INDEX_T ui32Prev;
+	/* index of next allocation in the list */
+	ALLOC_INDEX_T ui32Next;
+	/* annotation/name of this allocation */
+	IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN];
+} RECORD_ALLOCATION;
+
+/* each command in the circular buffer is prefixed with an 8-bit value
+ * denoting the command type
+ */
+typedef enum _COMMAND_TYPE_
+{
+	COMMAND_TYPE_NONE,
+	COMMAND_TYPE_TIMESTAMP,
+	COMMAND_TYPE_MAP_ALL,
+	COMMAND_TYPE_UNMAP_ALL,
+	COMMAND_TYPE_MAP_RANGE,
+	COMMAND_TYPE_UNMAP_RANGE,
+	/* sentinel value */
+	COMMAND_TYPE_COUNT,
+} COMMAND_TYPE;
+
+/* Timestamp command:
+ * This command is inserted into the circular buffer to provide an updated
+ * timestamp.
+ * The nanosecond-accuracy timestamp is packed into a 56-bit integer, in order
+ * for the whole command to fit into 10 bytes.
+ */
+typedef struct __attribute__((__packed__))_COMMAND_TIMESTAMP_
+{
+	IMG_UINT8 aui8TimeNs[7];
+} COMMAND_TIMESTAMP;
+
+/* MAP_ALL command:
+ * This command denotes the allocation at the given index was wholly mapped
+ * in to the GPU MMU
+ */
+typedef struct __attribute__((__packed__))_COMMAND_MAP_ALL_
+{
+	ALLOC_INDEX_T uiAllocIndex;
+} COMMAND_MAP_ALL;
+
+/* UNMAP_ALL command:
+ * This command denotes the allocation at the given index was wholly unmapped
+ * from the GPU MMU
+ * Note: COMMAND_MAP_ALL and COMMAND_UNMAP_ALL commands have the same layout.
+ */
+typedef COMMAND_MAP_ALL COMMAND_UNMAP_ALL;
+
+// This shift allows room for 512GiB virtual memory regions at 4Kb pages.
+#define VM_RANGE_SHIFT 27
+
+/* packing attributes for the MAP_RANGE command */
+#define MAP_RANGE_MAX_START ((1 << VM_RANGE_SHIFT) - 1)
+#define MAP_RANGE_MAX_RANGE ((1 << VM_RANGE_SHIFT) - 1)
+
+/* MAP_RANGE command:
+ * Denotes a range of pages within the given allocation being mapped.
+ * The range is expressed as [Page Index] + [Page Count]
+ * This information is packed into a 72/88-bit struct, in order to make
+ * the command size 9/11 bytes.
+ */
+
+typedef struct __attribute__((__packed__))_COMMAND_MAP_RANGE_
+{
+	IMG_UINT8 aui8Data[7];
+	ALLOC_INDEX_T uiAllocIndex;
+} COMMAND_MAP_RANGE;
+
+/* UNMAP_RANGE command:
+ * Denotes a range of pages within the given allocation being unmapped.
+ * The range is expressed as [Page Index] + [Page Count]
+ * This information is packed into a 72/88-bit struct, in order to make
+ * the command size 9/11 bytes.
+ */
+typedef COMMAND_MAP_RANGE COMMAND_UNMAP_RANGE;
+
+/* wrapper structure for a command */
+typedef struct __attribute__((__packed__))_COMMAND_WRAPPER_
+{
+	IMG_UINT8 ui8Type;
+	union {
+		COMMAND_TIMESTAMP sTimeStamp;
+		COMMAND_MAP_ALL sMapAll;
+		COMMAND_UNMAP_ALL sUnmapAll;
+		COMMAND_MAP_RANGE sMapRange;
+		COMMAND_UNMAP_RANGE sUnmapRange;
+	} u;
+} COMMAND_WRAPPER;
+
+/* target size for the circular buffer of commands */
+#if (PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 < 5)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 is too low.
+#elif (PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 > 18)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 is too high.
+#else
+#define CIRCULAR_BUFFER_SIZE_KB (1 << PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2)
+#endif
+
+
+/* turn the circular buffer target size into a number of commands */
+#define CIRCULAR_BUFFER_NUM_COMMANDS ((CIRCULAR_BUFFER_SIZE_KB * 1024) / sizeof(COMMAND_WRAPPER))
+
+/* index value denoting the end of a list */
+#define END_OF_LIST 0xFFFFFFFF
+#define ALLOC_INDEX_TO_PTR(psDevHData, idx) (&((psDevHData)->sRecords.pasAllocations[idx]))
+#define CHECK_ALLOC_INDEX(idx) (idx < ALLOCATION_LIST_NUM_ENTRIES)
+
+/* wrapper structure for the allocation records and the commands circular buffer */
+typedef struct _RECORDS_
+{
+	RECORD_ALLOCATION *pasAllocations;
+	IMG_UINT32 ui32AllocationsListHead;
+
+	IMG_UINT32 ui32Head;
+	IMG_UINT32 ui32Tail;
+	COMMAND_WRAPPER *pasCircularBuffer;
+	/* Times the CB has wrapped back to start */
+	IMG_UINT64 ui64CBWrapCount;
+	/* Records of CB commands sent */
+	IMG_UINT64 ui64MapAllCount;//Incremented by InsertMapAllCommand()
+	IMG_UINT64 ui64UnMapAllCount;//Incremented by InsertUnmapAllCommand()
+	IMG_UINT64 ui64MapRangeCount;//Incremented by InsertMapRangeCommand()
+	IMG_UINT64 ui64UnMapRangeCount;//Incremented by InsertUnmapRangeCommand()
+	IMG_UINT64 ui64TimeStampCount;//Incremented by InsertTimeStampCommand()
+} RECORDS;
+
+typedef struct _DEVICEMEM_HISTORY_DATA_
+{
+	RECORDS sRecords;
+	POS_LOCK hLock;
+} DEVICEMEM_HISTORY_DATA;
+
+/* Maximum number of device instances supported. This should be DDK global */
+static DEVICEMEM_HISTORY_DATA *gapsDevicememHistoryData[PVRSRV_MAX_DEVICES] = { NULL };
+
+/* DevmemFindDataFromDev
+ *
+ * Return the address of the associated DEVICEMEM_HISTORY_DATA for the given
+ * device. If psDevNode associated unit is out of range we return NULL.
+ */
+static DEVICEMEM_HISTORY_DATA *DevmemFindDataFromDev(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DEVICEMEM_HISTORY_DATA *psDevmemData = NULL;
+
+	IMG_UINT32 uiUnit = psDevNode->sDevId.ui32InternalID;
+
+	PVR_ASSERT(uiUnit < PVRSRV_MAX_DEVICES);
+	if ((uiUnit < PVRSRV_MAX_DEVICES) && (gapsDevicememHistoryData[uiUnit] != NULL))
+	{
+		psDevmemData = gapsDevicememHistoryData[uiUnit];
+	}
+
+	return psDevmemData;
+}
+
+/* gsDevicememHistoryData is static, hLock is NULL unless
+ * EnablePageFaultDebug is set and DevicememHistoryInitKM()
+ * was called.
+ */
+static void DevicememHistoryLock(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	if (psDevHData->hLock)
+	{
+		OSLockAcquire(psDevHData->hLock);
+	}
+}
+
+static void DevicememHistoryUnlock(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	if (psDevHData->hLock)
+	{
+		OSLockRelease(psDevHData->hLock);
+	}
+}
+
+/* given a time stamp, calculate the age in nanoseconds */
+static IMG_UINT64 _CalculateAge(IMG_UINT64 ui64Now,
+						IMG_UINT64 ui64Then,
+						IMG_UINT64 ui64Max)
+{
+	if (ui64Now >= ui64Then)
+	{
+		/* no clock wrap */
+		return ui64Now - ui64Then;
+	}
+	else
+	{
+		/* clock has wrapped */
+		return (ui64Max - ui64Then) + ui64Now + 1;
+	}
+}
+
+/* AcquireCBSlot:
+ * Acquire the next slot in the circular buffer and
+ * move the circular buffer head along by one
+ * Returns a pointer to the acquired slot.
+ */
+static COMMAND_WRAPPER *AcquireCBSlot(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	COMMAND_WRAPPER *psSlot;
+
+	psSlot = &psDevHData->sRecords.pasCircularBuffer[psDevHData->sRecords.ui32Head];
+
+	psDevHData->sRecords.ui32Head =
+		(psDevHData->sRecords.ui32Head + 1)
+				% CIRCULAR_BUFFER_NUM_COMMANDS;
+
+	if (psDevHData->sRecords.ui32Head == 0)
+	{
+		psDevHData->sRecords.ui64CBWrapCount++;
+	}
+
+	return psSlot;
+}
+
+/* TimeStampPack:
+ * Packs the given timestamp value into the COMMAND_TIMESTAMP structure.
+ * This takes a 64-bit nanosecond timestamp and packs it in to a 56-bit
+ * integer in the COMMAND_TIMESTAMP command.
+ */
+static void TimeStampPack(COMMAND_TIMESTAMP *psTimeStamp, IMG_UINT64 ui64Now)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ARRAY_SIZE(psTimeStamp->aui8TimeNs); i++)
+	{
+		psTimeStamp->aui8TimeNs[i] = ui64Now & 0xFF;
+		ui64Now >>= 8;
+	}
+}
+
+/* packing a 64-bit nanosecond into a 7-byte integer loses the
+ * top 8 bits of data. This must be taken into account when
+ * comparing a full timestamp against an unpacked timestamp
+ */
+#define TIME_STAMP_MASK ((1LLU << 56) - 1)
+#define DO_TIME_STAMP_MASK(ns64) (ns64 & TIME_STAMP_MASK)
+
+/* TimeStampUnpack:
+ * Unpack the timestamp value from the given COMMAND_TIMESTAMP command
+ */
+static IMG_UINT64 TimeStampUnpack(COMMAND_TIMESTAMP *psTimeStamp)
+{
+	IMG_UINT64 ui64TimeNs = 0;
+	IMG_UINT32 i;
+
+	for (i = ARRAY_SIZE(psTimeStamp->aui8TimeNs); i > 0; i--)
+	{
+		ui64TimeNs <<= 8;
+		ui64TimeNs |= (IMG_UINT64) psTimeStamp->aui8TimeNs[i - 1];
+	}
+
+	return ui64TimeNs;
+}
+
+#if defined(PDUMP)
+
+static void EmitPDumpAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32AllocationIndex,
+                                RECORD_ALLOCATION *psAlloc)
+{
+	PDUMPCOMMENT(psDeviceNode,
+			"[SrvPFD] Allocation: %u"
+			" Addr: " IMG_DEV_VIRTADDR_FMTSPEC
+			" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+			" Page size: %u"
+			" PID: %u"
+			" Process: %s"
+			" Name: %s",
+			ui32AllocationIndex,
+			psAlloc->sDevVAddr.uiAddr,
+			psAlloc->uiSize,
+			1U << psAlloc->ui32Log2PageSize,
+			psAlloc->uiPID,
+			OSGetCurrentClientProcessNameKM(),
+			psAlloc->szName);
+}
+
+static void EmitPDumpMapUnmapAll(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 COMMAND_TYPE eType,
+                                 IMG_UINT32 ui32AllocationIndex)
+{
+	const IMG_CHAR *pszOpName;
+
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_ALL:
+			pszOpName = "MAP_ALL";
+			break;
+		case COMMAND_TYPE_UNMAP_ALL:
+			pszOpName = "UNMAP_ALL";
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapAll: Invalid type: %u",
+										eType));
+			return;
+
+	}
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "[SrvPFD] Op: %s Allocation: %u",
+	             pszOpName,
+	             ui32AllocationIndex);
+}
+
+static void EmitPDumpMapUnmapRange(PVRSRV_DEVICE_NODE *psDeviceNode,
+					COMMAND_TYPE eType,
+					IMG_UINT32 ui32AllocationIndex,
+					IMG_UINT32 ui32StartPage,
+					IMG_UINT32 ui32Count)
+{
+	const IMG_CHAR *pszOpName;
+
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_RANGE:
+			pszOpName = "MAP_RANGE";
+			break;
+		case COMMAND_TYPE_UNMAP_RANGE:
+			pszOpName = "UNMAP_RANGE";
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapRange: Invalid type: %u",
+										eType));
+			return;
+	}
+
+	PDUMPCOMMENT(psDeviceNode,
+                 "[SrvPFD] Op: %s Allocation: %u Start Page: %u Count: %u",
+                 pszOpName,
+                 ui32AllocationIndex,
+                 ui32StartPage,
+                 ui32Count);
+}
+
+#endif
+
+/* InsertTimeStampCommand:
+ * Insert a timestamp command into the circular buffer.
+ */
+static void InsertTimeStampCommand(IMG_UINT64 ui64Now, PVRSRV_DEVICE_NODE *psDevNode)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDevNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_TIMESTAMP;
+	psDevHData->sRecords.ui64TimeStampCount++;
+	TimeStampPack(&psCommand->u.sTimeStamp, ui64Now);
+}
+
+/* InsertMapAllCommand:
+ * Insert a "MAP_ALL" command for the given allocation into the circular buffer
+ */
+static void InsertMapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32AllocIndex)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_MAP_ALL;
+	psCommand->u.sMapAll.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64MapAllCount++;
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_MAP_ALL, ui32AllocIndex);
+#endif
+}
+
+/* InsertUnmapAllCommand:
+ * Insert a "UNMAP_ALL" command for the given allocation into the circular buffer
+ */
+static void InsertUnmapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32AllocIndex)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_UNMAP_ALL;
+	psCommand->u.sUnmapAll.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64UnMapAllCount++;
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_UNMAP_ALL, ui32AllocIndex);
+#endif
+}
+
+/* MapRangePack:
+ * Pack the given StartPage and Count values into the 75-bit representation
+ * in the MAP_RANGE command.
+ */
+static void MapRangePack(COMMAND_MAP_RANGE *psMapRange,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+
+{
+	IMG_UINT64 ui64Data;
+	IMG_UINT32 i;
+
+	/* we must encode the data into 54 bits:
+	 *   27 bits for the start page index
+	 *   27 bits for the range
+	*/
+
+	PVR_ASSERT(ui32StartPage <= MAP_RANGE_MAX_START);
+	PVR_ASSERT(ui32Count <= MAP_RANGE_MAX_RANGE);
+
+	ui32StartPage &= MAP_RANGE_MAX_START;
+	ui32Count &= MAP_RANGE_MAX_RANGE;
+
+	ui64Data = (((IMG_UINT64) ui32StartPage) << VM_RANGE_SHIFT) | ui32Count;
+
+	for (i = 0; i < ARRAY_SIZE(psMapRange->aui8Data); i++)
+	{
+		psMapRange->aui8Data[i] = ui64Data & 0xFF;
+		ui64Data >>= 8;
+	}
+}
+
+/* MapRangeUnpack:
+ * Unpack the StartPage and Count values from the 75-bit representation
+ * in the MAP_RANGE command.
+ */
+static void MapRangeUnpack(COMMAND_MAP_RANGE *psMapRange,
+						IMG_UINT32 *pui32StartPage,
+						IMG_UINT32 *pui32Count)
+{
+	IMG_UINT64 ui64Data = 0;
+	IMG_UINT32 i;
+
+	for (i = ARRAY_SIZE(psMapRange->aui8Data); i > 0; i--)
+	{
+		ui64Data <<= 8;
+		ui64Data |= (IMG_UINT64) psMapRange->aui8Data[i - 1];
+	}
+
+	*pui32StartPage =  (IMG_UINT32)(ui64Data >> VM_RANGE_SHIFT);
+	*pui32Count = (IMG_UINT32) ui64Data & (MAP_RANGE_MAX_RANGE);
+}
+
+/* InsertMapRangeCommand:
+ * Insert a MAP_RANGE command into the circular buffer with the given
+ * StartPage and Count values.
+ */
+static void InsertMapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32 ui32AllocIndex,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_MAP_RANGE;
+	psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64MapRangeCount++;
+
+	MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapRange(psDeviceNode,
+	                       COMMAND_TYPE_MAP_RANGE,
+	                       ui32AllocIndex,
+	                       ui32StartPage,
+	                       ui32Count);
+#endif
+}
+
+/* InsertUnmapRangeCommand:
+ * Insert a UNMAP_RANGE command into the circular buffer with the given
+ * StartPage and Count values.
+ */
+static void InsertUnmapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32 ui32AllocIndex,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_UNMAP_RANGE;
+	psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64UnMapRangeCount++;
+
+	MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapRange(psDeviceNode,
+	                       COMMAND_TYPE_UNMAP_RANGE,
+	                       ui32AllocIndex,
+	                       ui32StartPage,
+	                       ui32Count);
+#endif
+}
+
+/* InsertAllocationToList:
+ * Helper function for the allocation list.
+ * Inserts the given allocation at the head of the list, whose current head is
+ * pointed to by pui32ListHead
+ */
+static void InsertAllocationToList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
+{
+	RECORD_ALLOCATION *psAlloc;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = IMG_CONTAINER_OF(pui32ListHead, DEVICEMEM_HISTORY_DATA,
+	                              sRecords.ui32AllocationsListHead);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	if (*pui32ListHead == END_OF_LIST)
+	{
+		/* list is currently empty, so just replace it */
+		*pui32ListHead = ui32Alloc;
+		psAlloc->ui32Next = psAlloc->ui32Prev = *pui32ListHead;
+	}
+	else
+	{
+		RECORD_ALLOCATION *psHeadAlloc;
+		RECORD_ALLOCATION *psTailAlloc;
+
+		psHeadAlloc = ALLOC_INDEX_TO_PTR(psDevHData, *pui32ListHead);
+		psTailAlloc = ALLOC_INDEX_TO_PTR(psDevHData, psHeadAlloc->ui32Prev);
+
+		/* make the new alloc point forwards to the previous head */
+		psAlloc->ui32Next = *pui32ListHead;
+		/* make the new alloc point backwards to the previous tail */
+		psAlloc->ui32Prev = psHeadAlloc->ui32Prev;
+
+		/* the head is now our new alloc */
+		*pui32ListHead = ui32Alloc;
+
+		/* the old head now points back to the new head */
+		psHeadAlloc->ui32Prev = *pui32ListHead;
+
+		/* the tail now points forward to the new head */
+		psTailAlloc->ui32Next = ui32Alloc;
+	}
+}
+
+static void InsertAllocationToBusyList(DEVICEMEM_HISTORY_DATA *psDevHData,
+                                       IMG_UINT32 ui32Alloc)
+{
+	InsertAllocationToList(&psDevHData->sRecords.ui32AllocationsListHead, ui32Alloc);
+}
+
+/* RemoveAllocationFromList:
+ * Helper function for the allocation list.
+ * Removes the given allocation from the list, whose head is
+ * pointed to by pui32ListHead
+ */
+static void RemoveAllocationFromList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
+{
+	RECORD_ALLOCATION *psAlloc;
+
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = IMG_CONTAINER_OF(pui32ListHead, DEVICEMEM_HISTORY_DATA,
+	                              sRecords.ui32AllocationsListHead);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	/* if this is the only element in the list then just make the list empty */
+	if ((*pui32ListHead == ui32Alloc) && (psAlloc->ui32Next == ui32Alloc))
+	{
+		*pui32ListHead = END_OF_LIST;
+	}
+	else
+	{
+		RECORD_ALLOCATION *psPrev, *psNext;
+
+		psPrev = ALLOC_INDEX_TO_PTR(psDevHData, psAlloc->ui32Prev);
+		psNext = ALLOC_INDEX_TO_PTR(psDevHData, psAlloc->ui32Next);
+
+		/* remove the allocation from the list */
+		psPrev->ui32Next = psAlloc->ui32Next;
+		psNext->ui32Prev = psAlloc->ui32Prev;
+
+		/* if this allocation is the head then update the head */
+		if (*pui32ListHead == ui32Alloc)
+		{
+			*pui32ListHead = psAlloc->ui32Prev;
+		}
+	}
+}
+
+static void RemoveAllocationFromBusyList(DEVICEMEM_HISTORY_DATA *psDevHData, IMG_UINT32 ui32Alloc)
+{
+	RemoveAllocationFromList(&psDevHData->sRecords.ui32AllocationsListHead, ui32Alloc);
+}
+
+/* TouchBusyAllocation:
+ * Move the given allocation to the head of the list
+ */
+static void TouchBusyAllocation(DEVICEMEM_HISTORY_DATA *psDevHData, IMG_UINT32 ui32Alloc)
+{
+	RemoveAllocationFromBusyList(psDevHData, ui32Alloc);
+	InsertAllocationToBusyList(psDevHData, ui32Alloc);
+}
+
+/* GetOldestBusyAllocation:
+ * Returns the index of the oldest allocation in the MRU list
+ */
+static IMG_UINT32 GetOldestBusyAllocation(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 ui32Alloc;
+	RECORD_ALLOCATION *psAlloc;
+
+	if (psDevHData != NULL)
+	{
+		ui32Alloc = psDevHData->sRecords.ui32AllocationsListHead;
+	}
+	else
+	{
+		ui32Alloc = END_OF_LIST;	/* Default if no psDevHData */
+	}
+
+	if (ui32Alloc == END_OF_LIST)
+	{
+		return END_OF_LIST;
+	}
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	return psAlloc->ui32Prev;
+}
+
+static IMG_UINT32 GetFreeAllocation(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 ui32Alloc;
+
+	ui32Alloc = GetOldestBusyAllocation(psDevHData);
+
+	return ui32Alloc;
+}
+
+
+/* InitialiseAllocation:
+ * Initialise the given allocation structure with the given properties
+ */
+static void InitialiseAllocation(RECORD_ALLOCATION *psAlloc,
+							const IMG_CHAR *pszName,
+							IMG_UINT64 ui64Serial,
+							IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT32 ui32Log2PageSize)
+{
+	OSStringSafeCopy(psAlloc->szName, pszName, sizeof(psAlloc->szName));
+	psAlloc->ui64Serial = ui64Serial;
+	psAlloc->uiPID = uiPID;
+	psAlloc->sDevVAddr = sDevVAddr;
+	psAlloc->uiSize = uiSize;
+	psAlloc->ui32Log2PageSize = ui32Log2PageSize;
+	psAlloc->ui64CreationTime = OSClockns64();
+}
+
+/* CreateAllocation:
+ * Creates a new allocation with the given properties then outputs the
+ * index of the allocation
+ */
+static PVRSRV_ERROR CreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+							const IMG_CHAR *pszName,
+							IMG_UINT64 ui64Serial,
+							IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 *puiAllocationIndex)
+{
+	IMG_UINT32 ui32Alloc;
+	RECORD_ALLOCATION *psAlloc;
+
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	ui32Alloc = GetFreeAllocation(psDevHData);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	InitialiseAllocation(ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc),
+						pszName,
+						ui64Serial,
+						uiPID,
+						sDevVAddr,
+						uiSize,
+						ui32Log2PageSize);
+
+	/* put the newly initialised allocation at the front of the MRU list */
+	TouchBusyAllocation(psDevHData, ui32Alloc);
+
+	*puiAllocationIndex = ui32Alloc;
+
+#if defined(PDUMP)
+	EmitPDumpAllocation(psDeviceNode, ui32Alloc, psAlloc);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/* MatchAllocation:
+ * Tests if the allocation at the given index matches the supplied properties.
+ * Returns IMG_TRUE if it is a match, otherwise IMG_FALSE.
+ */
+static IMG_BOOL MatchAllocation(DEVICEMEM_HISTORY_DATA *psDevHData,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT64 ui64Serial,
+						IMG_DEV_VIRTADDR sDevVAddr,
+						IMG_DEVMEM_SIZE_T uiSize,
+						const IMG_CHAR *pszName,
+						IMG_UINT32 ui32Log2PageSize)
+{
+	RECORD_ALLOCATION *psAlloc;
+
+	if (psDevHData == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocationIndex);
+
+	return (psAlloc->ui64Serial == ui64Serial) &&
+	       (psAlloc->sDevVAddr.uiAddr == sDevVAddr.uiAddr) &&
+	       (psAlloc->uiSize == uiSize) &&
+	       (psAlloc->ui32Log2PageSize == ui32Log2PageSize) &&
+	       (OSStringNCompare(psAlloc->szName, pszName, DEVMEM_ANNOTATION_MAX_LEN) == 0);
+}
+
+/* FindOrCreateAllocation:
+ * Convenience function.
+ * Given a set of allocation properties (serial, DevVAddr, size, name, etc),
+ * this function will look for an existing record of this allocation and
+ * create the allocation if there is no existing record
+ */
+static PVRSRV_ERROR FindOrCreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_UINT32 ui32AllocationIndexHint,
+							IMG_UINT64 ui64Serial,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char *pszName,
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_PID uiPID,
+							IMG_UINT32 *pui32AllocationIndexOut,
+							IMG_BOOL *pbCreated)
+{
+	IMG_UINT32 ui32AllocationIndex;
+	PVRSRV_ERROR eError;
+
+	if (ui32AllocationIndexHint != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE)
+	{
+		IMG_BOOL bHaveAllocation;
+
+		/* first, try to match against the index given by the client.
+		 * if the caller provided a hint but the allocation record is no longer
+		 * there, it must have been purged, so go ahead and create a new allocation
+		 */
+		bHaveAllocation = MatchAllocation(DevmemFindDataFromDev(psDeviceNode),
+								ui32AllocationIndexHint,
+								ui64Serial,
+								sDevVAddr,
+								uiSize,
+								pszName,
+								ui32Log2PageSize);
+		if (bHaveAllocation)
+		{
+			*pbCreated = IMG_FALSE;
+			*pui32AllocationIndexOut = ui32AllocationIndexHint;
+			return PVRSRV_OK;
+		}
+	}
+
+	/* if there is no record of the allocation then we
+	 * create it now
+	 */
+	eError = CreateAllocation(psDeviceNode,
+					pszName,
+					ui64Serial,
+					uiPID,
+					sDevVAddr,
+					uiSize,
+					ui32Log2PageSize,
+					&ui32AllocationIndex);
+
+	if (eError == PVRSRV_OK)
+	{
+		*pui32AllocationIndexOut = ui32AllocationIndex;
+		*pbCreated = IMG_TRUE;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed to create record for allocation %s",
+								__func__,
+								pszName));
+	}
+
+	return eError;
+}
+
+/* GenerateMapUnmapCommandsForSparsePMR:
+ * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the PMR's
+ * current mapping table
+ *
+ * PMR: The PMR whose mapping table to read.
+ * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
+ * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
+ *
+ * This function goes through every page in the PMR's mapping table and looks for
+ * virtually contiguous ranges to record as being mapped or unmapped.
+ */
+static void GenerateMapUnmapCommandsForSparsePMR(PMR *psPMR,
+							IMG_UINT32 ui32AllocIndex,
+							IMG_BOOL bMap)
+{
+	PMR_MAPPING_TABLE *psMappingTable;
+	IMG_UINT32 ui32DonePages = 0;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 i;
+	IMG_BOOL bInARun = IMG_FALSE;
+	IMG_UINT32 ui32CurrentStart = 0;
+	IMG_UINT32 ui32RunCount = 0;
+
+	psMappingTable = PMR_GetMappingTable(psPMR);
+	ui32NumPages = psMappingTable->ui32NumPhysChunks;
+
+	if (ui32NumPages == 0)
+	{
+		/* nothing to do */
+		return;
+	}
+
+	for (i = 0; i < psMappingTable->ui32NumLogicalChunks; i++)
+	{
+		if (psMappingTable->aui32Translation[i] != TRANSLATION_INVALID)
+		{
+			if (!bInARun)
+			{
+				bInARun = IMG_TRUE;
+				ui32CurrentStart = i;
+				ui32RunCount = 1;
+			}
+			else
+			{
+				ui32RunCount++;
+			}
+		}
+
+		if (bInARun)
+		{
+			/* test if we need to end this current run and generate the command,
+			 * either because the next page is not virtually contiguous
+			 * to the current page, we have reached the maximum range,
+			 * or this is the last page in the mapping table
+			 */
+			if ((psMappingTable->aui32Translation[i] == TRANSLATION_INVALID) ||
+				(ui32RunCount == MAP_RANGE_MAX_RANGE) ||
+				(i == (psMappingTable->ui32NumLogicalChunks - 1)))
+			{
+				if (bMap)
+				{
+					InsertMapRangeCommand(PMR_DeviceNode(psPMR),
+					                      ui32AllocIndex,
+					                      ui32CurrentStart,
+					                      ui32RunCount);
+				}
+				else
+				{
+					InsertUnmapRangeCommand(PMR_DeviceNode(psPMR),
+					                        ui32AllocIndex,
+					                        ui32CurrentStart,
+					                        ui32RunCount);
+				}
+
+				ui32DonePages += ui32RunCount;
+
+				if (ui32DonePages == ui32NumPages)
+				{
+					break;
+				}
+
+				bInARun = IMG_FALSE;
+			}
+		}
+	}
+
+}
+
+/* GenerateMapUnmapCommandsForChangeList:
+ * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the
+ * list of page change (page map or page unmap) indices given.
+ *
+ * ui32NumPages: Number of pages which have changed.
+ * pui32PageList: List of indices of the pages which have changed.
+ * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
+ * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
+ *
+ * This function goes through every page in the list and looks for
+ * virtually contiguous ranges to record as being mapped or unmapped.
+ */
+static void GenerateMapUnmapCommandsForChangeList(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_UINT32 ui32NumPages,
+							IMG_UINT32 *pui32PageList,
+							IMG_UINT32 ui32AllocIndex,
+							IMG_BOOL bMap)
+{
+	IMG_UINT32 i;
+	IMG_BOOL bInARun = IMG_FALSE;
+	IMG_UINT32 ui32CurrentStart = 0;
+	IMG_UINT32 ui32RunCount = 0;
+
+	for (i = 0; i < ui32NumPages; i++)
+	{
+		if (!bInARun)
+		{
+			bInARun = IMG_TRUE;
+			ui32CurrentStart = pui32PageList[i];
+		}
+
+		ui32RunCount++;
+
+		 /* we flush if:
+		 * - the next page in the list is not one greater than the current page
+		 * - this is the last page in the list
+		 * - we have reached the maximum range size
+		 */
+		if ((i == (ui32NumPages - 1)) ||
+			((pui32PageList[i] + 1) != pui32PageList[i + 1]) ||
+			(ui32RunCount == MAP_RANGE_MAX_RANGE))
+		{
+			if (bMap)
+			{
+				InsertMapRangeCommand(psDeviceNode,
+									ui32AllocIndex,
+									ui32CurrentStart,
+									ui32RunCount);
+			}
+			else
+			{
+				InsertUnmapRangeCommand(psDeviceNode,
+									ui32AllocIndex,
+									ui32CurrentStart,
+									ui32RunCount);
+			}
+
+			bInARun = IMG_FALSE;
+			ui32RunCount = 0;
+		}
+	}
+}
+
+/* DevicememHistoryMapKM:
+ * Entry point for when an allocation is mapped into the MMU GPU
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_BOOL bSparse = PMR_IsSparse(psPMR);
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	if (!bSparse)
+	{
+		InsertMapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
+	}
+	else
+	{
+		GenerateMapUnmapCommandsForSparsePMR(psPMR,
+								ui32AllocationIndex,
+								IMG_TRUE);
+	}
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+static void VRangeInsertMapUnmapCommands(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_BOOL bMap,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							const IMG_CHAR *pszName)
+{
+	PVR_UNREFERENCED_PARAMETER(pszName);
+
+	while (ui32NumPages > 0)
+	{
+		IMG_UINT32 ui32PagesToAdd;
+
+		ui32PagesToAdd = MIN(ui32NumPages, MAP_RANGE_MAX_RANGE);
+
+		if (ui32StartPage > MAP_RANGE_MAX_START)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Cannot record %s range beginning at page "
+									"%u on allocation %s",
+									bMap ? "map" : "unmap",
+									ui32StartPage,
+									pszName));
+			return;
+		}
+
+		if (bMap)
+		{
+			InsertMapRangeCommand(psDeviceNode,
+								ui32AllocationIndex,
+								ui32StartPage,
+								ui32PagesToAdd);
+		}
+		else
+		{
+			InsertUnmapRangeCommand(psDeviceNode,
+								ui32AllocationIndex,
+								ui32StartPage,
+								ui32PagesToAdd);
+		}
+
+		ui32StartPage += ui32PagesToAdd;
+		ui32NumPages -= ui32PagesToAdd;
+	}
+}
+
+PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
+						PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_DEV_VIRTADDR sBaseDevVAddr,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32NumPages,
+						IMG_DEVMEM_SIZE_T uiAllocSize,
+						const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+						IMG_UINT32 ui32Log2PageSize,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+							ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(psDeviceNode,
+						ui32AllocationIndex,
+						0,
+						sBaseDevVAddr,
+						uiAllocSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	VRangeInsertMapUnmapCommands(psDeviceNode,
+						IMG_TRUE,
+						ui32AllocationIndex,
+						sBaseDevVAddr,
+						ui32StartPage,
+						ui32NumPages,
+						szName);
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+
+}
+
+PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
+						PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_DEV_VIRTADDR sBaseDevVAddr,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32NumPages,
+						IMG_DEVMEM_SIZE_T uiAllocSize,
+						const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+						IMG_UINT32 ui32Log2PageSize,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+							ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(psDeviceNode,
+						ui32AllocationIndex,
+						0,
+						sBaseDevVAddr,
+						uiAllocSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	VRangeInsertMapUnmapCommands(psDeviceNode,
+						IMG_FALSE,
+						ui32AllocationIndex,
+						sBaseDevVAddr,
+						ui32StartPage,
+						ui32NumPages,
+						szName);
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+
+
+/* DevicememHistoryUnmapKM:
+ * Entry point for when an allocation is unmapped from the MMU GPU
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_BOOL bSparse = PMR_IsSparse(psPMR);
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	if (!bSparse)
+	{
+		InsertUnmapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
+	}
+	else
+	{
+		GenerateMapUnmapCommandsForSparsePMR(psPMR,
+								ui32AllocationIndex,
+								IMG_FALSE);
+	}
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+/* DevicememHistorySparseChangeKM:
+ * Entry point for when a sparse allocation is changed, such that some of the
+ * pages within the sparse allocation are mapped or unmapped.
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocPageCount: Number of pages which have been mapped.
+ * paui32AllocPageIndices: Indices of pages which have been mapped.
+ * ui32FreePageCount: Number of pages which have been unmapped.
+ * paui32FreePageIndices: Indices of pages which have been unmapped.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocPageCount,
+							IMG_UINT32 *paui32AllocPageIndices,
+							IMG_UINT32 ui32FreePageCount,
+							IMG_UINT32 *paui32FreePageIndices,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if (!PMRValidateSize((IMG_UINT64) ui32AllocPageCount << ui32Log2PageSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				 "PMR size exceeds limit #Chunks: %u ChunkSz %"IMG_UINT64_FMTSPECX"",
+				 ui32AllocPageCount,
+				 (IMG_UINT64) 1ULL << ui32Log2PageSize);
+		return PVRSRV_ERROR_PMR_TOO_LARGE;
+	}
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
+							ui32AllocPageCount,
+							paui32AllocPageIndices,
+							ui32AllocationIndex,
+							IMG_TRUE);
+
+	GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
+							ui32FreePageCount,
+							paui32FreePageIndices,
+							ui32AllocationIndex,
+							IMG_FALSE);
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+
+}
+
+/* CircularBufferIterateStart:
+ * Initialise local state for iterating over the circular buffer
+ */
+static void CircularBufferIterateStart(DEVICEMEM_HISTORY_DATA *psHData, IMG_UINT32 *pui32Head, IMG_UINT32 *pui32Iter)
+{
+	*pui32Head = psHData->sRecords.ui32Head;
+
+	if (*pui32Head != 0)
+	{
+		*pui32Iter = *pui32Head - 1;
+	}
+	else
+	{
+		*pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
+	}
+}
+
+/* CircularBufferIteratePrevious:
+ * Iterate to the previous item in the circular buffer.
+ * This is called repeatedly to iterate over the whole circular buffer.
+ */
+static COMMAND_WRAPPER *CircularBufferIteratePrevious(DEVICEMEM_HISTORY_DATA *psHData,
+							IMG_UINT32 ui32Head,
+							IMG_UINT32 *pui32Iter,
+							COMMAND_TYPE *peType,
+							IMG_BOOL *pbLast)
+{
+	IMG_UINT8 *pui8Header;
+	COMMAND_WRAPPER *psOut = NULL;
+
+	psOut = psHData->sRecords.pasCircularBuffer + *pui32Iter;
+
+	pui8Header = (void *) psOut;
+
+	/* Check the command looks valid.
+	 * this condition should never happen, but check for it anyway
+	 * and try to handle it
+	 */
+	if (*pui8Header >= COMMAND_TYPE_COUNT)
+	{
+		/* invalid header detected. Circular buffer corrupted? */
+		PVR_DPF((PVR_DBG_ERROR, "CircularBufferIteratePrevious: "
+							"Invalid header: %u",
+							*pui8Header));
+		*pbLast = IMG_TRUE;
+		return NULL;
+	}
+
+	*peType = *pui8Header;
+
+	if (*pui32Iter != 0)
+	{
+		(*pui32Iter)--;
+	}
+	else
+	{
+		*pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
+	}
+
+
+	/* inform the caller this is the last command if either we have reached
+	 * the head (where we started) or if we have reached an empty command,
+	 * which means we have covered all populated entries
+	 */
+	if ((*pui32Iter == ui32Head) || (*peType == COMMAND_TYPE_NONE))
+	{
+		/* this is the final iteration */
+		*pbLast = IMG_TRUE;
+	}
+
+	return psOut;
+}
+
+/* MapUnmapCommandGetInfo:
+ * Helper function to get the address and mapping information from a MAP_ALL, UNMAP_ALL,
+ * MAP_RANGE or UNMAP_RANGE command
+ */
+static void MapUnmapCommandGetInfo(DEVICEMEM_HISTORY_DATA *psHData,
+					COMMAND_WRAPPER *psCommand,
+					COMMAND_TYPE eType,
+					IMG_DEV_VIRTADDR *psDevVAddrStart,
+					IMG_DEV_VIRTADDR *psDevVAddrEnd,
+					IMG_BOOL *pbMap,
+					IMG_UINT32 *pui32AllocIndex)
+{
+	if ((eType == COMMAND_TYPE_MAP_ALL) || ((eType == COMMAND_TYPE_UNMAP_ALL)))
+	{
+		COMMAND_MAP_ALL *psMapAll = &psCommand->u.sMapAll;
+		RECORD_ALLOCATION *psAlloc;
+
+		*pbMap = (eType == COMMAND_TYPE_MAP_ALL);
+		*pui32AllocIndex = psMapAll->uiAllocIndex;
+
+		psAlloc = ALLOC_INDEX_TO_PTR(psHData, psMapAll->uiAllocIndex);
+
+		*psDevVAddrStart = psAlloc->sDevVAddr;
+		psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr + psAlloc->uiSize - 1;
+	}
+	else if ((eType == COMMAND_TYPE_MAP_RANGE) || ((eType == COMMAND_TYPE_UNMAP_RANGE)))
+	{
+		COMMAND_MAP_RANGE *psMapRange = &psCommand->u.sMapRange;
+		RECORD_ALLOCATION *psAlloc;
+		IMG_UINT32 ui32StartPage, ui32Count;
+
+		*pbMap = (eType == COMMAND_TYPE_MAP_RANGE);
+		*pui32AllocIndex = psMapRange->uiAllocIndex;
+
+		psAlloc = ALLOC_INDEX_TO_PTR(psHData, psMapRange->uiAllocIndex);
+
+		MapRangeUnpack(psMapRange, &ui32StartPage, &ui32Count);
+
+		psDevVAddrStart->uiAddr = psAlloc->sDevVAddr.uiAddr +
+				((1ULL << psAlloc->ui32Log2PageSize) * ui32StartPage);
+
+		psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr +
+				((1ULL << psAlloc->ui32Log2PageSize) * ui32Count) - 1;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid command type: %u",
+								__func__,
+								eType));
+	}
+}
+
+void DevicememHistoryDumpRecordStats(PVRSRV_DEVICE_NODE *psDevNode,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+	psDevHData = DevmemFindDataFromDev(psDevNode);
+
+	if (psDevHData)
+	{
+		PVR_DUMPDEBUG_LOG("    DevmemHistoryRecordStats -"
+							  " CBWC:%"IMG_UINT64_FMTSPEC
+							  " MAC:%"IMG_UINT64_FMTSPEC
+							  " UMAC:%"IMG_UINT64_FMTSPEC
+							  " MRC:%"IMG_UINT64_FMTSPEC
+							  " UMRC:%"IMG_UINT64_FMTSPEC
+							  " TSC:%"IMG_UINT64_FMTSPEC
+							  " MAX:%"IMG_UINT64_FMTSPEC
+							  " CHD:%u",
+							  psDevHData->sRecords.ui64CBWrapCount,
+							  psDevHData->sRecords.ui64MapAllCount,
+							  psDevHData->sRecords.ui64UnMapAllCount,
+							  psDevHData->sRecords.ui64MapRangeCount,
+							  psDevHData->sRecords.ui64UnMapRangeCount,
+							  psDevHData->sRecords.ui64TimeStampCount,
+							  (IMG_UINT64)CIRCULAR_BUFFER_NUM_COMMANDS,
+							  psDevHData->sRecords.ui32Head);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("    DevmemHistoryRecordStats - None");
+	}
+}
+
+/* DevicememHistoryQuery:
+ * Entry point for rgxdebug to look up addresses relating to a page fault
+ */
+IMG_BOOL DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
+                               DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+                               IMG_UINT32 ui32PageSizeBytes,
+                               IMG_BOOL bMatchAnyAllocInPage)
+{
+	IMG_UINT32 ui32Head, ui32Iter;
+	COMMAND_TYPE eType = COMMAND_TYPE_NONE;
+	COMMAND_WRAPPER *psCommand = NULL;
+	IMG_BOOL bLast = IMG_FALSE;
+	IMG_UINT64 ui64StartTime = OSClockns64();
+	IMG_UINT64 ui64TimeNs = 0;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	/* initialise the results count for the caller */
+	psQueryOut->ui32NumResults = 0;
+	psQueryOut->ui64SearchCount = 0;
+
+	psDevHData = DevmemFindDataFromDev(psQueryIn->psDevNode);
+
+	if (psDevHData == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	/* if the search is constrained to a particular PID then we
+	 * first search the list of allocations to see if this
+	 * PID is known to us
+	 */
+	if (psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY)
+	{
+		IMG_UINT32 ui32Alloc;
+		ui32Alloc = psDevHData->sRecords.ui32AllocationsListHead;
+
+		while (ui32Alloc != END_OF_LIST)
+		{
+			RECORD_ALLOCATION *psAlloc;
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+			if (psAlloc->uiPID == psQueryIn->uiPID)
+			{
+				goto found_pid;
+			}
+
+			if (ui32Alloc == psDevHData->sRecords.ui32AllocationsListHead)
+			{
+				/* gone through whole list */
+				break;
+			}
+		}
+
+		/* PID not found, so we do not have any suitable data for this
+		 * page fault
+		 */
+		goto out_unlock;
+	}
+
+found_pid:
+
+	CircularBufferIterateStart(psDevHData, &ui32Head, &ui32Iter);
+
+	while (!bLast)
+	{
+		psCommand = CircularBufferIteratePrevious(psDevHData, ui32Head, &ui32Iter, &eType, &bLast);
+		if (psCommand == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: CircularBufferIteratePrevious returned NULL psCommand",
+			         __func__));
+			return IMG_FALSE;
+		}
+
+		if (eType == COMMAND_TYPE_TIMESTAMP)
+		{
+			ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
+			continue;
+		}
+
+		if ((eType == COMMAND_TYPE_MAP_ALL) ||
+			(eType == COMMAND_TYPE_UNMAP_ALL) ||
+			(eType == COMMAND_TYPE_MAP_RANGE) ||
+			(eType == COMMAND_TYPE_UNMAP_RANGE))
+		{
+			RECORD_ALLOCATION *psAlloc;
+			IMG_DEV_VIRTADDR sAllocStartAddrOrig, sAllocEndAddrOrig;
+			IMG_DEV_VIRTADDR sAllocStartAddr, sAllocEndAddr;
+			IMG_BOOL bMap;
+			IMG_UINT32 ui32AllocIndex;
+
+			MapUnmapCommandGetInfo(psDevHData,
+			                psCommand,
+							eType,
+							&sAllocStartAddrOrig,
+							&sAllocEndAddrOrig,
+							&bMap,
+							&ui32AllocIndex);
+
+			sAllocStartAddr = sAllocStartAddrOrig;
+			sAllocEndAddr = sAllocEndAddrOrig;
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocIndex);
+
+			/* skip this command if we need to search within
+			 * a particular PID, and this allocation is not from
+			 * that PID
+			 */
+			if ((psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY) &&
+				(psAlloc->uiPID != psQueryIn->uiPID))
+			{
+				continue;
+			}
+
+			/* if the allocation was created after this event, then this
+			 * event must be for an old/removed allocation, so skip it
+			 */
+			if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
+			{
+				continue;
+			}
+
+			psQueryOut->ui64SearchCount++;
+
+			/* if the caller wants us to match any allocation in the
+			 * same page as the allocation then tweak the real start/end
+			 * addresses of the allocation here
+			 */
+			if (bMatchAnyAllocInPage)
+			{
+				sAllocStartAddr.uiAddr = sAllocStartAddr.uiAddr & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
+				sAllocEndAddr.uiAddr = PVR_ALIGN(sAllocEndAddr.uiAddr, (IMG_UINT64)ui32PageSizeBytes);
+			}
+
+			if ((psQueryIn->sDevVAddr.uiAddr >= sAllocStartAddr.uiAddr) &&
+				(psQueryIn->sDevVAddr.uiAddr <  sAllocEndAddr.uiAddr))
+			{
+				DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult = &psQueryOut->sResults[psQueryOut->ui32NumResults];
+
+				OSStringSafeCopy(psResult->szString, psAlloc->szName, sizeof(psResult->szString));
+				psResult->sBaseDevVAddr = psAlloc->sDevVAddr;
+				psResult->uiSize = psAlloc->uiSize;
+				psResult->bMap = bMap;
+				psResult->ui64Age = _CalculateAge(ui64StartTime, ui64TimeNs, TIME_STAMP_MASK);
+				psResult->ui64When = ui64TimeNs;
+				/* write the responsible PID in the placeholder */
+				psResult->sProcessInfo.uiPID = psAlloc->uiPID;
+
+				if ((eType == COMMAND_TYPE_MAP_ALL) || (eType == COMMAND_TYPE_UNMAP_ALL))
+				{
+					psResult->bRange = IMG_FALSE;
+					psResult->bAll = IMG_TRUE;
+				}
+				else
+				{
+					psResult->bRange = IMG_TRUE;
+					MapRangeUnpack(&psCommand->u.sMapRange,
+										&psResult->ui32StartPage,
+										&psResult->ui32PageCount);
+					psResult->bAll = (psResult->ui32PageCount * (1U << psAlloc->ui32Log2PageSize))
+											== psAlloc->uiSize;
+					psResult->sMapStartAddr = sAllocStartAddrOrig;
+					psResult->sMapEndAddr = sAllocEndAddrOrig;
+				}
+
+				psQueryOut->ui32NumResults++;
+
+				if (psQueryOut->ui32NumResults == DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS)
+				{
+					break;
+				}
+			}
+		}
+	}
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return psQueryOut->ui32NumResults > 0;
+}
+
+#if defined(SUPPORT_RGX)
+static void DeviceMemHistoryFmt(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN],
+							IMG_PID uiPID,
+							const IMG_CHAR *pszName,
+							const IMG_CHAR *pszAction,
+							IMG_DEV_VIRTADDR sDevVAddrStart,
+							IMG_DEV_VIRTADDR sDevVAddrEnd,
+							IMG_UINT64 ui64TimeNs)
+{
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+				/* PID NAME MAP/UNMAP MIN-MAX SIZE AbsUS AgeUS*/
+				"%04u %-40s %-10s "
+				IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC " "
+				"0x%08" IMG_UINT64_FMTSPECX " "
+				"%013" IMG_UINT64_FMTSPEC, /* 13 digits is over 2 hours of ns */
+				uiPID,
+				pszName,
+				pszAction,
+				sDevVAddrStart.uiAddr,
+				sDevVAddrEnd.uiAddr,
+				sDevVAddrEnd.uiAddr - sDevVAddrStart.uiAddr + 1,
+				ui64TimeNs);
+}
+
+static void DeviceMemHistoryFmtHeader(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+				"%-4s %-40s %-6s   %10s   %10s   %8s %13s",
+				"PID",
+				"NAME",
+				"ACTION",
+				"ADDR MIN",
+				"ADDR MAX",
+				"SIZE",
+				"ABS NS");
+}
+
+static const char *CommandTypeToString(COMMAND_TYPE eType)
+{
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_ALL:
+			return "MapAll";
+		case COMMAND_TYPE_UNMAP_ALL:
+			return "UnmapAll";
+		case COMMAND_TYPE_MAP_RANGE:
+			return "MapRange";
+		case COMMAND_TYPE_UNMAP_RANGE:
+			return "UnmapRange";
+		case COMMAND_TYPE_TIMESTAMP:
+			return "TimeStamp";
+		default:
+			return "???";
+	}
+}
+
+static void DevicememHistoryPrintAll(OSDI_IMPL_ENTRY *psEntry)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	IMG_UINT32 ui32Iter;
+	IMG_UINT32 ui32Head;
+	IMG_BOOL bLast = IMG_FALSE;
+	IMG_UINT64 ui64TimeNs = 0;
+	IMG_UINT64 ui64StartTime = OSClockns64();
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+
+	DeviceMemHistoryFmtHeader(szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	CircularBufferIterateStart(psDevHData, &ui32Head, &ui32Iter);
+
+	while (!bLast)
+	{
+		COMMAND_WRAPPER *psCommand;
+		COMMAND_TYPE eType = COMMAND_TYPE_NONE;
+
+		psCommand = CircularBufferIteratePrevious(psDevHData, ui32Head, &ui32Iter,
+		                                          &eType, &bLast);
+		if (psCommand == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: CircularBufferIteratePrevious returned NULL psCommand",
+			         __func__));
+			return;
+		}
+
+		if (eType == COMMAND_TYPE_TIMESTAMP)
+		{
+			ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
+			continue;
+		}
+
+
+		if ((eType == COMMAND_TYPE_MAP_ALL) ||
+			(eType == COMMAND_TYPE_UNMAP_ALL) ||
+			(eType == COMMAND_TYPE_MAP_RANGE) ||
+			(eType == COMMAND_TYPE_UNMAP_RANGE))
+		{
+			RECORD_ALLOCATION *psAlloc;
+			IMG_DEV_VIRTADDR sDevVAddrStart, sDevVAddrEnd;
+			IMG_BOOL bMap;
+			IMG_UINT32 ui32AllocIndex;
+
+			MapUnmapCommandGetInfo(psDevHData,
+			                       psCommand,
+			                       eType,
+			                       &sDevVAddrStart,
+			                       &sDevVAddrEnd,
+			                       &bMap,
+			                       &ui32AllocIndex);
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocIndex);
+
+			if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
+			{
+				/* if this event relates to an allocation we
+				 * are no longer tracking then do not print it
+				 */
+				continue;
+			}
+
+			DeviceMemHistoryFmt(szBuffer,
+			                    psAlloc->uiPID,
+			                    psAlloc->szName,
+			                    CommandTypeToString(eType),
+			                    sDevVAddrStart,
+			                    sDevVAddrEnd,
+			                    ui64TimeNs);
+
+			DIPrintf(psEntry, "%s\n", szBuffer);
+		}
+	}
+
+	DIPrintf(psEntry, "\nTimestamp reference: %013" IMG_UINT64_FMTSPEC "\n",
+	         ui64StartTime);
+}
+
+static int DevicememHistoryPrintAllWrapper(OSDI_IMPL_ENTRY *psEntry,
+                                           void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)DIGetPrivData(psEntry);
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	/* Get the backing store associated with the device. If we are
+	 * called before the device has been started (i.e. FW loaded)
+	 * then we haven't yet had this data allocated.
+	 * Return to provide a NULL data stream to the consumer.
+	 */
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+	if (psDevHData == NULL)
+	{
+		return 0;
+	}
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	DevicememHistoryLock(psDevHData);
+	DevicememHistoryPrintAll(psEntry);
+	DevicememHistoryUnlock(psDevHData);
+
+	return 0;
+}
+#endif	/* defined(SUPPORT_RGX) */
+
+static PVRSRV_ERROR CreateRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	psDevHData->sRecords.pasAllocations =
+			OSAllocZMemNoStats(sizeof(RECORD_ALLOCATION) * ALLOCATION_LIST_NUM_ENTRIES);
+
+	PVR_RETURN_IF_NOMEM(psDevHData->sRecords.pasAllocations);
+
+	/* Allocated and initialise the circular buffer with zeros so every
+	 * command is initialised as a command of type COMMAND_TYPE_NONE. */
+	psDevHData->sRecords.pasCircularBuffer =
+			OSAllocZMemNoStats(sizeof(COMMAND_WRAPPER) * CIRCULAR_BUFFER_NUM_COMMANDS);
+
+	if (psDevHData->sRecords.pasCircularBuffer == NULL)
+	{
+		OSFreeMemNoStats(psDevHData->sRecords.pasAllocations);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void DestroyRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	OSFreeMemNoStats(psDevHData->sRecords.pasCircularBuffer);
+	OSFreeMemNoStats(psDevHData->sRecords.pasAllocations);
+}
+
+static void InitialiseRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 i;
+
+	/* initialise the allocations list */
+
+	psDevHData->sRecords.pasAllocations[0].ui32Prev = ALLOCATION_LIST_NUM_ENTRIES - 1;
+	psDevHData->sRecords.pasAllocations[0].ui32Next = 1;
+
+	for (i = 1; i < ALLOCATION_LIST_NUM_ENTRIES; i++)
+	{
+		psDevHData->sRecords.pasAllocations[i].ui32Prev = i - 1;
+		psDevHData->sRecords.pasAllocations[i].ui32Next = i + 1;
+	}
+
+	psDevHData->sRecords.pasAllocations[ALLOCATION_LIST_NUM_ENTRIES - 1].ui32Next = 0;
+
+	psDevHData->sRecords.ui32AllocationsListHead = 0;
+}
+
+static void DevicememHistoryDevDeInitUnit(IMG_UINT32 uiUnit);
+static PVRSRV_ERROR DevicememHistoryDevInitUnit(IMG_UINT32 uiUnit);
+
+PVRSRV_ERROR DevicememHistoryInitKM(void)
+{
+	IMG_UINT32 ui;
+
+	/* Zero-fill the gapsDevicememHistoryData array entries */
+	for (ui = 0; ui < PVRSRV_MAX_DEVICES; ui++)
+	{
+		gapsDevicememHistoryData[ui] = NULL;
+	}
+
+	return PVRSRV_OK;
+}
+
+void DevicememHistoryDeInitKM(void)
+{
+	IMG_UINT32 uiUnit;
+
+	/* Iterate over all potential units and remove their data.
+	 * DI entry is removed by DevicememHistoryDeviceDestroy()
+	 */
+	for (uiUnit = 0; uiUnit < PVRSRV_MAX_DEVICES; uiUnit++)
+	{
+		DevicememHistoryDevDeInitUnit(uiUnit);
+	}
+}
+
+/* Allocate DEVICEMEM_HISTORY_DATA entry for the specified unit */
+static PVRSRV_ERROR DevicememHistoryDevInitUnit(IMG_UINT32 uiUnit)
+{
+	PVRSRV_ERROR eError;
+	DEVICEMEM_HISTORY_DATA *psDevicememHistoryData;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		PVR_LOG_RETURN_IF_FALSE(uiUnit < PVRSRV_MAX_DEVICES, "Invalid Unit",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Valid unit, try and allocate and fill all structure members */
+
+	psDevicememHistoryData = OSAllocZMemNoStats(sizeof(DEVICEMEM_HISTORY_DATA));
+	PVR_RETURN_IF_NOMEM(psDevicememHistoryData);
+
+	eError = OSLockCreate(&psDevicememHistoryData->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", err_lock);
+
+	eError = CreateRecords(psDevicememHistoryData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateRecords", err_allocations);
+
+	InitialiseRecords(psDevicememHistoryData);
+
+	gapsDevicememHistoryData[uiUnit] = psDevicememHistoryData;
+
+	return PVRSRV_OK;
+
+err_allocations:
+	OSLockDestroy(psDevicememHistoryData->hLock);
+	psDevicememHistoryData->hLock = NULL;
+err_lock:
+	OSFreeMemNoStats(psDevicememHistoryData);
+	return eError;
+}
+
+/* Allocate DI entry for specified psDeviceNode */
+PVRSRV_ERROR DevicememHistoryDeviceCreate(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiUnit = psDeviceNode->sDevId.ui32InternalID;
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_DEBUG_INFO *psDevDebugInfo = &psDeviceNode->sDebugInfo;
+	DI_ITERATOR_CB sIterator = {.pfnShow = DevicememHistoryPrintAllWrapper};
+#endif
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Create the DI entry for the device's devicemem_history handle */
+
+
+#if defined(SUPPORT_RGX)
+	eError = DICreateEntry("devicemem_history", psDevDebugInfo->psGroup,
+	                       &sIterator, psDeviceNode,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psDevDebugInfo->psDevMemEntry);
+#endif	/* defined(SUPPORT_RGX) */
+
+	return eError;
+}
+
+/* Allocate the DEVICEMEM_HISTORY_DATA for specified psDeviceNode */
+PVRSRV_ERROR DevicememHistoryDeviceInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiUnit = psDeviceNode->sDevId.ui32InternalID;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = DevicememHistoryDevInitUnit(uiUnit);
+
+	return eError;
+}
+
+static void DevicememHistoryDevDeInitUnit(IMG_UINT32 uiUnit)
+{
+	DEVICEMEM_HISTORY_DATA *psDevicememHistoryData;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return;
+	}
+
+	psDevicememHistoryData = gapsDevicememHistoryData[uiUnit];
+
+	if (psDevicememHistoryData == NULL)
+	{
+		return;
+	}
+
+	DestroyRecords(psDevicememHistoryData);
+
+	if (psDevicememHistoryData->hLock != NULL)
+	{
+		OSLockDestroy(psDevicememHistoryData->hLock);
+		psDevicememHistoryData->hLock = NULL;
+	}
+
+	OSFreeMemNoStats(psDevicememHistoryData);
+	gapsDevicememHistoryData[uiUnit] = NULL;
+}
+
+void DevicememHistoryDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_DEBUG_INFO *psDevDebugInfo = &psDeviceNode->sDebugInfo;
+
+	/* Remove the DI entry associated with this device */
+	DIDestroyEntry(psDevDebugInfo->psDevMemEntry);
+#endif	/* defined(SUPPORT_RGX) */
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_server.c
new file mode 100644
index 000000000000..e26a7ab3b814
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_server.c
@@ -0,0 +1,4224 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server-side component of the Device Memory Management.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* our exported API */
+#include "devicemem_server.h"
+#include "devicemem_utils.h"
+#include "devicemem.h"
+
+#include "device.h" /* For device node */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "mmu_common.h"
+#include "pdump_km.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdumpdesc.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+#include "lock.h"
+
+#include "pvrsrv.h" /* for PVRSRVGetPVRSRVData() */
+
+
+#define DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE (1 << 0)
+#define DEVMEMHEAP_REFCOUNT_MIN 1
+#define DEVMEMHEAP_REFCOUNT_MAX IMG_INT32_MAX
+#define DEVMEMRESERVATION_ACQUISITION_MIN 0
+#define DEVMEMRESERVATION_ACQUISITION_MAX IMG_INT32_MAX
+#define DEVMEMRESERVATION_REFCOUNT_MAX IMG_INT32_MAX
+#define DEVMEM_PMR_RSRV_NODE_REFCOUNT_MAX 2
+#define DEVMEMCTX_REFCOUNT_MIN 1
+#define DEVMEMCTX_REFCOUNT_MAX IMG_INT32_MAX
+
+#define DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT 2
+
+struct _DEVMEMINT_CTX_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* MMU common code needs to have a context. There's a one-to-one
+	   correspondence between device memory context and MMU context,
+	   but we have the abstraction here so that we don't need to care
+	   what the MMU does with its context, and the MMU code need not
+	   know about us at all. */
+	MMU_CONTEXT *psMMUContext;
+
+	/* This handle is for devices that require notification when a new
+	   memory context is created and they need to store private data that
+	   is associated with the context. */
+	IMG_HANDLE hPrivData;
+
+	/* The following tracks UM applications that need to be notified of a
+	 * page fault.
+	 * Access to this list is protected by lock defined on a device node:
+	 * PVRSRV_DEVICE_NODE::hPageFaultNotifyLock. */
+	DLLIST_NODE sProcessNotifyListHead;
+	/* The following is a node for the list of registered devmem contexts.
+	 * Access to this list is protected by lock defined on a device node:
+	 * PVRSRV_DEVICE_NODE::hPageFaultNotifyLock. */
+	DLLIST_NODE sPageFaultNotifyListElem;
+
+	/* Device virtual address of a page fault on this context */
+	IMG_DEV_VIRTADDR sFaultAddress;
+
+	/* Bitfield stating which heaps were created on this context. */
+	IMG_UINT64 uiCreatedHeaps;
+
+	/* Context's reference count */
+	ATOMIC_T hRefCount;
+
+	/* General purpose flags */
+	IMG_UINT32 ui32Flags;
+};
+
+struct _DEVMEMINT_CTX_EXPORT_
+{
+	DEVMEMINT_CTX *psDevmemCtx;
+	PMR *psPMR;
+	ATOMIC_T hRefCount;
+	DLLIST_NODE sNode;
+};
+
+struct _DEVMEMINT_HEAP_
+{
+	struct _DEVMEMINT_CTX_ *psDevmemCtx;
+	IMG_DEV_VIRTADDR sBaseAddr;
+	IMG_DEV_VIRTADDR sLastAddr;
+
+	/* Private data for callback functions */
+	IMG_HANDLE hPrivData;
+
+	/* Callback function init */
+	PFN_HEAP_INIT pfnInit;
+
+	/* Callback function deinit */
+	PFN_HEAP_DEINIT pfnDeInit;
+
+	/* Heap's reference count */
+	ATOMIC_T uiRefCount;
+
+	/* Page shift of the heap */
+	IMG_UINT32 uiLog2PageSize;
+
+	/* Copy of the heap index from Device Heap Configuration module */
+	IMG_UINT32 uiHeapIndex;
+};
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+typedef enum _DEVMEM_RESERVATION_TYPE_
+{
+	DEVMEM_RES_TYPE_NONE,
+	DEVMEM_RES_TYPE_STD, /* Devmem Reservation */
+	DEVMEM_RES_TYPE_X, /* DevmemX Reservation */
+	DEVMEM_RES_TYPE_ZOMBIE /* Reservation Freed but still has references */
+} DEVMEM_RESERVATION_TYPE;
+
+/* Forward declare for callback definitions */
+typedef struct _DEVMEM_PMR_RSRV_NODE_ DEVMEM_PMR_RSRV_NODE;
+
+/* Forward declare for Remap functions */
+static PVRSRV_ERROR _RefPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode);
+static void _UnrefAndMaybeDestroyPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode);
+
+/* Callback definition for remaps */
+typedef PVRSRV_ERROR (*PFN_REMAP_PAGE_FN)(PMR *psPMR,
+                                          DLLIST_NODE *psMappingListHead,
+                                          DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                                          IMG_UINT32 ui32LogicalPgOffset);
+/* Callback definition for checking the page is in the mapping */
+typedef IMG_BOOL (*PFN_IS_PAGE_IN_MAP_FN)(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, IMG_UINT32 ui32LogicalPgOffset);
+/* Callback definition for free */
+typedef void (*PFN_FREE_PARENT_STRUCT_FN)(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode);
+
+/* Node linked to PMR for remap.
+ * Also used to control lifetime of struct in which it is embedded.
+ */
+typedef struct _DEVMEM_PMR_RSRV_NODE_
+{
+	/* List node given to the PMR to create the reservation link.*/
+	DLLIST_NODE sLinkNode;
+	/* Type of reservation, determines re-map path, should be
+	 * protected by reservation lock
+	 */
+	DEVMEM_RESERVATION_TYPE eType;
+
+	ATOMIC_T iRefCount;
+	PFN_REMAP_PAGE_FN pfnRemapPage;
+	PFN_IS_PAGE_IN_MAP_FN pfnIsPageInMap;
+	PFN_FREE_PARENT_STRUCT_FN pfnFreeParent;
+
+} DEVMEM_PMR_RSRV_NODE;
+
+/* DevmemX representation of a mapping on the reservation */
+typedef struct _DEVMEMX_MAPPING_
+{
+	/* Link node given to the PMR to create the link for remap */
+	DEVMEM_PMR_RSRV_NODE sPMRRsrvLinkNode;
+	DEVMEMXINT_RESERVATION *psRsrv;
+	PMR *psPMR;
+
+	/* Details of the mapped range */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+	IMG_UINT32 uiPhysPageOffset;
+	IMG_UINT32 uiVirtPageOffset;
+	IMG_UINT32 uiPageCount;
+} DEVMEMX_MAPPING;
+
+typedef struct _DEVMEM_MAPPING_
+{
+	DEVMEM_PMR_RSRV_NODE sPMRRsrvLinkNode;
+	DEVMEMINT_RESERVATION *psRsrv;
+} DEVMEM_MAPPING;
+#endif
+
+/**
+ * Indicates where the reservation will be used.
+ *
+ * Used as locking classes for DevMem reservations to resolve lockdep warnings.
+ */
+typedef enum _RESERVATION_USAGE_
+{
+	RESERVATION_USAGE_SERVER_FW = 0,  // Reservations used by the server / FW (excludes dma_bufs)
+	RESERVATION_USAGE_CLIENT          // Reservations used by client apps (includes dma_bufs)
+} RESERVATION_USAGE;
+
+struct _DEVMEMINT_RESERVATION_
+{
+	struct _DEVMEMINT_HEAP_ *psDevmemHeap;
+	IMG_DEV_VIRTADDR sBase;
+	IMG_DEVMEM_SIZE_T uiLength;
+
+	/* lock used to guard against potential race when freeing reservation, also protects
+	 * parallel operations occurring on the same reservation such as a (un)mapping operation
+	 * from UM and a remap operation from the Linux Kernel
+	 */
+	POS_LOCK hLock;
+	RESERVATION_USAGE eLockClass;
+
+	/* Indicator for ensuring range has not been externally acquired whilst attempting release.
+	 * Protects freeing the reservation while other device resources still have reference such as
+	 * FreeLists and ZSBuffer.
+	 */
+	IMG_INT32 i32DevResAcquisitionCount;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Mapping node representing the mapping on this reservation */
+	DEVMEM_MAPPING *psDevmemMapping;
+#endif
+	/* KM refcount used to protect the lifetime of the reservation itself. This ensures
+	 * synchronisation of use between UM usage and KM usage such as remap paths. We may need
+	 * to keep the reservation alive for these KM paths even once the UM has released it since
+	 * it is used to synchronise operations originating from these KM paths and between UM and KM.
+	 */
+	ATOMIC_T iLifetimeRefCount;
+
+	/* We keep a reference to the single PMR associated with this reservation
+	 * once mapped. When creating the reservation this is null. Used in
+	 * ChangeSparse to validate parameters. We could have a sparse PMR with
+	 * no backing and have it mapped to a reservation.
+	 */
+	PMR *psMappedPMR;
+
+	/* Array of bitfields of size `uiNumPages / MAP_MASK_SHIFT`.
+	 * This array represents the mapping between a PMR (psMappedPMR) and
+	 * the reservation. Each bit represents an index of a physical page - a value
+	 * 1 means the page is mapped and vice versa.
+	 */
+	IMG_UINT8 *pui8Map;
+	#define MAP_MASK_SHIFT 3
+};
+
+/*! Object representing a virtual range reservation and mapping between
+ * the virtual range and a set of PMRs.
+ *
+ * The physical allocations may be mapped entirely or partially to the entire
+ * or partial virtual range. */
+struct _DEVMEMXINT_RESERVATION_
+{
+	/*! Pointer to a device memory heap this reservation is made on. */
+	struct _DEVMEMINT_HEAP_ *psDevmemHeap;
+	/*! Base device virtual address of this reservation. */
+	IMG_DEV_VIRTADDR sBase;
+	/*! Size of this reservation (in bytes). */
+	IMG_DEVMEM_SIZE_T uiLength;
+	/*! Lock for protecting concurrent operations on the mapping. Also protects
+	 * parallel operations occurring on the same reservation such as a (un)mapping operation
+	 * from UM and a remap operation from the Linux Kernel. As such also protects mapping records
+	 * kept for remap operations.
+	 */
+	POS_LOCK hLock;
+
+	/* KM refcount used to protect the lifetime of the reservation itself. This ensures
+	 * synchronisation of use between UM usage and KM usage such as remap paths. We may need
+	 * to keep the reservation alive for these KM paths even once the UM has released it since
+	 * it is used to synchronise operations originating from these KM paths and between UM and KM.
+	 */
+	ATOMIC_T iLifetimeRefCount;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/*! Array of DevmemX Map Ranges of size `uiNumPages`. This array represents how the
+	 *  physical memory is mapped to the virtual range. Each entry in the array
+	 *  represents to one device page which means that one PMR may be spread
+	 *  across many indices and related to many ranges */
+	DEVMEMX_MAPPING *ppsDevmemXMapping[IMG_FLEX_ARRAY_MEMBER];
+#else
+
+	/*! Array of PMRs of size `uiNumPages`. This array represents how the
+	 *  physical memory is mapped to the virtual range. Each entry in the array
+	 *  represents to one device page which means that one PMR may be spread
+	 *  across many indices. */
+	PMR **ppsPMR;
+#endif
+};
+
+struct _DEVMEMINT_PF_NOTIFY_
+{
+	IMG_UINT32  ui32PID;
+	DLLIST_NODE sProcessNotifyListElem;
+};
+
+/** Computes division using log2 of divisor. */
+#define LOG2_DIV(x, log2) ((x) >> (log2))
+
+/** Computes modulo of a power of 2. */
+#define LOG2_MOD(x, log2) ((x) & ((1 << (log2)) - 1))
+
+static INLINE IMG_UINT32 _DevmemReservationPageCount(DEVMEMINT_RESERVATION *psRsrv);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationIsIndexMapped
+@Description    Checks whether a particular index in the reservation has been
+                mapped to a page in psMappedPMR.
+
+@Return         IMG_TRUE if mapped or IMG_FALSE if not.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntReservationIsIndexMapped(DEVMEMINT_RESERVATION *psReservation,
+                                                         IMG_UINT32 ui32Index)
+{
+	IMG_UINT32 ui32MapIndex = LOG2_DIV(ui32Index, MAP_MASK_SHIFT);
+
+	PVR_ASSERT(psReservation != NULL);
+	PVR_ASSERT(ui32Index < _DevmemReservationPageCount(psReservation));
+
+	return BIT_ISSET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationSetMappingIndex
+@Description    Sets an index of the reservation map to indicate a mapped or
+                unmapped PMR page.
+
+@Note           The reservations hLock must be acquired before calling this
+                function.
+
+@Return         None
+*/ /**************************************************************************/
+static void DevmemIntReservationSetMappingIndex(DEVMEMINT_RESERVATION *psReservation,
+                                                IMG_UINT32 ui32Index,
+                                                IMG_BOOL bMap)
+{
+	IMG_UINT32 ui32MapIndex = LOG2_DIV(ui32Index, MAP_MASK_SHIFT);
+
+	PVR_ASSERT(psReservation != NULL);
+	PVR_ASSERT(ui32Index < _DevmemReservationPageCount(psReservation));
+
+	if (bMap)
+	{
+		BIT_SET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+	}
+	else
+	{
+		BIT_UNSET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+	}
+}
+
+static INLINE IMG_UINT32
+_DevmemXReservationPageCount(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	return psRsrv->uiLength >> psRsrv->psDevmemHeap->uiLog2PageSize;
+}
+
+static INLINE IMG_DEV_VIRTADDR
+_DevmemXReservationPageAddress(DEVMEMXINT_RESERVATION *psRsrv, IMG_UINT32 uiVirtPageOffset)
+{
+	IMG_DEV_VIRTADDR sAddr = {
+		.uiAddr = psRsrv->sBase.uiAddr + ((IMG_UINT64)uiVirtPageOffset << psRsrv->psDevmemHeap->uiLog2PageSize)
+	};
+
+	return sAddr;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/****************************************
+ * Map & Remap synchronisation helpers  *
+ ****************************************/
+
+static void
+_AcquireReservationPMRLocks(POS_LOCK hRsrvLock, PMR* psPMR)
+{
+	OSLockAcquire(hRsrvLock);
+	PMRLockPMR(psPMR);
+}
+
+static void
+_AcquireReservationPMRLocksNested(POS_LOCK hRsrvLock, RESERVATION_USAGE eLockClass, PMR* psPMR)
+{
+	OSLockAcquireNested(hRsrvLock, eLockClass);
+	PMRLockPMR(psPMR);
+}
+
+static void
+_ReleaseReservationPMRLocks(POS_LOCK hRsrvLock, PMR* psPMR)
+{
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(hRsrvLock);
+}
+
+static void
+_DeschedAllowRemapToComplete(POS_LOCK hRsrvLock, PMR* psPMR)
+{
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(hRsrvLock);
+	OSReleaseThreadQuanta();
+	OSLockAcquire(hRsrvLock);
+	PMRLockPMR(psPMR);
+}
+
+static void
+_DeschedAllowRemapToCompleteNested(POS_LOCK hRsrvLock, RESERVATION_USAGE eLockClass, PMR* psPMR)
+{
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(hRsrvLock);
+	OSReleaseThreadQuanta();
+	OSLockAcquireNested(hRsrvLock, eLockClass);
+	PMRLockPMR(psPMR);
+}
+
+/*******************
+ * Remap functions *
+ *******************/
+
+static IMG_BOOL
+_CheckMultipleRelatedMappingsExist(DLLIST_NODE *psMappingListHead,
+                                                      IMG_UINT32 ui32LogicalPgOffset)
+{
+	PDLLIST_NODE pNext, pNode;
+	DEVMEM_PMR_RSRV_NODE *psFoundPMRResLinkNode = NULL;
+	IMG_BOOL bMultipleFound = IMG_FALSE;
+
+	dllist_foreach_node(psMappingListHead, pNode, pNext)
+	{
+		DEVMEM_PMR_RSRV_NODE *psPMRResLinkNode = IMG_CONTAINER_OF(pNode, DEVMEM_PMR_RSRV_NODE, sLinkNode);
+
+		if (psPMRResLinkNode->pfnIsPageInMap(psPMRResLinkNode, ui32LogicalPgOffset))
+		{
+			if (psFoundPMRResLinkNode == NULL)
+			{
+				psFoundPMRResLinkNode = psPMRResLinkNode;
+			}
+			else
+			{
+				/* We have found multiple mappings associated with this page,
+				 * we will reject the remap request
+				 */
+				bMultipleFound = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	return bMultipleFound;
+}
+
+static PVRSRV_ERROR
+_pfnDevmemRemapPageInPMR(PMR *psPMR,
+                         DLLIST_NODE *psMappingListHead,
+                         DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                         IMG_UINT32 ui32LogicalPgOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MAPPING* psMapping = IMG_CONTAINER_OF(psPMRRsrvNode, DEVMEM_MAPPING, sPMRRsrvLinkNode);
+	DEVMEMINT_RESERVATION *psRsrv = psMapping->psRsrv;
+	IMG_DEV_VIRTADDR sDevVAddr;
+
+	_AcquireReservationPMRLocksNested(psRsrv->hLock,
+	                                  psRsrv->eLockClass,
+	                                  psPMR);
+
+	PVR_ASSERT(psMapping->sPMRRsrvLinkNode.eType != DEVMEM_RES_TYPE_X);
+
+	/* First check if the Mapping is in Zombie state. This means UM has freed it between us
+	 * finding the related mapping on the PMR list and us preparing to action a remap here.
+	 */
+	if (psMapping->sPMRRsrvLinkNode.eType == DEVMEM_RES_TYPE_ZOMBIE)
+	{
+		/* This mapping has been deleted by UM */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	/* Now we have locked the reservation and the PMR, check if multiple mapping entries exist
+	 * on the list related to the page to remap. If there are multiple, we must reject the
+	 * request to avoid inconsistency in event of remap failure.
+	 */
+	if (_CheckMultipleRelatedMappingsExist(psMappingListHead, ui32LogicalPgOffset))
+	{
+		/* This PMR page has multiple associated mappings, reject migrate */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	sDevVAddr.uiAddr = psRsrv->sBase.uiAddr + ((IMG_UINT64)ui32LogicalPgOffset << psRsrv->psDevmemHeap->uiLog2PageSize);
+
+	eError = MMU_RemapPage(psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+	                       psRsrv->uiFlags,
+	                       sDevVAddr,
+	                       psRsrv->psDevmemHeap->uiLog2PageSize,
+	                       psPMR,
+	                       ui32LogicalPgOffset);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Devmem mapped PMR Remap page failed"));
+	}
+
+_Return:
+	_ReleaseReservationPMRLocks(psRsrv->hLock, psPMR);
+	return eError;
+}
+
+static IMG_BOOL
+_pfnDevmemCheckPMRPageInMappingRange(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, IMG_UINT32 ui32LogicalPgOffset)
+{
+	/* Devmem reservations can have 0 or 1 mappings, hence always return true if called for when mapping present.
+	 * We also don't support sparse mappings for movable pages, so the offset will always be valid for the
+	 * mapping should it exist.
+	 */
+	return IMG_TRUE;
+}
+
+static PVRSRV_ERROR
+_pfnDevmemXRemapPageInPMR(PMR *psPMR,
+                          DLLIST_NODE *psMappingListHead,
+                          DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                          IMG_UINT32 ui32LogicalPgOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMX_MAPPING *psDevmemXMapping = IMG_CONTAINER_OF(psPMRRsrvNode,
+	                                                     DEVMEMX_MAPPING,
+	                                                     sPMRRsrvLinkNode);
+	DEVMEMXINT_RESERVATION *psRsrv = psDevmemXMapping->psRsrv;
+	IMG_UINT32 uiVirtPageOffset;
+	IMG_DEV_VIRTADDR sDevVAddr;
+
+	_AcquireReservationPMRLocks(psRsrv->hLock, psPMR);
+
+	PVR_ASSERT(psDevmemXMapping->sPMRRsrvLinkNode.eType != DEVMEM_RES_TYPE_STD);
+
+	/* First check if the Mapping is in Zombie state. This means UM has freed it between us
+	 * finding the related mapping on the PMR list and us preparing to action a remap here.
+	 */
+	if (psDevmemXMapping->sPMRRsrvLinkNode.eType == DEVMEM_RES_TYPE_ZOMBIE)
+	{
+		/* This mapping has been deleted by UM */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	/* Next check if the mapping range is still valid for the PMR page, this could be changed
+	 * by a UM map overwriting part of an old PMR page range if we were already in the process
+	 * of mapping when remap occurred and we checked for the existence of related mappings.
+	 */
+	if (!psPMRRsrvNode->pfnIsPageInMap(psPMRRsrvNode, ui32LogicalPgOffset))
+	{
+		/* This range is no longer related to the page we wish to migrate. */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	/* Finally, check if multiple mapping entries exist on the list related to the page to remap.
+	 * If there are multiple, we must reject the request to avoid inconsistency in event of remap failure.
+	 */
+	if (_CheckMultipleRelatedMappingsExist(psMappingListHead, ui32LogicalPgOffset))
+	{
+		/* This PMR page has multiple associated mappings, reject migrate */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	uiVirtPageOffset = psDevmemXMapping->uiVirtPageOffset +
+	                   (ui32LogicalPgOffset - psDevmemXMapping->uiPhysPageOffset);
+
+	sDevVAddr = _DevmemXReservationPageAddress(psDevmemXMapping->psRsrv,
+	                                           uiVirtPageOffset);
+
+	eError =
+		MMU_RemapPage(psDevmemXMapping->psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+					  psDevmemXMapping->uiFlags,
+					  sDevVAddr,
+					  psDevmemXMapping->psRsrv->psDevmemHeap->uiLog2PageSize,
+					  psPMR,
+					  ui32LogicalPgOffset);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "DevmemX mapped PMR Remap page failed"));
+	}
+
+
+_Return:
+	_ReleaseReservationPMRLocks(psRsrv->hLock, psPMR);
+	return eError;
+}
+
+static IMG_BOOL
+_pfnDevmemXCheckPMRPageInMappingRange(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, IMG_UINT32 ui32LogicalPgOffset)
+{
+	DEVMEMX_MAPPING *psDevmemXMapping = IMG_CONTAINER_OF(psPMRRsrvNode,
+                                                         DEVMEMX_MAPPING,
+                                                         sPMRRsrvLinkNode);
+
+	if (ui32LogicalPgOffset >= psDevmemXMapping->uiPhysPageOffset + psDevmemXMapping->uiPageCount ||
+		ui32LogicalPgOffset < psDevmemXMapping->uiPhysPageOffset)
+	{
+		/* This range is not related to the page we wish to migrate. We return IMG_FALSE
+		 * because there is no operation to perform on this mapping.
+		 */
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+
+PVRSRV_ERROR
+DevmemIntRemapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset)
+{
+	DEVMEM_PMR_RSRV_NODE *psFoundPMRResLinkNode = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDLLIST_NODE pNext, pNode;
+
+	/* Lock the PMR whilst obtaining nodes from the list but not performing remap actions on them,
+	 * this allows us to protect lifetimes of the attachments */
+	PMRLockPMR(psPMR);
+
+	/* The PMR caller of this function has no perspective as to if a mapping
+	 * on its list is related to the page it wishes to remap and so we must iterate
+	 * over the list checking for related mappings. If we find a related mapping, pass
+	 * it onto the remap function for processing.
+	 */
+	dllist_foreach_node(psMappingListHead, pNode, pNext)
+	{
+		DEVMEM_PMR_RSRV_NODE *psPMRResLinkNode = IMG_CONTAINER_OF(pNode, DEVMEM_PMR_RSRV_NODE, sLinkNode);
+
+		if (psPMRResLinkNode->pfnIsPageInMap(psPMRResLinkNode, ui32LogicalPgOffset))
+		{
+			psFoundPMRResLinkNode = psPMRResLinkNode;
+			break;
+		}
+	}
+
+	if (psFoundPMRResLinkNode)
+	{
+		eError = _RefPMRRsrvNode(psFoundPMRResLinkNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntRemapPageInPMR.", ErrUnlockPMRExit);
+
+		PMRUnlockPMR(psPMR);
+
+		eError = psFoundPMRResLinkNode->pfnRemapPage(psPMR,
+		                                             psMappingListHead,
+		                                             psFoundPMRResLinkNode,
+		                                             ui32LogicalPgOffset);
+		PVR_GOTO_IF_ERROR(eError, ErrUnrefNodeExit);
+
+		_UnrefAndMaybeDestroyPMRRsrvNode(psFoundPMRResLinkNode);
+	}
+	else
+	{
+		PMRUnlockPMR(psPMR);
+	}
+
+	return eError;
+
+ErrUnlockPMRExit:
+	PMRUnlockPMR(psPMR);
+	return eError;
+
+ErrUnrefNodeExit:
+	_UnrefAndMaybeDestroyPMRRsrvNode(psFoundPMRResLinkNode);
+	return eError;
+}
+
+/**********************************************************
+ * DEVMEM_PMR_RSRV_NODE functions and lifetime management *
+ **********************************************************/
+
+static void _InitialisePMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                                   DEVMEM_RESERVATION_TYPE eType,
+                                   PFN_REMAP_PAGE_FN pfnRemapPage,
+                                   PFN_IS_PAGE_IN_MAP_FN pfnIsPageInMap,
+                                   PFN_FREE_PARENT_STRUCT_FN pfnFreeParent)
+{
+	/* Automatically start with a ref */
+	OSAtomicWrite(&psPMRRsrvNode->iRefCount, 1);
+
+	psPMRRsrvNode->eType = eType;
+	psPMRRsrvNode->pfnRemapPage = pfnRemapPage;
+	psPMRRsrvNode->pfnIsPageInMap = pfnIsPageInMap;
+	psPMRRsrvNode->pfnFreeParent = pfnFreeParent;
+}
+
+static
+PVRSRV_ERROR _RefPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psPMRRsrvNode->iRefCount, 1,
+	                                        DEVMEM_PMR_RSRV_NODE_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEM_PMR_RSRV_NODE_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the PMRRsrvNode, "
+		"reference count has overflowed.", __func__));
+		return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+void _UnrefAndMaybeDestroyPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	if (OSAtomicDecrement(&psPMRRsrvNode->iRefCount) == 0)
+	{
+		psPMRRsrvNode->pfnFreeParent(psPMRRsrvNode);
+	}
+}
+
+static
+void _LinkPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+	PMRLinkGPUMapping(psPMR, &psPMRRsrvNode->sLinkNode);
+}
+
+static
+void _UnlinkPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psPMRRsrvNode->eType = DEVMEM_RES_TYPE_ZOMBIE;
+	PMRUnlinkGPUMapping(psPMR, &psPMRRsrvNode->sLinkNode);
+}
+
+/***********************************
+ * Reservation lifetime management *
+ ***********************************/
+
+static PVRSRV_ERROR
+_DevmemReservationRef(DEVMEMINT_RESERVATION *psRsrv)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psRsrv->iLifetimeRefCount, 1,
+	                                        DEVMEMRESERVATION_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the reservation, "
+		"reference count has overflowed.", __func__));
+		return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_DevmemXReservationRef(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psRsrv->iLifetimeRefCount, 1,
+	                                        DEVMEMRESERVATION_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the reservation, "
+		"reference count has overflowed.", __func__));
+		return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+static void
+_DevmemReservationUnrefAndMaybeDestroy(DEVMEMINT_RESERVATION *psRsrv)
+{
+	if (OSAtomicDecrement(&psRsrv->iLifetimeRefCount) == 0)
+	{
+		OSLockDestroy(psRsrv->hLock);
+		OSFreeMem(psRsrv);
+	}
+}
+
+static void
+_DevmemXReservationUnrefAndMaybeDestroy(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	if (OSAtomicDecrement(&psRsrv->iLifetimeRefCount) == 0)
+	{
+		OSLockDestroy(psRsrv->hLock);
+		OSFreeMem(psRsrv);
+	}
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/*****************************
+ * Devmem mapping management *
+ *****************************/
+
+static void _pfnDevmemMappingFreeCallback(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	DEVMEM_MAPPING *psDevmemMapping;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psPMRRsrvNode != NULL, "psPMRRsrvNode");
+
+	psDevmemMapping = IMG_CONTAINER_OF(psPMRRsrvNode, DEVMEM_MAPPING, sPMRRsrvLinkNode);
+
+	_DevmemReservationUnrefAndMaybeDestroy(psDevmemMapping->psRsrv);
+	OSFreeMem(psDevmemMapping);
+}
+
+static DEVMEM_MAPPING* _AllocateDevmemMapping(DEVMEMINT_RESERVATION *psRsrv,
+                                              PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	DEVMEM_MAPPING *psDevmemMapping = OSAllocZMem(sizeof(*psDevmemMapping));
+	if (!psDevmemMapping)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()",
+		"psDevmemMapping",
+		__func__));
+
+		return NULL;
+	}
+
+	eError = _DevmemReservationRef(psRsrv);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psDevmemMapping);
+		return NULL;
+	}
+
+	_InitialisePMRRsrvNode(&psDevmemMapping->sPMRRsrvLinkNode,
+	                       DEVMEM_RES_TYPE_STD,
+	                       _pfnDevmemRemapPageInPMR,
+	                       _pfnDevmemCheckPMRPageInMappingRange,
+	                       _pfnDevmemMappingFreeCallback);
+
+	psDevmemMapping->psRsrv = psRsrv;
+
+	return psDevmemMapping;
+}
+
+/*****************************
+ * DevmemX mapping management *
+ *****************************/
+
+static void _FreePreAllocatedDevmemXMapping(DEVMEMX_MAPPING *psDevmemXMapping)
+{
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevmemXMapping != NULL, "psDevmemXMapping");
+
+	_DevmemXReservationUnrefAndMaybeDestroy(psDevmemXMapping->psRsrv);
+	OSFreeMem(psDevmemXMapping);
+}
+
+static void _pfnDevmemXMappingFreeCallback(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	DEVMEMX_MAPPING *psDevmemXMapping;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psPMRRsrvNode != NULL, "psPMRRsrvNode");
+
+	psDevmemXMapping = IMG_CONTAINER_OF(psPMRRsrvNode,
+	                                    DEVMEMX_MAPPING,
+	                                    sPMRRsrvLinkNode);
+
+	/* This pfn is called from either DestroyDevmemXMapping or the Remap path
+	 * dropping its final reference. At this point in both cases the mapping has
+	 * been removed from the list and by doing so de-initialised. Meaning we
+	 * can treat it as though it is in a pre-allocated state.
+	 */
+	_FreePreAllocatedDevmemXMapping(psDevmemXMapping);
+}
+
+static void _DestroyDevmemXMapping(DEVMEMX_MAPPING *psDevmemXMapping)
+{
+	PMRLockHeldAssert(psDevmemXMapping->psPMR);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevmemXMapping != NULL, "psDevmemXMapping");
+
+	_UnlinkPMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode, psDevmemXMapping->psPMR);
+	_UnrefAndMaybeDestroyPMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode);
+}
+
+static DEVMEMX_MAPPING* _PreAllocateDevmemXMapping(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMX_MAPPING *psDevmemXMapping = OSAllocZMem(sizeof(*psDevmemXMapping));
+	if (!psDevmemXMapping)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()",
+		"psDevmemXMapping",
+		__func__));
+
+		return NULL;
+	}
+
+	/* Take ref and set the reservation as we need to fail early */
+	eError = _DevmemXReservationRef(psRsrv);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psDevmemXMapping);
+		return NULL;
+	}
+	psDevmemXMapping->psRsrv = psRsrv;
+
+	_InitialisePMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode,
+	                       DEVMEM_RES_TYPE_X,
+	                       _pfnDevmemXRemapPageInPMR,
+	                       _pfnDevmemXCheckPMRPageInMappingRange,
+	                       _pfnDevmemXMappingFreeCallback);
+
+	return psDevmemXMapping;
+}
+
+static void _InitialiseDevmemXMapping(DEVMEMX_MAPPING *psDevmemXMapping,
+                                      PMR *psPMR,
+                                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                      IMG_UINT32 uiPhysPageOffset,
+                                      IMG_UINT32 uiVirtPageOffset,
+                                      IMG_UINT32 uiPageCount)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psDevmemXMapping->uiFlags = uiFlags;
+	psDevmemXMapping->uiPhysPageOffset = uiPhysPageOffset;
+	psDevmemXMapping->uiVirtPageOffset = uiVirtPageOffset;
+	psDevmemXMapping->uiPageCount = uiPageCount;
+	psDevmemXMapping->psPMR = psPMR;
+
+	_LinkPMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode, psPMR);
+}
+
+static DEVMEMX_MAPPING* _AllocateDevmemXMapping(DEVMEMXINT_RESERVATION *psRsrv,
+                                                PMR *psPMR,
+                                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                                IMG_UINT32 uiPhysPageOffset,
+                                                IMG_UINT32 uiVirtPageOffset,
+                                                IMG_UINT32 uiPageCount)
+{
+	DEVMEMX_MAPPING* psDevmemXMapping;
+
+	PMRLockHeldAssert(psPMR);
+
+	psDevmemXMapping = _PreAllocateDevmemXMapping(psRsrv);
+	if (!psDevmemXMapping)
+	{
+		return NULL;
+	}
+
+	_InitialiseDevmemXMapping(psDevmemXMapping,
+	                          psPMR,
+	                          uiFlags,
+	                          uiPhysPageOffset,
+	                          uiVirtPageOffset,
+	                          uiPageCount);
+
+	return psDevmemXMapping;
+}
+
+/* Return new range */
+static void _SplitDevmemXMapping(DEVMEMX_MAPPING *psSrcDevmemXMapping,
+                                 DEVMEMX_MAPPING *psSplitDevmemXMapping,
+                                 IMG_UINT32 uiNewMappingOffset)
+{
+	IMG_UINT32 uiCorrectedPageCount;
+	IMG_UINT32 uiNewMappingPageCount;
+	IMG_UINT32 uiNewMappingPhysOffset;
+
+	PVR_ASSERT(psSrcDevmemXMapping != NULL);
+	PVR_ASSERT(psSplitDevmemXMapping != NULL);
+
+	uiCorrectedPageCount = uiNewMappingOffset - psSrcDevmemXMapping->uiVirtPageOffset;
+	uiNewMappingPageCount = (psSrcDevmemXMapping->uiVirtPageOffset + psSrcDevmemXMapping->uiPageCount) -
+	                      uiNewMappingOffset;
+	uiNewMappingPhysOffset = psSrcDevmemXMapping->uiPhysPageOffset + uiCorrectedPageCount;
+
+	PVR_LOG_GOTO_IF_FALSE(uiNewMappingOffset > psSrcDevmemXMapping->uiVirtPageOffset,
+	                      "New range offset precedes source mapping.",
+	                      ErrorExit);
+	PVR_LOG_GOTO_IF_FALSE(uiNewMappingOffset <= (psSrcDevmemXMapping->uiVirtPageOffset +
+	                                               psSrcDevmemXMapping->uiPageCount),
+	                      "New range offset exceeds source mapping.",
+	                      ErrorExit);
+
+	_InitialiseDevmemXMapping(psSplitDevmemXMapping,
+	                          psSrcDevmemXMapping->psPMR,
+	                          psSrcDevmemXMapping->uiFlags,
+	                          uiNewMappingPhysOffset,
+	                          uiNewMappingOffset,
+	                          uiNewMappingPageCount);
+
+	psSrcDevmemXMapping->uiPageCount = uiCorrectedPageCount;
+
+	return;
+
+ErrorExit:
+	/* This should never happen and indicates logic error */
+	PVR_DPF((PVR_DBG_FATAL, "Unable to split range for devmemx accounting!"));
+	OSWarnOn(1);
+}
+
+static void _AdjustDevmemXMappingStart(DEVMEMX_MAPPING *psSrcDevmemXMapping,
+                                       IMG_UINT32 uiNewMappingOffsetStart)
+{
+	IMG_UINT32 uiVirtualPagesDiff = (uiNewMappingOffsetStart - psSrcDevmemXMapping->uiVirtPageOffset);
+	IMG_UINT32 uiCorrectedPageCount = psSrcDevmemXMapping->uiPageCount - uiVirtualPagesDiff;
+	IMG_UINT32 uiCorrectedPhysOffset = psSrcDevmemXMapping->uiPhysPageOffset + uiVirtualPagesDiff;
+
+	psSrcDevmemXMapping->uiVirtPageOffset = uiNewMappingOffsetStart;
+	psSrcDevmemXMapping->uiPhysPageOffset = uiCorrectedPhysOffset;
+	psSrcDevmemXMapping->uiPageCount = uiCorrectedPageCount;
+}
+
+static void _AdjustDevmemXMappingEnd(DEVMEMX_MAPPING *psSrcDevmemXMapping,
+                                     IMG_UINT32 uiNewMappingOffsetEnd)
+{
+	IMG_UINT32 uiCorrectedPageCount = uiNewMappingOffsetEnd - psSrcDevmemXMapping->uiVirtPageOffset;
+
+	psSrcDevmemXMapping->uiPageCount = uiCorrectedPageCount;
+}
+
+static void _ReplaceCentreSplitExistingMapping(DEVMEMXINT_RESERVATION *psRsrv,
+                                               IMG_UINT32 uiVirtPageOffset,
+                                               IMG_UINT32 uiPageCount,
+                                               DEVMEMX_MAPPING *psReplaceMapping,
+                                               DEVMEMX_MAPPING *psSplitMappings[2],
+                                               IMG_UINT32 *uiDeferPMRUnlockPhyCount)
+{
+	IMG_UINT32 i;
+	DEVMEMX_MAPPING *psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+	PMR *psCurrentPMR = psCurrentMapping->psPMR;
+
+	/* If the psCurrentMapping references the same PMR we are replacing it with we
+	 * already have the PMR locked above, else we need to lock the PMR we are adjusting.
+	 */
+	if (psReplaceMapping &&
+	    psCurrentPMR == psReplaceMapping->psPMR)
+	{
+		_SplitDevmemXMapping(psCurrentMapping, psSplitMappings[0], uiVirtPageOffset);
+
+		_SplitDevmemXMapping(psSplitMappings[0], psSplitMappings[1], uiVirtPageOffset + uiPageCount);
+
+		_DestroyDevmemXMapping(psSplitMappings[0]);
+
+		*uiDeferPMRUnlockPhyCount = uiPageCount;
+	}
+	else
+	{
+		PMRLockPMR(psCurrentPMR);
+
+		_SplitDevmemXMapping(psCurrentMapping, psSplitMappings[0], uiVirtPageOffset);
+
+		_SplitDevmemXMapping(psSplitMappings[0], psSplitMappings[1], uiVirtPageOffset + uiPageCount);
+
+		_DestroyDevmemXMapping(psSplitMappings[0]);
+
+		PMRUnlockPMR(psCurrentPMR);
+
+		/* Drop references on PMR we have replaced */
+		PMRUnlockSysPhysAddressesN(psCurrentPMR, uiPageCount);
+	}
+
+	/* Update replaced centre region */
+	for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+	{
+		psRsrv->ppsDevmemXMapping[i] = psReplaceMapping;
+	}
+
+	/* Update split after mappings with new entry */
+	for (i = psSplitMappings[1]->uiVirtPageOffset;
+	     i < psSplitMappings[1]->uiVirtPageOffset + psSplitMappings[1]->uiPageCount; i++)
+	{
+		psRsrv->ppsDevmemXMapping[i] = psSplitMappings[1];
+	}
+}
+
+static void _AdjustOrDestroyReplacedMapping(DEVMEMX_MAPPING *psCurrentMapping,
+                                            IMG_UINT32 uiIterStart,
+                                            IMG_UINT32 uiIterEnd)
+{
+	__maybe_unused PMR *psCurrentPMR = psCurrentMapping->psPMR;
+
+	PMRLockHeldAssert(psCurrentPMR);
+
+	/* Adjust or free the map range we just replaced.
+	 *
+	 * 3 Cases:
+	 * New range replaces all existing range
+	 * New range replaces from end of exist range. (This can only happen on first iter)
+	 * New range replaces from start of existing range.
+	 * */
+	if (psCurrentMapping->uiVirtPageOffset == uiIterStart &&
+	    psCurrentMapping->uiVirtPageOffset + psCurrentMapping->uiPageCount == uiIterEnd)
+	{
+		_DestroyDevmemXMapping(psCurrentMapping);
+	}
+	else if (psCurrentMapping->uiVirtPageOffset < uiIterStart)
+	{
+		_AdjustDevmemXMappingEnd(psCurrentMapping, uiIterStart);
+	}
+	else if (uiIterEnd < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount))
+	{
+		_AdjustDevmemXMappingStart(psCurrentMapping, uiIterEnd);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_FATAL, "Unable to perform range record adjustment"));
+	}
+}
+
+static void _ReplaceReservationMappingRecords(DEVMEMXINT_RESERVATION *psRsrv,
+                                             IMG_UINT32 uiVirtPageOffset,
+                                             IMG_UINT32 uiPageCount,
+                                             DEVMEMX_MAPPING *psReplaceMapping,
+                                             IMG_UINT32 *uiDeferPMRUnlockPhyCount)
+{
+	IMG_UINT32 i;
+
+	for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount);)
+	{
+		IMG_UINT32 j = i;
+		IMG_UINT32 uiConsolidate = 0;
+		DEVMEMX_MAPPING *psCurrentMapping = psRsrv->ppsDevmemXMapping[i];
+		PMR *psCurrentPMR = psCurrentMapping->psPMR;
+
+		/* If we have attempted to delete a virtual record from an empty page entry */
+		if (psReplaceMapping == NULL &&
+		    psCurrentMapping == NULL)
+		{
+			i++;
+			continue;
+		}
+		/* If we are adding a new record to the current range reference */
+		else if (psCurrentMapping == NULL)
+		{
+			psRsrv->ppsDevmemXMapping[i] = psReplaceMapping;
+			i++;
+			continue;
+		}
+
+		/* Iterate over records with the same CurrentMapRange and replace them
+		 * in the Map Range array
+		 */
+		do
+		{
+			psRsrv->ppsDevmemXMapping[j] = psReplaceMapping;
+			uiConsolidate++;
+			j++;
+		}
+		while (j < uiVirtPageOffset + uiPageCount &&
+		      psCurrentMapping == psRsrv->ppsDevmemXMapping[j]
+		);
+
+		/* If the psCurrentMapping references the same PMR we are replacing it with we
+		 * already have the PMR locked above, else we need to lock the PMR we are adjusting.
+		 */
+		if (psReplaceMapping &&
+		    psCurrentPMR == psReplaceMapping->psPMR)
+		{
+			_AdjustOrDestroyReplacedMapping(psCurrentMapping, i, j);
+
+			/* We can't UnlockPhyAddrs for this mapping since we have locked it
+			 * in the layer above, defer until we are done with all adjustments
+			 * to this PMR.
+			 */
+			*uiDeferPMRUnlockPhyCount += uiConsolidate;
+		}
+		else
+		{
+			PMRLockPMR(psCurrentPMR);
+			_AdjustOrDestroyReplacedMapping(psCurrentMapping, i, j);
+			PMRUnlockPMR(psCurrentPMR);
+
+			/* Drop references on PMR we have replaced */
+			PMRUnlockSysPhysAddressesN(psCurrentPMR, uiConsolidate);
+		}
+
+		i += uiConsolidate;
+	}
+}
+
+static void _DeleteReservationMappingRecords(DEVMEMXINT_RESERVATION *psRsrv,
+                                             IMG_UINT32 uiVirtPageOffset,
+                                             IMG_UINT32 uiPageCount)
+{
+	__maybe_unused IMG_UINT32 uiDeferUnlockPhyCount = 0;
+	_ReplaceReservationMappingRecords(psRsrv,
+	                                  uiVirtPageOffset,
+	                                  uiPageCount,
+	                                  NULL,
+	                                  &uiDeferUnlockPhyCount);
+
+	/* We don't have a PMR locked when calling this function so there will be
+	 * no UnlockPhys to defer.
+	 */
+	PVR_ASSERT(uiDeferUnlockPhyCount == 0);
+}
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxAcquire
+@Description    Acquire a reference to the provided device memory context.
+@Return         IMG_TRUE on success, IMG_FALSE on overflow
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntCtxAcquire(DEVMEMINT_CTX *psDevmemCtx)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psDevmemCtx->hRefCount, 1,
+	                                        DEVMEMCTX_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "context, reference count has overflowed.", __func__));
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxRelease
+@Description    Release the reference to the provided device memory context.
+                If this is the last reference which was taken then the
+                memory context will be freed.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void DevmemIntCtxRelease(DEVMEMINT_CTX *psDevmemCtx)
+{
+	if (OSAtomicDecrement(&psDevmemCtx->hRefCount) == 0)
+	{
+		/* The last reference has gone, destroy the context */
+		PVRSRV_DEVICE_NODE *psDevNode = psDevmemCtx->psDevNode;
+		DLLIST_NODE *psNode, *psNodeNext;
+
+		/* Protect removing the node from the list in case it's being accessed
+		 * by DevmemIntPFNotify(). */
+		OSWRLockAcquireWrite(psDevNode->hPageFaultNotifyLock);
+		/* If this context is in the list registered for a debugger, remove
+		 * from that list */
+		if (dllist_node_is_in_list(&psDevmemCtx->sPageFaultNotifyListElem))
+		{
+			dllist_remove_node(&psDevmemCtx->sPageFaultNotifyListElem);
+		}
+		/* It should be safe to release the lock here (as long as
+		 * DevmemIntPFNotify() protects accessing memory contexts as well). */
+		OSWRLockReleaseWrite(psDevNode->hPageFaultNotifyLock);
+
+		/* If there are any PIDs registered for page fault notification.
+		 * Loop through the registered PIDs and free each one */
+		dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+		{
+			DEVMEMINT_PF_NOTIFY *psNotifyNode =
+				IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+			dllist_remove_node(psNode);
+			OSFreeMem(psNotifyNode);
+		}
+
+		if (psDevNode->pfnUnregisterMemoryContext)
+		{
+			psDevNode->pfnUnregisterMemoryContext(psDevmemCtx->hPrivData);
+		}
+		MMU_ContextDestroy(psDevmemCtx->psMMUContext);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Freed memory context %p",
+				 __func__, psDevmemCtx));
+		OSFreeMem(psDevmemCtx);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapAcquire
+@Description    Acquire a reference to the provided device memory heap.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntHeapAcquire(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psDevmemHeap->uiRefCount, 1,
+	                                        DEVMEMHEAP_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMHEAP_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "heap, reference count has overflowed.", __func__));
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapRelease
+@Description    Release the reference to the provided device memory heap.
+                If this is the last reference which was taken then the
+                memory context will be freed.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void DevmemIntHeapRelease(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	IMG_INT32 iOldValue = OSAtomicSubtractUnless(&psDevmemHeap->uiRefCount, 1,
+	                                             DEVMEMHEAP_REFCOUNT_MIN);
+
+	if (iOldValue == DEVMEMHEAP_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "heap, reference count has underflowed.", __func__));
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationAcquire
+@Description    Acquire a reference to the provided device memory reservation.
+                Prevents releasing of the reservation if external device
+                resource components still require it.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+IMG_BOOL DevmemIntReservationAcquire(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+	IMG_BOOL bSuccess = IMG_TRUE;
+
+	OSLockAcquireNested(psDevmemReservation->hLock, psDevmemReservation->eLockClass);
+
+	if (psDevmemReservation->i32DevResAcquisitionCount == DEVMEMRESERVATION_ACQUISITION_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "reservation, acquisition count has overflowed.", __func__));
+
+		bSuccess = IMG_FALSE;
+	}
+	else
+	{
+		psDevmemReservation->i32DevResAcquisitionCount++;
+	}
+
+	OSLockRelease(psDevmemReservation->hLock);
+
+	return bSuccess;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationRelease
+@Description    Release the reference to the provided device memory reservation.
+                Once these references have been released the
+                reservation is allowed to be released from UM.
+@Return         None.
+*/ /**************************************************************************/
+void DevmemIntReservationRelease(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+	OSLockAcquireNested(psDevmemReservation->hLock, psDevmemReservation->eLockClass);
+
+	if (psDevmemReservation->i32DevResAcquisitionCount == DEVMEMRESERVATION_ACQUISITION_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to release the device memory "
+		         "reservation, acquisition count has underflowed.", __func__));
+
+		/* for better debugging */
+		PVR_ASSERT(psDevmemReservation->i32DevResAcquisitionCount == DEVMEMRESERVATION_ACQUISITION_MIN);
+	}
+	else
+	{
+		psDevmemReservation->i32DevResAcquisitionCount--;
+	}
+
+	OSLockRelease(psDevmemReservation->hLock);
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetImportHandle
+@Description    For given exportable memory descriptor returns PMR handle.
+@Return         Memory is exportable - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+                            IMG_HANDLE *phImport)
+{
+	PVRSRV_ERROR eError;
+
+	if ((GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION, e0);
+	}
+
+	/* A new handle means a new import tracking the PMR.
+	 * Hence the source PMR memory layout should be marked fixed
+	 * to make sure the importer view of the memory is the same as
+	 * the exporter throughout its lifetime */
+	PMR_SetLayoutFixed((PMR *)psMemDesc->psImport->hPMR, IMG_TRUE);
+
+	*phImport = psMemDesc->psImport->hPMR;
+	return PVRSRV_OK;
+
+e0:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetHeapHandle
+@Description    For given reservation returns the Heap handle.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetHeapHandle(DEVMEMINT_RESERVATION *psReservation,
+                          IMG_HANDLE *phHeap)
+{
+	if (psReservation == NULL || phHeap == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*phHeap = psReservation->psDevmemHeap;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetContext
+@Description    For given heap returns the context.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetContext(DEVMEMINT_HEAP *psDevmemHeap,
+					   DEVMEMINT_CTX **ppsDevmemCtxPtr)
+{
+	if (psDevmemHeap == NULL || ppsDevmemCtxPtr == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*ppsDevmemCtxPtr = psDevmemHeap->psDevmemCtx;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetPrivData
+@Description    For given context returns the private data handle.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetPrivData(DEVMEMINT_CTX *psDevmemCtx,
+						IMG_HANDLE *phPrivData)
+{
+	if (psDevmemCtx == NULL || phPrivData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*phPrivData = psDevmemCtx->hPrivData;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxCreate
+@Description    Creates and initialises a device memory context.
+@Return         valid Device Memory context handle - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntCtxCreate(CONNECTION_DATA *psConnection,
+                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                   IMG_BOOL bKernelFWMemoryCtx,
+                   DEVMEMINT_CTX **ppsDevmemCtxPtr,
+                   IMG_HANDLE *hPrivData,
+                   IMG_UINT32 *pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtx;
+	IMG_HANDLE hPrivDataInt = NULL;
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = psDeviceNode->pfnGetMMUDeviceAttributes(psDeviceNode,
+	                                                                           bKernelFWMemoryCtx);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s", __func__));
+
+	/* Only allow request for a kernel context that comes from a direct bridge
+	 * (psConnection == NULL). Only the FW/KM Ctx is created over the direct bridge. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(!bKernelFWMemoryCtx || psConnection == NULL,
+	                                "bKernelFWMemoryCtx && psConnection");
+
+	/*
+	 * Ensure that we are safe to perform unaligned accesses on memory
+	 * we mark write-combine, as the compiler might generate
+	 * instructions operating on this memory which require this
+	 * assumption to be true.
+	 */
+	PVR_ASSERT(OSIsWriteCombineUnalignedSafe());
+
+	/* allocate a Devmem context */
+	psDevmemCtx = OSAllocMem(sizeof(*psDevmemCtx));
+	PVR_LOG_GOTO_IF_NOMEM(psDevmemCtx, eError, fail_alloc);
+
+	OSAtomicWrite(&psDevmemCtx->hRefCount, 1);
+	psDevmemCtx->psDevNode = psDeviceNode;
+
+	/* Call down to MMU context creation */
+
+	eError = MMU_ContextCreate(psConnection,
+	                           psDeviceNode,
+	                           &psDevmemCtx->psMMUContext,
+	                           psMMUDevAttrs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_ContextCreate", fail_mmucontext);
+
+	if (psDeviceNode->pfnRegisterMemoryContext)
+	{
+		eError = psDeviceNode->pfnRegisterMemoryContext(psDeviceNode, psDevmemCtx->psMMUContext, &hPrivDataInt);
+		PVR_LOG_GOTO_IF_ERROR(eError, "pfnRegisterMemoryContext", fail_register);
+	}
+
+	/* Store the private data as it is required to unregister the memory context */
+	psDevmemCtx->hPrivData = hPrivDataInt;
+	*hPrivData = hPrivDataInt;
+	*ppsDevmemCtxPtr = psDevmemCtx;
+
+	/* Pass the CPU cache line size through the bridge to the user mode as it can't be queried in user mode.*/
+	*pui32CPUCacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+
+	/* Initialise the PID notify list */
+	dllist_init(&(psDevmemCtx->sProcessNotifyListHead));
+	psDevmemCtx->sPageFaultNotifyListElem.psNextNode = NULL;
+	psDevmemCtx->sPageFaultNotifyListElem.psPrevNode = NULL;
+
+	/* Initialise page fault address */
+	psDevmemCtx->sFaultAddress.uiAddr = 0ULL;
+
+	/* Initialise flags */
+	psDevmemCtx->ui32Flags = 0;
+
+	psDevmemCtx->uiCreatedHeaps = 0;
+
+	return PVRSRV_OK;
+
+fail_register:
+	MMU_ContextDestroy(psDevmemCtx->psMMUContext);
+fail_mmucontext:
+	OSFreeMem(psDevmemCtx);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapCreate
+@Description    Creates and initialises a device memory heap.
+@Return         valid Device Memory heap handle - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntHeapCreate(DEVMEMINT_CTX *psDevmemCtx,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    DEVMEMINT_HEAP **ppsDevmemHeapPtr)
+{
+	DEVMEMINT_HEAP *psDevmemHeap;
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sBlueprintHeapBaseAddr;
+	IMG_DEVMEM_SIZE_T uiBlueprintHeapLength;
+	IMG_DEVMEM_SIZE_T uiBlueprintResRgnLength;
+	IMG_UINT32 ui32BlueprintLog2DataPageSize;
+	IMG_UINT32 ui32BlueprintLog2ImportAlignment;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s", __func__));
+
+	if (!DevmemIntCtxAcquire(psDevmemCtx))
+	{
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	/* Retrieve page size and base addr match the heap blueprint */
+	eError = HeapCfgHeapDetails(NULL,
+	                            psDevmemCtx->psDevNode,
+	                            uiHeapConfigIndex,
+	                            uiHeapIndex,
+	                            0, NULL,
+	                            &sBlueprintHeapBaseAddr,
+	                            &uiBlueprintHeapLength,
+	                            &uiBlueprintResRgnLength,
+	                            &ui32BlueprintLog2DataPageSize,
+	                            &ui32BlueprintLog2ImportAlignment);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get details for HeapConfig:%d HeapIndex:%d.",
+				 __func__, uiHeapConfigIndex, uiHeapIndex));
+		goto ErrorCtxRelease;
+	}
+
+	/* uiHeapConfigIndex and uiHeapIndex are validated in HeapCfgHeapDetails()
+	 * so it should be safe to use here without additional checks. We must assert
+	 * though that the index is less than the number of bits in uiCreatedHeaps
+	 * bitfield (we assume 8 bits in a byte and bitfield width of 64). */
+	PVR_ASSERT(uiHeapIndex < sizeof(psDevmemCtx->uiCreatedHeaps) * 8);
+
+	if (BIT_ISSET(psDevmemCtx->uiCreatedHeaps, uiHeapIndex))
+	{
+		eError = PVRSRV_ERROR_ALREADY_EXISTS;
+		goto ErrorCtxRelease;
+	}
+
+	/* allocate the heap object */
+	psDevmemHeap = OSAllocMem(sizeof(*psDevmemHeap));
+	PVR_LOG_GOTO_IF_NOMEM(psDevmemHeap, eError, ErrorCtxRelease);
+
+	psDevmemHeap->psDevmemCtx = psDevmemCtx;
+	psDevmemHeap->uiLog2PageSize = ui32BlueprintLog2DataPageSize;
+	psDevmemHeap->sBaseAddr = sBlueprintHeapBaseAddr;
+	/* Store the last accessible address as our LastAddr. We can access
+	 * every address between sBlueprintHeapBaseAddr and
+	 * sBlueprintHeapBaseAddr + uiBlueprintHeapLength - 1
+	 */
+	psDevmemHeap->sLastAddr.uiAddr = sBlueprintHeapBaseAddr.uiAddr + uiBlueprintHeapLength - 1;
+	psDevmemHeap->uiHeapIndex = uiHeapIndex;
+
+	OSAtomicWrite(&psDevmemHeap->uiRefCount, 1);
+
+	eError = HeapCfgGetCallbacks(psDevmemCtx->psDevNode,
+	                             uiHeapConfigIndex,
+	                             uiHeapIndex,
+	                             &psDevmemHeap->pfnInit,
+	                             &psDevmemHeap->pfnDeInit);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get callbacks for HeapConfig:%d HeapIndex:%d.",
+				 __func__, uiHeapConfigIndex, uiHeapIndex));
+		goto ErrorFreeDevmemHeap;
+	}
+
+	if (psDevmemHeap->pfnInit != NULL)
+	{
+		eError = psDevmemHeap->pfnInit(psDevmemCtx->psDevNode,
+		                               psDevmemHeap,
+		                               &psDevmemHeap->hPrivData);
+		PVR_GOTO_IF_ERROR(eError, ErrorFreeDevmemHeap);
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: sBaseAddr = %" IMG_UINT64_FMTSPECX ", "
+	        "sLastAddr = %" IMG_UINT64_FMTSPECX, __func__,
+	        psDevmemHeap->sBaseAddr.uiAddr, psDevmemHeap->sLastAddr.uiAddr));
+
+	BIT_SET(psDevmemCtx->uiCreatedHeaps, uiHeapIndex);
+
+	*ppsDevmemHeapPtr = psDevmemHeap;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevmemHeap:
+	OSFreeMem(psDevmemHeap);
+ErrorCtxRelease:
+	DevmemIntCtxRelease(psDevmemCtx);
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR ReserveRangeParamValidation(DEVMEMINT_HEAP *psDevmemHeap,
+                                                       IMG_DEV_VIRTADDR sReservationVAddr,
+                                                       IMG_DEVMEM_SIZE_T uiVirtualSize)
+{
+	IMG_DEV_VIRTADDR sLastReserveAddr;
+	IMG_UINT64 ui64InvalidSizeMask = (1 << psDevmemHeap->uiLog2PageSize) - 1;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap != NULL, "psDevmemHeap");
+
+	sLastReserveAddr.uiAddr = sReservationVAddr.uiAddr + uiVirtualSize - 1;
+
+	/* Check that the requested address is not less than the base address of the heap. */
+	if (sReservationVAddr.uiAddr < psDevmemHeap->sBaseAddr.uiAddr)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"sReservationVAddr ("IMG_DEV_VIRTADDR_FMTSPEC") is invalid! "
+			"Must be greater or equal to "IMG_DEV_VIRTADDR_FMTSPEC,
+			sReservationVAddr.uiAddr,
+			psDevmemHeap->sBaseAddr.uiAddr);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check the allocation size is valid (must be page granular). */
+	if ((uiVirtualSize & ui64InvalidSizeMask) != 0 || uiVirtualSize == 0)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"uiVirtualSize ("IMG_DEVMEM_SIZE_FMTSPEC") is invalid! Must a multiple of %u and greater than 0",
+			uiVirtualSize,
+			1 << psDevmemHeap->uiLog2PageSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiVirtualSize > PMR_MAX_SUPPORTED_SIZE)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"uiVirtualSize must be less than or equal to the max PMR size ("
+			IMG_DEVMEM_SIZE_FMTSPEC")",
+			PMR_MAX_SUPPORTED_SIZE);
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check that requested address + size fits in our heap. */
+	if (sLastReserveAddr.uiAddr > psDevmemHeap->sLastAddr.uiAddr)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"sReservationVAddr ("IMG_DEV_VIRTADDR_FMTSPEC") is invalid! "
+			"Must be lower than "IMG_DEV_VIRTADDR_FMTSPEC,
+			sReservationVAddr.uiAddr,
+			psDevmemHeap->sLastAddr.uiAddr - uiVirtualSize + 1);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemXIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                       IMG_DEV_VIRTADDR sReservationVAddr,
+                       IMG_DEVMEM_SIZE_T uiVirtualSize,
+                       DEVMEMXINT_RESERVATION **ppsRsrv)
+{
+	DEVMEMXINT_RESERVATION *psRsrv;
+	IMG_UINT32 uiNumPages;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(ppsRsrv != NULL);
+
+	eError = ReserveRangeParamValidation(psDevmemHeap,
+	                                     sReservationVAddr,
+	                                     uiVirtualSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "ReserveRangeParamValidation");
+
+
+	if (!DevmemIntHeapAcquire(psDevmemHeap))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorReturnError);
+	}
+
+	uiNumPages = uiVirtualSize >> psDevmemHeap->uiLog2PageSize;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psRsrv = OSAllocZMem(sizeof(*psRsrv) + sizeof(*psRsrv->ppsDevmemXMapping) * uiNumPages);
+	PVR_LOG_GOTO_IF_NOMEM(psRsrv, eError, ErrorUnreferenceHeap);
+#else
+	psRsrv = OSAllocZMem(sizeof(*psRsrv->ppsPMR) * uiNumPages + sizeof(*psRsrv));
+	PVR_LOG_GOTO_IF_NOMEM(psRsrv, eError, ErrorUnreferenceHeap);
+
+	psRsrv->ppsPMR = IMG_OFFSET_ADDR(psRsrv, sizeof(*psRsrv));
+#endif
+
+	OSAtomicWrite(&psRsrv->iLifetimeRefCount, 1);
+
+	eError = OSLockCreate(&psRsrv->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeReservation);
+
+	psRsrv->sBase = sReservationVAddr;
+	psRsrv->uiLength = uiVirtualSize;
+
+	eError = MMU_Alloc(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                   uiVirtualSize,
+	                   0, /* IMG_UINT32 uiProtFlags */
+	                   0, /* alignment is n/a since we supply devvaddr */
+	                   &sReservationVAddr,
+	                   psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrorDestroyLock);
+
+	/* since we supplied the virt addr, MMU_Alloc shouldn't have
+	   chosen a new one for us */
+	PVR_ASSERT(sReservationVAddr.uiAddr == psRsrv->sBase.uiAddr);
+
+	psRsrv->psDevmemHeap = psDevmemHeap;
+	*ppsRsrv = psRsrv;
+
+	return PVRSRV_OK;
+
+ErrorDestroyLock:
+	OSLockDestroy(psRsrv->hLock);
+ErrorFreeReservation:
+	OSFreeMem(psRsrv);
+ErrorUnreferenceHeap:
+	DevmemIntHeapRelease(psDevmemHeap);
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntUnreserveRange(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	IMG_UINT32 ui32FirstMappedIdx = IMG_UINT32_MAX; // Initialise with invalid value.
+	IMG_UINT32 ui32ContigPageCount = 0;
+	IMG_UINT32 ui32PageIdx;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* No need to lock the mapping here since this is a handle destruction path which can not be
+	 * executed while there are outstanding handle lookups, i.e. other operations are performed
+	 * on the mapping. Bridge and handle framework also make sure this path can also not be executed
+	 * concurrently. */
+
+	for (ui32PageIdx = 0; ui32PageIdx < _DevmemXReservationPageCount(psRsrv); ui32PageIdx++)
+	{
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		if (psRsrv->ppsDevmemXMapping[ui32PageIdx] != NULL)
+#else
+		if (psRsrv->ppsPMR[ui32PageIdx] != NULL)
+#endif
+		{
+			if (ui32ContigPageCount == 0)
+			{
+#if defined(DEBUG)
+				if (ui32FirstMappedIdx == IMG_UINT32_MAX)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Reservation was not unmapped! The reservation will "
+					         "be unmapped before proceeding.",
+					         __func__));
+				}
+#endif
+				ui32FirstMappedIdx = ui32PageIdx;
+			}
+
+			ui32ContigPageCount++;
+		}
+		else
+		{
+			if (ui32ContigPageCount != 0)
+			{
+				eError = DevmemXIntUnmapPages(psRsrv,
+				                              ui32FirstMappedIdx,
+				                              ui32ContigPageCount);
+				PVR_RETURN_IF_ERROR(eError);
+			}
+
+			ui32ContigPageCount = 0;
+		}
+	}
+
+	if (ui32ContigPageCount != 0)
+	{
+		eError = DevmemXIntUnmapPages(psRsrv,
+		                              ui32FirstMappedIdx,
+		                              ui32ContigPageCount);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	MMU_Free(psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+	         psRsrv->sBase,
+	         psRsrv->uiLength,
+	         psRsrv->psDevmemHeap->uiLog2PageSize);
+
+	/* Don't bother with refcount on reservation, as a reservation only ever
+	 * holds one mapping, so we directly decrement the refcount on the heap
+	 * instead.
+	 * Function will print an error if the heap could not be unreferenced. */
+	DevmemIntHeapRelease(psRsrv->psDevmemHeap);
+
+	/* We have partially destroyed the object but we may need to keep the wrapper object
+	 * around since it may be referenced in KM threads where we need the lock to avoid
+	 * race conditions. UM threads will no longer have a handle to this object when this
+	 * bridge call exits.
+	 */
+	_DevmemXReservationUnrefAndMaybeDestroy(psRsrv);
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+DevmemValidateFlags(PMR *psPMR, PVRSRV_MEMALLOCFLAGS_T uiMapFlags)
+{
+	PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (PVRSRV_CHECK_GPU_READABLE(uiMapFlags) && !PVRSRV_CHECK_GPU_READABLE(uiPMRFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not GPU readable.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+	if (PVRSRV_CHECK_GPU_WRITEABLE(uiMapFlags) && !PVRSRV_CHECK_GPU_WRITEABLE(uiPMRFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not GPU writeable.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+	if ((uiMapFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK) !=
+	    (uiPMRFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR's device specific flags don't match mapping flags.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntMapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                   PMR *psPMR,
+                   IMG_UINT32 uiPageCount,
+                   IMG_UINT32 uiPhysPageOffset,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   IMG_UINT32 uiVirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPMRMaxChunkCount = PMR_LogicalSize(psPMR) >> PMR_GetLog2Contiguity(psPMR);
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	IMG_UINT32 uiLog2PageSize = psDevmemHeap->uiLog2PageSize;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	DEVMEMX_MAPPING *psNewMapping;
+	DEVMEMX_MAPPING *apsPotentialSplitMapping[DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT] = {0};
+	DEVMEMX_MAPPING *psCurrentMapping;
+	IMG_UINT32 uiLockedPMRDeferUnlockPhyCount = 0;
+#endif
+
+	/* Test uiPageCount+uiPhysPageOffset will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiPageCount + (IMG_UINT64)uiPhysPageOffset) <= (IMG_UINT64)IMG_UINT32_MAX, "uiPageCount+uiPhysPageOffset exceeds IMG_UINT32_MAX");
+	/* Test we do not exceed the PMR's maximum physical extent (in pages) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uiPageCount + uiPhysPageOffset) <= uiPMRMaxChunkCount, "uiPageCount+uiPhysPageOffset");
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	/* The range is not valid for the given virtual descriptor */
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+	PVR_LOG_RETURN_IF_FALSE((uiFlags & ~PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK) == 0,
+	                        "invalid flags", PVRSRV_ERROR_INVALID_FLAGS);
+	PVR_LOG_RETURN_IF_FALSE(!PMR_IsSparse(psPMR),
+		                    "PMR is Sparse, devmemx PMRs should be non-sparse", PVRSRV_ERROR_INVALID_FLAGS);
+	PVR_LOG_RETURN_IF_FALSE(!(PMR_Flags(psPMR) & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC),
+		                    "PMR allocation is deferred, devmemx PMRs can not be deferred", PVRSRV_ERROR_INVALID_FLAGS);
+
+	if (uiLog2PageSize > PMR_GetLog2Contiguity(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device heap and PMR have incompatible "
+		        "Log2Contiguity (%u - %u). PMR contiguity must be a multiple "
+		        "of the heap contiguity!", __func__, uiLog2PageSize,
+		        PMR_GetLog2Contiguity(psPMR)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = DevmemValidateFlags(psPMR, uiFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemValidateFlags");
+
+	OSLockAcquire(psRsrv->hLock);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+
+	if (psCurrentMapping != NULL && /* Does a mapping currently exist in this entry */
+	   /* Does the start of the current mapping precede the requested offset */
+	   psCurrentMapping->uiVirtPageOffset < uiVirtPageOffset &&
+	   /* Does the end of the mapping precede the requested offset + page count */
+	   uiVirtPageOffset + uiPageCount < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount - 1))
+	{
+		apsPotentialSplitMapping[0] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[0] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlockRsrv);
+		}
+
+		apsPotentialSplitMapping[1] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[1] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrFreeFirstSplitMapping);
+		}
+	}
+	/* Lock address `uiPageCount` times. This will also take N references on
+	 * the PMR. */
+	eError = PMRLockSysPhysAddressesN(psPMR, uiPageCount);
+	PVR_GOTO_IF_ERROR(eError, ErrFreeSecondSplitMapping);
+
+	PMRLockPMR(psPMR);
+
+	psNewMapping = _AllocateDevmemXMapping(psRsrv,
+	                                       psPMR,
+	                                       uiFlags,
+	                                       uiPhysPageOffset,
+	                                       uiVirtPageOffset,
+	                                       uiPageCount);
+	if (!psNewMapping)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_AllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlockPMR);
+	}
+
+	do
+	{
+		eError = MMU_MapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+							  uiFlags,
+							  _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+							  psPMR,
+							  uiPhysPageOffset,
+							  uiPageCount,
+							  NULL,
+							  psDevmemHeap->uiLog2PageSize);
+		if (eError == PVRSRV_OK)
+		{
+			break;
+		}
+		else if (eError == PVRSRV_ERROR_RETRY)
+		{
+			/* This path may request this call to retry mapping at a later time, if this happens we
+			 * need to relinquish the synchronisation resources to allow migrate to complete and
+			 * try again
+			 */
+			_DeschedAllowRemapToComplete(psRsrv->hLock, psPMR);
+		}
+		else
+		{
+			/* Unexpected Error code path */
+			goto ErrFreeNewMapping;
+		}
+	} while (eError == PVRSRV_ERROR_RETRY);
+
+	/* If this has overwritten the middle of an existing mapping */
+	if (apsPotentialSplitMapping[0] != NULL)
+	{
+		_ReplaceCentreSplitExistingMapping(psRsrv,
+										   uiVirtPageOffset,
+										   uiPageCount,
+										   psNewMapping,
+										   apsPotentialSplitMapping,
+										   &uiLockedPMRDeferUnlockPhyCount);
+	}
+	else
+	{
+		_ReplaceReservationMappingRecords(psRsrv,
+										  uiVirtPageOffset,
+										  uiPageCount,
+										  psNewMapping,
+										  &uiLockedPMRDeferUnlockPhyCount);
+	}
+	PMRUnlockPMR(psPMR);
+
+	/* Drop any deferred references on PMR we had locked */
+	if (uiLockedPMRDeferUnlockPhyCount != 0)
+	{
+		PMRUnlockSysPhysAddressesN(psPMR, uiLockedPMRDeferUnlockPhyCount);
+	}
+
+#else
+
+	/* Lock address `uiPageCount` times. This will also take N references on
+	 * the PMR. */
+	eError = PMRLockSysPhysAddressesN(psPMR, uiPageCount);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlockRsrv);
+
+	PMRLockPMR(psPMR);
+
+	eError = MMU_MapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                      uiFlags,
+	                      _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                      psPMR,
+	                      uiPhysPageOffset,
+	                      uiPageCount,
+	                      NULL,
+	                      psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlockPMR);
+
+	PMRUnlockPMR(psPMR);
+
+	{
+		IMG_UINT32 i;
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+			{
+				PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psRsrv->ppsPMR[i]);
+				PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+			}
+			}
+
+			psRsrv->ppsPMR[i] = psPMR;
+		}
+	}
+#endif
+
+	OSLockRelease(psRsrv->hLock);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrFreeNewMapping:
+	_DestroyDevmemXMapping(psNewMapping);
+ErrUnlockPMR:
+	PMRUnlockPMR(psPMR);
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddressesN(psPMR, uiPageCount);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+ErrFreeSecondSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[1]);
+ErrFreeFirstSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[0]);
+#else
+ErrUnlockPMR:
+	PMRUnlockPMR(psPMR);
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddressesN(psPMR, uiPageCount);
+		PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+	}
+#endif
+ErrUnlockRsrv:
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntUnmapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                     IMG_UINT32 uiVirtPageOffset,
+                     IMG_UINT32 uiPageCount)
+{
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	DEVMEMX_MAPPING *apsPotentialSplitMapping[DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT] = {0};
+	DEVMEMX_MAPPING *psCurrentMapping;
+#endif
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+
+	OSLockAcquire(psRsrv->hLock);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+	if (psCurrentMapping != NULL &&/* Does a mapping currently exist in this entry */
+	    /* Does the start of the current mapping precede the requested offset */
+	    psCurrentMapping->uiVirtPageOffset < uiVirtPageOffset &&
+	    /* Does the end of the mapping precede the requested offset + page count */
+	    uiVirtPageOffset + uiPageCount < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount - 1))
+	{
+		apsPotentialSplitMapping[0] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[0] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlock);
+		}
+
+		apsPotentialSplitMapping[1] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[1] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrFreeFirstSplitMapping);
+		}
+	}
+#endif
+
+	/* Unmap the pages and mark them invalid in the MMU PTE */
+	eError = MMU_UnmapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                        0,
+	                        _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                        uiPageCount,
+	                        NULL,
+	                        psDevmemHeap->uiLog2PageSize,
+	                        0);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrFreeSecondSplitMapping);
+
+	/* If this has overwritten the middle of an existing mapping */
+	if (apsPotentialSplitMapping[0] != NULL)
+	{
+		__maybe_unused IMG_UINT32 uiDeferUnlockPhyCount = 0;
+
+		_ReplaceCentreSplitExistingMapping(psRsrv,
+		                                   uiVirtPageOffset,
+		                                   uiPageCount,
+		                                   NULL,
+		                                   apsPotentialSplitMapping,
+		                                   &uiDeferUnlockPhyCount);
+		/* We don't have a PMR locked when calling this function so there will be
+		 * no UnlockPhys to defer.
+		 */
+		PVR_ASSERT(uiDeferUnlockPhyCount == 0);
+	}
+	else
+	{
+		_DeleteReservationMappingRecords(psRsrv,
+										 uiVirtPageOffset,
+										 uiPageCount);
+	}
+#else
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+	{
+		IMG_UINT32 i;
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+				PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psRsrv->ppsPMR[i]);
+				PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+
+				psRsrv->ppsPMR[i] = NULL;
+			}
+		}
+	}
+#endif
+
+	OSLockRelease(psRsrv->hLock);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrFreeSecondSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[1]);
+ErrFreeFirstSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[0]);
+#endif
+ErrUnlock:
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntMapVRangeToBackingPage(DEVMEMXINT_RESERVATION *psRsrv,
+                                 IMG_UINT32 uiPageCount,
+                                 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                 IMG_UINT32 uiVirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	DEVMEMX_MAPPING *apsPotentialSplitMapping[DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT] = {0};
+	DEVMEMX_MAPPING *psCurrentMapping;
+#endif
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	/* The range is not valid for the given virtual descriptor */
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+	PVR_LOG_RETURN_IF_FALSE((uiFlags & ~(PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK |
+	                                     PVRSRV_MEMALLOCFLAG_ZERO_BACKING)) == 0,
+	                        "invalid flags", PVRSRV_ERROR_INVALID_FLAGS);
+
+	OSLockAcquire(psRsrv->hLock);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+	if (psCurrentMapping != NULL &&/* Does a mapping currently exist in this entry */
+	    /* Does the start of the current mapping precede the requested offset */
+	    psCurrentMapping->uiVirtPageOffset < uiVirtPageOffset &&
+	    /* Does the end of the mapping precede the requested offset + page count*/
+	    uiVirtPageOffset + uiPageCount < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount - 1))
+	{
+		apsPotentialSplitMapping[0] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[0] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlock);
+		}
+
+		apsPotentialSplitMapping[1] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[1] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrFreeFirstSplitMapping);
+		}
+	}
+#endif
+
+	eError = MMUX_MapVRangeToBackingPage(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                                     uiFlags,
+	                                     _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                                     uiPageCount,
+	                                     psDevmemHeap->uiLog2PageSize);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrFreeSecondSplitMapping);
+
+	/* If this has overwritten the middle of an existing mapping */
+	if (apsPotentialSplitMapping[0] != NULL)
+	{
+		IMG_UINT32 uiDeferUnlockPhyCount = 0;
+
+		_ReplaceCentreSplitExistingMapping(psRsrv,
+										   uiVirtPageOffset,
+										   uiPageCount,
+										   NULL,
+										   apsPotentialSplitMapping,
+										   &uiDeferUnlockPhyCount);
+		/* We don't have a PMR locked when calling this function so there will be
+		 * no UnlockPhys to defer.
+		 */
+		PVR_ASSERT(uiDeferUnlockPhyCount == 0);
+	}
+	else
+	{
+		_DeleteReservationMappingRecords(psRsrv,
+										 uiVirtPageOffset,
+										 uiPageCount);
+	}
+#else
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+
+	{
+		IMG_UINT32 i;
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+				eError = PMRUnlockSysPhysAddresses(psRsrv->ppsPMR[i]);
+				PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+				psRsrv->ppsPMR[i] = NULL;
+			}
+		}
+	}
+#endif
+
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrFreeSecondSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[1]);
+ErrFreeFirstSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[0]);
+#endif
+ErrUnlock:
+	OSLockRelease(psRsrv->hLock);
+	return eError;
+}
+
+static INLINE IMG_UINT32
+_DevmemReservationPageCount(DEVMEMINT_RESERVATION *psRsrv)
+{
+	return psRsrv->uiLength >> psRsrv->psDevmemHeap->uiLog2PageSize;
+}
+
+PVRSRV_ERROR
+DevmemIntMapPMR(DEVMEMINT_RESERVATION *psReservation, PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+	/* number of pages (device pages) that allocation spans */
+	IMG_UINT32 ui32NumDevPages;
+	/* device virtual address of start of allocation */
+	IMG_DEV_VIRTADDR sReservationVAddr;
+	/* and its length */
+	IMG_DEVMEM_SIZE_T uiVirtualSize;
+	IMG_UINT32 uiLog2HeapContiguity = psReservation->psDevmemHeap->uiLog2PageSize;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags = psReservation->uiFlags;
+	IMG_BOOL bIsSparse = IMG_FALSE;
+	void *pvTmpBuf = NULL;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psReservation->psMappedPMR == NULL, "psReservation");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(PMR_LogicalSize(psPMR) == psReservation->uiLength, "psPMR logical size");
+
+	if (uiLog2HeapContiguity > PMR_GetLog2Contiguity(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Device heap and PMR have incompatible contiguity (%u - %u). "
+		         "Heap contiguity must be a multiple of the heap contiguity!",
+		         __func__,
+		         uiLog2HeapContiguity,
+		         PMR_GetLog2Contiguity(psPMR) ));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, ErrorReturnError);
+	}
+
+	eError = DevmemValidateFlags(psPMR, uiMapFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemValidateFlags", ErrorReturnError);
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+
+	uiVirtualSize = psReservation->uiLength;
+
+	ui32NumDevPages = 0xffffffffU & ( ( (uiVirtualSize - 1) >> uiLog2HeapContiguity) + 1);
+	PVR_ASSERT((IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity == uiVirtualSize);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorReleaseResLock);
+
+	PMRLockPMR(psPMR);
+
+	sReservationVAddr = psReservation->sBase;
+
+	/*Check if the PMR that needs to be mapped is sparse */
+	bIsSparse = PMR_IsSparse(psPMR);
+	if (bIsSparse)
+	{
+		IMG_DEV_PHYADDR *psDevPAddr;
+		IMG_BOOL *pbValid;
+
+		pvTmpBuf = OSAllocMem((sizeof(IMG_DEV_PHYADDR) + sizeof(IMG_BOOL)) *
+		                      ui32NumDevPages);
+		PVR_LOG_GOTO_IF_NOMEM(pvTmpBuf, eError, ErrorUnlockPhysAddr);
+
+		psDevPAddr = IMG_OFFSET_ADDR(pvTmpBuf, 0);
+		pbValid = IMG_OFFSET_ADDR(pvTmpBuf, ui32NumDevPages * sizeof(IMG_DEV_PHYADDR));
+
+		/* N.B. We pass mapping permission flags to MMU_MapPages and let
+		 * it reject the mapping if the permissions on the PMR are not compatible. */
+		eError = MMU_MapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                      uiMapFlags,
+		                      sReservationVAddr,
+		                      psPMR,
+		                      0,
+		                      ui32NumDevPages,
+		                      NULL,
+		                      uiLog2HeapContiguity);
+		PVR_GOTO_IF_ERROR(eError, ErrorFreeSparseTmpBuf);
+
+		/* Determine which entries of the PMR are valid */
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2HeapContiguity,
+		                         ui32NumDevPages,
+		                         0,
+		                         psDevPAddr,
+		                         pbValid,
+		                         DEVICE_USE | MAPPING_USE);
+		/* PVRSRV_ERROR_RETRY with SUPPORT_LINUX_OSPAGE_MIGRATION feature
+		 * enabled will not be returned here even with the use of
+		 * DEVICE_USE | MAPPING_USE because the feature does not
+		 * support migratable sparse PMRs.
+		 */
+		PVR_GOTO_IF_ERROR(eError, ErrorUnmap);
+
+		for (i = 0; i < ui32NumDevPages; i++)
+		{
+			if (DevmemIntReservationIsIndexMapped(psReservation, i))
+			{
+				eError = PMRUnrefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+			}
+
+			if (pbValid[i])
+			{
+				eError = PMRRefPMR(psPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRRefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_TRUE);
+			}
+		}
+
+		OSFreeMem(pvTmpBuf);
+	}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	else
+	{
+		do
+		{
+			eError = MMU_MapPMRFast(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+			                        sReservationVAddr,
+			                        psPMR,
+			                        (IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity,
+			                        uiMapFlags,
+			                        uiLog2HeapContiguity);
+			if (eError == PVRSRV_OK)
+			{
+				break;
+			}
+			else if (eError == PVRSRV_ERROR_RETRY)
+			{
+				/* This path may request this call to retry mapping at a later time, if this happens we
+				 * need to relinquish the synchronisation resources to allow migrate to complete and
+				 * try again
+				 */
+				_DeschedAllowRemapToCompleteNested(psReservation->hLock,
+				                                   psReservation->eLockClass,
+				                                   psPMR);
+			}
+			else
+			{
+				/* Unexpected Error code path */
+				goto ErrorUnlockPhysAddr;
+			}
+		} while (eError == PVRSRV_ERROR_RETRY);
+	}
+#else
+	else
+	{
+		eError = MMU_MapPMRFast(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        sReservationVAddr,
+		                        psPMR,
+		                        (IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity,
+		                        uiMapFlags,
+		                        uiLog2HeapContiguity);
+		PVR_GOTO_IF_ERROR(eError, ErrorUnlockPhysAddr);
+	}
+#endif
+
+	psReservation->psMappedPMR = psPMR;
+
+	/* Increase reservation association count so we know if multiple mappings have been created
+	 * on the PMR, also link the mapping to the PMR if required.
+	 */
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psReservation->psDevmemMapping = _AllocateDevmemMapping(psReservation,
+	                                                        psPMR);
+	PVR_LOG_GOTO_IF_NOMEM(psReservation->psDevmemMapping, eError, ErrorUnsetMappedPMR);
+	_LinkPMRRsrvNode(&psReservation->psDevmemMapping->sPMRRsrvLinkNode, psPMR);
+#else
+	PMRLinkGPUMapping(psPMR);
+#endif
+
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(psReservation->hLock);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrorUnsetMappedPMR:
+	psReservation->psMappedPMR = NULL;
+#endif
+ErrorUnmap:
+	(void) MMU_UnmapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+	                      0,
+	                      sReservationVAddr,
+	                      ui32NumDevPages,
+	                      NULL,
+	                      uiLog2HeapContiguity,
+	                      0);
+ErrorFreeSparseTmpBuf:
+	if (pvTmpBuf)
+	{
+		OSFreeMem(pvTmpBuf);
+	}
+
+ErrorUnlockPhysAddr:
+	PMRUnlockPMR(psPMR);
+	{
+		PVRSRV_ERROR eError1 = PVRSRV_OK;
+		eError1 = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError1, "PMRUnlockSysPhysAddresses");
+	}
+
+ErrorReleaseResLock:
+	OSLockRelease(psReservation->hLock);
+
+ErrorReturnError:
+	PVR_ASSERT (eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntGetReservationData(DEVMEMINT_RESERVATION* psReservation, PMR** ppsPMR, IMG_DEV_VIRTADDR* psDevVAddr)
+{
+	/* Reservation might not have a PMR if a mapping was not yet performed */
+	if (psReservation->psMappedPMR == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevVAddr->uiAddr = psReservation->sBase.uiAddr;
+	*ppsPMR = psReservation->psMappedPMR;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntReserveRangeAndMapPMR(CONNECTION_DATA *psConnectionData,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               DEVMEMINT_HEAP *psDevmemHeap,
+                               IMG_DEV_VIRTADDR sReservationVAddr,
+                               IMG_DEVMEM_SIZE_T uiVirtualSize,
+                               PMR *psPMR,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               DEVMEMINT_RESERVATION **ppsReservation)
+{
+	PVRSRV_ERROR eError, eUnreserveError;
+
+	eError = DevmemIntReserveRange(psConnectionData,
+	                               psDeviceNode,
+	                               psDevmemHeap,
+	                               sReservationVAddr,
+	                               uiVirtualSize,
+	                               uiFlags,
+	                               ppsReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorReturnError);
+
+	eError = DevmemIntMapPMR(*ppsReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	eUnreserveError = DevmemIntUnreserveRange(*ppsReservation);
+	*ppsReservation = NULL;
+	PVR_LOG_IF_ERROR(eUnreserveError, "DevmemIntUnreserveRange");
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnmapPMR(DEVMEMINT_RESERVATION *psReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeap = psReservation->psDevmemHeap;
+	/* device virtual address of start of allocation */
+	IMG_DEV_VIRTADDR sReservationVAddr;
+	/* number of pages (device pages) that allocation spans */
+	IMG_UINT32 ui32NumDevPages;
+	IMG_BOOL bIsSparse = IMG_FALSE;
+	IMG_UINT32 i;
+
+	PVR_RETURN_IF_INVALID_PARAM(psReservation->psMappedPMR != NULL);
+
+	ui32NumDevPages = _DevmemReservationPageCount(psReservation);
+	sReservationVAddr = psReservation->sBase;
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+	PMRLockPMR(psReservation->psMappedPMR);
+
+	bIsSparse = PMR_IsSparse(psReservation->psMappedPMR);
+
+	if (bIsSparse)
+	{
+		eError = MMU_UnmapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        0,
+		                        sReservationVAddr,
+		                        ui32NumDevPages,
+		                        NULL,
+		                        psDevmemHeap->uiLog2PageSize,
+		                        0);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+
+		/* We are unmapping the whole PMR */
+		for (i = 0; i < ui32NumDevPages; i++)
+		{
+			if (DevmemIntReservationIsIndexMapped(psReservation, i))
+			{
+				/* All PMRs in the range should be the same, set local PMR
+				 * for Unlocking phys addrs later */
+				eError = PMRUnrefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+			}
+		}
+	}
+	else
+	{
+		eError = MMU_UnmapPMRFast(psDevmemHeap->psDevmemCtx->psMMUContext,
+		                          sReservationVAddr,
+		                          ui32NumDevPages,
+		                          psDevmemHeap->uiLog2PageSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPMRFast", ErrUnlock);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	_UnlinkPMRRsrvNode(&psReservation->psDevmemMapping->sPMRRsrvLinkNode, psReservation->psMappedPMR);
+	PMRUnlockPMR(psReservation->psMappedPMR);
+
+	_UnrefAndMaybeDestroyPMRRsrvNode(&psReservation->psDevmemMapping->sPMRRsrvLinkNode);
+#else
+	PMRUnlinkGPUMapping(psReservation->psMappedPMR);
+	PMRUnlockPMR(psReservation->psMappedPMR);
+#endif
+
+	eError = PMRUnlockSysPhysAddresses(psReservation->psMappedPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+
+	psReservation->psMappedPMR = NULL;
+
+	OSLockRelease(psReservation->hLock);
+
+	return PVRSRV_OK;
+
+ErrUnlock:
+	PMRUnlockPMR(psReservation->psMappedPMR);
+	OSLockRelease(psReservation->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntReserveRange(CONNECTION_DATA *psConnectionData,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      DEVMEMINT_HEAP *psDevmemHeap,
+                      IMG_DEV_VIRTADDR sReservationVAddr,
+                      IMG_DEVMEM_SIZE_T uiVirtualSize,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      DEVMEMINT_RESERVATION **ppsReservationPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservation;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT64 ui64MapSize;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_ASSERT(ppsReservationPtr != NULL);
+
+	eError = ReserveRangeParamValidation(psDevmemHeap,
+	                                     sReservationVAddr,
+	                                     uiVirtualSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "ReserveRangeParamValidation");
+
+	if (!DevmemIntHeapAcquire(psDevmemHeap))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW,
+		                    ErrorReturnError);
+	}
+
+	uiNumPages = uiVirtualSize >> psDevmemHeap->uiLog2PageSize;
+
+	/* allocate memory to record the reservation info */
+	ui64MapSize = sizeof(*psReservation->pui8Map) * LOG2_DIV(uiNumPages, MAP_MASK_SHIFT);
+	if (LOG2_MOD(uiNumPages, MAP_MASK_SHIFT) != 0)
+	{
+		ui64MapSize += 1;
+	}
+
+	psReservation = OSAllocZMem(sizeof(*psReservation) + ui64MapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psReservation, eError, ErrorUnreference);
+
+	/* Create lock */
+	eError = OSLockCreate(&psReservation->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeReservation);
+
+	/* Determine where the caller originated from. This is to prevent lockdep
+	 * warnings between reservations that originate from different sides of the
+	 * bridge. */
+	if (psConnectionData == NULL)
+	{
+		psReservation->eLockClass = RESERVATION_USAGE_SERVER_FW;
+	}
+	else
+	{
+		psReservation->eLockClass = RESERVATION_USAGE_CLIENT;
+	}
+
+	/* Initialise external device resource use indicator */
+	psReservation->i32DevResAcquisitionCount = DEVMEMRESERVATION_ACQUISITION_MIN;
+	/* Initialise UM-KM lifetime refcount */
+	OSAtomicWrite(&psReservation->iLifetimeRefCount, 1);
+
+	psReservation->uiFlags = uiFlags;
+	psReservation->sBase = sReservationVAddr;
+	psReservation->uiLength = uiVirtualSize;
+	psReservation->pui8Map = IMG_OFFSET_ADDR(psReservation, sizeof(*psReservation));
+
+	eError = MMU_Alloc(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                   uiVirtualSize,
+	                   0, /* IMG_UINT32 uiProtFlags */
+	                   0, /* alignment is n/a since we supply devvaddr */
+	                   &sReservationVAddr,
+	                   psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrorDestroyLock);
+
+	/* since we supplied the virt addr, MMU_Alloc shouldn't have
+	   chosen a new one for us */
+	PVR_ASSERT(sReservationVAddr.uiAddr == psReservation->sBase.uiAddr);
+
+	psReservation->psDevmemHeap = psDevmemHeap;
+	*ppsReservationPtr = psReservation;
+
+	return PVRSRV_OK;
+
+	/*
+	 *  error exit paths follow
+	 */
+
+ErrorDestroyLock:
+	OSLockDestroy(psReservation->hLock);
+
+ErrorFreeReservation:
+	OSFreeMem(psReservation);
+
+ErrorUnreference:
+	/* if fails there's not much to do (the function will print an error) */
+	DevmemIntHeapRelease(psDevmemHeap);
+
+ErrorReturnError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnreserveRange(DEVMEMINT_RESERVATION *psReservation)
+{
+	IMG_UINT32 i;
+	DEVMEMINT_HEAP *psDevmemHeap = psReservation->psDevmemHeap;
+	PVRSRV_ERROR eError;
+
+	if (psReservation->psMappedPMR != NULL)
+	{
+		/* No warning to be emitted as this is expected behaviour for the
+		 * Devmem interface. */
+		eError = DevmemIntUnmapPMR(psReservation);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevmemIntUnmapPMR");
+	}
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+
+	if (psReservation->i32DevResAcquisitionCount != DEVMEMRESERVATION_ACQUISITION_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but still has existing dev res acquisitions "
+		         "(%d), free existing acquisitions first.", __func__,
+		         psReservation->i32DevResAcquisitionCount));
+
+		OSLockRelease(psReservation->hLock);
+
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	OSLockRelease(psReservation->hLock);
+
+	MMU_Free(psDevmemHeap->psDevmemCtx->psMMUContext,
+	         psReservation->sBase,
+	         psReservation->uiLength,
+	         psDevmemHeap->uiLog2PageSize);
+
+	for (i = 0; i < _DevmemReservationPageCount(psReservation); i++)
+	{
+		if (DevmemIntReservationIsIndexMapped(psReservation, i))
+		{
+			eError = PMRUnrefPMR(psReservation->psMappedPMR);
+			PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+			DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+		}
+	}
+
+	DevmemIntHeapRelease(psDevmemHeap);
+
+	/* We have partially destroyed the object but we may need to keep the wrapper object
+	 * around since it may be referenced in KM threads where we need the lock to avoid
+	 * race conditions. UM threads will no longer have a handle to this object when this
+	 * bridge call exits.
+	 */
+	_DevmemReservationUnrefAndMaybeDestroy(psReservation);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+DevmemIntHeapDestroy(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	if (psDevmemHeap->pfnDeInit != NULL)
+	{
+		psDevmemHeap->pfnDeInit(psDevmemHeap->hPrivData);
+		psDevmemHeap->pfnDeInit = NULL;
+	}
+
+	if (OSAtomicRead(&psDevmemHeap->uiRefCount) != DEVMEMHEAP_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but still has existing references "
+		         "(%d), free existing reservations & mappings first.", __func__,
+		         OSAtomicRead(&psDevmemHeap->uiRefCount)));
+
+		/*
+		 * Try again later when you've freed all the memory
+		 *
+		 * Note:
+		 * We don't expect the application to retry (after all this call would
+		 * succeed if the client had freed all the memory which it should have
+		 * done before calling this function). However, given there should be
+		 * an associated handle, when the handle base is destroyed it will free
+		 * any allocations leaked by the client and then it will retry this call,
+		 * which should then succeed.
+		 */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	PVR_ASSERT(OSAtomicRead(&psDevmemHeap->uiRefCount) == DEVMEMHEAP_REFCOUNT_MIN);
+
+	BIT_UNSET(psDevmemHeap->psDevmemCtx->uiCreatedHeaps, psDevmemHeap->uiHeapIndex);
+
+	DevmemIntCtxRelease(psDevmemHeap->psDevmemCtx);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freed heap %p", __func__, psDevmemHeap));
+	OSFreeMem(psDevmemHeap);
+
+	return PVRSRV_OK;
+}
+
+IMG_DEV_VIRTADDR
+DevmemIntHeapGetBaseAddr(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	PVR_ASSERT(psDevmemHeap != NULL);
+
+	return psDevmemHeap->sBaseAddr;
+}
+
+static PVRSRV_ERROR
+DevmemIntChangeSparseValidateParams(PMR *psPMR,
+                                    IMG_UINT32 ui32AllocPageCount,
+                                    IMG_UINT32 ui32FreePageCount,
+                                    IMG_UINT32 ui32LogicalChunkCount,
+                                    SPARSE_MEM_RESIZE_FLAGS uiSparseFlags)
+{
+	PMRLockHeldAssert(psPMR);
+
+	/* Ensure a PMR has been mapped to this reservation. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiSparseFlags & SPARSE_RESIZE_BOTH, "uiSparseFlags");
+
+	if (!PMR_IsSparse(psPMR) || PMR_IsMemLayoutFixed(psPMR) ||
+	    PMR_IsClientCpuMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR cannot be changed because one or more of the following"
+		         " were true: !PMR_IsSparse() = %s, PMR_IsMemLayoutFixed() = %s,"
+		         " PMR_IsClientCpuMapped() = %s",
+		         __func__,
+		         !PMR_IsSparse(psPMR) ? "true" : "false",
+		         PMR_IsMemLayoutFixed(psPMR) ? "true" : "false",
+		         PMR_IsClientCpuMapped(psPMR) ? "true" : "false"));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (PMR_IsGpuMultiMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR cannot be changed because PMR_IsGpuMultiMapped() = true",
+		         __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		PVR_LOG_RETURN_IF_INVALID_PARAM(ui32AllocPageCount != 0, "ui32AllocPageCount");
+		PVR_LOG_RETURN_IF_FALSE(ui32AllocPageCount <= ui32LogicalChunkCount,
+		                        "ui32AllocPageCount is invalid",
+		                        PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE);
+	}
+
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		PVR_LOG_RETURN_IF_INVALID_PARAM(ui32FreePageCount != 0, "ui32FreePageCount");
+		PVR_LOG_RETURN_IF_FALSE(ui32FreePageCount <= ui32LogicalChunkCount,
+		                        "ui32FreePageCount is invalid",
+		                        PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+DevmemIntValidateSparsePMRIndices(IMG_UINT32 ui32PMRLogicalChunkCount,
+                                  IMG_UINT32 *paui32LogicalIndices,
+                                  IMG_UINT32 ui32LogicalIndexCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT8 *paui8TrackedIndices;
+	IMG_UINT32 ui32AllocSize;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32PMRLogicalChunkCount != 0);
+	PVR_ASSERT(paui32LogicalIndices != NULL);
+	PVR_ASSERT(ui32LogicalIndexCount != 0 && ui32LogicalIndexCount <= ui32PMRLogicalChunkCount);
+
+	ui32AllocSize = LOG2_DIV(ui32PMRLogicalChunkCount, 3);
+	if (LOG2_MOD(ui32PMRLogicalChunkCount, 3) != 0)
+	{
+		++ui32AllocSize;
+	}
+
+	paui8TrackedIndices = OSAllocZMem(ui32AllocSize);
+	if (paui8TrackedIndices == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	for (i = 0; i < ui32LogicalIndexCount; i++)
+	{
+		IMG_UINT32 ui32LogicalIndex = paui32LogicalIndices[i];
+
+		if (ui32LogicalIndex >= ui32PMRLogicalChunkCount)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Reservation index %u is OOB",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+			break;
+		}
+
+		if (BIT_ISSET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3)))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Duplicate index found: %u",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+
+		BIT_SET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3));
+	}
+
+	OSFreeMem(paui8TrackedIndices);
+	return eError;
+}
+
+static PVRSRV_ERROR
+DevmemIntComputeVirtualIndicesFromLogical(DEVMEMINT_RESERVATION *psReservation,
+                                          IMG_UINT32 ui32LogicalIndexCount,
+                                          IMG_UINT32 *paui32LogicalIndices,
+                                          IMG_UINT32 *pui32VirtualIndexCount,
+                                          IMG_UINT32 **ppaui32VirtualIndices)
+{
+	IMG_UINT32 ui32OrderDiff = PMR_GetLog2Contiguity(psReservation->psMappedPMR) -
+		psReservation->psDevmemHeap->uiLog2PageSize;
+
+	if (ui32OrderDiff == 0)
+	{
+		*pui32VirtualIndexCount = ui32LogicalIndexCount;
+		*ppaui32VirtualIndices = paui32LogicalIndices;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		IMG_UINT32 ui32PagesPerOrder = 1 << ui32OrderDiff;
+		IMG_UINT32 *paui32VirtualIndices;
+		IMG_UINT32 i = 0;
+
+		paui32VirtualIndices = OSAllocMem(*pui32VirtualIndexCount * sizeof(IMG_UINT32));
+		PVR_RETURN_IF_NOMEM(paui32VirtualIndices);
+
+		for (i = 0; i < ui32LogicalIndexCount; i++)
+		{
+			IMG_UINT32 ui32LogicalIndex = paui32LogicalIndices[i];
+			IMG_UINT32 ui32PageOffset;
+
+			for (ui32PageOffset = 0; ui32PageOffset < ui32PagesPerOrder; ui32PageOffset++)
+			{
+				IMG_UINT32 ui32VirtAddr = ui32LogicalIndex * ui32PagesPerOrder + ui32PageOffset;
+				paui32VirtualIndices[i * ui32PagesPerOrder + ui32PageOffset] = ui32VirtAddr;
+			}
+		}
+
+		*ppaui32VirtualIndices = paui32VirtualIndices;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntChangeSparse(IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiSparseFlags,
+                      DEVMEMINT_RESERVATION *psReservation)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	IMG_UINT32 ui32Log2PMRContiguity;
+	IMG_UINT32 ui32Log2HeapContiguity;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+
+	IMG_UINT32 *pai32MapIndices;
+	IMG_UINT32 *pai32UnmapIndices;
+	IMG_UINT32 uiMapPageCount;
+	IMG_UINT32 uiUnmapPageCount;
+
+	IMG_UINT32 ui32LogicalChunkCount;
+	PMR *psPMR;
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+
+	uiFlags = psReservation->uiFlags;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psReservation->psMappedPMR != NULL, eError, InvalidPMRErr);
+	psPMR = psReservation->psMappedPMR;
+
+	PMRLockPMR(psPMR);
+
+	ui32LogicalChunkCount = PMR_GetLogicalChunkCount(psPMR);
+	ui32Log2PMRContiguity = PMR_GetLog2Contiguity(psPMR);
+	ui32Log2HeapContiguity = psReservation->psDevmemHeap->uiLog2PageSize;
+
+	eError = DevmemIntChangeSparseValidateParams(psPMR,
+	                                             ui32AllocPageCount,
+	                                             ui32FreePageCount,
+	                                             ui32LogicalChunkCount,
+	                                             uiSparseFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntChangeSparseValidateParams", e0);
+
+	/* This is check is made in DevmemIntMapPMR - no need to do it again in release. */
+	PVR_ASSERT(ui32Log2HeapContiguity <= ui32Log2PMRContiguity);
+
+	pai32MapIndices = pai32AllocIndices;
+	pai32UnmapIndices = pai32FreeIndices;
+	uiMapPageCount = ui32AllocPageCount;
+	uiUnmapPageCount = ui32FreePageCount;
+
+	/* Pre check free indices against reservation given */
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		IMG_UINT32 i;
+
+		eError = DevmemIntValidateSparsePMRIndices(ui32LogicalChunkCount,
+		                                           pai32FreeIndices,
+		                                           ui32FreePageCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntValidateSparsePMRIndices", e0);
+
+		eError = DevmemIntComputeVirtualIndicesFromLogical(psReservation,
+		                                                   ui32FreePageCount,
+		                                                   pai32FreeIndices,
+		                                                   &uiUnmapPageCount,
+		                                                   &pai32UnmapIndices);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntComputeVirtualIndicesFromLogical", e1);
+
+		for (i = 0; i < uiUnmapPageCount; i++)
+		{
+			IMG_UINT32 ui32VirtIndex = pai32UnmapIndices[i];
+			IMG_BOOL bIsMapped;
+
+			bIsMapped = DevmemIntReservationIsIndexMapped(psReservation,
+			                                              ui32VirtIndex);
+			if (!bIsMapped)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Reservation index %u is not mapped into the reservation",
+				         __func__,
+				         ui32VirtIndex));
+
+				PVR_GOTO_WITH_ERROR(eError,
+				                    PVRSRV_ERROR_DEVICEMEM_NO_MAPPING,
+				                    e1);
+			}
+		}
+	}
+
+	/* Pre check alloc indices against reservation given */
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		IMG_UINT32 i;
+
+		eError = DevmemIntValidateSparsePMRIndices(ui32LogicalChunkCount,
+		                                           pai32AllocIndices,
+		                                           ui32AllocPageCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntValidateSparsePMRIndices", e1);
+
+		eError = DevmemIntComputeVirtualIndicesFromLogical(psReservation,
+		                                                   ui32AllocPageCount,
+		                                                   pai32AllocIndices,
+		                                                   &uiMapPageCount,
+		                                                   &pai32MapIndices);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntComputeVirtualIndicesFromLogical", e1);
+
+		for (i = 0; i < uiMapPageCount; i++)
+		{
+			IMG_UINT32 ui32VirtIndex = pai32MapIndices[i];
+			IMG_BOOL bIsMapped;
+
+			bIsMapped = DevmemIntReservationIsIndexMapped(psReservation,
+			                                              ui32VirtIndex);
+			if (bIsMapped)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Reservation index %u is mapped into the reservation",
+				         __func__,
+				         ui32VirtIndex));
+
+				PVR_GOTO_WITH_ERROR(eError,
+				                    PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED,
+				                    e1);
+			}
+		}
+	}
+
+	/* Invalidate the page table entries before freeing the physical pages. */
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		PMR_FLAGS_T uiPMRFlags;
+		IMG_UINT32 i;
+
+		/*Get the flags*/
+		uiPMRFlags = PMR_Flags(psPMR);
+
+		/* Unmap the pages and mark them invalid in the MMU PTE */
+		eError = MMU_UnmapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        uiFlags,
+		                        psReservation->sBase,
+		                        uiUnmapPageCount,
+		                        pai32UnmapIndices,
+		                        ui32Log2HeapContiguity,
+		                        uiPMRFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", e1);
+
+		for (i = 0; i < uiUnmapPageCount; i++)
+		{
+			IMG_UINT32 uiIndex = pai32UnmapIndices[i];
+
+			if (DevmemIntReservationIsIndexMapped(psReservation, uiIndex))
+			{
+				eError = PMRUnrefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation,
+				                                    uiIndex,
+				                                    IMG_FALSE);
+			}
+		}
+	}
+
+	/* Do the PMR specific changes */
+	eError = PMR_ChangeSparseMemUnlocked(psPMR,
+	                                     ui32AllocPageCount,
+	                                     pai32AllocIndices,
+	                                     ui32FreePageCount,
+	                                     pai32FreeIndices,
+	                                     uiSparseFlags);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: Failed to do PMR specific changes.",
+		         __func__));
+		goto e1;
+	}
+
+	/* Wire the pages tables that got allocated */
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		IMG_UINT32 i;
+
+		/* Map the pages and mark them Valid in the MMU PTE */
+		eError = MMU_MapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                      uiFlags,
+		                      psReservation->sBase,
+		                      psPMR,
+		                      0,
+		                      uiMapPageCount,
+		                      pai32MapIndices,
+		                      ui32Log2HeapContiguity);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: Failed to map alloc indices.",
+			         __func__));
+			goto e1;
+		}
+
+		for (i = 0; i < uiMapPageCount; i++)
+		{
+			IMG_UINT32 uiIndex = pai32MapIndices[i];
+
+			if (!DevmemIntReservationIsIndexMapped(psReservation, uiIndex))
+			{
+				PVRSRV_ERROR eError2 = PMRRefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError2, "PMRRefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation,
+				                                    uiIndex,
+				                                    IMG_TRUE);
+			}
+		}
+	}
+
+e1:
+	if (pai32MapIndices != pai32AllocIndices)
+	{
+		OSFreeMem(pai32MapIndices);
+	}
+	if (pai32UnmapIndices != pai32FreeIndices)
+	{
+		OSFreeMem(pai32UnmapIndices);
+	}
+e0:
+	PMRUnlockPMR(psPMR);
+InvalidPMRErr:
+	OSLockRelease(psReservation->hLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxDestroy
+@Description    Destroy that created by DevmemIntCtxCreate
+@Input          psDevmemCtx   Device Memory context
+@Return         cannot fail.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntCtxDestroy(DEVMEMINT_CTX *psDevmemCtx)
+{
+	/*
+	   We can't determine if we should be freeing the context here
+	   as a refcount!=1 could be due to either the fact that heap(s)
+	   remain with allocations on them, or that this memory context
+	   has been exported.
+	   As the client couldn't do anything useful with this information
+	   anyway and the fact that the refcount will ensure we only
+	   free the context when _all_ references have been released
+	   don't bother checking and just return OK regardless.
+	   */
+	DevmemIntCtxRelease(psDevmemCtx);
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntGetVDevAddrPageSize
+@Description    Get the page size for a virtual address.
+@Input			psDevNode
+@Input			sDevAddr			Get the page size for this virtual address.
+@Output			puiLog2HeapPageSize	On success returns log2 of the page size.
+@Return         Failure code if the virtual address is outside any heap.
+*/ /**************************************************************************/
+static
+PVRSRV_ERROR DevmemIntGetVDevAddrPageSize(PVRSRV_DEVICE_NODE *psDevNode,
+										  IMG_DEV_VIRTADDR sDevAddr,
+										  IMG_PUINT32 puiLog2HeapPageSize)
+{
+	IMG_UINT32 i, j, uiLog2HeapPageSize = 0;
+	DEVICE_MEMORY_INFO *psDinfo = &psDevNode->sDevMemoryInfo;
+	DEVMEM_HEAP_CONFIG *psConfig = psDinfo->psDeviceMemoryHeapConfigArray;
+
+	IMG_BOOL bFound = IMG_FALSE;
+
+	for (i = 0;
+		 i < psDinfo->uiNumHeapConfigs && !bFound;
+		 i++)
+	{
+		for (j = 0;
+			 j < psConfig[i].uiNumHeaps  && !bFound;
+			 j++)
+		{
+			IMG_DEV_VIRTADDR uiBase =
+					psConfig[i].psHeapBlueprintArray[j].sHeapBaseAddr;
+			IMG_DEVMEM_SIZE_T uiSize =
+					psConfig[i].psHeapBlueprintArray[j].uiHeapLength;
+
+			if ((sDevAddr.uiAddr >= uiBase.uiAddr) &&
+				(sDevAddr.uiAddr < (uiBase.uiAddr + uiSize)))
+			{
+				uiLog2HeapPageSize =
+						psConfig[i].psHeapBlueprintArray[j].uiLog2DataPageSize;
+				bFound = IMG_TRUE;
+			}
+		}
+	}
+
+	if (uiLog2HeapPageSize == 0)
+	{
+		return PVRSRV_ERROR_INVALID_GPU_ADDR;
+	}
+
+	*puiLog2HeapPageSize = uiLog2HeapPageSize;
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntIsVDevAddrValid
+@Description    Checks if a virtual address is valid for access.
+@Input          psConnection
+@Input			psDevNode
+@Input			psDevmemCtx			Device Memory context
+@Input			sDevAddr			Virtual address to check.
+@Return         Failure code if the virtual address is invalid.
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntIsVDevAddrValid(CONNECTION_DATA * psConnection,
+                                      PVRSRV_DEVICE_NODE *psDevNode,
+                                      DEVMEMINT_CTX *psDevMemContext,
+                                      IMG_DEV_VIRTADDR sDevAddr)
+{
+	IMG_UINT32 uiLog2HeapPageSize = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = DevmemIntGetVDevAddrPageSize(psDevNode,
+										  sDevAddr,
+										  &uiLog2HeapPageSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	return MMU_IsVDevAddrValid(psDevMemContext->psMMUContext,
+	                           uiLog2HeapPageSize,
+	                           sDevAddr) ? PVRSRV_OK : PVRSRV_ERROR_INVALID_GPU_ADDR;
+}
+
+PVRSRV_ERROR
+DevmemIntInvalidateFBSCTable(DEVMEMINT_CTX *psDevMemContext,
+                             IMG_UINT64 ui64FBSCEntryMask)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psDevMemContext->psDevNode;
+	MMU_CONTEXT *psMMUContext = psDevMemContext->psMMUContext;
+
+	if (psDevNode->pfnInvalFBSCTable)
+	{
+		return psDevNode->pfnInvalFBSCTable(psDevNode,
+		                                    psMMUContext,
+		                                    ui64FBSCEntryMask);
+	}
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR DevmemIntGetFaultAddress(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDevNode,
+                                      DEVMEMINT_CTX *psDevMemContext,
+                                      IMG_DEV_VIRTADDR *psFaultAddress)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	if ((psDevMemContext->ui32Flags & DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE) == 0)
+	{
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	*psFaultAddress = psDevMemContext->sFaultAddress;
+	psDevMemContext->ui32Flags &= ~DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE;
+
+	return PVRSRV_OK;
+}
+
+static POSWR_LOCK g_hExportCtxListLock;
+static DLLIST_NODE g_sExportCtxList;
+
+PVRSRV_ERROR
+DevmemIntInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	dllist_init(&g_sExportCtxList);
+
+	eError = OSWRLockCreate(&g_hExportCtxListLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntDeInit(void)
+{
+	PVR_ASSERT(dllist_is_empty(&g_sExportCtxList));
+
+	OSWRLockDestroy(g_hExportCtxListLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntExportCtx(DEVMEMINT_CTX *psContext,
+                   PMR *psPMR,
+                   DEVMEMINT_CTX_EXPORT **ppsContextExport)
+{
+	DEVMEMINT_CTX_EXPORT *psCtxExport;
+	PVRSRV_ERROR eError;
+
+	psCtxExport = OSAllocMem(sizeof(DEVMEMINT_CTX_EXPORT));
+	PVR_LOG_RETURN_IF_NOMEM(psCtxExport, "psCtxExport");
+
+	if (!DevmemIntCtxAcquire(psContext))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorFreeCtxExport);
+	}
+
+	eError = PMRRefPMR(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorIntCtxRelease);
+
+	/* Now that the source PMR is exported, the layout
+	 * can't change as there could be outstanding importers
+	 * This is to make sure both exporter and importers view of
+	 * the memory is same */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+	psCtxExport->psDevmemCtx = psContext;
+	psCtxExport->psPMR = psPMR;
+	OSWRLockAcquireWrite(g_hExportCtxListLock);
+	dllist_add_to_tail(&g_sExportCtxList, &psCtxExport->sNode);
+	OSWRLockReleaseWrite(g_hExportCtxListLock);
+
+	*ppsContextExport = psCtxExport;
+
+	return PVRSRV_OK;
+
+ErrorIntCtxRelease:
+	DevmemIntCtxRelease(psContext);
+ErrorFreeCtxExport:
+	OSFreeMem(psCtxExport);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnexportCtx(DEVMEMINT_CTX_EXPORT *psContextExport)
+{
+	PVRSRV_ERROR eError;
+
+	OSWRLockAcquireWrite(g_hExportCtxListLock);
+	dllist_remove_node(&psContextExport->sNode);
+	OSWRLockReleaseWrite(g_hExportCtxListLock);
+
+	eError = PMRUnrefPMR(psContextExport->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+	DevmemIntCtxRelease(psContextExport->psDevmemCtx);
+
+	OSFreeMem(psContextExport);
+
+	/* Unable to find exported context, return error */
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntAcquireRemoteCtx(PMR *psPMR,
+                          DEVMEMINT_CTX **ppsContext,
+                          IMG_HANDLE *phPrivData)
+{
+	PDLLIST_NODE psListNode, psListNodeNext;
+	DEVMEMINT_CTX_EXPORT *psCtxExport;
+
+	OSWRLockAcquireRead(g_hExportCtxListLock);
+	/* Find context from list using PMR as key */
+	dllist_foreach_node(&g_sExportCtxList, psListNode, psListNodeNext)
+	{
+		psCtxExport = IMG_CONTAINER_OF(psListNode, DEVMEMINT_CTX_EXPORT, sNode);
+		if (psCtxExport->psPMR == psPMR)
+		{
+			if (!DevmemIntCtxAcquire(psCtxExport->psDevmemCtx))
+			{
+				OSWRLockReleaseRead(g_hExportCtxListLock);
+
+				return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+			}
+			*ppsContext = psCtxExport->psDevmemCtx;
+			*phPrivData = psCtxExport->psDevmemCtx->hPrivData;
+
+			OSWRLockReleaseRead(g_hExportCtxListLock);
+
+			/* PMR should have been already exported to import it
+			 * If a PMR is exported, its immutable and the same is
+			 * checked here */
+			PVR_ASSERT(IMG_TRUE == PMR_IsMemLayoutFixed(psPMR));
+
+			return PVRSRV_OK;
+		}
+	}
+	OSWRLockReleaseRead(g_hExportCtxListLock);
+
+	/* Unable to find exported context, return error */
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed to acquire remote context. Could not retrieve context with given PMR",
+			__func__));
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntRegisterPFNotify
+@Description    Registers a PID to be notified when a page fault occurs on a
+                specific device memory context.
+@Input          psDevmemCtx    The context to be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntRegisterPFNotifyKM(DEVMEMINT_CTX *psDevmemCtx,
+                                         IMG_BOOL      bRegister)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	DLLIST_NODE         *psNode, *psNodeNext;
+	DEVMEMINT_PF_NOTIFY *psNotifyNode;
+	IMG_BOOL            bPresent = IMG_FALSE;
+	PVRSRV_ERROR        eError;
+	IMG_PID             ui32PID;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemCtx, "psDevmemCtx");
+
+	/* We can be certain that the memory context is valid and not freed during
+	 * the call time of this function as it's assured by the handle framework
+	 * (while the handle is looked up it cannot be freed). Therefore we can
+	 * safely retrieve the pointer to the device node and acquire the
+	 * hPageFaultNotifyLock lock.
+	 */
+	psDevNode = psDevmemCtx->psDevNode;
+
+	/* Acquire write lock to avoid resource free while
+	 * sMemoryContextPageFaultNotifyListHead and sProcessNotifyListHead
+	 * are being accessed.
+	 */
+	OSWRLockAcquireWrite(psDevNode->hPageFaultNotifyLock);
+
+	if (bRegister)
+	{
+		/* If this is the first PID in the list, the device memory context
+		 * needs to be registered for notification */
+		if (dllist_is_empty(&psDevmemCtx->sProcessNotifyListHead))
+		{
+			dllist_add_to_tail(&psDevNode->sMemoryContextPageFaultNotifyListHead,
+			                   &psDevmemCtx->sPageFaultNotifyListElem);
+		}
+	}
+
+	/* Obtain current client PID */
+	ui32PID = OSGetCurrentClientProcessIDKM();
+
+	/* Loop through the registered PIDs and check whether this one is
+	 * present */
+	dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+	{
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+
+		if (psNotifyNode->ui32PID == ui32PID)
+		{
+			bPresent = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (bRegister)
+	{
+		if (bPresent)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to register a PID that is already registered",
+			         __func__));
+			eError = PVRSRV_ERROR_PID_ALREADY_REGISTERED;
+			goto err_already_registered;
+		}
+
+		psNotifyNode = OSAllocMem(sizeof(*psNotifyNode));
+		if (psNotifyNode == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unable to allocate memory for the notify list",
+			          __func__));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto err_out_of_mem;
+		}
+		psNotifyNode->ui32PID = ui32PID;
+		/* Write lock is already held */
+		dllist_add_to_tail(&(psDevmemCtx->sProcessNotifyListHead), &(psNotifyNode->sProcessNotifyListElem));
+	}
+	else
+	{
+		if (!bPresent)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to unregister a PID that is not registered",
+			         __func__));
+			eError = PVRSRV_ERROR_PID_NOT_REGISTERED;
+			goto err_not_registered;
+		}
+		/* Write lock is already held */
+		dllist_remove_node(psNode);
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+		OSFreeMem(psNotifyNode);
+
+		/* If the last process in the list is being unregistered, then also
+		 * unregister the device memory context from the notify list. */
+		if (dllist_is_empty(&psDevmemCtx->sProcessNotifyListHead))
+		{
+			dllist_remove_node(&psDevmemCtx->sPageFaultNotifyListElem);
+		}
+	}
+	eError = PVRSRV_OK;
+
+err_already_registered:
+err_out_of_mem:
+err_not_registered:
+	OSWRLockReleaseWrite(psDevNode->hPageFaultNotifyLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntPFNotify
+@Description    Notifies any processes that have registered themselves to be
+                notified when a page fault happens on a specific device memory
+                context.
+@Input          *psDevNode           The device node.
+@Input          ui64FaultedPCAddress The page catalogue address that faulted.
+@Input          sFaultAddress        The address that triggered the fault.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntPFNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                               IMG_UINT64         ui64FaultedPCAddress,
+                               IMG_DEV_VIRTADDR   sFaultAddress)
+{
+	DLLIST_NODE         *psNode, *psNodeNext;
+	DEVMEMINT_PF_NOTIFY *psNotifyNode;
+	PVRSRV_ERROR        eError;
+	DEVMEMINT_CTX       *psDevmemCtx = NULL;
+	IMG_BOOL            bFailed = IMG_FALSE;
+
+	/* Protect access both to sMemoryContextPageFaultNotifyListHead and
+	 * to sProcessNotifyListHead. Those lists must be accessed atomically
+	 * in relation to each other, otherwise we risk accessing context that
+	 * might have already been destroyed. */
+	OSWRLockAcquireRead(psDevNode->hPageFaultNotifyLock);
+	if (dllist_is_empty(&(psDevNode->sMemoryContextPageFaultNotifyListHead)))
+	{
+		OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+		return PVRSRV_OK;
+	}
+
+	dllist_foreach_node(&(psDevNode->sMemoryContextPageFaultNotifyListHead), psNode, psNodeNext)
+	{
+		DEVMEMINT_CTX *psThisContext =
+			IMG_CONTAINER_OF(psNode, DEVMEMINT_CTX, sPageFaultNotifyListElem);
+		IMG_DEV_PHYADDR sPCDevPAddr;
+
+		eError = MMU_AcquireBaseAddr(psThisContext->psMMUContext, &sPCDevPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "MMU_AcquireBaseAddr");
+			OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+			return eError;
+		}
+
+		if (sPCDevPAddr.uiAddr == ui64FaultedPCAddress)
+		{
+			psDevmemCtx = psThisContext;
+			break;
+		}
+	}
+
+	if (psDevmemCtx == NULL)
+	{
+		/* Not found, just return */
+		OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+		return PVRSRV_OK;
+	}
+
+	/*
+	 * Store the first occurrence of a page fault address,
+	 * until that address is consumed by a client.
+	 */
+	if ((psDevmemCtx->ui32Flags & DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE) == 0)
+	{
+		psDevmemCtx->sFaultAddress = sFaultAddress;
+		psDevmemCtx->ui32Flags |= DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE;
+	}
+
+	/* Loop through each registered PID and send a signal to the process */
+	dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+	{
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+
+		eError = OSDebugSignalPID(psNotifyNode->ui32PID);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unable to signal process for PID: %u",
+			         __func__,
+			         psNotifyNode->ui32PID));
+
+			bFailed = IMG_TRUE;
+		}
+	}
+	OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+
+	if (bFailed)
+	{
+		return PVRSRV_ERROR_SIGNAL_FAILED;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(PDUMP)
+typedef struct _DEVMEMINT_PDUMP_VALID_REGION_
+{
+	DLLIST_NODE sNode;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+} DEVMEMINT_PDUMP_VALID_REGION;
+
+IMG_UINT32 DevmemIntMMUContextID(DEVMEMINT_CTX *psDevMemContext)
+{
+	IMG_UINT32 ui32MMUContextID;
+	MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32MMUContextID, PDUMP_FLAGS_CONTINUOUS);
+	return ui32MMUContextID;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize,
+                              DLLIST_NODE *psValidRegionsList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2HeapPageSize;
+	IMG_UINT32 uiHeapPageSize;
+	IMG_DEV_VIRTADDR sValidStart, sValidEnd, sCurrent, sEnd, sStartPage;
+
+	/* Get the page size for heap containing the start virtual address. */
+	eError = DevmemIntGetVDevAddrPageSize(psDeviceNode,
+	                                      sDevAddrStart,
+	                                      &uiLog2HeapPageSize);
+	PVR_RETURN_IF_ERROR(eError);
+
+	uiHeapPageSize = 1 << uiLog2HeapPageSize;
+
+	/* Iterate every page in the region to dump... */
+	sValidStart.uiAddr = sValidEnd.uiAddr = 0; /* Start/end of the current region which is valid to read. */
+	sStartPage.uiAddr = sDevAddrStart.uiAddr & ~((IMG_UINT64) uiHeapPageSize - 1); /* Page aligned start of the region to dump. */
+	sCurrent = sStartPage;
+	sEnd.uiAddr = sDevAddrStart.uiAddr + uiSize; /* End of the region to dump. */
+
+	while (IMG_TRUE)
+	{
+		IMG_BOOL bAtEnd = sCurrent.uiAddr >= sEnd.uiAddr;
+		IMG_BOOL bValid = IMG_FALSE;
+
+		if (!bAtEnd)
+		{
+			/* Check if the page starting at the current address is valid for reading. */
+			eError = DevmemIntIsVDevAddrValid(psConnection,
+			                                  psDeviceNode,
+			                                  psDevmemCtx,
+			                                  sCurrent);
+			if (eError == PVRSRV_OK)
+			{
+				/* If the current valid region is empty then set the start
+				 * to the current page. */
+				if (sValidStart.uiAddr == 0)
+				{
+					if (sCurrent.uiAddr == sStartPage.uiAddr)
+					{
+						/* Use the start of the region to dump if it doesn't
+						 * start page aligned. */
+						sValidStart = sDevAddrStart;
+					}
+					else
+					{
+						sValidStart = sCurrent;
+					}
+				}
+				/* Set the end of the valid region. */
+				sValidEnd.uiAddr = sCurrent.uiAddr + uiHeapPageSize;
+				/* Restrict to the region to dump. */
+				if (sValidEnd.uiAddr > sEnd.uiAddr)
+				{
+					sValidEnd = sEnd;
+				}
+				bValid = IMG_TRUE;
+			}
+			/* Move to the next page. */
+			sCurrent.uiAddr += uiHeapPageSize;
+		}
+
+		/* If the current page is invalid or we've reached the end of the region
+		 * to dump then pdump the current valid region. */
+		if (!bValid && sValidEnd.uiAddr > sValidStart.uiAddr)
+		{
+			DEVMEMINT_PDUMP_VALID_REGION *psRegion = OSAllocMem(sizeof(*psRegion));
+			PVR_LOG_GOTO_IF_NOMEM(psRegion, eError, ErrFreeRegions);
+
+			psRegion->sDevVAddr = sValidStart;
+			psRegion->uiSize = sValidEnd.uiAddr - sValidStart.uiAddr;
+
+			dllist_add_to_tail(psValidRegionsList, &psRegion->sNode);
+
+			sValidStart.uiAddr = sValidEnd.uiAddr = 0;
+		}
+
+		if (bAtEnd)
+		{
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+
+ErrFreeRegions:
+	DevmemIntPDumpFreeValidRegions(psValidRegionsList);
+	return eError;
+}
+
+void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psValidRegionsList)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(psValidRegionsList, psThis, psNext)
+	{
+		DEVMEMINT_PDUMP_VALID_REGION *psRegion =
+		    IMG_CONTAINER_OF(psThis, DEVMEMINT_PDUMP_VALID_REGION, sNode);
+
+		dllist_remove_node(psThis);
+		OSFreeMem(psRegion);
+	}
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+	DLLIST_NODE *psThis, *psNext;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+	                                    &uiPDumpMMUCtx,
+	                                    ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* The following SYSMEM refers to the 'MMU Context', hence it
+	 * should be the MMU context, not the PMR, that says what the PDump
+	 * MemSpace tag is?
+	 * From a PDump P.O.V. it doesn't matter which name space we use as long
+	 * as that MemSpace is used on the 'MMU Context' we're dumping from
+	 */
+
+	dllist_foreach_node(psDevAddrRegions, psThis, psNext)
+	{
+		DEVMEMINT_PDUMP_VALID_REGION *psRegion =
+		    IMG_CONTAINER_OF(psThis, DEVMEMINT_PDUMP_VALID_REGION, sNode);
+
+		eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+		                     psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+		                     uiPDumpMMUCtx,
+		                     psRegion->sDevVAddr,
+		                     psRegion->uiSize,
+		                     pszFilename,
+		                     ui32FileOffset,
+		                     ui32PDumpFlags);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		ui32FileOffset += psRegion->uiSize;
+
+		dllist_remove_node(psThis);
+		OSFreeMem(psRegion);
+	}
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 ui32ArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+	IMG_UINT32 uiLog2HeapPageSize;
+	IMG_UINT32 uiHeapPageSize;
+	IMG_DEV_VIRTADDR sValidStart, sValidEnd, sCurrent, sEnd, sStartPage;
+	IMG_UINT64 ui64PageMask;
+
+	PVR_UNREFERENCED_PARAMETER(ui32ArraySize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+			&uiPDumpMMUCtx,
+			ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/*
+	  Get the page size for heap containing the start virtual address.
+	 */
+	eError = DevmemIntGetVDevAddrPageSize(psDeviceNode,
+										  sDevAddrStart,
+										  &uiLog2HeapPageSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+	uiHeapPageSize = 1 << uiLog2HeapPageSize;
+	ui64PageMask = uiHeapPageSize - 1;
+
+	/*
+	  Iterate every page in the region to dump...
+	 */
+	sValidStart.uiAddr = sValidEnd.uiAddr = 0; /* Start/end of the current region which is valid to read. */
+	sStartPage.uiAddr = sDevAddrStart.uiAddr & ~ui64PageMask; /* Page aligned start of the region to dump. */
+	sCurrent = sStartPage;
+	sEnd.uiAddr = sDevAddrStart.uiAddr + uiSize; /* End of the region to dump. */
+	for (;;)
+	{
+		IMG_BOOL bAtEnd = sCurrent.uiAddr >= sEnd.uiAddr;
+		IMG_BOOL bValid = IMG_FALSE;
+
+		if (!bAtEnd)
+		{
+			/* Check if the page starting at the current address is valid for reading. */
+			eError = DevmemIntIsVDevAddrValid(psConnection,
+											  psDeviceNode,
+											  psDevmemCtx,
+											  sCurrent);
+			if (eError == PVRSRV_OK)
+			{
+				/* If the current valid region is empty then set the start to the current page. */
+				if (sValidStart.uiAddr == 0)
+				{
+					if (sCurrent.uiAddr == sStartPage.uiAddr)
+					{
+						/* Use the start of the region to dump if it doesn't start page aligned. */
+						sValidStart = sDevAddrStart;
+					}
+					else
+					{
+						sValidStart = sCurrent;
+					}
+				}
+				/* Set the end of the valid region. */
+				sValidEnd.uiAddr = sCurrent.uiAddr + uiHeapPageSize;
+				/* Restrict to the region to dump. */
+				if (sValidEnd.uiAddr > sEnd.uiAddr)
+				{
+					sValidEnd = sEnd;
+				}
+				bValid = IMG_TRUE;
+			}
+			/* Move to the next page. */
+			sCurrent.uiAddr += uiHeapPageSize;
+		}
+		/*
+		  If the current page is invalid or we've reached the end of the region to dump then pdump the current valid region.
+		 */
+		if (!bValid && sValidEnd.uiAddr > sValidStart.uiAddr)
+		{
+			IMG_DEVMEM_SIZE_T uiValidSize = sValidEnd.uiAddr - sValidStart.uiAddr;
+			eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+								 psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+								 uiPDumpMMUCtx,
+								 sValidStart,
+								 uiValidSize,
+								 pszFilename,
+								 ui32FileOffset,
+								 ui32PDumpFlags);
+			PVR_ASSERT(eError == PVRSRV_OK);
+
+			ui32FileOffset += uiValidSize;
+
+			sValidStart.uiAddr = sValidEnd.uiAddr = 0;
+		}
+
+		if (bAtEnd)
+		{
+			break;
+		}
+	}
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          DEVMEMINT_CTX *psDevmemCtx,
+                                          IMG_DEV_VIRTADDR sDevAddrStart,
+                                          IMG_DEVMEM_SIZE_T uiSize,
+                                          const IMG_CHAR *pszFilename,
+                                          IMG_UINT32 ui32FileOffset,
+                                          IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+	                                    &uiPDumpMMUCtx,
+	                                    ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* The following SYSMEM refers to the 'MMU Context', hence it
+	 * should be the MMU context, not the PMR, that says what the PDump
+	 * MemSpace tag is?
+	 * From a PDump P.O.V. it doesn't matter which name space we use as long
+	 * as that MemSpace is used on the 'MMU Context' we're dumping from
+	 */
+	eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+	                     psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+	                     uiPDumpMMUCtx,
+	                     sDevAddrStart,
+	                     uiSize,
+	                     pszFilename,
+	                     ui32FileOffset,
+	                     ui32PDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+							  PVRSRV_DEVICE_NODE *psDeviceNode,
+							  DEVMEMINT_CTX *psDevMemContext,
+							  IMG_UINT32 ui32Size,
+							  const IMG_CHAR *pszFileName,
+							  IMG_DEV_VIRTADDR sData,
+							  IMG_UINT32 ui32DataSize,
+							  IMG_UINT32 ui32LogicalWidth,
+							  IMG_UINT32 ui32LogicalHeight,
+							  IMG_UINT32 ui32PhysicalWidth,
+							  IMG_UINT32 ui32PhysicalHeight,
+							  PDUMP_PIXEL_FORMAT ePixFmt,
+							  IMG_MEMLAYOUT eMemLayout,
+							  IMG_FB_COMPRESSION eFBCompression,
+							  const IMG_UINT32 *paui32FBCClearColour,
+							  PDUMP_FBC_SWIZZLE eFBCSwizzle,
+							  IMG_DEV_VIRTADDR sHeader,
+							  IMG_UINT32 ui32HeaderSize,
+							  IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32 ui32ContextID;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	eError = MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32ContextID, ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_AcquirePDumpMMUContext");
+
+	eError = PDumpImageDescriptor(psDeviceNode,
+									ui32ContextID,
+									(IMG_CHAR *)pszFileName,
+									sData,
+									ui32DataSize,
+									ui32LogicalWidth,
+									ui32LogicalHeight,
+									ui32PhysicalWidth,
+									ui32PhysicalHeight,
+									ePixFmt,
+									eMemLayout,
+									eFBCompression,
+									paui32FBCClearColour,
+									eFBCSwizzle,
+									sHeader,
+									ui32HeaderSize,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "PDumpImageDescriptor");
+
+	/* Don't care about return value */
+	(void) MMU_ReleasePDumpMMUContext(psDevMemContext->psMMUContext, ui32PDumpFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+							 PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_CTX *psDevMemContext,
+							 IMG_UINT32 ui32Size,
+							 const IMG_CHAR *pszFileName,
+							 IMG_DEV_VIRTADDR sData,
+							 IMG_UINT32 ui32DataSize,
+							 IMG_UINT32 ui32HeaderType,
+							 IMG_UINT32 ui32ElementType,
+							 IMG_UINT32 ui32ElementCount,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32 ui32ContextID;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	if ((ui32HeaderType != IBIN_HEADER_TYPE) &&
+		(ui32HeaderType != DATA_HEADER_TYPE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid header type (%u)",
+		         __func__,
+		         ui32HeaderType));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32ContextID, ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_AcquirePDumpMMUContext");
+
+	eError = PDumpDataDescriptor(psDeviceNode,
+									ui32ContextID,
+									(IMG_CHAR *)pszFileName,
+									sData,
+									ui32DataSize,
+									ui32HeaderType,
+									ui32ElementType,
+									ui32ElementCount,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "PDumpDataDescriptor");
+
+	/* Don't care about return value */
+	(void) MMU_ReleasePDumpMMUContext(psDevMemContext->psMMUContext, ui32PDumpFlags);
+
+	return eError;
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.c
new file mode 100644
index 000000000000..9255e8622412
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.c
@@ -0,0 +1,881 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via pvrdebug.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "allocmem.h"
+#include "hash.h"
+#include "hash_functions.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "lock.h"
+#include "osfunc_common.h"
+#include "osfunc.h" /* for thread */
+#include "tlstream.h"
+#include "dllist.h"
+
+#include "osdi_impl.h"
+#include "di_impl_brg.h"
+#include "di_impl_brg_intern.h"
+#include "pvr_dicommon.h"
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+#include "pvrsrv.h"
+#endif
+
+#define ENTRIES_TABLE_INIT_SIZE 64
+#define STREAM_BUFFER_SIZE 0x4000 /* 16KB */
+#define STREAM_LINE_LENGTH 512
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define WRITER_THREAD_SLEEP_TIMEOUT 0ULL
+#else
+#define WRITER_THREAD_SLEEP_TIMEOUT 28800000000ULL
+#endif
+#define WRITER_THREAD_DESTROY_TIMEOUT 100000ULL
+#define WRITER_THREAD_DESTROY_RETRIES 10U
+
+#define WRITE_RETRY_COUNT 10      /* retry a write to a TL buffer 10 times */
+#define WRITE_RETRY_WAIT_TIME 100 /* wait 10ms between write retries */
+
+typedef enum THREAD_STATE
+{
+	THREAD_STATE_NULL,
+	THREAD_STATE_ALIVE,
+	THREAD_STATE_TERMINATED,
+} THREAD_STATE;
+
+static struct DIIB_IMPL
+{
+	HASH_TABLE *psEntriesTable;    /*!< Table of entries. */
+	POSWR_LOCK psEntriesLock;      /*!< Protects psEntriesTable. */
+	IMG_HANDLE hWriterThread;
+	IMG_HANDLE hWriterEventObject;
+	ATOMIC_T eThreadState;
+
+	DLLIST_NODE sWriterQueue;
+	POS_LOCK psWriterLock;         /*!< Protects sWriterQueue. */
+} *_g_psImpl;
+
+struct DIIB_GROUP
+{
+	const IMG_CHAR *pszName;
+	struct DIIB_GROUP *psParentGroup;
+};
+
+struct DIIB_ENTRY
+{
+	struct DIIB_GROUP *psParentGroup;
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+	DI_ENTRY_TYPE eType;
+	IMG_CHAR pszFullPath[DI_IMPL_BRG_PATH_LEN];
+	void *pvPrivData;
+
+	POS_LOCK hLock; /*!< Protects access to entry's iterator. */
+};
+
+struct DI_CONTEXT_TAG
+{
+	IMG_HANDLE hStream;
+	ATOMIC_T iRefCnt;
+	IMG_BOOL bClientConnected; /*!< Indicated that the client is or is not
+	                                connected to the DI. */
+};
+
+struct DIIB_WORK_ITEM
+{
+	DI_CONTEXT *psContext;
+	DIIB_ENTRY *psEntry;
+	IMG_UINT64 ui64Size;
+	IMG_UINT64 ui64Offset;
+
+	DLLIST_NODE sQueueElement;
+};
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteWithRetires(void *pvNativeHandle, const IMG_CHAR *pszStr,
+                              IMG_UINT uiLen)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iRetry = 0;
+	IMG_UINT32 ui32Flags = TL_FLAG_NO_WRITE_FAILED;
+
+	do
+	{
+		/* Try to write to the buffer but don't inject MOST_RECENT_WRITE_FAILED
+		 * packet in case of failure because we're going to retry. */
+		eError = TLStreamWriteRetFlags(pvNativeHandle, (IMG_UINT8 *) pszStr,
+		                               uiLen, &ui32Flags);
+		if (eError == PVRSRV_ERROR_STREAM_FULL)
+		{
+			// wait to give the client a change to read
+			OSSleepms(WRITE_RETRY_WAIT_TIME);
+		}
+	}
+	while (eError == PVRSRV_ERROR_STREAM_FULL && iRetry++ < WRITE_RETRY_COUNT);
+
+	/* One last try to write to the buffer. In this case upon failure
+	 * a MOST_RECENT_WRITE_FAILED packet will be inject to the buffer to
+	 * indicate data loss. */
+	if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		eError = TLStreamWrite(pvNativeHandle, (IMG_UINT8 *) pszStr, uiLen);
+	}
+
+	PVR_LOG_IF_ERROR(eError, "TLStreamWrite");
+}
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	_WriteWithRetires(pvNativeHandle, pvData, uiSize);
+}
+
+__printf(2, 0)
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	IMG_CHAR pcBuffer[STREAM_LINE_LENGTH];
+	IMG_UINT uiLen = OSVSNPrintf(pcBuffer, sizeof(pcBuffer) - 1, pszFmt, pArgs);
+	pcBuffer[uiLen] = '\0';
+
+	_WriteWithRetires(pvNativeHandle, pcBuffer, uiLen + 1);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	_WriteWithRetires(pvNativeHandle, pszStr, OSStringLength(pszStr) + 1);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvNativeHandle);
+	return IMG_FALSE;
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- entry operations --------------------------------------------------- */
+
+static PVRSRV_ERROR _ContextUnrefAndMaybeDestroy(DI_CONTEXT *psContext)
+{
+	if (OSAtomicDecrement(&psContext->iRefCnt) == 0)
+	{
+		TLStreamClose(psContext->hStream);
+		OSFreeMem(psContext);
+	}
+
+	return PVRSRV_OK;
+}
+
+static IMG_INT64 _ReadGeneric(const DI_CONTEXT *psContext, DIIB_ENTRY *psEntry)
+{
+	IMG_INT64 iRet = 0;
+	IMG_UINT64 ui64Pos = 0;
+	DI_ITERATOR_CB *psIter = &psEntry->sIterCb;
+	OSDI_IMPL_ENTRY *psImplEntry = &psEntry->sImplEntry;
+	PVRSRV_ERROR eError;
+
+	if (psIter->pfnStart != NULL)
+	{
+		/* this is a full sequence of the operation */
+		void *pvData = psIter->pfnStart(psImplEntry, &ui64Pos);
+
+		while (pvData != NULL && psContext->bClientConnected)
+		{
+			iRet = psIter->pfnShow(psImplEntry, pvData);
+			if (iRet < 0)
+			{
+				break;
+			}
+
+			pvData = psIter->pfnNext(psImplEntry, pvData, &ui64Pos);
+		}
+
+		psIter->pfnStop(psImplEntry, pvData);
+	}
+	else if (psIter->pfnShow != NULL)
+	{
+		/* this is a simplified sequence of the operation */
+		iRet = psIter->pfnShow(psImplEntry, NULL);
+	}
+
+	eError = TLStreamMarkEOS(psImplEntry->pvNative, IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamMarkEOS", return_error_);
+
+	return iRet;
+
+return_error_:
+	return -1;
+}
+
+static IMG_INT64 _ReadRndAccess(DIIB_ENTRY *psEntry, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT8 *pui8Buffer;
+	IMG_HANDLE hStream = psEntry->sImplEntry.pvNative;
+
+	if (psEntry->sIterCb.pfnRead == NULL)
+	{
+		return -1;
+	}
+
+	eError = TLStreamReserve(hStream, &pui8Buffer, ui64Count);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamReserve", return_error_);
+
+	psEntry->sIterCb.pfnRead((IMG_CHAR *) pui8Buffer, ui64Count, pui64Pos,
+	                         pvData);
+
+	eError = TLStreamCommit(hStream, ui64Count);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCommit", return_error_);
+
+	eError = TLStreamMarkEOS(psEntry->sImplEntry.pvNative, IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamMarkEOS", return_error_);
+
+	return 0;
+
+return_error_:
+	return -1;
+}
+
+static void _WriterThread(void *pvArg)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	DLLIST_NODE *psNode;
+
+	PVR_UNREFERENCED_PARAMETER(pvArg);
+
+	eError = OSEventObjectOpen(_g_psImpl->hWriterEventObject, &hEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#ifdef PVRSRV_FORCE_UNLOAD_IF_BAD_STATE
+	while (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK &&
+	       OSAtomicRead(&_g_psImpl->eThreadState) == THREAD_STATE_ALIVE)
+#else
+	while (OSAtomicRead(&_g_psImpl->eThreadState) == THREAD_STATE_ALIVE)
+#endif
+	{
+		struct DIIB_WORK_ITEM *psItem = NULL;
+
+		OSLockAcquire(_g_psImpl->psWriterLock);
+		/* Get element from list tail so that we always get the oldest element
+		 * (elements are added to head). */
+		while ((psNode = dllist_get_prev_node(&_g_psImpl->sWriterQueue)) != NULL)
+		{
+			IMG_INT64 i64Ret;
+			DIIB_ENTRY *psEntry;
+			OSDI_IMPL_ENTRY *psImplEntry;
+
+			dllist_remove_node(psNode);
+			OSLockRelease(_g_psImpl->psWriterLock);
+
+			psItem = IMG_CONTAINER_OF(psNode, struct DIIB_WORK_ITEM,
+			                          sQueueElement);
+
+			psEntry = psItem->psEntry;
+			psImplEntry = &psItem->psEntry->sImplEntry;
+
+			/* if client has already disconnected we can just drop this item */
+			if (psItem->psContext->bClientConnected)
+			{
+
+				PVR_ASSERT(psItem->psContext->hStream != NULL);
+
+				psImplEntry->pvNative = psItem->psContext->hStream;
+
+				if (psEntry->eType == DI_ENTRY_TYPE_GENERIC)
+				{
+					i64Ret = _ReadGeneric(psItem->psContext, psEntry);
+					PVR_LOG_IF_FALSE(i64Ret >= 0, "generic access read operation "
+					                 "failed");
+				}
+				else if (psEntry->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+				{
+					IMG_UINT64 ui64Pos = psItem->ui64Offset;
+
+					i64Ret = _ReadRndAccess(psEntry, psItem->ui64Size, &ui64Pos,
+					                        psEntry->pvPrivData);
+					PVR_LOG_IF_FALSE(i64Ret >= 0, "random access read operation "
+					                 "failed");
+				}
+				else
+				{
+					PVR_ASSERT(psEntry->eType == DI_ENTRY_TYPE_GENERIC ||
+					           psEntry->eType == DI_ENTRY_TYPE_RANDOM_ACCESS);
+				}
+
+				psImplEntry->pvNative = NULL;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "client reading entry \"%s\" has "
+				        "disconnected", psEntry->pszFullPath));
+			}
+
+			_ContextUnrefAndMaybeDestroy(psItem->psContext);
+			OSFreeMemNoStats(psItem);
+
+			OSLockAcquire(_g_psImpl->psWriterLock);
+		}
+		OSLockRelease(_g_psImpl->psWriterLock);
+
+		eError = OSEventObjectWaitKernel(hEvent, WRITER_THREAD_SLEEP_TIMEOUT);
+		if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_LOG_ERROR(eError, "OSEventObjectWaitKernel");
+		}
+	}
+
+	OSLockAcquire(_g_psImpl->psWriterLock);
+	/* clear the queue if there are any items pending */
+	while ((psNode = dllist_get_prev_node(&_g_psImpl->sWriterQueue)) != NULL)
+	{
+		struct DIIB_WORK_ITEM *psItem = IMG_CONTAINER_OF(psNode,
+		                                                 struct DIIB_WORK_ITEM,
+		                                                 sQueueElement);
+
+		dllist_remove_node(psNode);
+		_ContextUnrefAndMaybeDestroy(psItem->psContext);
+		OSFreeMem(psItem);
+	}
+	OSLockRelease(_g_psImpl->psWriterLock);
+
+	eError = OSEventObjectClose(hEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_TERMINATED);
+}
+
+/* ----- DI internal API ---------------------------------------------------- */
+
+DIIB_ENTRY *DIImplBrgFind(const IMG_CHAR *pszPath)
+{
+	DIIB_ENTRY *psEntry;
+
+	OSWRLockAcquireRead(_g_psImpl->psEntriesLock);
+	psEntry = (void *) HASH_Retrieve_Extended(_g_psImpl->psEntriesTable,
+	                                          (IMG_CHAR *) pszPath);
+	OSWRLockReleaseRead(_g_psImpl->psEntriesLock);
+
+	return psEntry;
+}
+
+/* ----- DI bridge interface ------------------------------------------------ */
+
+static PVRSRV_ERROR _CreateStream(IMG_CHAR *pszStreamName, IMG_HANDLE *phStream)
+{
+	IMG_UINT32 iRet;
+	IMG_HANDLE hStream;
+	PVRSRV_ERROR eError;
+
+	/* for now only one stream can be created. Should we be able to create
+	 * per context stream? */
+	iRet = OSSNPrintf(pszStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+	                  "di_stream_%x", OSGetCurrentClientProcessIDKM());
+	if (iRet >= PRVSRVTL_MAX_STREAM_NAME_SIZE)
+	{
+		/* this check is superfluous because it can never happen but in case
+		 * someone changes the definition of PRVSRVTL_MAX_STREAM_NAME_SIZE
+		 * handle this case */
+		pszStreamName[0] = '\0';
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	eError = TLStreamCreate(&hStream, pszStreamName, STREAM_BUFFER_SIZE,
+	                        TL_OPMODE_DROP_NEWER, NULL, NULL, NULL, NULL,
+	                        NULL, NULL);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*phStream = hStream;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DICreateContextKM(IMG_CHAR *pszStreamName, DI_CONTEXT **ppsContext)
+{
+	PVRSRV_ERROR eError;
+	DI_CONTEXT *psContext;
+	IMG_HANDLE hStream = NULL;
+	THREAD_STATE eTState;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsContext != NULL, "ppsContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszStreamName != NULL, "pszStreamName");
+
+	psContext = OSAllocMem(sizeof(*psContext));
+	PVR_LOG_GOTO_IF_NOMEM(psContext, eError, return_);
+
+	eError = _CreateStream(pszStreamName, &hStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CreateStream", free_desc_);
+
+	psContext->hStream = hStream;
+	/* indicated to the write thread if the client is still connected and
+	 * waiting for the data */
+	psContext->bClientConnected = IMG_TRUE;
+	OSAtomicWrite(&psContext->iRefCnt, 1);
+
+	eTState = OSAtomicCompareExchange(&_g_psImpl->eThreadState,
+	                                  THREAD_STATE_NULL,
+	                                  THREAD_STATE_ALIVE);
+
+	/* if the thread has not been started yet do it */
+	if (eTState == THREAD_STATE_NULL)
+	{
+		PVR_ASSERT(_g_psImpl->hWriterThread == NULL);
+
+		eError = OSThreadCreate(&_g_psImpl->hWriterThread, "di_writer",
+		                        _WriterThread, NULL, IMG_FALSE, NULL);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreate", free_close_stream_);
+	}
+
+	*ppsContext = psContext;
+
+	return PVRSRV_OK;
+
+free_close_stream_:
+	TLStreamClose(psContext->hStream);
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_TERMINATED);
+free_desc_:
+	OSFreeMem(psContext);
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIDestroyContextKM(DI_CONTEXT *psContext)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+
+	/* pass the information to the write thread that the client has
+	 * disconnected */
+	psContext->bClientConnected = IMG_FALSE;
+
+	return _ContextUnrefAndMaybeDestroy(psContext);
+}
+
+PVRSRV_ERROR DIReadEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT64 ui64Offset, IMG_UINT64 ui64Size)
+{
+	PVRSRV_ERROR eError;
+	struct DIIB_WORK_ITEM *psItem;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszEntryPath != NULL, "pszEntryPath");
+
+	/* 'no stats' to avoid acquiring the process stats locks */
+	psItem = OSAllocMemNoStats(sizeof(*psItem));
+	PVR_LOG_GOTO_IF_NOMEM(psItem, eError, return_);
+
+	psItem->psContext = psContext;
+	psItem->psEntry = DIImplBrgFind(pszEntryPath);
+	PVR_LOG_GOTO_IF_FALSE_VA(psItem->psEntry != NULL, free_item_,
+	                         "entry %s does not exist", pszEntryPath);
+	psItem->ui64Size = ui64Size;
+	psItem->ui64Offset = ui64Offset;
+
+	/* increment ref count on the context so that it doesn't get freed
+	 * before it gets processed by the writer thread. */
+	if (OSAtomicAddUnless(&psContext->iRefCnt, 1, IMG_INT32_MAX) == IMG_INT32_MAX)
+	{
+		/* Job is not scheduled to the writer queue; there are too many waiting. */
+		PVR_LOG_GOTO_WITH_ERROR("OSAtomicAddUnless", eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, overflow_);
+	}
+
+	OSLockAcquire(_g_psImpl->psWriterLock);
+	dllist_add_to_head(&_g_psImpl->sWriterQueue, &psItem->sQueueElement);
+	OSLockRelease(_g_psImpl->psWriterLock);
+
+	eError = OSEventObjectSignal(_g_psImpl->hWriterEventObject);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+
+	return PVRSRV_OK;
+
+free_item_:
+	eError = PVRSRV_ERROR_NOT_FOUND;
+overflow_:
+	OSFreeMemNoStats(psItem);
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIWriteEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT32 ui32ValueSize, const IMG_CHAR *pszValue)
+{
+	DIIB_ENTRY *psEntry;
+	DI_PFN_WRITE pfnEntryPuts;
+	IMG_INT64 i64Length = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszEntryPath != NULL, "pszEntryPath");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszValue != NULL, "pszValue");
+
+	psEntry = DIImplBrgFind(pszEntryPath);
+	PVR_LOG_RETURN_IF_FALSE_VA(psEntry != NULL, PVRSRV_ERROR_NOT_FOUND,
+	                         "entry %s does not exist", pszEntryPath);
+
+	pfnEntryPuts = psEntry->sIterCb.pfnWrite;
+	if (pfnEntryPuts != NULL)
+	{
+		i64Length = pfnEntryPuts(pszValue, ui32ValueSize, (IMG_UINT64*)&i64Length, psEntry->pvPrivData);
+
+		/* To deal with -EINVAL being returned */
+		PVR_LOG_RETURN_IF_INVALID_PARAM(i64Length >= 0, pszValue);
+	}
+	else
+	{
+		PVR_LOG_MSG(PVR_DBG_WARNING, "Unable to write to Entry. Write callback not enabled");
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _listName(uintptr_t k,
+                               uintptr_t v,
+                               void* hStream)
+{
+	PVRSRV_ERROR eError;
+	DIIB_ENTRY *psEntry;
+	IMG_UINT32 ui32Size;
+	IMG_CHAR aszName[DI_IMPL_BRG_PATH_LEN];
+
+	psEntry = (DIIB_ENTRY*) v;
+	PVR_ASSERT(psEntry != NULL);
+	PVR_UNREFERENCED_PARAMETER(k);
+
+	ui32Size = OSSNPrintf(aszName, DI_IMPL_BRG_PATH_LEN, "%s\n", psEntry->pszFullPath);
+	PVR_LOG_IF_FALSE(ui32Size > 5, "ui32Size too small, Error suspected!");
+	eError = TLStreamWrite(hStream, (IMG_UINT8 *)aszName, ui32Size+1);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR DIListAllEntriesKM(DI_CONTEXT *psContext)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+
+	OSWRLockAcquireRead(_g_psImpl->psEntriesLock);
+	eError = HASH_Iterate(_g_psImpl->psEntriesTable, _listName, psContext->hStream);
+	OSWRLockReleaseRead(_g_psImpl->psEntriesLock);
+	PVR_LOG_IF_ERROR(eError, "HASH_Iterate_Extended");
+
+	eError = TLStreamMarkEOS(psContext->hStream, IMG_FALSE);
+	return eError;
+}
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	PVRSRV_ERROR eError;
+
+	_g_psImpl = OSAllocMem(sizeof(*_g_psImpl));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psImpl, eError, return_);
+
+	_g_psImpl->psEntriesTable = HASH_Create_Extended(ENTRIES_TABLE_INIT_SIZE,
+	                                                 DI_IMPL_BRG_PATH_LEN,
+	                                                 HASH_Djb2_Hash,
+	                                                 HASH_Djb2_Compare);
+	PVR_LOG_GOTO_IF_NOMEM(_g_psImpl->psEntriesTable, eError, free_impl_);
+
+	eError = OSWRLockCreate(&_g_psImpl->psEntriesLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSCreateLock", free_table_);
+
+	eError = OSLockCreate(&_g_psImpl->psWriterLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSCreateLock", free_entries_lock_);
+
+	eError = OSEventObjectCreate("DI_WRITER_EO",
+	                             &_g_psImpl->hWriterEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", free_writer_lock_);
+
+	_g_psImpl->hWriterThread = NULL;
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_NULL);
+
+	dllist_init(&_g_psImpl->sWriterQueue);
+
+	return PVRSRV_OK;
+
+free_writer_lock_:
+	OSLockDestroy(_g_psImpl->psWriterLock);
+free_entries_lock_:
+	OSWRLockDestroy(_g_psImpl->psEntriesLock);
+free_table_:
+	HASH_Delete_Extended(_g_psImpl->psEntriesTable, IMG_FALSE);
+free_impl_:
+	OSFreeMem(_g_psImpl);
+	_g_psImpl = NULL;
+return_:
+	return eError;
+}
+
+static void _DeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	THREAD_STATE eTState;
+
+	eTState = OSAtomicCompareExchange(&_g_psImpl->eThreadState,
+	                                  THREAD_STATE_ALIVE,
+	                                  THREAD_STATE_TERMINATED);
+
+	if (eTState == THREAD_STATE_ALIVE)
+	{
+		if (_g_psImpl->hWriterEventObject != NULL)
+		{
+			eError = OSEventObjectSignal(_g_psImpl->hWriterEventObject);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(WRITER_THREAD_DESTROY_TIMEOUT)
+		{
+			eError = OSThreadDestroy(_g_psImpl->hWriterThread);
+			if (eError == PVRSRV_OK)
+			{
+				break;
+			}
+			OSWaitus(WRITER_THREAD_DESTROY_TIMEOUT/WRITER_THREAD_DESTROY_RETRIES);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (_g_psImpl->hWriterEventObject != NULL)
+	{
+		eError = OSEventObjectDestroy(_g_psImpl->hWriterEventObject);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	HASH_Delete_Extended(_g_psImpl->psEntriesTable, IMG_FALSE);
+	OSLockDestroy(_g_psImpl->psWriterLock);
+	OSWRLockDestroy(_g_psImpl->psEntriesLock);
+	OSFreeMem(_g_psImpl);
+	_g_psImpl = NULL;
+}
+
+/* Recursively traverses the ancestors list up to the root group and
+ * appends their names preceded by "/" to the path in reverse order
+ * (root group's name first and psGroup group's name last).
+ * Returns current offset in the path (the current path length without the
+ * NUL character). If there is no more space in the path returns -1
+ * to indicate an error (the path is too long to fit into the buffer). */
+static ssize_t _BuildGroupPath(IMG_CHAR *pszPath, const DIIB_GROUP *psGroup)
+{
+	ssize_t iOff;
+	ssize_t iCopiedCnt;
+	size_t  uFreeCnt;
+
+	PVR_RETURN_IF_FALSE(psGroup != NULL, 0);
+
+	PVR_ASSERT(pszPath != NULL);
+
+	iOff = _BuildGroupPath(pszPath, psGroup->psParentGroup);
+	PVR_RETURN_IF_FALSE(iOff != -1, -1);
+
+	uFreeCnt = DI_IMPL_BRG_PATH_LEN - iOff;
+	PVR_RETURN_IF_FALSE(uFreeCnt > 1, -1);
+
+	pszPath[iOff++] = '/';
+	--uFreeCnt;
+
+	iCopiedCnt = OSStringSafeCopy(&pszPath[iOff], psGroup->pszName, uFreeCnt);
+	PVR_RETURN_IF_FALSE(iCopiedCnt >= 0, -1);
+	iOff += iCopiedCnt;
+
+	return iOff;
+}
+
+static PVRSRV_ERROR _BuildEntryPath(IMG_CHAR *pszPath, const IMG_CHAR *pszName,
+                                    const DIIB_GROUP *psGroup)
+{
+	ssize_t iOff = _BuildGroupPath(pszPath, psGroup);
+	ssize_t iCopiedCnt;
+	size_t  uFreeCnt;
+
+	PVR_RETURN_IF_FALSE(iOff != -1, PVRSRV_ERROR_INVALID_OFFSET);
+
+	uFreeCnt = DI_IMPL_BRG_PATH_LEN - iOff;
+	PVR_RETURN_IF_FALSE(uFreeCnt > 1, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	pszPath[iOff++] = '/';
+	--uFreeCnt;
+
+	iCopiedCnt = OSStringSafeCopy(&pszPath[iOff], pszName, uFreeCnt);
+	PVR_RETURN_IF_FALSE(iCopiedCnt >= 0, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _CreateEntry(const IMG_CHAR *pszName,
+                                 DI_ENTRY_TYPE eType,
+                                 const DI_ITERATOR_CB *psIterCb,
+                                 void *pvPrivData,
+                                 void *pvParentGroup,
+                                 void **pvEntry)
+{
+	DIIB_GROUP *psParentGroup = pvParentGroup;
+	DIIB_ENTRY *psEntry;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvEntry != NULL, "pvEntry");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentGroup != NULL, "pvParentGroup");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, return_);
+	}
+
+	psEntry = OSAllocMem(sizeof(*psEntry));
+	PVR_LOG_GOTO_IF_NOMEM(psEntry, eError, return_);
+
+	eError = OSLockCreate(&psEntry->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", free_entry_);
+
+	psEntry->eType = eType;
+	psEntry->sIterCb = *psIterCb;
+	psEntry->pvPrivData = pvPrivData;
+	psEntry->psParentGroup = psParentGroup;
+	psEntry->pszFullPath[0] = '\0';
+
+	psEntry->sImplEntry.pvPrivData = pvPrivData;
+	psEntry->sImplEntry.pvNative = NULL;
+	psEntry->sImplEntry.psCb = &_g_sEntryCallbacks;
+
+	eError = _BuildEntryPath(psEntry->pszFullPath, pszName,
+	                         psEntry->psParentGroup);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s() failed in _BuildEntryPath() for \"%s\" "
+		        "entry", __func__, pszName));
+		goto destroy_lock_;
+	}
+
+	OSWRLockAcquireWrite(_g_psImpl->psEntriesLock);
+	eError = HASH_Insert_Extended(_g_psImpl->psEntriesTable,
+	                              psEntry->pszFullPath,
+	                              (uintptr_t) psEntry) ?
+	         PVRSRV_OK : PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+	OSWRLockReleaseWrite(_g_psImpl->psEntriesLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HASH_Insert_Extended failed", destroy_lock_);
+
+	*pvEntry = psEntry;
+
+	return PVRSRV_OK;
+
+destroy_lock_:
+	OSLockDestroy(psEntry->hLock);
+free_entry_:
+	OSFreeMem(psEntry);
+return_:
+	return eError;
+}
+
+static void _DestroyEntry(void *pvEntry)
+{
+	DIIB_ENTRY *psEntry = pvEntry;
+	PVR_ASSERT(psEntry != NULL);
+
+	OSWRLockAcquireWrite(_g_psImpl->psEntriesLock);
+	HASH_Remove_Extended(_g_psImpl->psEntriesTable, psEntry->pszFullPath);
+	OSWRLockReleaseWrite(_g_psImpl->psEntriesLock);
+
+	OSLockDestroy(psEntry->hLock);
+	OSFreeMem(psEntry);
+}
+
+static PVRSRV_ERROR _CreateGroup(const IMG_CHAR *pszName,
+                                 void *pvParentGroup,
+                                 void **ppvGroup)
+{
+	DIIB_GROUP *psNewGroup;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvGroup != NULL, "ppvGroup");
+
+	psNewGroup = OSAllocMem(sizeof(*psNewGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psNewGroup, "OSAllocMem");
+
+	psNewGroup->pszName = pszName;
+	psNewGroup->psParentGroup = pvParentGroup;
+
+	*ppvGroup = psNewGroup;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyGroup(void *pvGroup)
+{
+	DIIB_GROUP *psGroup = pvGroup;
+	PVR_ASSERT(psGroup != NULL);
+
+	OSFreeMem(psGroup);
+}
+
+PVRSRV_ERROR PVRDIImplBrgRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateEntry,
+		.pfnDestroyEntry = _DestroyEntry,
+		.pfnCreateGroup = _CreateGroup,
+		.pfnDestroyGroup = _DestroyGroup
+	};
+
+	return DIRegisterImplementation("impl_brg", &sImplCb);
+}
+
+/******************************************************************************
+ End of file (di_impl_brg.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.h b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.h
new file mode 100644
index 000000000000..b8cff25d94ec
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_IMPL_BRG_H
+#define PVR_IMPL_BRG_H
+
+#include "pvrsrv_error.h"
+#include "lock_types.h"
+
+typedef struct DI_CONTEXT_TAG DI_CONTEXT;
+typedef struct DI_ENTRY_DESC DI_ENTRY_DESC;
+
+PVRSRV_ERROR PVRDIImplBrgRegister(void);
+
+/*! @Function DICreateContextKM
+ *
+ * @Description
+ * Creates DI context which among others also creates a TL stream for reading
+ * entries.
+ *
+ * @Output pszStreamName: name of the TL stream created in this context
+ * @Output ppsContext: pointer to the new context
+ *
+ * @Return PVRSRV_ERROR error code
+ *         PVRSRV_OK in case of a success
+ *         PVRSRV_ERROR_INVALID_PARAMS if any of the parameters is invalid
+ *         PVRSRV_ERROR_OUT_OF_MEMORY if any of the memory allocations failed
+ *         error codes returned by TLStreamCreate()
+ */
+PVRSRV_ERROR DICreateContextKM(IMG_CHAR *pszStreamName,
+                               DI_CONTEXT **ppsContext);
+
+/*! @Function DIDestroyContextKM
+ *
+ * @Description
+ * Destroy the DI context and all underlying dependencies.
+ *
+ * @Input psContext: pointer to the context
+ *
+ * @Return PVRSRV_ERROR error code
+ *         PVRSRV_OK in case of a success
+ *         PVRSRV_ERROR_INVALID_PARAMS if invalid context pointer given
+ */
+PVRSRV_ERROR DIDestroyContextKM(DI_CONTEXT *psContext);
+
+PVRSRV_ERROR DIReadEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT64 ui64Offset, IMG_UINT64 ui64Size);
+
+PVRSRV_ERROR DIWriteEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                            IMG_UINT32 ui32ValueSize, const IMG_CHAR *pszValue);
+
+PVRSRV_ERROR DIListAllEntriesKM(DI_CONTEXT *psContext);
+
+
+#endif /* PVR_IMPL_BRG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg_intern.h b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg_intern.h
new file mode 100644
index 000000000000..5e11cac59854
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg_intern.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info internal interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_IMPL_BRG_INTERN_H
+#define PVR_IMPL_BRG_INTERN_H
+
+typedef struct DIIB_GROUP DIIB_GROUP;
+typedef struct DIIB_ENTRY DIIB_ENTRY;
+
+/*! @Function DIImplBrgFind
+ *
+ * @Description
+ * Retrieves an entry based on a given path.
+ *
+ * @Input pszPath: Full entry path in form of
+ *                 /rootGroup/.../parentGroup/entryName.
+ *
+ * @Return Returns entry object if exists or NULL otherwise.
+ */
+DIIB_ENTRY *DIImplBrgFind(const IMG_CHAR *pszPath);
+
+#endif /* PVR_IMPL_BRG_INTERN_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_server.c
new file mode 100644
index 000000000000..47046d7ec3a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_server.c
@@ -0,0 +1,800 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Info framework functions and types.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "di_server.h"
+#include "osdi_impl.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "lock.h"
+#include "allocmem.h"
+#include "osfunc.h"
+
+#define ROOT_GROUP_NAME PVR_DRM_NAME
+
+/*! Implementation object. */
+typedef struct DI_IMPL
+{
+	const IMG_CHAR *pszName;       /*<! name of the implementation */
+	OSDI_IMPL_CB sCb;              /*<! implementation callbacks */
+	IMG_BOOL bInitialised;         /*<! set to IMG_TRUE after implementation
+	                                    is initialised */
+
+	DLLIST_NODE sListNode;         /*<! node element of the global list of all
+	                                    implementations */
+} DI_IMPL;
+
+/*! Wrapper object for objects originating from derivative implementations.
+ * This wraps both entries and groups native implementation objects. */
+typedef struct DI_NATIVE_HANDLE
+{
+	void *pvHandle;                /*!< opaque handle to the native object */
+	DI_IMPL *psDiImpl;             /*!< implementation pvHandle is associated
+	                                    with */
+	DLLIST_NODE sListNode;         /*!< node element of native handles list */
+} DI_NATIVE_HANDLE;
+
+/*! Debug Info entry object.
+ *
+ * Depending on the implementation this can be represented differently. For
+ * example for the DebugFS this translates to a file.
+ */
+struct DI_ENTRY
+{
+	const IMG_CHAR *pszName;       /*!< name of the entry */
+	void *pvPrivData;              /*! handle to entry's private data */
+	DI_ENTRY_TYPE eType;           /*! entry type */
+	DI_ITERATOR_CB sIterCb;        /*!< iterator interface for the entry */
+
+	DLLIST_NODE sListNode;         /*!< node element of group's entry list */
+	DLLIST_NODE sNativeHandleList; /*!< list of native handles belonging to this
+	                                    entry */
+};
+
+/*! Debug Info group object.
+ *
+ * Depending on the implementation this can be represented differently. For
+ * example for the DebugFS this translates to a directory.
+ */
+struct DI_GROUP
+{
+	IMG_CHAR *pszName;               /*!< name of the group */
+	const struct DI_GROUP *psParent; /*!< parent groups */
+
+	DLLIST_NODE sListNode;           /*!< node element of group's group list */
+	DLLIST_NODE sGroupList;          /*!< list of groups (children) that belong
+	                                      to this group */
+	DLLIST_NODE sEntryList;          /*!< list of entries (children) that belong
+	                                      to this group */
+	DLLIST_NODE sNativeHandleList;   /*!< list of native handles belonging to
+	                                      this group */
+};
+
+/* List of all registered implementations. */
+static DECLARE_DLLIST(_g_sImpls);
+
+/* Root group for the DI entries and groups. This group is used as a root
+ * group for all other groups and entries if during creation a parent groups
+ * is not given. */
+static DI_GROUP *_g_psRootGroup;
+
+/* Protects access to _g_sImpls and _g_psRootGroup */
+static POS_LOCK _g_hLock;
+
+PVRSRV_ERROR DIInit(void)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__) && defined(__KERNEL__)
+	eError = OSLockCreateNoStats(&_g_hLock);
+#else
+	eError = OSLockCreate(&_g_hLock);
+#endif
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", return_);
+
+	_g_psRootGroup = OSAllocMemNoStats(sizeof(*_g_psRootGroup));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psRootGroup, eError, destroy_lock_);
+
+	_g_psRootGroup->pszName = OSAllocMemNoStats(sizeof(ROOT_GROUP_NAME));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psRootGroup->pszName, eError, cleanup_name_);
+	OSStringSafeCopy(_g_psRootGroup->pszName, ROOT_GROUP_NAME,
+				  sizeof(ROOT_GROUP_NAME));
+
+	dllist_init(&_g_psRootGroup->sListNode);
+	dllist_init(&_g_psRootGroup->sGroupList);
+	dllist_init(&_g_psRootGroup->sEntryList);
+	dllist_init(&_g_psRootGroup->sNativeHandleList);
+
+	return PVRSRV_OK;
+
+cleanup_name_:
+	OSFreeMemNoStats(_g_psRootGroup);
+destroy_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(_g_hLock);
+#else
+	OSLockDestroy(_g_hLock);
+#endif
+return_:
+	return eError;
+}
+
+/* Destroys the whole tree of group and entries for a given group as a root. */
+static void _DeInitGroupRecursively(DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+		DIDestroyEntry(psThisEntry);
+	}
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		_DeInitGroupRecursively(psThisGroup);
+	}
+
+	DIDestroyGroup(psGroup);
+}
+
+void DIDeInit(void)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	OSLockAcquire(_g_hLock);
+
+	if (!dllist_is_empty(&_g_psRootGroup->sGroupList) ||
+	    !dllist_is_empty(&_g_psRootGroup->sEntryList))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: entries or groups still exist during "
+		        "de-initialisation process, destroying all", __func__));
+	}
+
+	_DeInitGroupRecursively(_g_psRootGroup);
+	_g_psRootGroup = NULL;
+
+	/* Remove all of the implementations. */
+	dllist_foreach_node(&_g_sImpls, psThis, psNext)
+	{
+		DI_IMPL *psDiImpl = IMG_CONTAINER_OF(psThis, DI_IMPL, sListNode);
+
+		if (psDiImpl->bInitialised)
+		{
+			psDiImpl->sCb.pfnDeInit();
+			psDiImpl->bInitialised = IMG_FALSE;
+		}
+
+		dllist_remove_node(&psDiImpl->sListNode);
+		OSFreeMem(psDiImpl);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	/* all resources freed so free the lock itself too */
+
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(_g_hLock);
+#else
+	OSLockDestroy(_g_hLock);
+#endif
+}
+
+static IMG_BOOL _ValidateIteratorCb(const DI_ITERATOR_CB *psIterCb,
+                                    DI_ENTRY_TYPE eType)
+{
+	IMG_UINT32 uiFlags = 0;
+
+	if (psIterCb == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	if (eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		uiFlags |= psIterCb->pfnShow != NULL ? BIT(0) : 0;
+		uiFlags |= psIterCb->pfnStart != NULL ? BIT(1) : 0;
+		uiFlags |= psIterCb->pfnStop != NULL ? BIT(2) : 0;
+		uiFlags |= psIterCb->pfnNext != NULL ? BIT(3) : 0;
+
+		/* either only pfnShow or all callbacks need to be set */
+		if (uiFlags != BIT(0) && !BITMASK_HAS(uiFlags, 0x0f))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else if (eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		uiFlags |= psIterCb->pfnRead != NULL ? BIT(0) : 0;
+		uiFlags |= psIterCb->pfnSeek != NULL ? BIT(1) : 0;
+
+		/* either only pfnRead or all callbacks need to be set */
+		if (uiFlags != BIT(0) && !BITMASK_HAS(uiFlags, 0x03))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static PVRSRV_ERROR _CreateNativeEntry(DI_ENTRY *psEntry,
+                                       const DI_NATIVE_HANDLE *psNativeParent)
+{
+	PVRSRV_ERROR eError;
+	DI_IMPL *psImpl = psNativeParent->psDiImpl;
+
+	DI_NATIVE_HANDLE *psNativeEntry = OSAllocMem(sizeof(*psNativeEntry));
+	PVR_LOG_GOTO_IF_NOMEM(psNativeEntry, eError, return_);
+
+	eError = psImpl->sCb.pfnCreateEntry(psEntry->pszName,
+	                                    psEntry->eType,
+	                                    &psEntry->sIterCb,
+	                                    psEntry->pvPrivData,
+	                                    psNativeParent->pvHandle,
+	                                    &psNativeEntry->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateEntry", free_memory_);
+
+	psNativeEntry->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psEntry->sNativeHandleList, &psNativeEntry->sListNode);
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeEntry);
+return_:
+	return eError;
+}
+
+static void _DestroyNativeEntry(DI_NATIVE_HANDLE *psNativeEntry)
+{
+	dllist_remove_node(&psNativeEntry->sListNode);
+	OSFreeMem(psNativeEntry);
+}
+
+PVRSRV_ERROR DICreateEntry(const IMG_CHAR *pszName,
+                           DI_GROUP *psGroup,
+                           const DI_ITERATOR_CB *psIterCb,
+                           void *pvPriv,
+                           DI_ENTRY_TYPE eType,
+                           DI_ENTRY **ppsEntry)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_ENTRY *psEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(_ValidateIteratorCb(psIterCb, eType),
+	                                "psIterCb");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsEntry != NULL, "psEntry");
+
+	psEntry = OSAllocMem(sizeof(*psEntry));
+	PVR_LOG_RETURN_IF_NOMEM(psEntry, "OSAllocMem");
+
+	if (psGroup == NULL)
+	{
+		psGroup = _g_psRootGroup;
+	}
+
+	psEntry->pszName = pszName;
+	psEntry->pvPrivData = pvPriv;
+	psEntry->eType = eType;
+	psEntry->sIterCb = *psIterCb;
+	dllist_init(&psEntry->sNativeHandleList);
+
+	OSLockAcquire(_g_hLock);
+
+	dllist_add_to_tail(&psGroup->sEntryList, &psEntry->sListNode);
+
+	/* Iterate over all of the native handles of parent group to create
+	 * the entry for every registered implementation. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		eError = _CreateNativeEntry(psEntry, psNativeGroup);
+		PVR_GOTO_IF_ERROR(eError, cleanup_);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	*ppsEntry = psEntry;
+
+	return PVRSRV_OK;
+
+cleanup_:
+	OSLockRelease(_g_hLock);
+
+	/* Something went wrong so if there were any native entries created remove
+	 * them from the list, free them and free the DI entry itself. */
+	dllist_foreach_node(&psEntry->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeEntry =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		_DestroyNativeEntry(psNativeEntry);
+	}
+
+	OSFreeMem(psEntry);
+
+	return eError;
+}
+
+void DIDestroyEntry(DI_ENTRY *psEntry)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psEntry != NULL,
+	                             "psEntry invalid in DIDestroyEntry()");
+
+	/* Iterate through all of the native entries of the DI entry, remove
+	 * them from the list and then destroy them. After that, destroy the
+	 * DI entry itself. */
+	dllist_foreach_node(&psEntry->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNative = IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE,
+		                                              sListNode);
+
+		/* The implementation must ensure that entry is not removed if any
+		 * operations are being executed on the entry. If this is the case
+		 * the implementation should block until all of them are finished
+		 * and prevent any further operations.
+		 * This will guarantee proper synchronisation between the DI framework
+		 * and underlying implementations and prevent destruction/access
+		 * races. */
+		psNative->psDiImpl->sCb.pfnDestroyEntry(psNative->pvHandle);
+		dllist_remove_node(&psNative->sListNode);
+		OSFreeMem(psNative);
+	}
+
+	dllist_remove_node(&psEntry->sListNode);
+
+	OSFreeMem(psEntry);
+}
+
+static PVRSRV_ERROR _CreateNativeGroup(DI_GROUP *psGroup,
+                                       const DI_NATIVE_HANDLE *psNativeParent,
+                                       DI_NATIVE_HANDLE **ppsNativeGroup)
+{
+	PVRSRV_ERROR eError;
+	DI_IMPL *psImpl = psNativeParent->psDiImpl;
+
+	DI_NATIVE_HANDLE *psNativeGroup = OSAllocMem(sizeof(*psNativeGroup));
+	PVR_LOG_GOTO_IF_NOMEM(psNativeGroup, eError, return_);
+
+	eError = psImpl->sCb.pfnCreateGroup(psGroup->pszName,
+	                                    psNativeParent->pvHandle,
+	                                    &psNativeGroup->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateGroup", free_memory_);
+
+	psNativeGroup->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psGroup->sNativeHandleList, &psNativeGroup->sListNode);
+
+	*ppsNativeGroup = psNativeGroup;
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeGroup);
+return_:
+	return eError;
+}
+
+static void _DestroyNativeGroup(DI_NATIVE_HANDLE *psNativeEntry)
+{
+	dllist_remove_node(&psNativeEntry->sListNode);
+	OSFreeMem(psNativeEntry);
+}
+
+PVRSRV_ERROR DICreateGroup(const IMG_CHAR *pszName,
+                           DI_GROUP *psParent,
+                           DI_GROUP **ppsGroup)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_GROUP *psGroup;
+	size_t uSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsGroup != NULL, "ppsDiGroup");
+
+	psGroup = OSAllocMem(sizeof(*psGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psGroup, "OSAllocMem");
+
+	if (psParent == NULL)
+	{
+		psParent = _g_psRootGroup;
+	}
+
+	uSize = OSStringLength(pszName) + 1;
+	psGroup->pszName = OSAllocMem(uSize * sizeof(*psGroup->pszName));
+	PVR_LOG_GOTO_IF_NOMEM(psGroup->pszName, eError, cleanup_name_);
+	OSStringSafeCopy(psGroup->pszName, pszName, uSize);
+
+	psGroup->psParent = psParent;
+	dllist_init(&psGroup->sGroupList);
+	dllist_init(&psGroup->sEntryList);
+	dllist_init(&psGroup->sNativeHandleList);
+
+	OSLockAcquire(_g_hLock);
+
+	dllist_add_to_tail(&psParent->sGroupList, &psGroup->sListNode);
+
+	/* Iterate over all of the native handles of parent group to create
+	 * the group for every registered implementation. */
+	dllist_foreach_node(&psParent->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup = NULL, *psNativeParent =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		eError = _CreateNativeGroup(psGroup, psNativeParent, &psNativeGroup);
+		PVR_GOTO_IF_ERROR(eError, cleanup_);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	*ppsGroup = psGroup;
+
+	return PVRSRV_OK;
+
+cleanup_:
+	OSLockRelease(_g_hLock);
+
+	/* Something went wrong so if there were any native groups created remove
+	 * them from the list, free them and free the DI group itself. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		dllist_remove_node(&psNativeGroup->sListNode);
+		OSFreeMem(psNativeGroup);
+	}
+
+	OSFreeMem(psGroup->pszName);
+cleanup_name_:
+	OSFreeMem(psGroup);
+
+	return eError;
+}
+
+void DIDestroyGroup(DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psGroup != NULL,
+	                             "psGroup invalid in DIDestroyGroup()");
+
+	/* Iterate through all of the native groups of the DI group, remove
+	 * them from the list and then destroy them. After that destroy the
+	 * DI group itself. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNative = IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE,
+		                                              sListNode);
+
+		psNative->psDiImpl->sCb.pfnDestroyGroup(psNative->pvHandle);
+		dllist_remove_node(&psNative->sListNode);
+		OSFreeMem(psNative);
+	}
+
+	dllist_remove_node(&psGroup->sListNode);
+
+	if (psGroup == _g_psRootGroup)
+	{
+		OSFreeMemNoStats(psGroup->pszName);
+		OSFreeMemNoStats(psGroup);
+	}
+	else
+	{
+		OSFreeMem(psGroup->pszName);
+		OSFreeMem(psGroup);
+	}
+}
+
+void *DIGetPrivData(const OSDI_IMPL_ENTRY *psEntry)
+{
+	PVR_ASSERT(psEntry != NULL);
+
+	return psEntry->pvPrivData;
+}
+
+void DIWrite(const OSDI_IMPL_ENTRY *psEntry, const void *pvData,
+             IMG_UINT32 uiSize)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnWrite != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnWrite(psEntry->pvNative, pvData, uiSize);
+}
+
+void DIPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt, ...)
+{
+	va_list args;
+
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnVPrintf != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	va_start(args, pszFmt);
+	psEntry->psCb->pfnVPrintf(psEntry->pvNative, pszFmt, args);
+	va_end(args);
+}
+
+void DIVPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt,
+               va_list pArgs)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnVPrintf != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnVPrintf(psEntry->pvNative, pszFmt, pArgs);
+}
+
+void DIPuts(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszStr)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnPuts != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnPuts(psEntry->pvNative, pszStr);
+}
+
+IMG_BOOL DIHasOverflowed(const OSDI_IMPL_ENTRY *psEntry)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnHasOverflowed != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	return psEntry->psCb->pfnHasOverflowed(psEntry->pvNative);
+}
+
+/* ---- OS implementation API ---------------------------------------------- */
+
+static IMG_BOOL _ValidateImplCb(const OSDI_IMPL_CB *psImplCb)
+{
+	PVR_GOTO_IF_FALSE(psImplCb->pfnInit != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDeInit != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnCreateGroup != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDestroyGroup != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnCreateEntry != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDestroyEntry != NULL, failed_);
+
+	return IMG_TRUE;
+
+failed_:
+	return IMG_FALSE;
+}
+
+/* Walks the tree of groups and entries and create all of the native handles
+ * for the given implementation for all of the already existing groups and
+ * entries. */
+static PVRSRV_ERROR _InitNativeHandlesRecursively(DI_IMPL *psImpl,
+                                               DI_GROUP *psGroup,
+                                               DI_NATIVE_HANDLE *psNativeParent)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_NATIVE_HANDLE *psNativeGroup;
+
+	psNativeGroup = OSAllocMem(sizeof(*psNativeGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psNativeGroup, "OSAllocMem");
+
+	eError = psImpl->sCb.pfnCreateGroup(psGroup->pszName,
+	                           psNativeParent ? psNativeParent->pvHandle : NULL,
+	                           &psNativeGroup->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateGroup", free_memory_);
+
+	psNativeGroup->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psGroup->sNativeHandleList,
+	                   &psNativeGroup->sListNode);
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		// and then walk the new group
+		eError = _InitNativeHandlesRecursively(psImpl, psThisGroup,
+		                                       psNativeGroup);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_InitNativeHandlesRecursively");
+	}
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+
+		eError = _CreateNativeEntry(psThisEntry, psNativeGroup);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_CreateNativeEntry");
+	}
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeGroup);
+
+	return eError;
+}
+
+/* Walks the tree of groups and entries and destroys all of the native handles
+ * for the given implementation. */
+static void _DeInitNativeHandlesRecursively(DI_IMPL *psImpl, DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+
+		// free all of the native entries that belong to this implementation
+		dllist_foreach_node(&psThisEntry->sNativeHandleList, psThis, psNext)
+		{
+			DI_NATIVE_HANDLE *psNativeEntry =
+			        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+			if (psNativeEntry->psDiImpl == psImpl)
+			{
+				_DestroyNativeEntry(psNativeEntry);
+				// there can be only one entry on the list for a given
+				// implementation
+				break;
+			}
+		}
+	}
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		// and then walk the new group
+		_DeInitNativeHandlesRecursively(psImpl, psThisGroup);
+	}
+
+	// free all of the native entries that belong to this implementation
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		if (psNativeGroup->psDiImpl == psImpl)
+		{
+			_DestroyNativeGroup(psNativeGroup);
+			// there can be only one entry on the list for a given
+			// implementation
+			break;
+		}
+	}
+}
+
+static PVRSRV_ERROR _InitImpl(DI_IMPL *psImpl)
+{
+	PVRSRV_ERROR eError;
+	// DI_NATIVE_HANDLE *psNativeGroup;
+
+	eError = psImpl->sCb.pfnInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->pfnInit()", return_);
+
+	/* if the implementation is being created after any groups or entries
+	 * have been created we need to walk the current tree and create
+	 * native groups and entries for all of the existing ones */
+	eError = _InitNativeHandlesRecursively(psImpl, _g_psRootGroup, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_InitNativeHandlesRecursively",
+	                      free_native_handles_and_deinit_);
+
+	psImpl->bInitialised = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+free_native_handles_and_deinit_:
+	/* something went wrong so we need to walk the tree and remove all of the
+	 * native entries and groups that we've created before we can destroy
+	 * the implementation */
+	_DeInitNativeHandlesRecursively(psImpl, _g_psRootGroup);
+	psImpl->sCb.pfnDeInit();
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIRegisterImplementation(const IMG_CHAR *pszName,
+                                      const OSDI_IMPL_CB *psImplCb)
+{
+	DI_IMPL *psImpl;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(_ValidateImplCb(psImplCb), "psImplCb");
+	/* if root group does not exist it can mean 2 things:
+	 * - DIInit() was not called so initialisation order is incorrect and needs
+	 *   to be fixed
+	 * - DIInit() failed but if that happens we should never make it here */
+	PVR_ASSERT(_g_psRootGroup != NULL);
+
+	psImpl = OSAllocMem(sizeof(*psImpl));
+	PVR_LOG_RETURN_IF_NOMEM(psImpl, "OSAllocMem");
+
+	psImpl->pszName = pszName;
+	psImpl->sCb = *psImplCb;
+
+	OSLockAcquire(_g_hLock);
+
+	eError = _InitImpl(psImpl);
+	if (eError != PVRSRV_OK)
+	{
+		/* implementation could not be initialised so remove it from the
+		 * list, free the memory and forget about it */
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: could not initialise \"%s\" debug "
+		        "info implementation, discarding", __func__,
+		        psImpl->pszName));
+
+		goto free_impl_;
+	}
+
+	psImpl->bInitialised = IMG_TRUE;
+
+	dllist_add_to_tail(&_g_sImpls, &psImpl->sListNode);
+
+	OSLockRelease(_g_hLock);
+
+	return PVRSRV_OK;
+
+free_impl_:
+	OSLockRelease(_g_hLock);
+
+	OSFreeMem(psImpl);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/dma_km.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/dma_km.c
new file mode 100644
index 000000000000..6e5ae4915847
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/dma_km.c
@@ -0,0 +1,564 @@
+/*************************************************************************/ /*!
+@File           dma_km.c
+@Title          kernel side of dma transfer scheduling
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements server side code for allowing DMA transfers between
+                cpu and device memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <asm/current.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#endif
+
+#include "pmr.h"
+#include "log2.h"
+#include "device.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "dma_km.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "allocmem.h"
+#include "process_stats.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+#include "ri_server.h"
+#endif
+#include "devicemem.h"
+#include "pvrsrv_apphint.h"
+#include "pvrsrv_sync_server.h"
+#include "km_apphint_defs.h"
+#include "di_server.h"
+#include "dma_flags.h"
+
+/* This header must always be included last */
+#if defined(__linux__)
+#include "kernel_compatibility.h"
+#endif
+
+typedef struct _SERVER_CLEANUP_DATA_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	CONNECTION_DATA *psConnection;
+	IMG_UINT32 uiNumDMA;
+	IMG_UINT32 uiCount;
+	SYNC_TIMELINE_OBJ sTimelineObject;
+	void* pvChan;
+	PMR** ppsPMR;
+} SERVER_CLEANUP_DATA;
+
+#if !defined(NO_HARDWARE)
+static void Cleanup(void* pvCleanupData, IMG_BOOL bAdvanceTimeline)
+{
+	IMG_UINT i;
+	PVRSRV_ERROR eError;
+	SERVER_CLEANUP_DATA* psCleanupData = (SERVER_CLEANUP_DATA*) pvCleanupData;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Server Cleanup thread entry (%p)", pvCleanupData));
+#endif
+
+	for (i=0; i<psCleanupData->uiCount; i++)
+	{
+		eError = PMRUnlockSysPhysAddresses(psCleanupData->ppsPMR[i]);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+	}
+
+	/* Advance timeline */
+	if (psCleanupData->sTimelineObject.pvTlObj && bAdvanceTimeline)
+	{
+		eError = SyncSWTimelineAdvanceKM(psCleanupData->psDevNode, &psCleanupData->sTimelineObject);
+		PVR_LOG_IF_ERROR(eError, "SyncSWTimelineAdvanceKM");
+		eError = SyncSWTimelineReleaseKM(&psCleanupData->sTimelineObject);
+		PVR_LOG_IF_ERROR(eError, "SyncSWTimelineReleaseKM");
+	}
+
+	OSAtomicDecrement(&psCleanupData->psConnection->ui32NumDmaTransfersInFlight);
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Decremented to %d", OSAtomicRead(&psCleanupData->psConnection->ui32NumDmaTransfersInFlight)));
+#endif
+	eError = OSEventObjectSignal(psCleanupData->psConnection->hDmaEventObject);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSEventObjectSignal failed: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+	}
+
+
+	OSFreeMem(psCleanupData->ppsPMR);
+	OSFreeMem(psCleanupData);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode, CONNECTION_DATA *psConnectionData)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+	if (!psDevConfig->bHasDma)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pfnSlaveDMAGetChan, "pfnSlaveDMAGetChan");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pfnSlaveDMAFreeChan, "pfnSlaveDMAFreeChan");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pszDmaTxChanName, "pszDmaTxChanName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pszDmaRxChanName, "pszDmaRxChanName");
+
+	eError = OSEventObjectCreate("Dma transfer cleanup event object",
+								 &psConnectionData->hDmaEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", dma_init_error1);
+
+	OSLockAcquire(psDeviceNode->hConnectionsLock);
+
+	if (psDeviceNode->ui32RefCountDMA == 0)
+	{
+		psDeviceNode->hDmaTxChan =
+			psDevConfig->pfnSlaveDMAGetChan(psDevConfig,
+											psDevConfig->pszDmaTxChanName);
+		if (!psDeviceNode->hDmaTxChan)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_RESOURCE_UNAVAILABLE, dma_init_error2);
+		}
+		psDeviceNode->hDmaRxChan =
+			psDevConfig->pfnSlaveDMAGetChan(psDevConfig,
+											psDevConfig->pszDmaRxChanName);
+		if (!psDeviceNode->hDmaRxChan)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_RESOURCE_UNAVAILABLE, dma_init_error3);
+		}
+
+		eError = OSLockCreate(&psDeviceNode->hDmaTxLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(hDmaTxLock)", dma_init_error_txlock);
+
+		eError = OSLockCreate(&psDeviceNode->hDmaRxLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(hDmaRxLock)", dma_init_error_rxlock);
+	}
+
+	psDeviceNode->ui32RefCountDMA++;
+
+	OSLockRelease(psDeviceNode->hConnectionsLock);
+
+	OSAtomicWrite(&psConnectionData->ui32NumDmaTransfersInFlight, 0);
+	psConnectionData->bAcceptDmaRequests = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+dma_init_error_rxlock:
+	OSLockDestroy(psDeviceNode->hDmaTxLock);
+dma_init_error_txlock:
+	psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaRxChan);
+dma_init_error3:
+	psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaTxChan);
+dma_init_error2:
+	OSLockRelease(psDeviceNode->hConnectionsLock);
+	OSEventObjectDestroy(psConnectionData->hDmaEventObject);
+dma_init_error1:
+	return eError;
+}
+
+static void WaitForOutstandingDma(CONNECTION_DATA *psConnectionData)
+{
+
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	IMG_UINT32 ui32Tries = 100;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR,
+					"Waiting on %d DMA transfers in flight...", OSAtomicRead(&psConnectionData->ui32NumDmaTransfersInFlight)));
+#endif
+
+	eError = OSEventObjectOpen(psConnectionData->hDmaEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return;
+	}
+
+	while (OSAtomicRead(&psConnectionData->ui32NumDmaTransfersInFlight) != 0)
+	{
+		/*
+		#define DMA_TRANSFER_TIMEOUT_US (5000000ULL)
+
+		This currently doesn't work properly. Wait time is not as requested.
+		Using OSSleepms instead
+
+		OSEventObjectWaitKernel(hEvent, DMA_TRANSFER_TIMEOUT_US);
+		*/
+		OSSleepms(50);
+		if (!ui32Tries)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Timeout while waiting on outstanding DMA transfers!", __func__));
+			break;
+		}
+
+		ui32Tries--;
+	}
+
+	OSEventObjectClose(hEvent);
+}
+
+IMG_EXPORT void
+PVRSRVDeInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode, CONNECTION_DATA *psConnectionData)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+
+	if (!psDevConfig->bHasDma)
+	{
+		return;
+	}
+
+	OSLockAcquire(psDeviceNode->hDmaTxLock);
+	OSLockAcquire(psDeviceNode->hDmaRxLock);
+
+	psConnectionData->bAcceptDmaRequests = IMG_FALSE;
+
+	OSLockRelease(psDeviceNode->hDmaRxLock);
+	OSLockRelease(psDeviceNode->hDmaTxLock);
+
+	WaitForOutstandingDma(psConnectionData);
+
+	OSLockAcquire(psDeviceNode->hConnectionsLock);
+
+	if (psDeviceNode->ui32RefCountDMA == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid ref count (%d)", __func__, psDeviceNode->ui32RefCountDMA));
+		OSLockRelease(psDeviceNode->hConnectionsLock);
+		return;
+	}
+
+	if (--psDeviceNode->ui32RefCountDMA == 0)
+	{
+		if (psDeviceNode->hDmaRxLock != NULL)
+		{
+			OSLockDestroy(psDeviceNode->hDmaRxLock);
+		}
+
+		if (psDeviceNode->hDmaTxLock != NULL)
+		{
+			OSLockDestroy(psDeviceNode->hDmaTxLock);
+		}
+
+		psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaRxChan);
+		psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaTxChan);
+	}
+
+	OSLockRelease(psDeviceNode->hConnectionsLock);
+
+	OSEventObjectDestroy(psConnectionData->hDmaEventObject);
+}
+
+IMG_EXPORT PVRSRV_ERROR
+DmaDeviceParams(CONNECTION_DATA *psConnection,
+				PVRSRV_DEVICE_NODE *psDevNode,
+				IMG_UINT32 *ui32DmaBuffAlign,
+				IMG_UINT32 *ui32DmaTransferMult)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+	*ui32DmaBuffAlign = psDevConfig->ui32DmaAlignment;
+	*ui32DmaTransferMult = psDevConfig->ui32DmaTransferUnit;
+
+	return PVRSRV_OK;
+}
+
+IMG_EXPORT PVRSRV_ERROR
+DmaSparseMappingTable(PMR *psPMR,
+					  IMG_DEVMEM_OFFSET_T uiOffset,
+					  IMG_UINT32 ui32SizeInPages,
+					  IMG_BOOL *pbTable)
+{
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_DEV_PHYADDR *psDevPhyAddr;
+		IMG_BOOL *pbValid;
+
+		psDevPhyAddr = OSAllocZMem(ui32SizeInPages * sizeof(IMG_CPU_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPhyAddr, eError, err1);
+
+		pbValid = OSAllocZMem(ui32SizeInPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, err2);
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", err3);
+
+		eError = PMR_DevPhysAddr(psPMR,
+								 OSGetPageShift(),
+								 ui32SizeInPages,
+								 uiOffset,
+								 psDevPhyAddr,
+								 pbValid,
+								 CPU_USE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", err3);
+
+		PMRUnlockSysPhysAddresses(psPMR);
+
+		memcpy(pbTable, pbValid, ui32SizeInPages * sizeof(IMG_BOOL));
+
+err3:
+		OSFreeMem(pbValid);
+err2:
+		OSFreeMem(psDevPhyAddr);
+err1:
+		return eError;
+}
+
+IMG_EXPORT PVRSRV_ERROR
+DmaTransfer(CONNECTION_DATA *psConnection,
+		    PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 uiNumDMAs,
+			PMR** ppsPMR,
+			IMG_UINT64 *puiAddress,
+			IMG_DEVMEM_OFFSET_T *puiOffset,
+			IMG_DEVMEM_SIZE_T *puiSize,
+			IMG_UINT32 uiFlags,
+			PVRSRV_TIMELINE iUpdateFenceTimeline)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(NO_HARDWARE)
+	/* On nohw the kernel call just advances the timeline to signal completion */
+
+	SYNC_TIMELINE_OBJ sSwTimeline = {NULL, PVRSRV_NO_TIMELINE};
+
+	if (iUpdateFenceTimeline != PVRSRV_NO_TIMELINE)
+	{
+		eError = SyncSWGetTimelineObj(iUpdateFenceTimeline, &sSwTimeline);
+		PVR_LOG_RETURN_IF_ERROR(eError, "SyncSWGetTimelineObj");
+
+		eError = SyncSWTimelineAdvanceKM(psDevNode, &sSwTimeline);
+		PVR_LOG_RETURN_IF_ERROR(eError, "SyncSWTimelineAdvanceKM");
+
+		eError = SyncSWTimelineReleaseKM(&sSwTimeline);
+		PVR_LOG_RETURN_IF_ERROR(eError, "SyncSWTimelineReleaseKM");
+	}
+
+	return PVRSRV_OK;
+
+#else
+	IMG_DEV_PHYADDR *psDevPhyAddr;
+	IMG_DMA_ADDR *psDmaAddr;
+	IMG_BOOL *pbValid;
+	IMG_UINT32 i;
+	PVRSRV_DEVICE_CONFIG* psDevConfig = psDevNode->psDevConfig;
+	void* pvChan = NULL;
+	SERVER_CLEANUP_DATA* psServerData;
+	void*  pvOSData;
+	POS_LOCK hChanLock = uiFlags & (DMA_FLAG_MEM_TO_DEV) ? psDevNode->hDmaTxLock : psDevNode->hDmaRxLock;
+
+	for (i=0; i<uiNumDMAs; i++)
+	{
+		PMR* psPMR = ppsPMR[i];
+		PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+		IMG_BOOL bIsPrivate = PhysHeapGetFlags(PMR_PhysHeap(psPMR)) & PHYS_HEAP_USAGE_GPU_PRIVATE;
+		IMG_BOOL bWrite = uiFlags & DMA_FLAG_MEM_TO_DEV;
+
+		if (PMR_GetType(psPMR) == PMR_TYPE_OSMEM)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("PMR is type OSMEM, expected device memory.", eError, PVRSRV_ERROR_NOT_SUPPORTED, error_return);
+		}
+
+		if (!bIsPrivate)
+		{
+			if (( bWrite && !PVRSRV_CHECK_CPU_WRITEABLE(uiPMRFlags)) ||
+			    (!bWrite && !PVRSRV_CHECK_CPU_READABLE(uiPMRFlags)))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Incorrect CPU access parameters on PMR", eError, PVRSRV_ERROR_INVALID_PARAMS, error_return);
+			}
+		}
+		else
+		{
+			/* Treat GPU private PMRs differently since they can't be mapped by the client.
+			   Check access permissions based on GPU flags. */
+			if (( bWrite && !PVRSRV_CHECK_GPU_WRITEABLE(uiPMRFlags)) ||
+			    (!bWrite && !PVRSRV_CHECK_GPU_READABLE(uiPMRFlags))  ||
+			     uiPMRFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Incorrect GPU access parameters on PMR", eError, PVRSRV_ERROR_INVALID_PARAMS, error_return);
+			}
+		}
+	}
+
+	OSLockAcquire(hChanLock);
+
+	if (!psConnection->bAcceptDmaRequests)
+	{
+		OSLockRelease(hChanLock);
+		return PVRSRV_OK;
+	}
+
+	OSAtomicIncrement(&psConnection->ui32NumDmaTransfersInFlight);
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Incremented to %d", OSAtomicRead(&psConnection->ui32NumDmaTransfersInFlight)));
+#endif
+	psServerData = OSAllocZMem(sizeof(SERVER_CLEANUP_DATA));
+	PVR_LOG_GOTO_IF_NOMEM(psServerData, eError, e0);
+
+	pvChan = uiFlags & (DMA_FLAG_MEM_TO_DEV) ? psDevNode->hDmaTxChan : psDevNode->hDmaRxChan;
+	if (!pvChan)
+	{
+		eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		PVR_LOG_GOTO_IF_ERROR(eError, "Error acquiring DMA channel", e1);
+	}
+
+	if (iUpdateFenceTimeline != PVRSRV_NO_TIMELINE)
+	{
+		eError = SyncSWGetTimelineObj(iUpdateFenceTimeline, &psServerData->sTimelineObject);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncSWGetTimelineObj", e1);
+	}
+
+	psServerData->uiCount = 0;
+	psServerData->psDevNode = psDevNode;
+	psServerData->psConnection = psConnection;
+	psServerData->pvChan = pvChan;
+	psServerData->ppsPMR = OSAllocZMem(sizeof(PMR*) * uiNumDMAs);
+	PVR_LOG_GOTO_IF_NOMEM(psServerData->ppsPMR, eError, e2);
+
+	eError = OSDmaAllocData(psDevNode, uiNumDMAs, &pvOSData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSDmaAllocData failed", e3);
+
+	for (i=0; i<uiNumDMAs; i++)
+	{
+		IMG_UINT32 ui32SizeInPages;
+		IMG_UINT32 uiOffsetInPage = puiOffset[i] & (OSGetPageSize() - 1);
+		ui32SizeInPages = (puiSize[i] + uiOffsetInPage + OSGetPageSize() - 1) >> OSGetPageShift();
+
+		psDmaAddr = OSAllocZMem(ui32SizeInPages * sizeof(IMG_DMA_ADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDmaAddr, eError, loop_e0);
+
+		psDevPhyAddr = OSAllocZMem(ui32SizeInPages * sizeof(IMG_CPU_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPhyAddr, eError, loop_e1);
+
+		pbValid = OSAllocZMem(ui32SizeInPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, loop_e2);
+
+		eError = PMRLockSysPhysAddresses(ppsPMR[i]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", loop_e3);
+
+		psServerData->ppsPMR[i] = ppsPMR[i];
+
+		eError = PMR_DevPhysAddr(ppsPMR[i],
+								 OSGetPageShift(),
+								 ui32SizeInPages,
+								 puiOffset[i],
+								 psDevPhyAddr,
+								 pbValid,
+								 CPU_USE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", loop_e4);
+
+		psDevConfig->pfnDevPhysAddr2DmaAddr(psDevConfig,
+											psDmaAddr,
+											psDevPhyAddr,
+											pbValid,
+											ui32SizeInPages,
+											PMR_IsSparse(ppsPMR[i]));
+
+		if (!PMR_IsSparse(ppsPMR[i]))
+		{
+			eError = OSDmaPrepareTransfer(psDevNode,
+										  pvChan,
+										  &psDmaAddr[0], (IMG_UINT64*)puiAddress[i],
+										  puiSize[i], (uiFlags & DMA_FLAG_MEM_TO_DEV), pvOSData,
+										  psServerData, Cleanup, (i == 0));
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSDmaPrepareTransfer", loop_e4);
+			psServerData->uiCount++;
+
+		}
+		else
+		{
+			eError = OSDmaPrepareTransferSparse(psDevNode, pvChan,
+												psDmaAddr, pbValid,
+												(IMG_UINT64*)puiAddress[i], puiSize[i],
+												uiOffsetInPage, ui32SizeInPages,
+												(uiFlags & DMA_FLAG_MEM_TO_DEV),
+												pvOSData, psServerData,
+												Cleanup, (i == 0));
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSDmaPrepareTransferSparse", loop_e4);
+			psServerData->uiCount++;
+		}
+
+		OSFreeMem(pbValid);
+		OSFreeMem(psDevPhyAddr);
+		OSFreeMem(psDmaAddr);
+
+		continue;
+
+loop_e4:
+		PMRUnlockSysPhysAddresses(ppsPMR[i]);
+loop_e3:
+		OSFreeMem(pbValid);
+loop_e2:
+		OSFreeMem(psDevPhyAddr);
+loop_e1:
+		OSFreeMem(psDmaAddr);
+loop_e0:
+		break;
+	}
+
+	if (psServerData->uiCount == uiNumDMAs)
+	{
+		OSDmaSubmitTransfer(psDevNode, pvOSData, pvChan, (uiFlags & DMA_FLAG_SYNCHRONOUS));
+	}
+	else
+	{
+		/* One of the transfers could not be programmed, roll back */
+		OSDmaForceCleanup(psDevNode, pvChan, pvOSData, psServerData, Cleanup);
+	}
+	OSLockRelease(hChanLock);
+	return eError;
+
+e3:
+	OSFreeMem(psServerData->ppsPMR);
+e2:
+	if (iUpdateFenceTimeline != PVRSRV_NO_TIMELINE)
+	{
+		SyncSWTimelineReleaseKM(&psServerData->sTimelineObject);
+	}
+e1:
+	OSFreeMem(psServerData);
+e0:
+	OSLockRelease(hChanLock);
+error_return:
+	return eError;
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/handle.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/handle.c
new file mode 100644
index 000000000000..7ebd38861963
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/handle.c
@@ -0,0 +1,2522 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Handle Manager
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide resource handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+/* See handle.h for a description of the handle API. */
+
+/*
+ * The implementation supports movable handle structures, allowing the address
+ * of a handle structure to change without having to fix up pointers in
+ * any of the handle structures. For example, the linked list mechanism
+ * used to link subhandles together uses handle array indices rather than
+ * pointers to the structures themselves.
+ */
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "handle.h"
+#include "handle_impl.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "connection_server.h"
+
+#define	HANDLE_HASH_TAB_INIT_SIZE 32
+#define HANDLE_PROC_HANDLE_HASH_INIT_SIZE 10
+
+#define	TEST_FLAG(v, f) BITMASK_HAS(v, f)
+#define	TEST_ALLOC_FLAG(psHandleData, f) BITMASK_HAS((psHandleData)->eFlag, f)
+
+
+/* Linked list structure. Used for both the list head and list items */
+typedef struct _HANDLE_LIST_
+{
+	IMG_HANDLE hPrev;
+	IMG_HANDLE hNext;
+	IMG_HANDLE hParent;
+} HANDLE_LIST;
+
+typedef struct _HANDLE_DATA_
+{
+	/* The handle that represents this structure */
+	IMG_HANDLE hHandle;
+
+	/* Handle type */
+	PVRSRV_HANDLE_TYPE eType;
+
+	/* Flags specified when the handle was allocated */
+	PVRSRV_HANDLE_ALLOC_FLAG eFlag;
+
+	/* Pointer to the data that the handle represents */
+	void *pvData;
+
+	/*
+	 * Callback specified at handle allocation time to
+	 * release/destroy/free the data represented by the
+	 * handle when it's reference count reaches 0. This
+	 * should always be NULL for subhandles.
+	 */
+	PFN_HANDLE_RELEASE pfnReleaseData;
+
+	/* List head for subhandles of this handle */
+	HANDLE_LIST sChildren;
+
+	/* List entry for sibling subhandles */
+	HANDLE_LIST sSiblings;
+
+	/* Reference count of lookups made. It helps track which resources are in
+	 * use in concurrent bridge calls. */
+	IMG_INT32 iLookupCount;
+	/* State of a handle. If the handle was already destroyed this is false.
+	 * If this is false and iLookupCount is 0 the pfnReleaseData callback is
+	 * called on the handle. */
+	IMG_BOOL bCanLookup;
+
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	/* Store the handle base used for this handle, so we
+	 * can later access the handle base lock (or check if
+	 * it has been already acquired)
+	 */
+	PVRSRV_HANDLE_BASE *psBase;
+#endif
+
+} HANDLE_DATA;
+
+struct _HANDLE_BASE_
+{
+	/* Pointer to a handle implementations base structure */
+	HANDLE_IMPL_BASE *psImplBase;
+
+	/*
+	 * Pointer to handle hash table.
+	 * The hash table is used to do reverse lookups, converting data
+	 * pointers to handles.
+	 */
+	HASH_TABLE *psHashTab;
+
+	/* Type specific (connection/global/process) Lock handle */
+	POS_LOCK hLock;
+
+	/* Can be connection, process, global */
+	PVRSRV_HANDLE_BASE_TYPE eType;
+};
+
+/*
+ * The key for the handle hash table is an array of three elements, the
+ * pointer to the resource, the resource type and the parent handle (or
+ * NULL if there is no parent). The eHandKey enumeration gives the
+ * array indices of the elements making up the key.
+ */
+enum eHandKey
+{
+	HAND_KEY_DATA = 0,
+	HAND_KEY_TYPE,
+	HAND_KEY_PARENT,
+	HAND_KEY_LEN		/* Must be last item in list */
+};
+
+/* HAND_KEY is the type of the hash table key */
+typedef uintptr_t HAND_KEY[HAND_KEY_LEN];
+
+typedef struct FREE_HANDLE_DATA_TAG
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	PVRSRV_HANDLE_TYPE eHandleFreeType;
+	/* timing data (ns) to release bridge lock upon the deadline */
+	IMG_UINT64 ui64TimeStart;
+	IMG_UINT64 ui64MaxBridgeTime;
+} FREE_HANDLE_DATA;
+
+typedef struct FREE_KERNEL_HANDLE_DATA_TAG
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	HANDLE_DATA *psProcessHandleData;
+	IMG_HANDLE hKernelHandle;
+} FREE_KERNEL_HANDLE_DATA;
+
+/* Stores a pointer to the function table of the handle back-end in use */
+static HANDLE_IMPL_FUNCTAB const *gpsHandleFuncs;
+
+static POS_LOCK gKernelHandleLock;
+static IMG_BOOL gbLockInitialised = IMG_FALSE;
+/* Pointer to process handle base currently being freed */
+static PVRSRV_HANDLE_BASE *g_psProcessHandleBaseBeingFreed;
+/* Lock for the process handle base table */
+static POS_LOCK g_hProcessHandleBaseLock;
+/* Hash table with process handle bases */
+static HASH_TABLE *g_psProcessHandleBaseTable;
+
+void LockHandle(PVRSRV_HANDLE_BASE *psBase)
+{
+	OSLockAcquire(psBase->hLock);
+}
+
+void UnlockHandle(PVRSRV_HANDLE_BASE *psBase)
+{
+	OSLockRelease(psBase->hLock);
+}
+
+/*
+ * Kernel handle base structure. This is used for handles that are not
+ * allocated on behalf of a particular process.
+ */
+PVRSRV_HANDLE_BASE *gpsKernelHandleBase = NULL;
+
+/* Increase the lookup reference count on the given handle.
+ * The handle lock must already be acquired.
+ * Returns: the reference count after the increment
+ */
+static inline IMG_UINT32 HandleGet(HANDLE_DATA *psHandleData)
+{
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	if (!OSLockIsLocked(psHandleData->psBase->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle lock is not locked", __func__));
+		OSDumpStack();
+	}
+#endif
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = %u, iLookupCount %d -> %d",
+	        __func__, psHandleData->bCanLookup, psHandleData->iLookupCount,
+	        psHandleData->iLookupCount + 1));
+#endif /* DEBUG_REFCNT */
+
+	PVR_ASSERT(psHandleData->bCanLookup);
+
+	return ++psHandleData->iLookupCount;
+}
+
+/* Decrease the lookup reference count on the given handle.
+ * The handle lock must already be acquired.
+ * Returns: the reference count after the decrement
+ */
+static inline IMG_UINT32 HandlePut(HANDLE_DATA *psHandleData)
+{
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	if (!OSLockIsLocked(psHandleData->psBase->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle lock is not locked", __func__));
+		OSDumpStack();
+	}
+#endif
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = %u, iLookupCount %d -> %d",
+	        __func__, psHandleData->bCanLookup, psHandleData->iLookupCount,
+	        psHandleData->iLookupCount - 1));
+#endif /* DEBUG_REFCNT */
+
+	/* psHandleData->bCanLookup can be false at this point */
+	PVR_ASSERT(psHandleData->iLookupCount > 0);
+
+	return --psHandleData->iLookupCount;
+}
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+static const IMG_CHAR *HandleTypeToString(PVRSRV_HANDLE_TYPE eType)
+{
+	#define HANDLETYPE(x) \
+			case PVRSRV_HANDLE_TYPE_##x: \
+				return #x;
+	switch (eType)
+	{
+		#include "handle_types.h"
+		#undef HANDLETYPE
+
+		default:
+			return "INVALID";
+	}
+}
+
+static const IMG_CHAR *HandleBaseTypeToString(PVRSRV_HANDLE_BASE_TYPE eType)
+{
+	#define HANDLEBASETYPE(x) \
+			case PVRSRV_HANDLE_BASE_TYPE_##x: \
+				return #x;
+	switch (eType)
+	{
+		HANDLEBASETYPE(CONNECTION);
+		HANDLEBASETYPE(PROCESS);
+		HANDLEBASETYPE(GLOBAL);
+		#undef HANDLEBASETYPE
+
+		default:
+			return "INVALID";
+	}
+}
+#endif
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFree(PVRSRV_HANDLE_BASE *psBase,
+                                                   HANDLE_DATA *psHandleData,
+                                                   IMG_HANDLE hHandle,
+                                                   PVRSRV_HANDLE_TYPE eType);
+
+static PVRSRV_ERROR HandleFreePrivData(PVRSRV_HANDLE_BASE *psBase,
+                                       HANDLE_DATA *psHandleData,
+                                       IMG_HANDLE hHandle,
+                                       PVRSRV_HANDLE_TYPE eType);
+
+static PVRSRV_ERROR HandleFreeDestroy(PVRSRV_HANDLE_BASE *psBase,
+                                      HANDLE_DATA *psHandleData,
+                                      IMG_HANDLE hHandle,
+                                      PVRSRV_HANDLE_TYPE eType);
+
+/*!
+*******************************************************************************
+ @Function      GetHandleData
+ @Description   Get the handle data structure for a given handle
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type or PVRSRV_HANDLE_TYPE_NONE if the handle
+                        type is not to be checked.
+ @Output        ppsHandleData - pointer to a pointer to the handle data struct
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(GetHandleData)
+#endif
+static INLINE
+PVRSRV_ERROR GetHandleData(PVRSRV_HANDLE_BASE *psBase,
+			   HANDLE_DATA **ppsHandleData,
+			   IMG_HANDLE hHandle,
+			   PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError;
+
+	eError = gpsHandleFuncs->pfnGetHandleData(psBase->psImplBase,
+						  hHandle,
+						  (void **)&psHandleData);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/*
+	 * Unless PVRSRV_HANDLE_TYPE_NONE was passed in to this function,
+	 * check handle is of the correct type.
+	 */
+	if (unlikely(eType != PVRSRV_HANDLE_TYPE_NONE && eType != psHandleData->eType))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "GetHandleData: Type mismatch. Lookup request: Handle %p, type: %s (%u) but stored handle is type %s (%u)",
+			 hHandle,
+			 HandleTypeToString(eType),
+			 eType,
+			 HandleTypeToString(psHandleData->eType),
+			 psHandleData->eType));
+		return PVRSRV_ERROR_HANDLE_TYPE_MISMATCH;
+	}
+
+	/* Return the handle structure */
+	*ppsHandleData = psHandleData;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListInit
+ @Description   Initialise a linked list structure embedded in a handle
+                structure.
+ @Input         hHandle - handle containing the linked list structure
+                psList - pointer to linked list structure
+                hParent - parent handle or NULL
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListInit)
+#endif
+static INLINE
+void HandleListInit(IMG_HANDLE hHandle, HANDLE_LIST *psList, IMG_HANDLE hParent)
+{
+	psList->hPrev = hHandle;
+	psList->hNext = hHandle;
+	psList->hParent = hParent;
+}
+
+/*!
+*******************************************************************************
+ @Function      InitParentList
+ @Description   Initialise the children list head in a handle structure.
+                The children are the subhandles of this handle.
+ @Input         psHandleData - pointer to handle data structure
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitParentList)
+#endif
+static INLINE
+void InitParentList(HANDLE_DATA *psHandleData)
+{
+	IMG_HANDLE hParent = psHandleData->hHandle;
+
+	HandleListInit(hParent, &psHandleData->sChildren, hParent);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitChildEntry
+ @Description   Initialise the child list entry in a handle structure. The list
+                entry is used to link together subhandles of a given handle.
+ @Input         psHandleData - pointer to handle data structure
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitChildEntry)
+#endif
+static INLINE
+void InitChildEntry(HANDLE_DATA *psHandleData)
+{
+	HandleListInit(psHandleData->hHandle, &psHandleData->sSiblings, NULL);
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListIsEmpty
+ @Description   Determine whether a given linked list is empty.
+ @Input         hHandle - handle containing the list head
+                psList - pointer to the list head
+ @Return        IMG_TRUE if the list is empty, IMG_FALSE if it isn't.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListIsEmpty)
+#endif
+static INLINE
+IMG_BOOL HandleListIsEmpty(IMG_HANDLE hHandle, HANDLE_LIST *psList) /* Instead of passing in the handle can we not just do (psList->hPrev == psList->hNext) ? IMG_TRUE : IMG_FALSE ??? */
+{
+	IMG_BOOL bIsEmpty;
+
+	bIsEmpty = (IMG_BOOL)(psList->hNext == hHandle);
+
+#ifdef DEBUG
+	{
+		IMG_BOOL bIsEmpty2;
+
+		bIsEmpty2 = (IMG_BOOL)(psList->hPrev == hHandle);
+		PVR_ASSERT(bIsEmpty == bIsEmpty2);
+	}
+#endif
+
+	return bIsEmpty;
+}
+
+#ifdef DEBUG
+/*!
+*******************************************************************************
+ @Function      NoChildren
+ @Description   Determine whether a handle has any subhandles
+ @Input         psHandleData - pointer to handle data structure
+ @Return        IMG_TRUE if the handle has no subhandles, IMG_FALSE if it does.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(NoChildren)
+#endif
+static INLINE
+IMG_BOOL NoChildren(HANDLE_DATA *psHandleData)
+{
+	PVR_ASSERT(psHandleData->sChildren.hParent == psHandleData->hHandle);
+
+	return HandleListIsEmpty(psHandleData->hHandle, &psHandleData->sChildren);
+}
+
+/*!
+*******************************************************************************
+ @Function      NoParent
+ @Description   Determine whether a handle is a subhandle
+ @Input         psHandleData - pointer to handle data structure
+ @Return        IMG_TRUE if the handle is not a subhandle, IMG_FALSE if it is.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(NoParent)
+#endif
+static INLINE
+IMG_BOOL NoParent(HANDLE_DATA *psHandleData)
+{
+	if (HandleListIsEmpty(psHandleData->hHandle, &psHandleData->sSiblings))
+	{
+		PVR_ASSERT(psHandleData->sSiblings.hParent == NULL);
+
+		return IMG_TRUE;
+	}
+
+	PVR_ASSERT(psHandleData->sSiblings.hParent != NULL);
+	return IMG_FALSE;
+}
+#endif /*DEBUG*/
+
+/*!
+*******************************************************************************
+ @Function      ParentHandle
+ @Description   Determine the parent of a handle
+ @Input         psHandleData - pointer to handle data structure
+ @Return        Parent handle, or NULL if the handle is not a subhandle.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(ParentHandle)
+#endif
+static INLINE
+IMG_HANDLE ParentHandle(HANDLE_DATA *psHandleData)
+{
+	return psHandleData->sSiblings.hParent;
+}
+
+/*
+ * GetHandleListFromHandleAndOffset is used to generate either a
+ * pointer to the subhandle list head, or a pointer to the linked list
+ * structure of an item on a subhandle list.
+ * The list head is itself on the list, but is at a different offset
+ * in the handle structure to the linked list structure for items on
+ * the list. The two linked list structures are differentiated by
+ * the third parameter, containing the parent handle. The parent field
+ * in the list head structure references the handle structure that contains
+ * it. For items on the list, the parent field in the linked list structure
+ * references the parent handle, which will be different from the handle
+ * containing the linked list structure.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(GetHandleListFromHandleAndOffset)
+#endif
+static INLINE
+HANDLE_LIST *GetHandleListFromHandleAndOffset(PVRSRV_HANDLE_BASE *psBase,
+					      IMG_HANDLE hEntry,
+					      IMG_HANDLE hParent,
+					      size_t uiParentOffset,
+					      size_t uiEntryOffset)
+{
+	HANDLE_DATA *psHandleData = NULL;
+
+	PVR_ASSERT(psBase != NULL);
+
+	if (GetHandleData(psBase, &psHandleData, hEntry,
+	                  PVRSRV_HANDLE_TYPE_NONE) != PVRSRV_OK)
+	{
+		return NULL;
+	}
+
+	if (hEntry == hParent)
+	{
+		return (HANDLE_LIST *)IMG_OFFSET_ADDR(psHandleData, uiParentOffset);
+	}
+	else
+	{
+		return (HANDLE_LIST *)IMG_OFFSET_ADDR(psHandleData, uiEntryOffset);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListInsertBefore
+ @Description   Insert a handle before a handle currently on the list.
+ @Input         hEntry - handle to be inserted after
+                psEntry - pointer to handle structure to be inserted after
+                uiParentOffset - offset to list head struct in handle structure
+                hNewEntry - handle to be inserted
+                psNewEntry - pointer to handle structure of item to be inserted
+                uiEntryOffset - offset of list item struct in handle structure
+                hParent - parent handle of hNewEntry
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListInsertBefore)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListInsertBefore(PVRSRV_HANDLE_BASE *psBase,
+				    IMG_HANDLE hEntry,
+				    HANDLE_LIST *psEntry,
+				    size_t uiParentOffset,
+				    IMG_HANDLE hNewEntry,
+				    HANDLE_LIST *psNewEntry,
+				    size_t uiEntryOffset,
+				    IMG_HANDLE hParent)
+{
+	HANDLE_LIST *psPrevEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psEntry != NULL, "psEntry");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNewEntry != NULL, "psNewEntry");
+
+	psPrevEntry = GetHandleListFromHandleAndOffset(psBase,
+						       psEntry->hPrev,
+						       hParent,
+						       uiParentOffset,
+						       uiEntryOffset);
+	if (psPrevEntry == NULL)
+	{
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+
+	PVR_ASSERT(psNewEntry->hParent == NULL);
+	PVR_ASSERT(hEntry == psPrevEntry->hNext);
+
+#if defined(DEBUG)
+	{
+		HANDLE_LIST *psParentList;
+
+		psParentList = GetHandleListFromHandleAndOffset(psBase,
+								hParent,
+								hParent,
+								uiParentOffset,
+								uiParentOffset);
+		PVR_ASSERT(psParentList && psParentList->hParent == hParent);
+	}
+#endif /* defined(DEBUG) */
+
+	psNewEntry->hPrev = psEntry->hPrev;
+	psEntry->hPrev = hNewEntry;
+
+	psNewEntry->hNext = hEntry;
+	psPrevEntry->hNext = hNewEntry;
+
+	psNewEntry->hParent = hParent;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      AdoptChild
+ @Description   Assign a subhandle to a handle
+ @Input         psParentData - pointer to handle structure of parent handle
+                psChildData - pointer to handle structure of child subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(AdoptChild)
+#endif
+static INLINE
+PVRSRV_ERROR AdoptChild(PVRSRV_HANDLE_BASE *psBase,
+			HANDLE_DATA *psParentData,
+			HANDLE_DATA *psChildData)
+{
+	IMG_HANDLE hParent = psParentData->sChildren.hParent;
+
+	PVR_ASSERT(hParent == psParentData->hHandle);
+
+	return HandleListInsertBefore(psBase,
+				      hParent,
+				      &psParentData->sChildren,
+				      offsetof(HANDLE_DATA, sChildren),
+				      psChildData->hHandle,
+				      &psChildData->sSiblings,
+				      offsetof(HANDLE_DATA, sSiblings),
+				      hParent);
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListRemove
+ @Description   Remove a handle from a list
+ @Input         hEntry - handle to be removed
+                psEntry - pointer to handle structure of item to be removed
+                uiEntryOffset - offset of list item struct in handle structure
+                uiParentOffset - offset to list head struct in handle structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListRemove)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListRemove(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE hEntry,
+			      HANDLE_LIST *psEntry,
+			      size_t uiEntryOffset,
+			      size_t uiParentOffset)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psEntry != NULL, "psEntry");
+
+	if (!HandleListIsEmpty(hEntry, psEntry))
+	{
+		HANDLE_LIST *psPrev;
+		HANDLE_LIST *psNext;
+
+		psPrev = GetHandleListFromHandleAndOffset(psBase,
+							  psEntry->hPrev,
+							  psEntry->hParent,
+							  uiParentOffset,
+							  uiEntryOffset);
+		if (psPrev == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		psNext = GetHandleListFromHandleAndOffset(psBase,
+							  psEntry->hNext,
+							  psEntry->hParent,
+							  uiParentOffset,
+							  uiEntryOffset);
+		if (psNext == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		/*
+		 * The list head is on the list, and we don't want to
+		 * remove it.
+		 */
+		PVR_ASSERT(psEntry->hParent != NULL);
+
+		psPrev->hNext = psEntry->hNext;
+		psNext->hPrev = psEntry->hPrev;
+
+		HandleListInit(hEntry, psEntry, NULL);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      UnlinkFromParent
+ @Description   Remove a subhandle from its parents list
+ @Input         psHandleData - pointer to handle data structure of child
+                               subhandle.
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(UnlinkFromParent)
+#endif
+static INLINE
+PVRSRV_ERROR UnlinkFromParent(PVRSRV_HANDLE_BASE *psBase,
+			      HANDLE_DATA *psHandleData)
+{
+	return HandleListRemove(psBase,
+				psHandleData->hHandle,
+				&psHandleData->sSiblings,
+				offsetof(HANDLE_DATA, sSiblings),
+				offsetof(HANDLE_DATA, sChildren));
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListIterate
+ @Description   Iterate over the items in a list
+ @Input         psHead - pointer to list head
+                uiParentOffset - offset to list head struct in handle structure
+                uiEntryOffset - offset of list item struct in handle structure
+                pfnIterFunc - function to be called for each handle in the list
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListIterate)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListIterate(PVRSRV_HANDLE_BASE *psBase,
+			       HANDLE_LIST *psHead,
+			       size_t uiParentOffset,
+			       size_t uiEntryOffset,
+			       PVRSRV_ERROR (*pfnIterFunc)(PVRSRV_HANDLE_BASE *, IMG_HANDLE))
+{
+	IMG_HANDLE hHandle = psHead->hNext;
+	IMG_HANDLE hParent = psHead->hParent;
+	IMG_HANDLE hNext;
+
+	PVR_ASSERT(psHead->hParent != NULL);
+
+	/*
+	 * Follow the next chain from the list head until we reach
+	 * the list head again, which signifies the end of the list.
+	 */
+	while (hHandle != hParent)
+	{
+		HANDLE_LIST *psEntry;
+		PVRSRV_ERROR eError;
+
+		psEntry = GetHandleListFromHandleAndOffset(psBase,
+							   hHandle,
+							   hParent,
+							   uiParentOffset,
+							   uiEntryOffset);
+		if (psEntry == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		PVR_ASSERT(psEntry->hParent == psHead->hParent);
+
+		/*
+		 * Get the next index now, in case the list item is
+		 * modified by the iteration function.
+		 */
+		hNext = psEntry->hNext;
+
+		eError = (*pfnIterFunc)(psBase, hHandle);
+		PVR_RETURN_IF_ERROR(eError);
+
+		hHandle = hNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      IterateOverChildren
+ @Description   Iterate over the subhandles of a parent handle
+ @Input         psParentData - pointer to parent handle structure
+                pfnIterFunc - function to be called for each subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(IterateOverChildren)
+#endif
+static INLINE
+PVRSRV_ERROR IterateOverChildren(PVRSRV_HANDLE_BASE *psBase,
+				 HANDLE_DATA *psParentData,
+				 PVRSRV_ERROR (*pfnIterFunc)(PVRSRV_HANDLE_BASE *, IMG_HANDLE))
+{
+	 return HandleListIterate(psBase,
+				  &psParentData->sChildren,
+				  offsetof(HANDLE_DATA, sChildren),
+				  offsetof(HANDLE_DATA, sSiblings),
+				  pfnIterFunc);
+}
+
+/*!
+*******************************************************************************
+ @Function      ParentIfPrivate
+ @Description   Return the parent handle if the handle was allocated with
+                PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE, else return NULL.
+ @Input         psHandleData - pointer to handle data structure
+ @Return        Parent handle or NULL
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(ParentIfPrivate)
+#endif
+static INLINE
+IMG_HANDLE ParentIfPrivate(HANDLE_DATA *psHandleData)
+{
+	return TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE) ?
+			ParentHandle(psHandleData) : NULL;
+}
+
+/*!
+*******************************************************************************
+ @Function      InitKey
+ @Description   Initialise a hash table key for the current process
+ @Input         aKey - pointer to key
+                psBase - pointer to handle base structure
+                pvData - pointer to the resource the handle represents
+                eType - type of resource
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitKey)
+#endif
+static INLINE
+void InitKey(HAND_KEY aKey,
+	     PVRSRV_HANDLE_BASE *psBase,
+	     void *pvData,
+	     PVRSRV_HANDLE_TYPE eType,
+	     IMG_HANDLE hParent)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+
+	aKey[HAND_KEY_DATA] = (uintptr_t)pvData;
+	aKey[HAND_KEY_TYPE] = (uintptr_t)eType;
+	aKey[HAND_KEY_PARENT] = (uintptr_t)hParent;
+}
+
+/*!
+*******************************************************************************
+ @Function      FindHandle
+ @Description   Find handle corresponding to a resource pointer
+ @Input         psBase - pointer to handle base structure
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Return        The handle, or NULL if not found
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(FindHandle)
+#endif
+static INLINE
+IMG_HANDLE FindHandle(PVRSRV_HANDLE_BASE *psBase,
+		      void *pvData,
+		      PVRSRV_HANDLE_TYPE eType,
+		      IMG_HANDLE hParent)
+{
+	HAND_KEY aKey;
+
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+
+	InitKey(aKey, psBase, pvData, eType, hParent);
+
+	return (IMG_HANDLE) HASH_Retrieve_Extended(psBase->psHashTab, aKey);
+}
+
+/*!
+*******************************************************************************
+ @Function      AllocHandle
+ @Description   Allocate a new handle
+ @Input         phHandle - location for new handle
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                hParent - parent handle or NULL
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+static PVRSRV_ERROR AllocHandle(PVRSRV_HANDLE_BASE *psBase,
+				IMG_HANDLE *phHandle,
+				void *pvData,
+				PVRSRV_HANDLE_TYPE eType,
+				PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				IMG_HANDLE hParent,
+				PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	HANDLE_DATA *psNewHandleData;
+	IMG_HANDLE hHandle;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(psBase != NULL && psBase->psHashTab != NULL);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	if (!TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		/* Handle must not already exist */
+		PVR_ASSERT(FindHandle(psBase, pvData, eType, hParent) == NULL);
+	}
+
+	psNewHandleData = OSAllocZMem(sizeof(*psNewHandleData));
+	PVR_LOG_RETURN_IF_NOMEM(psNewHandleData, "OSAllocZMem");
+
+	eError = gpsHandleFuncs->pfnAcquireHandle(psBase->psImplBase, &hHandle,
+	                                          psNewHandleData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnAcquireHandle",
+	                  ErrorFreeHandleData);
+
+	/*
+	 * If a data pointer can be associated with multiple handles, we
+	 * don't put the handle in the hash table, as the data pointer
+	 * may not map to a unique handle
+	 */
+	if (!TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+
+		/* Initialise hash key */
+		InitKey(aKey, psBase, pvData, eType, hParent);
+
+		/* Put the new handle in the hash table */
+		eError = HASH_Insert_Extended(psBase->psHashTab, aKey, (uintptr_t)hHandle) ?
+		        PVRSRV_OK : PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "couldn't add handle to hash table",
+		                  ErrorReleaseHandle);
+	}
+
+	psNewHandleData->hHandle = hHandle;
+	psNewHandleData->eType = eType;
+	psNewHandleData->eFlag = eFlag;
+	psNewHandleData->pvData = pvData;
+	psNewHandleData->pfnReleaseData = pfnReleaseData;
+	psNewHandleData->iLookupCount = 0;
+	psNewHandleData->bCanLookup = IMG_TRUE;
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = true", __func__));
+#endif /* DEBUG_REFCNT */
+
+	InitParentList(psNewHandleData);
+#if defined(DEBUG)
+	PVR_ASSERT(NoChildren(psNewHandleData));
+#endif
+
+	InitChildEntry(psNewHandleData);
+#if defined(DEBUG)
+	PVR_ASSERT(NoParent(psNewHandleData));
+#endif
+
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	psNewHandleData->psBase = psBase;
+#endif
+
+	/* Return the new handle to the client */
+	*phHandle = psNewHandleData->hHandle;
+
+	return PVRSRV_OK;
+
+ErrorReleaseHandle:
+	(void)gpsHandleFuncs->pfnReleaseHandle(psBase->psImplBase, hHandle, NULL);
+
+ErrorFreeHandleData:
+	OSFreeMem(psNewHandleData);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandle
+ @Description   Allocate a handle
+ @Input         psBase - pointer to handle base structure
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase,
+			       IMG_HANDLE *phHandle,
+			       void *pvData,
+			       PVRSRV_HANDLE_TYPE eType,
+			       PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+			       PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVAllocHandleUnlocked(psBase, phHandle, pvData, eType, eFlag, pfnReleaseData);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandleUnlocked
+ @Description   Allocate a handle without acquiring/releasing the handle lock.
+                The function assumes you hold the lock when called.
+ @Input         phHandle - location for new handle
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+			       IMG_HANDLE *phHandle,
+			       void *pvData,
+			       PVRSRV_HANDLE_TYPE eType,
+			       PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+			       PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	*phHandle = NULL;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnReleaseData != NULL, "pfnReleaseData");
+
+	return AllocHandle(psBase, phHandle, pvData, eType, eFlag, NULL, pfnReleaseData);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocSubHandle
+ @Description   Allocate a subhandle
+ @Input         pvData - pointer to resource to be associated with the subhandle
+                eType - the type of resource
+                hParent - parent handle
+ @Output        phHandle - points to new subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData,
+				  PVRSRV_HANDLE_TYPE eType,
+				  PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				  IMG_HANDLE hParent)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVAllocSubHandleUnlocked(psBase, phHandle, pvData, eType, eFlag, hParent);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocSubHandleUnlocked
+ @Description   Allocate a subhandle without acquiring/releasing the handle
+                lock. The function assumes you hold the lock when called.
+ @Input         pvData - pointer to resource to be associated with the subhandle
+                eType - the type of resource
+                hParent - parent handle
+ @Output        phHandle - points to new subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocSubHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData,
+				  PVRSRV_HANDLE_TYPE eType,
+				  PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				  IMG_HANDLE hParent)
+{
+	HANDLE_DATA *psPHandleData = NULL;
+	HANDLE_DATA *psCHandleData = NULL;
+	IMG_HANDLE hParentKey;
+	IMG_HANDLE hHandle;
+	PVRSRV_ERROR eError;
+
+	*phHandle = NULL;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psBase, eError, Exit);
+
+	hParentKey = TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE) ? hParent : NULL;
+
+	/* Lookup the parent handle */
+	eError = GetHandleData(psBase, &psPHandleData, hParent, PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to get parent handle structure",
+	                  Exit);
+
+	eError = AllocHandle(psBase, &hHandle, pvData, eType, eFlag, hParentKey, NULL);
+	PVR_GOTO_IF_ERROR(eError, Exit);
+
+	eError = GetHandleData(psBase, &psCHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+	/* If we were able to allocate the handle then there should be no reason why we
+	 * can't also get it's handle structure. Otherwise something has gone badly wrong.
+	 */
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "Failed to get parent handle structure",
+	                  ExitFreeHandle);
+
+	/*
+	 * Get the parent handle structure again, in case the handle
+	 * structure has moved (depending on the implementation
+	 * of AllocHandle).
+	 */
+	eError = GetHandleData(psBase, &psPHandleData, hParent, PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to get parent handle structure",
+	                  ExitFreeHandle);
+
+	eError = AdoptChild(psBase, psPHandleData, psCHandleData);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "parent handle failed to adopt subhandle",
+	                  ExitFreeHandle);
+
+	*phHandle = hHandle;
+
+	return PVRSRV_OK;
+
+ExitFreeHandle:
+	PVRSRVDestroyHandleUnlocked(psBase, hHandle, eType);
+Exit:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFindHandle
+ @Description   Find handle corresponding to a resource pointer
+ @Input         pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Output        phHandle - points to returned handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE *phHandle,
+			      void *pvData,
+			      PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVFindHandleUnlocked(psBase, phHandle, pvData, eType);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFindHandleUnlocked
+ @Description   Find handle corresponding to a resource pointer without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Output        phHandle - points to the returned handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFindHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE *phHandle,
+			      void *pvData,
+			      PVRSRV_HANDLE_TYPE eType)
+{
+	IMG_HANDLE hHandle;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	/* See if there is a handle for this data pointer */
+	hHandle = FindHandle(psBase, pvData, eType, NULL);
+	if (hHandle == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error finding handle. Type %u",
+			 __func__,
+			 eType));
+
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	*phHandle = hHandle;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupHandle
+ @Description   Lookup the data pointer corresponding to a handle
+ @Input         hHandle - handle from client
+                eType - handle type
+                bRef - If TRUE, a reference will be added on the handle if the
+                       lookup is successful.
+ @Output        ppvData - points to the return data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase,
+				void **ppvData,
+				IMG_HANDLE hHandle,
+				PVRSRV_HANDLE_TYPE eType,
+				IMG_BOOL bRef)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVLookupHandleUnlocked(psBase, ppvData, hHandle, eType, bRef);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupHandleUnlocked
+ @Description   Lookup the data pointer corresponding to a handle without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         hHandle - handle from client
+                eType - handle type
+                bRef - If TRUE, a reference will be added on the handle if the
+                       lookup is successful.
+ @Output        ppvData - points to the returned data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+				void **ppvData,
+				IMG_HANDLE hHandle,
+				PVRSRV_HANDLE_TYPE eType,
+				IMG_BOOL bRef)
+{
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error looking up handle (%s) for base %p of type %s. Handle %p, type %s",
+			 __func__,
+			 PVRSRVGetErrorString(eError),
+			 psBase,
+			 HandleBaseTypeToString(psBase->eType),
+			 (void*) hHandle,
+			 HandleTypeToString(eType)));
+#if defined(DEBUG) || defined(PVRSRV_NEED_PVR_DPF)
+		OSDumpStack();
+#endif
+		return eError;
+	}
+
+	/* If bCanLookup is false it means that a destroy operation was already
+	 * called on this handle; therefore it can no longer be looked up. */
+	if (!psHandleData->bCanLookup)
+	{
+		return PVRSRV_ERROR_HANDLE_NOT_ALLOCATED;
+	}
+
+	if (bRef)
+	{
+		HandleGet(psHandleData);
+	}
+
+	*ppvData = psHandleData->pvData;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupSubHandle
+ @Description   Lookup the data pointer corresponding to a subhandle
+ @Input         hHandle - handle from client
+                eType - handle type
+                hAncestor - ancestor handle
+ @Output        ppvData - points to the returned data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase,
+				   void **ppvData,
+				   IMG_HANDLE hHandle,
+				   PVRSRV_HANDLE_TYPE eType,
+				   IMG_HANDLE hAncestor)
+{
+	HANDLE_DATA *psPHandleData = NULL;
+	HANDLE_DATA *psCHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	LockHandle(psBase);
+
+	eError = GetHandleData(psBase, &psCHandleData, hHandle, eType);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error looking up subhandle (%s). Handle %p, type %u",
+			 __func__,
+			 PVRSRVGetErrorString(eError),
+			 (void*) hHandle,
+			 eType));
+		OSDumpStack();
+		goto ExitUnlock;
+	}
+
+	/* Look for hAncestor among the handle's ancestors */
+	for (psPHandleData = psCHandleData; ParentHandle(psPHandleData) != hAncestor; )
+	{
+		eError = GetHandleData(psBase, &psPHandleData, ParentHandle(psPHandleData), PVRSRV_HANDLE_TYPE_NONE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "GetHandleData");
+			eError = PVRSRV_ERROR_INVALID_SUBHANDLE;
+			goto ExitUnlock;
+		}
+	}
+
+	*ppvData = psCHandleData->pvData;
+
+	eError = PVRSRV_OK;
+
+ExitUnlock:
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseHandle
+ @Description   Release a handle that is no longer needed
+ @Input         hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+void PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase,
+                         IMG_HANDLE hHandle,
+                         PVRSRV_HANDLE_TYPE eType)
+{
+	LockHandle(psBase);
+	PVRSRVReleaseHandleUnlocked(psBase, hHandle, eType);
+	UnlockHandle(psBase);
+}
+
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseHandleUnlocked
+ @Description   Release a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         hHandle - handle from client
+                eType - handle type
+******************************************************************************/
+void PVRSRVReleaseHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                 IMG_HANDLE hHandle,
+                                 PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(psBase != NULL);
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psBase != NULL, "invalid psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error (%s) looking up handle %p of type %s "
+		        "for base %p of type %s.", __func__, PVRSRVGetErrorString(eError),
+		        (void*) hHandle, HandleTypeToString(eType), psBase,
+		        HandleBaseTypeToString(psBase->eType)));
+
+		return;
+	}
+
+	PVR_ASSERT(psHandleData->bCanLookup);
+	PVR_ASSERT(psHandleData->iLookupCount > 0);
+
+	/* If there are still outstanding lookups for this handle or the handle
+	 * has not been destroyed yet, return early */
+	HandlePut(psHandleData);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVPurgeHandles
+ @Description   Purge handles for a given handle base
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPurgeHandles(PVRSRV_HANDLE_BASE *psBase)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	LockHandle(psBase);
+	eError = gpsHandleFuncs->pfnPurgeHandles(psBase->psImplBase);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFreeWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                                          IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleUnrefAndMaybeMarkForFree(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFree(PVRSRV_HANDLE_BASE *psBase,
+                                                   HANDLE_DATA *psHandleData,
+                                                   IMG_HANDLE hHandle,
+                                                   PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	/* If bCanLookup is false it means that the destructor was called more than
+	 * once on this handle. */
+	if (!psHandleData->bCanLookup)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle %p of type %s already freed.",
+		        __func__, psHandleData->hHandle,
+		        HandleTypeToString(psHandleData->eType)));
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	if (psHandleData->iLookupCount > 0)
+	{
+		return PVRSRV_ERROR_OBJECT_STILL_REFERENCED;
+	}
+
+	/* Mark this handle as freed only if it's no longer referenced by any
+	 * lookup. The user space should retry freeing this handle once there are
+	 * no outstanding lookups. */
+	psHandleData->bCanLookup = IMG_FALSE;
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = false, iLookupCount = %d", __func__,
+	        psHandleData->iLookupCount));
+#endif /* DEBUG_REFCNT */
+
+	/* Prepare children for destruction */
+	eError = IterateOverChildren(psBase, psHandleData,
+	                             HandleUnrefAndMaybeMarkForFreeWrapper);
+	if (eError != PVRSRV_OK)
+	{
+		/* If there was a problem destroying any of the child handles,
+		 * the parent handle still needs to be able to be looked-up */
+		psHandleData->bCanLookup = IMG_TRUE;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "HandleUnrefAndMaybeMarkForFreeWrapper");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HandleFreePrivDataWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                              IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleFreePrivData(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleFreePrivData(PVRSRV_HANDLE_BASE *psBase,
+                                       HANDLE_DATA *psHandleData,
+                                       IMG_HANDLE hHandle,
+                                       PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	/* Call the release data callback for each reference on the handle */
+	if (psHandleData->pfnReleaseData != NULL)
+	{
+		eError = psHandleData->pfnReleaseData(psHandleData->pvData);
+		if (eError != PVRSRV_OK)
+		{
+			if (PVRSRVIsRetryError(eError))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Got retry while calling release "
+						"data callback for handle %p of type = %s", __func__,
+						hHandle, HandleTypeToString(psHandleData->eType)));
+			}
+			else
+			{
+				PVR_LOG_ERROR(eError, "pfnReleaseData");
+			}
+
+			return eError;
+		}
+
+		/* we don't need this so make sure it's not called on
+		 * the pvData for the second time
+		 */
+		psHandleData->pfnReleaseData = NULL;
+	}
+
+	/* Free children's data */
+	eError = IterateOverChildren(psBase, psHandleData,
+	                             HandleFreePrivDataWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IterateOverChildren->HandleFreePrivData");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HandleFreeDestroyWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                             IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleFreeDestroy(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleFreeDestroy(PVRSRV_HANDLE_BASE *psBase,
+                                      HANDLE_DATA *psHandleData,
+                                      IMG_HANDLE hHandle,
+                                      PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psReleasedHandleData;
+	PVRSRV_ERROR eError;
+
+	eError = UnlinkFromParent(psBase, psHandleData);
+	PVR_LOG_RETURN_IF_ERROR(eError, "UnlinkFromParent");
+
+	if (!TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+		IMG_HANDLE hRemovedHandle;
+
+		InitKey(aKey, psBase, psHandleData->pvData, psHandleData->eType,
+		        ParentIfPrivate(psHandleData));
+
+		hRemovedHandle = (IMG_HANDLE) HASH_Remove_Extended(psBase->psHashTab,
+		                                                   aKey);
+
+		PVR_ASSERT(hRemovedHandle != NULL);
+		PVR_ASSERT(hRemovedHandle == psHandleData->hHandle);
+		PVR_UNREFERENCED_PARAMETER(hRemovedHandle);
+	}
+
+	/* Free children */
+	eError = IterateOverChildren(psBase, psHandleData, HandleFreeDestroyWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IterateOverChildren->HandleFreeDestroy");
+
+	eError = gpsHandleFuncs->pfnReleaseHandle(psBase->psImplBase,
+	                                          psHandleData->hHandle,
+	                                          (void **)&psReleasedHandleData);
+	OSFreeMem(psHandleData);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnReleaseHandle");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DestroyHandle(PVRSRV_HANDLE_BASE *psBase,
+                                  IMG_HANDLE hHandle,
+                                  PVRSRV_HANDLE_TYPE eType,
+                                  IMG_BOOL bReleaseLock)
+{
+	PVRSRV_ERROR eError;
+	HANDLE_DATA *psHandleData = NULL;
+
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	PVR_RETURN_IF_ERROR(eError);
+
+	eError = HandleUnrefAndMaybeMarkForFree(psBase, psHandleData, hHandle, eType);
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (bReleaseLock)
+	{
+		UnlockHandle(psBase);
+	}
+
+	eError = HandleFreePrivData(psBase, psHandleData, hHandle, eType);
+	if (eError != PVRSRV_OK)
+	{
+		if (bReleaseLock)
+		{
+			LockHandle(psBase);
+		}
+
+		/* If the data could not be freed due to a temporary condition the
+		 * handle must be kept alive so that the next destroy call can try again */
+		if (PVRSRVIsRetryError(eError))
+		{
+			psHandleData->bCanLookup = IMG_TRUE;
+		}
+
+		return eError;
+	}
+
+	if (bReleaseLock)
+	{
+		LockHandle(psBase);
+	}
+
+	return HandleFreeDestroy(psBase, psHandleData, hHandle, eType);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandle
+ @Description   Destroys a handle that is no longer needed. Will
+                acquiring the handle lock for duration of the call.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandle(PVRSRV_HANDLE_BASE *psBase,
+                                 IMG_HANDLE hHandle,
+                                 PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = DestroyHandle(psBase, hHandle, eType, IMG_FALSE);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandleUnlocked
+ @Description   Destroys a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                         IMG_HANDLE hHandle,
+                                         PVRSRV_HANDLE_TYPE eType)
+{
+	return DestroyHandle(psBase, hHandle, eType, IMG_FALSE);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandleStagedUnlocked
+ @Description   Destroys a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called. This function, unlike
+                PVRSRVDestroyHandleUnlocked(), releases the handle lock while
+                destroying handle private data. This is done to open the
+                bridge for other bridge calls.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandleStagedUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                               IMG_HANDLE hHandle,
+                                               PVRSRV_HANDLE_TYPE eType)
+{
+	return DestroyHandle(psBase, hHandle, eType, IMG_TRUE);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandleBase
+ @Description   Allocate a handle base structure for a process
+ @Input         eType - handle type
+ @Output        ppsBase - points to handle base structure pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandleBase(PVRSRV_HANDLE_BASE **ppsBase,
+                                   PVRSRV_HANDLE_BASE_TYPE eType)
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(gpsHandleFuncs != NULL, "handle management not initialised",
+	                  PVRSRV_ERROR_NOT_READY);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsBase != NULL, "ppsBase");
+
+	psBase = OSAllocZMem(sizeof(*psBase));
+	PVR_LOG_RETURN_IF_NOMEM(psBase, "psBase");
+
+	eError = OSLockCreate(&psBase->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeHandleBase);
+
+	psBase->eType = eType;
+
+	LockHandle(psBase);
+
+	eError = gpsHandleFuncs->pfnCreateHandleBase(&psBase->psImplBase);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnlock);
+
+	psBase->psHashTab = HASH_Create_Extended(HANDLE_HASH_TAB_INIT_SIZE,
+						 sizeof(HAND_KEY),
+						 HASH_Func_Default,
+						 HASH_Key_Comp_Default);
+	PVR_LOG_GOTO_IF_FALSE(psBase->psHashTab != NULL, "couldn't create data pointer"
+	                  " hash table", ErrorDestroyHandleBase);
+
+	*ppsBase = psBase;
+
+	UnlockHandle(psBase);
+
+	return PVRSRV_OK;
+
+ErrorDestroyHandleBase:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+	(void)gpsHandleFuncs->pfnDestroyHandleBase(psBase->psImplBase);
+
+ErrorUnlock:
+	UnlockHandle(psBase);
+	OSLockDestroy(psBase->hLock);
+
+ErrorFreeHandleBase:
+	OSFreeMem(psBase);
+
+	return eError;
+}
+
+#if defined(DEBUG)
+typedef struct _COUNT_HANDLE_DATA_
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	IMG_UINT32 uiHandleDataCount;
+} COUNT_HANDLE_DATA;
+
+/* Used to count the number of handles that have data associated with them */
+static PVRSRV_ERROR CountHandleDataWrapper(IMG_HANDLE hHandle, void *pvData)
+{
+	COUNT_HANDLE_DATA *psData = (COUNT_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData != NULL, "psData");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->psBase != NULL, "psData->psBase");
+
+	eError = GetHandleData(psData->psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData != NULL)
+	{
+		psData->uiHandleDataCount++;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Print a handle in the handle base. Used with the iterator callback. */
+static PVRSRV_ERROR ListHandlesInBase(IMG_HANDLE hHandle, void *pvData)
+{
+	PVRSRV_HANDLE_BASE *psBase = (PVRSRV_HANDLE_BASE*) pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData != NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "    Handle: %6u, CanLookup: %u, LookupCount: %3u, Type: %s (%u), pvData<%p>",
+		       (IMG_UINT32) (uintptr_t) psHandleData->hHandle, psHandleData->bCanLookup,
+		       psHandleData->iLookupCount, HandleTypeToString(psHandleData->eType),
+		       psHandleData->eType, psHandleData->pvData));
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(DEBUG) */
+
+static INLINE IMG_BOOL _CheckIfMaxTimeExpired(IMG_UINT64 ui64TimeStart, IMG_UINT64 ui64MaxBridgeTime)
+{
+	/* unsigned arithmetic is well defined so this will wrap around correctly */
+	return (IMG_BOOL)((OSClockns64() - ui64TimeStart) >= ui64MaxBridgeTime);
+}
+
+static PVRSRV_ERROR FreeKernelHandlesWrapperIterKernel(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_KERNEL_HANDLE_DATA *psData = (FREE_KERNEL_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psKernelHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	/* Get kernel handle data. */
+	eError = GetHandleData(KERNEL_HANDLE_BASE,
+			    &psKernelHandleData,
+			    hHandle,
+			    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psKernelHandleData->pvData == psData->psProcessHandleData->pvData)
+	{
+		/* This kernel handle belongs to our process handle. */
+		psData->hKernelHandle = hHandle;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FreeKernelHandlesWrapperIterProcess(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_KERNEL_HANDLE_DATA *psData = (FREE_KERNEL_HANDLE_DATA *)pvData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	/* Get process handle data. */
+	eError = GetHandleData(psData->psBase,
+			    &psData->psProcessHandleData,
+			    hHandle,
+			    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psData->psProcessHandleData->eFlag == PVRSRV_HANDLE_ALLOC_FLAG_MULTI
+#if defined(SUPPORT_INSECURE_EXPORT)
+		|| psData->psProcessHandleData->eType == PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT
+#endif
+		)
+	{
+		/* Only multi alloc process handles might be in kernel handle base. */
+		psData->hKernelHandle = NULL;
+		/* Iterate over kernel handles. */
+		eError = gpsHandleFuncs->pfnIterateOverHandles(KERNEL_HANDLE_BASE->psImplBase,
+									&FreeKernelHandlesWrapperIterKernel,
+									(void *)psData);
+		PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "failed to iterate over kernel handles",
+		                  eError);
+
+		if (psData->hKernelHandle)
+		{
+			/* Release kernel handle which belongs to our process handle. */
+			eError = gpsHandleFuncs->pfnReleaseHandle(KERNEL_HANDLE_BASE->psImplBase,
+						psData->hKernelHandle,
+						NULL);
+			PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "couldn't release kernel handle",
+			                  eError);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FreeHandleDataWrapper(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_HANDLE_DATA *psData = (FREE_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData != NULL, "psData");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->psBase != NULL, "psData->psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->eHandleFreeType != PVRSRV_HANDLE_TYPE_NONE,
+	                          "psData->eHandleFreeType");
+
+	eError = GetHandleData(psData->psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData == NULL || psHandleData->eType != psData->eHandleFreeType)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(psHandleData->bCanLookup && psHandleData->iLookupCount == 0);
+
+	if (psHandleData->bCanLookup)
+	{
+		if (psHandleData->pfnReleaseData != NULL)
+		{
+			eError = psHandleData->pfnReleaseData(psHandleData->pvData);
+			if (PVRSRVIsRetryError(eError))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Got retry while calling release "
+				        "data callback for handle %p of type = %s", __func__,
+				        hHandle, HandleTypeToString(psHandleData->eType)));
+
+				return eError;
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected error from pfnReleaseData (%s)",
+						 __func__,
+						 PVRSRVGetErrorString(eError)));
+			}
+		}
+
+		psHandleData->bCanLookup = IMG_FALSE;
+	}
+
+	if (!TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+		IMG_HANDLE hRemovedHandle;
+
+		InitKey(aKey,
+			psData->psBase,
+			psHandleData->pvData,
+			psHandleData->eType,
+			ParentIfPrivate(psHandleData));
+
+		hRemovedHandle = (IMG_HANDLE)HASH_Remove_Extended(psData->psBase->psHashTab, aKey);
+
+		PVR_ASSERT(hRemovedHandle != NULL);
+		PVR_ASSERT(hRemovedHandle == psHandleData->hHandle);
+		PVR_UNREFERENCED_PARAMETER(hRemovedHandle);
+	}
+
+	eError = gpsHandleFuncs->pfnSetHandleData(psData->psBase->psImplBase, hHandle, NULL);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSFreeMem(psHandleData);
+
+	/* If we reach the end of the time slice release we can release the global
+	 * lock, invoke the scheduler and reacquire the lock */
+	if ((psData->ui64MaxBridgeTime != 0) && _CheckIfMaxTimeExpired(psData->ui64TimeStart, psData->ui64MaxBridgeTime))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+			 "%s: Lock timeout (timeout: %" IMG_UINT64_FMTSPEC")",
+			 __func__,
+			 psData->ui64MaxBridgeTime));
+		UnlockHandle(psData->psBase);
+		/* Invoke the scheduler to check if other processes are waiting for the lock */
+		OSReleaseThreadQuanta();
+		LockHandle(psData->psBase);
+		/* Set again lock timeout and reset the counter */
+		psData->ui64TimeStart = OSClockns64();
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Lock acquired again", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/* The Ordered Array of PVRSRV_HANDLE_TYPE Enum Entries.
+ *
+ *   Some handles must be destroyed prior to other handles,
+ *   such relationships are established with respect to handle types.
+ *   Therefore elements of this array have to maintain specific order,
+ *   e.g. the PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET must be placed
+ *   before PVRSRV_HANDLE_TYPE_RGX_FREELIST.
+ *
+ *   If ordering is incorrect driver may fail on the ground of cleanup
+ *   routines. Unfortunately, we can mainly rely on the actual definition of
+ *   the array, there is no explicit information about all relationships
+ *   between handle types. These relationships do not necessarily come from
+ *   bridge-specified handle attributes such as 'sub handle' and 'parent
+ *   handle'. They may come from internal/private ref-counters contained by
+ *   objects referenced by our kernel handles.
+ *
+ *   For example, at the bridge level, PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET
+ *   and PVRSRV_HANDLE_TYPE_RGX_FREELIST have no explicit relationship, meaning
+ *   none of them is a sub-handle for the other.
+ *   However the freelist contains internal ref-count that is decremented by
+ *   the destroy routine for KM_HW_RT_DATASET.
+ *
+ *   BE CAREFUL when adding/deleting/moving handle types.
+ */
+static const PVRSRV_HANDLE_TYPE g_aeOrderedFreeList[] =
+{
+	PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+	PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+	PVRSRV_HANDLE_TYPE_RGX_FW_MEMDESC,
+	PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+	PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+	PVRSRV_HANDLE_TYPE_RGX_MEMORY_BLOCK,
+	PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+	PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RI_HANDLE,
+	PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+	PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+	PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+	PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_MAPPING,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+	PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+	PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_PAGELIST,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+	PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+	PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+	PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+	PVRSRV_HANDLE_TYPE_DC_BUFFER,
+	PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+	PVRSRV_HANDLE_TYPE_DC_DEVICE,
+	PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD,
+	PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+	PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+	PVRSRV_HANDLE_TYPE_MM_PLAT_CLEANUP
+};
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFreeKernelHandles
+ @Description   Free kernel handles which belongs to process handles
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFreeKernelHandles(PVRSRV_HANDLE_BASE *psBase)
+{
+	FREE_KERNEL_HANDLE_DATA sHandleData = {NULL};
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	LockHandle(psBase);
+
+	sHandleData.psBase = psBase;
+	/* Iterate over process handles. */
+	eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+								&FreeKernelHandlesWrapperIterProcess,
+								(void *)&sHandleData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnIterateOverHandles", ExitUnlock);
+
+	eError = PVRSRV_OK;
+
+ExitUnlock:
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/* Only called from sync_fallback_server.c */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+/*!
+*******************************************************************************
+ @Function      PVRSRVRetrieveProcessHandleBase
+ @Description   Returns a pointer to the process handle base for the current
+                process. If the current process is the cleanup thread, then the
+                process handle base for the process currently being cleaned up
+                is returned
+ @Return        Pointer to the process handle base, or NULL if not found.
+******************************************************************************/
+PVRSRV_HANDLE_BASE *PVRSRVRetrieveProcessHandleBase(void)
+{
+	PVRSRV_HANDLE_BASE *psHandleBase = NULL;
+	PROCESS_HANDLE_BASE *psProcHandleBase = NULL;
+	IMG_PID ui32PurgePid = PVRSRVGetPurgeConnectionPid();
+	IMG_PID uiCleanupPid = PVRSRVCleanupThreadGetPid();
+	uintptr_t uiCleanupTid = PVRSRVCleanupThreadGetTid();
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	/* Check to see if we're being called from the cleanup thread... */
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+	    (OSGetCurrentThreadID() == uiCleanupTid) &&
+	    (ui32PurgePid > 0))
+	{
+		/* Check to see if the cleanup thread has already removed the
+		 * process handle base from the HASH table.
+		 */
+		psHandleBase = g_psProcessHandleBaseBeingFreed;
+		/* psHandleBase shouldn't be null, as cleanup thread
+		 * should be removing this from the HASH table before
+		 * we get here, so assert if not.
+		 */
+		PVR_ASSERT(psHandleBase);
+	}
+	else
+	{
+		/* Not being called from the cleanup thread, so return the process
+		 * handle base for the current process.
+		 */
+		uintptr_t uiHashKey;
+
+		uiHashKey = OSAcquireCurrentPPIDResourceRefKM();
+		OSReleasePPIDResourceRefKM(uiHashKey);
+
+		psProcHandleBase = (PROCESS_HANDLE_BASE *)
+		    HASH_Retrieve(g_psProcessHandleBaseTable, uiHashKey);
+	}
+
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	if (psHandleBase == NULL && psProcHandleBase != NULL)
+	{
+		psHandleBase = psProcHandleBase->psHandleBase;
+	}
+	return psHandleBase;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAcquireProcessHandleBase
+ @Description   Increments reference count on the process handle base for the
+                current process and returns pointer to the base. If the handle
+                base does not exist it will be allocated.
+ @Output        ppsBase - pointer to a handle base for the current process
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAcquireProcessHandleBase(PROCESS_HANDLE_BASE **ppsBase)
+{
+	PROCESS_HANDLE_BASE *psBase;
+	PVRSRV_ERROR eError;
+	uintptr_t uiHashKey;
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	/* Acquire the process resource hash key (and take ref) */
+	uiHashKey = OSAcquireCurrentPPIDResourceRefKM();
+	PVR_ASSERT(uiHashKey != 0);
+
+	psBase = (PROCESS_HANDLE_BASE*) HASH_Retrieve(g_psProcessHandleBaseTable, uiHashKey);
+
+	/* In case there is none we are going to allocate one */
+	if (psBase == NULL)
+	{
+		IMG_BOOL bSuccess;
+
+		psBase = OSAllocZMem(sizeof(*psBase));
+		PVR_LOG_GOTO_IF_NOMEM(psBase, eError, ErrorUnlock);
+
+		/* Allocate handle base for this process */
+		eError = PVRSRVAllocHandleBase(&psBase->psHandleBase, PVRSRV_HANDLE_BASE_TYPE_PROCESS);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", ErrorFreeProcessHandleBase);
+
+		/* Insert the handle base into the global hash table */
+		psBase->uiHashKey = uiHashKey;
+		bSuccess = HASH_Insert(g_psProcessHandleBaseTable, uiHashKey, (uintptr_t)psBase);
+		PVR_LOG_GOTO_IF_FALSE(bSuccess, "HASH_Insert failed", ErrorFreeHandleBase);
+	}
+
+	OSAtomicIncrement(&psBase->iRefCount);
+
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	*ppsBase = psBase;
+
+	return PVRSRV_OK;
+
+ErrorFreeHandleBase:
+	eError = PVRSRV_ERROR_INVALID_PARAMS;
+	PVRSRVFreeHandleBase(psBase->psHandleBase, 0);
+ErrorFreeProcessHandleBase:
+	OSFreeMem(psBase);
+ErrorUnlock:
+	OSReleasePPIDResourceRefKM(uiHashKey);
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseProcessHandleBase
+ @Description   Decrements reference count on a process handle base psBase
+                for the current process. If the reference count reaches 0 the
+                process handle base will be freed.
+ @Input         psBase - pointer to a process handle base
+ @Inout         ui64MaxBridgeTime - maximum time a handle destroy operation
+                                    can hold the handle base lock (after that
+                                    time a lock will be release and reacquired
+                                    for another time slice)
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVReleaseProcessHandleBase(PROCESS_HANDLE_BASE *psBase,
+                                            IMG_UINT64 ui64MaxBridgeTime)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iRefCount;
+	uintptr_t uiHashValue = 0;
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	iRefCount = OSAtomicDecrement(&psBase->iRefCount);
+
+	if (iRefCount != 0)
+	{
+		/* Release the process resource hash key (drop ref) */
+		OSReleasePPIDResourceRefKM(psBase->uiHashKey);
+		OSLockRelease(g_hProcessHandleBaseLock);
+		return PVRSRV_OK;
+	}
+
+	/* in case the refcount becomes 0 we can remove the process handle base
+	 * and all related objects */
+
+	uiHashValue = HASH_Remove(g_psProcessHandleBaseTable, psBase->uiHashKey);
+	/* Release the process resource hash key (drop ref) */
+	OSReleasePPIDResourceRefKM(psBase->uiHashKey);
+	psBase->uiHashKey = 0;
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	PVR_LOG_RETURN_IF_FALSE(uiHashValue != 0, "HASH_Remove failed",
+	                        PVRSRV_ERROR_UNABLE_TO_REMOVE_HASH_VALUE);
+
+	eError = PVRSRVFreeKernelHandles(psBase->psHandleBase);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeKernelHandles");
+
+	eError = PVRSRVFreeHandleBase(psBase->psHandleBase, ui64MaxBridgeTime);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeHandleBase");
+
+	OSFreeMem(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFreeHandleBase
+ @Description   Free a handle base structure
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFreeHandleBase(PVRSRV_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime)
+{
+#if defined(DEBUG)
+	COUNT_HANDLE_DATA sCountData = {NULL};
+#endif
+	FREE_HANDLE_DATA sHandleData = {NULL};
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_PID uiCleanupPid = PVRSRVCleanupThreadGetPid();
+	uintptr_t uiCleanupTid = PVRSRVCleanupThreadGetTid();
+	IMG_UINT32 ui32ErrorCount = 0;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	LockHandle(psBase);
+
+	/* If this is a process handle base being freed by the cleanup
+	 * thread, store this in g_psProcessHandleBaseBeingFreed
+	 */
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+	    (OSGetCurrentThreadID() == uiCleanupTid) &&
+	    (psBase->eType == PVRSRV_HANDLE_BASE_TYPE_PROCESS))
+	{
+		g_psProcessHandleBaseBeingFreed = psBase;
+	}
+
+	sHandleData.psBase = psBase;
+	sHandleData.ui64TimeStart = OSClockns64();
+	sHandleData.ui64MaxBridgeTime = ui64MaxBridgeTime;
+
+
+#if defined(DEBUG)
+
+	sCountData.psBase = psBase;
+
+	eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+						       &CountHandleDataWrapper,
+						       (void *)&sCountData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnIterateOverHandles", ExitUnlock);
+
+	if (sCountData.uiHandleDataCount != 0)
+	{
+		IMG_BOOL bList = (IMG_BOOL)(sCountData.uiHandleDataCount < HANDLE_DEBUG_LISTING_MAX_NUM);
+
+		PVR_DPF((PVR_DBG_WARNING,
+			 "%s: %u remaining handles in handle base 0x%p "
+			 "(PVRSRV_HANDLE_BASE_TYPE %u).%s",
+			 __func__,
+			 sCountData.uiHandleDataCount,
+			 psBase,
+			 psBase->eType,
+			 bList ? "": " Skipping details, too many items..."));
+
+		if (bList)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "-------- Listing Handles --------"));
+			(void) gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+			                                             &ListHandlesInBase,
+			                                             psBase);
+			PVR_DPF((PVR_DBG_WARNING, "-------- Done Listing    --------"));
+		}
+	}
+
+#endif /* defined(DEBUG) */
+
+	/*
+	 * As we're freeing handles based on type, make sure all
+	 * handles have actually had their data freed to avoid
+	 * resources being leaked
+	 */
+	for (i = 0; i < ARRAY_SIZE(g_aeOrderedFreeList); i++)
+	{
+		sHandleData.eHandleFreeType = g_aeOrderedFreeList[i];
+
+		/* Make sure all handles have been freed before destroying the handle base */
+		eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+							       &FreeHandleDataWrapper,
+							       (void *)&sHandleData);
+
+		/* On retry error return without destroying handle base. Caller may retry. */
+		if (PVRSRVIsRetryError(eError))
+		{
+			PVR_GOTO_IF_ERROR(eError, ExitUnlock);
+		}
+
+		/* Retry is not possible. Continue freeing remaining handles. */
+		if (eError != PVRSRV_OK)
+		{
+			ui32ErrorCount++;
+		}
+	}
+
+	if (ui32ErrorCount > 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error freeing %d handles.", __func__, ui32ErrorCount));
+	}
+
+	if (psBase->psHashTab != NULL)
+	{
+		HASH_Delete(psBase->psHashTab);
+	}
+
+	eError = gpsHandleFuncs->pfnDestroyHandleBase(psBase->psImplBase);
+	PVR_GOTO_IF_ERROR(eError, ExitUnlock);
+
+	UnlockHandle(psBase);
+	OSLockDestroy(psBase->hLock);
+	OSFreeMem(psBase);
+
+	return eError;
+
+ExitUnlock:
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+		(OSGetCurrentThreadID() == uiCleanupTid))
+	{
+		g_psProcessHandleBaseBeingFreed = NULL;
+	}
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVHandleInit
+ @Description   Initialise handle management
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVHandleInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsKernelHandleBase == NULL);
+	PVR_ASSERT(gpsHandleFuncs == NULL);
+	PVR_ASSERT(g_hProcessHandleBaseLock == NULL);
+	PVR_ASSERT(g_psProcessHandleBaseTable == NULL);
+	PVR_ASSERT(!gbLockInitialised);
+
+	eError = OSLockCreate(&gKernelHandleLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate:1");
+
+	eError = OSLockCreate(&g_hProcessHandleBaseLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate:2", ErrorHandleDeinit);
+
+	gbLockInitialised = IMG_TRUE;
+
+	eError = PVRSRVHandleGetFuncTable(&gpsHandleFuncs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVHandleGetFuncTable", ErrorHandleDeinit);
+
+	eError = PVRSRVAllocHandleBase(&gpsKernelHandleBase,
+	                               PVRSRV_HANDLE_BASE_TYPE_GLOBAL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", ErrorHandleDeinit);
+
+	g_psProcessHandleBaseTable = HASH_Create(HANDLE_PROC_HANDLE_HASH_INIT_SIZE);
+	PVR_LOG_GOTO_IF_NOMEM(g_psProcessHandleBaseTable, eError, ErrorHandleDeinit);
+
+	eError = gpsHandleFuncs->pfnEnableHandlePurging(gpsKernelHandleBase->psImplBase);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnEnableHandlePurging", ErrorHandleDeinit);
+
+	return PVRSRV_OK;
+
+ErrorHandleDeinit:
+	(void) PVRSRVHandleDeInit();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVHandleDeInit
+ @Description   De-initialise handle management
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVHandleDeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (gpsHandleFuncs != NULL)
+	{
+		if (gpsKernelHandleBase != NULL)
+		{
+			eError = PVRSRVFreeHandleBase(gpsKernelHandleBase, 0 /* do not release bridge lock */);
+			if (eError == PVRSRV_OK)
+			{
+				gpsKernelHandleBase = NULL;
+			}
+			else
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVFreeHandleBase");
+			}
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+			gpsHandleFuncs = NULL;
+		}
+	}
+	else
+	{
+		/* If we don't have a handle function table we shouldn't have a handle base either */
+		PVR_ASSERT(gpsKernelHandleBase == NULL);
+	}
+
+	if (g_psProcessHandleBaseTable != NULL)
+	{
+		HASH_Delete(g_psProcessHandleBaseTable);
+		g_psProcessHandleBaseTable = NULL;
+	}
+
+	if (g_hProcessHandleBaseLock != NULL)
+	{
+		OSLockDestroy(g_hProcessHandleBaseLock);
+		g_hProcessHandleBaseLock = NULL;
+	}
+
+	if (gKernelHandleLock != NULL)
+	{
+		OSLockDestroy(gKernelHandleLock);
+		gbLockInitialised = IMG_FALSE;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (handle.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.c
new file mode 100644
index 000000000000..b16592ae5ebb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.c
@@ -0,0 +1,1223 @@
+/*************************************************************************/ /*!
+@File           htb_debug.c
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides kernel side debugFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxdevice.h"
+#include "htbserver.h"
+#include "htbuffer_types.h"
+#include "tlstream.h"
+#include "tlclient.h"
+#include "pvrsrv_tlcommon.h"
+#include "di_server.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "htb_debug.h"
+
+// Global data handles for buffer manipulation and processing
+
+typedef struct {
+	IMG_PBYTE	pBuf;		/* Raw data buffer from TL stream */
+	IMG_UINT32	uiBufLen;	/* Amount of data to process from 'pBuf' */
+	IMG_UINT32	uiTotal;	/* Total bytes processed */
+	IMG_UINT32	uiMsgLen;	/* Length of HTB message to be processed */
+	IMG_PBYTE	pCurr;		/* pointer to current message to be decoded */
+	IMG_CHAR	szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];	/* Output string */
+} HTB_Sentinel_t;
+
+typedef struct
+{
+	DI_ENTRY *psDumpHostDiEntry;  /* debug info entry */
+	HTB_Sentinel_t sSentinel;     /* private control structure for HTB DI
+	                                 operations */
+	IMG_HANDLE hStream;           /* stream handle for debugFS use */
+} HTB_DBG_INFO;
+
+static HTB_DBG_INFO g_sHTBData;
+
+// Comment out for extra debug level
+// #define HTB_CHATTY_PRINT(x) PVR_DPF(x)
+#define HTB_CHATTY_PRINT(x)
+
+typedef void (DI_PRINTF)(const OSDI_IMPL_ENTRY *, const IMG_CHAR *, ...);
+
+/******************************************************************************
+ * debugFS display routines
+ *****************************************************************************/
+static int HTBDumpBuffer(DI_PRINTF, OSDI_IMPL_ENTRY *, void *);
+
+static int _DebugHBTraceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	int	retVal;
+
+	PVR_ASSERT(psEntry != NULL);
+
+	/* psEntry should never be NULL */
+	if (psEntry == NULL)
+	{
+		return -1;
+	}
+
+	/* Ensure that we have a valid address to use to dump info from. If NULL we
+	 * return a failure code to terminate the DI read call. pvData is either
+	 * DI_START_TOKEN (for the initial call) or an HTB buffer address for
+	 * subsequent calls [returned from the NEXT function]. */
+	if (pvData == NULL)
+	{
+		return -1;
+	}
+
+	retVal = HTBDumpBuffer(DIPrintf, psEntry, pvData);
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %d", __func__, retVal));
+
+	return retVal;
+}
+
+static IMG_UINT32 idToLogIdx(IMG_UINT32);	/* Forward declaration */
+
+/*
+ * HTB_GetNextMessage
+ *
+ * Get next non-empty message block from the buffer held in pSentinel->pBuf
+ * If we exhaust the data buffer we refill it (after releasing the previous
+ * message(s) [only one non-NULL message, but PAD messages will get released
+ * as we traverse them].
+ *
+ * Input:
+ *	pSentinel		references the already acquired data buffer
+ *
+ * Output:
+ *	pSentinel
+ *		-> uiMsglen updated to the size of the non-NULL message
+ *
+ * Returns:
+ *	Address of first non-NULL message in the buffer (if any)
+ *	NULL if there is no further data available from the stream and the buffer
+ *	contents have been drained.
+ */
+static IMG_PBYTE HTB_GetNextMessage(HTB_Sentinel_t *pSentinel)
+{
+	void	*pNext, *pLast, *pStart, *pData = NULL;
+	void	*pCurrent;		/* Current processing point within buffer */
+	PVRSRVTL_PPACKETHDR	ppHdr;	/* Current packet header */
+	IMG_UINT32	uiHdrType;		/* Packet header type */
+	IMG_UINT32	uiMsgSize;		/* Message size of current packet (bytes) */
+	IMG_BOOL	bUnrecognizedErrorPrinted = IMG_FALSE;
+	IMG_UINT32	ui32Data;
+	IMG_UINT32	ui32LogIdx;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pSentinel != NULL);
+
+	pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+
+	pStart = pSentinel->pBuf;
+
+	pNext = pStart;
+	pSentinel->uiMsgLen = 0;	// Reset count for this message
+	uiMsgSize = 0;				// nothing processed so far
+	ui32LogIdx = HTB_SF_LAST;	// Loop terminator condition
+
+	do
+	{
+		/*
+		 * If we've drained the buffer we must RELEASE and ACQUIRE some more.
+		 */
+		if (pNext >= pLast)
+		{
+			eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream);
+			if (PVRSRV_OK != eError)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: %s FAILED '%s'", __func__,
+					"TLClientReleaseData", PVRSRVGETERRORSTRING(eError)));
+				return NULL;
+			}
+
+			eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+				g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+			if (PVRSRV_OK != eError)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'", __func__,
+					"TLClientAcquireData", PVRSRVGETERRORSTRING(eError)));
+				return NULL;
+			}
+
+			// Reset our limits - if we've returned an empty buffer we're done.
+			pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+			pStart = pSentinel->pBuf;
+			pNext = pStart;
+
+			if (pStart == NULL || pLast == NULL)
+			{
+				return NULL;
+			}
+		}
+
+		/*
+		 * We should have a header followed by data block(s) in the stream.
+		 */
+
+		pCurrent = pNext;
+		ppHdr = GET_PACKET_HDR(pCurrent);
+
+		if (ppHdr == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unexpected NULL packet in Host Trace buffer",
+			         __func__));
+			pSentinel->uiMsgLen += uiMsgSize;
+			return NULL;		// This should never happen
+		}
+
+		/*
+		 * This should *NEVER* fire. If it does it means we have got some
+		 * dubious packet header back from the HTB stream. In this case
+		 * the sensible thing is to abort processing and return to
+		 * the caller
+		 */
+		uiHdrType = GET_PACKET_TYPE(ppHdr);
+
+		PVR_ASSERT(uiHdrType < PVRSRVTL_PACKETTYPE_LAST &&
+			uiHdrType > PVRSRVTL_PACKETTYPE_UNDEF);
+
+		if (uiHdrType < PVRSRVTL_PACKETTYPE_LAST &&
+			uiHdrType > PVRSRVTL_PACKETTYPE_UNDEF)
+		{
+			/*
+			 * We have a (potentially) valid data header. We should see if
+			 * the associated packet header matches one of our expected
+			 * types.
+			 */
+			pNext = GET_NEXT_PACKET_ADDR(ppHdr);
+
+			PVR_ASSERT(pNext != NULL);
+
+			uiMsgSize = (IMG_UINT32)((size_t)pNext - (size_t)ppHdr);
+
+			pSentinel->uiMsgLen += uiMsgSize;
+
+			pData = GET_PACKET_DATA_PTR(ppHdr);
+
+			/*
+			 * Handle non-DATA packet types. These include PAD fields which
+			 * may have data associated and other types. We simply discard
+			 * these as they have no decodable information within them.
+			 */
+			if (uiHdrType != PVRSRVTL_PACKETTYPE_DATA)
+			{
+				/*
+				 * Now release the current non-data packet and proceed to the
+				 * next entry (if any).
+				 */
+				eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+				    g_sHTBData.hStream, uiMsgSize);
+
+				HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Packet Type %x "
+				                 "Length %u", __func__, uiHdrType, uiMsgSize));
+
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - '%s' message"
+						" size %u", __func__, "TLClientReleaseDataLess",
+						PVRSRVGETERRORSTRING(eError), uiMsgSize));
+				}
+
+				eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+					g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+				if (PVRSRV_OK != eError)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - %s Giving up",
+						__func__, "TLClientAcquireData",
+						PVRSRVGETERRORSTRING(eError)));
+
+					return NULL;
+				}
+				pSentinel->uiMsgLen = 0;
+				// Reset our limits - if we've returned an empty buffer we're done.
+				pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+				pStart = pSentinel->pBuf;
+				pNext = pStart;
+
+				if (pStart == NULL || pLast == NULL)
+				{
+					return NULL;
+				}
+				continue;
+			}
+			if (pData == NULL || pData >= pLast)
+			{
+				continue;
+			}
+			ui32Data = *(IMG_UINT32 *)pData;
+			ui32LogIdx = idToLogIdx(ui32Data);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Unexpected Header @%p value %x",
+				ppHdr, uiHdrType));
+
+			return NULL;
+		}
+
+		/*
+		 * Check if the unrecognized ID is valid and therefore, tracebuf
+		 * needs updating.
+		 */
+		if (HTB_SF_LAST == ui32LogIdx && HTB_LOG_VALIDID(ui32Data)
+			&& IMG_FALSE == bUnrecognizedErrorPrinted)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			    "%s: Unrecognised LOG value '%x' GID %x Params %d ID %x @ '%p'",
+			    __func__, ui32Data, HTB_SF_GID(ui32Data),
+			    HTB_SF_PARAMNUM(ui32Data), ui32Data & 0xfff, pData));
+			bUnrecognizedErrorPrinted = IMG_TRUE;
+		}
+
+	} while (HTB_SF_LAST == ui32LogIdx);
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning data @ %p Log value '%x'",
+	                 __func__, pCurrent, ui32Data));
+
+	return pCurrent;
+}
+
+/*
+ * HTB_GetFirstMessage
+ *
+ * Called from START to obtain the buffer address of the first message within
+ * pSentinel->pBuf. Will ACQUIRE data if the buffer is empty.
+ *
+ * Input:
+ *	pSentinel
+ *	pui64Pos			Offset within the debugFS file
+ *
+ * Output:
+ *	pSentinel->pCurr	Set to reference the first valid non-NULL message within
+ *						the buffer. If no valid message is found set to NULL.
+ *	pSentinel
+ *		->pBuf		if unset on entry
+ *		->uiBufLen	if pBuf unset on entry
+ *
+ * Side-effects:
+ *	HTB TL stream will be updated to bypass any zero-length PAD messages before
+ *	the first non-NULL message (if any).
+ */
+static void HTB_GetFirstMessage(HTB_Sentinel_t *pSentinel, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_ERROR	eError;
+
+	PVR_UNREFERENCED_PARAMETER(pui64Pos);
+
+	if (pSentinel == NULL)
+		return;
+
+	if (pSentinel->pBuf == NULL)
+	{
+		/* Acquire data */
+		pSentinel->uiMsgLen = 0;
+
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		    g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'",
+			    __func__, "TLClientAcquireData", PVRSRVGETERRORSTRING(eError)));
+
+			pSentinel->pBuf = NULL;
+			pSentinel->pCurr = NULL;
+		}
+		else
+		{
+			/*
+			 * If there is no data available we set pSentinel->pCurr to NULL
+			 * and return. This is expected behaviour if we've drained the
+			 * data and nothing else has yet been produced.
+			 */
+			if (pSentinel->uiBufLen == 0 || pSentinel->pBuf == NULL)
+			{
+				HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Empty Buffer @ %p",
+				                 __func__, pSentinel->pBuf));
+
+				pSentinel->pCurr = NULL;
+				return;
+			}
+		}
+	}
+
+	/* Locate next message within buffer. NULL => no more data to process */
+	pSentinel->pCurr = HTB_GetNextMessage(pSentinel);
+}
+
+/*
+ * _DebugHBTraceDIStart:
+ *
+ * Returns the address to use for subsequent 'Show', 'Next', 'Stop' file ops.
+ * Return DI_START_TOKEN for the very first call and allocate a sentinel for
+ * use by the 'Show' routine and its helpers.
+ * This is stored in the psEntry's private hook field.
+ *
+ * We obtain access to the TLstream associated with the HTB. If this doesn't
+ * exist (because no pvrdebug capture trace has been set) we simply return with
+ * a NULL value which will stop the DI traversal.
+ */
+static void *_DebugHBTraceDIStart(OSDI_IMPL_ENTRY *psEntry,
+                                  IMG_UINT64 *pui64Pos)
+{
+	HTB_Sentinel_t	*pSentinel = DIGetPrivData(psEntry);
+	PVRSRV_ERROR	eError;
+	IMG_UINT32		uiTLMode;
+	void			*retVal;
+	IMG_HANDLE		hStream;
+
+	/* The sentinel object should have been allocated during the creation
+	 * of the DI entry. If it's not there it means that something went
+	 * wrong. Return NULL in such case. */
+	if (pSentinel == NULL)
+	{
+		return NULL;
+	}
+
+	/* Check to see if the HTB stream has been configured yet. If not, there is
+	 * nothing to display so we just return NULL to stop the stream access.
+	 */
+	if (!HTBIsConfigured())
+	{
+		return NULL;
+	}
+
+	/* Open the stream in non-blocking mode so that we can determine if there
+	 * is no data to consume. Also disable the producer callback (if any) and
+	 * the open callback so that we do not generate spurious trace data when
+	 * accessing the stream.
+	 */
+	uiTLMode = PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING|
+			   PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK|
+			   PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK;
+
+	/* If two or more processes try to read from this file at the same time
+	 * the TLClientOpenStream() function will handle this by allowing only
+	 * one of them to actually open the stream. The other process will get
+	 * an error stating that the stream is already open. The open function
+	 * is threads safe. */
+	eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE, HTB_STREAM_NAME, uiTLMode,
+	                            &hStream);
+
+	if (eError == PVRSRV_ERROR_ALREADY_OPEN)
+	{
+		/* Stream allows only one reader so return error if it's already
+		 * opened. */
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Stream handle %p already "
+		                 "exists for %s", __func__, g_sHTBData.hStream,
+		                 HTB_STREAM_NAME));
+		return NULL;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		/*
+		 * No stream available so nothing to report
+		 */
+		return NULL;
+	}
+
+	/* There is a window where hStream can be NULL but the stream is already
+	 * opened. This shouldn't matter since the TLClientOpenStream() will make
+	 * sure that only one stream can be opened and only one process can reach
+	 * this place at a time. Also the .stop function will be always called
+	 * after this function returns so there should be no risk of stream
+	 * not being closed. */
+	PVR_ASSERT(g_sHTBData.hStream == NULL);
+	g_sHTBData.hStream = hStream;
+
+	/* We're starting the read operation so ensure we properly zero the
+	 * sentinel object. */
+	memset(pSentinel, 0, sizeof(*pSentinel));
+
+	/*
+	 * Find the first message location within pSentinel->pBuf
+	 * => for DI_START_TOKEN we must issue our first ACQUIRE, also for the
+	 * subsequent re-START calls (if any).
+	 */
+
+	HTB_GetFirstMessage(pSentinel, pui64Pos);
+
+	retVal = *pui64Pos == 0 ? DI_START_TOKEN : pSentinel->pCurr;
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %p, Stream %s @ %p",
+	                 __func__, retVal, HTB_STREAM_NAME, g_sHTBData.hStream));
+
+	return retVal;
+}
+
+/*
+ * _DebugTBTraceDIStop:
+ *
+ * Stop processing data collection and release any previously allocated private
+ * data structure if we have exhausted the previously filled data buffers.
+ */
+static void _DebugHBTraceDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+	IMG_UINT32 uiMsgLen;
+	PVRSRV_ERROR eError;
+
+	if (pSentinel == NULL)
+	{
+		return;
+	}
+
+	uiMsgLen = pSentinel->uiMsgLen;
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: MsgLen = %d", __func__, uiMsgLen));
+
+	/* If we get here the handle should never be NULL because
+	 * _DebugHBTraceDIStart() shouldn't allow that. */
+	if (g_sHTBData.hStream == NULL)
+	{
+		return;
+	}
+
+	if (uiMsgLen != 0)
+	{
+		eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+		                                 g_sHTBData.hStream, uiMsgLen);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - %s, nBytes %u",
+			        __func__, "TLClientReleaseDataLess",
+			        PVRSRVGETERRORSTRING(eError), uiMsgLen));
+		}
+	}
+
+	eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()",
+		        "TLClientCloseStream", PVRSRVGETERRORSTRING(eError),
+		        __func__));
+	}
+
+	g_sHTBData.hStream = NULL;
+}
+
+
+/*
+ * _DebugHBTraceDINext:
+ *
+ * This is where we release any acquired data which has been processed by the
+ * DIShow routine. If we have encountered a DI entry overflow we stop
+ * processing and return NULL. Otherwise we release the message that we
+ * previously processed and simply update our position pointer to the next
+ * valid HTB message (if any)
+ */
+static void *_DebugHBTraceDINext(OSDI_IMPL_ENTRY *psEntry, void *pvPriv,
+                                 IMG_UINT64 *pui64Pos)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+	IMG_UINT64 ui64CurPos;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+	if (pui64Pos)
+	{
+		ui64CurPos = *pui64Pos;
+		*pui64Pos = ui64CurPos + 1;
+	}
+
+	/* Determine if we've had an overflow on the previous 'Show' call. If so
+	 * we leave the previously acquired data in the queue (by releasing 0 bytes)
+	 * and return NULL to end this DI entry iteration.
+	 * If we have not overflowed we simply get the next HTB message and use that
+	 * for our display purposes. */
+
+	if (DIHasOverflowed(psEntry))
+	{
+		(void) TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream,
+		                               0);
+
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: OVERFLOW - returning NULL",
+		                 __func__));
+
+		return NULL;
+	}
+	else
+	{
+		eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+		                                 g_sHTBData.hStream,
+		                                 pSentinel->uiMsgLen);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s' @ %p Length %d",
+			        __func__, "TLClientReleaseDataLess",
+			        PVRSRVGETERRORSTRING(eError), pSentinel->pCurr,
+			        pSentinel->uiMsgLen));
+			PVR_DPF((PVR_DBG_WARNING, "%s: Buffer @ %p..%p", __func__,
+			        pSentinel->pBuf,
+			        (IMG_PBYTE) (pSentinel->pBuf + pSentinel->uiBufLen)));
+
+		}
+
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		                             g_sHTBData.hStream, &pSentinel->pBuf,
+		                             &pSentinel->uiBufLen);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'\nPrev message len %d",
+			        __func__, "TLClientAcquireData",
+			        PVRSRVGETERRORSTRING(eError), pSentinel->uiMsgLen));
+			pSentinel->pBuf = NULL;
+		}
+
+		pSentinel->uiMsgLen = 0; /* We don't (yet) know the message size */
+	}
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %p Msglen %d", __func__,
+	                 pSentinel->pBuf, pSentinel->uiMsgLen));
+
+	if (pSentinel->pBuf == NULL || pSentinel->uiBufLen == 0)
+	{
+		return NULL;
+	}
+
+	pSentinel->pCurr = HTB_GetNextMessage(pSentinel);
+
+	return pSentinel->pCurr;
+}
+
+/******************************************************************************
+ * HTB Dumping routines and definitions
+ *****************************************************************************/
+#define IS_VALID_FMT_STRING(FMT) (strchr(FMT, '%') != NULL)
+#define MAX_STRING_SIZE (128)
+
+typedef enum
+{
+	TRACEBUF_ARG_TYPE_INT,
+	TRACEBUF_ARG_TYPE_ERR,
+	TRACEBUF_ARG_TYPE_NONE
+} TRACEBUF_ARG_TYPE;
+
+/*
+ * Array of all Host Trace log IDs used to convert the tracebuf data
+ */
+typedef struct _HTB_TRACEBUF_LOG_ {
+	HTB_LOG_SFids eSFId;
+	IMG_CHAR      *pszName;
+	IMG_CHAR      *pszFmt;
+	IMG_UINT32    ui32ArgNum;
+	IMG_UINT32    ui32StrArgIdx; /* This is 1 indexed, if 0, there is not a string arg. */
+} HTB_TRACEBUF_LOG;
+
+static const HTB_TRACEBUF_LOG aLogs[] = {
+#define X(a, b, c, d, e, f) {HTB_LOG_CREATESFID(a,b,e,f), #c, d, e, f},
+	HTB_LOG_SFIDLIST
+#undef X
+};
+
+static const IMG_CHAR *aGroups[] = {
+#define X(A,B) #B,
+	HTB_LOG_SFGROUPLIST
+#undef X
+};
+static const IMG_UINT32 uiMax_aGroups = ARRAY_SIZE(aGroups) - 1;
+
+static TRACEBUF_ARG_TYPE ExtractOneArgFmt(IMG_CHAR **, IMG_CHAR *);
+/*
+ * ExtractOneArgFmt
+ *
+ * Scan the input 'printf-like' string *ppszFmt and return the next
+ * value string to be displayed. If there is no '%' format field in the
+ * string we return 'TRACEBUF_ARG_TYPE_NONE' and leave the input string
+ * untouched.
+ *
+ * Input
+ *	ppszFmt          reference to format string to be decoded
+ *	pszOneArgFmt     single field format from *ppszFmt
+ *
+ * Returns
+ *	TRACEBUF_ARG_TYPE_ERR       unrecognised argument
+ *	TRACEBUF_ARG_TYPE_INT       variable is of numeric type
+ *	TRACEBUF_ARG_TYPE_NONE      no variable reference in *ppszFmt
+ *
+ * Side-effect
+ *	*ppszFmt is updated to reference the next part of the format string
+ *	to be scanned
+ */
+static TRACEBUF_ARG_TYPE ExtractOneArgFmt(
+	IMG_CHAR **ppszFmt,
+	IMG_CHAR *pszOneArgFmt)
+{
+	IMG_CHAR          *pszFmt;
+	IMG_CHAR          *psT;
+	IMG_UINT32        ui32Count = MAX_STRING_SIZE;
+	IMG_UINT32        ui32OneArgSize;
+	TRACEBUF_ARG_TYPE eRet = TRACEBUF_ARG_TYPE_ERR;
+
+	if (NULL == ppszFmt)
+		return TRACEBUF_ARG_TYPE_ERR;
+
+	pszFmt = *ppszFmt;
+	if (NULL == pszFmt)
+		return TRACEBUF_ARG_TYPE_ERR;
+
+	/*
+	 * Find the first '%'
+	 * NOTE: we can be passed a simple string to display which will have no
+	 * parameters embedded within it. In this case we simply return
+	 * TRACEBUF_ARG_TYPE_NONE and the string contents will be the full pszFmt
+	 */
+	psT = strchr(pszFmt, '%');
+	if (psT == NULL)
+	{
+		return TRACEBUF_ARG_TYPE_NONE;
+	}
+
+	/* Find next conversion identifier after the initial '%' */
+	while ((*psT++) && (ui32Count-- > 0))
+	{
+		switch (*psT)
+		{
+			case 'd':
+			case 'i':
+			case 'o':
+			case 'u':
+			case 'x':
+			case 'X':
+			{
+				eRet = TRACEBUF_ARG_TYPE_INT;
+				goto _found_arg;
+			}
+			case 's':
+			{
+				eRet = TRACEBUF_ARG_TYPE_ERR;
+				goto _found_arg;
+			}
+		}
+	}
+
+	if ((psT == NULL) || (ui32Count == 0)) return TRACEBUF_ARG_TYPE_ERR;
+
+_found_arg:
+	ui32OneArgSize = psT - pszFmt + 1;
+	OSCachedMemCopy(pszOneArgFmt, pszFmt, ui32OneArgSize);
+	pszOneArgFmt[ui32OneArgSize] = '\0';
+
+	*ppszFmt = psT + 1;
+
+	return eRet;
+}
+
+static IMG_UINT32 idToLogIdx(IMG_UINT32 ui32CheckData)
+{
+	IMG_UINT32	i = 0;
+	for (i = 0; aLogs[i].eSFId != HTB_SF_LAST; i++)
+	{
+		if ( ui32CheckData == aLogs[i].eSFId )
+			return i;
+	}
+	/* Nothing found, return max value */
+	return HTB_SF_LAST;
+}
+
+/*
+ * DecodeHTB
+ *
+ * Decode the data buffer message located at pBuf. This should be a valid
+ * HTB message as we are provided with the start of the buffer. If empty there
+ * is no message to process. We update the uiMsgLen field with the size of the
+ * HTB message that we have processed so that it can be returned to the system
+ * on successful logging of the message to the output file.
+ *
+ *	Input
+ *		pSentinel reference to newly read data and pending completion data
+ *		          from a previous invocation [handle DI entry buffer overflow]
+ *		 -> pBuf         reference to raw data that we are to parse
+ *		 -> uiBufLen     total number of bytes of data available
+ *		 -> pCurr        start of message to decode
+ *
+ *		pvDumpDebugFile     output file
+ *		pfnDumpDebugPrintf  output generating routine
+ *
+ * Output
+ *		pSentinel
+ *		 -> uiMsgLen	length of the decoded message which will be freed to
+ *						the system on successful completion of the DI entry
+ *						update via _DebugHBTraceDINext(),
+ * Return Value
+ *		0				successful decode
+ *		-1				unsuccessful decode
+ */
+static int
+DecodeHTB(HTB_Sentinel_t *pSentinel, OSDI_IMPL_ENTRY *pvDumpDebugFile,
+          DI_PRINTF pfnDumpDebugPrintf)
+{
+	IMG_UINT32	ui32Data, ui32LogIdx, ui32ArgsCur;
+	IMG_CHAR	*pszFmt = NULL;
+	IMG_CHAR	aszOneArgFmt[MAX_STRING_SIZE];
+	IMG_BOOL	bUnrecognizedErrorPrinted = IMG_FALSE;
+
+	size_t	nPrinted;
+
+	void	*pNext, *pLast, *pStart, *pData = NULL;
+	PVRSRVTL_PPACKETHDR	ppHdr;	/* Current packet header */
+	IMG_UINT32	uiHdrType;		/* Packet header type */
+	IMG_UINT32	uiMsgSize;		/* Message size of current packet (bytes) */
+	IMG_BOOL	bPacketsDropped;
+
+	pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+	pStart = pSentinel->pCurr;
+
+	pSentinel->uiMsgLen = 0;	// Reset count for this message
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Buf @ %p..%p, Length = %d",
+	                 __func__, pStart, pLast, pSentinel->uiBufLen));
+
+	/*
+	 * We should have a DATA header with the necessary information following
+	 */
+	ppHdr = GET_PACKET_HDR(pStart);
+
+	if (ppHdr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Unexpected NULL packet in Host Trace buffer", __func__));
+		return -1;
+	}
+
+	uiHdrType = GET_PACKET_TYPE(ppHdr);
+	PVR_ASSERT(uiHdrType == PVRSRVTL_PACKETTYPE_DATA);
+
+	pNext = GET_NEXT_PACKET_ADDR(ppHdr);
+
+	PVR_ASSERT(pNext != NULL);
+
+	uiMsgSize = (IMG_UINT32)((size_t)pNext - (size_t)ppHdr);
+
+	pSentinel->uiMsgLen += uiMsgSize;
+
+	pData = GET_PACKET_DATA_PTR(ppHdr);
+
+	if (pData == NULL || pData >= pLast)
+	{
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: pData = %p, pLast = %p "
+		                 "Returning 0", __func__, pData, pLast));
+		return 0;
+	}
+
+	ui32Data = *(IMG_UINT32 *)pData;
+	ui32LogIdx = idToLogIdx(ui32Data);
+
+	/*
+	 * Check if the unrecognised ID is valid and therefore, tracebuf
+	 * needs updating.
+	 */
+	if (ui32LogIdx == HTB_SF_LAST)
+	{
+		if (HTB_LOG_VALIDID(ui32Data))
+		{
+			if (!bUnrecognizedErrorPrinted)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				    "%s: Unrecognised LOG value '%x' GID %x Params %d ID %x @ '%p'",
+				    __func__, ui32Data, HTB_SF_GID(ui32Data),
+				    HTB_SF_PARAMNUM(ui32Data), ui32Data & 0xfff, pData));
+				bUnrecognizedErrorPrinted = IMG_TRUE;
+			}
+
+			return 0;
+		}
+
+		PVR_DPF((PVR_DBG_ERROR,
+		    "%s: Unrecognised and invalid LOG value detected '%x'",
+		    __func__, ui32Data));
+
+		return -1;
+	}
+
+	/* The string format we are going to display */
+	/*
+	 * The display will show the header (log-ID, group-ID, number of params)
+	 * The maximum parameter list length = 15 (only 4bits used to encode)
+	 * so we need HEADER + 15 * sizeof(UINT32) and the displayed string
+	 * describing the event. We use a buffer in the per-process pSentinel
+	 * structure to hold the data.
+	 */
+	pszFmt = aLogs[ui32LogIdx].pszFmt;
+
+	/* add the message payload size to the running count */
+	ui32ArgsCur = HTB_SF_PARAMNUM(ui32Data);
+
+	/* Determine if we've over-filled the buffer and had to drop packets */
+	bPacketsDropped = CHECK_PACKETS_DROPPED(ppHdr);
+	if (bPacketsDropped ||
+		(uiHdrType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED))
+	{
+		/* Flag this as it is useful to know ... */
+
+		PVR_DUMPDEBUG_LOG("\n<========================== *** PACKETS DROPPED *** ======================>\n");
+	}
+
+	{
+		IMG_UINT32 ui32Timestampns, ui32PID, ui32TID;
+		IMG_UINT64 ui64Timestamp, ui64TimestampSec;
+		IMG_CHAR	*szBuffer = pSentinel->szBuffer;	// Buffer start
+		IMG_CHAR	*pszBuffer = pSentinel->szBuffer;	// Current place in buf
+		size_t		uBufBytesAvailable = sizeof(pSentinel->szBuffer);
+		IMG_UINT32	*pui32Data = (IMG_UINT32 *)pData;
+		IMG_UINT32	ui_aGroupIdx;
+
+		// Get PID field from data stream
+		pui32Data++;
+		ui32PID = *pui32Data;
+		// Get TID field from data stream
+		pui32Data++;
+		ui32TID = *pui32Data;
+		// Get Timestamp part 1 from data stream
+		pui32Data++;
+		ui64Timestamp = (IMG_UINT64) *pui32Data << 32;
+		// Get Timestamp part 2 from data stream
+		pui32Data++;
+		ui64Timestamp |= (IMG_UINT64) *pui32Data;
+		// Move to start of message contents data
+		pui32Data++;
+
+		/*
+		 * We need to snprintf the data to a local in-kernel buffer
+		 * and then PVR_DUMPDEBUG_LOG() that in one shot
+		 */
+		ui_aGroupIdx = MIN(HTB_SF_GID(ui32Data), uiMax_aGroups);
+
+		/* Divide by 1B to get seconds & mod using output var (nanosecond resolution)*/
+		ui64TimestampSec = OSDivide64r64(ui64Timestamp, 1000000000, &ui32Timestampns);
+
+		nPrinted = OSSNPrintf(szBuffer, uBufBytesAvailable, "%010"IMG_UINT64_FMTSPEC".%09u:%-5u-%-5u-%s> ",
+			ui64TimestampSec, ui32Timestampns, ui32PID, ui32TID, aGroups[ui_aGroupIdx]);
+		if (nPrinted >= uBufBytesAvailable)
+		{
+			PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+				" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+				uBufBytesAvailable);
+
+			nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+		}
+
+		PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+		/* Update where our next 'output' point in the buffer is */
+		pszBuffer += nPrinted;
+		uBufBytesAvailable -= nPrinted;
+
+		/*
+		 * Print one argument at a time as this simplifies handling variable
+		 * number of arguments. Special case handling for no arguments.
+		 * This is the case for simple format strings such as
+		 * HTB_SF_MAIN_KICK_UNCOUNTED.
+		 */
+		if (ui32ArgsCur == 0)
+		{
+			if (pszFmt)
+			{
+				const ssize_t iCopiedCnt =
+					OSStringSafeCopy(pszBuffer, pszFmt, uBufBytesAvailable);
+				if (iCopiedCnt < 0)
+				{
+					PVR_DUMPDEBUG_LOG("Buffer overrun - %zu required,"
+					                  " max space %zu\n",
+					                   OSStringLength(pszFmt),
+					                   uBufBytesAvailable);
+
+					/* Ensure we don't overflow buffer */
+					nPrinted = uBufBytesAvailable;
+				}
+				else
+				{
+					nPrinted = iCopiedCnt;
+				}
+				PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+				pszBuffer += nPrinted;
+				/* Don't update the uBufBytesAvailable as we have finished this
+				 * message decode. pszBuffer - szBuffer is the total amount of
+				 * data we have decoded.
+				 */
+			}
+		}
+		else
+		{
+			if (HTB_SF_GID(ui32Data) == HTB_GID_CTRL && HTB_SF_ID(ui32Data) == HTB_ID_MARK_SCALE)
+			{
+				IMG_UINT32 i;
+				IMG_UINT32 ui32ArgArray[HTB_MARK_SCALE_ARG_ARRAY_SIZE];
+				IMG_UINT64 ui64OSTS = 0;
+				IMG_UINT32 ui32OSTSRem = 0;
+				IMG_UINT64 ui64CRTS = 0;
+
+				/* Retrieve 6 args to an array */
+				for (i = 0; i < ARRAY_SIZE(ui32ArgArray); i++)
+				{
+					ui32ArgArray[i] = *pui32Data;
+					pui32Data++;
+					--ui32ArgsCur;
+				}
+
+				ui64OSTS = (IMG_UINT64) ui32ArgArray[HTB_ARG_OSTS_PT1] << 32 | ui32ArgArray[HTB_ARG_OSTS_PT2];
+				ui64CRTS = (IMG_UINT64) ui32ArgArray[HTB_ARG_CRTS_PT1] << 32 | ui32ArgArray[HTB_ARG_CRTS_PT2];
+
+				/* Divide by 1B to get seconds, remainder in nano seconds*/
+				ui64OSTS = OSDivide64r64(ui64OSTS, 1000000000, &ui32OSTSRem);
+
+				nPrinted = OSSNPrintf(pszBuffer,
+						              uBufBytesAvailable,
+						              "HTBFWMkSync Mark=%u OSTS=%010" IMG_UINT64_FMTSPEC ".%09u CRTS=%" IMG_UINT64_FMTSPEC " CalcClkSpd=%u\n",
+						              ui32ArgArray[HTB_ARG_SYNCMARK],
+						              ui64OSTS,
+						              ui32OSTSRem,
+						              ui64CRTS,
+						              ui32ArgArray[HTB_ARG_CLKSPD]);
+
+				if (nPrinted >= uBufBytesAvailable)
+				{
+					PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+						" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+						uBufBytesAvailable);
+					nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+				}
+
+				PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+				pszBuffer += nPrinted;
+				uBufBytesAvailable -= nPrinted;
+			}
+			else
+			{
+				while (IS_VALID_FMT_STRING(pszFmt) && (uBufBytesAvailable > 0))
+				{
+					IMG_UINT32 ui32TmpArg = *pui32Data;
+					TRACEBUF_ARG_TYPE eArgType;
+
+					eArgType = ExtractOneArgFmt(&pszFmt, aszOneArgFmt);
+
+					pui32Data++;
+					ui32ArgsCur--;
+
+					switch (eArgType)
+					{
+						case TRACEBUF_ARG_TYPE_INT:
+							nPrinted = OSSNPrintf(pszBuffer, uBufBytesAvailable,
+								aszOneArgFmt, ui32TmpArg);
+							break;
+
+						case TRACEBUF_ARG_TYPE_NONE:
+							{
+								const ssize_t iCopiedCnt =
+									OSStringSafeCopy(pszBuffer,
+									                 pszFmt,
+									                 uBufBytesAvailable);
+								if (iCopiedCnt < 0)
+								{
+									nPrinted = OSStringLength(pszFmt);
+								}
+								else
+								{
+									nPrinted = iCopiedCnt;
+								}
+							}
+							break;
+
+						default:
+							nPrinted = OSSNPrintf(pszBuffer, uBufBytesAvailable,
+								"Error processing arguments, type not "
+								"recognized (fmt: %s)", aszOneArgFmt);
+							break;
+					}
+					if (nPrinted >= uBufBytesAvailable)
+					{
+						PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+							" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+							uBufBytesAvailable);
+						nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+					}
+					PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+					pszBuffer += nPrinted;
+					uBufBytesAvailable -= nPrinted;
+				}
+				/* Display any remaining text in pszFmt string */
+				if (pszFmt)
+				{
+					const ssize_t iCopiedCnt =
+						OSStringSafeCopy(pszBuffer, pszFmt, uBufBytesAvailable);
+					if (iCopiedCnt < 0)
+					{
+						PVR_DUMPDEBUG_LOG("Buffer overrun - %zu required,"
+						                  " max space %zu\n",
+						                   OSStringLength(pszFmt),
+						                   uBufBytesAvailable);
+
+						/* Ensure we don't overflow buffer */
+						nPrinted = uBufBytesAvailable;
+					}
+					else
+					{
+						nPrinted = iCopiedCnt;
+					}
+					PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+					pszBuffer += nPrinted;
+					/* Don't update the uBufBytesAvailable as we have finished this
+					 * message decode. pszBuffer - szBuffer is the total amount of
+					 * data we have decoded.
+					 */
+				}
+			}
+		}
+
+		/* Update total bytes processed */
+		pSentinel->uiTotal += (pszBuffer - szBuffer);
+	}
+	return 0;
+}
+
+/*
+ * HTBDumpBuffer: Dump the Host Trace Buffer using the TLClient API
+ *
+ * This routine just parses *one* message from the buffer.
+ * The stream will be opened by the Start() routine, closed by the Stop() and
+ * updated for data consumed by this routine once we have DebugPrintf'd it.
+ * We use the new TLReleaseDataLess() routine which enables us to update the
+ * HTB contents with just the amount of data we have successfully processed.
+ * If we need to leave the data available we can call this with a 0 count.
+ * This will happen in the case of a buffer overflow so that we can reprocess
+ * any data which wasn't handled before.
+ *
+ * In case of overflow or an error we return -1 otherwise 0
+ *
+ * Input:
+ *  pfnPrintf           output routine to display data
+ *  psEntry             handle to debug frontend
+ *  pvData              data address to start dumping from
+ *                      (set by Start() / Next())
+ */
+static int HTBDumpBuffer(DI_PRINTF pfnPrintf, OSDI_IMPL_ENTRY *psEntry,
+                         void *pvData)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+
+	PVR_ASSERT(pvData != NULL);
+
+	if (pvData == DI_START_TOKEN)
+	{
+		if (pSentinel->pCurr == NULL)
+		{
+			HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: DI_START_TOKEN, "
+			                 "Empty buffer", __func__));
+			return 0;
+		}
+		PVR_ASSERT(pSentinel->pCurr != NULL);
+
+		/* Display a Header as we have data to process */
+		pfnPrintf(psEntry, "%-20s:%-5s-%-5s-%s  %s\n", "Timestamp", "PID", "TID", "Group>",
+		         "Log Entry");
+	}
+	else
+	{
+		if (pvData != NULL)
+		{
+			PVR_ASSERT(pSentinel->pCurr == pvData);
+		}
+	}
+
+	return DecodeHTB(pSentinel, psEntry, pfnPrintf);
+}
+
+
+/******************************************************************************
+ * External Entry Point routines ...
+ *****************************************************************************/
+/*************************************************************************/ /*!
+ @Function     HTB_CreateDIEntry
+
+ @Description  Create the debugFS entry-point for the host-trace-buffer
+
+ @Returns      eError          internal error code, PVRSRV_OK on success
+
+ */ /*************************************************************************/
+PVRSRV_ERROR HTB_CreateDIEntry_Impl(void)
+{
+	PVRSRV_ERROR eError;
+
+	DI_ITERATOR_CB sIterator = {
+		.pfnStart = _DebugHBTraceDIStart,
+		.pfnStop  = _DebugHBTraceDIStop,
+		.pfnNext  = _DebugHBTraceDINext,
+		.pfnShow  = _DebugHBTraceDIShow,
+	};
+
+	eError = DICreateEntry("host_trace", NULL, &sIterator,
+	                       &g_sHTBData.sSentinel,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &g_sHTBData.psDumpHostDiEntry);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DICreateEntry");
+
+	return PVRSRV_OK;
+}
+
+
+/*************************************************************************/ /*!
+ @Function     HTB_DestroyDIEntry
+
+ @Description  Destroy the debugFS entry-point created by earlier
+               HTB_CreateDIEntry() call.
+*/ /**************************************************************************/
+void HTB_DestroyDIEntry_Impl(void)
+{
+	if (g_sHTBData.psDumpHostDiEntry != NULL)
+	{
+		DIDestroyEntry(g_sHTBData.psDumpHostDiEntry);
+		g_sHTBData.psDumpHostDiEntry = NULL;
+	}
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.h b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.h
new file mode 100644
index 000000000000..cf18bbb2ac5d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@File           htb_debug.h
+@Title          Linux debugFS routine setup header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HTB_DEBUG_H
+#define HTB_DEBUG_H
+
+/**************************************************************************/ /*!
+ @Function     HTB_CreateDIEntry
+
+ @Description  Create the debugFS entry-point for the host-trace-buffer
+
+ @Returns      eError          internal error code, PVRSRV_OK on success
+
+ */ /**************************************************************************/
+PVRSRV_ERROR HTB_CreateDIEntry_Impl(void);
+
+/**************************************************************************/ /*!
+ @Function     HTB_DestroyFSEntry
+
+ @Description  Destroy the debugFS entry-point created by earlier
+               HTB_CreateDIEntry() call.
+*/ /**************************************************************************/
+void HTB_DestroyDIEntry_Impl(void);
+
+#if defined(PVRSRV_ENABLE_HTB)
+#define HTB_CreateDIEntry() HTB_CreateDIEntry_Impl()
+#define HTB_DestroyDIEntry() HTB_DestroyDIEntry_Impl()
+#else /* !PVRSRV_ENABLE_HTB */
+#define HTB_CreateDIEntry() PVRSRV_OK
+#define HTB_DestroyDIEntry()
+#endif /* PVRSRV_ENABLE_HTB */
+
+#endif /* HTB_DEBUG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/htbserver.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/htbserver.c
new file mode 100644
index 000000000000..9ed3283f52b2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/htbserver.c
@@ -0,0 +1,998 @@
+/*************************************************************************/ /*!
+@File           htbserver.c
+@Title          Host Trace Buffer server implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "htbserver.h"
+#include "htbuffer_types.h"
+#include "tlstream.h"
+#include "pvrsrv_tlcommon.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+
+/* size of circular buffer controlling the maximum number of concurrent PIDs logged */
+#define HTB_MAX_NUM_PID 8
+
+/* number of times to try rewriting a log entry */
+#define HTB_LOG_RETRY_COUNT 5
+
+#if defined(__linux__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+/*************************************************************************/ /*!
+  Host Trace Buffer control information structure
+*/ /**************************************************************************/
+typedef struct
+{
+	IMG_UINT32 ui32BufferSize;      /*!< Requested buffer size in bytes
+                                         Once set this may not be changed */
+
+	HTB_OPMODE_CTRL eOpMode;        /*!< Control what trace data is dropped if
+                                         the buffer is full.
+                                         Once set this may not be changed */
+
+/*	IMG_UINT32 ui32GroupEnable; */  /*!< Flags word controlling groups to be
+                                         logged */
+
+	IMG_UINT32 ui32LogLevel;        /*!< Log level to control messages logged */
+
+	IMG_UINT32 aui32EnablePID[HTB_MAX_NUM_PID]; /*!< PIDs to enable logging for
+                                                     a specific set of processes */
+
+	IMG_UINT32 ui32PIDCount;        /*!< Current number of PIDs being logged */
+
+	IMG_UINT32 ui32PIDHead;         /*!< Head of the PID circular buffer */
+
+	HTB_LOGMODE_CTRL eLogMode;      /*!< Logging mode control */
+
+	IMG_BOOL bLogDropSignalled;     /*!< Flag indicating if a log message has
+                                         been signalled as dropped */
+
+	/* synchronisation parameters */
+	IMG_UINT64 ui64SyncOSTS;
+	IMG_UINT64 ui64SyncCRTS;
+	IMG_UINT32 ui32SyncCalcClkSpd;
+	IMG_UINT32 ui32SyncMarker;
+
+	IMG_BOOL bInitDone;             /* Set by HTBInit, reset by HTBDeInit */
+
+	POS_SPINLOCK hRepeatMarkerLock;     /*!< Spinlock used in HTBLogKM to protect global variables
+	                                     (ByteCount, OSTS, CRTS ClkSpeed)
+	                                     from becoming inconsistent due to calls from
+	                                     both KM and UM */
+
+	IMG_UINT32 ui32ByteCount; /* Byte count used for triggering repeat sync point */
+	/* static variables containing details of previous sync point */
+	IMG_UINT64 ui64OSTS;
+	IMG_UINT64 ui64CRTS;
+	IMG_UINT32 ui32ClkSpeed;
+
+} HTB_CTRL_INFO;
+
+
+/*************************************************************************/ /*!
+*/ /**************************************************************************/
+static const IMG_UINT32 MapFlags[] =
+{
+	0,                    /* HTB_OPMODE_UNDEF = 0 */
+	TL_OPMODE_DROP_NEWER, /* HTB_OPMODE_DROPLATEST */
+	TL_OPMODE_DROP_OLDEST,/* HTB_OPMODE_DROPOLDEST */
+	TL_OPMODE_BLOCK       /* HTB_OPMODE_BLOCK */
+};
+
+static_assert(0 == HTB_OPMODE_UNDEF,      "Unexpected value for HTB_OPMODE_UNDEF");
+static_assert(1 == HTB_OPMODE_DROPLATEST, "Unexpected value for HTB_OPMODE_DROPLATEST");
+static_assert(2 == HTB_OPMODE_DROPOLDEST, "Unexpected value for HTB_OPMODE_DROPOLDEST");
+static_assert(3 == HTB_OPMODE_BLOCK,      "Unexpected value for HTB_OPMODE_BLOCK");
+
+static_assert(1 == TL_OPMODE_DROP_NEWER,  "Unexpected value for TL_OPMODE_DROP_NEWER");
+static_assert(2 == TL_OPMODE_DROP_OLDEST, "Unexpected value for TL_OPMODE_DROP_OLDEST");
+static_assert(3 == TL_OPMODE_BLOCK,       "Unexpected value for TL_OPMODE_BLOCK");
+
+static const IMG_UINT32 g_ui32TLBaseFlags; //TL_FLAG_NO_SIGNAL_ON_COMMIT
+
+/* Minimum TL buffer size.
+ * Large enough for around 60 worst case messages or 200 average messages
+ */
+#define HTB_TL_BUFFER_SIZE_MIN	(0x10000)
+
+/* Minimum concentration of HTB packets in a TL Stream is 60%
+ * If we just put the HTB header in the TL stream (12 bytes), the TL overhead
+ * is 8 bytes for its own header, so for the smallest possible (and most
+ * inefficient) packet we have 3/5 of the buffer used for actual HTB data.
+ * This shift is used as a guaranteed estimation on when to produce a repeat
+ * packet. By shifting the size of the buffer by 1 we effectively /2 this
+ * under the 60% boundary chance we may have overwritten the marker and thus
+ * guaranteed to always have a marker in the stream */
+#define HTB_MARKER_PREDICTION_THRESHOLD(val) (val >> 1)
+
+static HTB_CTRL_INFO g_sCtrl;
+static IMG_BOOL g_bConfigured = IMG_FALSE;
+static IMG_HANDLE g_hTLStream;
+
+static IMG_HANDLE hHtbDbgReqNotify;
+
+/************************************************************************/ /*!
+ @Function      _LookupFlags
+ @Description   Convert HTBuffer Operation mode to TLStream flags
+
+ @Input         eModeHTBuffer   Operation Mode
+
+ @Return        IMG_UINT32      TLStream FLags
+*/ /**************************************************************************/
+static IMG_UINT32
+_LookupFlags( HTB_OPMODE_CTRL eMode )
+{
+	return (eMode < ARRAY_SIZE(MapFlags)) ? MapFlags[eMode] : 0;
+}
+
+
+/************************************************************************/ /*!
+ @Function      _HTBLogDebugInfo
+ @Description   Debug dump handler used to dump the state of the HTB module.
+                Called for each verbosity level during a debug dump. Function
+                only prints state when called for High verbosity.
+
+ @Input         hDebugRequestHandle See PFN_DBGREQ_NOTIFY
+
+ @Input         ui32VerbLevel       See PFN_DBGREQ_NOTIFY
+
+ @Input         pfnDumpDebugPrintf  See PFN_DBGREQ_NOTIFY
+
+ @Input         pvDumpDebugFile     See PFN_DBGREQ_NOTIFY
+
+*/ /**************************************************************************/
+static void _HTBLogDebugInfo(
+		PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+		IMG_UINT32 ui32VerbLevel,
+		DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+		void *pvDumpDebugFile
+)
+{
+	PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+
+		if (g_bConfigured)
+		{
+			IMG_INT i;
+
+			PVR_DUMPDEBUG_LOG("------[ HTB Log state: On ]------");
+
+			PVR_DUMPDEBUG_LOG("HTB Log mode: %d", g_sCtrl.eLogMode);
+			PVR_DUMPDEBUG_LOG("HTB Log level: %d", g_sCtrl.ui32LogLevel);
+			PVR_DUMPDEBUG_LOG("HTB Buffer Opmode: %d", g_sCtrl.eOpMode);
+
+			for (i=0; i < HTB_FLAG_NUM_EL; i++)
+			{
+				PVR_DUMPDEBUG_LOG("HTB Log group %d: %x", i, g_auiHTBGroupEnable[i]);
+			}
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("------[ HTB Log state: Off ]------");
+		}
+	}
+}
+
+static IMG_UINT32 g_ui32HTBufferSize = HTB_TL_BUFFER_SIZE_MIN;
+
+/*
+ * AppHint access routine forward definitions
+ */
+static PVRSRV_ERROR _HTBSetLogGroup(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32);
+static PVRSRV_ERROR _HTBReadLogGroup(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32 *);
+
+static PVRSRV_ERROR	_HTBSetOpMode(const PVRSRV_DEVICE_NODE *, const void *,
+                                   IMG_UINT32);
+static PVRSRV_ERROR _HTBReadOpMode(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32 *);
+
+static void _OnTLReaderOpenCallback(void *);
+
+/************************************************************************/ /*!
+ @Function      HTBInit
+ @Description   Allocate and initialise the Host Trace Buffer
+                The buffer size may be changed by specifying
+                HTBufferSizeInKB=xxxx
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBInit_Impl(void)
+{
+	void			*pvAppHintState = NULL;
+	IMG_UINT32		ui32AppHintDefault;
+	IMG_UINT32		ui32BufBytes;
+	PVRSRV_ERROR	eError;
+
+	if (g_sCtrl.bInitDone)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "HTBInit: Driver already initialised"));
+		return PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+
+	/*
+	 * Buffer Size can be configured by specifying a value in the AppHint
+	 * This will only take effect at module load time so there is no query
+	 * or setting mechanism available.
+	 */
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HTBufferSizeInKB,
+										NULL,
+										NULL,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableHTBLogGroup,
+	                                    _HTBReadLogGroup,
+	                                    _HTBSetLogGroup,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HTBOperationMode,
+	                                    _HTBReadOpMode,
+	                                    _HTBSetOpMode,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+
+	/*
+	 * Now get whatever values have been configured for our AppHints
+	 */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = HTB_TL_BUFFER_SIZE_MIN / 1024;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+						 &ui32AppHintDefault, &g_ui32HTBufferSize);
+	OSFreeAppHintState(pvAppHintState);
+
+	ui32BufBytes = g_ui32HTBufferSize * 1024;
+
+	/* initialise rest of state */
+	g_sCtrl.ui32BufferSize =
+		(ui32BufBytes < HTB_TL_BUFFER_SIZE_MIN)
+		? HTB_TL_BUFFER_SIZE_MIN
+		: ui32BufBytes;
+	g_sCtrl.eOpMode = HTB_OPMODE_DROPOLDEST;
+	g_sCtrl.ui32LogLevel = 0;
+	g_sCtrl.ui32PIDCount = 0;
+	g_sCtrl.ui32PIDHead = 0;
+	g_sCtrl.eLogMode = HTB_LOGMODE_ALLPID;
+	g_sCtrl.bLogDropSignalled = IMG_FALSE;
+
+	eError = OSSpinLockCreate(&g_sCtrl.hRepeatMarkerLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSSpinLockCreate");
+
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&hHtbDbgReqNotify,
+			 _HTBLogDebugInfo, DEBUG_REQUEST_HTB, NULL);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify");
+
+	g_sCtrl.bInitDone = IMG_TRUE;
+
+	/* Log the current driver parameter setting for the HTBufferSizeInKB.
+	 * We do this here as there is no other infrastructure for obtaining
+	 * the value.
+	 */
+	if (g_ui32HTBufferSize != ui32AppHintDefault)
+	{
+		PVR_LOG(("Increasing HTBufferSize to %uKB", g_ui32HTBufferSize));
+	}
+
+
+	return PVRSRV_OK;
+}
+
+/************************************************************************/ /*!
+ @Function      HTBDeInit
+ @Description   Close the Host Trace Buffer and free all resources. Must
+                perform a no-op if already de-initialised.
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBDeInit_Impl( void )
+{
+	if (!g_sCtrl.bInitDone)
+		return PVRSRV_OK;
+
+	if (hHtbDbgReqNotify)
+	{
+		/* Not much we can do if it fails, driver unloading */
+		(void)PVRSRVUnregisterDriverDbgRequestNotify(hHtbDbgReqNotify);
+		hHtbDbgReqNotify = NULL;
+	}
+
+	if (g_hTLStream)
+	{
+		TLStreamClose( g_hTLStream );
+		g_hTLStream = NULL;
+	}
+
+	if (g_sCtrl.hRepeatMarkerLock != NULL)
+	{
+		OSSpinLockDestroy(g_sCtrl.hRepeatMarkerLock);
+		g_sCtrl.hRepeatMarkerLock = NULL;
+	}
+
+	g_sCtrl.bInitDone = IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+
+/*************************************************************************/ /*!
+ AppHint interface functions
+*/ /**************************************************************************/
+static
+PVRSRV_ERROR _HTBSetLogGroup(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const void *psPrivate,
+                             IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return HTBControlKM(1, &ui32Value, 0, 0,
+	                    HTB_LOGMODE_UNDEF, HTB_OPMODE_UNDEF);
+}
+
+static
+PVRSRV_ERROR _HTBReadLogGroup(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 *pui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value = g_auiHTBGroupEnable[0];
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR _HTBSetOpMode(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                           const void *psPrivate,
+                           IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return HTBControlKM(0, NULL, 0, 0, HTB_LOGMODE_UNDEF, ui32Value);
+}
+
+static
+PVRSRV_ERROR _HTBReadOpMode(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                            const void *psPrivate,
+                            IMG_UINT32 *pui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value = (IMG_UINT32)g_sCtrl.eOpMode;
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_HTB)
+static IMG_BOOL
+_ValidPID( IMG_UINT32 PID )
+{
+	IMG_UINT32 i;
+	for (i = 0; i < g_sCtrl.ui32PIDCount; i++)
+	{
+		if ( g_sCtrl.aui32EnablePID[i] == PID )
+		{
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+#endif	/* PVRSRV_ENABLE_HTB */
+
+/*************************************************************************/ /*!
+ @Function      HTBLogKM
+ @Description   Record a Host Trace Buffer log event
+
+ @Input         PID             The PID of the process the event is associated
+                                with. This is provided as an argument rather
+                                than querying internally so that events associated
+                                with a particular process, but performed by
+                                another can be logged correctly.
+
+ @Input         ui64TimeStamp   The timestamp to be associated with this log event
+
+ @Input         SF              The log event ID
+
+ @Input         ...             Log parameters
+
+ @Return        PVRSRV_OK       Success.
+
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+HTBLogKM(IMG_UINT32 PID,
+		IMG_UINT32 TID,
+		IMG_UINT64 ui64TimeStamp,
+		HTB_LOG_SFids SF,
+		va_list args
+)
+{
+#if defined(PVRSRV_ENABLE_HTB)
+
+	OS_SPINLOCK_FLAGS uiSpinLockFlags;
+	IMG_UINT32 ui32ReturnFlags = 0;
+	IMG_UINT32 ui32CurrentArg = 0;
+
+	/* Local snapshot variables of global counters */
+	IMG_UINT64 ui64OSTSSnap;
+	IMG_UINT64 ui64CRTSSnap;
+	IMG_UINT32 ui32ClkSpeedSnap;
+
+	/* format of messages is: SF:PID:TID:TIMEPT1:TIMEPT2:[PARn]*
+	 * Buffer is on the stack so we don't need a semaphore to guard it
+	 */
+	IMG_UINT32 aui32MessageBuffer[HTB_LOG_HEADER_SIZE+HTB_LOG_MAX_PARAMS];
+	IMG_UINT32 aui32Args[HTB_LOG_MAX_PARAMS + 1] = {0};
+
+	/* Min HTB size is HTB_TL_BUFFER_SIZE_MIN : 10000 bytes and Max message/
+	 * packet size is 4*(HTB_LOG_HEADER_SIZE+HTB_LOG_MAX_PARAMS) = 80 bytes,
+	 * hence with these constraints this design is unlikely to get
+	 * PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED error
+	 */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_ENABLED;
+	IMG_UINT32 ui32RetryCount = HTB_LOG_RETRY_COUNT;
+	IMG_UINT32 * pui32Message = aui32MessageBuffer;
+	IMG_UINT32 ui32NumArgs = HTB_SF_PARAMNUM(SF);
+	IMG_UINT32 ui32StrArg = HTB_SF_STRNUM(SF);
+	IMG_UINT32 ui32CurrentStringArg;
+
+	IMG_UINT32 ui32MessageSize = 4 * (HTB_LOG_HEADER_SIZE+ui32NumArgs);
+
+	if (ui32StrArg > 0)
+	{
+		ui32MessageSize += HTB_LOG_STR_ARG_SIZE - sizeof(IMG_UINT32);
+	}
+
+	PVR_ASSERT(ui32NumArgs <= HTB_LOG_MAX_PARAMS);
+	ui32NumArgs = (ui32NumArgs>HTB_LOG_MAX_PARAMS) ?
+			HTB_LOG_MAX_PARAMS : ui32NumArgs;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32NumArgs == HTB_SF_PARAMNUM(SF), eError, ReturnError);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32NumArgs <= HTB_LOG_MAX_PARAMS, eError, ReturnError);
+
+	/* Needs to be set up here because it's accessed from both `if` blocks below
+	 * and it needs to be pre-populated for both of them (pui32Message case and
+	 * HTB_SF_CTRL_FWSYNC_MARK_SCALE case). */
+
+	while (ui32CurrentArg < ui32NumArgs)
+	{
+		if (ui32StrArg != 0 && ui32CurrentArg == ui32StrArg - 1)
+		{
+			IMG_CHAR* strArg;
+			strArg = va_arg(args, IMG_CHAR*);
+
+			/* if a string is present, it will need more than one UINT32 words. */
+			ui32NumArgs += HTB_LOG_STR_ARG_NUM_WORDS - 1;
+
+			/* Ignore if filename is not supported. */
+			if (strcmp(strArg, "n/a") == 0)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "N/A\n"));
+				return PVRSRV_OK;
+			}
+
+			/* looping through the created string to get the substrings to encode. */
+			for (ui32CurrentStringArg = 0; ui32CurrentStringArg < HTB_LOG_STR_ARG_NUM_WORDS; ui32CurrentStringArg++)
+			{
+				IMG_UINT32 encodedString = 0;
+
+				if (*strArg == '\0')
+				{
+					aui32Args[ui32CurrentArg] = encodedString;
+					ui32CurrentArg++;
+				}
+				else
+				{
+					IMG_UINT32 currentSubstring;
+
+					for (currentSubstring = 0; currentSubstring < sizeof(IMG_UINT32); currentSubstring++)
+					{
+						IMG_UINT32 bitPos = currentSubstring * 8;
+
+						if (*strArg == '\0')
+						{
+							break;
+						}
+						else
+						{
+							encodedString |= (IMG_UINT32) *strArg << bitPos;
+							strArg++;
+						}
+					}
+
+					aui32Args[ui32CurrentArg] = encodedString;
+					ui32CurrentArg++;
+				}
+			}
+		}
+		else
+		{
+			aui32Args[ui32CurrentArg] = va_arg(args, IMG_UINT32);
+			ui32CurrentArg++;
+		}
+	}
+
+	if ( g_hTLStream
+			&& ( 0 == PID || ~0 == PID || HTB_LOGMODE_ALLPID == g_sCtrl.eLogMode || _ValidPID(PID) )
+/*			&& ( g_sCtrl.ui32GroupEnable & (0x1 << HTB_SF_GID(SF)) ) */
+/*			&& ( g_sCtrl.ui32LogLevel >= HTB_SF_LVL(SF) ) */
+			)
+	{
+		*pui32Message++ = SF;
+		*pui32Message++ = PID;
+		*pui32Message++ = TID;
+		*pui32Message++ = ((IMG_UINT32)((ui64TimeStamp>>32)&0xffffffff));
+		*pui32Message++ = ((IMG_UINT32)(ui64TimeStamp&0xffffffff));
+		for (ui32CurrentArg = 0; ui32CurrentArg < ui32NumArgs; ui32CurrentArg++)
+		{
+			pui32Message[ui32CurrentArg] = aui32Args[ui32CurrentArg];
+		}
+
+		eError = TLStreamWriteRetFlags( g_hTLStream, (IMG_UINT8*)aui32MessageBuffer, ui32MessageSize, &ui32ReturnFlags );
+
+		while ( PVRSRV_ERROR_NOT_READY == eError && ui32RetryCount-- )
+		{
+			OSReleaseThreadQuanta();
+			eError = TLStreamWriteRetFlags( g_hTLStream, (IMG_UINT8*)aui32MessageBuffer, ui32MessageSize, &ui32ReturnFlags );
+		}
+
+		if ( PVRSRV_OK == eError )
+		{
+			g_sCtrl.bLogDropSignalled = IMG_FALSE;
+		}
+		else if ( PVRSRV_ERROR_STREAM_FULL != eError || !g_sCtrl.bLogDropSignalled )
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()", "TLStreamWrite", PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		if ( PVRSRV_ERROR_STREAM_FULL == eError )
+		{
+			g_sCtrl.bLogDropSignalled = IMG_TRUE;
+		}
+
+	}
+
+	if (SF == HTB_SF_CTRL_FWSYNC_MARK_SCALE)
+	{
+		OSSpinLockAcquire(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+
+		/* If a marker is being placed reset byte count from last marker */
+		g_sCtrl.ui32ByteCount = 0;
+		g_sCtrl.ui64OSTS = (IMG_UINT64)aui32Args[HTB_ARG_OSTS_PT1] << 32 | aui32Args[HTB_ARG_OSTS_PT2];
+		g_sCtrl.ui64CRTS = (IMG_UINT64)aui32Args[HTB_ARG_CRTS_PT1] << 32 | aui32Args[HTB_ARG_CRTS_PT2];
+		g_sCtrl.ui32ClkSpeed = aui32Args[HTB_ARG_CLKSPD];
+
+		OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+	}
+	else
+	{
+		OSSpinLockAcquire(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+		/* Increase global count */
+		g_sCtrl.ui32ByteCount += ui32MessageSize;
+
+		/* Check if packet has overwritten last marker/rpt &&
+		   If the packet count is over half the size of the buffer */
+		if (ui32ReturnFlags & TL_FLAG_OVERWRITE_DETECTED &&
+				 g_sCtrl.ui32ByteCount > HTB_MARKER_PREDICTION_THRESHOLD(g_sCtrl.ui32BufferSize))
+		{
+			/* Take snapshot of global variables */
+			ui64OSTSSnap = g_sCtrl.ui64OSTS;
+			ui64CRTSSnap = g_sCtrl.ui64CRTS;
+			ui32ClkSpeedSnap = g_sCtrl.ui32ClkSpeed;
+			/* Reset global variable counter */
+			g_sCtrl.ui32ByteCount = 0;
+			OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+
+			/* Produce a repeat marker */
+			HTBSyncPartitionMarkerRepeat(g_sCtrl.ui32SyncMarker, ui64OSTSSnap, ui64CRTSSnap, ui32ClkSpeedSnap);
+		}
+		else
+		{
+			OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+		}
+	}
+
+ReturnError:
+	return eError;
+
+#else
+	/* HTB support is disabled. Just return PVRSRV_OK and do nothing. */
+	PVR_UNREFERENCED_PARAMETER(PID);
+	PVR_UNREFERENCED_PARAMETER(TID);
+	PVR_UNREFERENCED_PARAMETER(ui64TimeStamp);
+	PVR_UNREFERENCED_PARAMETER(SF);
+	PVR_UNREFERENCED_PARAMETER(args);
+	return PVRSRV_OK;
+#endif
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBLog
+ @Description   Record a Host Trace Buffer log event
+ @Input         PID     The PID of the process the event is associated
+                        with. This is provided as an argument rather
+                        than querying internally so that events
+                        associated with a particular process, but
+                        performed by another can be logged correctly.
+ @Input         ui64TimeStamp   The timestamp to be associated with this
+                                log event
+ @Input         SF              The log event ID
+ @Input         ...             Log parameters
+ @Return        PVRSRV_OK       Success.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+HTBLog(IMG_UINT32 PID, IMG_UINT32 TID, IMG_UINT64 ui64TimeStamp,
+		IMG_UINT32 SF, ...)
+{
+	PVRSRV_ERROR eError;
+
+	va_list args;
+	va_start(args, SF);
+	eError = HTBLogKM(PID, TID, ui64TimeStamp, SF, args);
+	va_end(args);
+	return eError;
+}
+
+static void
+_OnTLReaderOpenCallback( void *pvArg )
+{
+	if ( g_hTLStream )
+	{
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+		(void) HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+		              g_sCtrl.ui32SyncMarker,
+		              ((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)),
+		              ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+		              ((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)),
+		              ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+		              g_sCtrl.ui32SyncCalcClkSpd);
+	}
+
+	PVR_UNREFERENCED_PARAMETER(pvArg);
+}
+
+
+/*************************************************************************/ /*!
+ @Function      HTBControlKM
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control the behaviour of the data buffer
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBControlKM_Impl(
+	const IMG_UINT32 ui32NumFlagGroups,
+	const IMG_UINT32 * aui32GroupEnable,
+	const IMG_UINT32 ui32LogLevel,
+	const IMG_UINT32 ui32EnablePID,
+	const HTB_LOGMODE_CTRL eLogMode,
+	const HTB_OPMODE_CTRL eOpMode
+)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32RetryCount = HTB_LOG_RETRY_COUNT;
+	IMG_UINT32 i;
+	IMG_UINT64 ui64Time;
+	OSClockMonotonicns64(&ui64Time);
+
+	if ( !g_bConfigured && ui32NumFlagGroups )
+	{
+		eError = TLStreamCreate(
+				&g_hTLStream,
+				HTB_STREAM_NAME,
+				g_sCtrl.ui32BufferSize,
+				_LookupFlags(HTB_OPMODE_DROPOLDEST) | g_ui32TLBaseFlags,
+				_OnTLReaderOpenCallback, NULL, NULL, NULL, NULL, NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamCreate");
+		g_bConfigured = IMG_TRUE;
+	}
+
+	if (HTB_OPMODE_UNDEF != eOpMode && g_sCtrl.eOpMode != eOpMode)
+	{
+		g_sCtrl.eOpMode = eOpMode;
+		eError = TLStreamReconfigure(g_hTLStream, _LookupFlags(g_sCtrl.eOpMode | g_ui32TLBaseFlags));
+		while ( PVRSRV_ERROR_NOT_READY == eError && ui32RetryCount-- )
+		{
+			OSReleaseThreadQuanta();
+			eError = TLStreamReconfigure(g_hTLStream, _LookupFlags(g_sCtrl.eOpMode | g_ui32TLBaseFlags));
+		}
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamReconfigure");
+	}
+
+	if ( ui32EnablePID )
+	{
+		g_sCtrl.aui32EnablePID[g_sCtrl.ui32PIDHead] = ui32EnablePID;
+		g_sCtrl.ui32PIDHead++;
+		g_sCtrl.ui32PIDHead %= HTB_MAX_NUM_PID;
+		g_sCtrl.ui32PIDCount++;
+		if ( g_sCtrl.ui32PIDCount > HTB_MAX_NUM_PID )
+		{
+			g_sCtrl.ui32PIDCount = HTB_MAX_NUM_PID;
+		}
+	}
+
+	/* HTB_LOGMODE_ALLPID overrides ui32EnablePID */
+	if ( HTB_LOGMODE_ALLPID == eLogMode )
+	{
+		OSCachedMemSet(g_sCtrl.aui32EnablePID, 0, sizeof(g_sCtrl.aui32EnablePID));
+		g_sCtrl.ui32PIDCount = 0;
+		g_sCtrl.ui32PIDHead = 0;
+	}
+	if ( HTB_LOGMODE_UNDEF != eLogMode )
+	{
+		g_sCtrl.eLogMode = eLogMode;
+	}
+
+	if ( ui32NumFlagGroups )
+	{
+		for (i = 0; i < HTB_FLAG_NUM_EL && i < ui32NumFlagGroups; i++)
+		{
+			g_auiHTBGroupEnable[i] = aui32GroupEnable[i];
+		}
+		for (; i < HTB_FLAG_NUM_EL; i++)
+		{
+			g_auiHTBGroupEnable[i] = 0;
+		}
+	}
+
+	if ( ui32LogLevel )
+	{
+		g_sCtrl.ui32LogLevel = ui32LogLevel;
+	}
+
+	/* Dump the current configuration state */
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_OPMODE, g_sCtrl.eOpMode);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_ENABLE_GROUP, g_auiHTBGroupEnable[0]);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_LOG_LEVEL, g_sCtrl.ui32LogLevel);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_LOGMODE, g_sCtrl.eLogMode);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	for (i = 0; i < g_sCtrl.ui32PIDCount; i++)
+	{
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_ENABLE_PID, g_sCtrl.aui32EnablePID[i]);
+		PVR_LOG_IF_ERROR(eError, "HTBLog");
+	}
+	/* Else should never be hit as we set the spd when the power state is updated */
+	if (0 != g_sCtrl.ui32SyncMarker && 0 != g_sCtrl.ui32SyncCalcClkSpd)
+	{
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+				g_sCtrl.ui32SyncMarker,
+				((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+				((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+				g_sCtrl.ui32SyncCalcClkSpd);
+		PVR_LOG_IF_ERROR(eError, "HTBLog");
+	}
+
+	return eError;
+}
+
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarker
+ @Description   Write an HTB sync partition marker to the HTB log
+
+ @Input         ui33Marker      Marker value
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarker_Impl(
+	const IMG_UINT32 ui32Marker
+)
+{
+	g_sCtrl.ui32SyncMarker = ui32Marker;
+	if ( g_hTLStream )
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+
+		/* Else should never be hit as we set the spd when the power state is updated */
+		if (0 != g_sCtrl.ui32SyncCalcClkSpd)
+		{
+			eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+					ui32Marker,
+					((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+					((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+					g_sCtrl.ui32SyncCalcClkSpd);
+			PVR_WARN_IF_ERROR(eError, "HTBLog");
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarkerRepeat
+ @Description   Write a HTB sync partition marker to the HTB log, given
+                the previous values to repeat.
+
+ @Input         ui33Marker      Marker value
+ @Input         ui64SyncOSTS    previous OSTS
+ @Input         ui64SyncCRTS    previous CRTS
+ @Input         ui32ClkSpeed    previous Clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarkerRepeat_Impl(
+	const IMG_UINT32 ui32Marker,
+	const IMG_UINT64 ui64SyncOSTS,
+	const IMG_UINT64 ui64SyncCRTS,
+	const IMG_UINT32 ui32ClkSpeed
+)
+{
+	if ( g_hTLStream )
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+
+		/* Else should never be hit as we set the spd when the power state is updated */
+		if (0 != ui32ClkSpeed)
+		{
+			eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+					ui32Marker,
+					((IMG_UINT32)((ui64SyncOSTS>>32)&0xffffffffU)), ((IMG_UINT32)(ui64SyncOSTS&0xffffffffU)),
+					((IMG_UINT32)((ui64SyncCRTS>>32)&0xffffffffU)), ((IMG_UINT32)(ui64SyncCRTS&0xffffffffU)),
+					ui32ClkSpeed);
+			PVR_WARN_IF_ERROR(eError, "HTBLog");
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncScale
+ @Description   Write FW-Host synchronisation data to the HTB log when clocks
+                change or are re-calibrated
+
+ @Input         bLogValues      IMG_TRUE if value should be immediately written
+                                out to the log
+
+ @Input         ui32OSTS        OS Timestamp
+
+ @Input         ui32CRTS        Rogue timestamp
+
+ @Input         ui32CalcClkSpd  Calculated clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncScale_Impl(
+	const IMG_BOOL bLogValues,
+	const IMG_UINT64 ui64OSTS,
+	const IMG_UINT64 ui64CRTS,
+	const IMG_UINT32 ui32CalcClkSpd
+)
+{
+	g_sCtrl.ui64SyncOSTS = ui64OSTS;
+	g_sCtrl.ui64SyncCRTS = ui64CRTS;
+	g_sCtrl.ui32SyncCalcClkSpd = ui32CalcClkSpd;
+	if (g_hTLStream && bLogValues)
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+				g_sCtrl.ui32SyncMarker,
+				((IMG_UINT32)((ui64OSTS>>32)&0xffffffff)), ((IMG_UINT32)(ui64OSTS&0xffffffff)),
+				((IMG_UINT32)((ui64CRTS>>32)&0xffffffff)), ((IMG_UINT32)(ui64CRTS&0xffffffff)),
+				ui32CalcClkSpd);
+		/*
+		 * Don't spam the log with non-failure cases
+		 */
+		PVR_WARN_IF_ERROR(eError, "HTBLog");
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBLogSimple
+ @Description   Record a Host Trace Buffer log event with implicit PID and
+                Timestamp
+ @Input         SF              The log event ID
+ @Input         ...             Log parameters
+ @Return        PVRSRV_OK       Success.
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBLogSimple_Impl(IMG_UINT32 SF, ...)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64TimeStamp;
+	va_list args;
+	va_start(args, SF);
+	OSClockMonotonicns64(&ui64TimeStamp);
+	eError = HTBLogKM(OSGetCurrentProcessID(), OSGetCurrentThreadID(), ui64TimeStamp,
+			SF, args);
+	va_end(args);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBIsConfigured
+ @Description   Determine if HTB stream has been configured
+
+ @Input         none
+
+ @Return        IMG_FALSE       Stream has not been configured
+                IMG_TRUE        Stream has been configured
+
+*/ /**************************************************************************/
+IMG_BOOL
+HTBIsConfigured(void)
+{
+	return g_bConfigured;
+}
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/info_page_km.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/info_page_km.c
new file mode 100644
index 000000000000..79b03fe7e658
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/info_page_km.c
@@ -0,0 +1,142 @@
+/*************************************************************************/ /*!
+@File           info_page_km.c
+@Title          Kernel/User space shared memory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements general purpose shared memory between kernel driver
+                and user mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "info_page_defs.h"
+#include "info_page.h"
+#include "pvrsrv.h"
+#include "devicemem.h"
+#include "pmr.h"
+
+PVRSRV_ERROR InfoPageCreate(PVRSRV_DATA *psData)
+{
+    const PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+                                              PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+                                              PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+                                              PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL);
+    PVRSRV_ERROR eError;
+
+    PVR_ASSERT(psData != NULL);
+
+    /* Allocate single page of memory for driver information page */
+    eError = DevmemAllocateExportable(psData->psHostMemDeviceNode,
+                                      PVR_ALIGN(INFO_PAGE_TOTAL_SIZE, OSGetPageSize()),
+                                      OSGetPageSize(),
+                                      OSGetPageShift(),
+                                      uiMemFlags,
+                                      "PVRSRVInfoPage",
+                                      &psData->psInfoPageMemDesc);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+    eError =  DevmemAcquireCpuVirtAddr(psData->psInfoPageMemDesc,
+                                       (void **) &psData->pui32InfoPage);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+    /* Look-up the memory descriptor PMR handle */
+    eError = DevmemLocalGetImportHandle(psData->psInfoPageMemDesc,
+                                        (void **) &psData->psInfoPagePMR);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemLocalGetImportHandle", e0);
+
+    eError = OSLockCreate(&psData->hInfoPageLock);
+    PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+    /* Because the memory is allocated read only we need to explicitly set it to
+     * 0. The reason for this is that if the memory is allocated with
+     * ZERO_ON_ALLOC a WRITEABLE attribute is implicitly added to the flags (see
+     * DevmemValidateParams()). */
+    OSCachedMemSet(psData->pui32InfoPage, 0, OSGetPageSize());
+
+    return PVRSRV_OK;
+
+e0:
+    InfoPageDestroy(psData);
+    return eError;
+}
+
+void InfoPageDestroy(PVRSRV_DATA *psData)
+{
+    if (psData->psInfoPageMemDesc)
+    {
+        if (psData->pui32InfoPage != NULL)
+        {
+            DevmemReleaseCpuVirtAddr(psData->psInfoPageMemDesc);
+            psData->pui32InfoPage = NULL;
+        }
+
+        DevmemFree(psData->psInfoPageMemDesc);
+        psData->psInfoPageMemDesc = NULL;
+    }
+
+    if (psData->hInfoPageLock)
+    {
+        OSLockDestroy(psData->hInfoPageLock);
+        psData->hInfoPageLock = NULL;
+    }
+}
+
+PVRSRV_ERROR PVRSRVAcquireInfoPageKM(PMR **ppsPMR)
+{
+    PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+
+    PVR_LOG_RETURN_IF_FALSE(psData->psInfoPageMemDesc != NULL, "invalid MEMDESC"
+                      " handle", PVRSRV_ERROR_INVALID_PARAMS);
+    PVR_LOG_RETURN_IF_FALSE(psData->psInfoPagePMR != NULL, "invalid PMR handle",
+                      PVRSRV_ERROR_INVALID_PARAMS);
+
+    /* Copy the PMR import handle back */
+    *ppsPMR = psData->psInfoPagePMR;
+
+    /* Mark the PMR such that no layout changes can happen
+     * This is a fixed layout created during early stages of
+     * driver loading and shouldn't change later */
+    PMR_SetLayoutFixed(psData->psInfoPagePMR, IMG_TRUE);
+
+    return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVReleaseInfoPageKM(PMR *ppsPMR)
+{
+    /* Nothing to do here as PMR is singleton */
+    PVR_UNREFERENCED_PARAMETER(ppsPMR);
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/lists.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/lists.c
new file mode 100644
index 000000000000..e8e7088a3296
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/lists.c
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linked list shared functions implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implementation of the list iterators for types shared among
+                more than one file in the services code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "lists.h"
+
+/*===================================================================
+  LIST ITERATOR FUNCTIONS USED IN MORE THAN ONE FILE (those used just
+  once are implemented locally).
+  ===================================================================*/
+
+IMPLEMENT_LIST_ANY(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_ANY_2(PVRSRV_DEVICE_NODE, IMG_BOOL, IMG_FALSE)
+IMPLEMENT_LIST_ANY_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK)
+IMPLEMENT_LIST_ANY_VA(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_ANY_VA_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK)
+IMPLEMENT_LIST_FOR_EACH(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_FOR_EACH_VA(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_INSERT_TAIL(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_REMOVE(PVRSRV_DEVICE_NODE)
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/mmu_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/mmu_common.c
new file mode 100644
index 000000000000..e94d52a4ffe7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/mmu_common.c
@@ -0,0 +1,5343 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "dllist.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "htbserver.h"
+#include "pvr_ricommon.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+# include "process_stats.h"
+# include "proc_stats.h"
+#endif
+
+#if defined(PDUMP)
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#endif
+
+#include "physmem.h"
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "physmem_lma.h"
+#endif
+
+/*
+Major Interfaces to other modules:
+
+Let's keep this graph up-to-date:
+
+   +-----------+
+   | devicemem |
+   +-----------+
+         |
+   +============+
+   | mmu_common |
+   +============+
+         |
+         +-----------------+
+         |                 |
+    +---------+      +----------+
+    |   pmr   |      |  device  |
+    +---------+      +----------+
+ */
+
+#include "mmu_common.h"
+#include "pmr.h"
+#include "devicemem_server_utils.h"
+
+/* #define MMU_OBJECT_REFCOUNT_DEBUGING 1 */
+#if defined(MMU_OBJECT_REFCOUNT_DEBUGING)
+#define MMU_OBJ_DBG(x)	PVR_DPF(x)
+#else
+#define MMU_OBJ_DBG(x)
+#endif
+
+#define SCRATCH_PAGE 1
+#define DEV_ZERO_PAGE 2
+#if defined(PDUMP)
+#define SCRATCH_PAGE_STR "SCRATCH_PAGE"
+#define DEV_ZERO_PAGE_STR "DEV_ZERO_PAGE"
+#endif
+#define PVR_SCRATCH_PAGE_INIT_VALUE 0
+#define PVR_ZERO_PAGE_INIT_VALUE  0
+
+/*!
+ * Refcounted structure that is shared between the context and
+ * the cleanup thread items.
+ * It is used to keep track of all cleanup items and whether the creating
+ * MMU context has been destroyed and therefore is not allowed to be
+ * accessed any more.
+ *
+ * The cleanup thread is used to defer the freeing of the page tables
+ * because we have to make sure that the MMU cache has been invalidated.
+ * If we don't take care of this the MMU might partially access cached
+ * and uncached tables which might lead to inconsistencies and in the
+ * worst case to MMU pending faults on random memory.
+ */
+typedef struct _MMU_CTX_CLEANUP_DATA_
+{
+	/*! Refcount to know when this structure can be destroyed */
+	ATOMIC_T iRef;
+	/*! Protect items in this structure, especially the refcount */
+	POS_LOCK hCleanupLock;
+	/*! List of all cleanup items currently in flight */
+	DLLIST_NODE sMMUCtxCleanupItemsHead;
+	/*! Was the MMU context destroyed and should not be accessed any more? */
+	IMG_BOOL bMMUContextExists;
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/*! Associated OSid for this context */
+	IMG_UINT32 ui32OSid;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+} MMU_CTX_CLEANUP_DATA;
+
+
+/*!
+ * Structure holding one or more page tables that need to be
+ * freed after the MMU cache has been flushed which is signalled when
+ * the stored sync has a value that is <= the required value.
+ */
+typedef struct _MMU_CLEANUP_ITEM_
+{
+	/*! Cleanup thread data */
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+	/*! List to hold all the MMU_MEMORY_MAPPINGs, i.e. page tables */
+	DLLIST_NODE sMMUMappingHead;
+	/*! Node of the cleanup item list for the context */
+	DLLIST_NODE sMMUCtxCleanupItem;
+	/* Pointer to the cleanup meta data */
+	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData;
+	/* Sync to query if the MMU cache was flushed */
+	PVRSRV_CLIENT_SYNC_PRIM *psSync;
+	/*! The update value of the sync to signal that the cache was flushed */
+	IMG_UINT32 uiRequiredSyncVal;
+	/*! The update value of the power off counter */
+	IMG_UINT32 uiRequiredPowerOffCounter;
+	/*! The device node needed to free the page tables */
+	PVRSRV_DEVICE_NODE *psDevNode;
+} MMU_CLEANUP_ITEM;
+
+/*!
+	All physical allocations and frees are relative to this context, so
+	we would get all the allocations of PCs, PDs, and PTs from the same
+	RA.
+
+	We have one per MMU context in case we have mixed UMA/LMA devices
+	within the same system.
+ */
+typedef struct _MMU_PHYSMEM_CONTEXT_
+{
+	/*! Associated MMU_CONTEXT */
+	struct _MMU_CONTEXT_ *psMMUContext;
+
+	/*! Parent device node */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/*! Refcount so we know when to free up the arena */
+	IMG_UINT32 uiNumAllocations;
+
+	/*! Arena from which physical memory is derived */
+	RA_ARENA *psPhysMemRA;
+	/*! Arena name */
+	IMG_CHAR *pszPhysMemRAName;
+	/*! Size of arena name string */
+	size_t uiPhysMemRANameAllocSize;
+
+	/*! Meta data for deferred cleanup */
+	MMU_CTX_CLEANUP_DATA *psCleanupData;
+	/*! Temporary list of all deferred MMU_MEMORY_MAPPINGs. */
+	DLLIST_NODE sTmpMMUMappingHead;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	IMG_UINT32 ui32OSid;
+	IMG_UINT32 ui32OSidReg;
+	IMG_BOOL   bOSidAxiProt;
+#endif
+
+} MMU_PHYSMEM_CONTEXT;
+
+/*!
+	Mapping structure for MMU memory allocation
+ */
+typedef struct _MMU_MEMORY_MAPPING_
+{
+	/*! Physmem context to allocate from */
+	MMU_PHYSMEM_CONTEXT		*psContext;
+	/*! OS/system Handle for this allocation */
+	PG_HANDLE				sMemHandle;
+	/*! CPU virtual address of this allocation */
+	void					*pvCpuVAddr;
+	/*! Device physical address of this allocation */
+	IMG_DEV_PHYADDR			sDevPAddr;
+	/*! Size of this allocation */
+	size_t					uiSize;
+	/*! Number of current mappings of this allocation */
+	IMG_UINT32				uiCpuVAddrRefCount;
+	/*! Node for the defer free list */
+	DLLIST_NODE				sMMUMappingItem;
+} MMU_MEMORY_MAPPING;
+
+/*!
+	Memory descriptor for MMU objects. There can be more than one memory
+	descriptor per MMU memory allocation.
+ */
+typedef struct _MMU_MEMORY_DESC_
+{
+	/* NB: bValid is set if this descriptor describes physical
+	   memory.  This allows "empty" descriptors to exist, such that we
+	   can allocate them in batches. */
+	/*! Does this MMU object have physical backing */
+	IMG_BOOL				bValid;
+	/*! Device Physical address of physical backing */
+	IMG_DEV_PHYADDR			sDevPAddr;
+	/*! CPU virtual address of physical backing */
+	void					*pvCpuVAddr;
+	/*! Mapping data for this MMU object */
+	MMU_MEMORY_MAPPING		*psMapping;
+	/*! Memdesc offset into the psMapping */
+	IMG_UINT32 uiOffset;
+	/*! Size of the Memdesc */
+	IMG_UINT32 uiSize;
+} MMU_MEMORY_DESC;
+
+/*!
+	MMU levelx structure. This is generic and is used
+	for all levels (PC, PD, PT).
+ */
+typedef struct _MMU_Levelx_INFO_
+{
+	/*! The Number of entries in this level */
+	IMG_UINT32 ui32NumOfEntries;
+
+	/*! Number of times this level has been reference. Note: For Level1 (PTE)
+	    we still take/drop the reference when setting up the page tables rather
+	    then at map/unmap time as this simplifies things */
+	IMG_UINT32 ui32RefCount;
+
+	/*! MemDesc for this level */
+	MMU_MEMORY_DESC sMemDesc;
+
+	/*! Array of infos for the next level. Must be last member in structure */
+	struct _MMU_Levelx_INFO_ *apsNextLevel[IMG_FLEX_ARRAY_MEMBER];
+} MMU_Levelx_INFO;
+
+/*!
+	MMU context structure
+ */
+struct _MMU_CONTEXT_
+{
+	/*! Originating Connection */
+	CONNECTION_DATA *psConnection;
+
+	/*! Device MMU attribute descriptions */
+	MMU_DEVICEATTRIBS *psDevAttrs;
+
+	/*! For allocation and deallocation of the physical memory where
+	    the pagetables live */
+	struct _MMU_PHYSMEM_CONTEXT_ *psPhysMemCtx;
+
+#if defined(PDUMP)
+	/*! PDump context ID (required for PDump commands with virtual addresses) */
+	IMG_UINT32 uiPDumpContextID;
+
+	/*! The refcount of the PDump context ID */
+	IMG_UINT32 ui32PDumpContextIDRefCount;
+#endif
+
+	/*! MMU cache invalidation flags. For Rogue, only one set of flags are
+	 * maintained (in the KernelMMUCtx) and used for all context page table
+	 * updates, while in Volcanic each context maintains its own flags
+	 * for the PTs it contains. This is due to the different memory hierarchy
+	 * designs in use between the architectures, See SLC_VIVT feature. */
+	ATOMIC_T sCacheFlags;
+
+	/*! Lock to ensure exclusive access when manipulating the MMU context or
+	 * reading and using its content
+	 */
+	POS_LOCK hLock;
+
+	/*! Base level info structure. Must be last member in structure */
+	MMU_Levelx_INFO sBaseLevelInfo;
+	/* NO OTHER MEMBERS AFTER THIS STRUCTURE ! */
+};
+
+/*
+ * Only the kernel created (direct bridge) firmware memory context will
+ * have a NULL connection and all other application memory context get a
+ * valid connection object.
+ */
+#define _MMU_IS_FWKM_CTX(_ctx) ((_ctx)->psConnection == NULL)
+
+/*
+ * Used to determine if the MMU Ctx provided is the FWKM memory context
+ * and if it belongs to the VZ Guest.
+ */
+#define _MMU_IS_FWKM_CTX_VZGUEST(_ctx) (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, _ctx->psPhysMemCtx->psDevNode) && _MMU_IS_FWKM_CTX(_ctx))
+
+static const IMG_DEV_PHYADDR gsBadDevPhyAddr = {MMU_BAD_PHYS_ADDR};
+
+#if defined(DEBUG)
+#include "log2.h"
+#endif
+
+
+static PVRSRV_ERROR
+MMU_UnmapPagesUnlocked(MMU_CONTEXT *psMMUContext,
+                       PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                       IMG_DEV_VIRTADDR sDevVAddrBase,
+                       IMG_UINT32 ui32PageCount,
+                       IMG_UINT32 *pai32FreeIndices,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags);
+
+static PVRSRV_ERROR
+MMU_UnmapPMRFastUnlocked(MMU_CONTEXT *psMMUContext,
+                         IMG_DEV_VIRTADDR sDevVAddrBase,
+                         IMG_UINT32 ui32PageCount,
+                         IMG_UINT32 uiLog2PageSize);
+
+/*****************************************************************************
+ *                          Utility functions                                *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _FreeMMUMapping
+
+@Description    Free a given dllist of MMU_MEMORY_MAPPINGs and the page tables
+                they represent.
+
+@Input          psDevNode           Device node
+
+@Input          psTmpMMUMappingHead List of MMU_MEMORY_MAPPINGs to free
+ */
+/*****************************************************************************/
+static void
+_FreeMMUMapping(PVRSRV_DEVICE_NODE *psDevNode,
+                PDLLIST_NODE psTmpMMUMappingHead)
+{
+	PDLLIST_NODE psNode, psNextNode;
+
+	/* Free the current list unconditionally */
+	dllist_foreach_node(psTmpMMUMappingHead,
+	                    psNode,
+	                    psNextNode)
+	{
+		MMU_MEMORY_MAPPING *psMapping = IMG_CONTAINER_OF(psNode,
+		                                                 MMU_MEMORY_MAPPING,
+		                                                 sMMUMappingItem);
+
+		PhysHeapPagesFree(psDevNode->psMMUPhysHeap, &psMapping->sMemHandle);
+		dllist_remove_node(psNode);
+		OSFreeMem(psMapping);
+	}
+}
+
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+/*************************************************************************/ /*!
+@Function       _CleanupThread_FreeMMUMapping
+
+@Description    Function to be executed by the cleanup thread to free
+                MMU_MEMORY_MAPPINGs after the MMU cache has been invalidated.
+
+                This function will request a MMU cache invalidate once and
+                retry to free the MMU_MEMORY_MAPPINGs until the invalidate
+                has been executed.
+
+                If the memory context that created this cleanup item has been
+                destroyed in the meantime this function will directly free the
+                MMU_MEMORY_MAPPINGs without waiting for any MMU cache
+                invalidation.
+
+@Input          pvData           Cleanup data in form of a MMU_CLEANUP_ITEM
+
+@Return         PVRSRV_OK if successful otherwise PVRSRV_ERROR_RETRY
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR
+_CleanupThread_FreeMMUMapping(void* pvData)
+{
+	PVRSRV_ERROR eError;
+	MMU_CLEANUP_ITEM *psCleanup = (MMU_CLEANUP_ITEM *)pvData;
+	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData = psCleanup->psMMUCtxCleanupData;
+	PVRSRV_DEVICE_NODE *psDevNode = psCleanup->psDevNode;
+	IMG_BOOL bFreeNow;
+
+	OSLockAcquire(psMMUCtxCleanupData->hCleanupLock);
+
+	/* Don't attempt to free anything when the context has been destroyed.
+	 * Especially don't access any device specific structures any more!*/
+	if (!psMMUCtxCleanupData->bMMUContextExists)
+	{
+		OSFreeMem(psCleanup);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_OK, e0);
+	}
+
+	/* Has there been a power off since the cleanup item was queued? */
+	bFreeNow = PVRSRVHasCounter32Advanced(psDevNode->uiPowerOffCounter,
+	                                      psCleanup->uiRequiredPowerOffCounter);
+	if (bFreeNow)
+	{
+		goto freeNow;
+	}
+
+	if (psCleanup->psSync == NULL)
+	{
+		IMG_UINT32 uiSyncVal;
+		/* Kick to invalidate the MMU caches and get sync info */
+		eError = psDevNode->pfnMMUCacheInvalidateKick(psDevNode,
+		                                              &uiSyncVal);
+		if (eError != PVRSRV_OK)
+		{
+			OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+			return PVRSRV_ERROR_RETRY;
+		}
+		psCleanup->psSync = psDevNode->psMMUCacheSyncPrim;
+		/* If we have no sync reference we treat this as a simple FREE.
+		 * We cannot retry as there will never be a sync-prim to kick / invalidate.
+		 */
+		if (psCleanup->psSync == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device %u has no MMUCacheSyncPrim",
+			         __func__, psDevNode->sDevId.ui32InternalID));
+			bFreeNow = IMG_TRUE;
+			goto freeNow;
+		}
+	}
+
+	/* Has the invalidate executed (sync is updated when the Firmware performs
+	 * the invalidate)? */
+	bFreeNow = PVRSRVHasCounter32Advanced(OSReadDeviceMem32(psCleanup->psSync->pui32LinAddr),
+	                                      psCleanup->uiRequiredSyncVal);
+
+#if defined(NO_HARDWARE)
+	/* In NOHW the syncs will never be updated so just free the tables */
+	bFreeNow = IMG_TRUE;
+#endif
+	/* If the Invalidate operation is not completed, check if the operation timed out */
+	if (!bFreeNow)
+	{
+		IMG_UINT32 uiTimeStart = psCleanup->sCleanupThreadFn.ui32TimeStart;
+		IMG_UINT32 uiTimeEnd = psCleanup->sCleanupThreadFn.ui32TimeEnd;
+
+		/* If the time left for the completion of invalidate operation is
+		 * within 500ms of time-out, consider the operation as timed out */
+		if ((uiTimeEnd - uiTimeStart - 500) <= (OSClockms() - uiTimeStart))
+		{
+			/* Consider the operation is timed out */
+			bFreeNow = IMG_TRUE;
+		}
+	}
+
+freeNow:
+	/* Free if the invalidate operation completed or the operation itself timed out */
+	if (bFreeNow)
+	{
+		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);
+
+		dllist_remove_node(&psCleanup->sMMUCtxCleanupItem);
+
+		OSFreeMem(psCleanup);
+
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_RETRY;
+	}
+
+e0:
+
+	/* If this cleanup task has been successfully executed we can
+	 * decrease the context cleanup data refcount. Successfully
+	 * means here that the MMU_MEMORY_MAPPINGs have been freed by
+	 * either this cleanup task of when the MMU context has been
+	 * destroyed. */
+	if (eError == PVRSRV_OK)
+	{
+		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+
+		if (OSAtomicDecrement(&psMMUCtxCleanupData->iRef) == 0)
+		{
+			OSLockDestroy(psMMUCtxCleanupData->hCleanupLock);
+			OSFreeMem(psMMUCtxCleanupData);
+		}
+	}
+	else
+	{
+		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+	}
+
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _SetupCleanup_FreeMMUMapping
+
+@Description    Setup a cleanup item for the cleanup thread that will
+                kick off a MMU invalidate request and free the associated
+                MMU_MEMORY_MAPPINGs when the invalidate was successful.
+
+@Input          psPhysMemCtx        The current MMU physmem context
+ */
+/*****************************************************************************/
+static void
+_SetupCleanup_FreeMMUMapping(MMU_PHYSMEM_CONTEXT *psPhysMemCtx)
+{
+
+	MMU_CLEANUP_ITEM *psCleanupItem;
+	MMU_CTX_CLEANUP_DATA *psCleanupData = psPhysMemCtx->psCleanupData;
+	PVRSRV_DEVICE_NODE *psDevNode = psPhysMemCtx->psDevNode;
+
+	if (dllist_is_empty(&psPhysMemCtx->sTmpMMUMappingHead))
+	{
+		goto e0;
+	}
+
+#if defined(PDUMP)
+	/* Free the page tables immediately in case of pdump, which avoids
+	 * changing script files due to the additional invalidation kick */
+	goto e1;
+#endif
+
+	/* Don't defer the freeing if we are currently unloading the driver
+	 * or if the sync has been destroyed */
+	if (PVRSRVGetPVRSRVData()->bUnload ||
+			psDevNode->psMMUCacheSyncPrim == NULL)
+	{
+		goto e1;
+	}
+
+	/* Allocate a cleanup item */
+	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
+	if (!psCleanupItem)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to get memory for deferred page table cleanup. "
+				"Freeing tables immediately",
+				__func__));
+		goto e1;
+	}
+
+	/* Set sync to NULL to indicate we did not interact with
+	 * the FW yet. Kicking off an MMU cache invalidate should
+	 * be done in the cleanup thread to not waste time here. */
+	psCleanupItem->psSync = NULL;
+	psCleanupItem->uiRequiredSyncVal = psDevNode->ui32NextMMUInvalidateUpdate;
+	psCleanupItem->uiRequiredPowerOffCounter = psDevNode->uiPowerOffCounterNext;
+	psCleanupItem->psDevNode = psDevNode;
+	psCleanupItem->psMMUCtxCleanupData = psCleanupData;
+
+	OSAtomicIncrement(&psCleanupData->iRef);
+
+	/* Move the page tables to free to the cleanup item */
+	dllist_replace_head(&psPhysMemCtx->sTmpMMUMappingHead,
+	                    &psCleanupItem->sMMUMappingHead);
+
+	/* Add the cleanup item itself to the context list */
+	dllist_add_to_tail(&psCleanupData->sMMUCtxCleanupItemsHead,
+	                   &psCleanupItem->sMMUCtxCleanupItem);
+
+	/* Setup the cleanup thread data and add the work item */
+	psCleanupItem->sCleanupThreadFn.pfnFree = _CleanupThread_FreeMMUMapping;
+	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
+	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+	psCleanupItem->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_MMU;
+	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
+	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);
+
+	PVRSRVCleanupThreadAddWork(psDevNode, &psCleanupItem->sCleanupThreadFn);
+
+	return;
+
+e1:
+	/* Free the page tables now */
+	_FreeMMUMapping(psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+e0:
+	return;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _CalcPCEIdx
+
+@Description    Calculate the page catalogue index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page catalogue index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPCEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPCIndexMask)
+			>> psDevVAddrConfig->uiPCIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _CalcPDEIdx
+
+@Description    Calculate the page directory index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page directory index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPDEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPDIndexMask)
+			>> psDevVAddrConfig->uiPDIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _CalcPTEIdx
+
+@Description    Calculate the page entry index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page entry index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPTEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+	sTmpDevVAddr.uiAddr -= psDevVAddrConfig->uiOffsetInBytes;
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPTIndexMask)
+			>> psDevVAddrConfig->uiPTIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+/*************************************************************************/ /*!
+@Function       _GetParityBit
+
+@Description    Calculate parity bit
+
+@Input          uiSrc           Value to calculate the parity bit from.
+
+@Return         The parity bit
+ */
+/*****************************************************************************/
+static inline IMG_UINT64 _GetParityBit(IMG_UINT64 uiSrc)
+{
+	uiSrc ^= uiSrc >> 32U;
+	uiSrc ^= uiSrc >> 16U;
+	uiSrc ^= uiSrc >> 8U;
+	uiSrc ^= uiSrc >> 4U;
+	uiSrc ^= uiSrc >> 2U;
+	uiSrc ^= uiSrc >> 1U;
+
+	return (uiSrc & 1U);
+}
+
+/*****************************************************************************
+ *         MMU memory allocation/management functions (mem desc)             *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMem_RAImportAlloc
+
+@Description    Imports MMU Px memory into the RA. This is where the
+                actual allocation of physical memory happens.
+
+@Input          hArenaHandle    Handle that was passed in during the
+                                creation of the RA
+
+@Input          uiSize          Size of the memory to import
+
+@Input          uiFlags         Flags that where passed in the allocation.
+
+@Input          uBaseAlignment  Alignment for the base returned, not used
+                                in this context.
+
+@Output         puiBase         The address of where to insert this import
+
+@Output         puiActualSize   The actual size of the import
+
+@Output         phPriv          Handle which will be passed back when
+                                this import is freed
+
+@Return         PVRSRV_OK if import alloc was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _MMU_PhysMem_RAImportAlloc(RA_PERARENA_HANDLE hArenaHandle,
+                                               RA_LENGTH_T uiSize,
+                                               RA_FLAGS_T uiFlags,
+                                               RA_LENGTH_T uBaseAlignment,
+                                               const IMG_CHAR *pszAnnotation,
+                                               RA_IMPORT *psImport)
+{
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx = (MMU_PHYSMEM_CONTEXT *)hArenaHandle;
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psPhysMemCtx->psDevNode;
+	MMU_MEMORY_MAPPING *psMapping;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPid = 0;
+
+	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	PVR_ASSERT(psDevNode != NULL);
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode, eError, e0);
+
+	psMapping = OSAllocMem(sizeof(MMU_MEMORY_MAPPING));
+	PVR_GOTO_IF_NOMEM(psMapping, eError, e0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	uiPid = psDevNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ?
+	        PVR_SYS_ALLOC_PID : OSGetCurrentClientProcessIDKM();
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/*
+	 * Store the OSid in the PG_HANDLE.uiOSid field for use by the
+	 * pfnDevPxFree() routine.
+	 */
+	psMapping->sMemHandle.uiOSid = psPhysMemCtx->ui32OSid;
+	eError = PhysHeapPagesAllocGPV(psDevNode->psMMUPhysHeap,
+	                               TRUNCATE_64BITS_TO_SIZE_T(uiSize),
+	                               &psMapping->sMemHandle,
+	                               &psMapping->sDevPAddr,
+	                               psPhysMemCtx->ui32OSid,
+	                               uiPid);
+#else
+	eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+	                            TRUNCATE_64BITS_TO_SIZE_T(uiSize),
+	                            &psMapping->sMemHandle,
+	                            &psMapping->sDevPAddr,
+	                            uiPid);
+#endif
+	if (eError != PVRSRV_OK)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		PVRSRVStatsUpdateOOMStat(NULL, psDevNode, PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+					  OSGetCurrentClientProcessIDKM());
+#endif
+		goto e1;
+	}
+
+	psMapping->psContext = psPhysMemCtx;
+	psMapping->uiSize = TRUNCATE_64BITS_TO_SIZE_T(uiSize);
+
+	psMapping->uiCpuVAddrRefCount = 0;
+
+	psImport->hPriv = (RA_PERISPAN_HANDLE) psMapping;
+
+	/* Note: This assumes this memory never gets paged out */
+	psImport->base = (RA_BASE_T)psMapping->sDevPAddr.uiAddr;
+	psImport->uSize = uiSize;
+
+	#if (MAP_UNCACHED_MEM)
+		arch_sync_dma_for_device(page_to_phys((struct page*)psMapping->sMemHandle.u.pvHandle), PAGE_ALIGN(uiSize), DMA_TO_DEVICE);
+	#endif
+	return PVRSRV_OK;
+e1:
+	OSFreeMem(psMapping);
+e0:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMem_RAImportFree
+
+@Description    Imports MMU Px memory into the RA. This is where the
+                actual free of physical memory happens.
+
+@Input          hArenaHandle    Handle that was passed in during the
+                                creation of the RA
+
+@Input          puiBase         The address of where to insert this import
+
+@Output         phPriv          Private data that the import alloc provided
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _MMU_PhysMem_RAImportFree(RA_PERARENA_HANDLE hArenaHandle,
+                                      RA_BASE_T uiBase,
+                                      RA_PERISPAN_HANDLE hPriv)
+{
+	MMU_MEMORY_MAPPING *psMapping = (MMU_MEMORY_MAPPING *)hPriv;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx = (MMU_PHYSMEM_CONTEXT *)hArenaHandle;
+
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+
+	/* Check we have dropped all CPU mappings */
+	PVR_ASSERT(psMapping->uiCpuVAddrRefCount == 0);
+
+	/* Add mapping to defer free list */
+	psMapping->psContext = NULL;
+	dllist_add_to_tail(&psPhysMemCtx->sTmpMMUMappingHead, &psMapping->sMMUMappingItem);
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMemAlloc
+
+@Description    Allocates physical memory for MMU objects
+
+@Input          psPhysMemCtx    Physmem context to do the allocation from
+
+@Output         psMemDesc       Allocation description
+
+@Input          uiBytes         Size of the allocation in bytes
+
+@Input          uiAlignment     Alignment requirement of this allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+
+static PVRSRV_ERROR _MMU_PhysMemAlloc(MMU_PHYSMEM_CONTEXT *psPhysMemCtx,
+                                      MMU_MEMORY_DESC *psMemDesc,
+                                      size_t uiBytes,
+                                      size_t uiAlignment)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiPhysAddr;
+
+	PVR_RETURN_IF_INVALID_PARAM(psMemDesc);
+	PVR_RETURN_IF_INVALID_PARAM(!psMemDesc->bValid);
+
+	eError = RA_Alloc(psPhysMemCtx->psPhysMemRA,
+	                  uiBytes,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0, /* flags */
+	                  uiAlignment,
+	                  "",
+	                  &uiPhysAddr,
+	                  NULL,
+	                  (RA_PERISPAN_HANDLE *)&psMemDesc->psMapping);
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "RA_Alloc");
+
+	psMemDesc->bValid = IMG_TRUE;
+	psMemDesc->pvCpuVAddr = NULL;
+	psMemDesc->sDevPAddr.uiAddr = (IMG_UINT64) uiPhysAddr;
+
+	if (psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
+	{
+		eError = PhysHeapPagesMap(psPhysMemCtx->psDevNode->psMMUPhysHeap,
+		                          &psMemDesc->psMapping->sMemHandle,
+		                          psMemDesc->psMapping->uiSize,
+		                          &psMemDesc->psMapping->sDevPAddr,
+		                          &psMemDesc->psMapping->pvCpuVAddr);
+		if (eError != PVRSRV_OK)
+		{
+			RA_Free(psPhysMemCtx->psPhysMemRA, psMemDesc->sDevPAddr.uiAddr);
+			return eError;
+		}
+	}
+
+	psMemDesc->psMapping->uiCpuVAddrRefCount++;
+	psMemDesc->uiOffset = (psMemDesc->sDevPAddr.uiAddr - psMemDesc->psMapping->sDevPAddr.uiAddr);
+	psMemDesc->pvCpuVAddr = (IMG_UINT8 *)psMemDesc->psMapping->pvCpuVAddr + psMemDesc->uiOffset;
+	psMemDesc->uiSize = uiBytes;
+	PVR_ASSERT(psMemDesc->pvCpuVAddr != NULL);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMemFree
+
+@Description    Allocates physical memory for MMU objects
+
+@Input          psPhysMemCtx    Physmem context to do the free on
+
+@Input          psMemDesc       Allocation description
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _MMU_PhysMemFree(MMU_PHYSMEM_CONTEXT *psPhysMemCtx,
+                             MMU_MEMORY_DESC *psMemDesc)
+{
+	RA_BASE_T uiPhysAddr;
+
+	PVR_ASSERT(psMemDesc->bValid);
+
+	if (--psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
+	{
+		PhysHeapPagesUnMap(psPhysMemCtx->psDevNode->psMMUPhysHeap,
+		                   &psMemDesc->psMapping->sMemHandle,
+		                   psMemDesc->psMapping->pvCpuVAddr);
+	}
+
+	psMemDesc->pvCpuVAddr = NULL;
+
+	uiPhysAddr = psMemDesc->sDevPAddr.uiAddr;
+	RA_Free(psPhysMemCtx->psPhysMemRA, uiPhysAddr);
+
+	psMemDesc->bValid = IMG_FALSE;
+}
+
+
+/*****************************************************************************
+ *              MMU object allocation/management functions                   *
+ *****************************************************************************/
+
+static INLINE PVRSRV_ERROR _MMU_ConvertDevMemFlags(IMG_BOOL bInvalidate,
+                                                   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                                                   MMU_PROTFLAGS_T *puiMMUProtFlags,
+                                                   MMU_CONTEXT *psMMUContext)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiGPUCacheMode;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Do flag conversion between devmem flags and MMU generic flags */
+	if (bInvalidate == IMG_TRUE)
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_INVALID;
+		return eError; /* OK */
+	}
+
+	/* Convert to valid flags for valid mapping */
+	*puiMMUProtFlags |= ((uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
+			>> PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
+			<< MMU_PROTFLAGS_DEVICE_OFFSET;
+
+	if (PVRSRV_CHECK_GPU_READABLE(uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+	}
+	if (PVRSRV_CHECK_GPU_WRITEABLE(uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_WRITEABLE;
+	}
+
+	eError = DevmemDeviceCacheMode(uiMappingFlags, &uiGPUCacheMode);
+	PVR_RETURN_IF_ERROR(eError);
+
+	switch (uiGPUCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC:
+			break;
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHED:
+			*puiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Wrong parameters",
+					__func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (DevmemDeviceCacheCoherency(psDevNode, uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_CACHE_COHERENT;
+	}
+
+	if (psDevNode->pfnMMUTweakProtFlags)
+	{
+		psDevNode->pfnMMUTweakProtFlags(psDevNode, psMMUContext->psDevAttrs, uiMappingFlags, puiMMUProtFlags);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _PxMemAlloc
+
+@Description    Allocates physical memory for MMU objects, initialises
+                and PDumps it.
+
+@Input          psMMUContext    MMU context
+
+@Input          uiNumEntries    Number of entries to allocate
+
+@Input          psConfig        MMU Px config
+
+@Input          eMMULevel       MMU level that allocation is for
+
+@Output         psMemDesc       Description of allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _PxMemAlloc(MMU_CONTEXT *psMMUContext,
+                                IMG_UINT32 uiNumEntries,
+                                const MMU_PxE_CONFIG *psConfig,
+                                MMU_LEVEL eMMULevel,
+                                MMU_MEMORY_DESC *psMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+                                IMG_DEV_VIRTADDR* psRunningDevVAddr,
+                                IMG_UINT32 uiLog2DataPageSize,
+#endif
+                                IMG_UINT32 uiLog2Align)
+{
+	PVRSRV_ERROR eError;
+	size_t uiBytes;
+	size_t uiAlign;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	PVR_ASSERT(psConfig->uiBytesPerEntry != 0);
+
+	uiBytes = uiNumEntries * psConfig->uiBytesPerEntry;
+	/* We need here the alignment of the previous level because that is the entry for we generate here */
+	uiAlign = 1 << uiLog2Align;
+
+	/*
+	 * If the hardware specifies an alignment requirement for a page table then
+	 * it also requires that all memory up to the next aligned address is
+	 * zeroed.
+	 *
+	 * Failing to do this can result in uninitialised data outside of the actual
+	 * page table range being read by the MMU and treated as valid, e.g. the
+	 * pending flag.
+	 *
+	 * Typically this will affect 1MiB, 2MiB PT pages which have a size of 16
+	 * and 8 bytes respectively but an alignment requirement of 64 bytes each.
+	 */
+	uiBytes = PVR_ALIGN(uiBytes, uiAlign);
+
+	/* allocate the object */
+	eError = _MMU_PhysMemAlloc(psMMUContext->psPhysMemCtx,
+	                           psMemDesc, uiBytes, uiAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_MMU_PhysMemAlloc", eError, PVRSRV_ERROR_OUT_OF_MEMORY, e0);
+	}
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	/* If parity bit is needed, set it accordingly to avoid parity errors along with page-faults. */
+	if (psConfig->uiParityBitMask && eMMULevel == MMU_LEVEL_1 && psRunningDevVAddr != NULL)
+	{
+		IMG_UINT32 uiParity = (IMG_UINT32)_GetParityBit(psRunningDevVAddr->uiAddr ^ 0ULL);
+
+		/* Only calculate parity for first address in the PT.
+		   This tells us which of the two patterns to use for the rest of the PT. */
+		OSCachedMemCopy(psMemDesc->pvCpuVAddr, psMMUContext->psDevAttrs->pui64PrecomputedAllocParity[uiParity], uiBytes);
+
+		/* Increment running dev virtual address */
+		psRunningDevVAddr->uiAddr += (1 << uiLog2DataPageSize) * uiNumEntries;
+	}
+	else
+#endif
+	{
+		/*
+			Clear the object
+			Note: if any MMUs are cleared with non-zero values then will need a
+			custom clear function
+			Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
+			unlikely
+		*/
+		OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, uiBytes);
+	}
+
+	eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+	                            &psMemDesc->psMapping->sMemHandle,
+	                            psMemDesc->uiOffset,
+	                            psMemDesc->uiSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Alloc MMU object");
+
+	PDumpMMUMalloc(psDevNode,
+	               psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	               eMMULevel,
+	               &psMemDesc->sDevPAddr,
+	               uiBytes,
+	               uiAlign,
+	               psMMUContext->psDevAttrs->eMMUType);
+
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      eMMULevel,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      0,
+	                      uiNumEntries,
+	                      NULL, NULL, 0, /* pdump symbolic info is irrelevant here */
+	                      psConfig->uiBytesPerEntry,
+	                      uiLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      0, /* Unused - Parity bit values are taken directly from PTE memory when PTE is 0-initialised or invalid */
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      psMMUContext->psDevAttrs->eMMUType);
+#else
+	PVR_UNREFERENCED_PARAMETER(eMMULevel);
+#endif
+
+	return PVRSRV_OK;
+e1:
+	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx,
+	                 psMemDesc);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _PxMemFree
+
+@Description    Frees physical memory for MMU objects, de-initialises
+                and PDumps it.
+
+@Input          psMemDesc       Description of allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+
+static void _PxMemFree(MMU_CONTEXT *psMMUContext,
+                       MMU_MEMORY_DESC *psMemDesc, MMU_LEVEL eMMULevel)
+{
+#if defined(MMU_CLEARMEM_ON_FREE)
+	/*
+		Clear the MMU object
+		Note: if any MMUs are cleared with non-zero values then will need a
+		custom clear function
+		Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
+		unlikely
+	 */
+	OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, psMemDesc->ui32Bytes);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Clear MMU object before freeing it");
+#endif
+#endif/* MMU_CLEARMEM_ON_FREE */
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Free MMU object");
+	PDumpMMUFree(psMMUContext->psPhysMemCtx->psDevNode,
+	             psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	             eMMULevel,
+	             &psMemDesc->sDevPAddr,
+	             psMMUContext->psDevAttrs->eMMUType);
+#else
+	PVR_UNREFERENCED_PARAMETER(eMMULevel);
+#endif
+	/* free the PC */
+	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx, psMemDesc);
+}
+
+static INLINE PVRSRV_ERROR _SetupPTE(MMU_CONTEXT *psMMUContext,
+                                     MMU_Levelx_INFO *psLevel,
+                                     IMG_UINT32 uiIndex,
+                                     const MMU_PxE_CONFIG *psConfig,
+                                     const IMG_DEV_PHYADDR *psDevPAddr,
+                                     const IMG_DEV_VIRTADDR *psDevVAddr,
+                                     IMG_BOOL bUnmap,
+#if defined(PDUMP)
+                                     const IMG_CHAR *pszMemspaceName,
+                                     const IMG_CHAR *pszSymbolicAddr,
+                                     IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
+#endif
+                                     IMG_UINT64 uiProtFlags)
+{
+	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
+	IMG_UINT64 ui64PxE64;
+	IMG_UINT64 uiAddr = psDevPAddr->uiAddr;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_BOOL bParity = psConfig->uiParityBitMask;
+#else
+	IMG_BOOL bParity = psConfig->uiParityBitMask && !bUnmap;
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(bParity);
+#endif
+#endif
+
+	if (psDevNode->pfnValidateOrTweakPhysAddrs)
+	{
+		PVRSRV_ERROR eErr = psDevNode->pfnValidateOrTweakPhysAddrs(psDevNode,
+		                                                           psMMUContext->psDevAttrs,
+		                                                           &uiAddr);
+		/* return if error */
+		PVR_LOG_RETURN_IF_ERROR(eErr, "_SetupPTE");
+	}
+
+	/* Calculate Entry */
+	ui64PxE64 =    uiAddr /* Calculate the offset to that base */
+			>> psConfig->uiAddrLog2Align /* Shift away the useless bits, because the alignment is very coarse and we address by alignment */
+			<< psConfig->uiAddrShift /* Shift back to fit address in the Px entry */
+			& psConfig->uiAddrMask; /* Delete unused bits */
+
+	ui64PxE64 |= uiProtFlags;
+
+	/* Add parity */
+	if (bParity)
+	{
+		ui64PxE64 |= _GetParityBit(psDevVAddr->uiAddr ^ psDevPAddr->uiAddr) << psConfig->uiParityBitShift;
+	}
+
+	/* Set the entry */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+		pui64Px[uiIndex] = ui64PxE64;
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+		/* assert that the result fits into 32 bits before writing
+		   it into the 32-bit array with a cast */
+		PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+		PVR_ASSERT(!bParity);
+
+		pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
+	}
+	else
+	{
+		return PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
+	}
+
+	/* Log modification */
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+	        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+	        uiIndex, MMU_LEVEL_1,
+	        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+	        !bUnmap);
+
+	PVR_UNREFERENCED_PARAMETER(bUnmap); /* when HTBLOGK is disabled */
+
+#if defined(PDUMP)
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      MMU_LEVEL_1,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      uiIndex,
+	                      1,
+	                      pszMemspaceName,
+	                      pszSymbolicAddr,
+	                      uiSymbolicAddrOffset,
+	                      psConfig->uiBytesPerEntry,
+	                      psConfig->uiAddrLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      bParity ? _GetParityBit(psDevVAddr->uiAddr ^ 0) : 0,
+	                      psConfig->uiParityBitShift,
+	                      bParity ? psConfig->uiParityBitMask : 0,
+	                      psMMUContext->psDevAttrs->eMMUType);
+#endif /*PDUMP*/
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _SetupPxE
+
+@Description    Sets up an entry of an MMU object to point to the
+                provided address
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel         Level info for MMU object
+
+@Input          uiIndex         Index into the MMU object to setup
+
+@Input          psConfig        MMU Px config
+
+@Input          eMMULevel       Level of MMU object
+
+@Input          psDevPAddr      Address to setup the MMU object to point to
+
+@Input          pszMemspaceName Name of the PDump memory space that the entry
+                                will point to
+
+@Input          pszSymbolicAddr PDump symbolic address that the entry will
+                                point to
+
+@Input          uiProtFlags     MMU protection flags
+ */
+/*****************************************************************************/
+static void _SetupPxE(MMU_CONTEXT *psMMUContext,
+                      MMU_Levelx_INFO *psLevel,
+                      IMG_UINT32 uiIndex,
+                      const MMU_PxE_CONFIG *psConfig,
+                      MMU_LEVEL eMMULevel,
+                      const IMG_DEV_PHYADDR *psDevPAddr,
+#if defined(PDUMP)
+                      const IMG_CHAR *pszMemspaceName,
+                      const IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
+#endif
+                      MMU_PROTFLAGS_T uiProtFlags,
+                      IMG_UINT32 uiLog2DataPageSize)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
+
+	IMG_UINT32 (*pfnDerivePxEProt4)(IMG_UINT32);
+	IMG_UINT64 (*pfnDerivePxEProt8)(IMG_UINT32, IMG_UINT32);
+
+	if (!psDevPAddr)
+	{
+		/* Invalidate entry */
+		if (~uiProtFlags & MMU_PROTFLAGS_INVALID)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Error, no physical address specified, but not invalidating entry"));
+			uiProtFlags |= MMU_PROTFLAGS_INVALID;
+		}
+		psDevPAddr = &gsBadDevPhyAddr;
+	}
+	else
+	{
+		if (uiProtFlags & MMU_PROTFLAGS_INVALID)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "A physical address was specified when requesting invalidation of entry"));
+			uiProtFlags |= MMU_PROTFLAGS_INVALID;
+		}
+	}
+
+	switch (eMMULevel)
+	{
+		case MMU_LEVEL_3:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePCEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePCEProt8;
+			break;
+
+		case MMU_LEVEL_2:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePDEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePDEProt8;
+			break;
+
+		case MMU_LEVEL_1:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePTEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePTEProt8;
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level", __func__));
+			PVR_ASSERT(0);
+			return;
+	}
+
+	/* How big is a PxE in bytes? */
+	/* Filling the actual Px entry with an address */
+	switch (psConfig->uiBytesPerEntry)
+	{
+		case 4:
+		{
+			IMG_UINT32 *pui32Px;
+			IMG_UINT64 ui64PxE64;
+
+			pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+			ui64PxE64 = psDevPAddr->uiAddr               /* Calculate the offset to that base */
+					>> psConfig->uiAddrLog2Align /* Shift away the unnecessary bits of the address */
+					<< psConfig->uiAddrShift     /* Shift back to fit address in the Px entry */
+					& psConfig->uiAddrMask;      /* Delete unused higher bits */
+
+			ui64PxE64 |= (IMG_UINT64)pfnDerivePxEProt4(uiProtFlags);
+			/* assert that the result fits into 32 bits before writing
+			   it into the 32-bit array with a cast */
+			PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+
+			/* We should never invalidate an invalid page */
+			if (uiProtFlags & MMU_PROTFLAGS_INVALID)
+			{
+				PVR_ASSERT(pui32Px[uiIndex] != ui64PxE64);
+			}
+			pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+			        uiIndex, eMMULevel,
+			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
+			break;
+		}
+		case 8:
+		{
+			IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+			IMG_UINT64 ui64PxE64;
+
+			ui64PxE64 = psDevPAddr->uiAddr             /* Calculate the offset to that base */
+					>> psConfig->uiAddrLog2Align  /* Shift away the unnecessary bits of the address */
+					<< psConfig->uiAddrShift      /* Shift back to fit address in the Px entry */
+					& psConfig->uiAddrMask;       /* Delete unused higher bits */
+			ui64PxE64 |= pfnDerivePxEProt8(uiProtFlags, uiLog2DataPageSize);
+
+			pui64Px[uiIndex] = ui64PxE64;
+
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+			        uiIndex, eMMULevel,
+			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
+			break;
+		}
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: PxE size not supported (%d) for level %d",
+					__func__, psConfig->uiBytesPerEntry, eMMULevel));
+			PVR_ASSERT(0);
+			return;
+	}
+
+#if defined(PDUMP)
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      eMMULevel,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      uiIndex,
+	                      1,
+	                      pszMemspaceName,
+	                      pszSymbolicAddr,
+	                      uiSymbolicAddrOffset,
+	                      psConfig->uiBytesPerEntry,
+	                      psConfig->uiAddrLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      psMMUContext->psDevAttrs->eMMUType);
+#endif
+
+	psDevNode->pfnMMUCacheInvalidate(psDevNode, psMMUContext,
+	                                 eMMULevel,
+	                                 uiProtFlags & MMU_PROTFLAGS_INVALID);
+
+	return;
+}
+
+/*****************************************************************************
+ *                   MMU host control functions (Level Info)                 *
+ *****************************************************************************/
+
+
+/*************************************************************************/ /*!
+@Function       _MMU_FreeLevel
+
+@Description    Recursively frees the specified range of Px entries. If any
+                level has its last reference dropped then the MMU object
+                memory and the MMU_Levelx_Info will be freed.
+
+				At each level we might be crossing a boundary from one Px to
+				another. The values for auiStartArray should be by used for
+				the first call into each level and the values in auiEndArray
+				should only be used in the last call for each level.
+				In order to determine if this is the first/last call we pass
+				in bFirst and bLast.
+				When one level calls down to the next only if bFirst/bLast is set
+				and it's the first/last iteration of the loop at its level will
+				bFirst/bLast set for the next recursion.
+				This means that each iteration has the knowledge of the previous
+				level which is required.
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel                 Level info on which to free the
+                                        specified range
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          auiEntriesPerPxArray    Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          pui32CurrentLevel       Pointer to a variable which is set to our
+                                        current level
+
+@Input          uiStartIndex            Start index of the range to free
+
+@Input          uiEndIndex              End index of the range to free
+
+@Input			bFirst                  This is the first call for this level
+
+@Input			bLast                   This is the last call for this level
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static IMG_BOOL _MMU_FreeLevel(MMU_CONTEXT *psMMUContext,
+                               MMU_Levelx_INFO *psLevel,
+                               IMG_UINT32 auiStartArray[],
+                               IMG_UINT32 auiEndArray[],
+                               IMG_UINT32 auiEntriesPerPxArray[],
+                               const MMU_PxE_CONFIG *apsConfig[],
+                               MMU_LEVEL aeMMULevel[],
+                               IMG_UINT32 *pui32CurrentLevel,
+                               IMG_UINT32 uiStartIndex,
+                               IMG_UINT32 uiEndIndex,
+                               IMG_BOOL bFirst,
+                               IMG_BOOL bLast,
+                               IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 uiThisLevel = *pui32CurrentLevel;
+	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel];
+	IMG_UINT32 i;
+	IMG_BOOL bFreed = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Call before parameter check for debugging purposes. */
+	MMU_OBJ_DBG((PVR_DBG_ERROR,
+	             "_MMU_FreeLevel: level = %u, range %u - %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], uiStartIndex, uiEndIndex,
+	             (psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX)));
+
+	/* Parameter checks */
+	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
+
+	if (psLevel == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level data", __func__));
+		goto ErrReturn;
+	}
+
+	for (i = uiStartIndex; i < uiEndIndex; i++)
+	{
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			MMU_Levelx_INFO *psNextLevel = psLevel->apsNextLevel[i];
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			/* Recurse into the next level */
+			(*pui32CurrentLevel)++;
+			if (_MMU_FreeLevel(psMMUContext, psNextLevel, auiStartArray,
+			                   auiEndArray, auiEntriesPerPxArray,
+			                   apsConfig, aeMMULevel, pui32CurrentLevel,
+			                   uiNextStartIndex, uiNextEndIndex,
+			                   bNextFirst, bNextLast, uiLog2DataPageSize))
+			{
+				/* Un-wire the entry */
+				_SetupPxE(psMMUContext,
+				          psLevel,
+				          i,
+				          psConfig,
+				          aeMMULevel[uiThisLevel],
+				          NULL,
+#if defined(PDUMP)
+				          NULL,	/* Only required for data page */
+				          NULL,	/* Only required for data page */
+				          0,      /* Only required for data page */
+#endif
+				          MMU_PROTFLAGS_INVALID,
+				          uiLog2DataPageSize);
+
+				/* Free table of the level below, pointed to by this table entry.
+				 * We don't destroy the table inside the above _MMU_FreeLevel call because we
+				 * first have to set the table entry of the level above to invalid. */
+				_PxMemFree(psMMUContext, &psNextLevel->sMemDesc, aeMMULevel[*pui32CurrentLevel]);
+				OSFreeMem(psNextLevel);
+
+				/* The level below us is empty, drop the refcount and clear the pointer */
+				psLevel->ui32RefCount--;
+				psLevel->apsNextLevel[i] = NULL;
+
+				/* Check we haven't wrapped around */
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+			}
+			(*pui32CurrentLevel)--;
+		}
+		else
+		{
+			psLevel->ui32RefCount--;
+		}
+
+		/*
+		   Free this level if it is no longer referenced, unless it's the base
+		   level in which case it's part of the MMU context and should be freed
+		   when the MMU context is freed
+		 */
+		if ((psLevel->ui32RefCount == 0) && (psLevel != &psMMUContext->sBaseLevelInfo))
+		{
+			bFreed = IMG_TRUE;
+		}
+	}
+
+	/* Level one flushing is done when we actually write the table entries */
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		PVRSRV_ERROR eError;
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                   &psLevel->sMemDesc.psMapping->sMemHandle,
+		                   uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                   (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+ErrReturn:
+	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_FreeLevel end: level = %u, refcount = %u",
+	             aeMMULevel[uiThisLevel],
+	             bFreed ? 0 : (psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX)));
+
+	return bFreed;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_AllocLevel
+
+@Description    Recursively allocates the specified range of Px entries. If any
+                level has its last reference dropped then the MMU object
+                memory and the MMU_Levelx_Info will be freed.
+
+				At each level we might be crossing a boundary from one Px to
+				another. The values for auiStartArray should be by used for
+				the first call into each level and the values in auiEndArray
+				should only be used in the last call for each level.
+				In order to determine if this is the first/last call we pass
+				in bFirst and bLast.
+				When one level calls down to the next only if bFirst/bLast is set
+				and it's the first/last iteration of the loop at its level will
+				bFirst/bLast set for the next recursion.
+				This means that each iteration has the knowledge of the previous
+				level which is required.
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel                 Level info on which to free the
+                                        specified range
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          auiEntriesPerPxArray    Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          pui32CurrentLevel       Pointer to a variable which is set to our
+                                        current level
+
+@Input          uiStartIndex            Start index of the range to free
+
+@Input          uiEndIndex              End index of the range to free
+
+@Input			bFirst                  This is the first call for this level
+
+@Input			bLast                   This is the last call for this level
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _MMU_AllocLevel(MMU_CONTEXT *psMMUContext,
+                                    MMU_Levelx_INFO *psLevel,
+                                    IMG_UINT32 auiStartArray[],
+                                    IMG_UINT32 auiEndArray[],
+                                    IMG_UINT32 auiEntriesPerPxArray[],
+                                    const MMU_PxE_CONFIG *apsConfig[],
+                                    MMU_LEVEL aeMMULevel[],
+                                    IMG_UINT32 *pui32CurrentLevel,
+                                    IMG_UINT32 uiStartIndex,
+                                    IMG_UINT32 uiEndIndex,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+                                    IMG_DEV_VIRTADDR* psRunningDevVAddr,
+#endif
+                                    IMG_BOOL bFirst,
+                                    IMG_BOOL bLast,
+                                    IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 uiThisLevel = *pui32CurrentLevel; /* Starting with 0 */
+	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel]; /* The table config for the current level */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	IMG_UINT32 uiAllocState = 99; /* Debug info to check what progress was made in the function. Updated during this function. */
+	IMG_UINT32 i;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Parameter check */
+	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
+
+	MMU_OBJ_DBG((PVR_DBG_ERROR,
+	             "_MMU_AllocLevel: level = %u, range %u - %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], uiStartIndex, uiEndIndex,
+	             psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX));
+
+	/* Go from uiStartIndex to uiEndIndex through the Px */
+	for (i = uiStartIndex;i < uiEndIndex;i++)
+	{
+		/* Only try an allocation if this is not the last level */
+		/*Because a PT allocation is already done while setting the entry in PD */
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If there is already a next Px level existing, do not allocate it */
+			if (!psLevel->apsNextLevel[i])
+			{
+				MMU_Levelx_INFO *psNextLevel;
+				IMG_UINT32 ui32AllocSize;
+				IMG_UINT32 uiNextEntries;
+
+				/* Allocate and setup the next level */
+				uiNextEntries = auiEntriesPerPxArray[uiThisLevel + 1];
+				ui32AllocSize = sizeof(MMU_Levelx_INFO);
+				if (aeMMULevel[uiThisLevel + 1] != MMU_LEVEL_1)
+				{
+					ui32AllocSize += IMG_FLEX_ARRAY_SIZE(sizeof(MMU_Levelx_INFO *), uiNextEntries);
+				}
+				psNextLevel = OSAllocZMem(ui32AllocSize);
+				if (psNextLevel == NULL)
+				{
+					uiAllocState = 0;
+					goto e0;
+				}
+
+				/* Hook in this level for next time */
+				psLevel->apsNextLevel[i] = psNextLevel;
+
+				psNextLevel->ui32NumOfEntries = uiNextEntries;
+				psNextLevel->ui32RefCount = 0;
+				/* Allocate Px memory for a sub level*/
+				eError = _PxMemAlloc(psMMUContext, uiNextEntries, apsConfig[uiThisLevel + 1],
+				                     aeMMULevel[uiThisLevel + 1],
+				                     &psNextLevel->sMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+				                     psRunningDevVAddr,
+				                     uiLog2DataPageSize,
+#endif
+				                     psConfig->uiAddrLog2Align
+);
+				if (eError != PVRSRV_OK)
+				{
+					uiAllocState = 1;
+					goto e0;
+				}
+
+				/* Wire up the entry */
+				_SetupPxE(psMMUContext,
+				          psLevel,
+				          i,
+				          psConfig,
+				          aeMMULevel[uiThisLevel],
+				          &psNextLevel->sMemDesc.sDevPAddr,
+#if defined(PDUMP)
+				          NULL, /* Only required for data page */
+				          NULL, /* Only required for data page */
+				          0,    /* Only required for data page */
+#endif
+				          0,
+				          uiLog2DataPageSize);
+
+				psLevel->ui32RefCount++;
+			}
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+			else
+			{
+				/* The level structure already exists, increment running device virtual address
+				   This is necessary for correct parity bit calculation on further allocated page tables */
+				if (apsConfig[aeMMULevel[MMU_LEVEL_1]]->uiParityBitMask && aeMMULevel[uiThisLevel+1] == MMU_LEVEL_1)
+				{
+					psRunningDevVAddr->uiAddr += psLevel->apsNextLevel[i]->ui32NumOfEntries * (1 << uiLog2DataPageSize);
+				}
+			}
+#endif
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			/* Recurse into the next level */
+			(*pui32CurrentLevel)++;
+			eError = _MMU_AllocLevel(psMMUContext, psLevel->apsNextLevel[i],
+			                         auiStartArray,
+			                         auiEndArray,
+			                         auiEntriesPerPxArray,
+			                         apsConfig,
+			                         aeMMULevel,
+			                         pui32CurrentLevel,
+			                         uiNextStartIndex,
+			                         uiNextEndIndex,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+
+			                         psRunningDevVAddr,
+#endif
+			                         bNextFirst,
+			                         bNextLast,
+			                         uiLog2DataPageSize);
+			(*pui32CurrentLevel)--;
+			if (eError != PVRSRV_OK)
+			{
+				uiAllocState = 3;
+				goto e0;
+			}
+		}
+		else
+		{
+			/* All we need to do for level 1 is bump the refcount */
+			psLevel->ui32RefCount++;
+		}
+
+		if (psLevel->ui32RefCount > psLevel->ui32NumOfEntries)
+		{
+			/* Given how the reference counting is implemented for MMU_LEVEL_2
+			 * and MMU_LEVEL_3 this should never happen for those levels. Only
+			 * in case of MMU_LEVEL_1 this should be possible (e.g. when someone
+			 * takes multiple reservations of the same range). In such case
+			 * return error to prevent reference count to rollover. */
+			uiAllocState = 4;
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, e0);
+		}
+	}
+
+	/* Level one flushing is done when we actually write the table entries */
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, e1);
+	}
+
+	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_AllocLevel end: level = %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], psLevel->ui32RefCount));
+	return PVRSRV_OK;
+
+e0:
+	/* Confirm that we've not come down this route unexpectedly */
+	PVR_ASSERT(uiAllocState!=99);
+	PVR_DPF((PVR_DBG_ERROR, "_MMU_AllocLevel: Error %d allocating Px for level %d in stage %d"
+			,eError, aeMMULevel[uiThisLevel], uiAllocState));
+
+	/* In case of an error process current `i` in special way without
+	 * recursively calling `_MMU_FreeLevel()`. */
+
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		/* If this is not a PT (so it's a PC or PD) it means that an error
+		 * happened either during PT or PD allocation/setup.
+		 *
+		 * - If the error happened during PT allocation/setup it means that
+		 *   we're at the PD level now and we already run below `else` clause
+		 *   for the failing PT and the `for` loop below for the other PTs. This
+		 *   means that all of the PTs that have been referenced during this
+		 *   operation have been dereferenced. So we just need to free that
+		 *   failing PT here and the rest of them in the loop below.
+		 * - If the error happened during PD allocation/setup it means that
+		 *   we're at the PC level now and we already run below `else` clause
+		 *   for the failing PD and the `for` loop below for the other PDs. This
+		 *   means that all of the PTs that have been allocated during this
+		 *   operation for the failing PD have been freed. So we just need to
+		 *   free that failing PD and the rest of them in a recursive manner
+		 *   in the loop below. */
+		if (uiAllocState >= 3)
+		{
+			if (psLevel->apsNextLevel[i] != NULL &&
+			    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+			{
+				psLevel->ui32RefCount--;
+			}
+		}
+		if (uiAllocState >= 2)
+		{
+			if (psLevel->apsNextLevel[i] != NULL &&
+			    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+			{
+				_PxMemFree(psMMUContext, &psLevel->apsNextLevel[i]->sMemDesc,
+				           aeMMULevel[uiThisLevel + 1]);
+			}
+		}
+		if (psLevel->apsNextLevel[i] != NULL &&
+		    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+		{
+			OSFreeMem(psLevel->apsNextLevel[i]);
+			psLevel->apsNextLevel[i] = NULL;
+		}
+	}
+	else
+	{
+		/* This is a PT which means that we just need to dereference it. It's
+		 * going to be freed on the PD level in this error path in the `if`
+		 * clause above. */
+		psLevel->ui32RefCount--;
+	}
+
+	/* Check we haven't wrapped around */
+	PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+
+e1:
+	i--;
+
+	/* The start value of index variable i is not initialised on purpose.
+	 * This clean-up loop deinitialises what was already initialised in
+	 * reverse order, so the i index already has the correct value.
+	 */
+	for (/* i already set */; i >= uiStartIndex && i < uiEndIndex; i--)
+	{
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			(*pui32CurrentLevel)++;
+			if (_MMU_FreeLevel(psMMUContext, psLevel->apsNextLevel[i],
+			                   auiStartArray, auiEndArray,
+			                   auiEntriesPerPxArray, apsConfig,
+			                   aeMMULevel, pui32CurrentLevel,
+			                   uiNextStartIndex, uiNextEndIndex,
+			                   bNextFirst, bNextLast, uiLog2DataPageSize))
+			{
+				_PxMemFree(psMMUContext, &psLevel->apsNextLevel[i]->sMemDesc,
+				           aeMMULevel[uiThisLevel + 1]);
+				OSFreeMem(psLevel->apsNextLevel[i]);
+				psLevel->apsNextLevel[i] = NULL;
+
+				psLevel->ui32RefCount--;
+
+				/* Check we haven't wrapped around */
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+			}
+			(*pui32CurrentLevel)--;
+		}
+		else
+		{
+			/* We should never come down this path, but it's here
+			   for completeness */
+			psLevel->ui32RefCount--;
+
+			/* Check we haven't wrapped around */
+			PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+		}
+	}
+
+	return eError;
+}
+
+/*****************************************************************************
+ *                   MMU page table functions                                *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetLevelData
+
+@Description    Get the all the level data and calculates the indexes for the
+                specified address range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Log2 of the page size to use
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          uiEntriesPerPxArray     Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          ppsMMUDevVAddrConfig    Device virtual address config
+
+@Input			phPriv					Private data of page size config
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static void _MMU_GetLevelData(MMU_CONTEXT *psMMUContext,
+                              IMG_DEV_VIRTADDR sDevVAddrStart,
+                              IMG_DEV_VIRTADDR sDevVAddrEnd,
+                              IMG_UINT32 uiLog2DataPageSize,
+                              IMG_UINT32 auiStartArray[],
+                              IMG_UINT32 auiEndArray[],
+                              IMG_UINT32 auiEntriesPerPx[],
+                              const MMU_PxE_CONFIG *apsConfig[],
+                              MMU_LEVEL aeMMULevel[],
+                              const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+                              IMG_HANDLE *phPriv)
+{
+	const MMU_PxE_CONFIG *psMMUPDEConfig;
+	const MMU_PxE_CONFIG *psMMUPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i = 0;
+
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
+	                                                 &psMMUPDEConfig,
+	                                                 &psMMUPTEConfig,
+	                                                 ppsMMUDevVAddrConfig,
+	                                                 phPriv);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "GetPageSizeConfiguration");
+
+	psDevVAddrConfig = *ppsMMUDevVAddrConfig;
+
+	if (psDevVAddrConfig->uiPCIndexMask != 0)
+	{
+		auiStartArray[i] = _CalcPCEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+		auiEndArray[i] = _CalcPCEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPC;
+		apsConfig[i] = psDevAttrs->psBaseConfig;
+		aeMMULevel[i] = MMU_LEVEL_3;
+		i++;
+	}
+
+	if (psDevVAddrConfig->uiPDIndexMask != 0)
+	{
+		auiStartArray[i] = _CalcPDEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+		auiEndArray[i] = _CalcPDEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPD;
+		if (i == 0)
+		{
+			apsConfig[i] = psDevAttrs->psBaseConfig;
+		}
+		else
+		{
+			apsConfig[i] = psMMUPDEConfig;
+		}
+		aeMMULevel[i] = MMU_LEVEL_2;
+		i++;
+	}
+
+	/*
+		There is always a PTE entry so we have a slightly different behaviour than above.
+		E.g. for 2 MB pages the uiPTIndexMask is 0x0000000000 but still there
+		is a PT with one entry.
+
+	 */
+	auiStartArray[i] = _CalcPTEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+	if (psDevVAddrConfig->uiPTIndexMask !=0)
+	{
+		auiEndArray[i] = _CalcPTEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+	}
+	else
+	{
+		/*
+			If the PTE mask is zero it means there is only 1 PTE and thus, as an
+			an exclusive bound, the end array index is equal to the start index + 1.
+		 */
+
+		auiEndArray[i] = auiStartArray[i] + 1;
+	}
+
+	auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPT;
+
+	if (i == 0)
+	{
+		apsConfig[i] = psDevAttrs->psBaseConfig;
+	}
+	else
+	{
+		apsConfig[i] = psMMUPTEConfig;
+	}
+	aeMMULevel[i] = MMU_LEVEL_1;
+}
+
+static void _MMU_PutLevelData(MMU_CONTEXT *psMMUContext, IMG_HANDLE hPriv)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+}
+
+/*************************************************************************/ /*!
+@Function       _AllocPageTables
+
+@Description    Allocate page tables and any higher level MMU objects required
+                for the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Page size of the data pages
+
+@Return         PVRSRV_OK if the allocation was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR
+_AllocPageTables(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrStart,
+                 IMG_DEV_VIRTADDR sDevVAddrEnd,
+                 IMG_UINT32 uiLog2DataPageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
+	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
+	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
+	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+	IMG_UINT32 ui32CurrentLevel = 0;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_DEV_VIRTADDR sRunningDevVAddrStart;
+	IMG_BOOL bSetParity;
+#endif
+
+	PVR_DPF((PVR_DBG_ALLOC,
+			"_AllocPageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+			sDevVAddrStart.uiAddr,
+			sDevVAddrEnd.uiAddr
+	));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode,
+	             "Allocating page tables for %"IMG_UINT64_FMTSPEC" bytes virtual range: "
+	             IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+	             (IMG_UINT64)sDevVAddrEnd.uiAddr - (IMG_UINT64)sDevVAddrStart.uiAddr,
+	             (IMG_UINT64)sDevVAddrStart.uiAddr,
+	             (IMG_UINT64)sDevVAddrEnd.uiAddr);
+#endif
+
+	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
+	                  (IMG_UINT32) uiLog2DataPageSize, auiStartArray, auiEndArray,
+	                  auiEntriesPerPx, apsConfig, aeMMULevel,
+	                  &psDevVAddrConfig, &hPriv);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	bSetParity = apsConfig[aeMMULevel[MMU_LEVEL_1]]->uiParityBitMask != 0;
+	if (bSetParity)
+	{
+		/* If parity bit needs to be written for PTEs save the first VA of the PT */
+		sRunningDevVAddrStart.uiAddr = sDevVAddrStart.uiAddr & ~((IMG_UINT64)(auiEntriesPerPx[MMU_LEVEL_1] * (1 << uiLog2DataPageSize)) - 1);
+	}
+
+#endif
+
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_ALLOC,
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));
+
+	eError = _MMU_AllocLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
+	                         auiStartArray, auiEndArray, auiEntriesPerPx,
+	                         apsConfig, aeMMULevel, &ui32CurrentLevel,
+	                         auiStartArray[0], auiEndArray[0],
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	                         &sRunningDevVAddrStart,
+#endif
+	                         IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP) && defined(DEBUG)
+	if (bSetParity)
+	{
+		PVR_ASSERT(sRunningDevVAddrStart.uiAddr >= sDevVAddrEnd.uiAddr);
+	}
+#endif
+
+	_MMU_PutLevelData(psMMUContext, hPriv);
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreePageTables
+
+@Description    Free page tables and any higher level MMU objects at are no
+                longer referenced for the specified virtual range.
+                This will fill the temporary free list of the MMU context which
+                needs cleanup after the call.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Page size of the data pages
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _FreePageTables(MMU_CONTEXT *psMMUContext,
+                            IMG_DEV_VIRTADDR sDevVAddrStart,
+                            IMG_DEV_VIRTADDR sDevVAddrEnd,
+                            IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
+	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
+	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
+	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
+	IMG_UINT32 ui32CurrentLevel = 0;
+	IMG_HANDLE hPriv;
+
+	PVR_DPF((PVR_DBG_ALLOC,
+			"_FreePageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+			sDevVAddrStart.uiAddr,
+			sDevVAddrEnd.uiAddr
+	));
+
+	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
+	                  uiLog2DataPageSize, auiStartArray, auiEndArray,
+	                  auiEntriesPerPx, apsConfig, aeMMULevel,
+	                  &psDevVAddrConfig, &hPriv);
+
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_FREE,
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));
+
+	/* ignoring return code, in this case there should be no references
+	 * to the level anymore, and at this stage there is nothing to do with
+	 * the return status */
+	(void) _MMU_FreeLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
+	                      auiStartArray, auiEndArray, auiEntriesPerPx,
+	                      apsConfig, aeMMULevel, &ui32CurrentLevel,
+	                      auiStartArray[0], auiEndArray[0],
+	                      IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);
+
+	_MMU_PutLevelData(psMMUContext, hPriv);
+}
+
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetPTInfo
+
+@Description    Get the PT level information and PT entry index for the specified
+                virtual address
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          psDevVAddr              Device virtual address to get the PTE info
+                                        from.
+
+@Input          psDevVAddrConfig        The current virtual address config obtained
+                                        by another function call before.
+
+@Output         psLevel                 Level info of the PT
+
+@Output         pui32PTEIndex           Index into the PT the address corresponds to
+
+@Return         IMG_TRUE if the operation was successful and IMG_FALSE otherwise
+ */
+/*****************************************************************************/
+static INLINE IMG_BOOL _MMU_GetPTInfo(MMU_CONTEXT                *psMMUContext,
+                                      IMG_DEV_VIRTADDR            sDevVAddr,
+                                      const MMU_DEVVADDR_CONFIG  *psDevVAddrConfig,
+                                      MMU_Levelx_INFO           **ppsLevel,
+                                      IMG_UINT32                 *pui32PTEIndex)
+{
+	MMU_Levelx_INFO *psLocalLevel = NULL;
+	MMU_LEVEL eMMULevel = psMMUContext->psDevAttrs->psBaseConfig->ePxLevel;
+	const MMU_LEVEL eMMUBaseLevel = eMMULevel;
+	IMG_UINT32 uiPCEIndex;
+	IMG_UINT32 uiPDEIndex;
+
+	if ((eMMULevel <= MMU_LEVEL_0) || (eMMULevel >= MMU_LEVEL_LAST))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTEInfo: Invalid MMU level"));
+		PVR_ASSERT(0);
+	}
+
+	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
+	{
+		if (eMMULevel == MMU_LEVEL_3)
+		{
+			/* find the page directory pointed by the PCE */
+			uiPCEIndex = _CalcPCEIdx (sDevVAddr, psDevVAddrConfig,
+			                          IMG_FALSE);
+			psLocalLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiPCEIndex];
+		}
+
+		if (eMMULevel == MMU_LEVEL_2)
+		{
+			/* find the page table pointed by the PDE */
+			uiPDEIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+			if (psLocalLevel == NULL)
+			{
+				return IMG_FALSE;
+			}
+
+			psLocalLevel = psLocalLevel->apsNextLevel[uiPDEIndex];
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			/* find PTE index into page table */
+			*pui32PTEIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLocalLevel == NULL)
+			{
+				if (eMMUBaseLevel == eMMULevel)
+				{
+					/* if the MMU only supports one level return the base level */
+					psLocalLevel = &psMMUContext->sBaseLevelInfo;
+				}
+				else
+				{
+					return IMG_FALSE;
+				}
+			}
+		}
+	}
+
+	*ppsLevel = psLocalLevel;
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetPTConfig
+
+@Description    Get the level config. Call _MMU_PutPTConfig after use!
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiLog2DataPageSize      Log 2 of the page size
+
+@Output         ppsConfig               Config of the PTE
+
+@Output         phPriv                  Private data handle to be passed back
+                                        when the info is put
+
+@Output         ppsDevVAddrConfig       Config of the device virtual addresses
+
+@Return         None
+ */
+/*****************************************************************************/
+static INLINE void _MMU_GetPTConfig(MMU_CONTEXT               *psMMUContext,
+                                    IMG_UINT32                  uiLog2DataPageSize,
+                                    const MMU_PxE_CONFIG      **ppsConfig,
+                                    IMG_HANDLE                 *phPriv,
+                                    const MMU_DEVVADDR_CONFIG **ppsDevVAddrConfig)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+
+	if (psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
+	                                            &psPDEConfig,
+	                                            &psPTEConfig,
+	                                            &psDevVAddrConfig,
+	                                            phPriv) != PVRSRV_OK)
+	{
+		/*
+		   There should be no way we got here unless uiLog2DataPageSize
+		   has changed after the MMU_Alloc call (in which case it's a bug in
+		   the MM code)
+		 */
+		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTConfig: Could not get valid page size config"));
+		PVR_ASSERT(0);
+	}
+
+	*ppsConfig = psPTEConfig;
+	*ppsDevVAddrConfig = psDevVAddrConfig;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PutPTConfig
+
+@Description    Put the level info. Has to be called after _MMU_GetPTConfig to
+                ensure correct refcounting.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          phPriv                  Private data handle created by
+                                        _MMU_GetPTConfig.
+
+@Return         None
+ */
+/*****************************************************************************/
+static INLINE void _MMU_PutPTConfig(MMU_CONTEXT *psMMUContext,
+                                    IMG_HANDLE hPriv)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	if (psDevAttrs->pfnPutPageSizeConfiguration(hPriv) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Could not put page size config",
+				__func__));
+		PVR_ASSERT(0);
+	}
+}
+
+/* scratch / zero pages */
+
+static INLINE PVRSRV_ERROR _MMU_GetDefPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                           IMG_INT uiDefPage,
+                                           PVRSRV_DEF_PAGE **ppsDefPage,
+                                           IMG_CHAR **ppcDefPageName)
+{
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(ppcDefPageName);
+#endif
+
+	switch (uiDefPage)
+	{
+		case SCRATCH_PAGE:
+		{
+			*ppsDefPage = &psDevNode->sScratchPage;
+#if defined(PDUMP)
+			*ppcDefPageName = SCRATCH_PAGE_STR;
+#endif
+			break;
+		}
+		case DEV_ZERO_PAGE:
+		{
+			*ppsDefPage = &psDevNode->sDevZeroPage;
+#if defined(PDUMP)
+			*ppcDefPageName = DEV_ZERO_PAGE_STR;
+#endif
+			break;
+		}
+		default:
+		{
+			/* Invalid pcDefPageName */
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MMU_AllocBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                          IMG_INT uiInitValue,
+                                          IMG_INT uiDefPage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR	sDevPAddr = {0};
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+
+	eError = _MMU_GetDefPage(psDevNode,
+	                         uiDefPage,
+	                         &psDefPage,
+	                         &pcDefPageName);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr != MMU_BAD_PHYS_ADDR)
+	{
+		goto UnlockAndReturn;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Alloc %s page object", pcDefPageName);
+#endif
+
+	/* Allocate the scratch / zero page required for physical backing
+	 * of virtual ranges
+	 */
+	eError = DevPhysMemAlloc(psDevNode,
+	                         (1 << psDefPage->ui32Log2PgSize),
+	                         0,
+	                         uiInitValue,
+	                         IMG_TRUE,
+#if defined(PDUMP)
+	                         psDevNode->psMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                         pcDefPageName,
+	                         &psDefPage->hPdumpPg,
+#endif
+	                         PVR_SYS_ALLOC_PID,
+	                         &psDefPage->sPageHandle,
+	                         &sDevPAddr);
+
+	psDefPage->ui64PgPhysAddr = sDevPAddr.uiAddr;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _MMU_GetBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                        IMG_UINT64 *pui64PgPhysAddr,
+                                        IMG_INT uiDefPage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+
+	eError = _MMU_GetDefPage(psDevNode,
+	                         uiDefPage,
+	                         &psDefPage,
+	                         &pcDefPageName);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr == MMU_BAD_PHYS_ADDR)
+	{
+		eError = PVRSRV_ERROR_NOT_FOUND;
+		PVR_DPF((PVR_DBG_ERROR,"%s: %s (uiDefPage=%d)",
+		         __func__, PVRSRVGetErrorString(eError), uiDefPage));
+		goto UnlockAndReturn;
+	}
+
+	if (pui64PgPhysAddr)
+		*pui64PgPhysAddr = psDefPage->ui64PgPhysAddr;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+
+	return eError;
+}
+
+static void _MMU_FreeBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                 IMG_INT uiDefPage)
+{
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+	PVRSRV_ERROR eErr;
+
+	eErr = _MMU_GetDefPage(psDevNode,
+	                       uiDefPage,
+	                       &psDefPage,
+	                       &pcDefPageName);
+	if (eErr != PVRSRV_OK)
+		return;
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr == MMU_BAD_PHYS_ADDR)
+	{
+		goto UnlockAndReturn;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Free %s page object", pcDefPageName);
+#endif
+
+	DevPhysMemFree(psDevNode,
+#if defined(PDUMP)
+	               psDefPage->hPdumpPg,
+#endif
+	               &psDefPage->sPageHandle);
+
+#if defined(PDUMP)
+	psDefPage->hPdumpPg = NULL;
+#endif
+	psDefPage->ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+}
+
+
+/*****************************************************************************
+ *                     Public interface functions                            *
+ *****************************************************************************/
+
+/*
+	MMU_InitDevice
+*/
+PVRSRV_ERROR MMU_InitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Set the order to 0 */
+	psDevNode->sScratchPage.sPageHandle.uiOrder = 0;
+	psDevNode->sDevZeroPage.sPageHandle.uiOrder = 0;
+
+	/* Set the size of the Scratch and Zero pages to largest page size */
+	if (psDevNode->ui32Non4KPageSizeLog2 != 0)
+	{
+		psDevNode->sScratchPage.ui32Log2PgSize = psDevNode->ui32Non4KPageSizeLog2;
+		psDevNode->sDevZeroPage.ui32Log2PgSize = psDevNode->ui32Non4KPageSizeLog2;
+	}
+	else
+	{
+		psDevNode->sScratchPage.ui32Log2PgSize = OSGetPageShift();
+		psDevNode->sDevZeroPage.ui32Log2PgSize = OSGetPageShift();
+	}
+
+	/* Set the Scratch page phys addr */
+	psDevNode->sScratchPage.ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+	/* Set the Zero page phys addr */
+	psDevNode->sDevZeroPage.ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+	/* The lock can be acquired from MISR (Z-buffer) path */
+	eError = OSLockCreate(&psDevNode->sScratchPage.psPgLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.Scratch", ErrReturnError);
+
+	/* Create the lock for zero page */
+	eError = OSLockCreate(&psDevNode->sDevZeroPage.psPgLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.Zero", ErrFreeScratchPageLock);
+
+#ifdef PDUMP
+	psDevNode->sScratchPage.hPdumpPg = NULL;
+	psDevNode->sDevZeroPage.hPdumpPg = NULL;
+#endif /* PDUMP */
+
+	eError = _MMU_AllocBackingPage(psDevNode,
+	                               PVR_SCRATCH_PAGE_INIT_VALUE,
+	                               SCRATCH_PAGE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_AllocBackingPage.Scratch", ErrFreeZeroPageLock);
+
+	eError = _MMU_AllocBackingPage(psDevNode,
+	                               PVR_ZERO_PAGE_INIT_VALUE,
+	                               DEV_ZERO_PAGE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_AllocBackingPage.Zero", ErrFreeScratchPage);
+
+	return PVRSRV_OK;
+
+ErrFreeScratchPage:
+	_MMU_FreeBackingPage(psDevNode, SCRATCH_PAGE);
+ErrFreeZeroPageLock:
+	OSLockDestroy(psDevNode->sDevZeroPage.psPgLock);
+	psDevNode->sDevZeroPage.psPgLock = NULL;
+ErrFreeScratchPageLock:
+	OSLockDestroy(psDevNode->sScratchPage.psPgLock);
+	psDevNode->sScratchPage.psPgLock = NULL;
+ErrReturnError:
+	return eError;
+}
+
+/*
+	MMU_DeInitDevice
+*/
+void MMU_DeInitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode)
+{
+	if (psDevNode->sScratchPage.psPgLock != NULL)
+	{
+		_MMU_FreeBackingPage(psDevNode, SCRATCH_PAGE);
+
+		OSLockDestroy(psDevNode->sScratchPage.psPgLock);
+		psDevNode->sScratchPage.psPgLock = NULL;
+	}
+
+	if (psDevNode->sDevZeroPage.psPgLock)
+	{
+		_MMU_FreeBackingPage(psDevNode, DEV_ZERO_PAGE);
+
+
+		OSLockDestroy(psDevNode->sDevZeroPage.psPgLock);
+		psDevNode->sDevZeroPage.psPgLock = NULL;
+	}
+}
+
+/*
+	MMU_ContextCreate
+ */
+PVRSRV_ERROR
+MMU_ContextCreate(CONNECTION_DATA *psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  MMU_CONTEXT **ppsMMUContext,
+                  MMU_DEVICEATTRIBS *psDevAttrs)
+{
+	MMU_CONTEXT *psMMUContext;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	const MMU_PxE_CONFIG *psConfig;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx;
+	IMG_UINT32 ui32BaseObjects;
+	IMG_UINT32 ui32Size;
+	IMG_CHAR sBuf[40];
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "MMU context create");
+#endif
+
+	psConfig = psDevAttrs->psBaseConfig;
+	psDevVAddrConfig = psDevAttrs->psTopLevelDevVAddrConfig;
+
+	switch (psDevAttrs->psBaseConfig->ePxLevel)
+	{
+		case MMU_LEVEL_3:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPC;
+			break;
+
+		case MMU_LEVEL_2:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPD;
+			break;
+
+		case MMU_LEVEL_1:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPT;
+			break;
+
+		default:
+			PVR_LOG_GOTO_WITH_ERROR("psDevAttrs->psBaseConfig->ePxLevel", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+	}
+
+	/* Allocate the MMU context with the Level 1 Px info's */
+	ui32Size = sizeof(MMU_CONTEXT) +
+			((ui32BaseObjects - 1) * sizeof(MMU_Levelx_INFO *));
+
+	psMMUContext = OSAllocZMem(ui32Size);
+	PVR_LOG_GOTO_IF_NOMEM(psMMUContext, eError, e0);
+
+#if defined(PDUMP)
+	/* Clear the refcount */
+	psMMUContext->ui32PDumpContextIDRefCount = 0;
+#endif
+	/* Record Device specific attributes in the context for subsequent use */
+	psMMUContext->psDevAttrs = psDevAttrs;
+
+	/*
+	  Allocate physmem context and set it up
+	 */
+	psPhysMemCtx = OSAllocZMem(sizeof(MMU_PHYSMEM_CONTEXT));
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx, eError, e1);
+
+	psMMUContext->psPhysMemCtx = psPhysMemCtx;
+	psMMUContext->psConnection = psConnection;
+
+	psPhysMemCtx->psDevNode = psDevNode;		/* Needed for Direct Bridge case */
+	psPhysMemCtx->psMMUContext = psMMUContext;	/* Back-link to self */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/* Save the app-specific values for external reference via MMU_GetOSids. */
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+	{
+		psPhysMemCtx->ui32OSid     = psConnection->ui32OSid;
+		psPhysMemCtx->ui32OSidReg  = psConnection->ui32OSidReg;
+		psPhysMemCtx->bOSidAxiProt = psConnection->bOSidAxiProtReg;
+	}
+	else
+	{
+		/* Direct Bridge calling sequence e.g. Firmware */
+		psPhysMemCtx->ui32OSid     = 0;
+		psPhysMemCtx->ui32OSidReg  = 0;
+		psPhysMemCtx->bOSidAxiProt = IMG_FALSE;
+	}
+#endif
+
+	OSSNPrintf(sBuf, sizeof(sBuf), "pgtables %p", psPhysMemCtx);
+	psPhysMemCtx->uiPhysMemRANameAllocSize = OSStringLength(sBuf)+1;
+	psPhysMemCtx->pszPhysMemRAName = OSAllocMem(psPhysMemCtx->uiPhysMemRANameAllocSize);
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx->pszPhysMemRAName, eError, e2);
+
+	OSStringSafeCopy(psPhysMemCtx->pszPhysMemRAName, sBuf, psPhysMemCtx->uiPhysMemRANameAllocSize);
+
+	psPhysMemCtx->psPhysMemRA = RA_Create(psPhysMemCtx->pszPhysMemRAName,
+	                                      /* subsequent import */
+	                                      PhysHeapGetPageShift(psDevNode->psMMUPhysHeap),
+	                                      RA_LOCKCLASS_2,
+	                                      _MMU_PhysMem_RAImportAlloc,
+	                                      _MMU_PhysMem_RAImportFree,
+	                                      psPhysMemCtx, /* priv */
+	                                      RA_POLICY_DEFAULT);
+	if (psPhysMemCtx->psPhysMemRA == NULL)
+	{
+		OSFreeMem(psPhysMemCtx->pszPhysMemRAName);
+		psPhysMemCtx->pszPhysMemRAName = NULL;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, e3);
+	}
+
+	/* Setup cleanup meta data to check if a MMU context
+	 * has been destroyed and should not be accessed anymore */
+	psPhysMemCtx->psCleanupData = OSAllocMem(sizeof(*(psPhysMemCtx->psCleanupData)));
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx->psCleanupData, eError, e4);
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/* Record the originating OSid for all allocation / free for this context */
+	psPhysMemCtx->psCleanupData->ui32OSid = psPhysMemCtx->ui32OSid;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+	OSLockCreate(&psPhysMemCtx->psCleanupData->hCleanupLock);
+	psPhysMemCtx->psCleanupData->bMMUContextExists = IMG_TRUE;
+	dllist_init(&psPhysMemCtx->psCleanupData->sMMUCtxCleanupItemsHead);
+	OSAtomicWrite(&psPhysMemCtx->psCleanupData->iRef, 1);
+
+	/* allocate the base level object */
+	/*
+	   Note: Although this is not required by the this file until
+	         the 1st allocation is made, a device specific callback
+	         might request the base object address so we allocate
+	         it up front.
+	         In VZ we only need to allocate the FW/KM ctx on the
+	         Host Driver as all Guest tables are allocated and
+	         pre-mapped into the host FW Memory Context.
+	         Drivers with security support and premapped heaps
+	         rely on the TEE to manage the Firmware mappings.
+	 */
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#else
+	if (!_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+#endif
+	{
+		if (_PxMemAlloc(psMMUContext,
+						ui32BaseObjects,
+						psConfig,
+						psDevAttrs->psBaseConfig->ePxLevel,
+						&psMMUContext->sBaseLevelInfo.sMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+						NULL,
+						0U,
+#endif
+						psDevAttrs->ui32BaseAlign))
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PxMemAlloc", eError, PVRSRV_ERROR_OUT_OF_MEMORY, e5);
+		}
+	}
+
+	dllist_init(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
+
+	psMMUContext->sBaseLevelInfo.ui32NumOfEntries = ui32BaseObjects;
+	psMMUContext->sBaseLevelInfo.ui32RefCount = 0;
+
+	/* Support cores that need to adjust the base level Px */
+	if (psDevNode->pfnMMUTopLevelPxWorkarounds != NULL)
+	{
+		PVRSRV_ERROR eError;
+		MMU_MEMORY_DESC  *psMemDesc  = &psMMUContext->sBaseLevelInfo.sMemDesc;
+
+		psDevNode->pfnMMUTopLevelPxWorkarounds(psConnection,
+		                                     psDevNode,
+		                                     psMemDesc->sDevPAddr,
+		                                     psMemDesc->pvCpuVAddr);
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psMemDesc->psMapping->sMemHandle,
+		                            psMemDesc->uiOffset,
+		                            psMemDesc->uiSize);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+	eError = OSLockCreate(&psMMUContext->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e6);
+
+	/* return context */
+	*ppsMMUContext = psMMUContext;
+
+	return PVRSRV_OK;
+
+e6:
+	_PxMemFree(psMMUContext, &psMMUContext->sBaseLevelInfo.sMemDesc, psDevAttrs->psBaseConfig->ePxLevel);
+e5:
+	OSFreeMem(psPhysMemCtx->psCleanupData);
+e4:
+	RA_Delete(psPhysMemCtx->psPhysMemRA);
+e3:
+	OSFreeMem(psPhysMemCtx->pszPhysMemRAName);
+e2:
+	OSFreeMem(psPhysMemCtx);
+e1:
+	OSFreeMem(psMMUContext);
+e0:
+	return eError;
+}
+
+/*
+	MMU_ContextDestroy
+ */
+void
+MMU_ContextDestroy (MMU_CONTEXT *psMMUContext)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PDLLIST_NODE psNode, psNextNode;
+
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psMMUContext->psPhysMemCtx->psDevNode;
+	MMU_CTX_CLEANUP_DATA *psCleanupData = psMMUContext->psPhysMemCtx->psCleanupData;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Enter", __func__));
+
+	if (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
+	{
+		/* There should be no way to get here with live pages unless
+		   there is a bug in this module or the MM code */
+		PVR_ASSERT(psMMUContext->sBaseLevelInfo.ui32RefCount == 0);
+	}
+
+	/* Cleanup lock must be acquired before MMUContext lock. Reverse order
+	 * may lead to a deadlock and is reported by lockdep. */
+	OSLockAcquire(psCleanupData->hCleanupLock);
+	OSLockAcquire(psMMUContext->hLock);
+
+	/* Free the top level MMU object - will be put on defer free list.
+	 * This has to be done before the step below that will empty the
+	 * defer-free list. */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#else
+	if (!_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+#endif
+	{
+		_PxMemFree(psMMUContext,
+		           &psMMUContext->sBaseLevelInfo.sMemDesc,
+		           psMMUContext->psDevAttrs->psBaseConfig->ePxLevel);
+	}
+
+	/* Empty the temporary defer-free list of Px */
+	_FreeMMUMapping(psDevNode, &psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
+	PVR_ASSERT(dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead));
+
+	/* Empty the defer free list so the cleanup thread will
+	 * not have to access any MMU context related structures anymore */
+	dllist_foreach_node(&psCleanupData->sMMUCtxCleanupItemsHead,
+	                    psNode,
+	                    psNextNode)
+	{
+		MMU_CLEANUP_ITEM *psCleanup = IMG_CONTAINER_OF(psNode,
+		                                               MMU_CLEANUP_ITEM,
+		                                               sMMUCtxCleanupItem);
+
+		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);
+
+		dllist_remove_node(psNode);
+	}
+	PVR_ASSERT(dllist_is_empty(&psCleanupData->sMMUCtxCleanupItemsHead));
+
+	psCleanupData->bMMUContextExists = IMG_FALSE;
+
+	/* Free physmem context */
+	RA_Delete(psMMUContext->psPhysMemCtx->psPhysMemRA);
+	psMMUContext->psPhysMemCtx->psPhysMemRA = NULL;
+	OSFreeMem(psMMUContext->psPhysMemCtx->pszPhysMemRAName);
+	psMMUContext->psPhysMemCtx->pszPhysMemRAName = NULL;
+
+	OSFreeMem(psMMUContext->psPhysMemCtx);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	OSLockRelease(psCleanupData->hCleanupLock);
+
+	if (OSAtomicDecrement(&psCleanupData->iRef) == 0)
+	{
+		OSLockDestroy(psCleanupData->hCleanupLock);
+		OSFreeMem(psCleanupData);
+	}
+
+	OSLockDestroy(psMMUContext->hLock);
+
+	/* free the context itself. */
+	OSFreeMem(psMMUContext);
+	/*not nulling pointer, copy on stack*/
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Exit", __func__));
+}
+
+/*
+	MMU_Alloc
+ */
+PVRSRV_ERROR
+MMU_Alloc (MMU_CONTEXT *psMMUContext,
+           IMG_DEVMEM_SIZE_T uSize,
+           IMG_UINT32 uiProtFlags,
+           IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
+           IMG_DEV_VIRTADDR *psDevVAddr,
+           IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevVAddrEnd;
+
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	MMU_DEVICEATTRIBS *psDevAttrs;
+	IMG_HANDLE hPriv;
+
+#if !defined(DEBUG)
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uDevVAddrAlignment);
+#endif
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: uSize=" IMG_DEVMEM_SIZE_FMTSPEC
+			", uiProtFlags=0x%x, align="IMG_DEVMEM_ALIGN_FMTSPEC,
+			__func__, uSize, uiProtFlags, uDevVAddrAlignment));
+
+	/* check params */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevVAddr, "psDevVAddr");
+
+	psDevAttrs = psMMUContext->psDevAttrs;
+
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2PageSize,
+	                                                 &psPDEConfig,
+	                                                 &psPTEConfig,
+	                                                 &psDevVAddrConfig,
+	                                                 &hPriv);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnGetPageSizeConfiguration");
+
+	/* size and alignment must be datapage granular */
+	if (((psDevVAddr->uiAddr & psDevVAddrConfig->uiPageOffsetMask) != 0)
+			|| ((uSize & psDevVAddrConfig->uiPageOffsetMask) != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: invalid address or size granularity",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sDevVAddrEnd = *psDevVAddr;
+	sDevVAddrEnd.uiAddr += uSize;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (_MMU_IS_FWKM_CTX(psMMUContext))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver relying on firmware mappings created by the TEE.", __func__));
+	}
+	else
+#endif
+	{
+		eError = _AllocPageTables(psMMUContext, *psDevVAddr, sDevVAddrEnd, uiLog2PageSize);
+	}
+	OSLockRelease(psMMUContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "_AllocPageTables");
+		return PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES;
+	}
+
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_Free
+ */
+void
+MMU_Free (MMU_CONTEXT *psMMUContext,
+          IMG_DEV_VIRTADDR sDevVAddr,
+          IMG_DEVMEM_SIZE_T uiSize,
+          IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_DEV_VIRTADDR sDevVAddrEnd;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx;
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+#endif
+
+
+	PVR_ASSERT(psMMUContext != NULL);
+	PVR_LOG_RETURN_VOID_IF_FALSE(psMMUContext != NULL, "psMMUContext");
+
+	psPhysMemCtx = psMMUContext->psPhysMemCtx;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freeing DevVAddr " IMG_DEV_VIRTADDR_FMTSPEC,
+			__func__, sDevVAddr.uiAddr));
+
+	/* ensure the address range to free is inside the heap */
+	sDevVAddrEnd = sDevVAddr;
+	sDevVAddrEnd.uiAddr += uiSize;
+
+	/* The Cleanup lock has to be taken before the MMUContext hLock to
+	 * prevent deadlock scenarios. It is necessary only for parts of
+	 * _SetupCleanup_FreeMMUMapping though.*/
+	OSLockAcquire(psPhysMemCtx->psCleanupData->hCleanupLock);
+
+	OSLockAcquire(psMMUContext->hLock);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#endif
+	{
+		_FreePageTables(psMMUContext,
+		                sDevVAddr,
+		                sDevVAddrEnd,
+		                uiLog2DataPageSize);
+	}
+
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+	eError = PVRSRVGetDevicePowerState(psPhysMemCtx->psDevNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		/* Treat unknown power state as ON. */
+		ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+	}
+
+	if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF)
+	{
+		_FreeMMUMapping(psPhysMemCtx->psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+	}
+	else
+	{
+		_SetupCleanup_FreeMMUMapping(psMMUContext->psPhysMemCtx);
+	}
+#else
+	if (!dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead))
+	{
+		_FreeMMUMapping(psPhysMemCtx->psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+	}
+#endif
+	OSLockRelease(psMMUContext->hLock);
+
+	OSLockRelease(psMMUContext->psPhysMemCtx->psCleanupData->hCleanupLock);
+}
+
+PVRSRV_ERROR
+MMU_MapPages(MMU_CONTEXT *psMMUContext,
+             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+             IMG_DEV_VIRTADDR sDevVAddrBase,
+             PMR *psPMR,
+             IMG_UINT32 ui32PhysPgOffset,
+             IMG_UINT32 ui32MapPageCount,
+             IMG_UINT32 *paui32MapIndices,
+             IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hPriv;
+
+	MMU_Levelx_INFO *psLevel = NULL;
+
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+
+	IMG_UINT32 uiPTEIndex = 0;
+	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
+	IMG_UINT32 uiLoop = 0;
+	IMG_UINT32 ui32MappedCount = 0;
+	IMG_DEVMEM_OFFSET_T uiPgOffset = 0;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0, uiDefProtFlags=0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *psDevPAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+	IMG_BOOL *pbValid;
+	IMG_BOOL bValid;
+	IMG_BOOL bScratchBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
+	IMG_BOOL bNeedBacking = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+#if defined(PDUMP)
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
+
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Wire up Page Table entries to point to the Data Pages (%"IMG_INT64_FMTSPECd" bytes)",
+	             (IMG_UINT64)(ui32MapPageCount * uiPageSize));
+#endif /*PDUMP*/
+
+	/* Validate the most essential parameters */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext != NULL, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR != NULL, "psPMR");
+
+	psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Allocate memory for page-frame-numbers and validity states,
+	   N.B. assert could be triggered by an illegal uiSizeBytes */
+	if (ui32MapPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psDevPAddr = OSAllocMem(ui32MapPageCount * sizeof(IMG_DEV_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPAddr, eError, ErrReturnError);
+
+		pbValid = OSAllocMem(ui32MapPageCount * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrFreePAddrMappingArray);
+	}
+	else
+	{
+		psDevPAddr = asDevPAddr;
+		pbValid	= abValid;
+	}
+
+	/* Get the Device physical addresses of the pages we are trying to map
+	 * In the case of non indexed mapping we can get all addresses at once */
+
+	/* DevmemX callers */
+	if (NULL == paui32MapIndices)
+	{
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2HeapPageSize,
+		                         ui32MapPageCount,
+		                         ((IMG_DEVMEM_OFFSET_T) ui32PhysPgOffset << uiLog2HeapPageSize),
+		                         psDevPAddr,
+		                         pbValid,
+		                         DEVICE_USE | MAPPING_USE);
+		/* Since offsets are always used and never indices, we are safe to
+		 * receive retry here as this path won't have performed any mappings
+		 * or overwrites thus far. Retry may be returned due to the combination of
+		 * DEVICE_USE | MAPPING_USE
+		 */
+		PVR_GOTO_IF_ERROR(eError, ErrFreeValidArray);
+	}
+
+	/*Get the Page table level configuration */
+	_MMU_GetPTConfig(psMMUContext,
+	                 (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig,
+	                 &hPriv,
+	                 &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, ErrPutPTConfig);
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
+		                                                                  uiLog2HeapPageSize);
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrPutPTConfig);
+	}
+
+	if (PMR_IsSparse(psPMR))
+	{
+		/* We know there will not be 4G number of PMR's */
+		bScratchBacking = PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(PMR_Flags(psPMR));
+		if (bScratchBacking)
+		{
+			bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(PMR_Flags(psPMR));
+		}
+	}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRMarkForDeferFree(psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
+	{
+
+#if defined(PDUMP)
+		IMG_DEVMEM_OFFSET_T uiNextSymName;
+#endif /*PDUMP*/
+
+		/* Devmem Map/Change sparse callers */
+		if (NULL != paui32MapIndices)
+		{
+			uiPgOffset = paui32MapIndices[uiLoop];
+
+			/*Calculate the Device Virtual Address of the page */
+			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (uiPgOffset * uiPageSize);
+
+			/* Get the physical address to map */
+			eError = PMR_DevPhysAddr(psPMR,
+			                         uiLog2HeapPageSize,
+			                         1,
+			                         uiPgOffset * uiPageSize,
+			                         &sDevPAddr,
+			                         &bValid,
+			                         DEVICE_USE | MAPPING_USE);
+			/* PVRSRV_ERROR_RETRY with SUPPORT_LINUX_OSPAGE_MIGRATION feature
+			 * enabled will not be returned here even with the use of
+			 * DEVICE_USE | MAPPING_USE because the feature does not
+			 * support migratable sparse PMRs.
+			 */
+			PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+		}
+		else
+		{
+			uiPgOffset = uiLoop + ui32PhysPgOffset;
+			sDevPAddr = psDevPAddr[uiLoop];
+			bValid = pbValid[uiLoop];
+		}
+
+		uiDefProtFlags = uiProtFlags;
+		/*
+			The default value of the entry is invalid so we don't need to mark
+			it as such if the page wasn't valid, we just advance pass that address
+		 */
+		if (bValid || bScratchBacking)
+		{
+			if (!bValid)
+			{
+				if (bZeroBacking)
+				{
+					eError = _MMU_GetBackingPage(psDevNode,
+					                             &sDevPAddr.uiAddr,
+					                             DEV_ZERO_PAGE);
+					PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+					                      ErrUnlockAndUnmapPages);
+
+					/* Ensure the zero back page PTE is read only */
+					uiDefProtFlags = uiProtFlagsReadOnly;
+				}
+				else
+				{
+					eError = _MMU_GetBackingPage(psDevNode,
+					                             &sDevPAddr.uiAddr,
+					                             SCRATCH_PAGE);
+					PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+					                      ErrUnlockAndUnmapPages);
+				}
+			}
+			else
+			{
+				/* check the physical alignment of the memory to map */
+				PVR_ASSERT((sDevPAddr.uiAddr & (uiPageSize-1)) == 0);
+			}
+
+#if defined(DEBUG)
+			{
+				IMG_INT32	i32FeatureVal = 0;
+				IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);
+
+				i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+				do {
+					/* i32FeatureVal can be negative for cases where this feature is undefined
+					 * In that situation we need to bail out than go ahead with debug comparison */
+					if (0 > i32FeatureVal)
+						break;
+
+					if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"%s Failed. The physical address bitlength (%u)"
+								" is greater than the chip can handle (%d).",
+								__func__, ui32BitLength, i32FeatureVal));
+
+						PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto ErrUnlockAndUnmapPages;
+					}
+				} while (0);
+			}
+#endif /*DEBUG*/
+
+#if defined(PDUMP)
+			if (bValid)
+			{
+				eError = PMR_PDumpSymbolicAddr(psPMR, uiPgOffset * uiPageSize,
+				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
+				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
+				                               &uiSymbolicAddrOffset,
+				                               &uiNextSymName);
+				PVR_LOG_IF_ERROR(eError, "PMR_PDumpSymbolicAddr");
+			}
+#endif /*PDUMP*/
+
+			psPrevLevel = psLevel;
+			/* Calculate PT index and get new table descriptor */
+			if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+			                    &psLevel, &uiPTEIndex))
+			{
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+				                    ErrUnlockAndUnmapPages);
+			}
+
+			if (psPrevLevel == psLevel)
+			{
+				/*
+				 * Sparse allocations may have page offsets which
+				 * decrement as well as increment, so make sure we
+				 * update the range we will flush correctly.
+				 */
+				if (uiPTEIndex > uiFlushEnd)
+					uiFlushEnd = uiPTEIndex;
+				else if (uiPTEIndex < uiFlushStart)
+					uiFlushStart = uiPTEIndex;
+			}
+			else
+			{
+				/* Flush if we moved to another psLevel, i.e. page table */
+				if (psPrevLevel != NULL)
+				{
+					eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+					                            &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+					                            uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+					                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+					PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+				}
+
+				uiFlushStart = uiPTEIndex;
+				uiFlushEnd = uiFlushStart;
+			}
+
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
+			        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+			        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));
+
+			/* Set the PT entry with the specified address and protection flags */
+			eError = _SetupPTE(psMMUContext,
+			                   psLevel,
+			                   uiPTEIndex,
+			                   psConfig,
+			                   &sDevPAddr,
+			                   &sDevVAddr,
+			                   IMG_FALSE,
+#if defined(PDUMP)
+			                   (bValid)?aszMemspaceName:(psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName),
+			                          ((bValid)?aszSymbolicAddress:((bZeroBacking)?DEV_ZERO_PAGE_STR:SCRATCH_PAGE_STR)),
+			                          (bValid)?uiSymbolicAddrOffset:0,
+#endif /*PDUMP*/
+			                    uiDefProtFlags);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockAndUnmapPages);
+
+			if (bValid)
+			{
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+				PVR_DPF ((PVR_DBG_MESSAGE,
+						"%s: devVAddr=" IMG_DEV_VIRTADDR_FMTSPEC ", "
+						"size=" IMG_DEVMEM_OFFSET_FMTSPEC,
+						__func__,
+						sDevVAddr.uiAddr,
+						uiPgOffset * uiPageSize));
+
+				ui32MappedCount++;
+			}
+		}
+
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(pbValid);
+		OSFreeMem(psDevPAddr);
+	}
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Wired up %d Page Table entries (out of %d)", ui32MappedCount, ui32MapPageCount);
+#endif /*PDUMP*/
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	if (PMR_IsSparse(psPMR) && PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiMappingFlags))
+	{
+		bNeedBacking = IMG_TRUE;
+	}
+
+	(void) MMU_UnmapPagesUnlocked(psMMUContext,
+	                              (bNeedBacking) ? uiMappingFlags : 0,
+	                              sDevVAddrBase,
+	                              uiLoop,
+	                              paui32MapIndices,
+	                              uiLog2HeapPageSize,
+	                              uiMappingFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+ErrFreeValidArray:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(pbValid);
+	}
+ErrFreePAddrMappingArray:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+ErrReturnError:
+	return eError;
+}
+
+static PVRSRV_ERROR
+MMU_UnmapPagesUnlocked(MMU_CONTEXT *psMMUContext,
+                       PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                       IMG_DEV_VIRTADDR sDevVAddrBase,
+                       IMG_UINT32 ui32PageCount,
+                       IMG_UINT32 *pai32FreeIndices,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPTEIndex = 0, ui32Loop=0;
+	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+	IMG_HANDLE hPriv;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0, uiMMUReadOnlyProtFlags = 0;
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_DEV_PHYADDR sBackingPgDevPhysAddr;
+	IMG_BOOL bUnmap = IMG_TRUE, bScratchBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+#if defined(PDUMP)
+	const IMG_CHAR *pcBackingPageName = NULL;
+
+	PDUMPCOMMENT(psDevNode,
+	             "Invalidate %d entries in page tables for virtual range: 0x%010"IMG_UINT64_FMTSPECX" to 0x%010"IMG_UINT64_FMTSPECX,
+	             ui32PageCount,
+	             (IMG_UINT64)sDevVAddr.uiAddr,
+	             ((IMG_UINT64)sDevVAddr.uiAddr) + (uiPageSize*ui32PageCount)-1);
+#endif
+
+	PVR_ASSERT(OSLockIsLocked(psMMUContext->hLock));
+
+	bScratchBacking = PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiMemAllocFlags);
+	bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(uiMemAllocFlags);
+
+	if (bZeroBacking)
+	{
+		/* Ensure the zero backing page has been created */
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sBackingPgDevPhysAddr.uiAddr,
+		                             DEV_ZERO_PAGE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_MMU_GetBackingPage (zero)");
+	}
+	else
+	{
+		/* Ensure the scratch backing page has been created */
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sBackingPgDevPhysAddr.uiAddr,
+		                             SCRATCH_PAGE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_MMU_GetBackingPage (scratch)");
+	}
+
+#if defined(PDUMP)
+	if (bScratchBacking)
+	{
+		pcBackingPageName = bZeroBacking ? DEV_ZERO_PAGE_STR : SCRATCH_PAGE_STR;
+	}
+#endif
+
+	bUnmap = (uiMappingFlags)? !bScratchBacking : IMG_TRUE;
+
+	eError = _MMU_ConvertDevMemFlags(bUnmap,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Get PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	uiMMUReadOnlyProtFlags = (uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE) | MMU_PROTFLAGS_READABLE;
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUReadOnlyProtFlags);
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUReadOnlyProtFlags, uiLog2PageSize);
+	}
+
+	/* Unmap page by page */
+	while (ui32Loop < ui32PageCount)
+	{
+		if (NULL != pai32FreeIndices)
+		{
+			/*Calculate the Device Virtual Address of the page */
+			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr +
+					pai32FreeIndices[ui32Loop] * (IMG_UINT64) uiPageSize;
+		}
+
+		psPrevLevel = psLevel;
+		/* Calculate PT index and get new table descriptor */
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND, e0);
+		}
+
+		if (psPrevLevel == psLevel)
+		{
+			/*
+			 * Sparse allocations may have page offsets which
+			 * decrement as well as increment, so make sure we
+			 * update the range we will flush correctly.
+			 */
+			if (uiPTEIndex > uiFlushEnd)
+				uiFlushEnd = uiPTEIndex;
+			else if (uiPTEIndex < uiFlushStart)
+				uiFlushStart = uiPTEIndex;
+		}
+		else
+		{
+			/* Flush if we moved to another psLevel, i.e. page table */
+			if (psPrevLevel != NULL)
+			{
+				PVRSRV_ERROR eError;
+				eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+				                   &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+				                   uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+				                   (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+				PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+			}
+
+			uiFlushStart = uiPTEIndex;
+			uiFlushEnd = uiFlushStart;
+		}
+
+		HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
+		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr));
+
+		/* Set the PT entry to invalid and poison it with a bad address */
+		eError = _SetupPTE(psMMUContext,
+		                   psLevel,
+		                   uiPTEIndex,
+		                   psConfig,
+		                   (bScratchBacking)? &sBackingPgDevPhysAddr : &gsBadDevPhyAddr,
+		                   &sDevVAddr,
+		                   bUnmap,
+#if defined(PDUMP)
+		                   (bScratchBacking)? (psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName): NULL,
+		                   pcBackingPageName,
+		                   0U,
+#endif
+		                   (bZeroBacking)? uiProtFlagsReadOnly: uiProtFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Check we haven't wrapped around */
+		PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+		ui32Loop++;
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		PVRSRV_ERROR eError;
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                   &psLevel->sMemDesc.psMapping->sMemHandle,
+		                   uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                   (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+e0:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table "
+	         "with error %u", __func__, eError));
+
+	return eError;
+}
+
+/*
+	MMU_UnmapPages
+ */
+PVRSRV_ERROR
+MMU_UnmapPages(MMU_CONTEXT *psMMUContext,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_DEV_VIRTADDR sDevVAddrBase,
+               IMG_UINT32 ui32PageCount,
+               IMG_UINT32 *pai32FreeIndices,
+               IMG_UINT32 uiLog2PageSize,
+               PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = MMU_UnmapPagesUnlocked(psMMUContext,
+	                                uiMappingFlags,
+	                                sDevVAddrBase,
+	                                ui32PageCount,
+	                                pai32FreeIndices,
+	                                uiLog2PageSize,
+	                                uiMemAllocFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+MMUX_MapVRangeToBackingPage(MMU_CONTEXT *psMMUContext,
+                          PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                          IMG_DEV_VIRTADDR sDevVAddrBase,
+                          IMG_UINT32 ui32MapPageCount,
+                          IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError;
+
+	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+	IMG_UINT32 uiPTEIndex = 0;
+
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+
+	IMG_UINT32 uiLoop = 0;
+
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	IMG_BOOL bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(uiMappingFlags);
+
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+
+	IMG_DEV_PHYADDR sDevPAddr;
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/*Get the Page table level configuration */
+	_MMU_GetPTConfig(psMMUContext,
+	                 (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig,
+	                 &hPriv,
+	                 &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, ErrPutPTConfig);
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
+		                                                                  uiLog2HeapPageSize);
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrPutPTConfig);
+	}
+
+	if (bZeroBacking)
+	{
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sDevPAddr.uiAddr,
+		                             DEV_ZERO_PAGE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+		                      ErrPutPTConfig);
+	}
+	else
+	{
+		eError = _MMU_GetBackingPage(psDevNode,
+									 &sDevPAddr.uiAddr,
+									 SCRATCH_PAGE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+							  ErrPutPTConfig);
+	}
+
+#if defined(DEBUG)
+	{
+		IMG_INT32	i32FeatureVal = 0;
+		IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);
+
+		i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+		do {
+			/* i32FeatureVal can be negative for cases where this feature is undefined
+			 * In that situation we need to bail out than go ahead with debug comparison */
+			if (0 > i32FeatureVal)
+				break;
+
+			if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s Failed. The physical address bitlength (%u)"
+						" is greater than the chip can handle (%d).",
+						__func__, ui32BitLength, i32FeatureVal));
+
+				PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto ErrPutPTConfig;
+			}
+		} while (0);
+	}
+#endif /*DEBUG*/
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
+	{
+		psPrevLevel = psLevel;
+		/* Calculate PT index and get new table descriptor */
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+			                    ErrUnlockAndUnmapPages);
+		}
+
+		if (psPrevLevel == psLevel)
+		{
+			/*
+			 * Sparse allocations may have page offsets which
+			 * decrement as well as increment, so make sure we
+			 * update the range we will flush correctly.
+			 */
+			if (uiPTEIndex > uiFlushEnd)
+				uiFlushEnd = uiPTEIndex;
+			else if (uiPTEIndex < uiFlushStart)
+				uiFlushStart = uiPTEIndex;
+		}
+		else
+		{
+			/* Flush if we moved to another psLevel, i.e. page table */
+			if (psPrevLevel != NULL)
+			{
+				eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+				                            &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+				                            uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+				                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+				PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+			}
+
+			uiFlushStart = uiPTEIndex;
+			uiFlushEnd = uiFlushStart;
+		}
+
+		HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
+		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+		        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));
+
+		/* Set the PT entry with the specified address and protection flags */
+		eError = _SetupPTE(psMMUContext,
+		                   psLevel,
+		                   uiPTEIndex,
+		                   psConfig,
+		                   &sDevPAddr,
+		                   &sDevVAddr,
+		                   IMG_FALSE,
+#if defined(PDUMP)
+		                   psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+		                   bZeroBacking ? DEV_ZERO_PAGE_STR : SCRATCH_PAGE_STR,
+		                   0,
+#endif /*PDUMP*/
+		                   bZeroBacking ? uiProtFlagsReadOnly : uiProtFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockAndUnmapPages);
+
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	(void) MMU_UnmapPagesUnlocked(psMMUContext,
+	                              0,
+	                              sDevVAddrBase,
+	                              uiLoop,
+	                              NULL,
+	                              uiLog2HeapPageSize,
+	                              uiMappingFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+	return eError;
+}
+
+PVRSRV_ERROR
+MMU_MapPMRFast(MMU_CONTEXT *psMMUContext,
+               IMG_DEV_VIRTADDR sDevVAddrBase,
+               PMR *psPMR,
+               IMG_DEVMEM_SIZE_T uiSizeBytes,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+
+	IMG_UINT32 i, uiChunkStart, uiLastPTEIndex, uiNumEntriesToWrite;
+	IMG_UINT32 ui32PagesDone=0, uiPTEIndex=0;
+
+	IMG_UINT8 uiAddrLog2Align, uiAddrShift, uiParityShift;
+	IMG_UINT64 uiAddrMask, uiProtFlags;
+	IMG_UINT32 uiBytesPerEntry;
+	IMG_UINT64 uiParityBit = 0;
+	IMG_BOOL bSetParity = IMG_FALSE;
+	IMG_DEV_VIRTADDR sDevVAddrRunning, sDevVAddrBaseCopy = sDevVAddrBase;
+
+	IMG_UINT64* pui64LevelBase;
+	IMG_UINT32* pui32LevelBase;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 uiNumPages = uiSizeBytes >> uiLog2HeapPageSize;
+
+
+#if defined(PVRSRV_ENABLE_HTB)
+	IMG_BOOL bHTBLog =
+	    HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_PMRMAP) ||
+	    HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_TABLE);
+#endif
+
+	IMG_BOOL bValidateOrTweak = psDevNode->pfnValidateOrTweakPhysAddrs ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDUMP)
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
+	IMG_UINT32 ui32MappedCount = 0;
+	IMG_DEV_VIRTADDR sDevVAddrRunningPdump;
+	PDUMPCOMMENT(psDevNode, "Wire up Page Table entries to point to the Data Pages (%d bytes)", uiNumPages << uiLog2HeapPageSize);
+#endif /*PDUMP*/
+
+	PVR_ASSERT (psMMUContext != NULL);
+	PVR_ASSERT (psPMR != NULL);
+	PVR_ASSERT((IMG_DEVMEM_OFFSET_T)uiNumPages << uiLog2HeapPageSize == uiSizeBytes);
+
+	/* Get general PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, put_mmu_context);
+
+	uiAddrLog2Align = psConfig->uiAddrLog2Align;
+	uiAddrShift = psConfig->uiAddrShift;
+	uiAddrMask = psConfig->uiAddrMask;
+	uiBytesPerEntry = psConfig->uiBytesPerEntry;
+
+	bSetParity = psConfig->uiParityBitMask;
+	uiParityShift = psConfig->uiParityBitShift;
+
+	sDevVAddrRunning.uiAddr = sDevVAddrBase.uiAddr;
+#if defined(PDUMP)
+	sDevVAddrRunningPdump.uiAddr = sDevVAddrBase.uiAddr;
+#endif
+	if (uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+	}
+	else if (uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_MMU_CONFIG_IS_WRONG, put_mmu_context);
+	}
+
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRMarkForDeferFree(psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	do
+	{
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddrBase, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+			                    unlock_mmu_context);
+		}
+
+		pui64LevelBase = (IMG_UINT64*)psLevel->sMemDesc.pvCpuVAddr;
+		pui32LevelBase = (IMG_UINT32*)psLevel->sMemDesc.pvCpuVAddr;
+
+		uiLastPTEIndex = MIN(uiPTEIndex + uiNumPages - ui32PagesDone, psDevVAddrConfig->uiNumEntriesPT);
+		uiNumEntriesToWrite = uiLastPTEIndex - uiPTEIndex;
+
+		for (uiChunkStart = 0; uiChunkStart < uiNumEntriesToWrite; uiChunkStart += PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			IMG_UINT32 uiNumPagesInBlock = MIN(uiNumEntriesToWrite - uiChunkStart, PMR_MAX_TRANSLATION_STACK_ALLOC);
+
+			/* With feature SUPPORT_LINUX_OSPAGE_MIGRATION:
+			 * This call is allowed to fail with retry part way through mapping.
+			 * Since this path is only used in Devicemem for fully contiguous PMR mappings
+			 * we know a PMR is always either fully mapped or not at all and cannot be
+			 * overwritten. On failure due to migrate we can unmap our progress and retry later.
+			 * Retry may be returned due to the combination of DEVICE_USE | MAPPING_USE.
+			 */
+			eError = PMR_DevPhysAddr(psPMR,
+			                         uiLog2HeapPageSize,
+			                         uiNumPagesInBlock,
+			                         (IMG_UINT64) (ui32PagesDone + uiChunkStart) << uiLog2HeapPageSize,
+			                         asDevPAddr,
+			                         abValid,
+			                         DEVICE_USE | MAPPING_USE);
+			PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+
+			if (bValidateOrTweak)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					PVRSRV_ERROR eError = psDevNode->pfnValidateOrTweakPhysAddrs(psDevNode,
+					                                                             psMMUContext->psDevAttrs,
+					                                                             &asDevPAddr[i].uiAddr);
+					PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+				}
+			}
+
+#if defined(DEBUG)
+			{
+				IMG_INT32 i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+
+				if (i32FeatureVal >= 0)
+				{
+					for (i=0; i<uiNumPagesInBlock; i++)
+					{
+						IMG_UINT32 ui32BitLength = FloorLog2(asDevPAddr[i].uiAddr);
+
+						if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+							        "%s Failed. The physical address bitlength (%u)"
+							        " is greater than the chip can handle (%d).",
+							        __func__, ui32BitLength, i32FeatureVal));
+
+							PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+							PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, unlock_mmu_context);
+						}
+					}
+				}
+			}
+#endif /*DEBUG*/
+
+			if (uiBytesPerEntry == 8)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					if (bSetParity)
+					{
+						uiParityBit = _GetParityBit(sDevVAddrRunning.uiAddr ^ asDevPAddr[i].uiAddr);
+						uiParityBit <<= uiParityShift;
+
+						sDevVAddrRunning.uiAddr += (1 << uiLog2HeapPageSize);
+					}
+
+					pui64LevelBase[uiPTEIndex + uiChunkStart + i] =
+						(((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags | uiParityBit;
+				}
+			}
+			else if (uiBytesPerEntry == 4)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+					/* assert that the result fits into 32 bits before writing
+					it into the 32-bit array with a cast */
+					IMG_UINT64 ui64PxE64 =
+					    (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+					PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+					PVR_ASSERT(!bSetParity);
+#endif
+
+					pui32LevelBase[uiPTEIndex + uiChunkStart + i] =
+					    (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+				}
+			}
+
+#if defined(PDUMP)
+			for (i=0; i<uiNumPagesInBlock; i++)
+			{
+				IMG_DEVMEM_OFFSET_T uiNextSymName;
+				IMG_UINT64 uiParityBit = psConfig->uiParityBitMask ? _GetParityBit(sDevVAddrRunningPdump.uiAddr ^ 0ULL) : 0;
+
+				eError = PMR_PDumpSymbolicAddr(psPMR, (ui32PagesDone + uiChunkStart + i) << uiLog2HeapPageSize,
+				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
+				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
+				                               &uiSymbolicAddrOffset,
+				                               &uiNextSymName);
+				PVR_LOG_IF_ERROR(eError, "PMR_PDumpSymbolicAddr");
+				ui32MappedCount++;
+
+				PDumpMMUDumpPxEntries(psDevNode,
+				                      MMU_LEVEL_1,
+				                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+				                      psLevel->sMemDesc.pvCpuVAddr,
+				                      psLevel->sMemDesc.sDevPAddr,
+				                      uiPTEIndex + uiChunkStart + i,
+				                      1,
+				                      aszMemspaceName,
+				                      aszSymbolicAddress,
+				                      uiSymbolicAddrOffset,
+				                      uiBytesPerEntry,
+				                      uiAddrLog2Align,
+				                      uiAddrShift,
+				                      uiAddrMask,
+				                      psConfig->uiProtMask,
+				                      psConfig->uiValidEnMask,
+				                      0,
+				                      uiParityBit,
+				                      psConfig->uiParityBitShift,
+				                      psConfig->uiParityBitMask,
+				                      psMMUContext->psDevAttrs->eMMUType);
+
+				sDevVAddrRunningPdump.uiAddr += (1 << uiLog2HeapPageSize);
+			}
+#endif /*PDUMP*/
+
+#if defined(PVRSRV_ENABLE_HTB)
+			if (bHTBLog)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					IMG_DEV_VIRTADDR sDevVAddr;
+					IMG_UINT64 ui64PxE64 = (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+					sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (ui32PagesDone + uiChunkStart + i)  * (1 << uiLog2HeapPageSize);
+
+					if (bSetParity)
+					{
+						ui64PxE64 |= _GetParityBit(sDevVAddr.uiAddr ^ asDevPAddr[i].uiAddr) << uiParityShift;
+					}
+
+					HTBLOGK(HTB_SF_MMU_PAGE_OP_PMRMAP,
+					        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+					        HTBLOG_U64_BITS_HIGH(asDevPAddr[i].uiAddr), HTBLOG_U64_BITS_LOW(asDevPAddr[i].uiAddr));
+
+					HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+					        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+					        uiPTEIndex + uiChunkStart + i, MMU_LEVEL_1,
+					        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+					        IMG_FALSE);
+				}
+			}
+#endif
+		}
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiPTEIndex * uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiNumEntriesToWrite) * uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+
+		sDevVAddrBase.uiAddr += uiNumEntriesToWrite * (1 << uiLog2HeapPageSize);
+		ui32PagesDone += uiNumEntriesToWrite;
+
+	} while (ui32PagesDone < uiNumPages);
+
+	OSLockRelease(psMMUContext->hLock);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Wired up %d Page Table entries (out of %d)", ui32MappedCount, uiNumPages);
+#endif /*PDUMP*/
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return PVRSRV_OK;
+
+unlock_mmu_context:
+	/* Unmap starting from the address passed as an argument. */
+	(void) MMU_UnmapPMRFastUnlocked(psMMUContext,
+	                                sDevVAddrBaseCopy,
+	                                uiNumPages,
+	                                uiLog2HeapPageSize);
+	OSLockRelease(psMMUContext->hLock);
+put_mmu_context:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+MMU_UnmapPMRFastUnlocked(MMU_CONTEXT *psMMUContext,
+                         IMG_DEV_VIRTADDR sDevVAddrBase,
+                         IMG_UINT32 ui32PageCount,
+                         IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiPTEIndex = 0, uiLastPTEIndex = 0, ui32PagesDone=0, i, uiNumEntriesToWrite;
+	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
+	MMU_Levelx_INFO *psLevel = NULL;
+	IMG_HANDLE hPriv;
+	void* pvPTStart;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_UINT64 uiProtFlags = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	IMG_UINT64 uiEntry = 0;
+
+	IMG_UINT64 uiParityBit = 0;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_DEV_VIRTADDR sDevVAddrStartOfTable;
+	IMG_UINT32 uiParityPatternIdx;
+	IMG_BOOL bSetParity = IMG_FALSE;
+	IMG_DEV_VIRTADDR sDevVAddrRunning;
+	IMG_UINT64 ui64BadPhysAddrParity = 0;
+#endif
+	IMG_UINT64 ui64BadPhysAddr = 0;
+
+#if defined(PVRSRV_ENABLE_HTB)
+	IMG_BOOL bLog;
+#endif
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	PVR_ASSERT(OSLockIsLocked(psMMUContext->hLock));
+
+	eError = _MMU_ConvertDevMemFlags(IMG_TRUE,
+	                                 0,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Get PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	bSetParity = psConfig->uiParityBitMask != 0;
+	sDevVAddrRunning.uiAddr = sDevVAddrBase.uiAddr;
+#endif
+
+	/* Callback to get device specific protection flags */
+
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
+		ui64BadPhysAddr = gsBadDevPhyAddr.uiAddr & ~(psConfig->uiProtMask | psConfig->uiParityBitMask);
+		/* Fill the entry with a bad address but leave space for protection flags and parity bit */
+		uiEntry = ui64BadPhysAddr | uiProtFlags;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		ui64BadPhysAddrParity = _GetParityBit(ui64BadPhysAddr) << psConfig->uiParityBitShift;
+#endif
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		PVR_ASSERT(!bSetParity);
+#endif
+		/* Fill the entry with a bad address but leave space for protection flags */
+		uiEntry = (((IMG_UINT32) gsBadDevPhyAddr.uiAddr) & ~psConfig->uiProtMask) | (IMG_UINT32) uiProtFlags;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: The page table entry byte length is not supported",
+				__func__));
+		goto e0;
+	}
+
+#if defined(PVRSRV_ENABLE_HTB)
+	bLog = HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_UNMAP) ||
+		   HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_TABLE);
+#endif
+
+	do
+	{
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND, e0);
+		}
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		sDevVAddrStartOfTable.uiAddr = sDevVAddr.uiAddr - uiPTEIndex * (1 << uiLog2PageSize);
+		uiParityPatternIdx = _GetParityBit(sDevVAddrStartOfTable.uiAddr);
+#endif
+		pvPTStart = psLevel->sMemDesc.pvCpuVAddr;
+
+		uiLastPTEIndex = MIN(uiPTEIndex + ui32PageCount - ui32PagesDone, psDevVAddrConfig->uiNumEntriesPT);
+		uiNumEntriesToWrite = uiLastPTEIndex - uiPTEIndex;
+
+		if (psConfig->uiBytesPerEntry == 8)
+		{
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+				if (bSetParity)
+				{
+					uiParityBit = psMMUContext->psDevAttrs->pui64PrecomputedAllocParity[uiParityPatternIdx][i] ^ ui64BadPhysAddrParity;
+					sDevVAddrRunning.uiAddr += (1 << uiLog2PageSize);
+				}
+#endif
+				((IMG_UINT64*)pvPTStart)[i] = uiEntry | uiParityBit;
+			}
+		}
+		else
+		{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+			PVR_ASSERT(psConfig->uiBytesPerEntry == 4);
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+			PVR_ASSERT(!bSetParity);
+#endif
+#endif
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+				((IMG_UINT32*)pvPTStart)[i] = (IMG_UINT32) uiEntry;
+			}
+		}
+
+#if defined(PVRSRV_ENABLE_HTB)
+		/* Log separately to avoid polluting the fast path */
+		if (bLog)
+		{
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+				IMG_DEV_VIRTADDR sDevVAddrToLog;
+				sDevVAddrToLog.uiAddr = sDevVAddr.uiAddr + (i - uiPTEIndex) * uiPageSize;
+
+				/* Log modifications */
+				HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
+						HTBLOG_U64_BITS_HIGH(sDevVAddrToLog.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrToLog.uiAddr));
+
+				HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+						HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+						uiPTEIndex, MMU_LEVEL_1,
+						HTBLOG_U64_BITS_HIGH(uiEntry), HTBLOG_U64_BITS_LOW(uiEntry),
+						IMG_FALSE);
+			}
+		}
+#endif
+
+#if defined(PDUMP)
+		PDumpMMUDumpPxEntries(psDevNode,
+		                      MMU_LEVEL_1,
+		                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+		                      psLevel->sMemDesc.pvCpuVAddr,
+		                      psLevel->sMemDesc.sDevPAddr,
+		                      uiPTEIndex,
+		                      uiNumEntriesToWrite,
+		                      NULL,
+		                      NULL,
+		                      0,
+		                      psConfig->uiBytesPerEntry,
+		                      psConfig->uiAddrLog2Align,
+		                      psConfig->uiAddrShift,
+		                      psConfig->uiAddrMask,
+		                      psConfig->uiProtMask,
+		                      psConfig->uiValidEnMask,
+		                      0,
+		                      0, /* Unused - Parity bit values are taken directly from PTE memory when bad phys addr is used */
+		                      0, /* Unused */
+		                      0, /* Unused */
+		                      psMMUContext->psDevAttrs->eMMUType);
+#endif /*PDUMP*/
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+						&psLevel->sMemDesc.psMapping->sMemHandle,
+						uiPTEIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+						(uiNumEntriesToWrite) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+
+		sDevVAddr.uiAddr += uiNumEntriesToWrite * uiPageSize;
+		ui32PagesDone += uiNumEntriesToWrite;
+
+	} while (ui32PagesDone < ui32PageCount);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+e0:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table with error %u",
+	         __func__, eError));
+
+	return eError;
+}
+
+/*
+    MMU_UnmapPMRFast
+ */
+PVRSRV_ERROR
+MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrBase,
+                 IMG_UINT32 ui32PageCount,
+                 IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = MMU_UnmapPMRFastUnlocked(psMMUContext,
+                                      sDevVAddrBase,
+                                      ui32PageCount,
+                                      uiLog2PageSize);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+PVRSRV_ERROR
+MMU_RemapPage(MMU_CONTEXT *psMMUContext,
+              PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+              IMG_DEV_VIRTADDR sDevVAddr,
+              IMG_UINT32 uiLog2HeapPageSize,
+              PMR *psOriginPMR,
+              IMG_UINT32 ui32LogicalPgOffset)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	const MMU_PxE_CONFIG *psConfig;
+	IMG_HANDLE hPriv;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT32 uiPTEIndex = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+
+	IMG_UINT64 uiProtFlags = 0;
+	IMG_UINT64 ui64PrevPxE64;
+	IMG_UINT32 ui32PrevPxE32;
+
+	IMG_DEV_PHYADDR sTargetDevPAddr;
+	IMG_BOOL bValid = IMG_FALSE;
+
+	/* Validate the most essential parameters */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext != NULL, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psOriginPMR != NULL, "psPMR");
+
+	psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	_MMU_GetPTConfig(psMMUContext, uiLog2HeapPageSize, &psConfig, &hPriv, &psDevVAddrConfig);
+
+	if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+	                    &psLevel, &uiPTEIndex))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_MMU_GetPTInfo", eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+		                        ErrPutPTConfig);
+	}
+
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psLevel->sMemDesc.pvCpuVAddr;
+		ui32PrevPxE32 = pui32Px[uiPTEIndex];
+
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psLevel->sMemDesc.pvCpuVAddr;
+		ui64PrevPxE64 = pui64Px[uiPTEIndex];
+
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+	}
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = PMR_DevPhysAddr(psOriginPMR,
+	                         uiLog2HeapPageSize,
+	                         1,
+	                         ((IMG_DEVMEM_OFFSET_T) ui32LogicalPgOffset << uiLog2HeapPageSize),
+	                         &sTargetDevPAddr,
+	                         &bValid,
+	                         DEVICE_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", ErrUnlockContext);
+
+	if (!bValid)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("PMR_DevPhysAddr",
+		                        eError,
+		                        PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE,
+		                        ErrUnlockContext);
+	}
+
+	/* Set the PT entry with the specified address and protection flags */
+	eError = _SetupPTE(psMMUContext,
+	                   psLevel,
+	                   uiPTEIndex,
+	                   psConfig,
+	                   &sTargetDevPAddr,
+	                   &sDevVAddr,
+	                   IMG_FALSE,
+#if defined(PDUMP)
+	                   /*PDUMP Not supported for remap */
+	                   NULL,
+	                   NULL,
+	                   0,
+#endif /*PDUMP*/
+	                   uiProtFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockContext);
+
+	eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+	                            &psLevel->sMemDesc.psMapping->sMemHandle,
+	                            uiPTEIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+	                            psConfig->uiBytesPerEntry);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapPagesClean", ErrUnlockAndUnmapPages);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs, this queues invalidation, actioning of this
+	 * should be ensured complete by the caller before further GPU
+	 * workloads continue.
+	 */
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	/* Reset old PTE in place */
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psLevel->sMemDesc.pvCpuVAddr;
+		pui32Px[uiPTEIndex] = ui32PrevPxE32;
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psLevel->sMemDesc.pvCpuVAddr;
+		pui64Px[uiPTEIndex] = ui64PrevPxE64;
+	}
+
+ErrUnlockContext:
+	OSLockRelease(psMMUContext->hLock);
+
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return eError;
+}
+#endif
+
+/*
+	MMU_AcquireBaseAddr
+ */
+PVRSRV_ERROR
+MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr)
+{
+	if (!psMMUContext)
+	{
+		psPhysAddr->uiAddr = 0;
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED,
+		                        "MMU_AcquireBaseAddr");
+	}
+
+	*psPhysAddr = psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_AcquireCPUBaseAddr
+ */
+PVRSRV_ERROR
+MMU_AcquireCPUBaseAddr(MMU_CONTEXT *psMMUContext, void **ppvCPUVAddr)
+{
+	if (!psMMUContext)
+	{
+		*ppvCPUVAddr = NULL;
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED,
+		                        "MMU_AcquireCPUBaseAddr");
+	}
+
+	*ppvCPUVAddr = psMMUContext->sBaseLevelInfo.sMemDesc.pvCpuVAddr;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_ReleaseBaseAddr
+ */
+void
+MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext)
+{
+	PVR_UNREFERENCED_PARAMETER(psMMUContext);
+}
+
+/*
+	MMU_AppendCacheFlags
+*/
+
+void MMU_AppendCacheFlags(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32AppendFlags)
+{
+	PVR_ASSERT(psMMUContext != NULL);
+
+	if (psMMUContext == NULL)
+	{
+		return;
+	}
+
+	OSAtomicOr(&psMMUContext->sCacheFlags, (IMG_INT)ui32AppendFlags);
+}
+
+/*
+	MMU_GetAndResetCacheFlags
+*/
+IMG_UINT32 MMU_GetAndResetCacheFlags(MMU_CONTEXT *psMMUContext)
+{
+	IMG_UINT32 uiFlags;
+
+	PVR_ASSERT(psMMUContext != NULL);
+	if (psMMUContext == NULL)
+	{
+		return 0;
+	}
+
+	uiFlags = (IMG_UINT32) OSAtomicExchange(&psMMUContext->sCacheFlags, 0);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* kick cleanup thread to free all zombie PMRs residing in the device's
+	 * zombie list */
+	if (PMRQueueZombiesForCleanup(psMMUContext->psPhysMemCtx->psDevNode))
+	{
+		BITMASK_SET(uiFlags, RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL | RGXFWIF_MMUCACHEDATA_FLAGS_PT);
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	return uiFlags;
+}
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+/*
+    MMU_GetOSids
+ */
+
+void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 *pui32OSid, IMG_UINT32 *pui32OSidReg, IMG_BOOL *pbOSidAxiProt)
+{
+	*pui32OSid     = psMMUContext->psPhysMemCtx->ui32OSid;
+	*pui32OSidReg  = psMMUContext->psPhysMemCtx->ui32OSidReg;
+	*pbOSidAxiProt = psMMUContext->psPhysMemCtx->bOSidAxiProt;
+}
+
+#endif
+
+static IMG_BOOL _MMUGetPxEFaultLevelData(const MMU_CONTEXT *psMMUContext,
+		                                 IMG_DEV_VIRTADDR *psDevVAddr,
+		                                 const MMU_PxE_CONFIG *psConfig,
+		                                 void *pvCpuVAddr,
+		                                 IMG_UINT32 ui32PxIndex,
+		                                 MMU_LEVEL_DATA *psMMULevelDataOut,
+		                                 IMG_UINT32 *ui32Log2PageSizeOut)
+{
+	static const IMG_CHAR *apszMMUValidStr[4] =  {
+			/*--*/ "not valid",
+			/*-V*/ "valid",
+			/*P-*/ "pending",
+			/*PV*/ "inconsistent (pending and valid)"  };
+	#define _MMU_VALID_STR(_cfg, _entry) \
+		(apszMMUValidStr[((((_entry)&(_cfg->uiPendingEnMask))!=0) << 1)| \
+		                 ((((_entry)&(_cfg->uiValidEnMask))!=0) << 0)])
+	#define _MMU_MASK_VALID_FOR_32BITS(_cfg) \
+		((_cfg->uiPendingEnMask | _cfg->uiValidEnMask) <= 0xFFFFFFFF)
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Ptr = pvCpuVAddr;
+
+		psMMULevelDataOut->ui64Address = pui32Ptr[ui32PxIndex];
+
+		/* Check if we are dealing with a PxE entry where these bits are in the first 32 bits */
+		/* But if we know it is a 4 byte entry, why check this? */
+		if (_MMU_MASK_VALID_FOR_32BITS(psConfig))
+		{
+			psMMULevelDataOut->psDebugStr = _MMU_VALID_STR(psConfig, pui32Ptr[ui32PxIndex] & psConfig->uiProtMask);
+		}
+		else
+		{
+			psMMULevelDataOut->psDebugStr = "";
+			PVR_DPF((PVR_DBG_ERROR, "Invalid %sE masks for 32-bit entry", psConfig->pszPxLevelStr));
+			return IMG_FALSE;
+		}
+
+		if (ui32Log2PageSizeOut != NULL)
+		{
+			if (psDevAttrs->pfnGetPageSizeFromPDE4(pui32Ptr[ui32PxIndex], ui32Log2PageSizeOut) != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from the %sE", psConfig->pszPxLevelStr));
+				return IMG_FALSE;
+			}
+		}
+	}
+	else
+	{
+		IMG_UINT64 *pui64Ptr = pvCpuVAddr;
+
+		psMMULevelDataOut->ui64Address = pui64Ptr[ui32PxIndex];
+		psMMULevelDataOut->psDebugStr  = _MMU_VALID_STR(psConfig, pui64Ptr[ui32PxIndex] & psConfig->uiProtMask);
+
+		if (ui32Log2PageSizeOut != NULL)
+		{
+			/* MMU_VERSION >= 4 */
+			if (psDevAttrs->pfnGetPageSizeFromVirtAddr != NULL)
+			{
+				if (psDevAttrs->pfnGetPageSizeFromVirtAddr(psMMUContext->psPhysMemCtx->psDevNode, *psDevVAddr, ui32Log2PageSizeOut) != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from device VA"));
+					return IMG_FALSE;
+				}
+			}
+			/* MMU_VERSION < 4 */
+			else if (psDevAttrs->pfnGetPageSizeFromPDE8(pui64Ptr[ui32PxIndex], ui32Log2PageSizeOut) != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from the %sE", psConfig->pszPxLevelStr));
+				return IMG_FALSE;
+			}
+		}
+	}
+
+	return IMG_TRUE;
+}
+
+/*
+	MMU_CheckFaultAddress
+ */
+void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
+                           IMG_DEV_VIRTADDR *psDevVAddr,
+                           MMU_FAULT_DATA *psOutFaultData)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	MMU_LEVEL	eMMULevel = psDevAttrs->psBaseConfig->ePxLevel;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_PxE_CONFIG *psMMUPDEConfig;
+	const MMU_PxE_CONFIG *psMMUPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psMMUDevVAddrConfig;
+	IMG_HANDLE hPriv;
+	MMU_Levelx_INFO *psLevel = NULL;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 uiIndex;
+	IMG_UINT32 ui32PCIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32PDIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32PTIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32Log2PageSize;
+	MMU_FAULT_DATA sMMUFaultData = {0};
+	MMU_LEVEL_DATA *psMMULevelData;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	/*
+		At this point we don't know the page size so assume it's 4K.
+		When we get the PD level (MMU_LEVEL_2) we can check to see
+		if this assumption is correct.
+	 */
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(12,
+	                                                 &psMMUPDEConfig,
+	                                                 &psMMUPTEConfig,
+	                                                 &psMMUDevVAddrConfig,
+	                                                 &hPriv);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("Failed to get the page size info for log2 page sizeof 12"));
+	}
+
+	psLevel = &psMMUContext->sBaseLevelInfo;
+	psConfig = psDevAttrs->psBaseConfig;
+
+	sMMUFaultData.eTopLevel = psDevAttrs->psBaseConfig->ePxLevel;
+	sMMUFaultData.eType = MMU_FAULT_TYPE_NON_PM;
+
+
+	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
+	{
+		if (eMMULevel == MMU_LEVEL_3)
+		{
+			/* Determine the PC index */
+			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexMask;
+			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexShift;
+			ui32PCIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PCIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_3];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PCIndex;
+
+			if (ui32PCIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PCIndex,
+					                        psMMULevelData, NULL);
+
+			psLevel = psLevel->apsNextLevel[ui32PCIndex];
+			if (!psLevel)
+			{
+				break;
+			}
+			psConfig = psMMUPDEConfig;
+			continue; /* continue to the next level */
+		}
+
+
+		if (eMMULevel == MMU_LEVEL_2)
+		{
+			/* Determine the PD index */
+			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexMask;
+			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexShift;
+			ui32PDIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PDIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_2];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PDIndex;
+
+			if (ui32PDIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PDIndex,
+					                        psMMULevelData, &ui32Log2PageSize);
+
+			/*
+				We assumed the page size was 4K, now we have the actual size
+				from the PDE we can confirm if our assumption was correct.
+				Until now it hasn't mattered as the PC and PD are the same
+				regardless of the page size
+			 */
+			if (ui32Log2PageSize != 12)
+			{
+				/* Put the 4K page size data */
+				psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+
+				/* Get the correct size data */
+				eError = psDevAttrs->pfnGetPageSizeConfiguration(ui32Log2PageSize,
+				                                                 &psMMUPDEConfig,
+				                                                 &psMMUPTEConfig,
+				                                                 &psMMUDevVAddrConfig,
+				                                                 &hPriv);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_LOG(("Failed to get the page size info for log2 page sizeof %d", ui32Log2PageSize));
+					break;
+				}
+			}
+
+			psLevel = psLevel->apsNextLevel[ui32PDIndex];
+			if (!psLevel)
+			{
+				break;
+			}
+			psConfig = psMMUPTEConfig;
+			continue; /* continue to the next level */
+		}
+
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			/* Determine the PT index */
+			uiIndex = psDevVAddr->uiAddr & psMMUDevVAddrConfig->uiPTIndexMask;
+			uiIndex = uiIndex >> psMMUDevVAddrConfig->uiPTIndexShift;
+			ui32PTIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PTIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_1];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PTIndex;
+
+			if (ui32PTIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PTIndex,
+					                        psMMULevelData, NULL);
+			break;
+		}
+
+		PVR_LOG(("Unsupported MMU setup: %d", eMMULevel));
+		break;
+	}
+
+	/* Put the page size data back */
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+	OSLockRelease(psMMUContext->hLock);
+
+	*psOutFaultData = sMMUFaultData;
+}
+
+static IMG_UINT64 MMU_GetVDevAddrPTE(MMU_CONTEXT *psMMUContext,
+                                     const MMU_PxE_CONFIG *psConfig,
+                                     const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                                     IMG_DEV_VIRTADDR sDevVAddr,
+                                     IMG_BOOL *pbStatusOut)
+{
+	MMU_Levelx_INFO *psLevel = NULL;
+	IMG_UINT32 uiIndex = 0;
+	IMG_BOOL bStatus = IMG_FALSE;
+	IMG_UINT64 ui64Entry = 0;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	switch (psMMUContext->psDevAttrs->psBaseConfig->ePxLevel)
+	{
+		case MMU_LEVEL_3:
+			uiIndex = _CalcPCEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+			psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
+			if (psLevel == NULL)
+				break;
+
+			__fallthrough;
+		case MMU_LEVEL_2:
+			uiIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLevel != NULL)
+				psLevel = psLevel->apsNextLevel[uiIndex];
+			else
+				psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
+
+			if (psLevel == NULL)
+				break;
+
+			__fallthrough;
+		case MMU_LEVEL_1:
+			uiIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLevel == NULL)
+				psLevel = &psMMUContext->sBaseLevelInfo;
+
+			ui64Entry = ((IMG_UINT64 *)psLevel->sMemDesc.pvCpuVAddr)[uiIndex];
+			bStatus = ui64Entry & psConfig->uiValidEnMask;
+
+			break;
+		default:
+			PVR_LOG(("MMU_IsVDevAddrValid: Unsupported MMU setup"));
+			break;
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	*pbStatusOut = bStatus;
+
+	return ui64Entry;
+}
+
+IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
+                             IMG_UINT32 uiLog2PageSize,
+                             IMG_DEV_VIRTADDR sDevVAddr)
+{
+	IMG_BOOL bStatus;
+	const MMU_PxE_CONFIG *psConfig;
+	IMG_HANDLE hPriv;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	_MMU_GetPTConfig(psMMUContext, uiLog2PageSize, &psConfig, &hPriv, &psDevVAddrConfig);
+
+	MMU_GetVDevAddrPTE(psMMUContext,
+	                   psConfig,
+	                   psDevVAddrConfig,
+	                   sDevVAddr,
+	                   &bStatus);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return bStatus;
+}
+
+#if defined(PDUMP)
+/*
+	MMU_ContextDerivePCPDumpSymAddr
+ */
+PVRSRV_ERROR MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
+                                             IMG_CHAR *pszPDumpSymbolicNameBuffer,
+                                             size_t uiPDumpSymbolicNameBufferSize)
+{
+	size_t uiCount;
+	IMG_UINT64 ui64PhysAddr;
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	if (!psMMUContext->sBaseLevelInfo.sMemDesc.bValid)
+	{
+		/* We don't have any allocations.  You're not allowed to ask
+		 * for the page catalogue base address until you've made at
+		 * least one allocation.
+		 */
+		return PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR;
+	}
+
+	ui64PhysAddr = (IMG_UINT64)psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr.uiAddr;
+
+	PVR_ASSERT(uiPDumpSymbolicNameBufferSize >= (IMG_UINT32)(21 + OSStringLength(psDevId->pszPDumpDevName)));
+
+	/* Page table Symbolic Name is formed from page table phys addr
+	   prefixed with MMUPT_. */
+	uiCount = OSSNPrintf(pszPDumpSymbolicNameBuffer,
+	                     uiPDumpSymbolicNameBufferSize,
+	                     ":%s:%s%016"IMG_UINT64_FMTSPECX,
+	                     psDevId->pszPDumpDevName,
+	                     psMMUContext->sBaseLevelInfo.sMemDesc.bValid?"MMUPC_":"XXX",
+	                     ui64PhysAddr);
+
+	if (uiCount + 1 > uiPDumpSymbolicNameBufferSize)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_PDumpWritePageCatBase
+ */
+PVRSRV_ERROR
+MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                          const IMG_CHAR *pszSpaceName,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32WordSize,
+                          IMG_UINT32 ui32AlignShift,
+                          IMG_UINT32 ui32Shift,
+                          PDUMP_FLAGS_T uiPdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszPageCatBaseSymbolicAddr[100];
+	const IMG_CHAR *pszPDumpDevName = psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName;
+
+	eError = MMU_ContextDerivePCPDumpSymAddr(psMMUContext,
+	                                         &aszPageCatBaseSymbolicAddr[0],
+	                                         sizeof(aszPageCatBaseSymbolicAddr));
+	if (eError == PVRSRV_OK)
+	{
+		eError = PDumpWriteSymbAddress(psMMUContext->psPhysMemCtx->psDevNode,
+		                               pszSpaceName,
+		                               uiOffset,
+		                               aszPageCatBaseSymbolicAddr,
+		                               0, /* offset -- Could be non-zero for var. pgsz */
+		                               pszPDumpDevName,
+		                               ui32WordSize,
+		                               ui32AlignShift,
+		                               ui32Shift,
+		                               uiPdumpFlags | PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	return eError;
+}
+
+/*
+	MMU_AcquirePDumpMMUContext
+ */
+PVRSRV_ERROR MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                                        IMG_UINT32 *pui32PDumpMMUContextID,
+                                        IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	if (!psMMUContext->ui32PDumpContextIDRefCount)
+	{
+		PDUMP_MMU_ALLOC_MMUCONTEXT(psMMUContext->psPhysMemCtx->psDevNode,
+		                           psDevId->pszPDumpDevName,
+		                           psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr,
+		                           psMMUContext->psDevAttrs->eMMUType,
+		                           &psMMUContext->uiPDumpContextID,
+		                           ui32PDumpFlags);
+	}
+
+	psMMUContext->ui32PDumpContextIDRefCount++;
+	*pui32PDumpMMUContextID = psMMUContext->uiPDumpContextID;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_ReleasePDumpMMUContext
+ */
+PVRSRV_ERROR MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+	                                        IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	PVR_ASSERT(psMMUContext->ui32PDumpContextIDRefCount != 0);
+	psMMUContext->ui32PDumpContextIDRefCount--;
+
+	if (psMMUContext->ui32PDumpContextIDRefCount == 0)
+	{
+		PDUMP_MMU_FREE_MMUCONTEXT(psMMUContext->psPhysMemCtx->psDevNode,
+		                          psDevId->pszPDumpDevName,
+		                          psMMUContext->uiPDumpContextID,
+		                          ui32PDumpFlags);
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+PVRSRV_ERROR MMU_CacheInvalidateKick(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                     IMG_UINT32 *puiRequiredSyncValue)
+{
+	IMG_UINT32 uiRequiredSyncValue;
+	PVRSRV_ERROR eError;
+
+	eError = psDeviceNode->pfnMMUCacheInvalidateKick(psDeviceNode, &uiRequiredSyncValue);
+
+	if (puiRequiredSyncValue != NULL)
+	{
+		*puiRequiredSyncValue = uiRequiredSyncValue;
+	}
+
+	return eError;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/******************************************************************************
+ End of file (mmu_common.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_mmu.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_mmu.c
new file mode 100644
index 000000000000..64a5740f52f5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_mmu.c
@@ -0,0 +1,946 @@
+/*************************************************************************/ /*!
+@File
+@Title          MMU PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common PDump (MMU specific) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if defined(PDUMP)
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pdump_mmu.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+
+#define MAX_PDUMP_MMU_CONTEXTS	(10)
+static IMG_UINT32 guiPDumpMMUContextAvailabilityMask = (1<<MAX_PDUMP_MMU_CONTEXTS)-1;
+
+
+#define MMUPX_FMT(X) ((X<3) ? ((X<2) ?  "MMUPT_\0" : "MMUPD_\0") : "MMUPC_\0")
+#define MIPSMMUPX_FMT(X) ((X<3) ? ((X<2) ?  "MIPSMMUPT_\0" : "MIPSMMUPD_\0") : "MIPSMMUPC_\0")
+
+
+/* Array used to look-up debug strings from MMU_LEVEL */
+static const IMG_CHAR * const apszMMULevelStringLookup[MMU_LEVEL_LAST] =
+{
+	"MMU_LEVEL_0",
+	"PAGE_TABLE",
+	"PAGE_DIRECTORY",
+	"PAGE_CATALOGUE",
+};
+
+static PVRSRV_ERROR
+_ContiguousPDumpBytes(PVRSRV_DEVICE_NODE *psDeviceNode,
+		      const IMG_CHAR *pszSymbolicName,
+		      IMG_UINT32 ui32SymAddrOffset,
+		      IMG_BOOL bFlush,
+		      IMG_UINT32 uiNumBytes,
+		      void *pvBytes,
+		      IMG_UINT32 ui32Flags)
+{
+	static const IMG_CHAR *pvBeyondLastPointer;
+	static const IMG_CHAR *pvBasePointer;
+	static IMG_UINT32 ui32BeyondLastOffset;
+	static IMG_UINT32 ui32BaseOffset;
+	static IMG_UINT32 uiAccumulatedBytes;
+	IMG_UINT32 ui32ParamOutPos;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_AND_FILE_STRING();
+
+	/* Caller has PDUMP_LOCK */
+
+	if (!bFlush && uiAccumulatedBytes > 0)
+	{
+		/* do some tests for contiguity.  If it fails, we flush anyway */
+		if (pvBeyondLastPointer != pvBytes ||
+		    ui32SymAddrOffset != ui32BeyondLastOffset
+		    /* NB: ought to check that symbolic name agrees too, but
+		       we know this always to be the case in the current use-case */
+			)
+		{
+			bFlush = IMG_TRUE;
+		}
+	}
+
+	/* Flush if necessary */
+	if (bFlush && uiAccumulatedBytes > 0)
+	{
+		eErr = PDumpWriteParameter(psDeviceNode,
+					   (IMG_UINT8 *)(uintptr_t)pvBasePointer,
+					   uiAccumulatedBytes, ui32Flags,
+					   &ui32ParamOutPos, pszFileName);
+		if (eErr == PVRSRV_OK)
+		{
+			eErr = PDumpSNPrintf(hScript, ui32MaxLenScript,
+					     "LDB %s:0x%X 0x%X 0x%X %s",
+					     /* dest */
+					     pszSymbolicName,
+					     ui32BaseOffset,
+					     /* size */
+					     uiAccumulatedBytes,
+					     /* file offset */
+					     ui32ParamOutPos,
+					     /* filename */
+					     pszFileName);
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpSNPrintf", ErrOut);
+
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+		else if (eErr != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+		{
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpWriteParameter", ErrOut);
+		}
+		else
+		{
+			/* else Write to parameter file prevented under the flags and
+			 * current state of the driver so skip write to script and error IF.
+			 * this is normal e.g. no in capture range for example.
+			 */
+			eErr = PVRSRV_OK;
+		}
+
+		uiAccumulatedBytes = 0;
+	}
+
+	/* Initialise offsets and pointers if necessary */
+	if (uiAccumulatedBytes == 0)
+	{
+		ui32BaseOffset = ui32BeyondLastOffset = ui32SymAddrOffset;
+		pvBeyondLastPointer = pvBasePointer = (const IMG_CHAR *)pvBytes;
+	}
+
+	/* Accumulate some bytes */
+	ui32BeyondLastOffset += uiNumBytes;
+	pvBeyondLastPointer += uiNumBytes;
+	uiAccumulatedBytes += uiNumBytes;
+
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_AND_FILE_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUMalloc
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+			    const IMG_CHAR *pszPDumpDevName,
+			    MMU_LEVEL eMMULevel,
+			    IMG_DEV_PHYADDR *psDevPAddr,
+			    IMG_UINT32 ui32Size,
+			    IMG_UINT32 ui32Align,
+			    PDUMP_MMU_TYPE eMMUType)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA;
+	IMG_UINT64 ui64SymbolicAddr;
+	IMG_CHAR *pszMMUPX;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_GOTO_IF_INVALID_PARAM(eMMULevel < MMU_LEVEL_LAST, eErr, ErrOut);
+
+	PDUMP_LOCK(ui32Flags);
+
+	/*
+	  Write a comment to the PDump2 script streams indicating the memory allocation
+	*/
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "-- CALLOC :%s:%s Size=0x%08X Alignment=0x%08X 0x0 DevPAddr=0x%08"IMG_UINT64_FMTSPECX,
+			     pszPDumpDevName,
+			     apszMMULevelStringLookup[eMMULevel],
+			     ui32Size,
+			     ui32Align,
+			     psDevPAddr->uiAddr);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	/*
+	  construct the symbolic address
+	*/
+	ui64SymbolicAddr = (IMG_UINT64)psDevPAddr->uiAddr;
+
+	/*
+	  Write to the MMU script stream indicating the memory allocation
+	*/
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(eMMULevel);
+	}
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "CALLOC :%s:%s%016"IMG_UINT64_FMTSPECX" 0x%X 0x%X 0x0",
+			     pszPDumpDevName,
+			     pszMMUPX,
+			     ui64SymbolicAddr,
+			     ui32Size,
+			     ui32Align
+			     /* don't need this sDevPAddr.uiAddr*/);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32Flags);
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUFree
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+			const IMG_CHAR *pszPDumpDevName,
+			  MMU_LEVEL eMMULevel,
+			  IMG_DEV_PHYADDR *psDevPAddr,
+			  PDUMP_MMU_TYPE eMMUType)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT64 ui64SymbolicAddr;
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA;
+	IMG_CHAR *pszMMUPX;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_GOTO_IF_INVALID_PARAM(eMMULevel < MMU_LEVEL_LAST, eErr, ErrOut);
+
+	PDUMP_LOCK(ui32Flags);
+	/*
+	  Write a comment to the PDUMP2 script streams indicating the memory free
+	*/
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "-- FREE :%s:%s",
+			     pszPDumpDevName, apszMMULevelStringLookup[eMMULevel]);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	/*
+	  construct the symbolic address
+	*/
+	ui64SymbolicAddr = (IMG_UINT64)psDevPAddr->uiAddr;
+
+	/*
+	  Write to the MMU script stream indicating the memory free
+	*/
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(eMMULevel);
+	}
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "FREE :%s:%s%016"IMG_UINT64_FMTSPECX,
+			     pszPDumpDevName,
+			     pszMMUPX,
+			     ui64SymbolicAddr);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32Flags);
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/*******************************************************************************************************
+ * Function Name  : PDumpPTBaseObjectToMem64
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write from the baseobject
+ *					for MIPS MMU device type
+********************************************************************************************************/
+PVRSRV_ERROR PDumpPTBaseObjectToMem64(const IMG_CHAR *pszPDumpDevName,
+				      PMR *psPMRDest,
+				      IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+				      IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+				      IMG_UINT32 ui32Flags,
+				      MMU_LEVEL eMMULevel,
+				      IMG_UINT64 ui64PxSymAddr,
+				      IMG_UINT64 ui64PxOffset)
+{
+
+	IMG_CHAR aszMemspaceNameDest[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameDest[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetDest;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameDest;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRDest,
+				     uiLogicalOffsetDest,
+				     PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+				     aszMemspaceNameDest,
+				     PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+				     aszSymbolicNameDest,
+				     &uiPDumpSymbolicOffsetDest,
+				     &uiNextSymNameDest);
+
+	PVR_GOTO_IF_ERROR(eErr, Err);
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+						 "WRW64 :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:%s%016"IMG_UINT64_FMTSPECX":0x%"IMG_UINT64_FMTSPECX,
+						 aszMemspaceNameDest, aszSymbolicNameDest, uiPDumpSymbolicOffsetDest,
+						 pszPDumpDevName, MIPSMMUPX_FMT(eMMULevel), ui64PxSymAddr, ui64PxOffset);
+
+	PVR_GOTO_IF_ERROR(eErr, Err);
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(PMR_DeviceNode(psPMRDest), hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+Err:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUDumpPxEntries
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUDumpPxEntries(PVRSRV_DEVICE_NODE *psDeviceNode,
+				   MMU_LEVEL eMMULevel,
+				   const IMG_CHAR *pszPDumpDevName,
+				   void *pvPxMem,
+				   IMG_DEV_PHYADDR sPxDevPAddr,
+				   IMG_UINT32 uiFirstEntry,
+				   IMG_UINT32 uiNumEntries,
+				   const IMG_CHAR *pszMemspaceName,
+				   const IMG_CHAR *pszSymbolicAddr,
+				   IMG_UINT64 uiSymbolicAddrOffset,
+				   IMG_UINT32 uiBytesPerEntry,
+				   IMG_UINT32 uiLog2Align,
+				   IMG_UINT32 uiAddrShift,
+				   IMG_UINT64 uiAddrMask,
+				   IMG_UINT64 uiPxEProtMask,
+				   IMG_UINT64 uiDataValidEnable,
+				   IMG_UINT32 ui32Flags,
+				   IMG_UINT32 ui32VAParity,
+				   IMG_UINT32 ui32ParityShift,
+				   IMG_UINT64 ui64ParityMask,
+				   PDUMP_MMU_TYPE eMMUType)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT64 ui64PxSymAddr;
+	IMG_UINT64 ui64PxEValueSymAddr;
+	IMG_UINT32 ui32SymAddrOffset = 0;
+	IMG_UINT32 *pui32PxMem;
+	IMG_UINT64 *pui64PxMem;
+	IMG_BOOL   bPxEValid;
+	IMG_UINT32 uiPxEIdx;
+	IMG_INT32  iShiftAmount;
+	IMG_CHAR   *pszWrwSuffix = NULL;
+	void *pvRawBytes = NULL;
+	IMG_CHAR aszPxSymbolicAddr[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_UINT64 ui64PxE64;
+	IMG_UINT64 ui64Protflags64;
+	IMG_CHAR *pszMMUPX;
+
+	PDUMP_GET_SCRIPT_STRING();
+	ui32Flags |= (PDUMP_FLAGS_BLKDATA | PDUMP_FLAGS_CONTINUOUS);
+
+	eErr = PDumpReady();
+	if (eErr != PVRSRV_OK)
+	{
+		/* Mask suspension from caller as this is terminal & logged */
+		eErr = (eErr == PVRSRV_ERROR_PDUMP_NOT_ACTIVE) ? PVRSRV_OK : eErr;
+		goto ErrOut;
+	}
+
+	PVR_GOTO_IF_INVALID_PARAM(pvPxMem, eErr, ErrOut);
+
+	/*
+	  create the symbolic address of the Px
+	*/
+	ui64PxSymAddr = sPxDevPAddr.uiAddr;
+
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(eMMULevel);
+	}
+
+	OSSNPrintf(aszPxSymbolicAddr,
+	           PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	           ":%s:%s%016"IMG_UINT64_FMTSPECX,
+	           pszPDumpDevName,
+	           pszMMUPX,
+	           ui64PxSymAddr);
+
+	PDUMP_LOCK(ui32Flags);
+
+	/*
+	  traverse PxEs, dumping entries
+	*/
+	for (uiPxEIdx = uiFirstEntry;
+	     uiPxEIdx < uiFirstEntry + uiNumEntries;
+	     uiPxEIdx++)
+	{
+		IMG_BOOL bLastEntry = (uiPxEIdx == uiFirstEntry + uiNumEntries - 1);
+		/* Calc the symbolic address offset of the PxE location
+		   This is what we have to add to the table address to get to a certain entry */
+		ui32SymAddrOffset = (uiPxEIdx*uiBytesPerEntry);
+
+		/* Calc the symbolic address of the PxE value and HW protflags */
+		/* just read it here */
+		switch (uiBytesPerEntry)
+		{
+			case 4:
+			{
+				pui32PxMem = pvPxMem;
+				ui64PxE64 = pui32PxMem[uiPxEIdx];
+				pszWrwSuffix = "";
+				pvRawBytes = &pui32PxMem[uiPxEIdx];
+				break;
+			}
+			case 8:
+			{
+				pui64PxMem = pvPxMem;
+				ui64PxE64 = pui64PxMem[uiPxEIdx];
+				pszWrwSuffix = "64";
+				pvRawBytes = &pui64PxMem[uiPxEIdx];
+				break;
+			}
+			default:
+			{
+				PVR_DPF((PVR_DBG_ERROR, "PDumpMMUPxEntries: error"));
+				ui64PxE64 = 0;
+				//!!error
+				break;
+			}
+		}
+
+		ui64PxEValueSymAddr = (ui64PxE64 & uiAddrMask) >> uiAddrShift << uiLog2Align;
+		ui64Protflags64 = ui64PxE64 & uiPxEProtMask;
+		bPxEValid = (ui64Protflags64 & uiDataValidEnable) ? IMG_TRUE : IMG_FALSE;
+
+		if (!bPxEValid)
+		{
+			/* If the entry was "invalid", simply write the actual
+			   value found to the memory location */
+			eErr = _ContiguousPDumpBytes(psDeviceNode,aszPxSymbolicAddr,
+			                             ui32SymAddrOffset, IMG_FALSE,
+			                             uiBytesPerEntry, pvRawBytes,
+			                             ui32Flags);
+			if (bLastEntry && eErr == PVRSRV_OK)
+			{
+				goto done;
+			}
+			else if (eErr == PVRSRV_OK)
+			{
+				continue;
+			}
+			else
+			{
+				goto ErrUnlock;
+			}
+		}
+
+		_ContiguousPDumpBytes(psDeviceNode, aszPxSymbolicAddr,
+		                      ui32SymAddrOffset, IMG_TRUE,
+		                      0, NULL,
+		                      ui32Flags);
+
+		iShiftAmount = (IMG_INT32)(uiLog2Align - uiAddrShift);
+
+		/* First put the symbolic representation of the actual
+		   address of the entry into a pdump internal register */
+		/* MOV seemed cleaner here, since (a) it's 64-bit; (b) the
+		   target is not memory.  However, MOV cannot do the
+		   "reference" of the symbolic address.  Apparently WRW is
+		   correct. */
+
+		if (pszSymbolicAddr == NULL)
+		{
+			pszSymbolicAddr = "none";
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			if (iShiftAmount == 0)
+			{
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "WRW%s :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:%s:0x%"IMG_UINT64_FMTSPECX" | 0x%"IMG_UINT64_FMTSPECX"\n",
+				                     pszWrwSuffix,
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     pszMMUPX,
+				                     ui64PxSymAddr,
+				                     ui32SymAddrOffset,
+				                     /* src */
+				                     pszMemspaceName,
+				                     pszSymbolicAddr,
+				                     uiSymbolicAddrOffset,
+				                     /* ORing prot flags */
+				                     ui64Protflags64);
+			}
+			else
+			{
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                    "WRW :%s:$1 :%s:%s:0x%"IMG_UINT64_FMTSPECX"\n",
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     /* src */
+				                     pszMemspaceName,
+				                     pszSymbolicAddr,
+				                     uiSymbolicAddrOffset);
+			}
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+			/* OR a parity bit */
+			if (ui64ParityMask)
+			{
+				IMG_UINT32 uiStateWord =
+					((0x1U << PDUMP_SET_PARITY_STATE_WORD_VERSION_SHIFT) & PDUMP_SET_PARITY_STATE_WORD_VERSION_MASK) |
+					((ui32ParityShift << PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_SHIFT) & PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_MASK) |
+					((ui32VAParity << PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_SHIFT) & PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_MASK);
+
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "CMD:SetParity :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:%s:0x%"IMG_UINT64_FMTSPECX" 0x%08X\n",
+				                     /* Dest PTE entry tag */
+				                     pszPDumpDevName,
+				                     pszMMUPX,
+				                     ui64PxSymAddr,
+				                     ui32SymAddrOffset,
+				                     /* Source tag to calc PA parity from */
+				                     pszMemspaceName,
+				                     pszSymbolicAddr,
+				                     uiSymbolicAddrOffset,
+				                     /* State word */
+				                     uiStateWord);
+
+				PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+				PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+			}
+		}
+		else
+		{
+			if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+			{
+				pszMMUPX = MIPSMMUPX_FMT(eMMULevel - 1);
+			}
+			else
+			{
+				pszMMUPX = MMUPX_FMT(eMMULevel - 1);
+			}
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "WRW :%s:$1 :%s:%s%016"IMG_UINT64_FMTSPECX":0x0",
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src */
+			                     pszPDumpDevName,
+			                     pszMMUPX,
+			                     ui64PxEValueSymAddr);
+			if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+			{
+				pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+			}
+			else
+			{
+				pszMMUPX = MMUPX_FMT(eMMULevel);
+			}
+
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+
+
+		/* Now shift it to the right place, if necessary: */
+		/* Now shift that value down, by the "Align shift"
+		   amount, to get it into units (ought to assert that
+		   we get an integer - i.e. we don't shift any bits
+		   off the bottom, don't know how to do PDUMP
+		   assertions yet) and then back up by the right
+		   amount to get it into the position of the field.
+		   This is optimised into a single shift right by the
+		   difference between the two. */
+		if (iShiftAmount > 0)
+		{
+			/* Page X Address is specified in units larger
+			   than the position in the PxE would suggest. */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "SHR :%s:$1 :%s:$1 0x%X",
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src A */
+			                     pszPDumpDevName,
+			                     /* src B */
+			                     iShiftAmount);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+		else if (iShiftAmount < 0)
+		{
+			/* Page X Address is specified in units smaller
+			   than the position in the PxE would suggest. */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "SHL :%s:$1 :%s:$1 0x%X",
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src A */
+			                     pszPDumpDevName,
+			                     /* src B */
+			                     -iShiftAmount);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			if (iShiftAmount != 0)
+			{
+				/* Now we can "or" in the protection flags */
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "OR :%s:$1 :%s:$1 0x%"IMG_UINT64_FMTSPECX,
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     /* src A */
+				                     pszPDumpDevName,
+				                     /* src B */
+				                     ui64Protflags64);
+				PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+				PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "WRW%s :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:$1",
+				                     pszWrwSuffix,
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     pszMMUPX,
+				                     ui64PxSymAddr,
+				                     ui32SymAddrOffset,
+				                     /* src */
+				                     pszPDumpDevName);
+				PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+				PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+			}
+		}
+		else
+		{
+			/* Now we can "or" in the protection flags */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "OR :%s:$1 :%s:$1 0x%"IMG_UINT64_FMTSPECX,
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src A */
+			                     pszPDumpDevName,
+			                     /* src B */
+			                     ui64Protflags64);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+			/* Finally, we write the register into the actual PxE */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "WRW%s :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:$1",
+			                     pszWrwSuffix,
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     pszMMUPX,
+			                     ui64PxSymAddr,
+			                     ui32SymAddrOffset,
+			                     /* src */
+			                     pszPDumpDevName);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+	}
+
+done:
+	/* flush out any partly accumulated stuff for LDB */
+	_ContiguousPDumpBytes(psDeviceNode, aszPxSymbolicAddr,
+	                      ui32SymAddrOffset, IMG_TRUE,
+	                      0, NULL,
+	                      ui32Flags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32Flags);
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : _PdumpAllocMMUContext
+ * Inputs         : pui32MMUContextID
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : pdump util to allocate MMU contexts
+**************************************************************************/
+static PVRSRV_ERROR _PdumpAllocMMUContext(IMG_UINT32 *pui32MMUContextID)
+{
+	IMG_UINT32 i;
+
+	/* there are MAX_PDUMP_MMU_CONTEXTS contexts available, find one */
+	for (i=0; i<MAX_PDUMP_MMU_CONTEXTS; i++)
+	{
+		if ((guiPDumpMMUContextAvailabilityMask & (1U << i)))
+		{
+			/* mark in use */
+			guiPDumpMMUContextAvailabilityMask &= ~(1U << i);
+			*pui32MMUContextID = i;
+			return PVRSRV_OK;
+		}
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "_PdumpAllocMMUContext: no free MMU context ids"));
+
+	return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+}
+
+/**************************************************************************
+ * Function Name  : _PdumpFreeMMUContext
+ * Inputs         : ui32MMUContextID
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : pdump util to free MMU contexts
+**************************************************************************/
+static PVRSRV_ERROR _PdumpFreeMMUContext(IMG_UINT32 ui32MMUContextID)
+{
+	if (ui32MMUContextID < MAX_PDUMP_MMU_CONTEXTS)
+	{
+		/* free the id */
+		PVR_ASSERT (!(guiPDumpMMUContextAvailabilityMask & (1U << ui32MMUContextID)));
+		guiPDumpMMUContextAvailabilityMask |= (1U << ui32MMUContextID);
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "_PdumpFreeMMUContext: MMU context ids invalid"));
+
+	return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUAllocMMUContext
+ * Inputs         :
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Alloc MMU Context
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUAllocMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+				const IMG_CHAR *pszPDumpMemSpaceName,
+				     IMG_DEV_PHYADDR sPCDevPAddr,
+				     PDUMP_MMU_TYPE eMMUType,
+				     IMG_UINT32 *pui32MMUContextID,
+				     IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT64 ui64PCSymAddr;
+	IMG_CHAR *pszMMUPX;
+	IMG_UINT32 ui32MMUContextID;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = _PdumpAllocMMUContext(&ui32MMUContextID);
+	if (eErr != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eErr, "_PdumpAllocMMUContext");
+		PVR_DBG_BREAK;
+		goto ErrOut;
+	}
+
+	/*
+	  create the symbolic address of the PC
+	*/
+	ui64PCSymAddr = sPCDevPAddr.uiAddr;
+
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(1);
+		/* Giving it a mock value until the Pdump player implements
+		   the support for the MIPS microAptiv MMU*/
+		eMMUType = PDUMP_MMU_TYPE_VARPAGE_40BIT;
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(3);
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "MMU :%s:v%d %d :%s:%s%016"IMG_UINT64_FMTSPECX,
+			     /* mmu context */
+			     pszPDumpMemSpaceName,
+			     ui32MMUContextID,
+			     /* mmu type */
+			     eMMUType,
+			     /* PC base address */
+			     pszPDumpMemSpaceName,
+			     pszMMUPX,
+			     ui64PCSymAddr);
+	if (eErr != PVRSRV_OK)
+	{
+		PVR_DBG_BREAK;
+		goto ErrOut;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	/* return the MMU Context ID */
+	*pui32MMUContextID = ui32MMUContextID;
+
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUFreeMMUContext
+ * Inputs         :
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Free MMU Context
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUFreeMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+				    const IMG_CHAR *pszPDumpMemSpaceName,
+				    IMG_UINT32 ui32MMUContextID,
+				    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	PDUMP_GET_SCRIPT_STRING();
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "-- Clear MMU Context for memory space %s", pszPDumpMemSpaceName);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode,hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+	                     ui32MaxLen,
+	                     "MMU :%s:v%d",
+	                     pszPDumpMemSpaceName,
+	                     ui32MMUContextID);
+
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = _PdumpFreeMMUContext(ui32MMUContextID);
+	PVR_LOG_GOTO_IF_ERROR(eErr, "_PdumpFreeMMUContext", ErrUnlock);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+PVRSRV_ERROR
+PDumpMMUSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+	    const IMG_CHAR *pszPDumpMemNamespace,
+	    IMG_UINT32 uiPDumpMMUCtx,
+	    IMG_DEV_VIRTADDR sDevAddrStart,
+	    IMG_DEVMEM_SIZE_T uiSize,
+	    const IMG_CHAR *pszFilename,
+	    IMG_UINT32 uiFileOffset,
+	    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	// "SAB :%s:v%x:0x%010llX 0x%08X 0x%08X %s.bin",
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpReady();
+	if (eErr != PVRSRV_OK)
+	{
+		/* Mask suspension from caller as this is terminal & logged */
+		eErr = (eErr == PVRSRV_ERROR_PDUMP_NOT_ACTIVE) ? PVRSRV_OK : eErr;
+		goto ErrOut;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "SAB :%s:v%x:" IMG_DEV_VIRTADDR_FMTSPEC " "
+			     IMG_DEVMEM_SIZE_FMTSPEC " "
+			     "0x%x %s.bin\n",
+			     pszPDumpMemNamespace,
+			     uiPDumpMMUCtx,
+			     sDevAddrStart.uiAddr,
+			     uiSize,
+			     uiFileOffset,
+			     pszFilename);
+	PVR_GOTO_IF_ERROR(eErr, ErrOut);
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+#endif /* #if defined(PDUMP) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_physmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_physmem.c
new file mode 100644
index 000000000000..e25b1f40e069
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_physmem.c
@@ -0,0 +1,742 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common PDump (PMR specific) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /***************************************************************************/
+
+#if defined(PDUMP)
+
+#if defined(__linux__)
+#include <linux/ctype.h>
+#else
+#include <ctype.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "pdump_physmem.h"
+#include "pdump_km.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+
+#include "pvrsrv.h"
+
+/* #define MAX_PDUMP_MMU_CONTEXTS	(10) */
+/* static IMG_UINT32 guiPDumpMMUContextAvailabilityMask = (1<<MAX_PDUMP_MMU_CONTEXTS)-1; */
+
+
+struct _PDUMP_PHYSMEM_INFO_T_
+{
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+	IMG_UINT64 ui64Size;
+	IMG_UINT32 ui32Align;
+	IMG_UINT32 ui32SerialNum;
+};
+
+static IMG_BOOL _IsAllowedSym(IMG_CHAR sym)
+{
+	/* Numbers, Characters or '_' are allowed */
+	if (isalnum(sym) || sym == '_')
+		return IMG_TRUE;
+	else
+		return IMG_FALSE;
+}
+
+static IMG_BOOL _IsLowerCaseSym(IMG_CHAR sym)
+{
+	if (sym >= 'a' && sym <= 'z')
+		return IMG_TRUE;
+	else
+		return IMG_FALSE;
+}
+
+void PDumpMakeStringValid(IMG_CHAR *pszString,
+                          IMG_UINT32 ui32StrLen)
+{
+	IMG_UINT32 i;
+
+	if (pszString)
+	{
+		for (i = 0; i < ui32StrLen; i++)
+		{
+			if (_IsAllowedSym(pszString[i]))
+			{
+				if (_IsLowerCaseSym(pszString[i]))
+					pszString[i] = pszString[i]-32;
+				else
+					pszString[i] = pszString[i];
+			}
+			else
+			{
+				pszString[i] = '_';
+			}
+		}
+	}
+}
+
+/**************************************************************************
+ * Function Name  : PDumpGetSymbolicAddr
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+ **************************************************************************/
+PVRSRV_ERROR PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                                  IMG_CHAR **ppszSymbolicAddress)
+{
+	PDUMP_PHYSMEM_INFO_T *psPDumpAllocationInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(hPhysmemPDumpHandle);
+
+	psPDumpAllocationInfo = (PDUMP_PHYSMEM_INFO_T *)hPhysmemPDumpHandle;
+	*ppszSymbolicAddress = psPDumpAllocationInfo->aszSymbolicAddress;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpMalloc
+@Description    Builds and writes an allocation command to pdump output. Whilst
+                writing the thread is locked.
+
+@Input          psDeviceNode        A pointer to a device node.
+@Input          pszDevSpace         Device space string.
+@Input          pszSymbolicAddress  Name of the allocation.
+@Input          ui64Size            String size.
+@Input          uiAlign             Command alignment.
+@Input          bInitialise         Should the command initialise the allocation.
+@Input          ui8InitValue        The value memory is initialised to.
+@Input          phHandlePtr         PDump allocation handle.
+@Input          ui32PDumpFlags      PDump allocation flags.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         const IMG_CHAR *pszDevSpace,
+                         const IMG_CHAR *pszSymbolicAddress,
+                         IMG_UINT64 ui64Size,
+                         IMG_DEVMEM_ALIGN_T uiAlign,
+                         IMG_BOOL bInitialise,
+                         IMG_UINT8 ui8InitValue,
+                         IMG_HANDLE *phHandlePtr,
+                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	eError = PDumpMallocUnlocked(psDeviceNode,
+	                             pszDevSpace,
+	                             pszSymbolicAddress,
+	                             ui64Size,
+	                             uiAlign,
+	                             bInitialise,
+	                             ui8InitValue,
+	                             phHandlePtr,
+	                             ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpMallocUnlocked
+@Description    Builds and writes an allocation command to pdump output. Whilst
+                writing the thread remains unlocked.
+
+@Input          psDeviceNode        A pointer to a device node.
+@Input          pszDevSpace         Device space string.
+@Input          pszSymbolicAddress  Name of the allocation.
+@Input          ui64Size            String size.
+@Input          uiAlign             Command alignment.
+@Input          bInitialise         Should the command initialise the allocation.
+@Input          ui8InitValue        The value memory is initialised to.
+@Input          phHandlePtr         PDump allocation handle.
+@Input          ui32PDumpFlags      PDump allocation flags.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const IMG_CHAR *pszDevSpace,
+                                 const IMG_CHAR *pszSymbolicAddress,
+                                 IMG_UINT64 ui64Size,
+                                 IMG_DEVMEM_ALIGN_T uiAlign,
+                                 IMG_BOOL bInitialise,
+                                 IMG_UINT8 ui8InitValue,
+                                 IMG_HANDLE *phHandlePtr,
+                                 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDUMP_PHYSMEM_INFO_T *psPDumpAllocationInfo;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psPDumpAllocationInfo = OSAllocMem(sizeof*psPDumpAllocationInfo);
+	PVR_ASSERT(psPDumpAllocationInfo != NULL);
+
+	/*
+	 * PDUMP_CONT and PDUMP_PERSIST flag can't set together.
+	 */
+	if (ui32PDumpFlags == PDUMP_NONE)
+	{
+		/*
+			Set continuous flag because there is no way of knowing beforehand which
+			allocation is needed for playback of the captured range.
+		*/
+		ui32PDumpFlags |= PDUMP_FLAGS_CONTINUOUS;
+	}
+
+	ui32PDumpFlags |= PDUMP_FLAGS_BLKDATA;
+
+	/*
+		construct the symbolic address
+	 */
+
+	OSSNPrintf(psPDumpAllocationInfo->aszSymbolicAddress,
+	           sizeof(psPDumpAllocationInfo->aszSymbolicAddress),
+	           ":%s:%s",
+	           pszDevSpace,
+	           pszSymbolicAddress);
+
+	/*
+		Write to the MMU script stream indicating the memory allocation
+	*/
+	if (bInitialise)
+	{
+		eError = PDumpSNPrintf(hScript, ui32MaxLen, "CALLOC %s 0x%"IMG_UINT64_FMTSPECX" 0x%"IMG_UINT64_FMTSPECX" 0x%X\n",
+		                          psPDumpAllocationInfo->aszSymbolicAddress,
+		                          ui64Size,
+		                          uiAlign,
+		                          ui8InitValue);
+	}
+	else
+	{
+		eError = PDumpSNPrintf(hScript, ui32MaxLen, "MALLOC %s 0x%"IMG_UINT64_FMTSPECX" 0x%"IMG_UINT64_FMTSPECX"\n",
+		                          psPDumpAllocationInfo->aszSymbolicAddress,
+		                          ui64Size,
+		                          uiAlign);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psPDumpAllocationInfo);
+		goto _return;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	psPDumpAllocationInfo->ui64Size = ui64Size;
+	psPDumpAllocationInfo->ui32Align = TRUNCATE_64BITS_TO_32BITS(uiAlign);
+
+	*phHandlePtr = (IMG_HANDLE)psPDumpAllocationInfo;
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpFree
+@Description    Writes a FREE command for an allocation handle to the pdump out2
+                stream. When writing to the output stream the thread is locked.
+
+@Input          psDeviceNode                A pointer to a device node.
+@Input          hPDumpAllocationInfoHandle  A PDump allocation handle.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_LOCK(PDUMP_FLAGS_NONE);
+	eError = PDumpFreeUnlocked(psDeviceNode, hPDumpAllocationInfoHandle);
+	PDUMP_UNLOCK(PDUMP_FLAGS_NONE);
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpFreeUnlocked
+@Description    Writes a FREE command for an allocation handle to the pdump
+                out2 stream. When writing to the output stream the thread
+                remains unlocked.
+
+@Input          psDeviceNode                A pointer to a device node.
+@Input          hPDumpAllocationInfoHandle  A PDump allocation handle.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDUMP_PHYSMEM_INFO_T *psPDumpAllocationInfo;
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psPDumpAllocationInfo = (PDUMP_PHYSMEM_INFO_T *)hPDumpAllocationInfoHandle;
+
+	/*
+		Write to the MMU script stream indicating the memory free
+	 */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "FREE %s\n",
+	                          psPDumpAllocationInfo->aszSymbolicAddress);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	OSFreeMem(psPDumpAllocationInfo);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW32(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT32 ui32Value,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                       ui32MaxLen,
+	                       "WRW :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                       PMR_VALUE32_FMTSPEC,
+	                       pszDevSpace,
+	                       pszSymbolicName,
+	                       uiOffset,
+	                       ui32Value);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW32InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "WRW :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " %s",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          pszInternalVar);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRRDW32MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "RDW %s :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC,
+	                          pszInternalVar,
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW64(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT64 ui64Value,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                       ui32MaxLen,
+	                       "WRW64 :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                       PMR_VALUE64_FMTSPEC,
+	                       pszDevSpace,
+	                       pszSymbolicName,
+	                       uiOffset,
+	                       ui64Value);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW64InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "WRW64 :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " %s",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          pszInternalVar);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRRDW64MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "RDW64 %s :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC,
+	                          pszInternalVar,
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRLDB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "LDB :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          IMG_DEVMEM_SIZE_FMTSPEC " "
+	                          PDUMP_FILEOFFSET_FMTSPEC " %s\n",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          uiSize,
+	                          uiFileOffset,
+	                          pszFilename);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR PDumpPMRSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         const IMG_CHAR *pszDevSpace,
+                         const IMG_CHAR *pszSymbolicName,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         const IMG_CHAR *pszFileName,
+                         IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiPDumpFlags;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	uiPDumpFlags = 0;
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "SAB :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          IMG_DEVMEM_SIZE_FMTSPEC " "
+	                          "0x%08X %s.bin\n",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          uiSize,
+	                          uiFileOffset,
+	                          pszFileName);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRPOL(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_UINT32 ui32Value,
+            IMG_UINT32 ui32Mask,
+            PDUMP_POLL_OPERATOR eOperator,
+            IMG_UINT32 uiCount,
+            IMG_UINT32 uiDelay,
+            PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "POL :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          "0x%08X 0x%08X %d %d %d\n",
+	                          pszMemspaceName,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          ui32Value,
+	                          ui32Mask,
+	                          eOperator,
+	                          uiCount,
+	                          uiDelay);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDUMP_FLAGS_T uiPDumpFlags = 0;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "CBP :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          IMG_DEVMEM_OFFSET_FMTSPEC " " IMG_DEVMEM_SIZE_FMTSPEC " " IMG_DEVMEM_SIZE_FMTSPEC "\n",
+	                          pszMemspaceName,
+	                          pszSymbolicName,
+	                          uiReadOffset,
+	                          uiWriteOffset,
+	                          uiPacketSize,
+	                          uiBufferSize);
+
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpWriteParameterBlob(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiNumBytes,
+                        PDUMP_FLAGS_T uiPDumpFlags,
+                        IMG_CHAR *pszFilenameOut,
+                        size_t uiFilenameBufSz,
+                        PDUMP_FILEOFFSET_T *puiOffsetOut)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
+	PVR_UNREFERENCED_PARAMETER(uiFilenameBufSz);
+
+	PVR_ASSERT(uiNumBytes > 0);
+
+	eError = PDumpReady();
+	if (eError != PVRSRV_OK)
+	{
+		/* Mask suspension from caller as this is terminal & logged */
+		eError = (eError == PVRSRV_ERROR_PDUMP_NOT_ACTIVE) ?
+				PVRSRV_ERROR_PDUMP_NOT_ALLOWED :
+				eError;
+		return eError;
+	}
+
+	PVR_ASSERT(uiFilenameBufSz <= PDUMP_PARAM_MAX_FILE_NAME);
+
+	PDUMP_LOCK(uiPDumpFlags);
+
+	eError = PDumpWriteParameter(psDeviceNode, pcBuffer, uiNumBytes,
+	                             uiPDumpFlags, puiOffsetOut, pszFilenameOut);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+	if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_PDUMP_NOT_ALLOWED))
+	{
+		PVR_LOG_RETURN_IF_ERROR(eError, "PDumpWriteParameter");
+	}
+	/* else Write to parameter file Ok or Prevented under the flags or
+	 * current state of the driver so skip further writes and let caller know.
+	 */
+	return eError;
+}
+
+#endif /* PDUMP */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_server.c
new file mode 100644
index 000000000000..4b42e9c207a0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_server.c
@@ -0,0 +1,5740 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common Server PDump functions layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(PDUMP)
+
+#if defined(__linux__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "pvrversion.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "pdump_physmem.h"
+#include "hash.h"
+#include "connection_server.h"
+#include "services_km.h"
+#include <powervr/buffer_attribs.h>
+#include "os_apphint.h"
+
+/* pdump headers */
+#include "tlstream.h"
+#include "pdump_km.h"
+
+#include "pdumpdesc.h"
+#include "rgxpdump.h"
+
+/* Allow temporary buffer size override */
+#if !defined(PDUMP_TEMP_BUFFER_SIZE)
+#define PDUMP_TEMP_BUFFER_SIZE (64 * 1024U)
+#endif
+
+#define	PTR_PLUS(t, p, x) ((t)(((IMG_CHAR *)(p)) + (x)))
+#define	VPTR_PLUS(p, x) PTR_PLUS(void *, p, x)
+#define	VPTR_INC(p, x) ((p) = VPTR_PLUS(p, x))
+#define MAX_PDUMP_MMU_CONTEXTS	(32)
+
+#define PRM_FILE_SIZE_MAX	0x7FDFFFFFU /*!< Default maximum file size to split output files, 2GB-2MB as fwrite limits it to 2GB-1 on 32bit systems */
+
+#define MAX_PDUMP_WRITE_RETRIES	200	/*!< Max number of retries to dump pdump data in to respective buffers */
+
+/* 'Magic' cookie used in this file only, where no psDeviceNode is available
+ * but writing to the PDump log should be permitted
+ */
+#define PDUMP_MAGIC_COOKIE 0x9E0FF
+
+static ATOMIC_T		g_sConnectionCount;
+
+/*
+ * Structure to store some essential attributes of a PDump stream buffer.
+ */
+typedef struct
+{
+	IMG_CHAR*  pszName;                     /*!< Name of the PDump TL Stream buffer */
+	IMG_HANDLE hTL;                         /*!< Handle of created TL stream buffer */
+	IMG_UINT32 ui32BufferSize;              /*!< The size of the buffer in bytes */
+	IMG_UINT32 ui32BufferFullRetries;       /*!< The number of times the buffer got full */
+	IMG_UINT32 ui32BufferFullAborts;        /*!< The number of times we failed to write data */
+	IMG_UINT32 ui32HighestRetriesWatermark; /*!< Max number of retries try to dump pdump data */
+	IMG_UINT32 ui32MaxAllowedWriteSize;     /*!< Max allowed write packet size */
+} PDUMP_STREAM;
+
+typedef struct
+{
+	PDUMP_STREAM sInitStream;   /*!< Driver initialisation PDump stream */
+	PDUMP_STREAM sMainStream;   /*!< App framed PDump stream */
+	PDUMP_STREAM sDeinitStream; /*!< Driver/HW de-initialisation PDump stream */
+	PDUMP_STREAM sBlockStream;  /*!< Block mode PDump block data stream - currently its script only */
+} PDUMP_CHANNEL;
+
+typedef struct
+{
+	PDUMP_CHANNEL sCh;         /*!< Channel handles */
+	IMG_UINT32    ui32FileIdx; /*!< File index gets incremented on script out file split */
+} PDUMP_SCRIPT;
+
+typedef struct
+{
+	IMG_UINT32    ui32Init;    /*!< Count of bytes written to the init phase stream */
+	IMG_UINT32    ui32Main;    /*!< Count of bytes written to the main stream */
+	IMG_UINT32    ui32Deinit;  /*!< Count of bytes written to the deinit stream */
+	IMG_UINT32    ui32Block;   /*!< Count of bytes written to the block stream */
+} PDUMP_CHANNEL_WOFFSETS;
+
+typedef struct
+{
+	PDUMP_CHANNEL          sCh;             /*!< Channel handles */
+	PDUMP_CHANNEL_WOFFSETS sWOff;           /*!< Channel file write offsets */
+	IMG_UINT32             ui32FileIdx;     /*!< File index used when file size limit reached and a new file is started, parameter channel only */
+	IMG_UINT32             ui32MaxFileSize; /*!< Maximum file size for parameter files */
+
+	PDUMP_FILEOFFSET_T     uiZeroPageOffset; /*!< Offset of the zero page in the parameter file */
+	size_t                 uiZeroPageSize;   /*!< Size of the zero page in the parameter file */
+	IMG_CHAR               szZeroPageFilename[PDUMP_PARAM_MAX_FILE_NAME]; /*< PRM file name where the zero page was pdumped */
+} PDUMP_PARAMETERS;
+
+/* PDump global connection count - used to determine when/if the last
+ * connection (from a PDump generating app) has been closed.
+ * This is used to key the AUTO_TERMINATED behaviour if enabled.
+ */
+static ATOMIC_T gPDumpNumConnex;
+
+/* PDump lock to keep pdump write atomic.
+ * Which will protect g_PDumpScript & g_PDumpParameters pdump
+ * specific shared variable.
+ */
+static POS_LOCK g_hPDumpWriteLock;
+
+static PDUMP_SCRIPT     g_PDumpScript    = { {
+		{	PDUMP_SCRIPT_INIT_STREAM_NAME,   NULL,
+			PDUMP_SCRIPT_INIT_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_SCRIPT_MAIN_STREAM_NAME,   NULL,
+			PDUMP_SCRIPT_MAIN_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_SCRIPT_DEINIT_STREAM_NAME, NULL,
+			PDUMP_SCRIPT_DEINIT_STREAM_SIZE, 0, 0, 0 },
+		{	PDUMP_SCRIPT_BLOCK_STREAM_NAME,   NULL,
+			PDUMP_SCRIPT_BLOCK_STREAM_SIZE,   0, 0, 0 },
+		}, 0 };
+static PDUMP_PARAMETERS g_PDumpParameters = { {
+		{	PDUMP_PARAM_INIT_STREAM_NAME,   NULL,
+			PDUMP_PARAM_INIT_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_PARAM_MAIN_STREAM_NAME,   NULL,
+			PDUMP_PARAM_MAIN_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_PARAM_DEINIT_STREAM_NAME, NULL,
+			PDUMP_PARAM_DEINIT_STREAM_SIZE, 0, 0, 0 },
+		{	PDUMP_PARAM_BLOCK_STREAM_NAME,   NULL,
+			PDUMP_PARAM_BLOCK_STREAM_SIZE,   0, 0, 0 },
+		}, {0, 0, 0, 0}, 0, PRM_FILE_SIZE_MAX};
+
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+/* counter increments each time debug write is called */
+ATOMIC_T g_sEveryLineCounter;
+#endif
+
+// #define PDUMP_DEBUG_TRANSITION
+#if defined(PDUMP_DEBUG_TRANSITION)
+# define DEBUG_OUTFILES_COMMENT(dev, fmt, ...) (void)PDumpCommentWithFlags(dev, PDUMP_FLAGS_CONTINUOUS, fmt, __VA_ARGS__)
+#else
+# define DEBUG_OUTFILES_COMMENT(dev, fmt, ...)
+#endif
+
+#if defined(PDUMP_DEBUG) || defined(REFCOUNT_DEBUG)
+# define PDUMP_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+# define PDUMP_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+/* Prototype for the test/debug state dump routine used in debugging */
+#if defined(PDUMP_TRACE_STATE)
+void PDumpCommonDumpState(void);
+#endif
+
+
+/*****************************************************************************/
+/* PDump Control Module Definitions                                          */
+/*****************************************************************************/
+
+/*
+ * struct _PDUMP_CAPTURE_RANGE_ is interpreted differently in different modes of PDump
+ *
+ * Non-Block mode:
+ *    ui32Start     - Start frame number of range
+ *    ui32End       - End frame number of range
+ *    ui32Interval  - Frame sample rate interval
+ *
+ * Block mode:
+ *    ui32Start     - If set to '0', first PDump-block will of minimal (i.e. PDUMP_BLOCKLEN_MIN)
+ *                    length, else all blocks will be of block-length provided
+ *
+ *    ui32End       - By default this is set to PDUMP_FRAME_MAX so that Blocked PDump
+ *                    will be captured indefinitely till stopped externally. On force capture
+ *                    stop, this will be set to (ui32CurrentFrame + 1) to stop capture from
+ *                    next frame onwards
+ *
+ *    ui32Interval  - This will be interpreted as PDump block-length provided
+ **/
+typedef struct _PDUMP_CAPTURE_RANGE_
+{
+	IMG_UINT32 ui32Start;
+	IMG_UINT32 ui32End;
+	IMG_UINT32 ui32Interval;
+} PDUMP_CAPTURE_RANGE;
+
+/* PDump Block mode specific controls */
+typedef struct _PDUMP_BLOCK_CTRL_
+{
+	IMG_UINT32 ui32BlockLength;       /*!< PDump block length in term of number of frames per block */
+	IMG_UINT32 ui32CurrentBlock;      /*!< Current block number */
+} PDUMP_BLOCK_CTRL;
+
+/*! PDump common module State Machine states */
+typedef enum _PDUMP_SM_
+{
+	PDUMP_SM_UNINITIALISED,           /*!< Starting state */
+	PDUMP_SM_INITIALISING,            /*!< Module is initialising */
+	PDUMP_SM_READY,                   /*!< Module is initialised and ready */
+	PDUMP_SM_READY_CLIENT_CONNECTED,  /*!< Module is ready and capture client connected */
+	PDUMP_SM_FORCED_SUSPENDED,        /*!< Module forced error, PDumping suspended, this is to force driver reload before next capture */
+	PDUMP_SM_ERROR_SUSPENDED,         /*!< Module fatal error, PDumping suspended semi-final state */
+	PDUMP_SM_DEINITIALISED            /*!< Final state */
+} PDUMP_SM;
+
+/*! PDump control flags */
+#define FLAG_IS_DRIVER_IN_INIT_PHASE 0x1  /*! Control flag that keeps track of State of driver initialisation phase */
+#define FLAG_IS_IN_CAPTURE_RANGE     0x2  /*! Control flag that keeps track of Current capture status, is current frame in range */
+#define FLAG_IS_IN_CAPTURE_INTERVAL  0x4  /*! Control flag that keeps track of Current capture status, is current frame in an interval where no capture takes place. */
+#define FLAG_IS_AUTO_TERMINATED      0x8  /*! Control flag that indicates app has auto-terminated. */
+
+#define CHECK_PDUMP_CONTROL_FLAG(PDUMP_CONTROL_FLAG) BITMASK_HAS(g_PDumpCtrl.ui32Flags, PDUMP_CONTROL_FLAG)
+#define SET_PDUMP_CONTROL_FLAG(PDUMP_CONTROL_FLAG)   BITMASK_SET(g_PDumpCtrl.ui32Flags, PDUMP_CONTROL_FLAG)
+#define UNSET_PDUMP_CONTROL_FLAG(PDUMP_CONTROL_FLAG) BITMASK_UNSET(g_PDumpCtrl.ui32Flags, PDUMP_CONTROL_FLAG)
+
+/* No direct access to members from outside the control module - please */
+typedef struct _PDUMP_CTRL_STATE_
+{
+	PDUMP_SM            eServiceState;      /*!< State of the pdump_common module */
+	IMG_UINT32          ui32Flags;
+
+	IMG_UINT32          ui32DefaultCapMode; /*!< Capture mode of the dump */
+	PDUMP_CAPTURE_RANGE sCaptureRange;      /*|< The capture range for capture mode 'framed' */
+	IMG_UINT32          ui32CurrentFrame;   /*!< Current frame number */
+
+	PDUMP_BLOCK_CTRL    sBlockCtrl;         /*!< Pdump block mode ctrl data */
+
+	POS_LOCK            hLock;              /*!< Exclusive lock to this structure */
+	IMG_PID             InPowerTransitionPID;/*!< pid of thread requesting power transition */
+	IMG_UINT32          ui32TimeoutFrequency;/*!< Timer frequency for checking process existence */
+} PDUMP_CTRL_STATE;
+
+static PDUMP_CTRL_STATE g_PDumpCtrl =
+{
+	PDUMP_SM_UNINITIALISED,
+
+	FLAG_IS_DRIVER_IN_INIT_PHASE,
+
+	PDUMP_CAPMODE_UNSET,
+	{
+		PDUMP_FRAME_UNSET,
+		PDUMP_FRAME_UNSET,
+		0
+	},
+	0,
+
+	{
+		0,
+		PDUMP_BLOCKNUM_INVALID,
+	},
+
+	NULL,
+	0,
+	0
+};
+
+static IMG_HANDLE g_PDumpTimerID;
+
+static void PDumpAssertWriteLockHeld(void);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+
+/*************************************************************************/ /*!
+ @Function		PDumpCreateIncVarNameStr
+ @Description	When 64 bit register access is split between two 32 bit
+	accesses, it needs two PDump Internal variables to store register value.
+	This function creates the string for the second PDump Internal variable
+	for example if Passed Variable name is :SYSMEM:$1 this function will
+	generate the string :SYSMEM:$2
+
+ @Input	pszInternalVar	String for PDump internal variable in use
+
+ @Return IMG_CHAR*  String for second PDump internal variable to be used
+*/ /**************************************************************************/
+static INLINE IMG_CHAR* PDumpCreateIncVarNameStr(const IMG_CHAR* pszInternalVar)
+{
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32Size = (IMG_UINT32)OSStringLength(pszInternalVar);
+	if (ui32Size == 0)
+	{
+		return NULL;
+	}
+
+	ui32Size++;
+	pszPDumpVarName = (IMG_CHAR*)OSAllocMem((ui32Size) * sizeof(IMG_CHAR));
+	if (pszPDumpVarName == NULL)
+	{
+		return NULL;
+	}
+
+	OSStringSafeCopy(pszPDumpVarName, pszInternalVar, ui32Size);
+	/* Increase the number on the second variable */
+	pszPDumpVarName[ui32Size-2] += 1;
+	return pszPDumpVarName;
+}
+
+/*************************************************************************/ /*!
+ @Function		PDumpFreeIncVarNameStr
+ @Description	Free the string created by function PDumpCreateIncVarNameStr
+
+ @Input	pszPDumpVarName String to free
+
+ @Return	void
+*/ /**************************************************************************/
+static INLINE void PDumpFreeIncVarNameStr(IMG_CHAR* pszPDumpVarName)
+{
+	if (pszPDumpVarName != NULL)
+	{
+		OSFreeMem(pszPDumpVarName);
+	}
+}
+#endif
+
+static PVRSRV_ERROR PDumpCtrlInit(void)
+{
+	g_PDumpCtrl.eServiceState = PDUMP_SM_INITIALISING;
+
+	/* Create lock for PDUMP_CTRL_STATE struct, which is shared between pdump client
+	   and PDumping app. This lock will help us serialize calls from pdump client
+	   and PDumping app */
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&g_PDumpCtrl.hLock), "OSLockCreate");
+
+	return PVRSRV_OK;
+}
+
+static void PDumpCtrlDeInit(void)
+{
+	if (g_PDumpCtrl.hLock)
+	{
+		OSLockDestroy(g_PDumpCtrl.hLock);
+		g_PDumpCtrl.hLock = NULL;
+	}
+}
+
+static INLINE void PDumpCtrlLockAcquire(void)
+{
+	OSLockAcquire(g_PDumpCtrl.hLock);
+}
+
+static INLINE void PDumpCtrlLockRelease(void)
+{
+	OSLockRelease(g_PDumpCtrl.hLock);
+}
+
+static INLINE PDUMP_SM PDumpCtrlGetModuleState(void)
+{
+	return g_PDumpCtrl.eServiceState;
+}
+
+PVRSRV_ERROR PDumpValidateUMFlags(PDUMP_FLAGS_T uiFlags)
+{
+	/* If these flags are or'd together, they are invalid */
+	return ((uiFlags & (~(PDUMP_BLKDATA | PDUMP_CONT))) != 0) ? PVRSRV_ERROR_INVALID_PARAMS : PVRSRV_OK;
+}
+
+PVRSRV_ERROR PDumpReady(void)
+{
+	switch (PDumpCtrlGetModuleState())
+	{
+	case PDUMP_SM_READY:
+	case PDUMP_SM_READY_CLIENT_CONNECTED:
+		return PVRSRV_OK;
+
+	case PDUMP_SM_FORCED_SUSPENDED:
+	case PDUMP_SM_ERROR_SUSPENDED:
+		return PVRSRV_ERROR_PDUMP_NOT_ACTIVE;
+
+	case PDUMP_SM_UNINITIALISED:
+	case PDUMP_SM_INITIALISING:
+	case PDUMP_SM_DEINITIALISED:
+		return PVRSRV_ERROR_PDUMP_NOT_AVAILABLE;
+
+	default:
+		/* Bad state */
+		PVR_ASSERT(1);
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+}
+
+
+/******************************************************************************
+	NOTE:
+	The following PDumpCtrl*** functions require the PDUMP_CTRL_STATE lock be
+	acquired BEFORE they are called. This is because the PDUMP_CTRL_STATE data
+	is shared between the PDumping App and the PDump client, hence an exclusive
+	access is required. The lock can be acquired and released by using the
+	PDumpCtrlLockAcquire & PDumpCtrlLockRelease functions respectively.
+******************************************************************************/
+
+static void PDumpCtrlUpdateCaptureStatus(void)
+{
+	if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_FRAMED)
+	{
+		if ((g_PDumpCtrl.ui32CurrentFrame >= g_PDumpCtrl.sCaptureRange.ui32Start) &&
+			(g_PDumpCtrl.ui32CurrentFrame <= g_PDumpCtrl.sCaptureRange.ui32End))
+		{
+			if (((g_PDumpCtrl.ui32CurrentFrame - g_PDumpCtrl.sCaptureRange.ui32Start) % g_PDumpCtrl.sCaptureRange.ui32Interval) == 0)
+			{
+				SET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+				UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+			}
+			else
+			{
+				UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+				SET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+			}
+		}
+		else
+		{
+			UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+			UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+		}
+	}
+	else if ((g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_CONTINUOUS) || (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_BLOCKED))
+	{
+		SET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+	}
+	else if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_UNSET)
+	{
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+	}
+	else
+	{
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+		PVR_DPF((PVR_DBG_ERROR, "PDumpCtrlUpdateCaptureStatus: Unexpected capture mode (%x)", g_PDumpCtrl.ui32DefaultCapMode));
+	}
+
+}
+
+static INLINE IMG_UINT32 PDumpCtrlCapModIsBlocked(void)
+{
+	return (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_BLOCKED);
+}
+
+static INLINE IMG_UINT32 PDumpCtrlMinimalFirstBlock(void)
+{
+	/* If ui32Start is set to zero, first block length will be set to minimum
+	 * (i.e. PDUMP_BLOCKLEN_MIN), else it will be of same length as that of
+	 * rest of the blocks (i.e. ui32BlockLength)
+	 *
+	 * Having shorter first block reduces playback time of final capture.
+	 * */
+
+	return (PDumpCtrlCapModIsBlocked() && (g_PDumpCtrl.sCaptureRange.ui32Start == 0));
+}
+
+static void PDumpCtrlSetBlock(IMG_UINT32 ui32BlockNum)
+{
+	g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock = PDumpCtrlCapModIsBlocked()? ui32BlockNum : PDUMP_BLOCKNUM_INVALID;
+}
+
+static INLINE IMG_UINT32 PDumpCtrlGetBlock(void)
+{
+	return PDumpCtrlCapModIsBlocked()? g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock : PDUMP_BLOCKNUM_INVALID;
+}
+
+static PVRSRV_ERROR PDumpCtrlForcedStop(void)
+{
+	/* In block-mode on forced stop request, capture will be stopped after (current_frame + 1)th frame number.
+	 * This ensures that DumpAfterRender always be called on last frame before exiting the PDump capturing
+	 * */
+	g_PDumpCtrl.sCaptureRange.ui32End = g_PDumpCtrl.ui32CurrentFrame + 1;
+
+	return PVRSRV_OK;
+}
+
+static INLINE IMG_BOOL PDumpCtrlIsCaptureForceStopped(void)
+{
+	return (PDumpCtrlCapModIsBlocked() && (g_PDumpCtrl.ui32CurrentFrame > g_PDumpCtrl.sCaptureRange.ui32End));
+}
+
+static void PDumpCtrlSetCurrentFrame(IMG_UINT32 ui32Frame)
+{
+	g_PDumpCtrl.ui32CurrentFrame = ui32Frame;
+
+	PDumpCtrlUpdateCaptureStatus();
+
+	/* Force PDump module to suspend PDumping on forced capture stop */
+	if ((PDumpCtrlGetModuleState() != PDUMP_SM_FORCED_SUSPENDED) && PDumpCtrlIsCaptureForceStopped())
+	{
+		PVR_LOG(("PDump forced capture stop received. Suspend PDumping to force driver reload before next capture."));
+		g_PDumpCtrl.eServiceState = PDUMP_SM_FORCED_SUSPENDED;
+	}
+#if defined(PDUMP_TRACE_STATE)
+	PDumpCommonDumpState();
+#endif
+}
+
+static void PDumpCtrlSetDefaultCaptureParams(IMG_UINT32 ui32Mode, IMG_UINT32 ui32Start, IMG_UINT32 ui32End, IMG_UINT32 ui32Interval, IMG_UINT32 ui32AutoTermTimeout)
+{
+	/* Set the capture range to that supplied by the PDump client tool
+	 */
+	g_PDumpCtrl.ui32DefaultCapMode = ui32Mode;
+	g_PDumpCtrl.sCaptureRange.ui32Start = ui32Start;
+	g_PDumpCtrl.sCaptureRange.ui32End = ui32End;
+	g_PDumpCtrl.sCaptureRange.ui32Interval = ui32Interval;
+
+	/* Disable / Enable AUTO_TERMINATE behaviour if AutoTermTimeout is set */
+	if (ui32AutoTermTimeout == 0U)
+	{
+		if (g_PDumpTimerID != NULL)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = OSDisableTimer(g_PDumpTimerID);
+			PVR_LOG_IF_ERROR(eError, "OSDisableTimer");
+
+			/* Now destroy it too */
+			eError = OSRemoveTimer(g_PDumpTimerID);
+			PVR_LOG_IF_ERROR(eError, "OSRemoveTimer");
+
+			g_PDumpTimerID = NULL;
+		}
+		g_PDumpCtrl.ui32TimeoutFrequency = 0U;
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED);
+	}
+
+	/* Set pdump block mode ctrl variables */
+	g_PDumpCtrl.sBlockCtrl.ui32BlockLength = (ui32Mode == PDUMP_CAPMODE_BLOCKED)? ui32Interval : 0; /* ui32Interval is interpreted as block length */
+	g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock = PDUMP_BLOCKNUM_INVALID;
+
+	/* Change module state to record capture client connected */
+	if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_UNSET)
+		g_PDumpCtrl.eServiceState = PDUMP_SM_READY;
+	else
+		g_PDumpCtrl.eServiceState = PDUMP_SM_READY_CLIENT_CONNECTED;
+
+	/* Reset the current frame on reset of the capture range, the helps to
+	 * avoid inter-pdump start frame issues when the driver is not reloaded.
+	 * No need to call PDumpCtrlUpdateCaptureStatus() direct as the set
+	 * current frame call will.
+	 */
+	PDumpCtrlSetCurrentFrame(0);
+
+}
+
+static IMG_UINT32 PDumpCtrlGetCurrentFrame(void)
+{
+	return g_PDumpCtrl.ui32CurrentFrame;
+}
+
+static INLINE IMG_BOOL PDumpCtrlCaptureOn(void)
+{
+	return ((g_PDumpCtrl.eServiceState == PDUMP_SM_READY_CLIENT_CONNECTED) &&
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE)) ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE IMG_BOOL PDumpCtrlCaptureInInterval(void)
+{
+	return ((g_PDumpCtrl.eServiceState == PDUMP_SM_READY_CLIENT_CONNECTED) &&
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL)) ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE IMG_BOOL PDumpCtrlCaptureRangePast(void)
+{
+	return (g_PDumpCtrl.ui32CurrentFrame > g_PDumpCtrl.sCaptureRange.ui32End);
+}
+
+static IMG_BOOL PDumpCtrlIsLastCaptureFrame(void)
+{
+	if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_FRAMED)
+	{
+		/* Is the next capture frame within the range end limit? */
+		if ((g_PDumpCtrl.ui32CurrentFrame + g_PDumpCtrl.sCaptureRange.ui32Interval) > g_PDumpCtrl.sCaptureRange.ui32End)
+		{
+			return IMG_TRUE;
+		}
+	}
+	else if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_BLOCKED)
+	{
+		if (g_PDumpCtrl.ui32CurrentFrame == g_PDumpCtrl.sCaptureRange.ui32End)
+		{
+			return IMG_TRUE;
+		}
+	}
+	/* Return false for all other conditions: framed mode but not last frame,
+	 * continuous mode; unset mode.
+	 */
+	return IMG_FALSE;
+}
+
+static INLINE IMG_BOOL PDumpCtrlInitPhaseComplete(void)
+{
+	return !CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE);
+}
+
+static INLINE void PDumpCtrlSetInitPhaseComplete(IMG_BOOL bIsComplete)
+{
+	PDUMP_HERE_VAR;
+
+	if (bIsComplete)
+	{
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE);
+		PDUMP_HEREA(102);
+	}
+	else
+	{
+		SET_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE);
+		PDUMP_HEREA(103);
+	}
+}
+
+static INLINE void PDumpCtrlPowerTransitionStart(void)
+{
+	g_PDumpCtrl.InPowerTransitionPID = OSGetCurrentProcessID();
+}
+
+static INLINE void PDumpCtrlPowerTransitionEnd(void)
+{
+	g_PDumpCtrl.InPowerTransitionPID = 0;
+}
+
+static INLINE IMG_PID PDumpCtrlInPowerTransitionPID(void)
+{
+	return g_PDumpCtrl.InPowerTransitionPID;
+}
+
+static INLINE IMG_BOOL PDumpCtrlInPowerTransition(void)
+{
+	IMG_BOOL bPDumpInPowerTransition = IMG_FALSE;
+	if (PDumpCtrlInPowerTransitionPID())
+	{
+		bPDumpInPowerTransition = IMG_TRUE;
+	}
+	return bPDumpInPowerTransition;
+}
+
+static PVRSRV_ERROR PDumpCtrlGetState(IMG_UINT64 *ui64State)
+{
+	PDUMP_SM eState;
+
+	*ui64State = 0;
+
+	if (PDumpCtrlCaptureOn())
+	{
+		*ui64State |= PDUMP_STATE_CAPTURE_FRAME;
+	}
+
+	if (PDumpCtrlCaptureInInterval())
+	{
+		*ui64State |= PDUMP_STATE_CAPTURE_IN_INTERVAL;
+	}
+
+	eState = PDumpCtrlGetModuleState();
+
+	if (eState == PDUMP_SM_READY_CLIENT_CONNECTED)
+	{
+		*ui64State |= PDUMP_STATE_CONNECTED;
+	}
+
+	if (eState == PDUMP_SM_ERROR_SUSPENDED)
+	{
+		*ui64State |= PDUMP_STATE_SUSPENDED;
+	}
+
+	if (CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED))
+	{
+		*ui64State |= PDUMP_STATE_APP_TERMINATED;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PDumpSetAutoTerminate(IMG_UINT32 ui32TimeoutFrequency)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bEnable = (ui32TimeoutFrequency != 0U) ? IMG_TRUE : IMG_FALSE;
+
+	eError = PDumpReady();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PDumpReady");
+
+	g_PDumpCtrl.ui32TimeoutFrequency = ui32TimeoutFrequency;
+
+	if (bEnable)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: ENABLING Auto Termination - Timeout %u",
+		         __func__, ui32TimeoutFrequency));
+		PDUMP_REFCOUNT_PRINT(("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                      &gPDumpNumConnex, OSAtomicRead(&gPDumpNumConnex)));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: DISABLING Auto Termination",
+		         __func__));
+		PDUMP_REFCOUNT_PRINT(("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                      &gPDumpNumConnex,
+		                      OSAtomicRead(&gPDumpNumConnex)));
+	}
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+	End of PDumpCtrl*** functions
+******************************************************************************/
+
+/*
+	Wrapper functions which need to be exposed in pdump_km.h for use in other
+	pdump_*** modules safely. These functions call the specific PDumpCtrl layer
+	function after acquiring the PDUMP_CTRL_STATE lock, hence making the calls
+	from other modules hassle free by avoiding the acquire/release CtrlLock
+	calls.
+*/
+
+static INLINE void PDumpModuleTransitionState(PDUMP_SM eNewState)
+{
+	PDumpCtrlLockAcquire();
+	g_PDumpCtrl.eServiceState = eNewState;
+	PDumpCtrlLockRelease();
+}
+
+void PDumpPowerTransitionStart(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PDumpCtrlLockAcquire();
+		PDumpCtrlPowerTransitionStart();
+		PDumpCtrlLockRelease();
+	}
+}
+
+void PDumpPowerTransitionEnd(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PDumpCtrlLockAcquire();
+		PDumpCtrlPowerTransitionEnd();
+		PDumpCtrlLockRelease();
+	}
+}
+
+static PVRSRV_ERROR PDumpGetCurrentBlockKM(IMG_UINT32 *pui32BlockNum)
+{
+	PDumpCtrlLockAcquire();
+	*pui32BlockNum = PDumpCtrlGetBlock();
+	PDumpCtrlLockRelease();
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL PDumpIsClientConnected(void)
+{
+	IMG_BOOL bPDumpClientConnected;
+
+	PDumpCtrlLockAcquire();
+	bPDumpClientConnected = (PDumpCtrlGetModuleState() == PDUMP_SM_READY_CLIENT_CONNECTED);
+	PDumpCtrlLockRelease();
+
+	return bPDumpClientConnected;
+}
+
+/* Prototype write allowed for exposure in PDumpCheckFlagsWrite */
+static IMG_BOOL PDumpWriteAllowed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32Flags, IMG_UINT32* ui32ExitHere);
+
+IMG_BOOL PDumpCheckFlagsWrite(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_UINT32 ui32Flags)
+{
+	return PDumpWriteAllowed(psDeviceNode, ui32Flags, NULL);
+}
+
+/*****************************************************************************/
+/* PDump Common Write Layer just above common Transport Layer                */
+/*****************************************************************************/
+
+
+/*!
+ * \name	_PDumpOSGetStreamOffset
+ */
+static IMG_BOOL _PDumpSetSplitMarker(IMG_HANDLE hStream, IMG_BOOL bRemoveOld)
+{
+	PVRSRV_ERROR eError;
+	/* We have to indicate the reader that we wish to split. Insert an EOS packet in the TL stream */
+	eError = TLStreamMarkEOS(hStream, bRemoveOld);
+
+	/* If unsuccessful, return false */
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "TLStreamMarkEOS");
+
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+IMG_BOOL PDumpIsDevicePermitted(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if ((void*)psDeviceNode == (void*)PDUMP_MAGIC_COOKIE)
+	{
+		/* Always permit PDumping if passed 'magic' cookie */
+		return IMG_TRUE;
+	}
+
+	if (psDeviceNode)
+	{
+		if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_CREATED)
+		{
+			PVR_DPF((PVR_DBG_FATAL,"%s: PDump output requested for Device %d "
+			         "before device created. Not permitted - please fix driver.",
+			         __func__, psDeviceNode->sDevId.ui32InternalID));
+			return IMG_FALSE;
+		}
+		if ((psDeviceNode->sDevId.ui32InternalID > PVRSRV_MAX_DEVICES) ||
+		    ((psPVRSRVData->ui32PDumpBoundDevice < PVRSRV_MAX_DEVICES) &&
+		     (psDeviceNode->sDevId.ui32InternalID != psPVRSRVData->ui32PDumpBoundDevice)))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else
+	{
+		/* Assert if provided with a NULL psDeviceNode */
+		OSDumpStack();
+		PVR_ASSERT(psDeviceNode);
+		return IMG_FALSE;
+	}
+	return IMG_TRUE;
+}
+
+/*
+	Checks in this method were seeded from the original PDumpWriteILock()
+	and DBGDrivWriteCM() and have grown since to ensure PDump output
+	matches legacy output.
+	Note: the order of the checks in this method is important as some
+	writes have multiple pdump flags set!
+ */
+static IMG_BOOL PDumpWriteAllowed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32Flags, IMG_UINT32* ui32ExitHere)
+{
+	PDUMP_HERE_VAR;
+
+	/* No writes if for a different device than the PDump-bound device
+	 *  NB. psDeviceNode may be NULL if called during initialisation
+	 */
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PDUMP_HERE(5);
+		goto returnFalse;
+	}
+
+	/* PDUMP_FLAGS_CONTINUOUS and PDUMP_FLAGS_PERSISTENT can't come together. */
+	PVR_ASSERT(IMG_FALSE == ((ui32Flags & PDUMP_FLAGS_CONTINUOUS) &&
+		                     (ui32Flags & PDUMP_FLAGS_PERSISTENT)));
+
+	/* Lock down the PDUMP_CTRL_STATE struct before calling the following
+	   PDumpCtrl*** functions. This is to avoid updates to the Control data
+	   while we are reading from it */
+	PDumpCtrlLockAcquire();
+
+	/* No writes if in framed mode and range pasted */
+	if (PDumpCtrlCaptureRangePast())
+	{
+		PDUMP_HERE(10);
+		goto unlockAndReturnFalse;
+	}
+
+	/* No writes while PDump is not ready or is suspended */
+	if (PDumpReady() != PVRSRV_OK)
+	{
+		PDUMP_HERE(11);
+		goto unlockAndReturnFalse;
+	}
+
+	/* Prevent PDumping during a power transition */
+	if (PDumpCtrlInPowerTransition())
+	{	/* except when it's flagged */
+		if (ui32Flags & PDUMP_FLAGS_POWER)
+		{
+			PDUMP_HERE(20);
+			goto unlockAndReturnTrue;
+		}
+		else if (PDumpCtrlInPowerTransitionPID() == OSGetCurrentProcessID())
+		{
+			PDUMP_HERE(16);
+			goto unlockAndReturnFalse;
+		}
+	}
+
+	/* Always allow dumping in init phase and when persistent flagged */
+	if (ui32Flags & PDUMP_FLAGS_PERSISTENT)
+	{
+		PDUMP_HERE(12);
+		goto unlockAndReturnTrue;
+	}
+	if (!PDumpCtrlInitPhaseComplete())
+	{
+		PDUMP_HERE(15);
+		goto unlockAndReturnTrue;
+	}
+
+	/* The following checks are made when the driver has completed initialisation */
+	/* No last/deinit statements allowed when not in initialisation phase */
+	else /* init phase over */
+	{
+		if (ui32Flags & PDUMP_FLAGS_DEINIT)
+		{
+			PVR_ASSERT(0);
+			PDUMP_HERE(17);
+			goto unlockAndReturnFalse;
+		}
+	}
+
+	/* If PDump client connected allow continuous flagged writes */
+	if (PDUMP_IS_CONTINUOUS(ui32Flags))
+	{
+		if (PDumpCtrlGetModuleState() != PDUMP_SM_READY_CLIENT_CONNECTED) /* Is client connected? */
+		{
+			PDUMP_HERE(13);
+			goto unlockAndReturnFalse;
+		}
+		PDUMP_HERE(14);
+		goto unlockAndReturnTrue;
+	}
+
+	/* If in a capture interval but a write is still required.
+	 * Force write out if FLAGS_INTERVAL has been set and we are in
+	 * a capture interval */
+	if (ui32Flags & PDUMP_FLAGS_INTERVAL)
+	{
+		if (PDumpCtrlCaptureInInterval()){
+			PDUMP_HERE(21);
+			goto unlockAndReturnTrue;
+		}
+	}
+
+	/*
+		If no flags are provided then it is FRAMED output and the frame
+		range must be checked matching expected behaviour.
+	 */
+	if (!PDumpCtrlCaptureOn())
+	{
+		PDUMP_HERE(18);
+		goto unlockAndReturnFalse;
+	}
+
+	PDUMP_HERE(19);
+
+unlockAndReturnTrue:
+	/* Allow the write to take place */
+
+	PDumpCtrlLockRelease();
+	return IMG_TRUE;
+
+unlockAndReturnFalse:
+	PDumpCtrlLockRelease();
+returnFalse:
+	if (ui32ExitHere != NULL)
+	{
+		*ui32ExitHere = here;
+	}
+	return IMG_FALSE;
+}
+
+
+/*************************************************************************/ /*!
+ @Function		PDumpWriteToBuffer
+ @Description	Write the supplied data to the PDump stream buffer and attempt
+                to handle any buffer full conditions to ensure all the data
+                requested to be written, is.
+
+ @Input			psDeviceNode The device the PDump pertains to
+ @Input			psStream	The address of the PDump stream buffer to write to
+ @Input			pui8Data    Pointer to the data to be written
+ @Input			ui32BCount	Number of bytes to write
+ @Input			ui32Flags	PDump statement flags.
+
+ @Return		IMG_UINT32  Actual number of bytes written, may be less than
+                            ui32BCount when buffer full condition could not
+                            be avoided.
+*/ /**************************************************************************/
+static IMG_UINT32 PDumpWriteToBuffer(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     PDUMP_STREAM* psStream,
+                                     IMG_UINT8 *pui8Data,
+                                     IMG_UINT32 ui32BCount,
+                                     IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32	ui32BytesToBeWritten;
+	IMG_UINT32	ui32Off = 0;
+	IMG_BYTE *pbyDataBuffer;
+	IMG_UINT32 ui32BytesAvailable = 0;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiRetries = 0;
+
+	/* Check PDump stream validity */
+	if (psStream->hTL == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PDumpWriteToBuffer: PDump stream '%s' is invalid", psStream->pszName));
+		return 0;
+	}
+
+	/* This API always called by holding pdump write lock
+	 * to ensure atomic pdump write happened and
+	 * if not holding pdump lock then assert.
+	 */
+	PDumpAssertWriteLockHeld();
+
+	/* No need to check size of data to write as this is asserted
+	 * higher up in the call stack as 1KB and 16KB for each channel
+	 * respectively. */
+
+	while (ui32BCount > 0)
+	{
+		ui32BytesToBeWritten = MIN ( ui32BCount, psStream->ui32MaxAllowedWriteSize );
+
+		eError = TLStreamReserve2(psStream->hTL, &pbyDataBuffer, ui32BytesToBeWritten, 0, &ui32BytesAvailable, NULL);
+		if (eError == PVRSRV_ERROR_STREAM_FULL)
+		{
+			psStream->ui32BufferFullRetries++;
+
+			/*! Retry write2 only if available bytes is at least 1024 or more. */
+			if (ui32BytesAvailable >= 0x400)
+			{
+				ui32BytesToBeWritten = ui32BytesAvailable;
+				PVR_DPF((PVR_DBG_WARNING, "PDumpWriteToBuffer: TL buffer '%s' retrying write2=%u out of %u", psStream->pszName, ui32BytesToBeWritten, ui32BCount));
+				eError = TLStreamReserve(psStream->hTL, &pbyDataBuffer, ui32BytesToBeWritten);
+				/*! Not expected to get PVRSRV_ERROR_STREAM_FULL error and other error may get */
+				PVR_ASSERT(eError != PVRSRV_ERROR_STREAM_FULL);
+			}
+			else
+			{
+				uiRetries++;
+				PVR_DPF((PVR_DBG_WARNING, "PDumpWriteToBuffer: TL buffer '%s' full, rq=%u, av=%u, retrying write", psStream->pszName, ui32BCount, ui32BytesAvailable));
+
+				/* Check if we are out of retries , if so then print warning */
+				if (uiRetries >= MAX_PDUMP_WRITE_RETRIES)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+					         "PDumpWriteToBuffer: PDump writes blocked to dump %d bytes, %s TLBuffers full for %d seconds, check system",
+					         ui32BCount,
+					         psStream->pszName,
+					         ((200 * uiRetries)/1000)));
+
+					if (uiRetries > psStream->ui32HighestRetriesWatermark)
+					{
+						psStream->ui32HighestRetriesWatermark = uiRetries;
+					}
+
+					psStream->ui32BufferFullAborts++;
+					uiRetries = 0;
+
+					/* As uiRetries exceed max write retries that means,
+					 * something went wrong in system and thus suspend pdump.
+					 */
+					PDumpModuleTransitionState(PDUMP_SM_ERROR_SUSPENDED);
+					return 0;
+				}
+
+				OSSleepms(100);
+				continue;
+			}
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+			PVR_ASSERT(pbyDataBuffer != NULL);
+
+			OSDeviceMemCopy((void*)pbyDataBuffer, pui8Data + ui32Off, ui32BytesToBeWritten);
+
+			eError = TLStreamCommit(psStream->hTL, ui32BytesToBeWritten);
+			if (PVRSRV_OK != eError)
+			{
+				return 0;
+			}
+
+			if (uiRetries > psStream->ui32HighestRetriesWatermark)
+			{
+				psStream->ui32HighestRetriesWatermark = uiRetries;
+			}
+
+			uiRetries = 0;
+			ui32Off += ui32BytesToBeWritten;
+			ui32BCount -= ui32BytesToBeWritten;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToBuffer: TLStreamReserve2(%s) unrecoverable error %s", psStream->pszName, PVRSRVGETERRORSTRING(eError)));
+			/* Fatal -suspend PDump to prevent flooding kernel log buffer */
+			PVR_LOG(("Unrecoverable error, PDump suspended!"));
+
+			PDumpModuleTransitionState(PDUMP_SM_ERROR_SUSPENDED);
+			return 0;
+		}
+
+		/*
+		   if the capture range is unset
+		   (which is detected via PDumpWriteAllowed())
+		*/
+
+		if (!PDumpWriteAllowed(psDeviceNode, ui32Flags, NULL))
+		{
+			psStream->ui32BufferFullAborts++;
+			break;
+		}
+	}
+
+	return ui32Off;
+}
+
+/*************************************************************************/ /*!
+ @Function      PDumpWriteToChannel
+ @Description   Write the supplied data to the PDump channel specified obeying
+                flags to write to the necessary channel buffers.
+
+ @Input         psDeviceNode The device the PDump pertains to
+ @Input         psChannel   Address of the script or parameter channel object
+ @Input/Output  psWOff      Address of the channel write offsets object to
+                            update on successful writing
+ @Input         pui8Data    Pointer to the data to be written
+ @Input         ui32Size    Number of bytes to write
+ @Input         ui32Flags   PDump statement flags, they may be clear (no flags)
+                            or persistent flagged and they determine how the
+                            which implies framed data, continuous flagged, data
+                            is output. On the first test app run after driver
+                            load, the Display Controller dumps a resource that
+                            is persistent and this needs writing to both the
+                            init (persistent) and main (continuous) channel
+                            buffers to ensure the data is dumped in subsequent
+                            test runs without reloading the driver.
+                            In subsequent runs the PDump client 'freezes' the
+                            init buffer so that only one dump of persistent
+                            data for the "extended init phase" is captured to
+                            the init buffer.
+ @Return        IMG_BOOL    True when the data has been consumed, false otherwise
+*/ /**************************************************************************/
+static IMG_BOOL PDumpWriteToChannel(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    PDUMP_CHANNEL* psChannel,
+                                    PDUMP_CHANNEL_WOFFSETS* psWOff,
+                                    IMG_UINT8* pui8Data,
+                                    IMG_UINT32 ui32Size,
+                                    IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32 ui32BytesWritten = 0;
+	PDUMP_HERE_VAR;
+
+	PDUMP_HERE(210);
+
+	/* At this point, PDumpWriteAllowed() has returned TRUE (or called from
+	 * PDumpParameterChannelZeroedPageBlock() during driver init) we know the
+	 * write must proceed because:
+	 * - pdump is not suspended and
+	 * - there is not an ongoing power transition or POWER override flag is set or
+	 * - in driver init phase with ANY flag set or
+	 * - post init with the pdump client connected and
+	 * -   - PERSIST flag is present, xor
+	 * -   - the CONTINUOUS flag is present, xor
+	 * -   - in capture frame range
+	 */
+	PDumpAssertWriteLockHeld();
+
+	/* Dump data to deinit buffer when flagged as deinit */
+	if (ui32Flags & PDUMP_FLAGS_DEINIT)
+	{
+		PDUMP_HERE(211);
+		ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+		                                      &psChannel->sDeinitStream,
+		                                      pui8Data, ui32Size, ui32Flags);
+		if (ui32BytesWritten != ui32Size)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: DEINIT Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+			PDUMP_HERE(212);
+			return IMG_FALSE;
+		}
+
+		if (psWOff)
+		{
+			psWOff->ui32Deinit += ui32Size;
+		}
+
+	}
+	else
+	{
+		IMG_BOOL bDumpedToInitAlready = IMG_FALSE;
+		IMG_BOOL bMainStreamData = IMG_FALSE;
+		PDUMP_STREAM*  psStream = NULL;
+		IMG_UINT32* pui32Offset = NULL;
+
+		/* Always append persistent data to init phase so it's available on
+		 * subsequent app runs, but also to the main stream if client connected */
+		if (ui32Flags & PDUMP_FLAGS_PERSISTENT)
+		{
+			PDUMP_HERE(213);
+			ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+			                                      &psChannel->sInitStream,
+			                                      pui8Data, ui32Size, ui32Flags);
+			if (ui32BytesWritten != ui32Size)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: PERSIST Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+				PDUMP_HERE(214);
+				return IMG_FALSE;
+			}
+
+			bDumpedToInitAlready = IMG_TRUE;
+			if (psWOff)
+			{
+				psWOff->ui32Init += ui32Size;
+			}
+
+			/* Don't write continuous data if client not connected */
+			if (PDumpCtrlGetModuleState() != PDUMP_SM_READY_CLIENT_CONNECTED)
+			{
+				return IMG_TRUE;
+			}
+		}
+
+		/* Prepare to write the data to the main stream for
+		 * persistent, continuous or framed data. Override and use init
+		 * stream if driver still in init phase and we have not written
+		 * to it yet.*/
+		PDumpCtrlLockAcquire();
+		if (!PDumpCtrlInitPhaseComplete() && !bDumpedToInitAlready)
+		{
+			PDUMP_HERE(215);
+			psStream = &psChannel->sInitStream;
+			if (psWOff)
+			{
+				pui32Offset = &psWOff->ui32Init;
+			}
+		}
+		else
+		{
+			PDUMP_HERE(216);
+			psStream = &psChannel->sMainStream;
+			if (psWOff)
+			{
+				pui32Offset = &psWOff->ui32Main;
+			}
+			bMainStreamData = IMG_TRUE;
+
+		}
+		PDumpCtrlLockRelease();
+
+		if (PDumpCtrlCapModIsBlocked() && bMainStreamData && !psWOff)
+		{
+			/* if PDUMP_FLAGS_BLKDATA flag is set in Blocked mode, Make copy of Main script stream data to Block script stream as well */
+			if (ui32Flags & PDUMP_FLAGS_BLKDATA)
+			{
+				PDUMP_HERE(217);
+				ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+				                                      &psChannel->sBlockStream,
+				                                      pui8Data, ui32Size, ui32Flags);
+				if (ui32BytesWritten != ui32Size)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: BLOCK Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+					PDUMP_HERE(218);
+					return IMG_FALSE;
+				}
+			}
+		}
+
+		/* Write the data to the stream */
+		ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+		                                      psStream, pui8Data,
+		                                      ui32Size, ui32Flags);
+		if (ui32BytesWritten != ui32Size)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: MAIN Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+			PDUMP_HERE(219);
+			return IMG_FALSE;
+		}
+
+		if (pui32Offset)
+		{
+			*pui32Offset += ui32BytesWritten;
+		}
+	}
+
+	return IMG_TRUE;
+}
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+
+static IMG_UINT32 _GenerateChecksum(void *pvData, size_t uiSize)
+{
+	IMG_UINT32 ui32Sum = 0;
+	IMG_UINT32 *pui32Data = pvData;
+	IMG_UINT8 *pui8Data = pvData;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32LeftOver;
+
+	for (i = 0; i < uiSize / sizeof(IMG_UINT32); i++)
+	{
+		ui32Sum += pui32Data[i];
+	}
+
+	ui32LeftOver = uiSize % sizeof(IMG_UINT32);
+
+	while (ui32LeftOver)
+	{
+		ui32Sum += pui8Data[uiSize - ui32LeftOver];
+		ui32LeftOver--;
+	}
+
+	return ui32Sum;
+}
+
+#endif
+
+PVRSRV_ERROR PDumpWriteParameter(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT8 *pui8Data,
+                                 IMG_UINT32 ui32Size,
+                                 IMG_UINT32 ui32Flags,
+                                 IMG_UINT32* pui32FileOffset,
+                                 IMG_CHAR* aszFilenameStr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bPDumpCtrlInitPhaseComplete = IMG_FALSE;
+	IMG_UINT32 here = 0;
+	IMG_INT32 iCount;
+
+	PDumpAssertWriteLockHeld();
+
+	PVR_ASSERT(pui8Data && (ui32Size!=0));
+	PVR_ASSERT(pui32FileOffset && aszFilenameStr);
+
+	PDUMP_HERE(1);
+
+	/* Check if write can proceed? */
+	if (!PDumpWriteAllowed(psDeviceNode, ui32Flags, &here))
+	{
+		/* Abort write for the above reason but indicate what happened to
+		 * caller to avoid disrupting the driver, caller should treat it as OK
+		 * but skip any related PDump writes to the script file. */
+		return PVRSRV_ERROR_PDUMP_NOT_ALLOWED;
+	}
+
+	PDUMP_HERE(2);
+
+	PDumpCtrlLockAcquire();
+	bPDumpCtrlInitPhaseComplete = PDumpCtrlInitPhaseComplete();
+	PDumpCtrlLockRelease();
+
+	if (!bPDumpCtrlInitPhaseComplete || (ui32Flags & PDUMP_FLAGS_PERSISTENT))
+	{
+		PDUMP_HERE(3);
+
+		/* Init phase stream not expected to get above the file size max */
+		PVR_ASSERT(g_PDumpParameters.sWOff.ui32Init < g_PDumpParameters.ui32MaxFileSize);
+
+		/* Return the file write offset at which the parameter data was dumped */
+		*pui32FileOffset = g_PDumpParameters.sWOff.ui32Init;
+	}
+	else
+	{
+		PDUMP_HERE(4);
+
+		/* Do we need to signal the PDump client that a split is required? */
+		if (g_PDumpParameters.sWOff.ui32Main + ui32Size > g_PDumpParameters.ui32MaxFileSize)
+		{
+			PDUMP_HERE(5);
+			_PDumpSetSplitMarker(g_PDumpParameters.sCh.sMainStream.hTL, IMG_FALSE);
+			g_PDumpParameters.ui32FileIdx++;
+			g_PDumpParameters.sWOff.ui32Main = 0;
+		}
+
+		/* Return the file write offset at which the parameter data was dumped */
+		*pui32FileOffset = g_PDumpParameters.sWOff.ui32Main;
+	}
+
+	/* Create the parameter file name, based on index, to be used in the script */
+	if (g_PDumpParameters.ui32FileIdx == 0)
+	{
+		iCount = OSSNPrintf(aszFilenameStr, PDUMP_PARAM_MAX_FILE_NAME, PDUMP_PARAM_0_FILE_NAME);
+	}
+	else
+	{
+		PDUMP_HERE(6);
+		iCount = OSSNPrintf(aszFilenameStr, PDUMP_PARAM_MAX_FILE_NAME, PDUMP_PARAM_N_FILE_NAME, g_PDumpParameters.ui32FileIdx);
+	}
+
+	PVR_LOG_GOTO_IF_FALSE(((iCount != -1) && (iCount < PDUMP_PARAM_MAX_FILE_NAME)), "OSSNPrintf", errExit);
+
+	/* Write the parameter data to the parameter channel */
+	eError = PVRSRV_ERROR_PDUMP_BUFFER_FULL;
+	if (!PDumpWriteToChannel(psDeviceNode, &g_PDumpParameters.sCh,
+	                         &g_PDumpParameters.sWOff, pui8Data,
+	                         ui32Size, ui32Flags))
+	{
+		PDUMP_HERE(7);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpWrite", errExit);
+	}
+#if defined(PDUMP_DEBUG_OUTFILES)
+	else
+	{
+		IMG_UINT32 ui32Checksum;
+		PDUMP_GET_SCRIPT_STRING();
+
+		ui32Checksum = _GenerateChecksum(pui8Data, ui32Size);
+
+		/* CHK CHKSUM SIZE PRMOFFSET PRMFILE */
+		eError = PDumpSNPrintf(hScript, ui32MaxLen, "-- CHK 0x%08X 0x%08X 0x%08X %s",
+									ui32Checksum,
+									ui32Size,
+									*pui32FileOffset,
+									aszFilenameStr);
+		PVR_GOTO_IF_ERROR(eError, errExit);
+
+		PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		PDUMP_RELEASE_SCRIPT_STRING();
+	}
+#endif
+
+	return PVRSRV_OK;
+
+errExit:
+	return eError;
+}
+
+
+IMG_BOOL PDumpWriteScript(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          IMG_HANDLE hString, IMG_UINT32 ui32Flags)
+{
+	PDUMP_HERE_VAR;
+
+	PVR_ASSERT(hString);
+
+	PDumpAssertWriteLockHeld();
+
+	PDUMP_HERE(201);
+
+#if defined(DEBUG)
+	/* Since buffer sizes and buffer writing/reading are a balancing act to
+	 * avoid buffer full errors, check here our assumption on the maximum write size.
+	 */
+	{
+		IMG_UINT32 ui32Size = (IMG_UINT32) OSStringLength((const IMG_CHAR *)hString);
+		if (ui32Size > 0x400) // 1KB
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDUMP large script write %u bytes", ui32Size));
+			OSDumpStack();
+		}
+	}
+#endif
+
+	if (!PDumpWriteAllowed(psDeviceNode, ui32Flags, NULL))
+	{
+		/* Abort write for the above reasons but indicated it was OK to
+		 * caller to avoid disrupting the driver */
+		return IMG_TRUE;
+	}
+
+	if (PDumpCtrlCapModIsBlocked())
+	{
+		if (ui32Flags & PDUMP_FLAGS_FORCESPLIT)
+		{
+			IMG_UINT32 ui32CurrentBlock;
+
+			PDumpGetCurrentBlockKM(&ui32CurrentBlock);
+			/* Keep Main stream script output files belongs to first and last block only */
+			if (ui32CurrentBlock == 1)
+			{
+				/* To keep first(0th) block, do not remove old script file while
+				 * splitting to second(1st) block (i.e. bRemoveOld=IMG_FALSE).
+				 * */
+				_PDumpSetSplitMarker(g_PDumpScript.sCh.sMainStream.hTL, IMG_FALSE);
+			}
+			else
+			{
+				/* Previous block's Main script output file will be removed
+				 * before splitting to next
+				 * */
+				_PDumpSetSplitMarker(g_PDumpScript.sCh.sMainStream.hTL, IMG_TRUE);
+			}
+
+			/* Split Block stream output file
+			 *
+			 * We are keeping block script output files from all PDump blocks.
+			 * */
+			_PDumpSetSplitMarker(g_PDumpScript.sCh.sBlockStream.hTL, IMG_FALSE);
+			g_PDumpScript.ui32FileIdx++;
+		}
+	}
+
+	return PDumpWriteToChannel(psDeviceNode, &g_PDumpScript.sCh, NULL,
+	                          (IMG_UINT8*) hString,
+	                          (IMG_UINT32) OSStringLength((IMG_CHAR*) hString),
+	                          ui32Flags);
+}
+
+
+/*****************************************************************************/
+
+
+struct _PDUMP_CONNECTION_DATA_ {
+	ATOMIC_T                  sRefCount;
+	POS_LOCK                  hLock;                       /*!< Protects access to sListHead. */
+	DLLIST_NODE               sListHead;
+	IMG_UINT32                ui32LastSetFrameNumber;
+	PDUMP_TRANSITION_EVENT    eLastEvent;                  /*!< Last processed transition event */
+	PDUMP_TRANSITION_EVENT    eFailedEvent;                /*!< Failed transition event to retry */
+	PFN_PDUMP_SYNCBLOCKS      pfnPDumpSyncBlocks;          /*!< Callback to PDump sync blocks */
+	void                      *hSyncPrivData;              /*!< Sync private data */
+};
+
+static PDUMP_CONNECTION_DATA * _PDumpConnectionAcquire(PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psPDumpConnectionData->sRefCount);
+
+	PDUMP_REFCOUNT_PRINT("%s: PDump connection %p, refcount = %d", __func__,
+	                     psPDumpConnectionData, iRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+
+	return psPDumpConnectionData;
+}
+
+static void _PDumpConnectionRelease(PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psPDumpConnectionData->sRefCount);
+	if (iRefCount == 0)
+	{
+		OSLockDestroy(psPDumpConnectionData->hLock);
+		PVR_ASSERT(dllist_is_empty(&psPDumpConnectionData->sListHead));
+		OSFreeMem(psPDumpConnectionData);
+	}
+
+	PDUMP_REFCOUNT_PRINT("%s: PDump connection %p, refcount = %d", __func__,
+	                     psPDumpConnectionData, iRefCount);
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInitStreams
+ * Outputs        : None
+ * Returns        :
+ * Description    : Create the PDump streams
+******************************************************************************/
+static PVRSRV_ERROR PDumpInitStreams(PDUMP_CHANNEL* psParam, PDUMP_CHANNEL* psScript)
+{
+
+	PVRSRV_ERROR   eError;
+	TL_STREAM_INFO sTLStreamInfo;
+
+	/* TL - Create the streams */
+
+	/**************************** Parameter stream ***************************/
+
+	/* Parameter - Init */
+	eError = TLStreamCreate(&psParam->sInitStream.hTL,
+				psParam->sInitStream.pszName, psParam->sInitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ParamInit", end);
+
+	TLStreamInfo(psParam->sInitStream.hTL, &sTLStreamInfo);
+	psParam->sInitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Parameter - Main */
+	eError = TLStreamCreate(&psParam->sMainStream.hTL,
+				psParam->sMainStream.pszName, psParam->sMainStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER ,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ParamMain", param_main_failed);
+
+	TLStreamInfo(psParam->sMainStream.hTL, &sTLStreamInfo);
+	psParam->sMainStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Parameter - Deinit */
+	eError = TLStreamCreate(&psParam->sDeinitStream.hTL,
+				psParam->sDeinitStream.pszName,	psParam->sDeinitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ParamDeinit", param_deinit_failed);
+
+	TLStreamInfo(psParam->sDeinitStream.hTL, &sTLStreamInfo);
+	psParam->sDeinitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Parameter - Block */
+	/* As in current implementation Block script stream is just a filtered
+	 * Main script stream using PDUMP_FLAGS_BLKDATA flag, no separate
+	 * Parameter stream is needed. Block script will be referring to the
+	 * same Parameters as that of Main script stream.
+	 */
+
+	/***************************** Script streams ****************************/
+
+	/* Script - Init */
+	eError = TLStreamCreate(&psScript->sInitStream.hTL,
+				psScript->sInitStream.pszName, psScript->sInitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptInit", script_init_failed);
+
+	TLStreamInfo(psScript->sInitStream.hTL, &sTLStreamInfo);
+	psScript->sInitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Script - Main */
+	eError = TLStreamCreate(&psScript->sMainStream.hTL,
+				psScript->sMainStream.pszName, psScript->sMainStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptMain", script_main_failed);
+
+	TLStreamInfo(psScript->sMainStream.hTL, &sTLStreamInfo);
+	psScript->sMainStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Script - Deinit */
+	eError = TLStreamCreate(&psScript->sDeinitStream.hTL,
+				psScript->sDeinitStream.pszName, psScript->sDeinitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptDeinit", script_deinit_failed);
+
+	TLStreamInfo(psScript->sDeinitStream.hTL, &sTLStreamInfo);
+	psScript->sDeinitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Script - Block */
+	eError = TLStreamCreate(&psScript->sBlockStream.hTL,
+				psScript->sBlockStream.pszName, psScript->sBlockStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptBlock", script_block_failed);
+
+	TLStreamInfo(psScript->sBlockStream.hTL, &sTLStreamInfo);
+	psScript->sBlockStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	return PVRSRV_OK;
+
+script_block_failed:
+	TLStreamClose(psScript->sDeinitStream.hTL);
+
+script_deinit_failed:
+	TLStreamClose(psScript->sMainStream.hTL);
+
+script_main_failed:
+	TLStreamClose(psScript->sInitStream.hTL);
+
+script_init_failed:
+	TLStreamClose(psParam->sDeinitStream.hTL);
+
+param_deinit_failed:
+	TLStreamClose(psParam->sMainStream.hTL);
+
+param_main_failed:
+	TLStreamClose(psParam->sInitStream.hTL);
+
+end:
+	return eError;
+}
+/******************************************************************************
+ * Function Name  : PDumpDeInitStreams
+ * Inputs         : psParam, psScript
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Deinitialises the PDump streams
+******************************************************************************/
+static void PDumpDeInitStreams(PDUMP_CHANNEL* psParam, PDUMP_CHANNEL* psScript)
+{
+	/* Script streams */
+	TLStreamClose(psScript->sDeinitStream.hTL);
+	TLStreamClose(psScript->sMainStream.hTL);
+	TLStreamClose(psScript->sInitStream.hTL);
+	TLStreamClose(psScript->sBlockStream.hTL);
+
+	/* Parameter streams */
+	TLStreamClose(psParam->sDeinitStream.hTL);
+	TLStreamClose(psParam->sMainStream.hTL);
+	TLStreamClose(psParam->sInitStream.hTL);
+
+}
+
+/******************************************************************************
+ * Function Name  : PDumpParameterChannelZeroedPageBlock
+ * Inputs         : psDeviceNode
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Set up the zero page block in the parameter stream
+******************************************************************************/
+static PVRSRV_ERROR PDumpParameterChannelZeroedPageBlock(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT8 aui8Zero[32] = { 0 };
+	size_t uiBytesToWrite;
+	PVRSRV_ERROR eError;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+	IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, GeneralNon4KHeapPageSize,
+			&ui32AppHintDefault, &ui32GeneralNon4KHeapPageSize);
+	OSFreeAppHintState(pvAppHintState);
+
+	/* ZeroPageSize can't be smaller than page size */
+	g_PDumpParameters.uiZeroPageSize = MAX(ui32GeneralNon4KHeapPageSize, OSGetPageSize());
+
+	/* ensure the zero page size of a multiple of the zero source on the stack */
+	PVR_ASSERT(g_PDumpParameters.uiZeroPageSize % sizeof(aui8Zero) == 0);
+
+	/* the first write gets the parameter file name and stream offset,
+	 * then subsequent writes do not need to know this as the data is
+	 * contiguous in the stream
+	 */
+	PDUMP_LOCK(0);
+	eError = PDumpWriteParameter(psDeviceNode, aui8Zero,
+							sizeof(aui8Zero),
+							0,
+							&g_PDumpParameters.uiZeroPageOffset,
+							g_PDumpParameters.szZeroPageFilename);
+
+	/* Also treat PVRSRV_ERROR_PDUMP_NOT_ALLOWED as an error in this case
+	 * as it should never happen since all writes during driver Init are
+	 * allowed.
+	*/
+	PVR_GOTO_IF_ERROR(eError, err_write);
+
+	uiBytesToWrite = g_PDumpParameters.uiZeroPageSize - sizeof(aui8Zero);
+
+	while (uiBytesToWrite)
+	{
+		IMG_BOOL bOK;
+
+		bOK = PDumpWriteToChannel(psDeviceNode,
+								  &g_PDumpParameters.sCh,
+								  &g_PDumpParameters.sWOff,
+								  aui8Zero,
+								  sizeof(aui8Zero), 0);
+
+		if (!bOK)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PDUMP_BUFFER_FULL, err_write);
+		}
+
+		uiBytesToWrite -= sizeof(aui8Zero);
+	}
+
+err_write:
+	PDUMP_UNLOCK(0);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to initialise parameter stream zero block"));
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpGetParameterZeroPageInfo
+ * Inputs         : None
+ * Outputs        : puiZeroPageOffset: set to the offset of the zero page
+ *                : puiZeroPageSize: set to the size of the zero page
+ *                : ppszZeroPageFilename: set to a pointer to the PRM file name
+ *                :                       containing the zero page
+ * Returns        : None
+ * Description    : Get information about the zero page
+******************************************************************************/
+void PDumpGetParameterZeroPageInfo(PDUMP_FILEOFFSET_T *puiZeroPageOffset,
+					size_t *puiZeroPageSize,
+					const IMG_CHAR **ppszZeroPageFilename)
+{
+		*puiZeroPageOffset = g_PDumpParameters.uiZeroPageOffset;
+		*puiZeroPageSize = g_PDumpParameters.uiZeroPageSize;
+		*ppszZeroPageFilename = g_PDumpParameters.szZeroPageFilename;
+}
+
+
+PVRSRV_ERROR PDumpInitCommon(void)
+{
+	PVRSRV_ERROR eError;
+	PDUMP_HERE_VAR;
+
+	PDUMP_HEREA(2010);
+
+	/* Initialised with default initial value */
+	OSAtomicWrite(&g_sConnectionCount, 0);
+#if defined(PDUMP_DEBUG_OUTFILES)
+	OSAtomicWrite(&g_sEveryLineCounter, 1);
+#endif
+
+	eError = OSLockCreate(&g_hPDumpWriteLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", errRet);
+
+	/* Initialise PDump control module in common layer, also sets
+	 * state to PDUMP_SM_INITIALISING.
+	 */
+	eError = PDumpCtrlInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCtrlInit", errRetLock);
+
+	/* Call environment specific PDump initialisation Part 2*/
+	eError = PDumpInitStreams(&g_PDumpParameters.sCh, &g_PDumpScript.sCh);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpInitStreams", errRetCtrl);
+
+	/* PDump now ready for write calls */
+	PDumpModuleTransitionState(PDUMP_SM_READY);
+
+	PDUMP_HEREA(2011);
+
+	/* Test PDump initialised and ready by logging driver details */
+	eError = PDumpCommentWithFlags((PVRSRV_DEVICE_NODE*)PDUMP_MAGIC_COOKIE,
+	                               PDUMP_FLAGS_CONTINUOUS,
+	                               "Driver Product Version: %s - %s (%s)",
+	                               PVRVERSION_STRING, PVR_BUILD_DIR, PVR_BUILD_TYPE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCommentWithFlags", errRetState);
+
+	eError = PDumpCommentWithFlags((PVRSRV_DEVICE_NODE*)PDUMP_MAGIC_COOKIE,
+	                               PDUMP_FLAGS_CONTINUOUS,
+	                               "Start of Init Phase");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCommentWithFlags", errRetState);
+
+	eError = PDumpParameterChannelZeroedPageBlock((PVRSRV_DEVICE_NODE*)PDUMP_MAGIC_COOKIE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpParameterChannelZeroedPageBlock", errRetState);
+
+	PDUMP_HEREA(2012);
+ret:
+	return eError;
+
+
+errRetState:
+	PDumpModuleTransitionState(PDUMP_SM_UNINITIALISED);
+	PDumpDeInitStreams(&g_PDumpParameters.sCh, &g_PDumpScript.sCh);
+errRetCtrl:
+	PDumpCtrlDeInit();
+errRetLock:
+	OSLockDestroy(g_hPDumpWriteLock);
+	PDUMP_HEREA(2013);
+errRet:
+	goto ret;
+}
+void PDumpDeInitCommon(void)
+{
+	PDUMP_HERE_VAR;
+
+	PDUMP_HEREA(2020);
+
+	/* Suspend PDump as we want PDumpWriteAllowed to deliberately fail during PDump deinit */
+	PDumpModuleTransitionState(PDUMP_SM_DEINITIALISED);
+
+	/*Call environment specific PDump Deinitialisation */
+	PDumpDeInitStreams(&g_PDumpParameters.sCh, &g_PDumpScript.sCh);
+
+	/* DeInit the PDUMP_CTRL_STATE data */
+	PDumpCtrlDeInit();
+
+	/* take down the global PDump lock */
+	OSLockDestroy(g_hPDumpWriteLock);
+}
+
+void PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32PDumpBoundDevice = PVRSRVGetPVRSRVData()->ui32PDumpBoundDevice;
+
+	/* Stop the init phase for the PDump-bound device only */
+	if (psDeviceNode->sDevId.ui32InternalID == ui32PDumpBoundDevice)
+	{
+		/* output this comment to indicate init phase ending OSs */
+		PDUMPCOMMENT(psDeviceNode, "Stop Init Phase");
+
+		PDumpCtrlLockAcquire();
+		PDumpCtrlSetInitPhaseComplete(IMG_TRUE);
+		PDumpCtrlLockRelease();
+	}
+}
+
+PVRSRV_ERROR PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame)
+{
+	PDumpCtrlLockAcquire();
+	*pbIsLastCaptureFrame = PDumpCtrlIsLastCaptureFrame();
+	PDumpCtrlLockRelease();
+
+	return PVRSRV_OK;
+}
+
+
+
+typedef struct _PDUMP_Transition_DATA_
+{
+	PFN_PDUMP_TRANSITION        pfnCallback;
+	void                        *hPrivData;
+	void                        *pvDevice;
+	PDUMP_CONNECTION_DATA       *psPDumpConnectionData;
+	DLLIST_NODE                 sNode;
+} PDUMP_Transition_DATA;
+
+PVRSRV_ERROR PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+											  PFN_PDUMP_TRANSITION pfnCallback,
+											  void *hPrivData,
+											  void *pvDevice,
+											  void **ppvHandle)
+{
+	PDUMP_Transition_DATA *psData;
+	PVRSRV_ERROR eError;
+
+	psData = OSAllocMem(sizeof(*psData));
+	PVR_GOTO_IF_NOMEM(psData, eError, fail_alloc);
+
+	/* Setup the callback and add it to the list for this process */
+	psData->pfnCallback = pfnCallback;
+	psData->hPrivData = hPrivData;
+	psData->pvDevice = pvDevice;
+
+	OSLockAcquire(psPDumpConnectionData->hLock);
+	dllist_add_to_head(&psPDumpConnectionData->sListHead, &psData->sNode);
+	OSLockRelease(psPDumpConnectionData->hLock);
+
+	/* Take a reference on the connection so it doesn't get freed too early */
+	psData->psPDumpConnectionData =_PDumpConnectionAcquire(psPDumpConnectionData);
+	*ppvHandle = psData;
+
+	return PVRSRV_OK;
+
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void PDumpUnregisterTransitionCallback(void *pvHandle)
+{
+	PDUMP_Transition_DATA *psData = pvHandle;
+
+	OSLockAcquire(psData->psPDumpConnectionData->hLock);
+	dllist_remove_node(&psData->sNode);
+	OSLockRelease(psData->psPDumpConnectionData->hLock);
+	_PDumpConnectionRelease(psData->psPDumpConnectionData);
+	OSFreeMem(psData);
+}
+
+typedef struct _PDUMP_Transition_DATA_FENCE_SYNC_
+{
+	PFN_PDUMP_TRANSITION_FENCE_SYNC         pfnCallback;
+	void                                    *hPrivData;
+} PDUMP_Transition_DATA_FENCE_SYNC;
+
+PVRSRV_ERROR PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+							  PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback, void **ppvHandle)
+{
+	PDUMP_Transition_DATA_FENCE_SYNC *psData;
+	PVRSRV_ERROR eError;
+
+	psData = OSAllocMem(sizeof(*psData));
+	PVR_GOTO_IF_NOMEM(psData, eError, fail_alloc_exit);
+
+	/* Setup the callback and add it to the list for this process */
+	psData->pfnCallback = pfnCallback;
+	psData->hPrivData = hPrivData;
+
+	*ppvHandle = psData;
+	return PVRSRV_OK;
+
+fail_alloc_exit:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle)
+{
+	PDUMP_Transition_DATA_FENCE_SYNC *psData = pvHandle;
+
+	OSFreeMem(psData);
+}
+
+static PVRSRV_ERROR _PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                 PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+	                                 PDUMP_TRANSITION_EVENT eEvent,
+	                                 IMG_UINT32 ui32PDumpFlags)
+{
+	DLLIST_NODE *psNode, *psNext;
+	PVRSRV_ERROR eError;
+
+	/* Only call the callbacks if we've really got new event */
+	if ((eEvent != psPDumpConnectionData->eLastEvent) && (eEvent != PDUMP_TRANSITION_EVENT_NONE))
+	{
+		OSLockAcquire(psPDumpConnectionData->hLock);
+
+		dllist_foreach_node(&psPDumpConnectionData->sListHead, psNode, psNext)
+		{
+			PDUMP_Transition_DATA *psData =
+				IMG_CONTAINER_OF(psNode, PDUMP_Transition_DATA, sNode);
+
+			eError = psData->pfnCallback(psData->hPrivData, psData->pvDevice, eEvent, ui32PDumpFlags);
+
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease(psPDumpConnectionData->hLock);
+				psPDumpConnectionData->eFailedEvent = eEvent; /* Save failed event to retry */
+				return eError;
+			}
+		}
+		OSLockRelease(psPDumpConnectionData->hLock);
+
+		/* PDump sync blocks:
+		 *
+		 * Client sync prims are managed in blocks.
+		 *
+		 * sync-blocks gets re-dumped each time we enter into capture range or
+		 * enter into new PDump block. Ensure that live-FW thread and app-thread
+		 * are synchronised before this.
+		 *
+		 * At playback time, script-thread and sim-FW threads needs to be
+		 * synchronised before re-loading sync-blocks.
+		 * */
+		psPDumpConnectionData->pfnPDumpSyncBlocks(psDeviceNode, psPDumpConnectionData->hSyncPrivData, eEvent);
+
+		if (psDeviceNode->hTransition)
+		{
+			PDUMP_Transition_DATA_FENCE_SYNC *psData = (PDUMP_Transition_DATA_FENCE_SYNC*)psDeviceNode->hTransition;
+			psData->pfnCallback(psData->hPrivData, eEvent);
+		}
+
+		psPDumpConnectionData->eLastEvent = eEvent;
+		psPDumpConnectionData->eFailedEvent = PDUMP_TRANSITION_EVENT_NONE; /* Clear failed event on success */
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _PDumpBlockTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                          PDUMP_TRANSITION_EVENT eEvent,
+                                          IMG_UINT32 ui32PDumpFlags)
+{
+
+	/* Need to follow following sequence for Block transition:
+	 *
+	 * (1) _PDumpTransition with BLOCK_FINISHED event for current block
+	 * (2) Split MAIN and Block script files
+	 * (3) _PDumpTransition with BLOCK_STARTED event for new block
+	 *
+	 * */
+
+	PVRSRV_ERROR        eError;
+	IMG_UINT32          ui32CurrentBlock;
+	IMG_UINT32          ui32Flags = (PDUMP_FLAGS_BLKDATA | PDUMP_FLAGS_CONTINUOUS); /* Internal Block mode specific PDump flags */
+
+	PDumpGetCurrentBlockKM(&ui32CurrentBlock);
+
+	if (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_FINISHED)
+	{
+		/* (1) Current block has finished */
+		eError = _PDumpTransition(psDeviceNode,
+			                      psPDumpConnectionData,
+			                      PDUMP_TRANSITION_EVENT_BLOCK_FINISHED,
+			                      ui32PDumpFlags);
+		PVR_RETURN_IF_ERROR(eError);
+
+		(void) PDumpCommentWithFlags(psDeviceNode, ui32Flags,
+		                             "}PDUMP_BLOCK_END_0x%08X",
+		                             ui32CurrentBlock - 1); /* Add pdump-block end marker */
+
+		/* (2) Split MAIN and BLOCK script out files on current pdump-block end */
+		ui32Flags |= PDUMP_FLAGS_FORCESPLIT;
+
+		(void) PDumpCommentWithFlags(psDeviceNode, ui32Flags,
+		                             "PDUMP_BLOCK_START_0x%08X{",
+		                             ui32CurrentBlock); /* Add pdump-block start marker */
+	}
+
+	/* (3) New block has started */
+	return _PDumpTransition(psDeviceNode,
+		                    psPDumpConnectionData,
+		                    PDUMP_TRANSITION_EVENT_BLOCK_STARTED,
+		                    ui32PDumpFlags);
+}
+
+
+PVRSRV_ERROR PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                             PDUMP_TRANSITION_EVENT eEvent,
+                             IMG_UINT32 ui32PDumpFlags)
+{
+	if ((eEvent == PDUMP_TRANSITION_EVENT_BLOCK_FINISHED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		/* Block mode transition events */
+		PVR_ASSERT(PDumpCtrlCapModIsBlocked());
+		return _PDumpBlockTransition(psDeviceNode, psPDumpConnectionData, eEvent, ui32PDumpFlags);
+	}
+	else
+	{
+		/* Non-block mode transition events */
+		return _PDumpTransition(psDeviceNode, psPDumpConnectionData, eEvent, ui32PDumpFlags);
+	}
+}
+
+static PVRSRV_ERROR PDumpIsCaptureFrame(IMG_BOOL *bInCaptureRange)
+{
+	IMG_UINT64 ui64State = 0;
+	PVRSRV_ERROR eError;
+
+	eError = PDumpCtrlGetState(&ui64State);
+
+	*bInCaptureRange = (ui64State & PDUMP_STATE_CAPTURE_FRAME) ? IMG_TRUE : IMG_FALSE;
+
+	return eError;
+}
+
+PVRSRV_ERROR PDumpGetStateKM(IMG_UINT64 *ui64State)
+{
+	PVRSRV_ERROR eError;
+
+	PDumpCtrlLockAcquire();
+	eError = PDumpCtrlGetState(ui64State);
+	PDumpCtrlLockRelease();
+
+	return eError;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpUpdateBlockCtrlStatus
+ * Inputs         : ui32Frame - frame number
+ * Outputs        : None
+ * Returns        : IMG_TRUE if Block transition is required, else IMG_FALSE
+ * Description    : Updates Block Ctrl status and checks if block transition
+ *                  is required or not
+******************************************************************************/
+static INLINE IMG_BOOL PDumpUpdateBlockCtrlStatus(IMG_UINT32 ui32Frame)
+{
+	IMG_BOOL bForceBlockTransition;
+
+	/* Default length of first block will be PDUMP_BLOCKLEN_MIN.
+	 * User can force it to be same as block length provided (i.e. ui32BlockLength)
+	 * through pdump client.
+	 *
+	 * Here is how blocks will be created.
+	 *
+	 * Assume,
+	 * ui32BlockLength = 20
+	 * PDUMP_BLOCKLEN_MIN = 10
+	 *
+	 * Then different pdump blocks will have following number of frames in it:
+	 *
+	 * if(!PDumpCtrlMinimalFirstBlock())
+	 * {
+	 *		//pdump -b<block len>
+	 *		block 0 -> 00...09        -->minimal first block
+	 *		block 1 -> 10...29
+	 *		block 2 -> 30...49
+	 *		block 3 -> 50...69
+	 *		...
+	 * }
+	 * else
+	 * {
+	 *		//pdump -bf<block len>
+	 *		block 0 -> 00...19
+	 *		block 1 -> 20...39
+	 *		block 2 -> 40...59
+	 *		block 3 -> 60...79
+	 *		...
+	 * }
+	 *
+	 * */
+
+	if (PDumpCtrlMinimalFirstBlock())
+	{
+		bForceBlockTransition = ((ui32Frame >= PDUMP_BLOCKLEN_MIN) && !((ui32Frame - PDUMP_BLOCKLEN_MIN) % g_PDumpCtrl.sBlockCtrl.ui32BlockLength)) || (ui32Frame == 0);
+	}
+	else
+	{
+		bForceBlockTransition = !(ui32Frame % g_PDumpCtrl.sBlockCtrl.ui32BlockLength);
+	}
+
+	if (bForceBlockTransition) /* Entering in new pdump-block */
+	{
+		/* Update block number
+		 *
+		 * Logic below is to maintain block number and frame number mappings
+		 * in case of some applications where setFrame(0) gets called twice
+		 * at the start.
+		 * */
+		PDumpCtrlLockAcquire();
+		PDumpCtrlSetBlock((ui32Frame == 0)? 0 : (PDumpCtrlGetBlock() + 1));
+		PDumpCtrlLockRelease();
+
+		if (ui32Frame > 0) /* Do not do transition on first frame itself */
+		{
+			return IMG_TRUE; /* Transition */
+		}
+	}
+	return IMG_FALSE; /* No transition */
+}
+
+PVRSRV_ERROR PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* If call is not for the pdump-bound device, return immediately
+	 * taking no action.
+	 */
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	if (!PDumpCtrlCapModIsBlocked())
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: This call is valid only in Block mode of PDump i.e. pdump -b<block-len>", __func__));
+		return PVRSRV_ERROR_PDUMP_NOT_ALLOWED;
+	}
+
+	(void) PDumpCommentWithFlags(psDeviceNode,
+	                             PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA,
+	                             "PDdump forced STOP capture request received at frame %u",
+	                             g_PDumpCtrl.ui32CurrentFrame);
+
+	PDumpCtrlLockAcquire();
+	eError = PDumpCtrlForcedStop();
+	PDumpCtrlLockRelease();
+
+	return eError;
+}
+
+static PVRSRV_ERROR _PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Frame)
+{
+	PDUMP_CONNECTION_DATA *psPDumpConnectionData = psConnection->psPDumpConnectionData;
+	PDUMP_TRANSITION_EVENT eTransitionEvent = PDUMP_TRANSITION_EVENT_NONE;
+	IMG_BOOL bWasInCaptureRange = IMG_FALSE;
+	IMG_BOOL bIsInCaptureRange = IMG_FALSE;
+	PVRSRV_ERROR eError;
+
+	/*
+		Note:
+		As we can't test to see if the new frame will be in capture range
+		before we set the frame number and we don't want to roll back the
+		frame number if we fail then we have to save the "transient" data
+		which decides if we're entering or exiting capture range along
+		with a failure boolean so we know what's required on a retry
+	*/
+	if (psPDumpConnectionData->ui32LastSetFrameNumber != ui32Frame)
+	{
+		(void) PDumpCommentWithFlags(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                             "Set pdump frame %u", ui32Frame);
+
+		/*
+			The boolean values below decide if the PDump transition
+			should trigger because of the current context setting the
+			frame number, hence the functions below should execute
+			atomically and do not give a chance to some other context
+			to transition
+		*/
+		PDumpCtrlLockAcquire();
+
+		PDumpIsCaptureFrame(&bWasInCaptureRange);
+		PDumpCtrlSetCurrentFrame(ui32Frame);
+		PDumpIsCaptureFrame(&bIsInCaptureRange);
+
+		PDumpCtrlLockRelease();
+
+		psPDumpConnectionData->ui32LastSetFrameNumber = ui32Frame;
+
+		/* Check for any transition event only if client is connected */
+		if (PDumpIsClientConnected())
+		{
+			if (!bWasInCaptureRange && bIsInCaptureRange)
+			{
+				eTransitionEvent = PDUMP_TRANSITION_EVENT_RANGE_ENTERED;
+			}
+			else if (bWasInCaptureRange && !bIsInCaptureRange)
+			{
+				eTransitionEvent = PDUMP_TRANSITION_EVENT_RANGE_EXITED;
+			}
+
+			if (PDumpCtrlCapModIsBlocked())
+			{
+				/* Update block ctrl status and check for block transition */
+				if (PDumpUpdateBlockCtrlStatus(ui32Frame))
+				{
+					PVR_ASSERT(eTransitionEvent == PDUMP_TRANSITION_EVENT_NONE); /* Something went wrong, can't handle two events at same time */
+					eTransitionEvent = PDUMP_TRANSITION_EVENT_BLOCK_FINISHED;
+				}
+			}
+		}
+	}
+	else if (psPDumpConnectionData->eFailedEvent != PDUMP_TRANSITION_EVENT_NONE)
+	{
+		/* Load the Transition data so we can try again */
+		eTransitionEvent = psPDumpConnectionData->eFailedEvent;
+	}
+	else
+	{
+		/* New frame is the same as the last frame set and the last
+		 * transition succeeded, no need to perform another transition.
+		 */
+		return PVRSRV_OK;
+	}
+
+	if (eTransitionEvent != PDUMP_TRANSITION_EVENT_NONE)
+	{
+		DEBUG_OUTFILES_COMMENT(psDeviceNode, "PDump transition event(%u)-begin frame %u (post)", eTransitionEvent, ui32Frame);
+		eError = PDumpTransition(psDeviceNode, psPDumpConnectionData, eTransitionEvent, PDUMP_FLAGS_NONE);
+		DEBUG_OUTFILES_COMMENT(psDeviceNode, "PDump transition event(%u)-complete frame %u (post)", eTransitionEvent, ui32Frame);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32Frame)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* If call is not for the pdump-bound device, return immediately
+	 * taking no action.
+	 */
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(PDUMP_TRACE_STATE)
+	PVR_DPF((PVR_DBG_WARNING, "PDumpSetFrameKM: ui32Frame( %d )", ui32Frame));
+#endif
+
+	DEBUG_OUTFILES_COMMENT(psDeviceNode, "(pre) Set pdump frame %u", ui32Frame);
+
+	eError = _PDumpSetFrameKM(psConnection, psDeviceNode, ui32Frame);
+	if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+	{
+		PVR_LOG_ERROR(eError, "_PDumpSetFrameKM");
+	}
+
+	DEBUG_OUTFILES_COMMENT(psDeviceNode, "(post) Set pdump frame %u", ui32Frame);
+
+	return eError;
+}
+
+PVRSRV_ERROR PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                             IMG_UINT32* pui32Frame)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/*
+		It may be safe to avoid acquiring this lock here as all the other calls
+		which read/modify current frame will wait on the PDump Control bridge
+		lock first. Also, in no way as of now, does the PDumping app modify the
+		current frame through a call which acquires the global bridge lock.
+		Still, as a legacy we acquire and then read.
+	*/
+	PDumpCtrlLockAcquire();
+
+	*pui32Frame = PDumpCtrlGetCurrentFrame();
+
+	PDumpCtrlLockRelease();
+	return eError;
+}
+
+PVRSRV_ERROR PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_UINT32 ui32Mode,
+                                            IMG_UINT32 ui32Start,
+                                            IMG_UINT32 ui32End,
+                                            IMG_UINT32 ui32Interval,
+                                            IMG_UINT32 ui32MaxParamFileSize,
+                                            IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* NB. We choose not to check that the device is the pdump-bound
+	 * device here, as this particular bridge call is made only from the pdump
+	 * tool itself (which may only connect to the bound device).
+	 */
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	eError = PDumpReady();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PDumpReady");
+
+	/* Validate parameters */
+	if ((ui32End < ui32Start) || (ui32Mode > PDUMP_CAPMODE_MAX))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (ui32Mode == PDUMP_CAPMODE_BLOCKED)
+	{
+		if ((ui32Interval < PDUMP_BLOCKLEN_MIN) || (ui32Interval > PDUMP_BLOCKLEN_MAX))
+		{
+			/* Force client to set ui32Interval (i.e. block length) in valid range */
+			eError = PVRSRV_ERROR_PDUMP_INVALID_BLOCKLEN;
+		}
+
+		if (ui32End != PDUMP_FRAME_MAX)
+		{
+			/* Force client to set ui32End to PDUMP_FRAME_MAX */
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	else if ((ui32Mode != PDUMP_CAPMODE_UNSET) && (ui32Interval < 1))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "PDumpSetDefaultCaptureParamsKM");
+
+	/*
+	   Acquire PDUMP_CTRL_STATE struct lock before modifications as a
+	   PDumping app may be reading the state data for some checks
+	*/
+	PDumpCtrlLockAcquire();
+	PDumpCtrlSetDefaultCaptureParams(ui32Mode, ui32Start, ui32End, ui32Interval, ui32AutoTermTimeout);
+	PDumpSetAutoTerminate(ui32AutoTermTimeout);
+	PDumpCtrlLockRelease();
+
+	if (ui32MaxParamFileSize == 0)
+	{
+		g_PDumpParameters.ui32MaxFileSize = PRM_FILE_SIZE_MAX;
+	}
+	else
+	{
+		g_PDumpParameters.ui32MaxFileSize = ui32MaxParamFileSize;
+	}
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpReg32
+ * Inputs         : pszPDumpDevName, Register offset, and value to write
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+******************************************************************************/
+PVRSRV_ERROR PDumpReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_CHAR	*pszPDumpRegName,
+						IMG_UINT32	ui32Reg,
+						IMG_UINT32	ui32Data,
+						IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X 0x%08X", pszPDumpRegName, ui32Reg, ui32Data);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpReg64
+ * Inputs         : pszPDumpDevName, Register offset, and value to write
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+******************************************************************************/
+PVRSRV_ERROR PDumpReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_CHAR	*pszPDumpRegName,
+						IMG_UINT32	ui32Reg,
+						IMG_UINT64	ui64Data,
+						IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Data >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Data);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X 0x%08X",
+					pszPDumpRegName, ui32Reg, ui32LowerValue);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X 0x%08X",
+					pszPDumpRegName, ui32Reg + 4, ui32UpperValue);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:0x%08X 0x%010" IMG_UINT64_FMTSPECX, pszPDumpRegName, ui32Reg, ui64Data);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToReg64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+ *                  from a register label
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32RegDst,
+                                  IMG_UINT32 ui32RegSrc,
+                                  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:0x%08X :%s:0x%08X", pszPDumpRegName, ui32RegDst, pszPDumpRegName, ui32RegSrc);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToMem32
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write
+ *                  from a register label
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToMem32(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:0x%08X",aszMemspaceName, aszSymbolicName,
+							uiPDumpSymbolicOffset, pszPDumpRegName, ui32Reg);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToMem64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write
+ *                  from a register label
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToMem64(IMG_CHAR *pszPDumpRegName,
+								  IMG_UINT32 ui32Reg,
+								  PMR *psPMR,
+								  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+								  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:0x%08X",aszMemspaceName, aszSymbolicName,
+							uiPDumpSymbolicOffset, pszPDumpRegName, ui32Reg);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpPhysHandleToInternalVar64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents an internal var
+                    write using a PDump pages handle
+******************************************************************************/
+PVRSRV_ERROR PDumpPhysHandleToInternalVar64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_CHAR *pszInternalVar,
+                                            IMG_HANDLE hPdumpPages,
+                                            IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR *pszSymbolicName;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpGetSymbolicAddr(hPdumpPages,
+	                            &pszSymbolicName);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+	                        "WRW %s %s:0x%llX",
+	                        pszInternalVar, pszSymbolicName, 0llu);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s 0x%X", pszPDumpVarName, 0);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#endif
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpMemLabelToInternalVar64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents an internal var
+ *                  write using a memory label
+******************************************************************************/
+PVRSRV_ERROR PDumpMemLabelToInternalVar64(IMG_CHAR *pszInternalVar,
+                                          PMR *psPMR,
+                                          IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                          IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:%s:0x%"IMG_UINT64_FMTSPECX, pszInternalVar,
+							aszMemspaceName, aszSymbolicName, uiPDumpSymbolicOffset);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:%s:0x%"IMG_UINT64_FMTSPECX, pszPDumpVarName,
+							aszMemspaceName, aszSymbolicName, uiPDumpSymbolicOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "SHR %s %s 0x20", pszPDumpVarName, pszPDumpVarName);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#endif
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalVarToMemLabel
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory label
+ *                  write using an internal var
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalVarToMemLabel(PMR *psPMR,
+                                        IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:%s:0x%"IMG_UINT64_FMTSPECX" %s",
+							aszMemspaceName, aszSymbolicName, uiPDumpSymbolicOffset, pszInternalVar);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s 0x%X", pszPDumpVarName, 0);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#endif
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteRegORValueOp
+
+ @Description
+
+ Emits the PDump commands for the logical OR operation
+ Var <- Var OR Value
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarORValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const IMG_CHAR *pszInternalVariable,
+                                    const IMG_UINT64 ui64Value,
+                                    const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Value >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Value);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			"OR %s %s 0x%X",
+#else
+			"OR %s %s 0x%"IMG_UINT64_FMTSPECX,
+#endif
+			pszInternalVariable,
+			pszInternalVariable,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			ui32LowerValue
+#else
+			ui64Value
+#endif
+			);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVariable);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"OR %s %s 0x%X",
+			pszPDumpVarName,
+			pszPDumpVarName,
+			ui32UpperValue);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+#endif
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteVarORVarOp
+
+ @Description
+
+ Emits the PDump commands for the logical OR operation
+ Var <- Var OR Var2
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarORVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const IMG_CHAR *pszInternalVar,
+                                  const IMG_CHAR *pszInternalVar2,
+                                  const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"OR %s %s %s",
+			pszInternalVar,
+			pszInternalVar,
+			pszInternalVar2);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteVarANDVarOp
+
+ @Description
+
+ Emits the PDump commands for the logical AND operation
+ Var <- Var AND Var2
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarANDVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const IMG_CHAR *pszInternalVar,
+                                   const IMG_CHAR *pszInternalVar2,
+                                   const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"AND %s %s %s",
+			pszInternalVar,
+			pszInternalVar,
+			pszInternalVar2);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToInternalVar
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which writes a register label into
+ *                  an internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR *pszPDumpRegName,
+                                        IMG_UINT32 ui32Reg,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:0x%08X", pszInternalVar, pszPDumpRegName, ui32Reg);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:0x%08X", pszPDumpVarName, pszPDumpRegName, ui32Reg + 4);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+#endif
+
+	PDUMP_UNLOCK(ui32Flags);
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalVarToReg32
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+ *                  from an internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalVarToReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X %s", pszPDumpRegName, ui32Reg, pszInternalVar);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalVarToReg64
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+ *                  from an internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalVarToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+	PDUMP_GET_SCRIPT_STRING()
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X %s", pszPDumpRegName, ui32Reg, pszInternalVar);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X %s", pszPDumpRegName, ui32Reg + 4, pszPDumpVarName);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:0x%08X %s", pszPDumpRegName, ui32Reg, pszInternalVar);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+
+
+/******************************************************************************
+ * Function Name  : PDumpMemLabelToMem32
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write from
+ *                  a memory label
+******************************************************************************/
+PVRSRV_ERROR PDumpMemLabelToMem32(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceNameSource[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameSource[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_CHAR aszMemspaceNameDest[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameDest[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetSource;
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetDest;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameSource;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameDest;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMRSource);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRSource,
+	                             uiLogicalOffsetSource,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameSource,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameSource,
+	                             &uiPDumpSymbolicOffsetSource,
+	                             &uiNextSymNameSource);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRDest,
+	                             uiLogicalOffsetDest,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameDest,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameDest,
+	                             &uiPDumpSymbolicOffsetDest,
+	                             &uiNextSymNameDest);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+	                     "WRW :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:%s:0x%"IMG_UINT64_FMTSPECX,
+	                     aszMemspaceNameDest, aszSymbolicNameDest,
+	                     uiPDumpSymbolicOffsetDest, aszMemspaceNameSource,
+	                     aszSymbolicNameSource, uiPDumpSymbolicOffsetSource);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpMemLabelToMem64
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write from
+ *                  a memory label
+******************************************************************************/
+PVRSRV_ERROR PDumpMemLabelToMem64(PMR *psPMRSource,
+								  PMR *psPMRDest,
+								  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+								  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+								  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceNameSource[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameSource[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_CHAR aszMemspaceNameDest[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameDest[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetSource;
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetDest;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameSource;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameDest;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMRSource);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRSource,
+	                             uiLogicalOffsetSource,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameSource,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameSource,
+	                             &uiPDumpSymbolicOffsetSource,
+	                             &uiNextSymNameSource);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRDest,
+	                             uiLogicalOffsetDest,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameDest,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameDest,
+	                             &uiPDumpSymbolicOffsetDest,
+	                             &uiNextSymNameDest);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+	                     "WRW64 :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:%s:0x%"IMG_UINT64_FMTSPECX,
+	                     aszMemspaceNameDest, aszSymbolicNameDest,
+	                     uiPDumpSymbolicOffsetDest, aszMemspaceNameSource,
+	                     aszSymbolicNameSource, uiPDumpSymbolicOffsetSource);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteVarSHRValueOp
+
+ @Description
+
+ Emits the PDump commands for the logical SHR operation
+ Var <-  Var SHR Value
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarSHRValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Value >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Value);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			"SHR %s %s 0x%X",
+#else
+			"SHR %s %s 0x%"IMG_UINT64_FMTSPECX,
+#endif
+			pszInternalVariable,
+			pszInternalVariable,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			ui32LowerValue
+#else
+			ui64Value
+#endif
+			);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVariable);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"SHR %s %s 0x%X",
+			pszPDumpVarName,
+			pszPDumpVarName,
+			ui32UpperValue);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+#endif
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteRegANDValueOp
+
+ @Description
+
+ Emits the PDump commands for the logical AND operation
+ Var <-  Var AND Value
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarANDValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Value >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Value);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			"AND %s %s 0x%X",
+#else
+			"AND %s %s 0x%"IMG_UINT64_FMTSPECX,
+#endif
+			pszInternalVariable,
+			pszInternalVariable,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			ui32LowerValue
+#else
+			ui64Value
+#endif
+			);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVariable);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"AND %s %s 0x%X",
+			pszPDumpVarName,
+			pszPDumpVarName,
+			ui32UpperValue);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+#endif
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpSAW
+ * Inputs         : pszDevSpaceName -- device space from which to output
+ *                  ui32Offset -- offset value from register base
+ *                  ui32NumSaveBytes -- number of bytes to output
+ *                  pszOutfileName -- name of file to output to
+ *                  ui32OutfileOffsetByte -- offset into output file to write
+ *                  uiPDumpFlags -- flags to pass to PDumpOSWriteScript
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Dumps the contents of a register bank into a file
+ *                  NB: ui32NumSaveBytes must be divisible by 4
+******************************************************************************/
+PVRSRV_ERROR PDumpSAW(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR           *pszDevSpaceName,
+                      IMG_UINT32         ui32HPOffsetBytes,
+                      IMG_UINT32         ui32NumSaveBytes,
+                      IMG_CHAR           *pszOutfileName,
+                      IMG_UINT32         ui32OutfileOffsetByte,
+                      PDUMP_FLAGS_T      uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	PVR_DPF((PVR_DBG_ERROR, "PDumpSAW"));
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "SAW :%s:0x%x 0x%x 0x%x %s\n",
+	                          pszDevSpaceName,
+	                          ui32HPOffsetBytes,
+	                          ui32NumSaveBytes / (IMG_UINT32)sizeof(IMG_UINT32),
+	                          ui32OutfileOffsetByte,
+	                          pszOutfileName);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDumpSAW PDumpSNPrintf failed: eError=%u", eError));
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eError;
+	}
+
+	PDUMP_LOCK(uiPDumpFlags);
+	if (! PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDumpSAW PDumpWriteScript failed!"));
+	}
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpRegPolKM
+ * Inputs         : Description of what this register read is trying to do
+ *					pszPDumpDevName
+ *					Register offset
+ *					expected value
+ *					mask for that value
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents a register read
+ *					with the expected value
+******************************************************************************/
+PVRSRV_ERROR PDumpRegPolKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+						   IMG_CHAR				*pszPDumpRegName,
+						   IMG_UINT32			ui32RegAddr,
+						   IMG_UINT32			ui32RegValue,
+						   IMG_UINT32			ui32Mask,
+						   IMG_UINT32			ui32Flags,
+						   PDUMP_POLL_OPERATOR	eOperator)
+{
+	/* Timings correct for Linux and XP */
+	/* Timings should be passed in */
+	#define POLL_DELAY			1000U
+	#define POLL_COUNT_LONG		(2000000000U / POLL_DELAY)
+	#define POLL_COUNT_SHORT	(1000000U / POLL_DELAY)
+
+	PVRSRV_ERROR eErr;
+	IMG_UINT32	ui32PollCount;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	ui32PollCount = POLL_COUNT_LONG;
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "POL :%s:0x%08X 0x%08X 0x%08X %d %u %d",
+							pszPDumpRegName, ui32RegAddr, ui32RegValue,
+							ui32Mask, eOperator, ui32PollCount, POLL_DELAY);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/*!
+ * \name	PDumpOSVerifyLineEnding
+ */
+static void _PDumpVerifyLineEnding(IMG_HANDLE hBuffer, IMG_UINT32 ui32BufferSizeMax)
+{
+	IMG_UINT32 ui32Count;
+	IMG_CHAR* pszBuf = hBuffer;
+
+	/* strlen */
+	ui32Count = OSStringNLength(pszBuf, ui32BufferSizeMax);
+
+	/* Put \n sequence at the end if it isn't already there */
+	if ((ui32Count >= 1) && (pszBuf[ui32Count-1] != '\n') && (ui32Count<ui32BufferSizeMax))
+	{
+		pszBuf[ui32Count] = '\n';
+		ui32Count++;
+		pszBuf[ui32Count] = '\0';
+	}
+}
+
+
+/* Never call direct, needs caller to hold OS Lock.
+ * Use PDumpCommentWithFlags() from within the server.
+ * Clients call this via the bridge and PDumpCommentKM().
+ */
+static PVRSRV_ERROR _PDumpWriteComment(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_CHAR *pszComment,
+                                       IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_DEBUG_OUTFILES)
+	IMG_CHAR pszTemp[PVRSRV_PDUMP_MAX_COMMENT_SIZE+80];
+	IMG_INT32 iCount;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_ASSERT(pszComment != NULL);
+
+	if (OSStringNLength(pszComment, ui32MaxLen) == 0)
+	{
+		/* PDumpOSVerifyLineEnding silently fails if pszComment is too short to
+		   actually hold the line endings that it's trying to enforce, so
+		   short circuit it and force safety */
+		pszComment = "\n";
+	}
+	else
+	{
+		/* Put line ending sequence at the end if it isn't already there */
+		_PDumpVerifyLineEnding(pszComment, ui32MaxLen);
+	}
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+	/* Prefix comment with PID and line number */
+	iCount = OSSNPrintf(pszTemp, PVRSRV_PDUMP_MAX_COMMENT_SIZE+80, "%u %u:%lu %s: %s",
+		OSAtomicRead(&g_sEveryLineCounter),
+		OSGetCurrentClientProcessIDKM(),
+		(unsigned long)OSGetCurrentClientThreadIDKM(),
+		OSGetCurrentClientProcessNameKM(),
+		pszComment);
+	if ((iCount < 0) || (iCount >= (PVRSRV_PDUMP_MAX_COMMENT_SIZE+80)))
+	{
+		eErr = PVRSRV_ERROR_PDUMP_BUF_OVERFLOW;
+	}
+
+	/* Append the comment to the script stream */
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "-- %s",
+		pszTemp);
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "-- %s",
+		pszComment);
+#endif
+	if ((eErr != PVRSRV_OK) &&
+		(eErr != PVRSRV_ERROR_PDUMP_BUF_OVERFLOW))
+	{
+		PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpSNPrintf", ErrUnlock);
+	}
+
+	if (!PDumpWriteScript(psDeviceNode, hScript, ui32Flags))
+	{
+		if (PDUMP_IS_CONTINUOUS(ui32Flags))
+		{
+			eErr = PVRSRV_ERROR_PDUMP_BUFFER_FULL;
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpWriteScript", ErrUnlock);
+		}
+		else
+		{
+			eErr = PVRSRV_ERROR_CMD_NOT_PROCESSED;
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpWriteScript", ErrUnlock);
+		}
+	}
+
+ErrUnlock:
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentKM
+ * Inputs         : ui32CommentSize, pszComment, ui32Flags
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Dumps a pre-formatted comment, primarily called from the
+ *                : bridge.
+******************************************************************************/
+PVRSRV_ERROR PDumpCommentKM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT32 ui32CommentSize,
+                            IMG_CHAR *pszComment,
+                            IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32CommentSize); /* Generated bridge code appends null char to pszComment. */
+
+	PDUMP_LOCK(ui32Flags);
+
+	eErr = _PDumpWriteComment(psDeviceNode, pszComment, ui32Flags);
+
+	PDUMP_UNLOCK(ui32Flags);
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlagsNoLockVA
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : args      - pre-started va_list args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comment, caller need to acquire pdump lock
+ *                  explicitly before calling this function
+******************************************************************************/
+static PVRSRV_ERROR PDumpCommentWithFlagsNoLockVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32 ui32Flags,
+                                           const IMG_CHAR * pszFormat, va_list args)
+{
+	IMG_INT32 iCount;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	PDUMP_GET_MSG_STRING();
+
+	/* Construct the string */
+	iCount = OSVSNPrintf(pszMsg, ui32MaxLen, pszFormat, args);
+	PVR_LOG_GOTO_IF_FALSE(((iCount != -1) && (iCount < ui32MaxLen)), "OSVSNPrintf", exit);
+
+	eErr = _PDumpWriteComment(psDeviceNode, pszMsg, ui32Flags);
+
+exit:
+	PDUMP_RELEASE_MSG_STRING();
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlagsNoLock
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : ... - args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comment, caller need to acquire pdump lock
+ *                  explicitly before calling this function.
+******************************************************************************/
+__printf(3, 4)
+static PVRSRV_ERROR PDumpCommentWithFlagsNoLock(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_UINT32 ui32Flags,
+                                         IMG_CHAR *pszFormat, ...)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	va_list args;
+
+	va_start(args, pszFormat);
+	PDumpCommentWithFlagsNoLockVA(psDeviceNode, ui32Flags, pszFormat, args);
+	va_end(args);
+
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlags
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : ... - args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comments
+******************************************************************************/
+PVRSRV_ERROR PDumpCommentWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_UINT32 ui32Flags,
+                                   IMG_CHAR * pszFormat, ...)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	va_list args;
+
+	va_start(args, pszFormat);
+	PDumpCommentWithFlagsVA(psDeviceNode, ui32Flags, pszFormat, args);
+	va_end(args);
+
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlagsVA
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : args      - pre-started va_list args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comments
+******************************************************************************/
+PVRSRV_ERROR PDumpCommentWithFlagsVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Flags,
+                                     const IMG_CHAR * pszFormat, va_list args)
+{
+	IMG_INT32 iCount;
+	PVRSRV_ERROR eErr = PVRSRV_ERROR_INVALID_PARAMS;
+	PDUMP_GET_MSG_STRING();
+
+	/* Construct the string */
+	iCount = OSVSNPrintf(pszMsg, ui32MaxLen, pszFormat, args);
+	PVR_LOG_GOTO_IF_FALSE(((iCount != -1) && (iCount < ui32MaxLen)), "OSVSNPrintf", exit);
+
+	PDUMP_LOCK(ui32Flags);
+	eErr = _PDumpWriteComment(psDeviceNode, pszMsg, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+exit:
+	PDUMP_RELEASE_MSG_STRING();
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCOMCommand
+ * Inputs         : ui32PDumpFlags - PDump flags
+ *			: pszPdumpStr - string for COM command
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : PDumps a COM command
+******************************************************************************/
+PVRSRV_ERROR PDumpCOMCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags,
+                             const IMG_CHAR * pszPdumpStr)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "COM %s\n", pszPdumpStr);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ * Function Name  : PDumpPanic
+ * Inputs         : ui32PanicNo - Unique number for panic condition
+ *				  : pszPanicMsg - Panic reason message limited to ~90 chars
+ *				  : pszPPFunc   - Function name string where panic occurred
+ *				  : ui32PPline  - Source line number where panic occurred
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : PDumps a panic assertion. Used when the host driver
+ *                : detects a condition that will lead to an invalid PDump
+ *                : script that cannot be played back off-line.
+ */ /*************************************************************************/
+PVRSRV_ERROR PDumpPanic(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32      ui32PanicNo,
+						IMG_CHAR*       pszPanicMsg,
+						const IMG_CHAR* pszPPFunc,
+						IMG_UINT32      ui32PPline)
+{
+	PVRSRV_ERROR   eError = PVRSRV_OK;
+	PDUMP_FLAGS_T  uiPDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+	PDUMP_GET_SCRIPT_STRING();
+
+	/* Log the panic condition to the live kern.log in both REL and DEB mode
+	 * to aid user PDump troubleshooting. */
+	PVR_LOG(("PDUMP PANIC %08x: %s", ui32PanicNo, pszPanicMsg));
+	PVR_DPF((PVR_DBG_MESSAGE, "PDUMP PANIC start %s:%d", pszPPFunc, ui32PPline));
+
+	/* Check the supplied panic reason string is within length limits */
+	PVR_ASSERT(OSStringLength(pszPanicMsg)+sizeof("PANIC   ") < PVRSRV_PDUMP_MAX_COMMENT_SIZE-1);
+
+	/* Obtain lock to keep the multi-line
+	 * panic statement together in a single atomic write */
+	PDUMP_BLKSTART(uiPDumpFlags);
+
+
+	/* Write -- Panic start (Function:line) */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "-- Panic start (%s:%d)", pszPPFunc, ui32PPline);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpSNPrintf", e1);
+	(void)PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+	/* Write COM messages */
+	eError = PDumpCOMCommand(psDeviceNode, uiPDumpFlags,
+				  "**** Script invalid and not compatible with off-line playback. ****");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCOMCommand", e1);
+
+	eError = PDumpCOMCommand(psDeviceNode, uiPDumpFlags,
+				  "**** Check test parameters and driver configuration, stop imminent. ****");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCOMCommand", e1);
+
+	/* Write PANIC no msg command */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "PANIC %08x %s", ui32PanicNo, pszPanicMsg);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpSNPrintf", e1);
+	(void)PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+	/* Write -- Panic end */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "-- Panic end");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpSNPrintf", e1);
+	(void)PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+e1:
+	PDUMP_BLKEND(uiPDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ * Function Name  : PDumpCaptureError
+ * Inputs         : ui32ErrorNo - Unique number for panic condition
+ *                : pszErrorMsg - Panic reason message limited to ~90 chars
+ *                : pszPPFunc   - Function name string where panic occurred
+ *                : ui32PPline  - Source line number where panic occurred
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : PDumps an error string to the script file to interrupt
+ *                : play back to inform user of a fatal issue that occurred
+ *                : during PDump capture.
+ */ /*************************************************************************/
+PVRSRV_ERROR PDumpCaptureError(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PVRSRV_ERROR       ui32ErrorNo,
+                               IMG_CHAR*          pszErrorMsg,
+                               const IMG_CHAR     *pszPPFunc,
+                               IMG_UINT32         ui32PPline)
+{
+	IMG_CHAR*       pszFormatStr = "DRIVER_ERROR: %3d: %s";
+	PDUMP_FLAGS_T   uiPDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	/* Need to return an error using this macro */
+	PDUMP_GET_SCRIPT_STRING();
+
+	/* Check the supplied panic reason string is within length limits */
+	PVR_ASSERT(OSStringLength(pszErrorMsg)+sizeof(pszFormatStr) < PVRSRV_PDUMP_MAX_COMMENT_SIZE-1);
+
+	/* Write driver error message to the script file */
+	(void) PDumpSNPrintf(hScript, ui32MaxLen, pszFormatStr, ui32ErrorNo, pszErrorMsg);
+
+	/* Obtain lock to keep the multi-line
+	 * panic statement together in a single atomic write */
+	PDUMP_LOCK(uiPDumpFlags);
+	(void) PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpImageDescriptor
+
+ @Description
+
+ Dumps an OutputImage command and its associated header info.
+
+ @Input    psDeviceNode			: device
+ @Input    ui32MMUContextID		: MMU context
+ @Input    pszSABFileName		: filename string
+
+ @Return   PVRSRV_ERROR			:
+
+******************************************************************************/
+PVRSRV_ERROR PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32MMUContextID,
+									IMG_CHAR *pszSABFileName,
+									IMG_DEV_VIRTADDR sData,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_DEV_VIRTADDR sHeader,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32PDumpFlags)
+{
+#if !defined(SUPPORT_RGX)
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#else
+	PVRSRV_ERROR  eErr = PVRSRV_OK;
+	IMG_CHAR      *pszPDumpDevName = psDeviceNode->sDevId.pszPDumpDevName;
+	IMG_BYTE      abyPDumpDesc[IMAGE_HEADER_SIZE];
+	IMG_UINT32    ui32ParamOutPos, ui32SABOffset = 0;
+	IMG_BOOL      bRawImageData = IMG_FALSE;
+
+	PDUMP_GET_SCRIPT_AND_FILE_STRING();
+
+	if (pszSABFileName == NULL)
+	{
+		eErr = PVRSRV_ERROR_INVALID_PARAMS;
+		goto error_release_script;
+	}
+
+	/* Writing image descriptor to persistent buffer is not permitted */
+	if (ui32PDumpFlags & PDUMP_FLAGS_PERSISTENT)
+	{
+		goto error_release_script;
+	}
+
+	/* Prepare OutputImage descriptor header */
+	eErr = RGXPDumpPrepareOutputImageDescriptorHdr(psDeviceNode,
+									ui32HeaderSize,
+									ui32DataSize,
+									ui32LogicalWidth,
+									ui32LogicalHeight,
+									ui32PhysicalWidth,
+									ui32PhysicalHeight,
+									ePixFmt,
+									eMemLayout,
+									eFBCompression,
+									paui32FBCClearColour,
+									eFBCSwizzle,
+									&(abyPDumpDesc[0]));
+	PVR_LOG_GOTO_IF_ERROR(eErr, "RGXPDumpPrepareOutputImageDescriptorHdr", error_release_script);
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	PDumpCommentWithFlagsNoLock(psDeviceNode, ui32PDumpFlags, "Dump Image descriptor");
+
+	bRawImageData =
+		 (ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_YUV8
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV12
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV8
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV8
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV10
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV10
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_VY0UY1_8888
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_UY0VY1_8888
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_Y0UY1V_8888
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_Y0VY1U_8888);
+
+
+	if (bRawImageData)
+	{
+		IMG_UINT32 ui32ElementType;
+		IMG_UINT32 ui32ElementCount;
+
+		PDumpCommentWithFlagsNoLock(psDeviceNode, ui32PDumpFlags,
+		                            "YUV data. Switching from OutputImage to SAB. Width=0x%08X Height=0x%08X",
+		                            ui32LogicalWidth, ui32LogicalHeight);
+
+		PDUMP_UNLOCK(ui32PDumpFlags);
+
+		PDUMP_RELEASE_SCRIPT_AND_FILE_STRING();
+
+		ui32ElementType = 0;
+		ui32ElementCount = 0;
+
+		/* Switch to CMD:OutputData with IBIN header. */
+		return PDumpDataDescriptor(psDeviceNode,
+								   ui32MMUContextID,
+								   pszSABFileName,
+								   sData,
+								   ui32DataSize,
+								   IBIN_HEADER_TYPE,
+								   ui32ElementType,
+								   ui32ElementCount,
+								   ui32PDumpFlags);
+	}
+
+	/* Write OutputImage descriptor header to parameter file */
+	eErr = PDumpWriteParameter(psDeviceNode,
+							   abyPDumpDesc,
+							   IMAGE_HEADER_SIZE,
+							   ui32PDumpFlags,
+							   &ui32ParamOutPos,
+							   pszFileName);
+	if (eErr != PVRSRV_OK)
+	{
+		if (eErr != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+		{
+			PDUMP_ERROR(psDeviceNode, eErr,
+			            "Failed to write device allocation to parameter file");
+			PVR_LOG_ERROR(eErr, "PDumpWriteParameter");
+		}
+		else
+		{
+			/*
+			 * Write to parameter file prevented under the flags and
+			 * current state of the driver so skip write to script and return.
+			 */
+			eErr = PVRSRV_OK;
+		}
+		goto error;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"MALLOC :%s:BINHEADER 0x%08X 0x%08X\n",
+							pszPDumpDevName,
+							IMAGE_HEADER_SIZE,
+							IMAGE_HEADER_SIZE);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"LDB :%s:BINHEADER:0x00 0x%08x 0x%08x %s\n",
+							pszPDumpDevName,
+							IMAGE_HEADER_SIZE,
+							ui32ParamOutPos,
+							pszFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:BINHEADER:0x00 0x%08X 0x00000000 %s.bin\n",
+							pszPDumpDevName,
+							IMAGE_HEADER_SIZE,
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	ui32SABOffset += IMAGE_HEADER_SIZE;
+
+	/*
+	 * Write out the header section if image is FB compressed
+	 */
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		eErr = PDumpSNPrintf(hScript,
+								ui32MaxLenScript,
+								"SAB :%s:v%x:0x%010"IMG_UINT64_FMTSPECX" 0x%08X 0x%08X %s.bin\n",
+								pszPDumpDevName,
+								ui32MMUContextID,
+								(IMG_UINT64)sHeader.uiAddr,
+								ui32HeaderSize,
+								ui32SABOffset,
+								pszSABFileName);
+		PVR_GOTO_IF_ERROR(eErr, error);
+		PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+		ui32SABOffset += ui32HeaderSize;
+	}
+
+	/*
+	 * Now dump out the actual data associated with the surface
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:v%x:0x%010"IMG_UINT64_FMTSPECX" 0x%08X 0x%08X %s.bin\n",
+							pszPDumpDevName,
+							ui32MMUContextID,
+							(IMG_UINT64)sData.uiAddr,
+							ui32DataSize,
+							ui32SABOffset,
+							pszSABFileName);
+
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	/*
+	 * The OutputImage command is required to trigger processing of the output
+	 * data
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"CMD:OutputImage %s.bin\n",
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"FREE :%s:BINHEADER\n",
+							pszPDumpDevName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+error:
+	PDUMP_UNLOCK(ui32PDumpFlags);
+error_release_script:
+	PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()
+	return eErr;
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpDataDescriptor
+
+ @Description
+
+ Dumps an OutputData command and its associated header info.
+
+ @Input    psDeviceNode         : device
+ @Input    ui32MMUContextID     : MMU context
+ @Input    pszSABFileName       : filename string
+ @Input    sData                : GPU virtual address of data
+ @Input    ui32HeaderType       : Header type
+ @Input    ui32DataSize         : Data size
+ @Input    ui32ElementType      : Element type being dumped
+ @Input    ui32ElementCount     : Number of elements to be dumped
+ @Input    ui32PDumpFlags       : PDump flags
+
+ @Return   PVRSRV_ERROR         :
+
+******************************************************************************/
+PVRSRV_ERROR PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32MMUContextID,
+									IMG_CHAR *pszSABFileName,
+									IMG_DEV_VIRTADDR sData,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_UINT32 ui32PDumpFlags)
+{
+#if !defined(SUPPORT_RGX)
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#else
+	PVRSRV_ERROR   eErr = PVRSRV_OK;
+	IMG_CHAR       *pszPDumpDevName = psDeviceNode->sDevId.pszPDumpDevName;
+	IMG_BYTE       abyPDumpDesc[DATA_HEADER_SIZE];
+	IMG_UINT32     ui32ParamOutPos, ui32SABOffset = 0;
+	IMG_UINT32     ui32HeaderSize;
+
+	PDUMP_GET_SCRIPT_AND_FILE_STRING();
+
+	PVR_GOTO_IF_INVALID_PARAM(pszSABFileName, eErr, error_release_script);
+
+	if (ui32HeaderType == DATA_HEADER_TYPE)
+	{
+		ui32HeaderSize = DATA_HEADER_SIZE;
+	}
+	else if (ui32HeaderType == IBIN_HEADER_TYPE)
+	{
+		ui32HeaderSize = IBIN_HEADER_SIZE;
+	}
+	else
+	{
+		PVR_GOTO_WITH_ERROR(eErr, PVRSRV_ERROR_INVALID_PARAMS, error_release_script);
+	}
+
+	/* Writing data descriptor to persistent buffer is not permitted */
+	if (ui32PDumpFlags & PDUMP_FLAGS_PERSISTENT)
+	{
+		goto error_release_script;
+	}
+
+	/* Prepare OutputData descriptor header */
+	eErr = RGXPDumpPrepareOutputDataDescriptorHdr(psDeviceNode,
+									ui32HeaderType,
+									ui32DataSize,
+									ui32ElementType,
+									ui32ElementCount,
+									&(abyPDumpDesc[0]));
+	PVR_LOG_GOTO_IF_ERROR(eErr, "RGXPDumpPrepareOutputDataDescriptorHdr", error_release_script);
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	PDumpCommentWithFlagsNoLock(psDeviceNode, ui32PDumpFlags, "Dump Data descriptor");
+
+	/* Write OutputImage command header to parameter file */
+	eErr = PDumpWriteParameter(psDeviceNode,
+							   abyPDumpDesc,
+							   ui32HeaderSize,
+							   ui32PDumpFlags,
+							   &ui32ParamOutPos,
+							   pszFileName);
+	if (eErr != PVRSRV_OK)
+	{
+		if (eErr != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+		{
+			PDUMP_ERROR(psDeviceNode, eErr,
+			            "Failed to write device allocation to parameter file");
+			PVR_LOG_ERROR(eErr, "PDumpWriteParameter");
+		}
+		else
+		{
+			/*
+			 * Write to parameter file prevented under the flags and
+			 * current state of the driver so skip write to script and return.
+			 */
+			eErr = PVRSRV_OK;
+		}
+		goto error;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"MALLOC :%s:BINHEADER 0x%08X 0x%08X\n",
+							pszPDumpDevName,
+							ui32HeaderSize,
+							ui32HeaderSize);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"LDB :%s:BINHEADER:0x00 0x%08x 0x%08x %s\n",
+							pszPDumpDevName,
+							ui32HeaderSize,
+							ui32ParamOutPos,
+							pszFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:BINHEADER:0x00 0x%08X 0x00000000 %s.bin\n",
+							pszPDumpDevName,
+							ui32HeaderSize,
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	ui32SABOffset += ui32HeaderSize;
+
+	/*
+	 * Now dump out the actual data associated
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:v%x:0x%010"IMG_UINT64_FMTSPECX" 0x%08X 0x%08X %s.bin\n",
+							pszPDumpDevName,
+							ui32MMUContextID,
+							(IMG_UINT64)sData.uiAddr,
+							ui32DataSize,
+							ui32SABOffset,
+							pszSABFileName);
+
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	/*
+	 * The OutputData command is required to trigger processing of the output
+	 * data
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"CMD:OutputData %s.bin\n",
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"FREE :%s:BINHEADER\n",
+							pszPDumpDevName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+error:
+	PDUMP_UNLOCK(ui32PDumpFlags);
+error_release_script:
+	PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()
+	return eErr;
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpReadRegKM
+
+ @Description
+
+ Dumps a read from a device register to a file
+
+ @Input    psConnection			: connection info
+ @Input    pszFileName
+ @Input    ui32FileOffset
+ @Input    ui32Address
+ @Input    ui32Size
+ @Input    ui32PDumpFlags
+
+ @Return   PVRSRV_ERROR			:
+
+******************************************************************************/
+PVRSRV_ERROR PDumpReadRegKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszFileName,
+                            IMG_UINT32 ui32FileOffset,
+                            IMG_UINT32 ui32Address,
+                            IMG_UINT32 ui32Size,
+                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"SAB :%s:0x%08X 0x%08X %s",
+			pszPDumpRegName,
+			ui32Address,
+			ui32FileOffset,
+			pszFileName);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegRead32ToInternalVar
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which reads register into an
+ *                  internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead32ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_CHAR *pszPDumpRegName,
+							IMG_UINT32 ui32Reg,
+							IMG_CHAR *pszInternalVar,
+							IMG_UINT32 ui32Flags)
+
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLen,
+							"RDW %s :%s:0x%08X",
+							pszInternalVar,
+							pszPDumpRegName,
+							ui32Reg);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ @name		PDumpRegRead32
+ @brief		Dump 32-bit register read to script
+ @param		pszPDumpDevName - pdump device name
+ @param		ui32RegOffset - register offset
+ @param		ui32Flags - pdump flags
+ @return	Error
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead32(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_CHAR *pszPDumpRegName,
+							const IMG_UINT32 ui32RegOffset,
+							IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW :%s:0x%X",
+							pszPDumpRegName,
+							ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ @name      PDumpRegRead64ToInternalVar
+ @brief     Read 64-bit register into an internal variable
+ @param     pszPDumpDevName - pdump device name
+ @param     ui32RegOffset - register offset
+ @param     ui32Flags - pdump flags
+ @return    Error
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead64ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszInternalVar,
+                            const IMG_UINT32 ui32RegOffset,
+                            IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+	PDUMP_GET_SCRIPT_STRING();
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW %s :%s:0x%X",
+	                     pszInternalVar,
+	                     pszPDumpRegName,
+	                     ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW %s :%s:0x%X",
+	                        pszPDumpVarName,
+	                        pszPDumpRegName,
+	                        ui32RegOffset + 4);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW64 %s :%s:0x%X",
+	                        pszInternalVar,
+	                        pszPDumpRegName,
+	                        ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ @name		PDumpRegRead64
+ @brief		Dump 64-bit register read to script
+ @param		pszPDumpDevName - pdump device name
+ @param		ui32RegOffset - register offset
+ @param		ui32Flags - pdump flags
+ @return	Error
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead64(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_CHAR *pszPDumpRegName,
+							const IMG_UINT32 ui32RegOffset,
+							IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW :%s:0x%X",
+							pszPDumpRegName, ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW :%s:0x%X",
+							pszPDumpRegName, ui32RegOffset + 4);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW64 :%s:0x%X",
+							pszPDumpRegName,
+							ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ FUNCTION	: PDumpWriteShiftedMaskedValue
+
+ PURPOSE	: Emits the PDump commands for writing a masked shifted address
+              into another location
+
+ PARAMETERS	: PDump symbolic name and offset of target word
+              PDump symbolic name and offset of source address
+              right shift amount
+              left shift amount
+              mask
+
+ RETURNS	: None
+******************************************************************************/
+PVRSRV_ERROR
+PDumpWriteShiftedMaskedValue(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const IMG_CHAR *pszDestRegspaceName,
+                             const IMG_CHAR *pszDestSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiDestOffset,
+                             const IMG_CHAR *pszRefRegspaceName,
+                             const IMG_CHAR *pszRefSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiRefOffset,
+                             IMG_UINT32 uiSHRAmount,
+                             IMG_UINT32 uiSHLAmount,
+                             IMG_UINT32 uiMask,
+                             IMG_DEVMEM_SIZE_T uiWordSize,
+                             IMG_UINT32 uiPDumpFlags)
+{
+	PVRSRV_ERROR         eError;
+
+	/* Suffix of WRW command in PDump (i.e. WRW or WRW64) */
+	const IMG_CHAR       *pszWrwSuffix;
+
+	/* Internal PDump register used for interim calculation */
+	const IMG_CHAR       *pszPDumpIntRegSpace;
+	IMG_UINT32           uiPDumpIntRegNum;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	if ((uiWordSize != 4) && (uiWordSize != 8))
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	pszWrwSuffix = (uiWordSize == 8) ? "64" : "";
+
+	/* Should really "Acquire" a pdump register here */
+	pszPDumpIntRegSpace = pszDestRegspaceName;
+	uiPDumpIntRegNum = 1;
+
+	PDUMP_LOCK(uiPDumpFlags);
+	eError = PDumpSNPrintf(hScript,
+	               ui32MaxLen,
+	               /* Should this be "MOV" instead? */
+	               "WRW :%s:$%d :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC "\n",
+	               /* dest */
+	               pszPDumpIntRegSpace,
+	               uiPDumpIntRegNum,
+	               /* src */
+	               pszRefRegspaceName,
+	               pszRefSymbolicName,
+	               uiRefOffset);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+	if (uiSHRAmount > 0)
+	{
+		eError = PDumpSNPrintf(hScript,
+		               ui32MaxLen,
+		               "SHR :%s:$%d :%s:$%d 0x%X\n",
+		               /* dest */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src A */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src B */
+		               uiSHRAmount);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+	if (uiSHLAmount > 0)
+	{
+		eError = PDumpSNPrintf(hScript,
+		               ui32MaxLen,
+		               "SHL :%s:$%d :%s:$%d 0x%X\n",
+		               /* dest */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src A */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src B */
+		               uiSHLAmount);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+	if (uiMask != (1ULL << (8*uiWordSize))-1)
+	{
+		eError = PDumpSNPrintf(hScript,
+		               ui32MaxLen,
+		               "AND :%s:$%d :%s:$%d 0x%X\n",
+		               /* dest */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src A */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src B */
+		               uiMask);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+	eError = PDumpSNPrintf(hScript,
+	               ui32MaxLen,
+	               "WRW%s :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " :%s:$%d\n",
+	               pszWrwSuffix,
+	               /* dest */
+	               pszDestRegspaceName,
+	               pszDestSymbolicName,
+	               uiDestOffset,
+	               /* src */
+	               pszPDumpIntRegSpace,
+	               uiPDumpIntRegNum);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(uiPDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PDumpWriteSymbAddress(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      const IMG_CHAR *pszDestSpaceName,
+                      IMG_DEVMEM_OFFSET_T uiDestOffset,
+                      const IMG_CHAR *pszRefSymbolicName,
+                      IMG_DEVMEM_OFFSET_T uiRefOffset,
+                      const IMG_CHAR *pszPDumpDevName,
+                      IMG_UINT32 ui32WordSize,
+                      IMG_UINT32 ui32AlignShift,
+                      IMG_UINT32 ui32Shift,
+                      IMG_UINT32 uiPDumpFlags)
+{
+	const IMG_CHAR       *pszWrwSuffix = "";
+	PVRSRV_ERROR         eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	if (ui32WordSize == 8)
+	{
+		pszWrwSuffix = "64";
+	}
+
+	PDUMP_LOCK(uiPDumpFlags);
+
+	if (ui32AlignShift != ui32Shift)
+	{
+		/* Write physical address into a variable */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "WRW%s :%s:$1 %s:" IMG_DEVMEM_OFFSET_FMTSPEC "\n",
+				       pszWrwSuffix,
+				       /* dest */
+				       pszPDumpDevName,
+				       /* src */
+				       pszRefSymbolicName,
+				       uiRefOffset);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+		/* apply address alignment */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "SHR :%s:$1 :%s:$1 0x%X",
+				       /* dest */
+				       pszPDumpDevName,
+				       /* src A */
+				       pszPDumpDevName,
+				       /* src B */
+				       ui32AlignShift);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+		/* apply address shift */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "SHL :%s:$1 :%s:$1 0x%X",
+				       /* dest */
+				       pszPDumpDevName,
+				       /* src A */
+				       pszPDumpDevName,
+				       /* src B */
+				       ui32Shift);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+
+		/* write result to register */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "WRW%s :%s:0x%08X :%s:$1",
+				       pszWrwSuffix,
+				       pszDestSpaceName,
+				       (IMG_UINT32)uiDestOffset,
+				       pszPDumpDevName);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+	else
+	{
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "WRW%s :%s:" IMG_DEVMEM_OFFSET_FMTSPEC " %s:" IMG_DEVMEM_OFFSET_FMTSPEC "\n",
+				       pszWrwSuffix,
+				       /* dest */
+				       pszDestSpaceName,
+				       uiDestOffset,
+				       /* src */
+				       pszRefSymbolicName,
+				       uiRefOffset);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+symbAddress_error:
+	PDUMP_UNLOCK(uiPDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return eError;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpIDLWithFlags
+ * Inputs         : Idle time in clocks
+ * Outputs        : None
+ * Returns        : Error
+ * Description    : Dump IDL command to script
+******************************************************************************/
+PVRSRV_ERROR PDumpIDLWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Clocks, IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "IDL %u", ui32Clocks);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpIDL
+ * Inputs         : Idle time in clocks
+ * Outputs        : None
+ * Returns        : Error
+ * Description    : Dump IDL command to script
+******************************************************************************/
+PVRSRV_ERROR PDumpIDL(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32Clocks)
+{
+	return PDumpIDLWithFlags(psDeviceNode, ui32Clocks, PDUMP_FLAGS_CONTINUOUS);
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegBasedCBP
+ * Inputs         : pszPDumpRegName, ui32RegOffset, ui32WPosVal, ui32PacketSize
+ *                  ui32BufferSize, ui32Flags
+ * Outputs        : None
+ * Returns        : Error
+ * Description    : Dump CBP command to script
+******************************************************************************/
+PVRSRV_ERROR PDumpRegBasedCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+							  IMG_CHAR		*pszPDumpRegName,
+							  IMG_UINT32	ui32RegOffset,
+							  IMG_UINT32	ui32WPosVal,
+							  IMG_UINT32	ui32PacketSize,
+							  IMG_UINT32	ui32BufferSize,
+							  IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			 ui32MaxLen,
+			 "CBP :%s:0x%08X 0x%08X 0x%08X 0x%08X",
+			 pszPDumpRegName,
+			 ui32RegOffset,
+			 ui32WPosVal,
+			 ui32PacketSize,
+			 ui32BufferSize);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PDumpTRG(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR *pszMemSpace,
+                      IMG_UINT32 ui32MMUCtxID,
+                      IMG_UINT32 ui32RegionID,
+                      IMG_BOOL bEnable,
+                      IMG_UINT64 ui64VAddr,
+                      IMG_UINT64 ui64LenBytes,
+                      IMG_UINT32 ui32XStride,
+                      IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	if (bEnable)
+	{
+		eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+				     "TRG :%s:v%u %u 0x%08"IMG_UINT64_FMTSPECX" 0x%08"IMG_UINT64_FMTSPECX" %u",
+				     pszMemSpace, ui32MMUCtxID, ui32RegionID,
+				     ui64VAddr, ui64LenBytes, ui32XStride);
+	}
+	else
+	{
+		eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+				     "TRG :%s:v%u %u",
+				     pszMemSpace, ui32MMUCtxID, ui32RegionID);
+
+	}
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpConnectionNotify
+ * Description    : Called by the srvcore to tell PDump core that the
+ *                  PDump capture and control client has connected
+******************************************************************************/
+void PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(TL_BUFFER_STATS)
+	PVRSRV_ERROR		eErr;
+#endif
+
+	OSAtomicIncrement(&g_sConnectionCount);
+
+	/* Reset the parameter file attributes */
+	g_PDumpParameters.sWOff.ui32Main = g_PDumpParameters.sWOff.ui32Init;
+	g_PDumpParameters.ui32FileIdx = 0;
+
+	/* Reset the script file attributes */
+	g_PDumpScript.ui32FileIdx = 0;
+
+	/* The Main script & parameter buffers should be empty after the previous
+	 * PDump capture if it completed correctly.
+	 * When PDump client is not connected, writes are prevented to Main
+	 * buffers in PDumpWriteAllowed() since no capture range, no client,
+	 * no writes to Main buffers for continuous flagged and regular writes.
+	 */
+	if (!TLStreamOutOfData(g_PDumpParameters.sCh.sMainStream.hTL)) /* !empty */
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDump Main parameter buffer not empty, capture will be corrupt!"));
+	}
+	if (!TLStreamOutOfData(g_PDumpScript.sCh.sMainStream.hTL)) /* !empty */
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDump Main script buffer not empty, capture will be corrupt!"));
+	}
+
+#if defined(TL_BUFFER_STATS)
+	eErr = TLStreamResetProducerByteCount(g_PDumpParameters.sCh.sMainStream.hTL, g_PDumpParameters.sWOff.ui32Init);
+	PVR_LOG_IF_ERROR(eErr, "TLStreamResetByteCount Parameter Main");
+
+	eErr = TLStreamResetProducerByteCount(g_PDumpScript.sCh.sMainStream.hTL, 0);
+	PVR_LOG_IF_ERROR(eErr, "TLStreamResetByteCount Script Main");
+#endif
+
+	if (psDeviceNode->pfnPDumpInitDevice)
+	{
+		/* Reset pdump according to connected device */
+		psDeviceNode->pfnPDumpInitDevice(psDeviceNode);
+	}
+}
+
+/******************************************************************************
+ * Function Name  : PDumpDisconnectionNotify
+ * Description    : Called by the connection_server to tell PDump core that
+ *                  the PDump capture and control client has disconnected
+******************************************************************************/
+void PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eErr;
+
+	if (PDumpCtrlCaptureOn())
+	{
+		PVR_LOG(("pdump killed, capture files may be invalid or incomplete!"));
+
+		/* Disable capture in server, in case PDump client was killed and did
+		 * not get a chance to reset the capture parameters.
+		 * Will set module state back to READY.
+		 */
+		eErr = PDumpSetDefaultCaptureParamsKM(NULL, psDeviceNode, PDUMP_CAPMODE_UNSET,
+						      PDUMP_FRAME_UNSET, PDUMP_FRAME_UNSET, 0, 0, 0);
+		PVR_LOG_IF_ERROR(eErr, "PDumpSetDefaultCaptureParamsKM");
+	}
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegCondStr
+ * Inputs         : Description of what this register read is trying to do
+ *					pszPDumpDevName
+ *					Register offset
+ *					expected value
+ *					mask for that value
+ * Outputs        : PDump conditional test for use with 'IF' and 'DOW'
+ * Returns        : None
+ * Description    : Create a PDUMP conditional test. The string is allocated
+ *					on the heap and should be freed by the caller on success.
+******************************************************************************/
+PVRSRV_ERROR PDumpRegCondStr(IMG_CHAR            **ppszPDumpCond,
+                             IMG_CHAR            *pszPDumpRegName,
+                             IMG_UINT32          ui32RegAddr,
+                             IMG_UINT32          ui32RegValue,
+                             IMG_UINT32          ui32Mask,
+                             IMG_UINT32          ui32Flags,
+                             PDUMP_POLL_OPERATOR eOperator)
+{
+	IMG_UINT32	ui32PollCount;
+
+	PDUMP_GET_MSG_STRING();
+
+	ui32PollCount = POLL_COUNT_SHORT;
+
+	if (0 == OSSNPrintf(pszMsg, ui32MaxLen, ":%s:0x%08X 0x%08X 0x%08X %d %u %d",
+						pszPDumpRegName, ui32RegAddr, ui32RegValue,
+						ui32Mask, eOperator, ui32PollCount, POLL_DELAY))
+	{
+		PDUMP_RELEASE_MSG_STRING()
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	*ppszPDumpCond = pszMsg;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalValCondStr
+ * Inputs         : Description of what this register read is trying to do
+ *					pszPDumpDevName
+ *					Internal variable
+ *					expected value
+ *					mask for that value
+ * Outputs        : PDump conditional test for use with 'IF' and 'DOW'
+ * Returns        : None
+ * Description    : Create a PDUMP conditional test. The string is allocated
+ *					on the heap and should be freed by the caller on success.
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalValCondStr(IMG_CHAR            **ppszPDumpCond,
+                                     IMG_CHAR            *pszInternalVar,
+                                     IMG_UINT32          ui32RegValue,
+                                     IMG_UINT32          ui32Mask,
+                                     IMG_UINT32          ui32Flags,
+                                     PDUMP_POLL_OPERATOR eOperator)
+{
+	IMG_UINT32	ui32PollCount;
+
+	PDUMP_GET_MSG_STRING();
+
+	ui32PollCount = POLL_COUNT_SHORT;
+
+	if (0 == OSSNPrintf(pszMsg, ui32MaxLen, "%s 0x%08X 0x%08X %d %u %d",
+						pszInternalVar, ui32RegValue,
+						ui32Mask, eOperator, ui32PollCount, POLL_DELAY))
+	{
+		PDUMP_RELEASE_MSG_STRING()
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	*ppszPDumpCond = pszMsg;
+
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpIfKM
+ * Inputs         : pszPDumpCond - string for condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents IF command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpIfKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "IF %s\n", pszPDumpCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpElseKM
+ * Inputs         : pszPDumpCond - string for condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents ELSE command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpElseKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "ELSE %s\n", pszPDumpCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpFiKM
+ * Inputs         : pszPDumpCond - string for condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents FI command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpFiKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "FI %s\n", pszPDumpCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpStartDoLoopKM
+ * Inputs         : None
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents SDO command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpStartDoLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "SDO");
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpEndDoWhileLoopKM
+ * Inputs         : pszPDumpWhileCond - string for loop condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents DOW command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpEndDoWhileLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_CHAR *pszPDumpWhileCond,
+                                   IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "DOW %s\n", pszPDumpWhileCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+
+void PDumpLock(void)
+{
+	OSLockAcquire(g_hPDumpWriteLock);
+}
+void PDumpUnlock(void)
+{
+	OSLockRelease(g_hPDumpWriteLock);
+}
+static void PDumpAssertWriteLockHeld(void)
+{
+	/* It is expected to be g_hPDumpWriteLock is locked at this point. */
+	PVR_ASSERT(OSLockIsLocked(g_hPDumpWriteLock));
+}
+
+#if defined(PDUMP_TRACE_STATE)
+void PDumpCommonDumpState(void)
+{
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.hTL (In, Mn, De, Bk) ( %p, %p, %p, %p )",
+			g_PDumpScript.sCh.sInitStream.hTL, g_PDumpScript.sCh.sMainStream.hTL, g_PDumpScript.sCh.sDeinitStream.hTL, g_PDumpScript.sCh.sBlockStream.hTL));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.ui32BufferFullRetries (In, Mn, De, Bk) ( %5d, %5d, %5d, %5d )",
+			g_PDumpScript.sCh.sInitStream.ui32BufferFullRetries,
+			g_PDumpScript.sCh.sMainStream.ui32BufferFullRetries,
+			g_PDumpScript.sCh.sDeinitStream.ui32BufferFullRetries,
+			g_PDumpScript.sCh.sBlockStream.ui32BufferFullRetries));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.ui32BufferFullAborts (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+				g_PDumpScript.sCh.sInitStream.ui32BufferFullAborts,
+				g_PDumpScript.sCh.sMainStream.ui32BufferFullAborts,
+				g_PDumpScript.sCh.sDeinitStream.ui32BufferFullAborts,
+				g_PDumpScript.sCh.sBlockStream.ui32BufferFullAborts));
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.ui32HighestRetriesWatermark (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+				g_PDumpScript.sCh.sInitStream.ui32HighestRetriesWatermark,
+				g_PDumpScript.sCh.sMainStream.ui32HighestRetriesWatermark,
+				g_PDumpScript.sCh.sDeinitStream.ui32HighestRetriesWatermark,
+			    g_PDumpScript.sCh.sBlockStream.ui32HighestRetriesWatermark));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.ui32FileIdx( %d )", g_PDumpScript.ui32FileIdx));
+
+
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.hTL (In, Mn, De, Bk) ( %p, %p, %p, %p )",
+			g_PDumpParameters.sCh.sInitStream.hTL, g_PDumpParameters.sCh.sMainStream.hTL, g_PDumpParameters.sCh.sDeinitStream.hTL, g_PDumpParameters.sCh.sBlockStream.hTL));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.ui32BufferFullRetries (In, Mn, De, Bk) ( %5d, %5d, %5d, %5d )",
+			g_PDumpParameters.sCh.sInitStream.ui32BufferFullRetries,
+			g_PDumpParameters.sCh.sMainStream.ui32BufferFullRetries,
+			g_PDumpParameters.sCh.sDeinitStream.ui32BufferFullRetries,
+			g_PDumpParameters.sCh.sBlockStream.ui32BufferFullRetries));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.ui32BufferFullAborts (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+			g_PDumpParameters.sCh.sInitStream.ui32BufferFullAborts,
+			g_PDumpParameters.sCh.sMainStream.ui32BufferFullAborts,
+			g_PDumpParameters.sCh.sDeinitStream.ui32BufferFullAborts,
+			g_PDumpParameters.sCh.sBlockStream.ui32BufferFullAborts));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.ui32HighestRetriesWatermark (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+				g_PDumpParameters.sCh.sInitStream.ui32HighestRetriesWatermark,
+				g_PDumpParameters.sCh.sMainStream.ui32HighestRetriesWatermark,
+				g_PDumpParameters.sCh.sDeinitStream.ui32HighestRetriesWatermark,
+				g_PDumpParameters.sCh.sBlockStream.ui32HighestRetriesWatermark));
+
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sWOff.* (In, Mn, De, Bk) ( %d, %d, %d, %d )",
+			g_PDumpParameters.sWOff.ui32Init, g_PDumpParameters.sWOff.ui32Main, g_PDumpParameters.sWOff.ui32Deinit, g_PDumpParameters.sWOff.ui32Block));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.ui32FileIdx( %d )", g_PDumpParameters.ui32FileIdx));
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpCtrl( %p ) eServiceState( %d ), IsDriverInInitPhase( %s ) ui32Flags( %x )",
+			&g_PDumpCtrl, g_PDumpCtrl.eServiceState, CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE) ? "yes" : "no", g_PDumpCtrl.ui32Flags));
+	PVR_LOG(("--- PDUMP COMMON: ui32DefaultCapMode( %d ) ui32CurrentFrame( %d )",
+			g_PDumpCtrl.ui32DefaultCapMode, g_PDumpCtrl.ui32CurrentFrame));
+	PVR_LOG(("--- PDUMP COMMON: sCaptureRange.ui32Start( %x ) sCaptureRange.ui32End( %x ) sCaptureRange.ui32Interval( %u )",
+			g_PDumpCtrl.sCaptureRange.ui32Start, g_PDumpCtrl.sCaptureRange.ui32End, g_PDumpCtrl.sCaptureRange.ui32Interval));
+	PVR_LOG(("--- PDUMP COMMON: IsInCaptureRange( %s ) IsInCaptureInterval( %s ) InPowerTransition( %d )",
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE) ? "yes" : "no",
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL) ? "yes" : "no",
+			PDumpCtrlInPowerTransition()));
+	PVR_LOG(("--- PDUMP COMMON: sBlockCtrl.ui32BlockLength( %d ), sBlockCtrl.ui32CurrentBlock( %d )",
+			g_PDumpCtrl.sBlockCtrl.ui32BlockLength, g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock));
+}
+#endif /* defined(PDUMP_TRACE_STATE) || defined(PVR_TESTING_UTILS) */
+
+static void PDumpStartTimer(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *hSyncPrivData,
+                                     PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                                     PDUMP_CONNECTION_DATA **ppsPDumpConnectionData)
+{
+	PDUMP_CONNECTION_DATA *psPDumpConnectionData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(ppsPDumpConnectionData != NULL);
+	PVR_ASSERT(pfnPDumpSyncBlocks != NULL);
+	PVR_ASSERT(hSyncPrivData != NULL);
+
+	psPDumpConnectionData = OSAllocMem(sizeof(*psPDumpConnectionData));
+	PVR_GOTO_IF_NOMEM(psPDumpConnectionData, eError, fail_alloc);
+
+	eError = OSLockCreate(&psPDumpConnectionData->hLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lockcreate);
+
+	dllist_init(&psPDumpConnectionData->sListHead);
+	OSAtomicWrite(&psPDumpConnectionData->sRefCount, 1);
+	psPDumpConnectionData->ui32LastSetFrameNumber = PDUMP_FRAME_UNSET;
+	psPDumpConnectionData->eLastEvent = PDUMP_TRANSITION_EVENT_NONE;
+	psPDumpConnectionData->eFailedEvent = PDUMP_TRANSITION_EVENT_NONE;
+
+	/*
+	 * Although we don't take a ref count here, handle base destruction
+	 * will ensure that any resource that might trigger us to do a Transition
+	 * will have been freed before the sync blocks which are keeping the sync
+	 * connection data alive.
+	 */
+	psPDumpConnectionData->hSyncPrivData = hSyncPrivData;
+	psPDumpConnectionData->pfnPDumpSyncBlocks = pfnPDumpSyncBlocks;
+
+	*ppsPDumpConnectionData = psPDumpConnectionData;
+
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		IMG_INT iRefCount;
+
+		/* Add this new reference to the global count of active connections */
+		iRefCount = OSAtomicIncrement(&gPDumpNumConnex);
+		PDUMP_REFCOUNT_PRINT("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                     &gPDumpNumConnex, iRefCount);
+
+		if (((iRefCount > 1) &&
+			(g_PDumpCtrl.ui32TimeoutFrequency != 0U)) &&
+			(g_PDumpTimerID == NULL))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: Starting Timeout Chain now, refcnt = %d",
+			         __func__, iRefCount));
+			PDumpStartTimer(psDeviceNode);
+		}
+	}
+
+	return PVRSRV_OK;
+
+fail_lockcreate:
+	OSFreeMem(psPDumpConnectionData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	IMG_INT iRefCount;
+
+	_PDumpConnectionRelease(psPDumpConnectionData);
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		/* Remove this connection from the global count */
+		iRefCount = OSAtomicDecrement(&gPDumpNumConnex);
+		PDUMP_REFCOUNT_PRINT("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                     &gPDumpNumConnex, iRefCount);
+		PVR_ASSERT(iRefCount >= 0);
+	}
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+
+/*!
+ * \name	PDumpSNPrintf
+ */
+PVRSRV_ERROR PDumpSNPrintf(IMG_HANDLE hBuf, IMG_UINT32 ui32ScriptSizeMax, IMG_CHAR* pszFormat, ...)
+{
+	IMG_CHAR* pszBuf = hBuf;
+	IMG_INT32 n;
+	va_list	vaArgs;
+
+	va_start(vaArgs, pszFormat);
+
+	n = OSVSNPrintf(pszBuf, ui32ScriptSizeMax, pszFormat, vaArgs);
+
+	va_end(vaArgs);
+
+	if (n>=(IMG_INT32)ui32ScriptSizeMax || n==-1)	/* glibc >= 2.1 or glibc 2.0 */
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Buffer overflow detected, pdump output may be incomplete."));
+
+		return PVRSRV_ERROR_PDUMP_BUF_OVERFLOW;
+	}
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+	OSAtomicIncrement(&g_sEveryLineCounter);
+#endif
+
+	/* Put line ending sequence at the end if it isn't already there */
+	_PDumpVerifyLineEnding(pszBuf, ui32ScriptSizeMax);
+
+	return PVRSRV_OK;
+}
+
+void PDumpTimerCB(void *pvData);
+void PDumpTimerCB(void *pvData)
+{
+	IMG_INT iRefCount;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Already flagged as TERMINATED",
+		         __func__));
+	}
+
+	/* Simply check to see if the global connection count indicates that all
+	 * subsequent applications requiring PDump logging have terminated.
+	 * In a quiescent state we will have a singleton pdump utility still
+	 * connected.
+	 */
+	if ((iRefCount = OSAtomicRead(&gPDumpNumConnex)) == 1)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: No connections active (%d), flagging as AUTO_TERMINATED",
+		         __func__, iRefCount));
+		SET_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED);
+	}
+}
+
+/*
+ * FunctionName  PDumpStartTimer
+ * Description   Start an OSTimer chain running to scan for active connections
+ *               being present in the connections associated with the given
+ *               psDeviceNode. Only started if we have AutoTerminate flagged
+ *               in the internal PDump state.
+ * Inputs        psDeviceNode   associated device node to scan for connections
+ * Returns       nothing
+ */
+static void PDumpStartTimer(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: DeviceID %u not valid", __func__,
+		         psDeviceNode->sDevId.ui32InternalID));
+		return;
+	}
+
+	if (g_PDumpCtrl.ui32TimeoutFrequency == 0U)
+	{
+		return;
+	}
+
+	if (g_PDumpTimerID != NULL)
+	{
+		eError = OSDisableTimer(g_PDumpTimerID);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSDisableTimer");
+		eError = OSRemoveTimer(g_PDumpTimerID);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSRemoveTimer");
+	}
+
+	g_PDumpTimerID = OSAddTimer(PDumpTimerCB, NULL, g_PDumpCtrl.ui32TimeoutFrequency * 1000U);
+
+	if (g_PDumpTimerID != NULL)
+	{
+		eError = OSEnableTimer(g_PDumpTimerID);
+
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEnableTimer");
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Timer %p now active.", __func__,
+		         g_PDumpTimerID));
+	}
+}
+
+#endif /* defined(PDUMP) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physheap.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physheap.c
new file mode 100644
index 000000000000..c5c603024b54
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physheap.c
@@ -0,0 +1,2331 @@
+/*************************************************************************/ /*!
+@File           physheap.c
+@Title          Physical heap management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Management functions for the physical heap(s). A heap contains
+                all the information required by services when using memory from
+                that heap (such as CPU <> Device physical address translation).
+                A system must register one heap but can have more then one which
+                is why a heap must register with a (system) unique ID.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "physheap.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "physmem.h"
+#include "physmem_hostmem.h"
+#include "physmem_lma.h"
+#include "physmem_dlm.h"
+#include "physmem_ima.h"
+#include "physmem_osmem.h"
+#include "debug_common.h"
+
+struct _PHYS_HEAP_
+{
+	/*! The type of this heap */
+	PHYS_HEAP_TYPE			eType;
+
+	/*! The allocation policy for this heap */
+	PHYS_HEAP_POLICY uiPolicy;
+
+	/* Config flags */
+	PHYS_HEAP_USAGE_FLAGS		ui32UsageFlags;
+
+	/* OOM Detection state */
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	ATOMIC_T					sOOMDetected;
+#endif
+
+	/*! Pointer to device node struct */
+	PPVRSRV_DEVICE_NODE         psDevNode;
+	/*! PDump name of this physical memory heap */
+	IMG_CHAR					*pszPDumpMemspaceName;
+	/*! Physheap name of this physical memory heap */
+	IMG_CHAR					aszName[PHYS_HEAP_NAME_SIZE];
+	/*! Private data for the translate routines */
+	IMG_HANDLE					hPrivData;
+	/*! Function callbacks */
+	PHYS_HEAP_FUNCTIONS			*psMemFuncs;
+
+	/*! Refcount */
+	IMG_UINT32					ui32RefCount;
+
+	/*! Implementation specific */
+	PHEAP_IMPL_DATA             pvImplData;
+	PHEAP_IMPL_FUNCS            *psImplFuncs;
+
+	/*! Pointer to next physical heap */
+	struct _PHYS_HEAP_		*psNext;
+
+#if defined(SUPPORT_STATIC_IPA)
+	/*! IPA Policy value from Heap Config */
+	IMG_UINT32                  ui32IPAPolicyValue;
+
+	/*! IPA Clear Mask value from Heap Config */
+	IMG_UINT32                  ui32IPAClearMask;
+
+	/*! IPA Bit Shift value from Heap Config */
+	IMG_UINT32                  ui32IPAShift;
+#endif /* defined(SUPPORT_STATIC_IPA) */
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	/* Pointer to the Spas region the physheap is part of
+	 * or NULL. */
+	PHYS_HEAP_SPAS_REGION      *psSpasRegion;
+	/* The Spas regions list of physheaps.
+	 * Each physheap is only part of at most 1 Spas region. */
+	DLLIST_NODE                 sSpasSibling;
+#endif
+};
+
+#if defined(REFCOUNT_DEBUG)
+#define PHYSHEAP_REFCOUNT_PRINT(fmt, ...)	\
+	PVRSRVDebugPrintf(PVR_DBG_WARNING,	\
+			  __FILE__,		\
+			  __LINE__,		\
+			  fmt,			\
+			  __VA_ARGS__)
+#else
+#define PHYSHEAP_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+#define IsOOMError(err) ((err == PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES) | \
+						 (err == PVRSRV_ERROR_OUT_OF_MEMORY) | \
+						 (err == PVRSRV_ERROR_PMR_TOO_LARGE))
+
+typedef enum _PVR_LAYER_HEAP_ACTION_
+{
+	PVR_LAYER_HEAP_ACTION_IGNORE,          /* skip heap during heap init */
+	PVR_LAYER_HEAP_ACTION_INSTANTIATE,     /* instantiate heap but don't acquire */
+	PVR_LAYER_HEAP_ACTION_INITIALISE       /* instantiate and acquire */
+
+} PVR_LAYER_HEAP_ACTION;
+
+typedef struct PHYS_HEAP_PROPERTIES_TAG
+{
+	PVRSRV_PHYS_HEAP eFallbackHeap;
+	PVR_LAYER_HEAP_ACTION ePVRLayerAction;
+	IMG_BOOL bUserModeAlloc;
+} PHYS_HEAP_PROPERTIES;
+
+/* NOTE: Table entries and order must match enum PVRSRV_PHYS_HEAP to ensure
+ * correct operation of PhysHeapCreatePMR().
+ */
+static PHYS_HEAP_PROPERTIES gasHeapProperties[PVRSRV_PHYS_HEAP_LAST] =
+{
+	/* eFallbackHeap,               ePVRLayerAction,                   bUserModeAlloc */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* DEFAULT */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* CPU_LOCAL */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_LOCAL */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_PRIVATE */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_MAIN */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_FALSE }, /* EXTERNAL */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_FALSE }, /* GPU_COHERENT */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_SECURE */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_CONFIG */
+    {  PVRSRV_PHYS_HEAP_FW_MAIN,    PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_CODE */
+    {  PVRSRV_PHYS_HEAP_FW_MAIN,    PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PRIV_DATA */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP_PT */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP0, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP0 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP1, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP1 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP2, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP2 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP3, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP3 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP4, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP4 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP5, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP5 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP6, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP6 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP7, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP7 */
+    {  PVRSRV_PHYS_HEAP_WRAP,       PVR_LAYER_HEAP_ACTION_INSTANTIATE, IMG_FALSE }, /* WRAP */
+    {  PVRSRV_PHYS_HEAP_DISPLAY,    PVR_LAYER_HEAP_ACTION_INSTANTIATE, IMG_FALSE }, /* DISPLAY */
+};
+
+static_assert((ARRAY_SIZE(gasHeapProperties) == PVRSRV_PHYS_HEAP_LAST),
+	"Size or order of gasHeapProperties entries incorrect for PVRSRV_PHYS_HEAP enum");
+
+static IMG_BOOL PhysHeapCreatedByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+static IMG_BOOL PhysHeapAcquiredByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+
+/**
+ * ! IMPORTANT !
+ * Do not change this string array unless the usage flag definitions in
+ * physheap_config.h have changed.
+ *
+ * NOTE: Use DebugCommonFlagStrings or GetPhysHeapUsageString to get
+ * usage flags string.
+ */
+static const IMG_FLAGS2DESC g_asPhysHeapUsageFlagStrings[] =
+{
+	{PHYS_HEAP_USAGE_CPU_LOCAL,		"CPU_LOCAL"},
+	{PHYS_HEAP_USAGE_GPU_LOCAL,		"GPU_LOCAL"},
+	{PHYS_HEAP_USAGE_GPU_PRIVATE,	"GPU_PRIVATE"},
+	{PHYS_HEAP_USAGE_EXTERNAL,		"EXTERNAL"},
+	{PHYS_HEAP_USAGE_GPU_COHERENT,	"GPU_COHERENT"},
+	{PHYS_HEAP_USAGE_GPU_SECURE,	"GPU_SECURE"},
+	{PHYS_HEAP_USAGE_FW_SHARED,		"FW_SHARED"},
+	{PHYS_HEAP_USAGE_FW_PRIVATE,	"FW_PRIVATE"},
+	{PHYS_HEAP_USAGE_FW_CODE,		"FW_CODE"},
+	{PHYS_HEAP_USAGE_FW_PRIV_DATA,	"FW_PRIV_DATA"},
+	{PHYS_HEAP_USAGE_FW_PREMAP_PT,	"FW_PREMAP_PT"},
+	{PHYS_HEAP_USAGE_FW_PREMAP,		"FW_PREMAP"},
+	{PHYS_HEAP_USAGE_WRAP,			"WRAP"},
+	{PHYS_HEAP_USAGE_DISPLAY,		"DISPLAY"},
+	{PHYS_HEAP_USAGE_DLM,			"DLM"}
+};
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCheckValidUsageFlags
+@Description    Checks if any bits were set outside of the valid ones within
+                PHYS_HEAP_USAGE_FLAGS.
+
+@Input          ui32PhysHeapUsage  The value of the usage flag.
+
+@Return         True or False depending on whether there were only valid bits set.
+*/ /**************************************************************************/
+static inline IMG_BOOL PhysHeapCheckValidUsageFlags(PHYS_HEAP_USAGE_FLAGS ui32PhysHeapUsage)
+{
+	return !(ui32PhysHeapUsage & ~PHYS_HEAP_USAGE_MASK);
+}
+
+/*************************************************************************/ /*!
+@Function       GetPhysHeapUsageString
+@Description    This function is used to create a comma separated string of all
+                usage flags passed in as a bitfield.
+
+@Input          ui32UsageFlags     The bitfield of usage flags.
+@Input          ui32Size           The size of the memory pointed to by
+                                   pszUsageString.
+@Output         pszUsageString     A pointer to memory where the created string
+                                   will be stored.
+
+@Return         If successful PVRSRV_OK, else a PVRSRV_ERROR.
+*/ /**************************************************************************/
+static PVRSRV_ERROR GetPhysHeapUsageString(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags,
+                                          IMG_UINT32 ui32Size,
+                                          IMG_CHAR *const pszUsageString)
+{
+	IMG_UINT32 i;
+	IMG_BOOL bFirst = IMG_TRUE;
+	size_t uiSize = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszUsageString != NULL, "pszUsageString");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui32Size > 0, "ui32Size");
+
+	/* Initialise the string to be null terminated at the beginning */
+	pszUsageString[0] = '\0';
+
+	if (ui32UsageFlags == 0)
+	{
+		const ssize_t iCopiedCnt = OSStringSafeCopy(pszUsageString, "NONE", (size_t)ui32Size);
+		PVR_LOG_RETURN_IF_FALSE((iCopiedCnt >= 0), "OSStringSafeCopy", PVRSRV_ERROR_OUT_OF_MEMORY);
+
+		return PVRSRV_OK;
+	}
+
+	/* Process from left to right. */
+	for (i = (sizeof(PHYS_HEAP_USAGE_FLAGS) * BITS_PER_BYTE - 1); i > 0; i--)
+	{
+		IMG_UINT32 ui32Flag = BIT(i);
+
+		if (BITMASK_HAS(ui32UsageFlags, ui32Flag))
+		{
+			IMG_CHAR pszString[32] = "\0";
+
+			if (PhysHeapCheckValidUsageFlags(ui32Flag))
+			{
+				DebugCommonFlagStrings(pszString,
+				                   sizeof(pszString),
+				                   g_asPhysHeapUsageFlagStrings,
+				                   ARRAY_SIZE(g_asPhysHeapUsageFlagStrings),
+				                   ui32Flag);
+			}
+			else
+			{
+				uiSize = OSStringLCat(pszString,
+				                      "INVALID",
+				                      sizeof(pszString));
+				PVR_LOG_RETURN_IF_FALSE((uiSize < sizeof(pszString)), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+			}
+
+			if (!bFirst)
+			{
+				uiSize = OSStringLCat(pszUsageString,
+				                      ", ",
+				                      (size_t)ui32Size);
+				PVR_LOG_RETURN_IF_FALSE((uiSize < ui32Size), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+			}
+			else
+			{
+				bFirst = IMG_FALSE;
+			}
+
+			uiSize = OSStringLCat(pszUsageString,
+			                      pszString,
+			                      (size_t)ui32Size);
+			PVR_LOG_RETURN_IF_FALSE((uiSize < ui32Size), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapPrintHeapProperties
+@Description    This function is used to print properties
+                of the specified physheap.
+
+@Input          psPhysHeap          The physheap to create the string from.
+@Input          pfnDumpDebugPrintf  The specified print function that should be
+                                    used to dump any debug information
+                                    (see PVRSRVDebugRequest).
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+
+@Return         If successful PVRSRV_OK, else a PVRSRV_ERROR.
+*/ /**************************************************************************/
+static PVRSRV_ERROR PhysHeapPrintHeapProperties(PHYS_HEAP *psPhysHeap,
+                                                IMG_BOOL bDefaultHeap,
+                                                DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                                void *pvDumpDebugFile)
+{
+	static const IMG_CHAR *const pszTypeStrings[] = {
+	#define X(_name) #_name,
+		PHYS_HEAP_TYPE_LIST
+	#undef X
+	};
+
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	IMG_CHAR pszUsageString[127] = "\0";
+	PVRSRV_ERROR eError;
+
+	if (psPhysHeap->eType >= ARRAY_SIZE(pszTypeStrings))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PhysHeap at address %p eType is not a PHYS_HEAP_TYPE",
+		         psPhysHeap));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAPINFO, failure);
+	}
+
+	psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+	                                               &ui64TotalSize,
+	                                               &ui64FreeSize);
+
+	eError = GetPhysHeapUsageString(psPhysHeap->ui32UsageFlags,
+	                                sizeof(pszUsageString),
+	                                pszUsageString);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GetPhysHeapUsageString", failure);
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC ", Spas Region: %p"
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap) (psPhysHeap->psSpasRegion)
+#else
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC "%s"
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap) ""
+#endif
+
+	if ((psPhysHeap->eType == PHYS_HEAP_TYPE_LMA) ||
+	    (psPhysHeap->eType == PHYS_HEAP_TYPE_DLM))
+	{
+		IMG_CPU_PHYADDR sCPUPAddr;
+		IMG_DEV_PHYADDR sGPUPAddr;
+
+		PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetCPUPAddr != NULL);
+		PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetDevPAddr != NULL);
+
+		eError = psPhysHeap->psImplFuncs->pfnGetCPUPAddr(psPhysHeap->pvImplData,
+		                                                 &sCPUPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "pfnGetCPUPAddr");
+			sCPUPAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(IMG_UINT64_MAX);
+		}
+
+		eError = psPhysHeap->psImplFuncs->pfnGetDevPAddr(psPhysHeap->pvImplData,
+		                                                 &sGPUPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "pfnGetDevPAddr");
+			sGPUPAddr.uiAddr = IMG_UINT64_MAX;
+		}
+
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "CPU PA Base: " CPUPHYADDR_UINT_FMTSPEC", "
+		                  "GPU PA Base: 0x%08"IMG_UINT64_FMTSPECx", "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B"
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC,
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  CPUPHYADDR_FMTARG(sCPUPAddr.uiAddr),
+		                  sGPUPAddr.uiAddr,
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize,
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap));
+	}
+	else if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+	{
+		IMG_CHAR pszSpanString[128] = "\0";
+		void *pvIterHandle = NULL;
+
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B Spans:",
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize);
+
+		while (psPhysHeap->psImplFuncs->pfnGetHeapSpansStringIter(psPhysHeap->pvImplData,
+		                                                                   pszSpanString,
+		                                                                   sizeof(pszSpanString),
+		                                                                   &pvIterHandle))
+		{
+			PVR_DUMPDEBUG_LOG("%s", pszSpanString);
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B"
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC,
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize,
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap));
+	}
+
+#undef PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC
+#undef PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG
+
+	return PVRSRV_OK;
+
+failure:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDebugRequest
+@Description    This function is used to output debug information for a given
+                device's PhysHeaps.
+@Input          pfnDbgRequestHandle Data required by this function that is
+                                    passed through the RegisterDeviceDbgRequestNotify
+                                    function.
+@Input          ui32VerbLevel       The maximum verbosity of the debug request.
+@Input          pfnDumpDebugPrintf  The specified print function that should be
+                                    used to dump any debug information
+                                    (see PVRSRVDebugRequest).
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+@Return         void
+*/ /**************************************************************************/
+static void PhysHeapDebugRequest(PVRSRV_DBGREQ_HANDLE pfnDbgRequestHandle,
+                                  IMG_UINT32 ui32VerbLevel,
+                                  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                  void *pvDumpDebugFile)
+{
+	PPVRSRV_DEVICE_NODE psDeviceNode = (PPVRSRV_DEVICE_NODE)pfnDbgRequestHandle;
+	PHYS_HEAP *psPhysHeap;
+
+	PVR_UNREFERENCED_PARAMETER(ui32VerbLevel);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDeviceNode != NULL,
+	                             "Phys Heap debug request failed. psDeviceNode was NULL");
+
+	PVR_DUMPDEBUG_LOG("------[ Device ID: %d - Phys Heaps ]------",
+	                  psDeviceNode->sDevId.i32KernelDeviceID);
+
+	for (psPhysHeap = psDeviceNode->psPhysHeapList; psPhysHeap != NULL; psPhysHeap = psPhysHeap->psNext)
+	{
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_BOOL bDefaultHeap = psPhysHeap == psDeviceNode->apsPhysHeap[psDeviceNode->psDevConfig->eDefaultHeap];
+
+		eError = PhysHeapPrintHeapProperties(psPhysHeap,
+		                                     bDefaultHeap,
+		                                     pfnDumpDebugPrintf,
+		                                     pvDumpDebugFile);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "PhysHeapCreateProperties");
+			continue;
+		}
+	}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	OSLockAcquire(psDeviceNode->hPMRZombieListLock);
+	PVR_DUMPDEBUG_LOG("PMR Zombie Count: %u, PMR Zombie Count In Cleanup: %u",
+	                  psDeviceNode->uiPMRZombieCount,
+	                  psDeviceNode->uiPMRZombieCountInCleanup);
+	OSLockRelease(psDeviceNode->hPMRZombieListLock);
+#endif
+	PVR_DUMPDEBUG_LOG("PMR Live Count: %d", PMRGetLiveCount());
+}
+
+/*************************************************************************/ /*!
+@Function       HeapCfgUsedByPVRLayer
+@Description    Checks if a physheap config must be handled by the PVR Layer
+@Input          psConfig PhysHeapConfig
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+static IMG_BOOL HeapCfgUsedByPVRLayer(PHYS_HEAP_CONFIG *psConfig)
+{
+	PVRSRV_PHYS_HEAP eHeap;
+	IMG_BOOL bPVRHeap = IMG_FALSE;
+
+	/* Heaps are triaged for initialisation by either
+	 * the PVR Layer or the device-specific heap handler. */
+	for (eHeap = PVRSRV_PHYS_HEAP_DEFAULT;
+		 eHeap < PVRSRV_PHYS_HEAP_LAST;
+		 eHeap++)
+	{
+		if ((BIT_ISSET(psConfig->ui32UsageFlags, eHeap) &&
+		     PhysHeapCreatedByPVRLayer(eHeap)))
+		{
+			bPVRHeap = IMG_TRUE;
+			break;
+		}
+	}
+
+	return bPVRHeap;
+}
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateDLMIMAHeapsFromConfig
+@Description    Create new heaps for a device from DLM and IMA configs.
+                This function will both create and construct the link
+                between them.
+@Input          psDevNode      Pointer to device node struct
+@Input          pasConfigs     Pointer to array of Heap configurations.
+@Input          ui32NumConfigs Number of configurations in array.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PhysHeapCreateDLMIMAHeapsFromConfig(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PHYS_HEAP_CONFIG *pasConfigs,
+                                    IMG_UINT32 ui32NumConfigs)
+{
+	/* The DLM heaps must be created before IMA heaps and the config order is not well-defined.
+	 * So for each DLM heap create it and then create the IMA heaps associated. */
+
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiDLMIdx, uiIMAIdx;
+	PHYS_HEAP_POLICY uiIMAPolicy = OSIsMapPhysNonContigSupported() ? PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG : PHYS_HEAP_POLICY_DEFAULT;
+	PHYS_HEAP *psDLMHeap;
+
+	/* Iterate and create the DLM heaps */
+	for (uiDLMIdx = 0; uiDLMIdx < ui32NumConfigs; uiDLMIdx++)
+	{
+		if (pasConfigs[uiDLMIdx].eType == PHYS_HEAP_TYPE_DLM)
+		{
+			IMG_UINT32 uiHeapCount = 0;
+			eError = PhysmemCreateHeapDLM(psDevNode,
+			                               PHYS_HEAP_POLICY_DEFAULT,
+			                               &pasConfigs[uiDLMIdx],
+			                               pasConfigs[uiDLMIdx].uConfig.sDLM.pszHeapName,
+			                               &psDLMHeap);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeapDLM");
+
+			/* Then iterate and create the IMA heaps linked to this DLM heap */
+			for (uiIMAIdx = 0; uiIMAIdx < ui32NumConfigs; uiIMAIdx++)
+			{
+				if (pasConfigs[uiIMAIdx].eType == PHYS_HEAP_TYPE_IMA &&
+				    pasConfigs[uiIMAIdx].uConfig.sIMA.uiDLMHeapIdx == uiDLMIdx)
+				{
+					PhysmemCreateHeapIMA(psDevNode,
+					                     uiIMAPolicy,
+					                     &pasConfigs[uiIMAIdx],
+					                     pasConfigs[uiIMAIdx].uConfig.sIMA.pszHeapName,
+					                     psDLMHeap,
+					                     pasConfigs[uiDLMIdx].uConfig.sDLM.ui32Log2PMBSize,
+					                     NULL);
+					PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeapIMA");
+					uiHeapCount++;
+				}
+			}
+
+			if (uiHeapCount == 0)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "DLM phys heap config %d: No connected IMA heaps. Phys heap will go unused.", uiDLMIdx));
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+PhysHeapDestroyDLMIMAHeaps(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PHYS_HEAP *psNode = psDevNode->psPhysHeapList;
+
+	/* We need to ensure IMA heaps are destroyed before DLM so that
+	 * we don't cause RA leaks by freeing the DLM first.
+	 */
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+
+		if (psTmp->eType == PHYS_HEAP_TYPE_IMA)
+		{
+			PhysHeapDestroy(psTmp);
+		}
+	}
+
+	/* Reset the loop */
+	psNode = psDevNode->psPhysHeapList;
+
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+
+		if (psTmp->eType == PHYS_HEAP_TYPE_DLM)
+		{
+			PhysHeapDestroy(psTmp);
+		}
+	}
+}
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateDeviceHeapsFromConfigs
+@Description    Create new heaps for a device from configs.
+@Input          psDevNode      Pointer to device node struct
+@Input          pasConfigs     Pointer to array of Heap configurations.
+@Input          ui32NumConfigs Number of configurations in array.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PhysHeapCreateDeviceHeapsFromConfigs(PPVRSRV_DEVICE_NODE psDevNode,
+                                     PHYS_HEAP_CONFIG *pasConfigs,
+                                     IMG_UINT32 ui32NumConfigs)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	psDevNode->psPhysHeapList = NULL;
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+	/* DLM/IMA heaps must be constructed in a specific order */
+	eError = PhysHeapCreateDLMIMAHeapsFromConfig(psDevNode,
+	                                             pasConfigs,
+	                                             ui32NumConfigs);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateDLMIMAHeapsFromConfig");
+#endif
+
+
+	for (i = 0; i < ui32NumConfigs; i++)
+	{
+		/* A PhysHeapConfig can have multiple usage flags. If any flag in a
+		 * heap's set points to a heap type that is handled by the PVR Layer
+		 * then we assume that a single heap is shared between multiple
+		 * allocators and it is safe to instantiate it here. If the heap
+		 * is not marked to be initialised by the PVR Layer, leave it
+		 * to the device specific handler.
+		 * DLM Heaps have usage flags but don't have element in the
+		 * fallback table, this check will prevent instantiate them twice.
+		 */
+		if (HeapCfgUsedByPVRLayer(&pasConfigs[i]))
+		{
+			eError = PhysHeapCreateHeapFromConfig(psDevNode, &pasConfigs[i], NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+		}
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST)
+	/* For a temporary device node there will never be a debug dump
+	 * request targeting it */
+	if (psDevNode->hDebugTable != NULL)
+#endif
+	{
+		eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hPhysHeapDbgReqNotify,
+		                                              psDevNode,
+		                                              PhysHeapDebugRequest,
+		                                              DEBUG_REQUEST_SYS,
+		                                              psDevNode);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify");
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PhysHeapCreateHeapFromConfig(PVRSRV_DEVICE_NODE *psDevNode,
+							 PHYS_HEAP_CONFIG *psConfig,
+							 PHYS_HEAP **ppsPhysHeap)
+{
+	PVRSRV_ERROR eResult;
+
+	if (psConfig->eType == PHYS_HEAP_TYPE_UMA
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+		|| psConfig->eType == PHYS_HEAP_TYPE_WRAP
+#endif
+		)
+	{
+		eResult = PhysmemCreateHeapOSMEM(psDevNode,
+		                                 PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG,
+		                                 psConfig,
+		                                 ppsPhysHeap);
+	}
+	else if ((psConfig->eType == PHYS_HEAP_TYPE_LMA) ||
+	         (psConfig->eType == PHYS_HEAP_TYPE_DMA))
+	{
+		PHYS_HEAP_POLICY uiHeapPolicy;
+
+		if (psDevNode->pfnPhysHeapGetLMAPolicy != NULL)
+		{
+			uiHeapPolicy = psDevNode->pfnPhysHeapGetLMAPolicy(psConfig->ui32UsageFlags, psDevNode);
+		}
+		else
+		{
+			uiHeapPolicy = OSIsMapPhysNonContigSupported() ?
+			               PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG :
+			               PHYS_HEAP_POLICY_DEFAULT;
+		}
+
+		eResult = PhysmemCreateHeapLMA(psDevNode,
+		                               uiHeapPolicy,
+		                               psConfig,
+		                               (psConfig->eType == PHYS_HEAP_TYPE_LMA) ?
+		                                "GPU LMA (Sys)" :
+		                                "GPU LMA DMA (Sys)",
+		                               ppsPhysHeap);
+	}
+	else if ((psConfig->eType == PHYS_HEAP_TYPE_DLM) ||
+	         (psConfig->eType == PHYS_HEAP_TYPE_IMA))
+	{
+		/* These heaps have already been instantiated in
+		 * PhysHeapCreateDLMIMAHeapsFromConfig
+		 */
+		eResult = PVRSRV_OK;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s Invalid phys heap type: %d",
+				 __func__, psConfig->eType));
+		eResult = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eResult;
+}
+
+#define PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE (IMG_UINT64_C(0x100000) * IMG_UINT64_C(32)) /* 32MB */
+
+static PVRSRV_ERROR PVRSRVValidatePhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT32 ui32FlagsAccumulate = 0;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_FALSE(psDevConfig->ui32PhysHeapCount > 0,
+							"Device config must specify at least one phys heap config.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		/* Flags that may be used by multiple heaps. */
+		const PHYS_HEAP_USAGE_FLAGS uiDuplicateFlags = PHYS_HEAP_USAGE_DLM;
+		PHYS_HEAP_CONFIG *psHeapConf = &psDevConfig->pasPhysHeaps[i];
+
+		PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->ui32UsageFlags != 0,
+		                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+		                           "Phys heap config %d: must specify usage flags.", i);
+
+		PVR_LOG_RETURN_IF_FALSE_VA(((ui32FlagsAccumulate & ~uiDuplicateFlags) & psHeapConf->ui32UsageFlags) == 0,
+		                             PVRSRV_ERROR_PHYSHEAP_CONFIG,
+		                             "Phys heap config %d: duplicate usage flags.", i);
+
+		ui32FlagsAccumulate |= psHeapConf->ui32UsageFlags;
+
+		if (BITMASK_ANY((1U << psDevConfig->eDefaultHeap), PHYS_HEAP_USAGE_MASK) &&
+		    BITMASK_ANY((1U << psDevConfig->eDefaultHeap), psHeapConf->ui32UsageFlags))
+		{
+			switch (psHeapConf->eType)
+			{
+#if defined(__KERNEL__)
+				case PHYS_HEAP_TYPE_DMA:
+#endif
+				case PHYS_HEAP_TYPE_LMA:
+				{
+					if (psHeapConf->uConfig.sLMA.uiSize < PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE)
+					{
+						/* Output message if default heap is LMA and smaller than recommended minimum. */
+						PVR_DPF((PVR_DBG_ERROR, "%s: Size of default heap is 0x%" IMG_UINT64_FMTSPECX
+						         " (recommended minimum heap size is 0x%" IMG_UINT64_FMTSPECX ")",
+						         __func__, psHeapConf->uConfig.sLMA.uiSize,
+						         PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE));
+					}
+					break;
+				}
+				case PHYS_HEAP_TYPE_IMA:
+				{
+					/* Check if chained DLM has more than default LMA size.
+					 * This is not perfect as other connected IMAs could reserve all the memory. */
+					IMG_UINT64 uiSize = psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].uConfig.sDLM.uiSize;
+
+					if (uiSize < PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Default heap has access to 0x%" IMG_UINT64_FMTSPECX
+						         " (recommended minimum heap size is 0x%" IMG_UINT64_FMTSPECX ")",
+						         __func__, uiSize,
+						         PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE));
+					}
+					break;
+				}
+				case PHYS_HEAP_TYPE_DLM:
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Phys heap config %d: "
+					         "A DLM heap cannot be the default heap.",
+					         __func__, i));
+					return PVRSRV_ERROR_PHYSHEAP_CONFIG;
+				}
+				default:
+					break;
+			}
+		}
+
+#if !defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+		if ((psHeapConf->eType == PHYS_HEAP_TYPE_IMA) || (psHeapConf->eType == PHYS_HEAP_TYPE_DLM))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Phys heap config %d: "
+			         "Dynamic phys heaps are not supported! "
+			         "Please enable PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS",
+			         __func__, i));
+			return PVRSRV_ERROR_PHYSHEAP_CONFIG;
+		}
+#else /* PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS */
+		if (psHeapConf->eType == PHYS_HEAP_TYPE_DLM)
+		{
+			PVR_LOG_RETURN_IF_FALSE_VA((psHeapConf->ui32UsageFlags & PHYS_HEAP_USAGE_DLM) == PHYS_HEAP_USAGE_DLM,
+			                            PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                            "Phys heap config %d: DLM heap must specify the DLM usage flag.", i);
+
+			/* If a DLM heap has less space than a PMB, it cannot export a single PMB. The PMB size should be lowered. */
+			PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->uConfig.sDLM.uiSize >= psHeapConf->uConfig.sDLM.ui32Log2PMBSize,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: Size of DLM heap is 0x%" IMG_UINT64_FMTSPECX
+			                           " but the PMB size is 0x%" IMG_UINT64_FMTSPECX
+			                           ". The total size must be greater than or equal to the PMB size.",
+			                           i, psHeapConf->uConfig.sDLM.uiSize,
+			                           IMG_UINT64_C(1) << psHeapConf->uConfig.sDLM.ui32Log2PMBSize);
+		}
+
+		/* IMA heaps must point to a DLM heap. */
+		if (psHeapConf->eType == PHYS_HEAP_TYPE_IMA)
+		{
+			PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->uConfig.sIMA.uiDLMHeapIdx < psDevConfig->ui32PhysHeapCount,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: IMA heap is trying to link to a DLM heap out of bounds. "
+			                           "Requested Heap Index: %d, Heap Array Count: %d",
+			                           i, psHeapConf->uConfig.sIMA.uiDLMHeapIdx, psDevConfig->ui32PhysHeapCount);
+			PVR_LOG_RETURN_IF_FALSE_VA(psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].eType == PHYS_HEAP_TYPE_DLM,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: IMA heap is trying to link to a NON-DLM heap type. "
+			                           "Requested Heap Idx: %d, Heap Type: %d",
+			                           i, psHeapConf->uConfig.sIMA.uiDLMHeapIdx,
+			                           psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].eType);
+		}
+#endif /* PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS */
+
+	}
+
+
+	if (psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_GPU_LOCAL)
+	{
+		PVR_LOG_RETURN_IF_FALSE(((ui32FlagsAccumulate & PHYS_HEAP_USAGE_GPU_LOCAL) != 0) ,
+							"Device config must specify GPU local phys heap config.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+	else if (psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_CPU_LOCAL)
+	{
+		PVR_LOG_RETURN_IF_FALSE(((ui32FlagsAccumulate & PHYS_HEAP_USAGE_CPU_LOCAL) != 0) ,
+						"Device config must specify CPU local phys heap config.",
+						PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+static void DestroyGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       CreateGpuVirtValArenas
+@Description    Create virtualization validation arenas
+@Input          psDeviceNode The device node
+@Return         PVRSRV_ERROR PVRSRV_OK on success
+*/ /**************************************************************************/
+static PVRSRV_ERROR CreateGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* aui64OSidMin and aui64OSidMax are what we program into HW registers.
+	   The values are different from base/size of arenas. */
+	IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS];
+	IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS];
+	PHYS_HEAP_CONFIG *psGPULocalHeap = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig, PHYS_HEAP_USAGE_GPU_LOCAL);
+	IMG_DEV_PHYADDR sGPULocalCardBase = PhysHeapConfigGetCardBase(psGPULocalHeap);
+	IMG_UINT64 uiGPULocalSize = PhysHeapConfigGetSize(psGPULocalHeap);
+	PHYS_HEAP_CONFIG *psDisplayHeap = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig, PHYS_HEAP_USAGE_DISPLAY);
+
+	IMG_UINT64 uPrivateRABase;
+	IMG_UINT64 uPrivateRASize;
+	IMG_UINT64 uSharedRABase;
+	IMG_UINT64 uSharedRASize;
+	IMG_UINT64 uSharedRegionBase;
+	IMG_UINT64 uSharedRegionSize;
+	IMG_UINT32 i;
+
+	//PVR_LOG_RETURN_IF_FALSE((psDeviceNode->psOSSharedArena == NULL), "Validation RAs already created.", PVRSRV_OK);
+
+	/* Shared region is fixed size, the remaining space is divided amongst OSes */
+	uPrivateRASize = uiGPULocalSize - PVR_ALIGN(GPUVIRT_SIZEOF_SHARED, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+	uPrivateRASize /= GPUVIRT_VALIDATION_NUM_OS;
+	uPrivateRASize = uPrivateRASize & ~((IMG_UINT64)OSGetPageSize() - 1ULL); /* Align, round down */
+	uSharedRASize = uiGPULocalSize - uPrivateRASize * GPUVIRT_VALIDATION_NUM_OS;
+
+	PVR_LOG(("GPUVIRT_VALIDATION: GPU_LOCAL heap base (base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ")"
+			 " split into %u private regions of size 0x%" IMG_UINT64_FMTSPECX " and one shared region of size 0x%" IMG_UINT64_FMTSPECX " ",
+			 sGPULocalCardBase.uiAddr, uiGPULocalSize, GPUVIRT_VALIDATION_NUM_OS, uPrivateRASize, uSharedRASize));
+
+	/* If a display heap config exists, include the display heap in the non-secure regions */
+	if (psDisplayHeap)
+	{
+		IMG_DEV_PHYADDR sGPUDisplayCardBase = PhysHeapConfigGetCardBase(psDisplayHeap);
+		IMG_UINT64 uiGPUDisplaySize = PhysHeapConfigGetSize(psDisplayHeap);
+
+		PVR_LOG(("GPUVIRT_VALIDATION: DISPLAY heap (base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ") merged into shared region.",
+		         sGPUDisplayCardBase.uiAddr,
+		         uiGPUDisplaySize));
+
+		if (sGPUDisplayCardBase.uiAddr > sGPULocalCardBase.uiAddr)
+		{
+			/*
+			 * ---------------------------------------------------------------------------------------------------
+			 * |        .        .        .     GPU LOCAL HEAP       .        .        .        |  DISPLAY HEAP  |
+			 * ---------------------------------------------------------------------------------------------------
+			 *  \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \_______________/
+			 *   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7   SHARED      DISPLAY
+			 *                                                                            \____SHARED REGION____/
+			 *  OSID 0 has full access over everything
+			 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+			 */
+			if ((sGPULocalCardBase.uiAddr + uiGPULocalSize) != sGPUDisplayCardBase.uiAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: GPU Local heap and display heap must be adjacent in physical memory.", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			uPrivateRABase = sGPULocalCardBase.uiAddr;
+			uSharedRABase = sGPULocalCardBase.uiAddr + uiGPULocalSize - uSharedRASize;
+			uSharedRegionBase = uSharedRABase;
+		}
+		else
+		{
+			/*
+			 * ---------------------------------------------------------------------------------------------------
+			 * |  DISPLAY HEAP   |        .        .        .     GPU LOCAL HEAP       .        .        .       |
+			 * ---------------------------------------------------------------------------------------------------
+			 *  \_______________/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/
+			 *        DISPLAY      SHARED   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7
+			 *   \____SHARED REGION____/
+			 *
+			 *  OSID 0 has full access over everything
+			 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+			 */
+			if ((sGPUDisplayCardBase.uiAddr + uiGPUDisplaySize) != sGPULocalCardBase.uiAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: GPU Local heap and display heap must be adjacent in physical memory.", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			uSharedRABase = sGPULocalCardBase.uiAddr;
+			uPrivateRABase = sGPULocalCardBase.uiAddr + uSharedRASize;
+			uSharedRegionBase = sGPUDisplayCardBase.uiAddr;
+		}
+
+		uSharedRegionSize = uSharedRASize + uiGPUDisplaySize;
+	}
+	else
+	{
+		/*
+		 * ----------------------------------------------------------------------------------
+		 * |        .        .        .     GPU LOCAL HEAP       .        .        .        |
+		 * ----------------------------------------------------------------------------------
+		 *  \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/
+		 *   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7   SHARED
+		 *
+		 *  OSID 0 has full access over everything
+		 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+		 */
+		uPrivateRABase = sGPULocalCardBase.uiAddr;
+		uSharedRABase = uPrivateRABase + uPrivateRASize * GPUVIRT_VALIDATION_NUM_OS;
+		uSharedRASize = uiGPULocalSize - (uSharedRABase - uPrivateRABase);
+		uSharedRegionSize = uSharedRASize;
+		uSharedRegionBase = uSharedRABase;
+	}
+
+	if (uPrivateRASize < GPUVIRT_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Per-OSID private memory regions are too small (current: 0x%" IMG_UINT64_FMTSPECX ""
+								" required: 0x%" IMG_UINT64_FMTSPECX "). Increase GPU Local heap size.",
+								__func__, uPrivateRASize, GPUVIRT_MIN_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uSharedRASize < GPUVIRT_SIZEOF_SHARED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Memory region shared between all OSIDs is too small (current: 0x%" IMG_UINT64_FMTSPECX ""
+								" required: 0x%" IMG_UINT64_FMTSPECX "). Increase GPU Local heap size.",
+								__func__, uSharedRASize, GPUVIRT_SIZEOF_SHARED));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	FOREACH_VALIDATION_OSID(i)
+	{
+		IMG_CHAR aszOSRAName[RA_MAX_NAME_LENGTH];
+
+		PVR_LOG(("GPUVIRT_VALIDATION: create arena Private OSID: %u, base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ".", i, uPrivateRABase, uPrivateRASize));
+
+		OSSNPrintf(aszOSRAName, RA_MAX_NAME_LENGTH, "GPUVIRT_OS%u", i);
+
+		psDeviceNode->psOSidSubArena[i] = RA_Create_With_Span(aszOSRAName,
+		                                                      OSGetPageShift(),
+		                                                      0,
+		                                                      uPrivateRABase,
+		                                                      uPrivateRASize,
+		                                                      RA_POLICY_DEFAULT);
+		PVR_GOTO_IF_FALSE(psDeviceNode->psOSidSubArena[i] != NULL, FreeArenas);
+		aui64OSidMin[GPUVIRT_VAL_REGION_SECURE][i] = uPrivateRABase;
+
+		if (i == 0)
+		{
+			/* Host OSID0 has access to all regions */
+			aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i] = uiGPULocalSize - 1ULL;
+		}
+		else
+		{
+			/* Guest OSIDs are limited to their private range */
+			aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i] = uPrivateRABase + uPrivateRASize - 1ULL;
+		}
+
+		/* uSharedRegionSize includes display heap */
+		aui64OSidMin[GPUVIRT_VAL_REGION_SHARED][i] = uSharedRegionBase;
+		aui64OSidMax[GPUVIRT_VAL_REGION_SHARED][i] = uSharedRegionBase + uSharedRegionSize - 1ULL;
+
+		uPrivateRABase += uPrivateRASize;
+	}
+
+	PVR_LOG(("GPUVIRT_VALIDATION: create arena Shared, base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ".", uSharedRABase, uSharedRASize));
+
+	/* uSharedRASize does not include display heap */
+	psDeviceNode->psOSSharedArena = RA_Create_With_Span("GPUVIRT_SHARED",
+	                                                    OSGetPageShift(),
+	                                                    0,
+	                                                    uSharedRABase,
+	                                                    uSharedRASize,
+	                                                    RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_FALSE(psDeviceNode->psOSSharedArena != NULL, FreeArenas);
+
+	FOREACH_VALIDATION_OSID(i)
+	{
+		PVR_LOG(("GPUVIRT_VALIDATION: HW regions OSID %u: Private Region = [0x%" IMG_UINT64_FMTSPECX "..0x%" IMG_UINT64_FMTSPECX "]"
+														" Shared Region = [0x%" IMG_UINT64_FMTSPECX "..0x%" IMG_UINT64_FMTSPECX "]",
+				 i,
+				 aui64OSidMin[GPUVIRT_VAL_REGION_SECURE][i],
+				 aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i],
+				 aui64OSidMin[GPUVIRT_VAL_REGION_SHARED][i],
+				 aui64OSidMax[GPUVIRT_VAL_REGION_SHARED][i]));
+	}
+
+	if (psDeviceNode->psDevConfig->pfnSysInitFirewall != NULL)
+	{
+		psDeviceNode->psDevConfig->pfnSysInitFirewall(psDeviceNode->psDevConfig->hSysData, aui64OSidMin, aui64OSidMax);
+	}
+
+	return PVRSRV_OK;
+
+FreeArenas:
+	DestroyGpuVirtValArenas(psDeviceNode);
+
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+}
+
+/*
+ * Counter-part to CreateGpuVirtValArenas.
+ */
+static void DestroyGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32	uiCounter = 0;
+
+	FOREACH_VALIDATION_OSID(uiCounter)
+	{
+		if (uiCounter == RGXFW_HOST_DRIVER_ID)
+		{
+			/*
+			 * NOTE: We overload psOSidSubArena[0] into the psLocalMemArena so we must
+			 * not free it here as it gets cleared later.
+			 */
+			continue;
+		}
+
+		if (psDeviceNode->psOSidSubArena[uiCounter] == NULL)
+		{
+			continue;
+		}
+		RA_Delete(psDeviceNode->psOSidSubArena[uiCounter]);
+	}
+
+	if (psDeviceNode->psOSSharedArena != NULL)
+	{
+		RA_Delete(psDeviceNode->psOSSharedArena);
+	}
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       PhysHeapMMUPxSetup
+@Description    Setup MMU Px allocation function pointers.
+@Input          psDeviceNode Pointer to device node struct
+@Return         PVRSRV_ERROR PVRSRV_OK on success.
+*/ /**************************************************************************/
+static PVRSRV_ERROR PhysHeapMMUPxSetup(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PHYS_HEAP_TYPE eHeapType;
+	PVRSRV_ERROR eError;
+
+	eError = PhysHeapAcquireByID(psDeviceNode->psDevConfig->eDefaultHeap,
+	                                      psDeviceNode, &psDeviceNode->psMMUPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorDeinit);
+
+	eHeapType = PhysHeapGetType(psDeviceNode->psMMUPhysHeap);
+
+	if (eHeapType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: GPU physical heap uses OS System memory (UMA)", __func__));
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		PVR_DPF((PVR_DBG_ERROR, "%s: Virtualisation Validation builds are currently only"
+								 " supported on systems with local memory (LMA).", __func__));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto ErrorDeinit;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: GPU physical heap uses local memory managed by the driver (LMA)", __func__));
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		eError = CreateGpuVirtValArenas(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "CreateGpuVirtValArenas", ErrorDeinit);
+#endif
+	}
+
+	return PVRSRV_OK;
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapMMUPxDeInit
+@Description    Deinit after PhysHeapMMUPxSetup.
+@Input          psDeviceNode Pointer to device node struct
+*/ /**************************************************************************/
+static void PhysHeapMMUPxDeInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/* Remove local LMA subarenas */
+	DestroyGpuVirtValArenas(psDeviceNode);
+#endif	/* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+	if (psDeviceNode->psMMUPhysHeap != NULL)
+	{
+		PhysHeapRelease(psDeviceNode->psMMUPhysHeap);
+		psDeviceNode->psMMUPhysHeap = NULL;
+	}
+}
+
+PVRSRV_ERROR PhysHeapInitDeviceHeaps(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP ePhysHeap;
+
+	eError = OSLockCreate(&psDeviceNode->hPhysHeapLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	eError = PVRSRVValidatePhysHeapConfig(psDevConfig);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVValidatePhysHeapConfig");
+
+	eError = PhysHeapCreateDeviceHeapsFromConfigs(psDeviceNode,
+	                                              psDevConfig->pasPhysHeaps,
+	                                              psDevConfig->ui32PhysHeapCount);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreateDeviceHeapsFromConfigs", ErrorDeinit);
+
+	/* Must loop from the 2nd heap to the last */
+	PVR_ASSERT(PVRSRV_PHYS_HEAP_DEFAULT == 0);
+	for (ePhysHeap = (PVRSRV_PHYS_HEAP)(PVRSRV_PHYS_HEAP_DEFAULT+1); ePhysHeap < PVRSRV_PHYS_HEAP_LAST; ePhysHeap++)
+	{
+		if (PhysHeapAcquiredByPVRLayer(ePhysHeap))
+		{
+			eError = PhysHeapAcquireByID(ePhysHeap, psDeviceNode, &psDeviceNode->apsPhysHeap[ePhysHeap]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorDeinit);
+		}
+	}
+
+	if (PhysHeapValidateDefaultHeapExists(psDeviceNode))
+	{
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapValidateDefaultHeapExists", ErrorDeinit);
+	}
+
+	eError = PhysHeapMMUPxSetup(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapMMUPxSetup", ErrorDeinit);
+
+	return PVRSRV_OK;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+
+	return eError;
+}
+
+void PhysHeapDeInitDeviceHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_PHYS_HEAP ePhysHeapIdx;
+	IMG_UINT32 i;
+
+	PhysHeapMMUPxDeInit(psDeviceNode);
+
+	/* Release heaps */
+	for (ePhysHeapIdx = PVRSRV_PHYS_HEAP_DEFAULT;
+		 ePhysHeapIdx < ARRAY_SIZE(psDeviceNode->apsPhysHeap);
+		 ePhysHeapIdx++)
+	{
+		if (psDeviceNode->apsPhysHeap[ePhysHeapIdx])
+		{
+			PhysHeapRelease(psDeviceNode->apsPhysHeap[ePhysHeapIdx]);
+		}
+	}
+
+	FOREACH_SUPPORTED_DRIVER(i)
+	{
+		if (psDeviceNode->apsFWPremapPhysHeap[i])
+		{
+			PhysHeapDestroy(psDeviceNode->apsFWPremapPhysHeap[i]);
+			psDeviceNode->apsFWPremapPhysHeap[i] = NULL;
+		}
+	}
+
+	PhysHeapDestroyDeviceHeaps(psDeviceNode);
+
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+}
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+static void PhysHeapSpasRegionLock(PHYS_HEAP_SPAS_REGION *psSpasRegion)
+{
+	while (OSAtomicCompareExchange(&psSpasRegion->ui32Lock, 0, 1) != 0);
+	OSMemoryBarrier(NULL);
+}
+static void PhysHeapSpasRegionUnlock(PHYS_HEAP_SPAS_REGION *psSpasRegion)
+{
+	OSMemoryBarrier(NULL);
+	OSAtomicWrite(&psSpasRegion->ui32Lock, 0);
+}
+
+/* Insert the physheap into a given SPAS. */
+static void PhysHeapSpasInsert(PHYS_HEAP* psPhysHeap, PHYS_HEAP_SPAS_REGION *psSpasRegion)
+{
+	if (psSpasRegion == NULL)
+		return; /* Not in a SPAS region. */
+	/* A physheap may only be a part of a single SPAS region. */
+	PVR_ASSERT(psPhysHeap->psSpasRegion == NULL);
+
+	psPhysHeap->psSpasRegion = psSpasRegion;
+
+	PhysHeapSpasRegionLock(psSpasRegion);
+	dllist_add_to_head(&psSpasRegion->sListHead, &psPhysHeap->sSpasSibling);
+	PhysHeapSpasRegionUnlock(psSpasRegion);
+}
+
+/* Remove the PhysHeap from its SPAS region if it is in one. */
+static void PhysHeapSpasRemove(PHYS_HEAP *psPhysHeap)
+{
+	PHYS_HEAP_SPAS_REGION *psSpasRegion = psPhysHeap->psSpasRegion;
+
+	if (psSpasRegion == NULL)
+		return; /* Not in a region. */
+
+	PhysHeapSpasRegionLock(psSpasRegion);
+	psPhysHeap->psSpasRegion = NULL;
+	dllist_remove_node(&psPhysHeap->sSpasSibling);
+	PhysHeapSpasRegionUnlock(psSpasRegion);
+
+	psPhysHeap->psSpasRegion = NULL;
+}
+
+PVRSRV_ERROR PhysHeapSpasWithDevice(PHYS_HEAP* psFromPhysHeap,
+                                    PPVRSRV_DEVICE_NODE psToDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_FOUND;
+	PDLLIST_NODE psNode, psNextNode;
+
+	if (psFromPhysHeap->psSpasRegion == NULL)
+		return eError;
+
+	PhysHeapSpasRegionLock(psFromPhysHeap->psSpasRegion);
+	dllist_foreach_node(&psFromPhysHeap->psSpasRegion->sListHead, psNode, psNextNode)
+	{
+		PHYS_HEAP *psPhysHeapCursor = IMG_CONTAINER_OF(psNode, PHYS_HEAP, sSpasSibling);
+		if (psPhysHeapCursor->psDevNode == psToDevNode)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_OK, exit_unlock_);
+		}
+	}
+exit_unlock_:
+	PhysHeapSpasRegionUnlock(psFromPhysHeap->psSpasRegion);
+	return eError;
+}
+
+static IMG_UINT64 PhysHeapSpasDeviceBitmap(PHYS_HEAP* psPhysHeap)
+{
+	PDLLIST_NODE psNode, psNextNode;
+	IMG_UINT64 uiDevices = 0;
+
+	static_assert((sizeof(IMG_UINT64) * 8) >= PVRSRV_MAX_DEVICES, "PhysHeapSpasDeviceBitmap() requires updating");
+
+	if (psPhysHeap->psSpasRegion == NULL)
+		return uiDevices;
+
+	PhysHeapSpasRegionLock(psPhysHeap->psSpasRegion);
+	dllist_foreach_node(&psPhysHeap->psSpasRegion->sListHead, psNode, psNextNode)
+	{
+		PHYS_HEAP *psPhysHeapCursor = IMG_CONTAINER_OF(psNode, PHYS_HEAP, sSpasSibling);
+		uiDevices |= IMG_UINT64_C(1) << psPhysHeapCursor->psDevNode->sDevId.ui32InternalID;
+	}
+	PhysHeapSpasRegionUnlock(psPhysHeap->psSpasRegion);
+	return uiDevices;
+}
+#endif
+
+PVRSRV_ERROR PhysHeapCreate(PPVRSRV_DEVICE_NODE psDevNode,
+							  PHYS_HEAP_CONFIG *psConfig,
+							  PHYS_HEAP_POLICY uiPolicy,
+							  PHEAP_IMPL_DATA pvImplData,
+							  PHEAP_IMPL_FUNCS *psImplFuncs,
+							  PHYS_HEAP **ppsPhysHeap)
+{
+	PHYS_HEAP *psNew;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	if (psConfig->eType == PHYS_HEAP_TYPE_UNKNOWN)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psImplFuncs != NULL, "psImplFuncs");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psImplFuncs->pfnCreatePMR != NULL ||
+	                                psImplFuncs->pfnCreatePMB,
+	                                "psImplFuncs->pfnCreatePMR || psImplFuncs->pfnCreatePMB");
+
+	psNew = OSAllocMem(sizeof(PHYS_HEAP));
+	PVR_RETURN_IF_NOMEM(psNew);
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	psNew->psSpasRegion = NULL;
+#endif
+
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		psNew->psMemFuncs = psConfig->uConfig.sLMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sLMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sLMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sLMA.pszHeapName) ? psConfig->uConfig.sLMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+#if defined(PVRSRV_ENABLE_XD_MEM)
+		PhysHeapSpasInsert(psNew, psConfig->uConfig.sLMA.psSpasRegion);
+#endif
+		break;
+	case PHYS_HEAP_TYPE_IMA:
+		psNew->psMemFuncs = psConfig->uConfig.sIMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sIMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sIMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sIMA.pszHeapName) ? psConfig->uConfig.sIMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_DMA:
+		psNew->psMemFuncs = psConfig->uConfig.sDMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sDMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sDMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sDMA.pszHeapName) ? psConfig->uConfig.sDMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_UMA:
+		psNew->psMemFuncs = psConfig->uConfig.sUMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sUMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sUMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sUMA.pszHeapName) ? psConfig->uConfig.sUMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+#if defined(PVRSRV_ENABLE_XD_MEM)
+		PhysHeapSpasInsert(psNew, psConfig->uConfig.sUMA.psSpasRegion);
+#endif
+		break;
+	case PHYS_HEAP_TYPE_DLM:
+		psNew->psMemFuncs = psConfig->uConfig.sDLM.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sDLM.hPrivData;
+		psNew->pszPDumpMemspaceName = "None";
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sDLM.pszHeapName) ? psConfig->uConfig.sDLM.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+	case PHYS_HEAP_TYPE_WRAP:
+		psNew->psMemFuncs = psConfig->uConfig.sWRAP.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sWRAP.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sWRAP.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sWRAP.pszHeapName) ? psConfig->uConfig.sWRAP.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+#endif
+	default:
+		PVR_LOG_ERROR(PVRSRV_ERROR_NOT_IMPLEMENTED, "psConfig->eType not implemented");
+	}
+
+	psNew->eType = psConfig->eType;
+	psNew->ui32UsageFlags = psConfig->ui32UsageFlags;
+	psNew->uiPolicy = uiPolicy;
+	psNew->ui32RefCount = 0;
+	psNew->psDevNode = psDevNode;
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	OSAtomicWrite(&psNew->sOOMDetected, IMG_FALSE);
+#endif
+	psNew->pvImplData = pvImplData;
+	psNew->psImplFuncs = psImplFuncs;
+
+#if defined(SUPPORT_STATIC_IPA)
+	{
+		IMG_UINT8  ui8Val;
+
+		/* Ensure we do not cause an address fault by accessing beyond
+		 * the end of the psConfig->sIPAConfig structure.
+		 */
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyDefault;
+		psNew->ui32IPAPolicyValue = (IMG_UINT32)ui8Val;
+
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyMask;
+		psNew->ui32IPAClearMask = (IMG_UINT32)ui8Val;
+
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyShift;
+		psNew->ui32IPAShift = (IMG_UINT32)ui8Val;
+		PVR_LOG_VA(PVR_DBG_MESSAGE, "%s: Physheap <%p> ['%s'] Config @ <%p> IPA = [0x%x, 0x%x, 0x%x]",
+		           __func__, psNew, psNew->aszName,
+		           psConfig, psNew->ui32IPAPolicyValue,
+		           psNew->ui32IPAClearMask, psNew->ui32IPAShift);
+	}
+#endif
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psNew;
+	}
+
+	psNew->psNext = psDevNode->psPhysHeapList;
+	psDevNode->psPhysHeapList = psNew;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNew);
+}
+
+void PhysHeapDestroyDeviceHeaps(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psNode;
+
+	if (psDevNode->hPhysHeapDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hPhysHeapDbgReqNotify);
+	}
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+	/* DLM/IMA heaps must be destroyed in a specific order */
+	PhysHeapDestroyDLMIMAHeaps(psDevNode);
+#endif
+
+	psNode = psDevNode->psPhysHeapList;
+
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+		PhysHeapDestroy(psTmp);
+	}
+}
+
+void PhysHeapDestroy(PHYS_HEAP *psPhysHeap)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PPVRSRV_DEVICE_NODE psDevNode = psPhysHeap->psDevNode;
+
+	PVR_DPF_ENTERED1(psPhysHeap);
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK)
+#endif
+	{
+		PVR_ASSERT(psPhysHeap->ui32RefCount == 0);
+	}
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	PhysHeapSpasRemove(psPhysHeap);
+#endif
+
+	if (psDevNode->psPhysHeapList == psPhysHeap)
+	{
+		psDevNode->psPhysHeapList = psPhysHeap->psNext;
+	}
+	else
+	{
+		PHYS_HEAP *psTmp = psDevNode->psPhysHeapList;
+
+		while (psTmp->psNext != psPhysHeap)
+		{
+			psTmp = psTmp->psNext;
+		}
+		psTmp->psNext = psPhysHeap->psNext;
+	}
+
+	if (psImplFuncs->pfnDestroyData != NULL)
+	{
+		psImplFuncs->pfnDestroyData(psPhysHeap->pvImplData);
+	}
+
+	OSFreeMem(psPhysHeap);
+
+	PVR_DPF_RETURN;
+}
+
+static void _PhysHeapCountUserModeHeaps(PPVRSRV_DEVICE_NODE psDevNode,
+										PHYS_HEAP_USAGE_FLAGS ui32UsageFlags)
+{
+	PVRSRV_PHYS_HEAP eHeap;
+
+	for (eHeap = PVRSRV_PHYS_HEAP_DEFAULT;
+		 eHeap <= PVRSRV_PHYS_HEAP_LAST;
+		 eHeap++)
+	{
+		if (BIT_ISSET(ui32UsageFlags, eHeap) &&
+			PhysHeapUserModeAlloc(eHeap))
+		{
+			psDevNode->ui32UserAllocHeapCount++;
+			break;
+		}
+	}
+}
+
+PVRSRV_ERROR PhysHeapAcquire(PHYS_HEAP *psPhysHeap)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+
+	psPhysHeap->ui32RefCount++;
+
+	/* When acquiring a heap for the 1st time, perform a check and
+	 * calculate the total number of user accessible physical heaps */
+	if (psPhysHeap->ui32RefCount == 1)
+	{
+		_PhysHeapCountUserModeHeaps(psPhysHeap->psDevNode,
+									psPhysHeap->ui32UsageFlags);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PHYS_HEAP * _PhysHeapFindHeapOrFallback(PVRSRV_PHYS_HEAP ePhysHeap,
+								   PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psPhysHeapNode = psDevNode->psPhysHeapList;
+	PVRSRV_PHYS_HEAP eFallback;
+
+	/* Swap the default heap alias for the system's real default PhysHeap */
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	/* Check cache of PhysHeaps to see if it has been resolved before */
+	if (psDevNode->apsPhysHeap[ePhysHeap] != NULL)
+	{
+		return psDevNode->apsPhysHeap[ePhysHeap];
+	}
+
+	/* Cache not ready, carry out search with fallback */
+	while (psPhysHeapNode)
+	{
+		if (BIT_ISSET(psPhysHeapNode->ui32UsageFlags, ePhysHeap))
+		{
+			return psPhysHeapNode;
+		}
+
+		psPhysHeapNode = psPhysHeapNode->psNext;
+	}
+
+	/* Find fallback PhysHeap */
+	eFallback = gasHeapProperties[ePhysHeap].eFallbackHeap;
+	if (ePhysHeap == eFallback)
+	{
+		return NULL;
+	}
+	else
+	{
+		return _PhysHeapFindHeapOrFallback(eFallback, psDevNode);
+	}
+}
+
+/*
+ * Acquire heap, no fallback, no recursion: single loop acquisition
+ */
+static PHYS_HEAP* _PhysHeapFindRealHeapNoFallback(PVRSRV_PHYS_HEAP ePhysHeap,
+                                          PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psPhysHeapNode = psDevNode->psPhysHeapList;
+
+	/* Swap the default heap alias for the system's real default PhysHeap */
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	/* Check cache of PhysHeaps to see if it has been resolved before.
+	 * The physheap must initialised to access its cache. */
+	if (psDevNode->apsPhysHeap[ePhysHeap] != NULL &&
+	    BIT_ISSET(psDevNode->apsPhysHeap[ePhysHeap]->ui32UsageFlags, ePhysHeap))
+	{
+		return psDevNode->apsPhysHeap[ePhysHeap];
+	}
+
+	/* Cache not ready, carry out search for real PhysHeap, no fallback */
+	while (psPhysHeapNode)
+	{
+		if (BIT_ISSET(psPhysHeapNode->ui32UsageFlags, ePhysHeap))
+		{
+			return psPhysHeapNode;
+		}
+
+		psPhysHeapNode = psPhysHeapNode->psNext;
+	}
+	return NULL;
+}
+
+PVRSRV_ERROR PhysHeapAcquireByID(PVRSRV_PHYS_HEAP eDevPhysHeap,
+								 PPVRSRV_DEVICE_NODE psDevNode,
+								 PHYS_HEAP **ppsPhysHeap)
+{
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(eDevPhysHeap < PVRSRV_PHYS_HEAP_LAST, "eDevPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	PVR_DPF_ENTERED1(ui32Flags);
+
+	OSLockAcquire(psDevNode->hPhysHeapLock);
+
+	psPhysHeap = _PhysHeapFindHeapOrFallback(eDevPhysHeap, psDevNode);
+
+	if (psPhysHeap != NULL)
+	{
+		psPhysHeap->ui32RefCount++;
+		PHYSHEAP_REFCOUNT_PRINT("%s: Heap %p, refcount = %d",
+								__func__, psPhysHeap, psPhysHeap->ui32RefCount);
+
+		/* When acquiring a heap for the 1st time, perform a check and
+		 * calculate the total number of user accessible physical heaps */
+		if (psPhysHeap->ui32RefCount == 1)
+		{
+			_PhysHeapCountUserModeHeaps(psDevNode, BIT(eDevPhysHeap));
+		}
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_PHYSHEAP_ID_INVALID;
+	}
+
+	OSLockRelease(psDevNode->hPhysHeapLock);
+
+	*ppsPhysHeap = psPhysHeap;
+	PVR_DPF_RETURN_RC1(eError, *ppsPhysHeap);
+}
+
+void PhysHeapRelease(PHYS_HEAP *psPhysHeap)
+{
+	PVR_DPF_ENTERED1(psPhysHeap);
+
+	OSLockAcquire(psPhysHeap->psDevNode->hPhysHeapLock);
+	psPhysHeap->ui32RefCount--;
+	PHYSHEAP_REFCOUNT_PRINT("%s: Heap %p, refcount = %d",
+							__func__, psPhysHeap, psPhysHeap->ui32RefCount);
+	OSLockRelease(psPhysHeap->psDevNode->hPhysHeapLock);
+
+	PVR_DPF_RETURN;
+}
+
+PHEAP_IMPL_DATA PhysHeapGetImplData(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->pvImplData;
+}
+
+PHYS_HEAP_TYPE PhysHeapGetType(PHYS_HEAP *psPhysHeap)
+{
+	PVR_ASSERT(psPhysHeap->eType != PHYS_HEAP_TYPE_UNKNOWN);
+	return psPhysHeap->eType;
+}
+
+PHYS_HEAP_POLICY PhysHeapGetPolicy(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->uiPolicy;
+}
+
+PHYS_HEAP_USAGE_FLAGS PhysHeapGetFlags(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32UsageFlags;
+}
+
+const IMG_CHAR *PhysHeapName(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->aszName;
+}
+
+IMG_BOOL PhysHeapValidateDefaultHeapExists(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_PHYS_HEAP eDefaultHeap = psDevNode->psDevConfig->eDefaultHeap;
+
+	return ((psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_DEFAULT] != NULL) &&
+			((psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_DEFAULT] ==
+			  psDevNode->apsPhysHeap[eDefaultHeap])));
+}
+
+#if defined(SUPPORT_STATIC_IPA)
+IMG_UINT32 PhysHeapGetIPAValue(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAPolicyValue;
+}
+
+IMG_UINT32 PhysHeapGetIPAMask(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAClearMask;
+}
+
+IMG_UINT32 PhysHeapGetIPAShift(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAShift;
+}
+#endif
+
+/*
+ * This function will set the psDevPAddr to whatever the system layer
+ * has set it for the referenced region.
+ * It will not fail if the psDevPAddr is invalid.
+ */
+PVRSRV_ERROR PhysHeapGetDevPAddr(PHYS_HEAP *psPhysHeap,
+									   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetDevPAddr != NULL)
+	{
+		eResult = psImplFuncs->pfnGetDevPAddr(psPhysHeap->pvImplData,
+											  psDevPAddr);
+	}
+
+	return eResult;
+}
+
+/*
+ * This function will set the psCpuPAddr to whatever the system layer
+ * has set it for the referenced region.
+ * It will not fail if the psCpuPAddr is invalid.
+ */
+PVRSRV_ERROR PhysHeapGetCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetCPUPAddr != NULL)
+	{
+		eResult = psImplFuncs->pfnGetCPUPAddr(psPhysHeap->pvImplData,
+											  psCpuPAddr);
+	}
+
+	return eResult;
+}
+
+PVRSRV_ERROR PhysHeapGetSize(PHYS_HEAP *psPhysHeap,
+								   IMG_UINT64 *puiSize)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetSize != NULL)
+	{
+		eResult = psImplFuncs->pfnGetSize(psPhysHeap->pvImplData,
+										  puiSize);
+	}
+
+	return eResult;
+}
+
+/* Used to add the total and free sizes of an IMA heap and the backing
+ * DLM heap
+ */
+static void PhysHeapIMAGetMemInfo(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 *puiTotalSize,
+                                  IMG_UINT64 *puiFreeSize)
+{
+	PHYS_HEAP *psDLMHeap;
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	IMG_UINT64 ui64DLMTotalSize;
+	IMG_UINT64 ui64DLMFreeSize;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(
+		psPhysHeap->eType == PHYS_HEAP_TYPE_IMA,
+		"Physheap type not IMA");
+
+	/* Obtain the DLM heap chained to this IMA heap */
+	PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetHeapDLMBacking);
+	psPhysHeap->psImplFuncs->pfnGetHeapDLMBacking(psPhysHeap->pvImplData,
+	                                              &psDLMHeap);
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDLMHeap != NULL, "pfnGetHeapDLMBacking");
+
+	/* Obtain the memstats for the current IMA heap */
+	PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetFactoryMemStats);
+	psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+	                                               &ui64TotalSize,
+	                                               &ui64FreeSize);
+
+	/* Obtain the memstats for the chained DLM heap */
+	PVR_ASSERT(psDLMHeap->psImplFuncs->pfnGetFactoryMemStats);
+	psDLMHeap->psImplFuncs->pfnGetFactoryMemStats(psDLMHeap->pvImplData,
+	                                              &ui64DLMTotalSize,
+	                                              &ui64DLMFreeSize);
+
+	/* Total size is just the DLM heap size as the IMA heap imports a
+	 * subset of that total size.
+	 * Free size is a combination of the two because DLM heap may
+	 * have spare PMBs to provide and the IMA heap may have space
+	 * in an imported PMB.
+	 */
+	*puiTotalSize = ui64DLMTotalSize;
+	*puiFreeSize = ui64FreeSize + ui64DLMFreeSize;
+}
+
+PVRSRV_ERROR
+PhysHeapGetMemInfo(PVRSRV_DEVICE_NODE *psDevNode,
+                   IMG_UINT32 ui32PhysHeapCount,
+                   PVRSRV_PHYS_HEAP *paePhysHeapID,
+                   PHYS_HEAP_MEM_STATS_PTR paPhysHeapMemStats)
+{
+	IMG_UINT32 i = 0;
+	PHYS_HEAP *psPhysHeap;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui32PhysHeapCount <= MAX_USER_MODE_ALLOC_PHYS_HEAPS, "ui32PhysHeapCount invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(paePhysHeapID != NULL, "paePhysHeapID invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(paPhysHeapMemStats != NULL, "paPhysHeapMemStats invalid");
+
+	for (i = 0; i < ui32PhysHeapCount; i++)
+	{
+		if (paePhysHeapID[i] >= PVRSRV_PHYS_HEAP_LAST)
+		{
+			return PVRSRV_ERROR_PHYSHEAP_ID_INVALID;
+		}
+
+		psPhysHeap = _PhysHeapFindRealHeapNoFallback(paePhysHeapID[i], psDevNode);
+
+		paPhysHeapMemStats[i].ui32PhysHeapFlags = 0;
+		paPhysHeapMemStats[i].ui64DevicesInSPAS = 0;
+
+		if (psPhysHeap && PhysHeapUserModeAlloc(paePhysHeapID[i])
+				&& psPhysHeap->psImplFuncs->pfnGetFactoryMemStats)
+		{
+			if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+			{
+				PhysHeapIMAGetMemInfo(psPhysHeap,
+				                      &paPhysHeapMemStats[i].ui64TotalSize,
+				                      &paPhysHeapMemStats[i].ui64FreeSize);
+			}
+			else
+			{
+				psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+				                                               &paPhysHeapMemStats[i].ui64TotalSize,
+				                                               &paPhysHeapMemStats[i].ui64FreeSize);
+			}
+
+			if (paePhysHeapID[i] == psDevNode->psDevConfig->eDefaultHeap)
+			{
+				paPhysHeapMemStats[i].ui32PhysHeapFlags |= PVRSRV_PHYS_HEAP_FLAGS_IS_DEFAULT;
+			}
+#if defined(PVRSRV_ENABLE_XD_MEM)
+			paPhysHeapMemStats[i].ui64DevicesInSPAS = PhysHeapSpasDeviceBitmap(psPhysHeap);
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+			paPhysHeapMemStats[i].ePhysHeapType = psPhysHeap->eType;
+		}
+		else
+		{
+			paPhysHeapMemStats[i].ui64TotalSize = 0;
+			paPhysHeapMemStats[i].ui64FreeSize = 0;
+			paPhysHeapMemStats[i].ePhysHeapType = PHYS_HEAP_TYPE_UNKNOWN;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+void PhysHeapCpuPAddrToDevPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	psPhysHeap->psMemFuncs->pfnCpuPAddrToDevPAddr(psPhysHeap->hPrivData,
+												 ui32NumOfAddr,
+												 psDevPAddr,
+												 psCpuPAddr);
+}
+
+void PhysHeapDevPAddrToCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr,
+								IMG_DEV_PHYADDR *psDevPAddr)
+{
+	psPhysHeap->psMemFuncs->pfnDevPAddrToCpuPAddr(psPhysHeap->hPrivData,
+												 ui32NumOfAddr,
+												 psCpuPAddr,
+												 psDevPAddr);
+}
+
+IMG_CHAR *PhysHeapPDumpMemspaceName(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->pszPDumpMemspaceName;
+}
+
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+static inline void _LogOOMDetection(IMG_BOOL isOOMDetected, PHYS_HEAP *psPhysHeap, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	IMG_BOOL bExistingVal = OSAtomicExchange(&psPhysHeap->sOOMDetected, isOOMDetected);
+	PVRSRV_PHYS_HEAP ePhysIdx = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	if (bExistingVal != isOOMDetected)
+	{
+		PVR_LOG(("Device: %d Physheap: %s OOM: %s",
+				(psPhysHeap->psDevNode->sDevId.ui32InternalID),
+				g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel,
+				(isOOMDetected) ? "Detected" : "Resolved"));
+	}
+}
+#endif
+
+PVRSRV_ERROR PhysHeapCreatePMR(PHYS_HEAP *psPhysHeap,
+							   struct _CONNECTION_DATA_ *psConnection,
+							   IMG_DEVMEM_SIZE_T uiSize,
+							   IMG_UINT32 ui32NumPhysChunks,
+							   IMG_UINT32 ui32NumVirtChunks,
+							   IMG_UINT32 *pui32MappingTable,
+							   IMG_UINT32 uiLog2PageSize,
+							   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							   const IMG_CHAR *pszAnnotation,
+							   IMG_PID uiPid,
+							   PMR **ppsPMRPtr,
+							   IMG_UINT32 ui32PDumpFlags,
+							   PVRSRV_MEMALLOCFLAGS_T *puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	IMG_UINT64 uiFreeBytes;
+	PVRSRV_PHYS_HEAP eDemotionPhysIdx;
+	PVRSRV_MEMALLOCFLAGS_T uiDemotionFlags = uiFlags;
+	PVRSRV_PHYS_HEAP ePhysIdx = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+	PHYS_HEAP *psDemotionHeap = NULL;
+#endif
+	eError = psImplFuncs->pfnCreatePMR(psPhysHeap,
+									 psConnection,
+									 uiSize,
+									 ui32NumPhysChunks,
+									 ui32NumVirtChunks,
+									 pui32MappingTable,
+									 uiLog2PageSize,
+									 uiFlags,
+									 pszAnnotation,
+									 uiPid,
+									 ppsPMRPtr,
+									 ui32PDumpFlags);
+
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	/* Check for OOM error, return if otherwise */
+	_LogOOMDetection(((IsOOMError(eError)) ? IMG_TRUE : IMG_FALSE), psPhysHeap, uiFlags);
+	if (eError == PVRSRV_OK)
+	{
+		if (puiOutFlags)
+		{
+			*puiOutFlags = uiFlags;
+		}
+		return eError;
+	}
+	PVR_LOG_RETURN_IF_FALSE((IsOOMError(eError)), "Failed to allocate PMR", eError);
+
+	/* Skip logic and return if mandate flag is set */
+	if (PVRSRV_CHECK_MANDATED_PHYSHEAP(uiFlags))
+	{
+		return eError;
+	}
+
+	/* Demotion only occurs on CPU_LOCAL,GPU_LOCAL,GPU_PRIVATE */
+	if (ePhysIdx > PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+	{
+		return eError;
+	}
+
+	eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	for (eDemotionPhysIdx = (PVRSRV_PHYS_HEAP)(ePhysIdx-1); eDemotionPhysIdx != PVRSRV_PHYS_HEAP_DEFAULT; eDemotionPhysIdx--)
+	{
+		PVRSRV_CHANGE_PHYS_HEAP_HINT(eDemotionPhysIdx, uiDemotionFlags);
+		PVR_LOG_IF_FALSE_VA(PVR_DBG_MESSAGE, (ePhysIdx-eDemotionPhysIdx < 2), "Demoted from %s to CPU_LOCAL. "
+						"Expect Performance to be affected!", g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel);
+		psDemotionHeap = _PhysHeapFindRealHeapNoFallback(eDemotionPhysIdx, psPhysHeap->psDevNode);
+
+		/* Either no alternative available, or allocation already failed on selected heap */
+		if (psDemotionHeap == NULL || psPhysHeap == psDemotionHeap)
+		{
+			continue;
+		}
+
+		if (PhysHeapFreeMemCheck(psDemotionHeap, uiSize, &uiFreeBytes) != PVRSRV_OK)
+		{
+			_LogOOMDetection(IMG_TRUE, psDemotionHeap, uiDemotionFlags);
+			continue;
+		}
+
+		psImplFuncs = psDemotionHeap->psImplFuncs;
+		eError = psImplFuncs->pfnCreatePMR(psDemotionHeap,
+						 psConnection,
+						 uiSize,
+						 ui32NumPhysChunks,
+						 ui32NumVirtChunks,
+						 pui32MappingTable,
+						 uiLog2PageSize,
+						 uiDemotionFlags,
+						 pszAnnotation,
+						 uiPid,
+						 ppsPMRPtr,
+						 ui32PDumpFlags);
+		_LogOOMDetection(((IsOOMError(eError)) ? IMG_TRUE : IMG_FALSE), psDemotionHeap, uiDemotionFlags);
+
+		if (eError == PVRSRV_OK)
+		{
+			if (puiOutFlags)
+			{
+				*puiOutFlags = uiDemotionFlags;
+			}
+			break;
+		}
+	}
+	if (eError == PVRSRV_OK)
+	{
+		/* Success demotion worked error Ok - emit warning. */
+		PVR_LOG_VA(PVR_DBG_WARNING, "PhysHeap(%s) failed to allocate PMR. Demoted to %s" ,
+						g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel,
+						g_asPhysHeapUsageFlagStrings[eDemotionPhysIdx-1].pszLabel);
+	}
+	else
+	{
+		/* Unable to create PMR (Heap not found or CreatePMR failed) - emit Error */
+		PVR_LOG_VA(PVR_DBG_ERROR, "Error raised %s : Unable to %s." ,
+		                          PVRSRVGETERRORSTRING(eError),
+		                          (psDemotionHeap == NULL) ?  "find heaps for demotion" :
+		                                                      "allocate PMR via Demotion heap");
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		{
+			PPVRSRV_DEVICE_NODE psDevNode = PhysHeapDeviceNode(psPhysHeap);
+			OSLockAcquire(psDevNode->hPMRZombieListLock);
+			PVR_LOG_VA(PVR_DBG_ERROR, "PMR Zombie Count: %u, PMR Zombie Count In Cleanup: %u",
+			                          psDevNode->uiPMRZombieCount,
+			                          psDevNode->uiPMRZombieCountInCleanup);
+			OSLockRelease(psDevNode->hPMRZombieListLock);
+		}
+#endif
+	}
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PhysHeapCreatePMB(PHYS_HEAP *psPhysHeap,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               const IMG_CHAR *pszAnnotation,
+                               PMB **ppsPMRPtr,
+                               RA_BASE_T *puiBase,
+                               RA_LENGTH_T *puiSize)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		psPhysHeap->eType == PHYS_HEAP_TYPE_DLM,
+		"Physheap type not DLM",
+		PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT(psImplFuncs->pfnCreatePMB);
+
+	return psImplFuncs->pfnCreatePMB(psPhysHeap,
+	                                 uiSize,
+	                                 pszAnnotation,
+	                                 ppsPMRPtr,
+	                                 puiBase,
+	                                 puiSize);
+}
+
+PPVRSRV_DEVICE_NODE PhysHeapDeviceNode(PHYS_HEAP *psPhysHeap)
+{
+	PVR_ASSERT(psPhysHeap != NULL);
+
+	return psPhysHeap->psDevNode;
+}
+
+static IMG_BOOL PhysHeapCreatedByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return (gasHeapProperties[ePhysHeap].ePVRLayerAction != PVR_LAYER_HEAP_ACTION_IGNORE);
+}
+
+static IMG_BOOL PhysHeapAcquiredByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return (gasHeapProperties[ePhysHeap].ePVRLayerAction == PVR_LAYER_HEAP_ACTION_INITIALISE);
+}
+
+IMG_BOOL PhysHeapUserModeAlloc(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return gasHeapProperties[ePhysHeap].bUserModeAlloc;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+PVRSRV_ERROR PhysHeapPagesAllocGPV(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_UINT32 ui32OSid, IMG_PID uiPid)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesAllocGPV != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesAllocGPV(psPhysHeap,
+		                                        uiSize, psMemHandle, psDevPAddr, ui32OSid, uiPid);
+	}
+
+	return eResult;
+}
+#endif
+
+PVRSRV_ERROR PhysHeapPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+								PG_HANDLE *psMemHandle,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_PID uiPid)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesAlloc != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesAlloc(psPhysHeap,
+		                                       uiSize, psMemHandle, psDevPAddr, uiPid);
+	}
+
+	return eResult;
+}
+
+void PhysHeapPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_ASSERT(psImplFuncs->pfnPagesFree != NULL);
+
+	if (psImplFuncs->pfnPagesFree != NULL)
+	{
+		psImplFuncs->pfnPagesFree(psPhysHeap,
+		                          psMemHandle);
+	}
+}
+
+PVRSRV_ERROR PhysHeapPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle, size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+							  void **pvPtr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesMap != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesMap(psPhysHeap,
+		                                   pshMemHandle, uiSize, psDevPAddr, pvPtr);
+	}
+
+	return eResult;
+}
+
+void PhysHeapPagesUnMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_ASSERT(psImplFuncs->pfnPagesUnMap != NULL);
+
+	if (psImplFuncs->pfnPagesUnMap != NULL)
+	{
+		psImplFuncs->pfnPagesUnMap(psPhysHeap,
+		                           psMemHandle, pvPtr);
+	}
+}
+
+PVRSRV_ERROR PhysHeapPagesClean(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle,
+							  IMG_UINT32 uiOffset,
+							  IMG_UINT32 uiLength)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesClean != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesClean(psPhysHeap,
+		                                     pshMemHandle, uiOffset, uiLength);
+	}
+
+	return eResult;
+}
+
+IMG_UINT32 PhysHeapGetPageShift(PHYS_HEAP *psPhysHeap)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	IMG_UINT32 ui32PageShift = 0;
+
+	PVR_ASSERT(psImplFuncs->pfnGetPageShift != NULL);
+
+	if (psImplFuncs->pfnGetPageShift != NULL)
+	{
+		ui32PageShift = psImplFuncs->pfnGetPageShift();
+	}
+
+	return ui32PageShift;
+}
+
+PVRSRV_ERROR PhysHeapFreeMemCheck(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 ui64MinRequiredMem,
+                                  IMG_UINT64 *pui64FreeMem)
+{
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui64FreeMem != NULL, "pui64FreeMem");
+
+
+	if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+	{
+		PhysHeapIMAGetMemInfo(psPhysHeap->pvImplData,
+		                            &ui64TotalSize,
+		                            &ui64FreeSize);
+	}
+	else
+	{
+		psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+		                                               &ui64TotalSize,
+		                                               &ui64FreeSize);
+	}
+
+	*pui64FreeMem = ui64FreeSize;
+	if (ui64MinRequiredMem >= *pui64FreeMem)
+	{
+		eError = PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (physheap.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem.c
new file mode 100644
index 000000000000..36dfff83c0e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem.c
@@ -0,0 +1,866 @@
+/*************************************************************************/ /*!
+@File           physmem.c
+@Title          Physmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common entry point for creation of RAM backed PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "device.h"
+#include "physmem.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "pdump_physmem.h"
+#include "pdump_km.h"
+#include "pvr_ricommon.h"
+#include "allocmem.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#if defined(DEBUG)
+static IMG_UINT32 PMRAllocFail;
+
+#if defined(__linux__)
+#include <linux/moduleparam.h>
+
+module_param(PMRAllocFail, uint, 0644);
+MODULE_PARM_DESC(PMRAllocFail, "When number of PMR allocs reaches "
+				 "this value, it will fail (default value is 0 which "
+				 "means that alloc function will behave normally).");
+#endif /* defined(__linux__) */
+#endif /* defined(DEBUG) */
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#include "proc_stats.h"
+#endif
+
+/** Computes division using log2 of divisor. */
+#define LOG2_DIV(x, log2) ((x) >> (log2))
+
+/** Computes modulo of a power of 2. */
+#define LOG2_MOD(x, log2) ((x) & ((1 << (log2)) - 1))
+
+PVRSRV_ERROR DevPhysMemAlloc(PVRSRV_DEVICE_NODE	*psDevNode,
+                             IMG_UINT32 ui32MemSize,
+                             IMG_UINT32 ui32Log2Align,
+                             const IMG_UINT8 u8Value,
+                             IMG_BOOL bInitPage,
+#if defined(PDUMP)
+                             const IMG_CHAR *pszDevSpace,
+                             const IMG_CHAR *pszSymbolicAddress,
+                             IMG_HANDLE *phHandlePtr,
+#endif
+                             IMG_PID uiPid,
+                             IMG_HANDLE hMemHandle,
+                             IMG_DEV_PHYADDR *psDevPhysAddr)
+{
+	void *pvCpuVAddr;
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	IMG_CHAR szFilenameOut[PDUMP_PARAM_MAX_FILE_NAME];
+	PDUMP_FILEOFFSET_T uiOffsetOut;
+	IMG_UINT32 ui32PageSize;
+	IMG_UINT32 ui32PDumpMemSize = ui32MemSize;
+	PVRSRV_ERROR ePDumpError;
+#endif
+	PG_HANDLE *psMemHandle;
+	IMG_UINT64 uiMask;
+	IMG_DEV_PHYADDR sDevPhysAddr_int;
+
+	psMemHandle = hMemHandle;
+
+	/* Allocate the pages */
+	eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+	                            TRUNCATE_64BITS_TO_SIZE_T(ui32MemSize),
+	                            psMemHandle,
+	                            &sDevPhysAddr_int,
+	                            uiPid);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnDevPxAlloc:1");
+
+	/* Check to see if the page allocator returned pages with our desired
+	 * alignment, which is not unlikely
+	 */
+	uiMask = IMG_PAGE2BYTES32(ui32Log2Align) - 1;
+	if (ui32Log2Align && (sDevPhysAddr_int.uiAddr & uiMask))
+	{
+		/* use over allocation instead */
+		PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+
+		ui32MemSize += (IMG_UINT32) uiMask;
+		eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+		                            TRUNCATE_64BITS_TO_SIZE_T(ui32MemSize),
+		                            psMemHandle,
+		                            &sDevPhysAddr_int,
+		                            uiPid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "pfnDevPxAlloc:2");
+
+		sDevPhysAddr_int.uiAddr += uiMask;
+		sDevPhysAddr_int.uiAddr &= ~uiMask;
+	}
+	*psDevPhysAddr = sDevPhysAddr_int;
+
+#if defined(PDUMP)
+	ui32PageSize = ui32Log2Align? IMG_PAGE2BYTES32(ui32Log2Align) : OSGetPageSize();
+	eError = PDumpMalloc(psDevNode,
+	                     pszDevSpace,
+	                     pszSymbolicAddress,
+	                     ui32PDumpMemSize,
+	                     ui32PageSize,
+	                     IMG_FALSE,
+	                     0,
+	                     phHandlePtr,
+	                     PDUMP_NONE);
+	if (PVRSRV_OK != eError)
+	{
+		PDUMPCOMMENT(psDevNode, "Allocating pages failed");
+		*phHandlePtr = NULL;
+	}
+	ePDumpError = eError;
+#endif
+
+	if (bInitPage)
+	{
+		/*Map the page to the CPU VA space */
+		eError = PhysHeapPagesMap(psDevNode->psMMUPhysHeap,
+		                          psMemHandle,
+		                          ui32MemSize,
+		                          &sDevPhysAddr_int,
+		                          &pvCpuVAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_LOG_ERROR(eError, "DevPxMap");
+			PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+			return eError;
+		}
+
+		/*Fill the memory with given content */
+		OSDeviceMemSet(pvCpuVAddr, u8Value, ui32MemSize);
+		#if (MAP_UNCACHED_MEM)
+			arch_sync_dma_for_device(page_to_phys((struct page*)psMemHandle->u.pvHandle), PAGE_ALIGN(ui32MemSize), DMA_TO_DEVICE);
+		#endif
+		/*Map the page to the CPU VA space */
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            psMemHandle,
+		                            0,
+		                            ui32MemSize);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_LOG_ERROR(eError, "DevPxClean");
+			PhysHeapPagesUnMap(psDevNode->psMMUPhysHeap, psMemHandle, pvCpuVAddr);
+			PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+			return eError;
+		}
+
+#if defined(PDUMP)
+		if (ePDumpError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+		{
+			/* PDumping of the page contents can be done in two ways
+			 * 1. Store the single byte init value to the .prm file
+			 *    and load the same value to the entire dummy page buffer
+			 *    This method requires lot of LDB's inserted into the out2.txt
+			 *
+			 * 2. Store the entire contents of the buffer to the .prm file
+			 *    and load them back.
+			 *    This only needs a single LDB instruction in the .prm file
+			 *    and chosen this method
+			 *    size of .prm file might go up but that's not huge at least
+			 *    for this allocation
+			 */
+			/* Write the buffer contents to the prm file */
+			eError = PDumpWriteParameterBlob(psDevNode,
+											 pvCpuVAddr,
+											 ui32PDumpMemSize,
+											 PDUMP_FLAGS_CONTINUOUS,
+											 szFilenameOut,
+											 sizeof(szFilenameOut),
+											 &uiOffsetOut);
+			if (PVRSRV_OK == eError)
+			{
+				/* Load the buffer back to the allocated memory when playing the pdump */
+				eError = PDumpPMRLDB(psDevNode,
+									 pszDevSpace,
+									 pszSymbolicAddress,
+									 0,
+									 ui32PDumpMemSize,
+									 szFilenameOut,
+									 uiOffsetOut,
+									 PDUMP_FLAGS_CONTINUOUS);
+				if (PVRSRV_OK != eError)
+				{
+					PDUMP_ERROR(psDevNode, eError, "Failed to write LDB statement to script file");
+					PVR_LOG_ERROR(eError, "PDumpPMRLDB");
+				}
+			}
+			else if (eError != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+			{
+				PDUMP_ERROR(psDevNode, eError, "Failed to write device allocation to parameter file");
+				PVR_LOG_ERROR(eError, "PDumpWriteParameterBlob");
+			}
+			else
+			{
+				/* Else write to parameter file prevented under the flags and
+				 * current state of the driver so skip write to script and error IF.
+				 * This is expected e.g., if not in the capture range.
+				 */
+				eError = PVRSRV_OK;
+			}
+		}
+#endif
+
+		/* Unmap the page */
+		PhysHeapPagesUnMap(psDevNode->psMMUPhysHeap,
+		                   psMemHandle,
+		                   pvCpuVAddr);
+	}
+
+	return PVRSRV_OK;
+}
+
+void DevPhysMemFree(PVRSRV_DEVICE_NODE *psDevNode,
+#if defined(PDUMP)
+							IMG_HANDLE hPDUMPMemHandle,
+#endif
+							IMG_HANDLE	hMemHandle)
+{
+	PG_HANDLE *psMemHandle;
+
+	psMemHandle = hMemHandle;
+	PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+#if defined(PDUMP)
+	if (NULL != hPDUMPMemHandle)
+	{
+		PDumpFree(psDevNode, hPDUMPMemHandle);
+	}
+#endif
+
+}
+
+PVRSRV_ERROR PhysMemValidateMappingTable(IMG_UINT32 ui32TotalNumVirtChunks,
+                                         IMG_UINT32 ui32IndexCount,
+                                         const IMG_UINT32 *pui32MappingTable)
+{
+	IMG_UINT8 *paui8TrackedIndices;
+	IMG_UINT32 ui32BytesToTrackIndicies;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Allocate memory for a bitmask to track indices.
+	 * We allocate 'n' bytes with 1 bit representing each index, to allow
+	 * us to check for any repeated entries in pui32MappingTable.
+	 */
+	ui32BytesToTrackIndicies = LOG2_DIV(ui32TotalNumVirtChunks, 3);
+	if (LOG2_MOD(ui32TotalNumVirtChunks, 3) != 0)
+	{
+		++ui32BytesToTrackIndicies;
+	}
+	paui8TrackedIndices = OSAllocZMem(ui32BytesToTrackIndicies);
+	if (paui8TrackedIndices == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	for (i = 0; i < ui32IndexCount; i++)
+	{
+		IMG_UINT32 ui32LogicalIndex = pui32MappingTable[i];
+
+		/* Check that index is within the bounds of the allocation */
+		if (ui32LogicalIndex >= ui32TotalNumVirtChunks)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Index %u is OOB",
+			         __func__,
+			         ui32LogicalIndex));
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+
+		/* Check that index is not repeated */
+		if (BIT_ISSET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3)))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Duplicate index found: %u",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+		BIT_SET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3));
+	}
+
+	OSFreeMem(paui8TrackedIndices);
+
+	return eError;
+}
+
+/* Checks the input parameters and adjusts them if possible and necessary */
+PVRSRV_ERROR PhysMemValidateParams(PVRSRV_DEVICE_NODE *psDevNode,
+                                   IMG_UINT32 ui32NumPhysChunks,
+                                   IMG_UINT32 ui32NumVirtChunks,
+                                   IMG_UINT32 *pui32MappingTable,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   IMG_PID uiPid,
+                                   IMG_UINT32 *puiLog2AllocPageSize,
+                                   IMG_DEVMEM_SIZE_T *puiSize)
+{
+	IMG_UINT32 uiLog2AllocPageSize = *puiLog2AllocPageSize;
+	IMG_UINT32 ui32PageSize = IMG_PAGE2BYTES32(uiLog2AllocPageSize);
+	IMG_DEVMEM_SIZE_T uiSize = *puiSize;
+	/* Sparse if we have different number of virtual and physical chunks plus
+	 * in general all allocations with more than one virtual chunk */
+	IMG_BOOL bIsSparse = (ui32NumVirtChunks != ui32NumPhysChunks ||
+			ui32NumVirtChunks > 1) ? IMG_TRUE : IMG_FALSE;
+
+	/* Sparse allocations must be backed immediately as the requested
+	 * pui32MappingTable is not retained in any structure if not immediately
+	 * actioned on allocation.
+	 */
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags) && bIsSparse)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid to specify ON_DEMAND for a sparse allocation: 0x%" IMG_UINT64_FMTSPECX, __func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags) && uiPid == PVR_SYS_ALLOC_PID)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot move system allocated resources PID = PVR_SYS_ALLOC_PID",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags) &&
+	    PVRSRV_CHECK_OS_LINUX_DENY_MOVE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot move denied movable allocation",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif
+
+	if (ui32NumVirtChunks == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of virtual chunks cannot be 0",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Protect against invalid page sizes */
+	if ((ui32PageSize & psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask) == 0)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR, "Page size of %u is invalid", ui32PageSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Range check of the alloc size */
+	if (!PMRValidateSize(uiSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				   "PMR size exceeds limit #Chunks: %u ChunkSz 0x%08X",
+				   ui32NumVirtChunks,
+				   IMG_PAGE2BYTES32(uiLog2AllocPageSize));
+		return PVRSRV_ERROR_PMR_TOO_LARGE;
+	}
+
+	/* Fail if requesting coherency on one side but uncached on the other */
+	if (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
+	    (PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
+				"Please use GPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+	    (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
+				"Please use CPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags) && PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Zero on Alloc and Poison on Alloc are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (bIsSparse)
+	{
+		/* For sparse we need correct parameters like a suitable page size....  */
+		if (OSGetPageShift() > uiLog2AllocPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid log2-contiguity for sparse allocation. "
+					"Requested %u, required minimum %zd",
+					__func__,
+					uiLog2AllocPageSize,
+					OSGetPageShift() ));
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		if (IMG_PAGES2BYTES64(ui32NumVirtChunks, uiLog2AllocPageSize) != uiSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Total alloc size (%#" IMG_UINT64_FMTSPECx ") "
+					 "is not equal to virtual chunks * chunk size "
+					 "(%#" IMG_UINT64_FMTSPECx ")",
+					 __func__,
+					 uiSize,
+					 IMG_PAGES2BYTES64(ui32NumVirtChunks, uiLog2AllocPageSize)));
+
+			return PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
+		}
+
+		if (ui32NumPhysChunks > ui32NumVirtChunks)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Number of physical chunks (%u) must not be greater "
+					"than number of virtual chunks (%u)",
+					__func__,
+					ui32NumPhysChunks,
+					ui32NumVirtChunks));
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	else
+	{
+		/*
+		 * Silently round up alignment/pagesize if request was less that PAGE_SHIFT
+		 * because it would never be harmful for memory to be _more_ contiguous that
+		 * was desired.
+		 */
+		uiLog2AllocPageSize = OSGetPageShift() > uiLog2AllocPageSize ?
+				OSGetPageShift() : uiLog2AllocPageSize;
+
+		/* Same for total size */
+		uiSize = PVR_ALIGN(uiSize, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+	}
+
+	if ((uiSize & (IMG_PAGE2BYTES32(uiLog2AllocPageSize) - 1)) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Total size (%#" IMG_UINT64_FMTSPECx ") "
+		        "must be a multiple of the requested contiguity (%u)",
+		        __func__,
+		        uiSize,
+		        IMG_PAGE2BYTES32(uiLog2AllocPageSize)));
+		return PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
+	}
+
+	/* Parameter validation - Mapping table entries */
+	{
+		PVRSRV_ERROR eErr = PhysMemValidateMappingTable(ui32NumVirtChunks,
+		                                                ui32NumPhysChunks,
+		                                                pui32MappingTable);
+		PVR_RETURN_IF_ERROR(eErr);
+	}
+
+	*puiLog2AllocPageSize = uiLog2AllocPageSize;
+	*puiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _DevPhysHeapFromFlags(PVRSRV_MEMALLOCFLAGS_T uiFlags,
+										  PVRSRV_PHYS_HEAP *peDevPhysHeap,
+										  PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_PHYS_HEAP eHeap = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	switch (eHeap)
+	{
+		case PVRSRV_PHYS_HEAP_FW_PREMAP0:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP1:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP2:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP3:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP4:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP5:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP6:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP7:
+		{
+			/* keep heap (with check) */
+			PVR_RETURN_IF_INVALID_PARAM(!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode));
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_LAST:
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+		default:
+		{
+			break;
+		}
+	}
+
+	*peDevPhysHeap = eHeap;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void _PromoteToCpuCached(PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)) == 0)
+	{
+		/* We don't need to upgrade if we don't map into the CPU */
+		return;
+	}
+
+	/* Clear the existing CPU cache flags */
+	*puiFlags &= ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK);
+
+	/* Add CPU cached flags */
+	*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT;
+}
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR_direct(CONNECTION_DATA *psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2AllocPageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMRPtr,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	PVRSRV_PHYS_HEAP ePhysHeapIdx;
+	PVRSRV_MEMALLOCFLAGS_T uiPMRFlags = uiFlags;
+	uiPid = (psConnection != NULL) ? OSGetCurrentClientProcessIDKM() : uiPid;
+
+	/* This is where we would expect to validate the uiAnnotationLength parameter
+	   (to confirm it is sufficient to store the string in pszAnnotation plus a
+	   terminating NULL). However, we do not make reference to this value when
+	   we copy the string in PMRCreatePMR() - instead there we use strlcpy()
+	   to copy at most chars and ensure whatever is copied is null-terminated.
+	   The parameter is only used by the generated bridge code.
+	 */
+	PVR_UNREFERENCED_PARAMETER(uiAnnotationLength);
+
+	if (PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY)
+	{
+		_PromoteToCpuCached(&uiPMRFlags);
+	}
+
+	eError = PhysMemValidateParams(psDevNode,
+	                               ui32NumPhysChunks,
+	                               ui32NumVirtChunks,
+	                               pui32MappingTable,
+	                               uiFlags,
+	                               uiPid,
+	                               &uiLog2AllocPageSize,
+	                               &uiSize);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(SUPPORT_STATIC_IPA)
+#if !defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	/* Do not permit IPA PMR allocation flags to be passed through to the
+	 * new PMR.
+	 */
+	uiPMRFlags &= ~PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+#endif
+#endif
+
+#if defined(PDUMP)
+	eError = PDumpValidateUMFlags(ui32PDumpFlags);
+	PVR_RETURN_IF_ERROR(eError);
+#endif
+
+	for (i = 0; i < ui32NumPhysChunks; i++)
+	{
+		PVR_LOG_RETURN_IF_FALSE(pui32MappingTable[i] < ui32NumVirtChunks,
+		                        "Mapping table value exceeds ui32NumVirtChunks",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = _DevPhysHeapFromFlags(uiFlags, &ePhysHeapIdx, psDevNode);
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeapIdx = psDevNode->psDevConfig->eDefaultHeap;
+		PVRSRV_CHANGE_PHYS_HEAP_HINT(ePhysHeapIdx, uiPMRFlags);
+	}
+
+	if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_GPU_LOCAL)
+	{
+		if ((uiFlags & PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK) == 0)
+		{
+			ePhysHeapIdx = PVRSRV_PHYS_HEAP_GPU_PRIVATE;
+			PVRSRV_SET_PHYS_HEAP_HINT(GPU_PRIVATE, uiPMRFlags);
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Consider explicit use of GPU_PRIVATE for PMR %s."
+			        " Implicit conversion to GPU PRIVATE performed",
+			        __func__, pszAnnotation));
+		}
+		else if (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+				 PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig))
+		{
+			ePhysHeapIdx = PVRSRV_PHYS_HEAP_GPU_COHERENT;
+			PVRSRV_SET_PHYS_HEAP_HINT(GPU_COHERENT, uiPMRFlags);
+		}
+	}
+	else if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+	{
+		if (uiFlags & PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid flags for PMR %s!"
+			        " Client requested GPU_PRIVATE physical heap with CPU access flags.",
+			        __func__, pszAnnotation));
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+	}
+
+	if (NULL == psDevNode->apsPhysHeap[ePhysHeapIdx])
+	{
+		/* In case a heap hasn't been acquired for this type, return invalid heap error */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Requested allocation on device node (%p) from "
+		        "an invalid heap (HeapIndex=%d)",
+		        __func__, psDevNode, ePhysHeapIdx));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+#if defined(DEBUG)
+	if (PMRAllocFail > 0)
+	{
+		static IMG_UINT32 ui32AllocCount = 1;
+
+		if (ui32AllocCount < PMRAllocFail)
+		{
+			ui32AllocCount++;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s failed on %d allocation.",
+			         __func__, ui32AllocCount));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+#endif /* defined(DEBUG) */
+
+	/* If the driver is in an 'init' state all of the allocated memory
+	 * should be attributed to the driver (PID 1) rather than to the
+	 * process those allocations are made under. Same applies to the memory
+	 * allocated for the Firmware. */
+	if (psDevNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+	    PVRSRV_CHECK_FW_MAIN(uiFlags))
+	{
+		uiPid = PVR_SYS_ALLOC_PID;
+	}
+
+	/* ePhysHeapIdx and PhysHeap hint in uiPMRFlags match and provide the
+	 * intended PhysHeap to use at this point, but systems vary so the next
+	 * call may fallback (apsPhysHeap[]) or demote (OutOfMem) and not be the
+	 * heap that was intended, e.g. GPU_PRIVATE index may fallback to GPU_LOCAL,
+	 * GPU_LOCAL may demote to CPU_LOCAL.
+	 * On output uiPMRFlags show the PhysHeap finally used.
+	 */
+	eError = PhysHeapCreatePMR(psDevNode->apsPhysHeap[ePhysHeapIdx],
+							   psConnection,
+							   uiSize,
+							   ui32NumPhysChunks,
+							   ui32NumVirtChunks,
+							   pui32MappingTable,
+							   uiLog2AllocPageSize,
+							   uiPMRFlags,
+							   pszAnnotation,
+							   uiPid,
+							   ppsPMRPtr,
+							   ui32PDumpFlags,
+							   &uiPMRFlags);
+
+	if (puiPMRFlags != NULL)
+	{
+		*puiPMRFlags = uiPMRFlags;
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	if (eError != PVRSRV_OK)
+	{
+		PVRSRVStatsUpdateOOMStat(psConnection,
+		                          psDevNode,
+		                          PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+		                          OSGetCurrentClientProcessIDKM());
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR(CONNECTION_DATA *psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2AllocPageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMRPtr,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags)
+{
+	PVRSRV_PHYS_HEAP ePhysHeap = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ePhysHeap < PVRSRV_PHYS_HEAP_LAST, "uiFlags");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiAnnotationLength != 0, "uiAnnotationLength");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	if (!PhysHeapUserModeAlloc(ePhysHeap))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid phys heap hint: %d.", __func__, ePhysHeap));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PhysmemNewRamBackedPMR_direct(psConnection,
+	                                     psDevNode,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     uiAnnotationLength,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags,
+	                                     puiPMRFlags);
+}
+
+PVRSRV_ERROR
+PVRSRVGetDefaultPhysicalHeapKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDevNode,
+                               PVRSRV_PHYS_HEAP *peHeap)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	*peHeap = psDevNode->psDevConfig->eDefaultHeap;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfoKM(CONNECTION_DATA *psConnection,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           IMG_UINT32 ui32PhysHeapCount,
+                           PVRSRV_PHYS_HEAP *paePhysHeapID,
+                           PHYS_HEAP_MEM_STATS_V1 *paPhysHeapMemStats)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysHeapCount);
+	PVR_UNREFERENCED_PARAMETER(paePhysHeapID);
+	PVR_UNREFERENCED_PARAMETER(paPhysHeapMemStats);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfo2KM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                            IMG_UINT32 ui32PhysHeapCount,
+                            PVRSRV_PHYS_HEAP *paePhysHeapID,
+                            PHYS_HEAP_MEM_STATS_V2 *paPhysHeapMemStats)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	return PhysHeapGetMemInfo(psDevNode,
+	                          ui32PhysHeapCount,
+	                          paePhysHeapID,
+	                          paPhysHeapMemStats);
+}
+
+/* 'Wrapper' function to call PMRImportPMR(), which first checks the PMR is
+ * for the current device. This avoids the need to do this in pmr.c, which
+ * would then need PVRSRV_DEVICE_NODE (defining this type in pmr.h causes a
+ * typedef redefinition issue).
+ */
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR
+PhysmemImportPMR(CONNECTION_DATA *psConnection,
+             PVRSRV_DEVICE_NODE *psDevNode,
+             PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (PMRGetExportDeviceNode(psPMRExport) != psDevNode)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device", __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	return PMRImportPMR(psPMRExport,
+	                    uiPassword,
+	                    uiSize,
+	                    uiLog2Contig,
+	                    ppsPMR);
+}
+#endif /* if defined(SUPPORT_INSECURE_EXPORT) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_cpumap_history.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_cpumap_history.c
new file mode 100644
index 000000000000..6975926a45d8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_cpumap_history.c
@@ -0,0 +1,455 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR Mapping History for OS managed memory
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This module allows for an OS independent method of capture of
+                mapping history for OS memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/mem_types.h>
+#include "di_common.h"
+#include "di_server.h"
+#include "pvr_notifier.h"
+#include "physmem_cpumap_history.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#define MAPPING_HISTORY_CB_NUM_ENTRIES 10000
+#if defined(DEBUG)
+#define MAX_MAPPING_ANNOT_STR PVR_ANNOTATION_MAX_LEN
+#else
+#define MAX_MAPPING_ANNOT_STR 32
+#endif
+
+typedef struct _MAPPING_RECORD_ MAPPING_RECORD; /* Forward declaration */
+typedef void (*PFN_MAPPING_RECORD_STRING)(MAPPING_RECORD *psRecord,
+                                          IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN]);
+
+typedef struct _MAP_DATA_
+{
+	IMG_CHAR aszAnnotation[MAX_MAPPING_ANNOT_STR];
+	IMG_PID uiPID;
+	IMG_CPU_VIRTADDR pvAddress;
+	IMG_CPU_PHYADDR sCpuPhyAddr;
+	IMG_UINT32 ui32CPUCacheFlags;
+	size_t uiMapOffset; /* Mapping offset used when we don't map the whole PMR */
+	IMG_UINT32 ui32PageCount; /* # pages mapped */
+} MAP_DATA;
+
+typedef struct _UNMAP_DATA_
+{
+	IMG_CPU_VIRTADDR pvAddress;
+	IMG_CPU_PHYADDR sCpuPhyAddr;
+	IMG_UINT32 ui32CPUCacheFlags;
+	IMG_UINT32 ui32PageCount;
+} UNMAP_DATA;
+
+struct _MAPPING_RECORD_
+{
+	enum MAPPING_OP
+	{
+		UNSET = 0,
+		MAP,
+		UNMAP,
+	} etype;
+	union
+	{
+		MAP_DATA sMapData;
+		UNMAP_DATA sUnmapData;
+	} u;
+	PFN_MAPPING_RECORD_STRING pfnRecordString;
+};
+
+typedef struct _RECORDS_
+{
+	/* CB of mapping records that will be overwritten by newer entries */
+	MAPPING_RECORD *pasMapRecordsCB;
+	/* Current head of CB, used to get next slot, newest record +1 */
+	IMG_UINT32 ui32Head;
+	/* Current tail of CB, oldest record */
+	IMG_UINT32 ui32Tail;
+	/* Have we overwritten any records */
+	IMG_BOOL bOverwrite;
+} RECORDS;
+
+typedef struct _PHYSMEM_CPUMAP_HISTORY_DATA_
+{
+	DI_ENTRY *psDIEntry;
+	RECORDS sRecords;
+	POS_LOCK hLock;
+	IMG_HANDLE hDbgNotifier;
+
+} PHYSMEM_CPUMAP_HISTORY_DATA;
+
+typedef struct _MAPPING_HISTORY_ITERATOR_
+{
+	IMG_UINT32 ui32Start;
+	IMG_UINT32 ui32Finish;
+	IMG_UINT32 ui32Current;
+} MAPPING_HISTORY_ITERATOR;
+
+static PHYSMEM_CPUMAP_HISTORY_DATA gsMappingHistoryData;
+
+static PVRSRV_ERROR CreateMappingRecords(void)
+{
+	gsMappingHistoryData.sRecords.pasMapRecordsCB =
+	    OSAllocMem(sizeof(MAPPING_RECORD) * MAPPING_HISTORY_CB_NUM_ENTRIES);
+	PVR_RETURN_IF_NOMEM(gsMappingHistoryData.sRecords.pasMapRecordsCB);
+
+	return PVRSRV_OK;
+}
+
+static void InitMappingRecordCB(void)
+{
+	gsMappingHistoryData.sRecords.ui32Head = 0;
+	gsMappingHistoryData.sRecords.ui32Tail = 0;
+	gsMappingHistoryData.sRecords.bOverwrite = IMG_FALSE;
+}
+
+static void DestroyMappingRecords(void)
+{
+	OSFreeMem(gsMappingHistoryData.sRecords.pasMapRecordsCB);
+}
+
+static void MappingHistoryLock(void)
+{
+	if (gsMappingHistoryData.hLock)
+	{
+		OSLockAcquire(gsMappingHistoryData.hLock);
+	}
+}
+
+static void MappingHistoryUnlock(void)
+{
+	if (gsMappingHistoryData.hLock)
+	{
+		OSLockRelease(gsMappingHistoryData.hLock);
+	}
+}
+
+static MAPPING_HISTORY_ITERATOR CreateMappingHistoryIterator(void)
+{
+	MAPPING_HISTORY_ITERATOR sIter;
+
+	sIter.ui32Start = gsMappingHistoryData.sRecords.ui32Tail;
+	sIter.ui32Current = gsMappingHistoryData.sRecords.ui32Tail;
+	sIter.ui32Finish = gsMappingHistoryData.sRecords.ui32Head;
+
+	return sIter;
+}
+
+static IMG_BOOL MappingHistoryHasRecords(void)
+{
+	if ((gsMappingHistoryData.sRecords.ui32Head ==
+	     gsMappingHistoryData.sRecords.ui32Tail) &&
+	    !gsMappingHistoryData.sRecords.bOverwrite)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static IMG_BOOL MappingHistoryIteratorNext(MAPPING_HISTORY_ITERATOR *psIter)
+{
+	psIter->ui32Current = (psIter->ui32Current + 1) % MAPPING_HISTORY_CB_NUM_ENTRIES;
+
+	if (psIter->ui32Current == psIter->ui32Finish)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+#define MAPPING_ITER_TO_PTR(psIter) (&(gsMappingHistoryData.sRecords.pasMapRecordsCB[psIter.ui32Current]))
+
+static MAPPING_RECORD* AcquireMappingHistoryCBSlot(void)
+{
+	MAPPING_RECORD *psSlot;
+
+	psSlot = &gsMappingHistoryData.sRecords.pasMapRecordsCB[gsMappingHistoryData.sRecords.ui32Head];
+
+	gsMappingHistoryData.sRecords.ui32Head =
+	    (gsMappingHistoryData.sRecords.ui32Head + 1)
+	    % MAPPING_HISTORY_CB_NUM_ENTRIES;
+
+	if (!gsMappingHistoryData.sRecords.bOverwrite)
+	{
+		if (gsMappingHistoryData.sRecords.ui32Head == gsMappingHistoryData.sRecords.ui32Tail)
+		{
+			gsMappingHistoryData.sRecords.bOverwrite = IMG_TRUE;
+		}
+	}
+	else
+	{
+		gsMappingHistoryData.sRecords.ui32Tail = gsMappingHistoryData.sRecords.ui32Head;
+	}
+
+	return psSlot;
+}
+
+static IMG_CHAR* CacheFlagsToStr(IMG_UINT32 ui32CPUCacheFlags)
+{
+	switch (ui32CPUCacheFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			return "UC";
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			return "WC";
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+			return "CA";
+
+		default:
+			return "UN";
+	}
+}
+
+static void MapRecordString(MAPPING_RECORD *psRecord,
+                            IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(*pszBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+	           "%-32s %-8u 0x%p "CPUPHYADDR_UINT_FMTSPEC" %-5u (%-2s) %-12lu %-8u",
+	           psRecord->u.sMapData.aszAnnotation,
+	           psRecord->u.sMapData.uiPID,
+	           psRecord->u.sMapData.pvAddress,
+	           CPUPHYADDR_FMTARG(psRecord->u.sMapData.sCpuPhyAddr.uiAddr),
+	           psRecord->u.sMapData.ui32CPUCacheFlags,
+	           CacheFlagsToStr(psRecord->u.sMapData.ui32CPUCacheFlags),
+	           (unsigned long) psRecord->u.sMapData.uiMapOffset,
+	           psRecord->u.sMapData.ui32PageCount);
+}
+
+static void UnMapRecordString(MAPPING_RECORD *psRecord,
+                              IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(*pszBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+	           "%-41s 0x%p "CPUPHYADDR_UINT_FMTSPEC" %-5u (%-2s) %-12s %-8u",
+	           "UnMap", /* PADDING */
+	           psRecord->u.sUnmapData.pvAddress,
+	           CPUPHYADDR_FMTARG(psRecord->u.sUnmapData.sCpuPhyAddr.uiAddr),
+	           psRecord->u.sUnmapData.ui32CPUCacheFlags,
+	           CacheFlagsToStr(psRecord->u.sUnmapData.ui32CPUCacheFlags),
+	           "", /* PADDING */
+	           psRecord->u.sUnmapData.ui32PageCount);
+}
+
+static void MappingHistoryGetHeaderString(IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(*pszBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+	           "%-32s %-8s %-18s %-18s %-9s %-8s %-8s",
+	           "PMRAnnotation",
+	           "PID",
+	           "CpuVirtAddr",
+	           "CpuPhyAddr",
+	           "CacheFlags",
+	           "PMRMapOffset",
+	           "ui32PageCount");
+}
+
+static void MappingHistoryPrintAll(OSDI_IMPL_ENTRY *psEntry)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	MAPPING_HISTORY_ITERATOR sIter = CreateMappingHistoryIterator();
+
+	if (!MappingHistoryHasRecords())
+	{
+		DIPrintf(psEntry, "No Records...\n");
+		return;
+	}
+
+	MappingHistoryGetHeaderString(&szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	do
+	{
+		MAPPING_RECORD *psRecord = MAPPING_ITER_TO_PTR(sIter);
+		psRecord->pfnRecordString(psRecord, &szBuffer);
+		DIPrintf(psEntry, "%s\n", szBuffer);
+
+	}
+	while (MappingHistoryIteratorNext(&sIter));
+}
+
+static int MappingHistoryPrintAllWrapper(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	MappingHistoryLock();
+	MappingHistoryPrintAll(psEntry);
+	MappingHistoryUnlock();
+
+	return 0;
+}
+
+static void MappingHistoryDebugDump(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                    IMG_UINT32 ui32VerbLevel,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MAX))
+	{
+		MAPPING_HISTORY_ITERATOR sIter = CreateMappingHistoryIterator();
+		IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+
+		PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+		PVR_DUMPDEBUG_LOG("------[ Physmem CPU Map History ]------");
+
+		if (!MappingHistoryHasRecords())
+		{
+			PVR_DUMPDEBUG_LOG("No Records...");
+			return;
+		}
+
+		MappingHistoryGetHeaderString(&szBuffer);
+		PVR_DUMPDEBUG_LOG("%s", szBuffer);
+
+		do
+		{
+			MAPPING_RECORD *psRecord = MAPPING_ITER_TO_PTR(sIter);
+			psRecord->pfnRecordString(psRecord, &szBuffer);
+			PVR_DUMPDEBUG_LOG("%s", szBuffer);
+		}
+		while (MappingHistoryIteratorNext(&sIter));
+	}
+}
+
+PVRSRV_ERROR CPUMappingHistoryInit(void)
+{
+	PVRSRV_ERROR eError;
+	DI_ITERATOR_CB sIterator =
+	{ .pfnShow = MappingHistoryPrintAllWrapper };
+
+	eError = OSLockCreate(&gsMappingHistoryData.hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", error);
+
+	eError = CreateMappingRecords();
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateMappingRecords", error);
+
+	InitMappingRecordCB();
+
+	eError = DICreateEntry("cpumap_history", NULL, &sIterator, NULL,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &gsMappingHistoryData.psDIEntry);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", error);
+
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&gsMappingHistoryData.hDbgNotifier,
+	                                              MappingHistoryDebugDump,
+	                                              DEBUG_REQUEST_SRV,
+	                                              NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterDriverDbgRequestNotify", error);
+
+	return PVRSRV_OK;
+
+error:
+	CPUMappingHistoryDeInit();
+	return eError;
+}
+
+void CPUMappingHistoryDeInit(void)
+{
+	if (gsMappingHistoryData.hDbgNotifier != NULL)
+	{
+		PVRSRVUnregisterDriverDbgRequestNotify(gsMappingHistoryData.hDbgNotifier);
+		gsMappingHistoryData.hDbgNotifier = NULL;
+	}
+
+	if (gsMappingHistoryData.psDIEntry != NULL)
+	{
+		DIDestroyEntry(gsMappingHistoryData.psDIEntry);
+		gsMappingHistoryData.psDIEntry = NULL;
+	}
+
+	if (gsMappingHistoryData.sRecords.pasMapRecordsCB)
+	{
+		DestroyMappingRecords();
+		gsMappingHistoryData.sRecords.pasMapRecordsCB = NULL;
+	}
+
+	if (gsMappingHistoryData.hLock != NULL)
+	{
+		OSLockDestroy(gsMappingHistoryData.hLock);
+		gsMappingHistoryData.hLock = NULL;
+	}
+}
+
+void InsertMappingRecord(const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPID,
+                         IMG_CPU_VIRTADDR pvAddress,
+                         IMG_CPU_PHYADDR sCpuPhyAddr,
+                         IMG_UINT32 ui32CPUCacheFlags,
+                         size_t uiMapOffset,
+                         IMG_UINT32 ui32PageCount)
+{
+	MAPPING_RECORD *psRecord;
+
+	psRecord = AcquireMappingHistoryCBSlot();
+	psRecord->etype = MAP;
+	psRecord->pfnRecordString = MapRecordString;
+
+	OSStringSafeCopy(psRecord->u.sMapData.aszAnnotation, pszAnnotation, MAX_MAPPING_ANNOT_STR);
+	psRecord->u.sMapData.uiPID = uiPID;
+	psRecord->u.sMapData.pvAddress = pvAddress;
+	psRecord->u.sMapData.ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psRecord->u.sMapData.sCpuPhyAddr = sCpuPhyAddr;
+	psRecord->u.sMapData.uiMapOffset = uiMapOffset;
+	psRecord->u.sMapData.ui32PageCount = ui32PageCount;
+}
+
+void InsertUnMappingRecord(IMG_CPU_VIRTADDR pvAddress,
+                           IMG_CPU_PHYADDR sCpuPhyAddr,
+                           IMG_UINT32 ui32CPUCacheFlags,
+                           IMG_UINT32 ui32PageCount)
+{
+	MAPPING_RECORD *psRecord;
+
+	psRecord = AcquireMappingHistoryCBSlot();
+	psRecord->etype = UNMAP;
+	psRecord->pfnRecordString = UnMapRecordString;
+
+	psRecord->u.sUnmapData.pvAddress = pvAddress;
+	psRecord->u.sUnmapData.ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psRecord->u.sUnmapData.sCpuPhyAddr = sCpuPhyAddr;
+	psRecord->u.sUnmapData.ui32PageCount = ui32PageCount;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_dlm.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_dlm.c
new file mode 100644
index 000000000000..07a92b2973df
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_dlm.c
@@ -0,0 +1,341 @@
+/*************************************************************************/ /*!
+@File           physmem_dlm.c
+@Title          Dedicated Local Memory allocator for Physical Memory Blocks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for Dedicated local memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+#include "physmem_dlm.h"
+
+typedef struct PHYSMEM_DLM_DATA_TAG {
+	RA_ARENA           *psRA;
+	IMG_CPU_PHYADDR    sStartAddr;
+	IMG_DEV_PHYADDR    sCardBase;
+	IMG_UINT64         uiSize;
+	IMG_UINT32         uiLog2PMBSize;
+} PHYSMEM_DLM_DATA;
+
+/* PMB (Physical Memory Block) */
+struct _PMB_
+{
+	RA_ARENA *pArena;
+	RA_BASE_T uiBase;
+	RA_LENGTH_T uiSize;
+	const IMG_CHAR *pszAnnotation;
+};
+
+/* PMBCreatePMB
+ *
+ * Creates a new PMB used to represent a block of memory
+ * obtained from a DLM heap.
+ */
+static
+PVRSRV_ERROR PMBCreatePMB(RA_ARENA *pArena,
+                          RA_LENGTH_T uiSize,
+                          RA_LENGTH_T uiAlignment,
+                          const IMG_CHAR *pszAnnotation,
+                          PMB **ppsPMB)
+{
+	PVRSRV_ERROR eError;
+	PMB* psPMB = OSAllocMem(sizeof(*psPMB));
+	PVR_LOG_GOTO_IF_NOMEM(psPMB, eError, error_Return);
+
+	eError = RA_Alloc(pArena,
+	                  uiSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                         /* No flags */
+	                  uiAlignment,
+	                  pszAnnotation,
+	                  &psPMB->uiBase,
+	                  &psPMB->uiSize,
+	                  NULL);                     /* No private handle */
+	PVR_LOG_GOTO_IF_ERROR(eError, "RA_Alloc", error_FreePMB);
+
+	psPMB->pArena = pArena;
+	psPMB->pszAnnotation = pszAnnotation;
+
+	*ppsPMB = psPMB;
+
+	return PVRSRV_OK;
+
+error_FreePMB:
+	OSFreeMem(psPMB);
+error_Return:
+	return eError;
+}
+
+void
+PMBDestroy(PMB *psPMB)
+{
+	PVR_LOG_RETURN_VOID_IF_FALSE(psPMB, "psPMB NULL");
+
+	RA_Free(psPMB->pArena, psPMB->uiBase);
+	OSFreeMem(psPMB);
+}
+
+
+const IMG_CHAR *
+PMBGetAnnotation(PMB *psPMB)
+{
+	if (psPMB == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"psPMB in %s",__func__));
+		return "";
+	}
+
+	return psPMB->pszAnnotation;
+}
+
+/* DLM */
+
+static void PFNGetLocalRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                                 IMG_UINT64 *pui64TotalSize,
+                                 IMG_UINT64 *pui64FreeSize)
+{
+	RA_USAGE_STATS sRAUsageStats;
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_VOID_IF_FALSE(pvImplData, "pvImplData NULL");
+
+	RA_Get_Usage_Stats(psDLMData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+/*
+* This function will set the psDevPAddr to whatever the system layer
+* has set it for the referenced heap.
+* It will not fail if the psDevPAddr is invalid.
+*/
+static PVRSRV_ERROR
+PFNGetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvImplData != NULL, "pvImplData");
+
+	*psDevPAddr = psDLMData->sCardBase;
+
+	return PVRSRV_OK;
+}
+
+/*
+* This function will set the psCpuPAddr to whatever the system layer
+* has set it for the referenced heap.
+* It will not fail if the psCpuPAddr is invalid.
+*/
+static PVRSRV_ERROR
+PFNGetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvImplData != NULL, "pvImplData");
+
+	*psCpuPAddr = psDLMData->sStartAddr;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PFNGetSize(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *puiSize)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvImplData != NULL, "pvImplData");
+
+	*puiSize = psDLMData->uiSize;
+
+	return PVRSRV_OK;
+}
+
+static IMG_UINT32
+PFNGetPageShift(void)
+{
+	return PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT;
+}
+
+static PVRSRV_ERROR
+CreateArenas(PHYSMEM_DLM_DATA *psDLMData, IMG_CHAR *pszLabel, PHYS_HEAP_POLICY uiPolicy)
+{
+	psDLMData->psRA = RA_Create_With_Span(pszLabel,
+	                             OSGetPageShift(),
+	                             psDLMData->sStartAddr.uiAddr,
+	                             psDLMData->sCardBase.uiAddr,
+	                             psDLMData->uiSize,
+	                             RA_POLICY_DEFAULT);
+	PVR_LOG_RETURN_IF_NOMEM(psDLMData->psRA, "RA_Create_With_Span");
+
+	return PVRSRV_OK;
+}
+
+static void
+DestroyArenas(PHYSMEM_DLM_DATA *psDLMData)
+{
+	/* Remove RAs and RA names for dedicated local memory */
+	if (psDLMData->psRA)
+	{
+		RA_Delete(psDLMData->psRA);
+		psDLMData->psRA = NULL;
+	}
+}
+
+static void
+PFNDestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_VOID_IF_FALSE(pvImplData, "pvImplData NULL");
+
+	DestroyArenas(pvImplData);
+
+	OSFreeMem(psDLMData);
+}
+
+static PVRSRV_ERROR
+PFNPhysmemNewLocalRamBackedPMB(PHYS_HEAP *psPhysHeap,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            const IMG_CHAR *pszAnnotation,
+                            PMB **ppPMBPtr,
+                            RA_BASE_T *puiBase,
+                            RA_LENGTH_T *puiSize)
+{
+	PHYSMEM_DLM_DATA *psDLMData;
+	PVRSRV_ERROR eError;
+	PMB* pPMB;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppPMBPtr != NULL, "ppPMBPtr");
+
+	/* Check size is aligned to page size */
+	if (uiSize & ((1 << PFNGetPageShift()) - 1ULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: uiSize %" IMG_UINT64_FMTSPEC " is not aligned to page size: %u",
+		         __func__,
+		         uiSize,
+		         1 << PFNGetPageShift()));
+		return PVRSRV_ERROR_PMB_NOT_PAGE_MULTIPLE;
+	}
+
+	PVR_ASSERT(PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_DLM);
+
+	psDLMData = (PHYSMEM_DLM_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	eError = PMBCreatePMB(psDLMData->psRA, uiSize, IMG_UINT64_C(1) << psDLMData->uiLog2PMBSize, pszAnnotation, &pPMB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMBCreatePMB", error_Return);
+
+	*ppPMBPtr = pPMB;
+	*puiBase = pPMB->uiBase;
+	*puiSize = pPMB->uiSize;
+
+	return PVRSRV_OK;
+
+error_Return:
+	return eError;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = &PFNDestroyImplData,
+	.pfnGetDevPAddr = &PFNGetDevPAddr,
+	.pfnGetCPUPAddr = &PFNGetCPUPAddr,
+	.pfnGetSize = &PFNGetSize,
+	.pfnGetPageShift = &PFNGetPageShift,
+	.pfnGetFactoryMemStats = &PFNGetLocalRamMemStats,
+	.pfnCreatePMB = &PFNPhysmemNewLocalRamBackedPMB,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapDLM(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_DLM_DATA *psDLMData;
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConfig != NULL, "psConfig");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_DLM);
+
+	psDLMData = OSAllocMem(sizeof(*psDLMData));
+	PVR_LOG_RETURN_IF_NOMEM(psDLMData, "OSAllocMem");
+
+	psDLMData->sStartAddr = PhysHeapConfigGetStartAddr(psConfig);
+	psDLMData->sCardBase = PhysHeapConfigGetCardBase(psConfig);
+	psDLMData->uiSize = PhysHeapConfigGetSize(psConfig);
+	psDLMData->uiLog2PMBSize = psConfig->uConfig.sDLM.ui32Log2PMBSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDLMData->uiLog2PMBSize >= OSGetPageShift(), "ui32Log2PMBSize must be greater than or equal to OSPageSize");
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psDLMData,
+							&_sPHEAPImplFuncs,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", error_FreeDlmData);
+
+	eError = CreateArenas(psDLMData, pszLabel, uiPolicy);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateArenas", error_FreePhysHeap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+error_FreePhysHeap:
+	PhysHeapDestroy(psPhysHeap);
+error_FreeDlmData:
+	OSFreeMem(psDLMData);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_extmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_extmem.c
new file mode 100644
index 000000000000..d365524f9a9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_extmem.c
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File           physmem_extmem.c
+@Title          Physmem - External Memory Import Support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common entry point for Importing Externally allocated Memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv.h"
+#include "physmem_extmem.h"
+
+
+#if defined(LMA)
+#pragma message "WARNING! You are compiling SUPPORT_WRAP_EXTMEM on a platform with LMA. " \
+	"Make sure the memory you wrap is actually accessible by the GPU!"
+#endif
+
+PVRSRV_ERROR
+PhysmemWrapExtMem(CONNECTION_DATA * psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT64 pvCpuVAddr,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  PMR **ppsPMRPtr)
+{
+	return PhysmemWrapExtMemOS(psConnection,
+	                           psDevNode,
+	                           uiSize,
+	                           (IMG_CPU_VIRTADDR)(uintptr_t)pvCpuVAddr,
+	                           uiFlags,
+	                           ppsPMRPtr);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_hostmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_hostmem.c
new file mode 100644
index 000000000000..6c87632bbea5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_hostmem.c
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File           physmem_hostmem.c
+@Title          Host memory device node functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Functions relevant to device memory allocations made from host
+                mem device node.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "physmem_hostmem.h"
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "pvrsrv_device.h"
+#include "physheap.h"
+#include "physmem_osmem.h"
+
+static void HostMemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void HostMemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr);
+
+/* heap callbacks for host driver's device's heap */
+static PHYS_HEAP_FUNCTIONS gsHostMemDevPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	HostMemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	HostMemDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_DEVICE_CONFIG gsHostMemDevConfig[];
+
+/* heap configuration for host driver's device */
+static PHYS_HEAP_CONFIG gsPhysHeapConfigHostMemDevice[] =
+{
+	{
+		.eType = PHYS_HEAP_TYPE_UMA,
+		.ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL,
+		.uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM",
+		.uConfig.sUMA.psMemFuncs = &gsHostMemDevPhysHeapFuncs,
+		.uConfig.sUMA.pszHeapName = "uma_cpu_local",
+		.uConfig.sUMA.hPrivData = (IMG_HANDLE)&gsHostMemDevConfig[0]
+	}
+};
+
+/* device configuration for host driver's device */
+static PVRSRV_DEVICE_CONFIG gsHostMemDevConfig[] =
+{
+	{
+		.pszName = "HostMemDevice",
+		.eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE,
+		.pasPhysHeaps = &gsPhysHeapConfigHostMemDevice[0],
+		.ui32PhysHeapCount = ARRAY_SIZE(gsPhysHeapConfigHostMemDevice),
+	}
+};
+
+static void HostMemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void HostMemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+}
+
+PVRSRV_ERROR HostMemDeviceCreate(PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = &gsHostMemDevConfig[0];
+
+	/* Assert ensures HostMemory device isn't already created and
+	 * that data is initialised */
+	PVR_ASSERT(*ppsDeviceNode == NULL);
+
+	/* for now, we only know a single heap (UMA) config for host device */
+	PVR_ASSERT(psDevConfig->ui32PhysHeapCount == 1 &&
+				psDevConfig->pasPhysHeaps[0].eType == PHYS_HEAP_TYPE_UMA);
+
+	/* N.B.- In case of any failures in this function, we just return error to
+	   the caller, as clean-up is taken care by _HostMemDeviceDestroy function */
+
+	psDeviceNode = OSAllocZMem(sizeof(*psDeviceNode) + sizeof(*psDeviceNode->psMMUDevAttrs));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "OSAllocZMem");
+
+	/* early save return pointer to aid clean-up */
+	*ppsDeviceNode = psDeviceNode;
+
+	psDeviceNode->sDevId.ui32InternalID = PVRSRV_HOST_DEVICE_ID;
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDeviceNode->psPhysHeapList = NULL;
+
+	eError = OSLockCreate(&psDeviceNode->hPhysHeapLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", LockCreateErr);
+
+	eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+										  &psDevConfig->pasPhysHeaps[0],
+										  NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig", PhysHeapCreateErr);
+
+	/* Only CPU local heap is valid on host-mem DevNode, so enable minimal callbacks */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_CPU_LOCAL,
+								 psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire", AcquirePhysHeapErr);
+
+	psDeviceNode->psMMUDevAttrs = (MMU_DEVICEATTRIBS*)(psDeviceNode + 1);
+	psDeviceNode->psMMUDevAttrs->ui32ValidPageSizeMask = OSGetPageSize();
+
+	dllist_init(&psDeviceNode->sCleanupThreadWorkList);
+
+	return PVRSRV_OK;
+
+AcquirePhysHeapErr:
+	PhysHeapDestroyDeviceHeaps(psDeviceNode);
+PhysHeapCreateErr:
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+LockCreateErr:
+	OSFreeMem(psDeviceNode);
+
+	return eError;
+}
+
+void HostMemDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (!psDeviceNode)
+	{
+		return;
+	}
+	else
+	{
+		if (psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL])
+		{
+			PhysHeapRelease(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL]);
+		}
+
+		PhysHeapDestroyDeviceHeaps(psDeviceNode);
+	}
+
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+
+	OSFreeMem(psDeviceNode);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ima.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ima.c
new file mode 100644
index 000000000000..87595e1c8391
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ima.c
@@ -0,0 +1,655 @@
+/*************************************************************************/ /*!
+@File           physmem_ima.c
+@Title          Import memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of memory management. This module is responsible for
+                implementing the function callbacks for local card memory when
+                used under a shared heap system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "device.h"
+#include "osfunc.h"
+#include "physmem_ima.h"
+#include "physmem_dlm.h"
+#include "physmem_ramem.h"
+
+typedef struct PHYSMEM_IMA_DATA_TAG {
+	PHYS_HEAP          *psPhysHeap;
+	RA_ARENA           *psRA;
+	PHYS_HEAP          *pDLMHeap;
+	IMG_UINT32          uiLog2PMBSize;
+	IMG_UINT32          uiReservedPMBs;
+	PMB               **ppsReservedPMBs;
+} PHYSMEM_IMA_DATA;
+
+static PVRSRV_ERROR
+IMAGetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+               IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_ARENA_ITERATOR *psRAIter = RA_IteratorAcquire(psIMAData->psRA, IMG_FALSE);
+	RA_ITERATOR_DATA sData = {0};
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_NOMEM(psRAIter, "RA_IteratorAcquire");
+
+	if (!RA_IteratorNext(psRAIter, &sData))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("RA_IteratorNext",
+		                        eError,
+		                        PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR,
+		                        err_free_iter);
+	}
+
+	psDevPAddr->uiAddr = sData.uiAddr;
+
+err_free_iter:
+	RA_IteratorRelease(psRAIter);
+	return eError;
+}
+
+static PVRSRV_ERROR
+IMAGetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+               IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	IMG_DEV_PHYADDR sDevPAddr;
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	PVRSRV_ERROR eError = IMAGetDevPAddr(pvImplData, &sDevPAddr);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IMAGetDevPAddr");
+
+	PhysHeapDevPAddrToCpuPAddr(
+		psIMAData->pDLMHeap,
+		1,
+		psCpuPAddr,
+		&sDevPAddr
+	);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+IMAGetSize(PHEAP_IMPL_DATA pvImplData,
+           IMG_UINT64 *puiSize)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psIMAData->psRA, &sRAUsageStats);
+
+	*puiSize = sRAUsageStats.ui64TotalArenaSize;
+
+	return PVRSRV_OK;
+}
+
+static void IMAPhysmemGetRAMemRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                                          IMG_UINT64 *pui64TotalSize,
+                                          IMG_UINT64 *pui64FreeSize)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psIMAData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+/*
+ * We iterate on this function multiple times using the IterHandle.
+ * First iteration will create the IterHandle.
+ * We return IMG_TRUE when there are iterations remaining.
+ * We return IMG_FALSE when there are no iterations remaining, this
+ * will also free the IterHandle.
+ * Callers must always iterate until false to ensure IterHandle is
+ * freed.
+ * */
+static IMG_BOOL IMAGetHeapSpansStringIter(PHEAP_IMPL_DATA pvImplData,
+                                          IMG_CHAR *ppszStrBuf,
+                                          IMG_UINT32 uiStrBufSize,
+                                          void **ppvIterHandle)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_ITERATOR_DATA sData = {0};
+	PVRSRV_ERROR eError;
+	IMG_BOOL bIter = IMG_FALSE;
+
+	/* If we haven't been given an IterHandle */
+	if (!*ppvIterHandle)
+	{
+		*ppvIterHandle = RA_IteratorAcquire(psIMAData->psRA, IMG_TRUE);
+		PVR_LOG_GOTO_IF_NOMEM(*ppvIterHandle, eError, return_false);
+	}
+
+	bIter = RA_IteratorNextSpan(*ppvIterHandle, &sData);
+	if (bIter)
+	{
+		IMG_DEV_PHYADDR sRangeBase = {sData.uiAddr};
+		IMG_CHAR aszCpuAddr[18] = "Non-Addressable"; /* length of CPUPHYADDR_UINT_FMTSPEC */
+		IMG_UINT64 uiRangeSize = sData.uiSize;
+		IMG_INT32 iCount;
+
+		/* Cannot get the CPU addr for private heaps. Display "Non-Addressable" instead. */
+		if ((PhysHeapGetFlags(psIMAData->psPhysHeap) & PHYS_HEAP_USAGE_GPU_PRIVATE) != PHYS_HEAP_USAGE_GPU_PRIVATE)
+		{
+			IMG_CPU_PHYADDR sCPURangeBase;
+			PhysHeapDevPAddrToCpuPAddr(psIMAData->pDLMHeap,
+			                           1,
+			                           &sCPURangeBase,
+			                           &sRangeBase);
+			OSSNPrintf(aszCpuAddr, ARRAY_SIZE(aszCpuAddr), CPUPHYADDR_UINT_FMTSPEC, CPUPHYADDR_FMTARG(sCPURangeBase.uiAddr));
+		}
+
+		iCount = OSSNPrintf(ppszStrBuf,
+							uiStrBufSize,
+							"                            " /* padding */
+							"CPU PA Base: %s, "
+							"GPU PA Base: 0x%08"IMG_UINT64_FMTSPECx", "
+							"Size: %"IMG_UINT64_FMTSPEC"B",
+							aszCpuAddr,
+							sRangeBase.uiAddr,
+							uiRangeSize);
+		if (!(0 < iCount && iCount < (IMG_INT32)uiStrBufSize))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "OSSNPrintf in %s(), "
+			                        "Heap Span print may be corrupt!", __func__));
+		}
+		return IMG_TRUE;
+	}
+	/* else end iteration and free the iter handle */
+
+	RA_IteratorRelease(*ppvIterHandle);
+
+return_false:
+	return IMG_FALSE;
+}
+
+static void
+IMAGetHeapDLMBacking(PHEAP_IMPL_DATA pvImplData,
+                     PHYS_HEAP **psDLMPhysHeap)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	*psDLMPhysHeap = psIMAData->pDLMHeap;
+}
+
+/* When uiSize > DLM PMB size, allocates a single physically contiguous Huge PMB to satisfy the import.
+ * A huge PMB is a PMB with a size larger than the DLM PMB size (1 << psIMAData->uiLog2PMBSize).
+ * The requested import size will be rounded up to a multiple of the DLM PMB size. */
+static PVRSRV_ERROR IMAImportDLMAllocHuge(RA_PERARENA_HANDLE hArenaHandle,
+                                          RA_LENGTH_T uiSize,
+                                          RA_FLAGS_T uiFlags,
+                                          RA_LENGTH_T uBaseAlignment,
+                                          const IMG_CHAR *pszAnnotation,
+                                          RA_IMPORT *psImport)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*) hArenaHandle;
+	PHYS_HEAP *psPhysHeap = psIMAData->pDLMHeap;
+	PMB *pPMB;
+	IMG_UINT64 uiPMBSizeBytes;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+	PVR_ASSERT(psIMAData->uiLog2PMBSize != 0);
+	/* Round up the allocation size to the nearest multiple of the
+	 * DLM PMB size (1 << psIMAData->uiLog2PMBSize).
+	 * If this is greater than DLM PMB size, a huge PMB will be imported. */
+	uiPMBSizeBytes = PVR_ALIGN(uiSize, IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize);
+
+	eError = PhysHeapCreatePMB(psPhysHeap,
+	                           uiPMBSizeBytes,
+	                           pszAnnotation,
+	                           &pPMB,
+	                           &psImport->base,
+	                           &psImport->uSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreatePMB");
+
+	psImport->hPriv = pPMB;
+
+	return PVRSRV_OK;
+}
+
+/* When uiSize <= DLM PMB size, this returns a single PMB, otherwise
+ * it requests multiple PMBs from the connected DLM heap.
+ * The PMBs will be RA_Free'd once there are no busy segments inside of the span.
+ * This will trigger the normal imp_free callback.
+ * This strategy relies on RA_POLICY_ALLOC_ALLOW_NONCONTIG
+ * and that all spans in the IMA heap are all the same size of 1 PMB, otherwise
+ * we might add a span that is not used, invalidating the invariants of the RA. */
+static PVRSRV_ERROR IMAImportDLMAllocMulti(RA_PERARENA_HANDLE hArenaHandle,
+                                           RA_LENGTH_T uiSize,
+                                           RA_FLAGS_T uiFlags,
+                                           RA_LENGTH_T uBaseAlignment,
+                                           const IMG_CHAR *pszAnnotation,
+                                           IMG_UINT32 *puiImportsCount,
+                                           RA_IMPORT **ppsImports)
+{
+	PVRSRV_ERROR eError;
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*) hArenaHandle;
+	PHYS_HEAP *psPhysHeap = psIMAData->pDLMHeap;
+
+	IMG_UINT32 i;
+	IMG_UINT32 uiPMBCount;
+
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+	PVR_ASSERT(psIMAData->uiLog2PMBSize != 0);
+
+	/* Round up the allocation size to the nearest multiple of the
+	 * DLM PMB size (1 << psIMAData->uiLog2PMBSize). */
+	uiSize = PVR_ALIGN(uiSize, IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize);
+	/* Get the number of PMBs required to make the allocation. */
+	uiPMBCount = uiSize >> psIMAData->uiLog2PMBSize;
+
+	/* Try use the provided ppsImports array instead of allocating a new array
+	 * if the array is large enough to hold all the new PMBs. */
+	if (uiPMBCount > *puiImportsCount)
+	{
+		*ppsImports = OSAllocMem(uiPMBCount * sizeof(**ppsImports));
+		PVR_LOG_RETURN_IF_NOMEM(ppsImports, "ppsImports");
+	}
+	*puiImportsCount = uiPMBCount;
+
+	/* Create the PMB that will be become the spans. */
+	for (i = 0; i < uiPMBCount; i++)
+	{
+		RA_IMPORT *psImport = &(*ppsImports)[i];
+		eError = PhysHeapCreatePMB(psPhysHeap,
+		                           IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize,
+		                           pszAnnotation,
+		                           (PMB**) &psImport->hPriv,
+		                           &psImport->base,
+		                           &psImport->uSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreatePMB", err_FreePMBs);
+	}
+
+	return PVRSRV_OK;
+
+/* Free i PMBs from the array. */
+err_FreePMBs:
+	uiPMBCount = i;
+	for (i = 0; i < uiPMBCount; i++)
+	{
+		PMBDestroy((*ppsImports)[i].hPriv);
+	}
+	return eError;
+}
+
+static void IMAImportDLMFree(RA_PERARENA_HANDLE hArenaHandle,
+                             RA_BASE_T uiBase,
+                             RA_PERISPAN_HANDLE hPriv)
+{
+	PMB *pPMB = (PMB*) hPriv;
+	PVR_ASSERT(pPMB != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(hArenaHandle);
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+
+	PMBDestroy(pPMB);
+}
+
+static void
+FreeReservedMemory(PHYSMEM_IMA_DATA *psIMAData)
+{
+	IMG_UINT32 i;
+	if (psIMAData->ppsReservedPMBs)
+	{
+		for (i = 0; i < psIMAData->uiReservedPMBs; i++)
+		{
+			/* The PMBs will be RA_Free'd by RA_Delete,
+			 * so it doesn't need to be done manually */
+			PMBDestroy(psIMAData->ppsReservedPMBs[i]);
+		}
+		OSFreeMem(psIMAData->ppsReservedPMBs);
+	}
+}
+
+static PVRSRV_ERROR
+AllocateReservedMemory(PHYSMEM_IMA_DATA *psIMAData,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       IMG_UINT32 uiReservedPMBs)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bSuccess;
+	IMG_UINT32 i;
+
+	IMG_UINT64 uiPMBSize = IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize;
+	IMG_UINT32 uiPMBCount = uiReservedPMBs;
+
+	RA_BASE_T uiBase;
+	RA_LENGTH_T uiReserveActSize;
+
+	if (uiReservedPMBs == 0)
+	{
+		psIMAData->uiReservedPMBs = 0;
+		psIMAData->ppsReservedPMBs = NULL;
+		return PVRSRV_OK;
+	}
+
+	/* If we do not support non-contiguous, we can allocate a single huge PMB.
+	 * We cannot do this for non-contiguous as IMAImportDLMAllocMulti requires all spans in the IMA-RA
+	 * to be exactly the same: 1 << psIMAData->uiLog2PMBSize. */
+	if ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		uiPMBSize *= uiPMBCount;
+		uiPMBCount = 1;
+	}
+
+	/* Count the number of PMBs that have been successfully created. */
+	psIMAData->uiReservedPMBs = 0;
+	psIMAData->ppsReservedPMBs = OSAllocMem(uiPMBCount * sizeof(*psIMAData->ppsReservedPMBs));
+	PVR_LOG_RETURN_IF_NOMEM(psIMAData->ppsReservedPMBs, "OSAllocMem");
+
+	for (i = 0; i < uiPMBCount; i++)
+	{
+		eError = PhysHeapCreatePMB(psIMAData->pDLMHeap,
+		                           uiPMBSize,
+		                           "PMB Reserved",
+		                           &psIMAData->ppsReservedPMBs[i],
+		                           &uiBase,
+		                           &uiReserveActSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreatePMB", err_FreeReserved);
+
+		psIMAData->uiReservedPMBs++;
+
+		bSuccess = RA_Add(psIMAData->psRA,
+		                  uiBase,
+		                  uiReserveActSize,
+		                  0,
+		                  (RA_PERISPAN_HANDLE) psIMAData->ppsReservedPMBs[i]);
+		PVR_LOG_GOTO_IF_FALSE(bSuccess, "RA_Add", err_AddFailed);
+	}
+
+	return PVRSRV_OK;
+
+err_AddFailed:
+    eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+err_FreeReserved:
+	FreeReservedMemory(psIMAData);
+	return eError;
+}
+
+static PVRSRV_ERROR
+CreateIMAArena(PHYSMEM_IMA_DATA *psIMAData,
+               IMG_CHAR *pszLabel,
+               PHYS_HEAP_POLICY uiPolicy,
+               IMG_UINT32 uiReservedPMBs)
+{
+	/* In practice an IMA heap only differs from LMA in the fact it can import more memory
+	 * when it has expended its current extent. */
+
+	/* If non contiguous mapping is available then we should allow that for an IMA heap.*/
+	IMG_UINT32 ui32RAPolicy =
+	    ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	    ? RA_POLICY_ALLOC_ALLOW_NONCONTIG : RA_POLICY_DEFAULT;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psIMAData != NULL);
+
+	if ((ui32RAPolicy & RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == RA_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		psIMAData->psRA = RA_CreateMulti(pszLabel,
+		                                 OSGetPageShift(),
+		                                 RA_LOCKCLASS_1,
+		                                 IMAImportDLMAllocMulti,
+		                                 IMAImportDLMFree,
+		                                 psIMAData,
+		                                 ui32RAPolicy);
+	}
+	else
+	{
+		psIMAData->psRA = RA_Create(pszLabel,
+		                            OSGetPageShift(),
+		                            RA_LOCKCLASS_1,
+		                            IMAImportDLMAllocHuge,
+		                            IMAImportDLMFree,
+		                            psIMAData,
+		                            ui32RAPolicy);
+	}
+	PVR_LOG_RETURN_IF_NOMEM(psIMAData->psRA, "RA_Create");
+
+	eError = AllocateReservedMemory(psIMAData, uiPolicy, uiReservedPMBs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "AllocateReservedMemory", err_ra_free);
+
+	return PVRSRV_OK;
+
+err_ra_free:
+	RA_Delete(psIMAData->psRA);
+	psIMAData->psRA = NULL;
+	return eError;
+}
+
+static void
+DestroyIMAArena(PHYSMEM_IMA_DATA *psIMAData)
+{
+	PVR_ASSERT(psIMAData != NULL);
+
+	/* Locked imports should be implicitly freed has a part of
+	 * the RA_Delete assuming there are no allocations remaining on
+	 * the import
+	 */
+	FreeReservedMemory(psIMAData);
+
+	/* Remove RAs and RA names for local card memory */
+	if (psIMAData->psRA)
+	{
+		RA_Delete(psIMAData->psRA);
+		psIMAData->psRA = NULL;
+	}
+}
+
+static void
+IMADestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+
+	DestroyIMAArena(pvImplData);
+
+	OSFreeMem(psIMAData);
+}
+
+static PVRSRV_ERROR
+PhysmemGetArenaIMA(PHYS_HEAP *psPhysHeap,
+                   RA_ARENA **ppsArena)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	PVR_LOG_RETURN_IF_FALSE(psIMAData != NULL, "psIMAData", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	*ppsArena = psIMAData->psRA;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+IMAPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap,
+                       size_t uiSize,
+                       PG_HANDLE *psMemHandle,
+                       IMG_DEV_PHYADDR *psDevPAddr,
+                       IMG_PID uiPid)
+{
+	PVRSRV_ERROR eError;
+
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+
+	eError = PhysmemGetArenaIMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLocalMem");
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = (1 << ui32Log2NumPages) * OSGetPageSize();
+
+	eError = RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                    psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "LocalDoPhyContigPagesAlloc");
+
+	return eError;
+}
+
+static void
+IMAPhyContigPagesFree(PHYS_HEAP *psPhysHeap,
+                      PG_HANDLE *psMemHandle)
+{
+	RA_ARENA	*pArena;
+
+	PhysmemGetArenaIMA(psPhysHeap, &pArena);
+
+	RAMemDoPhyContigPagesFree(pArena,
+	                          psMemHandle);
+}
+
+static PVRSRV_ERROR
+IMAPhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                               CONNECTION_DATA *psConnection,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               IMG_UINT32 ui32NumPhysChunks,
+                               IMG_UINT32 ui32NumVirtChunks,
+                               IMG_UINT32 *pui32MappingTable,
+                               IMG_UINT32 uiLog2AllocPageSize,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               const IMG_CHAR *pszAnnotation,
+                               IMG_PID uiPid,
+                               PMR **ppsPMRPtr,
+                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	RA_ARENA *pArena;
+
+	eError = PhysmemGetArenaIMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaIMA");
+
+	eError = PhysmemNewRAMemRamBackedPMR(psPhysHeap,
+	                                     pArena,
+	                                     psConnection,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewLocalRamBackedPMR");
+
+	return PVRSRV_OK;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncsIMA =
+{
+	.pfnDestroyData = &IMADestroyImplData,
+	.pfnGetDevPAddr = &IMAGetDevPAddr,
+	.pfnGetCPUPAddr = &IMAGetCPUPAddr,
+	.pfnGetSize = &IMAGetSize,
+	.pfnGetPageShift = &RAMemGetPageShift,
+	.pfnGetFactoryMemStats = &IMAPhysmemGetRAMemRamMemStats,
+	.pfnGetHeapSpansStringIter = &IMAGetHeapSpansStringIter,
+	.pfnGetHeapDLMBacking = &IMAGetHeapDLMBacking,
+	.pfnCreatePMR = &IMAPhysmemNewRAMemRamBackedPMR,
+	.pfnPagesAlloc = &IMAPhyContigPagesAlloc,
+	.pfnPagesFree = &IMAPhyContigPagesFree,
+	.pfnPagesMap = &RAMemPhyContigPagesMap,
+	.pfnPagesUnMap = &RAMemPhyContigPagesUnmap,
+	.pfnPagesClean = &RAMemPhyContigPagesClean,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapIMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP *psDLMHeap,
+                     IMG_UINT32 uiLog2PMBSize,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_IMA_DATA *psIMAData;
+	PHYS_HEAP *psPhysHeap;
+	IMG_UINT32 uiPMBStartingMultiple = psConfig->uConfig.sIMA.ui32PMBStartingMultiple;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConfig != NULL, "psConfig");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_IMA);
+
+	psIMAData = OSAllocMem(sizeof(*psIMAData));
+	PVR_LOG_RETURN_IF_NOMEM(psIMAData, "OSAllocMem");
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psIMAData,
+							&_sPHEAPImplFuncsIMA,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", err_free_ima_data);
+
+	psIMAData->psPhysHeap = psPhysHeap;
+	psIMAData->pDLMHeap = psDLMHeap;
+	psIMAData->uiLog2PMBSize = uiLog2PMBSize;
+
+	eError = CreateIMAArena(psIMAData,
+	                        pszLabel,
+	                        uiPolicy,
+	                        uiPMBStartingMultiple);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateIMAArena", err_free_physheap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+err_free_physheap:
+	PhysHeapDestroy(psPhysHeap);
+err_free_ima_data:
+	OSFreeMem(psIMAData);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_lma.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_lma.c
new file mode 100644
index 000000000000..3075166b5366
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_lma.c
@@ -0,0 +1,523 @@
+/*************************************************************************/ /*!
+@File           physmem_lma.c
+@Title          Local card memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for local card memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "physmem_lma.h"
+#include "pvr_debug.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "device.h"
+#include "osfunc.h"
+#include "physmem_ramem.h"
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "rgxutils.h"
+#endif
+
+typedef struct PHYSMEM_LMA_DATA_TAG {
+	RA_ARENA           *psRA;
+	IMG_CPU_PHYADDR    sStartAddr;
+	IMG_DEV_PHYADDR    sCardBase;
+	IMG_UINT64         uiSize;
+} PHYSMEM_LMA_DATA;
+
+/*
+ * This function will set the psDevPAddr to whatever the system layer
+ * has set it for the referenced heap.
+ * It will not fail if the psDevPAddr is invalid.
+ */
+static PVRSRV_ERROR
+_GetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*psDevPAddr = psLMAData->sCardBase;
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This function will set the psCpuPAddr to whatever the system layer
+ * has set it for the referenced heap.
+ * It will not fail if the psCpuPAddr is invalid.
+ */
+static PVRSRV_ERROR
+_GetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*psCpuPAddr = psLMAData->sStartAddr;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_GetSize(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *puiSize)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*puiSize = psLMAData->uiSize;
+
+	return PVRSRV_OK;
+}
+
+static void PhysmemGetRAMemRamMemStats(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *pui64TotalSize,
+		 IMG_UINT64 *pui64FreeSize)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psLMAData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+static PVRSRV_ERROR
+PhysmemGetArenaLMA(PHYS_HEAP *psPhysHeap,
+				   RA_ARENA **ppsArena)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	PVR_LOG_RETURN_IF_FALSE(psLMAData != NULL, "psLMAData", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	*ppsArena = psLMAData->psRA;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_CreateArenas(PHEAP_IMPL_DATA pvImplData, IMG_CHAR *pszLabel, PHYS_HEAP_POLICY uiPolicy)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	IMG_UINT32 ui32RAPolicy =
+	    ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	    ? RA_POLICY_ALLOC_ALLOW_NONCONTIG : RA_POLICY_DEFAULT;
+
+	psLMAData->psRA = RA_Create_With_Span(pszLabel,
+	                             OSGetPageShift(),
+	                             psLMAData->sStartAddr.uiAddr,
+	                             psLMAData->sCardBase.uiAddr,
+	                             psLMAData->uiSize,
+	                             ui32RAPolicy);
+	PVR_LOG_RETURN_IF_NOMEM(psLMAData->psRA, "RA_Create_With_Span");
+
+	return PVRSRV_OK;
+}
+
+static void
+_DestroyArenas(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	/* Remove RAs and RA names for local card memory */
+	if (psLMAData->psRA)
+	{
+		RA_Delete(psLMAData->psRA);
+		psLMAData->psRA = NULL;
+	}
+}
+
+static void
+_DestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	_DestroyArenas(pvImplData);
+
+	OSFreeMem(psLMAData);
+}
+
+struct _PHYS_HEAP_ITERATOR_ {
+	PHYS_HEAP *psPhysHeap;
+	RA_ARENA_ITERATOR *psRAIter;
+
+	IMG_UINT64 uiTotalSize;
+	IMG_UINT64 uiInUseSize;
+};
+
+PVRSRV_ERROR LMA_HeapIteratorCreate(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_PHYS_HEAP ePhysHeap,
+                                    PHYS_HEAP_ITERATOR **ppsIter)
+{
+	PVRSRV_ERROR eError;
+	PHYSMEM_LMA_DATA *psLMAData;
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	PHYS_HEAP *psPhysHeap = NULL;
+	RA_USAGE_STATS sStats;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsIter != NULL, "ppsIter");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	eError = PhysHeapAcquireByID(ePhysHeap, psDevNode, &psPhysHeap);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquireByID");
+
+	PVR_LOG_GOTO_IF_FALSE(PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA,
+	                      "PhysHeap must be of LMA type", release_heap);
+
+	psLMAData = (PHYSMEM_LMA_DATA *) PhysHeapGetImplData(psPhysHeap);
+
+	psHeapIter = OSAllocMem(sizeof(*psHeapIter));
+	PVR_LOG_GOTO_IF_NOMEM(psHeapIter, eError, release_heap);
+
+	psHeapIter->psPhysHeap = psPhysHeap;
+	psHeapIter->psRAIter = RA_IteratorAcquire(psLMAData->psRA, IMG_FALSE);
+	PVR_LOG_GOTO_IF_NOMEM(psHeapIter->psRAIter, eError, free_heap_iter);
+
+	/* get heap usage */
+	RA_Get_Usage_Stats(psLMAData->psRA, &sStats);
+
+	psHeapIter->uiTotalSize = sStats.ui64TotalArenaSize;
+	psHeapIter->uiInUseSize = sStats.ui64TotalArenaSize - sStats.ui64FreeArenaSize;
+
+	*ppsIter = psHeapIter;
+
+	return PVRSRV_OK;
+
+free_heap_iter:
+	OSFreeMem(psHeapIter);
+
+release_heap:
+	PhysHeapRelease(psPhysHeap);
+
+	return eError;
+}
+
+void LMA_HeapIteratorDestroy(PHYS_HEAP_ITERATOR *psIter)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psHeapIter != NULL, "psHeapIter is NULL");
+
+	PhysHeapRelease(psHeapIter->psPhysHeap);
+	RA_IteratorRelease(psHeapIter->psRAIter);
+	OSFreeMem(psHeapIter);
+}
+
+PVRSRV_ERROR LMA_HeapIteratorReset(PHYS_HEAP_ITERATOR *psIter)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeapIter != NULL, "ppsIter");
+
+	RA_IteratorReset(psHeapIter->psRAIter);
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL LMA_HeapIteratorNext(PHYS_HEAP_ITERATOR *psIter,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              IMG_UINT64 *puiSize)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+	RA_ITERATOR_DATA sData = {0};
+
+	if (psHeapIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "psHeapIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	if (!RA_IteratorNext(psHeapIter->psRAIter, &sData))
+	{
+		return IMG_FALSE;
+	}
+
+	PVR_ASSERT(sData.uiSize != 0);
+
+	psDevPAddr->uiAddr = sData.uiAddr;
+	*puiSize = sData.uiSize;
+
+	return IMG_TRUE;
+}
+
+PVRSRV_ERROR LMA_HeapIteratorGetHeapStats(PHYS_HEAP_ITERATOR *psIter,
+                                          IMG_UINT64 *puiTotalSize,
+                                          IMG_UINT64 *puiInUseSize)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeapIter != NULL, "psHeapIter");
+
+	*puiTotalSize = psHeapIter->uiTotalSize;
+	*puiInUseSize = psHeapIter->uiInUseSize;
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+static PVRSRV_ERROR
+LMA_PhyContigPagesAllocGPV(PHYS_HEAP *psPhysHeap,
+                           size_t uiSize,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_UINT32 ui32OSid,
+                           IMG_PID uiPid)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	PVR_ASSERT(ui32OSid < GPUVIRT_VALIDATION_NUM_OS);
+	if (ui32OSid >= GPUVIRT_VALIDATION_NUM_OS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Arena index %u defaulting to 0",
+		        __func__, ui32OSid));
+		ui32OSid = 0;
+	}
+
+	pArena = psDevNode->psOSidSubArena[ui32OSid];
+
+	if (psMemHandle->uiOSid != ui32OSid)
+	{
+		PVR_LOG(("%s: Unexpected OSid value %u - expecting %u", __func__,
+		        psMemHandle->uiOSid, ui32OSid));
+	}
+
+	psMemHandle->uiOSid = ui32OSid;		/* For Free() use */
+
+	eError =  RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                     psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "RAMemDoPhyContigPagesAlloc");
+
+	return eError;
+}
+#endif
+
+static PVRSRV_ERROR
+LMA_PhyContigPagesAlloc(PHYS_HEAP *psPhysHeap,
+                         size_t uiSize,
+                         PG_HANDLE *psMemHandle,
+                         IMG_DEV_PHYADDR *psDevPAddr,
+                         IMG_PID uiPid)
+{
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	IMG_UINT32 ui32OSid = 0;
+	return LMA_PhyContigPagesAllocGPV(psPhysHeap, uiSize, psMemHandle, psDevPAddr,
+									  ui32OSid, uiPid);
+#else
+	PVRSRV_ERROR eError;
+
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+
+	eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLMA");
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	eError = RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                    psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "RAMemDoPhyContigPagesAlloc");
+
+	return eError;
+#endif
+}
+
+static void
+LMA_PhyContigPagesFree(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle)
+{
+	RA_ARENA	*pArena;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	RA_BASE_T uiCardAddr = (RA_BASE_T) psMemHandle->u.ui64Handle;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	IMG_UINT32	ui32OSid = psMemHandle->uiOSid;
+
+	/*
+	 * The Arena ID is set by the originating allocation, and maintained via
+	 * the call stacks into this function. We have a limited range of IDs
+	 * and if the passed value falls outside this we simply treat it as a
+	 * 'global' arena ID of 0. This is where all default OS-specific allocations
+	 * are created.
+	 */
+	PVR_ASSERT(ui32OSid < GPUVIRT_VALIDATION_NUM_OS);
+	if (ui32OSid >= GPUVIRT_VALIDATION_NUM_OS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Arena index %u PhysAddr 0x%"
+		         IMG_UINT64_FMTSPECx " Reverting to Arena 0", __func__,
+		         ui32OSid, uiCardAddr));
+		/*
+		 * No way of determining what we're trying to free so default to the
+		 * global default arena index 0.
+		 */
+		ui32OSid = 0;
+	}
+
+	pArena = psDevNode->psOSidSubArena[ui32OSid];
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: (GPU Virtualisation) Freeing 0x%"
+	        IMG_UINT64_FMTSPECx ", Arena %u", __func__,
+	        uiCardAddr, ui32OSid));
+
+#else
+	PVRSRV_ERROR eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysmemGetArenaLMA");
+#endif
+
+	RAMemDoPhyContigPagesFree(pArena, psMemHandle);
+}
+
+static PVRSRV_ERROR
+LMAPhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                               CONNECTION_DATA *psConnection,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               IMG_UINT32 ui32NumPhysChunks,
+                               IMG_UINT32 ui32NumVirtChunks,
+                               IMG_UINT32 *pui32MappingTable,
+                               IMG_UINT32 uiLog2AllocPageSize,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               const IMG_CHAR *pszAnnotation,
+                               IMG_PID uiPid,
+                               PMR **ppsPMRPtr,
+                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	RA_ARENA *pArena;
+
+	eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLMA");
+
+	eError = PhysmemNewRAMemRamBackedPMR(psPhysHeap,
+	                                     pArena,
+	                                     psConnection,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewLocalRamBackedPMR");
+
+	return PVRSRV_OK;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = &_DestroyImplData,
+	.pfnGetDevPAddr = &_GetDevPAddr,
+	.pfnGetCPUPAddr = &_GetCPUPAddr,
+	.pfnGetSize = &_GetSize,
+	.pfnGetPageShift = &RAMemGetPageShift,
+	.pfnGetFactoryMemStats = &PhysmemGetRAMemRamMemStats,
+	.pfnCreatePMR = &LMAPhysmemNewRAMemRamBackedPMR,
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	.pfnPagesAllocGPV = &LMA_PhyContigPagesAllocGPV,
+#endif
+	.pfnPagesAlloc = &LMA_PhyContigPagesAlloc,
+	.pfnPagesFree = &LMA_PhyContigPagesFree,
+	.pfnPagesMap = &RAMemPhyContigPagesMap,
+	.pfnPagesUnMap = &RAMemPhyContigPagesUnmap,
+	.pfnPagesClean = &RAMemPhyContigPagesClean,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapLMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_LMA_DATA *psLMAData;
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_LMA ||
+	           psConfig->eType == PHYS_HEAP_TYPE_DMA);
+
+	psLMAData = OSAllocMem(sizeof(*psLMAData));
+	PVR_LOG_RETURN_IF_NOMEM(psLMAData, "OSAllocMem");
+
+	psLMAData->sStartAddr = PhysHeapConfigGetStartAddr(psConfig);
+	psLMAData->sCardBase = PhysHeapConfigGetCardBase(psConfig);
+	psLMAData->uiSize = PhysHeapConfigGetSize(psConfig);
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psLMAData,
+							&_sPHEAPImplFuncs,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", err_free_lma_data);
+
+	eError = _CreateArenas(psLMAData, pszLabel, uiPolicy);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CreateArenas", err_free_physheap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+err_free_physheap:
+	PhysHeapDestroy(psPhysHeap);
+err_free_lma_data:
+	OSFreeMem(psLMAData);
+	return eError;
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_osmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_osmem.c
new file mode 100644
index 000000000000..cef1ea620821
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_osmem.c
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File           physmem_osmem.c
+@Title          OS Memory PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                OS memory PMR factory API that must be defined so that the
+                common & device layer code in the Services Server can allocate
+                new PMRs back with pages from the OS page allocator. Applicable
+                for UMA based platforms, such platforms must implement this API
+                in the OS Porting layer, in the "env" directory for that
+                system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/* services/server/include/ */
+#include "physheap.h"
+#include "osfunc.h"
+#include "physmem_osmem.h"
+
+
+static IMG_UINT32 PhysHeapOSGetPageShift(void)
+{
+	return (IMG_UINT32)OSGetPageShift();
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = NULL,
+	.pfnGetFactoryMemStats = PhysmemGetOSRamMemStats,
+	.pfnCreatePMR = PhysmemNewOSRamBackedPMR,
+	.pfnPagesAlloc = &OSPhyContigPagesAlloc,
+	.pfnPagesFree = &OSPhyContigPagesFree,
+	.pfnPagesMap = &OSPhyContigPagesMap,
+	.pfnPagesUnMap = &OSPhyContigPagesUnmap,
+	.pfnPagesClean = &OSPhyContigPagesClean,
+	.pfnGetPageShift = &PhysHeapOSGetPageShift,
+};
+
+
+PVRSRV_ERROR
+PhysmemCreateHeapOSMEM(PVRSRV_DEVICE_NODE *psDevNode,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       PHYS_HEAP_CONFIG *psConfig,
+                       PHYS_HEAP **ppsPhysHeap)
+{
+	return PhysHeapCreate(psDevNode,
+	                      psConfig,
+	                      uiPolicy,
+	                      NULL,
+	                      &_sPHEAPImplFuncs,
+	                      ppsPhysHeap);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ramem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ramem.c
new file mode 100644
index 000000000000..6597042770d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ramem.c
@@ -0,0 +1,2609 @@
+/*************************************************************************/ /*!
+@File           physmem_ramem.c
+@Title          Resource allocator managed PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                RA managed memory PMR factory API that is shared between local
+                physheap implementations (LMA & IMA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "physmem_ramem.h"
+#include "physheap.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "connection_server.h"
+#include "device.h"
+#include "devicemem_server_utils.h"
+#include "osfunc.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "rgx_pdump_panics.h"
+#include "pdump_km.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "rgxutils.h"
+#endif
+
+#if defined(INTEGRITY_OS)
+#include "mm.h"
+#include "integrity_memobject.h"
+#endif
+
+#include "physmem_dlm.h"
+
+
+#if defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+#define PHYSHEAP_DPF_LVL PVR_DBG_ERROR
+#else
+#define PHYSHEAP_DPF_LVL PVR_DBG_WARNING
+#endif
+
+/* Common Physheap Callback implementations */
+
+IMG_UINT32
+RAMemGetPageShift(void)
+{
+	return PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT;
+}
+
+PVRSRV_ERROR
+RAMemDoPhyContigPagesAlloc(RA_ARENA *pArena,
+                           size_t uiSize,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_PID uiPid)
+{
+	RA_BASE_T uiCardAddr = 0;
+	RA_LENGTH_T uiActualSize;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2NumPages;
+
+#if defined(DEBUG)
+	static IMG_UINT32 ui32MaxLog2NumPagesHistory = 0;
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* Firmware heaps on VZ are usually LMA, and several 64kb allocations need to
+	 * be made on Guest drivers: FwGuardPage, FwSysInit, FwConnectionCtl, FwOsInit.
+	 * Increase the maximum number to avoid driver warnings. */
+	IMG_UINT32	ui32MaxLog2NumPages = 7;	/* 128 pages => 512KB */
+#else
+	IMG_UINT32	ui32MaxLog2NumPages = 4;	/*  16 pages =>  64KB */
+#endif
+	ui32MaxLog2NumPages = MAX(ui32MaxLog2NumPages, OSGetOrder(1 << psDevNode->ui32Non4KPageSizeLog2));
+#else /* defined(DEBUG) */
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+#endif /* defined(DEBUG) */
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	eError = RA_Alloc(pArena,
+	                  uiSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                         /* No flags */
+	                  uiSize,
+	                  "RAMemPhyContigPagesAlloc",
+	                  &uiCardAddr,
+	                  &uiActualSize,
+	                  NULL);                     /* No private handle */
+
+	if (eError != PVRSRV_OK)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(pArena, &sRAStats);
+
+		PVR_DPF((PVR_DBG_ERROR,
+				"Failed to Allocate size = 0x"IMG_SIZE_FMTSPECX", align = 0x"
+				IMG_SIZE_FMTSPECX" Arena Free Space 0x%"IMG_UINT64_FMTSPECX,
+				uiSize, uiSize,	sRAStats.ui64FreeArenaSize));
+		return eError;
+	}
+
+	PVR_ASSERT(uiSize == uiActualSize);
+
+	psMemHandle->u.ui64Handle = uiCardAddr;
+	psDevPAddr->uiAddr = (IMG_UINT64) uiCardAddr;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+	                                    uiSize,
+	                                    uiCardAddr,
+	                                    uiPid);
+#else
+	{
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = psDevPAddr->uiAddr;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+									 NULL,
+									 sCpuPAddr,
+									 uiSize,
+									 uiPid
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#else /* PVRSRV_ENABLE_PROCESS_STATS */
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: (GPU Virtualisation) Allocated 0x" IMG_SIZE_FMTSPECX " at 0x%"
+	        IMG_UINT64_FMTSPECX ", Arena ID %u",
+	        __func__, uiSize, psDevPAddr->uiAddr, psMemHandle->uiOSid));
+#endif
+
+#if defined(DEBUG)
+	if (ui32Log2NumPages > ui32MaxLog2NumPages && ui32Log2NumPages > ui32MaxLog2NumPagesHistory)
+	{
+		PVR_ASSERT((ui32Log2NumPages <= ui32MaxLog2NumPages));
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ui32MaxLog2NumPages = %u, increasing to %u", __func__,
+		        ui32MaxLog2NumPages, ui32Log2NumPages ));
+		ui32MaxLog2NumPagesHistory = ui32Log2NumPages;
+	}
+#endif /* defined(DEBUG) */
+	psMemHandle->uiOrder = ui32Log2NumPages;
+
+	return eError;
+}
+
+void
+RAMemDoPhyContigPagesFree(RA_ARENA *pArena,
+                          PG_HANDLE *psMemHandle)
+{
+	RA_BASE_T uiCardAddr = (RA_BASE_T) psMemHandle->u.ui64Handle;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+	                                      (IMG_UINT64)uiCardAddr);
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+									(IMG_UINT64)uiCardAddr,
+									OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	RA_Free(pArena, uiCardAddr);
+	psMemHandle->uiOrder = 0;
+}
+
+PVRSRV_ERROR
+RAMemPhyContigPagesMap(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle,
+                       size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                       void **pvPtr)
+{
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_UINT32 ui32NumPages = (1 << psMemHandle->uiOrder);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+
+	PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPAddr, psDevPAddr);
+	*pvPtr = OSMapPhysToLin(sCpuPAddr,
+							ui32NumPages * OSGetPageSize(),
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+	PVR_RETURN_IF_NOMEM(*pvPtr);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+	                            ui32NumPages * OSGetPageSize(),
+	                            OSGetCurrentClientProcessIDKM());
+#else
+	{
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+									 *pvPtr,
+									 sCpuPAddr,
+									 ui32NumPages * OSGetPageSize(),
+									 OSGetCurrentClientProcessIDKM()
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#endif
+	return PVRSRV_OK;
+}
+
+void
+RAMemPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap,
+                         PG_HANDLE *psMemHandle,
+                         void *pvPtr)
+{
+	IMG_UINT32 ui32NumPages = (1 << psMemHandle->uiOrder);
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+		                            ui32NumPages * OSGetPageSize(),
+		                            OSGetCurrentClientProcessIDKM());
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+	                                (IMG_UINT64)(uintptr_t)pvPtr,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	OSUnMapPhysToLin(pvPtr, ui32NumPages * OSGetPageSize());
+}
+
+PVRSRV_ERROR
+RAMemPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                         PG_HANDLE *psMemHandle,
+                         IMG_UINT32 uiOffset,
+                         IMG_UINT32 uiLength)
+{
+	/* No need to flush because we map as uncached */
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+	PVR_UNREFERENCED_PARAMETER(psMemHandle);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiLength);
+
+	return PVRSRV_OK;
+}
+
+/* Local memory allocation routines */
+
+/* Assert that the conversions between the RA base type and the device
+ * physical address are safe.
+ */
+static_assert(sizeof(IMG_DEV_PHYADDR) == sizeof(RA_BASE_T),
+              "Size IMG_DEV_PHYADDR != RA_BASE_T");
+
+/* Since 0x0 is a valid DevPAddr, we rely on max 64-bit value to be an invalid
+ * page address */
+#define INVALID_PAGE_ADDR ~((IMG_UINT64)0x0)
+#define ZERO_PAGE_VALUE 0
+
+typedef struct _PMR_KERNEL_MAP_HANDLE_ {
+	void *vma;
+	void *pvKernelAddress;
+	/* uiSize has 2 uses:
+	 * In Physically contiguous case it is used to track size of the mapping
+	 * for free.
+	 * In Physically sparse case it is used to determine free path to use, single page
+	 * sparse mapping or multi page
+	 */
+	size_t uiSize;
+} PMR_KERNEL_MAPPING;
+
+typedef struct _PMR_LMALLOCARRAY_DATA_ {
+
+#define FLAG_ZERO              (0U)
+#define FLAG_POISON_ON_FREE    (1U)
+#define FLAG_POISON_ON_ALLOC   (2U)
+#define FLAG_ONDEMAND          (3U)
+#define FLAG_SPARSE            (4U)
+#define FLAG_PHYS_CONTIG       (5U)
+#define FLAG_ZOMBIE            (6U)
+
+	IMG_PID uiPid;
+
+	/*
+	 * N.B Chunks referenced in this struct commonly are
+	 * to OS page sized. But in reality it is dependent on
+	 * the uiLog2ChunkSize.
+	 * Chunks will always be one 1 << uiLog2ChunkSize in size.
+	 * */
+
+	/*
+	 * The number of chunks currently allocated in the PMR.
+	 */
+	IMG_INT32 iNumChunksAllocated;
+
+	/*
+	 * Total number of (Virtual) chunks supported by this PMR.
+	 */
+	IMG_UINT32 uiTotalNumChunks;
+
+	/* The number of chunks to next be allocated for the PMR.
+	 * This will initially be the number allocated at first alloc
+	 * but may be changed in later calls to change sparse.
+	 * It represents the number of chunks to next be allocated.
+	 * This is used to store this value because we have the ability to
+	 * defer allocation.
+	 */
+	IMG_UINT32 uiChunksToAlloc;
+
+	/*
+	 * Log2 representation of the chunksize.
+	 */
+	IMG_UINT32 uiLog2ChunkSize;
+
+	/* Physical heap and arena pointers for this allocation */
+	PHYS_HEAP* psPhysHeap;
+	RA_ARENA* psArena;
+	PVRSRV_MEMALLOCFLAGS_T uiAllocFlags;
+
+	/*
+	   Connection data for this requests' originating process. NULL for
+	   direct-bridge originating calls
+	 */
+	CONNECTION_DATA *psConnection;
+
+	/*
+	 * Allocation flags related to the pages:
+	 * Zero              - Should we Zero memory on alloc
+	 * Poison on free    - Should we Poison the memory on free.
+	 * Poison on alloc   - Should we Poison the memory on alloc.
+	 * On demand         - Is the allocation on Demand i.e Do we defer allocation to time of use.
+	 * Sparse            - Is the PMR sparse.
+	 * Phys Contig       - Is the alloc Physically contiguous
+	 * Zombie            - Is zombie
+	 * */
+	IMG_UINT32 ui32Flags;
+
+	RA_BASE_ARRAY_T aBaseArray; /* Array of RA Bases */
+
+} PMR_LMALLOCARRAY_DATA;
+
+static PVRSRV_ERROR
+_FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+             IMG_UINT32 *pui32FreeIndices,
+             IMG_UINT32 ui32FreeChunkCount);
+
+static PVRSRV_ERROR _MapPhysicalContigAlloc(PHYS_HEAP *psPhysHeap,
+                                            RA_BASE_ARRAY_T paBaseArray,
+                                            size_t uiSize,
+                                            PMR_FLAGS_T ulFlags,
+                                            PMR_KERNEL_MAPPING *psMapping)
+{
+	IMG_UINT32 ui32CPUCacheFlags;
+	PVRSRV_ERROR eError;
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+	sDevPAddr.uiAddr = RA_BASE_STRIP_GHOST_BIT(*paBaseArray);
+
+	eError = DevmemCPUCacheMode(ulFlags, &ui32CPUCacheFlags);
+	PVR_RETURN_IF_ERROR(eError);
+
+	PhysHeapDevPAddrToCpuPAddr(psPhysHeap,
+	                           1,
+	                           &sCpuPAddr,
+	                           &sDevPAddr);
+
+	psMapping->pvKernelAddress = OSMapPhysToLin(sCpuPAddr, uiSize, ui32CPUCacheFlags);
+	PVR_LOG_RETURN_IF_FALSE(psMapping->pvKernelAddress,
+	                        "OSMapPhyToLin: out of VM Mem",
+	                        PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING);
+	psMapping->vma = NULL;
+	psMapping->uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MapPhysicalSparseAlloc(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                                            RA_BASE_ARRAY_T paBaseArray,
+                                            size_t uiSize,
+                                            PMR_KERNEL_MAPPING *psMapping)
+{
+	IMG_UINT32 uiChunkCount = uiSize >> psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_CPU_PHYADDR uiPages[PMR_MAX_TRANSLATION_STACK_ALLOC], *puiPages;
+	PVRSRV_ERROR eError;
+	size_t uiPageShift = OSGetPageShift();
+	IMG_UINT32 uiOSPagesPerChunkShift = psLMAllocArrayData->uiLog2ChunkSize - uiPageShift;
+	IMG_UINT32 uiOSPageCount = uiChunkCount << uiOSPagesPerChunkShift;
+
+	if (uiOSPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPages = OSAllocZMem(sizeof(IMG_CPU_PHYADDR) * uiOSPageCount);
+		PVR_RETURN_IF_NOMEM(puiPages);
+	}
+	else
+	{
+		puiPages = &uiPages[0];
+	}
+
+	if (uiOSPagesPerChunkShift == 0)
+	{
+		IMG_UINT32 i;
+		PhysHeapDevPAddrToCpuPAddr(psLMAllocArrayData->psPhysHeap,
+								   uiChunkCount,
+								   puiPages,
+								   (IMG_DEV_PHYADDR *)paBaseArray);
+
+		/* If the ghost bit is present then the addrs returned will be off by 1
+		 * Strip the ghost bit to correct to real page aligned addresses.
+		 * */
+		for (i = 0; i < uiChunkCount; i++)
+		{
+			puiPages[i].uiAddr = RA_BASE_STRIP_GHOST_BIT(puiPages[i].uiAddr);
+		}
+	}
+	else
+	{
+		IMG_UINT32 i = 0, j = 0, index = 0;
+		for (i = 0; i < uiChunkCount; i++)
+		{
+			IMG_UINT32 ui32OSPagesPerDeviceChunk = (1 << uiOSPagesPerChunkShift);
+			IMG_DEV_PHYADDR uiDevAddr;
+			uiDevAddr.uiAddr = RA_BASE_STRIP_GHOST_BIT(paBaseArray[i]);
+			for (j = 0; j < ui32OSPagesPerDeviceChunk; j++)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psLMAllocArrayData->psPhysHeap,
+										   1,
+										   &puiPages[index],
+										   &uiDevAddr);
+				uiDevAddr.uiAddr += (1ULL << uiPageShift);
+				index++;
+			}
+		}
+	}
+
+	eError = OSMapPhysArrayToLin(puiPages,
+	                             uiOSPageCount,
+	                             &psMapping->pvKernelAddress,
+	                             &psMapping->vma);
+	if (eError == PVRSRV_OK)
+	{
+		psMapping->uiSize = uiSize;
+	}
+
+	if (puiPages != &uiPages[0])
+	{
+		OSFreeMem(puiPages);
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR _MapPMRKernel(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                                  RA_BASE_ARRAY_T paBaseArray,
+                                  size_t uiSize,
+                                  PMR_FLAGS_T ulFlags,
+                                  PMR_KERNEL_MAPPING *psMapping)
+{
+	PVRSRV_ERROR eError;
+	PHYS_HEAP *psPhysHeap = psLMAllocArrayData->psPhysHeap;
+	if (!BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* Physically Contig */
+		if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			eError = _MapPhysicalContigAlloc(psPhysHeap,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 ulFlags,
+			                                 psMapping);
+		}
+		/* Physically Sparse */
+		else
+		{
+			eError = _MapPhysicalSparseAlloc(psLMAllocArrayData,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 psMapping);
+		}
+	}
+	else
+	{
+		/* Sparse Alloc Single Chunk */
+		if (uiSize == IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize))
+		{
+			eError = _MapPhysicalContigAlloc(psPhysHeap,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 ulFlags,
+			                                 psMapping);
+		}
+		/* Sparse Alloc Multi Chunk */
+		else
+		{
+			eError = _MapPhysicalSparseAlloc(psLMAllocArrayData,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 psMapping);
+		}
+	}
+
+	return eError;
+}
+
+static void _UnMapPhysicalContigAlloc(PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	OSUnMapPhysToLin(psKernelMapping->pvKernelAddress, psKernelMapping->uiSize);
+}
+
+static void _UnMapPhysicalSparseAlloc(PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	OSUnMapPhysArrayToLin(psKernelMapping->pvKernelAddress,
+	                   psKernelMapping->vma);
+}
+
+static void _UnMapPMRKernel(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                            PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	if (!BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* Physically Contig */
+		if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			_UnMapPhysicalContigAlloc(psKernelMapping);
+		}
+		/* Physically Sparse */
+		else
+		{
+			_UnMapPhysicalSparseAlloc(psKernelMapping);
+		}
+	}
+	else
+	{
+		/* Sparse Alloc Single Chunk */
+		if (psKernelMapping->uiSize == IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize))
+		{
+			_UnMapPhysicalContigAlloc(psKernelMapping);
+		}
+		/* Sparse Alloc Multi Chunk */
+		else
+		{
+			_UnMapPhysicalSparseAlloc(psKernelMapping);
+		}
+	}
+}
+
+static PVRSRV_ERROR
+_PhysPgMemSet(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+              RA_BASE_ARRAY_T paBaseArray,
+              size_t uiSize,
+              IMG_BYTE ui8SetValue)
+{
+	PVRSRV_ERROR eError;
+	PMR_KERNEL_MAPPING sKernelMapping;
+
+	eError = _MapPMRKernel(psLMAllocArrayData,
+	                       paBaseArray,
+	                       uiSize,
+	                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+	                       &sKernelMapping);
+	PVR_GOTO_IF_ERROR(eError, map_failed);
+
+	OSCachedMemSetWMB(sKernelMapping.pvKernelAddress, ui8SetValue, uiSize);
+
+	_UnMapPMRKernel(psLMAllocArrayData, &sKernelMapping);
+
+	return PVRSRV_OK;
+
+map_failed:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to poison/zero allocation"));
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPageArray(PMR_SIZE_T uiSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 uiLog2AllocPageSize,
+                  IMG_UINT32 ui32Flags,
+                  PHYS_HEAP* psPhysHeap,
+                  RA_ARENA* pArena,
+                  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+                  IMG_PID uiPid,
+                  PMR_LMALLOCARRAY_DATA **ppsPageArrayDataPtr,
+                  CONNECTION_DATA *psConnection)
+{
+	PMR_LMALLOCARRAY_DATA *psPageArrayData = NULL;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiNumPages;
+
+	PVR_ASSERT(!BIT_ISSET(ui32Flags, FLAG_ZERO) || !BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC));
+	PVR_ASSERT(OSGetPageShift() <= uiLog2AllocPageSize);
+
+	/* Use of cast below is justified by the assertion that follows to
+	prove that no significant bits have been truncated */
+	uiNumPages = (IMG_UINT32)(((uiSize - 1) >> uiLog2AllocPageSize) + 1);
+	PVR_ASSERT(((PMR_SIZE_T)uiNumPages << uiLog2AllocPageSize) == uiSize);
+
+	psPageArrayData = OSAllocMem(sizeof(PMR_LMALLOCARRAY_DATA) + IMG_FLEX_ARRAY_SIZE(sizeof(RA_BASE_T), uiNumPages));
+	PVR_GOTO_IF_NOMEM(psPageArrayData, eError, errorOnAllocArray);
+
+	if (BIT_ISSET(ui32Flags, FLAG_SPARSE))
+	{
+		/* Since no pages are allocated yet, initialise page addresses to INVALID_PAGE_ADDR */
+		OSCachedMemSet(psPageArrayData->aBaseArray,
+					   0xFF,
+					   sizeof(RA_BASE_T) *
+					   uiNumPages);
+	}
+	else
+	{
+		/* Base pointers have been allocated for the full PMR in case we require a non
+		 * physically contiguous backing for the virtually contiguous allocation but the most
+		 * common case will be contiguous and so only require the first Base to be present
+		 */
+		psPageArrayData->aBaseArray[0] = INVALID_BASE_ADDR;
+	}
+
+	psPageArrayData->uiTotalNumChunks = uiNumPages;
+	psPageArrayData->uiChunksToAlloc = BIT_ISSET(ui32Flags, FLAG_SPARSE) ? ui32NumPhysChunks : uiNumPages;
+	psPageArrayData->uiLog2ChunkSize = uiLog2AllocPageSize;
+
+	psPageArrayData->psConnection = psConnection;
+	psPageArrayData->uiPid = uiPid;
+	psPageArrayData->iNumChunksAllocated = 0;
+	psPageArrayData->ui32Flags = ui32Flags;
+	psPageArrayData->psPhysHeap = psPhysHeap;
+	psPageArrayData->psArena = pArena;
+	psPageArrayData->uiAllocFlags = uiAllocFlags;
+
+	*ppsPageArrayDataPtr = psPageArrayData;
+
+	return PVRSRV_OK;
+
+/*
+  error exit path follows:
+*/
+
+errorOnAllocArray:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPagesContig(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiChunksToAlloc, uiLog2ChunkSize);
+	IMG_BOOL bPhysContig;
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+
+	eError = RA_AllocMulti(psPageArrayData->psArena,
+	                  ui64PhysSize,
+	                  uiLog2ChunkSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                       /* No flags */
+	                  "LMA_Page_Alloc",
+	                  psPageArrayData->aBaseArray,
+	                  psPageArrayData->uiTotalNumChunks,
+	                  &bPhysContig);
+
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"Contig: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long)ui64PhysSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+	if (bPhysContig)
+	{
+		BIT_SET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+			psPageArrayData->aBaseArray[0]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoison);
+	}
+
+	if (BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       ZERO_PAGE_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnZero);
+	}
+
+	psPageArrayData->iNumChunksAllocated += psPageArrayData->uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, ui64PhysSize, psPageArrayData->uiPid);
+#else
+	if (bPhysContig)
+	{
+		IMG_CPU_PHYADDR sLocalCpuPAddr;
+		sLocalCpuPAddr.uiAddr = (IMG_UINT64) psPageArrayData->aBaseArray[0];
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+								 NULL,
+								 sLocalCpuPAddr,
+								 IMG_PAGES2BYTES64(psPageArrayData->uiTotalNumChunks, uiLog2ChunkSize),
+								 psPageArrayData->uiPid
+								 DEBUG_MEMSTATS_VALUES);
+	}
+	else
+	{
+		IMG_UINT32 i, j;
+		IMG_CPU_PHYADDR sLocalCpuPAddr;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks;)
+		{
+			IMG_UINT32 ui32AllocSizeInChunks = 1;
+
+			for (j = i;
+			     j + 1 != psPageArrayData->uiTotalNumChunks &&
+			     RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+			     j++)
+			{
+				ui32AllocSizeInChunks++;
+			}
+
+			sLocalCpuPAddr.uiAddr = (IMG_UINT64) psPageArrayData->aBaseArray[i];
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 IMG_PAGES2BYTES64(ui32AllocSizeInChunks, uiLog2ChunkSize),
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+
+			i += ui32AllocSizeInChunks;
+		}
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnZero:
+errorOnPoison:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMulti(psPageArrayData->psArena,
+	              psPageArrayData->aBaseArray,
+	              psPageArrayData->uiTotalNumChunks);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+ * Fully allocated variant of sparse allocation does not take in as argument an
+ * array of indices. It is used in cases where the amount of chunks to allocate is
+ * the same as the total the PMR can represent. I.E when we want to fully populate
+ * a sparse PMR.
+ */
+static PVRSRV_ERROR
+_AllocLMPagesSparseFull(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiChunksToAlloc, uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+
+	eError = RA_AllocMultiSparse(psPageArrayData->psArena,
+	                  uiLog2ChunkSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                       /* No flags */
+	                  "LMA_Page_Alloc",
+	                  psPageArrayData->aBaseArray,
+	                  psPageArrayData->uiTotalNumChunks,
+	                  NULL, /* No indices given meaning allocate full base array using chunk count below */
+	                  psPageArrayData->uiChunksToAlloc);
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"SparseFull: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long)ui64PhysSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+		"(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+		psPageArrayData->aBaseArray[0]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoison);
+	}
+
+	if (BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       ZERO_PAGE_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnZero);
+	}
+
+	psPageArrayData->iNumChunksAllocated += psPageArrayData->uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, ui64PhysSize, psPageArrayData->uiPid);
+#else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			IMG_CPU_PHYADDR sLocalCpuPAddr;
+			sLocalCpuPAddr.uiAddr =
+				(IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[i]);
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 1 << uiLog2ChunkSize,
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+		}
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnZero:
+errorOnPoison:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMulti(psPageArrayData->psArena,
+	              psPageArrayData->aBaseArray,
+	              psPageArrayData->uiTotalNumChunks);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPagesSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT32 uiChunksToAlloc = psPageArrayData->uiChunksToAlloc;
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+	if (!pui32MapTable)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("pui32MapTable", eError, PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY, errorOnRAAlloc);
+	}
+
+#if defined(DEBUG)
+	/*
+	 * This block performs validation of the mapping table input in the following ways:
+	 * Check that each index in the mapping table does not exceed the number of the chunks
+	 * the whole PMR supports.
+	 * Check that each index given by the mapping table is not already allocated.
+	 * Check that there are no duplicated indices given in the mapping table.
+	 */
+	{
+		IMG_UINT32 i;
+		IMG_BOOL bIssueDetected = IMG_FALSE;
+		PVRSRV_ERROR eMapCheckError;
+
+		for (i = 0; i < uiChunksToAlloc; i++)
+		{
+			if (pui32MapTable[i] >= psPageArrayData->uiTotalNumChunks)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Page alloc request Index out of bounds for PMR @0x%p",
+						__func__,
+						psPageArrayData));
+				eMapCheckError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+
+			if (!RA_BASE_IS_INVALID(psPageArrayData->aBaseArray[pui32MapTable[i]]))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Mapping already exists Index %u Mapping index %u",
+						__func__,
+						i,
+						pui32MapTable[i]));
+				eMapCheckError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+
+			if (RA_BASE_IS_SPARSE_PREP(psPageArrayData->aBaseArray[pui32MapTable[i]]))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Mapping already exists in mapping table given Index %u Mapping index %u",
+						__func__,
+						i,
+						pui32MapTable[i]));
+				eMapCheckError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+			else
+			{
+				/* Set the To Prep value so we can detect duplicated map indices */
+				psPageArrayData->aBaseArray[pui32MapTable[i]] = RA_BASE_SPARSE_PREP_ALLOC_ADDR;
+			}
+		}
+		/* Unwind the Alloc Prep Values */
+		if (bIssueDetected)
+		{
+			/* We don't want to affect the index of the issue seen
+			 * as it could be a valid mapping. If it is a duplicated
+			 * mapping in the given table then we will clean-up the
+			 * previous instance anyway.
+			 */
+			IMG_UINT32 uiUnwind = i;
+
+			for (i = 0; i < uiUnwind; i++)
+			{
+				psPageArrayData->aBaseArray[pui32MapTable[i]] = INVALID_BASE_ADDR;
+			}
+
+			PVR_GOTO_WITH_ERROR(eError, eMapCheckError, errorOnRAAlloc);
+		}
+	}
+#endif
+
+	eError = RA_AllocMultiSparse(psPageArrayData->psArena,
+	                              psPageArrayData->uiLog2ChunkSize,
+	                              RA_NO_IMPORT_MULTIPLIER,
+	                              0,
+	                              "LMA_Page_Alloc",
+	                              psPageArrayData->aBaseArray,
+	                              psPageArrayData->uiTotalNumChunks,
+	                              pui32MapTable,
+	                              uiChunksToAlloc);
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"Sparse: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long) uiChunksToAlloc << uiLog2ChunkSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+	        psPageArrayData->aBaseArray[pui32MapTable[0]]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC) || BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		IMG_UINT32 i, ui32Index = 0;
+		for (i = 0; i < uiChunksToAlloc; i++)
+		{
+			ui32Index = pui32MapTable[i];
+
+			eError = _PhysPgMemSet(psPageArrayData,
+			                       &psPageArrayData->aBaseArray[ui32Index],
+			                       ui64ChunkSize,
+			                       BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC) ? PVRSRV_POISON_ON_ALLOC_VALUE :
+			                                                                    ZERO_PAGE_VALUE);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoisonZero);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+	                            uiChunksToAlloc << uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiChunksToAlloc; i++)
+		{
+			IMG_UINT32 ui32Index = pui32MapTable[i];
+			IMG_CPU_PHYADDR sLocalCpuPAddr;
+			sLocalCpuPAddr.uiAddr =
+				(IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[ui32Index]);
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 ui64ChunkSize,
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+		}
+	}
+#endif
+#endif
+
+	psPageArrayData->iNumChunksAllocated += uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnPoisonZero:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMultiSparse(psPageArrayData->psArena,
+	                    psPageArrayData->aBaseArray,
+	                    psPageArrayData->uiTotalNumChunks,
+	                    psPageArrayData->uiLog2ChunkSize,
+	                    pui32MapTable,
+	                    &uiChunksToAlloc);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR
+_AllocLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(NULL != psPageArrayData);
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	if (psPageArrayData->uiTotalNumChunks <
+			(psPageArrayData->iNumChunksAllocated + psPageArrayData->uiChunksToAlloc))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Pages requested to allocate don't fit PMR alloc Size. "
+				"Allocated: %u + Requested: %u > Total Allowed: %u",
+				psPageArrayData->iNumChunksAllocated,
+				psPageArrayData->uiChunksToAlloc,
+				psPageArrayData->uiTotalNumChunks));
+		return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+	}
+
+	/* If we have a non-backed sparse PMR then we can just return */
+	if (psPageArrayData->uiChunksToAlloc == 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+							"%s: Non-Backed Sparse PMR Created: %p.",
+							__func__,
+							psPageArrayData));
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	{
+		IMG_UINT32 ui32OSid=0;
+		PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPageArrayData->psPhysHeap);
+
+		/* Obtain the OSid specific data from our connection handle */
+		if (psPageArrayData->psConnection != NULL)
+		{
+			ui32OSid = psPageArrayData->psConnection->ui32OSid;
+		}
+
+		/* Replace the given RA Arena with OS Arena */
+		if (PVRSRV_CHECK_SHARED_BUFFER(psPageArrayData->uiAllocFlags))
+		{
+			psPageArrayData->psArena = psDevNode->psOSSharedArena;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "(GPU Virtualization Validation): Giving from shared mem"));
+		}
+		else
+		{
+			psPageArrayData->psArena = psDevNode->psOSidSubArena[ui32OSid];
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "(GPU Virtualization Validation): Giving from OS slot %d",
+					 ui32OSid));
+		}
+	}
+#endif
+
+	/*
+	 * 3 cases:
+	 * Sparse allocation populating the whole PMR.
+	 * [**********]
+	 * Sparse allocation partially populating the PMR at given indices.
+	 * [*** *** **]
+	 * Contiguous allocation.
+	 * [**********]
+	 *
+	 * Note: Separate cases are required for 1 and 3 due to memstats tracking.
+	 * In Contiguous case we can track the block as a single memstat record as we know
+	 * we will also free in that size record.
+	 * Sparse allocations require a memstat record per chunk as they can be arbitrarily
+	 * free'd.
+	 */
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		if (psPageArrayData->uiTotalNumChunks == psPageArrayData->uiChunksToAlloc &&
+		    !pui32MapTable)
+		{
+			eError = _AllocLMPagesSparseFull(psPageArrayData);
+		}
+		else
+		{
+			eError = _AllocLMPagesSparse(psPageArrayData, pui32MapTable);
+		}
+	}
+	else
+	{
+		eError = _AllocLMPagesContig(psPageArrayData);
+	}
+
+	return eError;
+}
+
+static void
+_FreeLMPageArray(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"physmem_lma.c: freed local memory array structure for PMR @0x%p",
+			psPageArrayData));
+
+	OSFreeMem(psPageArrayData);
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesContig(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	RA_ARENA *pArena = psPageArrayData->psArena;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiTotalNumChunks, psPageArrayData->uiLog2ChunkSize);
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated != 0);
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated ==
+	           psPageArrayData->uiTotalNumChunks);
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_POISON_ON_FREE))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+							   psPageArrayData->aBaseArray,
+							   ui64PhysSize,
+							   PVRSRV_POISON_ON_FREE_VALUE);
+		PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            ui64PhysSize,
+	                            psPageArrayData->uiPid);
+#else
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		PVRSRVStatsRemoveMemAllocRecord(uiStat,
+		                                (IMG_UINT64) psPageArrayData->aBaseArray[0],
+		                                psPageArrayData->uiPid);
+	}
+	else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+			{
+				PVRSRVStatsRemoveMemAllocRecord(uiStat,
+				                                (IMG_UINT64) psPageArrayData->aBaseArray[i],
+				                                psPageArrayData->uiPid);
+			}
+		}
+	}
+#endif
+#endif
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		eError = RA_FreeMulti(pArena,
+							  psPageArrayData->aBaseArray,
+							  1);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+	}
+	else
+	{
+		eError = RA_FreeMulti(pArena,
+							  psPageArrayData->aBaseArray,
+							  psPageArrayData->iNumChunksAllocated);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+	}
+
+	psPageArrayData->iNumChunksAllocated = 0;
+
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: freed %"IMG_UINT64_FMTSPEC" local memory for PMR @0x%p",
+			__func__,
+			ui64PhysSize,
+			psPageArrayData));
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesRemainingSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(psPageArrayData->uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+	IMG_BOOL bPoisonOnFree = (BIT_ISSET(ui32Flags, FLAG_POISON_ON_FREE));
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            psPageArrayData->iNumChunksAllocated << psPageArrayData->uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#endif
+
+	for (i = 0; i < psPageArrayData->uiTotalNumChunks;)
+	{
+		if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+		{
+			IMG_UINT32 j;
+			IMG_UINT32 ui32AccumulatedChunks = 1;
+
+			for (j = i;
+				 j + 1 != psPageArrayData->uiTotalNumChunks &&
+				 RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+				 j++)
+			{
+				ui32AccumulatedChunks++;
+			}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+			for (j = i; j < (i + ui32AccumulatedChunks); j++)
+			{
+				PVRSRVStatsRemoveMemAllocRecord(uiStat,
+				                                RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[j]),
+				                                psPageArrayData->uiPid);
+				if (bPoisonOnFree)
+#else
+			for (j = i; j < (i + ui32AccumulatedChunks) && bPoisonOnFree; j++)
+			{
+#endif
+				{
+					eError = _PhysPgMemSet(psPageArrayData,
+										   &psPageArrayData->aBaseArray[j],
+										   ui64ChunkSize,
+										   PVRSRV_POISON_ON_FREE_VALUE);
+					PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+				}
+			}
+
+			eError = RA_FreeMulti(psPageArrayData->psArena,
+			                       &psPageArrayData->aBaseArray[i],
+			                       ui32AccumulatedChunks);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+
+			psPageArrayData->iNumChunksAllocated -= ui32AccumulatedChunks;
+			i += ui32AccumulatedChunks;
+		}
+		else if (RA_BASE_IS_INVALID(psPageArrayData->aBaseArray[i]))
+		{
+			i++;
+		}
+	}
+
+	/* We have freed all allocations in the previous loop */
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+                   IMG_UINT32 *pui32FreeIndices,
+                   IMG_UINT32 ui32FreeChunkCount)
+{
+	RA_ARENA *pArena = psPageArrayData->psArena;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+	IMG_UINT32 uiActualFreeCount = ui32FreeChunkCount;
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated != 0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < ui32FreeChunkCount; i++)
+		{
+			IMG_UINT32 ui32Index = pui32FreeIndices[i];
+
+			PVRSRVStatsRemoveMemAllocRecord(uiStat,
+			                                (IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(
+			                                psPageArrayData->aBaseArray[ui32Index]),
+			                                psPageArrayData->uiPid);
+		}
+	}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_FREE))
+	{
+		IMG_UINT32 i, ui32Index = 0;
+		for (i = 0; i < ui32FreeChunkCount; i++)
+		{
+			ui32Index = pui32FreeIndices[i];
+
+			eError = _PhysPgMemSet(psPageArrayData,
+								   &psPageArrayData->aBaseArray[ui32Index],
+								   ui64ChunkSize,
+								   PVRSRV_POISON_ON_FREE_VALUE);
+			PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+		}
+	}
+
+	eError = RA_FreeMultiSparse(pArena,
+	                             psPageArrayData->aBaseArray,
+	                             psPageArrayData->uiTotalNumChunks,
+	                             uiLog2ChunkSize,
+	                             pui32FreeIndices,
+	                             &uiActualFreeCount);
+	psPageArrayData->iNumChunksAllocated -= uiActualFreeCount;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            uiActualFreeCount << psPageArrayData->uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#endif
+	if (eError == PVRSRV_ERROR_RA_FREE_INVALID_CHUNK)
+	{
+		/* Log the RA error but convert it to PMR level to match the interface,
+		 * this is important because other PMR factories may not use the RA but
+		 * still return error, returning a PMR based error
+		 * keeps the interface agnostic to implementation behaviour.
+		 */
+		PVR_LOG_IF_ERROR(eError, "RA_FreeMultiSparse");
+		return PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMultiSparse");
+
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: freed %" IMG_UINT64_FMTSPEC " local memory for PMR @0x%p",
+			__func__,
+			(uiActualFreeCount * ui64ChunkSize),
+			psPageArrayData));
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRFreeZombiePagesRAMem(PMR_IMPL_ZOMBIEPAGES pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psZombiePageArray = pvPriv;
+
+	eError = _FreeLMPages(psZombiePageArray, NULL, 0);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	_FreeLMPageArray(psZombiePageArray);
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+
+/* Allocates a new PMR_LMALLOCARRAY_DATA object and fills it with
+ * pages to be extracted from psSrcPageArrayData.
+ */
+static PVRSRV_ERROR
+_ExtractPages(PMR_LMALLOCARRAY_DATA *psSrcPageArrayData,
+			  IMG_UINT32 *pai32ExtractIndices,
+			  IMG_UINT32 ui32ExtractPageCount,
+			  PMR_LMALLOCARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ExtractPageCountSaved;
+	PMR_LMALLOCARRAY_DATA* psDstPageArrayData;
+
+	/* Alloc PMR_LMALLOCARRAY_DATA for the extracted pages */
+	eError = _AllocLMPageArray(ui32ExtractPageCount << psSrcPageArrayData->uiLog2ChunkSize,
+	                           ui32ExtractPageCount,
+	                           psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->ui32Flags,
+	                           psSrcPageArrayData->psPhysHeap,
+	                           psSrcPageArrayData->psArena,
+	                           psSrcPageArrayData->uiAllocFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData,
+	                           psSrcPageArrayData->psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_AllocLMPageArray", alloc_error);
+
+	ui32ExtractPageCountSaved = ui32ExtractPageCount;
+	/* Transfer pages from source base array to newly allocated page array */
+	eError = RA_TransferMultiSparseIndices(psSrcPageArrayData->psArena,
+	                                       psSrcPageArrayData->aBaseArray,
+	                                       psSrcPageArrayData->uiTotalNumChunks,
+	                                       psDstPageArrayData->aBaseArray,
+	                                       psDstPageArrayData->uiTotalNumChunks,
+	                                       psSrcPageArrayData->uiLog2ChunkSize,
+	                                       pai32ExtractIndices,
+	                                       &ui32ExtractPageCountSaved);
+	PVR_LOG_GOTO_IF_FALSE((eError == PVRSRV_OK) && (ui32ExtractPageCountSaved == ui32ExtractPageCount),
+	                      "RA_TransferMultiSparseIndices failed",
+	                      transfer_error);
+
+
+	/* Update page counts */
+	psSrcPageArrayData->iNumChunksAllocated -= ui32ExtractPageCount;
+	psDstPageArrayData->iNumChunksAllocated += ui32ExtractPageCount;
+
+	*psOutPageArrayData = psDstPageArrayData;
+
+	return PVRSRV_OK;
+transfer_error:
+	_FreeLMPageArray(psDstPageArrayData);
+alloc_error:
+	return eError;
+}
+
+/* Extracts all allocated pages referenced psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted
+ * pages information.
+ */
+static PVRSRV_ERROR
+_ExtractAllPages(PMR_LMALLOCARRAY_DATA *psSrcPageArrayData,
+				 PMR_LMALLOCARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA* psDstPageArrayData;
+	IMG_UINT32 ui32IdxSrc, ui32IdxDst;
+
+	if (psSrcPageArrayData->iNumChunksAllocated == 0)
+	{
+		/* Do nothing if psSrcPageArrayData contains no allocated pages */
+		return PVRSRV_OK;
+	}
+
+	/* Alloc PMR_LMALLOCARRAY_DATA for the extracted pages */
+	eError = _AllocLMPageArray(psSrcPageArrayData->iNumChunksAllocated << psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->iNumChunksAllocated,
+	                           psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->ui32Flags,
+	                           psSrcPageArrayData->psPhysHeap,
+	                           psSrcPageArrayData->psArena,
+	                           psSrcPageArrayData->uiAllocFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData,
+	                           psSrcPageArrayData->psConnection);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "_AllocLMPageArray failed in %s", __func__);
+
+	/* Now do the transfer */
+	ui32IdxDst=0;
+	for (ui32IdxSrc=0; ((ui32IdxDst<psSrcPageArrayData->iNumChunksAllocated) &&
+	                    (psDstPageArrayData->iNumChunksAllocated<psSrcPageArrayData->iNumChunksAllocated)); ui32IdxSrc++)
+	{
+		if (psSrcPageArrayData->aBaseArray[ui32IdxSrc] != INVALID_BASE_ADDR)
+		{
+			psDstPageArrayData->aBaseArray[ui32IdxDst++] = psSrcPageArrayData->aBaseArray[ui32IdxSrc];
+			psSrcPageArrayData->aBaseArray[ui32IdxSrc] = INVALID_BASE_ADDR;
+			psDstPageArrayData->iNumChunksAllocated++;
+		}
+	}
+
+	/* Update src page count */
+	psSrcPageArrayData->iNumChunksAllocated = 0;
+
+	*psOutPageArrayData = psDstPageArrayData;
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+             IMG_UINT32 *pui32FreeIndices,
+             IMG_UINT32 ui32FreeChunkCount)
+{
+	PVRSRV_ERROR eError;
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		if (!pui32FreeIndices)
+		{
+			eError =  _FreeLMPagesRemainingSparse(psPageArrayData);
+		}
+		else
+		{
+			eError = _FreeLMPagesSparse(psPageArrayData, pui32FreeIndices, ui32FreeChunkCount);
+		}
+	}
+	else
+	{
+		eError = _FreeLMPagesContig(psPageArrayData);
+	}
+
+	return eError;
+}
+
+/*
+ *
+ * Implementation of PMR callback functions
+ *
+ */
+
+/* destructor func is called after last reference disappears, but
+   before PMR itself is freed. */
+static void
+PMRFinalizeLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+
+	/* We can't free pages until now. */
+	if (psLMAllocArrayData->iNumChunksAllocated != 0)
+	{
+		eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
+		PVR_LOG_IF_ERROR(eError, "_FreeLMPages");
+		PVR_ASSERT (eError == PVRSRV_OK);
+	}
+
+	_FreeLMPageArray(psLMAllocArrayData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static inline void _TransferToMemZombieRecord_LmaPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+                                                       RA_BASE_T uiBase)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = {
+		.uiAddr = RA_BASE_STRIP_GHOST_BIT(uiBase)
+	};
+
+	PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+	                                  PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES,
+	                                  sCPUPhysAddr.uiAddr,
+	                                  psPageArrayData->uiPid
+	                                  DEBUG_MEMSTATS_VALUES);
+}
+#endif
+
+static PVRSRV_ERROR PMRZombifyLocalMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+	PMR_LMALLOCARRAY_DATA *psPageArrayData = pvPriv;
+
+	BIT_SET(psPageArrayData->ui32Flags, FLAG_ZOMBIE);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_PID uiPid = psPageArrayData->uiPid;
+		IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+		IMG_UINT64 uiSize = BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE) ?
+			(IMG_UINT64) psPageArrayData->iNumChunksAllocated << uiLog2ChunkSize :
+			(IMG_UINT64) psPageArrayData->uiTotalNumChunks << uiLog2ChunkSize;
+
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, uiSize, uiPid);
+		PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES, uiSize, uiPid);
+	}
+#else /* !defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* _FreeLMPagesRemainingSparse path */
+
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+			{
+				IMG_UINT32 j;
+				IMG_UINT32 ui32AccumulatedChunks = 1;
+
+				for (j = i;
+				     j + 1 != psPageArrayData->uiTotalNumChunks &&
+				         RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+				     j++)
+				{
+					ui32AccumulatedChunks++;
+				}
+
+				for (j = i; j < (i + ui32AccumulatedChunks); j++)
+				{
+					_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+					                                    psPageArrayData->aBaseArray[j]);
+				}
+			}
+		}
+	}
+	else
+	{
+		/* _FreeLMPagesContig path */
+
+		if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+			                                    psPageArrayData->aBaseArray[0]);
+		}
+		else
+		{
+			IMG_UINT32 i;
+
+			for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+			{
+				if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+				{
+					_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+					                                    psPageArrayData->aBaseArray[i]);
+				}
+			}
+
+		}
+	}
+#endif /* !defined(PVRSRV_ENABLE_MEMORY_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* callback function for locking the system physical page addresses.
+   As we are LMA there is nothing to do as we control physical memory. */
+static PVRSRV_ERROR
+PMRLockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData;
+
+	psLMAllocArrayData = pvPriv;
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_ONDEMAND))
+	{
+		/* Allocate Memory for deferred allocation */
+		eError = _AllocLMPages(psLMAllocArrayData, NULL);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+                                  PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_LMALLOCARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+	*ppvZombiePages = NULL;
+#endif
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_ONDEMAND))
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (psLMAllocArrayData->iNumChunksAllocated == 0)
+		{
+			*ppvZombiePages = NULL;
+			return PVRSRV_OK;
+		}
+
+		eError = _ExtractAllPages(psLMAllocArrayData,
+		                          &psExtractedPagesPageArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractAllPages", e0);
+
+		if (psExtractedPagesPageArray)
+		{
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyLocalMem(psExtractedPagesPageArray, NULL);
+			PVR_WARN_IF_ERROR(eError, "PMRZombifyLocalMem");
+		}
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		/* Free Memory for deferred allocation */
+		eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
+		PVR_RETURN_IF_ERROR(eError);
+#endif
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+#endif
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+/* N.B. It is assumed that PMRLockSysPhysAddressesLocalMem() is called _before_ this function! */
+static PVRSRV_ERROR
+PMRSysPhysAddrLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+					   IMG_UINT32 ui32Log2PageSize,
+					   IMG_UINT32 ui32NumOfPages,
+					   IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					   IMG_UINT64 ui64IPAPolicyValue,
+					   IMG_UINT64 ui64IPAClearMask,
+#endif
+					   IMG_BOOL *pbValid,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	IMG_UINT32 idx;
+	IMG_UINT32 uiLog2AllocSize;
+	IMG_UINT64 uiAllocIndex;
+	IMG_DEVMEM_OFFSET_T uiInAllocOffset;
+	IMG_UINT32 uiNumAllocs = psLMAllocArrayData->uiTotalNumChunks;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (psLMAllocArrayData->uiLog2ChunkSize < ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "for incompatible contiguity %u!",
+		         __func__,
+		         ui32Log2PageSize));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	PVR_ASSERT(psLMAllocArrayData->uiLog2ChunkSize != 0);
+	PVR_ASSERT(ui32Log2PageSize >= RA_BASE_FLAGS_LOG2);
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		for (idx=0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				psDevPAddr[idx].uiAddr = psLMAllocArrayData->aBaseArray[0] + puiOffset[idx];
+#if defined(SUPPORT_STATIC_IPA)
+				/* Modify the physical address with the associated IPA values */
+				psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+				psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+			}
+		}
+	}
+	else
+	{
+		uiLog2AllocSize = psLMAllocArrayData->uiLog2ChunkSize;
+
+		for (idx=0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				uiAllocIndex = puiOffset[idx] >> uiLog2AllocSize;
+				uiInAllocOffset = puiOffset[idx] - (uiAllocIndex << uiLog2AllocSize);
+
+				PVR_LOG_RETURN_IF_FALSE(uiAllocIndex < uiNumAllocs,
+										"puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+				PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2AllocSize));
+
+				/* The base may or may not be a ghost base, but we don't care,
+				 * we just need the real representation of the base.
+				 */
+				psDevPAddr[idx].uiAddr = RA_BASE_STRIP_GHOST_BIT(
+					psLMAllocArrayData->aBaseArray[uiAllocIndex]) + uiInAllocOffset;
+#if defined(SUPPORT_STATIC_IPA)
+				/* Modify the physical address with the associated IPA values */
+				psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+				psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+								 size_t uiOffset,
+								 size_t uiSize,
+								 void **ppvKernelAddressOut,
+								 IMG_HANDLE *phHandleOut,
+								 PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	PMR_KERNEL_MAPPING *psKernelMapping;
+	RA_BASE_T *paBaseArray;
+	IMG_UINT32 ui32ChunkIndex = 0;
+	size_t uiOffsetMask = uiOffset;
+
+	IMG_UINT32 uiLog2ChunkSize = psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT64 ui64PhysSize;
+
+	PVR_ASSERT(psLMAllocArrayData);
+	PVR_ASSERT(ppvKernelAddressOut);
+	PVR_ASSERT(phHandleOut);
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		IMG_UINT32 i;
+		/* Locate the desired physical chunk to map in */
+		ui32ChunkIndex = uiOffset >> psLMAllocArrayData->uiLog2ChunkSize;
+
+		if (OSIsMapPhysNonContigSupported())
+		{
+			/* If a size hasn't been supplied assume we are mapping a single page */
+			IMG_UINT32 uiNumChunksToMap;
+
+			/* This is to support OSMapPMR originated parameters */
+			if (uiOffset == 0 && uiSize == 0)
+			{
+				uiNumChunksToMap = psLMAllocArrayData->iNumChunksAllocated;
+			}
+			else
+			{
+				uiNumChunksToMap = uiSize >> psLMAllocArrayData->uiLog2ChunkSize;
+			}
+
+			/* Check we are attempting to map at least a chunk in size */
+			if (uiNumChunksToMap < 1)
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_INVALID_PARAMS, "uiNumChunksToMap < 1");
+			}
+
+			/* Check contiguous region doesn't exceed size of PMR */
+			if (ui32ChunkIndex + (uiNumChunksToMap - 1) > psLMAllocArrayData->uiTotalNumChunks)
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_INVALID_PARAMS,
+				                        "Mapping range exceeds total num chunks in PMR");
+			}
+
+			/* Check the virtually contiguous region given is physically backed */
+			for (i = ui32ChunkIndex; i < ui32ChunkIndex + uiNumChunksToMap; i++)
+			{
+				if (RA_BASE_IS_INVALID(psLMAllocArrayData->aBaseArray[i]))
+				{
+					PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY, "Sparse contiguity check");
+				}
+			}
+			/* Size of virtually contiguous sparse alloc */
+			ui64PhysSize = IMG_PAGES2BYTES64(uiNumChunksToMap, psLMAllocArrayData->uiLog2ChunkSize);
+		}
+		else
+		{
+			size_t uiStart = uiOffset;
+			size_t uiEnd = uiOffset + uiSize - 1;
+			size_t uiChunkMask = ~(IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize) - 1);
+
+			/* We can still map if only one chunk is required */
+			if ((uiStart & uiChunkMask) != (uiEnd & uiChunkMask))
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY, "Sparse contiguity check");
+			}
+			/* Map a single chunk */
+			ui64PhysSize = ui64ChunkSize;
+		}
+
+		paBaseArray = &psLMAllocArrayData->aBaseArray[ui32ChunkIndex];
+
+		/* Offset mask to be used for address offsets within a chunk */
+		uiOffsetMask = (1U << psLMAllocArrayData->uiLog2ChunkSize) - 1;
+	}
+	else
+	{
+		paBaseArray = psLMAllocArrayData->aBaseArray;
+		ui64PhysSize = IMG_PAGES2BYTES64(psLMAllocArrayData->uiTotalNumChunks, uiLog2ChunkSize);
+	}
+
+	PVR_ASSERT(ui32ChunkIndex < psLMAllocArrayData->uiTotalNumChunks);
+
+	psKernelMapping = OSAllocMem(sizeof(*psKernelMapping));
+	PVR_RETURN_IF_NOMEM(psKernelMapping);
+
+	eError = _MapPMRKernel(psLMAllocArrayData,
+	                       paBaseArray,
+	                       ui64PhysSize,
+	                       ulFlags,
+	                       psKernelMapping);
+	if (eError == PVRSRV_OK)
+	{
+		/* uiOffset & uiOffsetMask is used to get the kernel addr within the page */
+		*ppvKernelAddressOut = ((IMG_CHAR *) psKernelMapping->pvKernelAddress) + (uiOffset & uiOffsetMask);
+		*phHandleOut = psKernelMapping;
+	}
+	else
+	{
+		OSFreeMem(psKernelMapping);
+		PVR_LOG_ERROR(eError, "_MapPMRKernel");
+	}
+
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+                                                IMG_HANDLE hHandle)
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = (PMR_LMALLOCARRAY_DATA *) pvPriv;
+	PMR_KERNEL_MAPPING *psKernelMapping = (PMR_KERNEL_MAPPING *) hHandle;
+
+	PVR_ASSERT(psLMAllocArrayData);
+	PVR_ASSERT(psKernelMapping);
+
+	_UnMapPMRKernel(psLMAllocArrayData,
+	                psKernelMapping);
+
+	OSFreeMem(psKernelMapping);
+}
+
+static PVRSRV_ERROR
+CopyBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+				  IMG_DEVMEM_OFFSET_T uiOffset,
+				  IMG_UINT8 *pcBuffer,
+				  size_t uiBufSz,
+				  size_t *puiNumBytes,
+				  void (*pfnCopyBytes)(IMG_UINT8 *pcBuffer,
+									   IMG_UINT8 *pcPMR,
+									   size_t uiSize))
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	size_t uiBytesCopied;
+	size_t uiBytesToCopy;
+	size_t uiBytesCopyableFromAlloc;
+	PMR_KERNEL_MAPPING sMapping;
+	IMG_UINT8 *pcKernelPointer = NULL;
+	size_t uiBufferOffset;
+	IMG_UINT64 uiAllocIndex;
+	IMG_DEVMEM_OFFSET_T uiInAllocOffset;
+	IMG_UINT32 uiLog2ChunkSize = psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT64 ui64PhysSize;
+	PVRSRV_ERROR eError;
+
+	uiBytesCopied = 0;
+	uiBytesToCopy = uiBufSz;
+	uiBufferOffset = 0;
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		while (uiBytesToCopy > 0)
+		{
+			/* we have to map one alloc in at a time */
+			PVR_ASSERT(psLMAllocArrayData->uiLog2ChunkSize != 0);
+			uiAllocIndex = uiOffset >> psLMAllocArrayData->uiLog2ChunkSize;
+			uiInAllocOffset = uiOffset - (uiAllocIndex << psLMAllocArrayData->uiLog2ChunkSize);
+			uiBytesCopyableFromAlloc = uiBytesToCopy;
+			if (uiBytesCopyableFromAlloc + uiInAllocOffset > (1ULL << psLMAllocArrayData->uiLog2ChunkSize))
+			{
+				uiBytesCopyableFromAlloc = TRUNCATE_64BITS_TO_SIZE_T((1ULL << psLMAllocArrayData->uiLog2ChunkSize)-uiInAllocOffset);
+			}
+			/* Mapping a single chunk at a time */
+			ui64PhysSize = ui64ChunkSize;
+
+			PVR_ASSERT(uiBytesCopyableFromAlloc != 0);
+			PVR_ASSERT(uiAllocIndex < psLMAllocArrayData->uiTotalNumChunks);
+			PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2ChunkSize));
+
+			eError = _MapPMRKernel(psLMAllocArrayData,
+			                       &psLMAllocArrayData->aBaseArray[uiAllocIndex],
+			                       ui64PhysSize,
+			                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+			                       &sMapping);
+			PVR_GOTO_IF_ERROR(eError, e0);
+			pcKernelPointer = sMapping.pvKernelAddress;
+			pfnCopyBytes(&pcBuffer[uiBufferOffset], &pcKernelPointer[uiInAllocOffset], uiBytesCopyableFromAlloc);
+
+			_UnMapPMRKernel(psLMAllocArrayData,
+			                &sMapping);
+
+			uiBufferOffset += uiBytesCopyableFromAlloc;
+			uiBytesToCopy -= uiBytesCopyableFromAlloc;
+			uiOffset += uiBytesCopyableFromAlloc;
+			uiBytesCopied += uiBytesCopyableFromAlloc;
+		}
+	}
+	else
+	{
+		ui64PhysSize = IMG_PAGES2BYTES64(psLMAllocArrayData->uiTotalNumChunks, uiLog2ChunkSize);
+		PVR_ASSERT((uiOffset + uiBufSz) <= ui64PhysSize);
+		PVR_ASSERT(ui64ChunkSize != 0);
+		eError = _MapPMRKernel(psLMAllocArrayData,
+		                       psLMAllocArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+		                       &sMapping);
+		PVR_GOTO_IF_ERROR(eError, e0);
+		pcKernelPointer = sMapping.pvKernelAddress;
+		pfnCopyBytes(pcBuffer, &pcKernelPointer[uiOffset], uiBufSz);
+
+		_UnMapPMRKernel(psLMAllocArrayData,
+		                &sMapping);
+
+		uiBytesCopied = uiBufSz;
+	}
+	*puiNumBytes = uiBytesCopied;
+	return PVRSRV_OK;
+e0:
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+static void ReadLocalMem(IMG_UINT8 *pcBuffer,
+						 IMG_UINT8 *pcPMR,
+						 size_t uiSize)
+{
+	/* the memory is mapped as WC (and also aligned to page size) so we can
+	 * safely call "Cached" memcpy */
+	OSCachedMemCopy(pcBuffer, pcPMR, uiSize);
+}
+
+static PVRSRV_ERROR
+PMRReadBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+				  IMG_DEVMEM_OFFSET_T uiOffset,
+				  IMG_UINT8 *pcBuffer,
+				  size_t uiBufSz,
+				  size_t *puiNumBytes)
+{
+	return CopyBytesLocalMem(pvPriv,
+							 uiOffset,
+							 pcBuffer,
+							 uiBufSz,
+							 puiNumBytes,
+							 ReadLocalMem);
+}
+
+static void WriteLocalMem(IMG_UINT8 *pcBuffer,
+						  IMG_UINT8 *pcPMR,
+						  size_t uiSize)
+{
+	/* the memory is mapped as WC (and also aligned to page size) so we can
+	 * safely call "Cached" memcpy but need to issue a write memory barrier
+	 * to flush the write buffers after */
+	OSCachedMemCopyWMB(pcPMR, pcBuffer, uiSize);
+}
+
+static PVRSRV_ERROR
+PMRWriteBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+					  IMG_DEVMEM_OFFSET_T uiOffset,
+					  IMG_UINT8 *pcBuffer,
+					  size_t uiBufSz,
+					  size_t *puiNumBytes)
+{
+	return CopyBytesLocalMem(pvPriv,
+							 uiOffset,
+							 pcBuffer,
+							 uiBufSz,
+							 puiNumBytes,
+							 WriteLocalMem);
+}
+
+/*************************************************************************/ /*!
+@Function       PMRChangeSparseMemLocalMem
+@Description    This function Changes the sparse mapping by allocating and
+                freeing of pages. It also changes the GPU maps accordingly.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PMRChangeSparseMemLocalMem(PMR_IMPL_PRIVDATA pPriv,
+                           const PMR *psPMR,
+                           IMG_UINT32 ui32AllocPageCount,
+                           IMG_UINT32 *pai32AllocIndices,
+                           IMG_UINT32 ui32FreePageCount,
+                           IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+                           PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+                           IMG_UINT32 uiFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+	IMG_UINT32 ui32AdtnlAllocPages = 0;
+	IMG_UINT32 ui32AdtnlFreePages = 0;
+	IMG_UINT32 ui32CommonRequstCount = 0;
+	IMG_UINT32 ui32Loop = 0;
+	IMG_UINT32 ui32Index = 0;
+	IMG_UINT32 uiAllocpgidx;
+	IMG_UINT32 uiFreepgidx;
+
+	PMR_LMALLOCARRAY_DATA *psPMRPageArrayData = (PMR_LMALLOCARRAY_DATA *)pPriv;
+	IMG_UINT32 uiLog2ChunkSize = psPMRPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+
+#if defined(DEBUG)
+	IMG_BOOL bPoisonFail = IMG_FALSE;
+	IMG_BOOL bZeroFail = IMG_FALSE;
+#endif
+
+	/* Fetch the Page table array represented by the PMR */
+	RA_BASE_T *paBaseArray = psPMRPageArrayData->aBaseArray;
+	PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+	/* The incoming request is classified into two operations independent of
+	 * each other: alloc & free chunks.
+	 * These operations can be combined with two mapping operations as well
+	 * which are GPU & CPU space mappings.
+	 *
+	 * From the alloc and free chunk requests, the net amount of chunks to be
+	 * allocated or freed is computed. Chunks that were requested to be freed
+	 * will be reused to fulfil alloc requests.
+	 *
+	 * The order of operations is:
+	 * 1. Allocate new Chunks.
+	 * 2. Move the free chunks from free request to alloc positions.
+	 * 3. Free the rest of the chunks not used for alloc
+	 *
+	 * Alloc parameters are validated at the time of allocation
+	 * and any error will be handled then. */
+
+	if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
+	{
+		ui32CommonRequstCount = (ui32AllocPageCount > ui32FreePageCount) ?
+				ui32FreePageCount : ui32AllocPageCount;
+
+		PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free chunks not supported");
+	}
+
+	if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
+	{
+		ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequstCount;
+	}
+	else
+	{
+		ui32AllocPageCount = 0;
+	}
+
+	if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
+	{
+		ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequstCount;
+	}
+	else
+	{
+		ui32FreePageCount = 0;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(
+	    (ui32CommonRequstCount | ui32AdtnlAllocPages | ui32AdtnlFreePages) != 0,
+	    "Invalid combination of parameters: ui32CommonRequstCount,"
+	    " ui32AdtnlAllocPages and ui32AdtnlFreePages.",
+	    PVRSRV_ERROR_INVALID_PARAMS
+	);
+
+	{
+		/* Validate the free page indices */
+		if (ui32FreePageCount)
+		{
+			if (pai32FreeIndices != NULL)
+			{
+				for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
+				{
+					uiFreepgidx = pai32FreeIndices[ui32Loop];
+
+					if (uiFreepgidx >= psPMRPageArrayData->uiTotalNumChunks)
+					{
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+					}
+
+					if (RA_BASE_IS_INVALID(paBaseArray[uiFreepgidx]))
+					{
+						PVR_LOG_GOTO_WITH_ERROR("paBaseArray[uiFreepgidx]", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Given non-zero free count but missing indices array",
+				         __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+		}
+
+		/* The following block of code verifies any issues with common alloc chunk indices */
+		for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
+		{
+			uiAllocpgidx = pai32AllocIndices[ui32Loop];
+			if (uiAllocpgidx >= psPMRPageArrayData->uiTotalNumChunks)
+			{
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+			}
+
+			if ((!RA_BASE_IS_INVALID(paBaseArray[uiAllocpgidx])) ||
+					(psPMRMapTable->aui32Translation[uiAllocpgidx] != TRANSLATION_INVALID))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Trying to allocate already allocated page again", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+			}
+		}
+
+		ui32Loop = 0;
+
+		/* Allocate new chunks */
+		if (0 != ui32AdtnlAllocPages)
+		{
+			/* Say how many chunks to allocate */
+			psPMRPageArrayData->uiChunksToAlloc = ui32AdtnlAllocPages;
+
+			eError = _AllocLMPages(psPMRPageArrayData, pai32AllocIndices);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_AllocLMPages", e0);
+
+			/* Mark the corresponding chunks of translation table as valid */
+			for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
+			{
+				psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
+			}
+
+			psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages;
+		}
+
+		ui32Index = ui32Loop;
+		ui32Loop = 0;
+
+		/* Move the corresponding free chunks to alloc request */
+		eError = RA_SwapSparseMem(psPMRPageArrayData->psArena,
+		                           paBaseArray,
+		                           psPMRPageArrayData->uiTotalNumChunks,
+		                           psPMRPageArrayData->uiLog2ChunkSize,
+		                           &pai32AllocIndices[ui32Index],
+		                           &pai32FreeIndices[ui32Loop],
+		                           ui32CommonRequstCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RA_SwapSparseMem", unwind_alloc);
+
+		for (ui32Loop = 0; ui32Loop < ui32CommonRequstCount; ui32Loop++, ui32Index++)
+		{
+			uiAllocpgidx = pai32AllocIndices[ui32Index];
+			uiFreepgidx  = pai32FreeIndices[ui32Loop];
+
+			psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
+			psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
+
+			/* Be sure to honour the attributes associated with the allocation
+			 * such as zeroing, poisoning etc. */
+			if (BIT_ISSET(psPMRPageArrayData->ui32Flags, FLAG_POISON_ON_ALLOC))
+			{
+				eError = _PhysPgMemSet(psPMRPageArrayData,
+				                       &psPMRPageArrayData->aBaseArray[uiAllocpgidx],
+				                       ui64ChunkSize,
+				                       PVRSRV_POISON_ON_ALLOC_VALUE);
+
+				/* Consider this as a soft failure and go ahead but log error to kernel log */
+				if (eError != PVRSRV_OK)
+				{
+#if defined(DEBUG)
+					bPoisonFail = IMG_TRUE;
+#endif
+				}
+			}
+
+			if (BIT_ISSET(psPMRPageArrayData->ui32Flags, FLAG_ZERO))
+			{
+				eError = _PhysPgMemSet(psPMRPageArrayData,
+									   &psPMRPageArrayData->aBaseArray[uiAllocpgidx],
+									   ui64ChunkSize,
+									   ZERO_PAGE_VALUE);
+				/* Consider this as a soft failure and go ahead but log error to kernel log */
+				if (eError != PVRSRV_OK)
+				{
+#if defined(DEBUG)
+					/* Don't think we need to zero any chunks further */
+					bZeroFail = IMG_TRUE;
+#endif
+				}
+			}
+		}
+
+		/* Free or zombie the additional free chunks */
+		if (0 != ui32AdtnlFreePages)
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			PMR_LMALLOCARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+			eError = _ExtractPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages, &psExtractedPagesPageArray);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractPages", e0);
+
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyLocalMem(psExtractedPagesPageArray, NULL);
+			PVR_LOG_IF_ERROR(eError, "PMRZombifyLocalMem");
+
+			*ppvZombiePages = psExtractedPagesPageArray;
+#else
+			eError = _FreeLMPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_FreeLMPages", e0);
+#endif /* SUPPORT_PMR_PAGES_DEFERRED_FREE */
+			ui32Index = ui32Loop;
+			ui32Loop = 0;
+
+			while (ui32Loop++ < ui32AdtnlFreePages)
+			{
+				/* Set the corresponding mapping table entry to invalid address */
+				psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Index++]] = TRANSLATION_INVALID;
+			}
+
+			psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages;
+		}
+	}
+
+#if defined(DEBUG)
+	if (IMG_TRUE == bPoisonFail)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error in poisoning the chunk", __func__));
+	}
+
+	if (IMG_TRUE == bZeroFail)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error in zeroing the chunk", __func__));
+	}
+#endif
+
+	return PVRSRV_OK;
+
+unwind_alloc:
+	_FreeLMPages(psPMRPageArrayData, pai32AllocIndices, ui32Index);
+
+	for (ui32Loop = 0; ui32Loop < ui32Index; ui32Loop++)
+	{
+		psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = TRANSLATION_INVALID;
+	}
+
+e0:
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRLMAFuncTab = {
+	.pfnLockPhysAddresses = &PMRLockSysPhysAddressesLocalMem,
+	.pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesLocalMem,
+	.pfnDevPhysAddr = &PMRSysPhysAddrLocalMem,
+	.pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataLocalMem,
+	.pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataLocalMem,
+	.pfnReadBytes = &PMRReadBytesLocalMem,
+	.pfnWriteBytes = &PMRWriteBytesLocalMem,
+	.pfnChangeSparseMem = &PMRChangeSparseMemLocalMem,
+	.pfnMMap = NULL,
+	.pfnFinalize = &PMRFinalizeLocalMem,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	.pfnFreeZombiePages = &PMRFreeZombiePagesRAMem,
+#endif
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = &PMRZombifyLocalMem,
+#endif
+};
+
+PVRSRV_ERROR
+PhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                            RA_ARENA *pArena,
+                            CONNECTION_DATA *psConnection,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_UINT32 ui32NumPhysChunks,
+                            IMG_UINT32 ui32NumVirtChunks,
+                            IMG_UINT32 *pui32MappingTable,
+                            IMG_UINT32 uiLog2AllocPageSize,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                            const IMG_CHAR *pszAnnotation,
+                            IMG_PID uiPid,
+                            PMR **ppsPMRPtr,
+                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	PMR *psPMR = NULL;
+	PMR_LMALLOCARRAY_DATA *psPrivData = NULL;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_UINT32 ui32LMAllocFlags = 0;
+
+	/* This path is checking for the type of PMR to create, if sparse we
+	 * have to perform additional validation as we can only map sparse ranges
+	 * if the os functionality to do so is present. We can also only map virtually
+	 * contiguous sparse regions. Non backed gaps in a range cannot be mapped.
+	 */
+	if (ui32NumPhysChunks != ui32NumVirtChunks || ui32NumVirtChunks > 1)
+	{
+		if (PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags) &&
+		   !OSIsMapPhysNonContigSupported())
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: LMA kernel mapping functions not available "
+					"for physically discontiguous memory.",
+					__func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, errorOnParam);
+		}
+		BIT_SET(ui32LMAllocFlags, FLAG_SPARSE);
+	}
+
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_ONDEMAND);
+	}
+
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_ZERO);
+	}
+
+	if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_POISON_ON_ALLOC);
+	}
+
+#if defined(DEBUG)
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_POISON_ON_FREE);
+	}
+#endif
+
+	/* Create Array structure that holds the physical pages */
+	eError = _AllocLMPageArray(uiSize,
+	                           ui32NumPhysChunks,
+	                           uiLog2AllocPageSize,
+	                           ui32LMAllocFlags,
+	                           psPhysHeap,
+	                           pArena,
+	                           uiFlags,
+	                           uiPid,
+	                           &psPrivData,
+	                           psConnection);
+	PVR_GOTO_IF_ERROR(eError, errorOnAllocPageArray);
+
+	if (!BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND))
+	{
+		/* Allocate the physical pages */
+		eError = _AllocLMPages(psPrivData, pui32MappingTable);
+		PVR_GOTO_IF_ERROR(eError, errorOnAllocPages);
+	}
+
+	/* In this instance, we simply pass flags straight through.
+
+	   Generically, uiFlags can include things that control the PMR
+	   factory, but we don't need any such thing (at the time of
+	   writing!), and our caller specifies all PMR flags so we don't
+	   need to meddle with what was given to us.
+	*/
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+	/* check no significant bits were lost in cast due to different
+	   bit widths for flags */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND))
+	{
+		PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (LMA)");
+	}
+
+	eError = PMRCreatePMR(psPhysHeap,
+						  uiSize,
+						  ui32NumPhysChunks,
+						  ui32NumVirtChunks,
+						  pui32MappingTable,
+						  uiLog2AllocPageSize,
+						  uiPMRFlags,
+						  pszAnnotation,
+						  &_sPMRLMAFuncTab,
+						  psPrivData,
+						  PMR_TYPE_LMA,
+						  &psPMR,
+						  ui32PDumpFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRCreatePMR", errorOnCreate);
+
+	*ppsPMRPtr = psPMR;
+	return PVRSRV_OK;
+
+errorOnCreate:
+	if (!BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND) && psPrivData->iNumChunksAllocated)
+	{
+		eError2 = _FreeLMPages(psPrivData, NULL, 0);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+errorOnAllocPages:
+	_FreeLMPageArray(psPrivData);
+
+errorOnAllocPageArray:
+errorOnParam:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pmr.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pmr.c
new file mode 100644
index 000000000000..588b076a4ed5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pmr.c
@@ -0,0 +1,5468 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                the "PMR" abstraction.  A PMR (Physical Memory Resource)
+                represents some unit of physical memory which is
+                allocated/freed/mapped/unmapped as an indivisible unit
+                (higher software levels provide an abstraction above that
+                to deal with dividing this down into smaller manageable units).
+                Importantly, this module knows nothing of virtual memory, or
+                of MMUs etc., with one excusable exception.  We have the
+                concept of a "page size", which really means nothing in
+                physical memory, but represents a "contiguity quantum" such
+                that the higher level modules which map this memory are able
+                to verify that it matches the needs of the page size for the
+                virtual realm into which it is being mapped.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "devicemem_server_utils.h"
+#include "devicemem_server.h"
+
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "pmr_impl.h"
+#include "pmr_os.h"
+#include "pmr_env.h"
+#include "pvrsrv.h"
+
+#include "allocmem.h"
+#include "lock.h"
+#include "uniq_key_splay_tree.h"
+
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "secure_export.h"
+#include "ossecure_export.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+/* ourselves */
+#include "pmr.h"
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#include "proc_stats.h"
+#endif
+
+#include "pdump_km.h"
+
+#define PMR_FLAG_INTERNAL_SPARSE_ALLOC     (1 << 0)
+#define PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE (1 << 1)
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PMR_FLAG_INTERNAL_DEFER_FREE       (1 << 2)
+#define PMR_FLAG_INTERNAL_IS_ZOMBIE        (1 << 3)
+
+/* Indicates PMR should be destroyed immediately and not deferred. */
+#define PMR_NO_ZOMBIE_FENCE IMG_UINT64_MAX
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Memalloc flags can be converted into pmr, ra or psplay flags.
+ * Ensure flags types are same size.
+ */
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(PMR_FLAGS_T),
+			  "Mismatch memalloc and pmr flags type size.");
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(RA_FLAGS_T),
+			  "Mismatch memalloc and ra flags type size.");
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(IMG_PSPLAY_FLAGS_T),
+			  "Mismatch memalloc and psplay flags type size.");
+
+/* A "context" for the physical memory block resource allocator.
+ *
+ * Context is probably the wrong word.
+ *
+ * There is almost certainly only one of these, ever, in the system.
+ * But, let's keep the notion of a context anyway, "just-in-case".
+ */
+static struct _PMR_CTX_
+{
+	/* For debugging, and PDump, etc., let's issue a forever incrementing
+	 * serial number to each allocation.
+	 */
+	IMG_UINT64 uiNextSerialNum;
+
+	/* For security, we only allow a PMR to be mapped if the caller knows
+	 * its key. We can pseudo-randomly generate keys
+	 */
+	IMG_UINT64 uiNextKey;
+
+	/* For debugging only, I guess: Number of live PMRs */
+	ATOMIC_T uiNumLivePMRs;
+
+	/* Lock for this structure */
+	POS_LOCK hLock;
+
+	/* In order to seed the uiNextKey, we enforce initialisation at driver
+	 * load time. Also, we can debug check at driver unload that the PMR
+	 * count is zero.
+	 */
+	IMG_BOOL bModuleInitialised;
+} _gsSingletonPMRContext = { 1, 0, {0}, NULL, IMG_FALSE };
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+typedef enum _PMR_ZOMBIE_TYPE_ {
+	PMR_ZOMBIE_TYPE_PMR,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_ZOMBIE_TYPE_PAGES,
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	PMR_ZOMBIE_TYPE_DEVICE_IMPORT,
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+} PMR_ZOMBIE_TYPE;
+
+typedef struct _PMR_HEADER_
+{
+	/* List node used to put the header on the zombie list
+	 * (psDevNode->sPMRZombieList). */
+	DLLIST_NODE sZombieNode;
+
+	PMR_ZOMBIE_TYPE eZombieType;
+} PMR_HEADER;
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/*
+ * A structure describing zombie pages.
+ */
+typedef struct _PMR_ZOMBIE_PAGES_
+{
+	PMR_HEADER sHeader;
+	PMR_IMPL_ZOMBIEPAGES pvFactoryPages;
+	PFN_FREE_ZOMBIE_PAGES_FN pfnFactoryFreeZombies;
+} PMR_ZOMBIE_PAGES;
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+typedef enum _PMR_STATE_
+{
+	PMR_STATE_INIT,
+	PMR_STATE_ACTIVE,
+	PMR_STATE_PAGES_IN_MIGRATE
+} PMR_STATE;
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+typedef struct _PMR_DEVICE_IMPORT_
+{
+	PMR_HEADER sHeader;             /* psDevNode zombie queue list node. */
+	DLLIST_NODE sNext;              /* PMR::sXDeviceImports list node. */
+	PVRSRV_DEVICE_NODE *psDevNode;  /* Device this import is representing. */
+	PMR *psParent;                  /* PMR the import belongs to. */
+} PMR_DEVICE_IMPORT;
+#endif
+
+/* A PMR. One per physical allocation. May be "shared".
+ *
+ * "shared" is ambiguous. We need to be careful with terminology.
+ * There are two ways in which a PMR may be "shared" and we need to be sure
+ * that we are clear which we mean.
+ *
+ * i)   multiple small allocations living together inside one PMR.
+ *
+ * ii)  one single allocation filling a PMR but mapped into multiple memory
+ *      contexts.
+ *
+ * This is more important further up the stack - at this level, all we care is
+ * that the PMR is being referenced multiple times.
+ */
+struct _PMR_
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* A Common header structure shared between PMR and PMR-like PMR_ZOMBIE_PAGES object */
+	PMR_HEADER sHeader;
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	/* This object is strictly refcounted. References include:
+	 * - mapping
+	 * - live handles (to this object)
+	 * - live export handles
+	 * (thus it is normal for allocated and exported memory to have a refcount of 3)
+	 * The object is destroyed when and only when the refcount reaches 0
+	 */
+
+	/* Physical address translation (device <> cpu) is done on a per device
+	 * basis which means we need the physical heap info
+	 */
+	PHYS_HEAP *psPhysHeap;
+
+	/* Reference count of the PMR. */
+	IMG_UINT32 uiRefCount;
+	/* Lock protecting reference counting (uiRefCount). */
+	POS_SPINLOCK hRefCountLock;
+
+	/* Client CPU mapping count - this is the number of times the PMR has been
+	 * mapped by the client to the CPU. It is used to determine when it
+	 * is safe to permit modification of a sparse allocation's layout.
+	 * Note that the process of mapping also increments uiRefCount
+	 * independently (as that is used to determine when a PMR may safely
+	 * be destroyed).
+	 */
+	ATOMIC_T iClientCpuMapCount;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+	/* Kernel CPU mapping count - number of times the PMR has been
+	 * mapped into the kernel. Used to signal if migrate is allowed
+	 * to occur.
+	 */
+	ATOMIC_T iKernelCpuMapCount;
+	/*
+	 * GPU mappings associated with PMR.
+	 * Must be protected with PMR lock.
+	 */
+	DLLIST_NODE sGpuMappingListHead;
+
+	/*
+	 * Current state of PMR
+	 * Must be protected by PMR lock.
+	 * Used to describe the current state of the PMR and determine which interactions
+	 * are possible. Pages could be in progress of migration during which physical
+	 * addresses cannot be requested.
+	 */
+	PMR_STATE eState;
+#endif
+
+	/* Count of how many reservations refer to this
+	 * PMR as a part of a GPU mapping. Must be protected
+	 * by PMR lock.
+	 */
+	IMG_INT32 iAssociatedResCount;
+
+	/* Lock count - this is the number of times PMRLockSysPhysAddresses()
+	 * has been called, less the number of PMRUnlockSysPhysAddresses()
+	 * calls. This is arguably here for debug reasons only, as the refcount
+	 * is already incremented as a matter of course.
+	 * Really, this just allows us to trap protocol errors: i.e. calling
+	 * PMRSysPhysAddr(), without a lock, or calling
+	 * PMRUnlockSysPhysAddresses() too many or too few times.
+	 */
+	ATOMIC_T iLockCount;
+
+	/* Lock for this structure */
+	POS_LOCK hLock;
+	/* Protects: `uiInternalFlags` & `uiDevImportBitmap` */
+	POS_SPINLOCK hBitmapLock;
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+	/* See PMR_ImportedDevicesMask()
+	 * Protected by hBitmapLock. */
+	IMG_UINT64 uiDevImportBitmap;
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	/* List of PMR_DEVICE_IMPORT's */
+	DLLIST_NODE sXDeviceImports;
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	/* Incrementing serial number to each allocation. */
+	IMG_UINT64 uiSerialNum;
+
+	/* For security, we only allow a PMR to be mapped if the caller knows
+	 * its key. We can pseudo-randomly generate keys
+	 */
+	PMR_PASSWORD_T uiKey;
+
+	/* Callbacks for per-flavour functions */
+	const PMR_IMPL_FUNCTAB *psFuncTab;
+
+	/* Data associated with the "subtype" */
+	PMR_IMPL_PRIVDATA pvFlavourData;
+
+	/* Environment specific data */
+	DECLARE_PMR_ENV_DATA(sEnvData)
+
+	/* What kind of PMR do we have? */
+	PMR_IMPL_TYPE eFlavour;
+
+	/* And for pdump */
+	const IMG_CHAR *pszPDumpDefaultMemspaceName;
+
+	/* Allocation annotation */
+	IMG_CHAR szAnnotation[DEVMEM_ANNOTATION_MAX_LEN];
+
+#if defined(PDUMP)
+
+	IMG_HANDLE hPDumpAllocHandle;
+
+	IMG_UINT32 uiNumPDumpBlocks;
+#endif
+
+	/* Logical size of allocation. "logical", because a PMR can represent
+	 * memory that will never physically exist.  This is the amount of
+	 * virtual space that the PMR would consume when it's mapped into a
+	 * virtual allocation.
+	 */
+	PMR_SIZE_T uiLogicalSize;
+
+	/* Mapping table for the allocation.
+	 * PMR's can be sparse in which case not all the "logic" addresses in
+	 * it are valid. We need to know which addresses are and aren't valid
+	 * when mapping or reading the PMR.
+	 * The mapping table translates "logical" offsets into physical offsets
+	 * which is what we always pass to the PMR factory (so it doesn't have
+	 * to be concerned about sparseness issues)
+	 */
+	PMR_MAPPING_TABLE *psMappingTable;
+
+	/* Minimum Physical Contiguity Guarantee.  Might be called "page size",
+	 * but that would be incorrect, as page size is something meaningful
+	 * only in virtual realm. This contiguity guarantee provides an
+	 * inequality that can be verified/asserted/whatever to ensure that
+	 * this PMR conforms to the page size requirement of the place the PMR
+	 * gets mapped. (May be used to select an appropriate heap in variable
+	 * page size systems)
+	 *
+	 * The absolutely necessary condition is this:
+	 *
+	 *    device MMU page size <= actual physical contiguity.
+	 *
+	 * We go one step further in order to be able to provide an early
+	 * warning / early compatibility check and say this:
+	 *
+	 *     device MMU page size <=
+	 *         2**(uiLog2ContiguityGuarantee) <=
+	 *             actual physical contiguity.
+	 *
+	 * In this way, it is possible to make the page table reservation
+	 * in the device MMU without even knowing the granularity of the
+	 * physical memory (i.e. useful for being able to allocate virtual
+	 * before physical)
+	 */
+	PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee;
+
+	/* Flags. We store a copy of the "PMR flags" (usually a subset of the
+	 * flags given at allocation time) and return them to any caller of
+	 * PMR_Flags(). The intention of these flags is that the ones stored
+	 * here are used to represent permissions, such that no one is able
+	 * to map a PMR in a mode in which they are not allowed, e.g.,
+	 * writeable for a read-only PMR, etc.
+	 */
+	PMR_FLAGS_T uiFlags;
+
+	/* Various flags informing about PMR's state:
+	 *
+	 * SPARSE_ALLOC:
+	 *   indicates whether this PMR has been allocated as sparse.
+	 *   The condition for this variable to be set at allocation time is:
+	 *   (numVirtChunks != numPhysChunks) || (numVirtChunks > 1)
+	 *
+	 * NO_LAYOUT_CHANGE:
+	 *   Flag that conveys mutability of the PMR:
+	 *   - set:   indicates the PMR is immutable (no more memory changes)
+	 *   - unset: means the memory layout associated with the PMR is mutable
+	 *
+	 *   A PMR is always mutable by default but is marked immutable on the
+	 *   first export for the rest of its life.
+	 *
+	 *   Also, any PMRs that track the same memory through imports are
+	 *   marked immutable as well.
+	 *
+	 * DEFER_FREE:
+	 *   If present the PMR is marked to be freed by the CleanupThread.
+	 *
+	 * IS_ZOMBIE:
+	 *   Indicates if the PMR is in the zombie state (marked for free in the
+	 *   CleanupThread). */
+	IMG_UINT32 uiInternalFlags;
+
+	/* Do we really need this?
+	 * For now we'll keep it, until we know we don't.
+	 * NB: this is not the "memory context" in client terms - this is
+	 * _purely_ the "PMR" context, of which there is almost certainly only
+	 * ever one per system as a whole, but we'll keep the concept anyway,
+	 * just-in-case.
+	 */
+	struct _PMR_CTX_ *psContext;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Stored handle to PMR RI entry */
+	void		*hRIHandle;
+#endif
+};
+
+/* Do we need a struct for the export handle?
+ * I'll use one for now, but if nothing goes in it, we'll lose it
+ */
+struct _PMR_EXPORT_
+{
+	struct _PMR_ *psPMR;
+};
+
+struct _PMR_PAGELIST_
+{
+	struct _PMR_ *psReferencePMR;
+};
+
+static INLINE IMG_UINT32
+_GetRef(const PMR *const psPMR)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT32 uiRefCount;
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+	uiRefCount = psPMR->uiRefCount;
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return uiRefCount;
+}
+
+#if defined(PDUMP)
+static INLINE IMG_BOOL _IsHostDevicePMR(const PMR *const psPMR)
+{
+	const PVRSRV_DEVICE_NODE *psDevNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	return psPMR->psPhysHeap == psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL];
+}
+
+static void
+PDumpPMRFreePMR(PMR *psPMR,
+                IMG_DEVMEM_SIZE_T uiSize,
+                IMG_DEVMEM_ALIGN_T uiBlockSize,
+                IMG_UINT32 uiLog2Contiguity,
+                IMG_HANDLE hPDumpAllocationInfoHandle);
+
+static void
+PDumpPMRMallocPMR(PMR *psPMR,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT32 ui32ChunkSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 ui32NumLogicalChunks,
+                  IMG_UINT32 *puiMappingTable,
+                  IMG_UINT32 uiLog2Contiguity,
+                  IMG_BOOL bInitialise,
+                  IMG_UINT8 ui8InitValue,
+                  IMG_HANDLE *phPDumpAllocInfoOut,
+                  IMG_UINT32 ui32PDumpFlags);
+
+static void
+PDumpPMRChangeSparsePMR(PMR *psPMR,
+                        IMG_UINT32 uiBlockSize,
+                        IMG_UINT32 ui32AllocPageCount,
+                        IMG_UINT32 *pai32AllocIndices,
+                        IMG_UINT32 ui32FreePageCount,
+                        IMG_UINT32 *pai32FreeIndices,
+                        IMG_BOOL bInitialise,
+                        IMG_UINT8 ui8InitValue,
+                        IMG_HANDLE *phPDumpAllocInfoOut);
+#endif /* defined PDUMP */
+
+IMG_INT32 PMRGetLiveCount(void)
+{
+	return OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs);
+}
+
+PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR)
+{
+	PPVRSRV_DEVICE_NODE psReturnedDeviceNode = NULL;
+
+	PVR_ASSERT(psExportPMR != NULL);
+	if (psExportPMR)
+	{
+		PVR_ASSERT(psExportPMR->psPMR != NULL);
+		if (psExportPMR->psPMR)
+		{
+			IMG_UINT32 uiRefCount = _GetRef(psExportPMR->psPMR);
+			if (uiRefCount > 0)
+			{
+				psReturnedDeviceNode = PMR_DeviceNode(psExportPMR->psPMR);
+			}
+#ifdef PVRSRV_NEED_PVR_ASSERT
+			else
+			{
+				PVR_ASSERT(IMG_FALSE);
+			}
+#endif
+		}
+	}
+
+	return psReturnedDeviceNode;
+}
+
+static PVRSRV_ERROR
+_PMRCreate(PMR_SIZE_T uiLogicalSize,
+           IMG_UINT32 ui32NumPhysChunks,
+           IMG_UINT32 ui32NumLogicalChunks,
+           IMG_UINT32 *pui32MappingTable,
+           PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+           PMR_FLAGS_T uiFlags,
+           PMR **ppsPMR)
+{
+	void *pvPMRLinAddr;
+	PMR *psPMR;
+	PMR_MAPPING_TABLE *psMappingTable;
+	struct _PMR_CTX_ *psContext;
+	IMG_UINT32 i, ui32Temp = 0;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bSparse = IMG_FALSE;
+	PMR_SIZE_T uiChunkSize;
+
+	psContext = &_gsSingletonPMRContext;
+
+	/* Do we have a sparse allocation? */
+	if ( (ui32NumLogicalChunks != ui32NumPhysChunks) ||
+			(ui32NumLogicalChunks > 1) )
+	{
+		bSparse = IMG_TRUE;
+		uiChunkSize = 1ULL << uiLog2ContiguityGuarantee;
+	}
+	else
+	{
+		uiChunkSize = uiLogicalSize;
+	}
+
+	/* Extra checks required for sparse PMRs */
+	if (bSparse)
+	{
+		/* Check the logical size and chunk information agree with each other */
+		if (uiLogicalSize != (uiChunkSize * ui32NumLogicalChunks))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Bad mapping size (uiLogicalSize = 0x%llx, uiChunkSize = 0x%llx, ui32NumLogicalChunks = %d)",
+					__func__, (unsigned long long)uiLogicalSize, (unsigned long long)uiChunkSize, ui32NumLogicalChunks));
+			return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+		}
+	}
+
+	pvPMRLinAddr = OSAllocMem(sizeof(*psPMR) + sizeof(*psMappingTable) + IMG_FLEX_ARRAY_SIZE(sizeof(IMG_UINT32), ui32NumLogicalChunks));
+	PVR_RETURN_IF_NOMEM(pvPMRLinAddr);
+
+	psPMR = (PMR *) pvPMRLinAddr;
+	psMappingTable = IMG_OFFSET_ADDR(pvPMRLinAddr, sizeof(*psPMR));
+
+	/* Setup the mapping table */
+	psMappingTable->uiChunkSize = uiChunkSize;
+	psMappingTable->ui32NumLogicalChunks = ui32NumLogicalChunks;
+	psMappingTable->ui32NumPhysChunks = ui32NumPhysChunks;
+	OSCachedMemSet(&psMappingTable->aui32Translation[0], 0xFF, sizeof(psMappingTable->aui32Translation[0])*
+	               ui32NumLogicalChunks);
+	for (i=0; i<ui32NumPhysChunks; i++)
+	{
+		ui32Temp = pui32MappingTable[i];
+		PVR_ASSERT(ui32Temp < ui32NumLogicalChunks);
+		psMappingTable->aui32Translation[ui32Temp] = ui32Temp;
+	}
+
+	eError = OSLockCreate(&psPMR->hLock);
+	PVR_GOTO_IF_ERROR(eError, ErrFreePMR);
+
+	eError = OSSpinLockCreate(&psPMR->hBitmapLock);
+	PVR_GOTO_IF_ERROR(eError, ErrFreePMRLock);
+
+	eError = OSSpinLockCreate(&psPMR->hRefCountLock);
+	PVR_GOTO_IF_ERROR(eError, ErrFreeBitmapLock);
+
+	/* Setup the PMR */
+	psPMR->uiRefCount = 0;
+	OSAtomicWrite(&psPMR->iClientCpuMapCount, 0);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	OSAtomicWrite(&psPMR->iKernelCpuMapCount, 0);
+	dllist_init(&psPMR->sGpuMappingListHead);
+#endif
+	psPMR->iAssociatedResCount = 0;
+
+	/* If allocation is not made on demand, it will be backed now and
+	 * backing will not be removed until the PMR is destroyed, therefore
+	 * we can initialise the iLockCount to 1 rather than 0.
+	 */
+	OSAtomicWrite(&psPMR->iLockCount, (PVRSRV_CHECK_ON_DEMAND(uiFlags) ? 0 : 1));
+
+	psPMR->psContext = psContext;
+	psPMR->uiLogicalSize = uiLogicalSize;
+	psPMR->uiLog2ContiguityGuarantee = uiLog2ContiguityGuarantee;
+	psPMR->uiFlags = uiFlags;
+	psPMR->psMappingTable = psMappingTable;
+	psPMR->uiInternalFlags = bSparse ? PMR_FLAG_INTERNAL_SPARSE_ALLOC : 0;
+	psPMR->szAnnotation[0] = '\0';
+	PMR_ENV_INITIALIZE(psPMR, sEnvData);
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+	psPMR->uiDevImportBitmap = 0;
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	psPMR->sHeader.eZombieType = PMR_ZOMBIE_TYPE_PMR;
+	dllist_init(&psPMR->sHeader.sZombieNode);
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	dllist_init(&psPMR->sXDeviceImports);
+#endif /* defined(SUPPORT_DEVICE_IMPORT_DEFERRED_FREE) */
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	psPMR->hRIHandle = NULL;
+#endif
+	OSLockAcquire(psContext->hLock);
+	psPMR->uiKey = psContext->uiNextKey;
+	psPMR->uiSerialNum = psContext->uiNextSerialNum;
+	psContext->uiNextKey = (0x80200003 * psContext->uiNextKey)
+								^ (0xf00f0081 * (uintptr_t)pvPMRLinAddr);
+	psContext->uiNextSerialNum++;
+	*ppsPMR = psPMR;
+	OSLockRelease(psContext->hLock);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+
+	/* Increment live PMR count */
+	OSAtomicIncrement(&psContext->uiNumLivePMRs);
+
+	return PVRSRV_OK;
+
+ErrFreeBitmapLock:
+	OSSpinLockDestroy(psPMR->hBitmapLock);
+ErrFreePMRLock:
+	OSLockDestroy(psPMR->hLock);
+ErrFreePMR:
+	OSFreeMem(psPMR);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_Ref(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+
+	if (psPMR->uiRefCount == 0)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Ref Count == 0 PMR: @0x%p Annot: %s",
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+	else if (psPMR->uiRefCount >= IMG_UINT32_MAX - uiRefCount)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Ref Count >= IMG_UINT32_MAX PMR: @0x%p "
+		                        "Annot: %s RefCount: %u",
+		                        psPMR,
+		                        psPMR->szAnnotation,
+		                        uiRefCount));
+		OSWarnOn(1);
+
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	psPMR->uiRefCount += uiRefCount;
+
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_Unref(PMR *psPMR, IMG_INT32 uiRefCount, IMG_UINT32 *pui32RefCount)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+
+	if (psPMR->uiRefCount == 0)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Unref Count = 0 PMR: @0x%p Annot: %s "
+		                        "RefCount: %u",
+		                        psPMR,
+		                        psPMR->szAnnotation,
+		                        uiRefCount));
+		OSWarnOn(1);
+
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	psPMR->uiRefCount -= uiRefCount;
+
+	if (pui32RefCount != NULL)
+	{
+		*pui32RefCount = psPMR->uiRefCount;
+	}
+
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+#if defined(DEBUG)
+void PMRLockHeldAssert(const PMR *psPMR)
+{
+	OSLockHeldAssert(psPMR->hLock);
+}
+#endif
+
+void
+PMRLockPMR(const PMR *psPMR)
+{
+	OSLockAcquire(psPMR->hLock);	/* Uses same lock as PhysAddresses */
+}
+
+void
+PMRUnlockPMR(const PMR *psPMR)
+{
+	OSLockRelease(psPMR->hLock);	/* Uses same lock as PhysAddresses */
+}
+
+static INLINE void _IntFlagSet(PMR *psPMR, const IMG_UINT32 uiValue)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_SET(psPMR->uiInternalFlags, uiValue);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+
+static INLINE void _IntFlagClr(PMR *psPMR, const IMG_UINT32 uiValue)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_UNSET(psPMR->uiInternalFlags, uiValue);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+
+static INLINE IMG_BOOL _IntFlagIsSet(const PMR *psPMR, const IMG_UINT32 uiValue)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_BOOL bIsSet;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	bIsSet = BITMASK_HAS(psPMR->uiInternalFlags, uiValue);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return bIsSet;
+}
+
+static INLINE void
+_FactoryLock(const PMR_IMPL_FUNCTAB *psFuncTable)
+{
+	if (psFuncTable->pfnGetPMRFactoryLock != NULL)
+	{
+		psFuncTable->pfnGetPMRFactoryLock();
+	}
+}
+
+static INLINE void
+_FactoryUnlock(const PMR_IMPL_FUNCTAB *psFuncTable)
+{
+	if (psFuncTable->pfnReleasePMRFactoryLock != NULL)
+	{
+		psFuncTable->pfnReleasePMRFactoryLock();
+	}
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/* Protects:
+ * - `psDevNode->sPMRZombieList`
+ * - `uiPMRZombieCount`
+ * - `uiPMRZombieCountInCleanup`
+ *
+ * and all `PMR_ZOMBIE_CLEANUP_ITEM::sZombieList` where
+ * `PMR_ZOMBIE_CLEANUP_ITEM::psDevNode == psDevNode` */
+static INLINE void
+_ZombieListLock(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OSLockAcquire(psDevNode->hPMRZombieListLock);
+}
+
+static INLINE void
+_ZombieListUnlock(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OSLockRelease(psDevNode->hPMRZombieListLock);
+}
+
+static IMG_BOOL _IsDeviceOnAndOperating(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+
+	eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		/* Treat unknown power state as ON. */
+		ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+	}
+
+	/* The device does not accept zombies when its power is OFF as
+	 * the cache invalidation comes as a given. */
+	return !( ePowerState == PVRSRV_DEV_POWER_STATE_OFF
+	       || psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+static IMG_UINT64
+_DeviceImportBitmapGet(const PMR *psPMR)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_UINT64 uiDevImportBitmap;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	uiDevImportBitmap = psPMR->uiDevImportBitmap;
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return uiDevImportBitmap;
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(PVRSRV_ENABLE_XD_MEM) && !defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static void
+_DeviceImportBitmapSet(PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_SET(psPMR->uiDevImportBitmap, IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) && !defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static void
+_DeviceImportBitmapClr(PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_UNSET(psPMR->uiDevImportBitmap, IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+
+static IMG_BOOL
+_DeviceImportBitmapIsSet(const PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_BOOL bIsSet;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	bIsSet = BITMASK_HAS(psPMR->uiDevImportBitmap,
+	                     IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return bIsSet;
+}
+
+static IMG_BOOL
+/* Atomically, return if the `psDevNode` is set in the bitmap and then set it. */
+_DeviceImportBitmapFetchAndSet(PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_BOOL bIsSet;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	bIsSet = BITMASK_HAS(psPMR->uiDevImportBitmap,
+	                     IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	BITMASK_SET(psPMR->uiDevImportBitmap,
+	            IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return bIsSet;
+}
+
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static PVRSRV_ERROR
+_DeviceImportRegister(PMR *psPMR, PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_DEVICE_IMPORT *psImport;
+
+	PVR_ASSERT(psPMR);
+	PVR_ASSERT(psDevNode);
+	PVR_ASSERT(PMR_DeviceNode(psPMR) != psDevNode);
+
+	/* Explicitly reject:
+	 * - PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC
+	 * - !PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE
+	 * as XD PMRs don't have support for
+	 * SUPPORT_PMR_PAGES_DEFERRED_FREE. */
+	if (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ||
+	    !_IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE))
+	{
+		eError = PVRSRV_ERROR_PMR_NOT_PERMITTED;
+		PVR_LOG_ERROR(eError,
+		              "PVRSRV_CHECK_ON_DEMAND || !PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE");
+		return eError;
+	}
+
+	/* Check if the device is already imported */
+	if (_DeviceImportBitmapFetchAndSet(psPMR, psDevNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	psImport = OSAllocMem(sizeof(*psImport));
+	PVR_LOG_RETURN_IF_NOMEM(psImport, "PMR_DEVICE_IMPORT");
+
+	psImport->psParent = psPMR;
+	psImport->psDevNode = psDevNode;
+	dllist_init(&psImport->sHeader.sZombieNode);
+	psImport->sHeader.eZombieType = PMR_ZOMBIE_TYPE_DEVICE_IMPORT;
+
+	PMRLockPMR(psPMR);
+	dllist_add_to_tail(&psPMR->sXDeviceImports, &psImport->sNext);
+	PMRUnlockPMR(psPMR);
+
+	return eError;
+}
+
+static void
+_DeviceImportFreeImportZombie(PMR_DEVICE_IMPORT *psImport)
+{
+	PVR_ASSERT(_DeviceImportBitmapIsSet(psImport->psParent, psImport->psDevNode));
+	_DeviceImportBitmapClr(psImport->psParent, psImport->psDevNode);
+
+	PMRLockPMR(psImport->psParent);
+	dllist_remove_node(&psImport->sNext);
+	PMRUnlockPMR(psImport->psParent);
+
+	OSFreeMem(psImport);
+}
+
+static IMG_BOOL
+_DeviceImportEnqueueZombie(PMR_DEVICE_IMPORT *psImport)
+{
+	PVR_ASSERT(_DeviceImportBitmapIsSet(psImport->psParent, psImport->psDevNode));
+
+	if (!_IsDeviceOnAndOperating(psImport->psDevNode))
+	{
+		_DeviceImportFreeImportZombie(psImport);
+		return IMG_FALSE;
+	}
+
+	_ZombieListLock(psImport->psDevNode);
+	dllist_add_to_tail(&psImport->psDevNode->sPMRZombieList,
+	                   &psImport->sHeader.sZombieNode);
+	psImport->psDevNode->uiPMRZombieCount++;
+	_ZombieListUnlock(psImport->psDevNode);
+
+	return IMG_TRUE;
+}
+
+static void
+_DeviceImportsReviveZombies(PMR *psPMR)
+{
+	PDLLIST_NODE psNode, psNext;
+	PMR_DEVICE_IMPORT *psImport;
+
+	dllist_foreach_node(&psPMR->sXDeviceImports, psNode, psNext)
+	{
+		psImport = IMG_CONTAINER_OF(psNode, PMR_DEVICE_IMPORT, sNext);
+		_ZombieListLock(psImport->psDevNode);
+		if (!dllist_is_empty(&psImport->sHeader.sZombieNode))
+		{
+			dllist_remove_node(&psImport->sHeader.sZombieNode);
+			psImport->psDevNode->uiPMRZombieCount--;
+		}
+		_ZombieListUnlock(psImport->psDevNode);
+	}
+}
+
+static IMG_BOOL
+_DeviceImportsEnqueueZombies(PMR *psPMR)
+{
+	PDLLIST_NODE psNode, psNext;
+	PMR_DEVICE_IMPORT *psImport;
+	IMG_BOOL bEnqueued = IMG_FALSE;
+
+	PMRLockPMR(psPMR);
+
+	dllist_foreach_node(&psPMR->sXDeviceImports, psNode, psNext)
+	{
+		psImport = IMG_CONTAINER_OF(psNode, PMR_DEVICE_IMPORT, sNext);
+		bEnqueued |= _DeviceImportEnqueueZombie(psImport);
+	}
+
+	PMRUnlockPMR(psPMR);
+
+	return bEnqueued;
+}
+
+static void
+_DeviceImportsUnregisterAll(PMR *psPMR)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	PDLLIST_NODE psNode, psNext;
+
+	PMRLockPMR(psPMR);
+	dllist_foreach_node(&psPMR->sXDeviceImports, psNode, psNext)
+	{
+		PMR_DEVICE_IMPORT *psImport = IMG_CONTAINER_OF(psNode, PMR_DEVICE_IMPORT, sNext);
+		PVR_ASSERT(_DeviceImportBitmapIsSet(psPMR, psImport->psDevNode));
+		OSFreeMem(psImport);
+	}
+	dllist_init(&psPMR->sXDeviceImports);
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	psPMR->uiDevImportBitmap = 0;
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+	PMRUnlockPMR(psPMR);
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+static void
+_PMRDestroy(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	PMR_ENV_FINALIZE(psPMR, sEnvData);
+
+	if (psPMR->psFuncTab->pfnFinalize != NULL)
+	{
+		psPMR->psFuncTab->pfnFinalize(psPMR->pvFlavourData);
+	}
+
+#if defined(PDUMP)
+	/* if allocation is done on the host node don't include it in the PDUMP */
+	if (!_IsHostDevicePMR(psPMR))
+	{
+		PDumpPMRFreePMR(psPMR,
+		                psPMR->uiLogicalSize,
+		                IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee),
+		                psPMR->uiLog2ContiguityGuarantee,
+		                psPMR->hPDumpAllocHandle);
+	}
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	/* This PMR is about to be destroyed, update its mmap stats record (if present)
+	 * to avoid dangling pointer. Additionally, this is required because mmap stats
+	 * are identified by PMRs and a new PMR down the line "might" get the same address
+	 * as the one we're about to free and we'd like 2 different entries in mmaps
+	 * stats for such cases */
+	MMapStatsRemovePMR(psPMR);
+#endif
+
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	/* If not backed on demand, iLockCount should be 1 otherwise it should be 0 */
+	PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1));
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Delete RI entry */
+	if (psPMR->hRIHandle)
+	{
+		PVRSRV_ERROR eError = RIDeletePMREntryKM(psPMR->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "RIDeletePMREntryKM");
+		/* continue destroying the PMR */
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	_DeviceImportsUnregisterAll(psPMR);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	/* Decrement live PMR count. Probably only of interest for debugging */
+	PVR_ASSERT(OSAtomicRead(&psPMR->psContext->uiNumLivePMRs) > 0);
+	OSAtomicDecrement(&psPMR->psContext->uiNumLivePMRs);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Detect programming errors here, either a reference on the PMR
+	 * has not been taken by a user or the mapping records added
+	 * to the list head have not been destroyed correctly.
+	 */
+	PVR_ASSERT(dllist_is_empty(&psPMR->sGpuMappingListHead));
+#endif
+
+	OSSpinLockDestroy(psPMR->hRefCountLock);
+	OSSpinLockDestroy(psPMR->hBitmapLock);
+	OSLockDestroy(psPMR->hLock);
+	OSFreeMem(psPMR);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static INLINE PMR_ZOMBIE_TYPE
+PMR_GetZombieTypeFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return psPMRHeader->eZombieType;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static INLINE PMR_ZOMBIE_PAGES*
+PMR_GetZombiePagesFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR_ZOMBIE_PAGES, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static INLINE PMR_DEVICE_IMPORT*
+PMR_GetDeviceImportFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR_DEVICE_IMPORT, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+static INLINE PMR*
+PMR_GetPMRFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_UnrefAndMaybeDestroy(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	const PMR_IMPL_FUNCTAB *psFuncTable;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_BOOL bQueuedDeviceImports = IMG_FALSE;
+#endif
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	psFuncTable = psPMR->psFuncTab;
+
+	_FactoryLock(psFuncTable);
+
+	eError = _Unref(psPMR, uiRefCount, &uiRefCount);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_Unref", ErrFactoryUnlock);
+
+	if (uiRefCount > 0)
+	{
+		/* PMR is still referenced so just return */
+		_FactoryUnlock(psFuncTable);
+		return PVRSRV_OK;
+	}
+
+#if !defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Don't defer PMR destruction in NoHW and PDUMP drivers. */
+	_PMRDestroy(psPMR);
+#else /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+	psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+
+	/* PMRs that are not marked for deferred free can be freed right away.
+	 * Those are the PMRs that have not been mapped to the device.
+	 * All PMRs that have been mapped to the device need to go through
+	 * the defer free path unless the power is OFF for the PMR's device
+	 * and for all of the device imports. If power is OFF
+	 * the cache invalidation comes as a given. */
+	if (!_IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_DEFER_FREE))
+	{
+		_PMRDestroy(psPMR);
+		goto exit_;
+	}
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	bQueuedDeviceImports = _DeviceImportsEnqueueZombies(psPMR);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	if (!bQueuedDeviceImports
+	    && !_IsDeviceOnAndOperating(psDevNode)
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	    && !PVRSRV_CHECK_OS_LINUX_MOVABLE(psPMR->uiFlags)
+#endif
+	    )
+	{
+		_PMRDestroy(psPMR);
+	}
+	else
+	{
+		/* Defer freeing the PMR until the Firmware invalidates the caches. */
+		_ZombieListLock(psDevNode);
+
+		_IntFlagSet(psPMR, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+
+		dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMR->sHeader.sZombieNode);
+		psDevNode->uiPMRZombieCount++;
+
+		/* PMR pages are accounted by the driver/process stats. Those stats
+		 * are available on page level hence they need to be adjusted by
+		 * the factories. This is done by the pfnZombify callback.
+		 * Operation needs to be done while holding hPMRZombieListLock
+		 * to prevent CleanupThread from freeing pages while memory stats
+		 * accounting is ongoing. */
+		if (psPMR->psFuncTab->pfnZombify != NULL)
+		{
+			eError = psPMR->psFuncTab->pfnZombify(psPMR->pvFlavourData, psPMR);
+			PVR_LOG_IF_ERROR(eError, "pfnZombify");
+		}
+
+		_ZombieListUnlock(psDevNode);
+	}
+exit_:
+#endif /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	_FactoryUnlock(psFuncTable);
+
+	return PVRSRV_OK;
+
+ErrFactoryUnlock:
+	_FactoryUnlock(psFuncTable);
+
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+typedef struct _PMR_ZOMBIE_CLEANUP_ITEM_
+{
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+	DLLIST_NODE sZombieList;                     /*!< See _ZombieListLock */
+	PPVRSRV_DEVICE_NODE psDevNode;
+	PVRSRV_CLIENT_SYNC_PRIM *psSync;
+	IMG_UINT32 uiRequiredSyncValue;
+	IMG_UINT32 uiRequiredPowerOffCounter;
+} PMR_ZOMBIE_CLEANUP_ITEM;
+
+static INLINE IMG_BOOL
+_CanNotFreeZombies(const PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem)
+{
+	const PVRSRV_DEVICE_NODE *psDevNode = psCleanupItem->psDevNode;
+
+	/* For a zombie PMR to be eligible to be freed either the GPU MMU caches
+	 * need to be flushed (the Firmware updates the sync) or the GPU power needs
+	 * to be off. */
+	return !PVRSRVHasCounter32Advanced(OSReadDeviceMem32(psCleanupItem->psSync->pui32LinAddr),
+	                                   psCleanupItem->uiRequiredSyncValue) &&
+	       !PVRSRVHasCounter32Advanced(psDevNode->uiPowerOffCounter,
+	                                   psCleanupItem->uiRequiredPowerOffCounter);
+}
+
+static PVRSRV_ERROR _PmrZombieCleanup(void *pvData)
+{
+	PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem = pvData;
+	DLLIST_NODE *psNode;
+	DLLIST_NODE sRetryHead;
+	IMG_UINT32 uiRetryCount = 0;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (_CanNotFreeZombies(psCleanupItem))
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+	dllist_init(&sRetryHead);
+
+	do
+	{
+		/* hPMRZombieListLock will prevent removing a node while the list is
+		 * processed. If the lock is already acquired by other process which
+		 * intends to remove an item from the list it'll assure the list
+		 * consistency.
+		 * If this thread acquires the lock first it's possible that another
+		 * thread might be holding PMR factory lock. */
+
+		_ZombieListLock(psCleanupItem->psDevNode);
+		psNode = dllist_get_next_node(&psCleanupItem->sZombieList);
+		_ZombieListUnlock(psCleanupItem->psDevNode);
+
+		if (psNode == NULL)
+		{
+			continue;
+		}
+
+		switch (PMR_GetZombieTypeFromNode(psNode))
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_PAGES:
+			{
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psNode);
+				eError = psZombiePages->pfnFactoryFreeZombies(psZombiePages->pvFactoryPages);
+				_ZombieListLock(psCleanupItem->psDevNode);
+				dllist_remove_node(psNode);
+				psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+				_ZombieListUnlock(psCleanupItem->psDevNode);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Cannot free zombie pages! Skipping object %p", psZombiePages));
+					dllist_add_to_tail(&sRetryHead, psNode);
+					uiRetryCount++;
+				}
+				else
+				{
+					OSFreeMem(psZombiePages);
+				}
+				break;
+			}
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_DEVICE_IMPORT:
+			{
+				PMR_DEVICE_IMPORT *psImport = PMR_GetDeviceImportFromNode(psNode);
+				_ZombieListLock(psCleanupItem->psDevNode);
+				dllist_remove_node(psNode);
+				psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+				_ZombieListUnlock(psCleanupItem->psDevNode);
+				_DeviceImportFreeImportZombie(psImport);
+				break;
+			}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+			case PMR_ZOMBIE_TYPE_PMR:
+			{
+				PMR* psPMR = PMR_GetPMRFromNode(psNode);
+				const PMR_IMPL_FUNCTAB *psFuncTable = psPMR->psFuncTab;
+
+				_FactoryLock(psFuncTable);
+				_ZombieListLock(psCleanupItem->psDevNode);
+				/* It is possible that the element might have been removed so
+				 * we have to check if the PMR is still a zombie.
+				 * It's also possible that the PMR has been revived
+				 * (PMRReviveZombieAndRef()), mapped, unmapped and zombified
+				 * again while the lock was not held.
+				 * Considering above only immediately free the PMR if the
+				 * PMR is still a part of this cleanup item. */
+				if (psNode == dllist_get_next_node(&psCleanupItem->sZombieList))
+				{
+					dllist_remove_node(psNode);
+					psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+					/* The PMR cannot be freed if other devices are
+					 * still waiting for the cache flush. */
+					if (_DeviceImportBitmapGet(psPMR) != 0)
+					{
+						/* Request it to be retried and continue
+						 * to the next zombie item. */
+						dllist_add_to_tail(&sRetryHead, psNode);
+						uiRetryCount++;
+						_ZombieListUnlock(psCleanupItem->psDevNode);
+						_FactoryUnlock(psFuncTable);
+						continue;
+					}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+					/* Unlock here to avoid locking dependency with the power lock.
+					 * It's okay to do it here since the factory lock is the one
+					 * that needs to be held during PMR destruction. */
+					_ZombieListUnlock(psCleanupItem->psDevNode);
+					_PMRDestroy(psPMR);
+				}
+				else
+				{
+					_ZombieListUnlock(psCleanupItem->psDevNode);
+				}
+				_FactoryUnlock(psFuncTable);
+				break;
+			}
+		}
+	} while (psNode != NULL);
+
+	if (uiRetryCount)
+	{
+		eError = PVRSRV_ERROR_RETRY;
+		_ZombieListLock(psCleanupItem->psDevNode);
+		/* Add the retry items back to this cleanup item for when the
+		 * cleanup item is retried. Oldest items will reside at the head of
+		 * the list. The cleanup item will be placed at the back of the cleanup
+		 * queue to process other dependencies first. */
+		dllist_insert_list_at_head(&psCleanupItem->sZombieList, &sRetryHead);
+		psCleanupItem->psDevNode->uiPMRZombieCountInCleanup += uiRetryCount;
+		_ZombieListUnlock(psCleanupItem->psDevNode);
+	}
+	else
+	{
+		OSFreeMem(psCleanupItem);
+	}
+
+	return eError;
+}
+
+IMG_BOOL PMRQueueZombiesForCleanup(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem;
+
+	/* Don't defer the freeing if we are currently unloading the driver
+	 * or if the sync has been destroyed */
+	if (psPVRSRVData->bUnload || psDevNode->psMMUCacheSyncPrim == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	_ZombieListLock(psDevNode);
+
+	if (dllist_is_empty(&psDevNode->sPMRZombieList))
+	{
+		_ZombieListUnlock(psDevNode);
+		return IMG_FALSE;
+	}
+
+	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
+	if (psCleanupItem == NULL)
+	{
+		_ZombieListUnlock(psDevNode);
+		return IMG_FALSE;
+	}
+
+	psCleanupItem->sCleanupThreadFn.pfnFree = _PmrZombieCleanup;
+	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
+	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+	psCleanupItem->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_PMR;
+	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
+	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);
+
+	psCleanupItem->psDevNode = psDevNode;
+	psCleanupItem->psSync = psDevNode->psMMUCacheSyncPrim;
+	psCleanupItem->uiRequiredSyncValue = psDevNode->ui32NextMMUInvalidateUpdate;
+	psCleanupItem->uiRequiredPowerOffCounter = psDevNode->uiPowerOffCounterNext;
+
+	/* This moves the zombie list to the cleanup item. */
+	dllist_replace_head(&psDevNode->sPMRZombieList, &psCleanupItem->sZombieList);
+	psDevNode->uiPMRZombieCountInCleanup += psDevNode->uiPMRZombieCount;
+	psDevNode->uiPMRZombieCount = 0;
+
+	OSLockRelease(psDevNode->hPMRZombieListLock);
+
+	PVRSRVCleanupThreadAddWork(psDevNode, &psCleanupItem->sCleanupThreadFn);
+
+	return IMG_TRUE;
+}
+
+void
+PMRReviveZombieAndRef(PMR *psPMR)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	DLLIST_NODE *psThis, *psNext;
+	IMG_BOOL bIsOnZombieList = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	psDeviceNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+
+	/* If this was on a list then it's brought back to life. */
+	_ZombieListLock(psDeviceNode);
+
+	/* Need to reference this PMR since it was about to be destroyed and its
+	 * reference count must be 0 (can't use _Ref() due to the warning). */
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+	psPMR->uiRefCount++;
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+#if  defined(DEBUG)
+	PVR_LOG(("%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+#endif
+
+	/* If we got to this point the PMR must be on a list. If it's not
+	 * it should mean a race of some sort. */
+	PVR_ASSERT(!dllist_is_empty(&psPMR->sHeader.sZombieNode));
+
+	/* For the sake of correct accounting check if the PMR is in the zombie
+	 * list or in the cleanup item. */
+	dllist_foreach_node(&psDeviceNode->sPMRZombieList, psThis, psNext)
+	{
+		if (psThis == &psPMR->sHeader.sZombieNode)
+		{
+			bIsOnZombieList = IMG_TRUE;
+			break;
+		}
+	}
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	if (_DeviceImportBitmapGet(psPMR) != 0) {
+		PMRLockPMR(psPMR);
+		_DeviceImportsReviveZombies(psPMR);
+		PMRUnlockPMR(psPMR);
+	}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	/* Revive the PMR (remove it from the zombie list) and therefore
+	 * prevent it's destruction. */
+	dllist_remove_node(&psPMR->sHeader.sZombieNode);
+	_IntFlagClr(psPMR, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+
+	if (bIsOnZombieList)
+	{
+		psDeviceNode->uiPMRZombieCount--;
+	}
+	else
+	{
+		psDeviceNode->uiPMRZombieCountInCleanup--;
+	}
+
+	_ZombieListUnlock(psDeviceNode);
+}
+
+void
+PMRMarkForDeferFree(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	if (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags))
+	{
+		/* If PMR pages are allocated on demand the freeing is handled
+		 * by `SUPPORT_PMR_PAGES_DEFERRED_FREE` path in
+		 * `PMRUnlockSysPhysAddressesNested()`. */
+		return;
+	}
+
+	_IntFlagSet(psPMR, PMR_FLAG_INTERNAL_DEFER_FREE);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static INLINE IMG_BOOL _PMRIsSparse(const PMR *psPMR)
+{
+	return _IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_SPARSE_ALLOC);
+}
+
+PVRSRV_ERROR
+PMRCreatePMR(PHYS_HEAP *psPhysHeap,
+             PMR_SIZE_T uiLogicalSize,
+             IMG_UINT32 ui32NumPhysChunks,
+             IMG_UINT32 ui32NumLogicalChunks,
+             IMG_UINT32 *pui32MappingTable,
+             PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+             PMR_FLAGS_T uiFlags,
+             const IMG_CHAR *pszAnnotation,
+             const PMR_IMPL_FUNCTAB *psFuncTab,
+             PMR_IMPL_PRIVDATA pvPrivData,
+             PMR_IMPL_TYPE eType,
+             PMR **ppsPMRPtr,
+             IMG_UINT32 ui32PDumpFlags)
+{
+	PMR *psPMR = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+
+	eError = _PMRCreate(uiLogicalSize,
+	                    ui32NumPhysChunks,
+	                    ui32NumLogicalChunks,
+	                    pui32MappingTable,
+	                    uiLog2ContiguityGuarantee,
+	                    uiFlags,
+	                    &psPMR);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	psPMR->psPhysHeap = psPhysHeap;
+	psPMR->psFuncTab = psFuncTab;
+	psPMR->pszPDumpDefaultMemspaceName = PhysHeapPDumpMemspaceName(psPhysHeap);
+	psPMR->pvFlavourData = pvPrivData;
+	psPMR->eFlavour = eType;
+	psPMR->uiRefCount = 1;
+
+	OSStringSafeCopy(psPMR->szAnnotation, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PDUMP)
+	/* if allocation was done on the host node don't include it in the PDUMP */
+	if (!_IsHostDevicePMR(psPMR))
+	{
+		PMR_FLAGS_T uiFlags = psPMR->uiFlags;
+		IMG_BOOL bInitialise = IMG_FALSE;
+		IMG_UINT8 ui8InitValue = 0;
+
+		if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+		{
+			bInitialise = IMG_TRUE;
+		}
+		else if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+		{
+			ui8InitValue = (IMG_UINT8)PVRSRV_POISON_ON_ALLOC_VALUE;
+			bInitialise = IMG_TRUE;
+		}
+
+		PDumpPMRMallocPMR(psPMR,
+		                  uiLogicalSize,
+		                  IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee),
+		                  ui32NumPhysChunks,
+		                  ui32NumLogicalChunks,
+		                  pui32MappingTable,
+		                  uiLog2ContiguityGuarantee,
+		                  bInitialise,
+		                  ui8InitValue,
+		                  &psPMR->hPDumpAllocHandle,
+		                  ui32PDumpFlags);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psPMR->eState = PMR_STATE_ACTIVE;
+#endif
+
+	*ppsPMRPtr = psPMR;
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRLockSysPhysAddresses(PMR *psPMR)
+{
+	return PMRLockSysPhysAddressesNested(psPMR, 1, 0);
+}
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount)
+{
+	return PMRLockSysPhysAddressesNested(psPMR, uiLockCount, 0);
+}
+
+PVRSRV_ERROR PMRLockSysPhysAddressesNested(PMR *psPMR,
+                                        IMG_UINT32 uiLockCount,
+                                        IMG_UINT32 ui32NestingLevel)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiCallbackValue;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	/* Note: taking this lock is not required to protect the PMR reference
+	 * count, because the PMR reference count is atomic. Rather, taking
+	 * the lock here guarantees that no caller will exit this function
+	 * without the underlying physical addresses being locked.
+	 */
+	OSLockAcquireNested(psPMR->hLock, ui32NestingLevel);
+	/* We also count the locks as references, so that the PMR is not freed
+	 * while someone is using a physical address.
+	 * "lock" here simply means incrementing the refcount. It means the
+	 * refcount is multipurpose, but that's okay. We only have to promise
+	 * that physical addresses are valid after this point, and remain valid
+	 * until the corresponding PMRUnlockSysPhysAddressesOSMem()
+	 */
+	eError = _Ref(psPMR, uiLockCount);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psPMR->hLock);
+		return eError;
+	}
+
+	/* Also count locks separately from other types of references, to
+	 * allow for debug assertions
+	 */
+
+	/* Only call callback if lock count transitions from 0 to 1 (or 1 to 2 if not
+	 * backed on demand) */
+	uiCallbackValue = uiLockCount + (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1);
+
+	if (OSAtomicAdd(&psPMR->iLockCount, uiLockCount) == uiCallbackValue)
+	{
+		if (psPMR->psFuncTab->pfnLockPhysAddresses != NULL)
+		{
+			/* must always have lock and unlock in pairs! */
+			PVR_ASSERT(psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL);
+
+			eError = psPMR->psFuncTab->pfnLockPhysAddresses(psPMR->pvFlavourData);
+
+			PVR_GOTO_IF_ERROR(eError, e1);
+		}
+	}
+	OSLockRelease(psPMR->hLock);
+
+	return PVRSRV_OK;
+
+e1:
+	OSAtomicSubtract(&psPMR->iLockCount, uiLockCount);
+	PVR_ASSERT(_GetRef(psPMR) != 0);
+	OSLockRelease(psPMR->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	(void) _Unref(psPMR, uiLockCount, NULL);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddresses(PMR *psPMR)
+{
+	return PMRUnlockSysPhysAddressesNested(psPMR, 1, 2);
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount)
+{
+	return PMRUnlockSysPhysAddressesNested(psPMR, uiLockCount, 2);
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesNested(PMR *psPMR,
+                             IMG_UINT32 uiLockCount,
+                             IMG_UINT32 ui32NestingLevel)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_IMPL_ZOMBIEPAGES pvZombiePages = NULL;
+	PMR_ZOMBIE_PAGES* psPMRZombiePages = NULL;
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+
+	/* Acquiring the lock here, as well as during the Lock operation ensures
+	 * the lock count hitting zero and the unlocking of the phys addresses is
+	 * an atomic operation
+	 */
+	OSLockAcquireNested(psPMR->hLock, ui32NestingLevel);
+
+	PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) >= uiLockCount);
+
+	if (OSAtomicSubtract(&psPMR->iLockCount, uiLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1))
+	{
+		if (psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL)
+		{
+			PVR_ASSERT(psPMR->psFuncTab->pfnLockPhysAddresses != NULL);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData,
+			                                                  &pvZombiePages);
+#else
+			eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData);
+#endif
+			PVR_LOG_IF_ERROR(eError, "pfnUnlockPhysAddresses");
+		}
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		psPMRZombiePages = OSAllocZMem(sizeof(PMR_ZOMBIE_PAGES));
+		PVR_GOTO_IF_NOMEM(psPMRZombiePages, eError, ErrRelockPhysAddresses);
+	}
+#endif
+
+	OSLockRelease(psPMR->hLock);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+		PVRSRV_DEVICE_NODE *psDevNode;
+
+		psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+		eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+		if (eError != PVRSRV_OK)
+		{
+			/* Treat unknown power state as ON. */
+			ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+		}
+
+		if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+			psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+		{
+			/* Free preallocated psPMRZombiePages as these won't be used*/
+			OSFreeMem(psPMRZombiePages);
+
+			eError = psPMR->psFuncTab->pfnFreeZombiePages(pvZombiePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "Error when trying to free zombies immediately.",
+			                      ErrReturn);
+		}
+		else
+		{
+			PVR_ASSERT(psPMRZombiePages != NULL);
+			psPMRZombiePages->sHeader.eZombieType = PMR_ZOMBIE_TYPE_PAGES;
+			psPMRZombiePages->pfnFactoryFreeZombies = psPMR->psFuncTab->pfnFreeZombiePages;
+			psPMRZombiePages->pvFactoryPages = pvZombiePages;
+
+			_ZombieListLock(psDevNode);
+			dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMRZombiePages->sHeader.sZombieNode);
+			psDevNode->uiPMRZombieCount++;
+			_ZombieListUnlock(psDevNode);
+		}
+	}
+	else
+	{
+		OSFreeMem(psPMRZombiePages);
+	}
+#endif
+
+	/* We also count the locks as references, so that the PMR is not
+	 * freed while someone is using a physical address.
+	 */
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	(void) _UnrefAndMaybeDestroy(psPMR, uiLockCount);
+
+	return eError;
+#else
+	return _UnrefAndMaybeDestroy(psPMR, uiLockCount);
+#endif
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+ErrRelockPhysAddresses:
+	{
+		IMG_UINT32 uiCallbackValue =
+		    uiLockCount + (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1);
+
+		if (OSAtomicAdd(&psPMR->iLockCount, uiLockCount) == uiCallbackValue)
+		{
+			if (psPMR->psFuncTab->pfnLockPhysAddresses != NULL)
+			{
+				eError = psPMR->psFuncTab->pfnLockPhysAddresses(psPMR->pvFlavourData);
+				PVR_LOG_IF_ERROR(eError, "pfnLockPhysAddresses");
+			}
+		}
+	}
+
+	OSLockRelease(psPMR->hLock);
+
+ErrReturn:
+	return eError;
+#endif
+}
+
+PVRSRV_ERROR
+PMRMakeLocalImportHandle(PMR *psPMR,
+                         PMR **ppsPMR)
+{
+	PVRSRV_ERROR eError = PMRRefPMR(psPMR);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppsPMR = psPMR;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRUnmakeLocalImportHandle(PMR *psPMR)
+{
+	return PMRUnrefPMR(psPMR);
+}
+
+/*
+	Note:
+	We pass back the PMR as it was passed in as a different handle type
+	(DEVMEM_MEM_IMPORT) and it allows us to change the import structure
+	type if we should need to embed any meta data in it.
+ */
+PVRSRV_ERROR
+PMRLocalImportPMR(PMR *psPMR,
+                  PMR **ppsPMR,
+                  IMG_DEVMEM_SIZE_T *puiSize,
+                  IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PVRSRV_ERROR eError = _Ref(psPMR, 1);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Return the PMR */
+	*ppsPMR = psPMR;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiAlign = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	return PVRSRV_OK;
+}
+
+inline IMG_UINT64
+PMRInternalGetUID(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiSerialNum;
+}
+
+PVRSRV_ERROR
+PMRGetUID(PMR *psPMR,
+          IMG_UINT64 *pui64UID)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	*pui64UID = PMRInternalGetUID(psPMR);
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR
+PMRExportPMR(PMR *psPMR,
+             PMR_EXPORT **ppsPMRExportPtr,
+             PMR_SIZE_T *puiSize,
+             PMR_LOG2ALIGN_T *puiLog2Contig,
+             PMR_PASSWORD_T *puiPassword)
+{
+	IMG_UINT64 uiPassword;
+	PMR_EXPORT *psPMRExport;
+	PVRSRV_ERROR eError;
+
+	uiPassword = psPMR->uiKey;
+
+	psPMRExport = OSAllocMem(sizeof(*psPMRExport));
+	PVR_RETURN_IF_NOMEM(psPMRExport);
+
+	psPMRExport->psPMR = psPMR;
+
+	eError = _Ref(psPMR, 1);
+	PVR_GOTO_IF_ERROR(eError, ErrFreePMRExport);
+
+	/* The layout of a PMR can't change once exported
+	 * to make sure the importers view of the memory is
+	 * the same as exporter. */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	*ppsPMRExportPtr = psPMRExport;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiLog2Contig = psPMR->uiLog2ContiguityGuarantee;
+	*puiPassword = uiPassword;
+
+	return PVRSRV_OK;
+
+ErrFreePMRExport:
+	OSFreeMem(psPMRExport);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMRUnexportPMR(PMR_EXPORT *psPMRExport)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMRExport != NULL);
+	PVR_ASSERT(psPMRExport->psPMR != NULL);
+	PVR_ASSERT(psPMRExport->psPMR->uiRefCount > 0);
+
+	eError = _UnrefAndMaybeDestroy(psPMRExport->psPMR, 1);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_UnrefAndMaybeDestroy");
+
+	OSFreeMem(psPMRExport);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PMRImportPMR(PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR)
+{
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMRExport->psPMR->uiRefCount > 0);
+
+	psPMR = psPMRExport->psPMR;
+
+	PVR_ASSERT(PMR_IsMemLayoutFixed(psPMR));
+
+	if (psPMR->uiKey != uiPassword)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"PMRImport: Import failed, password specified does not match the export"));
+		return PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR;
+	}
+
+	if (psPMR->uiLogicalSize != uiSize || psPMR->uiLog2ContiguityGuarantee != uiLog2Contig)
+	{
+		return PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES;
+	}
+
+	eError = _Ref(psPMR, 1);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppsPMR = psPMR;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRUnimportPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+
+#endif /* if defined(SUPPORT_INSECURE_EXPORT) */
+
+#if defined(SUPPORT_SECURE_EXPORT)
+PVRSRV_ERROR PMRSecureUnexportPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+
+static PVRSRV_ERROR _ReleaseSecurePMR(void *psExport)
+{
+	return PMRSecureUnexportPMR(psExport);
+}
+
+PVRSRV_ERROR PMRSecureExportPMR(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+                                PMR *psPMR,
+                                IMG_SECURE_TYPE *phSecure,
+                                PMR **ppsPMR,
+                                CONNECTION_DATA **ppsSecureConnection)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ppsSecureConnection);
+
+	/* We are acquiring reference to PMR here because OSSecureExport
+	 * releases bridge lock and PMR lock for a moment and we don't want PMR
+	 * to be removed by other thread in the meantime. */
+	eError = _Ref(psPMR, 1);
+	PVR_RETURN_IF_ERROR(eError);
+
+	eError = OSSecureExport("secure_pmr",
+	                        _ReleaseSecurePMR,
+	                        (void *) psPMR,
+	                        phSecure);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	*ppsPMR = psPMR;
+
+	/* Mark the PMR immutable once exported
+	 * This allows the importers and exporter to have
+	 * the same view of the memory */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	return PVRSRV_OK;
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	(void) _UnrefAndMaybeDestroy(psPMR, 1);
+	return eError;
+}
+
+PVRSRV_ERROR PMRSecureImportPMR(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE *psDevNode,
+                                IMG_SECURE_TYPE hSecure,
+                                PMR **ppsPMR,
+                                IMG_DEVMEM_SIZE_T *puiSize,
+                                IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMR;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = OSSecureImport(hSecure, (void **) &psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrReturnError);
+
+	PVR_LOG_RETURN_IF_FALSE(PhysHeapDeviceNode(psPMR->psPhysHeap) == psDevNode,
+	                        "PMR invalid for this device",
+	                        PVRSRV_ERROR_PMR_NOT_PERMITTED);
+
+	eError = _Ref(psPMR, 1);
+	PVR_GOTO_IF_ERROR(eError, ErrReturnError);
+
+	/* The PMR should be immutable once exported
+	 * This allows the importers and exporter to have
+	 * the same view of the memory */
+	PVR_ASSERT(PMR_IsMemLayoutFixed(psPMR));
+
+	/* Return the PMR */
+	*ppsPMR = psPMR;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiAlign = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+
+	return PVRSRV_OK;
+
+ErrReturnError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMRSecureUnimportPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR
+PMRStoreRIHandle(PMR *psPMR,
+                 void *hRIHandle)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	psPMR->hRIHandle = hRIHandle;
+	return PVRSRV_OK;
+}
+#endif
+
+static PVRSRV_ERROR
+_PMRAcquireKernelMappingData(PMR *psPMR,
+                             size_t uiLogicalOffset,
+                             size_t uiSize,
+                             void **ppvKernelAddressOut,
+                             size_t *puiLengthOut,
+                             IMG_HANDLE *phPrivOut,
+                             IMG_BOOL bMapSparse)
+{
+	PVRSRV_ERROR eError;
+	void *pvKernelAddress;
+	IMG_HANDLE hPriv;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	if (_PMRIsSparse(psPMR) && !bMapSparse)
+	{
+		/* Mapping of sparse allocations must be signalled. */
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	/* Acquire/Release functions must be overridden in pairs */
+	if (psPMR->psFuncTab->pfnAcquireKernelMappingData == NULL)
+	{
+		PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData == NULL);
+
+		/* If PMR implementation does not supply this pair of
+		 * functions, it means they do not permit the PMR to be mapped
+		 * into kernel memory at all
+		 */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PERMITTED, e0);
+	}
+	PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL);
+
+	eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+	                                                       uiLogicalOffset,
+	                                                       uiSize,
+	                                                       &pvKernelAddress,
+	                                                       &hPriv,
+	                                                       psPMR->uiFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	*ppvKernelAddressOut = pvKernelAddress;
+	if (uiSize == 0)
+	{
+		/* Zero size means map in the whole PMR ... */
+		*puiLengthOut = (size_t)psPMR->uiLogicalSize;
+	}
+	else if (uiSize > IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee))
+	{
+		/* ... map in the requested pages ... */
+		*puiLengthOut = uiSize;
+	}
+	else
+	{
+		/* ... otherwise we just map in one page */
+		*puiLengthOut = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	}
+	*phPrivOut = hPriv;
+
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRAcquireKernelMappingData(PMR *psPMR,
+                            size_t uiLogicalOffset,
+                            size_t uiSize,
+                            void **ppvKernelAddressOut,
+                            size_t *puiLengthOut,
+                            IMG_HANDLE *phPrivOut)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountIncr(psPMR);
+
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psPMR)))
+	{
+		do
+		{
+			eError = _PMRAcquireKernelMappingData(psPMR,
+			                                      uiLogicalOffset,
+			                                      uiSize,
+			                                      ppvKernelAddressOut,
+			                                      puiLengthOut,
+			                                      phPrivOut,
+			                                      IMG_FALSE);
+		}
+		while (eError == PVRSRV_ERROR_RETRY);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PMRAcquireKernelMappingData", error_fail_decr);
+	}
+	else
+#endif
+	{
+		eError = _PMRAcquireKernelMappingData(psPMR,
+		                                      uiLogicalOffset,
+		                                      uiSize,
+		                                      ppvKernelAddressOut,
+		                                      puiLengthOut,
+		                                      phPrivOut,
+		                                      IMG_FALSE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PMRAcquireKernelMappingData", error_fail_decr);
+	}
+
+	return eError;
+
+error_fail_decr:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountDecr(psPMR);
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRAcquireSparseKernelMappingData(PMR *psPMR,
+                                  size_t uiLogicalOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  size_t *puiLengthOut,
+                                  IMG_HANDLE *phPrivOut)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountIncr(psPMR);
+#endif
+
+	eError =  _PMRAcquireKernelMappingData(psPMR,
+	                                    uiLogicalOffset,
+	                                    uiSize,
+	                                    ppvKernelAddressOut,
+	                                    puiLengthOut,
+	                                    phPrivOut,
+	                                    IMG_TRUE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_PMRAcquireKernelMappingData", error_fail_decr);
+
+	return eError;
+
+error_fail_decr:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountDecr(psPMR);
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRReleaseKernelMappingData(PMR *psPMR,
+                            IMG_HANDLE hPriv)
+{
+	PVR_ASSERT (psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL);
+	PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL);
+
+
+	psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+	                                              hPriv);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountDecr(psPMR);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	_PMRLogicalOffsetToPhysicalOffset
+
+	Translate between the "logical" offset which the upper levels
+	provide and the physical offset which is what the PMR
+	factories works on.
+
+	As well as returning the physical offset we return the number of
+	bytes remaining till the next chunk and if this chunk is valid.
+
+	For multi-page operations, upper layers communicate their
+	Log2PageSize else argument is redundant (set to zero).
+ */
+
+static PVRSRV_ERROR
+_PMRLogicalOffsetToPhysicalOffset(const PMR *psPMR,
+                                  IMG_UINT32 ui32Log2PageSize,
+                                  IMG_UINT32 ui32NumOfPages,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_DEVMEM_OFFSET_T *puiPhysicalOffset,
+                                  IMG_UINT32 *pui32BytesRemain,
+                                  IMG_BOOL *bValid)
+{
+	PMR_MAPPING_TABLE *psMappingTable = psPMR->psMappingTable;
+	IMG_DEVMEM_OFFSET_T uiPageSize = 1ULL << ui32Log2PageSize;
+	IMG_DEVMEM_OFFSET_T uiOffset = uiLogicalOffset;
+	IMG_UINT64 ui64ChunkIndex;
+	IMG_UINT32 ui32Remain;
+	IMG_UINT32 idx;
+
+	/* Must be translating at least a page */
+	PVR_ASSERT(ui32NumOfPages > 0);
+
+	if (psMappingTable->ui32NumPhysChunks == psMappingTable->ui32NumLogicalChunks)
+	{
+		/* Fast path the common case, as logical and physical offsets are
+		 * equal we assume the ui32NumOfPages span is also valid */
+		*pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiOffset);
+		puiPhysicalOffset[0] = uiOffset;
+		bValid[0] = IMG_TRUE;
+
+		if (ui32NumOfPages > 1)
+		{
+			/* initial offset may not be page aligned, round down */
+			uiOffset &= ~(uiPageSize-1);
+			for (idx = 1; idx < ui32NumOfPages; idx++)
+			{
+				uiOffset += uiPageSize;
+				puiPhysicalOffset[idx] = uiOffset;
+				bValid[idx] = IMG_TRUE;
+			}
+		}
+	}
+	else
+	{
+		for (idx = 0; idx < ui32NumOfPages; idx++)
+		{
+			IMG_UINT32 uiTranslation;
+			const IMG_UINT32 uiChunkSize = psMappingTable->uiChunkSize;
+
+			ui64ChunkIndex = OSDivide64r64(
+					uiOffset,
+					TRUNCATE_64BITS_TO_32BITS(uiChunkSize),
+					&ui32Remain);
+
+			/* In some cases ui32NumOfPages can come from the user space which
+			 * means that the uiOffset could go out-of-bounds when the number
+			 * of pages is invalid. */
+			if (ui64ChunkIndex >= psMappingTable->ui32NumLogicalChunks)
+			{
+				return PVRSRV_ERROR_BAD_MAPPING;
+			}
+
+			uiTranslation = psMappingTable->aui32Translation[ui64ChunkIndex];
+
+			if (uiTranslation == TRANSLATION_INVALID)
+			{
+				bValid[idx] = IMG_FALSE;
+				/* explicitly set to an invalid value */
+				puiPhysicalOffset[idx] = IMG_UINT64_C(0xffffffffffffffff);
+			}
+			else
+			{
+				bValid[idx] = IMG_TRUE;
+				puiPhysicalOffset[idx] = uiTranslation * uiChunkSize + ui32Remain;
+			}
+
+			if (idx == 0)
+			{
+				*pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(uiChunkSize - ui32Remain);
+				/* initial offset may not be page aligned, round down */
+				uiOffset &= ~(uiPageSize-1);
+			}
+
+			uiOffset += uiPageSize;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_PMR_ReadBytesPhysical(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                       IMG_UINT8 *pcBuffer,
+                       size_t uiBufSz,
+                       size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError;
+
+	if (psPMR->psFuncTab->pfnReadBytes != NULL)
+	{
+		/* defer to callback if present */
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		eError = psPMR->psFuncTab->pfnReadBytes(psPMR->pvFlavourData,
+		                                        uiPhysicalOffset,
+		                                        pcBuffer,
+		                                        uiBufSz,
+		                                        puiNumBytes);
+		PMRUnlockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+	}
+	else if (psPMR->psFuncTab->pfnAcquireKernelMappingData)
+	{
+		/* "default" handler for reading bytes */
+
+		IMG_HANDLE hKernelMappingHandle;
+		IMG_UINT8 *pcKernelAddress;
+
+		eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+		                                                       (size_t) uiPhysicalOffset,
+		                                                       uiBufSz,
+		                                                       (void **)&pcKernelAddress,
+		                                                       &hKernelMappingHandle,
+		                                                       psPMR->uiFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Use the conservative 'DeviceMemCopy' here because we can't
+		 * know if this PMR will be mapped cached.
+		 */
+
+		OSDeviceMemCopy(&pcBuffer[0], pcKernelAddress, uiBufSz);
+		*puiNumBytes = uiBufSz;
+
+		psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+		                                              hKernelMappingHandle);
+	}
+	else
+	{
+		OSPanic();
+#ifndef __CHECKER__ /* exclude for Smatch */
+		PVR_LOG_GOTO_WITH_ERROR("psPMR->psFuncTab", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+#endif
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*puiNumBytes = 0;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMR_ReadBytes(PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT8 *pcBuffer,
+              size_t uiBufSz,
+              size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	size_t uiBytesCopied = 0;
+
+	/* Check for integer overflow as uiLogicalOffset might come from the client */
+	if (uiLogicalOffset + uiBufSz < uiLogicalOffset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize)
+	{
+		uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset);
+	}
+	PVR_ASSERT(uiBufSz > 0);
+	PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize);
+
+	/* PMR implementations can override this. If they don't, a "default"
+	 * handler uses kernel virtual mappings.  If the kernel can't
+	 * provide a kernel virtual mapping, this function fails.
+	 */
+	PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL ||
+	           psPMR->psFuncTab->pfnReadBytes != NULL);
+
+	while (uiBytesCopied != uiBufSz)
+	{
+		IMG_UINT32 ui32Remain;
+		size_t uiBytesToCopy;
+		size_t uiRead;
+		IMG_BOOL bValid;
+
+		eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+		                                           0,
+		                                           1,
+		                                           uiLogicalOffset,
+		                                           &uiPhysicalOffset,
+		                                           &ui32Remain,
+		                                           &bValid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+		/* Copy till either then end of the chunk or end
+		 * of the buffer
+		 */
+		uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain);
+
+		if (bValid)
+		{
+			/* Read the data from the PMR */
+			eError = _PMR_ReadBytesPhysical(psPMR,
+			                                uiPhysicalOffset,
+			                                &pcBuffer[uiBytesCopied],
+			                                uiBytesToCopy,
+			                                &uiRead);
+			if ((eError != PVRSRV_OK) || (uiRead != uiBytesToCopy))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to read chunk (eError = %s, uiRead = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")",
+						__func__,
+						PVRSRVGetErrorString(eError),
+						uiRead,
+						uiBytesToCopy));
+				/* Bail out as soon as we hit an error */
+				break;
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Invalid phys offset at logical offset (" IMG_DEVMEM_OFFSET_FMTSPEC ") logical size (" IMG_DEVMEM_OFFSET_FMTSPEC ")",
+					__func__,
+					uiLogicalOffset,
+					psPMR->uiLogicalSize));
+			/* Fill invalid chunks with 0 */
+			OSCachedMemSet(&pcBuffer[uiBytesCopied], 0, uiBytesToCopy);
+			uiRead = uiBytesToCopy;
+			eError = PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR;
+		}
+		uiLogicalOffset += uiRead;
+		uiBytesCopied += uiRead;
+	}
+
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+static PVRSRV_ERROR
+_PMR_WriteBytesPhysical(PMR *psPMR,
+                        IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiBufSz,
+                        size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError;
+
+	if (psPMR->psFuncTab->pfnWriteBytes != NULL)
+	{
+		/* defer to callback if present */
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		eError = psPMR->psFuncTab->pfnWriteBytes(psPMR->pvFlavourData,
+		                                         uiPhysicalOffset,
+		                                         pcBuffer,
+		                                         uiBufSz,
+		                                         puiNumBytes);
+		PMRUnlockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+	}
+	else if (psPMR->psFuncTab->pfnAcquireKernelMappingData)
+	{
+		/* "default" handler for reading bytes */
+
+		IMG_HANDLE hKernelMappingHandle;
+		IMG_UINT8 *pcKernelAddress;
+
+		eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+		                                                       (size_t) uiPhysicalOffset,
+		                                                       uiBufSz,
+		                                                       (void **)&pcKernelAddress,
+		                                                       &hKernelMappingHandle,
+		                                                       psPMR->uiFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Use the conservative 'DeviceMemCopy' here because we can't know
+		 * if this PMR will be mapped cached.
+		 */
+
+		OSDeviceMemCopy(pcKernelAddress, &pcBuffer[0], uiBufSz);
+		*puiNumBytes = uiBufSz;
+
+		psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+		                                              hKernelMappingHandle);
+	}
+	else
+	{
+		/* The write callback is optional as it's only required by the
+		 * debug tools
+		 */
+		OSPanic();
+#ifndef __CHECKER__ /* exclude for Smatch */
+		PVR_LOG_GOTO_WITH_ERROR("psPMR->psFuncTab", eError, PVRSRV_ERROR_PMR_NOT_PERMITTED, e0);
+#endif
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*puiNumBytes = 0;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMR_WriteBytes(PMR *psPMR,
+               IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+               IMG_UINT8 *pcBuffer,
+               size_t uiBufSz,
+               size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	size_t uiBytesCopied = 0;
+
+	/* Check for integer overflow as uiLogicalOffset might come from the client */
+	if (uiLogicalOffset + uiBufSz < uiLogicalOffset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize)
+	{
+		uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset);
+	}
+	PVR_ASSERT(uiBufSz > 0);
+	PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize);
+
+	/* PMR implementations can override this. If they don't, a "default"
+	 * handler uses kernel virtual mappings. If the kernel can't provide
+	 * a kernel virtual mapping, this function fails.
+	 */
+	PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL ||
+	           psPMR->psFuncTab->pfnWriteBytes != NULL);
+
+	while (uiBytesCopied != uiBufSz)
+	{
+		IMG_UINT32 ui32Remain;
+		size_t uiBytesToCopy;
+		size_t uiWrite;
+		IMG_BOOL bValid;
+
+		eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+		                                           0,
+		                                           1,
+		                                           uiLogicalOffset,
+		                                           &uiPhysicalOffset,
+		                                           &ui32Remain,
+		                                           &bValid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+		/* Copy till either then end of the chunk or end of the buffer
+		 */
+		uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain);
+
+		if (bValid)
+		{
+			/* Write the data to the PMR */
+			eError = _PMR_WriteBytesPhysical(psPMR,
+			                                 uiPhysicalOffset,
+			                                 &pcBuffer[uiBytesCopied],
+			                                 uiBytesToCopy,
+			                                 &uiWrite);
+			if ((eError != PVRSRV_OK) || (uiWrite != uiBytesToCopy))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to read chunk (eError = %s, uiWrite = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")",
+						__func__,
+						PVRSRVGetErrorString(eError),
+						uiWrite,
+						uiBytesToCopy));
+				/* Bail out as soon as we hit an error */
+				break;
+			}
+		}
+		else
+		{
+			/* Ignore writes to invalid pages */
+			uiWrite = uiBytesToCopy;
+		}
+		uiLogicalOffset += uiWrite;
+		uiBytesCopied += uiWrite;
+	}
+
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRMMapPMR(PMR *psPMR, PMR_MMAP_DATA pOSMMapData, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	/* if writeable mapping is requested on non-writeable PMR then fail */
+	PVR_RETURN_IF_FALSE(PVRSRV_CHECK_CPU_WRITEABLE(psPMR->uiFlags) ||
+	                    !PVRSRV_CHECK_CPU_WRITEABLE(uiFlags),
+	                    PVRSRV_ERROR_PMR_NOT_PERMITTED);
+
+	PVR_LOG_RETURN_IF_TRUE(PMR_PhysicalSize(psPMR) == 0, "PVRSRV_ERROR_BAD_MAPPING can not map PMR of 0 physical size", PVRSRV_ERROR_BAD_MAPPING);
+
+	if (psPMR->psFuncTab->pfnMMap)
+	{
+		return psPMR->psFuncTab->pfnMMap(psPMR->pvFlavourData, psPMR, pOSMMapData);
+	}
+
+	return OSMMapPMRGeneric(psPMR, pOSMMapData);
+}
+
+PVRSRV_ERROR
+PMRRefPMR(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _Ref(psPMR, 1);
+}
+
+PVRSRV_ERROR
+PMRUnrefPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+
+PVRSRV_ERROR
+PMRRefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _Ref(psPMR, uiRefCount);
+}
+
+PVRSRV_ERROR
+PMRUnrefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _UnrefAndMaybeDestroy(psPMR, uiRefCount);
+}
+
+#define PMR_MAPCOUNT_MIN 0
+#define PMR_MAPCOUNT_MAX IMG_INT32_MAX
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+PVRSRV_ERROR PMRTryRefPMR(PMR *psPMR)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+
+	if (psPMR->uiRefCount == 0 || psPMR->uiRefCount == IMG_UINT32_MAX)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	psPMR->uiRefCount++;
+
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+void
+PMRKernelCpuMapCountIncr(PMR *psPMR)
+{
+	if (OSAtomicAddUnless(&psPMR->iKernelCpuMapCount, 1, PMR_MAPCOUNT_MAX) == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iKernelCpuMapCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+void
+PMRKernelCpuMapCountDecr(PMR *psPMR)
+{
+	if (OSAtomicSubtractUnless(&psPMR->iKernelCpuMapCount, 1, PMR_MAPCOUNT_MIN) == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iKernelCpuMapCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iKernelCpuMapCount),
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+IMG_BOOL
+PMR_IsKernelCpuMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return (OSAtomicRead(&psPMR->iKernelCpuMapCount) > 0);
+}
+
+#endif /* #if defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+void
+PMRClientCpuMapCountIncr(PMR *psPMR)
+{
+	if (OSAtomicAddUnless(&psPMR->iClientCpuMapCount, 1, PMR_MAPCOUNT_MAX) == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iClientCpuMapCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+void
+PMRClientCpuMapCountDecr(PMR *psPMR)
+{
+	if (OSAtomicSubtractUnless(&psPMR->iClientCpuMapCount, 1, PMR_MAPCOUNT_MIN) == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iClientCpuMapCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iClientCpuMapCount),
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+IMG_BOOL
+PMR_IsClientCpuMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return (OSAtomicRead(&psPMR->iClientCpuMapCount) > 0);
+}
+
+static void
+PMRGpuResCountIncr(PMR *psPMR)
+{
+	if (psPMR->iAssociatedResCount == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iAssociatedResCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+		return;
+	}
+
+	psPMR->iAssociatedResCount++;
+}
+
+static void
+PMRGpuResCountDecr(PMR *psPMR)
+{
+	if (psPMR->iAssociatedResCount == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iAssociatedResCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        psPMR->iAssociatedResCount,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+		return;
+	}
+
+	psPMR->iAssociatedResCount--;
+}
+
+IMG_BOOL
+PMR_IsGpuMultiMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->iAssociatedResCount > 1;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRLinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode)
+#else
+void
+PMRLinkGPUMapping(PMR *psPMR)
+#endif
+{
+	PMRLockHeldAssert(psPMR);
+
+	PMRGpuResCountIncr(psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	dllist_add_to_head(&psPMR->sGpuMappingListHead, psMappingNode);
+#endif
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRUnlinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode)
+#else
+void
+PMRUnlinkGPUMapping(PMR *psPMR)
+#endif
+
+{
+	PMRLockHeldAssert(psPMR);
+
+	PMRGpuResCountDecr(psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	dllist_remove_node(psMappingNode);
+#endif
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRNotifyMigrateInProgress(PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psPMR->eState = PMR_STATE_PAGES_IN_MIGRATE;
+}
+
+void
+PMRNotifyMigrateComplete(PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psPMR->eState = PMR_STATE_ACTIVE;
+}
+
+PVRSRV_ERROR
+PMRRemapGPUPMR(PMR *psPMR, IMG_UINT32 ui32LogicalPgOffset)
+{
+	if (psPMR->eState != PMR_STATE_PAGES_IN_MIGRATE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Remap requested on PMR not in migrate state.", __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	return DevmemIntRemapPageInPMR(psPMR, &psPMR->sGpuMappingListHead, ui32LogicalPgOffset);
+}
+#endif
+
+PVRSRV_DEVICE_NODE *
+PMR_DeviceNode(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return PhysHeapDeviceNode(psPMR->psPhysHeap);
+}
+
+PMR_FLAGS_T
+PMR_Flags(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiFlags;
+}
+
+IMG_BOOL
+PMR_IsSparse(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return _PMRIsSparse(psPMR);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+IMG_BOOL
+PMR_IsZombie(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return _IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Function that alters the mutability property
+ * of the PMR
+ * Setting it to TRUE makes sure the PMR memory layout
+ * can't be changed through future calls */
+void
+PMR_SetLayoutFixed(PMR *psPMR, IMG_BOOL bFlag)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	if (bFlag)
+	{
+		_IntFlagSet(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+	}
+	else
+	{
+		_IntFlagClr(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+	}
+}
+
+IMG_BOOL PMR_IsMemLayoutFixed(PMR *psPMR)
+{
+
+	PVR_ASSERT(psPMR != NULL);
+
+	return _IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+}
+
+IMG_DEVMEM_SIZE_T
+PMR_PhysicalSize(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	/* iLockCount will be > 0 for any backed PMR (backed on demand or not) */
+	if (OSAtomicRead(&psPMR->iLockCount) > 0)
+	{
+		if (_PMRIsSparse(psPMR))
+		{
+			return psPMR->psMappingTable->uiChunkSize * psPMR->psMappingTable->ui32NumPhysChunks;
+		}
+		else
+		{
+			return psPMR->uiLogicalSize;
+		}
+	}
+
+	return 0;
+}
+
+PVRSRV_ERROR
+PMR_IsOffsetValid(const PMR *psPMR,
+                  IMG_UINT32 ui32Log2PageSize,
+                  IMG_UINT32 ui32NumOfPages,
+                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                  IMG_BOOL *pbValid)
+{
+	IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 aui32BytesRemain[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset;
+	IMG_UINT32 *pui32BytesRemain = aui32BytesRemain;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psPMR != NULL);
+	PVR_ASSERT(psPMR->uiLogicalSize >= uiLogicalOffset);
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T));
+		PVR_GOTO_IF_NOMEM(puiPhysicalOffset, eError, e0);
+
+		pui32BytesRemain = OSAllocMem(ui32NumOfPages * sizeof(IMG_UINT32));
+		PVR_GOTO_IF_NOMEM(pui32BytesRemain, eError, e0);
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           ui32Log2PageSize,
+	                                           ui32NumOfPages,
+	                                           uiLogicalOffset,
+	                                           puiPhysicalOffset,
+	                                           pui32BytesRemain,
+	                                           pbValid);
+	PVR_LOG_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+e0:
+	if (puiPhysicalOffset != auiPhysicalOffset && puiPhysicalOffset != NULL)
+	{
+		OSFreeMem(puiPhysicalOffset);
+	}
+
+	if (pui32BytesRemain != aui32BytesRemain && pui32BytesRemain != NULL)
+	{
+		OSFreeMem(pui32BytesRemain);
+	}
+
+	return eError;
+}
+
+PHYS_HEAP *
+PMR_PhysHeap(const PMR *psPMR)
+{
+	return psPMR->psPhysHeap;
+}
+
+PMR_MAPPING_TABLE *
+PMR_GetMappingTable(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->psMappingTable;
+
+}
+
+IMG_UINT32
+PMR_GetLog2Contiguity(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->uiLog2ContiguityGuarantee;
+}
+
+IMG_DEVMEM_SIZE_T
+PMR_LogicalSize(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiLogicalSize;
+}
+
+IMG_UINT32 PMR_GetLogicalChunkCount(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->psMappingTable->ui32NumLogicalChunks;
+}
+
+const IMG_CHAR *
+PMR_GetAnnotation(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->szAnnotation;
+}
+
+PMR_IMPL_TYPE
+PMR_GetType(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->eFlavour;
+}
+
+IMG_CHAR *
+PMR_GetTypeStr(const PMR *psPMR)
+{
+	static IMG_CHAR *pszFlavour[] = {
+#define X(type) #type
+		PMR_IMPL_TYPES
+#undef X
+	};
+
+	if (psPMR->eFlavour >= PMR_TYPE_LAST)
+	{
+		return "INVALID";
+	}
+
+	return pszFlavour[psPMR->eFlavour];
+}
+
+IMG_INT32
+PMR_GetRefCount(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _GetRef(psPMR);
+}
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRGetIPAPolicy(PMR *psPMR,
+                IMG_UINT8 *pui8IPAPolicy)
+{
+	IMG_UINT32 ui32FlagsIPAPolicy;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui8IPAPolicy != NULL, "pui8IPAPolicy");
+	ui32FlagsIPAPolicy = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+
+	*pui8IPAPolicy = (IMG_UINT8)ui32FlagsIPAPolicy;
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING) */
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRGetIPAInfo(PMR *psPMR, IMG_UINT32 *pui32IPAPolicy, IMG_UINT32 *pui32IPAShift,
+              IMG_UINT32 *pui32IPAMask, IMG_UINT32 *pui32IPAFlagsValue)
+{
+	IMG_UINT32 ui32IPAFlagsPolicyValue;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAPolicy != NULL, "pui32IPAPolicy");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAShift != NULL, "pui32IPAShift");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAMask != NULL, "pui32IPAMask");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAFlagsValue != NULL, "pui32IPAFlagsValue");
+
+	/* Get the underlying heap-provided default IPA settings (if any) */
+	*pui32IPAShift = PhysHeapGetIPAShift(psPMR->psPhysHeap);
+	*pui32IPAPolicy = PhysHeapGetIPAValue(psPMR->psPhysHeap);
+	*pui32IPAMask = PhysHeapGetIPAMask(psPMR->psPhysHeap);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: Shift, Policy, Mask for Heap %p = %d, %d, %d", __func__,
+	        psPMR->psPhysHeap, *pui32IPAShift, *pui32IPAPolicy, *pui32IPAMask));
+
+	/* Query the current PMR flags settings for current IPA policy */
+	ui32IPAFlagsPolicyValue = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+	*pui32IPAFlagsValue = ui32IPAFlagsPolicyValue;
+	return PVRSRV_OK;
+}
+#endif	/* PVRSRV_INTERNAL_IPA_FEATURE_TESTING */
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRModifyIPAPolicy(PMR *psPMR,
+                   IMG_UINT8 ui8NewIPAPolicy)
+{
+	IMG_UINT64 ui64IPAFlagsPolicyValue;
+	IMG_UINT32 ui32IPAPolicyMaskValue;
+
+	ui32IPAPolicyMaskValue = (IMG_UINT32)(PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK >> PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET);
+
+	/* ui8NewIPAPolicy must be between 0 .. uiMaskValue inclusive */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui8NewIPAPolicy <= ui32IPAPolicyMaskValue,
+	                                "ui8NewIPAPolicy");
+
+	/* Set the appropriate policy bits in the PMR */
+	ui64IPAFlagsPolicyValue = ((IMG_UINT64)(ui8NewIPAPolicy) << PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET) &
+	                           PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+
+	psPMR->uiFlags &= ~PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+	psPMR->uiFlags |= ui64IPAFlagsPolicyValue;
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: ui32IPAPolicy Mask = 0x%x, Value = 0x%x",
+	         __func__, ui32IPAPolicyMaskValue, (IMG_UINT32)ui8NewIPAPolicy));
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: uiFlags = 0x%016" IMG_UINT64_FMTSPECx ", ui64IPAFlags = 0x%016" IMG_UINT64_FMTSPECx, __func__, psPMR->uiFlags, ui64IPAFlagsPolicyValue));
+
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING) */
+
+/* must have called PMRLockSysPhysAddresses() before calling this! */
+PVRSRV_ERROR
+PMR_DevPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEV_PHYADDR *psDevAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage)
+{
+	IMG_UINT32 ui32Remain;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset;
+#if defined(SUPPORT_STATIC_IPA)
+	IMG_UINT32 ui32FlagsIPAPolicy;      /* Local value for the IPA policy */
+	IMG_UINT32 ui32IPAHeapShift;        /* Phys-heap bit-shift value */
+	IMG_UINT32 ui32IPAHeapPolicyValue;  /* Phys-heap default policy value */
+	IMG_UINT32 ui32IPAHeapClearMask;    /* Phys-heap ClearMask bitmask */
+	IMG_UINT64 ui64IPAPolicy;           /* IPAPolicy value to be applied to physical address(es) */
+	IMG_UINT64 ui64IPAClearMask;        /* IPAClearMask to be applied to physical address(es) */
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+	PVR_ASSERT(ui32NumOfPages > 0);
+	PVR_ASSERT(psPMR->psFuncTab->pfnDevPhysAddr != NULL);
+
+	if (OSAtomicRead(&psPMR->iLockCount) <= 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"Attempt to obtain DevPhysAddr of non-backed PMR"));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T));
+		PVR_RETURN_IF_NOMEM(puiPhysicalOffset);
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           ui32Log2PageSize,
+	                                           ui32NumOfPages,
+	                                           uiLogicalOffset,
+	                                           puiPhysicalOffset,
+	                                           &ui32Remain,
+	                                           pbValid);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset", FreeOffsetArray);
+
+#if defined(SUPPORT_STATIC_IPA)
+	/* Need to determine the values to pass into the pfnDevPhysAddr
+	 * for Intermediate Physical Address settings associated with
+	 * this PMR.
+	 * If the ui32FlagsIPAPolicy value is non-zero, the value will be used in
+	 * preference to the default value specified in the physheap config.
+	 * Whichever value is used the associated physheap configuration bit shift and
+	 * mask values are passed to the pfnDevPhysAddr PMR factory function to modify
+	 * the returned address(es).
+	 */
+
+	PVR_ASSERT(psPMR->psPhysHeap != NULL);
+
+	ui32IPAHeapShift = PhysHeapGetIPAShift(psPMR->psPhysHeap);
+	ui32IPAHeapPolicyValue = PhysHeapGetIPAValue(psPMR->psPhysHeap);
+	ui32IPAHeapClearMask = PhysHeapGetIPAMask(psPMR->psPhysHeap);
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	ui32FlagsIPAPolicy = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+	if (ui32FlagsIPAPolicy == 0U)
+	{
+		ui32FlagsIPAPolicy = ui32IPAHeapPolicyValue;
+	}
+#else
+	ui32FlagsIPAPolicy = ui32IPAHeapPolicyValue;	/* Use heap default values*/
+#endif	/* PVRSRV_INTERNAL_IPA_FEATURE_TESTING */
+	/* To handle the 'disabling' of IPAPolicy setting for some callers we
+	 * check to see if the uiPMRUsage is set to DEVICE_USE.
+	 * If so, we simply use the calculated shifts and policy values determined
+	 * above. If disabled BIT_ISSET(uiPMRUsage, CPU_USE) we pass 0 values to the PMR
+	 * factory which will result in no IPA modification being made to the
+	 * phys_heap physical addresses.
+	 */
+	if (unlikely(BITMASK_HAS(uiPMRUsage, CPU_USE)))
+	{
+		ui32IPAHeapClearMask = 0U;
+		ui32FlagsIPAPolicy = 0U;
+	}
+	ui64IPAPolicy = (IMG_UINT64)ui32FlagsIPAPolicy << ui32IPAHeapShift;
+	ui64IPAClearMask = (IMG_UINT64)ui32IPAHeapClearMask << ui32IPAHeapShift;
+#endif	/* SUPPORT_STATIC_IPA */
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* PMRs marked with migrate ability need to wait for migrate to complete
+	 * whilst in the CPU mapping path. PMR lock must be held in this path to
+	 * synchronise obtaining phys addrs.
+	 */
+	if (BITMASK_HAS(uiPMRUsage, MAPPING_USE) && BITMASK_HAS(uiPMRUsage, CPU_USE))
+	{
+		PMRLockHeldAssert(psPMR);
+
+		/* We must wait and retry until we are allowed to obtain phys addrs, no timeout
+		 * because we need this to complete and migrate is guaranteed to finish eventually
+		 */
+		while (1)
+		{
+			if (psPMR->eState == PMR_STATE_ACTIVE)
+			{
+				break;
+			}
+			/* Allow another thread to take the lock, this should allow migration to continue
+			 * and complete before we take physical addresses for use
+			 */
+			PMRUnlockPMR(psPMR);
+			OSReleaseThreadQuanta();
+			PMRLockPMR(psPMR);
+		}
+
+		eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+												  ui32Log2PageSize,
+												  ui32NumOfPages,
+												  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+												  ui64IPAPolicy,
+												  ui64IPAClearMask,
+#endif
+												  pbValid,
+												  psDevAddrPtr);
+	}
+	/* PMRs marked with migrate ability may need to retry an attempt at Device mappings
+	 * for a migrate to complete whilst in the mapping path. This is because we hold higher
+	 * synchronisation methods in device mapping paths that also need to release for the
+	 * migrate to complete.
+	 */
+	else if (BITMASK_HAS(uiPMRUsage, MAPPING_USE) && BITMASK_HAS(uiPMRUsage, DEVICE_USE))
+	{
+		PMRLockHeldAssert(psPMR);
+
+		/* If PMR is in migrate state then ask the requester to retry */
+		if (psPMR->eState != PMR_STATE_ACTIVE)
+		{
+			eError = PVRSRV_ERROR_RETRY;
+			goto FreeOffsetArray;
+		}
+
+		eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+												  ui32Log2PageSize,
+												  ui32NumOfPages,
+												  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+												  ui64IPAPolicy,
+												  ui64IPAClearMask,
+#endif
+												  pbValid,
+												  psDevAddrPtr);
+	}
+	else
+	/* All other paths that are not used in GPU mapping will never return
+	 * PVRSRV_ERROR_RETRY.
+	 */
+#endif
+	{
+		/* Sparse PMR may not always have the first page valid */
+		eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+												  ui32Log2PageSize,
+												  ui32NumOfPages,
+												  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+												  ui64IPAPolicy,
+												  ui64IPAClearMask,
+#endif
+												  pbValid,
+												  psDevAddrPtr);
+		PVR_GOTO_IF_ERROR(eError, FreeOffsetArray);
+	}
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+	/* Currently excluded from the default build because of performance
+	 * concerns.
+	 * We do not need this part in all systems because the GPU has the same
+	 * address view of system RAM as the CPU.
+	 * Alternatively this could be implemented as part of the PMR-factories
+	 * directly */
+	if (PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_UMA ||
+		PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_DMA)
+	{
+		IMG_UINT32 i;
+		IMG_DEV_PHYADDR sDevPAddrCorrected;
+
+		/* Copy the translated addresses to the correct array */
+		for (i = 0; i < ui32NumOfPages; i++)
+		{
+			PhysHeapCpuPAddrToDevPAddr(psPMR->psPhysHeap,
+									   1,
+									   &sDevPAddrCorrected,
+									   (IMG_CPU_PHYADDR *) &psDevAddrPtr[i]);
+			psDevAddrPtr[i].uiAddr = sDevPAddrCorrected.uiAddr;
+		}
+	}
+#endif
+
+
+FreeOffsetArray:
+	if (puiPhysicalOffset != auiPhysicalOffset)
+	{
+		OSFreeMem(puiPhysicalOffset);
+	}
+
+	return eError;
+}
+
+/* must have called PMRLockSysPhysAddresses() before calling this! */
+PVRSRV_ERROR
+PMR_CpuPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_CPU_PHYADDR *psCpuAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage)
+{
+	IMG_UINT32 idx;
+	PVRSRV_ERROR eError;
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *psDevPAddr = asDevPAddr;
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+		PVR_GOTO_IF_NOMEM(psDevPAddr, eError, e0);
+	}
+
+	eError = PMR_DevPhysAddr(psPMR, ui32Log2PageSize, ui32NumOfPages,
+	                         uiLogicalOffset, psDevPAddr, pbValid, uiPMRUsage);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	if (_PMRIsSparse(psPMR))
+	{
+		/* Loop over each page.
+		 * If Dev addr valid, populate the CPU addr from the Dev addr
+		 */
+		for (idx = 0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, 1, &psCpuAddrPtr[idx], &psDevPAddr[idx]);
+			}
+		}
+	}
+	else
+	{
+		/* In this case all addrs will be valid, so we can block translate */
+		PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, ui32NumOfPages, psCpuAddrPtr, psDevPAddr);
+	}
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+
+	return PVRSRV_OK;
+e1:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PMRLockPMR(psPMR);
+
+	eError = PMR_ChangeSparseMemUnlocked(psPMR,
+	                                     ui32AllocPageCount,
+	                                     pai32AllocIndices,
+	                                     ui32FreePageCount,
+	                                     pai32FreeIndices,
+	                                     uiSparseFlags);
+
+	PMRUnlockPMR(psPMR);
+
+	return eError;
+}
+
+PVRSRV_ERROR PMR_ChangeSparseMemUnlocked(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_IMPL_ZOMBIEPAGES pvZombiePages = NULL;
+	PMR_ZOMBIE_PAGES* psPMRZombiePages = NULL;
+#endif
+
+	if (PMR_IsMemLayoutFixed(psPMR) || PMR_IsClientCpuMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This PMR layout cannot be changed - PMR_IsMemLayoutFixed()=%c, _PMR_IsClientCpuMapped()=%c",
+				__func__,
+				PMR_IsMemLayoutFixed(psPMR) ? 'Y' : 'n',
+				PMR_IsClientCpuMapped(psPMR) ? 'Y' : 'n'));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (NULL == psPMR->psFuncTab->pfnChangeSparseMem)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This type of sparse PMR cannot be changed.",
+				__func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		/* Speculatively preallocate in order to simplify error handling later */
+		psPMRZombiePages = OSAllocZMem(sizeof(PMR_ZOMBIE_PAGES));
+		PVR_GOTO_IF_NOMEM(psPMRZombiePages, eError, e0);
+	}
+#endif
+
+	eError = psPMR->psFuncTab->pfnChangeSparseMem(psPMR->pvFlavourData,
+	                                              psPMR,
+	                                              ui32AllocPageCount,
+	                                              pai32AllocIndices,
+	                                              ui32FreePageCount,
+	                                              pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	                                              &pvZombiePages,
+#endif
+	                                              uiSparseFlags);
+	if (eError != PVRSRV_OK)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		if (eError == PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES)
+		{
+			PVRSRVStatsUpdateOOMStat(NULL,
+									  PMR_DeviceNode(psPMR),
+									  PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+									  OSGetCurrentClientProcessIDKM());
+		}
+#endif
+		goto e1;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+		PVRSRV_DEVICE_NODE *psDevNode;
+
+		psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+		eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+		if (eError != PVRSRV_OK)
+		{
+			/* Treat unknown power state as ON. */
+			ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+		}
+
+		if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+			psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+		{
+			/* Free preallocated psPMRZombiePages as these won't be used*/
+			OSFreeMem(psPMRZombiePages);
+
+			eError = psPMR->psFuncTab->pfnFreeZombiePages(pvZombiePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "Error when trying to free zombies immediately.", e0);
+		}
+		else
+		{
+			PVR_ASSERT(psPMRZombiePages != NULL);
+			psPMRZombiePages->sHeader.eZombieType = PMR_ZOMBIE_TYPE_PAGES;
+			psPMRZombiePages->pfnFactoryFreeZombies = psPMR->psFuncTab->pfnFreeZombiePages;
+			psPMRZombiePages->pvFactoryPages = pvZombiePages;
+
+			_ZombieListLock(psDevNode);
+			dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMRZombiePages->sHeader.sZombieNode);
+			psDevNode->uiPMRZombieCount++;
+			_ZombieListUnlock(psDevNode);
+		}
+	}
+	else
+	{
+		/* Free psPMRZombiePages as change sparse has not produced zombie pages */
+		OSFreeMem(psPMRZombiePages);
+	}
+#endif
+
+#if defined(PDUMP)
+	{
+		IMG_BOOL bInitialise = IMG_FALSE;
+		IMG_UINT8 ui8InitValue = 0;
+
+		if (PVRSRV_CHECK_ZERO_ON_ALLOC(PMR_Flags(psPMR)))
+		{
+			bInitialise = IMG_TRUE;
+		}
+		else if (PVRSRV_CHECK_POISON_ON_ALLOC(PMR_Flags(psPMR)))
+		{
+			ui8InitValue = (IMG_UINT8)PVRSRV_POISON_ON_ALLOC_VALUE;
+			bInitialise = IMG_TRUE;
+		}
+
+		PDumpPMRChangeSparsePMR(psPMR,
+		                        IMG_PAGE2BYTES32(psPMR->uiLog2ContiguityGuarantee),
+		                        ui32AllocPageCount,
+		                        pai32AllocIndices,
+		                        ui32FreePageCount,
+		                        pai32FreeIndices,
+		                        bInitialise,
+		                        ui8InitValue,
+		                        &psPMR->hPDumpAllocHandle);
+	}
+
+#endif
+
+	return PVRSRV_OK;
+e1:
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		OSFreeMem(psPMRZombiePages);
+	}
+e0:
+#endif
+	return eError;
+}
+
+#if defined(PDUMP)
+
+static PVRSRV_ERROR
+_PMR_PDumpSymbolicAddrPhysical(const PMR *psPMR,
+                               IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                               IMG_UINT32 ui32MemspaceNameLen,
+                               IMG_CHAR *pszMemspaceName,
+                               IMG_UINT32 ui32SymbolicAddrLen,
+                               IMG_CHAR *pszSymbolicAddr,
+                               IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                               IMG_DEVMEM_OFFSET_T *puiNextSymName)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (PVRSRV_CHECK_PHYS_HEAP(FW_CODE, psPMR->uiFlags) ||
+		PVRSRV_CHECK_PHYS_HEAP(FW_PRIV_DATA, psPMR->uiFlags) ||
+	    PVRSRV_CHECK_PHYS_HEAP(GPU_SECURE, psPMR->uiFlags))
+	{
+		OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+	else
+#endif
+	if (DevmemDeviceCacheCoherency(psDevNode, psPMR->uiFlags))
+	{
+		OSSNPrintf(pszMemspaceName,
+		           ui32MemspaceNameLen,
+		           PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+	else
+	{
+		OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+
+	OSSNPrintf(pszSymbolicAddr,
+	           ui32SymbolicAddrLen,
+	           PMR_SYMBOLICADDR_FMTSPEC,
+	           PMR_IsSparse(psPMR) ? PMR_SPARSE_PREFIX : PMR_DEFAULT_PREFIX,
+	           psPMR->uiSerialNum,
+	           uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR),
+	           psPMR->szAnnotation);
+
+	if (pszSymbolicAddr)
+	{
+		PDumpMakeStringValid(pszSymbolicAddr, OSStringLength(pszSymbolicAddr));
+	}
+
+
+	*puiNewOffset = uiPhysicalOffset & (IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR))-1);
+	*puiNextSymName = (IMG_DEVMEM_OFFSET_T) (((uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR))+1)
+			<< PMR_GetLog2Contiguity(psPMR));
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32MemspaceNameLen,
+                      IMG_CHAR *pszMemspaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                      IMG_DEVMEM_OFFSET_T *puiNextSymName
+)
+{
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	IMG_UINT32 ui32Remain;
+	IMG_BOOL bValid;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiLogicalOffset < psPMR->uiLogicalSize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           0,
+	                                           1,
+	                                           uiLogicalOffset,
+	                                           &uiPhysicalOffset,
+	                                           &ui32Remain,
+	                                           &bValid);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+	if (!bValid)
+	{
+		/* For sparse allocations, for a given logical address, there
+		 * may not be a physical memory backing, the virtual range can
+		 * still be valid.
+		 */
+		uiPhysicalOffset = uiLogicalOffset;
+	}
+
+	return _PMR_PDumpSymbolicAddrPhysical(psPMR,
+	                                      uiPhysicalOffset,
+	                                      ui32MemspaceNameLen,
+	                                      pszMemspaceName,
+	                                      ui32SymbolicAddrLen,
+	                                      pszSymbolicAddr,
+	                                      puiNewOffset,
+	                                      puiNextSymName);
+}
+
+/*!
+ * @brief Writes a WRW command to the script2 buffer, representing a
+ *        dword write to a physical allocation. Size is always
+ *        sizeof(IMG_UINT32).
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - offset
+ * @param ui32Value - value to write
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                       IMG_UINT32 ui32Value,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + sizeof(ui32Value) <= psPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+			<= uiPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the PMR */
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW32(PMR_DeviceNode(psPMR),
+	                       aszMemspaceName,
+	                       aszSymbolicName,
+	                       uiPDumpSymbolicOffset,
+	                       ui32Value,
+	                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a RDW followed by a WRW command to the pdump script to perform
+ *        an effective copy from memory to memory. Memory copied is of size
+ *        sizeof(IMG_UINT32)
+ *
+ * @param psDstPMR - PMR object representing allocation of destination
+ * @param uiDstLogicalOffset - destination offset
+ * @param psSrcPMR - PMR object representing allocation of source
+ * @param uiSrcLogicalOffset - source offset
+ * @param pszTmpVar - pdump temporary variable used during the copy
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem32(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	const IMG_UINT32 uiDstPMRPageSize = 1 << psDstPMR->uiLog2ContiguityGuarantee;
+	const IMG_UINT32 uiSrcPMRPageSize = 1 << psSrcPMR->uiLog2ContiguityGuarantee;
+
+	PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiSrcLogicalOffset & (uiSrcPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= uiSrcPMRPageSize));
+
+	PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiDstLogicalOffset & (uiDstPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= uiDstPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the source PMR */
+	eError = PMR_PDumpSymbolicAddr(psSrcPMR,
+	                               uiSrcLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Issue PDump read command */
+	eError = PDumpPMRRDW32MemToInternalVar(PMR_DeviceNode(psSrcPMR),
+	                                       pszTmpVar,
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+
+	eError = PMRLockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Get the symbolic address of the destination PMR */
+	eError = PMR_PDumpSymbolicAddr(psDstPMR,
+	                               uiDstLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW32InternalVarToMem(PMR_DeviceNode(psDstPMR),
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       pszTmpVar,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	eError = PMRUnlockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a WRW64 command to the script2 buffer, representing a
+ *        dword write to a physical allocation. Size is always
+ *        sizeof(IMG_UINT64).
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - offset
+ * @param ui64Value - value to write
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                       IMG_UINT64 ui64Value,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PMR_SIZE_T ui64PMRPageSize = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + sizeof(ui64Value) <= psPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (ui64PMRPageSize-1)) + sizeof(ui64Value))
+			<= ui64PMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the PMR */
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW64(PMR_DeviceNode(psPMR),
+	                       aszMemspaceName,
+	                       aszSymbolicName,
+	                       uiPDumpSymbolicOffset,
+	                       ui64Value,
+	                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a RDW64 followed by a WRW64 command to the pdump script to
+ *        perform an effective copy from memory to memory. Memory copied is of
+ *        size sizeof(IMG_UINT32)
+ *
+ * @param psDstPMR - PMR object representing allocation of destination
+ * @param uiDstLogicalOffset - destination offset
+ * @param psSrcPMR - PMR object representing allocation of source
+ * @param uiSrcLogicalOffset - source offset
+ * @param pszTmpVar - pdump temporary variable used during the copy
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem64(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	const PMR_SIZE_T ui64DstPMRPageSize = IMG_PAGE2BYTES64(psDstPMR->uiLog2ContiguityGuarantee);
+	const PMR_SIZE_T ui64SrcPMRPageSize = IMG_PAGE2BYTES64(psSrcPMR->uiLog2ContiguityGuarantee);
+
+	PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiSrcLogicalOffset & (ui64SrcPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= ui64SrcPMRPageSize));
+
+	PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiDstLogicalOffset & (ui64DstPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= ui64DstPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the source PMR */
+	eError = PMR_PDumpSymbolicAddr(psSrcPMR,
+	                               uiSrcLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Issue PDump read command */
+	eError = PDumpPMRRDW64MemToInternalVar(PMR_DeviceNode(psSrcPMR),
+	                                       pszTmpVar,
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+
+	eError = PMRLockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Get the symbolic address of the destination PMR */
+	eError = PMR_PDumpSymbolicAddr(psDstPMR,
+	                               uiDstLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW64InternalVarToMem(PMR_DeviceNode(psDstPMR),
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       pszTmpVar,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	eError = PMRUnlockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief PDumps the contents of the given allocation.
+ * If bZero is IMG_TRUE then the zero page in the parameter stream is used
+ * as the source of data, rather than the allocation's actual backing.
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - Offset to write at
+ * @param uiSize - Number of bytes to write
+ * @param uiPDumpFlags - PDump flags
+ * @param bZero - Use the PDump zero page as the source
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOutOffset;
+	IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName = 0;
+	const IMG_CHAR *pszParamStreamFileName;
+	PDUMP_FILEOFFSET_T uiParamStreamFileOffset;
+
+	/* required when !bZero */
+#define PMR_MAX_PDUMP_BUFSZ (1<<21)
+	IMG_CHAR aszParamStreamFilename[PDUMP_PARAM_MAX_FILE_NAME];
+	IMG_UINT8 *pcBuffer = NULL;
+	size_t ui64BufSz;
+	IMG_BOOL bValid;
+	IMG_DEVMEM_SIZE_T uiSizeRemain = uiSize;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(psDevNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Forcibly initialise the name to a 'NULL' 0-length string */
+	aszParamStreamFilename[0] = '\0';
+
+	PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize);
+
+	/* Check if pdump client is connected */
+	if (!PDumpCheckFlagsWrite(psDevNode,
+		                      PDUMP_FLAGS_CONTINUOUS))
+	{
+		/* Dumping of memory in Pdump buffer will be rejected for no client connected case.
+		 * So return early and save reading of data from PMR. */
+		return PVRSRV_OK;
+	}
+
+	/* Get the correct PDump stream file name */
+	if (bZero)
+	{
+		PDumpCommentWithFlags(psDevNode,
+		                      uiPDumpFlags,
+		                      "Zeroing allocation (" IMG_DEVMEM_SIZE_FMTSPEC " bytes)",
+		                      uiSize);
+
+		/* get the zero page information. it is constant for this function */
+		PDumpGetParameterZeroPageInfo(&uiParamStreamFileOffset,
+		                              &ui64BufSz,
+		                              &pszParamStreamFileName);
+	}
+	else
+	{
+
+		ui64BufSz = IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR));
+		PVR_ASSERT(IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR)) <= PMR_MAX_PDUMP_BUFSZ);
+
+		pcBuffer = OSAllocMem(ui64BufSz);
+
+		PVR_LOG_RETURN_IF_NOMEM(pcBuffer, "OSAllocMem");
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		pszParamStreamFileName = aszParamStreamFilename;
+	}
+
+	/* Loop over all touched symbolic addresses of the PMR and
+	 * emit LDBs to load the contents. */
+	while (uiCurrentOffset < (uiLogicalOffset + uiSize))
+	{
+		/* Get the correct symbolic name for the current offset */
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiCurrentOffset,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOutOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		PVR_ASSERT((uiNextSymName - uiCurrentOffset) <= ui64BufSz);
+
+		PMR_IsOffsetValid(psPMR,
+		                  0,
+		                  1,
+		                  uiCurrentOffset,
+		                  &bValid);
+
+		/* Either just LDB the zeros or read from the PMR and store that
+		 * in the pdump stream */
+		if (bValid)
+		{
+			size_t uiNumBytes;
+			IMG_BOOL bOk2Write = IMG_TRUE;
+
+			if (bZero)
+			{
+				uiNumBytes = MIN(uiSizeRemain, uiNextSymName - uiCurrentOffset);
+			}
+			else
+			{
+				IMG_DEVMEM_OFFSET_T uiReadOffset;
+				uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ?
+						uiLogicalOffset + uiSize - uiCurrentOffset :
+						uiNextSymName - uiCurrentOffset);
+
+				eError = PMR_ReadBytes(psPMR,
+				                       uiCurrentOffset,
+				                       pcBuffer,
+				                       uiReadOffset,
+				                       &uiNumBytes);
+				PVR_ASSERT(eError == PVRSRV_OK);
+
+				eError = PDumpWriteParameterBlob(psDevNode,
+				                          pcBuffer,
+				                          uiNumBytes,
+				                          uiPDumpFlags,
+				                          &aszParamStreamFilename[0],
+				                          sizeof(aszParamStreamFilename),
+				                          &uiParamStreamFileOffset);
+
+				if (eError == PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+				{
+					/* Write to parameter file prevented under the flags and
+					 * current state of the driver so skip further writes.
+					 */
+					eError = PVRSRV_OK;
+					bOk2Write = IMG_FALSE;	/* Do *NOT* write anything */
+				}
+				else if (eError != PVRSRV_OK)
+				{
+					PDUMP_ERROR(psDevNode,
+					            eError, "Failed to write PMR memory to parameter file");
+					bOk2Write = IMG_FALSE;	/* Do *NOT* write anything */
+				}
+			}
+
+			if (bOk2Write)
+			{
+				/* Emit the LDB command to the current symbolic address */
+				eError = PDumpPMRLDB(psDevNode,
+				                     aszMemspaceName,
+				                     aszSymbolicName,
+				                     uiOutOffset,
+				                     uiNumBytes,
+				                     pszParamStreamFileName,
+				                     uiParamStreamFileOffset,
+				                     uiPDumpFlags);
+			}
+			uiSizeRemain = uiSizeRemain - uiNumBytes;
+		}
+		uiCurrentOffset = uiNextSymName;
+	}
+
+	if (!bZero)
+	{
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		OSFreeMem(pcBuffer);
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOutOffset;
+	IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName = 0;
+	IMG_UINT32 uiCurrentFileOffset = uiFileOffset;
+
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize);
+
+	while (uiCurrentOffset < (uiLogicalOffset + uiSize))
+	{
+		IMG_DEVMEM_OFFSET_T uiReadOffset;
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiCurrentOffset,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOutOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		PVR_ASSERT(uiNextSymName <= psPMR->uiLogicalSize);
+
+		uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ?
+				uiLogicalOffset + uiSize - uiCurrentOffset :
+				uiNextSymName - uiCurrentOffset);
+
+		eError = PDumpPMRSAB(PMR_DeviceNode(psPMR),
+		                     aszMemspaceName,
+		                     aszSymbolicName,
+		                     uiOutOffset,
+		                     uiReadOffset,
+		                     pszFilename,
+		                     uiCurrentFileOffset);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		uiCurrentFileOffset += uiNextSymName - uiCurrentOffset;
+		uiCurrentOffset = uiNextSymName;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+			<= uiPMRPageSize));
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+#define _MEMPOLL_DELAY		(1000)
+#define _MEMPOLL_COUNT		(2000000000 / _MEMPOLL_DELAY)
+
+	eError = PDumpPMRPOL(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     ui32Value,
+	                     ui32Mask,
+	                     eOperator,
+	                     _MEMPOLL_COUNT,
+	                     _MEMPOLL_DELAY,
+	                     uiPDumpFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+	           < uiPMRPageSize));
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = PDumpPMRPOL(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     ui32Value,
+	                     ui32Mask,
+	                     eOperator,
+	                     1,
+	                     1,
+	                     uiPDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiReadOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	eError = PDumpPMRCBP(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     uiWriteOffset,
+	                     uiPacketSize,
+	                     uiBufferSize);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void
+PDumpPMRChangeSparsePMR(PMR *psPMR,
+                        IMG_UINT32 uiBlockSize,
+                        IMG_UINT32 ui32AllocPageCount,
+                        IMG_UINT32 *pai32AllocIndices,
+                        IMG_UINT32 ui32FreePageCount,
+                        IMG_UINT32 *pai32FreeIndices,
+                        IMG_BOOL bInitialise,
+                        IMG_UINT8 ui8InitValue,
+                        IMG_HANDLE *phPDumpAllocInfoOut)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE *phPDumpAllocInfo = (IMG_HANDLE*) psPMR->hPDumpAllocHandle;
+
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 i, uiIndex;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	/* Remove pages from the PMR */
+	for (i = 0; i < ui32FreePageCount; i++)
+	{
+		uiIndex = pai32FreeIndices[i];
+
+		eError = PDumpFree(psDevNode,
+		                   phPDumpAllocInfo[uiIndex]);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		phPDumpAllocInfo[uiIndex] = NULL;
+	}
+
+	/* Add new pages to the PMR */
+	for (i = 0; i < ui32AllocPageCount; i++)
+	{
+		uiIndex = pai32AllocIndices[i];
+
+		PVR_ASSERT(phPDumpAllocInfo[uiIndex] == NULL);
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiIndex * uiBlockSize,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpMalloc(psDevNode,
+		                     aszMemspaceName,
+		                     aszSymbolicName,
+		                     uiBlockSize,
+		                     uiBlockSize,
+		                     bInitialise,
+		                     ui8InitValue,
+		                     &phPDumpAllocInfo[uiIndex],
+		                     PDUMP_NONE);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* (IMG_HANDLE) <- (IMG_HANDLE*) */
+	*phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo;
+}
+
+static void
+PDumpPMRFreePMR(PMR *psPMR,
+                IMG_DEVMEM_SIZE_T uiSize,
+                IMG_DEVMEM_ALIGN_T uiBlockSize,
+                IMG_UINT32 uiLog2Contiguity,
+                IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	/* (IMG_HANDLE*) <- (IMG_HANDLE) */
+	IMG_HANDLE *ahPDumpAllocHandleArray = (IMG_HANDLE*) hPDumpAllocationInfoHandle;
+
+	PDUMP_LOCK(PDUMP_FLAGS_NONE);
+
+	for (i = 0; i < psPMR->uiNumPDumpBlocks; i++)
+	{
+		if (ahPDumpAllocHandleArray[i] != NULL)
+		{
+			eError = PDumpFreeUnlocked(PMR_DeviceNode(psPMR),
+			                           ahPDumpAllocHandleArray[i]);
+			PVR_ASSERT(eError == PVRSRV_OK);
+			ahPDumpAllocHandleArray[i] = NULL;
+		}
+	}
+
+	PDUMP_UNLOCK(PDUMP_FLAGS_NONE);
+	OSFreeMem(ahPDumpAllocHandleArray);
+}
+
+static void
+PDumpPMRMallocPMR(PMR *psPMR,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT32 ui32ChunkSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 ui32NumLogicalChunks,
+                  IMG_UINT32 *puiMappingTable,
+                  IMG_UINT32 uiLog2Contiguity,
+                  IMG_BOOL bInitialise,
+                  IMG_UINT8 ui8InitValue,
+                  IMG_HANDLE *phPDumpAllocInfoOut,
+                  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE *phPDumpAllocInfo;
+
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiNumPhysBlocks;
+	IMG_UINT32 uiNumVirtBlocks;
+	IMG_UINT32 i, uiIndex;
+
+	if (PMR_IsSparse(psPMR))
+	{
+		uiNumPhysBlocks = (ui32ChunkSize * ui32NumPhysChunks) >> uiLog2Contiguity;
+		/* Make sure we did not cut off anything */
+		PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == (ui32ChunkSize * ui32NumPhysChunks));
+	}
+	else
+	{
+		uiNumPhysBlocks = uiSize >> uiLog2Contiguity;
+		/* Make sure we did not cut off anything */
+		PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == uiSize);
+	}
+
+	uiNumVirtBlocks = uiSize >> uiLog2Contiguity;
+	PVR_ASSERT(uiNumVirtBlocks << uiLog2Contiguity == uiSize);
+
+	psPMR->uiNumPDumpBlocks = uiNumVirtBlocks;
+
+	phPDumpAllocInfo = (IMG_HANDLE*) OSAllocZMem(uiNumVirtBlocks * sizeof(IMG_HANDLE));
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	for (i = 0; i < uiNumPhysBlocks; i++)
+	{
+		uiIndex = PMR_IsSparse(psPMR) ? puiMappingTable[i] : i;
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiIndex * ui32ChunkSize,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpMallocUnlocked(PMR_DeviceNode(psPMR),
+		                             aszMemspaceName,
+		                             aszSymbolicName,
+		                             ui32ChunkSize,
+		                             ui32ChunkSize,
+		                             bInitialise,
+		                             ui8InitValue,
+		                             &phPDumpAllocInfo[uiIndex],
+		                             ui32PDumpFlags);
+		PVR_LOG_RETURN_VOID_IF_FALSE((eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE),
+		                             "PDumpPMRMalloc PDump capture bound to other device");
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	/* (IMG_HANDLE) <- (IMG_HANDLE*) */
+	*phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo;
+
+}
+#endif	/* PDUMP */
+
+
+void *PMRGetPrivateData(const PMR *psPMR,
+                        const PMR_IMPL_FUNCTAB *psFuncTab)
+{
+	return (psFuncTab == psPMR->psFuncTab) ? psPMR->pvFlavourData : NULL;
+}
+
+#define PMR_PM_WORD_SIZE 4
+
+PVRSRV_ERROR
+PMRWritePMPageList(/* Target PMR, offset, and length */
+		PMR *psPageListPMR,
+		IMG_DEVMEM_OFFSET_T uiTableOffset,
+		IMG_DEVMEM_SIZE_T  uiTableLength,
+		/* Referenced PMR, and "page" granularity */
+		PMR *psReferencePMR,
+		IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+		PMR_PAGELIST **ppsPageList)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiWordSize;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	PMR_FLAGS_T uiFlags = psPageListPMR->uiFlags;
+	PMR_PAGELIST *psPageList;
+#if defined(PDUMP)
+	IMG_CHAR aszTableEntryMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszTableEntrySymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiTableEntryPDumpOffset;
+	IMG_CHAR aszPageMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszPageSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPagePDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+#endif
+#if !defined(NO_HARDWARE)
+	PMR_SIZE_T ui64PageListPageSize = IMG_PAGE2BYTES64(psPageListPMR->uiLog2ContiguityGuarantee);
+	IMG_UINT64 uiPageListPMRPage = 0;
+	IMG_UINT64 uiPrevPageListPMRPage = 0;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+	IMG_UINT32 *pui32DataPtr = NULL;
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *pasDevAddrPtr;
+	IMG_BOOL *pbPageIsValid;
+#endif
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPageListPMR->psPhysHeap);
+	IMG_BOOL bCPUCacheSnoop =
+		(PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		 psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY);
+
+	uiWordSize = PMR_PM_WORD_SIZE;
+
+	/* check we're being asked to write the same number of 4-byte units as there are pages */
+	uiNumPages = (IMG_UINT32)(psReferencePMR->uiLogicalSize >> uiLog2PageSize);
+
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psReferencePMR->uiLogicalSize)
+	{
+		/* Strictly speaking, it's possible to provoke this error in two ways:
+			(i) if it's not a whole multiple of the page size; or
+			(ii) if there are more than 4 billion pages.
+			The latter is unlikely. :) but the check is required in order to justify the cast.
+		 */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, return_error);
+	}
+	uiWordSize = (IMG_UINT32)uiTableLength / uiNumPages;
+	if (uiNumPages * uiWordSize != uiTableLength)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, return_error);
+	}
+
+	/* Check for integer overflow */
+	PVR_GOTO_IF_INVALID_PARAM(uiTableOffset + uiTableLength > uiTableOffset, eError, return_error);
+	/* Check we're not being asked to write off the end of the PMR */
+	PVR_GOTO_IF_INVALID_PARAM(uiTableOffset + uiTableLength <= psPageListPMR->uiLogicalSize, eError, return_error);
+
+	/* the PMR into which we are writing must not be user CPU mappable: */
+	if (PVRSRV_CHECK_CPU_READABLE(uiFlags) || PVRSRV_CHECK_CPU_WRITEABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         (PMR_FLAGS_T)(uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE | PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE))));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Page list PMR allows CPU mapping (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* the PMR into which we are writing must not be user CPU cacheable: */
+	if (!bCPUCacheSnoop &&
+		(PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags) ||
+		 PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+		 PVRSRV_CHECK_CPU_CACHED(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         (PMR_FLAGS_T)(uiFlags &  PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Page list PMR allows CPU caching (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	if (_PMRIsSparse(psPageListPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPageListPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	if (_PMRIsSparse(psReferencePMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psReferencePMR", eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	psPageList = OSAllocMem(sizeof(PMR_PAGELIST));
+	PVR_LOG_GOTO_IF_NOMEM(psPageList, eError, return_error);
+
+	psPageList->psReferencePMR = psReferencePMR;
+
+	/* Need to lock down the physical addresses of the reference PMR */
+	/* N.B.  This also checks that the requested "contiguity" is achievable */
+	eError = PMRLockSysPhysAddresses(psReferencePMR);
+	PVR_GOTO_IF_ERROR(eError, free_page_list);
+
+#if !defined(NO_HARDWARE)
+	if (uiNumPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		pasDevAddrPtr = OSAllocMem(uiNumPages * sizeof(IMG_DEV_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(pasDevAddrPtr, eError, unlock_phys_addrs);
+
+		pbPageIsValid = OSAllocMem(uiNumPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbPageIsValid, eError, free_devaddr_array);
+	}
+	else
+	{
+		pasDevAddrPtr = asDevPAddr;
+		pbPageIsValid = abValid;
+	}
+
+	eError = PMR_DevPhysAddr(psReferencePMR, uiLog2PageSize, uiNumPages, 0,
+	                         pasDevAddrPtr, pbPageIsValid, DEVICE_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", free_valid_array);
+#endif
+
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		IMG_DEVMEM_OFFSET_T uiPMROffset = uiTableOffset + (uiWordSize * uiPageIndex);
+
+#if defined(PDUMP)
+		eError = PMR_PDumpSymbolicAddr(psPageListPMR,
+		                               uiPMROffset,
+		                               sizeof(aszTableEntryMemspaceName),
+		                               &aszTableEntryMemspaceName[0],
+		                               sizeof(aszTableEntrySymbolicName),
+		                               &aszTableEntrySymbolicName[0],
+		                               &uiTableEntryPDumpOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PMR_PDumpSymbolicAddr(psReferencePMR,
+		                               IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                               sizeof(aszPageMemspaceName),
+		                               &aszPageMemspaceName[0],
+		                               sizeof(aszPageSymbolicName),
+		                               &aszPageSymbolicName[0],
+		                               &uiPagePDumpOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpWriteShiftedMaskedValue(PMR_DeviceNode(psReferencePMR),
+		                                      /* destination */
+		                                      aszTableEntryMemspaceName,
+		                                      aszTableEntrySymbolicName,
+		                                      uiTableEntryPDumpOffset,
+		                                      /* source */
+		                                      aszPageMemspaceName,
+		                                      aszPageSymbolicName,
+		                                      uiPagePDumpOffset,
+		                                      /* shift right */
+		                                      uiLog2PageSize,
+		                                      /* shift left */
+		                                      0,
+		                                      /* mask */
+		                                      0xffffffff,
+		                                      /* word size */
+		                                      uiWordSize,
+		                                      /* flags */
+		                                      PDUMP_FLAGS_CONTINUOUS);
+		PVR_ASSERT(eError == PVRSRV_OK);
+#else
+		PVR_UNREFERENCED_PARAMETER(uiPMROffset);
+#endif
+
+#if !defined(NO_HARDWARE)
+
+		/*
+			We check for sparse PMR's at function entry, but as we can,
+			check that every page is valid
+		 */
+		PVR_ASSERT(pbPageIsValid[uiPageIndex]);
+		PVR_ASSERT(pasDevAddrPtr[uiPageIndex].uiAddr != 0);
+		PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000LL) == 0);
+
+		uiPageListPMRPage = uiPMROffset >> psReferencePMR->uiLog2ContiguityGuarantee;
+
+		if ((pui32DataPtr == NULL) || (uiPageListPMRPage != uiPrevPageListPMRPage))
+		{
+			size_t uiMappingOffset = uiPMROffset & (~(ui64PageListPageSize - 1));
+			size_t uiMappedSize;
+
+			/* If we already had a page list mapped, we need to unmap it... */
+			if (pui32DataPtr != NULL)
+			{
+				PMRReleaseKernelMappingData(psPageListPMR, hPrivData);
+			}
+
+			eError = PMRAcquireKernelMappingData(psPageListPMR,
+			                                     uiMappingOffset,
+			                                     ui64PageListPageSize,
+			                                     &pvKernAddr,
+			                                     &uiMappedSize,
+			                                     &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Error mapping page list PMR page (%" IMG_UINT64_FMTSPEC ") into kernel (%d)",
+						uiPageListPMRPage, eError));
+				goto free_valid_array;
+			}
+
+			uiPrevPageListPMRPage = uiPageListPMRPage;
+			PVR_ASSERT(uiMappedSize >= ui64PageListPageSize);
+			PVR_ASSERT(pvKernAddr != NULL);
+
+			pui32DataPtr = IMG_OFFSET_ADDR(pvKernAddr, (uiPMROffset & (ui64PageListPageSize - 1)));
+		}
+
+		PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000LL) == 0);
+
+		/* Write the physical page index into the page list PMR */
+		*pui32DataPtr++ = TRUNCATE_64BITS_TO_32BITS(pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize);
+
+		/* Last page so unmap */
+		if (uiPageIndex == (uiNumPages - 1))
+		{
+			PMRReleaseKernelMappingData(psPageListPMR, hPrivData);
+		}
+#endif
+	}
+
+	OSWriteMemoryBarrier(NULL);
+
+#if !defined(NO_HARDWARE)
+	if (pasDevAddrPtr != asDevPAddr)
+	{
+		OSFreeMem(pbPageIsValid);
+		OSFreeMem(pasDevAddrPtr);
+	}
+#endif
+	*ppsPageList = psPageList;
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+#if !defined(NO_HARDWARE)
+
+free_valid_array:
+	if (pbPageIsValid != abValid)
+	{
+		OSFreeMem(pbPageIsValid);
+	}
+
+free_devaddr_array:
+	if (pasDevAddrPtr != asDevPAddr)
+	{
+		OSFreeMem(pasDevAddrPtr);
+	}
+
+unlock_phys_addrs:
+	PMRUnlockSysPhysAddresses(psReferencePMR);
+#endif
+
+free_page_list:
+	OSFreeMem(psPageList);
+
+return_error:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMRUnwritePMPageList(PMR_PAGELIST *psPageList)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PMRUnlockSysPhysAddresses(psPageList->psReferencePMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+	OSFreeMem(psPageList);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRZeroingPMR(PMR *psPMR,
+              IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize)
+{
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	PMR_SIZE_T ui64PageSize = IMG_PAGE2BYTES64(uiLog2PageSize);
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	size_t uiMappedSize;
+
+	PVR_ASSERT(psPMR);
+
+	/* Calculate number of pages in this PMR */
+	uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize);
+
+	/* Verify the logical Size is a multiple or the physical page size */
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR is not a multiple of %" IMG_UINT64_FMTSPEC,
+		         __func__,
+		         ui64PageSize));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, MultiPage_Error);
+	}
+
+	if (_PMRIsSparse(psPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, Sparse_Error);
+	}
+
+	/* Scan through all pages of the PMR */
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		/* map the physical page (for a given PMR offset) into kernel space */
+		eError = PMRAcquireKernelMappingData(psPMR,
+		                                     IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                                     ui64PageSize,
+		                                     &pvKernAddr,
+		                                     &uiMappedSize,
+		                                     &hPrivData);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", AcquireKernelMapping_Error);
+
+		/* ensure the mapped page size is the same as the physical page size */
+		if (uiMappedSize != ui64PageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Physical Page size = 0x%08" IMG_UINT64_FMTSPECx ", Size of Mapping = 0x%016" IMG_UINT64_FMTSPECx,
+			         __func__,
+			         ui64PageSize,
+			         (IMG_UINT64)uiMappedSize));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, MappingSize_Error);
+		}
+
+		/* Use the conservative 'DeviceMemSet' here because we can't know
+		 * if this PMR will be mapped cached.
+		 */
+		OSDeviceMemSet(pvKernAddr, 0, ui64PageSize);
+
+		/* release mapping */
+		PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: Zeroing PMR %p done (num pages %u, page size %" IMG_UINT64_FMTSPEC ")",
+	         __func__,
+	         psPMR,
+	         uiNumPages,
+	         ui64PageSize));
+
+	return PVRSRV_OK;
+
+
+	/* Error handling */
+
+MappingSize_Error:
+	PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+AcquireKernelMapping_Error:
+Sparse_Error:
+MultiPage_Error:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRDumpPageList(PMR *psPMR,
+                IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize)
+{
+	IMG_DEV_PHYADDR sDevAddrPtr;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	IMG_BOOL bPageIsValid;
+	IMG_UINT32 ui32Col = 16;
+	IMG_UINT32 ui32SizePerCol = 11;
+	IMG_UINT32 ui32ByteCount = 0;
+	IMG_CHAR pszBuffer[16 /* ui32Col */ * 11 /* ui32SizePerCol */ + 1];
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Get number of pages */
+	uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize);
+
+	/* Verify the logical Size is a multiple or the physical page size */
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not a multiple of %" IMG_UINT64_FMTSPEC,
+		        __func__, (IMG_UINT64) (1ULL << uiLog2PageSize)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, MultiPage_Error);
+	}
+
+	if (_PMRIsSparse(psPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, Sparse_Error);
+	}
+
+	PVR_LOG(("    PMR %p, Number of pages %u, Log2PageSize %d", psPMR, uiNumPages, uiLog2PageSize));
+
+	/* Print the address of the physical pages */
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		/* Get Device physical Address */
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2PageSize,
+		                         1,
+		                         IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                         &sDevAddrPtr,
+		                         &bPageIsValid,
+		                         DEVICE_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PMR %p failed to get DevPhysAddr with error %u",
+					__func__,
+					psPMR,
+					eError));
+			goto DevPhysAddr_Error;
+		}
+
+		ui32ByteCount += OSSNPrintf(pszBuffer + ui32ByteCount, ui32SizePerCol + 1, "%08x ", (IMG_UINT32)(sDevAddrPtr.uiAddr >> uiLog2PageSize));
+		PVR_ASSERT(ui32ByteCount < ui32Col * ui32SizePerCol);
+
+		if (uiPageIndex % ui32Col == ui32Col-1)
+		{
+			PVR_LOG(("      Phys Page: %s", pszBuffer));
+			ui32ByteCount = 0;
+		}
+	}
+	if (ui32ByteCount > 0)
+	{
+		PVR_LOG(("      Phys Page: %s", pszBuffer));
+	}
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+DevPhysAddr_Error:
+Sparse_Error:
+MultiPage_Error:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+DEFINE_PMR_ENV_GET_DATA(psPMR, sEnvData)
+
+PVRSRV_ERROR
+PMRInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	/* Singleton PMR context already initialised */
+	if (_gsSingletonPMRContext.bModuleInitialised)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+	eError = OSLockCreate(&_gsSingletonPMRContext.hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", out);
+
+	_gsSingletonPMRContext.uiNextSerialNum = 1;
+
+	_gsSingletonPMRContext.uiNextKey = 0x8300f001 * (uintptr_t)&_gsSingletonPMRContext;
+
+	_gsSingletonPMRContext.bModuleInitialised = IMG_TRUE;
+
+	OSAtomicWrite(&_gsSingletonPMRContext.uiNumLivePMRs, 0);
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	eError = MMapStatsInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMapStatsInit", destroy_context_lock);
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+destroy_context_lock:
+	OSLockDestroy(_gsSingletonPMRContext.hLock);
+	_gsSingletonPMRContext.hLock = NULL;
+#endif
+out:
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRDeInit(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		goto out;
+	}
+
+	/* Singleton PMR context is not initialised */
+	if (!_gsSingletonPMRContext.bModuleInitialised)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	MMapStatsDeInit();
+#endif
+
+	if (OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error: %d live PMRs remain",
+				__func__,
+				OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs)));
+		PVR_DPF((PVR_DBG_ERROR, "%s: This is an unrecoverable error; a subsequent crash is inevitable",
+				__func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+	if (_gsSingletonPMRContext.hLock != NULL)
+	{
+		OSLockDestroy(_gsSingletonPMRContext.hLock);
+	}
+
+	_gsSingletonPMRContext.bModuleInitialised = IMG_FALSE;
+
+out:
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+PVRSRV_ERROR
+PMRInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDeviceNode->hPMRZombieListLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	dllist_init(&psDeviceNode->sPMRZombieList);
+	psDeviceNode->uiPMRZombieCount = 0;
+	psDeviceNode->uiPMRZombieCountInCleanup = 0;
+
+	return PVRSRV_OK;
+}
+
+void
+PMRFreeZombies(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	DECLARE_DLLIST(sZombieList);
+	DLLIST_NODE *psThis, *psNext;
+	IMG_INT32 uiZombieCount;
+
+	_ZombieListLock(psDeviceNode);
+	/* Move the zombie list to a local copy. The original list will become
+	 * an empty list. This will allow us to process the list without holding
+	 * the list lock. */
+	dllist_replace_head(&psDeviceNode->sPMRZombieList, &sZombieList);
+	uiZombieCount = psDeviceNode->uiPMRZombieCount;
+	psDeviceNode->uiPMRZombieCount = 0;
+	_ZombieListUnlock(psDeviceNode);
+
+	dllist_foreach_node(&sZombieList, psThis, psNext)
+	{
+		dllist_remove_node(psThis);
+		switch (PMR_GetZombieTypeFromNode(psThis))
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_PAGES:
+			{
+				PVRSRV_ERROR eError;
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psThis);
+
+				eError = psZombiePages->pfnFactoryFreeZombies(psZombiePages->pvFactoryPages);
+				if (eError != PVRSRV_OK)
+				{
+					/* In case of failure to free zombie pages, remove it from
+					* the sZombieList and add back to the original list. */
+					_ZombieListLock(psDeviceNode);
+					dllist_add_to_tail(&psDeviceNode->sPMRZombieList, psThis);
+					psDeviceNode->uiPMRZombieCount++;
+					_ZombieListUnlock(psDeviceNode);
+
+					PVR_DPF((PVR_DBG_ERROR, "Cannot free zombie pages!"));
+					continue;
+				}
+
+				OSFreeMem(psZombiePages);
+				break;
+			}
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_DEVICE_IMPORT:
+			{
+				PMR_DEVICE_IMPORT *psImport = PMR_GetDeviceImportFromNode(psThis);
+				_DeviceImportFreeImportZombie(psImport);
+				break;
+			}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+			case PMR_ZOMBIE_TYPE_PMR:
+			{
+				PMR *psPMR = PMR_GetPMRFromNode(psThis);
+				const PMR_IMPL_FUNCTAB *psFuncTable = psPMR->psFuncTab;
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+				/* The PMR cannot be freed as other devices are
+				 * still waiting for the cache flush. */
+				PMRLockPMR(psPMR);
+				if (_DeviceImportBitmapGet(psPMR) != 0)
+				{
+					PDLLIST_NODE psNodeImport;
+					PMR_DEVICE_IMPORT *psImport;
+					/* Transfer the ownership to a different
+					 * device queue that has not been processed yet.
+					 * There will be a PMR_DEVICE_IMPORT on the same
+					 * queue, however, this doesn't have any knock on affects as
+					 * it will be freed before the PMR is reached again. */
+					psNodeImport = dllist_get_next_node(&psPMR->sXDeviceImports);
+					PVR_ASSERT(psNodeImport);
+					psImport = IMG_CONTAINER_OF(psNodeImport, PMR_DEVICE_IMPORT, sNext);
+					_ZombieListLock(psImport->psDevNode);
+					dllist_add_to_tail(&psImport->psDevNode->sPMRZombieList, psThis);
+					psImport->psDevNode->uiPMRZombieCount++;
+					_ZombieListUnlock(psImport->psDevNode);
+					PMRUnlockPMR(psPMR);
+					break;
+				}
+				PMRUnlockPMR(psPMR);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+				_FactoryLock(psFuncTable);
+				_PMRDestroy(psPMR);
+				_FactoryUnlock(psFuncTable);
+				break;
+			}
+		}
+		uiZombieCount--;
+	}
+
+	PVR_ASSERT(uiZombieCount == 0);
+}
+
+void
+PMRDumpZombies(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	_ZombieListLock(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR, "Items in zombie list: %u",
+	        psDeviceNode->uiPMRZombieCount));
+
+	dllist_foreach_node(&psDeviceNode->sPMRZombieList, psThis, psNext)
+	{
+		switch (PMR_GetZombieTypeFromNode(psThis))
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_PAGES:
+			{
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psThis);
+				PVR_DPF((PVR_DBG_ERROR, "Zombie Pages = %p", psZombiePages));
+				break;
+			}
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_DEVICE_IMPORT:
+			{
+				PMR_DEVICE_IMPORT* psImport = PMR_GetDeviceImportFromNode(psThis);
+				PVR_DPF((PVR_DBG_ERROR, "Device Import = %p, DevID = %u, PMR = %px (%s)",
+				         psImport,
+				         psImport->psDevNode->sDevId.ui32InternalID,
+				         psImport->psParent,
+				         PMR_GetAnnotation(psImport->psParent)));
+				break;
+			}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+			case PMR_ZOMBIE_TYPE_PMR:
+			{
+				PMR *psPMR = PMR_GetPMRFromNode(psThis);
+				PVR_DPF((PVR_DBG_ERROR, "PMR = %px, Flavour = %s, Annotation: %s",
+						psPMR, PMR_GetTypeStr(psPMR), PMR_GetAnnotation(psPMR)));
+				break;
+			}
+		}
+	}
+
+	_ZombieListUnlock(psDeviceNode);
+}
+
+void
+PMRDeInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PMRFreeZombies(psDeviceNode);
+
+	OSLockDestroy(psDeviceNode->hPMRZombieListLock);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+IMG_UINT64
+PMR_ImportedDevicesMask(const PMR* psPMR)
+{
+	return _DeviceImportBitmapGet(psPMR) & ((IMG_UINT64_C(1) << PVRSRV_MAX_DEVICES) - 1);
+}
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+PVRSRV_ERROR
+PMR_RegisterDeviceImport(PMR* psPMR, PPVRSRV_DEVICE_NODE psDevNode)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVR_ASSERT(!PMR_IsZombie(psPMR));
+#endif /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	if (PMR_DeviceNode(psPMR) != psDevNode)
+	{
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+		PVRSRV_ERROR eError = _DeviceImportRegister(psPMR, psDevNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_DeviceImportRegister");
+#else
+		/* `_DeviceImportRegister` already sets the bitmap.
+		 * This is still needs to be set without device import zombie support
+		 * for debugging information, i.e. the RI. */
+		_DeviceImportBitmapSet(psPMR, psDevNode);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+	}
+	/* else: We explicitly don't add the PMR's dev node to the list because
+	 *       this bitmask lets us know if the PMR is cross device. It's not
+	 *       an error to register with the original dev node, as the user is
+	 *       declaring "The PMR is using `psDevNode`", not that it's a new
+	 *       devnode. */
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/power.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/power.c
new file mode 100644
index 000000000000..43647aa9c78c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/power.c
@@ -0,0 +1,1525 @@
+/*************************************************************************/ /*!
+@File           power.c
+@Title          Power management functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for power management functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if !defined(__linux__)
+#include <errno.h>
+#endif /* #if !defined(__linux__) */
+
+#include "pdump_km.h"
+#include "allocmem.h"
+#include "osfunc.h"
+
+#include "lock.h"
+#include "pvrsrv.h"
+#include "power.h"
+#include "pvr_debug.h"
+#include "htbserver.h"
+#include "di_server.h"
+
+#define OS_POWERLOCK_TIMEOUT_MS		(5000UL)
+#define OS_POWERLOCK_TIMEOUT_US		(OS_POWERLOCK_TIMEOUT_MS * 1000UL)
+#define OS_POWERLOCK_RETRY_COUNT	(50)
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+typedef struct _EXTRA_POWER_STATS_
+{
+	IMG_UINT64	ui64PreClockSpeedChangeDuration;
+	IMG_UINT64	ui64BetweenPreEndingAndPostStartingDuration;
+	IMG_UINT64	ui64PostClockSpeedChangeDuration;
+} EXTRA_POWER_STATS;
+
+/* For the power timing stats we need 16 variables to store all the
+ * combinations of forced/not forced, power-on/power-off, pre-power/post-power
+ * and device/system statistics
+ */
+#define NUM_POWER_STATS        (16)
+#define NUM_EXTRA_POWER_STATS	10
+
+typedef struct PVRSRV_POWER_STATS_TAG
+{
+	EXTRA_POWER_STATS               asClockSpeedChanges[NUM_EXTRA_POWER_STATS];
+	IMG_UINT64                      ui64PreClockSpeedChangeMark;
+	IMG_UINT64                      ui64FirmwareIdleDuration;
+	IMG_UINT32                      aui32PowerTimingStats[NUM_POWER_STATS];
+	IMG_UINT32                      ui32ClockSpeedIndexStart;
+	IMG_UINT32                      ui32ClockSpeedIndexEnd;
+	IMG_UINT32                      ui32FirmwareStartTimestamp;
+} PVRSRV_POWER_STATS;
+#endif
+
+struct _PVRSRV_POWER_DEV_TAG_
+{
+	PFN_PRE_POWER					pfnDevicePrePower;
+	PFN_POST_POWER					pfnDevicePostPower;
+	PFN_SYS_PRE_POWER				pfnSystemPrePower;
+	PFN_SYS_POST_POWER				pfnSystemPostPower;
+	PFN_PRE_CLOCKSPEED_CHANGE		pfnPreClockSpeedChange;
+	PFN_POST_CLOCKSPEED_CHANGE		pfnPostClockSpeedChange;
+	PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest;
+	PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest;
+	PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC	pfnForcedIdleCancelRequestAsync;
+	PFN_GPU_UNITS_POWER_CHANGE		pfnGPUUnitsPowerChange;
+	IMG_HANDLE						hSysData;
+	IMG_HANDLE						psDevNode;
+	PVRSRV_DEV_POWER_STATE			eDefaultPowerState;
+	ATOMIC_T						eCurrentPowerState;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_POWER_STATS				sPowerStats;
+#endif
+	/* Counts number of device idle references that have been taken,
+	 * must be protected by power lock.
+	 */
+	IMG_INT32                       iIdleReqInFlight;
+};
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+/*
+ * Power statistics related definitions
+ */
+
+/* For the mean time, use an exponentially weighted moving average with a
+ * 1/4 weighting for the new measurement.
+ */
+#define MEAN_TIME(A, B)     ( ((3*(A))/4) + ((1 * (B))/4) )
+
+#define UPDATE_TIME(time, newtime) \
+	((time) > 0 ? MEAN_TIME((time), (newtime)) : (newtime))
+
+/* Enum to be used as input to GET_POWER_STAT_INDEX */
+typedef enum
+{
+	DEVICE     = 0,
+	SYSTEM     = 1,
+	POST_POWER = 0,
+	PRE_POWER  = 2,
+	POWER_OFF  = 0,
+	POWER_ON   = 4,
+	NOT_FORCED = 0,
+	FORCED     = 8,
+} PVRSRV_POWER_STAT_TYPE;
+
+/* Macro used to access one of the power timing statistics inside an array */
+#define GET_POWER_STAT_INDEX(forced,powon,prepow,system) \
+	((forced) + (powon) + (prepow) + (system))
+
+void PVRSRVSetFirmwareStartTime(PVRSRV_POWER_DEV *psPowerDevice,
+						  IMG_UINT32 ui32Time)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->ui32FirmwareStartTimestamp =
+		UPDATE_TIME(psPowerStats->ui32FirmwareStartTimestamp,
+					ui32Time);
+}
+
+void PVRSRVSetFirmwareHandshakeIdleTime(PVRSRV_POWER_DEV *psPowerDevice,
+								  IMG_UINT64 ui64Duration)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->ui64FirmwareIdleDuration =
+		UPDATE_TIME(psPowerStats->ui64FirmwareIdleDuration,
+					ui64Duration);
+}
+
+static void _InsertPowerTimeStatistic(PVRSRV_POWER_DEV *psPowerDevice,
+									  IMG_UINT64 ui64SysStartTime, IMG_UINT64 ui64SysEndTime,
+									  IMG_UINT64 ui64DevStartTime, IMG_UINT64 ui64DevEndTime,
+									  IMG_BOOL bForced, IMG_BOOL bPowerOn, IMG_BOOL bPrePower)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+	IMG_UINT32 *pui32Stat;
+	IMG_UINT64 ui64DeviceDiff = ui64DevEndTime - ui64DevStartTime;
+	IMG_UINT64 ui64SystemDiff = ui64SysEndTime - ui64SysStartTime;
+	IMG_UINT32 ui32Index;
+
+	if (bPrePower)
+	{
+		HTBLOGK(HTB_SF_MAIN_PRE_POWER, bPowerOn, ui64DeviceDiff, ui64SystemDiff);
+	}
+	else
+	{
+		HTBLOGK(HTB_SF_MAIN_POST_POWER, bPowerOn, ui64SystemDiff, ui64DeviceDiff);
+	}
+
+	ui32Index = GET_POWER_STAT_INDEX(bForced ? FORCED : NOT_FORCED,
+	                                 bPowerOn ? POWER_ON : POWER_OFF,
+	                                 bPrePower ? PRE_POWER : POST_POWER,
+	                                 DEVICE);
+	pui32Stat = &psPowerStats->aui32PowerTimingStats[ui32Index];
+	*pui32Stat = UPDATE_TIME(*pui32Stat, ui64DeviceDiff);
+
+	ui32Index = GET_POWER_STAT_INDEX(bForced ? FORCED : NOT_FORCED,
+	                                 bPowerOn ? POWER_ON : POWER_OFF,
+	                                 bPrePower ? PRE_POWER : POST_POWER,
+	                                 SYSTEM);
+	pui32Stat = &psPowerStats->aui32PowerTimingStats[ui32Index];
+	*pui32Stat = UPDATE_TIME(*pui32Stat, ui64SystemDiff);
+}
+
+static void _InsertPowerTimeStatisticExtraPre(PVRSRV_POWER_DEV *psPowerDevice,
+											  IMG_UINT64 ui64StartTimer,
+											  IMG_UINT64 ui64Stoptimer)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64PreClockSpeedChangeDuration =
+		ui64Stoptimer - ui64StartTimer;
+
+	psPowerStats->ui64PreClockSpeedChangeMark = OSClockus();
+}
+
+static void _InsertPowerTimeStatisticExtraPost(PVRSRV_POWER_DEV *psPowerDevice,
+											   IMG_UINT64 ui64StartTimer,
+											   IMG_UINT64 ui64StopTimer)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+	IMG_UINT64 ui64Duration = ui64StartTimer - psPowerStats->ui64PreClockSpeedChangeMark;
+
+	PVR_ASSERT(psPowerStats->ui64PreClockSpeedChangeMark > 0);
+
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64BetweenPreEndingAndPostStartingDuration = ui64Duration;
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64PostClockSpeedChangeDuration = ui64StopTimer - ui64StartTimer;
+
+	psPowerStats->ui32ClockSpeedIndexEnd = (psPowerStats->ui32ClockSpeedIndexEnd + 1) % NUM_EXTRA_POWER_STATS;
+
+	if (psPowerStats->ui32ClockSpeedIndexEnd == psPowerStats->ui32ClockSpeedIndexStart)
+	{
+		psPowerStats->ui32ClockSpeedIndexStart = (psPowerStats->ui32ClockSpeedIndexStart + 1) % NUM_EXTRA_POWER_STATS;
+	}
+
+	psPowerStats->ui64PreClockSpeedChangeMark = 0;
+}
+
+static INLINE void _PowerStatsPrintGroup(IMG_UINT32 *pui32Stats,
+										 OSDI_IMPL_ENTRY *psEntry,
+										 PVRSRV_POWER_STAT_TYPE eForced,
+										 PVRSRV_POWER_STAT_TYPE ePowerOn)
+{
+	IMG_UINT32 ui32Index;
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, PRE_POWER, DEVICE);
+	DIPrintf(psEntry, "  Pre-Device:  %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, PRE_POWER, SYSTEM);
+	DIPrintf(psEntry, "  Pre-System:  %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, POST_POWER, SYSTEM);
+	DIPrintf(psEntry, "  Post-System: %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, POST_POWER, DEVICE);
+	DIPrintf(psEntry, "  Post-Device: %9u\n", pui32Stats[ui32Index]);
+}
+
+int PVRSRVPowerStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	PVRSRV_POWER_STATS *psPowerStats;
+	IMG_UINT32 *pui32Stats;
+	IMG_UINT32 ui32Idx;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psPowerDevice == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "reading power timing stats!"));
+		return -EIO;
+	}
+
+	psPowerStats = &psPowerDevice->sPowerStats;
+
+	pui32Stats = &psPowerStats->aui32PowerTimingStats[0];
+
+	DIPrintf(psEntry, "Forced Power-on Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, FORCED, POWER_ON);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Forced Power-off Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, FORCED, POWER_OFF);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Not Forced Power-on Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, NOT_FORCED, POWER_ON);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Not Forced Power-off Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, NOT_FORCED, POWER_OFF);
+	DIPrintf(psEntry, "\n");
+
+
+	DIPrintf(psEntry, "FW bootup time (timer ticks): %u\n", psPowerStats->ui32FirmwareStartTimestamp);
+	DIPrintf(psEntry, "Host Acknowledge Time for FW Idle Signal (timer ticks): %u\n", (IMG_UINT32)(psPowerStats->ui64FirmwareIdleDuration));
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Last %d Clock Speed Change Timers (nanoseconds):\n", NUM_EXTRA_POWER_STATS);
+	DIPrintf(psEntry, "Prepare DVFS\tDVFS Change\tPost DVFS\n");
+
+	for (ui32Idx = psPowerStats->ui32ClockSpeedIndexStart;
+		 ui32Idx != psPowerStats->ui32ClockSpeedIndexEnd;
+		 ui32Idx = (ui32Idx + 1) % NUM_EXTRA_POWER_STATS)
+	{
+		DIPrintf(psEntry, "%12" IMG_UINT64_FMTSPEC "\t%11" IMG_UINT64_FMTSPEC "\t%9" IMG_UINT64_FMTSPEC "\n",
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64PreClockSpeedChangeDuration,
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64BetweenPreEndingAndPostStartingDuration,
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64PostClockSpeedChangeDuration);
+	}
+
+	return 0;
+}
+
+#else /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+static void _InsertPowerTimeStatistic(PVRSRV_POWER_DEV *psPowerDevice,
+									  IMG_UINT64 ui64SysStartTime, IMG_UINT64 ui64SysEndTime,
+									  IMG_UINT64 ui64DevStartTime, IMG_UINT64 ui64DevEndTime,
+									  IMG_BOOL bForced, IMG_BOOL bPowerOn, IMG_BOOL bPrePower)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64SysStartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64SysEndTime);
+	PVR_UNREFERENCED_PARAMETER(ui64DevStartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64DevEndTime);
+	PVR_UNREFERENCED_PARAMETER(bForced);
+	PVR_UNREFERENCED_PARAMETER(bPowerOn);
+	PVR_UNREFERENCED_PARAMETER(bPrePower);
+}
+
+static void _InsertPowerTimeStatisticExtraPre(PVRSRV_POWER_DEV *psPowerDevice,
+											  IMG_UINT64 ui64StartTimer,
+											  IMG_UINT64 ui64Stoptimer)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTimer);
+	PVR_UNREFERENCED_PARAMETER(ui64Stoptimer);
+}
+
+static void _InsertPowerTimeStatisticExtraPost(PVRSRV_POWER_DEV *psPowerDevice,
+											   IMG_UINT64 ui64StartTimer,
+											   IMG_UINT64 ui64StopTimer)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTimer);
+	PVR_UNREFERENCED_PARAMETER(ui64StopTimer);
+}
+#endif
+
+const char *PVRSRVSysPowerStateToString(PVRSRV_SYS_POWER_STATE eState)
+{
+	switch (eState) {
+#define X(name, _) \
+		case PVRSRV_SYS_POWER_STATE_##name: \
+			return #name;
+		_PVRSRV_SYS_POWER_STATES
+#undef X
+		default:
+			return "unknown";
+	}
+}
+
+const char *PVRSRVDevPowerStateToString(PVRSRV_DEV_POWER_STATE eState)
+{
+	switch (eState) {
+		case PVRSRV_DEV_POWER_STATE_DEFAULT:
+			return "DEFAULT";
+		case PVRSRV_DEV_POWER_STATE_OFF:
+			return "OFF";
+		case PVRSRV_DEV_POWER_STATE_ON:
+			return "ON";
+		default:
+			return "unknown";
+	}
+}
+
+/*!
+  Typedef for a pointer to a function that will be called for re-acquiring
+  device powerlock after releasing it temporarily for some timeout period
+  in function PVRSRVDeviceIdleRequestKM
+ */
+#if defined(DEBUG)
+typedef PVRSRV_ERROR (*PFN_POWER_LOCK_ACQUIRE) (PPVRSRV_DEVICE_NODE psDevNode,
+                                                const char *pszFile, const unsigned int ui32LineNum);
+#else
+typedef PVRSRV_ERROR (*PFN_POWER_LOCK_ACQUIRE) (PPVRSRV_DEVICE_NODE psDevNode);
+#endif
+
+static inline IMG_UINT64 PVRSRVProcessStatsGetTimeNs(void)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	return OSClockns64();
+#else
+	return 0;
+#endif
+}
+
+static inline IMG_UINT64 PVRSRVProcessStatsGetTimeUs(void)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	return OSClockus();
+#else
+	return 0;
+#endif
+}
+
+/*!
+******************************************************************************
+
+ @Function	_IsSystemStatePowered
+
+ @Description	Tests whether a given system state represents powered-up.
+
+ @Input		eSystemPowerState : a system power state
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+static IMG_BOOL _IsSystemStatePowered(PVRSRV_SYS_POWER_STATE eSystemPowerState)
+{
+	return (eSystemPowerState == PVRSRV_SYS_POWER_STATE_ON);
+}
+
+/* We don't expect PID=0 to acquire device power-lock */
+#define PWR_LOCK_OWNER_PID_CLR_VAL 0
+
+PVRSRV_ERROR PVRSRVPowerLockInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDeviceNode->hPowerLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	return PVRSRV_OK;
+}
+
+void PVRSRVPowerLockDeInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	OSLockDestroy(psDeviceNode->hPowerLock);
+}
+
+IMG_BOOL PVRSRVPwrLockIsLockedByMe(PCPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	return OSLockIsLocked(psDeviceNode->hPowerLock) &&
+	       OSGetCurrentClientProcessIDKM() == psDeviceNode->uiPwrLockOwnerPID;
+}
+
+
+#if defined(DEBUG)
+static void _PVRSRVPowerLockOwner(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const char *pszFile, const unsigned int ui32LineNum)
+{
+	psDeviceNode->sPowerLockOwner.pszFile = pszFile;
+	psDeviceNode->sPowerLockOwner.ui32LineNum = ui32LineNum;
+	psDeviceNode->sPowerLockOwner.ui64Timestamp = OSClockus64();
+}
+#endif
+
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                   const char *pszFile, const unsigned int ui32LineNum)
+#else
+PVRSRV_ERROR PVRSRVPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+#endif
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+#if defined(DEBUG)
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+#endif
+
+	return PVRSRV_OK;
+}
+
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerTryLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      const char *pszFile, const unsigned int ui32LineNum)
+#else
+PVRSRV_ERROR PVRSRVPowerTryLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+#endif
+{
+	if (!(OSTryLockAcquire(psDeviceNode->hPowerLock)))
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+#if defined(DEBUG)
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVPowerTryLockWaitForTimeout(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Take power lock, retry if it's in use in another task. */
+	LOOP_UNTIL_TIMEOUT_US(OS_POWERLOCK_TIMEOUT_US)
+	{
+		eError = PVRSRVPowerTryLock(psDeviceNode);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			return eError;
+		}
+
+		OSSleepms(OS_POWERLOCK_TIMEOUT_MS / OS_POWERLOCK_RETRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError == PVRSRV_ERROR_RETRY)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function     _PVRSRVForcedPowerLock
+
+ @Description  Obtain the mutex for power transitions regardless of system
+               power state
+
+ @Return       Always returns PVRSRV_OK. Function prototype required same as
+               PFN_POWER_LOCK_ACQUIRE
+
+******************************************************************************/
+#if defined(DEBUG)
+#define _PVRSRVForcedPowerLock(DEV_NODE)	\
+	_PVRSRVForcedPowerLock_Debug(DEV_NODE, __FILE__, __LINE__)
+
+static PVRSRV_ERROR _PVRSRVForcedPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                 const char *pszFile, const unsigned int ui32LineNum)
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+
+	return PVRSRV_OK;
+}
+#else
+static PVRSRV_ERROR _PVRSRVForcedPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+}
+#endif
+
+void PVRSRVPowerUnlock(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDeviceNode));
+
+	/* Reset uiPwrLockOwnerPID before releasing lock */
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	OSLockRelease(psDeviceNode->hPowerLock);
+}
+
+IMG_BOOL PVRSRVDeviceIsDefaultStateOFF(PVRSRV_POWER_DEV *psPowerDevice)
+{
+	return (psPowerDevice->eDefaultPowerState == PVRSRV_DEV_POWER_STATE_OFF);
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceDefaultPowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+					PVRSRV_DEV_POWER_STATE eNewPowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psPowerDevice->eDefaultPowerState = eNewPowerState;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceCurrentPowerState(PVRSRV_POWER_DEV *psPowerDevice,
+					PVRSRV_DEV_POWER_STATE eNewPowerState)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEV_POWER_STATE eOldPowerState;
+#endif
+
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if !defined(SUPPORT_PMR_DEFERRED_FREE)
+	OSAtomicWrite(&psPowerDevice->eCurrentPowerState, eNewPowerState);
+#else
+	eOldPowerState = OSAtomicExchange(&psPowerDevice->eCurrentPowerState,
+	                                  eNewPowerState);
+
+	psDeviceNode = psPowerDevice->psDevNode;
+	PVR_ASSERT(psDeviceNode);
+
+	if (eNewPowerState == PVRSRV_DEV_POWER_STATE_OFF &&
+	    eNewPowerState != eOldPowerState)
+	{
+		psDeviceNode->uiPowerOffCounter = psDeviceNode->uiPowerOffCounterNext;
+
+		/* It's not really important to know if any zombies were queued. */
+		(void) PMRQueueZombiesForCleanup(psDeviceNode);
+
+		psDeviceNode->uiPowerOffCounterNext++;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ @Input       pfnPowerLockAcquire  : Function to re-acquire power-lock in-case
+                                     it was necessary to release it.
+*/
+static PVRSRV_ERROR _PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF    pfnIsDefaultStateOff,
+					IMG_BOOL                            bDeviceOffPermitted,
+					PFN_POWER_LOCK_ACQUIRE              pfnPowerLockAcquire)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	/* if pfnIsDefaultStateOff not provided or pfnIsDefaultStateOff(psPowerDev)
+	 * is true (which means that the default state is OFF) then force idle. */
+	if ((psPowerDev && psPowerDev->pfnForcedIdleRequest) &&
+	    (pfnIsDefaultStateOff == NULL || pfnIsDefaultStateOff(psPowerDev)))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = psPowerDev->pfnForcedIdleRequest(psPowerDev->psDevNode,
+			                                          bDeviceOffPermitted);
+			if (eError == PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+			{
+				PVRSRV_ERROR eErrPwrLockAcq;
+				/* FW denied idle request */
+				PVRSRVPowerUnlock(psDeviceNode);
+
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+
+#if defined(DEBUG)
+				eErrPwrLockAcq = pfnPowerLockAcquire(psDeviceNode, __FILE__, __LINE__);
+#else
+				eErrPwrLockAcq = pfnPowerLockAcquire(psDeviceNode);
+#endif
+				if (eErrPwrLockAcq != PVRSRV_OK)
+				{
+					/* We only understand PVRSRV_ERROR_RETRY, so assert on others.
+					 * Moreover, we've ended-up releasing the power-lock which was
+					 * originally "held" by caller before calling this function -
+					 * since this needs vigilant handling at call-site, we pass
+					 * back an explicit error, for caller(s) to "avoid" calling
+					 * PVRSRVPowerUnlock */
+					PVR_ASSERT(eErrPwrLockAcq == PVRSRV_ERROR_RETRY);
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to re-acquire power-lock "
+					         "(%s) after releasing it for a time-out",
+							 __func__, PVRSRVGetErrorString(eErrPwrLockAcq)));
+					return PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED;
+				}
+			}
+			else
+			{
+				/* idle request successful or some other error occurred, return */
+				break;
+			}
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*
+ * Wrapper function helps limiting calling complexity of supplying additional
+ * PFN_POWER_LOCK_ACQUIRE argument (required by _PVRSRVDeviceIdleRequestKM)
+ */
+inline PVRSRV_ERROR PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF      pfnIsDefaultStateOff,
+					IMG_BOOL                              bDeviceOffPermitted)
+{
+	return _PVRSRVDeviceIdleRequestKM(psDeviceNode,
+	                                  pfnIsDefaultStateOff,
+	                                  bDeviceOffPermitted,
+#if defined(DEBUG)
+	                                  PVRSRVPowerLock_Debug
+#else
+	                                  PVRSRVPowerLock
+#endif
+	                                  );
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleCancelRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+
+	if (psPowerDev && psPowerDev->pfnForcedIdleCancelRequest)
+	{
+		return psPowerDev->pfnForcedIdleCancelRequest(psPowerDev->psDevNode);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedGetKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	if ((++psPowerDev->iIdleReqInFlight) > 1)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return PVRSRV_OK;
+	}
+
+	eError = _PVRSRVDeviceIdleRequestKM(psDeviceNode,
+									  &PVRSRVDeviceIsDefaultStateOFF,
+									  IMG_TRUE,
+#if defined(DEBUG)
+									  PVRSRVPowerLock_Debug
+#else
+									  PVRSRVPowerLock
+#endif
+									  );
+	if (eError != PVRSRV_OK)
+	{
+		psPowerDev->iIdleReqInFlight--;
+	}
+
+	if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedPutAsyncKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPowerLock", ErrorOut);
+
+	if (psPowerDev &&
+	    (--psPowerDev->iIdleReqInFlight) == 0 &&
+	    psPowerDev->pfnForcedIdleCancelRequestAsync)
+	{
+		eError = psPowerDev->pfnForcedIdleCancelRequestAsync(psPowerDev->psDevNode);
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+ErrorOut:
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceSystemPrePowerStateKM
+
+ @Description
+
+ Perform device and system-specific processing required before a power
+ transition
+
+ @Input		psPowerDevice : Power device
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static
+PVRSRV_ERROR PVRSRVDeviceSystemPrePowerStateKM(PVRSRV_POWER_DEV			*psPowerDevice,
+											   PVRSRV_DEV_POWER_STATE	eNewPowerState,
+											   PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+	IMG_UINT64 ui64SysTimer1 = 0;
+	IMG_UINT64 ui64SysTimer2 = 0;
+	IMG_UINT64 ui64DevTimer1 = 0;
+	IMG_UINT64 ui64DevTimer2 = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(eNewPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	if (psPowerDevice->pfnDevicePrePower != NULL)
+	{
+		ui64DevTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		/* Call the device's power callback. */
+		eError = psPowerDevice->pfnDevicePrePower(psPowerDevice->psDevNode,
+												  eNewPowerState,
+												  eCurrentPowerState,
+												  ePwrFlags);
+
+		ui64DevTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	/* Do any required system-layer processing. */
+	if (psPowerDevice->pfnSystemPrePower != NULL)
+	{
+		ui64SysTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		eError = psPowerDevice->pfnSystemPrePower(psPowerDevice->hSysData,
+												  (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													PVRSRV_SYS_POWER_STATE_ON :
+													PVRSRV_SYS_POWER_STATE_OFF,
+												  (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													PVRSRV_SYS_POWER_STATE_ON :
+													PVRSRV_SYS_POWER_STATE_OFF,
+												  ePwrFlags);
+
+		ui64SysTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_GOTO_IF_ERROR(eError, ErrRestorePowerState);
+	}
+
+	_InsertPowerTimeStatistic(psPowerDevice, ui64SysTimer1, ui64SysTimer2,
+							 ui64DevTimer1, ui64DevTimer2,
+							 BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED),
+							 eNewPowerState == PVRSRV_DEV_POWER_STATE_ON,
+							 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+ErrRestorePowerState:
+	/* In a situation where pfnDevicePrePower() succeeded but pfnSystemPrePower()
+	 * failed we need to restore the device's power state from before the current
+	 * request. Otherwise it will result in an inconsistency between the device's
+	 * actual state and what the driver thinks the state is. */
+	{
+		PVRSRV_ERROR eError2 = PVRSRV_OK;
+
+		if (psPowerDevice->pfnDevicePrePower != NULL)
+		{
+			/* Call the device's power callback. */
+			eError2 = psPowerDevice->pfnDevicePrePower(psPowerDevice->psDevNode,
+			                                           eCurrentPowerState,
+			                                           eNewPowerState,
+			                                           ePwrFlags);
+			PVR_LOG_IF_ERROR(eError2, "pfnDevicePrePower");
+		}
+		if (eError2 == PVRSRV_OK && psPowerDevice->pfnDevicePostPower != NULL)
+		{
+			/* Call the device's power callback. */
+			eError2 = psPowerDevice->pfnDevicePostPower(psPowerDevice->psDevNode,
+			                                            eCurrentPowerState,
+			                                            eNewPowerState,
+			                                            ePwrFlags);
+			PVR_LOG_IF_ERROR(eError2, "pfnDevicePostPower");
+		}
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceSystemPostPowerStateKM
+
+ @Description
+
+ Perform device and system-specific processing required after a power
+ transition
+
+ @Input		psPowerDevice : Power device
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static
+PVRSRV_ERROR PVRSRVDeviceSystemPostPowerStateKM(PVRSRV_POWER_DEV		*psPowerDevice,
+												PVRSRV_DEV_POWER_STATE	eNewPowerState,
+												PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+	IMG_UINT64 ui64SysTimer1 = 0;
+	IMG_UINT64 ui64SysTimer2 = 0;
+	IMG_UINT64 ui64DevTimer1 = 0;
+	IMG_UINT64 ui64DevTimer2 = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(eNewPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	/* Do any required system-layer processing. */
+	if (psPowerDevice->pfnSystemPostPower != NULL)
+	{
+		ui64SysTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		eError = psPowerDevice->pfnSystemPostPower(psPowerDevice->hSysData,
+												   (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													 PVRSRV_SYS_POWER_STATE_ON :
+													 PVRSRV_SYS_POWER_STATE_OFF,
+												   (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													 PVRSRV_SYS_POWER_STATE_ON :
+													 PVRSRV_SYS_POWER_STATE_OFF,
+												   ePwrFlags);
+
+		ui64SysTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psPowerDevice->pfnDevicePostPower != NULL)
+	{
+		ui64DevTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		/* Call the device's power callback. */
+		eError = psPowerDevice->pfnDevicePostPower(psPowerDevice->psDevNode,
+												   eNewPowerState,
+												   eCurrentPowerState,
+												   ePwrFlags);
+
+		ui64DevTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	_InsertPowerTimeStatistic(psPowerDevice, ui64SysTimer1, ui64SysTimer2,
+							 ui64DevTimer1, ui64DevTimer2,
+							 BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED),
+							 eNewPowerState == PVRSRV_DEV_POWER_STATE_ON,
+							 IMG_FALSE);
+
+	PVRSRVSetDeviceCurrentPowerState(psPowerDevice, eNewPowerState);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVSetDevicePowerStateKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE eNewPowerState,
+										 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_ERROR	eError;
+	PVRSRV_DATA*    psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (!psPowerDevice)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (eNewPowerState == PVRSRV_DEV_POWER_STATE_DEFAULT)
+	{
+		eNewPowerState = psPowerDevice->eDefaultPowerState;
+	}
+
+	/* Call power function if the state change or if this is an OS request. */
+	if (OSAtomicRead(&psPowerDevice->eCurrentPowerState) != eNewPowerState ||
+	    BITMASK_ANY(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ | PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ))
+	{
+		eError = PVRSRVDeviceSystemPrePowerStateKM(psPowerDevice,
+												   eNewPowerState,
+												   ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		eError = PVRSRVDeviceSystemPostPowerStateKM(psPowerDevice,
+													eNewPowerState,
+													ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		psDeviceNode->eCurrentSysPowerState = eNewPowerState;
+
+		/* Signal Device Watchdog Thread about power mode change. */
+		if (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans++;
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			if (psPVRSRVData->ui32DevicesWatchdogTimeout == DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT)
+#endif
+			{
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+		}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+		else if (eNewPowerState == PVRSRV_DEV_POWER_STATE_OFF)
+		{
+			/* signal watchdog thread and give it a chance to switch to
+			 * longer / infinite wait time */
+			eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	return PVRSRV_OK;
+
+ErrorExit:
+
+	if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Transition to %d was denied, Flags=0x%08x",
+				 __func__, eNewPowerState, ePwrFlags));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Transition to %d FAILED (%s)",
+				 __func__, eNewPowerState, PVRSRVGetErrorString(eError)));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState,
+											 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT uiStage = 0;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+#endif
+
+	PVRSRV_DATA*    psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_POWER_DEV *psPowerDevice;
+	PVRSRV_DEV_POWER_STATE eNewDevicePowerState;
+	PFN_SYS_DEV_IS_DEFAULT_STATE_OFF pfnIsDefaultStateOff;
+
+	/* Require a proper power state */
+	if (eNewSysPowerState == PVRSRV_SYS_POWER_STATE_Unspecified)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		/* Power device is not initialised. */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device not initialised", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Prevent simultaneous SetPowerStateKM calls */
+	_PVRSRVForcedPowerLock(psDeviceNode);
+	psPowerDevice = psDeviceNode->psPowerDev;
+
+	eNewDevicePowerState = _IsSystemStatePowered(eNewSysPowerState)
+	    ? psPowerDevice->eDefaultPowerState : PVRSRV_DEV_POWER_STATE_OFF;
+
+	/* If setting devices to default state, force idle all devices whose default state is off */
+	pfnIsDefaultStateOff =
+	  (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_OFF) ? PVRSRVDeviceIsDefaultStateOFF : NULL;
+
+	/* No power transition requested, so do nothing */
+	if (eNewSysPowerState == psDeviceNode->eCurrentSysPowerState)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return PVRSRV_OK;
+	}
+
+	/* If the device is already off don't send the idle request. */
+	if ((psDeviceNode->eCurrentSysPowerState != PVRSRV_SYS_POWER_STATE_OFF) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		&& (eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+#endif
+		)
+	{
+		eError = _PVRSRVDeviceIdleRequestKM(psDeviceNode, pfnIsDefaultStateOff,
+#if defined(DEBUG)
+		                                    IMG_TRUE, _PVRSRVForcedPowerLock_Debug
+#else
+		                                    IMG_TRUE, _PVRSRVForcedPowerLock
+#endif
+		                                    );
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PVRSRVDeviceIdleRequestKM");
+			uiStage = 1;
+			goto ErrorExit;
+		}
+	}
+
+	/* Call power function if the state change or if this is an OS request. */
+	if (OSAtomicRead(&psPowerDevice->eCurrentPowerState) != eNewDevicePowerState ||
+	    BITMASK_ANY(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ | PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ))
+	{
+		eError = PVRSRVDeviceSystemPrePowerStateKM(psPowerDevice,
+												   eNewDevicePowerState,
+												   ePwrFlags);
+		uiStage = 2;
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		eError = PVRSRVDeviceSystemPostPowerStateKM(psPowerDevice,
+													eNewDevicePowerState,
+													ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		/* Signal Device Watchdog Thread about power mode change. */
+		if (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans++;
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			if (psPVRSRVData->ui32DevicesWatchdogTimeout == DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT)
+#endif
+			{
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+		}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+		else if (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_OFF)
+		{
+			/* signal watchdog thread and give it a chance to switch to
+			 * longer / infinite wait time */
+			eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	psDeviceNode->eCurrentSysPowerState = eNewSysPowerState;
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Transition from %s to %s FAILED (%s) at stage "
+	         "%u. Dumping debug info.", __func__,
+	         PVRSRVSysPowerStateToString(psDeviceNode->eCurrentSysPowerState),
+	         PVRSRVSysPowerStateToString(eNewSysPowerState),
+	         PVRSRVGetErrorString(eError), uiStage));
+
+	PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVSetSystemPowerState(PVRSRV_DEVICE_CONFIG *psDevConfig,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode = psDevConfig->psDevNode;
+	PVRSRV_SYS_POWER_STATE eCurrentSysPowerState;
+
+	PVRSRVGetSystemPowerState(psDevNode, &eCurrentSysPowerState);
+
+	/* no power transition requested, so do nothing */
+	if (eNewSysPowerState == eCurrentSysPowerState)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (psDevConfig->pfnPrePowerState != NULL)
+	{
+		eError = psDevConfig->pfnPrePowerState(psDevConfig->hSysData,
+												  eNewSysPowerState,
+												  eCurrentSysPowerState,
+												  PVRSRV_POWER_FLAGS_FORCED);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psDevConfig->pfnPostPowerState != NULL)
+	{
+		eError = psDevConfig->pfnPostPowerState(psDevConfig->hSysData,
+												   eNewSysPowerState,
+												   eCurrentSysPowerState,
+												   PVRSRV_POWER_FLAGS_FORCED);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psDevNode != NULL)
+	{
+		psDevNode->eCurrentSysPowerState = eNewSysPowerState;
+	}
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVSetPowerCallbacks(PPVRSRV_DEVICE_NODE				psDeviceNode,
+							 PVRSRV_POWER_DEV					*psPowerDevice,
+							 PFN_PRE_POWER						pfnDevicePrePower,
+							 PFN_POST_POWER						pfnDevicePostPower,
+							 PFN_SYS_PRE_POWER					pfnSystemPrePower,
+							 PFN_SYS_POST_POWER					pfnSystemPostPower,
+							 PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync)
+{
+	if (psPowerDevice != NULL)
+	{
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+		{
+			psPowerDevice->pfnSystemPrePower = NULL;
+			psPowerDevice->pfnSystemPostPower = NULL;
+		}
+		else
+		{
+			psPowerDevice->pfnSystemPrePower = pfnSystemPrePower;
+			psPowerDevice->pfnSystemPostPower = pfnSystemPostPower;
+		}
+
+		psPowerDevice->pfnDevicePrePower = pfnDevicePrePower;
+		psPowerDevice->pfnDevicePostPower = pfnDevicePostPower;
+		psPowerDevice->pfnForcedIdleRequest = pfnForcedIdleRequest;
+		psPowerDevice->pfnForcedIdleCancelRequest = pfnForcedIdleCancelRequest;
+		psPowerDevice->pfnForcedIdleCancelRequestAsync = pfnForcedIdleCancelRequestAsync;
+	}
+}
+
+PVRSRV_ERROR PVRSRVRegisterPowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode,
+									   PFN_PRE_POWER				pfnDevicePrePower,
+									   PFN_POST_POWER				pfnDevicePostPower,
+									   PFN_SYS_PRE_POWER			pfnSystemPrePower,
+									   PFN_SYS_POST_POWER			pfnSystemPostPower,
+									   PFN_PRE_CLOCKSPEED_CHANGE	pfnPreClockSpeedChange,
+									   PFN_POST_CLOCKSPEED_CHANGE	pfnPostClockSpeedChange,
+									   PFN_FORCED_IDLE_REQUEST	pfnForcedIdleRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync,
+									   PFN_GPU_UNITS_POWER_CHANGE	pfnGPUUnitsPowerChange,
+									   IMG_HANDLE					psDevNode,
+									   PVRSRV_DEV_POWER_STATE		eCurrentPowerState,
+									   PVRSRV_DEV_POWER_STATE		eDefaultPowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	PVR_ASSERT(!psDeviceNode->psPowerDev);
+
+	PVR_ASSERT(eCurrentPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+	PVR_ASSERT(eDefaultPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	psPowerDevice = OSAllocMem(sizeof(PVRSRV_POWER_DEV));
+	PVR_LOG_RETURN_IF_NOMEM(psPowerDevice, "psPowerDevice");
+
+	/* setup device for power manager */
+	PVRSRVSetPowerCallbacks(psDeviceNode,
+							psPowerDevice,
+							pfnDevicePrePower,
+							pfnDevicePostPower,
+							pfnSystemPrePower,
+							pfnSystemPostPower,
+							pfnForcedIdleRequest,
+							pfnForcedIdleCancelRequest,
+							pfnForcedIdleCancelRequestAsync);
+
+	psPowerDevice->pfnPreClockSpeedChange = pfnPreClockSpeedChange;
+	psPowerDevice->pfnPostClockSpeedChange = pfnPostClockSpeedChange;
+	psPowerDevice->pfnGPUUnitsPowerChange = pfnGPUUnitsPowerChange;
+	psPowerDevice->hSysData = psDeviceNode->psDevConfig->hSysData;
+	psPowerDevice->psDevNode = psDevNode;
+	PVRSRVSetDeviceCurrentPowerState(psPowerDevice, eCurrentPowerState);
+	psPowerDevice->eDefaultPowerState = eDefaultPowerState;
+	psPowerDevice->iIdleReqInFlight = 0;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	OSCachedMemSet(&psPowerDevice->sPowerStats, 0, sizeof(psPowerDevice->sPowerStats));
+#endif
+
+	psDeviceNode->psPowerDev = psPowerDevice;
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVRemovePowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	if (psDeviceNode->psPowerDev)
+	{
+		/* RGX device/firmware should now be powered down */
+		PVR_ASSERT(PVRSRVIsDevicePowered(psDeviceNode) == IMG_FALSE);
+		OSFreeMem(psDeviceNode->psPowerDev);
+		psDeviceNode->psPowerDev = NULL;
+	}
+}
+
+PVRSRV_ERROR PVRSRVGetDevicePowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+									   PPVRSRV_DEV_POWER_STATE pePowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	*pePowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL PVRSRVIsDevicePowered(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_DEV_POWER_STATE ePowerState;
+
+	if (PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState) != PVRSRV_OK)
+	{
+		return IMG_FALSE;
+	}
+
+	return (ePowerState == PVRSRV_DEV_POWER_STATE_ON);
+}
+
+PVRSRV_ERROR PVRSRVGetSystemPowerState(PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                   PPVRSRV_SYS_POWER_STATE peCurrentSysPowerState)
+{
+	if (psDeviceNode != NULL)
+	{
+		if (psDeviceNode->psDevConfig->pfnGpuDomainPower)
+		{
+			*peCurrentSysPowerState = psDeviceNode->psDevConfig->pfnGpuDomainPower(psDeviceNode);
+		}
+		else
+		{
+			*peCurrentSysPowerState = psDeviceNode->eCurrentSysPowerState;
+		}
+	}
+	else
+	{
+		/* assume power is off if no device node */
+		*peCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_OFF;
+		return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL PVRSRVIsSystemPowered(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_SYS_POWER_STATE eSysPowerState;
+
+	if (PVRSRVGetSystemPowerState(psDeviceNode, &eSysPowerState) != PVRSRV_OK)
+	{
+		return IMG_FALSE;
+	}
+
+	return (eSysPowerState == PVRSRV_SYS_POWER_STATE_ON);
+}
+
+PVRSRV_ERROR
+PVRSRVDevicePreClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                IMG_BOOL            bIdleDevice,
+                                void*               pvInfo)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	IMG_UINT64 ui64StartTimer, ui64StopTimer;
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+
+	PVR_UNREFERENCED_PARAMETER(pvInfo);
+
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	ui64StartTimer = PVRSRVProcessStatsGetTimeUs();
+
+	/* This mutex is released in PVRSRVDevicePostClockSpeedChange. */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	if ((eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) && bIdleDevice)
+	{
+		/* We can change the clock speed if the device is either IDLE or OFF */
+		eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_TRUE);
+
+		if (eError != PVRSRV_OK)
+		{
+			/* FW Can signal denied when busy with SPM or other work it can not idle */
+			if (eError != PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Error (%s) from %s()", __func__,
+				         PVRSRVGETERRORSTRING(eError), "PVRSRVDeviceIdleRequestKM"));
+			}
+			if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+			return eError;
+		}
+	}
+
+	eError = psPowerDevice->pfnPreClockSpeedChange(psPowerDevice->psDevNode,
+	                                               eCurrentPowerState);
+
+	ui64StopTimer = PVRSRVProcessStatsGetTimeUs();
+
+	_InsertPowerTimeStatisticExtraPre(psPowerDevice, ui64StartTimer, ui64StopTimer);
+
+	return eError;
+}
+
+void
+PVRSRVDevicePostClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                 IMG_BOOL            bIdleDevice,
+                                 void*               pvInfo)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	IMG_UINT64 ui64StartTimer, ui64StopTimer;
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+
+	PVR_UNREFERENCED_PARAMETER(pvInfo);
+
+	if (psPowerDevice == NULL)
+	{
+		return;
+	}
+
+	ui64StartTimer = PVRSRVProcessStatsGetTimeUs();
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	eError = psPowerDevice->pfnPostClockSpeedChange(psPowerDevice->psDevNode,
+													eCurrentPowerState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p failed (%s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+	}
+
+	if ((eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) && bIdleDevice)
+	{
+		eError = PVRSRVDeviceIdleCancelRequestKM(psDeviceNode);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceIdleCancelRequestKM");
+	}
+
+	/* This mutex was acquired in PVRSRVDevicePreClockSpeedChange. */
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	OSAtomicIncrement(&psDeviceNode->iNumClockSpeedChanges);
+
+	ui64StopTimer = PVRSRVProcessStatsGetTimeUs();
+
+	_InsertPowerTimeStatisticExtraPost(psPowerDevice, ui64StartTimer, ui64StopTimer);
+}
+
+PVRSRV_ERROR PVRSRVDeviceGPUUnitsPowerChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                             IMG_UINT32 ui32NewValue)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_POWER_DEV	*psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice)
+	{
+		PVRSRV_DEV_POWER_STATE eDevicePowerState;
+
+		eError = PVRSRVPowerLock(psDeviceNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+		eDevicePowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+		if (eDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			/* Device must be idle to change GPU unit(s) power state */
+			eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_FALSE);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+				if (eError == PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+				{
+					goto ErrorExit;
+				}
+				goto ErrorUnlockAndExit;
+			}
+		}
+
+		if (psPowerDevice->pfnGPUUnitsPowerChange != NULL)
+		{
+			PVRSRV_ERROR eError2 = psPowerDevice->pfnGPUUnitsPowerChange(psPowerDevice->psDevNode, ui32NewValue);
+
+			if (eError2 != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Device %p failed (%s)",
+				         __func__, psDeviceNode,
+				         PVRSRVGetErrorString(eError2)));
+			}
+		}
+
+		if (eDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			eError = PVRSRVDeviceIdleCancelRequestKM(psDeviceNode);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleCancelRequestKM", ErrorUnlockAndExit);
+		}
+
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return eError;
+
+ErrorUnlockAndExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+ErrorExit:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (power.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/process_stats.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/process_stats.c
new file mode 100644
index 000000000000..77441c578089
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/process_stats.c
@@ -0,0 +1,3492 @@
+/*************************************************************************/ /*!
+@File
+@Title          Process based statistics
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Manages a collection of statistics based around a process
+                and referenced via OS agnostic methods.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "lock.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "process_stats.h"
+#include "ri_server.h"
+#include "hash.h"
+#include "connection_server.h"
+#include "pvrsrv.h"
+#include "proc_stats.h"
+#include "pvr_ricommon.h"
+#include "di_server.h"
+#include "dllist.h"
+#if defined(__linux__)
+#include "trace_events.h"
+#endif
+
+#if defined(SUPPORT_LINUX_FDINFO) && defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "dkp_impl.h"
+
+#define PROC_STATS_FDINFO_ENABLED
+#endif
+
+/* Enabled OS Statistics entries: DEBUGFS on Linux, undefined for other OSs */
+#if defined(__linux__) && ( \
+	defined(PVRSRV_ENABLE_PERPID_STATS) || \
+	defined(PVRSRV_ENABLE_CACHEOP_STATS) || \
+	defined(PVRSRV_ENABLE_MEMORY_STATS) || \
+	defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) )
+#define ENABLE_DEBUGFS_PIDS
+#endif
+
+/* Enable GPU memory accounting tracepoint */
+#if defined(__linux__) && ( \
+	defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) )
+#define ENABLE_GPU_MEM_TRACEPOINT
+#endif
+
+/*
+ * Maximum history of process statistics that will be kept.
+ */
+#define MAX_DEAD_LIST_PROCESSES  (10)
+
+/*
+ * Maximum size of a process name to be displayed in process_stats.
+ */
+#define MAX_PROC_NAME_LEN  (16)
+
+/*
+ * Definition of all the strings used to format process based statistics.
+ */
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+/* Array of Process stat type defined using the X-Macro */
+#define X(stat_type, stat_str, drm_str) stat_str,
+static const IMG_CHAR *const pszProcessStatType[PVRSRV_PROCESS_STAT_TYPE_COUNT] = { PVRSRV_PROCESS_STAT_KEY };
+#undef X
+#define X(stat_type, stat_str) stat_str,
+static const IMG_CHAR *const pszDeviceStatType[PVRSRV_DEVICE_STAT_TYPE_COUNT] = { PVRSRV_DEVICE_STAT_KEY };
+#undef X
+#endif
+
+/* Array of Driver stat type defined using the X-Macro */
+#define X(stat_type, stat_str) stat_str,
+static const IMG_CHAR *const pszDriverStatType[PVRSRV_DRIVER_STAT_TYPE_COUNT] = { PVRSRV_DRIVER_STAT_KEY };
+#undef X
+
+/* structure used in hash table to track statistic entries */
+typedef struct {
+	size_t	   uiSizeInBytes;
+	IMG_PID	   uiPid;
+} _PVR_STATS_TRACKING_HASH_ENTRY;
+
+/* Function used internally to decrement tracked per-process statistic entries */
+static void _StatsDecrMemTrackedStat(_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry,
+                                     PVRSRV_MEM_ALLOC_TYPE eAllocType);
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+int RawProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+#endif
+int GlobalStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/* Note: all of the accesses to the global stats should be protected
+ * by the gsGlobalStats.hGlobalStatsLock lock. This means all of the
+ * invocations of macros *_GLOBAL_STAT_VALUE. */
+
+/* Macros for fetching stat values */
+#define GET_STAT_VALUE(ptr,var) (ptr)->i64StatValue[(var)]
+#define GET_GLOBAL_STAT_VALUE(idx) gsGlobalStats.ui64StatValue[idx]
+
+#define GET_GPUMEM_GLOBAL_STAT_VALUE() \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT)
+
+#define GET_GPUMEM_PERPID_STAT_VALUE(ptr) \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT)
+/*
+ * Macros for updating stat values.
+ */
+#define UPDATE_MAX_VALUE(a,b)					do { if ((b) > (a)) {(a) = (b);} } while (0)
+#define INCREASE_STAT_VALUE(ptr,var,val)		do { (ptr)->i64StatValue[(var)] += (IMG_INT64)(val); if ((ptr)->i64StatValue[(var)] > (ptr)->i64StatValue[(var##_MAX)]) {(ptr)->i64StatValue[(var##_MAX)] = (ptr)->i64StatValue[(var)];} } while (0)
+#define INCREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { (var).ui64StatValue[(idx)] += (IMG_UINT64)(val); if ((var).ui64StatValue[(idx)] > (var).ui64StatValue[(idx##_MAX)]) {(var).ui64StatValue[(idx##_MAX)] = (var).ui64StatValue[(idx)];} } while (0)
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+/* Allow stats to go negative */
+#define DECREASE_STAT_VALUE(ptr,var,val)		do { (ptr)->i64StatValue[(var)] -= (val); } while (0)
+#define DECREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { (var).ui64StatValue[(idx)] -= (val); } while (0)
+#else
+#define DECREASE_STAT_VALUE(ptr,var,val)		do { if ((ptr)->i64StatValue[(var)] >= (val)) { (ptr)->i64StatValue[(var)] -= (IMG_INT64)(val); } else { (ptr)->i64StatValue[(var)] = 0; } } while (0)
+#define DECREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { if ((var).ui64StatValue[(idx)] >= (val)) { (var).ui64StatValue[(idx)] -= (IMG_UINT64)(val); } else { (var).ui64StatValue[(idx)] = 0; } } while (0)
+#endif
+#define MAX_CACHEOP_STAT 16
+#define INCREMENT_CACHEOP_STAT_IDX_WRAP(x) ((x+1) >= MAX_CACHEOP_STAT ? 0 : (x+1))
+#define DECREMENT_CACHEOP_STAT_IDX_WRAP(x) ((x-1) < 0 ? (MAX_CACHEOP_STAT-1) : (x-1))
+
+/*
+ * Track the search of one process when PVRSRV_DEBUG_LINUX_MEMORY_STATS
+ * is enabled.
+ */
+typedef enum _PVRSRV_PROC_SEARCH_STATE_
+{
+	PVRSRV_PROC_NOTFOUND,
+	PVRSRV_PROC_FOUND,
+	PVRSRV_PROC_RESURRECTED,
+} PVRSRV_PROC_SEARCH_STATE;
+
+/*
+ * Structures for holding statistics...
+ */
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+typedef struct _PVRSRV_MEM_ALLOC_REC_
+{
+	PVRSRV_MEM_ALLOC_TYPE           eAllocType;
+	void*                           pvCpuVAddr;
+	IMG_CPU_PHYADDR	                sCpuPAddr;
+	size_t                          uiBytes;
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	void*                           pvAllocdFromFile;
+	IMG_UINT32                      ui32AllocdFromLine;
+#endif
+} PVRSRV_MEM_ALLOC_REC;
+
+typedef struct PVRSRV_MEM_ALLOC_PRINT_DATA_TAG
+{
+	OSDI_IMPL_ENTRY *psEntry;
+	IMG_PID	        pid;
+	IMG_UINT32      ui32NumEntries;
+} PVRSRV_MEM_ALLOC_PRINT_DATA;
+#endif
+
+typedef struct _PVRSRV_PROCESS_STATS_ {
+
+	/* Linked list pointers */
+	DLLIST_NODE                    sNode;
+
+	/* Create per process lock that need to be held
+	 * to edit of its members */
+	POS_LOCK                       hLock;
+
+	/* OS level process ID */
+	IMG_PID	                       pid;
+	IMG_CHAR                       processName[MAX_PROC_NAME_LEN];
+	IMG_UINT32                     ui32RefCount;
+
+	/* Process memory stats */
+	IMG_INT64                      i64StatValue[PVRSRV_PROCESS_STAT_TYPE_COUNT];
+	IMG_UINT32                     ui32StatAllocFlags;
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	struct _CACHEOP_STRUCT_ {
+		PVRSRV_CACHE_OP        uiCacheOp;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+		IMG_DEV_VIRTADDR       sDevVAddr;
+		IMG_DEV_PHYADDR        sDevPAddr;
+#endif
+		IMG_DEVMEM_SIZE_T      uiOffset;
+		IMG_DEVMEM_SIZE_T      uiSize;
+		IMG_UINT64             ui64ExecuteTime;
+		IMG_BOOL               bUserModeFlush;
+		IMG_BOOL               bIsFence;
+		IMG_PID                ownerPid;
+	}                              asCacheOp[MAX_CACHEOP_STAT];
+	IMG_INT32                      uiCacheOpWriteIndex;
+#endif
+
+	/* Other statistics structures */
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	HASH_TABLE* psMemoryRecords;
+#endif
+	/* Device stats */
+	IMG_UINT32                     ui32DevCount;
+	IMG_INT32                      ai32DevStats[][PVRSRV_DEVICE_STAT_TYPE_COUNT];
+} PVRSRV_PROCESS_STATS;
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+static PVRDKF_DKP_HANDLE ghDKPHandle;
+#endif
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+
+typedef struct _PVRSRV_OS_STAT_ENTRY_
+{
+	DI_GROUP *psStatsDIGroup;
+	DI_ENTRY *psProcessStatsDIEntry;
+	DI_ENTRY *psMemStatsDIEntry;
+	DI_ENTRY *psRIMemStatsDIEntry;
+	DI_ENTRY *psCacheOpStatsDIEntry;
+} PVRSRV_OS_STAT_ENTRY;
+
+static PVRSRV_OS_STAT_ENTRY gsLiveStatEntries;
+static PVRSRV_OS_STAT_ENTRY gsRetiredStatEntries;
+
+int GenericStatsPrintElementsLive(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+int GenericStatsPrintElementsRetired(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*
+ * Functions for printing the information stored...
+ */
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+void ProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                               PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+void MemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                           PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+void RIMemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                             PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void CacheOpStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                               PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+typedef void (PVRSRV_STATS_PRINT_ELEMENTS)(OSDI_IMPL_ENTRY *psEntry,
+                                           PVRSRV_PROCESS_STATS *psProcessStats);
+
+typedef enum
+{
+	PVRSRV_STAT_TYPE_PROCESS,
+	PVRSRV_STAT_TYPE_MEMORY,
+	PVRSRV_STAT_TYPE_RIMEMORY,
+	PVRSRV_STAT_TYPE_CACHEOP,
+	PVRSRV_STAT_TYPE_LAST
+} PVRSRV_STAT_TYPE;
+
+#define SEPARATOR_STR_LEN 166
+
+typedef struct _PVRSRV_STAT_PV_DATA_ {
+
+	PVRSRV_STAT_TYPE eStatType;
+	PVRSRV_STATS_PRINT_ELEMENTS* pfnStatsPrintElements;
+	IMG_CHAR szLiveStatsHeaderStr[SEPARATOR_STR_LEN + 1];
+	IMG_CHAR szRetiredStatsHeaderStr[SEPARATOR_STR_LEN + 1];
+
+} PVRSRV_STAT_PV_DATA;
+
+static PVRSRV_STAT_PV_DATA g_StatPvDataArr[] = {
+						{ PVRSRV_STAT_TYPE_PROCESS,  NULL, " Process"               , " Process"               },
+						{ PVRSRV_STAT_TYPE_MEMORY,   NULL, " Memory Allocation"     , " Memory Allocation"     },
+						{ PVRSRV_STAT_TYPE_RIMEMORY, NULL, " Resource Allocation"   , " Resource Allocation"   },
+						{ PVRSRV_STAT_TYPE_CACHEOP,  NULL, " Cache Maintenance Ops" , " Cache Maintenance Ops" }
+					      };
+
+#define GET_STAT_ENTRY_ID(STAT_TYPE) &g_StatPvDataArr[(STAT_TYPE)]
+
+/* Generic header strings */
+static const IMG_CHAR g_szLiveHeaderStr[]    = " Statistics for LIVE Processes ";
+static const IMG_CHAR g_szRetiredHeaderStr[] = " Statistics for RETIRED Processes ";
+
+/* Separator string used for separating stats for different PIDs */
+static IMG_CHAR g_szSeparatorStr[SEPARATOR_STR_LEN + 1] = "";
+
+static inline void
+_prepareStatsHeaderString(IMG_CHAR *pszStatsSpecificStr, const IMG_CHAR* pszGenericHeaderStr)
+{
+	IMG_UINT32 ui32NumSeparators;
+	IMG_CHAR szStatsHeaderFooterStr[75];
+
+	/* Prepare text content of the header in a local string */
+	OSStringSafeCopy(szStatsHeaderFooterStr, pszStatsSpecificStr, ARRAY_SIZE(szStatsHeaderFooterStr));
+	OSStringLCat(szStatsHeaderFooterStr, pszGenericHeaderStr, ARRAY_SIZE(szStatsHeaderFooterStr));
+
+	/* Write all '-' characters to the header string */
+	memset(pszStatsSpecificStr, '-', SEPARATOR_STR_LEN);
+	pszStatsSpecificStr[SEPARATOR_STR_LEN] = '\0';
+
+	/* Find the spot for text content in the header string */
+	ui32NumSeparators = (SEPARATOR_STR_LEN - OSStringLength(szStatsHeaderFooterStr)) >> 1;
+
+	/* Finally write the text content */
+	OSSNPrintf(pszStatsSpecificStr + ui32NumSeparators,
+		   OSStringLength(szStatsHeaderFooterStr),
+		   "%s", szStatsHeaderFooterStr);
+
+	/* Overwrite the '\0' character added by OSSNPrintf() */
+	if (OSStringLength(szStatsHeaderFooterStr) > 0)
+	{
+		pszStatsSpecificStr[ui32NumSeparators + OSStringLength(szStatsHeaderFooterStr) - 1] = ' ';
+	}
+}
+
+static inline void
+_prepareSeparatorStrings(void)
+{
+	IMG_UINT32 i;
+
+	/* Prepare header strings for each stat type */
+	for (i = 0; i < PVRSRV_STAT_TYPE_LAST; ++i)
+	{
+		_prepareStatsHeaderString(g_StatPvDataArr[i].szLiveStatsHeaderStr, g_szLiveHeaderStr);
+		_prepareStatsHeaderString(g_StatPvDataArr[i].szRetiredStatsHeaderStr, g_szRetiredHeaderStr);
+	}
+
+	/* Prepare separator string to separate stats for different PIDs */
+	memset(g_szSeparatorStr, '-', SEPARATOR_STR_LEN);
+	g_szSeparatorStr[SEPARATOR_STR_LEN] = '\0';
+}
+
+static inline void
+_prepareStatsPrivateData(void)
+{
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_PROCESS].pfnStatsPrintElements = ProcessStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_MEMORY].pfnStatsPrintElements = MemStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_RIMEMORY].pfnStatsPrintElements = RIMemStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_CACHEOP].pfnStatsPrintElements = CacheOpStatsPrintElements;
+#endif
+
+	_prepareSeparatorStrings();
+}
+
+#endif
+
+/*
+ * Global Boolean to flag when the statistics are ready to monitor
+ * memory allocations.
+ */
+static IMG_BOOL bProcessStatsInitialised = IMG_FALSE;
+
+/*
+ * Linked lists for process stats. Live stats are for processes which are still running
+ * and the dead list holds those that have exited.
+ */
+static DLLIST_NODE gsLiveList;
+static DLLIST_NODE gsDeadList;
+
+static POS_LOCK g_psLinkedListLock;
+/* Lockdep feature in the kernel cannot differentiate between different instances of same lock type.
+ * This allows it to group all such instances of the same lock type under one class
+ * The consequence of this is that, if lock acquisition is nested on different instances, it generates
+ * a false warning message about the possible occurrence of deadlock due to recursive lock acquisition.
+ * Hence we create the following sub classes to explicitly appraise Lockdep of such safe lock nesting */
+#define PROCESS_LOCK_SUBCLASS_CURRENT	1
+#if defined(ENABLE_DEBUGFS_PIDS)
+/*
+ * Pointer to OS folder to hold PID folders.
+ */
+static DI_GROUP *psProcStatsDIGroup;
+#endif
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+static DI_ENTRY *psProcStatsDIEntry;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+/* Global driver PID stats registration handle */
+static IMG_HANDLE g_hDriverProcessStats;
+#endif
+
+/* Global driver-data folders */
+typedef struct _GLOBAL_STATS_
+{
+	IMG_UINT64 ui64StatValue[PVRSRV_DRIVER_STAT_TYPE_COUNT];
+	POS_LOCK   hGlobalStatsLock;
+} GLOBAL_STATS;
+
+static DI_ENTRY *psGlobalMemDIEntry;
+static GLOBAL_STATS gsGlobalStats;
+
+#define HASH_INITIAL_SIZE 5
+/* A hash table used to store the size of any vmalloc'd allocation
+ * against its address (not needed for kmallocs as we can use ksize()) */
+static HASH_TABLE* gpsSizeTrackingHashTable;
+static POS_LOCK	 gpsSizeTrackingHashTableLock;
+
+static PVRSRV_ERROR _RegisterProcess(IMG_HANDLE *phProcessStats, IMG_PID ownerPid);
+
+static void _DestroyProcessStat(PVRSRV_PROCESS_STATS* psProcessStats);
+
+static void _DecreaseProcStatValueUnlocked(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                                           PVRSRV_PROCESS_STATS* psProcessStats,
+                                           IMG_UINT64 uiBytes);
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStatsInLiveList
+@Description    Searches the Live Process List for a statistics structure that
+                matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStatsInLiveList(IMG_PID pid)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats;
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+
+		if (psProcessStats->pid == pid)
+		{
+			return psProcessStats;
+		}
+	}
+	return NULL;
+} /* _FindProcessStatsInLiveList */
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStatsInDeadList
+@Description    Searches the Dead Process List for a statistics structure that
+                matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStatsInDeadList(IMG_PID pid)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats;
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+
+		if (psProcessStats->pid == pid)
+		{
+			return psProcessStats;
+		}
+	}
+	return NULL;
+} /* _FindProcessStatsInDeadList */
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStats
+@Description    Searches the Live and Dead Process Lists for a statistics
+                structure that matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStats(IMG_PID pid)
+{
+	PVRSRV_PROCESS_STATS* psProcessStats = _FindProcessStatsInLiveList(pid);
+
+	if (psProcessStats == NULL)
+	{
+		psProcessStats = _FindProcessStatsInDeadList(pid);
+	}
+
+	return psProcessStats;
+} /* _FindProcessStats */
+
+/*************************************************************************/ /*!
+@Function       _CompressMemoryUsage
+@Description    Reduces memory usage by deleting old statistics data.
+                This function requires that the list lock is not held!
+*/ /**************************************************************************/
+static void
+_CompressMemoryUsage(void)
+{
+	IMG_INT32 i32ItemsRemaining;
+	DLLIST_NODE *psNode, *psNext;
+	DLLIST_NODE sToBeFreedHead;
+
+	/*
+	 * We hold the lock whilst checking the list, but we'll release it
+	 * before freeing memory (as that will require the lock too)!
+	 */
+	OSLockAcquire(g_psLinkedListLock);
+
+	/* Check that the dead list is not bigger than the max size... */
+	i32ItemsRemaining = MAX_DEAD_LIST_PROCESSES;
+
+	dllist_init(&sToBeFreedHead);
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		i32ItemsRemaining--;
+		if (i32ItemsRemaining < 0)
+		{
+			/* This is the last allowed process, cut the linked list here! */
+			dllist_remove_node(psNode);
+			dllist_add_to_tail(&sToBeFreedHead, psNode);
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	dllist_foreach_node(&sToBeFreedHead, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS *psProcessStatsToBeFreed;
+		psProcessStatsToBeFreed = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		RIDeleteEntriesForPID(psProcessStatsToBeFreed->pid);
+#endif
+		_DestroyProcessStat(psProcessStatsToBeFreed);
+	}
+} /* _CompressMemoryUsage */
+
+/* These functions move the process stats from the live to the dead list.
+ * _MoveProcessToDeadList moves the entry in the global lists and
+ * it needs to be protected by g_psLinkedListLock.
+ * _MoveProcessToDeadList performs the OS calls and it
+ * shouldn't be used under g_psLinkedListLock because this could generate a
+ * lockdep warning. */
+static void
+_MoveProcessToDeadList(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	/* Take the element out of the live list and append to the dead list... */
+	PVR_ASSERT(psProcessStats != NULL);
+	dllist_remove_node(&psProcessStats->sNode);
+	dllist_add_to_head(&gsDeadList, &psProcessStats->sNode);
+} /* _MoveProcessToDeadList */
+
+/* These functions move the process stats from the dead to the live list.
+ * _MoveProcessToLiveList moves the entry in the global lists and
+ * it needs to be protected by g_psLinkedListLock.
+ * _MoveProcessToLiveList performs the OS calls and it
+ * shouldn't be used under g_psLinkedListLock because this could generate a
+ * lockdep warning. */
+static void
+_MoveProcessToLiveList(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	/* Take the element out of the live list and append to the dead list... */
+	PVR_ASSERT(psProcessStats != NULL);
+	dllist_remove_node(&psProcessStats->sNode);
+	dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+} /* _MoveProcessToLiveList */
+
+static PVRSRV_ERROR
+_AllocateProcessStats(PVRSRV_PROCESS_STATS **ppsProcessStats, IMG_PID ownerPid)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PROCESS_STATS *psProcessStats;
+	PVRSRV_DATA	*psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DevCount = 0;
+
+	if (psPVRSRVData != NULL)
+	{
+		ui32DevCount = psPVRSRVData->ui32RegisteredDevices;
+	}
+
+	psProcessStats = OSAllocZMemNoStats(sizeof(PVRSRV_PROCESS_STATS) +
+	                                    ui32DevCount * PVRSRV_DEVICE_STAT_TYPE_COUNT * sizeof(IMG_INT32));
+	PVR_RETURN_IF_NOMEM(psProcessStats);
+
+	psProcessStats->pid             = ownerPid;
+	psProcessStats->ui32RefCount    = 1;
+	psProcessStats->ui32DevCount    = ui32DevCount;
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	psProcessStats->psMemoryRecords = HASH_Create(HASH_INITIAL_SIZE);
+	PVR_GOTO_IF_NOMEM(psProcessStats->psMemoryRecords, eError, free_process_stats);
+#endif
+
+	eError = OSLockCreateNoStats(&psProcessStats->hLock);
+	PVR_GOTO_IF_ERROR(eError, destroy_mem_recs);
+
+	*ppsProcessStats = psProcessStats;
+	return PVRSRV_OK;
+
+destroy_mem_recs:
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	HASH_Delete(psProcessStats->psMemoryRecords);
+free_process_stats:
+#endif
+	OSFreeMemNoStats(psProcessStats);
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+}
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static PVRSRV_ERROR _FreeMemStatsEntry(uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	PVRSRV_MEM_ALLOC_REC *psRecord = (PVRSRV_MEM_ALLOC_REC *)(uintptr_t)v;
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	PVR_DPF((PVR_DBG_WARNING, "Mem Stats Record not freed: 0x%" IMG_UINT64_FMTSPECx " %p, size="IMG_SIZE_FMTSPEC", %s:%d",
+			 (IMG_UINT64)(k), psRecord, psRecord->uiBytes,
+			 (IMG_CHAR*)psRecord->pvAllocdFromFile, psRecord->ui32AllocdFromLine));
+#else
+	PVR_UNREFERENCED_PARAMETER(k);
+#endif
+	OSFreeMemNoStats(psRecord);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _DestroyProcessStat
+@Description    Frees memory and resources held by a process statistic.
+@Input          psProcessStats  Process stats to destroy.
+*/ /**************************************************************************/
+static void
+_DestroyProcessStat(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	PVR_ASSERT(psProcessStats != NULL);
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+	/* Free the memory statistics... */
+	HASH_Iterate(psProcessStats->psMemoryRecords, (HASH_pfnCallback)_FreeMemStatsEntry, NULL);
+	HASH_Delete(psProcessStats->psMemoryRecords);
+
+	OSLockRelease(psProcessStats->hLock);
+#endif
+
+	/*Destroy the lock */
+	OSLockDestroyNoStats(psProcessStats->hLock);
+
+	/* Free the memory... */
+	OSFreeMemNoStats(psProcessStats);
+} /* _DestroyProcessStat */
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+static inline void
+_createStatsFiles(PVRSRV_OS_STAT_ENTRY* psStatsEntries,
+                  DI_PFN_SHOW pfnStatsShow)
+{
+	PVRSRV_ERROR eError;
+	DI_ITERATOR_CB sIterator = {.pfnShow = pfnStatsShow};
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	eError = DICreateEntry("process_stats", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_PROCESS),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psProcessStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (1)");
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	eError = DICreateEntry("cache_ops_exec", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_CACHEOP),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psCacheOpStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (2)");
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	eError = DICreateEntry("mem_area", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_MEMORY),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psMemStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (3)");
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = DICreateEntry("gpu_mem_area", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_RIMEMORY),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psRIMemStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (4)");
+#endif
+}
+
+static inline void
+_createStatisticsEntries(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DICreateGroup("proc_stats", NULL, &psProcStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (1)");
+	eError = DICreateGroup("live_pids_stats", psProcStatsDIGroup,
+                           &gsLiveStatEntries.psStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (2)");
+	eError = DICreateGroup("retired_pids_stats", psProcStatsDIGroup,
+                           &gsRetiredStatEntries.psStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (3)");
+
+	_createStatsFiles(&gsLiveStatEntries, GenericStatsPrintElementsLive);
+	_createStatsFiles(&gsRetiredStatEntries, GenericStatsPrintElementsRetired);
+
+	_prepareStatsPrivateData();
+}
+
+static inline void
+_removeStatsFiles(PVRSRV_OS_STAT_ENTRY* psStatsEntries)
+{
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	DIDestroyEntry(psStatsEntries->psProcessStatsDIEntry);
+	psStatsEntries->psProcessStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	DIDestroyEntry(psStatsEntries->psCacheOpStatsDIEntry);
+    psStatsEntries->psCacheOpStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	DIDestroyEntry(psStatsEntries->psMemStatsDIEntry);
+	psStatsEntries->psMemStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	DIDestroyEntry(psStatsEntries->psRIMemStatsDIEntry);
+	psStatsEntries->psRIMemStatsDIEntry = NULL;
+#endif
+}
+
+static inline void
+_removeStatisticsEntries(void)
+{
+	_removeStatsFiles(&gsLiveStatEntries);
+	_removeStatsFiles(&gsRetiredStatEntries);
+
+	DIDestroyGroup(gsLiveStatEntries.psStatsDIGroup);
+	gsLiveStatEntries.psStatsDIGroup = NULL;
+	DIDestroyGroup(gsRetiredStatEntries.psStatsDIGroup);
+	gsRetiredStatEntries.psStatsDIGroup = NULL;
+	DIDestroyGroup(psProcStatsDIGroup);
+	psProcStatsDIGroup = NULL;
+}
+#endif
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+
+#define GENERATE_DKP_PRINT(_handle, _key, _value)                   \
+	PVRDKPOutput((_handle),                                         \
+	             "%s:%-*s%"IMG_UINT64_FMTSPEC"\n",                  \
+	             (_key),                                            \
+	             (int)(50 - sizeof((_key))),                        \
+	              "",                                               \
+	             (IMG_UINT64)(_value))
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#define GENERATE_PROCESS_STAT_FDINFO(_dkp_handle, _stat_type, _key, _process_stats) \
+	do { \
+		IMG_UINT32 ui32Value = MAX(0, (_process_stats)->i64StatValue[(_stat_type)]); \
+		if (((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) || \
+			((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES)) \
+		{ \
+			ui32Value = RITotalAllocProcessUnlocked((_process_stats)->pid, \
+			                                        ((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) ? PHYS_HEAP_TYPE_LMA : PHYS_HEAP_TYPE_UMA); \
+		} \
+		GENERATE_DKP_PRINT((_dkp_handle), (_key), ui32Value); \
+	} while (0)
+#else
+#define GENERATE_PROCESS_STAT_FDINFO(_dkp_handle, _stat_type, _key, _process_stats) \
+	GENERATE_DKP_PRINT((_dkp_handle), (_key), MAX(0, (_process_stats)->i64StatValue[(_stat_type)]));
+#endif
+
+static void _ProcessStatsDKPShow(PVRSRV_DEVICE_NODE *psDevNode,
+                                 int pid,
+                                 IMG_HANDLE hPrivData)
+{
+	PVRSRV_PROCESS_STATS *psProcessStats = _FindProcessStatsInLiveList((IMG_PID)pid);
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	if (psProcessStats == NULL)
+	{
+		/* Just return if the specified process is no longer in the process
+		 * stats list.
+		 */
+		return;
+	}
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-memory-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_MEMORY
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-shared-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_SHARED
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-total-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_TOTAL
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-resident-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_RESIDENT
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-purgeable-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_PURGEABLE
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-active-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_ACTIVE
+	#undef X
+}
+
+#undef GENERATE_PROCESS_STAT_FDINFO
+#undef GENERATE_DKP_PRINT
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsInitialise
+@Description    Entry point for initialising the statistics module.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVStatsInitialise(void)
+{
+	PVRSRV_ERROR error;
+
+	PVR_ASSERT(g_psLinkedListLock == NULL);
+	PVR_ASSERT(gpsSizeTrackingHashTable == NULL);
+	PVR_ASSERT(bProcessStatsInitialised == IMG_FALSE);
+
+	/* We need a lock to protect the linked lists... */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&g_psLinkedListLock);
+#else
+	error = OSLockCreate(&g_psLinkedListLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, return_);
+
+	/* We also need a lock to protect the hash table used for size tracking. */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&gpsSizeTrackingHashTableLock);
+#else
+	error = OSLockCreate(&gpsSizeTrackingHashTableLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, destroy_linked_list_lock_);
+
+	/* We also need a lock to protect the GlobalStat counters */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&gsGlobalStats.hGlobalStatsLock);
+#else
+	error = OSLockCreate(&gsGlobalStats.hGlobalStatsLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, destroy_hashtable_lock_);
+
+	/* Flag that we are ready to start monitoring memory allocations. */
+
+	gpsSizeTrackingHashTable = HASH_Create(HASH_INITIAL_SIZE);
+	PVR_GOTO_IF_NOMEM(gpsSizeTrackingHashTable, error, destroy_stats_lock_);
+
+	dllist_init(&gsLiveList);
+	dllist_init(&gsDeadList);
+
+	bProcessStatsInitialised = IMG_TRUE;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Register our 'system' PID to hold driver-wide alloc stats */
+	_RegisterProcess(&g_hDriverProcessStats, PVR_SYS_ALLOC_PID);
+#endif
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+	_createStatisticsEntries();
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = RawProcessStatsPrintElements};
+		error = DICreateEntry("memtrack_stats", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &psProcStatsDIEntry);
+		PVR_LOG_IF_ERROR(error, "DICreateEntry (1)");
+	}
+#endif
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = GlobalStatsPrintElements};
+		error = DICreateEntry("driver_stats", NULL, &sIterator, NULL,
+		                      DI_ENTRY_TYPE_GENERIC, &psGlobalMemDIEntry);
+		PVR_LOG_IF_ERROR(error, "DICreateEntry (3)");
+	}
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+	/* We might error here, but drm usage keys related to memory are optional.
+	 * We'll notify that the DKP registration has failed but continue as if nothing
+	 * happened. */
+	{
+		error = PVRSRVRegisterDKP(NULL,
+		                          "proc-stats",
+		                          &_ProcessStatsDKPShow,
+		                          DKP_CONNECTION_FLAG_SERVICES,
+		                          &ghDKPHandle);
+		if (error != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Failed to register process stats DKP", __func__));
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+destroy_stats_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(gsGlobalStats.hGlobalStatsLock);
+#else
+	OSLockDestroy(gsGlobalStats.hGlobalStatsLock);
+#endif
+	gsGlobalStats.hGlobalStatsLock = NULL;
+destroy_hashtable_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(gpsSizeTrackingHashTableLock);
+#else
+	OSLockDestroy(gpsSizeTrackingHashTableLock);
+#endif
+	gpsSizeTrackingHashTableLock = NULL;
+destroy_linked_list_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(g_psLinkedListLock);
+#else
+	OSLockDestroy(g_psLinkedListLock);
+#endif
+	g_psLinkedListLock = NULL;
+return_:
+	return error;
+
+}
+
+static PVRSRV_ERROR _DumpAllVMallocEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+#if defined(PVRSRV_NEED_PVR_DPF) || defined(DOXYGEN)
+	_PVR_STATS_TRACKING_HASH_ENTRY *psNewTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)(uintptr_t)v;
+	IMG_UINT64 uiCpuVAddr = (IMG_UINT64)k;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: " IMG_SIZE_FMTSPEC " bytes @ 0x%" IMG_UINT64_FMTSPECx " (PID %u)", __func__,
+	         psNewTrackingHashEntry->uiSizeInBytes,
+	         uiCpuVAddr,
+	         psNewTrackingHashEntry->uiPid));
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsDestroy
+@Description    Method for destroying the statistics module data.
+*/ /**************************************************************************/
+void
+PVRSRVStatsDestroy(void)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_ASSERT(bProcessStatsInitialised);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Deregister our 'system' PID which holds driver-wide alloc stats */
+	PVRSRVStatsDeregisterProcess(g_hDriverProcessStats);
+#endif
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+	if (ghDKPHandle != NULL)
+	{
+		PVRSRV_ERROR eError = PVRSRVUnRegisterDKP(NULL, ghDKPHandle);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to unregister process stats DKP", __func__));
+		}
+	}
+#endif
+
+	/* Stop monitoring memory allocations... */
+	bProcessStatsInitialised = IMG_FALSE;
+
+	/* Destroy the locks... */
+	if (g_psLinkedListLock != NULL)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(g_psLinkedListLock);
+#else
+		OSLockDestroy(g_psLinkedListLock);
+#endif
+		g_psLinkedListLock = NULL;
+	}
+
+	/* Free the live and dead lists... */
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		dllist_remove_node(&psProcessStats->sNode);
+		_DestroyProcessStat(psProcessStats);
+	}
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		dllist_remove_node(&psProcessStats->sNode);
+		_DestroyProcessStat(psProcessStats);
+	}
+
+	if (gpsSizeTrackingHashTable != NULL)
+	{
+		/* Dump all remaining entries in HASH table (list any remaining vmallocs) */
+		HASH_Iterate(gpsSizeTrackingHashTable, (HASH_pfnCallback)_DumpAllVMallocEntries, NULL);
+		HASH_Delete(gpsSizeTrackingHashTable);
+	}
+	if (gpsSizeTrackingHashTableLock != NULL)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(gpsSizeTrackingHashTableLock);
+#else
+		OSLockDestroy(gpsSizeTrackingHashTableLock);
+#endif
+		gpsSizeTrackingHashTableLock = NULL;
+	}
+
+	if (NULL != gsGlobalStats.hGlobalStatsLock)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(gsGlobalStats.hGlobalStatsLock);
+#else
+		OSLockDestroy(gsGlobalStats.hGlobalStatsLock);
+#endif
+		gsGlobalStats.hGlobalStatsLock = NULL;
+	}
+
+}
+
+void
+PVRSRVStatsDestroyDI(void)
+{
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+	if (psProcStatsDIEntry != NULL)
+	{
+		DIDestroyEntry(psProcStatsDIEntry);
+		psProcStatsDIEntry = NULL;
+	}
+#endif
+
+	/* Destroy the global data entry */
+	if (psGlobalMemDIEntry != NULL)
+	{
+		DIDestroyEntry(psGlobalMemDIEntry);
+		psGlobalMemDIEntry = NULL;
+	}
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+	_removeStatisticsEntries();
+#endif
+}
+
+static void _decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize;
+#endif
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	ui64InitialSize = GET_GPUMEM_GLOBAL_STAT_VALUE();
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_KMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			break;
+#endif
+
+		default:
+			PVR_ASSERT(0);
+			break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_GLOBAL_STAT_VALUE();
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemGlobal(0, ui64Size);
+		}
+	}
+#endif
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+}
+
+static void _increase_global_stat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize;
+#endif
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	ui64InitialSize = GET_GPUMEM_GLOBAL_STAT_VALUE();
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_KMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			break;
+#endif
+
+		default:
+			PVR_ASSERT(0);
+			break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_GLOBAL_STAT_VALUE();
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemGlobal(0, ui64Size);
+		}
+	}
+#endif
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+}
+
+static PVRSRV_ERROR
+_RegisterProcess(IMG_HANDLE *phProcessStats, IMG_PID ownerPid)
+{
+	PVRSRV_PROCESS_STATS*	psProcessStats=NULL;
+	PVRSRV_ERROR			eError;
+
+	PVR_ASSERT(phProcessStats != NULL);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Register process PID %d [%s]",
+			__func__, ownerPid, (ownerPid == PVR_SYS_ALLOC_PID)
+			? "system" : OSGetCurrentClientProcessNameKM()));
+
+	/* Check the PID has not already moved to the dead list... */
+	OSLockAcquire(g_psLinkedListLock);
+	psProcessStats = _FindProcessStatsInDeadList(ownerPid);
+	if (psProcessStats != NULL)
+	{
+		/* Move it back onto the live list! */
+		_MoveProcessToLiveList(psProcessStats);
+	}
+	else
+	{
+		/* Check the PID is not already registered in the live list... */
+		psProcessStats = _FindProcessStatsInLiveList(ownerPid);
+	}
+
+	/* If the PID is on the live list then just increment the ref count and return... */
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		psProcessStats->ui32RefCount++;
+
+		OSLockRelease(psProcessStats->hLock);
+		OSLockRelease(g_psLinkedListLock);
+
+		*phProcessStats = psProcessStats;
+
+		return PVRSRV_OK;
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	/* Allocate a new node structure and initialise it... */
+	eError = _AllocateProcessStats(&psProcessStats, ownerPid);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	OSStringSafeCopy(psProcessStats->processName, OSGetCurrentProcessName(), MAX_PROC_NAME_LEN);
+
+	/* Add it to the live list... */
+	OSLockAcquire(g_psLinkedListLock);
+	dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+	OSLockRelease(g_psLinkedListLock);
+
+	/* Done */
+	*phProcessStats = (IMG_HANDLE) psProcessStats;
+
+	return PVRSRV_OK;
+
+e0:
+	*phProcessStats = (IMG_HANDLE) NULL;
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+} /* _RegisterProcess */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsRegisterProcess
+@Description    Register a process into the list statistics list.
+@Output         phProcessStats  Handle to the process to be used to deregister.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVStatsRegisterProcess(IMG_HANDLE* phProcessStats)
+{
+	return _RegisterProcess(phProcessStats, OSGetCurrentClientProcessIDKM());
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsDeregisterProcess
+@Input          hProcessStats  Handle to the process returned when registered.
+@Description    Method for destroying the statistics module data.
+*/ /**************************************************************************/
+void
+PVRSRVStatsDeregisterProcess(IMG_HANDLE hProcessStats)
+{
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Deregister process entered PID %d [%s]",
+			__func__, OSGetCurrentClientProcessIDKM(),
+			OSGetCurrentProcessName()));
+
+	if (hProcessStats != (IMG_HANDLE) NULL)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = (PVRSRV_PROCESS_STATS*) hProcessStats;
+
+		/* Lower the reference count, if zero then move it to the dead list */
+		OSLockAcquire(g_psLinkedListLock);
+		if (psProcessStats->ui32RefCount > 0)
+		{
+			OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+			psProcessStats->ui32RefCount--;
+
+#if !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+			if (psProcessStats->ui32RefCount == 0)
+			{
+				OSLockRelease(psProcessStats->hLock);
+				_MoveProcessToDeadList(psProcessStats);
+			}else
+#endif
+			{
+				OSLockRelease(psProcessStats->hLock);
+			}
+		}
+		OSLockRelease(g_psLinkedListLock);
+
+		/* Check if the dead list needs to be reduced */
+		_CompressMemoryUsage();
+	}
+} /* PVRSRVStatsDeregisterProcess */
+
+PVRSRV_ERROR PVRSRVStatsDeviceConnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+	IMG_PID ownerPid = OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS*	psProcessStats;
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStatsInLiveList(ownerPid);
+
+	if (psProcessStats != NULL)
+	{
+		if ((ui32DevID < psProcessStats->ui32DevCount) ||
+		    (ui32DevID == 0 && psProcessStats->ui32DevCount == 0))
+		{
+			psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]++;
+			UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_MAX_CONNECTIONS],
+			                 psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device index %d is greater than device count %d for PID %d.",
+			         __func__, ui32DevID, psProcessStats->ui32DevCount, ownerPid));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Process %d not found.",
+				 __func__, ownerPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVStatsDeviceDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+	IMG_PID	currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA* psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_PID currentPid = OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS*	psProcessStats;
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+	}
+
+	if (psProcessStats != NULL)
+	{
+		if ((ui32DevID < psProcessStats->ui32DevCount) ||
+		    (ui32DevID == 0 && psProcessStats->ui32DevCount == 0))
+		{
+			psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]--;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device index %d is greater than device count %d for PID %d.",
+					 __func__, ui32DevID, psProcessStats->ui32DevCount, currentPid));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Process %d not found.",
+				 __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+/* Assumes PVRSRV_PROCESS_STATS lock (psProcessStats->hLock) has already been taken */
+static void
+_IncreaseProcStatValueUnlocked(PVRSRV_PROCESS_STATS* psProcessStats,
+                               PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                               size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+#endif
+
+	/* Update the memory watermarks... */
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_KMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMALLOC, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		default:
+		{
+			PVR_ASSERT(0);
+		}
+		break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemPerProcess(0, psProcessStats->pid,
+			                                 ui64Size);
+		}
+	}
+#endif
+}
+
+/* Assumes PVRSRV_PROCESS_STATS lock (psProcessStats->hLock) has already been taken */
+static void
+_DecreaseProcStatValueUnlocked(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                               PVRSRV_PROCESS_STATS* psProcessStats,
+                               IMG_UINT64 uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_KMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMALLOC, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMALLOC] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		default:
+		{
+			PVR_ASSERT(0);
+		}
+		break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemPerProcess(0, psProcessStats->pid, ui64Size);
+		}
+	}
+#endif
+}
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static INLINE IMG_BOOL
+_CheckKeyPhysAddr(PVRSRV_MEM_ALLOC_TYPE eAllocType)
+{
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+#endif
+			return IMG_TRUE;
+		default:
+			return IMG_FALSE;
+	}
+}
+#endif
+
+void
+PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+							 void *pvCpuVAddr,
+							 IMG_CPU_PHYADDR sCpuPAddr,
+							 size_t uiBytes,
+							 IMG_PID currentPid
+							 DEBUG_MEMSTATS_PARAMS)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_MEM_ALLOC_REC*  psRecord = NULL;
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+	__maybe_unused PVRSRV_PROC_SEARCH_STATE eProcSearch = PVRSRV_PROC_FOUND;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/*
+	 * To prevent a recursive loop, we make the memory allocations for our
+	 * memstat records via OSAllocMemNoStats(), which does not try to
+	 * create a memstat record entry.
+	 */
+
+	/* Allocate the memory record... */
+	psRecord = OSAllocZMemNoStats(sizeof(PVRSRV_MEM_ALLOC_REC));
+	if (psRecord == NULL)
+	{
+		return;
+	}
+
+	psRecord->eAllocType       = eAllocType;
+	psRecord->pvCpuVAddr       = pvCpuVAddr;
+	psRecord->sCpuPAddr.uiAddr = sCpuPAddr.uiAddr;
+	psRecord->uiBytes          = uiBytes;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	psRecord->pvAllocdFromFile = pvAllocFromFile;
+	psRecord->ui32AllocdFromLine = ui32AllocFromLine;
+#endif
+
+	_increase_global_stat(eAllocType, uiBytes);
+	/* Lock while we find the correct process... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+			if (!psProcessStats)
+			{
+				psProcessStats = _FindProcessStatsInDeadList(currentPid);
+				eProcSearch = PVRSRV_PROC_RESURRECTED;
+			}
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		if (!psProcessStats)
+		{
+			psProcessStats = _FindProcessStatsInDeadList(currentPid);
+			eProcSearch = PVRSRV_PROC_RESURRECTED;
+		}
+	}
+
+	if (psProcessStats == NULL)
+	{
+		eProcSearch = PVRSRV_PROC_NOTFOUND;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat increment called for 'unknown' process PID(%d)",
+				 __func__, currentPid));
+
+		if (_AllocateProcessStats(&psProcessStats, currentPid) != PVRSRV_OK)
+		{
+			OSLockRelease(g_psLinkedListLock);
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s UNABLE TO CREATE process_stats entry for pid %d [%s] (" IMG_SIZE_FMTSPEC " bytes)",
+			        __func__, currentPid, OSGetCurrentProcessName(), uiBytes));
+			goto free_record;
+		}
+
+		/* Add it to the live list... */
+		dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+
+		OSLockRelease(g_psLinkedListLock);
+
+#else /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+		OSLockRelease(g_psLinkedListLock);
+		goto free_record;
+#endif /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+	}
+	else
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		if (eProcSearch == PVRSRV_PROC_RESURRECTED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat incremented on 'dead' process PID(%d)",
+				 __func__, currentPid));
+			/* Move process from dead list to live list */
+			_MoveProcessToLiveList(psProcessStats);
+		}
+#endif
+		OSLockRelease(g_psLinkedListLock);
+	}
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_UINT64 ui64Key;
+
+		if (_CheckKeyPhysAddr(eAllocType))
+		{
+			ui64Key = psRecord->sCpuPAddr.uiAddr;
+		}
+		else
+		{
+			ui64Key = (IMG_UINT64)psRecord->pvCpuVAddr;
+		}
+
+		/* Insert the memory record... */
+		if (!HASH_Insert(psProcessStats->psMemoryRecords, ui64Key, (uintptr_t)psRecord))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s UNABLE TO CREATE mem stats record for pid %d [%s] (" IMG_SIZE_FMTSPEC " bytes)",
+					 __func__, currentPid, OSGetCurrentProcessName(), uiBytes));
+		}
+	}
+#endif
+
+	/* Update the memory watermarks... */
+	_IncreaseProcStatValueUnlocked(psProcessStats,
+	                               eAllocType,
+	                               uiBytes);
+
+	OSLockRelease(psProcessStats->hLock);
+
+	return;
+
+free_record:
+	_decrease_global_stat(eAllocType, uiBytes);
+	if (psRecord != NULL)
+	{
+		OSFreeMemNoStats(psRecord);
+	}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	PVR_UNREFERENCED_PARAMETER(eAllocType);
+	PVR_UNREFERENCED_PARAMETER(pvCpuVAddr);
+	PVR_UNREFERENCED_PARAMETER(sCpuPAddr);
+	PVR_UNREFERENCED_PARAMETER(uiBytes);
+	PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+} /* PVRSRVStatsAddMemAllocRecord */
+
+void
+PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								IMG_UINT64 ui64Key,
+								IMG_PID currentPid)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+	PVRSRV_MEM_ALLOC_REC*  psRecord		  = NULL;
+	IMG_BOOL			   bFound	      = IMG_FALSE;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/* Lock while we find the correct process and remove this record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+		OSLockRelease(psProcessStats->hLock);
+		bFound = psRecord != NULL;
+	}
+
+	/* If not found, we need to do a full search in case it was allocated to a different PID... */
+	if (!bFound)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStatsAlreadyChecked = psProcessStats;
+		DLLIST_NODE *psNode, *psNext;
+
+		/* Search all live lists first... */
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			if (psProcessStats != psProcessStatsAlreadyChecked)
+			{
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+				psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+				OSLockRelease(psProcessStats->hLock);
+				bFound = psRecord != NULL;
+			}
+
+			if (bFound)
+			{
+				break;
+			}
+		}
+
+		/* If not found, then search all dead lists next... */
+		if (!bFound)
+		{
+			dllist_foreach_node(&gsDeadList, psNode, psNext)
+			{
+				psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+				if (psProcessStats != psProcessStatsAlreadyChecked)
+				{
+					OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+					psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+					OSLockRelease(psProcessStats->hLock);
+					bFound = psRecord != NULL;
+				}
+
+				if (bFound)
+				{
+					break;
+				}
+			}
+		}
+	}
+
+	/* Update the watermark and remove this record... */
+	if (bFound)
+	{
+		_decrease_global_stat(eAllocType, psRecord->uiBytes);
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               psRecord->uiBytes);
+
+		OSLockRelease(psProcessStats->hLock);
+		OSLockRelease(g_psLinkedListLock);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		/* If all stats are now zero, remove the entry for this thread */
+		if (psProcessStats->ui32StatAllocFlags == 0)
+		{
+			OSLockAcquire(g_psLinkedListLock);
+			_MoveProcessToDeadList(psProcessStats);
+			OSLockRelease(g_psLinkedListLock);
+
+			/* Check if the dead list needs to be reduced */
+			_CompressMemoryUsage();
+		}
+#endif
+		/*
+		 * Free the record outside the lock so we don't deadlock and so we
+		 * reduce the time the lock is held.
+		 */
+		OSFreeMemNoStats(psRecord);
+	}
+	else
+	{
+		OSLockRelease(g_psLinkedListLock);
+	}
+
+#else
+	PVR_UNREFERENCED_PARAMETER(eAllocType);
+	PVR_UNREFERENCED_PARAMETER(ui64Key);
+	PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif
+} /* PVRSRVStatsRemoveMemAllocRecord */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void
+PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eCurrentType,
+                                  PVRSRV_MEM_ALLOC_TYPE eTargetType,
+                                  IMG_UINT64 ui64Key,
+                                  IMG_PID currentPid
+                                  DEBUG_MEMSTATS_PARAMS)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+	PVRSRV_MEM_ALLOC_REC*  psRecord		  = NULL;
+	IMG_BOOL			   bFound	      = IMG_FALSE;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	if (_CheckKeyPhysAddr(eCurrentType) != _CheckKeyPhysAddr(eTargetType))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Key type used for current type must match target type, "
+		                        "Record transfer failed!"));
+		return;
+	}
+
+	/* Lock while we find the correct process and remove this record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+		OSLockRelease(psProcessStats->hLock);
+		bFound = psRecord != NULL;
+	}
+
+	/* If not found, we need to do a full search in case it was allocated to a different PID... */
+	if (!bFound)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStatsAlreadyChecked = psProcessStats;
+		DLLIST_NODE *psNode, *psNext;
+
+		/* Search all live lists first... */
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			if (psProcessStats != psProcessStatsAlreadyChecked)
+			{
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+				psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+				OSLockRelease(psProcessStats->hLock);
+				bFound = psRecord != NULL;
+			}
+
+			if (bFound)
+			{
+				break;
+			}
+		}
+	}
+
+	if (bFound)
+	{
+		/* Update the current type watermark by reducing */
+		_decrease_global_stat(eCurrentType, psRecord->uiBytes);
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		_DecreaseProcStatValueUnlocked(eCurrentType,
+		                               psProcessStats,
+		                               psRecord->uiBytes);
+
+		/* Change the type of the record */
+		psRecord->eAllocType = eTargetType;
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+		/* Adjust where the record transfer originated */
+		psRecord->pvAllocdFromFile = pvAllocFromFile;
+		psRecord->ui32AllocdFromLine = ui32AllocFromLine;
+#endif
+
+		/* Update the target watermark by increasing */
+		_IncreaseProcStatValueUnlocked(psProcessStats,
+		                               eTargetType,
+		                               psRecord->uiBytes);
+		OSLockRelease(psProcessStats->hLock);
+
+		_increase_global_stat(eTargetType, psRecord->uiBytes);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+#else
+PVR_UNREFERENCED_PARAMETER(eCurrentType);
+PVR_UNREFERENCED_PARAMETER(eTargetType);
+PVR_UNREFERENCED_PARAMETER(ui64Key);
+PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+void
+PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									size_t uiBytes,
+									IMG_UINT64 uiCpuVAddr,
+									IMG_PID uiPid)
+{
+	IMG_BOOL bRes = IMG_FALSE;
+	_PVR_STATS_TRACKING_HASH_ENTRY *psNewTrackingHashEntry = NULL;
+
+	if (!bProcessStatsInitialised || (gpsSizeTrackingHashTable == NULL))
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/* Alloc untracked memory for the new hash table entry */
+	psNewTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)OSAllocMemNoStats(sizeof(*psNewTrackingHashEntry));
+	if (psNewTrackingHashEntry == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "*** %s : @ line %d Failed to alloc memory for psNewTrackingHashEntry!",
+				 __func__, __LINE__));
+		return;
+	}
+
+	/* Fill-in the size of the allocation and PID of the allocating process */
+	psNewTrackingHashEntry->uiSizeInBytes = uiBytes;
+	psNewTrackingHashEntry->uiPid = uiPid;
+	OSLockAcquire(gpsSizeTrackingHashTableLock);
+	/* Insert address of the new struct into the hash table */
+	bRes = HASH_Insert(gpsSizeTrackingHashTable, uiCpuVAddr, (uintptr_t)psNewTrackingHashEntry);
+	OSLockRelease(gpsSizeTrackingHashTableLock);
+	if (bRes)
+	{
+		PVRSRVStatsIncrMemAllocStat(eAllocType, uiBytes, uiPid);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "*** %s : @ line %d HASH_Insert() failed!",
+				 __func__, __LINE__));
+		/* Free the memory allocated for psNewTrackingHashEntry, as we
+		 * failed to insert it into the Hash table.
+		 */
+		OSFreeMemNoStats(psNewTrackingHashEntry);
+	}
+}
+
+void
+PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                            size_t uiBytes,
+                            IMG_PID currentPid)
+
+{
+	IMG_PID				  currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		  psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS* psProcessStats = NULL;
+	__maybe_unused PVRSRV_PROC_SEARCH_STATE eProcSearch = PVRSRV_PROC_FOUND;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	_increase_global_stat(eAllocType, uiBytes);
+	OSLockAcquire(g_psLinkedListLock);
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+			if (!psProcessStats)
+			{
+				psProcessStats = _FindProcessStatsInDeadList(currentPid);
+				eProcSearch = PVRSRV_PROC_RESURRECTED;
+			}
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		if (!psProcessStats)
+		{
+			psProcessStats = _FindProcessStatsInDeadList(currentPid);
+			eProcSearch = PVRSRV_PROC_RESURRECTED;
+		}
+	}
+
+	if (psProcessStats == NULL)
+	{
+		eProcSearch = PVRSRV_PROC_NOTFOUND;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat increment called for 'unknown' process PID(%d)",
+				 __func__, currentPid));
+
+		if (bProcessStatsInitialised)
+		{
+			if (_AllocateProcessStats(&psProcessStats, currentPid) != PVRSRV_OK)
+			{
+				OSLockRelease(g_psLinkedListLock);
+				return;
+			}
+			/* Add it to the live list... */
+			dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+		}
+#else
+		OSLockRelease(g_psLinkedListLock);
+#endif /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+
+	}
+
+	if (psProcessStats != NULL)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		if (eProcSearch == PVRSRV_PROC_RESURRECTED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "%s: Process stat incremented on 'dead' process PID(%d)",
+					 __func__, currentPid));
+
+			/* Move process from dead list to live list */
+			_MoveProcessToLiveList(psProcessStats);
+		}
+#endif
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Release the list lock as soon as we acquire the process lock,
+		 * this ensures if the process is in deadlist the entry cannot be
+		 * deleted or modified
+		 */
+		OSLockRelease(g_psLinkedListLock);
+
+		/* Update the memory watermarks... */
+		_IncreaseProcStatValueUnlocked(psProcessStats,
+		                               eAllocType,
+		                               uiBytes);
+
+		OSLockRelease(psProcessStats->hLock);
+	}
+}
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+int RawProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_PROCESS_STATS *psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	DIPrintf(psEntry,
+	         "%s,%s,%s,%s,%s,%s,%s\n",
+	         "PID",
+	         "MemoryUsageKMalloc",           // PVRSRV_PROCESS_STAT_TYPE_KMALLOC
+	         "MemoryUsageAllocPTMemoryUMA",  // PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA
+	         "MemoryUsageAllocPTMemoryLMA",  // PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA
+	         "MemoryUsageAllocGPUMemLMA",    // PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES
+	         "MemoryUsageAllocGPUMemUMA",    // PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES
+	         "MemoryUsageDmaBufImport");     // PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+		{
+			DIPrintf(psEntry,
+			         "%d,%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd","
+			         "%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd","
+			         "%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd"\n",
+			         psProcessStats->pid,
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT]);
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+} /* RawProcessStatsPrintElements */
+#endif
+
+void
+PVRSRVStatsDecrMemKAllocStat(size_t uiBytes,
+                             IMG_PID decrPID)
+{
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE_KMALLOC, uiBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(decrPID);
+
+	if (psProcessStats != NULL)
+	{
+		/* Decrement the kmalloc memory stat... */
+		DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+		DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+static void
+_StatsDecrMemTrackedStat(_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry,
+                        PVRSRV_MEM_ALLOC_TYPE eAllocType)
+{
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(eAllocType, psTrackingHashEntry->uiSizeInBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(psTrackingHashEntry->uiPid);
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Decrement the memory stat... */
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               psTrackingHashEntry->uiSizeInBytes);
+		OSLockRelease(psProcessStats->hLock);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+void
+PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									  IMG_UINT64 uiCpuVAddr)
+{
+	_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry = NULL;
+
+	if (!bProcessStatsInitialised || (gpsSizeTrackingHashTable == NULL))
+	{
+		return;
+	}
+
+	OSLockAcquire(gpsSizeTrackingHashTableLock);
+	psTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)HASH_Remove(gpsSizeTrackingHashTable, uiCpuVAddr);
+	OSLockRelease(gpsSizeTrackingHashTableLock);
+	if (psTrackingHashEntry)
+	{
+		_StatsDecrMemTrackedStat(psTrackingHashEntry, eAllocType);
+		OSFreeMemNoStats(psTrackingHashEntry);
+	}
+}
+
+void
+PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+							size_t uiBytes,
+							IMG_PID currentPid)
+{
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(eAllocType, uiBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Release the list lock as soon as we acquire the process lock,
+		 * this ensures if the process is in deadlist the entry cannot be
+		 * deleted or modified
+		 */
+		OSLockRelease(g_psLinkedListLock);
+		/* Update the memory watermarks... */
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               uiBytes);
+		OSLockRelease(psProcessStats->hLock);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		/* If all stats are now zero, remove the entry for this thread */
+		if (psProcessStats->ui32StatAllocFlags == 0)
+		{
+			OSLockAcquire(g_psLinkedListLock);
+			_MoveProcessToDeadList(psProcessStats);
+			OSLockRelease(g_psLinkedListLock);
+
+			/* Check if the dead list needs to be reduced */
+			_CompressMemoryUsage();
+		}
+#endif
+	}else{
+		OSLockRelease(g_psLinkedListLock);
+	}
+}
+
+/* For now we do not want to expose the global stats API
+ * so we wrap it into this specific function for pooled pages.
+ * As soon as we need to modify the global stats directly somewhere else
+ * we want to replace these functions with more general ones.
+ */
+void
+PVRSRVStatsIncrMemAllocPoolStat(size_t uiBytes)
+{
+	_increase_global_stat(PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES, uiBytes);
+}
+
+void
+PVRSRVStatsDecrMemAllocPoolStat(size_t uiBytes)
+{
+	_decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES, uiBytes);
+}
+
+PVRSRV_ERROR
+PVRSRVStatsUpdateOOMStat(CONNECTION_DATA *psConnection,
+						  PVRSRV_DEVICE_NODE *psDeviceNode,
+						  IMG_UINT32 ui32OOMStatType,
+						  IMG_PID pidOwner)
+{
+	PVRSRV_DEVICE_STAT_TYPE eOOMStatType = (PVRSRV_DEVICE_STAT_TYPE) ui32OOMStatType;
+	IMG_PID	pidCurrent = pidOwner;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32OOMStatType >= PVRSRV_DEVICE_STAT_TYPE_COUNT)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(pidCurrent);
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[psDeviceNode->sDevId.ui32InternalID][eOOMStatType]++;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PVRSRVStatsUpdateOOMStat: Process not found for Pid=%d", pidCurrent));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+} /* PVRSRVStatsUpdateOOMStat */
+
+void
+PVRSRVStatsUpdateRenderContextStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32TotalNumPartialRenders,
+									IMG_UINT32 ui32TotalNumOutOfMemory,
+									IMG_UINT32 ui32NumTAStores,
+									IMG_UINT32 ui32Num3DStores,
+									IMG_UINT32 ui32NumCDMStores,
+									IMG_UINT32 ui32NumTDMStores,
+									IMG_UINT32 ui32NumRayStores,
+									IMG_PID pidOwner)
+{
+	IMG_PID	pidCurrent = pidOwner;
+
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(pidCurrent);
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_PRS]       += ui32TotalNumPartialRenders;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_OOMS]      += ui32TotalNumOutOfMemory;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_TA_STORES] += ui32NumTAStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_3D_STORES] += ui32Num3DStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_CDM_STORES]+= ui32NumCDMStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_TDM_STORES]+= ui32NumTDMStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_RAY_STORES]+= ui32NumRayStores;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PVRSRVStatsUpdateRenderContextStats: Process not found for Pid=%d", pidCurrent));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateRenderContextStats */
+
+void
+PVRSRVStatsUpdateZSBufferStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+							   IMG_UINT32 ui32NumReqByApp,
+							   IMG_UINT32 ui32NumReqByFW,
+							   IMG_PID owner)
+{
+	IMG_PID				  currentPid = (owner==0)?OSGetCurrentClientProcessIDKM():owner;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_APP] += ui32NumReqByApp;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_FW] += ui32NumReqByFW;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Process not found for Pid=%d", __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateZSBufferStats */
+
+void
+PVRSRVStatsUpdateFreelistStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+							   IMG_UINT32 ui32NumGrowReqByApp,
+							   IMG_UINT32 ui32NumGrowReqByFW,
+							   IMG_UINT32 ui32InitFLPages,
+							   IMG_UINT32 ui32NumHighPages,
+							   IMG_PID ownerPid)
+{
+	IMG_PID				  currentPid = (ownerPid!=0)?ownerPid:OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_APP] += ui32NumGrowReqByApp;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_FW] += ui32NumGrowReqByFW;
+
+		UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_PAGES_INIT],
+						 (IMG_INT32) ui32InitFLPages);
+
+		UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_MAX_PAGES],
+						 (IMG_INT32) ui32NumHighPages);
+		OSLockRelease(psProcessStats->hLock);
+
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Process not found for Pid=%d", __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateFreelistStats */
+
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+
+int
+GenericStatsPrintElementsLive(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_STAT_PV_DATA *psStatType = DIGetPrivData(psEntry);
+	PVRSRV_PROCESS_STATS* psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	PVR_ASSERT(psStatType->pfnStatsPrintElements != NULL);
+
+	DIPrintf(psEntry, "%s\n", psStatType->szLiveStatsHeaderStr);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (dllist_is_empty(&gsLiveList))
+	{
+		DIPrintf(psEntry, "No Stats to display\n%s\n", g_szSeparatorStr);
+	}
+	else
+	{
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			psStatType->pfnStatsPrintElements(psEntry, psProcessStats);
+			DIPrintf(psEntry, "%s\n", g_szSeparatorStr);
+		}
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+}
+
+int
+GenericStatsPrintElementsRetired(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_STAT_PV_DATA *psStatType = DIGetPrivData(psEntry);
+	PVRSRV_PROCESS_STATS* psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	PVR_ASSERT(psStatType->pfnStatsPrintElements != NULL);
+
+	DIPrintf(psEntry, "%s\n", psStatType->szRetiredStatsHeaderStr);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (dllist_is_empty(&gsDeadList))
+	{
+		DIPrintf(psEntry, "No Stats to display\n%s\n", g_szSeparatorStr);
+	}
+	else
+	{
+		dllist_foreach_node(&gsDeadList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			psStatType->pfnStatsPrintElements(psEntry, psProcessStats);
+			DIPrintf(psEntry, "%s\n", g_szSeparatorStr);
+		}
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+/*************************************************************************/ /*!
+@Function       ProcessStatsPrintElements
+@Description    Prints all elements for this process statistic record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+ProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                          PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_UINT32 ui32StatNumber;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RILockAcquireKM();
+#endif
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+	DIPrintf(psEntry, "PID %u - %s\n", psProcessStats->pid, psProcessStats->processName);
+
+	/* Print device stats table PVRSRV_DEVICE_STAT_TYPE */
+	if (psProcessStats->ui32DevCount > 0)
+	{
+		IMG_UINT32 i;
+
+		for (ui32StatNumber = 0;
+			 ui32StatNumber < ARRAY_SIZE(pszDeviceStatType);
+			 ui32StatNumber++)
+		{
+			if (OSStringNCompare(pszDeviceStatType[ui32StatNumber], "", 1) != 0)
+			{
+				DIPrintf(psEntry, "%-34s",
+						 pszDeviceStatType[ui32StatNumber]);
+
+				for (i = 0; i < psProcessStats->ui32DevCount; i++)
+				{
+					if (i == 0)
+					{
+						DIPrintf(psEntry, "%10d",
+								 psProcessStats->ai32DevStats[i][ui32StatNumber]);
+					}
+					else
+					{
+						DIPrintf(psEntry, ",%d",
+								 psProcessStats->ai32DevStats[i][ui32StatNumber]);
+					}
+				}
+			}
+
+			DIPrintf(psEntry, "\n");
+		}
+	}
+
+	/* Print process memory stats table PVRSRV_PROCESS_STAT_TYPE */
+	for (ui32StatNumber = 0;
+	     ui32StatNumber < ARRAY_SIZE(pszProcessStatType);
+	     ui32StatNumber++)
+	{
+		if (OSStringNCompare(pszProcessStatType[ui32StatNumber], "", 1) != 0)
+		{
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+			if ((ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) ||
+			    (ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES))
+			{
+				/* get the stat from RI */
+				IMG_INT32 ui32Total = RITotalAllocProcessUnlocked(psProcessStats->pid,
+									    (ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES)
+									    ? PHYS_HEAP_TYPE_LMA : PHYS_HEAP_TYPE_UMA);
+
+				DIPrintf(psEntry, "%-34s%10d %8dK\n",
+						 pszProcessStatType[ui32StatNumber], ui32Total, ui32Total>>10);
+			}
+			else
+#endif
+			{
+				if (ui32StatNumber >= PVRSRV_PROCESS_STAT_TYPE_KMALLOC &&
+					ui32StatNumber <= PVRSRV_PROCESS_STAT_TYPE_TOTAL_MAX)
+				{
+					DIPrintf(psEntry, "%-34s%10"IMG_INT64_FMTSPECd" %8"IMG_INT64_FMTSPECd"K\n",
+					         pszProcessStatType[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber] >> 10);
+				}
+				else
+				{
+					DIPrintf(psEntry, "%-34s%10"IMG_INT64_FMTSPECd"\n",
+					         pszProcessStatType[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber]);
+				}
+			}
+		}
+	}
+
+	OSLockRelease(psProcessStats->hLock);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RILockReleaseKM();
+#endif
+} /* ProcessStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void
+PVRSRVStatsUpdateCacheOpStats(PVRSRV_CACHE_OP uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEV_PHYADDR sDevPAddr,
+#endif
+							IMG_DEVMEM_SIZE_T uiOffset,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT64 ui64ExecuteTime,
+							IMG_BOOL bUserModeFlush,
+							IMG_PID ownerPid)
+{
+	IMG_PID				  currentPid = (ownerPid!=0)?ownerPid:OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+
+	if (psProcessStats != NULL)
+	{
+		IMG_INT32 Idx;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		/* Look-up next buffer write index */
+		Idx = psProcessStats->uiCacheOpWriteIndex;
+		psProcessStats->uiCacheOpWriteIndex = INCREMENT_CACHEOP_STAT_IDX_WRAP(Idx);
+
+		/* Store all CacheOp meta-data */
+		psProcessStats->asCacheOp[Idx].uiCacheOp = uiCacheOp;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+		psProcessStats->asCacheOp[Idx].sDevVAddr = sDevVAddr;
+		psProcessStats->asCacheOp[Idx].sDevPAddr = sDevPAddr;
+#endif
+		psProcessStats->asCacheOp[Idx].uiOffset = uiOffset;
+		psProcessStats->asCacheOp[Idx].uiSize = uiSize;
+		psProcessStats->asCacheOp[Idx].bUserModeFlush = bUserModeFlush;
+		psProcessStats->asCacheOp[Idx].ui64ExecuteTime = ui64ExecuteTime;
+
+		OSLockRelease(psProcessStats->hLock);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateCacheOpStats */
+
+/*************************************************************************/ /*!
+@Function       CacheOpStatsPrintElements
+@Description    Prints all elements for this process statistic CacheOp record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+CacheOpStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                          PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_CHAR  *pszCacheOpType, *pszFlushType, *pszFlushMode;
+	IMG_INT32 i32WriteIdx, i32ReadIdx;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	#define CACHEOP_RI_PRINTF_HEADER \
+		"%-10s %-10s %-5s %-16s %-16s %-10s %-10s %-12s\n"
+	#define CACHEOP_RI_PRINTF		\
+		"%-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-12llu\n"
+#else
+	#define CACHEOP_PRINTF_HEADER	\
+		"%-10s %-10s %-5s %-10s %-10s %-12s\n"
+	#define CACHEOP_PRINTF			\
+		"%-10s %-10s %-5s 0x%-8llx 0x%-8llx %-12llu\n"
+#endif
+
+	DIPrintf(psEntry, "PID %u - %s\n", psProcessStats->pid, psProcessStats->processName);
+
+	/* File header info */
+	DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+					CACHEOP_RI_PRINTF_HEADER,
+#else
+					CACHEOP_PRINTF_HEADER,
+#endif
+					"CacheOp",
+					"Type",
+					"Mode",
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+					"DevVAddr",
+					"DevPAddr",
+#endif
+					"Offset",
+					"Size",
+					"Time (us)");
+
+	/* Take a snapshot of write index, read backwards in buffer
+	   and wrap round at boundary */
+	i32WriteIdx = psProcessStats->uiCacheOpWriteIndex;
+	for (i32ReadIdx = DECREMENT_CACHEOP_STAT_IDX_WRAP(i32WriteIdx);
+		 i32ReadIdx != i32WriteIdx;
+		 i32ReadIdx = DECREMENT_CACHEOP_STAT_IDX_WRAP(i32ReadIdx))
+	{
+		IMG_UINT64 ui64ExecuteTime = psProcessStats->asCacheOp[i32ReadIdx].ui64ExecuteTime;
+		IMG_DEVMEM_SIZE_T ui64NumOfPages = psProcessStats->asCacheOp[i32ReadIdx].uiSize >> OSGetPageShift();
+
+		if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			pszFlushType = "RBF.Fast";
+		}
+		else
+		{
+			pszFlushType = "RBF.Slow";
+		}
+
+		if (psProcessStats->asCacheOp[i32ReadIdx].bUserModeFlush)
+		{
+			pszFlushMode = "UM";
+		}
+		else
+		{
+			pszFlushMode = "KM";
+		}
+
+		switch (psProcessStats->asCacheOp[i32ReadIdx].uiCacheOp)
+		{
+			case PVRSRV_CACHE_OP_NONE:
+				pszCacheOpType = "None";
+				break;
+			case PVRSRV_CACHE_OP_CLEAN:
+				pszCacheOpType = "Clean";
+				break;
+			case PVRSRV_CACHE_OP_INVALIDATE:
+				pszCacheOpType = "Invalidate";
+				break;
+			case PVRSRV_CACHE_OP_FLUSH:
+				pszCacheOpType = "Flush";
+				break;
+			default:
+				pszCacheOpType = "Unknown";
+				break;
+		}
+
+		DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							CACHEOP_RI_PRINTF,
+#else
+							CACHEOP_PRINTF,
+#endif
+							pszCacheOpType,
+							pszFlushType,
+							pszFlushMode,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							psProcessStats->asCacheOp[i32ReadIdx].sDevVAddr.uiAddr,
+							psProcessStats->asCacheOp[i32ReadIdx].sDevPAddr.uiAddr,
+#endif
+							psProcessStats->asCacheOp[i32ReadIdx].uiOffset,
+							psProcessStats->asCacheOp[i32ReadIdx].uiSize,
+							ui64ExecuteTime);
+		}
+
+} /* CacheOpStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static PVRSRV_ERROR _PrintMemStatsEntry(uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	IMG_UINT32	ui32VAddrFields = sizeof(void*)/sizeof(IMG_UINT32);
+	IMG_UINT32	ui32PAddrFields = sizeof(IMG_CPU_PHYADDR)/sizeof(IMG_UINT32);
+	IMG_UINT32 ui32ItemNumber;
+	PVRSRV_MEM_ALLOC_REC *psRecord = (PVRSRV_MEM_ALLOC_REC *)(uintptr_t)v;
+	PVRSRV_MEM_ALLOC_PRINT_DATA *psPrintData = (PVRSRV_MEM_ALLOC_PRINT_DATA *)pvPriv;
+	OSDI_IMPL_ENTRY *psEntry = psPrintData->psEntry;
+
+	if (psRecord != NULL)
+	{
+		IMG_BOOL bPrintStat = IMG_TRUE;
+
+		DIPrintf(psEntry, "%-5d  ", psPrintData->pid);
+
+		switch (psRecord->eAllocType)
+		{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:             DIPrintf(psEntry, "KMALLOC             "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:             DIPrintf(psEntry, "VMALLOC             "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:  DIPrintf(psEntry, "ALLOC_PAGES_PT_LMA  "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:  DIPrintf(psEntry, "ALLOC_PAGES_PT_UMA  "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:      DIPrintf(psEntry, "IOREMAP_PT_LMA      "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:         DIPrintf(psEntry, "VMAP_PT_UMA         "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:     DIPrintf(psEntry, "ALLOC_LMA_PAGES     "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:    DIPrintf(psEntry, "ZOMBIE_LMA_PAGES    "); break;
+#endif
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:     DIPrintf(psEntry, "ALLOC_UMA_PAGES     "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:    DIPrintf(psEntry, "ZOMBIE_UMA_PAGES    "); break;
+#endif
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:   DIPrintf(psEntry, "MAP_UMA_LMA_PAGES   "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:      DIPrintf(psEntry, "DMA_BUF_IMPORT      "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:      DIPrintf(psEntry, "DMA_BUF_ZOMBIE      "); break;
+#endif
+		default:                                        DIPrintf(psEntry, "INVALID             "); break;
+		}
+
+		if (bPrintStat)
+		{
+			for (ui32ItemNumber = 0; ui32ItemNumber < ui32VAddrFields; ui32ItemNumber++)
+			{
+				DIPrintf(psEntry, "%08x", *(((IMG_UINT32*) &psRecord->pvCpuVAddr) + ui32VAddrFields - ui32ItemNumber - 1));
+			}
+			DIPrintf(psEntry, "  ");
+
+			for (ui32ItemNumber = 0; ui32ItemNumber < ui32PAddrFields; ui32ItemNumber++)
+			{
+				DIPrintf(psEntry, "%08x", *(((IMG_UINT32*) &psRecord->sCpuPAddr.uiAddr) + ui32PAddrFields - ui32ItemNumber - 1));
+			}
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+			DIPrintf(psEntry, "  " IMG_SIZE_FMTSPEC, psRecord->uiBytes);
+
+			DIPrintf(psEntry, "  %s", (IMG_CHAR*) psRecord->pvAllocdFromFile);
+
+			DIPrintf(psEntry, "  %d\n", psRecord->ui32AllocdFromLine);
+#else
+			DIPrintf(psEntry, "  " IMG_SIZE_FMTSPEC "\n", psRecord->uiBytes);
+#endif
+		}
+
+		psPrintData->ui32NumEntries++;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       MemStatsPrintElements
+@Description    Prints all elements for the memory statistic record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+MemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                      PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_UINT32	ui32VAddrFields = sizeof(void*)/sizeof(IMG_UINT32);
+	IMG_UINT32	ui32PAddrFields = sizeof(IMG_CPU_PHYADDR)/sizeof(IMG_UINT32);
+	IMG_UINT32 ui32ItemNumber;
+	PVRSRV_MEM_ALLOC_PRINT_DATA sPrintData;
+
+	sPrintData.psEntry = psEntry;
+	sPrintData.pid = psProcessStats->pid;
+	sPrintData.ui32NumEntries = 0;
+
+	/* Write the header... */
+	DIPrintf(psEntry, "PID    ");
+
+	DIPrintf(psEntry, "Type                VAddress");
+	for (ui32ItemNumber = 1;  ui32ItemNumber < ui32VAddrFields;  ui32ItemNumber++)
+	{
+		DIPrintf(psEntry, "        ");
+	}
+
+	DIPrintf(psEntry, "  PAddress");
+	for (ui32ItemNumber = 1;  ui32ItemNumber < ui32PAddrFields;  ui32ItemNumber++)
+	{
+		DIPrintf(psEntry, "        ");
+	}
+
+	DIPrintf(psEntry, "  Size(bytes)\n");
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+	HASH_Iterate(psProcessStats->psMemoryRecords, (HASH_pfnCallback)_PrintMemStatsEntry, &sPrintData);
+	OSLockRelease(psProcessStats->hLock);
+
+	if (sPrintData.ui32NumEntries == 0)
+	{
+		DIPrintf(psEntry, "%-5d\n", psProcessStats->pid);
+	}
+} /* MemStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+/*************************************************************************/ /*!
+@Function       RIMemStatsPrintElements
+@Description    Prints all elements for the RI Memory record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void RIMemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                             PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_CHAR   *pszStatFmtText  = NULL;
+	IMG_HANDLE *pRIHandle       = NULL;
+
+	/* Acquire RI lock */
+	RILockAcquireKM();
+
+	/*
+	 * Loop through the RI system to get each line of text.
+	 */
+	while (RIGetListEntryKM(psProcessStats->pid,
+							&pRIHandle,
+							&pszStatFmtText))
+	{
+		DIPrintf(psEntry, "%s", pszStatFmtText);
+	}
+
+	/* Release RI lock */
+	RILockReleaseKM();
+
+} /* RIMemStatsPrintElements */
+#endif
+
+#endif
+
+int GlobalStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	IMG_UINT32 ui32StatNumber;
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+	for (ui32StatNumber = 0;
+	     ui32StatNumber < ARRAY_SIZE(pszDriverStatType);
+	     ui32StatNumber++)
+	{
+		if (OSStringNCompare(pszDriverStatType[ui32StatNumber], "", 1) != 0)
+		{
+			DIPrintf(psEntry, "%-34s%12" IMG_UINT64_FMTSPEC "\n",
+				    pszDriverStatType[ui32StatNumber],
+				    GET_GLOBAL_STAT_VALUE(ui32StatNumber));
+		}
+	}
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVFindProcessMemStats
+@Description    Using the provided PID find memory stats for that process.
+                Memstats will be provided for live/connected processes only.
+                Memstat values provided by this API relate only to the physical
+                memory allocated by the process and does not relate to any of
+                the mapped or imported memory.
+@Input          pid                 Process to search for.
+@Input          ArraySize           Size of the array where memstat
+                                    records will be stored
+@Input          bAllProcessStats    Flag to denote if stats for
+                                    individual process are requested
+                                    stats for all processes are
+                                    requested
+@Input          MemoryStats         Handle to the memory where memstats
+                                    are stored.
+@Output         Memory statistics records for the requested pid.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVFindProcessMemStats(IMG_PID pid,
+                                       IMG_UINT32 ui32ArrSize,
+                                       IMG_BOOL bAllProcessStats,
+                                       IMG_UINT64 *pui64MemoryStats)
+{
+	IMG_INT i;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui64MemoryStats, "pui64MemoryStats");
+
+	if (bAllProcessStats)
+	{
+		PVR_LOG_RETURN_IF_FALSE(ui32ArrSize == PVRSRV_DRIVER_STAT_TYPE_COUNT,
+			"MemStats array size is incorrect",
+			PVRSRV_ERROR_INVALID_PARAMS);
+
+		OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+		for (i = 0; i < ui32ArrSize; i++)
+		{
+			pui64MemoryStats[i] = GET_GLOBAL_STAT_VALUE(i);
+		}
+
+		OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrSize == PVRSRV_PROCESS_STAT_TYPE_COUNT,
+		"MemStats array size is incorrect",
+		PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	/* Search for the given PID in the Live List */
+	psProcessStats = _FindProcessStatsInLiveList(pid);
+
+	if (psProcessStats == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Process %d not found. This process may not be live anymore.", (IMG_INT)pid));
+		OSLockRelease(g_psLinkedListLock);
+
+		return PVRSRV_ERROR_PROCESS_NOT_FOUND;
+	}
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+	for (i = 0; i < ui32ArrSize; i++)
+	{
+		pui64MemoryStats[i] = psProcessStats->i64StatValue[i];
+	}
+	OSLockRelease(psProcessStats->hLock);
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+
+} /* PVRSRVFindProcessMemStats */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetProcessMemUsage
+@Description    Calculate allocated kernel and graphics memory for all live or
+                connected processes. Memstat values provided by this API relate
+                only to the physical memory allocated by the process and does
+                not relate to any of the mapped or imported memory.
+@Output         pui64TotalMem                   Total memory usage for all live
+                                                PIDs connected to the driver.
+@Output         pui32NumberOfLivePids           Number of live pids currently
+                                                connected to the server.
+@Output         ppsPerProcessMemUsageData       Handle to an array of
+                                                PVRSRV_PER_PROCESS_MEM_USAGE,
+                                                number of elements defined by
+                                                pui32NumberOfLivePids.
+@Return         PVRSRV_OK                       Success
+                PVRSRV_ERROR_PROCESS_NOT_FOUND  No live processes.
+                PVRSRV_ERROR_OUT_OF_MEMORY      Failed to allocate memory for
+                                                ppsPerProcessMemUsageData.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVGetProcessMemUsage(IMG_UINT64 *pui64TotalMem,
+									  IMG_UINT32 *pui32NumberOfLivePids,
+									  PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsageData)
+{
+	IMG_UINT32 ui32NumberOfLivePids = 0;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_PROCESS_NOT_FOUND;
+	PVRSRV_PER_PROCESS_MEM_USAGE* psPerProcessMemUsageData = NULL;
+	DLLIST_NODE *psNode, *psNext;
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+	*pui64TotalMem = GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_KMALLOC) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_VMALLOC) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA);
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		ui32NumberOfLivePids++;
+	}
+
+	if (ui32NumberOfLivePids > 0)
+	{
+		/* Use OSAllocZMemNoStats to prevent deadlock. */
+		psPerProcessMemUsageData = OSAllocZMemNoStats(ui32NumberOfLivePids * sizeof(*psPerProcessMemUsageData));
+
+		if (psPerProcessMemUsageData)
+		{
+			PVRSRV_PROCESS_STATS* psProcessStats = NULL;
+			IMG_UINT32 ui32Counter = 0;
+
+			dllist_foreach_node(&gsLiveList, psNode, psNext)
+			{
+				psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+				psPerProcessMemUsageData[ui32Counter].ui32Pid = (IMG_UINT32)psProcessStats->pid;
+
+				psPerProcessMemUsageData[ui32Counter].ui64KernelMemUsage =
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMALLOC];
+
+				psPerProcessMemUsageData[ui32Counter].ui64GraphicsMemUsage =
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES];
+
+				OSLockRelease(psProcessStats->hLock);
+				ui32Counter++;
+			}
+			eError = PVRSRV_OK;
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+	*pui32NumberOfLivePids = ui32NumberOfLivePids;
+	*ppsPerProcessMemUsageData = psPerProcessMemUsageData;
+
+	return eError;
+
+} /* PVRSRVGetProcessMemUsage */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvr_notifier.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvr_notifier.c
new file mode 100644
index 000000000000..fd9653a11fb4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvr_notifier.c
@@ -0,0 +1,671 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR notifier interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "allocmem.h"
+#include "dllist.h"
+
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrversion.h"
+#include "connection_server.h"
+
+#include "osfunc.h"
+#include "sofunc_pvr.h"
+
+#define PVR_DUMP_DRIVER_INFO(x, y)														\
+	PVR_DUMPDEBUG_LOG("%s info: %d.%d @ %8d (%s) build options: 0x%08x",				\
+					   (x),																\
+					   PVRVERSION_UNPACK_MAJ((y).ui32BuildVersion),						\
+					   PVRVERSION_UNPACK_MIN((y).ui32BuildVersion),						\
+					   (y).ui32BuildRevision,											\
+					   (BUILD_TYPE_DEBUG == (y).ui32BuildType) ? "debug":"release",		\
+					   (y).ui32BuildOptions);
+
+#if !defined(WINDOW_SYSTEM)
+#define WINDOW_SYSTEM "Unknown"
+#endif
+
+#define IS_DECLARED(x) (x[0] != '\0')
+
+/*************************************************************************/ /*!
+Command Complete Notifier Interface
+*/ /**************************************************************************/
+
+typedef struct PVRSRV_CMDCOMP_NOTIFY_TAG
+{
+	PVRSRV_CMDCOMP_HANDLE	hCmdCompHandle;
+	PFN_CMDCOMP_NOTIFY		pfnCmdCompleteNotify;
+	DLLIST_NODE				sListNode;
+} PVRSRV_CMDCOMP_NOTIFY;
+
+/* Head of the list of callbacks called when command complete happens */
+static DLLIST_NODE g_sCmdCompNotifyHead;
+static POSWR_LOCK g_hCmdCompNotifyLock;
+
+PVRSRV_ERROR
+PVRSRVCmdCompleteInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSWRLockCreate(&g_hCmdCompNotifyLock);
+	PVR_RETURN_IF_ERROR(eError);
+
+	dllist_init(&g_sCmdCompNotifyHead);
+
+	return PVRSRV_OK;
+}
+
+void
+PVRSRVCmdCompleteDeinit(void)
+{
+	/* Check that all notify function have been unregistered */
+	if (!dllist_is_empty(&g_sCmdCompNotifyHead))
+	{
+		PDLLIST_NODE psNode;
+
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Command complete notify list is not empty!", __func__));
+
+		/* Clean up any stragglers */
+		psNode = dllist_get_next_node(&g_sCmdCompNotifyHead);
+		while (psNode)
+		{
+			PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+			dllist_remove_node(psNode);
+
+			psNotify = IMG_CONTAINER_OF(psNode, PVRSRV_CMDCOMP_NOTIFY, sListNode);
+			OSFreeMem(psNotify);
+
+			psNode = dllist_get_next_node(&g_sCmdCompNotifyHead);
+		}
+	}
+
+	if (g_hCmdCompNotifyLock)
+	{
+		OSWRLockDestroy(g_hCmdCompNotifyLock);
+	}
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterCmdCompleteNotify(IMG_HANDLE *phNotify,
+								PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify,
+								PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phNotify, "phNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnCmdCompleteNotify, "pfnCmdCompleteNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hCmdCompHandle, "hCmdCompHandle");
+
+	psNotify = OSAllocMem(sizeof(*psNotify));
+	PVR_LOG_RETURN_IF_NOMEM(psNotify, "psNotify");
+
+	/* Set-up the notify data */
+	psNotify->hCmdCompHandle = hCmdCompHandle;
+	psNotify->pfnCmdCompleteNotify = pfnCmdCompleteNotify;
+
+	/* Add it to the list of Notify functions */
+	OSWRLockAcquireWrite(g_hCmdCompNotifyLock);
+	dllist_add_to_tail(&g_sCmdCompNotifyHead, &psNotify->sListNode);
+	OSWRLockReleaseWrite(g_hCmdCompNotifyLock);
+
+	*phNotify = psNotify;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterCmdCompleteNotify(IMG_HANDLE hNotify)
+{
+	PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+	psNotify = (PVRSRV_CMDCOMP_NOTIFY *) hNotify;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNotify, "hNotify");
+
+	OSWRLockAcquireWrite(g_hCmdCompNotifyLock);
+	dllist_remove_node(&psNotify->sListNode);
+	OSWRLockReleaseWrite(g_hCmdCompNotifyLock);
+
+	OSFreeMem(psNotify);
+
+	return PVRSRV_OK;
+}
+
+void
+PVRSRVNotifyCommandCompletion(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle)
+{
+#if !defined(NO_HARDWARE)
+	DLLIST_NODE *psNode, *psNext;
+
+	/* Call notify callbacks to check if blocked work items can now proceed */
+	OSWRLockAcquireRead(g_hCmdCompNotifyLock);
+	dllist_foreach_node(&g_sCmdCompNotifyHead, psNode, psNext)
+	{
+		PVRSRV_CMDCOMP_NOTIFY *psNotify =
+			IMG_CONTAINER_OF(psNode, PVRSRV_CMDCOMP_NOTIFY, sListNode);
+
+		if (hCmdCompCallerHandle != psNotify->hCmdCompHandle)
+		{
+			psNotify->pfnCmdCompleteNotify(psNotify->hCmdCompHandle);
+		}
+	}
+	OSWRLockReleaseRead(g_hCmdCompNotifyLock);
+#endif
+}
+
+inline void
+PVRSRVSignalDriverWideEO(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectSignal(psPVRSRVData->hGlobalEventObject);
+	}
+	/* Cleanup Thread could be waiting on Cleanup event object,
+	 * signal it as well to ensure work is processed
+	 */
+	if (psPVRSRVData->hCleanupEventObject && (OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued) != 0))
+	{
+		OSEventObjectSignal(psPVRSRVData->hCleanupEventObject);
+	}
+}
+
+inline void
+PVRSRVCheckStatus(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle)
+{
+	PVRSRVNotifyCommandCompletion(hCmdCompCallerHandle);
+	PVRSRVSignalDriverWideEO();
+}
+
+/*************************************************************************/ /*!
+Debug Notifier Interface
+*/ /**************************************************************************/
+
+/* Lockdep sees both locks as the same class due to same struct used thus warns
+ * about a possible deadlock (false positive),
+ * using nested api we can supply separate Class'
+ * */
+#define DN_LOCKCLASS_DRIVER 0
+#define DN_LOCKCLASS_DEVICE 1
+
+typedef struct DEBUG_REQUEST_ENTRY_TAG
+{
+	IMG_UINT32      ui32RequesterID;
+	DLLIST_NODE     sListHead;
+} DEBUG_REQUEST_ENTRY;
+
+typedef struct DEBUG_REQUEST_TABLE_TAG
+{
+	POSWR_LOCK              hLock;
+	DEBUG_REQUEST_ENTRY     asEntry[IMG_FLEX_ARRAY_MEMBER];
+} DEBUG_REQUEST_TABLE;
+
+typedef struct DEBUG_REQUEST_NOTIFY_TAG
+{
+	IMG_HANDLE              hDebugTable;
+	PVRSRV_DBGREQ_HANDLE    hDbgRequestHandle;
+	PFN_DBGREQ_NOTIFY       pfnDbgRequestNotify;
+	IMG_UINT32              ui32RequesterID;
+	DLLIST_NODE             sListNode;
+} DEBUG_REQUEST_NOTIFY;
+
+static DEBUG_REQUEST_TABLE *g_psDriverDebugTable;
+
+static const IMG_UINT32 g_aui32DebugOrderTable[] = {
+	DEBUG_REQUEST_SRV,
+	DEBUG_REQUEST_RGX,
+	DEBUG_REQUEST_SYS,
+	DEBUG_REQUEST_APPHINT,
+	DEBUG_REQUEST_HTB,
+	DEBUG_REQUEST_DC,
+	DEBUG_REQUEST_SYNCCHECKPOINT,
+	DEBUG_REQUEST_SYNCTRACKING,
+	DEBUG_REQUEST_ANDROIDSYNC,
+	DEBUG_REQUEST_FALLBACKSYNC,
+	DEBUG_REQUEST_LINUXFENCE
+};
+static const IMG_UINT32 g_ui32DebugOrderTableReqCount = ARRAY_SIZE(g_aui32DebugOrderTable);
+
+static PVRSRV_ERROR
+_RegisterDebugTableI(DEBUG_REQUEST_TABLE **ppsDebugTable)
+{
+	DEBUG_REQUEST_TABLE *psDebugTable;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	if (*ppsDebugTable)
+	{
+		return PVRSRV_ERROR_DBGTABLE_ALREADY_REGISTERED;
+	}
+
+	psDebugTable = OSAllocMem(sizeof(DEBUG_REQUEST_TABLE) +
+	                          IMG_FLEX_ARRAY_SIZE(sizeof(DEBUG_REQUEST_ENTRY), g_ui32DebugOrderTableReqCount));
+	PVR_RETURN_IF_NOMEM(psDebugTable);
+
+	eError = OSWRLockCreate(&psDebugTable->hLock);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeDebugTable);
+
+	/* Init the list heads */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		psDebugTable->asEntry[i].ui32RequesterID = g_aui32DebugOrderTable[i];
+		dllist_init(&psDebugTable->asEntry[i].sListHead);
+	}
+
+	*ppsDebugTable = psDebugTable;
+
+	return PVRSRV_OK;
+
+ErrorFreeDebugTable:
+	OSFreeMem(psDebugTable);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgTable(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	return _RegisterDebugTableI((DEBUG_REQUEST_TABLE**)&psDevNode->hDebugTable);
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgTable(void)
+{
+	return _RegisterDebugTableI(&g_psDriverDebugTable);
+}
+
+static void _UnregisterDbgTableI(DEBUG_REQUEST_TABLE **ppsDebugTable)
+{
+	DEBUG_REQUEST_TABLE *psDebugTable;
+	IMG_UINT32 i;
+
+	PVR_ASSERT(*ppsDebugTable);
+	psDebugTable = *ppsDebugTable;
+	*ppsDebugTable = NULL;
+
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		if (!dllist_is_empty(&psDebugTable->asEntry[i].sListHead))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Found registered callback(s) on %d",
+					 __func__, i));
+		}
+	}
+
+	OSWRLockDestroy(psDebugTable->hLock);
+	psDebugTable->hLock = NULL;
+
+	OSFreeMem(psDebugTable);
+}
+
+void
+PVRSRVUnregisterDeviceDbgTable(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	_UnregisterDbgTableI((DEBUG_REQUEST_TABLE**)&psDevNode->hDebugTable);
+	PVR_ASSERT(!psDevNode->hDebugTable);
+}
+
+void
+PVRSRVUnregisterDriverDbgTable(void)
+{
+	_UnregisterDbgTableI(&g_psDriverDebugTable);
+	PVR_ASSERT(!g_psDriverDebugTable);
+}
+
+static PVRSRV_ERROR
+_RegisterDbgRequestNotifyI(IMG_HANDLE *phNotify,
+		                   DEBUG_REQUEST_TABLE *psDebugTable,
+		                   PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+		                   IMG_UINT32 ui32RequesterID,
+		                   PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	DEBUG_REQUEST_NOTIFY *psNotify;
+	PDLLIST_NODE psHead = NULL;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phNotify, "phNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDebugTable, "psDebugTable");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnDbgRequestNotify, "pfnDbRequestNotify");
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psNotify = OSAllocMemNoStats(sizeof(*psNotify));
+	PVR_LOG_RETURN_IF_NOMEM(psNotify, "psNotify");
+
+	/* Set-up the notify data */
+	psNotify->hDebugTable = psDebugTable;
+	psNotify->hDbgRequestHandle = hDbgRequestHandle;
+	psNotify->pfnDbgRequestNotify = pfnDbgRequestNotify;
+	psNotify->ui32RequesterID = ui32RequesterID;
+
+	/* Lock down all the lists */
+	OSWRLockAcquireWrite(psDebugTable->hLock);
+
+	/* Find which list to add it to */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		if (psDebugTable->asEntry[i].ui32RequesterID == ui32RequesterID)
+		{
+			psHead = &psDebugTable->asEntry[i].sListHead;
+		}
+	}
+
+	/* Failed to find debug requester */
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psHead, eError, ErrorReleaseLock);
+
+	/* Add it to the list of Notify functions */
+	dllist_add_to_tail(psHead, &psNotify->sListNode);
+
+	/* Unlock the lists */
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+
+	*phNotify = psNotify;
+
+	return PVRSRV_OK;
+
+ErrorReleaseLock:
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+	OSFreeMem(psNotify);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgRequestNotify(IMG_HANDLE *phNotify,
+	                                 PVRSRV_DEVICE_NODE *psDevNode,
+	                                 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	                                 IMG_UINT32 ui32RequesterID,
+	                                 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode, "psDevNode");
+	if (!psDevNode->hDebugTable)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: psDevNode->hDebugTable not yet initialised!",
+				 __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	return _RegisterDbgRequestNotifyI(phNotify,
+			                          (DEBUG_REQUEST_TABLE *)psDevNode->hDebugTable,
+			                          pfnDbgRequestNotify,
+			                          ui32RequesterID,
+			                          hDbgRequestHandle);
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgRequestNotify(IMG_HANDLE *phNotify,
+	                                 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	                                 IMG_UINT32 ui32RequesterID,
+	                                 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	if (!g_psDriverDebugTable)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: g_psDriverDebugTable not yet initialised!",
+				 __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	return _RegisterDbgRequestNotifyI(phNotify,
+			                          g_psDriverDebugTable,
+			                          pfnDbgRequestNotify,
+			                          ui32RequesterID,
+			                          hDbgRequestHandle);
+}
+
+PVRSRV_ERROR
+SOPvrDbgRequestNotifyRegister(IMG_HANDLE *phNotify,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+							  IMG_UINT32 ui32RequesterID,
+							  PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	return PVRSRVRegisterDeviceDbgRequestNotify(phNotify,
+			psDevNode,
+			pfnDbgRequestNotify,
+			ui32RequesterID,
+			hDbgRequestHandle);
+}
+
+static PVRSRV_ERROR
+_UnregisterDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	DEBUG_REQUEST_NOTIFY *psNotify = (DEBUG_REQUEST_NOTIFY *) hNotify;
+	DEBUG_REQUEST_TABLE *psDebugTable;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNotify, "psNotify");
+
+	psDebugTable = (DEBUG_REQUEST_TABLE *) psNotify->hDebugTable;
+
+	OSWRLockAcquireWrite(psDebugTable->hLock);
+	dllist_remove_node(&psNotify->sListNode);
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+
+	OSFreeMemNoStats(psNotify);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterDeviceDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterDriverDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+PVRSRV_ERROR
+SOPvrDbgRequestNotifyUnregister(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+void
+PVRSRVDebugRequest(PVRSRV_DEVICE_NODE *psDevNode,
+				   IMG_UINT32 ui32VerbLevel,
+				   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				   void *pvDumpDebugFile)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	DEBUG_REQUEST_TABLE *psDebugTable =
+		(DEBUG_REQUEST_TABLE *) psDevNode->hDebugTable;
+	DEBUG_REQUEST_TABLE *psDriverDebugTable =
+		(DEBUG_REQUEST_TABLE *) g_psDriverDebugTable;
+	static const IMG_CHAR *apszVerbosityTable[] = { "Low", "Medium", "High" };
+	const IMG_CHAR *szVerbosityLevel;
+	const IMG_CHAR *Bit32 = "32 Bit", *Bit64 = "64 Bit";
+	IMG_UINT32 i;
+
+	static_assert(ARRAY_SIZE(apszVerbosityTable) == DEBUG_REQUEST_VERBOSITY_MAX+1,
+	              "Incorrect number of verbosity levels");
+
+	PVR_ASSERT(psDebugTable);
+	PVR_ASSERT(psDriverDebugTable);
+
+	if (ui32VerbLevel < ARRAY_SIZE(apszVerbosityTable))
+	{
+		szVerbosityLevel = apszVerbosityTable[ui32VerbLevel];
+	}
+	else
+	{
+		szVerbosityLevel = "unknown";
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Invalid verbosity level received", __func__));
+	}
+
+	PVR_DUMPDEBUG_LOG("------------[ PVR DBG: START (%s) ]------------",
+	                  szVerbosityLevel);
+
+#if defined(RGX_IRQ_HYPERV_HANDLER)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+#endif
+	{
+		OSDumpVersionInfo(pfnDumpDebugPrintf, pvDumpDebugFile);
+	}
+
+	PVR_DUMPDEBUG_LOG("DDK info: %s (%s) %s",
+	                  PVRVERSION_STRING, PVR_BUILD_TYPE, PVR_BUILD_DIR);
+
+	PVR_DUMPDEBUG_LOG("Time now: %" IMG_UINT64_FMTSPEC "us",
+	                  OSClockus64());
+
+	switch (psPVRSRVData->eServicesState)
+	{
+		case PVRSRV_SERVICES_STATE_OK:
+			PVR_DUMPDEBUG_LOG("Services State: OK");
+			break;
+		case PVRSRV_SERVICES_STATE_BAD:
+			PVR_DUMPDEBUG_LOG("Services State: BAD");
+			break;
+		case PVRSRV_SERVICES_STATE_UNDEFINED:
+			PVR_DUMPDEBUG_LOG("Services State: UNDEFINED");
+			break;
+		default:
+			PVR_DUMPDEBUG_LOG("Services State: UNKNOWN (%d)",
+			                  psPVRSRVData->eServicesState);
+			break;
+	}
+
+	PVR_DUMPDEBUG_LOG("Server Errors: %d",
+	          PVRSRV_KM_ERRORS);
+
+	PVRSRVConnectionDebugNotify(psDevNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+
+	PVR_DUMPDEBUG_LOG("------[ Driver Info ]------");
+
+	PVR_DUMPDEBUG_LOG("Comparison of UM/KM components: %s",
+	                 (psPVRSRVData->sDriverInfo.bIsNoMatch) ? "MISMATCH" : "MATCHING");
+
+	if (psPVRSRVData->sDriverInfo.ui8KMBitArch)
+	{
+		PVR_DUMPDEBUG_LOG("KM Arch: %s",(psPVRSRVData->sDriverInfo.ui8KMBitArch & BUILD_ARCH_64BIT) ? Bit64 : Bit32);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("KM Arch is undefined");
+	}
+
+	PVR_DUMPDEBUG_LOG("Driver Mode: %s",
+	                  PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDevNode) ? "Native" : (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDevNode)) ? "Host":"Guest");
+
+	if (psPVRSRVData->sDriverInfo.ui8UMSupportedArch)
+	{
+		if ((psPVRSRVData->sDriverInfo.ui8UMSupportedArch & BUILD_ARCH_BOTH) ==
+			BUILD_ARCH_BOTH)
+		{
+			PVR_DUMPDEBUG_LOG("UM Connected Clients Arch: %s and %s", Bit64, Bit32);
+
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("UM Connected Clients: %s",
+			                 (psPVRSRVData->sDriverInfo.ui8UMSupportedArch & BUILD_ARCH_64BIT) ? Bit64 : Bit32);
+		}
+	}
+
+	PVR_DUMP_DRIVER_INFO("UM", psPVRSRVData->sDriverInfo.sUMBuildInfo);
+	PVR_DUMP_DRIVER_INFO("KM", psPVRSRVData->sDriverInfo.sKMBuildInfo);
+
+	PVR_DUMPDEBUG_LOG("Window system: %s", (IS_DECLARED(WINDOW_SYSTEM)) ? (WINDOW_SYSTEM) : "Not declared");
+
+	PVR_DUMPDEBUG_LOG("Power lock status: %s", OSLockIsLocked(psDevNode->hPowerLock) ? "Locked" : "Free");
+#if defined(DEBUG)
+	if (OSLockIsLocked(psDevNode->hPowerLock))
+	{
+		/* Ensure this info is logged before the power lock is taken, in case
+		 * it's already in use. */
+		PVR_DUMPDEBUG_LOG("Power lock owner: PID = %u at timestamp %" IMG_UINT64_FMTSPEC " (%s:%u)",
+		                  psDevNode->uiPwrLockOwnerPID, psDevNode->sPowerLockOwner.ui64Timestamp,
+		                  psDevNode->sPowerLockOwner.pszFile, psDevNode->sPowerLockOwner.ui32LineNum);
+	}
+#endif
+
+	/* Driver debug table */
+	OSWRLockAcquireReadNested(psDriverDebugTable->hLock, DN_LOCKCLASS_DRIVER);
+	/* Device debug table*/
+	OSWRLockAcquireReadNested(psDebugTable->hLock, DN_LOCKCLASS_DEVICE);
+
+	/* For each requester in Driver and Device table */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		DLLIST_NODE *psNode;
+		DLLIST_NODE *psNext;
+
+		/* For each notifier on this requestor */
+		dllist_foreach_node(&psDriverDebugTable->asEntry[i].sListHead, psNode, psNext)
+		{
+			DEBUG_REQUEST_NOTIFY *psNotify =
+				IMG_CONTAINER_OF(psNode, DEBUG_REQUEST_NOTIFY, sListNode);
+			psNotify->pfnDbgRequestNotify(psNotify->hDbgRequestHandle, ui32VerbLevel,
+				                          pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+
+		/* For each notifier on this requestor */
+		dllist_foreach_node(&psDebugTable->asEntry[i].sListHead, psNode, psNext)
+		{
+			DEBUG_REQUEST_NOTIFY *psNotify =
+				IMG_CONTAINER_OF(psNode, DEBUG_REQUEST_NOTIFY, sListNode);
+			psNotify->pfnDbgRequestNotify(psNotify->hDbgRequestHandle, ui32VerbLevel,
+							pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+	}
+
+	OSWRLockReleaseRead(psDebugTable->hLock);
+	OSWRLockReleaseRead(psDriverDebugTable->hLock);
+
+	PVR_DUMPDEBUG_LOG("------------[ PVR DBG: END ]------------");
+
+	if (!pfnDumpDebugPrintf)
+	{
+		/* Only notify OS of an issue if the caller requested it,
+		 * using a NULL pointer in pfnDumpDebugPrintf. */
+		OSWarnOn(IMG_TRUE);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv.c
new file mode 100644
index 000000000000..934593ff3ae5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv.c
@@ -0,0 +1,3871 @@
+/*************************************************************************/ /*!
+@File
+@Title          core services functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for core services functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#define CLEANUP_TYPE_STRINGS
+
+#include "img_defs.h"
+#include "rgxdebug_common.h"
+#include "handle.h"
+
+#include "pmr.h"
+#include "connection_server.h"
+#include "osconnection_server.h"
+#include "pdump_km.h"
+#include "ra.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "srvcore.h"
+#include "services_km.h"
+#include "pvrsrv_device.h"
+#include "pvr_debug.h"
+#include "debug_common.h"
+#include "pvr_notifier.h"
+#include "sync.h"
+#include "sync_server.h"
+#include "sync_checkpoint.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint_init.h"
+#include "devicemem.h"
+#include "cache_km.h"
+#include "info_page.h"
+#include "info_page_defs.h"
+#include "pvrsrv_bridge_init.h"
+#include "devicemem_server.h"
+#include "km_apphint_defs.h"
+#include "di_server.h"
+#include "di_impl_brg.h"
+#include "htb_debug.h"
+#include "dma_km.h"
+#include "pmr.h"
+
+#include "log2.h"
+
+#include "lists.h"
+#include "dllist.h"
+#include "syscommon.h"
+#include "sysvalidation.h"
+#include "hash_functions.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+#include "physmem_hostmem.h"
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+#include "physmem_cpumap_history.h"
+#endif
+
+#include "tlintern.h"
+#include "htbserver.h"
+#include "rgxmulticore.h"
+
+
+//#define MULTI_DEVICE_BRINGUP
+
+#if defined(MULTI_DEVICE_BRINGUP)
+#define MULTI_DEVICE_BRINGUP_DPF(msg, ...) PVR_DPF((PVR_DBG_MESSAGE, msg, __VA_ARGS__))
+#else
+#define MULTI_DEVICE_BRINGUP_DPF(msg, ...)
+#endif
+
+#if defined(SUPPORT_RGX)
+#include "rgxinit.h"
+#include "rgxhwperf.h"
+#include "rgxfwutils.h"
+#include "rgx_bridge_init.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#include "vz_vmm_pvz.h"
+
+#include "devicemem_history_server.h"
+
+#if defined(SUPPORT_LINUX_DVFS)
+#include "pvr_dvfs_device.h"
+#elif defined(SUPPORT_PDVFS)
+#include "pvr_dvfs_common.h"
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+
+#include "rgx_options.h"
+#include "srvinit.h"
+#include "rgxutils.h"
+
+#include "os_apphint.h"
+#include "pvrsrv_apphint.h"
+
+#include "pvrsrv_tlstreams.h"
+#include "tlstream.h"
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+#include "physmem_test.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#if defined(__linux__)
+#include "km_apphint.h"
+#endif /* defined(__linux__) */
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define INFINITE_SLEEP_TIMEOUT 0ULL
+#endif
+
+/*! Wait 100ms before retrying deferred clean-up again */
+#define CLEANUP_THREAD_WAIT_RETRY_TIMEOUT 100000ULL
+
+/*! Wait 8hrs when no deferred clean-up required. Allows a poll several times
+ * a day to check for any missed clean-up. */
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT INFINITE_SLEEP_TIMEOUT
+#else
+#define CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT 28800000000ULL
+#endif
+
+/*! When unloading try a few times to free everything remaining on the list */
+#define CLEANUP_THREAD_UNLOAD_RETRY 4
+
+#define PVRSRV_TL_CTRL_STREAM_SIZE 4096
+
+static PVRSRV_DATA	*gpsPVRSRVData;
+static IMG_UINT32 g_ui32InitFlags;
+
+/* mark which parts of Services were initialised */
+#define		INIT_DATA_ENABLE_PDUMPINIT	0x1U
+
+#define CLEANUP_QUEUE_TOTAL_DPF "QUEUED: %1.5d "
+#define CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE sizeof(CLEANUP_QUEUE_TOTAL_DPF)+1
+
+#define CLEANUP_STRING_SUMMARY_MAX_LEN (CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE+(PVRSRV_CLEANUP_TYPE_LAST * CLEANUP_TYPE_ITEM_DPF_MAX_SIZE))
+
+static IMG_CHAR *_ConcatCleanupString(IMG_CHAR *cleanupString)
+{
+	IMG_UINT32 uiLoop;
+	IMG_UINT32 strSize = CLEANUP_STRING_SUMMARY_MAX_LEN;
+
+	PVRSRV_DATA *psPVRSRVData =  PVRSRVGetPVRSRVData();
+
+	OSSNPrintf(cleanupString, CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE, CLEANUP_QUEUE_TOTAL_DPF,
+				OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued));
+
+	PVR_ASSERT(PVRSRV_CLEANUP_TYPE_CONNECTION == 1);
+
+	for (uiLoop=PVRSRV_CLEANUP_TYPE_CONNECTION; (strSize > CLEANUP_TYPE_ITEM_DPF_MAX_SIZE) && (uiLoop < PVRSRV_CLEANUP_TYPE_LAST); uiLoop++)
+	{
+		IMG_CHAR acTempQueued[CLEANUP_TYPE_ITEM_DPF_MAX_SIZE];
+		OSSNPrintf(acTempQueued, CLEANUP_TYPE_ITEM_DPF_MAX_SIZE, CLEANUP_TYPE_ITEM_DPF,
+				PVRSRVGetCleanupName(uiLoop),
+				OSAtomicRead(&psPVRSRVData->i32CleanupItemTypes[uiLoop]));
+		OSStringLCat(cleanupString, acTempQueued, strSize);
+	}
+
+	return cleanupString;
+}
+
+/* Callback to dump info of cleanup thread in debug_dump */
+static void CleanupThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                                  void *pvDumpDebugFile)
+{
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_DUMPDEBUG_LOG("    Number of deferred cleanup items: %s", _ConcatCleanupString(acCleanupString));
+
+	PVR_DUMPDEBUG_LOG("    Number of deferred cleanup items dropped after "
+					  "retry limit reached : %d",
+					  OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsNotCompleted));
+}
+
+static void _CleanupThreadDecrementStats(PVRSRV_DATA *psPVRSRVData,
+                                         PVRSRV_CLEANUP_TYPE eCleanupType)
+{
+#if defined(DEBUG)
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+#endif
+
+	PVR_ASSERT(OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued) >= 0);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "BEFORE REMOVING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+	OSAtomicDecrement(&psPVRSRVData->i32NumCleanupItemsQueued);
+
+	if ((eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) || ((eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item passed: %d", eCleanupType));
+	}
+	else
+	{
+		PVR_ASSERT(OSAtomicRead(&psPVRSRVData->i32CleanupItemTypes[eCleanupType]) >= 0);
+		OSAtomicDecrement(&psPVRSRVData->i32CleanupItemTypes[eCleanupType]);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "AFTER REMOVING ----- %s", _ConcatCleanupString(acCleanupString)));
+}
+
+#if defined(DEBUG)
+static void _CleanupThreadWorkListDump(PVRSRV_DATA *psPVRSRVData)
+{
+	DLLIST_NODE *psNode;
+	DLLIST_NODE *psNextNode;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+	char pszCleanupLog[128];
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* Iterate over all devices. */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		if (dllist_is_empty(&psDeviceNode->sCleanupThreadWorkList))
+		{
+			OSSNPrintf(pszCleanupLog, 128, "Dev_%p: CLEAN", psDeviceNode);
+			PVR_LOG(("%s", pszCleanupLog));
+			continue;
+		}
+
+		OSSNPrintf(pszCleanupLog, 128, "Dev_%p: TASKS", psDeviceNode);
+		PVR_LOG(("%s", pszCleanupLog));
+
+		/* Iterate over all cleanup items. */
+		dllist_foreach_node(&psDeviceNode->sCleanupThreadWorkList, psNode, psNextNode)
+		{
+			PVRSRV_CLEANUP_THREAD_WORK *psData;
+
+			psData = IMG_CONTAINER_OF(psNode, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+
+			PVR_ASSERT(psData != NULL);
+
+			if ((psData->eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) ||
+				((psData->eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item found: %d.", psData->eCleanupType));
+				continue;
+			}
+
+			OSSNPrintf(pszCleanupLog, 128, "+ %p: type %u, depends-HW %s (%s:%d)",
+					   psData,
+					   psData->eCleanupType,
+					   (psData->bDependsOnHW) ? "Yes" : " No",
+					   psData->pszFun, psData->ui32LineNum);
+			PVR_LOG(("%s", pszCleanupLog));
+		}
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+}
+#endif
+
+/* Add work to the cleanup thread work list.
+ * The work item will be executed by the cleanup thread
+ */
+#if defined(DEBUG)
+void PVRSRVCleanupThreadAddWork_Debug(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_CLEANUP_THREAD_WORK *psData,
+	const char *pszFun, const unsigned int ui32LineNum)
+#else
+void PVRSRVCleanupThreadAddWork_Int(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_CLEANUP_THREAD_WORK *psData)
+#endif
+{
+#if defined(DEBUG)
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+#endif
+
+	PVRSRV_DATA *psPVRSRVData;
+	PVRSRV_ERROR eError;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psData != NULL);
+
+	if ((psData->eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) ||
+		((psData->eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item passed: %d.", psData->eCleanupType));
+	}
+
+	psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "AFTER QUEUEING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+
+	/* Cleanup thread work item added during device destructing:
+	   1. bDependsOnHW: Item may use device resources. Not safe to clean.
+	   2. !bDependsOnHW: Clean item immediately to prevent memory leak. */
+	if (psPVRSRVData->bUnload
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		|| psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK
+#endif
+		|| (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING && !psData->bDependsOnHW))
+	{
+		CLEANUP_THREAD_FN pfnFree = psData->pfnFree;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Cleanup thread has already quit: doing work immediately"));
+
+		eError = pfnFree(psData->pvData);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "BEFORE ADDING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to free resource "
+						"(callback " IMG_PFN_FMTSPEC "). "
+						"Immediate free will not be retried.",
+						pfnFree));
+		}
+	}
+	else
+	{
+		/*
+		 * Access psData before putting it in the work list.
+		 * Cleanup thread will free psData after it is done with the cleanup.
+		 * We should consider psData potentially freed after it was put into the worklist.
+		 */
+
+		OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsQueued);
+		OSAtomicIncrement(&psPVRSRVData->i32CleanupItemTypes[psData->eCleanupType]);
+#if defined(DEBUG)
+		psData->pszFun = pszFun;
+		psData->ui32LineNum = ui32LineNum;
+#endif
+
+		/* add this work item to the list */
+		OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+		OSAtomicIncrement(&psDeviceNode->i32NumCleanupItems);
+		dllist_add_to_tail(&psDeviceNode->sCleanupThreadWorkList, &psData->sNode);
+		OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+		/* signal the cleanup thread to ensure this item gets processed */
+		eError = OSEventObjectSignal(psPVRSRVData->hCleanupEventObject);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "AFTER ADDING ----- %s", _ConcatCleanupString(acCleanupString)));
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+	}
+}
+
+void PVRSRVCleanupThreadWaitForDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(DEBUG)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+#endif
+	IMG_INT32 i32NumCleanupItems;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+
+	if (gpsPVRSRVData->hCleanupThread == NULL)
+	{
+		return;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+	{
+		PVRSRV_ERROR eError;
+
+		if (gpsPVRSRVData->hCleanupEventObject)
+		{
+			eError = OSEventObjectSignal(gpsPVRSRVData->hCleanupEventObject);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+
+		i32NumCleanupItems = OSAtomicRead(&psDeviceNode->i32NumCleanupItems);
+
+		if (i32NumCleanupItems == 0)
+		{
+#if defined(DEBUG)
+			OS_SPINLOCK_FLAGS uiFlags;
+
+			OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+			PVR_LOG_IF_FALSE(dllist_is_empty(&psDeviceNode->sCleanupThreadWorkList),
+							 "Cleanup thread work list is not empty");
+
+			OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+#endif
+			break;
+		}
+
+		OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US / OS_THREAD_DESTROY_RETRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (i32NumCleanupItems != 0)
+	{
+		PVR_LOG(("Failed to flush device cleanup queue. %d items remain.", i32NumCleanupItems));
+	}
+
+#if defined(DEBUG)
+	_CleanupThreadWorkListDump(psPVRSRVData);
+#endif
+}
+
+static INLINE DLLIST_NODE *_CleanupThreadWorkListLast(PVRSRV_DATA *psPVRSRVData)
+{
+	DLLIST_NODE *psNode = NULL;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* We treat the Host device node as the last node in the list. */
+	psNode = dllist_get_prev_node(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList);
+	if (psNode != NULL)
+	{
+		OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+		return psNode;
+	}
+
+	/* Iterate over all devices and find the last node of the last device. */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		DLLIST_NODE *psCurrNode = dllist_get_prev_node(&psDeviceNode->sCleanupThreadWorkList);
+		if (psCurrNode != NULL)
+		{
+			psNode = psCurrNode;
+		}
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psNode;
+}
+
+/* Pop an item from the head of the cleanup thread work lists.
+ *
+ * This pops an item in a round robin manner:
+ *
+ * 1. It starts from the first device and moves to next device if no cleanup
+ *    item found.
+ * 2. If no items were found in the regular devices bound lists it moves to the
+ *    Host device list.
+ * 3. In case there was nothing in the Host device items list but there was
+ *    something in the devices lists, it returns first found device bound item
+ *
+ * If `*ppsDeviceNode` is not NULL the lookup of the next cleanup item will
+ * start from the next devices after `*ppsDeviceNode`. This makes sure that
+ * we always pop a first item from each device and move to the next device.
+ * We prevent "starvation" of some devices this way.
+ */
+static INLINE DLLIST_NODE *_CleanupThreadWorkListPop(PVRSRV_DATA *psPVRSRVData,
+                                                     PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL, *psFirstDevice = NULL;
+	DLLIST_NODE *psNode = NULL, *psFirstNode = NULL;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* always need to start from the beginning of the list in case some of the
+	 * devices were already destroyed */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		/* not safe to run cleanup during deinit */
+		if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+		{
+			continue;
+		}
+
+		psNode = dllist_get_next_node(&psDeviceNode->sCleanupThreadWorkList);
+
+		/* remember first non-NULL node and device in case no items are found
+		 * on later devices */
+		if (psNode != NULL && psFirstNode == NULL)
+		{
+			psFirstNode = psNode;
+			psFirstDevice = psDeviceNode;
+		}
+
+		/* to ensure that further device don't get starved skip the last device
+		 * until all devices are processed and then move to non-device bound
+		 * items */
+		if (*ppsDeviceNode != NULL) {
+			if (psDeviceNode == *ppsDeviceNode)
+			{
+				*ppsDeviceNode = NULL;
+			}
+
+			/* in case this is the last device and next iteration exits the
+			 * loop */
+			psNode = NULL;
+
+			continue;
+		}
+
+		if (psNode != NULL)
+		{
+			*ppsDeviceNode = psDeviceNode;
+			dllist_remove_node(psNode);
+
+			break;
+		}
+	}
+
+	/* if no item found in the regular devices check also the Host device */
+	if (psNode == NULL)
+	{
+		psNode = dllist_get_next_node(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList);
+		if (psNode != NULL)
+		{
+			dllist_remove_node(psNode);
+		}
+		*ppsDeviceNode = psPVRSRVData->psHostMemDeviceNode;
+	}
+
+	/* if no item found for the Host device check if there was a cleanup item
+	 * in one of the previous devices, this starts processing items from the
+	 * beginning without one extra empty call to "wrap around" */
+	if (psNode == NULL && psFirstNode != NULL)
+	{
+		psNode = psFirstNode;
+		*ppsDeviceNode = psFirstDevice;
+		dllist_remove_node(psNode);
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psNode;
+}
+
+/* Process the cleanup thread work list */
+static IMG_BOOL _CleanupThreadProcessWorkList(PVRSRV_DATA *psPVRSRVData,
+                                              IMG_BOOL *pbUseHWTimeout)
+{
+	DLLIST_NODE *psNodeIter, *psNodeLast;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bNeedRetry = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_CLEANUP_TYPE eCleanupType;
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+
+	/* Reset HWTimeout Flag */
+	*pbUseHWTimeout = IMG_FALSE;
+
+	psNodeLast = _CleanupThreadWorkListLast(psPVRSRVData);
+	if (psNodeLast == NULL)
+	{
+		/* no elements to clean up */
+		return IMG_FALSE;
+	}
+
+	/* any callback functions which return error will be
+	 * moved to the back of the list, and additional items can be added
+	 * to the list at any time so we ensure we only iterate from the
+	 * head of the list to the current tail (since the tail may always
+	 * be changing)
+	 */
+	do
+	{
+		PVRSRV_CLEANUP_THREAD_WORK *psData;
+		CLEANUP_THREAD_FN pfnFree;
+		IMG_BOOL bRetry = IMG_FALSE;
+
+		psNodeIter = _CleanupThreadWorkListPop(psPVRSRVData, &psDeviceNode);
+		if (psNodeIter == NULL)
+		{
+			return IMG_FALSE;
+		}
+
+		psData = IMG_CONTAINER_OF(psNodeIter, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+
+		/* get the function pointer address here so we have access to it
+		 * in order to report the error in case of failure, without having
+		 * to depend on psData not having been freed
+		 */
+		pfnFree = psData->pfnFree;
+		eCleanupType = psData->eCleanupType;
+		eError = pfnFree(psData->pvData);
+
+		if (eError != PVRSRV_OK)
+		{
+			/* Move to back of the list, if this item's
+			 * retry count hasn't hit zero.
+			 */
+			if (CLEANUP_THREAD_IS_RETRY_TIMEOUT(psData))
+			{
+				if (CLEANUP_THREAD_RETRY_TIMEOUT_NOT_REACHED(psData))
+				{
+					bNeedRetry = IMG_TRUE;
+					bRetry = IMG_TRUE;
+					/* If any items require retry and are HW dependent
+					 * use the HW timeout
+					 */
+					if (psData->bDependsOnHW)
+					{
+						*pbUseHWTimeout = psData->bDependsOnHW;
+					}
+				}
+			}
+			else
+			{
+				if (psData->ui32RetryCount > 0)
+				{
+					psData->ui32RetryCount--;
+					bNeedRetry = IMG_TRUE;
+					bRetry = IMG_TRUE;
+					/* If any items require retry and are HW dependent
+					 * use the HW timeout
+					 */
+					if (psData->bDependsOnHW)
+					{
+						*pbUseHWTimeout = psData->bDependsOnHW;
+					}
+				}
+			}
+
+			/* If the work depends on HW then we should add it to the back of the list,
+			 * the cleanup thread will sleep for longer if required and the next MISR
+			 * from the device will wake the task again in which it might be ready.
+			 */
+			if (bRetry || psData->bDependsOnHW)
+			{
+				/* If any items on the work list depend on HW
+				 * and didn't get cleaned up.
+				 */
+				OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+				dllist_add_to_tail(&psDeviceNode->sCleanupThreadWorkList, psNodeIter);
+				OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to free resource (callback " IMG_PFN_FMTSPEC "). "
+				        "Retry limit reached", pfnFree));
+				OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsNotCompleted);
+				OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+				/* Dropping item */
+				_CleanupThreadDecrementStats(psPVRSRVData, eCleanupType);
+			}
+		}
+		else
+		{
+			/* Ok returned */
+			OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+			_CleanupThreadDecrementStats(psPVRSRVData, eCleanupType);
+		}
+	} while (psNodeIter != NULL && psNodeIter != psNodeLast);
+
+	return bNeedRetry;
+}
+
+// #define CLEANUP_DPFL PVR_DBG_WARNING
+#define CLEANUP_DPFL    PVR_DBG_MESSAGE
+
+/* Create/initialise data required by the cleanup thread,
+ * before the cleanup thread is started
+ */
+static PVRSRV_ERROR _CleanupThreadPrepare(PVRSRV_DATA *psPVRSRVData)
+{
+	PVRSRV_ERROR eError;
+
+	/* Create the clean up event object */
+
+	eError = OSEventObjectCreate("PVRSRV_CLEANUP_EVENTOBJECT", &gpsPVRSRVData->hCleanupEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Exit);
+
+	/* initialise the mutex and linked list required for the cleanup thread work list */
+
+	eError = OSSpinLockCreate(&psPVRSRVData->hCleanupThreadWorkListLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", Exit);
+
+Exit:
+	return eError;
+}
+
+static void CleanupThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_BOOL     bRetryWorkList = IMG_FALSE;
+	IMG_BOOL     bUseHWTimeout = IMG_FALSE;
+	IMG_HANDLE	 hOSEvent;
+	PVRSRV_ERROR eRc;
+	IMG_UINT32 uiUnloadRetry = 0;
+	DLLIST_NODE *psNodeIter, *psNodeLast;
+
+	/* Store the process id (pid) of the clean-up thread */
+	psPVRSRVData->cleanupThreadPid = OSGetCurrentProcessID();
+	psPVRSRVData->cleanupThreadTid = OSGetCurrentThreadID();
+
+	PVR_DPF((CLEANUP_DPFL, "CleanupThread: thread starting... "));
+
+	/* Open an event on the clean up event object so we can listen on it,
+	 * abort the clean up thread and driver if this fails.
+	 */
+	eRc = OSEventObjectOpen(psPVRSRVData->hCleanupEventObject, &hOSEvent);
+	PVR_ASSERT(eRc == PVRSRV_OK);
+
+	/* While the driver is in a good state and is not being unloaded
+	 * try to free any deferred items when signalled
+	 */
+	while (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
+	{
+		IMG_UINT64 ui64Timeoutus;
+		if (psPVRSRVData->bUnload)
+		{
+			if (dllist_is_empty(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList) ||
+			    uiUnloadRetry > CLEANUP_THREAD_UNLOAD_RETRY)
+			{
+				break;
+			}
+			uiUnloadRetry++;
+		}
+
+		/* Wait until signalled for deferred clean up OR wait for a
+		 * short period if the previous deferred clean up was not able
+		 * to release all the resources before trying again.
+		 * Bridge lock re-acquired on our behalf before the wait call returns.
+		 */
+
+		if (bRetryWorkList && bUseHWTimeout)
+		{
+			/* If item depends on HW we are
+			 * waiting for GPU work to finish, so
+			 * use MAX_HW_TIME_US as timeout (this
+			 * will be set appropriately when
+			 * running on systems with emulated
+			 * hardware, etc).
+			 */
+			ui64Timeoutus = MAX_HW_TIME_US;
+		}
+		else
+		{
+			/* Use the default retry timeout. */
+			ui64Timeoutus = CLEANUP_THREAD_WAIT_RETRY_TIMEOUT;
+		}
+
+		eRc = OSEventObjectWaitKernel(hOSEvent,
+				bRetryWorkList ?
+				ui64Timeoutus :
+				CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT);
+		if (eRc == PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_DPF((CLEANUP_DPFL, "CleanupThread: wait timeout"));
+		}
+		else if (eRc == PVRSRV_OK)
+		{
+			PVR_DPF((CLEANUP_DPFL, "CleanupThread: wait OK, signal received"));
+		}
+		else
+		{
+			PVR_LOG_ERROR(eRc, "OSEventObjectWaitKernel");
+		}
+
+		bRetryWorkList = _CleanupThreadProcessWorkList(psPVRSRVData,
+		                                               &bUseHWTimeout);
+	}
+
+	psNodeLast = _CleanupThreadWorkListLast(psPVRSRVData);
+	if (psNodeLast != NULL)
+	{
+		do
+		{
+			PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+			PVRSRV_CLEANUP_THREAD_WORK *psData;
+			psNodeIter = _CleanupThreadWorkListPop(psPVRSRVData, &psDeviceNode);
+			if (psNodeIter == NULL)
+			{
+				break;
+			}
+
+			psData = IMG_CONTAINER_OF(psNodeIter, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+			OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsNotCompleted);
+			OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+			/* Dropping item */
+			_CleanupThreadDecrementStats(psPVRSRVData, psData->eCleanupType);
+		}
+		while (psNodeIter != NULL && psNodeIter != psNodeLast);
+
+		PVR_DPF((PVR_DBG_ERROR, "Cleanup Thread Failed to free %d resources", OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsNotCompleted)));
+	}
+
+	OSSpinLockDestroy(psPVRSRVData->hCleanupThreadWorkListLock);
+
+	eRc = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eRc, "OSEventObjectClose");
+
+	PVR_DPF((CLEANUP_DPFL, "CleanupThread: thread ending... "));
+}
+
+IMG_PID PVRSRVCleanupThreadGetPid(void)
+{
+	return gpsPVRSRVData->cleanupThreadPid;
+}
+
+uintptr_t PVRSRVCleanupThreadGetTid(void)
+{
+	return gpsPVRSRVData->cleanupThreadTid;
+}
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*
+ * Firmware is unresponsive.
+ * The Host will initiate a recovery process during which the
+ * Firmware and GPU are reset and returned to a working state.
+ */
+static PVRSRV_ERROR HandleFwHostSideRecovery(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32CtxIdx = 0U;
+	IMG_UINT32 ui32Nodes  = 0U;
+
+	OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+	/* Get the number of nodes in a linked list */
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		++ui32Nodes;
+	}
+
+	/* Any client contexts active at the moment? */
+	if (ui32Nodes > 0U)
+	{
+		/* Free the active context buffer previously allocated */
+		if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+			psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+		}
+
+		/* Setup allocations to store the active contexts */
+		eError = RGXSetupFwAllocation(psDevInfo,
+				RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+				(ui32Nodes + 1) * sizeof(RGXFWIF_ACTIVE_CONTEXT_BUF_DATA),
+				"FwSysActiveContextBufData",
+				&psDevInfo->psRGXFWIfActiveContextBufDesc,
+				(void *)  &psDevInfo->psRGXFWIfSysInit->sActiveContextBufBase.ui32Addr,
+				(void **) &psDevInfo->psRGXFWIfActiveContextBuf,
+				RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation",Error);
+
+		/* List of contexts to be rekicked by FW powering up the device */
+		dllist_foreach_node_backwards(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+		{
+			psDevInfo->psRGXFWIfActiveContextBuf[ui32CtxIdx].psContext =
+					RGXGetFWCommonContextAddrFromServerCommonCtx(psDevInfo, psNode);
+			++ui32CtxIdx;
+		}
+		/* Null context as the terminator marker */
+		psDevInfo->psRGXFWIfActiveContextBuf[ui32CtxIdx].psContext.ui32Addr = 0;
+	}
+	OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+
+	/* Host can't expect a response on power-down request as FW is in BAD state */
+	eError = PVRSRVSetDeviceCurrentPowerState(psDeviceNode->psPowerDev, PVRSRV_DEV_POWER_STATE_OFF);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVSetDeviceCurrentPowerState OFF", Error);
+
+	/* Flag to be set to notify FW while recovering from crash */
+	psDevInfo->psRGXFWIfSysInit->bFwHostRecoveryMode = IMG_TRUE;
+
+	/* Flush here because we have setup a fw alloc addr in the structure earlier */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit, FLUSH);
+
+	/* Power-on the device resetting GPU & FW */
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+			PVRSRV_DEV_POWER_STATE_ON,
+			PVRSRV_POWER_FLAGS_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVSetDevicePowerStateKM ON", Error);
+	OSLockRelease(psDeviceNode->hPowerLock);
+
+Error:
+	return eError;
+}
+#endif
+
+
+void PVRSRVDeviceSetState(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_STATE eNewDevState)
+{
+	if (eNewDevState == psDeviceNode->eDevState)
+	{
+		return;
+	}
+
+	switch (eNewDevState)
+	{
+		case PVRSRV_DEVICE_STATE_PCI_ERROR:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetDebugDevStateString(eNewDevState)));
+			psDeviceNode->eDevState = eNewDevState;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+			PMRFreeZombies(psDeviceNode);
+#endif
+			break;
+		}
+		case PVRSRV_DEVICE_STATE_CREATING:
+		case PVRSRV_DEVICE_STATE_CREATED:
+		case PVRSRV_DEVICE_STATE_ACTIVE:
+		case PVRSRV_DEVICE_STATE_FROZEN:
+		case PVRSRV_DEVICE_STATE_DEINIT:
+		case PVRSRV_DEVICE_STATE_DESTRUCTING:
+		case PVRSRV_DEVICE_STATE_BAD:
+		{
+			/* PCI_ERROR is a terminal state. Reload driver to recover. */
+			if (eNewDevState != PVRSRV_DEVICE_STATE_PCI_ERROR)
+			{
+				psDeviceNode->eDevState = eNewDevState;
+			}
+			break;
+		}
+		case PVRSRV_DEVICE_STATE_UNDEFINED:
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unknown state (%d)", __func__, eNewDevState));
+			break;
+		}
+	}
+}
+
+static void DevicesWatchdogThread_ForEachVaCb(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  va_list va)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+#endif
+	PVRSRV_DEVICE_HEALTH_STATUS *pePreviousHealthStatus, eHealthStatus;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_DEBUG_DUMP_STATUS eDebugDumpState;
+	IMG_BOOL bCheckAfterTimePassed;
+
+	pePreviousHealthStatus = va_arg(va, PVRSRV_DEVICE_HEALTH_STATUS *);
+	/* IMG_BOOL (i.e. bool) is converted to int during default argument promotion
+	 * in variadic argument list. Thus, fetch an int to get IMG_BOOL */
+	bCheckAfterTimePassed = (IMG_BOOL) va_arg(va, int);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		return;
+	}
+
+	/* Block here if needed */
+	PVRSRVBlockIfFrozen(psDeviceNode);
+
+	if (psDeviceNode->pfnUpdateHealthStatus != NULL)
+	{
+		eError = psDeviceNode->pfnUpdateHealthStatus(psDeviceNode, bCheckAfterTimePassed);
+		PVR_WARN_IF_ERROR(eError, "pfnUpdateHealthStatus");
+	}
+	eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+	if (eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		if (eHealthStatus != *pePreviousHealthStatus)
+		{
+#if defined(SUPPORT_RGX)
+			if (!(psDevInfo->ui32DeviceFlags &
+				  RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN))
+#else
+			/* In this case we don't have an RGX device */
+			if (eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED)
+#endif
+			{
+				PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: "
+						 "Device status not OK!!!"));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX,
+								   NULL, NULL);
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+				HandleFwHostSideRecovery(psDeviceNode);
+#endif
+			}
+		}
+	}
+
+	*pePreviousHealthStatus = eHealthStatus;
+
+	/* Have we received request from FW to capture debug dump(could be due to HWR) */
+	eDebugDumpState = (PVRSRV_DEVICE_DEBUG_DUMP_STATUS)OSAtomicCompareExchange(
+						&psDeviceNode->eDebugDumpRequested,
+						PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE,
+						PVRSRV_DEVICE_DEBUG_DUMP_NONE);
+	if (PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE == eDebugDumpState)
+	{
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	}
+
+}
+
+#if defined(SUPPORT_RGX)
+static void HWPerfPeriodicHostEventsThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+
+	eError = OSEventObjectOpen(psPVRSRVData->hHWPerfHostPeriodicEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload && !psPVRSRVData->bHWPerfHostThreadStop)
+#else
+	while (!psPVRSRVData->bUnload && !psPVRSRVData->bHWPerfHostThreadStop)
+#endif
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode;
+		IMG_BOOL bInfiniteSleep = IMG_TRUE;
+
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)psPVRSRVData->ui32HWPerfHostThreadTimeout * 1000);
+		if (eError == PVRSRV_OK && (psPVRSRVData->bUnload || psPVRSRVData->bHWPerfHostThreadStop))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "HWPerfPeriodicHostEventsThread: Shutdown event received."));
+			break;
+		}
+
+		for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+		     psDeviceNode != NULL;
+		     psDeviceNode = psDeviceNode->psNext)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+			/* If the psDevInfo or hHWPerfHostStream are NULL it most
+			 * likely means that this device or stream has not been
+			 * initialised yet, so just skip */
+			if (psDevInfo == NULL || psDevInfo->hHWPerfHostStream == NULL)
+			{
+				continue;
+			}
+
+			/* Check if the HWPerf host stream is open for reading before writing
+			 * a packet, this covers cases where the event filter is not zeroed
+			 * before a reader disconnects. */
+			if (TLStreamIsOpenForReading(psDevInfo->hHWPerfHostStream))
+			{
+				/* As long as any of the streams is opened don't go into
+				 * indefinite sleep. */
+				bInfiniteSleep = IMG_FALSE;
+#if defined(SUPPORT_RGX)
+				RGXSRV_HWPERF_HOST_INFO(psDevInfo, RGX_HWPERF_INFO_EV_MEM64_USAGE);
+#endif
+			}
+		}
+
+		if (bInfiniteSleep)
+		{
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			psPVRSRVData->ui32HWPerfHostThreadTimeout = INFINITE_SLEEP_TIMEOUT;
+#else
+			/* Use an 8 hour timeout if indefinite sleep is not supported. */
+			psPVRSRVData->ui32HWPerfHostThreadTimeout = 60 * 60 * 8 * 1000;
+#endif
+		}
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+#endif
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+typedef enum
+{
+	DWT_ST_INIT,
+	DWT_ST_SLEEP_POWERON,
+	DWT_ST_SLEEP_POWEROFF,
+	DWT_ST_SLEEP_DEFERRED,
+	DWT_ST_FINAL
+} DWT_STATE;
+
+typedef enum
+{
+	DWT_SIG_POWERON,
+	DWT_SIG_POWEROFF,
+	DWT_SIG_TIMEOUT,
+	DWT_SIG_UNLOAD,
+	DWT_SIG_ERROR
+} DWT_SIGNAL;
+
+static inline IMG_BOOL _DwtIsPowerOn(PVRSRV_DATA *psPVRSRVData)
+{
+	return List_PVRSRV_DEVICE_NODE_IMG_BOOL_Any(psPVRSRVData->psDeviceNodeList,
+	                                            PVRSRVIsDevicePowered);
+}
+
+static inline void _DwtCheckHealthStatus(PVRSRV_DATA *psPVRSRVData,
+                                         PVRSRV_DEVICE_HEALTH_STATUS *peStatus,
+                                         IMG_BOOL bTimeOut)
+{
+	List_PVRSRV_DEVICE_NODE_ForEach_va(psPVRSRVData->psDeviceNodeList,
+	                                   DevicesWatchdogThread_ForEachVaCb,
+	                                   peStatus,
+	                                   bTimeOut);
+}
+
+static DWT_SIGNAL _DwtWait(PVRSRV_DATA *psPVRSRVData, IMG_HANDLE hOSEvent,
+                           IMG_UINT32 ui32Timeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64) ui32Timeout * 1000);
+
+
+	if (eError == PVRSRV_OK)
+	{
+		if (psPVRSRVData->bUnload)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Shutdown event"
+			        " received."));
+			return DWT_SIG_UNLOAD;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power state "
+			        "change event received."));
+
+			if (_DwtIsPowerOn(psPVRSRVData))
+			{
+				return DWT_SIG_POWERON;
+			}
+			else
+			{
+				return DWT_SIG_POWEROFF;
+			}
+		}
+	}
+	else if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		return DWT_SIG_TIMEOUT;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: Error (%d) when"
+	        " waiting for event!", eError));
+	return DWT_SIG_ERROR;
+}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+
+static void DevicesWatchdogThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	PVRSRV_DEVICE_HEALTH_STATUS ePreviousHealthStatus = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	DWT_STATE eState = DWT_ST_INIT;
+	const IMG_UINT32 ui32OnTimeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	const IMG_UINT32 ui32OffTimeout = INFINITE_SLEEP_TIMEOUT;
+#else
+	IMG_UINT32 ui32Timeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	/* Flag used to defer the sleep timeout change by 1 loop iteration.
+	 * This helps to ensure at least two health checks are performed before a long sleep.
+	 */
+	IMG_BOOL bDoDeferredTimeoutChange = IMG_FALSE;
+#endif
+
+	PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power off sleep time: %d.",
+			DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT));
+
+	/* Open an event on the devices watchdog event object so we can listen on it
+	   and abort the devices watchdog thread. */
+	eError = OSEventObjectOpen(psPVRSRVData->hDevicesWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+	/* Loop continuously checking the device status every few seconds. */
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload)
+#else
+	while (!psPVRSRVData->bUnload)
+#endif
+	{
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+		switch (eState)
+		{
+			case DWT_ST_INIT:
+			{
+				if (_DwtIsPowerOn(psPVRSRVData))
+				{
+					eState = DWT_ST_SLEEP_POWERON;
+				}
+				else
+				{
+					eState = DWT_ST_SLEEP_POWEROFF;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_POWERON:
+			{
+				DWT_SIGNAL eSignal = _DwtWait(psPVRSRVData, hOSEvent,
+				                                    ui32OnTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_POWEROFF:
+						eState = DWT_ST_SLEEP_DEFERRED;
+						break;
+					case DWT_SIG_TIMEOUT:
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_TRUE);
+						/* self-transition */
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_POWEROFF:
+			{
+				DWT_SIGNAL eSignal = _DwtWait(psPVRSRVData, hOSEvent,
+				                                    ui32OffTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						eState = DWT_ST_SLEEP_POWERON;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_POWEROFF:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_TIMEOUT:
+						/* self-transition */
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_TRUE);
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_DEFERRED:
+			{
+				DWT_SIGNAL eSignal =_DwtWait(psPVRSRVData, hOSEvent,
+				                                   ui32OnTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						eState = DWT_ST_SLEEP_POWERON;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_POWEROFF:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_TIMEOUT:
+						eState = DWT_ST_SLEEP_POWEROFF;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_FINAL:
+				/* the loop should terminate on next spin if this state is
+				 * reached so nothing to do here. */
+				break;
+		}
+
+#else /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+		IMG_BOOL bPwrIsOn = IMG_FALSE;
+		IMG_BOOL bTimeOut = IMG_FALSE;
+
+		/* Wait time between polls (done at the start of the loop to allow devices
+		   to initialise) or for the event signal (shutdown or power on). */
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+
+		if (eError == PVRSRV_OK)
+		{
+			if (psPVRSRVData->bUnload)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Shutdown event received."));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power state change event received."));
+			}
+		}
+		else if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			/* If timeout do nothing otherwise print warning message. */
+			PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: "
+					"Error (%d) when waiting for event!", eError));
+		}
+		else
+		{
+			bTimeOut = IMG_TRUE;
+		}
+
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		List_PVRSRV_DEVICE_NODE_ForEach_va(psPVRSRVData->psDeviceNodeList,
+		                                   DevicesWatchdogThread_ForEachVaCb,
+		                                   &ePreviousHealthStatus,
+		                                   bTimeOut);
+		bPwrIsOn = List_PVRSRV_DEVICE_NODE_IMG_BOOL_Any(psPVRSRVData->psDeviceNodeList,
+														PVRSRVIsDevicePowered);
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+		if (bPwrIsOn || psPVRSRVData->ui32DevicesWatchdogPwrTrans)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans = 0;
+			ui32Timeout = psPVRSRVData->ui32DevicesWatchdogTimeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+			bDoDeferredTimeoutChange = IMG_FALSE;
+		}
+		else
+		{
+			/* First, check if the previous loop iteration signalled a need to change the timeout period */
+			if (bDoDeferredTimeoutChange)
+			{
+				ui32Timeout = psPVRSRVData->ui32DevicesWatchdogTimeout = DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT;
+				bDoDeferredTimeoutChange = IMG_FALSE;
+			}
+			else
+			{
+				/* Signal that we need to change the sleep timeout in the next loop iteration
+				 * to allow the device health check code a further iteration at the current
+				 * sleep timeout in order to determine bad health (e.g. stalled cCCB) by
+				 * comparing past and current state snapshots */
+				bDoDeferredTimeoutChange = IMG_TRUE;
+			}
+		}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+#if defined(SUPPORT_AUTOVZ)
+static void AutoVzWatchdogThread_ForEachCb(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (psDeviceNode->pfnUpdateAutoVzWatchdog != NULL)
+	{
+		psDeviceNode->pfnUpdateAutoVzWatchdog(psDeviceNode);
+	}
+}
+
+static void AutoVzWatchdogThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Timeout = PVR_AUTOVZ_WDG_PERIOD_MS / 3;
+
+	/* Open an event on the devices watchdog event object so we can listen on it
+	   and abort the devices watchdog thread. */
+	eError = OSEventObjectOpen(psPVRSRVData->hAutoVzWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload)
+#else
+	while (!psPVRSRVData->bUnload)
+#endif
+	{
+		/* Wait time between polls (done at the start of the loop to allow devices
+		   to initialise) or for the event signal (shutdown or power on). */
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+
+		List_PVRSRV_DEVICE_NODE_ForEach(psPVRSRVData->psDeviceNodeList,
+		                                AutoVzWatchdogThread_ForEachCb);
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_DATA *PVRSRVGetPVRSRVData(void)
+{
+	return gpsPVRSRVData;
+}
+
+static PVRSRV_ERROR InitialiseInfoPageTimeouts(PVRSRV_DATA *psPVRSRVData)
+{
+	if (NULL == psPVRSRVData)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_VALUE_RETRIES] = WAIT_TRY_COUNT;
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_VALUE_TIMEOUT_MS] =
+		((MAX_HW_TIME_US / 10000) + 1000);
+		/* TIMEOUT_INFO_VALUE_TIMEOUT_MS resolves to...
+			vp       : 2000  + 1000
+			emu      : 2000  + 1000
+			rgx_nohw : 50    + 1000
+			plato    : 30000 + 1000 (VIRTUAL_PLATFORM or EMULATOR)
+			           50    + 1000 (otherwise)
+		*/
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_CONDITION_RETRIES] = 5;
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_CONDITION_TIMEOUT_MS] =
+		((MAX_HW_TIME_US / 10000) + 100);
+		/* TIMEOUT_INFO_CONDITION_TIMEOUT_MS resolves to...
+			vp       : 2000  + 100
+			emu      : 2000  + 100
+			rgx_nohw : 50    + 100
+			plato    : 30000 + 100 (VIRTUAL_PLATFORM or EMULATOR)
+			           50    + 100 (otherwise)
+		*/
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_TASK_QUEUE_RETRIES] = 10;
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_TASK_QUEUE_FLUSH_TIMEOUT_MS] =
+		MAX_HW_TIME_US / 1000U;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PopulateInfoPageBridges(PVRSRV_DATA *psPVRSRVData)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psPVRSRVData);
+
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_PVR_BRIDGES] = gui32PVRBridges;
+
+#if defined(SUPPORT_RGX)
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_RGX_BRIDGES] = gui32RGXBridges;
+#else
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_RGX_BRIDGES] = 0;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static void _ThreadsDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDbgRequestHandle,
+                                       IMG_UINT32 ui32VerbLevel,
+                                       DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                       void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(hDbgRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+		PVR_DUMPDEBUG_LOG("------[ Server Thread Summary ]------");
+		OSThreadDumpInfo(pfnDumpDebugPrintf, pvDumpDebugFile);
+	}
+}
+
+#define PVRVZMODE_STR_SIZE_MAX (10U)
+#define PVRVZMODE_BUFFER_SIZE (((PVRSRV_MAX_DEVICES)*(PVRVZMODE_STR_SIZE_MAX))+1)
+
+static const IMG_CHAR *const asModeStrings[] =
+{
+	"native",
+	"host",
+	"guest",
+	"default"
+};
+
+static void _InitDefaultVzDriverMode(PVRSRV_DATA *psPVRSRVData, void *pvAppHintState)
+{
+	char *pszMode, pszModeAppHint[PVRVZMODE_BUFFER_SIZE];
+	static const IMG_CHAR *pszDefault = "";
+	PVRSRV_DRIVER_MODE aeInitModes[PVRSRV_MAX_DEVICES];
+	IMG_BOOL bRet;
+	IMG_UINT ui32Dev;
+
+	for (ui32Dev = 0; ui32Dev < PVRSRV_MAX_DEVICES; ui32Dev++)
+	{
+		aeInitModes[ui32Dev] = DRIVER_MODE_DEFAULT;
+	}
+
+	pszMode = pszModeAppHint;
+	pszModeAppHint[0] = '\0';
+
+	bRet = (IMG_BOOL) OSGetAppHintSTRING(APPHINT_NO_DEVICE,
+						pvAppHintState,
+						DriverMode,
+						pszDefault,
+						pszModeAppHint,
+						PVRVZMODE_BUFFER_SIZE);
+	PVR_GOTO_IF_FALSE(bRet, uninit_default);
+
+	ui32Dev = 0;
+
+	while (*pszMode && (ui32Dev < PVRSRV_MAX_DEVICES))
+	{
+		PVRSRV_DRIVER_MODE eMode;
+
+		while (*pszMode == ',')
+		{
+			/* skip commas */
+			pszMode++;
+		}
+
+		for (eMode = DRIVER_MODE_NATIVE; eMode <= DRIVER_MODE_DEFAULT; eMode++)
+		{
+			/* if no matching strings found, device's mode remains DEFAULT */
+			if (OSStringNCompare(pszMode,
+								 asModeStrings[eMode],
+								 OSStringLength(asModeStrings[eMode])) == 0)
+			{
+				aeInitModes[ui32Dev] = eMode;
+			}
+		}
+
+		while ((*pszMode != '\0') && (*pszMode != ','))
+		{
+			/* advance until the next command or string end */
+			pszMode++;
+		}
+
+		ui32Dev++;
+	}
+
+uninit_default:
+	for (ui32Dev = 0; ui32Dev < PVRSRV_MAX_DEVICES; ui32Dev++)
+	{
+		psPVRSRVData->aeModuleParamDriverMode[ui32Dev] = aeInitModes[ui32Dev];
+	}
+}
+
+PVRSRV_ERROR
+PVRSRVCommonDriverInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVRSRV_DATA	*psPVRSRVData = NULL;
+
+	IMG_UINT32 ui32AppHintCleanupThreadPriority;
+	IMG_UINT32 ui32AppHintWatchdogThreadPriority;
+	IMG_BOOL bEnablePageFaultDebug;
+	IMG_BOOL bEnableFullSyncTracking;
+
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+
+	/*
+	 * As this function performs one time driver initialisation, use the
+	 * Services global device-independent data to determine whether or not
+	 * this function has already been called.
+	 */
+	if (gpsPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Driver already initialised", __func__));
+		return PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+
+#if defined(SUPPORT_RGX)
+	RGXBridgeDriverInit();
+#endif
+
+	eError = DIInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	eError = PVRSRVStatsInitialise();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	eError = PVRDIImplBrgRegister();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	eError = HTB_CreateDIEntry();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/*
+	 * Allocate the device-independent data. Do NOT track this allocation.
+	 */
+	psPVRSRVData = OSAllocZMemNoStats(sizeof(*gpsPVRSRVData));
+	PVR_GOTO_IF_NOMEM(psPVRSRVData, eError, Error);
+
+	/* Now it is set up, point gpsPVRSRVData to the actual data */
+	gpsPVRSRVData = psPVRSRVData;
+
+	eError = OSWRLockCreate(&gpsPVRSRVData->hDeviceNodeListLock);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Register the driver context debug table */
+	eError = PVRSRVRegisterDriverDbgTable();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Register the Server Thread Debug notifier */
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&gpsPVRSRVData->hThreadsDbgReqNotify,
+		                                          _ThreadsDebugRequestNotify,
+		                                          DEBUG_REQUEST_SRV,
+		                                          NULL);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/*
+	 * Initialise the server bridges
+	 */
+	eError = ServerBridgeInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = DevmemIntInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = DebugCommonInitDriver();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = BridgeDispatcherInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Init any OS specific's */
+	eError = OSInitEnvData();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Early init. server cache maintenance */
+	eError = CacheOpInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RIInitKM();
+#endif
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG;
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnablePageFaultDebug,
+			&bAppHintDefault, &bEnablePageFaultDebug);
+	OSFreeAppHintState(pvAppHintState);
+
+	if (bEnablePageFaultDebug)
+	{
+		eError = DevicememHistoryInitKM();
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevicememHistoryInitKM", Error);
+	}
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	eError = CPUMappingHistoryInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	eError = PMRInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = DCInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	/* Initialise overall system state */
+	gpsPVRSRVData->eServicesState = PVRSRV_SERVICES_STATE_OK;
+
+	/* Create an event object */
+	eError = OSEventObjectCreate("PVRSRV_GLOBAL_EVENTOBJECT", &gpsPVRSRVData->hGlobalEventObject);
+	PVR_GOTO_IF_ERROR(eError, Error);
+	gpsPVRSRVData->ui32GEOConsecutiveTimeouts = 0;
+
+	eError = PVRSRVCmdCompleteInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = PVRSRVHandleInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = PVRSRV_APPHINT_CLEANUPTHREADPRIORITY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, CleanupThreadPriority,
+	                     &ui32AppHintDefault, &ui32AppHintCleanupThreadPriority);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, WatchdogThreadPriority,
+	                     &ui32AppHintDefault, &ui32AppHintWatchdogThreadPriority);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnableFullSyncTracking,
+			&bAppHintDefault, &bEnableFullSyncTracking);
+
+	_InitDefaultVzDriverMode(gpsPVRSRVData, pvAppHintState);
+
+
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	eError = HostMemDeviceCreate(&gpsPVRSRVData->psHostMemDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = _CleanupThreadPrepare(gpsPVRSRVData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CleanupThreadPrepare", Error);
+
+	/* Create a thread which is used to do the deferred cleanup */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hCleanupThread,
+	                                "pvr_defer_free",
+	                                CleanupThread,
+	                                CleanupThreadDumpInfo,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                ui32AppHintCleanupThreadPriority);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:1", Error);
+
+	/* Create the devices watchdog event object */
+	eError = OSEventObjectCreate("PVRSRV_DEVICESWATCHDOG_EVENTOBJECT", &gpsPVRSRVData->hDevicesWatchdogEvObj);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Error);
+
+	/* Create a thread which is used to detect fatal errors */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hDevicesWatchdogThread,
+	                                "pvr_device_wdg",
+	                                DevicesWatchdogThread,
+	                                NULL,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                ui32AppHintWatchdogThreadPriority);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:2", Error);
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Create the devices watchdog event object */
+	eError = OSEventObjectCreate("PVRSRV_AUTOVZ_WATCHDOG_EVENTOBJECT", &gpsPVRSRVData->hAutoVzWatchdogEvObj);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Error);
+
+	/* Create a thread that maintains the FW-KM connection by regularly updating the virtualization watchdog */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hAutoVzWatchdogThread,
+	                                "pvr_autovz_wdg",
+	                                AutoVzWatchdogThread,
+	                                NULL,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                OS_THREAD_HIGHEST_PRIORITY);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:3", Error);
+#endif /* SUPPORT_AUTOVZ */
+
+#if defined(SUPPORT_RGX)
+	eError = OSLockCreate(&gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", Error);
+#endif
+
+	/* Initialise the Transport Layer */
+	eError = TLInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Initialise pdump */
+	eError = PDUMPINIT();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	g_ui32InitFlags |= INIT_DATA_ENABLE_PDUMPINIT;
+
+	/* Initialise TL control stream */
+	eError = TLStreamCreate(&psPVRSRVData->hTLCtrlStream,
+	                        PVRSRV_TL_CTRL_STREAM,
+	                        PVRSRV_TL_CTRL_STREAM_SIZE,
+	                        TL_OPMODE_DROP_OLDEST,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+                            NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "TLStreamCreate");
+		psPVRSRVData->hTLCtrlStream = NULL;
+	}
+
+	eError = InfoPageCreate(psPVRSRVData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InfoPageCreate", Error);
+
+
+	/* Initialise the Timeout Info */
+	eError = InitialiseInfoPageTimeouts(psPVRSRVData);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = PopulateInfoPageBridges(psPVRSRVData);
+
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	if (bEnableFullSyncTracking)
+	{
+		psPVRSRVData->pui32InfoPage[DEBUG_FEATURE_FLAGS] |= DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED;
+	}
+	if (bEnablePageFaultDebug)
+	{
+		psPVRSRVData->pui32InfoPage[DEBUG_FEATURE_FLAGS] |= DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED;
+	}
+
+	{
+		IMG_UINT64 *pui64InfoPage;
+
+		pui64InfoPage = IMG_OFFSET_ADDR_DW(psPVRSRVData->pui32InfoPage,
+		                                   DEVMEM_INFO_PHYS_BUF_MAX_SIZE);
+		*pui64InfoPage = PMR_MAX_SUPPORTED_SIZE;
+	}
+
+	/* Initialise the Host Trace Buffer */
+	eError = HTBInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(SUPPORT_RGX)
+	RGXHWPerfClientInitAppHintCallbacks();
+#endif
+
+	eError = OSLockCreate(&psPVRSRVData->hClientStreamTableLock);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	psPVRSRVData->psClientStreamTable = HASH_Create_Extended(16, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+	                                                         HASH_Djb2_Hash, HASH_Djb2_Compare);
+	PVR_GOTO_IF_NOMEM(psPVRSRVData->psClientStreamTable, eError, Error);
+
+	/* Late init. client cache maintenance via info. page */
+	eError = CacheOpInit2();
+	PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpInit2", Error);
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	eError = SyncFbRegisterSyncFunctions();
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncFbRegisterSyncFunctions", Error);
+#endif
+
+#if defined(PDUMP)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	/* If initialising the device on first connection, we will
+	 * bind PDump capture to the first device we connect to later.
+	 */
+	psPVRSRVData->ui32PDumpBoundDevice = PVRSRV_MAX_DEVICES;
+#else
+	/* If not initialising the device on first connection, bind PDump
+	 * capture to device 0. This is because we need to capture PDump
+	 * during device initialisation but only want to capture PDump for
+	 * a single device (by default, device 0).
+	 */
+	psPVRSRVData->ui32PDumpBoundDevice = 0;
+#endif
+#endif
+
+	return 0;
+
+Error:
+	PVRSRVCommonDriverDeInit();
+	return eError;
+}
+
+void
+PVRSRVCommonDriverDeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bEnablePageFaultDebug = IMG_FALSE;
+
+	if (gpsPVRSRVData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: missing device-independent data",
+				 __func__));
+		return;
+	}
+
+	if (gpsPVRSRVData->psClientStreamTable != NULL)
+	{
+		HASH_Delete(gpsPVRSRVData->psClientStreamTable);
+	}
+
+	if (gpsPVRSRVData->hClientStreamTableLock != NULL)
+	{
+		OSLockDestroy(gpsPVRSRVData->hClientStreamTableLock);
+	}
+
+	if (gpsPVRSRVData->pui32InfoPage != NULL)
+	{
+		bEnablePageFaultDebug = GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED;
+	}
+
+	gpsPVRSRVData->bUnload = IMG_TRUE;
+
+#if defined(SUPPORT_RGX)
+	PVRSRVDestroyHWPerfHostThread();
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock)
+	{
+		OSLockDestroy(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+		gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock = NULL;
+	}
+#endif
+
+	if (gpsPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectSignal(gpsPVRSRVData->hGlobalEventObject);
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Stop and cleanup the devices watchdog thread */
+	if (gpsPVRSRVData->hAutoVzWatchdogThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hAutoVzWatchdogEvObj)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hAutoVzWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hAutoVzWatchdogThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hAutoVzWatchdogThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hAutoVzWatchdogEvObj)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hAutoVzWatchdogEvObj);
+		gpsPVRSRVData->hAutoVzWatchdogEvObj = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+#endif /* SUPPORT_AUTOVZ */
+
+	/* Stop and cleanup the devices watchdog thread */
+	if (gpsPVRSRVData->hDevicesWatchdogThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hDevicesWatchdogEvObj)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hDevicesWatchdogThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hDevicesWatchdogThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hDevicesWatchdogEvObj)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hDevicesWatchdogEvObj);
+		gpsPVRSRVData->hDevicesWatchdogEvObj = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	/* Stop and cleanup the deferred clean up thread, event object and
+	 * deferred context list.
+	 */
+	if (gpsPVRSRVData->hCleanupThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hCleanupEventObject)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hCleanupEventObject);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hCleanupThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hCleanupThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hCleanupEventObject)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hCleanupEventObject);
+		gpsPVRSRVData->hCleanupEventObject = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	/* Tear down the HTB before PVRSRVHandleDeInit() removes its TL handle */
+	/* HTB De-init happens in device de-registration currently */
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	/* Tear down CacheOp framework information page first */
+	CacheOpDeInit2();
+
+	/* Clean up information page */
+	InfoPageDestroy(gpsPVRSRVData);
+
+	/* Close the TL control plane stream. */
+	if (gpsPVRSRVData->hTLCtrlStream != NULL)
+	{
+		TLStreamClose(gpsPVRSRVData->hTLCtrlStream);
+	}
+
+	/* deinitialise pdump */
+	if ((g_ui32InitFlags & INIT_DATA_ENABLE_PDUMPINIT) > 0)
+	{
+		PDUMPDEINIT();
+	}
+
+	/* Clean up Transport Layer resources that remain */
+	TLDeInit();
+
+	HostMemDeviceDestroy(gpsPVRSRVData->psHostMemDeviceNode);
+	gpsPVRSRVData->psHostMemDeviceNode = NULL;
+
+	eError = PVRSRVHandleDeInit();
+	PVR_LOG_IF_ERROR(eError, "PVRSRVHandleDeInit");
+
+	/* destroy event object */
+	if (gpsPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectDestroy(gpsPVRSRVData->hGlobalEventObject);
+		gpsPVRSRVData->hGlobalEventObject = NULL;
+	}
+
+	PVRSRVCmdCompleteDeinit();
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = DCDeInit();
+	PVR_LOG_IF_ERROR(eError, "DCDeInit");
+#endif
+
+	eError = PMRDeInit();
+	PVR_LOG_IF_ERROR(eError, "PMRDeInit");
+
+	BridgeDispatcherDeinit();
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	CPUMappingHistoryDeInit();
+#endif
+
+	if (bEnablePageFaultDebug)
+	{
+		/* Clear all allocated history tracking data */
+		DevicememHistoryDeInitKM();
+	}
+
+	CacheOpDeInit();
+
+	OSDeInitEnvData();
+
+	(void) DevmemIntDeInit();
+
+	ServerBridgeDeInit();
+
+	HTB_DestroyDIEntry();
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDestroyDI();		/* Stage 1 freeing */
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+	DebugCommonDeInitDriver();
+
+	DIDeInit();
+
+	if (gpsPVRSRVData->hThreadsDbgReqNotify)
+	{
+		PVRSRVUnregisterDriverDbgRequestNotify(gpsPVRSRVData->hThreadsDbgReqNotify);
+	}
+
+	PVRSRVUnregisterDriverDbgTable();
+
+	OSWRLockDestroy(gpsPVRSRVData->hDeviceNodeListLock);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDestroy();		/* Stage 2 freeing */
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RIDeInitKM();
+#endif
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+	OSFreeMemNoStats(gpsPVRSRVData);		/* Not trackable */
+
+	gpsPVRSRVData = NULL;
+}
+
+static void _SysDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	/* Only dump info once */
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hDebugRequestHandle;
+
+	PVR_UNREFERENCED_PARAMETER(ui32VerbLevel);
+
+	PVR_DUMPDEBUG_LOG("------[ System Summary Device ID:%d ]------", psDeviceNode->sDevId.ui32InternalID);
+
+	switch (psDeviceNode->eCurrentSysPowerState)
+	{
+		case PVRSRV_SYS_POWER_STATE_OFF:
+			PVR_DUMPDEBUG_LOG("Device System Power State: OFF");
+			break;
+		case PVRSRV_SYS_POWER_STATE_ON:
+			PVR_DUMPDEBUG_LOG("Device System Power State: ON");
+			break;
+		default:
+			PVR_DUMPDEBUG_LOG("Device System Power State: UNKNOWN (%d)",
+							   psDeviceNode->eCurrentSysPowerState);
+			break;
+	}
+
+	PVR_DUMPDEBUG_LOG("MaxHWTOut: %dus, WtTryCt: %d, WDGTOut(on,off): (%dms,%dms)",
+	                  MAX_HW_TIME_US, WAIT_TRY_COUNT, DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT, DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT);
+
+	SysDebugInfo(psDeviceNode->psDevConfig, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+PHYS_HEAP_CONFIG* PVRSRVFindPhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										   PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		if (BITMASK_HAS(psDevConfig->pasPhysHeaps[i].ui32UsageFlags, ui32Flags))
+		{
+			return &psDevConfig->pasPhysHeaps[i];
+		}
+	}
+
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVAcquireInternalID(IMG_UINT32 *pui32InternalID)
+{
+	IMG_UINT32 ui32InternalID = 0;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	for (ui32InternalID = 0;
+		 ui32InternalID < PVRSRV_MAX_DEVICES;
+		 ui32InternalID++)
+	{
+		if (PVRSRVGetDeviceInstance(ui32InternalID) == NULL)
+		{
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (bFound)
+	{
+		*pui32InternalID = ui32InternalID;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		return PVRSRV_ERROR_NO_FREE_DEVICEIDS_AVAILABLE;
+	}
+}
+
+PVRSRV_ERROR PVRSRVCommonDeviceCreate(void *pvOSDevice,
+											 IMG_INT32 i32KernelDeviceID,
+											 PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_DATA				*psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR			eError;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig;
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+	IMG_UINT32				ui32InternalID;
+
+	void *pvAppHintState    = NULL;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_HANDLE				hProcessStats;
+#endif
+	IMG_BOOL				bAppHintDefault;
+	IMG_BOOL				bEnablePageFaultDebug = IMG_FALSE;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL				bAutoVzGPUPowerdown   = IMG_FALSE;
+#endif
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceCreate: DevId %d", i32KernelDeviceID);
+
+	/* Determine if we've got EnablePageFaultDebug set or not */
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnablePageFaultDebug,
+			&bAppHintDefault, &bEnablePageFaultDebug);
+
+#if defined(SUPPORT_AUTOVZ)
+	bAppHintDefault = PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, AutoVzGPUPowerdown,
+			&bAppHintDefault, &bAutoVzGPUPowerdown);
+#endif
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	psDeviceNode = OSAllocZMemNoStats(sizeof(*psDeviceNode));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "psDeviceNode");
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Allocate process statistics */
+	eError = PVRSRVStatsRegisterProcess(&hProcessStats);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsRegisterProcess", ErrorFreeDeviceNode);
+#endif
+
+	dllist_init(&psDeviceNode->sCleanupThreadWorkList);
+	OSAtomicWrite(&psDeviceNode->i32NumCleanupItems, 0);
+
+	/* Record setting of EnablePageFaultDebug in device-node */
+	psDeviceNode->bEnablePFDebug = bEnablePageFaultDebug;
+#if defined(SUPPORT_AUTOVZ)
+	psDeviceNode->bAutoVzAllowGPUPowerdown = bAutoVzGPUPowerdown;
+#endif
+	psDeviceNode->sDevId.i32KernelDeviceID = i32KernelDeviceID;
+	eError = PVRSRVAcquireInternalID(&ui32InternalID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireInternalID", ErrorDeregisterStats);
+
+	eError = SysDevInit(pvOSDevice, &psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysDevInit", ErrorDeregisterStats);
+
+	/* Lock down the InternalID for this device by saving the value into
+	 * its device node. This is done after SysDevInit to allow the system
+	 * layer to identify the current device's the internal ID the same way. */
+	psDeviceNode->sDevId.ui32InternalID = ui32InternalID;
+
+	PVR_ASSERT(psDevConfig);
+	PVR_ASSERT(psDevConfig->pvOSDevice == pvOSDevice);
+	PVR_ASSERT(!psDevConfig->psDevNode);
+
+	if ((psDevConfig->eDefaultHeap != PVRSRV_PHYS_HEAP_GPU_LOCAL) &&
+	    (psDevConfig->eDefaultHeap != PVRSRV_PHYS_HEAP_CPU_LOCAL))
+	{
+		PVR_LOG_MSG(PVR_DBG_ERROR, "DEFAULT Heap is invalid, "
+		                           "it must be GPU_LOCAL or CPU_LOCAL");
+		PVR_LOG_GOTO_IF_ERROR(eError, "SysDevInit", ErrorDeregisterStats);
+	}
+	PVR_DPF((PVR_DBG_MESSAGE, "Device PhysHeap Default: %s",
+		(psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_CPU_LOCAL)
+		? "CPU_LOCAL" : "GPU_LOCAL"));
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATING);
+
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDevConfig->psDevNode = psDeviceNode;
+
+	if (psDevConfig->pfnGpuDomainPower)
+	{
+		psDeviceNode->eCurrentSysPowerState = psDevConfig->pfnGpuDomainPower(psDeviceNode);
+	}
+	else
+	{
+		/* If the System Layer doesn't provide a function to query the power state
+		 * of the system hardware, assume the system starting state below. */
+
+#if defined(SUPPORT_AUTOVZ)
+		psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_ON;
+#else
+		psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_OFF;
+#endif
+	}
+
+
+	/* Next update value will be 0xFFFFFFF7 since sync prim starts with 0xFFFFFFF6.
+	 * Has to be set before call to PMRInitDevice(). */
+	psDeviceNode->ui32NextMMUInvalidateUpdate = 0xFFFFFFF7U;
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+	psDeviceNode->uiPowerOffCounter = 0;
+	psDeviceNode->uiPowerOffCounterNext = 1;
+#endif
+
+	eError = PVRSRVRegisterDeviceDbgTable(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorPvzConnectionDeInit);
+
+	eError = PVRSRVPowerLockInit(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnregisterDbgTable);
+
+	eError = PhysHeapInitDeviceHeaps(psDeviceNode, psDevConfig);
+	PVR_GOTO_IF_ERROR(eError, ErrorPowerLockDeInit);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	eError = PMRInitDevice(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorPhysHeapDeInitDeviceHeaps);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_RGX)
+	/* Requirements:
+	 *  registered GPU and FW local heaps */
+	/*  debug table */
+	eError = RGXRegisterDevice(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXRegisterDevice");
+		eError = PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+		goto ErrorPMRDeInitDevice;
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	if (PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		IMG_UINT32 ui32AppHintDefault = 0;
+		IMG_UINT32 ui32AppHintPhysMemTestPasses;
+
+		/* Read AppHint - Configurable memory test pass count */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, PhysMemTestPasses,
+				&ui32AppHintDefault, &ui32AppHintPhysMemTestPasses);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		if (ui32AppHintPhysMemTestPasses > 0)
+		{
+			eError = PhysMemTest(psDeviceNode, psDevConfig, ui32AppHintPhysMemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysMemTest", ErrorDeInitRgx);
+		}
+	}
+#endif
+#endif
+
+	/* Inform the device layer PhysHeaps are now initialised so that device
+	 * specific heaps can be obtained along with carrying out any Vz setup. */
+	if (psDeviceNode->pfnPhysMemDeviceHeapsInit != NULL)
+	{
+		eError = psDeviceNode->pfnPhysMemDeviceHeapsInit(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorPMRDeInitDevice);
+	}
+
+	/* Carry out initialisation of a dedicated FW MMU data, if the FW CPU has
+	 * an MMU separate to the GPU MMU e.g. MIPS based FW. */
+	if (psDeviceNode->pfnFwMMUInit != NULL)
+	{
+		eError = psDeviceNode->pfnFwMMUInit(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorFwMMUDeinit);
+	}
+
+	eError = SyncServerInit(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorDeInitRgx);
+
+	eError = SyncCheckpointInit(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncCheckpointInit", ErrorSyncCheckpointInit);
+
+	/*
+	 * This is registered before doing device specific initialisation to ensure
+	 * generic device information is dumped first during a debug request.
+	 */
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDeviceNode->hDbgReqNotify,
+		                                          psDeviceNode,
+		                                          _SysDebugRequestNotify,
+		                                          DEBUG_REQUEST_SYS,
+		                                          psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify", ErrorRegDbgReqNotify);
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	eError = InitDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InitDVFS", ErrorDVFSInitFail);
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+	eError = InitPDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InitPDVFS", ErrorPDVFSInitFail);
+#endif
+
+	OSAtomicWrite(&psDeviceNode->iNumClockSpeedChanges, 0);
+
+	OSWRLockCreate(&psDeviceNode->hPageFaultNotifyLock);
+	if (psDeviceNode->hPageFaultNotifyLock == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for PF notify list",
+		        __func__));
+		goto ErrorPageFaultLockFailCreate;
+	}
+
+	dllist_init(&psDeviceNode->sMemoryContextPageFaultNotifyListHead);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Registered device %p", psDeviceNode));
+	PVR_DPF((PVR_DBG_MESSAGE, "Register bank address = 0x%08lx",
+			 (unsigned long)psDevConfig->sRegsCpuPBase.uiAddr));
+	PVR_DPF((PVR_DBG_MESSAGE, "IRQ = %d", psDevConfig->ui32IRQ));
+
+/* SUPPORT_ALT_REGBASE is defined for rogue cores only */
+#if defined(SUPPORT_RGX) && defined(SUPPORT_ALT_REGBASE)
+	{
+		IMG_DEV_PHYADDR sRegsGpuPBase;
+
+		PhysHeapCpuPAddrToDevPAddr(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+		                           1,
+		                           &sRegsGpuPBase,
+		                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+
+		PVR_LOG(("%s: Using alternate Register bank GPU address: 0x%08lx (orig: 0x%08lx)", __func__,
+		         (unsigned long)psDevConfig->sAltRegsGpuPBase.uiAddr,
+		         (unsigned long)sRegsGpuPBase.uiAddr));
+	}
+#endif
+
+	eError = DebugCommonInitDevice(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DebugCommonInitDevice",
+	                      ErrorDestroyMemoryContextPageFaultNotifyListLock);
+
+	/* Create the devicemem_history hook for the device. We need to
+	 * have the debug-info instantiated before calling this.
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		eError = DevicememHistoryDeviceCreate(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevicememHistoryDeviceCreate", ErrorDebugCommonDeInitDevice);
+	}
+
+#if defined(__linux__)
+	/* Register the device specific AppHints so individual AppHints can be
+	 * configured before the FW is initialised. This must be called after
+	 * DebugCommonInitDevice() above as it depends on the created gpuXX/apphint
+	 * DI Group.
+	 */
+	{
+		int iError = pvr_apphint_device_register(psDeviceNode);
+		PVR_LOG_IF_FALSE(iError == 0, "pvr_apphint_device_register() failed");
+	}
+#endif /* defined(__linux__) */
+
+#if defined(SUPPORT_RGX)
+	RGXHWPerfInitAppHintCallbacks(psDeviceNode);
+#endif
+
+	/* Finally insert the device into the dev-list and set it as active */
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_InsertTail(&psPVRSRVData->psDeviceNodeList,
+									   psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices++;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Initialise the paravirtualised connection */
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		PvzConnectionInit(psDevConfig);
+		PVR_GOTO_IF_ERROR(eError, ErrorSysDevDeInit);
+	}
+
+	BIT_SET(psDevConfig->psDevNode->ui32VmState, RGXFW_HOST_DRIVER_ID);
+
+	*ppsDeviceNode = psDeviceNode;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	/* Register the DVFS device now the device node is present in the dev-list */
+	eError = RegisterDVFSDevice(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RegisterDVFSDevice", ErrorRegisterDVFSDeviceFail);
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+#endif
+
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATED);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+ErrorRegisterDVFSDeviceFail:
+	/* Remove the device from the list */
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_Remove(psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices--;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+#if defined(__linux__)
+	pvr_apphint_device_unregister(psDeviceNode);
+#endif /* defined(__linux__) */
+
+	/* Remove the devicemem_history hook if we created it */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		DevicememHistoryDeviceDestroy(psDeviceNode);
+	}
+#endif
+
+ErrorDebugCommonDeInitDevice:
+	DebugCommonDeInitDevice(psDeviceNode);
+
+ErrorDestroyMemoryContextPageFaultNotifyListLock:
+	OSWRLockDestroy(psDeviceNode->hPageFaultNotifyLock);
+	psDeviceNode->hPageFaultNotifyLock = NULL;
+
+ErrorPageFaultLockFailCreate:
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+ErrorDVFSInitFail:
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+ErrorPDVFSInitFail:
+#endif
+
+	if (psDeviceNode->hDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceNode->hDbgReqNotify);
+	}
+
+ErrorRegDbgReqNotify:
+	SyncCheckpointDeinit(psDeviceNode);
+
+ErrorSyncCheckpointInit:
+	SyncServerDeinit(psDeviceNode);
+
+ErrorDeInitRgx:
+#if defined(SUPPORT_RGX)
+	DevDeInitRGX(psDeviceNode);
+#endif
+ErrorFwMMUDeinit:
+ErrorPMRDeInitDevice:
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRDeInitDevice(psDeviceNode);
+ErrorPhysHeapDeInitDeviceHeaps:
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+ErrorPowerLockDeInit:
+	PVRSRVPowerLockDeInit(psDeviceNode);
+ErrorUnregisterDbgTable:
+	PVRSRVUnregisterDeviceDbgTable(psDeviceNode);
+ErrorPvzConnectionDeInit:
+	psDevConfig->psDevNode = NULL;
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		PvzConnectionDeInit(psDevConfig);
+	}
+ErrorSysDevDeInit:
+	SysDevDeInit(psDevConfig);
+ErrorDeregisterStats:
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+ErrorFreeDeviceNode:
+#endif
+	OSFreeMemNoStats(psDeviceNode);
+
+	return eError;
+}
+
+#if defined(SUPPORT_RGX)
+static PVRSRV_ERROR _SetDeviceFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL bValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXSetDeviceFlags((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                            ui32Flag, bValue);
+
+	return eResult;
+}
+
+static PVRSRV_ERROR _ReadDeviceFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL *pbValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+	IMG_UINT32 ui32State;
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXGetDeviceFlags((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                            &ui32State);
+
+	if (PVRSRV_OK == eResult)
+	{
+		*pbValue = (ui32State & ui32Flag)? IMG_TRUE: IMG_FALSE;
+	}
+
+	return eResult;
+}
+static PVRSRV_ERROR _SetStateFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                  const void *psPrivate, IMG_BOOL bValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXStateFlagCtrl((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                           ui32Flag, NULL, bValue);
+
+	return eResult;
+}
+
+static PVRSRV_ERROR _ReadStateFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL *pbValue)
+{
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+	IMG_UINT32 ui32State;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevice->pvDevice;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags,
+	                           INVALIDATE);
+	ui32State = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+
+	if (pbValue)
+	{
+		*pbValue = (ui32State & ui32Flag)? IMG_TRUE: IMG_FALSE;
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR PVRSRVCommonDeviceInitialise(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_BOOL bInitSuccessful = IMG_FALSE;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_HANDLE hProcessStats;
+#endif
+	PVRSRV_ERROR eError;
+
+	PDUMPCOMMENT(psDeviceNode, "Common Device Initialisation");
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceInitialise: DevId %d", psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_CREATED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device already initialised", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Allocate an OSEventObject to use in the Freeze / Thaw transitioning to
+	 * allow LBIST requests to be serviced. This object is held for the life of
+	 * this device-node.
+	 */
+	eError = OSEventObjectCreate("PVRSRV_DEVICE_THREAD_EVENT_OBJECT",
+	                             &psDeviceNode->hDeviceThreadEvObj);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSEventObjectCreate");
+
+	eError = OSEventObjectOpen(psDeviceNode->hDeviceThreadEvObj,
+	                           &psDeviceNode->hDeviceFreezeThaw);
+
+	if (PVRSRV_OK != eError)
+	{
+		OSEventObjectDestroy(psDeviceNode->hDeviceThreadEvObj);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "OSEventObjectOpen");
+	}
+
+	/* Initial zero-set for the number of Frozen threads */
+	OSAtomicWrite(&psDeviceNode->iFreezeCount, 0);
+	OSAtomicWrite(&psDeviceNode->iTotalFreezes, 0);
+
+	/* Initial zero-set for number of active threads on this device */
+	OSAtomicWrite(&psDeviceNode->iThreadsActive, 0);
+
+	/* Allocate devmem_history backing store for the device if we have
+	 * EnablePageFaultDebug set
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		eError = DevicememHistoryDeviceInit(psDeviceNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevicememHistoryDeviceInit");
+	}
+
+#if defined(PDUMP)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	{
+		PVRSRV_DATA *psSRVData = PVRSRVGetPVRSRVData();
+
+		/* If first connection, bind this and future PDump clients to use this device */
+		if (psSRVData->ui32PDumpBoundDevice == PVRSRV_MAX_DEVICES)
+		{
+			psSRVData->ui32PDumpBoundDevice = psDeviceNode->sDevId.ui32InternalID;
+		}
+	}
+#endif
+#endif
+
+	/* Initialise Connection_Data access mechanism */
+	dllist_init(&psDeviceNode->sConnections);
+	eError = OSLockCreate(&psDeviceNode->hConnectionsLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	/* Allocate process statistics */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	eError = PVRSRVStatsRegisterProcess(&hProcessStats);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVStatsRegisterProcess");
+#endif
+
+	eError = MMU_InitDevice(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_InitDevice");
+
+#if defined(SUPPORT_RGX)
+	eError = RGXInit(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInit", Exit);
+#endif
+
+	bInitSuccessful = IMG_TRUE;
+
+#if defined(SUPPORT_RGX)
+Exit:
+#endif
+	eError = PVRSRVDeviceFinalise(psDeviceNode, bInitSuccessful);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceFinalise");
+
+#if defined(SUPPORT_RGX)
+	if (bInitSuccessful && (eError == PVRSRV_OK))
+	{
+		RGXInitMultiCoreInfo(psDeviceNode);
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableClockGating,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  APPHINT_OF_DRIVER_NO_DEVICE,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_DISABLE_CLKGATING_EN));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableDMOverlap,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  APPHINT_OF_DRIVER_NO_DEVICE,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_DISABLE_DM_OVERLAP));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_AssertOnHWRTrigger,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_AssertOutOfMemory,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_CheckMList,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_CHECK_MLIST_EN));
+		}
+
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableFEDLogging,
+		                                  _ReadDeviceFlag, _SetDeviceFlag,
+		                                  psDeviceNode,
+		                                  (void*)((uintptr_t)RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN));
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_ZeroFreelist,
+		                                  _ReadDeviceFlag, _SetDeviceFlag,
+		                                  psDeviceNode,
+		                                  (void*)((uintptr_t)RGXKM_DEVICE_STATE_ZERO_FREELIST));
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisablePDumpPanic,
+		                                  RGXQueryPdumpPanicDisable, RGXSetPdumpPanicDisable,
+		                                  psDeviceNode,
+		                                  NULL);
+	}
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+#endif
+
+	return eError;
+}
+
+void PVRSRVCommonDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DATA				*psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR			eError;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL				bForceUnload = IMG_FALSE;
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+	if ((PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK) ||
+		(eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_DEAD) ||
+		(eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+	{
+		bForceUnload = IMG_TRUE;
+	}
+#endif
+
+	/* Remove DI hook for the devicemem_history for this device (if any).
+	 * The associated devicemem_history buffers are freed by the final
+	 * call to DevicememHistoryDeInitKM() as they are used asynchronously
+	 * by other parts of the DDK.
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		DevicememHistoryDeviceDestroy(psDeviceNode);
+	}
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceDestroy: DevId %d", psDeviceNode->sDevId.i32KernelDeviceID);
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_DEINIT);
+
+	PVRSRVCleanupThreadWaitForDevice(psDeviceNode);
+
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_Remove(psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices--;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_DESTRUCTING);
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	UnregisterDVFSDevice(psDeviceNode);
+#endif
+
+#if defined(__linux__)
+	pvr_apphint_device_unregister(psDeviceNode);
+#endif /* defined(__linux__) */
+
+	DebugCommonDeInitDevice(psDeviceNode);
+
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	SyncFbDeregisterDevice(psDeviceNode);
+#endif
+	/* Counter part to what gets done in PVRSRVDeviceFinalise */
+	if (psDeviceNode->hSyncCheckpointContext)
+	{
+		SyncCheckpointContextDestroy(psDeviceNode->hSyncCheckpointContext);
+		psDeviceNode->hSyncCheckpointContext = NULL;
+	}
+	if (psDeviceNode->hSyncPrimContext)
+	{
+		if (psDeviceNode->psMMUCacheSyncPrim)
+		{
+			PVRSRV_CLIENT_SYNC_PRIM *psSync = psDeviceNode->psMMUCacheSyncPrim;
+
+			/* Ensure there are no pending MMU Cache Ops in progress before freeing this sync. */
+			eError = PVRSRVPollForValueKM(psDeviceNode,
+			                              psSync->pui32LinAddr,
+			                              psDeviceNode->ui32NextMMUInvalidateUpdate-1,
+			                              0xFFFFFFFF,
+			                              POLL_FLAG_LOG_ERROR,
+			                              NULL);
+			PVR_LOG_IF_ERROR(eError, "PVRSRVPollForValueKM");
+
+			/* Important to set the device node pointer to NULL
+			 * before we free the sync-prim to make sure we don't
+			 * defer the freeing of the sync-prim's page tables itself.
+			 * The sync is used to defer the MMU page table
+			 * freeing. */
+			psDeviceNode->psMMUCacheSyncPrim = NULL;
+
+			/* Free general purpose sync primitive */
+			SyncPrimFree(psSync);
+		}
+
+		SyncPrimContextDestroy(psDeviceNode->hSyncPrimContext);
+		psDeviceNode->hSyncPrimContext = NULL;
+	}
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError == PVRSRV_OK)
+	{
+		IMG_BOOL bHasPowerLock = IMG_TRUE;
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		/*
+		 * Firmware probably not responding if bForceUnload is set, but we still want to unload the
+		 * driver.
+		 */
+		if (!bForceUnload)
+#endif
+		{
+			/* Force idle device */
+			eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_TRUE);
+			PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+			if (eError == PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				bHasPowerLock = IMG_FALSE;
+			}
+		}
+
+		if (bHasPowerLock)
+		{
+			/* Power down the device if necessary */
+			eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+												 PVRSRV_DEV_POWER_STATE_OFF,
+												 PVRSRV_POWER_FLAGS_FORCED);
+			PVRSRVPowerUnlock(psDeviceNode);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVSetDevicePowerStateKM");
+
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+		}
+	}
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	DeinitDVFS(psDeviceNode);
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+	DeinitPDVFS(psDeviceNode);
+#endif
+
+	if (psDeviceNode->hDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceNode->hDbgReqNotify);
+	}
+
+	SyncCheckpointDeinit(psDeviceNode);
+
+	SyncServerDeinit(psDeviceNode);
+
+	MMU_DeInitDevice(psDeviceNode);
+
+#if defined(SUPPORT_RGX)
+	DevDeInitRGX(psDeviceNode);
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* must be called before PhysHeapDeInitDeviceHeaps() */
+	PMRDeInitDevice(psDeviceNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+	PVRSRVPowerLockDeInit(psDeviceNode);
+
+	PVRSRVUnregisterDeviceDbgTable(psDeviceNode);
+
+	if (psDeviceNode->hPageFaultNotifyLock != NULL)
+	{
+		OSWRLockDestroy(psDeviceNode->hPageFaultNotifyLock);
+	}
+
+	/* Release the Connection-Data lock as late as possible. */
+	if (psDeviceNode->hConnectionsLock)
+	{
+		OSLockDestroy(psDeviceNode->hConnectionsLock);
+	}
+
+	/* Release the hDeviceThreadEvObj as late as possible. */
+	if (psDeviceNode->hDeviceThreadEvObj)
+	{
+		OSEventObjectClose(psDeviceNode->hDeviceFreezeThaw);
+
+		OSEventObjectDestroy(psDeviceNode->hDeviceThreadEvObj);
+	}
+
+	PVR_ASSERT(OSAtomicRead(&psDeviceNode->iFreezeCount) == 0);
+
+	psDeviceNode->psDevConfig->psDevNode = NULL;
+
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		PvzConnectionDeInit(psDeviceNode->psDevConfig);
+	}
+	SysDevDeInit(psDeviceNode->psDevConfig);
+
+	OSFreeMemNoStats(psDeviceNode);
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceFinalise
+@Description  Performs the final parts of device initialisation.
+@Input        psDeviceNode            Device node of the device to finish
+                                      initialising
+@Input        bInitSuccessful         Whether or not device specific
+                                      initialisation was successful
+@Return       PVRSRV_ERROR     PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceFinalise(PVRSRV_DEVICE_NODE *psDeviceNode,
+											   IMG_BOOL bInitSuccessful)
+{
+	PVRSRV_ERROR eError;
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	if (bInitSuccessful)
+	{
+		eError = SyncCheckpointContextCreate(psDeviceNode,
+											 &psDeviceNode->hSyncCheckpointContext);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncCheckpointContextCreate", ErrorExit);
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+		eError = SyncFbRegisterDevice(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+#endif
+		eError = SyncPrimContextCreate(psDeviceNode,
+									   &psDeviceNode->hSyncPrimContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "SyncPrimContextCreate");
+			SyncCheckpointContextDestroy(psDeviceNode->hSyncCheckpointContext);
+			goto ErrorExit;
+		}
+
+		/* Allocate MMU cache invalidate sync */
+		eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
+							   &psDeviceNode->psMMUCacheSyncPrim,
+							   "pvrsrv dev MMU cache");
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncPrimAlloc", ErrorExit);
+
+		/* Set the sync prim value to a much higher value near the
+		 * wrapping range. This is so any wrapping bugs would be
+		 * seen early in the driver start-up.
+		 */
+		SyncPrimSet(psDeviceNode->psMMUCacheSyncPrim, 0xFFFFFFF6UL);
+
+		eError = PVRSRVPowerLock(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPowerLock", ErrorExit);
+
+		/*
+		 * Always ensure a single power on command appears in the pdump. This
+		 * should be the only power related call outside of PDUMPPOWCMDSTART
+		 * and PDUMPPOWCMDEND.
+		 */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_FORCED);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set device %p power state to 'on' (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			PVRSRVPowerUnlock(psDeviceNode);
+			goto ErrorExit;
+		}
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+		eError = ValidateFWOnLoad(psDeviceNode->pvDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "ValidateFWOnLoad");
+			PVRSRVPowerUnlock(psDeviceNode);
+			return eError;
+		}
+#endif
+
+		/*
+		 * Guest driver must do a runtime compatibility check against the
+		 * data provided by the Firmware.
+		 */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Failed compatibility check for device %p (%s)",
+						 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVPowerUnlock(psDeviceNode);
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				goto ErrorExit;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+
+		/* Force the device to idle if its default power state is off */
+		eError = PVRSRVDeviceIdleRequestKM(psDeviceNode,
+										   &PVRSRVDeviceIsDefaultStateOFF,
+										   IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+			if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+			goto ErrorExit;
+		}
+
+		/* Place device into its default power state. */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_DEFAULT,
+											 PVRSRV_POWER_FLAGS_FORCED);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set device %p into its default power state (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			goto ErrorExit;
+		}
+
+		PVRSRVPowerUnlock(psDeviceNode);
+
+		/*
+		 * If PDUMP is enabled and RGX device is supported, then initialise the
+		 * performance counters that can be further modified in PDUMP. Then,
+		 * before ending the init phase of the pdump, drain the commands put in
+		 * the kCCB during the init phase.
+		 */
+#if defined(SUPPORT_RGX)
+#if defined(PDUMP)
+		{
+			eError = RGXInitHWPerfCounters(psDeviceNode);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitHWPerfCounters", ErrorExit);
+
+			eError = RGXPdumpDrainKCCB(psDevInfo,
+									   psDevInfo->psKernelCCBCtlLocal->ui32WriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", ErrorExit);
+		}
+#endif
+#endif /* defined(SUPPORT_RGX) */
+		/* Now that the device(s) are fully initialised set them as active */
+		PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_ACTIVE);
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		/* Initialisation failed so set the device(s) into a bad state */
+		PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_BAD);
+		eError = PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	/* Give PDump control a chance to end the init phase, depends on OS */
+	PDUMPENDINITPHASE(psDeviceNode);
+	return eError;
+
+ErrorExit:
+	/* Initialisation failed so set the device(s) into a bad state */
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_BAD);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* Only check devices which specify a compatibility check callback */
+	if (psDeviceNode->pfnInitDeviceCompatCheck)
+		return psDeviceNode->pfnInitDeviceCompatCheck(psDeviceNode);
+	else
+		return PVRSRV_OK;
+}
+
+/*
+	PollForValueKM
+*/
+static
+PVRSRV_ERROR PollForValueKM (volatile IMG_UINT32 __iomem           *pui32LinMemAddr,
+										  IMG_UINT32                ui32Value,
+										  IMG_UINT32                ui32Mask,
+										  IMG_UINT32                ui32Timeoutus,
+										  IMG_UINT32                ui32PollPeriodus,
+										  POLL_FLAGS                ePollFlags,
+										  PFN_INVALIDATE_CACHEFUNC  pfnFwInvalidate)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(pui32LinMemAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(ui32Timeoutus);
+	PVR_UNREFERENCED_PARAMETER(ui32PollPeriodus);
+	PVR_UNREFERENCED_PARAMETER(ePollFlags);
+	return PVRSRV_OK;
+#else
+	IMG_UINT32 ui32ActualValue = 0xFFFFFFFFU; /* Initialiser only required to prevent incorrect warning */
+
+	LOOP_UNTIL_TIMEOUT_US(ui32Timeoutus)
+	{
+		if (pfnFwInvalidate)
+		{
+			pfnFwInvalidate((const volatile void __force *)pui32LinMemAddr,
+			                sizeof(*pui32LinMemAddr),
+			                PVRSRV_CACHE_OP_INVALIDATE);
+		}
+
+		ui32ActualValue = OSReadHWReg32((void __iomem *)pui32LinMemAddr, 0) & ui32Mask;
+
+		if (ui32ActualValue == ui32Value)
+		{
+			return PVRSRV_OK;
+		}
+
+		if (gpsPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+#define ONE_MSEC_IN_USECS	(1000U)
+
+		if (ui32PollPeriodus <= ONE_MSEC_IN_USECS)
+		{
+			OSWaitus(ui32PollPeriodus);
+		}
+		else
+		{
+			OSSleepms(ui32PollPeriodus / ONE_MSEC_IN_USECS);
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (BITMASK_HAS(ePollFlags, POLL_FLAG_LOG_ERROR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PollForValueKM: Timeout. Expected 0x%x but found 0x%x (mask 0x%x).",
+		         ui32Value, ui32ActualValue, ui32Mask));
+	}
+
+	return PVRSRV_ERROR_TIMEOUT;
+#endif /* NO_HARDWARE */
+}
+
+
+/*
+	PVRSRVPollForValueKM
+*/
+PVRSRV_ERROR PVRSRVPollForValueKM (PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem   *pui32LinMemAddr,
+												IMG_UINT32                     ui32Value,
+												IMG_UINT32                     ui32Mask,
+												POLL_FLAGS                     ePollFlags,
+												PFN_INVALIDATE_CACHEFUNC       pfnFwInvalidate)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PollForValueKM(pui32LinMemAddr, ui32Value, ui32Mask,
+						  MAX_HW_TIME_US,
+						  MAX_HW_TIME_US/WAIT_TRY_COUNT,
+						  ePollFlags,
+						  pfnFwInvalidate);
+	if (eError != PVRSRV_OK && BITMASK_HAS(ePollFlags, POLL_FLAG_DEBUG_DUMP))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		                        __func__, PVRSRVGetErrorString(eError),
+								pui32LinMemAddr, ui32Value));
+		PVRSRVDebugRequest(psDevNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVWaitForValueKM(volatile IMG_UINT32 __iomem   *pui32LinMemAddr,
+                     IMG_UINT32                    ui32Value,
+                     IMG_UINT32                    ui32Mask,
+                     PFN_INVALIDATE_CACHEFUNC      pfnFwInvalidate)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(pui32LinMemAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	return PVRSRV_OK;
+#else
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eErrorWait;
+	IMG_UINT32 ui32ActualValue;
+
+	eError = OSEventObjectOpen(psPVRSRVData->hGlobalEventObject, &hOSEvent);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectOpen", EventObjectOpenError);
+
+	eError = PVRSRV_ERROR_TIMEOUT; /* Initialiser for following loop */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (pfnFwInvalidate)
+		{
+			pfnFwInvalidate((const volatile void __force *)pui32LinMemAddr,
+			                sizeof(*pui32LinMemAddr),
+			                PVRSRV_CACHE_OP_INVALIDATE);
+		}
+
+		ui32ActualValue = (OSReadDeviceMem32(pui32LinMemAddr) & ui32Mask);
+
+		if (ui32ActualValue == ui32Value)
+		{
+			/* Expected value has been found */
+			eError = PVRSRV_OK;
+			break;
+		}
+		else if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			/* Services in bad state, don't wait any more */
+			eError = PVRSRV_ERROR_NOT_READY;
+			break;
+		}
+		else
+		{
+			/* wait for event and retry */
+			eErrorWait = OSEventObjectWait(hOSEvent);
+			if (eErrorWait != PVRSRV_OK  &&  eErrorWait != PVRSRV_ERROR_TIMEOUT)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Failed with error %d. Found value 0x%x but was expected "
+				         "to be 0x%x (Mask 0x%08x). Retrying",
+						 __func__,
+						 eErrorWait,
+						 ui32ActualValue,
+						 ui32Value,
+						 ui32Mask));
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	OSEventObjectClose(hOSEvent);
+
+	/* One last check in case the object wait ended after the loop timeout... */
+	if (eError != PVRSRV_OK &&
+	    (OSReadDeviceMem32(pui32LinMemAddr) & ui32Mask) == ui32Value)
+	{
+		eError = PVRSRV_OK;
+	}
+
+	/* Provide event timeout information to aid the Device Watchdog Thread... */
+	if (eError == PVRSRV_OK)
+	{
+		psPVRSRVData->ui32GEOConsecutiveTimeouts = 0;
+	}
+	else if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		psPVRSRVData->ui32GEOConsecutiveTimeouts++;
+	}
+
+EventObjectOpenError:
+
+	return eError;
+
+#endif /* NO_HARDWARE */
+}
+
+int PVRSRVGetDriverStatus(void)
+{
+	return PVRSRVGetPVRSRVData()->eServicesState;
+}
+
+/*
+	PVRSRVSystemHasCacheSnooping
+*/
+IMG_BOOL PVRSRVSystemHasCacheSnooping(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if ((psDevConfig->eCacheSnoopingMode != PVRSRV_DEVICE_SNOOP_NONE) &&
+		(psDevConfig->eCacheSnoopingMode != PVRSRV_DEVICE_SNOOP_EMULATED))
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemSnoopingIsEmulated(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_EMULATED)
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemSnoopingOfCPUCache(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if ((psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_CPU_ONLY) ||
+		(psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_CROSS))
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemHasNonMappableLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	return psDevConfig->bHasNonMappableLocalMemory;
+}
+
+/*
+	PVRSRVSystemWaitCycles
+*/
+void PVRSRVSystemWaitCycles(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT32 ui32Cycles)
+{
+	/* Delay in us */
+	IMG_UINT32 ui32Delayus = 1;
+
+	/* obtain the device freq */
+	if (psDevConfig->pfnClockFreqGet != NULL)
+	{
+		IMG_UINT32 ui32DeviceFreq;
+
+		ui32DeviceFreq = psDevConfig->pfnClockFreqGet(psDevConfig->hSysData);
+
+		ui32Delayus = (ui32Cycles*1000000)/ui32DeviceFreq;
+
+		if (ui32Delayus == 0)
+		{
+			ui32Delayus = 1;
+		}
+	}
+
+	OSWaitus(ui32Delayus);
+}
+
+static void *
+PVRSRVSystemInstallDeviceLISR_Match_AnyVaCb(PVRSRV_DEVICE_NODE *psDeviceNode,
+											va_list va)
+{
+	void *pvOSDevice = va_arg(va, void *);
+
+	if (psDeviceNode->psDevConfig->pvOSDevice == pvOSDevice)
+	{
+		return psDeviceNode;
+	}
+
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVSystemInstallDeviceLISR(void *pvOSDevice,
+										   IMG_UINT32 ui32IRQ,
+										   const IMG_CHAR *pszName,
+										   PFN_LISR pfnLISR,
+										   void *pvData,
+										   IMG_HANDLE *phLISRData)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =
+		List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+									   &PVRSRVSystemInstallDeviceLISR_Match_AnyVaCb,
+									   pvOSDevice);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (!psDeviceNode)
+	{
+		/* Device can't be found in the list so it isn't in the system */
+		PVR_DPF((PVR_DBG_ERROR, "%s: device %p with irq %d is not present",
+				 __func__, pvOSDevice, ui32IRQ));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	return SysInstallDeviceLISR(psDeviceNode->psDevConfig->hSysData, ui32IRQ,
+								pszName, pfnLISR, pvData, phLISRData);
+}
+
+PVRSRV_ERROR PVRSRVSystemUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return SysUninstallDeviceLISR(hLISRData);
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION) && defined(EMULATOR)
+/* functions only used on rogue, but header defining them is common */
+void SetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState)
+{
+	SysSetAxiProtOSid(hSysData, ui32OSid, bState);
+}
+
+void SetTrustedDeviceAceEnabled(IMG_HANDLE hSysData)
+{
+	SysSetTrustedDeviceAceEnabled(hSysData);
+}
+#endif
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR PVRSRVCreateHWPerfHostThread(IMG_UINT32 ui32Timeout)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!ui32Timeout)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	OSLockAcquire(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+
+	/* Create only once */
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread == NULL)
+	{
+		/* Create the HWPerf event object */
+		eError = OSEventObjectCreate("PVRSRV_HWPERFHOSTPERIODIC_EVENTOBJECT", &gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectCreate");
+
+		if (eError == PVRSRV_OK)
+		{
+			gpsPVRSRVData->bHWPerfHostThreadStop = IMG_FALSE;
+			gpsPVRSRVData->ui32HWPerfHostThreadTimeout = ui32Timeout;
+			/* Create a thread which is used to periodically emit host stream packets */
+			eError = OSThreadCreate(&gpsPVRSRVData->hHWPerfHostPeriodicThread,
+				"pvr_hwperf_host",
+				HWPerfPeriodicHostEventsThread,
+				NULL, IMG_TRUE, gpsPVRSRVData);
+			PVR_LOG_IF_ERROR(eError, "OSThreadCreate");
+		}
+	}
+	/* If the thread has already been created then just update the timeout and wake up thread */
+	else
+	{
+		gpsPVRSRVData->ui32HWPerfHostThreadTimeout = ui32Timeout;
+		eError = OSEventObjectSignal(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+	}
+
+	OSLockRelease(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDestroyHWPerfHostThread(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	OSLockAcquire(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+
+	/* Stop and cleanup the HWPerf periodic thread */
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread)
+	{
+		if (gpsPVRSRVData->hHWPerfHostPeriodicEvObj)
+		{
+			gpsPVRSRVData->bHWPerfHostThreadStop = IMG_TRUE;
+			eError = OSEventObjectSignal(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			eError = OSThreadDestroy(gpsPVRSRVData->hHWPerfHostPeriodicThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hHWPerfHostPeriodicThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+
+		if (gpsPVRSRVData->hHWPerfHostPeriodicEvObj)
+		{
+			eError = OSEventObjectDestroy(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+			gpsPVRSRVData->hHWPerfHostPeriodicEvObj = NULL;
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+		}
+	}
+
+	OSLockRelease(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	return eError;
+}
+#endif
+
+/*
+ * Scan the list of known devices until we find the specific instance or
+ * exhaust the list
+ */
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstance(IMG_UINT32 uiInstance)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	if (uiInstance >= gpsPVRSRVData->ui32RegisteredDevices)
+	{
+		return NULL;
+	}
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (psDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     psDevNode != NULL; psDevNode = psDevNode->psNext)
+	{
+		if (uiInstance == psDevNode->sDevId.ui32InternalID)
+		{
+			break;
+		}
+	}
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	return psDevNode;
+}
+
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstanceByKernelDevID(IMG_INT32 i32OSInstance)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (psDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     psDevNode != NULL; psDevNode = psDevNode->psNext)
+	{
+		if (i32OSInstance == psDevNode->sDevId.i32KernelDeviceID)
+		{
+			MULTI_DEVICE_BRINGUP_DPF("%s: Found DevId %d. Retrieving node.", __func__, i32OSInstance);
+			break;
+		}
+		else
+		{
+			MULTI_DEVICE_BRINGUP_DPF("%s: Searching for DevId %d: Id %d not matching", __func__, i32OSInstance, psDevNode->sDevId.i32KernelDeviceID);
+		}
+	}
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	if (psDevNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: DevId %d not found.", __func__, i32OSInstance));
+	}
+	return psDevNode;
+}
+
+#define _FROZEN 1		/* Device is already frozen */
+#define _NOT_FROZEN 0	/* Device is not frozen */
+
+/* Freeze the specified device if not already frozen */
+PVRSRV_ERROR PVRSRVDeviceFreeze(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL     bHasPowerLock = IMG_FALSE;
+
+	/* Verify that given argument *IS* a recognised device node */
+	PVRSRV_DEVICE_NODE *lpsDevNode;
+
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (lpsDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     lpsDevNode != NULL && lpsDevNode != psDevNode;
+	     lpsDevNode = lpsDevNode->psNext)
+		;
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	PVR_LOG_RETURN_IF_FALSE((lpsDevNode == psDevNode), "Device node not known",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Mark the device as 'frozen' if not already marked */
+	if (OSAtomicCompareExchange(&psDevNode->eFrozen, _NOT_FROZEN,
+	                            _FROZEN) == _FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p already frozen", __func__,
+		        psDevNode));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure there are no other threads active within this device. */
+	while (OSAtomicRead(&psDevNode->iThreadsActive) > 0)
+	{
+		OSReleaseThreadQuanta();	/* Let other threads execute */
+	}
+
+	/* Attempt to idle the GPU without power-down. If this fails, we try with
+	 * a potential power-down.
+	 */
+	eError = PVRSRVPowerLock(psDevNode);
+	if (eError != PVRSRV_OK)
+	{
+		/* Device is powered down, we can continue
+		 * as there are no in-flight requests in the GPU.
+		 */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unable to Idle Device (%p) [%u/%d]"
+		        " Device powered down",
+		        __func__, psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		bHasPowerLock = IMG_TRUE;
+
+		eError = PVRSRVDeviceIdleRequestKM(psDevNode,
+		                                   &PVRSRVDeviceIsDefaultStateOFF,
+		                                   IMG_TRUE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Failed to Idle. Need to remove the pending _FROZEN state. */
+		(void) OSAtomicExchange(&psDevNode->eFrozen, _NOT_FROZEN);
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unable to Idle Device (%p) [%u/%d]",
+		        __func__, psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+
+		if (bHasPowerLock)
+		{
+			PVRSRVPowerUnlock(psDevNode);
+		}
+		return eError;
+	}
+
+	/* Now change the device-state to not be ACTIVE until we unfreeze */
+	if (psDevNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected Device State %u for device %p",
+		        __func__, psDevNode->eDevState, psDevNode));
+
+		/* Reset device to UNFROZEN */
+		(void) OSAtomicExchange(&psDevNode->eFrozen, _NOT_FROZEN);
+
+		if (bHasPowerLock)
+		{
+			(void) PVRSRVDeviceIdleCancelRequestKM(psDevNode);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+		else
+		{
+			(void) PVRSRVSetDeviceSystemPowerState(psDevNode,
+	                                               PVRSRV_SYS_POWER_STATE_ON,
+	                                               PVRSRV_POWER_FLAGS_NONE);
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_FROZEN);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Device (%p) [%u/%d] FROZEN", __func__,
+	        psDevNode,
+	        psDevNode->sDevId.ui32InternalID,
+	        psDevNode->sDevId.i32KernelDeviceID));
+
+	OSAtomicIncrement(&psDevNode->iTotalFreezes);
+
+	if (bHasPowerLock)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Unfreeze / Thaw the specified device if frozen */
+PVRSRV_ERROR PVRSRVDeviceThaw(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *lpsDevNode;
+	IMG_BOOL           bHasPowerLock = IMG_FALSE;
+
+	/* Verify that the given argument *IS* a recognised device node */
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (lpsDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     lpsDevNode != NULL && lpsDevNode != psDevNode;
+	     lpsDevNode = lpsDevNode->psNext)
+		;
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	PVR_LOG_RETURN_IF_FALSE((lpsDevNode == psDevNode), "Device node not known",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Unfreeze the device */
+	if (OSAtomicCompareExchange(&psDevNode->eFrozen, _FROZEN,
+	                            _NOT_FROZEN) == _NOT_FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device (%p) [%u/%d] already unfrozen",
+		        __func__,
+		        psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDevNode->eDevState != PVRSRV_DEVICE_STATE_FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Unexpected Device state %u for device (%p) [%u/%d]",
+		        __func__, psDevNode->eDevState, psDevNode,
+		        psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+	}
+	else
+	{
+		PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_ACTIVE);
+
+		PVR_DPF((PVR_DBG_VERBOSE, "%s: Device (%p) [%u/%d] UNFROZEN", __func__,
+		        psDevNode,
+		        psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+	}
+
+	/* Now unblock the device by clearing any ForcedIdle state and/or
+	 * powering-on the device if needed.
+	 */
+	eError = PVRSRVPowerLock(psDevNode);
+
+	bHasPowerLock = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
+
+	if (bHasPowerLock)
+	{
+		eError = PVRSRVDeviceIdleCancelRequestKM(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Could not cancel Idle state: Device (%p) [%u/%d] '%s'",
+			        __func__,
+			        psDevNode,
+			        psDevNode->sDevId.ui32InternalID,
+			        psDevNode->sDevId.i32KernelDeviceID,
+			        PVRSRVGetErrorString(eError)));
+		}
+
+		PVRSRVPowerUnlock(psDevNode);
+	}
+	else
+	{
+		/* Force device back into POWER_STATE_ON as it must already be OFF. */
+		(void) PVRSRVSetDeviceSystemPowerState(psDevNode,
+	                                           PVRSRV_SYS_POWER_STATE_ON,
+	                                           PVRSRV_POWER_FLAGS_NONE);
+	}
+
+	/* Unblock any waiting threads */
+	while (OSAtomicRead(&psDevNode->iFreezeCount) > 0)
+	{
+		eError = OSEventObjectSignal(psDevNode->hDeviceThreadEvObj);
+		if (OSAtomicRead(&psDevNode->iFreezeCount) > 0)
+		{
+			/* Sleep for 1ms to allow waiter to receive signal */
+			OSSleepms(1U);
+		}
+	}
+
+	/* Ensure that any blocked queues get rescheduled if we've woken up a
+	 * waiter.
+	 */
+	PVRSRVCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_DRIVER_MODE PVRSRVGetVzModeByDevNum(IMG_UINT32 ui32DevNum)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DRIVER_MODE eRetMode = DRIVER_MODE_NATIVE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Iterate over all devices. */
+	for (psDevNode = psPVRSRVData->psDeviceNodeList;
+		 psDevNode != NULL;
+		 psDevNode = psDevNode->psNext)
+	{
+		if (psDevNode->sDevId.ui32InternalID == ui32DevNum)
+		{
+			eRetMode = psDevNode->psDevConfig->eDriverMode;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	return eRetMode;
+}
+
+/*****************************************************************************
+ End of file (pvrsrv.c)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_bridge_init.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_bridge_init.c
new file mode 100644
index 000000000000..13beb3f5c454
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_bridge_init.c
@@ -0,0 +1,379 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_bridge_init.h"
+#include "srvcore.h"
+
+/* These will go when full bridge gen comes in */
+#if defined(PDUMP)
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+#endif
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+#endif
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+#if defined(SUPPORT_DMA_TRANSFER)
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+#endif
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+PVRSRV_ERROR InitRGXTQBridge(void);
+void DeinitRGXTQBridge(void);
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void);
+void DeinitRGXBREAKPOINTBridge(void);
+#endif
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+#endif
+#endif /* SUPPORT_RGX */
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+#if defined(SUPPORT_SECURE_EXPORT)
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+#endif
+#if defined(PVRSRV_ENABLE_HTB)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+#endif
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+#endif
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+PVRSRV_ERROR InitVALIDATIONBridge(void);
+void DeinitVALIDATIONBridge(void);
+#endif
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+#if defined(SUPPORT_WRAP_EXTMEM)
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+#endif
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+#endif
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+#if defined(SUPPORT_DI_BRG_IMPL)
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+#endif
+
+PVRSRV_ERROR
+ServerBridgeInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	BridgeDispatchTableStartOffsetsInit();
+
+	eError = InitSRVCOREBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSRVCOREBridge");
+
+	eError = InitSYNCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCBridge");
+
+#if defined(PDUMP)
+	eError = InitPDUMPCTRLBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPCTRLBridge");
+#endif
+
+	eError = InitMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitMMBridge");
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+	eError = InitCMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitCMMBridge");
+#endif
+
+#if defined(PDUMP)
+	eError = InitPDUMPMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPMMBridge");
+
+	eError = InitPDUMPBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPBridge");
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = InitDCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDCBridge");
+#endif
+
+	eError = InitCACHEBridge();
+	PVR_LOG_IF_ERROR(eError, "InitCACHEBridge");
+
+#if defined(SUPPORT_SECURE_EXPORT)
+	eError = InitSMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSMMBridge");
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+	eError = InitHTBUFFERBridge();
+	PVR_LOG_IF_ERROR(eError, "InitHTBUFFERBridge");
+#endif
+
+	eError = InitPVRTLBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPVRTLBridge");
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = InitRIBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRIBridge");
+#endif
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+	eError = InitVALIDATIONBridge();
+	PVR_LOG_IF_ERROR(eError, "InitVALIDATIONBridge");
+#endif
+
+
+	eError = InitDEVICEMEMHISTORYBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDEVICEMEMHISTORYBridge");
+
+	eError = InitSYNCTRACKINGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCTRACKINGBridge");
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	eError = InitDMABridge();
+	PVR_LOG_IF_ERROR(eError, "InitDMABridge");
+#endif
+
+#if defined(SUPPORT_RGX)
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+	eError = InitRGXTQBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTQBridge");
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+	eError = InitRGXTA3DBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTA3DBridge");
+
+	#if defined(SUPPORT_USC_BREAKPOINT)
+	eError = InitRGXBREAKPOINTBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXBREAKPOINTBridge");
+#endif
+
+	eError = InitRGXFWDBGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXFWDBGBridge");
+
+#if defined(PDUMP)
+	eError = InitRGXPDUMPBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXPDUMPBridge");
+#endif
+
+	eError = InitRGXHWPERFBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXHWPERFBridge");
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+	eError = InitRGXREGCONFIGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXREGCONFIGBridge");
+#endif
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	eError = InitRGXKICKSYNCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXKICKSYNCBridge");
+#endif
+
+	eError = InitRGXTIMERQUERYBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTIMERQUERYBridge");
+
+#endif /* SUPPORT_RGX */
+
+#if defined(SUPPORT_WRAP_EXTMEM)
+	eError = InitMMEXTMEMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitMMEXTMEMBridge");
+#endif
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	eError = InitSYNCFALLBACKBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCFALLBACKBridge");
+#endif
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	eError = InitDIBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDIBridge");
+#endif
+
+	eError = OSPlatformBridgeInit();
+	PVR_LOG_IF_ERROR(eError, "OSPlatformBridgeInit");
+
+	return eError;
+}
+
+void ServerBridgeDeInit(void)
+{
+	OSPlatformBridgeDeInit();
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	DeinitDIBridge();
+#endif
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	DeinitSYNCFALLBACKBridge();
+#endif
+
+#if defined(SUPPORT_WRAP_EXTMEM)
+	DeinitMMEXTMEMBridge();
+#endif
+
+	DeinitSRVCOREBridge();
+
+	DeinitSYNCBridge();
+
+#if defined(PDUMP)
+	DeinitPDUMPCTRLBridge();
+#endif
+
+	DeinitMMBridge();
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+	DeinitCMMBridge();
+#endif
+
+#if defined(PDUMP)
+	DeinitPDUMPMMBridge();
+
+	DeinitPDUMPBridge();
+#endif
+
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	DeinitDCBridge();
+#endif
+
+	DeinitCACHEBridge();
+
+#if defined(SUPPORT_SECURE_EXPORT)
+	DeinitSMMBridge();
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+	DeinitHTBUFFERBridge();
+#endif
+
+	DeinitPVRTLBridge();
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+	DeinitVALIDATIONBridge();
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	DeinitRIBridge();
+#endif
+
+	DeinitDEVICEMEMHISTORYBridge();
+
+	DeinitSYNCTRACKINGBridge();
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	DeinitDMABridge();
+#endif
+
+#if defined(SUPPORT_RGX)
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+	DeinitRGXTQBridge();
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+	DeinitRGXTA3DBridge();
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+	DeinitRGXBREAKPOINTBridge();
+#endif
+
+	DeinitRGXFWDBGBridge();
+
+#if defined(PDUMP)
+	DeinitRGXPDUMPBridge();
+#endif
+
+	DeinitRGXHWPERFBridge();
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+	DeinitRGXREGCONFIGBridge();
+#endif
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DeinitRGXKICKSYNCBridge();
+#endif
+
+	DeinitRGXTIMERQUERYBridge();
+#endif /* SUPPORT_RGX */
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_pool.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_pool.c
new file mode 100644
index 000000000000..9fa95ea98756
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_pool.c
@@ -0,0 +1,260 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services pool implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a generic pool implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "lock.h"
+#include "dllist.h"
+#include "allocmem.h"
+
+struct _PVRSRV_POOL_
+{
+	POS_LOCK hLock;
+	/* total max number of permitted entries in the pool */
+	IMG_UINT uiMaxEntries;
+	/* currently number of pool entries created. these may be in the pool
+	 * or in-use
+	 */
+	IMG_UINT uiNumBusy;
+	/* number of not-in-use entries currently free in the pool */
+	IMG_UINT uiNumFree;
+
+	DLLIST_NODE sFreeList;
+
+	const IMG_CHAR *pszName;
+
+	PVRSRV_POOL_ALLOC_FUNC *pfnAlloc;
+	PVRSRV_POOL_FREE_FUNC *pfnFree;
+	void *pvPrivData;
+};
+
+typedef struct _PVRSRV_POOL_ENTRY_
+{
+	DLLIST_NODE sNode;
+	void *pvData;
+} PVRSRV_POOL_ENTRY;
+
+PVRSRV_ERROR PVRSRVPoolCreate(PVRSRV_POOL_ALLOC_FUNC *pfnAlloc,
+					PVRSRV_POOL_FREE_FUNC *pfnFree,
+					IMG_UINT32 ui32MaxEntries,
+					const IMG_CHAR *pszName,
+					void *pvPrivData,
+					PVRSRV_POOL **ppsPool)
+{
+	PVRSRV_POOL *psPool;
+	PVRSRV_ERROR eError;
+
+	psPool = OSAllocMem(sizeof(PVRSRV_POOL));
+	PVR_GOTO_IF_NOMEM(psPool, eError, err_alloc);
+
+	eError = OSLockCreate(&psPool->hLock);
+
+	PVR_GOTO_IF_ERROR(eError, err_lock_create);
+
+	psPool->uiMaxEntries = ui32MaxEntries;
+	psPool->uiNumBusy = 0;
+	psPool->uiNumFree = 0;
+	psPool->pfnAlloc = pfnAlloc;
+	psPool->pfnFree = pfnFree;
+	psPool->pvPrivData = pvPrivData;
+	psPool->pszName = pszName;
+
+	dllist_init(&psPool->sFreeList);
+
+	*ppsPool = psPool;
+
+	return PVRSRV_OK;
+
+err_lock_create:
+	OSFreeMem(psPool);
+err_alloc:
+	return eError;
+}
+
+static PVRSRV_ERROR _DestroyPoolEntry(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_ENTRY *psEntry)
+{
+	psPool->pfnFree(psPool->pvPrivData, psEntry->pvData);
+	OSFreeMem(psEntry);
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVPoolDestroy(PVRSRV_POOL *psPool)
+{
+	if (psPool->uiNumBusy != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Attempt to destroy pool %s "
+						"with %u entries still in use",
+						__func__,
+						psPool->pszName,
+						psPool->uiNumBusy));
+		return;
+	}
+
+	OSLockDestroy(psPool->hLock);
+
+	if (psPool->uiNumFree)
+	{
+		PVRSRV_POOL_ENTRY *psEntry;
+		DLLIST_NODE *psChosenNode;
+
+		psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+
+		while (psChosenNode)
+		{
+			dllist_remove_node(psChosenNode);
+
+			psEntry = IMG_CONTAINER_OF(psChosenNode, PVRSRV_POOL_ENTRY, sNode);
+			_DestroyPoolEntry(psPool, psEntry);
+
+			psPool->uiNumFree--;
+
+			psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+		}
+
+		PVR_ASSERT(psPool->uiNumFree == 0);
+	}
+
+	OSFreeMem(psPool);
+}
+
+static PVRSRV_ERROR _CreateNewPoolEntry(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_ENTRY **ppsEntry)
+{
+	PVRSRV_POOL_ENTRY *psNewEntry;
+	PVRSRV_ERROR eError;
+
+	psNewEntry = OSAllocMem(sizeof(PVRSRV_POOL_ENTRY));
+	PVR_GOTO_IF_NOMEM(psNewEntry, eError, err_allocmem);
+
+	dllist_init(&psNewEntry->sNode);
+
+	eError = psPool->pfnAlloc(psPool->pvPrivData, &psNewEntry->pvData);
+
+	PVR_GOTO_IF_ERROR(eError, err_pfn_alloc);
+
+	*ppsEntry = psNewEntry;
+
+	return PVRSRV_OK;
+
+err_pfn_alloc:
+	OSFreeMem(psNewEntry);
+err_allocmem:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVPoolGet(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_TOKEN *hToken,
+					void **ppvDataOut)
+{
+	PVRSRV_POOL_ENTRY *psEntry;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psChosenNode;
+
+	OSLockAcquire(psPool->hLock);
+
+	psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+	if (unlikely(psChosenNode == NULL))
+	{
+		/* no available elements in the pool. try to create one */
+
+		eError = _CreateNewPoolEntry(psPool, &psEntry);
+
+		PVR_GOTO_IF_ERROR(eError, out_unlock);
+	}
+	else
+	{
+		dllist_remove_node(psChosenNode);
+
+		psEntry = IMG_CONTAINER_OF(psChosenNode, PVRSRV_POOL_ENTRY, sNode);
+
+		psPool->uiNumFree--;
+	}
+
+#if defined(DEBUG)
+	/* Don't poison the IN buffer as that is copied from client and would be
+	 * waste of cycles.
+	 */
+	OSCachedMemSet(((IMG_PBYTE)psEntry->pvData)+PVRSRV_MAX_BRIDGE_IN_SIZE,
+			PVRSRV_POISON_ON_ALLOC_VALUE, PVRSRV_MAX_BRIDGE_OUT_SIZE);
+#endif
+
+	psPool->uiNumBusy++;
+	*hToken = psEntry;
+	*ppvDataOut = psEntry->pvData;
+
+out_unlock:
+	OSLockRelease(psPool->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVPoolPut(PVRSRV_POOL *psPool, PVRSRV_POOL_TOKEN hToken)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_POOL_ENTRY *psEntry = hToken;
+
+	PVR_ASSERT(psPool->uiNumBusy > 0);
+
+	OSLockAcquire(psPool->hLock);
+
+	/* put this entry in the pool if the pool has space,
+	 * otherwise free it
+	 */
+	if (psPool->uiNumFree < psPool->uiMaxEntries)
+	{
+		dllist_add_to_tail(&psPool->sFreeList, &psEntry->sNode);
+		psPool->uiNumFree++;
+	}
+	else
+	{
+		eError = _DestroyPoolEntry(psPool, psEntry);
+	}
+
+	psPool->uiNumBusy--;
+
+	OSLockRelease(psPool->hLock);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/ri_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/ri_server.c
new file mode 100644
index 000000000000..370e41dd4553
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/ri_server.c
@@ -0,0 +1,2638 @@
+/*************************************************************************/ /*!
+@File           ri_server.c
+@Title          Resource Information (RI) server implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) server functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#include "img_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+
+#include "srvkm.h"
+#include "lock.h"
+
+/* services/include */
+#include "pvr_ricommon.h"
+
+/* services/server/include/ */
+#include "ri_server.h"
+
+/* services/include/shared/ */
+#include "hash.h"
+/* services/shared/include/ */
+#include "dllist.h"
+
+#include "pmr.h"
+#include "physheap.h"
+
+/* include/device.h */
+#include "device.h"
+
+#if !defined(RI_UNIT_TEST)
+#include "pvrsrv.h"
+#endif
+
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+#define USE_RI_LOCK		1
+
+/*
+ * Initial size use for Hash table. (Used to index the RI list entries).
+ */
+#define _RI_INITIAL_HASH_TABLE_SIZE	64
+
+/*
+ * If this define is set to 1, details of the linked lists (addresses,
+ * prev/next ptrs, etc) are also output when function RIDumpList() is called.
+ */
+#define _DUMP_LINKEDLIST_INFO		0
+
+
+typedef IMG_UINT64 _RI_BASE_T;
+
+
+/* No +1 in SIZE macros since sizeof includes \0 byte in size */
+
+#define RI_PROC_BUF_SIZE    16
+#define RI_ANNO_BUF_SIZE    80
+#define RI_ANNO_FRMT_SIZE (sizeof(RI_ANNO_FRMT))
+
+#define RI_DEV_ID_BUF_SIZE  4
+
+#define RI_MEMDESC_SUM_FRMT     "PID:%d %s MEMDESCs Alloc'd:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K) + "\
+                                                  "Imported:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K) = "\
+                                                     "Total:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K)\n"
+#define RI_MEMDESC_SUM_BUF_SIZE (sizeof(RI_MEMDESC_SUM_FRMT)+5+RI_PROC_BUF_SIZE+30+60)
+
+
+#define RI_PMR_SUM_FRMT     "PID:%d %s PMRs Alloc'd:0x%010" IMG_UINT64_FMTSPECx ", %" IMG_UINT64_FMTSPEC "K  "\
+                                        "[Physical: 0x%010" IMG_UINT64_FMTSPECx ", %" IMG_UINT64_FMTSPEC "K]\n"
+#define RI_PMR_SUM_BUF_SIZE (sizeof(RI_PMR_SUM_FRMT)+(20+40))
+
+#define RI_FREED_BY_DRIVER "{Freed by KM}"
+
+#define RI_PMR_ENTRY_IMPORTED_FRMT_BEGIN "{Imported to DEV:"
+#define RI_PMR_ENTRY_IMPORTED_FRMT_END   "\b}"
+#define RI_PMR_ENTRY_IMPORTED_DEV_FRMT "%2u,"
+#define RI_PMR_ENTRY_IMPORTED_DEV_SIZE 3
+#define RI_PMR_ENTRY_IMPORTED_BUF_SIZE (sizeof(RI_PMR_ENTRY_IMPORTED_FRMT_BEGIN)+sizeof(RI_PMR_ENTRY_IMPORTED_FRMT_END) + PVRSRV_MAX_DEVICES * RI_PMR_ENTRY_IMPORTED_DEV_SIZE)
+
+#define RI_PMR_ENTRY_FRMT      "%%sPID:%%-5d DEV:%%s <%%p>\t%%-%ds\t%%-%ds\t0x%%010" IMG_UINT64_FMTSPECx "\t[0x%%010" IMG_UINT64_FMTSPECx "]\t%%s%%s%%c"
+#define RI_PMR_ENTRY_BUF_SIZE  (sizeof(RI_PMR_ENTRY_FRMT)+(3+5+RI_DEV_ID_BUF_SIZE+16+(PVR_ANNOTATION_MAX_LEN/2)+PHYS_HEAP_NAME_SIZE+10+10)+ sizeof(RI_FREED_BY_DRIVER))
+#define RI_PMR_ENTRY_FRMT_SIZE (sizeof(RI_PMR_ENTRY_FRMT))
+
+/* Use %5d rather than %d so the output aligns in server/kernel.log, debugFS sees extra spaces */
+#define RI_MEMDESC_ENTRY_PROC_FRMT        "[%5d:%s]"
+#define RI_MEMDESC_ENTRY_PROC_BUF_SIZE    (sizeof(RI_MEMDESC_ENTRY_PROC_FRMT)+5+16)
+
+#define RI_SYS_ALLOC_IMPORT_FRMT      "{Import from PID %d}"
+#define RI_SYS_ALLOC_IMPORT_FRMT_SIZE (sizeof(RI_SYS_ALLOC_IMPORT_FRMT)+5)
+static IMG_CHAR g_szSysAllocImport[RI_SYS_ALLOC_IMPORT_FRMT_SIZE];
+
+#define RI_MEMDESC_ENTRY_IMPORT_FRMT     "{Import from PID %d}"
+#define RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE (sizeof(RI_MEMDESC_ENTRY_IMPORT_FRMT)+5)
+
+#define RI_MEMDESC_ENTRY_FRMT      "%%sPID:%%-5d DEV:%%s 0x%%010" IMG_UINT64_FMTSPECx "\t%%-%ds %%s\t0x%%010" IMG_UINT64_FMTSPECx "\t<%%p> %%s%%s %%s%%c"
+#define RI_MEMDESC_ENTRY_BUF_SIZE  (sizeof(RI_MEMDESC_ENTRY_FRMT)+(3+5+RI_DEV_ID_BUF_SIZE+10+PVR_ANNOTATION_MAX_LEN+RI_MEMDESC_ENTRY_PROC_BUF_SIZE+16+\
+                                               RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE+RI_SYS_ALLOC_IMPORT_FRMT_SIZE)) + sizeof(RI_FREED_BY_DRIVER)
+#define RI_MEMDESC_ENTRY_FRMT_SIZE (sizeof(RI_MEMDESC_ENTRY_FRMT))
+
+
+#define RI_FRMT_SIZE_MAX (MAX(RI_MEMDESC_ENTRY_BUF_SIZE,\
+                              MAX(RI_PMR_ENTRY_BUF_SIZE,\
+                                  MAX(RI_MEMDESC_SUM_BUF_SIZE,\
+                                      RI_PMR_SUM_BUF_SIZE))))
+
+typedef struct _RI_PMR_INFO_
+{
+	uintptr_t uiAddr;
+	PHYS_HEAP *psHeap;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	size_t uiLogicalSize;
+	size_t uiPhysicalSize;
+	IMG_CHAR *pszAnnotation;
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	IMG_UINT64 uiXDevices;
+#endif
+} RI_PMR_INFO;
+
+
+#define RI_DEVID_MASK						0xFF /* Highest value is PVRSRV_HOST_DEVICE_ID[255] */
+
+/* Flags for Memdescs(RI_SUBLIST) */
+#define RI_IMPORT_FLAG						(8)  /* Indicates the PMR is an import */
+#define RI_SUBALLOC_FLAG					(9)  /* Indicates the PMR is sub-allocatable */
+
+/* Flags for PMRs(RI_LIST) */
+#define RI_SYSALLOC_PMR_FLAG				(10) /* Indicates the PMR belongs to the 'system' process */
+#define RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG	(11) /* Indicates size counted prior, to prevent double reads */
+
+/* Shared flags */
+#define RI_RACC_FLAG						(12) /* Freed by Driver after disconnect */
+#define RI_VALID_FLAG						(13) /* Valid on creation, Only invalid after delete */
+#define RI_HAS_PMR_INFO						(14) /* Entry has RI_PMR_INFO object rather than reference to a PMR */
+
+#define GET_DEVICE_ID(entry)			((entry)->ui16Flags & RI_DEVID_MASK)
+#define SET_DEVICE_ID(entry,id)			BITMASK_SET((entry)->ui16Flags, (id & RI_DEVID_MASK))
+#define IS_HOST_DEVICE(entry)			(((entry)->ui16Flags & RI_DEVID_MASK) == PVRSRV_HOST_DEVICE_ID)
+
+#define IS_IMPORT(entry)				BIT_ISSET((entry)->ui16Flags, RI_IMPORT_FLAG)
+#define IS_SUBALLOC(entry)				BIT_ISSET((entry)->ui16Flags, RI_SUBALLOC_FLAG)
+
+#define IS_SYSPMR(entry)				BIT_ISSET((entry)->ui16Flags, RI_SYSALLOC_PMR_FLAG)
+#define IS_COUNTED_BY_DEBUGFS(entry)	BIT_ISSET((entry)->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG)
+
+#define IS_RACC(entry)					BIT_ISSET((entry)->ui16Flags, RI_RACC_FLAG)
+#define IS_VALID(entry)					BIT_ISSET((entry)->ui16Flags, RI_VALID_FLAG)
+#define HAS_PMR_INFO(entry)				BIT_ISSET((entry)->ui16Flags, RI_HAS_PMR_INFO)
+
+/* Structure used to make linked sublist of memory allocations (MEMDESC) */
+struct _RI_SUBLIST_ENTRY_
+{
+	DLLIST_NODE				sListNode;
+	DLLIST_NODE				sProcListNode; /* Sublist entries that share the same PID.
+	                                          IMPORTANT: This list does not contain a head node! */
+	CONNECTION_DATA			*psConnection;
+	struct _RI_LIST_ENTRY_	*psRI;
+	IMG_CHAR				*pszTextB; /* Annotation+(NUL)+ProcName+(NUL) */
+	IMG_DEV_VIRTADDR		sVAddr;
+	IMG_UINT64				ui64Offset;
+	IMG_UINT64				ui64Size;
+	IMG_PID					pid;
+	IMG_UINT16				ui16TextBLength; /* Total length of Annotation+(NUL)+ProcName+(NUL) */
+	IMG_UINT16				ui16Flags; /* Refer to above */
+};
+
+/*
+ * Structure used to make linked list of PMRs. Sublists of allocations
+ * (MEMDESCs) made from these PMRs are chained off these entries.
+ */
+struct _RI_LIST_ENTRY_
+{
+	DLLIST_NODE				sListNode;
+	DLLIST_NODE				sSubListFirst;
+	union {
+		PMR *psPMR;
+		RI_PMR_INFO *psPmrInfo;
+	} pmr_info;
+	IMG_PID					pid;
+	IMG_UINT16				ui16SubListCount;
+	IMG_UINT16				ui16Flags; /* Refer to above */
+};
+
+#define GET_KEY(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (void*)(entry)->pmr_info.psPmrInfo \
+			: (void*)(entry)->pmr_info.psPMR \
+	)
+
+#define GET_ADDR(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (void*)(entry)->pmr_info.psPmrInfo->uiAddr \
+			: (void*)(entry)->pmr_info.psPMR \
+	)
+#define GET_HEAP(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->psHeap \
+			: PMR_PhysHeap((entry)->pmr_info.psPMR) \
+	)
+#define GET_DEVNODE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->psDeviceNode \
+			: (PVRSRV_DEVICE_NODE *) PMR_DeviceNode((entry)->pmr_info.psPMR) \
+	)
+#define GET_LOGICAL_SIZE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiLogicalSize \
+			: PMR_LogicalSize((entry)->pmr_info.psPMR) \
+	)
+#define GET_PHYSICAL_SIZE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiPhysicalSize \
+			: PMR_PhysicalSize((entry)->pmr_info.psPMR) \
+	)
+#define GET_NAME(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->pszAnnotation \
+			: PMR_GetAnnotation((entry)->pmr_info.psPMR) \
+	)
+#define GET_XDEVICES(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiXDevices \
+			: PMR_ImportedDevicesMask((entry)->pmr_info.psPMR) \
+	)
+
+/* pszTextB = <Memdesc Annotation>,0,<Proc Name>,0 */
+/* Retrieve the string pointer to the ProcName */
+#define GET_PROC(entry) ((IMG_CHAR*) (strchr((entry)->pszTextB, '\0') + 1))
+
+typedef struct _RI_LIST_ENTRY_ RI_LIST_ENTRY;
+typedef struct _RI_SUBLIST_ENTRY_ RI_SUBLIST_ENTRY;
+
+static HASH_TABLE	*g_pPMR2RIListHashTable;
+static HASH_TABLE	*g_PID2RISublistHashTable;
+
+static POS_LOCK		g_hRILock;
+
+/* Linked list of PMR allocations made against the PVR_SYS_ALLOC_PID and lock
+ * to prevent concurrent access to it.
+ */
+static POS_LOCK		g_hSysAllocPidListLock;
+static DLLIST_NODE	g_sSysAllocPidListHead;
+
+/*
+ * Flag used to indicate if RILock should be destroyed when final PMR entry is
+ * deleted, i.e. if RIDeInitKM() has already been called before that point but
+ * the handle manager has deferred deletion of RI entries.
+ */
+static IMG_BOOL		bRIDeInitDeferred = IMG_FALSE;
+
+/*
+ * Used as head of linked-list of PMR RI entries - this is useful when we wish
+ * to iterate all PMR list entries (when we don't have a PMR ref)
+ */
+static DLLIST_NODE	g_sClientsListHead;
+
+/* Function used to produce string containing info for MEMDESC RI entries (used for both debugfs and kernel log output) */
+static void _GenerateMEMDESCEntryString(RI_SUBLIST_ENTRY *psRISubEntry, IMG_BOOL bDebugFs, IMG_UINT16 ui16MaxStrLen, IMG_CHAR *pszEntryString);
+/* Function used to produce string containing info for PMR RI entries (used for both debugfs and kernel log output) */
+static void _GeneratePMREntryString(RI_LIST_ENTRY *psRIEntry, IMG_BOOL bDebugFs, IMG_UINT16 ui16MaxStrLen, IMG_CHAR *pszEntryString);
+
+static PVRSRV_ERROR _DumpAllEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DeleteAllEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DeleteAllProcEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DumpList(PMR *psPMR, IMG_PID pid);
+#define _RIOutput(x) PVR_LOG(x)
+
+static IMG_UINT32
+_ProcHashFunc(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+static IMG_UINT32
+_ProcHashFunc(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_UINT32 *p = (IMG_UINT32 *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = *p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+static IMG_BOOL
+_ProcHashComp(size_t uKeySize, void *pKey1, void *pKey2);
+
+static IMG_BOOL
+_ProcHashComp(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_UINT32 *p1 = (IMG_UINT32 *)pKey1;
+	IMG_UINT32 *p2 = (IMG_UINT32 *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static void _RILock(void)
+{
+#if (USE_RI_LOCK == 1)
+	OSLockAcquire(g_hRILock);
+#endif
+}
+
+static void _RIUnlock(void)
+{
+#if (USE_RI_LOCK == 1)
+	OSLockRelease(g_hRILock);
+#endif
+}
+
+/* This value maintains a count of the number of PMRs attributed to the
+ * PVR_SYS_ALLOC_PID. Access to this value is protected by g_hRILock, so it
+ * does not need to be an ATOMIC_T.
+ */
+static IMG_UINT32 g_ui32SysAllocPMRCount;
+
+static IMG_BOOL RICheckListHandle(RI_HANDLE hRIHandle)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *) hRIHandle, *psTableEntry;
+	void *pPMRHashKey;
+	IMG_BOOL bValid = IMG_FALSE;
+
+	if ((GET_DEVICE_ID(psRIEntry) > PVRSRV_MAX_DEVICES &&
+	     !IS_HOST_DEVICE(psRIEntry)) ||
+	    !g_pPMR2RIListHashTable)
+	{
+		//Shortcut check
+		return IMG_FALSE;
+	}
+
+	pPMRHashKey = GET_KEY(psRIEntry);
+
+	_RILock();
+
+	/* Look-up psPMR in Hash Table */
+	psTableEntry = (RI_LIST_ENTRY *) HASH_Retrieve_Extended(g_pPMR2RIListHashTable,
+	                                                        &pPMRHashKey);
+	if (psTableEntry != NULL)
+	{
+		bValid = IS_VALID(psTableEntry);
+	}
+
+	_RIUnlock();
+
+	return bValid;
+}
+
+static IMG_BOOL RICheckSubListHandle(RI_HANDLE hRIHandle)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry, *psTableEntry;
+	uintptr_t hashData;
+	IMG_PID pid;
+	IMG_BOOL bValid = IMG_FALSE;
+
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if ((GET_DEVICE_ID(psRISubEntry) > PVRSRV_MAX_DEVICES) || !g_PID2RISublistHashTable)
+	{
+		//Shortcut check
+		return bValid;
+	}
+	pid = psRISubEntry->pid;
+
+	_RILock();
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (hashData)
+	{
+		psTableEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+		bValid = IS_VALID(psTableEntry);
+	}
+
+	/* Release RI Lock */
+	_RIUnlock();
+	return bValid;
+}
+
+PVRSRV_ERROR RIInitKM(void)
+{
+	IMG_INT iCharsWritten;
+	PVRSRV_ERROR eError;
+
+	bRIDeInitDeferred = IMG_FALSE;
+
+	iCharsWritten = OSSNPrintf(g_szSysAllocImport,
+	            RI_SYS_ALLOC_IMPORT_FRMT_SIZE,
+	            RI_SYS_ALLOC_IMPORT_FRMT,
+	            PVR_SYS_ALLOC_PID);
+	PVR_LOG_IF_FALSE((iCharsWritten>0 && iCharsWritten<(IMG_INT32)RI_SYS_ALLOC_IMPORT_FRMT_SIZE),
+			"OSSNPrintf failed to initialise g_szSysAllocImport");
+
+	eError = OSLockCreate(&g_hSysAllocPidListLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: OSLockCreate (g_hSysAllocPidListLock) failed (returned %d)",
+		         __func__,
+		         eError));
+	}
+	dllist_init(&(g_sSysAllocPidListHead));
+#if (USE_RI_LOCK == 1)
+	eError = OSLockCreate(&g_hRILock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: OSLockCreate (g_hRILock) failed (returned %d)",
+		         __func__,
+		         eError));
+	}
+#endif
+	return eError;
+}
+
+void RIDeInitKM(void)
+{
+#if (USE_RI_LOCK == 1)
+	if (g_pPMR2RIListHashTable != NULL && HASH_Count(g_pPMR2RIListHashTable) > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: called with %d entries remaining - deferring OSLockDestroy()",
+		         __func__,
+		         HASH_Count(g_pPMR2RIListHashTable)));
+		bRIDeInitDeferred = IMG_TRUE;
+	}
+	else
+	{
+		OSLockDestroy(g_hRILock);
+		OSLockDestroy(g_hSysAllocPidListLock);
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RILockAcquireKM
+
+ @Description
+            Acquires the RI Lock (which protects the integrity of the RI
+            linked lists). Caller will be suspended until lock is acquired.
+
+ @Return	None
+
+******************************************************************************/
+void RILockAcquireKM(void)
+{
+	_RILock();
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RILockReleaseKM
+
+ @Description
+            Releases the RI Lock (which protects the integrity of the RI
+            linked lists).
+
+ @Return	None
+
+******************************************************************************/
+void RILockReleaseKM(void)
+{
+	_RIUnlock();
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWritePMREntryWithOwnerKM
+
+ @Description
+            Writes a new Resource Information list entry.
+            The new entry will be inserted at the head of the list of
+            PMR RI entries and assigned the values provided.
+
+ @input     psPMR - Reference (handle) to the PMR to which this reference relates
+
+ @input     ui32Owner - PID of the process which owns the allocation. This
+                        may not be the current process (e.g. a request to
+                        grow a buffer may happen in the context of a kernel
+                        thread, or we may import further resource for a
+                        suballocation made from the FW heap which can then
+                        also be utilized by other processes)
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWritePMREntryWithOwnerKM(PMR *psPMR,
+                                        IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR *pPMRHashKey = psPMR;
+	RI_LIST_ENTRY *psRIEntry;
+	uintptr_t hashData;
+
+	/* if Hash table has not been created, create it now */
+	if (!g_pPMR2RIListHashTable)
+	{
+		g_pPMR2RIListHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(PMR*), HASH_Func_Default, HASH_Key_Comp_Default);
+		g_PID2RISublistHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(IMG_PID), _ProcHashFunc, _ProcHashComp);
+	}
+	PVR_LOG_RETURN_IF_NOMEM(g_pPMR2RIListHashTable, "HASH_Create_Extended");
+	PVR_LOG_RETURN_IF_NOMEM(g_PID2RISublistHashTable, "HASH_Create_Extended");
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	psRIEntry = (RI_LIST_ENTRY *)hashData;
+	if (psRIEntry == NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)PMR_DeviceNode(psPMR);
+
+		/*
+		 * If failed to find a matching existing entry, create a new one
+		 */
+		psRIEntry = (RI_LIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_LIST_ENTRY));
+		PVR_LOG_GOTO_IF_NOMEM(psRIEntry, eError, exit_);
+		{
+			PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+			IMG_UINT32 ui32RICount = HASH_Count(g_pPMR2RIListHashTable);
+
+			/*
+			 * Add new RI Entry
+			 */
+			if (ui32RICount == 0)
+			{
+				/* Initialise PMR entry linked-list head */
+				dllist_init(&g_sClientsListHead);
+			}
+			else if (ui32RICount == IMG_UINT32_MAX)
+			{
+				PVR_LOG_GOTO_WITH_ERROR("ui32RICount", eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, exit_);
+			}
+
+			dllist_init (&(psRIEntry->sListNode));
+			dllist_init (&(psRIEntry->sSubListFirst));
+			BIT_SET(psRIEntry->ui16Flags, RI_VALID_FLAG);
+
+			/* Check if this PMR should be accounted for under the
+			 * PVR_SYS_ALLOC_PID debugFS entry. This should happen if
+			 * we are in the driver init phase, the flags indicate
+			 * this is a FW Main allocation (made from FW heap)
+			 * or the owner PID is PVR_SYS_ALLOC_PID.
+			 * Also record host dev node allocs on the system PID.
+			 */
+			if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+				PVRSRV_CHECK_FW_MAIN(uiPMRFlags) ||
+				ui32Owner == PVR_SYS_ALLOC_PID ||
+				psDeviceNode == PVRSRVGetPVRSRVData()->psHostMemDeviceNode)
+			{
+				BIT_SET(psRIEntry->ui16Flags, RI_SYSALLOC_PMR_FLAG);
+				psRIEntry->pid = PVR_SYS_ALLOC_PID;
+				OSLockAcquire(g_hSysAllocPidListLock);
+				/* Add this psRIEntry to the list of entries for PVR_SYS_ALLOC_PID */
+				dllist_add_to_tail(&g_sSysAllocPidListHead, (PDLLIST_NODE)&(psRIEntry->sListNode));
+				OSLockRelease(g_hSysAllocPidListLock);
+				g_ui32SysAllocPMRCount++;
+			}
+			else
+			{
+				psRIEntry->pid = ui32Owner;
+				dllist_add_to_tail(&g_sClientsListHead, (PDLLIST_NODE)&(psRIEntry->sListNode));
+			}
+		}
+
+		psRIEntry->pmr_info.psPMR = psPMR;
+		SET_DEVICE_ID(psRIEntry, psDeviceNode->sDevId.ui32InternalID);
+
+		/* Create index entry in Hash Table */
+		HASH_Insert_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey, (uintptr_t)psRIEntry);
+
+		/* Store phRIHandle in PMR structure, so it can delete the associated RI entry when it destroys the PMR */
+		PMRStoreRIHandle(psPMR, psRIEntry);
+	}
+exit_:
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWritePMREntryKM
+
+ @Description
+            Writes a new Resource Information list entry.
+            The new entry will be inserted at the head of the list of
+            PMR RI entries and assigned the values provided.
+
+ @input     psPMR - Reference (handle) to the PMR to which this reference relates
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWritePMREntryKM(PMR *psPMR)
+{
+	return RIWritePMREntryWithOwnerKM(psPMR,
+	                                  OSGetCurrentClientProcessIDKM());
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWriteMEMDESCEntryKM
+
+ @Description
+            Writes a new Resource Information sublist entry.
+            The new entry will be inserted at the head of the sublist of
+            the indicated PMR list entry, and assigned the values provided.
+
+ @input     psConnection - Reference to the Services connection
+ @input     psDeviceNode - Reference to the device node
+ @input     psPMR - Reference (handle) to the PMR to which this MEMDESC RI entry relates
+ @input     ui32TextBSize - Length of string provided in psz8TextB parameter
+ @input     psz8TextB - String describing this secondary reference (may be null)
+ @input     ui64Offset - Offset from the start of the PMR at which this allocation begins
+ @input     ui64Size - Size of this allocation
+ @input     uiFlags - Flags indicating if this is a sub-allocation or an import
+ @output    phRIHandle - Handle to the created RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWriteMEMDESCEntryKM(void* psConnection,
+                                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   PMR *psPMR,
+                                   IMG_UINT32 ui32TextBSize,
+                                   const IMG_CHAR *psz8TextB,
+                                   IMG_UINT64 ui64Offset,
+                                   IMG_UINT64 ui64Size,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   RI_HANDLE *phRIHandle)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry;
+	RI_LIST_ENTRY *psRIEntry;
+	PMR *pPMRHashKey = psPMR;
+	uintptr_t hashData;
+	IMG_PID	pid;
+
+	IMG_CHAR szProcName[RI_PROC_BUF_SIZE];
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	/* Check Hash tables have been created (meaning at least one PMR has been defined) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(g_pPMR2RIListHashTable, "g_pPMR2RIListHashTable");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(g_PID2RISublistHashTable, "g_PID2RISublistHashTable");
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phRIHandle, "phRIHandle");
+
+	/* Only allow request for a firmware context that comes from a direct bridge
+	 * (psConnection == NULL). */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(!(psConnection != NULL && PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags)),
+	                                "psConnection for firmware context");
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	*phRIHandle = NULL;
+
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	psRIEntry = (RI_LIST_ENTRY *)hashData;
+	if (!psRIEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		PVR_DPF((PVR_DBG_ERROR, "RI Entry not found for PMR: '%s'", PMR_GetAnnotation(psPMR)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_SUBLIST_ENTRY));
+	if (!psRISubEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		/* Error - no memory to allocate for new RI sublist entry */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	else
+	{
+		psRISubEntry->psRI = psRIEntry;
+		dllist_add_to_head(&(psRIEntry->sSubListFirst), &(psRISubEntry->sListNode));
+
+		/* Increment number of entries in sublist */
+		psRIEntry->ui16SubListCount++;
+		BIT_SET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+	}
+
+	if (PVRSRV_CHECK_RI_IMPORT(uiFlags))
+	{
+		BIT_SET(psRISubEntry->ui16Flags, RI_IMPORT_FLAG);
+	}
+	else
+	{
+		BIT_UNSET(psRISubEntry->ui16Flags, RI_IMPORT_FLAG);
+	}
+
+	if (PVRSRV_CHECK_RI_SUBALLOC(uiFlags))
+	{
+		BIT_SET(psRISubEntry->ui16Flags, RI_SUBALLOC_FLAG);
+	}
+	else
+	{
+		BIT_UNSET(psRISubEntry->ui16Flags, RI_SUBALLOC_FLAG);
+	}
+
+	/* If allocation is made during device or driver initialisation,
+	 * track the MEMDESC entry under PVR_SYS_ALLOC_PID, otherwise use
+	 * the current PID. */
+	{
+		PVRSRV_DEVICE_NODE *psRIDeviceNode = GET_DEVNODE(psRISubEntry->psRI);
+		IMG_INT iRet;
+
+		/* HostMemDevice doesn't update eDevState hence there's no need to test
+		 * for it. */
+		if (psRIDeviceNode == PVRSRVGetPVRSRVData()->psHostMemDeviceNode)
+		{
+			/* Imports on HostMemDev should be attributed to the importing
+			 * process. This way if an import is not freed before disconnect
+			 * it will be outlined in the gpu_mem_area for the given process.
+			 * Otherwise attribute the rest of the records to PVR_SYS_ALLOC_PID. */
+			if (IS_IMPORT(psRISubEntry))
+			{
+				psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+				iRet = OSStringSafeCopy(szProcName, OSGetCurrentClientProcessNameKM(),
+				                        RI_PROC_BUF_SIZE);
+			}
+			else
+			{
+				psRISubEntry->pid = PVR_SYS_ALLOC_PID;
+				iRet = OSStringSafeCopy(szProcName, "SysProc", RI_PROC_BUF_SIZE);
+
+				if (psRISubEntry->pid != psRISubEntry->psRI->pid)
+				{
+					PVR_LOG(("%s(1): current PID = %u, RI entry PID = %u, RI sub-entry PID = %u",
+					         __func__, OSGetCurrentClientProcessIDKM(), psRISubEntry->psRI->pid,
+					         psRISubEntry->pid));
+				}
+			}
+		}
+		else
+		{
+			/* All allocations done during device initialisation or belonging
+			 * to the Firmware should be attributed to PVR_SYS_ALLOC_PID. */
+			if (psRIDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+			    PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags))
+			{
+				psRISubEntry->pid = PVR_SYS_ALLOC_PID;
+				iRet = OSStringSafeCopy(szProcName, "SysProc", RI_PROC_BUF_SIZE);
+
+				if (psRISubEntry->pid != psRISubEntry->psRI->pid)
+				{
+					PVR_LOG(("%s(2): current PID = %u, RI entry PID = %u, RI sub-entry PID = %u",
+					         __func__, OSGetCurrentClientProcessIDKM(), psRISubEntry->psRI->pid,
+					         psRISubEntry->pid));
+				}
+			}
+			else
+			{
+				psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+				iRet = OSStringSafeCopy(szProcName, OSGetCurrentClientProcessNameKM(),
+				                        RI_PROC_BUF_SIZE);
+			}
+		}
+		if (iRet < 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: process name has been truncated from '%s' to '%s'",
+			         __func__, OSGetCurrentClientProcessNameKM(), szProcName));
+			psRISubEntry->ui16TextBLength = RI_PROC_BUF_SIZE - 1;
+		}
+		else
+		{
+			psRISubEntry->ui16TextBLength = iRet;
+		}
+
+		if (psz8TextB == NULL)
+		{
+			psz8TextB = "";
+			ui32TextBSize = 0;
+		}
+		if (ui32TextBSize > (RI_ANNO_BUF_SIZE))
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: TextBSize too long (%u). Text will be truncated "
+					 "to %d characters", __func__,
+					 ui32TextBSize, RI_ANNO_BUF_SIZE));
+			ui32TextBSize = RI_ANNO_BUF_SIZE;
+		}
+
+		/* copy TextB field data plus separator char and terminator */
+		psRISubEntry->ui16TextBLength += ui32TextBSize + 2;
+		psRISubEntry->pszTextB = OSAllocZMemNoStats(psRISubEntry->ui16TextBLength);
+		if (!psRISubEntry->pszTextB)
+		{
+			OSFreeMemNoStats(psRISubEntry);
+			_RIUnlock();
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		/* TextB is formatted as Annotation+NullTerm+ProcName so we can still
+		 * print annotation without proc name.
+		 * If any of the strings is too long it will be truncated. */
+		(void) OSStringSafeCopy(psRISubEntry->pszTextB, psz8TextB, ui32TextBSize + 1);
+		(void) OSStringSafeCopy(psRISubEntry->pszTextB + ui32TextBSize + 1, szProcName,
+		                        psRISubEntry->ui16TextBLength - ui32TextBSize - 1);
+
+		psRISubEntry->ui64Offset = ui64Offset;
+		psRISubEntry->ui64Size = ui64Size;
+		psRISubEntry->psConnection = psConnection;
+		dllist_init (&(psRISubEntry->sProcListNode));
+	}
+
+	/*
+	 *	Now insert this MEMDESC into the proc list
+	 */
+	/* look-up pid in Hash Table */
+	pid = psRISubEntry->pid;
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (!hashData)
+	{
+		/*
+		 * No allocations for this pid yet
+		 */
+		if (!HASH_Insert_Extended(g_PID2RISublistHashTable,
+		                          (void *) &pid,
+		                          (uintptr_t) &psRISubEntry->sProcListNode))
+		{
+			dllist_remove_node(&psRISubEntry->sListNode);
+			psRIEntry->ui16SubListCount--;
+			OSFreeMemNoStats(psRISubEntry->pszTextB);
+			OSFreeMemNoStats(psRISubEntry);
+
+			_RIUnlock();
+			return PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED;
+		}
+	}
+	else
+	{
+		/*
+		 * Insert allocation into pid allocations linked list
+		 */
+		PDLLIST_NODE currentNode = (PDLLIST_NODE)hashData;
+
+		/*
+		 * Insert new entry
+		 */
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sProcListNode));
+	}
+
+	*phRIHandle = (RI_HANDLE)psRISubEntry;
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWriteProcListEntryKM
+
+ @Description
+            Write a new entry in the process list directly. We have to do this
+            because there might be no, multiple or changing PMR handles.
+
+            In the common case we have a PMR that will be added to the PMR list
+            and one or several MemDescs that are associated to it in a sub-list.
+            Additionally these MemDescs will be inserted in the per-process list.
+
+            There might be special descriptors from e.g. new user APIs that
+            are associated with no or multiple PMRs and not just one.
+            These can be now added to the per-process list (as RI_SUBLIST_ENTRY)
+            directly with this function and won't be listed in the PMR list (RIEntry)
+            because there might be no PMR.
+
+            To remove entries from the per-process list, just use
+            RIDeleteMEMDESCEntryKM().
+
+ @input     psConnection - Reference to the Services connection
+ @input     psDeviceNode - Reference to the device node
+ @input     psz8TextB - String describing this secondary reference (may be null)
+ @input     ui64Size - Size of this allocation
+ @input     ui64DevVAddr - Virtual address of this entry
+ @output    phRIHandle - Handle to the created RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWriteProcListEntryKM(void* psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32TextBSize,
+                                    const IMG_CHAR *psz8TextB,
+                                    IMG_UINT64 ui64Size,
+                                    IMG_UINT64 ui64DevVAddr,
+                                    RI_HANDLE *phRIHandle)
+{
+	uintptr_t hashData = 0;
+	IMG_PID		pid;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_INT iRet;
+
+	if (!g_pPMR2RIListHashTable)
+	{
+		g_pPMR2RIListHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(PMR*), HASH_Func_Default, HASH_Key_Comp_Default);
+		g_PID2RISublistHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(IMG_PID), _ProcHashFunc, _ProcHashComp);
+
+		if (!g_pPMR2RIListHashTable || !g_PID2RISublistHashTable)
+		{
+			/* Error - no memory to allocate for Hash table(s) */
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	*phRIHandle = NULL;
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_SUBLIST_ENTRY));
+	if (!psRISubEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		/* Error - no memory to allocate for new RI sublist entry */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	BIT_SET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+
+	psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+	SET_DEVICE_ID(psRISubEntry, psDeviceNode->sDevId.ui32InternalID);
+
+	if (psz8TextB == NULL)
+	{
+		psz8TextB = "";
+		ui32TextBSize = 0;
+	}
+	if (ui32TextBSize > (RI_ANNO_BUF_SIZE))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: TextBSize too long (%u). Text will be truncated "
+				 "to %d characters", __func__,
+				 ui32TextBSize, RI_ANNO_BUF_SIZE));
+		ui32TextBSize = RI_ANNO_BUF_SIZE;
+	}
+
+	/* copy TextB field data */
+	psRISubEntry->ui16TextBLength = ui32TextBSize + RI_PROC_BUF_SIZE;
+	psRISubEntry->pszTextB = OSAllocZMemNoStats(psRISubEntry->ui16TextBLength);
+	if (!psRISubEntry->pszTextB)
+	{
+		OSFreeMemNoStats(psRISubEntry);
+		_RIUnlock();
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	/* TextB is formatted as Annotation+NullTerm+ProcName so we can still
+	 * print annotation without proc name.
+	 */
+	(void) OSStringSafeCopy(psRISubEntry->pszTextB, psz8TextB, ui32TextBSize + 1);
+	iRet = OSStringSafeCopy(psRISubEntry->pszTextB + ui32TextBSize + 1,
+	                        OSGetCurrentClientProcessNameKM(),
+	                        psRISubEntry->ui16TextBLength - ui32TextBSize - 1);
+	if (iRet < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: process name has been truncated from '%s' to '%s'",
+		         __func__, OSGetCurrentClientProcessNameKM(),
+		         psRISubEntry->pszTextB + ui32TextBSize + 1));
+	}
+	psRISubEntry->ui64Size = ui64Size;
+	psRISubEntry->sVAddr.uiAddr = ui64DevVAddr;
+	psRISubEntry->psConnection = psConnection;
+	dllist_init (&(psRISubEntry->sProcListNode));
+
+	/*
+	 *	Now insert this MEMDESC into the proc list
+	 */
+	/* look-up pid in Hash Table */
+	pid = psRISubEntry->pid;
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (!hashData)
+	{
+		/*
+		 * No allocations for this pid yet
+		 */
+		if (!HASH_Insert_Extended(g_PID2RISublistHashTable,
+		                          (void *) &pid,
+		                          (uintptr_t) &psRISubEntry->sProcListNode))
+		{
+			OSFreeMemNoStats(psRISubEntry->pszTextB);
+			OSFreeMemNoStats(psRISubEntry);
+			_RIUnlock();
+			return PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED;
+		}
+	}
+	else
+	{
+		/*
+		 * Insert allocation into pid allocations linked list
+		 */
+		PDLLIST_NODE currentNode = (PDLLIST_NODE)hashData;
+
+		/*
+		 * Insert new entry
+		 */
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sProcListNode));
+	}
+	*phRIHandle = (RI_HANDLE)psRISubEntry;
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIUpdateMEMDESCAddrKM
+
+ @Description
+            Update a Resource Information entry.
+
+ @input     hRIHandle - Handle of object whose reference info is to be updated
+ @input     sVAddr - New address for the RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIUpdateMEMDESCAddrKM(RI_HANDLE hRIHandle,
+								   IMG_DEV_VIRTADDR sVAddr)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hRIHandle, "hRIHandle");
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if (!IS_VALID(psRISubEntry))
+	{
+		/* Pointer does not point to valid structure */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	psRISubEntry->sVAddr.uiAddr = sVAddr.uiAddr;
+
+	/* Release RI lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RIDeletePMREntryUnlocked(RI_LIST_ENTRY *psRIEntry)
+{
+	PMR			*pPMRHashKey;
+	uintptr_t hashValue = 0;
+
+	/* Remove the HASH table index entry */
+	pPMRHashKey = GET_KEY(psRIEntry);
+	hashValue = HASH_Remove_Extended(g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "RI");
+
+	BIT_UNSET(psRIEntry->ui16Flags, RI_VALID_FLAG);
+
+	/* Remove PMR entry from linked-list of PMR entries */
+	dllist_remove_node((PDLLIST_NODE)&(psRIEntry->sListNode));
+
+	if (IS_SYSPMR(psRIEntry))
+	{
+		g_ui32SysAllocPMRCount--;
+	}
+
+	if (IS_RACC(psRIEntry))
+	{
+		OSFreeMemNoStats(psRIEntry->pmr_info.psPmrInfo);
+	}
+
+	/* Now, free the memory used to store the RI entry */
+	OSFreeMemNoStats(psRIEntry);
+	psRIEntry = NULL;
+
+	/* If the hash table is now empty we can delete the RI hash table */
+	if (HASH_Count(g_pPMR2RIListHashTable) == 0)
+	{
+		HASH_Delete(g_pPMR2RIListHashTable);
+		g_pPMR2RIListHashTable = NULL;
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RICreateAndSetPmrInfo(RI_LIST_ENTRY *const psRIEntry)
+{
+	uintptr_t uiEntry;
+	void* pPMRHashKey;
+	const IMG_CHAR *const pszAnnotation = PMR_GetAnnotation(psRIEntry->pmr_info.psPMR);
+	const IMG_UINT32 uiLength = OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN) + 1;
+
+	RI_PMR_INFO *psPmrInfo = OSAllocZMemNoStats(sizeof(*psPmrInfo) + uiLength);
+	PVR_LOG_RETURN_IF_NOMEM(psPmrInfo, "OSAllocZMemNoStats");
+
+	psPmrInfo->uiAddr = (uintptr_t) psRIEntry->pmr_info.psPMR;
+	psPmrInfo->psHeap = PMR_PhysHeap(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->psDeviceNode = (PVRSRV_DEVICE_NODE *) PMR_DeviceNode(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->uiLogicalSize = PMR_LogicalSize(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->uiPhysicalSize = PMR_PhysicalSize(psRIEntry->pmr_info.psPMR);
+
+	psPmrInfo->pszAnnotation = IMG_OFFSET_ADDR(psPmrInfo, sizeof(*psPmrInfo));
+	OSStringSafeCopy(psPmrInfo->pszAnnotation, pszAnnotation, uiLength);
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	psPmrInfo->uiXDevices = PMR_ImportedDevicesMask(psRIEntry->pmr_info.psPMR);
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+	/* Swap the key from the PMR address to the psPmrInfo's address. */
+
+	pPMRHashKey = GET_KEY(psRIEntry);
+	uiEntry = HASH_Remove_Extended(g_pPMR2RIListHashTable, (void*) &pPMRHashKey);
+	/* Should be removing itself to change the key. */
+	PVR_ASSERT(((void*) uiEntry) == ((void*) psRIEntry));
+
+	psRIEntry->pmr_info.psPmrInfo = psPmrInfo;
+	BIT_SET(psRIEntry->ui16Flags, RI_HAS_PMR_INFO);
+
+	pPMRHashKey = GET_KEY(psRIEntry);
+	HASH_Insert_Extended(g_pPMR2RIListHashTable, (void*) &pPMRHashKey, (uintptr_t) psRIEntry);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeletePMREntryKM
+
+ @Description
+            Delete a Resource Information entry.
+
+ @input     hRIHandle - Handle of object whose reference info is to be deleted
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeletePMREntryKM(RI_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_LIST_ENTRY *psRIEntry;
+
+	psRIEntry = (RI_LIST_ENTRY *) hRIHandle;
+
+	if (!RICheckListHandle(hRIHandle))
+	{
+		/* Pointer does not point to valid structure. */
+		PVR_LOG_GOTO_WITH_ERROR("RICheckListHandle", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrReturn);
+	}
+	else if (IS_RACC(psRIEntry))
+	{
+		/* Keep this entry so that it can be inspected as a memory leak in the
+		 * gpu_mem_area stats. */
+		return PVRSRV_OK;
+	}
+
+	if (psRIEntry->ui16SubListCount != 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%p not deleted. RIEntry(%s) still has %u allocation(s)",
+		         psRIEntry, GET_NAME(psRIEntry), psRIEntry->ui16SubListCount));
+
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP,
+		                    ErrCreateAndAssignPmrInfo);
+	}
+
+	_RILock();
+
+	eError = RIDeletePMREntryUnlocked(psRIEntry);
+
+	_RIUnlock();
+
+	/* If deInit has been deferred, we can now destroy the RI Lock */
+	if (bRIDeInitDeferred && (g_pPMR2RIListHashTable == NULL || HASH_Count(g_pPMR2RIListHashTable) == 0))
+	{
+		OSLockDestroy(g_hRILock);
+	}
+
+	return PVRSRV_OK;
+
+ErrCreateAndAssignPmrInfo:
+	{
+		PVRSRV_ERROR eError2;
+		_RILock();
+		eError2 = _RICreateAndSetPmrInfo(psRIEntry);
+		_RIUnlock();
+		PVR_LOG_RETURN_IF_ERROR(eError2, "_RICreateAndSetPmrInfo");
+	}
+ErrReturn:
+	return eError;
+}
+
+static PVRSRV_ERROR RIDeleteMemdescEntryUnlocked(RI_SUBLIST_ENTRY *psRISubEntry)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	uintptr_t hashData;
+	uintptr_t hashValue = 0;
+	IMG_PID pid;
+
+	/* For entries which do have a parent PMR remove the node from the sublist */
+	if (psRISubEntry->psRI)
+	{
+		psRIEntry = (RI_LIST_ENTRY *)psRISubEntry->psRI;
+
+		/* Now, remove entry from the sublist */
+		dllist_remove_node(&(psRISubEntry->sListNode));
+	}
+
+	BIT_UNSET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+
+	/* Remove the entry from the proc allocations linked list */
+	pid = psRISubEntry->pid;
+	/* If this is the only allocation for this pid, just remove it from the hash table */
+	if (dllist_get_next_node(&(psRISubEntry->sProcListNode)) == NULL)
+	{
+		hashValue = HASH_Remove_Extended(g_PID2RISublistHashTable, (void *)&pid);
+		PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "PID");
+		/* Delete the hash table if there are now no entries */
+		if (HASH_Count(g_PID2RISublistHashTable) == 0)
+		{
+			HASH_Delete(g_PID2RISublistHashTable);
+			g_PID2RISublistHashTable = NULL;
+		}
+	}
+	else
+	{
+		hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+		if ((PDLLIST_NODE)hashData == &(psRISubEntry->sProcListNode))
+		{
+			hashValue = HASH_Remove_Extended(g_PID2RISublistHashTable, (void *)&pid);
+			PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "PID");
+			HASH_Insert_Extended (g_PID2RISublistHashTable, (void *)&pid, (uintptr_t)dllist_get_next_node(&(psRISubEntry->sProcListNode)));
+		}
+	}
+	dllist_remove_node(&(psRISubEntry->sProcListNode));
+
+	/* Now, free the memory used to store the sublist entry */
+	OSFreeMemNoStats(psRISubEntry->pszTextB);
+	OSFreeMemNoStats(psRISubEntry);
+	psRISubEntry = NULL;
+
+	/*
+	 * Decrement number of entries in sublist if this MemDesc had a parent entry.
+	 */
+	if (psRIEntry)
+	{
+		psRIEntry->ui16SubListCount--;
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeleteMEMDESCEntryKM
+
+ @Description
+            Delete a Resource Information entry.
+            Entry can be from RIEntry list or ProcList.
+
+ @input     hRIHandle - Handle of object whose reference info is to be deleted
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeleteMEMDESCEntryKM(RI_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_SUBLIST_ENTRY *psRISubEntry;
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if (!RICheckSubListHandle(hRIHandle))
+	{
+		/* Pointer does not point to valid structure */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (IS_RACC(psRISubEntry))
+	{
+		return PVRSRV_OK;
+	}
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = RIDeleteMemdescEntryUnlocked(psRISubEntry);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	/*
+	 * Make the handle NULL once MEMDESC RI entry is deleted
+	 */
+	hRIHandle = NULL;
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeleteListKM
+
+ @Description
+            Delete all Resource Information entries and free associated
+            memory.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeleteListKM(void)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+
+	_RILock();
+
+	if (g_pPMR2RIListHashTable)
+	{
+		eResult = HASH_Iterate(g_pPMR2RIListHashTable, (HASH_pfnCallback)_DeleteAllEntries, NULL);
+		if (eResult == PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+		{
+			/*
+			 * PVRSRV_ERROR_RESOURCE_UNAVAILABLE is used to stop the Hash iterator when
+			 * the hash table gets deleted as a result of deleting the final PMR entry,
+			 * so this is not a real error condition...
+			 */
+			eResult = PVRSRV_OK;
+		}
+	}
+
+	/* After the run through the RIHashTable that holds the PMR entries there might be
+	 * still entries left in the per-process hash table because they were added with
+	 * RIWriteProcListEntryKM() and have no PMR parent associated.
+	 */
+	if (g_PID2RISublistHashTable)
+	{
+		eResult = HASH_Iterate(g_PID2RISublistHashTable, (HASH_pfnCallback) _DeleteAllProcEntries, NULL);
+		if (eResult == PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+		{
+			/*
+			 * PVRSRV_ERROR_RESOURCE_UNAVAILABLE is used to stop the Hash iterator when
+			 * the hash table gets deleted as a result of deleting the final PMR entry,
+			 * so this is not a real error condition...
+			 */
+			eResult = PVRSRV_OK;
+		}
+	}
+
+	_RIUnlock();
+
+	return eResult;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpListKM
+
+ @Description
+            Dumps out the contents of the RI List entry for the
+            specified PMR, and all MEMDESC allocation entries
+            in the associated sub linked list.
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @input     psPMR - PMR for which RI entry details are to be output
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpListKM(PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpList(psPMR, 0);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIGetListEntryKM
+
+ @Description
+            Returns pointer to a formatted string with details of the specified
+            list entry. If no entry exists (e.g. it may have been deleted
+            since the previous call), NULL is returned.
+
+ @input     pid - pid for which RI entry details are to be output
+ @input     ppHandle - handle to the entry, if NULL, the first entry will be
+                     returned.
+ @output    pszEntryString - string to be output for the entry
+ @output    hEntry - hEntry will be returned pointing to the next entry
+                     (or NULL if there is no next entry)
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+IMG_BOOL RIGetListEntryKM(IMG_PID pid,
+						  IMG_HANDLE **ppHandle,
+						  IMG_CHAR **ppszEntryString)
+{
+	RI_SUBLIST_ENTRY  *psRISubEntry = NULL;
+	RI_LIST_ENTRY  *psRIEntry = NULL;
+	uintptr_t     hashData = 0;
+	IMG_PID       hashKey  = pid;
+
+	static IMG_CHAR acStringBuffer[RI_FRMT_SIZE_MAX];
+
+	static IMG_UINT64 ui64TotalMemdescAlloc;
+	static IMG_UINT64 ui64TotalImport;
+	static IMG_UINT64 ui64TotalPMRAlloc;
+	static IMG_UINT64 ui64TotalPMRBacked;
+	static enum {
+		RI_GET_STATE_MEMDESCS_LIST_START,
+		RI_GET_STATE_MEMDESCS_SUMMARY,
+		RI_GET_STATE_PMR_LIST,
+		RI_GET_STATE_PMR_SUMMARY,
+		RI_GET_STATE_END,
+		RI_GET_STATE_LAST
+	} g_bNextGetState = RI_GET_STATE_MEMDESCS_LIST_START;
+
+	static DLLIST_NODE *psNode;
+	static DLLIST_NODE *psSysAllocNode;
+	static IMG_CHAR szProcName[RI_PROC_BUF_SIZE];
+	static IMG_UINT32 ui32ProcessedSysAllocPMRCount;
+
+	acStringBuffer[0] = '\0';
+
+	switch (g_bNextGetState)
+	{
+	case RI_GET_STATE_MEMDESCS_LIST_START:
+		/* look-up pid in Hash Table, to obtain first entry for pid */
+		hashData = HASH_Retrieve_Extended(g_PID2RISublistHashTable, (void *)&hashKey);
+		if (hashData)
+		{
+			if (*ppHandle)
+			{
+				psRISubEntry = (RI_SUBLIST_ENTRY *)*ppHandle;
+				if (!IS_VALID(psRISubEntry))
+				{
+					psRISubEntry = NULL;
+				}
+			}
+			else
+			{
+				psRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+				if (!IS_VALID(psRISubEntry))
+				{
+					psRISubEntry = NULL;
+				}
+			}
+		}
+
+		if (psRISubEntry)
+		{
+			PDLLIST_NODE psNextProcListNode = dllist_get_next_node(&psRISubEntry->sProcListNode);
+
+			if (IS_IMPORT(psRISubEntry))
+			{
+				ui64TotalImport += psRISubEntry->ui64Size;
+			}
+			else
+			{
+				ui64TotalMemdescAlloc += psRISubEntry->ui64Size;
+			}
+
+			_GenerateMEMDESCEntryString(psRISubEntry,
+										IMG_TRUE,
+										RI_MEMDESC_ENTRY_BUF_SIZE,
+										acStringBuffer);
+
+			if (szProcName[0] == '\0')
+			{
+				OSStringSafeCopy(szProcName, (pid == PVR_SYS_ALLOC_PID) ?
+						PVRSRV_MODNAME : GET_PROC(psRISubEntry), RI_PROC_BUF_SIZE);
+			}
+
+
+			*ppszEntryString = acStringBuffer;
+			*ppHandle        = (IMG_HANDLE)IMG_CONTAINER_OF(psNextProcListNode, RI_SUBLIST_ENTRY, sProcListNode);
+
+			if (psNextProcListNode == NULL ||
+				psNextProcListNode == (PDLLIST_NODE)hashData)
+			{
+				g_bNextGetState = RI_GET_STATE_MEMDESCS_SUMMARY;
+			}
+			/* else continue to list MEMDESCs */
+		}
+		else
+		{
+			if (ui64TotalMemdescAlloc == 0)
+			{
+				acStringBuffer[0] = '\0';
+				*ppszEntryString = acStringBuffer;
+				g_bNextGetState = RI_GET_STATE_MEMDESCS_SUMMARY;
+			}
+			/* else continue to list MEMDESCs */
+		}
+		break;
+
+	case RI_GET_STATE_MEMDESCS_SUMMARY:
+		OSSNPrintf(acStringBuffer,
+		           RI_MEMDESC_SUM_BUF_SIZE,
+		           RI_MEMDESC_SUM_FRMT,
+		           pid,
+		           szProcName,
+		           ui64TotalMemdescAlloc,
+		           ui64TotalMemdescAlloc >> 10,
+		           ui64TotalImport,
+		           ui64TotalImport >> 10,
+		           (ui64TotalMemdescAlloc + ui64TotalImport),
+		           (ui64TotalMemdescAlloc + ui64TotalImport) >> 10);
+
+		*ppszEntryString = acStringBuffer;
+		ui64TotalMemdescAlloc = 0;
+		ui64TotalImport = 0;
+		szProcName[0] = '\0';
+
+		g_bNextGetState = RI_GET_STATE_PMR_LIST;
+		break;
+
+	case RI_GET_STATE_PMR_LIST:
+		if (pid == PVR_SYS_ALLOC_PID)
+		{
+			OSLockAcquire(g_hSysAllocPidListLock);
+			acStringBuffer[0] = '\0';
+			if (!psSysAllocNode)
+			{
+				psSysAllocNode = &g_sSysAllocPidListHead;
+				ui32ProcessedSysAllocPMRCount = 0;
+			}
+			psSysAllocNode = dllist_get_next_node(psSysAllocNode);
+
+			if (szProcName[0] == '\0')
+			{
+				OSStringSafeCopy(szProcName, PVRSRV_MODNAME, RI_PROC_BUF_SIZE);
+			}
+			if (psSysAllocNode != NULL && psSysAllocNode != &g_sSysAllocPidListHead)
+			{
+				psRIEntry = IMG_CONTAINER_OF((PDLLIST_NODE)psSysAllocNode, RI_LIST_ENTRY, sListNode);
+				_GeneratePMREntryString(psRIEntry,
+										IMG_TRUE,
+										RI_PMR_ENTRY_BUF_SIZE,
+										acStringBuffer);
+				ui64TotalPMRAlloc += GET_LOGICAL_SIZE(psRIEntry);
+				ui64TotalPMRBacked += GET_PHYSICAL_SIZE(psRIEntry);
+
+				ui32ProcessedSysAllocPMRCount++;
+				if (ui32ProcessedSysAllocPMRCount > g_ui32SysAllocPMRCount+1)
+				{
+					g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+				}
+				/* else continue to list PMRs */
+			}
+			else
+			{
+				g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+			}
+			*ppszEntryString = (IMG_CHAR *)acStringBuffer;
+			OSLockRelease(g_hSysAllocPidListLock);
+		}
+		else
+		{
+			IMG_BOOL bPMRToDisplay = IMG_FALSE;
+
+			/* Iterate through the 'touched' PMRs and display details */
+			if (!psNode)
+			{
+				psNode = dllist_get_next_node(&g_sClientsListHead);
+			}
+			else
+			{
+				psNode = dllist_get_next_node(psNode);
+			}
+
+			while ((psNode != NULL && psNode != &g_sClientsListHead) &&
+					!bPMRToDisplay)
+			{
+				psRIEntry =	IMG_CONTAINER_OF(psNode, RI_LIST_ENTRY, sListNode);
+				if (psRIEntry->pid == pid)
+				{
+					/* This PMR was 'touched', so display details and unflag it*/
+					_GeneratePMREntryString(psRIEntry,
+											IMG_TRUE,
+											RI_PMR_ENTRY_BUF_SIZE,
+											acStringBuffer);
+					ui64TotalPMRAlloc += GET_LOGICAL_SIZE(psRIEntry);
+					ui64TotalPMRBacked += GET_PHYSICAL_SIZE(psRIEntry);
+
+					/* Remember the name of the process for 1 PMR for the summary */
+					if (szProcName[0] == '\0')
+					{
+						psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)), RI_SUBLIST_ENTRY, sListNode);
+						OSStringSafeCopy(szProcName, GET_PROC(psRISubEntry), RI_PROC_BUF_SIZE);
+					}
+					bPMRToDisplay = IMG_TRUE;
+				}
+				else
+				{
+					psNode = dllist_get_next_node(psNode);
+				}
+			}
+
+			if (psNode == NULL || (psNode == &g_sClientsListHead))
+			{
+				g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+			}
+			/* else continue listing PMRs */
+		}
+		break;
+
+	case RI_GET_STATE_PMR_SUMMARY:
+		OSSNPrintf(acStringBuffer,
+		           RI_PMR_SUM_BUF_SIZE,
+		           RI_PMR_SUM_FRMT,
+		           pid,
+		           szProcName,
+		           ui64TotalPMRAlloc,
+		           ui64TotalPMRAlloc >> 10,
+		           ui64TotalPMRBacked,
+		           ui64TotalPMRBacked >> 10);
+
+		*ppszEntryString = acStringBuffer;
+		ui64TotalPMRAlloc = 0;
+		ui64TotalPMRBacked = 0;
+		szProcName[0] = '\0';
+		psSysAllocNode = NULL;
+
+		g_bNextGetState = RI_GET_STATE_END;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "%s: Bad %d)",__func__, g_bNextGetState));
+
+		__fallthrough;
+	case RI_GET_STATE_END:
+		/* Reset state ready for the next gpu_mem_area file to display */
+		*ppszEntryString = NULL;
+		*ppHandle        = NULL;
+		psNode = NULL;
+		szProcName[0] = '\0';
+
+		g_bNextGetState = RI_GET_STATE_MEMDESCS_LIST_START;
+		return IMG_FALSE;
+		break;
+	}
+
+	return IMG_TRUE;
+}
+
+/* Function used to produce string containing info for MEMDESC RI entries (used for both debugfs and kernel log output) */
+static void _GenerateMEMDESCEntryString(RI_SUBLIST_ENTRY *psRISubEntry,
+                                        IMG_BOOL bDebugFs,
+                                        IMG_UINT16 ui16MaxStrLen,
+                                        IMG_CHAR *pszEntryString)
+{
+	IMG_CHAR szProc[RI_MEMDESC_ENTRY_PROC_BUF_SIZE];
+	IMG_CHAR szImport[RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE];
+	IMG_CHAR szEntryFormat[RI_MEMDESC_ENTRY_FRMT_SIZE];
+	const IMG_CHAR *pszAnnotationText;
+	IMG_PID uiRIPid = 0;
+	PMR* psRIPMR = NULL;
+	IMG_BOOL bSysPMR = IMG_FALSE;
+	IMG_BOOL bHostDevice = IS_HOST_DEVICE(psRISubEntry);
+	IMG_CHAR szDeviceID[RI_DEV_ID_BUF_SIZE];
+
+	if (psRISubEntry->psRI != NULL)
+	{
+		uiRIPid = psRISubEntry->psRI->pid;
+		psRIPMR = (PMR*)GET_ADDR(psRISubEntry->psRI);
+		bSysPMR = IS_SYSPMR(psRISubEntry->psRI);
+	}
+
+	OSSNPrintf(szEntryFormat,
+			RI_MEMDESC_ENTRY_FRMT_SIZE,
+			RI_MEMDESC_ENTRY_FRMT,
+			DEVMEM_ANNOTATION_MAX_LEN);
+
+	if (!bDebugFs)
+	{
+		/* we don't include process ID info for debugfs output */
+		OSSNPrintf(szProc,
+				RI_MEMDESC_ENTRY_PROC_BUF_SIZE,
+				RI_MEMDESC_ENTRY_PROC_FRMT,
+				psRISubEntry->pid,
+				GET_PROC(psRISubEntry));
+	}
+
+	if (!bHostDevice)
+	{
+		OSSNPrintf(szDeviceID,
+				   sizeof(szDeviceID),
+				   "%-3d",
+				   GET_DEVICE_ID(psRISubEntry));
+	}
+
+	if (IS_IMPORT(psRISubEntry) && psRIPMR)
+	{
+		OSSNPrintf((IMG_CHAR *)&szImport,
+		           RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE,
+		           RI_MEMDESC_ENTRY_IMPORT_FRMT,
+		           uiRIPid);
+		/* Set pszAnnotationText to that of the 'parent' PMR RI entry */
+		pszAnnotationText = GET_NAME(psRISubEntry->psRI);
+	}
+	else if (!IS_SUBALLOC(psRISubEntry) && psRIPMR)
+	{
+		/* Set pszAnnotationText to that of the 'parent' PMR RI entry */
+		pszAnnotationText = GET_NAME(psRISubEntry->psRI);
+	}
+	else
+	{
+		/* Set pszAnnotationText to that of the MEMDESC RI entry */
+		pszAnnotationText = psRISubEntry->pszTextB;
+	}
+
+	/* Don't print memdescs if they are local imports
+	 * (i.e. imported PMRs allocated by this process)
+	 */
+	if (bDebugFs &&
+		((psRISubEntry->sVAddr.uiAddr + psRISubEntry->ui64Offset) == 0) &&
+		(IS_IMPORT(psRISubEntry) && ((psRISubEntry->pid == uiRIPid)
+									 || (psRISubEntry->pid == PVR_SYS_ALLOC_PID))))
+	{
+		/* Don't print this entry */
+		pszEntryString[0] = '\0';
+	}
+	else
+	{
+		OSSNPrintf(pszEntryString,
+				   ui16MaxStrLen,
+				   szEntryFormat,
+				   (bDebugFs ? "" : "   "),
+				   psRISubEntry->pid,
+				   (bHostDevice ? "-  " : szDeviceID),
+				   (psRISubEntry->sVAddr.uiAddr + psRISubEntry->ui64Offset),
+				   pszAnnotationText,
+				   (bDebugFs ? "" : (char *)szProc),
+				   psRISubEntry->ui64Size,
+				   psRIPMR,
+				   (IS_IMPORT(psRISubEntry) ? (char *)&szImport : ""),
+				   (!IS_IMPORT(psRISubEntry) &&
+				    (bSysPMR) &&
+				    (psRISubEntry->pid != PVR_SYS_ALLOC_PID)) ? g_szSysAllocImport : "",
+				   (IS_RACC(psRISubEntry) ? RI_FREED_BY_DRIVER : ""),
+				   (bDebugFs ? '\n' : ' '));
+	}
+}
+
+/* Function used to produce string containing info for PMR RI entries (used for debugfs and kernel log output) */
+static void _GeneratePMREntryString(RI_LIST_ENTRY *psRIEntry,
+                                    IMG_BOOL bDebugFs,
+                                    IMG_UINT16 ui16MaxStrLen,
+                                    IMG_CHAR *pszEntryString)
+{
+	const IMG_CHAR*   pszAnnotationText;
+	const IMG_CHAR*   pszHeapText;
+
+	IMG_CHAR          szEntryFormat[RI_PMR_ENTRY_FRMT_SIZE];
+	IMG_BOOL          bHostDevice = IS_HOST_DEVICE(psRIEntry);
+	IMG_CHAR          szDeviceID[RI_DEV_ID_BUF_SIZE];
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	IMG_CHAR          szXDevices[RI_PMR_ENTRY_IMPORTED_BUF_SIZE];
+#else
+	IMG_CHAR          szXDevices[] = "";
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+	OSSNPrintf(szEntryFormat,
+			RI_PMR_ENTRY_FRMT_SIZE,
+			RI_PMR_ENTRY_FRMT,
+			(DEVMEM_ANNOTATION_MAX_LEN/2),
+			PHYS_HEAP_NAME_SIZE);
+
+	/* Set pszAnnotationText to that PMR RI entry */
+	pszAnnotationText = GET_NAME(psRIEntry);
+
+	/* Acquire PhysHeap Name to that PMR RI entry */
+	pszHeapText = PhysHeapName(GET_HEAP(psRIEntry));
+
+	if (!bHostDevice)
+	{
+		OSSNPrintf(szDeviceID,
+				   sizeof(szDeviceID),
+				   "%-3d",
+				   GET_DEVICE_ID(psRIEntry));
+	}
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	if (GET_XDEVICES(psRIEntry) != 0)
+	{
+		IMG_UINT64 uiXDevices = GET_XDEVICES(psRIEntry);
+		IMG_UINT8 i;
+		IMG_CHAR *pszHead = szXDevices;
+
+		pszHead += OSSNPrintf(pszHead,
+		                        sizeof(szXDevices) - (pszHead - szXDevices),
+		                        RI_PMR_ENTRY_IMPORTED_FRMT_BEGIN);
+		for (i = 0; i < PVRSRV_MAX_DEVICES; i++)
+		{
+			if (uiXDevices & (IMG_UINT64_C(1) << i))
+			{
+				pszHead += OSSNPrintf(pszHead,
+				                        sizeof(szXDevices) - (pszHead - szXDevices),
+				                        RI_PMR_ENTRY_IMPORTED_DEV_FRMT,
+				                        i);
+			}
+		}
+		pszHead += OSSNPrintf(pszHead,
+		                        sizeof(szXDevices) - (pszHead - szXDevices),
+		                        RI_PMR_ENTRY_IMPORTED_FRMT_END);
+	} else {
+		szXDevices[0] = '\0';
+	}
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+	OSSNPrintf(pszEntryString,
+	           ui16MaxStrLen,
+	           szEntryFormat,
+	           (bDebugFs ? "" : "   "),
+	           psRIEntry->pid,
+	           (bHostDevice ? "-  " : szDeviceID),
+	           GET_ADDR(psRIEntry),
+	           pszAnnotationText,
+	           pszHeapText,
+	           GET_LOGICAL_SIZE(psRIEntry),
+	           GET_PHYSICAL_SIZE(psRIEntry),
+	           (IS_RACC(psRIEntry) ? RI_FREED_BY_DRIVER : ""),
+			   szXDevices,
+	           (bDebugFs ? '\n' : ' '));
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_DumpList
+
+ @Description
+            Dumps out RI List entries according to parameters passed.
+
+ @input     psPMR - If not NULL, function will output the RI entries for
+                   the specified PMR only
+ @input     pid - If non-zero, the function will only output MEMDESC RI
+                  entries made by the process with ID pid.
+                  If zero, all MEMDESC RI entries will be output.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR _DumpList(PMR *psPMR, IMG_PID pid)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_UINT16 ui16SubEntriesParsed = 0;
+	uintptr_t hashData = 0;
+	IMG_PID hashKey;
+	PMR *pPMRHashKey = psPMR;
+	IMG_BOOL bDisplayedThisPMR = IMG_FALSE;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		if (pid != 0)
+		{
+			/* look-up pid in Hash Table */
+			hashKey = pid;
+			hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&hashKey);
+			if (hashData)
+			{
+				psRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+				if (psRISubEntry)
+				{
+					psRIEntry = psRISubEntry->psRI;
+				}
+			}
+		}
+		else
+		{
+			/* Look-up psPMR in Hash Table */
+			hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+			psRIEntry = (RI_LIST_ENTRY *)hashData;
+		}
+		if (!psRIEntry)
+		{
+			/* No entry found in hash table */
+			return PVRSRV_ERROR_NOT_FOUND;
+		}
+		while (psRIEntry)
+		{
+			bDisplayedThisPMR = IMG_FALSE;
+			/* Output details for RI entry */
+			if (!pid)
+			{
+				_RIOutput (("%s <%p> suballocs:%d size:0x%010" IMG_UINT64_FMTSPECx,
+				            GET_NAME(psRIEntry),
+				            GET_ADDR(psRIEntry),
+				            (IMG_UINT)psRIEntry->ui16SubListCount,
+				            GET_LOGICAL_SIZE(psRIEntry)));
+				bDisplayedThisPMR = IMG_TRUE;
+			}
+			ui16SubEntriesParsed = 0;
+			if (psRIEntry->ui16SubListCount)
+			{
+#if _DUMP_LINKEDLIST_INFO
+				_RIOutput (("RI LIST: {sSubListFirst.psNextNode:0x%p}\n",
+				            psRIEntry->sSubListFirst.psNextNode));
+#endif /* _DUMP_LINKEDLIST_INFO */
+				if (!pid)
+				{
+					psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)),
+					                                RI_SUBLIST_ENTRY, sListNode);
+				}
+				/* Traverse RI sublist and output details for each entry */
+				while (psRISubEntry)
+				{
+					if (psRIEntry)
+					{
+						if ((ui16SubEntriesParsed >= psRIEntry->ui16SubListCount))
+						{
+							break;
+						}
+						if (!bDisplayedThisPMR)
+						{
+							_RIOutput (("%s <%p> suballocs:%d size:0x%010" IMG_UINT64_FMTSPECx,
+								    GET_NAME(psRIEntry),
+								    GET_ADDR(psRIEntry),
+								    (IMG_UINT)psRIEntry->ui16SubListCount,
+								    GET_LOGICAL_SIZE(psRIEntry)));
+							bDisplayedThisPMR = IMG_TRUE;
+						}
+					}
+#if _DUMP_LINKEDLIST_INFO
+					_RIOutput (("RI LIST:    [this subentry:0x%p]\n",psRISubEntry));
+					_RIOutput (("RI LIST:     psRI:0x%p\n",psRISubEntry->psRI));
+#endif /* _DUMP_LINKEDLIST_INFO */
+
+					{
+						IMG_CHAR szEntryString[RI_MEMDESC_ENTRY_BUF_SIZE];
+
+						_GenerateMEMDESCEntryString(psRISubEntry,
+						                            IMG_FALSE,
+						                            RI_MEMDESC_ENTRY_BUF_SIZE,
+						                            szEntryString);
+						_RIOutput (("%s",szEntryString));
+					}
+
+					if (pid)
+					{
+						if ((dllist_get_next_node(&(psRISubEntry->sProcListNode)) == NULL) ||
+							(dllist_get_next_node(&(psRISubEntry->sProcListNode)) == (PDLLIST_NODE)hashData))
+						{
+							psRISubEntry = NULL;
+						}
+						else
+						{
+							psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sProcListNode)),
+							                                RI_SUBLIST_ENTRY, sProcListNode);
+							if (psRISubEntry)
+							{
+								if (psRIEntry != psRISubEntry->psRI)
+								{
+									/*
+									 * The next MEMDESC in the process linked list is in a different PMR
+									 */
+									psRIEntry = psRISubEntry->psRI;
+									bDisplayedThisPMR = IMG_FALSE;
+								}
+							}
+						}
+					}
+					else
+					{
+						ui16SubEntriesParsed++;
+						psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sListNode)),
+						                                RI_SUBLIST_ENTRY, sListNode);
+					}
+				}
+			}
+			if (!pid && psRIEntry)
+			{
+				if (ui16SubEntriesParsed != psRIEntry->ui16SubListCount)
+				{
+					/*
+					 * Output error message as sublist does not contain the
+					 * number of entries indicated by sublist count
+					 */
+					_RIOutput (("RI ERROR: RI sublist contains %d entries, not %d entries\n",
+					            ui16SubEntriesParsed, psRIEntry->ui16SubListCount));
+				}
+				else if (psRIEntry->ui16SubListCount && !dllist_get_next_node(&(psRIEntry->sSubListFirst)))
+				{
+					/*
+					 * Output error message as sublist is empty but sublist count
+					 * is not zero
+					 */
+					_RIOutput (("RI ERROR: ui16SubListCount=%d for empty RI sublist\n",
+					            psRIEntry->ui16SubListCount));
+				}
+			}
+			psRIEntry = NULL;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpAllKM
+
+ @Description
+            Dumps out the contents of all RI List entries (i.e. for all
+            MEMDESC allocations for each PMR).
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpAllKM(void)
+{
+	if (g_pPMR2RIListHashTable)
+	{
+		return HASH_Iterate(g_pPMR2RIListHashTable, (HASH_pfnCallback)_DumpAllEntries, NULL);
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpProcessKM
+
+ @Description
+            Dumps out the contents of all MEMDESC RI List entries (for every
+            PMR) which have been allocate by the specified process only.
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpProcessKM(IMG_PID pid)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 dummyPMR;
+
+	if (!g_PID2RISublistHashTable)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpList((PMR *)&dummyPMR, pid);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_TotalAllocsForProcess
+
+ @Description
+            Totals all PMR physical backing for given process.
+
+ @input     pid - ID of process.
+
+ @input     ePhysHeapType - type of Physical Heap for which to total allocs
+
+ @Return	Size of all physical backing for PID's PMRs allocated from the
+            specified heap type (in bytes).
+
+******************************************************************************/
+static IMG_INT32 _TotalAllocsForProcess(const IMG_PID pid, const PHYS_HEAP_TYPE ePhysHeapType)
+{
+	IMG_INT32 i32TotalPhysical = 0;
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		if (pid == PVR_SYS_ALLOC_PID)
+		{
+			DLLIST_NODE *psSysAllocNode = NULL;
+
+			OSLockAcquire(g_hSysAllocPidListLock);
+
+			for (psSysAllocNode = dllist_get_next_node(&g_sSysAllocPidListHead);
+			     psSysAllocNode != NULL && psSysAllocNode != &g_sSysAllocPidListHead;
+			     psSysAllocNode = dllist_get_next_node(psSysAllocNode))
+			{
+				const RI_LIST_ENTRY *const psRIEntry =
+					IMG_CONTAINER_OF((PDLLIST_NODE)psSysAllocNode, RI_LIST_ENTRY, sListNode);
+
+				/* Exclude RACC entries from the stats. This memory should have
+				 * been already freed during connection destruction so they should
+				 * not affect figure showing memory usage. Also RACC entries existence
+				 * is to show unfreed memory in `gpu_mem_area`, not to affect other
+				 * process stats. Finally if the RI entry exists only due to sub-entries
+				 * referencing it but the underlying PMR has been freed don't include
+				 * it in the total stats. */
+				if (PhysHeapGetType(GET_HEAP(psRIEntry)) == ePhysHeapType &&
+				    !IS_RACC(psRIEntry) && !HAS_PMR_INFO(psRIEntry))
+				{
+					IMG_UINT64 ui64PhysicalSize = GET_PHYSICAL_SIZE(psRIEntry);
+
+					if (((IMG_UINT64)i32TotalPhysical + ui64PhysicalSize > IMG_INT32_MAX))
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: i32TotalPhysical exceeding size for i32",
+						         __func__));
+					}
+					i32TotalPhysical += (IMG_INT32)(ui64PhysicalSize & 0x00000000ffffffff);
+				}
+			}
+
+			OSLockRelease(g_hSysAllocPidListLock);
+		}
+		else
+		{
+			RI_LIST_ENTRY *psRIEntry = NULL;
+			RI_SUBLIST_ENTRY *psInitialRISubEntry = NULL, *psRISubEntry = NULL;
+			uintptr_t hashData = 0;
+
+			if (pid != 0)
+			{
+				/* look-up pid in Hash Table */
+				IMG_PID hashKey = pid;
+				hashData = HASH_Retrieve_Extended(g_PID2RISublistHashTable, (void *)&hashKey);
+				if (hashData)
+				{
+					psInitialRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psInitialRISubEntry != NULL)
+					{
+						psRISubEntry = psInitialRISubEntry;
+						psRIEntry = psInitialRISubEntry->psRI;
+					}
+				}
+			}
+
+			while (psRISubEntry != NULL && psRIEntry != NULL)
+			{
+				DLLIST_NODE *psNextNode;
+
+				if (!IS_IMPORT(psRISubEntry) &&
+				    !IS_RACC(psRISubEntry) &&
+					!IS_COUNTED_BY_DEBUGFS(psRIEntry) &&
+				    !IS_SYSPMR(psRIEntry) &&
+				    (PhysHeapGetType(GET_HEAP(psRIEntry)) == ePhysHeapType))
+				{
+					IMG_UINT64 ui64PhysicalSize = GET_PHYSICAL_SIZE(psRIEntry);
+
+					if (((IMG_UINT64)i32TotalPhysical + ui64PhysicalSize > IMG_INT32_MAX))
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: i32TotalPhysical exceeding size for i32",
+						         __func__));
+					}
+
+					i32TotalPhysical += (IMG_INT32)(ui64PhysicalSize & 0x00000000ffffffff);
+					BIT_SET(psRIEntry->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG);
+				}
+
+				psNextNode = dllist_get_next_node(&(psRISubEntry->sProcListNode));
+				if (psNextNode == NULL || psNextNode == (PDLLIST_NODE)hashData)
+				{
+					psRISubEntry = NULL;
+					psRIEntry = NULL;
+				}
+				else
+				{
+					psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psRISubEntry)
+					{
+						psRIEntry = psRISubEntry->psRI;
+					}
+				}
+			}
+
+			psRISubEntry = psInitialRISubEntry;
+			if (psRISubEntry)
+			{
+				psRIEntry = psRISubEntry->psRI;
+			}
+
+			while (psRISubEntry != NULL && psRIEntry != NULL)
+			{
+				const DLLIST_NODE *const psNextNode =
+				    dllist_get_next_node(&(psRISubEntry->sProcListNode));
+
+				BIT_UNSET(psRIEntry->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG);
+
+				if (psNextNode == NULL || psNextNode == (PDLLIST_NODE)hashData)
+				{
+					psRISubEntry = NULL;
+					psRIEntry = NULL;
+				}
+				else
+				{
+					psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psRISubEntry)
+					{
+						psRIEntry = psRISubEntry->psRI;
+					}
+				}
+			}
+		}
+	}
+
+	return i32TotalPhysical;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RITotalAllocProcessKM
+
+ @Description
+            Returns the total of allocated GPU memory (backing for PMRs)
+            which has been allocated from the specific heap by the specified
+            process only.
+
+ @Return	Amount of physical backing allocated (in bytes)
+
+******************************************************************************/
+IMG_INT32 RITotalAllocProcessUnlocked(IMG_PID pid, PHYS_HEAP_TYPE ePhysHeapType)
+{
+	IMG_INT32 i32BackingTotal = 0;
+
+	if (g_PID2RISublistHashTable)
+	{
+		i32BackingTotal = _TotalAllocsForProcess(pid, ePhysHeapType);
+	}
+	return i32BackingTotal;
+}
+
+#if defined(DEBUG)
+/*!
+*******************************************************************************
+
+ @Function	_DumpProcessList
+
+ @Description
+            Dumps out RI List entries according to parameters passed.
+
+ @input     psPMR - If not NULL, function will output the RI entries for
+                   the specified PMR only
+ @input     pid - If non-zero, the function will only output MEMDESC RI
+                  entries made by the process with ID pid.
+                  If zero, all MEMDESC RI entries will be output.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR _DumpProcessList(PMR *psPMR,
+									 IMG_PID pid,
+									 IMG_UINT64 ui64Offset,
+									 IMG_DEV_VIRTADDR *psDevVAddr)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_UINT16 ui16SubEntriesParsed = 0;
+	uintptr_t hashData = 0;
+	PMR *pPMRHashKey = psPMR;
+
+	psDevVAddr->uiAddr = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		PVR_ASSERT(psPMR && pid);
+
+		/* Look-up psPMR in Hash Table */
+		hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+		psRIEntry = (RI_LIST_ENTRY *)hashData;
+
+		if (!psRIEntry)
+		{
+			/* No entry found in hash table */
+			return PVRSRV_ERROR_NOT_FOUND;
+		}
+
+		if (psRIEntry->ui16SubListCount)
+		{
+			psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)),
+											RI_SUBLIST_ENTRY, sListNode);
+
+			/* Traverse RI sublist and output details for each entry */
+			while (psRISubEntry && (ui16SubEntriesParsed < psRIEntry->ui16SubListCount))
+			{
+				if (pid == psRISubEntry->pid)
+				{
+					IMG_UINT64 ui64StartOffset = psRISubEntry->ui64Offset;
+					IMG_UINT64 ui64EndOffset = psRISubEntry->ui64Offset + psRISubEntry->ui64Size;
+
+					if (ui64Offset >= ui64StartOffset && ui64Offset < ui64EndOffset)
+					{
+						psDevVAddr->uiAddr = psRISubEntry->sVAddr.uiAddr;
+						return PVRSRV_OK;
+					}
+				}
+
+				ui16SubEntriesParsed++;
+				psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sListNode)),
+												RI_SUBLIST_ENTRY, sListNode);
+			}
+		}
+	}
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpProcessListKM
+
+ @Description
+            Dumps out selected contents of all MEMDESC RI List entries (for a
+            PMR) which have been allocate by the specified process only.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpProcessListKM(PMR *psPMR,
+								 IMG_PID pid,
+								 IMG_UINT64 ui64Offset,
+								 IMG_DEV_VIRTADDR *psDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	if (!g_PID2RISublistHashTable)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpProcessList(psPMR,
+							  pid,
+							  ui64Offset,
+							  psDevVAddr);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+#endif
+
+static PVRSRV_ERROR _MarkRACCEntries(uintptr_t k, uintptr_t v, void *psConnection)
+{
+	/* This is not a head node. See sProcListNode. */
+	DLLIST_NODE *psListStart = (DLLIST_NODE *) v;
+	DLLIST_NODE *psNextNode = psListStart;
+
+	PVR_UNREFERENCED_PARAMETER(k);
+
+	do
+	{
+		RI_SUBLIST_ENTRY *psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY,
+		                                                  sProcListNode);
+
+		if (psRISubEntry->psConnection == psConnection)
+		{
+			RI_LIST_ENTRY *psRIEntry = psRISubEntry->psRI;
+
+			BIT_SET(psRISubEntry->ui16Flags, RI_RACC_FLAG);
+
+			/* RI sub-entry may not have an RI entry associated with it. If that's the
+			 * case just skip processing it. */
+			if (psRIEntry != NULL)
+			{
+				if (!IS_RACC(psRIEntry) &&
+				    !HAS_PMR_INFO(psRIEntry) &&
+				    /* Mark as RACC only if the entry doesn't belong to the system process.
+				     * System process allocations are alive for the whole driver lifetime
+					 * hence they will always exist at a connection closed for every process
+					 * that references them. */
+				    psRIEntry->pid != PVR_SYS_ALLOC_PID)
+				{
+					PVRSRV_ERROR eError;
+
+					eError = _RICreateAndSetPmrInfo(psRIEntry);
+					PVR_LOG_RETURN_IF_ERROR(eError, "_RICreateAndSetPmrInfo");
+
+					BIT_SET(psRIEntry->ui16Flags, RI_RACC_FLAG);
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "RIEntry(%s) is already RACC", GET_NAME(psRIEntry)));
+				}
+			}
+		}
+
+		psNextNode = dllist_get_next_node(psNextNode);
+	} while (psNextNode != NULL && psNextNode != psListStart);
+
+	return PVRSRV_OK;
+}
+
+void RIConnectionClosed(void* psConnection)
+{
+	PVRSRV_ERROR eError;
+	if (g_PID2RISublistHashTable)
+	{
+		_RILock();
+		eError = HASH_Iterate(g_PID2RISublistHashTable, (HASH_pfnCallback)_MarkRACCEntries, psConnection);
+		PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "_MarkRACCEntries");
+		_RIUnlock();
+	}
+}
+
+static PVRSRV_ERROR DeleteRACCEntry(RI_SUBLIST_ENTRY *psRISubEntry)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_LIST_ENTRY *psRIEntry = psRISubEntry->psRI;
+
+	PVR_LOG_IF_FALSE_VA(PVR_DBG_WARNING, IS_RACC(psRISubEntry), "Non-RACC entry(%s)",
+	                    psRISubEntry->pszTextB);
+
+	eError = RIDeleteMemdescEntryUnlocked(psRISubEntry);
+	PVR_LOG_IF_ERROR(eError, "RIDeleteMemdescEntryUnlocked");
+
+	/* RI sub-entry may not have an RI entry associated with it. If that's the
+	 * case just skip processing it. */
+	if (psRIEntry != NULL)
+	{
+		if (psRIEntry->ui16SubListCount)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%p: More than 1 sublist present! (%s)",
+			         psRIEntry, __func__));
+		}
+		else
+		{
+			eError = RIDeletePMREntryUnlocked(psRIEntry);
+			PVR_LOG_IF_ERROR(eError, "RIDeletePMREntryUnlocked");
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RIDeleteEntriesForPID(IMG_PID pid)
+{
+	/* This is not a head node. See sProcListNode. */
+	DLLIST_NODE *psListStart;
+	DLLIST_NODE *psNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_PID hashKey  = pid;
+
+	_RILock();
+
+	PVR_LOG_GOTO_IF_FALSE(g_PID2RISublistHashTable != NULL, "g_PID2RISublistHashTable is NULL", ErrHashError);
+
+	psListStart = (DLLIST_NODE *) HASH_Retrieve_Extended(g_PID2RISublistHashTable,
+	                                                    (void *) &hashKey);
+	PVR_GOTO_IF_FALSE(psListStart != NULL, ExitUnlockAndReturn);
+	psNode = psListStart;
+
+	do
+	{
+		DLLIST_NODE *psNext = dllist_get_next_node(psNode);
+		RI_SUBLIST_ENTRY *psRISubEntry = IMG_CONTAINER_OF(psNode,
+		                                                  RI_SUBLIST_ENTRY,
+		                                                  sProcListNode);
+
+		eError = DeleteRACCEntry(psRISubEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DeleteRACCEntry", ExitUnlockAndReturn);
+
+		psNode = psNext;
+	} while (psNode != NULL && psNode != psListStart);
+
+	goto ExitUnlockAndReturn;
+
+ErrHashError:
+    eError = PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE;
+
+ExitUnlockAndReturn:
+	_RIUnlock();
+
+	return eError;
+}
+
+static PVRSRV_ERROR _DumpAllEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *)v;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	return RIDumpListKM(GET_ADDR(psRIEntry));
+}
+
+static PVRSRV_ERROR _DeleteAllEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *)v;
+	RI_SUBLIST_ENTRY *psRISubEntry;
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	while ((eResult == PVRSRV_OK) && (psRIEntry->ui16SubListCount > 0))
+	{
+		psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)), RI_SUBLIST_ENTRY, sListNode);
+		eResult = RIDeleteMEMDESCEntryKM((RI_HANDLE)psRISubEntry);
+	}
+	if (eResult == PVRSRV_OK)
+	{
+		eResult = RIDeletePMREntryKM((RI_HANDLE)psRIEntry);
+		/*
+		 * If we've deleted the Hash table, return
+		 * an error to stop the iterator...
+		 */
+		if (!g_pPMR2RIListHashTable)
+		{
+			eResult = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		}
+	}
+	return eResult;
+}
+
+static PVRSRV_ERROR _DeleteAllProcEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry = (RI_SUBLIST_ENTRY *)v;
+	PVRSRV_ERROR eResult;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	eResult = RIDeleteMEMDESCEntryKM((RI_HANDLE) psRISubEntry);
+	if (eResult == PVRSRV_OK && !g_PID2RISublistHashTable)
+	{
+		/*
+		 * If we've deleted the Hash table, return
+		 * an error to stop the iterator...
+		 */
+		eResult = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	return eResult;
+}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/srvcore.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/srvcore.c
new file mode 100644
index 000000000000..203c89adda15
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/srvcore.c
@@ -0,0 +1,1621 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements core PVRSRV API, server side
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types_check.h"
+#include "pvr_debug.h"
+#include "ra.h"
+#include "pvr_bridge.h"
+#include "connection_server.h"
+#include "device.h"
+#include "htbserver.h"
+
+#include "pdump_km.h"
+
+#include "srvkm.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "log2.h"
+
+#include "srvcore.h"
+#include "pvrsrv.h"
+#include "power.h"
+
+#include "os_apphint.h"
+
+#if defined(SUPPORT_RGX)
+#include "rgxdevice.h"
+#include "rgxinit.h"
+#include "rgx_compat_bvnc.h"
+#endif
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+#include "lock.h"
+#include "osfunc.h"
+#include "device_connection.h"
+#include "process_stats.h"
+#include "pvrsrv_pool.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "physmem_lma.h"
+#include "services_km.h"
+#endif
+
+#include "pvrsrv_tlstreams.h"
+#include "tlstream.h"
+
+#if defined(PVRSRV_MISSING_NO_SPEC_IMPL)
+#pragma message ("There is no implementation of OSConfineArrayIndexNoSpeculation() - see osfunc.h")
+#endif
+
+/* For the purpose of maintainability, it is intended that this file should not
+ * contain any OS specific #ifdefs. Please find a way to add e.g.
+ * an osfunc.c abstraction or override the entire function in question within
+ * env,*,pvr_bridge_k.c
+ */
+
+PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY g_BridgeDispatchTable[BRIDGE_DISPATCH_TABLE_ENTRY_COUNT] = { {.pfFunction = DummyBW,} ,};
+
+#define PVR_DISPATCH_OFFSET_FIRST_FUNC			0
+#define PVR_DISPATCH_OFFSET_LAST_FUNC			1
+#define PVR_DISPATCH_OFFSET_ARRAY_MAX			2
+
+#define PVRSRV_CLIENT_TL_STREAM_SIZE_DEFAULT PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE
+
+static IMG_UINT16 g_BridgeDispatchTableStartOffsets[BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT][PVR_DISPATCH_OFFSET_ARRAY_MAX];
+
+
+#define PVRSRV_MAX_POOLED_BRIDGE_BUFFERS 8	/*!< Initial number of pooled bridge buffers */
+
+static PVRSRV_POOL *g_psBridgeBufferPool;	/*! Pool of bridge buffers */
+
+
+#if defined(DEBUG_BRIDGE_KM)
+/* a lock used for protecting bridge call timing calculations
+ * for calls which do not acquire a lock
+ */
+static POS_LOCK g_hStatsLock;
+PVRSRV_BRIDGE_GLOBAL_STATS g_BridgeGlobalStats;
+
+void BridgeGlobalStatsLock(void)
+{
+	OSLockAcquire(g_hStatsLock);
+}
+
+void BridgeGlobalStatsUnlock(void)
+{
+	OSLockRelease(g_hStatsLock);
+}
+#endif
+
+void BridgeDispatchTableStartOffsetsInit(void)
+{
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEFAULT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEFAULT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SRVCORE][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SRVCORE][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED1][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RESERVED1_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED1][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED2][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RESERVED2_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED2][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RESERVED2_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPCTRL][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPCTRL][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMPLAT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMPLAT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_CMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_CMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMP][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMP][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMABUF][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMABUF][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CACHE][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CACHE][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_CACHE_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PVRTL][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PVRTL][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RI][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RI_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RI][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RI_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_VALIDATION][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_VALIDATION][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_TUTILS][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_TUTILS_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_TUTILS][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEVICEMEMHISTORY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEVICEMEMHISTORY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_HTBUFFER][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_HTBUFFER][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DCPLAT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DCPLAT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMEXTMEM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMEXTMEM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCTRACKING][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCTRACKING][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCFALLBACK][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCFALLBACK][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DI][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DI_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DI][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DI_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMA][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DMA_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMA][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DMA_DISPATCH_LAST;
+#if defined(SUPPORT_RGX)
+	/* Need a gap here to start next entry at element 128 */
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXCMP][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXCMP][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTA3D][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTA3D][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXBREAKPOINT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXBREAKPOINT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXFWDBG][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXFWDBG][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXPDUMP][PVR_DISPATCH_OFFSET_FIRST_FUNC]= PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXPDUMP][PVR_DISPATCH_OFFSET_LAST_FUNC]= PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXHWPERF][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXHWPERF][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXREGCONFIG][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXREGCONFIG][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXKICKSYNC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXKICKSYNC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ2][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ2][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTIMERQUERY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTIMERQUERY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXRAY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXRAY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST;
+#endif
+}
+
+#if defined(DEBUG_BRIDGE_KM)
+
+#if defined(INTEGRITY_OS)
+void PVRSRVPrintBridgeStats(void)
+{
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Remainder;
+
+	BridgeGlobalStatsLock();
+
+	printf("Total Bridge call count = %u\n"
+		   "Total number of bytes copied via copy_from_user = %u\n"
+		   "Total number of bytes copied via copy_to_user = %u\n"
+		   "Total number of bytes copied via copy_*_user = %u\n\n"
+		   "%3s: %-60s | %-48s | %10s | %20s | %20s | %20s | %20s\n",
+		   g_BridgeGlobalStats.ui32IOCTLCount,
+		   g_BridgeGlobalStats.ui32TotalCopyFromUserBytes,
+		   g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		   g_BridgeGlobalStats.ui32TotalCopyFromUserBytes + g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		   "#",
+		   "Bridge Name",
+		   "Wrapper Function",
+		   "Call Count",
+		   "copy_from_user (B)",
+		   "copy_to_user (B)",
+		   "Total Time (us)",
+		   "Max Time (us)");
+
+	/* Is the item asked for (starts at 0) a valid table index? */
+	for ( ui32Index=0; ui32Index < BRIDGE_DISPATCH_TABLE_ENTRY_COUNT; ui32Index++ )
+	{
+		PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psEntry = &g_BridgeDispatchTable[ui32Index];
+		printf("%3d: %-60s   %-48s   %-10u   %-20u   %-20u   %-20llu   %-20llu\n",
+			   (IMG_UINT32)(((size_t)psEntry-(size_t)g_BridgeDispatchTable)/sizeof(*g_BridgeDispatchTable)),
+			   psEntry->pszIOCName,
+			   (psEntry->pfFunction != NULL) ? psEntry->pszFunctionName : "(null)",
+			   psEntry->ui32CallCount,
+			   psEntry->ui32CopyFromUserTotalBytes,
+			   psEntry->ui32CopyToUserTotalBytes,
+			   (unsigned long long) OSDivide64r64(psEntry->ui64TotalTimeNS, 1000, &ui32Remainder),
+			   (unsigned long long) OSDivide64r64(psEntry->ui64MaxTimeNS, 1000, &ui32Remainder));
+
+
+	}
+
+	BridgeGlobalStatsUnlock();
+}
+#endif
+
+PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+					IMG_UINT32 ui32DispatchTableEntry,
+					void *pvDest,
+					void __user *pvSrc,
+					IMG_UINT32 ui32Size)
+{
+	BridgeGlobalStatsLock();
+	g_BridgeDispatchTable[ui32DispatchTableEntry].ui32CopyFromUserTotalBytes+=ui32Size;
+	g_BridgeGlobalStats.ui32TotalCopyFromUserBytes+=ui32Size;
+	BridgeGlobalStatsUnlock();
+
+	return OSBridgeCopyFromUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+				  IMG_UINT32 ui32DispatchTableEntry,
+				  void __user *pvDest,
+				  void *pvSrc,
+				  IMG_UINT32 ui32Size)
+{
+	BridgeGlobalStatsLock();
+	g_BridgeDispatchTable[ui32DispatchTableEntry].ui32CopyToUserTotalBytes+=ui32Size;
+	g_BridgeGlobalStats.ui32TotalCopyToUserBytes+=ui32Size;
+	BridgeGlobalStatsUnlock();
+
+	return OSBridgeCopyToUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+#endif
+
+/**************************************************************************/ /*!
+@Function       DeviceDefaultPhysHeapFreeMemCheck
+
+@Description    Check if the required amount of free space is available in the
+                Default PhysHeap for a connection to be made.
+
+@Input          psDeviceNode    The device the connection is being
+                                made on.
+@Input          ui32MinMemInMBs The minimum memory required to be
+                                available in the Default PhysHeap.
+
+@Return         PVRSRV_OK if successful else a PVRSRV_ERROR.
+*/ /***************************************************************************/
+static PVRSRV_ERROR DeviceDefaultPhysHeapFreeMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                      IMG_UINT32 ui32MinMemInMBs)
+{
+	PHYS_HEAP *psDefaultHeap = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL, "psDeviceNode");
+
+	psDefaultHeap = psDeviceNode->apsPhysHeap[psDeviceNode->psDevConfig->eDefaultHeap];
+	if (psDefaultHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's default PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psDefaultHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psDefaultHeap,
+		                              MB2B(ui32MinMemInMBs),
+		                              &ui64FreePhysHeapMem);
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Default PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"MB "
+				"Minimum required: %uMB",
+				B2MB(ui64FreePhysHeapMem),
+				ui32MinMemInMBs));
+		}
+	}
+
+	return eError;
+}
+
+/**************************************************************************/ /*!
+@Function       CheckConnectionPhysHeapMem
+
+@Description    Check if there is enough memory in the PhysHeaps to allow a
+                connection to be made.
+
+@Input          psConnection    The connection being made.
+
+@Return         PVRSRV_OK if successful else a PVRSRV_ERROR.
+*/ /***************************************************************************/
+static PVRSRV_ERROR CheckConnectionPhysHeapMem(CONNECTION_DATA *psConnection)
+{
+	IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION;
+	IMG_UINT32 ui32AppHintPhysHeapMinMemOnConnection = 0;
+	void *pvAppHintState = NULL;
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_INVALID_PARAM(psConnection);
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, PhysHeapMinMemOnConnection,
+		&ui32AppHintDefault, &ui32AppHintPhysHeapMinMemOnConnection);
+	OSFreeAppHintState(pvAppHintState);
+
+	psDeviceNode = OSGetDevNode(psConnection);
+
+	if (ui32AppHintPhysHeapMinMemOnConnection != 0)
+	{
+		eError = DeviceDefaultPhysHeapFreeMemCheck(psDeviceNode,
+		                                           ui32AppHintPhysHeapMinMemOnConnection);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DeviceDefaultPhysHeapFreeMemCheck");
+
+		if (psDeviceNode->pfnCheckForSufficientFWPhysMem != NULL
+		    && RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION > 0)
+		{
+			eError = psDeviceNode->pfnCheckForSufficientFWPhysMem(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "pfnCheckForSufficientFWPhysMem");
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVConnectKM(CONNECTION_DATA *psConnection,
+				PVRSRV_DEVICE_NODE * psDeviceNode,
+				IMG_UINT32 ui32Flags,
+				IMG_UINT32 ui32ClientBuildOptions,
+				IMG_UINT32 ui32ClientDDKVersion,
+				IMG_UINT32 ui32ClientDDKBuild,
+				IMG_UINT8  *pui8KernelArch,
+				IMG_UINT32 *pui32CapabilityFlags,
+				IMG_UINT64 *ui64PackedBvnc)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32ServerBuildOptions;
+	IMG_UINT32			ui32DDKVersion, ui32DDKBuild;
+	PVRSRV_DATA			*psSRVData = NULL;
+	IMG_UINT64			ui64ProcessVASpaceSize = OSGetCurrentProcessVASpaceSize();
+	static IMG_BOOL		bIsFirstConnection=IMG_FALSE;
+#if defined(SUPPORT_RGX)
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#endif
+
+	/* Check the minimum free PhysHeap memory is available before allowing
+	 * the connection to succeed */
+	eError = CheckConnectionPhysHeapMem(psConnection);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(SUPPORT_RGX)
+	/* Gather BVNC information to output to UM */
+
+	*ui64PackedBvnc = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+	                       psDevInfo->sDevFeatureCfg.ui32V,
+	                       psDevInfo->sDevFeatureCfg.ui32N,
+	                       psDevInfo->sDevFeatureCfg.ui32C);
+#else
+	*ui64PackedBvnc = 0;
+#endif /* defined(SUPPORT_RGX)*/
+
+	/* Clear the flags */
+	*pui32CapabilityFlags = 0;
+
+	psSRVData = PVRSRVGetPVRSRVData();
+
+	psConnection->ui32ClientFlags = ui32Flags;
+
+	/*Set flags to pass back to the client showing which cache coherency is available.*/
+	/* Is the system snooping of caches emulated in software? */
+	if (PVRSRVSystemSnoopingIsEmulated(psDeviceNode->psDevConfig))
+	{
+		*pui32CapabilityFlags |= PVRSRV_CACHE_COHERENT_EMULATE_FLAG;
+	}
+	else
+	{
+		/*Set flags to pass back to the client showing which cache coherency is available.*/
+		/*Is the system CPU cache coherent?*/
+		if (PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig))
+		{
+			*pui32CapabilityFlags |= PVRSRV_CACHE_COHERENT_DEVICE_FLAG;
+		}
+	}
+
+	/* Has the system device non-mappable local memory?*/
+	if (PVRSRVSystemHasNonMappableLocalMemory(psDeviceNode->psDevConfig))
+	{
+		*pui32CapabilityFlags |= PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG;
+	}
+
+	/* Is system using FBCDC v31? */
+	if (psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode))
+	{
+		*pui32CapabilityFlags |= PVRSRV_FBCDC_V3_1_USED;
+	}
+
+	/* Set flags to indicate shared-virtual-memory (SVM) allocation availability */
+	if (! psDeviceNode->ui64GeneralSVMHeapTopVA || ! ui64ProcessVASpaceSize)
+	{
+		*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED;
+	}
+	else
+	{
+		if (ui64ProcessVASpaceSize <= psDeviceNode->ui64GeneralSVMHeapTopVA)
+		{
+			*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED;
+		}
+		else
+		{
+			/* This can happen when processor has more virtual address bits
+			   than device (i.e. alloc is not always guaranteed to succeed) */
+			*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL;
+		}
+	}
+
+	/* Is the system DMA capable? */
+	if (psDeviceNode->psDevConfig->bHasDma)
+	{
+		*pui32CapabilityFlags |= PVRSRV_SYSTEM_DMA_USED;
+	}
+
+#if defined(SUPPORT_RGX) && defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	/* For GPUs with lossy TFBC support, is system using lossy control group 1? */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		if (psDeviceNode->pfnGetTFBCLossyGroup(psDeviceNode) == 1)
+		{
+			*pui32CapabilityFlags |= PVRSRV_TFBC_LOSSY_GROUP_1;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+{
+	IMG_UINT32 ui32OSid = 0, ui32OSidReg = 0;
+	IMG_BOOL   bOSidAxiProtReg = IMG_FALSE;
+
+	ui32OSid    = (ui32Flags & SRV_VIRTVAL_FLAG_OSID_MASK)    >> (VIRTVAL_FLAG_OSID_SHIFT);
+	ui32OSidReg = (ui32Flags & SRV_VIRTVAL_FLAG_OSIDREG_MASK) >> (VIRTVAL_FLAG_OSIDREG_SHIFT);
+
+#if defined(EMULATOR)
+{
+    /* AXI_ACELITE is only supported on rogue cores - volcanic cores all support full ACE
+     * and don't want to compile the code below (RGX_FEATURE_AXI_ACELITE_BIT_MASK is not
+     * defined for volcanic cores).
+     */
+
+     PVRSRV_RGXDEV_INFO *psDevInfo;
+     psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#else
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACE))
+#endif
+	{
+		IMG_UINT32 ui32OSidAxiProtReg = 0, ui32OSidAxiProtTD = 0;
+
+		ui32OSidAxiProtReg = (ui32Flags & SRV_VIRTVAL_FLAG_AXIPREG_MASK) >> (VIRTVAL_FLAG_AXIPREG_SHIFT);
+		ui32OSidAxiProtTD  = (ui32Flags & SRV_VIRTVAL_FLAG_AXIPTD_MASK)  >> (VIRTVAL_FLAG_AXIPTD_SHIFT);
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"[AxiProt & Virt]: Setting bOSidAxiProt of Emulator's Trusted Device for Catbase %d to %s",
+				ui32OSidReg,
+				(ui32OSidAxiProtTD == 1)?"TRUE":"FALSE"));
+
+		bOSidAxiProtReg = ui32OSidAxiProtReg == 1;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"[AxiProt & Virt]: Setting bOSidAxiProt of FW's Register for Catbase %d to %s",
+				ui32OSidReg,
+				bOSidAxiProtReg?"TRUE":"FALSE"));
+
+		SetAxiProtOSid(psDeviceNode->psDevConfig->hSysData, ui32OSidReg, ui32OSidAxiProtTD);
+	}
+}
+#endif /* defined(EMULATOR) */
+
+	/* We now know the OSid, OSidReg and bOSidAxiProtReg setting for this
+	 * connection. We can access these from wherever we have a connection
+	 * reference and do not need to traverse an arbitrary linked-list to
+	 * obtain them. The settings are process-specific.
+	 */
+	psConnection->ui32OSid = ui32OSid;
+	psConnection->ui32OSidReg = ui32OSidReg;
+	psConnection->bOSidAxiProtReg = bOSidAxiProtReg;
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "[GPU Virtualization Validation]: OSIDs: %d, %d",
+	         ui32OSid,
+	         ui32OSidReg));
+}
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Only enabled if enabled in the UM */
+	if (!(ui32ClientBuildOptions & RGX_BUILD_OPTIONS_KM & OPTIONS_WORKLOAD_ESTIMATION_EN))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Workload Estimation disabled. Not enabled in UM",
+				__func__));
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/* Only enabled if enabled in the UM */
+	if (!(ui32ClientBuildOptions & RGX_BUILD_OPTIONS_KM & OPTIONS_PDVFS_EN))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Proactive DVFS disabled. Not enabled in UM",
+		         __func__));
+	}
+#endif
+
+	ui32DDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+	ui32DDKBuild = PVRVERSION_BUILD;
+
+	if (ui32Flags & SRV_FLAGS_CLIENT_64BIT_COMPAT)
+	{
+		psSRVData->sDriverInfo.ui8UMSupportedArch |= BUILD_ARCH_64BIT;
+	}
+	else
+	{
+		psSRVData->sDriverInfo.ui8UMSupportedArch |= BUILD_ARCH_32BIT;
+	}
+
+	if (IMG_FALSE == bIsFirstConnection)
+	{
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions = (RGX_BUILD_OPTIONS_KM);
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions = ui32ClientBuildOptions;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildVersion = ui32DDKVersion;
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildVersion = ui32ClientDDKVersion;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildRevision = ui32DDKBuild;
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildRevision = ui32ClientDDKBuild;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildType =
+				((RGX_BUILD_OPTIONS_KM) & OPTIONS_DEBUG_EN) ? BUILD_TYPE_DEBUG : BUILD_TYPE_RELEASE;
+
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildType =
+				(ui32ClientBuildOptions & OPTIONS_DEBUG_EN) ? BUILD_TYPE_DEBUG : BUILD_TYPE_RELEASE;
+
+		if (sizeof(void *) == POINTER_SIZE_64BIT)
+		{
+			psSRVData->sDriverInfo.ui8KMBitArch |= BUILD_ARCH_64BIT;
+		}
+		else
+		{
+			psSRVData->sDriverInfo.ui8KMBitArch |= BUILD_ARCH_32BIT;
+		}
+	}
+
+	/* Masking out every option that is not kernel specific*/
+	ui32ClientBuildOptions &= RGX_BUILD_OPTIONS_MASK_KM;
+
+	/*
+	 * Validate the build options
+	 */
+	ui32ServerBuildOptions = (RGX_BUILD_OPTIONS_KM);
+	if (ui32ServerBuildOptions != ui32ClientBuildOptions)
+	{
+		IMG_UINT32			ui32ServerBuildOptionsMismatch = ui32ServerBuildOptions ^ ui32ClientBuildOptions;
+		IMG_UINT32			ui32ClientBuildOptionsMismatch = ui32ServerBuildOptionsMismatch;
+
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask the debug flag option out as we do support combinations of debug vs release in um & km*/
+		ui32ServerBuildOptionsMismatch &= KM_OPTIONS_STRICT;
+		ui32ClientBuildOptionsMismatch &= UM_OPTIONS_STRICT;
+#endif
+		if ( (ui32ClientBuildOptions & ui32ClientBuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) %s: Mismatch in client-side and KM driver build options; "
+				"extra options present in client-side driver: (0x%x). Please check rgx_options.h",
+				__func__,
+				ui32ClientBuildOptions & ui32ClientBuildOptionsMismatch));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH, chk_exit);
+		}
+
+		if ( (ui32ServerBuildOptions & ui32ServerBuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) %s: Mismatch in client-side and KM driver build options; "
+				"extra options present in KM driver: (0x%x). Please check rgx_options.h",
+				__func__,
+				ui32ServerBuildOptions & ui32ServerBuildOptionsMismatch ));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH, chk_exit);
+		}
+		if (IMG_FALSE == bIsFirstConnection)
+		{
+			PVR_LOG(("%s: COMPAT_TEST: Client-side (0x%04x) (%s) and KM driver (0x%04x) (%s) build options differ.",
+																			__func__,
+																			ui32ClientBuildOptions,
+																			(psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildType)?"release":"debug",
+																			ui32ServerBuildOptions,
+																			(psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildType)?"release":"debug"));
+		}else{
+			PVR_DPF((PVR_DBG_WARNING, "%s: COMPAT_TEST: Client-side (0x%04x) and KM driver (0x%04x) build options differ.",
+																		__func__,
+																		ui32ClientBuildOptions,
+																		ui32ServerBuildOptions));
+
+		}
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: Client-side and KM driver build options match. [ OK ]", __func__));
+	}
+
+	/*
+	 * Validate DDK version
+	 */
+	if (ui32ClientDDKVersion != ui32DDKVersion)
+	{
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+		PVR_LOG(("(FAIL) %s: Incompatible driver DDK version (%u.%u) / client DDK version (%u.%u).",
+				__func__,
+				PVRVERSION_MAJ, PVRVERSION_MIN,
+				PVRVERSION_UNPACK_MAJ(ui32ClientDDKVersion),
+				PVRVERSION_UNPACK_MIN(ui32ClientDDKVersion)));
+		PVR_DBG_BREAK;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DDK_VERSION_MISMATCH, chk_exit);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: driver DDK version (%u.%u) and client DDK version (%u.%u) match. [ OK ]",
+				__func__,
+				PVRVERSION_MAJ, PVRVERSION_MIN, PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	/* Create stream for every connection except for the special clients
+	 * that don't need it e.g.: recipients of HWPerf data. */
+	if (!(psConnection->ui32ClientFlags & SRV_NO_HWPERF_CLIENT_STREAM))
+	{
+		IMG_CHAR acStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+		OSSNPrintf(acStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+		           PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC,
+		           psDeviceNode->sDevId.i32KernelDeviceID,
+		           psConnection->pid);
+
+		eError = TLStreamCreate(&psConnection->hClientTLStream,
+		                        acStreamName,
+		                        PVRSRV_CLIENT_TL_STREAM_SIZE_DEFAULT,
+		                        TL_OPMODE_DROP_NEWER |
+		                        TL_FLAG_ALLOCATE_ON_FIRST_OPEN,
+		                        NULL, NULL, NULL, NULL, NULL, NULL);
+		if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_ALREADY_EXISTS)
+		{
+			PVR_LOG_ERROR(eError, "TLStreamCreate");
+			psConnection->hClientTLStream = NULL;
+		}
+		else if (eError == PVRSRV_OK)
+		{
+			/* Set "tlctrl" stream as a notification channel. This channel is
+			 * is used to notify recipients about stream open/close (by writer)
+			 * actions (and possibly other actions in the future). */
+			eError = TLStreamSetNotifStream(psConnection->hClientTLStream,
+			                                psSRVData->hTLCtrlStream);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "TLStreamSetNotifStream");
+				TLStreamClose(psConnection->hClientTLStream);
+				psConnection->hClientTLStream = NULL;
+			}
+		}
+
+		/* Reset error status. We don't want to propagate any errors from here. */
+		eError = PVRSRV_OK;
+		PVR_DPF((PVR_DBG_MESSAGE, "Created stream \"%s\".", acStreamName));
+	}
+
+	/*
+	 * Validate DDK build
+	 */
+	if (ui32ClientDDKBuild != ui32DDKBuild)
+	{
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+		PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch in driver DDK revision (%d) / client DDK revision (%d).",
+				__func__, ui32DDKBuild, ui32ClientDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		PVR_DBG_BREAK;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DDK_BUILD_MISMATCH, chk_exit);
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: driver DDK revision (%d) and client DDK revision (%d) match. [ OK ]",
+				__func__, ui32DDKBuild, ui32ClientDDKBuild));
+	}
+
+#if defined(PDUMP)
+	/* Success so far so is it the PDump client that is connecting? */
+	if (ui32Flags & SRV_FLAGS_PDUMPCTRL)
+	{
+		if (psDeviceNode->sDevId.ui32InternalID == psSRVData->ui32PDumpBoundDevice)
+		{
+			PDumpConnectionNotify(psDeviceNode);
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+			PVR_DPF((PVR_DBG_ERROR, "%s: PDump requested for device %u but only permitted for device %u",
+					__func__, psDeviceNode->sDevId.ui32InternalID, psSRVData->ui32PDumpBoundDevice));
+			goto chk_exit;
+		}
+	}
+	else
+	{
+		/* Warn if the app is connecting to a device PDump won't be able to capture */
+		if (psDeviceNode->sDevId.ui32InternalID != psSRVData->ui32PDumpBoundDevice)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: NB. App running on device %d won't be captured by PDump (must be on device %u)",
+					__func__, psDeviceNode->sDevId.ui32InternalID, psSRVData->ui32PDumpBoundDevice));
+		}
+	}
+#endif
+
+	PVR_ASSERT(pui8KernelArch != NULL);
+
+	if (psSRVData->sDriverInfo.ui8KMBitArch & BUILD_ARCH_64BIT)
+	{
+		*pui8KernelArch = 64;
+	}
+	else
+	{
+		*pui8KernelArch = 32;
+	}
+
+	bIsFirstConnection = IMG_TRUE;
+
+#if defined(DEBUG_BRIDGE_KM)
+	{
+		int ii;
+
+		/* dump dispatch table offset lookup table */
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: g_BridgeDispatchTableStartOffsets[0-%lu] entries:", __func__, BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT - 1));
+		for (ii=0; ii < BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT; ii++)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "g_BridgeDispatchTableStartOffsets[%d]: %u", ii, g_BridgeDispatchTableStartOffsets[ii][PVR_DISPATCH_OFFSET_FIRST_FUNC]));
+		}
+	}
+#endif
+
+#if defined(PDUMP)
+	if (!(ui32Flags & SRV_FLAGS_PDUMPCTRL))
+	{
+		IMG_UINT64 ui64PDumpState = 0;
+
+		PDumpGetStateKM(&ui64PDumpState);
+		if (ui64PDumpState & PDUMP_STATE_CONNECTED)
+		{
+			*pui32CapabilityFlags |= PVRSRV_PDUMP_IS_RECORDING;
+		}
+	}
+#endif
+
+chk_exit:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVDisconnectKM(void)
+{
+#if defined(INTEGRITY_OS) && defined(DEBUG_BRIDGE_KM)
+	PVRSRVPrintBridgeStats();
+#endif
+	/* just return OK, per-process data is cleaned up by resmgr */
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       PVRSRVAcquireGlobalEventObjectKM
+@Description    Acquire the global event object.
+@Output         phGlobalEventObject    On success, points to the global event
+                                       object handle
+@Return         PVRSRV_ERROR           PVRSRV_OK on success or an error
+                                       otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR
+PVRSRVAcquireGlobalEventObjectKM(IMG_HANDLE *phGlobalEventObject)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	*phGlobalEventObject = psPVRSRVData->hGlobalEventObject;
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       PVRSRVReleaseGlobalEventObjectKM
+@Description    Release the global event object.
+@Output         hGlobalEventObject    Global event object handle
+@Return         PVRSRV_ERROR          PVRSRV_OK on success or an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR
+PVRSRVReleaseGlobalEventObjectKM(IMG_HANDLE hGlobalEventObject)
+{
+	PVR_ASSERT(PVRSRVGetPVRSRVData()->hGlobalEventObject == hGlobalEventObject);
+
+	return PVRSRV_OK;
+}
+
+static void _DumpDebugUMReqPrintWrapper(void *pvPriv, const IMG_CHAR *pszFmt, ...)
+{
+	va_list pvArgs;
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+	va_start(pvArgs, pszFmt);
+	PVRSRVReleasePrintfVArgs(pszFmt, pvArgs);
+	va_end(pvArgs);
+}
+
+/*
+	PVRSRVDumpDebugInfoKM
+*/
+PVRSRV_ERROR
+PVRSRVDumpDebugInfoKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDeviceNode,
+					  IMG_UINT32 ui32VerbLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32VerbLevel > DEBUG_REQUEST_VERBOSITY_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	PVR_LOG(("User requested PVR debug info"));
+
+	PVRSRVDebugRequest(psDeviceNode, ui32VerbLevel, _DumpDebugUMReqPrintWrapper, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVGetDevClockSpeedKM
+*/
+PVRSRV_ERROR
+PVRSRVGetDevClockSpeedKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_PUINT32  pui32RGXClockSpeed)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVR_ASSERT(psDeviceNode->pfnDeviceClockSpeed != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = psDeviceNode->pfnDeviceClockSpeed(psDeviceNode, pui32RGXClockSpeed);
+	PVR_WARN_IF_ERROR(eError, "pfnDeviceClockSpeed");
+
+	return eError;
+}
+
+
+/*
+	PVRSRVHWOpTimeoutKM
+*/
+PVRSRV_ERROR
+PVRSRVHWOpTimeoutKM(CONNECTION_DATA *psConnection,
+					PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(PVRSRV_RESET_ON_HWTIMEOUT)
+	PVR_LOG(("User requested OS reset"));
+	OSPanic();
+#endif
+	PVR_LOG(("HW operation timeout, dump server info"));
+	PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	return PVRSRV_OK;
+}
+
+
+IMG_INT
+DummyBW(IMG_UINT32 ui32DispatchTableEntry,
+		IMG_UINT8 *psBridgeIn,
+		IMG_UINT8 *psBridgeOut,
+		CONNECTION_DATA *psConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(psBridgeIn);
+	PVR_UNREFERENCED_PARAMETER(psBridgeOut);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(DEBUG_BRIDGE_KM)
+	PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE ERROR: ui32DispatchTableEntry %u (%s) mapped to "
+			 "Dummy Wrapper (probably not what you want!)",
+			 __func__, ui32DispatchTableEntry, g_BridgeDispatchTable[ui32DispatchTableEntry].pszIOCName));
+#else
+	PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE ERROR: ui32DispatchTableEntry %u mapped to "
+			 "Dummy Wrapper (probably not what you want!)",
+			 __func__, ui32DispatchTableEntry));
+#endif
+	return PVRSRV_ERROR_BRIDGE_ENOTTY;
+}
+
+PVRSRV_ERROR PVRSRVAlignmentCheckKM(CONNECTION_DATA *psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32AlignChecksSize,
+                                    IMG_UINT32 aui32AlignChecks[])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if !defined(NO_HARDWARE)
+
+	PVR_ASSERT(psDeviceNode->pfnAlignmentCheck != NULL);
+	return psDeviceNode->pfnAlignmentCheck(psDeviceNode, ui32AlignChecksSize,
+	                                       aui32AlignChecks);
+
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32AlignChecksSize);
+	PVR_UNREFERENCED_PARAMETER(aui32AlignChecks);
+
+	return PVRSRV_OK;
+
+#endif /* !defined(NO_HARDWARE) */
+
+}
+
+PVRSRV_ERROR PVRSRVGetDeviceStatusKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 *pui32DeviceStatus)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* First try to update the status. */
+	if (psDeviceNode->pfnUpdateHealthStatus != NULL)
+	{
+		PVRSRV_ERROR eError = psDeviceNode->pfnUpdateHealthStatus(psDeviceNode,
+		                                                          IMG_FALSE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "PVRSRVGetDeviceStatusKM: Failed to "
+					 "check for device status (%d)", eError));
+
+			/* Return unknown status and error because we don't know what
+			 * happened and if the status is valid. */
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_UNKNOWN;
+			return eError;
+		}
+	}
+
+	switch (OSAtomicRead(&psDeviceNode->eHealthStatus))
+	{
+		case PVRSRV_DEVICE_HEALTH_STATUS_OK:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_OK;
+			return PVRSRV_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_NOT_RESPONDING;
+			return PVRSRV_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:
+		case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:
+		case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_DEVICE_ERROR;
+			return PVRSRV_OK;
+		default:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_UNKNOWN;
+			return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+}
+
+PVRSRV_ERROR PVRSRVGetMultiCoreInfoKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32CapsSize,
+                                      IMG_UINT32 *pui32NumCores,
+                                      IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32CapsSize > 0)
+	{
+		/* Clear the buffer to ensure no uninitialised data is returned to UM
+		 * if the pfn call below does not write to the whole array, or is null.
+		 */
+		memset(pui64Caps, 0x00, (ui32CapsSize * sizeof(IMG_UINT64)));
+	}
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		eError = psDeviceNode->pfnGetMultiCoreInfo(psDeviceNode, ui32CapsSize, pui32NumCores, pui64Caps);
+	}
+	return eError;
+}
+
+
+/*!
+ * *****************************************************************************
+ * @brief A wrapper for removing entries in the g_BridgeDispatchTable array.
+ *		  All this does is zero the entry to allow for a full table re-population
+ *		  later.
+ *
+ * @param ui32BridgeGroup
+ * @param ui32Index
+ *
+ * @return
+ ********************************************************************************/
+void
+UnsetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup, IMG_UINT32 ui32Index)
+{
+	ui32Index += g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+
+	g_BridgeDispatchTable[ui32Index].pfFunction = NULL;
+	g_BridgeDispatchTable[ui32Index].hBridgeLock = NULL;
+#if defined(DEBUG_BRIDGE_KM)
+	g_BridgeDispatchTable[ui32Index].pszIOCName = NULL;
+	g_BridgeDispatchTable[ui32Index].pszFunctionName = NULL;
+	g_BridgeDispatchTable[ui32Index].pszBridgeLockName = NULL;
+	g_BridgeDispatchTable[ui32Index].ui32CallCount = 0;
+	g_BridgeDispatchTable[ui32Index].ui32CopyFromUserTotalBytes = 0;
+	g_BridgeDispatchTable[ui32Index].ui64TotalTimeNS = 0;
+	g_BridgeDispatchTable[ui32Index].ui64MaxTimeNS = 0;
+#endif
+}
+
+/*!
+ * *****************************************************************************
+ * @brief A wrapper for filling in the g_BridgeDispatchTable array that does
+ *		  error checking.
+ *
+ * @param ui32Index
+ * @param pszIOCName
+ * @param pfFunction
+ * @param pszFunctionName
+ * @param hBridgeLock
+ * @param pszBridgeLockName
+ * @param ui32InBufferSize
+ * @param ui32OutBufferSize
+ *
+ * @return
+ ********************************************************************************/
+void
+_SetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+					   IMG_UINT32 ui32Index,
+					   const IMG_CHAR *pszIOCName,
+					   BridgeWrapperFunction pfFunction,
+					   const IMG_CHAR *pszFunctionName,
+					   POS_LOCK hBridgeLock,
+					   const IMG_CHAR *pszBridgeLockName,
+					   IMG_UINT32 ui32InBufferSize,
+					   IMG_UINT32 ui32OutBufferSize)
+{
+	static IMG_UINT32 ui32PrevIndex = IMG_UINT32_MAX;		/* -1 */
+
+#if !defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE) && !defined(DEBUG_BRIDGE_KM)
+	PVR_UNREFERENCED_PARAMETER(pszFunctionName);
+	PVR_UNREFERENCED_PARAMETER(pszBridgeLockName);
+#endif
+
+	ui32Index += g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+	/* Enable this to dump out the dispatch table entries */
+	PVR_DPF((PVR_DBG_WARNING, "%s: g_BridgeDispatchTableStartOffsets[%d]=%d", __func__, ui32BridgeGroup, g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC]));
+	PVR_DPF((PVR_DBG_WARNING, "%s: %d %s %s %s", __func__, ui32Index, pszIOCName, pszFunctionName, pszBridgeLockName));
+#endif
+
+	/* Any gaps are sub-optimal in-terms of memory usage, but we are mainly
+	 * interested in spotting any large gap of wasted memory that could be
+	 * accidentally introduced.
+	 *
+	 * This will currently flag up any gaps > 5 entries.
+	 *
+	 * NOTE: This shouldn't be debug only since switching from debug->release
+	 * etc is likely to modify the available ioctls and thus be a point where
+	 * mistakes are exposed. This isn't run at a performance critical time.
+	 */
+	if ((ui32PrevIndex != IMG_UINT32_MAX) &&
+		((ui32Index >= ui32PrevIndex + DISPATCH_TABLE_GAP_THRESHOLD) ||
+		 (ui32Index <= ui32PrevIndex)))
+	{
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: There is a gap in the dispatch table between indices %u (%s) and %u (%s)",
+				 __func__, ui32PrevIndex, g_BridgeDispatchTable[ui32PrevIndex].pszIOCName,
+				 ui32Index, pszIOCName));
+#else
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: There is a gap in the dispatch table between indices %u and %u (%s)",
+				 __func__, (IMG_UINT)ui32PrevIndex, (IMG_UINT)ui32Index, pszIOCName));
+#endif
+	}
+
+	if (ui32Index >= BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Index %u (%s) out of range",
+				 __func__, (IMG_UINT)ui32Index, pszIOCName));
+
+#if defined(DEBUG_BRIDGE_KM)
+		PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE_DISPATCH_TABLE_ENTRY_COUNT = %lu",
+				 __func__, BRIDGE_DISPATCH_TABLE_ENTRY_COUNT));
+#if defined(SUPPORT_RGX)
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST));
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGX_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGX_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGX_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGX_LAST));
+#endif
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST));
+#endif
+
+		OSPanic();
+	}
+
+	/* Panic if the previous entry has been overwritten as this is not allowed!
+	 * NOTE: This shouldn't be debug only since switching from debug->release
+	 * etc is likely to modify the available ioctls and thus be a point where
+	 * mistakes are exposed. This isn't run at a performance critical time.
+	 */
+	if (g_BridgeDispatchTable[ui32Index].pfFunction)
+	{
+		if (g_BridgeDispatchTable[ui32Index].pfFunction != pfFunction)
+		{
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+			PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Adding dispatch table entry for %s clobbers an existing entry for %s (current pfn=<%p>, new pfn=<%p>)",
+				 __func__, pszIOCName, g_BridgeDispatchTable[ui32Index].pszIOCName,
+				 (void*)g_BridgeDispatchTable[ui32Index].pfFunction, (void*)pfFunction));
+#else
+			PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Adding dispatch table entry for %s clobbers an existing entry (index=%u). (current pfn=<%p>, new pfn=<%p>)",
+				 __func__, pszIOCName, ui32Index,
+				 (void*)g_BridgeDispatchTable[ui32Index].pfFunction, (void*)pfFunction));
+			PVR_DPF((PVR_DBG_WARNING, "NOTE: Enabling DEBUG_BRIDGE_KM_DISPATCH_TABLE may help debug this issue."));
+#endif
+			OSPanic();
+		}
+	}
+	else
+	{
+		g_BridgeDispatchTable[ui32Index].pfFunction = pfFunction;
+		g_BridgeDispatchTable[ui32Index].hBridgeLock = hBridgeLock;
+		g_BridgeDispatchTable[ui32Index].ui32InBufferSize = ui32InBufferSize;
+		g_BridgeDispatchTable[ui32Index].ui32OutBufferSize = ui32OutBufferSize;
+#if defined(DEBUG_BRIDGE_KM)
+		g_BridgeDispatchTable[ui32Index].pszIOCName = pszIOCName;
+		g_BridgeDispatchTable[ui32Index].pszFunctionName = pszFunctionName;
+		g_BridgeDispatchTable[ui32Index].pszBridgeLockName = pszBridgeLockName;
+		g_BridgeDispatchTable[ui32Index].ui32CallCount = 0;
+		g_BridgeDispatchTable[ui32Index].ui32CopyFromUserTotalBytes = 0;
+		g_BridgeDispatchTable[ui32Index].ui64TotalTimeNS = 0;
+		g_BridgeDispatchTable[ui32Index].ui64MaxTimeNS = 0;
+#endif
+	}
+
+	ui32PrevIndex = ui32Index;
+}
+
+static PVRSRV_ERROR _BridgeBufferAlloc(void *pvPrivData, void **pvOut)
+{
+	PVR_UNREFERENCED_PARAMETER(pvPrivData);
+
+	*pvOut = OSAllocZMem(PVRSRV_MAX_BRIDGE_IN_SIZE +
+	                     PVRSRV_MAX_BRIDGE_OUT_SIZE);
+	PVR_RETURN_IF_NOMEM(*pvOut);
+
+	return PVRSRV_OK;
+}
+
+static void _BridgeBufferFree(void *pvPrivData, void *pvFreeData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvPrivData);
+
+	OSFreeMem(pvFreeData);
+}
+
+PVRSRV_ERROR BridgeDispatcherInit(void)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(DEBUG_BRIDGE_KM)
+	eError = OSLockCreate(&g_hStatsLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", errorLockCreateFailed);
+#endif
+
+	eError = PVRSRVPoolCreate(_BridgeBufferAlloc,
+	                          _BridgeBufferFree,
+	                          PVRSRV_MAX_POOLED_BRIDGE_BUFFERS,
+	                          "Bridge buffer pool",
+	                          NULL,
+	                          &g_psBridgeBufferPool);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPoolCreate", erroPoolCreateFailed);
+
+	return PVRSRV_OK;
+
+erroPoolCreateFailed:
+#if defined(DEBUG_BRIDGE_KM)
+	OSLockDestroy(g_hStatsLock);
+	g_hStatsLock = NULL;
+errorLockCreateFailed:
+#endif
+	return eError;
+}
+
+void BridgeDispatcherDeinit(void)
+{
+	if (g_psBridgeBufferPool)
+	{
+		PVRSRVPoolDestroy(g_psBridgeBufferPool);
+		g_psBridgeBufferPool = NULL;
+	}
+
+#if defined(DEBUG_BRIDGE_KM)
+	if (g_hStatsLock)
+	{
+		OSLockDestroy(g_hStatsLock);
+		g_hStatsLock = NULL;
+	}
+#endif
+}
+
+PVRSRV_ERROR BridgedDispatchKM(CONNECTION_DATA * psConnection,
+                          PVRSRV_BRIDGE_PACKAGE   * psBridgePackageKM)
+{
+
+	void       * psBridgeIn=NULL;
+	void       * psBridgeOut=NULL;
+	BridgeWrapperFunction pfBridgeHandler;
+	IMG_UINT32   ui32DispatchTableEntry, ui32GroupBoundary;
+	PVRSRV_ERROR err = PVRSRV_OK;
+#if !defined(INTEGRITY_OS)
+	PVRSRV_POOL_TOKEN hBridgeBufferPoolToken = NULL;
+#endif
+	IMG_UINT32 ui32Timestamp = OSClockus();
+#if defined(DEBUG_BRIDGE_KM)
+	IMG_UINT64	ui64TimeStart;
+	IMG_UINT64	ui64TimeEnd;
+	IMG_UINT64	ui64TimeDiff;
+#endif
+	IMG_UINT32	ui32DispatchTableIndex, ui32DispatchTableEntryIndex;
+
+#if defined(DEBUG_BRIDGE_KM_STOP_AT_DISPATCH)
+	PVR_DBG_BREAK;
+#endif
+
+#if !defined(PVRSRV_ENABLE_HTB)
+	PVR_UNREFERENCED_PARAMETER(ui32Timestamp);
+#endif
+
+	if (psBridgePackageKM->ui32BridgeID >= BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Out of range dispatch table group ID: %d",
+		        __func__, psBridgePackageKM->ui32BridgeID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+	ui32DispatchTableIndex = OSConfineArrayIndexNoSpeculation(psBridgePackageKM->ui32BridgeID, BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT);
+
+	ui32DispatchTableEntry = g_BridgeDispatchTableStartOffsets[ui32DispatchTableIndex][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+	ui32GroupBoundary = g_BridgeDispatchTableStartOffsets[ui32DispatchTableIndex][PVR_DISPATCH_OFFSET_LAST_FUNC];
+
+	/* bridge function is not implemented in this build */
+	if (0 == ui32DispatchTableEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Dispatch table entry=%d, boundary = %d, (bridge module %d, function %d)",
+		         __func__,
+		         ui32DispatchTableEntry,
+		         ui32GroupBoundary,
+		         psBridgePackageKM->ui32BridgeID,
+		         psBridgePackageKM->ui32FunctionID));
+		/* this points to DummyBW() which returns PVRSRV_ERROR_ENOTTY */
+		err = g_BridgeDispatchTable[ui32DispatchTableEntry].pfFunction(ui32DispatchTableEntry,
+				  psBridgeIn,
+				  psBridgeOut,
+				  psConnection);
+		goto return_error;
+	}
+	if ((ui32DispatchTableEntry + psBridgePackageKM->ui32FunctionID) > ui32GroupBoundary)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Dispatch table entry=%d, boundary = %d, (bridge module %d, function %d)",
+		         __func__,
+		         ui32DispatchTableEntry,
+		         ui32GroupBoundary,
+		         psBridgePackageKM->ui32BridgeID,
+		         psBridgePackageKM->ui32FunctionID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+	ui32DispatchTableEntry += psBridgePackageKM->ui32FunctionID;
+	ui32DispatchTableEntryIndex = OSConfineArrayIndexNoSpeculation(ui32DispatchTableEntry, ui32GroupBoundary+1);
+	if (BRIDGE_DISPATCH_TABLE_ENTRY_COUNT <= ui32DispatchTableEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dispatch table entry=%d, entry count = %lu,"
+		        " (bridge module %d, function %d)", __func__,
+		        ui32DispatchTableEntry, BRIDGE_DISPATCH_TABLE_ENTRY_COUNT,
+		        psBridgePackageKM->ui32BridgeID,
+		        psBridgePackageKM->ui32FunctionID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+#if defined(DEBUG_BRIDGE_KM)
+	BridgeGlobalStatsLock();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s idx:%d mod:%d, func:%d",
+			__func__, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].pszIOCName,
+			ui32DispatchTableEntryIndex, psBridgePackageKM->ui32BridgeID, psBridgePackageKM->ui32FunctionID));
+
+	g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32CallCount++;
+	g_BridgeGlobalStats.ui32IOCTLCount++;
+	BridgeGlobalStatsUnlock();
+#endif
+
+	if (psBridgePackageKM->ui32InBufferSize != g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32InBufferSize ||
+	    psBridgePackageKM->ui32OutBufferSize != g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32OutBufferSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Bridge buffer sizes mismatch! "
+		        "In: User(%u), Kernel(%u) - Out: User(%u), Kernel(%u)",
+		        __func__,
+		        psBridgePackageKM->ui32InBufferSize, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32InBufferSize,
+		        psBridgePackageKM->ui32OutBufferSize, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32OutBufferSize));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+	if (g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock != NULL)
+	{
+		OSLockAcquire(g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock);
+	}
+#if !defined(INTEGRITY_OS)
+	/* try to acquire a bridge buffer from the pool */
+
+	err = PVRSRVPoolGet(g_psBridgeBufferPool,
+			&hBridgeBufferPoolToken,
+			&psBridgeIn);
+	PVR_LOG_GOTO_IF_ERROR(err, "PVRSRVPoolGet", unlock_and_return_error);
+
+	psBridgeOut = ((IMG_BYTE *) psBridgeIn) + PVRSRV_MAX_BRIDGE_IN_SIZE;
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+	ui64TimeStart = OSClockns64();
+#endif
+
+#if !defined(INTEGRITY_OS)
+	if ((CopyFromUserWrapper (psConnection,
+							  ui32DispatchTableEntryIndex,
+							  psBridgeIn,
+							  psBridgePackageKM->pvParamIn,
+							  psBridgePackageKM->ui32InBufferSize) != PVRSRV_OK)
+#if defined(__QNXNTO__)
+/* For Neutrino, the output bridge buffer acts as an input as well */
+					|| (CopyFromUserWrapper(psConnection,
+											ui32DispatchTableEntryIndex,
+											psBridgeOut,
+											(void *)((uintptr_t)psBridgePackageKM->pvParamIn + psBridgePackageKM->ui32InBufferSize),
+											psBridgePackageKM->ui32OutBufferSize) != PVRSRV_OK)
+#endif
+		) /* end of if-condition */
+	{
+		PVR_LOG_GOTO_WITH_ERROR("CopyFromUserWrapper", err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+	}
+#else
+	psBridgeIn = psBridgePackageKM->pvParamIn;
+	psBridgeOut = psBridgePackageKM->pvParamOut;
+#endif
+
+	pfBridgeHandler =
+		(BridgeWrapperFunction)g_BridgeDispatchTable[ui32DispatchTableEntryIndex].pfFunction;
+
+	if (pfBridgeHandler == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui32DispatchTableEntry = %d is not a registered function!",
+				 __func__, ui32DispatchTableEntry));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+	}
+
+	/* pfBridgeHandler functions do not fail and return an IMG_INT.
+	 * The value returned is either 0 or PVRSRV_OK (0).
+	 * In the event this changes an error may be +ve or -ve,
+	 * so try to return something consistent here.
+	 */
+	if (0 != pfBridgeHandler(ui32DispatchTableEntryIndex,
+						  psBridgeIn,
+						  psBridgeOut,
+						  psConnection)
+		)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("pfBridgeHandler", err, PVRSRV_ERROR_BRIDGE_EPERM, unlock_and_return_error);
+	}
+
+	/*
+	   This should always be true as a.t.m. all bridge calls have to
+	   return an error message, but this could change so we do this
+	   check to be safe.
+	*/
+#if !defined(INTEGRITY_OS)
+	if (psBridgePackageKM->ui32OutBufferSize > 0)
+	{
+		if (CopyToUserWrapper (psConnection,
+						ui32DispatchTableEntryIndex,
+						psBridgePackageKM->pvParamOut,
+						psBridgeOut,
+						psBridgePackageKM->ui32OutBufferSize) != PVRSRV_OK)
+		{
+			PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+		}
+	}
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+	ui64TimeEnd = OSClockns64();
+
+	ui64TimeDiff = ui64TimeEnd - ui64TimeStart;
+
+	BridgeGlobalStatsLock();
+
+	g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64TotalTimeNS += ui64TimeDiff;
+
+	if (ui64TimeDiff > g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64MaxTimeNS)
+	{
+		g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64MaxTimeNS = ui64TimeDiff;
+	}
+
+	BridgeGlobalStatsUnlock();
+#endif
+
+unlock_and_return_error:
+
+	if (g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock != NULL)
+	{
+		OSLockRelease(g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock);
+	}
+
+#if !defined(INTEGRITY_OS)
+	if (hBridgeBufferPoolToken != NULL)
+	{
+		err = PVRSRVPoolPut(g_psBridgeBufferPool,
+				hBridgeBufferPoolToken);
+		PVR_LOG_IF_ERROR(err, "PVRSRVPoolPut");
+	}
+#endif
+
+return_error:
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: returning (err = %d)", __func__, err));
+	}
+	/* ignore transport layer bridge to avoid HTB flooding */
+	if (psBridgePackageKM->ui32BridgeID != PVRSRV_BRIDGE_PVRTL)
+	{
+		if (err)
+		{
+			HTBLOGK(HTB_SF_BRG_BRIDGE_CALL_ERR, ui32Timestamp,
+			        psBridgePackageKM->ui32BridgeID,
+			        psBridgePackageKM->ui32FunctionID, err);
+		}
+		else
+		{
+			HTBLOGK(HTB_SF_BRG_BRIDGE_CALL, ui32Timestamp,
+			        psBridgePackageKM->ui32BridgeID,
+			        psBridgePackageKM->ui32FunctionID);
+		}
+	}
+
+	return err;
+}
+
+PVRSRV_ERROR PVRSRVFindProcessMemStatsKM(IMG_PID pid, IMG_UINT32 ui32ArrSize, IMG_BOOL bAllProcessStats, IMG_UINT64 *pui64MemStatArray)
+{
+#if !defined(__QNXNTO__)
+	return PVRSRVFindProcessMemStats(pid,
+					ui32ArrSize,
+					bAllProcessStats,
+					pui64MemStatArray);
+#else
+	PVR_DPF((PVR_DBG_ERROR, "This functionality is not yet implemented for this platform"));
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+
+}
+
+void PVRSRVBlockIfFrozen(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Short-circuit if we're not marked as frozen */
+	if (OSAtomicRead(&psDevNode->eFrozen) == 0)
+	{
+		return;
+	}
+
+	while ((psDevNode->eDevState == PVRSRV_DEVICE_STATE_FROZEN) &&
+	       ((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_TIMEOUT)))
+	{
+		IMG_HANDLE hEvent;
+
+		eError = OSEventObjectOpen(psDevNode->hDeviceThreadEvObj, &hEvent);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Failed to open event object (%d)", __func__,
+			        eError));
+
+			/* Continue with the loop by resetting to PVRSRV_OK. This makes it
+			 * a busy wait (with a 10ms delay) if we cannot grab an EvObj
+			 * reference.
+			 */
+			eError = PVRSRV_OK;
+			OSSleepms(10U);		/* Allow failure (OOM etc.) to resolve */
+			continue;
+		}
+
+		if (hEvent != NULL)
+		{
+			/* Register that we've got an interest in the device */
+			(void) OSAtomicIncrement(&psDevNode->iFreezeCount);
+
+			eError = OSEventObjectWait(hEvent);
+
+			OSEventObjectClose(hEvent);
+
+			(void) OSAtomicDecrement(&psDevNode->iFreezeCount);
+		}
+	}
+
+	/* We expect to exit the above loop only when the device is no longer
+	 * FROZEN. If we are still marked as frozen that is an unexpected error
+	 * so log it.
+	 */
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_FROZEN)
+	{
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectWait");
+		if (eError != PVRSRV_OK)
+		{
+			OSDumpStack();
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_checkpoint.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_checkpoint.c
new file mode 100644
index 000000000000..a72ec7dc1809
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_checkpoint.c
@@ -0,0 +1,3454 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation checkpoint interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server side code for services synchronisation interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "dllist.h"
+#include "sync.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+#include "sync_checkpoint_init.h"
+#include "lock.h"
+#include "log2.h"
+#include "pvrsrv.h"
+#include "pdump_km.h"
+#include "info_page.h"
+#include "os_apphint.h"
+#include "rgxfwutils.h"
+
+#include "pvrsrv_sync_km.h"
+#include "rgxhwperf.h"
+
+
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/* Enable this to turn on debug relating to the creation and
+   resolution of contexts */
+#define ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG 0
+
+/* Enable this to turn on debug relating to the creation and
+   resolution of fences */
+#define ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint
+   allocation and freeing */
+#define ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint
+   enqueuing and signalling */
+#define ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint pool */
+#define ENABLE_SYNC_CHECKPOINT_POOL_DEBUG 0
+
+/* Enable this to turn on debug relating to sync checkpoint UFO
+   lookup */
+#define ENABLE_SYNC_CHECKPOINT_UFO_DEBUG 0
+
+/* Enable this to turn on sync checkpoint deferred cleanup debug
+ * (for syncs we have been told to free but which have some
+ * outstanding FW operations remaining (enqueued in CCBs)
+ */
+#define ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG 0
+
+#else
+
+#define ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_POOL_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_UFO_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG 0
+
+#endif
+
+/* Maximum number of deferred sync checkpoint signal/error received for atomic context */
+#define SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL 500
+
+/* Set the size of the sync checkpoint pool (not used if 0).
+ * A pool will be maintained for each sync checkpoint context.
+ * SYNC_CHECKPOINT_POOL_LIMIT must be a power of 2 (POT),
+ * as the pool wrap mask is calculated using it.
+ */
+#define SYNC_CHECKPOINT_POOL_LIMIT	1024
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT & (SYNC_CHECKPOINT_POOL_LIMIT - 1)) != 0
+#error "SYNC_CHECKPOINT_POOL_LIMIT must be power of 2."
+#endif
+
+/*
+	This defines the maximum amount of synchronisation memory
+	that can be allocated per sync checkpoint context.
+	In reality this number is meaningless as we would run out
+	of synchronisation memory before we reach this limit, but
+	we need to provide a size to the span RA.
+ */
+#define MAX_SYNC_CHECKPOINT_MEM  (4 * 1024 * 1024)
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+/* Flags used to indicate state of pool */
+#define SYNC_CHECKPOINT_POOL_FULL  (1)
+#define SYNC_CHECKPOINT_POOL_VALID (1 << 7)
+#define CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags & SYNC_CHECKPOINT_POOL_FULL
+#define CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags & SYNC_CHECKPOINT_POOL_VALID
+#define SET_CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags |= SYNC_CHECKPOINT_POOL_FULL
+#define SET_CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags |= SYNC_CHECKPOINT_POOL_VALID
+#define CLEAR_CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags &= ~SYNC_CHECKPOINT_POOL_FULL
+#define CLEAR_CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags &= ~SYNC_CHECKPOINT_POOL_VALID
+#endif
+struct _SYNC_CHECKPOINT_CONTEXT_CTL_
+{
+	SHARED_DEV_CONNECTION					psDeviceNode;
+	/*
+	 *  Used as head of linked-list of sync checkpoints for which
+	 *  SyncCheckpointFree() has been called, but have outstanding
+	 *  FW operations (enqueued in CCBs)
+	 *  This list will be check whenever a SyncCheckpointFree() is
+	 *  called, and when SyncCheckpointContextDestroy() is called.
+	 */
+	DLLIST_NODE								sDeferredCleanupListHead;
+	/* Lock to protect the deferred cleanup list */
+	POS_SPINLOCK							hDeferredCleanupListLock;
+
+	/* Counters to provide stats for number of checkpoints used at any one time */
+	IMG_UINT32                              ui32CurrentInUseSyncCheckpoints;
+	IMG_UINT32                              ui32MaxInUseSyncCheckpoints;
+	IMG_UINT32                              ui32CurrentInUseMirroredSyncCPs;
+	IMG_UINT32                              ui32MaxInUseMirroredSyncCPs;
+	/* Lock to protect the checkpoint stats */
+	POS_SPINLOCK							hSyncCheckpointStatsLock;
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	IMG_UINT32								ui32SyncCheckpointPoolSize;    /*! Allocated size of the pool */
+	IMG_UINT32								ui32SyncCheckpointPoolCount;   /*! Number of checkpoints currently in pool */
+	IMG_UINT32								ui32SyncCheckpointPoolWp;      /*! Pool write pointer */
+	IMG_UINT32								ui32SyncCheckpointPoolRp;      /*! Pool read pointer */
+#if defined(PDUMP)
+	SYNC_CHECKPOINT_BLOCK					*psSyncCheckpointPoolBlock;    /*! Block that all pool checkpoints come from */
+	IMG_BOOL								bAllocateFromCheckpointPool;   /*! PDumps only use the pool once it has allocated all its checkpoints */
+#endif
+	POS_SPINLOCK							hSyncCheckpointPoolLock;       /*! Lock to protect access to pool control data */
+	IMG_UINT8								ui8PoolStateFlags;             /*! Flags to indicate state of pool */
+	/*! Array of SYNC_CHECKPOINTs. Must be last member in structure */
+	SYNC_CHECKPOINT *apsSyncCheckpointPool[IMG_FLEX_ARRAY_MEMBER];   /*! The allocated checkpoint pool */
+#endif
+}; /*_SYNC_CHECKPOINT_CONTEXT_CTL is already typedef-ed in sync_checkpoint_internal.h */
+
+struct SYNC_CHECKPOINT_CONTEXT_TAG
+{
+#if defined(PDUMP)
+	DLLIST_NODE						sSyncCheckpointBlockListHead;          /*!< List head for the sync chkpt blocks in this context*/
+	DLLIST_NODE						sListNode;				               /*!< List node for the sync chkpt context list*/
+	POS_LOCK						hSyncCheckpointBlockListLock;          /*!< sync chkpt blocks list lock*/
+#endif
+	RA_ARENA						*psSpanRA;                             /*!< RA used for span management of SubAllocRA */
+	RA_ARENA						*psSubAllocRA;                         /*!< RA context */
+	_PSYNC_CHECKPOINT_CONTEXT_CTL	psContextCtl;
+	ATOMIC_T						hCheckpointCount;                      /*!< Checkpoint count for this context */
+	ATOMIC_T						hRefCount;                             /*!< Ref count for this context */
+}; /*_SYNC_CHECKPOINT_CONTEXT is already typedef-ed in sync_checkpoint_internal.h */
+
+/* this is the max number of sync checkpoint records we will search or dump
+ * at any time.
+ */
+#define SYNC_CHECKPOINT_RECORD_LIMIT 20000
+
+#define DECREMENT_WITH_WRAP(value, sz) ((value) ? ((value) - 1) : ((sz) - 1))
+
+struct SYNC_CHECKPOINT_RECORD
+{
+	SYNC_CHECKPOINT_BLOCK	*psSyncCheckpointBlock;	/*!< handle to SYNC_CHECKPOINT_BLOCK */
+	IMG_UINT32				ui32SyncOffset;			/*!< offset to sync in block */
+	IMG_UINT32				ui32FwBlockAddr;
+	IMG_PID					uiPID;
+	IMG_UINT32				ui32UID;
+	IMG_UINT64				ui64OSTime;
+	DLLIST_NODE				sNode;
+	IMG_CHAR				szClassName[PVRSRV_SYNC_NAME_LENGTH];
+	PSYNC_CHECKPOINT		pSyncCheckpt;
+};
+
+static PFN_SYNC_CHECKPOINT_STRUCT *g_psSyncCheckpointPfnStruct = NULL;
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static SYNC_CHECKPOINT *_GetCheckpointFromPool(_SYNC_CHECKPOINT_CONTEXT *psContext);
+static IMG_BOOL _PutCheckpointInPool(SYNC_CHECKPOINT *psSyncCheckpoint);
+static IMG_UINT32 _CleanCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext);
+#endif
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+static IMG_UINT32 gui32NumSyncCheckpointContexts = 0;
+#endif
+
+/* Defined values to indicate status of sync checkpoint, which is
+ * stored in the memory of the structure */
+#define SYNC_CHECKPOINT_PATTERN_IN_USE 0x1a1aa
+#define SYNC_CHECKPOINT_PATTERN_IN_POOL 0x2b2bb
+#define SYNC_CHECKPOINT_PATTERN_FREED 0x3c3cc
+
+#if defined(SUPPORT_RGX)
+static inline void RGXSRVHWPerfSyncCheckpointUFOIsSignalled(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               SYNC_CHECKPOINT *psSyncCheckpointInt, IMG_UINT32 ui32FenceSyncFlags)
+{
+	if (RGXHWPerfHostIsEventEnabled(psDevInfo, RGX_HWPERF_HOST_UFO)
+	    && !(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		RGX_HWPERF_UFO_EV eEv;
+		RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+		if (psSyncCheckpointInt)
+		{
+			IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+			if ((ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ||
+				(ui32State == PVRSRV_SYNC_CHECKPOINT_ERRORED))
+			{
+				sSyncData.sCheckSuccess.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+				sSyncData.sCheckSuccess.ui32Value = ui32State;
+				eEv = RGX_HWPERF_UFO_EV_CHECK_SUCCESS;
+			}
+			else
+			{
+				sSyncData.sCheckFail.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+				sSyncData.sCheckFail.ui32Value = ui32State;
+				sSyncData.sCheckFail.ui32Required = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				eEv = RGX_HWPERF_UFO_EV_CHECK_FAIL;
+			}
+			RGXHWPerfHostPostUfoEvent(psDevInfo, eEv, &sSyncData,
+			    (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+		}
+	}
+}
+
+static inline void RGXSRVHWPerfSyncCheckpointUFOUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               SYNC_CHECKPOINT *psSyncCheckpointInt, IMG_UINT32 ui32FenceSyncFlags)
+{
+	if (RGXHWPerfHostIsEventEnabled(psDevInfo, RGX_HWPERF_HOST_UFO)
+	    && !(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+		if (psSyncCheckpointInt)
+		{
+			sSyncData.sUpdate.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+			sSyncData.sUpdate.ui32OldValue = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+			sSyncData.sUpdate.ui32NewValue = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+			RGXHWPerfHostPostUfoEvent(psDevInfo, RGX_HWPERF_UFO_EV_UPDATE, &sSyncData,
+			    (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+		}
+	}
+}
+#endif
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordAdd(PSYNC_CHECKPOINT_RECORD_HANDLE *phRecord,
+	                    SYNC_CHECKPOINT_BLOCK *hSyncCheckpointBlock,
+	                    IMG_UINT32 ui32FwBlockAddr,
+	                    IMG_UINT32 ui32SyncOffset,
+	                    IMG_UINT32 ui32UID,
+	                    IMG_UINT32 ui32ClassNameSize,
+	                    const IMG_CHAR *pszClassName, PSYNC_CHECKPOINT pSyncCheckpt);
+static PVRSRV_ERROR
+_SyncCheckpointRecordRemove(PSYNC_CHECKPOINT_RECORD_HANDLE hRecord);
+static void _SyncCheckpointState(PDLLIST_NODE psNode,
+                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile);
+static void _SyncCheckpointDebugRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                        IMG_UINT32 ui32VerbLevel,
+                                        DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                        void *pvDumpDebugFile);
+static PVRSRV_ERROR _SyncCheckpointRecordListInit(PVRSRV_DEVICE_NODE *psDevNode);
+static void _SyncCheckpointRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+#if defined(PDUMP)
+static void
+MISRHandler_PdumpDeferredSyncSignalPoster(void *pvData);
+static PVRSRV_ERROR _SyncCheckpointAllocPDump(PVRSRV_DEVICE_NODE *psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint);
+static PVRSRV_ERROR _SyncCheckpointUpdatePDump(PPVRSRV_DEVICE_NODE psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint, IMG_UINT32 ui32Status, IMG_UINT32 ui32FenceSyncFlags);
+static PVRSRV_ERROR _SyncCheckpointPDumpTransition(void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+#endif
+
+/* Unique incremental ID assigned to sync checkpoints when allocated */
+static IMG_UINT32 g_SyncCheckpointUID;
+
+static void _CheckDeferredCleanupList(_SYNC_CHECKPOINT_CONTEXT *psContext);
+
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContextInt = (_SYNC_CHECKPOINT_CONTEXT *) psContext;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContextInt->psContextCtl;
+	IMG_UINT32 ui32RefCt = OSAtomicRead(&psContextInt->hRefCount);
+
+	if (ui32RefCt == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_INVALID_CONTEXT,
+		              "SyncCheckpointContextUnref context already freed");
+	}
+	else if (OSAtomicDecrement(&psContextInt->hRefCount) == 0)
+	{
+#if defined(PDUMP)
+		PPVRSRV_DEVICE_NODE psDeviceNode = psCtxCtl->psDeviceNode;
+#endif
+
+		/* SyncCheckpointContextDestroy only when no longer referenced */
+		OSSpinLockDestroy(psCtxCtl->hDeferredCleanupListLock);
+		psCtxCtl->hDeferredCleanupListLock = NULL;
+
+		OSSpinLockDestroy(psCtxCtl->hSyncCheckpointStatsLock);
+		psCtxCtl->hSyncCheckpointStatsLock = NULL;
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+		if (psCtxCtl->ui32SyncCheckpointPoolCount)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			        "%s called for context<%p> with %d sync checkpoints still"
+			        " in the pool",
+			        __func__,
+			        (void *) psContext,
+			        psCtxCtl->ui32SyncCheckpointPoolCount));
+		}
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+		OSSpinLockDestroy(psCtxCtl->hSyncCheckpointPoolLock);
+		psCtxCtl->hSyncCheckpointPoolLock = NULL;
+#endif
+		OSFreeMem(psContextInt->psContextCtl);
+		RA_Delete(psContextInt->psSpanRA);
+		RA_Delete(psContextInt->psSubAllocRA);
+
+#if defined(PDUMP)
+		PVR_ASSERT(dllist_is_empty(&psContext->sSyncCheckpointBlockListHead));
+
+		OSLockAcquire(psDeviceNode->hSyncCheckpointContextListLock);
+		dllist_remove_node(&psContext->sListNode);
+		OSLockRelease(psDeviceNode->hSyncCheckpointContextListLock);
+
+		OSLockDestroy(psContext->hSyncCheckpointBlockListLock);
+#endif
+
+		OSFreeMem(psContext);
+	}
+}
+
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContextInt = (_SYNC_CHECKPOINT_CONTEXT *)psContext;
+	IMG_UINT32 ui32RefCt = OSAtomicRead(&psContextInt->hRefCount);
+
+	if (ui32RefCt == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_INVALID_CONTEXT,
+		              "SyncCheckpointContextRef context use after free");
+	}
+	else
+	{
+		OSAtomicIncrement(&psContextInt->hRefCount);
+	}
+}
+
+/*
+	Internal interfaces for management of synchronisation block memory
+ */
+static PVRSRV_ERROR
+_AllocSyncCheckpointBlock(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                          SYNC_CHECKPOINT_BLOCK    **ppsSyncBlock)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	SYNC_CHECKPOINT_BLOCK *psSyncBlk;
+	PVRSRV_ERROR eError;
+
+	psSyncBlk = OSAllocMem(sizeof(*psSyncBlk));
+	PVR_LOG_GOTO_IF_NOMEM(psSyncBlk, eError, fail_alloc);
+
+	psSyncBlk->psContext = psContext;
+
+	/* Allocate sync checkpoint block */
+	psDevNode = psContext->psContextCtl->psDeviceNode;
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psDevNode, eError, fail_alloc_ufo_block);
+
+	eError = psDevNode->pfnAllocUFOBlock(psDevNode,
+	                                     SYNC_CHECKPOINT_POOL_LIMIT * sizeof(SYNC_CHECKPOINT_FW_OBJ),
+	                                     &psSyncBlk->hMemDesc,
+	                                     &psSyncBlk->ui32FirmwareAddr,
+	                                     &psSyncBlk->ui32SyncBlockSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnAllocUFOBlock", fail_alloc_ufo_block);
+
+	eError = DevmemAcquireCpuVirtAddr(psSyncBlk->hMemDesc,
+	                                  (void **) &psSyncBlk->pui32LinAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail_devmem_acquire);
+
+	OSAtomicWrite(&psSyncBlk->hRefCount, 1);
+
+	OSLockCreate(&psSyncBlk->hLock);
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Allocated Sync Checkpoint UFO block (FirmwareVAddr = 0x%08x)",
+	                      psSyncBlk->ui32FirmwareAddr);
+#if defined(PDUMP)
+	OSLockAcquire(psContext->hSyncCheckpointBlockListLock);
+	dllist_add_to_tail(&psContext->sSyncCheckpointBlockListHead, &psSyncBlk->sListNode);
+	OSLockRelease(psContext->hSyncCheckpointBlockListLock);
+#endif
+
+	*ppsSyncBlock = psSyncBlk;
+	return PVRSRV_OK;
+
+fail_devmem_acquire:
+	psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->hMemDesc);
+fail_alloc_ufo_block:
+	OSFreeMem(psSyncBlk);
+fail_alloc:
+	return eError;
+}
+
+static void
+_FreeSyncCheckpointBlock(SYNC_CHECKPOINT_BLOCK *psSyncBlk)
+{
+	OSLockAcquire(psSyncBlk->hLock);
+	if (0 == OSAtomicDecrement(&psSyncBlk->hRefCount))
+	{
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncBlk->psContext;
+		PVRSRV_DEVICE_NODE *psDevNode = psContext->psContextCtl->psDeviceNode;
+
+#if defined(PDUMP)
+		OSLockAcquire(psSyncBlk->psContext->hSyncCheckpointBlockListLock);
+		dllist_remove_node(&psSyncBlk->sListNode);
+		OSLockRelease(psSyncBlk->psContext->hSyncCheckpointBlockListLock);
+#endif
+		DevmemReleaseCpuVirtAddr(psSyncBlk->hMemDesc);
+		psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->hMemDesc);
+		OSLockRelease(psSyncBlk->hLock);
+		OSLockDestroy(psSyncBlk->hLock);
+		psSyncBlk->hLock = NULL;
+		OSFreeMem(psSyncBlk);
+	}
+	else
+	{
+		OSLockRelease(psSyncBlk->hLock);
+	}
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointBlockImport(RA_PERARENA_HANDLE hArena,
+                           RA_LENGTH_T uSize,
+                           RA_FLAGS_T uFlags,
+                           RA_LENGTH_T uBaseAlignment,
+                           const IMG_CHAR *pszAnnotation,
+                           RA_IMPORT *psImport)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = hArena;
+	SYNC_CHECKPOINT_BLOCK *psSyncBlock = NULL;
+	RA_LENGTH_T uiSpanSize;
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(uFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM((hArena != NULL), "hArena");
+
+	/* Check we've not be called with an unexpected size */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uSize == sizeof(SYNC_CHECKPOINT_FW_OBJ)), "uSize");
+
+	/*
+		Ensure the sync checkpoint context doesn't go away while we have
+		sync blocks attached to it.
+	 */
+	SyncCheckpointContextRef((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+	/* Allocate the block of memory */
+	eError = _AllocSyncCheckpointBlock(psContext, &psSyncBlock);
+	PVR_GOTO_IF_ERROR(eError, fail_syncblockalloc);
+
+	/* Allocate a span for it */
+	eError = RA_Alloc(psContext->psSpanRA,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  pszAnnotation,
+	                  &psSyncBlock->uiSpanBase,
+	                  &uiSpanSize,
+	                  NULL);
+	PVR_GOTO_IF_ERROR(eError, fail_spanalloc);
+
+	/*
+		There is no reason the span RA should return an allocation larger
+		then we request
+	 */
+	PVR_LOG_IF_FALSE((uiSpanSize == psSyncBlock->ui32SyncBlockSize),
+	                 "uiSpanSize invalid");
+
+	psImport->base = psSyncBlock->uiSpanBase;
+	psImport->uSize = psSyncBlock->ui32SyncBlockSize;
+	psImport->hPriv = psSyncBlock;
+	return PVRSRV_OK;
+
+fail_spanalloc:
+	_FreeSyncCheckpointBlock(psSyncBlock);
+fail_syncblockalloc:
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+	return eError;
+}
+
+static void
+_SyncCheckpointBlockUnimport(RA_PERARENA_HANDLE hArena,
+                             RA_BASE_T uiBase,
+                             RA_PERISPAN_HANDLE hImport)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = hArena;
+	SYNC_CHECKPOINT_BLOCK   *psSyncBlock = hImport;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE((psContext != NULL), "hArena invalid");
+	PVR_LOG_RETURN_VOID_IF_FALSE((psSyncBlock != NULL), "hImport invalid");
+	PVR_LOG_RETURN_VOID_IF_FALSE((uiBase == psSyncBlock->uiSpanBase), "uiBase invalid");
+
+	/* Free the span this import is using */
+	RA_Free(psContext->psSpanRA, uiBase);
+
+	/* Free the sync checkpoint block */
+	_FreeSyncCheckpointBlock(psSyncBlock);
+
+	/*	Drop our reference to the sync checkpoint context */
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+}
+
+static INLINE IMG_UINT32 _SyncCheckpointGetOffset(SYNC_CHECKPOINT *psSyncInt)
+{
+	IMG_UINT64 ui64Temp;
+
+	ui64Temp = (IMG_UINT64)psSyncInt->uiAllocatedAddr -
+	           (IMG_UINT64)psSyncInt->psSyncCheckpointBlock->uiSpanBase;
+	PVR_ASSERT(ui64Temp<IMG_UINT32_MAX);
+	return (IMG_UINT32)ui64Temp;
+}
+
+/* Used by SyncCheckpointContextCreate() below */
+static INLINE IMG_UINT32 _Log2(IMG_UINT32 ui32Align)
+{
+	PVR_ASSERT(IsPower2(ui32Align));
+	return ExactLog2(ui32Align);
+}
+
+/*
+	External interfaces
+ */
+
+PVRSRV_ERROR
+SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns)
+{
+	PVR_ASSERT(psSyncCheckpointPfns != NULL);
+
+	if (g_psSyncCheckpointPfnStruct)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but already initialised", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	g_psSyncCheckpointPfnStruct = psSyncCheckpointPfns;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+SyncCheckpointResolveFence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PVRSRV_FENCE hFence, IMG_UINT32 *pui32NumSyncCheckpoints,
+                           PSYNC_CHECKPOINT **papsSyncCheckpoints,
+                           IMG_UINT64 *pui64FenceUID,
+                           PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+#if defined(PDUMP)
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	_PSYNC_CHECKPOINT_CONTEXT_CTL	psContextCtl;
+#endif
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceResolve))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_pfnFenceResolve is NULL");
+		return eError;
+	}
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	psContextCtl = psSyncCheckpointContext->psContextCtl;
+
+	PVR_DPF((PVR_DBG_WARNING,
+	        "%s: Checkpoint %p DeviceID = [%u/%d]\n", __func__,
+	        psSyncCheckpointContext,
+	        psContextCtl->psDeviceNode->sDevId.ui32InternalID,
+	        psContextCtl->psDeviceNode->sDevId.i32KernelDeviceID));
+#endif
+
+	if (papsSyncCheckpoints)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceResolve(
+		                           psSyncCheckpointContext,
+		                           hFence,
+		                           pui32NumSyncCheckpoints,
+		                           papsSyncCheckpoints,
+		                           pui64FenceUID);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceResolve");
+
+#if defined(PDUMP)
+	if (*papsSyncCheckpoints)
+	{
+		for (i = 0; i < *pui32NumSyncCheckpoints; i++)
+		{
+			psSyncCheckpoint = (SYNC_CHECKPOINT *)(*papsSyncCheckpoints)[i];
+			psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	if (*pui32NumSyncCheckpoints > MAX_SYNC_CHECKPOINTS_PER_FENCE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: g_psSyncCheckpointPfnStruct->pfnFenceResolve() returned too many checkpoints (%u > MAX_SYNC_CHECKPOINTS_PER_FENCE=%u)",
+				__func__, *pui32NumSyncCheckpoints, MAX_SYNC_CHECKPOINTS_PER_FENCE));
+
+		/* Free resources after error */
+		if (*papsSyncCheckpoints)
+		{
+			for (i = 0; i < *pui32NumSyncCheckpoints; i++)
+			{
+				SyncCheckpointDropRef((*papsSyncCheckpoints)[i]);
+			}
+
+			SyncCheckpointFreeCheckpointListMem(*papsSyncCheckpoints);
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	{
+		IMG_UINT32 ii;
+
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: g_psSyncCheckpointPfnStruct->pfnFenceResolve() for fence %d returned the following %d checkpoints:",
+				__func__,
+				hFence,
+				*pui32NumSyncCheckpoints));
+
+		for (ii=0; ii<*pui32NumSyncCheckpoints; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(*papsSyncCheckpoints +  ii);
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s:   *papsSyncCheckpoints[%d]:<%p>",
+					__func__,
+					ii,
+					(void*)psNextCheckpoint));
+		}
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointResolveExportFence(PVRSRV_FENCE hExportFence,
+                                 PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                 PSYNC_CHECKPOINT *ppsSyncCheckpoint,
+                                 PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnExportFenceResolve))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+		        __func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "pfnExportFenceResolve is NULL");
+		return eError;
+	}
+
+	if (ppsSyncCheckpoint)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnExportFenceResolve(
+		                           hExportFence,
+		                           psSyncCheckpointContext,
+		                           ppsSyncCheckpoint);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnExportFenceResolve");
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s: g_psSyncCheckpointPfnStruct->pfnExportFenceResolve() for fence %d returned the following checkpoint:",
+		        __func__,
+		        hExportFence));
+
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s:   psSyncCheckpoint:<%p>",
+		         __func__,
+		         (void*)*ppsSyncCheckpoint));
+	}
+#endif
+
+#if defined(PDUMP)
+	if (*ppsSyncCheckpoint)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT *)(*ppsSyncCheckpoint);
+		psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointCreateFence(PVRSRV_DEVICE_NODE *psDevNode,
+                          const IMG_CHAR *pszFenceName,
+                          PVRSRV_TIMELINE hTimeline,
+                          PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                          PVRSRV_FENCE *phNewFence,
+                          IMG_UINT64 *puiUpdateFenceUID,
+                          void **ppvFenceFinaliseData,
+                          PSYNC_CHECKPOINT *psNewSyncCheckpoint,
+                          void **ppvTimelineUpdateSyncPrim,
+                          IMG_UINT32 *pui32TimelineUpdateValue,
+                          PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceCreate))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceCreate is NULL");
+	}
+	else
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceCreate(
+		                          psDevNode,
+		                          pszFenceName,
+		                          hTimeline,
+		                          psSyncCheckpointContext,
+		                          phNewFence,
+		                          puiUpdateFenceUID,
+		                          ppvFenceFinaliseData,
+		                          psNewSyncCheckpoint,
+		                          ppvTimelineUpdateSyncPrim,
+		                          pui32TimelineUpdateValue);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s failed to create new fence<%p> for timeline<%d> using "
+					"sync checkpoint context<%p>, psNewSyncCheckpoint=<%p>, eError=%s",
+					__func__,
+					(void*)phNewFence,
+					hTimeline,
+					(void*)psSyncCheckpointContext,
+					(void*)psNewSyncCheckpoint,
+					PVRSRVGetErrorString(eError)));
+		}
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s [%u/%d] created new fence<%d> for timeline<%d> "
+					"using sync checkpoint context<%p>, "
+					"new sync_checkpoint=<%p>",
+					__func__,
+					psDevNode->sDevId.ui32InternalID,
+					psDevNode->sDevId.i32KernelDeviceID,
+					*phNewFence,
+					hTimeline,
+					(void*)psSyncCheckpointContext,
+					(void*)*psNewSyncCheckpoint));
+		}
+#endif
+
+#if defined(PDUMP)
+		if (eError == PVRSRV_OK)
+		{
+			SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT*)(*psNewSyncCheckpoint);
+			if (psSyncCheckpoint)
+			{
+				psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+			}
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointRollbackFenceData(PVRSRV_FENCE hFence, void *pvFinaliseData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceDataRollback)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceDataRollback is NULL");
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: called to rollback fence data <%p>",
+				__func__,
+				pvFinaliseData));
+#endif
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceDataRollback(
+		            hFence, pvFinaliseData);
+		PVR_LOG_IF_ERROR(eError,
+		                 "g_psSyncCheckpointPfnStruct->pfnFenceDataRollback returned error");
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointFinaliseFence(PPVRSRV_DEVICE_NODE psDevNode,
+                            PVRSRV_FENCE hFence,
+                            void *pvFinaliseData,
+                            PSYNC_CHECKPOINT psSyncCheckpoint,
+                            const IMG_CHAR *pszName)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceFinalise)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Warning (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED) (this is permitted)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: called to finalise fence <%d>",
+				__func__,
+				hFence));
+#endif
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceFinalise(hFence, pvFinaliseData);
+		PVR_LOG_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceFinalise returned error");
+
+		RGXSRV_HWPERF_ALLOC_FENCE(psDevNode, OSGetCurrentClientProcessIDKM(), hFence,
+		                          SyncCheckpointGetFirmwareAddr(psSyncCheckpoint),
+		                          pszName, OSStringLength(pszName));
+	}
+	return eError;
+}
+
+void
+SyncCheckpointFreeCheckpointListMem(void *pvCheckpointListMem)
+{
+	if (g_psSyncCheckpointPfnStruct->pfnFreeCheckpointListMem)
+	{
+		g_psSyncCheckpointPfnStruct->pfnFreeCheckpointListMem(pvCheckpointListMem);
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointNoHWUpdateTimelines(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines is NULL");
+	}
+	else
+	{
+		g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines(pvPrivateData);
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointNoHWSignalExportFence(PVRSRV_FENCE iExportFenceToSignal)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence is NULL");
+	}
+	else
+	{
+		g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence(iExportFenceToSignal);
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointDumpInfoOnStalledUFOs(IMG_UINT32 ui32NumUFOs, IMG_UINT32 *pui32Vaddrs, IMG_UINT32 *pui32NumSyncOwnedUFOs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_FALSE((pui32NumSyncOwnedUFOs != NULL), "pui32NumSyncOwnedUFOs invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs)
+	{
+		*pui32NumSyncOwnedUFOs = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs is NULL");
+	}
+	else
+	{
+		*pui32NumSyncOwnedUFOs = g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs(ui32NumUFOs, pui32Vaddrs);
+		PVR_LOG(("%d sync checkpoint%s owned by %s in stalled context",
+		         *pui32NumSyncOwnedUFOs, *pui32NumSyncOwnedUFOs==1 ? "" : "s",
+		         g_psSyncCheckpointPfnStruct->pszImplName));
+	}
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocSyncCheckpoint(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                     SYNC_CHECKPOINT **ppsSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psNewSyncCheckpoint = NULL;
+	PVRSRV_ERROR eError;
+
+	/* Allocate sync checkpoint */
+	psNewSyncCheckpoint = OSAllocMem(sizeof(*psNewSyncCheckpoint));
+	PVR_LOG_RETURN_IF_NOMEM(psNewSyncCheckpoint, "OSAllocMem"); /* Sets OOM error code */
+
+	eError = RA_Alloc(psContext->psSubAllocRA,
+                      sizeof(*psNewSyncCheckpoint->psSyncCheckpointFwObj),
+                      RA_NO_IMPORT_MULTIPLIER,
+                      0,
+                      sizeof(IMG_UINT32),
+                      NULL,
+                      &psNewSyncCheckpoint->uiAllocatedAddr,
+                      NULL,
+                      (RA_PERISPAN_HANDLE *) &psNewSyncCheckpoint->psSyncCheckpointBlock);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RA_Alloc", fail_ra_alloc);
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+	        "%s CALLED RA_Alloc(), psSubAllocRA=<%p>, ui32SpanAddr=0x%llx",
+	        __func__,
+	        (void*)psContext->psSubAllocRA,
+	        psNewSyncCheckpoint->uiAllocatedAddr));
+#endif
+	psNewSyncCheckpoint->psSyncCheckpointFwObj =
+	        (volatile SYNC_CHECKPOINT_FW_OBJ*)(void *)(psNewSyncCheckpoint->psSyncCheckpointBlock->pui32LinAddr +
+	                (_SyncCheckpointGetOffset(psNewSyncCheckpoint)/sizeof(IMG_UINT32)));
+	psNewSyncCheckpoint->ui32FWAddr = psNewSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+	                                  _SyncCheckpointGetOffset(psNewSyncCheckpoint) + 1;
+	OSAtomicIncrement(&psNewSyncCheckpoint->psSyncCheckpointBlock->psContext->hCheckpointCount);
+
+#if defined(DEBUG)
+	psNewSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_USE;
+#endif
+	psNewSyncCheckpoint->sListNode.psPrevNode = NULL;
+	psNewSyncCheckpoint->sListNode.psNextNode = NULL;
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s called to allocate new sync checkpoint<%p> for context<%p>",
+	         __func__, (void*)psNewSyncCheckpoint, (void*)psContext));
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s                    psSyncCheckpointFwObj<%p>",
+	         __func__, (void*)psNewSyncCheckpoint->psSyncCheckpointFwObj));
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s                    psSyncCheckpoint FwAddr=0x%x",
+	         __func__, SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psNewSyncCheckpoint)));
+#endif
+	*ppsSyncCheckpoint = psNewSyncCheckpoint;
+	return PVRSRV_OK;
+
+fail_ra_alloc:
+	OSFreeMem(psNewSyncCheckpoint);
+	return eError;
+}
+
+/* Poisons and frees the checkpoint
+ * Decrements context refcount. */
+static void _FreeSyncCheckpoint(SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+
+	psSyncCheckpoint->sCheckpointUFOAddr.ui32Addr = 0;
+	psSyncCheckpoint->psSyncCheckpointFwObj = NULL;
+#if defined(DEBUG)
+	psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_FREED;
+#endif
+
+	RA_Free(psSyncCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA,
+	        psSyncCheckpoint->uiAllocatedAddr);
+	psSyncCheckpoint->psSyncCheckpointBlock = NULL;
+
+	OSFreeMem(psSyncCheckpoint);
+
+	OSAtomicDecrement(&psContext->hCheckpointCount);
+}
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static PVRSRV_ERROR
+_PrepopulateSyncCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                               IMG_UINT32 ui32InitPoolSize)
+{
+	IMG_UINT32 ui32SyncCheckpoint;
+	SYNC_CHECKPOINT *psNewSyncCheckpoint;
+	PVRSRV_ERROR eError;
+
+	/* Allocate sync checkpoints and place in the pool */
+	for (ui32SyncCheckpoint=0; ui32SyncCheckpoint<ui32InitPoolSize; ui32SyncCheckpoint++)
+	{
+		eError = _AllocSyncCheckpoint(psContext, &psNewSyncCheckpoint);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_AllocSyncCheckpoint");
+
+#if defined(PDUMP)
+		if (ui32SyncCheckpoint == 0)
+		{
+			psContext->psContextCtl->psSyncCheckpointPoolBlock = psNewSyncCheckpoint->psSyncCheckpointBlock;
+		}
+#endif
+
+		if (!_PutCheckpointInPool(psNewSyncCheckpoint))
+		{
+			_FreeSyncCheckpoint(psNewSyncCheckpoint);
+		}
+	}
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,"%s: Sync checkpoint pool [%d/%d]",
+	         __func__, psContext->psContextCtl->ui32SyncCheckpointPoolCount,
+	         psContext->psContextCtl->ui32SyncCheckpointPoolSize));
+#endif
+	return PVRSRV_OK;
+}
+#endif /* if (SYNC_CHECKPOINT_POOL_LIMIT > 0) */
+
+PVRSRV_ERROR
+SyncCheckpointContextCreate(PPVRSRV_DEVICE_NODE psDevNode,
+                            PSYNC_CHECKPOINT_CONTEXT *ppsSyncCheckpointContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = NULL;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *psContextCtl = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR azTempName[PVRSRV_SYNC_NAME_LENGTH] = {0};
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if !defined(PDUMP)
+	void *pvAppHintState = NULL;
+	const IMG_UINT32 ui32DefaultMaxPoolLog2Size = PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2;
+	const IMG_UINT32 ui32DefaultInitPoolLog2Size = PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2;
+#endif
+	IMG_UINT32 ui32MaxPoolLog2Size;
+	IMG_UINT32 ui32InitPoolLog2Size;
+	IMG_UINT32 ui32InitPoolSize = 0;
+#endif
+	IMG_UINT32 ui32MaxPoolSize = 0;
+
+	PVR_LOG_RETURN_IF_FALSE((ppsSyncCheckpointContext != NULL),
+	                  "ppsSyncCheckpointContext invalid",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if defined(PDUMP)
+	/*
+	   Pdumps use the maximum pool size to limit checkpoint address reuse and start with
+	   a minimum pool to avoid extra work allocating checkpoints which might not be used.
+	   Since during a pdump capture the pool is setup once during the pdump init phase
+	   and not emptied / de-initialised between captures we need to map at least one
+	   checkpoint at creation time, so that the pdump records the CPU mapping of the
+	   pool. Not doing this would lead to errors on subsequent pdump captures because
+	   only the first pdump would contain the mapping.
+	*/
+	ui32MaxPoolLog2Size = 10;
+	ui32InitPoolLog2Size = 0;
+	ui32InitPoolSize = 1;
+#else
+	/* Read AppHints to determine the size of the sync checkpoint pool, if specified */
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, SyncCheckpointPoolMaxLog2,
+	                     &ui32DefaultMaxPoolLog2Size, &ui32MaxPoolLog2Size);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, SyncCheckpointPoolInitLog2,
+	                     &ui32DefaultInitPoolLog2Size, &ui32InitPoolLog2Size);
+	OSFreeAppHintState(pvAppHintState);
+#endif
+
+	if (ui32MaxPoolLog2Size > 0)
+	{
+		ui32MaxPoolSize = 1 << ui32MaxPoolLog2Size;
+		if (ui32MaxPoolSize > SYNC_CHECKPOINT_POOL_LIMIT)
+		{
+			ui32MaxPoolSize = SYNC_CHECKPOINT_POOL_LIMIT;
+		}
+	}
+#endif
+
+	psContext = OSAllocMem(sizeof(*psContext));
+	PVR_LOG_GOTO_IF_NOMEM(psContext, eError, fail_alloc); /* Sets OOM error code */
+
+	/* psContextCtl includes allocation for the sync checkpoint pool) */
+	psContextCtl = OSAllocMem(sizeof(*psContextCtl) + IMG_FLEX_ARRAY_SIZE(sizeof(SYNC_CHECKPOINT*), ui32MaxPoolSize));
+	PVR_LOG_GOTO_IF_NOMEM(psContextCtl, eError, fail_alloc2); /* Sets OOM error code */
+
+	eError = OSSpinLockCreate(&psContextCtl->hDeferredCleanupListLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate:1", fail_create_deferred_cleanup_lock);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	eError = OSSpinLockCreate(&psContextCtl->hSyncCheckpointPoolLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate:2", fail_create_pool_lock);
+#endif
+
+	dllist_init(&psContextCtl->sDeferredCleanupListHead);
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	psContextCtl->ui32SyncCheckpointPoolCount = 0;
+	psContextCtl->ui32SyncCheckpointPoolWp = 0;
+	psContextCtl->ui32SyncCheckpointPoolRp = 0;
+#if defined(PDUMP)
+	psContextCtl->psSyncCheckpointPoolBlock = NULL;
+	psContextCtl->bAllocateFromCheckpointPool = IMG_FALSE;
+#endif
+	psContextCtl->ui8PoolStateFlags = SYNC_CHECKPOINT_POOL_VALID;
+#endif
+	psContextCtl->psDeviceNode = (SHARED_DEV_CONNECTION)psDevNode;
+
+	/*
+		Create the RA for sub-allocations of the sync checkpoints
+
+		Note:
+		The import size doesn't matter here as the server will pass
+		back the blocksize when it does the import which overrides
+		what we specify here.
+	 */
+	OSSNPrintf(azTempName, PVRSRV_SYNC_NAME_LENGTH,"Sync Prim RA-%p", psContext);
+	psContext->psSubAllocRA = RA_Create(azTempName,
+	                                    /* Params for imports */
+	                                    _Log2(sizeof(IMG_UINT32)),
+	                                    RA_LOCKCLASS_3,
+	                                    _SyncCheckpointBlockImport,
+	                                    _SyncCheckpointBlockUnimport,
+	                                    psContext,
+	                                    RA_POLICY_DEFAULT);
+	PVR_LOG_GOTO_IF_NOMEM(psContext->psSubAllocRA, eError, fail_suballoc);
+
+	/*
+		Create the span-management RA
+
+		The RA requires that we work with linear spans. For our use
+		here we don't require this behaviour as we're always working
+		within offsets of blocks (imports). However, we need to keep
+		the RA happy so we create the "span" management RA which
+		ensures that all are imports are added to the RA in a linear
+		fashion
+	 */
+	OSSNPrintf(azTempName, PVRSRV_SYNC_NAME_LENGTH, "Sync Prim span RA-%p", psContext);
+	psContext->psSpanRA = RA_Create(azTempName,
+	                                /* Params for imports */
+	                                0,
+	                                RA_LOCKCLASS_1,
+	                                NULL,
+	                                NULL,
+	                                NULL,
+	                                RA_POLICY_DEFAULT);
+	PVR_LOG_GOTO_IF_NOMEM(psContext->psSpanRA, eError, fail_span);
+
+	if (!RA_Add(psContext->psSpanRA, 0, MAX_SYNC_CHECKPOINT_MEM, 0, NULL))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		PVR_LOG_ERROR(eError, "SyncCheckpointContextCreate call to RA_Add(span) failed");
+		goto fail_span_add;
+	}
+
+	OSAtomicWrite(&psContext->hRefCount, 1);
+	OSAtomicWrite(&psContext->hCheckpointCount, 0);
+
+	psContext->psContextCtl = psContextCtl;
+
+	*ppsSyncCheckpointContext = (PSYNC_CHECKPOINT_CONTEXT)psContext;
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s: created psSyncCheckpointContext=<%p> (%d contexts exist)",
+			__func__,
+			(void*)*ppsSyncCheckpointContext,
+			++gui32NumSyncCheckpointContexts));
+#endif
+
+#if defined(PDUMP)
+	dllist_init(&psContext->sSyncCheckpointBlockListHead);
+
+	eError = OSLockCreate(&psContext->hSyncCheckpointBlockListLock);
+	PVR_GOTO_IF_ERROR(eError, fail_span_add);
+
+	OSLockAcquire(psDevNode->hSyncCheckpointContextListLock);
+	dllist_add_to_tail(&psDevNode->sSyncCheckpointContextListHead, &psContext->sListNode);
+	OSLockRelease(psDevNode->hSyncCheckpointContextListLock);
+#endif
+
+	psContextCtl->ui32CurrentInUseSyncCheckpoints = 0;
+	psContextCtl->ui32MaxInUseSyncCheckpoints = 0;
+	psContextCtl->ui32CurrentInUseMirroredSyncCPs = 0;
+	psContextCtl->ui32MaxInUseMirroredSyncCPs = 0;
+	eError = OSSpinLockCreate(&psContextCtl->hSyncCheckpointStatsLock);
+	PVR_GOTO_IF_ERROR(eError, fail_span_stat);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	/* Pre-populate the sync checkpoint pool, if specified */
+	psContextCtl->ui32SyncCheckpointPoolSize = ui32MaxPoolSize;
+
+	/* Ensure ui32MaxPoolSize is a POT and does not exceed SYNC_CHECKPOINT_POOL_LIMIT,
+	 * and ui32InitPoolSize does not exceed ui32MaxPoolSize.
+	 */
+	if (psContextCtl->ui32SyncCheckpointPoolSize > SYNC_CHECKPOINT_POOL_LIMIT)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: AppHint SyncCheckpointPoolMaxLog2(%d) would exceed "
+		                          "SYNC_CHECKPOINT_POOL_LIMIT(%d) - limiting to %d",
+		                          __func__, ui32MaxPoolLog2Size,
+		                          SYNC_CHECKPOINT_POOL_LIMIT, SYNC_CHECKPOINT_POOL_LIMIT));
+		psContextCtl->ui32SyncCheckpointPoolSize = SYNC_CHECKPOINT_POOL_LIMIT;
+	}
+
+	if (ui32InitPoolLog2Size > 0)
+	{
+		ui32InitPoolSize = 1 << ui32InitPoolLog2Size;
+	}
+	if (ui32InitPoolSize > psContextCtl->ui32SyncCheckpointPoolSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: AppHint SyncCheckpointPoolInitLog2(%d) would exceed "
+		                          "ui32SyncCheckpointPoolSize(%d) - limiting to %d",
+		                          __func__, ui32InitPoolLog2Size,
+		                          psContextCtl->ui32SyncCheckpointPoolSize,
+		                          psContextCtl->ui32SyncCheckpointPoolSize));
+		ui32InitPoolSize = psContextCtl->ui32SyncCheckpointPoolSize;
+	}
+
+	if (ui32InitPoolSize > 0)
+	{
+		eError = _PrepopulateSyncCheckpointPool(psContext, ui32InitPoolSize);
+		PVR_LOG_RETURN_IF_ERROR_VA(eError, "_PrepopulateSyncCheckpointPool(%d)", ui32InitPoolSize);
+	}
+#endif
+
+	return PVRSRV_OK;
+fail_span_stat:
+#if defined(PDUMP)
+	OSLockDestroy(psContext->hSyncCheckpointBlockListLock);
+	psContext->hSyncCheckpointBlockListLock = NULL;
+#endif
+fail_span_add:
+	RA_Delete(psContext->psSpanRA);
+fail_span:
+	RA_Delete(psContext->psSubAllocRA);
+fail_suballoc:
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	OSSpinLockDestroy(psContextCtl->hSyncCheckpointPoolLock);
+	psContextCtl->hSyncCheckpointPoolLock = NULL;
+fail_create_pool_lock:
+#endif
+	OSSpinLockDestroy(psContextCtl->hDeferredCleanupListLock);
+	psContextCtl->hDeferredCleanupListLock = NULL;
+fail_create_deferred_cleanup_lock:
+	OSFreeMem(psContextCtl);
+fail_alloc2:
+	OSFreeMem(psContext);
+fail_alloc:
+	return eError;
+}
+
+PVRSRV_ERROR SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_INT iRf = 0;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncCheckpointContext != NULL),
+	                  "psSyncCheckpointContext invalid",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevNode = (PVRSRV_DEVICE_NODE *)psContext->psContextCtl->psDeviceNode;
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s: destroying psSyncCheckpointContext=<%p> (now have %d contexts)",
+			__func__,
+			(void*)psSyncCheckpointContext,
+			--gui32NumSyncCheckpointContexts));
+#endif
+
+	_CheckDeferredCleanupList(psContext);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	if (psContext->psContextCtl->ui32SyncCheckpointPoolCount > 0)
+	{
+		IMG_UINT32 ui32NumFreedFromPool = _CleanCheckpointPool(psContext);
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s freed %d sync checkpoints that were still in the pool for context<%p>",
+				__func__,
+				ui32NumFreedFromPool,
+				(void*)psContext));
+#else
+		PVR_UNREFERENCED_PARAMETER(ui32NumFreedFromPool);
+#endif
+	}
+#endif
+
+	iRf = OSAtomicRead(&psContext->hCheckpointCount);
+
+	if (iRf != 0)
+	{
+		OS_SPINLOCK_FLAGS uiFlags;
+
+		/* Note, this is not a permanent error as the caller may retry later */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s <%p> attempted with active references (iRf=%d), "
+				"may be the result of a race",
+				__func__,
+				(void*)psContext,
+				iRf));
+
+		eError = PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT;
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		{
+			DLLIST_NODE *psNode, *psNext;
+
+			dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+			{
+				SYNC_CHECKPOINT *psSyncCheckpoint = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+				bool bDeferredFree = dllist_node_is_in_list(&psSyncCheckpoint->sDeferredFreeListNode);
+#if defined(DEBUG)
+				IMG_UINT32 ui32State = psSyncCheckpoint->psSyncCheckpointFwObj->ui32State;
+#endif
+
+				/* Line below avoids build error in release builds (where PVR_DPF is not defined) */
+				PVR_UNREFERENCED_PARAMETER(bDeferredFree);
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s syncCheckpoint<%p> ID=%d, %s, refs=%d, state=%s, fwaddr=%#08x, enqCount:%d, FWCount:%d %s",
+						__func__,
+						(void*)psSyncCheckpoint,
+						psSyncCheckpoint->ui32UID,
+						psSyncCheckpoint->azName,
+						OSAtomicRead(&psSyncCheckpoint->hRefCount),
+						ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED ?
+								"PVRSRV_SYNC_CHECKPOINT_SIGNALLED" :
+								ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE ?
+										"PVRSRV_SYNC_CHECKPOINT_ACTIVE" : "PVRSRV_SYNC_CHECKPOINT_ERRORED",
+						psSyncCheckpoint->ui32FWAddr,
+						OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+						psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+						bDeferredFree ? "(deferred free)" : ""));
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+				gui32NumSyncCheckpointContexts++;
+#endif
+			}
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+	else
+	{
+		SyncCheckpointContextUnref(psSyncCheckpointContext);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                    PVRSRV_TIMELINE hTimeline,
+                    PVRSRV_FENCE hFence,
+                    const IMG_CHAR *pszCheckpointName,
+                    PSYNC_CHECKPOINT *ppsSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psNewSyncCheckpoint = NULL;
+	_SYNC_CHECKPOINT_CONTEXT *psSyncContextInt = (_SYNC_CHECKPOINT_CONTEXT*)psSyncContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncContext != NULL), "psSyncContext invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((ppsSyncCheckpoint != NULL), "ppsSyncCheckpoint invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevNode = (PVRSRV_DEVICE_NODE *)psSyncContextInt->psContextCtl->psDeviceNode;
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+	PVR_DPF((PVR_DBG_WARNING, "%s Entry, Getting checkpoint from pool",
+			 __func__));
+#endif
+	psNewSyncCheckpoint = _GetCheckpointFromPool(psSyncContextInt);
+	if (!psNewSyncCheckpoint)
+	{
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s     checkpoint pool empty - will have to allocate",
+				 __func__));
+#endif
+	}
+#endif
+	/* If pool is empty (or not defined) alloc the new sync checkpoint */
+	if (!psNewSyncCheckpoint)
+	{
+		eError = _AllocSyncCheckpoint(psSyncContextInt, &psNewSyncCheckpoint);
+		PVR_LOG_GOTO_IF_NOMEM(psNewSyncCheckpoint, eError, fail_alloc); /* Sets OOM error code */
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0) && defined(PDUMP)
+		/* If the pool is not yet enabled and we allocated all the block, then the pool is ready to use. */
+		if (!psSyncContextInt->psContextCtl->bAllocateFromCheckpointPool)
+		{
+			RA_USAGE_STATS  sRAUsageStats;
+
+			RA_Get_Usage_Stats(psSyncContextInt->psSubAllocRA, &sRAUsageStats);
+
+			if (sRAUsageStats.ui64FreeArenaSize < sizeof(SYNC_CHECKPOINT_FW_OBJ))
+			{
+				psSyncContextInt->psContextCtl->bAllocateFromCheckpointPool = IMG_TRUE;
+			}
+		}
+#endif
+	}
+
+	OSSpinLockAcquire(psSyncContextInt->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+	if (++psSyncContextInt->psContextCtl->ui32CurrentInUseSyncCheckpoints > psSyncContextInt->psContextCtl->ui32MaxInUseSyncCheckpoints)
+	{
+		psSyncContextInt->psContextCtl->ui32MaxInUseSyncCheckpoints = psSyncContextInt->psContextCtl->ui32CurrentInUseSyncCheckpoints;
+	}
+	if ((hTimeline == SYNC_CHECKPOINT_MIRRORED_CHECKPOINT) &&
+	    ++psSyncContextInt->psContextCtl->ui32CurrentInUseMirroredSyncCPs > psSyncContextInt->psContextCtl->ui32MaxInUseMirroredSyncCPs)
+	{
+		psSyncContextInt->psContextCtl->ui32MaxInUseMirroredSyncCPs = psSyncContextInt->psContextCtl->ui32CurrentInUseMirroredSyncCPs;
+	}
+	OSSpinLockRelease(psSyncContextInt->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+	psNewSyncCheckpoint->hTimeline = hTimeline;
+	OSAtomicWrite(&psNewSyncCheckpoint->hRefCount, 1);
+	OSAtomicWrite(&psNewSyncCheckpoint->hEnqueuedCCBCount, 0);
+	psNewSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount = 0;
+	psNewSyncCheckpoint->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_ACTIVE;
+	psNewSyncCheckpoint->uiProcess = OSGetCurrentClientProcessIDKM();
+	OSCachedMemSet(&psNewSyncCheckpoint->sDeferredFreeListNode, 0, sizeof(psNewSyncCheckpoint->sDeferredFreeListNode));
+
+	if (pszCheckpointName)
+	{
+		/* Copy over the checkpoint name annotation */
+		OSStringSafeCopy(psNewSyncCheckpoint->azName, pszCheckpointName, PVRSRV_SYNC_NAME_LENGTH);
+	}
+	else
+	{
+		/* No sync checkpoint name annotation */
+		psNewSyncCheckpoint->azName[0] = '\0';
+	}
+
+	/* Store sync checkpoint FW address in PRGXFWIF_UFO_ADDR struct */
+	psNewSyncCheckpoint->sCheckpointUFOAddr.ui32Addr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psNewSyncCheckpoint);
+
+	/* Assign unique ID to this sync checkpoint */
+	psNewSyncCheckpoint->ui32UID = g_SyncCheckpointUID++;
+
+#if defined(PDUMP)
+	/* Flushing deferred fence signals to pdump */
+	MISRHandler_PdumpDeferredSyncSignalPoster(psDevNode);
+
+	_SyncCheckpointAllocPDump(psDevNode, psNewSyncCheckpoint);
+#endif
+
+	RGXSRV_HWPERF_ALLOC_SYNC_CP(psDevNode, psNewSyncCheckpoint->hTimeline,
+	                            OSGetCurrentClientProcessIDKM(),
+	                            hFence,
+	                            psNewSyncCheckpoint->ui32FWAddr,
+	                            psNewSyncCheckpoint->azName,
+	                            OSStringLength(psNewSyncCheckpoint->azName));
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		IMG_CHAR szChkptName[PVRSRV_SYNC_NAME_LENGTH];
+
+		if (pszCheckpointName)
+		{
+			/* Copy the checkpoint name annotation into a fixed-size array */
+			OSStringSafeCopy(szChkptName, pszCheckpointName, PVRSRV_SYNC_NAME_LENGTH);
+		}
+		else
+		{
+			/* No checkpoint name annotation */
+			szChkptName[0] = 0;
+		}
+		/* record this sync */
+		eError = _SyncCheckpointRecordAdd(&psNewSyncCheckpoint->hRecord,
+		                                 psNewSyncCheckpoint->psSyncCheckpointBlock,
+		                                 psNewSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr,
+		                                 _SyncCheckpointGetOffset(psNewSyncCheckpoint),
+		                                 psNewSyncCheckpoint->ui32UID,
+		                                 OSStringNLength(szChkptName, PVRSRV_SYNC_NAME_LENGTH),
+		                                 szChkptName, (PSYNC_CHECKPOINT)psNewSyncCheckpoint);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync checkpoint record \"%s\" (%s)",
+					__func__,
+					szChkptName,
+					PVRSRVGetErrorString(eError)));
+			psNewSyncCheckpoint->hRecord = NULL;
+			/* note the error but continue without affecting driver operation */
+		}
+	}
+
+	/* Add the sync checkpoint to the device list */
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_add_to_head(&psDevNode->sSyncCheckpointSyncsList,
+	                   &psNewSyncCheckpoint->sListNode);
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+	*ppsSyncCheckpoint = (PSYNC_CHECKPOINT)psNewSyncCheckpoint;
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Exit(Ok), psNewSyncCheckpoint->ui32UID=%d <%p>",
+			__func__,
+			psNewSyncCheckpoint->ui32UID,
+			(void*)psNewSyncCheckpoint));
+#endif
+	return PVRSRV_OK;
+
+fail_alloc:
+	return eError;
+}
+
+static void SyncCheckpointUnref(SYNC_CHECKPOINT *psSyncCheckpointInt)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psContext = psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+	psDevNode = (PVRSRV_DEVICE_NODE *)psContext->psContextCtl->psDeviceNode;
+
+	/*
+	 * Without this reference, the context may be destroyed as soon
+	 * as _FreeSyncCheckpoint is called, but the context is still
+	 * needed when _CheckDeferredCleanupList is called at the end
+	 * of this function.
+	 */
+	SyncCheckpointContextRef((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+#if defined(DEBUG)
+	PVR_ASSERT(psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE);
+#endif
+	if (!OSAtomicRead(&psSyncCheckpointInt->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncCheckpointUnref sync checkpoint already freed"));
+	}
+	else if (0 == OSAtomicDecrement(&psSyncCheckpointInt->hRefCount))
+	{
+		/* If the firmware has serviced all enqueued references to the sync checkpoint, free it */
+		if (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount ==
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)))
+		{
+			PVRSRV_ERROR eError;
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s No outstanding FW ops and hRef is zero, deleting SyncCheckpoint..",
+					__func__));
+#endif
+
+			if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+				&& psSyncCheckpointInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = _SyncCheckpointRecordRemove(psSyncCheckpointInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "_SyncCheckpointRecordRemove");
+			}
+
+			/* Remove the sync checkpoint from the global list */
+			OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+			dllist_remove_node(&psSyncCheckpointInt->sListNode);
+			OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+			RGXSRV_HWPERF_FREE(psDevNode, SYNC_CP, psSyncCheckpointInt->ui32FWAddr);
+
+			OSSpinLockAcquire(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+			psContext->psContextCtl->ui32CurrentInUseSyncCheckpoints--;
+			if (psSyncCheckpointInt->hTimeline == SYNC_CHECKPOINT_MIRRORED_CHECKPOINT)
+			{
+				psContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs--;
+			}
+			OSSpinLockRelease(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s attempting to return sync checkpoint to the pool",
+					__func__));
+#endif
+			if (!_PutCheckpointInPool(psSyncCheckpointInt))
+#endif
+			{
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s pool is full, so just free it",
+						__func__));
+#endif
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s CALLING RA_Free(psSyncCheckpoint(ID:%d)<%p>), psSubAllocRA=<%p>, ui32AllocatedAddr=0x%llx",
+						__func__,
+						psSyncCheckpointInt->ui32UID,
+						(void*)psSyncCheckpointInt,
+						(void*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext->psSubAllocRA,
+						psSyncCheckpointInt->uiAllocatedAddr));
+#endif
+				_FreeSyncCheckpoint(psSyncCheckpointInt);
+			}
+		}
+		else
+		{
+			OS_SPINLOCK_FLAGS uiFlags;
+#if ((ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s Outstanding FW ops hEnqueuedCCBCount=%d != FwObj->ui32FwRefCount=%d "
+					"- DEFERRING CLEANUP psSyncCheckpoint(ID:%d)<%p>",
+					__func__,
+					OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+					psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount,
+					psSyncCheckpointInt->ui32UID,
+					(void*)psSyncCheckpointInt));
+#endif
+			/* Add the sync checkpoint to the deferred free list */
+			OSSpinLockAcquire(psContext->psContextCtl->hDeferredCleanupListLock, uiFlags);
+			dllist_add_to_tail(&psContext->psContextCtl->sDeferredCleanupListHead,
+			                   &psSyncCheckpointInt->sDeferredFreeListNode);
+			OSSpinLockRelease(psContext->psContextCtl->hDeferredCleanupListLock, uiFlags);
+		}
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s psSyncCheckpoint(ID:%d)<%p>, hRefCount decremented to %d",
+				__func__,
+				psSyncCheckpointInt->ui32UID,
+				(void*)psSyncCheckpointInt,
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount))));
+#endif
+	}
+
+	/* See if any sync checkpoints in the deferred cleanup list can be freed */
+	_CheckDeferredCleanupList(psContext);
+
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+}
+
+void SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Entry,  psSyncCheckpoint(ID:%d)<%p>, hRefCount=%d, psSyncCheckpoint->ui32ValidationCheck=0x%x",
+			__func__,
+			psSyncCheckpointInt->ui32UID,
+			(void*)psSyncCheckpoint,
+			(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount)),
+			psSyncCheckpointInt->ui32ValidationCheck));
+#else
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Entry,  psSyncCheckpoint(ID:%d)<%p>, hRefCount=%d",
+			__func__,
+			psSyncCheckpointInt->ui32UID,
+			(void*)psSyncCheckpoint,
+			(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount))));
+#endif
+#endif
+	SyncCheckpointUnref(psSyncCheckpointInt);
+}
+
+void
+SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+			RGXSRVHWPerfSyncCheckpointUFOUpdate(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+
+#if defined(SUPPORT_RGX) && defined(PDUMP)
+			_SyncCheckpointUpdatePDump(psContext->psContextCtl->psDeviceNode, psSyncCheckpointInt, PVRSRV_SYNC_CHECKPOINT_SIGNALLED, ui32FenceSyncFlags);
+#endif
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s asked to set PVRSRV_SYNC_CHECKPOINT_SIGNALLED(%d) for (psSyncCheckpointInt->ui32UID=%d), "
+					 "when value is already %d",
+					 __func__,
+					 PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+					 psSyncCheckpointInt->ui32UID,
+					 ui32State));
+		}
+	}
+}
+
+void
+SyncCheckpointSignalNoHW(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+			RGXSRVHWPerfSyncCheckpointUFOUpdate(psDevInfo, psSyncCheckpointInt, PVRSRV_FENCE_FLAG_NONE);
+#endif
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+		}
+		else
+		{
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s asked to set PVRSRV_SYNC_CHECKPOINT_SIGNALLED(%d) for (psSyncCheckpointInt->ui32UID=%d), "
+					"when value is already %d",
+					__func__,
+					PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+					psSyncCheckpointInt->ui32UID,
+					ui32State));
+#endif
+		}
+	}
+}
+
+void
+SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+			if (!(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+			{
+				RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+				sSyncData.sUpdate.ui32FWAddr = SyncCheckpointGetFirmwareAddr(psSyncCheckpoint);
+				sSyncData.sUpdate.ui32OldValue = ui32State;
+				sSyncData.sUpdate.ui32NewValue = PVRSRV_SYNC_CHECKPOINT_ERRORED;
+
+				RGXSRV_HWPERF_UFO(psDevInfo, RGX_HWPERF_UFO_EV_UPDATE, &sSyncData,
+				                  (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+			}
+#endif
+
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_ERRORED;
+
+#if defined(SUPPORT_RGX) && defined(PDUMP)
+			_SyncCheckpointUpdatePDump(psContext->psContextCtl->psDeviceNode, psSyncCheckpointInt, PVRSRV_SYNC_CHECKPOINT_ERRORED, ui32FenceSyncFlags);
+#endif
+		}
+	}
+}
+
+IMG_BOOL SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+#if defined(SUPPORT_RGX)
+		PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+		RGXSRVHWPerfSyncCheckpointUFOIsSignalled(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+
+		switch (ui32State)
+		{
+			case PVRSRV_SYNC_CHECKPOINT_UNDEF:
+			case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			{
+				break;
+			}
+			case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			{
+				bRet = IMG_TRUE;
+				break;
+			}
+			default:
+			{
+
+				PVRSRV_DEVICE_NODE *psDeviceNode = psContext->psContextCtl->psDeviceNode;
+
+#if defined(SUPPORT_RGX)
+				RGXUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+#endif
+				if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: PCI error - state=0x%x, ID=%d, %s, fwaddr=%#08x",
+							 __func__, ui32State, psSyncCheckpoint->ui32UID,
+							 psSyncCheckpoint->azName, psSyncCheckpoint->ui32FWAddr));
+					bRet = IMG_TRUE;
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: unknown state (0x%x) - ID=%d, %s, fwaddr=%#08x",
+							 __func__, ui32State, psSyncCheckpoint->ui32UID,
+							 psSyncCheckpoint->azName, psSyncCheckpoint->ui32FWAddr));
+				}
+				break;
+			}
+
+		}
+
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s called for psSyncCheckpoint<%p>, returning %d",
+				__func__,
+				(void*)psSyncCheckpoint,
+				bRet));
+#endif
+	}
+	return bRet;
+}
+
+IMG_BOOL
+SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+#if defined(SUPPORT_RGX)
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+		PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+		RGXSRVHWPerfSyncCheckpointUFOIsSignalled(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+		bRet = (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_ERRORED);
+
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s called for psSyncCheckpoint<%p>, returning %d",
+				__func__,
+				(void*)psSyncCheckpoint,
+				bRet));
+#endif
+	}
+	return bRet;
+}
+
+const IMG_CHAR *
+SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid", "Null");
+
+	switch (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State)
+	{
+		case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			return "Signalled";
+		case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			return "Active";
+		case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			return "Errored";
+		case PVRSRV_SYNC_CHECKPOINT_UNDEF:
+			return "Undefined";
+		default:
+			return "Unknown";
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	PVRSRV_ERROR eRet = PVRSRV_OK;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncCheckpoint, "psSyncCheckpoint");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount),
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount)+1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	OSAtomicIncrement(&psSyncCheckpointInt->hRefCount);
+
+	return eRet;
+}
+
+PVRSRV_ERROR
+SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	PVRSRV_ERROR eRet = PVRSRV_OK;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncCheckpoint, "psSyncCheckpoint");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount),
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount)-1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	SyncCheckpointUnref(psSyncCheckpointInt);
+
+	return eRet;
+}
+
+void
+SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psSyncCheckpointInt != NULL, "psSyncCheckpoint");
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(psSyncCheckpointInt);
+#else /* !defined(NO_HARDWARE) */
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			(void*)psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+			OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)+1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	OSAtomicIncrement(&psSyncCheckpointInt->hEnqueuedCCBCount);
+#endif
+}
+
+PRGXFWIF_UFO_ADDR*
+SyncCheckpointGetRGXFWIFUFOAddr(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if defined(DEBUG)
+	if (psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE)
+#endif
+	{
+		return &psSyncCheckpointInt->sCheckpointUFOAddr;
+	}
+#if defined(DEBUG)
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called for psSyncCheckpoint<%p>, but ui32ValidationCheck=0x%x",
+				__func__,
+				(void*)psSyncCheckpointInt,
+				psSyncCheckpointInt->ui32ValidationCheck));
+	}
+#endif
+
+invalid_chkpt:
+	return NULL;
+}
+
+IMG_UINT32
+SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	IMG_UINT32 ui32Ret = 0;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if defined(DEBUG)
+	if (psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE)
+#endif
+	{
+		ui32Ret = psSyncCheckpointInt->ui32FWAddr;
+	}
+#if defined(DEBUG)
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called for psSyncCheckpoint<%p>, but ui32ValidationCheck=0x%x",
+				__func__,
+				(void*)psSyncCheckpointInt,
+				psSyncCheckpointInt->ui32ValidationCheck));
+	}
+#endif
+
+invalid_chkpt:
+	return ui32Ret;
+}
+
+IMG_UINT32
+SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	IMG_UINT32 ui32Ret = 0U;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s returning ID for sync checkpoint<%p>",
+			__func__,
+			(void*)psSyncCheckpointInt));
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s (validationCheck=0x%x)",
+			__func__,
+			psSyncCheckpointInt->ui32ValidationCheck));
+#endif
+#endif
+	ui32Ret = psSyncCheckpointInt->ui32UID;
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s (ui32UID=0x%x)",
+			__func__,
+			psSyncCheckpointInt->ui32UID));
+#endif
+
+invalid_chkpt:
+	return ui32Ret;
+}
+
+PVRSRV_TIMELINE
+SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+	return psSyncCheckpointInt->hTimeline;
+
+invalid_chkpt:
+	return 0;
+}
+
+
+IMG_UINT32
+SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount);
+}
+
+IMG_UINT32
+SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return OSAtomicRead(&psSyncCheckpointInt->hRefCount);
+}
+
+IMG_PID
+SyncCheckpointGetCreator(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return psSyncCheckpointInt->uiProcess;
+}
+
+IMG_UINT32 SyncCheckpointStateFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt;
+	PDLLIST_NODE psNode, psNext;
+	IMG_UINT32 ui32State = 0;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+	{
+		psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+		if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+		{
+			ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+			break;
+		}
+	}
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	return ui32State;
+}
+
+void SyncCheckpointErrorFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt;
+	PDLLIST_NODE psNode, psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s called to error UFO with ui32FWAddr=%d",
+			__func__,
+			ui32FwAddr));
+#endif
+
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+	{
+		psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+		if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+		{
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s calling SyncCheckpointError for sync checkpoint <%p>",
+					__func__,
+					(void*)psSyncCheckpointInt));
+#endif
+			/* Mark as errored */
+			SyncCheckpointError((PSYNC_CHECKPOINT)psSyncCheckpointInt, PVRSRV_FENCE_FLAG_NONE);
+			break;
+		}
+	}
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+}
+
+void SyncCheckpointRollbackFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr)
+{
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s called to rollback UFO with ui32FWAddr=0x%x",
+			__func__,
+			ui32FwAddr));
+#endif
+#if !defined(NO_HARDWARE)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpointInt = NULL;
+		PDLLIST_NODE psNode = NULL, psNext = NULL;
+		OS_SPINLOCK_FLAGS uiFlags;
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+		{
+			psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+			if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+			{
+#if ((ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)) || (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s called for psSyncCheckpointInt<%p> %d->%d",
+				        __func__,
+				        (void *) psSyncCheckpointInt,
+				        OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+				        OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount) - 1));
+#endif
+				OSAtomicDecrement(&psSyncCheckpointInt->hEnqueuedCCBCount);
+				break;
+			}
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+#endif
+}
+
+PVRSRV_ERROR SyncCheckpointRollbackExportFence(PVRSRV_FENCE hExportFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnExportFenceRollback))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+		        __func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "pfnExportFenceRollback is NULL");
+		return eError;
+	}
+
+	if (hExportFence != PVRSRV_NO_FENCE)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnExportFenceRollback(hExportFence);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnExportFenceRollback");
+
+	return eError;
+}
+
+static void _SyncCheckpointState(PDLLIST_NODE psNode,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	SYNC_CHECKPOINT *psSyncCheckpoint = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+
+	if (psSyncCheckpoint->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+	{
+		PVR_DUMPDEBUG_LOG("\t- ID = %d, FWAddr = 0x%08x, r%d:e%d:f%d: %s",
+		                   psSyncCheckpoint->ui32UID,
+		                   psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+		                   _SyncCheckpointGetOffset(psSyncCheckpoint),
+		                   OSAtomicRead(&psSyncCheckpoint->hRefCount),
+		                   OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+		                   psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+		                   psSyncCheckpoint->azName);
+	}
+}
+
+static void _SyncCheckpointDebugRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	DLLIST_NODE *psNode, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psDevNode->hSyncCheckpointContext == NULL) return;
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		PVR_DUMPDEBUG_LOG("------[ Active Sync Checkpoints ]------");
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+		PVR_DUMPDEBUG_LOG("(SyncCP Counts: InUse:%d Max:%d)",
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32CurrentInUseSyncCheckpoints,
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32MaxInUseSyncCheckpoints);
+		PVR_DUMPDEBUG_LOG("(SyncCP Mirrored Counts: InUse:%d Max:%d)",
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs,
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32MaxInUseMirroredSyncCPs);
+		OSSpinLockRelease(psDevNode->hSyncCheckpointContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+		{
+			_SyncCheckpointState(psNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointInit(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+#endif
+
+	eError = OSSpinLockCreate(&psDevNode->hSyncCheckpointListLock);
+	PVR_RETURN_IF_ERROR(eError);
+
+	dllist_init(&psDevNode->sSyncCheckpointSyncsList);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncCheckpointNotify,
+		                                          psDevNode,
+		                                          _SyncCheckpointDebugRequest,
+		                                          DEBUG_REQUEST_SYNCCHECKPOINT,
+		                                          (PVRSRV_DBGREQ_HANDLE)psDevNode);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		_SyncCheckpointRecordListInit(psDevNode);
+	}
+
+#if defined(PDUMP)
+	eError = OSSpinLockCreate(&psDevInfo->hSyncCheckpointSignalSpinLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+		goto e1;
+	}
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointSignalLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hSyncCheckpointSignalLock = NULL;
+		goto e2;
+	}
+
+	psDevNode->pui8DeferredSyncCPSignal = OSAllocMem(SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL
+	                                                      * sizeof(_SYNC_CHECKPOINT_DEFERRED_SIGNAL));
+	PVR_GOTO_IF_NOMEM(psDevNode->pui8DeferredSyncCPSignal, eError, e3);
+
+	psDevNode->ui16SyncCPWriteIdx = 0;
+	psDevNode->ui16SyncCPReadIdx = 0;
+
+	eError = OSInstallMISR(&psDevNode->pvSyncCPMISR,
+					MISRHandler_PdumpDeferredSyncSignalPoster,
+					psDevNode,
+					"RGX_PdumpDeferredSyncSignalPoster");
+	PVR_GOTO_IF_ERROR(eError, e4);
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointContextListLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hSyncCheckpointContextListLock = NULL;
+		goto e5;
+	}
+
+
+	dllist_init(&psDevNode->sSyncCheckpointContextListHead);
+
+	eError = PDumpRegisterTransitionCallbackFenceSync(psDevNode,
+								_SyncCheckpointPDumpTransition,
+								&psDevNode->hTransition);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hTransition = NULL;
+		goto e6;
+	}
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PDUMP)
+e6:
+	OSLockDestroy(psDevNode->hSyncCheckpointContextListLock);
+	psDevNode->hSyncCheckpointContextListLock = NULL;
+e5:
+	(void) OSUninstallMISR(psDevNode->pvSyncCPMISR);
+	psDevNode->pvSyncCPMISR = NULL;
+e4:
+	if (psDevNode->pui8DeferredSyncCPSignal)
+	{
+		OSFreeMem(psDevNode->pui8DeferredSyncCPSignal);
+		psDevNode->pui8DeferredSyncCPSignal = NULL;
+	}
+e3:
+	OSLockDestroy(psDevNode->hSyncCheckpointSignalLock);
+	psDevNode->hSyncCheckpointSignalLock = NULL;
+e2:
+	OSSpinLockDestroy(psDevInfo->hSyncCheckpointSignalSpinLock);
+	psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+e1:
+	_SyncCheckpointRecordListDeinit(psDevNode);
+#endif
+e0:
+	OSSpinLockDestroy(psDevNode->hSyncCheckpointListLock);
+	psDevNode->hSyncCheckpointListLock = NULL;
+
+	return eError;
+}
+
+void SyncCheckpointDeinit(PPVRSRV_DEVICE_NODE psDevNode)
+{
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+	PDumpUnregisterTransitionCallbackFenceSync(psDevNode->hTransition);
+	psDevNode->hTransition = NULL;
+
+	if (psDevNode->hSyncCheckpointContextListLock)
+	{
+		OSLockDestroy(psDevNode->hSyncCheckpointContextListLock);
+		psDevNode->hSyncCheckpointContextListLock = NULL;
+	}
+
+	if (psDevNode->pvSyncCPMISR)
+	{
+		(void) OSUninstallMISR(psDevNode->pvSyncCPMISR);
+		psDevNode->pvSyncCPMISR = NULL;
+	}
+
+	if (psDevNode->pui8DeferredSyncCPSignal)
+	{
+		OSFreeMem(psDevNode->pui8DeferredSyncCPSignal);
+		psDevNode->pui8DeferredSyncCPSignal = NULL;
+	}
+	if (psDevNode->hSyncCheckpointSignalLock)
+	{
+		OSLockDestroy(psDevNode->hSyncCheckpointSignalLock);
+		psDevNode->hSyncCheckpointSignalLock = NULL;
+	}
+	if (psDevInfo->hSyncCheckpointSignalSpinLock)
+	{
+		OSSpinLockDestroy(psDevInfo->hSyncCheckpointSignalSpinLock);
+		psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+	}
+#endif
+
+	PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncCheckpointNotify);
+	psDevNode->hSyncCheckpointNotify = NULL;
+	OSSpinLockDestroy(psDevNode->hSyncCheckpointListLock);
+	psDevNode->hSyncCheckpointListLock = NULL;
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		_SyncCheckpointRecordListDeinit(psDevNode);
+	}
+}
+
+void SyncCheckpointRecordLookup(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr,
+                                IMG_CHAR * pszSyncInfo, size_t len)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (!pszSyncInfo)
+	{
+		return;
+	}
+
+	pszSyncInfo[0] = '\0';
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+	{
+		struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+		if ((psSyncCheckpointRec->ui32FwBlockAddr + psSyncCheckpointRec->ui32SyncOffset + 1) == ui32FwAddr)
+		{
+			SYNC_CHECKPOINT_BLOCK *psSyncCheckpointBlock = psSyncCheckpointRec->psSyncCheckpointBlock;
+			if (psSyncCheckpointBlock && psSyncCheckpointBlock->pui32LinAddr)
+			{
+				void *pSyncCheckpointAddr = IMG_OFFSET_ADDR(psSyncCheckpointBlock->pui32LinAddr,
+													psSyncCheckpointRec->ui32SyncOffset);
+				OSSNPrintf(pszSyncInfo, len, "%s Checkpoint:%05u (%s)",
+				           (*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ?
+				                   "SIGNALLED" :
+				                   ((*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_ERRORED) ?
+				                           "ERRORED" : "ACTIVE"),
+				                           psSyncCheckpointRec->uiPID,
+				                           psSyncCheckpointRec->szClassName);
+			}
+			else
+			{
+				OSSNPrintf(pszSyncInfo, len, "Checkpoint:%05u (%s)",
+				           psSyncCheckpointRec->uiPID,
+				           psSyncCheckpointRec->szClassName);
+			}
+
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	if (!bFound && (psDevNode->ui32SyncCheckpointRecordCountHighWatermark == SYNC_CHECKPOINT_RECORD_LIMIT))
+	{
+		OSSNPrintf(pszSyncInfo, len, "(Record may be lost)");
+	}
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordAdd(
+			PSYNC_CHECKPOINT_RECORD_HANDLE * phRecord,
+			SYNC_CHECKPOINT_BLOCK *hSyncCheckpointBlock,
+			IMG_UINT32 ui32FwBlockAddr,
+			IMG_UINT32 ui32SyncOffset,
+			IMG_UINT32 ui32UID,
+			IMG_UINT32 ui32ClassNameSize,
+			const IMG_CHAR *pszClassName, PSYNC_CHECKPOINT pSyncCheckpt)
+{
+	struct SYNC_CHECKPOINT_RECORD * psSyncRec;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)hSyncCheckpointBlock->psContext;
+	PVRSRV_DEVICE_NODE *psDevNode = psContext->psContextCtl->psDeviceNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_INVALID_PARAM(phRecord);
+
+	*phRecord = NULL;
+
+	psSyncRec = OSAllocMem(sizeof(*psSyncRec));
+	PVR_LOG_GOTO_IF_NOMEM(psSyncRec, eError, fail_alloc); /* Sets OOM error code */
+
+	psSyncRec->psSyncCheckpointBlock = hSyncCheckpointBlock;
+	psSyncRec->ui32SyncOffset = ui32SyncOffset;
+	psSyncRec->ui32FwBlockAddr = ui32FwBlockAddr;
+	psSyncRec->ui64OSTime = OSClockns64();
+	psSyncRec->uiPID = OSGetCurrentProcessID();
+	psSyncRec->ui32UID = ui32UID;
+	psSyncRec->pSyncCheckpt = pSyncCheckpt;
+	if (pszClassName)
+	{
+		if (ui32ClassNameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			ui32ClassNameSize = PVRSRV_SYNC_NAME_LENGTH;
+		/* Copy over the class name annotation */
+		OSStringSafeCopy(psSyncRec->szClassName, pszClassName, ui32ClassNameSize);
+	}
+	else
+	{
+		/* No class name annotation */
+		psSyncRec->szClassName[0] = 0;
+	}
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	if (psDevNode->ui32SyncCheckpointRecordCount < SYNC_CHECKPOINT_RECORD_LIMIT)
+	{
+		dllist_add_to_head(&psDevNode->sSyncCheckpointRecordList, &psSyncRec->sNode);
+		psDevNode->ui32SyncCheckpointRecordCount++;
+
+		if (psDevNode->ui32SyncCheckpointRecordCount > psDevNode->ui32SyncCheckpointRecordCountHighWatermark)
+		{
+			psDevNode->ui32SyncCheckpointRecordCountHighWatermark = psDevNode->ui32SyncCheckpointRecordCount;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync checkpoint record \"%s\". %u records already exist.",
+				__func__,
+				pszClassName,
+				psDevNode->ui32SyncCheckpointRecordCount));
+		OSFreeMem(psSyncRec);
+		psSyncRec = NULL;
+		eError = PVRSRV_ERROR_TOOMANYBUFFERS;
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	*phRecord = (PSYNC_CHECKPOINT_RECORD_HANDLE)psSyncRec;
+
+fail_alloc:
+	return eError;
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordRemove(PSYNC_CHECKPOINT_RECORD_HANDLE hRecord)
+{
+	struct SYNC_CHECKPOINT_RECORD **ppFreedSync;
+	struct SYNC_CHECKPOINT_RECORD *pSync = (struct SYNC_CHECKPOINT_RECORD*)hRecord;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = pSync->psSyncCheckpointBlock->psContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRecord);
+
+	psDevNode = psContext->psContextCtl->psDeviceNode;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+
+	dllist_remove_node(&pSync->sNode);
+
+	if (psDevNode->uiSyncCheckpointRecordFreeIdx >= PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: psDevNode->uiSyncCheckpointRecordFreeIdx out of range",
+				__func__));
+		psDevNode->uiSyncCheckpointRecordFreeIdx = 0;
+	}
+	ppFreedSync = &psDevNode->apsSyncCheckpointRecordsFreed[psDevNode->uiSyncCheckpointRecordFreeIdx];
+	psDevNode->uiSyncCheckpointRecordFreeIdx =
+			(psDevNode->uiSyncCheckpointRecordFreeIdx + 1) % PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN;
+
+	if (*ppFreedSync)
+	{
+		OSFreeMem(*ppFreedSync);
+	}
+	pSync->psSyncCheckpointBlock = NULL;
+	pSync->ui64OSTime = OSClockns64();
+	*ppFreedSync = pSync;
+
+	psDevNode->ui32SyncCheckpointRecordCount--;
+
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	return PVRSRV_OK;
+}
+
+#define NS_IN_S (1000000000UL)
+static void _SyncCheckpointRecordPrint(struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec,
+                                       IMG_UINT64 ui64TimeNow,
+                                       DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                       void *pvDumpDebugFile)
+{
+	SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT *)psSyncCheckpointRec->pSyncCheckpt;
+	SYNC_CHECKPOINT_BLOCK *psSyncCheckpointBlock = psSyncCheckpointRec->psSyncCheckpointBlock;
+	IMG_UINT64 ui64DeltaS;
+	IMG_UINT32 ui32DeltaF;
+	IMG_UINT64 ui64Delta = ui64TimeNow - psSyncCheckpointRec->ui64OSTime;
+	ui64DeltaS = OSDivide64(ui64Delta, NS_IN_S, &ui32DeltaF);
+
+	if (psSyncCheckpointBlock && psSyncCheckpointBlock->pui32LinAddr)
+	{
+		void *pSyncCheckpointAddr;
+		pSyncCheckpointAddr = IMG_OFFSET_ADDR(psSyncCheckpointBlock->pui32LinAddr,
+											psSyncCheckpointRec->ui32SyncOffset);
+
+		PVR_DUMPDEBUG_LOG("\t%05u %05" IMG_UINT64_FMTSPEC ".%09u %010u "
+		                  "FWAddr=0x%08x (r%d:e%d:f%d) State=%s (%s)",
+		                  psSyncCheckpointRec->uiPID,
+		                  ui64DeltaS, ui32DeltaF, psSyncCheckpointRec->ui32UID,
+		                  (psSyncCheckpointRec->ui32FwBlockAddr+psSyncCheckpointRec->ui32SyncOffset),
+		                  OSAtomicRead(&psSyncCheckpoint->hRefCount),
+	                      OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+	                      psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+		                  (*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ?
+		                          "SIGNALLED" :
+		                          ((*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_ERRORED) ?
+		                                  "ERRORED" : "ACTIVE"),
+		                                  psSyncCheckpointRec->szClassName);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("\t%05u %05" IMG_UINT64_FMTSPEC ".%09u %010u FWAddr=0x%08x State=<null_ptr> (%s)",
+		                  psSyncCheckpointRec->uiPID,
+		                  ui64DeltaS, ui32DeltaF, psSyncCheckpointRec->ui32UID,
+		                  (psSyncCheckpointRec->ui32FwBlockAddr+psSyncCheckpointRec->ui32SyncOffset),
+		                  psSyncCheckpointRec->szClassName
+		);
+	}
+}
+
+static void _SyncCheckpointRecordRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                         IMG_UINT32 ui32VerbLevel,
+                                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                         void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	IMG_UINT64 ui64TimeNowS;
+	IMG_UINT32 ui32TimeNowF;
+	IMG_UINT64 ui64TimeNow = OSClockns64();
+	DLLIST_NODE *psNode, *psNext;
+
+	ui64TimeNowS = OSDivide64(ui64TimeNow, NS_IN_S, &ui32TimeNowF);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+
+		OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+
+		PVR_DUMPDEBUG_LOG("Dumping allocated sync checkpoints. Allocated: %u High watermark: %u (time ref %05" IMG_UINT64_FMTSPEC ".%09u)",
+		                  psDevNode->ui32SyncCheckpointRecordCount,
+		                  psDevNode->ui32SyncCheckpointRecordCountHighWatermark,
+		                  ui64TimeNowS,
+		                  ui32TimeNowF);
+		if (psDevNode->ui32SyncCheckpointRecordCountHighWatermark == SYNC_CHECKPOINT_RECORD_LIMIT)
+		{
+			PVR_DUMPDEBUG_LOG("Warning: Record limit (%u) was reached. Some sync checkpoints may not have been recorded in the debug information.",
+			                  SYNC_CHECKPOINT_RECORD_LIMIT);
+		}
+		PVR_DUMPDEBUG_LOG("\t%-5s %-15s %-10s %-17s %-14s (%s)",
+		                  "PID", "Time Delta (s)", "UID", "Address", "State", "Annotation");
+
+		dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+		{
+			struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec =
+					IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+			_SyncCheckpointRecordPrint(psSyncCheckpointRec, ui64TimeNow,
+			                           pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+
+		PVR_DUMPDEBUG_LOG("Dumping all recently freed sync checkpoints @ %05" IMG_UINT64_FMTSPEC ".%09u",
+		                  ui64TimeNowS,
+		                  ui32TimeNowF);
+		PVR_DUMPDEBUG_LOG("\t%-5s %-15s %-10s %-17s %-14s (%s)",
+		                  "PID", "Time Delta (s)", "UID", "Address", "State", "Annotation");
+		for (i = DECREMENT_WITH_WRAP(psDevNode->uiSyncCheckpointRecordFreeIdx, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN);
+				i != psDevNode->uiSyncCheckpointRecordFreeIdx;
+				i = DECREMENT_WITH_WRAP(i, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN))
+		{
+			if (psDevNode->apsSyncCheckpointRecordsFreed[i])
+			{
+				_SyncCheckpointRecordPrint(psDevNode->apsSyncCheckpointRecordsFreed[i],
+				                           ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+			else
+			{
+				break;
+			}
+		}
+		OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+	}
+}
+#undef NS_IN_S
+static PVRSRV_ERROR _SyncCheckpointRecordListInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointRecordLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lock_create);
+	dllist_init(&psDevNode->sSyncCheckpointRecordList);
+
+	psDevNode->ui32SyncCheckpointRecordCount = 0;
+	psDevNode->ui32SyncCheckpointRecordCountHighWatermark = 0;
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncCheckpointRecordNotify,
+	                                              psDevNode,
+	                                              _SyncCheckpointRecordRequest,
+	                                              DEBUG_REQUEST_SYNCCHECKPOINT,
+	                                              (PVRSRV_DBGREQ_HANDLE)psDevNode);
+	PVR_GOTO_IF_ERROR(eError, fail_dbg_register);
+
+	return PVRSRV_OK;
+
+fail_dbg_register:
+	OSLockDestroy(psDevNode->hSyncCheckpointRecordLock);
+fail_lock_create:
+	return eError;
+}
+
+static void _SyncCheckpointRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DLLIST_NODE *psNode, *psNext;
+	int i;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+	{
+		struct SYNC_CHECKPOINT_RECORD *pSyncCheckpointRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+
+		dllist_remove_node(psNode);
+		OSFreeMem(pSyncCheckpointRec);
+	}
+
+	for (i = 0; i < PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN; i++)
+	{
+		if (psDevNode->apsSyncCheckpointRecordsFreed[i])
+		{
+			OSFreeMem(psDevNode->apsSyncCheckpointRecordsFreed[i]);
+			psDevNode->apsSyncCheckpointRecordsFreed[i] = NULL;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	if (psDevNode->hSyncCheckpointRecordNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncCheckpointRecordNotify);
+	}
+	OSLockDestroy(psDevNode->hSyncCheckpointRecordLock);
+}
+
+#if defined(PDUMP)
+
+static PVRSRV_ERROR
+_SyncCheckpointAllocPDump(PVRSRV_DEVICE_NODE *psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Allocated Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+	                      psSyncCheckpoint->azName,
+	                      psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+	                      psSyncCheckpoint->sCheckpointUFOAddr.ui32Addr);
+
+	DevmemPDumpLoadMemValue32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+	                          _SyncCheckpointGetOffset(psSyncCheckpoint),
+	                          PVRSRV_SYNC_CHECKPOINT_ACTIVE,
+	                          PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointUpdatePDump(PPVRSRV_DEVICE_NODE psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint, IMG_UINT32 ui32Status, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bSleepAllowed = (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	 */
+	if (bSleepAllowed)
+	{
+		if (ui32Status == PVRSRV_SYNC_CHECKPOINT_ERRORED)
+		{
+			PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+					"Errored Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+					psSyncCheckpoint->azName,
+					psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+					(psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+					_SyncCheckpointGetOffset(psSyncCheckpoint)));
+		}
+		else
+		{
+			PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+					"Signalled Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+					psSyncCheckpoint->azName,
+					psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+					(psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+					_SyncCheckpointGetOffset(psSyncCheckpoint)));
+		}
+
+		DevmemPDumpLoadMemValue32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+		                          _SyncCheckpointGetOffset(psSyncCheckpoint),
+		                          ui32Status,
+		                          PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		_SYNC_CHECKPOINT_DEFERRED_SIGNAL *psSyncData;
+		OS_SPINLOCK_FLAGS uiFlags;
+		IMG_UINT16 ui16NewWriteIdx;
+
+		OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+
+		ui16NewWriteIdx = GET_CP_CB_NEXT_IDX(psDevNode->ui16SyncCPWriteIdx);
+		if (ui16NewWriteIdx == psDevNode->ui16SyncCPReadIdx)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR Deferred SyncCheckpointSignal CB is full)",
+				__func__));
+		}
+		else
+		{
+			psSyncData = GET_CP_CB_BASE(psDevNode->ui16SyncCPWriteIdx);
+			psSyncData->asSyncCheckpoint = *psSyncCheckpoint;
+			psSyncData->ui32Status = ui32Status;
+			psDevNode->ui16SyncCPWriteIdx = ui16NewWriteIdx;
+		}
+
+		OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+
+		OSScheduleMISR(psDevNode->pvSyncCPMISR);
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+MISRHandler_PdumpDeferredSyncSignalPoster(void *pvData)
+{
+	PPVRSRV_DEVICE_NODE psDevNode = (PPVRSRV_DEVICE_NODE) pvData;
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT16 ui16ReadIdx, ui16WriteIdx;
+	_SYNC_CHECKPOINT_DEFERRED_SIGNAL *psSyncData;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointSignalLock);
+
+	OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+	/* Snapshot current write and read offset of CB */
+	ui16WriteIdx = psDevNode->ui16SyncCPWriteIdx;
+	ui16ReadIdx = psDevNode->ui16SyncCPReadIdx;
+
+	OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+	/* CB is empty */
+	if (ui16WriteIdx == ui16ReadIdx)
+	{
+		OSLockRelease(psDevNode->hSyncCheckpointSignalLock);
+		return;
+	}
+	do
+	{
+		/* Read item in the CB and flush it to pdump */
+		psSyncData = GET_CP_CB_BASE(ui16ReadIdx);
+		_SyncCheckpointUpdatePDump(psDevNode, &psSyncData->asSyncCheckpoint, psSyncData->ui32Status, PVRSRV_FENCE_FLAG_NONE);
+		ui16ReadIdx = GET_CP_CB_NEXT_IDX(psDevNode->ui16SyncCPReadIdx);
+		/* Increment read offset in CB as one item is flushed to pdump */
+		OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+		psDevNode->ui16SyncCPReadIdx = ui16ReadIdx;
+		OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+		/* Call to this function will flush all the items present in CB
+		 * when this function is called i.e. use snapshot of WriteOffset
+		 * taken at the beginning in this function and iterate till Write != Read */
+	} while (ui16WriteIdx != ui16ReadIdx);
+
+	OSLockRelease(psDevNode->hSyncCheckpointSignalLock);
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR PVRSRVSyncCheckpointSignalledPDumpPolKM(PVRSRV_FENCE hFence)
+{
+	PVRSRV_ERROR eError;
+	PSYNC_CHECKPOINT *apsCheckpoints = NULL;
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+
+	if (hFence != PVRSRV_NO_FENCE)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnSyncFenceGetCheckpoints(hFence, &uiNumCheckpoints, &apsCheckpoints);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_pfnFenceGetCheckpoints");
+
+	if (uiNumCheckpoints)
+	{
+		/* Flushing deferred fence signals to pdump */
+		psSyncCheckpoint = (SYNC_CHECKPOINT *)apsCheckpoints[0];
+		psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+		MISRHandler_PdumpDeferredSyncSignalPoster(psContext->psContextCtl->psDeviceNode);
+	}
+
+	for (i=0; i < uiNumCheckpoints; i++)
+	{
+		psSyncCheckpoint = (SYNC_CHECKPOINT *)apsCheckpoints[i];
+		if (psSyncCheckpoint->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED)
+		{
+			psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+			PDUMPCOMMENTWITHFLAGS(psContext->psContextCtl->psDeviceNode,
+							psSyncCheckpoint->ui32PDumpFlags,
+							"Wait for Fence %s (ID:%d)",
+							psSyncCheckpoint->azName,
+							psSyncCheckpoint->ui32UID);
+
+			eError = DevmemPDumpDevmemPol32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+								_SyncCheckpointGetOffset(psSyncCheckpoint),
+								PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+								0xFFFFFFFF,
+								PDUMP_POLL_OPERATOR_EQUAL,
+								psSyncCheckpoint->ui32PDumpFlags);
+			PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+		}
+	}
+
+
+
+	/* Free the memory that was allocated for the sync checkpoint list returned */
+	if (apsCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsCheckpoints);
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* #if defined(PDUMP) */
+
+static PVRSRV_ERROR
+_SyncCheckpointPDumpTransition(void *pvData, PDUMP_TRANSITION_EVENT eEvent)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+	DLLIST_NODE *psNode, *psNext;
+	DLLIST_NODE *psNode1, *psNext1;
+	PPVRSRV_DEVICE_NODE psDevNode = (PPVRSRV_DEVICE_NODE) pvData;
+
+	if ((eEvent == PDUMP_TRANSITION_EVENT_RANGE_ENTERED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		OSLockAcquire(psDevNode->hSyncCheckpointContextListLock);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointContextListHead, psNode, psNext)
+		{
+			psContext = IMG_CONTAINER_OF(psNode, _SYNC_CHECKPOINT_CONTEXT, sListNode);
+
+			OSLockAcquire(psContext->hSyncCheckpointBlockListLock);
+			dllist_foreach_node(&psContext->sSyncCheckpointBlockListHead, psNode1, psNext1)
+			{
+				SYNC_CHECKPOINT_BLOCK *psSyncBlk =
+					IMG_CONTAINER_OF(psNode1, SYNC_CHECKPOINT_BLOCK, sListNode);
+				DevmemPDumpLoadMem(psSyncBlk->hMemDesc,
+							   0,
+							   psSyncBlk->ui32SyncBlockSize,
+							   PDUMP_FLAGS_CONTINUOUS);
+			}
+			OSLockRelease(psContext->hSyncCheckpointBlockListLock);
+		}
+		OSLockRelease(psDevNode->hSyncCheckpointContextListLock);
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+static void _CheckDeferredCleanupList(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE*)psCtxCtl->psDeviceNode;
+	DECLARE_DLLIST(sCleanupList);
+	DLLIST_NODE *psNode, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_ERROR eError;
+
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called", __func__));
+#endif
+
+	/* Check the deferred cleanup list and free any sync checkpoints we can */
+	OSSpinLockAcquire(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+
+	if (dllist_is_empty(&psCtxCtl->sDeferredCleanupListHead))
+	{
+		OSSpinLockRelease(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING, "%s: Defer free list is empty", __func__));
+#endif
+		/* if list is empty then we have nothing to do here */
+		return;
+	}
+
+	dllist_foreach_node(&psCtxCtl->sDeferredCleanupListHead, psNode, psNext)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpointInt =
+				IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sDeferredFreeListNode);
+
+		if (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount ==
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)))
+		{
+			if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+				&& psSyncCheckpointInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = _SyncCheckpointRecordRemove(psSyncCheckpointInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "_SyncCheckpointRecordRemove");
+			}
+
+			/* Move the sync checkpoint from the deferred free list to local list */
+			dllist_remove_node(&psSyncCheckpointInt->sDeferredFreeListNode);
+			/* It's not an ideal solution to traverse list of checkpoints-to-free
+			 * twice but it allows us to avoid holding the lock for too long */
+			dllist_add_to_tail(&sCleanupList, &psSyncCheckpointInt->sDeferredFreeListNode);
+		}
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s psSyncCheckpoint '%s'' (ID:%d)<%p>), "
+			        "still pending (enq=%d,FWRef=%d)", __func__,
+			        psSyncCheckpointInt->azName, psSyncCheckpointInt->ui32UID,
+			        (void*)psSyncCheckpointInt,
+			        (IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)),
+			        psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+		}
+#endif
+	}
+
+	OSSpinLockRelease(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+
+	dllist_foreach_node(&sCleanupList, psNode, psNext) {
+		SYNC_CHECKPOINT *psSyncCheckpointInt =
+				IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sDeferredFreeListNode);
+
+		/* Remove the sync checkpoint from the global list */
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_remove_node(&psSyncCheckpointInt->sListNode);
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+		RGXSRV_HWPERF_FREE(psDevNode, SYNC_CP, psSyncCheckpointInt->ui32FWAddr);
+
+		/* Unref the checkpoint in use */
+		OSSpinLockAcquire(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+		psContext->psContextCtl->ui32CurrentInUseSyncCheckpoints--;
+		if (psSyncCheckpointInt->hTimeline == SYNC_CHECKPOINT_MIRRORED_CHECKPOINT)
+		{
+			psContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs--;
+		}
+		OSSpinLockRelease(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s attempting to return sync(ID:%d),%p> to pool",
+		        __func__,
+		        psSyncCheckpointInt->ui32UID,
+		        (void *) psSyncCheckpointInt));
+#endif
+		if (!_PutCheckpointInPool(psSyncCheckpointInt))
+#endif
+		{
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING, "%s pool is full, so just free it",
+			        __func__));
+#endif
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+		}
+		else
+		{
+#endif
+			PVR_DPF((PVR_DBG_WARNING,
+			       "%s psSyncCheckpoint '%s'' (ID:%d)<%p>), still pending (enq=%d,FWRef=%d)",
+			        __func__,
+			        psSyncCheckpointInt->azName,
+			        psSyncCheckpointInt->ui32UID,
+			        (void*)psSyncCheckpointInt,
+			        (IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)),
+			        psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+			_FreeSyncCheckpoint(psSyncCheckpointInt);
+		}
+	}
+}
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static SYNC_CHECKPOINT *_GetCheckpointFromPool(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psCtxCtl->ui32SyncCheckpointPoolSize == 0)
+	{
+		goto pool_not_used;
+	}
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* Check if we can allocate from the pool */
+	if (CHECKPOINT_POOL_VALID(psCtxCtl) &&
+#if defined(PDUMP)
+	    psCtxCtl->bAllocateFromCheckpointPool &&
+#endif
+	    (psCtxCtl->ui32SyncCheckpointPoolCount > 0))
+	{
+		/* Get the next sync checkpoint from the pool */
+		psSyncCheckpoint = psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolRp];
+		psCtxCtl->ui32SyncCheckpointPoolRp =
+		        (psCtxCtl->ui32SyncCheckpointPoolRp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount--;
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+#if defined(DEBUG)
+		psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_USE;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s checkpoint(old ID:%d)<-POOL(%d/%d), psContext=<%p>, "
+		        "poolRp=%d, poolWp=%d",
+		        __func__,
+		        psSyncCheckpoint->ui32UID,
+		        psCtxCtl->ui32SyncCheckpointPoolCount,
+		         psCtxCtl->ui32SyncCheckpointPoolSize,
+		        (void *) psContext,
+		        psCtxCtl->ui32SyncCheckpointPoolRp,
+		        psCtxCtl->ui32SyncCheckpointPoolWp));
+#endif
+	}
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+pool_not_used:
+	return psSyncCheckpoint;
+}
+
+static IMG_BOOL _PutCheckpointInPool(SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	IMG_BOOL bReturnedToPool = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psCtxCtl->ui32SyncCheckpointPoolSize == 0)
+	{
+		return IMG_FALSE;
+	}
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* Check if pool has space */
+	if (CHECKPOINT_POOL_VALID(psCtxCtl) &&
+#if defined(PDUMP)
+			(psSyncCheckpoint->psSyncCheckpointBlock == psCtxCtl->psSyncCheckpointPoolBlock) &&
+#endif
+			!(CHECKPOINT_POOL_FULL(psCtxCtl)))
+	{
+		/* Put the sync checkpoint into the next write slot in the pool */
+		psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolWp] = psSyncCheckpoint;
+		psCtxCtl->ui32SyncCheckpointPoolWp =
+		        (psCtxCtl->ui32SyncCheckpointPoolWp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount++;
+		/* Update if the checkpoint that was just added filled up the pool */
+		if (psCtxCtl->ui32SyncCheckpointPoolWp == psCtxCtl->ui32SyncCheckpointPoolRp)
+		{
+			SET_CHECKPOINT_POOL_FULL(psCtxCtl);
+#if defined(PDUMP)
+			/* The pool is now fully allocated and ready to use. */
+			psCtxCtl->bAllocateFromCheckpointPool = IMG_TRUE;
+#endif
+		}
+		bReturnedToPool = IMG_TRUE;
+		psSyncCheckpoint->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_UNDEF;
+#if defined(DEBUG)
+		psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_POOL;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s checkpoint(ID:%d)->POOL(%d/%d), poolRp=%d, poolWp=%d",
+		        __func__,
+		        psSyncCheckpoint->ui32UID,
+		        psCtxCtl->ui32SyncCheckpointPoolCount,
+		        psCtxCtl->ui32SyncCheckpointPoolSize,
+		        psCtxCtl->ui32SyncCheckpointPoolRp,
+		        psCtxCtl->ui32SyncCheckpointPoolWp));
+#endif
+	}
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	return bReturnedToPool;
+}
+
+static IMG_UINT32 _CleanCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	SYNC_CHECKPOINT *psCheckpoint = NULL;
+	DECLARE_DLLIST(sCleanupList);
+	DLLIST_NODE *psThis, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT32 ui32ItemsFreed = 0, ui32NullScpCount = 0;
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* While the pool still contains sync checkpoints, free them */
+	while (CHECKPOINT_POOL_VALID(psCtxCtl) && psCtxCtl->ui32SyncCheckpointPoolCount > 0)
+	{
+		/* Get the sync checkpoint from the next read slot in the pool */
+		psCheckpoint = psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolRp];
+		psCtxCtl->ui32SyncCheckpointPoolRp =
+		        (psCtxCtl->ui32SyncCheckpointPoolRp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount--;
+
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+
+		if (psCheckpoint)
+		{
+			PVR_ASSERT(!dllist_node_is_in_list(&psCheckpoint->sListNode));
+			/* before checkpoints are added to the pool they are removed
+			 * from the list so it's safe to use sListNode here */
+			dllist_add_to_head(&sCleanupList, &psCheckpoint->sListNode);
+		}
+		else
+		{
+			ui32NullScpCount++;
+		}
+	}
+
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* go through the local list and free all of the sync checkpoints */
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s psContext=<%p>, ui8PoolStateFlags=0x%x, "
+	        "uiSyncCheckpointPoolCount=%d", __func__, (void *) psContext,
+	        psCtxCtl->ui8PoolStateFlags, psCtxCtl->ui32SyncCheckpointPoolCount));
+
+	if (ui32NullScpCount > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s pool contained %u NULL entries", __func__,
+		        ui32NullScpCount));
+	}
+#endif
+
+	dllist_foreach_node(&sCleanupList, psThis, psNext)
+	{
+		psCheckpoint = IMG_CONTAINER_OF(psThis, SYNC_CHECKPOINT, sListNode);
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+#if defined(DEBUG)
+		if (psCheckpoint->ui32ValidationCheck != SYNC_CHECKPOINT_PATTERN_IN_POOL)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s pool contains invalid entry "
+			        "(ui32ValidationCheck=0x%x)", __func__,
+			        psCheckpoint->ui32ValidationCheck));
+		}
+#endif
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint(ID:%d)",
+		        __func__, psCheckpoint->ui32UID));
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->ui32ValidationCheck=0x%x",
+		        __func__, psCheckpoint->ui32ValidationCheck));
+#endif
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->uiAllocatedAddr=0x%llx",
+		        __func__, psCheckpoint->uiAllocatedAddr));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock->psContext=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock->psContext));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA));
+
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s CALLING RA_Free(psSyncCheckpoint(ID:%d)<%p>), "
+		        "psSubAllocRA=<%p>, ui32SpanAddr=0x%llx",
+		        __func__,
+		        psCheckpoint->ui32UID,
+		        (void *) psCheckpoint,
+		        (void *) psCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA,
+		        psCheckpoint->uiAllocatedAddr));
+#endif
+
+		dllist_remove_node(psThis);
+
+		_FreeSyncCheckpoint(psCheckpoint);
+		ui32ItemsFreed++;
+	}
+
+	return ui32ItemsFreed;
+}
+#endif /* (SYNC_CHECKPOINT_POOL_LIMIT > 0) */
+
+IMG_BOOL SyncCheckpointCommonDeviceIDs(PSYNC_CHECKPOINT_CONTEXT psContext,
+									   IMG_HANDLE hDevRef)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDevRef;
+
+	if (unlikely((hDevRef == NULL) || (psContext == NULL)))
+	{
+		return IMG_TRUE;
+	}
+
+	return (psContext->psContextCtl->psDeviceNode->sDevId.i32KernelDeviceID ==
+	        psDevNode->sDevId.i32KernelDeviceID) ? IMG_TRUE : IMG_FALSE;
+}
+
+PVRSRV_ERROR SyncCheckpointGetCounters(PPVRSRV_DEVICE_NODE psDevNode,
+                                       IMG_UINT32 *puiInUse,
+                                       IMG_UINT32 *puiMax,
+                                       IMG_UINT32 *puiXDInUse,
+                                       IMG_UINT32 *puiXDMax)
+{
+	PSYNC_CHECKPOINT_CONTEXT psSyncContext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_LOG_RETURN_IF_FALSE((psDevNode != NULL), "psDevNode invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_RETURN_IF_FALSE((psDevNode->hSyncCheckpointContext != NULL), PVRSRV_OK);
+
+	psSyncContext = (PSYNC_CHECKPOINT_CONTEXT)psDevNode->hSyncCheckpointContext;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncContext != NULL), "psSyncContext invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiInUse != NULL), "puiInUse invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiMax != NULL), "puiMax invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiXDInUse != NULL), "puiXDInUse invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiXDMax != NULL), "puiXDMax invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSSpinLockAcquire(psSyncContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+	*puiInUse = psSyncContext->psContextCtl->ui32CurrentInUseSyncCheckpoints;
+	*puiMax = psSyncContext->psContextCtl->ui32MaxInUseSyncCheckpoints;
+	*puiXDInUse = psSyncContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs;
+	*puiXDMax = psSyncContext->psContextCtl->ui32MaxInUseMirroredSyncCPs;
+	OSSpinLockRelease(psSyncContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_fallback_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_fallback_server.c
new file mode 100644
index 000000000000..47abd65c83ab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_fallback_server.c
@@ -0,0 +1,2959 @@
+/*************************************************************************/ /*!
+@File           sync_fallback_server.c
+@Title          Fallback implementation of server fence sync interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    The server implementation of software fallback synchronisation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(DEBUG)
+#define PVR_DPF_FUNCTION_TRACE_ON 1
+#endif
+
+#include "pvr_debug.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_sync_km.h"
+#include "pvrsrv_sync_server.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint_internal.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "handle.h"
+#include "pvrsrv.h"
+#include "hash.h"
+#include "rgxhwperf.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+
+
+#include "ossecure_export.h"
+
+/* Refcounting debug.
+ * Define SYNC_FB_REF_DEBUG to print out a reference count log. */
+// #define SYNC_FB_REF_DEBUG 1
+
+#if defined(SYNC_FB_REF_DEBUG)
+#define PRINT_REF(type, opchr, pRef, ptr, name, info, num) \
+	PVR_LOG(("        %s REF(%c) -> %d - %6s: %-5u - %-30s (0x%p)", type, opchr, pRef, info, num, name, ptr))
+#else
+#define PRINT_REF(type, opchr, pRef, ptr, name, info, num)
+#endif
+
+#define REF_SET(type, pRef, val, ptr, name, info, num) OSAtomicWrite(pRef, val); PRINT_REF(type, '=', val, ptr, name, info, num)
+#define REF_INC(type, pRef,      ptr, name, info, num) OSAtomicIncrement(pRef); PRINT_REF(type, '+', OSAtomicRead(pRef), ptr, name, info, num)
+#define REF_DEC(type, pRef,      ptr, name, info, num) OSAtomicDecrement(pRef); PRINT_REF(type, '-', OSAtomicRead(pRef), ptr, name, info, num)
+
+/* Timelines */
+#define TL_REF_SET(pRef, val, ptr) REF_SET("TL", pRef, val, ptr, ptr->pszName, "UID", (IMG_UINT64) ptr->iUID)
+#define TL_REF_INC(pRef, ptr)      REF_INC("TL", pRef,      ptr, ptr->pszName, "UID", (IMG_UINT64) ptr->iUID)
+#define TL_REF_DEC(pRef, ptr)      REF_DEC("TL", pRef,      ptr, ptr->pszName, "UID", (IMG_UINT64) ptr->iUID)
+
+/* Fences */
+#define FENCE_REF_SET(pRef, val, ptr) REF_SET("FE", pRef, val, ptr, ptr->pszName, "#Syncs", ptr->uiNumSyncs)
+#define FENCE_REF_INC(pRef, ptr)      REF_INC("FE", pRef,      ptr, ptr->pszName, "#Syncs", ptr->uiNumSyncs)
+#define FENCE_REF_DEC(pRef, ptr)      REF_DEC("FE", pRef,      ptr, ptr->pszName, "#Syncs", ptr->uiNumSyncs)
+
+/* SyncPt */
+#define PT_REF_SET(pRef, val, ptr) REF_SET("PT", pRef, val, ptr, ptr->psTl->pszName, "SeqNum" ,ptr->uiSeqNum)
+#define PT_REF_INC(pRef, ptr)      REF_INC("PT", pRef,      ptr, ptr->psTl->pszName, "SeqNum" ,ptr->uiSeqNum)
+#define PT_REF_DEC(pRef, ptr)      REF_DEC("PT", pRef,      ptr, ptr->psTl->pszName, "SeqNum" ,ptr->uiSeqNum)
+
+
+/* Simple prints for error and warning */
+#define ERR(msg, ...) PVR_DPF((PVR_DBG_ERROR, \
+                          "%s: " msg, \
+                          __func__, \
+                          ##__VA_ARGS__));
+
+#define WRN(msg, ...) PVR_DPF((PVR_DBG_WARNING, \
+                          "%s: " msg, \
+                          __func__, \
+                          ##__VA_ARGS__));
+
+// #define SYNC_FB_DEBUG 1
+#if defined(SYNC_FB_DEBUG)
+#define DBG(...) PVR_LOG(__VA_ARGS__)
+#else
+#define DBG(...)
+#endif
+
+/* Functions for timelines */
+typedef IMG_BOOL (*PFN_SYNC_PT_HAS_SIGNALLED)(PVRSRV_SYNC_PT *psSyncPt);
+typedef void (*PFN_SYNC_FREE) (IMG_HANDLE hAttachedSync);
+
+
+/* The states a SyncPt can be in */
+typedef enum
+{
+	PVRSRV_SYNC_NOT_SIGNALLED,     /*!< sync pt has not yet signalled */
+	PVRSRV_SYNC_SIGNALLED,         /*!< sync pt has signalled */
+	PVRSRV_SYNC_ERRORED            /*!< sync pt has errored*/
+} PVRSRV_SYNC_STATE;
+
+typedef enum
+{
+	PVRSRV_SYNC_HANDLE_UNKNOWN,
+	PVRSRV_SYNC_HANDLE_PVR,
+	PVRSRV_SYNC_HANDLE_SW
+} PVRSRV_SYNC_HANDLE_TYPE;
+
+typedef struct _PVRSRV_SYNC_SIGNAL_CB_
+{
+	DLLIST_NODE sCallbackNode;
+	IMG_HANDLE  hAttachedSync;
+	IMG_HANDLE  hPrivData;
+	PVRSRV_ERROR (*pfnSignal)(IMG_HANDLE hAttachedSync,
+	                          PVRSRV_SYNC_STATE eState);
+	PFN_SYNC_FREE pfnSyncFree;
+
+} PVRSRV_SYNC_SIGNAL_CB;
+
+struct _PVRSRV_SYNC_PT_
+{
+	/* The timeline this sync pt is associated with */
+	PVRSRV_TIMELINE_SERVER	*psTl;
+	IMG_UINT32				uiSeqNum;
+	/* Refcount */
+	ATOMIC_T				iRef;
+	/* Timeline list. Take TL lock! */
+	DLLIST_NODE				sTlSyncList;
+	/* Timeline active list. Take TL lock! */
+	DLLIST_NODE				sTlSyncActiveList;
+
+	/* List of callbacks to signal attached syncs.
+	 *
+	 * THE FIRST ITEM OF THIS LIST DEFINES THE FLAVOUR OF THE SYNC PT
+	 * AND MUST BE CREATED TOGETHER WITH THE SYNC PT!
+	 * Usually this is done when creating a fence.
+	 * E.g. if PVR has been asked to create a fence we would
+	 * create a sync pt for it with an attached sync checkpoint.
+	 *
+	 * In case someone waits for this sync pt who is not able
+	 * to access the first item, a new foreign sync
+	 * needs to be attached that can be read by the waiter.
+	 * This might be the case if a second device is involved that cannot
+	 * access sync checkpoints of another device or a device that needs
+	 * to wait for a different sync type that it is not able to read
+	 * e.g. a SW sync */
+	DLLIST_NODE				sSignalCallbacks;
+	/* Can have a PVRSRV_SYNC_STATE */
+	ATOMIC_T				iStatus;
+	/* PID of the sync pt creator, used for cleanup-unblocking */
+	IMG_UINT32				uiPID;
+};
+
+/* Definition representing an attached SW sync pt.
+ * This is the counterpart to the SYNC_CHECKPOINTS for syncs that get
+ * signalled by the CPU. */
+typedef struct _PVRSRV_SYNC_PT_SW_
+{
+	 IMG_BOOL bSignalled;
+} PVRSRV_SYNC_PT_SW;
+
+/*! Possible states for a PVRSRV_FENCE */
+typedef enum
+{
+    PVRSRV_FENCE_NOT_SIGNALLED,             /*!< fence has not yet signalled (not all components have signalled) */
+    PVRSRV_FENCE_SIGNALLED                  /*!< fence has signalled (all components have signalled/errored) */
+} PVRSRV_FENCE_STATE;
+
+struct _PVRSRV_FENCE_SERVER_
+{
+	IMG_UINT32			uiNumSyncs;
+	PVRSRV_SYNC_PT		**apsFenceSyncList;
+	ATOMIC_T			iRef;
+	/* Only written to when waiter checks if fence is met */
+	ATOMIC_T			iStatus;
+	IMG_INT64			iUID;
+	IMG_CHAR			pszName[SYNC_FB_FENCE_MAX_LENGTH];
+	PVRSRV_DEVICE_NODE	*psDevNode;
+	DLLIST_NODE			sFenceListNode;
+};
+
+struct _PVRSRV_FENCE_EXPORT_
+{
+	PVRSRV_FENCE_SERVER *psFence;
+};
+
+typedef struct _PVRSRV_TIMELINE_OPS_
+{
+	/* Supposed to be called when someone queries the TL
+	 * to update its active generic syncs */
+	PFN_SYNC_PT_HAS_SIGNALLED pfnSyncPtHasSignalled;
+} PVRSRV_TIMELINE_OPS;
+
+struct _PVRSRV_TIMELINE_SERVER_
+{
+	/* Never take the fence lock after this one */
+	POS_LOCK			hTlLock;
+	/* Timeline list. Contains all sync pts of the timeline that
+	 * were not destroyed. Signalled or unsignalled. Take TL lock! */
+	DLLIST_NODE			sSyncList;
+	/* Timeline active list. Contains all sync pts of the timeline
+	 * that were not signalled yet.
+	 * Before removing node, check if it's still in list. Take TL lock! */
+	DLLIST_NODE			sSyncActiveList;
+	IMG_CHAR			pszName[SYNC_FB_TIMELINE_MAX_LENGTH];
+	ATOMIC_T			iRef;
+	PVRSRV_TIMELINE_OPS sTlOps;
+	DLLIST_NODE			sTlList;
+	/* This ID helps to order the sync pts in a fence when merging */
+	IMG_INT64			iUID;
+	/* The sequence number of the last sync pt created */
+	ATOMIC_T			iSeqNum;
+	/* The sequence number of the latest signalled sync pt */
+	ATOMIC_T			iLastSignalledSeqNum;
+	/* The PID of the process which created this timeline */
+	IMG_PID				uiPID;
+};
+
+typedef struct _SYNC_FB_CONTEXT_DEVICE_LIST_
+{
+	DLLIST_NODE sDeviceListNode;
+	IMG_HANDLE hDBGNotify;
+	PVRSRV_DEVICE_NODE *psDevice;
+} SYNC_FB_CONTEXT_DEVICE_LIST;
+
+typedef struct _SYNC_FB_CONTEXT_
+{
+	IMG_HANDLE hSyncEventObject;
+	IMG_HANDLE hCMDNotify;
+	DLLIST_NODE sDeviceList;
+	DLLIST_NODE sTlList;
+	POS_LOCK hFbContextLock;
+	DLLIST_NODE sFenceList; /* protected by hFbContextLock */
+	HASH_TABLE *sCheckpointHashTable; /* protected by hFbContextLock */
+	PFN_SYNC_CHECKPOINT_STRUCT sSyncCheckpointReg;
+} SYNC_FB_CONTEXT;
+
+/* GLOBALS */
+static SYNC_FB_CONTEXT gsSyncFbContext;
+
+/* Declarations */
+static void _SyncFbTimelineAcquire(PVRSRV_TIMELINE_SERVER *psTl);
+static void _SyncFbFenceAcquire(PVRSRV_FENCE_SERVER *psFence);
+static PVRSRV_ERROR _SyncFbSyncPtSignalAttached(PVRSRV_SYNC_PT *psSyncPt,
+                                        PVRSRV_SYNC_STATE eSignal);
+static PVRSRV_ERROR SyncFbFenceRollbackPVR(PVRSRV_FENCE iFence, void *pvFenceData);
+static PVRSRV_ERROR _SyncFbSyncPtSignalPVR(IMG_HANDLE hSync,
+                                           PVRSRV_SYNC_STATE eState);
+static PVRSRV_ERROR _SyncFbSyncPtSignalSW(IMG_HANDLE hSync,
+                                          PVRSRV_SYNC_STATE eState);
+static IMG_BOOL _SyncFbFenceSyncsHaveSignalled(PVRSRV_FENCE_SERVER *psFence);
+static IMG_BOOL _SyncFbSyncPtHasSignalled(PVRSRV_SYNC_PT *psSyncPt);
+static IMG_BOOL _SyncFbSyncPtHasSignalledPVR(PVRSRV_SYNC_PT *psSyncPt);
+static IMG_BOOL _SyncFbSyncPtHasSignalledSW(PVRSRV_SYNC_PT *psSyncPt);
+static IMG_BOOL _SyncFbFenceAddPt(PVRSRV_FENCE_SERVER *psFence,
+                                  IMG_UINT32 *i,
+                                  PVRSRV_SYNC_PT *psSyncPt);
+static PVRSRV_ERROR _SyncFbSWTimelineFenceCreate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                 PVRSRV_TIMELINE_SERVER *psTl,
+                                                 const IMG_CHAR *pszFenceName,
+                                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                                 IMG_UINT64 *pui64SyncPtIdx);
+static PVRSRV_ERROR _SyncSWTimelineAdvanceSigErr(PVRSRV_TIMELINE_SERVER *psTl,
+                                                 PVRSRV_SYNC_STATE eState,
+                                                 IMG_UINT64 *pui64SyncPtIdx);
+static void _SyncSWTimelineCheckForUnsignalledPts(PVRSRV_TIMELINE_SERVER *psTl);
+
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         GENERIC FUNCTIONS                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+
+/* Add a fence to the global fence list */
+static inline void _SyncFbFenceListAdd(PVRSRV_FENCE_SERVER *psFence)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_add_to_tail(&gsSyncFbContext.sFenceList, &psFence->sFenceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Remove a fence from the global fence list */
+static inline void _SyncFbFenceListDel(PVRSRV_FENCE_SERVER *psFence)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_remove_node(&psFence->sFenceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Add a timeline to the global timeline list */
+static inline void _SyncFbFTimelineListAdd(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_add_to_tail(&gsSyncFbContext.sTlList, &psTl->sTlList);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Remove a timeline from the global timeline list */
+static inline void _SyncFbTimelineListDel(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_remove_node(&psTl->sTlList);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Signal the sync event object to wake up waiters */
+static PVRSRV_ERROR _SyncFbSignalEO(void)
+{
+	PVRSRV_ERROR eError;
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(gsSyncFbContext.hSyncEventObject != NULL);
+
+	eError = OSEventObjectSignal(gsSyncFbContext.hSyncEventObject);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Retrieve the process handle base for the calling PID to look up sync objects */
+static PVRSRV_ERROR _SyncFbGetProcHandleBase(PVRSRV_HANDLE_BASE **ppsHandleBase)
+{
+	*ppsHandleBase = PVRSRVRetrieveProcessHandleBase();
+
+	if (*ppsHandleBase == NULL)
+	{
+		ERR("Failed to retrieve process handle base");
+		return PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Look up a handle in the process handle base of the calling PID */
+static PVRSRV_ERROR _SyncFbLookupProcHandle(IMG_HANDLE hHandle,
+                                            PVRSRV_HANDLE_TYPE eType,
+                                            IMG_BOOL bRefHandle,
+                                            void **ppvData,
+                                            PVRSRV_HANDLE_BASE **ppsBase)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHandleBase;
+
+	eError = _SyncFbGetProcHandleBase(&psHandleBase);
+
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	DBG(("%s: Handle Base: %p", __func__, psHandleBase));
+
+	eError = PVRSRVLookupHandle(psHandleBase,
+	                            ppvData,
+	                            hHandle,
+	                            eType,
+	                            bRefHandle);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*ppsBase = psHandleBase;
+
+	return PVRSRV_OK;
+
+e1:
+	return eError;
+}
+
+/* Destroy a handle in case a resource has not been registered with
+ * the resource manager */
+static PVRSRV_ERROR _SyncFbDestroyHandle(IMG_HANDLE hHandle,
+                                         PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHandleBase;
+
+	eError = _SyncFbGetProcHandleBase(&psHandleBase);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	eError = PVRSRVDestroyHandle(psHandleBase, hHandle, eType);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDestroyHandle", e1);
+
+	return PVRSRV_OK;
+
+e1:
+	return eError;
+}
+
+/* Currently unused */
+/*
+static PVRSRV_ERROR _SyncFbFindProcHandle(void *pvData,
+                                          PVRSRV_HANDLE_TYPE eType,
+                                          IMG_HANDLE *phHandle,
+                                          PVRSRV_HANDLE_BASE **ppsBase)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHandleBase;
+
+	PVR_DPF_ENTERED;
+
+	eError = _SyncFbGetProcHandleBase(&psHandleBase);
+
+	PVR_GOTO_IF_ERROR(eError, eExit);
+
+	eError = PVRSRVFindHandle(psHandleBase,
+	                          phHandle,
+	                          pvData,
+							  eType);
+	PVR_GOTO_IF_ERROR(eError, eExit);
+
+	*ppsBase = psHandleBase;
+
+	PVR_DPF_RETURN_OK;
+
+eExit:
+	PVR_DPF_RETURN_RC(eError);
+}
+*/
+
+/* Returns the type of a sync point determined by its registered
+ * signalling callback. Type can be e.g. a PVR sync point containing
+ * sync checkpoints or a software sync point*/
+static PVRSRV_SYNC_HANDLE_TYPE _SyncFbSyncPtHandleType(PVRSRV_SYNC_SIGNAL_CB *psCb)
+{
+	if (psCb == NULL)
+		return PVRSRV_SYNC_HANDLE_UNKNOWN;
+
+	if (psCb->pfnSignal == &_SyncFbSyncPtSignalPVR)
+		return PVRSRV_SYNC_HANDLE_PVR;
+
+	if (psCb->pfnSignal == &_SyncFbSyncPtSignalSW)
+		return PVRSRV_SYNC_HANDLE_SW;
+
+	return PVRSRV_SYNC_HANDLE_UNKNOWN;
+}
+
+static PVRSRV_SYNC_HANDLE_TYPE _SyncFbTimelineHandleType(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	if (psTl == NULL)
+		return PVRSRV_SYNC_HANDLE_UNKNOWN;
+
+	if (psTl->sTlOps.pfnSyncPtHasSignalled == &_SyncFbSyncPtHasSignalledPVR)
+		return PVRSRV_SYNC_HANDLE_PVR;
+
+	if (psTl->sTlOps.pfnSyncPtHasSignalled == &_SyncFbSyncPtHasSignalledSW)
+		return PVRSRV_SYNC_HANDLE_SW;
+
+	return PVRSRV_SYNC_HANDLE_UNKNOWN;
+}
+
+/* Print info about a sync point to the debug dump log */
+static void _SyncFbDebugRequestPrintSyncPt(PVRSRV_SYNC_PT *psSyncPt,
+                                           IMG_BOOL bPrintTl,
+                                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                           void *pvDumpDebugFile)
+{
+	PDLLIST_NODE psCBNode, psNextCBNode;
+	PVRSRV_SYNC_SIGNAL_CB *psCb;
+
+	if (bPrintTl)
+	{
+		PVR_DUMPDEBUG_LOG(" - SyncPt: SeqNum: %u, Refs: %d, Timeline: %-9s <%#"IMG_UINT64_FMTSPECx">, %-9s - <0x%p>",
+						  psSyncPt->uiSeqNum,
+						  OSAtomicRead(&psSyncPt->iRef),
+						  psSyncPt->psTl->pszName,
+						  psSyncPt->psTl->iUID,
+						  OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_SIGNALLED ? "Signalled" :
+							OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_ERRORED ? "Errored" : "Active",
+						  psSyncPt);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG(" - SyncPt: SeqNum: %u, Refs: %d, %-9s - <0x%p>",
+						  psSyncPt->uiSeqNum,
+						  OSAtomicRead(&psSyncPt->iRef),
+						  OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_SIGNALLED ? "Signalled" :
+							OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_ERRORED ? "Errored" : "Active",
+						  psSyncPt);
+	}
+
+	/* ... all attached syncs to that sync point*/
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks,
+	                    psCBNode,
+	                    psNextCBNode)
+	{
+		psCb = IMG_CONTAINER_OF(psCBNode,
+		                        PVRSRV_SYNC_SIGNAL_CB,
+		                        sCallbackNode);
+
+		switch (_SyncFbSyncPtHandleType(psCb))
+		{
+			case PVRSRV_SYNC_HANDLE_PVR:
+			{
+				PSYNC_CHECKPOINT pCP = psCb->hAttachedSync;
+				PVR_DUMPDEBUG_LOG("    - CbType: PVR-Checkpoint, ID: %u, FWAddr: %#08x, Enq: %d, Ref: %d, %-9s - <0x%p>",
+				                  SyncCheckpointGetId(pCP),
+				                  SyncCheckpointGetFirmwareAddr(pCP),
+				                  SyncCheckpointGetEnqueuedCount(pCP),
+				                  SyncCheckpointGetReferenceCount(pCP),
+				                  SyncCheckpointGetStateString(pCP),
+				                  pCP);
+				break;
+			}
+			case PVRSRV_SYNC_HANDLE_SW:
+			{
+				PVRSRV_SYNC_PT_SW *psSWPt = psCb->hAttachedSync;
+				PVR_DUMPDEBUG_LOG("    - CbType: SW-Syncpoint, %-9s - <0x%p>",
+				                  psSWPt->bSignalled ? "Signalled" : "Active",
+				                  psSWPt);
+				break;
+			}
+			case PVRSRV_SYNC_HANDLE_UNKNOWN:
+			default:
+				PVR_DUMPDEBUG_LOG("    - CbType: Unknown - <0x%p>",
+				                  psCb->hAttachedSync);
+		}
+	}
+}
+
+/* Function registered with the debug dump mechanism. Prints out all timelines
+ * with pending syncs. */
+static void _SyncFbDebugRequest(IMG_HANDLE hDebugRequestHandle,
+                                IMG_UINT32 ui32VerbLevel,
+                                DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+
+		PDLLIST_NODE psTlNode, psNextTlNode;
+		PVRSRV_TIMELINE_SERVER *psTl;
+
+		PDLLIST_NODE psFenceNode, psNextFenceNode;
+		PVRSRV_FENCE_SERVER *psFence;
+
+		PDLLIST_NODE psPtNode, psNextPtNode;
+		PVRSRV_SYNC_PT *psSyncPt;
+
+		OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+		PVR_DUMPDEBUG_LOG("------[ Fallback Fence Sync: timelines ]------");
+
+		/* Iterate over all timelines */
+		dllist_foreach_node(&gsSyncFbContext.sTlList, psTlNode, psNextTlNode)
+		{
+
+			psTl = IMG_CONTAINER_OF(psTlNode,
+			                        PVRSRV_TIMELINE_SERVER,
+			                        sTlList);
+
+			OSLockAcquire(psTl->hTlLock);
+
+			PVR_DUMPDEBUG_LOG("Timeline: %s, SeqNum: %d/%d - <%#"IMG_UINT64_FMTSPECx">",
+			                  psTl->pszName,
+			                  OSAtomicRead(&psTl->iLastSignalledSeqNum),
+			                  OSAtomicRead(&psTl->iSeqNum),
+			                  psTl->iUID);
+
+			/* ... all active sync points in the timeline */
+			dllist_foreach_node(&psTl->sSyncActiveList, psPtNode, psNextPtNode)
+			{
+
+				psSyncPt = IMG_CONTAINER_OF(psPtNode,
+				                            PVRSRV_SYNC_PT,
+				                            sTlSyncActiveList);
+
+				_SyncFbDebugRequestPrintSyncPt(psSyncPt,
+				                               IMG_FALSE,
+				                               pfnDumpDebugPrintf,
+				                               pvDumpDebugFile);
+
+			}
+			OSLockRelease(psTl->hTlLock);
+		}
+
+		PVR_DUMPDEBUG_LOG("------[ Fallback Fence Sync: fences ]------");
+
+		/* Iterate over all fences */
+		dllist_foreach_node(&gsSyncFbContext.sFenceList,
+							psFenceNode,
+							psNextFenceNode)
+		{
+			psFence = IMG_CONTAINER_OF(psFenceNode,
+			                           PVRSRV_FENCE_SERVER,
+			                           sFenceListNode);
+
+			PVR_DUMPDEBUG_LOG("Fence: %s, %-9s - <%#"IMG_UINT64_FMTSPECx">",
+			                  psFence->pszName,
+			                  _SyncFbFenceSyncsHaveSignalled(psFence) ?
+			                      "Signalled" : "Active",
+				               psFence->iUID);
+
+			/* ... all sync points in the fence */
+			for (i = 0; i < psFence->uiNumSyncs; i++)
+			{
+				_SyncFbDebugRequestPrintSyncPt(psFence->apsFenceSyncList[i],
+                                               IMG_TRUE,
+				                               pfnDumpDebugPrintf,
+				                               pvDumpDebugFile);
+			}
+		}
+
+		OSLockRelease(gsSyncFbContext.hFbContextLock);
+	}
+
+}
+
+/* Notify callback that is called as part of the RGX MISR e.g. after FW
+ * signalled the host that work completed. */
+static void _SyncFbTimelineUpdate_NotifyCMD(void *psSyncFbContext)
+{
+	PVRSRV_TIMELINE_SERVER *psTl;
+	PVRSRV_SYNC_PT *psSyncPt;
+	PDLLIST_NODE psTlList = &gsSyncFbContext.sTlList;
+	PDLLIST_NODE psCurrentTl, psNextTl;
+	PDLLIST_NODE psCurrentPt, psNextPt;
+	IMG_BOOL bSignalled = IMG_FALSE, bSignal = IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psSyncFbContext);
+
+	PVR_DPF_ENTERED;
+
+	/* Outer loop over all timelines */
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_foreach_node(psTlList, psCurrentTl, psNextTl)
+	{
+		psTl = IMG_CONTAINER_OF(psCurrentTl,
+		                        PVRSRV_TIMELINE_SERVER,
+		                        sTlList);
+
+		/* Inner loop over all SyncPts in the timeline.
+		 * Check & Update all active SyncPts */
+		OSLockAcquire(psTl->hTlLock);
+		dllist_foreach_node(&psTl->sSyncActiveList, psCurrentPt, psNextPt)
+		{
+			psSyncPt = IMG_CONTAINER_OF(psCurrentPt,
+			                            PVRSRV_SYNC_PT,
+			                            sTlSyncActiveList);
+
+			/* If the SyncPt has been signalled we have to
+			 * update all attached syncs */
+			bSignalled = psTl->sTlOps.pfnSyncPtHasSignalled(psSyncPt);
+			if (bSignalled)
+			{
+				/* Wake up waiters after releasing the locks */
+				bSignal = IMG_TRUE;
+
+				/* Remove the SyncPt from the active list of the timeline. */
+				dllist_remove_node(psCurrentPt);
+			}
+			else
+			{
+				/* No need to check further points on this timeline because
+				 * this sync pt will be signalled first */
+				break;
+			}
+
+		}/* End inner loop */
+		OSLockRelease(psTl->hTlLock);
+
+	} /* End outer loop */
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	if (bSignal)
+	{
+		PVR_LOG_IF_ERROR(_SyncFbSignalEO(), "_SyncFbSignalEO");
+	}
+
+	PVR_DPF_RETURN;
+}
+
+static IMG_UINT32
+_SyncCheckpointFWAddrHash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_UINT32 *p = (IMG_UINT32 *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = *p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+static IMG_BOOL
+_SyncCheckpointFWAddrCompare(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_UINT32 *p1 = (IMG_UINT32 *)pKey1;
+	IMG_UINT32 *p2 = (IMG_UINT32 *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+#if defined(PDUMP)
+static PVRSRV_ERROR SyncFbFenceGetCheckpoints(PVRSRV_FENCE hFence, IMG_UINT32 *puiNumCheckpoints,
+		                                PSYNC_CHECKPOINT **papsCheckpoints)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psFence;
+	PVRSRV_HANDLE_BASE *psHBase;
+	PSYNC_CHECKPOINT *apsCheckpoints;
+	PSYNC_CHECKPOINT psCheckpoint;
+	PVRSRV_SYNC_SIGNAL_CB *psSyncCB;
+	PVRSRV_SYNC_PT *psSyncPt;
+	PDLLIST_NODE psNode;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+
+	if (hFence == PVRSRV_NO_FENCE)
+	{
+		*puiNumCheckpoints = 0;
+		eError = PVRSRV_OK;
+		goto e0;
+	}
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) hFence,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                                 IMG_TRUE,
+	                                 (void**)&psFence,
+	                                 &psHBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	apsCheckpoints = OSAllocMem(sizeof(*apsCheckpoints) * psFence->uiNumSyncs);
+	PVR_LOG_GOTO_IF_NOMEM(apsCheckpoints, eError, e1);
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+	/* Increase refcount to make sure fence is not destroyed while waiting */
+	_SyncFbFenceAcquire(psFence);
+
+	/* Go through all syncs and add them to the list */
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		psSyncPt = psFence->apsFenceSyncList[i];
+
+		psNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psSyncCB = IMG_CONTAINER_OF(psNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+
+		if (_SyncFbSyncPtHandleType(psSyncCB) == PVRSRV_SYNC_HANDLE_PVR)
+		{
+			psCheckpoint = (PSYNC_CHECKPOINT) psSyncCB->hAttachedSync;
+			apsCheckpoints[uiNumCheckpoints++] = psCheckpoint;
+		}
+	}
+
+	SyncFbFenceRelease(psFence);
+
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	*puiNumCheckpoints = uiNumCheckpoints;
+	*papsCheckpoints = apsCheckpoints;
+
+e1:
+	PVRSRVReleaseHandle(psHBase,
+	                    (IMG_HANDLE) (uintptr_t) hFence,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+e0:
+	return eError;
+}
+#endif
+
+PVRSRV_ERROR SyncFbRegisterSyncFunctions(void)
+{
+	/* Initialise struct and register with sync_checkpoint.c */
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceResolve = &SyncFbFenceResolvePVR;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceCreate = &SyncFbFenceCreatePVR;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceDataRollback = &SyncFbFenceRollbackPVR;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceFinalise = NULL; /* no fence finalise function required */
+	gsSyncFbContext.sSyncCheckpointReg.pfnNoHWUpdateTimelines = &_SyncFbTimelineUpdate_NotifyCMD;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFreeCheckpointListMem = OSFreeMem;
+	gsSyncFbContext.sSyncCheckpointReg.pfnDumpInfoOnStalledUFOs = &SyncFbDumpInfoOnStalledUFOs;
+	OSStringSafeCopy(gsSyncFbContext.sSyncCheckpointReg.pszImplName, "SyncFb", SYNC_CHECKPOINT_IMPL_MAX_STRLEN);
+#if defined(PDUMP)
+	gsSyncFbContext.sSyncCheckpointReg.pfnSyncFenceGetCheckpoints = &SyncFbFenceGetCheckpoints;
+#endif
+
+	return SyncCheckpointRegisterFunctions(&gsSyncFbContext.sSyncCheckpointReg);
+}
+
+PVRSRV_ERROR SyncFbRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	SYNC_FB_CONTEXT_DEVICE_LIST *psNewDeviceEntry;
+
+	PVR_DPF_ENTERED;
+
+	/* Initialise the sync fallback context */
+	if (gsSyncFbContext.hSyncEventObject == NULL)
+	{
+		eError = OSEventObjectCreate("Sync event object",
+		                             &gsSyncFbContext.hSyncEventObject);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		dllist_init(&gsSyncFbContext.sTlList);
+		dllist_init(&gsSyncFbContext.sFenceList);
+		dllist_init(&gsSyncFbContext.sDeviceList);
+		gsSyncFbContext.sCheckpointHashTable = HASH_Create_Extended(64, sizeof(IMG_UINT32), _SyncCheckpointFWAddrHash, _SyncCheckpointFWAddrCompare);
+
+		eError = OSLockCreate(&gsSyncFbContext.hFbContextLock);
+		PVR_GOTO_IF_ERROR(eError, e2);
+
+		eError = PVRSRVRegisterCmdCompleteNotify(&gsSyncFbContext.hCMDNotify,
+		                                         &_SyncFbTimelineUpdate_NotifyCMD,
+		                                         &gsSyncFbContext);
+		PVR_GOTO_IF_ERROR(eError, e3);
+	}
+
+	psNewDeviceEntry = OSAllocMem(sizeof(*psNewDeviceEntry));
+	PVR_GOTO_IF_NOMEM(psNewDeviceEntry, eError, e4);
+
+	psNewDeviceEntry->psDevice = psDeviceNode;
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_add_to_tail(&gsSyncFbContext.sDeviceList, &psNewDeviceEntry->sDeviceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psNewDeviceEntry->hDBGNotify,
+	                                              psDeviceNode,
+	                                              _SyncFbDebugRequest,
+	                                              DEBUG_REQUEST_FALLBACKSYNC,
+	                                              NULL);
+	PVR_GOTO_IF_ERROR(eError, e5);
+
+	PVR_DPF_RETURN_RC(eError);
+
+
+e5:
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_remove_node(&psNewDeviceEntry->sDeviceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+	OSFreeMem(psNewDeviceEntry);
+e4:
+	PVRSRVUnregisterCmdCompleteNotify(gsSyncFbContext.hCMDNotify);
+e3:
+	OSLockDestroy(gsSyncFbContext.hFbContextLock);
+e2:
+	HASH_Delete_Extended(gsSyncFbContext.sCheckpointHashTable, IMG_TRUE);
+	OSEventObjectDestroy(gsSyncFbContext.hSyncEventObject);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbDeregisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_FB_CONTEXT_DEVICE_LIST *psDeviceEntry;
+	PDLLIST_NODE psNode, psNext;
+
+	PVR_DPF_ENTERED;
+
+	/* Return if Init was never called */
+	if (gsSyncFbContext.hSyncEventObject == NULL)
+		goto e1;
+
+	/* Check device list for the given device and remove it */
+	dllist_foreach_node(&gsSyncFbContext.sDeviceList, psNode, psNext)
+	{
+		psDeviceEntry = IMG_CONTAINER_OF(psNode,
+		                                 SYNC_FB_CONTEXT_DEVICE_LIST,
+		                                 sDeviceListNode);
+
+		if (psDeviceEntry->psDevice == psDeviceNode)
+		{
+			PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceEntry->hDBGNotify);
+
+			OSLockAcquire(gsSyncFbContext.hFbContextLock);
+			dllist_remove_node(psNode);
+			OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+			OSFreeMem(psDeviceEntry);
+			break;
+		}
+	}
+
+	/* If there are still devices registered with us don't deinit module */
+	if (!dllist_is_empty(&gsSyncFbContext.sDeviceList))
+	{
+		goto e1;
+	}
+
+	PVRSRVUnregisterCmdCompleteNotify(gsSyncFbContext.hCMDNotify);
+
+	HASH_Delete_Extended(gsSyncFbContext.sCheckpointHashTable, IMG_TRUE);
+	eError = OSEventObjectDestroy(gsSyncFbContext.hSyncEventObject);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+	gsSyncFbContext.hSyncEventObject = NULL;
+
+	OSLockDestroy(gsSyncFbContext.hFbContextLock);
+
+e1:
+	return eError;
+}
+
+/* HOLD TL LOCK!
+ * Creates a new sync point on a timeline */
+static PVRSRV_ERROR _SyncFbSyncPtCreate(PVRSRV_SYNC_PT **ppsSyncPt,
+                                       PVRSRV_TIMELINE_SERVER *psTl,
+                                       IMG_UINT32 uiSeqNumber)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_SYNC_PT *psNewSyncPt;
+
+	PVR_DPF_ENTERED;
+
+	psNewSyncPt = OSAllocMem(sizeof(*psNewSyncPt));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSyncPt, eError, e1);
+
+	psNewSyncPt->psTl = psTl;
+	OSAtomicWrite(&psNewSyncPt->iStatus, PVRSRV_SYNC_NOT_SIGNALLED);
+
+	psNewSyncPt->uiSeqNum = uiSeqNumber;
+	psNewSyncPt->uiPID = OSGetCurrentClientProcessIDKM();
+	PT_REF_SET(&psNewSyncPt->iRef, 1,psNewSyncPt);
+
+	dllist_init(&psNewSyncPt->sTlSyncList);
+	dllist_init(&psNewSyncPt->sTlSyncActiveList);
+	dllist_init(&psNewSyncPt->sSignalCallbacks);
+
+	/* Increment Tl ref due to new checkpoint*/
+	_SyncFbTimelineAcquire(psTl);
+
+	dllist_add_to_tail(&psTl->sSyncList, &psNewSyncPt->sTlSyncList);
+	dllist_add_to_tail(&psTl->sSyncActiveList, &psNewSyncPt->sTlSyncActiveList);
+
+	*ppsSyncPt = psNewSyncPt;
+
+	PVR_DPF_RETURN_OK;
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Increment sync point refcount */
+static void _SyncFbSyncPtAcquire(PVRSRV_SYNC_PT *psSyncPt)
+{
+	PT_REF_INC(&psSyncPt->iRef, psSyncPt);
+}
+
+/* Release and maybe destroy sync point if refcount is 0 */
+static PVRSRV_ERROR _SyncFbSyncPtRelease(PVRSRV_SYNC_PT *psSyncPt,
+                                         IMG_BOOL bError)
+{
+	PVRSRV_ERROR eError;
+	PDLLIST_NODE psNode, psNext;
+	PVRSRV_SYNC_SIGNAL_CB *psSyncCB;
+	IMG_INT iRef;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	iRef = PT_REF_DEC(&psSyncPt->iRef, psSyncPt);
+	if (iRef != 0)
+	{
+		eError = PVRSRV_OK;
+		goto e1;
+	}
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	/* Remove all checkpoints from the ufo lookup table.*/
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks, psNode, psNext)
+	{
+		psSyncCB = IMG_CONTAINER_OF(psNode,
+		                            PVRSRV_SYNC_SIGNAL_CB,
+		                            sCallbackNode);
+
+		if (_SyncFbSyncPtHandleType(psSyncCB) == PVRSRV_SYNC_HANDLE_PVR)
+		{
+			IMG_UINT32 ui32FwAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT) psSyncCB->hAttachedSync);
+
+			DBG(("%s: Removing key <%#08x>", __func__, ui32FwAddr));
+			HASH_Remove_Extended(gsSyncFbContext.sCheckpointHashTable, &ui32FwAddr);
+		}
+	}
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	OSLockAcquire(psSyncPt->psTl->hTlLock);
+	if (dllist_node_is_in_list(&psSyncPt->sTlSyncActiveList))
+		dllist_remove_node(&psSyncPt->sTlSyncActiveList);
+
+	dllist_remove_node(&psSyncPt->sTlSyncList);
+
+	if (bError)
+	{
+		_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_ERRORED);
+	}
+	OSLockRelease(psSyncPt->psTl->hTlLock);
+
+	/* Remove all attached nodes and signal them.*/
+	while (!dllist_is_empty(&psSyncPt->sSignalCallbacks))
+	{
+		psNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psSyncCB = IMG_CONTAINER_OF(psNode,
+		                            PVRSRV_SYNC_SIGNAL_CB,
+		                            sCallbackNode);
+
+		psSyncCB->pfnSyncFree(psSyncCB->hAttachedSync);
+		dllist_remove_node(&psSyncCB->sCallbackNode);
+		OSFreeMem(psSyncCB);
+	}
+
+	eError = SyncFbTimelineRelease(psSyncPt->psTl);
+	PVR_LOG_IF_ERROR(eError, "SyncFbTimelineRelease");
+
+	OSFreeMem(psSyncPt);
+
+	PVR_DPF_RETURN_OK;
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* HOLD TL LOCK!
+ * Mark all attached syncs of a sync point with the state eSignal */
+static PVRSRV_ERROR _SyncFbSyncPtSignalAttached(PVRSRV_SYNC_PT *psSyncPt,
+                                                PVRSRV_SYNC_STATE eSignal)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK, eRet;
+	PDLLIST_NODE psCurrentCB, psNextCB;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	if (dllist_is_empty(&psSyncPt->sSignalCallbacks))
+	{
+		ERR("Sync pt has no attached syncs. Make sure to attach one "
+		    "when creating a new sync pt to define its flavour");
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e1;
+	}
+
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks, psCurrentCB, psNextCB)
+	{
+		psCB = IMG_CONTAINER_OF(psCurrentCB,
+		                        PVRSRV_SYNC_SIGNAL_CB,
+		                        sCallbackNode);
+		eRet = psCB->pfnSignal(psCB->hAttachedSync, eSignal);
+		if (eRet != PVRSRV_OK)
+		{
+			ERR("Failed to signal an attached sync, system might block!");
+			eError = eRet;
+			/* Don't jump to exit but try to signal remaining syncs */
+		}
+	}
+
+e1:
+	PVR_DPF_RETURN_RC1(eError, psSyncPt);
+}
+
+/* HOLD TL LOCK!
+ * Mark all a sync point with the state eSignal */
+static PVRSRV_ERROR _SyncFbSyncPtSignal(PVRSRV_SYNC_PT *psSyncPt, PVRSRV_SYNC_STATE eSignal)
+{
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	OSAtomicWrite(&psSyncPt->iStatus, eSignal);
+
+	if (psSyncPt->uiSeqNum >
+	    OSAtomicRead(&psSyncPt->psTl->iLastSignalledSeqNum))
+	{
+		OSAtomicWrite(&psSyncPt->psTl->iLastSignalledSeqNum,
+		              psSyncPt->uiSeqNum);
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/* Check whether all syncs in a fence were signalled */
+static IMG_BOOL _SyncFbFenceSyncsHaveSignalled(PVRSRV_FENCE_SERVER *psFence)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		if (OSAtomicRead(&psFence->apsFenceSyncList[i]->iStatus) ==
+			PVRSRV_SYNC_NOT_SIGNALLED)
+		{
+			PVR_DPF_RETURN_RC1(IMG_FALSE, psFence);
+		}
+	}
+
+	OSAtomicWrite(&psFence->iStatus,
+	              PVRSRV_FENCE_SIGNALLED);
+
+	PVR_DPF_RETURN_RC1(IMG_TRUE, psFence);
+}
+
+/* Increment timeline refcount */
+static void _SyncFbTimelineAcquire(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	TL_REF_INC(&psTl->iRef, psTl);
+}
+
+PVRSRV_ERROR SyncFbTimelineRelease(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	IMG_INT iRef;
+
+	PVR_DPF_ENTERED1(psTl);
+
+	iRef = TL_REF_DEC(&psTl->iRef, psTl);
+	if (iRef != 0)
+	{
+		PVR_DPF_RETURN_OK;
+	}
+
+	if (_SyncFbTimelineHandleType(psTl) == PVRSRV_SYNC_HANDLE_SW)
+	{
+		_SyncSWTimelineCheckForUnsignalledPts(psTl);
+	}
+
+	_SyncFbTimelineListDel(psTl);
+
+	OSLockDestroy(psTl->hTlLock);
+
+#if defined(DEBUG)
+	psTl->sTlOps.pfnSyncPtHasSignalled = NULL;
+	psTl->hTlLock = NULL;
+#endif
+
+	OSFreeMem(psTl);
+
+	PVR_DPF_RETURN_OK;
+}
+
+/* Increment fence refcount */
+static void _SyncFbFenceAcquire(PVRSRV_FENCE_SERVER *psFence)
+{
+	FENCE_REF_INC(&psFence->iRef, psFence);
+}
+
+PVRSRV_ERROR SyncFbFenceRelease(PVRSRV_FENCE_SERVER *psFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK, eRet;
+	IMG_INT iRef;
+	IMG_UINT32 i;
+	IMG_BOOL bError = IMG_FALSE;
+	IMG_BOOL bCleanup = IMG_FALSE;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	/* If cleanup thread, force cleanup of SW fences, otherwise
+	 * simply decrement psFence->iRef
+	 */
+	if (OSGetCurrentClientProcessIDKM() == PVRSRVGetPVRSRVData()->cleanupThreadPid)
+	{
+		IMG_UINT32 uiSyncPt;
+
+		/* Force erroring of any SW sync pts this fence may contain */
+		for (uiSyncPt=0; uiSyncPt < psFence->uiNumSyncs; uiSyncPt++)
+		{
+			PVRSRV_SYNC_PT *psSyncPt = psFence->apsFenceSyncList[uiSyncPt];
+
+			/* If this is a SW sync point from a timeline which was created
+			 * by the process being cleaned-up, then error it
+			 */
+			if ((_SyncFbTimelineHandleType(psSyncPt->psTl) == PVRSRV_SYNC_HANDLE_SW) &&
+			    (PVRSRVGetPurgeConnectionPid() == psSyncPt->psTl->uiPID))
+			{
+				OSLockAcquire(psSyncPt->psTl->hTlLock);
+				_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_ERRORED);
+				OSLockRelease(psSyncPt->psTl->hTlLock);
+			}
+		}
+	}
+
+	iRef = FENCE_REF_DEC(&psFence->iRef, psFence);
+	if (iRef != 0)
+	{
+		goto e1;
+	}
+
+	PDUMPCOMMENTWITHFLAGS(psFence->psDevNode, 0,
+	                      "Destroy Fence %s (ID:%"IMG_UINT64_FMTSPEC")",
+	                      psFence->pszName,
+	                      psFence->iUID);
+
+	if (OSGetCurrentClientProcessIDKM() ==
+			PVRSRVGetPVRSRVData()->cleanupThreadPid)
+	{
+		bCleanup = IMG_TRUE;
+	}
+
+	_SyncFbFenceListDel(psFence);
+
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		PVRSRV_SYNC_PT *psSyncPt = psFence->apsFenceSyncList[i];
+
+		if (bCleanup &&
+				_SyncFbTimelineHandleType(psSyncPt->psTl) == PVRSRV_SYNC_HANDLE_SW)
+		{
+			bError = IMG_TRUE;
+		}
+
+		eRet = _SyncFbSyncPtRelease(psSyncPt,
+		                            bError);
+		if (eRet != PVRSRV_OK)
+		{
+			ERR("Error when releasing SyncPt, this might leak memory")
+			eError = eRet;
+			/* Try to continue and release the other sync pts, return error */
+		}
+	}
+
+#if defined(DEBUG)
+	{
+		for (i = 0; i < psFence->uiNumSyncs; i++)
+		{
+			psFence->apsFenceSyncList[i] = NULL;
+		}
+		psFence->uiNumSyncs = 0;
+	}
+#endif
+
+	OSFreeMem(psFence->apsFenceSyncList);
+	OSFreeMem(psFence);
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbFenceDup(PVRSRV_FENCE_SERVER *psInFence,
+                            PVRSRV_FENCE_SERVER **ppsOutFence)
+{
+	PVR_DPF_ENTERED1(psInFence);
+
+	FENCE_REF_INC(&psInFence->iRef, psInFence);
+
+	PDUMPCOMMENTWITHFLAGS(psInFence->psDevNode, 0,
+	                      "Dup Fence %s (ID:%"IMG_UINT64_FMTSPEC").",
+	                      psInFence->pszName,
+	                      psInFence->iUID);
+
+	*ppsOutFence = psInFence;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, *ppsOutFence);
+}
+
+static IMG_BOOL _SyncFbSyncPtHasSignalled(PVRSRV_SYNC_PT *psSyncPt)
+{
+	return psSyncPt->psTl->sTlOps.pfnSyncPtHasSignalled(psSyncPt);
+}
+
+static IMG_BOOL _SyncFbFenceAddPt(PVRSRV_FENCE_SERVER *psFence,
+                                  IMG_UINT32 *i,
+                                  PVRSRV_SYNC_PT *psSyncPt)
+{
+	/*
+	 * If the fence is signalled there is no need to add it to the fence.
+	 * One exception is PDUMP drivers where we need to make sure we
+	 * set up proper synchronisation in the pdump stream.
+	 */
+#if !defined(PDUMP)
+	if (_SyncFbSyncPtHasSignalled(psSyncPt)) return IMG_FALSE;
+#endif
+	_SyncFbSyncPtAcquire(psSyncPt);
+	psFence->apsFenceSyncList[*i] = psSyncPt;
+	(*i)++;
+	return IMG_TRUE;
+}
+
+PVRSRV_ERROR SyncFbFenceMerge(PVRSRV_FENCE_SERVER *psInFence1,
+                              PVRSRV_FENCE_SERVER *psInFence2,
+                              IMG_UINT32 uiFenceNameSize,
+                              const IMG_CHAR *pszFenceName,
+                              PVRSRV_FENCE_SERVER **ppsOutFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	IMG_UINT32 i, i1, i2;
+	IMG_UINT32 uiFenceSyncListSize;
+
+	PVR_DPF_ENTERED;
+
+	psNewFence = OSAllocMem(sizeof(*psNewFence));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence, eError, e1);
+
+	psNewFence->psDevNode = psInFence1->psDevNode;
+
+	uiFenceSyncListSize = sizeof(*(psNewFence->apsFenceSyncList)) *
+			(psInFence1->uiNumSyncs + psInFence2->uiNumSyncs);
+
+	psNewFence->apsFenceSyncList = OSAllocMem(uiFenceSyncListSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence->apsFenceSyncList, eError, e2);
+
+	if (uiFenceNameSize == 1)
+	{
+		OSSNPrintf(psNewFence->pszName,
+		           SYNC_FB_FENCE_MAX_LENGTH,
+		           "Fence-Merged");
+	}
+	else
+	{
+		if (pszFenceName)
+		{
+			OSStringSafeCopy(psNewFence->pszName,
+						  pszFenceName,
+						  SYNC_FB_FENCE_MAX_LENGTH);
+		}
+		else
+		{
+			psNewFence->pszName[0] = '\0';
+		}
+	}
+
+	/* Add sync pts from input fence 1 & 2
+	 * - no duplicates in one timeline
+	 * - sync pts in one fence are ordered by timeline UID
+	 *
+	 * */
+	for (i = 0, i1 = 0, i2 = 0;
+	     i1 < psInFence1->uiNumSyncs && i2 < psInFence2->uiNumSyncs;)
+	{
+		PVRSRV_SYNC_PT *psSyncPt1 = psInFence1->apsFenceSyncList[i1];
+		PVRSRV_SYNC_PT *psSyncPt2 = psInFence2->apsFenceSyncList[i2];
+
+		/* Adding sync pts in order of their timeline UID, smaller ID first */
+		if (psSyncPt1->psTl->iUID <
+            psSyncPt2->psTl->iUID)
+		{
+			_SyncFbFenceAddPt(psNewFence, &i, psSyncPt1);
+			i1++;
+		}
+		else if (psSyncPt1->psTl->iUID >
+		         psSyncPt2->psTl->iUID)
+		{
+			_SyncFbFenceAddPt(psNewFence, &i, psSyncPt2);
+			i2++;
+		}
+		/* In case the timeline UID is the same just add the point that is
+		 * later on that timeline. */
+		else
+		{
+			/* --> Some C magic to find out if 'a' is a point later in the
+			 * timeline than 'b', wrap around is taken into account:
+			 *			(a - b <= ((IMG_INT)(~0U>>1)) ) */
+			if ( psSyncPt1->uiSeqNum - psSyncPt2->uiSeqNum <=
+			    ((IMG_INT)(~0U>>1)) )
+			{
+				_SyncFbFenceAddPt(psNewFence, &i, psSyncPt1);
+			}
+			else
+			{
+				_SyncFbFenceAddPt(psNewFence, &i, psSyncPt2);
+			}
+
+			i1++;
+			i2++;
+		}
+	}
+
+	/* Add the remaining syncs pts to the fence. At this point we only enter
+	 * either the first or the second loop because one fence has
+	 * more sync pts than the other.
+	 */
+	for (; i1 < psInFence1->uiNumSyncs; i1++)
+	{
+		_SyncFbFenceAddPt(psNewFence, &i, psInFence1->apsFenceSyncList[i1]);
+	}
+
+	for (; i2 < psInFence2->uiNumSyncs; i2++)
+	{
+		_SyncFbFenceAddPt(psNewFence, &i, psInFence2->apsFenceSyncList[i2]);
+	}
+
+	/* Fill remaining fields */
+	psNewFence->uiNumSyncs = i;
+	psNewFence->iUID = (IMG_INT64)(uintptr_t) psNewFence;
+	FENCE_REF_SET(&psNewFence->iRef, 1, psNewFence);
+
+	OSAtomicWrite(&psNewFence->iStatus, PVRSRV_SYNC_NOT_SIGNALLED);
+
+	_SyncFbFenceListAdd(psNewFence);
+
+	PDUMPCOMMENTWITHFLAGS(psInFence1->psDevNode, 0,
+	                      "Merge Fence1 %s (ID:%"IMG_UINT64_FMTSPEC"), Fence2 %s (ID:%"IMG_UINT64_FMTSPEC") "
+	                      "to Fence %s (ID:%"IMG_UINT64_FMTSPEC")",
+	                      psInFence1->pszName,
+	                      psInFence1->iUID,
+	                      psInFence2->pszName,
+	                      psInFence2->iUID,
+	                      psNewFence->pszName,
+	                      psNewFence->iUID);
+
+	*ppsOutFence = psNewFence;
+
+	PVR_DPF_RETURN_RC1(eError, *ppsOutFence);
+
+e2:
+	OSFreeMem(psNewFence);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+PVRSRV_ERROR SyncFbFenceWait(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 ui32TimeoutInMs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_HANDLE hOSEvent;
+	IMG_UINT32 t1 = 0, t2 = 0;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	/* Increase refcount to make sure fence is not destroyed while waiting */
+	_SyncFbFenceAcquire(psFence);
+
+	if (OSAtomicRead(&psFence->iStatus) == PVRSRV_FENCE_NOT_SIGNALLED)
+	{
+		PVRSRV_ERROR eErrorClose;
+
+		/* If the status of the fence is not signalled it could mean that
+		 * there are actually syncs still pending or that we have not
+		 * checked yet whether the syncs were met, therefore do the
+		 * check now and return in case they are. If they are not, go
+		 * to sleep and wait. */
+
+		if (_SyncFbFenceSyncsHaveSignalled(psFence))
+		{
+			goto e1;
+		}
+		else if (ui32TimeoutInMs == 0)
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			goto e1;
+		}
+
+		eError = OSEventObjectOpen(gsSyncFbContext.hSyncEventObject,
+		                           &hOSEvent);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		while (!_SyncFbFenceSyncsHaveSignalled(psFence) && ui32TimeoutInMs)
+		{
+			t1 = OSClockms();
+			/* Wait for EO to be signalled */
+			eError = OSEventObjectWaitTimeout(hOSEvent,
+			                                  ui32TimeoutInMs * 1000);
+			t2 = OSClockms();
+
+			if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_TIMEOUT)
+			{
+				break;
+			}
+
+
+			/* Reduce timeout by the time we have just waited */
+			if (ui32TimeoutInMs < (t2-t1))
+			{
+				ui32TimeoutInMs = 0;
+			}
+			else
+			{
+				ui32TimeoutInMs -= (t2-t1);
+			}
+		}
+
+		eErrorClose = OSEventObjectClose(hOSEvent);
+		if (eErrorClose != PVRSRV_OK)
+		{
+			ERR("Unable to close Event Object");
+
+			/* Do not overwrite previous error
+			 * if it was something else than PVRSRV_OK */
+			if (eError == PVRSRV_OK)
+			{
+				eError = eErrorClose;
+			}
+		}
+	}
+e1:
+
+	SyncFbFenceRelease(psFence);
+
+	PVR_DPF_RETURN_RC1(eError, psFence);
+}
+
+PVRSRV_ERROR SyncFbFenceDump(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiLine,
+                             IMG_UINT32 uiFileNameLength,
+                             const IMG_CHAR *pszFile,
+                             IMG_UINT32 uiModuleLength,
+                             const IMG_CHAR *pszModule,
+                             IMG_UINT32 uiDescLength,
+                             const IMG_CHAR *pszDesc)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(uiFileNameLength);
+	PVR_UNREFERENCED_PARAMETER(uiModuleLength);
+	PVR_UNREFERENCED_PARAMETER(uiDescLength);
+
+	PVR_DPF_ENTERED1(psFence);
+
+	PVR_LOG(("  Fence dump request from:"));
+#if defined(DEBUG)
+	PVR_LOG(("    %s (%s:%u)", pszModule, pszFile, uiLine));
+#else
+	PVR_LOG(("    %s (location only available in debug build)", pszModule));
+	PVR_UNREFERENCED_PARAMETER(pszFile);
+	PVR_UNREFERENCED_PARAMETER(uiLine);
+#endif
+	PVR_LOG(("  Desc: %s", pszDesc));
+	PVR_LOG(("---------------- FENCE ----------------"));
+	PVR_LOG(("%s (UID: %"IMG_UINT64_FMTSPEC")", psFence->pszName, psFence->iUID));
+
+	PVR_LOG(("  Signalled: %s",
+	        _SyncFbFenceSyncsHaveSignalled(psFence)?"Yes":"No"));
+	PVR_LOG(("  Ref: %d", OSAtomicRead(&psFence->iRef) ));
+
+	PVR_LOG(("  Sync Points:"));
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		PVRSRV_SYNC_PT *psSP = psFence->apsFenceSyncList[i];
+		PVR_LOG(("    Point %u)", i));
+		PVR_LOG(("      On timeline:     %s (UID: %"IMG_UINT64_FMTSPEC")",
+		         psSP->psTl->pszName, psSP->psTl->iUID));
+		PVR_LOG(("      Sequence number: %u", psSP->uiSeqNum));
+		PVR_LOG(("      Signalled:       %s",
+		        psSP->psTl->sTlOps.pfnSyncPtHasSignalled(psSP)? "Yes":"No"));
+		PVR_LOG(("      Ref:             %d", OSAtomicRead(&psSP->iRef)));
+	}
+	PVR_LOG(("----------------------------------------"));
+
+	PVR_DPF_RETURN_RC1(eError, psFence);
+}
+
+static PVRSRV_ERROR _SyncFbTimelineCreate(PFN_SYNC_PT_HAS_SIGNALLED pfnHasPtSignalled,
+                                          IMG_UINT32 uiTimelineNameSize,
+                                          const IMG_CHAR *pszTimelineName,
+                                          PVRSRV_TIMELINE_SERVER **ppsTimeline)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_TIMELINE_SERVER *psNewTl;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ppsTimeline, eError, e1);
+
+	psNewTl = OSAllocMem(sizeof(*psNewTl));
+	PVR_LOG_GOTO_IF_NOMEM(psNewTl, eError, e2);
+
+	eError = OSLockCreate(&psNewTl->hTlLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e3);
+
+	if (uiTimelineNameSize == 1)
+	{
+		OSSNPrintf(psNewTl->pszName,
+		           SYNC_FB_TIMELINE_MAX_LENGTH,
+		           "TL-%s-%d",
+		           OSGetCurrentClientProcessNameKM(),
+		           OSGetCurrentClientProcessIDKM());
+	}
+	else
+	{
+		if (pszTimelineName)
+		{
+			OSStringSafeCopy((IMG_CHAR*) psNewTl->pszName,
+			              pszTimelineName,
+			              SYNC_FB_TIMELINE_MAX_LENGTH);
+		}
+		else
+		{
+			psNewTl->pszName[0] = '\0';
+		}
+	}
+
+	dllist_init(&psNewTl->sSyncList);
+	dllist_init(&psNewTl->sSyncActiveList);
+	dllist_init(&psNewTl->sTlList);
+
+	psNewTl->sTlOps.pfnSyncPtHasSignalled = pfnHasPtSignalled;
+	psNewTl->iUID = (IMG_INT64)(uintptr_t) psNewTl; /* Not unique throughout the driver lifetime */
+	OSAtomicWrite(&psNewTl->iSeqNum, 0);
+	OSAtomicWrite(&psNewTl->iLastSignalledSeqNum, 0);
+	psNewTl->uiPID = OSGetCurrentClientProcessIDKM();
+
+	/* Set initial refcount value */
+	TL_REF_SET(&psNewTl->iRef, 1, psNewTl);
+
+	_SyncFbFTimelineListAdd(psNewTl);
+
+	*ppsTimeline = psNewTl;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNewTl);
+
+e3:
+	OSFreeMem(psNewTl);
+e2:
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         PVR SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+/* Free a PVR sync point with its sync checkpoint */
+static void _SyncFbSyncPtFreePVR(IMG_HANDLE hSync)
+{
+	PVR_DPF_ENTERED1(hSync);
+
+	SyncCheckpointFree((PSYNC_CHECKPOINT) hSync);
+
+	PVR_DPF_RETURN;
+}
+
+
+/* Mark a sync checkpoint with the given state.
+ * MAKE SURE TO WAKE UP FW AFTER CALLING THIS */
+static PVRSRV_ERROR _SyncFbSyncPtSignalPVR(IMG_HANDLE hSync,
+                                           PVRSRV_SYNC_STATE eState)
+{
+	PSYNC_CHECKPOINT psSyncCheck = (PSYNC_CHECKPOINT) hSync;
+
+	PVR_DPF_ENTERED1(hSync);
+
+	if (!SyncCheckpointIsSignalled(psSyncCheck, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		switch (eState)
+		{
+			case PVRSRV_SYNC_SIGNALLED:
+				SyncCheckpointSignal(psSyncCheck, PVRSRV_FENCE_FLAG_NONE);
+				break;
+			case PVRSRV_SYNC_ERRORED:
+				SyncCheckpointError(psSyncCheck, PVRSRV_FENCE_FLAG_NONE);
+				break;
+			default:
+				ERR("Passed unknown sync state, "
+						"please use a valid one for signalling.");
+				return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, hSync);
+}
+
+/* Check whether the native sync of the SyncPt has signalled.
+ *
+ * HOLD TL LOCK!
+ */
+static IMG_BOOL _SyncFbSyncPtHasSignalledPVR(PVRSRV_SYNC_PT *psSyncPt)
+{
+	PDLLIST_NODE psCBNode;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+	PSYNC_CHECKPOINT psSyncCheck;
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	/* If the SyncPt is not signalled yet,
+	 * check whether the first attached sync has.
+	 *
+	 * Change SyncPt state to signalled or errored if yes.
+	 * Also notify other attached syncs.
+	 */
+	if (OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_NOT_SIGNALLED)
+	{
+		/* List must have at least the device sync attached if we are called */
+		PVR_ASSERT(!dllist_is_empty(&psSyncPt->sSignalCallbacks));
+
+		/* Retrieve the first sync checkpoint of that sync pt */
+		psCBNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psCB = IMG_CONTAINER_OF(psCBNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+		psSyncCheck = (PSYNC_CHECKPOINT) psCB->hAttachedSync;
+
+		if (SyncCheckpointIsSignalled(psSyncCheck, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		{
+			_SyncFbSyncPtSignal(psSyncPt, PVRSRV_SYNC_SIGNALLED);
+
+			/* Signal all other attached syncs */
+			PVR_LOG_IF_ERROR(_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_SIGNALLED),
+			                 "_SyncFbSyncPtSignalAttached");
+
+			bRet = IMG_TRUE;
+		}
+
+		PVR_DPF_RETURN_RC1(bRet, psSyncPt);
+	}
+	else
+	{
+		PVR_DPF_RETURN_RC1(IMG_TRUE, psSyncPt);
+	}
+}
+
+PVRSRV_ERROR SyncFbTimelineCreatePVR(IMG_UINT32 uiTimelineNameSize,
+                                     const IMG_CHAR *pszTimelineName,
+                                     PVRSRV_TIMELINE_SERVER **ppsTimeline)
+{
+	return _SyncFbTimelineCreate(&_SyncFbSyncPtHasSignalledPVR,
+	                             uiTimelineNameSize,
+	                             pszTimelineName,
+	                             ppsTimeline);
+}
+
+PVRSRV_ERROR SyncFbFenceCreatePVR(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const IMG_CHAR *pszName,
+                                  PVRSRV_TIMELINE iTl,
+                                  PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                  PVRSRV_FENCE *piOutFence,
+                                  IMG_UINT64 *puiFenceUID,
+                                  void **ppvFenceFinaliseData,
+                                  PSYNC_CHECKPOINT *ppsOutCheckpoint,
+                                  void **ppvTimelineUpdateSync,
+                                  IMG_UINT32 *puiTimelineUpdateValue)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	PVRSRV_SYNC_PT *psNewSyncPt = NULL;
+	PVRSRV_SYNC_SIGNAL_CB *psNewSyncSignalCB;
+	PVRSRV_HANDLE_BASE	*psHandleBase;
+	PVRSRV_TIMELINE_SERVER *psTl;
+	IMG_HANDLE hOutFence;
+	IMG_UINT32 ui32FwAddr;
+
+	PVR_UNREFERENCED_PARAMETER(ppvTimelineUpdateSync);
+	PVR_UNREFERENCED_PARAMETER(puiTimelineUpdateValue);
+
+	PVR_DPF_ENTERED;
+
+	/* The fallback implementation does not need to finalise
+	 * the fence, so set the ppvFenceFinaliseData to NULL
+	 * (if provided)
+	 */
+	if (ppvFenceFinaliseData != NULL )
+	{
+		*ppvFenceFinaliseData = NULL;
+	}
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pszName, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(piOutFence, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ppsOutCheckpoint, eError, e0);
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) iTl,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	                                 IMG_TRUE,
+	                                 (void**) &psTl,
+	                                 &psHandleBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (unlikely(_SyncFbTimelineHandleType(psTl) != PVRSRV_SYNC_HANDLE_PVR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_SyncFbTimelineHandleType", eError, PVRSRV_ERROR_INVALID_PARAMS, e1);
+	}
+
+	/* Allocate:
+	 *		Fence
+	 *		Sync Signal CB
+	 *		SyncPt List
+	 *		Sync Checkpoint
+	 *		SyncPt
+	 *		Handle
+	 *	Setup
+	 */
+	psNewFence = OSAllocMem(sizeof(*psNewFence));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence, eError, e2);
+
+	psNewSyncSignalCB = OSAllocMem(sizeof(*psNewSyncSignalCB));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSyncSignalCB, eError, e3);
+
+	psNewFence->apsFenceSyncList = OSAllocMem(sizeof(*(psNewFence->apsFenceSyncList)));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence->apsFenceSyncList, eError, e4);
+
+	psNewFence->psDevNode = psDeviceNode;
+
+	/* Lock down TL until new point is fully created and inserted */
+	OSLockAcquire(psTl->hTlLock);
+
+	eError = _SyncFbSyncPtCreate(&psNewSyncPt,
+	                             psTl,
+	                             OSAtomicIncrement(&psTl->iSeqNum));
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		OSLockRelease(psTl->hTlLock);
+		ERR("Cannot allocate SyncPt.");
+		goto e5;
+	}
+
+	eError = SyncCheckpointAlloc(psSyncCheckpointContext,
+	                             iTl,
+	                             PVRSRV_NO_FENCE,
+	                             pszName,
+	                             ppsOutCheckpoint);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		OSLockRelease(psTl->hTlLock);
+		ERR("Cannot allocate SyncCheckpoint.");
+		goto e6;
+	}
+
+	ui32FwAddr = SyncCheckpointGetFirmwareAddr(*ppsOutCheckpoint);
+
+	/* Init Sync Signal CB */
+	psNewSyncSignalCB->hAttachedSync = (IMG_HANDLE) *ppsOutCheckpoint;
+	psNewSyncSignalCB->pfnSignal = &_SyncFbSyncPtSignalPVR;
+	psNewSyncSignalCB->pfnSyncFree = &_SyncFbSyncPtFreePVR;
+	psNewSyncSignalCB->hPrivData = (IMG_HANDLE) psSyncCheckpointContext;
+
+	dllist_add_to_tail(&psNewSyncPt->sSignalCallbacks,
+	                   &psNewSyncSignalCB->sCallbackNode);
+
+	OSLockRelease(psTl->hTlLock);
+
+	DBG(("%s: Inserting key <%p> = %p", __func__, *ppsOutCheckpoint, psNewSyncPt));
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	HASH_Insert_Extended(gsSyncFbContext.sCheckpointHashTable, (void *)&ui32FwAddr, (uintptr_t) psNewSyncPt);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	/* Init Fence */
+	OSStringSafeCopy(psNewFence->pszName,
+	              pszName,
+	              SYNC_FB_FENCE_MAX_LENGTH);
+
+	psNewFence->apsFenceSyncList[0] = psNewSyncPt;
+	psNewFence->uiNumSyncs = 1;
+	FENCE_REF_SET(&psNewFence->iRef, 1, psNewFence);
+	OSAtomicWrite(&psNewFence->iStatus, PVRSRV_SYNC_NOT_SIGNALLED);
+	psNewFence->iUID = (IMG_INT64)(uintptr_t) psNewFence; /* Not unique throughout the driver lifetime */
+
+	eError = PVRSRVAllocHandle(psHandleBase,
+	                           &hOutFence,
+	                           (void*) psNewFence,
+	                           PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                           PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+	                           (PFN_HANDLE_RELEASE) &SyncFbFenceRelease);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandle", e7);
+
+	_SyncFbFenceListAdd(psNewFence);
+
+	PDUMPCOMMENTWITHFLAGS(psNewFence->psDevNode, 0,
+	                      "Allocated PVR Fence %s (ID:%"IMG_UINT64_FMTSPEC") with Checkpoint (ID:%d) "
+	                      "on Timeline %s (ID:%"IMG_UINT64_FMTSPEC")",
+	                      psNewFence->pszName,
+	                      psNewFence->iUID,
+	                      SyncCheckpointGetId(psNewSyncSignalCB->hAttachedSync),
+	                      psTl->pszName,
+	                      psTl->iUID);
+
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iTl,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+
+	*puiFenceUID = psNewFence->iUID;
+	*piOutFence = (PVRSRV_FENCE) (uintptr_t) hOutFence;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNewFence);
+
+e7:
+	SyncCheckpointFree(*ppsOutCheckpoint);
+e6:
+	_SyncFbSyncPtRelease(psNewSyncPt, IMG_FALSE);
+e5:
+	OSFreeMem(psNewFence->apsFenceSyncList);
+e4:
+	OSFreeMem(psNewSyncSignalCB);
+e3:
+	OSFreeMem(psNewFence);
+e2:
+e1:
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iTl,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+e0:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Resolve caller has to free the sync checkpoints and free the
+ * array that holds the pointers. */
+PVRSRV_ERROR SyncFbFenceResolvePVR(PSYNC_CHECKPOINT_CONTEXT psContext,
+                                   PVRSRV_FENCE iFence,
+                                   IMG_UINT32 *puiNumCheckpoints,
+                                   PSYNC_CHECKPOINT **papsCheckpoints,
+                                   IMG_UINT64 *pui64FenceUID)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psFence;
+	PVRSRV_HANDLE_BASE *psHBase;
+	PSYNC_CHECKPOINT *apsCheckpoints;
+	PSYNC_CHECKPOINT psCheckpoint;
+	PVRSRV_SYNC_SIGNAL_CB *psSyncCB, *psNewSyncCB;
+	PVRSRV_SYNC_PT *psSyncPt;
+	PDLLIST_NODE psNode;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+
+	PVR_DPF_ENTERED;
+
+	if (iFence == PVRSRV_NO_FENCE)
+	{
+		*puiNumCheckpoints = 0;
+		eError = PVRSRV_OK;
+		goto e0;
+	}
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) iFence,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                                 IMG_TRUE,
+	                                 (void**)&psFence,
+	                                 &psHBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	apsCheckpoints = OSAllocMem(sizeof(*apsCheckpoints) * psFence->uiNumSyncs);
+	PVR_LOG_GOTO_IF_NOMEM(apsCheckpoints, eError, e1);
+
+	/* Go through all syncs and add them to the list */
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		psNewSyncCB = NULL;
+		psSyncPt = psFence->apsFenceSyncList[i];
+
+		/* Don't skip signalled fences on PDUMP to make sure we set up proper
+		   synchronisation in the pdump stream. */
+#if !defined(PDUMP)
+		if (_SyncFbSyncPtHasSignalled(psSyncPt))
+		{
+			continue;
+		}
+#endif
+
+		OSLockAcquire(gsSyncFbContext.hFbContextLock);
+		OSLockAcquire(psSyncPt->psTl->hTlLock);
+		psNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psSyncCB = IMG_CONTAINER_OF(psNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+
+		/* If we have a sync checkpoint AND
+		 * it uses the same context as the given one,
+		 * just add the checkpoint to the resolve list.*/
+		if ((_SyncFbSyncPtHandleType(psSyncCB) == PVRSRV_SYNC_HANDLE_PVR) &&
+		    (psContext == (PSYNC_CHECKPOINT_CONTEXT) psSyncCB->hPrivData))
+		{
+			psCheckpoint = (PSYNC_CHECKPOINT) psSyncCB->hAttachedSync;
+		}
+		/* Else create a new sync checkpoint in the given context */
+		else
+		{
+			IMG_UINT32 ui32FwAddr;
+
+			eError = SyncCheckpointAlloc(psContext,
+			                             SYNC_CHECKPOINT_FOREIGN_CHECKPOINT,
+			                             iFence,
+			                             psFence->pszName,
+			                             &psCheckpoint);
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease(psSyncPt->psTl->hTlLock);
+				OSLockRelease(gsSyncFbContext.hFbContextLock);
+				goto e2;
+			}
+
+			psNewSyncCB = OSAllocMem(sizeof(*psNewSyncCB));
+			if (psNewSyncCB == NULL)
+			{
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				OSLockRelease(psSyncPt->psTl->hTlLock);
+				OSLockRelease(gsSyncFbContext.hFbContextLock);
+				goto e3;
+			}
+
+			psNewSyncCB->hAttachedSync = (IMG_HANDLE) psCheckpoint;
+			psNewSyncCB->hPrivData = (IMG_HANDLE) psContext;
+			psNewSyncCB->pfnSignal = &_SyncFbSyncPtSignalPVR;
+			psNewSyncCB->pfnSyncFree = &_SyncFbSyncPtFreePVR;
+			dllist_add_to_tail(&psFence->apsFenceSyncList[i]->sSignalCallbacks,
+			                   &psNewSyncCB->sCallbackNode);
+
+			/* Insert the checkpoint into the firmware address lookup table,
+			 * in case this checkpoint gets errored by the host driver. */
+			ui32FwAddr = SyncCheckpointGetFirmwareAddr(psCheckpoint);
+
+			DBG(("%s: Inserting key (fwAddr0x%x)<%p> = %p", __func__, ui32FwAddr, psCheckpoint, psSyncPt));
+			HASH_Insert_Extended(gsSyncFbContext.sCheckpointHashTable, (void *)&ui32FwAddr, (uintptr_t) psSyncPt);
+
+			/* If the existing sync pt has already been signalled, then signal
+			 * this new sync too */
+			if (_SyncFbSyncPtHasSignalled(psFence->apsFenceSyncList[i]))
+			{
+				_SyncFbSyncPtSignalPVR(psNewSyncCB->hAttachedSync, PVRSRV_SYNC_SIGNALLED);
+			}
+		}
+		OSLockRelease(psSyncPt->psTl->hTlLock);
+		OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+		/* Take a reference, resolve caller is responsible
+		 * to drop it after use */
+		eError = SyncCheckpointTakeRef(psCheckpoint);
+		PVR_GOTO_IF_ERROR(eError, e4);
+
+		apsCheckpoints[uiNumCheckpoints++] = psCheckpoint;
+	}
+
+	*pui64FenceUID = psFence->iUID;
+	*puiNumCheckpoints = uiNumCheckpoints;
+	*papsCheckpoints = apsCheckpoints;
+
+	PVRSRVReleaseHandle(psHBase,
+	                    (IMG_HANDLE) (uintptr_t) iFence,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+
+	PVR_DPF_RETURN_OK;
+
+e4:
+	if (psNewSyncCB)
+		OSFreeMem(psNewSyncCB);
+e3:
+	SyncCheckpointFree(psCheckpoint);
+e2:
+	for (; i > 0; i--)
+	{
+		SyncCheckpointDropRef(apsCheckpoints[i-1]);
+		SyncCheckpointFree(apsCheckpoints[i-1]);
+	}
+
+	OSFreeMem(apsCheckpoints);
+e1:
+	PVRSRVReleaseHandle(psHBase,
+	                    (IMG_HANDLE) (uintptr_t) iFence,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+e0:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* In case something went wrong after FenceCreate we can roll back (destroy)
+ * the fence in the server */
+static PVRSRV_ERROR SyncFbFenceRollbackPVR(PVRSRV_FENCE iFence, void *pvFenceData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+	PVR_UNREFERENCED_PARAMETER(pvFenceData);
+
+	PVR_GOTO_IF_INVALID_PARAM(iFence != PVRSRV_NO_FENCE, eError, e1);
+
+	eError = _SyncFbDestroyHandle((IMG_HANDLE) (uintptr_t) iFence,
+	                              PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	PVR_DPF_RETURN_OK;
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Dump debug info on syncs relating to any of the FWAddrs in the
+ * given array. This is called when rgx_ccb.c determines we have a
+ * stalled CCB, so this debug will aid in debugging which sync(s)
+ * have failed to signal.
+ */
+IMG_UINT32 SyncFbDumpInfoOnStalledUFOs(IMG_UINT32 nr_ufos, IMG_UINT32 *vaddrs)
+{
+	IMG_UINT32 ui32NumFallbackUfos = 0;
+	PDLLIST_NODE psFenceNode, psNextFenceNode;
+	PVRSRV_FENCE_SERVER *psFence;
+	IMG_UINT32 *pui32NextFWAddr = vaddrs;
+	IMG_UINT32 ui32CurrentUfo;
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+	for (ui32CurrentUfo=0; ui32CurrentUfo<nr_ufos; ui32CurrentUfo++)
+	{
+		if (pui32NextFWAddr)
+		{
+			/* Iterate over all fences */
+			dllist_foreach_node(&gsSyncFbContext.sFenceList,
+								psFenceNode,
+								psNextFenceNode)
+			{
+				IMG_UINT32 i;
+				IMG_BOOL bFenceDumped = IMG_FALSE;
+				IMG_UINT32			  ui32SyncPtBitmask = 0;
+				IMG_UINT32			  ui32SyncCheckpointFWAddr = 0;
+				PVRSRV_SYNC_PT		  *psSyncPt = NULL;
+
+				psFence = IMG_CONTAINER_OF(psFenceNode,
+										   PVRSRV_FENCE_SERVER,
+										   sFenceListNode);
+
+				/* ... all sync points in the fence */
+				for (i = 0; i < psFence->uiNumSyncs; i++)
+				{
+					PDLLIST_NODE psCBNode, psNextCBNode;
+
+					psSyncPt = psFence->apsFenceSyncList[i];
+
+					dllist_foreach_node(&psSyncPt->sSignalCallbacks,
+										psCBNode,
+										psNextCBNode)
+					{
+						PVRSRV_SYNC_SIGNAL_CB *psCb = IMG_CONTAINER_OF(psCBNode,
+						                                               PVRSRV_SYNC_SIGNAL_CB,
+						                                               sCallbackNode);
+
+						switch (_SyncFbSyncPtHandleType(psCb))
+						{
+							case PVRSRV_SYNC_HANDLE_PVR:
+							{
+								ui32SyncCheckpointFWAddr = SyncCheckpointGetFirmwareAddr(psCb->hAttachedSync);
+								ui32SyncPtBitmask |= 1;
+								break;
+							}
+							case PVRSRV_SYNC_HANDLE_SW:
+							{
+								ui32SyncPtBitmask |= 2;
+								break;
+							}
+							default:
+								break;
+						}
+					}
+				}
+
+				if ((ui32SyncPtBitmask == 0x3) &&
+					(ui32SyncCheckpointFWAddr == *pui32NextFWAddr))
+				{
+					/* Print fence info (if not already done so) */
+					if (!bFenceDumped)
+					{
+						PVR_LOG(("Fence: %s, ID: %"IMG_UINT64_FMTSPEC", %s - (0x%p)",
+								  psFence->pszName,
+								  psFence->iUID,
+								  _SyncFbFenceSyncsHaveSignalled(psFence) ?
+										  "Signalled" : "Pending  ",
+								  psFence));
+						bFenceDumped = IMG_TRUE;
+					}
+					_SyncFbDebugRequestPrintSyncPt(psSyncPt,
+					                               IMG_TRUE,
+												   NULL,
+												   NULL);
+					ui32NumFallbackUfos++;
+				}
+			}
+			pui32NextFWAddr++;
+		}
+	}
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	return ui32NumFallbackUfos;
+}
+
+static inline PVRSRV_SYNC_STATE _SyncFbCheckpointToSyncState(PVRSRV_SYNC_CHECKPOINT_STATE eState)
+{
+	switch (eState)
+	{
+		case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			return PVRSRV_SYNC_SIGNALLED;
+		case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			return PVRSRV_SYNC_ERRORED;
+		case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			return PVRSRV_SYNC_NOT_SIGNALLED;
+		default:
+			PVR_ASSERT("Unknown sync checkpoint state");
+			return PVRSRV_SYNC_ERRORED;
+	}
+}
+
+IMG_BOOL SyncFbCheckpointHasSignalled(IMG_UINT32 ui32FwAddr, IMG_UINT32 ui32Value)
+{
+	PVRSRV_SYNC_PT *psSyncPt;
+	PVRSRV_SYNC_STATE eSignal;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+	PDLLIST_NODE psCurrentCB, psNextCB;
+
+	PVR_DPF_ENTERED1(ui32FwAddr);
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+	DBG(("%s: Looking up key (fwAddr=0x%x)", __func__, ui32FwAddr));
+	psSyncPt = (PVRSRV_SYNC_PT *) HASH_Retrieve_Extended(gsSyncFbContext.sCheckpointHashTable, (void *) &ui32FwAddr);
+	DBG(("%s: Found sync pt <%p>", __func__, psSyncPt));
+
+	if (!psSyncPt)
+	{
+		/* Sync Fallback does not know about this UFO, skip processing it. */
+		goto err_release;
+	}
+
+	eSignal = _SyncFbCheckpointToSyncState(ui32Value);
+	_SyncFbSyncPtSignal(psSyncPt, eSignal);
+
+	/*
+	 * Signal all attachments except for the one representing the signalled
+	 * checkpoint.
+	 */
+
+	if (dllist_is_empty(&psSyncPt->sSignalCallbacks))
+	{
+		ERR("Sync pt has no attached syncs. Make sure to attach one "
+		    "when creating a new sync pt to define its flavour");
+	}
+
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks, psCurrentCB, psNextCB)
+	{
+		psCB = IMG_CONTAINER_OF(psCurrentCB,
+		                        PVRSRV_SYNC_SIGNAL_CB,
+		                        sCallbackNode);
+
+		if (_SyncFbSyncPtHandleType(psCB) == PVRSRV_SYNC_HANDLE_PVR &&
+		    SyncCheckpointGetFirmwareAddr(psCB->hAttachedSync) == ui32FwAddr)
+		{
+			continue;
+		}
+
+		if (psCB->pfnSignal(psCB->hAttachedSync, eSignal) != PVRSRV_OK)
+		{
+			ERR("Failed to signal an attached sync, system might block!");
+			/* Don't jump to exit but try to signal remaining syncs */
+		}
+	}
+
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	PVR_DPF_RETURN_RC(IMG_TRUE);
+
+err_release:
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+	PVR_DPF_RETURN_RC(IMG_FALSE);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         SW SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+/* Free a SW sync point with its sync checkpoint */
+static void _SyncFbSyncPtFreeSW(IMG_HANDLE hSync)
+{
+	PVR_DPF_ENTERED1(hSync);
+
+	OSFreeMem(hSync);
+
+	PVR_DPF_RETURN;
+}
+
+static IMG_BOOL _SyncFbSyncPtHasSignalledSW(PVRSRV_SYNC_PT *psSyncPt)
+{
+	PDLLIST_NODE psCBNode;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+	PVRSRV_SYNC_PT_SW *psSWSyncPt;
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	/* If the SyncPt has not been signalled yet,
+	 * check whether the first attached sync has.
+	 *
+	 * Change SyncPt state to signalled or errored if yes.
+	 * Also notify other attached syncs.
+	 */
+	if (OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_NOT_SIGNALLED)
+	{
+		/* List must have at least the device sync attached if we are called */
+		PVR_ASSERT(!dllist_is_empty(&psSyncPt->sSignalCallbacks));
+
+		/* Retrieve the first sync checkpoint of that sync pt */
+		psCBNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psCB = IMG_CONTAINER_OF(psCBNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+		psSWSyncPt = (PVRSRV_SYNC_PT_SW*) psCB->hAttachedSync;
+
+		if (psSWSyncPt->bSignalled)
+		{
+			_SyncFbSyncPtSignal(psSyncPt, PVRSRV_SYNC_SIGNALLED);
+
+			/* Signal all other attached syncs */
+			PVR_LOG_IF_ERROR(_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_SIGNALLED),
+			                 "_SyncFbSyncPtSignalAttached");
+
+			bRet = IMG_TRUE;
+		}
+
+		PVR_DPF_RETURN_RC1(bRet, psSyncPt);
+	}
+	else
+	{
+		PVR_DPF_RETURN_RC1(IMG_TRUE, psSyncPt);
+	}
+}
+
+/* Mark an attached sw sync pt with the given state.
+ * MAKE SURE TO WAKE UP FW AFTER CALLING THIS (if enqueued)*/
+static PVRSRV_ERROR _SyncFbSyncPtSignalSW(IMG_HANDLE hSync,
+                                          PVRSRV_SYNC_STATE eState)
+{
+	PVRSRV_SYNC_PT_SW *psSWSyncPt = (PVRSRV_SYNC_PT_SW*) hSync;
+
+	PVR_DPF_ENTERED1(hSync);
+
+	if (!psSWSyncPt->bSignalled)
+	{
+		switch (eState)
+		{
+			case PVRSRV_SYNC_SIGNALLED:
+			case PVRSRV_SYNC_ERRORED:
+				psSWSyncPt->bSignalled = IMG_TRUE;
+				break;
+			default:
+				ERR("Passed unknown sync state (%d), "
+						"please use a valid one for signalling.",
+						eState);
+				return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, hSync);
+}
+
+PVRSRV_ERROR SyncFbTimelineCreateSW(IMG_UINT32 uiTimelineNameSize,
+                                    const IMG_CHAR *pszTimelineName,
+                                    PVRSRV_TIMELINE_SERVER **ppsTimeline)
+{
+	return _SyncFbTimelineCreate(&_SyncFbSyncPtHasSignalledSW,
+	                             uiTimelineNameSize,
+	                             pszTimelineName,
+	                             ppsTimeline);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       SOFTWARE_TIMELINE FUNCTIONS                         */
+/*                                                                           */
+/*****************************************************************************/
+static PVRSRV_ERROR _SyncFbSWTimelineFenceCreate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          PVRSRV_TIMELINE_SERVER *psTl,
+                                          const IMG_CHAR *pszFenceName,
+                                          PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                          IMG_UINT64 *pui64SyncPtIdx)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	PVRSRV_SYNC_PT *psNewSyncPt = NULL;
+	PVRSRV_SYNC_PT_SW *psNewSWSyncPt;
+	PVRSRV_SYNC_SIGNAL_CB *psNewSyncSignalCB;
+	IMG_INT iNextSeqNum;
+
+	if (_SyncFbTimelineHandleType(psTl) != PVRSRV_SYNC_HANDLE_SW)
+	{
+		ERR("Passed timeline is not a SW timeline.");
+		eError = PVRSRV_ERROR_NOT_SW_TIMELINE;
+		goto e1;
+	}
+
+	/* Allocate:
+	 *		Fence
+	 *		Sync Signal CB
+	 *		SyncPt List
+	 *		SW Sync
+	 *		SyncPt
+	 *		Handle
+	 *	Setup
+	 */
+	psNewFence = OSAllocMem(sizeof(*psNewFence));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence, eError, e1);
+
+	psNewFence->psDevNode = psDeviceNode;
+
+	psNewSyncSignalCB = OSAllocMem(sizeof(*psNewSyncSignalCB));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSyncSignalCB, eError, e2);
+
+	psNewFence->apsFenceSyncList = OSAllocMem(sizeof(*(psNewFence->apsFenceSyncList)));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence->apsFenceSyncList, eError, e3);
+
+	psNewSWSyncPt = OSAllocMem(sizeof(*psNewSWSyncPt));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSWSyncPt, eError, e4);
+
+	/* Lock down TL until new point is fully created and inserted */
+	OSLockAcquire(psTl->hTlLock);
+
+	/* Sample our next sync pt value - we won't actually increment
+	 * iSeqNum for this SW timeline until we know the fence has been
+	 * successfully created.
+	 */
+	iNextSeqNum = OSAtomicRead(&psTl->iSeqNum) + 1;
+
+	eError = _SyncFbSyncPtCreate(&psNewSyncPt, psTl, iNextSeqNum);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psTl->hTlLock);
+		goto e5;
+	}
+
+	if (OSAtomicRead(&psTl->iLastSignalledSeqNum) < psNewSyncPt->uiSeqNum)
+	{
+		psNewSWSyncPt->bSignalled = IMG_FALSE;
+	}
+	else
+	{
+		psNewSWSyncPt->bSignalled = IMG_TRUE;
+		OSAtomicWrite(&psNewSyncPt->iStatus, PVRSRV_SYNC_SIGNALLED);
+	}
+
+	/* Init Sync Signal CB */
+	psNewSyncSignalCB->hAttachedSync = (IMG_HANDLE) psNewSWSyncPt;
+	psNewSyncSignalCB->pfnSignal = &_SyncFbSyncPtSignalSW;
+	psNewSyncSignalCB->pfnSyncFree = &_SyncFbSyncPtFreeSW;
+	psNewSyncSignalCB->hPrivData = NULL;
+
+	dllist_add_to_tail(&psNewSyncPt->sSignalCallbacks,
+					   &psNewSyncSignalCB->sCallbackNode);
+
+	/* Now that the fence has been created, increment iSeqNum */
+	OSAtomicIncrement(&psTl->iSeqNum);
+
+	OSLockRelease(psTl->hTlLock);
+
+	if (pszFenceName)
+	{
+		/* Init Fence */
+		OSStringSafeCopy(psNewFence->pszName,
+		              pszFenceName,
+		              SYNC_FB_FENCE_MAX_LENGTH);
+	}
+	else
+	{
+		psNewFence->pszName[0] = '\0';
+	}
+
+	psNewFence->apsFenceSyncList[0] = psNewSyncPt;
+	psNewFence->uiNumSyncs = 1;
+	FENCE_REF_SET(&psNewFence->iRef, 1, psNewFence);
+	OSAtomicWrite(&psNewFence->iStatus, PVRSRV_FENCE_NOT_SIGNALLED);
+	psNewFence->iUID = (IMG_INT64)(uintptr_t) psNewFence;
+
+	_SyncFbFenceListAdd(psNewFence);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+						  "Allocated SW Fence %s (ID:%"IMG_UINT64_FMTSPEC") with sequence number %u "
+						  "on Timeline %s (ID:%"IMG_UINT64_FMTSPEC")",
+						  psNewFence->pszName,
+						  psNewFence->iUID,
+						  psNewSyncPt->uiSeqNum,
+						  psTl->pszName,
+						  psTl->iUID);
+
+	*ppsOutputFence = psNewFence;
+	if (pui64SyncPtIdx != NULL)
+	{
+		*pui64SyncPtIdx = psNewSyncPt->uiSeqNum;
+	}
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNewFence);
+
+e5:
+	OSFreeMem(psNewSWSyncPt);
+e4:
+	OSFreeMem(psNewFence->apsFenceSyncList);
+e3:
+	OSFreeMem(psNewSyncSignalCB);
+e2:
+	OSFreeMem(psNewFence);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Kernel mode function (SyncFb implementation) to create fence on a SW timeline */
+PVRSRV_ERROR SyncFbSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           PVRSRV_TIMELINE iSWTimeline,
+                                           const IMG_CHAR *pszFenceName,
+                                           PVRSRV_FENCE *piOutputFence,
+                                           IMG_UINT64 *pui64SyncPtIdx)
+{
+
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	PVRSRV_HANDLE_BASE	*psHandleBase;
+	PVRSRV_TIMELINE_SERVER *psTl;
+	IMG_HANDLE hOutFence;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(piOutputFence, eError, e0);
+
+	/* Lookup up the ST Timeline (and take a reference on it while
+	 * we are creating the new sync pt and fence)
+	 */
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) iSWTimeline,
+									 PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+									 IMG_TRUE,
+									 (void**) &psTl,
+									 &psHandleBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	eError = _SyncFbSWTimelineFenceCreate(psDeviceNode, psTl,
+	                                      pszFenceName,
+	                                      &psNewFence,
+	                                      pui64SyncPtIdx);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	eError = PVRSRVAllocHandle(psHandleBase,
+							   &hOutFence,
+							   (void*) psNewFence,
+							   PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+							   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							   (PFN_HANDLE_RELEASE) &SyncFbFenceRelease);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	/* Drop the reference we took on the timeline earlier */
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iSWTimeline,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+
+	*piOutputFence = (PVRSRV_FENCE) (uintptr_t) hOutFence;
+
+	return PVRSRV_OK;
+
+e2:
+	/* Release the fence we created, as we failed to
+	 * allocate a handle for it */
+	SyncFbFenceRelease(psNewFence);
+
+e1:
+	/* Drop the reference we took on the timeline earlier */
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iSWTimeline,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+e0:
+	PVR_DPF_RETURN_RC(eError);
+
+}
+
+/* Client (bridge) interface to the SyncSWTimelineFenceCreateKM() function */
+PVRSRV_ERROR SyncFbFenceCreateSW(CONNECTION_DATA *psConnection,
+                                 PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 PVRSRV_TIMELINE_SERVER *psTimeline,
+                                 IMG_UINT32 uiFenceNameSize,
+                                 const IMG_CHAR *pszFenceName,
+                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                 IMG_UINT64 *pui64SyncPtIdx)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(uiFenceNameSize);
+
+	eError =  _SyncFbSWTimelineFenceCreate(psDeviceNode,
+	                                       psTimeline,
+	                                       pszFenceName,
+	                                       ppsOutputFence,
+	                                       pui64SyncPtIdx);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _SyncSWTimelineAdvanceSigErr(PVRSRV_TIMELINE_SERVER *psTl,
+                                                 PVRSRV_SYNC_STATE eState,
+                                                 IMG_UINT64 *pui64SyncPtIdx)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDLLIST_NODE psPtNode, psNextNode;
+	PVRSRV_SYNC_PT *psSyncPt;
+	IMG_INT32 uiTlSeqNum;
+
+	PVR_DPF_ENTERED1(psTl);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psTl, eError, errExit);
+
+	if (_SyncFbTimelineHandleType(psTl) != PVRSRV_SYNC_HANDLE_SW)
+	{
+		ERR("Passed timeline is not a SW timeline.");
+		eError = PVRSRV_ERROR_NOT_SW_TIMELINE;
+		goto errExit;
+	}
+
+	OSLockAcquire(psTl->hTlLock);
+
+	/* Don't allow incrementing of SW timeline beyond its last created pt */
+	if (OSAtomicRead(&psTl->iLastSignalledSeqNum) == OSAtomicRead(&psTl->iSeqNum))
+	{
+		DBG(("%s: !! TL<%p> (%d->%d/%d) !!", __func__, (void*)psTl, OSAtomicRead(&psTl->iLastSignalledSeqNum), OSAtomicRead(&psTl->iLastSignalledSeqNum)+1, OSAtomicRead(&psTl->iSeqNum)));
+		WRN("Attempt to advance SW timeline beyond last created point.");
+		eError = PVRSRV_ERROR_SW_TIMELINE_AT_LATEST_POINT;
+		goto errUnlockAndExit;
+	}
+
+	uiTlSeqNum = OSAtomicIncrement(&psTl->iLastSignalledSeqNum);
+
+	/* Go through list of active sync pts and
+	 * signal the points that are met now */
+	dllist_foreach_node(&psTl->sSyncActiveList, psPtNode, psNextNode)
+	{
+		psSyncPt = IMG_CONTAINER_OF(psPtNode,
+		                            PVRSRV_SYNC_PT,
+		                            sTlSyncActiveList);
+		if (psSyncPt->uiSeqNum <= uiTlSeqNum)
+		{
+			_SyncFbSyncPtSignal(psSyncPt, eState);
+
+			/* Signal all other attached syncs */
+			PVR_LOG_IF_ERROR(_SyncFbSyncPtSignalAttached(psSyncPt, eState),
+			                 "_SyncFbSyncPtSignalAttached");
+
+			dllist_remove_node(psPtNode);
+		}
+	}
+
+	OSLockRelease(psTl->hTlLock);
+
+	if (pui64SyncPtIdx != NULL)
+	{
+		*pui64SyncPtIdx = uiTlSeqNum;
+	}
+
+	/* A completed SW operation may un-block the GPU */
+	PVRSRVCheckStatus(NULL);
+
+	eError = _SyncFbSignalEO();
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbSignalEO", errExit);
+
+	PVR_DPF_RETURN_OK;
+
+errUnlockAndExit:
+	OSLockRelease(psTl->hTlLock);
+errExit:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static void _SyncSWTimelineCheckForUnsignalledPts(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	IMG_UINT32 ui32TlSeqNum = OSAtomicRead(&psTl->iSeqNum);
+	IMG_UINT32 ui32TlLastSigSeqNum = OSAtomicRead(&psTl->iLastSignalledSeqNum);
+
+	while (ui32TlLastSigSeqNum < ui32TlSeqNum)
+	{
+		ui32TlLastSigSeqNum++;
+		PVR_DPF((PVR_DBG_WARNING,"%s: Found unsignalled SW timeline <%p> '%s' sync pt (%d/%d)",__func__, psTl, psTl->pszName, ui32TlLastSigSeqNum, ui32TlSeqNum));
+		_SyncSWTimelineAdvanceSigErr(psTl, PVRSRV_SYNC_ERRORED, NULL);
+	}
+}
+
+/* kernel mode function to advance a SW timeline */
+PVRSRV_ERROR SyncFbSWTimelineAdvanceKM(void *pvSWTimelineObj, IMG_UINT64 *pui64SyncPtIdx)
+{
+	return _SyncSWTimelineAdvanceSigErr(pvSWTimelineObj,
+	                                    PVRSRV_SYNC_SIGNALLED,
+	                                    pui64SyncPtIdx);
+}
+
+/* Client (bridge) interface to the SyncSWTimelineAdvanceKM() function */
+PVRSRV_ERROR SyncFbTimelineAdvanceSW(PVRSRV_TIMELINE_SERVER *psTimeline,
+                                     IMG_UINT64 *pui64SyncPtIdx)
+{
+	return _SyncSWTimelineAdvanceSigErr(psTimeline,
+	                                    PVRSRV_SYNC_SIGNALLED,
+	                                    pui64SyncPtIdx);
+}
+
+PVRSRV_ERROR SyncFbFenceReleaseKM(void *pvFenceObj)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED1(pvFenceObj);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pvFenceObj, eError, e0);
+
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER*) pvFenceObj);
+
+e0:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbSWGetTimelineObj(PVRSRV_TIMELINE iSWTimeline,
+                                    void **ppvSWTimelineObj)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHB;
+
+	PVR_DPF_ENTERED1(iSWTimeline);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(iSWTimeline != PVRSRV_NO_TIMELINE, eError, err_out);
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE)(uintptr_t) iSWTimeline,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	                                 IMG_FALSE,
+	                                 ppvSWTimelineObj,
+	                                 &psHB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbLookupProcHandle", err_out);
+
+	_SyncFbTimelineAcquire((PVRSRV_TIMELINE_SERVER*) *ppvSWTimelineObj);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+PVRSRV_ERROR SyncFbGetFenceObj(PVRSRV_FENCE iFence,
+                               void **ppvFenceObj)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHB;
+
+	PVR_DPF_ENTERED1(iFence);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(iFence != PVRSRV_NO_FENCE, eError, err_out);
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE)(uintptr_t) iFence,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                                 IMG_FALSE,
+	                                 ppvFenceObj,
+	                                 &psHB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbLookupProcHandle", err_out);
+
+	_SyncFbFenceAcquire((PVRSRV_FENCE_SERVER*) *ppvFenceObj);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbDumpFenceKM(void *pvSWFenceObj,
+	                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	                           void *pvDumpDebugFile)
+{
+	PVRSRV_FENCE_SERVER *psFence = (PVRSRV_FENCE_SERVER *) pvSWFenceObj;
+	IMG_UINT32 i;
+
+	PVR_DUMPDEBUG_LOG("Fence: %s, %-9s - <%#"IMG_UINT64_FMTSPECx">",
+	                  psFence->pszName,
+	                  _SyncFbFenceSyncsHaveSignalled(psFence) ?
+	                      "Signalled" : "Active",
+	                  psFence->iUID);
+
+	/* ... all sync points in the fence */
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		_SyncFbDebugRequestPrintSyncPt(psFence->apsFenceSyncList[i],
+		                               IMG_TRUE,
+		                               pfnDumpDebugPrintf,
+		                               pvDumpDebugFile);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SyncFbSWDumpTimelineKM(void *pvSWTimelineObj,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	PVRSRV_TIMELINE_SERVER *psTl = (PVRSRV_TIMELINE_SERVER *) pvSWTimelineObj;
+
+	OSLockAcquire(psTl->hTlLock);
+
+	PVR_DUMPDEBUG_LOG("TL: %s, SeqNum: %d/%d - <%#"IMG_UINT64_FMTSPECx">",
+	                  psTl->pszName,
+	                  OSAtomicRead(&psTl->iLastSignalledSeqNum),
+	                  OSAtomicRead(&psTl->iSeqNum),
+	                  psTl->iUID);
+
+	OSLockRelease(psTl->hTlLock);
+
+	return PVRSRV_OK;
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       IMPORT/EXPORT FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+static PVRSRV_ERROR _SyncFbFenceExport(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppsExport)
+{
+	PVRSRV_FENCE_EXPORT *psExport;
+	PVRSRV_ERROR eError;
+
+	psExport = OSAllocMem(sizeof(*psExport));
+	PVR_LOG_GOTO_IF_NOMEM(psExport, eError, err_out);
+
+	_SyncFbFenceAcquire(psFence);
+
+	psExport->psFence = psFence;
+	*ppsExport = psExport;
+
+	eError = PVRSRV_OK;
+err_out:
+	return eError;
+}
+
+static PVRSRV_ERROR _SyncFbFenceExportDestroy(PVRSRV_FENCE_EXPORT *psExport)
+{
+	PVRSRV_ERROR eError;
+
+	eError = SyncFbFenceRelease(psExport->psFence);
+	PVR_LOG_IF_ERROR(eError, "SyncFbFenceRelease");
+
+	OSFreeMem(psExport);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _SyncFbFenceImport(PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFenceOut)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psFence;
+
+	psFence = psImport->psFence;
+	_SyncFbFenceAcquire(psFence);
+
+	*psFenceOut = psFence;
+
+	eError = PVRSRV_OK;
+	return eError;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR SyncFbFenceExportInsecure(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppsExport)
+{
+	return _SyncFbFenceExport(psFence, ppsExport);
+}
+
+PVRSRV_ERROR SyncFbFenceExportDestroyInsecure(PVRSRV_FENCE_EXPORT *psExport)
+{
+	return _SyncFbFenceExportDestroy(psExport);
+}
+
+PVRSRV_ERROR SyncFbFenceImportInsecure(CONNECTION_DATA *psConnection,
+                                       PVRSRV_DEVICE_NODE *psDevice,
+                                       PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFenceOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	return _SyncFbFenceImport(psImport, psFenceOut);
+}
+#endif /* defined(SUPPORT_INSECURE_EXPORT) */
+
+PVRSRV_ERROR SyncFbFenceExportDestroySecure(PVRSRV_FENCE_EXPORT *psExport)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED1(psExport);
+
+	eError = _SyncFbFenceExportDestroy(psExport);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _SyncFbReleaseSecureExport(void *pvExport)
+{
+	return SyncFbFenceExportDestroySecure(pvExport);
+}
+
+PVRSRV_ERROR SyncFbFenceExportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE * psDevNode,
+                                     PVRSRV_FENCE_SERVER *psFence,
+                                     IMG_SECURE_TYPE *phSecure,
+                                     PVRSRV_FENCE_EXPORT **ppsExport,
+                                     CONNECTION_DATA **ppsSecureConnection)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_EXPORT *psExport;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ppsSecureConnection);
+
+	eError = _SyncFbFenceExport(psFence, &psExport);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbFenceExport", err_out);
+
+	/* Transform it into a secure export */
+	eError = OSSecureExport("fallback_fence",
+	                        _SyncFbReleaseSecureExport,
+	                        (void *) psExport,
+	                        phSecure);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSecureExport", err_export);
+
+	*ppsExport = psExport;
+	PVR_DPF_RETURN_OK;
+err_export:
+	_SyncFbFenceExportDestroy(psExport);
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbFenceImportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDevice,
+                                     IMG_SECURE_TYPE hSecure,
+                                     PVRSRV_FENCE_SERVER **ppsFence)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_EXPORT *psImport;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	PVR_DPF_ENTERED1(hSecure);
+
+	eError = OSSecureImport(hSecure, (void **) &psImport);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSecureImport", err_out);
+
+	eError = _SyncFbFenceImport(psImport, ppsFence);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                            TESTING FUNCTIONS                              */
+/*                                                                           */
+/*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_server.c
new file mode 100644
index 000000000000..1746d946aca1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_server.c
@@ -0,0 +1,1230 @@
+/*************************************************************************/ /*!
+@File           sync_server.c
+@Title          Server side synchronisation functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side functions that for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "sync_server.h"
+#include "allocmem.h"
+#include "device.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pdump.h"
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "pdump_km.h"
+#include "sync.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+#include "htbserver.h"
+#include "rgxhwperf.h"
+#include "info_page.h"
+
+#include "sync_checkpoint_internal.h"
+#include "sync_checkpoint.h"
+
+/* Include this to obtain MAX_SYNC_CHECKPOINTS_PER_FENCE */
+#include "sync_checkpoint_external.h"
+
+/* Include this to obtain PVRSRV_MAX_DEV_VARS */
+#include "pvrsrv_devvar.h"
+
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "ossecure_export.h"
+#endif
+
+/* Set this to enable debug relating to the construction and maintenance of the sync address list */
+#define SYNC_ADDR_LIST_DEBUG 0
+
+/* Set maximum number of FWAddrs that can be accommodated in a SYNC_ADDR_LIST.
+ * This should allow for PVRSRV_MAX_DEV_VARS dev vars plus
+ * MAX_SYNC_CHECKPOINTS_PER_FENCE sync checkpoints for check fences.
+ * The same SYNC_ADDR_LIST is also used to hold UFOs for updates. While this
+ * may need to accommodate the additional sync prim update returned by Native
+ * sync implementation (used for timeline debug), the size calculated from
+ * PVRSRV_MAX_DEV_VARS+MAX_SYNC_CHECKPOINTS_PER_FENCE should be ample.
+ */
+#define PVRSRV_MAX_SYNC_ADDR_LIST_SIZE (PVRSRV_MAX_DEV_VARS+MAX_SYNC_CHECKPOINTS_PER_FENCE)
+/* Check that helper functions will not be preparing longer lists of
+ * UFOs than the FW can handle.
+ */
+static_assert(PVRSRV_MAX_SYNC_ADDR_LIST_SIZE <= RGXFWIF_CCB_CMD_MAX_UFOS,
+              "PVRSRV_MAX_SYNC_ADDR_LIST_SIZE > RGXFWIF_CCB_CMD_MAX_UFOS.");
+
+/* Max number of syncs allowed in a sync prim op */
+#define SYNC_PRIM_OP_MAX_SYNCS 1024
+
+struct _SYNC_PRIMITIVE_BLOCK_
+{
+	PVRSRV_DEVICE_NODE	*psDevNode;
+	DEVMEM_MEMDESC		*psMemDesc;
+	IMG_UINT32			*pui32LinAddr;
+	IMG_UINT32			ui32BlockSize;		/*!< Size of the Sync Primitive Block */
+	ATOMIC_T			sRefCount;
+	DLLIST_NODE			sConnectionNode;
+	SYNC_CONNECTION_DATA *psSyncConnectionData;	/*!< Link back to the sync connection data if there is one */
+	PRGXFWIF_UFO_ADDR		uiFWAddr;	/*!< The firmware address of the sync prim block */
+};
+
+struct _SYNC_CONNECTION_DATA_
+{
+	DLLIST_NODE	sListHead;  /*!< list of sync block associated with / created against this connection */
+	ATOMIC_T	sRefCount;  /*!< number of references to this object */
+	POS_LOCK	hLock;      /*!< lock protecting the list of sync blocks */
+};
+
+#define DECREMENT_WITH_WRAP(value, sz) ((value) ? ((value) - 1) : ((sz) - 1))
+
+/* this is the max number of syncs we will search or dump
+ * at any time.
+ */
+#define SYNC_RECORD_LIMIT 20000
+
+enum SYNC_RECORD_TYPE
+{
+	SYNC_RECORD_TYPE_UNKNOWN = 0,
+	SYNC_RECORD_TYPE_CLIENT,
+	SYNC_RECORD_TYPE_SERVER,
+};
+
+struct SYNC_RECORD
+{
+	PVRSRV_DEVICE_NODE		*psDevNode;
+	SYNC_PRIMITIVE_BLOCK	*psServerSyncPrimBlock;	/*!< handle to _SYNC_PRIMITIVE_BLOCK_ */
+	IMG_UINT32				ui32SyncOffset;			/*!< offset to sync in block */
+	IMG_UINT32				ui32FwBlockAddr;
+	IMG_PID					uiPID;
+	IMG_UINT64				ui64OSTime;
+	enum SYNC_RECORD_TYPE	eRecordType;
+	DLLIST_NODE				sNode;
+	IMG_CHAR				szClassName[PVRSRV_SYNC_NAME_LENGTH];
+};
+
+#if defined(SYNC_DEBUG) || defined(REFCOUNT_DEBUG)
+#define SYNC_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define SYNC_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+#if defined(SYNC_DEBUG)
+#define SYNC_UPDATES_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define SYNC_UPDATES_PRINT(fmt, ...)
+#endif
+
+/*!
+*****************************************************************************
+ @Function      : SyncPrimitiveBlockToFWAddr
+
+ @Description   : Given a pointer to a sync primitive block and an offset,
+                  returns the firmware address of the sync.
+
+ @Input           psSyncPrimBlock : Sync primitive block which contains the sync
+ @Input           ui32Offset      : Offset of sync within the sync primitive block
+ @Output          psAddrOut       : Absolute FW address of the sync is written out through
+                                    this pointer
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncPrimitiveBlockToFWAddr(SYNC_PRIMITIVE_BLOCK *psSyncPrimBlock,
+							IMG_UINT32 ui32Offset,
+						PRGXFWIF_UFO_ADDR *psAddrOut)
+{
+	/* check offset is legal */
+	if (unlikely((ui32Offset >= psSyncPrimBlock->ui32BlockSize) ||
+		(ui32Offset % sizeof(IMG_UINT32))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncPrimitiveBlockToFWAddr: parameters check failed"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAddrOut->ui32Addr = psSyncPrimBlock->uiFWAddr.ui32Addr + ui32Offset;
+	return PVRSRV_OK;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListGrow
+
+ @Description   : Grow the SYNC_ADDR_LIST so it can accommodate the given
+                  number of syncs, up to a maximum of PVRSRV_MAX_SYNC_PRIMS.
+
+ @Input           psList       : The SYNC_ADDR_LIST to grow
+ @Input           ui32NumSyncs : The number of sync addresses to be able to hold
+ @Return :        PVRSRV_OK on success
+*****************************************************************************/
+
+static PVRSRV_ERROR SyncAddrListGrow(SYNC_ADDR_LIST *psList, IMG_UINT32 ui32NumSyncs)
+{
+	if (unlikely(ui32NumSyncs > PVRSRV_MAX_SYNC_ADDR_LIST_SIZE))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui32NumSyncs=%u > PVRSRV_MAX_SYNC_ADDR_LIST_SIZE=%u", __func__, ui32NumSyncs, PVRSRV_MAX_SYNC_ADDR_LIST_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s:     Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	if (ui32NumSyncs > psList->ui32NumSyncs)
+	{
+		if (psList->pasFWAddrs == NULL)
+		{
+			psList->pasFWAddrs = OSAllocMem(sizeof(PRGXFWIF_UFO_ADDR) * PVRSRV_MAX_SYNC_ADDR_LIST_SIZE);
+			PVR_RETURN_IF_NOMEM(psList->pasFWAddrs);
+		}
+
+		psList->ui32NumSyncs = ui32NumSyncs;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s:     Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	return PVRSRV_OK;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListInit
+
+ @Description   : Initialise a SYNC_ADDR_LIST structure ready for use
+
+ @Input           psList        : The SYNC_ADDR_LIST structure to initialise
+ @Return        : None
+*****************************************************************************/
+
+void
+SyncAddrListInit(SYNC_ADDR_LIST *psList)
+{
+	psList->ui32NumSyncs = 0;
+	psList->pasFWAddrs   = NULL;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListDeinit
+
+ @Description   : Frees any resources associated with the given SYNC_ADDR_LIST
+
+ @Input           psList        : The SYNC_ADDR_LIST structure to deinitialise
+ @Return        : None
+*****************************************************************************/
+
+void
+SyncAddrListDeinit(SYNC_ADDR_LIST *psList)
+{
+	if (psList->pasFWAddrs != NULL)
+	{
+		OSFreeMem(psList->pasFWAddrs);
+	}
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListPopulate
+
+ @Description   : Populate the given SYNC_ADDR_LIST with the FW addresses
+                  of the syncs given by the SYNC_PRIMITIVE_BLOCKs and sync offsets
+
+ @Input           ui32NumSyncs    : The number of syncs being passed in
+ @Input           apsSyncPrimBlock: Array of pointers to SYNC_PRIMITIVE_BLOCK structures
+                                    in which the syncs are based
+ @Input           paui32SyncOffset: Array of offsets within each of the sync primitive blocks
+                                    where the syncs are located
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncAddrListPopulate(SYNC_ADDR_LIST *psList,
+						IMG_UINT32 ui32NumSyncs,
+						SYNC_PRIMITIVE_BLOCK **apsSyncPrimBlock,
+						IMG_UINT32 *paui32SyncOffset)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	if (ui32NumSyncs > psList->ui32NumSyncs)
+	{
+		eError = SyncAddrListGrow(psList, ui32NumSyncs);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	psList->ui32NumSyncs = ui32NumSyncs;
+
+	for (i = 0; i < ui32NumSyncs; i++)
+	{
+		eError = SyncPrimitiveBlockToFWAddr(apsSyncPrimBlock[i],
+								paui32SyncOffset[i],
+								&psList->pasFWAddrs[i]);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+SyncAddrListAppendSyncPrim(SYNC_ADDR_LIST          *psList,
+						   PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32FwAddr = 0;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs));
+#endif
+	/* Ensure there's room in psList for the additional sync prim update */
+	eError = SyncAddrListGrow(psList, psList->ui32NumSyncs + 1);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	SyncPrimGetFirmwareAddr(psSyncPrim, &ui32FwAddr);
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Appending sync prim <%p> UFO addr (0x%x) to psList[->pasFWAddrss[%d]", __func__, (void*)psSyncPrim, ui32FwAddr, psList->ui32NumSyncs-1));
+#endif
+	psList->pasFWAddrs[psList->ui32NumSyncs-1].ui32Addr = ui32FwAddr;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	{
+		IMG_UINT32 iii;
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: psList->ui32NumSyncs=%d", __func__, psList->ui32NumSyncs));
+		for (iii=0; iii<psList->ui32NumSyncs; iii++)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: psList->pasFWAddrs[%d].ui32Addr=0x%x", __func__, iii, psList->pasFWAddrs[iii].ui32Addr));
+		}
+	}
+#endif
+e0:
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d", __func__, (void*)psList, psList->ui32NumSyncs));
+#endif
+	return eError;
+}
+
+
+static PVRSRV_ERROR
+_AppendCheckpoints(SYNC_ADDR_LIST *psList,
+				   IMG_UINT32 ui32NumCheckpoints,
+				   PSYNC_CHECKPOINT *apsSyncCheckpoint,
+				   IMG_BOOL bDeRefCheckpoints)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32SyncCheckpointIndex;
+	IMG_UINT32 ui32RollbackSize = psList->ui32NumSyncs;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+#endif
+	/* Ensure there's room in psList for the sync checkpoints */
+	eError = SyncAddrListGrow(psList, psList->ui32NumSyncs + ui32NumCheckpoints);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: * * * * ERROR * * * * Trying to SyncAddrListGrow(psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+		goto e0;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: (ui32NumCheckpoints=%d) (psList->ui32NumSyncs is now %d) array already contains %d FWAddrs:", __func__, ui32NumCheckpoints, psList->ui32NumSyncs, ui32RollbackSize));
+	if (ui32RollbackSize > 0)
+	{
+		{
+			IMG_UINT32 kk;
+			for (kk=0; kk<ui32RollbackSize; kk++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:    <%p>psList->pasFWAddrs[%d].ui32Addr = %u(0x%x)", __func__,
+						 (void*)&psList->pasFWAddrs[kk], kk,
+						 psList->pasFWAddrs[kk].ui32Addr, psList->pasFWAddrs[kk].ui32Addr));
+			}
+		}
+	}
+	PVR_DPF((PVR_DBG_ERROR, "%s: apsSyncCheckpoint=<%p>, apsSyncCheckpoint[0] = <%p>", __func__, (void*)apsSyncCheckpoint, (void*)apsSyncCheckpoint[0]));
+#endif
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		psList->pasFWAddrs[ui32RollbackSize + ui32SyncCheckpointIndex].ui32Addr = SyncCheckpointGetFirmwareAddr(apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s:  SyncCheckpointCCBEnqueued(<%p>)", __func__, (void*)apsSyncCheckpoint[ui32SyncCheckpointIndex]));
+		PVR_DPF((PVR_DBG_ERROR, "%s:                           ID:%d", __func__, SyncCheckpointGetId((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex])));
+#endif
+		SyncCheckpointCCBEnqueued((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		if (bDeRefCheckpoints)
+		{
+			/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+			SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		}
+	}
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	if (psList->ui32NumSyncs > 0)
+	{
+		IMG_UINT32 kk;
+		for (kk=0; kk<psList->ui32NumSyncs; kk++)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s:    <%p>psList->pasFWAddrs[%d].ui32Addr = %u(0x%x)", __func__,
+			         (void*)&psList->pasFWAddrs[kk], kk,
+			         psList->pasFWAddrs[kk].ui32Addr, psList->pasFWAddrs[kk].ui32Addr));
+		}
+	}
+#endif
+	return eError;
+
+e0:
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		if (bDeRefCheckpoints)
+		{
+			/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+			SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		}
+	}
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+#endif
+	return eError;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListAppendCheckpoints
+
+ @Description   : Append the FW addresses of the sync checkpoints given in
+                  the PSYNC_CHECKPOINTs array to the given SYNC_ADDR_LIST
+
+ @Input           ui32NumSyncCheckpoints : The number of sync checkpoints
+                                           being passed in
+ @Input           apsSyncCheckpoint : Array of PSYNC_CHECKPOINTs whose details
+                                      are to be appended to the SYNC_ADDR_LIST
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+PVRSRV_ERROR
+SyncAddrListAppendCheckpoints(SYNC_ADDR_LIST *psList,
+								IMG_UINT32 ui32NumCheckpoints,
+								PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	return _AppendCheckpoints(psList, ui32NumCheckpoints, apsSyncCheckpoint, IMG_FALSE);
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListAppendAndDeRefCheckpoints
+
+ @Description   : Append the FW addresses of the sync checkpoints given in
+                  the PSYNC_CHECKPOINTs array to the given SYNC_ADDR_LIST.
+                  A reference is dropped for each of the checkpoints.
+
+ @Input           ui32NumSyncCheckpoints : The number of sync checkpoints
+                                           being passed in
+ @Input           apsSyncCheckpoint : Array of PSYNC_CHECKPOINTs whose details
+                                      are to be appended to the SYNC_ADDR_LIST
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+PVRSRV_ERROR
+SyncAddrListAppendAndDeRefCheckpoints(SYNC_ADDR_LIST *psList,
+									  IMG_UINT32 ui32NumCheckpoints,
+									  PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	return _AppendCheckpoints(psList, ui32NumCheckpoints, apsSyncCheckpoint, IMG_TRUE);
+}
+
+void
+SyncAddrListDeRefCheckpoints(IMG_UINT32 ui32NumCheckpoints,
+							 PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	IMG_UINT32 ui32SyncCheckpointIndex;
+
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+		SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+	}
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListRollbackCheckpoints
+
+ @Description   : Rollback the enqueued count of each sync checkpoint in
+                  the given SYNC_ADDR_LIST. This needs to be done in the
+                  event of the kick call failing, so that the reference
+                  taken on each sync checkpoint on the firmware's behalf
+                  is dropped.
+
+ @Input           psList        : The SYNC_ADDR_LIST structure containing
+                                  sync checkpoints to be rolled back
+
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncAddrListRollbackCheckpoints(PVRSRV_DEVICE_NODE *psDevNode, SYNC_ADDR_LIST *psList)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32SyncIndex;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: called (psList=<%p>)", __func__, (void*)psList));
+#endif
+	if (psList)
+	{
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: psList->ui32NumSyncs=%d", __func__, psList->ui32NumSyncs));
+#endif
+		for (ui32SyncIndex=0; ui32SyncIndex<psList->ui32NumSyncs; ui32SyncIndex++)
+		{
+			if (psList->pasFWAddrs[ui32SyncIndex].ui32Addr & 0x1)
+			{
+				SyncCheckpointRollbackFromUFO(psDevNode, psList->pasFWAddrs[ui32SyncIndex].ui32Addr);
+			}
+		}
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncRecordAddKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDevNode,
+					  SYNC_RECORD_HANDLE *phRecord,
+					  SYNC_PRIMITIVE_BLOCK *hServerSyncPrimBlock,
+					  IMG_UINT32 ui32FwBlockAddr,
+					  IMG_UINT32 ui32SyncOffset,
+					  IMG_BOOL bServerSync,
+					  IMG_UINT32 ui32ClassNameSize,
+					  const IMG_CHAR *pszClassName)
+{
+	struct SYNC_RECORD * psSyncRec;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	RGXSRV_HWPERF_ALLOC(psDevNode, SYNC,
+	                    ui32FwBlockAddr + ui32SyncOffset,
+	                    pszClassName,
+	                    ui32ClassNameSize);
+
+	PVR_RETURN_IF_INVALID_PARAM(phRecord);
+
+	*phRecord = NULL;
+
+	psSyncRec = OSAllocMem(sizeof(*psSyncRec));
+	PVR_GOTO_IF_NOMEM(psSyncRec, eError, fail_alloc);
+
+	psSyncRec->psDevNode = psDevNode;
+	psSyncRec->psServerSyncPrimBlock = hServerSyncPrimBlock;
+	psSyncRec->ui32SyncOffset = ui32SyncOffset;
+	psSyncRec->ui32FwBlockAddr = ui32FwBlockAddr;
+	psSyncRec->ui64OSTime = OSClockns64();
+	psSyncRec->uiPID = OSGetCurrentProcessID();
+	psSyncRec->eRecordType = bServerSync? SYNC_RECORD_TYPE_SERVER: SYNC_RECORD_TYPE_CLIENT;
+
+	if (pszClassName)
+	{
+		if (ui32ClassNameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			ui32ClassNameSize = PVRSRV_SYNC_NAME_LENGTH;
+		/* Copy over the class name annotation */
+		OSStringSafeCopy(psSyncRec->szClassName, pszClassName, ui32ClassNameSize);
+	}
+	else
+	{
+		/* No class name annotation */
+		psSyncRec->szClassName[0] = 0;
+	}
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	if (psDevNode->ui32SyncServerRecordCount < SYNC_RECORD_LIMIT)
+	{
+		dllist_add_to_head(&psDevNode->sSyncServerRecordList, &psSyncRec->sNode);
+		psDevNode->ui32SyncServerRecordCount++;
+
+		if (psDevNode->ui32SyncServerRecordCount > psDevNode->ui32SyncServerRecordCountHighWatermark)
+		{
+			psDevNode->ui32SyncServerRecordCountHighWatermark = psDevNode->ui32SyncServerRecordCount;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync record \"%s\". %u records already exist.",
+											__func__,
+											pszClassName,
+											psDevNode->ui32SyncServerRecordCount));
+		OSFreeMem(psSyncRec);
+		psSyncRec = NULL;
+		eError = PVRSRV_ERROR_TOOMANYBUFFERS;
+	}
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	*phRecord = (SYNC_RECORD_HANDLE)psSyncRec;
+
+fail_alloc:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncRecordRemoveByHandleKM(
+			SYNC_RECORD_HANDLE hRecord)
+{
+	struct SYNC_RECORD **ppFreedSync;
+	struct SYNC_RECORD *pSync = (struct SYNC_RECORD*)hRecord;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRecord);
+
+	psDevNode = pSync->psDevNode;
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+
+	RGXSRV_HWPERF_FREE(psDevNode, SYNC, pSync->ui32FwBlockAddr + pSync->ui32SyncOffset);
+
+	dllist_remove_node(&pSync->sNode);
+
+	if (psDevNode->uiSyncServerRecordFreeIdx >= PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: freed sync record index out of range",
+				 __func__));
+		psDevNode->uiSyncServerRecordFreeIdx = 0;
+	}
+	ppFreedSync = &psDevNode->apsSyncServerRecordsFreed[psDevNode->uiSyncServerRecordFreeIdx];
+	psDevNode->uiSyncServerRecordFreeIdx =
+		(psDevNode->uiSyncServerRecordFreeIdx + 1) % PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN;
+
+	if (*ppFreedSync)
+	{
+		OSFreeMem(*ppFreedSync);
+	}
+	pSync->psServerSyncPrimBlock = NULL;
+	pSync->ui64OSTime = OSClockns64();
+	*ppFreedSync = pSync;
+
+	psDevNode->ui32SyncServerRecordCount--;
+
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncAllocEventKM(CONNECTION_DATA *psConnection,
+			PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_BOOL bServerSync,
+			IMG_UINT32 ui32FWAddr,
+			IMG_UINT32 ui32ClassNameSize,
+			const IMG_CHAR *pszClassName)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	RGXSRV_HWPERF_ALLOC(psDevNode, SYNC, ui32FWAddr, pszClassName, ui32ClassNameSize);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncFreeEventKM(CONNECTION_DATA *psConnection,
+			PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 ui32FWAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	RGXSRV_HWPERF_FREE(psDevNode, SYNC, ui32FWAddr);
+
+	return PVRSRV_OK;
+}
+
+static
+void _SyncConnectionRef(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psSyncConnectionData->sRefCount);
+
+	SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+						__func__, psSyncConnectionData, iRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+static
+void _SyncConnectionUnref(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psSyncConnectionData->sRefCount);
+	if (iRefCount == 0)
+	{
+		SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+		                    __func__, psSyncConnectionData, iRefCount);
+
+		PVR_ASSERT(dllist_is_empty(&psSyncConnectionData->sListHead));
+		OSLockDestroy(psSyncConnectionData->hLock);
+		OSFreeMem(psSyncConnectionData);
+	}
+	else
+	{
+		SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+		                    __func__, psSyncConnectionData, iRefCount);
+		PVR_ASSERT(iRefCount > 0);
+	}
+}
+
+static
+void _SyncConnectionAddBlock(CONNECTION_DATA *psConnection, SYNC_PRIMITIVE_BLOCK *psBlock)
+{
+	if (psConnection)
+	{
+		SYNC_CONNECTION_DATA *psSyncConnectionData = psConnection->psSyncConnectionData;
+
+		/*
+			Make sure the connection doesn't go away. It doesn't matter that we will release
+			the lock between as the refcount and list don't have to be atomic w.r.t. to each other
+		*/
+		_SyncConnectionRef(psSyncConnectionData);
+
+		OSLockAcquire(psSyncConnectionData->hLock);
+		dllist_add_to_head(&psSyncConnectionData->sListHead, &psBlock->sConnectionNode);
+		OSLockRelease(psSyncConnectionData->hLock);
+		psBlock->psSyncConnectionData = psSyncConnectionData;
+	}
+	else
+	{
+		psBlock->psSyncConnectionData = NULL;
+	}
+}
+
+static
+void _SyncConnectionRemoveBlock(SYNC_PRIMITIVE_BLOCK *psBlock)
+{
+	SYNC_CONNECTION_DATA *psSyncConnectionData = psBlock->psSyncConnectionData;
+
+	if (psBlock->psSyncConnectionData)
+	{
+		OSLockAcquire(psSyncConnectionData->hLock);
+		dllist_remove_node(&psBlock->sConnectionNode);
+		OSLockRelease(psSyncConnectionData->hLock);
+
+		_SyncConnectionUnref(psBlock->psSyncConnectionData);
+	}
+}
+
+static inline
+void _DoPrimBlockFree(SYNC_PRIMITIVE_BLOCK *psSyncBlk)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psSyncBlk->psDevNode;
+
+	SYNC_REFCOUNT_PRINT("%s: Sync block %p, refcount = %d (remove)",
+	                    __func__, psSyncBlk, OSAtomicRead(&psSyncBlk->sRefCount));
+
+	PVR_ASSERT(OSAtomicRead(&psSyncBlk->sRefCount) == 1);
+
+	_SyncConnectionRemoveBlock(psSyncBlk);
+	DevmemReleaseCpuVirtAddr(psSyncBlk->psMemDesc);
+	psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->psMemDesc);
+	OSFreeMem(psSyncBlk);
+}
+
+PVRSRV_ERROR
+PVRSRVAllocSyncPrimitiveBlockKM(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+								SYNC_PRIMITIVE_BLOCK **ppsSyncBlk,
+								IMG_UINT32 *puiSyncPrimVAddr,
+								IMG_UINT32 *puiSyncPrimBlockSize,
+								PMR        **ppsSyncPMR)
+{
+	SYNC_PRIMITIVE_BLOCK *psNewSyncBlk;
+	PVRSRV_ERROR eError;
+
+	psNewSyncBlk = OSAllocMem(sizeof(SYNC_PRIMITIVE_BLOCK));
+	PVR_GOTO_IF_NOMEM(psNewSyncBlk, eError, e0);
+
+	psNewSyncBlk->psDevNode = psDevNode;
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS, "Allocate UFO block");
+
+	eError = psDevNode->pfnAllocUFOBlock(psDevNode,
+										 sizeof(IMG_UINT32),
+										 &psNewSyncBlk->psMemDesc,
+										 &psNewSyncBlk->uiFWAddr.ui32Addr,
+										 &psNewSyncBlk->ui32BlockSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = psNewSyncBlk->uiFWAddr.ui32Addr;
+
+	eError = DevmemAcquireCpuVirtAddr(psNewSyncBlk->psMemDesc,
+									  (void **) &psNewSyncBlk->pui32LinAddr);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	eError = DevmemLocalGetImportHandle(psNewSyncBlk->psMemDesc, (void **) ppsSyncPMR);
+
+	PVR_GOTO_IF_ERROR(eError, e3);
+
+	OSAtomicWrite(&psNewSyncBlk->sRefCount, 1);
+
+	/* If there is a connection pointer then add the new block onto it's list */
+	_SyncConnectionAddBlock(psConnection, psNewSyncBlk);
+
+	*ppsSyncBlk = psNewSyncBlk;
+	*puiSyncPrimBlockSize = psNewSyncBlk->ui32BlockSize;
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+						  "Allocated UFO block (FirmwareVAddr = 0x%08x)",
+						  *puiSyncPrimVAddr);
+
+	return PVRSRV_OK;
+
+e3:
+	DevmemReleaseCpuVirtAddr(psNewSyncBlk->psMemDesc);
+e2:
+	psDevNode->pfnFreeUFOBlock(psDevNode, psNewSyncBlk->psMemDesc);
+e1:
+	OSFreeMem(psNewSyncBlk);
+e0:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVFreeSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk)
+{
+
+	/* This function is an alternative to the above without reference counting.
+	 * With the removal of sync prim ops for server syncs we no longer have to
+	 * reference count prim blocks as the reference will never be incremented /
+	 * decremented by a prim op */
+	_DoPrimBlockFree(psSyncBlk);
+	return PVRSRV_OK;
+}
+
+static INLINE IMG_BOOL _CheckSyncIndex(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+                                       IMG_UINT32 ui32Index)
+{
+	if (psSyncBlk->ui32BlockSize == 0)
+	{
+		return IMG_FALSE;
+	}
+
+	return (ui32Index < psSyncBlk->ui32BlockSize / sizeof(IMG_UINT32));
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimSetKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+                    IMG_UINT32 ui32Index,
+                    IMG_UINT32 ui32Value)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncBlk != NULL, "psSyncBlk");
+
+	if (_CheckSyncIndex(psSyncBlk, ui32Index))
+	{
+		psSyncBlk->pui32LinAddr[ui32Index] = ui32Value;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVSyncPrimSetKM: Index %u out of range for "
+		         "0x%08X byte sync block (value 0x%08X)",
+		         ui32Index,
+		         psSyncBlk->ui32BlockSize,
+		         ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	*/
+	DevmemPDumpLoadMemValue32(psSyncBlk->psMemDesc,
+					   ui32Offset,
+					   ui32Value,
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset)
+{
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	*/
+	DevmemPDumpLoadMem(psSyncBlk->psMemDesc,
+					   ui32Offset,
+					   sizeof(IMG_UINT32),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	DevmemPDumpDevmemPol32(psSyncBlk->psMemDesc,
+						   ui32Offset,
+						   ui32Value,
+						   ui32Mask,
+						   eOperator,
+						   ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize)
+{
+	DevmemPDumpCBP(psSyncBlk->psMemDesc,
+				   ui32Offset,
+				   uiWriteOffset,
+				   uiPacketSize,
+				   uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif
+
+/* SyncRegisterConnection */
+PVRSRV_ERROR SyncRegisterConnection(SYNC_CONNECTION_DATA **ppsSyncConnectionData)
+{
+	SYNC_CONNECTION_DATA *psSyncConnectionData;
+	PVRSRV_ERROR eError;
+
+	psSyncConnectionData = OSAllocMem(sizeof(SYNC_CONNECTION_DATA));
+	if (psSyncConnectionData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+
+	eError = OSLockCreate(&psSyncConnectionData->hLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lockcreate);
+	dllist_init(&psSyncConnectionData->sListHead);
+	OSAtomicWrite(&psSyncConnectionData->sRefCount, 1);
+
+	*ppsSyncConnectionData = psSyncConnectionData;
+	return PVRSRV_OK;
+
+fail_lockcreate:
+	OSFreeMem(psSyncConnectionData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* SyncUnregisterConnection */
+void SyncUnregisterConnection(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	_SyncConnectionUnref(psSyncConnectionData);
+}
+
+void SyncConnectionPDumpSyncBlocks(PVRSRV_DEVICE_NODE *psDevNode, void *hSyncPrivData, PDUMP_TRANSITION_EVENT eEvent)
+{
+	if ((eEvent == PDUMP_TRANSITION_EVENT_RANGE_ENTERED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		SYNC_CONNECTION_DATA *psSyncConnectionData = hSyncPrivData;
+		DLLIST_NODE *psNode, *psNext;
+
+		OSLockAcquire(psSyncConnectionData->hLock);
+
+		PDUMPCOMMENT(psDevNode, "Dump client Sync Prim state");
+		dllist_foreach_node(&psSyncConnectionData->sListHead, psNode, psNext)
+		{
+			SYNC_PRIMITIVE_BLOCK *psSyncBlock =
+				IMG_CONTAINER_OF(psNode, SYNC_PRIMITIVE_BLOCK, sConnectionNode);
+
+			DevmemPDumpLoadMem(psSyncBlock->psMemDesc,
+					0,
+					psSyncBlock->ui32BlockSize,
+					PDUMP_FLAGS_CONTINUOUS);
+		}
+
+		OSLockRelease(psSyncConnectionData->hLock);
+	}
+}
+
+void SyncRecordLookup(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 ui32FwAddr,
+					  IMG_CHAR * pszSyncInfo, size_t len)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_INT iEnd;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (!pszSyncInfo)
+	{
+		return;
+	}
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	pszSyncInfo[0] = '\0';
+
+	dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+	{
+		struct SYNC_RECORD *psSyncRec =
+			IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+		if ((psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset) == ui32FwAddr
+			&& SYNC_RECORD_TYPE_UNKNOWN != psSyncRec->eRecordType
+			&& psSyncRec->psServerSyncPrimBlock
+			&& psSyncRec->psServerSyncPrimBlock->pui32LinAddr
+			)
+		{
+			IMG_UINT32 *pui32SyncAddr;
+			pui32SyncAddr = psSyncRec->psServerSyncPrimBlock->pui32LinAddr
+				+ (psSyncRec->ui32SyncOffset/sizeof(IMG_UINT32));
+			iEnd = OSSNPrintf(pszSyncInfo, len, "Cur=0x%08x %s:%05u (%s)",
+				*pui32SyncAddr,
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				psSyncRec->szClassName
+				);
+			if (iEnd >= 0 && iEnd < len)
+			{
+				pszSyncInfo[iEnd] = '\0';
+			}
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	if (!bFound && (psDevNode->ui32SyncServerRecordCountHighWatermark == SYNC_RECORD_LIMIT))
+	{
+		OSSNPrintf(pszSyncInfo, len, "(Record may be lost)");
+	}
+}
+
+#define NS_IN_S (1000000000UL)
+static void _SyncRecordPrint(struct SYNC_RECORD *psSyncRec,
+					IMG_UINT64 ui64TimeNow,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	SYNC_PRIMITIVE_BLOCK *psSyncBlock = psSyncRec->psServerSyncPrimBlock;
+
+	if (SYNC_RECORD_TYPE_UNKNOWN != psSyncRec->eRecordType)
+	{
+		IMG_UINT64 ui64DeltaS;
+		IMG_UINT32 ui32DeltaF;
+		IMG_UINT64 ui64Delta = ui64TimeNow - psSyncRec->ui64OSTime;
+		ui64DeltaS = OSDivide64(ui64Delta, NS_IN_S, &ui32DeltaF);
+
+		if (psSyncBlock && psSyncBlock->pui32LinAddr)
+		{
+			IMG_UINT32 *pui32SyncAddr;
+			pui32SyncAddr = psSyncBlock->pui32LinAddr
+				+ (psSyncRec->ui32SyncOffset/sizeof(IMG_UINT32));
+
+			PVR_DUMPDEBUG_LOG("\t%s %05u %05" IMG_UINT64_FMTSPEC ".%09u FWAddr=0x%08x Val=0x%08x (%s)",
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				ui64DeltaS, ui32DeltaF,
+				(psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset),
+				*pui32SyncAddr,
+				psSyncRec->szClassName
+				);
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("\t%s %05u %05" IMG_UINT64_FMTSPEC ".%09u FWAddr=0x%08x Val=<null_ptr> (%s)",
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				ui64DeltaS, ui32DeltaF,
+				(psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset),
+				psSyncRec->szClassName
+				);
+		}
+	}
+}
+
+static void _SyncRecordRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	IMG_UINT64 ui64TimeNowS;
+	IMG_UINT32 ui32TimeNowF;
+	IMG_UINT64 ui64TimeNow = OSClockns64();
+	DLLIST_NODE *psNode, *psNext;
+
+	ui64TimeNowS = OSDivide64(ui64TimeNow, NS_IN_S, &ui32TimeNowF);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+		OSLockAcquire(psDevNode->hSyncServerRecordLock);
+
+		PVR_DUMPDEBUG_LOG("Dumping all allocated syncs. Allocated: %u High watermark: %u @ %05" IMG_UINT64_FMTSPEC ".%09u",
+										psDevNode->ui32SyncServerRecordCount,
+										psDevNode->ui32SyncServerRecordCountHighWatermark,
+										ui64TimeNowS,
+										ui32TimeNowF);
+		if (psDevNode->ui32SyncServerRecordCountHighWatermark == SYNC_RECORD_LIMIT)
+		{
+			PVR_DUMPDEBUG_LOG("Warning: Record limit (%u) was reached. Some sync checkpoints may not have been recorded in the debug information.",
+                                                                                                                SYNC_RECORD_LIMIT);
+		}
+
+		PVR_DUMPDEBUG_LOG("\t%-6s %-5s %-15s %-17s %-14s (%s)",
+					"Type", "PID", "Time Delta (s)", "Address", "Value", "Annotation");
+
+		dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+		{
+			struct SYNC_RECORD *psSyncRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+			_SyncRecordPrint(psSyncRec, ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+
+		PVR_DUMPDEBUG_LOG("Dumping all recently freed syncs @ %05" IMG_UINT64_FMTSPEC ".%09u",
+						  ui64TimeNowS, ui32TimeNowF);
+		PVR_DUMPDEBUG_LOG("\t%-6s %-5s %-15s %-17s %-14s (%s)",
+					"Type", "PID", "Time Delta (s)", "Address", "Value", "Annotation");
+		for (i = DECREMENT_WITH_WRAP(psDevNode->uiSyncServerRecordFreeIdx, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN);
+			 i != psDevNode->uiSyncServerRecordFreeIdx;
+			 i = DECREMENT_WITH_WRAP(i, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN))
+		{
+			if (psDevNode->apsSyncServerRecordsFreed[i])
+			{
+				_SyncRecordPrint(psDevNode->apsSyncServerRecordsFreed[i],
+								 ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+			else
+			{
+				break;
+			}
+		}
+
+		OSLockRelease(psDevNode->hSyncServerRecordLock);
+	}
+}
+#undef NS_IN_S
+
+static PVRSRV_ERROR SyncRecordListInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	psDevNode->ui32SyncServerRecordCount = 0;
+	psDevNode->ui32SyncServerRecordCountHighWatermark = 0;
+
+	eError = OSLockCreate(&psDevNode->hSyncServerRecordLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lock_create);
+	dllist_init(&psDevNode->sSyncServerRecordList);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncServerRecordNotify,
+		                                          psDevNode,
+		                                          _SyncRecordRequest,
+		                                          DEBUG_REQUEST_SYNCTRACKING,
+		                                          psDevNode);
+
+	PVR_GOTO_IF_ERROR(eError, fail_dbg_register);
+
+	return PVRSRV_OK;
+
+fail_dbg_register:
+	OSLockDestroy(psDevNode->hSyncServerRecordLock);
+fail_lock_create:
+	return eError;
+}
+
+static void SyncRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DLLIST_NODE *psNode, *psNext;
+	int i;
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+	{
+		struct SYNC_RECORD *pSyncRec =
+			IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+
+		dllist_remove_node(psNode);
+		OSFreeMem(pSyncRec);
+	}
+
+	for (i = 0; i < PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN; i++)
+	{
+		if (psDevNode->apsSyncServerRecordsFreed[i])
+		{
+			OSFreeMem(psDevNode->apsSyncServerRecordsFreed[i]);
+			psDevNode->apsSyncServerRecordsFreed[i] = NULL;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	if (psDevNode->hSyncServerRecordNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncServerRecordNotify);
+	}
+	OSLockDestroy(psDevNode->hSyncServerRecordLock);
+}
+
+PVRSRV_ERROR SyncServerInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		eError = SyncRecordListInit(psDevNode);
+		PVR_GOTO_IF_ERROR(eError, fail_record_list);
+	}
+
+	return PVRSRV_OK;
+
+fail_record_list:
+	return eError;
+}
+
+void SyncServerDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		SyncRecordListDeinit(psDevNode);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/tlintern.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlintern.c
new file mode 100644
index 000000000000..3bd6b0af1718
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlintern.c
@@ -0,0 +1,442 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer functions available to driver components in
+                the driver.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "allocmem.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "devicemem.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "tlintern.h"
+
+/*
+ * Make functions
+ */
+PTL_STREAM_DESC
+TLMakeStreamDesc(PTL_SNODE f1, IMG_UINT32 f2, IMG_HANDLE f3)
+{
+	PTL_STREAM_DESC ps = OSAllocZMem(sizeof(TL_STREAM_DESC));
+	if (ps == NULL)
+	{
+		return NULL;
+	}
+	ps->psNode = f1;
+	ps->ui32Flags = f2;
+	ps->hReadEvent = f3;
+	ps->uiRefCount = 1;
+
+	if (f2 & PVRSRV_STREAM_FLAG_READ_LIMIT)
+	{
+		ps->ui32ReadLimit = f1->psStream->ui32Write;
+	}
+	return ps;
+}
+
+PTL_SNODE
+TLMakeSNode(IMG_HANDLE f2, TL_STREAM *f3, TL_STREAM_DESC *f4)
+{
+	PTL_SNODE ps = OSAllocZMem(sizeof(TL_SNODE));
+	if (ps == NULL)
+	{
+		return NULL;
+	}
+	ps->hReadEventObj = f2;
+	ps->psStream = f3;
+	ps->psRDesc = f4;
+	f3->psNode = ps;
+	return ps;
+}
+
+/*
+ * Transport Layer Global top variables and functions
+ */
+static TL_GLOBAL_DATA sTLGlobalData;
+
+TL_GLOBAL_DATA *TLGGD(void) /* TLGetGlobalData() */
+{
+	return &sTLGlobalData;
+}
+
+/* TLInit must only be called once at driver initialisation.
+ * An assert is provided to check this condition on debug builds.
+ */
+PVRSRV_ERROR
+TLInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(sTLGlobalData.hTLGDLock == NULL && sTLGlobalData.hTLEventObj == NULL);
+
+	/* Allocate a lock for TL global data, to be used while updating the TL data.
+	 * This is for making TL global data multi-thread safe */
+	eError = OSLockCreate(&sTLGlobalData.hTLGDLock);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	/* Allocate the event object used to signal global TL events such as
+	 * a new stream created */
+	eError = OSEventObjectCreate("TLGlobalEventObj", &sTLGlobalData.hTLEventObj);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	PVR_DPF_RETURN_OK;
+
+/* Don't allow the driver to start up on error */
+e1:
+	OSLockDestroy (sTLGlobalData.hTLGDLock);
+	sTLGlobalData.hTLGDLock = NULL;
+e0:
+	PVR_DPF_RETURN_RC (eError);
+}
+
+static void RemoveAndFreeStreamNode(PTL_SNODE psRemove)
+{
+	TL_GLOBAL_DATA*  psGD = TLGGD();
+	PTL_SNODE*       last;
+	PTL_SNODE        psn;
+	PVRSRV_ERROR     eError;
+
+	PVR_DPF_ENTERED;
+
+	/* Unlink the stream node from the master list */
+	PVR_ASSERT(psGD->psHead);
+	last = &psGD->psHead;
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn == psRemove)
+		{
+			/* Other calling code may have freed and zeroed the pointers */
+			if (psn->psRDesc)
+			{
+				OSFreeMem(psn->psRDesc);
+				psn->psRDesc = NULL;
+			}
+			if (psn->psStream)
+			{
+				OSFreeMem(psn->psStream);
+				psn->psStream = NULL;
+			}
+			*last = psn->psNext;
+			break;
+		}
+		last = &psn->psNext;
+	}
+
+	/* Release the event list object owned by the stream node */
+	if (psRemove->hReadEventObj)
+	{
+		eError = OSEventObjectDestroy(psRemove->hReadEventObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+		psRemove->hReadEventObj = NULL;
+	}
+
+	/* Release the memory of the stream node */
+	OSFreeMem(psRemove);
+
+	PVR_DPF_RETURN;
+}
+
+static void FreeGlobalData(void)
+{
+	PTL_SNODE psCurrent = sTLGlobalData.psHead;
+	PTL_SNODE psNext;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	/* Clean up the SNODE list */
+	if (psCurrent)
+	{
+		while (psCurrent)
+		{
+			psNext = psCurrent->psNext;
+
+			/* Other calling code may have freed and zeroed the pointers */
+			if (psCurrent->psRDesc)
+			{
+				OSFreeMem(psCurrent->psRDesc);
+				psCurrent->psRDesc = NULL;
+			}
+			if (psCurrent->psStream)
+			{
+				OSFreeMem(psCurrent->psStream);
+				psCurrent->psStream = NULL;
+			}
+
+			/* Release the event list object owned by the stream node */
+			if (psCurrent->hReadEventObj)
+			{
+				eError = OSEventObjectDestroy(psCurrent->hReadEventObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+				psCurrent->hReadEventObj = NULL;
+			}
+
+			OSFreeMem(psCurrent);
+			psCurrent = psNext;
+		}
+
+		sTLGlobalData.psHead = NULL;
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void
+TLDeInit(void)
+{
+	PVR_DPF_ENTERED;
+
+	if (sTLGlobalData.uiClientCnt)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TLDeInit transport layer but %d client streams are still connected", sTLGlobalData.uiClientCnt));
+		sTLGlobalData.uiClientCnt = 0;
+	}
+
+	FreeGlobalData();
+
+	/* Clean up the TL global event object */
+	if (sTLGlobalData.hTLEventObj)
+	{
+		OSEventObjectDestroy(sTLGlobalData.hTLEventObj);
+		sTLGlobalData.hTLEventObj = NULL;
+	}
+
+	/* Destroy the TL global data lock */
+	if (sTLGlobalData.hTLGDLock)
+	{
+		OSLockDestroy (sTLGlobalData.hTLGDLock);
+		sTLGlobalData.hTLGDLock = NULL;
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void TLAddStreamNode(PTL_SNODE psAdd)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psAdd);
+	psAdd->psNext = TLGGD()->psHead;
+	TLGGD()->psHead = psAdd;
+
+	PVR_DPF_RETURN;
+}
+
+PTL_SNODE TLFindStreamNodeByName(const IMG_CHAR *pszName)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(pszName);
+
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn->psStream && OSStringNCompare(psn->psStream->szName, pszName, PRVSRVTL_MAX_STREAM_NAME_SIZE)==0)
+		{
+			PVR_DPF_RETURN_VAL(psn);
+		}
+	}
+
+	PVR_DPF_RETURN_VAL(NULL);
+}
+
+PTL_SNODE TLFindStreamNodeByDesc(PTL_STREAM_DESC psDesc)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDesc);
+
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn->psRDesc == psDesc || psn->psWDesc == psDesc)
+		{
+			PVR_DPF_RETURN_VAL(psn);
+		}
+	}
+	PVR_DPF_RETURN_VAL(NULL);
+}
+
+IMG_UINT32 TLDiscoverStreamNodes(const IMG_CHAR *pszNamePattern,
+                          IMG_CHAR aaszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+                          IMG_UINT32 ui32Max)
+{
+	TL_GLOBAL_DATA *psGD = TLGGD();
+	PTL_SNODE psn;
+	IMG_UINT32 ui32Count = 0;
+	size_t uiLen;
+
+	PVR_ASSERT(pszNamePattern);
+
+	if ((uiLen = OSStringLength(pszNamePattern)) == 0)
+		return 0;
+
+	for (psn = psGD->psHead; psn; psn = psn->psNext)
+	{
+		if (OSStringNCompare(pszNamePattern, psn->psStream->szName, uiLen) != 0)
+			continue;
+
+		/* If aaszStreams is NULL we only count how many string match
+		 * the given pattern. If it's a valid pointer we also return
+		 * the names. */
+		if (aaszStreams != NULL)
+		{
+			if (ui32Count >= ui32Max)
+				break;
+
+			/* all of names are shorter than MAX and null terminated */
+			OSStringSafeCopy(aaszStreams[ui32Count], psn->psStream->szName,
+			              PRVSRVTL_MAX_STREAM_NAME_SIZE);
+		}
+
+		ui32Count++;
+	}
+
+	return ui32Count;
+}
+
+PTL_SNODE TLFindAndGetStreamNodeByDesc(PTL_STREAM_DESC psDesc)
+{
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	psn = TLFindStreamNodeByDesc(psDesc);
+	if (psn == NULL)
+		PVR_DPF_RETURN_VAL(NULL);
+
+	PVR_ASSERT(psDesc == psn->psWDesc);
+
+	psn->uiWRefCount++;
+	psDesc->uiRefCount++;
+
+	PVR_DPF_RETURN_VAL(psn);
+}
+
+void TLReturnStreamNode(PTL_SNODE psNode)
+{
+	psNode->uiWRefCount--;
+	psNode->psWDesc->uiRefCount--;
+
+	PVR_ASSERT(psNode->uiWRefCount > 0);
+	PVR_ASSERT(psNode->psWDesc->uiRefCount > 0);
+}
+
+IMG_BOOL TLTryRemoveStreamAndFreeStreamNode(PTL_SNODE psRemove)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRemove);
+
+	/* If there is a client connected to this stream, defer stream's deletion */
+	if (psRemove->psRDesc != NULL || psRemove->psWDesc != NULL)
+	{
+		PVR_DPF_RETURN_VAL(IMG_FALSE);
+	}
+
+	/* Remove stream from TL_GLOBAL_DATA's list and free stream node */
+	psRemove->psStream = NULL;
+	RemoveAndFreeStreamNode(psRemove);
+
+	PVR_DPF_RETURN_VAL(IMG_TRUE);
+}
+
+IMG_BOOL TLUnrefDescAndTryFreeStreamNode(PTL_SNODE psNodeToRemove,
+                                          PTL_STREAM_DESC psSD)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psNodeToRemove);
+	PVR_ASSERT(psSD);
+
+	/* Decrement reference count. For descriptor obtained by reader it must
+	 * reach 0 (only single reader allowed) and for descriptors obtained by
+	 * writers it must reach value greater or equal to 0 (multiple writers
+	 * model). */
+	psSD->uiRefCount--;
+
+	if (psSD == psNodeToRemove->psRDesc)
+	{
+		PVR_ASSERT(0 == psSD->uiRefCount);
+		/* Remove stream descriptor (i.e. stream reader context) */
+		psNodeToRemove->psRDesc = NULL;
+	}
+	else if (psSD == psNodeToRemove->psWDesc)
+	{
+		PVR_ASSERT(0 <= psSD->uiRefCount);
+
+		psNodeToRemove->uiWRefCount--;
+
+		/* Remove stream descriptor if reference == 0 */
+		if (0 == psSD->uiRefCount)
+		{
+			psNodeToRemove->psWDesc = NULL;
+		}
+	}
+
+	/* Do not Free Stream Node if there is a write reference (a producer
+	 * context) to the stream */
+	if (NULL != psNodeToRemove->psRDesc || NULL != psNodeToRemove->psWDesc ||
+	    0 != psNodeToRemove->uiWRefCount)
+	{
+		PVR_DPF_RETURN_VAL(IMG_FALSE);
+	}
+
+	/* Make stream pointer NULL to prevent it from being destroyed in
+	 * RemoveAndFreeStreamNode. Cleanup of stream should be done by the
+	 * calling context */
+	psNodeToRemove->psStream = NULL;
+	RemoveAndFreeStreamNode(psNodeToRemove);
+
+	PVR_DPF_RETURN_VAL(IMG_TRUE);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/tlserver.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlserver.c
new file mode 100644
index 000000000000..5c8359271508
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlserver.c
@@ -0,0 +1,755 @@
+/*************************************************************************/ /*!
+@File
+@Title          KM server Transport Layer implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main bridge APIs for Transport Layer client functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+
+/*#define PVR_DPF_FUNCTION_TRACE_ON 1*/
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "connection_server.h"
+#include "allocmem.h"
+#include "devicemem.h"
+
+#include "tlintern.h"
+#include "tlstream.h"
+#include "tlserver.h"
+
+#include "pvrsrv_tlstreams.h"
+#define NO_STREAM_WAIT_PERIOD_US 2000000ULL
+#define NO_DATA_WAIT_PERIOD_US    500000ULL
+#define NO_ACQUIRE               0xffffffffU
+
+
+/*
+ * Transport Layer Client API Kernel-Mode bridge implementation
+ */
+PVRSRV_ERROR
+TLServerOpenStreamKM(const IMG_CHAR*   pszName,
+                     IMG_UINT32        ui32Mode,
+                     PTL_STREAM_DESC*  ppsSD,
+                     PMR**             ppsTLPMR)
+{
+	PVRSRV_ERROR	eError = PVRSRV_OK;
+	PVRSRV_ERROR	eErrorEO = PVRSRV_OK;
+	PTL_SNODE		psNode;
+	PTL_STREAM		psStream;
+	TL_STREAM_DESC *psNewSD = NULL;
+	IMG_HANDLE		hEvent;
+	IMG_BOOL		bIsWriteOnly = ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	                               IMG_TRUE : IMG_FALSE;
+	IMG_BOOL		bResetOnOpen = ui32Mode & PVRSRV_STREAM_FLAG_RESET_ON_OPEN ?
+	                               IMG_TRUE : IMG_FALSE;
+	IMG_BOOL		bNoOpenCB    = ui32Mode & PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK ?
+	                               IMG_TRUE : IMG_FALSE;
+	PTL_GLOBAL_DATA psGD = TLGGD();
+
+#if defined(PVR_DPF_FUNCTION_TRACE_ON)
+	PVR_DPF((PVR_DBG_CALLTRACE, "--> %s:%d entered (%s, %x)", __func__, __LINE__, pszName, ui32Mode));
+#endif
+
+	PVR_ASSERT(pszName);
+
+	/* Acquire TL_GLOBAL_DATA lock here, as if the following TLFindStreamNodeByName
+	 * returns NON NULL PTL_SNODE, we try updating the global data client count and
+	 * PTL_SNODE's psRDesc and we want to make sure the TL_SNODE is valid (eg. has
+	 * not been deleted) while we are updating it
+	 */
+	OSLockAcquire (psGD->hTLGDLock);
+
+	psNode = TLFindStreamNodeByName(pszName);
+	if ((psNode == NULL) && (ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WAIT))
+	{	/* Blocking code to wait for stream to be created if it does not exist */
+		eError = OSEventObjectOpen(psGD->hTLEventObj, &hEvent);
+		PVR_LOG_GOTO_IF_ERROR (eError, "OSEventObjectOpen", e0);
+
+		do
+		{
+			if ((psNode = TLFindStreamNodeByName(pszName)) == NULL)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "Stream %s does not exist, waiting...", pszName));
+
+				/* Release TL_GLOBAL_DATA lock before sleeping */
+				OSLockRelease (psGD->hTLGDLock);
+
+				/* Will exit OK or with timeout, both cases safe to ignore */
+				eErrorEO = OSEventObjectWaitTimeout(hEvent, NO_STREAM_WAIT_PERIOD_US);
+
+				/* Acquire lock after waking up */
+				OSLockAcquire (psGD->hTLGDLock);
+			}
+		}
+		while ((psNode == NULL) && (eErrorEO == PVRSRV_OK));
+
+		eError = OSEventObjectClose(hEvent);
+		PVR_LOG_GOTO_IF_ERROR (eError, "OSEventObjectClose", e0);
+	}
+
+	/* Make sure we have found a stream node after wait/search */
+	if (psNode == NULL)
+	{
+		/* Did we exit the wait with timeout, inform caller */
+		if (eErrorEO == PVRSRV_ERROR_TIMEOUT)
+		{
+			eError = eErrorEO;
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_NOT_FOUND;
+			PVR_DPF((PVR_DBG_ERROR, "Stream \"%s\" does not exist", pszName));
+		}
+		goto e0;
+	}
+
+	psStream = psNode->psStream;
+
+	/* Allocate memory for the stream. The memory will be allocated with the
+	 * first call. */
+	eError = TLAllocSharedMemIfNull(psStream);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to allocate memory for stream"
+				" \"%s\"", pszName));
+		goto e0;
+	}
+
+	if (bIsWriteOnly)
+	{
+
+		/* If psWDesc == NULL it means that this is the first attempt
+		 * to open stream for write. If yes create the descriptor or increment
+		 * reference count otherwise. */
+		if (psNode->psWDesc == NULL)
+		{
+			psNewSD = TLMakeStreamDesc(psNode, ui32Mode, NULL);
+			psNode->psWDesc = psNewSD;
+		}
+		else
+		{
+			psNewSD = psNode->psWDesc;
+			psNode->psWDesc->uiRefCount++;
+		}
+
+		PVR_LOG_GOTO_IF_NOMEM(psNewSD, eError, e0);
+
+		psNode->uiWRefCount++;
+	}
+	else
+	{
+		/* Only one reader per stream supported */
+		if (psNode->psRDesc != NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Cannot open \"%s\" stream, stream already"
+			        " opened", pszName));
+			eError = PVRSRV_ERROR_ALREADY_OPEN;
+			goto e0;
+		}
+
+		/* Create an event handle for this client to wait on when no data in
+		 * stream buffer. */
+		eError = OSEventObjectOpen(psNode->hReadEventObj, &hEvent);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "OSEventObjectOpen");
+			eError = PVRSRV_ERROR_UNABLE_TO_CREATE_EVENT;
+			goto e0;
+		}
+
+		psNewSD = TLMakeStreamDesc(psNode, ui32Mode, hEvent);
+		psNode->psRDesc = psNewSD;
+
+		if (!psNewSD)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Not possible to make a new stream descriptor"));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e1;
+		}
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+		        "TLServerOpenStreamKM evList=%p, evObj=%p",
+		        psNode->hReadEventObj,
+		        psNode->psRDesc->hReadEvent));
+	}
+
+	/* Copy the import handle back to the user mode API to enable access to
+	 * the stream buffer from user-mode process. */
+	eError = DevmemLocalGetImportHandle(TLStreamGetBufferPointer(psStream),
+	                                    (void**) ppsTLPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemLocalGetImportHandle", e2);
+
+	psGD->uiClientCnt++;
+
+	/* Global data updated. Now release global lock */
+	OSLockRelease (psGD->hTLGDLock);
+
+	*ppsSD = psNewSD;
+
+	if (bResetOnOpen)
+	{
+		TLStreamReset(psStream);
+	}
+
+	/* This callback is executed only on reader open. There are some actions
+	 * executed on reader open that don't make much sense for writers e.g.
+	 * injection on time synchronisation packet into the stream. */
+	if (!bIsWriteOnly && psStream->pfOnReaderOpenCallback != NULL && !bNoOpenCB)
+	{
+		psStream->pfOnReaderOpenCallback(psStream->pvOnReaderOpenUserData);
+	}
+
+	/* psNode->uiWRefCount is set to '1' on stream create so the first open
+	 * is '2'. */
+	if (bIsWriteOnly && psStream->psNotifStream != NULL &&
+	    psNode->uiWRefCount == 2)
+	{
+		TLStreamMarkStreamOpen(psStream);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Stream %s opened for %s", __func__, pszName,
+	        ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ? "write" : "read"));
+
+	PVR_DPF_RETURN_OK;
+
+e2:
+	OSFreeMem(psNewSD);
+e1:
+	if (!bIsWriteOnly)
+		OSEventObjectClose(hEvent);
+e0:
+	OSLockRelease (psGD->hTLGDLock);
+	PVR_DPF_RETURN_RC (eError);
+}
+
+PVRSRV_ERROR
+TLServerCloseStreamKM(PTL_STREAM_DESC psSD)
+{
+	PVRSRV_ERROR    eError = PVRSRV_OK;
+	PTL_GLOBAL_DATA psGD = TLGGD();
+	PTL_SNODE		psNode;
+	PTL_STREAM	psStream;
+	IMG_BOOL	bDestroyStream;
+	IMG_BOOL	bIsWriteOnly = psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	                           IMG_TRUE : IMG_FALSE;
+	IMG_BOOL	bNoOpenCB    = psSD->ui32Flags & PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK ?
+	                           IMG_TRUE : IMG_FALSE;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since the descriptor is valid, the stream should not have been made NULL */
+	PVR_ASSERT (psNode->psStream);
+
+	/* Save the stream's reference in-case its destruction is required after this
+	 * client is removed */
+	psStream = psNode->psStream;
+
+	/* Acquire TL_GLOBAL_DATA lock as the following TLRemoveDescAndTryFreeStreamNode
+	 * call will update the TL_SNODE's descriptor value */
+	OSLockAcquire (psGD->hTLGDLock);
+
+	/* Close event handle because event object list might be destroyed in
+	 * TLUnrefDescAndTryFreeStreamNode(). */
+	if (!bIsWriteOnly)
+	{
+		/* Reset the read position on close if the stream requires it. */
+		TLStreamResetReadPos(psStream);
+
+		/* Close and free the event handle resource used by this descriptor */
+		eError = OSEventObjectClose(psSD->hReadEvent);
+		if (eError != PVRSRV_OK)
+		{
+			/* Log error but continue as it seems best */
+			PVR_LOG_ERROR(eError, "OSEventObjectClose");
+			eError = PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+		}
+	}
+	else if (psNode->uiWRefCount == 2 && psStream->psNotifStream != NULL)
+	{
+		/* psNode->uiWRefCount is set to '1' on stream create so the last close
+		 * before destruction is '2'. */
+		TLStreamMarkStreamClose(psStream);
+	}
+
+	/* Remove descriptor from stream object/list */
+	bDestroyStream = TLUnrefDescAndTryFreeStreamNode (psNode, psSD);
+
+	/* Check the counter is sensible after input data validated. */
+	PVR_ASSERT(psGD->uiClientCnt > 0);
+	psGD->uiClientCnt--;
+
+	/* This callback is executed only on reader close. */
+	if (!bIsWriteOnly && psStream->pfOnReaderCloseCallback != NULL && !bNoOpenCB)
+	{
+		psStream->pfOnReaderCloseCallback(psStream->pvOnReaderCloseUserData);
+	}
+
+	OSLockRelease (psGD->hTLGDLock);
+
+	/* Destroy the stream if its TL_SNODE was removed from TL_GLOBAL_DATA */
+	if (bDestroyStream)
+	{
+		TLStreamDestroy (psStream);
+		psStream = NULL;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Stream closed", __func__));
+
+	/* Free the descriptor if ref count reaches 0. */
+	if (psSD->uiRefCount == 0)
+	{
+		/* Free the stream descriptor object */
+		OSFreeMem(psSD);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerReserveStreamKM(PTL_STREAM_DESC psSD,
+                        IMG_UINT32* ui32BufferOffset,
+                        IMG_UINT32 ui32Size,
+                        IMG_UINT32 ui32SizeMin,
+                        IMG_UINT32* pui32Available)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+	IMG_UINT8* pui8Buffer = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire the global lock. We have to be sure that no one modifies
+	 * the list while we are looking for our stream. */
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	/* The TL writers that currently land here are at a very low to none risk
+	 * to breach max TL packet size constraint (even if there is no reader
+	 * connected to the TL stream and hence eventually will cause the TL stream
+	 * to be full). Hence no need to know the status of TL stream reader
+	 * connection.
+	 */
+	eError = TLStreamReserve2(psNode->psStream, &pui8Buffer, ui32Size,
+				  ui32SizeMin, pui32Available, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Failed to reserve %u (%u, %u) bytes in the stream, error %s.",
+				ui32Size, ui32SizeMin, *pui32Available, PVRSRVGETERRORSTRING(eError)));
+	}
+	else if (pui8Buffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Not enough space in the stream."));
+		eError = PVRSRV_ERROR_STREAM_FULL;
+	}
+	else
+	{
+		*ui32BufferOffset = pui8Buffer - psNode->psStream->pbyBuffer;
+		PVR_ASSERT(*ui32BufferOffset < psNode->psStream->ui32Size);
+	}
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerCommitStreamKM(PTL_STREAM_DESC psSD,
+                       IMG_UINT32 ui32Size)
+{
+	TL_GLOBAL_DATA*	psGD = TLGGD();
+	PTL_SNODE psNode;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire the global lock. We have to be sure that no one modifies
+	 * the list while we are looking for our stream. */
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	eError = TLStreamCommit(psNode->psStream, ui32Size);
+	PVR_LOG_IF_ERROR(eError, "TLStreamCommit");
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerDiscoverStreamsKM(const IMG_CHAR *pszNamePattern,
+                          IMG_UINT32 ui32Size,
+                          IMG_CHAR *pszStreams,
+                          IMG_UINT32 *pui32NumFound)
+{
+	PTL_SNODE psNode = NULL;
+	IMG_CHAR (*paszStreams)[PRVSRVTL_MAX_STREAM_NAME_SIZE] =
+			(IMG_CHAR (*)[PRVSRVTL_MAX_STREAM_NAME_SIZE]) (void *)pszStreams;
+
+	if (*pszNamePattern == '\0')
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (ui32Size % PRVSRVTL_MAX_STREAM_NAME_SIZE != 0)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	/* Quick exit if there are no streams */
+	if (TLGGD()->psHead == NULL)
+	{
+		*pui32NumFound = 0;
+		return PVRSRV_OK;
+	}
+
+	OSLockAcquire(TLGGD()->hTLGDLock);
+
+	*pui32NumFound = TLDiscoverStreamNodes(pszNamePattern, paszStreams,
+	                                  ui32Size / PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	/* Find "tlctrl" stream and reset it */
+	psNode = TLFindStreamNodeByName(PVRSRV_TL_CTRL_STREAM);
+	if (psNode != NULL)
+		TLStreamReset(psNode->psStream);
+
+	OSLockRelease(TLGGD()->hTLGDLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+TLServerAcquireDataKM(PTL_STREAM_DESC psSD,
+                      IMG_UINT32*     puiReadOffset,
+                      IMG_UINT32*     puiReadLen)
+{
+	PVRSRV_ERROR    eError = PVRSRV_OK;
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	IMG_UINT32      uiTmpOffset;
+	IMG_UINT32      uiTmpLen = 0;
+	PTL_SNODE       psNode;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	TL_COUNTER_INC(psSD->ui32AcquireCount);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* If we are here, the stream will never be made NULL until this context itself
+	 * calls TLRemoveDescAndTryFreeStreamNode(). This is because the producer will
+	 * fail to make the stream NULL (by calling TLTryRemoveStreamAndFreeStreamNode)
+	 * when a valid stream descriptor is present (i.e. a client is connected).
+	 * Hence, no checks for stream being NON NULL are required after this. */
+	PVR_ASSERT (psNode->psStream);
+
+	psSD->ui32ReadLen = 0;	/* Handle NULL read returns */
+
+	do
+	{
+		uiTmpLen = TLStreamAcquireReadPos(psNode->psStream, psSD->ui32Flags & PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK, &uiTmpOffset);
+
+		/* Check we have not already exceeded read limit with just offset
+		 * regardless of data length to ensure the client sees the RC */
+		if (psSD->ui32Flags & PVRSRV_STREAM_FLAG_READ_LIMIT)
+		{
+			/* Check to see if we are reading beyond the read limit */
+			if (uiTmpOffset >= psSD->ui32ReadLimit)
+			{
+				PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_READLIMIT_REACHED);
+			}
+		}
+
+		if (uiTmpLen > 0)
+		{ /* Data found */
+
+			/* Check we have not already exceeded read limit offset+len */
+			if (psSD->ui32Flags & PVRSRV_STREAM_FLAG_READ_LIMIT)
+			{
+				/* Adjust the read length if it goes beyond the read limit
+				 * limit always guaranteed to be on packet */
+				if ((uiTmpOffset + uiTmpLen) >= psSD->ui32ReadLimit)
+				{
+					uiTmpLen = psSD->ui32ReadLimit - uiTmpOffset;
+				}
+			}
+
+			*puiReadOffset = uiTmpOffset;
+			*puiReadLen = uiTmpLen;
+			psSD->ui32ReadLen = uiTmpLen;	/* Save the original data length in the stream desc */
+			PVR_DPF_RETURN_OK;
+		}
+		else if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING))
+		{ /* No data found blocking */
+
+			/* Instead of doing a complete sleep for `NO_DATA_WAIT_PERIOD_US` us, we sleep in chunks
+			 * of 168 ms. In a "deferred" signal scenario from writer, this gives us a chance to
+			 * wake-up (timeout) early and continue reading in-case some data is available */
+			IMG_UINT64 ui64WaitInChunksUs = MIN(NO_DATA_WAIT_PERIOD_US, 168000ULL);
+			IMG_BOOL bDataFound = IMG_FALSE;
+
+			TL_COUNTER_INC(psSD->ui32NoDataSleep);
+
+			LOOP_UNTIL_TIMEOUT_US(NO_DATA_WAIT_PERIOD_US)
+			{
+				eError = OSEventObjectWaitTimeout(psSD->hReadEvent, ui64WaitInChunksUs);
+				if (eError == PVRSRV_OK)
+				{
+					bDataFound = IMG_TRUE;
+					TL_COUNTER_INC(psSD->ui32Signalled);
+					break;
+				}
+				else if (eError == PVRSRV_ERROR_TIMEOUT)
+				{
+					if (TLStreamOutOfData(psNode->psStream))
+					{
+						/* Return on timeout if stream empty, else let while exit and return data */
+						continue;
+					}
+					else
+					{
+						bDataFound = IMG_TRUE;
+						TL_COUNTER_INC(psSD->ui32TimeoutData);
+						PVR_DPF((PVR_DBG_MESSAGE, "%s: Data found at timeout. Current BuffUt = %u",
+												 __func__, TLStreamGetUT(psNode->psStream)));
+						break;
+					}
+				}
+				else
+				{ /* Some other system error with event objects */
+					PVR_DPF_RETURN_RC(eError);
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (bDataFound)
+			{
+				continue;
+			}
+			else
+			{
+				TL_COUNTER_INC(psSD->ui32TimeoutEmpty);
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+		}
+		else
+		{ /* No data non-blocking */
+			TL_COUNTER_INC(psSD->ui32NoData);
+
+			/* When no-data in non-blocking mode, uiReadOffset should be set to NO_ACQUIRE
+			 * signifying there's no need of Release call */
+			*puiReadOffset = NO_ACQUIRE;
+			*puiReadLen = 0;
+			PVR_DPF_RETURN_OK;
+		}
+	}
+	while (1);
+}
+
+PVRSRV_ERROR
+TLServerReleaseDataKM(PTL_STREAM_DESC psSD,
+                      IMG_UINT32      uiReadOffset,
+                      IMG_UINT32      uiReadLen)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+
+	PVR_DPF_ENTERED;
+
+	/* Unreferenced in release builds */
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+
+	PVR_ASSERT(psSD);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	if ((uiReadLen % PVRSRVTL_PACKET_ALIGNMENT != 0))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "TLReleaseDataKM uiReadOffset=%d, uiReadLen=%d", uiReadOffset, uiReadLen));
+
+	/* Move read position on to free up space in stream buffer */
+	PVR_DPF_RETURN_RC(TLStreamAdvanceReadPos(psNode->psStream, uiReadLen, psSD->ui32ReadLen));
+}
+
+PVRSRV_ERROR
+TLServerWriteDataKM(PTL_STREAM_DESC psSD,
+                    IMG_UINT32 ui32Size,
+                    IMG_BYTE* pui8Data)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	eError = TLStreamWrite(psNode->psStream, pui8Data, ui32Size);
+	/* propagate error up but don't print anything here */
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ End of file (tlserver.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/tlstream.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlstream.c
new file mode 100644
index 000000000000..29cc42a802fe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlstream.c
@@ -0,0 +1,1686 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer API implementation.
+                These functions are provided to driver components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "allocmem.h"
+#include "devicemem.h"
+#include "pvrsrv_error.h"
+#include "sysinfo.h"
+#include "osfunc.h"
+#include "log2.h"
+
+#include "tlintern.h"
+#include "tlstream.h"
+
+#include "pvrsrv.h"
+
+#if !defined(EVENT_OBJECT_TIMEOUT_US)
+#error EVENT_OBJECT_TIMEOUT_US should be defined sysinfo.h
+#endif
+
+#define READ_PENDING_TIMEOUT_US 100000ULL
+
+/*! Compute maximum TL packet size for this stream. Max packet size will be
+ * minimum of PVRSRVTL_MAX_PACKET_SIZE and (BufferSize / 2.5). This computation
+ * is required to avoid a corner case that was observed when TL buffer size is
+ * smaller than twice of TL max packet size and read, write index are positioned
+ * in such a way that the TL packet (write packet + padding packet) size is may
+ * be bigger than the buffer size itself.
+ */
+#define GET_TL_MAX_PACKET_SIZE( bufSize ) PVRSRVTL_ALIGN( MIN( PVRSRVTL_MAX_PACKET_SIZE, ( 2 * bufSize ) / 5 ) )
+
+/* Given the state of the buffer it returns a number of bytes that the client
+ * can use for a successful allocation. */
+static INLINE IMG_UINT32 suggestAllocSize(IMG_UINT32 ui32LRead,
+                                          IMG_UINT32 ui32LWrite,
+                                          IMG_UINT32 ui32CBSize,
+                                          IMG_UINT32 ui32ReqSizeMin,
+                                          IMG_UINT32 ui32MaxPacketSize)
+{
+	IMG_UINT32 ui32AvSpace = 0;
+
+	/* This could be written in fewer lines using the ? operator but it
+		would not be kind to potential readers of this source at all. */
+	if (ui32LRead > ui32LWrite)                          /* Buffer WRAPPED */
+	{
+		if ((ui32LRead - ui32LWrite) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32LRead - ui32LWrite - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+	}
+	else                                                  /* Normal, no wrap */
+	{
+		if ((ui32CBSize - ui32LWrite) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32CBSize - ui32LWrite - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+		else if ((ui32LRead - 0) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32LRead - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+	}
+	/* The max size of a TL packet currently is UINT16. adjust accordingly */
+	return MIN(ui32AvSpace, ui32MaxPacketSize);
+}
+
+/* Returns bytes left in the buffer. Negative if there is not any.
+ * two 8b aligned values are reserved, one for the write failed buffer flag
+ * and one to be able to distinguish the buffer full state to the buffer
+ * empty state.
+ * Always returns free space -8 even when the "write failed" packet may be
+ * already in the stream before this write. */
+static INLINE IMG_INT
+circbufSpaceLeft(IMG_UINT32 ui32Read, IMG_UINT32 ui32Write, IMG_UINT32 ui32size)
+{
+	/* We need to reserve 8b (one packet) in the buffer to be able to tell empty
+	 * buffers from full buffers and one more for packet write fail packet */
+	if (ui32Read > ui32Write)
+	{
+		return (IMG_INT)ui32Read - (IMG_INT)ui32Write - (IMG_INT)BUFFER_RESERVED_SPACE;
+	}
+	else
+	{
+		return (IMG_INT)ui32size - ((IMG_INT)ui32Write - (IMG_INT)ui32Read) - (IMG_INT)BUFFER_RESERVED_SPACE;
+	}
+}
+
+IMG_UINT32 TLStreamGetUT(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	IMG_UINT32 ui32LRead = psStream->ui32Read, ui32LWrite = psStream->ui32Write;
+
+	if (ui32LWrite >= ui32LRead)
+	{
+		return (ui32LWrite-ui32LRead);
+	}
+	else
+	{
+		return (psStream->ui32Size-ui32LRead+ui32LWrite);
+	}
+}
+
+PVRSRV_ERROR TLAllocSharedMemIfNull(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	PVRSRV_ERROR eError;
+
+	/* CPU Local memory used as these buffers are not accessed by the device.
+	 * CPU Uncached write combine memory used to improve write performance,
+	 * memory barrier added in TLStreamCommit to ensure data written to memory
+	 * before CB write point is updated before consumption by the reader.
+	 */
+	IMG_CHAR pszBufferLabel[PRVSRVTL_MAX_STREAM_NAME_SIZE + 20];
+	PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                                    PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                                    PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                                    PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                                    PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL) |  /* TL for now is only used by host driver, so cpulocal mem suffices */
+	                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/* Exit if memory has already been allocated. */
+	if (psStream->pbyBuffer != NULL)
+		return PVRSRV_OK;
+
+	OSSNPrintf(pszBufferLabel, sizeof(pszBufferLabel), "TLStreamBuf-%s",
+	           psStream->szName);
+
+	/* Use HostMemDeviceNode instead of psStream->psDevNode to benefit from faster
+	 * accesses to CPU local memory. When the framework to access CPU_LOCAL device
+	 * memory from GPU is fixed, we'll switch back to use psStream->psDevNode for
+	 * TL buffers */
+	eError = DevmemAllocateExportable((IMG_HANDLE)PVRSRVGetPVRSRVData()->psHostMemDeviceNode,
+	                                  (IMG_DEVMEM_SIZE_T) psStream->ui32Size,
+	                                  (IMG_DEVMEM_ALIGN_T) OSGetPageSize(),
+	                                  ExactLog2(OSGetPageSize()),
+	                                  uiMemFlags,
+	                                  pszBufferLabel,
+	                                  &psStream->psStreamMemDesc);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+	eError = DevmemAcquireCpuVirtAddr(psStream->psStreamMemDesc,
+	                                  (void**) &psStream->pbyBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", e1);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFree(psStream->psStreamMemDesc);
+e0:
+	return eError;
+}
+
+void TLFreeSharedMem(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+
+	if (psStream->pbyBuffer != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psStream->psStreamMemDesc);
+		psStream->pbyBuffer = NULL;
+	}
+	if (psStream->psStreamMemDesc != NULL)
+	{
+		DevmemFree(psStream->psStreamMemDesc);
+		psStream->psStreamMemDesc = NULL;
+	}
+}
+
+/* Special space left routine for TL_FLAG_PERMANENT_NO_WRAP streams */
+static INLINE IMG_UINT
+bufSpaceLeft(IMG_UINT32 ui32Read, IMG_UINT32 ui32Write, IMG_UINT32 ui32size)
+{
+	 /* buffers from full buffers and one more for packet write fail packet */
+	PVR_ASSERT(ui32Read<=ui32Write);
+	return ui32size - ui32Write;
+}
+
+/*******************************************************************************
+ * TL Server public API implementation.
+ ******************************************************************************/
+PVRSRV_ERROR
+TLStreamCreate(IMG_HANDLE *phStream,
+               const IMG_CHAR *szStreamName,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32StreamFlags,
+               TL_STREAM_ONREADEROPENCB pfOnReaderOpenCB,
+               void *pvOnReaderOpenUD,
+               TL_STREAM_ONREADERCLOSECB pfOnReaderCloseCB,
+               void *pvOnReaderCloseUD,
+               TL_STREAM_SOURCECB pfProducerCB,
+               void *pvProducerUD)
+{
+	PTL_STREAM    psTmp;
+	PVRSRV_ERROR  eError;
+	IMG_HANDLE    hEventList;
+	PTL_SNODE     psn;
+	TL_OPMODE     eOpMode;
+
+	PVR_DPF_ENTERED;
+	/* Parameter checks: non NULL handler required */
+	if (NULL == phStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	if (szStreamName == NULL || *szStreamName == '\0' ||
+	    OSStringLength(szStreamName) >= PRVSRVTL_MAX_STREAM_NAME_SIZE)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eOpMode = ui32StreamFlags & TL_OPMODE_MASK;
+	if (( eOpMode <= TL_OPMODE_UNDEF ) || ( eOpMode >= TL_OPMODE_LAST ))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OpMode for TL stream is invalid"));
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Acquire TL_GLOBAL_DATA lock here because, if the following TLFindStreamNodeByName()
+	 * returns NULL, a new TL_SNODE will be added to TL_GLOBAL_DATA's TL_SNODE list */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Check if there already exists a stream with this name. */
+	psn = TLFindStreamNodeByName( szStreamName );
+	if (NULL != psn)
+	{
+		eError = PVRSRV_ERROR_ALREADY_EXISTS;
+		goto e0;
+	}
+
+	/* Allocate stream structure container (stream struct) for the new stream */
+	psTmp = OSAllocZMem(sizeof(TL_STREAM));
+	if (NULL == psTmp)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	OSStringSafeCopy(psTmp->szName, szStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	if (ui32StreamFlags & TL_FLAG_FORCE_FLUSH)
+	{
+		psTmp->bWaitForEmptyOnDestroy = IMG_TRUE;
+	}
+
+	psTmp->bNoSignalOnCommit = (ui32StreamFlags&TL_FLAG_NO_SIGNAL_ON_COMMIT) ? IMG_TRUE : IMG_FALSE;
+	psTmp->bNoWrapPermanent = (ui32StreamFlags&TL_FLAG_PERMANENT_NO_WRAP) ? IMG_TRUE : IMG_FALSE;
+
+	psTmp->eOpMode = eOpMode;
+	if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+	{
+		/* Only allow drop properties to be mixed with no-wrap type streams
+		 * since space does not become available when reads take place hence
+		 * no point blocking.
+		 */
+		if (psTmp->bNoWrapPermanent)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e1;
+		}
+	}
+
+	/* Additional synchronisation object required for some streams e.g. blocking */
+	eError = OSEventObjectCreate(NULL, &psTmp->hProducerEventObj);
+	PVR_GOTO_IF_ERROR(eError, e1);
+	/* Create an event handle for this kind of stream */
+	eError = OSEventObjectOpen(psTmp->hProducerEventObj, &psTmp->hProducerEvent);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	psTmp->pfOnReaderOpenCallback = pfOnReaderOpenCB;
+	psTmp->pvOnReaderOpenUserData = pvOnReaderOpenUD;
+	psTmp->pfOnReaderCloseCallback = pfOnReaderCloseCB;
+	psTmp->pvOnReaderCloseUserData = pvOnReaderCloseUD;
+	/* Remember producer supplied CB and data for later */
+	psTmp->pfProducerCallback = (void(*)(void))pfProducerCB;
+	psTmp->pvProducerUserData = pvProducerUD;
+
+	psTmp->psNotifStream = NULL;
+
+	/* Round the requested bytes to a multiple of array elements' size, eg round 3 to 4 */
+	psTmp->ui32Size = PVRSRVTL_ALIGN(ui32Size);
+
+	/* Signalling from TLStreamCommit is deferred until buffer is slightly (~12%) filled */
+	psTmp->ui32ThresholdUsageForSignal = psTmp->ui32Size >> 3;
+	psTmp->ui32MaxPacketSize = GET_TL_MAX_PACKET_SIZE(psTmp->ui32Size);
+	psTmp->ui32Read = 0;
+	psTmp->ui32Write = 0;
+	psTmp->ui32Pending = NOTHING_PENDING;
+	psTmp->bReadPending = IMG_FALSE;
+	psTmp->bSignalPending = IMG_FALSE;
+
+#if defined(TL_BUFFER_STATS)
+	OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 0);
+	/* Setting MAX possible value for "minimum" time to full,
+	 * helps in the logic which calculates this time */
+	psTmp->ui32MinTimeToFullInUs = IMG_UINT32_MAX;
+#endif
+
+	/* Memory will be allocated on first connect to the stream */
+	if (!(ui32StreamFlags & TL_FLAG_ALLOCATE_ON_FIRST_OPEN))
+	{
+		/* Allocate memory for the circular buffer and export it to user space. */
+		eError = TLAllocSharedMemIfNull(psTmp);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TLAllocSharedMem", e3);
+	}
+
+	/* Synchronisation object to synchronise with user side data transfers. */
+	eError = OSEventObjectCreate(psTmp->szName, &hEventList);
+	PVR_GOTO_IF_ERROR(eError, e4);
+
+	eError = OSLockCreate (&psTmp->hStreamWLock);
+	PVR_GOTO_IF_ERROR(eError, e5);
+
+	eError = OSLockCreate (&psTmp->hReadLock);
+	PVR_GOTO_IF_ERROR(eError, e6);
+
+	/* Now remember the stream in the global TL structures */
+	psn = TLMakeSNode(hEventList, (TL_STREAM *)psTmp, NULL);
+	PVR_GOTO_IF_NOMEM(psn, eError, e7);
+
+	/* Stream node created, now reset the write reference count to 1
+	 * (i.e. this context's reference) */
+	psn->uiWRefCount = 1;
+
+	TLAddStreamNode(psn);
+
+	/* Release TL_GLOBAL_DATA lock as the new TL_SNODE is now added to the list */
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	/* Best effort signal, client wait timeout will ultimately let it find the
+	 * new stream if this fails, acceptable to avoid clean-up as it is tricky
+	 * at this point */
+	(void) OSEventObjectSignal(TLGGD()->hTLEventObj);
+
+	/* Pass the newly created stream handle back to caller */
+	*phStream = (IMG_HANDLE)psTmp;
+	PVR_DPF_RETURN_OK;
+
+e7:
+	OSLockDestroy(psTmp->hReadLock);
+e6:
+	OSLockDestroy(psTmp->hStreamWLock);
+e5:
+	OSEventObjectDestroy(hEventList);
+e4:
+	TLFreeSharedMem(psTmp);
+e3:
+	OSEventObjectClose(psTmp->hProducerEvent);
+e2:
+	OSEventObjectDestroy(psTmp->hProducerEventObj);
+e1:
+	OSFreeMem(psTmp);
+e0:
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+void TLStreamReset(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	IMG_HANDLE hEventWaitForWriterToComplete;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psStream != NULL);
+
+	OSLockAcquire(psStream->hStreamWLock);
+
+	while (psStream->ui32Pending != NOTHING_PENDING)
+	{
+		/* We're in the middle of a write so we cannot reset the stream.
+		 * We are going to wait until the data is committed. Release lock while
+		 * we're here. */
+		OSLockRelease(psStream->hStreamWLock);
+
+		eError = OSEventObjectOpen(psStream->psNode->hReadEventObj,
+		                           &hEventWaitForWriterToComplete);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+		/* Event when psStream->bNoSignalOnCommit is set we can still use
+		 * the timeout capability of event object API (time in us). */
+		eError = OSEventObjectWaitTimeout(hEventWaitForWriterToComplete, 100);
+		if (eError != PVRSRV_ERROR_TIMEOUT && eError != PVRSRV_OK)
+		{
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectWaitTimeout", TimeoutError);
+		}
+
+		eError = OSEventObjectClose(hEventWaitForWriterToComplete);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectClose");
+
+		OSLockAcquire(psStream->hStreamWLock);
+
+		/* Either timeout occurred or the stream has been signalled.
+		 * If former we have to check if the data was committed and if latter
+		 * if the stream hasn't been re-reserved. Either way we have to go
+		 * back to the condition.
+		 * If the stream has been released we'll exit with the lock held so
+		 * we can finally go and reset the stream. */
+	}
+
+	psStream->ui32Read = 0;
+	psStream->ui32Write = 0;
+	/* we know that ui32Pending already has correct value (no need to set) */
+
+	OSLockRelease(psStream->hStreamWLock);
+
+	return;
+
+TimeoutError:
+	eError = OSEventObjectClose(hEventWaitForWriterToComplete);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+PVRSRV_ERROR
+TLStreamSetNotifStream(IMG_HANDLE hStream, IMG_HANDLE hNotifStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+
+	if (hStream == NULL || hNotifStream == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psStream->psNotifStream = (PTL_STREAM) hNotifStream;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+TLStreamReconfigure(
+		IMG_HANDLE hStream,
+		IMG_UINT32 ui32StreamFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PTL_STREAM psTmp;
+	TL_OPMODE eOpMode;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eOpMode = ui32StreamFlags & TL_OPMODE_MASK;
+	if (( eOpMode <= TL_OPMODE_UNDEF ) || ( eOpMode >= TL_OPMODE_LAST ))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OpMode for TL stream is invalid"));
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Prevent the TL Stream buffer from being written to
+	 * while its mode is being reconfigured
+	 */
+	OSLockAcquire (psTmp->hStreamWLock);
+	if (NOTHING_PENDING != psTmp->ui32Pending)
+	{
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_READY);
+	}
+	psTmp->ui32Pending = 0;
+	OSLockRelease (psTmp->hStreamWLock);
+
+	psTmp->eOpMode = eOpMode;
+	if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+	{
+		/* Only allow drop properties to be mixed with no-wrap type streams
+		 * since space does not become available when reads take place hence
+		 * no point blocking.
+		 */
+		if (psTmp->bNoWrapPermanent)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e1;
+		}
+	}
+
+	OSLockAcquire (psTmp->hStreamWLock);
+	psTmp->ui32Pending = NOTHING_PENDING;
+	OSLockRelease (psTmp->hStreamWLock);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLStreamOpen(IMG_HANDLE     *phStream,
+             const IMG_CHAR *szStreamName)
+{
+	PTL_SNODE psTmpSNode;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == phStream || NULL == szStreamName)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Acquire the TL_GLOBAL_DATA lock first to ensure,
+	 * the TL_STREAM while returned and being modified,
+	 * is not deleted by some other context */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Search for a stream node with a matching stream name */
+	psTmpSNode = TLFindStreamNodeByName(szStreamName);
+
+	if (NULL == psTmpSNode)
+	{
+		OSLockRelease (TLGGD()->hTLGDLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_FOUND);
+	}
+
+	if (psTmpSNode->psStream->psNotifStream != NULL &&
+	    psTmpSNode->uiWRefCount == 1)
+	{
+		TLStreamMarkStreamOpen(psTmpSNode->psStream);
+	}
+
+	/* The TL_SNODE->uiWRefCount governs the presence of this node in the
+	 * TL_GLOBAL_DATA list i.e. when uiWRefCount falls to zero we try removing
+	 * this node from the TL_GLOBAL_DATA list. Hence, is protected using the
+	 * TL_GLOBAL_DATA lock and not TL_STREAM lock */
+	psTmpSNode->uiWRefCount++;
+
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	/* Return the stream handle to the caller */
+	*phStream = (IMG_HANDLE)psTmpSNode->psStream;
+
+	PVR_DPF_RETURN_VAL(PVRSRV_OK);
+}
+
+void
+TLStreamClose(IMG_HANDLE hStream)
+{
+	PTL_STREAM	psTmp;
+	IMG_BOOL	bDestroyStream;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "TLStreamClose failed as NULL stream handler passed, nothing done."));
+		PVR_DPF_RETURN;
+	}
+
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Acquire TL_GLOBAL_DATA lock for updating the reference count as this will be required
+	 * in-case this TL_STREAM node is to be deleted */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Decrement write reference counter of the stream */
+	psTmp->psNode->uiWRefCount--;
+
+	if (0 != psTmp->psNode->uiWRefCount)
+	{
+		/* The stream is still being used in other context(s) do not destroy
+		 * anything */
+
+		/* uiWRefCount == 1 means that stream was closed for write. Next
+		 * close is pairing TLStreamCreate(). Send notification to indicate
+		 * that no writer are connected to the stream any more. */
+		if (psTmp->psNotifStream != NULL && psTmp->psNode->uiWRefCount == 1)
+		{
+			TLStreamMarkStreamClose(psTmp);
+		}
+
+		OSLockRelease (TLGGD()->hTLGDLock);
+		PVR_DPF_RETURN;
+	}
+	else
+	{
+		/* Now we try removing this TL_STREAM from TL_GLOBAL_DATA */
+
+		if (psTmp->bWaitForEmptyOnDestroy)
+		{
+			/* We won't require the TL_STREAM lock to be acquired here for accessing its read
+			 * and write offsets. REASON: We are here because there is no producer context
+			 * referencing this TL_STREAM, hence its ui32Write offset won't be changed now.
+			 * Also, the update of ui32Read offset is not protected by locks */
+			while (psTmp->ui32Read != psTmp->ui32Write)
+			{
+				/* Release lock before sleeping */
+				OSLockRelease (TLGGD()->hTLGDLock);
+
+				OSEventObjectWaitTimeout(psTmp->hProducerEvent, EVENT_OBJECT_TIMEOUT_US);
+
+				OSLockAcquire (TLGGD()->hTLGDLock);
+
+				/* Ensure destruction of stream is still required */
+				if (0 != psTmp->psNode->uiWRefCount)
+				{
+					OSLockRelease (TLGGD()->hTLGDLock);
+					PVR_DPF_RETURN;
+				}
+			}
+		}
+
+		/* Try removing the stream from TL_GLOBAL_DATA */
+		bDestroyStream = TLTryRemoveStreamAndFreeStreamNode (psTmp->psNode);
+
+		OSLockRelease (TLGGD()->hTLGDLock);
+
+		if (bDestroyStream)
+		{
+			/* Destroy the stream if it was removed from TL_GLOBAL_DATA */
+			TLStreamDestroy (psTmp);
+			psTmp = NULL;
+		}
+		PVR_DPF_RETURN;
+	}
+}
+
+/*
+ * DoTLSetPacketHeader
+ *
+ * Ensure that whenever we update a Header we always add the RESERVED field
+ */
+static inline void DoTLSetPacketHeader(PVRSRVTL_PPACKETHDR, IMG_UINT32);
+static inline void
+DoTLSetPacketHeader(PVRSRVTL_PPACKETHDR pHdr,
+				IMG_UINT32 ui32Val)
+{
+	PVR_ASSERT(((size_t)pHdr & (size_t)(PVRSRVTL_PACKET_ALIGNMENT - 1)) == 0);
+
+	/* Check that this is a correctly aligned packet header. */
+	if (((size_t)pHdr & (size_t)(PVRSRVTL_PACKET_ALIGNMENT - 1)) != 0)
+	{
+		/* Should return an error because the header is misaligned */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Misaligned header @ %p", __func__, pHdr));
+		pHdr->uiTypeSize = ui32Val;
+	}
+	else
+	{
+		pHdr->uiTypeSize = ui32Val;
+		pHdr->uiReserved = PVRSRVTL_PACKETHDR_RESERVED;
+	}
+}
+
+static PVRSRV_ERROR
+DoTLStreamReserve(IMG_HANDLE hStream,
+				IMG_UINT8 **ppui8Data,
+				IMG_UINT32 ui32ReqSize,
+				IMG_UINT32 ui32ReqSizeMin,
+				PVRSRVTL_PACKETTYPE ePacketType,
+				IMG_UINT32* pui32AvSpace,
+				IMG_UINT32* pui32Flags)
+{
+	PTL_STREAM psTmp;
+	IMG_UINT32 *pui32Buf, ui32LRead, ui32LWrite, ui32LPending, lReqSizeAligned, lReqSizeActual, ui32CreateFreeSpace;
+	IMG_UINT32 ui32InputFlags = 0;
+	IMG_INT pad, iFreeSpace;
+	IMG_UINT8 *pui8IncrRead = NULL;
+	PVRSRVTL_PPACKETHDR pHdr;
+
+	PVR_DPF_ENTERED;
+	if (pui32AvSpace) *pui32AvSpace = 0;
+	if (pui32Flags)
+	{
+		ui32InputFlags = *pui32Flags;
+		*pui32Flags = 0;
+	}
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Assert used as the packet type parameter is currently only provided
+	 * by the TL APIs, not the calling client */
+	PVR_ASSERT((PVRSRVTL_PACKETTYPE_UNDEF < ePacketType) && (PVRSRVTL_PACKETTYPE_LAST >= ePacketType));
+
+	/* The buffer is only used in "rounded" (aligned) chunks */
+	lReqSizeAligned = PVRSRVTL_ALIGN(ui32ReqSize);
+
+	/* Lock the stream before reading it's pending value, because if pending is set
+	 * to NOTHING_PENDING, we update the pending value such that subsequent calls to
+	 * this function from other context(s) fail with PVRSRV_ERROR_NOT_READY */
+	OSLockAcquire (psTmp->hStreamWLock);
+
+#if defined(TL_BUFFER_STATS)
+	/* If writing into an empty buffer, start recording time-to-full */
+	if (psTmp->ui32Read == psTmp->ui32Write)
+	{
+		OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 1);
+		psTmp->ui32TimeStart = OSClockus();
+	}
+
+	if (ui32ReqSize > psTmp->ui32MaxReserveWatermark)
+	{
+		psTmp->ui32MaxReserveWatermark = ui32ReqSize;
+	}
+#endif
+
+	/* Get a local copy of the stream buffer parameters */
+	ui32LRead  = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+	ui32LPending = psTmp->ui32Pending;
+
+	/* Multiple pending reserves are not supported. */
+	if (NOTHING_PENDING != ui32LPending)
+	{
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_READY);
+	}
+
+	if (psTmp->ui32MaxPacketSize < lReqSizeAligned)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Requested Size: %u > TL Max Packet size: %u", lReqSizeAligned, psTmp->ui32MaxPacketSize));
+		psTmp->ui32Pending = NOTHING_PENDING;
+		if (pui32AvSpace)
+		{
+			*pui32AvSpace = suggestAllocSize(ui32LRead, ui32LWrite, psTmp->ui32Size, ui32ReqSizeMin, psTmp->ui32MaxPacketSize);
+			if (*pui32AvSpace == 0 && psTmp->eOpMode == TL_OPMODE_DROP_OLDEST)
+			{
+				*pui32AvSpace = psTmp->ui32MaxPacketSize;
+				PVR_DPF((PVR_DBG_MESSAGE, "Opmode is Drop_Oldest, so Available Space changed to: %u", *pui32AvSpace));
+			}
+		}
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED);
+	}
+
+	/* Prevent other threads from entering this region before we are done
+	 * updating the pending value and write offset (in case of padding). This
+	 * is not exactly a lock but a signal for other contexts that there is a
+	 * TLStreamCommit operation pending on this stream */
+	psTmp->ui32Pending = 0;
+
+	OSLockRelease (psTmp->hStreamWLock);
+
+	/* If there is enough contiguous space following the current Write
+	 * position then no padding is required */
+	if (  psTmp->ui32Size
+		< ui32LWrite + lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR) )
+	{
+		pad = psTmp->ui32Size - ui32LWrite;
+	}
+	else
+	{
+		pad = 0;
+	}
+
+	lReqSizeActual = lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR) + pad;
+	if (psTmp->bNoWrapPermanent)
+	{
+		iFreeSpace = bufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+	else
+	{
+		iFreeSpace = circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+
+	if (iFreeSpace < (IMG_INT) lReqSizeActual)
+	{
+		/* If this is a blocking reserve and there is not enough space then wait. */
+		if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+		{
+			/* Stream create should stop us entering here when
+			 * psTmp->bNoWrapPermanent is true as it does not make sense to
+			 * block on permanent data streams. */
+			PVR_ASSERT(psTmp->bNoWrapPermanent == IMG_FALSE);
+			while ( ( circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size)
+				 <(IMG_INT) lReqSizeActual ) )
+			{
+				/* The TL bridge is lockless now, so changing to OSEventObjectWait() */
+				OSEventObjectWait(psTmp->hProducerEvent);
+				// update local copies.
+				ui32LRead  = psTmp->ui32Read;
+				ui32LWrite = psTmp->ui32Write;
+			}
+		}
+		/* Data overwriting, also insert PACKETS_DROPPED flag into existing packet */
+		else if (psTmp->eOpMode == TL_OPMODE_DROP_OLDEST)
+		{
+			OSLockAcquire(psTmp->hReadLock);
+
+			while (psTmp->bReadPending)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "Waiting for the pending read operation to complete."));
+				OSLockRelease(psTmp->hReadLock);
+#if defined(TL_BUFFER_STATS)
+				TL_COUNTER_INC(psTmp->ui32CntWriteWaits);
+#endif
+				(void) OSEventObjectWaitTimeout(psTmp->hProducerEvent, READ_PENDING_TIMEOUT_US);
+				OSLockAcquire(psTmp->hReadLock);
+			}
+
+#if defined(TL_BUFFER_STATS)
+			TL_COUNTER_INC(psTmp->ui32CntWriteSuccesses);
+#endif
+			ui32LRead = psTmp->ui32Read;
+
+			if ( circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size)
+			     < (IMG_INT) lReqSizeActual )
+			{
+				ui32CreateFreeSpace = 5 * (psTmp->ui32Size / 100);
+				if (ui32CreateFreeSpace < lReqSizeActual)
+				{
+					ui32CreateFreeSpace = lReqSizeActual;
+				}
+
+				while (ui32CreateFreeSpace > (IMG_UINT32)circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size))
+				{
+					pui8IncrRead = &psTmp->pbyBuffer[ui32LRead];
+					ui32LRead += (sizeof(PVRSRVTL_PACKETHDR) + PVRSRVTL_ALIGN( GET_PACKET_DATA_LEN(pui8IncrRead) ));
+
+					/* Check if buffer needs to wrap */
+					if (ui32LRead >= psTmp->ui32Size)
+					{
+						ui32LRead = 0;
+					}
+				}
+				psTmp->ui32Read = ui32LRead;
+				pui8IncrRead = &psTmp->pbyBuffer[psTmp->ui32Read];
+
+				pHdr = GET_PACKET_HDR(pui8IncrRead);
+				DoTLSetPacketHeader(pHdr, SET_PACKETS_DROPPED(pHdr));
+			}
+			/* else fall through as there is enough space now to write the data */
+
+			OSLockRelease(psTmp->hReadLock);
+			/* If we accepted a flag var set the OVERWRITE bit*/
+			if (pui32Flags) *pui32Flags |= TL_FLAG_OVERWRITE_DETECTED;
+		}
+		/* No data overwriting, insert write_failed flag and return */
+		else if (psTmp->eOpMode == TL_OPMODE_DROP_NEWER)
+		{
+			/* Caller should not try to use ppui8Data,
+			 * NULLify to give user a chance of avoiding memory corruption */
+			*ppui8Data = NULL;
+
+			/* This flag should not be inserted two consecutive times, so
+			 * check the last ui32 in case it was a packet drop packet. */
+			pui32Buf = ui32LWrite
+					  ?
+					    (void *)&psTmp->pbyBuffer[ui32LWrite - sizeof(PVRSRVTL_PACKETHDR)]
+					   : // Previous four bytes are not guaranteed to be a packet header...
+					    (void *)&psTmp->pbyBuffer[psTmp->ui32Size - PVRSRVTL_PACKET_ALIGNMENT];
+
+			pHdr = GET_PACKET_HDR(pui32Buf);
+			if ( PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED
+				 !=
+				 GET_PACKET_TYPE( pHdr ) && (ui32InputFlags & TL_FLAG_NO_WRITE_FAILED) == 0)
+			{
+				/* Insert size-stamped packet header */
+				pui32Buf = (void *)&psTmp->pbyBuffer[ui32LWrite];
+				pHdr = GET_PACKET_HDR(pui32Buf);
+				DoTLSetPacketHeader(pHdr, PVRSRVTL_SET_PACKET_WRITE_FAILED);
+				ui32LWrite += sizeof(PVRSRVTL_PACKETHDR);
+				ui32LWrite %= psTmp->ui32Size;
+				iFreeSpace -= sizeof(PVRSRVTL_PACKETHDR);
+			}
+
+			OSLockAcquire (psTmp->hStreamWLock);
+			psTmp->ui32Write = ui32LWrite;
+			psTmp->ui32Pending = NOTHING_PENDING;
+			OSLockRelease (psTmp->hStreamWLock);
+
+			if (pui32AvSpace)
+			{
+				*pui32AvSpace = suggestAllocSize(ui32LRead, ui32LWrite, psTmp->ui32Size, ui32ReqSizeMin, psTmp->ui32MaxPacketSize);
+			}
+
+			/* Inform call of permanent stream misuse, no space left,
+			 * the size of the stream will need to be increased. */
+			if (psTmp->bNoWrapPermanent)
+			{
+				PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE);
+			}
+
+			PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_FULL);
+		}
+	}
+
+	/* The easy case: buffer has enough space to hold the requested packet (data + header) */
+
+	/* Should we treat the buffer as non-circular buffer? */
+	if (psTmp->bNoWrapPermanent)
+	{
+		iFreeSpace = bufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+	else
+	{
+		iFreeSpace = circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+
+	if (iFreeSpace >= (IMG_INT) lReqSizeActual)
+	{
+		if (pad)
+		{
+			/* Inserting padding packet. */
+			pui32Buf = (void *)&psTmp->pbyBuffer[ui32LWrite];
+			pHdr = GET_PACKET_HDR(pui32Buf);
+			DoTLSetPacketHeader(pHdr,
+				PVRSRVTL_SET_PACKET_PADDING(pad-sizeof(PVRSRVTL_PACKETHDR)));
+
+			/* CAUTION: the used pad value should always result in a properly
+			 *          aligned ui32LWrite pointer, which in this case is 0 */
+			ui32LWrite = (ui32LWrite + pad) % psTmp->ui32Size;
+			/* Detect unaligned pad value */
+			PVR_ASSERT(ui32LWrite == 0);
+		}
+		/* Insert size-stamped packet header */
+		pui32Buf = (void *) &psTmp->pbyBuffer[ui32LWrite];
+
+		pHdr = GET_PACKET_HDR(pui32Buf);
+		DoTLSetPacketHeader(pHdr,
+			PVRSRVTL_SET_PACKET_HDR(ui32ReqSize, ePacketType));
+
+		/* return the next position in the buffer to the user */
+		*ppui8Data = &psTmp->pbyBuffer[ ui32LWrite+sizeof(PVRSRVTL_PACKETHDR) ];
+
+		/* update pending offset: size stamp + data */
+		ui32LPending = lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR);
+	}
+	else
+	{
+		OSLockAcquire (psTmp->hStreamWLock);
+		psTmp->ui32Pending = NOTHING_PENDING;
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire stream lock for updating stream parameters */
+	OSLockAcquire (psTmp->hStreamWLock);
+	psTmp->ui32Write = ui32LWrite;
+	psTmp->ui32Pending = ui32LPending;
+	OSLockRelease (psTmp->hStreamWLock);
+
+#if defined(TL_BUFFER_STATS)
+	TL_COUNTER_INC(psTmp->ui32CntNumWriteSuccess);
+#endif
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamReserve(IMG_HANDLE hStream,
+				IMG_UINT8 **ppui8Data,
+				IMG_UINT32 ui32Size)
+{
+	return DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32Size, PVRSRVTL_PACKETTYPE_DATA, NULL, NULL);
+}
+
+PVRSRV_ERROR
+TLStreamReserve2(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size,
+                IMG_UINT32 ui32SizeMin,
+                IMG_UINT32* pui32Available,
+                IMG_BOOL* pbIsReaderConnected)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32SizeMin, PVRSRVTL_PACKETTYPE_DATA, pui32Available, NULL);
+	if (eError != PVRSRV_OK && pbIsReaderConnected != NULL)
+	{
+		*pbIsReaderConnected = TLStreamIsOpenForReading(hStream);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+TLStreamReserveReturnFlags(IMG_HANDLE hStream,
+						   IMG_UINT8  **ppui8Data,
+						   IMG_UINT32 ui32Size,
+						   IMG_UINT32* pui32Flags)
+{
+	return DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32Size, PVRSRVTL_PACKETTYPE_DATA, NULL, pui32Flags);
+}
+
+PVRSRV_ERROR
+TLStreamCommit(IMG_HANDLE hStream, IMG_UINT32 ui32ReqSize)
+{
+	PTL_STREAM psTmp;
+	IMG_UINT32 ui32LRead, ui32OldWrite, ui32LWrite, ui32LPending;
+	PVRSRV_ERROR eError;
+
+#if defined(TL_BUFFER_STATS)
+	IMG_UINT32 ui32UnreadBytes;
+#endif
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Get a local copy of the stream buffer parameters */
+	ui32LRead = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+	ui32LPending = psTmp->ui32Pending;
+
+	ui32OldWrite = ui32LWrite;
+
+	// Space in buffer is aligned
+	ui32ReqSize = PVRSRVTL_ALIGN(ui32ReqSize) + sizeof(PVRSRVTL_PACKETHDR);
+
+	/* Check pending reserver and ReqSize + packet header size. */
+	if ((ui32LPending == NOTHING_PENDING) || (ui32ReqSize > ui32LPending))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_MISUSE);
+	}
+
+	/* Update pointer to written data. */
+	ui32LWrite = (ui32LWrite + ui32ReqSize) % psTmp->ui32Size;
+
+	/* and reset LPending to 0 since data are now submitted */
+	ui32LPending = NOTHING_PENDING;
+
+#if defined(TL_BUFFER_STATS)
+	/* Calculate new number of bytes unread */
+	if (ui32LWrite > ui32LRead)
+	{
+		ui32UnreadBytes = (ui32LWrite-ui32LRead);
+	}
+	else if (ui32LWrite < ui32LRead)
+	{
+		ui32UnreadBytes = (psTmp->ui32Size-ui32LRead+ui32LWrite);
+	}
+	else
+	{ /* else equal, ignore */
+		ui32UnreadBytes = 0;
+	}
+
+	/* Calculate high water mark for debug purposes */
+	if (ui32UnreadBytes > psTmp->ui32BufferUt)
+	{
+		psTmp->ui32BufferUt = ui32UnreadBytes;
+	}
+#endif
+
+	/* Memory barrier required to ensure prior data written by writer is
+	 * flushed from WC buffer to main memory. */
+	OSWriteMemoryBarrier(NULL);
+
+	/* Acquire stream lock to ensure other context(s) (if any)
+	 * wait on the lock (in DoTLStreamReserve) for consistent values
+	 * of write offset and pending value */
+	OSLockAcquire (psTmp->hStreamWLock);
+
+	/* Update stream buffer parameters to match local copies */
+	psTmp->ui32Write = ui32LWrite;
+	psTmp->ui32Pending = ui32LPending;
+
+	/* Ensure write pointer is flushed */
+	OSWriteMemoryBarrier(&psTmp->ui32Write);
+
+	TL_COUNTER_ADD(psTmp->ui32ProducerByteCount, ui32ReqSize);
+	TL_COUNTER_INC(psTmp->ui32NumCommits);
+
+#if defined(TL_BUFFER_STATS)
+	/* IF there has been no-reader since first reserve on an empty-buffer,
+	 * AND current utilisation is considerably high (90%), calculate the
+	 * time taken to fill up the buffer */
+	if ((OSAtomicRead(&psTmp->bNoReaderSinceFirstReserve) == 1) &&
+	    (TLStreamGetUT(psTmp) >= 90 * psTmp->ui32Size/100))
+	{
+		IMG_UINT32 ui32TimeToFullInUs = OSClockus() - psTmp->ui32TimeStart;
+		if (psTmp->ui32MinTimeToFullInUs > ui32TimeToFullInUs)
+		{
+			psTmp->ui32MinTimeToFullInUs = ui32TimeToFullInUs;
+		}
+		/* Following write ensures ui32MinTimeToFullInUs doesn't lose its
+		 * real (expected) value in case there is no reader until next Commit call */
+		OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 0);
+	}
+#endif
+
+	if (!psTmp->bNoSignalOnCommit)
+	{
+		/* If we have transitioned from an empty buffer to a non-empty buffer, we
+		 * must signal possibly waiting consumer. BUT, let the signal be "deferred"
+		 * until buffer is at least 'ui32ThresholdUsageForSignal' bytes full. This
+		 * avoids a race between OSEventObjectSignal and OSEventObjectWaitTimeout
+		 * (in TLServerAcquireDataKM), where a "signal" might happen before "wait",
+		 * resulting into signal being lost and stream-reader waiting even though
+		 * buffer is no-more empty */
+		if (ui32OldWrite == ui32LRead)
+		{
+			psTmp->bSignalPending = IMG_TRUE;
+		}
+
+		if (psTmp->bSignalPending && (TLStreamGetUT(psTmp) >= psTmp->ui32ThresholdUsageForSignal))
+		{
+			TL_COUNTER_INC(psTmp->ui32SignalsSent);
+			psTmp->bSignalPending = IMG_FALSE;
+
+			/* Signal consumers that may be waiting */
+			eError = OSEventObjectSignal(psTmp->psNode->hReadEventObj);
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease (psTmp->hStreamWLock);
+				PVR_DPF_RETURN_RC(eError);
+			}
+		}
+		else
+		{
+			TL_COUNTER_INC(psTmp->ui32SignalNotSent);
+		}
+	}
+	OSLockRelease (psTmp->hStreamWLock);
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamWrite(IMG_HANDLE hStream, IMG_UINT8 *pui8Src, IMG_UINT32 ui32Size)
+{
+	IMG_BYTE *pbyDest = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = TLStreamReserve(hStream, &pbyDest, ui32Size);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	else
+	{
+		OSDeviceMemCopy((void*)pbyDest, (void*)pui8Src, ui32Size);
+		eError = TLStreamCommit(hStream, ui32Size);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF_RETURN_RC(eError);
+		}
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamWriteRetFlags(IMG_HANDLE hStream, IMG_UINT8 *pui8Src, IMG_UINT32 ui32Size, IMG_UINT32 *pui32Flags){
+	IMG_BYTE *pbyDest = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = TLStreamReserveReturnFlags(hStream, &pbyDest, ui32Size, pui32Flags);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	else
+	{
+		OSDeviceMemCopy((void*)pbyDest, (void*)pui8Src, ui32Size);
+		eError = TLStreamCommit(hStream, ui32Size);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF_RETURN_RC(eError);
+		}
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+void TLStreamInfo(IMG_HANDLE hStream, PTL_STREAM_INFO psInfo)
+{
+	IMG_DEVMEM_SIZE_T actual_req_size;
+	IMG_DEVMEM_ALIGN_T align = 4; /* Low fake value so the real value can be obtained */
+
+	actual_req_size = 2;
+	/* ignore error as OSGetPageShift() should always return correct value */
+	(void) DevmemExportalignAdjustSizeAndAlign(OSGetPageShift(), &actual_req_size, &align);
+
+	psInfo->headerSize = sizeof(PVRSRVTL_PACKETHDR);
+	psInfo->minReservationSize = sizeof(IMG_UINT32);
+	psInfo->pageSize = (IMG_UINT32)(actual_req_size);
+	psInfo->pageAlign = (IMG_UINT32)(align);
+	psInfo->maxTLpacketSize = ((PTL_STREAM)hStream)->ui32MaxPacketSize;
+}
+
+PVRSRV_ERROR
+TLStreamMarkEOS(IMG_HANDLE psStream, IMG_BOOL bRemoveOld)
+{
+	PTL_STREAM   psTmp;
+	PVRSRV_ERROR eError;
+	IMG_UINT8*   pData;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psTmp = (PTL_STREAM)psStream;
+
+	/* Do not support EOS packets on permanent stream buffers at present,
+	 * EOS is best used with streams where data is consumed. */
+	if (psTmp->bNoWrapPermanent)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_MISUSE);
+	}
+
+	if (bRemoveOld)
+	{
+		eError = DoTLStreamReserve(psStream, &pData, 0, 0, PVRSRVTL_PACKETTYPE_MARKER_EOS_REMOVEOLD, NULL, NULL);
+	}
+	else
+	{
+		eError = DoTLStreamReserve(psStream, &pData, 0, 0, PVRSRVTL_PACKETTYPE_MARKER_EOS, NULL, NULL);
+	}
+
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	PVR_DPF_RETURN_RC(TLStreamCommit(psStream, 0));
+}
+
+
+static PVRSRV_ERROR
+_TLStreamMarkOC(IMG_HANDLE hStream, PVRSRVTL_PACKETTYPE ePacketType)
+{
+	PVRSRV_ERROR eError;
+	PTL_STREAM psStream = hStream;
+	IMG_UINT32 ui32Size;
+	IMG_UINT8 *pData;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	if (NULL == psStream->psNotifStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_NOTIF_STREAM);
+	}
+
+	ui32Size = OSStringLength(psStream->szName) + 1;
+
+	eError = DoTLStreamReserve(psStream->psNotifStream, &pData, ui32Size,
+	                           ui32Size, ePacketType, NULL, NULL);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	OSDeviceMemCopy(pData, psStream->szName, ui32Size);
+
+	PVR_DPF_RETURN_RC(TLStreamCommit(psStream->psNotifStream, ui32Size));
+}
+
+PVRSRV_ERROR
+TLStreamMarkStreamOpen(IMG_HANDLE psStream)
+{
+	return _TLStreamMarkOC(psStream, PVRSRVTL_PACKETTYPE_STREAM_OPEN_FOR_WRITE);
+}
+
+PVRSRV_ERROR
+TLStreamMarkStreamClose(IMG_HANDLE psStream)
+{
+	return _TLStreamMarkOC(psStream, PVRSRVTL_PACKETTYPE_STREAM_CLOSE_FOR_WRITE);
+}
+
+PVRSRV_ERROR
+TLStreamSync(IMG_HANDLE psStream)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PTL_STREAM   psTmp;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)psStream;
+
+	/* If read client exists and has opened stream in blocking mode,
+	 * signal when data is available to read. */
+	if (psTmp->psNode->psRDesc &&
+		 (!(psTmp->psNode->psRDesc->ui32Flags & PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING)) &&
+			psTmp->ui32Read != psTmp->ui32Write)
+	{
+		TL_COUNTER_INC(psTmp->ui32ManSyncs);
+		eError = OSEventObjectSignal(psTmp->psNode->hReadEventObj);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+IMG_BOOL
+TLStreamIsOpenForReading(IMG_HANDLE hStream)
+{
+	PTL_STREAM psTmp;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+
+	PVR_DPF_RETURN_VAL(psTmp->psNode->psRDesc != NULL);
+}
+
+IMG_BOOL
+TLStreamOutOfData(IMG_HANDLE hStream)
+{
+	PTL_STREAM psTmp;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+
+	/* If both pointers are equal then the buffer is empty */
+	PVR_DPF_RETURN_VAL(psTmp->ui32Read == psTmp->ui32Write);
+}
+
+
+PVRSRV_ERROR
+TLStreamResetProducerByteCount(IMG_HANDLE hStream, IMG_UINT32 ui32Value)
+{
+	PTL_STREAM   psTmp;
+	IMG_UINT32   ui32LRead, ui32LWrite;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+	ui32LRead = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+
+	if (ui32LRead != ui32LWrite)
+	{
+		eErr = PVRSRV_ERROR_STREAM_MISUSE;
+	}
+#if defined(TL_BUFFER_STATS)
+	psTmp->ui32ProducerByteCount = ui32Value;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+#endif
+	PVR_DPF_RETURN_RC(eErr);
+}
+/*
+ * Internal stream APIs to server part of Transport Layer, declared in
+ * header tlintern.h. Direct pointers to stream objects are used here as
+ * these functions are internal.
+ */
+IMG_UINT32
+TLStreamAcquireReadPos(PTL_STREAM psStream,
+                       IMG_BOOL bDisableCallback,
+                       IMG_UINT32* puiReadOffset)
+{
+	IMG_UINT32 uiReadLen = 0;
+	IMG_UINT32 ui32LRead, ui32LWrite;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+	PVR_ASSERT(puiReadOffset);
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		if (!OSTryLockAcquire(psStream->hReadLock))
+		{
+			/*
+			 * This is a normal event when the system is under load.
+			 * An example of how to produce this is to run testrunner /
+			 * regression/ddk_test_seq2_host_fw_mem.conf with HTB / pvrhtbd
+			 * configured as
+			 *
+			 * # pvrdebug -log trace -loggroups main,pow,debug \
+			 * -hostloggroups main,ctrl,sync,brg -hostlogtype dropoldest
+			 *
+			 * # pvrhtbd -hostloggroups main,ctrl,sync,brg
+			 *
+			 * We will see a small number of these collisions but as this is
+			 * an expected calling path, and an expected return code, we drop
+			 * the severity to just be a debug MESSAGE instead of WARNING
+			 */
+			PVR_DPF((PVR_DBG_MESSAGE,
+			    "%s: Read lock on stream '%s' is acquired by some writer, "
+			    "hence reader failed to acquire read lock.", __func__,
+			    psStream->szName));
+#if defined(TL_BUFFER_STATS)
+			TL_COUNTER_INC(psStream->ui32CntReadFails);
+#endif
+			PVR_DPF_RETURN_VAL(0);
+		}
+	}
+
+#if defined(TL_BUFFER_STATS)
+	TL_COUNTER_INC(psStream->ui32CntReadSuccesses);
+#endif
+
+	/* Grab a local copy */
+	ui32LRead = psStream->ui32Read;
+	ui32LWrite = psStream->ui32Write;
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		psStream->bReadPending = IMG_TRUE;
+		OSLockRelease(psStream->hReadLock);
+	}
+
+	/* No data available and CB defined - try and get data */
+	if ((ui32LRead == ui32LWrite) && psStream->pfProducerCallback && !bDisableCallback)
+	{
+		PVRSRV_ERROR eRc;
+		IMG_UINT32   ui32Resp = 0;
+
+		eRc = ((TL_STREAM_SOURCECB)psStream->pfProducerCallback)(psStream, TL_SOURCECB_OP_CLIENT_EOS,
+				&ui32Resp, psStream->pvProducerUserData);
+		PVR_LOG_IF_ERROR(eRc, "TLStream->pfProducerCallback");
+
+		ui32LWrite = psStream->ui32Write;
+	}
+
+	/* No data available... */
+	if (ui32LRead == ui32LWrite)
+	{
+		if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+		{
+			psStream->bReadPending = IMG_FALSE;
+		}
+		PVR_DPF_RETURN_VAL(0);
+	}
+
+#if defined(TL_BUFFER_STATS)
+	/* The moment reader knows it will see a non-zero data, it marks its presence in writer's eyes */
+	OSAtomicWrite (&psStream->bNoReaderSinceFirstReserve, 0);
+#endif
+
+	/* Data is available to read... */
+	*puiReadOffset = ui32LRead;
+
+	/*PVR_DPF((PVR_DBG_VERBOSE,
+	 *		"TLStreamAcquireReadPos Start before: Write:%d, Read:%d, size:%d",
+	 *		ui32LWrite, ui32LRead, psStream->ui32Size));
+	 */
+
+	if (ui32LRead > ui32LWrite)
+	{	/* CB has wrapped around. */
+		PVR_ASSERT(!psStream->bNoWrapPermanent);
+		/* Return the first contiguous piece of memory, ie [ReadLen,EndOfBuffer]
+		 * and let a subsequent AcquireReadPos read the rest of the Buffer */
+		/*PVR_DPF((PVR_DBG_VERBOSE, "TLStreamAcquireReadPos buffer has wrapped"));*/
+		uiReadLen = psStream->ui32Size - ui32LRead;
+		TL_COUNTER_INC(psStream->ui32AcquireRead2);
+	}
+	else
+	{	/* CB has not wrapped */
+		uiReadLen = ui32LWrite - ui32LRead;
+		TL_COUNTER_INC(psStream->ui32AcquireRead1);
+	}
+
+	PVR_DPF_RETURN_VAL(uiReadLen);
+}
+
+PVRSRV_ERROR
+TLStreamAdvanceReadPos(PTL_STREAM psStream,
+                       IMG_UINT32 uiReadLen,
+                       IMG_UINT32 uiOrigReadLen)
+{
+	IMG_UINT32 uiNewReadPos;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	/*
+	 * This API does not use Read lock as 'bReadPending' is sufficient
+	 * to keep Read index safe by preventing a write from updating the
+	 * index and 'bReadPending' itself is safe as it can only be modified
+	 * by readers and there can be only one reader in action at a time.
+	 */
+
+	/* Update the read offset by the length provided in a circular manner.
+	 * Assuming the update to be atomic hence, avoiding use of locks
+	 */
+	uiNewReadPos = (psStream->ui32Read + uiReadLen) % psStream->ui32Size;
+
+	/* Must validate length is on a packet boundary, for
+	 * TLReleaseDataLess calls.
+	 */
+	if (uiReadLen != uiOrigReadLen) /* buffer not empty */
+	{
+		PVRSRVTL_PPACKETHDR psHdr = GET_PACKET_HDR(psStream->pbyBuffer+uiNewReadPos);
+		PVRSRVTL_PACKETTYPE eType = GET_PACKET_TYPE(psHdr);
+
+		if ((psHdr->uiReserved != PVRSRVTL_PACKETHDR_RESERVED) ||
+			(eType == PVRSRVTL_PACKETTYPE_UNDEF) ||
+			(eType >= PVRSRVTL_PACKETTYPE_LAST))
+		{
+			PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_ALIGNMENT);
+		}
+		/* else OK, on a packet boundary */
+	}
+	/* else no check needed */
+
+	psStream->ui32Read = uiNewReadPos;
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		psStream->bReadPending = IMG_FALSE;
+	}
+
+	/* notify reserves that may be pending */
+	/* The producer event object is used to signal the StreamReserve if the TL
+	 * Buffer is in blocking mode and is full.
+	 * Previously this event was only signalled if the buffer was created in
+	 * blocking mode. Since the buffer mode can now change dynamically the event
+	 * is signalled every time to avoid any potential race where the signal is
+	 * required, but not produced.
+	 */
+	{
+		PVRSRV_ERROR eError;
+		eError = OSEventObjectSignal(psStream->hProducerEventObj);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "Error in TLStreamAdvanceReadPos: OSEventObjectSignal returned:%u",
+					 eError));
+			/* We've failed to notify the producer event. This means there may
+			 * be a delay in generating more data to be consumed until the next
+			 * Write() generating action occurs.
+			 */
+		}
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE,
+			 "TLStreamAdvanceReadPos Read now at: %d",
+			psStream->ui32Read));
+	PVR_DPF_RETURN_OK;
+}
+
+void
+TLStreamResetReadPos(PTL_STREAM psStream)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	if (psStream->bNoWrapPermanent)
+	{
+
+		/* Update the read offset by the length provided in a circular manner.
+		 * Assuming the update to be atomic hence, avoiding use of locks */
+		psStream->ui32Read = 0;
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+				 "TLStreamResetReadPos Read now at: %d",
+					psStream->ui32Read));
+	}
+	else
+	{
+		/* else for other stream types this is a no-op */
+		PVR_DPF((PVR_DBG_VERBOSE,
+				"No need to reset read position of circular tlstream"));
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void
+TLStreamDestroy (PTL_STREAM psStream)
+{
+	PVR_ASSERT (psStream);
+
+	OSLockDestroy (psStream->hStreamWLock);
+	OSLockDestroy (psStream->hReadLock);
+
+	OSEventObjectClose(psStream->hProducerEvent);
+	OSEventObjectDestroy(psStream->hProducerEventObj);
+
+	TLFreeSharedMem(psStream);
+	OSFreeMem(psStream);
+}
+
+DEVMEM_MEMDESC*
+TLStreamGetBufferPointer(PTL_STREAM psStream)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	PVR_DPF_RETURN_VAL(psStream->psStreamMemDesc);
+}
+
+/*
+ * Determine the maximum transfer size which will fit into the specified
+ * L2 consumer stream.
+ * This is a point-in-time snapshot which compares the given requested transfer
+ * size and returns the minimum of that vs the available buffer space within
+ * the L2 stream. This is the largest amount of data that can be successfully
+ * copied in the stream without truncation occurring.
+ * Worst-case scenario is that we copy fewer bytes than the maximum
+ * actually available, but we will never copy too much data.
+ */
+IMG_UINT32
+TLStreamGetMaxTransfer(IMG_UINT32 uiXferSize, IMG_HANDLE hConsumerStream)
+{
+	PTL_STREAM  psConsumer = (PTL_STREAM)hConsumerStream;
+	IMG_UINT32  ui32MaxTransfer = 0U;
+
+	IMG_UINT32  ui32Xfer;
+
+	/* Local copies */
+	IMG_UINT32	ui32Read = psConsumer->ui32Read;
+	IMG_UINT32	ui32Write = psConsumer->ui32Write;
+
+	if (ui32Write >= ui32Read)
+	{
+		/* Can transfer Write .. End-of-buffer + Read bytes at start */
+		ui32Xfer = psConsumer->ui32Size - ui32Write + ui32Read;
+	}
+	else
+	{
+		/* Can transfer Read - Write bytes maximum */
+		ui32Xfer = ui32Read - ui32Write;
+	}
+
+	PVR_ASSERT(ui32Xfer <= psConsumer->ui32Size);
+
+	ui32MaxTransfer = MIN(ui32Xfer, uiXferSize);
+
+	return ui32MaxTransfer;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_client.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_client.c
new file mode 100644
index 000000000000..c989cea789ea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_client.c
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File			vmm_pvz_client.c
+@Title          VM manager client para-virtualization
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides VMM client para-virtualization APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#include "vmm_impl.h"
+#include "vz_vmm_pvz.h"
+#include "vmm_pvz_client.h"
+
+
+static inline void
+PvzClientLockAcquire(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockAcquire(psPVRSRVData->hPvzConnectionLock);
+}
+
+static inline void
+PvzClientLockRelease(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockRelease(psPVRSRVData->hPvzConnectionLock);
+}
+
+/*
+ * ===========================================================
+ *  The following client para-virtualization (pvz) functions
+ *  are exclusively called by guests to initiate a pvz call
+ *  to the host via hypervisor (guest -> vm manager -> host)
+ * ===========================================================
+ */
+
+PVRSRV_ERROR
+PvzClientMapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_PHYADDR sDevPAddr;
+	VMM_PVZ_CONNECTION *psVmmPvz;
+	PHYS_HEAP *psFwPhysHeap = psDevConfig->psDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+
+	eError = PhysHeapGetDevPAddr(psFwPhysHeap, &sDevPAddr);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+	{
+		IMG_DEV_PHYADDR sDevPAddrTranslated;
+
+		/* If required, perform a software translation between CPU and Device physical addresses. */
+		PhysHeapCpuPAddrToDevPAddr(psFwPhysHeap, 1, &sDevPAddrTranslated, (IMG_CPU_PHYADDR *)&sDevPAddr);
+		sDevPAddr.uiAddr = sDevPAddrTranslated.uiAddr;
+	}
+#endif
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapGetDevPAddr");
+	PVR_LOG_RETURN_IF_FALSE((sDevPAddr.uiAddr != 0), "PhysHeapGetDevPAddr", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psVmmPvz = PvzConnectionAcquire();
+	PvzClientLockAcquire();
+
+	eError = psVmmPvz->sClientFuncTab.pfnMapDevPhysHeap(RGX_FIRMWARE_RAW_HEAP_SIZE, sDevPAddr.uiAddr);
+
+	PvzClientLockRelease();
+	PvzConnectionRelease(psVmmPvz);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzClientUnmapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	VMM_PVZ_CONNECTION *psVmmPvz = PvzConnectionAcquire();
+	PVR_ASSERT(psVmmPvz);
+
+	PvzClientLockAcquire();
+
+	PVR_ASSERT(psVmmPvz->sClientFuncTab.pfnUnmapDevPhysHeap);
+
+	eError = psVmmPvz->sClientFuncTab.pfnUnmapDevPhysHeap();
+
+	PvzClientLockRelease();
+	PvzConnectionRelease(psVmmPvz);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (vmm_pvz_client.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_server.c
new file mode 100644
index 000000000000..7e67d397a908
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_server.c
@@ -0,0 +1,251 @@
+/*************************************************************************/ /*!
+@File			vmm_pvz_server.c
+@Title          VM manager server para-virtualization handlers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides VMM server para-virtz handler APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxfwutils.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vz_vmm_pvz.h"
+#include "vmm_pvz_server.h"
+
+static inline void
+PvzServerLockAcquire(void)
+{
+#if !defined(FPGA)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockAcquire(psPVRSRVData->hPvzConnectionLock);
+#endif
+}
+
+static inline void
+PvzServerLockRelease(void)
+{
+#if !defined(FPGA)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockRelease(psPVRSRVData->hPvzConnectionLock);
+#endif
+}
+
+#define VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID) do {							\
+	if ((ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED) ||								\
+		(ui32DriverID < RGXFW_GUEST_DRIVER_ID_START))								\
+	{																				\
+		PVR_DPF((PVR_DBG_ERROR,														\
+					"%s: Invalid OSID %u. Supported Guest OSID range: %u - %u",		\
+					__func__,														\
+					ui32DriverID,													\
+					RGXFW_GUEST_DRIVER_ID_START,									\
+					RGX_NUM_DRIVERS_SUPPORTED-1));									\
+		return PVRSRV_ERROR_INVALID_PARAMS;											\
+	}																				\
+	if (PVRSRVGetDeviceInstance(ui32DevID) == NULL)									\
+	{																				\
+		PVR_DPF((PVR_DBG_ERROR,														\
+					"%s: Invalid Device ID %u.",									\
+					__func__,														\
+					ui32DevID));													\
+		return PVRSRV_ERROR_INVALID_PARAMS;											\
+	}																				\
+} while (false);
+
+/*
+ * ===========================================================
+ *  The following server para-virtualization (pvz) functions
+ *  are exclusively called by the VM manager (hypervisor) on
+ *  behalf of guests to complete guest pvz calls
+ *  (guest -> vm manager -> host)
+ * ===========================================================
+ */
+
+PVRSRV_ERROR
+PvzServerMapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						IMG_UINT32 ui32DevID,
+						IMG_UINT64 ui64Size,
+						IMG_UINT64 ui64PAddr)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && !defined(FPGA)
+		/*
+		 * Reject hypercall if called on a system configured at build time to
+		 * preallocate the Guest's firmware heaps from static carveout memory.
+		 */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Host PVZ config: Does not match with Guest PVZ config."
+		         " Host preallocates the Guest's FW physheap from static memory carveouts at startup.", __func__));
+		return PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+#else
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+
+	PvzServerLockAcquire();
+
+#if defined(SUPPORT_RGX)
+	if (IsVmOnline(ui32DriverID, ui32DevID))
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstance(ui32DevID);
+		IMG_DEV_PHYADDR sDevPAddr = {ui64PAddr};
+		IMG_UINT32 sync;
+
+		eError = RGXFwRawHeapAllocMap(psDeviceNode, ui32DriverID, sDevPAddr, ui64Size);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", e0);
+
+		/* Invalidate MMU cache in preparation for a kick from this Guest */
+		eError = psDeviceNode->pfnMMUCacheInvalidateKick(psDeviceNode, &sync);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMUCacheInvalidateKick", e0);
+
+		/* Everything is ready for the firmware to start interacting with this OS */
+		eError = RGXFWSetFwOsState(psDeviceNode->pvDevice, ui32DriverID, RGXFWIF_OS_ONLINE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Paravirtualized request to map Guest%u's Firmware heap"
+								" rejected: Guest not allowed to run.", __func__, ui32DriverID));
+		eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+	}
+e0:
+#endif /* defined(SUPPORT_RGX) */
+	PvzServerLockRelease();
+
+	return eError;
+#endif
+}
+
+PVRSRV_ERROR
+PvzServerUnmapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						  IMG_UINT32 ui32DevID)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && !defined(FPGA)
+		/*
+		 * Reject hypercall if called on a system configured at built time to
+		 * preallocate the Guest's firmware heaps from static carveout memory.
+		 */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Host PVZ config: Does not match with Guest PVZ config."
+		         " Host preallocates the Guest's FW physheap from static memory carveouts at startup.", __func__));
+		return PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+#else
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+
+	PvzServerLockAcquire();
+
+#if defined(SUPPORT_RGX)
+	if (IsVmOnline(ui32DriverID, ui32DevID))
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+		/* Order firmware to offload this OS' data and stop accepting commands from it */
+		eError = RGXFWSetFwOsState(psDeviceNode->pvDevice, ui32DriverID, RGXFWIF_OS_OFFLINE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXFWSetFwOsState", e0);
+
+		/* it is now safe to remove the Guest's memory mappings */
+		RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+	}
+e0:
+#endif
+
+	PvzServerLockRelease();
+
+	return eError;
+#endif
+}
+
+/*
+ * ============================================================
+ *  The following server para-virtualization (pvz) functions
+ *  are exclusively called by the VM manager (hypervisor) to
+ *  pass side band information to the host (vm manager -> host)
+ * ============================================================
+ */
+
+PVRSRV_ERROR
+PvzServerOnVmOnline(IMG_UINT32 ui32DriverID,
+					IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzOnVmOnline(ui32DriverID, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzServerOnVmOffline(IMG_UINT32 ui32DriverID,
+					 IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzOnVmOffline(ui32DriverID, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzServerVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+					  IMG_UINT32 ui32ParamValue,
+					  IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(RGXFW_GUEST_DRIVER_ID_START, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzVMMConfigure(eVMMParamType, ui32ParamValue, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (vmm_pvz_server.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_pvz.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_pvz.c
new file mode 100644
index 000000000000..bb7c8f6426c6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_pvz.c
@@ -0,0 +1,201 @@
+/*************************************************************************/ /*!
+@File           vz_vmm_pvz.c
+@Title          VM manager para-virtualization APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager para-virtualization management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "vz_vmm_pvz.h"
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR
+PvzConnectionValidate(void)
+{
+	VMM_PVZ_CONNECTION *psVmmPvz;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/*
+	 * Acquire the underlying VM manager PVZ connection & validate it.
+	 */
+	psVmmPvz = PvzConnectionAcquire();
+	if (psVmmPvz == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Unable to acquire PVZ connection", __func__));
+		eError = PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+		goto e0;
+	}
+
+	/* Log which PVZ setup type is being used by driver */
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/*
+	 *  Static PVZ bootstrap setup
+	 *
+	 *  This setup uses carve-out memory, has no hypercall mechanism & does not support
+	 *  out-of-order initialisation of host/guest VMs/drivers. The host driver has all
+	 *  the information needed to initialize all Drivers firmware state when it's loaded
+	 *  and its PVZ layer must mark all guest Drivers as being online as part of its PVZ
+	 *  initialisation. Having no out-of-order initialisation support, the guest driver
+	 *  can only submit a workload to the device after the host driver has completely
+	 *  initialized the firmware, the VZ hypervisor/VM setup must guarantee this.
+	 */
+	PVR_LOG(("Using static PVZ bootstrap setup"));
+#else
+	/*
+	 *  Dynamic PVZ bootstrap setup
+	 *
+	 *  This setup uses guest memory, has PVZ hypercall mechanism & supports out-of-order
+	 *  initialisation of host/guest VMs/drivers. The host driver initializes only its
+	 *  own Driver-0 firmware state when its loaded and each guest driver will use its PVZ
+	 *  interface to hypercall to the host driver to both synchronise its initialisation
+	 *  so it does not submit any workload to the firmware before the host driver has
+	 *  had a chance to initialize the firmware and to also initialize its own Driver-x
+	 *  firmware state.
+	 */
+	PVR_LOG(("Using dynamic PVZ bootstrap setup"));
+#endif
+
+	PvzConnectionRelease(psVmmPvz);
+e0:
+	return eError;
+}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+PVRSRV_ERROR PvzConnectionInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+
+#if (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_DPF((PVR_DBG_ERROR, "This kernel driver does not support virtualization. Please rebuild with RGX_NUM_DRIVERS_SUPPORTED > 1"));
+	eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	goto e0;
+#else
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if ((psPVRSRVData->hPvzConnection != NULL) &&
+		(psPVRSRVData->hPvzConnectionLock != NULL))
+	{
+		eError = PVRSRV_OK;
+		PVR_DPF((PVR_DBG_MESSAGE, "PVzConnection already initialised."));
+		goto e0;
+	}
+
+	/* Create para-virtualization connection lock */
+	eError = OSLockCreate(&psPVRSRVData->hPvzConnectionLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	/* Create VM manager para-virtualization connection */
+	eError = VMMCreatePvzConnection((VMM_PVZ_CONNECTION **)&psPVRSRVData->hPvzConnection, psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockDestroy(psPVRSRVData->hPvzConnectionLock);
+		psPVRSRVData->hPvzConnectionLock = NULL;
+
+		PVR_LOG_ERROR(eError, "VMMCreatePvzConnection");
+		goto e0;
+	}
+
+	/* Ensure pvz connection is configured correctly */
+	eError = PvzConnectionValidate();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PvzConnectionValidate");
+#endif
+e0:
+	return eError;
+}
+
+void PvzConnectionDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDN;
+	IMG_BOOL bCanDestroyPvzData = IMG_TRUE;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	for (psDN = psPVRSRVData->psDeviceNodeList; psDN != NULL; psDN = psDN->psNext)
+	{
+		if ((psDN->psDevConfig != psDevConfig) &&
+			(!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDN)))
+		{
+			/* if any other virtual devices are present keep the pvz data */
+			bCanDestroyPvzData = IMG_FALSE;
+			break;
+		}
+	}
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (bCanDestroyPvzData)
+	{
+		if ((psPVRSRVData->hPvzConnection == NULL) &&
+			(psPVRSRVData->hPvzConnectionLock == NULL))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "PVzConnection already deinitialised."));
+			return;
+		}
+
+		VMMDestroyPvzConnection(psPVRSRVData->hPvzConnection, psDevConfig);
+		psPVRSRVData->hPvzConnection = NULL;
+
+		OSLockDestroy(psPVRSRVData->hPvzConnectionLock);
+		psPVRSRVData->hPvzConnectionLock = NULL;
+	}
+}
+
+VMM_PVZ_CONNECTION* PvzConnectionAcquire(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVR_ASSERT(psPVRSRVData->hPvzConnection != NULL);
+	return psPVRSRVData->hPvzConnection;
+}
+
+void PvzConnectionRelease(VMM_PVZ_CONNECTION *psParaVz)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	/* Nothing to do, just validate the pointer we're passed back */
+	PVR_ASSERT(psParaVz == psPVRSRVData->hPvzConnection);
+}
+
+/******************************************************************************
+ End of file (vz_vmm_pvz.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_vm.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_vm.c
new file mode 100644
index 000000000000..bc737f5acd81
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_vm.c
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File			vz_vmm_vm.c
+@Title          System virtualization VM support APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System virtualization VM support functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv.h"
+#include "pvrsrv_error.h"
+#include "vz_vm.h"
+#include "rgxfwutils.h"
+#include "rgxfwdbg.h"
+
+bool IsVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		return false;
+	}
+	else
+	{
+		return BIT_ISSET(psDevNode->ui32VmState, ui32DriverID);
+	}
+}
+
+PVRSRV_ERROR PvzOnVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVRSRV_ERROR       eError          = PVRSRV_ERROR_INVALID_PARAMS;
+#else
+	PVRSRV_ERROR       eError          = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+	else if (BIT_ISSET(psDevNode->ui32VmState, ui32DriverID))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: DriverID %u on Device %u is already enabled.",
+				 __func__, ui32DriverID, ui32DevID));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_CREATED)
+	{
+		/* Firmware not initialized yet, do it here */
+		eError = PVRSRVCommonDeviceInitialise(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: failed to initialize firmware (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+	}
+
+	eError = RGXFWHealthCheckCmd(psDevNode->pvDevice);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	BIT_SET(psDevNode->ui32VmState, ui32DriverID);
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* Everything is ready for the firmware to start interacting with this OS */
+	eError = RGXFWSetFwOsState(psDevNode->pvDevice, ui32DriverID, RGXFWIF_OS_ONLINE);
+#endif
+
+e0:
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PvzOnVmOffline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVRSRV_ERROR       eError          = PVRSRV_ERROR_INVALID_PARAMS;
+#else
+	PVRSRV_ERROR      eError          = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+	else if (!BIT_ISSET(psDevNode->ui32VmState, ui32DriverID))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: DriverID %u on Device %u is already disabled.",
+				 __func__, ui32DriverID, ui32DevID));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	psDevInfo = psDevNode->pvDevice;
+	if (psDevInfo == NULL)
+	{
+		eError = PVRSRV_ERROR_INVALID_DEVINFO;
+		goto e0;
+	}
+
+	eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+	if (eError == PVRSRV_OK)
+	{
+		BIT_UNSET(psDevNode->ui32VmState, ui32DriverID);
+	}
+
+e0:
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PvzVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+							 IMG_UINT32 ui32ParamValue,
+							 IMG_UINT32 ui32DevID)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+
+	psDevInfo = psDevNode->pvDevice;
+	if (psDevInfo == NULL)
+	{
+		eError = PVRSRV_ERROR_INVALID_DEVINFO;
+		goto e0;
+	}
+
+	switch (eVMMParamType)
+	{
+		case VMM_CONF_PRIO_DRV0:
+		case VMM_CONF_PRIO_DRV1:
+		case VMM_CONF_PRIO_DRV2:
+		case VMM_CONF_PRIO_DRV3:
+		case VMM_CONF_PRIO_DRV4:
+		case VMM_CONF_PRIO_DRV5:
+		case VMM_CONF_PRIO_DRV6:
+		case VMM_CONF_PRIO_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType;
+			IMG_UINT32 ui32Prio = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverPriorityKM(NULL, psDevNode, ui32DriverID, ui32Prio);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_HCS_DEADLINE:
+		{
+			IMG_UINT32 ui32HCSDeadline = ui32ParamValue;
+			eError = RGXFWSetHCSDeadline(psDevInfo, ui32HCSDeadline);
+			break;
+		}
+		case VMM_CONF_ISOLATION_GROUP_DRV0:
+		case VMM_CONF_ISOLATION_GROUP_DRV1:
+		case VMM_CONF_ISOLATION_GROUP_DRV2:
+		case VMM_CONF_ISOLATION_GROUP_DRV3:
+		case VMM_CONF_ISOLATION_GROUP_DRV4:
+		case VMM_CONF_ISOLATION_GROUP_DRV5:
+		case VMM_CONF_ISOLATION_GROUP_DRV6:
+		case VMM_CONF_ISOLATION_GROUP_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType - VMM_CONF_ISOLATION_GROUP_DRV0;
+			IMG_UINT32 ui32IsolationGroup = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverIsolationGroupKM(NULL, psDevNode, ui32DriverID, ui32IsolationGroup);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_TIME_SLICE_DRV0:
+		case VMM_CONF_TIME_SLICE_DRV1:
+		case VMM_CONF_TIME_SLICE_DRV2:
+		case VMM_CONF_TIME_SLICE_DRV3:
+		case VMM_CONF_TIME_SLICE_DRV4:
+		case VMM_CONF_TIME_SLICE_DRV5:
+		case VMM_CONF_TIME_SLICE_DRV6:
+		case VMM_CONF_TIME_SLICE_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType - VMM_CONF_TIME_SLICE_DRV0;
+			IMG_UINT32 ui32TSPercentage = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverTimeSliceKM(NULL, psDevNode, ui32DriverID, ui32TSPercentage);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_TIME_SLICE_INTERVAL:
+		{
+			IMG_UINT32 ui32TSIntervalMs = ui32ParamValue;
+
+			eError = PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(NULL, psDevNode, ui32TSIntervalMs);
+			break;
+		}
+		case VMM_CONF_VZ_CONNECTION_COOLDOWN_PERIOD:
+		{
+			IMG_UINT32 ui32VzConnectionCooldownPeriodInSec = ui32ParamValue;
+			eError = RGXFWSetVzConnectionCooldownPeriod(psDevInfo, ui32VzConnectionCooldownPeriodInSec);
+			break;
+		}
+		default:
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+e0:
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(eVMMParamType);
+	PVR_UNREFERENCED_PARAMETER(ui32ParamValue);
+	PVR_UNREFERENCED_PARAMETER(ui32DevID);
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+}
+
+/******************************************************************************
+ End of file (vz_vmm_vm.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.c
new file mode 100644
index 000000000000..1de4f08dd2b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.c
@@ -0,0 +1,166 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR device dependent bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements device dependent PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_bridge_init.h"
+#include "rgxdevice.h"
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+PVRSRV_ERROR InitRGXRAYBridge(void);
+void DeinitRGXRAYBridge(void);
+#endif
+
+/* Reference counts for device-conditional
+ * bridges. This ensures that bridges remain
+ * valid while there are still devices using
+ * them.
+ */
+static ATOMIC_T i32RGXCMPBridgeRefCt;
+static ATOMIC_T i32RGXTQ2BridgeRefCt;
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+static ATOMIC_T i32RGXRayBridgeRefCt;
+#endif
+static IMG_BOOL bAtomicsInitialised = IMG_FALSE;
+
+void RGXBridgeDriverInit(void)
+{
+	if (!bAtomicsInitialised)
+	{
+		bAtomicsInitialised = IMG_TRUE;
+		OSAtomicWrite(&i32RGXCMPBridgeRefCt, 0);
+		OSAtomicWrite(&i32RGXTQ2BridgeRefCt, 0);
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+		OSAtomicWrite(&i32RGXRayBridgeRefCt, 0);
+#endif
+	}
+}
+
+PVRSRV_ERROR RGXRegisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	if (!bAtomicsInitialised)
+	{
+		eError = PVRSRV_ERROR_NOT_INITIALISED;
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXBridgeRefCts");
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		if (OSAtomicIncrement(&i32RGXCMPBridgeRefCt) == 1)
+		{
+			eError = InitRGXCMPBridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXCMPBridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXCMPBridge");
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		if (OSAtomicIncrement(&i32RGXTQ2BridgeRefCt) == 1)
+		{
+			eError = InitRGXTQ2Bridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXTQ2BridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXTQ2Bridge");
+		}
+	}
+
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 0)
+	{
+		if (OSAtomicIncrement(&i32RGXRayBridgeRefCt) == 1)
+		{
+			eError = InitRGXRAYBridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXRayBridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXRAYBridge");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+void RGXUnregisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		if (OSAtomicDecrement(&i32RGXCMPBridgeRefCt) == 0)
+		{
+			DeinitRGXCMPBridge();
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		if (OSAtomicDecrement(&i32RGXTQ2BridgeRefCt) == 0)
+		{
+			DeinitRGXTQ2Bridge();
+		}
+	}
+
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 0)
+	{
+		if (OSAtomicDecrement(&i32RGXRayBridgeRefCt) == 0)
+		{
+			DeinitRGXRAYBridge();
+		}
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.h
new file mode 100644
index 000000000000..8622848b51b1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR device dependent bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements device dependent PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_BRIDGE_INIT_H)
+#define RGX_BRIDGE_INIT_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "device.h"
+#include "rgxdevice.h"
+
+void RGXBridgeDriverInit(void);
+PVRSRV_ERROR RGXRegisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo);
+void RGXUnregisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#endif /* RGX_BRIDGE_INIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_heaps_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_heaps_server.h
new file mode 100644
index 000000000000..a1dd31113fa3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_heaps_server.h
@@ -0,0 +1,86 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap (server) definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if !defined RGX_HEAPS_SERVER_H
+#define RGX_HEAPS_SERVER_H
+
+#include "img_types.h"
+#include "rgx_heaps.h"
+
+/*
+ *  Supported log2 page size values for RGX_GENERAL_NON_4K_HEAP_ID
+ */
+#define RGX_HEAP_PAGE_SHIFTS_DEF \
+	X(4KB, 12U) \
+	X(16KB, 14U) \
+	X(64KB, 16U) \
+	X(256KB, 18U) \
+	X(1MB, 20U) \
+	X(2MB, 21U)
+
+typedef enum RGX_HEAP_PAGE_SHIFTS_TAG
+{
+#define X(_name, _shift) RGX_HEAP_ ## _name ## _PAGE_SHIFT = _shift,
+	RGX_HEAP_PAGE_SHIFTS_DEF
+#undef X
+} RGX_HEAP_PAGE_SHIFTS;
+
+/* Base and size alignment 2MB */
+#define RGX_HEAP_BASE_SIZE_ALIGN 0x200000UL
+#define RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT 0x8000UL
+
+/*************************************************************************/ /*!
+@Function       RGXGetValidHeapPageSizeMask
+@Description    Returns a bitmask indicating all supported virtual heap page sizes.
+
+@Return         IMG_UINT32  A 32-bit mask with enabled bits indicating valid
+                            page sizes.
+*/ /**************************************************************************/
+static inline IMG_UINT32 RGXGetValidHeapPageSizeMask(void)
+{
+	/* Generates a bit mask with the values of RGX_HEAP_PAGE_SHIFTS_DEF.
+	 * 0 is required for the first shift to properly bitwise OR. */
+#define X(_name, _shift) | (1 << _shift)
+	return 0 RGX_HEAP_PAGE_SHIFTS_DEF;
+#undef X
+}
+
+#endif /* RGX_HEAPS_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.c
new file mode 100644
index 000000000000..902cbe0318d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.c
@@ -0,0 +1,292 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Breakpoint routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Breakpoint routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxbreakpoint.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxmem.h"
+#include "device.h"
+#include "sync_internal.h"
+#include "pdump_km.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR PVRSRVRGXSetBreakpointKM(CONNECTION_DATA    * psConnection,
+                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                      IMG_HANDLE           hMemCtxPrivData,
+                                      RGXFWIF_DM           eFWDataMaster,
+                                      IMG_UINT64           ui64TempSpillingAddr,
+                                      IMG_UINT32           ui32BPAddr,
+                                      IMG_UINT32           ui32HandlerAddr,
+                                      IMG_UINT32           ui32DataMaster)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet)
+	{
+		eError = PVRSRV_ERROR_BP_ALREADY_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPAddr = ui32BPAddr;
+	sBPCmd.uCmdData.sBPData.ui32HandlerAddr = ui32HandlerAddr;
+	sBPCmd.uCmdData.sBPData.ui32BPDM = ui32DataMaster;
+	sBPCmd.uCmdData.sBPData.ui64SpillAddr = ui64TempSpillingAddr;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_WRITE | RGXFWIF_BPDATA_FLAGS_ENABLE;
+	sBPCmd.uCmdData.sBPData.eDM = eFWDataMaster;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          eFWDataMaster,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+	psDevInfo->eBPDM = eFWDataMaster;
+	psDevInfo->bBPSet = IMG_TRUE;
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXClearBreakpointKM(CONNECTION_DATA    * psConnection,
+                                        PVRSRV_DEVICE_NODE * psDeviceNode,
+                                        IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPAddr = 0;
+	sBPCmd.uCmdData.sBPData.ui32HandlerAddr = 0;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_WRITE | RGXFWIF_BPDATA_FLAGS_CTL;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+	psDevInfo->bBPSet = IMG_FALSE;
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXEnableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                         PVRSRV_DEVICE_NODE * psDeviceNode,
+                                         IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet == IMG_FALSE)
+	{
+		eError = PVRSRV_ERROR_BP_NOT_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_CTL | RGXFWIF_BPDATA_FLAGS_ENABLE;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDisableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet == IMG_FALSE)
+	{
+		eError = PVRSRV_ERROR_BP_NOT_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_CTL;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXOverallocateBPRegistersKM(CONNECTION_DATA    * psConnection,
+                                                PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                IMG_UINT32           ui32TempRegs,
+                                                IMG_UINT32           ui32SharedRegs)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_REGS;
+	sBPCmd.uCmdData.sBPData.ui32TempRegs = ui32TempRegs;
+	sBPCmd.uCmdData.sBPData.ui32SharedRegs = ui32SharedRegs;
+	sBPCmd.uCmdData.sBPData.psFWMemContext.ui32Addr = 0U;
+	sBPCmd.uCmdData.sBPData.eDM = RGXFWIF_DM_GP;
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxbreakpoint.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.h
new file mode 100644
index 000000000000..2470de250914
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.h
@@ -0,0 +1,142 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX breakpoint functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX breakpoint functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXBREAKPOINT_H)
+#define RGXBREAKPOINT_H
+
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetBreakpointKM
+
+ @Description
+	Server-side implementation of RGXSetBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input eDataMaster - Data Master to schedule command for
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32BPAddr - Address of breakpoint
+ @Input ui32HandlerAddr - Address of breakpoint handler
+ @Input ui32BPCtl - Breakpoint controls
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetBreakpointKM(CONNECTION_DATA    * psConnection,
+                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                      IMG_HANDLE           hMemCtxPrivData,
+                                      RGXFWIF_DM           eFWDataMaster,
+                                      IMG_UINT64           ui64TempSpillingAddr,
+                                      IMG_UINT32           ui32BPAddr,
+                                      IMG_UINT32           ui32HandlerAddr,
+                                      IMG_UINT32           ui32DataMaster);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXClearBreakpointKM
+
+ @Description
+	Server-side implementation of RGXClearBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXClearBreakpointKM(CONNECTION_DATA    * psConnection,
+                                        PVRSRV_DEVICE_NODE * psDeviceNode,
+                                        IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXEnableBreakpointKM
+
+ @Description
+	Server-side implementation of RGXEnableBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXEnableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                         PVRSRV_DEVICE_NODE * psDeviceNode,
+                                         IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDisableBreakpointKM
+
+ @Description
+	Server-side implementation of RGXDisableBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDisableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXOverallocateBPRegistersKM
+
+ @Description
+	Server-side implementation of RGXOverallocateBPRegisters
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui32TempRegs - Number of temporary registers to overallocate
+ @Input ui32SharedRegs - Number of shared registers to overallocate
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXOverallocateBPRegistersKM(CONNECTION_DATA    * psConnection,
+                                                PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                IMG_UINT32           ui32TempRegs,
+                                                IMG_UINT32           ui32SharedRegs);
+#endif /* RGXBREAKPOINT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.c
new file mode 100644
index 000000000000..9f82ff1260bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.c
@@ -0,0 +1,960 @@
+/*************************************************************************/ /*!
+@File
+@Title          BVNC handling specific routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Functions used for BVNC related work
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxbvnc.h"
+#define RGXBVNC_C
+#include "rgx_bvnc_table_km.h"
+#undef RGXBVNC_C
+#include "os_apphint.h"
+#include "pvrsrv.h"
+#include "pdump_km.h"
+#include "rgx_compat_bvnc.h"
+#include "allocmem.h"
+
+#define RGX_FEATURE_TRUE_VALUE_TYPE_UINT16 (RGX_FEATURE_VALUE_TYPE_UINT16 >> RGX_FEATURE_TYPE_BIT_SHIFT)
+#define RGX_FEATURE_TRUE_VALUE_TYPE_UINT32 (RGX_FEATURE_VALUE_TYPE_UINT32 >> RGX_FEATURE_TYPE_BIT_SHIFT)
+#define RGXBVNC_BUFFER_SIZE (((PVRSRV_MAX_DEVICES)*(RGX_BVNC_STR_SIZE_MAX))+1)
+
+/* This function searches the given array for a given search value */
+static IMG_UINT64* _RGXSearchBVNCTable( IMG_UINT64 *pui64Array,
+								IMG_UINT uiEnd,
+								IMG_UINT64 ui64SearchValue,
+								IMG_UINT uiColCount)
+{
+	IMG_UINT uiStart = 0, index;
+	IMG_UINT64 value, *pui64Ptr = NULL;
+
+	while (uiStart < uiEnd)
+	{
+		index = (uiStart + uiEnd)/2;
+		pui64Ptr = pui64Array + (index * uiColCount);
+		value = *(pui64Ptr);
+
+		if (value == ui64SearchValue)
+		{
+			return pui64Ptr;
+		}
+
+		if (value > ui64SearchValue)
+		{
+			uiEnd = index;
+		}else
+		{
+			uiStart = index + 1;
+		}
+	}
+	return NULL;
+}
+#define RGX_SEARCH_BVNC_TABLE(t, b) (_RGXSearchBVNCTable((IMG_UINT64*)(t), \
+                                ARRAY_SIZE(t), (b), \
+                                sizeof((t)[0])/sizeof(IMG_UINT64)) )
+
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@brief		This function reads the (P)BVNC core_ID register and extracts
+			the BVNC configuration. Supports the old scheme and the newer
+			PBVNC scheme.
+@param		psDeviceNode - Device Node pointer
+@param		ui32CoreNum  - Core/bank number (0 for single core)
+@param		pB           - Address of branch value (output)
+@param		pV           - Address of version value (output)
+@param		pN           - Address of number of clusters/scalable shading units value (output)
+@param		pC           - Address of configuration value (output)
+@return		BVNC encoded in 64-bit value, 16-bits per field
+*/ /**************************************************************************/
+static
+IMG_UINT64 _RGXReadBVNCFromReg(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32CoreNum,
+							   IMG_UINT32 *pB, IMG_UINT32 *pV, IMG_UINT32 *pN, IMG_UINT32 *pC)
+{
+	IMG_UINT64 ui64BVNC;
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_CORE_ID__PBVNC)
+	/* Core ID reading code for Rogue */
+
+	/* Read the BVNC, in to new way first, if B not set, use old scheme */
+	ui64BVNC = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID__PBVNC + (ui32CoreNum << 16));
+
+	if (GET_B(ui64BVNC))
+	{
+		B = GET_PBVNC_B(ui64BVNC);
+		V = GET_PBVNC_V(ui64BVNC);
+		N = GET_PBVNC_N(ui64BVNC);
+		C = GET_PBVNC_C(ui64BVNC);
+	}
+	else
+	{
+		IMG_UINT64 ui32CoreID, ui32CoreRev;
+		ui32CoreRev = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_REVISION + (ui32CoreNum << 16));
+		ui32CoreID = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID + (ui32CoreNum << 16));
+		B = (ui32CoreRev & ~RGX_CR_CORE_REVISION_MAJOR_CLRMSK) >>
+												RGX_CR_CORE_REVISION_MAJOR_SHIFT;
+		V = (ui32CoreRev & ~RGX_CR_CORE_REVISION_MINOR_CLRMSK) >>
+												RGX_CR_CORE_REVISION_MINOR_SHIFT;
+		N = (ui32CoreID & ~RGX_CR_CORE_ID_CONFIG_N_CLRMSK) >>
+												RGX_CR_CORE_ID_CONFIG_N_SHIFT;
+		C = (ui32CoreID & ~RGX_CR_CORE_ID_CONFIG_C_CLRMSK) >>
+												RGX_CR_CORE_ID_CONFIG_C_SHIFT;
+		ui64BVNC = rgx_bvnc_pack(B, V, N, C);
+	}
+#else
+	/* Core ID reading code for Volcanic */
+
+	ui64BVNC = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID + (ui32CoreNum << 16));
+
+	B = (ui64BVNC & ~RGX_CR_CORE_ID_BRANCH_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_BRANCH_ID_SHIFT;
+	V = (ui64BVNC & ~RGX_CR_CORE_ID_VERSION_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_VERSION_ID_SHIFT;
+	N = (ui64BVNC & ~RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_CLRMSK) >>
+											RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_SHIFT;
+	C = (ui64BVNC & ~RGX_CR_CORE_ID_CONFIG_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_CONFIG_ID_SHIFT;
+#endif
+
+	*pB = B; *pV = V; *pN = N; *pC = C;
+	return ui64BVNC;
+}
+#endif
+
+#if defined(DEBUG) || defined(SUPPORT_PERFORMANCE_RUN)
+
+#define PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, szShortName, Feature)															\
+	if ( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] != RGX_FEATURE_VALUE_DISABLED )			\
+		{ PVR_LOG(("%s %d", szShortName, psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX])); }		\
+	else																\
+		{ PVR_LOG(("%s N/A", szShortName)); }
+
+static void _RGXBvncDumpParsedConfig(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT64 ui64Mask = 0, ui32IdOrNameIdx = 1;
+
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NC:       ", NUM_CLUSTERS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "CSF:      ", CDM_CONTROL_STREAM_FORMAT);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "FBCDCA:   ", FBCDC_ARCHITECTURE);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "META:     ", META);
+#if defined(RGX_FEATURE_META_COREMEM_BANKS_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MCMB:     ", META_COREMEM_BANKS);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MCMS:     ", META_COREMEM_SIZE);
+#if defined(RGX_FEATURE_META_DMA_CHANNEL_COUNT_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MDMACnt:  ", META_DMA_CHANNEL_COUNT);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NIIP:     ", NUM_ISP_IPP_PIPES);
+#if defined(RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NIPS:     ", NUM_ISP_PER_SPU);
+#endif
+#if defined(RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "PPS:      ", PBE_PER_SPU);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NSPU:     ", NUM_SPU);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "PBW:      ", PHYS_BUS_WIDTH);
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "STEArch:  ", SCALABLE_TE_ARCH);
+#endif
+#if defined(RGX_FEATURE_SCALABLE_VCE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SVCEA:    ", SCALABLE_VCE);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SLCBanks: ", SLC_BANKS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SLCCLS:   ", SLC_CACHE_LINE_SIZE_BITS);
+	PVR_LOG(("SLCSize:   %d",  psDevInfo->sDevFeatureCfg.ui32SLCSizeInBytes));
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "VASB:     ", VIRTUAL_ADDRESS_SPACE_BITS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NOSIDS:   ", NUM_OSIDS);
+
+#if defined(FEATURE_NO_VALUES_NAMES_MAX_IDX)
+	/* Dump the features with no values */
+	ui64Mask = psDevInfo->sDevFeatureCfg.ui64Features;
+	while (ui64Mask)
+	{
+		if (ui64Mask & 0x01)
+		{
+			if (ui32IdOrNameIdx <= FEATURE_NO_VALUES_NAMES_MAX_IDX)
+			{
+				PVR_LOG(("%s", gaszFeaturesNoValuesNames[ui32IdOrNameIdx - 1]));
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				         "Feature with Mask doesn't exist: 0x%016" IMG_UINT64_FMTSPECx,
+				         ((IMG_UINT64)1 << (ui32IdOrNameIdx - 1))));
+			}
+		}
+		ui64Mask >>= 1;
+		ui32IdOrNameIdx++;
+	}
+#endif
+
+#if defined(ERNSBRNS_IDS_MAX_IDX)
+	/* Dump the ERN and BRN flags for this core */
+	ui64Mask = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
+	ui32IdOrNameIdx = 1;
+
+	while (ui64Mask)
+	{
+		if (ui64Mask & 0x1)
+		{
+			if (ui32IdOrNameIdx <= ERNSBRNS_IDS_MAX_IDX)
+			{
+				PVR_LOG(("ERN/BRN : %d", gaui64ErnsBrnsIDs[ui32IdOrNameIdx - 1]));
+			}
+			else
+			{
+				PVR_LOG(("Unknown ErnBrn bit: 0x%0" IMG_UINT64_FMTSPECx, ((IMG_UINT64)1 << (ui32IdOrNameIdx - 1))));
+			}
+		}
+		ui64Mask >>= 1;
+		ui32IdOrNameIdx++;
+	}
+#endif
+
+#if !defined(ERNSBRNS_IDS_MAX_IDX) && !defined(FEATURE_NO_VALUES_NAMES_MAX_IDX)
+	PVR_UNREFERENCED_PARAMETER(ui64Mask);
+	PVR_UNREFERENCED_PARAMETER(ui32IdOrNameIdx);
+#endif
+
+}
+#endif
+
+static PVRSRV_ERROR _RGXBvncParseFeatureValues(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                               IMG_UINT64 *pui64Cfg)
+{
+	IMG_UINT32 ui32Index;
+
+	/* Read the feature values for the runtime BVNC */
+	for (ui32Index = 0; ui32Index < RGX_FEATURE_WITH_VALUES_MAX_IDX; ui32Index++)
+	{
+		IMG_UINT16 ui16BitPosition = aui16FeaturesWithValuesBitPositions[ui32Index];
+		IMG_UINT64 ui64PackedValues = pui64Cfg[2 + ui16BitPosition / 64];
+		IMG_UINT16 ui16ValueIndex = (ui64PackedValues & aui64FeaturesWithValuesBitMasks[ui32Index]) >> (ui16BitPosition % 64);
+
+		if (ui16ValueIndex >= gaFeaturesValuesMaxIndexes[ui32Index])
+		{
+			/* This case should never be reached */
+			psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] = RGX_FEATURE_VALUE_INVALID;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Feature with index (%d) decoded wrong value index (%d)", __func__, ui32Index, ui16ValueIndex));
+			PVR_ASSERT(ui16ValueIndex < gaFeaturesValuesMaxIndexes[ui32Index]);
+			return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+
+		switch (ui16BitPosition >> RGX_FEATURE_TYPE_BIT_SHIFT)
+		{
+			case RGX_FEATURE_TRUE_VALUE_TYPE_UINT16:
+			{
+				IMG_UINT16 *pui16FeatureValues = (IMG_UINT16*)gaFeaturesValues[ui32Index];
+				if (pui16FeatureValues[ui16ValueIndex] == (IMG_UINT16)RGX_FEATURE_VALUE_DISABLED)
+				{
+					psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+						RGX_FEATURE_VALUE_DISABLED;
+				}
+				else
+				{
+					psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+						pui16FeatureValues[ui16ValueIndex];
+				}
+				break;
+			}
+			case RGX_FEATURE_TRUE_VALUE_TYPE_UINT32:
+				psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+					((IMG_UINT32*)gaFeaturesValues[ui32Index])[ui16ValueIndex];
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Feature with index %d has invalid feature type",
+				         __func__,
+				         ui32Index));
+				return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+	}
+
+
+#if defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX)
+	/* Code path for Volcanic */
+
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = RGXFWIF_DM_CDM+1;
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_RAY+1);
+	}
+#if defined(SUPPORT_AGP)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM2+1);
+#if defined(SUPPORT_AGP4)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM4+1);
+#endif
+#endif
+
+	/* Get the max number of dusts in the core */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) == 1)
+		{
+			/* per SPU power island */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = MAX(1, (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) / 2));
+		}
+		else if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) >= 2)
+		{
+			/* per Cluster power island */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS);
+		}
+		else
+		{
+			/* All volcanic cores support power islanding */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Power island feature version not found!", __func__));
+			PVR_ASSERT(0);
+			return PVRSRV_ERROR_FEATURE_DISABLED;
+		}
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+			RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+		{
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RT_RAC_PER_SPU))
+			{
+				psDevInfo->sDevFeatureCfg.ui32MAXRACCount = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU);
+			}
+			else
+			{
+				psDevInfo->sDevFeatureCfg.ui32MAXRACCount = 1;
+			}
+		}
+	}
+	else
+	{
+		/* This case should never be reached as all cores have clusters */
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of clusters feature value missing!", __func__));
+		PVR_ASSERT(0);
+		return PVRSRV_ERROR_FEATURE_DISABLED;
+	}
+#else /* defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX) */
+	/* Code path for Rogue */
+
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = RGXFWIF_DM_CDM+1;
+#if defined(SUPPORT_AGP)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM2+1);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_META_IDX] = RGX_FEATURE_VALUE_DISABLED;
+	}
+
+	/* Get the max number of dusts in the core */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS))
+	{
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = MAX(1, (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) / 2));
+	}
+	else
+	{
+		/* This case should never be reached as all cores have clusters */
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of clusters feature value missing!", __func__));
+		PVR_ASSERT(0);
+		return PVRSRV_ERROR_FEATURE_DISABLED;
+	}
+#endif /* defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX) */
+
+	/* Transform the META coremem size info in bytes */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_META_COREMEM_SIZE_IDX] *= 1024;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RGXBvncAcquireAppHint(IMG_CHAR *pszBVNC, const IMG_UINT32 ui32RGXDevCount)
+{
+	const IMG_CHAR *pszAppHintDefault = PVRSRV_APPHINT_RGXBVNC;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32BVNCCount = 0;
+	IMG_BOOL bRet;
+	IMG_CHAR *pszBVNCAppHint;
+	IMG_CHAR *pszCurrentBVNC;
+	pszBVNCAppHint = (IMG_CHAR *)OSAllocMem(RGXBVNC_BUFFER_SIZE);
+	if (pszBVNCAppHint == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pszBVNCAppHint[0] = '\0';
+
+	pszCurrentBVNC = pszBVNCAppHint;
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bRet = (IMG_BOOL)OSGetAppHintSTRING(APPHINT_NO_DEVICE,
+						pvAppHintState,
+						RGXBVNC,
+						pszAppHintDefault,
+						pszBVNCAppHint,
+						RGXBVNC_BUFFER_SIZE
+	);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	if (!bRet || (pszBVNCAppHint[0] == '\0'))
+	{
+		OSFreeMem(pszBVNCAppHint);
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC module param list: %s",__func__, pszBVNCAppHint));
+
+	while (*pszCurrentBVNC != '\0')
+	{
+		IMG_CHAR *pszNext = pszCurrentBVNC;
+
+		if (ui32BVNCCount >= PVRSRV_MAX_DEVICES)
+		{
+			break;
+		}
+
+		while (1)
+		{
+			if (*pszNext == ',')
+			{
+				pszNext[0] = '\0';
+				pszNext++;
+				break;
+			} else if (*pszNext == '\0')
+			{
+				break;
+			}
+			pszNext++;
+		}
+
+		if (ui32BVNCCount == ui32RGXDevCount)
+		{
+			OSStringSafeCopy(pszBVNC, pszCurrentBVNC, RGX_BVNC_STR_SIZE_MAX);
+			OSFreeMem(pszBVNCAppHint);
+			return PVRSRV_OK;
+		}
+
+		ui32BVNCCount++;
+		pszCurrentBVNC = pszNext;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Given module parameters list is shorter than "
+	"number of actual devices", __func__));
+
+	/* If only one BVNC parameter is specified, the same is applied for all RGX
+	 * devices detected */
+	if (1 == ui32BVNCCount)
+	{
+		OSStringSafeCopy(pszBVNC, pszBVNCAppHint, RGX_BVNC_STR_SIZE_MAX);
+	}
+
+	OSFreeMem(pszBVNCAppHint);
+
+	return PVRSRV_OK;
+}
+
+/* Function that parses the BVNC List passed as module parameter */
+static PVRSRV_ERROR _RGXBvncParseList(IMG_UINT32 *pB,
+									  IMG_UINT32 *pV,
+									  IMG_UINT32 *pN,
+									  IMG_UINT32 *pC,
+									  const IMG_UINT32 ui32RGXDevCount)
+{
+	IMG_CHAR aszBVNCString[RGX_BVNC_STR_SIZE_MAX];
+	IMG_CHAR *pcTemp, *pcNext;
+	PVRSRV_ERROR eError;
+
+	aszBVNCString[0] = '\0';
+
+	/* 4 components of a BVNC string is B, V, N & C */
+#define RGX_BVNC_INFO_PARAMS (4)
+
+	eError = _RGXBvncAcquireAppHint(aszBVNCString, ui32RGXDevCount);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	if ('\0' == aszBVNCString[0])
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+
+	/* Parse the given RGX_BVNC string */
+	pcTemp = &aszBVNCString[0];
+	pcNext = strchr(pcTemp, '.');
+	if (pcNext == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+
+	*pcNext = '\0';
+	if (OSStringToUINT32(pcTemp, 0, pB) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	/* remove any 'p' from the V string, as this will
+	 * cause the call to OSStringToUINT32 to fail
+	 */
+	pcNext = strchr(pcTemp, 'p');
+	if (pcNext)
+	{
+		/* found one- - changing to '\0' */
+		*pcNext = '\0';
+		/* Move to next '.' */
+		pcNext++;
+	}
+	else
+	{
+		/* none found, so find next '.' and change to '\0' */
+		pcNext = strchr(pcTemp, '.');
+		if (pcNext == NULL)
+		{
+			return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+		*pcNext = '\0';
+	}
+	if (OSStringToUINT32(pcTemp, 0, pV) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	pcNext = strchr(pcTemp, '.');
+	if (pcNext == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	*pcNext = '\0';
+	if (OSStringToUINT32(pcTemp, 0, pN) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	if (OSStringToUINT32(pcTemp, 0, pC) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	PVR_LOG(("BVNC module parameter honoured: %d.%d.%d.%d", *pB, *pV, *pN, *pC));
+
+	return PVRSRV_OK;
+}
+
+#if !defined(NO_HARDWARE)
+/*
+ * This function obtains the SLCSize from the physical device for GPUs which provide
+ * this information. If the GPU does not provide support we return a value of 0 which will
+ * result in the BVNC supplied definition being used to provide the SLCSize.
+ * Must only be called from driver-live with hardware powered-on.
+ */
+static IMG_UINT32 _RGXBvncReadSLCSize(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT64 ui64SLCSize = 0ULL;
+
+#if defined(RGX_CR_SLC_SIZE_IN_KB)
+	/* Rogue hardware */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_SIZE_CONFIGURABLE))
+	{
+		ui64SLCSize = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SLC_SIZE_IN_KB);
+		if (ui64SLCSize == 0ULL)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Unexpected 0 SLC size. Using default", __func__));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: RGX_CR_SIZE_IN_KB = %u", __func__,
+			        (IMG_UINT32) ui64SLCSize));
+		}
+	}
+#else
+	/* Volcanic hardware */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_SIZE_ADJUSTMENT))
+	{
+		ui64SLCSize = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SLC_STATUS2);
+		ui64SLCSize &= ~RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_CLRMSK;
+		ui64SLCSize >>= RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_SHIFT;
+
+		if (ui64SLCSize == 0ULL)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Unexpected 0 SLC size. Using default", __func__));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: SLC_SIZE_IN_KILOBYTES = %u", __func__,
+			        (IMG_UINT32) ui64SLCSize));
+		}
+	}
+#endif
+
+	return (IMG_UINT32)ui64SLCSize * 1024U;
+}
+#endif /* !defined(NO_HARDWARE) */
+
+/* This function detects the Rogue variant and configures the essential
+ * config info associated with such a device.
+ * The config info includes features, errata, etc
+ */
+PVRSRV_ERROR RGXBvncInitialiseConfiguration(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	static IMG_UINT32 ui32RGXDevCnt = 0;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT64 ui64BVNC=0;
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+	IMG_UINT64 *pui64Cfg = NULL;
+	IMG_UINT32 ui32Cores = 1U;
+	IMG_UINT32 ui32SLCSize = 0;
+
+	/* Check for load time RGX BVNC parameter */
+	eError = _RGXBvncParseList(&B,&V,&N,&C, ui32RGXDevCnt);
+	if (PVRSRV_OK == eError)
+	{
+		PVR_LOG(("Read BVNC " RGX_BVNC_STR_FMTSPEC
+				" from driver load parameter", B, V, N, C));
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+		PVR_LOG_IF_FALSE((pui64Cfg != NULL), "Driver parameter BVNC configuration not found!");
+	}
+
+	{
+		void *pvAppHintState = NULL;
+		const IMG_BOOL bAppHintDefault = PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC;
+
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							IgnoreHWReportedBVNC,
+							&bAppHintDefault,
+							&psDevInfo->bIgnoreHWReportedBVNC);
+		OSFreeAppHintState(pvAppHintState);
+	}
+
+#if !defined(NO_HARDWARE)
+
+	/* Try to detect the RGX BVNC from the HW device */
+	if ((NULL == pui64Cfg) && !psDevInfo->bIgnoreHWReportedBVNC)
+	{
+		IMG_BOOL bPowerDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+		}
+
+		/* Read the BVNC from HW */
+		_RGXReadBVNCFromReg(psDeviceNode, 0 /*core0*/, &B, &V, &N, &C);
+
+		PVR_LOG(("Read BVNC " RGX_BVNC_STR_FMTSPEC
+				" from HW device registers", B, V, N, C));
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Read the number of cores in the system for newer BVNC (Branch ID > 20) */
+			if (B > 20)
+			{
+				ui32Cores = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM);
+			}
+		}
+
+		/* Obtain the SLC size from the device */
+		ui32SLCSize = _RGXBvncReadSLCSize(psDeviceNode);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: SLC Size reported as %u", __func__, ui32SLCSize));
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		if (ui64BVNC != 0)
+		{
+			pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+			PVR_LOG_IF_FALSE((pui64Cfg != NULL), "HW device BVNC configuration not found!");
+		}
+		else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/*
+			 * On host OS we should not get here as CORE_ID should not be zero, so flag an error.
+			 * On older cores, guest OS only has CORE_ID if defined(RGX_FEATURE_COREID_PER_OS)
+			 */
+			PVR_LOG_ERROR(PVRSRV_ERROR_DEVICE_REGISTER_FAILED, "CORE_ID register returns zero. Unknown BVNC");
+		}
+	}
+#endif
+
+#if defined(RGX_BVNC_KM_B) && defined(RGX_BVNC_KM_N) && defined(RGX_BVNC_KM_C)
+	if (NULL == pui64Cfg)
+	{
+		IMG_CHAR acVStr[5] = RGX_BVNC_KM_V_ST;
+
+		/* We reach here if the HW is not present,
+		 * or we are running in a guest OS with no COREID_PER_OS feature,
+		 * or HW is unstable during register read giving invalid values,
+		 * or runtime detection has been disabled - fall back to compile time BVNC
+		 */
+		B = RGX_BVNC_KM_B;
+		N = RGX_BVNC_KM_N;
+		C = RGX_BVNC_KM_C;
+
+		/* Clear any 'p' that may have been in RGX_BVNC_KM_V_ST,
+		 * as OSStringToUINT32() will otherwise return an error.
+		 */
+		if (acVStr[strlen(acVStr)-1] == 'p')
+		{
+			acVStr[strlen(acVStr)-1] = '\0';
+		}
+
+		if (OSStringToUINT32(&acVStr[0], 0, &V) != PVRSRV_OK)
+		{
+			V = 0;
+		}
+		PVR_LOG(("Reverting to compile time BVNC %s", RGX_BVNC_KM));
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+		PVR_LOG_IF_FALSE((pui64Cfg != NULL), "Compile time BVNC configuration not found!");
+	}
+#endif /* defined(RGX_BVNC) */
+
+	/* Have we failed to identify the BVNC to use? */
+	if (NULL == pui64Cfg)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: BVNC Detection and feature lookup failed. "
+		    "Unsupported BVNC: 0x%016" IMG_UINT64_FMTSPECx, __func__, ui64BVNC));
+		return PVRSRV_ERROR_BVNC_UNSUPPORTED;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC Feature found config: 0x%016"
+	    IMG_UINT64_FMTSPECx " 0x%016" IMG_UINT64_FMTSPECx " 0x%016"
+	    IMG_UINT64_FMTSPECx " 0x%016" IMG_UINT64_FMTSPECx "\n", __func__,
+	    pui64Cfg[0], pui64Cfg[1], pui64Cfg[2], pui64Cfg[3]));
+
+	/* Parsing feature config depends on available features on the core
+	 * hence this parsing should always follow the above feature assignment */
+	psDevInfo->sDevFeatureCfg.ui64Features = pui64Cfg[1];
+	eError = _RGXBvncParseFeatureValues(psDevInfo, pui64Cfg);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Add 'V' to the packed BVNC value to get the BVNC ERN and BRN config. */
+	ui64BVNC = BVNC_PACK(B,V,N,C);
+	pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaErnsBrns, ui64BVNC);
+	if (NULL == pui64Cfg)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: BVNC ERN/BRN lookup failed. "
+		    "Unsupported BVNC: 0x%016" IMG_UINT64_FMTSPECx, __func__, ui64BVNC));
+		psDevInfo->sDevFeatureCfg.ui64ErnsBrns = 0;
+		return PVRSRV_ERROR_BVNC_UNSUPPORTED;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC ERN/BRN Cfg: 0x%016" IMG_UINT64_FMTSPECx
+	    " 0x%016" IMG_UINT64_FMTSPECx, __func__, *pui64Cfg, pui64Cfg[1]));
+	psDevInfo->sDevFeatureCfg.ui64ErnsBrns = pui64Cfg[1];
+
+	psDevInfo->sDevFeatureCfg.ui32B = B;
+	psDevInfo->sDevFeatureCfg.ui32V = V;
+	psDevInfo->sDevFeatureCfg.ui32N = N;
+	psDevInfo->sDevFeatureCfg.ui32C = C;
+
+
+	/*
+	 * Store the SLCSize in the device info field. If 0 it means the device uses the BVNC
+	 * values so grab them here as we've already populated the internal structures.
+	 */
+	if (ui32SLCSize == 0U)
+	{
+		ui32SLCSize = RGX_GET_FEATURE_VALUE(psDevInfo, SLC_SIZE_IN_KILOBYTES) * 1024U;
+
+		/* Verify that we have a valid value returned from the BVNC */
+		PVR_ASSERT(ui32SLCSize != 0U);
+	}
+	psDevInfo->sDevFeatureCfg.ui32SLCSizeInBytes = ui32SLCSize;
+
+	/* Message to confirm configuration look up was a success */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT) &&
+		!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(NO_HARDWARE)
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32Cores);
+			PVR_LOG(("RGX Device registered with BVNC " RGX_BVNC_STR_FMTSPEC,
+					B, V, N, C));
+		}
+#else
+		{
+			PVR_LOG(("RGX Device registered BVNC " RGX_BVNC_STR_FMTSPEC
+					" with %u %s in the system", B ,V ,N ,C, ui32Cores ,
+					((ui32Cores == 1U)?"core":"cores")));
+		}
+#endif
+	}
+	else
+	{
+		PVR_LOG(("RGX Device registered with BVNC " RGX_BVNC_STR_FMTSPEC,
+					B, V, N, C));
+	}
+
+	ui32RGXDevCnt++;
+
+#if defined(DEBUG) || defined(SUPPORT_PERFORMANCE_RUN)
+	_RGXBvncDumpParsedConfig(psDeviceNode);
+#endif
+	return PVRSRV_OK;
+}
+
+/*
+ * This function checks if a particular feature is available on the given rgx device */
+IMG_BOOL RGXBvncCheckFeatureSupported(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64FeatureMask)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->sDevFeatureCfg.ui64Features & ui64FeatureMask)
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+/*
+ * This function returns the value of a feature on the given rgx device */
+IMG_INT32 RGXBvncGetSupportedFeatureValue(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_FEATURE_WITH_VALUE_INDEX eFeatureIndex)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	if (eFeatureIndex >= RGX_FEATURE_WITH_VALUES_MAX_IDX)
+	{
+		return -1;
+	}
+
+	if (psDevInfo->sDevFeatureCfg.ui32FeaturesValues[eFeatureIndex] == RGX_FEATURE_VALUE_DISABLED)
+	{
+		return -1;
+	}
+
+	return psDevInfo->sDevFeatureCfg.ui32FeaturesValues[eFeatureIndex];
+}
+
+/**************************************************************************/ /*!
+@Function       RGXVerifyBVNC
+@Description    Checks that the device's BVNC registers have the correct values.
+@Input          psDeviceNode		Device node
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+#define NUM_RGX_CORE_IDS    8
+PVRSRV_ERROR RGXVerifyBVNC(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT64 ui64MatchBVNC;
+#if !defined(NO_HARDWARE)
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+#endif
+	IMG_UINT32 i;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	/* The device info */
+	psDevInfo = psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "PDUMP VERIFY CORE_ID registers for all OSIDs\n");
+
+	/* construct the value to match against */
+	if ((ui64GivenBVNC | ui64CoreIdMask) == 0) /* both zero means use configured DDK value */
+	{
+		ui64MatchBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+									psDevInfo->sDevFeatureCfg.ui32V,
+									psDevInfo->sDevFeatureCfg.ui32N,
+									psDevInfo->sDevFeatureCfg.ui32C);
+	}
+	else
+	{
+		/* use the value in CORE_ID for any zero elements in the BVNC */
+#if !defined(NO_HARDWARE)
+		IMG_UINT64 ui64BVNC = _RGXReadBVNCFromReg(psDeviceNode, 0, &B, &V, &N, &C);
+		ui64MatchBVNC = (ui64GivenBVNC & ~ui64CoreIdMask) | (ui64BVNC & ui64CoreIdMask);
+#else
+		ui64MatchBVNC = 0;
+#endif
+	}
+	PVR_LOG(("matchBVNC %d.%d.%d.%d",
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_B) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_V) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_N) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_C) & 0xffff)));
+
+	/* read in all the CORE_ID registers */
+	for (i = 0; i < NUM_RGX_CORE_IDS; ++i)
+	{
+#if !defined(NO_HARDWARE)
+		IMG_UINT64 ui64BVNC = _RGXReadBVNCFromReg(psDeviceNode, i, &B, &V, &N, &C);
+		PVR_LOG(("CORE_ID%d returned %d.%d.%d.%d", i, B, V, N, C));
+
+		if (ui64BVNC != ui64MatchBVNC)
+		{
+			eError = PVRSRV_ERROR_BVNC_MISMATCH;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid CORE_ID%d %d.%d.%d.%d, Expected %d.%d.%d.%d", __func__, i,
+					B, V, N, C,
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_B) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_V) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_N) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_C) & 0xffff)));
+			break;
+		}
+#endif
+
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.h
new file mode 100644
index 000000000000..64c418bc4edb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          BVNC handling specific header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the BVNC related work
+                (see hwdefs/km/rgx_bvnc_table_km.h and
+                hwdefs/km/rgx_bvnc_defs_km.h
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXBVNC_H)
+#define RGXBVNC_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "rgxdevice.h"
+
+/*************************************************************************/ /*!
+@brief		This function detects the Rogue variant and configures the
+			essential config info associated with such a device.
+			The config info includes features, errata, etc
+@param		psDeviceNode - Device Node pointer
+@return		PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXBvncInitialiseConfiguration(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@brief		This function checks if a particular feature is available on
+			the given rgx device
+@param		psDeviceNode - Device Node pointer
+@param		ui64FeatureMask - feature to be checked
+@return		true if feature is supported, false otherwise
+*/ /**************************************************************************/
+IMG_BOOL RGXBvncCheckFeatureSupported(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64FeatureMask);
+
+/*************************************************************************/ /*!
+@brief		This function returns the value of a feature on the given
+			rgx device
+@param		psDeviceNode - Device Node pointer
+@param		ui64FeatureMask - feature for which to return the value
+@return		the value for the specified feature
+*/ /**************************************************************************/
+IMG_INT32 RGXBvncGetSupportedFeatureValue(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_FEATURE_WITH_VALUE_INDEX eFeatureIndex);
+
+/*************************************************************************/ /*!
+@brief		This function validates that the BVNC values in CORE_ID regs are
+			consistent and correct.
+@param		psDeviceNode - Device Node pointer
+@param		GivenBVNC - BVNC to be verified against as supplied by caller
+@param		CoreIdMask - mask of components to pull from CORE_ID register
+@return		success or fail
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXVerifyBVNC(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask);
+
+#endif /* RGXBVNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.c
new file mode 100644
index 000000000000..8c45398a97bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.c
@@ -0,0 +1,2863 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX CCB routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX CCB routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgx_memallocflags.h"
+#include "devicemem_pdump.h"
+#include "dllist.h"
+#if defined(__linux__)
+#include "trace_events.h"
+#endif
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "rgxutils.h"
+#include "info_page.h"
+#include "rgxtimerquery.h"
+
+#if defined(PVRSRV_FORCE_FLUSH_CCCB_ON_KICK)
+#include "cache_km.h"
+#endif
+
+/*
+ *  Uncomment PVRSRV_ENABLE_CCCB_UTILISATION_INFO define for verbose
+ *  info and statistics regarding CCB usage.
+ */
+//#define PVRSRV_ENABLE_CCCB_UTILISATION_INFO
+
+/* Default threshold (as a percentage) for the PVRSRV_ENABLE_CCCB_UTILISATION_INFO feature. */
+#define PVRSRV_ENABLE_CCCB_UTILISATION_INFO_THRESHOLD  (90)
+
+/*
+ * Defines the number of fence updates to record so that future fences in the
+ * CCB. Can be checked to see if they are already known to be satisfied.
+ */
+#define RGX_CCCB_FENCE_UPDATE_LIST_SIZE  (32)
+
+#define RGX_UFO_PTR_ADDR(ufoptr) \
+	(((ufoptr)->puiAddrUFO.ui32Addr) & 0xFFFFFFFC)
+
+#define GET_CCB_SPACE(WOff, ROff, CCBSize) \
+	((((ROff) - (WOff)) + ((CCBSize) - 1)) & ((CCBSize) - 1))
+
+#define UPDATE_CCB_OFFSET(Off, PacketSize, CCBSize) \
+	(Off) = (((Off) + (PacketSize)) & ((CCBSize) - 1))
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_THRESHOLD 0x1
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED 0x2
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_FULL_CCB 0x4
+
+typedef struct _RGX_CLIENT_CCB_UTILISATION_
+{
+	/* the threshold in bytes.
+	 * when the CCB utilisation hits the threshold then we will print
+	 * a warning message.
+	 */
+	IMG_UINT32 ui32ThresholdBytes;
+	/* Maximum cCCB usage at some point in time */
+	IMG_UINT32 ui32HighWaterMark;
+	/* keep track of the warnings already printed.
+	 * bit mask of PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_xyz
+	 */
+	IMG_UINT32 ui32Warnings;
+	/* Keep track how many times CCB was full.
+	 * Counters are reset after every grow.
+	 */
+	IMG_UINT32 ui32CCBFull;
+	IMG_UINT32 ui32CCBAcquired;
+} RGX_CLIENT_CCB_UTILISATION;
+
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+struct _RGX_CLIENT_CCB_ {
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;				/*!< CPU mapping of the CCB control structure used by the fw */
+	void						*pvClientCCB;					/*!< CPU mapping of the CCB */
+	DEVMEM_MEMDESC				*psClientCCBMemDesc;			/*!< MemDesc for the CCB */
+	DEVMEM_MEMDESC				*psClientCCBCtrlMemDesc;		/*!< MemDesc for the CCB control */
+	IMG_UINT32					ui32HostWriteOffset;			/*!< CCB write offset from the driver side */
+	IMG_UINT32					ui32LastPDumpWriteOffset;		/*!< CCB write offset from the last time we submitted a command in capture range */
+	IMG_UINT32					ui32LastROff;					/*!< Last CCB Read offset to help detect any CCB wedge */
+	IMG_UINT32					ui32LastWOff;					/*!< Last CCB Write offset to help detect any CCB wedge */
+	IMG_UINT32					ui32ByteCount;					/*!< Count of the number of bytes written to CCCB */
+	IMG_UINT32					ui32LastByteCount;				/*!< Last value of ui32ByteCount to help detect any CCB wedge */
+	IMG_UINT32					ui32Size;						/*!< Size of the CCB */
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	POS_LOCK					hCCBGrowLock;					/*!< Prevents CCB Grow while DumpCCB() is called and vice versa */
+	IMG_UINT32					ui32VirtualAllocSize;			/*!< Virtual size of the CCB */
+	IMG_UINT32					ui32ChunkSize;					/*!< CCB Sparse allocation chunk size */
+	IMG_PUINT32					pui32MappingTable;				/*!< Mapping table for sparse allocation of the CCB */
+#endif
+	DLLIST_NODE					sNode;							/*!< Node used to store this CCB on the per connection list */
+	PDUMP_CONNECTION_DATA		*psPDumpConnectionData;			/*!< Pointer to the per connection data in which we reside */
+	void						*hTransition;					/*!< Handle for Transition callback */
+	IMG_CHAR					szName[MAX_CLIENT_CCB_NAME];	/*!< Name of this client CCB */
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;			/*!< Parent server common context that this CCB belongs to */
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	RGX_CCB_REQUESTOR_TYPE		eRGXCCBRequestor;
+	RGX_CLIENT_CCB_UTILISATION	sUtilisation;					/*!< CCB utilisation data */
+#endif
+#if defined(DEBUG)
+	IMG_UINT32					ui32UpdateEntries;				/*!< Number of Fence Updates in asFenceUpdateList */
+	RGXFWIF_UFO					asFenceUpdateList[RGX_CCCB_FENCE_UPDATE_LIST_SIZE]; /*!< List of recent updates written in this CCB */
+#endif
+	IMG_UINT32					ui32CCBFlags;                   /*!< Bitmask for various flags relating to CCB. Bit defines in rgxccb.h */
+};
+
+/* Forms a table, with array of strings for each requestor type (listed in RGX_CCB_REQUESTORS X macro), to be used for
+   DevMemAllocation comments and PDump comments. Each tuple in the table consists of 3 strings:
+	{ "FwClientCCB:" <requestor_name>, "FwClientCCBControl:" <requestor_name>, <requestor_name> },
+   The first string being used as comment when allocating ClientCCB for the given requestor, the second for CCBControl
+   structure, and the 3rd one for use in PDUMP comments. The number of tuples in the table must adhere to the following
+   build assert. */
+const IMG_CHAR *const aszCCBRequestors[][3] =
+{
+#define REQUESTOR_STRING(prefix,req) #prefix ":" #req
+#define FORM_REQUESTOR_TUPLE(req) { REQUESTOR_STRING(FwClientCCB,req), REQUESTOR_STRING(FwClientCCBControl,req), #req },
+	RGX_CCB_REQUESTORS(FORM_REQUESTOR_TUPLE)
+#undef FORM_REQUESTOR_TUPLE
+};
+
+PVRSRV_ERROR RGXCCBPDumpDrainCCB(RGX_CLIENT_CCB *psClientCCB,
+						IMG_UINT32 ui32PDumpFlags)
+{
+
+	IMG_UINT32 ui32PollOffset;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+#endif
+
+	/* Command to a finished CCB stream and FW is drained to empty
+	 * out remaining commands until R==W.
+	 */
+	ui32PollOffset = psClientCCB->ui32LastPDumpWriteOffset;
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+						  ui32PDumpFlags,
+						  "cCCB(%s@%p): Draining CCB rgxfw_roff == woff (%d)",
+						  psClientCCB->szName,
+						  psClientCCB,
+						  ui32PollOffset);
+
+	return DevmemPDumpDevmemPol32(psClientCCB->psClientCCBCtrlMemDesc,
+									offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+									ui32PollOffset,
+									0xffffffff,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXCCBPDumpSyncCCB
+
+ PURPOSE	: Synchronise Client CCBs from both live and playback contexts.
+               Waits for live-FW to empty live-CCB.
+               Waits for sim-FW to empty sim-CCB by adding POL
+
+ PARAMETERS	: psClientCCB		- The client CCB
+              ui32PDumpFlags    - PDump flags
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXCCBPDumpSyncCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Wait for the live FW to catch up/empty CCB. This is done by returning
+	 * retry which will get pushed back out to Services client where it
+	 * waits on the event object and then resubmits the command.
+	 */
+	if (psClientCCB->psClientCCBCtrl->ui32ReadOffset != psClientCCB->ui32HostWriteOffset)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Wait for the sim FW to catch up/empty sim CCB.
+	 * We drain whenever capture range is entered, even if no commands
+	 * have been issued on this CCB when out of capture range. We have to
+	 * wait for commands that might have been issued in the last capture
+	 * range to finish so the connection's sync block snapshot dumped after
+	 * all the PDumpTransition callbacks have been execute doesn't clobber
+	 * syncs which the sim FW is currently working on.
+	 *
+	 * Although this is sub-optimal for play-back - while out of capture
+	 * range for every continuous operation we synchronise the sim
+	 * play-back processing the script and the sim FW, there is no easy
+	 * solution. Not all modules that work with syncs register a
+	 * PDumpTransition callback and thus we have no way of knowing if we
+	 * can skip this sim CCB drain and sync block dump or not.
+	 */
+
+	eError = RGXCCBPDumpDrainCCB(psClientCCB, ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "RGXCCBPDumpDrainCCB");
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Live CCB and simulation CCB now empty, FW idle on CCB in both
+	 * contexts.
+	 */
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXCCBPDumpFastForwardCCB
+
+ PURPOSE	: Fast-forward sim-CCB and live-CCB offsets to live app-thread
+              values.
+               This helps to skip any commands submitted when out of capture
+               range and start with first command in capture range in both
+               live and playback contexts. In case of Block mode, this helps
+               to playback any intermediate PDump block directly after first
+               block.
+
+
+ PARAMETERS	: psClientCCB		- The client CCB
+			  ui32PDumpFlags    - PDump flags
+
+ RETURNS	: void
+******************************************************************************/
+static void RGXCCBPDumpFastForwardCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32PDumpFlags)
+{
+	volatile RGXFWIF_CCCB_CTL *psCCBCtl = psClientCCB->psClientCCBCtrl;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+#endif
+
+	/* Make sure that we have synced live-FW and live-App threads */
+	PVR_ASSERT(psCCBCtl->ui32ReadOffset == psClientCCB->ui32HostWriteOffset);
+
+	psCCBCtl->ui32ReadOffset = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32DepOffset = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32WriteOffset = psClientCCB->ui32HostWriteOffset;
+
+	psCCBCtl->ui32ReadOffset2 = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32ReadOffset3 = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32ReadOffset4 = psClientCCB->ui32HostWriteOffset;
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+			ui32PDumpFlags,
+			"cCCB(%s@%p): Fast-forward from %d to %d",
+			psClientCCB->szName,
+			psClientCCB,
+			psClientCCB->ui32LastPDumpWriteOffset,
+			psClientCCB->ui32HostWriteOffset);
+
+	DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+			0,
+			sizeof(RGXFWIF_CCCB_CTL),
+			ui32PDumpFlags);
+
+	/* Although we've entered capture range for this process connection
+	 * we might not do any work on this CCB so update the
+	 * ui32LastPDumpWriteOffset to reflect where we got to for next
+	 * time so we start the drain from where we got to last time.
+	 */
+	psClientCCB->ui32LastPDumpWriteOffset = psClientCCB->ui32HostWriteOffset;
+
+}
+
+static PVRSRV_ERROR _RGXCCBPDumpTransition(void *pvData, void *pvDevice, PDUMP_TRANSITION_EVENT eEvent, IMG_UINT32 ui32PDumpFlags)
+{
+	RGX_CLIENT_CCB *psClientCCB = (RGX_CLIENT_CCB *) pvData;
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) pvDevice;
+#else
+	PVR_UNREFERENCED_PARAMETER(pvDevice);
+#endif
+
+	/* Block mode:
+	 * Here is block structure at transition (ui32BlockLength=N frames):
+	 *
+	 * ...
+	 * ...
+	 * PDUMP_BLOCK_START_0x0000000x{
+	 *   <Fast-forward sim-CCCB>
+	 *   <Re-dump SyncBlocks>
+	 *   ...
+	 *   ...
+	 *   ... (N frames data)
+	 *   ...
+	 *   ...
+	 *   <(1) Drain sim-KCCB>                     ''|
+	 *   <(2) Sync live and sim CCCB>               |
+	 * }PDUMP_BLOCK_END_0x0000000x                  | <- BlockTransition Steps
+	 *   <(3) Split MAIN and BLOCK stream script>   |
+	 * PDUMP_BLOCK_START_0x0000000y{                |
+	 *   <(4) Fast-forward sim-CCCB>                |
+	 *   <(5) Re-dump SyncBlocks>                 ,,|
+	 *   ...
+	 *   ...
+	 *   ... (N frames data)
+	 *   ...
+	 *   ...
+	 *   <Drain sim-KCCB>
+	 *   <Sync live and sim CCCB>
+	 * }PDUMP_BLOCK_END_0x0000000y
+	 * ...
+	 * ...
+	 *
+	 * Steps (3) and (5) are done in pdump_server.c
+	 * */
+	switch (eEvent)
+	{
+		case PDUMP_TRANSITION_EVENT_RANGE_ENTERED:
+			{
+				/* We're about to transition into capture range and we've submitted
+				 * new commands since the last time we entered capture range so drain
+				 * the live CCB and simulation (sim) CCB as required, i.e. leave CCB
+				 * idle in both live and sim contexts.
+				 * This requires the host driver to ensure the live FW & the sim FW
+				 * have both emptied out the remaining commands until R==W (CCB empty).
+				 */
+
+				eError = RGXCCBPDumpSyncCCB(psClientCCB, ui32PDumpFlags);
+				PVR_RETURN_IF_ERROR(eError);
+
+				if (psClientCCB->ui32LastPDumpWriteOffset != psClientCCB->ui32HostWriteOffset)
+				{
+					/* If new commands have been written when out of capture range in
+					 * the live CCB then we need to fast forward the sim CCBCtl
+					 * offsets past uncaptured commands. This is done by PDUMPing
+					 * the CCBCtl memory to align sim values with the live CCBCtl
+					 * values. Both live and sim FWs can start with the 1st command
+					 * which is in the new capture range.
+					 */
+					RGXCCBPDumpFastForwardCCB(psClientCCB, ui32PDumpFlags);
+				}
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_RANGE_APPEND:
+			{
+				/* Capture range already entered. Skip dump sync and fast forward.
+				 */
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_RANGE_EXITED:
+			{
+				/* Nothing to do */
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_BLOCK_FINISHED:
+			{
+				/* (1) Drain KCCB from current block before starting new:
+				 *
+				 * At playback, this will ensure that sim-FW drains all commands in KCCB
+				 * belongs to current block before 'jumping' to any future commands (from
+				 * next block). This will synchronise script-thread and sim-FW thread KCCBs
+				 * at end of each pdump block.
+				 *
+				 * This will additionally force redump of KCCBCtl structure at start of next/new block.
+				 * */
+
+#if defined(PDUMP)
+				eError = RGXPdumpDrainKCCB(psDevInfo, psDevInfo->psKernelCCBCtl->ui32WriteOffset);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXPdumpDrainKCCB");
+#endif
+
+				/* (2) Synchronise Client CCBs from live and playback contexts before starting new block:
+				 *
+				 * This operation will,
+				 * a. Force synchronisation between app-thread and live-FW thread (i.e. Wait
+				 *    for live-FW to empty live Client CCB).
+				 *
+				 * b. Next, it will dump poll command to drain Client CCB at end of every
+				 *    pdump block. At playback time this will synchronise sim-FW and
+				 *    script-thread Client CCBs at end of each block.
+				 *
+				 * This is to ensure that all commands in CCB from current block are processed
+				 * before moving on to future commands.
+				 * */
+
+				eError = RGXCCBPDumpSyncCCB(psClientCCB, ui32PDumpFlags);
+				PVR_RETURN_IF_ERROR(eError);
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_BLOCK_STARTED:
+			{
+				/* (4) Fast-forward CCB write offsets to current live values:
+				 *
+				 * We have already synchronised live-FW and app-thread above at end of each
+				 * block (in Step 2a above), now fast-forward Client CCBCtl write offsets to that of
+				 * current app-thread values at start of every block. This will allow us to
+				 * skip any intermediate pdump blocks and start with last (or any next) block
+				 * immediately after first pdump block.
+				 * */
+
+				RGXCCBPDumpFastForwardCCB(psClientCCB, ui32PDumpFlags);
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_NONE:
+			/* Invalid event for transition */
+		default:
+			{
+				/* Unknown Transition event */
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+	}
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+
+static INLINE void _RGXInitCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	psClientCCB->sUtilisation.ui32HighWaterMark = 0; /* initialize ui32HighWaterMark level to zero */
+	psClientCCB->sUtilisation.ui32ThresholdBytes = (psClientCCB->ui32Size *
+							PVRSRV_ENABLE_CCCB_UTILISATION_INFO_THRESHOLD)	/ 100;
+	psClientCCB->sUtilisation.ui32Warnings = 0;
+	psClientCCB->sUtilisation.ui32CCBAcquired = 0;
+	psClientCCB->sUtilisation.ui32CCBFull = 0;
+}
+
+static INLINE void _RGXCCBUtilisationEvent(RGX_CLIENT_CCB *psClientCCB,
+						IMG_UINT32 ui32WarningType,
+						IMG_UINT32 ui32CmdSize)
+{
+	/* in VERBOSE mode we will print a message for each different
+	 * event type as they happen.
+	 */
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	if (!(psClientCCB->sUtilisation.ui32Warnings & ui32WarningType))
+	{
+		if (ui32WarningType == PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED)
+		{
+			PVR_LOG(("Failed to acquire CCB space for %u byte command:", ui32CmdSize));
+		}
+
+		PVR_LOG(("%s: Client CCB (%s) watermark (%u) hit %d%% of its allocation size (%u)",
+									__func__,
+									psClientCCB->szName,
+									psClientCCB->sUtilisation.ui32HighWaterMark,
+									psClientCCB->sUtilisation.ui32HighWaterMark * 100 / psClientCCB->ui32Size,
+									psClientCCB->ui32Size));
+
+		/* record that we have issued a warning of this type */
+		psClientCCB->sUtilisation.ui32Warnings |= ui32WarningType;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psClientCCB);
+	PVR_UNREFERENCED_PARAMETER(ui32WarningType);
+	PVR_UNREFERENCED_PARAMETER(ui32CmdSize);
+#endif
+}
+
+/* Check the current CCB utilisation. Print a one-time warning message if it is above the
+ * specified threshold
+ */
+static INLINE void _RGXCheckCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	/* Print a warning message if the cCCB watermark is above the threshold value */
+	if (psClientCCB->sUtilisation.ui32HighWaterMark >= psClientCCB->sUtilisation.ui32ThresholdBytes)
+	{
+		_RGXCCBUtilisationEvent(psClientCCB,
+					PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_THRESHOLD,
+					0);
+	}
+}
+
+/* Update the cCCB high watermark level if necessary */
+static void _RGXUpdateCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	IMG_UINT32 ui32FreeSpace, ui32MemCurrentUsage;
+
+	RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+	                           INVALIDATE);
+
+	ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+									  psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+									  psClientCCB->ui32Size);
+	ui32MemCurrentUsage = psClientCCB->ui32Size - ui32FreeSpace;
+
+	if (ui32MemCurrentUsage > psClientCCB->sUtilisation.ui32HighWaterMark)
+	{
+		psClientCCB->sUtilisation.ui32HighWaterMark = ui32MemCurrentUsage;
+
+		/* The high water mark has increased. Check if it is above the
+		 * threshold so we can print a warning if necessary.
+		 */
+		_RGXCheckCCBUtilisation(psClientCCB);
+	}
+}
+
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+PVRSRV_ERROR RGXCreateCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+						  IMG_UINT32			ui32CCBSizeLog2,
+						  IMG_UINT32			ui32CCBMaxSizeLog2,
+						  IMG_UINT32			ui32ContextFlags,
+						  CONNECTION_DATA		*psConnectionData,
+						  RGX_CCB_REQUESTOR_TYPE		eRGXCCBRequestor,
+						  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+						  RGX_CLIENT_CCB		**ppsClientCCB,
+						  DEVMEM_MEMDESC		**ppsClientCCBMemDesc,
+						  DEVMEM_MEMDESC		**ppsClientCCBCtrlMemDesc)
+{
+	PVRSRV_ERROR	eError = PVRSRV_OK;
+	PVRSRV_MEMALLOCFLAGS_T	uiClientCCBMemAllocFlags, uiClientCCBCtlMemAllocFlags;
+	IMG_UINT32		ui32FWMainLog2PageSize = DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap);
+	IMG_UINT32		ui32ChunkSize = (1U << ui32FWMainLog2PageSize);
+	IMG_UINT32		ui32AllocSize = MAX((1U << ui32CCBSizeLog2), ui32ChunkSize);
+	IMG_UINT32		ui32MinAllocSize = MAX((1U << MIN_SAFE_CCB_SIZE_LOG2), ui32ChunkSize);
+	RGX_CLIENT_CCB	*psClientCCB;
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	IMG_UINT32		ui32NumChunks = ui32AllocSize / ui32ChunkSize;
+	IMG_UINT32		ui32VirtualAllocSize = (1U << ui32CCBMaxSizeLog2);
+	IMG_UINT32		ui32NumVirtChunks = ui32VirtualAllocSize / ui32ChunkSize;
+	IMG_UINT32		i;
+
+	/* For the allocation request to be valid, at least one page is required.
+	 * This is relevant on systems where the page size is greater than the client CCB size. */
+	ui32NumVirtChunks = MAX(1, ui32NumVirtChunks);
+	PVR_ASSERT((ui32ChunkSize >= (1U << PAGE_SHIFT)));
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32CCBMaxSizeLog2);
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+	/* All client CCBs should be at-least of the "minimum" size and not to exceed "maximum" */
+	if ((ui32CCBSizeLog2 < MIN_SAFE_CCB_SIZE_LOG2) ||
+		(ui32CCBSizeLog2 > MAX_SAFE_CCB_SIZE_LOG2))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: %s CCB size is invalid (%d). Should be from %d to %d",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         ui32CCBSizeLog2, MIN_SAFE_CCB_SIZE_LOG2, MAX_SAFE_CCB_SIZE_LOG2));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if ((ui32CCBMaxSizeLog2 < ui32CCBSizeLog2) ||
+		(ui32CCBMaxSizeLog2 > MAX_SAFE_CCB_SIZE_LOG2))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: %s CCB maximum size is invalid (%d). Should be from %d to %d",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         ui32CCBMaxSizeLog2, ui32CCBSizeLog2, MAX_SAFE_CCB_SIZE_LOG2));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+
+	psClientCCB = OSAllocMem(sizeof(*psClientCCB));
+	if (psClientCCB == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+	psClientCCB->psServerCommonContext = psServerCommonContext;
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	psClientCCB->ui32VirtualAllocSize = 0;
+	psClientCCB->pui32MappingTable = NULL;
+	psClientCCB->ui32ChunkSize = ui32ChunkSize;
+#endif
+
+	uiClientCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+								PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+								PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+								PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+#if defined(DEBUG)
+								PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC |
+#endif
+								PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+								PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	uiClientCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+								PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+								PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+								PVRSRV_MEMALLOCFLAG_CPU_UNCACHED |
+								PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+								PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+								PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	/* If connection data indicates Sync Lockup Recovery (SLR) should be disabled,
+	 * or if the caller has set ui32ContextFlags to disable SLR for this context,
+	 * indicate this in psClientCCB->ui32CCBFlags.
+	 */
+	if ((psConnectionData->ui32ClientFlags & SRV_FLAGS_CLIENT_SLR_DISABLED) ||
+	    (ui32ContextFlags & RGX_CONTEXT_FLAG_DISABLESLR))
+	{
+		BIT_SET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGXFW cCCB");
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if (BITMASK_HAS(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN))
+	{
+		PHYS_HEAP *psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+		PHYS_HEAP_POLICY uiHeapPolicy = PhysHeapGetPolicy(psPhysHeap);
+
+		psClientCCB->ui32VirtualAllocSize = ui32VirtualAllocSize;
+
+		if (uiHeapPolicy != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+		{
+			psClientCCB->pui32MappingTable = NULL;
+			/*
+			 * On LMA sparse memory can't be mapped to kernel without support for non physically
+			 * sparse allocations.
+			 * To work around this whole ccb memory is allocated at once as contiguous.
+			 */
+			eError = DevmemFwAllocate(psDevInfo,
+									ui32VirtualAllocSize,
+									uiClientCCBMemAllocFlags,
+									aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+									&psClientCCB->psClientCCBMemDesc);
+		}
+		else
+		{
+			/*
+			 * Growing CCB is doubling the size. Last grow would require only ui32NumVirtChunks/2 new chunks
+			 * because another ui32NumVirtChunks/2 is already allocated.
+			 * Sometimes initial chunk count would be higher (when CCB size is equal to CCB maximum size) so MAX is needed.
+			 */
+			psClientCCB->pui32MappingTable = OSAllocMem(MAX(ui32NumChunks, ui32NumVirtChunks/2) * sizeof(IMG_UINT32));
+			if (psClientCCB->pui32MappingTable == NULL)
+			{
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_mtable;
+			}
+
+			for (i = 0; i < ui32NumChunks; i++)
+			{
+				psClientCCB->pui32MappingTable[i] = i;
+			}
+
+			eError = DevmemFwAllocateSparse(psDevInfo,
+											ui32VirtualAllocSize,
+											ui32NumChunks,
+											ui32NumVirtChunks,
+											psClientCCB->pui32MappingTable,
+											uiClientCCBMemAllocFlags,
+											aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+											&psClientCCB->psClientCCBMemDesc);
+		}
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+		psClientCCB->pui32MappingTable = NULL;
+		psClientCCB->ui32VirtualAllocSize = 0;
+	}
+
+	if (!BITMASK_HAS(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN) ||
+	    (eError != PVRSRV_OK))
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+	{
+		/* Allocate ui32AllocSize, or the next best POT allocation */
+		do
+		{
+			eError = DevmemFwAllocate(psDevInfo,
+									ui32AllocSize,
+									uiClientCCBMemAllocFlags,
+									aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+									&psClientCCB->psClientCCBMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				/* Failed to allocate - ensure CCB grow is disabled from
+				 * now on for this device.
+				 */
+				BITMASK_UNSET(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+
+				/* Failed to allocate, try next POT down */
+				ui32AllocSize >>= 1;
+			}
+		} while ((eError != PVRSRV_OK) && (ui32AllocSize > ui32MinAllocSize));
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate RGX client CCB (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_alloc_ccb;
+	}
+
+	OSSNPrintf(psClientCCB->szName, MAX_CLIENT_CCB_NAME, "%s-P%lu-T%lu-%s",
+									aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+									(unsigned long) OSGetCurrentClientProcessIDKM(),
+									(unsigned long) OSGetCurrentClientThreadIDKM(),
+									OSGetCurrentClientProcessNameKM());
+
+	if (ui32AllocSize < (1U << ui32CCBSizeLog2))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unable to allocate %d bytes for RGX client CCB (%s) but allocated %d bytes",
+		         __func__,
+		         (1U << ui32CCBSizeLog2),
+		         psClientCCB->szName,
+		         ui32AllocSize));
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									  &psClientCCB->pvClientCCB);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map RGX client CCB (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_map_ccb;
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGXFW cCCB control");
+	eError = DevmemFwAllocate(psDevInfo,
+										sizeof(RGXFWIF_CCCB_CTL),
+										uiClientCCBCtlMemAllocFlags,
+										aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_CONTROL_STRING],
+										&psClientCCB->psClientCCBCtrlMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate RGX client CCB control (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_alloc_ccbctrl;
+	}
+
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc,
+									  (void **) &psClientCCB->psClientCCBCtrl);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map RGX client CCB control (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_map_ccbctrl;
+	}
+
+	/* psClientCCBCtrlMemDesc was zero alloc'd so no need to initialise offsets. */
+	psClientCCB->psClientCCBCtrl->ui32WrapMask = ui32AllocSize - 1;
+
+	/* Flush the whole struct since other parts are implicitly init (zero'd) */
+	RGXFwSharedMemCacheOpPtr(psClientCCB->psClientCCBCtrl,
+	                         FLUSH);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "cCCB control");
+	DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCCB_CTL),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	psClientCCB->ui32HostWriteOffset = 0;
+	psClientCCB->ui32LastPDumpWriteOffset = 0;
+	psClientCCB->ui32Size = ui32AllocSize;
+	psClientCCB->ui32LastROff = ui32AllocSize - 1;
+	psClientCCB->ui32ByteCount = 0;
+	psClientCCB->ui32LastByteCount = 0;
+	BIT_UNSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	eError = OSLockCreate(&psClientCCB->hCCBGrowLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create hCCBGrowLock (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_create_ccbgrow_lock;
+	}
+#endif
+#if defined(DEBUG)
+	psClientCCB->ui32UpdateEntries = 0;
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXInitCCBUtilisation(psClientCCB);
+	psClientCCB->eRGXCCBRequestor = eRGXCCBRequestor;
+#endif
+	eError = PDumpRegisterTransitionCallback(psConnectionData->psPDumpConnectionData,
+											  _RGXCCBPDumpTransition,
+											  psClientCCB,
+											  psDevInfo,
+											  &psClientCCB->hTransition);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_pdumpreg;
+	}
+
+	/*
+	 * Note:
+	 * Save the PDump specific structure, which is ref counted unlike
+	 * the connection data, to ensure it's not freed too early
+	 */
+	psClientCCB->psPDumpConnectionData = psConnectionData->psPDumpConnectionData;
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "New RGXFW cCCB(%s@%p) created",
+				 psClientCCB->szName,
+				 psClientCCB);
+
+	*ppsClientCCB = psClientCCB;
+	*ppsClientCCBMemDesc = psClientCCB->psClientCCBMemDesc;
+	*ppsClientCCBCtrlMemDesc = psClientCCB->psClientCCBCtrlMemDesc;
+	return PVRSRV_OK;
+
+fail_pdumpreg:
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockDestroy(psClientCCB->hCCBGrowLock);
+fail_create_ccbgrow_lock:
+#endif
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc);
+fail_map_ccbctrl:
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBCtrlMemDesc);
+fail_alloc_ccbctrl:
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+fail_map_ccb:
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBMemDesc);
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+fail_alloc_ccb:
+	if (psClientCCB->pui32MappingTable)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+	}
+fail_alloc_mtable:
+#else
+fail_alloc_ccb:
+#endif
+	OSFreeMem(psClientCCB);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void RGXDestroyCCB(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_CLIENT_CCB *psClientCCB)
+{
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	if (psClientCCB->sUtilisation.ui32CCBFull)
+	{
+		PVR_LOG(("CCBUtilisationInfo: GPU %s command buffer was full %d times out of %d. "
+				"This is not an error but the application may not run optimally.",
+				aszCCBRequestors[psClientCCB->eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+				psClientCCB->sUtilisation.ui32CCBFull,
+				psClientCCB->sUtilisation.ui32CCBAcquired));
+	}
+#endif
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockDestroy(psClientCCB->hCCBGrowLock);
+#endif
+	PDumpUnregisterTransitionCallback(psClientCCB->hTransition);
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBCtrlMemDesc);
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBMemDesc);
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if (psClientCCB->pui32MappingTable)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+	}
+#endif
+	OSFreeMem(psClientCCB);
+}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+static PVRSRV_ERROR _RGXCCBMemChangeSparse(RGX_CLIENT_CCB *psClientCCB,
+										  IMG_UINT32 ui32AllocPageCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32	 i;
+
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+
+	for (i = 0; i < ui32AllocPageCount; i++)
+	{
+		psClientCCB->pui32MappingTable[i] = ui32AllocPageCount + i;
+	}
+
+	/* Double the CCB size (CCB must be POT) by adding ui32AllocPageCount new pages */
+	eError = DeviceMemChangeSparse(psClientCCB->psClientCCBMemDesc,
+									ui32AllocPageCount,
+									psClientCCB->pui32MappingTable,
+									0,
+									NULL,
+									SPARSE_RESIZE_ALLOC);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to grow RGX client CCB (%s)",
+				PVRSRVGetErrorString(eError)));
+
+		if (DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									&psClientCCB->pvClientCCB) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to reacquire CCB mapping"));
+			psClientCCB->pvClientCCB = NULL;
+		}
+
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									&psClientCCB->pvClientCCB);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to map RGX client CCB (%s)",
+				PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+PVRSRV_ERROR RGXCheckSpaceCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32CmdSize)
+{
+	IMG_UINT32 ui32FreeSpace;
+
+	/* Check that the CCB can hold this command + padding */
+	if ((ui32CmdSize + PADDING_COMMAND_SIZE + 1) > psClientCCB->ui32Size)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Command size (%d bytes) too big for CCB"
+		        " (%d bytes)", ui32CmdSize, psClientCCB->ui32Size));
+		return PVRSRV_ERROR_CMD_TOO_BIG;
+	}
+
+	/*
+		Check we don't overflow the end of the buffer and make sure we have
+		enough space for the padding command. If we don't have enough space
+		(including the minimum amount for the padding command) we need to make
+		sure we insert a padding command now and wrap before adding the main
+		command.
+	*/
+
+	/* Invalidate read offset */
+	RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+	                           INVALIDATE);
+	if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) <= psClientCCB->ui32Size)
+	{
+		ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+		                              psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+		                              psClientCCB->ui32Size);
+
+		/* Don't allow all the space to be used */
+		if (ui32FreeSpace > ui32CmdSize)
+		{
+			return PVRSRV_OK;
+		}
+
+		goto e_retry;
+	}
+	else
+	{
+		IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+		ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+		                              psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+		                              psClientCCB->ui32Size);
+
+		/* Check there is space for both the command and the padding command */
+		if (ui32FreeSpace > ui32Remain + ui32CmdSize)
+		{
+			return PVRSRV_OK;
+		}
+
+		goto e_retry;
+	}
+
+e_retry:
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXCCBUtilisationEvent(psClientCCB,
+	            PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_FULL_CCB,
+	            ui32CmdSize);
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+	return PVRSRV_ERROR_RETRY;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireCCB
+
+ PURPOSE	: Obtains access to write some commands to a CCB
+
+ PARAMETERS	: psClientCCB		- The client CCB
+			  ui32CmdSize		- How much space is required
+			  ppvBufferSpace	- Pointer to space in the buffer
+			  ui32PDumpFlags - Should this be PDump continuous?
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXAcquireCCB(RGX_CLIENT_CCB *psClientCCB,
+										IMG_UINT32		ui32CmdSize,
+										void			**ppvBufferSpace,
+										IMG_UINT32		ui32PDumpFlags)
+{
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	IMG_UINT32	ui32RetryCount = 2;
+#endif
+
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_BOOL	bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, ui32PDumpFlags);
+	IMG_BOOL	bPDumpFlagsContinuous = PDUMP_IS_CONTINUOUS(ui32PDumpFlags);
+
+	/*
+		PDumpSetFrame will detect as we Transition into capture range for
+		frame based data but if we are PDumping continuous data then we
+		need to inform the PDump layer ourselves
+
+		First check is to confirm we are in continuous mode
+		Second check is to confirm the pdump client is connected and ready.
+		Third check is to confirm we are not in capture range.
+	*/
+	if (bPDumpFlagsContinuous &&
+		bPDumpEnabled &&
+		!PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_NONE))
+	{
+		PDUMP_TRANSITION_EVENT eTransitionEvent = BIT_ISSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN) ?
+			PDUMP_TRANSITION_EVENT_RANGE_APPEND : PDUMP_TRANSITION_EVENT_RANGE_ENTERED;
+
+		eError = PDumpTransition(psDeviceNode,
+		                         psClientCCB->psPDumpConnectionData,
+		                         eTransitionEvent,
+		                         ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	/* Check that the CCB can hold this command + padding */
+	if ((ui32CmdSize + PADDING_COMMAND_SIZE + 1) > psClientCCB->ui32Size)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Command size (%d bytes) too big for CCB (%d bytes)",
+								ui32CmdSize, psClientCCB->ui32Size));
+		return PVRSRV_ERROR_CMD_TOO_BIG;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	while (ui32RetryCount--)
+#endif
+	{
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+		psClientCCB->sUtilisation.ui32CCBAcquired++;
+#endif
+
+		/*
+			Check we don't overflow the end of the buffer and make sure we have
+			enough space for the padding command. We don't have enough space (including the
+			minimum amount for the padding command) we will need to make sure we insert a
+			padding command now and wrap before adding the main command.
+		*/
+
+		if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) <= psClientCCB->ui32Size)
+		{
+			/* The command can fit without wrapping... */
+			IMG_UINT32 ui32FreeSpace;
+
+#if defined(PDUMP)
+			/* Wait for sufficient CCB space to become available */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+								  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+								  ui32CmdSize, psClientCCB->ui32HostWriteOffset,
+								  psClientCCB->szName);
+			DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+						offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+						psClientCCB->ui32HostWriteOffset,
+						ui32CmdSize,
+						psClientCCB->ui32Size);
+#endif
+
+			RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+			                           INVALIDATE);
+			ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+										psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+										psClientCCB->ui32Size);
+
+			/* Can command fit? */
+			if (ui32FreeSpace > ui32CmdSize)
+			{
+				*ppvBufferSpace = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+				return PVRSRV_OK;
+			}
+			/* There is not enough free space in CCB. */
+			goto e_retry;
+		}
+		else
+		{
+			/*
+				We're at the end of the buffer without enough contiguous space.
+				The command cannot fit without wrapping, we need to insert a
+				padding command and wrap. We need to do this in one go otherwise
+				we would be leaving unflushed commands and forcing the client to
+				deal with flushing the padding command but not the command they
+				wanted to write. Therefore we either do all or nothing.
+			*/
+			RGXFWIF_CCB_CMD_HEADER *psHeader;
+			IMG_UINT32 ui32FreeSpace;
+			IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+			RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+			                           INVALIDATE);
+			/* Check this is a growable CCB */
+			if (psClientCCB->ui32VirtualAllocSize > 0)
+			{
+				PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+
+				ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+											psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+											psClientCCB->ui32Size);
+				/*
+				 * Check if CCB should grow or be wrapped.
+				 * Wrap CCB if there is no need for grow (CCB is half empty) or CCB can't grow,
+				 * and when is free space for command and padding.
+				 */
+				if (((ui32FreeSpace > psClientCCB->ui32Size/2) || (psClientCCB->ui32Size == psClientCCB->ui32VirtualAllocSize)) &&
+					(ui32FreeSpace > ui32Remain + ui32CmdSize))
+				{
+					/* Wrap CCB */
+					psHeader = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+					psHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PADDING;
+					psHeader->ui32CmdSize = ui32Remain - sizeof(RGXFWIF_CCB_CMD_HEADER);
+
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags,
+										  "cCCB(%p): Padding cmd %d", psClientCCB, psHeader->ui32CmdSize);
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+										psClientCCB->ui32HostWriteOffset,
+										ui32Remain,
+										ui32PDumpFlags);
+					}
+#endif
+
+					*ppvBufferSpace = psClientCCB->pvClientCCB;
+					return PVRSRV_OK;
+				}
+				else if ((psClientCCB->ui32Size < psClientCCB->ui32VirtualAllocSize) &&
+				         (psClientCCB->ui32HostWriteOffset >= psClientCCB->psClientCCBCtrl->ui32ReadOffset))
+				{
+					/* Grow CCB */
+					PHYS_HEAP *psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+					PHYS_HEAP_POLICY uiHeapPolicy = PhysHeapGetPolicy(psPhysHeap);
+					PVRSRV_ERROR eErr = PVRSRV_OK;
+
+					/* Something went wrong if we are here a second time */
+					PVR_ASSERT(ui32RetryCount != 0);
+					OSLockAcquire(psClientCCB->hCCBGrowLock);
+
+					/*
+					 * On LMA sparse memory can't be mapped to kernel without support for non physically
+					 * sparse allocations.
+					 * To work around this whole ccb memory was allocated at once as contiguous.
+					 * In such case below sparse change is not needed because memory is already allocated.
+					 */
+					if (uiHeapPolicy == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+					{
+						IMG_UINT32 ui32AllocChunkCount = psClientCCB->ui32Size / psClientCCB->ui32ChunkSize;
+
+						eErr = _RGXCCBMemChangeSparse(psClientCCB, ui32AllocChunkCount);
+					}
+
+					/* Setup new CCB size */
+					if (eErr == PVRSRV_OK)
+					{
+						psClientCCB->ui32Size += psClientCCB->ui32Size;
+					}
+					else
+					{
+						PVR_LOG(("%s: Client CCB (%s) grow failed (%s)", __func__, psClientCCB->szName, PVRSRVGetErrorString(eErr)));
+						OSLockRelease(psClientCCB->hCCBGrowLock);
+						goto e_retry;
+					}
+
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "cCCB update for grow");
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+											offsetof(RGXFWIF_CCCB_CTL, ui32WrapMask),
+											sizeof(psClientCCB->psClientCCBCtrl->ui32WrapMask),
+											ui32PDumpFlags);
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+											offsetof(RGX_CLIENT_CCB, ui32Size),
+											sizeof(psClientCCB->ui32Size),
+											ui32PDumpFlags);
+					}
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+					PVR_LOG(("%s: Client CCB (%s) grew to %u", __func__, psClientCCB->szName, psClientCCB->ui32Size));
+					/* Reset counters */
+					_RGXInitCCBUtilisation(psClientCCB);
+#endif
+
+					/* CCB doubled the size so retry now. */
+					OSLockRelease(psClientCCB->hCCBGrowLock);
+				}
+				else
+				{
+					/* CCB can't grow anymore and can't be wrapped */
+#if defined(PDUMP)
+					/* Wait for sufficient CCB space to become available */
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+										  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+										  ui32Remain, psClientCCB->ui32HostWriteOffset,
+										  psClientCCB->szName);
+					DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+								offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+								psClientCCB->ui32HostWriteOffset,
+								ui32Remain,
+								psClientCCB->ui32Size);
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+										  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+										  ui32CmdSize, 0 /*ui32HostWriteOffset after wrap */,
+										  psClientCCB->szName);
+					DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+								offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+								0 /*ui32HostWriteOffset after wrap */,
+								ui32CmdSize,
+								psClientCCB->ui32Size);
+					/* CCB has now space for our command so try wrapping again. Retry now. */
+#else /* defined(PDUMP) */
+					goto e_retry;
+#endif /* defined(PDUMP) */
+				}
+			}
+			else
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+			{
+#if defined(PDUMP)
+				/* Wait for sufficient CCB space to become available */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+									  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+									  ui32Remain, psClientCCB->ui32HostWriteOffset,
+									  psClientCCB->szName);
+				DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+							offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+							psClientCCB->ui32HostWriteOffset,
+							ui32Remain,
+							psClientCCB->ui32Size);
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+									  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+									  ui32CmdSize, 0 /*ui32HostWriteOffset after wrap */,
+									  psClientCCB->szName);
+				DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+							offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+							0 /*ui32HostWriteOffset after wrap */,
+							ui32CmdSize,
+							psClientCCB->ui32Size);
+#endif
+				RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+				                           INVALIDATE);
+				ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+											psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+											psClientCCB->ui32Size);
+
+				if (ui32FreeSpace > ui32Remain + ui32CmdSize)
+				{
+					psHeader = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+					psHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PADDING;
+					psHeader->ui32CmdSize = ui32Remain - sizeof(RGXFWIF_CCB_CMD_HEADER);
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "cCCB(%p): Padding cmd %d", psClientCCB, psHeader->ui32CmdSize);
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+										psClientCCB->ui32HostWriteOffset,
+										ui32Remain,
+										ui32PDumpFlags);
+					}
+#endif
+
+					*ppvBufferSpace = psClientCCB->pvClientCCB;
+					return PVRSRV_OK;
+				}
+
+				goto e_retry;
+			}
+		}
+	}
+e_retry:
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	psClientCCB->sUtilisation.ui32CCBFull++;
+	_RGXCCBUtilisationEvent(psClientCCB,
+				PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED,
+				ui32CmdSize);
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+	return PVRSRV_ERROR_RETRY;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXReleaseCCB
+
+ PURPOSE	: Release a CCB that we have been writing to.
+
+ PARAMETERS	: psDevData			- device data
+			  psCCB				- the CCB
+
+ RETURNS	: None
+******************************************************************************/
+void RGXReleaseCCB(RGX_CLIENT_CCB *psClientCCB,
+								IMG_UINT32		ui32CmdSize,
+								IMG_UINT32		ui32PDumpFlags)
+{
+	IMG_UINT32 ui32NewWriteOffset;
+
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_BOOL	bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, ui32PDumpFlags);
+	IMG_BOOL	bPDumpFlagsContinuous = PDUMP_IS_CONTINUOUS(ui32PDumpFlags);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockAcquire(psClientCCB->hCCBGrowLock);
+#endif
+	/*
+	 * If a padding command was needed then we should now move ui32HostWriteOffset
+	 * forward. The command has already be dumped (if bPDumpEnabled).
+	 */
+	if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) > psClientCCB->ui32Size)
+	{
+		IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+		UPDATE_CCB_OFFSET(psClientCCB->ui32HostWriteOffset,
+						  ui32Remain,
+						  psClientCCB->ui32Size);
+		psClientCCB->ui32ByteCount += ui32Remain;
+	}
+
+#if defined(PDUMP)
+	/* Dump the CCB data */
+	if (bPDumpEnabled)
+	{
+		DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+						   psClientCCB->ui32HostWriteOffset,
+						   ui32CmdSize,
+						   ui32PDumpFlags);
+	}
+#endif
+
+	/*
+	 * Check if there any fences being written that will already be
+	 * satisfied by the last written update command in this CCB. At the
+	 * same time we can ASSERT that all sync addresses are not NULL.
+	 */
+#if defined(DEBUG)
+	{
+		void *pvBufferStart = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+		void *pvBufferEnd   = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset + ui32CmdSize);
+		IMG_BOOL  bMessagePrinted  = IMG_FALSE;
+
+		/* Walk through the commands in this section of CCB being released... */
+		while (pvBufferStart < pvBufferEnd)
+		{
+			RGXFWIF_CCB_CMD_HEADER *psCmdHeader = pvBufferStart;
+
+			if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_UPDATE)
+			{
+				/* If an UPDATE then record the values in case an adjacent fence uses it. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				psClientCCB->ui32UpdateEntries = 0;
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+					if (psClientCCB->ui32UpdateEntries < RGX_CCCB_FENCE_UPDATE_LIST_SIZE)
+					{
+						psClientCCB->asFenceUpdateList[psClientCCB->ui32UpdateEntries++] = *psUFOPtr++;
+					}
+				}
+			}
+			else if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE)
+			{
+				/* If a FENCE then check the values against the last UPDATE issued. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+
+					if (bMessagePrinted == IMG_FALSE)
+					{
+						RGXFWIF_UFO *psUpdatePtr = psClientCCB->asFenceUpdateList;
+						IMG_UINT32  ui32UpdateIndex;
+
+						for (ui32UpdateIndex = 0; ui32UpdateIndex < psClientCCB->ui32UpdateEntries; ui32UpdateIndex++)
+						{
+							if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+							{
+								if (RGX_UFO_PTR_ADDR(psUFOPtr) == RGX_UFO_PTR_ADDR(psUpdatePtr))
+								{
+									PVR_DPF((PVR_DBG_MESSAGE, "Redundant sync checkpoint check found in cCCB(%p) - 0x%x -> 0x%x",
+											psClientCCB, RGX_UFO_PTR_ADDR(psUFOPtr), psUFOPtr->ui32Value));
+									bMessagePrinted = IMG_TRUE;
+									break;
+								}
+							}
+							else
+							{
+								if (psUFOPtr->puiAddrUFO.ui32Addr == psUpdatePtr->puiAddrUFO.ui32Addr  &&
+									psUFOPtr->ui32Value == psUpdatePtr->ui32Value)
+								{
+									PVR_DPF((PVR_DBG_MESSAGE, "Redundant fence check found in cCCB(%p) - 0x%x -> 0x%x",
+											psClientCCB, psUFOPtr->puiAddrUFO.ui32Addr, psUFOPtr->ui32Value));
+									bMessagePrinted = IMG_TRUE;
+									break;
+								}
+							}
+							psUpdatePtr++;
+						}
+					}
+
+					psUFOPtr++;
+				}
+			}
+			else if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR)
+			{
+				/* For all other UFO ops check the UFO address is not NULL. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+					psUFOPtr++;
+				}
+			}
+
+			/* Move to the next command in this section of CCB being released... */
+			pvBufferStart = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER) + psCmdHeader->ui32CmdSize);
+		}
+	}
+#endif /* REDUNDANT_SYNCS_DEBUG */
+
+
+#if defined(PVRSRV_FORCE_FLUSH_CCCB_ON_KICK)
+	{
+		DEVMEM_MEMDESC* psClientCCBMemDesc = psClientCCB->psClientCCBMemDesc;
+		void *pvClientCCBAddr = psClientCCB->pvClientCCB;
+		PMR *psClientCCBMemDescPMR = NULL;
+		IMG_DEVMEM_OFFSET_T uiPMROffset;
+
+		DevmemGetPMRData(psClientCCBMemDesc,
+			             (IMG_HANDLE*)&psClientCCBMemDescPMR,
+			             &uiPMROffset);
+
+		CacheOpValExec(psClientCCBMemDescPMR,
+					   (IMG_UINT64)(uintptr_t) pvClientCCBAddr,
+					   uiPMROffset,
+					   psClientCCBMemDesc->uiAllocSize,
+					   PVRSRV_CACHE_OP_FLUSH);
+
+	}
+#endif
+
+	ui32NewWriteOffset = psClientCCB->ui32HostWriteOffset;
+	/*
+	 * Update the CCB write offset.
+	 */
+	UPDATE_CCB_OFFSET(ui32NewWriteOffset, ui32CmdSize,
+	                  psClientCCB->ui32Size);
+	psClientCCB->ui32ByteCount += ui32CmdSize;
+
+	/* Flush the CCB data */
+	RGXFwSharedMemFlushCCB(psClientCCB->pvClientCCB,
+	                       psClientCCB->ui32HostWriteOffset,
+	                       ui32NewWriteOffset,
+	                       psClientCCB->psClientCCBCtrl->ui32WrapMask + 1);
+
+	psClientCCB->ui32HostWriteOffset = ui32NewWriteOffset;
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXUpdateCCBUtilisation(psClientCCB);
+#endif
+	/*
+		PDumpSetFrame will detect as we Transition out of capture range for
+		frame based data but if we are PDumping continuous data then we
+		need to inform the PDump layer ourselves
+
+		First check is to confirm we are in continuous mode
+		Second check is to confirm the pdump client is connected and ready.
+		Third check is to confirm we are not in capture range.
+	*/
+#if defined(PDUMP)
+	if (bPDumpFlagsContinuous &&
+		bPDumpEnabled &&
+		!PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_NONE))
+	{
+		PVRSRV_ERROR eError;
+
+		/* Only Transitioning into capture range can cause an error */
+		eError = PDumpTransition(psDeviceNode,
+		                         psClientCCB->psPDumpConnectionData,
+		                         PDUMP_TRANSITION_EVENT_RANGE_EXITED,
+		                         ui32PDumpFlags);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	if (bPDumpEnabled)
+	{
+		/* Update the PDump write offset to show we PDumped this command */
+		psClientCCB->ui32LastPDumpWriteOffset = psClientCCB->ui32HostWriteOffset;
+	}
+#endif
+
+#if defined(NO_HARDWARE)
+	/*
+		The firmware is not running, it cannot update these; we do here instead.
+	*/
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32DepOffset = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset2 = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset3 = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset4 = psClientCCB->ui32HostWriteOffset;
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psClientCCB->hCCBGrowLock);
+#endif
+}
+
+IMG_UINT32 RGXGetHostWriteOffsetCCB(RGX_CLIENT_CCB *psClientCCB)
+{
+	return psClientCCB->ui32HostWriteOffset;
+}
+
+IMG_UINT32 RGXGetWrapMaskCCB(RGX_CLIENT_CCB *psClientCCB)
+{
+	return psClientCCB->ui32Size-1;
+}
+
+void RGXSetCCBFlags(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32		ui32Flags)
+{
+	if ((ui32Flags & RGX_CONTEXT_FLAG_DISABLESLR))
+	{
+		BIT_SET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+	else
+	{
+		BIT_UNSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+}
+
+void RGXCmdHelperInitCmdCCB_CommandSize(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                        IMG_UINT64 ui64FBSCEntryMask,
+                                        IMG_UINT32 ui32ClientFenceCount,
+                                        IMG_UINT32 ui32ClientUpdateCount,
+                                        IMG_UINT32 ui32CmdSize,
+                                        PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                                        PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                                        PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                                        RGX_CCB_CMD_HELPER_DATA *psCmdHelperData)
+{
+	IMG_UINT32 ui32FenceCmdSize = 0;
+	IMG_UINT32 ui32UpdateCmdSize = 0;
+
+	/* Init the generated data members */
+	psCmdHelperData->ui32FBSCInvalCmdSize = 0;
+	psCmdHelperData->ui64FBSCEntryMask = 0;
+	psCmdHelperData->ui32PreTimeStampCmdSize = 0;
+	psCmdHelperData->ui32PostTimeStampCmdSize = 0;
+	psCmdHelperData->ui32RMWUFOCmdSize = 0;
+
+	/* Total FBSC invalidate command size (header plus command data) */
+	if (ui64FBSCEntryMask != 0)
+	{
+		psCmdHelperData->ui32FBSCInvalCmdSize =
+			        RGX_CCB_FWALLOC_ALIGN(sizeof(psCmdHelperData->ui64FBSCEntryMask) +
+			                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+		psCmdHelperData->ui64FBSCEntryMask = ui64FBSCEntryMask;
+	}
+
+	/* total DM command size (header plus command data) */
+
+	psCmdHelperData->ui32DMCmdSize =
+	        RGX_CCB_FWALLOC_ALIGN(ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+	if (ui32ClientFenceCount != 0)
+	{
+		ui32FenceCmdSize =
+		        RGX_CCB_FWALLOC_ALIGN(ui32ClientFenceCount * sizeof(RGXFWIF_UFO) +
+		                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+	}
+
+	if (ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateCmdSize =
+		        RGX_CCB_FWALLOC_ALIGN(ui32ClientUpdateCount * sizeof(RGXFWIF_UFO) +
+		                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+	}
+
+	if (ppPreAddr && (ppPreAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32PreTimeStampCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER)
+			+ PVR_ALIGN(sizeof(RGXFWIF_DEV_VIRTADDR), RGXFWIF_FWALLOC_ALIGN);
+	}
+
+	if (ppPostAddr && (ppPostAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32PostTimeStampCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER)
+			+ PVR_ALIGN(sizeof(RGXFWIF_DEV_VIRTADDR), RGXFWIF_FWALLOC_ALIGN);
+	}
+
+	if (ppRMWUFOAddr && (ppRMWUFOAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32RMWUFOCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_UFO);
+	}
+
+	psCmdHelperData->ui32TotalSize =
+		ui32FenceCmdSize +
+		psCmdHelperData->ui32FBSCInvalCmdSize +
+		psCmdHelperData->ui32DMCmdSize +
+		ui32UpdateCmdSize +
+		psCmdHelperData->ui32PreTimeStampCmdSize +
+		psCmdHelperData->ui32PostTimeStampCmdSize +
+		psCmdHelperData->ui32RMWUFOCmdSize;
+
+	psCmdHelperData->ui32DMCmdOffset =
+		ui32FenceCmdSize +
+		psCmdHelperData->ui32PreTimeStampCmdSize +
+		psCmdHelperData->ui32FBSCInvalCmdSize;
+}
+
+/*
+	Work out how much space this command will require
+*/
+void RGXCmdHelperInitCmdCCB_OtherData(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                                      RGX_CLIENT_CCB            *psClientCCB,
+                                      IMG_UINT32                ui32ClientFenceCount,
+                                      PRGXFWIF_UFO_ADDR         *pauiFenceUFOAddress,
+                                      IMG_UINT32                *paui32FenceValue,
+                                      IMG_UINT32                ui32ClientUpdateCount,
+                                      PRGXFWIF_UFO_ADDR         *pauiUpdateUFOAddress,
+                                      IMG_UINT32                *paui32UpdateValue,
+                                      IMG_UINT32                ui32CmdSize,
+                                      IMG_PBYTE                 pui8DMCmd,
+                                      PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                                      PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                                      PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                                      RGXFWIF_CCB_CMD_TYPE      eType,
+                                      IMG_UINT32                ui32ExtJobRef,
+                                      IMG_UINT32                ui32IntJobRef,
+                                      IMG_UINT32                ui32PDumpFlags,
+                                      RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                                      IMG_CHAR                  *pszCommandName,
+                                      IMG_BOOL                  bCCBStateOpen,
+                                      RGX_CCB_CMD_HELPER_DATA   *psCmdHelperData)
+{
+	/* Job reference values */
+	psCmdHelperData->ui32ExtJobRef = ui32ExtJobRef;
+	psCmdHelperData->ui32IntJobRef = ui32IntJobRef;
+
+	/* Save the data we require in the submit call */
+	psCmdHelperData->psClientCCB = psClientCCB;
+#if defined(PDUMP)
+	psCmdHelperData->ui32PDumpFlags = ui32PDumpFlags;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+	psCmdHelperData->pszCommandName = pszCommandName;
+	if (bCCBStateOpen)
+	{
+		BIT_SET(psCmdHelperData->psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+	}
+	else
+	{
+		BIT_UNSET(psCmdHelperData->psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+	}
+
+	/* Client sync data */
+	psCmdHelperData->ui32ClientFenceCount = ui32ClientFenceCount;
+	psCmdHelperData->pauiFenceUFOAddress = pauiFenceUFOAddress;
+	psCmdHelperData->paui32FenceValue = paui32FenceValue;
+	psCmdHelperData->ui32ClientUpdateCount = ui32ClientUpdateCount;
+	psCmdHelperData->pauiUpdateUFOAddress = pauiUpdateUFOAddress;
+	psCmdHelperData->paui32UpdateValue = paui32UpdateValue;
+
+	/* Command data */
+	psCmdHelperData->ui32CmdSize = ui32CmdSize;
+	psCmdHelperData->pui8DMCmd = pui8DMCmd;
+	psCmdHelperData->eType = eType;
+
+	if (ppPreAddr)
+	{
+		psCmdHelperData->pPreTimestampAddr = *ppPreAddr;
+	}
+
+	if (ppPostAddr)
+	{
+		psCmdHelperData->pPostTimestampAddr = *ppPostAddr;
+	}
+
+	if (ppRMWUFOAddr)
+	{
+		psCmdHelperData->pRMWUFOAddr = *ppRMWUFOAddr;
+	}
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+			"%s Command Server Init on FWCtx %08x", pszCommandName,
+			FWCommonContextGetFWAddress(psClientCCB->psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Workload Data added */
+		psCmdHelperData->psWorkEstKickData = psWorkEstKickData;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(psWorkEstKickData);
+#endif
+}
+
+/*
+	Work out how much space this command will require
+*/
+void RGXCmdHelperInitCmdCCB(PVRSRV_RGXDEV_INFO		*psDevInfo,
+                            RGX_CLIENT_CCB            *psClientCCB,
+                            IMG_UINT64                ui64FBSCEntryMask,
+                            IMG_UINT32                ui32ClientFenceCount,
+                            PRGXFWIF_UFO_ADDR         *pauiFenceUFOAddress,
+                            IMG_UINT32                *paui32FenceValue,
+                            IMG_UINT32                ui32ClientUpdateCount,
+                            PRGXFWIF_UFO_ADDR         *pauiUpdateUFOAddress,
+                            IMG_UINT32                *paui32UpdateValue,
+                            IMG_UINT32                ui32CmdSize,
+                            IMG_PBYTE                 pui8DMCmd,
+                            PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                            PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                            PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                            RGXFWIF_CCB_CMD_TYPE      eType,
+                            IMG_UINT32                ui32ExtJobRef,
+                            IMG_UINT32                ui32IntJobRef,
+                            IMG_UINT32                ui32PDumpFlags,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                            IMG_CHAR                  *pszCommandName,
+                            IMG_BOOL                  bCCBStateOpen,
+                            RGX_CCB_CMD_HELPER_DATA   *psCmdHelperData)
+{
+	RGXCmdHelperInitCmdCCB_CommandSize(psDevInfo,
+	                                 ui64FBSCEntryMask,
+	                                 ui32ClientFenceCount,
+	                                 ui32ClientUpdateCount,
+	                                 ui32CmdSize,
+	                                 ppPreAddr,
+	                                 ppPostAddr,
+	                                 ppRMWUFOAddr,
+	                                 psCmdHelperData);
+
+	RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+	                                 psClientCCB,
+	                                 ui32ClientFenceCount,
+	                                 pauiFenceUFOAddress,
+	                                 paui32FenceValue,
+	                                 ui32ClientUpdateCount,
+	                                 pauiUpdateUFOAddress,
+	                                 paui32UpdateValue,
+	                                 ui32CmdSize,
+	                                 pui8DMCmd,
+	                                 ppPreAddr,
+	                                 ppPostAddr,
+	                                 ppRMWUFOAddr,
+	                                 eType,
+	                                 ui32ExtJobRef,
+	                                 ui32IntJobRef,
+	                                 ui32PDumpFlags,
+	                                 psWorkEstKickData,
+	                                 pszCommandName,
+	                                 bCCBStateOpen,
+	                                 psCmdHelperData);
+}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*************************************************************************/ /*!
+@Function       RGXIsValidWorkloadEstCCBCommand
+
+@Description    Checks if command type can be used for workload estimation
+
+@Input          eType       Command type to check
+
+
+@Return        IMG_BOOL
+******************************************************************************/
+FORCE_INLINE IMG_BOOL RGXIsValidWorkloadEstCCBCommand(RGXFWIF_CCB_CMD_TYPE eType)
+{
+	switch (eType)
+	{
+			case RGXFWIF_CCB_CMD_TYPE_GEOM:
+			case RGXFWIF_CCB_CMD_TYPE_3D:
+			case RGXFWIF_CCB_CMD_TYPE_CDM:
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+			case RGXFWIF_CCB_CMD_TYPE_RAY:
+#endif
+			case RGXFWIF_CCB_CMD_TYPE_TQ_TDM:
+				return IMG_TRUE;
+			default:
+				PVR_ASSERT(IMG_FALSE);
+				return IMG_FALSE;
+	}
+}
+#endif
+
+static inline void RGXWriteCmdHeader(void *pvCCB, IMG_UINT32 eCmdType, IMG_UINT32 ui32TotalSize,
+									 IMG_UINT32 ui32ExtJobRef, IMG_UINT32 ui32IntJobRef,
+									 RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData)
+{
+	RGXFWIF_CCB_CMD_HEADER sCmdHeader;
+
+	sCmdHeader.eCmdType = eCmdType;
+	sCmdHeader.ui32CmdSize = ui32TotalSize - sizeof(RGXFWIF_CCB_CMD_HEADER);
+	sCmdHeader.ui32ExtJobRef = ui32ExtJobRef;
+	sCmdHeader.ui32IntJobRef = ui32IntJobRef;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (psWorkEstKickData != NULL &&
+		RGXIsValidWorkloadEstCCBCommand(eCmdType))
+	{
+		sCmdHeader.sWorkEstKickData = *psWorkEstKickData;
+	}
+	else
+	{
+		sCmdHeader.sWorkEstKickData.ui16ReturnDataIndex = 0;
+		sCmdHeader.sWorkEstKickData.ui64Deadline = 0;
+		sCmdHeader.sWorkEstKickData.ui32CyclesPrediction = 0;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psWorkEstKickData);
+#endif
+
+	OSCachedMemCopy(pvCCB, &sCmdHeader, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+}
+
+/*
+	Reserve space in the CCB and fill in the command and client sync data
+*/
+PVRSRV_ERROR RGXCmdHelperAcquireCmdCCB(IMG_UINT32 ui32CmdCount,
+									   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+	void *pvStartPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(asCmdHelperData->psClientCCB->psServerCommonContext);
+	RGXFWIF_UFO asUFOs[RGXFWIF_CCB_CMD_MAX_UFOS];
+
+	/*
+		Check the number of fences & updates are valid.
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = &asCmdHelperData[i];
+
+		if (psCmdHelperData->ui32ClientFenceCount > RGXFWIF_CCB_CMD_MAX_UFOS ||
+		    psCmdHelperData->ui32ClientUpdateCount > RGXFWIF_CCB_CMD_MAX_UFOS)
+		{
+			return PVRSRV_ERROR_TOO_MANY_SYNCS;
+		}
+	}
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	ui32AllocSize = RGXCmdHelperGetCommandSize(ui32CmdCount, asCmdHelperData);
+
+#if defined(PDUMP)
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		if ((asCmdHelperData[0].ui32PDumpFlags ^ asCmdHelperData[i].ui32PDumpFlags) & PDUMP_FLAGS_CONTINUOUS)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PDump continuous is not consistent (%s != %s) for command %d",
+					 __func__,
+					 PDUMP_IS_CONTINUOUS(asCmdHelperData[0].ui32PDumpFlags)?"IMG_TRUE":"IMG_FALSE",
+					 PDUMP_IS_CONTINUOUS(asCmdHelperData[i].ui32PDumpFlags)?"IMG_TRUE":"IMG_FALSE",
+					 ui32CmdCount));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	/*
+		Acquire space in the CCB for all the command(s).
+	*/
+	eError = RGXAcquireCCB(asCmdHelperData[0].psClientCCB,
+						   ui32AllocSize,
+						   &pvStartPtr,
+						   asCmdHelperData[0].ui32PDumpFlags);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		return eError;
+	}
+
+	/*
+		For each command fill in the fence, DM, and update command
+
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = & asCmdHelperData[i];
+		void *pvCmdPtr;
+#if defined(PDUMP)
+		IMG_UINT32 ui32CtxAddr = FWCommonContextGetFWAddress(asCmdHelperData->psClientCCB->psServerCommonContext).ui32Addr;
+		IMG_UINT32 ui32CcbWoff = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(asCmdHelperData->psClientCCB->psServerCommonContext));
+#endif
+
+		if (psCmdHelperData->ui32ClientFenceCount+psCmdHelperData->ui32ClientUpdateCount != 0)
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "Start of %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+
+		pvCmdPtr = pvStartPtr;
+
+		/*
+			Create the fence command.
+		*/
+		if (psCmdHelperData->ui32ClientFenceCount > 0)
+		{
+			IMG_UINT k, uiNextValueIndex;
+			IMG_UINT32 ui32FenceCmdSize =
+				RGX_CCB_FWALLOC_ALIGN(psCmdHelperData->ui32ClientFenceCount * sizeof(RGXFWIF_UFO) +
+									  sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_FENCE,
+							  ui32FenceCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			/* Fill in the client fences */
+			uiNextValueIndex = 0;
+			for (k = 0; k < psCmdHelperData->ui32ClientFenceCount; k++)
+			{
+				PVR_ASSERT(k < RGXFWIF_CCB_CMD_MAX_UFOS);
+
+				asUFOs[k].puiAddrUFO = psCmdHelperData->pauiFenceUFOAddress[k];
+
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(psCmdHelperData->pauiFenceUFOAddress[k].ui32Addr))
+				{
+					asUFOs[k].ui32Value = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				}
+				else
+				{
+					/* Only increment uiNextValueIndex for non sync checkpoints
+					 * (as paui32FenceValue only contains values for sync prims)
+					 */
+					asUFOs[k].ui32Value = psCmdHelperData->paui32FenceValue[uiNextValueIndex++];
+				}
+
+#if defined(SYNC_COMMAND_DEBUG)
+				PVR_DPF((PVR_DBG_ERROR, "%s client sync fence - 0x%x -> 0x%x",
+						psCmdHelperData->psClientCCB->szName, asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value));
+#endif
+				PDUMPCOMMENT(psDevInfo->psDeviceNode,
+							 ".. %s client sync fence - 0x%x -> 0x%x",
+							 psCmdHelperData->psClientCCB->szName,
+							 asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value);
+
+
+			}
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&asUFOs, psCmdHelperData->ui32ClientFenceCount * sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, ui32FenceCmdSize);
+		}
+
+		/*
+			Create the FBSC invalidate command.
+		*/
+		if (psCmdHelperData->ui32FBSCInvalCmdSize)
+		{
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE,
+							  psCmdHelperData->ui32FBSCInvalCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->ui64FBSCEntryMask,
+							sizeof(psCmdHelperData->ui64FBSCEntryMask));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32FBSCInvalCmdSize);
+		}
+
+		/*
+		  Create the pre DM timestamp commands. Pre and Post timestamp commands are supposed to
+		  sandwich the DM cmd. The padding code with the CCB wrap upsets the FW if we don't have
+		  the task type bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+		*/
+		if (psCmdHelperData->ui32PreTimeStampCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP,
+							  psCmdHelperData->ui32PreTimeStampCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->pPreTimestampAddr.ui32Addr,
+							sizeof(psCmdHelperData->pPreTimestampAddr.ui32Addr));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32PreTimeStampCmdSize);
+
+		}
+
+		/*
+			Create the DM command
+		*/
+		if (psCmdHelperData->ui32DMCmdSize)
+		{
+			PVR_ASSERT(psCmdHelperData->ui32DMCmdSize == sizeof(RGXFWIF_CCB_CMD_HEADER) + psCmdHelperData->ui32CmdSize);
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  psCmdHelperData->eType,
+							  psCmdHelperData->ui32DMCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)) ? NULL : psCmdHelperData->psWorkEstKickData);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							psCmdHelperData->pui8DMCmd,
+							psCmdHelperData->ui32CmdSize);
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32DMCmdSize);
+		}
+
+
+		if (psCmdHelperData->ui32PostTimeStampCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP,
+							  psCmdHelperData->ui32PostTimeStampCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->pPostTimestampAddr.ui32Addr,
+							sizeof(psCmdHelperData->pPostTimestampAddr.ui32Addr));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32PostTimeStampCmdSize);
+
+		}
+
+
+		if (psCmdHelperData->ui32RMWUFOCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE,
+							  psCmdHelperData->ui32RMWUFOCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			asUFOs[0].puiAddrUFO = psCmdHelperData->pRMWUFOAddr;
+			asUFOs[0].ui32Value = 0;
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							asUFOs, sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32RMWUFOCmdSize);
+		}
+
+		/*
+			Create the update command.
+		*/
+		if (psCmdHelperData->ui32ClientUpdateCount > 0)
+		{
+			IMG_UINT k, uiNextValueIndex;
+			IMG_UINT32 ui32UpdateCmdSize =
+				RGX_CCB_FWALLOC_ALIGN(psCmdHelperData->ui32ClientUpdateCount * sizeof(RGXFWIF_UFO) +
+									  sizeof(RGXFWIF_CCB_CMD_HEADER));
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_UPDATE,
+							  ui32UpdateCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			/* Fill in the client updates */
+			uiNextValueIndex = 0;
+			for (k = 0; k < psCmdHelperData->ui32ClientUpdateCount; k++)
+			{
+				PVR_ASSERT(k < RGXFWIF_CCB_CMD_MAX_UFOS);
+
+				asUFOs[k].puiAddrUFO = psCmdHelperData->pauiUpdateUFOAddress[k];
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(psCmdHelperData->pauiUpdateUFOAddress[k].ui32Addr))
+				{
+					asUFOs[k].ui32Value = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				}
+				else
+				{
+					/* Only increment uiNextValueIndex for non sync checkpoints
+					 * (as paui32UpdateValue only contains values for sync prims)
+					 */
+					asUFOs[k].ui32Value = psCmdHelperData->paui32UpdateValue[uiNextValueIndex++];
+				}
+
+#if defined(SYNC_COMMAND_DEBUG)
+				PVR_DPF((PVR_DBG_ERROR, "%s client sync update - 0x%x -> 0x%x",
+						psCmdHelperData->psClientCCB->szName, asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value));
+#endif
+				PDUMPCOMMENT(psDevInfo->psDeviceNode,
+							 ".. %s client sync update - 0x%x -> 0x%x",
+							 psCmdHelperData->psClientCCB->szName,
+							 asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value);
+
+			}
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&asUFOs, psCmdHelperData->ui32ClientUpdateCount * sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, ui32UpdateCmdSize);
+		}
+
+		/* Set the start pointer for the next iteration around the loop */
+		pvStartPtr = IMG_OFFSET_ADDR(pvStartPtr, psCmdHelperData->ui32TotalSize);
+
+		if (psCmdHelperData->ui32ClientFenceCount+psCmdHelperData->ui32ClientUpdateCount != 0)
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "End of %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+		else
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "No %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	Fill in the server syncs data and release the CCB space
+*/
+void RGXCmdHelperReleaseCmdCCB(IMG_UINT32 ui32CmdCount,
+							   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+							   const IMG_CHAR *pcszDMName,
+							   IMG_UINT32 ui32CtxAddr)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	IMG_BOOL bTraceChecks = trace_rogue_are_fence_checks_traced();
+	IMG_BOOL bTraceUpdates = trace_rogue_are_fence_updates_traced();
+#endif
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	ui32AllocSize = RGXCmdHelperGetCommandSize(ui32CmdCount, asCmdHelperData);
+	/*
+		For each command write PDump comments and emit FTrace events for fence
+		checks and updates if they exist.
+	*/
+	for (i=0;i<ui32CmdCount;i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = &asCmdHelperData[i];
+#if defined(PDUMP) || (defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS))
+		PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psCmdHelperData->psClientCCB->psServerCommonContext);
+#endif
+
+#if (!defined(__linux__) || !defined(PDUMP))
+		PVR_UNREFERENCED_PARAMETER(psCmdHelperData);
+#endif
+
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		if (bTraceChecks)
+		{
+			trace_rogue_fence_checks(psCmdHelperData->pszCommandName,
+									 pcszDMName,
+									 psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+									 ui32CtxAddr,
+									 psCmdHelperData->psClientCCB->ui32HostWriteOffset + ui32AllocSize,
+									 psCmdHelperData->ui32ClientFenceCount,
+									 psCmdHelperData->pauiFenceUFOAddress,
+									 psCmdHelperData->paui32FenceValue);
+		}
+		if (bTraceUpdates)
+		{
+			trace_rogue_fence_updates(psCmdHelperData->pszCommandName,
+									  pcszDMName,
+									  psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+									  ui32CtxAddr,
+									  psCmdHelperData->psClientCCB->ui32HostWriteOffset + ui32AllocSize,
+									  psCmdHelperData->ui32ClientUpdateCount,
+									  psCmdHelperData->pauiUpdateUFOAddress,
+									  psCmdHelperData->paui32UpdateValue);
+		}
+#else
+	PVR_UNREFERENCED_PARAMETER(pcszDMName);
+	PVR_UNREFERENCED_PARAMETER(ui32CtxAddr);
+#endif
+
+		/*
+			All the commands have been filled in so release the CCB space.
+			The FW still won't run this command until we kick it
+		*/
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+				psCmdHelperData->ui32PDumpFlags,
+				"%s Command Server Release on FWCtx %08x",
+				psCmdHelperData->pszCommandName, ui32CtxAddr);
+	}
+
+	RGXReleaseCCB(asCmdHelperData[0].psClientCCB,
+				  ui32AllocSize,
+				  asCmdHelperData[0].ui32PDumpFlags);
+
+	BIT_UNSET(asCmdHelperData[0].psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+}
+
+IMG_UINT32 RGXCmdHelperGetCommandSize(IMG_UINT32              ui32CmdCount,
+                                      RGX_CCB_CMD_HELPER_DATA *asCmdHelperData)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		ui32AllocSize += asCmdHelperData[i].ui32TotalSize;
+	}
+
+	return ui32AllocSize;
+}
+
+/* Work out how much of an offset there is to a specific command. */
+IMG_UINT32 RGXCmdHelperGetCommandOffset(RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+                                        IMG_UINT32              ui32Cmdindex)
+{
+	return RGXCmdHelperGetCommandSize(ui32Cmdindex, asCmdHelperData);
+}
+
+/* Returns the offset of the data master command from a write offset */
+IMG_UINT32 RGXCmdHelperGetDMCommandHeaderOffset(RGX_CCB_CMD_HELPER_DATA *psCmdHelperData)
+{
+	return psCmdHelperData->ui32DMCmdOffset;
+}
+
+static const char *_CCBCmdTypename(RGXFWIF_CCB_CMD_TYPE cmdType)
+{
+	switch (cmdType)
+	{
+		case RGXFWIF_CCB_CMD_TYPE_GEOM: return "TA";
+		case RGXFWIF_CCB_CMD_TYPE_3D: return "3D";
+		case RGXFWIF_CCB_CMD_TYPE_3D_PR: return "3D_PR";
+		case RGXFWIF_CCB_CMD_TYPE_CDM: return "CDM";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_3D: return "TQ_3D";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_2D: return "TQ_2D";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_TDM: return "TQ_TDM";
+		case RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE: return "FBSC_INVALIDATE";
+		case RGXFWIF_CCB_CMD_TYPE_NULL: return "NULL";
+		case RGXFWIF_CCB_CMD_TYPE_FENCE: return "FENCE";
+		case RGXFWIF_CCB_CMD_TYPE_UPDATE: return "UPDATE";
+		case RGXFWIF_CCB_CMD_TYPE_FENCE_PR: return "FENCE_PR";
+		case RGXFWIF_CCB_CMD_TYPE_PRIORITY: return "PRIORITY";
+		case RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP: return "PRE_TIMESTAMP";
+		case RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE: return "RMW_UPDATE";
+		case RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP: return "POST_TIMESTAMP";
+		case RGXFWIF_CCB_CMD_TYPE_PADDING: return "PADDING";
+
+		default:
+			PVR_ASSERT(IMG_FALSE);
+		break;
+	}
+
+	return "INVALID";
+}
+
+PVRSRV_ERROR CheckForStalledCCB(PVRSRV_DEVICE_NODE *psDevNode, RGX_CLIENT_CCB *psCurrentClientCCB, RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;
+	IMG_UINT32					ui32SampledRdOff, ui32SampledDpOff, ui32SampledWrOff, ui32WrapMask;
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+
+	if (psCurrentClientCCB == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "CheckForStalledCCB: CCCB is NULL"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	/* If CCB grow is enabled, take the lock while sampling offsets
+	 * (to guard against a grow happening mid-sample)
+	 */
+	OSLockAcquire(psCurrentClientCCB->hCCBGrowLock);
+#endif
+	/* NB. use psCurrentClientCCB->ui32Size as basis for wrap mask (rather than psClientCCBCtrl->ui32WrapMask)
+	 * as if CCB grow happens, psCurrentClientCCB->ui32Size will have been updated but
+	 * psClientCCBCtrl->ui32WrapMask is only updated once the firmware sees the CCB has grown.
+	 * If we use the wrong value, we might incorrectly determine that the offsets are invalid.
+	 */
+	ui32WrapMask = RGXGetWrapMaskCCB(psCurrentClientCCB);
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	ui32SampledRdOff = psClientCCBCtrl->ui32ReadOffset;
+	ui32SampledDpOff = psClientCCBCtrl->ui32DepOffset;
+	ui32SampledWrOff = psCurrentClientCCB->ui32HostWriteOffset;
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+
+	if (ui32SampledRdOff > ui32WrapMask ||
+		ui32SampledDpOff > ui32WrapMask ||
+		ui32SampledWrOff > ui32WrapMask)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "CheckForStalledCCB: CCCB has invalid offset (ROFF=%d DOFF=%d WOFF=%d)",
+				ui32SampledRdOff, ui32SampledDpOff, ui32SampledWrOff));
+		return PVRSRV_ERROR_INVALID_OFFSET;
+	}
+
+	if (ui32SampledRdOff != ui32SampledWrOff &&
+				psCurrentClientCCB->ui32LastROff != psCurrentClientCCB->ui32LastWOff &&
+				ui32SampledRdOff == psCurrentClientCCB->ui32LastROff &&
+				(psCurrentClientCCB->ui32ByteCount - psCurrentClientCCB->ui32LastByteCount) < psCurrentClientCCB->ui32Size)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+
+		/* Only log a stalled CCB if GPU is idle (any state other than POW_ON is considered idle).
+		 * Guest drivers do not initialize psRGXFWIfFwSysData, so they assume FW internal state is ON. */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+		                           INVALIDATE);
+		if (((psDevInfo->psRGXFWIfFwSysData == NULL) || (psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_ON)) &&
+			(psDevInfo->ui32SLRHoldoffCounter == 0))
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Don't log this by default unless debugging since a higher up
+			 * function will log the stalled condition. Helps avoid double
+			 * messages in the log.
+			 */
+			PVR_DPF((PVR_DBG_ERROR, "%s (%s): CCCB has not progressed (ROFF=%d DOFF=%d WOFF=%d) for \"%s\"",
+					__func__, pszStalledAction, ui32SampledRdOff,
+					ui32SampledDpOff, ui32SampledWrOff,
+					psCurrentClientCCB->szName));
+			eError = PVRSRV_ERROR_CCCB_STALLED;
+
+			{
+				void				*pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+				RGXFWIF_CCB_CMD_HEADER	*psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledRdOff);
+				PVRSRV_RGXDEV_INFO		*psDevInfo = FWCommonContextGetRGXDevInfo(psCurrentClientCCB->psServerCommonContext);
+
+				/* Special case - if readOffset is on a PADDING packet, CCB has wrapped.
+				 * In this case, skip over the PADDING packet.
+				 */
+				if (psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_PADDING)
+				{
+					psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff,
+					                                             ((ui32SampledRdOff +
+					                                               psCommandHeader->ui32CmdSize +
+					                                               sizeof(RGXFWIF_CCB_CMD_HEADER))
+					                                              & psCurrentClientCCB->psClientCCBCtrl->ui32WrapMask));
+				}
+
+				/* Only try to recover a 'stalled' context (ie one waiting on a fence), as some work (eg compute) could
+				 * take a long time to complete, during which time the CCB ptrs would not advance.
+				 */
+				if (((psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE) ||
+				     (psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR)) &&
+				    (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff)))
+				{
+					/* Acquire the cCCB recovery lock */
+					OSLockAcquire(psDevInfo->hCCBRecoveryLock);
+
+					if (!psDevInfo->pvEarliestStalledClientCCB)
+					{
+						psDevInfo->pvEarliestStalledClientCCB = (void*)psCurrentClientCCB;
+						psDevInfo->ui32OldestSubmissionOrdinal = psCommandHeader->ui32IntJobRef;
+					}
+					else
+					{
+						/* Check if this fence cmd header has an older submission stamp than the one we are currently considering unblocking
+						 * (account for submission stamp wrap by checking diff is less than 0x80000000) - if it is older, then this becomes
+						 * our preferred fence to be unblocked/
+						 */
+						if ((psCommandHeader->ui32IntJobRef < psDevInfo->ui32OldestSubmissionOrdinal) &&
+						    ((psDevInfo->ui32OldestSubmissionOrdinal - psCommandHeader->ui32IntJobRef) < 0x8000000))
+						{
+							psDevInfo->pvEarliestStalledClientCCB = (void*)psCurrentClientCCB;
+							psDevInfo->ui32OldestSubmissionOrdinal = psCommandHeader->ui32IntJobRef;
+						}
+					}
+
+					/* Release the cCCB recovery lock */
+					OSLockRelease(psDevInfo->hCCBRecoveryLock);
+				}
+			}
+		}
+	}
+
+	psCurrentClientCCB->ui32LastROff = ui32SampledRdOff;
+	psCurrentClientCCB->ui32LastWOff = ui32SampledWrOff;
+	psCurrentClientCCB->ui32LastByteCount = psCurrentClientCCB->ui32ByteCount;
+
+	return eError;
+}
+
+void DumpCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+			PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+			RGX_CLIENT_CCB *psCurrentClientCCB,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	volatile RGXFWIF_CCCB_CTL *psClientCCBCtrl;
+	void *pvClientCCBBuff;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32DepOffset;
+	IMG_UINT32 ui32EndOffset;
+	IMG_UINT32 ui32WrapMask;
+	IMG_CHAR * pszState = "Ready";
+
+	/* Ensure hCCBGrowLock is acquired before reading
+	 * psCurrentClientCCB->pvClientCCB as a CCB grow
+	 * could remap the virtual addresses.
+	 */
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockAcquire(psCurrentClientCCB->hCCBGrowLock);
+#endif
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+	ui32EndOffset = psCurrentClientCCB->ui32HostWriteOffset;
+	OSMemoryBarrier(NULL);
+	ui32Offset = psClientCCBCtrl->ui32ReadOffset;
+	ui32DepOffset = psClientCCBCtrl->ui32DepOffset;
+	/* NB. Use psCurrentClientCCB->ui32Size as basis for wrap mask (rather
+	 * than psClientCCBCtrl->ui32WrapMask) as if CCB grow happened,
+	 * psCurrentClientCCB->ui32Size will have been updated but
+	 * psClientCCBCtrl->ui32WrapMask is only updated once the firmware
+	 * sees the CCB has grown. If we use the wrong value, ui32NextOffset
+	 * can end up being wrapped prematurely and pointing to garbage.
+	 */
+	ui32WrapMask = RGXGetWrapMaskCCB(psCurrentClientCCB);
+
+	PVR_DUMPDEBUG_LOG("FWCtx 0x%08X (%s)", sFWCommonContext.ui32Addr, psCurrentClientCCB->szName);
+	if (ui32Offset == ui32EndOffset)
+	{
+		PVR_DUMPDEBUG_LOG("  `--<Empty>");
+	}
+
+	if (ui32Offset > ui32WrapMask)
+	{
+		PVR_DUMPDEBUG_LOG("  `--<Invalid offset>");
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+		OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+		return;
+	}
+
+	while (ui32Offset != ui32EndOffset)
+	{
+		RGXFWIF_CCB_CMD_HEADER *psCmdHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32Offset);
+		IMG_UINT32 ui32NextOffset = (ui32Offset + psCmdHeader->ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER)) & ui32WrapMask;
+		IMG_BOOL bLastCommand = (ui32NextOffset == ui32EndOffset)? IMG_TRUE: IMG_FALSE;
+		IMG_BOOL bLastUFO;
+		#define CCB_SYNC_INFO_LEN 80
+		IMG_CHAR pszSyncInfo[CCB_SYNC_INFO_LEN];
+		IMG_UINT32 ui32NoOfUpdates, i;
+		RGXFWIF_UFO *psUFOPtr;
+
+		ui32NoOfUpdates = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+		psUFOPtr = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32Offset + sizeof(RGXFWIF_CCB_CMD_HEADER));
+		pszSyncInfo[0] = '\0';
+
+		if (ui32Offset == ui32DepOffset)
+		{
+			pszState = "Waiting";
+		}
+
+		PVR_DUMPDEBUG_LOG("  %s--%s %s @ %u Int=%u Ext=%u",
+			bLastCommand? "`": "|",
+			pszState, _CCBCmdTypename(psCmdHeader->eCmdType),
+			ui32Offset, psCmdHeader->ui32IntJobRef, psCmdHeader->ui32ExtJobRef
+			);
+
+		/* switch on type and write checks and updates */
+		switch (psCmdHeader->eCmdType)
+		{
+			case RGXFWIF_CCB_CMD_TYPE_UPDATE:
+			case RGXFWIF_CCB_CMD_TYPE_FENCE:
+			case RGXFWIF_CCB_CMD_TYPE_FENCE_PR:
+			{
+				for (i = 0; i < ui32NoOfUpdates; i++, psUFOPtr++)
+				{
+					bLastUFO = (ui32NoOfUpdates-1 == i)? IMG_TRUE: IMG_FALSE;
+
+					if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+					{
+						if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+						{
+							SyncCheckpointRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+						else
+						{
+							SyncRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+					}
+
+					PVR_DUMPDEBUG_LOG("  %s  %s--Addr:0x%08x Val=0x%08x %s",
+						bLastCommand? " ": "|",
+						bLastUFO? "`": "|",
+						psUFOPtr->puiAddrUFO.ui32Addr, psUFOPtr->ui32Value,
+						pszSyncInfo
+						);
+				}
+				break;
+			}
+			case RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE:
+			{
+				for (i = 0; i < ui32NoOfUpdates; i++, psUFOPtr++)
+				{
+					bLastUFO = (ui32NoOfUpdates-1 == i)? IMG_TRUE: IMG_FALSE;
+
+					if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+					{
+						if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+						{
+							SyncCheckpointRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+						else
+						{
+							SyncRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+					}
+
+					PVR_DUMPDEBUG_LOG("  %s  %s--Addr:0x%08x Val++ %s",
+						bLastCommand? " ": "|",
+						bLastUFO? "`": "|",
+						psUFOPtr->puiAddrUFO.ui32Addr,
+						pszSyncInfo
+						);
+				}
+				break;
+			}
+			default:
+				break;
+		}
+
+		/* Check the command size is valid, otherwise if corruption was present this loop might hang... */
+		if ((ui32Offset == ui32NextOffset) ||
+		    ((ui32Offset + psCmdHeader->ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER)) > ui32WrapMask+1))
+		{
+			PVR_DUMPDEBUG_LOG("  `--Invalid CCB offset!");
+			break;
+		}
+
+		ui32Offset = ui32NextOffset;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+}
+
+void DumpFirstCCBCmd(PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+				RGX_CLIENT_CCB *psCurrentClientCCB,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;
+	void					*pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+	IMG_UINT32					ui32SampledWrOff = psCurrentClientCCB->ui32HostWriteOffset;
+	IMG_UINT32					ui32SampledRdOff;
+	IMG_UINT32					ui32SampledDepOff;
+
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	ui32SampledRdOff = psClientCCBCtrl->ui32ReadOffset;
+	ui32SampledDepOff = psClientCCBCtrl->ui32DepOffset;
+
+	if ((ui32SampledRdOff == ui32SampledDepOff) &&
+		(ui32SampledRdOff != ui32SampledWrOff))
+	{
+		volatile RGXFWIF_CCB_CMD_HEADER *psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledRdOff);
+		RGXFWIF_CCB_CMD_TYPE	eCommandType = psCommandHeader->eCmdType;
+		volatile void				*pvPtr = psCommandHeader;
+
+		/* CCB is stalled on a fence... */
+		if ((eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE) || (eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR))
+		{
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+			PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psCurrentClientCCB->psServerCommonContext);
+			IMG_UINT32 ui32Val;
+#endif
+			RGXFWIF_UFO *psUFOPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader));
+			IMG_UINT32 jj;
+
+			/* Display details of the fence object on which the context is pending */
+			PVR_DUMPDEBUG_LOG("FWCtx 0x%08X @ %d (%s) pending on %s:",
+							   sFWCommonContext.ui32Addr,
+							   ui32SampledRdOff,
+							   psCurrentClientCCB->szName,
+							   _CCBCmdTypename(eCommandType));
+			for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+			{
+#if !defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+				PVR_DUMPDEBUG_LOG("  Addr:0x%08x  Value=0x%08x",psUFOPtr[jj].puiAddrUFO.ui32Addr, psUFOPtr[jj].ui32Value);
+#else
+				ui32Val = 0;
+				RGXReadFWModuleAddr(psDevInfo, psUFOPtr[jj].puiAddrUFO.ui32Addr, &ui32Val);
+				PVR_DUMPDEBUG_LOG("  Addr:0x%08x Value(Host)=0x%08x Value(FW)=0x%08x",
+				                   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+				                   psUFOPtr[jj].ui32Value, ui32Val);
+#endif
+			}
+
+			/* Advance psCommandHeader past the FENCE to the next command header (this will be the TA/3D command that is fenced) */
+			pvPtr = IMG_OFFSET_ADDR(psUFOPtr, psCommandHeader->ui32CmdSize);
+			psCommandHeader = pvPtr;
+			if (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff))
+			{
+				PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X fenced command is of type %s",sFWCommonContext.ui32Addr, _CCBCmdTypename(psCommandHeader->eCmdType));
+				/* Advance psCommandHeader past the TA/3D to the next command header (this will possibly be an UPDATE) */
+				pvPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader) + psCommandHeader->ui32CmdSize);
+				psCommandHeader = pvPtr;
+				/* If the next command is an update, display details of that so we can see what would then become unblocked */
+				if (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff))
+				{
+					eCommandType = psCommandHeader->eCmdType;
+
+					if (eCommandType == RGXFWIF_CCB_CMD_TYPE_UPDATE)
+					{
+						psUFOPtr = IMG_OFFSET_ADDR(psCommandHeader, sizeof(*psCommandHeader));
+						PVR_DUMPDEBUG_LOG(" preventing %s:",_CCBCmdTypename(eCommandType));
+						for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+						{
+#if !defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+							PVR_DUMPDEBUG_LOG("  Addr:0x%08x  Value=0x%08x",psUFOPtr[jj].puiAddrUFO.ui32Addr, psUFOPtr[jj].ui32Value);
+#else
+							ui32Val = 0;
+							RGXReadFWModuleAddr(psDevInfo, psUFOPtr[jj].puiAddrUFO.ui32Addr, &ui32Val);
+							PVR_DUMPDEBUG_LOG("  Addr:0x%08x Value(Host)=0x%08x Value(FW)=0x%08x",
+							                   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							                   psUFOPtr[jj].ui32Value,
+							                   ui32Val);
+#endif
+						}
+					}
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X has no further commands",sFWCommonContext.ui32Addr);
+				}
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X has no further commands",sFWCommonContext.ui32Addr);
+			}
+		}
+	}
+}
+
+void DumpStalledContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_CLIENT_CCB *psStalledClientCCB;
+
+	PVR_ASSERT(psDevInfo);
+
+	psStalledClientCCB = (RGX_CLIENT_CCB *)psDevInfo->pvEarliestStalledClientCCB;
+
+	if (psStalledClientCCB)
+	{
+		volatile RGXFWIF_CCCB_CTL *psClientCCBCtrl;
+		IMG_UINT32 ui32SampledDepOffset;
+		void                 *pvPtr;
+		RGXFWIF_CCB_CMD_HEADER    *psCommandHeader;
+		RGXFWIF_CCB_CMD_TYPE      eCommandType;
+
+		RGXFwSharedMemCacheOpValue(psStalledClientCCB->psClientCCBCtrl->ui32DepOffset,
+		                           INVALIDATE);
+		psClientCCBCtrl = psStalledClientCCB->psClientCCBCtrl;
+		ui32SampledDepOffset = psClientCCBCtrl->ui32DepOffset;
+		/* No need to invalidate CCCB as FW doesn't write to it, only read. */
+		pvPtr = IMG_OFFSET_ADDR(psStalledClientCCB->pvClientCCB, ui32SampledDepOffset);
+		psCommandHeader = pvPtr;
+		eCommandType = psCommandHeader->eCmdType;
+
+		if ((eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE) || (eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR))
+		{
+			RGXFWIF_UFO *psUFOPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader));
+			IMG_UINT32 jj;
+			IMG_UINT32 ui32NumUnsignalledUFOs = 0;
+			IMG_UINT32 ui32UnsignalledUFOVaddrs[PVRSRV_MAX_SYNCS];
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+			                         INVALIDATE);
+			if (!psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.aszCCBName[0])
+			{
+				OSClockMonotonicns64(&psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui64Timestamp);
+				psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui32NumUFOs = (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO));
+				psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui32FWCtxAddr = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr;
+				OSStringSafeCopy(psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.aszCCBName,
+				              psStalledClientCCB->szName,
+				              MAX_CLIENT_CCB_NAME);
+			}
+			else
+			{
+				OSClockMonotonicns64(&psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui64Timestamp);
+				psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui32NumUFOs = (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO));
+				psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui32FWCtxAddr = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr;
+				OSStringSafeCopy(psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].aszCCBName,
+				              psStalledClientCCB->szName,
+				              MAX_CLIENT_CCB_NAME);
+				psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp = (psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp + 1) % PVR_SLR_LOG_ENTRIES;
+			}
+			psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested++;
+			/* flush write buffers for psRGXFWIfFwOsData */
+			OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp]);
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+			                         FLUSH);
+#endif
+			PVR_LOG(("Fence found on context 0x%x '%s' @ %d has %d UFOs",
+			         FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr,
+			         psStalledClientCCB->szName, ui32SampledDepOffset,
+			         (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO))));
+
+			for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+			{
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT((RGXFWIF_UFO *)&psUFOPtr[jj]))
+				{
+					IMG_UINT32 ui32ReadValue = SyncCheckpointStateFromUFO(psDevInfo->psDeviceNode,
+					                                           psUFOPtr[jj].puiAddrUFO.ui32Addr);
+					PVR_LOG(("  %d/%d FWAddr 0x%x requires 0x%x (currently 0x%x)", jj+1,
+							   (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO)),
+							   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							   psUFOPtr[jj].ui32Value,
+							   ui32ReadValue));
+					/* If fence is unmet, dump debug info on it */
+					if (ui32ReadValue != psUFOPtr[jj].ui32Value)
+					{
+						/* Add to our list to pass to pvr_sync */
+						ui32UnsignalledUFOVaddrs[ui32NumUnsignalledUFOs] = psUFOPtr[jj].puiAddrUFO.ui32Addr;
+						ui32NumUnsignalledUFOs++;
+					}
+				}
+				else
+				{
+					PVR_LOG(("  %d/%d FWAddr 0x%x requires 0x%x", jj+1,
+							   (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO)),
+							   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							   psUFOPtr[jj].ui32Value));
+				}
+			}
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_FALLBACK_FENCE_SYNC)
+			if (ui32NumUnsignalledUFOs > 0)
+			{
+				IMG_UINT32 ui32NumSyncsOwned;
+				PVRSRV_ERROR eErr = SyncCheckpointDumpInfoOnStalledUFOs(ui32NumUnsignalledUFOs, &ui32UnsignalledUFOVaddrs[0], &ui32NumSyncsOwned);
+
+				PVR_LOG_IF_ERROR(eErr, "SyncCheckpointDumpInfoOnStalledUFOs() call failed.");
+			}
+#endif
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+			if (BIT_ISSET(psStalledClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED))
+			{
+				PRGXFWIF_FWCOMMONCONTEXT psContext = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext);
+
+				PVR_LOG(("SLR disabled for FWCtx 0x%08X", psContext.ui32Addr));
+			}
+			else
+			{
+				if (ui32NumUnsignalledUFOs > 0)
+				{
+					RGXFWIF_KCCB_CMD sSignalFencesCmd;
+
+					sSignalFencesCmd.eCmdType = RGXFWIF_KCCB_CMD_FORCE_UPDATE;
+					sSignalFencesCmd.ui32KCCBFlags = 0;
+					sSignalFencesCmd.uCmdData.sForceUpdateData.psContext = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext);
+					sSignalFencesCmd.uCmdData.sForceUpdateData.ui32CCBFenceOffset = ui32SampledDepOffset;
+
+					PVR_LOG(("Forced update command issued for FWCtx 0x%08X", sSignalFencesCmd.uCmdData.sForceUpdateData.psContext.ui32Addr));
+
+					RGXScheduleCommand(FWCommonContextGetRGXDevInfo(psStalledClientCCB->psServerCommonContext),
+					                   RGXFWIF_DM_GP,
+					                   &sSignalFencesCmd,
+					                   PDUMP_FLAGS_CONTINUOUS);
+				}
+			}
+#endif
+		}
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+}
+
+/******************************************************************************
+ End of file (rgxccb.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.h
new file mode 100644
index 000000000000..e9edd9a37bc3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.h
@@ -0,0 +1,357 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Circular Command Buffer functionality.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Circular Command Buffer functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXCCB_H)
+#define RGXCCB_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "rgxdefs_km.h"
+#include "pvr_notifier.h"
+
+#define MAX_CLIENT_CCB_NAME	30
+#define SYNC_FLAG_MASK_ALL  IMG_UINT32_MAX
+
+/*
+ * This size is to be used when a client CCB is found to consume very
+ * negligible space (e.g. a few hundred bytes to few KBs - less than a page).
+ * In such a case, instead of allocating CCB of size of only a few KBs, we
+ * allocate at-least this much to be future risk-free.
+ */
+#define MIN_SAFE_CCB_SIZE_LOG2         13  /* 8K (2 Pages) */
+#define MAX_SAFE_CCB_SIZE_LOG2         18  /* 256K (64 Pages) */
+
+#define RGX_TQ3D_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D
+static_assert(RGX_TQ3D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TQ3D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ3D CCB size is invalid");
+#define RGX_TQ3D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ3D
+static_assert(RGX_TQ3D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D
+	&& RGX_TQ3D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ3D max CCB size is invalid");
+
+#define RGX_TQ2D_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D
+static_assert(RGX_TQ2D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TQ2D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ2D CCB size is invalid");
+#define RGX_TQ2D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ2D
+static_assert(RGX_TQ2D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D &&
+	RGX_TQ2D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ2D max CCB size is invalid");
+
+#define RGX_CDM_CCB_SIZE_LOG2          PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM
+static_assert(RGX_CDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_CDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "CDM CCB size is invalid");
+#define RGX_CDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_CDM
+static_assert(RGX_CDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM &&
+	RGX_CDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "CDM max CCB size is invalid");
+
+#define RGX_TA_CCB_SIZE_LOG2           PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA
+static_assert(RGX_TA_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TA_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TA CCB size is invalid");
+#define RGX_TA_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TA
+static_assert(RGX_TA_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA &&
+	RGX_TA_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TA max CCB size is invalid");
+
+#define RGX_3D_CCB_SIZE_LOG2           PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D
+static_assert(RGX_3D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_3D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "3D CCB size is invalid");
+#define RGX_3D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_3D
+static_assert(RGX_3D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D &&
+	RGX_3D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "3D max CCB size is invalid");
+
+#define RGX_KICKSYNC_CCB_SIZE_LOG2     PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC
+static_assert(RGX_KICKSYNC_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_KICKSYNC_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "KickSync CCB size is invalid");
+#define RGX_KICKSYNC_CCB_MAX_SIZE_LOG2	PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_KICKSYNC
+static_assert(RGX_KICKSYNC_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC &&
+	RGX_KICKSYNC_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "KickSync max CCB size is invalid");
+
+#define RGX_TDM_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM
+static_assert(RGX_TDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TDM CCB size is invalid");
+#define RGX_TDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TDM
+static_assert(RGX_TDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM &&
+	RGX_TDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TDM max CCB size is invalid");
+
+#define RGX_RDM_CCB_SIZE_LOG2          PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM
+static_assert(RGX_RDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_RDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "RDM CCB size is invalid");
+#define RGX_RDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_RDM
+static_assert(RGX_RDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM &&
+	RGX_RDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "RDM max CCB size is invalid");
+
+typedef struct _RGX_CLIENT_CCB_ RGX_CLIENT_CCB;
+
+/*
+	This structure is declared here as it's allocated on the heap by
+	the callers
+*/
+
+typedef struct _RGX_CCB_CMD_HELPER_DATA_ {
+	/* Data setup at command init time */
+	RGX_CLIENT_CCB			*psClientCCB;
+	IMG_CHAR				*pszCommandName;
+	IMG_UINT32				ui32PDumpFlags;
+
+	IMG_UINT32				ui32ClientFenceCount;
+	PRGXFWIF_UFO_ADDR		*pauiFenceUFOAddress;
+	IMG_UINT32				*paui32FenceValue;
+	IMG_UINT32				ui32ClientUpdateCount;
+	PRGXFWIF_UFO_ADDR		*pauiUpdateUFOAddress;
+	IMG_UINT32				*paui32UpdateValue;
+	RGXFWIF_CCB_CMD_TYPE	eType;
+	IMG_UINT32				ui32CmdSize;
+	IMG_UINT8				*pui8DMCmd;
+	IMG_UINT32				ui32FBSCInvalCmdSize;
+	IMG_UINT32				ui32DMCmdSize;
+	IMG_UINT32				ui32TotalSize;
+	IMG_UINT32				ui32DMCmdOffset;
+
+	/* data for FBSC invalidate command */
+	IMG_UINT64				ui64FBSCEntryMask;
+
+	/* timestamp commands */
+	PRGXFWIF_TIMESTAMP_ADDR pPreTimestampAddr;
+	IMG_UINT32              ui32PreTimeStampCmdSize;
+	PRGXFWIF_TIMESTAMP_ADDR pPostTimestampAddr;
+	IMG_UINT32              ui32PostTimeStampCmdSize;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+	IMG_UINT32              ui32RMWUFOCmdSize;
+
+	/* Job reference fields */
+	IMG_UINT32				ui32ExtJobRef;
+	IMG_UINT32				ui32IntJobRef;
+
+	/* FW Memdesc for Workload information */
+	RGXFWIF_WORKEST_KICK_DATA	*psWorkEstKickData;
+
+} RGX_CCB_CMD_HELPER_DATA;
+
+#define PADDING_COMMAND_SIZE	(sizeof(RGXFWIF_CCB_CMD_HEADER))
+
+
+#define RGX_CCB_REQUESTORS(TYPE) \
+	/* for debugging purposes */ TYPE(UNDEF)	\
+	TYPE(TA)	\
+	TYPE(3D)	\
+	TYPE(CDM)	\
+	TYPE(SH)	\
+	TYPE(RS)	\
+	TYPE(TQ_3D)	\
+	TYPE(TQ_2D)	\
+	TYPE(TQ_TDM)	\
+	TYPE(KICKSYNC)	\
+	TYPE(RAY)	\
+
+/* Forms an enum constant for each type present in RGX_CCB_REQUESTORS list. The enum is mainly used as
+   an index to the aszCCBRequestors table defined in rgxccb.c. The total number of enums must adhere
+   to the following build assert.
+*/
+typedef enum _RGX_CCB_REQUESTOR_TYPE_
+{
+#define CONSTRUCT_ENUM(req) REQ_TYPE_##req,
+	RGX_CCB_REQUESTORS (CONSTRUCT_ENUM)
+#undef CONSTRUCT_ENUM
+
+	/* should always be at the end */
+	REQ_TYPE_TOTAL_COUNT,
+} RGX_CCB_REQUESTOR_TYPE;
+
+/* Tuple describing the columns of the following table */
+typedef enum _RGX_CCB_REQUESTOR_TUPLE_
+{
+	REQ_RGX_FW_CLIENT_CCB_STRING,          /* Index to comment to be dumped in DevMemAllocs when allocating FirmwareClientCCB for this requestor */
+	REQ_RGX_FW_CLIENT_CCB_CONTROL_STRING,  /* Index to comment to be dumped in DevMemAllocs when allocating FirmwareClientCCBControl for this requestor */
+	REQ_PDUMP_COMMENT,                     /* Index to comment to be dumped in PDUMPs */
+
+	/* should always be at the end */
+	REQ_TUPLE_CARDINALITY,
+} RGX_CCB_REQUESTOR_TUPLE;
+
+/* Unpack U8 values from U32. */
+#define U32toU8_Unpack1(U32Packed) (U32Packed & 0xFF)
+#define U32toU8_Unpack2(U32Packed) ((U32Packed>>8) & 0xFF)
+#define U32toU8_Unpack3(U32Packed) ((U32Packed>>16) & 0xFF)
+#define U32toU8_Unpack4(U32Packed) ((U32Packed>>24) & 0xFF)
+
+/* Defines for bit meanings within the ui32CCBFlags member of struct _RGX_CLIENT_CCB_
+ *
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *   31                             10
+ *    |                             ||
+ *    ------------------------------XX
+ *  Bit   Meaning
+ *    0 = If set, CCB is still open and commands will be appended to it
+ *    1 = If set, do not perform Sync Lockup Recovery (SLR) for this CCB
+ */
+#define CCB_FLAGS_CCB_STATE_OPEN (0)  /*!< This bit is set to indicate CCB is in the 'Open' state. */
+#define CCB_FLAGS_SLR_DISABLED   (1)  /*!< This bit is set to disable Sync Lockup Recovery (SLR) for this CCB. */
+
+
+/*	Table containing an array of strings for each requestor type in the list of RGX_CCB_REQUESTORS. In addition to its use in
+	this module (rgxccb.c), this table is also used to access string to be dumped in PDUMP comments, hence, marking it extern for
+	use in other modules.
+*/
+extern const IMG_CHAR *const aszCCBRequestors[][REQ_TUPLE_CARDINALITY];
+
+PVRSRV_ERROR RGXCCBPDumpDrainCCB(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR RGXCreateCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+						  IMG_UINT32			ui32CCBSizeLog2,
+						  IMG_UINT32			ui32CCBMaxSizeLog2,
+						  IMG_UINT32			ui32ContextFlags,
+						  CONNECTION_DATA		*psConnectionData,
+						  RGX_CCB_REQUESTOR_TYPE	eCCBRequestor,
+						  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+						  RGX_CLIENT_CCB		**ppsClientCCB,
+						  DEVMEM_MEMDESC		**ppsClientCCBMemDesc,
+						  DEVMEM_MEMDESC		**ppsClientCCBCtlMemDesc);
+
+void RGXDestroyCCB(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_CLIENT_CCB *psClientCCB);
+
+PVRSRV_ERROR RGXCheckSpaceCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32CmdSize);
+
+PVRSRV_ERROR RGXAcquireCCB(RGX_CLIENT_CCB *psClientCCB,
+						   IMG_UINT32		ui32CmdSize,
+						   void				**ppvBufferSpace,
+						   IMG_UINT32		ui32PDumpFlags);
+
+void RGXReleaseCCB(RGX_CLIENT_CCB *psClientCCB,
+								IMG_UINT32		ui32CmdSize,
+								IMG_UINT32		ui32PDumpFlags);
+
+IMG_UINT32 RGXGetHostWriteOffsetCCB(RGX_CLIENT_CCB *psClientCCB);
+IMG_UINT32 RGXGetWrapMaskCCB(RGX_CLIENT_CCB *psClientCCB);
+
+void RGXSetCCBFlags(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32		ui32Flags);
+
+void RGXCmdHelperInitCmdCCB_CommandSize(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT64 ui64FBSCEntryMask,
+                                        IMG_UINT32 ui32ClientFenceCount,
+                                        IMG_UINT32 ui32ClientUpdateCount,
+                                        IMG_UINT32 ui32CmdSize,
+                                        PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                                        PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                                        PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                                        RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void RGXCmdHelperInitCmdCCB_OtherData(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      RGX_CLIENT_CCB *psClientCCB,
+                                      IMG_UINT32 ui32ClientFenceCount,
+                                      PRGXFWIF_UFO_ADDR *pauiFenceUFOAddress,
+                                      IMG_UINT32 *paui32FenceValue,
+                                      IMG_UINT32 ui32ClientUpdateCount,
+                                      PRGXFWIF_UFO_ADDR *pauiUpdateUFOAddress,
+                                      IMG_UINT32 *paui32UpdateValue,
+                                      IMG_UINT32 ui32CmdSize,
+                                      IMG_PBYTE pui8DMCmd,
+                                      PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                                      PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                                      PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                                      RGXFWIF_CCB_CMD_TYPE eType,
+                                      IMG_UINT32 ui32ExtJobRef,
+                                      IMG_UINT32 ui32IntJobRef,
+                                      IMG_UINT32 ui32PDumpFlags,
+                                      RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                                      IMG_CHAR *pszCommandName,
+                                      IMG_BOOL bCCBStateOpen,
+                                      RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void RGXCmdHelperInitCmdCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+							RGX_CLIENT_CCB          *psClientCCB,
+                            IMG_UINT64              ui64FBSCEntryMask,
+                            IMG_UINT32              ui32ClientFenceCount,
+                            PRGXFWIF_UFO_ADDR       *pauiFenceUFOAddress,
+                            IMG_UINT32              *paui32FenceValue,
+                            IMG_UINT32              ui32ClientUpdateCount,
+                            PRGXFWIF_UFO_ADDR       *pauiUpdateUFOAddress,
+                            IMG_UINT32              *paui32UpdateValue,
+                            IMG_UINT32              ui32CmdSize,
+                            IMG_UINT8               *pui8DMCmd,
+                            PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                            PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                            PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                            RGXFWIF_CCB_CMD_TYPE    eType,
+                            IMG_UINT32              ui32ExtJobRef,
+                            IMG_UINT32              ui32IntJobRef,
+                            IMG_UINT32              ui32PDumpFlags,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                            IMG_CHAR                *pszCommandName,
+                            IMG_BOOL                bCCBStateOpen,
+                            RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+PVRSRV_ERROR RGXCmdHelperAcquireCmdCCB(IMG_UINT32 ui32CmdCount,
+									   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData);
+
+void RGXCmdHelperReleaseCmdCCB(IMG_UINT32 ui32CmdCount,
+							   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+							   const IMG_CHAR *pcszDMName,
+							   IMG_UINT32 ui32CtxAddr);
+
+IMG_UINT32 RGXCmdHelperGetCommandSize(IMG_UINT32 ui32CmdCount,
+								   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData);
+
+IMG_UINT32 RGXCmdHelperGetCommandOffset(RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+                                        IMG_UINT32              ui32Cmdindex);
+
+IMG_UINT32 RGXCmdHelperGetDMCommandHeaderOffset(RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void DumpFirstCCBCmd(PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+				RGX_CLIENT_CCB  *psCurrentClientCCB,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile);
+
+void DumpCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+			PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+			RGX_CLIENT_CCB *psCurrentClientCCB,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile);
+
+PVRSRV_ERROR CheckForStalledCCB(PVRSRV_DEVICE_NODE *psDevNode, RGX_CLIENT_CCB  *psCurrentClientCCB, RGX_KICK_TYPE_DM eKickTypeDM);
+
+void DumpStalledContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif /* RGXCCB_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.c
new file mode 100644
index 000000000000..65ff0cf76d53
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.c
@@ -0,0 +1,1872 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Compute routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Compute routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvkm.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxcompute.h"
+#include "rgx_bvnc_defs_km.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "htbserver.h"
+#include "rgxshader.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_memallocflags.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_CMP_UFO_DUMP	0
+
+//#define CMP_CHECKPOINT_DEBUG 1
+
+#if defined(CMP_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+struct _RGX_SERVER_COMPUTE_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	DEVMEM_MEMDESC				*psFWComputeContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC				*psFWComputeContextStateMemDesc;
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListFence;
+	SYNC_ADDR_LIST				sSyncAddrListUpdate;
+	POS_LOCK					hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+PVRSRV_ERROR PVRSRVRGXCreateComputeContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE			*psDeviceNode,
+											 IMG_INT32					i32Priority,
+											 IMG_UINT32					ui32FrameworkCommandSize,
+											 IMG_PBYTE					pbyFrameworkCommand,
+											 IMG_HANDLE					hMemCtxPrivData,
+											 IMG_UINT32					ui32StaticComputeContextStateSize,
+											 IMG_PBYTE					pStaticComputeContextState,
+											 IMG_UINT32					ui32PackedCCBSizeU88,
+											 IMG_UINT32					ui32ContextFlags,
+											 IMG_UINT64					ui64RobustnessAddress,
+											 IMG_UINT32					ui32MaxDeadlineMS,
+											 RGX_SERVER_COMPUTE_CONTEXT	**ppsComputeContext)
+{
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext;
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+	RGXFWIF_FWCOMPUTECONTEXT	*psFWComputeContext;
+	IMG_UINT32					ui32CCBAllocSizeLog2, ui32CCBMaxAllocSizeLog2;
+
+	/* Prepare cleanup struct */
+	*ppsComputeContext = NULL;
+
+	psComputeContext = OSAllocZMem(sizeof(*psComputeContext));
+	if (psComputeContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW compute context, this has the CDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWCOMPUTECONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwComputeContext",
+			&psComputeContext->psFWComputeContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwcomputecontext;
+	}
+
+	eError = OSLockCreate(&psComputeContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	psComputeContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware compute context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_COMPUTECTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwComputeContextState",
+							  &psComputeContext->psFWComputeContextStateMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate firmware GPU context suspend state (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_contextsuspendalloc;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitCompute(psDevInfo, &psComputeContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psComputeContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psComputeContext->psFWFrameworkMemDesc,
+				pbyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psComputeContext->psFWFrameworkMemDesc;
+	}
+
+	ui32CCBAllocSizeLog2 = U32toU8_Unpack1(ui32PackedCCBSizeU88);
+	ui32CCBMaxAllocSizeLog2 = U32toU8_Unpack2(ui32PackedCCBSizeU88);
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_CDM,
+									 RGXFWIF_DM_CDM,
+									 hMemCtxPrivData,
+									 psComputeContext->psFWComputeContextMemDesc,
+									 offsetof(RGXFWIF_FWCOMPUTECONTEXT, sCDMContext),
+									 psFWMemContextMemDesc,
+									 psComputeContext->psFWComputeContextStateMemDesc,
+									 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_CDM_CCB_SIZE_LOG2,
+									 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_CDM_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 i32Priority,
+									 ui32MaxDeadlineMS,
+									 ui64RobustnessAddress,
+									 &sInfo,
+									 &psComputeContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc,
+			(void **)&psFWComputeContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	OSDeviceMemCopy(&psFWComputeContext->sStaticComputeContextState, pStaticComputeContextState, ui32StaticComputeContextStateSize);
+	DevmemPDumpLoadMem(psComputeContext->psFWComputeContextMemDesc, 0, sizeof(RGXFWIF_FWCOMPUTECONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpValue(psFWComputeContext->sStaticComputeContextState, FLUSH);
+	DevmemReleaseCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psComputeContext->psBufferSyncContext =
+			pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+					"rogue-cdm");
+	if (IS_ERR(psComputeContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to create buffer_sync context (err=%ld)",
+				__func__, PTR_ERR(psComputeContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psComputeContext->sSyncAddrListFence);
+	SyncAddrListInit(&psComputeContext->sSyncAddrListUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hComputeCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sComputeCtxtListHead), &(psComputeContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hComputeCtxListLock);
+	}
+
+	*ppsComputeContext = psComputeContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	FWCommonContextFree(psComputeContext->psServerCommonContext);
+fail_contextalloc:
+fail_frameworkcopy:
+	if (psComputeContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextStateMemDesc);
+fail_contextsuspendalloc:
+	OSLockDestroy(psComputeContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextMemDesc);
+fail_fwcomputecontext:
+	OSFreeMem(psComputeContext);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyComputeContextKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext)
+{
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psComputeContext->psDeviceNode,
+											  psComputeContext->psServerCommonContext,
+											  RGXFWIF_DM_CDM,
+											  PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psComputeContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* remove after RGXFWRequestCommonContextCleanUp() because we might return
+	 * RETRY and don't want to be calling this twice */
+	if (psComputeContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psComputeContext->psBufferSyncContext);
+		psComputeContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_FWCOMPUTECONTEXT	*psFWComputeContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc,
+				(void **)&psFWComputeContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware compute context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+		RGXFwSharedMemCacheOpValue(psFWComputeContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+		ui32WorkEstCCBSubmitted = psFWComputeContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psComputeContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psComputeContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+	/* ... it has so we can free its resources */
+
+	OSWRLockAcquireWrite(psDevInfo->hComputeCtxListLock);
+	dllist_remove_node(&(psComputeContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hComputeCtxListLock);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		WorkEstDeInitCompute(psDevInfo, &psComputeContext->sWorkEstData);
+	}
+#endif
+
+	SyncAddrListDeinit(&psComputeContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psComputeContext->sSyncAddrListUpdate);
+
+	FWCommonContextFree(psComputeContext->psServerCommonContext);
+	psComputeContext->psServerCommonContext = NULL;
+
+	if (psComputeContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWFrameworkMemDesc);
+	}
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextStateMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextMemDesc);
+
+	OSLockDestroy(psComputeContext->hLock);
+	OSFreeMem(psComputeContext);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXKickCDMKM(RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext,
+								IMG_UINT32					ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK		**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								IMG_UINT32					*paui32ClientUpdateValue,
+								PVRSRV_FENCE				iCheckFence,
+								PVRSRV_TIMELINE				iUpdateTimeline,
+								PVRSRV_FENCE				*piUpdateFence,
+								IMG_CHAR					pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								PVRSRV_FENCE				iExportFenceToSignal,
+								IMG_UINT32					ui32CmdSize,
+								IMG_PBYTE					pui8DMCmd,
+								IMG_UINT32					ui32PDumpFlags,
+								IMG_UINT32					ui32ExtJobRef,
+								IMG_UINT32					ui32SyncPMRCount,
+								IMG_UINT32					*paui32SyncPMRFlags,
+								PMR							**ppsSyncPMRs,
+								IMG_UINT32					ui32NumWorkgroups,
+								IMG_UINT32					ui32NumWorkitems,
+								IMG_UINT64					ui64DeadlineInus,
+								IMG_PUINT32					pui32IntJobRef)
+{
+	RGXFWIF_KCCB_CMD		sCmpKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CDMCmdOffset = 0;
+	PVRSRV_RGXDEV_INFO      *psDevInfo = FWCommonContextGetRGXDevInfo(psComputeContext->psServerCommonContext);
+	RGX_CLIENT_CCB          *psClientCCB = FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext);
+	IMG_UINT32              ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32				ui32FWCtx;
+	IMG_BOOL				bCCBStateOpen = IMG_FALSE;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataCompute = {0};
+	IMG_UINT32 ui32CDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32CDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32CDMCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT64 ui64FBSCEntryMask = 0;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64 uiCheckFenceUID = 0;
+	IMG_UINT64 uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT psExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	CMD_COMMON *psComputeCmdCmn = IMG_OFFSET_ADDR(pui8DMCmd, 0);
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				"iUpdateTimeline=%d but piUpdateFence is NULL - PVRSRV_ERROR_INVALID_PARAMS",
+				__func__, iUpdateTimeline));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (pui32IntJobRef)
+	{
+		*pui32IntJobRef = ui32IntJobRef;
+	}
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	ui32IntClientUpdateCount = ui32ClientUpdateCount;
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListUpdate,
+									ui32ClientUpdateCount,
+									pauiClientUpdateUFODevVarBlock,
+									paui32ClientUpdateSyncOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount)
+	{
+		pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+	if (ui32SyncPMRCount != 0)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling "
+		          "pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psComputeContext->psBufferSyncContext,
+		    psComputeContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%s)",
+				        __func__, PVRSRVGetErrorString(eError)));
+			}
+
+			goto fail_resolve_input_fence;
+		}
+
+		/* Append buffer sync fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints "
+			          "to CDM Fence (&psTransferContext->sSyncAddrListFence=<%p>, "
+			          "pauiIntFenceUFOAddress=<%p>)...", __func__,
+			          ui32BufferFenceSyncCheckpointCount,
+			          (void *) &psComputeContext->sSyncAddrListFence ,
+			          (void *) pauiIntFenceUFOAddress));
+
+			SyncAddrListAppendAndDeRefCheckpoints(&psComputeContext->sSyncAddrListFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (pauiIntFenceUFOAddress == NULL)
+			{
+				pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		/* Append the update (from output fence) */
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+			                              1, &psBufferUpdateSyncCheckpoint);
+			if (pauiIntUpdateUFOAddress == NULL)
+			{
+				pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+		}
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+		PVR_UNREFERENCED_PARAMETER(ppsSyncPMRs);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers",
+		        __func__, ui32SyncPMRCount));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_populate_sync_addr_list;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										iCheckFence,
+										&ui32FenceSyncCheckpointCount,
+										&apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (%s)", __func__, PVRSRVGetErrorString(eError)));
+		goto fail_free_buffer_sync_data;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, iCheckFence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, ii, (void*)psNextCheckpoint, SyncCheckpointGetFirmwareAddr(psNextCheckpoint)));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psComputeContext->psDeviceNode,
+		                                   pszUpdateFenceName,
+										   iUpdateTimeline,
+										   psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s)", __func__, PVRSRVGetErrorString(eError)));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: iExportFenceToSignal=%d", __func__, iExportFenceToSignal));
+		/* Resolve the iExportFenceToSignal (if required) */
+		if (iExportFenceToSignal != PVRSRV_NO_FENCE)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+			eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+			                                          psComputeContext->psDeviceNode->hSyncCheckpointContext,
+			                                          &psExportFenceSyncCheckpoint,
+			                                          ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), psExportFenceSyncCheckpoint));
+				goto fail_resolve_export_fence;
+			}
+
+			/* Check that the export fence was not also included as part of the
+			 * check fence (which is an error and would lead to a stalled kick).
+			 */
+			if (ui32FenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Checking export fence is not part of check fence...", __func__));
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   ui32FenceSyncCheckpointCount=%d",
+						   __func__, ui32FenceSyncCheckpointCount));
+				if (ui32FenceSyncCheckpointCount > 0)
+				{
+					IMG_UINT32 iii;
+
+					for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+						if (apsFenceSyncCheckpoints[iii] == psExportFenceSyncCheckpoint)
+						{
+							CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, psExportFenceSyncCheckpoint));
+							eError = PVRSRV_ERROR_INVALID_PARAMS;
+							PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+							goto fail_check_fence_includes_export_fence;
+						}
+					}
+				}
+			}
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			if (paui32IntUpdateValue)
+			{
+				OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+			}
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the compute context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the compute context update list */
+			SyncAddrListAppendSyncPrim(&psComputeContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Compute CDM Fence (&psComputeContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psComputeContext->sSyncAddrListFence));
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			IMG_UINT32 iii;
+
+			for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+			}
+		}
+#endif
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32IntClientUpdateCount > 0)
+	{
+		IMG_UINT32 iii;
+		IMG_UINT32 *pui32Tmp = (IMG_UINT32*)paui32IntUpdateValue;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Dumping %d update values (paui32IntUpdateValue=<%p>)...", __func__, ui32IntClientUpdateCount, (void*)paui32IntUpdateValue));
+		for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: paui32IntUpdateValue[%d] = <%p>", __func__, iii, (void*)pui32Tmp));
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: *paui32IntUpdateValue[%d] = 0x%x", __func__, iii, *pui32Tmp));
+			pui32Tmp++;
+		}
+	}
+#endif
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+
+	if (psExportFenceSyncCheckpoint)
+	{
+		/* Append the update (from export fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psExportFenceSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psExportFenceSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_CMP_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping Compute (CDM) fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Compute (CDM) fence syncs (&psComputeContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psComputeContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Compute (CDM) update syncs (&psComputeContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psComputeContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+	/*
+	 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	 * in other words, take the value and set it to zero afterwards.
+	 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	 * as it must be ready at the time of context activation.
+	 */
+	{
+		eError = RGXExtractFBSCEntryMaskFromMMUContext(psComputeContext->psDeviceNode,
+													   FWCommonContextGetServerMMUCtx(psComputeContext->psServerCommonContext),
+													   &ui64FBSCEntryMask);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%s)", PVRSRVGetErrorString(eError)));
+			goto fail_cmdinvalfbsc;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psComputeContext->psDeviceNode))
+	{
+		sWorkloadCharacteristics.sCompute.ui32NumberOfWorkgroups = ui32NumWorkgroups;
+		sWorkloadCharacteristics.sCompute.ui32NumberOfWorkitems  = ui32NumWorkitems;
+
+		/* Prepare workload estimation */
+		WorkEstPrepare(psComputeContext->psDeviceNode->pvDevice,
+				&psComputeContext->sWorkEstData,
+				&psComputeContext->sWorkEstData.uWorkloadMatchingData.sCompute.sDataCDM,
+				RGXFWIF_CCB_CMD_TYPE_CDM,
+				&sWorkloadCharacteristics,
+				ui64DeadlineInus,
+				&sWorkloadKickDataCompute);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32NumWorkgroups);
+	PVR_UNREFERENCED_PARAMETER(ui32NumWorkitems);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psComputeContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       ui64FBSCEntryMask,
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+	                       &pPreAddr,
+	                       &pPostAddr,
+	                       &pRMWUFOAddr,
+	                       RGXFWIF_CCB_CMD_TYPE_CDM,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	                       &sWorkloadKickDataCompute,
+#else
+	                       NULL,
+#endif
+	                       "Compute",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+
+	/*
+		We should reserve space in the kernel CCB here and fill in the command
+		directly.
+		This is so if there isn't space in the kernel CCB we can return with
+		retry back to services client before we take any operations
+	*/
+
+	/*
+		We might only be kicking for flush out a padding packet so only submit
+		the command if the create was successful
+	*/
+
+	/*
+	  All the required resources are ready at this point, we can't fail so
+	  take the required server sync operations and commit all the resources
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	ui32CDMCmdOffset = RGXGetHostWriteOffsetCCB(psClientCCB);
+	RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "CDM", FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+	{
+		/* The following is used to determine the offset of the command header containing
+		   the workload estimation data so that can be accessed when the KCCB is read */
+		ui32CDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(asCmdHelperData);
+
+		ui32CDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32CDMCmdOffset < ui32CDMCmdOffsetWrapCheck)
+		{
+			ui32CDMWorkloadDataRO = ui32CDMCmdOffset;
+		}
+		else
+		{
+			ui32CDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/* Construct the kernel compute CCB command. */
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sCmpKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+	{
+		/* Store the offset to the CCCB command header so that it can be referenced
+		 * when the KCCB command reaches the FW */
+		sCmpKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32CDMWorkloadDataRO + ui32CDMCmdHeaderOffset;
+	}
+#endif
+
+	ui32FWCtx = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr;
+
+	if (psComputeCmdCmn)
+	{
+		HTBLOGK(HTB_SF_MAIN_KICK_CDM,
+				sCmpKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui32CDMCmdOffset,
+				psComputeCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef);
+	}
+
+	RGXSRV_HWPERF_ENQ(psComputeContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_CDM,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	/*
+	 * Submit the compute command to the firmware.
+	 */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandWithoutPowerLock(psComputeContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_CDM,
+									&sCmpKCCBCmd,
+									ui32PDumpFlags);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_schedulecmd;
+	}
+	else
+	{
+		PVRGpuTraceEnqueueEvent(psComputeContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_CDM);
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__,
+		           (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+	if (psExportFenceSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)psExportFenceSyncCheckpoint, SyncCheckpointGetId(psExportFenceSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(psExportFenceSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psExportFenceSyncCheckpoint);
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal=%d)", __func__, iExportFenceToSignal));
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psComputeContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pszUpdateFenceName);
+	}
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_schedulecmd:
+fail_acquirepowerlock:
+fail_cmdacquire:
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+fail_cmdinvalfbsc:
+#endif
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+fail_check_fence_includes_export_fence:
+fail_resolve_export_fence:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+
+fail_free_buffer_sync_data:
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+
+fail_resolve_input_fence:
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendCancelCmdKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext, IMG_UINT32 ui32FirstIntJobRefToCancel, IMG_UINT32 ui32LastIntJobRefToCancel)
+{
+	RGXFWIF_KCCB_CMD sCancelWorkKCCBCmd;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	sCancelWorkKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_CANCEL_WORK;
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.ui32FirstIntJobRefToCancel = ui32FirstIntJobRefToCancel;
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.ui32LastIntJobRefToCancel = ui32LastIntJobRefToCancel;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psComputeContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_CDM,
+		                            &sCancelWorkKCCBCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+
+		/* Iterate if we hit a PVRSRV_ERROR_KERNEL_CCB_FULL error */
+		if ((eError != PVRSRV_ERROR_RETRY) &&
+		    (eError != PVRSRV_ERROR_KERNEL_CCB_FULL))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Sending compute cancel command for context <%p>. Work with IntJobRef below ui32FirstValidIntJobRef=0x%x might be discarded.",
+	    psComputeContext->psServerCommonContext, ui32LastIntJobRefToCancel + 1));
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXFlushComputeDataKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext)
+{
+	RGXFWIF_KCCB_CMD sFlushCmd;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psComputeContext->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Submit Compute flush");
+#endif
+	sFlushCmd.eCmdType = RGXFWIF_KCCB_CMD_SLCFLUSHINVAL;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.bInval = IMG_FALSE;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.bDMContext = IMG_TRUE;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+									RGXFWIF_DM_CDM,
+									&sFlushCmd,
+									PDUMP_FLAGS_CONTINUOUS,
+									&ui32kCCBCommandSlot);
+		/* Iterate if we hit a PVRSRV_ERROR_KERNEL_CCB_FULL error */
+		if ((eError != PVRSRV_ERROR_RETRY) &&
+		    (eError != PVRSRV_ERROR_KERNEL_CCB_FULL))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If we hit a temporary KCCB exhaustion, return a RETRY to caller */
+		if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: Returning RETRY to caller", __func__));
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to schedule SLC flush command (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+		}
+	}
+	else
+	{
+		/* Wait for the SLC flush to complete */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Compute flush aborted (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+			goto error_exit;
+		}
+
+		if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] &
+		                  RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+	}
+
+error_exit:
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(RGX_SERVER_COMPUTE_CONTEXT  *psComputeContext)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+	IMG_UINT32 ui32ControlStreamFormat =
+		RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, CDM_CONTROL_STREAM_FORMAT) ?
+		RGX_GET_FEATURE_VALUE(psDevInfo, CDM_CONTROL_STREAM_FORMAT) : 0;
+
+	if (ui32ControlStreamFormat >= 2U && ui32ControlStreamFormat < 5U)
+	{
+
+		RGXFWIF_KCCB_CMD  sKCCBCmd;
+		PVRSRV_ERROR      eError;
+
+		OSLockAcquire(psComputeContext->hLock);
+
+		/* Schedule the firmware command */
+		sKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE;
+		sKCCBCmd.uCmdData.sWriteOffsetUpdateData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = RGXScheduleCommand(psComputeContext->psDeviceNode->pvDevice,
+										RGXFWIF_DM_CDM,
+										&sKCCBCmd,
+										PDUMP_FLAGS_NONE);
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to schedule the FW command %d (%s)",
+					__func__,
+					eError,
+					PVRSRVGetErrorString(eError)));
+		}
+
+		OSLockRelease(psComputeContext->hLock);
+		return eError;
+	}
+	else
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                  PVRSRV_DEVICE_NODE * psDeviceNode,
+												  RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = ContextSetPriority(psComputeContext->psServerCommonContext,
+								psConnection,
+								psComputeContext->psDeviceNode->pvDevice,
+								i32Priority,
+								RGXFWIF_DM_CDM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the compute context (%s)", __func__, PVRSRVGetErrorString(eError)));
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+/*
+ * PVRSRVRGXSetComputeContextPropertyKM
+ */
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPropertyKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                                  RGX_CONTEXT_PROPERTY eContextProperty,
+                                                  IMG_UINT64 ui64Input,
+                                                  IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psComputeContext->hLock);
+			eError = FWCommonContextSetFlags(psComputeContext->psServerCommonContext,
+			                                 ui32ContextFlags);
+			OSLockRelease(psComputeContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpComputeCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hComputeCtxListLock);
+	dllist_foreach_node(&psDevInfo->sComputeCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMPUTE_CONTEXT *psCurrentServerComputeCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMPUTE_CONTEXT, sListNode);
+		DumpFWCommonContextInfo(psCurrentServerComputeCtx->psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hComputeCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientComputeCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 ui32ContextBitMask = 0;
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hComputeCtxListLock);
+	dllist_foreach_node(&psDevInfo->sComputeCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMPUTE_CONTEXT *psCurrentServerComputeCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMPUTE_CONTEXT, sListNode);
+
+		if (CheckStalledClientCommonContext(psCurrentServerComputeCtx->psServerCommonContext, RGX_KICK_TYPE_DM_CDM)
+			== PVRSRV_ERROR_CCCB_STALLED)
+		{
+			ui32ContextBitMask |= RGX_KICK_TYPE_DM_CDM;
+		}
+	}
+	OSWRLockReleaseRead(psDevInfo->hComputeCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * PVRSRVRGXGetLastDeviceErrorKM
+ */
+PVRSRV_ERROR PVRSRVRGXGetLastDeviceErrorKM(CONNECTION_DATA    *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32         *ui32Error)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	*ui32Error = psDevInfo->eLastDeviceError;
+	psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_NONE;
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXKickTimestampQueryKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTimestampQueryKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                           PVRSRV_FENCE iCheckFence,
+										   PVRSRV_TIMELINE iUpdateTimeline,
+										   PVRSRV_FENCE *piUpdateFence,
+                                           IMG_CHAR pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                           IMG_UINT32 ui32CmdSize,
+                                           IMG_PBYTE pui8DMCmd,
+                                           IMG_UINT32 ui32ExtJobRef)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psComputeContext->psServerCommonContext);
+	RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext);
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	IMG_UINT32 ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32 ui32PDumpFlags = 0;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64 uiCheckFenceUID = 0;
+	IMG_UINT64 uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *paui32ClientUpdateValue = NULL;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+	PVRSRV_ERROR eError;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListUpdate,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount)
+	{
+		pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psComputeContext->psDeviceNode->hSyncCheckpointContext,
+	                                    iCheckFence,
+	                                    &ui32FenceSyncCheckpointCount,
+	                                    &apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (%s)", __func__, PVRSRVGetErrorString(eError)));
+		goto err_populate_sync_addr_list;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, ii, (void*)psNextCheckpoint, SyncCheckpointGetFirmwareAddr(psNextCheckpoint)));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psComputeContext->psDeviceNode,
+		                                   pszUpdateFenceName,
+										   iUpdateTimeline,
+										   psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s)", __func__, PVRSRVGetErrorString(eError)));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			if (paui32IntUpdateValue)
+			{
+				OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+			}
+
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the compute context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the compute context update list */
+			SyncAddrListAppendSyncPrim(&psComputeContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Compute CDM Fence (&psComputeContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psComputeContext->sSyncAddrListFence));
+
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+	}
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       0, /* empty ui64FBSCEntryMask */
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       PDUMP_FLAGS_NONE,
+	                       NULL,
+	                       "VkTimestamp",
+	                       IMG_FALSE, /* bCCBStateOpen */
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXCmdHelperAcquireCmdCCB", fail_cmdacquire);
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "CDM",
+	                          FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr);
+
+	/* Construct the kernel compute CCB command. */
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sCmpKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/*
+	 * Submit the RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP
+	 * command to the firmware.
+	 */
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandWithoutPowerLock(psComputeContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_CDM,
+		                            &sCmpKCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_schedulecmd;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__, (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif /* defined(NO_HARDWARE) */
+
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psComputeContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pszUpdateFenceName);
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned
+	 * by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_schedulecmd:
+fail_acquirepowerlock:
+fail_cmdacquire:
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXCDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXCDMReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxcompute.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.h
new file mode 100644
index 000000000000..b93c42efc044
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.h
@@ -0,0 +1,226 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX compute functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX compute functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXCOMPUTE_H)
+#define RGXCOMPUTE_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+
+
+typedef struct _RGX_SERVER_COMPUTE_CONTEXT_ RGX_SERVER_COMPUTE_CONTEXT;
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXCreateComputeContextKM
+@Description    Creates an RGX device context for submitting commands to CDM.
+@Input          psConnection          Device connection
+@Input          psDeviceNode          Services-managed device
+@Input          i32Priority           Scheduling priority for commands
+                                      on this context
+@Input          ui32FrameworkCommandSize
+                                      Framework command size
+@Input          pabyFrameworkCommand  Pointer to framework command
+@Input          hMemCtxPrivData       Private data
+@Input          ui32StaticComputeContextStateSize
+                                      Size of fixed compute context state
+@Input          pStaticComputeContextState
+                                      Compute context state
+@Input          ui32PackedCCBSizeU88  Packed CCB size. The first byte contains
+                                      the log2 CCB size and the second byte
+                                      the log2 maximum CCB size.
+@Input          ui32ContextFlags      Flags with context properties
+@Input          ui64RobustnessAddress Address for FW to signal a context reset
+@Input          ui32MaxDeadlineMS     Max deadline limit in MS that the
+                                      workload can run
+@Output         ppsComputeContext     Cleanup data
+@Return         PVRSRV_ERROR          Returns PVRSRV_OK or an error.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateComputeContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE			*psDeviceNode,
+											 IMG_INT32					i32Priority,
+											 IMG_UINT32					ui32FrameworkCommandSize,
+											 IMG_PBYTE					pabyFrameworkCommand,
+											 IMG_HANDLE					hMemCtxPrivData,
+											 IMG_UINT32					ui32StaticComputeContextStateSize,
+											 IMG_PBYTE					pStaticComputeContextState,
+											 IMG_UINT32					ui32PackedCCBSizeU88,
+											 IMG_UINT32					ui32ContextFlags,
+											 IMG_UINT64					ui64RobustnessAddress,
+											 IMG_UINT32					ui32MaxDeadlineMS,
+											 RGX_SERVER_COMPUTE_CONTEXT	**ppsComputeContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDestroyComputeContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyComputeContext
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyComputeContextKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXKickCDMKM
+
+ @Description
+	Server-side implementation of RGXKickCDM
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickCDMKM(RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext,
+								IMG_UINT32					ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK		**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								IMG_UINT32					*paui32ClientUpdateValue,
+								PVRSRV_FENCE				iCheckFence,
+								PVRSRV_TIMELINE				iUpdateTimeline,
+								PVRSRV_FENCE				*piUpdateFence,
+								IMG_CHAR					pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								PVRSRV_FENCE				iExportFenceToSignal,
+								IMG_UINT32					ui32CmdSize,
+								IMG_PBYTE					pui8DMCmd,
+								IMG_UINT32					ui32PDumpFlags,
+								IMG_UINT32					ui32ExtJobRef,
+								IMG_UINT32					ui32SyncPMRCount,
+								IMG_UINT32					*paui32SyncPMRFlags,
+								PMR							**ppsSyncPMRs,
+								IMG_UINT32					ui32NumWorkgroups,
+								IMG_UINT32					ui32NumWorkitems,
+								IMG_UINT64					ui64DeadlineInus,
+								IMG_PUINT32					pui32IntJobRef);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXFlushComputeDataKM
+
+ @Description
+	Server-side implementation of RGXFlushComputeData
+
+ @Input psComputeContext - Compute context to flush
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFlushComputeDataKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSendCancelCmd
+
+ @Description
+	Server-side implementation of RGXSendCancelCmd
+
+ @Input psComputeContext - Compute context to cancel work on
+ @Input ui32LastIntJobRefToCancel - Last IntJobRef to cancel
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSendCancelCmdKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                      IMG_UINT32 ui32FirstIntJobRefToCancel,
+                                      IMG_UINT32 ui32LastIntJobRefToCancel);
+
+
+/*!
+*******************************************************************************
+
+ @Function	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM
+ @Description   Server-side implementation of RGXNotifyComputeWriteOffsetUpdate
+
+ @Input         psComputeContext - Compute context to flush
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPriorityKM(CONNECTION_DATA *psConnection,
+												  PVRSRV_DEVICE_NODE *psDeviceNode,
+												  RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPropertyKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  RGX_CONTEXT_PROPERTY eContextProperty,
+												  IMG_UINT64 ui64Input,
+												  IMG_UINT64 *pui64Output);
+
+PVRSRV_ERROR PVRSRVRGXGetLastDeviceErrorKM(CONNECTION_DATA    *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32         *ui32Error);
+
+PVRSRV_ERROR PVRSRVRGXKickTimestampQueryKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                           PVRSRV_FENCE iCheckFence,
+										   PVRSRV_TIMELINE iUpdateTimeline,
+										   PVRSRV_FENCE *piUpdateFence,
+                                           IMG_CHAR pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                           IMG_UINT32 ui32CmdSize,
+                                           IMG_PBYTE pui8DMCmd,
+                                           IMG_UINT32 ui32ExtJobRef);
+
+/* Debug - Dump debug info of compute contexts on this device */
+void DumpComputeCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client compute contexts are stalled */
+IMG_UINT32 CheckForStalledClientComputeCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR PVRSRVRGXCDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+PVRSRV_ERROR PVRSRVRGXCDMReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+#endif /* RGXCOMPUTE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.c
new file mode 100644
index 000000000000..fca648b26f56
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.c
@@ -0,0 +1,3236 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_sf.h"
+#include "rgxfw_log_helper.h"
+#include "fwtrace_string.h"
+#include "rgxmmudefs_km.h"
+#include "rgxinit.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "rgxfwimageutils.h"
+#include "fwload.h"
+#include "rgx_options.h"
+#include "devicemem_history_server.h"
+#include "debug_common.h"
+#include "info_page.h"
+#include "osfunc.h"
+#if defined(SUPPORT_SOC_TIMER)
+#include "rgxtimecorr.h"
+#endif
+
+#define MAX_FW_DESCRIPTION_LENGTH	(600U)
+
+#define PVR_DUMP_FIRMWARE_INFO(x)														\
+	PVR_DUMPDEBUG_LOG("FW info: %d.%d @ %8d (%s) build options: 0x%08x",				\
+						PVRVERSION_UNPACK_MAJ((x).ui32DDKVersion),						\
+						PVRVERSION_UNPACK_MIN((x).ui32DDKVersion),						\
+						(x).ui32DDKBuild,												\
+						((x).ui32BuildOptions & OPTIONS_DEBUG_EN) ? "debug":"release",	\
+						(x).ui32BuildOptions);
+
+#define PVR_DUMP_FIRMWARE_INFO_HDR(x)												\
+	PVR_DUMPDEBUG_LOG("FW info: %d.%d @ %8d (%s) build options: 0x%08x",		\
+						(x).ui16PVRVersionMajor,								\
+						(x).ui16PVRVersionMinor,								\
+						(x).ui32PVRVersionBuild,								\
+						((x).ui32Flags & OPTIONS_DEBUG_EN) ? "debug":"release",	\
+						(x).ui32Flags);
+
+typedef struct {
+	IMG_UINT16     ui16Mask;
+	const IMG_CHAR *pszStr;
+} RGXFWT_DEBUG_INFO_MSKSTR; /* pair of bit mask and debug info message string */
+
+/*
+ *  Array of all the Firmware Trace log IDs used to convert the trace data.
+ */
+typedef struct _TRACEBUF_LOG_ {
+	RGXFW_LOG_SFids	eSFId;
+	const IMG_CHAR	*pszName;
+	const IMG_CHAR	*pszFmt;
+	IMG_UINT32		ui32ArgNum;
+} TRACEBUF_LOG;
+
+static const TRACEBUF_LOG aLogDefinitions[] =
+{
+#define X(a, b, c, d, e) {RGXFW_LOG_CREATESFID(a,b,e), #c, d, e},
+	RGXFW_LOG_SFIDLIST
+#undef X
+};
+
+static const IMG_FLAGS2DESC asCswOpts2Description[] =
+{
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST, " Fast CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM, " Medium CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW, " Slow CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY, " No Delay CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_MODE_RAND, " Random Csw enabled;"},
+	{RGXFWIF_INICFG_CTXSWITCH_SRESET_EN, " SoftReset;"},
+};
+
+static const IMG_FLAGS2DESC asMisc2Description[] =
+{
+	{RGXFWIF_INICFG_POW_RASCALDUST, " Power Rascal/Dust;"},
+	{RGXFWIF_INICFG_SPU_CLOCK_GATE, " SPU Clock Gating (requires Power Rascal/Dust);"},
+	{RGXFWIF_INICFG_HWPERF_EN, " HwPerf EN;"},
+	{RGXFWIF_INICFG_FBCDC_V3_1_EN, " FBCDCv3.1;"},
+	{RGXFWIF_INICFG_CHECK_MLIST_EN, " Check MList;"},
+	{RGXFWIF_INICFG_DISABLE_CLKGATING_EN, " ClockGating Off;"},
+	{RGXFWIF_INICFG_REGCONFIG_EN, " Register Config;"},
+	{RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY, " Assert on OOM;"},
+	{RGXFWIF_INICFG_HWP_DISABLE_FILTER, " HWP Filter Off;"},
+	{RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN, " CDM Random kill;"},
+	{RGXFWIF_INICFG_DISABLE_DM_OVERLAP, " DM Overlap Off;"},
+	{RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER, " Assert on HWR;"},
+	{RGXFWIF_INICFG_INJECT_ICS_FAULT, " Inject ICS Fault;"},
+	{RGXFWIF_INICFG_VALIDATE_IRQ, " Validate IRQ;"},
+	{RGXFWIF_INICFG_DISABLE_PDP_EN, " PDUMP Panic off;"},
+	{RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN, " SPU Pow mask change on;"},
+	{RGXFWIF_INICFG_WORKEST, " Workload Estim;"},
+	{RGXFWIF_INICFG_PDVFS, " PDVFS;"},
+	{RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP, " ISP v1 scheduling;"},
+	{RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP, " ISP v2 scheduling;"},
+	{RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER, " Validate SOC&USC timers;"},
+};
+
+static const IMG_FLAGS2DESC asFwOsCfg2Description[] =
+{
+	{RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN, " TDM;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN, " GEOM;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN, " 3D;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN, " CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN, " RDM;"},
+#endif
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM, " LowPrio TDM;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM, " LowPrio GEOM;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D, " LowPrio 3D;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM, " LowPrio CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM, " LowPrio RDM;"},
+#endif
+#if defined(SUPPORT_ICS)
+	{RGXFWIF_INICFG_OS_ICS_TDM_EN, " TDM;"},
+	{RGXFWIF_INICFG_OS_ICS_GEOM_EN, " GEOM;"},
+	{RGXFWIF_INICFG_OS_ICS_3D_EN, " 3D;"},
+	{RGXFWIF_INICFG_OS_ICS_CDM_EN, " CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_ICS_RDM_EN, " RDM;"},
+#endif
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_LONG, " Long FDTI profile;"},
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_MEDIUM, " Medium FDTI profile;"},
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_SHORT, " Short FDTI profile;"},
+#endif
+};
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+const IMG_CHAR * const gapszMipsPermissionPTFlags[4] =
+{
+	"    ",
+	"XI  ",
+	"RI  ",
+	"RIXI"
+};
+
+const IMG_CHAR * const gapszMipsCoherencyPTFlags[8] =
+{
+	"C",
+	"C",
+	" ",
+	"C",
+	"C",
+	"C",
+	"C",
+	" "
+};
+
+const IMG_CHAR * const gapszMipsDirtyGlobalValidPTFlags[8] =
+{
+	"   ",
+	"  G",
+	" V ",
+	" VG",
+	"D  ",
+	"D G",
+	"DV ",
+	"DVG"
+};
+
+#if !defined(NO_HARDWARE)
+/* Translation of MIPS exception encoding */
+typedef struct _MIPS_EXCEPTION_ENCODING_
+{
+	const IMG_CHAR *const pszStr;	/* Error type */
+	const IMG_BOOL bIsFatal;	/* Error is fatal or non-fatal */
+} MIPS_EXCEPTION_ENCODING;
+
+static const MIPS_EXCEPTION_ENCODING apsMIPSExcCodes[] =
+{
+	{"Interrupt", IMG_FALSE},
+	{"TLB modified exception", IMG_FALSE},
+	{"TLB exception (load/instruction fetch)", IMG_FALSE},
+	{"TLB exception (store)", IMG_FALSE},
+	{"Address error exception (load/instruction fetch)", IMG_TRUE},
+	{"Address error exception (store)", IMG_TRUE},
+	{"Bus error exception (instruction fetch)", IMG_TRUE},
+	{"Bus error exception (load/store)", IMG_TRUE},
+	{"Syscall exception", IMG_FALSE},
+	{"Breakpoint exception (FW assert)", IMG_FALSE},
+	{"Reserved instruction exception", IMG_TRUE},
+	{"Coprocessor Unusable exception", IMG_FALSE},
+	{"Arithmetic Overflow exception", IMG_FALSE},
+	{"Trap exception", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"Implementation-Specific Exception 1 (COP2)", IMG_FALSE},
+	{"CorExtend Unusable", IMG_FALSE},
+	{"Coprocessor 2 exceptions", IMG_FALSE},
+	{"TLB Read-Inhibit", IMG_TRUE},
+	{"TLB Execute-Inhibit", IMG_TRUE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"Reference to WatchHi/WatchLo address", IMG_FALSE},
+	{"Machine check", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"DSP Module State Disabled exception", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	/* Can only happen in MIPS debug mode */
+	{"Parity error", IMG_FALSE},
+	{NULL, IMG_FALSE}
+};
+
+static IMG_CHAR const *_GetMIPSExcString(IMG_UINT32 ui32ExcCode)
+{
+	if (ui32ExcCode >= sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Only %lu exceptions available in MIPS, %u is not a valid exception code",
+		         (unsigned long)sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING), ui32ExcCode));
+		return NULL;
+	}
+
+	return apsMIPSExcCodes[ui32ExcCode].pszStr;
+}
+#endif
+
+typedef struct _RGXMIPSFW_C0_DEBUG_TBL_ENTRY_
+{
+    IMG_UINT32 ui32Mask;
+    const IMG_CHAR * pszExplanation;
+} RGXMIPSFW_C0_DEBUG_TBL_ENTRY;
+
+#if !defined(NO_HARDWARE)
+static const RGXMIPSFW_C0_DEBUG_TBL_ENTRY sMIPS_C0_DebugTable[] =
+{
+    { RGXMIPSFW_C0_DEBUG_DSS,      "Debug single-step exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DBP,      "Debug software breakpoint exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBL,     "Debug data break exception occurred on a load" },
+    { RGXMIPSFW_C0_DEBUG_DDBS,     "Debug data break exception occurred on a store" },
+    { RGXMIPSFW_C0_DEBUG_DIB,      "Debug instruction break exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DINT,     "Debug interrupt exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DIBIMPR,  "Imprecise debug instruction break exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBLIMPR, "Imprecise debug data break load exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBSIMPR, "Imprecise debug data break store exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_IEXI,     "Imprecise error exception inhibit controls exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DBUSEP,   "Data access Bus Error exception pending" },
+    { RGXMIPSFW_C0_DEBUG_CACHEEP,  "Imprecise Cache Error pending" },
+    { RGXMIPSFW_C0_DEBUG_MCHECKP,  "Imprecise Machine Check exception pending" },
+    { RGXMIPSFW_C0_DEBUG_IBUSEP,   "Instruction fetch Bus Error exception pending" },
+    { (IMG_UINT32)RGXMIPSFW_C0_DEBUG_DBD,      "Debug exception occurred in branch delay slot" }
+};
+#endif
+#endif
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)*/
+
+#define NARGS_MASK ~(0xF<<16)
+static IMG_BOOL _FirmwareTraceIntegrityCheck(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile)
+{
+	const TRACEBUF_LOG *psLogDef = &aLogDefinitions[0];
+	IMG_BOOL bIntegrityOk = IMG_TRUE;
+
+	/*
+	 * For every log ID, check the format string and number of arguments is valid.
+	 */
+	while (psLogDef->eSFId != RGXFW_SF_LAST)
+	{
+		const TRACEBUF_LOG *psLogDef2;
+		const IMG_CHAR *pszString;
+		IMG_UINT32 ui32Count;
+
+		/*
+		 * Check the number of arguments matches the number of '%' in the string and
+		 * check that no string uses %s which is not supported as it requires a
+		 * pointer to memory that is not going to be valid.
+		 */
+		pszString = psLogDef->pszFmt;
+		ui32Count = 0;
+
+		while (*pszString != '\0')
+		{
+			if (*pszString++ == '%')
+			{
+				ui32Count++;
+				if (*pszString == 's')
+				{
+					bIntegrityOk = IMG_FALSE;
+					PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has an unsupported type not recognized (fmt: %%%c). Please fix.",
+									  psLogDef->pszName, *pszString);
+				}
+				else if (*pszString == '%')
+				{
+					/* Double % is a printable % sign and not a format string... */
+					ui32Count--;
+					pszString++;
+				}
+			}
+		}
+
+		if (ui32Count != psLogDef->ui32ArgNum)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has %d arguments but only %d are specified. Please fix.",
+			                  psLogDef->pszName, ui32Count, psLogDef->ui32ArgNum);
+		}
+
+		/* RGXDumpFirmwareTrace() has a hardcoded limit of supporting up to 20 arguments... */
+		if (ui32Count > 20)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has %d arguments but a maximum of 20 are supported. Please fix.",
+			                  psLogDef->pszName, ui32Count);
+		}
+
+		/* Check the id number is unique (don't take into account the number of arguments) */
+		ui32Count = 0;
+		psLogDef2 = &aLogDefinitions[0];
+
+		while (psLogDef2->eSFId != RGXFW_SF_LAST)
+		{
+			if ((psLogDef->eSFId & NARGS_MASK) == (psLogDef2->eSFId & NARGS_MASK))
+			{
+				ui32Count++;
+			}
+			psLogDef2++;
+		}
+
+		if (ui32Count != 1)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s id %x is not unique, there are %d more. Please fix.",
+			                  psLogDef->pszName, psLogDef->eSFId, ui32Count - 1);
+		}
+
+		/* Move to the next log ID... */
+		psLogDef++;
+	}
+
+	return bIntegrityOk;
+}
+
+void RGXDumpFirmwareTrace(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO  *psDevInfo)
+{
+	RGXFWIF_TRACEBUF  *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	static IMG_BOOL   bIntegrityCheckPassed = IMG_FALSE;
+
+	/* Check that the firmware trace is correctly defined... */
+	if (!bIntegrityCheckPassed)
+	{
+		bIntegrityCheckPassed = _FirmwareTraceIntegrityCheck(pfnDumpDebugPrintf, pvDumpDebugFile);
+		if (!bIntegrityCheckPassed)
+		{
+			return;
+		}
+	}
+
+	/* Dump FW trace information... */
+	if (psRGXFWIfTraceBufCtl != NULL)
+	{
+		IMG_UINT32  tid;
+
+		PVR_DUMPDEBUG_LOG("Device ID: %u", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+
+		/* Print the log type settings... */
+		if (psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+		{
+			PVR_DUMPDEBUG_LOG("Debug log type: %s ( " RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC ")",
+							  ((psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)?("trace"):("tbi")),
+							  RGXFWIF_LOG_ENABLED_GROUPS_LIST(psRGXFWIfTraceBufCtl->ui32LogType)
+							  );
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("Debug log type: none");
+		}
+
+		/* Print the decoded log for each thread... */
+		for (tid = 0;  tid < RGXFW_THREAD_NUM;  tid++)
+		{
+			RGXDumpFirmwareTraceDecoded(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPrepareExtraDebugInfo
+
+ @Description
+
+ Prepares debug info string by decoding ui16DebugInfo value passed
+
+ @Input pszBuffer	 - pointer to debug info string buffer
+
+ @Return   void
+
+******************************************************************************/
+static void RGXPrepareExtraDebugInfo(IMG_CHAR *pszBuffer, IMG_UINT32 ui32BufferSize, IMG_UINT16 ui16DebugInfo)
+{
+	const RGXFWT_DEBUG_INFO_MSKSTR aDebugInfoMskStr[] =
+	{
+#define X(a, b) {a, b},
+		RGXFWT_DEBUG_INFO_MSKSTRLIST
+#undef X
+	};
+
+	IMG_UINT32 ui32NumFields = sizeof(aDebugInfoMskStr)/sizeof(RGXFWT_DEBUG_INFO_MSKSTR);
+	IMG_UINT32 i;
+	IMG_BOOL   bHasExtraDebugInfo = IMG_FALSE;
+
+	/* Add prepend string */
+	OSStringSafeCopy(pszBuffer, RGXFWT_DEBUG_INFO_STR_PREPEND, ui32BufferSize);
+
+	/* Add debug info strings */
+	for (i = 0; i < ui32NumFields; i++)
+	{
+		if (ui16DebugInfo & aDebugInfoMskStr[i].ui16Mask)
+		{
+			if (bHasExtraDebugInfo)
+			{
+				OSStringLCat(pszBuffer, ", ", ui32BufferSize); /* Add comma separator */
+			}
+			OSStringLCat(pszBuffer, aDebugInfoMskStr[i].pszStr, ui32BufferSize);
+			bHasExtraDebugInfo = IMG_TRUE;
+		}
+	}
+
+	/* Add append string */
+	OSStringLCat(pszBuffer, RGXFWT_DEBUG_INFO_STR_APPEND, ui32BufferSize);
+}
+
+#define PVR_MAX_DEBUG_PARTIAL_LINES (40U)
+#define PVR_DUMPDEBUG_LOG_LINES(fmt, ...) \
+	if (!bPrintAllLines) { \
+		OSSNPrintf(&pszLineBuffer[ui32LastLineIdx * PVR_MAX_DEBUG_MESSAGE_LEN], PVR_MAX_DEBUG_MESSAGE_LEN, (fmt), ##__VA_ARGS__); \
+		ui32LineCount++; \
+		ui32LastLineIdx = ui32LineCount % PVR_MAX_DEBUG_PARTIAL_LINES; \
+	} else { \
+		PVR_UNREFERENCED_PARAMETER(pszLineBuffer); \
+		PVR_UNREFERENCED_PARAMETER(ui32LineCount); \
+		PVR_UNREFERENCED_PARAMETER(ui32LastLineIdx); \
+		PVR_DUMPDEBUG_LOG((fmt), ##__VA_ARGS__); \
+	}
+
+static void RGXDumpFirmwareTraceLines(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID,
+				bool bPrintAllLines)
+{
+	volatile IMG_UINT32  *pui32FWWrapCount;
+	volatile IMG_UINT32  *pui32FWTracePtr;
+	IMG_UINT32           *pui32TraceBuf;
+	IMG_UINT32           *pui32LocalTraceBuf = NULL;
+	IMG_UINT32           ui32HostWrapCount;
+	IMG_UINT32           ui32HostTracePtr;
+	IMG_UINT32           ui32Count           = 0;
+	IMG_UINT32           ui32LineCount       = 0;
+	IMG_UINT32           ui32LastLineIdx     = 0;
+	IMG_CHAR             *pszLineBuffer      = NULL;
+	IMG_UINT32           ui32TraceBufSizeInDWords;
+
+	RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID], INVALIDATE);
+	RGXFwSharedMemCacheOpExec(psDevInfo->apui32TraceBuffer[ui32TID],
+	                          psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	pui32FWWrapCount = &(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].ui32WrapCount);
+	pui32FWTracePtr  = &(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].ui32TracePointer);
+	pui32TraceBuf    = psDevInfo->apui32TraceBuffer[ui32TID];
+	ui32HostWrapCount = *pui32FWWrapCount;
+	ui32HostTracePtr  = *pui32FWTracePtr;
+
+	if (pui32TraceBuf == NULL)
+	{
+		/* trace buffer not yet allocated */
+		return;
+	}
+
+	if (!bPrintAllLines)
+	{
+		pszLineBuffer = OSAllocMem(PVR_MAX_DEBUG_MESSAGE_LEN * PVR_MAX_DEBUG_PARTIAL_LINES);
+		PVR_LOG_RETURN_VOID_IF_FALSE(pszLineBuffer != NULL, "pszLineBuffer alloc failed");
+	}
+
+	ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+	{
+		PVR_DUMPDEBUG_LOG_LINES("WARNING: Trace pointer (%d) greater than buffer size (%d).",
+								ui32HostTracePtr, ui32TraceBufSizeInDWords);
+		ui32HostTracePtr %= ui32TraceBufSizeInDWords;
+	}
+
+	/*
+	 *  Allocate a local copy of the trace buffer which will contain a static non-changing
+	 *  snapshot view of the buffer. This removes the issue of a fast GPU wrapping and
+	 *  overwriting the tail data of the buffer.
+	 */
+	pui32LocalTraceBuf = OSAllocMem(ui32TraceBufSizeInDWords * sizeof(IMG_UINT32));
+	if (pui32LocalTraceBuf != NULL)
+	{
+		memcpy(pui32LocalTraceBuf, pui32TraceBuf, ui32TraceBufSizeInDWords * sizeof(IMG_UINT32));
+		ui32HostTracePtr = *pui32FWTracePtr;
+		pui32TraceBuf = pui32LocalTraceBuf;
+	}
+
+	while (ui32Count < ui32TraceBufSizeInDWords)
+	{
+		IMG_UINT32  ui32Data, ui32DataToId;
+
+		/* Find the first valid log ID, skipping whitespace... */
+		do
+		{
+			IMG_UINT32 ui32ValidatedHostTracePtr;
+			ui32ValidatedHostTracePtr = OSConfineArrayIndexNoSpeculation(ui32HostTracePtr,
+			                                                             ui32TraceBufSizeInDWords);
+			ui32Data     = pui32TraceBuf[ui32ValidatedHostTracePtr];
+			ui32DataToId = idToStringID(ui32Data, SFs);
+
+			/* If an unrecognized id is found it may be inconsistent data or a firmware trace error. */
+			if (ui32DataToId == RGXFW_SF_LAST  &&  RGXFW_LOG_VALIDID(ui32Data))
+			{
+				PVR_DUMPDEBUG_LOG_LINES("WARNING: Unrecognized id (%x). From here on the trace might be wrong!", ui32Data);
+			}
+
+			/* Update the trace pointer... */
+			ui32HostTracePtr++;
+			if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+			{
+				ui32HostTracePtr = 0;
+				ui32HostWrapCount++;
+			}
+			ui32Count++;
+		} while ((RGXFW_SF_LAST == ui32DataToId)  &&
+				 ui32Count < ui32TraceBufSizeInDWords);
+
+		if (ui32Count < ui32TraceBufSizeInDWords)
+		{
+			IMG_CHAR   szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN] = "%" IMG_UINT64_FMTSPEC ":T%u-%s> ";
+			IMG_CHAR   szDebugInfoBuffer[RGXFWT_DEBUG_INFO_STR_MAXLEN] = "";
+			IMG_UINT64 ui64Timestamp;
+			IMG_UINT16 ui16DebugInfo;
+
+			/* If we hit the ASSERT message then this is the end of the log... */
+			if (ui32Data == RGXFW_SF_MAIN_ASSERT_FAILED)
+			{
+				PVR_DUMPDEBUG_LOG_LINES("ASSERTION %.*s failed at %.*s:%u",
+				                        RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.szInfo,
+				                        RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.szPath,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.ui32LineNum);
+				break;
+			}
+
+			ui64Timestamp = (IMG_UINT64)(pui32TraceBuf[(ui32HostTracePtr + 0) % ui32TraceBufSizeInDWords]) << 32 |
+							(IMG_UINT64)(pui32TraceBuf[(ui32HostTracePtr + 1) % ui32TraceBufSizeInDWords]);
+
+			ui16DebugInfo = (IMG_UINT16) ((ui64Timestamp & ~RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK) >> RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT);
+			ui64Timestamp = (ui64Timestamp & ~RGXFWT_TIMESTAMP_TIME_CLRMSK) >> RGXFWT_TIMESTAMP_TIME_SHIFT;
+
+			/*
+			 * Print the trace string and provide up to 20 arguments which
+			 * printf function will be able to use. We have already checked
+			 * that no string uses more than this.
+			 */
+			OSStringLCat(szBuffer, SFs[ui32DataToId].psName, PVR_MAX_DEBUG_MESSAGE_LEN);
+
+			/* Check and append any extra debug info available */
+			if (ui16DebugInfo)
+			{
+				/* Prepare debug info string */
+				RGXPrepareExtraDebugInfo(szDebugInfoBuffer, RGXFWT_DEBUG_INFO_STR_MAXLEN, ui16DebugInfo);
+
+				/* Append debug info string */
+				OSStringLCat(szBuffer, szDebugInfoBuffer, PVR_MAX_DEBUG_MESSAGE_LEN);
+			}
+
+			PVR_DUMPDEBUG_LOG_LINES(szBuffer, ui64Timestamp, ui32TID, groups[RGXFW_SF_GID(ui32Data)],
+			                        pui32TraceBuf[(ui32HostTracePtr +  2) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  3) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  4) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  5) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  6) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  7) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  8) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  9) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 10) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 11) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 12) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 13) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 14) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 15) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 16) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 17) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 18) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 19) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 20) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 21) % ui32TraceBufSizeInDWords]);
+
+			/* Update the trace pointer... */
+			ui32HostTracePtr = ui32HostTracePtr + 2 + RGXFW_SF_PARAMNUM(ui32Data);
+			if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+			{
+				ui32HostTracePtr = ui32HostTracePtr % ui32TraceBufSizeInDWords;
+				ui32HostWrapCount++;
+			}
+			ui32Count = (ui32Count + 2 + RGXFW_SF_PARAMNUM(ui32Data));
+
+			/* Has the FW trace buffer overtaken the host pointer during the last line printed??? */
+			if ((pui32LocalTraceBuf == NULL)  &&
+			    ((*pui32FWWrapCount > ui32HostWrapCount) ||
+			     ((*pui32FWWrapCount == ui32HostWrapCount) && (*pui32FWTracePtr > ui32HostTracePtr))))
+			{
+				/* Move forward to the oldest entry again... */
+				PVR_DUMPDEBUG_LOG_LINES(". . .");
+				ui32HostWrapCount = *pui32FWWrapCount;
+				ui32HostTracePtr  = *pui32FWTracePtr;
+			}
+		}
+	}
+
+	/* Free the local copy of the trace buffer if it was allocated... */
+	if (pui32LocalTraceBuf != NULL)
+	{
+		OSFreeMem(pui32LocalTraceBuf);
+	}
+
+	if (!bPrintAllLines)
+	{
+		IMG_UINT32 ui32FirstLineIdx;
+
+		if (ui32LineCount > PVR_MAX_DEBUG_PARTIAL_LINES)
+		{
+			ui32FirstLineIdx = ui32LastLineIdx;
+			ui32LineCount = PVR_MAX_DEBUG_PARTIAL_LINES;
+		}
+		else
+		{
+			ui32FirstLineIdx = 0;
+		}
+
+		for (ui32Count = 0; ui32Count < ui32LineCount; ui32Count++)
+		{
+			PVR_DUMPDEBUG_LOG("%s", &pszLineBuffer[((ui32FirstLineIdx + ui32Count) % PVR_MAX_DEBUG_PARTIAL_LINES) * PVR_MAX_DEBUG_MESSAGE_LEN]);
+		}
+
+		OSFreeMem(pszLineBuffer);
+	}
+}
+
+void RGXDumpFirmwareTraceDecoded(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	RGXDumpFirmwareTraceLines(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+	                          psRGXFWIfTraceBufCtl, ui32TID, true);
+}
+
+void RGXDumpFirmwareTracePartial(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	RGXDumpFirmwareTraceLines(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+	                          psRGXFWIfTraceBufCtl, ui32TID, false);
+}
+
+void RGXDumpFirmwareTraceBinary(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	IMG_UINT32	i;
+	IMG_BOOL	bPrevLineWasZero = IMG_FALSE;
+	IMG_BOOL	bLineIsAllZeros = IMG_FALSE;
+	IMG_UINT32	ui32CountLines = 0;
+	IMG_UINT32	*pui32TraceBuffer;
+	IMG_CHAR *pszLine;
+
+	RGXFwSharedMemCacheOpExec(psDevInfo->apui32TraceBuffer[ui32TID],
+	                          psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+	pui32TraceBuffer = psDevInfo->apui32TraceBuffer[ui32TID];
+
+/* Max number of DWords to be printed per line, in debug dump binary output */
+#define PVR_DD_FW_TRACEBUF_LINESIZE 30U
+	/* each element in the line is 8 characters plus a space.  The '+ 1' is because of the final trailing '\0'. */
+	pszLine = OSAllocMem(9 * PVR_DD_FW_TRACEBUF_LINESIZE + 1);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Out of mem allocating line string (size: %d)",
+				__func__,
+				9 * PVR_DD_FW_TRACEBUF_LINESIZE + 1));
+		return;
+	}
+
+	for (i = 0; i < psDevInfo->ui32TraceBufSizeInDWords; i += PVR_DD_FW_TRACEBUF_LINESIZE)
+	{
+		IMG_UINT32 k = 0;
+		IMG_UINT32 ui32Line = 0x0;
+		IMG_UINT32 ui32LineOffset = i*sizeof(IMG_UINT32);
+		IMG_CHAR   *pszBuf = pszLine;
+
+		for (k = 0; k < PVR_DD_FW_TRACEBUF_LINESIZE; k++)
+		{
+			if ((i + k) >= psDevInfo->ui32TraceBufSizeInDWords)
+			{
+				/* Stop reading when the index goes beyond trace buffer size. This condition is
+				 * hit during printing the last line in DD when ui32TraceBufSizeInDWords is not
+				 * a multiple of PVR_DD_FW_TRACEBUF_LINESIZE */
+				break;
+			}
+
+			ui32Line |= pui32TraceBuffer[i + k];
+
+			/* prepare the line to print it. The '+1' is because of the trailing '\0' added */
+			OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32TraceBuffer[i + k]);
+			pszBuf += 9; /* write over the '\0' */
+		}
+
+		bLineIsAllZeros = (ui32Line == 0x0);
+
+		if (bLineIsAllZeros)
+		{
+			if (bPrevLineWasZero)
+			{
+				ui32CountLines++;
+			}
+			else
+			{
+				bPrevLineWasZero = IMG_TRUE;
+				ui32CountLines = 1;
+				PVR_DUMPDEBUG_LOG("FWT[%08x]: 00000000 ... 00000000", ui32LineOffset);
+			}
+		}
+		else
+		{
+			if (bPrevLineWasZero  &&  ui32CountLines > 1)
+			{
+				PVR_DUMPDEBUG_LOG("FWT[...]: %d lines were all zero", ui32CountLines);
+			}
+			bPrevLineWasZero = IMG_FALSE;
+
+			PVR_DUMPDEBUG_LOG("FWT[%08x]:%s", ui32LineOffset, pszLine);
+		}
+	}
+
+	if (bPrevLineWasZero)
+	{
+		PVR_DUMPDEBUG_LOG("FWT[END]: %d lines were all zero", ui32CountLines);
+	}
+
+	OSFreeMem(pszLine);
+}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+void RGXDocumentFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				const IMG_UINT32 ui32FwVA,
+				const IMG_CPU_PHYADDR sCpuPA,
+				const IMG_DEV_PHYADDR sDevPA,
+				const IMG_UINT64 ui64PTE)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		PVR_DUMPDEBUG_LOG("|    0x%08X   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "%s%s%s   |",
+						  ui32FwVA,
+						  (IMG_UINT64) sCpuPA.uiAddr,
+						  sDevPA.uiAddr,
+						  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(ui64PTE)],
+						  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(ui64PTE)],
+						  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(ui64PTE)]);
+	}
+	else
+#endif
+	{
+		const char *pszSLCBypass =
+#if defined(RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN)
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN) ? "B" : " ";
+#else
+						  " ";
+#endif
+
+		/* META and RISCV use a subset of the GPU's virtual address space */
+		PVR_DUMPDEBUG_LOG("|    0x%08X   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "%s%s%s%s%s%s   |",
+						  ui32FwVA,
+						  (IMG_UINT64) sCpuPA.uiAddr,
+						  sDevPA.uiAddr,
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN)   ? "P" : " ",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_PM_SRC_EN)          ? "PM" : "  ",
+						  pszSLCBypass,
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_CC_EN)              ? "C" : " ",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_READ_ONLY_EN)       ? "RO" : "RW",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_VALID_EN)           ? "V" : " ");
+	}
+}
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+#if !defined(NO_HARDWARE)
+static PVRSRV_ERROR
+RGXPollMetaRegThroughSP(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32RegOffset,
+                        IMG_UINT32 ui32PollValue, IMG_UINT32 ui32Mask)
+{
+	IMG_UINT32 ui32RegValue, ui32NumPolls = 0;
+	PVRSRV_ERROR eError;
+
+	do
+	{
+		eError = RGXReadFWModuleAddr(psDevInfo, ui32RegOffset, &ui32RegValue);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	} while (((ui32RegValue & ui32Mask) != ui32PollValue) && (ui32NumPolls++ < 1000));
+
+	return ((ui32RegValue & ui32Mask) == ui32PollValue) ? PVRSRV_OK : PVRSRV_ERROR_RETRY;
+}
+
+PVRSRV_ERROR
+RGXReadMetaCoreReg(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32RegAddr, IMG_UINT32 *pui32RegVal)
+{
+	PVRSRV_ERROR eError;
+
+	/* Core Read Ready? */
+	eError = RGXPollMetaRegThroughSP(psDevInfo,
+	                                 META_CR_TXUXXRXRQ_OFFSET,
+	                                 META_CR_TXUXXRXRQ_DREADY_BIT,
+									 META_CR_TXUXXRXRQ_DREADY_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXPollMetaRegThroughSP");
+
+	/* Set the reg we are interested in reading */
+	eError = RGXWriteFWModuleAddr(psDevInfo, META_CR_TXUXXRXRQ_OFFSET,
+	                        ui32RegAddr | META_CR_TXUXXRXRQ_RDnWR_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXWriteFWModuleAddr");
+
+	/* Core Read Done? */
+	eError = RGXPollMetaRegThroughSP(psDevInfo,
+	                                 META_CR_TXUXXRXRQ_OFFSET,
+	                                 META_CR_TXUXXRXRQ_DREADY_BIT,
+									 META_CR_TXUXXRXRQ_DREADY_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXPollMetaRegThroughSP");
+
+	/* Read the value */
+	return RGXReadFWModuleAddr(psDevInfo, META_CR_TXUXXRXDT_OFFSET, pui32RegVal);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR _ValidateWithFWModule(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo,
+						RGXFWIF_DEV_VIRTADDR *psFWAddr,
+						void *pvHostCodeAddr,
+						IMG_UINT32 ui32MaxLen,
+						const IMG_CHAR *pszDesc,
+						IMG_UINT32 ui32StartOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32Value = 0;
+	IMG_UINT32 ui32FWCodeDevVAAddr = psFWAddr->ui32Addr + ui32StartOffset;
+	IMG_UINT32 *pui32FWCode = (IMG_UINT32*) IMG_OFFSET_ADDR(pvHostCodeAddr,ui32StartOffset);
+	IMG_UINT32 i;
+
+#if defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	ui32MaxLen -= ui32StartOffset;
+	ui32MaxLen /= sizeof(IMG_UINT32); /* Byte -> 32 bit words */
+
+	for (i = 0; i < ui32MaxLen; i++)
+	{
+		eError = RGXReadFWModuleAddr(psDevInfo, ui32FWCodeDevVAAddr, &ui32Value);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+#if defined(EMULATOR)
+		if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+#endif
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "0x%x: CPU 0x%08x, FW 0x%08x", i * 4, pui32FWCode[i], ui32Value));
+
+			if (pui32FWCode[i] != ui32Value)
+			{
+				PVR_DUMPDEBUG_LOG("%s: Mismatch while validating %s at offset 0x%x: CPU 0x%08x (%p), FW 0x%08x (%x)",
+					 __func__, pszDesc,
+					 (i * 4) + ui32StartOffset, pui32FWCode[i], pui32FWCode, ui32Value, ui32FWCodeDevVAAddr);
+				return PVRSRV_ERROR_FW_IMAGE_MISMATCH;
+			}
+		}
+
+		ui32FWCodeDevVAAddr += 4;
+	}
+
+	PVR_DUMPDEBUG_LOG("Match between Host and Firmware view of the %s", pszDesc);
+	return PVRSRV_OK;
+}
+#endif
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXValidateFWImage(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if !defined(SUPPORT_TRUSTED_DEVICE)
+	PVRSRV_ERROR eError;
+	IMG_UINT32 *pui32HostFWCode = NULL, *pui32HostFWCoremem = NULL;
+	OS_FW_IMAGE *psRGXFW = NULL;
+	const IMG_BYTE *pbRGXFirmware = NULL;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_UINT32 *pui32CodeMemoryPointer;
+#endif
+	RGXFWIF_DEV_VIRTADDR sFWAddr;
+	IMG_UINT32 ui32StartOffset = 0;
+
+#if defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		PVR_DUMPDEBUG_LOG("Validation of RISC-V FW code is disabled on emulator");
+		return PVRSRV_OK;
+	}
+#endif
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: RGX registers not mapped yet!", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	/* Load FW from system for code verification */
+	pui32HostFWCode = OSAllocZMem(psDevInfo->ui32FWCodeSizeInBytes);
+	if (pui32HostFWCode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed in allocating memory for FW code. "
+				"So skipping FW code verification",
+				__func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Coremem is not present on all GPU cores, so size can be zero */
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		pui32HostFWCoremem = OSAllocZMem(psDevInfo->ui32FWCorememCodeSizeInBytes);
+		if (pui32HostFWCoremem == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed in allocating memory for FW core code. "
+					"So skipping FW code verification",
+					__func__));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto freeHostFWCode;
+		}
+	}
+
+	/* Load FW image */
+	eError = RGXLoadAndGetFWData(psDevInfo->psDeviceNode, &psRGXFW);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to load FW image file (%s).",
+		         __func__, PVRSRVGetErrorString(eError)));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto cleanup_initfw;
+	}
+
+	pbRGXFirmware = (const IMG_BYTE *)OSFirmwareData(psRGXFW);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		eError = ProcessLDRCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+						(void*) pui32HostFWCode, NULL,
+						(void*) pui32HostFWCoremem, NULL, NULL);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = ProcessELFCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+		                                 pui32HostFWCode, NULL,
+		                                 NULL, NULL);
+	}
+#endif
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = ProcessELFCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+		                                 pui32HostFWCode, NULL,
+		                                 pui32HostFWCoremem, NULL);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed in parsing FW image file.", __func__));
+		goto cleanup_initfw;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, (void **)&pui32CodeMemoryPointer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Error in acquiring MIPS FW code memory area (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+
+		RGXFwSharedMemCacheOpExec(pui32CodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, PVRSRV_CACHE_OP_INVALIDATE);
+
+		if (OSMemCmp(pui32HostFWCode, pui32CodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes) == 0)
+		{
+			PVR_DUMPDEBUG_LOG("Match between Host and MIPS views of the FW code" );
+		}
+		else
+		{
+			IMG_UINT32 ui32Count = 10; /* Show only the first 10 mismatches */
+			IMG_UINT32 ui32Offset;
+
+			PVR_DUMPDEBUG_LOG("Mismatch between Host and MIPS views of the FW code");
+			for (ui32Offset = 0; (ui32Offset*4 < psDevInfo->ui32FWCodeSizeInBytes) || (ui32Count == 0); ui32Offset++)
+			{
+				if (pui32HostFWCode[ui32Offset] != pui32CodeMemoryPointer[ui32Offset])
+				{
+					PVR_DUMPDEBUG_LOG("At %d bytes, code should be 0x%x but it is instead 0x%x",
+					   ui32Offset*4, pui32HostFWCode[ui32Offset], pui32CodeMemoryPointer[ui32Offset]);
+					ui32Count--;
+				}
+			}
+		}
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+	else
+#endif
+	{
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			/* starting checking after BOOT LOADER config */
+			sFWAddr.ui32Addr = RGXFW_BOOTLDR_META_ADDR;
+
+			ui32StartOffset = RGXFW_MAX_BOOTLDR_OFFSET;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			/* Use bootloader code remap which is always configured before the FW is started */
+			if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+			{
+				sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE;
+			}
+			else
+#endif
+			{
+				sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+			}
+		}
+
+		eError = _ValidateWithFWModule(pfnDumpDebugPrintf, pvDumpDebugFile,
+						psDevInfo, &sFWAddr,
+						pui32HostFWCode, psDevInfo->ui32FWCodeSizeInBytes,
+						"FW code", ui32StartOffset);
+		if (eError != PVRSRV_OK)
+		{
+			goto cleanup_initfw;
+		}
+
+		/* Coremem is not present on all GPU cores, so may not be alloc'd */
+		if (pui32HostFWCoremem != NULL) // && psDevInfo->ui32FWCorememCodeSizeInBytes
+		{
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+			{
+				sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, META_COREMEM_CODE);
+			}
+			else
+			{
+				sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, RISCV_COREMEM_CODE);
+
+				/* Core must be halted while issuing abstract commands */
+				eError = RGXRiscvHalt(psDevInfo);
+				PVR_GOTO_IF_ERROR(eError, cleanup_initfw);
+			}
+
+			eError = _ValidateWithFWModule(pfnDumpDebugPrintf, pvDumpDebugFile,
+							psDevInfo, &sFWAddr,
+							pui32HostFWCoremem, psDevInfo->ui32FWCorememCodeSizeInBytes,
+							"FW coremem code", 0);
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+			{
+				eError = RGXRiscvResume(psDevInfo);
+				PVR_GOTO_IF_ERROR(eError, cleanup_initfw);
+			}
+		}
+	}
+
+cleanup_initfw:
+	if (psRGXFW)
+	{
+		OSUnloadFirmware(psRGXFW);
+	}
+
+	if (pui32HostFWCoremem)
+	{
+		OSFreeMem(pui32HostFWCoremem);
+	}
+freeHostFWCode:
+	if (pui32HostFWCode)
+	{
+		OSFreeMem(pui32HostFWCode);
+	}
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+PVRSRV_ERROR ValidateFWOnLoad(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_TRUSTED_DEVICE)
+	IMG_PBYTE pbCodeMemoryPointer;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR sFWAddr;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, (void **)&pbCodeMemoryPointer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	RGXFwSharedMemCacheOpExec(pbCodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, PVRSRV_CACHE_OP_INVALIDATE);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		sFWAddr.ui32Addr = RGXFW_BOOTLDR_META_ADDR;
+	}
+	else
+	{
+		PVR_ASSERT(RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR));
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+		{
+			sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE;
+		}
+		else
+#endif
+		{
+			sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+		}
+	};
+
+	eError = _ValidateWithFWModule(NULL, NULL, psDevInfo, &sFWAddr, pbCodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, "FW code", 0);
+	if (eError != PVRSRV_OK)
+	{
+		goto releaseFWCodeMapping;
+	}
+
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, (void **)&pbCodeMemoryPointer);
+		if (eError != PVRSRV_OK)
+		{
+			goto releaseFWCoreCodeMapping;
+		}
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, META_COREMEM_CODE);
+		}
+		else
+		{
+			PVR_ASSERT(RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR));
+			sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, RISCV_COREMEM_CODE);
+		}
+
+		eError = _ValidateWithFWModule(NULL, NULL, psDevInfo, &sFWAddr, pbCodeMemoryPointer,
+						psDevInfo->ui32FWCorememCodeSizeInBytes, "FW coremem code", 0);
+	}
+
+releaseFWCoreCodeMapping:
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+releaseFWCodeMapping:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+#endif
+}
+#endif
+
+static const IMG_CHAR* _RGXGetDebugDevPowerStateString(PVRSRV_DEV_POWER_STATE ePowerState)
+{
+	switch (ePowerState)
+	{
+		case PVRSRV_DEV_POWER_STATE_DEFAULT: return "DEFAULT";
+		case PVRSRV_DEV_POWER_STATE_OFF: return "OFF";
+		case PVRSRV_DEV_POWER_STATE_ON: return "ON";
+		default: return "UNKNOWN";
+	}
+}
+
+/*
+	Writes flags strings to an uninitialised buffer.
+*/
+static void _GetFwSysFlagsDescription(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, IMG_UINT32 ui32RawFlags)
+{
+	const IMG_CHAR szCswLabel[] = "Ctx switch options:";
+	size_t uLabelLen = sizeof(szCswLabel) - 1;
+	const size_t uiBytesPerDesc = (ui32DescSize - uLabelLen) / 2U - 1U;
+
+	OSStringSafeCopy(psDesc, szCswLabel, ui32DescSize);
+
+	DebugCommonFlagStrings(psDesc, uiBytesPerDesc + uLabelLen, asCswOpts2Description, ARRAY_SIZE(asCswOpts2Description), ui32RawFlags);
+	DebugCommonFlagStrings(psDesc, ui32DescSize, asMisc2Description, ARRAY_SIZE(asMisc2Description), ui32RawFlags);
+}
+
+static void _GetFwOsFlagsDescription(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, IMG_UINT32 ui32RawFlags)
+{
+	const IMG_CHAR szCswLabel[] = "Ctx switch:";
+	size_t uLabelLen = sizeof(szCswLabel) - 1;
+	const size_t uiBytesPerDesc = (ui32DescSize - uLabelLen) / 2U - 1U;
+
+	OSStringSafeCopy(psDesc, szCswLabel, ui32DescSize);
+
+	DebugCommonFlagStrings(psDesc, uiBytesPerDesc + uLabelLen, asFwOsCfg2Description, ARRAY_SIZE(asFwOsCfg2Description), ui32RawFlags);
+}
+
+
+typedef enum _DEVICEMEM_HISTORY_QUERY_INDEX_
+{
+	DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING,
+	DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED,
+	DEVICEMEM_HISTORY_QUERY_INDEX_NEXT,
+	DEVICEMEM_HISTORY_QUERY_INDEX_COUNT,
+} DEVICEMEM_HISTORY_QUERY_INDEX;
+
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintDevicememHistoryQueryResult
+
+ @Description
+
+ Print details of a single result from a DevicememHistory query
+
+ @Input pfnDumpDebugPrintf       - Debug printf function
+ @Input pvDumpDebugFile          - Optional file identifier to be passed to the
+                                   'printf' function if required
+ @Input psFaultProcessInfo       - The process info derived from the page fault
+ @Input psResult                 - The DevicememHistory result to be printed
+ @Input ui32Index                - The index of the result
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintDevicememHistoryQueryResult(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						RGXMEM_PROCESS_INFO *psFaultProcessInfo,
+						DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult,
+						IMG_UINT32 ui32Index,
+						const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+	RGXConvertOSTimestampToSAndNS(psResult->ui64When,
+							&ui64Seconds,
+							&ui64Nanoseconds);
+
+	if (psFaultProcessInfo->uiPID != RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		PVR_DUMPDEBUG_LOG("%s    [%u] Name: %s Base address: " IMG_DEV_VIRTADDR_FMTSPEC
+					" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+					" Operation: %s Modified: %" IMG_UINT64_FMTSPEC
+					" us ago (OS time %" IMG_UINT64_FMTSPEC
+					".%09" IMG_UINT64_FMTSPEC " s)",
+						pszIndent,
+						ui32Index,
+						psResult->szString,
+						psResult->sBaseDevVAddr.uiAddr,
+						psResult->uiSize,
+						psResult->bMap ? "Map": "Unmap",
+						OSDivide64r64(psResult->ui64Age, 1000, &ui32Remainder),
+						ui64Seconds,
+						ui64Nanoseconds);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s    [%u] Name: %s Base address: " IMG_DEV_VIRTADDR_FMTSPEC
+					" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+					" Operation: %s Modified: %" IMG_UINT64_FMTSPEC
+					" us ago (OS time %" IMG_UINT64_FMTSPEC
+					".%09" IMG_UINT64_FMTSPEC
+					") PID: %u (%s)",
+						pszIndent,
+						ui32Index,
+						psResult->szString,
+						psResult->sBaseDevVAddr.uiAddr,
+						psResult->uiSize,
+						psResult->bMap ? "Map": "Unmap",
+						OSDivide64r64(psResult->ui64Age, 1000, &ui32Remainder),
+						ui64Seconds,
+						ui64Nanoseconds,
+						psResult->sProcessInfo.uiPID,
+						psResult->sProcessInfo.szProcessName);
+	}
+
+	if (!psResult->bRange)
+	{
+		PVR_DUMPDEBUG_LOG("%s        Whole allocation was %s", pszIndent, psResult->bMap ? "mapped": "unmapped");
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s        Pages %u to %u (" IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC ") %s%s",
+										pszIndent,
+										psResult->ui32StartPage,
+										psResult->ui32StartPage + psResult->ui32PageCount - 1,
+										psResult->sMapStartAddr.uiAddr,
+										psResult->sMapEndAddr.uiAddr,
+										psResult->bAll ? "(whole allocation) " : "",
+										psResult->bMap ? "mapped": "unmapped");
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintDevicememHistoryQueryOut
+
+ @Description
+
+ Print details of all the results from a DevicememHistory query
+
+ @Input pfnDumpDebugPrintf       - Debug printf function
+ @Input pvDumpDebugFile          - Optional file identifier to be passed to the
+                                   'printf' function if required
+ @Input psFaultProcessInfo       - The process info derived from the page fault
+ @Input psQueryOut               - Storage for the query results
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintDevicememHistoryQueryOut(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						RGXMEM_PROCESS_INFO *psFaultProcessInfo,
+						DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+						const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 i;
+
+	if (psQueryOut->ui32NumResults == 0)
+	{
+		PVR_DUMPDEBUG_LOG("%s    No results", pszIndent);
+	}
+	else
+	{
+		for (i = 0; i < psQueryOut->ui32NumResults; i++)
+		{
+			_PrintDevicememHistoryQueryResult(pfnDumpDebugPrintf, pvDumpDebugFile,
+									psFaultProcessInfo,
+									&psQueryOut->sResults[i],
+									i,
+									pszIndent);
+		}
+	}
+}
+
+/* table of HW page size values and the equivalent */
+static const unsigned int aui32HWPageSizeTable[][2] =
+{
+	{ 0, PVRSRV_4K_PAGE_SIZE },
+	{ 1, PVRSRV_16K_PAGE_SIZE },
+	{ 2, PVRSRV_64K_PAGE_SIZE },
+	{ 3, PVRSRV_256K_PAGE_SIZE },
+	{ 4, PVRSRV_1M_PAGE_SIZE },
+	{ 5, PVRSRV_2M_PAGE_SIZE }
+};
+
+/*!
+*******************************************************************************
+
+ @Function	_PageSizeHWToBytes
+
+ @Description
+
+ Convert a HW page size value to its size in bytes
+
+ @Input ui32PageSizeHW     - The HW page size value
+
+ @Return   IMG_UINT32      The page size in bytes
+
+******************************************************************************/
+static IMG_UINT32 _PageSizeHWToBytes(IMG_UINT32 ui32PageSizeHW)
+{
+	if (ui32PageSizeHW > 5)
+	{
+		/* This is invalid, so return a default value as we cannot ASSERT in this code! */
+		return PVRSRV_4K_PAGE_SIZE;
+	}
+
+	return aui32HWPageSizeTable[ui32PageSizeHW][1];
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetDevicememHistoryData
+
+ @Description
+
+ Get the DevicememHistory results for the given PID and faulting device virtual address.
+ The function will query DevicememHistory for information about the faulting page, as well
+ as the page before and after.
+
+ @Input psDeviceNode       - The device which this allocation search should be made on
+ @Input uiPID              - The process ID to search for allocations belonging to
+ @Input sFaultDevVAddr     - The device address to search for allocations at/before/after
+ @Input asQueryOut         - Storage for the query results
+ @Input ui32PageSizeBytes  - Faulted page size in bytes
+
+ @Return IMG_BOOL          - IMG_TRUE if any results were found for this page fault
+
+******************************************************************************/
+static IMG_BOOL _GetDevicememHistoryData(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sFaultDevVAddr,
+							DEVICEMEM_HISTORY_QUERY_OUT asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_COUNT],
+							IMG_UINT32 ui32PageSizeBytes)
+{
+	DEVICEMEM_HISTORY_QUERY_IN sQueryIn;
+	IMG_BOOL bAnyHits = IMG_FALSE;
+
+	/* if the page fault originated in the firmware then the allocation may
+	 * appear to belong to any PID, because FW allocations are attributed
+	 * to the client process creating the allocation, so instruct the
+	 * devicemem_history query to search all available PIDs
+	 */
+	if (uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		sQueryIn.uiPID = DEVICEMEM_HISTORY_PID_ANY;
+	}
+	else
+	{
+		sQueryIn.uiPID = uiPID;
+	}
+
+	sQueryIn.psDevNode = psDeviceNode;
+	/* Query the DevicememHistory for all allocations in the previous page... */
+	sQueryIn.sDevVAddr.uiAddr = (sFaultDevVAddr.uiAddr & ~(IMG_UINT64)(ui32PageSizeBytes - 1)) - ui32PageSizeBytes;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING],
+	                          ui32PageSizeBytes, IMG_TRUE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+
+	/* Query the DevicememHistory for any record at the exact address... */
+	sQueryIn.sDevVAddr = sFaultDevVAddr;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED],
+	                          ui32PageSizeBytes, IMG_FALSE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+	else
+	{
+		/* If not matched then try matching any record in the faulting page... */
+		if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED],
+		                          ui32PageSizeBytes, IMG_TRUE))
+		{
+			bAnyHits = IMG_TRUE;
+		}
+	}
+
+	/* Query the DevicememHistory for all allocations in the next page... */
+	sQueryIn.sDevVAddr.uiAddr = (sFaultDevVAddr.uiAddr & ~(IMG_UINT64)(ui32PageSizeBytes - 1)) + ui32PageSizeBytes;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_NEXT],
+	                          ui32PageSizeBytes, IMG_TRUE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+
+	return bAnyHits;
+}
+
+/* stored data about one page fault */
+typedef struct _FAULT_INFO_
+{
+	/* the process info of the memory context that page faulted */
+	RGXMEM_PROCESS_INFO sProcessInfo;
+	IMG_DEV_VIRTADDR sFaultDevVAddr;
+	MMU_FAULT_DATA   sMMUFaultData;
+	DEVICEMEM_HISTORY_QUERY_OUT asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_COUNT];
+	/* the CR timer value at the time of the fault, recorded by the FW.
+	 * used to differentiate different page faults
+	 */
+	IMG_UINT64 ui64CRTimer;
+	/* time when this FAULT_INFO entry was added. used for timing
+	 * reference against the map/unmap information
+	 */
+	IMG_UINT64 ui64When;
+	IMG_UINT32 ui32FaultInfoFlags;
+} FAULT_INFO;
+
+/* history list of page faults.
+ * Keeps the first `n` page faults and the last `n` page faults, like the FW
+ * HWR log
+ */
+typedef struct _FAULT_INFO_LOG_
+{
+	IMG_UINT32 ui32Head;
+	/* the number of faults in this log need not correspond exactly to
+	 * the HWINFO number of the FW, as the FW HWINFO log may contain
+	 * non-page fault HWRs
+	 */
+	FAULT_INFO asFaults[RGXFWIF_HWINFO_MAX];
+} FAULT_INFO_LOG;
+
+#define FAULT_INFO_PROC_INFO   (0x1U)
+#define FAULT_INFO_DEVMEM_HIST (0x2U)
+
+static FAULT_INFO_LOG gsFaultInfoLog = { 0 };
+
+static void _FillAppForFWFaults(PVRSRV_RGXDEV_INFO *psDevInfo,
+							FAULT_INFO *psInfo)
+{
+	IMG_UINT32 i, j;
+
+	for (i = 0; i < DEVICEMEM_HISTORY_QUERY_INDEX_COUNT; i++)
+	{
+		for (j = 0; j < DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS; j++)
+		{
+			IMG_BOOL bFound;
+
+			RGXMEM_PROCESS_INFO *psProcInfo = &psInfo->asQueryOut[i].sResults[j].sProcessInfo;
+			bFound = RGXPCPIDToProcessInfo(psDevInfo,
+								psProcInfo->uiPID,
+								psProcInfo);
+			if (!bFound)
+			{
+				OSStringSafeCopy(psProcInfo->szProcessName,
+								"(unknown)",
+								sizeof(psProcInfo->szProcessName));
+			}
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintFaultInfo
+
+ @Description
+
+ Print all the details of a page fault from a FAULT_INFO structure
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psInfo               - The page fault occurrence to print
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintFaultInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					PVRSRV_DEVICE_NODE *psDevNode,
+					void *pvDumpDebugFile,
+					FAULT_INFO *psInfo,
+					const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+	RGXConvertOSTimestampToSAndNS(psInfo->ui64When, &ui64Seconds, &ui64Nanoseconds);
+
+	if (BITMASK_HAS(psInfo->ui32FaultInfoFlags, FAULT_INFO_PROC_INFO))
+	{
+		IMG_PID uiPID = (psInfo->sProcessInfo.uiPID == RGXMEM_SERVER_PID_FIRMWARE || psInfo->sProcessInfo.uiPID == RGXMEM_SERVER_PID_PM) ?
+							0 : psInfo->sProcessInfo.uiPID;
+
+		PVR_DUMPDEBUG_LOG("%sDevice memory history for page fault address " IMG_DEV_VIRTADDR_FMTSPEC
+							", CRTimer: 0x%016" IMG_UINT64_FMTSPECX
+							", PID: %u (%s, unregistered: %u) OS time: "
+							"%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+					pszIndent,
+					psInfo->sFaultDevVAddr.uiAddr,
+					psInfo->ui64CRTimer,
+					uiPID,
+					psInfo->sProcessInfo.szProcessName,
+					psInfo->sProcessInfo.bUnregistered,
+					ui64Seconds,
+					ui64Nanoseconds);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%sCould not find PID for device memory history on PC of the fault", pszIndent);
+	}
+
+	if (BITMASK_HAS(psInfo->ui32FaultInfoFlags, FAULT_INFO_DEVMEM_HIST))
+	{
+		for (i = DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING; i < DEVICEMEM_HISTORY_QUERY_INDEX_COUNT; i++)
+		{
+			const IMG_CHAR *pszWhich = NULL;
+
+			switch (i)
+			{
+				case DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING:
+					pszWhich = "Preceding page";
+					break;
+				case DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED:
+					pszWhich = "Faulted page";
+					break;
+				case DEVICEMEM_HISTORY_QUERY_INDEX_NEXT:
+					pszWhich = "Next page";
+					break;
+			}
+
+			PVR_DUMPDEBUG_LOG("%s  %s:", pszIndent, pszWhich);
+			_PrintDevicememHistoryQueryOut(pfnDumpDebugPrintf, pvDumpDebugFile,
+								&psInfo->sProcessInfo,
+								&psInfo->asQueryOut[i],
+								pszIndent);
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s  No matching Devmem History for fault address", pszIndent);
+		DevicememHistoryDumpRecordStats(psDevNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+		PVR_DUMPDEBUG_LOG("%s  Records Searched -"
+		                  " PP:%"IMG_UINT64_FMTSPEC
+		                  " FP:%"IMG_UINT64_FMTSPEC
+		                  " NP:%"IMG_UINT64_FMTSPEC,
+		                  pszIndent,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING].ui64SearchCount,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED].ui64SearchCount,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_NEXT].ui64SearchCount);
+	}
+}
+
+static void _RecordFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					FAULT_INFO *psInfo,
+					IMG_DEV_VIRTADDR sFaultDevVAddr,
+					IMG_DEV_PHYADDR sPCDevPAddr,
+					IMG_UINT64 ui64CRTimer,
+					IMG_UINT32 ui32PageSizeBytes)
+{
+	IMG_BOOL bFound = IMG_FALSE, bIsPMFault = IMG_FALSE;
+	RGXMEM_PROCESS_INFO sProcessInfo;
+
+	psInfo->ui32FaultInfoFlags = 0;
+	psInfo->sFaultDevVAddr = sFaultDevVAddr;
+	psInfo->ui64CRTimer = ui64CRTimer;
+	psInfo->ui64When = OSClockns64();
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		/* Check if this is PM fault */
+		if (psInfo->sMMUFaultData.eType == MMU_FAULT_TYPE_PM)
+		{
+			bIsPMFault = IMG_TRUE;
+			bFound = IMG_TRUE;
+			sProcessInfo.uiPID = RGXMEM_SERVER_PID_PM;
+			OSStringSafeCopy(sProcessInfo.szProcessName, "PM", sizeof(sProcessInfo.szProcessName));
+			sProcessInfo.szProcessName[sizeof(sProcessInfo.szProcessName) - 1] = '\0';
+			sProcessInfo.bUnregistered = IMG_FALSE;
+		}
+		else
+		{
+			/* look up the process details for the faulting page catalogue */
+			bFound = RGXPCAddrToProcessInfo(psDevInfo, sPCDevPAddr, &sProcessInfo);
+		}
+
+		if (bFound)
+		{
+			IMG_BOOL bHits;
+
+			psInfo->ui32FaultInfoFlags = FAULT_INFO_PROC_INFO;
+			psInfo->sProcessInfo = sProcessInfo;
+
+			if (bIsPMFault)
+			{
+				bHits = IMG_TRUE;
+			}
+			else
+			{
+				/* get any DevicememHistory data for the faulting address */
+				bHits = _GetDevicememHistoryData(psDevInfo->psDeviceNode,
+								 sProcessInfo.uiPID,
+								 sFaultDevVAddr,
+								 psInfo->asQueryOut,
+								 ui32PageSizeBytes);
+
+				if (bHits)
+				{
+					psInfo->ui32FaultInfoFlags |= FAULT_INFO_DEVMEM_HIST;
+
+					/* if the page fault was caused by the firmware then get information about
+					 * which client application created the related allocations.
+					 *
+					 * Fill in the process info data for each query result.
+					 */
+
+					if (sProcessInfo.uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+					{
+						_FillAppForFWFaults(psDevInfo, psInfo);
+					}
+				}
+			}
+		}
+	}
+}
+
+void RGXDumpFaultAddressHostView(MMU_FAULT_DATA *psFaultData,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const IMG_CHAR* pszIndent)
+{
+	MMU_LEVEL eTopLevel;
+	const IMG_CHAR szPageLevel[][4] = {"", "PTE", "PDE", "PCE" };
+	const IMG_CHAR szPageError[][3] = {"", "PT",  "PD",  "PC"  };
+
+	eTopLevel = psFaultData->eTopLevel;
+
+	if (psFaultData->eType == MMU_FAULT_TYPE_UNKNOWN)
+	{
+		PVR_DUMPDEBUG_LOG("%sNo live host MMU data available", pszIndent);
+		return;
+	}
+	else if (psFaultData->eType == MMU_FAULT_TYPE_PM)
+	{
+		PVR_DUMPDEBUG_LOG("%sPM faulted at PC address = 0x%016" IMG_UINT64_FMTSPECx, pszIndent, psFaultData->sLevelData[MMU_LEVEL_0].ui64Address);
+	}
+	else
+	{
+		MMU_LEVEL eCurrLevel;
+		PVR_ASSERT(eTopLevel < MMU_LEVEL_LAST);
+
+		for (eCurrLevel = eTopLevel; eCurrLevel > MMU_LEVEL_0; eCurrLevel--)
+		{
+			MMU_LEVEL_DATA *psMMULevelData = &psFaultData->sLevelData[eCurrLevel];
+			if (psMMULevelData->ui64Address)
+			{
+				if (psMMULevelData->uiBytesPerEntry == 4)
+				{
+					PVR_DUMPDEBUG_LOG("%s%s for index %d = 0x%08x and is %s",
+								pszIndent,
+								szPageLevel[eCurrLevel],
+								psMMULevelData->ui32Index,
+								(IMG_UINT) psMMULevelData->ui64Address,
+								psMMULevelData->psDebugStr);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("%s%s for index %d = 0x%016" IMG_UINT64_FMTSPECx " and is %s",
+								pszIndent,
+								szPageLevel[eCurrLevel],
+								psMMULevelData->ui32Index,
+								psMMULevelData->ui64Address,
+								psMMULevelData->psDebugStr);
+				}
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("%s%s index (%d) out of bounds (%d)",
+							pszIndent,
+							szPageError[eCurrLevel],
+							psMMULevelData->ui32Index,
+							psMMULevelData->ui32NumOfEntries);
+				break;
+			}
+		}
+	}
+
+}
+
+void RGXDumpFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile,
+					  const RGX_HWRINFO *psHWRInfo,
+                      IMG_UINT32 ui32ReadIndex,
+                      IMG_DEV_VIRTADDR *psFaultDevVAddr,
+                      IMG_DEV_PHYADDR *psPCDevPAddr,
+                      bool bPMFault,
+                      IMG_UINT32 ui32PageSize)
+{
+	FAULT_INFO *psInfo;
+
+	OSLockAcquire(psDevInfo->hDebugFaultInfoLock);
+
+	/* Find the matching Fault Info for this HWRInfo */
+	psInfo = &gsFaultInfoLog.asFaults[ui32ReadIndex];
+
+	/* if they do not match, we need to update the psInfo */
+	if ((psInfo->ui64CRTimer != psHWRInfo->ui64CRTimer) ||
+		(psInfo->sFaultDevVAddr.uiAddr != psFaultDevVAddr->uiAddr))
+	{
+		MMU_FAULT_DATA *psFaultData = &psInfo->sMMUFaultData;
+
+		psFaultData->eType = MMU_FAULT_TYPE_UNKNOWN;
+
+		if (bPMFault)
+		{
+			/* PM fault and we dump PC details only */
+			psFaultData->eTopLevel = MMU_LEVEL_0;
+			psFaultData->eType     = MMU_FAULT_TYPE_PM;
+			psFaultData->sLevelData[MMU_LEVEL_0].ui64Address = psPCDevPAddr->uiAddr;
+		}
+		else
+		{
+			RGXCheckFaultAddress(psDevInfo, psFaultDevVAddr, psPCDevPAddr, psFaultData);
+		}
+
+		_RecordFaultInfo(psDevInfo, psInfo,
+					*psFaultDevVAddr, *psPCDevPAddr, psHWRInfo->ui64CRTimer,
+					_PageSizeHWToBytes(ui32PageSize));
+
+	}
+
+	RGXDumpFaultAddressHostView(&psInfo->sMMUFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_NORMAL_INDENT);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		_PrintFaultInfo(pfnDumpDebugPrintf, psDevInfo->psDeviceNode, pvDumpDebugFile, psInfo, DD_NORMAL_INDENT);
+	}
+
+	OSLockRelease(psDevInfo->hDebugFaultInfoLock);
+}
+
+void RGXConvertOSTimestampToSAndNS(IMG_UINT64 ui64OSTimer,
+							IMG_UINT64 *pui64Seconds,
+							IMG_UINT64 *pui64Nanoseconds)
+{
+	IMG_UINT32 ui32Remainder;
+
+	*pui64Seconds = OSDivide64r64(ui64OSTimer, 1000000000, &ui32Remainder);
+	*pui64Nanoseconds = ui64OSTimer - (*pui64Seconds * 1000000000ULL);
+}
+
+#if !defined(NO_HARDWARE)
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _RGXMipsExtraDebug(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_MIPS_STATE *psMIPSState)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	IMG_UINT32 ui32RegRead;
+	IMG_UINT32 eError = PVRSRV_OK;
+	IMG_UINT32 volatile *pui32SyncFlag;
+
+	/* Acquire the NMI operations lock */
+	OSLockAcquire(psDevInfo->hNMILock);
+
+	/* Make sure the synchronisation flag is set to 0 */
+	pui32SyncFlag = &psDevInfo->psRGXFWIfSysInit->sMIPSState.ui32Sync;
+	*pui32SyncFlag = 0;
+
+	/* Readback performed as a part of memory barrier */
+	OSWriteMemoryBarrier(pui32SyncFlag);
+	RGXFwSharedMemCacheOpPtr(pui32SyncFlag,
+	                         FLUSH);
+
+
+	/* Enable NMI issuing in the MIPS wrapper */
+	OSWriteHWReg64(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_EN);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_ENABLE);
+
+	/* Check the MIPS is not in error state already (e.g. it is booting or an NMI has already been requested) */
+	ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_EXCEPTION_STATUS);
+	if ((ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN) || (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN))
+	{
+
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Issue NMI */
+	OSWriteHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_EVENT,
+				   RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_EN);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_EVENT);
+
+
+	/* Wait for NMI Taken to be asserted */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+									RGX_CR_MIPS_EXCEPTION_STATUS);
+		if (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if ((ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN) == 0)
+	{
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Allow the firmware to proceed */
+	*pui32SyncFlag = 1;
+
+	/* Readback performed as a part of memory barrier */
+	OSWriteMemoryBarrier(pui32SyncFlag);
+	RGXFwSharedMemCacheOpPtr(pui32SyncFlag,
+	                         FLUSH);
+
+
+	/* Wait for the FW to have finished the NMI routine */
+	ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+								RGX_CR_MIPS_EXCEPTION_STATUS);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+									RGX_CR_MIPS_EXCEPTION_STATUS);
+		if (!(ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	if (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN)
+	{
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Copy state */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->sMIPSState,
+	                           INVALIDATE);
+	OSDeviceMemCopy(psMIPSState, &psDevInfo->psRGXFWIfSysInit->sMIPSState, sizeof(*psMIPSState));
+
+	--(psMIPSState->ui32ErrorEPC);
+	--(psMIPSState->ui32EPC);
+
+	/* Disable NMI issuing in the MIPS wrapper */
+	OSWriteHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE,
+				   0);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_ENABLE);
+
+fail:
+	/* Release the NMI operations lock */
+	OSLockRelease(psDevInfo->hNMILock);
+	return eError;
+}
+
+/* Print decoded information from cause register */
+static void _RGXMipsDumpCauseDecode(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile,
+                                    IMG_UINT32 ui32Cause,
+                                    IMG_UINT32 ui32ErrorState)
+{
+#define INDENT "    "
+	const IMG_UINT32 ui32ExcCode = RGXMIPSFW_C0_CAUSE_EXCCODE(ui32Cause);
+	const IMG_CHAR * const pszException = _GetMIPSExcString(ui32ExcCode);
+
+	if (ui32ErrorState != 0 &&
+	    pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Cause exception: %s", pszException);
+	}
+
+	if (ui32Cause & RGXMIPSFW_C0_CAUSE_FDCIPENDING)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "FDC interrupt pending");
+	}
+
+	if (!(ui32Cause & RGXMIPSFW_C0_CAUSE_IV))
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Interrupt uses general interrupt vector");
+	}
+
+	if (ui32Cause & RGXMIPSFW_C0_CAUSE_PCIPENDING)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Performance Counter Interrupt pending");
+	}
+
+	/* Unusable Coproc exception */
+	if (ui32ExcCode == 11)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Unusable Coprocessor: %d", RGXMIPSFW_C0_CAUSE_UNUSABLE_UNIT(ui32Cause));
+	}
+
+#undef INDENT
+}
+
+static IMG_BOOL _IsFWCodeException(IMG_UINT32 ui32ExcCode)
+{
+	if (ui32ExcCode >= sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Only %lu exceptions available in MIPS, %u is not a valid exception code",
+		         (unsigned long)sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING), ui32ExcCode));
+		return IMG_FALSE;
+	}
+
+	return apsMIPSExcCodes[ui32ExcCode].bIsFatal;
+}
+
+static void _RGXMipsDumpDebugDecode(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32Debug,
+					IMG_UINT32 ui32DEPC)
+{
+	const IMG_CHAR *pszDException = NULL;
+	IMG_UINT32 i;
+#define INDENT "    "
+
+	if (!(ui32Debug & RGXMIPSFW_C0_DEBUG_DM))
+	{
+		return;
+	}
+
+	PVR_DUMPDEBUG_LOG("DEBUG                        :");
+
+	pszDException = _GetMIPSExcString(RGXMIPSFW_C0_DEBUG_EXCCODE(ui32Debug));
+
+	if (pszDException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Debug exception: %s", pszDException);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sMIPS_C0_DebugTable); ++i)
+	{
+		const RGXMIPSFW_C0_DEBUG_TBL_ENTRY * const psDebugEntry = &sMIPS_C0_DebugTable[i];
+
+		if (ui32Debug & psDebugEntry->ui32Mask)
+		{
+			PVR_DUMPDEBUG_LOG(INDENT "%s", psDebugEntry->pszExplanation);
+		}
+	}
+#undef INDENT
+	PVR_DUMPDEBUG_LOG("DEPC                    :0x%08X", ui32DEPC);
+}
+
+static inline void _GetMipsTLBPARanges(const RGX_MIPS_TLB_ENTRY *psTLBEntry,
+                                       const RGX_MIPS_REMAP_ENTRY *psRemapEntry0,
+                                       const RGX_MIPS_REMAP_ENTRY *psRemapEntry1,
+                                       IMG_UINT64 *pui64PA0Start,
+                                       IMG_UINT64 *pui64PA0End,
+                                       IMG_UINT64 *pui64PA1Start,
+                                       IMG_UINT64 *pui64PA1End)
+{
+	IMG_BOOL bUseRemapOutput = (psRemapEntry0 != NULL && psRemapEntry1 != NULL) ? IMG_TRUE : IMG_FALSE;
+	IMG_UINT64 ui64PageSize = RGXMIPSFW_TLB_GET_PAGE_SIZE(psTLBEntry->ui32TLBPageMask);
+
+	if ((psTLBEntry->ui32TLBLo0 & RGXMIPSFW_TLB_VALID) == 0)
+	{
+		/* Dummy values to fail the range checks later */
+		*pui64PA0Start = -1ULL;
+		*pui64PA0End   = -1ULL;
+	}
+	else if (bUseRemapOutput)
+	{
+		*pui64PA0Start = (IMG_UINT64)psRemapEntry0->ui32RemapAddrOut << 12;
+		*pui64PA0End   = *pui64PA0Start + ui64PageSize - 1;
+	}
+	else
+	{
+		*pui64PA0Start = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo0);
+		*pui64PA0End   = *pui64PA0Start + ui64PageSize - 1;
+	}
+
+	if ((psTLBEntry->ui32TLBLo1 & RGXMIPSFW_TLB_VALID) == 0)
+	{
+		/* Dummy values to fail the range checks later */
+		*pui64PA1Start = -1ULL;
+		*pui64PA1End   = -1ULL;
+	}
+	else if (bUseRemapOutput)
+	{
+		*pui64PA1Start = (IMG_UINT64)psRemapEntry1->ui32RemapAddrOut << 12;
+		*pui64PA1End   = *pui64PA1Start + ui64PageSize - 1;
+	}
+	else
+	{
+		*pui64PA1Start = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo1);
+		*pui64PA1End   = *pui64PA1Start + ui64PageSize - 1;
+	}
+}
+
+static void _CheckMipsTLBDuplicatePAs(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                      void *pvDumpDebugFile,
+                                      const RGX_MIPS_TLB_ENTRY *psTLB,
+                                      const RGX_MIPS_REMAP_ENTRY *psRemap)
+{
+	IMG_UINT64 ui64PA0StartI, ui64PA1StartI, ui64PA0StartJ, ui64PA1StartJ;
+	IMG_UINT64 ui64PA0EndI,   ui64PA1EndI,   ui64PA0EndJ,   ui64PA1EndJ;
+	IMG_UINT32 i, j;
+
+#define RANGES_OVERLAP(start0,end0,start1,end1)  ((start0) < (end1) && (start1) < (end0))
+
+	for (i = 0; i < RGXMIPSFW_NUMBER_OF_TLB_ENTRIES; i++)
+	{
+		_GetMipsTLBPARanges(&psTLB[i],
+		                    psRemap ? &psRemap[i] : NULL,
+		                    psRemap ? &psRemap[i + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+		                    &ui64PA0StartI, &ui64PA0EndI,
+		                    &ui64PA1StartI, &ui64PA1EndI);
+
+		for (j = i + 1; j < RGXMIPSFW_NUMBER_OF_TLB_ENTRIES; j++)
+		{
+			_GetMipsTLBPARanges(&psTLB[j],
+			                    psRemap ? &psRemap[j] : NULL,
+			                    psRemap ? &psRemap[j + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+			                    &ui64PA0StartJ, &ui64PA0EndJ,
+			                    &ui64PA1StartJ, &ui64PA1EndJ);
+
+			if (RANGES_OVERLAP(ui64PA0StartI, ui64PA0EndI, ui64PA0StartJ, ui64PA0EndJ) ||
+			    RANGES_OVERLAP(ui64PA0StartI, ui64PA0EndI, ui64PA1StartJ, ui64PA1EndJ) ||
+			    RANGES_OVERLAP(ui64PA1StartI, ui64PA1EndI, ui64PA0StartJ, ui64PA0EndJ) ||
+			    RANGES_OVERLAP(ui64PA1StartI, ui64PA1EndI, ui64PA1StartJ, ui64PA1EndJ)  )
+			{
+				PVR_DUMPDEBUG_LOG("Overlap between TLB entry %u and %u", i , j);
+			}
+		}
+	}
+}
+
+static inline IMG_UINT32 _GetMIPSRemapRegionSize(IMG_UINT32 ui32RegionSizeEncoding)
+{
+    return 1U << ((ui32RegionSizeEncoding + 1U) << 1U);
+}
+
+static inline void _RGXMipsDumpTLBEntry(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                        void *pvDumpDebugFile,
+                                        const RGX_MIPS_TLB_ENTRY *psTLBEntry,
+                                        const RGX_MIPS_REMAP_ENTRY *psRemapEntry0,
+                                        const RGX_MIPS_REMAP_ENTRY *psRemapEntry1,
+                                        IMG_UINT32 ui32Index)
+{
+	IMG_BOOL bDumpRemapEntries = (psRemapEntry0 != NULL && psRemapEntry1 != NULL) ? IMG_TRUE : IMG_FALSE;
+	IMG_UINT64 ui64PA0 = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo0);
+	IMG_UINT64 ui64PA1 = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo1);
+	IMG_UINT64 ui64Remap0AddrOut = 0, ui64Remap1AddrOut = 0;
+	IMG_UINT32 ui32Remap0AddrIn = 0, ui32Remap1AddrIn = 0;
+
+	if (bDumpRemapEntries)
+	{
+		/* RemapAddrIn is always 4k aligned and on 32 bit */
+		ui32Remap0AddrIn = psRemapEntry0->ui32RemapAddrIn << 12;
+		ui32Remap1AddrIn = psRemapEntry1->ui32RemapAddrIn << 12;
+
+		/* RemapAddrOut is always 4k aligned and on 32 or 36 bit */
+		ui64Remap0AddrOut = (IMG_UINT64)psRemapEntry0->ui32RemapAddrOut << 12;
+		ui64Remap1AddrOut = (IMG_UINT64)psRemapEntry1->ui32RemapAddrOut << 12;
+
+		/* If TLB and remap entries match, then merge them else, print them separately */
+		if ((IMG_UINT32)ui64PA0 == ui32Remap0AddrIn &&
+		    (IMG_UINT32)ui64PA1 == ui32Remap1AddrIn)
+		{
+			ui64PA0 = ui64Remap0AddrOut;
+			ui64PA1 = ui64Remap1AddrOut;
+			bDumpRemapEntries = IMG_FALSE;
+		}
+	}
+
+	PVR_DUMPDEBUG_LOG("%2u) VA 0x%08X (%3uk) -> PA0 0x%08" IMG_UINT64_FMTSPECx " %s%s%s, "
+	                                           "PA1 0x%08" IMG_UINT64_FMTSPECx " %s%s%s",
+	                  ui32Index,
+	                  psTLBEntry->ui32TLBHi,
+	                  RGXMIPSFW_TLB_GET_PAGE_SIZE(psTLBEntry->ui32TLBPageMask),
+	                  ui64PA0,
+	                  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(psTLBEntry->ui32TLBLo0)],
+	                  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(psTLBEntry->ui32TLBLo0)],
+	                  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(psTLBEntry->ui32TLBLo0)],
+	                  ui64PA1,
+	                  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(psTLBEntry->ui32TLBLo1)],
+	                  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(psTLBEntry->ui32TLBLo1)],
+	                  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(psTLBEntry->ui32TLBLo1)]);
+
+	if (bDumpRemapEntries)
+	{
+		PVR_DUMPDEBUG_LOG("    Remap %2u : IN 0x%08X (%3uk) => OUT 0x%08" IMG_UINT64_FMTSPECx,
+		                  ui32Index,
+		                  ui32Remap0AddrIn,
+		                  _GetMIPSRemapRegionSize(psRemapEntry0->ui32RemapRegionSize),
+		                  ui64Remap0AddrOut);
+
+		PVR_DUMPDEBUG_LOG("    Remap %2u : IN 0x%08X (%3uk) => OUT 0x%08" IMG_UINT64_FMTSPECx,
+		                  ui32Index + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES,
+		                  ui32Remap1AddrIn,
+		                  _GetMIPSRemapRegionSize(psRemapEntry1->ui32RemapRegionSize),
+		                  ui64Remap1AddrOut);
+	}
+}
+
+static RGX_MIPS_REMAP_ENTRY RGXDecodeMIPSRemap(IMG_UINT64 ui64RemapReg)
+{
+	RGX_MIPS_REMAP_ENTRY sRemapInfo;
+
+	sRemapInfo.ui32RemapAddrIn =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_SHIFT;
+
+	sRemapInfo.ui32RemapAddrOut =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_SHIFT;
+
+	sRemapInfo.ui32RemapRegionSize =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_SHIFT;
+
+	return sRemapInfo;
+}
+
+static void RGXDumpMIPSState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							 void *pvDumpDebugFile,
+							 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	RGX_MIPS_STATE sMIPSState = {0};
+	PVRSRV_ERROR eError;
+
+	eError = _RGXMipsExtraDebug(psDevInfo, &sMIPSState);
+	PVR_DUMPDEBUG_LOG("---- [ MIPS internal state ] ----");
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DUMPDEBUG_LOG("MIPS extra debug not available");
+	}
+	else
+	{
+		DDLOGVAL32("PC", sMIPSState.ui32ErrorEPC);
+		DDLOGVAL32("STATUS_REGISTER", sMIPSState.ui32StatusRegister);
+		DDLOGVAL32("CAUSE_REGISTER", sMIPSState.ui32CauseRegister);
+		_RGXMipsDumpCauseDecode(pfnDumpDebugPrintf, pvDumpDebugFile,
+								sMIPSState.ui32CauseRegister, sMIPSState.ui32ErrorState);
+		DDLOGVAL32("BAD_REGISTER", sMIPSState.ui32BadRegister);
+		DDLOGVAL32("EPC", sMIPSState.ui32EPC);
+		DDLOGVAL32("SP", sMIPSState.ui32SP);
+		DDLOGVAL32("BAD_INSTRUCTION", sMIPSState.ui32BadInstr);
+		_RGXMipsDumpDebugDecode(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+								sMIPSState.ui32Debug, sMIPSState.ui32DEPC);
+
+		{
+			IMG_UINT32 ui32Idx;
+			RGX_MIPS_REMAP_ENTRY *psMipsRemaps = NULL;
+
+			IMG_BOOL bCheckBRN63553WA =
+				RGX_IS_BRN_SUPPORTED(psDevInfo, 63553) &&
+				(OSReadHWReg32(pvRegsBaseKM, RGX_CR_MIPS_ADDR_REMAP5_CONFIG1) == (0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN));
+
+			IMG_BOOL bUseRemapRanges = RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32;
+
+			if (bUseRemapRanges)
+			{
+				psMipsRemaps = OSAllocMem(sizeof(RGX_MIPS_REMAP_ENTRY) * RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES);
+				PVR_LOG_RETURN_VOID_IF_FALSE(psMipsRemaps != NULL, "psMipsRemaps alloc failed.");
+			}
+
+			PVR_DUMPDEBUG_LOG("TLB                           :");
+
+			for (ui32Idx = 0; ui32Idx < ARRAY_SIZE(sMIPSState.asTLB); ui32Idx++)
+			{
+				if (bUseRemapRanges)
+				{
+					psMipsRemaps[ui32Idx] =
+							RGXDecodeMIPSRemap(sMIPSState.aui64Remap[ui32Idx]);
+
+					psMipsRemaps[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] =
+							RGXDecodeMIPSRemap(sMIPSState.aui64Remap[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES]);
+				}
+
+				_RGXMipsDumpTLBEntry(pfnDumpDebugPrintf,
+								     pvDumpDebugFile,
+								     &sMIPSState.asTLB[ui32Idx],
+								     (bUseRemapRanges) ? &psMipsRemaps[ui32Idx] : NULL,
+								     (bUseRemapRanges) ? &psMipsRemaps[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+								     ui32Idx);
+
+				if (bCheckBRN63553WA)
+				{
+					const RGX_MIPS_TLB_ENTRY *psTLBEntry = &sMIPSState.asTLB[ui32Idx];
+
+					#define BRN63553_TLB_IS_NUL(X)  (((X) & RGXMIPSFW_TLB_VALID) && (RGXMIPSFW_TLB_GET_PA(X) == 0x0))
+
+					if (BRN63553_TLB_IS_NUL(psTLBEntry->ui32TLBLo0) || BRN63553_TLB_IS_NUL(psTLBEntry->ui32TLBLo1))
+					{
+						PVR_DUMPDEBUG_LOG("BRN63553 WA present with a valid TLB entry mapping address 0x0.");
+					}
+				}
+			}
+
+			/* This implicitly also checks for overlaps between memory and regbank addresses */
+			_CheckMipsTLBDuplicatePAs(pfnDumpDebugPrintf,
+									  pvDumpDebugFile,
+									  sMIPSState.asTLB,
+									  bUseRemapRanges ? psMipsRemaps : NULL);
+
+			if (bUseRemapRanges)
+			{
+				/* Dump unmapped address if it was dumped in FW, otherwise it will be 0 */
+				if (sMIPSState.ui32UnmappedAddress)
+				{
+					PVR_DUMPDEBUG_LOG("Remap unmapped address => 0x%08X",
+									  sMIPSState.ui32UnmappedAddress);
+				}
+			}
+
+			if (psMipsRemaps != NULL)
+			{
+				OSFreeMem(psMipsRemaps);
+			}
+		}
+
+		/* Check FW code corruption in case of known errors */
+		if (_IsFWCodeException(RGXMIPSFW_C0_CAUSE_EXCCODE(sMIPSState.ui32CauseRegister)))
+		{
+			eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+			}
+		}
+	}
+	PVR_DUMPDEBUG_LOG("--------------------------------");
+}
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+static void _RGXDumpMetaSPExtraDebugInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+/* List of extra META Slave Port debug registers */
+#define RGX_META_SP_EXTRA_DEBUG \
+			X(RGX_CR_META_SP_MSLVCTRL0) \
+			X(RGX_CR_META_SP_MSLVCTRL1) \
+			X(RGX_CR_META_SP_MSLVDATAX) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL)
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+/* Order in these two initialisers and the one above must match */
+#define RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_EQ1_AND_MRUA_ACCESSES \
+			X(RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA)
+
+#define RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_GT1_AND_MRUA_ACCESSES \
+			X(RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA)
+#endif
+
+	IMG_UINT32 ui32Idx;
+	IMG_UINT32 ui32RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+	const IMG_UINT32* pui32DebugRegAddr;
+	const IMG_UINT32 aui32DebugRegAddr[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG
+#undef X
+		};
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	const IMG_UINT32 aui32DebugRegAddrUAHSV1[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_EQ1_AND_MRUA_ACCESSES
+#undef X
+		};
+
+	const IMG_UINT32 aui32DebugRegAddrUAHSGT1[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_GT1_AND_MRUA_ACCESSES
+#undef X
+		};
+#endif
+
+	const IMG_CHAR* apszDebugRegName[] = {
+#define X(A) #A,
+	RGX_META_SP_EXTRA_DEBUG
+#undef X
+	};
+
+	PVR_DUMPDEBUG_LOG("META Slave Port extra debug:");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* array of register offset values depends on feature. But don't augment names in apszDebugRegName */
+	PVR_ASSERT(sizeof(aui32DebugRegAddrUAHSGT1) == sizeof(aui32DebugRegAddr));
+	PVR_ASSERT(sizeof(aui32DebugRegAddrUAHSV1) == sizeof(aui32DebugRegAddr));
+	pui32DebugRegAddr = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+						((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ? (aui32DebugRegAddrUAHSGT1) : (aui32DebugRegAddrUAHSV1)) : aui32DebugRegAddr;
+#else
+	pui32DebugRegAddr = aui32DebugRegAddr;
+#endif
+
+	/* dump set of Slave Port debug registers */
+	for (ui32Idx = 0; ui32Idx < sizeof(aui32DebugRegAddr)/sizeof(IMG_UINT32); ui32Idx++)
+	{
+		const IMG_CHAR* pszRegName = apszDebugRegName[ui32Idx];
+
+		ui32RegAddr = pui32DebugRegAddr[ui32Idx];
+		ui32RegVal = OSReadUncheckedHWReg32(RGX_GET_RISCV_REGS_BASE(psDevInfo), ui32RegAddr);
+		PVR_DUMPDEBUG_LOG("  * %s: 0x%8.8X", pszRegName, ui32RegVal);
+	}
+}
+#endif
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+PVRSRV_ERROR RGXDumpRGXRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(NO_HARDWARE)
+	PVR_DUMPDEBUG_LOG("------[ RGX registers ]------");
+	PVR_DUMPDEBUG_LOG("(Not supported for NO_HARDWARE builds)");
+
+	return PVRSRV_OK;
+#else /* !defined(NO_HARDWARE) */
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32   ui32Meta = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? RGX_GET_FEATURE_VALUE(psDevInfo, META) : 0;
+#endif
+	IMG_BOOL     bFirmwarePerf;
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_DUMPDEBUG_LOG("------[ RGX registers ]------");
+	PVR_DUMPDEBUG_LOG("RGX Register Base Address (Linear):   0x%p", psDevInfo->pvRegsBaseKM);
+	PVR_DUMPDEBUG_LOG("RGX Register Base Address (Physical): 0x%08lX", (unsigned long)psDevInfo->sRegsPhysBase.uiAddr);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Linear): N/A in REE ");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Linear):   0x%p",
+							psDevInfo->pvSecureRegsBaseKM);
+#endif
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Physical): 0x%08lX",
+							(unsigned long)psDevInfo->sRegsPhysBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+#endif
+
+	/* Check if firmware perf was set at Init time */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->eFirmwarePerf,
+	                           INVALIDATE);
+	bFirmwarePerf = (psDevInfo->psRGXFWIfSysInit->eFirmwarePerf != FW_PERF_CONF_NONE);
+
+	RGXDumpCoreRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (ui32Meta)
+	{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		IMG_UINT32 ui32MSlvCtrl1Reg = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+				((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ?
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA :
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA) :
+				RGX_CR_META_SP_MSLVCTRL1;
+
+		/* Forcing bit 6 of MslvCtrl1 to 0 to avoid internal reg read going through the core */
+		OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32MSlvCtrl1Reg, 0x0);
+#else
+		/* Forcing bit 6 of MslvCtrl1 to 0 to avoid internal reg read going through the core */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL1, 0x0);
+#endif
+
+		DDLOG32(META_SP_MSLVIRQSTATUS);
+	}
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		RGXDumpMulticoreRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+	}
+
+	RGXDumpClkRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	DDLOG32(EVENT_STATUS);
+	DDLOG64(TIMER);
+
+	RGXDumpMMURegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	RGXDumpDMRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	DDLOG32(MTS_INTCTX);
+	DDLOG32(MTS_BGCTX);
+	DDLOG32(MTS_BGCTX_COUNTED_SCHEDULE);
+	DDLOG32(MTS_SCHEDULE);
+	DDLOG32(MTS_GPU_INT_STATUS);
+
+	RGXDumpSLCRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	RGXDumpMiscRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	PVR_DUMPDEBUG_LOG("FW Core Registers not available to REE");
+#else
+	if (ui32Meta)
+	{
+		IMG_BOOL bIsT0Enabled = IMG_FALSE, bIsFWFaulted = IMG_FALSE;
+		IMG_UINT32 ui32RegVal;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		IMG_UINT32 ui32MSlvIrqStatusReg = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+				((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ?
+					RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA :
+					RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA) :
+				RGX_CR_META_SP_MSLVIRQSTATUS;
+
+		PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, "META_SP_MSLVIRQSTATUS", OSReadUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32MSlvIrqStatusReg));
+#endif
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0ENABLE_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXENABLE", ui32RegVal);
+		if (ui32RegVal & META_CR_TXENABLE_ENABLE_BIT)
+		{
+			bIsT0Enabled = IMG_TRUE;
+		}
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0STATUS_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXSTATUS", ui32RegVal);
+
+		/* check for FW fault */
+		if (((ui32RegVal >> 20) & 0x3) == 0x2)
+		{
+			bIsFWFaulted = IMG_TRUE;
+		}
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0DEFR_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXDEFR", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_PC, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 PC", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_PCX, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 PCX", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_SP, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 SP", ui32RegVal);
+
+		if ((ui32Meta == MTP218) || (ui32Meta == MTP219))
+		{
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1ENABLE_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXENABLE", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1STATUS_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXSTATUS", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1DEFR_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXDEFR", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_PC, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 PC", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_PCX, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 PCX", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_SP, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 SP", ui32RegVal);
+		}
+
+		if (bFirmwarePerf)
+		{
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_PERF_COUNT0, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("META_CR_PERF_COUNT0", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_PERF_COUNT1, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("META_CR_PERF_COUNT1", ui32RegVal);
+		}
+
+		if (bIsT0Enabled & bIsFWFaulted)
+		{
+			eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+			}
+		}
+		else if (bIsFWFaulted)
+		{
+			PVR_DUMPDEBUG_LOG("Skipping FW code memory corruption checking as META is disabled");
+		}
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		DDLOG32(MIPS_ADDR_REMAP1_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP1_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP2_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP2_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP3_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP3_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP4_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP4_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP5_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP5_CONFIG2);
+		DDLOG64(MIPS_WRAPPER_CONFIG);
+		DDLOG32(MIPS_EXCEPTION_STATUS);
+
+		RGXDumpMIPSState(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+	}
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = RGXDumpRISCVState(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION) */
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		DDLOGVAL32("TFBC_VERSION", RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION));
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		DDLOGVAL32("TFBC_COMPRESSION_CONTROL", psDevInfo->psRGXFWIfSysInit->ui32TFBCCompressionControl);
+	}
+#endif
+
+	return eError;
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+_METASPError:
+	PVR_DUMPDEBUG_LOG("Dump Slave Port debug information");
+	_RGXDumpMetaSPExtraDebugInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	return eError;
+#endif
+#endif /* defined(NO_HARDWARE) */
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDebugRequestProcess
+
+ @Description
+
+ This function will print out the debug for the specified level of verbosity
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+ @Input ui32VerbLevel       - Verbosity level
+
+ @Return   void
+
+******************************************************************************/
+static
+void RGXDebugRequestProcess(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32VerbLevel)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_DEV_POWER_STATE  ePowerState;
+#if defined(SUPPORT_SOC_TIMER)
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#endif
+	IMG_BOOL                bRGXPoweredON;
+	RGXFWIF_TRACEBUF        *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	const RGXFWIF_OSDATA    *psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+	IMG_BOOL                bPwrLockAlreadyHeld;
+	IMG_BOOL                bPwrLockAcquired = IMG_TRUE;
+	IMG_UINT8               ui8FwOsCount;
+
+	bPwrLockAlreadyHeld = PVRSRVPwrLockIsLockedByMe(psDeviceNode);
+	if (!bPwrLockAlreadyHeld)
+	{
+		/* Only acquire the power-lock if not already held by the calling context */
+		eError = PVRSRVPowerTryLockWaitForTimeout(psDeviceNode);
+		if (eError == PVRSRV_ERROR_TIMEOUT)
+		{
+			bPwrLockAcquired = IMG_FALSE;
+#if defined(DEBUG)
+			PVR_DPF((PVR_DBG_WARNING, "Power lock owner: PID = %u at timestamp %" IMG_UINT64_FMTSPEC " (%s:%u)",
+		             psDeviceNode->uiPwrLockOwnerPID, psDeviceNode->sPowerLockOwner.ui64Timestamp,
+		             psDeviceNode->sPowerLockOwner.pszFile, psDeviceNode->sPowerLockOwner.ui32LineNum));
+#endif
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire lock (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+#if !defined(DEBUG_OVERRIDE_PWRLOCK)
+			/* Bail out if another thread has the power lock and may be
+			 * powering down the device. */
+			return;
+#endif
+		}
+		else if (eError != PVRSRV_OK)
+		{
+			return;
+		}
+	}
+	/* This should satisfy all accesses below */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                           INVALIDATE);
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error retrieving RGX power state. No debug info dumped.",
+				__func__));
+		goto Exit;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	{
+		PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						   1, RGX_NUM_DRIVERS_SUPPORTED);
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Device ID:%d Start ]------", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+	bRGXPoweredON = (ePowerState == PVRSRV_DEV_POWER_STATE_ON);
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Info ]------");
+	PVR_DUMPDEBUG_LOG("Device Node (Info): %p (%p)", psDevInfo->psDeviceNode, psDevInfo);
+	DevicememHistoryDumpRecordStats(psDevInfo->psDeviceNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+	PVR_DUMPDEBUG_LOG("RGX BVNC: %d.%d.%d.%d (%s)", psDevInfo->sDevFeatureCfg.ui32B,
+											   psDevInfo->sDevFeatureCfg.ui32V,
+											   psDevInfo->sDevFeatureCfg.ui32N,
+											   psDevInfo->sDevFeatureCfg.ui32C,
+											   PVR_ARCH_NAME);
+	PVR_DUMPDEBUG_LOG("RGX Device State: %s", PVRSRVGetDebugDevStateString(psDeviceNode->eDevState));
+	PVR_DUMPDEBUG_LOG("RGX Power State: %s", _RGXGetDebugDevPowerStateString(ePowerState));
+
+	PVR_DUMPDEBUG_LOG("RGX Health Status: %s, Reason: %s",PVRSRVGetDebugHealthStatusString(OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus)),
+						PVRSRVGetDebugHealthReasonString(OSAtomicRead(&psDevInfo->psDeviceNode->eHealthReason)));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		if (psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated)
+		{
+			PVR_DUMP_FIRMWARE_INFO(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks);
+			ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount);
+			}
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("FW info: UNINITIALIZED");
+		}
+	}
+	else
+	{
+		PVR_DUMP_FIRMWARE_INFO_HDR(psDevInfo->sFWInfoHeader);
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode) && BITMASK_HAS(psDevInfo->sFWInfoHeader.ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psDevInfo->sFWInfoHeader.ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount);
+			}
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TILE_REGION_PROTECTION))
+	{
+#if defined(SUPPORT_TRP)
+		PVR_DUMPDEBUG_LOG("TRP: HW support - Yes; SW enabled");
+#else
+		PVR_DUMPDEBUG_LOG("TRP: HW support - Yes; SW disabled");
+#endif
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("TRP: HW support - No");
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, WORKGROUP_PROTECTION))
+	{
+#if defined(SUPPORT_WGP)
+		PVR_DUMPDEBUG_LOG("WGP: HW support - Yes; SW enabled");
+#else
+		PVR_DUMPDEBUG_LOG("WGP: HW support - Yes; SW disabled");
+#endif
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("WGP: HW support - No");
+	}
+
+#if defined(SUPPORT_SOC_TIMER)
+	if (psDevConfig && psDevConfig->pfnSoCTimerRead)
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*)psDevConfig->hDevData;
+		RGX_TIMING_INFORMATION *psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+		IMG_UINT32 ui32Remainder;
+		IMG_UINT64 ui64CurrentSoCTime = psDevConfig->pfnSoCTimerRead(psDevConfig->hSysData);
+		IMG_UINT64 ui64CurrentSoCTimeInNS =
+			OSDivide64r64(ui64CurrentSoCTime * SECONDS_TO_MICROSECONDS, RGXFWIF_CONVERT_TO_KHZ(psRGXTimingInfo->ui32SOCClockSpeed), &ui32Remainder);
+		IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+		RGXConvertOSTimestampToSAndNS(ui64CurrentSoCTimeInNS, &ui64Seconds, &ui64Nanoseconds);
+		PVR_DUMPDEBUG_LOG("SoC timer counter: 0x%" IMG_UINT64_FMTSPECx, ui64CurrentSoCTime);
+		PVR_DUMPDEBUG_LOG("SoC timer: %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC " seconds",
+				ui64Seconds, ui64Nanoseconds);
+	}
+#endif
+
+	RGXDumpRGXDebugSummary(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, bRGXPoweredON);
+
+	/* Dump out the kernel CCB. */
+	{
+		const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		const RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+
+		if (psKCCBCtl != NULL)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Kernel CCB WO:0x%X RO:0x%X",
+							  psKCCBCtlLocal->ui32WriteOffset,
+							  psKCCBCtl->ui32ReadOffset);
+		}
+	}
+
+	/* Dump out the firmware CCB. */
+	{
+		const RGXFWIF_CCB_CTL *psFCCBCtl = psDevInfo->psFirmwareCCBCtl;
+		const RGXFWIF_CCB_CTL *psFCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psFirmwareCCBCtl, INVALIDATE);
+
+		if (psFCCBCtl != NULL)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Firmware CCB WO:0x%X RO:0x%X",
+							   psFCCBCtl->ui32WriteOffset,
+							   psFCCBCtlLocal->ui32ReadOffset);
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Dump out the Workload estimation CCB. */
+		const RGXFWIF_CCB_CTL *psWorkEstCCBCtl = psDevInfo->psWorkEstFirmwareCCBCtl;
+		const RGXFWIF_CCB_CTL *psWorkEstCCBCtlLocal = psDevInfo->psWorkEstFirmwareCCBCtlLocal;
+
+		if (psWorkEstCCBCtl != NULL)
+		{
+			RGXFwSharedMemCacheOpPtr(psWorkEstCCBCtl, INVALIDATE);
+			PVR_DUMPDEBUG_LOG("RGX WorkEst CCB WO:0x%X RO:0x%X",
+							  psWorkEstCCBCtl->ui32WriteOffset,
+							  psWorkEstCCBCtlLocal->ui32ReadOffset);
+		}
+	}
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psFwOsData,
+	                         INVALIDATE);
+
+	if (psFwOsData != NULL)
+	{
+		/* Dump the KCCB commands executed */
+		PVR_DUMPDEBUG_LOG("RGX Kernel CCB commands executed = %d",
+						  psFwOsData->ui32KCCBCmdsExecuted);
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+		/* Dump the number of times we have performed a forced UFO update,
+		 * and (if non-zero) the timestamp of the most recent occurrence/
+		 */
+		PVR_DUMPDEBUG_LOG("RGX SLR: Forced UFO updates requested = %d",
+						  psFwOsData->ui32ForcedUpdatesRequested);
+		if (psFwOsData->ui32ForcedUpdatesRequested > 0)
+		{
+			IMG_UINT8 ui8Idx;
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			if (psFwOsData->ui64LastForcedUpdateTime > 0ULL)
+			{
+				RGXConvertOSTimestampToSAndNS(psFwOsData->ui64LastForcedUpdateTime, &ui64Seconds, &ui64Nanoseconds);
+				PVR_DUMPDEBUG_LOG("RGX SLR: (most recent forced update was around %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ")",
+								  ui64Seconds, ui64Nanoseconds);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("RGX SLR: (unable to force update as fence contained no sync checkpoints)");
+			}
+			/* Dump SLR log */
+			if (psFwOsData->sSLRLogFirst.aszCCBName[0])
+			{
+				RGXConvertOSTimestampToSAndNS(psFwOsData->sSLRLogFirst.ui64Timestamp, &ui64Seconds, &ui64Nanoseconds);
+				PVR_DUMPDEBUG_LOG("RGX SLR:{%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC
+								  "} Fence found on context 0x%x [%u/%d] '%.*s' has %d UFOs",
+								  ui64Seconds, ui64Nanoseconds,
+								  psFwOsData->sSLRLogFirst.ui32FWCtxAddr,
+								  psDeviceNode->sDevId.ui32InternalID,
+								  psDeviceNode->sDevId.i32KernelDeviceID,
+								  PVR_SLR_LOG_STRLEN, psFwOsData->sSLRLogFirst.aszCCBName,
+								  psFwOsData->sSLRLogFirst.ui32NumUFOs);
+			}
+			for (ui8Idx=0; ui8Idx<PVR_SLR_LOG_ENTRIES;ui8Idx++)
+			{
+				if (psFwOsData->sSLRLog[ui8Idx].aszCCBName[0])
+				{
+					RGXConvertOSTimestampToSAndNS(psFwOsData->sSLRLog[ui8Idx].ui64Timestamp, &ui64Seconds, &ui64Nanoseconds);
+					PVR_DUMPDEBUG_LOG("RGX SLR:[%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC
+									  "] Fence found on context 0x%x '%.*s' has %d UFOs",
+									  ui64Seconds, ui64Nanoseconds,
+									  psFwOsData->sSLRLog[ui8Idx].ui32FWCtxAddr,
+									  PVR_SLR_LOG_STRLEN, psFwOsData->sSLRLog[ui8Idx].aszCCBName,
+									  psFwOsData->sSLRLog[ui8Idx].ui32NumUFOs);
+				}
+			}
+		}
+#else
+		PVR_DUMPDEBUG_LOG("RGX SLR: Disabled");
+#endif
+
+		/* Dump the error counts */
+		PVR_DUMPDEBUG_LOG("RGX Errors: WGP:%d, TRP:%d",
+						  psDevInfo->sErrorCounts.ui32WGPErrorCount,
+						  psDevInfo->sErrorCounts.ui32TRPErrorCount);
+
+		/* Dump the IRQ info for threads or OS IDs */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* only Host has access to registers containing IRQ counters */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+#endif
+		{
+			IMG_UINT32 ui32idx;
+
+			for_each_irq_cnt(ui32idx)
+			{
+				IMG_UINT32 ui32IrqCnt;
+
+				get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+				if (ui32IrqCnt)
+				{
+					PVR_DUMPDEBUG_LOG(MSG_IRQ_CNT_TYPE "%u: FW IRQ count = %u", ui32idx, ui32IrqCnt);
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+					if (ui32idx == RGXFW_HOST_DRIVER_ID)
+#endif
+					{
+						PVR_DUMPDEBUG_LOG("Last sampled IRQ count in LISR = %u", psDevInfo->aui32SampleIRQCount[ui32idx]);
+					}
+				}
+			}
+		}
+	}
+
+	/* Dump the FW Sys config flags on the Host */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+		IMG_CHAR sFwSysFlagsDescription[MAX_FW_DESCRIPTION_LENGTH];
+
+		if (!psFwSysData)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Fw Sys Data is not mapped into CPU space", __func__));
+			goto Exit;
+		}
+
+		RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags,
+		                           INVALIDATE);
+
+		_GetFwSysFlagsDescription(sFwSysFlagsDescription, MAX_FW_DESCRIPTION_LENGTH, psFwSysData->ui32ConfigFlags);
+		PVR_DUMPDEBUG_LOG("FW System config flags = 0x%08X (%s)", psFwSysData->ui32ConfigFlags, sFwSysFlagsDescription);
+	}
+
+	/* Dump the FW OS config flags */
+	{
+		IMG_CHAR sFwOsFlagsDescription[MAX_FW_DESCRIPTION_LENGTH];
+
+		if (!psFwOsData)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Fw Os Data is not mapped into CPU space", __func__));
+			goto Exit;
+		}
+
+		_GetFwOsFlagsDescription(sFwOsFlagsDescription, MAX_FW_DESCRIPTION_LENGTH, psFwOsData->ui32FwOsConfigFlags);
+		PVR_DUMPDEBUG_LOG("FW OS config flags = 0x%08X (%s)", psFwOsData->ui32FwOsConfigFlags, sFwOsFlagsDescription);
+	}
+
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDumpRGXRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: RGXDumpRGXRegisters failed (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG(" (!) %s. No registers dumped", PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? "Guest Mode of operation" : "RGX power is down");
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX FW Trace Info ]------");
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_INT tid;
+		/* Dump FW trace information */
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			RGX_FWT_LOGTYPE eFWTLogType = psDevInfo->eDebugDumpFWTLogType;
+
+			if (eFWTLogType == RGX_FWT_LOGTYPE_NONE)
+			{
+				PVR_DUMPDEBUG_LOG("Firmware trace printing disabled.");
+			}
+			else
+			{
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+
+				for (tid = 0; tid < RGXFW_THREAD_NUM; tid++)
+				{
+					IMG_UINT32 *pui32TraceBuffer;
+
+					if (psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+					{
+						PVR_DUMPDEBUG_LOG("Debug log type: %s ( " RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC ")",
+										  ((psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)?("trace"):("tbi")),
+										  RGXFWIF_LOG_ENABLED_GROUPS_LIST(psRGXFWIfTraceBufCtl->ui32LogType)
+										  );
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("Debug log type: none");
+					}
+
+					pui32TraceBuffer = psDevInfo->apui32TraceBuffer[tid];
+
+					/* Skip if trace buffer is not allocated */
+					if (pui32TraceBuffer == NULL)
+					{
+						PVR_DUMPDEBUG_LOG("RGX FW thread %d: Trace buffer not yet allocated",tid);
+						continue;
+					}
+
+					RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[tid].ui32TracePointer, INVALIDATE);
+					PVR_DUMPDEBUG_LOG("------[ RGX FW thread %d trace START ]------", tid);
+					PVR_DUMPDEBUG_LOG("FWT[traceptr]: %X", psRGXFWIfTraceBufCtl->sTraceBuf[tid].ui32TracePointer);
+					PVR_DUMPDEBUG_LOG("FWT[tracebufsize]: %X", psDevInfo->ui32TraceBufSizeInDWords);
+
+					if (eFWTLogType == RGX_FWT_LOGTYPE_BINARY)
+					{
+						RGXDumpFirmwareTraceBinary(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+					else if (eFWTLogType == RGX_FWT_LOGTYPE_DECODED)
+					{
+						RGXDumpFirmwareTraceDecoded(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+					else if (eFWTLogType == RGX_FWT_LOGTYPE_PARTIAL)
+					{
+						RGXDumpFirmwareTracePartial(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+
+					PVR_DUMPDEBUG_LOG("------[ RGX FW thread %d trace END ]------", tid);
+				}
+			}
+		}
+
+		{
+			if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+			{
+				PVR_DUMPDEBUG_LOG("------[ Full CCB Status ]------");
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("------[ FWCtxs Next CMD ]------");
+			}
+
+			RGXDumpAllContextInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Device ID:%d End ]------", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+Exit:
+	if (!bPwrLockAlreadyHeld && bPwrLockAcquired)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+}
+
+/*!
+ ******************************************************************************
+
+ @Function	RGXDebugRequestNotify
+
+ @Description Dump the debug data for RGX
+
+ ******************************************************************************/
+static void RGXDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDbgRequestHandle,
+		IMG_UINT32 ui32VerbLevel,
+		DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+		void *pvDumpDebugFile)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = hDbgRequestHandle;
+
+	/* Only action the request if we've fully init'ed */
+	if (psDevInfo->bDevInit2Done)
+	{
+		RGXDebugRequestProcess(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, ui32VerbLevel);
+	}
+}
+
+PVRSRV_ERROR RGXDebugInit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return PVRSRVRegisterDeviceDbgRequestNotify(&psDevInfo->hDbgReqNotify,
+		                                        psDevInfo->psDeviceNode,
+		                                        RGXDebugRequestNotify,
+		                                        DEBUG_REQUEST_RGX,
+		                                        psDevInfo);
+}
+
+PVRSRV_ERROR RGXDebugDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (psDevInfo->hDbgReqNotify)
+	{
+		return PVRSRVUnregisterDeviceDbgRequestNotify(psDevInfo->hDbgReqNotify);
+	}
+
+	/* No notifier registered */
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxdebug_common.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.h
new file mode 100644
index 000000000000..f7707b87fa9f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.h
@@ -0,0 +1,433 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX debug header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEBUG_COMMON_H)
+#define RGXDEBUG_COMMON_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "rgxdevice.h"
+#include "rgxfwmemctx.h"
+
+#define DD_NORMAL_INDENT   "    "
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+extern const IMG_CHAR * const gapszMipsPermissionPTFlags[4];
+extern const IMG_CHAR * const gapszMipsCoherencyPTFlags[8];
+extern const IMG_CHAR * const gapszMipsDirtyGlobalValidPTFlags[8];
+#endif
+
+/**
+ * Debug utility macro for printing FW IRQ count and Last sampled IRQ count in
+ * LISR for each RGX FW thread.
+ * Macro takes pointer to PVRSRV_RGXDEV_INFO as input.
+ */
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+#define for_each_irq_cnt(ui32idx)   FOREACH_SUPPORTED_DRIVER(ui32idx)
+
+#define get_irq_cnt_val(ui32Dest, ui32idx, psRgxDevInfo) \
+	do { \
+		extern const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS]; \
+		ui32Dest = PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo) ? 0 : OSReadHWReg32((psRgxDevInfo)->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[ui32idx]); \
+	} while (false)
+
+#define MSG_IRQ_CNT_TYPE "OS"
+
+#else
+
+#define for_each_irq_cnt(ui32idx) \
+	for (ui32idx = 0; ui32idx < RGXFW_THREAD_NUM; ui32idx++)
+
+#define get_irq_cnt_val(ui32Dest, ui32idx, psRgxDevInfo) \
+	do { \
+		RGXFwSharedMemCacheOpValue(psRgxDevInfo->psRGXFWIfFwOsData->aui32InterruptCount[ui32idx], \
+		                           INVALIDATE); \
+		ui32Dest = (psRgxDevInfo)->psRGXFWIfFwOsData->aui32InterruptCount[ui32idx]; \
+	} while (false)
+#define MSG_IRQ_CNT_TYPE "Thread"
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+static inline void RGXDEBUG_PRINT_IRQ_COUNT(PVRSRV_RGXDEV_INFO* psRgxDevInfo)
+{
+#if defined(PVRSRV_NEED_PVR_DPF) && defined(DEBUG)
+	IMG_UINT32 ui32idx;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		IMG_UINT32 ui32IrqCnt;
+
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psRgxDevInfo);
+
+		PVR_DPF((DBGPRIV_VERBOSE, MSG_IRQ_CNT_TYPE
+		         " %u FW IRQ count = %u", ui32idx, ui32IrqCnt));
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		if (ui32idx == RGXFW_HOST_DRIVER_ID)
+#endif
+		{
+			PVR_DPF((DBGPRIV_VERBOSE, "Last sampled IRQ count in LISR = %u",
+			        (psRgxDevInfo)->aui32SampleIRQCount[ui32idx]));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psRgxDevInfo);
+#endif /* PVRSRV_NEED_PVR_DPF */
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpFirmwareTrace
+
+ @Description Dumps the decoded version of the firmware trace buffer.
+
+ Dump useful debugging info
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+
+ @Return   void
+
+******************************************************************************/
+void RGXDumpFirmwareTrace(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO  *psDevInfo);
+
+void RGXDumpFirmwareTraceBinary(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+void RGXDumpFirmwareTracePartial(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+void RGXDumpFirmwareTraceDecoded(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+
+/* Helper macros to emit data */
+#define REG32_FMTSPEC   "%-30s: 0x%08X"
+#define REG64_FMTSPEC   "%-30s: 0x%016" IMG_UINT64_FMTSPECX
+#define DDLOG32(R)      PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, #R, OSReadHWReg32(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOG64(R)      PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadHWReg64(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOGUNCHECKED64(R)      PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadUncheckedHWReg64(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOG32_DPX(R)  PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, #R, OSReadHWReg32(pvRegsBaseKM, DPX_CR_##R));
+#define DDLOG64_DPX(R)  PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadHWReg64(pvRegsBaseKM, DPX_CR_##R));
+#define DDLOGVAL32(S,V) PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, S, V);
+
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpRGXRegisters
+
+ @Description
+
+ Dumps an extensive list of RGX registers required for debugging
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+
+ @Return PVRSRV_ERROR         PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXDumpRGXRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#if !defined(NO_HARDWARE)
+/*!
+*******************************************************************************
+
+ @Function	RGXReadMetaCoreReg
+
+ @Description  Read a META core register's value
+
+ @Input psDevInfo      RGX device info
+ @Input ui32RegAddr    Register address to read from
+ @Output pui32RegVal   Pointer to the resulting register value
+
+ @Return PVRSRV_ERROR  PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXReadMetaCoreReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT32 ui32RegAddr,
+                                IMG_UINT32 *pui32RegVal);
+
+/*!
+*******************************************************************************
+
+ @Function  RGXValidateFWImage
+
+ @Description  Validate the currently running firmware
+               against the firmware image
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo	         - RGX device info
+
+ @Return PVRSRV_ERROR  PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXValidateFWImage(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+/*!
+*******************************************************************************
+
+ @Function     ValidateFWOnLoad
+
+ @Description  Compare the Firmware image as seen from the CPU point of view
+               against the same memory area as seen from the firmware point
+               of view after first power up.
+
+ @Input        psDevInfo - Device Info
+
+ @Return       PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR ValidateFWOnLoad(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpRGXDebugSummary
+
+ @Description
+
+ Dump a summary in human readable form with the RGX state
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo	         - RGX device info
+ @Input bRGXPoweredON        - IMG_TRUE if RGX device is on
+
+ @Return   void
+
+******************************************************************************/
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON);
+
+/*!
+*******************************************************************************
+
+ @Function RGXDebugInit
+
+ @Description
+
+ Setup debug requests, calls into PVRSRVRegisterDeviceDbgRequestNotify
+
+ @Input          psDevInfo            RGX device info
+ @Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+
+******************************************************************************/
+PVRSRV_ERROR RGXDebugInit(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function RGXDebugDeinit
+
+ @Description
+
+ Remove debug requests, calls into PVRSRVUnregisterDeviceDbgRequestNotify
+
+ @Output         phNotify             Points to debug notifier handle
+ @Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+
+******************************************************************************/
+PVRSRV_ERROR RGXDebugDeinit(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetFwMapping
+
+ @Description    Retrieve any of the CPU Physical Address, Device Physical
+                Address or the raw value of the page table entry associated
+                with the firmware virtual address given.
+
+ @Input          psDevInfo           Pointer to device info
+ @Input          pfnDumpDebugPrintf  The debug printf function
+ @Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+ @Input          ui32FwVA            The Fw VA that needs decoding
+ @Output         psCpuPA             Pointer to the resulting CPU PA
+ @Output         psDevPA             Pointer to the resulting Dev PA
+ @Output         pui64PTE            Pointer to the raw Page Table Entry value
+
+ @Return         void
+
+******************************************************************************/
+void RGXDocumentFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          const IMG_UINT32 ui32FwVA,
+                          const IMG_CPU_PHYADDR sCpuPA,
+                          const IMG_DEV_PHYADDR sDevPA,
+                          const IMG_UINT64 ui64PTE);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXConvertOSTimestampToSAndNS
+
+ @Description    Convert the OS time to seconds and nanoseconds
+
+ @Input          ui64OSTimer       OS time to convert
+ @Output         pui64Seconds      Pointer to the resulting seconds
+ @Output         pui64Nanoseconds  Pointer to the resulting nanoseconds
+
+ @Return         void
+
+******************************************************************************/
+void RGXConvertOSTimestampToSAndNS(IMG_UINT64 ui64OSTimer,
+							IMG_UINT64 *pui64Seconds,
+							IMG_UINT64 *pui64Nanoseconds);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDumpAllContextInfo
+
+ @Description    Dump debug info of all contexts on a device
+
+ @Input          psDevInfo           Pointer to device info
+ @Input          pfnDumpDebugPrintf  The debug printf function
+ @Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+ @Input          ui32VerbLevel       Verbosity level
+
+ @Return         void
+
+******************************************************************************/
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpFaultAddressHostView
+
+ @Description
+
+ Dump FW HWR fault status in human readable form.
+
+ @Input ui32Index            - Index of global Fault info
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Return   void
+
+******************************************************************************/
+void RGXDumpFaultAddressHostView(MMU_FAULT_DATA *psFaultData,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const IMG_CHAR* pszIndent);
+
+void RGXDumpFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGX_HWRINFO *psHWRInfo,
+					IMG_UINT32 ui32ReadIndex,
+					IMG_DEV_VIRTADDR *psFaultDevVAddr,
+					IMG_DEV_PHYADDR *psPCDevPAddr,
+					bool bPMFault,
+					IMG_UINT32 ui32PageSize);
+
+#endif /* RGXDEBUG_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.c
new file mode 100644
index 000000000000..62a33fb9995d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.c
@@ -0,0 +1,810 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware common context utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware common context utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxfwcmnctx.h"
+#include "rgxfwutils.h"
+#include "devicemem_pdump.h"
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#include "rogue_trace_events.h"
+#endif
+
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#include "linux/cred.h"
+#include "linux/pid.h"
+#endif
+/*
+ * Maximum length of time a DM can run for before the DM will be marked
+ * as out-of-time. CDM has an increased value due to longer running kernels.
+ *
+ * These deadlines are increased on FPGA, EMU and VP due to the slower
+ * execution time of these platforms. PDUMPS are also included since they
+ * are often run on EMU, FPGA or in CSim.
+ */
+#if defined(FPGA) || defined(EMULATOR) || defined(VIRTUAL_PLATFORM) || defined(PDUMP)
+#define RGXFWIF_MAX_WORKLOAD_DEADLINE_MS     (480000)
+#define RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS (10800000)
+#else
+#define RGXFWIF_MAX_WORKLOAD_DEADLINE_MS     (40000)
+#define RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS (600000)
+#endif
+#define RGXFWIF_MAX_RDM_WORKLOAD_DEADLINE_MS (36000000)
+
+struct _RGX_SERVER_COMMON_CONTEXT_ {
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	DEVMEM_MEMDESC *psFWCommonContextMemDesc;
+	PRGXFWIF_FWCOMMONCONTEXT sFWCommonContextFWAddr;
+	SERVER_MMU_CONTEXT *psServerMMUContext;
+	DEVMEM_MEMDESC *psFWMemContextMemDesc;
+	DEVMEM_MEMDESC *psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC *psContextStateMemDesc;
+	RGX_CLIENT_CCB *psClientCCB;
+	DEVMEM_MEMDESC *psClientCCBMemDesc;
+	DEVMEM_MEMDESC *psClientCCBCtrlMemDesc;
+	IMG_BOOL bCommonContextMemProvided;
+	IMG_UINT32 ui32ContextID;
+	DLLIST_NODE sListNode;
+	RGX_CONTEXT_RESET_REASON eLastResetReason;
+	IMG_UINT32 ui32LastResetJobRef;
+	IMG_INT32 i32Priority;
+	RGX_CCB_REQUESTOR_TYPE eRequestor;
+};
+
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+static IMG_UINT32 _GetUID(IMG_PID pid)
+{
+	struct task_struct *psTask;
+	struct pid *psPid;
+
+	psPid = find_get_pid((pid_t)pid);
+	if (!psPid)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to lookup PID %u.",
+		                        __func__, pid));
+		return 0;
+	}
+
+	psTask = get_pid_task(psPid, PIDTYPE_PID);
+	if (!psTask)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get pid task for PID %u.",
+		                        __func__, pid));
+	}
+	put_pid(psPid);
+
+	return psTask ? from_kuid(&init_user_ns, psTask->cred->uid) : 0;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _CheckPriority
+@Description    Check if priority is allowed for requestor type
+@Input          psDevInfo    pointer to DevInfo struct
+@Input          i32Priority Requested priority
+@Input          eRequestor   Requestor type specifying data master
+@Return         PVRSRV_ERROR PVRSRV_OK on success
+*/ /**************************************************************************/
+static PVRSRV_ERROR _CheckPriority(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_INT32 i32Priority,
+								   RGX_CCB_REQUESTOR_TYPE eRequestor)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Only contexts from a single PID allowed with real time priority (highest priority) */
+	if (i32Priority == RGX_CTX_PRIORITY_REALTIME)
+	{
+		DLLIST_NODE *psNode, *psNext;
+
+		OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+
+		dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+		{
+			RGX_SERVER_COMMON_CONTEXT *psThisContext =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+			if (psThisContext->i32Priority == RGX_CTX_PRIORITY_REALTIME &&
+				psThisContext->eRequestor == eRequestor &&
+				RGXGetPIDFromServerMMUContext(psThisContext->psServerMMUContext) != OSGetCurrentClientProcessIDKM())
+			{
+				PVR_LOG(("Only one process can have contexts with real time priority"));
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				break;
+			}
+		}
+
+		OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
+									 PVRSRV_DEVICE_NODE *psDeviceNode,
+									 RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
+									 RGXFWIF_DM eDM,
+									 SERVER_MMU_CONTEXT *psServerMMUContext,
+									 DEVMEM_MEMDESC *psAllocatedMemDesc,
+									 IMG_UINT32 ui32AllocatedOffset,
+									 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+									 DEVMEM_MEMDESC *psContextStateMemDesc,
+									 IMG_UINT32 ui32CCBAllocSizeLog2,
+									 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+									 IMG_UINT32 ui32ContextFlags,
+									 IMG_INT32 i32Priority,
+									 IMG_UINT32 ui32MaxDeadlineMS,
+									 IMG_UINT64 ui64RobustnessAddress,
+									 RGX_COMMON_CONTEXT_INFO *psInfo,
+									 RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_COMMON_CONTEXT *psServerCommonContext;
+	RGXFWIF_FWCOMMONCONTEXT sFWCommonContext = {{0}};
+	IMG_UINT32 ui32FWCommonContextOffset;
+	IMG_UINT8 *pui8Ptr;
+	PVRSRV_ERROR eError;
+
+	/*
+	 * Allocate all the resources that are required
+	 */
+	psServerCommonContext = OSAllocMem(sizeof(*psServerCommonContext));
+	if (psServerCommonContext == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+
+	psServerCommonContext->psDevInfo = psDevInfo;
+	psServerCommonContext->psServerMMUContext = psServerMMUContext;
+
+	if (psAllocatedMemDesc)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+					 "Using existing MemDesc for Rogue firmware %s context (offset = %d)",
+					 aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+					 ui32AllocatedOffset);
+		ui32FWCommonContextOffset = ui32AllocatedOffset;
+		psServerCommonContext->psFWCommonContextMemDesc = psAllocatedMemDesc;
+		psServerCommonContext->bCommonContextMemProvided = IMG_TRUE;
+	}
+	else
+	{
+		/* Allocate device memory for the firmware context */
+		PDUMPCOMMENT(psDeviceNode,
+					 "Allocate Rogue firmware %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
+		eError = DevmemFwAllocate(psDevInfo,
+								sizeof(sFWCommonContext),
+								RGX_FWCOMCTX_ALLOCFLAGS,
+								"FwContext",
+								&psServerCommonContext->psFWCommonContextMemDesc);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to allocate firmware %s context (%s)",
+			         __func__,
+			         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+			         PVRSRVGetErrorString(eError)));
+			goto fail_contextalloc;
+		}
+		ui32FWCommonContextOffset = 0;
+		psServerCommonContext->bCommonContextMemProvided = IMG_FALSE;
+	}
+
+	/* Record this context so we can refer to it if the FW needs to tell us it was reset. */
+	psServerCommonContext->eLastResetReason    = RGX_CONTEXT_RESET_REASON_NONE;
+	psServerCommonContext->ui32LastResetJobRef = 0;
+	psServerCommonContext->ui32ContextID       = psDevInfo->ui32CommonCtxtCurrentID++;
+
+	/*
+	 * Temporarily map the firmware context to the kernel and initialise it
+	 */
+	eError = DevmemAcquireCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc,
+	                                  (void **)&pui8Ptr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map firmware %s context to CPU (%s)",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         PVRSRVGetErrorString(eError)));
+		goto fail_cpuvirtacquire;
+	}
+
+	/* Allocate the client CCB */
+	eError = RGXCreateCCB(psDevInfo,
+						  ui32CCBAllocSizeLog2,
+						  ui32CCBMaxAllocSizeLog2,
+						  ui32ContextFlags,
+						  psConnection,
+						  eRGXCCBRequestor,
+						  psServerCommonContext,
+						  &psServerCommonContext->psClientCCB,
+						  &psServerCommonContext->psClientCCBMemDesc,
+						  &psServerCommonContext->psClientCCBCtrlMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: failed to create CCB for %s context (%s)",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         PVRSRVGetErrorString(eError)));
+		goto fail_allocateccb;
+	}
+
+	sFWCommonContext.eDM = eDM;
+	BITMASK_SET(sFWCommonContext.ui32CompatFlags, RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT);
+
+	/* Set the firmware CCB device addresses in the firmware common context */
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psCCB,
+						  psServerCommonContext->psClientCCBMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail_cccbfwaddr);
+
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psCCBCtl,
+						  psServerCommonContext->psClientCCBCtrlMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail_cccbctrlfwaddr);
+
+#if defined(RGX_FEATURE_META_DMA_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		RGXSetMetaDMAAddress(&sFWCommonContext.sCCBMetaDMAAddr,
+							 psServerCommonContext->psClientCCBMemDesc,
+							 &sFWCommonContext.psCCB,
+							 0);
+	}
+#endif
+
+	/* Set the memory context device address */
+	psServerCommonContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psFWMemContext,
+						  psFWMemContextMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", fail_fwmemctxfwaddr);
+
+	/* Set the framework register updates address */
+	psServerCommonContext->psFWFrameworkMemDesc = psInfo->psFWFrameworkMemDesc;
+	if (psInfo->psFWFrameworkMemDesc != NULL)
+	{
+		eError = RGXSetFirmwareAddress(&sFWCommonContext.psRFCmd,
+				psInfo->psFWFrameworkMemDesc,
+				0, RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:4", fail_fwframeworkfwaddr);
+	}
+	else
+	{
+		/* This should never be touched in this contexts without a framework
+		 * memdesc, but ensure it is zero so we see crashes if it is.
+		 */
+		sFWCommonContext.psRFCmd.ui32Addr = 0;
+	}
+
+	eError = _CheckPriority(psDevInfo, i32Priority, eRGXCCBRequestor);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CheckPriority", fail_checkpriority);
+
+	psServerCommonContext->i32Priority = i32Priority;
+	psServerCommonContext->eRequestor = eRGXCCBRequestor;
+
+	sFWCommonContext.i32Priority = i32Priority;
+	sFWCommonContext.ui32PrioritySeqNum = 0;
+
+	if (eDM == RGXFWIF_DM_CDM)
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS);
+	}
+	else if (eDM == RGXFWIF_DM_RAY)
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_RDM_WORKLOAD_DEADLINE_MS);
+	}
+	else
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_WORKLOAD_DEADLINE_MS);
+	}
+
+	sFWCommonContext.ui64RobustnessAddress = ui64RobustnessAddress;
+
+	/* Store a references to Server Common Context and PID for notifications back from the FW. */
+	sFWCommonContext.ui32ServerCommonContextID = psServerCommonContext->ui32ContextID;
+	sFWCommonContext.ui32PID                   = OSGetCurrentClientProcessIDKM();
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	sFWCommonContext.ui32UID                   = _GetUID(sFWCommonContext.ui32PID);
+#endif
+	OSStringSafeCopy(sFWCommonContext.szProcName, psConnection->pszProcName, RGXFW_PROCESS_NAME_LEN);
+
+	/* Set the firmware GPU context state buffer */
+	psServerCommonContext->psContextStateMemDesc = psContextStateMemDesc;
+	if (psContextStateMemDesc)
+	{
+		eError = RGXSetFirmwareAddress(&sFWCommonContext.psContextState,
+							  psContextStateMemDesc,
+							  0,
+							  RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:5", fail_ctxstatefwaddr);
+	}
+
+	OSCachedMemCopy(IMG_OFFSET_ADDR(pui8Ptr, ui32FWCommonContextOffset), &sFWCommonContext, sizeof(sFWCommonContext));
+	RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pui8Ptr, ui32FWCommonContextOffset),
+	                          sizeof(sFWCommonContext),
+	                          PVRSRV_CACHE_OP_FLUSH);
+
+	/*
+	 * Dump the created context
+	 */
+	PDUMPCOMMENT(psDeviceNode,
+				 "Dump %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
+	DevmemPDumpLoadMem(psServerCommonContext->psFWCommonContextMemDesc,
+					   ui32FWCommonContextOffset,
+					   sizeof(sFWCommonContext),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* We've finished the setup so release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
+
+	/* Map this allocation into the FW */
+	eError = RGXSetFirmwareAddress(&psServerCommonContext->sFWCommonContextFWAddr,
+						  psServerCommonContext->psFWCommonContextMemDesc,
+						  ui32FWCommonContextOffset,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:6", fail_fwcommonctxfwaddr);
+
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	{
+		IMG_UINT32 ui32FWAddr;
+		switch (eDM) {
+			case RGXFWIF_DM_GEOM:
+				ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
+						psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, sTAContext));
+				break;
+			case RGXFWIF_DM_3D:
+				ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
+						psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, s3DContext));
+				break;
+			default:
+				ui32FWAddr = psServerCommonContext->sFWCommonContextFWAddr.ui32Addr;
+				break;
+		}
+
+		trace_rogue_create_fw_context(OSGetCurrentClientProcessNameKM(),
+									  aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+									  psDeviceNode->sDevId.ui32InternalID,
+									  ui32FWAddr);
+	}
+#endif
+
+	/*Add the node to the list when finalised */
+	OSWRLockAcquireWrite(psDevInfo->hCommonCtxtListLock);
+	dllist_add_to_tail(&(psDevInfo->sCommonCtxtListHead), &(psServerCommonContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hCommonCtxtListLock);
+
+	*ppsServerCommonContext = psServerCommonContext;
+	return PVRSRV_OK;
+
+fail_fwcommonctxfwaddr:
+	if (psContextStateMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psContextStateMemDesc);
+	}
+fail_ctxstatefwaddr:
+fail_checkpriority:
+	if (psInfo->psFWFrameworkMemDesc != NULL)
+	{
+		RGXUnsetFirmwareAddress(psInfo->psFWFrameworkMemDesc);
+	}
+fail_fwframeworkfwaddr:
+	RGXUnsetFirmwareAddress(psFWMemContextMemDesc);
+fail_fwmemctxfwaddr:
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
+fail_cccbctrlfwaddr:
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
+fail_cccbfwaddr:
+	RGXDestroyCCB(psDevInfo, psServerCommonContext->psClientCCB);
+fail_allocateccb:
+	DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
+fail_cpuvirtacquire:
+	if (!psServerCommonContext->bCommonContextMemProvided)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psServerCommonContext->psFWCommonContextMemDesc);
+		psServerCommonContext->psFWCommonContextMemDesc = NULL;
+	}
+fail_contextalloc:
+	OSFreeMem(psServerCommonContext);
+fail_alloc:
+	return eError;
+}
+
+void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+
+	OSWRLockAcquireWrite(psServerCommonContext->psDevInfo->hCommonCtxtListLock);
+	/* Remove the context from the list of all contexts. */
+	dllist_remove_node(&psServerCommonContext->sListNode);
+	OSWRLockReleaseWrite(psServerCommonContext->psDevInfo->hCommonCtxtListLock);
+
+	/*
+		Unmap the context itself and then all its resources
+	*/
+
+	/* Unmap the FW common context */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
+	/* Umap context state buffer (if there was one) */
+	if (psServerCommonContext->psContextStateMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psServerCommonContext->psContextStateMemDesc);
+	}
+	/* Unmap the framework buffer */
+	if (psServerCommonContext->psFWFrameworkMemDesc != NULL)
+	{
+		RGXUnsetFirmwareAddress(psServerCommonContext->psFWFrameworkMemDesc);
+	}
+	/* Unmap client CCB and CCB control */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
+	/* Unmap the memory context */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psFWMemContextMemDesc);
+
+	/* Destroy the client CCB */
+	RGXDestroyCCB(psServerCommonContext->psDevInfo, psServerCommonContext->psClientCCB);
+
+
+	/* Free the FW common context (if there was one) */
+	if (!psServerCommonContext->bCommonContextMemProvided)
+	{
+		DevmemFwUnmapAndFree(psServerCommonContext->psDevInfo,
+						psServerCommonContext->psFWCommonContextMemDesc);
+		psServerCommonContext->psFWCommonContextMemDesc = NULL;
+	}
+	/* Free the hosts representation of the common context */
+	OSFreeMem(psServerCommonContext);
+}
+
+PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->sFWCommonContextFWAddr;
+}
+
+RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psClientCCB;
+}
+
+SERVER_MMU_CONTEXT *FWCommonContextGetServerMMUCtx(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psServerMMUContext;
+}
+
+RGX_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                                               IMG_UINT32 *pui32LastResetJobRef)
+{
+	RGX_CONTEXT_RESET_REASON eLastResetReason;
+
+	PVR_ASSERT(psServerCommonContext != NULL);
+	PVR_ASSERT(pui32LastResetJobRef != NULL);
+
+	/* Take the most recent reason & job ref and reset for next time... */
+	eLastResetReason      = psServerCommonContext->eLastResetReason;
+	*pui32LastResetJobRef = psServerCommonContext->ui32LastResetJobRef;
+	psServerCommonContext->eLastResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+	psServerCommonContext->ui32LastResetJobRef = 0;
+
+	if (eLastResetReason == RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "A Hard Context Switch was triggered on the GPU to ensure Quality of Service."));
+	}
+
+	return eLastResetReason;
+}
+
+PVRSRV_RGXDEV_INFO* FWCommonContextGetRGXDevInfo(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psDevInfo;
+}
+
+PVRSRV_ERROR RGXGetFWCommonContextAddrFromServerMMUCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													   SERVER_MMU_CONTEXT *psServerMMUContext,
+													   PRGXFWIF_FWCOMMONCONTEXT *psFWCommonContextFWAddr)
+{
+	DLLIST_NODE *psNode, *psNext;
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMMON_CONTEXT *psThisContext =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+		if (psThisContext->psServerMMUContext == psServerMMUContext)
+		{
+			psFWCommonContextFWAddr->ui32Addr = psThisContext->sFWCommonContextFWAddr.ui32Addr;
+			return PVRSRV_OK;
+		}
+	}
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+PRGXFWIF_FWCOMMONCONTEXT RGXGetFWCommonContextAddrFromServerCommonCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													                  DLLIST_NODE *psNode)
+{
+	RGX_SERVER_COMMON_CONTEXT *psThisContext =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+	return FWCommonContextGetFWAddress(psThisContext);
+}
+
+PVRSRV_ERROR FWCommonContextSetFlags(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                     IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (BITMASK_ANY(ui32ContextFlags, ~RGX_CONTEXT_FLAGS_WRITEABLE_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Context flag(s) invalid or not writeable (%d)",
+		         __func__, ui32ContextFlags));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else
+	{
+		RGXSetCCBFlags(psServerCommonContext->psClientCCB,
+		               ui32ContextFlags);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
+								CONNECTION_DATA *psConnection,
+								PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_INT32 i32Priority,
+								RGXFWIF_DM eDM)
+{
+	IMG_UINT32				ui32CmdSize;
+	IMG_UINT8				*pui8CmdPtr;
+	RGXFWIF_KCCB_CMD		sPriorityCmd = { 0 };
+	RGXFWIF_CCB_CMD_HEADER	*psCmdHeader;
+	RGXFWIF_CMD_PRIORITY	*psCmd;
+	PVRSRV_ERROR			eError;
+	RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psContext);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = _CheckPriority(psDevInfo, i32Priority, psContext->eRequestor);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CheckPriority", fail_checkpriority);
+
+	/*
+		Get space for command
+	*/
+	ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_CMD_PRIORITY));
+
+	eError = RGXAcquireCCB(psClientCCB,
+						   ui32CmdSize,
+						   (void **) &pui8CmdPtr,
+						   PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire space for client CCB", __func__));
+		}
+		goto fail_ccbacquire;
+	}
+
+	/*
+		Write the command header and command
+	*/
+	psCmdHeader = IMG_OFFSET_ADDR(pui8CmdPtr, 0);
+	psCmdHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PRIORITY;
+	psCmdHeader->ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CMD_PRIORITY));
+	pui8CmdPtr += sizeof(*psCmdHeader);
+
+	psCmd = IMG_OFFSET_ADDR(pui8CmdPtr, 0);
+	psCmd->i32Priority = i32Priority;
+	pui8CmdPtr += sizeof(*psCmd);
+
+	/*
+		We should reserve space in the kernel CCB here and fill in the command
+		directly.
+		This is so if there isn't space in the kernel CCB we can return with
+		retry back to services client before we take any operations
+	*/
+
+	/*
+		Submit the command
+	*/
+	RGXReleaseCCB(psClientCCB,
+				  ui32CmdSize,
+				  PDUMP_FLAGS_CONTINUOUS);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release space in client CCB", __func__));
+		return eError;
+	}
+
+	/* Construct the priority command. */
+	sPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sPriorityCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psContext);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	sPriorityCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									eDM,
+									&sPriorityCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit set priority command with error (%u)",
+				__func__,
+				eError));
+		goto fail_cmdacquire;
+	}
+
+	psContext->i32Priority = i32Priority;
+
+	return PVRSRV_OK;
+
+fail_ccbacquire:
+fail_checkpriority:
+fail_cmdacquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	if (psCurrentServerCommonContext == NULL)
+	{
+		/* the context has already been freed so there is nothing to do here */
+		return PVRSRV_OK;
+	}
+
+	return CheckForStalledCCB(psCurrentServerCommonContext->psDevInfo->psDeviceNode,
+	                          psCurrentServerCommonContext->psClientCCB,
+	                          eKickTypeDM);
+}
+
+void DumpFWCommonContextInfo(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
+                             DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             IMG_UINT32 ui32VerbLevel)
+{
+	if (psCurrentServerCommonContext == NULL)
+	{
+		/* the context has already been freed so there is nothing to do here */
+		return;
+	}
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+		/* If high verbosity requested, dump whole CCB */
+		DumpCCB(psCurrentServerCommonContext->psDevInfo,
+		        psCurrentServerCommonContext->sFWCommonContextFWAddr,
+		        psCurrentServerCommonContext->psClientCCB,
+		        pfnDumpDebugPrintf,
+		        pvDumpDebugFile);
+	}
+	else
+	{
+		/* Otherwise, only dump first command in the CCB */
+		DumpFirstCCBCmd(psCurrentServerCommonContext->sFWCommonContextFWAddr,
+		                      psCurrentServerCommonContext->psClientCCB,
+		                      pfnDumpDebugPrintf,
+		                      pvDumpDebugFile);
+	}
+}
+
+void FWCommonContextListSetLastResetReason(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                           IMG_UINT32 *pui32ErrorPid,
+                                           const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_SERVER_COMMON_CONTEXT *psServerCommonContext = NULL;
+	IMG_UINT32 ui32ErrorPid = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMMON_CONTEXT *psThisContext =
+		    IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+		/* If the notification applies to all contexts update reset info
+		* for all contexts, otherwise only do so for the appropriate ID.
+		*/
+		if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS)
+		{
+			/* Notification applies to all contexts */
+			psThisContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
+			psThisContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+		}
+		else
+		{
+			/* Notification applies to one context only */
+			if (psThisContext->ui32ContextID == psCmdContextResetNotification->ui32ServerCommonContextID)
+			{
+				psServerCommonContext = psThisContext;
+				psServerCommonContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
+				psServerCommonContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+				ui32ErrorPid = RGXGetPIDFromServerMMUContext(psServerCommonContext->psServerMMUContext);
+				break;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+
+	if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: All contexts reset (Reason=%d, JobRef=0x%08x)",
+				__func__,
+				(IMG_UINT32)(psCmdContextResetNotification->eResetReason),
+				psCmdContextResetNotification->ui32ResetJobRef));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Context 0x%p reset (ID=0x%08x, Reason=%d, JobRef=0x%08x)",
+				__func__,
+				psServerCommonContext,
+				psCmdContextResetNotification->ui32ServerCommonContextID,
+				(IMG_UINT32)(psCmdContextResetNotification->eResetReason),
+				psCmdContextResetNotification->ui32ResetJobRef));
+	}
+
+	if (pui32ErrorPid)
+	{
+		*pui32ErrorPid = ui32ErrorPid;
+	}
+}
+
+/******************************************************************************
+ End of file (rgxfwcmnctx.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.h
new file mode 100644
index 000000000000..eba2fac705da
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware common context utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware common context utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWCMNCTX_H
+#define RGXFWCMNCTX_H
+
+#include "connection_server.h"
+#include "device.h"
+#include "rgxccb.h"
+#include "rgx_common.h"
+#include "devicemem_typedefs.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+
+/*************************************************************************/ /*!
+@Function       FWCommonContextAllocate
+
+@Description    Allocate a FW common context. This allocates the HW memory
+                for the context, the CCB and wires it all together.
+
+@Input          psConnection            Connection this context is being created on
+@Input          psDeviceNode            Device node to create the FW context on
+                                        (must be RGX device node)
+@Input          eRGXCCBRequestor        RGX_CCB_REQUESTOR_TYPE enum constant which
+                                        represents the requestor of this FWCC
+@Input          eDM                     Data Master type
+@Input          psServerMMUContext      Server MMU memory context.
+@Input          psAllocatedMemDesc      Pointer to pre-allocated MemDesc to use
+                                        as the FW context or NULL if this function
+                                        should allocate it
+@Input          ui32AllocatedOffset     Offset into pre-allocate MemDesc to use
+                                        as the FW context. If psAllocatedMemDesc
+                                        is NULL then this parameter is ignored
+@Input          psFWMemContextMemDesc   MemDesc of the FW memory context this
+                                        common context resides on
+@Input          psContextStateMemDesc   FW context state (context switch) MemDesc
+@Input          ui32CCBAllocSizeLog2    Size of the CCB for this context
+@Input          ui32CCBMaxAllocSizeLog2 Maximum size to which CCB can grow for this context
+@Input          ui32ContextFlags        Flags which specify properties of the context
+@Input          i32Priority             Priority of the context
+@Input          ui32MaxDeadlineMS       Max deadline limit in MS that the workload can run
+@Input          ui64RobustnessAddress   Address for FW to signal a context reset
+@Input          psInfo                  Structure that contains extra info
+                                        required for the creation of the context
+                                        (elements might change from core to core)
+@Return         PVRSRV_OK if the context was successfully created
+*/ /**************************************************************************/
+PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
+									 PVRSRV_DEVICE_NODE *psDeviceNode,
+									 RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
+									 RGXFWIF_DM eDM,
+									 SERVER_MMU_CONTEXT *psServerMMUContext,
+									 DEVMEM_MEMDESC *psAllocatedMemDesc,
+									 IMG_UINT32 ui32AllocatedOffset,
+									 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+									 DEVMEM_MEMDESC *psContextStateMemDesc,
+									 IMG_UINT32 ui32CCBAllocSizeLog2,
+									 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+									 IMG_UINT32 ui32ContextFlags,
+									 IMG_INT32 i32Priority,
+									 IMG_UINT32 ui32MaxDeadlineMS,
+									 IMG_UINT64 ui64RobustnessAddress,
+									 RGX_COMMON_CONTEXT_INFO *psInfo,
+									 RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext);
+
+
+void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+SERVER_MMU_CONTEXT *FWCommonContextGetServerMMUCtx(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+RGX_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                                           IMG_UINT32 *pui32LastResetJobRef);
+
+PVRSRV_RGXDEV_INFO* FWCommonContextGetRGXDevInfo(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+PVRSRV_ERROR RGXGetFWCommonContextAddrFromServerMMUCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													   SERVER_MMU_CONTEXT *psServerMMUContext,
+													   PRGXFWIF_FWCOMMONCONTEXT *psFWCommonContextFWAddr);
+
+PRGXFWIF_FWCOMMONCONTEXT RGXGetFWCommonContextAddrFromServerCommonCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													                  DLLIST_NODE *psNode);
+
+PVRSRV_ERROR FWCommonContextSetFlags(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                     IMG_UINT32 ui32ContextFlags);
+
+PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
+								CONNECTION_DATA *psConnection,
+								PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_INT32 i32Priority,
+								RGXFWIF_DM eDM);
+
+PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM);
+
+void DumpFWCommonContextInfo(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
+                             DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             IMG_UINT32 ui32VerbLevel);
+
+void FWCommonContextListSetLastResetReason(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                           IMG_UINT32 *pui32ErrorPid,
+                                           const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification);
+
+#endif /* RGXFWCMNCTX_H */
+/******************************************************************************
+ End of file (rgxfwcmnctx.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.c
new file mode 100644
index 000000000000..2271adf3ea9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.c
@@ -0,0 +1,666 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debugging and miscellaneous functions server implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel services functions for debugging and other
+                miscellaneous functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvr_debug.h"
+#include "rgxfwdbg.h"
+#include "rgxfwutils.h"
+#include "rgxta3d.h"
+#include "pdump_km.h"
+#include "mmu_common.h"
+#include "devicemem_server.h"
+#include "osfunc.h"
+#include "vmm_pvz_server.h"
+#include "vz_vm.h"
+#if defined(PDUMP)
+#include "devicemem_pdump.h"
+#endif
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugQueryFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 *pui32RGXFWLogType)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!psDeviceNode || !pui32RGXFWLogType)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+	psDevInfo = psDeviceNode->pvDevice;
+
+	if (!psDevInfo || !psDevInfo->psRGXFWIfTraceBufCtl)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+	*pui32RGXFWLogType = psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType;
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetFWLogKM(
+	const CONNECTION_DATA * psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32RGXFWLogType)
+{
+	RGXFWIF_KCCB_CMD sLogTypeUpdateCmd;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevInfo;
+	IMG_UINT32 ui32OldRGXFWLogTpe;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo = psDeviceNode->pvDevice;
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+	ui32OldRGXFWLogTpe = psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType;
+
+	/* check log type is valid */
+	if (ui32RGXFWLogType & ~RGXFWIF_LOG_TYPE_MASK)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquire(psDevInfo->hRGXFWIfBufInitLock);
+
+	/* set the new log type and ensure the new log type is written to memory
+	 * before requesting the FW to read it
+	 */
+	psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = ui32RGXFWLogType;
+	OSMemoryBarrier(&psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+
+	/* Allocate firmware trace buffer resource(s) if not already done */
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo, RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	/* Check if LogType is TBI then allocate resource on demand and copy
+	 * SFs to it
+	 */
+	else if (RGXTBIBufferIsInitRequired(psDevInfo))
+	{
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+	}
+
+	/* TBI buffer address will be 0 if not initialised */
+	sLogTypeUpdateCmd.uCmdData.sTBIBuffer = psDevInfo->sRGXFWIfTBIBuffer;
+#else
+	sLogTypeUpdateCmd.uCmdData.sTBIBuffer.ui32Addr = 0;
+#endif
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate resource on-demand. Reverting to old value",
+		         __func__));
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = ui32OldRGXFWLogTpe;
+		OSMemoryBarrier(&psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType);
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+
+		OSLockRelease(psDevInfo->hRGXFWIfBufInitLock);
+
+		return eError;
+	}
+
+	OSLockRelease(psDevInfo->hRGXFWIfBufInitLock);
+
+	eError = PVRSRVPowerLock((PPVRSRV_DEVICE_NODE) psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire power lock (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState((PPVRSRV_DEVICE_NODE) psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of logType value */
+		sLogTypeUpdateCmd.eCmdType = RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sLogTypeUpdateCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock( (PPVRSRV_DEVICE_NODE) psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the LogType value to be updated in FW */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetHCSDeadlineKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32HCSDeadlineMS)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWSetHCSDeadline(psDevInfo, ui32HCSDeadlineMS);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugMapGuestHeapKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32DriverID,
+	IMG_UINT64 ui64GuestHeapBase)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID = psDeviceNode->sDevId.ui32InternalID;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		if (ui64GuestHeapBase == IMG_UINT64_MAX)
+		{
+			/* unmap heap and set DriverID to offline */
+			eError = PvzServerUnmapDevPhysHeap(ui32DriverID, ui32DeviceID);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap()");
+			eError = PvzServerOnVmOffline(ui32DriverID, ui32DeviceID);
+		}
+		else
+		{
+			/* set DriverID online if necessary and map firmware heap */
+			if (!IsVmOnline(ui32DriverID, ui32DeviceID))
+			{
+				eError = PvzServerOnVmOnline(ui32DriverID, ui32DeviceID);
+				PVR_LOG_RETURN_IF_ERROR(eError, "PvzServerOnVmOnline()");
+			}
+
+			eError = PvzServerMapDevPhysHeap(ui32DriverID, ui32DeviceID, RGX_FIRMWARE_RAW_HEAP_SIZE, ui64GuestHeapBase);
+		}
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		PVR_DPF((PVR_DBG_ERROR, " %s: Driver must run in Host mode to support Guest Mapping operations\n", __func__));
+	}
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32DriverID);
+	PVR_UNREFERENCED_PARAMETER(ui64GuestHeapBase);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32TSIntervalMs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzTSIntervalCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	sVzTSIntervalCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs = ui32TSIntervalMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the timeslice interval inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32TSIntervalMs),
+							  ui32TSIntervalMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzTSIntervalCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32TSPercentage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sVzTimeSliceCmd = { 0 };
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_INT32 ui32TSPercentageMax = 0;
+	IMG_UINT32 ui32DriverIDLoop;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_NOT_INITIALISED);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, PVRSRV_ERROR_NOT_INITIALISED);
+
+	/*
+	 * Each time slice is a number between 0 -> 100.
+	 * Use '0' to disable time slice based CSW for the driver.
+	 */
+	 /* Check if the sum exceeds PVRSRV_VZ_TIME_SLICE_MAX */
+	if (ui32TSPercentage)
+	{
+		PVR_RETURN_IF_FALSE(ui32TSPercentage <= PVRSRV_VZ_TIME_SLICE_MAX, PVRSRV_ERROR_INVALID_PARAMS);
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverIDLoop)
+		{
+			if (ui32DriverID != ui32DriverIDLoop)
+			{
+				ui32TSPercentageMax += psRuntimeCfg->aui32TSPercentage[ui32DriverIDLoop];
+			}
+			else
+			{
+				ui32TSPercentageMax += ui32TSPercentage;
+			}
+
+			PVR_RETURN_IF_FALSE(ui32TSPercentageMax <= PVRSRV_VZ_TIME_SLICE_MAX, PVRSRV_ERROR_INVALID_PARAMS);
+		}
+	}
+
+	sVzTimeSliceCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE;
+	psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] = ui32TSPercentage;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the timeslice of DriverID %u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, aui32TSPercentage) + (ui32DriverID * sizeof(ui32TSPercentage)),
+							  ui32TSPercentage,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzTimeSliceCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverPriorityKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_INT32   i32DriverPriority)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzPriorityCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((i32DriverPriority & ~RGXFW_VZ_PRIORITY_MASK) != 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sVzPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE;
+	psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID] = i32DriverPriority;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the priority of DriverID %u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ai32DriverPriority) + (ui32DriverID * sizeof(i32DriverPriority)),
+							  i32DriverPriority,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzPriorityCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverIsolationGroupKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32DriverIsolationGroup)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzIsolationGroupCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sVzIsolationGroupCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE;
+	psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = ui32DriverIsolationGroup;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the isolation group of DriverID%u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, aui32DriverIsolationGroup) + (ui32DriverID * sizeof(ui32DriverIsolationGroup)),
+							  ui32DriverIsolationGroup,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+				RGXFWIF_DM_GP,
+				&sVzIsolationGroupCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetOSNewOnlineStateKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32OSNewState)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_OS_STATE_CHANGE eState;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eState = (ui32OSNewState) ? (RGXFWIF_OS_ONLINE) : (RGXFWIF_OS_OFFLINE);
+	return RGXFWSetFwOsState(psDevInfo, ui32DriverID, eState);
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32DriverID);
+	PVR_UNREFERENCED_PARAMETER(ui32OSNewState);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPHRConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWConfigPHR(psDevInfo,
+	                      ui32PHRMode);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugWdgConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWConfigWdg(psDevInfo,
+	                      ui32WdgPeriodUs);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugDumpFreelistPageListKM(
+	CONNECTION_DATA * psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (dllist_is_empty(&psDevInfo->sFreeListHead))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG(("---------------[ Begin Freelist Page List Dump ]------------------"));
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		RGXDumpFreeListPageList(psFreeList);
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	PVR_LOG(("----------------[ End Freelist Page List Dump ]-------------------"));
+
+	return PVRSRV_OK;
+
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInjectFaultKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWInjectFault(psDevInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOffKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+#if defined(SUPPORT_AUTOVZ)
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	return PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+					       PVRSRV_SYS_POWER_STATE_OFF,
+					       PVRSRV_POWER_FLAGS_NONE);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOnKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+					       PVRSRV_SYS_POWER_STATE_ON,
+					       PVRSRV_POWER_FLAGS_NONE);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWSetVzConnectionCooldownPeriod(psDevInfo, ui32VzConnectionCooldownPeriodInSec);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.h
new file mode 100644
index 000000000000..59c48ed5f4f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.h
@@ -0,0 +1,160 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debugging and miscellaneous functions server interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel services functions for debugging and other
+                miscellaneous functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXFWDBG_H)
+#define RGXFWDBG_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "device.h"
+#include "pmr.h"
+
+#include "connection_server.h"
+
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInitFWImageKM(
+	PMR *psFWImgDestPMR,
+	PMR *psFWImgSrcPMR,
+	IMG_UINT64 ui64FWImgLen,
+	PMR *psFWImgSigPMR,
+	IMG_UINT64 ui64FWSigLen);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugQueryFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 *pui32RGXFWLogType);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32RGXFWLogType);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetHCSDeadlineKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32HCSDeadlineMS);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32TSPercentage);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32TSIntervalMs);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverPriorityKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_INT32   i32DriverPriority);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverIsolationGroupKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32DriverIsolationGroup);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetOSNewOnlineStateKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32OSNewState);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugMapGuestHeapKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32DriverID,
+	IMG_UINT64 ui64GuestHeapBase);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPHRConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32PHRMode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugWdgConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32WdgPeriodUs);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugDumpFreelistPageListKM(
+	CONNECTION_DATA * psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInjectFaultKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOffKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOnKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32VzConnectionCooldownPeriodInSec);
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.c
new file mode 100644
index 000000000000..afc69f7228f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.c
@@ -0,0 +1,1165 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Firmware image utilities used at init time
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services Firmware image utilities used at init time
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxfwimageutils.h"
+#include "pvrversion.h"
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+#include "rgxfwutils.h"
+#endif
+
+/************************************************************************
+* FW layout information
+************************************************************************/
+#define MAX_NUM_ENTRIES (8)
+static RGX_FW_LAYOUT_ENTRY asRGXFWLayoutTable[MAX_NUM_ENTRIES];
+static IMG_UINT32 ui32LayoutEntryNum;
+
+
+static RGX_FW_LAYOUT_ENTRY* GetTableEntry(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		if (asRGXFWLayoutTable[i].eId == eId)
+		{
+			return &asRGXFWLayoutTable[i];
+		}
+	}
+
+	RGXErrorLog(hPrivate, "%s: id %u not found, returning entry 0\n",
+	            __func__, eId);
+
+	return &asRGXFWLayoutTable[0];
+}
+
+/*!
+*******************************************************************************
+
+ @Function      FindMMUSegment
+
+ @Description   Given a 32 bit FW address attempt to find the corresponding
+                pointer to FW allocation
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         ui32OffsetIn             : 32 bit FW address
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem code
+ @Input         uiHostAddrOut            : CPU pointer equivalent to ui32OffsetIn
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR FindMMUSegment(const void *hPrivate,
+                                   IMG_UINT32 ui32OffsetIn,
+                                   void *pvHostFWCodeAddr,
+                                   void *pvHostFWDataAddr,
+                                   void *pvHostFWCorememCodeAddr,
+                                   void *pvHostFWCorememDataAddr,
+                                   void **uiHostAddrOut)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		if ((ui32OffsetIn >= asRGXFWLayoutTable[i].ui32BaseAddr) &&
+		    (ui32OffsetIn < (asRGXFWLayoutTable[i].ui32BaseAddr + asRGXFWLayoutTable[i].ui32AllocSize)))
+		{
+			switch (asRGXFWLayoutTable[i].eType)
+			{
+				case FW_CODE:
+					*uiHostAddrOut = pvHostFWCodeAddr;
+					break;
+
+				case FW_DATA:
+					if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+					{
+						*uiHostAddrOut = RGXCalculateHostFWDataAddress(hPrivate, pvHostFWDataAddr);
+					}
+					else
+					{
+						*uiHostAddrOut = pvHostFWDataAddr;
+					}
+					break;
+
+				case FW_COREMEM_CODE:
+					*uiHostAddrOut = pvHostFWCorememCodeAddr;
+					break;
+
+				case FW_COREMEM_DATA:
+					*uiHostAddrOut = pvHostFWCorememDataAddr;
+					break;
+
+				default:
+					return PVRSRV_ERROR_INIT_FAILURE;
+			}
+
+			goto found;
+		}
+	}
+
+	return PVRSRV_ERROR_INIT_FAILURE;
+
+found:
+	if (*uiHostAddrOut == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Add offset to pointer to FW allocation now that allocation is found */
+
+	/* Direct Mem write to mapped memory */
+	ui32OffsetIn -= asRGXFWLayoutTable[i].ui32BaseAddr;
+	ui32OffsetIn += asRGXFWLayoutTable[i].ui32AllocOffset;
+
+	*(IMG_UINT8 **)uiHostAddrOut += ui32OffsetIn;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureSegID
+
+ @Description   Configures a single segment of the Segment MMU
+                (base, limit and out_addr)
+
+ @Input         hPrivate        : Implementation specific data
+ @Input         ui64SegOutAddr  : Segment output base address (40 bit devVaddr)
+ @Input         ui32SegBase     : Segment input base address (32 bit FW address)
+ @Input         ui32SegLimit    : Segment size
+ @Input         ui32SegID       : Segment ID
+ @Input         pszName         : Segment name
+ @Input         ppui32BootConf  : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureSegID(const void *hPrivate,
+                                IMG_UINT64 ui64SegOutAddr,
+                                IMG_UINT32 ui32SegBase,
+                                IMG_UINT32 ui32SegLimit,
+                                IMG_UINT32 ui32SegID,
+                                IMG_UINT32 **ppui32BootConf)
+{
+	IMG_UINT32 *pui32BootConf = *ppui32BootConf;
+	IMG_UINT32 ui32SegOutAddr0 = ui64SegOutAddr & 0x00000000FFFFFFFFUL;
+	IMG_UINT32 ui32SegOutAddr1 = (ui64SegOutAddr >> 32) & 0x00000000FFFFFFFFUL;
+
+	/* META segments have a minimum size */
+	IMG_UINT32 ui32LimitOff = (ui32SegLimit < RGXFW_SEGMMU_ALIGN) ?
+	                          RGXFW_SEGMMU_ALIGN : ui32SegLimit;
+	/* the limit is an offset, therefore off = size - 1 */
+	ui32LimitOff -= 1;
+
+	RGXCommentLog(hPrivate,
+	              "* Seg%d: meta_addr = 0x%08x, devv_addr = 0x%" IMG_UINT64_FMTSPECx ", limit = 0x%x",
+	              ui32SegID,
+	              ui32SegBase,
+	              ui64SegOutAddr,
+	              ui32LimitOff);
+
+	ui32SegBase |= RGXFW_SEGMMU_ALLTHRS_WRITEABLE;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_BASE(ui32SegID);
+	*pui32BootConf++ = ui32SegBase;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_LIMIT(ui32SegID);
+	*pui32BootConf++ = ui32LimitOff;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_OUTA0(ui32SegID);
+	*pui32BootConf++ = ui32SegOutAddr0;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_OUTA1(ui32SegID);
+	*pui32BootConf++ = ui32SegOutAddr1;
+
+	*ppui32BootConf = pui32BootConf;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureSegMMU
+
+ @Description   Configures META's Segment MMU
+
+ @Input         hPrivate             : Implementation specific data
+ @Input         psFWCodeDevVAddrBase : FW code base device virtual address
+ @Input         psFWDataDevVAddrBase : FW data base device virtual address
+ @Input         ppui32BootConf       : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureSegMMU(const void       *hPrivate,
+                                 IMG_DEV_VIRTADDR *psFWCodeDevVAddrBase,
+                                 IMG_DEV_VIRTADDR *psFWDataDevVAddrBase,
+                                 IMG_UINT32       **ppui32BootConf)
+{
+	IMG_UINT64 ui64SegOutAddrTop;
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(psFWCodeDevVAddrBase);
+
+	/* Configure Segment MMU */
+	RGXCommentLog(hPrivate, "********** FW configure Segment MMU **********");
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SLC_VIVT))
+	{
+		ui64SegOutAddrTop = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(MMU_CONTEXT_MAPPING_FWPRIV);
+	}
+	else
+#endif
+	{
+		ui64SegOutAddrTop = RGXFW_SEGMMU_OUTADDR_TOP_SLC(MMU_CONTEXT_MAPPING_FWPRIV, RGXFW_SEGMMU_META_BIFDM_ID);
+	}
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		/*
+		 * FW code is using the bootloader segment which is already configured on boot.
+		 * FW coremem code and data don't use the segment MMU.
+		 * Only the FW data segment needs to be configured.
+		 */
+
+		if (asRGXFWLayoutTable[i].eType == FW_DATA)
+		{
+			IMG_UINT64 ui64SegOutAddr;
+			IMG_UINT32 ui32SegId = RGXFW_SEGMMU_DATA_ID;
+
+			ui64SegOutAddr = (psFWDataDevVAddrBase->uiAddr | ui64SegOutAddrTop) +
+			                  asRGXFWLayoutTable[i].ui32AllocOffset;
+
+			RGXFWConfigureSegID(hPrivate,
+			                    ui64SegOutAddr,
+			                    asRGXFWLayoutTable[i].ui32BaseAddr,
+			                    asRGXFWLayoutTable[i].ui32AllocSize,
+			                    ui32SegId,
+			                    ppui32BootConf); /*write the sequence to the bootldr */
+
+			break;
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureMetaCaches
+
+ @Description   Configure and enable the Meta instruction and data caches
+
+ @Input         hPrivate          : Implementation specific data
+ @Input         ui32NumThreads    : Number of FW threads in use
+ @Input         ppui32BootConf    : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureMetaCaches(const void *hPrivate,
+                                     IMG_UINT32 ui32NumThreads,
+                                     IMG_UINT32 **ppui32BootConf)
+{
+	IMG_UINT32 *pui32BootConf = *ppui32BootConf;
+	IMG_UINT32 ui32DCacheT0, ui32ICacheT0;
+	IMG_UINT32 ui32DCacheT1, ui32ICacheT1;
+	IMG_UINT32 ui32DCacheT2, ui32ICacheT2;
+	IMG_UINT32 ui32DCacheT3, ui32ICacheT3;
+
+#define META_CR_MMCU_LOCAL_EBCTRL                        (0x04830600)
+#define META_CR_MMCU_LOCAL_EBCTRL_ICWIN                  (0x3 << 14)
+#define META_CR_MMCU_LOCAL_EBCTRL_DCWIN                  (0x3 << 6)
+#define META_CR_SYSC_DCPART(n)                           (0x04830200 + (n)*0x8)
+#define META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE         (0x1 << 31)
+#define META_CR_SYSC_ICPART(n)                           (0x04830220 + (n)*0x8)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF  (0x8 << 16)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE       (0xF)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE       (0x7)
+#define META_CR_MMCU_DCACHE_CTRL                         (0x04830018)
+#define META_CR_MMCU_ICACHE_CTRL                         (0x04830020)
+#define META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN           (0x1)
+
+	RGXCommentLog(hPrivate, "********** Meta caches configuration *********");
+
+	/* Initialise I/Dcache settings */
+	ui32DCacheT0 = ui32DCacheT1 = (IMG_UINT32)META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE;
+	ui32DCacheT2 = ui32DCacheT3 = (IMG_UINT32)META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE;
+	ui32ICacheT0 = ui32ICacheT1 = ui32ICacheT2 = ui32ICacheT3 = 0;
+
+	if (ui32NumThreads == 1)
+	{
+		ui32DCacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE;
+		ui32ICacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE;
+	}
+	else
+	{
+		ui32DCacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE;
+		ui32ICacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE;
+
+		ui32DCacheT1 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE |
+		                META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF;
+		ui32ICacheT1 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE |
+		                META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF;
+	}
+
+	/* Local region MMU enhanced bypass: WIN-3 mode for code and data caches */
+	*pui32BootConf++ = META_CR_MMCU_LOCAL_EBCTRL;
+	*pui32BootConf++ = META_CR_MMCU_LOCAL_EBCTRL_ICWIN |
+	                   META_CR_MMCU_LOCAL_EBCTRL_DCWIN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_LOCAL_EBCTRL,
+	              META_CR_MMCU_LOCAL_EBCTRL_ICWIN | META_CR_MMCU_LOCAL_EBCTRL_DCWIN);
+
+	/* Data cache partitioning thread 0 to 3 */
+	*pui32BootConf++ = META_CR_SYSC_DCPART(0);
+	*pui32BootConf++ = ui32DCacheT0;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(1);
+	*pui32BootConf++ = ui32DCacheT1;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(2);
+	*pui32BootConf++ = ui32DCacheT2;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(3);
+	*pui32BootConf++ = ui32DCacheT3;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(0), ui32DCacheT0);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(1), ui32DCacheT1);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(2), ui32DCacheT2);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(3), ui32DCacheT3);
+
+	/* Enable data cache hits */
+	*pui32BootConf++ = META_CR_MMCU_DCACHE_CTRL;
+	*pui32BootConf++ = META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_DCACHE_CTRL,
+	              META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN);
+
+	/* Instruction cache partitioning thread 0 to 3 */
+	*pui32BootConf++ = META_CR_SYSC_ICPART(0);
+	*pui32BootConf++ = ui32ICacheT0;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(1);
+	*pui32BootConf++ = ui32ICacheT1;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(2);
+	*pui32BootConf++ = ui32ICacheT2;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(3);
+	*pui32BootConf++ = ui32ICacheT3;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(0), ui32ICacheT0);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(1), ui32ICacheT1);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(2), ui32ICacheT2);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(3), ui32ICacheT3);
+
+	/* Enable instruction cache hits */
+	*pui32BootConf++ = META_CR_MMCU_ICACHE_CTRL;
+	*pui32BootConf++ = META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_ICACHE_CTRL,
+	              META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN);
+
+	*pui32BootConf++ = 0x040000C0;
+	*pui32BootConf++ = 0;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x", 0x040000C0, 0);
+
+	*ppui32BootConf = pui32BootConf;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessLDRCommandStream
+
+ @Description   Process the output of the Meta toolchain in the .LDR format
+                copying code and data sections into their final location and
+                passing some information to the Meta bootloader
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbLDR                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+ @Input         ppui32BootConf           : Pointer to bootloader data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessLDRCommandStream(const void *hPrivate,
+                                     const IMG_BYTE* pbLDR,
+                                     void* pvHostFWCodeAddr,
+                                     void* pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr,
+                                     IMG_UINT32 **ppui32BootConf)
+{
+	RGX_META_LDR_BLOCK_HDR *psHeader = (RGX_META_LDR_BLOCK_HDR *) pbLDR;
+	RGX_META_LDR_L1_DATA_BLK *psL1Data =
+	    (RGX_META_LDR_L1_DATA_BLK*) ((IMG_UINT8 *) pbLDR + psHeader->ui32SLData);
+
+	IMG_UINT32 *pui32BootConf  = ppui32BootConf ? *ppui32BootConf : NULL;
+	IMG_UINT32 ui32CorememSize = RGXGetFWCorememSize(hPrivate);
+
+	RGXCommentLog(hPrivate, "**********************************************");
+	RGXCommentLog(hPrivate, "************** Begin LDR Parsing *************");
+	RGXCommentLog(hPrivate, "**********************************************");
+
+	while (psL1Data != NULL)
+	{
+		if (RGX_META_LDR_BLK_IS_COMMENT(psL1Data->ui16Cmd))
+		{
+			/* Don't process comment blocks */
+			goto NextBlock;
+		}
+
+		switch (psL1Data->ui16Cmd & RGX_META_LDR_CMD_MASK)
+		{
+			case RGX_META_LDR_CMD_LOADMEM:
+			{
+				RGX_META_LDR_L2_DATA_BLK *psL2Block =
+				    (RGX_META_LDR_L2_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->aui32CmdData[1]);
+				IMG_UINT32 ui32Offset = psL1Data->aui32CmdData[0];
+				IMG_UINT32 ui32DataSize = psL2Block->ui16Length - 6 /* L2 Tag length and checksum */;
+				void *pvWriteAddr;
+				PVRSRV_ERROR eError;
+
+				if (!RGX_META_IS_COREMEM_CODE(ui32Offset, ui32CorememSize) &&
+				    !RGX_META_IS_COREMEM_DATA(ui32Offset, ui32CorememSize))
+				{
+					/* Global range is aliased to local range */
+					ui32Offset &= ~META_MEM_GLOBAL_RANGE_BIT;
+				}
+
+				eError = FindMMUSegment(hPrivate,
+				                        ui32Offset,
+				                        pvHostFWCodeAddr,
+				                        pvHostFWDataAddr,
+				                        pvHostFWCorememCodeAddr,
+				                        pvHostFWCorememDataAddr,
+				                        &pvWriteAddr);
+
+				if (eError != PVRSRV_OK)
+				{
+					RGXErrorLog(hPrivate,
+					            "ProcessLDRCommandStream: Addr 0x%x (size: %d) not found in any segment",
+					            ui32Offset, ui32DataSize);
+					return eError;
+				}
+
+				/* Write to FW allocation only if available */
+				if (pvWriteAddr)
+				{
+					RGXMemCopy(hPrivate,
+					           pvWriteAddr,
+					           psL2Block->aui32BlockData,
+					           ui32DataSize);
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+					RGXFwSharedMemCacheOpExec(pvWriteAddr, ui32DataSize, PVRSRV_CACHE_OP_FLUSH);
+#endif
+				}
+
+				break;
+			}
+			case RGX_META_LDR_CMD_LOADCORE:
+			case RGX_META_LDR_CMD_LOADMMREG:
+			{
+				return PVRSRV_ERROR_INIT_FAILURE;
+			}
+			case RGX_META_LDR_CMD_START_THREADS:
+			{
+				/* Don't process this block */
+				break;
+			}
+			case RGX_META_LDR_CMD_ZEROMEM:
+			{
+				IMG_UINT32 ui32Offset = psL1Data->aui32CmdData[0];
+				IMG_UINT32 ui32ByteCount = psL1Data->aui32CmdData[1];
+				void *pvWriteAddr;
+				PVRSRV_ERROR  eError;
+
+				if (RGX_META_IS_COREMEM_DATA(ui32Offset, ui32CorememSize))
+				{
+					/* cannot zero coremem directly */
+					break;
+				}
+
+				/* Global range is aliased to local range */
+				ui32Offset &= ~META_MEM_GLOBAL_RANGE_BIT;
+
+				eError = FindMMUSegment(hPrivate,
+				                        ui32Offset,
+				                        pvHostFWCodeAddr,
+				                        pvHostFWDataAddr,
+				                        pvHostFWCorememCodeAddr,
+				                        pvHostFWCorememDataAddr,
+				                        &pvWriteAddr);
+
+				if (eError != PVRSRV_OK)
+				{
+					RGXErrorLog(hPrivate,
+					            "ProcessLDRCommandStream: Addr 0x%x (size: %d) not found in any segment",
+					            ui32Offset, ui32ByteCount);
+					return eError;
+				}
+
+				/* Write to FW allocation only if available */
+				if (pvWriteAddr)
+				{
+					RGXMemSet(hPrivate, pvWriteAddr, 0, ui32ByteCount);
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+					RGXFwSharedMemCacheOpExec(pvWriteAddr, ui32ByteCount, PVRSRV_CACHE_OP_FLUSH);
+#endif
+				}
+
+				break;
+			}
+			case RGX_META_LDR_CMD_CONFIG:
+			{
+				RGX_META_LDR_L2_DATA_BLK *psL2Block =
+				    (RGX_META_LDR_L2_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->aui32CmdData[0]);
+				RGX_META_LDR_CFG_BLK *psConfigCommand = (RGX_META_LDR_CFG_BLK*) psL2Block->aui32BlockData;
+				IMG_UINT32 ui32L2BlockSize = psL2Block->ui16Length - 6 /* L2 Tag length and checksum */;
+				IMG_UINT32 ui32CurrBlockSize = 0;
+
+				while (ui32L2BlockSize)
+				{
+					switch (psConfigCommand->ui32Type)
+					{
+						case RGX_META_LDR_CFG_PAUSE:
+						case RGX_META_LDR_CFG_READ:
+						{
+							ui32CurrBlockSize = 8;
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+						case RGX_META_LDR_CFG_WRITE:
+						{
+							IMG_UINT32 ui32RegisterOffset = psConfigCommand->aui32BlockData[0];
+							IMG_UINT32 ui32RegisterValue  = psConfigCommand->aui32BlockData[1];
+
+							/* Only write to bootloader if we got a valid
+							 * pointer to the FW code allocation
+							 */
+							if (pui32BootConf)
+							{
+								/* Do register write */
+								*pui32BootConf++ = ui32RegisterOffset;
+								*pui32BootConf++ = ui32RegisterValue;
+							}
+
+							RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+							              ui32RegisterOffset, ui32RegisterValue);
+
+							ui32CurrBlockSize = 12;
+							break;
+						}
+						case RGX_META_LDR_CFG_MEMSET:
+						case RGX_META_LDR_CFG_MEMCHECK:
+						{
+							ui32CurrBlockSize = 20;
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+						default:
+						{
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+					}
+					ui32L2BlockSize -= ui32CurrBlockSize;
+					psConfigCommand = (RGX_META_LDR_CFG_BLK*) (((IMG_UINT8*) psConfigCommand) + ui32CurrBlockSize);
+				}
+
+				break;
+			}
+			default:
+			{
+				return PVRSRV_ERROR_INIT_FAILURE;
+			}
+		}
+
+NextBlock:
+
+		if (psL1Data->ui32Next == 0xFFFFFFFF)
+		{
+			psL1Data = NULL;
+		}
+		else
+		{
+			psL1Data = (RGX_META_LDR_L1_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->ui32Next);
+		}
+	}
+
+	if (pui32BootConf)
+	{
+		*ppui32BootConf = pui32BootConf;
+	}
+
+	RGXCommentLog(hPrivate, "**********************************************");
+	RGXCommentLog(hPrivate, "************** End Loader Parsing ************");
+	RGXCommentLog(hPrivate, "**********************************************");
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessELFCommandStream
+
+ @Description   Process a file in .ELF format copying code and data sections
+                into their final location
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbELF                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessELFCommandStream(const void *hPrivate,
+                                     const IMG_BYTE *pbELF,
+                                     void *pvHostFWCodeAddr,
+                                     void *pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr)
+{
+	IMG_UINT32 ui32Entry;
+	IMG_ELF_HDR *psHeader = (IMG_ELF_HDR *)pbELF;
+	IMG_ELF_PROGRAM_HDR *psProgramHeader =
+	    (IMG_ELF_PROGRAM_HDR *)(pbELF + psHeader->ui32Ephoff);
+	PVRSRV_ERROR eError;
+
+	for (ui32Entry = 0; ui32Entry < psHeader->ui32Ephnum; ui32Entry++, psProgramHeader++)
+	{
+		void *pvWriteAddr;
+
+		/* Only consider loadable entries in the ELF segment table */
+		if (psProgramHeader->ui32Ptype != ELF_PT_LOAD) continue;
+
+		eError = FindMMUSegment(hPrivate,
+		                        psProgramHeader->ui32Pvaddr,
+		                        pvHostFWCodeAddr,
+		                        pvHostFWDataAddr,
+		                        pvHostFWCorememCodeAddr,
+		                        pvHostFWCorememDataAddr,
+		                        &pvWriteAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate,
+			            "%s: Addr 0x%x (size: %d) not found in any segment",__func__,
+			            psProgramHeader->ui32Pvaddr,
+			            psProgramHeader->ui32Pfilesz);
+			return eError;
+		}
+
+		/* Write to FW allocation only if available */
+		if (pvWriteAddr)
+		{
+			RGXMemCopy(hPrivate,
+			           pvWriteAddr,
+			           (IMG_PBYTE)(pbELF + psProgramHeader->ui32Poffset),
+			           psProgramHeader->ui32Pfilesz);
+
+			RGXMemSet(hPrivate,
+			          (IMG_PBYTE)pvWriteAddr + psProgramHeader->ui32Pfilesz,
+			          0,
+			          psProgramHeader->ui32Pmemsz - psProgramHeader->ui32Pfilesz);
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+			RGXFwSharedMemCacheOpExec(pvWriteAddr, psProgramHeader->ui32Pmemsz, PVRSRV_CACHE_OP_FLUSH);
+#endif
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_UINT32 RGXGetFWImageSectionOffset(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32AllocOffset;
+}
+
+IMG_UINT32 RGXGetFWImageSectionMaxSize(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32MaxSize;
+}
+
+IMG_UINT32 RGXGetFWImageSectionAllocSize(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32AllocSize;
+}
+
+IMG_UINT32 RGXGetFWImageSectionAddress(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32BaseAddr;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion1(const void *hPrivate,
+                                         const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	/* Applicable to any FW_INFO_VERSION */
+	if (psInfoHeader->ui32LayoutEntrySize != sizeof(RGX_FW_LAYOUT_ENTRY))
+	{
+		RGXErrorLog(hPrivate, "%s: FW layout entry sizes mismatch (expected: %u, found: %u)",
+		            __func__,
+		            (IMG_UINT32) sizeof(RGX_FW_LAYOUT_ENTRY),
+		            psInfoHeader->ui32LayoutEntrySize);
+	}
+
+	/* Applicable to any FW_INFO_VERSION */
+	if (psInfoHeader->ui32LayoutEntryNum > MAX_NUM_ENTRIES)
+	{
+		RGXErrorLog(hPrivate, "%s: Not enough storage for the FW layout table (max: %u entries, found: %u)",
+		            __func__,
+		            MAX_NUM_ENTRIES,
+		            psInfoHeader->ui32LayoutEntryNum);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Applicable to any FW_INFO_VERSION */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		if (psInfoHeader->ui32FwPageSize != RGXGetOSPageSize(hPrivate))
+		{
+			RGXErrorLog(hPrivate, "%s: FW page size mismatch (expected: %u, found: %u)",
+			            __func__,
+			            (IMG_UINT32) RGXGetOSPageSize(hPrivate),
+			            psInfoHeader->ui32FwPageSize);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	if (psInfoHeader->ui32InfoVersion != FW_INFO_VERSION)
+	{
+		/* Not an error because RGX_FW_INFO_HEADER is now versioned. It can grow
+		 * incrementally and it must be backwards compatible.
+		 */
+		RGXCommentLog(hPrivate, "%s: FW info version mismatch (expected: %u, found: %u)",
+		              __func__,
+		              (IMG_UINT32) FW_INFO_VERSION,
+		              psInfoHeader->ui32InfoVersion);
+		goto exit_version1_validation;
+	}
+
+	if (psInfoHeader->ui32HeaderLen != sizeof(RGX_FW_INFO_HEADER))
+	{
+		RGXErrorLog(hPrivate, "%s: FW info header sizes mismatch (expected: %u, found: %u)",
+		            __func__,
+		            (IMG_UINT32) sizeof(RGX_FW_INFO_HEADER),
+		            psInfoHeader->ui32HeaderLen);
+	}
+
+exit_version1_validation:
+	return PVRSRV_OK;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion2(const void *hPrivate,
+                                         const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	if (psInfoHeader->ui16PVRVersionMajor != PVRVERSION_MAJ ||
+	    psInfoHeader->ui16PVRVersionMinor != PVRVERSION_MIN)
+	{
+		RGXErrorLog(hPrivate, "%s: KM and FW version mismatch (expected: %u.%u, found: %u.%u)",
+		            __func__,
+		            PVRVERSION_MAJ,
+		            PVRVERSION_MIN,
+		            psInfoHeader->ui16PVRVersionMajor,
+		            psInfoHeader->ui16PVRVersionMinor);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion(const void *hPrivate,
+                                        const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	PVRSRV_ERROR eError;
+
+	switch (psInfoHeader->ui32InfoVersion)
+	{
+		default:
+			__fallthrough;
+		case 2:
+			eError = RGXValidateFWHeaderVersion2(hPrivate, psInfoHeader);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			__fallthrough;
+		case 1:
+			eError = RGXValidateFWHeaderVersion1(hPrivate, psInfoHeader);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			break;
+		case 0:
+			RGXErrorLog(hPrivate, "%s: invalid FW_INFO_VERSION", __func__);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetFWImageAllocSize(const void *hPrivate,
+                                    const IMG_BYTE    *pbRGXFirmware,
+                                    const IMG_UINT32  ui32RGXFirmwareSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWDataAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememDataAllocSize,
+                                    RGX_FW_INFO_HEADER *psFWInfoHeader)
+{
+	RGX_FW_INFO_HEADER *psInfoHeader;
+	const IMG_BYTE *pbRGXFirmwareInfo;
+	const IMG_BYTE *pbRGXFirmwareLayout;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	if (pbRGXFirmware == NULL || ui32RGXFirmwareSize == 0 || ui32RGXFirmwareSize <= FW_BLOCK_SIZE)
+	{
+		RGXErrorLog(hPrivate, "%s: Invalid FW binary at %p, size %u",
+		            __func__, pbRGXFirmware, ui32RGXFirmwareSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/*
+	 * Acquire pointer to the FW info header within the FW image.
+	 * The format of the header in the FW image might not be the one expected
+	 * by the driver, but the driver should still be able to correctly read
+	 * the information below, as long as new/incompatible elements are added
+	 * at the end of the header (they will be ignored by the driver).
+	 */
+
+	pbRGXFirmwareInfo = pbRGXFirmware + ui32RGXFirmwareSize - FW_BLOCK_SIZE;
+	psInfoHeader = (RGX_FW_INFO_HEADER*)pbRGXFirmwareInfo;
+
+	eError = RGXValidateFWHeaderVersion(hPrivate, psInfoHeader);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	ui32LayoutEntryNum = psInfoHeader->ui32LayoutEntryNum;
+
+
+	/*
+	 * Copy FW layout table from FW image to local array.
+	 * One entry is copied at a time and the copy is limited to what the driver
+	 * expects to find in it. Assuming that new/incompatible elements
+	 * are added at the end of each entry, the loop below adapts the table
+	 * in the FW image into the format expected by the driver.
+	 */
+
+	pbRGXFirmwareLayout = pbRGXFirmwareInfo + psInfoHeader->ui32HeaderLen;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		RGX_FW_LAYOUT_ENTRY *psOutEntry = &asRGXFWLayoutTable[i];
+
+		RGX_FW_LAYOUT_ENTRY *psInEntry = (RGX_FW_LAYOUT_ENTRY*)
+			(pbRGXFirmwareLayout + i * psInfoHeader->ui32LayoutEntrySize);
+
+		RGXMemCopy(hPrivate,
+		           (void*)psOutEntry,
+		           (void*)psInEntry,
+		           sizeof(RGX_FW_LAYOUT_ENTRY));
+	}
+
+
+	/* Calculate how much memory the FW needs for its code and data segments */
+
+	*puiFWCodeAllocSize = 0;
+	*puiFWDataAllocSize = 0;
+	*puiFWCorememCodeAllocSize = 0;
+	*puiFWCorememDataAllocSize = 0;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		switch (asRGXFWLayoutTable[i].eType)
+		{
+			case FW_CODE:
+				*puiFWCodeAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_DATA:
+				*puiFWDataAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_COREMEM_CODE:
+				*puiFWCorememCodeAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_COREMEM_DATA:
+				*puiFWCorememDataAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			default:
+				RGXErrorLog(hPrivate, "%s: Unknown FW section type %u\n",
+				            __func__, asRGXFWLayoutTable[i].eType);
+				break;
+		}
+	}
+
+	*psFWInfoHeader = *psInfoHeader;
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXProcessFWImage(const void *hPrivate,
+                               const IMG_BYTE *pbRGXFirmware,
+                               void *pvFWCode,
+                               void *pvFWData,
+                               void *pvFWCorememCode,
+                               void *pvFWCorememData,
+                               PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMIPS = IMG_FALSE;
+	IMG_BOOL bRISCV = RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR);
+	IMG_BOOL bMETA;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	bMIPS = (IMG_BOOL)RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS);
+#endif
+	bMETA = (IMG_BOOL)(!bMIPS && !bRISCV);
+
+	if (bMETA)
+	{
+		IMG_UINT32 *pui32BootConf = NULL;
+		/* Skip bootloader configuration if a pointer to the FW code
+		 * allocation is not available
+		 */
+		if (pvFWCode)
+		{
+			/* This variable points to the bootloader code which is mostly
+			 * a sequence of <register address,register value> pairs
+			 */
+			pui32BootConf = ((IMG_UINT32*) pvFWCode) + RGXFW_BOOTLDR_CONF_OFFSET;
+
+			/* Slave port and JTAG accesses are privileged */
+			*pui32BootConf++ = META_CR_SYSC_JTAG_THREAD;
+			*pui32BootConf++ = META_CR_SYSC_JTAG_THREAD_PRIV_EN;
+
+			RGXFWConfigureSegMMU(hPrivate,
+			                     &puFWParams->sMeta.sFWCodeDevVAddr,
+			                     &puFWParams->sMeta.sFWDataDevVAddr,
+			                     &pui32BootConf);
+		}
+
+		/* Process FW image data stream */
+		eError = ProcessLDRCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 pvFWCorememCode,
+		                                 pvFWCorememData,
+		                                 &pui32BootConf);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		/* Skip bootloader configuration if a pointer to the FW code
+		 * allocation is not available
+		 */
+		if (pvFWCode)
+		{
+			IMG_UINT32 ui32NumThreads   = puFWParams->sMeta.ui32NumThreads;
+
+			if ((ui32NumThreads == 0) || (ui32NumThreads > 2))
+			{
+				RGXErrorLog(hPrivate,
+				            "ProcessFWImage: Wrong Meta threads configuration, using one thread only");
+
+				ui32NumThreads = 1;
+			}
+
+			RGXFWConfigureMetaCaches(hPrivate,
+			                         ui32NumThreads,
+			                         &pui32BootConf);
+
+			/* Signal the end of the conf sequence */
+			*pui32BootConf++ = 0x0;
+			*pui32BootConf++ = 0x0;
+
+			if (puFWParams->sMeta.uiFWCorememCodeSize && (puFWParams->sMeta.sFWCorememCodeFWAddr.ui32Addr != 0))
+			{
+				*pui32BootConf++ = puFWParams->sMeta.sFWCorememCodeFWAddr.ui32Addr;
+				*pui32BootConf++ = puFWParams->sMeta.uiFWCorememCodeSize;
+			}
+			else
+			{
+				*pui32BootConf++ = 0;
+				*pui32BootConf++ = 0;
+			}
+
+#if defined(RGX_FEATURE_META_DMA_BIT_MASK)
+			if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_DMA))
+			{
+				*pui32BootConf++ = (IMG_UINT32) (puFWParams->sMeta.sFWCorememCodeDevVAddr.uiAddr >> 32);
+				*pui32BootConf++ = (IMG_UINT32) puFWParams->sMeta.sFWCorememCodeDevVAddr.uiAddr;
+			}
+			else
+#endif
+			{
+				*pui32BootConf++ = 0;
+				*pui32BootConf++ = 0;
+			}
+		}
+	}
+#if defined(RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES)
+	else if (bMIPS)
+	{
+		/* Process FW image data stream */
+		eError = ProcessELFCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 NULL,
+		                                 NULL);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		if (pvFWData)
+		{
+			RGXMIPSFW_BOOT_DATA *psBootData = (RGXMIPSFW_BOOT_DATA*)
+				/* To get a pointer to the bootloader configuration data start from a pointer to the FW image... */
+				IMG_OFFSET_ADDR(pvFWData,
+				/* ... jump to the boot/NMI data page... */
+				(RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA)
+				/* ... and then jump to the bootloader data offset within the page */
+				+ RGXMIPSFW_BOOTLDR_CONF_OFFSET));
+
+			/* Rogue Registers physical address */
+			psBootData->ui64RegBase = puFWParams->sMips.sGPURegAddr.uiAddr;
+
+			/* MIPS Page Table physical address */
+			psBootData->ui32PTLog2PageSize = puFWParams->sMips.ui32FWPageTableLog2PageSize;
+			psBootData->ui32PTNumPages     = puFWParams->sMips.ui32FWPageTableNumPages;
+			psBootData->aui64PTPhyAddr[0U] = puFWParams->sMips.asFWPageTableAddr[0U].uiAddr;
+			psBootData->aui64PTPhyAddr[1U] = puFWParams->sMips.asFWPageTableAddr[1U].uiAddr;
+			psBootData->aui64PTPhyAddr[2U] = puFWParams->sMips.asFWPageTableAddr[2U].uiAddr;
+			psBootData->aui64PTPhyAddr[3U] = puFWParams->sMips.asFWPageTableAddr[3U].uiAddr;
+
+			/* MIPS Stack Pointer Physical Address */
+			psBootData->ui64StackPhyAddr = puFWParams->sMips.sFWStackAddr.uiAddr;
+
+			/* Reserved for future use */
+			psBootData->ui32Reserved1 = 0;
+			psBootData->ui32Reserved2 = 0;
+		}
+	}
+#endif /* #if defined(RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES) */
+	else
+	{
+		/* Process FW image data stream */
+		eError = ProcessELFCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 pvFWCorememCode,
+		                                 pvFWCorememData);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		if (pvFWData)
+		{
+			RGXRISCVFW_BOOT_DATA *psBootData = (RGXRISCVFW_BOOT_DATA*)
+				IMG_OFFSET_ADDR(pvFWData, RGXRISCVFW_BOOTLDR_CONF_OFFSET);
+
+			psBootData->ui64CorememCodeDevVAddr = puFWParams->sRISCV.sFWCorememCodeDevVAddr.uiAddr;
+			psBootData->ui32CorememCodeFWAddr   = puFWParams->sRISCV.sFWCorememCodeFWAddr.ui32Addr;
+			psBootData->ui32CorememCodeSize     = puFWParams->sRISCV.uiFWCorememCodeSize;
+
+			psBootData->ui64CorememDataDevVAddr = puFWParams->sRISCV.sFWCorememDataDevVAddr.uiAddr;
+			psBootData->ui32CorememDataFWAddr   = puFWParams->sRISCV.sFWCorememDataFWAddr.ui32Addr;
+			psBootData->ui32CorememDataSize     = puFWParams->sRISCV.uiFWCorememDataSize;
+		}
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.h
new file mode 100644
index 000000000000..b4cedb8d3271
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.h
@@ -0,0 +1,224 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for Services Firmware image utilities used at init time
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for Services Firmware image utilities used at init time
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWIMAGEUTILS_H
+#define RGXFWIMAGEUTILS_H
+
+/* The routines declared here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when DRM security is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary.
+ */
+#include "rgxlayer.h"
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionOffset
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return offset of a Firmware section, relative to the beginning
+               of the code or data allocation (depending on the section id)
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionOffset(const void *hPrivate,
+                                      RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionMaxSize
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return maximum size (not allocation size) of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionMaxSize(const void *hPrivate,
+                                       RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionAllocSize
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return allocation size of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionAllocSize(const void *hPrivate,
+                                         RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionAddress
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return base address of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionAddress(const void *hPrivate,
+                                       RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageAllocSize
+
+ @Description  Return size of Firmware code/data/coremem code allocations
+
+ @Input        hPrivate            : Implementation specific data
+ @Input        pbRGXFirmware       : Pointer to FW binary
+ @Input        ui32RGXFirmwareSize : FW binary size
+ @Output       puiFWCodeAllocSize  : Code size
+ @Output       puiFWDataAllocSize  : Data size
+ @Output       puiFWCorememCodeAllocSize : Coremem code size (0 if N/A)
+ @Output       puiFWCorememDataAllocSize : Coremem data size (0 if N/A)
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXGetFWImageAllocSize(const void *hPrivate,
+                                    const IMG_BYTE    *pbRGXFirmware,
+                                    const IMG_UINT32  ui32RGXFirmwareSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWDataAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememDataAllocSize,
+                                    RGX_FW_INFO_HEADER *psFWInfoHeader);
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessLDRCommandStream
+
+ @Description   Process the output of the Meta toolchain in the .LDR format
+                copying code and data sections into their final location and
+                passing some information to the Meta bootloader
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbLDR                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+ @Input         ppui32BootConf           : Pointer to bootloader data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessLDRCommandStream(const void *hPrivate,
+                                     const IMG_BYTE* pbLDR,
+                                     void* pvHostFWCodeAddr,
+                                     void* pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr,
+                                     IMG_UINT32 **ppui32BootConf);
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessELFCommandStream
+
+ @Description   Process a file in .ELF format copying code and data sections
+                into their final location
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbELF                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessELFCommandStream(const void *hPrivate,
+                                     const IMG_BYTE *pbELF,
+                                     void *pvHostFWCodeAddr,
+                                     void *pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXProcessFWImage
+
+ @Description  Process the Firmware binary blob copying code and data
+               sections into their final location and passing some
+               information to the Firmware bootloader.
+               If a pointer to the final memory location for FW code or data
+               is not valid (NULL) then the relative section will not be
+               processed.
+
+ @Input        hPrivate        : Implementation specific data
+ @Input        pbRGXFirmware   : Pointer to FW blob
+ @Input        pvFWCode        : Pointer to FW code
+ @Input        pvFWData        : Pointer to FW data
+ @Input        pvFWCorememCode : Pointer to FW coremem code
+ @Input        pvFWCorememData : Pointer to FW coremem data
+ @Input        puFWParams      : Parameters used by the FW at boot time
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessFWImage(const void *hPrivate,
+                               const IMG_BYTE *pbRGXFirmware,
+                               void *pvFWCode,
+                               void *pvFWData,
+                               void *pvFWCorememCode,
+                               void *pvFWCorememData,
+                               PVRSRV_FW_BOOT_PARAMS *puFWParams);
+
+#endif /* RGXFWIMAGEUTILS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwmemctx.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwmemctx.h
new file mode 100644
index 000000000000..49f37b1c27aa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwmemctx.h
@@ -0,0 +1,156 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware Memctx routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for operations on FWKM Shared memory context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWMEMCTX_H
+#define RGXFWMEMCTX_H
+
+#include "device.h"
+#include "rgx_memallocflags.h"
+#include "pvrsrv.h"
+#include "cache_ops.h"
+#include "cache_km.h"
+#include "pvr_debug.h"
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+/*
+ * RGXFwSharedMemCPUCacheMode()
+ * We upgrade allocations on ARM64 Linux when CPU Cache snooping is enabled.
+ * This is because of the Linux direct mapping causing interference due to PIPT
+ * cache. All allocations are normally UCWC but snooping can return a bad value from the
+ * direct mapping as it is cached. Upgrade our allocations to cached to prevent bad cached
+ * values but in turn we require flushing.
+ */
+static INLINE void RGXFwSharedMemCPUCacheMode(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)) == 0)
+	{
+		/* We don't need to upgrade if we don't map into the CPU */
+		return;
+	}
+
+	/* Clear the existing CPU cache flags */
+	*puiFlags &= ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK);
+
+	if (PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig))
+	{
+		*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT;
+	}
+	else
+	{
+		*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+	}
+}
+
+#define RGXFwSharedMemCheckSnoopMode(psDeviceConfig) PVR_ASSERT(PVRSRVSystemSnoopingOfCPUCache(psDeviceConfig))
+
+/*
+ * FWSharedMemCacheOpExec()
+ * This is the CPU data-cache maintenance interface for FW shared allocations.
+ * We have to be very careful that the VAs supplied to this function are
+ * sensible as to not cause a kernel oops. Given that this should only be
+ * used for allocations used for the FW this should be guaranteed.
+ */
+static INLINE PVRSRV_ERROR RGXFwSharedMemCacheOpExec(const volatile void *pvVirtStart,
+                                                     IMG_UINT64 uiSize,
+                                                     PVRSRV_CACHE_OP uiCacheOp)
+{
+	IMG_UINT64 uiEndAddr = (IMG_UINT64) pvVirtStart + uiSize;
+	IMG_CPU_PHYADDR uiUnusedPhysAddr = {.uiAddr = 0};
+
+	if (!pvVirtStart || uiSize == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return CacheOpExec(NULL,
+					   (void*) pvVirtStart,
+					   (void*) uiEndAddr,
+					   uiUnusedPhysAddr,
+					   uiUnusedPhysAddr,
+					   uiCacheOp);
+}
+
+#define RGXFwSharedMemCacheOpValue(value, cacheop) (RGXFwSharedMemCacheOpExec(&value, sizeof(value), PVRSRV_CACHE_OP_##cacheop))
+#define RGXFwSharedMemCacheOpPtr(ptr, cacheop) (RGXFwSharedMemCacheOpExec(ptr, sizeof(*ptr), PVRSRV_CACHE_OP_##cacheop))
+#define RGXFwSharedMemCacheOpExecPfn RGXFwSharedMemCacheOpExec
+
+static INLINE void RGXFwSharedMemFlushCCB(void *pvCCBVirtAddr,
+                                          IMG_UINT64 uiStart,
+                                          IMG_UINT64 uiFinish,
+                                          IMG_UINT64 uiLimit)
+{
+	if (uiFinish >= uiStart)
+	{
+		/* Flush the CCB data */
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, uiStart),
+		                          uiFinish - uiStart,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+	else
+	{
+		/* CCCB wrapped around - flush the pre and post wrap boundary separately */
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, uiStart),
+		                          uiLimit - uiStart,
+		                          PVRSRV_CACHE_OP_FLUSH);
+
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, 0),
+		                          uiFinish,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+}
+#else
+#define RGXFwSharedMemCPUCacheMode(...)
+#define RGXFwSharedMemCheckSnoopMode(...)
+/* NULL value required for function callbacks */
+#define RGXFwSharedMemCacheOpExec(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpValue(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpPtr(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpExecPfn NULL
+#define RGXFwSharedMemFlushCCB(...)
+#endif
+
+
+#endif /* RGXFWMEMCTX_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.c
new file mode 100644
index 000000000000..555691010517
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.c
@@ -0,0 +1,1074 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware RISC-V utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware RISC-V utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGX_GET_DMI_REG(psDevInfo, value) \
+	((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? \
+	 RGX_CR_FWCORE_DMI_##value##__HOST_SECURITY_GEQ4 : RGX_CR_FWCORE_DMI_##value)
+#else
+#define RGX_GET_DMI_REG(psDevInfo, value) RGX_CR_FWCORE_DMI_##value
+#endif
+
+/*
+ * RGXRiscvHalt
+ */
+PVRSRV_ERROR RGXRiscvHalt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_DMCONTROL_Reg = RGX_GET_DMI_REG(psDevInfo, DMCONTROL);
+	__maybe_unused IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Halt RISC-V FW");
+
+	/* Send halt request (no need to select one or more harts on this RISC-V core) */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN |
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until hart is halted */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DMSTATUS_Reg,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	/* Clear halt request */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send halt request (no need to select one or more harts on this RISC-V core) */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN |
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+
+	/* Wait until hart is halted */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_DMSTATUS_Reg/sizeof(IMG_UINT32),
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Hart not halted (0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Clear halt request */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXRiscvIsHalted
+ */
+IMG_BOOL RGXRiscvIsHalted(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	/* Assume the core is always halted in nohw */
+	return IMG_TRUE;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+	IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+	return (OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg) &
+	        RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN) != 0U;
+#endif
+}
+
+/*
+ * RGXRiscvResume
+ */
+PVRSRV_ERROR RGXRiscvResume(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_DMCONTROL_Reg = RGX_GET_DMI_REG(psDevInfo, DMCONTROL);
+	__maybe_unused IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Resume RISC-V FW");
+
+	/* Send resume request (no need to select one or more harts on this RISC-V core) */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN |
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until hart is resumed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DMSTATUS_Reg,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	/* Clear resume request */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send resume request (no need to select one or more harts on this RISC-V core) */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN |
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+
+	/* Wait until hart is resumed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_DMSTATUS_Reg/sizeof(IMG_UINT32),
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Hart not resumed (0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Clear resume request */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvCheckAbstractCmdError
+
+@Description    Check for RISC-V abstract command errors and clear them
+
+@Input          psDevInfo    Pointer to GPU device info
+
+@Return         RGXRISCVFW_ABSTRACT_CMD_ERR
+******************************************************************************/
+static RGXRISCVFW_ABSTRACT_CMD_ERR RGXRiscvCheckAbstractCmdError(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXRISCVFW_ABSTRACT_CMD_ERR eCmdErr;
+
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	eCmdErr = RISCV_ABSTRACT_CMD_NO_ERROR;
+
+	/* Check error status */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            RISCV_ABSTRACT_CMD_NO_ERROR << RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT,
+	            ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	void __iomem *pvRegsBaseKM = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Check error status */
+	eCmdErr = (OSReadUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_ABSTRACTCS_Reg)
+	          & ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK)
+	          >> RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT;
+
+	if (eCmdErr != RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RISC-V FW abstract command error %u", eCmdErr));
+
+		/* Clear the error (note CMDERR field is write-1-to-clear) */
+		OSWriteUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_ABSTRACTCS_Reg,
+		               ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK);
+	}
+#endif
+
+	return eCmdErr;
+}
+
+/*
+ * RGXRiscReadReg
+ */
+PVRSRV_ERROR RGXRiscvReadReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading HW registers is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send abstract register read command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_READ |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	               ui32RegAddr);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckAbstractCmdError(psDevInfo) == RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		/* Read register value */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+
+/*
+ * RGXRiscvPollReg
+ */
+PVRSRV_ERROR RGXRiscvPollReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V register 0x%x (expected 0x%08x)",
+	                      ui32RegAddr, ui32Value);
+
+	/* Send abstract register read command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_READ |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	           ui32RegAddr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckAbstractCmdError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling HW registers is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+/*
+ * RGXRiscvWriteReg
+ */
+PVRSRV_ERROR RGXRiscvWriteReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32RegAddr,
+                              IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V register 0x%x (value 0x%08x)",
+	                      ui32RegAddr, ui32Value);
+
+	/* Prepare data to be written to register */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract register write command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_WRITE |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	           ui32RegAddr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare data to be written to register */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg, ui32Value);
+
+	/* Send abstract register write command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_WRITE |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	               ui32RegAddr);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvCheckSysBusError
+
+@Description    Check for RISC-V system bus errors and clear them
+
+@Input          psDevInfo    Pointer to GPU device info
+
+@Return         RGXRISCVFW_SYSBUS_ERR
+******************************************************************************/
+static __maybe_unused RGXRISCVFW_SYSBUS_ERR RGXRiscvCheckSysBusError(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXRISCVFW_SYSBUS_ERR eSBError;
+
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	eSBError = RISCV_SYSBUS_NO_ERROR;
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            RISCV_SYSBUS_NO_ERROR << RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT,
+	            ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	void __iomem *pvRegsBaseKM = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	eSBError = (OSReadUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_SBCS_Reg)
+	         & ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK)
+	         >> RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT;
+
+	if (eSBError != RISCV_SYSBUS_NO_ERROR)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RISC-V FW system bus error %u", eSBError));
+
+		/* Clear the error (note SBERROR field is write-1-to-clear) */
+		OSWriteUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_SBCS_Reg,
+		               ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK);
+	}
+#endif
+
+	return eSBError;
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadAbstractMem
+
+@Description    Read a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvReadAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading memory is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare read address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA1_Reg, ui32Addr);
+
+	/* Send abstract memory read command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_READ |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckAbstractCmdError(psDevInfo) == RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		/* Read memory value */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollAbstractMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvPollAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V address 0x%x (expected 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Prepare read address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA1_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract memory read command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_READ |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckAbstractCmdError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling memory is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadSysBusMem
+
+@Description    Read a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvReadSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading memory is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_SBCS_Reg,
+	               (RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT) |
+	               RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN);
+
+	/* Perform read */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBADDRESS0_Reg, ui32Addr);
+
+	/* Wait until system bus is idle */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_SBCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: System Bus did not go idle in time (sbcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckSysBusError(psDevInfo) == RISCV_SYSBUS_NO_ERROR)
+	{
+		/* Read value from debug system bus */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollSysBusMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvPollSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBDATA0_Reg = RGX_GET_DMI_REG(psDevInfo, SBDATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V address 0x%x (expected 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBCS_Reg,
+	           (RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT) |
+	           RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Perform read */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_SBADDRESS0_Reg,
+	           ui32Addr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until system bus is idle */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckSysBusError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBDATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling memory is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+#if !defined(EMULATOR)
+/*
+ * RGXRiscvReadMem
+ */
+PVRSRV_ERROR RGXRiscvReadMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 *pui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvReadAbstractMem(psDevInfo, ui32Addr, pui32Value);
+	}
+
+	return RGXRiscvReadSysBusMem(psDevInfo, ui32Addr, pui32Value);
+}
+#endif /* !defined(EMULATOR) */
+
+/*
+ * RGXRiscvPollMem
+ */
+PVRSRV_ERROR RGXRiscvPollMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 ui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvPollAbstractMem(psDevInfo, ui32Addr, ui32Value);
+	}
+
+	return RGXRiscvPollSysBusMem(psDevInfo, ui32Addr, ui32Value);
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteAbstractMem
+
+@Description    Write a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvWriteAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V address 0x%x (value 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Prepare write address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA1_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write data */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract register write command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_WRITE |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare write address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA1_Reg, ui32Addr);
+
+	/* Prepare write data */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg, ui32Value);
+
+	/* Send abstract memory write command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_WRITE |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteSysBusMem
+
+@Description    Write a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvWriteSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBDATA0_Reg = RGX_GET_DMI_REG(psDevInfo, SBDATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V address 0x%x (value 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBCS_Reg,
+	           RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBADDRESS0_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write data and initiate write */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBDATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until system bus is idle */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Configure system bus for 32 bit accesses */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_SBCS_Reg,
+	               RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT);
+
+	/* Prepare write address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBADDRESS0_Reg, ui32Addr);
+
+	/* Prepare write data and initiate write */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBDATA0_Reg, ui32Value);
+
+	/* Wait until system bus is idle */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_SBCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: System Bus did not go idle in time (sbcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXRiscvWriteMem
+ */
+PVRSRV_ERROR RGXRiscvWriteMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32Addr,
+                              IMG_UINT32 ui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvWriteAbstractMem(psDevInfo, ui32Addr, ui32Value);
+	}
+
+	return RGXRiscvWriteSysBusMem(psDevInfo, ui32Addr, ui32Value);
+}
+#endif /* !defined(EMULATOR) */
+
+/*
+ * RGXRiscvDmiOp
+ */
+PVRSRV_ERROR RGXRiscvDmiOp(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT64 *pui64DMI)
+{
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(pui64DMI);
+
+	/* Accessing DM registers is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define DMI_BASE     ((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4 : RGX_CR_FWCORE_DMI_RESERVED00)
+#else
+#define DMI_BASE     RGX_CR_FWCORE_DMI_RESERVED00
+#endif
+#define DMI_STRIDE  (RGX_CR_FWCORE_DMI_RESERVED01 - RGX_CR_FWCORE_DMI_RESERVED00)
+#define DMI_REG(r)  ((DMI_BASE) + (DMI_STRIDE) * (r))
+
+#define DMI_OP_SHIFT            0U
+#define DMI_OP_MASK             0x3ULL
+#define DMI_DATA_SHIFT          2U
+#define DMI_DATA_MASK           0x3FFFFFFFCULL
+#define DMI_ADDRESS_SHIFT       34U
+#define DMI_ADDRESS_MASK        0xFC00000000ULL
+
+#define DMI_OP_NOP	            0U
+#define DMI_OP_READ	            1U
+#define DMI_OP_WRITE	        2U
+#define DMI_OP_RESERVED	        3U
+
+#define DMI_OP_STATUS_SUCCESS	0U
+#define DMI_OP_STATUS_RESERVED	1U
+#define DMI_OP_STATUS_FAILED	2U
+#define DMI_OP_STATUS_BUSY	    3U
+
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64Op, ui64Address, ui64Data;
+
+	ui64Op      = (*pui64DMI & DMI_OP_MASK) >> DMI_OP_SHIFT;
+	ui64Address = (*pui64DMI & DMI_ADDRESS_MASK) >> DMI_ADDRESS_SHIFT;
+	ui64Data    = (*pui64DMI & DMI_DATA_MASK) >> DMI_DATA_SHIFT;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		ui64Op = DMI_OP_STATUS_FAILED;
+		goto dmiop_update;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to retrieve RGX power state (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		ui64Op = DMI_OP_STATUS_FAILED;
+		goto dmiop_release_lock;
+	}
+
+	if (ePowerState == PVRSRV_DEV_POWER_STATE_ON)
+	{
+		void __iomem *pvRegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+		switch (ui64Op)
+		{
+			case DMI_OP_NOP:
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			case DMI_OP_WRITE:
+				OSWriteUncheckedHWReg32(pvRegsBase,
+						DMI_REG(ui64Address),
+						(IMG_UINT32)ui64Data);
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			case DMI_OP_READ:
+				ui64Data = (IMG_UINT64)OSReadUncheckedHWReg32(pvRegsBase,
+						DMI_REG(ui64Address));
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "%s: unknown op %u", __func__, (IMG_UINT32)ui64Op));
+				ui64Op = DMI_OP_STATUS_FAILED;
+				break;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Accessing RISC-V Debug Module is not "
+					"possible while the GPU is powered off", __func__));
+
+		ui64Op = DMI_OP_STATUS_FAILED;
+	}
+
+dmiop_release_lock:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+dmiop_update:
+	*pui64DMI = (ui64Op << DMI_OP_SHIFT) |
+		(ui64Address << DMI_ADDRESS_SHIFT) |
+		(ui64Data << DMI_DATA_SHIFT);
+
+	return eError;
+#endif
+}
+
+/******************************************************************************
+ End of file (rgxfwriscv.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.h
new file mode 100644
index 000000000000..7821c167fa2a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.h
@@ -0,0 +1,217 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware RISC-V utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware RISC-V utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWRISCV_H
+#define RGXFWRISCV_H
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGXRISCVFW_GET_REMAP_SECURE(r)          (RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4 + ((r) * 8U))
+#define RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE    (RGXRISCVFW_GET_REMAP_SECURE(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_REMAP_SECURE    (RGXRISCVFW_GET_REMAP_SECURE(RGXRISCVFW_BOOTLDR_DATA_REGION))
+#define RGX_GET_RISCV_REGS_BASE(psDevInfo) \
+	((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? \
+	 (psDevInfo)->pvSecureRegsBaseKM : (psDevInfo)->pvRegsBaseKM)
+#else
+#define RGX_GET_RISCV_REGS_BASE(psDevInfo) ((psDevInfo)->pvRegsBaseKM)
+#endif
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvHalt
+
+@Description    Halt the RISC-V FW core (required for certain operations
+                done through Debug Module)
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvHalt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvIsHalted
+
+@Description    Check if the RISC-V FW is halted
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         IMG_BOOL
+******************************************************************************/
+IMG_BOOL RGXRiscvIsHalted(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvResume
+
+@Description    Resume the RISC-V FW core
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvResume(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadReg
+
+@Description    Read a value from the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvReadReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollReg
+
+@Description    Poll for a value from the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvPollReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteReg
+
+@Description    Write a value to the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvWriteReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32RegAddr,
+                              IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvPollMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 ui32Value);
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadMem
+
+@Description    Read a value at the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvReadMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteMem
+
+@Description    Write a value to the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvWriteMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32Addr,
+                              IMG_UINT32 ui32Value);
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvDmiOp
+
+@Description    Acquire the powerlock and perform an operation on the RISC-V
+                Debug Module Interface, but only if the GPU is powered on.
+
+@Input          psDevInfo       Pointer to device info
+@InOut          pui64DMI        Encoding of a request for the RISC-V Debug
+                                Module with same format as the 'dmi' register
+                                from the RISC-V debug specification (v0.13+).
+                                On return, this is updated with the result of
+                                the request, encoded the same way.
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvDmiOp(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT64 *pui64DMI);
+
+#endif /* RGXFWRISCV_H */
+/******************************************************************************
+ End of file (rgxfwriscv.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwtrace_strings.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwtrace_strings.c
new file mode 100644
index 000000000000..d95050871994
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwtrace_strings.c
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File           rgxfwtrace_strings.c
+@Title          RGX Firmware trace strings
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgx_fwif_sf.h"
+#include "fwtrace_string.h"
+
+/* The tuple pairs that will be generated using XMacros will be stored here.
+ * This macro definition must match the definition of SFids in rgx_fwif_sf.h
+ */
+const RGXKM_STID_FMT SFs[]= {
+#define X(a, b, c, d, e) { RGXFW_LOG_CREATESFID(a,b,e), d },
+	RGXFW_LOG_SFIDLIST
+#undef X
+};
+
+const IMG_UINT32 g_ui32SFsCount = ARRAY_SIZE(SFs);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwutils.h
new file mode 100644
index 000000000000..6752bc95fa18
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwutils.h
@@ -0,0 +1,1298 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWUTILS_H
+#define RGXFWUTILS_H
+
+#include "rgx_memallocflags.h"
+#include "log2.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "devicemem.h"
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "connection_server.h"
+#include "rgxta3d.h"
+#include "devicemem_utils.h"
+#include "rgxmem.h"
+#include "rgxfwmemctx.h"
+#include "rgxinit_apphints.h"
+
+#define RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT   "FwRawDriverID%d" /*!< RGX Raw Firmware Heap identifier */
+
+static INLINE PVRSRV_ERROR _SelectDevMemHeap(PVRSRV_RGXDEV_INFO *psDevInfo,
+											 PVRSRV_MEMALLOCFLAGS_T *puiFlags,
+											 DEVMEM_HEAP **ppsFwHeap)
+{
+	PVRSRV_PHYS_HEAP ePhysHeap = (PVRSRV_PHYS_HEAP)PVRSRV_GET_PHYS_HEAP_HINT(*puiFlags);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (ePhysHeap)
+	{
+		case PVRSRV_PHYS_HEAP_FW_CODE:
+		case PVRSRV_PHYS_HEAP_FW_PRIV_DATA:
+		case PVRSRV_PHYS_HEAP_FW_MAIN:
+		{
+			*ppsFwHeap = psDevInfo->psFirmwareMainHeap;
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_FW_CONFIG:
+		{
+			*ppsFwHeap = psDevInfo->psFirmwareConfigHeap;
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_FW_PREMAP0:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP1:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP2:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP3:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP4:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP5:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP6:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP7:
+		{
+			IMG_UINT32 ui32DriverID = ePhysHeap - PVRSRV_PHYS_HEAP_FW_PREMAP0;
+
+			PVR_LOG_RETURN_IF_INVALID_PARAM(ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED, "ui32DriverID");
+			*ppsFwHeap = psDevInfo->psPremappedFwRawHeap[ui32DriverID];
+			break;
+		}
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid phys heap", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			break;
+		}
+	}
+
+	return eError;
+}
+
+/*
+ * Firmware-only allocation (which are initialised by the host) must be aligned to the SLC cache line size.
+ * This is because firmware-only allocations are GPU_CACHE_INCOHERENT and this causes problems
+ * if two allocations share the same cache line; e.g. the initialisation of the second allocation won't
+ * make it into the SLC cache because it has been already loaded when accessing the content of the first allocation.
+ */
+static INLINE PVRSRV_ERROR DevmemFwAllocate(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_DEVMEM_SIZE_T uiSize,
+											PVRSRV_MEMALLOCFLAGS_T uiFlags,
+											const IMG_CHAR *pszText,
+											DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_DEVMEM_ALIGN_T uiAlign;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) && (pszText[0] == 'F') && (pszText[1] == 'w'));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	if (psFwHeap == psDevInfo->psFirmwareConfigHeap)
+	{
+		/*
+		 * All structures allocated from the Firmware Config sub-heap must start at the same pre-determined
+		 * offsets, regardless of the system's page size (e.g. 4k,16k,64k). The alignment requirement is
+		 * satisfied for virtual addresses during the mapping stage. Physical allocations do not take
+		 * alignment into consideration.
+		 * VZ drivers usually preallocate and pre-map the entire Firmware heap range. Any allocations from
+		 * this heap are physical allocations only, having their device VAs derived from their PAs. This makes
+		 * it impossible to fulfil alignment requirements.
+		 * To work around this limitation, allocation sizes are rounded to the nearest multiple of 64kb,
+		 * regardless of the actual size of object.
+		 */
+		uiAlign = RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY;
+
+		uiSize = PVR_ALIGN(uiSize, RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY);
+	}
+	else
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			/* Aligning FW based allocations for MIPS based rogue cores at cache line boundary(16 bytes) instead
+			 * of SLC(64 bytes) to have more compact memory with less waste and hopefully save some TLB misses.
+			 * MIPS CPU cores alignment.
+			 */
+			uiAlign = RGXMIPSFW_MICROAPTIVEAP_CACHELINE_SIZE;
+		}
+		else
+#endif
+		{
+			/* Non-MIPS CPU cores alignment */
+			uiAlign = (GET_ROGUE_CACHE_LINE_SIZE(RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS)));
+		}
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+	eError = DevmemAllocateAndMap(psFwHeap,
+				uiSize,
+				uiAlign,
+				uiFlags,
+				pszText,
+				ppsMemDescPtr,
+				&sTmpDevVAddr);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static INLINE PVRSRV_ERROR DevmemFwAllocateExportable(PVRSRV_DEVICE_NODE *psDeviceNode,
+													  IMG_DEVMEM_SIZE_T uiSize,
+													  IMG_DEVMEM_ALIGN_T uiAlign,
+													  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+													  const IMG_CHAR *pszText,
+													  DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_UINT32 ui32HeapLog2PageSize;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) &&
+			(pszText[0] == 'F') && (pszText[1] == 'w') &&
+			(pszText[2] == 'E') && (pszText[3] == 'x'));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS cores */
+		ui32HeapLog2PageSize = ExactLog2(uiAlign);
+	}
+	else
+#endif
+	{
+		/* Meta and RiscV cores */
+		ui32HeapLog2PageSize = DevmemGetHeapLog2PageSize(psFwHeap);
+	}
+
+	eError = DevmemAllocateExportable(psDeviceNode,
+									  uiSize,
+									  uiAlign,
+									  ui32HeapLog2PageSize,
+									  uiFlags,
+									  pszText,
+									  ppsMemDescPtr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW DevmemAllocateExportable failed (%u)", eError));
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	/*
+		We need to map it so the heap for this allocation
+		is set
+	*/
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+							   psFwHeap,
+							   &sTmpDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsMemDescPtr);
+		PVR_DPF((PVR_DBG_ERROR, "FW DevmemMapToDevice failed (%u)", eError));
+	}
+
+	PVR_DPF_RETURN_RC1(eError, *ppsMemDescPtr);
+}
+
+static INLINE PVRSRV_ERROR DevmemFwAllocateSparse(PVRSRV_RGXDEV_INFO *psDevInfo,
+												IMG_DEVMEM_SIZE_T uiSize,
+												IMG_UINT32 ui32NumPhysChunks,
+												IMG_UINT32 ui32NumVirtChunks,
+												IMG_UINT32 *pui32MappingTable,
+												PVRSRV_MEMALLOCFLAGS_T uiFlags,
+												const IMG_CHAR *pszText,
+												DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_UINT32 ui32Align;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) && (pszText[0] == 'F') && (pszText[1] == 'w'));
+	ui32Align = GET_ROGUE_CACHE_LINE_SIZE(RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+	eError = DevmemAllocateSparse(psDevInfo->psDeviceNode,
+								uiSize,
+								ui32NumPhysChunks,
+								ui32NumVirtChunks,
+								pui32MappingTable,
+								ui32Align,
+								DevmemGetHeapLog2PageSize(psFwHeap),
+								uiFlags | PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING,
+								pszText,
+								ppsMemDescPtr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	/*
+		We need to map it so the heap for this allocation
+		is set
+	*/
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+				   psFwHeap,
+				   &sTmpDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsMemDescPtr);
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+static INLINE void DevmemFwUnmapAndFree(PVRSRV_RGXDEV_INFO *psDevInfo,
+								DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_DPF_ENTERED1(psMemDesc);
+
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	DevmemReleaseDevVirtAddr(psMemDesc);
+	DevmemFree(psMemDesc);
+
+	PVR_DPF_RETURN;
+}
+
+/*
+ * This function returns the value of the hardware register RGX_CR_TIMER
+ * which is a timer counting in ticks.
+ */
+
+static INLINE IMG_UINT64 RGXReadHWTimerReg(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 ui64Time = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TIMER);
+
+	/*
+	*  In order to avoid having to issue three 32-bit reads to detect the
+	*  lower 32-bits wrapping, the MSB of the low 32-bit word is duplicated
+	*  in the MSB of the high 32-bit word. If the wrap happens, we just read
+	*  the register again (it will not wrap again so soon).
+	*/
+	if ((ui64Time ^ (ui64Time << 32)) & ~RGX_CR_TIMER_BIT31_CLRMSK)
+	{
+		ui64Time = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TIMER);
+	}
+
+	return (ui64Time & ~RGX_CR_TIMER_VALUE_CLRMSK) >> RGX_CR_TIMER_VALUE_SHIFT;
+}
+
+/*
+ * This FW Common Context is only mapped into kernel for initialisation and cleanup purposes.
+ * Otherwise this allocation is only used by the FW.
+ * Therefore the GPU cache doesn't need coherency, and write-combine will
+ * suffice on the CPU side (WC buffer will be flushed at the first kick)
+ */
+#define RGX_FWCOMCTX_ALLOCFLAGS      (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                      PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)| \
+                                      PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                      PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                      PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+#define RGX_FWCODEDATA_ALLOCFLAGS    (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                      PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | \
+                                      PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)
+
+#define RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                         PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                         PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+#define RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CONFIG))
+
+#define RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+/* Firmware memory that is not accessible by the CPU. */
+#define RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                             PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                             PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC)
+
+/* Firmware shared memory that is supposed to be read-only to the CPU.
+ * In reality it isn't due to ZERO_ON_ALLOC which enforces CPU_WRITEABLE
+ * flag on the allocations. */
+#define RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC)
+
+/* data content being kept from previous boot cycles from physical memory must not be cleared during allocation */
+#define RGX_AUTOVZ_KEEP_FW_DATA_MASK(bKeepMem) ((bKeepMem) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL))
+
+/******************************************************************************
+ * RGXSetFirmwareAddress Flags
+ *****************************************************************************/
+#define RFW_FWADDR_FLAG_NONE		(0)			/*!< Void flag */
+#define RFW_FWADDR_NOREF_FLAG		(1U << 0)	/*!< It is safe to immediately release the reference to the pointer,
+												  otherwise RGXUnsetFirmwareAddress() must be call when finished. */
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferIsInitRequired
+
+@Description    Returns true if the firmware trace buffer is not allocated and
+                might be required by the firmware soon. Trace buffer allocated
+                on-demand to reduce RAM footprint on systems not needing
+                firmware trace.
+
+@Input          psDevInfo RGX device info
+
+@Return         IMG_BOOL  Whether on-demand allocation(s) is/are needed or not
+*/ /**************************************************************************/
+FORCE_INLINE IMG_BOOL RGXTraceBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	RGXFwSharedMemCacheOpValue(psTraceBufCtl->ui32LogType, INVALIDATE);
+
+	/* The firmware expects a trace buffer only when:
+	 *	- Logtype is "trace" AND
+	 *	- at least one LogGroup is configured
+	 *	- the Driver Mode is not Guest
+	 */
+	if ((psDevInfo->psRGXFWIfTraceBufferMemDesc[0] == NULL)
+		&& (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)
+		&& (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+		&& !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo, PVRSRV_MEMALLOCFLAGS_T uiAllocFlags);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferIsInitRequired
+
+@Description    Returns true if the firmware tbi buffer is not allocated and
+                might be required by the firmware soon. TBI buffer allocated
+                on-demand to reduce RAM footprint on systems not needing
+                tbi.
+
+@Input          psDevInfo RGX device info
+
+@Return         IMG_BOOL  Whether on-demand allocation(s) is/are needed or not
+*/ /**************************************************************************/
+FORCE_INLINE IMG_BOOL RGXTBIBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	RGXFwSharedMemCacheOpValue(psTraceBufCtl->ui32LogType, INVALIDATE);
+
+	/* The firmware expects a tbi buffer only when:
+	 *	- Logtype is "tbi"
+	 */
+	if ((psDevInfo->psRGXFWIfTBIBufferMemDesc == NULL)
+		 && (psTraceBufCtl->ui32LogType & ~RGXFWIF_LOG_TYPE_TRACE)
+		 && (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                              RGX_INIT_APPHINTS        *psApphints,
+                              IMG_UINT32               ui32ConfigFlags,
+                              IMG_UINT32               ui32ConfigFlagsExt,
+                              IMG_UINT32               ui32FwOsCfgFlags);
+
+
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO   *psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags);
+
+/*************************************************************************/ /*!
+@Function       RGXSetFirmwareAddress
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          ppDest          Address of the pointer to set
+@Input          psSrc           MemDesc describing the pointer
+@Input          ui32Flags       Any combination of RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+								   DEVMEM_MEMDESC		*psSrc,
+								   IMG_UINT32			uiOffset,
+								   IMG_UINT32			ui32Flags);
+
+
+/*************************************************************************/ /*!
+@Function       RGXSetMetaDMAAddress
+
+@Description    Fills a Firmware structure used to setup the Meta DMA with two
+                pointers to the same data, one on 40 bit and one on 32 bit
+                (pointer in the FW memory space).
+
+@Input          ppDest          Address of the structure to set
+@Input          psSrcMemDesc    MemDesc describing the pointer
+@Input          psSrcFWDevVAddr Firmware memory space pointer
+
+@Return         void
+*/ /**************************************************************************/
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset);
+
+
+/*************************************************************************/ /*!
+@Function       RGXUnsetFirmwareAddress
+
+@Description    Unsets a pointer in a firmware data structure
+
+@Input          psSrc           MemDesc describing the pointer
+
+@Return         void
+*/ /**************************************************************************/
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc);
+
+/*!
+*******************************************************************************
+@Function       RGXScheduleProcessQueuesKM
+
+@Description    Software command complete handler
+                (sends uncounted kicks for all the DMs through the MISR)
+
+@Input          hCmdCompHandle  RGX device node
+
+@Return         None
+******************************************************************************/
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle);
+
+
+/*!
+*******************************************************************************
+
+@Function       RGXInstallProcessQueuesMISR
+
+@Description    Installs the MISR to handle Process Queues operations
+
+@Input          phMISR          Pointer to the MISR handler
+@Input          psDeviceNode    RGX Device node
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll);
+
+/*************************************************************************/ /*!
+@Function       RGXSendCommandWithPowLockAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM without honouring
+                pending cache operations but taking the power lock.
+
+@Input          psDevInfo       Device Info
+@Input          psKCCBCmd       The cmd to send.
+@Input          ui32PDumpFlags  Pdump flags
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32			*pui32CmdKCCBSlot);
+
+#define RGXSendCommandWithPowLock(psDevInfo, psKCCBCmd, ui32PDumpFlags) \
+  RGXSendCommandWithPowLockAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, NULL)
+
+/*************************************************************************/ /*!
+@Function       RGXSendCommandAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM without honouring
+                pending cache operations or the power lock.
+                The function flushes any deferred KCCB commands first.
+
+@Input          psDevInfo       Device Info
+@Input          psKCCBCmd       The cmd to send.
+@Input          uiPdumpFlags    PDump flags.
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+										  RGXFWIF_KCCB_CMD   *psKCCBCmd,
+										  PDUMP_FLAGS_T      uiPdumpFlags,
+										  IMG_UINT32         *pui32CmdKCCBSlot);
+
+#define RGXSendCommand(psDevInfo, psKCCBCmd, ui32PDumpFlags) \
+  RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, NULL)
+
+/*************************************************************************/ /*!
+@Function       _RGXScheduleCommandAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM and kicks the firmware but
+                first schedules any commands which have to happen before
+                handle
+
+@Input          psDevInfo          Device Info
+@Input          eDM                To which DM the cmd is sent.
+@Input          psKCCBCmd          The cmd to send.
+@Input          ui32PDumpFlags     PDump flags
+@Input          bCallerHasPwrLock  Caller already has power lock
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											   RGXFWIF_DM         eKCCBType,
+											   RGXFWIF_KCCB_CMD   *psKCCBCmd,
+											   IMG_UINT32         ui32PDumpFlags,
+											   IMG_UINT32         *pui32CmdKCCBSlot,
+											   IMG_BOOL           bCallerHasPwrLock);
+
+#define RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot) \
+	_RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot, IMG_FALSE)
+
+#define RGXScheduleCommand(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags) \
+  RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, NULL)
+
+#define RGXScheduleCommandWithoutPowerLock(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags) \
+	_RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, NULL, IMG_TRUE)
+
+/*************************************************************************/ /*!
+@Function       RGXWaitForKCCBSlotUpdate
+
+@Description    Waits until the required kCCB slot value is updated by the FW
+                (signifies command completion). Additionally, dumps a relevant
+                PDump poll command.
+
+@Input          psDevInfo       Device Info
+@Input          ui32SlotNum     The kCCB slot number to wait for an update on
+@Input          ui32PDumpFlags
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXFrameworkCopyCommand
+
+@Description    Copy framework command into FW addressable buffer
+
+@param          psDeviceNode
+@param          psFWFrameworkMemDesc
+@param          pbyGPUFRegisterList
+@param          ui32FrameworkRegisterSize
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+										   DEVMEM_MEMDESC   *psFWFrameworkMemDesc,
+										   IMG_PBYTE        pbyGPUFRegisterList,
+										   IMG_UINT32       ui32FrameworkRegisterSize);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXFrameworkCreateKM
+
+@Description    Create FW addressable buffer for framework
+
+@param          psDeviceNode
+@param          ppsFWFrameworkMemDesc
+@param          ui32FrameworkRegisterSize
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE * psDeviceNode,
+										DEVMEM_MEMDESC     ** ppsFWFrameworkMemDesc,
+										IMG_UINT32         ui32FrameworkRegisterSize);
+
+/*************************************************************************/ /*!
+@Function       RGXPollForGPCommandCompletion
+
+@Description    Polls for completion of a submitted GP command. Poll is done
+                on a value matching a masked read from the address.
+
+@Input          psDevNode       Pointer to device node struct
+@Input          pui32LinMemAddr CPU linear address to poll
+@Input          ui32Value       Required value
+@Input          ui32Mask        Mask
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE *psDevNode,
+									volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+									IMG_UINT32                   ui32Value,
+									IMG_UINT32                   ui32Mask);
+
+/*************************************************************************/ /*!
+@Function       RGXStateFlagCtrl
+
+@Description    Set and return FW internal state flags.
+
+@Input          psDevInfo       Device Info
+@Input          ui32Config      AppHint config flags
+@Output         pui32State      Current AppHint state flag configuration
+@Input          bSetNotClear    Set or clear the provided config flags
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32State,
+				IMG_BOOL bSetNotClear);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestCommonContextCleanUp
+
+@Description    Schedules a FW common context cleanup. The firmware doesn't
+                block waiting for the resource to become idle but rather
+                notifies the host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psServerCommonContext context to be cleaned up
+@Input          eDM             Data master, to which the cleanup command should
+                                be sent
+@Input          ui32PDumpFlags  PDump continuous flag
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestHWRTDataCleanUp
+
+@Description    Schedules a FW HWRTData memory cleanup. The firmware doesn't
+                block waiting for the resource to become idle but rather
+                notifies the host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psHWRTData      firmware address of the HWRTData for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+										 PRGXFWIF_HWRTDATA psHWRTData);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestFreeListCleanUp
+
+@Description    Schedules a FW FreeList cleanup. The firmware doesn't block
+                waiting for the resource to become idle but rather notifies the
+                host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psFWFreeList    firmware address of the FreeList for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDeviceNode,
+										 PRGXFWIF_FREELIST psFWFreeList);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestZSBufferCleanUp
+
+@Description    Schedules a FW ZS Buffer cleanup. The firmware doesn't block
+                waiting for the resource to become idle but rather notifies the
+                host that the resource is busy.
+
+@Input          psDevInfo       pointer to device node
+@Input          psFWZSBuffer    firmware address of the ZS Buffer for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer);
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetHCSDeadline
+
+@Description    Requests the Firmware to set a new Hard Context Switch timeout
+                deadline. Context switches that surpass that deadline cause the
+                system to kill the currently running workloads.
+
+@Input          psDeviceNode    pointer to device node
+@Input          ui32HCSDeadlineMs  The deadline in milliseconds.
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs);
+
+/*!
+*******************************************************************************
+@Function       RGXFWHealthCheckCmdInt
+
+@Description    Ping the firmware to check if it is responsive.
+
+@Input          psDevInfo          pointer to device info
+@Input          bCallerHasPwrLock  Caller already has power lock
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock);
+
+#define RGXFWHealthCheckCmd(psDevInfo)	\
+	RGXFWHealthCheckCmdInt(psDevInfo, IMG_FALSE)
+
+#define RGXFWHealthCheckCmdWithoutPowerLock(psDevInfo)	\
+	RGXFWHealthCheckCmdInt(psDevInfo, IMG_TRUE)
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetFwOsState
+
+@Description    Requests the Firmware to change the guest OS Online states.
+                This should be initiated by the VMM when a guest VM comes
+                online or goes offline. If offline, the FW offloads any current
+                resource from that DriverID. The request is repeated until the
+                FW has had time to free all the resources or has waited for
+                workloads to finish.
+
+@Input          psDevInfo       pointer to device info
+@Input          ui32DriverID    The driver whose state is being altered
+@Input          eOSOnlineState  The new state (Online or Offline)
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32DriverID,
+								RGXFWIF_OS_STATE_CHANGE eOSOnlineState);
+
+#if defined(SUPPORT_AUTOVZ)
+/*!
+*******************************************************************************
+@Function       RGXUpdateAutoVzWdgToken
+
+@Description    If the driver-firmware connection is active, read the
+                firmware's watchdog token and copy its value back into the OS
+                token. This indicates to the firmware that this driver is alive
+                and responsive.
+
+@Input          psDevInfo       pointer to device info
+******************************************************************************/
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXDisconnectAllGuests
+
+@Description    Send requests to FW to disconnect all guest connections.
+
+@Input          psDeviceNode    pointer to device node
+******************************************************************************/
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode);
+#endif
+
+/*!
+*******************************************************************************
+@Function       RGXFWConfigPHR
+
+@Description    Configure the Periodic Hardware Reset functionality
+
+@Input          psDevInfo       pointer to device info
+@Input          ui32PHRMode     desired PHR mode
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode);
+
+/*!
+*******************************************************************************
+@Function       RGXFWConfigWdg
+
+@Description    Configure the Safety watchdog trigger period
+
+@Input          psDevInfo        pointer to device info
+@Input          ui32WdgPeriodUs  requested period in microseconds
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32WdgPeriod);
+
+/*!
+*******************************************************************************
+@Function       RGXCheckFirmwareCCB
+
+@Description    Processes all commands that are found in the Firmware CCB.
+
+@Input          psDevInfo       pointer to device
+
+@Return         None
+******************************************************************************/
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXCheckForStalledClientContexts
+
+@Description    Checks all client contexts, for the device with device info
+                provided, to see if any are waiting for a fence to signal and
+                optionally force signalling of the fence for the context which
+                has been waiting the longest.
+                This function is called by RGXUpdateHealthStatus() and also
+                may be invoked from other trigger points.
+
+@Input          psDevInfo       pointer to device info
+@Input          bIgnorePrevious If IMG_TRUE, any stalled contexts will be
+                                indicated immediately, rather than only
+                                checking against any previous stalled contexts
+
+@Return         None
+******************************************************************************/
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious);
+
+/*!
+*******************************************************************************
+@Function       RGXUpdateHealthStatus
+
+@Description    Tests a number of conditions which might indicate a fatal error
+                has occurred in the firmware. The result is stored in the
+                device node eHealthStatus.
+
+@Input         psDevNode        Pointer to device node structure.
+@Input         bCheckAfterTimePassed  When TRUE, the function will also test
+                                for firmware queues and polls not changing
+                                since the previous test.
+
+                                Note: if not enough time has passed since the
+                                last call, false positives may occur.
+
+@Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed);
+
+#if defined(SUPPORT_AUTOVZ)
+/*!
+*******************************************************************************
+@Function       RGXUpdateAutoVzWatchdog
+
+@Description    Updates AutoVz watchdog that maintains the fw-driver connection
+
+@Input         psDevNode        Pointer to device node structure.
+******************************************************************************/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode);
+#endif /* SUPPORT_AUTOVZ */
+
+/*!
+*******************************************************************************
+@Function       AttachKickResourcesCleanupCtls
+
+@Description    Attaches the cleanup structures to a kick command so that
+                submission reference counting can be performed when the
+                firmware processes the command
+
+@Output         apsCleanupCtl   Array of CleanupCtl structure pointers to populate.
+@Output         pui32NumCleanupCtl  Number of CleanupCtl structure pointers written out.
+@Input          eDM             Which data master is the subject of the command.
+@Input          bKick           TRUE if the client originally wanted to kick this DM.
+@Input          psRTDataCleanup Optional RTData cleanup associated with the command.
+@Input          psZBuffer       Optional ZSBuffer associated with the command.
+
+@Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer);
+
+/*!
+*******************************************************************************
+@Function       RGXResetHWRLogs
+
+@Description    Resets the HWR Logs buffer
+                (the hardware recovery count is not reset)
+
+@Input          psDevNode       Pointer to the device
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*!
+*******************************************************************************
+@Function       RGXGetPhyAddr
+
+@Description    Get the physical address of a PMR at an offset within it
+
+@Input          psPMR           PMR of the allocation
+@Input          ui32LogicalOffset  Logical offset
+
+@Output         psPhyAddr       Physical address of the allocation
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid);
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+@Function       RGXPdumpDrainKCCB
+
+@Description    Wait for the firmware to execute all the commands in the kCCB
+
+@Input          psDevInfo       Pointer to the device
+@Input          ui32WriteOffset Woff we have to POL for the Roff to be equal to
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+							   IMG_UINT32 ui32WriteOffset);
+#endif /* PDUMP */
+
+/*!
+*******************************************************************************
+@Function       RGXFwRawHeapAllocMap
+
+@Description    Register and maps to device, a raw firmware physheap
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize);
+
+/*!
+*******************************************************************************
+@Function       RGXFwRawHeapUnmapFree
+
+@Description    Unregister and unmap from device, a raw firmware physheap
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID);
+
+/*!
+*******************************************************************************
+@Function       RGXReadFWModuleAddr
+
+@Description    Read a value at the given address in META or RISCV memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in META or RISCV memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                 IMG_UINT32 ui32Addr,
+                                 IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXWriteFWModuleAddr
+
+@Description    Write a value to the given address in META or RISC memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32 ui32MemAddr,
+                                  IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXGetFwMapping
+
+@Description    Retrieve any of the CPU Physical Address, Device Physical
+                Address or the raw value of the page table entry associated
+                with the firmware virtual address given.
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32FwVA        The Fw VA that needs decoding
+@Output         psCpuPA         Pointer to the resulting CPU PA
+@Output         psDevPA         Pointer to the resulting Dev PA
+@Output         pui64RawPTE     Pointer to the raw Page Table Entry value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                    IMG_UINT32 ui32FwVA,
+                                    IMG_CPU_PHYADDR *psCpuPA,
+                                    IMG_DEV_PHYADDR *psDevPA,
+                                    IMG_UINT64 *pui64RawPTE);
+
+/*!
+*******************************************************************************
+@Function       RGXFWInjectFault
+
+@Description    Injecting firmware fault to validate recovery through Host
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetVzConnectionCooldownPeriod
+
+@Description    Set Vz connection cooldown period
+
+@Input          psDevInfo                              pointer to device info
+@Input          ui32VzConnectionCooldownPeriodInSec    Cooldown period in secs
+******************************************************************************/
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(SUPPORT_AUTOVZ_HW_REGS) && !defined(SUPPORT_AUTOVZ)
+#error "VZ build configuration error: use of OS scratch registers supported only in AutoVz drivers."
+#endif
+
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+/* AutoVz with hw support */
+#define KM_GET_FW_CONNECTION(psDevInfo)				OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH3)
+#define KM_GET_OS_CONNECTION(psDevInfo)				OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH2)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH2, RGXFW_CONNECTION_OS_##val)
+
+#define KM_GET_FW_ALIVE_TOKEN(psDevInfo)			OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH1)
+#define KM_GET_OS_ALIVE_TOKEN(psDevInfo)			OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH0)
+#define KM_SET_OS_ALIVE_TOKEN(val, psDevInfo)		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH0, val)
+
+#define KM_ALIVE_TOKEN_CACHEOP(Target, CacheOp)
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)
+
+#else
+
+#if defined(SUPPORT_AUTOVZ)
+#define KM_GET_FW_ALIVE_TOKEN(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->ui32AliveFwToken)
+#define KM_GET_OS_ALIVE_TOKEN(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->ui32AliveOsToken)
+#define KM_SET_OS_ALIVE_TOKEN(val, psDevInfo)		do { \
+                                                    OSWriteDeviceMem32WithWMB((volatile IMG_UINT32 *) &psDevInfo->psRGXFWIfConnectionCtl->ui32AliveOsToken, val); \
+                                                    KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH); \
+                                                    } while (0)
+
+#define KM_ALIVE_TOKEN_CACHEOP(Target, CacheOp)     RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfConnectionCtl->ui32Alive##Target##Token, \
+													                           CacheOp);
+#endif /* defined(SUPPORT_AUTOVZ) */
+
+#if !defined(NO_HARDWARE) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* static, dynamic and AutoVz DDKs using shared memory */
+#define KM_GET_FW_CONNECTION(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->eConnectionFwState)
+#define KM_GET_OS_CONNECTION(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->eConnectionOsState)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)	do { \
+                                                OSWriteDeviceMem32WithWMB((void*)&psDevInfo->psRGXFWIfConnectionCtl->eConnectionOsState, RGXFW_CONNECTION_OS_##val); \
+                                                KM_CONNECTION_CACHEOP(Os, FLUSH); \
+                                                } while (0)
+
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)  RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfConnectionCtl->eConnection##Target##State, \
+												                           CacheOp);
+#else
+/* nohw & native */
+#define KM_GET_FW_CONNECTION(psDevInfo)			(RGXFW_CONNECTION_FW_ACTIVE)
+#define KM_GET_OS_CONNECTION(psDevInfo)			(RGXFW_CONNECTION_OS_ACTIVE)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED == 1) */
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+#define RGX_FIRST_RAW_HEAP_DRIVER_ID		RGXFW_HOST_DRIVER_ID
+#else
+#define RGX_FIRST_RAW_HEAP_DRIVER_ID		RGXFW_GUEST_DRIVER_ID_START
+#endif
+
+#define KM_OS_CONNECTION_IS(val, psDevInfo)		(KM_GET_OS_CONNECTION(psDevInfo) == RGXFW_CONNECTION_OS_##val)
+#define KM_FW_CONNECTION_IS(val, psDevInfo)		(KM_GET_FW_CONNECTION(psDevInfo) == RGXFW_CONNECTION_FW_##val)
+
+#endif /* RGXFWUTILS_H */
+/******************************************************************************
+ End of file (rgxfwutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf.h
new file mode 100644
index 000000000000..06d3c3ae0487
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf.h
@@ -0,0 +1,109 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX HWPerf functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHWPERF_H_
+#define RGXHWPERF_H_
+
+#include "rgxhwperf_common.h"
+
+/******************************************************************************
+ * RGX HW Performance Profiling API(s)
+ *****************************************************************************/
+
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+PVRSRV_ERROR PVRSRVRGXConfigMuxHWPerfCountersKM(
+	CONNECTION_DATA               *psConnection,
+	PVRSRV_DEVICE_NODE            *psDeviceNode,
+	IMG_UINT32                     ui32ArrayLen,
+	RGX_HWPERF_CONFIG_MUX_CNTBLK  *psBlockConfigs);
+
+
+PVRSRV_ERROR PVRSRVRGXConfigCustomCountersKM(
+	CONNECTION_DATA    * psConnection,
+	PVRSRV_DEVICE_NODE * psDeviceNode,
+	IMG_UINT16           ui16CustomBlockID,
+	IMG_UINT16           ui16NumCustomCounters,
+	IMG_UINT32         * pui32CustomCounterIDs);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+	CONNECTION_DATA       * psConnection,
+	PVRSRV_DEVICE_NODE    * psDeviceNode,
+	IMG_UINT32            ui32CtrlWord,
+	IMG_UINT32            ui32ArrayLen,
+	RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs);
+
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(CONNECTION_DATA *psConnection,
+                                                       PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                       const IMG_UINT32 ui32BlockID,
+                                                       RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                     RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                     IMG_UINT32 ui32BlockID,
+                                                     RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLength,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs);
+
+typedef struct HWPERF_STREAM_DESC_TAG HWPERF_STREAM_DESC;
+
+PVRSRV_ERROR PVRSRVRGXOpenHWPerfClientStreamKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               HWPERF_STREAM_DESC **ppsSD);
+PVRSRV_ERROR PVRSRVRGXCloseHWPerfClientStreamKM(HWPERF_STREAM_DESC *psSD);
+PVRSRV_ERROR PVRSRVRGXWriteHWPerfClientEventKM(HWPERF_STREAM_DESC *psSD,
+                                               IMG_UINT32 uiSize,
+                                               IMG_BYTE *puiData);
+
+#endif /* RGXHWPERF_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.c
new file mode 100644
index 000000000000..b440560f2141
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.c
@@ -0,0 +1,4318 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+#include "tlserver.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+#include "ospvr_gputrace.h"
+#endif
+
+#include "info_page_defs.h"
+
+/* This is defined by default to enable producer callbacks.
+ * Clients of the TL interface can disable the use of the callback
+ * with PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK. */
+#define SUPPORT_TL_PRODUCER_CALLBACK 1
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+/* Defines size of buffers returned from acquire/release calls */
+#define FW_STREAM_BUFFER_SIZE (0x80000)
+#define HOST_STREAM_BUFFER_SIZE (0x20000)
+
+/* Must be at least as large as two tl packets of maximum size */
+static_assert(HOST_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "HOST_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+static_assert(FW_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "FW_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+
+IMG_INTERNAL /*static inline*/ IMG_UINT32 RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **);
+static IMG_UINT64 RGXHWPerfFwSetEventFilterNoLock(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                                  IMG_UINT64 uiFilter);
+
+static inline IMG_UINT32
+RGXHWPerfGetPackets(IMG_UINT32  ui32BytesExp,
+                    IMG_UINT32  ui32AllowedSize,
+                    RGX_PHWPERF_V2_PACKET_HDR psCurPkt )
+{
+	IMG_UINT32 sizeSum = 0;
+	RGXFwSharedMemCacheOpValue(psCurPkt->ui32Size, INVALIDATE);
+
+	/* Traverse the array to find how many packets will fit in the available space. */
+	while ( sizeSum + RGX_HWPERF_GET_SIZE(psCurPkt) <= ui32BytesExp &&
+			sizeSum + RGX_HWPERF_GET_SIZE(psCurPkt) < ui32AllowedSize )
+	{
+		sizeSum += RGX_HWPERF_GET_SIZE(psCurPkt);
+		psCurPkt = RGX_HWPERF_GET_NEXT_PACKET(psCurPkt);
+		RGXFwSharedMemCacheOpValue(psCurPkt->ui32Size, INVALIDATE);
+	}
+
+	return sizeSum;
+}
+
+static inline void
+RGXSuspendHWPerfL2DataCopy(PVRSRV_RGXDEV_INFO* psDeviceInfo,
+                           RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                           IMG_BOOL bIsReaderConnected)
+{
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	if (!bIsReaderConnected)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s : HWPerf FW events enabled but L2 host buffer "
+		        "for stream %u is full and no reader is currently connected, suspending "
+		        "event collection. Connect a reader or restart driver to avoid event loss.",
+		        __func__, eL2StreamId));
+		psDeviceInfo->bSuspendHWPerfL2DataCopy[eL2StreamId] = IMG_TRUE;
+	}
+}
+
+static IMG_UINT32 RGXHWPerfCopyData(PVRSRV_RGXDEV_INFO *psDeviceInfo,
+                                    IMG_BYTE *pbSrcBuffer,
+                                    RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                    IMG_UINT32 uiBytesToCopy)
+{
+	IMG_BYTE *pbDestBuffer;
+	IMG_UINT32 uiBytesCopied = 0;
+	IMG_UINT32 uiFreeSpace;
+	IMG_UINT32 uiBytesToCopyMin = RGX_HWPERF_GET_SIZE(RGX_HWPERF_GET_PACKET(pbSrcBuffer));
+	IMG_BOOL bIsReaderConnected;
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hHWPerfDestStream;
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	hHWPerfDestStream = psDeviceInfo->hHWPerfStream[eL2StreamId];
+
+	PVR_DPF_ENTERED;
+
+	/* Try submitting all data in one TL packet. */
+	eError = TLStreamReserve2(hHWPerfDestStream, &pbDestBuffer, uiBytesToCopy, uiBytesToCopyMin,
+	                          &uiFreeSpace, &bIsReaderConnected);
+	if (eError == PVRSRV_OK)
+	{
+		RGXFwSharedMemCacheOpExec(pbSrcBuffer, uiBytesToCopy, PVRSRV_CACHE_OP_INVALIDATE);
+		OSDeviceMemCopy(pbDestBuffer, pbSrcBuffer, (size_t) uiBytesToCopy);
+
+		eError = TLStreamCommit(hHWPerfDestStream, uiBytesToCopy);
+		PVR_LOG_GOTO_IF_ERROR_VA(eError, ErrReturn, "TLStreamCommit() failed with error %d, "
+		                         "unable to copy packet from L1 to L2 buffer", eError);
+
+		/* Data were successfully written */
+		uiBytesCopied = (size_t) uiBytesToCopy;
+	}
+	else if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		/* There was not enough space for all data, copy as much as possible */
+		IMG_UINT32 uiSizeSum = RGXHWPerfGetPackets(uiBytesToCopy, uiFreeSpace,
+		                                           RGX_HWPERF_GET_PACKET(pbSrcBuffer));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Unable to reserve space (%d) in host buffer on first attempt, "
+		         "remaining free space: %d", uiBytesToCopy, uiFreeSpace));
+
+		if (uiSizeSum != 0)
+		{
+			eError = TLStreamReserve(hHWPerfDestStream, &pbDestBuffer, uiSizeSum);
+
+			if (eError == PVRSRV_OK)
+			{
+				RGXFwSharedMemCacheOpExec(pbSrcBuffer, uiSizeSum, PVRSRV_CACHE_OP_INVALIDATE);
+				OSDeviceMemCopy(pbDestBuffer, pbSrcBuffer, (size_t) uiSizeSum);
+
+				eError = TLStreamCommit(hHWPerfDestStream, uiSizeSum);
+				PVR_LOG_GOTO_IF_ERROR_VA(eError, ErrReturn, "TLStreamCommit() failed with error "
+				                         "%d, unable to copy packet from L1 to L2 buffer", eError);
+
+				/* uiSizeSum bytes of hwperf packets have been successfully written */
+				uiBytesCopied = uiSizeSum;
+			}
+			else if (eError == PVRSRV_ERROR_STREAM_FULL)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "Cannot write HWPerf packet into host buffer, check data "
+				         "in case of packet loss, remaining free space: %d", uiFreeSpace));
+				RGXSuspendHWPerfL2DataCopy(psDeviceInfo, eL2StreamId, bIsReaderConnected);
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "Cannot find space in host buffer, check data in case of "
+			         "packet loss, remaining free space: %d", uiFreeSpace));
+			RGXSuspendHWPerfL2DataCopy(psDeviceInfo, eL2StreamId, bIsReaderConnected);
+		}
+	}
+
+	/* Some other error occurred. Full error handled by caller, we returning the copied bytes count
+	 * to caller */
+	if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_STREAM_FULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "HWPerf enabled: Unexpected Error (%d) while copying FW buffer "
+		         "to destination buffer.", eError));
+	}
+
+ErrReturn:
+	/* Return the remaining packets left to be transported. */
+	PVR_DPF_RETURN_VAL(uiBytesCopied);
+}
+
+/*
+  RGXHWPerfGetMaxTransfer
+ */
+static IMG_UINT32 RGXHWPerfGetMaxTransfer(PVRSRV_RGXDEV_INFO *psDeviceInfo,
+                                          IMG_UINT32 ui32BytesExp,
+                                          IMG_UINT32 uiL2StreamCopyMask)
+{
+	IMG_UINT32 uiMaxXfer = ui32BytesExp;
+	IMG_UINT32 eL2StreamId;
+
+	for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+	{
+		if (BIT_ISSET(uiL2StreamCopyMask, eL2StreamId))
+		{
+			IMG_UINT32 uiMaxXferSize = TLStreamGetMaxTransfer(
+			                             ui32BytesExp,
+			                             psDeviceInfo->hHWPerfStream[eL2StreamId]);
+
+			if (uiMaxXferSize < uiMaxXfer)
+			{
+				/* New minimum size found, save it for later */
+#if defined(DEBUG)
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s(dev %u, len %u, mask %u) New/Old = [0x%x/0x%x]",
+				         __func__, eL2StreamId, ui32BytesExp,
+				         uiL2StreamCopyMask, uiMaxXferSize, uiMaxXfer));
+#endif
+
+				uiMaxXfer = uiMaxXferSize;
+			}
+#if defined(DEBUG)
+			else
+			{
+				PVR_DPF((PVR_DBG_VERBOSE,
+				         "%s(dev %u, len %u, mask %u) New/Old = [0x%x/0x%x]",
+				         __func__, eL2StreamId, ui32BytesExp,
+				         uiL2StreamCopyMask, uiMaxXferSize, uiMaxXfer));
+			}
+#endif
+		}
+	}
+
+	return uiMaxXfer;
+}
+
+/*
+	RGXHWPerfCopyDataL1toL2
+ */
+static IMG_UINT32 RGXHWPerfCopyDataL1toL2(PVRSRV_RGXDEV_INFO* psDeviceInfo,
+                                          IMG_BYTE   *pbFwBuffer,
+                                          IMG_UINT32 uiL1DataSize,
+                                          IMG_UINT32 uiL2StreamCopyMask)
+{
+	IMG_UINT32 eL2StreamId, ui32L2AvailableSpace, uiHWPerfBytesCopied = 0;
+
+	/* HWPERF_MISR_FUNC_DEBUG enables debug code for investigating HWPerf issues */
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	static IMG_UINT32 gui32Ordinal = IMG_UINT32_MAX;
+#endif
+
+	/* Invalidate initial packet header, type/size cast via RGX_HWPEF_GET_PACKET */
+	RGXFwSharedMemCacheOpPtr(RGX_HWPERF_GET_PACKET(pbFwBuffer), INVALIDATE);
+
+	PVR_DPF_ENTERED;
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	PVR_DPF((PVR_DBG_VERBOSE, "EVENTS to copy from 0x%p length:%05d mask:0x%2x",
+	         pbFwBuffer, uiL1DataSize, uiL2StreamCopyMask));
+#endif
+
+	/* Determine the maximum space available in all consumer (L2) streams.
+	 * This limits the amount of data that will be copied if we have multiple
+	 * L2 consumers registered. In this case we only transfer the amount of
+	 * data that can fit into all available L2 consumer streams and so we will
+	 * exert back-pressure onto the L1 buffer whenever one of the L2 consumer
+	 * streams gets filled.
+	 * This effectively lock-steps the L2 consumer streams together.
+	 */
+	ui32L2AvailableSpace = RGXHWPerfGetMaxTransfer(psDeviceInfo, uiL1DataSize, uiL2StreamCopyMask);
+
+	for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+	{
+		if (BIT_ISSET(uiL2StreamCopyMask, eL2StreamId))
+		{
+			IMG_UINT32 uiBytesCopied, uiPacketDataSize = ui32L2AvailableSpace;
+			IMG_UINT32 uiL2MaxPacketSize = psDeviceInfo->ui32L2BufMaxPacketSize[eL2StreamId];
+
+			/* Adjust the copied data size if the maximum packet size is smaller
+			 * than the available space or if the L1 data we want to read can't
+			 * fit into the available space. */
+			if (ui32L2AvailableSpace > uiL2MaxPacketSize || ui32L2AvailableSpace < uiL1DataSize)
+			{
+				uiPacketDataSize = RGXHWPerfGetPackets(ui32L2AvailableSpace, uiL2MaxPacketSize,
+				                                       RGX_HWPERF_GET_PACKET(pbFwBuffer));
+
+				if (uiPacketDataSize == 0)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Failed to write data into host buffer "
+					        "(%u) as packet is too big (%u) and can't fit into available "
+					        "space (%u)", eL2StreamId, uiL1DataSize,
+					        MIN(ui32L2AvailableSpace, uiL2MaxPacketSize)));
+
+					continue;
+				}
+			}
+
+			uiBytesCopied = RGXHWPerfCopyData(psDeviceInfo, pbFwBuffer, eL2StreamId,
+			                                  uiPacketDataSize);
+
+			uiHWPerfBytesCopied = MAX(uiBytesCopied, uiHWPerfBytesCopied);
+		}
+	}
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	if (uiHWPerfBytesCopied > 0)
+	{
+		/* Check the incoming buffer of data has not lost any packets */
+		IMG_BYTE *pbFwBufferIter = pbFwBuffer;
+		IMG_BYTE *pbFwBufferEnd = pbFwBuffer + uiHWPerfBytesCopied;
+
+		do
+		{
+			RGX_HWPERF_V2_PACKET_HDR *asCurPos = RGX_HWPERF_GET_PACKET(pbFwBufferIter);
+			IMG_UINT32 ui32CurOrdinal;
+			/* Invalidate HDR pointed to by asCurPos as we use both ordinal for detecting
+			 * lost packets and size for iteration.
+			 */
+			RGXFwSharedMemCacheOpPtr(asCurPos, INVALIDATE);
+			ui32CurOrdinal = asCurPos->ui32Ordinal;
+			if (gui32Ordinal != IMG_UINT32_MAX)
+			{
+				if ((gui32Ordinal+1) != ui32CurOrdinal)
+				{
+					if (gui32Ordinal < ui32CurOrdinal)
+					{
+						PVR_DPF((PVR_DBG_WARNING,
+								"HWPerf [%p] packets lost (%u packets) between ordinal %u...%u",
+								pbFwBufferIter,
+								ui32CurOrdinal - gui32Ordinal - 1,
+								gui32Ordinal,
+								ui32CurOrdinal));
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_WARNING,
+								"HWPerf [%p] packet ordinal out of sequence last: %u, current: %u",
+								pbFwBufferIter,
+								gui32Ordinal,
+								ui32CurOrdinal));
+					}
+				}
+			}
+			gui32Ordinal = asCurPos->ui32Ordinal;
+			pbFwBufferIter += RGX_HWPERF_GET_SIZE(asCurPos);
+		} while (pbFwBufferIter < pbFwBufferEnd);
+	}
+#endif
+
+	/* Return the remaining packets left to be transported. */
+	PVR_DPF_RETURN_VAL(uiHWPerfBytesCopied);
+}
+
+
+static INLINE IMG_UINT32 RGXHWPerfAdvanceRIdx(
+		const IMG_UINT32 ui32BufSize,
+		const IMG_UINT32 ui32Pos,
+		const IMG_UINT32 ui32Size)
+{
+	return ( ui32Pos + ui32Size < ui32BufSize ? ui32Pos + ui32Size : 0 );
+}
+
+
+/*
+	RGXHWPerfDataStore
+
+	This function copies HWPerf data from L1 buffer to all L2 streams.
+	The number of copied data is always the maximum read number of packets.
+	In case where one of the stream is not able to accept the same amount of
+	data as other streams it will suffer from gaps in the data.
+
+	To avoid these gaps, we will exert back-pressure on the L1 buffer by
+	pre-calculating the maximum amount of data that can be copied to all
+	L2 streams (currently a maximum of 2).
+ */
+static IMG_UINT32 RGXHWPerfDataStore(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFWIF_SYSDATA	*psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	IMG_BYTE*				psHwPerfInfo = psDevInfo->psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32SrcRIdx, ui32SrcWIdx, ui32SrcWrapCount;
+	IMG_UINT32				ui32BytesExp = 0, ui32BytesCopied = 0, ui32BytesCopiedSum = 0;
+	IMG_UINT32				uiStreamCopyMask = 0;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	IMG_UINT32				ui32BytesExpSum = 0;
+#endif
+
+	/* It's unlikely that we're ever going to have more than 32 consumers
+	 * for the Firmware L1 buffer but check just to be safe. */
+	static_assert(RGX_HWPERF_L2_STREAM_LAST <= sizeof(uiStreamCopyMask) * 8,
+	              "RGX_HWPERF_L2_STREAM_LAST cannot be greater than 32.")
+
+	PVR_DPF_ENTERED;
+
+	/* Caller should check this member is valid before calling */
+	{
+		RGX_HWPERF_L2_STREAM_ID eL2StreamId;
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+		IMG_UINT32 uiNotNullCount = 0;
+#endif
+		for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+		{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+			if (psDevInfo->hHWPerfStream[eL2StreamId] != NULL)
+			{
+				uiNotNullCount++;
+			}
+#endif
+			if (!psDevInfo->bSuspendHWPerfL2DataCopy[eL2StreamId])
+			{
+				BIT_SET(uiStreamCopyMask, eL2StreamId);
+			}
+		}
+
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+		/* At least one stream must exist. */
+		PVR_ASSERT(uiNotNullCount > 0);
+#endif
+
+		/* Only proceed if any of the streams are not suspended.
+		 * Build bit field representing each L2 stream's suspend status. This
+		 * will be passed down the stack for copy function to determine if the
+		 * data should or should not be copied. */
+		if (uiStreamCopyMask == 0)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s : Copying data to all L2 host buffers for FW events is "
+			        "suspended. Start at least one of the HWPerf consumers or restart the driver "
+			        "if HWPerf FW events are needed", __func__));
+
+			PVR_DPF_RETURN_VAL(0);
+		}
+	}
+
+	/* Invalidate partial region of struct */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl,
+	                           INVALIDATE);
+
+	/* Get a copy of the current
+	 *   read (first packet to read)
+	 *   write (empty location for the next write to be inserted)
+	 *   WrapCount (size in bytes of the buffer at or past end)
+	 * indexes of the FW buffer */
+	ui32SrcRIdx = psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx;
+	ui32SrcWIdx = psFwSysData->sHWPerfCtrl.ui32HWPerfWIdx;
+	OSMemoryBarrier(NULL);
+	ui32SrcWrapCount = psFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount;
+
+#if defined(HWPERF_MISR_FUNC_DEBUG) || defined(EMULATOR)
+	{
+		IMG_UINT32  ui32SrcBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+
+		if (ui32SrcRIdx >= ui32SrcBufSize || ui32SrcWIdx >= ui32SrcBufSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s : Invalid read/write offsets found! srcRIdx:%u srcWIdx:%u srcBufSize:%u",
+			        __func__, ui32SrcRIdx, ui32SrcWIdx, ui32SrcBufSize));
+
+			PVR_DPF_RETURN_VAL(ui32BytesCopiedSum);
+		}
+	}
+#endif
+
+	/* Is there any data in the buffer not yet retrieved? */
+	if ( ui32SrcRIdx != ui32SrcWIdx )
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfDataStore EVENTS found srcRIdx:%d srcWIdx: %d", ui32SrcRIdx, ui32SrcWIdx));
+
+		/* Is the write position higher than the read position? */
+		if ( ui32SrcWIdx > ui32SrcRIdx )
+		{
+			/* Yes, buffer has not wrapped */
+			ui32BytesExp = ui32SrcWIdx - ui32SrcRIdx;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+			ui32BytesExpSum += ui32BytesExp;
+#endif
+			ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+			                                          psHwPerfInfo + ui32SrcRIdx,
+			                                          ui32BytesExp,
+			                                          uiStreamCopyMask);
+
+			/* Advance the read index and the free bytes counter by the number
+			 * of bytes transported. Items will be left in buffer if not all data
+			 * could be transported. Exit to allow buffer to drain. */
+			OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                          RGXHWPerfAdvanceRIdx(psDevInfo->ui32RGXFWIfHWPerfBufSize,
+			                                               ui32SrcRIdx,
+			                                               ui32BytesCopied));
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                           FLUSH);
+
+			ui32BytesCopiedSum += ui32BytesCopied;
+		}
+		/* No, buffer has wrapped and write position is behind read position */
+		else
+		{
+			/* Byte count equal to
+			 *     number of bytes from read position to the end of the buffer,
+			 *   + data in the extra space in the end of the buffer. */
+			ui32BytesExp = ui32SrcWrapCount - ui32SrcRIdx;
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+			ui32BytesExpSum += ui32BytesExp;
+#endif
+			ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+			                                          psHwPerfInfo + ui32SrcRIdx,
+			                                          ui32BytesExp,
+			                                          uiStreamCopyMask);
+
+			/* Advance read index as before and Update the local copy of the
+			 * read index as it might be used in the last if branch*/
+			ui32SrcRIdx = RGXHWPerfAdvanceRIdx(
+					psDevInfo->ui32RGXFWIfHWPerfBufSize, ui32SrcRIdx,
+					ui32BytesCopied);
+
+			/* Update Wrap Count */
+			if ( ui32SrcRIdx == 0)
+			{
+				OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount,
+				                          psDevInfo->ui32RGXFWIfHWPerfBufSize);
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount,
+				                           FLUSH);
+			}
+			OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx, ui32SrcRIdx);
+
+			ui32BytesCopiedSum += ui32BytesCopied;
+
+			/* If all the data in the end of the array was copied, try copying
+			 * wrapped data in the beginning of the array, assuming there is
+			 * any and the RIdx was wrapped. */
+			if (   (ui32BytesCopied == ui32BytesExp)
+					&& (ui32SrcWIdx > 0)
+					&& (ui32SrcRIdx == 0) )
+			{
+				ui32BytesExp = ui32SrcWIdx;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+				ui32BytesExpSum += ui32BytesExp;
+#endif
+				ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+				                                          psHwPerfInfo,
+				                                          ui32BytesExp,
+				                                          uiStreamCopyMask);
+				/* Advance the FW buffer read position. */
+				psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = RGXHWPerfAdvanceRIdx(
+						psDevInfo->ui32RGXFWIfHWPerfBufSize, ui32SrcRIdx,
+						ui32BytesCopied);
+
+				ui32BytesCopiedSum += ui32BytesCopied;
+			}
+			/* This flush covers both writes above */
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                           FLUSH);
+		}
+#ifdef HWPERF_MISR_FUNC_DEBUG
+		if (ui32BytesCopiedSum != ui32BytesExpSum)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfDataStore: FW L1 RIdx:%u. Not all bytes copied to L2: %u bytes out of %u expected", psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx, ui32BytesCopiedSum, ui32BytesExpSum));
+		}
+#endif
+
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_VERBOSE, "RGXHWPerfDataStore NO EVENTS to transport"));
+	}
+
+	PVR_DPF_RETURN_VAL(ui32BytesCopiedSum);
+}
+
+/* Function called from MISR to copy data from L1 buffer to L2 streams. */
+PVRSRV_ERROR RGXHWPerfDataStoreCB(PVRSRV_DEVICE_NODE *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+	IMG_UINT32          ui32BytesCopied;
+
+	PVR_ASSERT(psDevInfo);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevInfo, PVRSRV_OK);
+
+	PVR_DPF_ENTERED;
+
+	psRgxDevInfo = psDevInfo->pvDevice;
+
+	/* Store FW event data if the destination buffer exists.*/
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->uiHWPerfStreamCount > 0)
+	{
+		ui32BytesCopied = RGXHWPerfDataStore(psRgxDevInfo);
+		if ( ui32BytesCopied )
+		{
+			/* It's possible that the HWPerf stream doesn't exist yet. It's
+			 * possible that only FTrace L2 stream has been created so far. */
+			if (psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] != NULL)
+			{
+				/* Signal consumers that packets may be available to read when
+				 * running from a HW kick, not when called by client APP thread
+				 * via the transport layer CB as this can lead to stream
+				 * corruption. */
+				eError = TLStreamSync(psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF]);
+				PVR_LOG_IF_ERROR(eError, "TLStreamSync");
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfDataStoreCB: Zero bytes copied"));
+			RGXDEBUG_PRINT_IRQ_COUNT(psRgxDevInfo);
+		}
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+/* Currently supported by default */
+#if !defined(NO_HARDWARE) && defined(SUPPORT_TL_PRODUCER_CALLBACK)
+static PVRSRV_ERROR RGXHWPerfTLCB(IMG_HANDLE hStream,
+                                  IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)pvUser;
+
+	PVR_UNREFERENCED_PARAMETER(hStream);
+	PVR_UNREFERENCED_PARAMETER(ui32Resp);
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	switch (ui32ReqOp)
+	{
+		case TL_SOURCECB_OP_CLIENT_EOS:
+			/* Keep HWPerf resource init check and use of
+			 * resources atomic, they may not be freed during use
+			 */
+
+			/* This solution is for avoiding a deadlock situation where -
+			 * in DoTLStreamReserve(), writer has acquired HWPerfLock and
+			 * ReadLock and is waiting on ReadPending (which will be reset
+			 * by reader), And
+			 * the reader after setting ReadPending in TLStreamAcquireReadPos(),
+			 * is waiting for HWPerfLock in RGXHWPerfTLCB().
+			 * So here in RGXHWPerfTLCB(), if HWPerfLock is already acquired we
+			 * will return to the reader without waiting to acquire HWPerfLock.
+			 */
+			if (!OSTryLockAcquire(psRgxDevInfo->hHWPerfLock))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "hHWPerfLock is already acquired, a write "
+						"operation might already be in process"));
+				return PVRSRV_OK;
+			}
+
+			if (psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] != NULL)
+			{
+				(void) RGXHWPerfDataStore(psRgxDevInfo);
+			}
+			OSLockRelease(psRgxDevInfo->hHWPerfLock);
+			break;
+
+		default:
+			break;
+	}
+
+	return eError;
+}
+#endif
+
+
+static void RGXHWPerfL1BufferDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	if (psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc)
+	{
+		if (psRgxDevInfo->psRGXFWIfHWPerfBuf != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+			psRgxDevInfo->psRGXFWIfHWPerfBuf = NULL;
+		}
+		DevmemFwUnmapAndFree(psRgxDevInfo, psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+		psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc = NULL;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInit
+
+@Description    Called during driver init for initialization of HWPerf module
+				in the Rogue device driver. This function keeps allocated
+				only the minimal necessary resources, which are required for
+				functioning of HWPerf server module.
+
+@Input          psRgxDevInfo	RGX Device Info
+
+@Return			PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PVR_DPF_ENTERED;
+
+	/* expecting a valid device info */
+	PVR_RETURN_IF_INVALID_PARAM(psRgxDevInfo != NULL);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_OK);
+
+	/* Create a lock for HWPerf server module used for serializing, L1 to L2
+	 * copy calls (e.g. in case of TL producer callback) and L1, L2 resource
+	 * allocation */
+	eError = OSLockCreate(&psRgxDevInfo->hHWPerfLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	/* Initialise only non-zero fields since psRgxDevInfo is zeroed
+	 * on allocation. */
+	for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+	{
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[i] = IMG_TRUE;
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfIsInitRequired
+
+@Description    Returns true if the HWperf firmware buffer (L1 buffer) and host
+                driver TL buffer (L2 buffer) are not already allocated. Caller
+                must possess hHWPerfLock lock before calling this
+                function so the state tested is not inconsistent.
+
+@Input          psRgxDevInfo RGX Device Info, on which init requirement is
+                checked.
+
+@Return         IMG_BOOL	Whether initialization (allocation) is required
+ */ /**************************************************************************/
+static INLINE IMG_BOOL RGXHWPerfIsInitRequired(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                               RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hHWPerfLock));
+
+#if !defined(NO_HARDWARE)
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Both L1 and L2 buffers are required (for HWPerf functioning) on driver
+	 * built for actual hardware (TC, EMU, etc.)
+	 */
+	return psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc == NULL ||
+	       psRgxDevInfo->hHWPerfStream[eL2StreamId] == NULL;
+#else
+	/* On a NO-HW driver L2 is not allocated. So, no point in checking its
+	 * allocation */
+	return psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc == NULL;
+#endif
+}
+#if !defined(NO_HARDWARE)
+static void _HWPerfFWOnReaderOpenCB(void *pvArg)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	PVRSRV_DEVICE_NODE* psDevNode = (PVRSRV_DEVICE_NODE*) psRgxDevInfo->psDeviceNode;
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	IMG_UINT32 ui32kCCBCommandSlot;
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDevNode);
+
+	/* Clear any previously suspended state for bSuspendHWPerfL2DataCopy as we
+	 * now have a reader attached so the data will be delivered upstream. */
+	if (psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF])
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Resuming HWPerf FW event collection.",
+		        __func__));
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF] = IMG_FALSE;
+	}
+
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG;
+	sKccbCmd.uCmdData.sHWPerfCtrl.eOpCode = RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV;
+	sKccbCmd.uCmdData.sHWPerfCtrl.ui64Mask = 0;
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevNode->pvDevice,
+											  RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to generate feature packet in "
+				"firmware (error = %d)", __func__, eError));
+		return;
+	}
+
+	eError = RGXWaitForKCCBSlotUpdate(psRgxDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+}
+
+static void _HWPerfFWOnReaderCloseCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF] = IMG_TRUE;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInitOnDemandL1Buffer
+
+@Description    This function allocates the HWperf firmware buffer (L1 buffer)
+                if HWPerf is enabled at driver load time. Otherwise, this
+                buffer is allocated on-demand as and when required. Caller must
+                possess hHWPerfLock lock before calling this function so the
+                state tested is not inconsistent if called outside of
+                initialisation.
+
+@Input          psRgxDevInfo RGX Device Info, on which init is done
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInitOnDemandL1Buffer(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_ERROR eError;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_DPF_ENTERED;
+
+	/* This function might be called more than once due to initialisation of
+	 * multiple consumers. Make sure that L1 is only ever initialised once. */
+	if (psRgxDevInfo->psRGXFWIfHWPerfBuf != NULL)
+	{
+		PVR_DPF_RETURN_OK;
+	}
+
+	/* Create the L1 HWPerf buffer on demand, read-only for the CPU
+	 * (except for the zero/poison operations) */
+	uiMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)
+						| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+						| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE
+						| PVRSRV_MEMALLOCFLAG_GPU_UNCACHED
+						| PVRSRV_MEMALLOCFLAG_CPU_READABLE
+						| PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC
+						| PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+#if defined(PDUMP) /* Helps show where the packet data ends */
+						| PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC
+#else /* Helps show corruption issues in driver-live */
+						| PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC
+#endif
+						| PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC
+						| PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	/* Allocate HWPerf FW L1 buffer */
+	eError = DevmemFwAllocate(psRgxDevInfo,
+	                          /* Pad it enough to hold the biggest variable sized packet. */
+	                          psRgxDevInfo->ui32RGXFWIfHWPerfBufSize+RGX_HWPERF_MAX_PACKET_SIZE,
+	                          uiMemAllocFlags,
+	                          "FwHWPerfBuffer",
+	                          &psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate kernel fw hwperf buffer (%u)",
+				__func__, eError));
+		goto ErrReturn;
+	}
+
+	/* Expecting the RuntimeCfg structure is mapped into CPU virtual memory.
+	 * Also, make sure the FW address is not already set */
+	PVR_ASSERT(psRgxDevInfo->psRGXFWIfRuntimeCfg && psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr == 0x0);
+
+	/* Meta cached flag removed from this allocation as it was found
+	 * FW performance was better without it. */
+	eError = RGXSetFirmwareAddress(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf,
+	                      psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                      0, RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrDeInitL1Buffer);
+
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+	RGXSetMetaDMAAddress(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfDMABuf,
+	                     psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                     &psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf,
+	                     0);
+#endif
+
+	/* flush write buffers for psRgxDevInfo->psRGXFWIfRuntimeCfg */
+	OSWriteMemoryBarrier(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr);
+	RGXFwSharedMemCacheOpValue(psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr, FLUSH);
+
+	eError = DevmemAcquireCpuVirtAddr(psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                                  (void**)&psRgxDevInfo->psRGXFWIfHWPerfBuf);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire kernel hwperf buffer (%u)",
+				__func__, eError));
+		goto ErrDeInitL1Buffer;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf buffer size in bytes: L1: %d",
+	         psRgxDevInfo->ui32RGXFWIfHWPerfBufSize));
+
+	PVR_DPF_RETURN_OK;
+
+ErrDeInitL1Buffer:
+	/* L1 buffer initialisation failures */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+ErrReturn:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInitOnDemandL2Stream
+
+@Description    This function allocates the HWperf firmware buffer (L1 buffer)
+                and host driver TL buffer (L2 buffer) if HWPerf is enabled at
+                driver load time. Otherwise, these buffers are allocated
+                on-demand as and when required. Caller must possess hHWPerfLock
+                lock before calling this function so the state tested is not
+                inconsistent if called outside of driver initialisation.
+
+@Input          psRgxDevInfo RGX Device Info, on which init is done
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInitOnDemandL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                           RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if !defined(NO_HARDWARE)
+	IMG_HANDLE hStream;
+	TL_STREAM_INFO sTLStreamInfo;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	PVR_DPF_ENTERED;
+
+#if !defined(NO_HARDWARE)
+	if (eL2StreamId == RGX_HWPERF_L2_STREAM_HWPERF)
+	{
+		/* On NO-HW driver, there is no MISR installed to copy data from L1 to L2. Hence,
+		 * L2 buffer is not allocated */
+		IMG_CHAR pszHWPerfStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 4];
+			/* + 4 is used to allow names up to "hwperf_fw_999", which is enough */
+
+		/* form the HWPerf stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfStreamName, sizeof(pszHWPerfStreamName), "%s%d",
+					   PVRSRV_TL_HWPERF_RGX_FW_STREAM,
+					   psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		eError = TLStreamCreate(&hStream,
+								pszHWPerfStreamName,
+								psRgxDevInfo->ui32RGXL2HWPerfBufSize,
+								TL_OPMODE_DROP_NEWER | TL_FLAG_NO_SIGNAL_ON_COMMIT,
+								_HWPerfFWOnReaderOpenCB, psRgxDevInfo,
+								_HWPerfFWOnReaderCloseCB, psRgxDevInfo,
+#if !defined(SUPPORT_TL_PRODUCER_CALLBACK)
+								NULL, NULL
+#else
+								/* Not enabled by default */
+								RGXHWPerfTLCB, psRgxDevInfo
+#endif
+								);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate", ErrClearStream);
+
+		eError = TLStreamSetNotifStream(hStream,
+										PVRSRVGetPVRSRVData()->hTLCtrlStream);
+		/* we can still discover host stream so leave it as is and just log error */
+		PVR_LOG_IF_ERROR(eError, "TLStreamSetNotifStream");
+
+		/* send the event here because host stream is implicitly opened for write
+		 * in TLStreamCreate and TLStreamOpen is never called (so the event is
+		 * never emitted) */
+		TLStreamMarkStreamOpen(hStream);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf buffer size in bytes: L1: %d  L2: %d",
+			psRgxDevInfo->ui32RGXFWIfHWPerfBufSize,
+			psRgxDevInfo->ui32RGXL2HWPerfBufSize));
+
+		psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] = hStream;
+		psRgxDevInfo->uiHWPerfStreamCount++;
+		PVR_ASSERT(psRgxDevInfo->uiHWPerfStreamCount <= RGX_HWPERF_L2_STREAM_LAST);
+	}
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+	else if (eL2StreamId == RGX_HWPERF_L2_STREAM_FTRACE)
+	{
+		eError = PVRGpuTraceInitStream(psRgxDevInfo);
+		PVR_LOG_IF_ERROR(eError, "PVRGpuTraceInitStream");
+	}
+#endif
+
+	TLStreamInfo(psRgxDevInfo->hHWPerfStream[eL2StreamId], &sTLStreamInfo);
+	psRgxDevInfo->ui32L2BufMaxPacketSize[eL2StreamId] = sTLStreamInfo.maxTLpacketSize;
+#else
+	psRgxDevInfo->hHWPerfStream[eL2StreamId] = NULL;
+#endif /* !defined(NO_HARDWARE) */
+
+	PVR_DPF_RETURN_OK;
+
+#if !defined(NO_HARDWARE)
+ErrClearStream: /* L2 buffer initialisation failures */
+	psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] = NULL;
+#endif
+	/* L1 buffer initialisation failures */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+void RGXHWPerfDeinitL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                             RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	IMG_HANDLE hStream;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+
+	hStream = psRgxDevInfo->hHWPerfStream[eL2StreamId];
+
+	/* Clean up the L2 buffer stream object if allocated */
+	if (hStream)
+	{
+		psRgxDevInfo->hHWPerfStream[eL2StreamId] = NULL;
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[eL2StreamId] = IMG_TRUE;
+		psRgxDevInfo->uiHWPerfStreamCount--;
+		PVR_ASSERT(psRgxDevInfo->uiHWPerfStreamCount < RGX_HWPERF_L2_STREAM_LAST);
+
+		/* send the event here because host stream is implicitly opened for
+		 * write in TLStreamCreate and TLStreamClose is never called (so the
+		 * event is never emitted) */
+		TLStreamMarkStreamClose(hStream);
+		TLStreamClose(hStream);
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void RGXHWPerfDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	/* Cleanup L1 buffer resources */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+
+	/* Cleanup the HWPerf server module lock resource */
+	if (psRgxDevInfo->hHWPerfLock)
+	{
+		OSLockDestroy(psRgxDevInfo->hHWPerfLock);
+		psRgxDevInfo->hHWPerfLock = NULL;
+	}
+}
+
+
+/******************************************************************************
+ * RGX HW Performance Profiling Server API(s)
+ *****************************************************************************/
+
+static PVRSRV_ERROR RGXHWPerfCtrlFwBuffer(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                          IMG_BOOL bToggle,
+                                          IMG_UINT64 ui64Mask)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevice = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_UINT64 ui64MaskValue = ui64Mask;
+	IMG_UINT64 ui64OldMaskValue;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Modify mask to include the default bit settings if it is non-zero */
+	if (!bToggle && (ui64Mask != 0ULL))
+	{
+		ui64MaskValue = ui64Mask | RGX_HWPERF_EVENT_MASK_DEFAULT;
+	}
+
+	/* If this method is being used whether to enable or disable
+	 * then the hwperf buffers (host and FW) are likely to be needed
+	 * eventually so create them, also helps unit testing. Buffers
+	 * allocated on demand to reduce RAM foot print on systems not
+	 * needing HWPerf resources.
+	 * Obtain lock first, test and init if required. */
+	OSLockAcquire(psDevice->hHWPerfLock);
+
+	if (!psDevice->bFirmwareInitialised)
+	{
+		/* No other initialisation can be done at this point until the FW is
+		 * initialised so unlock, log and return Ok so the caller knows
+		 * the filter was set. */
+		(void) RGXHWPerfFwSetEventFilterNoLock(psDevice, eL2StreamId, ui64MaskValue);
+		OSLockRelease(psDevice->hHWPerfLock);
+		goto done_;
+	}
+
+	if (RGXHWPerfIsInitRequired(psDevice, eL2StreamId))
+	{
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Initialisation of on-demand HWPerfFW "
+					"resources failed", __func__));
+			goto unlock_and_return;
+		}
+
+		/* if this fails it also cleans up L1 buffer */
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevice, eL2StreamId);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Initialisation of on-demand HWPerfFW "
+					"resources failed", __func__));
+			goto unlock_and_return;
+		}
+	}
+
+	/* Unlock here as no further HWPerf resources are used below that would be
+	 * affected if freed by another thread */
+	OSLockRelease(psDevice->hHWPerfLock);
+
+	/* Return if the filter is the same */
+	if (!bToggle && psDevice->ui64HWPerfFilter[eL2StreamId] == ui64MaskValue)
+	{
+		goto done_;
+	}
+
+	ui64OldMaskValue = psDevice->ui64HWPerfFilter[eL2StreamId];
+	ui64MaskValue = RGXHWPerfFwSetEventFilter(psDevice, eL2StreamId, bToggle
+	                                          ? psDevice->ui64HWPerfFilter[eL2StreamId] ^ ui64MaskValue
+	                                          : ui64MaskValue);
+
+	/* Prepare command parameters ... */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG;
+	sKccbCmd.uCmdData.sHWPerfCtrl.eOpCode = bToggle ? RGXFWIF_HWPERF_CTRL_TOGGLE : RGXFWIF_HWPERF_CTRL_SET;
+	sKccbCmd.uCmdData.sHWPerfCtrl.ui64Mask = ui64MaskValue;
+
+	/* Ask the FW to carry out the HWPerf configuration command */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set new HWPerfFW filter in "
+				"firmware (error = %d)", __func__, eError));
+		goto restore_mask_;
+	}
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", restore_mask_);
+
+done_:
+	return PVRSRV_OK;
+
+unlock_and_return:
+	OSLockRelease(psDevice->hHWPerfLock);
+
+	return eError;
+
+restore_mask_:
+	(void) RGXHWPerfFwSetEventFilter(psDevice, eL2StreamId, ui64OldMaskValue);
+
+	return eError;
+}
+
+#define HWPERF_HOST_MAX_DEFERRED_PACKETS 800
+
+static PVRSRV_ERROR RGXHWPerfCtrlHostBuffer(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_BOOL bToggle,
+                                            IMG_UINT32 ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevice = psDeviceNode->pvDevice;
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32 ui32OldFilter = psDevice->ui32HWPerfHostFilter;
+#endif
+
+	if (psDevice->hHWPerfHostStream == NULL)
+	{
+		eError = RGXHWPerfHostInitOnDemandResources(psDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Initialisation of on-demand HWPerfHost resources failed",
+					 __func__));
+			return eError;
+		}
+	}
+
+	OSLockAcquire(psDevice->hLockHWPerfHostStream);
+	psDevice->ui32HWPerfHostFilter = bToggle ?
+			psDevice->ui32HWPerfHostFilter ^ ui32Mask : ui32Mask;
+
+	// Deferred creation of host periodic events thread
+	if (psDevice->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HOST_INFO))
+	{
+		eError = PVRSRVCreateHWPerfHostThread(PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVCreateHWPerfHostThread");
+	}
+	else
+	{
+		eError = PVRSRVDestroyHWPerfHostThread();
+		PVR_LOG_IF_ERROR(eError, "PVRSRVDestroyHWPerfHostThread");
+	}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	// Log deferred events stats if filter changed from non-zero to zero
+	if ((ui32OldFilter != 0) && (psDevice->ui32HWPerfHostFilter == 0))
+	{
+		PVR_LOG(("HWPerfHost deferred events buffer high-watermark / size: (%u / %u)",
+				psDevice->ui32DEHighWatermark, HWPERF_HOST_MAX_DEFERRED_PACKETS));
+
+		PVR_LOG(("HWPerfHost deferred event retries: WaitForAtomicCtxPktHighWatermark(%u) "
+				"WaitForRightOrdPktHighWatermark(%u)",
+				psDevice->ui32WaitForAtomicCtxPktHighWatermark,
+				psDevice->ui32WaitForRightOrdPktHighWatermark));
+	}
+#endif
+
+	OSLockRelease(psDevice->hLockHWPerfHostStream);
+
+	if (bToggle)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfHost events (%x) have been TOGGLED",
+				ui32Mask));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfHost mask has been SET to (%x)",
+				ui32Mask));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXHWPerfCtrlClientBuffer(IMG_BOOL bToggle,
+                                              IMG_UINT32 ui32InfoPageIdx,
+                                              IMG_UINT32 ui32Mask)
+{
+	PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+
+	PVR_LOG_RETURN_IF_FALSE(ui32InfoPageIdx >= INFO_PAGE_HWPERF_BLOCK_START &&
+	                        ui32InfoPageIdx < INFO_PAGE_HWPERF_BLOCK_END,
+	                        "invalid info page index", PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquire(psData->hInfoPageLock);
+	psData->pui32InfoPage[ui32InfoPageIdx] = bToggle ?
+			psData->pui32InfoPage[ui32InfoPageIdx] ^ ui32Mask : ui32Mask;
+	OSLockRelease(psData->hInfoPageLock);
+
+	if (bToggle)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfClient (%u) events (%x) have been TOGGLED",
+				ui32InfoPageIdx, ui32Mask));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfClient (%u) mask has been SET to (%x)",
+				ui32InfoPageIdx, ui32Mask));
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(CONNECTION_DATA    *psConnection,
+                                                  PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                  RGX_HWPERF_BVNC    *psBVNC)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR        eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDeviceNode), "psDeviceNode invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = psDeviceNode->pvDevice;
+	eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, psBVNC);
+
+	return eError;
+}
+
+/*
+	AppHint interfaces
+ */
+static
+PVRSRV_ERROR RGXHWPerfSetFwFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT64 ui64Value)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	eError = RGXHWPerfCtrlFwBuffer(psDeviceNode, RGX_HWPERF_L2_STREAM_HWPERF,
+	                               IMG_FALSE, ui64Value);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Failed to set HWPerf firmware filter for device (%u)",
+		        psDeviceNode->sDevId.ui32InternalID));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfReadFwFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT64 *pui64Value)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pui64Value = psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF];
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfSetHostFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const void *psPrivate,
+                                    IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	eError = RGXHWPerfCtrlHostBuffer(psDeviceNode, IMG_FALSE, ui32Value);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Failed to set HWPerf firmware filter for device (%u)",
+		        psDeviceNode->sDevId.ui32InternalID));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfReadHostFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const void *psPrivate,
+                                     IMG_UINT32 *pui32Value)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32HWPerfHostFilter;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _ReadClientFilter(const PVRSRV_DEVICE_NODE *psDevice,
+                                      const void *psPrivData,
+                                      IMG_UINT32 *pui32Value)
+{
+	PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32Idx = (IMG_UINT32) (uintptr_t) psPrivData;
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	OSLockAcquire(psData->hInfoPageLock);
+	*pui32Value = psData->pui32InfoPage[ui32Idx];
+	OSLockRelease(psData->hInfoPageLock);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _WriteClientFilter(const PVRSRV_DEVICE_NODE *psDevice,
+                                       const void *psPrivData,
+                                       IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 ui32Idx = (IMG_UINT32) (uintptr_t) psPrivData;
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	return RGXHWPerfCtrlClientBuffer(IMG_FALSE, ui32Idx, ui32Value);
+}
+
+void RGXHWPerfInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersUINT64(APPHINT_ID_HWPerfFWFilter,
+	                                    RGXHWPerfReadFwFilter,
+	                                    RGXHWPerfSetFwFilter,
+	                                    psDeviceNode,
+	                                    NULL);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfHostFilter,
+	                                    RGXHWPerfReadHostFilter,
+	                                    RGXHWPerfSetHostFilter,
+	                                    psDeviceNode,
+	                                    NULL);
+}
+
+void RGXHWPerfClientInitAppHintCallbacks(void)
+{
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_Services,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_SERVICES_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_EGL,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_EGL_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenGLES,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_OPENGLES_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenCL,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_OPENCL_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_Vulkan,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_VULKAN_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenGL,
+		                                _ReadClientFilter,
+		                                _WriteClientFilter,
+		                                APPHINT_OF_DRIVER_NO_DEVICE,
+		                                (void *) HWPERF_FILTER_OPENGL_IDX);
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCountersKM(CONNECTION_DATA *psConnection,
+                                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                    const IMG_UINT32 ui32BlockID,
+                                                    RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psConfiguredCounters != NULL, "psConfiguredCounters is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXGetHWPerfCtl");
+
+	eError = PVRSRVRGXGetConfiguredHWPerfCounters(psDeviceNode,
+	                                              psHWPerfCtl,
+	                                              ui32BlockID,
+	                                              psConfiguredCounters);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRGXGetConfiguredHWPerfCounters");
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocksKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               const IMG_UINT32 ui32ArrayLen,
+                                               IMG_UINT32 *pui32BlockCount,
+                                               IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	IMG_UINT32 *pui32BlockIDs = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(pui32BlockCount != NULL, "pui32BlockCount is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXGetHWPerfCtl");
+
+	if (pui32EnabledBlockIDs != NULL)
+	{
+		pui32BlockIDs = OSAllocMem(sizeof(IMG_UINT32) * ui32ArrayLen);
+		if (pui32BlockIDs == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("OSAllocMem", eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+		}
+	}
+
+	eError = PVRSRVRGXGetEnabledHWPerfBlocks(psDeviceNode,
+	                                         psHWPerfCtl,
+	                                         ui32ArrayLen,
+	                                         pui32BlockCount,
+	                                         pui32BlockIDs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRGXGetEnabledHWPerfBlocks", Error);
+
+	if (pui32EnabledBlockIDs != NULL)
+	{
+		IMG_UINT32 i;
+		if (*pui32BlockCount > ui32ArrayLen)
+		{
+			*pui32BlockCount = 0;
+			PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+			PVR_LOG_GOTO_WITH_ERROR(__func__, eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+		}
+		else if (*pui32BlockCount < ui32ArrayLen)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen greater than the number of enabled blocks."));
+		}
+
+		for (i = 0; i < *pui32BlockCount; i++)
+		{
+			pui32EnabledBlockIDs[i] = pui32BlockIDs[i];
+		}
+	}
+
+Error:
+	if (pui32BlockIDs != NULL)
+	{
+		OSFreeMem(pui32BlockIDs);
+	}
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+static INLINE IMG_UINT32 _RGXHWPerfFixBufferSize(IMG_UINT32 ui32BufSizeKB)
+{
+	if (ui32BufSizeKB > HWPERF_HOST_TL_STREAM_SIZE_MAX)
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "RGXHWPerfHostInit: HWPerf Host buffer size "
+		         "value (%u) too big, using maximum (%u)",
+		         ui32BufSizeKB, HWPERF_HOST_TL_STREAM_SIZE_MAX));
+		return HWPERF_HOST_TL_STREAM_SIZE_MAX<<10;
+	}
+	else if (ui32BufSizeKB >= HWPERF_HOST_TL_STREAM_SIZE_MIN)
+	{
+		return ui32BufSizeKB<<10;
+	}
+	else if (ui32BufSizeKB > 0)
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "RGXHWPerfHostInit: HWPerf Host buffer size "
+		         "value (%u) too small, using minimum (%u)",
+		         ui32BufSizeKB, HWPERF_HOST_TL_STREAM_SIZE_MIN));
+		return HWPERF_HOST_TL_STREAM_SIZE_MIN<<10;
+	}
+	else
+	{
+		/* 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. */
+		return HWPERF_HOST_TL_STREAM_SIZE_DEFAULT<<10;
+	}
+}
+
+/******************************************************************************
+ * RGX HW Performance Host Stream API
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfHostInit
+
+@Description    Called during driver init for initialisation of HWPerfHost
+                stream in the Rogue device driver. This function keeps allocated
+                only the minimal necessary resources, which are required for
+                functioning of HWPerf server module.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfHostInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32BufSizeKB)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_INVALID_PARAM(psRgxDevInfo != NULL);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_OK);
+
+	eError = OSLockCreate(&psRgxDevInfo->hLockHWPerfHostStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", error);
+
+	psRgxDevInfo->hHWPerfHostStream = NULL;
+	psRgxDevInfo->ui32HWPerfHostFilter = 0; /* disable all events */
+	psRgxDevInfo->ui32HWPerfHostNextOrdinal = 1;
+	psRgxDevInfo->ui32HWPerfHostBufSize = _RGXHWPerfFixBufferSize(ui32BufSizeKB);
+	psRgxDevInfo->pvHostHWPerfMISR = NULL;
+	psRgxDevInfo->pui8DeferredEvents = NULL;
+	/* First packet has ordinal=1, so LastOrdinal=0 will ensure ordering logic
+	 * is maintained */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = 0;
+	psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+
+error:
+	return eError;
+}
+
+#define RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE \
+	((IMG_UINT32)(offsetof(RGX_HWPERF_HOST_CLIENT_INFO_DATA, uDetail) + \
+		sizeof(((RGX_HWPERF_HOST_CLIENT_INFO_DETAIL*)0)->sProcName.ui32Count)))
+
+static void _HWPerfHostOnConnectCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psDevice;
+	PVRSRV_ERROR eError;
+
+	RGXSRV_HWPERF_CLK_SYNC(pvArg);
+
+	psDevice = (PVRSRV_RGXDEV_INFO*) pvArg;
+
+	/* Handle the case where we may be being called as part of a multi-device
+	 * initialisation sequence. If the bDevInit2Done flag is not yet set we can
+	 * perform no action for this device. Simply return.
+	 */
+	if (!psDevice->bDevInit2Done)
+	{
+		return;
+	}
+
+	/* Handle the case where the RGX_HWPERF_HOST_INFO bit is set in the event filter
+	 * before the host stream is opened for reading by a HWPerf client.
+	 * Which can result in the host periodic thread sleeping for a long duration as TLStreamIsOpenForReading may return false. */
+	if (psDevice->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HOST_INFO))
+	{
+		eError = PVRSRVCreateHWPerfHostThread(PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVCreateHWPerfHostThread");
+	}
+
+	RGXSRV_HWPERF_DEVICE_INFO_FEATURES(psDevice);
+
+	if (RGXHWPerfHostIsEventEnabled(psDevice, RGX_HWPERF_HOST_CLIENT_INFO))
+	{
+		// GCC throws -Werror=frame-larger-than error if the frame size is > 1024 bytes,
+		// so use a heap allocation - is there an alternate solution?
+		IMG_BYTE *pbPktPayload = (IMG_BYTE*)OSAllocMem(RGX_HWPERF_MAX_PAYLOAD_SIZE);
+
+		if (pbPktPayload)
+		{
+			RGX_HWPERF_HOST_CLIENT_INFO_DATA *psHostClientInfo;
+			RGX_HWPERF_HOST_CLIENT_PROC_NAME *psProcName;
+			IMG_UINT32 ui32TotalPayloadSize, ui32NameLen, ui32ProcNamePktSize;
+			DLLIST_NODE *pNode, *pNext;
+
+			psHostClientInfo = IMG_OFFSET_ADDR(pbPktPayload,0);
+			psHostClientInfo->eType = RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME;
+			psHostClientInfo->uDetail.sProcName.ui32Count = 0U;
+			psProcName = psHostClientInfo->uDetail.sProcName.asProcNames;
+			ui32TotalPayloadSize = RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE;
+
+			OSLockAcquire(psDevice->psDeviceNode->hConnectionsLock);
+
+			// Announce current client connections to the reader
+			dllist_foreach_node(&psDevice->psDeviceNode->sConnections, pNode, pNext)
+			{
+				CONNECTION_DATA *psData = IMG_CONTAINER_OF(pNode, CONNECTION_DATA, sConnectionListNode);
+
+				ui32NameLen = OSStringLength(psData->pszProcName) + 1U;
+				ui32ProcNamePktSize = RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen);
+
+				// Unlikely case where we have too much data to fit into a single hwperf packet
+				if (ui32ProcNamePktSize + ui32TotalPayloadSize > RGX_HWPERF_MAX_PAYLOAD_SIZE)
+				{
+					RGXHWPerfHostPostRaw(psDevice, RGX_HWPERF_HOST_CLIENT_INFO, pbPktPayload, ui32TotalPayloadSize);
+
+					psHostClientInfo->uDetail.sProcName.ui32Count = 0U;
+					psProcName = psHostClientInfo->uDetail.sProcName.asProcNames;
+					ui32TotalPayloadSize = RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE;
+				}
+
+				// Setup packet data
+				psHostClientInfo->uDetail.sProcName.ui32Count++;
+				psProcName->uiClientPID = psData->pid;
+				psProcName->ui32Length = ui32NameLen;
+				(void)OSCachedMemCopy(psProcName->acName, psData->pszProcName, ui32NameLen);
+
+				psProcName = (RGX_HWPERF_HOST_CLIENT_PROC_NAME*)IMG_OFFSET_ADDR(psProcName, ui32ProcNamePktSize);
+				ui32TotalPayloadSize += ui32ProcNamePktSize;
+			}
+
+			OSLockRelease(psDevice->psDeviceNode->hConnectionsLock);
+			RGXHWPerfHostPostRaw(psDevice, RGX_HWPERF_HOST_CLIENT_INFO, pbPktPayload, ui32TotalPayloadSize);
+			OSFreeMem(pbPktPayload);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OUT OF MEMORY. Could not allocate memory for RGX_HWPERF_HOST_CLIENT_INFO_DATA packet.", __func__));
+		}
+	}
+}
+
+/* Avoiding a holder struct using fields below, as a struct gets along padding,
+ * packing, and other compiler dependencies, and we want a continuous stream of
+ * bytes for (header+data) for use in TLStreamWrite. See
+ * _HWPerfHostDeferredEventsEmitter().
+ *
+ * A deferred (UFO) packet is represented in memory as:
+ *     - IMG_BOOL                 --> Indicates whether a packet write is
+ *                                    "complete" by atomic context or not.
+ *     - RGX_HWPERF_V2_PACKET_HDR --.
+ *                                  |--> Fed together to TLStreamWrite for
+ *                                  |    deferred packet to be written to
+ *                                  |    HWPerfHost buffer
+ *     - RGX_HWPERF_HOST_UFO_DATA---`
+ *
+ * PS: Currently only UFO events are supported in deferred list */
+#define HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE (sizeof(IMG_BOOL) +\
+		sizeof(RGX_HWPERF_V2_PACKET_HDR) +\
+		sizeof(RGX_HWPERF_HOST_UFO_DATA))
+
+static void RGX_MISRHandler_HWPerfPostDeferredHostEvents(void *pvData);
+static void _HWPerfHostDeferredEventsEmitter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                             IMG_UINT32 ui32MaxOrdinal);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfHostInitOnDemandResources
+
+@Description    This function allocates the HWPerfHost buffer if HWPerf is
+                enabled at driver load time. Otherwise, these buffers are
+                allocated on-demand as and when required.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfHostInitOnDemandResources(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError;
+	/* 4 makes space up to "hwperf_host_999" streams */
+	IMG_CHAR pszHWPerfHostStreamName[sizeof(PVRSRV_TL_HWPERF_HOST_SERVER_STREAM) + 4];
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psRgxDevInfo->hHWPerfHostStream != NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf host stream already initialised"));
+		return PVRSRV_OK;
+	}
+
+	/* form the HWPerf host stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszHWPerfHostStreamName, sizeof(pszHWPerfHostStreamName), "%s%d",
+	               PVRSRV_TL_HWPERF_HOST_SERVER_STREAM,
+	               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to form HWPerf host stream name for device %d",
+				__func__,
+				psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = TLStreamCreate(&psRgxDevInfo->hHWPerfHostStream,
+	                        pszHWPerfHostStreamName, psRgxDevInfo->ui32HWPerfHostBufSize,
+	                        TL_OPMODE_DROP_NEWER,
+	                        _HWPerfHostOnConnectCB, psRgxDevInfo,
+	                        NULL, NULL, NULL, NULL);
+	PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamCreate");
+
+	eError = TLStreamSetNotifStream(psRgxDevInfo->hHWPerfHostStream,
+	                                PVRSRVGetPVRSRVData()->hTLCtrlStream);
+	/* we can still discover host stream so leave it as is and just log error */
+	PVR_LOG_IF_ERROR(eError, "TLStreamSetNotifStream");
+
+	/* send the event here because host stream is implicitly opened for write
+	 * in TLStreamCreate and TLStreamOpen is never called (so the event is
+	 * never emitted) */
+	eError = TLStreamMarkStreamOpen(psRgxDevInfo->hHWPerfHostStream);
+	PVR_LOG_IF_ERROR(eError, "TLStreamMarkStreamOpen");
+
+	/* HWPerfHost deferred events specific initialization */
+	eError = OSInstallMISR(&psRgxDevInfo->pvHostHWPerfMISR,
+	                       RGX_MISRHandler_HWPerfPostDeferredHostEvents,
+	                       psRgxDevInfo,
+	                       "RGX_HWPerfDeferredEventPoster");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR", err_install_misr);
+
+	eError = OSSpinLockCreate(&psRgxDevInfo->hHWPerfHostSpinLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate", err_spinlock_create);
+
+	psRgxDevInfo->pui8DeferredEvents = OSAllocMem(HWPERF_HOST_MAX_DEFERRED_PACKETS
+	                                              * HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE);
+	if (NULL == psRgxDevInfo->pui8DeferredEvents)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OUT OF MEMORY. Could not allocate memory for "
+				"HWPerfHost deferred events array", __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_alloc_deferred_events;
+	}
+	psRgxDevInfo->ui16DEReadIdx = 0;
+	psRgxDevInfo->ui16DEWriteIdx = 0;
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	psRgxDevInfo->ui32DEHighWatermark = 0;
+	psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark = 0;
+	psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark = 0;
+#endif
+
+	PVR_DPF((DBGPRIV_MESSAGE, "HWPerf Host buffer size is %uKB",
+			psRgxDevInfo->ui32HWPerfHostBufSize));
+
+	return PVRSRV_OK;
+
+err_alloc_deferred_events:
+	OSSpinLockDestroy(psRgxDevInfo->hHWPerfHostSpinLock);
+	psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+
+err_spinlock_create:
+	(void) OSUninstallMISR(psRgxDevInfo->pvHostHWPerfMISR);
+	psRgxDevInfo->pvHostHWPerfMISR = NULL;
+
+err_install_misr:
+	TLStreamMarkStreamClose(psRgxDevInfo->hHWPerfHostStream);
+	TLStreamClose(psRgxDevInfo->hHWPerfHostStream);
+	psRgxDevInfo->hHWPerfHostStream = NULL;
+
+	return eError;
+}
+
+void RGXHWPerfHostDeInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVR_ASSERT (psRgxDevInfo);
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+
+	if (psRgxDevInfo->pui8DeferredEvents)
+	{
+		OSFreeMem(psRgxDevInfo->pui8DeferredEvents);
+		psRgxDevInfo->pui8DeferredEvents = NULL;
+	}
+
+	if (psRgxDevInfo->hHWPerfHostSpinLock)
+	{
+		OSSpinLockDestroy(psRgxDevInfo->hHWPerfHostSpinLock);
+		psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+	}
+
+	if (psRgxDevInfo->pvHostHWPerfMISR)
+	{
+		(void) OSUninstallMISR(psRgxDevInfo->pvHostHWPerfMISR);
+		psRgxDevInfo->pvHostHWPerfMISR = NULL;
+	}
+
+	if (psRgxDevInfo->hHWPerfHostStream)
+	{
+		/* send the event here because host stream is implicitly opened for
+		 * write in TLStreamCreate and TLStreamClose is never called (so the
+		 * event is never emitted) */
+		TLStreamMarkStreamClose(psRgxDevInfo->hHWPerfHostStream);
+		TLStreamClose(psRgxDevInfo->hHWPerfHostStream);
+		psRgxDevInfo->hHWPerfHostStream = NULL;
+	}
+
+	if (psRgxDevInfo->hLockHWPerfHostStream)
+	{
+		OSLockDestroy(psRgxDevInfo->hLockHWPerfHostStream);
+		psRgxDevInfo->hLockHWPerfHostStream = NULL;
+	}
+}
+
+static IMG_UINT64 RGXHWPerfFwSetEventFilterNoLock(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                                  IMG_UINT64 uiFilter)
+{
+	IMG_UINT64 uiTmpFilter = 0;
+	IMG_UINT32 i;
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Set filter for the given L2 stream. */
+	psRgxDevInfo->ui64HWPerfFilter[eL2StreamId] = uiFilter;
+
+	/* Compute compound filter from all existing L2 streams' filters. */
+	for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+	{
+		uiTmpFilter |= psRgxDevInfo->ui64HWPerfFilter[i];
+	}
+
+	psRgxDevInfo->ui64HWPerfFwFilter = uiTmpFilter;
+
+#if !defined(NO_HARDWARE)
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerfFW mask has been SET to 0x%" IMG_UINT64_FMTSPECx
+	         " (stream %u value SET to 0x%" IMG_UINT64_FMTSPECx ")",
+	         psRgxDevInfo->ui64HWPerfFwFilter, eL2StreamId,
+	         psRgxDevInfo->ui64HWPerfFilter[eL2StreamId]));
+#endif
+
+	return uiTmpFilter;
+}
+
+IMG_UINT64 RGXHWPerfFwSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                     RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                     IMG_UINT64 uiFilter)
+{
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	uiFilter = RGXHWPerfFwSetEventFilterNoLock(psRgxDevInfo, eL2StreamId, uiFilter);
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+	return uiFilter;
+}
+
+inline void RGXHWPerfHostSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Filter)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+	psRgxDevInfo->ui32HWPerfHostFilter = ui32Filter;
+}
+
+inline IMG_BOOL RGXHWPerfHostIsEventEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo, RGX_HWPERF_HOST_EVENT_TYPE eEvent)
+{
+	PVR_ASSERT(psRgxDevInfo);
+	return (psRgxDevInfo->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(eEvent)) ? IMG_TRUE : IMG_FALSE;
+}
+
+#define MAX_RETRY_COUNT 80
+static inline void _PostFunctionPrologue(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_UINT32 ui32CurrentOrdinal)
+{
+	IMG_UINT32 ui32Retry = MAX_RETRY_COUNT;
+
+	PVR_ASSERT(psRgxDevInfo->hLockHWPerfHostStream != NULL);
+	PVR_ASSERT(psRgxDevInfo->hHWPerfHostStream != NULL);
+
+	OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+
+	/* First, flush pending events (if any) */
+	_HWPerfHostDeferredEventsEmitter(psRgxDevInfo, ui32CurrentOrdinal);
+
+	while ((ui32CurrentOrdinal != psRgxDevInfo->ui32HWPerfHostLastOrdinal + 1)
+		   && (--ui32Retry != 0))
+	{
+		/* Release lock and give a chance to a waiting context to emit the
+		 * expected packet */
+		OSLockRelease (psRgxDevInfo->hLockHWPerfHostStream);
+		OSSleepms(100);
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+	}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	if ((ui32Retry == 0) && !(psRgxDevInfo->bWarnedPktOrdinalBroke))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Will warn only once! Potential packet(s) lost after ordinal"
+				 " %u (Current ordinal = %u)",
+				 __func__,
+				 psRgxDevInfo->ui32HWPerfHostLastOrdinal, ui32CurrentOrdinal));
+		psRgxDevInfo->bWarnedPktOrdinalBroke = IMG_TRUE;
+	}
+
+	if (psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark < (MAX_RETRY_COUNT - ui32Retry))
+	{
+		psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark = MAX_RETRY_COUNT - ui32Retry;
+	}
+#endif
+}
+
+static inline void _PostFunctionEpilogue(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_UINT32 ui32CurrentOrdinal)
+{
+	/* update last ordinal emitted */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = ui32CurrentOrdinal;
+
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hLockHWPerfHostStream));
+	OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+}
+
+static inline IMG_UINT8 *_ReserveHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Size)
+{
+	IMG_UINT8 *pui8Dest;
+
+	PVRSRV_ERROR eError = TLStreamReserve(psRgxDevInfo->hHWPerfHostStream,
+	                                      &pui8Dest, ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not reserve space in %s buffer"
+				" (%d). Dropping packet.",
+				__func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+		return NULL;
+	}
+	PVR_ASSERT(pui8Dest != NULL);
+
+	return pui8Dest;
+}
+
+static inline void _CommitHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError = TLStreamCommit(psRgxDevInfo->hHWPerfHostStream,
+	                                     ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not commit data to %s"
+				" (%d)", __func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+	}
+}
+
+/* Returns IMG_TRUE if packet write passes, IMG_FALSE otherwise */
+static inline IMG_BOOL _WriteHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                          RGX_HWPERF_V2_PACKET_HDR *psHeader)
+{
+	PVRSRV_ERROR eError = TLStreamWrite(psRgxDevInfo->hHWPerfHostStream,
+	                                    IMG_OFFSET_ADDR(psHeader, 0), psHeader->ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not write packet in %s buffer"
+				" (%d). Dropping packet.",
+				__func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+	}
+
+	/* Regardless of whether write passed/failed, we consider it "written" */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = psHeader->ui32Ordinal;
+
+	return (eError == PVRSRV_OK);
+}
+
+/* Helper macros for deferred events operations */
+#define GET_DE_NEXT_IDX(_curridx) ((_curridx + 1) % HWPERF_HOST_MAX_DEFERRED_PACKETS)
+#define GET_DE_EVENT_BASE(_idx)   (IMG_OFFSET_ADDR(psRgxDevInfo->pui8DeferredEvents, \
+		(_idx) * HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE))
+
+#define GET_DE_EVENT_WRITE_STATUS(_base) ((IMG_BOOL*)((void *)(_base)))
+#define GET_DE_EVENT_DATA(_base)         (IMG_OFFSET_ADDR((_base), sizeof(IMG_BOOL)))
+
+/* Emits HWPerfHost event packets present in the deferred list stopping when one
+ * of the following cases is hit:
+ * case 1: Packet ordering breaks i.e. a packet found doesn't meet ordering
+ *         criteria (ordinal == last_ordinal + 1)
+ *
+ * case 2: A packet with ordinal > ui32MaxOrdinal is found
+ *
+ * case 3: Deferred list's (read == write) i.e. no more deferred packets.
+ *
+ * NOTE: Caller must possess the hLockHWPerfHostStream lock before calling
+ *       this function.*/
+static void _HWPerfHostDeferredEventsEmitter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                             IMG_UINT32 ui32MaxOrdinal)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader;
+	IMG_UINT32 ui32Retry;
+	IMG_UINT8  *pui8DeferredEvent;
+	IMG_BOOL   *pbPacketWritten;
+	IMG_BOOL   bWritePassed;
+
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hLockHWPerfHostStream));
+
+	while (psRgxDevInfo->ui16DEReadIdx != psRgxDevInfo->ui16DEWriteIdx)
+	{
+		pui8DeferredEvent = GET_DE_EVENT_BASE(psRgxDevInfo->ui16DEReadIdx);
+		pbPacketWritten   = GET_DE_EVENT_WRITE_STATUS(pui8DeferredEvent);
+		psHeader          = (RGX_HWPERF_V2_PACKET_HDR*) GET_DE_EVENT_DATA(pui8DeferredEvent);
+
+		for (ui32Retry = MAX_RETRY_COUNT; !(*pbPacketWritten) && (ui32Retry != 0); ui32Retry--)
+		{
+			/* Packet not yet written, re-check after a while. Wait for a short period as
+			 * atomic contexts are generally expected to finish fast */
+			OSWaitus(10);
+		}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+		if ((ui32Retry == 0) && !(psRgxDevInfo->bWarnedAtomicCtxPktLost))
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: Will warn only once. Dropping a deferred packet as atomic context"
+					 " took too long to write it",
+					 __func__));
+			psRgxDevInfo->bWarnedAtomicCtxPktLost = IMG_TRUE;
+		}
+
+		if (psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark < (MAX_RETRY_COUNT - ui32Retry))
+		{
+			psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark = MAX_RETRY_COUNT - ui32Retry;
+		}
+#endif
+
+		if (*pbPacketWritten)
+		{
+			if ((psHeader->ui32Ordinal > ui32MaxOrdinal) ||
+					(psHeader->ui32Ordinal != (psRgxDevInfo->ui32HWPerfHostLastOrdinal + 1)))
+			{
+				/* Leave remaining events to be emitted by next call to this function */
+				break;
+			}
+			bWritePassed = _WriteHWPerfStream(psRgxDevInfo, psHeader);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Atomic context packet lost!", __func__));
+			bWritePassed = IMG_FALSE;
+		}
+
+		/* Move on to next packet */
+		psRgxDevInfo->ui16DEReadIdx = GET_DE_NEXT_IDX(psRgxDevInfo->ui16DEReadIdx);
+
+		if (!bWritePassed // if write failed
+				&& ui32MaxOrdinal == IMG_UINT32_MAX // and we are from MISR
+				&& psRgxDevInfo->ui16DEReadIdx != psRgxDevInfo->ui16DEWriteIdx) // and there are more events
+		{
+			/* Stop emitting here and re-schedule MISR */
+			OSScheduleMISR(psRgxDevInfo->pvHostHWPerfMISR);
+			break;
+		}
+	}
+}
+
+static void RGX_MISRHandler_HWPerfPostDeferredHostEvents(void *pvData)
+{
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo = pvData;
+
+	OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+
+	/* Since we're called from MISR, there is no upper cap of ordinal to be emitted.
+	 * Send IMG_UINT32_MAX to signify all possible packets. */
+	_HWPerfHostDeferredEventsEmitter(psRgxDevInfo, IMG_UINT32_MAX);
+
+	OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+static inline void _UpdateDEBufferHighWatermark(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	IMG_UINT32 ui32DEWatermark;
+	IMG_UINT16 ui16LRead = psRgxDevInfo->ui16DEReadIdx;
+	IMG_UINT16 ui16LWrite = psRgxDevInfo->ui16DEWriteIdx;
+
+	if (ui16LWrite >= ui16LRead)
+	{
+		ui32DEWatermark = ui16LWrite - ui16LRead;
+	}
+	else
+	{
+		ui32DEWatermark = (HWPERF_HOST_MAX_DEFERRED_PACKETS - ui16LRead) + (ui16LWrite);
+	}
+
+	if (ui32DEWatermark > psRgxDevInfo->ui32DEHighWatermark)
+	{
+		psRgxDevInfo->ui32DEHighWatermark = ui32DEWatermark;
+	}
+}
+#endif
+
+/* @Description Gets the data/members that concerns the accuracy of a packet in HWPerfHost
+                buffer. Since the data returned by this function is required in both, an
+                atomic as well as a process/sleepable context, it is protected under spinlock
+
+   @Output      pui32Ordinal Pointer to ordinal number assigned to this packet
+   @Output      pui64Timestamp Timestamp value for this packet
+   @Output      ppui8Dest If the current context cannot sleep, pointer to a place in
+                          deferred events buffer where the packet data should be written.
+                          Don't care, otherwise.
+ */
+static void _GetHWPerfHostPacketSpecifics(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                          IMG_UINT32 *pui32Ordinal,
+                                          IMG_UINT64 *pui64Timestamp,
+                                          IMG_UINT8 **ppui8Dest,
+                                          IMG_BOOL    bSleepAllowed)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* Spin lock is required to avoid getting scheduled out by a higher priority
+	 * context while we're getting header specific details and packet place in
+	 * HWPerf buffer (when in atomic context) for ourselves */
+	OSSpinLockAcquire(psRgxDevInfo->hHWPerfHostSpinLock, uiFlags);
+
+	*pui32Ordinal = psRgxDevInfo->ui32HWPerfHostNextOrdinal++;
+	/* For HWPerf we always use monotonic clock. */
+	(void) OSClockMonotonicus64(pui64Timestamp);
+
+	if (!bSleepAllowed)
+	{
+		/* We're in an atomic context. So return the next position available in
+		 * deferred events buffer */
+		IMG_UINT16 ui16NewWriteIdx;
+		IMG_BOOL *pbPacketWritten;
+
+		PVR_ASSERT(ppui8Dest != NULL);
+
+		ui16NewWriteIdx = GET_DE_NEXT_IDX(psRgxDevInfo->ui16DEWriteIdx);
+		if (ui16NewWriteIdx == psRgxDevInfo->ui16DEReadIdx)
+		{
+			/* This shouldn't happen. HWPERF_HOST_MAX_DEFERRED_PACKETS should be
+			 * big enough to avoid any such scenario */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+			/* PVR_LOG/printk isn't recommended in atomic context. Perhaps we'll do
+			 * this debug output here when trace_printk support is added to DDK */
+//			PVR_LOG(("%s: No more space in deferred events buffer (%u/%u) W=%u,R=%u",
+//                     __func__, psRgxDevInfo->ui32DEHighWatermark,
+//					 HWPERF_HOST_MAX_DEFERRED_PACKETS, psRgxDevInfo->ui16DEWriteIdx,
+//					 psRgxDevInfo->ui16DEReadIdx));
+#endif
+			*ppui8Dest = NULL;
+		}
+		else
+		{
+			/* Return the position where deferred event would be written */
+			*ppui8Dest = GET_DE_EVENT_BASE(psRgxDevInfo->ui16DEWriteIdx);
+
+			/* Make sure packet write "state" is "write-pending" _before_ moving write
+			 * pointer forward */
+			pbPacketWritten = GET_DE_EVENT_WRITE_STATUS(*ppui8Dest);
+			*pbPacketWritten = IMG_FALSE;
+
+			psRgxDevInfo->ui16DEWriteIdx = ui16NewWriteIdx;
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+			_UpdateDEBufferHighWatermark(psRgxDevInfo);
+#endif
+		}
+	}
+
+	OSSpinLockRelease(psRgxDevInfo->hHWPerfHostSpinLock, uiFlags);
+}
+
+static inline void _SetupHostPacketHeader(IMG_UINT8 *pui8Dest,
+                                          RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+                                          IMG_UINT32 ui32Size,
+                                          IMG_UINT32 ui32Ordinal,
+                                          IMG_UINT64 ui64Timestamp)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader = (RGX_HWPERF_V2_PACKET_HDR *) ((void *)pui8Dest);
+
+	PVR_ASSERT(ui32Size<=RGX_HWPERF_MAX_PACKET_SIZE);
+
+	psHeader->ui32Ordinal = ui32Ordinal;
+	psHeader->ui64Timestamp = ui64Timestamp;
+	psHeader->ui32Sig = HWPERF_PACKET_V2B_SIG;
+	psHeader->eTypeId = RGX_HWPERF_MAKE_TYPEID(RGX_HWPERF_STREAM_ID1_HOST,
+	                                           eEvType, 0, 0, 0);
+	psHeader->ui32Size = ui32Size;
+}
+
+static inline void _SetupHostEnqPacketData(IMG_UINT8 *pui8Dest,
+                                           RGX_HWPERF_KICK_TYPE eEnqType,
+                                           IMG_UINT32 ui32Pid,
+                                           IMG_UINT32 ui32FWDMContext,
+                                           IMG_UINT32 ui32ExtJobRef,
+                                           IMG_UINT32 ui32IntJobRef,
+                                           PVRSRV_FENCE hCheckFence,
+                                           PVRSRV_FENCE hUpdateFence,
+                                           PVRSRV_TIMELINE hUpdateTimeline,
+                                           IMG_UINT64 ui64CheckFenceUID,
+                                           IMG_UINT64 ui64UpdateFenceUID,
+                                           IMG_UINT64 ui64DeadlineInus,
+                                           IMG_UINT32 ui32CycleEstimate)
+{
+	RGX_HWPERF_HOST_ENQ_DATA *psData = (RGX_HWPERF_HOST_ENQ_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->ui32EnqType = eEnqType;
+	psData->ui32PID = ui32Pid;
+	psData->ui32ExtJobRef = ui32ExtJobRef;
+	psData->ui32IntJobRef = ui32IntJobRef;
+	psData->ui32DMContext = ui32FWDMContext;
+	psData->hCheckFence = hCheckFence;
+	psData->hUpdateFence = hUpdateFence;
+	psData->hUpdateTimeline = hUpdateTimeline;
+	psData->ui64CheckFence_UID = ui64CheckFenceUID;
+	psData->ui64UpdateFence_UID = ui64UpdateFenceUID;
+	psData->ui64DeadlineInus = ui64DeadlineInus;
+	psData->ui32CycleEstimate = ui32CycleEstimate;
+}
+
+void RGXHWPerfHostPostRaw(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+						  RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+						  IMG_BYTE *pbPayload,
+						  IMG_UINT32 ui32PayloadSize)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32PktSize;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	PVR_ASSERT(ui32PayloadSize <= RGX_HWPERF_MAX_PAYLOAD_SIZE);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32PktSize = RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32PayloadSize);
+	pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32PktSize);
+
+	if (pui8Dest == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, eEvType, ui32PktSize, ui32Ordinal, ui64Timestamp);
+	OSDeviceMemCopy((IMG_UINT8*)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR)), pbPayload, ui32PayloadSize);
+	_CommitHWPerfStream(psRgxDevInfo, ui32PktSize);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+void RGXHWPerfHostPostEnqEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_KICK_TYPE eEnqType,
+                               IMG_UINT32 ui32Pid,
+                               IMG_UINT32 ui32FWDMContext,
+                               IMG_UINT32 ui32ExtJobRef,
+                               IMG_UINT32 ui32IntJobRef,
+                               PVRSRV_FENCE hCheckFence,
+                               PVRSRV_FENCE hUpdateFence,
+                               PVRSRV_TIMELINE hUpdateTimeline,
+                               IMG_UINT64 ui64CheckFenceUID,
+                               IMG_UINT64 ui64UpdateFenceUID,
+                               IMG_UINT64 ui64DeadlineInus,
+                               IMG_UINT32 ui32CycleEstimate )
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_ENQ_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_ENQ, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostEnqPacketData(pui8Dest,
+	                        eEnqType,
+	                        ui32Pid,
+	                        ui32FWDMContext,
+	                        ui32ExtJobRef,
+	                        ui32IntJobRef,
+	                        hCheckFence,
+	                        hUpdateFence,
+	                        hUpdateTimeline,
+	                        ui64CheckFenceUID,
+	                        ui64UpdateFenceUID,
+	                        ui64DeadlineInus,
+	                        ui32CycleEstimate);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _CalculateHostUfoPacketSize(RGX_HWPERF_UFO_EV eUfoType)
+{
+	IMG_UINT32 ui32Size =
+			(IMG_UINT32) offsetof(RGX_HWPERF_HOST_UFO_DATA, aui32StreamData);
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData;
+
+	switch (eUfoType)
+	{
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			ui32Size += sizeof(puData->sCheckSuccess);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			ui32Size += sizeof(puData->sCheckFail);
+			break;
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			ui32Size += sizeof(puData->sUpdate);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostUfoEvent: Invalid UFO"
+					" event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostUfoPacketData(IMG_UINT8 *pui8Dest,
+                                           RGX_HWPERF_UFO_EV eUfoType,
+                                           RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData)
+{
+	RGX_HWPERF_HOST_UFO_DATA *psData = (RGX_HWPERF_HOST_UFO_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData = (RGX_HWPERF_UFO_DATA_ELEMENT *)
+					 psData->aui32StreamData;
+
+	psData->eEvType = eUfoType;
+	/* HWPerfHost always emits 1 UFO at a time, since each UFO has 1-to-1 mapping
+	 * with an underlying DevNode, and each DevNode has a dedicated HWPerf buffer */
+	psData->ui32StreamInfo = RGX_HWPERF_MAKE_UFOPKTINFO(1,
+	                                                    offsetof(RGX_HWPERF_HOST_UFO_DATA, aui32StreamData));
+
+	switch (eUfoType)
+	{
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			puData->sCheckSuccess.ui32FWAddr =
+					psUFOData->sCheckSuccess.ui32FWAddr;
+			puData->sCheckSuccess.ui32Value =
+					psUFOData->sCheckSuccess.ui32Value;
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			puData->sCheckFail.ui32FWAddr =
+					psUFOData->sCheckFail.ui32FWAddr;
+			puData->sCheckFail.ui32Value =
+					psUFOData->sCheckFail.ui32Value;
+			puData->sCheckFail.ui32Required =
+					psUFOData->sCheckFail.ui32Required;
+			break;
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			puData->sUpdate.ui32FWAddr =
+					psUFOData->sUpdate.ui32FWAddr;
+			puData->sUpdate.ui32OldValue =
+					psUFOData->sUpdate.ui32OldValue;
+			puData->sUpdate.ui32NewValue =
+					psUFOData->sUpdate.ui32NewValue;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostUfoEvent: Invalid UFO"
+					" event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+void RGXHWPerfHostPostUfoEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_UFO_EV eUfoType,
+                               RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData,
+                               const IMG_BOOL bSleepAllowed)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = _CalculateHostUfoPacketSize(eUfoType);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_BOOL   *pbPacketWritten = NULL;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              &pui8Dest, bSleepAllowed);
+
+	if (bSleepAllowed)
+	{
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+		{
+			goto cleanup;
+		}
+	}
+	else
+	{
+		if (pui8Dest == NULL)
+		{
+			// Give-up if we couldn't get a place in deferred events buffer
+			goto cleanup;
+		}
+		pbPacketWritten = GET_DE_EVENT_WRITE_STATUS(pui8Dest);
+		pui8Dest = GET_DE_EVENT_DATA(pui8Dest);
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_UFO, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostUfoPacketData(pui8Dest, eUfoType, psUFOData);
+
+	if (bSleepAllowed)
+	{
+		_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+	}
+	else
+	{
+		*pbPacketWritten = IMG_TRUE;
+		OSScheduleMISR(psRgxDevInfo->pvHostHWPerfMISR);
+	}
+
+cleanup:
+	if (bSleepAllowed)
+	{
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+	}
+}
+
+#define UNKNOWN_SYNC_NAME "UnknownSync"
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH==PVRSRV_SYNC_NAME_LENGTH, "Sync class name max does not match Fence Sync name max");
+
+static inline IMG_UINT32 _FixNameAndCalculateHostAllocPacketSize(
+		RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+		const IMG_CHAR **ppsName,
+		IMG_UINT32 *ui32NameSize)
+{
+	RGX_HWPERF_HOST_ALLOC_DATA *psData;
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_ALLOC_DATA, uAllocDetail);
+
+	if (*ppsName != NULL && *ui32NameSize > 0)
+	{
+		/* if string longer than maximum cut it (leave space for '\0') */
+		if ((*ui32NameSize +1U) >= PVRSRV_SYNC_NAME_LENGTH)
+			*ui32NameSize = PVRSRV_SYNC_NAME_LENGTH;
+		else
+			*ui32NameSize += 1U;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfHostPostAllocEvent: Invalid"
+				" resource name given."));
+		*ppsName = UNKNOWN_SYNC_NAME;
+		*ui32NameSize = sizeof(UNKNOWN_SYNC_NAME);
+	}
+
+	switch (eAllocType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			ui32Size += sizeof(psData->uAllocDetail.sSyncAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			ui32Size += sizeof(psData->uAllocDetail.sFenceAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW:
+			ui32Size += sizeof(psData->uAllocDetail.sSWFenceAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			ui32Size += sizeof(psData->uAllocDetail.sSyncCheckPointAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostAllocEvent: Invalid alloc event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostAllocPacketData(IMG_UINT8 *pui8Dest,
+                                             RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                             RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail,
+                                             const IMG_CHAR *psName,
+                                             IMG_UINT32 ui32NameSize)
+{
+	RGX_HWPERF_HOST_ALLOC_DATA *psData = (RGX_HWPERF_HOST_ALLOC_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	IMG_CHAR *acName = NULL;
+
+	psData->ui32AllocType = eAllocType;
+
+	switch (eAllocType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			psData->uAllocDetail.sSyncAlloc.ui32FWAddr = puAllocDetail->sSyncAlloc.ui32FWAddr;
+			acName = psData->uAllocDetail.sSyncAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uAllocDetail.sFenceAlloc.uiPID = puAllocDetail->sFenceAlloc.uiPID;
+			psData->uAllocDetail.sFenceAlloc.hFence = puAllocDetail->sFenceAlloc.hFence;
+			psData->uAllocDetail.sFenceAlloc.ui32CheckPt_FWAddr = puAllocDetail->sFenceAlloc.ui32CheckPt_FWAddr;
+			acName = psData->uAllocDetail.sFenceAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW:
+			psData->uAllocDetail.sSWFenceAlloc.uiPID = puAllocDetail->sSWFenceAlloc.uiPID;
+			psData->uAllocDetail.sSWFenceAlloc.hSWFence = puAllocDetail->sSWFenceAlloc.hSWFence;
+			psData->uAllocDetail.sSWFenceAlloc.hSWTimeline = puAllocDetail->sSWFenceAlloc.hSWTimeline;
+			psData->uAllocDetail.sSWFenceAlloc.ui64SyncPtIndex = puAllocDetail->sSWFenceAlloc.ui64SyncPtIndex;
+			acName = psData->uAllocDetail.sSWFenceAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			psData->uAllocDetail.sSyncCheckPointAlloc.ui32CheckPt_FWAddr = puAllocDetail->sSyncCheckPointAlloc.ui32CheckPt_FWAddr;
+			psData->uAllocDetail.sSyncCheckPointAlloc.hTimeline = puAllocDetail->sSyncCheckPointAlloc.hTimeline;
+			psData->uAllocDetail.sSyncCheckPointAlloc.uiPID = puAllocDetail->sSyncCheckPointAlloc.uiPID;
+			psData->uAllocDetail.sSyncCheckPointAlloc.hFence = puAllocDetail->sSyncCheckPointAlloc.hFence;
+			acName = psData->uAllocDetail.sSyncCheckPointAlloc.acName;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostAllocEvent: Invalid alloc event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+
+
+	if (acName != NULL)
+	{
+		if (ui32NameSize)
+		{
+			OSStringSafeCopy(acName, psName, ui32NameSize);
+		}
+		else
+		{
+			/* In case no name was given make sure we don't access random
+			 * memory */
+			acName[0] = '\0';
+		}
+	}
+}
+
+void RGXHWPerfHostPostAllocEvent(PVRSRV_RGXDEV_INFO* psRgxDevInfo,
+                                 RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                 const IMG_CHAR *psName,
+                                 IMG_UINT32 ui32NameSize,
+                                 RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT32 ui32Size = _FixNameAndCalculateHostAllocPacketSize(eAllocType,
+	                                                              &psName,
+	                                                              &ui32NameSize);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_ALLOC, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+
+	_SetupHostAllocPacketData(pui8Dest,
+	                          eAllocType,
+	                          puAllocDetail,
+	                          psName,
+	                          ui32NameSize);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostFreePacketData(IMG_UINT8 *pui8Dest,
+                                            RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                            IMG_UINT64 ui64UID,
+                                            IMG_UINT32 ui32FWAddr)
+{
+	RGX_HWPERF_HOST_FREE_DATA *psData = (RGX_HWPERF_HOST_FREE_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->ui32FreeType = eFreeType;
+
+	switch (eFreeType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			psData->uFreeDetail.sSyncFree.ui32FWAddr = ui32FWAddr;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uFreeDetail.sFenceDestroy.ui64Fence_UID = ui64UID;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			psData->uFreeDetail.sSyncCheckPointFree.ui32CheckPt_FWAddr = ui32FWAddr;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostFreeEvent: Invalid free event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+}
+
+void RGXHWPerfHostPostFreeEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                IMG_UINT64 ui64UID,
+                                IMG_UINT32 ui32PID,
+                                IMG_UINT32 ui32FWAddr)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_FREE_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	PVR_UNREFERENCED_PARAMETER(ui32PID);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_FREE, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostFreePacketData(pui8Dest,
+	                         eFreeType,
+	                         ui64UID,
+	                         ui32FWAddr);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _FixNameAndCalculateHostModifyPacketSize(
+		RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+		const IMG_CHAR **ppsName,
+		IMG_UINT32 *ui32NameSize)
+{
+	RGX_HWPERF_HOST_MODIFY_DATA *psData;
+	RGX_HWPERF_HOST_MODIFY_DETAIL *puData;
+	IMG_UINT32 ui32Size = sizeof(psData->ui32ModifyType);
+
+	if (*ppsName != NULL && *ui32NameSize > 0)
+	{
+		/* first strip the terminator */
+		if ((*ppsName)[*ui32NameSize - 1] == '\0')
+			*ui32NameSize -= 1;
+		/* if string longer than maximum cut it (leave space for '\0') */
+		if (*ui32NameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			*ui32NameSize = PVRSRV_SYNC_NAME_LENGTH - 1;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfHostPostModifyEvent: Invalid"
+				" resource name given."));
+		*ppsName = UNKNOWN_SYNC_NAME;
+		*ui32NameSize = sizeof(UNKNOWN_SYNC_NAME) - 1;
+	}
+
+	switch (eModifyType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			ui32Size += sizeof(puData->sFenceMerge) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize + 1; /* +1 for '\0' */
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostModifyEvent: Invalid modify event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostModifyPacketData(IMG_UINT8 *pui8Dest,
+                                              RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                              IMG_UINT64 ui64NewUID,
+                                              IMG_UINT64 ui64UID1,
+                                              IMG_UINT64 ui64UID2,
+                                              const IMG_CHAR *psName,
+                                              IMG_UINT32 ui32NameSize)
+{
+	RGX_HWPERF_HOST_MODIFY_DATA *psData = (RGX_HWPERF_HOST_MODIFY_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	IMG_CHAR *acName = NULL;
+
+	psData->ui32ModifyType = eModifyType;
+
+	switch (eModifyType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uModifyDetail.sFenceMerge.ui64NewFence_UID = ui64NewUID;
+			psData->uModifyDetail.sFenceMerge.ui64InFence1_UID = ui64UID1;
+			psData->uModifyDetail.sFenceMerge.ui64InFence2_UID = ui64UID2;
+			acName = psData->uModifyDetail.sFenceMerge.acName;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostModifyEvent: Invalid modify event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+
+	if (acName != NULL)
+	{
+		if (ui32NameSize)
+		{
+			OSStringSafeCopy(acName, psName, ui32NameSize);
+		}
+		else
+		{
+			/* In case no name was given make sure we don't access random
+			 * memory */
+			acName[0] = '\0';
+		}
+	}
+}
+
+void RGXHWPerfHostPostModifyEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                  RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                  IMG_UINT64 ui64NewUID,
+                                  IMG_UINT64 ui64UID1,
+                                  IMG_UINT64 ui64UID2,
+                                  const IMG_CHAR *psName,
+                                  IMG_UINT32 ui32NameSize)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT32 ui32Size = _FixNameAndCalculateHostModifyPacketSize(eModifyType,
+	                                                               &psName,
+	                                                               &ui32NameSize);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_MODIFY, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostModifyPacketData(pui8Dest,
+	                           eModifyType,
+	                           ui64NewUID,
+	                           ui64UID1,
+	                           ui64UID2,
+	                           psName,
+	                           ui32NameSize);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostClkSyncPacketData(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT8 *pui8Dest)
+{
+	RGX_HWPERF_HOST_CLK_SYNC_DATA *psData = (RGX_HWPERF_HOST_CLK_SYNC_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psRgxDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32CurrIdx;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32CurrIdx = RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount);
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32CurrIdx], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32CurrIdx];
+
+	psData->ui64CRTimestamp = psTimeCorr->ui64CRTimeStamp;
+	psData->ui64OSSecondaryTimeStamp = psTimeCorr->ui64OSTimeStamp;
+	psData->ui64OSMonoTimestamp = psTimeCorr->ui64OSMonoTimeStamp;
+	psData->ui32ClockSpeed = psTimeCorr->ui32CoreClockSpeed;
+}
+
+void RGXHWPerfHostPostClkSyncEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size =
+			RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_CLK_SYNC_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	/* if the buffer for time correlation data is not yet available (possibly
+	 * device not initialised yet) skip this event */
+	if (psRgxDevInfo->psRGXFWIfGpuUtilFW == NULL)
+	{
+		return;
+	}
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_CLK_SYNC, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostClkSyncPacketData(psRgxDevInfo, pui8Dest);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostDeviceInfoPacketData(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_DEV_INFO_EV eEvType,
+                                                  RGX_HWPERF_HOST_DEV_INFO_DETAIL *puPacketData,
+                                                  IMG_UINT8 *pui8Dest)
+{
+	RGX_HWPERF_HOST_DEV_INFO_DATA *psData = (RGX_HWPERF_HOST_DEV_INFO_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->eEvType = eEvType;
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_DEV_INFO_EV_HEALTH:
+			if (puPacketData != NULL)
+			{
+				psData->uDevInfoDetail.sDeviceStatus.eDeviceHealthStatus =
+					puPacketData->sDeviceStatus.eDeviceHealthStatus;
+				psData->uDevInfoDetail.sDeviceStatus.eDeviceHealthReason =
+					puPacketData->sDeviceStatus.eDeviceHealthReason;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: puPacketData is invalid."));
+			}
+			break;
+		case RGX_HWPERF_DEV_INFO_EV_FEATURES:
+			{
+				PVRSRV_ERROR eError;
+				eError = RGXServerFeatureFlagsToHWPerfFlags(psRgxDevInfo,
+				                                            &psData->uDevInfoDetail.sBVNC);
+				PVR_LOG_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags");
+				psData->uDevInfoDetail.sBVNC.ui32BvncKmFeatureFlags |=
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+					RGX_HWPERF_FEATURE_ROGUE_FLAG;
+#elif defined(RGX_FEATURE_HWPERF_VOLCANIC)
+					RGX_HWPERF_FEATURE_VOLCANIC_FLAG;
+#else
+					0x0;
+#endif
+			}
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+static inline IMG_UINT32 _CalculateHostDeviceInfoPacketSize(RGX_HWPERF_DEV_INFO_EV eEvType)
+{
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_DEV_INFO_DATA, uDevInfoDetail);
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_DEV_INFO_EV_HEALTH:
+			ui32Size += sizeof(((RGX_HWPERF_HOST_DEV_INFO_DATA*)0)->uDevInfoDetail.sDeviceStatus);
+			break;
+		case RGX_HWPERF_DEV_INFO_EV_FEATURES:
+			ui32Size += sizeof(((RGX_HWPERF_HOST_DEV_INFO_DATA*)0)->uDevInfoDetail.sBVNC);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+void RGXHWPerfHostPostDeviceInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 RGX_HWPERF_DEV_INFO_EV eEvType,
+                                 RGX_HWPERF_HOST_DEV_INFO_DETAIL *puData)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Size;
+
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->hHWPerfHostStream != (IMG_HANDLE) NULL)
+	{
+		_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+		ui32Size = _CalculateHostDeviceInfoPacketSize(eEvType);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) != NULL)
+		{
+			_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_DEV_INFO, ui32Size, ui32Ordinal, ui64Timestamp);
+			_SetupHostDeviceInfoPacketData(psRgxDevInfo, eEvType, puData, pui8Dest);
+			_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+		}
+
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+}
+
+static inline void _SetupHostInfoPacketData(RGX_HWPERF_INFO_EV eEvType,
+												  IMG_UINT64 ui64TotalMemoryUsage,
+												  IMG_UINT32 ui32LivePids,
+												  PVRSRV_PER_PROCESS_MEM_USAGE *psPerProcessMemUsage,
+												  IMG_UINT8 *pui8Dest)
+{
+	IMG_INT i;
+	RGX_HWPERF_HOST_INFO_DATA *psData = (RGX_HWPERF_HOST_INFO_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->eEvType = eEvType;
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_INFO_EV_MEM64_USAGE:
+			psData->uInfoDetail.sMemUsageStats.ui64TotalMemoryUsage = ui64TotalMemoryUsage;
+
+			if (psPerProcessMemUsage)
+			{
+				for (i = 0; i < ui32LivePids; ++i)
+				{
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui32Pid = psPerProcessMemUsage[i].ui32Pid;
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui64KernelMemUsage = psPerProcessMemUsage[i].ui64KernelMemUsage;
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui64GraphicsMemUsage = psPerProcessMemUsage[i].ui64GraphicsMemUsage;
+				}
+			}
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+static inline IMG_UINT32 _CalculateHostInfoPacketSize(RGX_HWPERF_INFO_EV eEvType,
+															IMG_UINT64 *pui64TotalMemoryUsage,
+															IMG_UINT32 *pui32LivePids,
+															PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsage)
+{
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_INFO_DATA, uInfoDetail);
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_INFO_EV_MEM64_USAGE:
+#if !defined(__QNXNTO__)
+			if (PVRSRVGetProcessMemUsage(pui64TotalMemoryUsage, pui32LivePids, ppsPerProcessMemUsage) == PVRSRV_OK)
+			{
+				ui32Size += offsetof(RGX_HWPERF_HOST_INFO_DETAIL, sMemUsageStats.sPerProcessUsage)
+					+ ((*pui32LivePids) * sizeof(struct _RGX_HWPERF_HOST_INFO_PER_PROC_USAGE_));
+			}
+#else
+			PVR_DPF((PVR_DBG_ERROR, "This functionality is not yet implemented for this platform"));
+#endif
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+void RGXHWPerfHostPostInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_INFO_EV eEvType)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT64 ui64TotalMemoryUsage = 0;
+	PVRSRV_PER_PROCESS_MEM_USAGE *psPerProcessMemUsage = NULL;
+	IMG_UINT32 ui32LivePids = 0;
+
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->hHWPerfHostStream != (IMG_HANDLE) NULL)
+	{
+		_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+		ui32Size = _CalculateHostInfoPacketSize(eEvType, &ui64TotalMemoryUsage, &ui32LivePids, &psPerProcessMemUsage);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) != NULL)
+		{
+			_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_INFO, ui32Size, ui32Ordinal, ui64Timestamp);
+			_SetupHostInfoPacketData(eEvType, ui64TotalMemoryUsage, ui32LivePids, psPerProcessMemUsage, pui8Dest);
+			_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+		}
+
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+
+		if (psPerProcessMemUsage)
+			OSFreeMemNoStats(psPerProcessMemUsage); // psPerProcessMemUsage was allocated with OSAllocZMemNoStats
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+}
+
+static inline IMG_UINT32
+_CalculateHostFenceWaitPacketSize(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eWaitType)
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *psSizeCalculator;
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA, uDetail);
+
+	switch (eWaitType)
+	{
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN:
+			ui32Size += sizeof(psSizeCalculator->uDetail.sBegin);
+			break;
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END:
+			ui32Size += sizeof(psSizeCalculator->uDetail.sEnd);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid wait event type (%u)", __func__,
+			         eWaitType));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void
+_SetupHostFenceWaitPacketData(IMG_UINT8 *pui8Dest,
+                              RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eWaitType,
+                              IMG_PID uiPID,
+                              PVRSRV_FENCE hFence,
+                              IMG_UINT32 ui32Data)
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *psData = (RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *)
+	                IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->eType = eWaitType;
+	psData->uiPID = uiPID;
+	psData->hFence = hFence;
+
+	switch (eWaitType)
+	{
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN:
+			psData->uDetail.sBegin.ui32TimeoutInMs = ui32Data;
+			break;
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END:
+			psData->uDetail.sEnd.eResult =
+			    (RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT) ui32Data;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid fence-wait event type", __func__));
+			PVR_ASSERT(IMG_FALSE);
+	}
+}
+
+void RGXHWPerfHostPostFenceWait(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType,
+								IMG_PID uiPID,
+								PVRSRV_FENCE hFence,
+								IMG_UINT32 ui32Data)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32Size = _CalculateHostFenceWaitPacketSize(eType);
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_SYNC_FENCE_WAIT,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+	_SetupHostFenceWaitPacketData(pui8Dest, eType, uiPID, hFence, ui32Data);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _CalculateHostSWTimelineAdvPacketSize(void)
+{
+	IMG_UINT32 ui32Size = sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA);
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void
+_SetupHostSWTimelineAdvPacketData(IMG_UINT8 *pui8Dest,
+                                  IMG_PID uiPID,
+								  PVRSRV_TIMELINE hSWTimeline,
+								  IMG_UINT64 ui64SyncPtIndex)
+
+{
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA *psData = (RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA *)
+	                IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->uiPID = uiPID;
+	psData->hTimeline = hSWTimeline;
+	psData->ui64SyncPtIndex = ui64SyncPtIndex;
+}
+
+void RGXHWPerfHostPostSWTimelineAdv(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                    IMG_PID uiPID,
+                                    PVRSRV_TIMELINE hSWTimeline,
+                                    IMG_UINT64 ui64SyncPtIndex)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32Size = _CalculateHostSWTimelineAdvPacketSize();
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+	_SetupHostSWTimelineAdvPacketData(pui8Dest, uiPID, hSWTimeline, ui64SyncPtIndex);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+
+}
+
+void RGXHWPerfHostPostClientInfoProcName(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_PID uiPID,
+									     const IMG_CHAR *psName)
+{
+	RGX_HWPERF_HOST_CLIENT_INFO_DATA* psPkt;
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32NameLen;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32NameLen = OSStringLength(psName) + 1U;
+	ui32Size = RGX_HWPERF_MAKE_SIZE_VARIABLE(RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE
+		+ RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen));
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_CLIENT_INFO,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+
+	psPkt = (RGX_HWPERF_HOST_CLIENT_INFO_DATA*)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psPkt->eType = RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME;
+	psPkt->uDetail.sProcName.ui32Count = 1U;
+	psPkt->uDetail.sProcName.asProcNames[0].uiClientPID = uiPID;
+	psPkt->uDetail.sProcName.asProcNames[0].ui32Length = ui32NameLen;
+	(void)OSCachedMemCopy(psPkt->uDetail.sProcName.asProcNames[0].acName, psName, ui32NameLen);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGX_KM_HWPERF_DEVDATA *psDevData;
+	RGX_HWPERF_DEVICE *psNewHWPerfDevice;
+	RGX_HWPERF_CONNECTION* psHWPerfConnection;
+	IMG_BOOL bFWActive = IMG_FALSE;
+
+	/* avoid uninitialised data */
+	PVR_ASSERT(*ppsHWPerfConnection == NULL);
+	PVR_ASSERT(psPVRSRVData);
+
+	/* Allocate connection object */
+	psHWPerfConnection = OSAllocZMem(sizeof(*psHWPerfConnection));
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	/* early save the return pointer to aid clean-up if failure occurs */
+	*ppsHWPerfConnection = psHWPerfConnection;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	while (psDeviceNode)
+	{
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_NOT_IMPLEMENTED;
+		}
+
+		if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: HWPerf: Device not currently active. ID:%u",
+					 __func__,
+					 psDeviceNode->sDevId.i32KernelDeviceID));
+			psDeviceNode = psDeviceNode->psNext;
+			continue;
+		}
+		/* Create a list node to be attached to connection object's list */
+		psNewHWPerfDevice = OSAllocMem(sizeof(*psNewHWPerfDevice));
+		if (!psNewHWPerfDevice)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		/* Insert node at head of the list */
+		psNewHWPerfDevice->psNext = psHWPerfConnection->psHWPerfDevList;
+		psHWPerfConnection->psHWPerfDevList = psNewHWPerfDevice;
+
+		/* create a device data object for kernel server */
+		psDevData = OSAllocZMem(sizeof(*psDevData));
+		psNewHWPerfDevice->hDevData = (IMG_HANDLE)psDevData;
+		if (!psDevData)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		if (OSSNPrintf(psNewHWPerfDevice->pszName, sizeof(psNewHWPerfDevice->pszName),
+			           "hwperf_device_%d", psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf device name for device %d",
+					__func__,
+					psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		psDevData->psRgxDevNode = psDeviceNode;
+		psDevData->psRgxDevInfo = psDeviceNode->pvDevice;
+
+		psDeviceNode = psDeviceNode->psNext;
+
+		/* At least one device is active */
+		bFWActive = IMG_TRUE;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (!bFWActive)
+	{
+		return PVRSRV_ERROR_NOT_READY;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION *psHWPerfConnection)
+{
+	RGX_KM_HWPERF_DEVDATA *psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo;
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszHWPerfFwStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 5];
+	IMG_CHAR pszHWPerfHostStreamName[sizeof(PVRSRV_TL_HWPERF_HOST_SERVER_STREAM) + 5];
+	IMG_UINT32 ui32BufSize;
+
+	/* Disable producer callback by default for the Kernel API. */
+	IMG_UINT32 ui32StreamFlags = PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING |
+			PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK;
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+		psRgxDevInfo = psDevData->psRgxDevInfo;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* In the case where the AppHint has not been set we need to
+		 * initialise the HWPerf resources here. Allocated on-demand
+		 * to reduce RAM foot print on systems not needing HWPerf.
+		 */
+		OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+		if (RGXHWPerfIsInitRequired(psRgxDevInfo, RGX_HWPERF_L2_STREAM_HWPERF))
+		{
+			eError = RGXHWPerfInitOnDemandL1Buffer(psRgxDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfFW resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hHWPerfLock);
+				return eError;
+			}
+
+			/* if this fails it also cleans up L1 buffer */
+			eError = RGXHWPerfInitOnDemandL2Stream(psRgxDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfFW resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hHWPerfLock);
+				return eError;
+			}
+		}
+		OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+		if (psRgxDevInfo->hHWPerfHostStream == NULL)
+		{
+			eError = RGXHWPerfHostInitOnDemandResources(psRgxDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfHost resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+				return eError;
+			}
+		}
+		OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+		/* form the HWPerf stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfFwStreamName, sizeof(pszHWPerfFwStreamName), "%s%d",
+		               PVRSRV_TL_HWPERF_RGX_FW_STREAM,
+		               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+		/* Open the RGX TL stream for reading in this session */
+		eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+		                            pszHWPerfFwStreamName,
+		                            ui32StreamFlags,
+		                            &psDevData->hSD[RGX_HWPERF_STREAM_ID0_FW]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientOpenStream(RGX_HWPerf)");
+
+		/* form the HWPerf host stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfHostStreamName, sizeof(pszHWPerfHostStreamName), "%s%d",
+		               PVRSRV_TL_HWPERF_HOST_SERVER_STREAM,
+		               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf host stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Open the host TL stream for reading in this session */
+		eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+		                            pszHWPerfHostStreamName,
+		                            PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING,
+		                            &psDevData->hSD[RGX_HWPERF_STREAM_ID1_HOST]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientOpenStream(Host_HWPerf)");
+
+		/* Allocate a large enough buffer for use during the entire session to
+		 * avoid the need to resize in the Acquire call as this might be in an ISR
+		 * Choose size that can contain at least one packet.
+		 */
+		/* Allocate buffer for FW Stream */
+		ui32BufSize = FW_STREAM_BUFFER_SIZE;
+		psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW] = OSAllocMem(ui32BufSize);
+		if (psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW] == NULL)
+		{
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		psDevData->pHwpBufEnd[RGX_HWPERF_STREAM_ID0_FW] = psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW]+ui32BufSize;
+
+		/* Allocate buffer for Host Stream */
+		ui32BufSize = HOST_STREAM_BUFFER_SIZE;
+		psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST] = OSAllocMem(ui32BufSize);
+		if (psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST] == NULL)
+		{
+			OSFreeMem(psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW]);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		psDevData->pHwpBufEnd[RGX_HWPERF_STREAM_ID1_HOST] = psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST]+ui32BufSize;
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXHWPerfLazyConnect(ppsHWPerfConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfLazyConnect", e0);
+
+	eError = RGXHWPerfOpen(*ppsHWPerfConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfOpen", e1);
+
+	return PVRSRV_OK;
+
+e1: /* HWPerfOpen might have opened some, and then failed */
+	RGXHWPerfClose(*ppsHWPerfConnection);
+e0: /* LazyConnect might have allocated some resources and then failed,
+	 * make sure they are cleaned up */
+	RGXHWPerfFreeConnection(ppsHWPerfConnection);
+	return eError;
+}
+
+/*
+    PVRSRVRGXControlHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXControlHWPerfBlocksKM(
+		CONNECTION_DATA             * psConnection,
+		PVRSRV_DEVICE_NODE          * psDeviceNode,
+		IMG_BOOL                      bEnable,
+		IMG_UINT32                    ui32ArrayLen,
+		IMG_UINT16                  * psBlockIDs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_ASSERT(psDeviceNode);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBlockIDs != NULL, "psBlockIDs");
+	PVR_LOG_RETURN_IF_INVALID_PARAM((ui32ArrayLen>0) && (ui32ArrayLen <= RGXFWIF_HWPERF_CTRL_BLKS_MAX), "ui32ArrayLen");
+
+	psDevice = psDeviceNode->pvDevice;
+
+	/* Fill in the command structure with the parameters needed
+	 */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCtrlBlks.bEnable = bEnable;
+	sKccbCmd.uCmdData.sHWPerfCtrlBlks.ui32NumBlocks = ui32ArrayLen;
+
+	OSDeviceMemCopy(sKccbCmd.uCmdData.sHWPerfCtrlBlks.aeBlockIDs, psBlockIDs, sizeof(IMG_UINT16) * ui32ArrayLen);
+
+
+	/* Ask the FW to carry out the HWPerf configuration command
+	 */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot");
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+
+	if (bEnable)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks have been ENABLED", ui32ArrayLen));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks have been DISABLED", ui32ArrayLen));
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/*
+	PVRSRVRGXCtrlHWPerfKM
+ */
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		RGX_HWPERF_STREAM_ID     eStreamId,
+		IMG_BOOL                 bToggle,
+		IMG_UINT64               ui64Mask)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_ASSERT(psDeviceNode);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_DPF_ENTERED;
+
+	if (eStreamId == RGX_HWPERF_STREAM_ID0_FW)
+	{
+		return RGXHWPerfCtrlFwBuffer(psDeviceNode, RGX_HWPERF_L2_STREAM_HWPERF,
+		                             bToggle, ui64Mask);
+	}
+	else if (eStreamId == RGX_HWPERF_STREAM_ID1_HOST)
+	{
+		return RGXHWPerfCtrlHostBuffer(psDeviceNode, bToggle, (IMG_UINT32) ui64Mask);
+	}
+	else if (eStreamId == RGX_HWPERF_STREAM_ID2_CLIENT)
+	{
+		IMG_UINT32 ui32Index = (IMG_UINT32) (ui64Mask >> 32);
+		IMG_UINT32 ui32Mask = (IMG_UINT32) ui64Mask;
+
+		return RGXHWPerfCtrlClientBuffer(bToggle, ui32Index, ui32Mask);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXCtrlHWPerfKM: Unknown stream id."));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfFW(
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		RGX_HWPERF_L2_STREAM_ID  eL2StreamId,
+		IMG_UINT64               ui64Mask,
+		HWPERF_FILTER_OPERATION  eMaskOp)
+{
+	IMG_UINT64 uiTmpFilter;
+
+	PVR_DPF_ENTERED;
+	PVR_ASSERT(psDeviceNode);
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	uiTmpFilter =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui64HWPerfFilter[eL2StreamId];
+
+	switch (eMaskOp)
+	{
+		case HWPERF_FILTER_OPERATION_SET:
+			uiTmpFilter = ui64Mask;
+			break;
+		case HWPERF_FILTER_OPERATION_BIT_CLR:
+			uiTmpFilter &= ~ui64Mask;
+			break;
+		case HWPERF_FILTER_OPERATION_BIT_OR:
+			uiTmpFilter |= ui64Mask;
+			break;
+	}
+
+	PVR_DPF_RETURN_RC(RGXHWPerfCtrlFwBuffer(psDeviceNode, eL2StreamId, IMG_FALSE,
+	                  uiTmpFilter));
+}
+
+/*
+	PVRSRVRGXGetHWPerfTimeStampKM
+ */
+PVRSRV_ERROR PVRSRVRGXGetHWPerfTimeStampKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		IMG_UINT64              *pui64TimeStamp)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	*pui64TimeStamp = RGXTimeCorrGetClockus64(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE* psHWPerfDev;
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXCtrlHWPerfKM(NULL, psDevData->psRgxDevNode, eStreamId, bToggle, ui64Mask);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR RGXHWPerfToggleCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs,
+		IMG_BOOL     bToggle,
+		const char* szFunctionString);
+
+IMG_INTERNAL PVRSRV_ERROR RGXHWPerfToggleCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs,
+		IMG_BOOL     bToggle,
+		const char* szFunctionString)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE*     psHWPerfDev;
+
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !aeBlockIDs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXControlHWPerfBlocksKM(NULL,
+		                                        psDevData->psRgxDevNode,
+		                                        bToggle,
+		                                        ui32NumBlocks,
+		                                        aeBlockIDs);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, szFunctionString);
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32            ui32NumBlocks,
+		IMG_UINT16*           aeBlockIDs)
+{
+	return RGXHWPerfToggleCounters(psHWPerfConnection,
+			                        ui32NumBlocks,
+			                        aeBlockIDs,
+			                        IMG_FALSE,
+			                        __func__);
+}
+
+
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32            ui32NumBlocks,
+		IMG_UINT16*           aeBlockIDs)
+{
+	return RGXHWPerfToggleCounters(psHWPerfConnection,
+			                        ui32NumBlocks,
+			                        aeBlockIDs,
+			                        IMG_TRUE,
+			                        __func__);
+}
+
+
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen)
+{
+	PVRSRV_ERROR			eError;
+	RGX_KM_HWPERF_DEVDATA*	psDevData = (RGX_KM_HWPERF_DEVDATA*)hDevData;
+	IMG_PBYTE				pDataDest;
+	IMG_UINT32			ui32TlPackets = 0;
+	IMG_PBYTE			pBufferEnd;
+	PVRSRVTL_PPACKETHDR psHDRptr;
+	PVRSRVTL_PACKETTYPE ui16TlType;
+
+	/* Reset the output arguments in case we discover an error */
+	*ppBuf = NULL;
+	*pui32BufLen = 0;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psDevData || eStreamId >= RGX_HWPERF_MAX_STREAM_ID)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psDevData->pTlBuf[eStreamId] == NULL)
+	{
+		/* Acquire some data to read from the HWPerf TL stream */
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		                             psDevData->hSD[eStreamId],
+		                             &psDevData->pTlBuf[eStreamId],
+		                             &psDevData->ui32AcqDataLen[eStreamId]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientAcquireData");
+
+		psDevData->pTlBufPos[eStreamId] = psDevData->pTlBuf[eStreamId];
+	}
+
+	/* TL indicates no data exists so return OK and zero. */
+	if ((psDevData->pTlBufPos[eStreamId] == NULL) || (psDevData->ui32AcqDataLen[eStreamId] == 0))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Process each TL packet in the data buffer we have acquired */
+	pBufferEnd = psDevData->pTlBuf[eStreamId]+psDevData->ui32AcqDataLen[eStreamId];
+	pDataDest = psDevData->pHwpBuf[eStreamId];
+	psHDRptr = GET_PACKET_HDR(psDevData->pTlBufPos[eStreamId]);
+	psDevData->pTlBufRead[eStreamId] = psDevData->pTlBufPos[eStreamId];
+	while (psHDRptr < (PVRSRVTL_PPACKETHDR)((void *)pBufferEnd))
+	{
+		ui16TlType = GET_PACKET_TYPE(psHDRptr);
+		if (ui16TlType == PVRSRVTL_PACKETTYPE_DATA)
+		{
+			IMG_UINT16 ui16DataLen = GET_PACKET_DATA_LEN(psHDRptr);
+			if (0 == ui16DataLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfAcquireEvents: ZERO Data in TL data packet: %p", psHDRptr));
+			}
+			else
+			{
+				/* Check next packet does not fill buffer */
+				if (pDataDest + ui16DataLen > psDevData->pHwpBufEnd[eStreamId])
+				{
+					break;
+				}
+
+				/* For valid data copy it into the client buffer and move
+				 * the write position on */
+				OSDeviceMemCopy(pDataDest, GET_PACKET_DATA_PTR(psHDRptr), ui16DataLen);
+				pDataDest += ui16DataLen;
+			}
+		}
+		else if (ui16TlType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfAcquireEvents: Indication that the transport buffer was full"));
+		}
+		else
+		{
+			/* else Ignore padding packet type and others */
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfAcquireEvents: Ignoring TL packet, type %d", ui16TlType ));
+		}
+
+		/* Update loop variable to the next packet and increment counts */
+		psHDRptr = GET_NEXT_PACKET_ADDR(psHDRptr);
+		/* Updated to keep track of the next packet to be read. */
+		psDevData->pTlBufRead[eStreamId] = (IMG_PBYTE) ((void *)psHDRptr);
+		ui32TlPackets++;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "RGXHWPerfAcquireEvents: TL Packets processed %03d", ui32TlPackets));
+
+	psDevData->bRelease[eStreamId] = IMG_FALSE;
+	if (psHDRptr >= (PVRSRVTL_PPACKETHDR)((void *)pBufferEnd))
+	{
+		psDevData->bRelease[eStreamId] = IMG_TRUE;
+	}
+
+	/* Update output arguments with client buffer details and true length */
+	*ppBuf = psDevData->pHwpBuf[eStreamId];
+	*pui32BufLen = pDataDest - psDevData->pHwpBuf[eStreamId];
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId)
+{
+	PVRSRV_ERROR			eError = PVRSRV_OK;
+	RGX_KM_HWPERF_DEVDATA*	psDevData = (RGX_KM_HWPERF_DEVDATA*)hDevData;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psDevData || eStreamId >= RGX_HWPERF_MAX_STREAM_ID)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psDevData->bRelease[eStreamId])
+	{
+		/* Inform the TL that we are done with reading the data. */
+		eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, psDevData->hSD[eStreamId]);
+		psDevData->ui32AcqDataLen[eStreamId] = 0;
+		psDevData->pTlBuf[eStreamId] = NULL;
+	}
+	else
+	{
+		psDevData->pTlBufPos[eStreamId] = psDevData->pTlBufRead[eStreamId];
+	}
+	return eError;
+}
+
+
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo =
+			hDevData ? ((RGX_KM_HWPERF_DEVDATA*) hDevData)->psRgxDevInfo : NULL;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psRgxDevInfo)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid pointer to the RGX device",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	/* No need to take hHWPerfLock here since we are only reading data
+	 * from always existing integers to return to debugfs which is an
+	 * atomic operation.
+	 */
+	switch (eStreamId) {
+		case RGX_HWPERF_STREAM_ID0_FW:
+			*ui64Filter = psRgxDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF];
+			break;
+		case RGX_HWPERF_STREAM_ID1_HOST:
+			*ui64Filter = psRgxDevInfo->ui32HWPerfHostFilter;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid stream ID",
+					__func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	RGX_HWPERF_DEVICE *psHWPerfDev, *psHWPerfNextDev;
+	RGX_HWPERF_CONNECTION *psHWPerfConnection = *ppsHWPerfConnection;
+
+	/* if connection object itself is NULL, nothing to free */
+	if (psHWPerfConnection == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	psHWPerfNextDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfNextDev)
+	{
+		psHWPerfDev = psHWPerfNextDev;
+		psHWPerfNextDev = psHWPerfNextDev->psNext;
+
+		/* Free the session memory */
+		if (psHWPerfDev->hDevData)
+			OSFreeMem(psHWPerfDev->hDevData);
+		OSFreeMem(psHWPerfDev);
+	}
+	OSFreeMem(psHWPerfConnection);
+	*ppsHWPerfConnection = NULL;
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection)
+{
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	IMG_UINT uiStreamId;
+	PVRSRV_ERROR eError;
+
+	/* Check session connection is not zero */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		for (uiStreamId = 0; uiStreamId < RGX_HWPERF_MAX_STREAM_ID; uiStreamId++)
+		{
+			/* If the TL buffer exists they have not called ReleaseData
+			 * before disconnecting so clean it up */
+			if (psDevData->pTlBuf[uiStreamId])
+			{
+				/* TLClientReleaseData call and null out the buffer fields
+				 * and length */
+				eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, psDevData->hSD[uiStreamId]);
+				psDevData->ui32AcqDataLen[uiStreamId] = 0;
+				psDevData->pTlBuf[uiStreamId] = NULL;
+				PVR_LOG_IF_ERROR(eError, "TLClientReleaseData");
+				/* Packets may be lost if release was not required */
+				if (!psDevData->bRelease[uiStreamId])
+				{
+					PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfClose: Events in buffer waiting to be read, remaining events may be lost."));
+				}
+			}
+
+			/* Close the TL stream, ignore the error if it occurs as we
+			 * are disconnecting */
+			if (psDevData->hSD[uiStreamId])
+			{
+				eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+				                             psDevData->hSD[uiStreamId]);
+				PVR_LOG_IF_ERROR(eError, "TLClientCloseStream");
+				psDevData->hSD[uiStreamId] = NULL;
+			}
+
+			/* Free the client buffer used in session */
+			if (psDevData->pHwpBuf[uiStreamId])
+			{
+				OSFreeMem(psDevData->pHwpBuf[uiStreamId]);
+				psDevData->pHwpBuf[uiStreamId] = NULL;
+			}
+		}
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXHWPerfClose(*ppsHWPerfConnection);
+	PVR_LOG_IF_ERROR(eError, "RGXHWPerfClose");
+
+	eError = RGXHWPerfFreeConnection(ppsHWPerfConnection);
+	PVR_LOG_IF_ERROR(eError, "RGXHWPerfFreeConnection");
+
+	return eError;
+}
+
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp)
+{
+	IMG_UINT64 ui64CRDeltaToOSDeltaKNs;
+	IMG_UINT64 ui64EventOSTimestamp, deltaRgxTimer, delta_ns;
+
+	if (!(ui64CRTimeStamp) || !(ui32ClkSpeed) || !(ui64CorrCRTimeStamp) || !(ui64CorrOSTimeStamp))
+	{
+		return 0;
+	}
+
+	ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(ui32ClkSpeed);
+
+	/* RGX CR timer ticks delta */
+	deltaRgxTimer = ui64CRTimeStamp - ui64CorrCRTimeStamp;
+	/* RGX time delta in nanoseconds */
+	delta_ns = RGXFWIF_GET_DELTA_OSTIME_NS(deltaRgxTimer, ui64CRDeltaToOSDeltaKNs);
+	/* Calculate OS time of HWPerf event */
+	ui64EventOSTimestamp = ui64CorrOSTimeStamp + delta_ns;
+
+	return ui64EventOSTimestamp;
+}
+
+#define CLIENT_STREAM_REFCOUNT_MIN 0
+#define CLIENT_STREAM_REFCOUNT_MAX IMG_INT32_MAX
+
+struct HWPERF_STREAM_DESC_TAG
+{
+    TL_STREAM_DESC *psSD;
+    PMR *psPMR;
+    POS_LOCK hLock;
+    ATOMIC_T iRefCount;
+    IMG_UINT32 uiOrdinal;
+};
+
+PVRSRV_ERROR PVRSRVRGXOpenHWPerfClientStreamKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               HWPERF_STREAM_DESC **ppsSD)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	HWPERF_STREAM_DESC *psHwpSD;
+	TL_STREAM_DESC *psTlSD;
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+	IMG_CHAR acStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+
+	OSSNPrintf(acStreamName, sizeof(acStreamName),
+	           PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC,
+	           psDeviceNode->sDevId.i32KernelDeviceID,
+	           psConnection->pid);
+
+	OSLockAcquire(psPVRSRVData->hClientStreamTableLock);
+
+	psHwpSD = (void *) HASH_Retrieve_Extended(psPVRSRVData->psClientStreamTable,
+	                                          (void *) acStreamName);
+	if (psHwpSD != NULL)
+	{
+		IMG_INT32 iOldRefCount = OSAtomicAddUnless(&psHwpSD->iRefCount, 1,
+		                                           CLIENT_STREAM_REFCOUNT_MAX);
+		if (iOldRefCount == CLIENT_STREAM_REFCOUNT_MAX)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("HWPerf stream reference count overflow",
+			                        eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW,
+			                        ErrUnlockAndReturnError);
+		}
+
+		*ppsSD = psHwpSD;
+
+		OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+		return PVRSRV_OK;
+	}
+
+	psHwpSD = OSAllocMem(sizeof(*psHwpSD));
+	PVR_LOG_GOTO_IF_NOMEM(psHwpSD, eError, ErrUnlockAndReturnError);
+
+	eError = OSLockCreate(&psHwpSD->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLServerOpenStreamKM", ErrFreeSD);
+
+	eError = TLServerOpenStreamKM(acStreamName, PVRSRV_STREAM_FLAG_OPEN_WO,
+	                              &psTlSD, &psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLServerOpenStreamKM", ErrFreeLock);
+
+	psHwpSD->psSD = psTlSD;
+	psHwpSD->psPMR = psPMR;
+	psHwpSD->uiOrdinal = IMG_UINT32_MAX;
+	OSAtomicWrite(&psHwpSD->iRefCount, CLIENT_STREAM_REFCOUNT_MIN + 1);
+
+	if (!HASH_Insert_Extended(psPVRSRVData->psClientStreamTable, (void *) acStreamName,
+	                          (uintptr_t) psHwpSD))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("HASH_Insert", eError, PVRSRV_ERROR_OUT_OF_MEMORY,
+		                        ErrCloseStream);
+	}
+
+	OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+	*ppsSD = psHwpSD;
+
+	return PVRSRV_OK;
+
+ErrCloseStream:
+	TLServerCloseStreamKM(psTlSD);
+ErrFreeLock:
+	OSLockDestroy(psHwpSD->hLock);
+ErrFreeSD:
+	OSFreeMem(psHwpSD);
+ErrUnlockAndReturnError:
+	OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+	return eError;
+}
+
+/* This function is called from the handle framework's handle destruction
+ * path which means there should not be any lookup outstanding, i.e. the handle
+ * should not be used at the given moment. */
+PVRSRV_ERROR PVRSRVRGXCloseHWPerfClientStreamKM(HWPERF_STREAM_DESC *psSD)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_INT iOldRefCount;
+	PVRSRV_ERROR eError;
+	void *psElement;
+
+	PVR_ASSERT(psSD != NULL);
+
+	iOldRefCount = OSAtomicSubtractUnless(&psSD->iRefCount, 1, CLIENT_STREAM_REFCOUNT_MIN);
+	if (iOldRefCount == CLIENT_STREAM_REFCOUNT_MIN)
+	{
+		/* If the old value is 0 return error. This means that something went
+		 * wrong and the reference count was already 0 and resources were
+		 * freed. */
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+	else if (iOldRefCount != (CLIENT_STREAM_REFCOUNT_MIN + 1))
+	{
+		/* If the old value is not 1 then return ok, the stream is still used
+		 * by someone. If it's 1, then progress with the resources destruction
+		 * since the current value is 0. */
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(iOldRefCount > 0);
+
+	OSLockAcquire(psPVRSRVData->hClientStreamTableLock);
+
+	psElement = (void *) HASH_Remove_Extended(psPVRSRVData->psClientStreamTable,
+	                                          psSD->psSD->psNode->psStream->szName);
+	PVR_ASSERT(psElement != NULL);
+
+	OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+	eError = TLServerCloseStreamKM(psSD->psSD);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLServerCloseStreamKM", ErrReturnError);
+
+	OSLockDestroy(psSD->hLock);
+
+	OSFreeMem(psSD);
+
+	return PVRSRV_OK;
+
+ErrReturnError:
+	return eError;
+}
+
+/* This function is called from the handle framework's lookup path which means
+ * there should not be a valid pointer to the looked up resource, i.e. no
+ * destructor is allowed to run on the resource. */
+PVRSRV_ERROR PVRSRVRGXWriteHWPerfClientEventKM(HWPERF_STREAM_DESC *psSD,
+                                               IMG_UINT32 uiSize,
+                                               IMG_BYTE *puiData)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader = (RGX_HWPERF_V2_PACKET_HDR *) ((void *)puiData);
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSD != NULL);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiSize > sizeof(*psHeader), "uiSize");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeader->ui32Size == uiSize, "uiSize");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeader->ui32Sig == HWPERF_PACKET_V2B_SIG, "ui32Sig");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeader->ui32Ordinal == 0, "ui32Ordinal");
+
+	OSLockAcquire(psSD->hLock);
+
+	psHeader->ui32Ordinal = ++psSD->uiOrdinal;
+
+	eError = TLServerWriteDataKM(psSD->psSD, uiSize, puiData);
+
+	OSLockRelease(psSD->hLock);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf_common.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.h
new file mode 100644
index 000000000000..6f1b932f3322
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.h
@@ -0,0 +1,655 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX HWPerf functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHWPERF_COMMON_H_
+#define RGXHWPERF_COMMON_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#include "device.h"
+#include "connection_server.h"
+#include "rgxdevice.h"
+#include "rgx_hwperf.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(PVR_ARCH_VOLCANIC)
+#include "rgx_hwperf_table.h"
+#endif
+#include "cache_ops.h"
+#include "rgxfwmemctx.h"
+
+/* HWPerf host buffer size constraints in KBs */
+#define HWPERF_HOST_TL_STREAM_SIZE_DEFAULT PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB
+#define HWPERF_HOST_TL_STREAM_SIZE_MIN     (32U)
+#define HWPERF_HOST_TL_STREAM_SIZE_MAX     (3072U)
+
+/* Operations on HWPerf filter. */
+typedef enum HWPERF_FILTER_OPERATION_TAG
+{
+	HWPERF_FILTER_OPERATION_SET,
+	HWPERF_FILTER_OPERATION_BIT_CLR,
+	HWPERF_FILTER_OPERATION_BIT_OR,
+} HWPERF_FILTER_OPERATION;
+
+/******************************************************************************
+ * RGX HW Performance decode Bvnc Features for HWPerf
+ *****************************************************************************/
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+		                                        RGX_HWPERF_BVNC    *psBVNC);
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(CONNECTION_DATA    *psConnection,
+                                                  PVRSRV_DEVICE_NODE *psDeviceNode,
+											      RGX_HWPERF_BVNC    *psBVNC);
+
+/******************************************************************************
+ * RGX HW Performance Data Transport Routines
+ *****************************************************************************/
+
+PVRSRV_ERROR RGXHWPerfDataStoreCB(PVRSRV_DEVICE_NODE* psDevInfo);
+
+PVRSRV_ERROR RGXHWPerfInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+void RGXHWPerfDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+
+PVRSRV_ERROR RGXHWPerfInitOnDemandL1Buffer(PVRSRV_RGXDEV_INFO* psRgxDevInfo);
+PVRSRV_ERROR RGXHWPerfInitOnDemandL2Stream(PVRSRV_RGXDEV_INFO* psRgxDevInfo,
+                                           RGX_HWPERF_L2_STREAM_ID eL2StreamId);
+void RGXHWPerfDeinitL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                             RGX_HWPERF_L2_STREAM_ID eL2StreamId);
+void RGXHWPerfInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+void RGXHWPerfClientInitAppHintCallbacks(void);
+
+static INLINE PVRSRV_ERROR RGXAcquireHWPerfCtlCPUAddr(PVRSRV_DEVICE_NODE *psDevNode,
+                                                      RGXFWIF_HWPERF_CTL **ppsHWPerfCtl)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+	                                  (void**)ppsHWPerfCtl);
+	RGXFwSharedMemCacheOpPtr(ppsHWPerfCtl, INVALIDATE);
+
+	return eError;
+}
+
+static INLINE void RGXReleaseHWPerfCtlCPUAddr(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevNode != NULL, "psDevNode is invalid");
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevInfo != NULL, "psDevInfo invalid");
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCountersKM(CONNECTION_DATA *psConnection,
+                                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                    const IMG_UINT32 ui32BlockID,
+                                                    RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocksKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               const IMG_UINT32 ui32ArrayLen,
+                                               IMG_UINT32 *pui32BlockCount,
+                                               IMG_UINT32 *pui32EnabledBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling API(s)
+ *****************************************************************************/
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfKM(
+	CONNECTION_DATA      * psConnection,
+	PVRSRV_DEVICE_NODE   * psDeviceNode,
+	 RGX_HWPERF_STREAM_ID  eStreamId,
+	IMG_BOOL               bToggle,
+	IMG_UINT64             ui64Mask);
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfFW(
+	PVRSRV_DEVICE_NODE      *psDeviceNode,
+	RGX_HWPERF_L2_STREAM_ID  eL2StreamId,
+	IMG_UINT64               ui64Mask,
+	HWPERF_FILTER_OPERATION  eMaskOp);
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfTimeStampKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		IMG_UINT64              *pui64TimeStamp);
+
+PVRSRV_ERROR PVRSRVRGXControlHWPerfBlocksKM(
+	CONNECTION_DATA       * psConnection,
+	PVRSRV_DEVICE_NODE    * psDeviceNode,
+	IMG_BOOL              bEnable,
+	IMG_UINT32            ui32ArrayLen,
+	IMG_UINT16            * psBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Host Stream API
+ *****************************************************************************/
+
+static inline RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS
+RGXHWPerfConvDeviceHealthStatus(PVRSRV_DEVICE_HEALTH_STATUS eDeviceHealthStatus)
+{
+	switch (eDeviceHealthStatus)
+	{
+		case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:         return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED;
+		case PVRSRV_DEVICE_HEALTH_STATUS_OK:                return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:    return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+		case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:              return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_DEAD;
+		case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:             return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_FAULT;
+		default:                                            return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED;
+	}
+}
+
+static inline RGX_HWPERF_HOST_DEVICE_HEALTH_REASON
+RGXHWPerfConvDeviceHealthReason(PVRSRV_DEVICE_HEALTH_REASON eDeviceHealthReason)
+{
+	switch (eDeviceHealthReason)
+	{
+		case PVRSRV_DEVICE_HEALTH_REASON_NONE:                  return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_NONE;
+		case PVRSRV_DEVICE_HEALTH_REASON_ASSERTED:              return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_ASSERTED;
+		case PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING:          return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_POLL_FAILING;
+		case PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS:              return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_TIMEOUTS;
+		case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT:         return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+		case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED:         return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+		case PVRSRV_DEVICE_HEALTH_REASON_IDLING:                return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_IDLING;
+		case PVRSRV_DEVICE_HEALTH_REASON_RESTARTING:            return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_RESTARTING;
+		case PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS:    return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+		case PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR:             return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_PCI_ERROR;
+		default:                                                return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_UNDEFINED;
+	}
+}
+
+PVRSRV_ERROR RGXHWPerfHostInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32BufSizeKB);
+PVRSRV_ERROR RGXHWPerfHostInitOnDemandResources(PVRSRV_RGXDEV_INFO* psRgxDevInfo);
+void RGXHWPerfHostDeInit(PVRSRV_RGXDEV_INFO	*psRgxDevInfo);
+
+IMG_UINT64 RGXHWPerfFwSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                     RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                     IMG_UINT64 uiFilter);
+
+void RGXHWPerfHostSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 IMG_UINT32 ui32Filter);
+
+void RGXHWPerfHostPostRaw(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+						  RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+						  IMG_BYTE *pbPayload,
+						  IMG_UINT32 ui32PayloadSize);
+
+void RGXHWPerfHostPostEnqEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_KICK_TYPE eEnqType,
+                               IMG_UINT32 ui32Pid,
+                               IMG_UINT32 ui32FWDMContext,
+                               IMG_UINT32 ui32ExtJobRef,
+                               IMG_UINT32 ui32IntJobRef,
+                               PVRSRV_FENCE hCheckFence,
+                               PVRSRV_FENCE hUpdateFence,
+                               PVRSRV_TIMELINE hUpdateTimeline,
+                               IMG_UINT64 ui64CheckFenceUID,
+                               IMG_UINT64 ui64UpdateFenceUID,
+                               IMG_UINT64 ui64DeadlineInus,
+                               IMG_UINT32 ui32CycleEstimate);
+
+void RGXHWPerfHostPostAllocEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                 const IMG_CHAR *psName,
+                                 IMG_UINT32 ui32NameSize,
+                                 RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail);
+
+void RGXHWPerfHostPostFreeEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                IMG_UINT64 ui64UID,
+                                IMG_UINT32 ui32PID,
+                                IMG_UINT32 ui32FWAddr);
+
+void RGXHWPerfHostPostModifyEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                  RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                  IMG_UINT64 ui64NewUID,
+                                  IMG_UINT64 ui64UID1,
+                                  IMG_UINT64 ui64UID2,
+                                  const IMG_CHAR *psName,
+                                  IMG_UINT32 ui32NameSize);
+
+void RGXHWPerfHostPostUfoEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_UFO_EV eUfoType,
+                               RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData,
+							   const IMG_BOOL bSleepAllowed);
+
+void RGXHWPerfHostPostClkSyncEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+
+void RGXHWPerfHostPostDeviceInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_DEV_INFO_EV eEvType,
+								 RGX_HWPERF_HOST_DEV_INFO_DETAIL *psData);
+
+void RGXHWPerfHostPostInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_INFO_EV eEvType);
+
+void RGXHWPerfHostPostFenceWait(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType,
+								IMG_PID uiPID,
+								PVRSRV_FENCE hFence,
+								IMG_UINT32 ui32Data);
+
+void RGXHWPerfHostPostSWTimelineAdv(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                    IMG_PID uiPID,
+									PVRSRV_TIMELINE hSWTimeline,
+									IMG_UINT64 ui64SyncPtIndex);
+
+void RGXHWPerfHostPostClientInfoProcName(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_PID uiPID,
+									     const IMG_CHAR *psName);
+
+IMG_BOOL RGXHWPerfHostIsEventEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo, RGX_HWPERF_HOST_EVENT_TYPE eEvent);
+
+#if defined(PVR_ARCH_VOLCANIC)
+IMG_INTERNAL /*static inline*/ IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel);
+
+/*!
+*******************************************************************************
+ @Function    RGXHWPerfMaxDefinedBlks
+
+ @Description Return the number of valid block-IDs for the given device node
+
+ @Input       (PVRSRV_RGXDEV_INFO *)   pvDevice    device-node to query
+
+ @Returns     (IMG_UINT32)             Number of block-IDs (RGX_CNTBLK_ID)
+                                       valid for this device.
+******************************************************************************/
+IMG_INTERNAL IMG_UINT32
+RGXHWPerfMaxDefinedBlks(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#define _RGX_HWPERF_HOST_FILTER(CTX, EV) \
+		(((PVRSRV_RGXDEV_INFO *)CTX->psDeviceNode->pvDevice)->ui32HWPerfHostFilter \
+		& RGX_HWPERF_EVENT_MASK_VALUE(EV))
+
+#define _RGX_DEVICE_INFO_FROM_CTX(CTX) \
+		((PVRSRV_RGXDEV_INFO *)CTX->psDeviceNode->pvDevice)
+
+#define _RGX_DEVICE_INFO_FROM_NODE(DEVNODE) \
+		((PVRSRV_RGXDEV_INFO *)DEVNODE->pvDevice)
+
+/* Deadline and cycle estimate is not supported for all ENQ events */
+#define NO_DEADLINE 0
+#define NO_CYCEST   0
+
+
+#if defined(SUPPORT_RGX)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param C      Kick context
+ * @param P      Pid of kicking process
+ * @param X      Related FW context
+ * @param E      External job reference
+ * @param I      Job ID
+ * @param K      Kick type
+ * @param CF     Check fence handle
+ * @param UF     Update fence handle
+ * @param UT     Update timeline (on which above UF was created) handle
+ * @param CHKUID Check fence UID
+ * @param UPDUID Update fence UID
+ * @param D      Deadline
+ * @param CE     Cycle estimate
+ */
+#define RGXSRV_HWPERF_ENQ(C, P, X, E, I, K, CF, UF, UT, CHKUID, UPDUID, D, CE) \
+		do { \
+			if (_RGX_HWPERF_HOST_FILTER(C, RGX_HWPERF_HOST_ENQ)) \
+			{ \
+				RGXHWPerfHostPostEnqEvent(_RGX_DEVICE_INFO_FROM_CTX(C), \
+				                          (K), (P), (X), (E), (I), \
+				                          (CF), (UF), (UT), \
+				                          (CHKUID), (UPDUID), (D), (CE)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I Device Info pointer
+ * @param T Host UFO event type
+ * @param D Pointer to UFO data
+ * @param S Is sleeping allowed?
+ */
+#define RGXSRV_HWPERF_UFO(I, T, D, S) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_UFO)) \
+			{ \
+				RGXHWPerfHostPostUfoEvent((I), (T), (D), (S)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device node pointer
+ * @param T Host ALLOC event type
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC(D, T, FWADDR, N, Z) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSyncAlloc.ui32FWAddr = (FWADDR); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+											(N), (Z), &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param PID ID of allocating process
+ * @param FENCE PVRSRV_FENCE object
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_FENCE(D, PID, FENCE, FWADDR, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sFenceAlloc.uiPID = (PID); \
+				uAllocDetail.sFenceAlloc.hFence = (FENCE); \
+				uAllocDetail.sFenceAlloc.ui32CheckPt_FWAddr = (FWADDR); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * @param D Device Node pointer
+ * @param TL PVRSRV_TIMELINE on which CP is allocated
+ * @param PID Allocating process ID of this TL/FENCE
+ * @param FENCE PVRSRV_FENCE as passed to SyncCheckpointResolveFence OR PVRSRV_NO_FENCE
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_SYNC_CP(D, TL, PID, FENCE, FWADDR, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSyncCheckPointAlloc.ui32CheckPt_FWAddr = (FWADDR); \
+				uAllocDetail.sSyncCheckPointAlloc.hTimeline = (TL); \
+				uAllocDetail.sSyncCheckPointAlloc.uiPID = (PID); \
+				uAllocDetail.sSyncCheckPointAlloc.hFence = (FENCE); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * @param D Device Node pointer
+ * @param PID ID of allocating process
+ * @param SW_FENCE PVRSRV_FENCE object
+ * @param SW_TL PVRSRV_TIMELINE on which SW_FENCE is allocated
+ * @param SPI Sync point index on the SW_TL on which this SW_FENCE is allocated
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_SW_FENCE(D, PID, SW_FENCE, SW_TL, SPI, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSWFenceAlloc.uiPID = (PID); \
+				uAllocDetail.sSWFenceAlloc.hSWFence = (SW_FENCE); \
+				uAllocDetail.sSWFenceAlloc.hSWTimeline = (SW_TL); \
+				uAllocDetail.sSWFenceAlloc.ui64SyncPtIndex = (SPI); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param FWADDR sync firmware address
+ */
+#define RGXSRV_HWPERF_FREE(D, T, FWADDR) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_FREE)) \
+			{ \
+				RGXHWPerfHostPostFreeEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                           RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                           (0), (0), (FWADDR)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param UID ID of input object
+ * @param PID ID of allocating process
+ * @param FWADDR sync firmware address
+ */
+#define RGXSRV_HWPERF_FREE_FENCE_SYNC(D, T, UID, PID, FWADDR) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_FREE)) \
+			{ \
+				RGXHWPerfHostPostFreeEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                           RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                           (UID), (PID), (FWADDR)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param NEWUID ID of output object
+ * @param UID1 ID of first input object
+ * @param UID2 ID of second input object
+ * @param N string containing new object's name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_MODIFY_FENCE_SYNC(D, T, NEWUID, UID1, UID2, N, Z) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_MODIFY)) \
+			{ \
+				RGXHWPerfHostPostModifyEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                             RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                             (NEWUID), (UID1), (UID2), N, Z); \
+			} \
+		} while (0)
+
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I Device info pointer
+ */
+#define RGXSRV_HWPERF_CLK_SYNC(I) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_CLK_SYNC)) \
+			{ \
+				RGXHWPerfHostPostClkSyncEvent((I)); \
+			} \
+		} while (0)
+
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts a device info health event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ * @param H      Health status enum
+ * @param R      Health reason enum
+ */
+#define RGXSRV_HWPERF_DEVICE_INFO_HEALTH(I, H, R) \
+	do { \
+		if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_DEV_INFO)) \
+		{ \
+			RGX_HWPERF_HOST_DEV_INFO_DETAIL uDevDetail; \
+			uDevDetail.sDeviceStatus.eDeviceHealthStatus = RGXHWPerfConvDeviceHealthStatus(H); \
+			uDevDetail.sDeviceStatus.eDeviceHealthReason = RGXHWPerfConvDeviceHealthReason(R); \
+			RGXHWPerfHostPostDeviceInfo((I), RGX_HWPERF_DEV_INFO_EV_HEALTH, &uDevDetail); \
+		} \
+	} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts a device info features event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ */
+#define RGXSRV_HWPERF_DEVICE_INFO_FEATURES(I) \
+	do { \
+		if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_DEV_INFO)) \
+		{ \
+			RGXHWPerfHostPostDeviceInfo((I), RGX_HWPERF_DEV_INFO_EV_FEATURES, NULL); \
+		} \
+	} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ * @param T      Event type
+ */
+#define RGXSRV_HWPERF_HOST_INFO(I, T) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_INFO)) \
+	{ \
+		RGXHWPerfHostPostInfo((I), (T)); \
+	} \
+} while (0)
+
+/**
+ * @param I      Device info pointer
+ * @param T      Wait Event type
+ * @param PID    Process ID that the following fence belongs to
+ * @param F      Fence handle
+ * @param D      Data for this wait event type
+ */
+#define RGXSRV_HWPERF_SYNC_FENCE_WAIT(I, T, PID, F, D) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_SYNC_FENCE_WAIT)) \
+	{ \
+		RGXHWPerfHostPostFenceWait(I, RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_##T, \
+		                           (PID), (F), (D)); \
+	} \
+} while (0)
+
+/**
+ * @param I      Device info pointer
+ * @param PID    Process ID that the following timeline belongs to
+ * @param F      SW-timeline handle
+ * @param SPI    Sync-pt index where this SW-timeline has reached
+ */
+#define RGXSRV_HWPERF_SYNC_SW_TL_ADV(I, PID, SW_TL, SPI)\
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE)) \
+	{ \
+		RGXHWPerfHostPostSWTimelineAdv((I), (PID), (SW_TL), (SPI)); \
+	} \
+} while (0)
+
+/**
+ * @param D      Device Node pointer
+ * @param PID    Process ID that the following timeline belongs to
+ * @param N      Null terminated string containing the process name
+ */
+#define RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(D, PID, N) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_CLIENT_INFO)) \
+	{ \
+		RGXHWPerfHostPostClientInfoProcName(_RGX_DEVICE_INFO_FROM_NODE(D), (PID), (N)); \
+	} \
+} while (0)
+
+#else
+
+#define RGXSRV_HWPERF_ENQ(C, P, X, E, I, K, CF, UF, UT, CHKUID, UPDUID, D, CE)
+#define RGXSRV_HWPERF_UFO(I, T, D, S)
+#define RGXSRV_HWPERF_ALLOC(D, T, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_FENCE(D, PID, FENCE, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_SYNC_CP(D, TL, PID, FENCE, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_SW_FENCE(D, PID, SW_FENCE, SW_TL, SPI, N, Z)
+#define RGXSRV_HWPERF_FREE(D, T, FWADDR)
+#define RGXSRV_HWPERF_FREE_FENCE_SYNC(D, T, UID, PID, FWADDR)
+#define RGXSRV_HWPERF_MODIFY_FENCE_SYNC(D, T, NEWUID, UID1, UID2, N, Z)
+#define RGXSRV_HWPERF_CLK_SYNC(I)
+#define RGXSRV_HWPERF_DEVICE_INFO_HEALTH(I, H, R)
+#define RGXSRV_HWPERF_DEVICE_INFO_FEATURES(I)
+#define RGXSRV_HWPERF_HOST_INFO(I, T)
+#define RGXSRV_HWPERF_SYNC_FENCE_WAIT(I, T, PID, F, D)
+#define RGXSRV_HWPERF_SYNC_SW_TL_ADV(I, PID, SW_TL, SPI)
+#define RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(D, PID, N)
+
+#endif
+
+#endif /* RGXHWPERF_COMMON_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit.h
new file mode 100644
index 000000000000..807f6a0b6fce
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit.h
@@ -0,0 +1,313 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX initialisation header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXINIT_H)
+#define RGXINIT_H
+
+#include "connection_server.h"
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_bridge.h"
+#include "fwload.h"
+#include "rgxinit_apphints.h"
+
+#if defined(__linux__)
+#define OS_FW_VERIFY_FUNCTION OSVerifyFirmware
+#else
+#define OS_FW_VERIFY_FUNCTION NULL
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXInitDevPart2
+
+ @Description
+
+ Second part of server-side RGX initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInitDevPart2 (PVRSRV_DEVICE_NODE	*psDeviceNode,
+							  RGX_INIT_APPHINTS		*psApphints);
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    ui32FWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    ui32FWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen);
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXInitFirmware
+
+ @Description
+
+ Server-side RGX firmware initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags);
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXLoadAndGetFWData
+
+ @Description
+
+ Load FW and return pointer to FW data.
+
+ @Input psDeviceNode - device node
+
+ @Input ppsRGXFW - fw pointer
+
+ @Return PVRSRV_ERROR - PVRSRV_OK on success
+                        PVRSRV_ERROR_NOT_READY if filesystem is not ready
+                        PVRSRV_ERROR_NOT_FOUND if no suitable FW image found
+                        PVRSRV_ERROR_OUT_OF_MEMORY if unable to alloc memory for FW image
+                        PVRSRV_ERROR_NOT_AUTHENTICATED if FW image failed verification
+
+******************************************************************************/
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW);
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function	RGXInitHWPerfCounters
+
+ @Description
+
+ Initialisation of the performance counters
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE	*psDeviceNode);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXRegisterDevice
+
+ @Description
+
+ Registers the device with the system
+
+ @Input:   psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevBVNCString
+
+ @Description
+
+ Returns the Device BVNC string. It will allocate and fill it first, if necessary.
+
+ @Input:   psDevInfo - device info (must not be null)
+
+ @Return   IMG_PCHAR - pointer to BVNC string
+
+******************************************************************************/
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function	DevDeInitRGX
+
+ @Description
+
+ Reset and deinitialise Chip
+
+ @Input psDeviceNode - device info. structure
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+
+#if !defined(NO_HARDWARE)
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsRegister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Initialise data used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument). This function must be called only once for each
+               different user/handle.
+
+ @Input        phGpuUtilUser - Pointer to handle used to identify a user of
+                               RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser);
+
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsUnregister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Free data previously used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument).
+
+ @Input        hGpuUtilUser - Handle used to identify a user of
+                              RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*!
+************************************************************************************
+ @Function		RGXSystemGetFabricCoherency
+
+ @Description	Get the system fabric coherency for the device by reading default
+				configuration from device register, subject to AppHint overrides.
+
+ @Input			sRegsCpuPBase		: Device register CPU physical address base
+				ui32RegsSize		: Device register size
+				peDevFabricType		: Device memory bus fabric type
+				peCacheSnoopingMode : Fabric coherency override
+
+ @Return		PVRSRV_ERROR
+************************************************************************************/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDeviceConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode);
+#endif
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXInitCreateFWKernelMemoryContext
+
+ @Description   Called to perform initialisation during firmware kernel context
+                creation.
+
+ @Input         psDeviceNode  device node
+ ******************************************************************************/
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXDeInitDestroyFWKernelMemoryContext
+
+ @Description   Called to perform deinitialisation during firmware kernel
+                context destruction.
+
+ @Input         psDeviceNode  device node
+ ******************************************************************************/
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXHeapDerivePageSize
+
+ @Description   Ensure the desire page size is suitable for the RGX hardware
+
+ @Input         uiLog2PageSize target page log2 size
+
+ @Return        IMG_UINT32 valid page log2 size
+ ******************************************************************************/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXGetNon4KHeapPageShift
+
+ @Description   Retrieves the log2 page size of the General non 4k heap
+
+ @Output        pui32Log2Non4KPgShift page shift
+ ******************************************************************************/
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate,
+                                     IMG_UINT32 *pui32Log2Non4KPgShift);
+
+
+#endif /* RGXINIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit_apphints.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit_apphints.h
new file mode 100644
index 000000000000..f3bea6280a2e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit_apphints.h
@@ -0,0 +1,120 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services device initialisation settings
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device initialisation settings
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXINIT_APPHINTS_H)
+#define RGXINIT_APPHINTS_H
+
+#include "img_defs.h"
+
+#include "rgx_fwif_km.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+
+/*
+ * Container for all the apphints used by this module
+ */
+typedef struct _RGX_INIT_APPHINTS_
+{
+	IMG_BOOL   bEnableSignatureChecks;
+	IMG_UINT32 ui32SignatureChecksBufSize;
+
+	IMG_BOOL   bAssertOnOutOfMem;
+	IMG_BOOL   bAssertOnHWRTrigger;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32TFBCVersion;
+	IMG_UINT32 ui32TFBCCompressionControlGroup;
+	IMG_UINT32 ui32TFBCCompressionControlScheme;
+	IMG_BOOL   bTFBCCompressionControlYUVFormat;
+	IMG_BOOL   bTFBCCompressionControlLossyMinChannel;
+#endif
+	IMG_BOOL   bCheckMlist;
+	IMG_BOOL   bDisableClockGating;
+	IMG_BOOL   bDisableDMOverlap;
+	IMG_BOOL   bDisablePDP;
+	IMG_BOOL   bEnableDMKillRand;
+	IMG_BOOL   bEnableRandomCsw;
+	IMG_BOOL   bEnableSoftResetCsw;
+	IMG_BOOL   bHWPerfDisableCounterFilter;
+	IMG_UINT32 ui32DeviceFlags;
+	IMG_UINT32 ui32FilterFlags;
+	IMG_UINT32 ui32EnableFWContextSwitch;
+	IMG_UINT32 ui32FWContextSwitchProfile;
+	IMG_UINT32 ui32HWPerfFWBufSize;
+	IMG_UINT32 ui32HWPerfHostBufSize;
+	IMG_UINT32 ui32HWPerfFilter0;
+	IMG_UINT32 ui32HWPerfFilter1;
+	IMG_UINT32 ui32HWPerfHostFilter;
+	IMG_UINT32 ui32SecondaryOSClockSource;
+	IMG_UINT32 ui32HWRDebugDumpLimit;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32KCCBSizeLog2;
+#if defined(PVR_ARCH_VOLCANIC)
+	IMG_UINT32 ui32ISPSchedulingLatencyMode;
+#endif
+	FW_PERF_CONF eFirmwarePerf;
+	RGX_ACTIVEPM_CONF eRGXActivePMConf;
+	RGX_RD_POWER_ISLAND_CONF eRGXRDPowerIslandConf;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32 ui32AvailablePowUnitsMask;
+	IMG_UINT32 ui32AvailableRACMask;
+	IMG_BOOL bSPUClockGating;
+#endif
+	IMG_BOOL   bEnableTrustedDeviceAceConfig;
+	IMG_UINT32 ui32FWContextSwitchCrossDM;
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	IMG_UINT32 ui32PhysMemTestPasses;
+#endif
+	RGX_FWT_LOGTYPE eDebugDumpFWTLogType;
+#if defined(SUPPORT_ICS)
+	IMG_UINT32 ui32EnableIdleCycleStealing;
+	IMG_UINT32 ui32FDTI;
+	IMG_UINT32 ui32ICSThreshold;
+	IMG_BOOL   bTestModeOn;
+#endif
+} RGX_INIT_APPHINTS;
+
+#endif /* RGXINIT_APPHINTS_H */
+
+/******************************************************************************
+ End of file (rgxinit_apphints.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.c
new file mode 100644
index 000000000000..b7ab800400c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.c
@@ -0,0 +1,799 @@
+/*************************************************************************/ /*!
+@File           rgxkicksync.c
+@Title          Server side of the sync only kick API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxkicksync.h"
+
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "allocmem.h"
+#include "sync.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_KICKSYNC_UFO_DUMP	0
+
+//#define KICKSYNC_CHECKPOINT_DEBUG 1
+
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+
+struct _RGX_SERVER_KICKSYNC_CONTEXT_
+{
+	PVRSRV_DEVICE_NODE        * psDeviceNode;
+	RGX_SERVER_COMMON_CONTEXT * psServerCommonContext;
+	DLLIST_NODE                 sListNode;
+	SYNC_ADDR_LIST              sSyncAddrListFence;
+	SYNC_ADDR_LIST              sSyncAddrListUpdate;
+	POS_LOCK                    hLock;
+};
+
+
+PVRSRV_ERROR PVRSRVRGXCreateKickSyncContextKM(CONNECTION_DATA              *psConnection,
+                                              PVRSRV_DEVICE_NODE           *psDeviceNode,
+                                              IMG_HANDLE                    hMemCtxPrivData,
+                                              IMG_UINT32                    ui32PackedCCBSizeU88,
+                                              IMG_UINT32                    ui32ContextFlags,
+                                              RGX_SERVER_KICKSYNC_CONTEXT **ppsKickSyncContext)
+{
+	PVRSRV_RGXDEV_INFO          * psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC              * psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext;
+	RGX_COMMON_CONTEXT_INFO      sInfo;
+	PVRSRV_ERROR                 eError;
+	IMG_UINT32                   ui32CCBAllocSizeLog2, ui32CCBMaxAllocSizeLog2;
+
+	memset(&sInfo, 0, sizeof(sInfo));
+
+	/* Prepare cleanup struct */
+	* ppsKickSyncContext = NULL;
+	psKickSyncContext = OSAllocZMem(sizeof(*psKickSyncContext));
+	if (psKickSyncContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psKickSyncContext->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+									__func__,
+									PVRSRVGetErrorString(eError)));
+		goto err_lockcreate;
+	}
+
+	psKickSyncContext->psDeviceNode = psDeviceNode;
+
+	ui32CCBAllocSizeLog2 = U32toU8_Unpack1(ui32PackedCCBSizeU88);
+	ui32CCBMaxAllocSizeLog2 = U32toU8_Unpack2(ui32PackedCCBSizeU88);
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_KICKSYNC,
+									 RGXFWIF_DM_GP,
+									 hMemCtxPrivData,
+									 NULL,
+									 0,
+									 psFWMemContextMemDesc,
+									 NULL,
+									 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_KICKSYNC_CCB_SIZE_LOG2,
+									 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_KICKSYNC_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 0, /* priority */
+									 0, /* max deadline MS */
+									 0, /* robustness address */
+									 & sInfo,
+									 & psKickSyncContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	OSWRLockAcquireWrite(psDevInfo->hKickSyncCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sKickSyncCtxtListHead), &(psKickSyncContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hKickSyncCtxListLock);
+
+	SyncAddrListInit(&psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListInit(&psKickSyncContext->sSyncAddrListUpdate);
+
+	* ppsKickSyncContext = psKickSyncContext;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+	OSLockDestroy(psKickSyncContext->hLock);
+err_lockcreate:
+	OSFreeMem(psKickSyncContext);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXDestroyKickSyncContextKM(RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = psKickSyncContext->psDeviceNode->pvDevice;
+	PVRSRV_ERROR         eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psKickSyncContext->psDeviceNode,
+	                                          psKickSyncContext->psServerCommonContext,
+	                                          RGXFWIF_DM_GP,
+	                                          PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psKickSyncContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+
+	OSWRLockAcquireWrite(psDevInfo->hKickSyncCtxListLock);
+	dllist_remove_node(&(psKickSyncContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hKickSyncCtxListLock);
+
+	FWCommonContextFree(psKickSyncContext->psServerCommonContext);
+	psKickSyncContext->psServerCommonContext = NULL;
+
+	SyncAddrListDeinit(&psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psKickSyncContext->sSyncAddrListUpdate);
+
+	OSLockDestroy(psKickSyncContext->hLock);
+
+	OSFreeMem(psKickSyncContext);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetKickSyncContextPropertyKM(RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContext,
+                                                   RGX_CONTEXT_PROPERTY eContextProperty,
+                                                   IMG_UINT64 ui64Input,
+                                                   IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psKickSyncContext->hLock);
+			eError = FWCommonContextSetFlags(psKickSyncContext->psServerCommonContext,
+			                                 ui32ContextFlags);
+
+			OSLockRelease(psKickSyncContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpKickSyncCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hKickSyncCtxListLock);
+	dllist_foreach_node(&psDevInfo->sKickSyncCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_KICKSYNC_CONTEXT *psCurrentServerKickSyncCtx =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_KICKSYNC_CONTEXT, sListNode);
+
+		if (NULL != psCurrentServerKickSyncCtx->psServerCommonContext)
+		{
+			DumpFWCommonContextInfo(psCurrentServerKickSyncCtx->psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+	OSWRLockReleaseRead(psDevInfo->hKickSyncCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientKickSyncCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hKickSyncCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sKickSyncCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_KICKSYNC_CONTEXT *psCurrentServerKickSyncCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_KICKSYNC_CONTEXT, sListNode);
+
+		if (NULL != psCurrentServerKickSyncCtx->psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerKickSyncCtx->psServerCommonContext, RGX_KICK_TYPE_DM_GP) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_GP;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hKickSyncCtxListLock);
+	return ui32ContextBitMask;
+}
+
+PVRSRV_ERROR PVRSRVRGXKickSyncKM(RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext,
+                                 IMG_UINT32                    ui32ClientUpdateCount,
+                                 SYNC_PRIMITIVE_BLOCK       ** pauiClientUpdateUFODevVarBlock,
+                                 IMG_UINT32                  * paui32ClientUpdateOffset,
+                                 IMG_UINT32                  * paui32ClientUpdateValue,
+                                 PVRSRV_FENCE                  iCheckFence,
+                                 PVRSRV_TIMELINE               iUpdateTimeline,
+                                 PVRSRV_FENCE                * piUpdateFence,
+                                 IMG_CHAR                      szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                 IMG_UINT32                    ui32ExtJobRef)
+{
+	RGXFWIF_KCCB_CMD         sKickSyncKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA  asCmdHelperData[1];
+	PVRSRV_ERROR             eError;
+	PVRSRV_ERROR             eError2;
+	IMG_BOOL                 bCCBStateOpen = IMG_FALSE;
+	PRGXFWIF_UFO_ADDR        *pauiClientFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR        *pauiClientUpdateUFOAddress = NULL;
+	IMG_UINT32               ui32ClientFenceCount = 0;
+	IMG_UINT32               *paui32ClientFenceValue = NULL;
+	PVRSRV_FENCE             iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT32               ui32FWCtx = FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext).ui32Addr;
+	PVRSRV_RGXDEV_INFO       *psDevInfo = FWCommonContextGetRGXDevInfo(psKickSyncContext->psServerCommonContext);
+	RGX_CLIENT_CCB           *psClientCCB = FWCommonContextGetClientCCB(psKickSyncContext->psServerCommonContext);
+	IMG_UINT32               ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have dev var updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	OSLockAcquire(psKickSyncContext->hLock);
+	eError = SyncAddrListPopulate(&psKickSyncContext->sSyncAddrListUpdate,
+							ui32ClientUpdateCount,
+							pauiClientUpdateUFODevVarBlock,
+							paui32ClientUpdateOffset);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_syncaddrlist;
+	}
+
+	if (ui32ClientUpdateCount > 0)
+	{
+		pauiClientUpdateUFOAddress = psKickSyncContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	/* Ensure the string is null-terminated (Required for safety) */
+	szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	/* This will never be true if called from the bridge since piUpdateFence will always be valid */
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto out_unlock;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), "
+			   "psKickSyncContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+			   __func__, iCheckFence,
+			   (void*)psKickSyncContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psKickSyncContext->psDeviceNode->hSyncCheckpointContext,
+	                                    iCheckFence,
+	                                    &ui32FenceSyncCheckpointCount,
+	                                    &apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID,
+	                                    PDUMP_FLAGS_NONE);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_resolve_fence;
+	}
+
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: calling SyncCheckpointCreateFence (iUpdateTimeline=%d)...",
+				   __func__, iUpdateTimeline));
+		eError = SyncCheckpointCreateFence(psKickSyncContext->psDeviceNode,
+		                                   szUpdateFenceName,
+		                                   iUpdateTimeline,
+		                                   psKickSyncContext->psDeviceNode->hSyncCheckpointContext,
+		                                   &iUpdateFence,
+		                                   &uiUpdateFenceUID,
+		                                   &pvUpdateFenceFinaliseData,
+		                                   &psUpdateSyncCheckpoint,
+		                                   (void*)&psFenceTimelineUpdateSync,
+		                                   &ui32FenceTimelineUpdateValue,
+		                                   PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%d)",
+					   __func__, eError));
+			goto fail_create_output_fence;
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: ...returned from SyncCheckpointCreateFence "
+				   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+				   "ui32FenceTimelineUpdateValue=%u)",
+				   __func__, iUpdateFence, psFenceTimelineUpdateSync,
+				   ui32FenceTimelineUpdateValue));
+
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*paui32ClientUpdateValue) * (ui32ClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32ClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32ClientUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32ClientUpdateCount);
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32ClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32ClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32ClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x",
+							   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now append the timeline sync prim addr to the kicksync context update list */
+			SyncAddrListAppendSyncPrim(&psKickSyncContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+		}
+	}
+
+	/* Reset number of fence syncs in kicksync context fence list to 0 */
+	SyncAddrListPopulate(&psKickSyncContext->sSyncAddrListFence,
+	                     0, NULL, NULL);
+
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		/* Append the checks (from input fence) */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Append %d sync checkpoints to KickSync Fence "
+				   "(&psKickSyncContext->sSyncAddrListFence=<%p>)...",
+				   __func__, ui32FenceSyncCheckpointCount,
+				   (void*)&psKickSyncContext->sSyncAddrListFence));
+		SyncAddrListAppendCheckpoints(&psKickSyncContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+
+		pauiClientFenceUFOAddress = psKickSyncContext->sSyncAddrListFence.pasFWAddrs;
+
+		ui32ClientFenceCount += ui32FenceSyncCheckpointCount;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiClientFenceUFOAddress;
+
+				for (iii=0; iii<ui32ClientFenceCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientFenceUFOAddress[%d](<%p>) = 0x%x",
+							   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+	}
+
+	if (psUpdateSyncCheckpoint)
+	{
+		PVRSRV_ERROR eErr;
+
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Append 1 sync checkpoint to KickSync Update "
+				   "(&psKickSyncContext->sSyncAddrListUpdate=<%p>)...",
+				   __func__, (void*)&psKickSyncContext->sSyncAddrListUpdate));
+		eErr = SyncAddrListAppendCheckpoints(&psKickSyncContext->sSyncAddrListUpdate,
+											 1,
+											 &psUpdateSyncCheckpoint);
+		if (eErr != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:  ...done. SyncAddrListAppendCheckpoints() returned error (%d)",
+					   __func__, eErr));
+		}
+		else
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:  ...done.", __func__));
+		}
+		if (!pauiClientUpdateUFOAddress)
+		{
+			pauiClientUpdateUFOAddress = psKickSyncContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32ClientUpdateCount++;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiClientUpdateUFOAddress;
+
+			for (iii=0; iii<ui32ClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: pauiClientUpdateUFOAddress[%d](<%p>) = 0x%x",
+						   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+
+#if (ENABLE_KICKSYNC_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping KICKSYNC fence/updates syncs...",
+				 __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiClientFenceUFOAddress;
+			IMG_UINT32 *pui32TmpIntFenceValue = paui32ClientFenceValue;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiClientUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32ClientUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Prepared %d KickSync fence syncs "
+					 "(&psKickSyncContext->sSyncAddrListFence=<%p>, "
+					 "pauiClientFenceUFOAddress=<%p>):",
+					 __func__, ui32ClientFenceCount,
+					 (void*)&psKickSyncContext->sSyncAddrListFence,
+					 (void*)pauiClientFenceUFOAddress));
+			for (ii=0; ii<ui32ClientFenceCount; ii++)
+			{
+				if (psTmpIntFenceUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, "
+							 "CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+							 __func__, ii + 1, ui32ClientFenceCount,
+							 (void*)psTmpIntFenceUFOAddress,
+							 psTmpIntFenceUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+							 __func__, ii + 1, ui32ClientFenceCount,
+							 (void*)psTmpIntFenceUFOAddress,
+							 psTmpIntFenceUFOAddress->ui32Addr,
+							 *pui32TmpIntFenceValue,
+							 *pui32TmpIntFenceValue));
+					pui32TmpIntFenceValue++;
+				}
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Prepared %d KickSync update syncs "
+					 "(&psKickSyncContext->sSyncAddrListUpdate=<%p>, "
+					 "pauiClientUpdateUFOAddress=<%p>):",
+					 __func__, ui32ClientUpdateCount,
+					 (void*)&psKickSyncContext->sSyncAddrListUpdate,
+					 (void*)pauiClientUpdateUFOAddress));
+			for (ii=0; ii<ui32ClientUpdateCount; ii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:  Line %d, psTmpIntUpdateUFOAddress=<%p>",
+						   __func__, __LINE__,
+						   (void*)psTmpIntUpdateUFOAddress));
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:  Line %d, pui32TmpIntUpdateValue=<%p>",
+						   __func__, __LINE__,
+						   (void*)pui32TmpIntUpdateValue));
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, "
+							 "UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+							 __func__, ii + 1, ui32ClientUpdateCount,
+							 (void*)psTmpIntUpdateUFOAddress,
+							 psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d",
+							 __func__, ii + 1, ui32ClientUpdateCount,
+							 (void*)psTmpIntUpdateUFOAddress,
+							 psTmpIntUpdateUFOAddress->ui32Addr,
+							 *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       0, /* empty ui64FBSCEntryMask */
+	                       ui32ClientFenceCount,
+	                       pauiClientFenceUFOAddress,
+	                       paui32ClientFenceValue,
+	                       ui32ClientUpdateCount,
+	                       pauiClientUpdateUFOAddress,
+	                       paui32ClientUpdateValue,
+	                       0,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       RGXFWIF_CCB_CMD_TYPE_NULL,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       PDUMP_FLAGS_NONE,
+	                       NULL,
+	                       "KickSync",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+
+	/*
+	 *  We should reserve space in the kernel CCB here and fill in the command
+	 *  directly.
+	 *  This is so if there isn't space in the kernel CCB we can return with
+	 *  retry back to services client before we take any operations
+	 */
+
+	/*
+	 * All the required resources are ready at this point, we can't fail so
+	 * take the required server sync operations and commit all the resources
+	 */
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	RGXCmdHelperReleaseCmdCCB(1,
+							  asCmdHelperData,
+							  "KickSync",
+							  FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext).ui32Addr);
+
+	/* Construct the kernel kicksync CCB command. */
+	sKickSyncKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext);
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+	/*
+	 * Submit the kicksync command to the firmware.
+	 */
+	RGXSRV_HWPERF_ENQ(psKickSyncContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_SYNC,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError2 = RGXScheduleCommandWithoutPowerLock(psKickSyncContext->psDeviceNode->pvDevice,
+		                             RGXFWIF_DM_GP,
+		                             & sKickSyncKCCBCmd,
+		                             PDUMP_FLAGS_NONE);
+		if (eError2 != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	PVRGpuTraceEnqueueEvent(psKickSyncContext->psDeviceNode,
+	                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+	                        RGX_HWPERF_KICK_TYPE2_SYNC);
+
+	if (eError2 != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXKickSync failed to schedule kernel CCB command. (0x%x)",
+		         eError));
+		eError = eError2;
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier call
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdacquire;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim<%p> to %d",
+				   __func__, (void*)psFenceTimelineUpdateSync,
+				   ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	*piUpdateFence = iUpdateFence;
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psKickSyncContext->psDeviceNode, iUpdateFence,
+									pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, szUpdateFenceName);
+	}
+
+	OSLockRelease(psKickSyncContext->hLock);
+	return PVRSRV_OK;
+
+fail_acquirepowerlock:
+fail_cmdacquire:
+	SyncAddrListRollbackCheckpoints(psKickSyncContext->psDeviceNode, &psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psKickSyncContext->psDeviceNode, &psKickSyncContext->sSyncAddrListUpdate);
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+
+	/* Free memory allocated to hold update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+	}
+fail_alloc_update_values_mem:
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free memory allocated to hold the resolved fence's checkpoints */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+fail_resolve_fence:
+fail_syncaddrlist:
+out_unlock:
+	OSLockRelease(psKickSyncContext->hLock);
+	return eError;
+}
+
+/**************************************************************************//**
+ End of file (rgxkicksync.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.h
new file mode 100644
index 000000000000..57b49a03da5c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.h
@@ -0,0 +1,128 @@
+/*************************************************************************/ /*!
+@File           rgxkicksync.h
+@Title          Server side of the sync only kick API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXKICKSYNC_H)
+#define RGXKICKSYNC_H
+
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+#include "sync_server.h"
+#include "rgxdevice.h"
+
+
+typedef struct _RGX_SERVER_KICKSYNC_CONTEXT_ RGX_SERVER_KICKSYNC_CONTEXT;
+
+/**************************************************************************/ /*!
+@Function       DumpKickSyncCtxtsInfo
+@Description    Function that dumps debug info of kick sync ctxs on this device
+@Return         none
+*/ /**************************************************************************/
+void
+DumpKickSyncCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile,
+                      IMG_UINT32 ui32VerbLevel);
+
+/**************************************************************************/ /*!
+@Function       CheckForStalledClientKickSyncCtxt
+@Description    Function that checks if a kick sync client is stalled
+@Return         RGX_KICK_TYPE_DM_GP on stalled context. Otherwise, 0
+*/ /**************************************************************************/
+IMG_UINT32 CheckForStalledClientKickSyncCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXCreateKickSyncContextKM
+@Description    Server-side implementation of RGXCreateKicksyncContext
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXCreateKickSyncContextKM(CONNECTION_DATA              * psConnection,
+                                 PVRSRV_DEVICE_NODE           * psDeviceNode,
+                                 IMG_HANDLE                     hMemCtxPrivData,
+                                 IMG_UINT32                     ui32PackedCCBSizeU88,
+                                 IMG_UINT32                     ui32ContextFlags,
+                                 RGX_SERVER_KICKSYNC_CONTEXT ** ppsKicksyncContext);
+
+
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXDestroyKickSyncContextKM
+@Description    Server-side implementation of RGXDestroyKicksyncContext
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXDestroyKickSyncContextKM(RGX_SERVER_KICKSYNC_CONTEXT * psKicksyncContext);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXSetKickSyncContextPropertyKM
+@Description    Server-side implementation of RGXSetKickSyncContextProperty
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+ */ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetKickSyncContextPropertyKM(RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContext,
+                                                   RGX_CONTEXT_PROPERTY eContextProperty,
+                                                   IMG_UINT64 ui64Input,
+                                                   IMG_UINT64 *pui64Output);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXKickSyncKM
+@Description    Kicks a sync only command
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXKickSyncKM(RGX_SERVER_KICKSYNC_CONTEXT * psKicksyncContext,
+                    IMG_UINT32                    ui32ClientUpdateCount,
+                    SYNC_PRIMITIVE_BLOCK       ** pauiClientUpdateUFODevVarBlock,
+                    IMG_UINT32                  * paui32ClientUpdateDevVarOffset,
+                    IMG_UINT32                  * paui32ClientUpdateValue,
+                    PVRSRV_FENCE                  iCheckFence,
+                    PVRSRV_TIMELINE               iUpdateTimeline,
+                    PVRSRV_FENCE                * piUpdateFence,
+                    IMG_CHAR                      szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+
+                    IMG_UINT32                    ui32ExtJobRef);
+
+#endif /* RGXKICKSYNC_H */
+
+/**************************************************************************//**
+ End of file (rgxkicksync.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer.h
new file mode 100644
index 000000000000..3141db5c4488
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer.h
@@ -0,0 +1,920 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declaration of an interface layer used to abstract code that
+                can be compiled outside of the DDK, potentially in a
+                completely different OS.
+                All the headers included by this file must also be copied to
+                the alternative source tree.
+                All the functions declared here must have a DDK implementation
+                inside the DDK source tree (e.g. rgxlayer_impl.h/.c) and
+                another different implementation in case they are used outside
+                of the DDK.
+                All of the functions accept as a first parameter a
+                "const void *hPrivate" argument. It should be used to pass
+                around any implementation specific data required.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXLAYER_H)
+#define RGXLAYER_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "img_elf.h"
+#include "pvrsrv_error.h" /* includes pvrsrv_errors.h */
+#include "rgx_firmware_boot.h"
+#include "rgx_bvnc_defs_km.h"
+#include "rgx_fw_info.h"
+#include "rgx_common.h"
+#include "rgx_meta.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgx_mips.h"
+#endif
+#include "rgx_riscv.h"
+
+#include "rgxdefs_km.h"
+/* includes:
+ * rgx_cr_defs_km.h,
+ * RGX_BVNC_CORE_KM_HEADER (rgxcore_km_B.V.N.C.h),
+ * RGX_BNC_CONFIG_KM_HEADER (rgxconfig_km_B.V.N.C.h)
+ */
+
+/*
+ * Specific fields for RGX_CR_IDLE must not be polled in pdumps
+ * (technical reasons)
+ */
+#define CR_IDLE_UNSELECTED_MASK ((~RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK) |	\
+								 (~RGX_CR_SLC_IDLE_OWDB_CLRMSK) |			\
+								 (RGX_CR_SLC_IDLE_FBCDC_ARB_EN))
+
+
+/*!
+*******************************************************************************
+
+ @Function       RGXMemCopy
+
+ @Description    MemCopy implementation
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pvDst      : Pointer to the destination
+ @Input          pvSrc      : Pointer to the source location
+ @Input          uiSize     : The amount of memory to copy in bytes
+
+ @Return         void
+
+******************************************************************************/
+void RGXMemCopy(const void *hPrivate,
+                void *pvDst,
+                void *pvSrc,
+                size_t uiSize);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXMemSet
+
+ @Description    MemSet implementation
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pvDst      : Pointer to the start of the memory region
+ @Input          ui8Value   : The value to be written
+ @Input          uiSize     : The number of bytes to be set to ui8Value
+
+ @Return         void
+
+******************************************************************************/
+void RGXMemSet(const void *hPrivate,
+               void *pvDst,
+               IMG_UINT8 ui8Value,
+               size_t uiSize);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXCommentLog
+
+ @Description    Generic log function used for debugging or other purposes
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pszString  : Message to be printed
+ @Input          ...        : Variadic arguments
+
+ @Return         void
+
+******************************************************************************/
+__printf(2, 3)
+void RGXCommentLog(const void *hPrivate,
+                   const IMG_CHAR *pszString,
+                   ...);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXErrorLog
+
+ @Description    Generic error log function used for debugging or other purposes
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pszString  : Message to be printed
+ @Input          ...        : Variadic arguments
+
+ @Return         void
+
+******************************************************************************/
+__printf(2, 3)
+void RGXErrorLog(const void *hPrivate,
+                 const IMG_CHAR *pszString,
+                 ...);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function       RGXGetOSPageSize
+
+ @Description    Return Page size used on OS
+
+ @Input          hPrivate   : Implementation specific data
+
+ @Return         IMG_UINT32
+
+******************************************************************************/
+
+IMG_UINT32 RGXGetOSPageSize(const void *hPrivate);
+#endif
+
+/* This is used to check if a specific feature is enabled.
+ * Should be used instead of calling RGXDeviceHasFeature.  */
+#define RGX_DEVICE_HAS_FEATURE(hPrivate, Feature) \
+			RGXDeviceHasFeature(hPrivate, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if a specific feature with value is enabled.
+ * Should be used instead of calling RGXDeviceGetFeatureValue.  */
+#define RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, Feature) \
+			(RGXDeviceGetFeatureValue(hPrivate, RGX_FEATURE_##Feature##_IDX) >= 0)
+
+/* This is used to get the value of a specific feature from hPrivate.
+ * Should be used instead of calling RGXDeviceGetFeatureValue.  */
+#define RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, Feature) \
+			RGXDeviceGetFeatureValue(hPrivate, RGX_FEATURE_##Feature##_IDX)
+
+
+/* This is used to check if a specific ERN is enabled from hPrivate.
+ * Should be used instead of calling RGXDeviceHasErnBrn.  */
+#define RGX_DEVICE_HAS_ERN(hPrivate, ERN) \
+			RGXDeviceHasErnBrn(hPrivate, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if a specific BRN is enabled from hPrivate.
+ * Should be used instead of calling RGXDeviceHasErnBrn.  */
+#define RGX_DEVICE_HAS_BRN(hPrivate, BRN) \
+			RGXDeviceHasErnBrn(hPrivate, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+#define CLK_CTRL_FORCE_ON(X, Module) \
+			X = (((X) & RGX_CR_##Module##_CLRMSK) | RGX_CR_##Module##_ON)
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceGetFeatureValue
+
+ @Description    Checks if a device has a particular feature with values
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64Feature  : Feature with values to check
+
+ @Return         Value >= 0 if the given feature is available, -1 otherwise
+
+******************************************************************************/
+IMG_INT32 RGXDeviceGetFeatureValue(const void *hPrivate, IMG_UINT64 ui64Feature);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceHasFeature
+
+ @Description    Checks if a device has a particular feature
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64Feature  : Feature to check
+
+ @Return         IMG_TRUE if the given feature is available, IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDeviceHasFeature(const void *hPrivate, IMG_UINT64 ui64Feature);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceHasErnBrn
+
+ @Description    Checks if a device has a particular errata
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64ErnsBrns : Flags to check
+
+ @Return         IMG_TRUE if the given errata is available, IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDeviceHasErnBrn(const void *hPrivate, IMG_UINT64 ui64ErnsBrns);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetFWCorememSize
+
+ @Description    Get the FW coremem size
+
+ @Input          hPrivate   : Implementation specific data
+
+ @Return         FW coremem size
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWCorememSize(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXWriteReg32/64
+
+ @Description   Write a value to a 32/64 bit RGX register
+
+ @Input         hPrivate         : Implementation specific data
+ @Input         ui32RegAddr      : Register offset inside the register bank
+ @Input         ui32/64RegValue  : New register value
+
+ @Return        void
+
+******************************************************************************/
+void RGXWriteReg32(const void *hPrivate,
+                   IMG_UINT32 ui32RegAddr,
+                   IMG_UINT32 ui32RegValue);
+
+void RGXWriteReg64(const void *hPrivate,
+                   IMG_UINT32 ui32RegAddr,
+                   IMG_UINT64 ui64RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadReg32/64
+
+ @Description    Read a 32/64 bit RGX register
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui32RegAddr  : Register offset inside the register bank
+
+ @Return         Register value
+
+******************************************************************************/
+IMG_UINT32 RGXReadReg32(const void *hPrivate,
+                        IMG_UINT32 ui32RegAddr);
+
+IMG_UINT64 RGXReadReg64(const void *hPrivate,
+                        IMG_UINT32 ui32RegAddr);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadModifyWriteReg32
+
+ @Description    Read-modify-write a 32 bit RGX register
+
+ @Input          hPrivate     : Implementation specific data.
+ @Input          ui32RegAddr  : Register offset inside the register bank.
+ @Input          ui32RegValue : New register value.
+ @Input          ui32RegMask  : Keep the bits set in the mask.
+
+ @Return         Always returns PVRSRV_OK
+
+******************************************************************************/
+IMG_UINT32 RGXReadModifyWriteReg64(const void *hPrivate,
+                                   IMG_UINT32 ui32RegAddr,
+                                   IMG_UINT64 ui64RegValue,
+                                   IMG_UINT64 ui64RegKeepMask);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXPollReg32/64
+
+ @Description    Poll on a 32/64 bit RGX register until some bits are set/unset
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          ui32/64RegValue  : Value expected from the register
+ @Input          ui32/64RegMask   : Only the bits set in this mask will be
+                                    checked against uiRegValue
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV_ERROR_TIMEOUT if the poll takes too long
+
+******************************************************************************/
+PVRSRV_ERROR RGXPollReg32(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT32 ui32RegValue,
+                          IMG_UINT32 ui32RegMask);
+
+PVRSRV_ERROR RGXPollReg64(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64RegValue,
+                          IMG_UINT64 ui64RegMask);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXWriteMetaRegThroughSP
+
+ @Description    Write a register value using the META slave port
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          ui32RegValue     : Value written to the register
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadMetaRegThroughSP
+
+ @Description    Read a register value using the META slave port
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          *ui32RegValue    : Value read from the register
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXSetPoweredState
+
+ @Description    Declare if the device is powered or not
+
+ @Input          hPrivate    : Implementation specific data
+ @Input          bPowered    : true is powered, false otherwise
+
+ @Return         void
+
+******************************************************************************/
+void RGXSetPoweredState(const void *hPrivate, IMG_BOOL bPowered);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXWaitCycles
+
+ @Description    Wait for a number of GPU cycles and/or microseconds
+
+ @Input          hPrivate    : Implementation specific data
+ @Input          ui32Cycles  : Number of GPU cycles to wait for in pdumps,
+                               it can also be used when running driver-live
+                               if desired (ignoring the next parameter)
+ @Input          ui32WaitUs  : Number of microseconds to wait for when running
+                               driver-live
+
+ @Return         void
+
+******************************************************************************/
+void RGXWaitCycles(const void *hPrivate,
+                   IMG_UINT32 ui32Cycles,
+                   IMG_UINT32 ui32WaitUs);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireKernelMMUPC
+
+ @Description     Acquire the Kernel MMU Page Catalogue device physical address
+
+ @Input           hPrivate  : Implementation specific data
+ @Input           psPCAddr  : Returned page catalog address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireKernelMMUPC(const void *hPrivate, IMG_DEV_PHYADDR *psPCAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXWriteKernelMMUPC32/64
+
+ @Description     Write the Kernel MMU Page Catalogue to the 32/64 bit
+                  RGX register passed as argument.
+                  In a driver-live scenario without PDump these functions
+                  are the same as RGXWriteReg32/64 and they don't need
+                  to be reimplemented.
+
+ @Input           hPrivate        : Implementation specific data
+ @Input           ui32PCReg       : Register offset inside the register bank
+ @Input           ui32AlignShift  : PC register alignshift
+ @Input           ui32Shift       : PC register shift
+ @Input           ui32/64PCVal    : Page catalog value (aligned and shifted)
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+void RGXWriteKernelMMUPC64(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT64 ui64PCVal);
+#endif
+
+void RGXWriteKernelMMUPC32(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT32 ui32PCVal);
+#else /* defined(PDUMP) */
+
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGXWriteKernelMMUPC64(priv, pcreg, alignshift, shift, pcval) \
+	RGXWriteReg64(priv, pcreg, pcval)
+#endif
+
+#define RGXWriteKernelMMUPC32(priv, pcreg, alignshift, shift, pcval) \
+	RGXWriteReg32(priv, pcreg, pcval)
+
+#endif /* defined(PDUMP) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireGPURegsAddr
+
+ @Description     Acquire the GPU registers base device physical address
+
+ @Input           hPrivate       : Implementation specific data
+ @Input           psGPURegsAddr  : Returned GPU registers base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireGPURegsAddr(const void *hPrivate, IMG_DEV_PHYADDR *psGPURegsAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXMIPSWrapperConfig
+
+ @Description     Write GPU register bank transaction ID and MIPS boot mode
+                  to the MIPS wrapper config register (passed as argument).
+                  In a driver-live scenario without PDump this is the same as
+                  RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate          : Implementation specific data
+ @Input           ui32RegAddr       : Register offset inside the register bank
+ @Input           ui64GPURegsAddr   : GPU registers base address
+ @Input           ui32GPURegsAlign  : Register bank transactions alignment
+ @Input           ui32BootMode      : Mips BOOT ISA mode
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXMIPSWrapperConfig(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64GPURegsAddr,
+                          IMG_UINT32 ui32GPURegsAlign,
+                          IMG_UINT32 ui32BootMode);
+#else
+#define RGXMIPSWrapperConfig(priv, regaddr, gpuregsaddr, gpuregsalign, bootmode) \
+	RGXWriteReg64(priv, regaddr, ((gpuregsaddr) >> (gpuregsalign)) | (bootmode))
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS bootloader
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootRemapAddr  : Base address of the remapped bootloader
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psBootRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXBootRemapConfig
+
+ @Description     Configure the bootloader remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXBootRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXBootRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireCodeRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS code
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psCodeRemapAddr  : Base address of the remapped code
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireCodeRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psCodeRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXCodeRemapConfig
+
+ @Description     Configure the code remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXCodeRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXCodeRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireDataRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS data
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psDataRemapAddr  : Base address of the remapped data
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireDataRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psDataRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXDataRemapConfig
+
+ @Description     Configure the data remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXDataRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXDataRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireTrampolineRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS data
+                  accessed through remap region
+
+ @Input           hPrivate             : Implementation specific data
+ @Output          psTrampolineRemapAddr: Base address of the remapped data
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireTrampolineRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psTrampolineRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXTrampolineRemapConfig
+
+ @Description     Configure the trampoline remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXTrampolineRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXTrampolineRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+/*!
+*******************************************************************************
+
+ @Function        RGXDoFWSlaveBoot
+
+ @Description     Returns whether or not a FW Slave Boot is required
+                  while powering on
+
+ @Input           hPrivate       : Implementation specific data
+
+ @Return          IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXFabricCoherencyTest
+
+ @Description    Performs fabric coherency test
+
+ @Input          hPrivate         : Implementation specific data
+
+ @Return         PVRSRV_OK if the test succeeds,
+                 PVRSRV_ERROR_INIT_FAILURE if the test fails at some point
+
+******************************************************************************/
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDeviceSLCBanks
+
+ @Description    Returns the number of SLC banks used by the device
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Number of SLC banks
+
+******************************************************************************/
+IMG_UINT32 RGXGetDeviceSLCBanks(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDeviceCacheLineSize
+
+ @Description    Returns the device cache line size
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Cache line size
+
+******************************************************************************/
+IMG_UINT32 RGXGetDeviceCacheLineSize(const void *hPrivate);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDevicePhysBusWidth
+
+ @Description    Returns the device physical bus width
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Physical bus width
+
+******************************************************************************/
+IMG_UINT32 RGXGetDevicePhysBusWidth(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDevicePA0IsValid
+
+ @Description    Returns true if the device physical address 0x0 is a valid
+                 address and can be accessed by the GPU.
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         IMG_TRUE if device physical address 0x0 is a valid address,
+                 IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDevicePA0IsValid(const void *hPrivate);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootCodeAddr
+
+ @Description     Acquire the device virtual address of the RISCV boot code
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootCodeAddr   : Boot code base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootCodeAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootCodeAddr);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXCalculateHostFWDataAddress
+
+ @Description   Calculates the base host address of the RISCV firmware data
+
+ @Input         hPrivate         : Implementation specific data
+ @Input         pvHostFWDataAddr : Initial host address of the firmware data
+
+ @Return:       Base host address of the RISCV firmware data
+
+******************************************************************************/
+void *RGXCalculateHostFWDataAddress(const void *hPrivate, void *pvHostFWDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootDataAddr
+
+ @Description     Acquire the device virtual address of the RISCV boot data
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootDataAddr   : Boot data base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootDataAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXDeviceAckIrq
+
+ @Description   Checks the implementation specific IRQ status register,
+                clearing it if necessary and returning the IRQ status.
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return:       IRQ status
+
+******************************************************************************/
+IMG_BOOL RGXDeviceAckIrq(const void *hPrivate);
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*!
+*******************************************************************************
+
+@Function       RGXMMUInitRangeValue
+
+@Description    Returns the appropriate config value for each MMU range
+
+@Input          ui32MMURange   : Hardware MMU range to be initialised
+
+@Return         64-bit register value
+
+******************************************************************************/
+IMG_UINT64 RGXMMUInitRangeValue(IMG_UINT32 ui32MMURange);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGXLAYER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl.h
new file mode 100644
index 000000000000..b46efee8c71c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXLAYER_IMPL_H)
+#define RGXLAYER_IMPL_H
+
+#include "rgxlayer.h"
+#include "device_connection.h"
+
+typedef struct _RGX_LAYER_PARAMS_
+{
+	void *psDevInfo;
+	void *psDevConfig;
+#if defined(PDUMP)
+	IMG_UINT32 ui32PdumpFlags;
+#endif
+
+	IMG_DEV_PHYADDR sPCAddr;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEV_PHYADDR sGPURegAddr;
+	IMG_DEV_PHYADDR sBootRemapAddr;
+	IMG_DEV_PHYADDR sCodeRemapAddr;
+	IMG_DEV_PHYADDR sDataRemapAddr;
+	IMG_DEV_PHYADDR sTrampolineRemapAddr;
+	IMG_BOOL bDevicePA0IsValid;
+#endif
+} RGX_LAYER_PARAMS;
+
+#endif /* RGXLAYER_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl_common.c
new file mode 100644
index 000000000000..7eab6fde4dcd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl_common.c
@@ -0,0 +1,914 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwimageutils.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr) \
+	(((ui32RegAddr) < RGX_HOST_SECURE_REGBANK_OFFSET) ? \
+	 ((psDevInfo)->pvRegsBaseKM) : ((psDevInfo)->pvSecureRegsBaseKM))
+#else
+#define RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr) ((psDevInfo)->pvRegsBaseKM)
+#endif
+
+void RGXMemCopy(const void *hPrivate,
+                void *pvDst,
+                void *pvSrc,
+                size_t uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	OSDeviceMemCopy(pvDst, pvSrc, uiSize);
+}
+
+void RGXMemSet(const void *hPrivate,
+               void *pvDst,
+               IMG_UINT8 ui8Value,
+               size_t uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	OSDeviceMemSet(pvDst, ui8Value, uiSize);
+}
+
+void RGXCommentLog(const void *hPrivate,
+                   const IMG_CHAR *pszString,
+                   ...)
+{
+#if defined(PDUMP)
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	va_list argList;
+	va_start(argList, pszString);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	PDumpCommentWithFlagsVA(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS, pszString, argList);
+	va_end(argList);
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	PVR_UNREFERENCED_PARAMETER(pszString);
+#endif
+}
+
+void RGXErrorLog(const void *hPrivate,
+                 const IMG_CHAR *pszString,
+                 ...)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list argList;
+
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+
+	va_start(argList, pszString);
+	vsnprintf(szBuffer, sizeof(szBuffer), pszString, argList);
+	va_end(argList);
+
+	PVR_DPF((PVR_DBG_ERROR, "%s", szBuffer));
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+IMG_UINT32 RGXGetOSPageSize(const void *hPrivate)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	return OSGetPageSize();
+}
+#endif
+
+IMG_INT32 RGXDeviceGetFeatureValue(const void *hPrivate, IMG_UINT64 ui64Feature)
+{
+	IMG_INT32 i32Ret = -1;
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	if ((psDeviceNode->pfnGetDeviceFeatureValue))
+	{
+		i32Ret = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode, ui64Feature);
+	}
+
+	return i32Ret;
+}
+
+IMG_BOOL RGXDeviceHasFeature(const void *hPrivate, IMG_UINT64 ui64Feature)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->sDevFeatureCfg.ui64Features & ui64Feature) != 0;
+}
+
+IMG_UINT32 RGXGetFWCorememSize(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32CorememSize = 0;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		ui32CorememSize = RGX_GET_FEATURE_VALUE(psDevInfo, META_COREMEM_SIZE);
+	}
+
+	return ui32CorememSize;
+}
+
+void RGXWriteReg32(const void *hPrivate, IMG_UINT32 ui32RegAddr, IMG_UINT32 ui32RegValue)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		OSWriteUncheckedHWReg32(pvRegsBase, ui32RegAddr, ui32RegValue);
+	}
+
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	           ui32RegAddr, ui32RegValue, psParams->ui32PdumpFlags);
+}
+
+void RGXWriteReg64(const void *hPrivate, IMG_UINT32 ui32RegAddr, IMG_UINT64 ui64RegValue)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		OSWriteUncheckedHWReg64(pvRegsBase, ui32RegAddr, ui64RegValue);
+	}
+
+	PDUMPREG64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	           ui32RegAddr, ui64RegValue, psParams->ui32PdumpFlags);
+}
+
+IMG_UINT32 RGXReadReg32(const void *hPrivate, IMG_UINT32 ui32RegAddr)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+	IMG_UINT32 ui32RegValue;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW)
+	{
+		ui32RegValue = IMG_UINT32_MAX;
+	}
+	else
+#endif
+	{
+		ui32RegValue = OSReadUncheckedHWReg32(pvRegsBase, ui32RegAddr);
+	}
+
+	PDUMPREGREAD32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	               ui32RegAddr, psParams->ui32PdumpFlags);
+
+	return ui32RegValue;
+}
+
+IMG_UINT64 RGXReadReg64(const void *hPrivate, IMG_UINT32 ui32RegAddr)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+	IMG_UINT64 ui64RegValue;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW)
+	{
+		ui64RegValue = IMG_UINT64_MAX;
+	}
+	else
+#endif
+	{
+		ui64RegValue = OSReadUncheckedHWReg64(pvRegsBase, ui32RegAddr);
+	}
+
+	PDUMPREGREAD64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	               ui32RegAddr, PDUMP_FLAGS_CONTINUOUS);
+
+	return ui64RegValue;
+}
+
+IMG_UINT32 RGXReadModifyWriteReg64(const void *hPrivate,
+                                   IMG_UINT32 ui32RegAddr,
+                                   IMG_UINT64 uiRegValueNew,
+                                   IMG_UINT64 uiRegKeepMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+#if defined(PDUMP)
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+	/* only use the new values for bits we update according to the keep mask */
+	uiRegValueNew &= ~uiRegKeepMask;
+
+#if defined(PDUMP)
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store register offset to temp PDump variable */
+	PDumpRegRead64ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                            ":SYSMEM:$1", ui32RegAddr, ui32PDumpFlags);
+
+	/* Keep the bits set in the mask */
+	PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                        uiRegKeepMask, ui32PDumpFlags);
+
+	/* OR the new values */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                       uiRegValueNew, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                        ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+
+	{
+		IMG_UINT64 uiRegValue = OSReadUncheckedHWReg64(pvRegsBase, ui32RegAddr);
+		uiRegValue &= uiRegKeepMask;
+		OSWriteUncheckedHWReg64(pvRegsBase, ui32RegAddr, uiRegValue | uiRegValueNew);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollReg32(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT32 ui32RegValue,
+                          IMG_UINT32 ui32RegMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr),
+		                         ui32RegValue,
+		                         ui32RegMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg32: Poll for Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+	}
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr,
+	            ui32RegValue,
+	            ui32RegMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollReg64(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64RegValue,
+                          IMG_UINT64 ui64RegMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	/* Split lower and upper words */
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64RegValue >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64RegValue);
+	IMG_UINT32 ui32UpperMask = (IMG_UINT32) (ui64RegMask >> 32);
+	IMG_UINT32 ui32LowerMask = (IMG_UINT32) (ui64RegMask);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr + 4),
+		                         ui32UpperValue,
+		                         ui32UpperMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg64: Poll for upper part of Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr),
+		                         ui32LowerValue,
+		                         ui32LowerMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg64: Poll for lower part of Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+	}
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr + 4,
+	            ui32UpperValue,
+	            ui32UpperMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr,
+	            ui32LowerValue,
+	            ui32LowerMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+}
+
+void RGXSetPoweredState(const void *hPrivate, IMG_BOOL bPowered)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	psDevInfo->bRGXPowered = bPowered;
+}
+
+void RGXWaitCycles(const void *hPrivate, IMG_UINT32 ui32Cycles, IMG_UINT32 ui32TimeUs)
+{
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+	OSWaitus(ui32TimeUs);
+	PDUMPIDLWITHFLAGS(psDevInfo->psDeviceNode, ui32Cycles, PDUMP_FLAGS_CONTINUOUS);
+}
+
+void RGXAcquireKernelMMUPC(const void *hPrivate, IMG_DEV_PHYADDR *psPCAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psPCAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sPCAddr;
+}
+
+#if defined(PDUMP)
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+void RGXWriteKernelMMUPC64(const void *hPrivate,
+		IMG_UINT32 ui32PCReg,
+		IMG_UINT32 ui32PCRegAlignShift,
+		IMG_UINT32 ui32PCRegShift,
+		IMG_UINT64 ui64PCVal)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write the cat-base address */
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, ui32PCReg, ui64PCVal);
+
+	/* Pdump catbase address */
+	MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
+			RGX_PDUMPREG_NAME,
+			ui32PCReg,
+			8,
+			ui32PCRegAlignShift,
+			ui32PCRegShift,
+			PDUMP_FLAGS_CONTINUOUS);
+}
+#endif
+
+void RGXWriteKernelMMUPC32(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT32 ui32PCVal)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32PCReg);
+
+	/* Write the cat-base address */
+	OSWriteUncheckedHWReg32(pvRegsBase, ui32PCReg, ui32PCVal);
+
+	/* Pdump catbase address */
+	MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
+	                          RGX_PDUMPREG_NAME,
+	                          ui32PCReg,
+	                          4,
+	                          ui32PCRegAlignShift,
+	                          ui32PCRegShift,
+	                          PDUMP_FLAGS_CONTINUOUS);
+}
+#endif /* defined(PDUMP) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXAcquireGPURegsAddr(const void *hPrivate, IMG_DEV_PHYADDR *psGPURegsAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psGPURegsAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sGPURegAddr;
+}
+
+#if defined(PDUMP)
+void RGXMIPSWrapperConfig(const void *hPrivate,
+		IMG_UINT32 ui32RegAddr,
+		IMG_UINT64 ui64GPURegsAddr,
+		IMG_UINT32 ui32GPURegsAlign,
+		IMG_UINT32 ui32BootMode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
+			ui32RegAddr,
+			(ui64GPURegsAddr >> ui32GPURegsAlign) | ui32BootMode);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store register offset to temp PDump variable */
+	PDumpRegLabelToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                           ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	/* Align register transactions identifier */
+	PDumpWriteVarSHRValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                        ui32GPURegsAlign, ui32PDumpFlags);
+
+	/* Enable micromips instruction encoding */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                       ui32BootMode, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                        ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+}
+#endif
+
+void RGXAcquireBootRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psBootRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psBootRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sBootRemapAddr;
+}
+
+void RGXAcquireCodeRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psCodeRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psCodeRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sCodeRemapAddr;
+}
+
+void RGXAcquireDataRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psDataRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psDataRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sDataRemapAddr;
+}
+
+void RGXAcquireTrampolineRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psTrampolineRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psTrampolineRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sTrampolineRemapAddr;
+}
+
+#if defined(PDUMP)
+static inline
+void RGXWriteRemapConfig2Reg(void __iomem *pvRegs,
+		PMR *psPMR,
+		IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+		IMG_UINT32 ui32RegAddr,
+		IMG_UINT64 ui64PhyAddr,
+		IMG_UINT64 ui64PhyMask,
+		IMG_UINT64 ui64Settings)
+{
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_ASSERT(psPMR != NULL);
+	psDevNode = PMR_DeviceNode(psPMR);
+
+	OSWriteHWReg64(pvRegs, ui32RegAddr, (ui64PhyAddr & ui64PhyMask) | ui64Settings);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store memory offset to temp PDump variable */
+	PDumpMemLabelToInternalVar64(":SYSMEM:$1", psPMR,
+	                             uiLogicalOffset, ui32PDumpFlags);
+
+	/* Keep only the relevant bits of the output physical address */
+	PDumpWriteVarANDValueOp(psDevNode, ":SYSMEM:$1", ui64PhyMask, ui32PDumpFlags);
+
+	/* Extra settings for this remapped region */
+	PDumpWriteVarORValueOp(psDevNode, ":SYSMEM:$1", ui64Settings, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevNode, RGX_PDUMPREG_NAME, ui32RegAddr,
+	                        ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+}
+
+void RGXBootRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32BootRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWCodeMemDesc->uiOffset + ui32BootRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXCodeRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32CodeRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWCodeMemDesc->uiOffset + ui32CodeRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXDataRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32DataRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWDataMemDesc->uiOffset + ui32DataRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXTrampolineRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* write the register for real, without PDump */
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store the memory address in a PDump variable */
+	PDumpPhysHandleToInternalVar64(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			psDevInfo->psTrampoline->hPdumpPages,
+			ui32PDumpFlags);
+
+	/* Keep only the relevant bits of the input physical address */
+	PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			~RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_CLRMSK,
+			ui32PDumpFlags);
+
+	/* Enable bit */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+			ui32PDumpFlags);
+
+	/* Do the PDump register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode,
+			RGX_PDUMPREG_NAME,
+			ui32Config1RegAddr,
+			":SYSMEM:$1",
+			ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+
+	/* this can be written directly */
+	RGXWriteReg64(hPrivate,
+			ui32Config2RegAddr,
+			(ui64Config2PhyAddr & ui64Config2PhyMask) | ui64Config2Settings);
+}
+#endif /* defined(PDUMP) */
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+IMG_BOOL RGXDeviceHasErnBrn(const void *hPrivate, IMG_UINT64 ui64ErnsBrns)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->sDevFeatureCfg.ui64ErnsBrns & ui64ErnsBrns) != 0;
+}
+
+IMG_UINT32 RGXGetDeviceSLCBanks(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_BANKS))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, SLC_BANKS);
+}
+
+IMG_UINT32 RGXGetDeviceCacheLineSize(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_CACHE_LINE_SIZE_BITS))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+IMG_UINT32 RGXGetDevicePhysBusWidth(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PHYS_BUS_WIDTH))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH);
+}
+
+IMG_BOOL RGXDevicePA0IsValid(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return psDevInfo->sLayerParams.bDevicePA0IsValid;
+}
+#endif
+
+void RGXAcquireBootCodeAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootCodeAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	*psBootCodeAddr = psDevInfo->sFWCodeDevVAddrBase;
+}
+
+void RGXAcquireBootDataAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootDataAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	*psBootDataAddr = psDevInfo->sFWDataDevVAddrBase;
+}
+
+void *RGXCalculateHostFWDataAddress(const void *hPrivate, void *pvHostFWDataAddr)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT8 *ui8HostFWDataAddr = (IMG_UINT8*)pvHostFWDataAddr;
+	IMG_UINT32 ui32Offset = 0U;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		ui32Offset =
+			PVR_ALIGN(RGXGetFWImageSectionAllocSize(hPrivate, RISCV_UNCACHED_CODE),
+			          RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN) +
+			PVR_ALIGN(RGXGetFWImageSectionAllocSize(hPrivate, RISCV_CACHED_CODE),
+			          RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN);
+	}
+
+	ui8HostFWDataAddr -= ui32Offset;
+	return (void*)ui8HostFWDataAddr;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+
+	return pvHostFWDataAddr;
+#endif
+}
+
+IMG_BOOL RGXDeviceAckIrq(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->pfnRGXAckIrq != NULL) ?
+			psDevInfo->pfnRGXAckIrq(psDevInfo) : IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.c
new file mode 100644
index 000000000000..4df36ef5d1f0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.c
@@ -0,0 +1,973 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX memory context management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX memory context management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_server_utils.h"
+#include "devicemem_pdump.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_km.h"
+#include "rgxfwutils.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "sync_internal.h"
+#include "rgx_memallocflags.h"
+#include "rgx_bvnc_defs_km.h"
+#include "info_page.h"
+
+# include "rgxmmudefs_km.h"
+
+#if defined(PDUMP)
+#include "sync.h"
+#endif
+
+struct SERVER_MMU_CONTEXT_TAG
+{
+	DEVMEM_MEMDESC *psFWMemContextMemDesc;
+	PRGXFWIF_FWMEMCONTEXT sFWMemContextDevVirtAddr;
+	MMU_CONTEXT *psMMUContext;
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_UINT64 ui64FBSCEntryMask;
+	DLLIST_NODE sNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+}; /* SERVER_MMU_CONTEXT is typedef-ed in rgxmem.h */
+
+PVRSRV_ERROR RGXInvalidateFBSCTable(PVRSRV_DEVICE_NODE *psDeviceNode,
+									MMU_CONTEXT *psMMUContext,
+									IMG_UINT64 ui64FBSCEntryMask)
+{
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSWRLockAcquireRead(psDevInfo->hMemoryCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psIter = IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+		if (psIter->psMMUContext == psMMUContext)
+		{
+			psServerMMUContext = psIter;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hMemoryCtxListLock);
+
+	if (! psServerMMUContext)
+	{
+		return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+	}
+
+	/* Accumulate the FBSC invalidate request */
+	psServerMMUContext->ui64FBSCEntryMask |= ui64FBSCEntryMask;
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXExtractFBSCEntryMaskFromMMUContext
+ *
+ */
+PVRSRV_ERROR RGXExtractFBSCEntryMaskFromMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+												   SERVER_MMU_CONTEXT *psServerMMUContext,
+												   IMG_UINT64 *pui64FBSCEntryMask)
+{
+	if (!psServerMMUContext)
+	{
+		return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+	}
+
+	*pui64FBSCEntryMask = psServerMMUContext->ui64FBSCEntryMask;
+	psServerMMUContext->ui64FBSCEntryMask = 0;
+
+	return PVRSRV_OK;
+}
+
+void RGXMMUCacheInvalidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   MMU_CONTEXT *psMMUContext,
+						   MMU_LEVEL eMMULevel,
+						   IMG_BOOL bUnmap)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT32 ui32NewCacheFlags;
+
+	PVR_UNREFERENCED_PARAMETER(bUnmap);
+
+	switch (eMMULevel)
+	{
+		case MMU_LEVEL_3:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PC;
+
+			break;
+		case MMU_LEVEL_2:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PD;
+
+			break;
+		case MMU_LEVEL_1:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PT;
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+			if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT)))
+#endif
+			{
+				ui32NewCacheFlags |= RGXFWIF_MMUCACHEDATA_FLAGS_TLB;
+			}
+
+			break;
+		default:
+			ui32NewCacheFlags = 0;
+			PVR_ASSERT(0);
+
+			break;
+	}
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	/* For VIVT devices only accumulate the flags on the Firmware MMU context
+	 * since the Firmware/HW invalidates caches on every kick automatically. */
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT) ||
+	    psMMUContext == psDevInfo->psKernelMMUCtx)
+#endif
+	{
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32NewCacheFlags);
+	}
+}
+
+static
+PVRSRV_ERROR _PrepareAndSubmitCacheCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           RGXFWIF_DM eDM,
+                                           IMG_UINT32 ui32CacheFlags,
+                                           IMG_BOOL bInterrupt,
+                                           IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sFlushCmd;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	*pui32MMUInvalidateUpdate = psDeviceNode->ui32NextMMUInvalidateUpdate++;
+
+	/* Setup cmd and add the device nodes sync object */
+	sFlushCmd.eCmdType = RGXFWIF_KCCB_CMD_MMUCACHE;
+	sFlushCmd.uCmdData.sMMUCacheData.ui32MMUCacheSyncUpdateValue = *pui32MMUInvalidateUpdate;
+	SyncPrimGetFirmwareAddr(psDeviceNode->psMMUCacheSyncPrim,
+	                        &sFlushCmd.uCmdData.sMMUCacheData.sMMUCacheSync.ui32Addr);
+
+	/*
+	 *  Indicate if the firmware should signal command completion to the host.
+	 *  The sync update will always happen. This flag requests that the FW pass
+	 *  back the result of the KCCB command and interrupt the host. The KCCB
+	 *  response is not used but the interrupt signal to the host is as a way
+	 *  to know the sync update may have happened.
+	 */
+	if (bInterrupt)
+	{
+		ui32CacheFlags |= RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT;
+	}
+
+	sFlushCmd.uCmdData.sMMUCacheData.ui32CacheFlags = ui32CacheFlags;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Submit MMU flush and invalidate (flags = 0x%08x)",
+	                      ui32CacheFlags);
+#endif
+
+	/* Schedule MMU cache command */
+	eError = RGXSendCommand(psDevInfo,
+							&sFlushCmd,
+							PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule MMU cache command to "
+		         "DM=%d with error (%u)",
+		         __func__, eDM, eError));
+		psDeviceNode->ui32NextMMUInvalidateUpdate--;
+
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32CacheFlags);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXMMUCacheInvalidateKick(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32FWCacheFlags;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+		goto RGXMMUCacheInvalidateKick_exit;
+	}
+
+	/*
+	 * Atomically clear flags to ensure we never accidentally read state
+	 * inconsistently or overwrite valid cache flags with 0.
+	 */
+	ui32FWCacheFlags = MMU_GetAndResetCacheFlags(psDevInfo->psKernelMMUCtx);
+	if (ui32FWCacheFlags == 0)
+	{
+		/* Nothing to do if no cache ops pending */
+		eError = PVRSRV_OK;
+		goto _PowerUnlockAndReturnErr;
+	}
+
+	/* Ensure device is powered up before sending cache command */
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32FWCacheFlags);
+		goto _PowerUnlockAndReturnErr;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = _PrepareAndSubmitCacheCommand(psDeviceNode, RGXFWIF_DM_GP, ui32FWCacheFlags,
+											   IMG_TRUE, pui32MMUInvalidateUpdate);
+		if (!PVRSRVIsRetryError(eError))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_LOG_IF_ERROR(eError, "_PrepareAndSubmitCacheCommand");
+
+_PowerUnlockAndReturnErr:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+RGXMMUCacheInvalidateKick_exit:
+	return eError;
+}
+
+PVRSRV_ERROR RGXPreKickCacheCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+									RGXFWIF_DM eDM,
+									IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32FWCacheFlags;
+
+	/* Caller should ensure that power lock is held before calling this function */
+	PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+
+	/*
+	 * Atomically clear flags to ensure we never accidentally read state
+	 * inconsistently or overwrite valid cache flags with 0.
+	 */
+	ui32FWCacheFlags = MMU_GetAndResetCacheFlags(psDevInfo->psKernelMMUCtx);
+	if (ui32FWCacheFlags == 0)
+	{
+		/* Nothing to do if no cache ops pending */
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = _PrepareAndSubmitCacheCommand(psDeviceNode, eDM, ui32FWCacheFlags,
+		                                       IMG_FALSE, pui32MMUInvalidateUpdate);
+		if (!PVRSRVIsRetryError(eError))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "_PrepareAndSubmitCacheCommand");
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+/*
+ * RGXMapBRN71422TargetPhysicalAddress
+ *
+ * Only used on early Volcanic cores.
+ * Set-up a special MMU tree mapping with a single page that eventually points
+ * to RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR. This is supplied by the
+ * customer (in rgxdefs_km.h) and hence this WA is off by default.
+ *
+ * PC entries are 32b, with the last 4 bits being 0 except for the LSB bit that should be 1 (Valid). Addr is 4KB aligned.
+ * PD entries are 64b, with addr in bits 39:5 and everything else 0 except for LSB bit that is Valid. Addr is byte aligned?
+ * PT entries are 64b, with phy addr in bits 39:12 and everything else 0 except for LSB bit that is Valid. Addr is 4KB aligned.
+ * So, we can construct the page tables in a single page like this:
+ *   0x00 : PCE (PCE index 0)
+ *   0x04 : 0x0
+ *   0x08 : PDEa (PDE index 1)
+ *   0x0C : PDEb
+ *   0x10 : PTEa (PTE index 2)
+ *   0x14 : PTEb
+ *
+ * With the PCE and the PDE pointing to this same page.
+ * The VA address that we are mapping is therefore:
+ *  VA = PCE_idx*PCE_size + PDE_idx*PDE_size + PTE_idx*PTE_size =
+ *     =      0 * 1GB     +      1 * 2MB     +      2 * 4KB     =
+ *     =        0         +    0x20_0000     +      0x2000      =
+ *     = 0x00_0020_2000
+ */
+void RGXMapBRN71422TargetPhysicalAddress(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDevNode,
+                                         IMG_DEV_PHYADDR sPhysAddrL1Px,
+                                         void *pxL1PxCpuVAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32       *pui32Px    = pxL1PxCpuVAddr;
+	IMG_UINT64       *pui64Px    = pxL1PxCpuVAddr;
+	IMG_UINT64       ui64Entry;
+
+
+	/* Only setup the BRN71422 workaround if this is the FW memory
+	 * context and BRN present.
+	 */
+	if ((psConnection != NULL) || !RGX_IS_BRN_SUPPORTED(psDevInfo, 71422))
+	{
+		return;
+	}
+
+	/* Setup dummy mapping to fast constant time physical address. */
+	/* PCE points to PC */
+	ui64Entry = sPhysAddrL1Px.uiAddr;
+	ui64Entry = ui64Entry >> RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;
+	ui64Entry = ui64Entry << RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PC_DATA_VALID_EN;
+	pui32Px[0] = (IMG_UINT32) ui64Entry;
+
+	/* PDE points to PC */
+	ui64Entry = sPhysAddrL1Px.uiAddr;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PD_DATA_VALID_EN;
+	pui64Px[1] = ui64Entry;
+
+	/* PTE points to PAddr */
+	ui64Entry = RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PT_DATA_VALID_EN;
+	pui64Px[2] = ui64Entry;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Mapping the BRN71422 workaround to target physical address 0x%" IMG_UINT64_FMTSPECx ".",
+	         __func__, RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR));
+
+}
+#endif
+
+/* Common header, only needed for architectures with MIPS FW CPU */
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXMMUTweakProtFlags(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                  MMU_DEVICEATTRIBS *psDevAttrs,
+		                  PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+		                  MMU_PROTFLAGS_T *puiMMUProtFlags)
+{
+	/* If we are allocating on the MMU of the firmware processor, the
+	 * cached/uncached attributes must depend on the FIRMWARE_CACHED
+	 * allocation flag.
+	 */
+	if (psDevAttrs == psDevNode->psFirmwareMMUDevAttrs)
+	{
+		if (uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED))
+		{
+			*puiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
+		}
+		else
+		{
+			*puiMMUProtFlags &= ~MMU_PROTFLAGS_CACHED;
+
+		}
+		*puiMMUProtFlags &= ~MMU_PROTFLAGS_CACHE_COHERENT;
+	}
+}
+#endif
+
+
+/* page fault debug is the only current use case for needing to find process info
+ * after that process device memory context has been destroyed
+ */
+
+typedef struct _UNREGISTERED_MEMORY_CONTEXT_
+{
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_DEV_PHYADDR sPCDevPAddr;
+} UNREGISTERED_MEMORY_CONTEXT;
+
+/* must be a power of two */
+#define UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE (1 << 3)
+
+static UNREGISTERED_MEMORY_CONTEXT gasUnregisteredMemCtxs[UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE];
+static IMG_UINT32 gui32UnregisteredMemCtxsHead;
+
+/* record a device memory context being unregistered.
+ * the list of unregistered contexts can be used to find the PID and process name
+ * belonging to a memory context which has been destroyed
+ */
+static void _RecordUnregisteredMemoryContext(PVRSRV_RGXDEV_INFO *psDevInfo, SERVER_MMU_CONTEXT *psServerMMUContext)
+{
+	UNREGISTERED_MEMORY_CONTEXT *psRecord;
+
+	OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+	psRecord = &gasUnregisteredMemCtxs[gui32UnregisteredMemCtxsHead];
+
+	gui32UnregisteredMemCtxsHead = (gui32UnregisteredMemCtxsHead + 1)
+					& (UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1);
+
+	OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+
+	psRecord->uiPID = psServerMMUContext->uiPID;
+	if (MMU_AcquireBaseAddr(psServerMMUContext->psMMUContext, &psRecord->sPCDevPAddr) != PVRSRV_OK)
+	{
+		PVR_LOG(("_RecordUnregisteredMemoryContext: Failed to get PC address for memory context"));
+	}
+	OSStringSafeCopy(psRecord->szProcessName, psServerMMUContext->szProcessName, sizeof(psRecord->szProcessName));
+}
+
+
+void RGXUnregisterMemoryContext(IMG_HANDLE hPrivData)
+{
+	SERVER_MMU_CONTEXT *psServerMMUContext = hPrivData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psServerMMUContext->psDevInfo;
+
+#if defined(PDUMP)
+	{
+		RGXFWIF_DEV_VIRTADDR sFWAddr;
+
+		RGXSetFirmwareAddress(&sFWAddr,
+		                      psServerMMUContext->psFWMemContextMemDesc,
+		                      0,
+		                      RFW_FWADDR_NOREF_FLAG);
+
+		/*
+		 * MMU cache commands (always dumped) might have a pointer to this FW
+		 * memory context, wait until the FW has caught-up to the latest command.
+		 */
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+		             "Ensure FW has executed all MMU invalidations on FW memory "
+		             "context 0x%x before freeing it", sFWAddr.ui32Addr);
+		SyncPrimPDumpPol(psDevInfo->psDeviceNode->psMMUCacheSyncPrim,
+		                 psDevInfo->psDeviceNode->ui32NextMMUInvalidateUpdate - 1,
+		                 0xFFFFFFFF,
+		                 PDUMP_POLL_OPERATOR_GREATEREQUAL,
+		                 PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+	OSWRLockAcquireWrite(psDevInfo->hMemoryCtxListLock);
+	dllist_remove_node(&psServerMMUContext->sNode);
+	OSWRLockReleaseWrite(psDevInfo->hMemoryCtxListLock);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		_RecordUnregisteredMemoryContext(psDevInfo, psServerMMUContext);
+	}
+
+	/*
+	 * Release the page catalogue address acquired in RGXRegisterMemoryContext().
+	 */
+	MMU_ReleaseBaseAddr(NULL);
+
+	/*
+	 * Free the firmware memory context.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free FW memory context");
+	DevmemFwUnmapAndFree(psDevInfo, psServerMMUContext->psFWMemContextMemDesc);
+
+	OSFreeMem(psServerMMUContext);
+}
+
+/*
+ * RGXRegisterMemoryContext
+ */
+PVRSRV_ERROR RGXRegisterMemoryContext(PVRSRV_DEVICE_NODE	*psDeviceNode,
+									  MMU_CONTEXT			*psMMUContext,
+									  IMG_HANDLE			*hPrivData)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_FWMEMCONTEXT	*psFWMemContext;
+	DEVMEM_MEMDESC			*psFWMemContextMemDesc;
+	SERVER_MMU_CONTEXT *psServerMMUContext;
+
+	if (psDevInfo->psKernelMMUCtx == NULL)
+	{
+		/*
+		 * This must be the creation of the Kernel memory context. Take a copy
+		 * of the MMU context for use when programming the BIF.
+		 */
+		psDevInfo->psKernelMMUCtx = psMMUContext;
+	}
+	else
+	{
+		psServerMMUContext = OSAllocMem(sizeof(*psServerMMUContext));
+		if (psServerMMUContext == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_alloc_server_ctx;
+		}
+
+		psServerMMUContext->psDevInfo = psDevInfo;
+		psServerMMUContext->ui64FBSCEntryMask = 0;
+		psServerMMUContext->sFWMemContextDevVirtAddr.ui32Addr = 0;
+
+		/*
+			Allocate device memory for the firmware memory context for the new
+			application.
+		*/
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGX firmware memory context");
+		eError = DevmemFwAllocate(psDevInfo,
+								sizeof(*psFWMemContext),
+								RGX_FWCOMCTX_ALLOCFLAGS,
+								"FwMemoryContext",
+								&psFWMemContextMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to allocate firmware memory context (%u)",
+			         __func__,
+			         eError));
+			goto fail_alloc_fw_ctx;
+		}
+
+		/*
+			Temporarily map the firmware memory context to the kernel.
+		*/
+		eError = DevmemAcquireCpuVirtAddr(psFWMemContextMemDesc,
+										  (void **)&psFWMemContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map firmware memory context (%u)",
+			         __func__,
+			         eError));
+			goto fail_acquire_cpu_addr;
+		}
+
+		/*
+		 * Write the new memory context's page catalogue into the firmware memory
+		 * context for the client.
+		 */
+		eError = MMU_AcquireBaseAddr(psMMUContext, &psFWMemContext->sPCDevPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to acquire Page Catalogue address (%u)",
+			         __func__,
+			         eError));
+			DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+			goto fail_acquire_base_addr;
+		}
+
+		/*
+		 * Set default values for the rest of the structure.
+		 */
+		psFWMemContext->uiPageCatBaseRegSet = RGXFW_BIF_INVALID_PCSET;
+		psFWMemContext->uiBreakpointAddr = 0;
+		psFWMemContext->uiBPHandlerAddr = 0;
+		psFWMemContext->uiBreakpointCtl = 0;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+{
+		IMG_UINT32 ui32OSid = 0, ui32OSidReg = 0;
+		IMG_BOOL   bOSidAxiProt;
+
+		MMU_GetOSids(psMMUContext, &ui32OSid, &ui32OSidReg, &bOSidAxiProt);
+
+		psFWMemContext->ui32OSid     = ui32OSidReg;
+		psFWMemContext->bOSidAxiProt = bOSidAxiProt;
+}
+#endif
+
+#if defined(PDUMP)
+		{
+			IMG_CHAR			aszName[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+			IMG_DEVMEM_OFFSET_T uiOffset = 0;
+
+			/*
+			 * Dump the Mem context allocation
+			 */
+			DevmemPDumpLoadMem(psFWMemContextMemDesc, 0, sizeof(*psFWMemContext), PDUMP_FLAGS_CONTINUOUS);
+
+
+			/*
+			 * Obtain a symbolic addr of the mem context structure
+			 */
+			eError = DevmemPDumpPageCatBaseToSAddr(psFWMemContextMemDesc,
+												   &uiOffset,
+												   aszName,
+												   PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Failed to generate a Dump Page Catalogue address (%u)",
+				         __func__,
+				         eError));
+				DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+				goto fail_pdump_cat_base_addr;
+			}
+
+			/*
+			 * Dump the Page Cat tag in the mem context (symbolic address)
+			 */
+			eError = MMU_PDumpWritePageCatBase(psMMUContext,
+												aszName,
+												uiOffset,
+												8, /* 64-bit register write */
+												0,
+												0,
+												0);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Failed to acquire Page Catalogue address (%u)",
+				         __func__,
+				         eError));
+				DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+				goto fail_pdump_cat_base;
+			}
+		}
+#endif
+
+		/*
+		 * Release kernel address acquired above.
+		 */
+		RGXFwSharedMemCacheOpPtr(psFWMemContext, FLUSH);
+		DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+
+		/*
+		 * Store the process information for this device memory context
+		 * for use with the host page-fault analysis.
+		 */
+		psServerMMUContext->uiPID = OSGetCurrentClientProcessIDKM();
+		psServerMMUContext->psMMUContext = psMMUContext;
+		psServerMMUContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
+		OSStringSafeCopy(psServerMMUContext->szProcessName,
+		              OSGetCurrentClientProcessNameKM(),
+		              sizeof(psServerMMUContext->szProcessName));
+
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "New memory context: Process Name: %s PID: %u (0x%08X)",
+		                      psServerMMUContext->szProcessName,
+		                      psServerMMUContext->uiPID,
+		                      psServerMMUContext->uiPID);
+
+		OSWRLockAcquireWrite(psDevInfo->hMemoryCtxListLock);
+		dllist_add_to_tail(&psDevInfo->sMemoryContextList, &psServerMMUContext->sNode);
+		OSWRLockReleaseWrite(psDevInfo->hMemoryCtxListLock);
+
+		*hPrivData = psServerMMUContext;
+	}
+
+	return PVRSRV_OK;
+
+#if defined(PDUMP)
+fail_pdump_cat_base:
+fail_pdump_cat_base_addr:
+	MMU_ReleaseBaseAddr(NULL);
+#endif
+fail_acquire_base_addr:
+	/* Done before jumping to the fail point as the release is done before exit */
+fail_acquire_cpu_addr:
+	DevmemFwUnmapAndFree(psDevInfo, psServerMMUContext->psFWMemContextMemDesc);
+fail_alloc_fw_ctx:
+	OSFreeMem(psServerMMUContext);
+fail_alloc_server_ctx:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+DEVMEM_MEMDESC *RGXGetFWMemDescFromMemoryContextHandle(IMG_HANDLE hPriv)
+{
+	SERVER_MMU_CONTEXT *psMMUContext = (SERVER_MMU_CONTEXT *) hPriv;
+
+	return psMMUContext->psFWMemContextMemDesc;
+}
+
+void RGXSetFWMemContextDevVirtAddr(SERVER_MMU_CONTEXT *psServerMMUContext,
+						RGXFWIF_DEV_VIRTADDR	sFWMemContextAddr)
+{
+	psServerMMUContext->sFWMemContextDevVirtAddr.ui32Addr = sFWMemContextAddr.ui32Addr;
+}
+
+void RGXCheckFaultAddress(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_DEV_VIRTADDR *psDevVAddr,
+				IMG_DEV_PHYADDR *psDevPAddr,
+				MMU_FAULT_DATA *psOutFaultData)
+{
+	IMG_DEV_PHYADDR sPCDevPAddr;
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hMemoryCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psServerMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+
+		if (MMU_AcquireBaseAddr(psServerMMUContext->psMMUContext, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for memory context"));
+			continue;
+		}
+
+		if (psDevPAddr->uiAddr == sPCDevPAddr.uiAddr)
+		{
+			MMU_CheckFaultAddress(psServerMMUContext->psMMUContext, psDevVAddr, psOutFaultData);
+			goto out_unlock;
+		}
+	}
+
+	/* Lastly check for fault in the kernel allocated memory */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		if (MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for kernel memory context"));
+		}
+
+		if (psDevPAddr->uiAddr == sPCDevPAddr.uiAddr)
+		{
+			MMU_CheckFaultAddress(psDevInfo->psKernelMMUCtx, psDevVAddr, psOutFaultData);
+		}
+	}
+
+out_unlock:
+	OSWRLockReleaseRead(psDevInfo->hMemoryCtxListLock);
+}
+
+/* given the physical address of a page catalogue, searches for a corresponding
+ * MMU context and if found, provides the caller details of the process.
+ * Returns IMG_TRUE if a process is found.
+ */
+IMG_BOOL RGXPCAddrToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_PHYADDR sPCAddress,
+								RGXMEM_PROCESS_INFO *psInfo)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+
+	/* check if the input PC addr corresponds to an active memory context */
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psThisMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+		IMG_DEV_PHYADDR sPCDevPAddr;
+
+		if (MMU_AcquireBaseAddr(psThisMMUContext->psMMUContext, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for memory context"));
+			continue;
+		}
+
+		if (sPCAddress.uiAddr == sPCDevPAddr.uiAddr)
+		{
+			psServerMMUContext = psThisMMUContext;
+			break;
+		}
+	}
+
+	if (psServerMMUContext != NULL)
+	{
+		psInfo->uiPID = psServerMMUContext->uiPID;
+		OSStringSafeCopy(psInfo->szProcessName, psServerMMUContext->szProcessName, sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+	/* else check if the input PC addr corresponds to the firmware */
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_DEV_PHYADDR sKernelPCDevPAddr;
+		PVRSRV_ERROR eError;
+
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sKernelPCDevPAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for kernel memory context"));
+		}
+		else
+		{
+			if (sPCAddress.uiAddr == sKernelPCDevPAddr.uiAddr)
+			{
+				psInfo->uiPID = RGXMEM_SERVER_PID_FIRMWARE;
+				OSStringSafeCopy(psInfo->szProcessName, "Firmware", sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_FALSE;
+				bRet = IMG_TRUE;
+			}
+		}
+	}
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(bRet == IMG_FALSE))
+	{
+		/* no active memory context found with the given PC address.
+		 * Check the list of most recently freed memory contexts.
+		 */
+		const IMG_UINT32 ui32Mask = UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1;
+		IMG_UINT32 i, j;
+
+		OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+		/* iterate through the list of unregistered memory contexts
+		 * from newest (one before the head) to the oldest (the current head)
+		 */
+		for (i = (gui32UnregisteredMemCtxsHead - 1) & ui32Mask, j = 0;
+		     j < UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE;
+		     i = (i - 1) & ui32Mask, j++)
+		{
+			UNREGISTERED_MEMORY_CONTEXT *psRecord = &gasUnregisteredMemCtxs[i];
+
+			if (psRecord->sPCDevPAddr.uiAddr == sPCAddress.uiAddr)
+			{
+				psInfo->uiPID = psRecord->uiPID;
+				OSStringSafeCopy(psInfo->szProcessName, psRecord->szProcessName, sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_TRUE;
+				bRet = IMG_TRUE;
+				break;
+			}
+		}
+
+		OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	return bRet;
+}
+
+IMG_BOOL RGXPCPIDToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_PID uiPID,
+								RGXMEM_PROCESS_INFO *psInfo)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+
+	/* check if the input PID corresponds to an active memory context */
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psThisMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+
+		if (psThisMMUContext->uiPID == uiPID)
+		{
+			psServerMMUContext = psThisMMUContext;
+			break;
+		}
+	}
+
+	if (psServerMMUContext != NULL)
+	{
+		psInfo->uiPID = psServerMMUContext->uiPID;
+		OSStringSafeCopy(psInfo->szProcessName, psServerMMUContext->szProcessName, sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+	/* else check if the input PID corresponds to the firmware */
+	else if (uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		psInfo->uiPID = RGXMEM_SERVER_PID_FIRMWARE;
+		OSStringSafeCopy(psInfo->szProcessName, "Firmware", sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(bRet == IMG_FALSE))
+	{
+		/* if the PID didn't correspond to an active context or the
+		 * FW address then see if it matches a recently unregistered context
+		 */
+		const IMG_UINT32 ui32Mask = UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1;
+		IMG_UINT32 i, j;
+
+		OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+		for (i = (gui32UnregisteredMemCtxsHead - 1) & ui32Mask, j = 0;
+		     j < UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE;
+		     i = (i - 1) & ui32Mask, j++)
+		{
+			UNREGISTERED_MEMORY_CONTEXT *psRecord = &gasUnregisteredMemCtxs[i];
+
+			if (psRecord->uiPID == uiPID)
+			{
+				psInfo->uiPID = psRecord->uiPID;
+				OSStringSafeCopy(psInfo->szProcessName, psRecord->szProcessName, sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_TRUE;
+				bRet = IMG_TRUE;
+				break;
+			}
+		}
+
+		OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	return bRet;
+}
+
+IMG_PID RGXGetPIDFromServerMMUContext(SERVER_MMU_CONTEXT *psServerMMUContext)
+{
+	if (psServerMMUContext)
+	{
+		return psServerMMUContext->uiPID;
+	}
+	return 0;
+}
+
+/******************************************************************************
+ End of file (rgxmem.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.h
new file mode 100644
index 000000000000..2e959fc79eda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.h
@@ -0,0 +1,157 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX memory context management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for RGX memory context management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXMEM_H)
+#define RGXMEM_H
+
+#include "pvrsrv_error.h"
+#include "device.h"
+#include "mmu_common.h"
+#include "rgxdevice.h"
+
+#define RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME 16
+
+/* this PID denotes the firmware */
+#define RGXMEM_SERVER_PID_FIRMWARE 0xFFFFFFFF
+
+/* this PID denotes the PM */
+#define RGXMEM_SERVER_PID_PM 0xEFFFFFFF
+
+typedef struct _RGXMEM_PROCESS_INFO_
+{
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_BOOL bUnregistered;
+} RGXMEM_PROCESS_INFO;
+
+typedef struct SERVER_MMU_CONTEXT_TAG SERVER_MMU_CONTEXT;
+
+IMG_DEV_PHYADDR GetPC(MMU_CONTEXT * psContext);
+
+void RGXSetFWMemContextDevVirtAddr(SERVER_MMU_CONTEXT *psServerMMUContext,
+			RGXFWIF_DEV_VIRTADDR	sFWMemContextAddr);
+
+void RGXMMUSyncPrimAlloc(PVRSRV_DEVICE_NODE *psDevNode);
+void RGXMMUSyncPrimFree(void);
+
+PVRSRV_ERROR RGXInvalidateFBSCTable(PVRSRV_DEVICE_NODE *psDeviceNode,
+									MMU_CONTEXT *psMMUContext,
+									IMG_UINT64 ui64FBSCEntryMask);
+
+PVRSRV_ERROR RGXExtractFBSCEntryMaskFromMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+												   SERVER_MMU_CONTEXT *psServerMMUContext,
+												   IMG_UINT64 *pui64FBSCEntryMask);
+
+void RGXMMUCacheInvalidate(PVRSRV_DEVICE_NODE *psDevNode,
+						   MMU_CONTEXT *psMMUContext,
+						   MMU_LEVEL eMMULevel,
+						   IMG_BOOL bUnmap);
+
+/*************************************************************************/ /*!
+@Function       RGXMMUCacheInvalidateKick
+
+@Description    Sends a flush command to a particular DM but first takes
+                the power lock.
+
+@Input          psDevNode   Device Node pointer
+@Input          pui32NextMMUInvalidateUpdate
+
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXMMUCacheInvalidateKick(PVRSRV_DEVICE_NODE *psDevNode,
+                                       IMG_UINT32 *pui32NextMMUInvalidateUpdate);
+
+/*************************************************************************/ /*!
+@Function       RGXPreKickCacheCommand
+
+@Description    Sends a cache flush command to a particular DM without
+                honouring the power lock. It's the caller's responsibility
+                to ensure power lock is held before calling this function.
+
+@Input          psDevInfo   Device Info
+@Input          eDM			To which DM the cmd is sent.
+@Input          pui32MMUInvalidateUpdate
+
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPreKickCacheCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+									RGXFWIF_DM eDM,
+									IMG_UINT32 *pui32MMUInvalidateUpdate);
+
+/* Needed for Volcanic architectures with BRN71422 */
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+void RGXMapBRN71422TargetPhysicalAddress(struct _CONNECTION_DATA_ *psConnection,
+		                                 struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                                 IMG_DEV_PHYADDR sPhysAddrL1Px,
+		                                 void *pxL1PxCpuVAddr);
+#endif
+
+/* Needed for Rogue architecture where a MIPS FW CPU is used */
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXMMUTweakProtFlags(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                  MMU_DEVICEATTRIBS *psDevAttrs,
+		                  PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+		                  MMU_PROTFLAGS_T *uiMMUProtFlags);
+#endif
+
+void RGXUnregisterMemoryContext(IMG_HANDLE hPrivData);
+PVRSRV_ERROR RGXRegisterMemoryContext(PVRSRV_DEVICE_NODE	*psDevNode,
+									  MMU_CONTEXT			*psMMUContext,
+									  IMG_HANDLE			*hPrivData);
+
+DEVMEM_MEMDESC *RGXGetFWMemDescFromMemoryContextHandle(IMG_HANDLE hPriv);
+
+void RGXCheckFaultAddress(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_DEV_VIRTADDR *psDevVAddr,
+				IMG_DEV_PHYADDR *psDevPAddr,
+				MMU_FAULT_DATA *psOutFaultData);
+
+IMG_BOOL RGXPCAddrToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_PHYADDR sPCAddress,
+								RGXMEM_PROCESS_INFO *psInfo);
+
+IMG_BOOL RGXPCPIDToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_PID uiPID,
+                               RGXMEM_PROCESS_INFO *psInfo);
+
+IMG_PID RGXGetPIDFromServerMMUContext(SERVER_MMU_CONTEXT *psServerMMUContext);
+
+#endif /* RGXMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.c
new file mode 100644
index 000000000000..29049ad8c827
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.c
@@ -0,0 +1,1421 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+#include "rgxmmuinit.h"
+#include "rgxmmudefs_km.h"
+
+#include "rgxdevice.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "mmu_common.h"
+#include "pdump_mmu.h"
+
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "rgx_memallocflags.h"
+#include "rgx_heaps_server.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+
+/* useful macros */
+/* units represented in a bitfield */
+#define UNITS_IN_BITFIELD(Mask, Shift)	((Mask >> Shift) + 1)
+
+/*
+ * Bits of PT, PD and PC not involving addresses
+ */
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/* protection bits for MMU_VERSION <= 3 */
+#define RGX_MMUCTRL_PTE_PROTMASK	(RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN | \
+		~RGX_MMUCTRL_PT_DATA_AXCACHE_CLRMSK | \
+		RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PT_DATA_PM_SRC_EN | \
+		RGX_MMUCTRL_PT_DATA_CC_EN | \
+		RGX_MMUCTRL_PT_DATA_READ_ONLY_EN | \
+		RGX_MMUCTRL_PT_DATA_VALID_EN)
+#else
+#define RGX_MMUCTRL_PTE_PROTMASK	(RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN | \
+		RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PT_DATA_PM_SRC_EN | \
+		RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN | \
+		RGX_MMUCTRL_PT_DATA_CC_EN | \
+		RGX_MMUCTRL_PT_DATA_READ_ONLY_EN | \
+		RGX_MMUCTRL_PT_DATA_VALID_EN)
+#endif
+
+#define RGX_MMUCTRL_PDE_PROTMASK	(RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN | \
+		~RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK | \
+		RGX_MMUCTRL_PD_DATA_VALID_EN)
+
+#define RGX_MMUCTRL_PCE_PROTMASK	(RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PC_DATA_VALID_EN)
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*
+ * protection bits for MMU_VERSION >= 4
+ * MMU4 has no PENDING or PAGE_SIZE fields in PxE
+ */
+#define RGX_MMU4CTRL_PTE_PROTMASK	(RGX_MMUCTRL_PTE_PROTMASK & ~RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN)
+
+#define RGX_MMU4CTRL_PDE_PROTMASK	(RGX_MMUCTRL_PD_DATA_VALID_EN)
+
+#define RGX_MMU4CTRL_PCE_PROTMASK	(RGX_MMUCTRL_PC_DATA_VALID_EN)
+#endif
+
+
+static MMU_PxE_CONFIG sRGXMMUPCEConfig;
+static MMU_DEVVADDR_CONFIG sRGXMMUTopLevelDevVAddrConfig;
+
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_4KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_4KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_4KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig4KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_16KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_16KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_16KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig16KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_64KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_64KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_64KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig64KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_256KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_256KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig256KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 1MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_1MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_1MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_1MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig1MB;
+
+
+/*
+ *
+ *  Configuration for heaps with 2MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_2MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_2MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_2MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig2MB;
+
+
+/* Forward declaration of protection bits derivation functions, for
+   the following structure */
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags);
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/* protection bits derivation functions for MMUv4 */
+static IMG_UINT64 RGXMMU4DerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromVirtAddr(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												   IMG_DEV_VIRTADDR sDevVAddr,
+												   IMG_UINT32 *pui32Log2PageSize);
+#endif
+
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+		const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+		const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+		const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+		IMG_HANDLE *phPriv);
+
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv);
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+
+static MMU_DEVICEATTRIBS sRGXMMUDeviceAttributes;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+/* This is a basis for a pattern of parity bit values for consecutive VAs.
+   For each PT with 512 entries we'd get either this pattern or its reverse. */
+static const IMG_UINT8 ui8ParityPTPattern[512] = {
+    0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,
+    1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0,
+    0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1,
+    0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1,
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,
+    1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1,
+    0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1,
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0,
+    0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
+    0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1,
+    1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1};
+#endif
+
+PVRSRV_ERROR RGXMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK) || defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	IMG_BOOL bHaveParity = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MH_PARITY);
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	IMG_BOOL bHaveMMU4 = (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4);
+#define RGX_GET_MMUCTRL_PROTMASK(entry) \
+	(bHaveMMU4 ? RGX_MMU4CTRL_##entry##_PROTMASK : RGX_MMUCTRL_##entry##_PROTMASK)
+#else
+#define RGX_GET_MMUCTRL_PROTMASK(entry) RGX_MMUCTRL_##entry##_PROTMASK
+#endif
+
+	/* Setup of Px Entries:
+	 *
+	 *
+	 * PAGE TABLE (8 Byte):
+	 *
+	 * | 62              | 61...40         | 39...12 (varies) | 11...6          | 5             | 4      | 3               | 2               | 1         | 0     |
+	 * | PM/Meta protect | VP Page (39:18) | Physical Page    | VP Page (17:12) | Entry Pending | PM src | SLC Bypass Ctrl | Cache Coherency | Read Only | Valid |
+	 *
+	 *
+	 * PAGE DIRECTORY (8 Byte):
+	 *
+	 *  | 40            | 39...5  (varies)        | 4          | 3...1     | 0     |
+	 *  | Entry Pending | Page Table base address | (reserved) | Page Size | Valid |
+	 *
+	 *
+	 * PAGE CATALOGUE (4 Byte):
+	 *
+	 *  | 31...4                      | 3...2      | 1             | 0     |
+	 *  | Page Directory base address | (reserved) | Entry Pending | Valid |
+	 *
+	 */
+
+
+	/* Example how to get the PD address from a PC entry.
+	 * The procedure is the same for PD and PT entries to retrieve PT and Page addresses:
+	 *
+	 * 1) sRGXMMUPCEConfig.uiAddrMask applied to PC entry with '&':
+	 *  | 31...4   | 3...2      | 1             | 0     |
+	 *  | PD Addr  | 0          | 0             | 0     |
+	 *
+	 * 2) sRGXMMUPCEConfig.uiAddrShift applied with '>>':
+	 *  | 27...0   |
+	 *  | PD Addr  |
+	 *
+	 * 3) sRGXMMUPCEConfig.uiAddrLog2Align applied with '<<':
+	 *  | 39...0   |
+	 *  | PD Addr  |
+	 *
+	 */
+
+
+	sRGXMMUDeviceAttributes.pszMMUPxPDumpMemSpaceName =
+			PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+
+	/*
+	 * Setup sRGXMMUPCEConfig
+	 */
+	sRGXMMUPCEConfig.ePxLevel = MMU_LEVEL_3;
+	sRGXMMUPCEConfig.pszPxLevelStr = "PC";
+	sRGXMMUPCEConfig.uiBytesPerEntry = 4;     /* 32 bit entries */
+
+	sRGXMMUPCEConfig.uiAddrMask = 0xfffffff0; /* Mask to get significant address bits of PC entry i.e. the address of the PD */
+	sRGXMMUPCEConfig.uiAddrShift = 4;         /* Shift this many bits to get PD address */
+	sRGXMMUPCEConfig.uiAddrLog2Align = 12;    /* Alignment of PD physical addresses. */
+
+	sRGXMMUPCEConfig.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PCE); /* Mask to get the status bits */
+
+	sRGXMMUPCEConfig.uiProtShift = 0;                       /* Shift this many bits to get the status bits */
+
+	sRGXMMUPCEConfig.uiValidEnMask = RGX_MMUCTRL_PC_DATA_VALID_EN;     /* Mask to get entry valid bit of the PC */
+	sRGXMMUPCEConfig.uiValidEnShift = RGX_MMUCTRL_PC_DATA_VALID_SHIFT; /* Shift this many bits to get entry valid bit */
+
+	/*
+	 *  Setup sRGXMMUTopLevelDevVAddrConfig
+	 */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK; /* Mask to get PC index applied to a 40 bit virt. device address */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;  /* Shift a 40 bit virt. device address by this amount to get the PC index */
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask,
+			sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift));
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK; /* Mask to get PD index applied to a 40 bit virt. device address */
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;  /* Shift a 40 bit virt. device address by this amount to get the PD index */
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask,
+			sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift));
+
+	/*
+	 *
+	 *  Configuration for heaps with 4kB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20   12   11        0
+	 *      |         \ /       \ /       \ /           \
+	 *      |....PCE...|...PDE...|...PTE...|....VAddr...|
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_4KBDP
+	 */
+	sRGXMMUPDEConfig_4KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_4KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_4KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_4KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_4KBDP.uiAddrShift = 12;
+	sRGXMMUPDEConfig_4KBDP.uiAddrLog2Align = 12;
+
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_4KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_4KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_4KBDP
+	 */
+	sRGXMMUPTEConfig_4KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_4KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_4KBDP.uiAddrMask = IMG_UINT64_C(0xfffffff000);
+	sRGXMMUPTEConfig_4KBDP.uiAddrShift = 12;
+	sRGXMMUPTEConfig_4KBDP.uiAddrLog2Align = 12; /* Alignment of the physical addresses of the pages NOT PTs */
+
+	sRGXMMUPTEConfig_4KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_4KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_4KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_4KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_4KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_4KBDP
+	 */
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask = ~RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift = RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000000fff);
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig4KB
+	 */
+	gsPageSizeConfig4KB.psPDEConfig = &sRGXMMUPDEConfig_4KBDP;
+	gsPageSizeConfig4KB.psPTEConfig = &sRGXMMUPTEConfig_4KBDP;
+	gsPageSizeConfig4KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_4KBDP;
+	gsPageSizeConfig4KB.uiRefCount = 0;
+	gsPageSizeConfig4KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 16kB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20 14   13          0
+	 *      |         \ /       \ /     \ /             \
+	 *      |....PCE...|...PDE...|..PTE..|.....VAddr....|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_16KBDP
+	 */
+	sRGXMMUPDEConfig_16KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_16KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_16KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_16KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_16KBDP.uiAddrShift = 10;
+	sRGXMMUPDEConfig_16KBDP.uiAddrLog2Align = 10;
+
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_16KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_16KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_16KBDP
+	 */
+	sRGXMMUPTEConfig_16KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_16KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_16KBDP.uiAddrMask = IMG_UINT64_C(0xffffffc000);
+	sRGXMMUPTEConfig_16KBDP.uiAddrShift = 14;
+	sRGXMMUPTEConfig_16KBDP.uiAddrLog2Align = 14;
+
+	sRGXMMUPTEConfig_16KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_16KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_16KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_16KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_16KBDP
+	 */
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001fc000);
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift = 14;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift));
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000003fff);
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig16KB
+	 */
+	gsPageSizeConfig16KB.psPDEConfig = &sRGXMMUPDEConfig_16KBDP;
+	gsPageSizeConfig16KB.psPTEConfig = &sRGXMMUPTEConfig_16KBDP;
+	gsPageSizeConfig16KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_16KBDP;
+	gsPageSizeConfig16KB.uiRefCount = 0;
+	gsPageSizeConfig16KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 64kB Data-Page size
+	 *
+	 * Bit  39      30   29   21  20 16 15             0
+	 *      |         \ /       \ /   \ /               \
+	 *      |....PCE...|...PDE...|.PTE.|.....VAddr......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_64KBDP
+	 */
+	sRGXMMUPDEConfig_64KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_64KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_64KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_64KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_64KBDP.uiAddrShift = 8;
+	sRGXMMUPDEConfig_64KBDP.uiAddrLog2Align = 8;
+
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_64KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_64KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_64KBDP
+	 */
+	sRGXMMUPTEConfig_64KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_64KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_64KBDP.uiAddrMask = IMG_UINT64_C(0xffffff0000);
+	sRGXMMUPTEConfig_64KBDP.uiAddrShift = 16;
+	sRGXMMUPTEConfig_64KBDP.uiAddrLog2Align = 16;
+
+	sRGXMMUPTEConfig_64KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_64KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_64KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_64KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_64KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_64KBDP
+	 */
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001f0000);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift = 16;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000000ffff);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig64KB
+	 */
+	gsPageSizeConfig64KB.psPDEConfig = &sRGXMMUPDEConfig_64KBDP;
+	gsPageSizeConfig64KB.psPTEConfig = &sRGXMMUPTEConfig_64KBDP;
+	gsPageSizeConfig64KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_64KBDP;
+	gsPageSizeConfig64KB.uiRefCount = 0;
+	gsPageSizeConfig64KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 256kB Data-Page size
+	 *
+	 * Bit  39      30   29   21 20 18 17              0
+	 *      |         \ /       \|   |/                 \
+	 *      |....PCE...|...PDE...|PTE|.......VAddr......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_256KBDP
+	 */
+	sRGXMMUPDEConfig_256KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_256KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_256KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_256KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_256KBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_256KBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_256KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_256KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP
+	 */
+	sRGXMMUPTEConfig_256KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_256KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_256KBDP.uiAddrMask = IMG_UINT64_C(0xfffffc0000);
+	sRGXMMUPTEConfig_256KBDP.uiAddrShift = 18;
+	sRGXMMUPTEConfig_256KBDP.uiAddrLog2Align = 18;
+
+	sRGXMMUPTEConfig_256KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_256KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_256KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_256KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_256KBDP
+	 */
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001c0000);
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift = 18;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000003ffff);
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig256KB
+	 */
+	gsPageSizeConfig256KB.psPDEConfig = &sRGXMMUPDEConfig_256KBDP;
+	gsPageSizeConfig256KB.psPTEConfig = &sRGXMMUPTEConfig_256KBDP;
+	gsPageSizeConfig256KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_256KBDP;
+	gsPageSizeConfig256KB.uiRefCount = 0;
+	gsPageSizeConfig256KB.uiMaxRefCount = 0;
+
+	/*
+	 *
+	 *  Configuration for heaps with 1MB Data-Page size
+	 *
+	 * Bit  39      30   29   21 20  19               0
+	 *      |         \ /       \ | /                  \
+	 *      |....PCE...|...PDE...|.|........VAddr.......|
+	 *                           PTE
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_1MBDP
+	 */
+	sRGXMMUPDEConfig_1MBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_1MBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_1MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_1MBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	/*
+	 * The hardware requires that PT tables need be 1<<6 = 64 byte aligned even
+	 * if they contain fewer entries.
+	 */
+	sRGXMMUPDEConfig_1MBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_1MBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_1MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_1MBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_1MBDP
+	 */
+	sRGXMMUPTEConfig_1MBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_1MBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_1MBDP.uiAddrMask = IMG_UINT64_C(0xfffff00000);
+	sRGXMMUPTEConfig_1MBDP.uiAddrShift = 20;
+	sRGXMMUPTEConfig_1MBDP.uiAddrLog2Align = 20;
+
+	sRGXMMUPTEConfig_1MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_1MBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_1MBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_1MBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_1MBDP
+	 */
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask = IMG_UINT64_C(0x0000100000);
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift = 20;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetMask = IMG_UINT64_C(0x00000fffff);
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig1MB
+	 */
+	gsPageSizeConfig1MB.psPDEConfig = &sRGXMMUPDEConfig_1MBDP;
+	gsPageSizeConfig1MB.psPTEConfig = &sRGXMMUPTEConfig_1MBDP;
+	gsPageSizeConfig1MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_1MBDP;
+	gsPageSizeConfig1MB.uiRefCount = 0;
+	gsPageSizeConfig1MB.uiMaxRefCount = 0;
+
+	/*
+	 *
+	 *  Configuration for heaps with 2MB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20                0
+	 *      |         \ /       \ /                   \
+	 *      |....PCE...|...PDE...|.........VAddr.......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_2MBDP
+	 */
+	sRGXMMUPDEConfig_2MBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_2MBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_2MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_2MBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	/*
+	 * The hardware requires that PT tables need be 1<<6 = 64 byte aligned even
+	 * if they contain fewer entries.
+	 */
+	sRGXMMUPDEConfig_2MBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_2MBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_2MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_2MBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_2MBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_2MBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUPTEConfig_2MBDP
+	 */
+	sRGXMMUPTEConfig_2MBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_2MBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_2MBDP.uiAddrMask = IMG_UINT64_C(0xffffe00000);
+	sRGXMMUPTEConfig_2MBDP.uiAddrShift = 21;
+	sRGXMMUPTEConfig_2MBDP.uiAddrLog2Align = 21;
+
+	sRGXMMUPTEConfig_2MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_2MBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_2MBDP
+	 */
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask = IMG_UINT64_C(0x0000000000);
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift = 21;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetMask = IMG_UINT64_C(0x00001fffff);
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig2MB
+	 */
+	gsPageSizeConfig2MB.psPDEConfig = &sRGXMMUPDEConfig_2MBDP;
+	gsPageSizeConfig2MB.psPTEConfig = &sRGXMMUPTEConfig_2MBDP;
+	gsPageSizeConfig2MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_2MBDP;
+	gsPageSizeConfig2MB.uiRefCount = 0;
+	gsPageSizeConfig2MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUDeviceAttributes
+	 */
+	sRGXMMUDeviceAttributes.eMMUType = PDUMP_MMU_TYPE_VARPAGE_40BIT;
+	sRGXMMUDeviceAttributes.ui32BaseAlign = RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;
+	sRGXMMUDeviceAttributes.psBaseConfig = &sRGXMMUPCEConfig;
+	sRGXMMUDeviceAttributes.psTopLevelDevVAddrConfig = &sRGXMMUTopLevelDevVAddrConfig;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	{
+		IMG_UINT32 i;
+
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT   <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT  <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT  <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT   <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT   <= 512);
+
+		PVR_ASSERT(sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry   == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry  == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry  == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry   == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry   == 8);
+
+		sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[0] =
+		    OSAllocZMem(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT * sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry);
+		sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[1] =
+		    OSAllocZMem(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT * sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry);
+
+		/* Generate two precomputed pages in host memory for zero-initialised PTs with parity bits set. */
+		for (i=0; i<sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT; i++)
+		{
+			sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[0][i] =
+			    ((IMG_UINT64)(ui8ParityPTPattern[i]))         << RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+			sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[1][i] =
+			    ((IMG_UINT64)(ui8ParityPTPattern[i] ? 0 : 1)) << RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+		}
+	}
+#endif
+
+	/* Functions for deriving page table/dir/cat protection bits */
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt8 = RGXDerivePCEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt4 = RGXDerivePCEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXDerivePDEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt4 = RGXDerivePDEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt8 = RGXDerivePTEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt4 = RGXDerivePTEProt4;
+
+	/* Functions for establishing configurations for PDE/PTE/DEVVADDR
+	   on per-heap basis */
+	sRGXMMUDeviceAttributes.pfnGetPageSizeConfiguration = RGXGetPageSizeConfigCB;
+	sRGXMMUDeviceAttributes.pfnPutPageSizeConfiguration = RGXPutPageSizeConfigCB;
+
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE4 = RGXGetPageSizeFromPDE4;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXGetPageSizeFromPDE8;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromVirtAddr = NULL;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (bHaveMMU4)
+	{
+		/* override some of these functions for MMU4 as page size is not stored in PD entries */
+		sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXMMU4DerivePDEProt8;
+		sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXMMU4GetPageSizeFromPDE8;
+		sRGXMMUDeviceAttributes.pfnGetPageSizeFromVirtAddr = RGXMMU4GetPageSizeFromVirtAddr;
+	}
+#endif
+
+	sRGXMMUDeviceAttributes.ui32ValidPageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	psDeviceNode->psMMUDevAttrs = &sRGXMMUDeviceAttributes;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	psDeviceNode->pfnMMUGetContextID = NULL;
+#endif
+
+	psDeviceNode->psMMUDevAttrs = NULL;
+
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_MESSAGE, "Variable Page Size Heap Stats:"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 4K page heaps: %d",
+			gsPageSizeConfig4KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 4K page heaps (should be 0): %d",
+			gsPageSizeConfig4KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 16K page heaps: %d",
+			gsPageSizeConfig16KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 16K page heaps (should be 0): %d",
+			gsPageSizeConfig16KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 64K page heaps: %d",
+			gsPageSizeConfig64KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 64K page heaps (should be 0): %d",
+			gsPageSizeConfig64KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 256K page heaps: %d",
+			gsPageSizeConfig256KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 256K page heaps (should be 0): %d",
+			gsPageSizeConfig256KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 1M page heaps: %d",
+			gsPageSizeConfig1MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 1M page heaps (should be 0): %d",
+			gsPageSizeConfig1MB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 2M page heaps: %d",
+			gsPageSizeConfig2MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 2M page heaps (should be 0): %d",
+			gsPageSizeConfig2MB.uiRefCount));
+#endif
+	if (gsPageSizeConfig4KB.uiRefCount > 0 ||
+			gsPageSizeConfig16KB.uiRefCount > 0 ||
+			gsPageSizeConfig64KB.uiRefCount > 0 ||
+			gsPageSizeConfig256KB.uiRefCount > 0 ||
+			gsPageSizeConfig1MB.uiRefCount > 0 ||
+			gsPageSizeConfig2MB.uiRefCount > 0
+	)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXMMUInit_Unregister: Unbalanced MMU API Usage (Internal error)"));
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt4
+@Description    calculate the PCE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags)
+{
+	return (uiProtFlags & MMU_PROTFLAGS_INVALID)?0:RGX_MMUCTRL_PC_DATA_VALID_EN;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt8
+@Description    calculate the PCE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "8-byte PCE not supported on this device"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt4
+@Description    derive the PDE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PDE not supported on this device"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize The log2 of the required page size.
+                E.g, for 4KiB pages, this parameter must be 12.
+                For 2MiB pages, it must be set to 21.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ret_value = 0; /* 0 means invalid */
+
+	if (!(uiProtFlags & MMU_PROTFLAGS_INVALID)) /* if not invalid */
+	{
+		switch (uiLog2DataPageSize)
+		{
+		case RGX_HEAP_4KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB;
+			break;
+		case RGX_HEAP_16KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB;
+			break;
+		case RGX_HEAP_64KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB;
+			break;
+		case RGX_HEAP_256KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB;
+			break;
+		case RGX_HEAP_1MB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB;
+			break;
+		case RGX_HEAP_2MB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid parameter log2_page_size. Expected {12, 14, 16, 18, 20, 21}. Got [%u]",
+					__func__, uiLog2DataPageSize));
+		}
+	}
+	return ret_value;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt4
+@Description    calculate the PTE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PTE not supported on this device"));
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt8
+@Description    calculate the PTE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ui64MMUFlags=0;
+
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	if (((MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE) & uiProtFlags) == (MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE))
+	{
+		/* read/write */
+	}
+	else if (MMU_PROTFLAGS_READABLE & uiProtFlags)
+	{
+		/* read only */
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_READ_ONLY_EN;
+	}
+	else if (MMU_PROTFLAGS_WRITEABLE & uiProtFlags)
+	{
+		/* write only */
+		PVR_DPF((PVR_DBG_WARNING, "RGXDerivePTEProt8: write-only is not possible on this device"));
+	}
+	else if ((MMU_PROTFLAGS_INVALID & uiProtFlags) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt8: neither read nor write specified..."));
+	}
+
+	/* cache coherency */
+	if (MMU_PROTFLAGS_CACHE_COHERENT & uiProtFlags)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_CC_EN;
+	}
+
+#if defined(RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN)
+	/* cache setup */
+	if ((MMU_PROTFLAGS_CACHED & uiProtFlags) == 0)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN;
+	}
+#endif
+
+	if ((uiProtFlags & MMU_PROTFLAGS_INVALID) == 0)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_VALID_EN;
+	}
+
+	if (MMU_PROTFLAGS_DEVICE(PMMETA_PROTECT) & uiProtFlags)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN;
+	}
+
+#if defined(RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC)
+	/**
+	 * Always enable caching on the fabric level cache irrespective of type of
+	 * cache coherent interconnect and memory cache attributes.
+	 * This needs to be updated, if selective caching policy needs to be
+	 * implemented based on cache attributes requested by caller and based on
+	 * cache coherent interconnect.
+	 */
+	ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC;
+#endif
+
+	return ui64MMUFlags;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXGetPageSizeConfig
+@Description    Set up configuration for variable sized data pages.
+                RGXPutPageSizeConfigCB has to be called to ensure correct
+                refcounting.
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+		const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+		const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+		const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+		IMG_HANDLE *phPriv)
+{
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGX_HEAP_4KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	case RGX_HEAP_16KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig16KB;
+		break;
+	case RGX_HEAP_64KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGX_HEAP_256KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig256KB;
+		break;
+	case RGX_HEAP_1MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig1MB;
+		break;
+	case RGX_HEAP_2MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig2MB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXGetPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				uiLog2DataPageSize));
+		*phPriv = NULL;
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Refer caller's pointers to the data */
+	*ppsMMUPDEConfig = psPageSizeConfig->psPDEConfig;
+	*ppsMMUPTEConfig = psPageSizeConfig->psPTEConfig;
+	*ppsMMUDevVAddrConfig = psPageSizeConfig->psDevVAddrConfig;
+
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	/* Increment ref-count - not that we're allocating anything here
+       (I'm using static structs), but one day we might, so we want
+       the Get/Put code to be balanced properly */
+	psPageSizeConfig->uiRefCount++;
+
+	/* This is purely for debug statistics */
+	psPageSizeConfig->uiMaxRefCount = MAX(psPageSizeConfig->uiMaxRefCount,
+			psPageSizeConfig->uiRefCount);
+#endif
+
+	*phPriv = (IMG_HANDLE)(uintptr_t)uiLog2DataPageSize;
+	PVR_ASSERT (uiLog2DataPageSize == (IMG_UINT32)(uintptr_t)*phPriv);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPutPageSizeConfig
+@Description    Tells this code that the mmu module is done with the
+                configurations set in RGXGetPageSizeConfig.  This can
+                be a no-op.
+                Called after RGXGetPageSizeConfigCB.
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv)
+{
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+	IMG_UINT32 uiLog2DataPageSize;
+
+	uiLog2DataPageSize = (IMG_UINT32)(uintptr_t) hPriv;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGX_HEAP_4KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	case RGX_HEAP_16KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig16KB;
+		break;
+	case RGX_HEAP_64KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGX_HEAP_256KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig256KB;
+		break;
+	case RGX_HEAP_1MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig1MB;
+		break;
+	case RGX_HEAP_2MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig2MB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXPutPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				uiLog2DataPageSize));
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Ref-count here is not especially useful, but it's an extra
+	   check that the API is being used correctly */
+	psPageSizeConfig->uiRefCount--;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PDE not supported on this device"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	IMG_UINT64 ui64PageSizeBits = ui64PDE & (~RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK);
+
+	switch (ui64PageSizeBits)
+	{
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB:
+		*pui32Log2PageSize = RGX_HEAP_4KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB:
+		*pui32Log2PageSize = RGX_HEAP_16KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB:
+		*pui32Log2PageSize = RGX_HEAP_64KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB:
+		*pui32Log2PageSize = RGX_HEAP_256KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB:
+		*pui32Log2PageSize = RGX_HEAP_1MB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB:
+		*pui32Log2PageSize = RGX_HEAP_2MB_PAGE_SHIFT;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXGetPageSizeFromPDE8: Invalid page size bitfield %" IMG_UINT64_FMTSPECx " in PDE",
+				ui64PageSizeBits));
+
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*************************************************************************/ /*!
+@Function       RGXMMU4DerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize: ignored as MMU4 doesn't put page size in PD entries.
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXMMU4DerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ret_value = 0; /* 0 means invalid */
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	if (!(uiProtFlags & MMU_PROTFLAGS_INVALID)) /* if not invalid */
+	{
+		/*  page size in range config registers. Bits in PD entries are reserved */
+		ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	}
+	return ret_value;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXMMU4GetPageSizeFromPDE8
+@Description    The upper layers should be such that this function is never called
+                as pages size are not stored in PD entries for MMU4.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+
+	PVR_ASSERT(0 && "RGXMMU4GetPageSizeFromPDE8 called in error. MMU4 does not store page sizes in PDT.");
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXMMU4GetPageSizeFromVirtAddr
+@Description    Get page size by walking through range config registers
+                looking for a match against the virtual address.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromVirtAddr(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												   IMG_DEV_VIRTADDR sDevVAddr,
+												   IMG_UINT32 *pui32Log2PageSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+
+	/* assume default of 4KB page size */
+	*pui32Log2PageSize = 12;
+
+	/* Loop through the range registers looking for the given target address */
+	for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+	{
+		IMG_UINT64 ui64RangeVal = psDevInfo->aui64MMUPageSizeRangeValue[i];
+
+		if (ui64RangeVal != 0)
+		{
+			/* end addr in register is inclusive in the range so add 1 to move it over the end */
+			IMG_UINT64 ui64Base = ((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT;
+			IMG_UINT64 ui64End  = (((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_CLRMSK)
+												>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT) + 1)
+												<< RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT;
+
+			if ((sDevVAddr.uiAddr >= ui64Base) && (sDevVAddr.uiAddr < ui64End))
+			{
+				IMG_UINT32 ui32PageSizeField = (IMG_UINT32)((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_CLRMSK)
+														>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT);
+				if (ui32PageSizeField < 5)
+				{
+					*pui32Log2PageSize = (ui32PageSizeField << 1) + 12;   /* 12 (4K), 14 (16K), 16 (64K), 18 (256K), 20 (1MB) */
+				}
+				else if (ui32PageSizeField == 5)
+				{
+					*pui32Log2PageSize = 21;    /* 2MB */
+				}
+				else
+				{
+					eError = PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+				}
+				break;
+			}
+		}
+	}
+
+	return eError;
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.h
new file mode 100644
index 000000000000..b70ddad0cb64
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* NB: this file is not to be included arbitrarily.  It exists solely
+   for the linkage between rgxinit.c and rgxmmuinit.c, the former
+   being otherwise cluttered by the contents of the latter */
+
+#ifndef SRVKM_RGXMMUINIT_H
+#define SRVKM_RGXMMUINIT_H
+
+#include "device.h"
+#include "img_types.h"
+#include "mmu_common.h"
+#include "img_defs.h"
+
+PVRSRV_ERROR RGXMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR RGXMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* #ifndef SRVKM_RGXMMUINIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmulticore.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmulticore.h
new file mode 100644
index 000000000000..b45a20ab5e14
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmulticore.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.h
+@Title          Functions related to multicore devices
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose memory shared between kernel driver and user
+                mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXMULTICORE_H
+#define RGXMULTICORE_H
+
+#include "pvrsrv_error.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode);
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* RGXMULTICORE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump.h
new file mode 100644
index 000000000000..365269af59ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump.h
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX pdump Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX pdump functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxdevice.h"
+#include "device.h"
+#include "devicemem.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpSignatureBufferKM
+
+ @Description
+
+ Dumps TA and 3D signature and checksum buffers
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_UINT32 ui32PDumpFlags);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpCRCSignatureCheckKM
+
+ @Description
+
+ Poll on FBC/FBDC end-to-end signature status
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpCRCSignatureCheckKM(CONNECTION_DATA * psConnection,
+                                            PVRSRV_DEVICE_NODE * psDeviceNode,
+                                            IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpValCheckPreCommandKM
+
+ @Description
+
+ Poll on various GPU status/signature status for validation, before
+ sending the GPU command.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpValCheckPreCommandKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpValCheckPostCommandKM
+
+ @Description
+
+ Poll on various GPU status/signature status for validation, after
+ sending the GPU command.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpValCheckPostCommandKM(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE * psDeviceNode,
+                                              IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpTraceBufferKM
+
+ @Description
+
+ Dumps TA and 3D signature and checksum buffers
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpTraceBufferKM(CONNECTION_DATA * psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPDumpPrepareOutputImageDescriptorHdr
+
+ @Description
+
+ Dumps the header for an OutputImage command
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXPDumpPrepareOutputImageDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_PBYTE abyPDumpDesc);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPDumpPrepareOutputDataDescriptorHdr
+
+ @Description
+
+ Dumps the header for an OutputData command
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXPDumpPrepareOutputDataDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_PBYTE pbyPDumpDataHdr);
+
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVPDumpSignatureBufferKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVPDumpTraceBufferKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVPDumpTraceBufferKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+/******************************************************************************
+ End of file (rgxpdump.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.c
new file mode 100644
index 000000000000..fb1303c5f565
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.c
@@ -0,0 +1,121 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX common PDump functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxpdump_common.h"
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+
+PVRSRV_ERROR PDumpGetValidRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 DEVMEM_MEMDESC *psMemDesc,
+                                 IMG_UINT32 uiSize,
+                                 DLLIST_NODE *psRegionList)
+{
+	IMG_DEV_VIRTADDR sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset;
+
+	return DevmemIntPDumpGetValidRegions(
+		GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+		psDeviceNode,
+		psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+		sDevAddrStart,
+		uiSize,
+		psRegionList
+	);
+}
+
+void PDumpSaveToFileVirtual(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            DEVMEM_MEMDESC *psMemDesc,
+                            DLLIST_NODE *psRegionList,
+                            const IMG_CHAR *pszFileName,
+                            IMG_UINT32 uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DevmemIntPDumpSaveFromRegionListToFileVirtual(
+		GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+		psDeviceNode,
+		psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+		psRegionList,
+		pszFileName,
+		0,
+		uiPDumpFlags
+	);
+	PVR_LOG_IF_ERROR(eError, "DevmemIntPDumpSaveFromRegionListToFileVirtual");
+
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+}
+
+void PDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEVMEM_OFFSET_T uiOffset,
+                                      IMG_UINT32 uiSize,
+                                      const IMG_CHAR *pszFileName,
+                                      IMG_UINT32 uiFileOffset,
+                                      IMG_UINT32 uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset + uiOffset;
+
+	eError = DevmemIntPDumpSaveToFileVirtualNoValidate(
+		psDeviceNode,
+		psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+		sDevAddrStart,
+		uiSize,
+		pszFileName,
+		uiFileOffset,
+		uiPDumpFlags
+	);
+	PVR_LOG_IF_ERROR(eError, "DevmemIntPDumpSaveToFileVirtualNoValidate");
+
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.h
new file mode 100644
index 000000000000..8df9389a13ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.h
@@ -0,0 +1,106 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX common PDump functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXPDUMP_COMMON_H
+#define RGXPDUMP_COMMON_H
+
+#if defined(PDUMP)
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "devicemem_typedefs.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       PDumpGetValidRegion
+@Description    Checks valid regions within a given DEVMEM_MEMDESC and returns
+                a list containing region descriptors.
+@Input          psDeviceNode Pointer to the device node.
+@Input          psMemDesc    Pointer to the memory descriptor.
+@Input          uiSize       Size of the allocation.
+@Output         psRegionList List of valid regions.
+@Return         PVRSRV_ERROR PVRSRV_OK on success or error code on failure.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpGetValidRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 DEVMEM_MEMDESC *psMemDesc,
+                                 IMG_UINT32 uiSize,
+                                 DLLIST_NODE *psRegionList);
+
+/*************************************************************************/ /*!
+@Function       PDumpSaveToFileVirtual
+@Description    Emits SAB PDump command to file.
+@Input          psDeviceNode Pointer to the device node.
+@Input          psMemDesc    Pointer to the memory descriptor.
+@Input          psRegionList List of valid regions.
+@Input          pszFileName  Output file name.
+@Input          uiPDumpFlags PDump flags.
+*/ /**************************************************************************/
+void PDumpSaveToFileVirtual(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            DEVMEM_MEMDESC *psMemDesc,
+                            DLLIST_NODE *psRegionList,
+                            const IMG_CHAR *pszFileName,
+                            IMG_UINT32 uiPDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PDumpSaveToFileVirtualNoValidate
+@Description    Emits SAB PDump command to file.
+@Input          psDeviceNode Pointer to the device node.
+@Input          psMemDesc    Pointer to the memory descriptor.
+@Input          uiOffset     Offset within memory region.
+@Input          uiSize       Size of memory region.
+@Input          pszFileName  Output file name.
+@Input          uiFileOffset Offset within file.
+@Input          uiPDumpFlags PDump flags.
+*/ /**************************************************************************/
+void PDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEVMEM_OFFSET_T uiOffset,
+                                      IMG_UINT32 uiSize,
+                                      const IMG_CHAR *pszFileName,
+                                      IMG_UINT32 uiFileOffset,
+                                      IMG_UINT32 uiPDumpFlags);
+
+#endif /* PDUMP */
+
+#endif /* RGXPDUMP_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.c
new file mode 100644
index 000000000000..e96091ec8b23
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.c
@@ -0,0 +1,146 @@
+/*************************************************************************/ /*!
+@File           rgxpdvfs.c
+@Title          RGX Proactive DVFS Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode Proactive DVFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxpdvfs.h"
+#include "rgxfwutils.h"
+#include "rgxpower.h"
+#include "rgx_options.h"
+#include "rgxtimecorr.h"
+
+#define USEC_TO_MSEC 1000
+
+PVRSRV_ERROR PDVFSLimitMaxFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MaxOPPPoint)
+{
+	RGXFWIF_KCCB_CMD		sGPCCBCmd;
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CmdKCCBSlot;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	/* send feedback */
+	sGPCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ;
+	sGPCCBCmd.uCmdData.sPDVFSMaxFreqData.ui32MaxOPPPoint = ui32MaxOPPPoint;
+
+	/* Submit command to the firmware.  */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+		                                      &sGPCCBCmd,
+		                                      PDUMP_FLAGS_CONTINUOUS,
+		                                      &ui32CmdKCCBSlot);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR PDVFSLimitMinFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MinOPPPoint)
+{
+	RGXFWIF_KCCB_CMD		sGPCCBCmd;
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CmdKCCBSlot;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	/* send feedback */
+	sGPCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ;
+	sGPCCBCmd.uCmdData.sPDVFSMinFreqData.ui32MinOPPPoint = ui32MinOPPPoint;
+
+	/* Submit command to the firmware.  */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+		                                      &sGPCCBCmd,
+		                                      PDUMP_FLAGS_CONTINUOUS,
+		                                      &ui32CmdKCCBSlot);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+/*************************************************************************/ /*!
+@Function       RGXPDVFSCheckCoreClkRateChange
+@Description    Checks if core clock rate has changed since the last snap-shot.
+@Input          psDevInfo    A pointer to PVRSRV_RGXDEV_INFO.
+@Return         None.
+*/ /**************************************************************************/
+void RGXPDVFSCheckCoreClkRateChange(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 ui32CoreClkRate = *psDevInfo->pui32RGXFWIFCoreClkRate;
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return;
+	}
+
+	if (ui32CoreClkRate != 0 && psDevInfo->ui32CoreClkRateSnapshot != ui32CoreClkRate)
+	{
+		psDevInfo->ui32CoreClkRateSnapshot = ui32CoreClkRate;
+		RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo, ui32CoreClkRate);
+	}
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.h
new file mode 100644
index 000000000000..f2bec1184077
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           rgxpdvfs.h
+@Title          RGX Proactive DVFS Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the kernel mode Proactive DVFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXPDVFS_H
+#define RGXPDVFS_H
+
+#include "img_types.h"
+#include "rgxdevice.h"
+#include "rgx_options.h"
+#include "pvrsrv.h"
+
+
+static inline IMG_BOOL _PDVFSEnabled(void)
+{
+	PVRSRV_DATA *psSRVData = PVRSRVGetPVRSRVData();
+
+	if (psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions &
+	    psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions &
+	    OPTIONS_PDVFS_EN)
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR PDVFSLimitMaxFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MaxOPPPoint);
+
+PVRSRV_ERROR PDVFSLimitMinFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MinOPPPoint);
+
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+void RGXPDVFSCheckCoreClkRateChange(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#endif /* RGXPDVFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.c
new file mode 100644
index 000000000000..7f77edefe558
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.c
@@ -0,0 +1,1853 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific power routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "rgxpower.h"
+#include "rgxinit.h"
+#include "rgx_fwif_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "rgxdebug_common.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxtimecorr.h"
+#include "devicemem_utils.h"
+#include "htbserver.h"
+#include "rgxstartstop.h"
+#include "rgxfwimageutils.h"
+#include "sync.h"
+#include "rgxdefs_km.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+#include "pvr_dvfs_device.h"
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_FREQ)
+#include "pvr_gpufreq.h"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_FREQ) */
+
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+#include "rgxpdvfs.h"
+#endif
+
+static PVRSRV_ERROR RGXFWNotifyHostTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_KCCB_CMD sCmd;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CmdKCCBSlot;
+
+	/* Send the Timeout notification to the FW */
+	sCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_HOST_TIMEOUT;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	return eError;
+}
+
+static void _RGXUpdateGPUUtilStats(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_GPU_UTIL_FW *psUtilFW;
+	IMG_UINT64 ui64LastPeriod;
+	IMG_UINT64 ui64LastState;
+	IMG_UINT64 ui64LastReducedState;
+	IMG_UINT64 ui64LastTime;
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT64 ui64DMOSStatsCounter;
+
+	psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+	ui64TimeNow = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDevInfo->psDeviceNode));
+
+	/* Update counters to account for the time since the last update */
+	ui64LastState  = RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord);
+	ui64LastTime   = RGXFWIF_GPU_UTIL_GET_TIME(psUtilFW->ui64GpuLastWord);
+	ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
+	psUtilFW->aui64GpuStatsCounters[ui64LastState] += ui64LastPeriod;
+
+	/* Update state and time of the latest update */
+	psUtilFW->ui64GpuLastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(ui64TimeNow, ui64LastState);
+
+	/* convert last period into the same units as used by fw */
+	ui64TimeNow  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+
+	FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+	{
+		RGXFWIF_GPU_STATS *psStats = &psUtilFW->sStats[ui32DriverID];
+		RGXFWIF_DM eDM;
+
+		for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+		{
+			ui64LastState  = (IMG_UINT64)RGXFWIF_GPU_UTIL_GET_STATE32(psStats->aui32DMOSLastWord[eDM]);
+			ui64LastTime   = (IMG_UINT64)RGXFWIF_GPU_UTIL_GET_TIME32(psStats->aui32DMOSLastWord[eDM]) +
+			                 ((IMG_UINT64)psStats->aui32DMOSLastWordWrap[eDM] << 32);
+			ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
+			/* for states statistics per DM per driver we only care about the time in Active state,
+			so we "combine" other states (Idle and Blocked) together */
+			ui64LastReducedState = (ui64LastState == RGXFWIF_GPU_UTIL_STATE_ACTIVE) ?
+			                       RGXFWIF_GPU_UTIL_STATE_ACTIVE : RGXFWIF_GPU_UTIL_STATE_INACTIVE;
+			ui64DMOSStatsCounter = (IMG_UINT64)psStats->aaui32DMOSStatsCounters[eDM][ui64LastReducedState] + ui64LastPeriod;
+			psStats->aaui32DMOSStatsCounters[eDM][ui64LastReducedState] = (IMG_UINT32)(ui64DMOSStatsCounter & IMG_UINT32_MAX);
+			if (ui64DMOSStatsCounter > IMG_UINT32_MAX)
+			{
+				psStats->aaui32DMOSCountersWrap[eDM][ui64LastReducedState] += (IMG_UINT32)(ui64DMOSStatsCounter >> 32);
+			}
+
+			/* Update state and time of the latest update */
+			psStats->aui32DMOSLastWord[eDM] = RGXFWIF_GPU_UTIL_MAKE_WORD32((ui64TimeNow & (IMG_UINT64)IMG_UINT32_MAX), ui64LastState);
+			if (ui64TimeNow > IMG_UINT32_MAX)
+			{
+				if (psStats->aui32DMOSLastWordWrap[eDM] != (IMG_UINT32)(ui64TimeNow >> 32))
+				{
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64TimeNow >> 32);
+				}
+			}
+		}
+	}
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+
+	OSLockRelease(psDevInfo->hGPUUtilLock);
+}
+
+static INLINE PVRSRV_ERROR RGXDoStop(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDeviceNode->psDevConfig->pfnTDRGXStop == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPrePowerState: TDRGXStop not implemented!"));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	psDevInfo->bRGXPowered = IMG_FALSE;
+	eError = psDeviceNode->psDevConfig->pfnTDRGXStop(psDeviceNode->psDevConfig->hSysData);
+#else
+	eError = RGXStop(&psDevInfo->sLayerParams);
+#endif
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSendPowerOffKick
+@Description    Send a KCCB kick to power off the GPU FW. This function will wait
+                for completion of the command before exiting.
+
+@Input          psDeviceNode    The device node struct associated with the GPU.
+@Input          bForce          A boolean indicating if the power off command
+                                should be forced.
+
+@Return         Failure code if the virtual address is invalid.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXSendPowerOffKick(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_BOOL bForce)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_KCCB_CMD sPowCmd;
+	IMG_UINT32 ui32CmdKCCBSlot;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Send the Power off request to the FW */
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_OFF_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.bForced = bForce;
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+		         __func__));
+		return eError;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send Power off request",
+		         __func__));
+		return eError;
+	}
+
+	/* Wait for the firmware to complete processing. It cannot use PVRSRVWaitForValueKM as it relies
+	 * on the EventObject which is signalled in this MISR */
+	return RGXPollForGPCommandCompletion(psDeviceNode,
+	                                     psDevInfo->psPowSyncPrim->pui32LinAddr,
+	                                     0x1, 0xFFFFFFFF);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXFinalisePowerOff
+@Description    Finalises the GPU power transition to off.
+
+@Input          psDeviceNode    The device node struct associated with the GPU.
+
+@Return         Result code indicating the success or reason for failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXFinalisePowerOff(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_SYNC_IRQ)
+	IMG_UINT32 ui32idx;
+#endif
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if !defined(NO_HARDWARE)
+	/* Driver takes the VZ Fw-KM connection down, preventing the
+		* firmware from submitting further interrupts */
+	KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+	KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+#if defined(SUPPORT_SYNC_IRQ)
+	/* Wait for the pending IRQ handlers to complete. */
+	OSSyncIRQ(psDeviceNode->psDevConfig->ui32IRQ);
+#else
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	ui32idx = RGXFW_HOST_DRIVER_ID;
+#else
+	for_each_irq_cnt(ui32idx)
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+	{
+		IMG_UINT32 ui32IrqCnt;
+
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+
+		/* Wait for the pending FW processor to host interrupts to come back. */
+		eError = PVRSRVPollForValueKM(psDeviceNode,
+		                              (IMG_UINT32 __iomem *)&psDevInfo->aui32SampleIRQCount[ui32idx],
+		                              ui32IrqCnt,
+		                              0xffffffff,
+		                              POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+		                              NULL);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Wait for pending interrupts failed (DevID %u)." MSG_IRQ_CNT_TYPE " %u Host: %u, FW: %u",
+			         __func__,
+			         psDeviceNode->sDevId.ui32InternalID,
+			         ui32idx,
+			         psDevInfo->aui32SampleIRQCount[ui32idx],
+			         ui32IrqCnt));
+
+			RGX_WaitForInterruptsTimeout(psDevInfo);
+#if !defined(RGX_FW_IRQ_OS_COUNTERS)
+			break;
+#endif
+		}
+	}
+#endif /* SUPPORT_SYNC_IRQ */
+#endif /* NO_HARDWARE */
+
+	/* Update GPU frequency and timer correlation related data */
+	RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_POWER);
+
+	/* Update GPU state counters */
+	_RGXUpdateGPUUtilStats(psDevInfo);
+
+#if defined(SUPPORT_LINUX_DVFS)
+	eError = SuspendDVFS(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to suspend DVFS", __func__));
+		return eError;
+	}
+#endif
+
+	eError = RGXDoStop(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		/* Power down failures are treated as successful since the power was removed but logged. */
+		PVR_DPF((PVR_DBG_WARNING, "%s: RGXDoStop failed (%s)",
+		         __func__, PVRSRVGetErrorString(eError)));
+		psDevInfo->ui32ActivePMReqNonIdle++;
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPrePowerState
+@Description    Initial step for setting power state, to be followed by
+                RGXPostPowerState.
+
+@Input          psDeviceNode        The device node struct associated with the GPU.
+@Input          eNewPowerState      The power state the GPU is to transition to.
+@Input          eCurrentPowerState  The current power state of the GPU.
+@Input          ePwrFlags           Flags indicating the behaviour of the transition.
+
+@Return         Result code indicating the success or reason for failure.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPrePowerState(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              PVRSRV_DEV_POWER_STATE eNewPowerState,
+                              PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL              bDeviceOk;
+#endif
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psDevInfo != NULL);
+
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if ((eNewPowerState == eCurrentPowerState) ||
+	    (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON))
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	bDeviceOk = (OSAtomicRead(&psDeviceNode->eHealthStatus) == PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	if (bDeviceOk)
+#endif
+	{
+		IMG_BOOL bForce = IMG_FALSE;
+
+		if (BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED))
+		{
+			bForce = IMG_TRUE;
+		}
+
+		eError = RGXSendPowerOffKick(psDeviceNode, bForce);
+		if (eError == PVRSRV_ERROR_TIMEOUT)
+		{
+			/* timeout waiting for the FW to ack the request: return timeout */
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: Timeout waiting for powoff ack from the FW",
+			         __func__));
+			return eError;
+		}
+		else if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error waiting for powoff ack from the FW (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+			return PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
+		}
+	}
+
+	/* Check the Power state after the answer */
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState, INVALIDATE);
+	if ((psFwSysData->ePowState != RGXFWIF_POW_OFF)
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	    && (bDeviceOk)
+#endif
+		)
+	{
+		if (BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED))
+		{	/* It is an error for a forced request to be denied */
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failure to power off during a forced power off. FW: %d",
+			         __func__, psFwSysData->ePowState));
+		}
+
+		/* the sync was updated but the pow state isn't off -> the FW denied the transition */
+		return PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
+	}
+
+	return RGXFinalisePowerOff(psDeviceNode);
+}
+
+#if defined(SUPPORT_AUTOVZ)
+static PVRSRV_ERROR _RGXWaitForGuestsToDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TIMEOUT;
+	IMG_UINT32 ui32FwTimeout = (20 * SECONDS_TO_MICROSECONDS);
+
+	LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+	{
+		IMG_UINT32 ui32DriverID;
+		IMG_BOOL bGuestOnline = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+		                           INVALIDATE);
+
+		for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+			 ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED; ui32DriverID++)
+		{
+			RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+					psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+			if ((eGuestState == RGXFW_CONNECTION_FW_ACTIVE) ||
+				(eGuestState == RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD) ||
+				(eGuestState == RGXFW_CONNECTION_FW_FORCED_OFFLOAD))
+			{
+				bGuestOnline = IMG_TRUE;
+				PVR_DPF((PVR_DBG_WARNING, "%s: Guest OS %u still online.", __func__, ui32DriverID));
+			}
+		}
+
+		if (!bGuestOnline)
+		{
+			/* Allow Guests to finish reading Connection state registers before disconnecting. */
+			OSSleepms(100);
+
+			PVR_DPF((PVR_DBG_WARNING, "%s: All Guest connections are down. "
+									  "Host can power down the GPU.", __func__));
+			eError = PVRSRV_OK;
+			break;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Waiting for Guests to disconnect "
+									  "before powering down GPU.", __func__));
+
+			if (PVRSRVPwrLockIsLockedByMe(psDeviceNode))
+			{
+				/* Don't wait with the power lock held as this prevents the vz
+				 * watchdog thread from keeping the fw-km connection alive. */
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+		}
+
+		OSSleepms(10);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (!PVRSRVPwrLockIsLockedByMe(psDeviceNode))
+	{
+		/* Take back power lock after waiting for Guests */
+		eError = PVRSRVPowerLock(psDeviceNode);
+	}
+
+	return eError;
+}
+#endif /* defined(SUPPORT_AUTOVZ) */
+
+/*
+	RGXVzPrePowerState
+*/
+PVRSRV_ERROR RGXVzPrePowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                                PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                                PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", eError);
+
+	if (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering down */
+#if defined(SUPPORT_AUTOVZ)
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode) && (!psDeviceNode->bAutoVzFwIsUp || psDeviceNode->bAutoVzAllowGPUPowerdown))
+		{
+			if (psDeviceNode->bAutoVzFwIsUp)
+			{
+				/* bAutoVzAllowGPUPowerdown must be TRUE here and
+				 * bAutoVzFwIsUp=TRUE indicates that this is a powerdown event
+				 * so send requests to the FW to disconnect all guest connections
+				 * before GPU is powered down. */
+				eError = RGXDisconnectAllGuests(psDeviceNode);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXDisconnectAllGuests");
+			}
+
+			/* The Host must ensure all Guest drivers have disconnected from the GPU before powering it down.
+			 * Guest drivers regularly access hardware registers during runtime. If an attempt is made to
+			 * access a GPU register while the GPU is down, the SoC might lock up. */
+			eError = _RGXWaitForGuestsToDisconnect(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "_RGXWaitForGuestsToDisconnect");
+
+			if (psDeviceNode->bAutoVzAllowGPUPowerdown)
+			{
+				psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+			}
+
+			/* Temporarily restore all power callbacks used by the driver to fully power down the GPU.
+			 * Under AutoVz, power transitions requests (e.g. on driver deinitialisation and unloading)
+			 * are generally ignored and the GPU power state is unaffected. Special power requests like
+			 * those triggered by Suspend/Resume calls must reinstate the callbacks when needed. */
+			PVRSRVSetPowerCallbacks(psDeviceNode, psDeviceNode->psPowerDev,
+									&RGXVzPrePowerState, &RGXVzPostPowerState,
+									psDeviceNode->psDevConfig->pfnPrePowerState,
+									psDeviceNode->psDevConfig->pfnPostPowerState,
+									&RGXForcedIdleRequest, &RGXCancelForcedIdleRequest,
+									&RGXCancelForcedIdleRequestAsync);
+		}
+		else
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+			KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+			KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+			if (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+				KM_OS_CONNECTION_IS(ACTIVE, psDevInfo))
+			{
+				PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, 0, RGXFWIF_OS_OFFLINE);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+			}
+		}
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering down: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+	}
+	else if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering up */
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering up: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+
+	}
+
+	if (!(PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp)))
+	{
+		/* call regular device power function */
+		eError = RGXPrePowerState(psDeviceNode, eNewPowerState, eCurrentPowerState, ePwrFlags);
+	}
+
+	return eError;
+}
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+static PVRSRV_ERROR RGXVzWaitFirmwareReady(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	if (!KM_FW_CONNECTION_IS(READY, psDevInfo))
+	{
+		PVR_LOG(("%s: Firmware Connection is not in Ready state. Waiting for Firmware ...", __func__));
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(RGX_VZ_CONNECTION_TIMEOUT_US)
+	{
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+		{
+			PVR_LOG(("%s: Firmware Connection is Ready. Initialisation proceeding.", __func__));
+			break;
+		}
+		else
+		{
+			OSSleepms(10);
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	if (!KM_FW_CONNECTION_IS(READY, psDevInfo))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timed out waiting for the Firmware to enter Ready state.", __func__));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXVzPostPowerState
+*/
+PVRSRV_ERROR RGXVzPostPowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                                 PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                                 PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", eError);
+
+	if (!(PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp)))
+	{
+		if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+		{
+			KM_SET_OS_CONNECTION(READY, psDevInfo);
+			KM_CONNECTION_CACHEOP(Os, FLUSH);
+		}
+		/* call regular device power function */
+		eError = RGXPostPowerState(psDeviceNode, eNewPowerState, eCurrentPowerState, ePwrFlags);
+	}
+	else
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	if (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering down */
+		if (psDeviceNode->bAutoVzFwIsUp)
+		{
+			PVR_LOG(("%s: AutoVz Fw active, power not changed", __func__));
+			return eError;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering down: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+
+#if !defined(SUPPORT_AUTOVZ_HW_REGS)
+		/* The connection states must be reset on a GPU power cycle. If the states are kept
+		 * in hardware scratch registers, they will be cleared on power down. When using shared
+		 * memory the connection data must be explicitly cleared by the driver. */
+		OSCachedMemSetWMB(psDevInfo->psRGXFWIfConnectionCtl, 0, sizeof(RGXFWIF_CONNECTION_CTL));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfConnectionCtl, FLUSH);
+#endif /* defined(SUPPORT_AUTOVZ) && !defined(SUPPORT_AUTOVZ_HW_REGS) */
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+		{
+#if defined(SUPPORT_AUTOVZ)
+			/* AutoVz Guests attempting to suspend have updated their connections earlier in RGXVzPrePowerState.
+			 * Skip this redundant register write, as the Host could have powered down the GPU by now. */
+			if (psDeviceNode->bAutoVzFwIsUp)
+#endif
+			{
+				/* Take the VZ connection down to prevent firmware from submitting further interrupts */
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+			/* Power transition callbacks were not executed, update RGXPowered flag here */
+			psDevInfo->bRGXPowered = IMG_FALSE;
+		}
+	}
+	else if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering up */
+		IMG_UINT32 ui32FwTimeout = (3 * SECONDS_TO_MICROSECONDS);
+		volatile IMG_BOOL *pbUpdatedFlag;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                           INVALIDATE);
+		pbUpdatedFlag = &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated;
+
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering up: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Guests don't execute the power transition callbacks, so update their RGXPowered flag here */
+			psDevInfo->bRGXPowered = IMG_TRUE;
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+			/* Guest drivers expect the firmware to have set its end of the
+			 * connection to Ready state by now. */
+			eError = RGXVzWaitFirmwareReady(psDevInfo);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXVzWaitFirmwareReady()");
+#endif /* RGX_VZ_STATIC_CARVEOUT_FW_HEAPS */
+
+			OSWriteDeviceMem32WithWMB(pbUpdatedFlag, IMG_FALSE);
+
+			/* Kick an initial dummy command to make the firmware initialise all
+			 * its internal guest OS data structures and compatibility information.
+			 * Use the lower-level RGXSendCommand() for the job, to make
+			 * sure only 1 KCCB command is issued to the firmware.
+			 * The default RGXFWHealthCheckCmd() prefaces each HealthCheck command with
+			 * a pre-kick cache command which can interfere with the FW-KM init handshake. */
+			{
+				RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+				sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+				KM_SET_OS_CONNECTION(READY, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+				eError = RGXSendCommand(psDevInfo, &sCmpKCCBCmd, PDUMP_FLAGS_CONTINUOUS);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXSendCommand()");
+			}
+		}
+		else
+		{
+#if defined(SUPPORT_AUTOVZ)
+			/* Disable power callbacks that should not be run on virtualised drivers after the GPU
+			 * is fully initialised: system layer pre/post functions and driver idle requests.
+			 * The original device RGX Pre/Post functions are called from this Vz wrapper. */
+			PVRSRVSetPowerCallbacks(psDeviceNode, psDeviceNode->psPowerDev,
+									&RGXVzPrePowerState, &RGXVzPostPowerState,
+									NULL, NULL, NULL, NULL, NULL);
+
+			/* AutoVz Host driver reconnecting to running Firmware */
+			if (psDeviceNode->bAutoVzFwIsUp)
+			{
+				/* Firmware already running, send a KCCB command to establish the new connection */
+				RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+				sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+				eError = RGXVzWaitFirmwareReady(psDevInfo);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXVzWaitFirmwareReady()");
+
+				KM_SET_OS_CONNECTION(READY, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+				eError = RGXSendCommand(psDevInfo, &sCmpKCCBCmd, PDUMP_FLAGS_CONTINUOUS);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXSendCommand()");
+			}
+			else
+			{
+				/* During first-time boot the flag is set here, while subsequent reboots will already
+				 * have set it earlier in RGXInit. Set to true from this point on. */
+				psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+			}
+#endif
+		}
+
+		/* Wait for the firmware to accept and enable the connection with this OS by setting its state to Active */
+		LOOP_UNTIL_TIMEOUT_US(RGX_VZ_CONNECTION_TIMEOUT_US)
+		{
+			KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+			if (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo))
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Connection is Active. Initialisation proceeding.", __func__));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Connection is not in Active state. Waiting for Firmware ...", __func__));
+				OSSleepms(10);
+			}
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		if (!KM_FW_CONNECTION_IS(ACTIVE, psDevInfo))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Timed out waiting for the Firmware to enter Active state.", __func__));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* poll on the Firmware supplying the compatibility data */
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				if (*pbUpdatedFlag)
+				{
+					break;
+				}
+				OSSleepms(10);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			PVR_LOG_RETURN_IF_FALSE(*pbUpdatedFlag, "Firmware does not respond with compatibility data.", PVRSRV_ERROR_TIMEOUT);
+		}
+
+		KM_SET_OS_CONNECTION(ACTIVE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(TRACK_FW_BOOT)
+static INLINE void RGXCheckFWBootStage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	FW_BOOT_STAGE eStage;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		/* Boot stage temporarily stored to the register below */
+		eStage = OSReadHWReg32(psDevInfo->pvRegsBaseKM,
+		                       RGX_FW_BOOT_STAGE_REGISTER);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		IMG_BYTE *pbBootData;
+
+		if (PVRSRV_OK != DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+		                                          (void**)&pbBootData))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Could not acquire pointer to FW boot stage", __func__));
+			eStage = FW_BOOT_STAGE_NOT_AVAILABLE;
+		}
+		else
+		{
+			pbBootData += RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+
+			eStage = *(FW_BOOT_STAGE*)&pbBootData[RGXMIPSFW_BOOT_STAGE_OFFSET];
+
+			if (eStage == FW_BOOT_STAGE_TLB_INIT_FAILURE)
+			{
+				RGXMIPSFW_BOOT_DATA *psBootData =
+					(RGXMIPSFW_BOOT_DATA*) (pbBootData + RGXMIPSFW_BOOTLDR_CONF_OFFSET);
+
+				PVR_LOG(("MIPS TLB could not be initialised. Boot data info:"
+						 " num PT pages %u, log2 PT page size %u, PT page addresses"
+						 " %"IMG_UINT64_FMTSPECx " %"IMG_UINT64_FMTSPECx
+						 " %"IMG_UINT64_FMTSPECx " %"IMG_UINT64_FMTSPECx,
+						 psBootData->ui32PTNumPages,
+						 psBootData->ui32PTLog2PageSize,
+						 psBootData->aui64PTPhyAddr[0U],
+						 psBootData->aui64PTPhyAddr[1U],
+						 psBootData->aui64PTPhyAddr[2U],
+						 psBootData->aui64PTPhyAddr[3U]));
+			}
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		}
+	}
+#endif
+	else
+	{
+		eStage = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SCRATCH14);
+	}
+
+	PVR_LOG(("%s: FW reached boot stage %i/%i.",
+	         __func__, eStage, FW_BOOT_INIT_DONE));
+}
+#endif
+
+static INLINE PVRSRV_ERROR RGXDoStart(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+
+	if (psDevConfig->pfnTDRGXStart == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPostPowerState: TDRGXStart not implemented!"));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	eError = psDevConfig->pfnTDRGXStart(psDevConfig->hSysData);
+
+	if (eError == PVRSRV_OK)
+	{
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+#else
+	eError = RGXStart(&psDevInfo->sLayerParams);
+#endif
+
+	return eError;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+/*
+ * To validate the MTS unit we do the following:
+ *  - Immediately after firmware loading for each OSID
+ *    - Write the OSid to a memory location shared with FW
+ *    - Kick the register of that OSid
+ *         (Uncounted, DM 0)
+ *    - FW clears the memory location if OSid matches
+ *    - Host checks that memory location is cleared
+ *
+ *  See firmware/rgxfw_bg.c
+ */
+static PVRSRV_ERROR RGXVirtualisationPowerupSidebandTest(PVRSRV_DEVICE_NODE	 *psDeviceNode,
+														 RGXFWIF_SYSINIT *psFwSysInit,
+														 PVRSRV_RGXDEV_INFO	 *psDevInfo)
+{
+	IMG_UINT32 ui32ScheduleRegister;
+	IMG_UINT32 ui32OSid;
+	IMG_UINT32 ui32KickType;
+	IMG_UINT32 ui32OsRegBanksMapped = (psDeviceNode->psDevConfig->ui32RegsSize / RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+
+	/* Nothing to do if the device does not support GPU_VIRTUALISATION */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, GPU_VIRTUALISATION))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Testing per-os kick registers:"));
+
+	ui32OsRegBanksMapped = MIN(ui32OsRegBanksMapped, GPUVIRT_VALIDATION_NUM_OS);
+
+	if (ui32OsRegBanksMapped != RGXFW_MAX_NUM_OSIDS)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "The register bank mapped into kernel VA does not cover all OS' registers:"));
+		PVR_DPF((PVR_DBG_WARNING, "Maximum OS count = %d / Per-os register banks mapped = %d", RGXFW_MAX_NUM_OSIDS, ui32OsRegBanksMapped));
+		PVR_DPF((PVR_DBG_WARNING, "Only first %d MTS registers will be tested", ui32OsRegBanksMapped));
+	}
+
+	ui32KickType = RGX_CR_MTS_SCHEDULE_DM_DM0 | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED;
+
+	for (ui32OSid = 0; ui32OSid < ui32OsRegBanksMapped; ui32OSid++)
+	{
+		/* set Test field */
+		psFwSysInit->ui32OSKickTest = (ui32OSid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT;
+
+#if defined(PDUMP)
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, ui32OSKickTest),
+						   sizeof(psFwSysInit->ui32OSKickTest),
+						   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+		/* Force a read-back to memory to avoid posted writes on certain buses */
+		OSWriteMemoryBarrier(&psFwSysInit->ui32OSKickTest);
+		RGXFwSharedMemCacheOpValue(psFwSysInit->ui32OSKickTest, FLUSH);
+
+		/* kick register */
+		ui32ScheduleRegister = RGX_CR_MTS_SCHEDULE + (ui32OSid * RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+		PVR_DPF((PVR_DBG_MESSAGE, "  Testing OS: %u, Kick Reg: %X",
+				 ui32OSid,
+				 ui32ScheduleRegister));
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32ScheduleRegister, ui32KickType);
+
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS, "VZ sideband test, kicking MTS register %u", ui32OSid);
+
+		PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+				ui32ScheduleRegister, ui32KickType, PDUMP_FLAGS_CONTINUOUS);
+
+		DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							   offsetof(RGXFWIF_SYSINIT, ui32OSKickTest),
+							   0,
+							   0xFFFFFFFF,
+							   PDUMP_POLL_OPERATOR_EQUAL,
+							   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if !defined(NO_HARDWARE)
+		OSMemoryBarrier((IMG_BYTE*) psDevInfo->pvRegsBaseKM + ui32ScheduleRegister);
+
+		/* Wait test enable bit to be unset */
+		if (PVRSRVPollForValueKM(psDeviceNode,
+								 (volatile IMG_UINT32 __iomem *)&psFwSysInit->ui32OSKickTest,
+								 0,
+								 RGXFWIF_KICK_TEST_ENABLED_BIT,
+								 POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+								 RGXFwSharedMemCacheOpExecPfn) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Testing OS %u kick register failed: firmware did not clear test location (contents: 0x%X)",
+					 ui32OSid,
+					 psFwSysInit->ui32OSKickTest));
+
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Check that the value is what we expect */
+		if (psFwSysInit->ui32OSKickTest != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Testing OS %u kick register failed: firmware wrote 0x%X to test location",
+					 ui32OSid,
+					 psFwSysInit->ui32OSKickTest));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "    PASS"));
+#endif
+	}
+
+	PVR_LOG(("MTS passed sideband tests"));
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION_MTS) */
+
+
+/*
+	RGXPostPowerState
+*/
+PVRSRV_ERROR RGXPostPowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                               PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                               PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                               PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	if ((eNewPowerState == eCurrentPowerState) ||
+	    (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGXPostPowerState: Current state: %d, New state: %d",
+		             eCurrentPowerState, eNewPowerState);
+
+		return PVRSRV_OK;
+	}
+
+	/* Update timer correlation related data */
+	RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_POWER);
+
+	/* Update GPU state counters */
+	_RGXUpdateGPUUtilStats(psDevInfo);
+
+	eError = RGXDoStart(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDoStart", fail);
+
+	OSMemoryBarrier(NULL);
+
+
+	/*
+		* Check whether the FW has started by polling on bFirmwareStarted flag
+		*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->bFirmwareStarted,
+	                           INVALIDATE);
+	if (PVRSRVPollForValueKM(psDeviceNode,
+	                         (IMG_UINT32 __iomem *)&psDevInfo->psRGXFWIfSysInit->bFirmwareStarted,
+	                         IMG_TRUE,
+	                         0xFFFFFFFF,
+	                         POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+	                         RGXFwSharedMemCacheOpExecPfn) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPostPowerState: Polling for 'FW started' flag failed."));
+		eError = PVRSRV_ERROR_TIMEOUT;
+
+#if defined(TRACK_FW_BOOT)
+		RGXCheckFWBootStage(psDevInfo);
+#endif
+
+		/*
+		* When bFirmwareStarted fails some info may be gained by doing the following
+		* debug dump but unfortunately it could be potentially dangerous if the reason
+		* for not booting is the GPU power is not ON. However, if we have reached this
+		* point the System Layer has returned without errors, we assume the GPU power
+		* is indeed ON.
+		*/
+		RGXDumpRGXDebugSummary(NULL, NULL, psDeviceNode->pvDevice, IMG_TRUE);
+		RGXDumpRGXRegisters(NULL, NULL, psDeviceNode->pvDevice);
+
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPollForValueKM(bFirmwareStarted)", fail);
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS, "Wait for the Firmware to start.");
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfSysInitMemDesc,
+	                                offsetof(RGXFWIF_SYSINIT, bFirmwareStarted),
+	                                IMG_TRUE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXPostPowerState: problem pdumping POL for psRGXFWIfSysInitMemDesc (%d)",
+		         eError));
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", fail);
+	}
+
+#endif /* defined(PDUMP) */
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	eError = RGXVirtualisationPowerupSidebandTest(psDeviceNode, psDevInfo->psRGXFWIfSysInit, psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXVirtualisationPowerupSidebandTest", fail);
+#endif
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp,
+	                           INVALIDATE);
+	PVRSRVSetFirmwareStartTime(psDeviceNode->psPowerDev,
+	                           psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp,
+	                           FLUSH);
+#endif
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32MarkerVal,
+	                           INVALIDATE);
+	HTBSyncPartitionMarker(psDevInfo->psRGXFWIfSysInit->ui32MarkerVal);
+
+#if defined(SUPPORT_LINUX_DVFS)
+	eError = ResumeDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "ResumeDVFS", fail);
+#endif
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXPostPowerState: Current state: %d, New state: %d",
+	             eCurrentPowerState, eNewPowerState);
+
+	return eError;
+
+fail:
+	psDevInfo->bRGXPowered = IMG_FALSE;
+
+	return eError;
+}
+
+/*
+	RGXPreClockSpeedChange
+*/
+PVRSRV_ERROR RGXPreClockSpeedChange(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                    PVRSRV_DEV_POWER_STATE	eCurrentPowerState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	const PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused const RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "RGXPreClockSpeedChange: RGX clock speed was %uHz",
+			psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF) &&
+	    (psFwSysData->ePowState != RGXFWIF_POW_OFF))
+	{
+		/* Update GPU frequency and timer correlation related data */
+		RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+	}
+
+	return eError;
+}
+
+/*
+	RGXPostClockSpeedChange
+*/
+PVRSRV_ERROR RGXPostClockSpeedChange(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                     PVRSRV_DEV_POWER_STATE	eCurrentPowerState)
+{
+	const PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32NewClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	/* Update runtime configuration with the new value */
+	OSWriteDeviceMem32WithWMB(&psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed,
+	                          ui32NewClockSpeed);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed, FLUSH);
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+	if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF) &&
+	    (psFwSysData->ePowState != RGXFWIF_POW_OFF))
+	{
+		RGXFWIF_KCCB_CMD sCOREClkSpeedChangeCmd;
+		IMG_UINT32 ui32CmdKCCBSlot;
+
+		RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+
+		sCOREClkSpeedChangeCmd.eCmdType = RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE;
+		sCOREClkSpeedChangeCmd.uCmdData.sCoreClkSpeedChangeData.ui32NewClockSpeed = ui32NewClockSpeed;
+
+		PDUMPCOMMENT(psDeviceNode, "Scheduling CORE clock speed change command");
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+		                                      &sCOREClkSpeedChangeCmd,
+		                                      PDUMP_FLAGS_NONE,
+		                                      &ui32CmdKCCBSlot);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Scheduling CORE clock speed change command failed");
+			PVR_DPF((PVR_DBG_ERROR, "RGXPostClockSpeedChange: Scheduling KCCB command failed. Error:%u", eError));
+			return eError;
+		}
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_FREQ)
+		GpuTraceFrequency(psDeviceNode->sDevId.ui32InternalID,
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed);
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_FREQ) */
+
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXPostClockSpeedChange: RGX clock speed changed to %uHz",
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	RGXPowUnitsChange
+
+ @Description Change power units state
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32PowUnits : On Rogue: Number of DUSTs to make transition to.
+                          On Volcanic: Mask containing power state of SPUs.
+                          Each bit corresponds to an SPU and value must be non-zero.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPowUnitsChange(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32PowUnits)
+
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError;
+	RGXFWIF_KCCB_CMD	sPowUnitsChange;
+	IMG_UINT32			ui32AvailablePowUnits;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AvailablePowUnits = psDevInfo->ui32AvailablePowUnitsMask;
+
+	/**
+	 * Validate the input. At-least one PU must be powered on and all requested
+	 * PU's must be a subset of full PU mask.
+	 */
+	if ((ui32PowUnits == 0) || (ui32PowUnits & ~ui32AvailablePowUnits))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid Power Units mask requested (0x%X). Value should be non-zero and sub-set of 0x%X mask",
+				__func__,
+				ui32PowUnits,
+				ui32AvailablePowUnits));
+		return PVRSRV_ERROR_INVALID_SPU_MASK;
+	}
+#else
+	ui32AvailablePowUnits = psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount;
+
+	if (ui32PowUnits > ui32AvailablePowUnits)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid number of DUSTs (%u) while expecting value within <0,%u>",
+				__func__,
+				ui32PowUnits,
+				ui32AvailablePowUnits));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+
+	psRuntimeCfg->ui32PowUnitsState = ui32PowUnits;
+	OSWriteMemoryBarrier(&psRuntimeCfg->ui32PowUnitsState);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ui32PowUnitsState, FLUSH);
+
+#if !defined(NO_HARDWARE)
+	{
+		const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+		RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+		                           INVALIDATE);
+
+		if (psFwSysData->ePowState == RGXFWIF_POW_OFF)
+		{
+			return PVRSRV_OK;
+		}
+
+		if (psFwSysData->ePowState != RGXFWIF_POW_FORCED_IDLE)
+		{
+			eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Powered units state can not be changed when not IDLE",
+					 __func__));
+			return eError;
+		}
+	}
+#endif
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		return eError;
+	}
+
+	sPowUnitsChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowUnitsChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_NUM_UNITS_CHANGE;
+	sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32PowUnits = ui32PowUnits;
+#if defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX)
+	sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32RACUnits = 0;
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) >= 3)
+	{
+		sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32RACUnits =
+			(1U << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+	}
+#endif
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "Scheduling command to change power units state to 0x%X",
+	             ui32PowUnits);
+	eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+	                                      &sPowUnitsChange,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "Scheduling command to change power units state. Error:%u",
+		             eError);
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Scheduling KCCB to change power units state. Error:%u",
+				 __func__, eError));
+		return eError;
+	}
+
+	/* Wait for the firmware to answer. */
+	eError = RGXPollForGPCommandCompletion(psDeviceNode,
+	                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+								  0x1, 0xFFFFFFFF);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timeout waiting for idle request", __func__));
+		return eError;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "%s: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             __func__, psDevInfo->psPowSyncPrim->pui32LinAddr,
+	             RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ @Function	RGXAPMLatencyChange
+*/
+PVRSRV_ERROR RGXAPMLatencyChange(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 IMG_UINT32			ui32ActivePMLatencyms,
+                                 IMG_BOOL			bActivePMLatencyPersistant)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError;
+	RGXFWIF_RUNTIME_CFG	*psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	PVRSRV_DEV_POWER_STATE	ePowerState;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	/* Update runtime configuration with the new values and ensure the
+	 * new APM latency is written to memory before requesting the FW to
+	 * read it
+	 */
+	psRuntimeCfg->ui32ActivePMLatencyms = ui32ActivePMLatencyms;
+	psRuntimeCfg->bActivePMLatencyPersistant = bActivePMLatencyPersistant;
+	OSWriteMemoryBarrier(&psRuntimeCfg->bActivePMLatencyPersistant);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ui32ActivePMLatencyms, FLUSH);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->bActivePMLatencyPersistant, FLUSH);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF)
+	{
+		/* Power is off, APM latency will be read on next firmware boot */
+		return PVRSRV_OK;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire power lock (%u)",
+		         __func__, eError));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		RGXFWIF_KCCB_CMD	sActivePMLatencyChange;
+		sActivePMLatencyChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
+		sActivePMLatencyChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_APM_LATENCY_CHANGE;
+
+		PDUMPCOMMENT(psDeviceNode,
+		             "Scheduling command to change APM latency to %u",
+		             ui32ActivePMLatencyms);
+		eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+		                                      &sActivePMLatencyChange,
+		                                      PDUMP_FLAGS_NONE,
+		                                      &ui32CmdKCCBSlot);
+
+		if (eError != PVRSRV_OK)
+		{
+			PDUMPCOMMENT(psDeviceNode,
+			             "Scheduling command to change APM latency failed. Error:%u",
+			             eError);
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Scheduling KCCB to change APM latency failed. Error:%u",
+			         __func__, eError));
+			goto ErrorExit;
+		}
+	}
+
+ErrorExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	return eError;
+}
+
+/*
+	RGXActivePowerRequest
+*/
+PVRSRV_ERROR RGXActivePowerRequest(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+
+	psDevInfo->ui32ActivePMReqTotal++;
+
+	/* Powerlock to avoid further requests from racing with the FW hand-shake
+	 * from now on (previous kicks to this point are detected by the FW)
+	 * PVRSRVPowerLock is replaced with PVRSRVPowerTryLock to avoid
+	 * potential dead lock between PDumpWriteLock and PowerLock
+	 * during 'DriverLive + PDUMP=1 + EnableAPM=1'.
+	 */
+	eError = PVRSRVPowerTryLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			PVR_LOG_ERROR(eError, "PVRSRVPowerTryLock");
+		}
+		else
+		{
+			psDevInfo->ui32ActivePMReqRetry++;
+		}
+		goto _RGXActivePowerRequest_PowerLock_failed;
+	}
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	/* Check again for IDLE once we have the power lock */
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		PVRSRVSetFirmwareHandshakeIdleTime(psDeviceNode->psPowerDev,
+		                                   RGXReadHWTimerReg(psDevInfo)-psFwSysData->ui64StartIdleTime);
+#endif
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+		                                     PVRSRV_DEV_POWER_STATE_OFF,
+		                                     PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError == PVRSRV_OK)
+		{
+			psDevInfo->ui32ActivePMReqOk++;
+		}
+		else if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+		{
+			psDevInfo->ui32ActivePMReqDenied++;
+		}
+	}
+	else
+	{
+		psDevInfo->ui32ActivePMReqNonIdle++;
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_RGXActivePowerRequest_PowerLock_failed:
+
+	return eError;
+}
+/*
+	RGXForcedIdleRequest
+*/
+
+#define RGX_FORCED_IDLE_RETRY_COUNT 10
+
+PVRSRV_ERROR RGXForcedIdleRequest(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_BOOL bDeviceOffPermitted)
+{
+	PVRSRV_RGXDEV_INFO    *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD      sPowCmd;
+	PVRSRV_ERROR          eError;
+	IMG_UINT32            ui32RetryCount = 0;
+	IMG_UINT32            ui32CmdKCCBSlot;
+#if !defined(NO_HARDWARE)
+	const RGXFWIF_SYSDATA *psFwSysData;
+#endif
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if !defined(NO_HARDWARE)
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	/* Firmware already forced idle */
+	if (psFwSysData->ePowState == RGXFWIF_POW_FORCED_IDLE)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Firmware is not powered. Sometimes this is permitted, for instance we were forcing idle to power down. */
+	if (psFwSysData->ePowState == RGXFWIF_POW_OFF)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Firmware is powered OFF (bDeviceOffPermitted = %s)",
+				 bDeviceOffPermitted ? "Yes" : "No"));
+		return (bDeviceOffPermitted) ? PVRSRV_OK : PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
+	}
+#endif
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		return eError;
+	}
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_FORCE_IDLE;
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Sending forced idle command");
+
+	/* Send one forced IDLE command to GP */
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send idle request", __func__));
+		return eError;
+	}
+
+	/* Wait for GPU to finish current workload */
+	do {
+		eError = RGXPollForGPCommandCompletion(psDeviceNode,
+		                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+									  0x1, 0xFFFFFFFF);
+		if ((eError == PVRSRV_OK) || (ui32RetryCount == RGX_FORCED_IDLE_RETRY_COUNT))
+		{
+			break;
+		}
+		ui32RetryCount++;
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Request timeout. Retry %d of %d",
+				 __func__, ui32RetryCount, RGX_FORCED_IDLE_RETRY_COUNT));
+	} while (IMG_TRUE);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXFWNotifyHostTimeout(psDevInfo);
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Idle request failed. Firmware potentially left in forced idle state",
+				 __func__));
+		return eError;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* Check the firmware state for idleness */
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+	if (psFwSysData->ePowState != RGXFWIF_POW_FORCED_IDLE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "FW power state (%u) is not RGXFWIF_POW_FORCED_IDLE (%u)",
+				 psFwSysData->ePowState, RGXFWIF_POW_FORCED_IDLE));
+		return PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RGXSendCancelForceIdleCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                   IMG_UINT32         *ui32CmdKCCBSlot)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD	sPowCmd;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	                      "SyncPrimSet: Failed to set Power sync prim",
+	                      ErrorExit);
+
+	/* Send the IDLE request to the FW */
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_CANCEL_FORCED_IDLE;
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Sending cancel forced idle command");
+
+	/* Send cancel forced IDLE command to GP */
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      ui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGXCancelForcedIdleRequest: Failed to send cancel IDLE request for DM%d",
+		             RGXFWIF_DM_GP);
+		goto ErrorExit;
+	}
+
+ErrorExit:
+	return eError;
+}
+/*
+	RGXCancelForcedIdleRequest
+*/
+PVRSRV_ERROR RGXCancelForcedIdleRequest(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	eError = _RGXSendCancelForceIdleCommand(psDeviceNode, &ui32CmdKCCBSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	                      "_RGXSendCancelForceIdleCommand",
+	                      ErrorExit);
+
+	/* Wait for the firmware to answer. */
+	eError = RGXPollForGPCommandCompletion(psDeviceNode,
+	                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+								  1, 0xFFFFFFFF);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	                      "RGXPollForGPCommandCompletion: Timeout waiting for cancel idle request",
+	                      ErrorExit);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXCancelForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+	return eError;
+
+ErrorExit:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Firmware potentially left in forced idle state err: %u",
+	         __func__,
+	         eError));
+	return eError;
+}
+
+PVRSRV_ERROR RGXCancelForcedIdleRequestAsync(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = _RGXSendCancelForceIdleCommand(psDeviceNode, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Firmware potentially left in forced idle state err: %u",
+		         __func__,
+		         eError));
+	}
+
+	return eError;
+}
+
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+/*************************************************************************/ /*!
+@Function       PDVFSProcessCoreClkChangeRequest
+@Description    Processes a core clock rate change request.
+@Input          psDevInfo            A pointer to PVRSRV_RGXDEV_INFO.
+@Input          ui32CoreClockRate    New core clock rate.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeRequest(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = &psDevConfig->sDVFS.sDVFSDeviceCfg;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = ((RGX_DATA*)(psDevConfig->hDevData))->psRGXTimingInfo;
+	IMG_UINT32 ui32CoreClockRateCurrent = psRGXTimingInfo->ui32CoreClockSpeed;
+	const IMG_OPP *psOpp = NULL;
+	IMG_UINT32 ui32Index;
+	PVRSRV_ERROR eError;
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Core clock rate = %u", ui32CoreClockRate));
+
+	/* Find the matching OPP (Exact). */
+	for (ui32Index = 0; ui32Index < psDVFSDeviceCfg->ui32OPPTableSize; ui32Index++)
+	{
+		if (ui32CoreClockRate == psDVFSDeviceCfg->pasOPPTable[ui32Index].ui32Freq)
+		{
+			psOpp = &psDVFSDeviceCfg->pasOPPTable[ui32Index];
+			break;
+		}
+	}
+
+	if (! psOpp)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Frequency not present in OPP table - %u", ui32CoreClockRate));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = PVRSRVDevicePreClockSpeedChange(psDevInfo->psDeviceNode, psDVFSDeviceCfg->bIdleReq, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVDevicePreClockSpeedChange failed"));
+		return eError;
+	}
+
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+
+	/* Increasing frequency, change voltage first */
+	if (ui32CoreClockRate > ui32CoreClockRateCurrent)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDevConfig->hSysData, psOpp->ui32Volt);
+	}
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDevConfig->hSysData, ui32CoreClockRate);
+
+	/* Decreasing frequency, change frequency first */
+	if (ui32CoreClockRate < ui32CoreClockRateCurrent)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDevConfig->hSysData, psOpp->ui32Volt);
+	}
+
+	PVRSRVDevicePostClockSpeedChange(psDevInfo->psDeviceNode, psDVFSDeviceCfg->bIdleReq, NULL);
+
+	return PVRSRV_OK;
+}
+#else
+/*************************************************************************/ /*!
+@Function       PDVFSProcessCoreClkChangeNotification
+@Description    Processes a core clock rate change notification.
+@Input          psDevInfo            A pointer to PVRSRV_RGXDEV_INFO.
+@Input          ui32CoreClockRate    New core clock rate.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeNotification(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = ((RGX_DATA*)(psDevConfig->hDevData))->psRGXTimingInfo;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire lock (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDevInfo->psDeviceNode, &ePowerState);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire power state (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+		return eError;
+	}
+
+	/* Guest drivers do not initialize psRGXFWIfFwSysData */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	if ((ePowerState != PVRSRV_DEV_POWER_STATE_OFF)
+	    && ((psDevInfo->psRGXFWIfFwSysData == NULL) || (psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)))
+	{
+		/* Update GPU frequency and timer correlation related data */
+		RGXTimeCorrEnd(psDevInfo->psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+		psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+		RGXTimeCorrBegin(psDevInfo->psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+	}
+	else
+	{
+		psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	return PVRSRV_OK;
+}
+#endif
+#endif /* SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY */
+
+
+
+/******************************************************************************
+ End of file (rgxpower.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.h
new file mode 100644
index 000000000000..206f44db6cba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX power header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX power
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXPOWER_H)
+#define RGXPOWER_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "servicesext.h"
+#include "rgxdevice.h"
+
+
+/*!
+******************************************************************************
+
+ @Function	RGXPrePowerState
+
+ @Description
+
+ does necessary preparation before power state transition
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPrePowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+							  PVRSRV_DEV_POWER_STATE	eNewPowerState,
+							  PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+							  PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPostPowerState
+
+ @Description
+
+ does necessary preparation after power state transition
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPostPowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+							   PVRSRV_DEV_POWER_STATE	eNewPowerState,
+							   PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+							   PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXVzPrePowerState
+
+ @Description
+
+ does necessary preparation before power state transition on a vz driver
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXVzPrePowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+								PVRSRV_DEV_POWER_STATE	eNewPowerState,
+								PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+								PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXVzPostPowerState
+
+ @Description
+
+ does necessary preparation after power state transition on a vz driver
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXVzPostPowerState(PPVRSRV_DEVICE_NODE	psDeviceNode,
+								 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+								 PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+								 PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPreClockSpeedChange
+
+ @Description
+
+	Does processing required before an RGX clock speed change.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eCurrentPowerState : Power state of the device
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPreClockSpeedChange(PPVRSRV_DEVICE_NODE		psDeviceNode,
+									PVRSRV_DEV_POWER_STATE	eCurrentPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPostClockSpeedChange
+
+ @Description
+
+	Does processing required after an RGX clock speed change.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eCurrentPowerState : Power state of the device
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPostClockSpeedChange(PPVRSRV_DEVICE_NODE	psDeviceNode,
+									 PVRSRV_DEV_POWER_STATE	eCurrentPowerState);
+
+
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+/*!
+******************************************************************************
+
+ @Function	RGXProcessCoreClkChangeRequest
+
+ @Input	   psDevInfo : RGX Device Info
+ @Input	   ui32CoreClockRate : New clock frequency to send to system layer.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeRequest(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate);
+#define RGX_PROCESS_CORE_CLK_RATE_CHANGE(devinfo, clk)  RGXProcessCoreClkChangeRequest(devinfo, clk)
+
+#else
+/*!
+******************************************************************************
+
+ @Function	RGXProcessCoreClkChangeNotification
+
+ @Input	   psDevInfo : RGX Device Info
+ @Input	   ui32CoreClockRate : New clock frequency.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeNotification(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate);
+#define RGX_PROCESS_CORE_CLK_RATE_CHANGE(devinfo, clk)  RGXProcessCoreClkChangeNotification(devinfo, clk)
+#endif
+#endif /* SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY */
+
+/*!
+******************************************************************************
+
+ @Function	RGXPowUnitsChange
+
+ @Description Change power units state
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32PowUnits : On Rogue: Number of DUSTs to make transition to.
+                          On Volcanic: Mask containing power state of SPUs.
+                          Each bit corresponds to an SPU and value must be non-zero.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPowUnitsChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                               IMG_UINT32 ui32PowUnits);
+
+/*!
+******************************************************************************
+
+ @Function	RGXAPMLatencyChange
+
+ @Description
+
+	Changes the wait duration used before firmware indicates IDLE.
+	Reducing this value will cause the firmware to shut off faster and
+	more often but may increase bubbles in GPU scheduling due to the added
+	power management activity. If bPersistent is NOT set, APM latency will
+	return back to system default on power up.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32ActivePMLatencyms : Number of milliseconds to wait
+ @Input	   bActivePMLatencyPersistant : Set to ensure new value is not reset
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXAPMLatencyChange(PPVRSRV_DEVICE_NODE	psDeviceNode,
+				IMG_UINT32				ui32ActivePMLatencyms,
+				IMG_BOOL				bActivePMLatencyPersistant);
+
+/*!
+******************************************************************************
+
+ @Function	RGXActivePowerRequest
+
+ @Description Initiate a handshake with the FW to power off the GPU
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXActivePowerRequest(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	RGXForcedIdleRequest
+
+ @Description Initiate a handshake with the FW to idle the GPU
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Input    bDeviceOffPermitted : Set to indicate device state being off is not
+                                 erroneous.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXForcedIdleRequest(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  IMG_BOOL bDeviceOffPermitted);
+
+/*!
+******************************************************************************
+
+ @Function	RGXCancelForcedIdleRequest
+
+ @Description Send a request to cancel idle to the firmware.
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXCancelForcedIdleRequest(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	RGXCancelForcedIdleRequestAsync
+
+ @Description Send a request to cancel idle to the firmware.
+              Does not wait for response from FW.
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXCancelForcedIdleRequestAsync(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+
+#endif /* RGXPOWER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.c
new file mode 100644
index 000000000000..c2f681c70f12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.c
@@ -0,0 +1,331 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Register configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Regconfig routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxregconfig.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "device.h"
+#include "sync_internal.h"
+#include "pdump_km.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR PVRSRVRGXSetRegConfigTypeKM(CONNECTION_DATA * psDevConnection,
+                                         PVRSRV_DEVICE_NODE	 *psDeviceNode,
+                                         IMG_UINT8           ui8RegCfgType)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR          eError      = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO    *psDevInfo  = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG        *psRegCfg   = &psDevInfo->sRegConfig;
+	RGXFWIF_REG_CFG_TYPE  eRegCfgType = (RGXFWIF_REG_CFG_TYPE) ui8RegCfgType;
+
+	PVR_UNREFERENCED_PARAMETER(psDevConnection);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (eRegCfgType < psRegCfg->eRegCfgTypeToPush)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Register configuration requested (%d) is not valid since it has to be at least %d."
+			 " Configurations of different types need to go in order",
+			 __func__,
+			 eRegCfgType,
+			 psRegCfg->eRegCfgTypeToPush));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_INVALID_TYPE;
+	}
+
+	psRegCfg->eRegCfgTypeToPush = eRegCfgType;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui8RegCfgType);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXAddRegConfigKM(CONNECTION_DATA * psConnection,
+                                     PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                     IMG_UINT32		ui32RegAddr,
+                                     IMG_UINT64		ui64RegValue,
+                                     IMG_UINT64		ui64RegMask)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (psRegCfg->bEnabled)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Cannot add record whilst register configuration active.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_ENABLED;
+	}
+	if (psRegCfg->ui32NumRegRecords == RGXFWIF_REG_CFG_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Register configuration full.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_FULL;
+	}
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Addr = (IMG_UINT64) ui32RegAddr;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Value = ui64RegValue;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Mask = ui64RegMask;
+	sRegCfgCmd.uCmdData.sRegConfigData.eRegConfigType = psRegCfg->eRegCfgTypeToPush;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_ADD;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->ui32NumRegRecords++;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(ui64RegValue);
+	PVR_UNREFERENCED_PARAMETER(ui64RegMask);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXClearRegConfigKM(CONNECTION_DATA * psConnection,
+                                       PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (psRegCfg->bEnabled)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Attempt to clear register configuration whilst active.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_ENABLED;
+	}
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_CLEAR;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->ui32NumRegRecords = 0;
+	psRegCfg->eRegCfgTypeToPush = RGXFWIF_REG_CFG_TYPE_PWR_ON;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXEnableRegConfigKM(CONNECTION_DATA * psConnection,
+                                        PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_ENABLE;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->bEnabled = IMG_TRUE;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXDisableRegConfigKM(CONNECTION_DATA * psConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_DISABLE;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->bEnabled = IMG_FALSE;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+/******************************************************************************
+ End of file (rgxregconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.h
new file mode 100644
index 000000000000..b0921d98cb14
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.h
@@ -0,0 +1,130 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX register configuration functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX register configuration functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXREGCONFIG_H)
+#define RGXREGCONFIG_H
+
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetRegConfigTypeKM
+
+ @Description
+	Server-side implementation of RGXSetRegConfig
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui8RegPowerIsland - Reg configuration
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetRegConfigTypeKM(CONNECTION_DATA * psDevConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                         IMG_UINT8 ui8RegPowerIsland);
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetRegConfigKM
+
+ @Description
+	Server-side implementation of RGXSetRegConfig
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui64RegAddr - Register address
+ @Input ui64RegValue - Reg value
+ @Input ui64RegMask - Reg mask
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+
+PVRSRV_ERROR PVRSRVRGXAddRegConfigKM(CONNECTION_DATA * psConnection,
+                                     PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                     IMG_UINT32	ui64RegAddr,
+                                     IMG_UINT64	ui64RegValue,
+                                     IMG_UINT64	ui64RegMask);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXClearRegConfigKM
+
+ @Description
+	Server-side implementation of RGXClearRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXClearRegConfigKM(CONNECTION_DATA * psConnection,
+                                       PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXEnableRegConfigKM
+
+ @Description
+	Server-side implementation of RGXEnableRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXEnableRegConfigKM(CONNECTION_DATA * psConnection,
+                                        PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDisableRegConfigKM
+
+ @Description
+	Server-side implementation of RGXDisableRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDisableRegConfigKM(CONNECTION_DATA * psConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+#endif /* RGXREGCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.c
new file mode 100644
index 000000000000..8eace4b16c81
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.c
@@ -0,0 +1,321 @@
+/*************************************************************************/ /*!
+@File           rgxshader.c
+@Title          TQ Shader Load
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shader code and info are shared for all context on the device.
+                If allocation doesn't already exist, read shader data from file
+                and allocate PMR memory. PMR memory is not deallocated until
+                device deinit.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxshader.h"
+#include "osfunc_common.h"
+#include "rgxdevice.h"
+#include "pdump_km.h"
+#include "physmem.h"
+#include "ri_server.h"
+#include "pvr_ricommon.h"
+#include "pvrsrv.h"
+
+static void
+RGXShaderReadHeader(OS_FW_IMAGE *psShaderFW, RGX_SHADER_HEADER *psHeader)
+{
+	const void * pvData;
+
+	pvData = OSFirmwareData(psShaderFW);
+
+	OSDeviceMemCopy(psHeader, pvData, sizeof(RGX_SHADER_HEADER));
+}
+
+static size_t
+RGXShaderCLIMemSize(OS_FW_IMAGE *psShaderFW)
+{
+	RGX_SHADER_HEADER sHeader;
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	return sHeader.ui32SizeClientMem;
+}
+
+static size_t
+RGXShaderUSCMemSize(OS_FW_IMAGE *psShaderFW)
+{
+	RGX_SHADER_HEADER sHeader;
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	return sHeader.ui32SizeFragment;
+}
+
+static void *
+RGXShaderCLIMem(OS_FW_IMAGE *psShaderFW)
+{
+	return (void*)OSFirmwareData(psShaderFW);
+}
+
+static void *
+RGXShaderUSCMem(OS_FW_IMAGE *psShaderFW)
+{
+	IMG_PBYTE pui8Data;
+
+	pui8Data = (IMG_PBYTE)OSFirmwareData(psShaderFW);
+
+	pui8Data += RGXShaderCLIMemSize(psShaderFW);
+
+	return (void*) pui8Data;
+}
+
+#define RGX_SHADER_FILENAME_MAX_SIZE   ((sizeof(RGX_SH_FILENAME)+	\
+										 RGX_BVNC_STR_SIZE_MAX))
+
+static void
+_GetShaderFileName(PVRSRV_DEVICE_NODE * psDeviceNode,
+				   IMG_CHAR           * pszShaderFilenameStr,
+				   IMG_CHAR           * pszShaderpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSSNPrintf(pszShaderFilenameStr, RGX_SHADER_FILENAME_MAX_SIZE,
+			   RGX_SH_FILENAME "." RGX_BVNC_STR_FMTSPEC,
+			   psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			   psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C);
+
+	OSSNPrintf(pszShaderpFilenameStr, RGX_SHADER_FILENAME_MAX_SIZE,
+			   RGX_SH_FILENAME "." RGX_BVNC_STRP_FMTSPEC,
+			   psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			   psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C);
+}
+
+PVRSRV_ERROR
+PVRSRVTQLoadShaders(PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_FW_IMAGE        *psShaderFW;
+	RGX_SHADER_HEADER   sHeader;
+	IMG_UINT32          ui32MappingTable = 0;
+	IMG_UINT32          ui32NumPages;
+	IMG_CHAR            aszShaderFilenameStr[RGX_SHADER_FILENAME_MAX_SIZE];
+	IMG_CHAR            aszShaderpFilenameStr[RGX_SHADER_FILENAME_MAX_SIZE];
+	const IMG_CHAR      *pszShaderFilenameStr = aszShaderFilenameStr;
+	size_t              uiNumBytes;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	size_t              uiLog2PageSize = OSGetPageShift();
+	IMG_DEVMEM_SIZE_T   uiTQUSCMemSize;
+	PVRSRV_ERROR        eError;
+
+	_GetShaderFileName(psDeviceNode, aszShaderFilenameStr, aszShaderpFilenameStr);
+
+	eError = OSLoadFirmware(psDeviceNode, aszShaderFilenameStr, NULL, &psShaderFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		eError = OSLoadFirmware(psDeviceNode, aszShaderpFilenameStr,
+		                        NULL, &psShaderFW);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to load shader binary file %s (%s)",
+			         __func__,
+			         aszShaderpFilenameStr,
+			         PVRSRVGetErrorString(eError)));
+			eError = PVRSRV_ERROR_UNABLE_TO_FIND_RESOURCE;
+			goto failed_init;
+		}
+
+		pszShaderFilenameStr = aszShaderpFilenameStr;
+	}
+
+	PVR_LOG(("Shader binary image '%s' loaded", pszShaderFilenameStr));
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	if (sHeader.ui32Version != RGX_TQ_SHADERS_VERSION_PACK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: unsupported TQ shaders version: %d != %d",
+				 __func__, sHeader.ui32Version, RGX_TQ_SHADERS_VERSION_PACK));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto failed_firmware;
+	}
+
+	ui32NumPages = (sHeader.ui32SizeFragment / IMG_PAGE2BYTES32(uiLog2PageSize)) + 1;
+
+	uiTQUSCMemSize = ui32NumPages * IMG_PAGE2BYTES32(uiLog2PageSize);
+
+	if (uiTQUSCMemSize > RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TQ shaders out of space ("IMG_DEVMEM_SIZE_FMTSPEC" > 0x%X)",
+				 __func__, uiTQUSCMemSize, RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto failed_firmware;
+	}
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate TDM USC PMR Block (Pages %08X)", ui32NumPages);
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiTQUSCMemSize,
+									1,
+									1,
+									&ui32MappingTable,
+									uiLog2PageSize,
+									PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT
+									| PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER,
+									sizeof("tquscpmr"),
+									"tquscpmr",
+									PVR_SYS_ALLOC_PID,
+									(PMR**)&psDevInfo->hTQUSCSharedMem,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PhysmemNewRamBackedPMR (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_firmware;
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(psDevInfo->hTQUSCSharedMem, PVR_SYS_ALLOC_PID);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RIWritePMREntryWithOwnerKM (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+#endif
+
+	eError = PMR_WriteBytes(psDevInfo->hTQUSCSharedMem, 0, RGXShaderUSCMem(psShaderFW), RGXShaderUSCMemSize(psShaderFW), &uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PMR_WriteBytes (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+
+	PDUMPCOMMENT(psDeviceNode, "Load TDM USC PMR Block (Size "IMG_SIZE_FMTSPECX")", RGXShaderUSCMemSize(psShaderFW));
+	PMRPDumpLoadMem(psDevInfo->hTQUSCSharedMem, 0, RGXShaderUSCMemSize(psShaderFW), PDUMP_FLAGS_CONTINUOUS, false);
+
+	ui32NumPages = (sHeader.ui32SizeClientMem / IMG_PAGE2BYTES32(uiLog2PageSize)) + 1;
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate TDM Client PMR Block (Pages %08X)", ui32NumPages);
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psHostDevNode,
+									(IMG_DEVMEM_SIZE_T)ui32NumPages * IMG_PAGE2BYTES32(uiLog2PageSize),
+									1,
+									1,
+									&ui32MappingTable,
+									uiLog2PageSize,
+									PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+									| PVRSRV_MEMALLOCFLAG_CPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT
+									| PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER
+									| PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL),
+									sizeof("tqclipmr"),
+									"tqclipmr",
+									PVR_SYS_ALLOC_PID,
+									(PMR**)&psDevInfo->hTQCLISharedMem,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PhysmemNewRamBackedPMR (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(psDevInfo->hTQCLISharedMem, PVR_SYS_ALLOC_PID);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RIWritePMREntryWithOwnerKM (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_clipmr;
+	}
+#endif
+
+	eError = PMR_WriteBytes(psDevInfo->hTQCLISharedMem, 0, RGXShaderCLIMem(psShaderFW), RGXShaderCLIMemSize(psShaderFW), &uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PMR_WriteBytes (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_clipmr;
+	}
+
+	OSUnloadFirmware(psShaderFW);
+
+	PVR_ASSERT(psDevInfo->hTQUSCSharedMem != NULL);
+	PVR_ASSERT(psDevInfo->hTQCLISharedMem != NULL);
+
+	return PVRSRV_OK;
+
+failed_clipmr:
+	(void) PMRUnrefPMR(psDevInfo->hTQCLISharedMem);
+failed_uscpmr:
+	(void) PMRUnrefPMR(psDevInfo->hTQUSCSharedMem);
+failed_firmware:
+	OSUnloadFirmware(psShaderFW);
+failed_init:
+	return eError;
+}
+
+void
+PVRSRVTQAcquireShaders(PVRSRV_DEVICE_NODE  * psDeviceNode,
+					   PMR                ** ppsCLIPMRMem)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_ASSERT(psDevInfo->hTQCLISharedMem != NULL);
+
+	*ppsCLIPMRMem = psDevInfo->hTQCLISharedMem;
+}
+
+void PVRSRVTQUnloadShaders(PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	(void) PMRUnrefPMR(psDevInfo->hTQUSCSharedMem);
+	(void) PMRUnrefPMR(psDevInfo->hTQCLISharedMem);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.h
new file mode 100644
index 000000000000..4ba50fc3fc86
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File           rgxshader.h
+@Title          TQ Shader Load
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shader code and info are shared for all context on the device.
+                If allocation doesn't already exist, read shader data from file
+                and allocate PMR memory. PMR memory is not deallocated until
+                device deinit.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSHADER_H)
+#define RGXSHADER_H
+
+#include "fwload.h"
+#include "rgxtransfer_shader.h"
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQLoadShaders
+@Description    If PMR is not allocated, reads shader binary data from file
+                and allocates new PMR memory.
+@Input          psDeviceNode Device node
+@Return         PVRSRV_ERROR Returns PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVTQLoadShaders(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQAcquireShaders
+@Description    Get handle to ready allocated shader PMR memory
+@Input          psDeviceNode Device node
+@Output         ppsCLIPMRMem Shader data used by CPU client side.
+*/ /**************************************************************************/
+void
+PVRSRVTQAcquireShaders(PVRSRV_DEVICE_NODE  *psDeviceNode,
+                       PMR                **ppsCLIPMRMem);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQUnLoadShaders
+@Description    Unref PMR memory.
+@Input          psDeviceNode Device node
+*/ /**************************************************************************/
+void PVRSRVTQUnloadShaders(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* RGXSHADER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxstartstop.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxstartstop.h
new file mode 100644
index 000000000000..178afe2849a0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxstartstop.h
@@ -0,0 +1,84 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX start/stop header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX start/stop functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSTARTSTOP_H)
+#define RGXSTARTSTOP_H
+
+/* The routines declared here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when DRM security is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary.
+ */
+#include "rgxlayer.h"
+
+/*!
+*******************************************************************************
+
+ @Function      RGXStart
+
+ @Description   Perform GPU reset and initialisation
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXStart(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXStop
+
+ @Description   Stop Rogue in preparation for power down
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXStop(const void *hPrivate);
+
+#endif /* RGXSTARTSTOP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.c
new file mode 100644
index 000000000000..cec0597d0301
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.c
@@ -0,0 +1,184 @@
+/*************************************************************************/ /*!
+@File           rgxsyncutils.c
+@Title          RGX Sync Utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Sync helper functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxsyncutils.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "allocmem.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static
+void _DebugSyncValues(IMG_UINT32 *pui32UpdateValues,
+					  IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 iii;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (iii = 0; iii < ui32Count; iii++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+#else
+#define CHKPT_DBG(X)
+#endif
+
+
+PVRSRV_ERROR RGXSyncAppendTimelineUpdate(IMG_UINT32 ui32FenceTimelineUpdateValue,
+										 SYNC_ADDR_LIST	*psSyncList,
+										 SYNC_ADDR_LIST	*psPRSyncList,
+										 PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync,
+										 RGX_SYNC_DATA *psSyncData,
+										 IMG_BOOL bKick3D)
+{
+	IMG_UINT32 *pui32TimelineUpdateWOff = NULL;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+
+	IMG_UINT32 ui32ClientUpdateValueCount = psSyncData->ui32ClientUpdateValueCount;
+
+	/* Space for original client updates, and the one new update */
+	size_t uiUpdateSize = sizeof(*pui32IntAllocatedUpdateValues) * (ui32ClientUpdateValueCount + 1);
+
+	if (!bKick3D)
+	{
+		/* Additional space for one PR update, only the newest one */
+		uiUpdateSize += sizeof(*pui32IntAllocatedUpdateValues) * 1;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+		   "%s: About to allocate memory to hold updates in pui32IntAllocatedUpdateValues(<%p>)",
+		   __func__,
+		   (void*)pui32IntAllocatedUpdateValues));
+
+	/* Allocate memory to hold the list of update values (including our timeline update) */
+	pui32IntAllocatedUpdateValues = OSAllocMem(uiUpdateSize);
+	if (!pui32IntAllocatedUpdateValues)
+	{
+		/* Failed to allocate memory */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xcc, uiUpdateSize);
+	pui32TimelineUpdateWOff = pui32IntAllocatedUpdateValues;
+
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: Copying %d %s update values into pui32IntAllocatedUpdateValues(<%p>)",
+			   __func__,
+			   ui32ClientUpdateValueCount,
+			   bKick3D ? "TA/3D" : "TA/PR",
+			   (void*)pui32IntAllocatedUpdateValues));
+		/* Copy the update values into the new memory, then append our timeline update value */
+		OSCachedMemCopy(pui32TimelineUpdateWOff, psSyncData->paui32ClientUpdateValue, ui32ClientUpdateValueCount * sizeof(*psSyncData->paui32ClientUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(pui32TimelineUpdateWOff, ui32ClientUpdateValueCount);
+#endif
+
+		pui32TimelineUpdateWOff += ui32ClientUpdateValueCount;
+	}
+
+	/* Now set the additional update value and append the timeline sync prim addr to either the
+	 * render context 3D (or TA) update list
+	 */
+	CHKPT_DBG((PVR_DBG_ERROR,
+		   "%s: Appending the additional update value (0x%x) to psRenderContext->sSyncAddrList%sUpdate...",
+		   __func__,
+		   ui32FenceTimelineUpdateValue,
+		   bKick3D ? "TA/3D" : "TA/PR"));
+
+	/* Append the TA/3D update */
+	{
+		*pui32TimelineUpdateWOff++ = ui32FenceTimelineUpdateValue;
+		psSyncData->ui32ClientUpdateValueCount++;
+		psSyncData->ui32ClientUpdateCount++;
+		SyncAddrListAppendSyncPrim(psSyncList, psFenceTimelineUpdateSync);
+
+		if (!psSyncData->pauiClientUpdateUFOAddress)
+		{
+			psSyncData->pauiClientUpdateUFOAddress = psSyncList->pasFWAddrs;
+		}
+		/* Update paui32ClientUpdateValue to point to our new list of update values */
+		psSyncData->paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(pui32IntAllocatedUpdateValues, psSyncData->ui32ClientUpdateValueCount);
+#endif
+	}
+
+	if (!bKick3D)
+	{
+		/* Use the sSyncAddrList3DUpdate for PR (as it doesn't have one of its own) */
+		*pui32TimelineUpdateWOff++ = ui32FenceTimelineUpdateValue;
+		psSyncData->ui32ClientPRUpdateValueCount = 1;
+		psSyncData->ui32ClientPRUpdateCount = 1;
+		SyncAddrListAppendSyncPrim(psPRSyncList, psFenceTimelineUpdateSync);
+
+		if (!psSyncData->pauiClientPRUpdateUFOAddress)
+		{
+			psSyncData->pauiClientPRUpdateUFOAddress = psPRSyncList->pasFWAddrs;
+		}
+		/* Update paui32ClientPRUpdateValue to point to our new list of update values */
+		psSyncData->paui32ClientPRUpdateValue = &pui32IntAllocatedUpdateValues[psSyncData->ui32ClientUpdateValueCount];
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(psSyncData->paui32ClientPRUpdateValue, psSyncData->ui32ClientPRUpdateValueCount);
+#endif
+	}
+
+	/* Do not free the old psSyncData->ui32ClientUpdateValueCount,
+	 * as it was constant data passed through the bridge down to PVRSRVRGXKickTA3DKM() */
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.h
new file mode 100644
index 000000000000..bfc46937f185
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           rgxsyncutils.h
+@Title          RGX Sync Utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Sync helper functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXSYNCUTILS_H
+#define RGXSYNCUTILS_H
+
+#include "rgxdevice.h"
+#include "sync_server.h"
+#include "rgxdebug_common.h"
+#include "rgx_fwif_km.h"
+
+typedef struct _RGX_SYNC_DATA_
+{
+	PRGXFWIF_UFO_ADDR *pauiClientUpdateUFOAddress;
+	IMG_UINT32 *paui32ClientUpdateValue;
+	IMG_UINT32 ui32ClientUpdateValueCount;
+	IMG_UINT32 ui32ClientUpdateCount;
+
+	PRGXFWIF_UFO_ADDR *pauiClientPRUpdateUFOAddress;
+	IMG_UINT32 *paui32ClientPRUpdateValue;
+	IMG_UINT32 ui32ClientPRUpdateValueCount;
+	IMG_UINT32 ui32ClientPRUpdateCount;
+} RGX_SYNC_DATA;
+
+PVRSRV_ERROR RGXSyncAppendTimelineUpdate(IMG_UINT32 ui32FenceTimelineUpdateValue,
+										 SYNC_ADDR_LIST	*psSyncList,
+										 SYNC_ADDR_LIST	*psPRSyncList,
+										 PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync,
+										 RGX_SYNC_DATA *psSyncData,
+										 IMG_BOOL bKick3D);
+
+#endif /* RGXSYNCUTILS_H */
+
+/******************************************************************************
+ End of file (rgxsyncutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.c
new file mode 100644
index 000000000000..dd9d05ee18b2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.c
@@ -0,0 +1,1486 @@
+/*************************************************************************/ /*!
+@File           rgxtdmtransfer.c
+@Title          Device specific TDM transfer queue routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxtdmtransfer.h"
+#include "rgx_tq_shared.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_memallocflags.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxshader.h"
+
+#include "pdump_km.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TDM_UFO_DUMP	0
+
+//#define TDM_CHECKPOINT_DEBUG 1
+
+#if defined(TDM_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+typedef struct {
+	RGX_SERVER_COMMON_CONTEXT * psServerCommonContext;
+	IMG_INT32                   i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_TDM_DATA;
+
+
+struct _RGX_SERVER_TQ_TDM_CONTEXT_ {
+	PVRSRV_DEVICE_NODE      *psDeviceNode;
+	DEVMEM_MEMDESC          *psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC          *psFWTransferContextMemDesc;
+	IMG_UINT32              ui32Flags;
+	RGX_SERVER_TQ_TDM_DATA  sTDMData;
+	DLLIST_NODE             sListNode;
+	SYNC_ADDR_LIST          sSyncAddrListFence;
+	SYNC_ADDR_LIST          sSyncAddrListUpdate;
+	POS_LOCK		hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA       sWorkEstData;
+#endif
+};
+
+static PVRSRV_ERROR _CreateTDMTransferContext(
+	CONNECTION_DATA         * psConnection,
+	PVRSRV_DEVICE_NODE      * psDeviceNode,
+	DEVMEM_MEMDESC          * psAllocatedMemDesc,
+	IMG_UINT32                ui32AllocatedOffset,
+	SERVER_MMU_CONTEXT      * psServerMMUContext,
+	DEVMEM_MEMDESC          * psFWMemContextMemDesc,
+	IMG_INT32                 i32Priority,
+	IMG_INT32                 ui32MaxDeadlineMS,
+	RGX_COMMON_CONTEXT_INFO * psInfo,
+	RGX_SERVER_TQ_TDM_DATA  * psTDMData,
+	IMG_UINT32                ui32CCBAllocSizeLog2,
+	IMG_UINT32                ui32CCBMaxAllocSizeLog2,
+	IMG_UINT32                ui32ContextFlags,
+	IMG_UINT64                ui64RobustnessAddress)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psTDMData->psBufferSyncContext =
+			pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+			                               "rogue-tdm");
+	if (IS_ERR(psTDMData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to create buffer_sync context (err=%ld)",
+				__func__, PTR_ERR(psTDMData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = FWCommonContextAllocate(
+			psConnection,
+			psDeviceNode,
+			REQ_TYPE_TQ_TDM,
+			RGXFWIF_DM_TDM,
+			psServerMMUContext,
+			psAllocatedMemDesc,
+			ui32AllocatedOffset,
+			psFWMemContextMemDesc,
+			NULL,
+			ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TDM_CCB_SIZE_LOG2,
+			ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TDM_CCB_MAX_SIZE_LOG2,
+			ui32ContextFlags,
+			i32Priority,
+			ui32MaxDeadlineMS,
+			ui64RobustnessAddress,
+			psInfo,
+			&psTDMData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	psTDMData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(psTDMData->psBufferSyncContext);
+	psTDMData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+static PVRSRV_ERROR _DestroyTDMTransferContext(
+	RGX_SERVER_TQ_TDM_DATA  * psTDMData,
+	PVRSRV_DEVICE_NODE      * psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(
+		psDeviceNode,
+		psTDMData->psServerCommonContext,
+		RGXFWIF_DM_TDM,
+		PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free it's resources */
+	FWCommonContextFree(psTDMData->psServerCommonContext);
+	psTDMData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(psTDMData->psBufferSyncContext);
+	psTDMData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+
+/*
+ * PVRSRVCreateTransferContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXTDMCreateTransferContextKM(
+	CONNECTION_DATA            * psConnection,
+	PVRSRV_DEVICE_NODE         * psDeviceNode,
+	IMG_INT32                    i32Priority,
+	IMG_UINT32                   ui32FrameworkCommandSize,
+	IMG_PBYTE                    pabyFrameworkCommand,
+	IMG_HANDLE                   hMemCtxPrivData,
+	IMG_UINT32                   ui32PackedCCBSizeU88,
+	IMG_UINT32                   ui32ContextFlags,
+	IMG_UINT64                   ui64RobustnessAddress,
+	IMG_UINT32                   ui32MaxDeadlineMS,
+	RGX_SERVER_TQ_TDM_CONTEXT ** ppsTransferContext)
+{
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext;
+
+	DEVMEM_MEMDESC          * psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_RGXDEV_INFO      * psDevInfo = psDeviceNode->pvDevice;
+	RGX_COMMON_CONTEXT_INFO   sInfo = {NULL};
+	PVRSRV_ERROR              eError = PVRSRV_OK;
+
+	/* Allocate the server side structure */
+	*ppsTransferContext = NULL;
+	psTransferContext = OSAllocZMem(sizeof(*psTransferContext));
+	if (psTransferContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW transfer context, this has the TDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWTDMCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwTransferContext",
+			&psTransferContext->psFWTransferContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwtransfercontext;
+	}
+
+	eError = OSLockCreate(&psTransferContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_lockcreate;
+	}
+
+	psTransferContext->psDeviceNode = psDeviceNode;
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psTransferContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psTransferContext->psFWFrameworkMemDesc,
+				pabyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psTransferContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _CreateTDMTransferContext(psConnection,
+	                                   psDeviceNode,
+	                                   psTransferContext->psFWTransferContextMemDesc,
+	                                   offsetof(RGXFWIF_FWTDMCONTEXT, sTDMContext),
+	                                   hMemCtxPrivData,
+	                                   psFWMemContextMemDesc,
+	                                   i32Priority,
+	                                   ui32MaxDeadlineMS,
+	                                   &sInfo,
+	                                   &psTransferContext->sTDMData,
+									   U32toU8_Unpack1(ui32PackedCCBSizeU88),
+									   U32toU8_Unpack2(ui32PackedCCBSizeU88),
+	                                   ui32ContextFlags,
+	                                   ui64RobustnessAddress);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tdmtransfercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTDM(psDevInfo, &psTransferContext->sWorkEstData);
+	}
+#endif
+
+	SyncAddrListInit(&psTransferContext->sSyncAddrListFence);
+	SyncAddrListInit(&psTransferContext->sSyncAddrListUpdate);
+
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTDMCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+	*ppsTransferContext = psTransferContext;
+
+	return PVRSRV_OK;
+
+fail_tdmtransfercontext:
+fail_frameworkcopy:
+	if (psTransferContext->psFWFrameworkMemDesc != NULL)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	OSLockDestroy(psTransferContext->hLock);
+fail_lockcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+fail_fwtransfercontext:
+	OSFreeMem(psTransferContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*ppsTransferContext = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMDestroyTransferContextKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		RGXFWIF_FWTDMCONTEXT	*psFWTransferContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psTransferContext->psFWTransferContextMemDesc,
+										  (void **)&psFWTransferContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to map firmware transfer context (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+		RGXFwSharedMemCacheOpValue(psFWTransferContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+		ui32WorkEstCCBSubmitted = psFWTransferContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psTransferContext->psFWTransferContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psTransferContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					 __func__, ui32WorkEstCCBSubmitted,
+					 psTransferContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_remove_node(&(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+
+
+	eError = _DestroyTDMTransferContext(&psTransferContext->sTDMData,
+	                                    psTransferContext->psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_destroyTDM;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstDeInitTDM(psDevInfo, &psTransferContext->sWorkEstData);
+	}
+#endif
+
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+
+	SyncAddrListDeinit(&psTransferContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psTransferContext->sSyncAddrListUpdate);
+
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+
+	OSLockDestroy(psTransferContext->hLock);
+
+	OSFreeMem(psTransferContext);
+
+	return PVRSRV_OK;
+
+fail_destroyTDM:
+
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTDMCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+ * PVRSRVSubmitTQ3DKickKM
+ */
+/* Old bridge call for backwards compatibility. */
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransferKM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * paui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32                  ui32TDMCharacteristic1,
+	IMG_UINT32                  ui32TDMCharacteristic2,
+	IMG_UINT64                  ui64DeadlineInus)
+{
+	return PVRSRVRGXTDMSubmitTransfer3KM(psTransferContext,
+										 ui32PDumpFlags,
+										 ui32ClientUpdateCount,
+										 pauiClientUpdateUFODevVarBlock,
+										 paui32ClientUpdateSyncOffset,
+										 paui32ClientUpdateValue,
+										 iCheckFence,
+										 iUpdateTimeline,
+										 piUpdateFence,
+										 szUpdateFenceName,
+										 PVRSRV_NO_FENCE,
+										 ui32FWCommandSize,
+										 pui8FWCommand,
+										 ui32ExtJobRef,
+										 ui32SyncPMRCount,
+										 paui32SyncPMRFlags,
+										 ppsSyncPMRs,
+										 ui32TDMCharacteristic1,
+										 ui32TDMCharacteristic2,
+										 ui64DeadlineInus);
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransfer3KM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	PVRSRV_FENCE                iExportFenceToSignal,
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * paui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32                  ui32TDMCharacteristic1,
+	IMG_UINT32                  ui32TDMCharacteristic2,
+	IMG_UINT64                  ui64DeadlineInus)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	RGX_CCB_CMD_HELPER_DATA *psCmdHelper;
+	PRGXFWIF_UFO_ADDR * pauiIntFenceUFOAddress   = NULL;
+	PRGXFWIF_UFO_ADDR * pauiIntUpdateUFOAddress  = NULL;
+	IMG_UINT32          ui32IntClientFenceCount  = 0;
+	IMG_UINT32        * paui32IntUpdateValue     = paui32ClientUpdateValue;
+	IMG_UINT32          ui32IntClientUpdateCount = ui32ClientUpdateCount;
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE iUpdateFence = PVRSRV_NO_FENCE;
+	PVRSRV_RGXDEV_INFO  *psDevInfo = FWCommonContextGetRGXDevInfo(psTransferContext->sTDMData.psServerCommonContext);
+	RGX_CLIENT_CCB      *psClientCCB = FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext);
+	IMG_UINT32          ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+
+	IMG_UINT64          ui64FBSCEntryMask = 0;
+
+	IMG_BOOL bCCBStateOpen;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32 ui32CmdOffset = 0;
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTransfer = {0};
+	IMG_UINT32 ui32TDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32TDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32TDMCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+#endif
+
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT psExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(SUPPORT_WORKLOAD_ESTIMATION)
+	PVR_UNREFERENCED_PARAMETER(ui32TDMCharacteristic1);
+	PVR_UNREFERENCED_PARAMETER(ui32TDMCharacteristic2);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	if (ui32SyncPMRCount != 0)
+	{
+		if (!ppsSyncPMRs)
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* We can't allocate the required amount of stack space on all consumer architectures */
+	psCmdHelper = OSAllocMem(sizeof(RGX_CCB_CMD_HELPER_DATA));
+	if (psCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_allochelper;
+	}
+
+
+	/*
+		Init the command helper commands for all the prepares
+	*/
+	{
+		IMG_CHAR *pszCommandName;
+		RGXFWIF_CCB_CMD_TYPE eType;
+#if defined(SUPPORT_BUFFER_SYNC)
+		struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+
+		pszCommandName = "TQ-TDM";
+
+		if (ui32FWCommandSize == 0)
+		{
+			/* A NULL CMD for TDM is used to append updates to a non finished
+			 * FW command. bCCBStateOpen is used in case capture range is
+			 * entered on this command, to not drain CCB up to the Roff for this
+			 * command, but the finished command prior to this.
+			 */
+			bCCBStateOpen = IMG_TRUE;
+			eType = RGXFWIF_CCB_CMD_TYPE_NULL;
+		}
+		else
+		{
+			bCCBStateOpen = IMG_FALSE;
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_TDM;
+		}
+
+#if defined(SUPPORT_BUFFER_SYNC)
+		psBufferSyncContext = psTransferContext->sTDMData.psBufferSyncContext;
+#endif
+
+		eError = SyncAddrListPopulate(&psTransferContext->sSyncAddrListFence,
+		                              0,
+		                              NULL,
+		                              NULL);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_populate_sync_addr_list;
+		}
+
+		eError = SyncAddrListPopulate(&psTransferContext->sSyncAddrListUpdate,
+		                              ui32ClientUpdateCount,
+		                              pauiClientUpdateUFODevVarBlock,
+		                              paui32ClientUpdateSyncOffset);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_populate_sync_addr_list;
+		}
+		paui32IntUpdateValue    = paui32ClientUpdateValue;
+		pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+
+
+		if (ui32SyncPMRCount)
+		{
+#if defined(SUPPORT_BUFFER_SYNC)
+			int err;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling pvr_buffer_sync_resolve_and_create_fences", __func__));
+			err = pvr_buffer_sync_resolve_and_create_fences(psBufferSyncContext,
+			                                                psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                                ui32SyncPMRCount,
+			                                                ppsSyncPMRs,
+			                                                paui32SyncPMRFlags,
+			                                                &ui32BufferFenceSyncCheckpointCount,
+			                                                &apsBufferFenceSyncCheckpoints,
+			                                                &psBufferUpdateSyncCheckpoint,
+			                                                &psBufferSyncData);
+			if (err)
+			{
+				switch (err)
+				{
+					case -EINTR:
+						eError = PVRSRV_ERROR_RETRY;
+						break;
+					case -ENOMEM:
+						eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+						break;
+					default:
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						break;
+				}
+
+				if (eError != PVRSRV_ERROR_RETRY)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   pvr_buffer_sync_resolve_and_create_fences failed (%s)", __func__, PVRSRVGetErrorString(eError)));
+				}
+				goto fail_resolve_input_fence;
+			}
+
+			/* Append buffer sync fences */
+			if (ui32BufferFenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints to TQ Fence (&psTransferContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>)...", __func__, ui32BufferFenceSyncCheckpointCount, (void*)&psTransferContext->sSyncAddrListFence , (void*)pauiIntFenceUFOAddress));
+				SyncAddrListAppendAndDeRefCheckpoints(&psTransferContext->sSyncAddrListFence,
+				                                      ui32BufferFenceSyncCheckpointCount,
+				                                      apsBufferFenceSyncCheckpoints);
+				if (!pauiIntFenceUFOAddress)
+				{
+					pauiIntFenceUFOAddress = psTransferContext->sSyncAddrListFence.pasFWAddrs;
+				}
+				ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+			}
+
+			if (psBufferUpdateSyncCheckpoint)
+			{
+				/* Append the update (from output fence) */
+				SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+				                              1,
+				                              &psBufferUpdateSyncCheckpoint);
+				if (!pauiIntUpdateUFOAddress)
+				{
+					pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+				}
+				ui32IntClientUpdateCount++;
+			}
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+			PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+			PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __func__, ui32SyncPMRCount));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto fail_populate_sync_addr_list;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+		}
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(psTransferContext->psDeviceNode->hSyncCheckpointContext,
+		                                    iCheckFence,
+		                                    &ui32FenceSyncCheckpointCount,
+		                                    &apsFenceSyncCheckpoints,
+		                                    &uiCheckFenceUID,
+		                                    ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_resolve_input_fence;
+		}
+#if defined(TDM_CHECKPOINT_DEBUG)
+		{
+			IMG_UINT32 ii;
+			for (ii=0; ii<32; ii++)
+			{
+				PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints +  ii);
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint)); //psFenceSyncCheckpoints[ii]));
+			}
+		}
+#endif
+		/* Create the output fence (if required) */
+		if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+		{
+			eError = SyncCheckpointCreateFence(psTransferContext->psDeviceNode,
+			                                   szUpdateFenceName,
+			                                   iUpdateTimeline,
+			                                   psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                   &iUpdateFence,
+			                                   &uiUpdateFenceUID,
+			                                   &pvUpdateFenceFinaliseData,
+			                                   &psUpdateSyncCheckpoint,
+			                                   (void*)&psFenceTimelineUpdateSync,
+			                                   &ui32FenceTimelineUpdateValue,
+			                                   ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				goto fail_create_output_fence;
+			}
+
+			/* Resolve the iExportFenceToSignal (if required) */
+			if (iExportFenceToSignal != PVRSRV_NO_FENCE)
+			{
+				IMG_UINT32 iii;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+				eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+														  psTransferContext->psDeviceNode->hSyncCheckpointContext,
+														  &psExportFenceSyncCheckpoint,
+														  ui32PDumpFlags);
+				if (eError != PVRSRV_OK)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), psExportFenceSyncCheckpoint));
+					goto fail_resolve_export_fence;
+				}
+
+				/* Check that the export fence was not also included as part of the
+				 * check fence (which is an error and would lead to a stalled kick).
+				 */
+				for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+				{
+					if (apsFenceSyncCheckpoints[iii] == psExportFenceSyncCheckpoint)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, psExportFenceSyncCheckpoint));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+						goto fail_check_fence_includes_export_fence;
+					}
+				}
+			}
+
+			/* Append the sync prim update for the timeline (if required) */
+			if (psFenceTimelineUpdateSync)
+			{
+				IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+				/* Allocate memory to hold the list of update values (including our timeline update) */
+				pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				if (!pui32IntAllocatedUpdateValues)
+				{
+					/* Failed to allocate memory */
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto fail_alloc_update_values_mem;
+				}
+				OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				/* Copy the update values into the new memory, then append our timeline update value */
+				if (paui32IntUpdateValue)
+				{
+					OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+				}
+				/* Now set the additional update value */
+				pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+				*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+				ui32IntClientUpdateCount++;
+#if defined(TDM_CHECKPOINT_DEBUG)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Now append the timeline sync prim addr to the transfer context update list */
+				SyncAddrListAppendSyncPrim(&psTransferContext->sSyncAddrListUpdate,
+				                           psFenceTimelineUpdateSync);
+#if defined(TDM_CHECKPOINT_DEBUG)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+				paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+			}
+		}
+
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			/* Append the checks (from input fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to TQ Fence (&psTransferContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psTransferContext->sSyncAddrListFence));
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntFenceUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListFence,
+										  ui32FenceSyncCheckpointCount,
+										  apsFenceSyncCheckpoints);
+			if (!pauiIntFenceUFOAddress)
+			{
+				pauiIntFenceUFOAddress = psTransferContext->sSyncAddrListFence.pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+		if (psUpdateSyncCheckpoint)
+		{
+			/* Append the update (from output fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to TQ Update (&psTransferContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)&psTransferContext->sSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+			                              1,
+			                              &psUpdateSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		if (psExportFenceSyncCheckpoint)
+		{
+			/* Append the update (from export fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Transfer CDM Update (&psTransferContext->sSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)&psTransferContext->sSyncAddrListUpdate , (void*)psExportFenceSyncCheckpoint));
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+										  1,
+										  &psExportFenceSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+#if (ENABLE_TDM_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping TDM fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TDM fence syncs (&psTransferContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psTransferContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TDM update syncs (&psTransferContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psTransferContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+		RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psTransferContext->psDeviceNode->pvDevice,
+		                          &pPreAddr,
+		                          &pPostAddr,
+		                          &pRMWUFOAddr);
+
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+		/*
+		 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+		 * in other words, take the value and set it to zero afterwards.
+		 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+		 * as it must be ready at the time of context activation.
+		 */
+		{
+			eError = RGXExtractFBSCEntryMaskFromMMUContext(psTransferContext->psDeviceNode,
+														   FWCommonContextGetServerMMUCtx(psTransferContext->sTDMData.psServerCommonContext),
+														   &ui64FBSCEntryMask);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+				goto fail_invalfbsc;
+			}
+		}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			sWorkloadCharacteristics.sTransfer.ui32Characteristic1 = ui32TDMCharacteristic1;
+			sWorkloadCharacteristics.sTransfer.ui32Characteristic2 = ui32TDMCharacteristic2;
+
+			/* Prepare workload estimation */
+			WorkEstPrepare(psDeviceNode->pvDevice,
+					&psTransferContext->sWorkEstData,
+					&psTransferContext->sWorkEstData.uWorkloadMatchingData.sTransfer.sDataTDM,
+					eType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTransfer);
+		}
+#endif
+
+		/*
+			Create the command helper data for this command
+		*/
+		RGXCmdHelperInitCmdCCB(psDevInfo,
+		                       psClientCCB,
+		                       ui64FBSCEntryMask,
+		                       ui32IntClientFenceCount,
+		                       pauiIntFenceUFOAddress,
+		                       NULL,
+		                       ui32IntClientUpdateCount,
+		                       pauiIntUpdateUFOAddress,
+		                       paui32IntUpdateValue,
+		                       ui32FWCommandSize,
+		                       pui8FWCommand,
+		                       &pPreAddr,
+		                       &pPostAddr,
+		                       &pRMWUFOAddr,
+		                       eType,
+		                       ui32ExtJobRef,
+		                       ui32IntJobRef,
+		                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                       &sWorkloadKickDataTransfer,
+#else /* SUPPORT_WORKLOAD_ESTIMATION */
+		                       NULL,
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+		                       pszCommandName,
+		                       bCCBStateOpen,
+		                       psCmdHelper);
+
+	}
+
+	/*
+		Acquire space for all the commands in one go
+	*/
+
+	eError = RGXCmdHelperAcquireCmdCCB(1, psCmdHelper);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcmdacquire;
+	}
+
+
+	/*
+		We should acquire the kernel CCB(s) space here as the schedule could fail
+		and we would have to roll back all the syncs
+	*/
+
+	/*
+		Only do the command helper release (which takes the server sync
+		operations if the acquire succeeded
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		ui32CmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext));
+	}
+#endif
+	RGXCmdHelperReleaseCmdCCB(1,
+	                          psCmdHelper,
+	                          "TQ_TDM",
+	                          FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext).ui32Addr);
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* The following is used to determine the offset of the command header containing
+		   the workload estimation data so that can be accessed when the KCCB is read */
+		ui32TDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(psCmdHelper);
+
+		ui32TDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32CmdOffset < ui32TDMCmdOffsetWrapCheck)
+		{
+			ui32TDMWorkloadDataRO = ui32CmdOffset;
+		}
+		else
+		{
+			ui32TDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/*
+		Even if we failed to acquire the client CCB space we might still need
+		to kick the HW to process a padding packet to release space for us next
+		time round
+	*/
+	{
+		RGXFWIF_KCCB_CMD sTDMKCCBCmd;
+		IMG_UINT32 ui32FWAddr = FWCommonContextGetFWAddress(
+		        psTransferContext->sTDMData.psServerCommonContext).ui32Addr;
+
+		/* Construct the kernel 3D CCB command. */
+		sTDMKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		sTDMKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced
+			 * when the KCCB command reaches the FW */
+			sTDMKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32TDMWorkloadDataRO + ui32TDMCmdHeaderOffset;
+		}
+#endif
+
+		/* HTBLOGK(HTB_SF_MAIN_KICK_TDM, */
+		/*		s3DKCCBCmd.uCmdData.sCmdKickData.psContext, */
+		/*		ui323DCmdOffset); */
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext).ui32Addr,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQTDM,
+		                  iCheckFence,
+		                  iUpdateFence,
+		                  iUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  uiUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = RGXScheduleCommandWithoutPowerLock(psDeviceNode->pvDevice,
+			                             RGXFWIF_DM_TDM,
+			                             & sTDMKCCBCmd,
+			                             ui32PDumpFlags);
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXTDMSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError));
+			goto fail_2dcmdacquire;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWAddr, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQTDM);
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointSignalNoHW(psExportFenceSyncCheckpoint);
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	* piUpdateFence = iUpdateFence;
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, iUpdateFence, pvUpdateFenceFinaliseData,
+		                            psUpdateSyncCheckpoint, szUpdateFenceName);
+	}
+
+	OSFreeMem(psCmdHelper);
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+/*
+	No resources are created in this function so there is nothing to free
+	unless we had to merge syncs.
+	If we fail after the client CCB acquire there is still nothing to do
+	as only the client CCB release will modify the client CCB
+*/
+fail_2dcmdacquire:
+fail_acquirepowerlock:
+fail_3dcmdacquire:
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+fail_invalfbsc:
+#endif
+	SyncAddrListRollbackCheckpoints(psTransferContext->psDeviceNode, &psTransferContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psTransferContext->psDeviceNode, &psTransferContext->sSyncAddrListUpdate);
+
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+fail_alloc_update_values_mem:
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+fail_check_fence_includes_export_fence:
+fail_resolve_export_fence:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+
+fail_resolve_input_fence:
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+fail_populate_sync_addr_list:
+	OSFreeMem(psCmdHelper);
+fail_allochelper:
+
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	OSLockRelease(psTransferContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+	IMG_UINT32                 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD  sKCCBCmd;
+	PVRSRV_ERROR      eError;
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* Schedule the firmware command */
+	sKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE;
+	sKCCBCmd.uCmdData.sWriteOffsetUpdateData.psContext = FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psTransferContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_TDM,
+		                            &sKCCBCmd,
+		                            ui32PDumpFlags);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to schedule the FW command %d (%s)",
+				__func__, eError, PVRSRVGETERRORSTRING(eError)));
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                      RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+                                                      IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	if (psTransferContext->sTDMData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psTransferContext->sTDMData.psServerCommonContext,
+		                            psConnection,
+		                            psTransferContext->psDeviceNode->pvDevice,
+		                            i32Priority,
+		                            RGXFWIF_DM_TDM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority (%s)", __func__, PVRSRVGetErrorString(eError)));
+
+			OSLockRelease(psTransferContext->hLock);
+			return eError;
+		}
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPropertyKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+													  RGX_CONTEXT_PROPERTY eContextProperty,
+													  IMG_UINT64 ui64Input,
+													  IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psTransferContext->hLock);
+			eError = FWCommonContextSetFlags(psTransferContext->sTDMData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			OSLockRelease(psTransferContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpTDMTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                              void *pvDumpDebugFile,
+                              IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hTDMCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTDMCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_TDM_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_TDM_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerTransferCtx->sTDMData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTDMCtxListLock);
+}
+
+
+IMG_UINT32 CheckForStalledClientTDMTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hTDMCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTDMCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_TDM_CONTEXT *psCurrentServerTransferCtx =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_TDM_CONTEXT, sListNode);
+
+		if (CheckStalledClientCommonContext(
+				psCurrentServerTransferCtx->sTDMData.psServerCommonContext, RGX_KICK_TYPE_DM_TDM_2D)
+				== PVRSRV_ERROR_CCCB_STALLED) {
+			ui32ContextBitMask = RGX_KICK_TYPE_DM_TDM_2D;
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTDMCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/**************************************************************************//**
+ End of file (rgxtdmtransfer.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.h
new file mode 100644
index 000000000000..b0f003cf3ad3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@File           rgxtdmtransfer.h
+@Title          RGX Transfer queue 2 Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Transfer queue Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTDMTRANSFER_H)
+#define RGXTDMTRANSFER_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "connection_server.h"
+
+typedef struct _RGX_SERVER_TQ_TDM_CONTEXT_ RGX_SERVER_TQ_TDM_CONTEXT;
+
+
+PVRSRV_ERROR PVRSRVRGXTDMCreateTransferContextKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	IMG_INT32                   i32Priority,
+	IMG_UINT32                  ui32FrameworkCommandSize,
+	IMG_PBYTE                   pabyFrameworkCommand,
+	IMG_HANDLE                  hMemCtxPrivData,
+	IMG_UINT32					ui32PackedCCBSizeU88,
+	IMG_UINT32                  ui32ContextFlags,
+	IMG_UINT64					ui64RobustnessAddress,
+	IMG_UINT32                  ui32MaxDeadlineMS,
+	RGX_SERVER_TQ_TDM_CONTEXT **ppsTransferContext);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMDestroyTransferContextKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransferKM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * pui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32					ui32TDMCharacteristic1,
+	IMG_UINT32					ui32TDMCharacteristic2,
+	IMG_UINT64					ui64DeadlineInus);
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransfer3KM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	PVRSRV_FENCE                iExportFenceToSignal,
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * pui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32					ui32TDMCharacteristic1,
+	IMG_UINT32					ui32TDMCharacteristic2,
+	IMG_UINT64					ui64DeadlineInus);
+
+PVRSRV_ERROR PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+	IMG_UINT32                 ui32PDumpFlags);
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDeviceNode,
+												   RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+												   IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPropertyKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+													 RGX_CONTEXT_PROPERTY eContextProperty,
+													 IMG_UINT64 ui64Input,
+													 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of TDM transfer contexts on this device */
+void DumpTDMTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                              void *pvDumpDebugFile,
+                              IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client transfer contexts are stalled */
+IMG_UINT32 CheckForStalledClientTDMTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+
+#endif /* RGXTDMTRANSFER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.c
new file mode 100644
index 000000000000..669ed230d783
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.c
@@ -0,0 +1,791 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific time correlation and calibration routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific time correlation and calibration routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxtimecorr.h"
+#include "rgxfwutils.h"
+#include "htbserver.h"
+#include "pvrsrv_apphint.h"
+#include "rgxpower.h"
+
+/******************************************************************************
+ *
+ * - A calibration period is started on power-on and after a DVFS transition,
+ *   and it's closed before a power-off and before a DVFS transition
+ *   (so power-on -> dfvs -> dvfs -> power-off , power on -> dvfs -> dvfs...,
+ *   where each arrow is a calibration period).
+ *
+ * - The timers on the Host and on the FW are correlated at the beginning of
+ *   each period together with the current GPU frequency.
+ *
+ * - Correlation and calibration are also done at regular intervals using
+ *   a best effort approach.
+ *
+ *****************************************************************************/
+
+/*
+	AppHint interfaces
+*/
+
+static PVRSRV_ERROR _SetClock(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 ui32Value)
+{
+	static __maybe_unused const char* const apszClocks[] = {
+		"mono", "mono_raw", "sched"
+	};
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	if (ui32Value >= RGXTIMECORR_CLOCK_LAST)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid clock source type (%u)", ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING,
+	        "Setting time correlation clock from \"%s\" to \"%s\"",
+	        apszClocks[psDevInfo->ui32ClockSource],
+	        apszClocks[ui32Value]));
+
+	/* PVRSRVPowerLock() fails only when power is off. */
+	if (PVRSRVPowerLock((PVRSRV_DEVICE_NODE *) psDeviceNode) == PVRSRV_OK)
+	{
+		RGXTimeCorrEnd((PVRSRV_DEVICE_NODE *) psDeviceNode,
+		               RGXTIMECORR_EVENT_CLOCK_CHANGE);
+
+		psDevInfo->ui32ClockSource = ui32Value;
+
+		RGXTimeCorrBegin((PVRSRV_DEVICE_NODE *) psDeviceNode,
+		                 RGXTIMECORR_EVENT_CLOCK_CHANGE);
+
+		PVRSRVPowerUnlock((PVRSRV_DEVICE_NODE *)psDeviceNode);
+	}
+	else
+	{
+		/* Set the new clock source without updating the time correlation
+		 * data. This is going to be accounted for during the next power up. */
+		psDevInfo->ui32ClockSource = ui32Value;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _GetClock(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 *pui32Value)
+{
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	*pui32Value =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource;
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return PVRSRV_OK;
+}
+
+void RGXTimeCorrInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_SecondaryOSClockSource,
+	                                    _GetClock, _SetClock, psDeviceNode,
+	                                   NULL);
+}
+
+/*
+	End of AppHint interface
+*/
+
+IMG_UINT64 RGXTimeCorrGetClockns64(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT64 ui64Clock;
+
+	switch (((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource) {
+		case RGXTIMECORR_CLOCK_MONO:
+			return ((void) OSClockMonotonicns64(&ui64Clock), ui64Clock);
+		case RGXTIMECORR_CLOCK_MONO_RAW:
+			return OSClockMonotonicRawns64();
+		case RGXTIMECORR_CLOCK_SCHED:
+			return OSClockns64();
+		default:
+			PVR_ASSERT(IMG_FALSE);
+			return 0;
+	}
+}
+
+IMG_UINT64 RGXTimeCorrGetClockus64(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 rem;
+	return OSDivide64r64(RGXTimeCorrGetClockns64(psDeviceNode), 1000, &rem);
+}
+
+static IMG_UINT64 RGXTimeGetDeviceTimestampInTicks(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT64 ui64DeviceTimestamp;
+
+	/* In case of powerlock acquire fails or device is powered off, return
+	 * last device power off timestamp plus ticks corresponding to device off time. */
+	ui64DeviceTimestamp = psDeviceNode->ui64LastDeviceOffTimestamp +
+			       RGXTimeCorrDeltaOSNsToDeltaCR(psDeviceNode,
+						OSClockns64() - psDeviceNode->ui64LastDeviceOffHostTimestampNs);
+
+	if (PVRSRV_OK == PVRSRVPowerLock(psDeviceNode))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+
+		if (PVRSRVIsDevicePowered(psDeviceNode))
+		{
+			ui64DeviceTimestamp = RGXReadHWTimerReg(psDevInfo) + psGpuUtilFW->i64DeviceTimestampOffset;
+		}
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return ui64DeviceTimestamp;
+}
+
+void RGXGetTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+							RGXFWIF_TIME_CORR *psTimeCorrs,
+							IMG_UINT32 ui32NumOut)
+{
+	PVRSRV_RGXDEV_INFO    *psDevInfo     = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW   *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32CurrentIndex;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32CurrentIndex = psGpuUtilFW->ui32TimeCorrSeqCount;
+
+	RGXFwSharedMemCacheOpExec(&psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32CurrentIndex - ui32NumOut)],
+	                          sizeof(psGpuUtilFW->sTimeCorr[0]) * ui32NumOut,
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+	while (ui32NumOut--)
+	{
+		*(psTimeCorrs++) = psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32CurrentIndex)];
+		ui32CurrentIndex--;
+	}
+}
+
+static __maybe_unused const IMG_CHAR* _EventToString(RGXTIMECORR_EVENT eEvent)
+{
+	switch (eEvent)
+	{
+		case RGXTIMECORR_EVENT_POWER:
+			return "power";
+		case RGXTIMECORR_EVENT_DVFS:
+			return "dvfs";
+		case RGXTIMECORR_EVENT_PERIODIC:
+			return "periodic";
+		case RGXTIMECORR_EVENT_CLOCK_CHANGE:
+			return "clock source";
+		default:
+			return "n/a";
+	}
+}
+
+static inline IMG_UINT32 _RGXGetSystemLayerGPUClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+
+	return psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+}
+
+static inline IMG_UINT32 _RGXGetEstimatedGPUClockSpeed(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_GPU_DVFS_TABLE *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+
+	psTrackingData = &psGpuDVFSTable->asTrackingData[psGpuDVFSTable->ui32FreqIndex];
+
+	return psTrackingData->ui32EstCoreClockSpeed;
+}
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+static inline void _DumpTimerCorrelationHistory(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_GPU_DVFS_TABLE *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	IMG_UINT32 i = psGpuDVFSTable->ui32HistoryIndex;
+
+	PVR_DPF((PVR_DBG_ERROR, "Dumping history of timer correlation data (latest first):"));
+
+	do
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "  Begin times: OS %" IMG_UINT64_FMTSPEC ", CR %" IMG_UINT64_FMTSPEC ", "
+				 "End times: OS %" IMG_UINT64_FMTSPEC ", CR %" IMG_UINT64_FMTSPEC ", "
+				 "Core clk %u, Estimated clk %u",
+				 psGpuDVFSTable->asTrackingHistory[i].ui64BeginOSTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64BeginCRTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64EndOSTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64EndCRTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui32CoreClockSpeed,
+				 psGpuDVFSTable->asTrackingHistory[i].ui32EstCoreClockSpeed));
+
+		i = (i - 1) % RGX_GPU_FREQ_TRACKING_SIZE;
+
+	} while (i != psGpuDVFSTable->ui32HistoryIndex);
+}
+#endif
+
+static void _RGXMakeTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32NewSeqCount;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	RGXFWIF_TIME_CORR sTimeCorr = {0};
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32NewSeqCount = psGpuUtilFW->ui32TimeCorrSeqCount + 1;
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32NewSeqCount)], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32NewSeqCount)];
+
+	/*
+	 * The following reads must be done as close together as possible, because
+	 * they represent the same current time sampled from different clock sources.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		if (OSClockMonotonicns64(&sTimeCorr.ui64OSMonoTimeStamp) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "_RGXMakeTimeCorrData: System Monotonic Clock not available."));
+			PVR_ASSERT(0);
+		}
+	}
+
+	sTimeCorr.ui64CRTimeStamp = RGXReadHWTimerReg(psDevInfo);
+	sTimeCorr.ui64OSTimeStamp =
+		((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource == RGXTIMECORR_CLOCK_MONO
+		? sTimeCorr.ui64OSMonoTimeStamp
+		: RGXTimeCorrGetClockns64(psDeviceNode);
+	sTimeCorr.ui32CoreClockSpeed = _RGXGetEstimatedGPUClockSpeed(psDevInfo);
+	sTimeCorr.ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(sTimeCorr.ui32CoreClockSpeed);
+
+	if (sTimeCorr.ui64CRDeltaToOSDeltaKNs == 0)
+	{
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+		_DumpTimerCorrelationHistory(psDevInfo);
+#endif
+
+		/* Revert to original clock speed (error already printed) */
+		sTimeCorr.ui32CoreClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+		sTimeCorr.ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(sTimeCorr.ui32CoreClockSpeed);
+	}
+
+	OSCachedMemCopy(psTimeCorr, &sTimeCorr, sizeof(sTimeCorr));
+	/* Make sure the values are written to memory before updating the index of the current entry */
+	OSWriteMemoryBarrier(psTimeCorr);
+	RGXFwSharedMemCacheOpPtr(psTimeCorr, FLUSH);
+
+
+	/* Update the index of the current entry in the timer correlation array */
+	psGpuUtilFW->ui32TimeCorrSeqCount = ui32NewSeqCount;
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, FLUSH);
+
+
+	if (RGXTIMECORR_EVENT_POWER == eEvent && psDeviceNode->ui64LastDeviceOffTimestamp)
+	{
+		/* Capture effective timestamp offset on device power on */
+		psGpuUtilFW->i64DeviceTimestampOffset = psDeviceNode->ui64LastDeviceOffTimestamp - RGXReadHWTimerReg(psDevInfo) +
+								RGXTimeCorrDeltaOSNsToDeltaCR(psDeviceNode,
+									OSClockns64() - psDeviceNode->ui64LastDeviceOffHostTimestampNs);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "Timer correlation data (post %s event): OS %" IMG_UINT64_FMTSPEC " ns, "
+	         "CR %" IMG_UINT64_FMTSPEC ", GPU freq. %u Hz (given as %u Hz)",
+	         _EventToString(eEvent),
+	         sTimeCorr.ui64OSTimeStamp,
+	         sTimeCorr.ui64CRTimeStamp,
+	         RGXFWIF_ROUND_TO_KHZ(sTimeCorr.ui32CoreClockSpeed),
+	         _RGXGetSystemLayerGPUClockSpeed(psDeviceNode)));
+
+	/*
+	 * Don't log timing data to the HTB log after a power(-on) event.
+	 * Otherwise this will be logged before the HTB partition marker, breaking
+	 * the log sync grammar. This data will be automatically repeated when the
+	 * partition marker is written.
+	 */
+	HTBSyncScale(eEvent != RGXTIMECORR_EVENT_POWER,
+	             sTimeCorr.ui64OSTimeStamp,
+	             sTimeCorr.ui64CRTimeStamp,
+	             sTimeCorr.ui32CoreClockSpeed);
+}
+
+static void _RGXCheckTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+#if !defined(NO_HARDWARE) && !defined(VIRTUAL_PLATFORM) && defined(DEBUG)
+#define SCALING_FACTOR (10)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32Index;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	IMG_UINT64 ui64EstimatedTime, ui64CRTimeStamp, ui64OSTimeStamp;
+	IMG_UINT64 ui64CRTimeDiff, ui64OSTimeDiff;
+	IMG_INT64 i64Diff;
+	IMG_UINT32 ui32Ratio, ui32Remainder;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32Index = RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount);
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32Index], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32Index];
+
+	/*
+	 * The following reads must be done as close together as possible, because
+	 * they represent the same current time sampled from different clock sources.
+	 */
+	ui64CRTimeStamp = RGXReadHWTimerReg(psDevInfo);
+	ui64OSTimeStamp = RGXTimeCorrGetClockns64(psDeviceNode);
+
+	if ((ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp) < (1 << SCALING_FACTOR))
+	{
+		/*
+		 * Less than ~1us has passed since the timer correlation data was generated.
+		 * A time frame this short is probably not enough to get an estimate
+		 * of how good the timer correlation data was.
+		 * Skip calculations for the above reason and to avoid a division by 0 below.
+		 */
+		return;
+	}
+
+
+	/* Calculate an estimated timestamp based on the latest timer correlation data */
+	ui64CRTimeDiff = ui64CRTimeStamp - psTimeCorr->ui64CRTimeStamp;
+	ui64OSTimeDiff = RGXFWIF_GET_DELTA_OSTIME_NS(ui64CRTimeDiff,
+	                                             psTimeCorr->ui64CRDeltaToOSDeltaKNs);
+	ui64EstimatedTime = psTimeCorr->ui64OSTimeStamp + ui64OSTimeDiff;
+
+	/* Get difference between estimated timestamp and current timestamp, in ns */
+	i64Diff = ui64EstimatedTime - ui64OSTimeStamp;
+
+	/*
+	 * Calculate ratio between estimated time diff and real time diff:
+	 * ratio% : 100% = (OSestimate - OStimecorr) : (OSreal - OStimecorr)
+	 *
+	 * The operands are scaled down (approximately from ns to us) so at least
+	 * the divisor fits on 32 bit.
+	 */
+	ui32Ratio = OSDivide64(((ui64EstimatedTime - psTimeCorr->ui64OSTimeStamp) * 100ULL) >> SCALING_FACTOR,
+	                       (ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp) >> SCALING_FACTOR,
+	                       &ui32Remainder);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "Estimated timestamp check: diff %" IMG_INT64_FMTSPECd " ns over "
+	         "period %" IMG_UINT64_FMTSPEC " ns, estimated timer speed %u%%",
+	         i64Diff,
+	         ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp,
+	         ui32Ratio));
+
+	/* Warn if the estimated timestamp is not within +/- 1% of the current time */
+	if (ui32Ratio < 99 || ui32Ratio > 101)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Estimated timestamps generated in the last %" IMG_UINT64_FMTSPEC " ns "
+		         "were %s the real time (increasing at %u%% speed)",
+		         ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp,
+		         i64Diff > 0 ? "ahead of" : "behind",
+		         ui32Ratio));
+
+		/* Higher ratio == higher delta OS == higher delta CR == frequency higher than expected (and viceversa) */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Current GPU frequency %u Hz (given as %u Hz) is probably %s than expected",
+		         RGXFWIF_ROUND_TO_KHZ(psTimeCorr->ui32CoreClockSpeed),
+		         _RGXGetSystemLayerGPUClockSpeed(psDeviceNode),
+		         i64Diff > 0 ? "lower" : "higher"));
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psGpuDVFSTable);
+#endif
+}
+
+static inline IMG_UINT32 _RGXGPUFreqGetIndex(RGX_GPU_DVFS_TABLE *psGpuDVFSTable, IMG_UINT32 ui32CoreClockSpeed)
+{
+	IMG_UINT32 *paui32GPUFrequencies = psGpuDVFSTable->aui32GPUFrequency;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGX_GPU_DVFS_TABLE_SIZE; i++)
+	{
+		if (paui32GPUFrequencies[i] == ui32CoreClockSpeed)
+		{
+			return i;
+		}
+
+		if (paui32GPUFrequencies[i] == 0)
+		{
+			paui32GPUFrequencies[i] = ui32CoreClockSpeed;
+			return i;
+		}
+	}
+
+	i--;
+
+	/* PVR_DPF((PVR_DBG_ERROR, "GPU frequency table in the driver is full! "
+	         "Table size should be increased! Overriding last entry (%u) with %u",
+	         paui32GPUFrequencies[i], ui32CoreClockSpeed)); */
+
+	paui32GPUFrequencies[i] = ui32CoreClockSpeed;
+
+	return i;
+}
+
+static void _RGXGPUFreqCalibrationPeriodStart(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+	IMG_UINT32 ui32CoreClockSpeed, ui32Index;
+
+	IMG_UINT64 ui64CRTimestamp = RGXReadHWTimerReg(psDevInfo);
+	IMG_UINT64 ui64OSTimestamp = RGXTimeCorrGetClockus64(psDeviceNode);
+
+	psGpuDVFSTable->ui64CalibrationCRTimestamp = ui64CRTimestamp;
+	psGpuDVFSTable->ui64CalibrationOSTimestamp = ui64OSTimestamp;
+
+	ui32CoreClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+	ui32Index          = _RGXGPUFreqGetIndex(psGpuDVFSTable, ui32CoreClockSpeed);
+	psTrackingData     = &psGpuDVFSTable->asTrackingData[ui32Index];
+
+	/* Set the time needed to (re)calibrate the GPU frequency */
+	if (psTrackingData->ui32CalibrationCount == 0) /* We never met this frequency */
+	{
+		psTrackingData->ui32EstCoreClockSpeed = ui32CoreClockSpeed;
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US;
+	}
+	else if (psTrackingData->ui32CalibrationCount == 1) /* We calibrated this frequency only once */
+	{
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US;
+	}
+	else
+	{
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US;
+	}
+
+	/* Update the index to the DVFS table */
+	psGpuDVFSTable->ui32FreqIndex = ui32Index;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui32CoreClockSpeed    = ui32CoreClockSpeed;
+		psTrackingHistory->ui32EstCoreClockSpeed = psTrackingData->ui32EstCoreClockSpeed;
+		psTrackingHistory->ui64BeginCRTimestamp  = ui64CRTimestamp;
+		psTrackingHistory->ui64BeginOSTimestamp  = ui64OSTimestamp;
+		psTrackingHistory->ui64EndCRTimestamp    = 0ULL;
+		psTrackingHistory->ui64EndOSTimestamp    = 0ULL;
+	}
+#endif
+}
+
+static void _RGXGPUFreqCalibrationPeriodStop(PVRSRV_DEVICE_NODE *psDeviceNode,
+											 RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	IMG_UINT64 ui64CRTimestamp = RGXReadHWTimerReg(psDevInfo);
+	IMG_UINT64 ui64OSTimestamp = RGXTimeCorrGetClockus64(psDeviceNode);
+
+	psGpuDVFSTable->ui64CalibrationCRTimediff =
+	    ui64CRTimestamp - psGpuDVFSTable->ui64CalibrationCRTimestamp;
+	psGpuDVFSTable->ui64CalibrationOSTimediff =
+	    ui64OSTimestamp - psGpuDVFSTable->ui64CalibrationOSTimestamp;
+
+	/* Check if the current timer correlation data is good enough */
+	_RGXCheckTimeCorrData(psDeviceNode, psGpuDVFSTable);
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui64EndCRTimestamp = ui64CRTimestamp;
+		psTrackingHistory->ui64EndOSTimestamp = ui64OSTimestamp;
+	}
+#endif
+}
+
+static void _RGXGPUFreqCalibrationCalculate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            RGX_GPU_DVFS_TABLE *psGpuDVFSTable,
+                                            RGXTIMECORR_EVENT   eEvent)
+{
+#if !defined(DISABLE_GPU_FREQUENCY_CALIBRATION)
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+	IMG_UINT32 ui32EstCoreClockSpeed, ui32PrevCoreClockSpeed, ui32SysGPUClockSpeed;
+	IMG_INT32  i32Diff;
+	IMG_UINT32 ui32Remainder;
+
+	ui32SysGPUClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+	/*
+	 * Find out what the GPU frequency was in the last period.
+	 * This should return a value very close to the frequency passed by the system layer.
+	 */
+	ui32EstCoreClockSpeed =
+	    RGXFWIF_GET_GPU_CLOCK_FREQUENCY_HZ(psGpuDVFSTable->ui64CalibrationCRTimediff,
+	                                       psGpuDVFSTable->ui64CalibrationOSTimediff,
+	                                       ui32Remainder);
+
+	/* Update GPU frequency used by the driver for a given system layer frequency */
+	psTrackingData = &psGpuDVFSTable->asTrackingData[psGpuDVFSTable->ui32FreqIndex];
+
+	ui32PrevCoreClockSpeed = psTrackingData->ui32EstCoreClockSpeed;
+	psTrackingData->ui32EstCoreClockSpeed = ui32EstCoreClockSpeed;
+	psTrackingData->ui32CalibrationCount++;
+
+	i32Diff = (IMG_INT32) (ui32EstCoreClockSpeed - ui32PrevCoreClockSpeed);
+
+	if ((i32Diff < -1000000) || (i32Diff > 1000000))
+	{
+		/* Warn if the frequency changed by more than 1 MHz between recalculations */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "GPU frequency calibration of system layer frequency %u Hz (pre %s event): "
+		         "more than 1 MHz difference between old and new value "
+		         "(%u Hz -> %u Hz over %"  IMG_UINT64_FMTSPEC " us)",
+		         ui32SysGPUClockSpeed,
+		         _EventToString(eEvent),
+		         RGXFWIF_ROUND_TO_KHZ(ui32PrevCoreClockSpeed),
+		         RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed),
+		         psGpuDVFSTable->ui64CalibrationOSTimediff));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "GPU frequency calibration of system layer frequency %u Hz (pre %s event): "
+		         "%u Hz -> %u Hz done over %" IMG_UINT64_FMTSPEC " us",
+		         ui32SysGPUClockSpeed,
+		         _EventToString(eEvent),
+		         RGXFWIF_ROUND_TO_KHZ(ui32PrevCoreClockSpeed),
+		         RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed),
+		         psGpuDVFSTable->ui64CalibrationOSTimediff));
+	}
+
+	if (eEvent == RGXTIMECORR_EVENT_PERIODIC)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+
+		i32Diff = (IMG_INT32) (ui32EstCoreClockSpeed - ui32SysGPUClockSpeed);
+
+		/*
+		 * Notify the Firmware about unexpected frequency differences observed during
+		 * periodic frequency calibration events only.
+		 * Other events like PDVFS and power transitions are already likely to call
+		 * the pre/post clock callback directly to set frequencies as needed.
+		 * Platforms without PDVFS or APM need a method to correct the Firmware's
+		 * internal timing measurements.
+		 */
+		if (((i32Diff < -1000000) || (i32Diff > 1000000)) &&
+			(PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState) == PVRSRV_OK))
+		{
+			PVRSRV_ERROR eError = RGXPreClockSpeedChange((IMG_HANDLE)psDeviceNode, ePowerState);
+
+			PVR_LOG_IF_ERROR(eError, "RGXPreClockSpeedChange");
+			if (eError == PVRSRV_OK)
+			{
+				RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+
+				/* Update the internal core frequency variable and notify the Firmware of the change */
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed = RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed);
+
+				eError = RGXPostClockSpeedChange((IMG_HANDLE)psDeviceNode, ePowerState);
+				PVR_LOG_IF_ERROR(eError, "RGXPostClockSpeedChange");
+			}
+		}
+	}
+
+	/* Reset time deltas to avoid recalibrating the same frequency over and over again */
+	psGpuDVFSTable->ui64CalibrationCRTimediff = 0;
+	psGpuDVFSTable->ui64CalibrationOSTimediff = 0;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui32EstCoreClockSpeed = ui32EstCoreClockSpeed;
+		psGpuDVFSTable->ui32HistoryIndex =
+			(psGpuDVFSTable->ui32HistoryIndex + 1) % RGX_GPU_FREQ_TRACKING_SIZE;
+	}
+#endif
+
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psGpuDVFSTable);
+	PVR_UNREFERENCED_PARAMETER(eEvent);
+#endif
+}
+
+void RGXTimeCorrBegin(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_DEVICE_NODE  *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO  *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE  *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	_RGXGPUFreqCalibrationPeriodStart(psDeviceNode, psGpuDVFSTable);
+	_RGXMakeTimeCorrData(psDeviceNode, eEvent);
+}
+
+void RGXTimeCorrEnd(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_DEVICE_NODE  *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO  *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE  *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	_RGXGPUFreqCalibrationPeriodStop(psDeviceNode, psGpuDVFSTable);
+
+	if (psGpuDVFSTable->ui64CalibrationOSTimediff >= psGpuDVFSTable->ui32CalibrationPeriod)
+	{
+		_RGXGPUFreqCalibrationCalculate(psDeviceNode, psGpuDVFSTable, eEvent);
+	}
+
+	if (RGXTIMECORR_EVENT_POWER == eEvent)
+	{
+		RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+
+		/* Capture effective device power off timestamp */
+		psDeviceNode->ui64LastDeviceOffTimestamp = RGXReadHWTimerReg(psDevInfo) + psGpuUtilFW->i64DeviceTimestampOffset;
+		psDeviceNode->ui64LastDeviceOffHostTimestampNs = OSClockns64();
+	}
+}
+
+void RGXTimeCorrRestartPeriodic(IMG_HANDLE hDevHandle)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO     *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE     *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	IMG_UINT64             ui64TimeNow     = RGXTimeCorrGetClockus64(psDeviceNode);
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_DEFAULT;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	if (psGpuDVFSTable == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Required data not initialised yet", __func__));
+		return;
+	}
+
+	/* Check if it's the right time to recalibrate the GPU clock frequency */
+	if ((ui64TimeNow - psGpuDVFSTable->ui64CalibrationOSTimestamp) < psGpuDVFSTable->ui32CalibrationPeriod) return;
+
+	/* Try to acquire the powerlock, if not possible then don't wait */
+	if (PVRSRVPowerTryLock(psDeviceNode) != PVRSRV_OK) return;
+
+	/* If the GPU is off then we can't do anything */
+	PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+	if (ePowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return;
+	}
+
+	/* All checks passed, we can calibrate and correlate */
+	RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_PERIODIC);
+	RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_PERIODIC);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+/*
+	RGXTimeCorrGetClockSource
+*/
+RGXTIMECORR_CLOCK_TYPE RGXTimeCorrGetClockSource(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource;
+}
+
+/*
+	RGXTimeCorrSetClockSource
+*/
+PVRSRV_ERROR RGXTimeCorrSetClockSource(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       RGXTIMECORR_CLOCK_TYPE eClockType)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psRGXDevNode = psDeviceNode->pvDevice;
+	RGXTIMECORR_CLOCK_TYPE eLastClock = psRGXDevNode->ui32ClockSource;
+
+	eError = _SetClock(psDeviceNode, NULL, eClockType);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_SetClock");
+
+	psRGXDevNode->ui32LastClockSource = eLastClock;
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXCurrentTime(CONNECTION_DATA    *psConnection,
+                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT8           ui8TimestampType,
+                     IMG_UINT64         *pui64Time)
+{
+	RGX_QUERY_TIMESTAMP_TYPE eTimestampType = (RGX_QUERY_TIMESTAMP_TYPE)ui8TimestampType;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	switch (eTimestampType)
+	{
+		case RGX_QUERY_HOST_TIMESTAMP:
+		*pui64Time = RGXTimeCorrGetClockns64(psDeviceNode);
+		break;
+
+		case RGX_QUERY_DEVICE_TIMESTAMP:
+		*pui64Time = RGXTimeGetDeviceTimestampInTicks(psDeviceNode);
+		break;
+
+		default:
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		break;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxtimecorr.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.h
new file mode 100644
index 000000000000..9758e61909c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.h
@@ -0,0 +1,316 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX time correlation and calibration header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX time correlation and calibration routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTIMECORR_H)
+#define RGXTIMECORR_H
+
+#include "img_types.h"
+#include "device.h"
+#include "osfunc.h"
+#include "connection_server.h"
+#include "rgxdevice.h"
+
+typedef enum
+{
+	RGXTIMECORR_CLOCK_MONO,
+	RGXTIMECORR_CLOCK_MONO_RAW,
+	RGXTIMECORR_CLOCK_SCHED,
+
+	RGXTIMECORR_CLOCK_LAST
+} RGXTIMECORR_CLOCK_TYPE;
+
+typedef enum
+{
+	RGXTIMECORR_EVENT_POWER,
+	RGXTIMECORR_EVENT_DVFS,
+	RGXTIMECORR_EVENT_PERIODIC,
+	RGXTIMECORR_EVENT_CLOCK_CHANGE
+} RGXTIMECORR_EVENT;
+
+/*
+ * Calibrated GPU frequencies are rounded to the nearest multiple of 1 KHz
+ * before use, to reduce the noise introduced by calculations done with
+ * imperfect operands (correlated timers not sampled at exactly the same
+ * time, GPU CR timer incrementing only once every 256 GPU cycles).
+ * This also helps reducing the variation between consecutive calculations.
+ */
+#define RGXFWIF_CONVERT_TO_KHZ(freq)   (((freq) + 500) / 1000)
+#define RGXFWIF_ROUND_TO_KHZ(freq)    ((((freq) + 500) / 1000) * 1000)
+
+/* Constants used in different calculations */
+#define SECONDS_TO_MICROSECONDS          (1000000ULL)
+#define CRTIME_TO_CYCLES_WITH_US_SCALE   (RGX_CRTIME_TICK_IN_CYCLES * SECONDS_TO_MICROSECONDS)
+
+/*
+ * Use this macro to get a more realistic GPU core clock speed than the one
+ * given by the upper layers (used when doing GPU frequency calibration)
+ */
+#define RGXFWIF_GET_GPU_CLOCK_FREQUENCY_HZ(deltacr_us, deltaos_us, remainder) \
+    OSDivide64((deltacr_us) * CRTIME_TO_CYCLES_WITH_US_SCALE, (deltaos_us), &(remainder))
+
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetConversionFactor
+
+ @Description Generate constant used to convert a GPU time difference into
+              an OS time difference (for more info see rgx_fwif_km.h).
+
+ @Input       ui32ClockSpeed : GPU clock speed
+
+ @Return      0 on failure, conversion factor otherwise
+
+******************************************************************************/
+static inline IMG_UINT64 RGXTimeCorrGetConversionFactor(IMG_UINT32 ui32ClockSpeed)
+{
+	IMG_UINT32 ui32Remainder;
+
+	if (RGXFWIF_CONVERT_TO_KHZ(ui32ClockSpeed) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: GPU clock frequency %u is too low",
+				 __func__, ui32ClockSpeed));
+
+		return 0;
+	}
+
+	return OSDivide64r64(CRTIME_TO_CYCLES_WITH_US_SCALE << RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT,
+	                     RGXFWIF_CONVERT_TO_KHZ(ui32ClockSpeed), &ui32Remainder);
+}
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrDeltaOSNsToDeltaCR
+
+ @Description Convert OS timestamp difference in ns to device tick difference
+
+ @Input       psDeviceNode, ui64DeltaOSNs
+
+ @Return      0 on failure, deltaCR otherwise
+
+******************************************************************************/
+static inline IMG_UINT64 RGXTimeCorrDeltaOSNsToDeltaCR(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64DeltaOSNs)
+{
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	IMG_UINT32 ui32Remainder;
+
+	PVR_ASSERT(psDeviceNode && psDeviceNode->pvDevice);
+	psDevInfo = psDeviceNode->pvDevice;
+	psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	PVR_ASSERT(psGpuUtilFW);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount)];
+
+	if (psTimeCorr->ui64CRDeltaToOSDeltaKNs == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui64CRDeltaToOSDeltaKNs is 0", __func__));
+		return 0;
+	}
+
+	return OSDivide64r64(ui64DeltaOSNs << RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT,
+	                     psTimeCorr->ui64CRDeltaToOSDeltaKNs, &ui32Remainder);
+}
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrBegin
+
+ @Description Generate new timer correlation data, and start tracking
+              the current GPU frequency.
+
+ @Input       hDevHandle : RGX Device Node
+ @Input       eEvent     : Event associated with the beginning of a timer
+                           correlation period
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrBegin(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrEnd
+
+ @Description Stop tracking the CPU and GPU timers, and if possible
+              recalculate the GPU frequency to a value which makes the timer
+              correlation data more accurate.
+
+ @Input       hDevHandle : RGX Device Node
+ @Input       eEvent     : Event associated with the end of a timer
+                           correlation period
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrEnd(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrRestartPeriodic
+
+ @Description Perform actions from RGXTimeCorrEnd and RGXTimeCorrBegin,
+              but only if enough time has passed since the last timer
+              correlation data was generated.
+
+ @Input       hDevHandle : RGX Device Node
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrRestartPeriodic(IMG_HANDLE hDevHandle);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockns64
+
+ @Description Returns value of currently selected clock (in ns).
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock value from currently selected clock source
+
+******************************************************************************/
+IMG_UINT64 RGXTimeCorrGetClockns64(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockus64
+
+ @Description Returns value of currently selected clock (in us).
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock value from currently selected clock source
+
+******************************************************************************/
+IMG_UINT64 RGXTimeCorrGetClockus64(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockSource
+
+ @Description Returns currently selected clock source
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock source type
+
+******************************************************************************/
+RGXTIMECORR_CLOCK_TYPE RGXTimeCorrGetClockSource(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrSetClockSource
+
+ @Description Sets clock source for correlation data.
+
+ @Input       psDeviceNode : RGX Device Node
+ @Input       eClockType : clock source type
+
+ @Return      error code
+
+******************************************************************************/
+PVRSRV_ERROR RGXTimeCorrSetClockSource(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       RGXTIMECORR_CLOCK_TYPE eClockType);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrInitAppHintCallbacks
+
+ @Description Initialise apphint callbacks for timer correlation
+              related apphints.
+
+ @Input       psDeviceNode : RGX Device Node
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXGetTimeCorrData
+
+ @Description Get a number of the most recent time correlation data points
+
+ @Input       psDeviceNode : RGX Device Node
+ @Output      psTimeCorrs  : Output array of RGXFWIF_TIME_CORR elements
+                             for data to be written to
+ @Input       ui32NumOut   : Number of elements to be written out
+
+ @Return      void
+
+******************************************************************************/
+void RGXGetTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+							RGXFWIF_TIME_CORR *psTimeCorrs,
+							IMG_UINT32 ui32NumOut);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVRGXCurrentTime
+
+ @Description Server-side implementation of RGXCurrentTime
+
+ @Input       psConnection     : Connection handle
+ @Input       psDeviceNode     : RGX Device Node
+ @Input       ui8TimestampType : Timestamp type
+ @Output      pui64Time        : Timestamp
+
+ @Return      PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXCurrentTime(CONNECTION_DATA    *psConnection,
+                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT8           ui8TimestampType,
+                     IMG_UINT64         *pui64Time);
+#endif /* RGXTIMECORR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.c
new file mode 100644
index 000000000000..82900863ed26
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.c
@@ -0,0 +1,225 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Timer queries
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Timer queries
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxtimerquery.h"
+#include "rgxdevice.h"
+#include "rgxtimecorr.h"
+
+#include "rgxfwutils.h"
+#include "pdump_km.h"
+
+PVRSRV_ERROR
+PVRSRVRGXBeginTimerQueryKM(CONNECTION_DATA    * psConnection,
+                           PVRSRV_DEVICE_NODE * psDeviceNode,
+                           IMG_UINT32         ui32QueryId)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32QueryId >= RGX_MAX_TIMER_QUERIES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	psDevInfo->bSaveStart = IMG_TRUE;
+	psDevInfo->bSaveEnd   = IMG_TRUE;
+
+	/* clear the stamps, in case there is no Kick */
+	psDevInfo->pui64StartTimeById[ui32QueryId] = 0UL;
+	psDevInfo->pui64EndTimeById[ui32QueryId]   = 0UL;
+	OSWriteMemoryBarrier(&psDevInfo->pui64EndTimeById[ui32QueryId]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui64StartTimeById[ui32QueryId], FLUSH);
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui64EndTimeById[ui32QueryId], FLUSH);
+
+	/* save of the active query index */
+	psDevInfo->ui32ActiveQueryId = ui32QueryId;
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXEndTimerQueryKM(CONNECTION_DATA    * psConnection,
+                         PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	/* clear off the flags set by Begin(). Note that _START_TIME is
+	 * probably already cleared by Kick()
+	 */
+	psDevInfo->bSaveStart = IMG_FALSE;
+	psDevInfo->bSaveEnd   = IMG_FALSE;
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXQueryTimerKM(CONNECTION_DATA    * psConnection,
+                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                      IMG_UINT32         ui32QueryId,
+                      IMG_UINT64         * pui64StartTime,
+                      IMG_UINT64         * pui64EndTime)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT32         ui32Scheduled;
+	IMG_UINT32         ui32Completed;
+	PVRSRV_ERROR       eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32QueryId >= RGX_MAX_TIMER_QUERIES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	ui32Scheduled = psDevInfo->aui32ScheduledOnId[ui32QueryId];
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32CompletedById[ui32QueryId], INVALIDATE);
+	ui32Completed = psDevInfo->pui32CompletedById[ui32QueryId];
+
+	/* if there was no kick since the Begin() on this id we return 0-s as Begin cleared
+	 * the stamps. If there was no begin the returned data is undefined - but still
+	 * safe from services pov
+	 */
+	if (ui32Completed >= ui32Scheduled)
+	{
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui64StartTimeById[ui32QueryId], INVALIDATE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui64EndTimeById[ui32QueryId], INVALIDATE);
+		* pui64StartTime = psDevInfo->pui64StartTimeById[ui32QueryId];
+		* pui64EndTime   = psDevInfo->pui64EndTimeById[ui32QueryId];
+
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+	return eError;
+}
+
+
+
+/******************************************************************************
+ NOT BRIDGED/EXPORTED FUNCS
+******************************************************************************/
+
+void
+RGX_GetTimestampCmdHelper(PVRSRV_RGXDEV_INFO      * psDevInfo,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPreAddr,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPostAddr,
+                          PRGXFWIF_UFO_ADDR       * ppUpdate)
+{
+	if (ppPreAddr != NULL)
+	{
+		if (psDevInfo->bSaveStart)
+		{
+			/* drop the SaveStart on the first Kick */
+			psDevInfo->bSaveStart = IMG_FALSE;
+
+			RGXSetFirmwareAddress(ppPreAddr,
+			                      psDevInfo->psStartTimeMemDesc,
+			                      sizeof(IMG_UINT64) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+		}
+		else
+		{
+			ppPreAddr->ui32Addr = 0;
+		}
+	}
+
+	if (ppPostAddr != NULL && ppUpdate != NULL)
+	{
+		if (psDevInfo->bSaveEnd)
+		{
+			RGXSetFirmwareAddress(ppPostAddr,
+			                      psDevInfo->psEndTimeMemDesc,
+			                      sizeof(IMG_UINT64) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+
+			psDevInfo->aui32ScheduledOnId[psDevInfo->ui32ActiveQueryId]++;
+
+			RGXSetFirmwareAddress(ppUpdate,
+			                      psDevInfo->psCompletedMemDesc,
+			                      sizeof(IMG_UINT32) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+		}
+		else
+		{
+			ppUpdate->ui32Addr   = 0;
+			ppPostAddr->ui32Addr = 0;
+		}
+	}
+}
+
+
+/******************************************************************************
+ End of file (rgxtimerquery.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.h
new file mode 100644
index 000000000000..7a43e7966f88
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.h
@@ -0,0 +1,117 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Timer queries
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Timer queries functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_TIMERQUERIES_H)
+#define RGX_TIMERQUERIES_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "rgxdevice.h"
+
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXBeginTimerQueryKM
+@Description    Opens a new timer query.
+
+@Input          ui32QueryId an identifier between [ 0 and RGX_MAX_TIMER_QUERIES - 1 ]
+@Return         PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXBeginTimerQueryKM(CONNECTION_DATA    * psConnection,
+                           PVRSRV_DEVICE_NODE * psDeviceNode,
+                           IMG_UINT32         ui32QueryId);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXEndTimerQueryKM
+@Description    Closes a timer query
+
+                The lack of ui32QueryId argument expresses the fact that there
+                can't be overlapping queries open.
+@Return         PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXEndTimerQueryKM(CONNECTION_DATA    * psConnection,
+                         PVRSRV_DEVICE_NODE * psDeviceNode);
+
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXQueryTimerKM
+@Description    Queries the state of the specified timer
+
+@Input          ui32QueryId an identifier between [ 0 and RGX_MAX_TIMER_QUERIES - 1 ]
+@Out            pui64StartTime
+@Out            pui64EndTime
+@Return         PVRSRV_OK                         on success.
+                PVRSRV_ERROR_RESOURCE_UNAVAILABLE if the device is still busy with
+                                                  operations from the queried period
+                other error code                  otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXQueryTimerKM(CONNECTION_DATA    * psConnection,
+                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                      IMG_UINT32         ui32QueryId,
+                      IMG_UINT64         * pui64StartTime,
+                      IMG_UINT64         * pui64EndTime);
+
+
+
+/******************************************************************************
+ NON BRIDGED/EXPORTED interface
+******************************************************************************/
+
+/* get the relevant data from the Kick to the helper*/
+void
+RGX_GetTimestampCmdHelper(PVRSRV_RGXDEV_INFO      * psDevInfo,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPreAddr,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPostAddr,
+                          PRGXFWIF_UFO_ADDR       * ppUpdate);
+
+#endif /* RGX_TIMERQUERIES_H */
+
+/******************************************************************************
+ End of file (rgxtimerquery.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.c
new file mode 100644
index 000000000000..09fb841e0e60
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.c
@@ -0,0 +1,550 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "power.h"
+#include "pvrsrv.h"
+#include "sync_internal.h"
+#include "rgxfwutils.h"
+#include "rgxlayer.h"
+#include "rgxmmudefs_km.h"
+#include "rgxta3d.h"
+
+PVRSRV_ERROR RGXQueryAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 *pui32State)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psDevInfo = psDeviceNode->pvDevice;
+	*pui32State = psDevInfo->eActivePMConf;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 ui32State)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if !defined(NO_HARDWARE)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (RGX_ACTIVEPM_FORCE_OFF != ui32State)
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+#if !defined(NO_HARDWARE)
+	psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pvAPMISRData)
+	{
+		psDevInfo->eActivePMConf = RGX_ACTIVEPM_FORCE_OFF;
+		psDevInfo->pvAPMISRData = NULL;
+		eError = PVRSRVSetDeviceDefaultPowerState((PPVRSRV_DEVICE_NODE)psDeviceNode,
+		                                          PVRSRV_DEV_POWER_STATE_ON);
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXQueryPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL *pbDisabled)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	*pbDisabled = !psDevInfo->bPDPEnabled;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL bDisable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	psDevInfo->bPDPEnabled = !bDisable;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 *pui32DeviceFlags)
+{
+	if (!pui32DeviceFlags || !psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui32DeviceFlags = psDevInfo->ui32DeviceFlags;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_BOOL bSetNotClear)
+{
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((ui32Config & ~RGXKM_DEVICE_STATE_MASK) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Bits outside of device state mask set (input: 0x%x, mask: 0x%x)",
+				 __func__, ui32Config, RGXKM_DEVICE_STATE_MASK));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (bSetNotClear)
+	{
+		psDevInfo->ui32DeviceFlags |= ui32Config;
+	}
+	else
+	{
+		psDevInfo->ui32DeviceFlags &= ~ui32Config;
+	}
+
+	return PVRSRV_OK;
+}
+
+inline const char * RGXStringifyKickTypeDM(RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	PVR_ASSERT(eKickTypeDM < RGX_KICK_TYPE_DM_LAST);
+
+	switch (eKickTypeDM) {
+		case RGX_KICK_TYPE_DM_GP:
+			return "GP ";
+		case RGX_KICK_TYPE_DM_TDM_2D:
+			return "TDM/2D ";
+		case RGX_KICK_TYPE_DM_TA:
+			return "TA ";
+		case RGX_KICK_TYPE_DM_3D:
+			return "3D ";
+		case RGX_KICK_TYPE_DM_CDM:
+			return "CDM ";
+		case RGX_KICK_TYPE_DM_RTU:
+			return "RTU ";
+		case RGX_KICK_TYPE_DM_SHG:
+			return "SHG ";
+		case RGX_KICK_TYPE_DM_TQ2D:
+			return "TQ2D ";
+		case RGX_KICK_TYPE_DM_TQ3D:
+			return "TQ3D ";
+		default:
+			return "Invalid DM ";
+	}
+}
+
+PHYS_HEAP_POLICY RGXPhysHeapGetLMAPolicy(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP_POLICY ui32Policy;
+
+	if (OSIsMapPhysNonContigSupported())
+	{
+		ui32Policy = PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG;
+
+		if (BITMASK_ANY(ui32UsageFlags,
+			(PHYS_HEAP_USAGE_FW_SHARED    |
+			 PHYS_HEAP_USAGE_FW_PRIVATE   |
+			 PHYS_HEAP_USAGE_FW_PREMAP_PT |
+			 PHYS_HEAP_USAGE_FW_CODE      |
+			 PHYS_HEAP_USAGE_FW_PRIV_DATA)))
+		{
+			if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				/* Guest Firmware heaps are always premapped */
+				ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+			}
+#if defined(RGX_PREMAP_FW_HEAPS)
+			else if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+			{
+				/* All Firmware heaps are premapped under AutoVz*/
+				ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+			}
+#endif
+		}
+
+		if (BITMASK_ANY(ui32UsageFlags, PHYS_HEAP_USAGE_FW_PREMAP))
+		{
+			ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+		}
+	}
+	else
+	{
+		ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+	}
+
+	return ui32Policy;
+}
+
+IMG_BOOL RGXIsErrorAndDeviceRecoverable(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_ERROR *peError)
+{
+	IMG_BOOL bRecoverable = IMG_TRUE;
+
+	if (*peError == PVRSRV_OK)
+	{
+		/* No recovery required */
+		return IMG_FALSE;
+	}
+
+	if (!PVRSRVIsStatusRecoverable(OSAtomicRead(&psDeviceNode->eHealthStatus)))
+	{
+		bRecoverable = IMG_FALSE;
+	}
+	else
+	{
+		RGXUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+
+		if (!PVRSRVIsStatusRecoverable(OSAtomicRead(&psDeviceNode->eHealthStatus)))
+		{
+			bRecoverable = IMG_FALSE;
+		}
+	}
+
+	if (bRecoverable && !PVRSRVIsRetryError(*peError))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device is recoverable. Changing error type (%s) to retry.",
+				 __func__, PVRSRVGetErrorString(*peError)));
+		*peError = PVRSRV_ERROR_RETRY;
+	}
+
+	if (!bRecoverable && PVRSRVIsRetryError(*peError))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device is not recoverable. Error type should not be retry (%s).",
+				 __func__, PVRSRVGetErrorString(*peError)));
+		*peError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return bRecoverable;
+}
+
+/*
+ *  Function that returns the MList Size required for a given max PB size.
+ *
+ *  The maximum MList size required always depends on the maximum PB Size
+ *  chosen and must also take into account the additional pages that will
+ *  be provided by a local PB.
+ */
+IMG_UINT32 RGXCalcMListSize(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT64 ui64MaxLocalPBSize,
+                            IMG_UINT64 ui64MaxGlobalPBSize)
+{
+	IMG_UINT32  ui32PTEPages = 0, ui32PDEPages = 0, ui32PCEPages = 0, ui32MListSize = 0;
+	IMG_UINT32  ui32NumOfPipes = 1;
+	IMG_UINT64  ui64TotalPages = 0;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	/*
+	 *  Assert if Size of PB exceeds maximum theoretical limit
+	 *  RGX_PM_MAX_PB_VIRT_ADDR_SPACE represents the 16G address space#
+	 */
+	PVR_ASSERT(ui64MaxLocalPBSize+ui64MaxGlobalPBSize <= RGX_PM_MAX_PB_VIRT_ADDR_SPACE);
+
+	/* Calculate the total number of pages which is the number of Page table entries */
+	ui64TotalPages = ((ui64MaxLocalPBSize+ui64MaxGlobalPBSize)/RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	/* Calculate the total number of pages required for the PTE's (minimum of 1) */
+	ui32PTEPages = (IMG_UINT32)(ui64TotalPages/RGX_MMUCTRL_ENTRIES_PT_VALUE);
+	if (ui32PTEPages == 0U)
+	{
+		ui32PTEPages = 1;
+	}
+
+	/* Calculate the total number of pages required to hold the PDE's (minimum of 1) */
+	ui32PDEPages = ui32PTEPages/RGX_MMUCTRL_ENTRIES_PD_VALUE;
+	if (ui32PDEPages == 0U)
+	{
+		ui32PDEPages = 1;
+	}
+
+	/* Calculate the total number of pages required to hold the PCE's (minimum of 1) */
+	ui32PCEPages = ui32PDEPages/RGX_MMUCTRL_ENTRIES_PC_VALUE;
+	if (ui32PCEPages == 0U)
+	{
+		ui32PCEPages = 1;
+	}
+
+	/* Calculate the maximum number of TA/VCE pipes */
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	{
+		IMG_UINT32 ui32Val = RGX_GET_FEATURE_VALUE(psDevInfo, RGX_FEATURE_SCALABLE_TE_ARCH);
+		if (ui32Val > ui32NumOfPipes)
+		{
+			ui32NumOfPipes = ui32Val;
+		}
+	}
+#endif
+
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	{
+		IMG_UINT32 ui32Val = RGX_GET_FEATURE_VALUE(psDevInfo, RGX_FEATURE_SCALABLE_VCE);
+		if (ui32Val > ui32NumOfPipes)
+		{
+			ui32NumOfPipes = ui32Val;
+		}
+	}
+#endif
+
+	/*
+	 *  Calculate the MList size considering the total number of pages in PB are shared
+	 *  among all the PM address spaces...
+	 */
+	ui32MListSize = (ui32PCEPages + ui32PDEPages + ui32PTEPages) *
+					RGX_NUM_PM_ADDR_SPACES * ui32NumOfPipes * RGX_MLIST_ENTRY_STRIDE;
+
+	/* Round it off to the nearest page granularity */
+	ui32MListSize = PVR_ALIGN(ui32MListSize, RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	return ui32MListSize;
+}
+
+/*
+ * Critical PMRs are PMRs that are created by client that might contain physical page addresses.
+ * We need to validate if they were allocated with proper flags.
+ */
+PVRSRV_ERROR ValidateCriticalPMR(PMR* psPMR, IMG_DEVMEM_SIZE_T ui64MinSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	IMG_BOOL bCPUCacheSnoop =
+		(PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		 psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY);
+
+	PMR_FLAGS_T uiFlags = PMR_Flags(psPMR);
+
+	/* Critical PMR cannot be user CPU mappable */
+	if (PVRSRV_CHECK_CPU_READABLE(uiFlags) ||
+	    PVRSRV_CHECK_CPU_WRITEABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Critical PMR allows CPU mapping (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* Critical PMR must not be user CPU cacheable (unless snooping is on) */
+	if (!bCPUCacheSnoop &&
+	    (PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags) ||
+	     PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+	     PVRSRV_CHECK_CPU_CACHED(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Critical PMR allows CPU caching (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* Critical PMRs must be allocated with PMMETA_PROTECT */
+	if ((uiFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Critical PMR must have PMMETA_PROTECT set",
+		        __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Critical PMR must not have OS_LINUX_MOVABLE set",
+		        __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+#endif
+
+	if (PMR_LogicalSize(psPMR) < ui64MinSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Critical PMR doesn't have sufficient size",
+		         __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	return PVRSRV_OK;
+return_error:
+	return eError;
+}
+
+/* Check if all global freelists have the same size and if all local freelists have the same size.*/
+PVRSRV_ERROR ValidateFreeListSizes(RGX_FREELIST* apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                   IMG_UINT32*   pui32LocalFLMaxPages,
+                                   IMG_UINT32*   pui32GlobalFLMaxPages)
+{
+	IMG_UINT32 i,j;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32GlobalFLMaxPages = apsFreeLists[RGXFW_GLOBAL_FREELIST]->ui32MaxFLPages;
+	IMG_UINT32 ui32LocalFLMaxPages = apsFreeLists[RGXFW_LOCAL_FREELIST]->ui32MaxFLPages;
+	IMG_UINT32 ui32NumFLPerGD = RGXMKIF_NUM_RTDATA_FREELISTS/RGXMKIF_NUM_GEOMDATAS;
+
+	for (i=0; i<RGXMKIF_NUM_GEOMDATAS; i++)
+	{
+		/* Check if all local freelists have the same size */
+		if (apsFreeLists[ui32NumFLPerGD * i + RGXFW_LOCAL_FREELIST]->ui32MaxFLPages != ui32LocalFLMaxPages)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Check if all global freelists have the same size */
+		for (j=RGXFW_GLOBAL_FREELIST; j<ui32NumFLPerGD; j++)
+		{
+			if (apsFreeLists[ui32NumFLPerGD * i + j]->ui32MaxFLPages != ui32GlobalFLMaxPages)
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+		}
+	}
+
+	*pui32LocalFLMaxPages = ui32LocalFLMaxPages;
+	*pui32GlobalFLMaxPages = ui32GlobalFLMaxPages;
+
+	return eError;
+}
+
+PVRSRV_ERROR
+AcquireValidateRefCriticalBuffer(PVRSRV_DEVICE_NODE*     psDevNode,
+                                 DEVMEMINT_RESERVATION*  psReservation,
+                                 IMG_DEVMEM_SIZE_T       ui64MinSize,
+                                 PMR**                   ppsPMR,
+                                 IMG_DEV_VIRTADDR*       psDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psReservation))
+	{
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		PVR_LOG_GOTO_IF_ERROR_VA(eError, ReturnError,
+		    "%s: Failed to acquire reservation for critical buffer", __func__);
+	}
+
+	eError = DevmemIntGetReservationData(psReservation, ppsPMR, psDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR_VA(eError, RollbackReservation,
+	    "%s: Error from DevmemIntGetReservationData for critical buffer: %s",
+	    __func__, PVRSRVGetErrorString(eError));
+
+
+	/* Check buffer sizes and flags are as required */
+	eError = ValidateCriticalPMR(*ppsPMR, ui64MinSize);
+	PVR_LOG_GOTO_IF_ERROR_VA(eError, RollbackReservation,
+	    "%s: Validation of critical PMR failed: %s",
+	    __func__, PVRSRVGetErrorString(eError));
+
+	/* If no error on validation ref the PMR */
+	(void) PMRRefPMR(*ppsPMR);
+
+	return PVRSRV_OK;
+
+RollbackReservation:
+	DevmemIntReservationRelease(psReservation);
+ReturnError:
+	return eError;
+}
+
+void UnrefAndReleaseCriticalBuffer(DEVMEMINT_RESERVATION* psReservation)
+{
+	PVRSRV_ERROR eError;
+	PMR* psPMR;
+	IMG_DEV_VIRTADDR sDummy;
+	/* Skip error check. If this function is called it means we already
+	   Acquired a reservation and confirmed that mapping exists. */
+	eError = DevmemIntGetReservationData(psReservation, &psPMR, &sDummy);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError,
+	    "Error when trying to obtain reservation data in %s", __func__);
+
+	eError = PMRUnrefPMR(psPMR);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError,
+	    "Error on PMR unref in %s", __func__);
+
+	DevmemIntReservationRelease(psReservation);
+}
+
+/******************************************************************************
+ End of file (rgxutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.h
new file mode 100644
index 000000000000..cf5046433826
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific utility routines declarations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Inline functions/structures specific to RGX
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_error.h"
+#include "rgxta3d.h"
+
+/*!
+******************************************************************************
+
+ @Function      RGXQueryAPMState
+
+ @Description   Query the state of the APM configuration
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Output        pui32State   : The APM configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXQueryAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 *pui32State);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetAPMState
+
+ @Description   Set the APM configuration state. Currently only 'OFF' is
+                supported
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         ui32State    : The requested APM configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 ui32State);
+
+/*!
+******************************************************************************
+
+ @Function      RGXQueryPdumpPanicDisable
+
+ @Description   Get the PDump Panic Enable configuration state.
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         pbDisabled    : IMG_TRUE if PDump Panic is disabled
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXQueryPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL *pbDisabled);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetPdumpPanicDisable
+
+ @Description   Set the PDump Panic Enable flag
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         bDisable      : The requested configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL bDisable);
+
+/*!
+******************************************************************************
+
+ @Function      RGXGetDeviceFlags
+
+ @Description   Get the device flags for a given device
+
+ @Input         psDevInfo        : The device descriptor query
+
+ @Output        pui32DeviceFlags : The current state of the device flags
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXGetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 *pui32DeviceFlags);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetDeviceFlags
+
+ @Description   Set the device flags for a given device
+
+ @Input         psDevInfo : The device descriptor to modify
+
+ @Input         ui32Config : The device flags to modify
+
+ @Input         bSetNotClear : Set or clear the specified flags
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_BOOL bSetNotClear);
+
+/*!
+******************************************************************************
+
+ @Function    RGXStringifyKickTypeDM
+
+ @Description Gives the kick type DM name stringified
+
+ @Input       Kick type DM
+
+ @Return      Array containing the kick type DM name
+
+******************************************************************************/
+const char* RGXStringifyKickTypeDM(RGX_KICK_TYPE_DM eKickTypeDM);
+
+/*************************************************************************/ /*!
+
+@Function       RGXPhysHeapGetLMAPolicy
+
+@Description    Returns the optimal LMA allocation policy based on a heap's
+                usage flags
+
+@Input          ui32UsageFlags Flags specifying a heap's intended use
+@Input          psDeviceNode The device node.
+
+@Return         PHYS_HEAP_POLICY The recommended LMA policy
+
+*/ /**************************************************************************/
+PHYS_HEAP_POLICY RGXPhysHeapGetLMAPolicy(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#define RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(bitmask, eKickTypeDM) bitmask & eKickTypeDM ? RGXStringifyKickTypeDM(eKickTypeDM) : ""
+
+/*************************************************************************/ /*!
+@Function       RGXIsErrorAndDeviceRecoverable
+@Description    This function is used to check if device (and firmware) is in
+                a state that can be recovered from without a full reset of the
+                device.
+@Input          psDeviceNode The device node.
+@Input          peError      Pointer to error. Can be changed to retry type.
+@Return         IMG_BOOL   Return true if device is recoverable.
+*/ /**************************************************************************/
+IMG_BOOL RGXIsErrorAndDeviceRecoverable(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_ERROR *peError);
+
+/*
+ * To avoid repeated calls and avoid double frees, the error value is set to PVRSRV_OK
+ * if RGXIsErrorAndDeviceRecoverable is false.
+ */
+#define RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode, eError, cleanupFunc) \
+	do \
+	{ \
+		if (RGXIsErrorAndDeviceRecoverable(psDeviceNode, &eError)) \
+		{ \
+			return eError; \
+		} \
+		else if (eError != PVRSRV_OK) \
+		{ \
+			PVR_LOG(("%s: Unexpected error from " #cleanupFunc "(%s)", \
+					__func__, \
+					PVRSRVGetErrorString(eError))); \
+			/* Device is dead. \
+			 * Change error type to make callers destroy the resource handle. \
+			 * This is to prevent repeated calls to this function. \
+			 */ \
+			eError = PVRSRV_OK; \
+		} \
+	} while (false)
+
+/*************************************************************************/ /*!
+@Function       RGXCalcMListSize
+@Description    Function that calculates the MList Size required for
+                given local and global PB sizes.
+@Input          psDeviceNode The device node.
+@Input          ui64MaxLocalPBSize Maximum local PB size in bytes
+@Input          ui64MaxGlobalPBSize Maximum global PB size in bytes
+
+@Return         IMG_UINT32 Returns size of the mlist in bytes aligned to
+                RGX_BIF_PM_PHYSICAL_PAGE_SIZE.
+*/ /**************************************************************************/
+IMG_UINT32 RGXCalcMListSize(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT64 ui64MaxLocalPBSize,
+                            IMG_UINT64 ui64MaxGlobalPBSize);
+
+/*************************************************************************/ /*!
+@Function       ValidateCriticalPMR
+@Description    Validate if critical PMR has proper flags and size.
+@Input          psPMR Pointer to PMR to validate.
+@Input          ui64MinSize Minimum size that PMR needs to have
+
+@Return         PVRSRV_ERROR PVRSRV_OK if validation successful.
+                Appropriate error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR ValidateCriticalPMR(PMR* psPMR, IMG_DEVMEM_SIZE_T ui64MinSize);
+
+/*************************************************************************/ /*!
+@Function       ValidateFreeListSizes
+@Description    Helper function for RGXCreateHWRTDataSet
+                For the freelist array passed to RGXCreateHWRTDataSet, validate
+                if all global freelists have the same size and if all local
+                freelists have the same size. Return the sizes in output params.
+
+@Output         pui32LocalFLMaxPages Max number of pages for local freelist
+@Output         pui32GlobalFLMaxPages Max number of pages for global freelist
+
+@Return         PVRSRV_ERROR PVRSRV_OK if validation successful.
+                Appropriate error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR ValidateFreeListSizes(RGX_FREELIST* apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                   IMG_UINT32*   pui32LocalFLMaxPages,
+                                   IMG_UINT32*   pui32GlobalFLMaxPages);
+
+/*************************************************************************/ /*!
+@Function       AcquireValidateRefCriticalBuffer
+@Description    Helper function for RGXCreateHWRTDataSet
+                Acquire the reservation validate if the underlying PMR is
+                appropriate for use as critical buffer and ref it.
+@Input          psDevNode The device node.
+@Input          psReservation The reservation describing the critical buffer
+@Input          ui64MinSize Minimum size that buffer needs to have
+@Output         ppsPMR Pointer to be written with the PMR on success.
+@Output         psDevVAddr The device vaddress to be written on success.
+
+@Return         PVRSRV_ERROR PVRSRV_OK if validation successful.
+                Appropriate error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR AcquireValidateRefCriticalBuffer(PVRSRV_DEVICE_NODE*     psDevNode,
+                                              DEVMEMINT_RESERVATION*  psReservation,
+                                              IMG_DEVMEM_SIZE_T       ui64MinSize,
+                                              PMR**                   ppsPMR,
+                                              IMG_DEV_VIRTADDR*       psDevVAddr);
+
+
+/*************************************************************************/ /*!
+@Function       UnrefAndReleaseCriticalBuffer
+@Description    Helper function for RGXCreateHWRTDataSet
+                Unref the critical buffer and release the reservation object.
+@Input          psReservation The reservation describing the critical buffer
+
+*/ /**************************************************************************/
+void UnrefAndReleaseCriticalBuffer(DEVMEMINT_RESERVATION* psReservation);
+
+/******************************************************************************
+ End of file (rgxutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.c
new file mode 100644
index 000000000000..1143910295c5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.c
@@ -0,0 +1,673 @@
+/*************************************************************************/ /*!
+@File           rgxworkest.c
+@Title          RGX Workload Estimation Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxworkest.h"
+#include "rgxfwutils.h"
+#include "rgxpdvfs.h"
+#include "rgx_options.h"
+#include "device.h"
+#include "hash.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_SOC_TIMER)
+#include "rgxtimecorr.h"
+#endif
+
+#define ROUND_DOWN_TO_NEAREST_1024(number) (((number) >> 10) << 10)
+
+static inline IMG_BOOL _WorkEstEnabled(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions &
+	    psPVRSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions &
+	    OPTIONS_WORKLOAD_ESTIMATION_EN)
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+inline IMG_UINT32 _WorkEstDoHash(IMG_UINT32 ui32Input)
+{
+	IMG_UINT32 ui32HashPart;
+
+	/* Hash function borrowed from hash.c */
+	ui32HashPart = ui32Input;
+	ui32HashPart += (ui32HashPart << 12);
+	ui32HashPart ^= (ui32HashPart >> 22);
+	ui32HashPart += (ui32HashPart << 4);
+	ui32HashPart ^= (ui32HashPart >> 9);
+	ui32HashPart += (ui32HashPart << 10);
+	ui32HashPart ^= (ui32HashPart >> 2);
+	ui32HashPart += (ui32HashPart << 7);
+	ui32HashPart ^= (ui32HashPart >> 12);
+
+	return ui32HashPart;
+}
+
+/*! Hash functions for TA/3D workload estimation */
+IMG_BOOL WorkEstHashCompareTA3D(size_t uKeySize, void *pKey1, void *pKey2);
+IMG_UINT32 WorkEstHashFuncTA3D(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*! Hash functions for compute workload estimation */
+IMG_BOOL WorkEstHashCompareCompute(size_t uKeySize, void *pKey1, void *pKey2);
+IMG_UINT32 WorkEstHashFuncCompute(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*! Hash functions for TDM/transfer workload estimation */
+IMG_BOOL WorkEstHashCompareTDM(size_t uKeySize, void *pKey1, void *pKey2);
+IMG_UINT32 WorkEstHashFuncTDM(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+
+IMG_BOOL WorkEstHashCompareTA3D(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sTA3D.ui32RenderTargetSize  == psWorkload2->sTA3D.ui32RenderTargetSize &&
+		    psWorkload1->sTA3D.ui32NumberOfDrawCalls == psWorkload2->sTA3D.ui32NumberOfDrawCalls &&
+		    psWorkload1->sTA3D.ui32NumberOfIndices   == psWorkload2->sTA3D.ui32NumberOfIndices &&
+		    psWorkload1->sTA3D.ui32NumberOfMRTs      == psWorkload2->sTA3D.ui32NumberOfMRTs)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_UINT32 WorkEstHashFuncTA3D(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on multiple render target attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32RenderTargetSize);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32NumberOfDrawCalls);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32NumberOfIndices);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32NumberOfMRTs);
+
+	return ui32HashKey;
+}
+
+IMG_BOOL WorkEstHashCompareCompute(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sCompute.ui32NumberOfWorkgroups == psWorkload2->sCompute.ui32NumberOfWorkgroups &&
+		    psWorkload1->sCompute.ui32NumberOfWorkitems  == psWorkload2->sCompute.ui32NumberOfWorkitems)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_UINT32 WorkEstHashFuncCompute(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on multiple render target attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sCompute.ui32NumberOfWorkgroups);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sCompute.ui32NumberOfWorkitems);
+	return ui32HashKey;
+}
+
+IMG_BOOL WorkEstHashCompareTDM(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sTransfer.ui32Characteristic1 == psWorkload2->sTransfer.ui32Characteristic1 &&
+		    psWorkload1->sTransfer.ui32Characteristic2 == psWorkload2->sTransfer.ui32Characteristic2)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_UINT32 WorkEstHashFuncTDM(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on transfer src/dest attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTransfer.ui32Characteristic1);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTransfer.ui32Characteristic2);
+
+	return ui32HashKey;
+}
+
+void WorkEstHashLockCreate(POS_LOCK *ppsHashLock)
+{
+	if (*ppsHashLock == NULL)
+	{
+		OSLockCreate(ppsHashLock);
+	}
+}
+
+void WorkEstHashLockDestroy(POS_LOCK psWorkEstHashLock)
+{
+	if (psWorkEstHashLock != NULL)
+	{
+		OSLockDestroy(psWorkEstHashLock);
+		psWorkEstHashLock = NULL;
+	}
+}
+
+void WorkEstCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_WORKEST_FWCCB_CMD *psFwCCBCmd;
+	IMG_UINT8 *psFWCCB = psDevInfo->psWorkEstFirmwareCCB;
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psWorkEstFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psWorkEstFirmwareCCBCtlLocal;
+
+	RGXFwSharedMemCacheOpPtr(psFWCCBCtl, INVALIDATE);
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtl->ui32WriteOffset)
+	{
+		PVRSRV_ERROR eError;
+
+		/* Point to the next command */
+		psFwCCBCmd = (RGXFWIF_WORKEST_FWCCB_CMD *)((uintptr_t)psFWCCB + psFWCCBCtlLocal->ui32ReadOffset * sizeof(RGXFWIF_WORKEST_FWCCB_CMD));
+
+		eError = WorkEstRetire(psDevInfo, psFwCCBCmd);
+		PVR_LOG_IF_ERROR(eError, "WorkEstCheckFirmwareCCB: WorkEstRetire failed");
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+	}
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32ReadOffset, FLUSH);
+}
+
+PVRSRV_ERROR WorkEstPrepare(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                            WORKEST_HOST_DATA         *psWorkEstHostData,
+                            WORKLOAD_MATCHING_DATA    *psWorkloadMatchingData,
+                            const RGXFWIF_CCB_CMD_TYPE eDMCmdType,
+                            const RGX_WORKLOAD        *psWorkloadCharsIn,
+                            IMG_UINT64                ui64DeadlineInus,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData)
+{
+	RGX_WORKLOAD          *psWorkloadCharacteristics;
+	IMG_UINT64            *pui64CyclePrediction;
+	IMG_UINT64            ui64CurrentTime;
+	WORKEST_RETURN_DATA   *psReturnData;
+	IMG_UINT32            ui32ReturnDataWO;
+#if defined(SUPPORT_SOC_TIMER)
+	PVRSRV_DEVICE_CONFIG  *psDevConfig;
+	IMG_UINT64            ui64CurrentSoCTime;
+#endif
+	PVRSRV_ERROR          eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (!_WorkEstEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	if (eDMCmdType == RGXFWIF_CCB_CMD_TYPE_NULL)
+	{
+		/* No workload, only fence updates */
+		return PVRSRV_OK;
+	}
+
+#if !defined(PVRSRV_NEED_PVR_DPF)
+	PVR_UNREFERENCED_PARAMETER(eDMCmdType);
+#endif
+
+	/* Validate all required objects required for preparing work estimation */
+	PVR_LOG_RETURN_IF_FALSE(psDevInfo, "device info not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkEstHostData, "host data not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkloadMatchingData, "Workload Matching Data not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkloadMatchingData->psHashLock, "hash lock not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkloadMatchingData->psHashTable, "hash table not available", eError);
+
+#if defined(SUPPORT_SOC_TIMER)
+	psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	if (psDevConfig->pfnSoCTimerRead)
+	{
+		ui64CurrentSoCTime = psDevConfig->pfnSoCTimerRead(psDevConfig->hSysData);
+	}
+	else
+	{
+		/* Fallback to OS clock */
+		ui64CurrentSoCTime = 0;
+	}
+#endif
+
+	eError = OSClockMonotonicus64(&ui64CurrentTime);
+	PVR_LOG_RETURN_IF_ERROR(eError, "unable to access System Monotonic clock");
+
+	OSLockAcquire(psDevInfo->hWorkEstLock);
+
+	/* Select the next index for the return data and update it (is this thread safe?) */
+	ui32ReturnDataWO = psDevInfo->ui32ReturnDataWO;
+	psDevInfo->ui32ReturnDataWO = (ui32ReturnDataWO + 1) & RETURN_DATA_ARRAY_WRAP_MASK;
+
+	/* Index for the return data passed to/from the firmware. */
+	psWorkEstKickData->ui16ReturnDataIndex = ui32ReturnDataWO;
+	if (ui64DeadlineInus > ui64CurrentTime)
+	{
+		/* Rounding is done to reduce multiple deadlines with minor spread flooding the fw workload array. */
+#if defined(SUPPORT_SOC_TIMER)
+		if (psDevConfig->pfnSoCTimerRead)
+		{
+			RGX_DATA *psRGXData = (RGX_DATA*)psDevConfig->hDevData;
+			RGX_TIMING_INFORMATION *psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+			IMG_UINT32 ui32Remainder;
+			IMG_UINT64 ui64TimeDelta =
+				OSDivide64r64((ui64DeadlineInus - ui64CurrentTime) * psRGXTimingInfo->ui32SOCClockSpeed, SECONDS_TO_MICROSECONDS, &ui32Remainder);
+
+			psWorkEstKickData->ui64Deadline = ROUND_DOWN_TO_NEAREST_1024(ui64CurrentSoCTime + ui64TimeDelta);
+			PVR_DPF((PVR_DBG_MESSAGE, "Current SOC time: %llu. New deadline: %llu SOC ticks",
+					 ui64CurrentSoCTime, psWorkEstKickData->ui64Deadline));
+		}
+		else
+		{
+			psWorkEstKickData->ui64Deadline = ROUND_DOWN_TO_NEAREST_1024(ui64DeadlineInus);
+		}
+#else
+		psWorkEstKickData->ui64Deadline = ROUND_DOWN_TO_NEAREST_1024(ui64DeadlineInus);
+#endif
+	}
+	else
+	{
+		/* If deadline has already passed, assign zero to suggest full frequency */
+		psWorkEstKickData->ui64Deadline = 0;
+	}
+
+	/* Set up data for the return path to process the workload; the matching data is needed
+	   as it holds the hash data, the host data is needed for completion updates */
+	psReturnData = &psDevInfo->asReturnData[ui32ReturnDataWO];
+	psReturnData->psWorkloadMatchingData = psWorkloadMatchingData;
+	psReturnData->psWorkEstHostData = psWorkEstHostData;
+
+	/* The workload characteristic is needed in the return data for the matching
+	   of future workloads via the hash. */
+	psWorkloadCharacteristics = &psReturnData->sWorkloadCharacteristics;
+	memcpy(psWorkloadCharacteristics, psWorkloadCharsIn, sizeof(RGX_WORKLOAD));
+
+	OSLockRelease(psDevInfo->hWorkEstLock);
+
+	/* Acquire the lock to access hash */
+	OSLockAcquire(psWorkloadMatchingData->psHashLock);
+
+	/* Check if there is a prediction for this workload */
+	pui64CyclePrediction = (IMG_UINT64*) HASH_Retrieve(psWorkloadMatchingData->psHashTable,
+													   (uintptr_t)psWorkloadCharacteristics);
+
+	/* Release lock */
+	OSLockRelease(psWorkloadMatchingData->psHashLock);
+
+	if (pui64CyclePrediction != NULL)
+	{
+		/* Cycle prediction is available, store this prediction */
+		psWorkEstKickData->ui32CyclesPrediction = *pui64CyclePrediction;
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+		if (*pui64CyclePrediction >= IMG_UINT32_MAX)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Workload estimate overflow:"
+					" %" IMG_UINT64_FMTSPEC, *pui64CyclePrediction));
+		}
+
+		switch (eDMCmdType)
+		{
+			case RGXFWIF_CCB_CMD_TYPE_GEOM:
+			case RGXFWIF_CCB_CMD_TYPE_3D:
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: RT size = %u, draw count = %u, indices = %u, prediction = " IMG_DEVMEM_SIZE_FMTSPEC,
+					 __func__,
+					 psWorkloadCharacteristics->sTA3D.ui32RenderTargetSize,
+					 psWorkloadCharacteristics->sTA3D.ui32NumberOfDrawCalls,
+					 psWorkloadCharacteristics->sTA3D.ui32NumberOfIndices,
+					 *pui64CyclePrediction));
+				break;
+			case RGXFWIF_CCB_CMD_TYPE_CDM:
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Number of workgroups = %u, max workgroup size = %u, prediction = " IMG_DEVMEM_SIZE_FMTSPEC,
+					 __func__,
+					 psWorkloadCharacteristics->sCompute.ui32NumberOfWorkgroups,
+					 psWorkloadCharacteristics->sCompute.ui32NumberOfWorkitems,
+					 *pui64CyclePrediction));
+				break;
+			case RGXFWIF_CCB_CMD_TYPE_TQ_TDM:
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Dest size = %u, Pixel format ID = %u, prediction = " IMG_DEVMEM_SIZE_FMTSPEC,
+					 __func__,
+					 psWorkloadCharacteristics->sTransfer.ui32Characteristic1,
+					 psWorkloadCharacteristics->sTransfer.ui32Characteristic2,
+					 *pui64CyclePrediction));
+				break;
+
+			default:
+				break;
+		}
+#endif
+	}
+	else
+	{
+		/* There is no prediction */
+		psWorkEstKickData->ui32CyclesPrediction = 0;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR WorkEstRetire(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   RGXFWIF_WORKEST_FWCCB_CMD *psReturnCmd)
+{
+	RGX_WORKLOAD           *psWorkloadCharacteristics;
+	WORKLOAD_MATCHING_DATA *psWorkloadMatchingData;
+	IMG_UINT64             *paui64WorkloadHashData;
+	RGX_WORKLOAD           *pasWorkloadHashKeys;
+	IMG_UINT32             ui32HashArrayWO;
+	IMG_UINT64             *pui64CyclesTaken;
+	IMG_UINT64              ui64ActualCyclesTaken;
+	WORKEST_RETURN_DATA    *psReturnData;
+	WORKEST_HOST_DATA      *psWorkEstHostData;
+
+	if (!_WorkEstEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(psReturnCmd,
+	                        "WorkEstRetire: Missing return command",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	RGXFwSharedMemCacheOpPtr(psReturnCmd, INVALIDATE);
+
+	if (psReturnCmd->ui16ReturnDataIndex >= RETURN_DATA_ARRAY_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "WorkEstRetire: Handle reference out-of-bounds:"
+		        " %u >= %u",
+		        psReturnCmd->ui16ReturnDataIndex,
+		        RETURN_DATA_ARRAY_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquire(psDevInfo->hWorkEstLock);
+
+	/* Retrieve/validate the return data from this completed workload */
+	psReturnData = &psDevInfo->asReturnData[psReturnCmd->ui16ReturnDataIndex];
+	psWorkloadCharacteristics = &psReturnData->sWorkloadCharacteristics;
+	psWorkEstHostData = psReturnData->psWorkEstHostData;
+	PVR_LOG_GOTO_IF_FALSE(psWorkEstHostData,
+	                      "WorkEstRetire: Missing host data",
+	                      unlock_workest);
+
+	/* Skip if cycle data unavailable */
+	PVR_LOG_GOTO_IF_FALSE(psReturnCmd->ui32CyclesTaken,
+	                      "WorkEstRetire: Cycle data not available",
+	                      unlock_workest);
+
+	/* Retrieve/validate completed workload matching data */
+	psWorkloadMatchingData = psReturnData->psWorkloadMatchingData;
+	PVR_LOG_GOTO_IF_FALSE(psWorkloadMatchingData,
+	                      "WorkEstRetire: Missing matching data",
+	                      unlock_workest);
+	PVR_LOG_GOTO_IF_FALSE(psWorkloadMatchingData->psHashTable,
+	                      "WorkEstRetire: Missing hash",
+	                      unlock_workest);
+	PVR_LOG_GOTO_IF_FALSE(psWorkloadMatchingData->psHashLock,
+	                      "WorkEstRetire: Missing hash/lock",
+	                      unlock_workest);
+	paui64WorkloadHashData = psWorkloadMatchingData->aui64HashData;
+	pasWorkloadHashKeys = psWorkloadMatchingData->asHashKeys;
+	ui32HashArrayWO = psWorkloadMatchingData->ui32HashArrayWO;
+
+	OSLockRelease(psDevInfo->hWorkEstLock);
+
+	OSLockAcquire(psWorkloadMatchingData->psHashLock);
+
+	/* Update workload prediction by removing old hash entry (if any)
+	 * & inserting new hash entry */
+	pui64CyclesTaken = (IMG_UINT64*) HASH_Remove(psWorkloadMatchingData->psHashTable,
+												 (uintptr_t)psWorkloadCharacteristics);
+
+	if (paui64WorkloadHashData[ui32HashArrayWO] > 0)
+	{
+		/* Out-of-space so remove the oldest hash data before it becomes
+		 * overwritten */
+		RGX_WORKLOAD *psWorkloadHashKey = &pasWorkloadHashKeys[ui32HashArrayWO];
+		(void) HASH_Remove(psWorkloadMatchingData->psHashTable, (uintptr_t)psWorkloadHashKey);
+	}
+
+	ui64ActualCyclesTaken = (IMG_UINT64)psReturnCmd->ui16CyclesTakenHigh << 32U;
+	ui64ActualCyclesTaken += psReturnCmd->ui32CyclesTaken;
+
+	if (pui64CyclesTaken == NULL)
+	{
+		/* There is no existing entry for this workload characteristics,
+		 * store it */
+		paui64WorkloadHashData[ui32HashArrayWO] = ui64ActualCyclesTaken;
+		pasWorkloadHashKeys[ui32HashArrayWO] = *psWorkloadCharacteristics;
+	}
+	else
+	{
+		/* Found prior entry for workload characteristics, average with
+		 * completed; also reset the old value to 0 so it is known to be
+		 * invalid */
+		paui64WorkloadHashData[ui32HashArrayWO] = (*pui64CyclesTaken + ui64ActualCyclesTaken)/2;
+		pasWorkloadHashKeys[ui32HashArrayWO] = *psWorkloadCharacteristics;
+		*pui64CyclesTaken = 0;
+	}
+
+	/* Hash insertion should not fail but if it does best we can do is to exit
+	 * gracefully and not update the FW received counter */
+	if (IMG_TRUE != HASH_Insert((HASH_TABLE*)psWorkloadMatchingData->psHashTable,
+								(uintptr_t)&pasWorkloadHashKeys[ui32HashArrayWO],
+								(uintptr_t)&paui64WorkloadHashData[ui32HashArrayWO]))
+	{
+		PVR_ASSERT(0);
+		PVR_LOG(("WorkEstRetire: HASH_Insert failed"));
+	}
+
+#if defined(DEBUG)
+	/* Zero the current entry in the return data table.
+	 * Helps detect invalid ReturnDataIndex values from the
+	 * firmware before the hash table is corrupted. */
+	memset(psReturnData, 0, sizeof(WORKEST_RETURN_DATA));
+#endif
+
+	psWorkloadMatchingData->ui32HashArrayWO = (ui32HashArrayWO + 1) & WORKLOAD_HASH_WRAP_MASK;
+
+	OSLockRelease(psWorkloadMatchingData->psHashLock);
+
+	/* Update the received counter so that the FW is able to check as to whether
+	 * all the workloads connected to a render context are finished.
+	 * Note: needs to be done also for *unlock_workest* label below. */
+	psWorkEstHostData->ui32WorkEstCCBReceived++;
+
+	return PVRSRV_OK;
+
+unlock_workest:
+	OSLockRelease(psDevInfo->hWorkEstLock);
+
+	PVR_ASSERT(psWorkEstHostData);
+	if (psWorkEstHostData)
+	{
+		psWorkEstHostData->ui32WorkEstCCBReceived++;
+	}
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+void _WorkEstInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						 WORKLOAD_MATCHING_DATA *psWorkloadMatchingData,
+						 HASH_FUNC *pfnWorkEstHashFunc,
+						 HASH_KEY_COMP *pfnWorkEstHashCompare)
+{
+	HASH_TABLE *psWorkloadHashTable;
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	/* Create a lock to protect the per-DM hash table */
+	WorkEstHashLockCreate(&psWorkloadMatchingData->psHashLock);
+
+	/* Create hash table for the per-DM workload matching */
+	psWorkloadHashTable = HASH_Create_Extended(WORKLOAD_HASH_SIZE,
+											   sizeof(RGX_WORKLOAD *),
+											   pfnWorkEstHashFunc,
+											   pfnWorkEstHashCompare);
+	psWorkloadMatchingData->psHashTable = psWorkloadHashTable;
+}
+
+void _WorkEstDeInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   WORKLOAD_MATCHING_DATA *psWorkloadMatchingData)
+{
+	HASH_TABLE        *psWorkloadHashTable;
+	RGX_WORKLOAD      *pasWorkloadHashKeys;
+	RGX_WORKLOAD      *psWorkloadHashKey;
+	IMG_UINT64        *paui64WorkloadCycleData;
+	IMG_UINT32        ui32Itr;
+
+	/* Tear down per-DM hash */
+	pasWorkloadHashKeys = psWorkloadMatchingData->asHashKeys;
+	paui64WorkloadCycleData = psWorkloadMatchingData->aui64HashData;
+	psWorkloadHashTable = psWorkloadMatchingData->psHashTable;
+
+	if (psWorkloadHashTable)
+	{
+		for (ui32Itr = 0; ui32Itr < WORKLOAD_HASH_SIZE; ui32Itr++)
+		{
+			if (paui64WorkloadCycleData[ui32Itr] > 0)
+			{
+				psWorkloadHashKey = &pasWorkloadHashKeys[ui32Itr];
+				HASH_Remove(psWorkloadHashTable, (uintptr_t)psWorkloadHashKey);
+			}
+		}
+
+		HASH_Delete(psWorkloadHashTable);
+	}
+
+	/* Remove the hash lock */
+	WorkEstHashLockDestroy(psWorkloadMatchingData->psHashLock);
+}
+
+void WorkEstInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sTA3D.sDataTA,
+		(HASH_FUNC *)WorkEstHashFuncTA3D,
+		(HASH_KEY_COMP *)WorkEstHashCompareTA3D);
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sTA3D.sData3D,
+		(HASH_FUNC *)WorkEstHashFuncTA3D,
+		(HASH_KEY_COMP *)WorkEstHashCompareTA3D);
+}
+
+void WorkEstDeInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sTA3D.sDataTA);
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sTA3D.sData3D);
+}
+
+void WorkEstInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sCompute.sDataCDM,
+		(HASH_FUNC *)WorkEstHashFuncCompute,
+		(HASH_KEY_COMP *)WorkEstHashCompareCompute);
+}
+
+void WorkEstDeInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sCompute.sDataCDM);
+}
+
+void WorkEstInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sTransfer.sDataTDM,
+		(HASH_FUNC *)WorkEstHashFuncTDM,
+		(HASH_KEY_COMP *)WorkEstHashCompareTDM);
+}
+
+void WorkEstDeInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sTransfer.sDataTDM);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.h
new file mode 100644
index 000000000000..972c5199a3d8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.h
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File           rgxworkest.h
+@Title          RGX Workload Estimation Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXWORKEST_H
+#define RGXWORKEST_H
+
+#include "img_types.h"
+#include "rgxta3d.h"
+
+
+void WorkEstInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstDeInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstDeInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstDeInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+PVRSRV_ERROR WorkEstPrepare(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                            WORKEST_HOST_DATA         *psWorkEstHostData,
+                            WORKLOAD_MATCHING_DATA    *psWorkloadMatchingData,
+                            const RGXFWIF_CCB_CMD_TYPE eDMCmdType,
+                            const RGX_WORKLOAD        *psWorkloadCharsIn,
+                            IMG_UINT64                ui64DeadlineInus,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData);
+
+PVRSRV_ERROR WorkEstRetire(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   RGXFWIF_WORKEST_FWCCB_CMD *psReturnCmd);
+
+void WorkEstHashLockCreate(POS_LOCK *ppsHashLock);
+
+void WorkEstHashLockDestroy(POS_LOCK psHashLock);
+
+void WorkEstCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void _WorkEstInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						 WORKLOAD_MATCHING_DATA *psWorkloadMatchingData,
+						 HASH_FUNC *pfnWorkEstHashFunc,
+						 HASH_KEY_COMP *pfnWorkEstHashCompare);
+
+void _WorkEstDeInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						 WORKLOAD_MATCHING_DATA *psWorkloadMatchingData);
+
+inline IMG_UINT32 _WorkEstDoHash(IMG_UINT32 ui32Input);
+
+#endif /* RGXWORKEST_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdebug.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdebug.c
new file mode 100644
index 000000000000..93738fdf6f9a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdebug.c
@@ -0,0 +1,2580 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "cache_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "rgxutils.h"
+#include "tlstream.h"
+#include "rgxfwriscv.h"
+#include "pvrsrv.h"
+#include "services_km.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_utils.h"
+#include "rgx_fwif_km.h"
+#include "rgx_fwif_sf.h"
+#include "debug_common.h"
+
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxcompute.h"
+#include "rgxtransfer.h"
+#include "rgxtdmtransfer.h"
+#include "rgxtimecorr.h"
+#include "rgx_options.h"
+#include "rgxinit.h"
+#include "rgxlayer_impl.h"
+#include "devicemem_history_server.h"
+
+#define DD_SUMMARY_INDENT  ""
+
+#define RGX_DEBUG_STR_SIZE			(150U)
+
+#define RGX_CR_BIF_CAT_BASE0                              (0x1200U)
+#define RGX_CR_BIF_CAT_BASE1                              (0x1208U)
+
+#define RGX_CR_BIF_CAT_BASEN(n) \
+	RGX_CR_BIF_CAT_BASE0 + \
+	((RGX_CR_BIF_CAT_BASE1 - RGX_CR_BIF_CAT_BASE0) * n)
+
+
+#define RGXDBG_BIF_IDS \
+	X(BIF0)\
+	X(BIF1)\
+	X(TEXAS_BIF)\
+	X(DPX_BIF) \
+	X(FWCORE)
+
+#define RGXDBG_SIDEBAND_TYPES \
+	X(META)\
+	X(TLA)\
+	X(DMA)\
+	X(VDMM)\
+	X(CDM)\
+	X(IPP)\
+	X(PM)\
+	X(TILING)\
+	X(MCU)\
+	X(PDS)\
+	X(PBE)\
+	X(VDMS)\
+	X(IPF)\
+	X(ISP)\
+	X(TPF)\
+	X(USCS)\
+	X(PPP)\
+	X(VCE)\
+	X(TPF_CPF)\
+	X(IPF_CPF)\
+	X(FBCDC)
+
+typedef enum
+{
+#define X(NAME) RGXDBG_##NAME,
+	RGXDBG_BIF_IDS
+#undef X
+} RGXDBG_BIF_ID;
+
+typedef enum
+{
+#define X(NAME) RGXDBG_##NAME,
+	RGXDBG_SIDEBAND_TYPES
+#undef X
+} RGXDBG_SIDEBAND_TYPE;
+
+static const IMG_CHAR *const pszPowStateName[] =
+{
+#define X(NAME)	#NAME,
+	RGXFWIF_POW_STATES
+#undef X
+};
+
+static const IMG_CHAR *const pszBIFNames[] =
+{
+#define X(NAME)	#NAME,
+	RGXDBG_BIF_IDS
+#undef X
+};
+
+static const IMG_FLAGS2DESC asHwrState2Description[] =
+{
+	{RGXFWIF_HWR_HARDWARE_OK, " HWR OK;"},
+	{RGXFWIF_HWR_GENERAL_LOCKUP, " General lockup;"},
+	{RGXFWIF_HWR_DM_RUNNING_OK, " DM running ok;"},
+	{RGXFWIF_HWR_DM_STALLING, " DM stalling;"},
+	{RGXFWIF_HWR_FW_FAULT, " FW fault;"},
+	{RGXFWIF_HWR_RESTART_REQUESTED, " Restart requested;"},
+};
+
+static const IMG_FLAGS2DESC asDmState2Description[] =
+{
+	{RGXFWIF_DM_STATE_READY_FOR_HWR, " ready for hwr;"},
+	{RGXFWIF_DM_STATE_NEEDS_SKIP, " needs skip;"},
+	{RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP, " needs PR cleanup;"},
+	{RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR, " needs trace clear;"},
+	{RGXFWIF_DM_STATE_GUILTY_LOCKUP, " guilty lockup;"},
+	{RGXFWIF_DM_STATE_INNOCENT_LOCKUP, " innocent lockup;"},
+	{RGXFWIF_DM_STATE_GUILTY_OVERRUNING, " guilty overrunning;"},
+	{RGXFWIF_DM_STATE_INNOCENT_OVERRUNING, " innocent overrunning;"},
+	{RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH, " hard context switching;"},
+	{RGXFWIF_DM_STATE_GPU_ECC_HWR, " GPU ECC hwr;"},
+};
+
+static const IMG_CHAR * const apszFwOsStateName[RGXFW_CONNECTION_FW_STATE_COUNT] =
+{
+	"offline",
+	"ready",
+	"active",
+	"offloading",
+	"cooldown"
+};
+
+#if defined(PVR_ENABLE_PHR)
+static const IMG_FLAGS2DESC asPHRConfig2Description[] =
+{
+	{BIT_ULL(RGXFWIF_PHR_MODE_OFF), "off"},
+	{BIT_ULL(RGXFWIF_PHR_MODE_RD_RESET), "reset RD hardware"},
+};
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodePMPC
+
+ @Description
+
+ Return the name for the PM managed Page Catalogues
+
+ @Input ui32PC	 - Page Catalogue number
+
+ @Return   void
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodePMPC(IMG_UINT32 ui32PC)
+{
+	const IMG_CHAR* pszPMPC = " (-)";
+
+	switch (ui32PC)
+	{
+		case 0x8: pszPMPC = " (PM-VCE0)"; break;
+		case 0x9: pszPMPC = " (PM-TE0)"; break;
+		case 0xA: pszPMPC = " (PM-ZLS0)"; break;
+		case 0xB: pszPMPC = " (PM-ALIST0)"; break;
+		case 0xC: pszPMPC = " (PM-VCE1)"; break;
+		case 0xD: pszPMPC = " (PM-TE1)"; break;
+		case 0xE: pszPMPC = " (PM-ZLS1)"; break;
+		case 0xF: pszPMPC = " (PM-ALIST1)"; break;
+	}
+
+	return pszPMPC;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeBIFReqTags
+
+ @Description
+
+ Decode the BIF Tag ID and sideband data fields from BIF_FAULT_BANK_REQ_STATUS regs
+
+ @Input eBankID             - BIF identifier
+ @Input ui32TagID           - Tag ID value
+ @Input ui32TagSB           - Tag Sideband data
+ @Output ppszTagID          - Decoded string from the Tag ID
+ @Output ppszTagSB          - Decoded string from the Tag SB
+ @Output pszScratchBuf      - Buffer provided to the function to generate the debug strings
+ @Input ui32ScratchBufSize  - Size of the provided buffer
+
+ @Return   void
+
+******************************************************************************/
+#include "rgxmhdefs_km.h"
+
+static void _RGXDecodeBIFReqTagsXE(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								   IMG_UINT32	ui32TagID,
+								   IMG_UINT32	ui32TagSB,
+								   IMG_CHAR		**ppszTagID,
+								   IMG_CHAR		**ppszTagSB,
+								   IMG_CHAR		*pszScratchBuf,
+								   IMG_UINT32	ui32ScratchBufSize)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+	IMG_BOOL bNewTagEncoding = IMG_FALSE;
+
+	PVR_ASSERT(ppszTagID != NULL);
+	PVR_ASSERT(ppszTagSB != NULL);
+
+	/* tags updated for all cores (auto & consumer) with branch > 36 or only auto cores with branch = 36 */
+	if ((psDevInfo->sDevFeatureCfg.ui32B > 36) ||
+	    (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TILE_REGION_PROTECTION) && (psDevInfo->sDevFeatureCfg.ui32B == 36)))
+	{
+		bNewTagEncoding = IMG_TRUE;
+	}
+
+	switch (ui32TagID)
+	{
+		/* MMU tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MMU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_MMU:	    pszTagID = "MMU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:	pszTagID = "CPU MMU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:	pszTagID = "CPU IFU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:	pszTagID = "CPU LSU"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PT_REQUEST:		pszTagSB = "PT"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PD_REQUEST:		pszTagSB = "PD"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PC_REQUEST:		pszTagSB = "PC"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PT_REQUEST:	pszTagSB = "PM PT"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_REQUEST:	pszTagSB = "PM PD"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_REQUEST:	pszTagSB = "PM PC"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_WREQUEST:	pszTagSB = "PM PD W"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_WREQUEST:	pszTagSB = "PM PC W"; break;
+			}
+			break;
+		}
+
+		/* MIPS */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MIPS:
+		{
+			pszTagID = "MIPS";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_OPCODE_FETCH:	pszTagSB = "Opcode"; break;
+				case RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_DATA_ACCESS:	pszTagSB = "Data"; break;
+			}
+			break;
+		}
+
+		/* CDM tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0:	pszTagID = "CDM Stage 0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1:	pszTagID = "CDM Stage 1"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2:	pszTagID = "CDM Stage 2"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3:	pszTagID = "CDM Stage 3"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTROL_STREAM:	pszTagSB = "Control"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_INDIRECT_DATA:	pszTagSB = "Indirect"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_EVENT_DATA:		pszTagSB = "Event"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTEXT_STATE:	pszTagSB = "Context"; break;
+			}
+			break;
+		}
+
+		/* VDM tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0:	pszTagID = "VDM Stage 0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1:	pszTagID = "VDM Stage 1"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2:	pszTagID = "VDM Stage 2"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3:	pszTagID = "VDM Stage 3"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4:	pszTagID = "VDM Stage 4"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTROL:	pszTagSB = "Control"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STATE:		pszTagSB = "State"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_INDEX:		pszTagSB = "Index"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STACK:		pszTagSB = "Stack"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTEXT:	pszTagSB = "Context"; break;
+			}
+			break;
+		}
+
+		/* PDS */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PDS_0:
+			pszTagID = "PDS req 0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PDS_1:
+			pszTagID = "PDS req 1"; break;
+
+		/* MCU */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCA:
+			pszTagID = "MCU USCA"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCB:
+			pszTagID = "MCU USCB"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCC:
+			pszTagID = "MCU USCC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCD:
+			pszTagID = "MCU USCD"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCA:
+			pszTagID = "MCU PDS USCA"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCB:
+			pszTagID = "MCU PDS USCB"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCC:
+			pszTagID = "MCU PDS USCC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCD:
+			pszTagID = "MCU PDSUSCD"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDSRW:
+			pszTagID = "MCU PDS PDSRW"; break;
+
+		/* TCU */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TCU_0:
+			pszTagID = "TCU req 0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TCU_1:
+			pszTagID = "TCU req 1"; break;
+
+		/* FBCDC */
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_0:
+			pszTagID = bNewTagEncoding ? "TFBDC_TCU0" : "FBCDC0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_1:
+			pszTagID = bNewTagEncoding ? "TFBDC_ZLS0" : "FBCDC1"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_2:
+			pszTagID = bNewTagEncoding ? "TFBDC_TCU1" : "FBCDC2"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_3:
+			pszTagID = bNewTagEncoding ? "TFBDC_ZLS1" : "FBCDC3"; break;
+
+		/* USC Shared */
+		case RGX_MH_TAG_ENCODING_MH_TAG_USC:
+			pszTagID = "USCS"; break;
+
+		/* ISP */
+		case RGX_MH_TAG_ENCODING_MH_TAG_ISP_ZLS:
+			pszTagID = "ISP0 ZLS"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_ISP_DS:
+			pszTagID = "ISP0 DS"; break;
+
+		/* TPF */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF:
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS:
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF:           pszTagID = "TPF0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS:  pszTagID = "TPF0 DBIAS"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF:       pszTagID = "TPF0 SPF"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_PDS_STATE:	pszTagSB = "PDS state"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DEPTH_BIAS:	pszTagSB = "Depth bias"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_FLOOR_OFFSET_DATA:	pszTagSB = "Floor offset"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DELTA_DATA:	pszTagSB = "Delta"; break;
+			}
+			break;
+		}
+
+		/* IPF */
+		case RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ:
+		case RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ:      pszTagID = "IPF0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS:    pszTagID = "IPF0"; break;
+			}
+
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_ISP_IPP_PIPES))
+			{
+				if (ui32TagID < RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+				{
+					OSSNPrintf(pszScratchBuf, ui32ScratchBufSize, "CReq%d", ui32TagID);
+					pszTagSB = pszScratchBuf;
+				}
+				else if (ui32TagID < 2 * RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+				{
+					ui32TagID -= RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES);
+					OSSNPrintf(pszScratchBuf, ui32ScratchBufSize, "PReq%d", ui32TagID);
+					pszTagSB = pszScratchBuf;
+				}
+				else
+				{
+					switch (ui32TagSB - 2 * RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+					{
+						case 0:	pszTagSB = "RReq"; break;
+						case 1:	pszTagSB = "DBSC"; break;
+						case 2:	pszTagSB = "CPF"; break;
+						case 3:	pszTagSB = "Delta"; break;
+					}
+				}
+			}
+			break;
+		}
+
+		/* VDM Stage 5 (temporary) */
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG5:
+			pszTagID = "VDM Stage 5"; break;
+
+		/* TA */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PPP:
+			pszTagID = "PPP"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPWRTC:
+			pszTagID = "TPW RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TEACRTC:
+			pszTagID = "TEAC RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGRTC:
+			pszTagID = "PSG RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGREGION:
+			pszTagID = "PSG Region"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGSTREAM:
+			pszTagID = "PSG Stream"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPW:
+			pszTagID = "TPW"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPC:
+			pszTagID = "TPC"; break;
+
+		/* PM */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PM_ALLOC:
+		{
+			pszTagID = "PMA";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAFSTACK:	pszTagSB = "TA Fstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMLIST:		pszTagSB = "TA MList"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DFSTACK:	pszTagSB = "3D Fstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMLIST:		pszTagSB = "3D MList"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX0:		pszTagSB = "Context0"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX1:		pszTagSB = "Context1"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_MAVP:		pszTagSB = "MAVP"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_UFSTACK:		pszTagSB = "UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMMUSTACK:	pszTagSB = "TA MMUstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMMUSTACK:	pszTagSB = "3D MMUstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAUFSTACK:	pszTagSB = "TA UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DUFSTACK:	pszTagSB = "3D UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAVFP:		pszTagSB = "TA VFP"; break;
+			}
+			break;
+		}
+		case RGX_MH_TAG_ENCODING_MH_TAG_PM_DEALLOC:
+		{
+			pszTagID = "PMD";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAFSTACK:	pszTagSB = "TA Fstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMLIST:		pszTagSB = "TA MList"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DFSTACK:	pszTagSB = "3D Fstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMLIST:		pszTagSB = "3D MList"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX0:		pszTagSB = "Context0"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX1:		pszTagSB = "Context1"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_UFSTACK:		pszTagSB = "UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMMUSTACK:	pszTagSB = "TA MMUstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMMUSTACK:	pszTagSB = "3D MMUstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAUFSTACK:	pszTagSB = "TA UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DUFSTACK:	pszTagSB = "3D UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAVFP:		pszTagSB = "TA VFP"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DVFP:		pszTagSB = "3D VFP"; break;
+			}
+			break;
+		}
+
+		/* TDM */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TDM_DMA:
+		{
+			pszTagID = "TDM DMA";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTL_STREAM: pszTagSB = "Ctl stream"; break;
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTX_BUFFER: pszTagSB = "Ctx buffer"; break;
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_QUEUE_CTL:  pszTagSB = "Queue ctl"; break;
+			}
+			break;
+		}
+		case RGX_MH_TAG_ENCODING_MH_TAG_TDM_CTL:
+		{
+			pszTagID = "TDM CTL";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_FENCE:   pszTagSB = "Fence"; break;
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_CONTEXT: pszTagSB = "Context"; break;
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_QUEUE:   pszTagSB = "Queue"; break;
+			}
+			break;
+		}
+
+		/* PBE */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE0:
+			pszTagID = "PBE0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE1:
+			pszTagID = "PBE1"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE2:
+			pszTagID = "PBE2"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE3:
+			pszTagID = "PBE3"; break;
+	}
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+/* RISC-V pf tags */
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU  (0x00000001U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU  (0x00000002U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU  (0x00000003U)
+
+static void _RGXDecodeBIFReqTagsFwcore(PVRSRV_RGXDEV_INFO *psDevInfo,
+									   IMG_UINT32 ui32TagID,
+									   IMG_UINT32 ui32TagSB,
+									   IMG_CHAR **ppszTagID,
+									   IMG_CHAR **ppszTagSB)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		pszTagSB = "RISC-V";
+
+		switch (ui32TagID)
+		{
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:	pszTagID = "RISC-V MMU"; break;
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:	pszTagID = "RISC-V Instruction Fetch Unit"; break;
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:	pszTagID = "RISC-V Load/Store Unit"; break; /* Or Debug Module System Bus */
+		}
+	}
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+static void _RGXDecodeBIFReqTags(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								 RGXDBG_BIF_ID	eBankID,
+								 IMG_UINT32		ui32TagID,
+								 IMG_UINT32		ui32TagSB,
+								 IMG_CHAR		**ppszTagID,
+								 IMG_CHAR		**ppszTagSB,
+								 IMG_CHAR		*pszScratchBuf,
+								 IMG_UINT32		ui32ScratchBufSize)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+
+	PVR_ASSERT(ppszTagID != NULL);
+	PVR_ASSERT(ppszTagSB != NULL);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		if (eBankID == RGXDBG_FWCORE)
+		{
+			_RGXDecodeBIFReqTagsFwcore(psDevInfo, ui32TagID, ui32TagSB, ppszTagID, ppszTagSB);
+		}
+		else
+		{
+			_RGXDecodeBIFReqTagsXE(psDevInfo, ui32TagID, ui32TagSB, ppszTagID, ppszTagSB, pszScratchBuf, ui32ScratchBufSize);
+		}
+		return;
+	}
+
+	switch (ui32TagID)
+	{
+		case 0x0:
+		{
+			pszTagID = "MMU";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Table"; break;
+				case 0x1: pszTagSB = "Directory"; break;
+				case 0x2: pszTagSB = "Catalogue"; break;
+			}
+			break;
+		}
+		case 0x1:
+		{
+			pszTagID = "TLA";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Pixel data"; break;
+				case 0x1: pszTagSB = "Command stream data"; break;
+				case 0x2: pszTagSB = "Fence or flush"; break;
+			}
+			break;
+		}
+		case 0x2:
+		{
+			pszTagID = "HOST";
+			break;
+		}
+		case 0x3:
+		{
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+			{
+					pszTagID = "META";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "DCache - Thread 0"; break;
+						case 0x1: pszTagSB = "ICache - Thread 0"; break;
+						case 0x2: pszTagSB = "JTag - Thread 0"; break;
+						case 0x3: pszTagSB = "Slave bus - Thread 0"; break;
+						case 0x4: pszTagSB = "DCache - Thread "; break;
+						case 0x5: pszTagSB = "ICache - Thread 1"; break;
+						case 0x6: pszTagSB = "JTag - Thread 1"; break;
+						case 0x7: pszTagSB = "Slave bus - Thread 1"; break;
+					}
+			}
+			else if (RGX_IS_ERN_SUPPORTED(psDevInfo, 57596))
+			{
+				pszTagID="TCU";
+			}
+			else
+			{
+				/* Unreachable code */
+				PVR_ASSERT(IMG_FALSE);
+			}
+			break;
+		}
+		case 0x4:
+		{
+			pszTagID = "USC";
+			OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+			           "Cache line %d", (ui32TagSB & 0x3f));
+			pszTagSB = pszScratchBuf;
+			break;
+		}
+		case 0x5:
+		{
+			pszTagID = "PBE";
+			break;
+		}
+		case 0x6:
+		{
+			pszTagID = "ISP";
+			switch (ui32TagSB)
+			{
+				case 0x00: pszTagSB = "ZLS"; break;
+				case 0x20: pszTagSB = "Occlusion Query"; break;
+			}
+			break;
+		}
+		case 0x7:
+		{
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+			{
+				if (eBankID == RGXDBG_TEXAS_BIF)
+				{
+					pszTagID = "IPF";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "CPF"; break;
+						case 0x1: pszTagSB = "DBSC"; break;
+						case 0x2:
+						case 0x4:
+						case 0x6:
+						case 0x8: pszTagSB = "Control Stream"; break;
+						case 0x3:
+						case 0x5:
+						case 0x7:
+						case 0x9: pszTagSB = "Primitive Block"; break;
+					}
+				}
+				else
+				{
+					pszTagID = "IPP";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "Macrotile Header"; break;
+						case 0x1: pszTagSB = "Region Header"; break;
+					}
+				}
+			}
+			else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SIMPLE_INTERNAL_PARAMETER_FORMAT))
+			{
+				pszTagID = "IPF";
+				switch (ui32TagSB)
+				{
+					case 0x0: pszTagSB = "Region Header"; break;
+					case 0x1: pszTagSB = "DBSC"; break;
+					case 0x2: pszTagSB = "CPF"; break;
+					case 0x3: pszTagSB = "Control Stream"; break;
+					case 0x4: pszTagSB = "Primitive Block"; break;
+				}
+			}
+			else
+			{
+				pszTagID = "IPF";
+				switch (ui32TagSB)
+				{
+					case 0x0: pszTagSB = "Macrotile Header"; break;
+					case 0x1: pszTagSB = "Region Header"; break;
+					case 0x2: pszTagSB = "DBSC"; break;
+					case 0x3: pszTagSB = "CPF"; break;
+					case 0x4:
+					case 0x6:
+					case 0x8: pszTagSB = "Control Stream"; break;
+					case 0x5:
+					case 0x7:
+					case 0x9: pszTagSB = "Primitive Block"; break;
+				}
+			}
+			break;
+		}
+		case 0x8:
+		{
+			pszTagID = "CDM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Control Stream"; break;
+				case 0x1: pszTagSB = "Indirect Data"; break;
+				case 0x2: pszTagSB = "Event Write"; break;
+				case 0x3: pszTagSB = "Context State"; break;
+			}
+			break;
+		}
+		case 0x9:
+		{
+			pszTagID = "VDM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Control Stream"; break;
+				case 0x1: pszTagSB = "PPP State"; break;
+				case 0x2: pszTagSB = "Index Data"; break;
+				case 0x4: pszTagSB = "Call Stack"; break;
+				case 0x8: pszTagSB = "Context State"; break;
+			}
+			break;
+		}
+		case 0xA:
+		{
+			pszTagID = "PM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "PMA_TAFSTACK"; break;
+				case 0x1: pszTagSB = "PMA_TAMLIST"; break;
+				case 0x2: pszTagSB = "PMA_3DFSTACK"; break;
+				case 0x3: pszTagSB = "PMA_3DMLIST"; break;
+				case 0x4: pszTagSB = "PMA_PMCTX0"; break;
+				case 0x5: pszTagSB = "PMA_PMCTX1"; break;
+				case 0x6: pszTagSB = "PMA_MAVP"; break;
+				case 0x7: pszTagSB = "PMA_UFSTACK"; break;
+				case 0x8: pszTagSB = "PMD_TAFSTACK"; break;
+				case 0x9: pszTagSB = "PMD_TAMLIST"; break;
+				case 0xA: pszTagSB = "PMD_3DFSTACK"; break;
+				case 0xB: pszTagSB = "PMD_3DMLIST"; break;
+				case 0xC: pszTagSB = "PMD_PMCTX0"; break;
+				case 0xD: pszTagSB = "PMD_PMCTX1"; break;
+				case 0xF: pszTagSB = "PMD_UFSTACK"; break;
+				case 0x10: pszTagSB = "PMA_TAMMUSTACK"; break;
+				case 0x11: pszTagSB = "PMA_3DMMUSTACK"; break;
+				case 0x12: pszTagSB = "PMD_TAMMUSTACK"; break;
+				case 0x13: pszTagSB = "PMD_3DMMUSTACK"; break;
+				case 0x14: pszTagSB = "PMA_TAUFSTACK"; break;
+				case 0x15: pszTagSB = "PMA_3DUFSTACK"; break;
+				case 0x16: pszTagSB = "PMD_TAUFSTACK"; break;
+				case 0x17: pszTagSB = "PMD_3DUFSTACK"; break;
+				case 0x18: pszTagSB = "PMA_TAVFP"; break;
+				case 0x19: pszTagSB = "PMD_3DVFP"; break;
+				case 0x1A: pszTagSB = "PMD_TAVFP"; break;
+			}
+			break;
+		}
+		case 0xB:
+		{
+			pszTagID = "TA";
+			switch (ui32TagSB)
+			{
+				case 0x1: pszTagSB = "VCE"; break;
+				case 0x2: pszTagSB = "TPC"; break;
+				case 0x3: pszTagSB = "TE Control Stream"; break;
+				case 0x4: pszTagSB = "TE Region Header"; break;
+				case 0x5: pszTagSB = "TE Render Target Cache"; break;
+				case 0x6: pszTagSB = "TEAC Render Target Cache"; break;
+				case 0x7: pszTagSB = "VCE Render Target Cache"; break;
+				case 0x8: pszTagSB = "PPP Context State"; break;
+			}
+			break;
+		}
+		case 0xC:
+		{
+			pszTagID = "TPF";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "TPF0: Primitive Block"; break;
+				case 0x1: pszTagSB = "TPF0: Depth Bias"; break;
+				case 0x2: pszTagSB = "TPF0: Per Primitive IDs"; break;
+				case 0x3: pszTagSB = "CPF - Tables"; break;
+				case 0x4: pszTagSB = "TPF1: Primitive Block"; break;
+				case 0x5: pszTagSB = "TPF1: Depth Bias"; break;
+				case 0x6: pszTagSB = "TPF1: Per Primitive IDs"; break;
+				case 0x7: pszTagSB = "CPF - Data: Pipe 0"; break;
+				case 0x8: pszTagSB = "TPF2: Primitive Block"; break;
+				case 0x9: pszTagSB = "TPF2: Depth Bias"; break;
+				case 0xA: pszTagSB = "TPF2: Per Primitive IDs"; break;
+				case 0xB: pszTagSB = "CPF - Data: Pipe 1"; break;
+				case 0xC: pszTagSB = "TPF3: Primitive Block"; break;
+				case 0xD: pszTagSB = "TPF3: Depth Bias"; break;
+				case 0xE: pszTagSB = "TPF3: Per Primitive IDs"; break;
+				case 0xF: pszTagSB = "CPF - Data: Pipe 2"; break;
+			}
+			break;
+		}
+		case 0xD:
+		{
+			pszTagID = "PDS";
+			break;
+		}
+		case 0xE:
+		{
+			pszTagID = "MCU";
+			{
+				IMG_UINT32 ui32Burst = (ui32TagSB >> 5) & 0x7;
+				IMG_UINT32 ui32GroupEnc = (ui32TagSB >> 2) & 0x7;
+				IMG_UINT32 ui32Group = ui32TagSB & 0x3;
+
+				IMG_CHAR* pszBurst = "";
+				IMG_CHAR* pszGroupEnc = "";
+				IMG_CHAR* pszGroup = "";
+
+				switch (ui32Burst)
+				{
+					case 0x0:
+					case 0x1: pszBurst = "128bit word within the Lower 256bits"; break;
+					case 0x2:
+					case 0x3: pszBurst = "128bit word within the Upper 256bits"; break;
+					case 0x4: pszBurst = "Lower 256bits"; break;
+					case 0x5: pszBurst = "Upper 256bits"; break;
+					case 0x6: pszBurst = "512 bits"; break;
+				}
+				switch (ui32GroupEnc)
+				{
+					case 0x0: pszGroupEnc = "TPUA_USC"; break;
+					case 0x1: pszGroupEnc = "TPUB_USC"; break;
+					case 0x2: pszGroupEnc = "USCA_USC"; break;
+					case 0x3: pszGroupEnc = "USCB_USC"; break;
+					case 0x4: pszGroupEnc = "PDS_USC"; break;
+					case 0x5:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 > RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "PDSRW";
+						} else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "UPUC_USC";
+						}
+						break;
+					case 0x6:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "TPUC_USC";
+						}
+						break;
+					case 0x7:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "PDSRW";
+						}
+						break;
+				}
+				switch (ui32Group)
+				{
+					case 0x0: pszGroup = "Banks 0-3"; break;
+					case 0x1: pszGroup = "Banks 4-7"; break;
+					case 0x2: pszGroup = "Banks 8-11"; break;
+					case 0x3: pszGroup = "Banks 12-15"; break;
+				}
+
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+								"%s, %s, %s", pszBurst, pszGroupEnc, pszGroup);
+				pszTagSB = pszScratchBuf;
+			}
+			break;
+		}
+		case 0xF:
+		{
+			pszTagID = "FB_CDC";
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+			{
+				IMG_UINT32 ui32Req   = (ui32TagSB >> 0) & 0xf;
+				IMG_UINT32 ui32MCUSB = (ui32TagSB >> 4) & 0x3;
+				IMG_CHAR* pszReqOrig = "";
+
+				switch (ui32Req)
+				{
+					case 0x0: pszReqOrig = "FBC Request, originator ZLS"; break;
+					case 0x1: pszReqOrig = "FBC Request, originator PBE"; break;
+					case 0x2: pszReqOrig = "FBC Request, originator Host"; break;
+					case 0x3: pszReqOrig = "FBC Request, originator TLA"; break;
+					case 0x4: pszReqOrig = "FBDC Request, originator ZLS"; break;
+					case 0x5: pszReqOrig = "FBDC Request, originator MCU"; break;
+					case 0x6: pszReqOrig = "FBDC Request, originator Host"; break;
+					case 0x7: pszReqOrig = "FBDC Request, originator TLA"; break;
+					case 0x8: pszReqOrig = "FBC Request, originator ZLS Requester Fence"; break;
+					case 0x9: pszReqOrig = "FBC Request, originator PBE Requester Fence"; break;
+					case 0xa: pszReqOrig = "FBC Request, originator Host Requester Fence"; break;
+					case 0xb: pszReqOrig = "FBC Request, originator TLA Requester Fence"; break;
+					case 0xc: pszReqOrig = "Reserved"; break;
+					case 0xd: pszReqOrig = "Reserved"; break;
+					case 0xe: pszReqOrig = "FBDC Request, originator FBCDC(Host) Memory Fence"; break;
+					case 0xf: pszReqOrig = "FBDC Request, originator FBCDC(TLA) Memory Fence"; break;
+				}
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+				           "%s, MCU sideband 0x%X", pszReqOrig, ui32MCUSB);
+				pszTagSB = pszScratchBuf;
+			}
+			else
+			{
+				IMG_UINT32 ui32Req   = (ui32TagSB >> 2) & 0x7;
+				IMG_UINT32 ui32MCUSB = (ui32TagSB >> 0) & 0x3;
+				IMG_CHAR* pszReqOrig = "";
+
+				switch (ui32Req)
+				{
+					case 0x0: pszReqOrig = "FBC Request, originator ZLS";   break;
+					case 0x1: pszReqOrig = "FBC Request, originator PBE";   break;
+					case 0x2: pszReqOrig = "FBC Request, originator Host";  break;
+					case 0x3: pszReqOrig = "FBC Request, originator TLA";   break;
+					case 0x4: pszReqOrig = "FBDC Request, originator ZLS";  break;
+					case 0x5: pszReqOrig = "FBDC Request, originator MCU";  break;
+					case 0x6: pszReqOrig = "FBDC Request, originator Host"; break;
+					case 0x7: pszReqOrig = "FBDC Request, originator TLA";  break;
+				}
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+				           "%s, MCU sideband 0x%X", pszReqOrig, ui32MCUSB);
+				pszTagSB = pszScratchBuf;
+			}
+			break;
+		}
+	} /* switch (TagID) */
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMULevel
+
+ @Description
+
+ Return the name for the MMU level that faulted.
+
+ @Input ui32MMULevel	 - MMU level
+
+ @Return   IMG_CHAR* to the string describing the MMU level that faulted.
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodeMMULevel(IMG_UINT32 ui32MMULevel)
+{
+	const IMG_CHAR* pszMMULevel = "";
+
+	switch (ui32MMULevel)
+	{
+		case 0x0: pszMMULevel = " (Page Table)"; break;
+		case 0x1: pszMMULevel = " (Page Directory)"; break;
+		case 0x2: pszMMULevel = " (Page Catalog)"; break;
+		case 0x3: pszMMULevel = " (Cat Base Reg)"; break;
+	}
+
+	return pszMMULevel;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpRGXBIFBank
+
+ @Description
+
+ Dump BIF Bank state in human readable form.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo            - RGX device info
+ @Input eBankID              - BIF identifier
+ @Input ui64MMUStatus        - MMU Status register value
+ @Input ui64ReqStatus        - BIF request Status register value
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpRGXBIFBank(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					RGXDBG_BIF_ID eBankID,
+					IMG_UINT64 ui64MMUStatus,
+					IMG_UINT64 ui64ReqStatus,
+					const IMG_CHAR *pszIndent)
+{
+	if (ui64MMUStatus == 0x0)
+	{
+		PVR_DUMPDEBUG_LOG("%s - OK", pszBIFNames[eBankID]);
+	}
+	else
+	{
+		IMG_UINT32 ui32PageSize;
+		IMG_UINT32 ui32PC =
+			(ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+				RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+
+		/* Bank 0 & 1 share the same fields */
+		PVR_DUMPDEBUG_LOG("%s%s - FAULT:",
+						  pszIndent,
+						  pszBIFNames[eBankID]);
+
+		/* MMU Status */
+		{
+			IMG_UINT32 ui32MMUDataType =
+				(ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT;
+
+			IMG_BOOL bROFault = (ui64MMUStatus & RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN) != 0;
+			IMG_BOOL bProtFault = (ui64MMUStatus & RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN) != 0;
+
+			ui32PageSize = (ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+						RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+
+			PVR_DUMPDEBUG_LOG("%s  * MMU status (0x%016" IMG_UINT64_FMTSPECx "): PC = %d%s, Page Size = %d%s%s%s.",
+						pszIndent,
+						ui64MMUStatus,
+						ui32PC,
+						(ui32PC < 0x8)?"":_RGXDecodePMPC(ui32PC),
+						ui32PageSize,
+						(bROFault)?", Read Only fault":"",
+						(bProtFault)?", PM/META protection fault":"",
+						_RGXDecodeMMULevel(ui32MMUDataType));
+		}
+
+		/* Req Status */
+		{
+			IMG_CHAR *pszTagID;
+			IMG_CHAR *pszTagSB;
+			IMG_CHAR aszScratch[RGX_DEBUG_STR_SIZE];
+			IMG_BOOL bRead;
+			IMG_UINT32 ui32TagSB, ui32TagID;
+			IMG_UINT64 ui64Addr;
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+			{
+				bRead = (ui64ReqStatus & RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN) != 0;
+				ui32TagSB = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT;
+				ui32TagID = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT;
+			}
+			else
+			{
+				bRead = (ui64ReqStatus & RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN) != 0;
+				ui32TagSB = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT;
+				ui32TagID = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT;
+			}
+			ui64Addr = ((ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK) >>
+				RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT) <<
+				RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT;
+
+			_RGXDecodeBIFReqTags(psDevInfo, eBankID, ui32TagID, ui32TagSB, &pszTagID, &pszTagSB, &aszScratch[0], RGX_DEBUG_STR_SIZE);
+
+			PVR_DUMPDEBUG_LOG("%s  * Request (0x%016" IMG_UINT64_FMTSPECx
+						"): %s (%s), %s " IMG_DEV_VIRTADDR_FMTSPEC ".",
+							  pszIndent,
+							  ui64ReqStatus,
+			                  pszTagID,
+			                  pszTagSB,
+			                  (bRead)?"Reading from":"Writing to",
+			                  ui64Addr);
+		}
+	}
+}
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT mismatch!");
+
+static_assert((RGX_CR_MMU_FAULT_STATUS_CONTEXT_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_CONTEXT_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_ADDRESS_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_ADDRESS_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TAG_SB_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TAG_SB_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_REQ_ID_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_REQ_ID_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_LEVEL_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_LEVEL_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_RNW_EN == RGX_CR_MMU_FAULT_STATUS_META_RNW_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_FAULT_EN == RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+
+
+#if !defined(NO_HARDWARE)
+static inline IMG_CHAR const *_GetRISCVException(IMG_UINT32 ui32Mcause)
+{
+	switch (ui32Mcause)
+	{
+#define X(value, fatal, description) \
+		case value: \
+			if (fatal) \
+				return description; \
+			return NULL;
+
+		RGXRISCVFW_MCAUSE_TABLE
+#undef X
+
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "Invalid RISC-V FW mcause value 0x%08x", ui32Mcause));
+			return NULL;
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+static const IMG_FLAGS2DESC asHWErrorState[] =
+{
+	{RGX_HW_ERR_NA, "N/A"},
+	{RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL, "Primitive ID failure during DM kill."},
+};
+
+/*
+ *  Translate ID code to descriptive string.
+ *  Returns on the first match.
+ */
+static void _ID2Description(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, const IMG_FLAGS2DESC *psConvTable, IMG_UINT32 ui32TableSize, IMG_UINT32 ui32ID)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (ui32ID == psConvTable[ui32Idx].uiFlag)
+		{
+			OSStringSafeCopy(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+			return;
+		}
+	}
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWAssert
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psRGXFWIfTraceBufCtl - RGX FW trace buffer
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWAssert(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl)
+{
+	const IMG_CHAR *pszTraceAssertPath;
+	const IMG_CHAR *pszTraceAssertInfo;
+	IMG_INT32 ui32TraceAssertLine;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf, INVALIDATE);
+		pszTraceAssertPath = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szPath;
+		pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szInfo;
+		ui32TraceAssertLine = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.ui32LineNum;
+
+		/* print non-null assert strings */
+		if (*pszTraceAssertInfo)
+		{
+			PVR_DUMPDEBUG_LOG("FW-T%d Assert: %.*s (%.*s:%d)",
+			                  i, RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertPath, ui32TraceAssertLine);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWFaults
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psFwSysData       - RGX FW shared system data
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWFaults(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             const RGXFWIF_SYSDATA *psFwSysData)
+{
+	if (psFwSysData->ui32FWFaults > 0)
+	{
+		IMG_UINT32	ui32StartFault = psFwSysData->ui32FWFaults - RGXFWIF_FWFAULTINFO_MAX;
+		IMG_UINT32	ui32EndFault   = psFwSysData->ui32FWFaults - 1;
+		IMG_UINT32  ui32Index;
+
+		if (psFwSysData->ui32FWFaults < RGXFWIF_FWFAULTINFO_MAX)
+		{
+			ui32StartFault = 0;
+		}
+
+		for (ui32Index = ui32StartFault; ui32Index <= ui32EndFault; ui32Index++)
+		{
+			const RGX_FWFAULTINFO *psFaultInfo = &psFwSysData->sFaultInfo[ui32Index % RGXFWIF_FWFAULTINFO_MAX];
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			/* Split OS timestamp in seconds and nanoseconds */
+			RGXConvertOSTimestampToSAndNS(psFaultInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+			PVR_DUMPDEBUG_LOG("FW Fault %d: %.*s (%.*s:%d)",
+			                  ui32Index+1, RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szPath,
+			                  psFaultInfo->sFaultBuf.ui32LineNum);
+			PVR_DUMPDEBUG_LOG("            Data = 0x%016"IMG_UINT64_FMTSPECx", CRTimer = 0x%012"IMG_UINT64_FMTSPECx", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+			                  psFaultInfo->ui64Data,
+			                  psFaultInfo->ui64CRTimer,
+			                  ui64Seconds, ui64Nanoseconds);
+		}
+	}
+}
+
+static void _RGXDumpFWPoll(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_SYSDATA *psFwSysData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psFwSysData->aui32CrPollAddr[i])
+		{
+			PVR_DUMPDEBUG_LOG("T%u polling %s (reg:0x%08X mask:0x%08X)",
+			                  i,
+			                  ((psFwSysData->aui32CrPollAddr[i] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+			                  psFwSysData->aui32CrPollAddr[i] & ~RGXFW_POLL_TYPE_SET,
+			                  psFwSysData->aui32CrPollMask[i]);
+		}
+	}
+
+}
+
+static void _RGXDumpFWHWRInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							  void *pvDumpDebugFile,
+							  const RGXFWIF_SYSDATA *psFwSysData,
+							  const RGXFWIF_HWRINFOBUF *psHWRInfoBuf,
+							  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL          bAnyLocked = IMG_FALSE;
+	IMG_UINT32        dm, i;
+	IMG_UINT32        ui32LineSize;
+	IMG_CHAR          *pszLine, *pszTemp;
+	const IMG_CHAR    *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+	const IMG_CHAR    szMsgHeader[] = "Number of HWR: ";
+	const IMG_CHAR    szMsgFalse[] = "FALSE(";
+	IMG_CHAR          *pszLockupType = "";
+	const IMG_UINT32  ui32MsgHeaderCharCount = ARRAY_SIZE(szMsgHeader) - 1; /* size includes the null */
+	const IMG_UINT32  ui32MsgFalseCharCount = ARRAY_SIZE(szMsgFalse) - 1;
+	IMG_UINT32        ui32HWRRecoveryFlags;
+	IMG_UINT32        ui32ReadIndex;
+
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+	{
+		apszDmNames[RGXFWIF_DM_TDM] = "2D";
+	}
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] ||
+		    psHWRInfoBuf->aui32HwrDmOverranCount[dm])
+		{
+			bAnyLocked = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) && !bAnyLocked && (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_HARDWARE_OK))
+	{
+		/* No HWR situation, print nothing */
+		return;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_BOOL bAnyHWROccurred = IMG_FALSE;
+
+		for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+		{
+			if (psHWRInfoBuf->aui32HwrDmRecoveredCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm] !=0)
+				{
+					bAnyHWROccurred = IMG_TRUE;
+					break;
+				}
+		}
+
+		if (!bAnyHWROccurred)
+		{
+			return;
+		}
+	}
+
+/* <DM name + left parenthesis> + <UINT32 max num of digits> + <slash> + <UINT32 max num of digits> +
+   <plus> + <UINT32 max num of digits> + <right parenthesis + comma + space> */
+#define FWHWRINFO_DM_STR_SIZE (5U + 10U + 1U + 10U + 1U + 10U + 3U)
+
+	ui32LineSize = sizeof(IMG_CHAR) * (
+			ui32MsgHeaderCharCount +
+			(psDevInfo->sDevFeatureCfg.ui32MAXDMCount * FWHWRINFO_DM_STR_SIZE) +
+			ui32MsgFalseCharCount + 1 + (psDevInfo->sDevFeatureCfg.ui32MAXDMCount*6) + 1
+				/* 'FALSE(' + ')' + (UINT16 max num + comma) per DM + \0 */
+			);
+
+	pszLine = OSAllocMem(ui32LineSize);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Out of mem allocating line string (size: %d)",
+			__func__,
+			ui32LineSize));
+		return;
+	}
+
+	OSStringSafeCopy(pszLine, szMsgHeader, ui32LineSize);
+	pszTemp = pszLine + ui32MsgHeaderCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				FWHWRINFO_DM_STR_SIZE,
+				"%s(%u/%u+%u), ",
+				apszDmNames[dm],
+				psHWRInfoBuf->aui32HwrDmRecoveredCount[dm],
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm],
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm]);
+	}
+
+	OSStringLCat(pszLine, szMsgFalse, ui32LineSize);
+	pszTemp += ui32MsgFalseCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				10 + 1 + 1 /* UINT32 max num + comma + \0 */,
+				(dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1 ? "%u," : "%u)"),
+				psHWRInfoBuf->aui32HwrDmFalseDetectCount[dm]);
+	}
+
+	PVR_DUMPDEBUG_LOG("%s", pszLine);
+
+	OSFreeMem(pszLine);
+
+	/* Print out per HWR info */
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (dm == RGXFWIF_DM_GP)
+		{
+			PVR_DUMPDEBUG_LOG("DM %d (GP)", dm);
+		}
+		else
+		{
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+			{
+				IMG_UINT32 ui32HWRRecoveryFlags = psFwSysData->aui32HWRRecoveryFlags[dm];
+				IMG_CHAR sPerDmHwrDescription[RGX_DEBUG_STR_SIZE];
+				sPerDmHwrDescription[0] = '\0';
+
+				if (ui32HWRRecoveryFlags == RGXFWIF_DM_STATE_WORKING)
+				{
+					OSStringSafeCopy(sPerDmHwrDescription, " working;", RGX_DEBUG_STR_SIZE);
+				}
+				else
+				{
+					DebugCommonFlagStrings(sPerDmHwrDescription, RGX_DEBUG_STR_SIZE,
+						asDmState2Description, ARRAY_SIZE(asDmState2Description),
+						ui32HWRRecoveryFlags);
+				}
+				PVR_DUMPDEBUG_LOG("DM %d (HWRflags 0x%08x:%.*s)", dm, ui32HWRRecoveryFlags,
+								  RGX_DEBUG_STR_SIZE, sPerDmHwrDescription);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("DM %d", dm);
+			}
+		}
+
+		ui32ReadIndex = 0;
+		for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+		{
+			IMG_BOOL bPMFault = IMG_FALSE;
+			IMG_UINT32 ui32PC;
+			IMG_UINT32 ui32PageSize = 0;
+			IMG_DEV_PHYADDR sPCDevPAddr = { 0 };
+			const RGX_HWRINFO *psHWRInfo = &psHWRInfoBuf->sHWRInfo[ui32ReadIndex];
+
+			if (ui32ReadIndex >= RGXFWIF_HWINFO_MAX)
+			{
+				PVR_DUMPDEBUG_LOG("HWINFO index error: %u", ui32ReadIndex);
+				break;
+			}
+
+			if ((psHWRInfo->eDM == dm) && (psHWRInfo->ui32HWRNumber != 0))
+			{
+				IMG_CHAR aui8RecoveryNum[10+10+1];
+				IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+				IMG_BOOL bPageFault = IMG_FALSE;
+				IMG_DEV_VIRTADDR sFaultDevVAddr;
+
+				/* Split OS timestamp in seconds and nanoseconds */
+				RGXConvertOSTimestampToSAndNS(psHWRInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+				ui32HWRRecoveryFlags = psHWRInfo->ui32HWRRecoveryFlags;
+				if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_LOCKUP) { pszLockupType = ", Guilty Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_LOCKUP) { pszLockupType = ", Innocent Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_OVERRUNING) { pszLockupType = ", Guilty Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_OVERRUNING) { pszLockupType = ", Innocent Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH) { pszLockupType = ", Hard Context Switch"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_ECC_HWR) { pszLockupType = ", GPU ECC HWR"; }
+
+				OSSNPrintf(aui8RecoveryNum, sizeof(aui8RecoveryNum), "Recovery %d:", psHWRInfo->ui32HWRNumber);
+				if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+				{
+					PVR_DUMPDEBUG_LOG("  %s Core = %u, PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32CoreID,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+
+				if (psHWRInfo->eHWErrorCode != RGX_HW_ERR_NA)
+				{
+					IMG_CHAR sHWDebugInfo[RGX_DEBUG_STR_SIZE] = "";
+
+					_ID2Description(sHWDebugInfo, RGX_DEBUG_STR_SIZE, asHWErrorState, ARRAY_SIZE(asHWErrorState),
+						psHWRInfo->eHWErrorCode);
+					PVR_DUMPDEBUG_LOG("  HW error code = 0x%X: %s",
+									  psHWRInfo->eHWErrorCode, sHWDebugInfo);
+				}
+
+				pszTemp = &aui8RecoveryNum[0];
+				while (*pszTemp != '\0')
+				{
+					*pszTemp++ = ' ';
+				}
+
+				/* There's currently no time correlation for the Guest OSes on the Firmware so there's no point printing OS Timestamps on Guests */
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   ui64Seconds,
+									   ui64Nanoseconds,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+
+				if (psHWRInfo->ui64CRTimeHWResetFinish != 0)
+				{
+					if (psHWRInfo->ui64CRTimeFreelistReady != 0)
+					{
+						/* If ui64CRTimeFreelistReady is less than ui64CRTimeHWResetFinish it means APM kicked in and the time is not valid. */
+						if (psHWRInfo->ui64CRTimeHWResetFinish < psHWRInfo->ui64CRTimeFreelistReady)
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalRecoveryTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimeHWResetFinish)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimer)*256);
+						}
+						else
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = <not_timed>, TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+						}
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+										   aui8RecoveryNum,
+										   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+					}
+				}
+
+#if defined(HW_ERN_65104_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 65104))
+				{
+					PVR_DUMPDEBUG_LOG("    Active PDS DM USCs = 0x%08x", psHWRInfo->ui32PDSActiveDMUSCs);
+				}
+#endif
+
+#if defined(HW_ERN_69700_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 69700))
+				{
+					PVR_DUMPDEBUG_LOG("    DMs stalled waiting on PDS Store space = 0x%08x", psHWRInfo->ui32PDSStalledDMs);
+				}
+#endif
+
+				switch (psHWRInfo->eHWRType)
+				{
+					case RGX_HWRTYPE_BIF0FAULT:
+					case RGX_HWRTYPE_BIF1FAULT:
+					{
+						_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXFWIF_HWRTYPE_BIF_BANK_GET(psHWRInfo->eHWRType),
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+										DD_NORMAL_INDENT);
+
+						bPageFault = IMG_TRUE;
+						sFaultDevVAddr.uiAddr = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK);
+						ui32PC = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+								RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+						bPMFault = (ui32PC >= 8);
+						ui32PageSize = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+									RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+						sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+					}
+					break;
+					case RGX_HWRTYPE_TEXASBIF0FAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+						{
+							_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+										DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK);
+							ui32PC = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+									RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+							bPMFault = (ui32PC >= 8);
+							ui32PageSize = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+										RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_ECCFAULT:
+					{
+						PVR_DUMPDEBUG_LOG("    ECC fault GPU=0x%08x", psHWRInfo->uHWRData.sECCInfo.ui32FaultGPU);
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUFAULT:
+					{
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUMETAFAULT:
+					{
+					}
+					break;
+
+					case RGX_HWRTYPE_POLLFAILURE:
+					{
+						PVR_DUMPDEBUG_LOG("    T%u polling %s (reg:0x%08X mask:0x%08X last:0x%08X)",
+										  psHWRInfo->uHWRData.sPollInfo.ui32ThreadNum,
+										  ((psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & ~RGXFW_POLL_TYPE_SET,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollMask,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollLastValue);
+					}
+					break;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+					case RGX_HWRTYPE_MIPSTLBFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+						{
+							IMG_UINT32 ui32EntryLo = psHWRInfo->uHWRData.sTLBInfo.ui32EntryLo;
+
+							/* This is not exactly what the MMU code does, but the result should be the same */
+							const IMG_UINT32 ui32UnmappedEntry =
+								((IMG_UINT32)(MMU_BAD_PHYS_ADDR & 0xffffffff) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) | RGXMIPSFW_ENTRYLO_UNCACHED;
+
+							PVR_DUMPDEBUG_LOG("    MIPS TLB fault: BadVA = 0x%08X, EntryLo = 0x%08X"
+											  " (page PA 0x%" IMG_UINT64_FMTSPECx", V %u)",
+											  psHWRInfo->uHWRData.sTLBInfo.ui32BadVAddr,
+											  ui32EntryLo,
+											  RGXMIPSFW_TLB_GET_PA(ui32EntryLo),
+											  ui32EntryLo & RGXMIPSFW_TLB_VALID ? 1 : 0);
+
+							if (ui32EntryLo == ui32UnmappedEntry)
+							{
+								PVR_DUMPDEBUG_LOG("    Potential use-after-free detected");
+							}
+						}
+					}
+					break;
+#endif
+
+					case RGX_HWRTYPE_MMURISCVFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+						{
+							_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_FWCORE,
+											psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+											psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							bPMFault = IMG_FALSE;
+							sFaultDevVAddr.uiAddr =
+								(psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus &
+								 ~RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK);
+							ui32PageSize =
+								(psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus &
+								 ~RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+								RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_SHIFT;
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_OVERRUN:
+					case RGX_HWRTYPE_UNKNOWNFAILURE:
+					{
+						/* Nothing to dump */
+					}
+					break;
+
+					default:
+					{
+						PVR_DUMPDEBUG_LOG("    Unknown HWR Info type: 0x%x", psHWRInfo->eHWRType);
+					}
+					break;
+				}
+
+				if (bPageFault)
+				{
+					RGXDumpFaultInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psHWRInfo,
+					                 ui32ReadIndex, &sFaultDevVAddr, &sPCDevPAddr, bPMFault, ui32PageSize);
+				}
+
+			}
+
+			if (ui32ReadIndex == RGXFWIF_HWINFO_MAX_FIRST - 1)
+				ui32ReadIndex = psHWRInfoBuf->ui32WriteIndex;
+			else
+				ui32ReadIndex = (ui32ReadIndex + 1) - (ui32ReadIndex / RGXFWIF_HWINFO_LAST_INDEX) * RGXFWIF_HWINFO_MAX_LAST;
+		}
+	}
+}
+
+
+
+
+#if !defined(NO_HARDWARE)
+
+/*!
+*******************************************************************************
+
+ @Function	_CheckForPendingPage
+
+ @Description
+
+ Check if the MMU indicates it is blocked on a pending page
+ MMU4 does not support pending pages, so return false.
+
+ @Input psDevInfo	 - RGX device info
+
+ @Return   IMG_BOOL      - IMG_TRUE if there is a pending page
+
+******************************************************************************/
+static INLINE IMG_BOOL _CheckForPendingPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* MMU4 doesn't support pending pages */
+	return (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) < 4) &&
+		   (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY) & RGX_CR_MMU_ENTRY_PENDING_EN);
+#else
+	IMG_UINT32 ui32BIFMMUEntry;
+
+	ui32BIFMMUEntry = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY);
+
+	if (ui32BIFMMUEntry & RGX_CR_BIF_MMU_ENTRY_PENDING_EN)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetPendingPageInfo
+
+ @Description
+
+ Get information about the pending page from the MMU status registers
+
+ @Input psDevInfo	 - RGX device info
+ @Output psDevVAddr      - The device virtual address of the pending MMU address translation
+ @Output pui32CatBase    - The page catalog base
+
+ @Return   void
+
+******************************************************************************/
+static void _GetPendingPageInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_VIRTADDR *psDevVAddr,
+								IMG_UINT32 *pui32CatBase)
+{
+	IMG_UINT64 ui64BIFMMUEntryStatus;
+
+	ui64BIFMMUEntryStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY_STATUS);
+
+	psDevVAddr->uiAddr = (ui64BIFMMUEntryStatus & ~RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_CLRMSK);
+
+	*pui32CatBase = (ui64BIFMMUEntryStatus & ~RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_CLRMSK) >>
+								RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_SHIFT;
+}
+
+#endif
+
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON)
+{
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	IMG_UINT32 ui32DriverID;
+	const RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	/* space for the current clock speed and 3 previous */
+	RGXFWIF_TIME_CORR asTimeCorrs[4];
+	IMG_UINT32 ui32NumClockSpeedChanges;
+
+	/* Should invalidate all reads below including when passed to functions. */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, INVALIDATE);
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(bRGXPoweredON);
+#else
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		{
+			IMG_UINT64	ui64RegValMMUStatus, ui64RegValREQStatus;
+
+			ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK0_MMU_STATUS);
+			ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK0_REQ_STATUS);
+
+			_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_BIF0, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+
+			if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, SINGLE_BIF)))
+			{
+				ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK1_MMU_STATUS);
+				ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK1_REQ_STATUS);
+				_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_BIF1, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+			}
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+			{
+				ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS);
+				ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS);
+				_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_FWCORE, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+			}
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+			{
+				IMG_UINT32  ui32PhantomCnt = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) ?  RGX_REQ_NUM_PHANTOMS(RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS)) : 0;
+
+				if (ui32PhantomCnt > 1)
+				{
+					IMG_UINT32  ui32Phantom;
+					for (ui32Phantom = 0;  ui32Phantom < ui32PhantomCnt;  ui32Phantom++)
+					{
+						/* This can't be done as it may interfere with the FW... */
+						/*OSWriteHWReg64(RGX_CR_TEXAS_INDIRECT, ui32Phantom);*/
+
+						ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS);
+						ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS);
+
+						_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+					}
+				}else
+				{
+					ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS);
+					ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS);
+
+					_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+				}
+			}
+		}
+
+		if (_CheckForPendingPage(psDevInfo))
+		{
+			IMG_UINT32 ui32CatBase;
+			IMG_DEV_VIRTADDR sDevVAddr;
+
+			PVR_DUMPDEBUG_LOG("MMU Pending page: Yes");
+
+			_GetPendingPageInfo(psDevInfo, &sDevVAddr, &ui32CatBase);
+
+			if (ui32CatBase >= 8)
+			{
+				PVR_DUMPDEBUG_LOG("Cannot check address on PM cat base %u", ui32CatBase);
+			}
+			else
+			{
+				IMG_DEV_PHYADDR sPCDevPAddr;
+				MMU_FAULT_DATA sFaultData;
+				IMG_BOOL bIsValid = IMG_TRUE;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+				IMG_UINT64 ui64CBaseMapping;
+				IMG_UINT32 ui32CBaseMapCtxReg;
+
+				if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN);
+				}
+				else
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING_INVALID_EN);
+				}
+#else
+				sPCDevPAddr.uiAddr = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_CAT_BASEN(ui32CatBase));
+#endif
+
+				PVR_DUMPDEBUG_LOG("Checking device virtual address " IMG_DEV_VIRTADDR_FMTSPEC
+							" on cat base %u. PC Addr = 0x%" IMG_UINT64_FMTSPECx " is %s",
+								sDevVAddr.uiAddr,
+								ui32CatBase,
+								sPCDevPAddr.uiAddr,
+								bIsValid ? "valid":"invalid");
+				RGXCheckFaultAddress(psDevInfo, &sDevVAddr, &sPCDevPAddr, &sFaultData);
+				RGXDumpFaultAddressHostView(&sFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_SUMMARY_INDENT);
+			}
+		}
+	}
+#endif /* NO_HARDWARE */
+
+#if !defined(NO_HARDWARE)
+	/* Determine the type virtualisation support used */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+#if defined(SUPPORT_AUTOVZ)
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with HW register support");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with shared memory");
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with static Fw heap allocation");
+#endif /* defined(SUPPORT_AUTOVZ) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with dynamic Fw heap allocation");
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+	}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_CONNECTION_FW_STATE eFwState;
+		RGXFWIF_CONNECTION_OS_STATE eOsState;
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+		eFwState = KM_GET_FW_CONNECTION(psDevInfo);
+		eOsState = KM_GET_OS_CONNECTION(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation firmware connection state: %s (Fw=%s; OS=%s)",
+						  ((eFwState == RGXFW_CONNECTION_FW_ACTIVE) && (eOsState == RGXFW_CONNECTION_OS_ACTIVE)) ? ("UP") : ("DOWN"),
+						  (eFwState < RGXFW_CONNECTION_FW_STATE_COUNT) ? (apszFwOsStateName[eFwState]) : ("invalid"),
+						  (eOsState < RGXFW_CONNECTION_OS_STATE_COUNT) ? (apszFwOsStateName[eOsState]) : ("invalid"));
+
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32FwAliveTS;
+		IMG_UINT32 ui32OsAliveTS;
+
+		KM_ALIVE_TOKEN_CACHEOP(Fw, INVALIDATE);
+		KM_ALIVE_TOKEN_CACHEOP(Os, INVALIDATE);
+
+		ui32FwAliveTS = KM_GET_FW_ALIVE_TOKEN(psDevInfo);
+		ui32OsAliveTS = KM_GET_OS_ALIVE_TOKEN(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation watchdog timestamps (in GPU timer ticks): Fw=%u; OS=%u; diff(FW, OS) = %u",
+						  ui32FwAliveTS, ui32OsAliveTS, ui32FwAliveTS - ui32OsAliveTS);
+	}
+#endif
+#endif /* !defined(NO_HARDWARE) */
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_CHAR sHwrStateDescription[RGX_DEBUG_STR_SIZE];
+		IMG_BOOL bDriverIsolationEnabled = IMG_FALSE;
+		IMG_UINT32 ui32HostIsolationGroup;
+
+		sHwrStateDescription[0] = '\0';
+
+		DebugCommonFlagStrings(sHwrStateDescription, RGX_DEBUG_STR_SIZE,
+			asHwrState2Description, ARRAY_SIZE(asHwrState2Description),
+			psFwSysData->ui32HWRStateFlags);
+		PVR_DUMPDEBUG_LOG("RGX HWR State 0x%08x:%s", psFwSysData->ui32HWRStateFlags, sHwrStateDescription);
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: %s (APM %s: %d ok, %d denied, %d non-idle, %d retry, %d other, %d total. Latency: %u ms)",
+		                  (psFwSysData->ePowState < ARRAY_SIZE(pszPowStateName) ? pszPowStateName[psFwSysData->ePowState] : "???"),
+		                  (psDevInfo->pvAPMISRData)?"enabled":"disabled",
+		                  psDevInfo->ui32ActivePMReqOk - psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqDenied,
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqRetry,
+		                  psDevInfo->ui32ActivePMReqTotal -
+		                  psDevInfo->ui32ActivePMReqOk -
+		                  psDevInfo->ui32ActivePMReqDenied -
+		                  psDevInfo->ui32ActivePMReqRetry -
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqTotal,
+		                  psRuntimeCfg->ui32ActivePMLatencyms);
+
+		ui32NumClockSpeedChanges = (IMG_UINT32) OSAtomicRead(&psDevInfo->psDeviceNode->iNumClockSpeedChanges);
+		RGXGetTimeCorrData(psDevInfo->psDeviceNode, asTimeCorrs, ARRAY_SIZE(asTimeCorrs));
+
+		PVR_DUMPDEBUG_LOG("RGX DVFS: %u frequency changes. "
+		                  "Current frequency: %u.%03u MHz (sampled at %" IMG_UINT64_FMTSPEC " ns). "
+		                  "FW frequency: %u.%03u MHz.",
+		                  ui32NumClockSpeedChanges,
+		                  asTimeCorrs[0].ui32CoreClockSpeed / 1000000,
+		                  (asTimeCorrs[0].ui32CoreClockSpeed / 1000) % 1000,
+		                  asTimeCorrs[0].ui64OSTimeStamp,
+		                  psRuntimeCfg->ui32CoreClockSpeed / 1000000,
+		                  (psRuntimeCfg->ui32CoreClockSpeed / 1000) % 1000);
+		if (ui32NumClockSpeedChanges > 0)
+		{
+			PVR_DUMPDEBUG_LOG("          Previous frequencies: %u.%03u, %u.%03u, %u.%03u MHz (Sampled at "
+							"%" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ")",
+												asTimeCorrs[1].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[1].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[2].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[2].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[3].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[3].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[1].ui64OSTimeStamp,
+												asTimeCorrs[2].ui64OSTimeStamp,
+												asTimeCorrs[3].ui64OSTimeStamp);
+		}
+
+		ui32HostIsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID];
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			RGXFWIF_OS_RUNTIME_FLAGS sFwRunFlags = psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+			IMG_UINT32 ui32IsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID];
+			IMG_BOOL bMTSEnabled = IMG_FALSE;
+
+#if !defined(NO_HARDWARE)
+			if (bRGXPoweredON)
+			{
+				bMTSEnabled = (
+#if defined(FIX_HW_BRN_64502_BIT_MASK)
+								RGX_IS_BRN_SUPPORTED(psDevInfo, 64502) ||
+#endif
+								!RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_VIRTUALISATION)) ?
+								IMG_TRUE : ((OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE_ENABLE) & BIT(ui32DriverID)) != 0);
+			}
+#endif
+
+			PVR_DUMPDEBUG_LOG("RGX FW OS %u - State: %s; Freelists: %s%s; Priority: %u; Isolation group: %u;  Time Slice%s: %u%% (%ums); %s", ui32DriverID,
+							  apszFwOsStateName[sFwRunFlags.bfOsState],
+							  (sFwRunFlags.bfFLOk) ? "Ok" : "Not Ok",
+							  (sFwRunFlags.bfFLGrowPending) ? "; Grow Request Pending" : "",
+							  psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID],
+							  ui32IsolationGroup,
+							  (psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] != 0) ? "" : "*",
+							  psFwSysData->aui32TSMirror[ui32DriverID],
+							  (psFwSysData->aui32TSMirror[ui32DriverID] *
+							   psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs / 100),
+							  (bMTSEnabled) ? "MTS on;" : "MTS off;"
+							 );
+
+			if (ui32IsolationGroup != ui32HostIsolationGroup)
+			{
+				bDriverIsolationEnabled = IMG_TRUE;
+			}
+
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+			{
+				/* don't print guest information on native mode drivers */
+				break;
+			}
+		}
+
+#if defined(PVR_ENABLE_PHR)
+		{
+			IMG_CHAR sPHRConfigDescription[RGX_DEBUG_STR_SIZE];
+
+			sPHRConfigDescription[0] = '\0';
+			DebugCommonFlagStrings(sPHRConfigDescription, RGX_DEBUG_STR_SIZE,
+			                   asPHRConfig2Description, ARRAY_SIZE(asPHRConfig2Description),
+			                   BIT_ULL(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode));
+
+			PVR_DUMPDEBUG_LOG("RGX PHR configuration: (%d) %.*s", psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, RGX_DEBUG_STR_SIZE, sPHRConfigDescription);
+		}
+#endif
+
+		if (bRGXPoweredON && RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			if (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM) > 1U)
+			{
+				PVR_DUMPDEBUG_LOG("RGX MC Configuration: 0x%X (1:primary, 0:secondary)", psFwSysData->ui32McConfig);
+			}
+		}
+
+		if (bDriverIsolationEnabled)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Hard Context Switch deadline: %u ms", psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+		}
+
+		_RGXDumpFWAssert(pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl);
+		_RGXDumpFWFaults(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+		_RGXDumpFWPoll(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("RGX FW State: Unavailable under Guest Mode of operation");
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: Unavailable under Guest Mode of operation");
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+	_RGXDumpFWHWRInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData, psDevInfo->psRGXFWIfHWRInfoBufCtl, psDevInfo);
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+	/* Dump all non-zero values in lines of 8... */
+	{
+		IMG_CHAR    pszLine[(9*RGXFWIF_STATS_FRAMEWORK_LINESIZE)+1];
+		const IMG_UINT32 *pui32FWStatsBuf = psFwSysData->aui32FWStatsBuf;
+		IMG_UINT32  ui32Index1, ui32Index2;
+
+		PVR_DUMPDEBUG_LOG("STATS[START]: RGXFWIF_STATS_FRAMEWORK_MAX=%d", RGXFWIF_STATS_FRAMEWORK_MAX);
+		for (ui32Index1 = 0;  ui32Index1 < RGXFWIF_STATS_FRAMEWORK_MAX;  ui32Index1 += RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+		{
+			IMG_UINT32  ui32OrOfValues = 0;
+			IMG_CHAR    *pszBuf = pszLine;
+
+			/* Print all values in this line and skip if all zero... */
+			for (ui32Index2 = 0;  ui32Index2 < RGXFWIF_STATS_FRAMEWORK_LINESIZE;  ui32Index2++)
+			{
+				ui32OrOfValues |= pui32FWStatsBuf[ui32Index1+ui32Index2];
+				OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32FWStatsBuf[ui32Index1+ui32Index2]);
+				pszBuf += 9; /* write over the '\0' */
+			}
+
+			if (ui32OrOfValues != 0)
+			{
+				PVR_DUMPDEBUG_LOG("STATS[%08x]:%s", ui32Index1, pszLine);
+			}
+		}
+		PVR_DUMPDEBUG_LOG("STATS[END]");
+	}
+#endif
+}
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	RGXRISCVFW_STATE sRiscvState;
+	const IMG_CHAR *pszException;
+	PVRSRV_ERROR eError;
+
+	DDLOG64(FWCORE_MEM_CAT_BASE0);
+	DDLOG64(FWCORE_MEM_CAT_BASE1);
+	DDLOG64(FWCORE_MEM_CAT_BASE2);
+	DDLOG64(FWCORE_MEM_CAT_BASE3);
+	DDLOG64(FWCORE_MEM_CAT_BASE4);
+	DDLOG64(FWCORE_MEM_CAT_BASE5);
+	DDLOG64(FWCORE_MEM_CAT_BASE6);
+	DDLOG64(FWCORE_MEM_CAT_BASE7);
+
+	/* Limit dump to what is currently being used */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4);
+	}
+	else
+#endif
+	{
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG4);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG5);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG6);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG12);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG13);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG14);
+	}
+
+	DDLOG32(FWCORE_MEM_FAULT_MMU_STATUS);
+	DDLOG64(FWCORE_MEM_FAULT_REQ_STATUS);
+	DDLOG32(FWCORE_MEM_MMU_STATUS);
+	DDLOG32(FWCORE_MEM_READS_EXT_STATUS);
+	DDLOG32(FWCORE_MEM_READS_INT_STATUS);
+
+	PVR_DUMPDEBUG_LOG("---- [ RISC-V internal state ] ----");
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGXRiscvIsHalted(psDevInfo))
+	{
+		/* Avoid resuming the RISC-V FW as most operations
+		 * on the debug module require a halted core */
+		PVR_DUMPDEBUG_LOG("(skipping as RISC-V found halted)");
+		return PVRSRV_OK;
+	}
+#endif
+
+	eError = RGXRiscvHalt(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+#define X(name, address)												\
+	eError = RGXRiscvReadReg(psDevInfo, address, &sRiscvState.name);	\
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXRiscvReadReg", _RISCVDMError);	\
+	DDLOGVAL32(#name, sRiscvState.name);
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+
+	eError = RGXRiscvResume(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+	pszException = _GetRISCVException(sRiscvState.mcause);
+	if (pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG("RISC-V FW hit an exception: %s", pszException);
+
+		eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+		}
+	}
+
+	return PVRSRV_OK;
+
+_RISCVDMError:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to communicate with the Debug Module"));
+
+	return eError;
+}
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBVNC_COREID_REG))
+	{
+		DDLOG64(CORE_ID__PBVNC);
+	}
+	else
+	{
+		DDLOG32(CORE_ID);
+		DDLOG32(CORE_REVISION);
+	}
+	DDLOG32(DESIGNER_REV_FIELD1);
+	DDLOG32(DESIGNER_REV_FIELD2);
+	DDLOG64(CHANGESET_NUMBER);
+}
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(MULTICORE_SYSTEM);
+	DDLOG32(MULTICORE_GPU);
+}
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(CLK_CTRL);
+	DDLOG64(CLK_STATUS);
+	DDLOG64(CLK_CTRL2);
+	DDLOG64(CLK_STATUS2);
+}
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	{
+		DDLOG32(BIF_FAULT_BANK0_MMU_STATUS);
+		DDLOG64(BIF_FAULT_BANK0_REQ_STATUS);
+		DDLOG32(BIF_FAULT_BANK1_MMU_STATUS);
+		DDLOG64(BIF_FAULT_BANK1_REQ_STATUS);
+	}
+	DDLOG32(BIF_MMU_STATUS);
+	DDLOG32(BIF_MMU_ENTRY);
+	DDLOG64(BIF_MMU_ENTRY_STATUS);
+
+	{
+		if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE)))
+		{
+			DDLOG32(BIF_STATUS_MMU);
+			DDLOG32(BIF_READS_EXT_STATUS);
+			DDLOG32(BIF_READS_INT_STATUS);
+		}
+		DDLOG32(BIFPM_STATUS_MMU);
+		DDLOG32(BIFPM_READS_EXT_STATUS);
+		DDLOG32(BIFPM_READS_INT_STATUS);
+	}
+
+	{
+		DDLOG64(BIF_CAT_BASE_INDEX);
+		DDLOG64(BIF_CAT_BASE0);
+		DDLOG64(BIF_CAT_BASE1);
+		DDLOG64(BIF_CAT_BASE2);
+		DDLOG64(BIF_CAT_BASE3);
+		DDLOG64(BIF_CAT_BASE4);
+		DDLOG64(BIF_CAT_BASE5);
+		DDLOG64(BIF_CAT_BASE6);
+		DDLOG64(BIF_CAT_BASE7);
+	}
+
+	DDLOG32(BIF_CTRL_INVAL);
+	DDLOG32(BIF_CTRL);
+}
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	IMG_UINT32   ui32TACycles, ui323DCycles, ui32TAOr3DCycles, ui32TAAnd3DCycles;
+
+	DDLOG64(BIF_PM_CAT_BASE_VCE0);
+	DDLOG64(BIF_PM_CAT_BASE_TE0);
+	DDLOG64(BIF_PM_CAT_BASE_ALIST0);
+	DDLOG64(BIF_PM_CAT_BASE_VCE1);
+	DDLOG64(BIF_PM_CAT_BASE_TE1);
+	DDLOG64(BIF_PM_CAT_BASE_ALIST1);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		DDLOG32(MULTICORE_GEOMETRY_CTRL_COMMON);
+		DDLOG32(MULTICORE_FRAGMENT_CTRL_COMMON);
+		DDLOG32(MULTICORE_COMPUTE_CTRL_COMMON);
+	}
+
+	DDLOG32(PERF_TA_PHASE);
+	DDLOG32(PERF_TA_CYCLE);
+	DDLOG32(PERF_3D_PHASE);
+	DDLOG32(PERF_3D_CYCLE);
+
+	ui32TACycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_TA_CYCLE);
+	ui323DCycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_3D_CYCLE);
+	ui32TAOr3DCycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_TA_OR_3D_CYCLE);
+	ui32TAAnd3DCycles = ((ui32TACycles + ui323DCycles) > ui32TAOr3DCycles) ? (ui32TACycles + ui323DCycles - ui32TAOr3DCycles) : 0;
+	DDLOGVAL32("PERF_TA_OR_3D_CYCLE", ui32TAOr3DCycles);
+	DDLOGVAL32("PERF_TA_AND_3D_CYCLE", ui32TAAnd3DCycles);
+
+	DDLOG32(PERF_COMPUTE_PHASE);
+	DDLOG32(PERF_COMPUTE_CYCLE);
+
+	DDLOG32(PM_PARTIAL_RENDER_ENABLE);
+
+	DDLOG32(ISP_RENDER);
+	DDLOG64(TLA_STATUS);
+	DDLOG64(MCU_FENCE);
+
+	DDLOG32(VDM_CONTEXT_STORE_STATUS);
+	DDLOG64(VDM_CONTEXT_STORE_TASK0);
+	DDLOG64(VDM_CONTEXT_STORE_TASK1);
+	DDLOG64(VDM_CONTEXT_STORE_TASK2);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK0);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK1);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK2);
+
+	DDLOG32(ISP_CTL);
+	DDLOG32(ISP_STATUS);
+
+	DDLOG32(CDM_CONTEXT_STORE_STATUS);
+	DDLOG64(CDM_CONTEXT_PDS0);
+	DDLOG64(CDM_CONTEXT_PDS1);
+	DDLOG64(CDM_TERMINATE_PDS);
+	DDLOG64(CDM_TERMINATE_PDS1);
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 47025))
+	{
+		DDLOG64(CDM_CONTEXT_LOAD_PDS0);
+		DDLOG64(CDM_CONTEXT_LOAD_PDS1);
+	}
+}
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	{
+		DDLOG32(SLC_IDLE);
+		DDLOG32(SLC_STATUS0);
+		DDLOG64(SLC_STATUS1);
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_BANKS) && RGX_GET_FEATURE_VALUE(psDevInfo, SLC_BANKS))
+		{
+			DDLOG64(SLC_STATUS2);
+		}
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+		{
+			DDLOG64(SLC_CTRL_BYPASS);
+		}
+		else
+		{
+			DDLOG32(SLC_CTRL_BYPASS);
+		}
+		DDLOG64(SLC_CTRL_MISC);
+	}
+}
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(SIDEKICK_IDLE);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ROGUEXE) &&
+		RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER))
+	{
+		DDLOG32(SAFETY_EVENT_STATUS__ROGUEXE);
+		DDLOG32(MTS_SAFETY_EVENT_ENABLE__ROGUEXE);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER))
+	{
+		DDLOG32(FWCORE_WDT_CTRL);
+	}
+
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevInfo->psDeviceNode, LAYOUT_MARS) > 0)
+	{
+		DDLOG32(SCRATCH0);
+		DDLOG32(SCRATCH1);
+		DDLOG32(SCRATCH2);
+		DDLOG32(SCRATCH3);
+		DDLOG32(SCRATCH4);
+		DDLOG32(SCRATCH5);
+		DDLOG32(SCRATCH6);
+		DDLOG32(SCRATCH7);
+		DDLOG32(SCRATCH8);
+		DDLOG32(SCRATCH9);
+		DDLOG32(SCRATCH10);
+		DDLOG32(SCRATCH11);
+		DDLOG32(SCRATCH12);
+		DDLOG32(SCRATCH13);
+		DDLOG32(SCRATCH14);
+		DDLOG32(SCRATCH15);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#undef REG32_FMTSPEC
+#undef REG64_FMTSPEC
+#undef DDLOG32
+#undef DDLOG64
+#undef DDLOG32_DPX
+#undef DDLOG64_DPX
+#undef DDLOGVAL32
+
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32VerbLevel)
+{
+	DumpTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	DumpRenderCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DumpKickSyncCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		DumpComputeCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		DumpTDMTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+}
+
+/******************************************************************************
+ End of file (rgxdebug.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdevice.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdevice.h
new file mode 100644
index 000000000000..95e0a0b53cca
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdevice.h
@@ -0,0 +1,938 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX device node header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX device node
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEVICE_H)
+#define RGXDEVICE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_device_types.h"
+#include "mmu_common.h"
+#include "rgx_fwif_km.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "osfunc.h"
+#include "rgxlayer_impl.h"
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "hash.h"
+#endif
+typedef struct _RGX_SERVER_COMMON_CONTEXT_ RGX_SERVER_COMMON_CONTEXT;
+
+typedef struct {
+	DEVMEM_MEMDESC		*psFWFrameworkMemDesc;
+} RGX_COMMON_CONTEXT_INFO;
+
+
+/*!
+ ******************************************************************************
+ * Device state flags
+ *****************************************************************************/
+#define RGXKM_DEVICE_STATE_ZERO_FREELIST                          (0x1)  /*!< Zeroing the physical pages of reconstructed free lists */
+#define RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN                  (0x2)  /*!< Used to disable the Devices Watchdog logging */
+#define RGXKM_DEVICE_STATE_GPU_UNITS_POWER_CHANGE_EN              (0x4)  /*!< Used for validation to inject power units state change every DM kick */
+#define RGXKM_DEVICE_STATE_CCB_GROW_EN                            (0x8)  /*!< Used to indicate CCB grow is permitted */
+#define RGXKM_DEVICE_STATE_ENABLE_SPU_UNITS_POWER_MASK_CHANGE_EN  (0x10) /*!< Used for validation to enable SPU power state mask change */
+#define RGXKM_DEVICE_STATE_MASK                                   (0x1F)
+
+/*!
+ ******************************************************************************
+ * ECC RAM Fault Validation
+ *****************************************************************************/
+#define RGXKM_ECC_ERR_INJ_DISABLE 0
+#define RGXKM_ECC_ERR_INJ_SLC     1
+#define RGXKM_ECC_ERR_INJ_USC     2
+#define RGXKM_ECC_ERR_INJ_TPU     3
+#define RGXKM_ECC_ERR_INJ_RASCAL  4
+#define RGXKM_ECC_ERR_INJ_MARS    5
+
+#define RGXKM_ECC_ERR_INJ_INTERVAL 10U
+
+/*!
+ ******************************************************************************
+ * GPU DVFS Table
+ *****************************************************************************/
+
+#define RGX_GPU_DVFS_TABLE_SIZE                      32
+#define RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US       25000     /* Time required to calibrate a clock frequency the first time */
+#define RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US  150000    /* Time required for a recalibration after a DVFS transition */
+#define RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US    10000000  /* Time before the next periodic calibration and correlation */
+
+
+/*!
+ ******************************************************************************
+ * Global flags for driver validation
+ *****************************************************************************/
+#define RGX_VAL_FBDC_SIG_CHECK_NOERR_EN           (0x2U)  /*!< Enable FBDC signature check. Signatures must match */
+#define RGX_VAL_FBDC_SIG_CHECK_ERR_EN             (0x4U)  /*!< Enable FBDC signature check. Signatures must not match */
+#define RGX_VAL_GPUSTATEPIN_EN                    (0x8U)  /*!< Enable GPU state pin check */
+#define RGX_VAL_WGP_SIG_CHECK_NOERR_EN           (0x10U)  /*!< Enable WGP signature check. Signatures must match */
+#define RGX_VAL_WGP_SIG_CHECK_ERR_EN             (0x20U)  /*!< Enable WGP signature check. Signatures must not match */
+#define RGX_VAL_TRP_SIG_CHECK_NOERR_EN           (0x40U)  /*!< Enable TRP signature check. Signatures must match */
+#define RGX_VAL_TRP_SIG_CHECK_ERR_EN             (0x80U)  /*!< Enable TRP signature check. Signatures must not match */
+
+/*!
+ ******************************************************************************
+ * HWPerf L2 Stream ID type definition.
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_HWPERF_L2_STREAM_ID;
+/* HWPerf stream for Client HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_HWPERF 0U
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+/* HWPerf stream for FTrace HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_FTRACE 1U
+#define RGX_HWPERF_L2_STREAM_LAST   2U
+#else
+#define RGX_HWPERF_L2_STREAM_LAST   1U
+#endif
+
+typedef struct _GPU_FREQ_TRACKING_DATA_
+{
+	/* Core clock speed estimated by the driver */
+	IMG_UINT32 ui32EstCoreClockSpeed;
+
+	/* Amount of successful calculations of the estimated core clock speed */
+	IMG_UINT32 ui32CalibrationCount;
+} GPU_FREQ_TRACKING_DATA;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+#define RGX_GPU_FREQ_TRACKING_SIZE 16
+
+typedef struct
+{
+	IMG_UINT64 ui64BeginCRTimestamp;
+	IMG_UINT64 ui64BeginOSTimestamp;
+
+	IMG_UINT64 ui64EndCRTimestamp;
+	IMG_UINT64 ui64EndOSTimestamp;
+
+	IMG_UINT32 ui32EstCoreClockSpeed;
+	IMG_UINT32 ui32CoreClockSpeed;
+} GPU_FREQ_TRACKING_HISTORY;
+#endif
+
+typedef struct _RGX_GPU_DVFS_TABLE_
+{
+	/* Beginning of current calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimestamp;
+	IMG_UINT64 ui64CalibrationOSTimestamp;
+
+	/* Calculated calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimediff;
+	IMG_UINT64 ui64CalibrationOSTimediff;
+
+	/* Current calibration period (in us) */
+	IMG_UINT32 ui32CalibrationPeriod;
+
+	/* System layer frequency table and frequency tracking data */
+	IMG_UINT32 ui32FreqIndex;
+	IMG_UINT32 aui32GPUFrequency[RGX_GPU_DVFS_TABLE_SIZE];
+	GPU_FREQ_TRACKING_DATA asTrackingData[RGX_GPU_DVFS_TABLE_SIZE];
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	IMG_UINT32 ui32HistoryIndex;
+	GPU_FREQ_TRACKING_HISTORY asTrackingHistory[RGX_GPU_FREQ_TRACKING_SIZE];
+#endif
+} RGX_GPU_DVFS_TABLE;
+
+
+/*!
+ ******************************************************************************
+ * GPU utilisation statistics
+ *****************************************************************************/
+
+typedef struct _RGXFWIF_TEMP_GPU_UTIL_STATS_
+{
+	IMG_UINT64 aaaui64DMOSTmpCounters[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 aaui64DMOSTmpLastWord[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastState[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastPeriod[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastTime[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+} RGXFWIF_TEMP_GPU_UTIL_STATS;
+
+typedef struct _RGXFWIF_GPU_UTIL_STATS_
+{
+	IMG_BOOL   bValid;                /* If TRUE, statistics are valid.
+	                                     FALSE if the driver couldn't get reliable stats. */
+	IMG_UINT64 ui64GpuStatActive;     /* GPU active statistic */
+	IMG_UINT64 ui64GpuStatBlocked;    /* GPU blocked statistic */
+	IMG_UINT64 ui64GpuStatIdle;       /* GPU idle statistic */
+	IMG_UINT64 ui64GpuStatCumulative; /* Sum of active/blocked/idle stats */
+
+	IMG_UINT64 aaui64DMOSStatInactive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];   /* Per-DM per-OS sum of idle and blocked stats */
+	IMG_UINT64 aaui64DMOSStatActive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];     /* Per-DM per-OS active statistic */
+	IMG_UINT64 aaui64DMOSStatCumulative[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED]; /* Per-DM per-OS sum of active/blocked/idle stats */
+
+	IMG_UINT64 ui64TimeStamp;         /* Timestamp of the most recent sample of the GPU stats */
+
+	RGXFWIF_TEMP_GPU_UTIL_STATS sTempGpuStats; /* Temporary data used to calculate the per-DM per-OS statistics */
+} RGXFWIF_GPU_UTIL_STATS;
+
+
+typedef struct _RGX_REG_CONFIG_
+{
+	IMG_BOOL               bEnabled;
+	RGXFWIF_REG_CFG_TYPE   eRegCfgTypeToPush;
+	IMG_UINT32             ui32NumRegRecords;
+	POS_LOCK               hLock;
+} RGX_REG_CONFIG;
+
+typedef struct _PVRSRV_STUB_PBDESC_ PVRSRV_STUB_PBDESC;
+
+
+typedef struct _PVRSRV_DEVICE_FEATURE_CONFIG_
+{
+	IMG_UINT64 ui64ErnsBrns;
+	IMG_UINT64 ui64Features;
+	IMG_UINT32 ui32B;
+	IMG_UINT32 ui32V;
+	IMG_UINT32 ui32N;
+	IMG_UINT32 ui32C;
+	IMG_UINT32 ui32FeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX];
+	IMG_UINT32 ui32MAXDMCount;
+	IMG_UINT32 ui32MAXPowUnitCount;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	IMG_UINT32 ui32MAXRACCount;
+#endif
+	IMG_UINT32 ui32SLCSizeInBytes;
+	IMG_PCHAR  pszBVNCString;
+}PVRSRV_DEVICE_FEATURE_CONFIG;
+
+/* This is used to get the value of a specific feature.
+ * Note that it will assert if the feature is disabled or value is invalid. */
+#define RGX_GET_FEATURE_VALUE(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] )
+
+/* This is used to check if the feature value (e.g. with an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] < RGX_FEATURE_VALUE_DISABLED )
+
+/* This is used to check if the Boolean feature (e.g. WITHOUT an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_SUPPORTED(psDevInfo, Feature) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64Features, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if the ERN is available for the currently running BVNC or not */
+#define RGX_IS_ERN_SUPPORTED(psDevInfo, ERN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if the BRN is available for the currently running BVNC or not */
+#define RGX_IS_BRN_SUPPORTED(psDevInfo, BRN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+/* there is a corresponding define in rgxapi.h */
+#define RGX_MAX_TIMER_QUERIES 16U
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*!
+ * The host maintains a 512-deep cache of submitted workloads per device,
+ * i.e. a global look-up table for TA, 3D and compute (depending on the RGX
+ * hardware support present)
+ */
+
+/*
+ * For the workload estimation return data array, the max amount of commands the
+ * MTS can have is 255, therefore 512 (LOG2 = 9) is large enough to account for
+ * all corner cases
+ */
+#define RETURN_DATA_ARRAY_SIZE_LOG2 (9)
+#define RETURN_DATA_ARRAY_SIZE      ((1U) << RETURN_DATA_ARRAY_SIZE_LOG2)
+#define RETURN_DATA_ARRAY_WRAP_MASK (RETURN_DATA_ARRAY_SIZE - 1)
+
+#define WORKLOAD_HASH_SIZE_LOG2		6
+#define WORKLOAD_HASH_SIZE			((1U) << WORKLOAD_HASH_SIZE_LOG2)
+#define WORKLOAD_HASH_WRAP_MASK		(WORKLOAD_HASH_SIZE - 1)
+
+/*!
+ * Workload characteristics for supported data masters.
+ * All characteristics must match for the workload estimate to be used/updated.
+ */
+typedef union _RGX_WORKLOAD_
+{
+	struct
+	{
+		IMG_UINT32				ui32RenderTargetSize;
+		IMG_UINT32				ui32NumberOfDrawCalls;
+		IMG_UINT32				ui32NumberOfIndices;
+		IMG_UINT32				ui32NumberOfMRTs;
+	} sTA3D;
+
+	struct
+	{
+		IMG_UINT32				ui32NumberOfWorkgroups;
+		IMG_UINT32				ui32NumberOfWorkitems;
+	} sCompute;
+
+	struct
+	{
+		IMG_UINT32				ui32Characteristic1;
+		IMG_UINT32				ui32Characteristic2;
+	} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	struct
+	{
+		IMG_UINT32				ui32DispatchSize;
+		IMG_UINT32				ui32AccStructSize;
+	} sRay;
+#endif
+
+} RGX_WORKLOAD;
+
+/*!
+ * Host data used to match the return data (actual cycles count) to the
+ * submitted command packet.
+ * The hash table is a per-DM circular buffer containing a key based on the
+ * workload characteristics. On job completion, the oldest workload data
+ * is evicted if the CB is full and the driver matches the characteristics
+ * to the matching data.
+ *
+ * o If the driver finds a match the existing cycle estimate is averaged with
+ *   the actual cycles used.
+ * o Otherwise a new hash entry is created with the actual cycles for this
+ *   workload.
+ *
+ * Subsequently if a match is found during command submission, the estimate
+ * is passed to the scheduler, e.g. adjust the GPU frequency if PDVFS is enabled.
+ */
+typedef struct _WORKLOAD_MATCHING_DATA_
+{
+	POS_LOCK				psHashLock;
+	HASH_TABLE				*psHashTable;		/*! existing workload cycle estimates for this DM */
+	RGX_WORKLOAD			asHashKeys[WORKLOAD_HASH_SIZE];
+	IMG_UINT64				aui64HashData[WORKLOAD_HASH_SIZE];
+	IMG_UINT32				ui32HashArrayWO;	/*! track the most recent workload estimates */
+} WORKLOAD_MATCHING_DATA;
+
+/*!
+ * A generic container for the workload matching data for GPU contexts:
+ * rendering (TA, 3D), compute, etc.
+ */
+typedef struct _WORKEST_HOST_DATA_
+{
+	union
+	{
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTA;	/*!< matching data for TA commands */
+			WORKLOAD_MATCHING_DATA	sData3D;	/*!< matching data for 3D commands */
+		} sTA3D;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataCDM;	/*!< matching data for CDM commands */
+		} sCompute;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTDM;	/*!< matching data for TDM-TQ commands */
+		} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataRDM;	/*!< matching data for RDM commands */
+		} sRay;
+#endif
+	} uWorkloadMatchingData;
+
+	/*
+	 * This is a per-context property, hence the TA and 3D share the same
+	 * per render context counter.
+	 */
+	IMG_UINT32				ui32WorkEstCCBReceived;	/*!< Used to ensure all submitted work
+														 estimation commands are received
+														 by the host before clean up. */
+} WORKEST_HOST_DATA;
+
+/*!
+ * Entries in the list of submitted workloads, used when the completed command
+ * returns data to the host.
+ *
+ * - the matching data is needed as it holds the hash data
+ * - the host data is needed for completion updates, ensuring memory is not
+ *   freed while workload estimates are in-flight.
+ * - the workload characteristic is used in the hash table look-up.
+ */
+typedef struct _WORKEST_RETURN_DATA_
+{
+	WORKEST_HOST_DATA		*psWorkEstHostData;
+	WORKLOAD_MATCHING_DATA	*psWorkloadMatchingData;
+	RGX_WORKLOAD			sWorkloadCharacteristics;
+} WORKEST_RETURN_DATA;
+#endif
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+#define RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES    4
+#endif
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+typedef struct
+{
+#if defined(PDUMP)
+	IMG_HANDLE      hPdumpPages;
+#endif
+	PG_HANDLE       sPages;
+	IMG_DEV_PHYADDR sPhysAddr;
+} RGX_MIPS_ADDRESS_TRAMPOLINE;
+#endif
+
+
+/*!
+ ******************************************************************************
+ * RGX Device error counts
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_ERROR_COUNTS_
+{
+	IMG_UINT32 ui32WGPErrorCount;		/*!< count of the number of WGP checksum errors */
+	IMG_UINT32 ui32TRPErrorCount;		/*!< count of the number of TRP checksum errors */
+} PVRSRV_RGXDEV_ERROR_COUNTS;
+
+/*!
+ ******************************************************************************
+ * RGX Debug dump firmware trace log type
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_FWT_LOGTYPE;
+#define RGX_FWT_LOGTYPE_NONE    0U
+#define RGX_FWT_LOGTYPE_BINARY  1U
+#define RGX_FWT_LOGTYPE_DECODED 2U
+#define RGX_FWT_LOGTYPE_PARTIAL 3U
+
+/*!
+ ******************************************************************************
+ * RGX Device info
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_INFO_
+{
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG	sDevFeatureCfg;
+
+	IMG_BOOL				bDevInit2Done;
+
+	IMG_BOOL				bFirmwareInitialised;
+	IMG_BOOL				bPDPEnabled;
+
+	IMG_HANDLE				hDbgReqNotify;
+
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvRegsBaseKM;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvSecureRegsBaseKM;
+#endif
+
+	IMG_HANDLE				hRegMapping;
+
+	/* System physical address of device registers */
+	IMG_CPU_PHYADDR			sRegsPhysBase;
+	/* Register region size in bytes */
+	IMG_UINT32				ui32RegSize;
+
+	PVRSRV_STUB_PBDESC		*psStubPBDescListKM;
+
+	/* Firmware memory context info */
+	DEVMEM_CONTEXT			*psKernelDevmemCtx;
+	DEVMEM_HEAP				*psFirmwareMainHeap;
+	DEVMEM_HEAP				*psFirmwareConfigHeap;
+	MMU_CONTEXT				*psKernelMMUCtx;
+
+	void					*pvDeviceMemoryHeap;
+
+	/* Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBCtlMemDesc;      /*!< memdesc for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtl;             /*!< kernel mapping for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtlLocal;        /*!< cpu local copy of Kernel CCB control */
+	DEVMEM_MEMDESC			*psKernelCCBMemDesc;         /*!< memdesc for Kernel CCB */
+	IMG_UINT8				*psKernelCCB;                /*!< kernel mapping for Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBRtnSlotsMemDesc; /*!< Return slot array for Kernel CCB commands */
+	IMG_UINT32				*pui32KernelCCBRtnSlots;     /*!< kernel mapping for return slot array */
+
+	/* Firmware CCB */
+	DEVMEM_MEMDESC			*psFirmwareCCBCtlMemDesc;   /*!< memdesc for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtl;          /*!< kernel mapping for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtlLocal;     /*!< cpu local copy of Firmware CCB control */
+	DEVMEM_MEMDESC			*psFirmwareCCBMemDesc;      /*!< memdesc for Firmware CCB */
+	IMG_UINT8				*psFirmwareCCB;             /*!< kernel mapping for Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBCtlMemDesc;   /*!< memdesc for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtl;          /*!< kernel mapping for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtlLocal;     /*!< cpu local copy of Workload Estimation Firmware CCB control */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBMemDesc;      /*!< memdesc for Workload Estimation Firmware CCB */
+	IMG_UINT8				*psWorkEstFirmwareCCB;             /*!< kernel mapping for Workload Estimation Firmware CCB */
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	/* Counter dumping */
+	DEVMEM_MEMDESC			*psCounterBufferMemDesc;      /*!< mem desc for counter dumping buffer */
+	POS_LOCK				hCounterDumpingLock;          /*!< Lock for guarding access to counter dumping buffer */
+#endif
+
+	PVRSRV_MEMALLOCFLAGS_T  uiFWPoisonOnFreeFlag;           /*!< Flag for poisoning FW allocations when freed */
+
+	IMG_BOOL				bIgnoreHWReportedBVNC;			/*!< Ignore BVNC reported by HW */
+
+    /* multicore configuration information */
+    IMG_UINT32              ui32MultiCoreNumCores;      /* total cores primary + secondaries. 0 for non-multi core */
+    IMG_UINT32              ui32MultiCorePrimaryId;     /* primary core id for this device */
+    IMG_UINT64             *pui64MultiCoreCapabilities; /* capabilities for each core */
+
+	/*
+		if we don't preallocate the pagetables we must
+		insert newly allocated page tables dynamically
+	*/
+	void					*pvMMUContextList;
+
+	IMG_UINT32				ui32ClkGateStatusReg;
+	IMG_UINT32				ui32ClkGateStatusMask;
+
+	DEVMEM_MEMDESC			*psRGXFWCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCodeDevVAddrBase;
+	IMG_UINT32			ui32FWCodeSizeInBytes;
+	DEVMEM_MEMDESC			*psRGXFWDataMemDesc;
+	IMG_DEV_VIRTADDR		sFWDataDevVAddrBase;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	RGX_MIPS_ADDRESS_TRAMPOLINE	*psTrampoline;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWCorememCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememCodeDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememCodeFWAddr;
+	IMG_UINT32			ui32FWCorememCodeSizeInBytes;
+
+	DEVMEM_MEMDESC			*psRGXFWIfCorememDataStoreMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememDataStoreDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememDataStoreFWAddr;
+
+	DEVMEM_MEMDESC			*psRGXFWAlignChecksMemDesc;
+
+#if defined(PDUMP)
+	DEVMEM_MEMDESC			*psRGXFWSigTAChecksMemDesc;
+	IMG_UINT32				ui32SigTAChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSig3DChecksMemDesc;
+	IMG_UINT32				ui32Sig3DChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigTDMChecksMemDesc;
+	IMG_UINT32				ui32SigTDMChecksSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DEVMEM_MEMDESC			*psRGXFWSigRDMChecksMemDesc;
+	IMG_UINT32				ui32SigRDMChecksSize;
+#endif
+
+	IMG_BOOL				bDumpedKCCBCtlAlready;
+
+	POS_SPINLOCK			hSyncCheckpointSignalSpinLock;						/*!< Guards data shared between an atomic & sleepable-context */
+#endif
+
+	POS_LOCK				hRGXFWIfBufInitLock;								/*!< trace buffer lock for initialisation phase */
+
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufCtlMemDesc;						/*!< memdesc of trace buffer control structure */
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufferMemDesc[RGXFW_THREAD_NUM];		/*!< memdesc of actual FW trace (log) buffer(s) */
+	IMG_PUINT32				apui32TraceBuffer[RGXFW_THREAD_NUM];				/*!< Trace buffer address (Host address), to be used by host when reading from trace buffer */
+	IMG_UINT32				ui32TraceBufSizeInDWords;							/*!< CPU local copy of FW Trace buffer size in dwords */
+	RGXFWIF_TRACEBUF		*psRGXFWIfTraceBufCtl;								/*!< structure containing trace control data and actual trace buffer */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwSysDataMemDesc;							/*!< memdesc of the firmware-shared system data structure */
+	RGXFWIF_SYSDATA			*psRGXFWIfFwSysData;								/*!< structure containing km-firmware shared system data */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwOsDataMemDesc;							/*!< memdesc of the firmware-shared os structure */
+	RGXFWIF_OSDATA			*psRGXFWIfFwOsData;									/*!< structure containing km-firmware shared os data */
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	DEVMEM_MEMDESC			*psRGXFWIfTBIBufferMemDesc;							/*!< memdesc of actual FW TBI buffer */
+	RGXFWIF_DEV_VIRTADDR	sRGXFWIfTBIBuffer;									/*!< TBI buffer data */
+	IMG_UINT32				ui32FWIfTBIBufferSize;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWRInfoBufCtlMemDesc;
+	RGXFWIF_HWRINFOBUF		*psRGXFWIfHWRInfoBufCtl;
+	IMG_UINT32				ui32ClockSource;
+	IMG_UINT32				ui32LastClockSource;
+
+	DEVMEM_MEMDESC			*psRGXFWIfGpuUtilFWCtlMemDesc;
+	RGXFWIF_GPU_UTIL_FW     *psRGXFWIfGpuUtilFW;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfBufMemDesc;
+	IMG_BYTE				*psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32RGXFWIfHWPerfBufSize; /* in bytes */
+	IMG_UINT32				ui32RGXL2HWPerfBufSize;   /* in bytes */
+
+	DEVMEM_MEMDESC			*psRGXFWIfRegCfgMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfCountersMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfConnectionCtlMemDesc;
+	RGXFWIF_CONNECTION_CTL	*psRGXFWIfConnectionCtl;
+
+	DEVMEM_MEMDESC			*psRGXFWHeapGuardPageReserveMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfSysInitMemDesc;
+	RGXFWIF_SYSINIT			*psRGXFWIfSysInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfOsInitMemDesc;
+	RGXFWIF_OSINIT			*psRGXFWIfOsInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfRuntimeCfgMemDesc;
+	RGXFWIF_RUNTIME_CFG		*psRGXFWIfRuntimeCfg;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	DEVMEM_MEMDESC			*psRGXFWIfActiveContextBufDesc;
+	RGXFWIF_ACTIVE_CONTEXT_BUF_DATA		*psRGXFWIfActiveContextBuf;
+#endif
+
+	/* Premapped firmware memory context info */
+	DEVMEM_HEAP				*psPremappedFwRawHeap[RGX_NUM_DRIVERS_SUPPORTED];
+	DEVMEM_MEMDESC			*psPremappedFwRawMemDesc[RGX_NUM_DRIVERS_SUPPORTED];
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Array to store data needed for workload estimation when a workload
+	   has finished and its cycle time is returned to the host.	 */
+	WORKEST_RETURN_DATA     asReturnData[RETURN_DATA_ARRAY_SIZE];
+	IMG_UINT32              ui32ReturnDataWO;
+	POS_LOCK                hWorkEstLock;
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/**
+	 * Host memdesc and pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory on changing the core clock rate over
+	 * GPIO.
+	 * Note: Shared memory needs atomic access from Host driver and firmware,
+	 * hence size should not be greater than memory transaction granularity.
+	 * Currently it is chosen to be 32 bits.
+	 */
+	DEVMEM_MEMDESC			*psRGXFWIFCoreClkRateMemDesc;
+	volatile IMG_UINT32		*pui32RGXFWIFCoreClkRate;
+	/**
+	 * Last sampled core clk rate.
+	 */
+	volatile IMG_UINT32		ui32CoreClkRateSnapshot;
+#endif
+
+	/*
+	   HWPerf data for the RGX device
+	 */
+
+	POS_LOCK    hHWPerfLock;  /*! Critical section lock that protects HWPerf code
+	                           *  from multiple thread duplicate init/deinit
+	                           *  and loss/freeing of FW & Host resources while in
+	                           *  use in another thread e.g. MSIR. */
+
+	IMG_UINT64  ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_LAST];         /*! Event filter for FW events (settable by AppHint) */
+	IMG_HANDLE  hHWPerfStream[RGX_HWPERF_L2_STREAM_LAST];            /*! TL Stream buffer (L2) for firmware event stream */
+	IMG_UINT32  ui32L2BufMaxPacketSize[RGX_HWPERF_L2_STREAM_LAST];   /*! Max allowed packet size in FW HWPerf TL (L2) buffer */
+	IMG_BOOL    bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_LAST]; /*! Flag to indicate if copying HWPerf data is suspended */
+	IMG_UINT64  ui64HWPerfFwFilter;                                  /*! Event filter for FW events created from OR-ing ui64HWPerfFilter values. */
+	IMG_UINT32  uiHWPerfStreamCount;                                 /*! Value indicating if any of the HWPerf streams has been created */
+
+	IMG_UINT32  ui32HWPerfHostFilter;      /*! Event filter for HWPerfHost stream (settable by AppHint) */
+	POS_LOCK    hLockHWPerfHostStream;     /*! Lock guarding access to HWPerfHost stream from multiple threads */
+	IMG_HANDLE  hHWPerfHostStream;         /*! TL Stream buffer for host only event stream */
+	IMG_UINT32  ui32HWPerfHostBufSize;     /*! Host side buffer size in bytes */
+	IMG_UINT32  ui32HWPerfHostLastOrdinal; /*! Ordinal of the last packet emitted in HWPerfHost TL stream.
+	                                        *  Guarded by hLockHWPerfHostStream */
+	IMG_UINT32  ui32HWPerfHostNextOrdinal; /*! Ordinal number for HWPerfHost events. Guarded by hHWPerfHostSpinLock */
+	IMG_UINT8   *pui8DeferredEvents;       /*! List of HWPerfHost events yet to be emitted in the TL stream.
+	                                        *  Events generated from atomic context are deferred "emitted"
+											*  as the "emission" code can sleep */
+	IMG_UINT16  ui16DEReadIdx;             /*! Read index in the above deferred events buffer */
+	IMG_UINT16  ui16DEWriteIdx;            /*! Write index in the above deferred events buffer */
+	void        *pvHostHWPerfMISR;         /*! MISR to emit pending/deferred events in HWPerfHost TL stream */
+	POS_SPINLOCK hHWPerfHostSpinLock;      /*! Guards data shared between an atomic & sleepable-context */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32  ui32DEHighWatermark;       /*! High watermark of deferred events buffer usage. Protected by
+	                                        *! hHWPerfHostSpinLock */
+	/* Max number of times DeferredEmission waited for an atomic-context to "finish" packet write */
+	IMG_UINT32  ui32WaitForAtomicCtxPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an atomic-context packet loss (due to too long wait for "write" finish) */
+	IMG_BOOL    bWarnedAtomicCtxPktLost;
+	/* Max number of times DeferredEmission scheduled-out to give a chance to the right-ordinal packet to be emitted */
+	IMG_UINT32  ui32WaitForRightOrdPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an packet loss (due to too long wait for right ordinal to emit) */
+	IMG_BOOL    bWarnedPktOrdinalBroke;
+#endif
+
+	void        *pvGpuFtraceData;
+
+	/* Poll data for detecting firmware fatal errors */
+	IMG_UINT32				aui32CrLastPollCount[RGXFW_THREAD_NUM];
+	IMG_UINT32				ui32KCCBCmdsExecutedLastTime;
+	IMG_BOOL				bKCCBCmdsWaitingLastTime;
+	IMG_UINT32				ui32GEOTimeoutsLastTime;
+	IMG_UINT32				ui32InterruptCountLastTime;
+	IMG_UINT32				ui32MissingInterruptsLastTime;
+
+	/* Client stall detection */
+	IMG_UINT32				ui32StalledClientMask;
+
+	IMG_BOOL				bWorkEstEnabled;
+	IMG_BOOL				bPDVFSEnabled;
+
+	void					*pvLISRData;
+	void					*pvMISRData;
+	void					*pvAPMISRData;
+	RGX_ACTIVEPM_CONF		eActivePMConf;
+
+	volatile IMG_UINT32		aui32SampleIRQCount[RGXFW_THREAD_NUM];
+
+	DEVMEM_MEMDESC			*psRGXFaultAddressMemDesc;
+
+	DEVMEM_MEMDESC			*psSLC3FenceMemDesc;
+
+	/* If we do 10 deferred memory allocations per second, then the ID would wrap around after 13 years */
+	IMG_UINT32				ui32ZSBufferCurrID;	/*!< ID assigned to the next deferred devmem allocation */
+	IMG_UINT32				ui32FreelistCurrID;	/*!< ID assigned to the next freelist */
+
+	POS_LOCK				hLockZSBuffer;		/*!< Lock to protect simultaneous access to ZSBuffers */
+	DLLIST_NODE				sZSBufferHead;		/*!< List of on-demand ZSBuffers */
+	POS_LOCK				hLockFreeList;		/*!< Lock to protect simultaneous access to Freelists */
+	DLLIST_NODE				sFreeListHead;		/*!< List of growable Freelists */
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+	PVRSRV_CLIENT_SYNC_PRIM	*psPowSyncPrim;
+
+	IMG_UINT32				ui32ActivePMReqOk;
+	IMG_UINT32				ui32ActivePMReqDenied;
+	IMG_UINT32				ui32ActivePMReqNonIdle;
+	IMG_UINT32				ui32ActivePMReqRetry;
+	IMG_UINT32				ui32ActivePMReqTotal;
+
+	IMG_HANDLE				hProcessQueuesMISR;
+
+	IMG_UINT32				ui32DeviceFlags;		/*!< Flags to track general device state */
+
+	/* GPU DVFS Table */
+	RGX_GPU_DVFS_TABLE		*psGpuDVFSTable;
+
+	/* Pointer to function returning the GPU utilisation statistics since the last
+	 * time the function was called. Supports different users at the same time.
+	 *
+	 * psReturnStats [out]: GPU utilisation statistics (active high/active low/idle/blocked)
+	 *                      in microseconds since the last time the function was called
+	 *                      by a specific user (identified by hGpuUtilUser)
+	 *
+	 * Returns PVRSRV_OK in case the call completed without errors,
+	 * some other value otherwise.
+	 */
+	PVRSRV_ERROR (*pfnGetGpuUtilStats) (PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                    IMG_HANDLE hGpuUtilUser,
+	                                    RGXFWIF_GPU_UTIL_STATS *psReturnStats);
+
+	/* Pointer to function that checks if the physical GPU IRQ
+	 * line has been asserted and clears it if so */
+	IMG_BOOL (*pfnRGXAckIrq) (struct _PVRSRV_RGXDEV_INFO_ *psDevInfo);
+
+	POS_LOCK				hGPUUtilLock;
+
+	/* Register configuration */
+	RGX_REG_CONFIG			sRegConfig;
+
+	IMG_BOOL				bRGXPowered;
+	DLLIST_NODE				sMemoryContextList;
+
+	POSWR_LOCK				hRenderCtxListLock;
+	POSWR_LOCK				hComputeCtxListLock;
+	POSWR_LOCK				hTransferCtxListLock;
+	POSWR_LOCK				hTDMCtxListLock;
+	POSWR_LOCK				hMemoryCtxListLock;
+	POSWR_LOCK				hKickSyncCtxListLock;
+
+	/* Linked list of deferred KCCB commands due to a full KCCB.
+	 * Access to members sKCCBDeferredCommandsListHead and ui32KCCBDeferredCommandsCount
+	 * are protected by the hLockKCCBDeferredCommandsList spin lock. */
+	POS_SPINLOCK			hLockKCCBDeferredCommandsList; /*!< Protects deferred KCCB commands list */
+	DLLIST_NODE				sKCCBDeferredCommandsListHead;
+	IMG_UINT32				ui32KCCBDeferredCommandsCount; /*!< No of commands in the deferred list */
+
+	/* Linked lists of contexts on this device */
+	DLLIST_NODE				sRenderCtxtListHead;
+	DLLIST_NODE				sComputeCtxtListHead;
+	DLLIST_NODE				sTransferCtxtListHead;
+	DLLIST_NODE				sTDMCtxtListHead;
+	DLLIST_NODE				sKickSyncCtxtListHead;
+
+	DLLIST_NODE				sCommonCtxtListHead;
+	POSWR_LOCK				hCommonCtxtListLock;
+	IMG_UINT32				ui32CommonCtxtCurrentID;	/*!< ID assigned to the next common context */
+
+	POS_LOCK				hDebugFaultInfoLock;	/*!< Lock to protect the debug fault info list */
+	POS_LOCK				hMMUCtxUnregLock;		/*!< Lock to protect list of unregistered MMU contexts */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	POS_LOCK				hNMILock; /*!< Lock to protect NMI operations */
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32				ui32AvailablePowUnitsMask;
+	IMG_UINT32				ui32AvailableRACMask;
+#endif
+
+	RGX_LAYER_PARAMS		sLayerParams;
+
+	RGXFWIF_DM				eBPDM;					/*!< Current breakpoint data master */
+	IMG_BOOL				bBPSet;					/*!< A Breakpoint has been set */
+	POS_LOCK				hBPLock;				/*!< Lock for break point operations */
+
+	IMG_UINT32				ui32CoherencyTestsDone;
+
+	ATOMIC_T				iCCBSubmissionOrdinal; /* Rolling count used to indicate CCB submission order (all CCBs) */
+	POS_LOCK				hCCBRecoveryLock;      /* Lock to protect pvEarliestStalledClientCCB and ui32OldestSubmissionOrdinal variables */
+	void					*pvEarliestStalledClientCCB; /* Will point to cCCB command to unblock in the event of a stall */
+	IMG_UINT32				ui32OldestSubmissionOrdinal; /* Earliest submission ordinal of CCB entry found so far */
+	IMG_UINT32				ui32SLRHoldoffCounter;   /* Decremented each time health check is called until zero. SLR only happen when zero. */
+
+	POS_LOCK				hCCBStallCheckLock; /* Lock used to guard against multiple threads simultaneously checking for stalled CCBs */
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Firmware gcov buffer */
+	DEVMEM_MEMDESC			*psFirmwareGcovBufferMemDesc;      /*!< mem desc for Firmware gcov dumping buffer */
+	IMG_UINT32				ui32FirmwareGcovSize;
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* Value to store for each page size range config register in MMU4 */
+	IMG_UINT64				aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES];
+#endif
+
+
+
+	IMG_HANDLE				hTQCLISharedMem;		/*!< TQ Client Shared Mem PMR */
+	IMG_HANDLE				hTQUSCSharedMem;		/*!< TQ USC Shared Mem PMR */
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	DEVMEM_MEMDESC			*psRGXFWIfSecureBufMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfNonSecureBufMemDesc;
+#endif
+
+	/* Timer Queries */
+	IMG_UINT32				ui32ActiveQueryId;		/*!< id of the active line */
+	IMG_BOOL				bSaveStart;				/*!< save the start time of the next kick on the device*/
+	IMG_BOOL				bSaveEnd;				/*!< save the end time of the next kick on the device*/
+
+	DEVMEM_MEMDESC			*psStartTimeMemDesc;    /*!< memdesc for Start Times */
+	IMG_UINT64				*pui64StartTimeById;    /*!< CPU mapping of the above */
+
+	DEVMEM_MEMDESC			*psEndTimeMemDesc;      /*!< memdesc for End Timer */
+	IMG_UINT64				*pui64EndTimeById;      /*!< CPU mapping of the above */
+
+	IMG_UINT32				aui32ScheduledOnId[RGX_MAX_TIMER_QUERIES];	/*!< kicks Scheduled on QueryId */
+	DEVMEM_MEMDESC			*psCompletedMemDesc;	/*!< kicks Completed on QueryId */
+	IMG_UINT32				*pui32CompletedById;	/*!< CPU mapping of the above */
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	POS_LOCK				hTimerQueryLock;		/*!< lock to protect simultaneous access to timer query members */
+#endif
+
+	PVRSRV_RGXDEV_ERROR_COUNTS sErrorCounts;		/*!< struct containing device error counts */
+
+	IMG_UINT32				ui32HostSafetyEventMask;/*!< mask of the safety events handled by the driver */
+
+	RGX_CONTEXT_RESET_REASON	eLastDeviceError;	/*!< device error reported to client */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	PMR						*psGenHeapSecMem;		/*!< An allocation of secure memory mapped to
+													  the general devmem heap. The allocation is
+													  created and mapped at driver init. It's used for
+													  various purposes. See rgx_fwif_km.h for all use cases. */
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	DEVMEM_MEMDESC			*psRGXFWScratchBufMemDesc;
+#endif
+
+	RGX_FWT_LOGTYPE			eDebugDumpFWTLogType;
+
+	RGX_FW_INFO_HEADER      sFWInfoHeader;
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	IMG_UINT32              ui32TFBCLossyGroup;     /*!< TFBCCompressionControlGroup
+													  setting for those cores which support
+													  this feature. */
+#endif
+	RGXFWIF_GPU_UTIL_STATS	sGpuUtilStats;          /*!< GPU usage statistics */
+	POS_LOCK				hGpuUtilStatsLock;
+
+} PVRSRV_RGXDEV_INFO;
+
+
+
+typedef struct _RGX_TIMING_INFORMATION_
+{
+	/*! GPU default core clock speed in Hz */
+	IMG_UINT32			ui32CoreClockSpeed;
+
+	/*! Default SOC clock speed in Hz */
+	IMG_UINT32			ui32SOCClockSpeed;
+
+	/*! Active Power Management: GPU actively requests the host driver to be powered off */
+	IMG_BOOL			bEnableActivePM;
+
+	/*! Enable the GPU to power off internal Power Islands independently from the host driver */
+	IMG_BOOL			bEnableRDPowIsland;
+
+	/*! Active Power Management: Delay between the GPU idle and the request to the host */
+	IMG_UINT32			ui32ActivePMLatencyms;
+
+} RGX_TIMING_INFORMATION;
+
+typedef struct _RGX_DATA_
+{
+	/*! Timing information */
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo;
+} RGX_DATA;
+
+typedef enum _RGX_QUERY_TIMESTAMP_TYPE_
+{
+	RGX_QUERY_HOST_TIMESTAMP,
+	RGX_QUERY_DEVICE_TIMESTAMP,
+} RGX_QUERY_TIMESTAMP_TYPE;
+
+
+/*
+	RGX PDUMP register bank name (prefix)
+*/
+#define RGX_PDUMPREG_NAME		"RGXREG"
+
+#endif /* RGXDEVICE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxfwutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxfwutils.c
new file mode 100644
index 000000000000..2f65106d359e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxfwutils.c
@@ -0,0 +1,6563 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rogue firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Rogue firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+#include "cache_km.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxfwriscv.h"
+#include "rgx_options.h"
+#include "rgx_fwif_alignchecks.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_pdump_panics.h"
+#include "fwtrace_string.h"
+#include "rgxheapconfig.h"
+#include "pvrsrv.h"
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "rgxccb.h"
+#include "rgxcompute.h"
+#include "rgxtransfer.h"
+#include "rgxtdmtransfer.h"
+#include "rgxpower.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+#include "rgxmem.h"
+#include "rgxmmudefs_km.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#endif
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxutils.h"
+#include "rgxtimecorr.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "tlstream.h"
+#include "devicemem_server_utils.h"
+#include "htbserver.h"
+#include "rgx_bvnc_defs_km.h"
+#include "info_page.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#ifdef __linux__
+#include <linux/kernel.h>	/* sprintf */
+#else
+#include <stdio.h>
+#include <string.h>
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF)
+#include "pvr_dvfs_common.h"
+#endif
+
+
+#include "vz_vmm_pvz.h"
+#include "rgx_heaps.h"
+
+/*!
+ ******************************************************************************
+ * HWPERF
+ *****************************************************************************/
+/* Size of the Firmware L1 HWPERF buffer in bytes (2MB). Accessed by the
+ * Firmware and host driver. */
+#define RGXFW_HWPERF_L1_SIZE_MIN        (16U)
+#define RGXFW_HWPERF_L1_SIZE_DEFAULT    PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB
+#define RGXFW_HWPERF_L1_SIZE_MAX        (12288U)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#if defined(DEBUG)
+/* Catch the use of auto-increment when meta_registers_unpacked_accesses feature is
+ * present in case we ever use it. No WA exists so it must not be used */
+#define CHECK_HWBRN_68777(v) \
+	do { \
+		PVR_ASSERT(((v) & RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN) == 0); \
+	} while (0)
+#else
+#define CHECK_HWBRN_68777(v)
+#endif
+#endif
+
+/* Firmware CCB length */
+#if defined(NO_HARDWARE) && defined(PDUMP)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (10)
+#elif defined(SUPPORT_PDVFS) || defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (8)
+#else
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (5)
+#endif
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS] = {IRQ_COUNTER_STORAGE_REGS};
+#endif
+
+/* Workload Estimation Firmware CCB length */
+#define RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2   (7)
+
+/* Size of memory buffer for firmware gcov data
+ * The actual data size is several hundred kilobytes. The buffer is an order of magnitude larger. */
+#define RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE (4*1024*1024)
+
+typedef struct
+{
+	RGXFWIF_KCCB_CMD        sKCCBcmd;
+	DLLIST_NODE             sListNode;
+	PDUMP_FLAGS_T           uiPDumpFlags;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+} RGX_DEFERRED_KCCB_CMD;
+
+typedef struct
+{
+	IMG_INT32  i32Priority;
+	IMG_UINT32 ui32IsolationGroups;
+	IMG_UINT32 ui32TSPercentage;
+}RGX_QOS_DEFAULTS;
+
+#define RGX_QOS_DEFAULTS_INIT(osid) \
+	{RGX_DRIVERID_##osid##_DEFAULT_PRIORITY,\
+	RGX_DRIVERID_##osid##_DEFAULT_ISOLATION_GROUP,\
+	RGX_DRIVERID_##osid##_DEFAULT_TIME_SLICE}
+
+#if defined(PDUMP)
+/* ensure PIDs are 32-bit because a 32-bit PDump load is generated for the
+ * PID filter example entries
+ */
+static_assert(sizeof(IMG_PID) == sizeof(IMG_UINT32),
+		"FW PID filtering assumes the IMG_PID type is 32-bits wide as it "
+		"generates WRW commands for loading the PID values");
+#endif
+
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+static_assert(RGX_DRIVER_DEFAULT_TIME_SLICES_SUM <= PVRSRV_VZ_TIME_SLICE_MAX, "Invalid driverid time slice aggregate");
+#endif
+
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo);
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+
+static void __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
+{
+	/* This should *NOT* happen. Try to trace what caused this and avoid a NPE
+	 * with the Write/Read at the foot of the function.
+	 */
+	PVR_ASSERT((psDevInfo != NULL));
+	if (psDevInfo == NULL)
+	{
+		return;
+	}
+
+	/* Kick MTS to wake firmware. */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
+
+	/* Uncached device/IO mapping will ensure MTS kick leaves CPU, read back
+	 * will ensure it reaches the regbank via inter-connects (AXI, PCIe etc)
+	 * before continuing.
+	 */
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          ppsMemDesc      pointer to the allocation's memory descriptor
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO*  psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL bClearByMemset;
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiAllocFlags))
+	{
+		/* Under AutoVz the ZERO_ON_ALLOC flag is avoided as it causes the memory to
+		 * be allocated from a different PMR than an allocation without the flag.
+		 * When the content of an allocation needs to be recovered from physical memory
+		 * on a later driver reboot, the memory then cannot be zeroed but the allocation
+		 * addresses must still match.
+		 * If the memory requires clearing, perform a memset after the allocation. */
+		uiAllocFlags &= ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+		bClearByMemset = IMG_TRUE;
+	}
+	else
+	{
+		bClearByMemset = IMG_FALSE;
+	}
+#endif
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate %s", pszName);
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui32Size,
+							  uiAllocFlags,
+							  pszName,
+							  ppsMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate %u bytes for %s (%u)",
+				 __func__,
+				 ui32Size,
+				 pszName,
+				 eError));
+		goto fail_alloc;
+	}
+
+	if (psFwPtr)
+	{
+		eError = RGXSetFirmwareAddress(psFwPtr, *ppsMemDesc, 0, ui32DevVAFlags);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire firmware virtual address for %s (%u)",
+					 __func__,
+					 pszName,
+					 eError));
+			goto fail_fwaddr;
+		}
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	if ((bClearByMemset) || (ppvCpuPtr))
+#else
+	if (ppvCpuPtr)
+#endif
+	{
+		void *pvTempCpuPtr;
+
+		eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, &pvTempCpuPtr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to acquire CPU virtual address for %s (%u)",
+					__func__,
+					 pszName,
+					eError));
+			goto fail_cpuva;
+		}
+
+#if defined(SUPPORT_AUTOVZ)
+		if (bClearByMemset)
+		{
+			if (PVRSRV_CHECK_CPU_WRITE_COMBINE(uiAllocFlags))
+			{
+				OSCachedMemSetWMB(pvTempCpuPtr, 0, ui32Size);
+			}
+			else
+			{
+				OSDeviceMemSet(pvTempCpuPtr, 0, ui32Size);
+			}
+		}
+		if (ppvCpuPtr)
+#endif
+		{
+			*ppvCpuPtr = pvTempCpuPtr;
+		}
+#if defined(SUPPORT_AUTOVZ)
+		else
+		{
+			DevmemReleaseCpuVirtAddr(*ppsMemDesc);
+			pvTempCpuPtr = NULL;
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s set up at Fw VA 0x%x and CPU VA 0x%p with alloc flags 0x%" IMG_UINT64_FMTSPECX,
+			 __func__, pszName,
+			 (psFwPtr)   ? (psFwPtr->ui32Addr) : (0),
+			 (ppvCpuPtr) ? (*ppvCpuPtr)        : (NULL),
+			 uiAllocFlags));
+
+	return eError;
+
+fail_cpuva:
+	if (psFwPtr)
+	{
+		RGXUnsetFirmwareAddress(*ppsMemDesc);
+	}
+fail_fwaddr:
+	DevmemFree(*ppsMemDesc);
+fail_alloc:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHwPerfBufferSize
+
+@Description    Computes the effective size of the HW Perf Buffer
+@Input          ui32HWPerfFWBufSizeKB       Device Info struct
+@Return         HwPerfBufferSize
+*/ /**************************************************************************/
+static IMG_UINT32 GetHwPerfBufferSize(IMG_UINT32 ui32HWPerfFWBufSizeKB)
+{
+	IMG_UINT32 HwPerfBufferSize;
+
+	/* HWPerf: Determine the size of the FW buffer */
+	if (ui32HWPerfFWBufSizeKB == 0 ||
+			ui32HWPerfFWBufSizeKB == RGXFW_HWPERF_L1_SIZE_DEFAULT)
+	{
+		/* Under pvrsrvctl 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. Set it to the default
+		 * size, no logging.
+		 */
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_DEFAULT<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX))
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MAX<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN))
+	{
+		/* Size specified as in AppHint HWPerfFWBufSizeInKB */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Using HWPerf FW buffer size of %u KB",
+				__func__,
+				ui32HWPerfFWBufSizeKB));
+		HwPerfBufferSize = ui32HWPerfFWBufSizeKB<<10;
+	}
+	else
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MIN<<10;
+	}
+
+	return HwPerfBufferSize;
+}
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupSignatureChecks
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
+                                              DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
+                                              IMG_UINT32          ui32SigChecksBufSize,
+                                              RGXFWIF_SIGBUF_CTL* psSigBufCtl)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+								  ui32SigChecksBufSize,
+								  "FwSignatureChecks",
+								  ppsSigChecksMemDesc,
+								  &psSigBufCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	DevmemPDumpLoadMem(	*ppsSigChecksMemDesc,
+			0,
+			ui32SigChecksBufSize,
+			PDUMP_FLAGS_CONTINUOUS);
+
+	psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
+fail:
+	return eError;
+}
+#endif
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupFirmwareGcovBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupFirmwareGcovBuffer(PVRSRV_RGXDEV_INFO*			psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32FirmwareGcovBufferSize,
+		RGXFWIF_FIRMWARE_GCOV_CTL*	psFirmwareGcovCtl,
+		const IMG_CHAR*				pszBufferName)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for gcov */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32FirmwareGcovBufferSize,
+								  pszBufferName,
+								  ppsBufferMemDesc,
+								  &psFirmwareGcovCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psFirmwareGcovCtl->ui32Size = ui32FirmwareGcovBufferSize;
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+/*!
+ ******************************************************************************
+ @Function		RGXFWSetupCounterBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+ *****************************************************************************/
+static PVRSRV_ERROR RGXFWSetupCounterBuffer(PVRSRV_RGXDEV_INFO* psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32CounterDataBufferSize,
+		RGXFWIF_COUNTER_DUMP_CTL*	psCounterDumpCtl)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32CounterDataBufferSize,
+								  "FwCounterBuffer",
+								  ppsBufferMemDesc,
+								  &psCounterDumpCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psCounterDumpCtl->ui32SizeInDwords = ui32CounterDataBufferSize >> 2;
+	RGXFwSharedMemCacheOpValue(psCounterDumpCtl->ui32SizeInDwords, FLUSH);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      RGXFWSetupAlignChecks
+ @Description   This functions allocates and fills memory needed for the
+                aligns checks of the UM and KM structures shared with the
+                firmware. The format of the data in the memory is as follows:
+                    <number of elements in the KM array>
+                    <array of KM structures' sizes and members' offsets>
+                    <number of elements in the UM array>
+                    <array of UM structures' sizes and members' offsets>
+                The UM array is passed from the user side. Now the firmware is
+                responsible for filling this part of the memory. If that
+                happens the check of the UM structures will be performed
+                by the host driver on client's connect.
+                If the macro is not defined the client driver fills the memory
+                and the firmware checks for the alignment of all structures.
+ @Input			psDeviceNode
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_DEVICE_NODE *psDeviceNode,
+								RGXFWIF_DEV_VIRTADDR	*psAlignChecksDevFW)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32			aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
+	IMG_UINT32			ui32RGXFWAlignChecksTotal;
+	IMG_UINT32*			paui32AlignChecks;
+	PVRSRV_ERROR		eError;
+
+	/* In this case we don't know the number of elements in UM array.
+	 * We have to assume something so we assume RGXFW_ALIGN_CHECKS_UM_MAX.
+	 */
+	ui32RGXFWAlignChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
+	                            + RGXFW_ALIGN_CHECKS_UM_MAX * sizeof(IMG_UINT32)
+	                            + 2 * sizeof(IMG_UINT32);
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32RGXFWAlignChecksTotal,
+								  "FwAlignmentChecks",
+								  &psDevInfo->psRGXFWAlignChecksMemDesc,
+								  psAlignChecksDevFW,
+								  (void**) &paui32AlignChecks,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Copy the values */
+		*paui32AlignChecks++ = ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+		OSCachedMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0],
+		                sizeof(aui32RGXFWAlignChecksKM));
+		paui32AlignChecks += ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+
+		*paui32AlignChecks = 0;
+
+		OSWriteMemoryBarrier(paui32AlignChecks);
+		RGXFwSharedMemCacheOpExec(paui32AlignChecks - (ARRAY_SIZE(aui32RGXFWAlignChecksKM) + 1),
+		                          ui32RGXFWAlignChecksTotal,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+
+
+	DevmemPDumpLoadMem(	psDevInfo->psRGXFWAlignChecksMemDesc,
+						0,
+						ui32RGXFWAlignChecksTotal,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	if (psDevInfo->psRGXFWAlignChecksMemDesc != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
+		psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
+	}
+}
+
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+						   DEVMEM_MEMDESC		*psSrc,
+						   IMG_UINT32			uiExtraOffset,
+						   IMG_UINT32			ui32Flags)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	psDevVirtAddr;
+	PVRSRV_DEVICE_NODE	*psDeviceNode;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+
+	psDeviceNode = (PVRSRV_DEVICE_NODE *) DevmemGetConnection(psSrc);
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		IMG_UINT32          ui32Offset;
+		IMG_BOOL            bCachedInMETA;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+		IMG_UINT32          uiGPUCacheMode;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in META memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the META cache flags */
+		bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		/* Honour the SLC cache flags */
+		eError = DevmemDeviceCacheMode(uiDevFlags, &uiGPUCacheMode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemDeviceCacheMode", failDevCacheMode);
+
+		/*
+		 * Choose Meta virtual address based on Meta and SLC cacheability.
+		 */
+		ui32Offset += RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+		if (bCachedInMETA)
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+		}
+
+		if (PVRSRV_CHECK_GPU_CACHED(uiGPUCacheMode))
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_GOTO_IF_ERROR(eError, failDevVAAcquire);
+
+		ppDest->ui32Addr = (IMG_UINT32)((psDevVirtAddr.uiAddr + uiExtraOffset) & 0xFFFFFFFF);
+	}
+#endif
+	else
+	{
+		IMG_UINT32      ui32Offset;
+		IMG_BOOL        bCachedInRISCV;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in RISCV memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the RISCV cache flags */
+		bCachedInRISCV = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		if (bCachedInRISCV)
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_CACHED_DATA_BASE;
+		}
+		else
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_UNCACHED_DATA_BASE;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+
+	if ((ppDest->ui32Addr & 0x3U) != 0)
+	{
+		IMG_CHAR *pszAnnotation;
+		/* It is expected that the annotation returned by DevmemGetAnnotation() is always valid */
+		DevmemGetAnnotation(psSrc, &pszAnnotation);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: %s @ 0x%x is not aligned to 32 bit",
+				 __func__, pszAnnotation, ppDest->ui32Addr));
+
+		return PVRSRV_ERROR_INVALID_ALIGNMENT;
+	}
+
+	if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
+	{
+		DevmemReleaseDevVirtAddr(psSrc);
+	}
+
+	return PVRSRV_OK;
+
+failDevCacheMode:
+	DevmemReleaseDevVirtAddr(psSrc);
+failDevVAAcquire:
+	return eError;
+}
+
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	sDevVirtAddr;
+
+	eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
+	psDest->psDevVirtAddr.uiAddr += uiOffset;
+	psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
+
+	DevmemReleaseDevVirtAddr(psSrcMemDesc);
+}
+
+
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
+{
+	DevmemReleaseDevVirtAddr(psSrc);
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCB
+ @Description	Free the kernel or firmware CCB
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppvCCBCtlLocal
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+******************************************************************************/
+static void RGXFreeCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtl,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+					   DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+					   IMG_UINT8			**ppui8CCB,
+					   DEVMEM_MEMDESC		**ppsCCBMemDesc)
+{
+	if (*ppsCCBMemDesc != NULL)
+	{
+		if (*ppui8CCB != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBMemDesc);
+			*ppui8CCB = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBMemDesc);
+		*ppsCCBMemDesc = NULL;
+	}
+	if (*ppsCCBCtlMemDesc != NULL)
+	{
+		if (*ppsCCBCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBCtlMemDesc);
+			*ppsCCBCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBCtlMemDesc);
+		*ppsCCBCtlMemDesc = NULL;
+	}
+	if (*ppsCCBCtlLocal != NULL)
+	{
+		OSFreeMem(*ppsCCBCtlLocal);
+		*ppsCCBCtlLocal = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCBReturnSlots
+ @Description	Free the kernel CCB's return slot array and associated mappings
+ @Input			psDevInfo              Device Info struct
+ @Input			ppui32CCBRtnSlots      CPU mapping of slot array
+ @Input			ppsCCBRtnSlotsMemDesc  Slot array's device memdesc
+******************************************************************************/
+static void RGXFreeCCBReturnSlots(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32         **ppui32CCBRtnSlots,
+								  DEVMEM_MEMDESC     **ppsCCBRtnSlotsMemDesc)
+{
+	/* Free the return slot array if allocated */
+	if (*ppsCCBRtnSlotsMemDesc != NULL)
+	{
+		/* Before freeing, ensure the CPU mapping as well is released */
+		if (*ppui32CCBRtnSlots != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBRtnSlotsMemDesc);
+			*ppui32CCBRtnSlots = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBRtnSlotsMemDesc);
+		*ppsCCBRtnSlotsMemDesc = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXSetupCCB
+ @Description	Allocate and initialise a circular command buffer
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppui8CCB
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+ @Input			ui32NumCmdsLog2
+ @Input			ui32CmdSize
+ @Input			uiCCBMemAllocFlags
+ @Input			pszName
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								RGXFWIF_CCB_CTL		**ppsCCBCtl,
+								RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+								DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+								IMG_UINT8			**ppui8CCB,
+								DEVMEM_MEMDESC		**ppsCCBMemDesc,
+								PRGXFWIF_CCB_CTL	*psCCBCtlFWAddr,
+								PRGXFWIF_CCB		*psCCBFWAddr,
+								IMG_UINT32			ui32NumCmdsLog2,
+								IMG_UINT32			ui32CmdSize,
+								PVRSRV_MEMALLOCFLAGS_T uiCCBMemAllocFlags,
+								const IMG_CHAR		*pszName)
+{
+	PVRSRV_ERROR		eError;
+	RGXFWIF_CCB_CTL		*psCCBCtl;
+	IMG_UINT32		ui32CCBSize = (1U << ui32NumCmdsLog2);
+	IMG_CHAR		szCCBCtlName[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_INT32		iStrLen;
+
+	/* Append "Control" to the name for the control struct. */
+	iStrLen = OSSNPrintf(szCCBCtlName, sizeof(szCCBCtlName), "%sControl", pszName);
+	PVR_ASSERT(iStrLen < sizeof(szCCBCtlName));
+
+	if (unlikely(iStrLen < 0))
+	{
+		OSStringSafeCopy(szCCBCtlName, "FwCCBControl", DEVMEM_ANNOTATION_MAX_LEN);
+	}
+
+	/* Allocate memory for the CCB control.*/
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_CCB_CTL),
+								  szCCBCtlName,
+								  ppsCCBCtlMemDesc,
+								  psCCBCtlFWAddr,
+								  (void**) ppsCCBCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Allocate memory for the CCB.
+	 * (this will reference further command data in non-shared CCBs)
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  uiCCBMemAllocFlags,
+								  ui32CCBSize * ui32CmdSize,
+								  pszName,
+								  ppsCCBMemDesc,
+								  psCCBFWAddr,
+								  (void**) ppui8CCB,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Initialise the CCB control.
+	 */
+	psCCBCtl = OSAllocZMem(sizeof(*psCCBCtl));
+	PVR_LOG_GOTO_IF_NOMEM(psCCBCtl, eError, fail);
+
+	psCCBCtl->ui32WrapMask = ui32CCBSize - 1;
+
+	OSDeviceMemCopy(*ppsCCBCtl, psCCBCtl, sizeof(*psCCBCtl));
+	RGXFwSharedMemCacheOpPtr(psCCBCtl, FLUSH);
+
+	*ppsCCBCtlLocal = psCCBCtl;
+
+	/* Pdump the CCB control */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Initialise %s", szCCBCtlName);
+	DevmemPDumpLoadMem(*ppsCCBCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCB_CTL),
+					   0);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeCCB(psDevInfo,
+			   ppsCCBCtl,
+			   ppsCCBCtlLocal,
+			   ppsCCBCtlMemDesc,
+			   ppui8CCB,
+			   ppsCCBMemDesc);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+#endif
+	{
+		if (psDevInfo->psRGXFaultAddressMemDesc)
+		{
+			if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR) == PVRSRV_OK)
+			{
+				PMRUnlockSysPhysAddresses(psPMR);
+			}
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+			psDevInfo->psRGXFaultAddressMemDesc = NULL;
+		}
+	}
+}
+
+static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE *psDeviceNode, RGXFWIF_SYSINIT *psFwSysInit)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			*pui32MemoryVirtAddr;
+	IMG_UINT32			i;
+	size_t				ui32PageSize = OSGetPageSize();
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PMR					*psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	/* Allocate page of memory to use for page faults on non-blocking memory transactions.
+	 * Doesn't need to be cleared as it is initialised with the 0xDEADBEEF pattern below. */
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+			ui32PageSize,
+			ui32PageSize,
+			RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS & ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+			"FwExFaultAddress",
+			&psDevInfo->psRGXFaultAddressMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate mem for fault address (%u)",
+		         __func__, eError));
+		goto failFaultAddressDescAlloc;
+	}
+
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
+										  (void **)&pui32MemoryVirtAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire mem for fault address (%u)",
+					 __func__, eError));
+			goto failFaultAddressDescAqCpuVirt;
+		}
+
+		/* fill the page with a known pattern when booting the firmware */
+		for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
+		}
+
+		OSWriteMemoryBarrier(pui32MemoryVirtAddr);
+		RGXFwSharedMemCacheOpExec(pui32MemoryVirtAddr, ui32PageSize, PVRSRV_CACHE_OP_FLUSH);
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
+	}
+
+	eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Error getting PMR for fault address (%u)",
+		         __func__, eError));
+
+		goto failFaultAddressDescGetPMR;
+	}
+	else
+	{
+		IMG_BOOL bValid;
+		IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error locking physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescLockPhys;
+		}
+
+		eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize, 1, 0, &(psFwSysInit->sFaultPhysAddr), &bValid, DEVICE_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error getting physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescGetPhys;
+		}
+
+		if (!bValid)
+		{
+			psFwSysInit->sFaultPhysAddr.uiAddr = 0;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed getting physical address for fault address MemDesc - invalid page (0x%" IMG_UINT64_FMTSPECX ")",
+			         __func__, psFwSysInit->sFaultPhysAddr.uiAddr));
+
+			goto failFaultAddressDescGetPhys;
+		}
+	}
+
+	return PVRSRV_OK;
+
+failFaultAddressDescGetPhys:
+	PMRUnlockSysPhysAddresses(psPMR);
+
+failFaultAddressDescLockPhys:
+failFaultAddressDescGetPMR:
+failFaultAddressDescAqCpuVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+
+failFaultAddressDescAlloc:
+
+	return eError;
+}
+
+#if defined(PDUMP)
+/* Replace the DevPhy address with the one Pdump allocates at pdump_player run time */
+static PVRSRV_ERROR RGXPDumpFaultReadRegister(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	PMR *psFWInitPMR, *psFaultAddrPMR;
+	IMG_UINT32 ui32Dstoffset;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	psFWInitPMR = (PMR *)(psDevInfo->psRGXFWIfSysInitMemDesc->psImport->hPMR);
+	ui32Dstoffset = psDevInfo->psRGXFWIfSysInitMemDesc->uiOffset + offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr.uiAddr);
+
+	psFaultAddrPMR = (PMR *)(psDevInfo->psRGXFaultAddressMemDesc->psImport->hPMR);
+
+	eError = PDumpMemLabelToMem64(psFaultAddrPMR,
+				psFWInitPMR,
+				0,
+				ui32Dstoffset,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dump of Fault Page Phys address failed(%u)", __func__, eError));
+	}
+	return eError;
+}
+#endif
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW tbi buffer
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTBIBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTBIBufferMemDesc);
+	psDevInfo->psRGXFWIfTBIBufferMemDesc = NULL;
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = 0;
+	psDevInfo->ui32RGXL2HWPerfBufSize = 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferInitOnDemandResources
+
+@Description    Allocates the firmware TBI buffer required for reading SFs
+		strings and initialize it with SFs.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         i, ui32Len;
+	const IMG_UINT32   ui32FWTBIBufsize = g_ui32SFsCount * sizeof(RGXFW_STID_FMT);
+	RGXFW_STID_FMT     *psFW_SFs = NULL;
+
+	/* Firmware address should not be already set */
+	if (psDevInfo->sRGXFWIfTBIBuffer.ui32Addr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW address for FWTBI is already set. Resetting it with newly allocated one",
+		         __func__));
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS,
+								  ui32FWTBIBufsize,
+								  "FwTBIBuffer",
+								  &psDevInfo->psRGXFWIfTBIBufferMemDesc,
+								  &psDevInfo->sRGXFWIfTBIBuffer,
+								  (void**)&psFW_SFs,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* Copy SFs entries to FW buffer */
+	for (i = 0; i < g_ui32SFsCount; i++)
+	{
+		OSCachedMemCopy(&psFW_SFs[i].ui32Id, &SFs[i].ui32Id, sizeof(SFs[i].ui32Id));
+		ui32Len = OSStringLength(SFs[i].psName);
+		OSCachedMemCopy(psFW_SFs[i].sName, SFs[i].psName, MIN(ui32Len, IMG_SF_STRING_MAX_SIZE - 1));
+	}
+
+	/* flush write buffers for psFW_SFs */
+	OSWriteMemoryBarrier(psFW_SFs);
+
+	/* Set size of TBI buffer */
+	psDevInfo->ui32FWIfTBIBufferSize = ui32FWTBIBufsize;
+
+	/* release CPU mapping */
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTBIBufferMemDesc);
+
+	return PVRSRV_OK;
+fail:
+	RGXTBIBufferDeinit(psDevInfo);
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW trace buffer(s)
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTraceBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufferMemDesc[i])
+		{
+			if (psDevInfo->apui32TraceBuffer[i] != NULL)
+			{
+				DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+				psDevInfo->apui32TraceBuffer[i] = NULL;
+			}
+
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+			psDevInfo->psRGXFWIfTraceBufferMemDesc[i] = NULL;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferInitOnDemandResources
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+		info from the firmware.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo,
+												 PVRSRV_MEMALLOCFLAGS_T uiAllocFlags)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         ui32FwThreadNum;
+	IMG_UINT32         ui32DefaultTraceBufSize;
+	IMG_DEVMEM_SIZE_T  uiTraceBufSizeInBytes;
+	void               *pvAppHintState = NULL;
+	IMG_CHAR           pszBufferName[] = "FwTraceBuffer_Thread0";
+
+	/* Check AppHint value for module-param FWTraceBufSizeInDWords */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32DefaultTraceBufSize = RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,
+						 pvAppHintState,
+						 FWTraceBufSizeInDWords,
+						 &ui32DefaultTraceBufSize,
+						 &psDevInfo->ui32TraceBufSizeInDWords);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/* Write tracebuf size once to devmem */
+	psTraceBufCtl->ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (psDevInfo->ui32TraceBufSizeInDWords < RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS ||
+		psDevInfo->ui32TraceBufSizeInDWords > RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Requested trace buffer size (%u) out of its minimum (%u) & maximum (%u) range. Exiting error.",
+				  __func__,
+				  psDevInfo->ui32TraceBufSizeInDWords,
+				  RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS,
+				  RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS));
+		eError = PVRSRV_ERROR_OUT_OF_RANGE;
+		goto exit_error;
+	}
+
+	uiTraceBufSizeInBytes = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+
+	for (ui32FwThreadNum = 0; ui32FwThreadNum < RGXFW_THREAD_NUM; ui32FwThreadNum++)
+	{
+#if !defined(SUPPORT_AUTOVZ)
+		/* Ensure allocation API is only called when not already allocated */
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum] == NULL);
+		/* Firmware address should not be already set */
+		PVR_ASSERT(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer.ui32Addr == 0x0);
+#endif
+
+		/* update the firmware thread number in the Trace Buffer's name */
+		pszBufferName[sizeof(pszBufferName) - 2] += ui32FwThreadNum;
+
+		eError = RGXSetupFwAllocation(psDevInfo,
+									  uiAllocFlags,
+									  uiTraceBufSizeInBytes,
+									  pszBufferName,
+									  &psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
+									  &psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer,
+									  (void**)&psDevInfo->apui32TraceBuffer[ui32FwThreadNum],
+									  RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+	}
+
+	return PVRSRV_OK;
+
+fail:
+	RGXTraceBufferDeinit(psDevInfo);
+exit_error:
+	return eError;
+}
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       RGXPDumpLoadFWInitData
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+                info from the firmware.
+
+@Input          psDevInfo RGX device info
+ */ /*************************************************************************/
+static void RGXPDumpLoadFWInitData(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   RGX_INIT_APPHINTS  *psApphints,
+								   IMG_UINT32         ui32HWPerfCountersDataSize)
+{
+	IMG_UINT32 ui32ConfigFlags  = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+	IMG_UINT32 ui32FwOsCfgFlags = psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Dump RGXFW Init data");
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "(to enable rgxfw signatures place the following line after the RTCONF line)");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, asSigBufCtl),
+						   sizeof(RGXFWIF_SIGBUF_CTL)*(RGXFWIF_DM_MAX),
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump initial state of FW runtime configuration");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_RUNTIME_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw hwperfctl structure");
+	DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+							0,
+							ui32HWPerfCountersDataSize,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw trace control structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TRACEBUF),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware system data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware OS data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgx TBI buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTBIBufferMemDesc,
+					   0,
+					   psDevInfo->ui32FWIfTBIBufferSize,
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw register configuration buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRegCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_REG_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_USER_REGISTER_CONFIGURATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw system init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw os init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfOsInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* RGXFW Init structure needs to be loaded before we overwrite FaultPhysAddr, else this address patching won't have any effect */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Overwrite FaultPhysAddr of FwSysInit in pdump with actual physical address");
+	RGXPDumpFaultReadRegister(psDevInfo);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "RTCONF: run-time configuration");
+
+	/* Dump the config options so they can be edited. */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW system config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Rand mode:                      0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Soft Reset Enable:              0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable HWPerf:                             0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Rascal+Dust Power Island:                  0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( FBCDC Version 3.1 Enable:                  0x%08x)", RGXFWIF_INICFG_FBCDC_V3_1_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Check MList:                               0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable Auto Clock Gating:                 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+				 "( Try overlapping DM pipelines:              0x%08x)", RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable DM pipeline roadblocks:             0x%08x)", RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#endif
+				 "( Enable register configuration:             0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on TA Out-of-Memory:                0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable HWPerf custom counter filter:      0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable Ctx Switch profile mode: 0x%08x (none=b'000, fast=b'001, medium=b'010, slow=b'011, nodelay=b'100))", RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on HWR trigger (page fault, lockup, overrun or poll failure): 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable IRQ validation:                     0x%08x)", RGXFWIF_INICFG_VALIDATE_IRQ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPU power state mask change Enable:        0x%08x)", RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN);
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Workload Estimation:                0x%08x)", RGXFWIF_INICFG_WORKEST);
+#if defined(SUPPORT_PDVFS)
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Proactive DVFS:                     0x%08x)", RGXFWIF_INICFG_PDVFS);
+#endif /* defined(SUPPORT_PDVFS) */
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ISP Scheduling Mode (v1=b'01, v2=b'10):    0x%08x)", RGXFWIF_INICFG_ISPSCHEDMODE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Validate SOC & USC timers:                 0x%08x)", RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+							offsetof(RGXFWIF_SYSDATA, ui32ConfigFlags),
+							ui32ConfigFlags,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Extended FW system config options not used.)");
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW OS config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch TDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch GEOM Enable:                    0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch 3D Enable:                      0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch CDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch RDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN);
+#endif
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  2D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  TA Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  3D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch CDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch RDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM);
+#endif
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+							  offsetof(RGXFWIF_OSDATA, ui32FwOsConfigFlags),
+							  ui32FwOsCfgFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	{
+		PDUMP_FLAGS_T      ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+		IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+		IMG_BOOL           bRunTimeUpdate = IMG_FALSE;
+		IMG_UINT32         ui32DstOffset = psDevInfo->psRGXFWIfRuntimeCfgMemDesc->uiOffset + offsetof(RGXFWIF_RUNTIME_CFG, ui32PowUnitsState);
+		IMG_CHAR           aszPowUnitsMaskRegVar[] = ":SYSMEM:$1";
+		IMG_CHAR           aszPowUnitsEnable[] = "RUNTIME_POW_UNITS_MASK";
+		PMR                *psPMR = (PMR *)(psDevInfo->psRGXFWIfRuntimeCfgMemDesc->psImport->hPMR);
+
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPIF(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+		                      "Load initial value power units mask in FW runtime configuration");
+		DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  ui32DstOffset,
+								  psDevInfo->psRGXFWIfRuntimeCfg->ui32PowUnitsState,
+								  ui32PDumpFlags);
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPELSE(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+			PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags, "Read initial SPU mask value from HW registers");
+			PDumpRegRead32ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_SPU_ENABLE, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, aszPowUnitsMaskRegVar, ui32AllPowUnitsMask, ui32PDumpFlags);
+			PDumpInternalVarToMemLabel(psPMR, ui32DstOffset, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDUMPFI(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+	}
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Select one or more security tests here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW private data from non-FW contexts: 0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW code from non-FW contexts:         0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from non-secure memory:          0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from secure (non-FW) memory:     0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui32SecurityTestFlags),
+							  psDevInfo->psRGXFWIfSysInit->ui32SecurityTestFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_SECURITY_VALIDATION) */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
+				 RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+				 RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, sPIDFilter.eMode),
+							  psDevInfo->psRGXFWIfSysInit->sPIDFilter.eMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter PID/DriverID list (Up to %u entries. Terminate with a zero PID))",
+				 RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
+	{
+		IMG_UINT32 i;
+
+		/* generate a few WRWs in the pdump stream as an example */
+		for (i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
+		{
+			/*
+			 * Some compilers cannot cope with the uses of offsetof() below - the specific problem being the use of
+			 * a non-const variable in the expression, which it needs to be const. Typical compiler output is
+			 * "expression must have a constant value".
+			 */
+			const IMG_DEVMEM_OFFSET_T uiPIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].uiPID);
+
+			const IMG_DEVMEM_OFFSET_T uiDriverIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].ui32DriverID);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID and DriverID pair %u)", i);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiPIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(DriverID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiDriverIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	/*
+	 * Dump the log config so it can be edited.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the log config here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Log Type: TRACE mode using shared memory buffer: 0x00000001)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           TBI mode via external interface or sim support: 0x00000000)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           Note: TBI mode will hang on most hardware devices!)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MISC Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MISC);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VZ Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VZ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SAFETY Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SAFETY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VERBOSE Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VERBOSE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CUSTOMER Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CUSTOMER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+							  offsetof(RGXFWIF_TRACEBUF, ui32LogType),
+							  psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Set the HWPerf Filter config here, see \"hwperfbin2jsont -h\"");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui64HWPerfFilter),
+							  psDevInfo->psRGXFWIfSysInit->ui64HWPerfFilter,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Number of registers configurations for types(byte index): pow on(%d), dust change(%d), ta(%d), 3d(%d), cdm(%d), "
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 "tla(%d), "
+#endif
+				 "tdm(%d))",
+				 RGXFWIF_REG_CFG_TYPE_PWR_ON,
+				 RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,
+				 RGXFWIF_REG_CFG_TYPE_TA,
+				 RGXFWIF_REG_CFG_TYPE_3D,
+				 RGXFWIF_REG_CFG_TYPE_CDM,
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 RGXFWIF_REG_CFG_TYPE_TLA,
+#endif
+				 RGXFWIF_REG_CFG_TYPE_TDM);
+
+	{
+		IMG_UINT32 i;
+
+		/* Write 32 bits in each iteration as required by PDUMP WRW command */
+		for (i = 0; i < RGXFWIF_REG_CFG_TYPE_ALL; i += sizeof(IMG_UINT32))
+		{
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
+									offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[i]),
+									0,
+									PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "(Set registers here: address, mask, value)");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Mask),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
+}
+#endif /* defined(PDUMP) */
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwGuardPage
+
+ @Description Allocate a Guard Page at the start of a Guest's Main Heap
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwGuardPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)),
+								  OSGetPageSize(),
+								  "FwGuardPage",
+								  &psDevInfo->psRGXFWHeapGuardPageReserveMemDesc,
+								  NULL,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwSysData
+
+ @Description Sets up all system-wide firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwSysData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									  RGX_INIT_APPHINTS        *psApphints,
+									  IMG_UINT32               ui32ConfigFlags,
+									  IMG_UINT32               ui32ConfigFlagsExt,
+									  IMG_UINT32               ui32HWPerfCountersDataSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32         ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+	RGXFWIF_SYSINIT    *psFwSysInitScratch = NULL;
+#if defined(PDUMP)
+	IMG_UINT32         ui32SignatureChecksBufSize = psApphints->ui32SignatureChecksBufSize;
+#endif
+
+	psFwSysInitScratch = OSAllocZMem(sizeof(*psFwSysInitScratch));
+	PVR_LOG_GOTO_IF_NOMEM(psFwSysInitScratch, eError, fail);
+
+	/* Sys Fw init data */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              (RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+	                              PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSINIT),
+	                              "FwSysInitStructure",
+	                              &psDevInfo->psRGXFWIfSysInitMemDesc,
+	                              NULL,
+	                              (void**) &psDevInfo->psRGXFWIfSysInit,
+	                              RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Sys Init structure allocation", fail);
+
+	/* Setup Fault read register */
+	eError = RGXSetupFaultReadRegister(psDeviceNode, psFwSysInitScratch);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Fault read register setup", fail);
+
+#if defined(SUPPORT_AUTOVZ)
+	psFwSysInitScratch->ui32VzWdgPeriod = PVR_AUTOVZ_WDG_PERIOD_MS;
+#endif
+
+	/* RD Power Island */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
+		IMG_BOOL bEnableRDPowIsland = ((psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
+		                               (psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
+
+		ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
+	}
+
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		ui32ConfigFlags |= RGXFWIF_INICFG_WORKEST;
+#endif
+#if defined(SUPPORT_FW_OPP_TABLE)
+		{
+			RGXFWIF_OPP_INFO    *psOPPInfo;
+			IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg;
+
+			/* Pro-active DVFS depends on Workload Estimation */
+			psOPPInfo = &psFwSysInitScratch->sOPPInfo;
+			psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+#if defined(CONFIG_OF) && defined(CONFIG_PM_OPP) && !defined(NO_HARDWARE)
+			if (psDVFSDeviceCfg->bDTConfig)
+			{
+				/* OPP table configured from Device tree */
+				eError = DVFSCopyOPPTable(psDeviceNode,
+										  psOPPInfo,
+										  ARRAY_SIZE(psOPPInfo->asOPPValues));
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Unable to copy OPP table to FW init buffer (%u)", eError));
+					goto fail;
+				}
+			}
+#endif
+			if (!psDVFSDeviceCfg->bDTConfig)
+			{
+				PVR_LOG_IF_FALSE(psDVFSDeviceCfg->pasOPPTable, "RGXSetupFwSysData: Missing OPP Table");
+
+				if (psDVFSDeviceCfg->pasOPPTable != NULL)
+				{
+					if (psDVFSDeviceCfg->ui32OPPTableSize > ARRAY_SIZE(psOPPInfo->asOPPValues))
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								 "%s: OPP Table too large: Size = %u, Maximum size = %lu",
+								 __func__,
+								 psDVFSDeviceCfg->ui32OPPTableSize,
+								 (unsigned long)(ARRAY_SIZE(psOPPInfo->asOPPValues))));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto fail;
+					}
+
+					OSDeviceMemCopy(psOPPInfo->asOPPValues,
+									psDVFSDeviceCfg->pasOPPTable,
+									sizeof(psOPPInfo->asOPPValues));
+
+					psOPPInfo->ui32MaxOPPPoint = psDVFSDeviceCfg->ui32OPPTableSize - 1;
+
+				}
+			}
+
+#if defined(SUPPORT_PDVFS)
+			ui32ConfigFlags |= RGXFWIF_INICFG_PDVFS;
+#endif
+		}
+#endif /* defined(SUPPORT_FW_OPP_TABLE) */
+	}
+
+	/* FW trace control structure */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_TRACEBUF),
+								  "FwTraceCtlStruct",
+								  &psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								  &psFwSysInitScratch->sTraceBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfTraceBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Set initial firmware log type/group(s) */
+		if (psApphints->ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid initial log type (0x%X)",
+			         __func__, psApphints->ui32LogType));
+			goto fail;
+		}
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = psApphints->ui32LogType;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+	}
+
+	/* When PDUMP is enabled, ALWAYS allocate on-demand trace buffer resource
+	 * (irrespective of loggroup(s) enabled), given that logtype/loggroups can
+	 * be set during PDump playback in logconfig, at any point of time,
+	 * Otherwise, allocate only if required. */
+#if !defined(PDUMP)
+#if defined(SUPPORT_AUTOVZ)
+	/* always allocate trace buffer for AutoVz Host drivers to allow
+	 * deterministic addresses of all SysData structures */
+	if ((PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo)) || (RGXTraceBufferIsInitRequired(psDevInfo)))
+#else
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+#endif
+#endif
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo,
+													 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+													 RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp));
+	}
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXTraceBufferInitOnDemandResources", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSDATA),
+	                              "FwSysData",
+	                              &psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                              &psFwSysInitScratch->sFwSysData,
+	                              (void**) &psDevInfo->psRGXFWIfFwSysData,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* GPIO validation setup */
+	psFwSysInitScratch->eGPIOValidationMode = RGXFWIF_GPIO_VAL_OFF;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = RGXFWSetupCounterBuffer(psDevInfo,
+									 &psDevInfo->psCounterBufferMemDesc,
+									 OSGetPageSize(),
+									 &psFwSysInitScratch->sCounterDumpCtl);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Counter Buffer allocation", fail);
+
+	PVR_DPF((PVR_DBG_WARNING, "Counter buffer allocated at %p, size %zu Bytes.", psDevInfo->psCounterBufferMemDesc, OSGetPageSize()));
+#endif /* defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = RGXFWSetupFirmwareGcovBuffer(psDevInfo,
+	                                      &psDevInfo->psFirmwareGcovBufferMemDesc,
+	                                      RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE,
+	                                      &psFwSysInitScratch->sFirmwareGcovCtl,
+	                                      "FirmwareGcovBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware GCOV buffer allocation", fail);
+	psDevInfo->ui32FirmwareGcovSize = RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE;
+#endif /* defined(SUPPORT_FIRMWARE_GCOV) */
+
+#if defined(PDUMP)
+	/* Require a minimum amount of memory for the signature buffers */
+	if (ui32SignatureChecksBufSize < RGXFW_SIG_BUFFER_SIZE_MIN)
+	{
+		ui32SignatureChecksBufSize = RGXFW_SIG_BUFFER_SIZE_MIN;
+	}
+
+	/* Setup Signature and Checksum Buffers */
+	psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	psDevInfo->ui32SigTDMChecksSize = 0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* Buffer allocated only when feature present because all known TDM
+		 * signature registers are dependent on this feature being present */
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TDM Signature check setup", fail);
+		psDevInfo->ui32SigTDMChecksSize = ui32SignatureChecksBufSize;
+	}
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GEOM Signature check setup", fail);
+	psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "3D Signature check setup", fail);
+	psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
+
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RDM Signature check setup", fail);
+		psDevInfo->ui32SigRDMChecksSize = ui32SignatureChecksBufSize;
+	}
+#endif
+
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM].sBuffer.ui32Addr = 0x0;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY].sBuffer.ui32Addr = 0x0;
+#endif
+	}
+#endif /* defined(PDUMP) */
+
+	eError = RGXFWSetupAlignChecks(psDeviceNode,
+	                               &psFwSysInitScratch->sAlignChecks);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Alignment checks setup", fail);
+
+	psFwSysInitScratch->ui32FilterFlags = psApphints->ui32FilterFlags;
+
+	/* Fill the remaining bits of fw the init data */
+	psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
+	psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/* Fill the remaining bits of fw the init data */
+		psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_BRN_65273_HEAP_BASE;
+		psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_BRN_65273_HEAP_BASE;
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/* Core clock rate */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(IMG_UINT32),
+	                              "FwPDVFSCoreClkRate",
+	                              &psDevInfo->psRGXFWIFCoreClkRateMemDesc,
+	                              &psFwSysInitScratch->sCoreClockRate,
+	                              (void**) &psDevInfo->pui32RGXFWIFCoreClkRate,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDVFS core clock rate memory setup", fail);
+#endif
+	{
+	/* Timestamps */
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags =
+		PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+		PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
+		PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/*
+	  the timer query arrays
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate timer query arrays (FW)");
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwStartTimesArray",
+	                          & psDevInfo->psStartTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64StartTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwEndTimesArray",
+	                          & psDevInfo->psEndTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64EndTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags,
+	                          "FwCompletedOpsArray",
+	                          & psDevInfo->psCompletedMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to completed ops array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
+	                                  (void **)& psDevInfo->pui32CompletedById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map completed ops array",
+				__func__));
+		goto fail;
+	}
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	eError = OSLockCreate(&psDevInfo->hTimerQueryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate log for timer query",
+				__func__));
+		goto fail;
+	}
+#endif
+#if defined(SUPPORT_TBI_INTERFACE)
+#if !defined(PDUMP)
+	/* allocate only if required */
+	if (RGXTBIBufferIsInitRequired(psDevInfo))
+#endif /* !defined(PDUMP) */
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand TBI buffer resource
+		 * (irrespective of loggroup(s) enabled), given that logtype/loggroups
+		 * can be set during PDump playback in logconfig, at any point of time
+		 */
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXTBIBufferInitOnDemandResources", fail);
+	}
+
+	psFwSysInitScratch->sTBIBuf = psDevInfo->sRGXFWIfTBIBuffer;
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+	/* Allocate shared buffer for GPU utilisation.
+	 * Enable FIRMWARE_CACHED to reduce read latency in the FW.
+	 * The FW flushes the cache after any writes.
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_GPU_UTIL_FW),
+								  "FwGPUUtilisationBuffer",
+								  &psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc,
+								  &psFwSysInitScratch->sGpuUtilFWCtl,
+								  (void**) &psDevInfo->psRGXFWIfGpuUtilFW,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GPU Utilisation Buffer ctl allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_RUNTIME_CFG),
+								  "FwRuntimeCfg",
+								  &psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  &psFwSysInitScratch->sRuntimeCfg,
+								  (void**) &psDevInfo->psRGXFWIfRuntimeCfg,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware runtime configuration memory allocation", fail);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_REG_CFG),
+								  "FwRegisterConfigStructure",
+								  &psDevInfo->psRGXFWIfRegCfgMemDesc,
+								  &psFwSysInitScratch->sRegCfg,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware register user configuration structure allocation", fail);
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS | PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  RGXFW_SCRATCH_BUF_SIZE,
+								  "FwScratchBuf",
+								  &psDevInfo->psRGXFWScratchBufMemDesc,
+								  &psFwSysInitScratch->pbFwScratchBuf,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware scratch buffer allocation", fail);
+#endif
+
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = GetHwPerfBufferSize(psApphints->ui32HWPerfFWBufSize);
+
+	/* Host L2 HWPERF buffer size in bytes must be bigger than the L1 buffer
+	 * accessed by the FW. The MISR may try to write one packet the size of the L1
+	 * buffer in some scenarios. When logging is enabled in the MISR, it can be seen
+	 * if the L2 buffer hits a full condition. The closer in size the L2 and L1 buffers
+	 * are the more chance of this happening.
+	 * Size chosen to allow MISR to write an L1 sized packet and for the client
+	 * application/daemon to drain a L1 sized packet e.g. ~ 1.5*L1.
+	 */
+	psDevInfo->ui32RGXL2HWPerfBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize +
+	        (psDevInfo->ui32RGXFWIfHWPerfBufSize>>1);
+
+	/* Second stage initialisation or HWPerf, hHWPerfLock created in first
+	 * stage. See RGXRegisterDevice() call to RGXHWPerfInit(). */
+	if (psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF] == 0)
+	{
+		psFwSysInitScratch->ui64HWPerfFilter =
+		    RGXHWPerfFwSetEventFilter(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF,
+		                              (IMG_UINT64) psApphints->ui32HWPerfFilter0 |
+		                              ((IMG_UINT64) psApphints->ui32HWPerfFilter1 << 32));
+	}
+	else
+	{
+		/* The filter has already been modified. This can happen if
+		 * pvr/apphint/EnableFTraceGPU was enabled. */
+		psFwSysInitScratch->ui64HWPerfFilter = psDevInfo->ui64HWPerfFwFilter;
+	}
+
+#if !defined(PDUMP)
+	/* Allocate if HWPerf filter has already been set. This is possible either
+	 * by setting a proper AppHint or enabling GPU ftrace events. */
+	if (psFwSysInitScratch->ui64HWPerfFilter != 0)
+#endif
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand HWPerf resources
+		 * (irrespective of HWPerf enabled or not), given that HWPerf can be
+		 * enabled during PDump playback via RTCONF at any point of time. */
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL1Buffer", fail);
+
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL2Stream", fail);
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWCOMCTX_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32HWPerfCountersDataSize,
+								  "FwHWPerfControlStructure",
+								  &psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+								  &psFwSysInitScratch->sHWPerfCtl,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware HW Perf control struct allocation", fail);
+
+	psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_INICFG_DISABLE_PDP_EN)
+							  ? IMG_FALSE : IMG_TRUE;
+
+	psFwSysInitScratch->eFirmwarePerf = psApphints->eFirmwarePerf;
+
+#if defined(PDUMP)
+	/* default: no filter */
+	psFwSysInitScratch->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
+	psFwSysInitScratch->sPIDFilter.asItems[0].uiPID = 0;
+#endif
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	{
+		PVRSRV_MEMALLOCFLAGS_T uiFlags = RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS;
+		PVRSRV_SET_PHYS_HEAP_HINT(FW_PRIV_DATA, uiFlags);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate non-secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+											"FwExNonSecureBuffer",
+											&psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Non-secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbNonSecureBuffer,
+									   psDevInfo->psRGXFWIfNonSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											uiFlags,
+											"FwExSecureBuffer",
+											&psDevInfo->psRGXFWIfSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbSecureBuffer,
+									   psDevInfo->psRGXFWIfSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail);
+	}
+#endif /* SUPPORT_SECURITY_VALIDATION */
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) || RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		psFwSysInitScratch->ui32TFBCCompressionControl =
+			(ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >> RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+	}
+#endif /* RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK */
+
+	/* Initialize FW started flag */
+	psFwSysInitScratch->bFirmwareStarted = IMG_FALSE;
+	psFwSysInitScratch->ui32MarkerVal = 1;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = RGX_VZ_CONNECTION_COOLDOWN_PERIOD;
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		IMG_UINT32 ui32DriverID;
+
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+
+		/* Required info by FW to calculate the ActivePM idle timer latency */
+		psFwSysInitScratch->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psFwSysInitScratch->ui32InitialSOCClockSpeed = psRGXData->psRGXTimingInfo->ui32SOCClockSpeed;
+#endif
+		psFwSysInitScratch->ui32InitialActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
+
+		/* Initialise variable runtime configuration to the system defaults */
+		psRuntimeCfg->ui32CoreClockSpeed = psFwSysInitScratch->ui32InitialCoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psRuntimeCfg->ui32SOCClockSpeed = psFwSysInitScratch->ui32InitialSOCClockSpeed;
+#endif
+		psRuntimeCfg->ui32ActivePMLatencyms = psFwSysInitScratch->ui32InitialActivePMLatencyms;
+		psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
+		psRuntimeCfg->ui32HCSDeadlineMS = RGX_HCS_DEFAULT_DEADLINE_MS;
+
+		if ((RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US > 0U) && (RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US < 1000U))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_LOG_GOTO_IF_ERROR(eError,
+								  "RGXSetupFwSysData: RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US must be either 0 (disabled) or greater than 1000",
+								  fail);
+		}
+		else
+		{
+			psRuntimeCfg->ui32WdgPeriodUs = RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+		{
+			psRuntimeCfg->ai32DriverPriority[RGXFW_HOST_DRIVER_ID] = 0;
+			psRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID] = RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP;
+			psRuntimeCfg->aui32TSPercentage[RGXFW_HOST_DRIVER_ID] = (IMG_UINT8)RGX_DRIVERID_0_DEFAULT_TIME_SLICE;
+		}
+		else
+		{
+			const RGX_QOS_DEFAULTS asQosDefaults[RGXFW_MAX_NUM_OSIDS] = {
+					RGX_QOS_DEFAULTS_INIT(0),
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+					RGX_QOS_DEFAULTS_INIT(1),
+#if (RGXFW_MAX_NUM_OSIDS > 2)
+					RGX_QOS_DEFAULTS_INIT(2),
+					RGX_QOS_DEFAULTS_INIT(3),
+					RGX_QOS_DEFAULTS_INIT(4),
+					RGX_QOS_DEFAULTS_INIT(5),
+					RGX_QOS_DEFAULTS_INIT(6),
+					RGX_QOS_DEFAULTS_INIT(7),
+#if (RGXFW_MAX_NUM_OSIDS > 8)
+					RGX_QOS_DEFAULTS_INIT(8),
+					RGX_QOS_DEFAULTS_INIT(9),
+					RGX_QOS_DEFAULTS_INIT(10),
+					RGX_QOS_DEFAULTS_INIT(11),
+					RGX_QOS_DEFAULTS_INIT(12),
+					RGX_QOS_DEFAULTS_INIT(13),
+					RGX_QOS_DEFAULTS_INIT(14),
+					RGX_QOS_DEFAULTS_INIT(15),
+					RGX_QOS_DEFAULTS_INIT(16),
+					RGX_QOS_DEFAULTS_INIT(17),
+					RGX_QOS_DEFAULTS_INIT(18),
+					RGX_QOS_DEFAULTS_INIT(19),
+					RGX_QOS_DEFAULTS_INIT(20),
+					RGX_QOS_DEFAULTS_INIT(21),
+					RGX_QOS_DEFAULTS_INIT(22),
+					RGX_QOS_DEFAULTS_INIT(23),
+					RGX_QOS_DEFAULTS_INIT(24),
+					RGX_QOS_DEFAULTS_INIT(25),
+					RGX_QOS_DEFAULTS_INIT(26),
+					RGX_QOS_DEFAULTS_INIT(27),
+					RGX_QOS_DEFAULTS_INIT(28),
+					RGX_QOS_DEFAULTS_INIT(29),
+					RGX_QOS_DEFAULTS_INIT(30),
+					RGX_QOS_DEFAULTS_INIT(31),
+#if (RGXFW_MAX_NUM_OSIDS > 32)
+#error "Support for more than 32 OSIDs not implemented."
+#endif
+#endif /* RGXFW_MAX_NUM_OSIDS > 8 */
+#endif /* RGXFW_MAX_NUM_OSIDS > 2 */
+
+#endif /* RGXFW_MAX_NUM_OSIDS > 1 */
+			};
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				/* Set up initial priorities between different OSes */
+				psRuntimeCfg->ai32DriverPriority[ui32DriverID] = asQosDefaults[ui32DriverID].i32Priority;
+				psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = asQosDefaults[ui32DriverID].ui32IsolationGroups;
+				psRuntimeCfg->aui32TSPercentage[ui32DriverID] = (asQosDefaults[ui32DriverID].ui32TSPercentage <=
+																				PVRSRV_VZ_TIME_SLICE_MAX) ?
+																				asQosDefaults[ui32DriverID].ui32TSPercentage:(0);
+			}
+		}
+		psRuntimeCfg->ui32TSIntervalMs = RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL;
+
+#if defined(PVR_ENABLE_PHR) && defined(PDUMP)
+		psRuntimeCfg->ui32PHRMode = RGXFWIF_PHR_MODE_RD_RESET;
+#else
+		psRuntimeCfg->ui32PHRMode = 0;
+#endif
+
+		/* Initialize the PowUnitsState Field to Max Dusts */
+		psRuntimeCfg->ui32PowUnitsState = psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount;
+
+		/* flush write buffers for psDevInfo->psRGXFWIfRuntimeCfg */
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfRuntimeCfg);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, FLUSH);
+
+		/* Setup FW coremem data */
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psFwSysInitScratch->sCorememDataStore.pbyFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+		}
+
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags    = ui32ConfigFlags    & RGXFWIF_INICFG_ALL;
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt = ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_ALL;
+
+		/* Initialise GPU utilisation buffer */
+		{
+			IMG_UINT64 ui64LastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(), RGXFWIF_GPU_UTIL_STATE_IDLE);
+			RGXFWIF_DM eDM;
+			IMG_UINT64 ui64LastWordTimeShifted =
+			        RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64() >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT, RGXFWIF_GPU_UTIL_STATE_IDLE);
+			IMG_UINT32 ui32DriverID;
+
+			psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord = ui64LastWord;
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				RGXFWIF_GPU_STATS *psStats = &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID];
+
+				for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+				{
+					psStats->aui32DMOSLastWord[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted & IMG_UINT32_MAX);
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted >> 32);
+				}
+			}
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+		}
+
+		/* init HWPERF data */
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfUt = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfDropCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32FirstDropOrdinal = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32LastDropOrdinal = 0;
+
+		psDevInfo->psRGXFWIfFwSysData->ui32MemFaultCheck = 0;
+
+		// flush write buffers for psRGXFWIfFwSysData
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwSysData);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, FLUSH);
+
+		/*Send through the BVNC Feature Flags*/
+		eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, &psFwSysInitScratch->sBvncKmFeatureFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags", fail);
+
+		/* populate the real FwOsInit structure with the values stored in the scratch copy */
+		OSCachedMemCopyWMB(psDevInfo->psRGXFWIfSysInit, psFwSysInitScratch, sizeof(RGXFWIF_SYSINIT));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit,
+		                         FLUSH);
+	}
+
+	OSFreeMem(psFwSysInitScratch);
+
+
+	return PVRSRV_OK;
+
+fail:
+	if (psFwSysInitScratch)
+	{
+		OSFreeMem(psFwSysInitScratch);
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwOsData
+
+ @Description Sets up all os-specific firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwOsData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									 IMG_UINT32               ui32KCCBSizeLog2,
+									 IMG_UINT32               ui32HWRDebugDumpLimit,
+									 IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_OSINIT     sFwOsInitScratch;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSCachedMemSet(&sFwOsInitScratch, 0, sizeof(RGXFWIF_OSINIT));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXSetupFwGuardPage(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware heap guard pages", fail);
+	}
+
+	/* Memory tracking the connection state should be non-volatile and
+	 * is not cleared on allocation to prevent loss of pre-reset information */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS &
+								  ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+								  sizeof(RGXFWIF_CONNECTION_CTL),
+								  "FwConnectionCtl",
+								  &psDevInfo->psRGXFWIfConnectionCtlMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfConnectionCtl,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Connection Control structure allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED),
+								  sizeof(RGXFWIF_OSINIT),
+								  "FwOsInitStructure",
+								  &psDevInfo->psRGXFWIfOsInitMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfOsInit,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Os Init structure allocation", fail);
+
+	/* init HWR frame info */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_HWRINFOBUF),
+								  "FwHWRInfoBuffer",
+								  &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
+								  &sFwOsInitScratch.sRGXFWIfHWRInfoBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfHWRInfoBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HWR Info Buffer allocation", fail);
+
+	/* Might be uncached. Be conservative and use a DeviceMemSet */
+	OSDeviceMemSet(psDevInfo->psRGXFWIfHWRInfoBufCtl, 0, sizeof(RGXFWIF_HWRINFOBUF));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, FLUSH);
+
+	/* Allocate a sync for power management */
+	eError = SyncPrimContextCreate(psDevInfo->psDeviceNode,
+	                               &psDevInfo->hSyncPrimContext);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive context allocation", fail);
+
+	eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive allocation", fail);
+
+	/* Set up kernel CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psKernelCCBCtl,
+						 &psDevInfo->psKernelCCBCtlLocal,
+						 &psDevInfo->psKernelCCBCtlMemDesc,
+						 &psDevInfo->psKernelCCB,
+						 &psDevInfo->psKernelCCBMemDesc,
+						 &sFwOsInitScratch.psKernelCCBCtl,
+						 &sFwOsInitScratch.psKernelCCB,
+						 ui32KCCBSizeLog2,
+						 sizeof(RGXFWIF_KCCB_CMD),
+						 (RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS |
+						 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+						 "FwKernelCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB allocation", fail);
+
+	/* KCCB additionally uses a return slot array for FW to be able to send back
+	 * return codes for each required command
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  (1U << ui32KCCBSizeLog2) * sizeof(IMG_UINT32),
+								  "FwKernelCCBRtnSlots",
+								  &psDevInfo->psKernelCCBRtnSlotsMemDesc,
+								  &sFwOsInitScratch.psKernelCCBRtnSlots,
+								  (void**) &psDevInfo->pui32KernelCCBRtnSlots,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB return slot array allocation", fail);
+
+	/* Set up firmware CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psFirmwareCCBCtl,
+						 &psDevInfo->psFirmwareCCBCtlLocal,
+						 &psDevInfo->psFirmwareCCBCtlMemDesc,
+						 &psDevInfo->psFirmwareCCB,
+						 &psDevInfo->psFirmwareCCBMemDesc,
+						 &sFwOsInitScratch.psFirmwareCCBCtl,
+						 &sFwOsInitScratch.psFirmwareCCB,
+						 RGXFWIF_FWCCB_NUMCMDS_LOG2,
+						 sizeof(RGXFWIF_FWCCB_CMD),
+						 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+						 "FwCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware CCB allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_OSDATA),
+								  "FwOsData",
+								  &psDevInfo->psRGXFWIfFwOsDataMemDesc,
+								  &sFwOsInitScratch.sFwOsData,
+								  (void**) &psDevInfo->psRGXFWIfFwOsData,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags = ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_ALL;
+
+	eError = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim, &psDevInfo->psRGXFWIfFwOsData->sPowerSync.ui32Addr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Get Sync Prim FW address", fail);
+
+	/* flush write buffers for psRGXFWIfFwOsData */
+	OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwOsData);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         FLUSH);
+
+	sFwOsInitScratch.ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Set up Workload Estimation firmware CCB */
+		eError = RGXSetupCCB(psDevInfo,
+							 &psDevInfo->psWorkEstFirmwareCCBCtl,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+							 &psDevInfo->psWorkEstFirmwareCCB,
+							 &psDevInfo->psWorkEstFirmwareCCBMemDesc,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCBCtl,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCB,
+							 RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2,
+							 sizeof(RGXFWIF_WORKEST_FWCCB_CMD),
+							 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+							 "FwWEstCCB");
+		PVR_LOG_GOTO_IF_ERROR(eError, "Workload Estimation Firmware CCB allocation", fail);
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise the compatibility check data */
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sFWBVNC);
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sHWBVNC);
+	}
+
+	/* populate the real FwOsInit structure with the values stored in the scratch copy */
+	OSCachedMemCopyWMB(psDevInfo->psRGXFWIfOsInit, &sFwOsInitScratch, sizeof(RGXFWIF_OSINIT));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfOsInit,
+	                         FLUSH);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeFwOsData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFirmware
+
+ @Description Sets up all firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+							  RGX_INIT_APPHINTS        *psApphints,
+							  IMG_UINT32               ui32ConfigFlags,
+							  IMG_UINT32               ui32ConfigFlagsExt,
+							  IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32HWPerfCountersDataSize;
+
+	eError = RGXSetupFwOsData(psDeviceNode,
+							  psApphints->ui32KCCBSizeLog2,
+							  psApphints->ui32HWRDebugDumpLimit,
+							  ui32FwOsCfgFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware os data", fail);
+
+	ui32HWPerfCountersDataSize = sizeof(RGXFWIF_HWPERF_CTL);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest drivers do not configure system-wide firmware data */
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+	else
+	{
+		/* Native and Host drivers must initialise the firmware's system data */
+		eError = RGXSetupFwSysData(psDeviceNode,
+								   psApphints,
+								   ui32ConfigFlags,
+								   ui32ConfigFlagsExt,
+								   ui32HWPerfCountersDataSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware system data", fail);
+	}
+
+	psDevInfo->bFirmwareInitialised = IMG_TRUE;
+
+#if defined(PDUMP)
+	RGXPDumpLoadFWInitData(psDevInfo,
+	                       psApphints,
+	                       ui32HWPerfCountersDataSize);
+#endif /* PDUMP */
+
+fail:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwSysData
+
+ @Description Frees all system-wide firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	psDevInfo->bFirmwareInitialised = IMG_FALSE;
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc)
+	{
+		RGXFWFreeAlignChecks(psDevInfo);
+	}
+
+#if defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM) &&
+	    psDevInfo->psRGXFWSigTDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTDMChecksMemDesc);
+		psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigTAChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTAChecksMemDesc);
+		psDevInfo->psRGXFWSigTAChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSig3DChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSig3DChecksMemDesc);
+		psDevInfo->psRGXFWSig3DChecksMemDesc = NULL;
+	}
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1 &&
+	   psDevInfo->psRGXFWSigRDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigRDMChecksMemDesc);
+		psDevInfo->psRGXFWSigRDMChecksMemDesc = NULL;
+	}
+#endif
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	if (psDevInfo->psCounterBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCounterBufferMemDesc);
+		psDevInfo->psCounterBufferMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	if (psDevInfo->psFirmwareGcovBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psFirmwareGcovBufferMemDesc);
+		psDevInfo->psFirmwareGcovBufferMemDesc = NULL;
+	}
+#endif
+
+	RGXSetupFaultReadRegisterRollback(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfGpuUtilFW != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+			psDevInfo->psRGXFWIfGpuUtilFW = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+		psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfRuntimeCfg != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+			psDevInfo->psRGXFWIfRuntimeCfg = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+		psDevInfo->psRGXFWIfRuntimeCfgMemDesc = NULL;
+	}
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+		}
+	}
+
+	if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufCtl != NULL)
+		{
+			/* deinit/free the tracebuffer allocation */
+			RGXTraceBufferDeinit(psDevInfo);
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+			psDevInfo->psRGXFWIfTraceBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+		psDevInfo->psRGXFWIfTraceBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwSysDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwSysData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwSysDataMemDesc);
+			psDevInfo->psRGXFWIfFwSysData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwSysDataMemDesc);
+		psDevInfo->psRGXFWIfFwSysDataMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	if (psDevInfo->psRGXFWIfTBIBufferMemDesc)
+	{
+		RGXTBIBufferDeinit(psDevInfo);
+	}
+#endif
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	if (psDevInfo->psRGXFWIfRegCfgMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRegCfgMemDesc);
+		psDevInfo->psRGXFWIfRegCfgMemDesc = NULL;
+	}
+#endif
+	if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		psDevInfo->psRGXFWIfHWPerfCountersMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDevInfo->psRGXFWIfNonSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		psDevInfo->psRGXFWIfNonSecureBufMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSecureBufMemDesc);
+		psDevInfo->psRGXFWIfSecureBufMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	if (psDevInfo->psRGXFWIFCoreClkRateMemDesc)
+	{
+		if (psDevInfo->pui32RGXFWIFCoreClkRate != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+			psDevInfo->pui32RGXFWIFCoreClkRate = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+		psDevInfo->psRGXFWIFCoreClkRateMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+		psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwOsData
+
+ @Description Frees all os-specific firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFreeCCBReturnSlots(psDevInfo,
+	                      &psDevInfo->pui32KernelCCBRtnSlots,
+	                      &psDevInfo->psKernelCCBRtnSlotsMemDesc);
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psKernelCCBCtl,
+	           &psDevInfo->psKernelCCBCtlLocal,
+	           &psDevInfo->psKernelCCBCtlMemDesc,
+	           &psDevInfo->psKernelCCB,
+	           &psDevInfo->psKernelCCBMemDesc);
+
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psFirmwareCCBCtl,
+	           &psDevInfo->psFirmwareCCBCtlLocal,
+	           &psDevInfo->psFirmwareCCBCtlMemDesc,
+	           &psDevInfo->psFirmwareCCB,
+	           &psDevInfo->psFirmwareCCBMemDesc);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFreeCCB(psDevInfo,
+		           &psDevInfo->psWorkEstFirmwareCCBCtl,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+		           &psDevInfo->psWorkEstFirmwareCCB,
+		           &psDevInfo->psWorkEstFirmwareCCBMemDesc);
+	}
+#endif
+
+	if (psDevInfo->psPowSyncPrim != NULL)
+	{
+		SyncPrimFree(psDevInfo->psPowSyncPrim);
+		psDevInfo->psPowSyncPrim = NULL;
+	}
+
+	if (psDevInfo->hSyncPrimContext != (IMG_HANDLE) NULL)
+	{
+		SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
+		psDevInfo->hSyncPrimContext = (IMG_HANDLE) NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfHWRInfoBufCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+			psDevInfo->psRGXFWIfHWRInfoBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+		psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwOsDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwOsData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwOsDataMemDesc);
+			psDevInfo->psRGXFWIfFwOsData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwOsDataMemDesc);
+		psDevInfo->psRGXFWIfFwOsDataMemDesc = NULL;
+	}
+
+	if (psDevInfo->psCompletedMemDesc)
+	{
+		if (psDevInfo->pui32CompletedById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
+			psDevInfo->pui32CompletedById = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCompletedMemDesc);
+		psDevInfo->psCompletedMemDesc = NULL;
+	}
+	if (psDevInfo->psEndTimeMemDesc)
+	{
+		if (psDevInfo->pui64EndTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
+			psDevInfo->pui64EndTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psEndTimeMemDesc);
+		psDevInfo->psEndTimeMemDesc = NULL;
+	}
+	if (psDevInfo->psStartTimeMemDesc)
+	{
+		if (psDevInfo->pui64StartTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
+			psDevInfo->pui64StartTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psStartTimeMemDesc);
+		psDevInfo->psStartTimeMemDesc = NULL;
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	if (psDevInfo->hTimerQueryLock)
+	{
+		OSLockDestroy(psDevInfo->hTimerQueryLock);
+		psDevInfo->hTimerQueryLock = NULL;
+	}
+#endif
+
+	if (psDevInfo->psRGXFWHeapGuardPageReserveMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWHeapGuardPageReserveMemDesc);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFirmware
+
+ @Description Frees all the firmware-related allocations
+
+ @Input       psDevInfo
+******************************************************************************/
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFreeFwOsData(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfConnectionCtl)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtl = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfConnectionCtlMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInitMemDesc = NULL;
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+	if (psDevInfo->psRGXFWIfSysInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSysInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInitMemDesc = NULL;
+	}
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalKCCBRoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT32 ui32ReadOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update KCCB read offset */
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+	ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+	if (ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psKCCBCtlLocal->ui32ReadOffset = ui32ReadOffset;
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalFWCCBWoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT32 ui32WriteOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update FWCCB write offset */
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32WriteOffset, INVALIDATE);
+	ui32WriteOffset = psFWCCBCtl->ui32WriteOffset;
+
+	if (ui32WriteOffset > psFWCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psFWCCBCtlLocal->ui32WriteOffset = ui32WriteOffset;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireKernelCCBSlot
+
+ PURPOSE	: Attempts to obtain a slot in the Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32		*pui32Offset)
+{
+	IMG_UINT32 ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+#if defined(PDUMP)
+	/* Wait for sufficient CCB space to become available */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, 0,
+	                      "Wait for space to write kCCB woff=%u", psKCCBCtlLocal->ui32WriteOffset);
+	DevmemPDumpCBP(psDevInfo->psKernelCCBCtlMemDesc,
+	               offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+	               psKCCBCtlLocal->ui32WriteOffset,
+	               1,
+	               (psKCCBCtlLocal->ui32WrapMask + 1));
+#endif
+
+	if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+	}
+	*pui32Offset = ui32NextWriteOffset;
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXPollKernelCCBSlot
+
+ PURPOSE	: Poll for space in Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXPollKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32	ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+	if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+	{
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_OK;
+		}
+
+		/*
+		 * The following check doesn't impact performance, since the
+		 * CPU has to wait for the GPU anyway (full kernel CCB).
+		 */
+		if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return PVRSRV_ERROR_KERNEL_CCB_FULL;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXGetCmdMemCopySize
+
+ PURPOSE	: Calculates actual size of KCCB command getting used
+
+ PARAMETERS	: eCmdType     Type of KCCB command
+
+ RETURNS	: Returns actual size of KCCB command on success else zero
+******************************************************************************/
+static IMG_UINT32 RGXGetCmdMemCopySize(RGXFWIF_KCCB_CMD_TYPE eCmdType)
+{
+	/* First get offset of uCmdData inside the struct RGXFWIF_KCCB_CMD
+	 * This will account alignment requirement of uCmdData union
+	 *
+	 * Then add command-data size depending on command type to calculate actual
+	 * command size required to do mem copy
+	 *
+	 * NOTE: Make sure that uCmdData is the last member of RGXFWIF_KCCB_CMD struct.
+	 */
+	switch (eCmdType)
+	{
+		case RGXFWIF_KCCB_CMD_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_MMUCACHE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_MMUCACHEDATA);
+		}
+#if defined(SUPPORT_USC_BREAKPOINT)
+		case RGXFWIF_KCCB_CMD_BP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_BPDATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_SLCFLUSHINVAL:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_SLCFLUSHINVALDATA);
+		}
+		case RGXFWIF_KCCB_CMD_CLEANUP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CLEANUP_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_POW:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_POWER_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE:
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_ZSBUFFER_BACKING_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELIST_GS_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELISTS_RECONSTRUCTION_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_WRITE_OFFSET_UPDATE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FORCE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA);
+		}
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+		case RGXFWIF_KCCB_CMD_REGCONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_REGCONFIG_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS);
+		}
+#if defined(SUPPORT_PDVFS)
+		case RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_PDVFS_MAX_FREQ_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_OS_STATE_CHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COUNTER_DUMP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_COUNTER_DUMP_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_DA_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CORECLKSPEEDCHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL:
+		case RGXFWIF_KCCB_CMD_WDG_CFG:
+		case RGXFWIF_KCCB_CMD_PHR_CFG:
+		case RGXFWIF_KCCB_CMD_HEALTH_CHECK:
+		case RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL:
+		{
+			/* No command specific data */
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData);
+		}
+		case RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_DEV_VIRTADDR);
+		}
+		case RGXFWIF_KCCB_CMD_CANCEL_WORK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CANCEL_WORK_DATA);
+		}
+		default:
+		{
+			/* Invalid (OR) Unused (OR) Newly added command type */
+			return 0; /* Error */
+		}
+	}
+}
+
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+	                           INVALIDATE);
+	eError = PVRSRVWaitForValueKM(
+	              (IMG_UINT32 __iomem *)&psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFwSharedMemCacheOpExecPfn);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVWaitForValueKM");
+
+#if defined(PDUMP)
+	/* PDumping conditions same as RGXSendCommandRaw for the actual command and poll command to go in harmony */
+	if (PDumpCheckFlagsWrite(psDevInfo->psDeviceNode, ui32PDumpFlags))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Poll on KCCB slot %u for value %u (mask: 0x%x)", ui32SlotNum,
+					 RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED, RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED);
+
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+										ui32SlotNum * sizeof(IMG_UINT32),
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										PDUMP_POLL_OPERATOR_EQUAL,
+										ui32PDumpFlags);
+		PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO  *psDevInfo,
+									  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+									  IMG_UINT32          uiPDumpFlags,
+									  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE	*psDeviceNode = psDevInfo->psDeviceNode;
+	RGXFWIF_CCB_CTL		*psKCCBCtl;
+	RGXFWIF_CCB_CTL		*psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT8			*pui8KCCB = psDevInfo->psKernelCCB;
+	IMG_UINT32			ui32NewWriteOffset;
+	IMG_UINT32			ui32OldWriteOffset;
+	IMG_UINT32			ui32CmdMemCopySize;
+
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+#else
+	IMG_BOOL bContCaptureOn = PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER); /* client connected or in pdump init phase */
+	IMG_BOOL bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, uiPDumpFlags); /* Are we in capture range or continuous and not in a power transition */
+
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			if (!psDevInfo->bDumpedKCCBCtlAlready)
+			{
+				/* entering capture range */
+				psDevInfo->bDumpedKCCBCtlAlready = IMG_TRUE;
+
+				/* Wait for the live FW to catch up */
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: waiting on fw to catch-up, roff: %d, woff: %d",
+						__func__,
+						psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
+				PVRSRVPollForValueKM(psDeviceNode,
+				                     (IMG_UINT32 __iomem *)&psKCCBCtl->ui32ReadOffset,
+				                     ui32OldWriteOffset, 0xFFFFFFFF,
+				                     POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+				                     NULL);
+
+				/* Dump Init state of Kernel CCB control (read and write offset) */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+						"Initial state of kernel CCB Control, roff: %d, woff: %d",
+						psKCCBCtl->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset);
+
+				DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+						0,
+						sizeof(RGXFWIF_CCB_CTL),
+						uiPDumpFlags);
+			}
+		}
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		if ((likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+			(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo)))) ||
+			(KM_FW_CONNECTION_IS(READY, psDevInfo) && KM_OS_CONNECTION_IS(READY, psDevInfo)))
+		{
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: The firmware-driver connection is invalid:"
+									"driver state = %u / firmware state = %u;"
+									"expected READY (%u/%u) or ACTIVE (%u/%u) or in transition (%u/%u);",
+									__func__, KM_GET_OS_CONNECTION(psDevInfo), KM_GET_FW_CONNECTION(psDevInfo),
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_READY,
+									RGXFW_CONNECTION_OS_ACTIVE, RGXFW_CONNECTION_FW_ACTIVE,
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_ACTIVE));
+			eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+			goto _RGXSendCommandRaw_Exit;
+		}
+	}
+#endif
+
+	if (!OSLockIsLocked(psDeviceNode->hPowerLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called without power lock held!",
+				__func__));
+		PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+	}
+
+	/* Acquire a slot in the CCB */
+	eError = RGXAcquireKernelCCBSlot(psDevInfo, &ui32NewWriteOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto _RGXSendCommandRaw_Exit;
+	}
+
+	/* Calculate actual size of command to optimize device mem copy */
+	ui32CmdMemCopySize = RGXGetCmdMemCopySize(psKCCBCmd->eCmdType);
+	PVR_LOG_RETURN_IF_FALSE(ui32CmdMemCopySize !=0, "RGXGetCmdMemCopySize failed", PVRSRV_ERROR_INVALID_CCB_COMMAND);
+
+	/* Copy the command into the CCB */
+	OSCachedMemCopy(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)],
+	                psKCCBCmd, ui32CmdMemCopySize);
+	RGXFwSharedMemCacheOpExec(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)], ui32CmdMemCopySize, PVRSRV_CACHE_OP_FLUSH);
+
+	/* If non-NULL pui32CmdKCCBSlot passed-in, return the kCCB slot in which the command was enqueued */
+	if (pui32CmdKCCBSlot)
+	{
+		*pui32CmdKCCBSlot = ui32OldWriteOffset;
+
+		/* Each such command enqueue needs to reset the slot value first. This is so that a caller
+		 * doesn't get to see stale/false value in allotted slot */
+		OSWriteDeviceMem32WithWMB(&psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                          RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                           FLUSH);
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+							  "Reset kCCB slot number %u", ui32OldWriteOffset);
+		DevmemPDumpLoadMem(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+		                   ui32OldWriteOffset * sizeof(IMG_UINT32),
+						   sizeof(IMG_UINT32),
+						   uiPDumpFlags);
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Device (%p) KCCB slot %u reset with value %u for command type %x",
+		         __func__, psDevInfo, ui32OldWriteOffset, RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE, psKCCBCmd->eCmdType));
+	}
+
+	/* Memory barrier before KCCB write offset update. */
+	OSWriteMemoryBarrier(NULL);
+
+
+	/* Move past the current command */
+	psKCCBCtlLocal->ui32WriteOffset = ui32NewWriteOffset;
+	psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
+	/* Read-back of memory before Kick MTS */
+	OSWriteMemoryBarrier(&psKCCBCtl->ui32WriteOffset);
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32WriteOffset, FLUSH);
+
+#if defined(PDUMP)
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			/* Dump new Kernel CCB content */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCB cmd woff = %d",
+					ui32OldWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBMemDesc,
+					ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD),
+					ui32CmdMemCopySize,
+					uiPDumpFlags);
+
+			/* Dump new kernel CCB write offset */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCBCtl woff: %d",
+					ui32NewWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					sizeof(IMG_UINT32),
+					uiPDumpFlags);
+
+			/* mimic the read-back of the write from above */
+			DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					ui32NewWriteOffset,
+					0xFFFFFFFF,
+					PDUMP_POLL_OPERATOR_EQUAL,
+					uiPDumpFlags);
+		}
+		/* out of capture range */
+		else
+		{
+			eError = RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", _RGXSendCommandRaw_Exit);
+		}
+	}
+#endif
+
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags, "MTS kick for kernel CCB");
+	/*
+	 * Kick the MTS to schedule the firmware.
+	 */
+	__MTSScheduleWrite(psDevInfo, RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK);
+
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE,
+	           RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK, uiPDumpFlags);
+
+#if defined(SUPPORT_AUTOVZ)
+	RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+
+#if defined(NO_HARDWARE)
+	/* keep the roff updated because fw isn't there to update it */
+	psKCCBCtl->ui32ReadOffset = psKCCBCtlLocal->ui32WriteOffset;
+#endif
+
+_RGXSendCommandRaw_Exit:
+	return eError;
+}
+
+/******************************************************************************
+ FUNCTION	: _AllocDeferredCommand
+
+ PURPOSE	: Allocate a KCCB command and add it to KCCB deferred list
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: eKCCBType		Firmware Command type
+			: psKCCBCmd		Firmware Command
+			: uiPDumpFlags	Pdump flags
+
+ RETURNS	: PVRSRV_OK	If all went good, PVRSRV_ERROR_RETRY otherwise.
+******************************************************************************/
+static PVRSRV_ERROR _AllocDeferredCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                          RGXFWIF_KCCB_CMD   *psKCCBCmd,
+                                          IMG_UINT32         uiPDumpFlags)
+{
+	RGX_DEFERRED_KCCB_CMD *psDeferredCommand;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psDeferredCommand = OSAllocMem(sizeof(*psDeferredCommand));
+
+	if (!psDeferredCommand)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Deferring a KCCB command failed: allocation failure: requesting retry"));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeferredCommand->sKCCBcmd = *psKCCBCmd;
+	psDeferredCommand->uiPDumpFlags = uiPDumpFlags;
+	psDeferredCommand->psDevInfo = psDevInfo;
+
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_add_to_tail(&(psDevInfo->sKCCBDeferredCommandsListHead), &(psDeferredCommand->sListNode));
+	psDevInfo->ui32KCCBDeferredCommandsCount++;
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: _FreeDeferredCommand
+
+ PURPOSE	: Remove from the deferred list the sent deferred KCCB command
+
+ PARAMETERS	: psNode			Node in deferred list
+			: psDeferredKCCBCmd	KCCB Command to free
+
+ RETURNS	: None
+******************************************************************************/
+static void _FreeDeferredCommand(DLLIST_NODE *psNode, RGX_DEFERRED_KCCB_CMD *psDeferredKCCBCmd)
+{
+	dllist_remove_node(psNode);
+	psDeferredKCCBCmd->psDevInfo->ui32KCCBDeferredCommandsCount--;
+	OSFreeMem(psDeferredKCCBCmd);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXSendCommandsFromDeferredList
+
+ PURPOSE	: Try send KCCB commands in deferred list to KCCB
+		  Should be called by holding PowerLock
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: bPoll		Poll for space in KCCB
+
+ RETURNS	: PVRSRV_OK	If all commands in deferred list are sent to KCCB,
+			  PVRSRV_ERROR_KERNEL_CCB_FULL otherwise.
+******************************************************************************/
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+	RGX_DEFERRED_KCCB_CMD *psTempDeferredKCCBCmd;
+	DLLIST_NODE sCommandList;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDevInfo->psDeviceNode));
+
+	/* !!! Important !!!
+	 *
+	 * The idea of moving the whole list hLockKCCBDeferredCommandsList below
+	 * to the temporary list is only valid under the principle that all of the
+	 * operations are also protected by the power lock. It must be held
+	 * so that the order of the commands doesn't get messed up while we're
+	 * performing the operations on the local list.
+	 *
+	 * The necessity of releasing the hLockKCCBDeferredCommandsList comes from
+	 * the fact that _FreeDeferredCommand() is allocating memory and it can't
+	 * be done in atomic context (inside section protected by a spin lock).
+	 *
+	 * We're using spin lock here instead of mutex to quickly perform a check
+	 * if the list is empty in MISR without a risk that the MISR is going
+	 * to sleep due to a lock.
+	 */
+
+	/* move the whole list to a local list so it can be processed without lock */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_replace_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (dllist_is_empty(&sCommandList))
+		{
+			return PVRSRV_OK;
+		}
+
+		/* For every deferred KCCB command, try to send it*/
+		dllist_foreach_node(&sCommandList, psNode, psNext)
+		{
+			psTempDeferredKCCBCmd = IMG_CONTAINER_OF(psNode, RGX_DEFERRED_KCCB_CMD, sListNode);
+			eError = RGXSendCommandRaw(psTempDeferredKCCBCmd->psDevInfo,
+			                           &psTempDeferredKCCBCmd->sKCCBcmd,
+			                           psTempDeferredKCCBCmd->uiPDumpFlags,
+			                           NULL /* We surely aren't interested in kCCB slot number of deferred command */);
+			if (eError != PVRSRV_OK)
+			{
+				if (!bPoll)
+				{
+					eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+					goto cleanup_;
+				}
+				break;
+			}
+
+			_FreeDeferredCommand(psNode, psTempDeferredKCCBCmd);
+		}
+
+		if (bPoll)
+		{
+			PVRSRV_ERROR eErrPollForKCCBSlot;
+
+			/* Don't overwrite eError because if RGXPollKernelCCBSlot returns OK and the
+			 * outer loop times-out, we'll still want to return KCCB_FULL to caller
+			 */
+			eErrPollForKCCBSlot = RGXPollKernelCCBSlot(psDevInfo);
+			if (eErrPollForKCCBSlot == PVRSRV_ERROR_KERNEL_CCB_FULL)
+			{
+				eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+				goto cleanup_;
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+cleanup_:
+	/* if the local list is not empty put it back to the deferred list head
+	 * so that the old order of commands is retained */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_insert_list_at_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+										  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+										  IMG_UINT32          uiPDumpFlags,
+										  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	IMG_BOOL     bPoll = (pui32CmdKCCBSlot != NULL);
+	PVRSRV_ERROR eError;
+
+	/*
+	 * First try to Flush all the cmds in deferred list.
+	 *
+	 * We cannot defer an incoming command if the caller is interested in
+	 * knowing the command's kCCB slot: it plans to poll/wait for a
+	 * response from the FW just after the command is enqueued, so we must
+	 * poll for space to be available.
+	 */
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, bPoll);
+	if (eError == PVRSRV_OK)
+	{
+		eError = RGXSendCommandRaw(psDevInfo,
+								   psKCCBCmd,
+								   uiPDumpFlags,
+								   pui32CmdKCCBSlot);
+	}
+
+	/*
+	 * If we don't manage to enqueue one of the deferred commands or the command
+	 * passed as argument because the KCCB is full, insert the latter into the deferred commands list.
+	 * The deferred commands will also be flushed eventually by:
+	 *  - one more KCCB command sent for any DM
+	 *  - RGX_MISRHandler_CheckFWActivePowerState
+	 */
+	if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+	{
+		if (pui32CmdKCCBSlot == NULL)
+		{
+			eError = _AllocDeferredCommand(psDevInfo, psKCCBCmd, uiPDumpFlags);
+		}
+		else
+		{
+			/* Let the caller retry. Otherwise if we deferred the command and returned OK,
+			 * the caller can end up looking in a stale CCB slot.
+			 */
+			PVR_DPF((PVR_DBG_WARNING, "%s: Couldn't flush the deferred queue for a command (Type:%d) "
+			                        "- will be retried", __func__, psKCCBCmd->eCmdType));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32         *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+
+	/* Ensure Rogue is powered up before kicking MTS */
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: failed to acquire powerlock (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVPowerLock_Exit;
+	}
+
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition Rogue to ON (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+										  psKCCBCmd,
+										  ui32PDumpFlags,
+	                                      pui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to schedule command (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+#if defined(DEBUG)
+		/* PVRSRVDebugRequest must be called without powerlock */
+		PVRSRVPowerUnlock(psDeviceNode);
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+		goto _PVRSRVPowerLock_Exit;
+#endif
+	}
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_PVRSRVPowerLock_Exit:
+	return eError;
+}
+
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGX_MISRHandler_ScheduleProcessQueues
+
+ @Description - Sends uncounted kick to all the DMs (the FW will process all
+				the queue for all the DMs)
+******************************************************************************/
+static void RGX_MISRHandler_ScheduleProcessQueues(void *pvData)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR           eError;
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_OFF;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Check whether it's worth waking up the GPU */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guests are not permitted to change the device power state */
+		if ((eError != PVRSRV_OK) || (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+	else
+	{
+		if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			RGXFWIF_GPU_UTIL_FW    *psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+			IMG_BOOL               bGPUHasWorkWaiting;
+
+			/* Check whether it's worth waking up the GPU */
+			RGXFwSharedMemCacheOpValue(psUtilFW->ui64GpuLastWord, INVALIDATE);
+			bGPUHasWorkWaiting =
+			    (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
+
+			if (!bGPUHasWorkWaiting)
+			{
+				/* all queues are empty, don't wake up the GPU */
+				PVRSRVPowerUnlock(psDeviceNode);
+				return;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		/* wake up the GPU */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition device to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+
+	/* Memory barrier before Kick MTS */
+	OSWriteMemoryBarrier(NULL);
+
+	/* uncounted kick to the FW */
+	HTBLOGK(HTB_SF_MAIN_KICK_UNCOUNTED);
+	__MTSScheduleWrite(psDevInfo, (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return OSInstallMISR(phMISR,
+			RGX_MISRHandler_ScheduleProcessQueues,
+			psDeviceNode,
+			"RGX_ScheduleProcessQueues");
+}
+
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+                                              RGXFWIF_DM          eKCCBType,
+                                              RGXFWIF_KCCB_CMD    *psKCCBCmd,
+                                              IMG_UINT32          ui32PDumpFlags,
+                                              IMG_UINT32          *pui32CmdKCCBSlot,
+                                              IMG_BOOL            bCallerHasPwrLock)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMMUSyncUpdate;
+
+	/* Don't send the command/power up request if device not available. */
+	if (unlikely((psDevInfo == NULL) ||
+	             (psDevInfo->psDeviceNode == NULL) ||
+				 (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)))
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	/* Don't send the command/power up request if device in deinit phase.
+	 * The de-init thread could destroy the device whilst the power up
+	 * sequence below is accessing the HW registers.
+	 * Not yet safe to free resources. Caller should retry later.
+	 */
+	if (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	    psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+
+
+	if (!bCallerHasPwrLock)
+	{
+		/* PVRSRVPowerLock guarantees atomicity between commands. This is helpful
+		   in a scenario with several applications allocating resources. */
+		eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+
+			/* If system is found powered OFF, Retry scheduling the command */
+			if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+			{
+				eError = PVRSRV_ERROR_RETRY;
+			}
+
+			goto RGXScheduleCommand_exit;
+		}
+	}
+
+	if (unlikely(psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	             psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING))
+	{
+		/* If we have the power lock the device is valid but the deinit
+		 * thread could be waiting for the lock. */
+		eError = PVRSRV_ERROR_RETRY;
+		goto _PVRSRVInvalidDeviceError_Exit;
+	}
+
+	/* Ensure device is powered up before sending any commands */
+	PDUMPPOWCMDSTART(psDevInfo->psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDevInfo->psDeviceNode,
+	                                     PVRSRV_DEV_POWER_STATE_ON,
+	                                     PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXPreKickCacheCommand(psDevInfo, eKCCBType, &uiMMUSyncUpdate);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+_PVRSRVInvalidDeviceError_Exit:
+	if (!bCallerHasPwrLock)
+	{
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+	}
+RGXScheduleCommand_exit:
+	return eError;
+}
+
+/*
+ * RGXCheckFirmwareCCB
+ */
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT8 *psFWCCB = psDevInfo->psFirmwareCCB;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo) ||
+								 (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+								  (KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))),
+								 "FW-KM connection is down");
+#endif
+
+	eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+		return;
+	}
+
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtlLocal->ui32WriteOffset)
+	{
+		/* Point to the next command */
+		const RGXFWIF_FWCCB_CMD *psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtlLocal->ui32ReadOffset;
+		RGXFwSharedMemCacheOpPtr(psFwCCBCmd, INVALIDATE);
+
+
+		HTBLOGK(HTB_SF_MAIN_FWCCB_CMD, psFwCCBCmd->eCmdType);
+		switch (psFwCCBCmd->eCmdType)
+		{
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
+				}
+				RGXProcessRequestZSBufferBacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
+				}
+				RGXProcessRequestZSBufferUnbacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELIST_GROW, "Request to grow the free list");
+				}
+				RGXProcessRequestGrow(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
+				}
+
+				if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d/%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psDevInfo->psRGXFWIfHWRInfoBufCtl->ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+
+				RGXProcessRequestFreelistsReconstruction(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION:
+			{
+				/* Notify client drivers */
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+					const RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA *psCmdFwPagefault =
+							&psFwCCBCmd->uCmdData.sCmdFWPagefault;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+						sErrorData.uErrData.sFwPFErrData.sFWFaultAddr.uiAddr = psCmdFwPagefault->sFWFaultAddr.uiAddr;
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+														  &sErrorData);
+					}
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
+			{
+				const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification =
+						&psFwCCBCmd->uCmdData.sCmdContextResetNotification;
+				IMG_UINT32 ui32ErrorPid = 0;
+
+				FWCommonContextListSetLastResetReason(psDevInfo,
+				                                      &ui32ErrorPid,
+				                                      psCmdContextResetNotification);
+
+				/* Increment error counter (if appropriate) */
+				if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32WGPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32WGPErrorCount++;
+					}
+				}
+				else if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32TRPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32TRPErrorCount++;
+					}
+				}
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = psCmdContextResetNotification->eResetReason;
+						sErrorData.pid = ui32ErrorPid;
+
+						/* Populate error data according to reset reason */
+						switch (psCmdContextResetNotification->eResetReason)
+						{
+							case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+							case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+							{
+								sErrorData.uErrData.sChecksumErrData.ui32ExtJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+								sErrorData.uErrData.sChecksumErrData.eDM = psCmdContextResetNotification->eDM;
+								break;
+							}
+							default:
+							{
+								break;
+							}
+						}
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+						                                  &sErrorData);
+					}
+				}
+
+				/* Notify if a page fault */
+				if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF)
+				{
+					DevmemIntPFNotify(psDevInfo->psDeviceNode,
+							psCmdContextResetNotification->ui64PCAddress,
+							psCmdContextResetNotification->sFaultAddress);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
+			{
+				PVRSRV_ERROR eError;
+				PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+				OSAtomicWrite(&psDevInfo->psDeviceNode->eDebugDumpRequested, PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE);
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal FW Cmd debug dump event, dumping now instead", __func__));
+					PVRSRVDebugRequest(psDevInfo->psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				IMG_PID pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
+				IMG_INT32 i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
+
+				switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
+				{
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,i32AdjustmentValue,0,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,i32AdjustmentValue,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,i32AdjustmentValue,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,i32AdjustmentValue,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,i32AdjustmentValue,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,i32AdjustmentValue,0,pidTmp);
+						break;
+					}
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,0,i32AdjustmentValue,pidTmp);
+						break;
+					}
+#endif
+				}
+#endif
+				break;
+			}
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+			case RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE:
+			{
+				RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo,
+											  psFwCCBCmd->uCmdData.sCmdCoreClkRateChange.ui32CoreClkRate);
+				break;
+			}
+#endif
+			case RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
+			{
+				RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+				if (psDevInfo->psRGXFWIfFwSysData != NULL  &&
+				    psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)
+				{
+					PVRSRV_ERROR eError;
+
+					/* Power down... */
+					eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+															 PVRSRV_SYS_POWER_STATE_OFF,
+															 PVRSRV_POWER_FLAGS_NONE);
+					if (eError == PVRSRV_OK)
+					{
+						/* Clear the FW faulted flags... */
+						psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags &= ~(RGXFWIF_HWR_FW_FAULT|RGXFWIF_HWR_RESTART_REQUESTED);
+						OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags);
+						RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags,
+						                           FLUSH);
+
+						/* Power back up again... */
+						eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+																 PVRSRV_SYS_POWER_STATE_ON,
+																 PVRSRV_POWER_FLAGS_NONE);
+
+						/* Send a dummy KCCB command to ensure the FW wakes up and checks the queues... */
+						if (eError == PVRSRV_OK)
+						{
+							LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+							{
+								eError = RGXFWHealthCheckCmd(psDevInfo);
+								if (eError != PVRSRV_ERROR_RETRY)
+								{
+									break;
+								}
+								OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+							} END_LOOP_UNTIL_TIMEOUT_US();
+						}
+					}
+
+					/* Notify client drivers and system layer of FW fault */
+					{
+						PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+						PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+						/* Client notification of device error will be achieved by
+						 * clients calling UM function RGXGetLastDeviceError() */
+						psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+
+						/* Notify system layer */
+						if (psDevConfig->pfnSysDevErrorNotify)
+						{
+							PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+							sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+							psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+							                                  &sErrorData);
+						}
+					}
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Failed firmware restart (%s)",
+								 __func__, PVRSRVGetErrorString(eError)));
+					}
+				}
+				break;
+			}
+			default:
+			{
+				/* unknown command */
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unknown Command (eCmdType=0x%08x)",
+				         __func__, psFwCCBCmd->eCmdType));
+				/* Assert on magic value corruption */
+				PVR_ASSERT((((IMG_UINT32)psFwCCBCmd->eCmdType & RGX_CMD_MAGIC_DWORD_MASK) >> RGX_CMD_MAGIC_DWORD_SHIFT) == RGX_CMD_MAGIC_DWORD);
+			}
+		}
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		OSMemoryBarrier(NULL);
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+		OSWriteMemoryBarrier(NULL);
+
+		if (psFWCCBCtlLocal->ui32ReadOffset == psFWCCBCtlLocal->ui32WriteOffset)
+		{
+			eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+				return;
+			}
+		}
+	}
+}
+
+/*
+ * PVRSRVRGXFrameworkCopyCommand
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+		DEVMEM_MEMDESC	*psFWFrameworkMemDesc,
+		IMG_PBYTE		pbyGPUFRegisterList,
+		IMG_UINT32		ui32FrameworkRegisterSize)
+{
+	PVRSRV_ERROR	eError;
+	RGXFWIF_RF_REGISTERS	*psRFReg;
+
+	eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
+			(void **)&psRFReg);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	OSDeviceMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
+	RGXFwSharedMemCacheOpPtr(psRFReg, FLUSH);
+
+	/* Release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
+
+	/*
+	 * Dump the FW framework buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FWFramework buffer");
+	DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXFrameworkCreateKM
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+		DEVMEM_MEMDESC		**ppsFWFrameworkMemDesc,
+		IMG_UINT32			ui32FrameworkCommandSize)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+		Allocate device memory for the firmware GPU framework state.
+		Sufficient info to kick one or more DMs should be contained in this buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware framework state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+			ui32FrameworkCommandSize,
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwGPUFrameworkState",
+			ppsFWFrameworkMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware framework state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem *pui32LinMemAddr,
+												IMG_UINT32			ui32Value,
+												IMG_UINT32			ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32CurrentQueueLength, ui32MaxRetries;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDevNode->pvDevice;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+	ui32CurrentQueueLength = (psKCCBCtlLocal->ui32WrapMask+1 +
+					psKCCBCtlLocal->ui32WriteOffset -
+					psKCCBCtlLocal->ui32ReadOffset) & psKCCBCtlLocal->ui32WrapMask;
+	ui32CurrentQueueLength += psDevInfo->ui32KCCBDeferredCommandsCount;
+
+	for (ui32MaxRetries = ui32CurrentQueueLength + 1;
+				ui32MaxRetries > 0;
+				ui32MaxRetries--)
+	{
+
+		/*
+		 * PVRSRVPollForValueKM flags are set to POLL_FLAG_NONE in this case so that the function
+		 * does not generate an error message. In this case, the PollForValueKM is expected to
+		 * timeout as there is work ongoing on the GPU which may take longer than the timeout period.
+		 */
+		eError = PVRSRVPollForValueKM(psDevNode, pui32LinMemAddr, ui32Value, ui32Mask, POLL_FLAG_NONE, NULL);
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			break;
+		}
+
+		RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		         __func__, PVRSRVGetErrorString(eError),
+		         pui32LinMemAddr, ui32Value));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32ConfigState,
+				IMG_BOOL bSetNotClear)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	RGXFWIF_KCCB_CMD sStateFlagCmd = { 0 };
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_SYSDATA *psFwSysData;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (NULL == psFwSysData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Fw Sys Config is not mapped into CPU space", __func__));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* apply change and ensure the new data is written to memory
+	 * before requesting the FW to read it
+	 */
+	ui32Config = ui32Config & RGXFWIF_INICFG_ALL;
+	if (bSetNotClear)
+	{
+		psFwSysData->ui32ConfigFlags |= ui32Config;
+	}
+	else
+	{
+		psFwSysData->ui32ConfigFlags &= ~ui32Config;
+	}
+	OSWriteMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+	RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags, FLUSH);
+
+	/* return current/new value to caller */
+	if (pui32ConfigState)
+	{
+		*pui32ConfigState = psFwSysData->ui32ConfigFlags;
+	}
+
+	OSMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	/* notify FW to update setting */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of the value */
+		sStateFlagCmd.eCmdType = RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sStateFlagCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock(psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the value to be updated as the FW validates
+		 * the parameters and modifies the ui32ConfigFlags
+		 * accordingly
+		 * (for completeness as registered callbacks should also
+		 *  not permit invalid transitions)
+		 */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO	*psDevInfo,
+									   RGXFWIF_DM			eDM,
+									   RGXFWIF_KCCB_CMD		*psKCCBCmd,
+									   RGXFWIF_CLEANUP_TYPE	eCleanupType,
+									   IMG_UINT32			ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32kCCBCommandSlot;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus);
+
+	PVR_LOG_RETURN_IF_FALSE((eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+							(eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT),
+							"Cleanup aborted: Device in bad state", PVRSRV_OK);
+#endif
+
+
+	/* Clean-up commands sent during frame capture intervals must be dumped even when not in capture range... */
+	ui32PDumpFlags |= PDUMP_FLAGS_INTERVAL;
+
+	psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
+	psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
+
+	/*
+		Send the cleanup request to the firmware. If the resource is still busy
+		the firmware will tell us and we'll drop out with a retry.
+	*/
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+											  eDM,
+											  psKCCBCmd,
+											  ui32PDumpFlags,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR ,"RGXScheduleCommandAndGetKCCBSlot() failed (%s) in %s()",
+			         PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		goto fail_command;
+	}
+
+	/* Wait for command kCCB slot to be updated by FW */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+						  "Wait for the firmware to reply to the cleanup command");
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot,
+									  ui32PDumpFlags);
+	/*
+		If the firmware hasn't got back to us in a timely manner
+		then bail and let the caller retry the command.
+	 */
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: RGXWaitForKCCBSlotUpdate timed out. Dump debug information.",
+		         __func__));
+
+		eError = PVRSRV_ERROR_RETRY;
+#if defined(DEBUG)
+		PVRSRVDebugRequest(psDevInfo->psDeviceNode,
+				DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+#endif
+		goto fail_poll;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		goto fail_poll;
+	}
+
+#if defined(PDUMP)
+	/*
+	 * The cleanup request to the firmware will tell us if a given resource is busy or not.
+	 * If the RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY flag is set, this means that the resource is
+	 * still in use. In this case we return a PVRSRV_ERROR_RETRY error to the client drivers
+	 * and they will re-issue the cleanup request until it succeed.
+	 *
+	 * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
+	 * that cleanup requests are only submitted if the resource is unused.
+	 * If this is not the case, the following poll will block infinitely, making sure
+	 * the issue doesn't go unnoticed.
+	 */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+					"Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
+					eDM,
+					psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
+					psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+									ui32kCCBCommandSlot * sizeof(IMG_UINT32),
+									0,
+									RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+#endif
+
+	/*
+		If the command has was run but a resource was busy, then the request
+		will need to be retried.
+	*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot],
+	                           INVALIDATE);
+
+	if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY))
+	{
+		if (psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+		eError = PVRSRV_ERROR_RETRY;
+		goto fail_requestbusy;
+	}
+
+	return PVRSRV_OK;
+
+fail_requestbusy:
+fail_poll:
+fail_command:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	RGXRequestCommonContextCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD			sRCCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+	PRGXFWIF_FWCOMMONCONTEXT	psFWCommonContextFWAddr;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	/* Force retry if this context's CCB is currently being dumped
+	 * as part of the stalled CCB debug */
+	if (psDevInfo->pvEarliestStalledClientCCB == (void*)FWCommonContextGetClientCCB(psServerCommonContext))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: [%u/%d]: Forcing retry as psDevInfo->pvEarliestStalledClientCCB = psCtxClientCCB <%p>",
+		         __func__,
+		         psDeviceNode->sDevId.ui32InternalID,
+		         psDeviceNode->sDevId.i32KernelDeviceID,
+		         psDevInfo->pvEarliestStalledClientCCB));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psFWCommonContextFWAddr = FWCommonContextGetFWAddress(psServerCommonContext);
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Common ctx cleanup Request DM%d [context = 0x%08x]",
+			eDM, psFWCommonContextFWAddr.ui32Addr);
+	PDUMPCOMMENT(psDeviceNode, "Wait for CCB to be empty before common ctx cleanup");
+
+	RGXCCBPDumpDrainCCB(FWCommonContextGetClientCCB(psServerCommonContext), ui32PDumpFlags);
+#endif
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+									   eDM,
+									   &sRCCleanUpCmd,
+									   RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
+									   ui32PDumpFlags);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+/*
+ * RGXFWRequestHWRTDataCleanUp
+ */
+
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         PRGXFWIF_HWRTDATA psHWRTData)
+{
+	RGXFWIF_KCCB_CMD			sHWRTDataCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDeviceNode, "HW RTData cleanup Request [HWRTData = 0x%08x]", psHWRTData.ui32Addr);
+
+	sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
+
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+	                                   RGXFWIF_DM_GP,
+	                                   &sHWRTDataCleanUpCmd,
+	                                   RGXFWIF_CLEANUP_HWRTDATA,
+	                                   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a HWRTData cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestFreeListCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_FREELIST psFWFreeList)
+{
+	RGXFWIF_KCCB_CMD			sFLCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_GP,
+									   &sFLCleanUpCmd,
+									   RGXFWIF_CLEANUP_FREELIST,
+									   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a memory context cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestZSBufferCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer)
+{
+	RGXFWIF_KCCB_CMD			sZSBufferCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_3D,
+									   &sZSBufferCleanUpCmd,
+									   RGXFWIF_CLEANUP_ZSBUFFER,
+									   PDUMP_FLAGS_NONE);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS = ui32HCSDeadlineMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Hard Context Switching deadline inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32HCSDeadlineMS),
+							  ui32HCSDeadlineMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock)
+{
+	RGXFWIF_KCCB_CMD	sCmpKCCBCmd = { 0 };
+
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+	if (bCallerHasPwrLock)
+	{
+		return	RGXScheduleCommandWithoutPowerLock(psDevInfo,
+												   RGXFWIF_DM_GP,
+												   &sCmpKCCBCmd,
+												   PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		return	RGXScheduleCommand(psDevInfo,
+								   RGXFWIF_DM_GP,
+								   &sCmpKCCBCmd,
+								   PDUMP_FLAGS_CONTINUOUS);
+	}
+}
+
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32CBaseMapCtxReg;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+	else
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+#else
+	/*
+	 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+	 * FW crash from its memory context
+	 */
+	RGXWriteKernelMMUPC64(&psDevInfo->sLayerParams,
+				                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+				                      ((0xDEADBEEF
+				                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+				                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+				                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               IMG_UINT32 ui32DriverID,
+                               RGXFWIF_OS_STATE_CHANGE eOSOnlineState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sOSOnlineStateCmd = { 0 };
+	const RGXFWIF_SYSDATA *psFwSysData;
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	sOSOnlineStateCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.ui32DriverID = ui32DriverID;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = eOSOnlineState;
+
+#if defined(SUPPORT_AUTOVZ)
+	{
+		IMG_BOOL bConnectionDown = IMG_FALSE;
+
+		PVR_UNREFERENCED_PARAMETER(psFwSysData);
+		sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = RGXFWIF_OS_OFFLINE;
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			/* Send the offline command regardless if power lock is held or not.
+			 * Under AutoVz this is done during regular driver deinit, store-to-ram suspend
+			 * or (optionally) from a kernel panic callback. Deinit and suspend operations
+			 * take the lock in the rgx pre/post power functions as expected.
+			 * The kernel panic callback is a last resort way of letting the firmware know that
+			 * the VM is unrecoverable and the vz connection must be disabled. It cannot wait
+			 * on other kernel threads to finish and release the lock. */
+			eError = RGXSendCommand(psDevInfo,
+									&sOSOnlineStateCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Guests and Host going offline should wait for confirmation
+		 * from the Firmware of the state change. If this fails, break
+		 * the connection on the OS Driver's end as backup. */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (ui32DriverID == RGXFW_HOST_DRIVER_ID))
+		{
+			LOOP_UNTIL_TIMEOUT_US(SECONDS_TO_MICROSECONDS/2)
+			{
+				KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+				if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+				{
+					bConnectionDown = IMG_TRUE;
+					break;
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (!bConnectionDown)
+			{
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+		}
+	}
+#else
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* no reason for Guests to update their state or any other VM's.
+		 * This is the Hypervisor and Host driver's responsibility. */
+		return PVRSRV_OK;
+	}
+	else
+	{
+		const volatile RGXFWIF_OS_RUNTIME_FLAGS *psFwRunFlags;
+
+		PVR_ASSERT(psFwSysData != NULL);
+
+		psFwRunFlags = (const volatile RGXFWIF_OS_RUNTIME_FLAGS*) &psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+		/* Attempt several times until the FW manages to offload the OS */
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			IMG_UINT32 ui32kCCBCommandSlot;
+
+			/* Send request */
+			eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+													  RGXFWIF_DM_GP,
+													  &sOSOnlineStateCmd,
+													  PDUMP_FLAGS_CONTINUOUS,
+													  &ui32kCCBCommandSlot);
+			if (unlikely(eError == PVRSRV_ERROR_RETRY)) continue;
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommand", return_);
+
+			/* Wait for FW to process the cmd */
+			eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", return_);
+
+			/* read the OS state */
+			OSMemoryBarrier(NULL);
+			/* check if FW finished offloading the driver and is stopped */
+			if ((eOSOnlineState == RGXFWIF_OS_ONLINE && (psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_READY || psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_ACTIVE)) ||
+			    (eOSOnlineState == RGXFWIF_OS_OFFLINE && psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_OFFLINE))
+			{
+				eError = PVRSRV_OK;
+				break;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_TIMEOUT;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+
+return_ :
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgPHRCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE((ui32PHRMode == RGXFWIF_PHR_MODE_OFF) ||
+							(ui32PHRMode == RGXFWIF_PHR_MODE_RD_RESET),
+							"Invalid PHR Mode.", PVRSRV_ERROR_INVALID_PARAMS);
+
+	sCfgPHRCmd.eCmdType = RGXFWIF_KCCB_CMD_PHR_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode = ui32PHRMode;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Periodic Hardware Reset Mode inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32PHRMode),
+							  ui32PHRMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GP,
+		                            &sCfgPHRCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+							IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgWdgCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCfgWdgCmd.eCmdType = RGXFWIF_KCCB_CMD_WDG_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs = ui32WdgPeriodUs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the firmware watchdog period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32WdgPeriodUs),
+							  ui32WdgPeriodUs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sCfgWdgCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious)
+{
+	/* Attempt to detect and deal with any stalled client contexts.
+	 * bIgnorePrevious may be set by the caller if they know a context to be
+	 * stalled, as otherwise this function will only identify stalled
+	 * contexts which have not been previously reported.
+	 */
+
+	IMG_UINT32 ui32StalledClientMask = 0;
+
+	if (!(OSTryLockAcquire(psDevInfo->hCCBStallCheckLock)))
+	{
+		PVR_LOG(("RGXCheckForStalledClientContexts: Failed to acquire hCCBStallCheckLock, returning..."));
+		return;
+	}
+
+	ui32StalledClientMask |= CheckForStalledClientTransferCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientRenderCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientComputeCtxt(psDevInfo);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	ui32StalledClientMask |= CheckForStalledClientKickSyncCtxt(psDevInfo);
+#endif
+	/* If at least one DM stalled bit is different than before */
+	if (bIgnorePrevious || (psDevInfo->ui32StalledClientMask != ui32StalledClientMask))
+	{
+		if (ui32StalledClientMask > 0)
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Print all the stalled DMs */
+			PVR_LOG(("Possible stalled client RGX contexts detected: %s%s%s%s%s%s%s",
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_GP),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TDM_2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TA),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_3D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_CDM),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ3D)));
+
+			PVR_LOG(("Trying to identify stalled context...(%s) [%d]",
+			         pszStalledAction, bIgnorePrevious));
+
+			DumpStalledContextInfo(psDevInfo);
+		}
+		else
+		{
+			if (psDevInfo->ui32StalledClientMask> 0)
+			{
+				/* Indicate there are no stalled DMs */
+				PVR_LOG(("No further stalled client contexts exist"));
+			}
+		}
+		psDevInfo->ui32StalledClientMask = ui32StalledClientMask;
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+	OSLockRelease(psDevInfo->hCCBStallCheckLock);
+}
+
+/*
+	RGXUpdateHealthStatus
+*/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed)
+{
+	const PVRSRV_DATA*           psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
+	PVRSRV_RGXDEV_INFO*          psDevInfo;
+	const RGXFWIF_TRACEBUF*      psRGXFWIfTraceBufCtl;
+	const RGXFWIF_SYSDATA*       psFwSysData;
+	const RGXFWIF_OSDATA*        psFwOsData;
+	const RGXFWIF_CCB_CTL*       psKCCBCtl;
+	RGXFWIF_CCB_CTL*             psKCCBCtlLocal;
+	IMG_UINT32                   ui32ThreadCount;
+	IMG_BOOL                     bKCCBCmdsWaiting;
+	PVRSRV_ERROR                 eError;
+
+	PVR_ASSERT(psDevNode != NULL);
+	psDevInfo = psDevNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, PVRSRV_OK);
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_OK;
+	}
+
+	psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData,
+								 INVALIDATE);
+		psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         INVALIDATE);
+	psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+
+	/* If this is a quick update, then include the last current value... */
+	if (!bCheckAfterTimePassed)
+	{
+		eNewStatus = OSAtomicRead(&psDevNode->eHealthStatus);
+		eNewReason = OSAtomicRead(&psDevNode->eHealthReason);
+	}
+
+	/* Decrement the SLR holdoff counter (if non-zero) */
+	if (psDevInfo->ui32SLRHoldoffCounter > 0)
+	{
+		psDevInfo->ui32SLRHoldoffCounter--;
+	}
+
+	/* Take power lock, retry if it's in use in another task. */
+	eError = PVRSRVPowerTryLockWaitForTimeout(psDevNode);
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		/* Skip health status update if timeout */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Power lock timeout, increase OS_POWERLOCK_TIMEOUT_US.", __func__));
+		goto _RGXUpdateHealthStatus_Exit;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerTryLockWaitForTimeout");
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+		return PVRSRV_OK;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		/* On a PCI error all reads from the PCI bar may return 0xFFFFFFFF.
+		   This value is not valid for a core ID. */
+		if (psFwSysData->ui32MemFaultCheck == RGX_PCI_ERROR_VALUE_DWORD)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: PCI error", __func__));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR;
+			PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_PCI_ERROR);
+			PVRSRVPowerUnlock(psDevNode);
+			goto _RGXUpdateHealthStatus_Exit;
+		}
+	}
+
+	/* If Rogue is not powered on, just skip ahead and check for stalled client CCBs */
+	if (PVRSRVIsDevicePowered(psDevNode))
+	{
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			/*
+			   Firmware thread checks...
+			 */
+			for (ui32ThreadCount = 0; ui32ThreadCount < RGXFW_THREAD_NUM; ui32ThreadCount++)
+			{
+				const IMG_CHAR* pszTraceAssertInfo;
+
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf, INVALIDATE);
+				pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
+
+				/*
+				Check if the FW has hit an assert...
+				*/
+				if (*pszTraceAssertInfo != '\0')
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Firmware thread %d has asserted: %.*s (%.*s:%d)",
+							__func__, ui32ThreadCount, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							pszTraceAssertInfo, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+				{
+					/*
+					   Check the threads to see if they are in the same poll locations as last time...
+					*/
+					if (bCheckAfterTimePassed)
+					{
+						if (psFwSysData->aui32CrPollAddr[ui32ThreadCount] != 0  &&
+							psFwSysData->aui32CrPollCount[ui32ThreadCount] == psDevInfo->aui32CrLastPollCount[ui32ThreadCount])
+						{
+							PVR_DPF((PVR_DBG_WARNING, "%s: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
+									__func__, ui32ThreadCount,
+									((psFwSysData->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+									psFwSysData->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
+									psFwSysData->aui32CrPollMask[ui32ThreadCount]));
+							eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+							eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
+							PVRSRVPowerUnlock(psDevNode);
+							goto _RGXUpdateHealthStatus_Exit;
+						}
+						psDevInfo->aui32CrLastPollCount[ui32ThreadCount] = psFwSysData->aui32CrPollCount[ui32ThreadCount];
+					}
+				}
+			}
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+			{
+				/*
+				Check if the FW has faulted...
+				*/
+				if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_FW_FAULT)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+							"%s: Firmware has faulted and needs to restart",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_FAULT;
+					if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_RESTART_REQUESTED)
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_RESTARTING;
+					}
+					else
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_IDLING;
+					}
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+			}
+		}
+
+		/*
+		   Event Object Timeouts check...
+		*/
+		if (!bCheckAfterTimePassed)
+		{
+			if (psDevInfo->ui32GEOTimeoutsLastTime > 1 && psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Global Event Object Timeouts have risen (from %d to %d)",
+				         __func__,
+				         psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
+			}
+			psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
+		}
+
+		/*
+		   Check the Kernel CCB pointer is valid. If any commands were waiting last time, then check
+		   that some have executed since then.
+		*/
+		bKCCBCmdsWaiting = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+		psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+		if (psKCCBCtl != NULL && psKCCBCtlLocal != NULL)
+		{
+			/* update KCCB read offset */
+			RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+			psKCCBCtlLocal->ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+			if (psKCCBCtlLocal->ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask  ||
+				psKCCBCtlLocal->ui32WriteOffset > psKCCBCtlLocal->ui32WrapMask)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: KCCB has invalid offset (ROFF=%d WOFF=%d)",
+						__func__, psKCCBCtlLocal->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+			}
+
+			if (psKCCBCtlLocal->ui32ReadOffset != psKCCBCtlLocal->ui32WriteOffset)
+			{
+				bKCCBCmdsWaiting = IMG_TRUE;
+			}
+		}
+
+		if (bCheckAfterTimePassed && psFwOsData != NULL)
+		{
+			IMG_UINT32 ui32KCCBCmdsExecuted = psFwOsData->ui32KCCBCmdsExecuted;
+
+			if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
+			{
+				/*
+				   If something was waiting last time then the Firmware has stopped processing commands.
+				*/
+				if (psDevInfo->bKCCBCmdsWaitingLastTime)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: No KCCB commands executed since check!",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+				}
+
+				/*
+				   If no commands are currently pending and nothing happened since the last poll, then
+				   schedule a dummy command to ping the firmware so we know it is alive and processing.
+				*/
+				if ((!bKCCBCmdsWaiting) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+				{
+					/* Protect the PDumpLoadMem. RGXScheduleCommand() cannot take the
+					 * PMR lock itself, because some bridge functions will take the PMR lock
+					 * before calling RGXScheduleCommand
+					 */
+					PVRSRV_ERROR eError = RGXFWHealthCheckCmdWithoutPowerLock(psDevNode->pvDevice);
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Cannot schedule Health Check command! (0x%x)",
+								__func__, eError));
+					}
+					else
+					{
+						bKCCBCmdsWaiting = IMG_TRUE;
+					}
+				}
+			}
+
+			psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
+			psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
+		}
+	}
+
+	/*
+	   Interrupt counts check...
+	*/
+	if (bCheckAfterTimePassed  &&  psFwOsData != NULL)
+	{
+		IMG_UINT32  ui32LISRCount   = 0;
+		IMG_UINT32  ui32FWCount     = 0;
+		IMG_UINT32  ui32MissingInts = 0;
+
+		/* Add up the total number of interrupts issued, sampled/received and missed... */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* Only the Host OS has a sample count, so only one counter to check. */
+		ui32LISRCount += psDevInfo->aui32SampleIRQCount[RGXFW_HOST_DRIVER_ID];
+		ui32FWCount   += OSReadHWReg32(psDevInfo->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[RGXFW_HOST_DRIVER_ID]);
+#else
+		IMG_UINT32  ui32Index;
+
+		for (ui32Index = 0;  ui32Index < RGXFW_THREAD_NUM;  ui32Index++)
+		{
+			ui32LISRCount += psDevInfo->aui32SampleIRQCount[ui32Index];
+			ui32FWCount   += psFwOsData->aui32InterruptCount[ui32Index];
+		}
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+		if (ui32LISRCount < ui32FWCount)
+		{
+			ui32MissingInts = (ui32FWCount-ui32LISRCount);
+		}
+
+		if (ui32LISRCount == psDevInfo->ui32InterruptCountLastTime  &&
+		    ui32MissingInts >= psDevInfo->ui32MissingInterruptsLastTime  &&
+		    psDevInfo->ui32MissingInterruptsLastTime > 1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: LISR has not received the last %d interrupts",
+					__func__, ui32MissingInts));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+
+			/* Schedule the MISRs to help mitigate the problems of missing interrupts. */
+			OSScheduleMISR(psDevInfo->pvMISRData);
+			if (psDevInfo->pvAPMISRData != NULL)
+			{
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+		psDevInfo->ui32InterruptCountLastTime    = ui32LISRCount;
+		psDevInfo->ui32MissingInterruptsLastTime = ui32MissingInts;
+	}
+
+	/* Release power lock before RGXCheckForStalledClientContexts */
+	PVRSRVPowerUnlock(psDevNode);
+
+	/*
+	   Stalled CCB check...
+	*/
+	if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
+	{
+		RGXCheckForStalledClientContexts(psDevInfo, IMG_FALSE);
+	}
+
+	/* Notify client driver and system layer of any eNewStatus errors */
+	if (eNewStatus > PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		/* Client notification of device error will be achieved by
+		 * clients calling UM function RGXGetLastDeviceError() */
+		psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+
+		/* Notify system layer */
+		{
+			PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+			PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+			if (psDevConfig->pfnSysDevErrorNotify)
+			{
+				PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+				sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+				sErrorData.uErrData.sHostWdgData.ui32Status = (IMG_UINT32)eNewStatus;
+				sErrorData.uErrData.sHostWdgData.ui32Reason = (IMG_UINT32)eNewReason;
+
+				psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+				                                  &sErrorData);
+			}
+		}
+	}
+
+	/*
+	   Finished, save the new status...
+	*/
+_RGXUpdateHealthStatus_Exit:
+	OSAtomicWrite(&psDevNode->eHealthStatus, eNewStatus);
+	OSAtomicWrite(&psDevNode->eHealthReason, eNewReason);
+	RGXSRV_HWPERF_DEVICE_INFO_HEALTH(psDevInfo, eNewStatus, eNewReason);
+
+	/*
+	 * Attempt to service the HWPerf buffer to regularly transport idle/periodic
+	 * packets to host buffer.
+	 */
+	if (psDevNode->pfnServiceHWPerf != NULL)
+	{
+		PVRSRV_ERROR eError = psDevNode->pfnServiceHWPerf(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: "
+			         "Error occurred when servicing HWPerf buffer (%d)",
+			         __func__, eError));
+		}
+	}
+
+	/* Attempt to refresh timer correlation data */
+	RGXTimeCorrRestartPeriodic(psDevNode);
+
+	return PVRSRV_OK;
+} /* RGXUpdateHealthStatus */
+
+#if defined(SUPPORT_AUTOVZ)
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+		(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))))
+	{
+		/* read and write back the alive token value to confirm to the
+		 * virtualisation watchdog that this connection is healthy */
+		KM_SET_OS_ALIVE_TOKEN(KM_GET_FW_ALIVE_TOKEN(psDevInfo), psDevInfo);
+		KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH);
+	}
+}
+
+/*
+	RGXUpdateAutoVzWatchdog
+*/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode)
+{
+	if (likely(psDevNode != NULL))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+
+		if (unlikely((psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || !psDevInfo->bRGXPowered ||
+					  psDevInfo->pvRegsBaseKM == NULL || psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+					  psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)))
+		{
+			/* If the firmware is not initialised, stop here */
+			return;
+		}
+		else
+		{
+			PVRSRV_ERROR eError = PVRSRVPowerLock(psDevNode);
+			PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PVRSRVPowerLock");
+
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+	}
+}
+
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DriverID;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+	                           INVALIDATE);
+
+	for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+	     ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED;
+	     ui32DriverID++)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+			psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+		if (eGuestState == RGXFW_CONNECTION_FW_ACTIVE ||
+		    eGuestState == RGXFW_CONNECTION_FW_READY)
+		{
+			PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+		}
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer)
+{
+	PVRSRV_ERROR eError;
+	PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
+
+	PVR_ASSERT((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+	PVR_RETURN_IF_INVALID_PARAM((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+
+	if (bKick)
+	{
+		if (psKMHWRTDataSet)
+		{
+			PRGXFWIF_CLEANUP_CTL psCleanupCtl;
+
+			eError = RGXSetFirmwareAddress(&psCleanupCtl, psKMHWRTDataSet->psHWRTDataFwMemDesc,
+					offsetof(RGXFWIF_HWRTDATA, sCleanupState),
+					RFW_FWADDR_NOREF_FLAG);
+			PVR_RETURN_IF_ERROR(eError);
+
+			*(psCleanupCtlWrite++) = psCleanupCtl;
+		}
+
+		if (eDM == RGXFWIF_DM_3D)
+		{
+			RGXFWIF_PRBUFFER_TYPE eBufferType;
+			RGX_ZSBUFFER_DATA *psBuffer = NULL;
+
+			for (eBufferType = RGXFWIF_PRBUFFER_START; eBufferType < RGXFWIF_PRBUFFER_MAXSUPPORTED; eBufferType++)
+			{
+				switch (eBufferType)
+				{
+				case RGXFWIF_PRBUFFER_ZSBUFFER:
+					psBuffer = psZSBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MSAABUFFER:
+					psBuffer = psMSAAScratchBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MAXSUPPORTED:
+					psBuffer = NULL;
+					break;
+				}
+				if (psBuffer)
+				{
+					(psCleanupCtlWrite++)->ui32Addr = psBuffer->sZSBufferFWDevVAddr.ui32Addr +
+									offsetof(RGXFWIF_PRBUFFER, sCleanupState);
+					psBuffer = NULL;
+				}
+			}
+		}
+	}
+
+	*pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
+	PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO       *psDevInfo;
+	RGXFWIF_HWRINFOBUF       *psHWRInfoBuf;
+	IMG_UINT32               i;
+
+	if (psDevNode->pvDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVINFO;
+	}
+	psDevInfo = psDevNode->pvDevice;
+
+	psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+	for (i = 0 ; i < RGXFWIF_DM_MAX ; i++)
+	{
+		/* Reset the HWR numbers */
+		psHWRInfoBuf->aui32HwrDmLockedUpCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmFalseDetectCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmRecoveredCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmOverranCount[i] = 0;
+	}
+
+	for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+	{
+		psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
+	}
+
+	psHWRInfoBuf->ui32WriteIndex = 0;
+	psHWRInfoBuf->ui32DDReqCount = 0;
+
+	OSWriteMemoryBarrier(&psHWRInfoBuf->ui32DDReqCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid)
+{
+
+	PVRSRV_ERROR eError;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+								 ui32Log2PageSize,
+								 ui32NumOfPages,
+								 ui32LogicalOffset,
+								 psPhyAddr,
+								 bValid,
+								 DEVICE_USE);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR_DevPhysAddr failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRUnLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	return eError;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_OK);
+
+	if (psDevInfo->bDumpedKCCBCtlAlready)
+	{
+		/* exiting capture range or pdump block */
+		psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+		/* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+                              PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
+                              "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
+                              psDevInfo->psKernelCCBCtl,
+                              ui32WriteOffset,
+                              ui32WriteOffset);
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+				offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+				ui32WriteOffset,
+				0xffffffff,
+				PDUMP_POLL_OPERATOR_EQUAL,
+				PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: problem pdumping POL for kCCBCtl (%d)", __func__, eError));
+		}
+	}
+
+	return eError;
+
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXClientConnectCompatCheck_ClientAgainstFW
+
+ @Description
+
+ Check compatibility of client and firmware (build options)
+ at the connection time.
+
+ @Input psDeviceNode - device node
+ @Input ui32ClientBuildOptions - build options for the client
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+******************************************************************************/
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32ClientBuildOptions)
+{
+	IMG_UINT32		ui32BuildOptionsMismatch;
+	IMG_UINT32		ui32BuildOptionsFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_FW_INFO_HEADER *psFWInfoHeader;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psDevInfo == NULL || psDevInfo->psRGXFWIfOsInitMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Cannot acquire kernel fw compatibility check info, RGXFWIF_OSINIT structure not allocated.",
+		         __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *) &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+#endif
+
+	ui32BuildOptionsFW = psFWInfoHeader->ui32Flags;
+	ui32BuildOptionsMismatch = ui32ClientBuildOptions ^ ui32BuildOptionsFW;
+
+	if (ui32BuildOptionsMismatch != 0)
+	{
+		if ((ui32ClientBuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in client: (0x%x). Please check rgx_options.h",
+					 ui32ClientBuildOptions & ui32BuildOptionsMismatch ));
+		}
+
+		if ((ui32BuildOptionsFW & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFW & ui32BuildOptionsMismatch ));
+		}
+
+		return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware and client build options match. [ OK ]", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapAllocMap
+
+ @Description Register firmware heap for the specified driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID - Guest driver
+ @Input sDevPAddr    - Heap address
+ @Input ui64DevPSize - Heap size
+
+ @Return   PVRSRV_ERROR - PVRSRV_OK if heap setup was successful.
+
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR szRegionRAName[RA_MAX_NAME_LENGTH];
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_MEMALLOCFLAGS_T uiRawFwHeapAllocFlags = (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+													PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PREMAP0 + ui32DriverID));
+	PHYS_HEAP_CONFIG *psFwHeapConfig = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+	PHYS_HEAP_CONFIG sFwHeapConfig;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psFwHeapConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW_MAIN heap config not found."));
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	OSSNPrintf(szRegionRAName, sizeof(szRegionRAName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	if (!ui64DevPSize ||
+		!sDevPAddr.uiAddr ||
+		ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED ||
+		ui64DevPSize != RGX_FIRMWARE_RAW_HEAP_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid parameters for %s", szRegionRAName));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sFwHeapConfig = *psFwHeapConfig;
+	sFwHeapConfig.eType = PHYS_HEAP_TYPE_LMA;
+	sFwHeapConfig.ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP;
+	sFwHeapConfig.uConfig.sLMA.sStartAddr.uiAddr = 0;
+	sFwHeapConfig.uConfig.sLMA.sCardBase.uiAddr = sDevPAddr.uiAddr;
+	sFwHeapConfig.uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	eError = PhysmemCreateHeapLMA(psDeviceNode,
+	                              RGXPhysHeapGetLMAPolicy(sFwHeapConfig.ui32UsageFlags, psDeviceNode),
+	                              &sFwHeapConfig,
+	                              szRegionRAName,
+	                              &psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysmemCreateHeapLMA:PREMAP [%d]", ui32DriverID);
+
+	eError = PhysHeapAcquire(psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysHeapAcquire:PREMAP [%d]", ui32DriverID);
+
+	psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID] = psDeviceNode->apsFWPremapPhysHeap[ui32DriverID];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate and map raw firmware heap for DriverID: [%d]", ui32DriverID);
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* don't clear the heap of other guests on allocation */
+	uiRawFwHeapAllocFlags &= (ui32DriverID > RGXFW_HOST_DRIVER_ID) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL);
+#endif
+
+	/* if the firmware is already powered up, consider the firmware heaps are pre-mapped. */
+	if (psDeviceNode->bAutoVzFwIsUp)
+	{
+		uiRawFwHeapAllocFlags &= RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Allocation and mapping for Firmware heaps done by TEE.", __func__));
+#else
+	eError = DevmemFwAllocate(psDevInfo,
+							  RGX_FIRMWARE_RAW_HEAP_SIZE,
+							  uiRawFwHeapAllocFlags,
+							  psDevInfo->psPremappedFwRawHeap[ui32DriverID]->pszName,
+							  &psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+#endif
+
+	/* Mark this devmem heap as premapped so allocations will not require device mapping. */
+	DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+
+	if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+	{
+		/* if the Host's raw fw heap is premapped, mark its main & config sub-heaps accordingly
+		 * No memory allocated from these sub-heaps will be individually mapped into the device's
+		 * address space so they can remain marked permanently as premapped. */
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapUnmapFree
+
+ @Description Unregister firmware heap for the specified guest driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID
+
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* remove the premap status, so the heap can be unmapped and freed */
+	if (psDevInfo->psPremappedFwRawHeap[ui32DriverID])
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_FALSE);
+	}
+
+	if (psDevInfo->psPremappedFwRawMemDesc[ui32DriverID])
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+		psDevInfo->psPremappedFwRawMemDesc[ui32DriverID] = NULL;
+	}
+}
+
+/*
+	RGXReadMETAAddr
+*/
+static PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO	*psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem  *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem  *pui8RegBase = pvRegBase;
+	IMG_UINT32 ui32PollValue;
+	IMG_UINT32 ui32PollMask;
+	IMG_UINT32 ui32PollRegOffset;
+	IMG_UINT32 ui32ReadOffset;
+	IMG_UINT32 ui32WriteOffset;
+	IMG_UINT32 ui32WriteValue;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+	{
+		ui32PollValue = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollMask = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1;
+		ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0;
+		ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN;
+		ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX;
+	}
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteUncheckedHWReg32(pvRegBase, ui32WriteOffset, ui32WriteValue);
+	(void)OSReadUncheckedHWReg32(pvRegBase, ui32WriteOffset);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadUncheckedHWReg32(pvRegBase, ui32ReadOffset);
+#else
+	IMG_UINT8 __iomem  *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteHWReg32(
+			psDevInfo->pvRegsBaseKM,
+			RGX_CR_META_SP_MSLVCTRL0,
+			ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGXWriteMETAAddr
+*/
+static PVRSRV_ERROR RGXWriteMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 ui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem *pui8RegBase = pvRegBase;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA, ui32Value);
+		}
+		else
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA, ui32Value);
+		}
+	}
+	else
+	{
+		/* Wait for Slave Port to be Ready */
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+				(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Issue the Write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0); /* Fence write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT); /* Fence write */
+	}
+#else
+	IMG_UINT8 __iomem *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Write */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 *pui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXReadMETAAddr(psDevInfo, ui32FWAddr, pui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvReadMem(psDevInfo, ui32FWAddr, pui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 ui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXWriteMETAAddr(psDevInfo, ui32FWAddr, ui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvWriteMem(psDevInfo, ui32FWAddr, ui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32FwVA,
+                             IMG_CPU_PHYADDR *psCpuPA,
+                             IMG_DEV_PHYADDR *psDevPA,
+                             IMG_UINT64 *pui64RawPTE)
+{
+	PVRSRV_ERROR eError       = PVRSRV_OK;
+	IMG_CPU_PHYADDR sCpuPA    = {0U};
+	IMG_DEV_PHYADDR sDevPA    = {0U};
+	IMG_UINT64 ui64RawPTE     = 0U;
+	MMU_FAULT_DATA sFaultData = {0U};
+	MMU_CONTEXT *psFwMMUCtx   = psDevInfo->psKernelMMUCtx;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+	IMG_UINT32 ui32DriverID   = (ui32FwVA - ui32FwHeapBase) / RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT32 ui32HeapId;
+	PHYS_HEAP *psPhysHeap;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* MIPS uses the same page size as the OS, while others default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) ?
+	                             OSGetPageSize() : BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#else
+	/* default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#endif
+
+	IMG_UINT32 ui32PageOffset = (ui32FwVA & (ui32FwPageSize - 1));
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM((ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((psCpuPA != NULL) ||
+	                               (psDevPA != NULL) ||
+	                               (pui64RawPTE != NULL)),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((ui32FwVA >= ui32FwHeapBase) &&
+	                              (ui32FwVA < ui32FwHeapEnd)),
+	                              eError, ErrorExit);
+
+	ui32HeapId = (ui32DriverID == RGXFW_HOST_DRIVER_ID) ?
+	              PVRSRV_PHYS_HEAP_FW_MAIN : (PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID);
+	psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[ui32HeapId];
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS is equipped with a dedicated MMU */
+		RGXMipsCheckFaultAddress(psFwMMUCtx, ui32FwVA, &sFaultData);
+	}
+	else
+#endif
+	{
+		IMG_UINT64 ui64FwDataBaseMask;
+		IMG_DEV_VIRTADDR sDevVAddr;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			ui64FwDataBaseMask = ~(RGXFW_SEGMMU_DATA_META_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_BASE_ADDRESS);
+		}
+#if !defined(EMULATOR)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			ui64FwDataBaseMask = ~RGXRISCVFW_REGION_MASK;
+		}
+#endif
+		else
+		{
+			PVR_LOG_GOTO_WITH_ERROR("RGXGetFwMapping", eError, PVRSRV_ERROR_NOT_IMPLEMENTED, ErrorExit);
+		}
+
+		sDevVAddr.uiAddr = (ui32FwVA & ui64FwDataBaseMask) | RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Fw CPU shares a subset of the GPU's VA space */
+		MMU_CheckFaultAddress(psFwMMUCtx, &sDevVAddr, &sFaultData);
+	}
+
+	ui64RawPTE = sFaultData.sLevelData[MMU_LEVEL_1].ui64Address;
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		IMG_BOOL bValidPage = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+		                       BITMASK_HAS(ui64RawPTE, RGXMIPSFW_TLB_VALID) :
+		                       BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#else
+		IMG_BOOL bValidPage = BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#endif
+
+		if (!bValidPage)
+		{
+			/* don't report invalid pages */
+			eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+			sDevPA.uiAddr = ui32PageOffset + ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			                RGXMIPSFW_TLB_GET_PA(ui64RawPTE) :
+			                (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK));
+#else
+			sDevPA.uiAddr = ui32PageOffset + (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK);
+#endif
+
+			/* Only the Host's Firmware heap is present in the Host's CPU IPA space */
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPA, &sDevPA);
+			}
+			else
+			{
+				sCpuPA.uiAddr = 0U;
+			}
+		}
+	}
+
+	if (psCpuPA != NULL)
+	{
+		*psCpuPA = sCpuPA;
+	}
+
+	if (psDevPA != NULL)
+	{
+		*psDevPA = sDevPA;
+	}
+
+	if (pui64RawPTE != NULL)
+	{
+		*pui64RawPTE = ui64RawPTE;
+	}
+
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = ui32VzConnectionCooldownPeriodInSec;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Vz reconnect request cooldown period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32VzConnectionCooldownPeriodInSec),
+							  ui32VzConnectionCooldownPeriodInSec,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxfwutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxhwperf.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxhwperf.c
new file mode 100644
index 000000000000..e43fc499d9db
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxhwperf.c
@@ -0,0 +1,1030 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+
+#include "info_page_defs.h"
+
+/* This is defined by default to enable producer callbacks.
+ * Clients of the TL interface can disable the use of the callback
+ * with PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK. */
+#define SUPPORT_TL_PRODUCER_CALLBACK 1
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+/* Defines size of buffers returned from acquire/release calls */
+#define FW_STREAM_BUFFER_SIZE (0x80000)
+#define HOST_STREAM_BUFFER_SIZE (0x20000)
+
+/* Must be at least as large as two tl packets of maximum size */
+static_assert(HOST_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "HOST_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+static_assert(FW_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "FW_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+
+/******************************************************************************
+ * RGX HW Performance Profiling Server API(s)
+ *****************************************************************************/
+
+static IMG_BOOL RGXServerFeatureFlagsToHWPerfFlagsAddBlock(
+	RGX_HWPERF_BVNC_BLOCK	* const psBlocks,
+	IMG_UINT16				* const pui16Count,
+	const IMG_UINT16		ui16BlockID, /* see RGX_HWPERF_CNTBLK_ID */
+	const IMG_UINT16		ui16NumCounters,
+	const IMG_UINT16		ui16NumBlocks)
+{
+	const IMG_UINT16 ui16Count = *pui16Count;
+
+	if (ui16Count < RGX_HWPERF_MAX_BVNC_BLOCK_LEN)
+	{
+		RGX_HWPERF_BVNC_BLOCK * const psBlock = &psBlocks[ui16Count];
+
+		/* If the GROUP is non-zero, convert from e.g. RGX_CNTBLK_ID_USC0 to RGX_CNTBLK_ID_USC_ALL. The table stores the former (plus the
+		number of blocks and counters) but PVRScopeServices expects the latter (plus the number of blocks and counters). The conversion
+		could always be moved to PVRScopeServices, but it's less code this way. */
+		psBlock->ui16BlockID		= (ui16BlockID & RGX_CNTBLK_ID_GROUP_MASK) ? (ui16BlockID | RGX_CNTBLK_ID_UNIT_ALL_MASK) : ui16BlockID;
+		if ((ui16BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+		{
+			psBlock->ui16NumCounters	= RGX_CNTBLK_COUNTERS_MAX;
+		}
+		else
+		{
+			psBlock->ui16NumCounters	= ui16NumCounters;
+		}
+		psBlock->ui16NumBlocks		= ui16NumBlocks;
+
+		*pui16Count = ui16Count + 1;
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_HWPERF_BVNC *psBVNC)
+{
+	IMG_PCHAR pszBVNC;
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDevInfo), "psDevInfo invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((pszBVNC = RGXDevBVNCString(psDevInfo)))
+	{
+		size_t uiStringLength = OSStringNLength(pszBVNC, RGX_HWPERF_MAX_BVNC_LEN - 1);
+		OSStringSafeCopy(psBVNC->aszBvncString, pszBVNC, uiStringLength + 1);
+		memset(&psBVNC->aszBvncString[uiStringLength], 0, RGX_HWPERF_MAX_BVNC_LEN - uiStringLength);
+	}
+	else
+	{
+		*psBVNC->aszBvncString = 0;
+	}
+
+	psBVNC->ui32BvncKmFeatureFlags = 0x0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERFBUS_FLAG;
+	}
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_XT_TOP_INFRASTRUCTURE_FLAG;
+	}
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERF_COUNTER_BATCH))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERF_COUNTER_BATCH_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ROGUEXE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_ROGUEXE_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PBE2_IN_XE_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_MULTICORE_FLAG;
+	}
+
+#ifdef SUPPORT_WORKLOAD_ESTIMATION
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Not a part of BVNC feature line and so doesn't need the feature supported check */
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION;
+	}
+#endif
+
+	/* Define the HW counter block counts. */
+	{
+		RGX_HWPERF_BVNC_BLOCK					* const psBlocks	= psBVNC->aBvncBlocks;
+		IMG_UINT16								* const pui16Count	= &psBVNC->ui16BvncBlocks;
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	*asCntBlkTypeModel;
+		const IMG_UINT32						ui32CntBlkModelLen	= RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+		IMG_UINT32								ui32BlkCfgIdx;
+		size_t									uiCount;
+		IMG_BOOL								bOk					= IMG_TRUE;
+
+		// Initialise to zero blocks
+		*pui16Count = 0;
+
+		// Add all the blocks
+		for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+		{
+			const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	* const psCntBlkInfo = &asCntBlkTypeModel[ui32BlkCfgIdx];
+			RGX_HWPERF_CNTBLK_RT_INFO				sCntBlkRtInfo;
+			/* psCntBlkInfo->ui8NumUnits gives compile-time info. For BVNC agnosticism, we use this: */
+			if (psCntBlkInfo->pfnIsBlkPresent(psCntBlkInfo, psDevInfo, &sCntBlkRtInfo))
+			{
+				bOk &= RGXServerFeatureFlagsToHWPerfFlagsAddBlock(psBlocks, pui16Count, psCntBlkInfo->ui32CntBlkIdBase, psCntBlkInfo->ui8NumCounters, sCntBlkRtInfo.ui32NumUnits);
+			}
+		}
+
+		/* If this fails, consider why the static_assert didn't fail, and consider increasing RGX_HWPERF_MAX_BVNC_BLOCK_LEN */
+		PVR_ASSERT(bOk);
+
+		// Zero the remaining entries
+		uiCount = *pui16Count;
+		OSDeviceMemSet(&psBlocks[uiCount], 0, (RGX_HWPERF_MAX_BVNC_BLOCK_LEN - uiCount) * sizeof(*psBlocks));
+	}
+
+	/* The GPU core count is overwritten by the FW */
+	psBVNC->ui16BvncGPUCores = 0;
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVRGXConfigMuxHWPerfCountersKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigMuxHWPerfCountersKM(
+		CONNECTION_DATA               *psConnection,
+		PVRSRV_DEVICE_NODE            *psDeviceNode,
+		IMG_UINT32                     ui32ArrayLen,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK  *psBlockConfigs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	DEVMEM_MEMDESC*		psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_MUX_CNTBLK* psFwArray;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed
+	 */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.sBlockConfigs,
+	                      psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM parameters set, calling FW"));*/
+
+	/* Ask the FW to carry out the HWPerf configuration command
+	 */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM command scheduled for FW"));*/
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration
+	 */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM firmware completed"));*/
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED", ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+/*
+	PVRSRVRGXConfigCustomCountersReadingHWPerfKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigCustomCountersKM(
+		CONNECTION_DATA             * psConnection,
+		PVRSRV_DEVICE_NODE          * psDeviceNode,
+		IMG_UINT16                    ui16CustomBlockID,
+		IMG_UINT16                    ui16NumCustomCounters,
+		IMG_UINT32                  * pui32CustomCounterIDs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	DEVMEM_MEMDESC*		psFwSelectCntrsMemDesc = NULL;
+	IMG_UINT32*			psFwArray;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVRGXSelectCustomCountersKM: configure block %u to read %u counters", ui16CustomBlockID, ui16NumCustomCounters));
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS;
+	sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.ui16NumCounters = ui16NumCustomCounters;
+	sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.ui16CustomBlock = ui16CustomBlockID;
+
+	if (ui16NumCustomCounters > 0)
+	{
+		PVR_ASSERT(pui32CustomCounterIDs);
+
+		/* used for passing counters config to the Firmware, write-only for the CPU */
+		eError = DevmemFwAllocate(psDevice,
+		                          sizeof(IMG_UINT32) * ui16NumCustomCounters,
+		                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+		                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+		                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+		                          "FwHWPerfConfigCustomCounters",
+		                          &psFwSelectCntrsMemDesc);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+		eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.sCustomCounterIDs,
+		                      psFwSelectCntrsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+		eError = DevmemAcquireCpuVirtAddr(psFwSelectCntrsMemDesc, (void **)&psFwArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+		OSCachedMemCopyWMB(psFwArray, pui32CustomCounterIDs, sizeof(IMG_UINT32) * ui16NumCustomCounters);
+		DevmemPDumpLoadMem(psFwSelectCntrsMemDesc,
+		                   0,
+		                   sizeof(IMG_UINT32) * ui16NumCustomCounters,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	/* Push in the KCCB the command to configure the custom counters block */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail3);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXSelectCustomCountersKM: Command scheduled"));
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXSelectCustomCountersKM: FW operation completed"));
+
+	if (ui16NumCustomCounters > 0)
+	{
+		/* Release temporary memory used for block configuration */
+		RGXUnsetFirmwareAddress(psFwSelectCntrsMemDesc);
+		DevmemReleaseCpuVirtAddr(psFwSelectCntrsMemDesc);
+		DevmemFwUnmapAndFree(psDevice, psFwSelectCntrsMemDesc);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf custom counters %u reading will be sent with the next HW events", ui16NumCustomCounters));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	if (psFwSelectCntrsMemDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psFwSelectCntrsMemDesc);
+	}
+fail2:
+	if (psFwSelectCntrsMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psFwSelectCntrsMemDesc);
+	}
+fail1:
+	if (psFwSelectCntrsMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevice, psFwSelectCntrsMemDesc);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*
+	PVRSRVRGXConfigureHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+		CONNECTION_DATA          * psConnection,
+		PVRSRV_DEVICE_NODE       * psDeviceNode,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32ArrayLen,
+		RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs)
+{
+	PVRSRV_ERROR             eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD         sKccbCmd;
+	DEVMEM_MEMDESC           *psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_CNTBLK *psFwArray;
+	IMG_UINT32               ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO       *psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(ui32CtrlWord);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgDABlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_CNTBLK) * ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersDAConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgDABlks.sBlockConfigs,
+	                               psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/* Ask the FW to carry out the HWPerf configuration command. */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration. */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED",
+	         ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfConfigMuxCounters(
+		RGX_HWPERF_CONNECTION           *psHWPerfConnection,
+		IMG_UINT32					     ui32NumBlocks,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK	*asBlockConfigs)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !asBlockConfigs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXConfigMuxHWPerfCountersKM(NULL,
+		                                            psDevData->psRgxDevNode,
+		                                            ui32NumBlocks,
+		                                            asBlockConfigs);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXConfigMuxHWPerfCountersKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfConfigureAndEnableCustomCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT16              ui16CustomBlockID,
+		IMG_UINT16          ui16NumCustomCounters,
+		IMG_UINT32         *pui32CustomCounterIDs)
+{
+	PVRSRV_ERROR            eError;
+	RGX_HWPERF_DEVICE       *psHWPerfDev;
+
+	/* Validate input arguments supplied by the caller */
+	PVR_LOG_RETURN_IF_FALSE((NULL != psHWPerfConnection), "psHWPerfConnection invalid",
+	                   PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((0 != ui16NumCustomCounters), "uiNumBlocks invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((NULL != pui32CustomCounterIDs),"asBlockConfigs invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Check # of blocks */
+	PVR_LOG_RETURN_IF_FALSE((!(ui16CustomBlockID > RGX_HWPERF_MAX_CUSTOM_BLKS)),"ui16CustomBlockID invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Check # of counters */
+	PVR_LOG_RETURN_IF_FALSE((!(ui16NumCustomCounters > RGX_HWPERF_MAX_CUSTOM_CNTRS)),"ui16NumCustomCounters invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		RGX_KM_HWPERF_DEVDATA *psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		eError = PVRSRVRGXConfigCustomCountersKM(NULL,
+				                                 psDevData->psRgxDevNode,
+												 ui16CustomBlockID, ui16NumCustomCounters, pui32CustomCounterIDs);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXCtrlCustHWPerfKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHWPerfBlockTypeByID
+@Description    Lookup function to obtain a block type descriptor for a given
+                counter block identifier.
+@Input          psDevInfo                       A pointer to current device info.
+@Input          ui32BlockID                     The block ID for which a type
+                                                descriptor should be retrieved.
+@Return         RGXFW_HWPERF_CNTBLK_TYPE_MODEL  Block type descriptor.
+*/ /**************************************************************************/
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *
+GetHWPerfBlockTypeByID(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32BlockID)
+{
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	IMG_UINT32 ui32TableIdx = 0xFFFF;
+	RGX_HWPERF_CNTBLK_RT_INFO sRtInfo; /* Only used to satisfy pfnIsBlkPresent requirements. */
+
+#if defined(HWPERF_UNIFIED)
+	IMG_UINT32 uiBlockID = (IMG_UINT32)(ui32BlockID & ~(RGX_CNTBLK_ID_UNIT_ALL_MASK|RGX_CNTBLK_ID_DA_MASK));
+#else
+	IMG_UINT32 uiBlockID = (IMG_UINT32)(ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK);
+#endif
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	/* Is it a direct block? */
+	if (uiBlockID < RGX_CNTBLK_ID_DIRECT_LAST)
+	{
+		ui32TableIdx = uiBlockID;
+	}
+	/* Is it an indirect block */
+	else if ((uiBlockID > RGX_CNTBLK_ID_DIRECT_LAST) && (uiBlockID < RGX_CNTBLK_ID_LAST))
+	{
+		ui32TableIdx = RGX_CNTBLK_ID_DIRECT_LAST + (((uiBlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT) - 1U);
+	}
+	/* Unknown mapping from CNTBLK_ID to Table index */
+	else
+	{
+		return NULL;
+	}
+
+	PVR_ASSERT(ui32TableIdx < ui32CntBlkModelLen);
+
+	if (psDevInfo == NULL)
+	{
+		PVR_LOG(("psDevInfo invalid"));
+		return NULL;
+	}
+
+	if ((ui32TableIdx < ui32CntBlkModelLen) &&
+		(asCntBlkTypeModel[ui32TableIdx].pfnIsBlkPresent(&asCntBlkTypeModel[ui32TableIdx], psDevInfo, &sRtInfo) != IMG_FALSE))
+	{
+		return &asCntBlkTypeModel[ui32TableIdx];
+	}
+
+	/* Fall through, block not valid from run-time validation */
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(CONNECTION_DATA *psConnection,
+                                                       PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                       const IMG_UINT32 ui32BlockID,
+                                                       RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXAcquireHWPerfCtlCPUAddr");
+
+	eError = PVRSRVRGXGetConfiguredHWPerfMuxCounters(psDeviceNode,
+	                                                 psHWPerfCtl,
+	                                                 ui32BlockID,
+	                                                 psConfiguredMuxCounters);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRGXGetConfiguredHWPerfMuxCounters");
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                     RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                     IMG_UINT32 ui32BlockID,
+                                                     RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredMuxCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) < RGX_CNTBLK_ID_LAST)
+	{
+		RGXFWIF_HWPERF_CTL_BLK *psBlock = rgxfw_hwperf_get_block_ctl(ui32BlockID, psHWPerfCtl);
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *psBlkTypeDesc;
+		IMG_UINT32 i, ui32LastCountIdx = 0, ui8CurCountIdx = 0;
+		RGX_HWPERF_CONFIG_MUX_CNTBLK sBlockConfig;
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		if (!psBlock->ui32Enabled || !psBlock->ui32Valid)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not %s",
+			         ui32BlockID,
+			         !psBlock->ui32Enabled ? "enabled." : "configured."));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		psBlkTypeDesc = GetHWPerfBlockTypeByID(psDevInfo, psBlock->eBlockID);
+		if (psBlkTypeDesc == NULL)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->eBlockID;
+		sBlockConfig.ui8Mode = 0;
+
+		for (i = 0; ((psBlock->uiCounterMask >> i) != 0) &&
+		     (ui8CurCountIdx < psBlkTypeDesc->ui8NumCounters); i++)
+		{
+			if (psBlock->uiCounterMask & (1 << i))
+			{
+				IMG_UINT8 ui8Mode = 0;
+
+				ui8Mode = (psBlock->aui64CounterCfg[i] >> psBlkTypeDesc->ui8SelectRegModeShift) & 1U;
+				sBlockConfig.ui8Mode |= ui8Mode << ui32LastCountIdx;
+
+				sBlockConfig.aui8GroupSelect[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_SHIFT) & 0x1F;
+
+				sBlockConfig.aui16BitSelect[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BIT_SELECT_SHIFT) & 0x7FFF;
+
+#if defined(RGX_FEATURE_PERF_COUNTER_BATCH)
+				sBlockConfig.aui32BatchMax[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BATCH_MAX_SHIFT) & 0x1FFF;
+
+				sBlockConfig.aui32BatchMin[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BATCH_MIN_SHIFT) & 0x1FFF;
+#endif
+				ui32LastCountIdx++;
+				ui8CurCountIdx++;
+			}
+		}
+
+		sBlockConfig.ui8CounterSelect = (1 << ui32LastCountIdx) - 1;
+		*psConfiguredMuxCounters = sBlockConfig;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	RGX_HWPERF_CONFIG_CNTBLK sBlockConfig;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & RGX_CNTBLK_ID_CUSTOM_MASK) >= RGX_CNTBLK_ID_LAST)
+	{
+		/* Validate block ID */
+		switch (ui32BlockID)
+		{
+			case RGX_CNTBLK_ID_CUSTOM0:
+			case RGX_CNTBLK_ID_CUSTOM1:
+			case RGX_CNTBLK_ID_CUSTOM2:
+			case RGX_CNTBLK_ID_CUSTOM3:
+			case RGX_CNTBLK_ID_CUSTOM4_FW:
+			{
+				PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+				/* Check to see if this block is enabled */
+				if (psHWPerfCtl->ui32SelectedCountersBlockMask & (1 << (ui32BlockID & 0x0F)))
+				{
+					RGXFW_HWPERF_SELECT *psBlock = &psHWPerfCtl->SelCntr[ui32BlockID & 0x0F];
+
+					sBlockConfig.ui16BlockID = ui32BlockID;
+					sBlockConfig.ui16NumCounters = psBlock->ui32NumSelectedCounters;
+
+					for (i = 0; i < psBlock->ui32NumSelectedCounters; i++)
+					{
+						sBlockConfig.ui16Counters[i] = psBlock->aui32SelectedCountersIDs[i];
+					}
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+					PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+				}
+				break;
+			}
+			default:
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+			}
+		}
+	}
+#if defined(HWPERF_UNIFIED)
+	else if ((ui32BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+	{
+		RGXFWIF_HWPERF_DA_BLK *psBlock = rgxfw_hwperf_get_da_block_ctl(ui32BlockID, psHWPerfCtl);
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		if (!psBlock->uiEnabled)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->eBlockID;
+		sBlockConfig.ui16NumCounters = psBlock->uiNumCounters;
+
+		for (i = 0; i < psBlock->uiNumCounters; i++)
+		{
+			sBlockConfig.ui16Counters[i] = psBlock->aui32Counters[i];
+		}
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	if (eError == PVRSRV_OK)
+	{
+		*psConfiguredCounters = sBlockConfig;
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLen,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	IMG_UINT32 ui32LastIdx = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(pui32BlockCount != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	*pui32BlockCount = 0;
+
+	if (ui32ArrayLen > 0 && pui32EnabledBlockIDs == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen is greater than 0 but pui32EnabledBlockIDs is NULL"));
+	}
+
+	PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+	for (i = 0; i < RGX_HWPERF_MAX_MUX_BLKS; i++)
+	{
+		if (psHWPerfCtl->sBlkCfg[i].ui32Enabled && psHWPerfCtl->sBlkCfg[i].ui32Valid)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx + 1 > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sBlkCfg[i].eBlockID;
+			ui32LastIdx++;
+		}
+	}
+
+	for (i = 0; i < RGX_HWPERF_MAX_CUSTOM_BLKS; i++)
+	{
+		if (psHWPerfCtl->ui32SelectedCountersBlockMask == 0)
+		{
+			break;
+		}
+
+		if (psHWPerfCtl->ui32SelectedCountersBlockMask & (1 << i))
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx + 1 > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = RGX_CNTBLK_ID_CUSTOM0 + i;
+			ui32LastIdx++;
+		}
+	}
+
+#if defined(HWPERF_UNIFIED)
+	for (i = 0; i < RGX_HWPERF_MAX_DA_BLKS; i++)
+	{
+		if (psHWPerfCtl->sDABlkCfg[i].uiEnabled)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sDABlkCfg[i].eBlockID;
+			ui32LastIdx++;
+		}
+	}
+#endif
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxinit.c
new file mode 100644
index 000000000000..d6096e6a9681
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxinit.c
@@ -0,0 +1,5608 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_bridge_init.h"
+#include "rgx_bridge_init.h"
+#include "syscommon.h"
+#include "rgx_heaps_server.h"
+#include "rgxheapconfig.h"
+#include "rgxpower.h"
+#include "tlstream.h"
+#include "pvrsrv_tlstreams.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "pvr_ricommon.h"
+#endif
+
+#include "rgxinit.h"
+#include "rgxbvnc.h"
+#include "rgxmulticore.h"
+
+#include "pdump_km.h"
+#include "handle.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxmem.h"
+#include "sync_internal.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "rgxfwdbg.h"
+#include "info_page.h"
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxfwimageutils.h"
+#endif
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+#include "rgxmmuinit.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#include "physmem.h"
+#endif
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "rgxta3d.h"
+#include "rgxtimecorr.h"
+#include "rgxshader.h"
+
+#if defined(PDUMP)
+#include "rgxstartstop.h"
+#endif
+
+#include "rgx_fwif_alignchecks.h"
+#include "vmm_pvz_client.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+
+#if defined(PDUMP) && defined(SUPPORT_SECURITY_VALIDATION)
+#include "pdump_physmem.h"
+#endif
+
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_CHAR **ppszVersionString);
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PUINT32 pui32RGXClockSpeed);
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID);
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/* Services internal heap identification used in this file only */
+#define RGX_FIRMWARE_MAIN_HEAP_IDENT   "FwMain"   /*!< RGX Main Firmware Heap identifier */
+#define RGX_FIRMWARE_CONFIG_HEAP_IDENT "FwConfig" /*!< RGX Config firmware Heap identifier */
+
+#define RGX_MMU_PAGE_SIZE_4KB   (   4 * 1024)
+#define RGX_MMU_PAGE_SIZE_16KB  (  16 * 1024)
+#define RGX_MMU_PAGE_SIZE_64KB  (  64 * 1024)
+#define RGX_MMU_PAGE_SIZE_256KB ( 256 * 1024)
+#define RGX_MMU_PAGE_SIZE_1MB   (1024 * 1024)
+#define RGX_MMU_PAGE_SIZE_2MB   (2048 * 1024)
+#define RGX_MMU_PAGE_SIZE_MIN RGX_MMU_PAGE_SIZE_4KB
+#define RGX_MMU_PAGE_SIZE_MAX RGX_MMU_PAGE_SIZE_2MB
+
+#define VAR(x) #x
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@Function       SampleIRQCount
+@Description    Utility function taking snapshots of RGX FW interrupt count.
+@Input          psDevInfo    Device Info structure
+
+@Return         IMG_BOOL     Returns IMG_TRUE if RGX FW IRQ is not equal to
+                             sampled RGX FW IRQ count for any RGX FW thread.
+ */ /**************************************************************************/
+static INLINE IMG_BOOL SampleIRQCount(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bReturnVal = IMG_FALSE;
+	volatile IMG_UINT32 *pui32SampleIrqCount = psDevInfo->aui32SampleIRQCount;
+	IMG_UINT32 ui32IrqCnt;
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bReturnVal = IMG_TRUE;
+	}
+	else
+	{
+		get_irq_cnt_val(ui32IrqCnt, RGXFW_HOST_DRIVER_ID, psDevInfo);
+
+		if (ui32IrqCnt != pui32SampleIrqCount[RGXFW_THREAD_0])
+		{
+			pui32SampleIrqCount[RGXFW_THREAD_0] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#else
+	IMG_UINT32 ui32TID;
+
+	for_each_irq_cnt(ui32TID)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32TID, psDevInfo);
+
+		/* treat unhandled interrupts here to align host count with fw count */
+		if (pui32SampleIrqCount[ui32TID] != ui32IrqCnt)
+		{
+			pui32SampleIrqCount[ui32TID] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#endif
+
+	return bReturnVal;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHostSafetyEvents
+@Description    Returns the event status masked to keep only the safety
+                events handled by the Host
+@Input          psDevInfo    Device Info structure
+@Return         IMG_UINT32   Status of Host-handled safety events
+ */ /**************************************************************************/
+static INLINE IMG_UINT32 RGXHostSafetyEvents(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (psDevInfo->ui32HostSafetyEventMask == 0))
+	{
+		return 0;
+	}
+	else
+	{
+		IMG_UINT32 ui32SafetyEventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE);
+		return (ui32SafetyEventStatus & psDevInfo->ui32HostSafetyEventMask);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventCheck
+@Description    Clears the Event Status register and checks if any of the
+                safety events need Host handling
+@Input          psDevInfo    Device Info structure
+@Return         IMG_BOOL     Are there any safety events for Host to handle ?
+ */ /**************************************************************************/
+static INLINE IMG_BOOL RGXSafetyEventCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bSafetyEvent = IMG_FALSE;
+
+	if (psDevInfo->ui32HostSafetyEventMask != 0)
+	{
+		IMG_UINT32 ui32EventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_STATUS);
+
+		if (BIT_ISSET(ui32EventStatus, RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_SHIFT))
+		{
+			/* clear the safety event */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_CLEAR, RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_EN);
+
+			/* report if there is anything for the Host to handle */
+			bSafetyEvent = (RGXHostSafetyEvents(psDevInfo) != 0);
+		}
+	}
+
+	return bSafetyEvent;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventHandler
+@Description    Handles the Safety Events that the Host is responsible for
+@Input          psDevInfo    Device Info structure
+ */ /**************************************************************************/
+static void RGXSafetyEventHandler(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	if (psDevInfo->bRGXPowered)
+	{
+		RGX_CONTEXT_RESET_REASON eResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+		IMG_UINT32 ui32HostSafetyStatus = RGXHostSafetyEvents(psDevInfo);
+
+		if (ui32HostSafetyStatus != 0)
+		{
+			/* clear the safety bus events handled by the Host */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE, ui32HostSafetyStatus);
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_SHIFT))
+			{
+				IMG_UINT32 ui32FaultFlag;
+				IMG_UINT32 ui32FaultFW = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_STATUS);
+				IMG_UINT32 ui32CorrectedBitOffset = RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_SHIFT -
+													RGX_CR_FAULT_FW_STATUS_CPU_DETECT_SHIFT;
+
+				PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety fault status: 0x%X", __func__, ui32FaultFW));
+
+				for (ui32FaultFlag = 0; ui32FaultFlag < ui32CorrectedBitOffset; ui32FaultFlag++)
+				{
+					if (BIT_ISSET(ui32FaultFW, ui32FaultFlag))
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety hardware fault detected (0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_ERR;
+					}
+					else if BIT_ISSET(ui32FaultFW, ui32FaultFlag + ui32CorrectedBitOffset)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Firmware safety hardware fault corrected.(0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+
+						/* Only report this if we haven't detected a more serious error */
+						if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR)
+						{
+							eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_OK;
+						}
+					}
+				}
+
+				OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_CLEAR, ui32FaultFW);
+			}
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT))
+			{
+				volatile RGXFWIF_POW_STATE ePowState;
+
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+										   INVALIDATE);
+				ePowState = psDevInfo->psRGXFWIfFwSysData->ePowState;
+
+				if (ePowState != RGXFWIF_POW_OFF)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Safety Watchdog Trigger !", __func__));
+
+					/* Only report this if we haven't detected a more serious error */
+					if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR)
+					{
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_WATCHDOG;
+					}
+				}
+			}
+
+			/* Notify client and system layer of any error */
+			if (eResetReason != RGX_CONTEXT_RESET_REASON_NONE)
+			{
+				PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+				PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = eResetReason;
+
+				/* Notify system layer of any error */
+				if (psDevConfig->pfnSysDevErrorNotify)
+				{
+					PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+					sErrorData.eResetReason = eResetReason;
+
+					psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+													  &sErrorData);
+				}
+			}
+		}
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+static IMG_BOOL _WaitForInterruptsTimeoutCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32idx;
+#endif
+
+	RGXDEBUG_PRINT_IRQ_COUNT(psDevInfo);
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVR_DPF((PVR_DBG_ERROR,
+	        "Last RGX_LISRHandler State (DevID %u): 0x%08X Clock: %" IMG_UINT64_FMTSPEC,
+			 psDeviceNode->sDevId.ui32InternalID,
+			 psDeviceNode->sLISRExecutionInfo.ui32Status,
+			 psDeviceNode->sLISRExecutionInfo.ui64Clockns));
+
+	for_each_irq_cnt(ui32idx)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				MSG_IRQ_CNT_TYPE " %u: InterruptCountSnapshot: 0x%X",
+				ui32idx, psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[ui32idx]));
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR, "No further information available. Please enable PVRSRV_DEBUG_LISR_EXECUTION"));
+#endif
+
+	return SampleIRQCount(psDevInfo);
+}
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bScheduleMISR;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bScheduleMISR = IMG_TRUE;
+	}
+	else
+	{
+		bScheduleMISR = _WaitForInterruptsTimeoutCheck(psDevInfo);
+	}
+
+	if (bScheduleMISR)
+	{
+		OSScheduleMISR(psDevInfo->pvMISRData);
+
+		if (psDevInfo->pvAPMISRData != NULL)
+		{
+			OSScheduleMISR(psDevInfo->pvAPMISRData);
+		}
+	}
+}
+
+static inline IMG_BOOL RGXAckHwIrq(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_UINT32 ui32IRQStatusReg,
+								   IMG_UINT32 ui32IRQStatusEventMsk,
+								   IMG_UINT32 ui32IRQClearReg,
+								   IMG_UINT32 ui32IRQClearMask)
+{
+	IMG_UINT32 ui32IRQStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQStatusReg);
+
+	/* clear only the pending bit of the thread that triggered this interrupt */
+	ui32IRQClearMask &= ui32IRQStatus;
+
+	if (ui32IRQStatus & ui32IRQStatusEventMsk)
+	{
+		/* acknowledge and clear the interrupt */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQClearReg, ui32IRQClearMask);
+		return IMG_TRUE;
+	}
+	else
+	{
+		/* spurious interrupt */
+		return IMG_FALSE;
+	}
+}
+
+static __maybe_unused IMG_BOOL RGXAckIrqMETA(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static IMG_BOOL RGXAckIrqMIPS(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN);
+}
+#endif
+
+static IMG_BOOL RGXAckIrqDedicated(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+		/* status & clearing registers are available on both Host and Guests
+		 * and are agnostic of the Fw CPU type. Due to the remappings done by
+		 * the 2nd stage device MMU, all drivers assume they are accessing
+		 * register bank 0 */
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_IRQ_OS0_EVENT_STATUS,
+					   ~RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK,
+					   RGX_CR_IRQ_OS0_EVENT_CLEAR,
+					   ~RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK);
+}
+
+static PVRSRV_ERROR RGXSetAckIrq(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, IRQ_PER_OS)) && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo)))
+	{
+		psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psDevInfo->pfnRGXAckIrq = NULL;
+	}
+	else
+	{
+		/* native and host drivers must clear the unique GPU physical interrupt */
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqMIPS;
+		}
+		else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqMETA;
+		}
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU IRQ clearing mechanism not implemented "
+									"for this architecture.", __func__));
+			return PVRSRV_ERROR_NOT_IMPLEMENTED;
+		}
+	}
+
+#if defined(RGX_IRQ_HYPERV_HANDLER)
+	/* The hypervisor receives and acknowledges the GPU irq, then it injects an
+	 * irq only in the recipient OS. The KM driver doesn't handle the GPU irq line */
+	psDevInfo->pfnRGXAckIrq = NULL;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL RGX_LISRHandler(void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bIrqAcknowledged = IMG_FALSE;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	IMG_UINT32 ui32idx, ui32IrqCnt;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+		UPDATE_LISR_DBG_SNAPSHOT(ui32idx, ui32IrqCnt);
+	}
+
+	UPDATE_LISR_DBG_STATUS(RGX_LISR_INIT);
+	UPDATE_LISR_DBG_TIMESTAMP();
+#endif
+
+	UPDATE_LISR_DBG_COUNTER();
+
+	if (psDevInfo->bRGXPowered)
+	{
+		IMG_BOOL bSafetyEvent = RGXSafetyEventCheck(psDevInfo);
+
+		if ((psDevInfo->pfnRGXAckIrq == NULL) || psDevInfo->pfnRGXAckIrq(psDevInfo) || bSafetyEvent)
+		{
+			bIrqAcknowledged = IMG_TRUE;
+
+			if (bSafetyEvent || SampleIRQCount(psDevInfo))
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_PROCESSED);
+				UPDATE_MISR_DBG_COUNTER();
+
+				OSScheduleMISR(psDevInfo->pvMISRData);
+
+#if defined(SUPPORT_AUTOVZ)
+				RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+				if (psDevInfo->pvAPMISRData != NULL)
+				{
+					OSScheduleMISR(psDevInfo->pvAPMISRData);
+				}
+			}
+			else
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED);
+			}
+		}
+		else
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_NOT_TRIGGERED_BY_HW);
+		}
+	}
+	else
+	{
+#if defined(SUPPORT_AUTOVZ)
+		/* AutoVz drivers rebooting while the firmware is active must acknowledge
+		 * and clear the hw IRQ line before the RGXInit() has finished. */
+		if ((psDevInfo->pfnRGXAckIrq != NULL) &&
+			 psDevInfo->pfnRGXAckIrq(psDevInfo))
+		{
+			bIrqAcknowledged = IMG_TRUE;
+		}
+		else
+#endif
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_DEVICE_NOT_POWERED);
+		}
+	}
+
+	return bIrqAcknowledged;
+}
+
+static void RGX_MISR_ProcessKCCBDeferredList(PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* First check whether there are pending commands in Deferred KCCB List */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	if (dllist_is_empty(&psDevInfo->sKCCBDeferredCommandsListHead))
+	{
+		OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+		return;
+	}
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	/* Powerlock to avoid further Power transition requests
+	   while KCCB deferred list is being processed */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Try to send deferred KCCB commands Do not Poll from here*/
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s could not flush Deferred KCCB list, KCCB is full.",
+				 __func__));
+	}
+}
+
+static void RGX_MISRHandler_CheckFWActivePowerState(void *psDevice)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevice;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_ON || psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		/* The FW is IDLE and therefore could be shut down */
+		eError = RGXActivePowerRequest(psDeviceNode);
+
+		if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED))
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+					"%s: Failed RGXActivePowerRequest call (device: %p) with %s",
+					__func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+			else
+			{
+				/* Re-schedule the power down request as it was deferred. */
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+	}
+
+}
+
+/* Shorter defines to keep the code a bit shorter */
+#define GPU_IDLE       RGXFWIF_GPU_UTIL_STATE_IDLE
+#define GPU_ACTIVE     RGXFWIF_GPU_UTIL_STATE_ACTIVE
+#define GPU_BLOCKED    RGXFWIF_GPU_UTIL_STATE_BLOCKED
+#define GPU_INACTIVE   RGXFWIF_GPU_UTIL_STATE_INACTIVE
+#define MAX_ITERATIONS 64
+#define MAX_DIFF_TIME_NS (300000ULL)
+#define MAX_DIFF_DM_TIME_NS (MAX_DIFF_TIME_NS >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT)
+
+static PVRSRV_ERROR RGXGetGpuUtilStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_HANDLE hGpuUtilUser,
+                                       RGXFWIF_GPU_UTIL_STATS *psReturnStats)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_STATS sStats;
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+	IMG_UINT64 (*paaui64DMOSTmpCounters)[RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 (*paui64DMOSTmpLastWord)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastState)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastPeriod)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastTime)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT64 ui64TimeNowShifted;
+	IMG_UINT32 ui32Attempts;
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32MaxDMCount;
+	RGXFWIF_DM eDM;
+
+	/***** (1) Initialise return stats *****/
+
+	psReturnStats->bValid = IMG_FALSE;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psAggregateStats = hGpuUtilUser;
+
+	/* decrease by 1 to account for excluding GP DM from the statics */;
+	ui32MaxDMCount =  psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1;
+
+	/* Reset temporary counters used in the attempts loop */
+	paaui64DMOSTmpCounters  = &psAggregateStats->sTempGpuStats.aaaui64DMOSTmpCounters[0];
+	paui64DMOSTmpLastWord   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastWord[0];
+	paui64DMOSTmpLastState  = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastState[0];
+	paui64DMOSTmpLastPeriod = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastPeriod[0];
+	paui64DMOSTmpLastTime   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastTime[0];
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	/* Try to acquire GPU utilisation counters and repeat if the FW is in the middle of an update */
+	for (ui32Attempts = 0; ui32Attempts < 4; ui32Attempts++)
+	{
+		IMG_UINT64 aui64GpuTmpCounters[RGXFWIF_GPU_UTIL_STATE_NUM] = {0};
+		IMG_UINT64 ui64GpuLastPeriod = 0, ui64GpuLastWord = 0, ui64GpuLastState = 0, ui64GpuLastTime = 0;
+
+		/***** (2) Get latest data from shared area *****/
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			IMG_UINT64 aui64StatsCountersNew[RGXFWIF_GPU_UTIL_STATE_NUM];
+			IMG_UINT64 ui64GpuLastWordNew;
+			RGXFWIF_GPU_STATS sStatsNew;
+			IMG_UINT32 i = 0;
+
+			ui64GpuLastWord = 0;
+			ui64GpuLastState = 0;
+
+			OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+			/* Copy data from device memory */
+			memcpy(&sStatsNew, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+			memcpy(&ui64GpuLastWordNew, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+			memcpy(aui64StatsCountersNew, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64StatsCountersNew));
+
+			/*
+			 * First attempt at detecting if the FW is in the middle of an update.
+			 * This should also help if the FW is in the middle of a 64 bit variable update.
+			 * This loop must be fast. Faster than FW updates the stats.
+			 */
+			for (i = 0; i < MAX_ITERATIONS; i++)
+			{
+				IMG_UINT32 j,k;
+				IMG_BOOL bRetry = IMG_FALSE;
+
+				if (i > 0)
+				{
+					/* On retry keep previous data */
+					ui64GpuLastWordNew = ui64GpuLastWord;
+					memcpy(aui64StatsCountersNew, aui64GpuTmpCounters, sizeof(aui64StatsCountersNew));
+					memcpy(&sStatsNew, &sStats, sizeof(sStatsNew));
+				}
+
+				/* Copy data from device memory */
+				memcpy(&sStats, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+				memcpy(&ui64GpuLastWord, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+				memcpy(aui64GpuTmpCounters, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64GpuTmpCounters));
+
+				/* Check for abnormal time difference between reads */
+				if (RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord) - RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWordNew) > MAX_DIFF_TIME_NS)
+				{
+					bRetry = IMG_TRUE;
+					continue;
+				}
+
+				for (j = 0; j < RGXFWIF_GPU_UTIL_STATE_NUM; j++)
+				{
+					/* Check for abnormal time difference between reads */
+					if (aui64GpuTmpCounters[j] - aui64StatsCountersNew[j] > MAX_DIFF_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+				}
+
+				if (bRetry)
+				{
+					continue;
+				}
+
+				/* Check for DM counters wrapped or
+				   abnormal time difference between reads.
+				   The DM time is shifted by RGXFWIF_DM_OS_TIMESTAMP_SHIFT */
+				for (j = 0; j < RGXFWIF_GPU_UTIL_DM_MAX; j++)
+				{
+					if (sStats.aui32DMOSLastWordWrap[j] != sStatsNew.aui32DMOSLastWordWrap[j] ||
+						RGXFWIF_GPU_UTIL_GET_TIME32(sStats.aui32DMOSLastWord[j]) - RGXFWIF_GPU_UTIL_GET_TIME32(sStatsNew.aui32DMOSLastWord[j]) > MAX_DIFF_DM_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+
+					for (k = 0; k < RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM; k++)
+					{
+						if (sStats.aaui32DMOSCountersWrap[j][k] != sStatsNew.aaui32DMOSCountersWrap[j][k] ||
+							sStats.aaui32DMOSStatsCounters[j][k] - sStatsNew.aaui32DMOSStatsCounters[j][k] > MAX_DIFF_DM_TIME_NS)
+						{
+							bRetry = IMG_TRUE;
+							break;
+						}
+
+					}
+
+					if (bRetry)
+					{
+						break;
+					}
+				}
+
+				if (!bRetry)
+				{
+					/* Stats are good*/
+					break;
+				}
+			}
+
+			OSLockRelease(psDevInfo->hGPUUtilLock);
+
+			ui64GpuLastState = RGXFWIF_GPU_UTIL_GET_STATE(ui64GpuLastWord);
+
+			if (i == MAX_ITERATIONS)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						 "RGXGetGpuUtilStats could not get reliable data after trying %u times", i));
+
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+			{
+				paui64DMOSTmpLastWord[eDM][ui32DriverID]  =
+					((IMG_UINT64)sStats.aui32DMOSLastWordWrap[eDM] << 32) + sStats.aui32DMOSLastWord[eDM];
+				paui64DMOSTmpLastState[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_STATE(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				if (paui64DMOSTmpLastState[eDM][ui32DriverID] != GPU_ACTIVE)
+				{
+					paui64DMOSTmpLastState[eDM][ui32DriverID] = GPU_INACTIVE;
+				}
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE] = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_INACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_INACTIVE] << 32);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE]  = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_ACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_ACTIVE] << 32);
+			}
+
+		} /* FOREACH_SUPPORTED_DRIVER(ui32DriverID) */
+
+
+		/***** (3) Compute return stats *****/
+
+		/* Update temp counters to account for the time since the last update to the shared ones */
+		OSMemoryBarrier(NULL); /* Ensure the current time is read after the loop above */
+		ui64TimeNow    = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDeviceNode));
+
+		ui64GpuLastTime   = RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord);
+		ui64GpuLastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64GpuLastTime);
+		aui64GpuTmpCounters[ui64GpuLastState] += ui64GpuLastPeriod;
+
+		/* Get statistics for a user since its last request */
+		psReturnStats->ui64GpuStatIdle = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_IDLE],
+		                                                             psAggregateStats->ui64GpuStatIdle);
+		psReturnStats->ui64GpuStatActive = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_ACTIVE],
+		                                                               psAggregateStats->ui64GpuStatActive);
+		psReturnStats->ui64GpuStatBlocked = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_BLOCKED],
+		                                                                psAggregateStats->ui64GpuStatBlocked);
+		psReturnStats->ui64GpuStatCumulative = psReturnStats->ui64GpuStatIdle +
+		                                       psReturnStats->ui64GpuStatActive +
+		                                       psReturnStats->ui64GpuStatBlocked;
+
+		/* convert time into the same units as used by fw */
+		ui64TimeNowShifted  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+		for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+		{
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				paui64DMOSTmpLastTime[eDM][ui32DriverID]   = RGXFWIF_GPU_UTIL_GET_TIME(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				paui64DMOSTmpLastPeriod[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNowShifted , paui64DMOSTmpLastTime[eDM][ui32DriverID]);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][paui64DMOSTmpLastState[eDM][ui32DriverID]] += paui64DMOSTmpLastPeriod[eDM][ui32DriverID];
+				/* Get statistics for a user since its last request */
+				psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE],
+				                                                             psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE],
+				                                                               psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatCumulative[eDM][ui32DriverID] = psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] +
+				                                       psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+			}
+		}
+
+		if (psAggregateStats->ui64TimeStamp != 0)
+		{
+			IMG_UINT64 ui64TimeSinceLastCall = ui64TimeNow - psAggregateStats->ui64TimeStamp;
+			/* We expect to return at least 75% of the time since the last call in GPU stats */
+			IMG_UINT64 ui64MinReturnedStats = ui64TimeSinceLastCall - (ui64TimeSinceLastCall / 4);
+
+			/*
+			 * If the returned stats are substantially lower than the time since
+			 * the last call, then the Host might have read a partial update from the FW.
+			 * If this happens, try sampling the shared counters again.
+			 */
+			if (psReturnStats->ui64GpuStatCumulative < ui64MinReturnedStats)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s: Return stats (%" IMG_UINT64_FMTSPEC ") too low "
+				         "(call period %" IMG_UINT64_FMTSPEC ")",
+				         __func__, psReturnStats->ui64GpuStatCumulative, ui64TimeSinceLastCall));
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Attempt #%u has failed, trying again",
+				         __func__, ui32Attempts));
+				continue;
+			}
+		}
+
+		break;
+	}
+
+	/***** (4) Update aggregate stats for the current user *****/
+
+	psAggregateStats->ui64GpuStatIdle    += psReturnStats->ui64GpuStatIdle;
+	psAggregateStats->ui64GpuStatActive  += psReturnStats->ui64GpuStatActive;
+	psAggregateStats->ui64GpuStatBlocked += psReturnStats->ui64GpuStatBlocked;
+	psAggregateStats->ui64TimeStamp       = ui64TimeNow;
+
+	for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+	{
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]    += psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID];
+			psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]  += psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+		}
+	}
+
+	/***** (5) Convert return stats to microseconds *****/
+
+	psReturnStats->ui64GpuStatIdle       = OSDivide64(psReturnStats->ui64GpuStatIdle, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatActive     = OSDivide64(psReturnStats->ui64GpuStatActive, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatBlocked    = OSDivide64(psReturnStats->ui64GpuStatBlocked, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatCumulative = OSDivide64(psReturnStats->ui64GpuStatCumulative, 1000, &ui32Remainder);
+
+	/* Check that the return stats make sense */
+	if (psReturnStats->ui64GpuStatCumulative == 0)
+	{
+		/* We can enter here only if allocating the temporary stats
+		 * buffers failed, or all the RGXFWIF_GPU_UTIL_GET_PERIOD
+		 * returned 0. The latter could happen if the GPU frequency value
+		 * is not well calibrated and the FW is updating the GPU state
+		 * while the Host is reading it.
+		 * When such an event happens frequently, timers or the aggregate
+		 * stats might not be accurate...
+		 */
+#if defined(VIRTUAL_PLATFORM)
+		/* To avoid spamming the console logging system on emulated devices,
+		 * we special-case so that we will only produce a single message per
+		 * driver invocation. This should reduce the time spent logging
+		 * information which is not relevant for very slow timers found in
+		 * VP device configurations
+		 */
+		static IMG_BOOL bFirstTime = IMG_TRUE;
+
+		if (bFirstTime)
+		{
+			bFirstTime = IMG_FALSE;
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "RGXGetGpuUtilStats could not get reliable data."));
+#if defined(VIRTUAL_PLATFORM)
+		}
+#endif	/* defined(VIRTUAL_PLATFORM) */
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psReturnStats->bValid = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psAggregateStats = OSAllocZMemNoStats(sizeof(RGXFWIF_GPU_UTIL_STATS));
+	if (psAggregateStats == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phGpuUtilUser = psAggregateStats;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAggregateStats = hGpuUtilUser;
+	OSFreeMemNoStats(psAggregateStats);
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGX MISR Handler
+*/
+static void RGX_MISRHandler_Main (void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* Give the HWPerf service a chance to transfer some data from the FW
+	 * buffer to the host driver transport layer buffer.
+	 */
+	RGXHWPerfDataStoreCB(psDeviceNode);
+
+	/* Inform other services devices that we have finished an operation */
+	PVRSRVNotifyCommandCompletion(psDeviceNode);
+
+#if defined(SUPPORT_PDVFS) && defined(RGXFW_META_SUPPORT_2ND_THREAD)
+	/* Normally, firmware CCB only exists for the primary FW thread unless PDVFS
+	   is running on the second[ary] FW thread, here we process said CCB */
+	RGXPDVFSCheckCoreClkRateChange(psDeviceNode->pvDevice);
+#endif
+
+	/* Only execute SafetyEventHandler if RGX_FEATURE_SAFETY_EVENT is on */
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, ECC_RAMS) > 0 ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, WATCHDOG_TIMER))
+	{
+		/* Handle Safety events if necessary */
+		RGXSafetyEventHandler(psDeviceNode->pvDevice);
+	}
+
+	/* Signal the global event object */
+	PVRSRVSignalDriverWideEO();
+
+	/* Process the Firmware CCB for pending commands */
+	RGXCheckFirmwareCCB(psDeviceNode->pvDevice);
+
+	/* Calibrate the GPU frequency and recorrelate Host and GPU timers (done every few seconds) */
+	RGXTimeCorrRestartPeriodic(psDeviceNode);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Process Workload Estimation Specific commands from the FW */
+		WorkEstCheckFirmwareCCB(psDeviceNode->pvDevice);
+	}
+#endif
+
+	if (psDevInfo->pvAPMISRData == NULL)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+static PVRSRV_ERROR RGXPDumpBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+		PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psFWDataPMR;
+	RGXMIPSFW_BOOT_DATA *psBootData;
+	IMG_DEV_PHYADDR sTmpAddr;
+	IMG_UINT32 ui32BootConfOffset, ui32ParamOffset, i;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+	ui32BootConfOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+	ui32BootConfOffset += RGXMIPSFW_BOOTLDR_CONF_OFFSET;
+
+	/* The physical addresses used by a pdump player will be different
+	 * than the ones we have put in the MIPS bootloader configuration data.
+	 * We have to tell the pdump player to replace the original values with the real ones.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Pass new boot parameters to the FW");
+
+	/* Rogue Registers physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64RegBase);
+
+	eError = PDumpRegLabelToMem64(RGX_PDUMPREG_NAME,
+			0x0,
+			psFWDataPMR,
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of Rogue registers phy address failed (%u)", eError));
+		return eError;
+	}
+
+	/* Page Table physical Address */
+	eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sTmpAddr);
+	if (eError !=  PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXBootldrDataInit: MMU_AcquireBaseAddr failed (%u)",
+		         eError));
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+									 (void **)&psBootData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire pointer to FW data (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	psBootData = IMG_OFFSET_ADDR(psBootData, ui32BootConfOffset);
+
+	for (i = 0; i < psBootData->ui32PTNumPages; i++)
+	{
+		ui32ParamOffset = ui32BootConfOffset +
+			offsetof(RGXMIPSFW_BOOT_DATA, aui64PTPhyAddr[0])
+			+ i * sizeof(psBootData->aui64PTPhyAddr[0]);
+
+		eError = PDumpPTBaseObjectToMem64(psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				psFWDataPMR,
+				0,
+				ui32ParamOffset,
+				PDUMP_FLAGS_CONTINUOUS,
+				MMU_LEVEL_1,
+				sTmpAddr.uiAddr,
+				i << psBootData->ui32PTLog2PageSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of page tables phy address failed (%u)", eError));
+			return eError;
+		}
+	}
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+
+	/* Stack physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64StackPhyAddr);
+
+	eError = PDumpMemLabelToMem64(psFWDataPMR,
+			psFWDataPMR,
+			RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of stack phy address failed (%u)", eError));
+		return eError;
+	}
+
+	return eError;
+}
+#endif /* PDUMP */
+
+static PVRSRV_ERROR RGXSetPowerParams(PVRSRV_RGXDEV_INFO   *psDevInfo,
+                                      PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR sKernelMMUCtxPCAddr;
+	IMG_BOOL bPremappedFw;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVCFG, psDevConfig, PVRSRV_OK);
+
+	/* Save information used on power transitions for later
+	 * (when RGXStart and RGXStop are executed)
+	 */
+#if defined(PDUMP)
+	psDevInfo->sLayerParams.ui32PdumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	/* Rogue drivers with security support and premapped fw heaps
+	 * always have their fw heap premapped by the TEE */
+	bPremappedFw = IMG_TRUE;
+#else
+	/* If AutoVz firmware is up at this stage, the driver initialised it
+	 * during a previous life-cycle. The firmware's memory is already pre-mapped
+	 * and the MMU page tables reside in the predetermined memory carveout.
+	 * The Kernel MMU Context created in this life-cycle is a dummy structure
+	 * that is not used for mapping.
+	 * To program the Device's BIF with the correct PC address, use the base
+	 * address of the carveout reserved for MMU mappings as Kernel MMU PC Address */
+	bPremappedFw = psDevInfo->psDeviceNode->bAutoVzFwIsUp;
+#endif
+
+	if (bPremappedFw)
+	{
+		IMG_DEV_PHYADDR sDevPAddr;
+		PHYS_HEAP *psFwPageTableHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_RETURN_IF_FALSE((NULL != psFwPageTableHeap),
+								"Firmware Page Table heap not defined.",
+								PVRSRV_ERROR_INVALID_HEAP);
+
+		PhysHeapGetDevPAddr(psFwPageTableHeap, &sDevPAddr);
+		sKernelMMUCtxPCAddr.uiAddr = sDevPAddr.uiAddr;
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx,
+		                             &sKernelMMUCtxPCAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire Kernel MMU Ctx page catalog"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		PMR *psFWCodePMR = (PMR *)(psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR);
+		PMR *psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+		IMG_DEV_PHYADDR sPhyAddr;
+		IMG_BOOL bValid;
+
+#if defined(SUPPORT_ALT_REGBASE)
+		psDevInfo->sLayerParams.sGPURegAddr = psDevConfig->sAltRegsGpuPBase;
+#else
+		/* The physical address of the GPU registers needs to be translated
+		 * in case we are in a LMA scenario
+		 */
+		PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+				1,
+				&sPhyAddr,
+				&(psDevConfig->sRegsCpuPBase));
+
+		psDevInfo->sLayerParams.sGPURegAddr = sPhyAddr;
+#endif
+
+		/* Register bank must be aligned to 512KB (as per the core integration) to
+		 * prevent the FW accessing incorrect registers */
+		if ((psDevInfo->sLayerParams.sGPURegAddr.uiAddr & 0x7FFFFU) != 0U)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Register bank must be aligned to 512KB, but current address (0x%016"IMG_UINT64_FMTSPECX") is not",
+						psDevInfo->sLayerParams.sGPURegAddr.uiAddr));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE),
+				OSGetPageShift(), /* FW will be using the same page size as the OS */
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI code address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sBootRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWDataPMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI data address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sDataRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW exceptions address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sCodeRemapAddr = sPhyAddr;
+
+		psDevInfo->sLayerParams.sTrampolineRemapAddr.uiAddr = psDevInfo->psTrampoline->sPhysAddr.uiAddr;
+
+		psDevInfo->sLayerParams.bDevicePA0IsValid = psDevConfig->bDevicePA0IsValid;
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	/* Send information used on power transitions to the trusted device as
+	 * in this setup the driver cannot start/stop the GPU and perform resets
+	 */
+	if (psDevConfig->pfnTDSetPowerParams)
+	{
+		PVRSRV_TD_POWER_PARAMS sTDPowerParams;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+		    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			sTDPowerParams.sPCAddr = psDevInfo->sLayerParams.sPCAddr;
+		}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			sTDPowerParams.sGPURegAddr    = psDevInfo->sLayerParams.sGPURegAddr;
+			sTDPowerParams.sBootRemapAddr = psDevInfo->sLayerParams.sBootRemapAddr;
+			sTDPowerParams.sCodeRemapAddr = psDevInfo->sLayerParams.sCodeRemapAddr;
+			sTDPowerParams.sDataRemapAddr = psDevInfo->sLayerParams.sDataRemapAddr;
+		}
+#endif
+
+		eError = psDevConfig->pfnTDSetPowerParams(psDevConfig->hSysData,
+												  &sTDPowerParams);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: TDSetPowerParams not implemented!"));
+		eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	return eError;
+}
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*
+	RGXSystemGetFabricCoherency
+*/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode)
+{
+	IMG_CHAR *aszLabels[] = {"none", "acelite", "fullace", "unknown"};
+	PVRSRV_DEVICE_SNOOP_MODE eAppHintCacheSnoopingMode;
+	PVRSRV_DEVICE_SNOOP_MODE eDeviceCacheSnoopingMode;
+	IMG_UINT32 ui32AppHintFabricCoherency;
+	IMG_UINT32 ui32DeviceFabricCoherency;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+#if !defined(NO_HARDWARE)
+	void *pvRegsBaseKM;
+	IMG_BOOL bPowerDown = IMG_TRUE;
+	PVRSRV_ERROR eError;
+#endif
+
+	if (!sRegsCpuPBase.uiAddr || !ui32RegsSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Invalid RGX register base/size parameters"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(NO_HARDWARE)
+	pvRegsBaseKM = OSMapPhysToLin(sRegsCpuPBase, ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvRegsBaseKM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Failed to create RGX register mapping"));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	bPowerDown = ! PVRSRVIsSystemPowered(psDevConfig->psDevNode);
+
+	/* Power-up the device as required to read the registers */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+	}
+
+	/* AXI support within the SoC, bitfield COHERENCY_SUPPORT [1 .. 0]
+		value NO_COHERENCY        0x0 {SoC does not support any form of Coherency}
+		value ACE_LITE_COHERENCY  0x1 {SoC supports ACE-Lite or I/O Coherency}
+		value FULL_ACE_COHERENCY  0x2 {SoC supports full ACE or 2-Way Coherency} */
+	ui32DeviceFabricCoherency = OSReadHWReg32((void __iomem *)pvRegsBaseKM, RGX_CR_SOC_AXI);
+	PVR_LOG(("AXI fabric coherency (RGX_CR_SOC_AXI): 0x%x", ui32DeviceFabricCoherency));
+#if defined(DEBUG)
+	if (ui32DeviceFabricCoherency & ~((IMG_UINT32)RGX_CR_SOC_AXI_MASKFULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid RGX_CR_SOC_AXI value.", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+	ui32DeviceFabricCoherency &= ~((IMG_UINT32)RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK);
+	ui32DeviceFabricCoherency >>= RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+	}
+
+	/* UnMap Regs */
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32RegsSize);
+
+	switch (ui32DeviceFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_FULLACE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+	default:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_NONE;
+		break;
+	}
+#else /* !defined(NO_HARDWARE) */
+    *peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+    eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+    ui32DeviceFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY;
+#endif /* !defined(NO_HARDWARE) */
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, FabricCoherencyOverride,
+						 &ui32AppHintDefault, &ui32AppHintFabricCoherency);
+	OSFreeAppHintState(pvAppHintState);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	/* Temporarily disable coherency */
+	ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY;
+#endif
+
+	/* Suppress invalid AppHint value */
+	switch (ui32AppHintFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"Invalid FabricCoherencyOverride AppHint %d, ignoring",
+				ui32AppHintFabricCoherency));
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+		break;
+	}
+
+	if (ui32AppHintFabricCoherency < ui32DeviceFabricCoherency)
+	{
+		PVR_LOG(("Downgrading device fabric coherency from %s to %s",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+		eDeviceCacheSnoopingMode = eAppHintCacheSnoopingMode;
+	}
+	else if (ui32AppHintFabricCoherency > ui32DeviceFabricCoherency)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Cannot upgrade device fabric coherency from %s to %s, not supported by device!",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+
+		/* Override requested-for app-hint with actual app-hint value being used */
+		ui32AppHintFabricCoherency = ui32DeviceFabricCoherency;
+	}
+
+	*peCacheSnoopingMode = eDeviceCacheSnoopingMode;
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXSystemHasFBCDCVersion31
+*/
+static IMG_BOOL RGXSystemHasFBCDCVersion31(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(HW_ERN_66622_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 66622))
+#endif
+	{
+		{
+			if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+			{
+				return IMG_TRUE;
+			}
+		}
+	}
+#if defined(HW_ERN_66622_BIT_MASK)
+	else
+	{
+
+#if !defined(NO_HARDWARE)
+		if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: System uses FBCDC3.1 but GPU doesn't support it!",
+			         __func__));
+		}
+#endif
+	}
+#endif /* defined(HW_ERN_66622_BIT_MASK) */
+
+	return IMG_FALSE;
+}
+
+/*
+	RGXGetTFBCLossyGroup
+*/
+static IMG_UINT32 RGXGetTFBCLossyGroup(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	return psDevInfo->ui32TFBCLossyGroup;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return 0;
+#endif
+}
+
+/*
+	RGXDevMMUAttributes
+*/
+static MMU_DEVICEATTRIBS *RGXDevMMUAttributes(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              IMG_BOOL bKernelFWMemoryCtx)
+{
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = NULL;
+
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			psMMUDevAttrs = bKernelFWMemoryCtx ?
+			                psDeviceNode->psFirmwareMMUDevAttrs :
+			                psDeviceNode->psMMUDevAttrs;
+		}
+		else
+#endif
+		{
+			PVR_UNREFERENCED_PARAMETER(bKernelFWMemoryCtx);
+			psMMUDevAttrs = psDeviceNode->psMMUDevAttrs;
+		}
+	}
+
+	return psMMUDevAttrs;
+}
+
+/*
+	RGXDevSnoopMode
+*/
+static PVRSRV_DEVICE_SNOOP_MODE RGXDevSnoopMode(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+	{
+		return PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+	}
+
+	return PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+/*
+ * RGXInitDevPart2
+ */
+PVRSRV_ERROR RGXInitDevPart2(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                             RGX_INIT_APPHINTS	*psApphints)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEV_POWER_STATE	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32			ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32			ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 2");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		RGXPDumpBootldrData(psDeviceNode, psDevInfo);
+	}
+#endif
+#if defined(TIMING) || defined(DEBUG)
+	OSUserModeAccessToPerfCountersEn();
+#endif
+
+	/* Initialise Device Flags */
+	psDevInfo->ui32DeviceFlags = 0;
+	RGXSetDeviceFlags(psDevInfo, psApphints->ui32DeviceFlags, IMG_TRUE);
+
+	/* Allocate DVFS Table (needs to be allocated before GPU trace events
+	 *  component is initialised because there is a dependency between them) */
+	psDevInfo->psGpuDVFSTable = OSAllocZMem(sizeof(*(psDevInfo->psGpuDVFSTable)));
+	PVR_LOG_GOTO_IF_NOMEM(psDevInfo->psGpuDVFSTable, eError, ErrorExit);
+
+	if (psDevInfo->ui32HWPerfHostFilter == 0)
+	{
+		RGXHWPerfHostSetEventFilter(psDevInfo, psApphints->ui32HWPerfHostFilter);
+	}
+
+	/* If HWPerf enabled allocate all resources for the host side buffer. */
+	if (psDevInfo->ui32HWPerfHostFilter != 0)
+	{
+		if (RGXHWPerfHostInitOnDemandResources(psDevInfo) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "HWPerfHost buffer on demand"
+			        " initialisation failed."));
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise work estimation lock */
+		eError = OSLockCreate(&psDevInfo->hWorkEstLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(WorkEstLock)", ErrorExit);
+	}
+#endif
+
+	/* Initialise lists of ZSBuffers */
+	eError = OSLockCreate(&psDevInfo->hLockZSBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockZSBuffer)", ErrorExit);
+	dllist_init(&psDevInfo->sZSBufferHead);
+	psDevInfo->ui32ZSBufferCurrID = 1;
+
+	/* Initialise lists of growable Freelists */
+	eError = OSLockCreate(&psDevInfo->hLockFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockFreeList)", ErrorExit);
+	dllist_init(&psDevInfo->sFreeListHead);
+	psDevInfo->ui32FreelistCurrID = 1;
+
+	eError = OSLockCreate(&psDevInfo->hDebugFaultInfoLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(DebugFaultInfoLock)", ErrorExit);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		eError = OSLockCreate(&psDevInfo->hMMUCtxUnregLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(MMUCtxUnregLock)", ErrorExit);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = OSLockCreate(&psDevInfo->hNMILock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(NMILock)", ErrorExit);
+	}
+#endif
+
+	/* Setup GPU utilisation stats update callback */
+	eError = OSLockCreate(&psDevInfo->hGPUUtilLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(GPUUtilLock)", ErrorExit);
+#if !defined(NO_HARDWARE)
+	psDevInfo->pfnGetGpuUtilStats = RGXGetGpuUtilStats;
+#endif
+
+	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	psDevInfo->eActivePMConf = psApphints->eRGXActivePMConf;
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	/* Validate the SPU mask and initialize to number of SPUs to power up */
+	if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s:Invalid SPU mask (All=0x%X, Non Fused=0x%X). At-least one SPU must to be powered up.",
+		         __func__,
+		         ui32AllPowUnitsMask,
+		         psApphints->ui32AvailablePowUnitsMask));
+		PVR_LOG_GOTO_WITH_ERROR("ui32AvailablePowUnitsMask", eError, PVRSRV_ERROR_INVALID_SPU_MASK, ErrorExit);
+	}
+
+	psDevInfo->ui32AvailablePowUnitsMask = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+	psDevInfo->ui32AvailableRACMask = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* set-up the Active Power Mgmt callback */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableAPM = psRGXData->psRGXTimingInfo->bEnableActivePM;
+		IMG_BOOL bEnableAPM = ((psApphints->eRGXActivePMConf == RGX_ACTIVEPM_DEFAULT) && bSysEnableAPM) ||
+							   (psApphints->eRGXActivePMConf == RGX_ACTIVEPM_FORCE_ON);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1) && defined(SUPPORT_AUTOVZ)
+		/* The AutoVz driver enables a virtualisation watchdog not compatible with APM */
+		if (bEnableAPM && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Active Power Management disabled in AutoVz mode", __func__));
+			bEnableAPM = IMG_FALSE;
+		}
+
+		PVR_ASSERT(bEnableAPM == IMG_FALSE);
+#endif
+
+		if (bEnableAPM)
+		{
+			eError = OSInstallMISR(&psDevInfo->pvAPMISRData,
+					RGX_MISRHandler_CheckFWActivePowerState,
+					psDeviceNode,
+					"RGX_CheckFWActivePower");
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(APMISR)", ErrorExit);
+
+			/* Prevent the device being woken up before there is something to do. */
+			eDefaultPowerState = PVRSRV_DEV_POWER_STATE_OFF;
+		}
+	}
+#endif
+
+	psDevInfo->eDebugDumpFWTLogType = psApphints->eDebugDumpFWTLogType;
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableAPM,
+	                                    RGXQueryAPMState,
+	                                    RGXSetAPMState,
+	                                    psDeviceNode,
+	                                    NULL);
+
+	RGXTimeCorrInitAppHintCallbacks(psDeviceNode);
+
+	/* Register the device with the power manager */
+	eError = PVRSRVRegisterPowerDevice(psDeviceNode,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPrePowerState : &RGXVzPrePowerState,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPostPowerState : &RGXVzPostPowerState,
+	                                   psDevConfig->pfnPrePowerState, psDevConfig->pfnPostPowerState,
+	                                   &RGXPreClockSpeedChange, &RGXPostClockSpeedChange,
+	                                   &RGXForcedIdleRequest, &RGXCancelForcedIdleRequest, &RGXCancelForcedIdleRequestAsync,
+	                                   &RGXPowUnitsChange,
+	                                   (IMG_HANDLE)psDeviceNode,
+	                                   PVRSRV_DEV_POWER_STATE_OFF,
+	                                   eDefaultPowerState);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterPowerDevice", ErrorExit);
+
+	eError = RGXSetPowerParams(psDevInfo, psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetPowerParams", ErrorExit);
+
+
+#if defined(PDUMP)
+#if defined(NO_HARDWARE)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT, "Wait for the FW to signal idle");
+
+	/* Kick the FW once, in case it still needs to detect and set the idle state */
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+			   RGX_CR_MTS_SCHEDULE,
+			   RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK,
+			   PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                                offsetof(RGXFWIF_SYSDATA, ePowState),
+	                                RGXFWIF_POW_IDLE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", ErrorExit);
+#endif
+
+	/* Run RGXStop with the correct PDump flags to feed the last-frame deinit buffer */
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT,
+	                      "RGX deinitialisation commands");
+
+	psDevInfo->sLayerParams.ui32PdumpFlags |= PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW;
+
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXStop(&psDevInfo->sLayerParams);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXStop", ErrorExit);
+	}
+
+	psDevInfo->sLayerParams.ui32PdumpFlags &= ~(PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW);
+#endif
+
+#if !defined(NO_HARDWARE)
+	eError = RGXInstallProcessQueuesMISR(&psDevInfo->hProcessQueuesMISR, psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInstallProcessQueuesMISR", ErrorExit);
+
+	/* Register RGX to receive notifies when other devices complete some work */
+	PVRSRVRegisterCmdCompleteNotify(&psDeviceNode->hCmdCompNotify, &RGXScheduleProcessQueuesKM, psDeviceNode);
+
+	/* Register the interrupt handlers */
+	eError = OSInstallMISR(&psDevInfo->pvMISRData,
+			RGX_MISRHandler_Main,
+			psDeviceNode,
+			"RGX_Main");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(MISR)", ErrorExit);
+
+	/* Register appropriate mechanism for clearing hw interrupts */
+	eError = RGXSetAckIrq(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetAckIrq", ErrorExit);
+
+	eError = SysInstallDeviceLISR(psDevConfig->hSysData,
+								  psDevConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  RGX_LISRHandler,
+								  psDeviceNode,
+								  &psDevInfo->pvLISRData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysInstallDeviceLISR", ErrorExit);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(PDUMP)
+    if (PVRSRVSystemSnoopingOfCPUCache(psDevConfig))
+    {
+        PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+                              "System has CPU cache snooping");
+    }
+    else
+    {
+        PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+                              "System has NO cache snooping");
+    }
+#endif
+
+#if defined(RGX_FEATURE_COMPUTE_ONLY_BIT_MASK)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE_ONLY))
+#endif
+	{
+		eError = PVRSRVTQLoadShaders(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVTQLoadShaders", ErrorExit);
+	}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	eError = OSAllocateSecBuf(psDeviceNode, RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE, "SharedSecMem", &psDevInfo->psGenHeapSecMem);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSAllocateSecBuf", ErrorExit);
+#endif
+
+	psDevInfo->bDevInit2Done = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	DevPart2DeInitRGX(psDeviceNode);
+
+	return eError;
+}
+
+#define VZ_RGX_FW_FILENAME_SUFFIX ".vz"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#define RGX_64K_FW_FILENAME_SUFFIX ".64k"
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX) + sizeof(RGX_64K_FW_FILENAME_SUFFIX)))
+#else
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX)))
+#endif
+
+static void _GetFWFileName(PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_CHAR *pszFWFilenameStr,
+		IMG_CHAR *pszFWpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const IMG_CHAR * const pszFWFilenameSuffix =
+			PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) ? "" : VZ_RGX_FW_FILENAME_SUFFIX;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	const IMG_CHAR * const pszFWFilenameSuffix2 =
+			((OSGetPageSize() == RGX_MMU_PAGE_SIZE_64KB) &&
+			 RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+			? RGX_64K_FW_FILENAME_SUFFIX : "";
+#else
+	const IMG_CHAR * const pszFWFilenameSuffix2 = "";
+#endif
+
+	OSSNPrintf(pszFWFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STR_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+
+	OSSNPrintf(pszFWpFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STRP_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+}
+
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW)
+{
+	IMG_CHAR aszFWFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR aszFWpFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR *pszLoadedFwStr;
+	PVRSRV_ERROR eErr;
+
+	/* Prepare the image filenames to use in the following code */
+	_GetFWFileName(psDeviceNode, aszFWFilenameStr, aszFWpFilenameStr);
+
+	/* Get pointer to Firmware image */
+	pszLoadedFwStr = aszFWFilenameStr;
+	eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+	if (eErr == PVRSRV_ERROR_NOT_FOUND)
+	{
+		pszLoadedFwStr = aszFWpFilenameStr;
+		eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+		if (eErr == PVRSRV_ERROR_NOT_FOUND)
+		{
+			pszLoadedFwStr = RGX_FW_FILENAME;
+			eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+			if (eErr == PVRSRV_ERROR_NOT_FOUND)
+			{
+				PVR_DPF((PVR_DBG_FATAL, "All RGX Firmware image loads failed for '%s' (%s)",
+						aszFWFilenameStr, PVRSRVGetErrorString(eErr)));
+			}
+		}
+	}
+
+	if (eErr == PVRSRV_OK)
+	{
+		PVR_LOG(("RGX Firmware image '%s' loaded", pszLoadedFwStr));
+	}
+
+	return eErr;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* set up fw memory contexts */
+	PVRSRV_RGXDEV_INFO   *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	IMG_BOOL  bNativeFwUMAHeap = PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) &&
+	                             (PhysHeapGetType(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM]) == PHYS_HEAP_TYPE_UMA);
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)) && (!psDeviceNode->bAutoVzFwIsUp) && (!bNativeFwUMAHeap))
+	{
+		PHYS_HEAP *psFwPageTableHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_GOTO_IF_INVALID_PARAM((psFwPageTableHeap != NULL),
+		                              eError, failed_to_create_ctx);
+
+		/* Temporarily swap the MMU and default GPU physheap to allow the page
+		 * tables of all memory mapped by the FwKernel context to be placed
+		 * in a dedicated memory carveout. This should allow the firmware mappings to
+		 * persist after a Host kernel crash or driver reset. */
+		psDeviceNode->psMMUPhysHeap = psFwPageTableHeap;
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+	/* Set the device fabric coherency before FW context creation */
+	eError = RGXSystemGetFabricCoherency(psDevConfig,
+										 psDevConfig->sRegsCpuPBase,
+										 psDevConfig->ui32RegsSize,
+										 &psDeviceNode->eDevFabricType,
+										 &psDevConfig->eCacheSnoopingMode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed RGXSystemGetFabricCoherency (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+#endif
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	RGXFwSharedMemCheckSnoopMode(psDevConfig);
+
+	/* Create the memory context for the firmware. */
+	eError = DevmemCreateContext(psDeviceNode, DEVMEM_HEAPCFG_FORFW,
+	                             &psDevInfo->psKernelDevmemCtx);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemCreateContext (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_MAIN_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareMainHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_CONFIG_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareConfigHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+#if (defined(RGX_PREMAP_FW_HEAPS)) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_CHAR szHeapName[RA_MAX_NAME_LENGTH];
+
+			OSSNPrintf(szHeapName, sizeof(szHeapName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+			eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, szHeapName,
+										  &psDevInfo->psPremappedFwRawHeap[ui32DriverID]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "DevmemFindHeapByName", failed_to_find_heap);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && !bNativeFwUMAHeap)
+	{
+		IMG_DEV_PHYADDR sPhysHeapBase;
+		IMG_UINT32 ui32DriverID;
+		void *pvAppHintState = NULL;
+		IMG_UINT64 ui64DefaultHeapStride;
+		IMG_UINT64 ui64GuestHeapDevBaseStride;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64DefaultHeapStride = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							GuestFWHeapStride,
+							&ui64DefaultHeapStride,
+							&ui64GuestHeapDevBaseStride);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		eError = PhysHeapGetDevPAddr(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM], &sPhysHeapBase);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", failed_to_find_heap);
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_DEV_PHYADDR sRawFwHeapBase = {sPhysHeapBase.uiAddr + (ui32DriverID * ui64GuestHeapDevBaseStride)};
+
+			eError = RGXFwRawHeapAllocMap(psDeviceNode,
+										  ui32DriverID,
+										  sRawFwHeapBase,
+										  RGX_FIRMWARE_RAW_HEAP_SIZE);
+			if (eError != PVRSRV_OK)
+			{
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+				}
+				PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", failed_to_find_heap);
+			}
+		}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+		/* restore default Px setup */
+		psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* On setups with dynamically mapped Guest heaps, the Guest makes
+	 * a PVZ call to the Host to request the mapping during init. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PvzClientMapDevPhysHeap(psDevConfig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PvzClientMapDevPhysHeap", failed_to_find_heap);
+	}
+#endif /* !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+
+failed_to_find_heap:
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+	DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+	psDevInfo->psKernelDevmemCtx = NULL;
+failed_to_create_ctx:
+	return eError;
+}
+
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR       eError;
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(RGX_PREMAP_FW_HEAPS)
+		psDeviceNode->psMMUPhysHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		if (!psDeviceNode->bAutoVzFwIsUp)
+#endif
+		{
+			IMG_UINT32 ui32DriverID;
+
+			FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+			{
+				RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+			}
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		(void) PvzClientUnmapDevPhysHeap(psDeviceNode->psDevConfig);
+
+		if (psDevInfo->psFirmwareMainHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_FALSE);
+		}
+		if (psDevInfo->psFirmwareConfigHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_FALSE);
+		}
+	}
+#endif
+
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+
+	if (psDevInfo->psKernelDevmemCtx)
+	{
+		eError = DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+}
+
+static PVRSRV_ERROR RGXAlignmentCheck(PVRSRV_DEVICE_NODE *psDevNode,
+                                      IMG_UINT32 ui32AlignChecksSizeUM,
+                                      IMG_UINT32 aui32AlignChecksUM[])
+{
+	static const IMG_UINT32 aui32AlignChecksKM[] = {RGXFW_ALIGN_CHECKS_INIT_KM};
+	IMG_UINT32 ui32UMChecksOffset = ARRAY_SIZE(aui32AlignChecksKM) + 1;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32 i, *paui32FWAlignChecks;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Skip the alignment check if the driver is guest
+	   since there is no firmware to check against */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, eError);
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW Alignment Check Mem Descriptor is NULL",
+		         __func__));
+		return PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
+	                                  (void **) &paui32FWAlignChecks);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire kernel address for alignment checks (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	paui32FWAlignChecks += ui32UMChecksOffset;
+	/* Invalidate the size value, check the next region size (UM) and invalidate */
+	RGXFwSharedMemCacheOpPtr(paui32FWAlignChecks, INVALIDATE);
+	if (*paui32FWAlignChecks++ != ui32AlignChecksSizeUM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Mismatching sizes of RGXFW_ALIGN_CHECKS_INIT"
+		         " array between UM(%d) and FW(%d)",
+		         __func__,
+		         ui32AlignChecksSizeUM,
+		         *paui32FWAlignChecks));
+		eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		goto return_;
+	}
+
+	RGXFwSharedMemCacheOpExec(paui32FWAlignChecks,
+	                          ui32AlignChecksSizeUM * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	for (i = 0; i < ui32AlignChecksSizeUM; i++)
+	{
+		if (aui32AlignChecksUM[i] != paui32FWAlignChecks[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: size/offset mismatch in RGXFW_ALIGN_CHECKS_INIT[%d]"
+					" between UM(%d) and FW(%d)",
+					__func__, i, aui32AlignChecksUM[i], paui32FWAlignChecks[i]));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		}
+	}
+
+	if (eError == PVRSRV_ERROR_INVALID_ALIGNMENT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Check for FW/KM structure"
+				" alignment failed.", __func__));
+	}
+
+return_:
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXAllocateFWMemoryRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_DEVMEM_SIZE_T ui32Size,
+                                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                       const IMG_PCHAR pszText,
+                                       DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Align = OSGetPageShift();
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(SUPPORT_MIPS_CONTIGUOUS_FW_MEMORY)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		uiLog2Align = RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiLog2Align);
+#endif
+
+	uiMemAllocFlags = (uiMemAllocFlags |
+					   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+					   PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &
+	                   RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	uiMemAllocFlags &= PVRSRV_MEMALLOCFLAGS_TDFWMASK;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate FW %s memory", pszText);
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										ui32Size,
+										1ULL << uiLog2Align,
+										uiMemAllocFlags,
+										pszText,
+										ppsMemDescPtr);
+
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver
+
+ @Description
+
+ Validate the FW build options against KM driver build options (KM build options only)
+
+ Following check is redundant, because next check checks the same bits.
+ Redundancy occurs because if client-server are build-compatible and client-firmware are
+ build-compatible then server-firmware are build-compatible as well.
+
+ This check is left for clarity in error messages if any incompatibility occurs.
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+
+	IMG_UINT32			ui32BuildOptions, ui32BuildOptionsFWKMPart, ui32BuildOptionsMismatch;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+	IMG_UINT8               ui8FwOsCount;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32BuildOptions = (RGX_BUILD_OPTIONS_KM & RGX_BUILD_OPTIONS_MASK_FW);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32BuildOptionsFWKMPart = psFwOsInit->sRGXCompChecks.ui32BuildOptions & RGX_BUILD_OPTIONS_MASK_FW;
+
+		ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+		if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32BuildOptionsFWKMPart = psFWInfoHeader->ui32Flags & RGX_BUILD_OPTIONS_MASK_FW;
+
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo) && BITMASK_HAS(psFWInfoHeader->ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psFWInfoHeader->ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+				return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+			}
+		}
+	}
+
+	/* Check if the FW is missing support for any features required by the driver */
+	if (~ui32BuildOptionsFWKMPart & ui32BuildOptions)
+	{
+		ui32BuildOptionsMismatch = ui32BuildOptions ^ ui32BuildOptionsFWKMPart;
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask non-critical options out as we do support combining them in UM & KM */
+		ui32BuildOptionsMismatch &= FW_OPTIONS_STRICT;
+#endif
+		if ((ui32BuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and KM driver build options; "
+					 "extra options present in the KM driver: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptions & ui32BuildOptionsMismatch));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+
+		if ((ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware-side and KM driver build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch ));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "RGXDevInitCompatCheck: Firmware and KM driver build options differ."));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: Firmware and KM driver build options match. [ OK ]"));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK version against driver DDK version
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32			ui32KMDDKVersion;
+	IMG_UINT32			ui32FWDDKVersion;
+	PVRSRV_ERROR		eError;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32KMDDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKVersion = psFwOsInit->sRGXCompChecks.ui32DDKVersion;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+		ui32FWDDKVersion = PVRVERSION_PACK(psFWInfoHeader->ui16PVRVersionMajor, psFWInfoHeader->ui16PVRVersionMinor);
+	}
+
+	if (ui32FWDDKVersion != ui32KMDDKVersion)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Incompatible driver DDK version (%u.%u) / Firmware DDK version (%u.%u).",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_UNPACK_MAJ(ui32FWDDKVersion),
+				 PVRVERSION_UNPACK_MIN(ui32FWDDKVersion)));
+		eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK version (%u.%u) and Firmware DDK version (%u.%u) match. [ OK ]",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK build against driver DDK build
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32KMDDKBuild;
+	IMG_UINT32			ui32FWDDKBuild;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	ui32KMDDKBuild = PVRVERSION_BUILD;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKBuild = psFwOsInit->sRGXCompChecks.ui32DDKBuild;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32FWDDKBuild = psFWInfoHeader->ui32PVRVersionBuild;
+	}
+
+	if (ui32FWDDKBuild != ui32KMDDKBuild)
+	{
+		PVR_LOG(("(WARN) RGXDevInitCompatCheck: Different driver DDK build version (%d) / Firmware DDK build version (%d).",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		eError = PVRSRV_ERROR_DDK_BUILD_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK build version (%d) and Firmware DDK build version (%d) match. [ OK ]",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+	}
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_BVNC_FWAgainstDriver
+
+ @Description
+
+ Validate FW BVNC against driver BVNC
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_BVNC_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+
+	RGX_FW_INFO_HEADER	*psFWInfoHeader;
+	IMG_UINT64			ui64KMBVNC;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+	ui64KMBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+							   psDevInfo->sDevFeatureCfg.ui32V,
+							   psDevInfo->sDevFeatureCfg.ui32N,
+							   psDevInfo->sDevFeatureCfg.ui32C);
+
+	if (ui64KMBVNC != psFWInfoHeader->ui64BVNC)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in KM driver BVNC (%u.%u.%u.%u) and Firmware BVNC (%u.%u.%u.%u)",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_ERROR_BVNC_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: KM driver BVNC (%d.%d.%d.%d) and FW BVNC (%d.%d.%d.%d) match. [ OK ]",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevInitCompatCheck
+
+ @Description
+
+ Check compatibility of host driver and firmware (DDK and build options)
+ for RGX devices at services/device initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32			ui32FwTimeout = MAX_HW_TIME_US;
+
+		LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Flush covers this instance and the reads in the functions below */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                               INVALIDATE);
+		if (!*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU Firmware not responding: failed to supply compatibility info (%u)",
+					__func__, eError));
+
+			PVR_DPF((PVR_DBG_ERROR, "%s: Potential causes: firmware not initialised or the current Guest driver's "
+									"OsConfig initialisation data was not accepted by the firmware", __func__));
+			goto chk_exit;
+		}
+	}
+#endif /* defined(NO_HARDWARE) */
+
+	eError = RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDevInitCompatCheck_BVNC_FWAgainstDriver(psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			goto chk_exit;
+		}
+	}
+
+	eError = PVRSRV_OK;
+chk_exit:
+
+	return eError;
+}
+
+static void _RGXSoftResetToggle(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT64  ui64ResetValue1,
+                                IMG_UINT64  ui64ResetValue2)
+{
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, ui64ResetValue1);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
+}
+
+/**************************************************************************/ /*!
+@Function       RGXSoftReset
+@Description    Resets some modules of the RGX device
+@Input          psDeviceNode		Device node
+@Input          ui64ResetValue1 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET
+                                register).
+@Input          ui64ResetValue2 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET2
+                                register).
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT64  ui64ResetValue1,
+                                 IMG_UINT64  ui64ResetValue2)
+{
+	PVRSRV_RGXDEV_INFO        *psDevInfo;
+	IMG_BOOL	bSoftReset = IMG_FALSE;
+	IMG_UINT64	ui64SoftResetMask = 0;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+	if (((ui64ResetValue1 & ui64SoftResetMask) != ui64ResetValue1) || bSoftReset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Set in soft-reset */
+	_RGXSoftResetToggle(psDevInfo, ui64ResetValue1, ui64ResetValue2);
+
+	/* Take the modules out of reset... */
+	_RGXSoftResetToggle(psDevInfo, 0, 0);
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static const RGX_MIPS_ADDRESS_TRAMPOLINE sNullTrampoline;
+
+static void RGXFreeTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+			psDevInfo->psTrampoline->hPdumpPages,
+#endif
+			&psDevInfo->psTrampoline->sPages);
+
+	if (psDevInfo->psTrampoline != &sNullTrampoline)
+	{
+		OSFreeMem(psDevInfo->psTrampoline);
+	}
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+}
+
+#define RANGES_OVERLAP(x,y,size) (x < (y+size) && y < (x+size))
+#define TRAMPOLINE_ALLOC_MAX_RETRIES (3)
+
+static PVRSRV_ERROR RGXAllocTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT32 i, j;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_MIPS_ADDRESS_TRAMPOLINE *pasTrampoline[TRAMPOLINE_ALLOC_MAX_RETRIES];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate pages for trampoline");
+
+	/* Retry the allocation of the trampoline block (16KB), retaining any
+	 * previous allocations overlapping  with the target range until we get an
+	 * allocation that doesn't overlap with the target range.
+	 * Any allocation like this will require a maximum of 3 tries as we are
+	 * allocating a physical contiguous block of memory, not individual pages.
+	 * Free the unused allocations at the end only after the desired range
+	 * is obtained to prevent the alloc function from returning the same bad
+	 * range repeatedly.
+	 */
+	for (i = 0; i < TRAMPOLINE_ALLOC_MAX_RETRIES; i++)
+	{
+		pasTrampoline[i] = OSAllocMem(sizeof(RGX_MIPS_ADDRESS_TRAMPOLINE));
+		eError = DevPhysMemAlloc(psDeviceNode,
+				RGXMIPSFW_TRAMPOLINE_SIZE,
+				RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE,
+				0,         // (init) u8Value
+				IMG_FALSE, // bInitPage,
+#if defined(PDUMP)
+				psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				"TrampolineRegion",
+				&pasTrampoline[i]->hPdumpPages,
+#endif
+				PVR_SYS_ALLOC_PID,
+				&pasTrampoline[i]->sPages,
+				&pasTrampoline[i]->sPhysAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s failed (%u)",
+			         __func__,
+			         eError));
+			goto fail;
+		}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		/* Set the persistent uiOSid value so that we free from the correct
+		 * base arena when unloading the driver and freeing the trampoline.
+		 */
+		pasTrampoline[i]->sPages.uiOSid = 0;	/* Firmware global arena */
+#endif
+
+		if (!RANGES_OVERLAP(pasTrampoline[i]->sPhysAddr.uiAddr,
+				RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+				RGXMIPSFW_TRAMPOLINE_SIZE))
+		{
+			break;
+		}
+	}
+	if (TRAMPOLINE_ALLOC_MAX_RETRIES == i)
+	{
+		/* Failed to find a physical allocation after 3 attempts */
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s failed to allocate non-overlapping pages (%u)",
+				__func__, eError));
+		/* Fall through, clean up and return error. */
+	}
+	else
+	{
+		/* Remember the last physical block allocated, it will not be freed */
+		psDevInfo->psTrampoline = pasTrampoline[i];
+	}
+
+fail:
+	/* free all unused allocations */
+	for (j = 0; j < i; j++)
+	{
+		DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+				pasTrampoline[j]->hPdumpPages,
+#endif
+				&pasTrampoline[j]->sPages);
+		OSFreeMem(pasTrampoline[j]);
+	}
+
+	return eError;
+}
+
+#undef RANGES_OVERLAP
+#endif
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    uiFWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR        eError;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEVMEM_SIZE_T	uiDummyLen;
+	DEVMEM_MEMDESC		*psDummyMemDesc = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		eError = RGXAllocTrampoline(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Failed to allocate trampoline region (%u)",
+					eError));
+			goto failTrampolineMemDescAlloc;
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW code section
+	 */
+	uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+	                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE);
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWCodeLen,
+	                                   uiMemAllocFlags,
+	                                   "FwExCodeRegion",
+	                                   &psDevInfo->psRGXFWCodeMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc,
+	                                  &psDevInfo->sFWCodeDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAqDevVirt;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) || (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))))
+#endif
+	{
+		/*
+		 * The FW code must be the first allocation in the firmware heap, otherwise
+		 * the bootloader will not work (the FW will not be able to find the bootloader).
+		 */
+		PVR_ASSERT(psDevInfo->sFWCodeDevVAddrBase.uiAddr == RGX_FIRMWARE_RAW_HEAP_BASE);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/*
+		 * Allocate Dummy Pages so that Data segment allocation gets the same
+		 * device virtual address as specified in MIPS firmware linker script
+		 */
+		uiDummyLen = RGXGetFWImageSectionMaxSize(NULL, MIPS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_EXCEPTIONS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_BOOT_CODE) -
+				uiFWCodeLen; /* code actual size */
+
+		if (uiDummyLen > 0)
+		{
+			eError = DevmemFwAllocateExportable(psDeviceNode,
+					uiDummyLen,
+					OSGetPageSize(),
+					uiMemAllocFlags,
+					"FwExDummyPages",
+					&psDummyMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "Failed to allocate fw dummy mem (%u)",
+				         eError));
+				goto failDummyMemDescAlloc;
+			}
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW data section
+	 */
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWDataLen,
+	                                   RGX_FWCODEDATA_ALLOCFLAGS |
+	                                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA),
+	                                   "FwExDataRegion",
+	                                   &psDevInfo->psRGXFWDataMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+	                                  &psDevInfo->sFWDataDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAqDevVirt;
+	}
+
+	if (uiFWCorememCodeLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem code section
+		 */
+		uiMemAllocFlags = (RGX_FWCODEDATA_ALLOCFLAGS |
+		                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE)) &
+		                   ~PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE;
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+		                                   uiFWCorememCodeLen,
+		                                   uiMemAllocFlags,
+		                                   "FwExCorememCodeRegion",
+		                                   &psDevInfo->psRGXFWCorememCodeMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem code mem, size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememCodeLen, uiMemAllocFlags, eError));
+			goto failFWCorememCodeMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc,
+		                                  &psDevInfo->sFWCorememCodeDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem code (%u)",
+			         eError));
+			goto failFWCorememCodeMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememCodeFWAddr,
+		                      psDevInfo->psRGXFWCorememCodeMemDesc,
+		                      0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", failFWCorememCodeMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememCodeDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememCodeFWAddr.ui32Addr = 0;
+	}
+
+	if (uiFWCorememDataLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem data section
+		 */
+		uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+		                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA);
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+				uiFWCorememDataLen,
+				uiMemAllocFlags,
+				"FwExCorememDataRegion",
+				&psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem data mem, "
+			         "size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememDataLen,
+			         uiMemAllocFlags,
+			         eError));
+			goto failFWCorememDataMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				&psDevInfo->sFWCorememDataStoreDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem data (%u)",
+			         eError));
+			goto failFWCorememDataMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememDataStoreFWAddr,
+				psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", failFWCorememDataMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememDataStoreDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememDataStoreFWAddr.ui32Addr = 0;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Free Dummy Pages */
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failFWCorememDataMemDescFwAddr:
+failFWCorememDataMemDescAqDevVirt:
+	if (uiFWCorememDataLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+failFWCorememDataMemDescAlloc:
+failFWCorememCodeMemDescFwAddr:
+failFWCorememCodeMemDescAqDevVirt:
+	if (uiFWCorememCodeLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+failFWCorememCodeMemDescAlloc:
+failFWDataMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+	psDevInfo->psRGXFWDataMemDesc = NULL;
+failFWDataMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+failDummyMemDescAlloc:
+#endif
+failFWCodeMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+	psDevInfo->psRGXFWCodeMemDesc = NULL;
+failFWCodeMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		RGXFreeTrampoline(psDeviceNode);
+	}
+failTrampolineMemDescAlloc:
+#endif
+	return eError;
+}
+
+/*
+	AppHint parameter interface
+ */
+static
+PVRSRV_ERROR RGXFWTraceQueryFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	*pui32Value &= RGXFWIF_LOG_TYPE_GROUP_MASK;
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceQueryLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	if (PVRSRV_OK == eResult)
+	{
+		if (*pui32Value & RGXFWIF_LOG_TYPE_TRACE)
+		{
+			*pui32Value = 0; /* Trace */
+		}
+		else
+		{
+			*pui32Value = 1; /* TBI */
+		}
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const void *psPrivate,
+                                 IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType;
+
+	eResult = RGXFWTraceQueryLogType(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK == eResult)
+	{
+		if (0 == ui32RGXFWLogType)
+		{
+			BITMASK_SET(ui32Value, RGXFWIF_LOG_TYPE_TRACE);
+		}
+		eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32Value);
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType = ui32Value;
+
+	eResult = RGXFWTraceQueryFilter(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK != eResult)
+	{
+		return eResult;
+	}
+
+	/* 0 - trace, 1 - tbi */
+	if (0 == ui32Value)
+	{
+		BITMASK_SET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	else if (1 == ui32Value)
+	{
+		BITMASK_UNSET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid parameter %u specified to set FW log type AppHint.",
+		         __func__, ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32RGXFWLogType);
+	return eResult;
+}
+
+#if defined(DEBUG)
+static
+PVRSRV_ERROR RGXQueryFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL *pbValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pbValue = (PVRSRV_MEMALLOCFLAG_POISON_ON_FREE == psDevInfo->uiFWPoisonOnFreeFlag)
+		? IMG_TRUE
+		: IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXSetFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL bValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	psDevInfo->uiFWPoisonOnFreeFlag = bValue
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+ * RGXInitFirmware
+ */
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+#if defined(DEBUG)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bEnableFWPoisonOnFree = IMG_FALSE;
+#endif
+
+	eError = RGXSetupFirmware(psDeviceNode,
+	                          psApphints,
+	                          ui32ConfigFlags,
+	                          ui32ConfigFlagsExt,
+	                          ui32FwOsCfgFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXInitFirmwareKM: RGXSetupFirmware failed (%u)",
+		         eError));
+		goto failed_init_firmware;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableLogGroup,
+		                                    RGXFWTraceQueryFilter,
+		                                    RGXFWTraceSetFilter,
+		                                    psDeviceNode,
+		                                    NULL);
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_FirmwareLogType,
+		                                    RGXFWTraceQueryLogType,
+		                                    RGXFWTraceSetLogType,
+		                                    psDeviceNode,
+		                                    NULL);
+	}
+
+#if defined(DEBUG)
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFWPOISONONFREE;
+	OSGetAppHintBOOL(psDeviceNode,
+			pvAppHintState,
+			EnableFWPoisonOnFree,
+			&bAppHintDefault,
+			&bEnableFWPoisonOnFree);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFWPoisonOnFree,
+	                                   RGXQueryFWPoisonOnFree,
+	                                   RGXSetFWPoisonOnFree,
+	                                   psDeviceNode,
+	                                   NULL);
+
+	psDevInfo->uiFWPoisonOnFreeFlag = bEnableFWPoisonOnFree
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+#else
+	psDevInfo->uiFWPoisonOnFreeFlag = 0ULL;
+#endif
+
+	psDevInfo->ui32ClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	psDevInfo->ui32LastClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+
+	return PVRSRV_OK;
+
+failed_init_firmware:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* See device.h for function declaration */
+static PVRSRV_ERROR RGXAllocUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+									 IMG_UINT32 ui32RequestedSize,
+									 DEVMEM_MEMDESC **psMemDesc,
+									 IMG_UINT32 *puiSyncPrimVAddr,
+									 IMG_UINT32 *puiSyncPrimBlockSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
+	IMG_DEVMEM_ALIGN_T uiUFOBlockAlign = MAX(sizeof(IMG_UINT32), sizeof(SYNC_CHECKPOINT_FW_OBJ));
+	IMG_DEVMEM_SIZE_T uiUFOBlockSize = PVR_ALIGN(ui32RequestedSize, uiUFOBlockAlign);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Size and align are 'expanded' because we request an Exportalign allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiUFOBlockSize,
+	                                             &uiUFOBlockAlign);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										uiUFOBlockSize,
+										uiUFOBlockAlign,
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_UNCACHED,
+										"FwExUFOBlock",
+										psMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = RGXSetFirmwareAddress(&pFirmwareAddr, *psMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = pFirmwareAddr.ui32Addr;
+	*puiSyncPrimBlockSize = TRUNCATE_64BITS_TO_32BITS(uiUFOBlockSize);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, *psMemDesc);
+e0:
+	return eError;
+}
+
+/* See device.h for function declaration */
+static void RGXFreeUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+							DEVMEM_MEMDESC *psMemDesc)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGXUnsetFirmwareAddress(psMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psMemDesc);
+}
+
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	psDevInfo->bDevInit2Done = IMG_FALSE;
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	if (psDevInfo->psGenHeapSecMem != NULL)
+	{
+		OSFreeSecBuf(psDevInfo->psGenHeapSecMem);
+	}
+#endif
+
+#if defined(RGX_FEATURE_COMPUTE_ONLY_BIT_MASK)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE_ONLY))
+#endif
+	{
+		if ((psDevInfo->hTQUSCSharedMem != NULL) &&
+		    (psDevInfo->hTQCLISharedMem != NULL))
+		{
+			PVRSRVTQUnloadShaders(psDeviceNode);
+		}
+	}
+
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvLISRData != NULL)
+	{
+		(void) SysUninstallDeviceLISR(psDevInfo->pvLISRData);
+	}
+	if (psDevInfo->pvMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvMISRData);
+	}
+	if (psDevInfo->hProcessQueuesMISR != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->hProcessQueuesMISR);
+	}
+	if (psDevInfo->pvAPMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvAPMISRData);
+	}
+	if (psDeviceNode->hCmdCompNotify != NULL)
+	{
+		/* Cancel notifications to this device */
+		PVRSRVUnregisterCmdCompleteNotify(psDeviceNode->hCmdCompNotify);
+		psDeviceNode->hCmdCompNotify = NULL;
+	}
+#endif /* !NO_HARDWARE */
+
+	/* Remove the device from the power manager */
+	PVRSRVRemovePowerDevice(psDeviceNode);
+
+	psDevInfo->pfnGetGpuUtilStats = NULL;
+	if (psDevInfo->hGPUUtilLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hGPUUtilLock);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(psDevInfo->hNMILock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hNMILock);
+	}
+#endif
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(psDevInfo->hMMUCtxUnregLock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	if (psDevInfo->hDebugFaultInfoLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hDebugFaultInfoLock);
+	}
+
+	/* De-init Freelists/ZBuffers... */
+	if (psDevInfo->hLockFreeList != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockFreeList);
+	}
+
+	if (psDevInfo->hLockZSBuffer != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockZSBuffer);
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* De-init work estimation lock */
+		if (psDevInfo->hWorkEstLock != NULL)
+		{
+			OSLockDestroy(psDevInfo->hWorkEstLock);
+		}
+	}
+#endif
+
+	/* Free DVFS Table */
+	if (psDevInfo->psGpuDVFSTable != NULL)
+	{
+		OSFreeMem(psDevInfo->psGpuDVFSTable);
+		psDevInfo->psGpuDVFSTable = NULL;
+	}
+}
+
+/*
+	DevDeInitRGX
+ */
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO		*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	PVRSRV_ERROR			eError;
+	DEVICE_MEMORY_INFO		*psDevMemoryInfo;
+
+	if (!psDevInfo)
+	{
+		/* Can happen if DevInitRGX failed */
+		PVR_DPF((PVR_DBG_ERROR, "DevDeInitRGX: Null DevInfo"));
+		return PVRSRV_OK;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	RGXUnregisterBridges(psDevInfo);
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	OSLockDestroy(psDevInfo->hCounterDumpingLock);
+#endif
+
+	/* Unregister debug request notifiers first as they could depend on anything. */
+
+	RGXDebugDeinit(psDevInfo);
+
+	/* De-initialise in reverse order, so stage 2 init is undone first. */
+	if (psDevInfo->bDevInit2Done)
+	{
+		DevPart2DeInitRGX(psDeviceNode);
+	}
+
+	/* Unregister MMU related stuff */
+	eError = RGXMMUInit_Unregister(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevDeInitRGX: Failed RGXMMUInit_Unregister (0x%x)",
+		         eError));
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* Unregister MMU related stuff */
+		eError = RGXMipsMMUInit_Unregister(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed RGXMipsMMUInit_Unregister (0x%x)",
+			         eError));
+		}
+	}
+#endif
+
+	/* UnMap Regs */
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+						 psDevInfo->ui32RegSize);
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvRegsBaseKM = NULL;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+			(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* undo the VA offset performed in RGXRegisterDevice() to allow the allocation to be unmapped */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM, RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvSecureRegsBaseKM = NULL;
+	}
+#endif
+
+#if 0 /* not required at this time */
+	if (psDevInfo->hTimer)
+	{
+		eError = OSRemoveTimer(psDevInfo->hTimer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed to remove timer"));
+			return eError;
+		}
+		psDevInfo->hTimer = NULL;
+	}
+#endif
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+
+	RGXDeInitHeaps(psDevMemoryInfo, psDeviceNode);
+
+	if (psDevInfo->psRGXFWCodeMemDesc)
+	{
+		/* Free fw code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+		psDevInfo->psRGXFWCodeMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware code memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		if (psDevInfo->psTrampoline->sPages.u.pvHandle)
+		{
+			/* Free trampoline region */
+			PDUMPCOMMENT(psDeviceNode, "Freeing trampoline memory");
+			RGXFreeTrampoline(psDeviceNode);
+		}
+	}
+#endif
+
+	if (psDevInfo->psRGXFWDataMemDesc)
+	{
+		/* Free fw data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW data memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+		psDevInfo->psRGXFWDataMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware data memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+	if (psDevInfo->psRGXFWCorememCodeMemDesc)
+	{
+		/* Free fw core mem code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+	{
+		/* Free fw core mem data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem data store memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+
+	/*
+	   Free the firmware allocations.
+	 */
+	RGXFreeFirmware(psDevInfo);
+
+
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+
+	/* De-initialise non-device specific (TL) users of RGX device memory */
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+		{
+			RGXHWPerfDeinitL2Stream(psDevInfo, i);
+		}
+
+		RGXHWPerfDeinit(psDevInfo);
+	}
+
+	RGXDeInitDestroyFWKernelMemoryContext(psDeviceNode);
+
+	RGXHWPerfHostDeInit(psDevInfo);
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+
+	/* destroy the stalled CCB locks */
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+
+	/* destroy the context list locks */
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+	OSLockDestroy(psDevInfo->hBPLock);
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+
+	/* Free device BVNC string */
+	if (NULL != psDevInfo->sDevFeatureCfg.pszBVNCString)
+	{
+		OSFreeMem(psDevInfo->sDevFeatureCfg.pszBVNCString);
+	}
+
+
+	/* DeAllocate devinfo */
+	OSFreeMem(psDevInfo);
+
+	psDeviceNode->pvDevice = NULL;
+
+	return PVRSRV_OK;
+}
+
+#if defined(PDUMP)
+static
+PVRSRV_ERROR RGXResetPDump(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/* Takes a log2 page size parameter and calculates a suitable page size
+ * for the RGX heaps. Returns 0 if parameter is wrong.*/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize)
+{
+	IMG_BOOL bFound = IMG_FALSE;
+	IMG_UINT32 ui32PageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	/* OS page shift must be at least RGX_HEAP_4KB_PAGE_SHIFT,
+	 * max RGX_HEAP_2MB_PAGE_SHIFT, non-zero and a power of two*/
+	if (uiLog2PageSize == 0U ||
+	    (uiLog2PageSize < RGX_HEAP_4KB_PAGE_SHIFT) ||
+	    (uiLog2PageSize > RGX_HEAP_2MB_PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Provided incompatible log2 page size %u",
+				__func__,
+				uiLog2PageSize));
+		PVR_ASSERT(0);
+		return 0;
+	}
+
+	do
+	{
+		if ((IMG_PAGE2BYTES32(uiLog2PageSize) & ui32PageSizeMask) == 0)
+		{
+			/* We have to fall back to a smaller device
+			 * page size than given page size because there
+			 * is no exact match for any supported size. */
+			uiLog2PageSize -= 1U;
+		}
+		else
+		{
+			/* All good, RGX page size equals given page size
+			 * => use it as default for heaps */
+			bFound = IMG_TRUE;
+		}
+	} while (!bFound);
+
+	return uiLog2PageSize;
+}
+
+/* First 16-bits define possible types */
+#define HEAP_INST_VALUE_MASK     (0xFFFF)
+#define HEAP_INST_DEFAULT_VALUE  (1U)  /* Used to show either the heap is always instantiated by default (pfn = NULL)
+	                                      OR
+	                                      that this is the default configuration of the heap with an Alternative BRN */
+#define HEAP_INST_BRN_DEP_VALUE  (2U)  /* The inclusion of this heap is dependent on the brn being present */
+#define HEAP_INST_FEAT_DEP_VALUE (3U)  /* The inclusion of this heap is dependent on the feature being present */
+#define HEAP_INST_BRN_ALT_VALUE  (4U)  /* This entry is a possible alternative to the default determined by a BRN */
+#define HEAP_INST_FEAT_ALT_VALUE (5U)  /* The entry is a possible alternative to the default determined by a Feature define */
+
+/* Latter 16-bits define other flags we may need */
+#define HEAP_INST_NON4K_FLAG     (1 << 16U) /* This is a possible NON4K Entry and we should use the device
+                                               NON4K size when instantiating */
+
+typedef struct RGX_HEAP_INFO_TAG RGX_HEAP_INFO; // Forward declaration
+typedef IMG_BOOL (*PFN_IS_PRESENT)(PVRSRV_RGXDEV_INFO*, const RGX_HEAP_INFO*);
+typedef void (*PFN_HEAP_DYNAMIC)(PVRSRV_DEVICE_NODE*, RGX_HEAP_INFO*);
+
+struct RGX_HEAP_INFO_TAG
+{
+	IMG_CHAR           *pszName;
+	IMG_UINT64         ui64HeapBase;
+	IMG_DEVMEM_SIZE_T  uiHeapLength;
+	IMG_DEVMEM_SIZE_T  uiHeapReservedRegionLength;
+	IMG_UINT32         ui32Log2ImportAlignment;
+	PFN_IS_PRESENT     pfnIsHeapPresent;
+	PFN_HEAP_DYNAMIC   pfnDynamicBaseSize; /* May modify the psHeapInfo's base & length. May be NULL.
+	                                          Only called once if the heap is present otherwise never. */
+	PFN_HEAP_INIT      pfnInit;
+	PFN_HEAP_DEINIT    pfnDeInit;
+	IMG_UINT32         ui32HeapInstanceFlags;
+};
+/* RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES is the total amount of reserved space, to be specified in gasRGXHeapLayoutApp[] */
+#define RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES	(RGX_HEAP_UM_GENERAL_RESERVED_SIZE + RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE)
+
+/* Private heap data struct. */
+typedef struct RGX_HEAP_DATA_TAG
+{
+	DEVMEMINT_RESERVATION  *psMemReservation;
+} RGX_HEAP_DATA;
+
+static PVRSRV_ERROR HeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_HEAP *psDevmemHeap,
+							 PMR *psPMR,
+							 IMG_DEVMEM_SIZE_T uiSize,
+							 IMG_UINT64 ui64Offset,
+							 IMG_BOOL bWriteAble,
+							 IMG_HANDLE *phPrivData)
+{
+	RGX_HEAP_DATA *psHeapData;
+	IMG_DEV_VIRTADDR sCarveOutAddr;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap, "psDevmemHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phPrivData, "phPrivData");
+
+	psHeapData = OSAllocMem(sizeof(*psHeapData));
+	PVR_LOG_RETURN_IF_NOMEM(psHeapData, "psHeapData");
+
+	/* Map the per device secure mem PMR allocation to the general devmem heap carveout. */
+	sCarveOutAddr = DevmemIntHeapGetBaseAddr(psDevmemHeap);
+	sCarveOutAddr.uiAddr += ui64Offset;
+
+	eError = DevmemIntReserveRange(NULL,
+	                               NULL,
+	                               psDevmemHeap,
+	                               sCarveOutAddr,
+	                               uiSize,
+	                               PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                               (bWriteAble ? PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE : 0),
+	                               &psHeapData->psMemReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeHeapData);
+
+	eError = DevmemIntMapPMR(psHeapData->psMemReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	*phPrivData = (IMG_HANDLE)psHeapData;
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+ErrorFreeHeapData:
+	OSFreeMem(psHeapData);
+
+	return eError;
+}
+
+/* Deinit callback function for general heap. */
+static void HeapDeInit(IMG_HANDLE hPrivData)
+{
+	RGX_HEAP_DATA *psHeapData = (RGX_HEAP_DATA*)hPrivData;
+
+	PVR_ASSERT(hPrivData);
+
+	DevmemIntUnmapPMR(psHeapData->psMemReservation);
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+
+	OSFreeMem(psHeapData);
+}
+
+/* Init callback function for general heap. */
+static PVRSRV_ERROR USCHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_FALSE;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	uiSize = PMR_PhysicalSize(psDevInfo->hTQUSCSharedMem);
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->hTQUSCSharedMem,
+					  uiSize,
+					  RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+/* Init callback function for general heap. */
+static PVRSRV_ERROR GeneralHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_TRUE;
+	PVRSRV_ERROR eError;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->psGenHeapSecMem,
+					  RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE,
+					  RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#define GeneralHeapDeInit HeapDeInit
+#else
+/* Callbacks not used */
+#define GeneralHeapInit NULL
+#define GeneralHeapDeInit NULL
+#endif
+
+static void SVMHeapDynamic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           RGX_HEAP_INFO *psHeapInfo)
+{
+	IMG_UINT64 ui64OSPageSize = OSGetPageSize();
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+
+	/* Ensures the SVM heap has the correct alignment & size for any OS page size.
+	 *
+	 * The SVM heap's base must be the smallest possible address mappable by UM.
+	 * This is 32KB unless the page size is larger than 32KB. [1]
+	 * If the page size > 32KB, raise the SVM heap base to the next page boundary.
+	 * Also reduce the length to ensure it's still page aligned and doesn't go
+	 * into another heap.
+	 *
+	 * [1]: https://chromium.googlesource.com/chromium/src/+/fe24932ee14aa93e1fe4d3e7003b9362591a54d4/docs/security/faq.md#why-aren_t-null-pointer-dereferences-considered-security-bugs
+	 */
+	IMG_UINT64 ui64Base = PVR_ALIGN(psHeapInfo->ui64HeapBase, ui64OSPageSize);
+	IMG_UINT64 ui64BaseDiff = ui64Base - psHeapInfo->ui64HeapBase;
+	psHeapInfo->ui64HeapBase = ui64Base;
+	if (psHeapInfo->uiHeapLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapLength -= ui64BaseDiff;
+	if (psHeapInfo->uiHeapReservedRegionLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapReservedRegionLength -= ui64BaseDiff;
+
+	/* The device shared-virtual-memory heap address-space size is stored on the device for
+	   faster look-up without having to walk the device heap configuration structures during
+	   client device connection  (i.e. this size is relative to a zero-based offset) */
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = 0;
+	}
+	else
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = psHeapInfo->ui64HeapBase + psHeapInfo->uiHeapLength;
+	}
+}
+
+/* Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+static IMG_BOOL BRN65273IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		return (((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_ALT_VALUE) ||
+		        ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_DEP_VALUE)) ?
+		        IMG_TRUE : IMG_FALSE;
+	}
+	else
+	{
+		return ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_DEFAULT_VALUE) ? IMG_TRUE : IMG_FALSE;
+	}
+}
+#endif
+
+#if defined(FIX_HW_BRN_63142_BIT_MASK)
+static IMG_BOOL BRN63142IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 63142))
+	{
+		PVR_ASSERT((pksHeapInfo->ui64HeapBase & IMG_UINT64_C(0x3FFFFFFFF)) +
+		            pksHeapInfo->uiHeapLength == IMG_UINT64_C(0x400000000));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FBCDescriptorIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, FBC_MAX_DEFAULT_DESCRIPTORS))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_BOOL FBCLargeDescriptorIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, FBC_MAX_LARGE_DESCRIPTORS))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_BOOL TextureStateIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+#if defined(RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, BINDLESS_IMAGE_AND_TEXTURE_STATE))
+	{
+		return IMG_TRUE;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+#endif
+	return IMG_FALSE;
+}
+
+/* FW Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+static IMG_BOOL FWBRN65101IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap size and base at run time VS the current table instance
+	 */
+	IMG_UINT64 ui64MainSubHeapSize;
+
+	/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)))
+	{
+		if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101;
+		}
+		else
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL;
+		}
+	}
+	else
+	{
+		ui64MainSubHeapSize = RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE;
+	}
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->uiHeapLength == ui64MainSubHeapSize &&
+	        pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ?
+	        IMG_TRUE : IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FWVZConfigPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_CONFIG_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+/* Blueprint array. note: not all heaps are available to clients*/
+
+static RGX_HEAP_INFO gasRGXHeapLayoutApp[] =
+{
+	/* Name                             HeapBase                                 HeapLength                               HeapReservedRegionLength                     Log2ImportAlignment pfnIsHeapPresent             pfnDynamicBaseSize pfnInit           pfnDeInit          HeapInstanceFlags   */
+	{RGX_GENERAL_SVM_HEAP_IDENT,        RGX_GENERAL_SVM_HEAP_BASE,               RGX_GENERAL_SVM_HEAP_SIZE,               0,                                           0,                  NULL,                        SVMHeapDynamic,    NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_HEAP_BASE,                   RGX_GENERAL_HEAP_SIZE,                   RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  BRN65273IsPresent,           NULL,              GeneralHeapInit,  GeneralHeapDeInit, HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_BRN_65273_HEAP_BASE,         RGX_GENERAL_BRN_65273_HEAP_SIZE,         RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_HEAP_BASE,             RGX_GENERAL_NON4K_HEAP_SIZE,             0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_BRN_65273_HEAP_BASE,   RGX_GENERAL_NON4K_BRN_65273_HEAP_SIZE,   0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_HEAP_BASE,               RGX_PDSCODEDATA_HEAP_SIZE,               RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_BRN_65273_HEAP_BASE,     RGX_PDSCODEDATA_BRN_65273_HEAP_SIZE,     RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_RGNHDR_BRN_63142_HEAP_IDENT,   RGX_RGNHDR_BRN_63142_HEAP_BASE,          RGX_RGNHDR_BRN_63142_HEAP_SIZE,          0,                                           0,                  BRN63142IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_HEAP_BASE,                   RGX_USCCODE_HEAP_SIZE,                   RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_BRN_65273_HEAP_BASE,         RGX_USCCODE_BRN_65273_HEAP_SIZE,         RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_BRN_ALT_VALUE },
+	{RGX_TQ3DPARAMETERS_HEAP_IDENT,     RGX_TQ3DPARAMETERS_HEAP_BASE,            RGX_TQ3DPARAMETERS_HEAP_SIZE,            0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TQ3DPARAMETERS_HEAP_IDENT,     RGX_TQ3DPARAMETERS_BRN_65273_HEAP_BASE,  RGX_TQ3DPARAMETERS_BRN_65273_HEAP_SIZE,  0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_VK_CAPT_REPLAY_HEAP_IDENT,     RGX_VK_CAPT_REPLAY_HEAP_BASE,            RGX_VK_CAPT_REPLAY_HEAP_SIZE,            0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_HEAP_IDENT,              RGX_FBCDC_HEAP_BASE,                     RGX_FBCDC_HEAP_SIZE,                     0,                                           0,                  FBCDescriptorIsPresent,      NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_FBCDC_LARGE_HEAP_IDENT,        RGX_FBCDC_LARGE_HEAP_BASE,               RGX_FBCDC_LARGE_HEAP_SIZE,               0,                                           0,                  FBCLargeDescriptorIsPresent, NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_CMP_MISSION_RMW_HEAP_IDENT,    RGX_CMP_MISSION_RMW_HEAP_BASE,           RGX_CMP_MISSION_RMW_HEAP_SIZE,           0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_SAFETY_RMW_HEAP_IDENT,     RGX_CMP_SAFETY_RMW_HEAP_BASE,            RGX_CMP_SAFETY_RMW_HEAP_SIZE,            0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TEXTURE_STATE_HEAP_IDENT,      RGX_TEXTURE_STATE_HEAP_BASE,             RGX_TEXTURE_STATE_HEAP_SIZE,             0,                                           0,                  TextureStateIsPresent,       NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_HEAP_BASE,           RGX_VISIBILITY_TEST_HEAP_SIZE,           0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_BRN_65273_HEAP_BASE, RGX_VISIBILITY_TEST_BRN_65273_HEAP_SIZE, 0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_MMU_INIA_BRN_65273_HEAP_IDENT, RGX_MMU_INIA_BRN_65273_HEAP_BASE,        RGX_MMU_INIA_BRN_65273_HEAP_SIZE,        0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+	{RGX_MMU_INIB_BRN_65273_HEAP_IDENT, RGX_MMU_INIB_BRN_65273_HEAP_BASE,        RGX_MMU_INIB_BRN_65273_HEAP_SIZE,        0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+};
+
+static RGX_HEAP_INFO gasRGXHeapLayoutFW[] =
+{
+	/* Name                          HeapBase                        HeapLength                                      HeapReservedRegionLength Log2ImportAlignment pfnIsHeapPresent     pfnDynamicBaseSize pfnInit pfnDeInit HeapInstanceFlags*/
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE,            0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL,   0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101, 0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_BRN_ALT_VALUE},
+	{RGX_FIRMWARE_CONFIG_HEAP_IDENT, RGX_FIRMWARE_CONFIG_HEAP_BASE,  RGX_FIRMWARE_CONFIG_HEAP_SIZE,                  0,                       0,                  FWVZConfigPresent,   NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+};
+
+static INLINE IMG_BOOL IsFwHeapLayout(const RGX_HEAP_INFO *psHeapInfo)
+{
+	return psHeapInfo->pszName[0] == 'F' &&
+	       psHeapInfo->pszName[1] == 'w' ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE void CheckHeapAlignment(const RGX_HEAP_INFO *psHeapInfo,
+                                      const PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 uiAlignment = RGX_HEAP_BASE_SIZE_ALIGN - 1;
+
+	if (IsFwHeapLayout(psHeapInfo) ||
+	    RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/*
+		 * 1) Main heap starts at 2MB offset 0xEC10000000UL
+		 * 2) Config Sub heap is created at the end of the main heap making the entire unit start and end at 2MB offset aligned,
+		 *    these 2 heaps will always have the same page size
+		 *    There are no other heaps in between these two heaps, there is no risk for another devmem heap be created between them.
+		 */
+		return;
+	}
+
+	/* General SVM Heap must be placed below 2MiB boundary so we need to adjust
+	 * the validity condition. This is because an OS might return virtual
+	 * addresses below 2MiB threshold. By default (based on testing on Linux)
+	 * this is around 32KiB. */
+	if (OSStringNCompare(psHeapInfo->pszName, RGX_GENERAL_SVM_HEAP_IDENT,
+	                     sizeof(RGX_GENERAL_SVM_HEAP_IDENT)) == 0)
+	{
+		uiAlignment = RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT - 1;
+	}
+
+	/* All UM accessible heap bases should be aligned to 2MB */
+	if (psHeapInfo->ui64HeapBase & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Base not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Base: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Base (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->ui64HeapBase,
+		         psHeapInfo->ui64HeapBase,
+		         uiAlignment + 1));
+	}
+
+	/* All UM accessible heaps should also be size aligned to 2MB */
+	if (psHeapInfo->uiHeapLength & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Size not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Size (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_SIZE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->uiHeapLength,
+		         psHeapInfo->uiHeapLength,
+		         uiAlignment + 1));
+	}
+}
+
+/* Generic counting method. */
+static void _CountRequiredHeaps(PVRSRV_RGXDEV_INFO  *psDevInfo,
+	                            const RGX_HEAP_INFO  pksHeapInfo[],
+	                            IMG_UINT32           ui32HeapListSize,
+	                            IMG_UINT32*          ui32HeapCount)
+{
+	IMG_UINT32 i;
+
+	/* Loop over rows in the heap data array using callback to decide if we
+	 * should include the heap
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo = &pksHeapInfo[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		(*ui32HeapCount)++;
+	}
+}
+/* Generic heap instantiator */
+static void _InstantiateRequiredHeaps(PVRSRV_RGXDEV_INFO     *psDevInfo,
+                                      RGX_HEAP_INFO           psHeapInfos[],
+                                      IMG_UINT32              ui32HeapListSize,
+                                      const IMG_UINT32        ui32Log2RgxDefaultPageShift,
+                                      DEVMEM_HEAP_BLUEPRINT **psDeviceMemoryHeapCursor)
+{
+	IMG_UINT32 i;
+
+#if defined DEBUG
+	IMG_UINT32 ui32heapListCnt;
+	bool bHeapPageSizeMisMatch = false;
+
+	/*
+	 * To ensure all heaps within a 2MB region have the same page sizes
+	 */
+	for (ui32heapListCnt = 0; ui32heapListCnt < (ui32HeapListSize-1); ui32heapListCnt++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo1 = &psHeapInfos[ui32heapListCnt];
+		const RGX_HEAP_INFO *psHeapInfo2 = &psHeapInfos[ui32heapListCnt+1];
+
+		if (((psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) &&
+			((psHeapInfo1->ui64HeapBase + psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) ==
+			((psHeapInfo2->ui64HeapBase) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)))
+		{
+			if (psHeapInfo1->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				if (!(psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG))
+				{
+					bHeapPageSizeMisMatch = true;
+				}
+			}
+			else if (psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				bHeapPageSizeMisMatch = true;
+			}
+		}
+		if (bHeapPageSizeMisMatch)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Two Heap with Different Page Size allocated in the same PD space(2MB)\n"
+			         "Invalid Heaps 1) \"%s\" and 2) \"%s\"",
+			         __func__,
+			         psHeapInfo1->pszName,
+			         psHeapInfo2->pszName));
+		}
+	}
+#endif
+
+	/* We now have a list of the heaps to include and so we should loop over this
+	 * list and instantiate.
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		IMG_UINT32 ui32Log2DataPageSize = 0;
+		RGX_HEAP_INFO *psHeapInfo = &psHeapInfos[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		if (psHeapInfo->pfnDynamicBaseSize != NULL)
+		{
+			psHeapInfo->pfnDynamicBaseSize(psDevInfo->psDeviceNode, psHeapInfo);
+		}
+
+		if (psHeapInfo->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+		{
+			ui32Log2DataPageSize = psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2;
+		}
+		else
+		{
+			ui32Log2DataPageSize = ui32Log2RgxDefaultPageShift;
+		}
+
+		CheckHeapAlignment(psHeapInfo, psDevInfo);
+
+		HeapCfgBlueprintInit(psHeapInfo->pszName,
+		                     psHeapInfo->ui64HeapBase,
+		                     psHeapInfo->uiHeapLength,
+		                     psHeapInfo->uiHeapReservedRegionLength,
+		                     ui32Log2DataPageSize,
+		                     psHeapInfo->ui32Log2ImportAlignment,
+		                     psHeapInfo->pfnInit,
+		                     psHeapInfo->pfnDeInit,
+		                     *psDeviceMemoryHeapCursor);
+
+		(*psDeviceMemoryHeapCursor)++;
+	}
+}
+
+static PVRSRV_ERROR RGXInitHeaps(PVRSRV_RGXDEV_INFO *psDevInfo,
+	                             DEVICE_MEMORY_INFO *psNewMemoryInfo)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor;
+
+	IMG_UINT32 ui32AppHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutApp);
+	IMG_UINT32 ui32FWHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutFW);
+	IMG_UINT32 ui32CountedHeapSize;
+
+	IMG_UINT32 ui32AppHeapCount = 0;
+	IMG_UINT32 ui32FWHeapCount = 0;
+
+	IMG_UINT32 ui32Log2DefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	if (ui32Log2DefaultPageShift == 0)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2DefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2DefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e0;
+		}
+	}
+#endif
+
+	/* Count heaps required for the app heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutApp,
+		                ui32AppHeapListSize,
+		                &ui32AppHeapCount);
+
+	/* Count heaps required for the FW heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutFW,
+		                ui32FWHeapListSize,
+		                &ui32FWHeapCount);
+
+	ui32CountedHeapSize = (ui32AppHeapCount + ui32FWHeapCount + RGX_NUM_DRIVERS_SUPPORTED);
+
+	psNewMemoryInfo->psDeviceMemoryHeap = OSAllocMem(sizeof(DEVMEM_HEAP_BLUEPRINT) * ui32CountedHeapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeap, eError, e0);
+
+	/* Initialise the heaps */
+	psDeviceMemoryHeapCursor = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	/* Instantiate App Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutApp,
+	                          ui32AppHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* Instantiate FW Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutFW,
+	                          ui32FWHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* set the heap count */
+	psNewMemoryInfo->ui32HeapCount = (IMG_UINT32)(psDeviceMemoryHeapCursor - psNewMemoryInfo->psDeviceMemoryHeap);
+
+	/* Check we have allocated the correct # of heaps, minus any VZ heaps as these
+	 * have not been created at this point
+	 */
+	PVR_ASSERT(psNewMemoryInfo->ui32HeapCount == (ui32CountedHeapSize - RGX_NUM_DRIVERS_SUPPORTED));
+
+	/*
+	   In the new heap setup, we initialise 2 configurations:
+		1 - One will be for the firmware only (index 1 in array)
+			a. This primarily has the firmware heap in it.
+			b. It also has additional guest OSID firmware heap(s)
+				- Only if the number of support firmware OSID > 1
+		2 - Others shall be for clients only (index 0 in array)
+			a. This has all the other client heaps in it.
+	 */
+	psNewMemoryInfo->uiNumHeapConfigs = 2;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray = OSAllocMem(sizeof(DEVMEM_HEAP_CONFIG) * psNewMemoryInfo->uiNumHeapConfigs);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeapConfigArray, eError, e1);
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].pszName = "Default Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].uiNumHeaps = psNewMemoryInfo->ui32HeapCount - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].psHeapBlueprintArray = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].pszName = "Firmware Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps = RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].psHeapBlueprintArray = psDeviceMemoryHeapCursor - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4)
+	{
+		IMG_UINT32 i;
+		const IMG_UINT32 ui32GeneralNon4KHeapPageSize = (1 << psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2);
+		const IMG_UINT32 ui32RgxDefaultPageSize = (1 << RGXHeapDerivePageSize(OSGetPageShift()));
+
+		/*
+		 * Initialise all MMU Page Size Range Config register to the default page size
+		 * used by the OS, leaving the address range 0;
+		 */
+		for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[i] =
+					RGXMMUInit_GetConfigRangeValue(ui32RgxDefaultPageSize,
+												   0,
+												   (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+		}
+
+
+		/* set the last MMU config range covering the entire virtual memory to the OS's page size */
+		psDevInfo->aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES - 1] =
+				RGXMMUInit_GetConfigRangeValue(ui32RgxDefaultPageSize, 0, (1ULL << 40));
+
+		/*
+		 * If the Non4K heap has a different page size than the OS's page size
+		 * (used as default for all other heaps), configure one MMU config range
+		 * for the Non4K heap
+		 */
+		if (ui32GeneralNon4KHeapPageSize != ui32RgxDefaultPageSize)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[0] =
+					RGXMMUInit_GetConfigRangeValue(ui32GeneralNon4KHeapPageSize,
+												   RGX_GENERAL_NON4K_HEAP_BASE,
+												   RGX_GENERAL_NON4K_HEAP_SIZE);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		/* Create additional raw firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVINFO, psDevInfo)
+		{
+			eError = RGXInitFwRawHeap(psDevInfo, psDeviceMemoryHeapCursor, ui32DriverID);
+			if (eError != PVRSRV_OK)
+			{
+				/* if any allocation fails, free previously allocated heaps and abandon initialisation */
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+					psDeviceMemoryHeapCursor--;
+				}
+				goto e1;
+			}
+
+			/* Append additional firmware heaps to host driver firmware context heap configuration */
+			psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps += 1;
+
+			/* advance to the next heap */
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	return PVRSRV_OK;
+e1:
+	OSFreeMem(psNewMemoryInfo->psDeviceMemoryHeap);
+e0:
+	return eError;
+}
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+		DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor = psDevMemoryInfo->psDeviceMemoryHeap;
+
+		/* Delete all guest firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeapConfigArray);
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeap);
+}
+
+static PVRSRV_ERROR RGXInitSharedFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psSysHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* VZ heap validation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg != NULL,
+								"FW Main heap is required for VZ Guest.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+#endif
+
+	if (psSysHeapCfg != NULL)
+	{
+		/* Check FW_SHARED for multiple usage flags. Because FW_SHARED is divided
+		   into subheaps, shared usage with other heaps is not allowed.  */
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_SHARED,
+								"FW_SHARED phys heap config not specified with more than one usage."
+								"FW_SHARED heap must be exclusively used as FW_SHARED.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	if (psSysHeapCfg == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap not set", __func__));
+		/* Nothing to do. Default to the physheap fallback option */
+	}
+	else if (psSysHeapCfg->eType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses OS System memory (UMA)", __func__));
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psSysHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG] = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	}
+	else /* PHYS_HEAP_TYPE_LMA or PHYS_HEAP_TYPE_DMA */
+	{
+		PHYS_HEAP_CONFIG sFwMainHeapCfg, sFwCfgHeapCfg;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses local memory managed by the driver (LMA)", __func__));
+
+
+		/* Subheap layout: Main + (optional MIPS reserved range) + Config */
+		sFwMainHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		/* Reserve space for the Config heap */
+		sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+			/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)))
+			{
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+				if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101;
+				}
+				else
+#endif
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL;
+				}
+			}
+		}
+#endif
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwMainHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwMainHeapCfg,
+		                              "Fw Main subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:MAIN", ErrorDeinit);
+
+		sFwCfgHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		sFwCfgHeapCfg.uConfig.sLMA.sStartAddr.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+		sFwCfgHeapCfg.uConfig.sLMA.sCardBase.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		sFwCfgHeapCfg.uConfig.sLMA.uiSize = RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwCfgHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwCfgHeapCfg,
+		                              "Fw Cfg subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:CFG", ErrorDeinit);
+	}
+
+	/* Acquire FW heaps */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_MAIN, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_MAIN", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CONFIG, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CONFIG", ErrorDeinit);
+
+	return eError;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitPrivateFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psFwCodeHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_CODE);
+	PHYS_HEAP_CONFIG *psFwDataHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIV_DATA);
+	PHYS_HEAP_CONFIG *psFwPrivateHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIVATE);
+	PHYS_HEAP_CONFIG sFwPrivateTempCfg;
+
+	if (psFwPrivateHeapCfg != NULL)
+	{
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg == NULL) && (psFwDataHeapCfg == NULL),
+								"FW_PRIVATE and the FW_CODE & FW_PRIV_DATA usage flags "
+								"achieve the same goal and are mutually exclusive.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Fw code and data are both allocated from this unified heap */
+		sFwPrivateTempCfg = *psFwPrivateHeapCfg;
+		sFwPrivateTempCfg.ui32UsageFlags = PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+		psFwCodeHeapCfg = &sFwPrivateTempCfg;
+		psFwDataHeapCfg = &sFwPrivateTempCfg;
+	}
+
+	if ((psFwCodeHeapCfg == NULL) || (psFwDataHeapCfg == NULL))
+	{
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware code and data heaps must be either both defined
+			 * or both undefined. There is no point in isolating one but not
+			 * the other.*/
+			eError = PVRSRV_ERROR_PHYSHEAP_CONFIG;
+			PVR_LOG_GOTO_IF_ERROR(eError, "PrivateFwPhysHeap check", ErrorDeinit);
+		}
+		else
+		{
+			/* No dedicated heaps, default to the physheap fallback option */
+		}
+	}
+	else if (psFwCodeHeapCfg == psFwDataHeapCfg)
+	{
+		if (psFwCodeHeapCfg->ui32UsageFlags ==
+			(PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA))
+		{
+			/* Fw code and private data allocations come from the same system heap
+			 * Instantiate one physheap and share it between them. */
+
+			eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+												  psFwCodeHeapCfg,
+												  NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig");
+		}
+		else
+		{
+			/* Not an exclusive heap, can be used for other purposes (e.g. secure buffers).
+			 * Expect the PVR layer to have already created a heap for the other uses. */
+		}
+	}
+	else
+	{
+		/*
+		 * Separating private Firmware code and data is allowed for backwards compatibility
+		 * purposes. New platforms should use the unified FW_PRIVATE heap instead.
+		 *
+		 * Early security implementations on Rogue cores required separate FW_PRIV_DATA
+		 * and FW_CODE heaps, as access permissions to Firmware were granted differently
+		 * based on the transaction types (code or data).
+		 */
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_CODE) &&
+								(psFwDataHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PRIV_DATA),
+								"Dedicated private heaps for Fw code and "
+								"data must have one usage flag exclusively.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Dedicated Fw code heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwCodeHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		/* Dedicated Fw private data heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwDataHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+	/* When premapping distinct private and shared Firmware phys heaps
+	 * inside the same virtual devmem heap, their sizes must add up to
+	 * the fixed RGX_FIRMWARE_RAW_HEAP_SIZE for the premapping to work */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwSharedHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		PHYS_HEAP_USAGE_FW_SHARED);
+		IMG_UINT64 ui64FwCodeHeapSize = PhysHeapConfigGetSize(psFwCodeHeapCfg);
+		IMG_UINT64 ui64FwDataHeapSize = PhysHeapConfigGetSize(psFwDataHeapCfg);
+		IMG_UINT64 ui64FwSharedHeapSize = PhysHeapConfigGetSize(psFwSharedHeapCfg);
+		IMG_UINT64 ui64FwPrivateHeapSize;
+
+		PVR_LOG_GOTO_IF_FALSE((psFwCodeHeapCfg != NULL) && (psFwDataHeapCfg != NULL),
+							  "Security support requires Fw code and data memory be"
+							  " separate from the heap shared with the kernel driver.", FailDeinit);
+
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware allocations come from 2 different heaps */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize + ui64FwDataHeapSize;
+		}
+		else
+		{
+			/* Private Firmware allocations come from a single heap */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize;
+		}
+
+		PVR_LOG_GOTO_IF_FALSE((ui64FwSharedHeapSize + ui64FwPrivateHeapSize) == RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                      "Invalid firmware physical heap size.", FailDeinit);
+	}
+#endif
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CODE, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CODE]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CODE", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_PRIV_DATA, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PRIV_DATA]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_DATA", ErrorDeinit);
+
+	return eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+FailDeinit:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitFwPageTableHeap(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwPageTableHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		  PHYS_HEAP_USAGE_FW_PREMAP_PT);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg != NULL),
+								"The Firmware Page Table phys heap config not found.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PREMAP_PT),
+								"The Firmware Page Table heap must be used exclusively for this purpose",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_LMA) ||
+								(psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_DMA),
+								"The Firmware Page Table heap must be LMA or DMA memory.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->uConfig.sLMA.uiSize >= RGX_FIRMWARE_MAX_PAGETABLE_SIZE),
+								"The Firmware Page Table heap must be able to hold the maximum "
+								"number of pagetables needed to cover the Firmware's VA space.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwPageTableHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig:FwPageTableHeap");
+
+		eError = PhysHeapAcquire(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquire:FwPageTableHeap");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) */
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXInitFwPageTableHeap(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitFwPageTableHeap", ErrorDeinit);
+	eError = RGXInitSharedFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitSharedFwPhysHeaps", ErrorDeinit);
+	eError = RGXInitPrivateFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitPrivateFwPhysHeaps", ErrorDeinit);
+
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDeviceFWMainHeapMemCheck
+@Description    Checks the free memory in FW Main PhysHeap of a device to ensure
+                there is enough for a connection to be made.
+
+@Input          psDeviceNode    The device of the FW Main PhysHeap to be checked.
+
+@Return         On success PVRSRV_OK, else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDeviceFWMainHeapMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP *psFWMainPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psFWMainPhysHeap = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	if (psFWMainPhysHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's FW Main PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psFWMainPhysHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		const IMG_UINT32 ui32MinMemInKBs = RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION;
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psFWMainPhysHeap,
+		                              KB2B(ui32MinMemInKBs),
+		                              &ui64FreePhysHeapMem);
+
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FW_MAIN PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"KB "
+				"Minimum required: %uKB",
+				B2KB(ui64FreePhysHeapMem),
+				ui32MinMemInKBs));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate, IMG_UINT32 *pui32Log2Non4KPgShift)
+{
+	IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+	IMG_UINT32 uiLog2OSPageShift = OSGetPageShift();
+
+	/* We support Non4K pages only on platforms with 4KB pages. On all platforms
+	 * where OS pages are larger than 4KB we must ensure the non4K device memory
+	 * heap matches the page size used in all other device memory heaps, which
+	 * is the OS page size, see RGXHeapDerivePageSize. */
+	if (uiLog2OSPageShift > RGX_HEAP_4KB_PAGE_SHIFT)
+	{
+		*pui32Log2Non4KPgShift = RGXHeapDerivePageSize(uiLog2OSPageShift);
+	}
+	else
+	{
+		void *pvAppHintState = NULL;
+		IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+		IMG_UINT32 ui32Log2Non4KPgShift;
+
+		/* Get the page size for the dummy page from the NON4K heap apphint */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState,
+		                   GeneralNon4KHeapPageSize, &ui32AppHintDefault,
+		                   &ui32GeneralNon4KHeapPageSize);
+		OSFreeAppHintState(pvAppHintState);
+
+		/* Validate the specified parameter to be one of the supported values */
+		ui32Log2Non4KPgShift = RGXHeapDerivePageSize(ExactLog2(ui32GeneralNon4KHeapPageSize));
+		if (ui32AppHintDefault != ui32GeneralNon4KHeapPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid Non4K Page-size, default=%u, requested=%u,"
+			         " actual=%u, pageshift 0x%x",
+			         __func__, ui32AppHintDefault, ui32GeneralNon4KHeapPageSize,
+			         (1U << ui32Log2Non4KPgShift), ui32Log2Non4KPgShift));
+		}
+
+		if (ui32Log2Non4KPgShift == 0U)
+		{
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+
+		*pui32Log2Non4KPgShift = ui32Log2Non4KPgShift;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 71317))
+	{
+		if (*pui32Log2Non4KPgShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || *pui32Log2Non4KPgShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Page sizes of 2MB or 1MB cause page faults."));
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+#endif
+
+	/* Check the Non4k page size is at least the size of the OS page size
+	 * or larger. The Non4k page size also has to be a multiple of the OS page
+	 * size but since we have the log2 value from the apphint we know powers of 2
+	 * will always be multiples. If the Non4k page size is less than OS page size
+	 * we notify and upgrade the size.
+	 */
+	if (*pui32Log2Non4KPgShift < uiLog2OSPageShift)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "Non4K page size smaller than OS page size, upgrading to "
+		                          "match OS page size."));
+		*pui32Log2Non4KPgShift = uiLog2OSPageShift;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* RGXRegisterDevice
+ *
+ * WARNING!
+ *
+ * No PDUMP statements are allowed until device initialisation starts.
+ */
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	DEVICE_MEMORY_INFO *psDevMemoryInfo;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = HWPERF_HOST_TL_STREAM_SIZE_DEFAULT, ui32HWPerfHostBufSizeKB;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault, &ui32HWPerfHostBufSizeKB);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/*********************
+	 * Device node setup *
+	 *********************/
+	/* Setup static data and callbacks on the device agnostic device node */
+#if defined(PDUMP)
+	psDeviceNode->sDevId.pszPDumpRegName	= RGX_PDUMPREG_NAME;
+	psDeviceNode->sDevId.pszPDumpDevName	= PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+	psDeviceNode->pfnPDumpInitDevice = &RGXResetPDump;
+#endif /* PDUMP */
+
+	OSAtomicWrite(&psDeviceNode->eHealthStatus, PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	OSAtomicWrite(&psDeviceNode->eHealthReason, PVRSRV_DEVICE_HEALTH_REASON_NONE);
+
+	/* Configure MMU specific stuff */
+	RGXMMUInit_Register(psDeviceNode);
+
+	psDeviceNode->pfnInvalFBSCTable = NULL;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = NULL;
+
+	/* Callback for getting the MMU device attributes */
+	psDeviceNode->pfnGetMMUDeviceAttributes = RGXDevMMUAttributes;
+	psDeviceNode->pfnGetDeviceSnoopMode = RGXDevSnoopMode;
+	psDeviceNode->pfnMMUCacheInvalidate = RGXMMUCacheInvalidate;
+	psDeviceNode->pfnMMUCacheInvalidateKick = RGXMMUCacheInvalidateKick;
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = RGXMapBRN71422TargetPhysicalAddress;
+#else
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = NULL;
+#endif
+	/* pfnMMUTweakProtFlags is set later on once BVNC features are setup */
+	psDeviceNode->pfnMMUTweakProtFlags = NULL;
+
+	psDeviceNode->pfnInitDeviceCompatCheck	= &RGXDevInitCompatCheck;
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	/* Register callbacks for Unified Fence Objects */
+	psDeviceNode->pfnAllocUFOBlock = RGXAllocUFOBlock;
+	psDeviceNode->pfnFreeUFOBlock = RGXFreeUFOBlock;
+
+	/* Register callback for checking the device's health */
+	psDeviceNode->pfnUpdateHealthStatus = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? NULL : RGXUpdateHealthStatus;
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Register callback for updating the virtualization watchdog */
+	psDeviceNode->pfnUpdateAutoVzWatchdog = RGXUpdateAutoVzWatchdog;
+#endif
+
+	/* Register method to service the FW HWPerf buffer */
+	psDeviceNode->pfnServiceHWPerf = RGXHWPerfDataStoreCB;
+
+	/* Register callback for getting the device version information string */
+	psDeviceNode->pfnDeviceVersionString = RGXDevVersionString;
+
+	/* Register callback for getting the device clock speed */
+	psDeviceNode->pfnDeviceClockSpeed = RGXDevClockSpeed;
+
+	/* Register callback for soft resetting some device modules */
+	psDeviceNode->pfnSoftReset = RGXSoftReset;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnResetHWRLogs = RGXResetHWRLogs;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnVerifyBVNC = RGXVerifyBVNC;
+
+	/* Register callback for checking alignment of UM structures */
+	psDeviceNode->pfnAlignmentCheck = RGXAlignmentCheck;
+
+	/*Register callback for checking the supported features and getting the
+	 * corresponding values */
+	psDeviceNode->pfnCheckDeviceFeature = RGXBvncCheckFeatureSupported;
+	psDeviceNode->pfnGetDeviceFeatureValue = RGXBvncGetSupportedFeatureValue;
+
+	/* Callback for checking if system layer supports FBC 3.1 */
+	psDeviceNode->pfnHasFBCDCVersion31 = RGXSystemHasFBCDCVersion31;
+
+	/* Callback for getting TFBC configuration */
+	psDeviceNode->pfnGetTFBCLossyGroup = RGXGetTFBCLossyGroup;
+
+	/* Register callback for initialising device-specific physical memory heaps */
+	psDeviceNode->pfnPhysMemDeviceHeapsInit = RGXPhysMemDeviceHeapsInit;
+
+	/* Register callback for checking a device's FW Main physical heap for sufficient free memory */
+	psDeviceNode->pfnCheckForSufficientFWPhysMem = RGXDeviceFWMainHeapMemCheck;
+
+	/* Register callback for determining the appropriate LMA allocation policy for a phys heap */
+	psDeviceNode->pfnPhysHeapGetLMAPolicy = RGXPhysHeapGetLMAPolicy;
+
+	/*********************
+	 * Device info setup *
+	 *********************/
+	/* Allocate device control block */
+	psDevInfo = OSAllocZMem(sizeof(*psDevInfo));
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevInitRGXPart1 : Failed to alloc memory for DevInfo"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* initialise the layer parameters needed for early hw feature checks */
+	psDevInfo->sLayerParams.psDevInfo = psDevInfo;
+	psDevInfo->sLayerParams.psDevConfig = psDeviceNode->psDevConfig;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Default psTrampoline to point to null struct */
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+#endif
+
+	/* create locks for the context lists stored in the DevInfo structure.
+	 * these lists are modified on context create/destroy and read by the
+	 * watchdog thread
+	 */
+
+	eError = OSWRLockCreate(&(psDevInfo->hRenderCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create render context list lock", __func__));
+		goto e0;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hComputeCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create compute context list lock", __func__));
+		goto e1;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTransferCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create transfer context list lock", __func__));
+		goto e2;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTDMCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create TDM context list lock", __func__));
+		goto e3;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hKickSyncCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create kick sync context list lock", __func__));
+		goto e4;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hMemoryCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create memory context list lock", __func__));
+		goto e5;
+	}
+
+	eError = OSSpinLockCreate(&psDevInfo->hLockKCCBDeferredCommandsList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to KCCB deferred commands list lock", __func__));
+		goto e6;
+	}
+	dllist_init(&(psDevInfo->sKCCBDeferredCommandsListHead));
+
+	dllist_init(&(psDevInfo->sRenderCtxtListHead));
+	dllist_init(&(psDevInfo->sComputeCtxtListHead));
+	dllist_init(&(psDevInfo->sTransferCtxtListHead));
+	dllist_init(&(psDevInfo->sTDMCtxtListHead));
+	dllist_init(&(psDevInfo->sKickSyncCtxtListHead));
+
+	dllist_init(&(psDevInfo->sCommonCtxtListHead));
+	psDevInfo->ui32CommonCtxtCurrentID = 1;
+
+
+	eError = OSWRLockCreate(&psDevInfo->hCommonCtxtListLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create common context list lock", __func__));
+		goto e7;
+	}
+
+	eError = OSLockCreate(&psDevInfo->sRegConfig.hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create register configuration lock", __func__));
+		goto e8;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hBPLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for break points", __func__));
+		goto e9;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hRGXFWIfBufInitLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for trace buffers", __func__));
+		goto e10;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hCCBStallCheckLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB checking lock", __func__));
+		goto e11;
+	}
+	eError = OSLockCreate(&psDevInfo->hCCBRecoveryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB recovery lock", __func__));
+		goto e12;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hGpuUtilStatsLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create GPU stats lock", __func__));
+		goto e13;
+	}
+
+	dllist_init(&psDevInfo->sMemoryContextList);
+
+	/* initialise ui32SLRHoldoffCounter */
+	if (RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS > DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT)
+	{
+		psDevInfo->ui32SLRHoldoffCounter = RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS / DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	}
+	else
+	{
+		psDevInfo->ui32SLRHoldoffCounter = 0;
+	}
+
+	/* Setup static data and callbacks on the device specific device info */
+	psDevInfo->psDeviceNode		= psDeviceNode;
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+	psDevInfo->pvDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
+
+	/*
+	 * Map RGX Registers
+	 */
+	psDevInfo->ui32RegSize = psDeviceNode->psDevConfig->ui32RegsSize;
+	psDevInfo->sRegsPhysBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+#if !defined(NO_HARDWARE)
+	psDevInfo->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create RGX register mapping",
+		         __func__));
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto e14;
+	}
+#endif /* !NO_HARDWARE */
+
+	psDeviceNode->pvDevice = psDevInfo;
+
+	eError = RGXBvncInitialiseConfiguration(psDeviceNode);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Unsupported HW device detected by driver",
+		         __func__));
+		goto e15;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/*
+	 * We must now setup the SECURITY mappings if supported. We cannot
+	 * check on the features until we have reached here as the BVNC is
+	 * not setup before now.
+	 */
+#if !defined(NO_HARDWARE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+	{
+		IMG_CPU_PHYADDR sHostSecureRegBankBase = {psDeviceNode->psDevConfig->sRegsCpuPBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET};
+
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *) OSMapPhysToLin(sHostSecureRegBankBase,
+																		RGX_HOST_SECURE_REGBANK_SIZE,
+																		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (psDevInfo->pvSecureRegsBaseKM == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "PVRSRVRGXInitDevPart2KM: Failed to create RGX secure register mapping"));
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			goto e15;
+		}
+
+		/*
+		 * The secure register bank is mapped into the CPU VA space starting from
+		 * the base of the normal register bank + an offset of RGX_HOST_SECURE_REGBAK_OFFSET.
+		 * The hardware register addresses are all indexed from the base of the regular register bank.
+		 * For the RegBankBase+RegOffset computation to still be accurate for host-secure registers,
+		 * we need to compensate for offsets of registers in the secure bank
+		 */
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM - RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+	else
+	{
+		psDevInfo->pvSecureRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	}
+#endif /* !NO_HARDWARE */
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	psDeviceNode->pfnMMUTweakProtFlags = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			RGXMMUTweakProtFlags : NULL;
+#endif
+
+	eError = RGXGetNon4KHeapPageShift(&psDevInfo->sLayerParams,
+	                                &psDeviceNode->ui32Non4KPageSizeLog2);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXGetNon4KHeapPageSize", e16);
+
+	eError = RGXInitHeaps(psDevInfo, psDevMemoryInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto e16;
+	}
+
+	eError = RGXHWPerfInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInit", e16);
+
+	eError = RGXHWPerfHostInit(psDeviceNode->pvDevice, ui32HWPerfHostBufSizeKB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfHostInit", ErrorDeInitHWPerfFw);
+
+
+	/* Register callback for dumping debug info */
+	eError = RGXDebugInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDebugInit", e17);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Register callback for fw mmu init */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDeviceNode->pfnFwMMUInit = RGXMipsMMUInit_Register;
+	}
+#endif
+
+	if (NULL != psDeviceNode->psDevConfig->pfnSysDevFeatureDepInit)
+	{
+		psDeviceNode->psDevConfig->pfnSysDevFeatureDepInit(psDeviceNode->psDevConfig,
+				psDevInfo->sDevFeatureCfg.ui64Features);
+	}
+
+	/* Initialise the device dependent bridges */
+	eError = RGXRegisterBridges(psDevInfo);
+	PVR_LOG_IF_ERROR(eError, "RGXRegisterBridges");
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = OSLockCreate(&psDevInfo->hCounterDumpingLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for counter sampling.", __func__));
+		goto ErrorDeInitDeviceDepBridge;
+	}
+#endif
+
+	/* Initialise error counters */
+	memset(&psDevInfo->sErrorCounts, 0, sizeof(PVRSRV_RGXDEV_ERROR_COUNTS));
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+ErrorDeInitDeviceDepBridge:
+	RGXUnregisterBridges(psDevInfo);
+#endif
+
+e17:
+	RGXHWPerfHostDeInit(psDevInfo);
+ErrorDeInitHWPerfFw:
+	RGXHWPerfDeinit(psDevInfo);
+e16:
+#if !defined(NO_HARDWARE)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+		/* Adjust pvSecureRegsBaseKM if device has SECURITY_VERSION > 1 */
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		    (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* Undo the VA offset adjustment to unmap correct VAddr */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM,
+			                 RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+	}
+#endif
+#endif /* !NO_HARDWARE */
+e15:
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+		                 psDevInfo->ui32RegSize);
+	}
+e14:
+#endif /* !NO_HARDWARE */
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+e13:
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+e12:
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+e11:
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+e10:
+	OSLockDestroy(psDevInfo->hBPLock);
+e9:
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+e8:
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+e7:
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+e6:
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+e5:
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+e4:
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+e3:
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+e2:
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+e1:
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+e0:
+	OSFreeMem(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_PCHAR psz = psDevInfo->sDevFeatureCfg.pszBVNCString;
+	if (NULL == psz)
+	{
+		IMG_CHAR pszBVNCInfo[RGX_HWPERF_MAX_BVNC_LEN];
+		size_t uiBVNCStringSize;
+		size_t uiStringLength;
+
+		uiStringLength = OSSNPrintf(pszBVNCInfo, RGX_HWPERF_MAX_BVNC_LEN, "%d.%d.%d.%d",
+				psDevInfo->sDevFeatureCfg.ui32B,
+				psDevInfo->sDevFeatureCfg.ui32V,
+				psDevInfo->sDevFeatureCfg.ui32N,
+				psDevInfo->sDevFeatureCfg.ui32C);
+		PVR_ASSERT(uiStringLength < RGX_HWPERF_MAX_BVNC_LEN);
+
+		uiBVNCStringSize = (uiStringLength + 1) * sizeof(IMG_CHAR);
+		psz = OSAllocMem(uiBVNCStringSize);
+		if (NULL != psz)
+		{
+			OSCachedMemCopy(psz, pszBVNCInfo, uiBVNCStringSize);
+			psDevInfo->sDevFeatureCfg.pszBVNCString = psz;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Allocating memory for BVNC Info string failed",
+					__func__));
+		}
+	}
+
+	return psz;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDevVersionString
+@Description    Gets the version string for the given device node and returns
+                a pointer to it in ppszVersionString. It is then the
+                responsibility of the caller to free this memory.
+@Input          psDeviceNode            Device node from which to obtain the
+                                        version string
+@Output	        ppszVersionString	Contains the version string upon return
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR **ppszVersionString)
+{
+#if defined(NO_HARDWARE) || defined(EMULATOR)
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (SW)";
+#else
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (HW)";
+#endif
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_PCHAR pszBVNC;
+	size_t uiStringLength;
+
+	if (psDeviceNode == NULL || ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	pszBVNC = RGXDevBVNCString(psDevInfo);
+
+	if (NULL == pszBVNC)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	uiStringLength = OSStringLength(pszBVNC);
+	uiStringLength += (sizeof(szFormatString) - 2); /* sizeof includes the null, -2 for "%s" */
+	*ppszVersionString = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (*ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSSNPrintf(*ppszVersionString, uiStringLength, szFormatString,
+		pszBVNC);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       RGXDevClockSpeed
+@Description    Gets the clock speed for the given device node and returns
+                it in pui32RGXClockSpeed.
+@Input          psDeviceNode		Device node
+@Output         pui32RGXClockSpeed  Variable for storing the clock speed
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_PUINT32  pui32RGXClockSpeed)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* get clock speed */
+	*pui32RGXClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/*!
+ *******************************************************************************
+
+ @Function		RGXInitFwRawHeap
+
+ @Description	Called to perform additional initialisation
+ ******************************************************************************/
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID)
+{
+	IMG_UINT32 uiStringLength;
+	IMG_UINT32 uiStringLengthMax = 32;
+
+	IMG_UINT32 ui32Log2RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	PVR_RETURN_IF_FALSE(ui32Log2RgxDefaultPageShift != 0, PVRSRV_ERROR_INVALID_PARAMS);
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2RgxDefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2RgxDefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	uiStringLength = MIN(sizeof(RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT), uiStringLengthMax + 1);
+
+	/* Start by allocating memory for this DriverID heap identification string */
+	psDevMemHeap->pszName = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (psDevMemHeap->pszName == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Append the DriverID number to the RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT string */
+	OSSNPrintf((IMG_CHAR *)psDevMemHeap->pszName, uiStringLength, RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	/* Use the common blueprint template support function to initialise the heap */
+	HeapCfgBlueprintInit(psDevMemHeap->pszName,
+		                 RGX_FIRMWARE_RAW_HEAP_BASE + (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE),
+		                 RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                 0,
+		                 ui32Log2RgxDefaultPageShift,
+		                 0,
+		                 NULL,
+		                 NULL,
+		                 psDevMemHeap);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function		RGXDeInitFwRawHeap
+
+ @Description	Called to perform additional deinitialisation
+ ******************************************************************************/
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap)
+{
+	IMG_UINT64 uiBase = RGX_FIRMWARE_RAW_HEAP_BASE + RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT64 uiSpan = uiBase + ((RGX_NUM_DRIVERS_SUPPORTED - 1) * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	/* Safe to do as the guest firmware heaps are last in the list */
+	if (psDevMemHeap->sHeapBaseAddr.uiAddr >= uiBase &&
+	    psDevMemHeap->sHeapBaseAddr.uiAddr < uiSpan)
+	{
+		void *pszName = (void*)psDevMemHeap->pszName;
+		OSFreeMem(pszName);
+	}
+}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/******************************************************************************
+ End of file (rgxinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxlayer_impl.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxlayer_impl.c
new file mode 100644
index 000000000000..535326d7e786
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxlayer_impl.c
@@ -0,0 +1,478 @@
+/*************************************************************************/ /*!
+@File
+@Title          DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwimageutils.h"
+#include "cache_km.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+
+#define MAX_NUM_COHERENCY_TESTS  (10)
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	if (psDevInfo->ui32CoherencyTestsDone >= MAX_NUM_COHERENCY_TESTS)
+	{
+		return IMG_FALSE;
+	}
+
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	return PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig);
+}
+
+/*
+ * The fabric coherency test is performed when platform supports fabric coherency
+ * either in the form of ACE-lite or Full-ACE. This test is done quite early
+ * with the firmware processor quiescent and makes exclusive use of the slave
+ * port interface for reading/writing through the device memory hierarchy. The
+ * rationale for the test is to ensure that what the CPU writes to its dcache
+ * is visible to the GPU via coherency snoop miss/hit and vice-versa without
+ * any intervening cache maintenance by the writing agent.
+ */
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 *pui32FabricCohTestBufferCpuVA;
+	DEVMEM_MEMDESC *psFabricCohTestBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR sFabricCohTestBufferDevVA;
+	IMG_DEVMEM_SIZE_T uiFabricCohTestBlockSize = sizeof(IMG_UINT64);
+	IMG_DEVMEM_ALIGN_T uiFabricCohTestBlockAlign = sizeof(IMG_UINT64);
+	IMG_UINT32 ui32SLCCTRL = 0;
+	IMG_UINT32 ui32OddEven;
+	IMG_UINT32 ui32OddEvenSeed = 1;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bFullTestPassed = IMG_TRUE;
+	IMG_BOOL bExit = IMG_FALSE;
+#if defined(DEBUG)
+	IMG_BOOL bSubTestPassed = IMG_FALSE;
+#endif
+	enum TEST_TYPE {
+		CPU_WRITE_GPU_READ_SM=0, GPU_WRITE_CPU_READ_SM,
+		CPU_WRITE_GPU_READ_SH,   GPU_WRITE_CPU_READ_SH
+	} eTestType;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	PVR_LOG(("Starting fabric coherency test ....."));
+
+	{
+		/* Bypass the SLC when IO coherency is enabled */
+		ui32SLCCTRL = RGXReadReg32(hPrivate, RGX_CR_SLC_CTRL_BYPASS);
+		RGXWriteReg32(hPrivate,
+				RGX_CR_SLC_CTRL_BYPASS,
+				ui32SLCCTRL | RGX_CR_SLC_CTRL_BYPASS_BYP_CC_EN);
+	}
+
+	/* Size and align are 'expanded' because we request an export align allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiFabricCohTestBlockSize,
+	                                             &uiFabricCohTestBlockAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemExportalignAdjustSizeAndAlign() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	/* Allocate, acquire cpu address and set firmware address */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyTestBuffer",
+										&psFabricCohTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohTestBufferMemDesc, (void **) &pui32FabricCohTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e1;
+	}
+
+	/* Create a FW address which is uncached in the Meta DCache and in the SLC using the Meta bootloader segment.
+	   This segment is the only one configured correctly out of reset (when this test is meant to be executed) */
+
+	eError = RGXSetFirmwareAddress(&sFabricCohTestBufferDevVA,
+						  psFabricCohTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", e2);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	for (eTestType = CPU_WRITE_GPU_READ_SH; eTestType <= GPU_WRITE_CPU_READ_SH && bExit == IMG_FALSE; eTestType++)
+	{
+		IMG_CPU_PHYADDR sCpuPhyAddr;
+		IMG_BOOL bValid;
+		PMR *psPMR;
+
+		/* Acquire underlying PMR CpuPA in preparation for cache maintenance */
+		(void) DevmemLocalGetImportHandle(psFabricCohTestBufferMemDesc, (void**)&psPMR);
+		eError = PMR_CpuPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sCpuPhyAddr, &bValid, CPU_USE);
+		if (eError != PVRSRV_OK || bValid == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"PMR_CpuPhysAddr error: %s, exiting",
+					PVRSRVGetErrorString(eError)));
+			bExit = IMG_TRUE;
+			continue;
+		}
+
+		/* Here we do two passes mostly to account for the effects of using a different
+		   seed (i.e. ui32OddEvenSeed) value to read and write */
+		for (ui32OddEven = 1; ui32OddEven < 3 && bExit == IMG_FALSE; ui32OddEven++)
+		{
+			IMG_UINT32 i;
+
+#if defined(DEBUG)
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+
+			/* Do multiple sub-dword cache line tests */
+			for (i = 0; i < 2 && bExit == IMG_FALSE; i++)
+			{
+				IMG_UINT32 ui32FWAddr;
+				IMG_UINT32 ui32FWValue;
+				IMG_UINT32 ui32FWValue2;
+				IMG_CPU_PHYADDR sCpuPhyAddrStart;
+				IMG_CPU_PHYADDR sCpuPhyAddrEnd;
+				IMG_UINT32 ui32LastFWValue = ~0;
+				IMG_UINT32 ui32Offset = i * sizeof(IMG_UINT32);
+
+				/* Calculate next address and seed value to write/read from slave-port */
+				ui32FWAddr = sFabricCohTestBufferDevVA.ui32Addr + ui32Offset;
+				sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + ui32Offset;
+				sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr;
+				ui32OddEvenSeed += 1;
+
+				if (eTestType == GPU_WRITE_CPU_READ_SM || eTestType == GPU_WRITE_CPU_READ_SH)
+				{
+					/* Clean dcache to ensure there is no stale data in dcache that might over-write
+					   what we are about to write via slave-port here because if it drains from the CPU
+					   dcache before we read it, it would corrupt what we are going to read back via
+					   the CPU */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_FLUSH);
+
+					/* Calculate a new value to write */
+					ui32FWValue = i + ui32OddEvenSeed;
+
+					/* Write the value using the RGX slave-port interface */
+					eError = RGXWriteFWModuleAddr(psDevInfo, ui32FWAddr, ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXWriteFWModuleAddr error: %s, exiting",
+						          PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Read back value using RGX slave-port interface, this is used
+					   as a sort of memory barrier for the above write */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue2);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXReadFWModuleAddr error: %s, exiting",
+						         PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+					else if (ui32FWValue != ui32FWValue2)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW via SlavePort failed: expected: %x, got: %x",
+								i,
+								ui32FWValue,
+								ui32FWValue2));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Invalidate dcache to ensure that any prefetched data by the CPU from this memory
+					   region is discarded before we read (i.e. next read must trigger a cache miss).
+					   Previously there was snooping of device cache, where prefetching done by the CPU
+					   would reflect the most up to date datum writing by GPU into said location,
+					   that is to say prefetching was coherent so CPU d-flush was not needed */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_INVALIDATE);
+				}
+				else
+				{
+					IMG_UINT32 ui32RAWCpuValue;
+
+					/* Ensures line is in dcache */
+					ui32FWValue = IMG_UINT32_MAX;
+
+					/* Dirty allocation in dcache */
+					ui32RAWCpuValue = i + ui32OddEvenSeed;
+					pui32FabricCohTestBufferCpuVA[i] = i + ui32OddEvenSeed;
+
+					/* Flush possible cpu store-buffer(ing) on LMA */
+					OSWriteMemoryBarrier(&pui32FabricCohTestBufferCpuVA[i]);
+
+					if (eTestType == CPU_WRITE_GPU_READ_SM)
+					{
+						/* Flush dcache to force subsequent incoming CPU-bound snoop to miss so
+						   memory is coherent before the SlavePort reads */
+						sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+						CacheOpExec(psDevInfo->psDeviceNode,
+								(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+								(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+								sCpuPhyAddrStart,
+								sCpuPhyAddrEnd,
+								PVRSRV_CACHE_OP_FLUSH);
+					}
+
+					/* Read back value using RGX slave-port interface */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"RGXReadFWModuleAddr error: %s, exiting",
+								PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Being mostly paranoid here, verify that CPU RAW operation is valid
+					   after the above slave port read */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_FLUSH);
+					if (pui32FabricCohTestBufferCpuVA[i] != ui32RAWCpuValue)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW by CPU failed: expected: %x, got: %x",
+								i,
+								ui32RAWCpuValue,
+								pui32FabricCohTestBufferCpuVA[i]));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				/* Compare to see if sub-test passed */
+				if (pui32FabricCohTestBufferCpuVA[i] == ui32FWValue)
+				{
+#if defined(DEBUG)
+					bSubTestPassed = IMG_TRUE;
+#endif
+				}
+				else
+				{
+					bFullTestPassed = IMG_FALSE;
+					eError = PVRSRV_ERROR_INIT_FAILURE;
+#if defined(DEBUG)
+					bSubTestPassed = IMG_FALSE;
+#endif
+					if (ui32LastFWValue != ui32FWValue)
+					{
+#if defined(DEBUG)
+						PVR_LOG(("At Offset: %d, Expected: %x, Got: %x",
+								 i,
+								 (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i],
+								 (eTestType & 0x1) ? pui32FabricCohTestBufferCpuVA[i] : ui32FWValue));
+#endif
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"test encountered unexpected error, exiting"));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				ui32LastFWValue = (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i];
+			}
+
+#if defined(DEBUG)
+			if (bExit)
+			{
+				continue;
+			}
+
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+		}
+	}
+
+	RGXUnsetFirmwareAddress(psFabricCohTestBufferMemDesc);
+e2:
+	DevmemReleaseCpuVirtAddr(psFabricCohTestBufferMemDesc);
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohTestBufferMemDesc);
+
+e0:
+	{
+		/* Restore SLC bypass settings */
+		RGXWriteReg32(hPrivate, RGX_CR_SLC_CTRL_BYPASS, ui32SLCCTRL);
+	}
+
+	bFullTestPassed = bExit ? IMG_FALSE: bFullTestPassed;
+	if (bFullTestPassed)
+	{
+		PVR_LOG(("fabric coherency test: PASSED"));
+		psDevInfo->ui32CoherencyTestsDone = MAX_NUM_COHERENCY_TESTS + 1;
+	}
+	else
+	{
+		PVR_LOG(("fabric coherency test: FAILED"));
+		psDevInfo->ui32CoherencyTestsDone++;
+	}
+
+	return eError;
+}
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.c
new file mode 100644
index 000000000000..e0b17bf3dedd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.c
@@ -0,0 +1,1068 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxmipsmmuinit.h"
+
+#include "device.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "mmu_common.h"
+#include "pdump_mmu.h"
+#include "rgxheapconfig.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "rgx_memallocflags.h"
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "log2.h"
+
+/*
+ * Bits of PT, PD and PC not involving addresses
+ */
+
+/* Currently there is no page directory for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PDE_PROTMASK        0
+/* Currently there is no page catalog for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PCE_PROTMASK	     0
+
+
+static MMU_PxE_CONFIG sRGXMMUPCEConfig;
+static MMU_DEVVADDR_CONFIG sRGXMMUTopLevelDevVAddrConfig;
+
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_4KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_4KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_4KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig4KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_16KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_16KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_16KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig16KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_64KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_64KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_64KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig64KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_256KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_256KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig256KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 1MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_1MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_1MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_1MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig1MB;
+
+
+/*
+ *
+ *  Configuration for heaps with 2MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_2MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_2MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_2MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig2MB;
+
+
+/* Forward declaration of protection bits derivation functions, for
+   the following structure */
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags);
+
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+										   const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+										   const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+										   const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+										   IMG_HANDLE *phPriv);
+
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv);
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+
+static MMU_DEVICEATTRIBS sRGXMMUDeviceAttributes;
+
+/* Cached policy */
+static IMG_UINT32 gui32CachedPolicy;
+
+static PVRSRV_ERROR RGXCheckTrampolineAddrs(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                            MMU_DEVICEATTRIBS *psDevAttrs,
+                                            IMG_UINT64 *pui64Addr);
+
+PVRSRV_ERROR RGXMipsMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bPhysBusAbove32Bit = 0;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PHYS_BUS_WIDTH))
+	{
+		bPhysBusAbove32Bit = RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32;
+	}
+
+	sRGXMMUDeviceAttributes.pszMMUPxPDumpMemSpaceName =
+		PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+
+	/*
+	 * Setup sRGXMMUPCEConfig, no PC in MIPS MMU currently
+	 */
+	sRGXMMUPCEConfig.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPCEConfig.pszPxLevelStr = "UnD";
+	sRGXMMUPCEConfig.uiBytesPerEntry = 0; /* 32 bit entries */
+
+	sRGXMMUPCEConfig.uiAddrMask = 0; /* Mask to get significant address bits of PC entry */
+	sRGXMMUPCEConfig.uiAddrShift = 0; /* Shift this many bits to get PD address in PC entry */
+	sRGXMMUPCEConfig.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K; /* Alignment of PD AND PC */
+
+	sRGXMMUPCEConfig.uiProtMask = RGX_MIPS_MMUCTRL_PCE_PROTMASK; /* Mask to get the status bits of the PC */
+	sRGXMMUPCEConfig.uiProtShift = 0; /* Shift this many bits to have status bits starting with bit 0 */
+
+	sRGXMMUPCEConfig.uiValidEnMask = RGX_MIPS_MMUCTRL_PC_DATA_VALID_EN; /* Mask to get entry valid bit of the PC */
+	sRGXMMUPCEConfig.uiValidEnShift = RGX_MIPS_MMUCTRL_PC_DATA_VALID_SHIFT; /* Shift this many bits to have entry valid bit starting with bit 0 */
+
+	/*
+	 *  Setup sRGXMMUTopLevelDevVAddrConfig
+	 */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask = 0; /* Get the PC address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift = 0;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPC = 0;
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask = 0; /* Get the PD address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift = 0;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPD = 0;
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPTIndexMask = IMG_UINT64_C(0xfffffff000); /* Get the PT address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPTIndexShift = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_4KBDP. No PD in MIPS MMU currently
+	 */
+	sRGXMMUPDEConfig_4KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_4KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_4KBDP.uiBytesPerEntry = 0;
+
+	/* No PD used for MIPS */
+	sRGXMMUPDEConfig_4KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_4KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_4KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlMask = IMG_UINT64_C(0x0);
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiProtMask = RGX_MIPS_MMUCTRL_PDE_PROTMASK;
+	sRGXMMUPDEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiValidEnMask = RGX_MIPS_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_4KBDP.uiValidEnShift = RGX_MIPS_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_4KBDP.
+	 */
+	sRGXMMUPTEConfig_4KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_4KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+
+	if (bPhysBusAbove32Bit)
+	{
+		sRGXMMUPTEConfig_4KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT;
+	}
+	else
+	{
+		sRGXMMUPTEConfig_4KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY;
+	}
+
+	sRGXMMUPTEConfig_4KBDP.uiAddrShift = RGXMIPSFW_ENTRYLO_PFN_SHIFT;
+	sRGXMMUPTEConfig_4KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPTEConfig_4KBDP.uiProtMask = RGXMIPSFW_ENTRYLO_DVG | ~RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK |
+	                                    RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN | RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN;
+	sRGXMMUPTEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_4KBDP.uiValidEnMask = RGXMIPSFW_ENTRYLO_VALID_EN;
+	sRGXMMUPTEConfig_4KBDP.uiValidEnShift = RGXMIPSFW_ENTRYLO_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_4KBDP
+	 */
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPC = 0;
+
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask = ~RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift = RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000000fff);
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiOffsetInBytes = RGX_FIRMWARE_RAW_HEAP_BASE & IMG_UINT64_C(0x00ffffffff);
+
+	/*
+	 * Setup gsPageSizeConfig4KB
+	 */
+	gsPageSizeConfig4KB.psPDEConfig = &sRGXMMUPDEConfig_4KBDP;
+	gsPageSizeConfig4KB.psPTEConfig = &sRGXMMUPTEConfig_4KBDP;
+	gsPageSizeConfig4KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_4KBDP;
+	gsPageSizeConfig4KB.uiRefCount = 0;
+	gsPageSizeConfig4KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_16KBDP
+	 */
+	sRGXMMUPDEConfig_16KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_16KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_16KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiAddrShift = 0; /* These are for a page directory ENTRY, meaning the address of a PT cropped to suit the PD */
+	sRGXMMUPDEConfig_16KBDP.uiAddrLog2Align = 0; /* Alignment of the page tables NOT directories */
+
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_16KBDP. Not supported yet
+	 */
+	sRGXMMUPTEConfig_16KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_16KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiAddrShift = 0; /* These are for a page table ENTRY, meaning the address of a PAGE cropped to suit the PD */
+	sRGXMMUPTEConfig_16KBDP.uiAddrLog2Align = 0; /* Alignment of the pages NOT tables */
+
+	sRGXMMUPTEConfig_16KBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_16KBDP
+	 */
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig16KB
+	 */
+	gsPageSizeConfig16KB.psPDEConfig = &sRGXMMUPDEConfig_16KBDP;
+	gsPageSizeConfig16KB.psPTEConfig = &sRGXMMUPTEConfig_16KBDP;
+	gsPageSizeConfig16KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_16KBDP;
+	gsPageSizeConfig16KB.uiRefCount = 0;
+	gsPageSizeConfig16KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size. Not supported yet
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_64KBDP
+	 */
+	sRGXMMUPDEConfig_64KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_64KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_64KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_64KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_64KBDP.
+	 *
+	 */
+	sRGXMMUPTEConfig_64KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_64KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+	if (bPhysBusAbove32Bit)
+	{
+		sRGXMMUPTEConfig_64KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT;
+	}
+	else
+	{
+		sRGXMMUPTEConfig_64KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY;
+	}
+
+	/* Even while using 64K pages, MIPS still aligns addresses to 4K */
+	sRGXMMUPTEConfig_64KBDP.uiAddrShift = RGXMIPSFW_ENTRYLO_PFN_SHIFT;
+	sRGXMMUPTEConfig_64KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPTEConfig_64KBDP.uiProtMask = RGXMIPSFW_ENTRYLO_DVG | ~RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK |
+	                                     RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN | RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN;
+	sRGXMMUPTEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_64KBDP.uiValidEnMask = RGXMIPSFW_ENTRYLO_VALID_EN;
+	sRGXMMUPTEConfig_64KBDP.uiValidEnShift = RGXMIPSFW_ENTRYLO_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_64KBDP.
+	 */
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask = IMG_UINT64_C(0x00ffff0000);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_64K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000000ffff);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiOffsetInBytes = RGX_FIRMWARE_RAW_HEAP_BASE & IMG_UINT64_C(0x00ffffffff);
+
+	/*
+	 * Setup gsPageSizeConfig64KB.
+	 */
+	gsPageSizeConfig64KB.psPDEConfig = &sRGXMMUPDEConfig_64KBDP;
+	gsPageSizeConfig64KB.psPTEConfig = &sRGXMMUPTEConfig_64KBDP;
+	gsPageSizeConfig64KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_64KBDP;
+	gsPageSizeConfig64KB.uiRefCount = 0;
+	gsPageSizeConfig64KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size. Not supported yet
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_256KBDP
+	 */
+	sRGXMMUPDEConfig_256KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_256KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_256KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_256KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP
+	 */
+	sRGXMMUPTEConfig_256KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_256KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_256KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_256KBDP
+	 */
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig256KB
+	 */
+	gsPageSizeConfig256KB.psPDEConfig = &sRGXMMUPDEConfig_256KBDP;
+	gsPageSizeConfig256KB.psPTEConfig = &sRGXMMUPTEConfig_256KBDP;
+	gsPageSizeConfig256KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_256KBDP;
+	gsPageSizeConfig256KB.uiRefCount = 0;
+	gsPageSizeConfig256KB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUPDEConfig_1MBDP.  Not supported yet
+	 */
+	sRGXMMUPDEConfig_1MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_1MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_1MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_1MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_1MBDP
+	 */
+	sRGXMMUPTEConfig_1MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_1MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_1MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_1MBDP
+	 */
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig1MB
+	 */
+	gsPageSizeConfig1MB.psPDEConfig = &sRGXMMUPDEConfig_1MBDP;
+	gsPageSizeConfig1MB.psPTEConfig = &sRGXMMUPTEConfig_1MBDP;
+	gsPageSizeConfig1MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_1MBDP;
+	gsPageSizeConfig1MB.uiRefCount = 0;
+	gsPageSizeConfig1MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUPDEConfig_2MBDP. Not supported yet
+	 */
+	sRGXMMUPDEConfig_2MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_2MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_2MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_2MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_2MBDP
+	 */
+	sRGXMMUPTEConfig_2MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_2MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_2MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_2MBDP
+	 */
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig2MB
+	 */
+	gsPageSizeConfig2MB.psPDEConfig = &sRGXMMUPDEConfig_2MBDP;
+	gsPageSizeConfig2MB.psPTEConfig = &sRGXMMUPTEConfig_2MBDP;
+	gsPageSizeConfig2MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_2MBDP;
+	gsPageSizeConfig2MB.uiRefCount = 0;
+	gsPageSizeConfig2MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUDeviceAttributes
+	 */
+	sRGXMMUDeviceAttributes.eMMUType = PDUMP_MMU_TYPE_MIPS_MICROAPTIV;
+
+	/*
+	 * The page table fits in one or more big physically adjacent pages,
+	 * at most as big as the page table itself.
+	 * To calculate its alignment/page size, calculate the log2 size of the page
+	 * table taking into account all OSes, then round that down to a valid MIPS
+	 * log2 page size (12, 14, 16 for a 4K, 16K, 64K page size).
+	 */
+	sRGXMMUDeviceAttributes.ui32BaseAlign =
+		(CeilLog2(RGX_NUM_DRIVERS_SUPPORTED) + RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) & ~1U;
+
+	/* 256K alignment might be too hard to achieve, fall back to 64K */
+	sRGXMMUDeviceAttributes.ui32BaseAlign =
+		MIN(sRGXMMUDeviceAttributes.ui32BaseAlign, RGXMIPSFW_LOG2_PAGE_SIZE_64K);
+
+
+
+	/* The base configuration is set to 4kB pages*/
+	sRGXMMUDeviceAttributes.psBaseConfig = &sRGXMMUPTEConfig_4KBDP;
+	sRGXMMUDeviceAttributes.psTopLevelDevVAddrConfig = &sRGXMMUTopLevelDevVAddrConfig;
+
+	/* Functions for deriving page table/dir/cat protection bits */
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt8 = RGXDerivePCEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt4 = RGXDerivePCEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXDerivePDEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt4 = RGXDerivePDEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt8 = RGXDerivePTEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt4 = RGXDerivePTEProt4;
+
+	/* Functions for establishing configurations for PDE/PTE/DEVVADDR
+	   on per-heap basis */
+	sRGXMMUDeviceAttributes.pfnGetPageSizeConfiguration = RGXGetPageSizeConfigCB;
+	sRGXMMUDeviceAttributes.pfnPutPageSizeConfiguration = RGXPutPageSizeConfigCB;
+
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE4 = RGXGetPageSizeFromPDE4;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXGetPageSizeFromPDE8;
+
+	psDeviceNode->psFirmwareMMUDevAttrs = &sRGXMMUDeviceAttributes;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = RGXCheckTrampolineAddrs;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXCheckTrampolineAddrs(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                            MMU_DEVICEATTRIBS *psDevAttrs,
+                                            IMG_UINT64 *pui64Addr)
+{
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, MIPS))
+	{
+		/*
+		 * If mapping for the MIPS FW context, check for sensitive PAs
+		 */
+		if (psDevAttrs == psDevNode->psFirmwareMMUDevAttrs)
+		{
+			PVRSRV_RGXDEV_INFO *psDevice = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice;
+
+			if ((RGX_GET_FEATURE_VALUE(psDevice, PHYS_BUS_WIDTH) == 32) &&
+				 RGXMIPSFW_SENSITIVE_ADDR(*pui64Addr))
+			{
+				*pui64Addr = psDevice->psTrampoline->sPhysAddr.uiAddr + RGXMIPSFW_TRAMPOLINE_OFFSET(*pui64Addr);
+			}
+			/* FIX_HW_BRN_63553 is mainlined for all MIPS cores */
+			else if (*pui64Addr == 0x0 && !psDevice->sLayerParams.bDevicePA0IsValid)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s attempt to map addr 0x0 in the FW but 0x0 is not considered valid.", __func__));
+				return PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE;
+			}
+		}
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXMipsMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	psDeviceNode->pfnMMUGetContextID = NULL;
+#endif
+
+	psDeviceNode->psFirmwareMMUDevAttrs = NULL;
+
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_MESSAGE, "Variable Page Size Heap Stats:"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 4K page heaps: %d",
+			 gsPageSizeConfig4KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 4K page heaps (should be 0): %d",
+			 gsPageSizeConfig4KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 16K page heaps: %d",
+			 gsPageSizeConfig16KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 16K page heaps (should be 0): %d",
+			 gsPageSizeConfig16KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 64K page heaps: %d",
+			 gsPageSizeConfig64KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 64K page heaps (should be 0): %d",
+			 gsPageSizeConfig64KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 256K page heaps: %d",
+			 gsPageSizeConfig256KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 256K page heaps (should be 0): %d",
+			 gsPageSizeConfig256KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 1M page heaps: %d",
+			 gsPageSizeConfig1MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 1M page heaps (should be 0): %d",
+			 gsPageSizeConfig1MB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 2M page heaps: %d",
+			 gsPageSizeConfig2MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 2M page heaps (should be 0): %d",
+			 gsPageSizeConfig2MB.uiRefCount));
+#endif
+	if (gsPageSizeConfig4KB.uiRefCount > 0 ||
+		gsPageSizeConfig16KB.uiRefCount > 0 ||
+		gsPageSizeConfig64KB.uiRefCount > 0 ||
+		gsPageSizeConfig256KB.uiRefCount > 0 ||
+		gsPageSizeConfig1MB.uiRefCount > 0 ||
+		gsPageSizeConfig2MB.uiRefCount > 0
+		)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXMMUInit_Unregister: Unbalanced MMU API Usage (Internal error)"));
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt4
+@Description    calculate the PCE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_DPF((PVR_DBG_ERROR, "Page Catalog not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt8
+@Description    calculate the PCE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "Page Catalog not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt4
+@Description    derive the PDE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "Page Directory not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize The log2 of the required page size.
+				E.g, for 4KiB pages, this parameter must be 12.
+				For 2MiB pages, it must be set to 21.
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "Page Directory not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt4
+@Description    calculate the PTE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags)
+{
+	IMG_UINT32 ui32MMUFlags = 0;
+
+	if (((MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE) & uiProtFlags) == (MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE))
+	{
+		/* read/write */
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_DIRTY_EN;
+	}
+	else if (MMU_PROTFLAGS_READABLE & uiProtFlags)
+	{
+		/* read only */
+	}
+	else if (MMU_PROTFLAGS_WRITEABLE & uiProtFlags)
+	{
+		/* write only */
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN;
+	}
+	else if ((MMU_PROTFLAGS_INVALID & uiProtFlags) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt4: neither read nor write specified..."));
+	}
+
+	/* cache coherency */
+	if (MMU_PROTFLAGS_CACHE_COHERENT & uiProtFlags)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt4: cache coherency not supported for MIPS caches"));
+	}
+
+	/* cache setup */
+	if ((MMU_PROTFLAGS_CACHED & uiProtFlags) == 0)
+	{
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_UNCACHED;
+	}
+	else
+	{
+		ui32MMUFlags |= gui32CachedPolicy <<
+		                RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT;
+	}
+
+	if ((uiProtFlags & MMU_PROTFLAGS_INVALID) == 0)
+	{
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_VALID_EN;
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_GLOBAL_EN;
+	}
+
+	if (MMU_PROTFLAGS_DEVICE(PMMETA_PROTECT) & uiProtFlags)
+	{
+		/* PVR_DPF((PVR_DBG_WARNING, "RGXDerivePTEProt4: PMMETA Protect not existent for MIPS, option discarded")); */
+	}
+
+	return ui32MMUFlags;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt8
+@Description    calculate the PTE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "8-byte PTE not supported on this device"));
+
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXGetPageSizeConfig
+@Description    Set up configuration for variable sized data pages.
+				RGXPutPageSizeConfigCB has to be called to ensure correct
+				refcounting.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+										   const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+										   const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+										   const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+										   IMG_HANDLE *phPriv)
+{
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGXMIPSFW_LOG2_PAGE_SIZE_64K:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGXMIPSFW_LOG2_PAGE_SIZE_4K:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				 "RGXGetPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				 uiLog2DataPageSize));
+		*phPriv = NULL;
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Refer caller's pointers to the data */
+	*ppsMMUPDEConfig = psPageSizeConfig->psPDEConfig;
+	*ppsMMUPTEConfig = psPageSizeConfig->psPTEConfig;
+	*ppsMMUDevVAddrConfig = psPageSizeConfig->psDevVAddrConfig;
+
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	/* Increment ref-count - not that we're allocating anything here
+	   (I'm using static structs), but one day we might, so we want
+	   the Get/Put code to be balanced properly */
+	psPageSizeConfig->uiRefCount++;
+
+	/* This is purely for debug statistics */
+	psPageSizeConfig->uiMaxRefCount = MAX(psPageSizeConfig->uiMaxRefCount,
+										  psPageSizeConfig->uiRefCount);
+#endif
+
+	*phPriv = (IMG_HANDLE)(uintptr_t)uiLog2DataPageSize;
+	PVR_ASSERT (uiLog2DataPageSize == (IMG_UINT32)(uintptr_t)*phPriv);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPutPageSizeConfig
+@Description    Tells this code that the mmu module is done with the
+				configurations set in RGXGetPageSizeConfig.  This can
+				be a no-op.
+				Called after RGXGetPageSizeConfigCB.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv)
+{
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+	IMG_UINT32 uiLog2DataPageSize;
+
+	uiLog2DataPageSize = (IMG_UINT32)(uintptr_t) hPriv;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGXMIPSFW_LOG2_PAGE_SIZE_64K:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGXMIPSFW_LOG2_PAGE_SIZE_4K:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				 "RGXPutPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				 uiLog2DataPageSize));
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Ref-count here is not especially useful, but it's an extra
+	   check that the API is being used correctly */
+	psPageSizeConfig->uiRefCount--;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "PDE not supported on MIPS"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "PDE not supported on MIPS"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+void RGXMipsCheckFaultAddress(MMU_CONTEXT *psFwMMUCtx,
+                              IMG_UINT32 ui32FwVA,
+                              MMU_FAULT_DATA *psOutFaultData)
+{
+	IMG_UINT32 *pui32PageTable = NULL;
+	PVRSRV_ERROR eError = MMU_AcquireCPUBaseAddr(psFwMMUCtx, (void**) &pui32PageTable);
+	MMU_LEVEL_DATA *psMMULevelData;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32PageSize = OSGetPageSize();
+
+	/* MIPS Firmware CPU must use the same page size as the Host */
+	IMG_UINT32 ui32PTEIndex = ((ui32FwVA & ~(ui32PageSize - 1)) - ui32FwHeapBase) / ui32PageSize;
+
+	psOutFaultData->eTopLevel = MMU_LEVEL_1;
+	psOutFaultData->eType = MMU_FAULT_TYPE_NON_PM;
+
+	psMMULevelData = &psOutFaultData->sLevelData[MMU_LEVEL_1];
+	psMMULevelData->uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	psMMULevelData->ui32Index = ui32PTEIndex;
+	psMMULevelData->ui32NumOfEntries = RGX_FIRMWARE_RAW_HEAP_SIZE / ui32PageSize;
+
+	if ((eError == PVRSRV_OK) && (pui32PageTable != NULL))
+	{
+		psMMULevelData->ui64Address = pui32PageTable[ui32PTEIndex];
+	}
+	else
+	{
+		psMMULevelData->ui64Address = 0U;
+	}
+
+	psMMULevelData->psDebugStr = BITMASK_HAS(psMMULevelData->ui64Address,
+	                                         RGXMIPSFW_TLB_VALID) ?
+	                             ("valid") : ("not valid");
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.h
new file mode 100644
index 000000000000..b2b39402cafe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.h
@@ -0,0 +1,97 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation for the MIPS firmware
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* NB: this file is not to be included arbitrarily.  It exists solely
+   for the linkage between rgxinit.c and rgxmmuinit.c, the former
+   being otherwise cluttered by the contents of the latter */
+
+#ifndef SRVKM_RGXMIPSMMUINIT_H
+#define SRVKM_RGXMIPSMMUINIT_H
+
+#include "device.h"
+#include "img_types.h"
+#include "mmu_common.h"
+#include "img_defs.h"
+#include "rgx_mips.h"
+
+/*
+
+		Labelling of fields within virtual address. No PD and PC are used currently for
+		the MIPS MMU
+*/
+/*
+Page Table entry #
+*/
+#define RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_SHIFT        (12U)
+#define RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_CLRMSK       (IMG_UINT64_C(0XFFFFFFFF00000FFF))
+
+
+/* PC entries related definitions */
+/* No PC is currently used for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_EN            (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_SHIFT         (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_CLRMSK        (0U)
+
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_SHIFT     (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_CLRMSK    (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_EN        (0U)
+
+/* PD entries related definitions */
+/* No PD is currently used for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_EN            (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_SHIFT         (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_CLRMSK        (0U)
+
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_SHIFT     (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_CLRMSK    (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_EN        (0U)
+
+
+PVRSRV_ERROR RGXMipsMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR RGXMipsMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void RGXMipsCheckFaultAddress(MMU_CONTEXT *psFwMMUCtx,
+                              IMG_UINT32 ui32FwVA,
+                              MMU_FAULT_DATA *psOutFaultData);
+
+#endif /* #ifndef SRVKM_RGXMIPSMMUINIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmulticore.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmulticore.c
new file mode 100644
index 000000000000..447c50a78e00
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmulticore.c
@@ -0,0 +1,328 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.c
+@Title          Functions related to multicore devices
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxdefs_km.h"
+#include "pdump_km.h"
+#include "rgxmulticore.h"
+#include "multicore_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwmemctx.h"
+
+/*
+ * check that register defines match our hardcoded definitions.
+ * Rogue has these, volcanic does not.
+ */
+#if ((RGX_MULTICORE_CAPABILITY_FRAGMENT_EN != RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_EN) || \
+     (RGX_MULTICORE_CAPABILITY_GEOMETRY_EN != RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_EN) || \
+     (RGX_MULTICORE_CAPABILITY_COMPUTE_EN  != RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_EN) || \
+     (RGX_MULTICORE_CAPABILITY_PRIMARY_EN  != RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN) || \
+     (RGX_MULTICORE_ID_CLRMSK              != RGX_CR_MULTICORE_GPU_ID_CLRMSK))
+#error "Rogue definitions for RGX_CR_MULTICORE_GPU register have changed"
+#endif
+
+
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores,
+                                        IMG_UINT64 *pui64Caps);
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Return multicore information to clients.
+ * Return not supported on cores without multicore.
+ */
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32CapsSize,
+                                 IMG_UINT32 *pui32NumCores,
+                                 IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDevInfo->ui32MultiCoreNumCores == 0)
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+	else
+	{
+		*pui32NumCores = psDevInfo->ui32MultiCoreNumCores;
+		if (ui32CapsSize > 0)
+		{
+			if (ui32CapsSize < psDevInfo->ui32MultiCoreNumCores)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore caps buffer too small"));
+				eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			}
+			else
+			{
+				IMG_UINT32 i;
+
+				for (i = 0; i < psDevInfo->ui32MultiCoreNumCores; ++i)
+				{
+					pui64Caps[i] = psDevInfo->pui64MultiCoreCapabilities[i];
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Read multicore HW registers and fill in data structure for clients.
+ * Return not supported on cores without multicore.
+ */
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		/* we only set this up once */
+		return PVRSRV_OK;
+	}
+
+	/* defaults for non-multicore devices */
+	psDevInfo->ui32MultiCoreNumCores = 0;
+	psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	psDevInfo->pui64MultiCoreCapabilities = NULL;
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		IMG_BOOL bPowerWasDown;
+		IMG_UINT32 ui32MulticoreRegBankOffset = (1 << RGX_GET_FEATURE_VALUE(psDevInfo, XPU_MAX_REGBANKS_ADDR_WIDTH));
+		IMG_UINT32 ui32MulticoreGPUReg = RGX_CR_MULTICORE_GPU;
+		IMG_UINT32 ui32NumCores;
+		IMG_UINT32 i;
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(XPU_MAX_REGBANKS_ADDR_WIDTH)
+		/* Ensure the HOST_SECURITY reg bank definitions are correct */
+		if ((RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE) != ui32MulticoreRegBankOffset)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Register bank definitions for HOST_SECURITY don't match core's configuration.", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+#endif
+		bPowerWasDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			PVRSRVPowerLock(psDeviceNode);
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVSetSystemPowerState ON failed (%u)", __func__, eError));
+				PVRSRVPowerUnlock(psDeviceNode);
+				return eError;
+			}
+		}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			IMG_UINT32 ui32FwTimeout = MAX_HW_TIME_US;
+
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores,
+		                               INVALIDATE);
+				if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores))
+				{
+					/* No need to wait if the FW has already updated the values */
+					break;
+				}
+				OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores) == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore info not available for guest"));
+				return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+			}
+
+			ui32NumCores = psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores;
+
+			PVR_LOG(("RGX Guest Device initialised with %u %s",
+					 ui32NumCores, (ui32NumCores == 1U) ? "core" : "cores"));
+		}
+		else
+#endif
+		{
+			ui32NumCores = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM);
+		}
+#if !defined(NO_HARDWARE)
+		/* check that the number of cores reported is in-bounds */
+		if (ui32NumCores > (RGX_CR_MULTICORE_SYSTEM_MASKFULL >> RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "invalid return (%u) read from MULTICORE_SYSTEM", ui32NumCores));
+			return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+		}
+#else
+		/* for nohw set to max so clients can allocate enough memory for all pdump runs on any config */
+		ui32NumCores = RGX_MULTICORE_MAX_NOHW_CORES;
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "Multicore system has %u cores", ui32NumCores));
+		PDUMPCOMMENT(psDeviceNode, "RGX Multicore has %d cores\n", ui32NumCores);
+
+		/* allocate storage for capabilities */
+		psDevInfo->pui64MultiCoreCapabilities = OSAllocMem(ui32NumCores * sizeof(psDevInfo->pui64MultiCoreCapabilities[0]));
+		PVR_LOG_GOTO_IF_NOMEM(psDevInfo->pui64MultiCoreCapabilities, eError, err);
+
+		psDevInfo->ui32MultiCoreNumCores = ui32NumCores;
+
+		for (i = 0; i < ui32NumCores; ++i)
+		{
+	#if !defined(NO_HARDWARE)
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+			if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				psDevInfo->pui64MultiCoreCapabilities[i] = psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.aui64MultiCoreCapabilities[i];
+			}
+			else
+#endif
+			{
+				IMG_BOOL bMulticoreRegsMapped = (psDeviceNode->psDevConfig->ui32RegsSize > ui32MulticoreGPUReg);
+				void __iomem *pvCoreRegBase;
+				IMG_INT32 ui32MulticoreRegOffset;
+
+				if (bMulticoreRegsMapped)
+				{
+					pvCoreRegBase = psDevInfo->pvRegsBaseKM;
+					ui32MulticoreRegOffset = ui32MulticoreGPUReg;
+				}
+				else
+				{
+					/* the register bank of this core is not mapped */
+					IMG_CPU_PHYADDR sMultiCoreRegsBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+					sMultiCoreRegsBase.uiAddr += i*ui32MulticoreRegBankOffset;
+					pvCoreRegBase = (void __iomem *) OSMapPhysToLin(sMultiCoreRegsBase, psDeviceNode->psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+					PVR_LOG_GOTO_IF_NOMEM(pvCoreRegBase, eError, err);
+
+					/* adjust the register offset to point inside the newly mapped range */
+					ui32MulticoreRegOffset = RGX_CR_MULTICORE_GPU;
+				}
+
+				psDevInfo->pui64MultiCoreCapabilities[i] =
+							OSReadHWReg64(pvCoreRegBase, ui32MulticoreRegOffset) & RGX_CR_MULTICORE_GPU_MASKFULL;
+
+				if (!bMulticoreRegsMapped)
+				{
+					OSUnMapPhysToLin((void __force *) pvCoreRegBase, psDeviceNode->psDevConfig->ui32RegsSize);
+				}
+			}
+	#else
+			/* emulation for what we think caps are */
+			psDevInfo->pui64MultiCoreCapabilities[i] =
+							   i | ((i == 0) ? (RGX_MULTICORE_CAPABILITY_PRIMARY_EN
+											  | RGX_MULTICORE_CAPABILITY_GEOMETRY_EN) : 0)
+							   | RGX_MULTICORE_CAPABILITY_COMPUTE_EN
+							   | RGX_MULTICORE_CAPABILITY_FRAGMENT_EN;
+	#endif
+			PVR_DPF((PVR_DBG_MESSAGE, "Core %d has capabilities value 0x%x", i, (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[i] ));
+			PDUMPCOMMENT(psDeviceNode, "\tCore %d has caps 0x%08x\n", i,
+			             (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[i]);
+
+			if (psDevInfo->pui64MultiCoreCapabilities[i] & RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN)
+			{
+				psDevInfo->ui32MultiCorePrimaryId = (psDevInfo->pui64MultiCoreCapabilities[i]
+														& ~RGX_CR_MULTICORE_GPU_ID_CLRMSK)
+														>> RGX_CR_MULTICORE_GPU_ID_SHIFT;
+			}
+
+			ui32MulticoreGPUReg += ui32MulticoreRegBankOffset;
+		}
+
+        /* revert power state to what it was on entry to this function */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Register callback to return info about multicore setup to client bridge */
+		psDeviceNode->pfnGetMultiCoreInfo = RGXGetMultiCoreInfo;
+	}
+	else
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	return eError;
+
+err:
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+	return eError;
+}
+
+
+/*
+ * RGXDeinitMultiCoreInfo:
+ * Release resources and clear the MultiCore values in the DeviceNode.
+ */
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pui64MultiCoreCapabilities != NULL)
+	{
+		OSFreeMem(psDevInfo->pui64MultiCoreCapabilities);
+		psDevInfo->pui64MultiCoreCapabilities = NULL;
+		psDevInfo->ui32MultiCoreNumCores = 0;
+		psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	}
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxpdump.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxpdump.c
new file mode 100644
index 000000000000..409d901ff3a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxpdump.c
@@ -0,0 +1,776 @@
+/*************************************************************************/ /*!
+@File           rgxpdump.c
+@Title          Device specific pdump routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(PDUMP)
+#include "pvrsrv.h"
+#include "devicemem_utils.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+#include "rgxpdump.h"
+#include "rgxpdump_common.h"
+#include "rgx_bvnc_defs_km.h"
+#include "pdumpdesc.h"
+
+/*
+ * There are two different set of functions one for META/RISCV and one for MIPS
+ * because the Pdump player does not implement the support for
+ * the MIPS MMU yet. So for MIPS builds we cannot use DevmemPDumpSaveToFileVirtual,
+ * we have to use DevmemPDumpSaveToFile instead.
+ */
+static PVRSRV_ERROR _FWDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DECLARE_DLLIST(sFirmwareGcovBufferValidRegions);
+#endif
+	DECLARE_DLLIST(sSigTDMChecksValidRegions);
+	DECLARE_DLLIST(sSigTAChecksValidRegions);
+	DECLARE_DLLIST(sSig3DChecksValidRegions);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DECLARE_DLLIST(sSigRDMChecksValidRegions);
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psFirmwareGcovBufferMemDesc,
+	                             psDevInfo->ui32FirmwareGcovSize,
+	                             &sFirmwareGcovBufferValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrReturnError);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                             psDevInfo->ui32SigTDMChecksSize,
+		                             &sSigTDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeGcovBufferRegions);
+	}
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                             psDevInfo->ui32SigTAChecksSize,
+	                             &sSigTAChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTDMChecksRegions);
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                             psDevInfo->ui32Sig3DChecksSize,
+	                             &sSig3DChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTAChecksRegions);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+	    RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                             psDevInfo->ui32SigRDMChecksSize,
+		                             &sSigRDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSig3DChecksRegions);
+	}
+#endif
+
+	PDUMPIF(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+	PDUMPELSE(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Gcov */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Gcov Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psFirmwareGcovBufferMemDesc,
+	                       &sFirmwareGcovBufferValidRegions,
+	                       "firmware_gcov.img",
+	                       ui32PDumpFlags);
+#endif
+
+	/* TDM signatures */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                       &sSigTDMChecksValidRegions,
+		                       "out.tdmsig",
+		                       ui32PDumpFlags);
+	}
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                       &sSigTAChecksValidRegions,
+	                       "out.tasig",
+	                       ui32PDumpFlags);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                       &sSig3DChecksValidRegions,
+	                       "out.3dsig",
+	                       ui32PDumpFlags);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		/* RDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump RDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                       &sSigRDMChecksValidRegions,
+		                       "out.rdmsig",
+		                       ui32PDumpFlags);
+	}
+#endif
+
+	PDUMPFI(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+ErrFreeSig3DChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSig3DChecksValidRegions);
+#endif
+ErrFreeSigTAChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTAChecksValidRegions);
+ErrFreeSigTDMChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTDMChecksValidRegions);
+ErrFreeGcovBufferRegions:
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DevmemIntPDumpFreeValidRegions(&sFirmwareGcovBufferValidRegions);
+ErrReturnError:
+#endif
+	return eError;
+}
+
+static PVRSRV_ERROR _FWDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32	ui32ThreadNum, ui32Size, ui32OutFileOffset;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufThreadNumOff =
+				(IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufThreadNumOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+								0, /* 0 offset in the trace buffer mem desc */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, sTraceBuf) /* move to first element of sTraceBuf */
+									+ ui32ThreadNum * sizeof(RGXFWIF_TRACEBUF_SPACE) /* skip required number of sTraceBuf elements */
+									+ offsetof(RGXFWIF_TRACEBUF_SPACE, sAssertBuf), /* offset into its sAssertBuf, to be pdumped */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+	}
+
+	/* FW HWPerf buffer is always allocated when PDUMP is defined, irrespective of HWPerf events being enabled/disabled */
+	PVR_ASSERT(psDevInfo->psRGXFWIfHWPerfBufMemDesc);
+
+	/* Dump hwperf buffer */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+								 0,
+								 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+								 "out.hwperf",
+								 0,
+								 ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _MipsDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTAChecksMemDesc,
+								 0,
+								 psDevInfo->ui32SigTAChecksSize,
+								 "out.tasig",
+								 0);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSig3DChecksMemDesc,
+								 0,
+								 psDevInfo->ui32Sig3DChecksSize,
+								 "out.3dsig",
+								 0);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* TDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTDMChecksMemDesc,
+									 0,
+									 psDevInfo->ui32SigTDMChecksSize,
+									 "out.tdmsig",
+									 0);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MipsDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32		ui32ThreadNum, ui32Size, ui32OutFileOffset;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* Same again... */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufSpaceAssertBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF_SPACE *)0)->sAssertBuf);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+								0, /* 0 offset in the trace buffer mem desc */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff + uiTraceBufSpaceAssertBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+	}
+
+	/* Dump hwperf buffer */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+								 0,
+								 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+								 "out.hwperf",
+								 0);
+
+	return PVRSRV_OK;
+}
+#endif
+
+
+/*
+ * PVRSRVPDumpSignatureBufferKM
+ */
+PVRSRV_ERROR PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_UINT32			ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpSignatureBufferKM(psConnection,
+											  psDeviceNode,
+											  ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpSignatureBufferKM(psConnection,
+											psDeviceNode,
+											ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR PVRSRVPDumpCRCSignatureCheckKM(CONNECTION_DATA * psConnection,
+                                            PVRSRV_DEVICE_NODE * psDeviceNode,
+                                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+/*
+ * PVRSRVPDumpValCheckPreCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPreCommandKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVPDumpValCheckPostCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPostCommandKM(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE * psDeviceNode,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR PVRSRVPDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputImageDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_PBYTE pbyPDumpImageHdr)
+{
+	IMG_PUINT32 pui32Word;
+	IMG_UINT32 ui32HeaderDataSize;
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK) || defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK) || defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32TFBCControl = (psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >>
+																					  RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+#endif
+
+	/* Validate parameters */
+	if (((IMAGE_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((IMAGE_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	memset(pbyPDumpImageHdr, 0, IMAGE_HEADER_SIZE);
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpImageHdr, 0);
+	pui32Word[0] = (IMAGE_HEADER_TYPE << HEADER_WORD0_TYPE_SHIFT);
+	pui32Word[1] = (IMAGE_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+				   (IMAGE_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+
+	ui32HeaderDataSize = ui32DataSize;
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		ui32HeaderDataSize += ui32HeaderSize;
+	}
+	pui32Word[2] = ui32HeaderDataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+	pui32Word[3] = ui32LogicalWidth << IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT;
+	pui32Word[4] = ui32LogicalHeight << IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT;
+
+	pui32Word[5] = ePixFmt << IMAGE_HEADER_WORD5_FORMAT_SHIFT;
+
+	pui32Word[6] = ui32PhysicalWidth << IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT;
+	pui32Word[7] = ui32PhysicalHeight << IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT;
+
+	pui32Word[8] = IMAGE_HEADER_WORD8_STRIDE_POSITIVE | IMAGE_HEADER_WORD8_BIFTYPE_NONE;
+
+	switch (eMemLayout)
+	{
+	case IMG_MEMLAYOUT_STRIDED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_STRIDED;
+		break;
+	case IMG_MEMLAYOUT_TWIDDLED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_NTWIDDLE;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported memory layout - %d", eMemLayout));
+		return PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT;
+	}
+
+	pui32Word[9] = 0;
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		switch (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, FBCDC_ALGORITHM))
+		{
+		case 1:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_BASE;
+			break;
+		case 2:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V2;
+			break;
+		case 3:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2;
+			break;
+		case 4:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V4;
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_ON;
+			}
+
+			pui32Word[9] |= (eFBCSwizzle << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT) & IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK;
+
+			break;
+		case 50:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_TFBC;
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_25;
+			}
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_37;
+			}
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_50;
+			}
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_75;
+			}
+
+			pui32Word[9] |= (eFBCSwizzle << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT) & IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK;
+
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Unsupported algorithm - %d",
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, FBCDC_ALGORITHM)));
+			return PVRSRV_ERROR_NOT_ENABLED;
+		}
+	}
+
+	switch (GET_FBCDC_BLOCK_TYPE(eFBCompression))
+	{
+	case IMG_FB_COMPRESSION_NONE:
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_8x8:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_8X8;
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_16x4:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_16x4;
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_32x2:
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported compression mode - %d", eFBCompression));
+		return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+	}
+
+	pui32Word[10] = paui32FBCClearColour[0];
+	pui32Word[11] = paui32FBCClearColour[1];
+	pui32Word[12] = paui32FBCClearColour[2];
+	pui32Word[13] = paui32FBCClearColour[3];
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP */
+		IMG_UINT32 ui32TFBCGroup  = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK;
+		switch (ui32TFBCGroup)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50;
+				break;
+		}
+	}
+	else
+#endif
+	{
+		pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+	}
+
+#if defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME */
+		IMG_UINT32 ui32TFBCScheme = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK;
+		switch (ui32TFBCScheme)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_ALL;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "Unsupported TFBC compression control scheme - %d", ui32TFBCScheme));
+				return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+		}
+	}
+	else
+#endif
+	{
+		/* Should always be set to 2 ("TFBC delta standard only") on cores without this feature */
+		pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+	}
+
+#if defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_NATIVE_YUV10))
+	{
+		IMG_UINT32 ui32TFBCOverrideYUV10 = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN);
+
+		if (ui32TFBCOverrideYUV10)
+		{
+			pui32Word[14] |= IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN;
+		}
+	}
+#endif
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, TFBC_VERSION) >= 20U)
+		{
+			IMG_UINT32 ui32TFBCOverrideLossyMinChannel = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN);
+
+			if (ui32TFBCOverrideLossyMinChannel)
+			{
+				pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+			}
+		}
+		else
+		{
+			/* Should be set on TFBC version < 2.0 cores */
+			pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputDataDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_PBYTE pbyPDumpDataHdr)
+{
+	IMG_PUINT32 pui32Word;
+
+	/* Validate parameters */
+	if (((DATA_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((DATA_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpDataHdr, 0);
+
+	if (ui32HeaderType == DATA_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (DATA_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(DATA_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+		pui32Word[3] = ui32ElementType << DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT;
+		pui32Word[4] = ui32ElementCount << DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT;
+	}
+
+	if (ui32HeaderType == IBIN_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (IBIN_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(IBIN_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/******************************************************************************
+ End of file (rgxpdump.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxsrvinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxsrvinit.c
new file mode 100644
index 000000000000..77d63732e4e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxsrvinit.c
@@ -0,0 +1,1711 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvinit.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "km_apphint_defs.h"
+#include "htbuffer_types.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+
+#include "rgxinit_apphints.h"
+#include "rgxinit.h"
+#include "rgxmulticore.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "osfunc.h"
+
+#include "rgxdefs_km.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "rgx_fwif_hwperf.h"
+#include "rgx_hwperf_table.h"
+
+#include "fwload.h"
+#include "rgxlayer_impl.h"
+#include "rgxfwimageutils.h"
+#include "rgxfwutils.h"
+
+#include "rgx_hwperf.h"
+#include "rgx_bvnc_defs_km.h"
+
+#include "rgxdevice.h"
+
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#include "rgxdevice.h"
+#include "pvrsrv_device.h"
+#endif
+
+#define	HW_PERF_FILTER_DEFAULT         0x00000000 /* Default to no HWPerf */
+#define HW_PERF_FILTER_DEFAULT_ALL_ON  0xFFFFFFFF /* All events */
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+#define AVAIL_POW_UNITS_MASK_DEFAULT   (PVRSRV_APPHINT_HWVALAVAILABLESPUMASK)
+#define AVAIL_RAC_MASK_DEFAULT         (PVRSRV_APPHINT_HWVALAVAILABLERACMASK)
+#endif
+
+/* Kernel CCB size */
+
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE 4
+#endif
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE 16
+#endif
+
+#if PVRSRV_APPHINT_KCCB_SIZE_LOG2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too low.
+#elif PVRSRV_APPHINT_KCCB_SIZE_LOG2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too high.
+#endif
+
+
+#include "os_apphint.h"
+
+/*
+ * _ParseHTBAppHints:
+ *
+ * Generate necessary references to the globally visible AppHints which are
+ * declared in the above #include "km_apphint_defs.h"
+ * Without these local references some compiler tool-chains will treat
+ * unreferenced declarations as fatal errors. This function duplicates the
+ * HTB_specific apphint references which are made in htbserver.c:HTBInit()
+ * However, it makes absolutely *NO* use of these hints.
+ */
+static void
+_ParseHTBAppHints(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvAppHintState)
+{
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32OpMode;
+	IMG_UINT32 ui32BufferSize;
+
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEHTBLOGGROUP;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, EnableHTBLogGroup,
+	                     &ui32AppHintDefault, &ui32LogType);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBOPERATIONMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBOperationMode,
+	                     &ui32AppHintDefault, &ui32OpMode);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBUFFERSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+	                     &ui32AppHintDefault, &ui32BufferSize);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetFilterFlags
+
+ @Description   Initialise and return filter flags
+
+ @Input         bFilteringMode : Enable new TPU filtering mode
+ @Input         ui32TruncateMode : TPU Truncate mode
+
+ @Return        IMG_UINT32 : Filter flags
+
+******************************************************************************/
+static INLINE IMG_UINT32 GetFilterFlags(IMG_BOOL bFilteringMode, IMG_UINT32 ui32TruncateMode)
+{
+	IMG_UINT32 ui32FilterFlags = 0;
+
+	ui32FilterFlags |= bFilteringMode ? RGXFWIF_FILTCFG_NEW_FILTER_MODE : 0;
+	if (ui32TruncateMode == 2)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_INT;
+	}
+	else if (ui32TruncateMode == 3)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_HALF;
+	}
+
+	return ui32FilterFlags;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      InitDeviceFlags
+
+ @Description   Initialise and return device flags
+
+ @Input         psDeviceNode     : Pointer to device node
+ @Input         pvAppHintState   : Pointer to apphint state
+ @Input         psHints          : Apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void InitDeviceFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   void *pvAppHintState,
+                                   RGX_INIT_APPHINTS *psHints)
+{
+	IMG_UINT32 ui32DeviceFlags = 0;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bZeroFreelist;
+	IMG_BOOL bDisableFEDLogging;
+
+
+	bAppHintDefault = PVRSRV_APPHINT_ZEROFREELIST;
+	OSGetAppHintBOOL(psDeviceNode,  pvAppHintState,  ZeroFreelist,
+	                   &bAppHintDefault,         &bZeroFreelist);
+	ui32DeviceFlags |= bZeroFreelist ? RGXKM_DEVICE_STATE_ZERO_FREELIST : 0;
+
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEFEDLOGGING;
+	OSGetAppHintBOOL(psDeviceNode,    pvAppHintState,    DisableFEDLogging,
+	                   &bAppHintDefault,        &bDisableFEDLogging);
+	ui32DeviceFlags |= bDisableFEDLogging ? RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN : 0;
+
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	BITMASK_SET(ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+#endif
+
+	psHints->ui32DeviceFlags = ui32DeviceFlags;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetApphints
+
+ @Description   Read init time apphints and initialise apphints structure
+
+ @Input         psHints : Pointer to apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetApphints(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_INIT_APPHINTS *psHints)
+{
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+	IMG_UINT32 ui32ParamTemp;
+#if defined(__linux__)
+	IMG_UINT64 ui64AppHintDefault;
+#endif
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESIGNATURECHECKS;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSignatureChecks,
+	                     &bAppHintDefault,    &psHints->bEnableSignatureChecks);
+	ui32AppHintDefault = PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    SignatureChecksBufSize,
+	                     &ui32AppHintDefault,    &psHints->ui32SignatureChecksBufSize);
+
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTOUTOFMEMORY;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOutOfMemory,
+	                   &bAppHintDefault,        &psHints->bAssertOnOutOfMem);
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTONHWRTRIGGER;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOnHWRTrigger,
+	                   &bAppHintDefault,        &psHints->bAssertOnHWRTrigger);
+	bAppHintDefault = PVRSRV_APPHINT_CHECKMLIST;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    CheckMList,
+	                   &bAppHintDefault,        &psHints->bCheckMlist);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLECLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableClockGating,
+	                   &bAppHintDefault,        &psHints->bDisableClockGating);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEDMOVERLAP;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableDMOverlap,
+	                   &bAppHintDefault,        &psHints->bDisableDMOverlap);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEAPM;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    EnableAPM,
+	                     &ui32AppHintDefault,      &ui32ParamTemp);
+	psHints->eRGXActivePMConf = ui32ParamTemp;
+	bAppHintDefault = PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableCDMKillingRandMode,
+	                   &bAppHintDefault,      &psHints->bEnableDMKillRand);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableRandomContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableRandomCsw);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSoftResetContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableSoftResetCsw);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableFWContextSwitch,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableFWContextSwitch);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLERDPOWERISLAND;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableRDPowerIsland,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eRGXRDPowerIslandConf = ui32ParamTemp;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESPUCLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableSPUClockGating,
+	                   &bAppHintDefault,           &psHints->bSPUClockGating);
+#endif
+	ui32AppHintDefault = PVRSRV_APPHINT_FIRMWAREPERF;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FirmwarePerf,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eFirmwarePerf = ui32ParamTemp;
+	ui32AppHintDefault = PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FWContextSwitchProfile,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchProfile);
+
+	/*
+	 * HWPerf apphints *
+	 */
+	bAppHintDefault = PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    HWPerfDisableCustomCounterFilter,
+	                     &bAppHintDefault,    &psHints->bHWPerfDisableCounterFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfHostBufSize);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfFWBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfFWBufSize);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_KCCB_SIZE_LOG2;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,  KernelCCBSizeLog2,
+	                     &ui32AppHintDefault,    &psHints->ui32KCCBSizeLog2);
+
+	if (psHints->ui32KCCBSizeLog2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too low, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE;
+	}
+	else if (psHints->ui32KCCBSizeLog2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too high, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE;
+	}
+
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AppHintDefault = PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ISPSchedulingLatencyMode,
+	                     &ui32AppHintDefault,  &psHints->ui32ISPSchedulingLatencyMode);
+#endif
+
+#if defined(__linux__)
+	/* name changes */
+	{
+		IMG_UINT64 ui64Tmp;
+		bAppHintDefault = PVRSRV_APPHINT_DISABLEPDUMPPANIC;
+		OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,  DisablePDumpPanic,
+		                   &bAppHintDefault,           &psHints->bDisablePDP);
+		ui64AppHintDefault = PVRSRV_APPHINT_HWPERFFWFILTER;
+		OSGetAppHintUINT64(psDevInfo->psDeviceNode,  pvAppHintState,  HWPerfFWFilter,
+		                   &ui64AppHintDefault,        &ui64Tmp);
+		psHints->ui32HWPerfFilter0 = (IMG_UINT32)(ui64Tmp & 0xffffffffllu);
+		psHints->ui32HWPerfFilter1 = (IMG_UINT32)((ui64Tmp >> 32) & 0xffffffffllu);
+	}
+#endif
+
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTFILTER;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWPerfHostFilter,
+	                     &ui32AppHintDefault,          &psHints->ui32HWPerfHostFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    SecondaryOSClockSource,
+	                         &ui32AppHintDefault,      &psHints->ui32SecondaryOSClockSource);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWRDebugDumpLimit,
+	                     &ui32AppHintDefault,          &ui32ParamTemp);
+	psHints->ui32HWRDebugDumpLimit = MIN(ui32ParamTemp, RGXFWIF_HWR_DEBUG_DUMP_ALL);
+
+	{
+		IMG_BOOL bFilteringMode = IMG_FALSE;
+		IMG_UINT32 ui32TruncateMode = 0U;
+
+#if defined(HW_ERN_42290_BIT_MASK) && defined(RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42290) && RGX_IS_FEATURE_SUPPORTED(psDevInfo, TPU_FILTERING_MODE_CONTROL))
+#endif
+		{
+			bAppHintDefault = PVRSRV_APPHINT_NEWFILTERINGMODE;
+			OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  NewFilteringMode,
+			                   &bAppHintDefault,  &bFilteringMode);
+		}
+
+#if defined(HW_ERN_42606_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42606))
+#endif
+		{
+			ui32AppHintDefault = PVRSRV_APPHINT_TRUNCATEMODE;
+			OSGetAppHintUINT32(APPHINT_NO_DEVICE,  pvAppHintState,  TruncateMode,
+			                     &ui32AppHintDefault,  &ui32TruncateMode);
+		}
+
+		psHints->ui32FilterFlags = GetFilterFlags(bFilteringMode, ui32TruncateMode);
+	}
+
+#if defined(EMULATOR)
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#endif
+	{
+		bAppHintDefault = PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG;
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  EnableTrustedDeviceAceConfig,
+		                   &bAppHintDefault,     &psHints->bEnableTrustedDeviceAceConfig);
+	}
+#endif
+
+#if defined(__linux__)
+	ui32AppHintDefault = 0;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  FWContextSwitchCrossDM,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchCrossDM);
+#endif
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	ui32AppHintDefault = PVRSRV_APPHINT_PHYSMEMTESTPASSES;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  PhysMemTestPasses,
+	                     &ui32AppHintDefault,    &psHints->ui32PhysMemTestPasses);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	psHints->ui32AvailablePowUnitsMask = AVAIL_POW_UNITS_MASK_DEFAULT;
+	psHints->ui32AvailableRACMask = AVAIL_RAC_MASK_DEFAULT;
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	ui32AppHintDefault = PVRSRV_APPHINT_TFBCVERSION;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, TFBCVersionDowngrade,
+	 &ui32AppHintDefault, &psHints->ui32TFBCVersion);
+
+	if (ui32AppHintDefault != psHints->ui32TFBCVersion)
+	{
+		PVR_LOG(("TFBCVersionDowngrade set to %u", psHints->ui32TFBCVersion));
+	}
+
+	psHints->bTFBCCompressionControlLossyMinChannel = false;
+	psHints->bTFBCCompressionControlYUVFormat = false;
+	psHints->ui32TFBCCompressionControlScheme =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME;
+	psHints->ui32TFBCCompressionControlGroup =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	ui32AppHintDefault = PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  DebugDumpFWTLogType,
+	                     &ui32AppHintDefault,  &psHints->eDebugDumpFWTLogType);
+	if ((IMG_UINT32)psHints->eDebugDumpFWTLogType > RGX_FWT_LOGTYPE_PARTIAL)
+	{
+		psHints->eDebugDumpFWTLogType = RGX_FWT_LOGTYPE_NONE;
+		PVR_DPF((PVR_DBG_WARNING, "Invalid value for DebugDumpFWTLogType. Setting to 0 (disabled)."));
+	}
+
+	/*
+	 * FW logs apphints
+	 */
+	{
+		IMG_UINT32 ui32LogGroup, ui32TraceOrTBI;
+
+		ui32AppHintDefault = PVRSRV_APPHINT_ENABLELOGGROUP;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, EnableLogGroup, &ui32AppHintDefault, &ui32LogGroup);
+		ui32AppHintDefault = PVRSRV_APPHINT_FIRMWARELOGTYPE;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, FirmwareLogType, &ui32AppHintDefault, &ui32TraceOrTBI);
+
+		/* Defaulting to TRACE */
+		BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+		if (ui32TraceOrTBI == 1 /* TBI */)
+		{
+			if ((ui32LogGroup & RGXFWIF_LOG_TYPE_GROUP_MASK) == 0)
+			{
+				/* No groups configured - defaulting to MAIN group */
+				BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_GROUP_MAIN);
+			}
+			BITMASK_UNSET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+		}
+#endif
+		psHints->ui32LogType = ui32LogGroup;
+	}
+
+	_ParseHTBAppHints(psDevInfo->psDeviceNode, pvAppHintState);
+
+	InitDeviceFlags(psDevInfo->psDeviceNode, pvAppHintState, psHints);
+
+	OSFreeAppHintState(pvAppHintState);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      GetFWConfigFlags
+
+ @Description   Initialise and return FW config flags
+
+ @Input         psHints            : Apphints container
+ @Input         pui32FWConfigFlags : Pointer to config flags
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetFWConfigFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    RGX_INIT_APPHINTS *psHints,
+                                    IMG_UINT32 *pui32FWConfigFlags,
+                                    IMG_UINT32 *pui32FWConfigFlagsExt,
+                                    IMG_UINT32 *pui32FwOsCfgFlags)
+{
+	IMG_UINT32 ui32FWConfigFlags = 0;
+	IMG_UINT32 ui32FWConfigFlagsExt = 0;
+	IMG_UINT32 ui32FwOsCfgFlags = psHints->ui32FWContextSwitchCrossDM |
+	                              (psHints->ui32EnableFWContextSwitch & ~RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK);
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32FWConfigFlagsSupValExt = 0;
+	IMG_UINT32 ui32TFBCVersion = 0U;
+#endif
+
+	ui32FWConfigFlags |= psHints->bAssertOnOutOfMem ? RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY : 0;
+	ui32FWConfigFlags |= psHints->bAssertOnHWRTrigger ? RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER : 0;
+	ui32FWConfigFlags |= psHints->bCheckMlist ? RGXFWIF_INICFG_CHECK_MLIST_EN : 0;
+	ui32FWConfigFlags |= psHints->bDisableClockGating ? RGXFWIF_INICFG_DISABLE_CLKGATING_EN : 0;
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0) &&
+		!RGX_IS_FEATURE_SUPPORTED(psDevInfo, ERYX_TOP_INFRASTRUCTURE))
+	{
+		/* Pipeline DM roadblocks are currently enabled pre-Eryx. */
+		ui32FWConfigFlags |= RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN;
+	}
+#endif
+
+	ui32FWConfigFlags |= psHints->bDisableDMOverlap ? RGXFWIF_INICFG_DISABLE_DM_OVERLAP : 0;
+
+	if ((RGX_GET_FEATURE_VALUE(psDevInfo, SLC_SIZE_IN_KILOBYTES) <= 2))
+	{
+		ui32FWConfigFlags |= RGXFWIF_INICFG_DISABLE_DM_OVERLAP;
+	}
+
+	ui32FWConfigFlags |= psHints->bDisablePDP ? RGXFWIF_INICFG_DISABLE_PDP_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableDMKillRand ? RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableRandomCsw ? RGXFWIF_INICFG_CTXSWITCH_MODE_RAND : 0;
+	ui32FWConfigFlags |= psHints->bEnableSoftResetCsw ? RGXFWIF_INICFG_CTXSWITCH_SRESET_EN : 0;
+	ui32FWConfigFlags |= (psHints->ui32HWPerfFilter0 != 0 || psHints->ui32HWPerfFilter1 != 0) ? RGXFWIF_INICFG_HWPERF_EN : 0;
+	ui32FWConfigFlags |= psHints->bHWPerfDisableCounterFilter ? RGXFWIF_INICFG_HWP_DISABLE_FILTER : 0;
+	ui32FWConfigFlags |= (psHints->ui32FWContextSwitchProfile << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT) & RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK;
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32FWConfigFlags |= (psHints->ui32ISPSchedulingLatencyMode << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT) & RGXFWIF_INICFG_ISPSCHEDMODE_MASK;
+#endif
+
+	{
+		ui32FWConfigFlags |= psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode) ? RGXFWIF_INICFG_FBCDC_V3_1_EN : 0;
+	}
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+
+	/* Determine if we need to present a TFBC v1.0, v1.1 or native
+	 * behaviour. For V1.0 we need to set the following features:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x1
+	 *  TFBCCompressionControlScheme = 0x2
+	 *  TFBCCompressionControlGroup = 0x0
+	 * For V1.1 we need to set the following:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x0
+	 *  TFBCCompressionControlScheme = 0x1
+	 *  TFBCCompressionControlGroup = 0 / 1 (depends on LOSSY_37_PERCENT)
+	 * The gating for these values depends on whether the GPU supports
+	 * RGX_FEATURE_TFBC_VERSION = 20U
+	 */
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		ui32TFBCVersion = RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION);
+
+		if (ui32TFBCVersion >= 20U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 11:	/* TFBC Version 1.1 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = false;
+					psHints->ui32TFBCCompressionControlScheme = 1U;
+					psHints->ui32TFBCCompressionControlGroup =
+					    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.1, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+					         " Ignoring. Using value %u instead",
+					         __func__, psHints->ui32TFBCVersion,
+					         ui32TFBCVersion));
+					break;
+			}
+
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else if (ui32TFBCVersion == 11U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+				             " Ignoring. Using value %u instead",
+				             __func__, psHints->ui32TFBCVersion,
+				             ui32TFBCVersion));
+					break;
+			}
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else	/* TFBC v1.0 */
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32FWConfigFlagsSupValExt);
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = 0;
+#endif
+			if ((psHints->ui32TFBCVersion != 0U) &&
+			    (psHints->ui32TFBCVersion != ui32TFBCVersion))
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s: Cannot specify TFBC version %u"
+				         " on a version %u GPU core", __func__,
+				         psHints->ui32TFBCVersion, ui32TFBCVersion));
+			}
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	*pui32FWConfigFlags    = ui32FWConfigFlags;
+	*pui32FWConfigFlagsExt = ui32FWConfigFlagsExt;
+	*pui32FwOsCfgFlags     = ui32FwOsCfgFlags;
+}
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+/*!
+*******************************************************************************
+
+ @Function      RGXTDProcessFWImage
+
+ @Description   Fetch and send data used by the trusted device to complete
+                the FW image setup
+
+ @Input         psDeviceNode : Device node
+ @Input         psFWParams : Firmware and parameters used by the FW
+
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXTDProcessFWImage(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        PVRSRV_FW_PARAMS *psFWParams)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+	PVRSRV_ERROR eError;
+
+	if (psDevConfig->pfnTDSendFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TDSendFWImage not implemented!", __func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		if (psFWParams->uFWP.sMips.ui32FWPageTableNumPages > TD_MAX_NUM_MIPS_PAGETABLE_PAGES)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Number of page table pages %u greater "
+					 "than what is allowed by the TD interface (%u), FW might "
+					 "not work properly!", __func__,
+					 psFWParams->uFWP.sMips.ui32FWPageTableNumPages,
+					 TD_MAX_NUM_MIPS_PAGETABLE_PAGES));
+		}
+	}
+#endif
+
+	eError = psDevConfig->pfnTDSendFWImage(psDevConfig->hSysData, psFWParams);
+
+	return eError;
+}
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXAcquireMipsBootldrData
+
+ @Description   Acquire MIPS bootloader data parameters
+
+ @Input         psDeviceNode : Device node
+ @Input         puFWParams   : FW boot parameters
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireMipsBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+	MMU_DEVICEATTRIBS *psFWMMUDevAttrs = psDevInfo->psDeviceNode->psFirmwareMMUDevAttrs;
+	IMG_DEV_PHYADDR sAddr;
+	IMG_UINT32 ui32PTSize, i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bValid;
+
+	/* Rogue Registers physical address */
+#if defined(SUPPORT_ALT_REGBASE)
+	puFWParams->sMips.sGPURegAddr = psDeviceNode->psDevConfig->sAltRegsGpuPBase;
+#else
+	PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+	                           1,
+	                           &puFWParams->sMips.sGPURegAddr,
+	                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+#endif
+
+	/* MIPS Page Table physical address */
+	MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sAddr);
+
+	/* MIPS Page Table allocation is contiguous. Pass one or more addresses
+	 * to the FW depending on the Page Table size and alignment. */
+
+	ui32PTSize = (psFWMMUDevAttrs->psTopLevelDevVAddrConfig->uiNumEntriesPT)
+	             << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	ui32PTSize = PVR_ALIGN(ui32PTSize, 1U << psFWMMUDevAttrs->ui32BaseAlign);
+
+	puFWParams->sMips.ui32FWPageTableLog2PageSize = psFWMMUDevAttrs->ui32BaseAlign;
+	puFWParams->sMips.ui32FWPageTableNumPages = ui32PTSize >> psFWMMUDevAttrs->ui32BaseAlign;
+
+	if (puFWParams->sMips.ui32FWPageTableNumPages > 4U)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Page table cannot be mapped by the FW "
+		         "(size 0x%x, log2 page size %u, %u pages)",
+		         __func__, ui32PTSize, puFWParams->sMips.ui32FWPageTableLog2PageSize,
+		         puFWParams->sMips.ui32FWPageTableNumPages));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Confirm page alignment fits in 64-bits */
+	if (psFWMMUDevAttrs->ui32BaseAlign > 63)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid page alignment "
+		         "(psFWMMUDevAttrs->ui32BaseAlign = %u)",
+		         __func__, psFWMMUDevAttrs->ui32BaseAlign));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	for (i = 0; i < puFWParams->sMips.ui32FWPageTableNumPages; i++)
+	{
+		puFWParams->sMips.asFWPageTableAddr[i].uiAddr =
+		    sAddr.uiAddr + i * (1ULL << psFWMMUDevAttrs->ui32BaseAlign);
+	}
+
+	/* MIPS Stack Pointer Physical Address */
+	eError = RGXGetPhyAddr(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+	                       &puFWParams->sMips.sFWStackAddr,
+	                       RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+	                       OSGetPageShift(),
+	                       1,
+	                       &bValid);
+
+	return eError;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      InitFirmware
+
+ @Description   Allocate, initialise and pdump Firmware code and data memory
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitFirmware(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	OS_FW_IMAGE       *psRGXFW = NULL;
+	PVRSRV_FW_PARAMS  sFWParams = {0};
+
+	/* FW code memory */
+	IMG_DEVMEM_SIZE_T uiFWCodeAllocSize;
+	void              *pvFWCodeHostAddr;
+
+	/* FW data memory */
+	IMG_DEVMEM_SIZE_T uiFWDataAllocSize;
+	void              *pvFWDataHostAddr;
+
+	/* FW coremem code memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememCodeAllocSize;
+	void              *pvFWCorememCodeHostAddr = NULL;
+
+	/* FW coremem data memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememDataAllocSize;
+	void              *pvFWCorememDataHostAddr = NULL;
+
+	PVRSRV_FW_BOOT_PARAMS uFWParams;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_BOOL bUseSecureFWData =
+								RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	                            (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+	                             RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32) ||
+#endif
+	                            RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR);
+#endif
+
+	/*
+	 * Get pointer to Firmware image
+	 */
+	eError = RGXLoadAndGetFWData(psDeviceNode, &psRGXFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Error or confirmation message generated in RGXLoadAndGetFWData */
+		goto fw_load_fail;
+	}
+
+	/*
+	 * Get pointer and size
+	 */
+	sFWParams.pvFirmware       = OSFirmwareData(psRGXFW);
+	sFWParams.ui32FirmwareSize = OSFirmwareSize(psRGXFW);
+
+	/*
+	 * Allow it to be pre-processed by the platform hook
+	 */
+	if (psDeviceNode->psDevConfig->pfnPrepareFWImage != NULL)
+	{
+		eError = psDeviceNode->psDevConfig->pfnPrepareFWImage(
+				psDeviceNode->psDevConfig->hSysData, &sFWParams);
+		if (eError != PVRSRV_OK)
+			goto cleanup_initfw;
+	}
+
+	/*
+	 * Allocate Firmware memory
+	 */
+
+	eError = RGXGetFWImageAllocSize(&psDevInfo->sLayerParams,
+	                                sFWParams.pvFirmware,
+	                                sFWParams.ui32FirmwareSize,
+	                                &uiFWCodeAllocSize,
+	                                &uiFWDataAllocSize,
+	                                &uiFWCorememCodeAllocSize,
+	                                &uiFWCorememDataAllocSize,
+	                                &psDevInfo->sFWInfoHeader);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXGetFWImageAllocSize failed",
+		        __func__));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Initiate FW compatibility check for Native and Host.
+	 * Guest compatibility check must be done after FW boot.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed compatibility check for device %p (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+	}
+
+	psDevInfo->ui32FWCodeSizeInBytes = uiFWCodeAllocSize;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+    PVR_DPF((PVR_DBG_WARNING,
+            "%s: META DMA not available, disabling core memory code/data",
+            __func__));
+    uiFWCorememCodeAllocSize = 0;
+    uiFWCorememDataAllocSize = 0;
+#endif
+
+	psDevInfo->ui32FWCorememCodeSizeInBytes = uiFWCorememCodeAllocSize;
+
+	eError = RGXInitAllocFWImgMem(psDeviceNode,
+	                              uiFWCodeAllocSize,
+	                              uiFWDataAllocSize,
+	                              uiFWCorememCodeAllocSize,
+	                              uiFWCorememDataAllocSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXInitAllocFWImgMem failed (%d)",
+		        __func__,
+		        eError));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Acquire pointers to Firmware allocations
+	 */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCodeHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, &pvFWCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", cleanup_initfw);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWDataHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc, &pvFWDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_code);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCorememCodeHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, &pvFWCorememCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_data);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		pvFWCorememDataHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememDataAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc, &pvFWCorememDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_corememcode);
+	}
+
+	/*
+	 * Prepare FW boot parameters
+	 */
+	OSCachedMemSet(&uFWParams, 0, sizeof(PVRSRV_FW_BOOT_PARAMS));
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		uFWParams.sMeta.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sMeta.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeFWAddr = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sMeta.uiFWCorememCodeSize = uiFWCorememCodeAllocSize;
+		uFWParams.sMeta.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sMeta.sFWCorememDataFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+		uFWParams.sMeta.ui32NumThreads = 2;
+#else
+		uFWParams.sMeta.ui32NumThreads = 1;
+#endif
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = RGXAcquireMipsBootldrData(psDeviceNode, &uFWParams);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXAcquireMipsBootldrData failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+	}
+#endif
+	else
+	{
+		uFWParams.sRISCV.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+
+		uFWParams.sRISCV.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememCodeFWAddr   = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sRISCV.uiFWCorememCodeSize    = uiFWCorememCodeAllocSize;
+
+		uFWParams.sRISCV.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememDataFWAddr   = psDevInfo->sFWCorememDataStoreFWAddr;
+		uFWParams.sRISCV.uiFWCorememDataSize    = uiFWCorememDataAllocSize;
+	}
+
+
+	/*
+	 * Process the Firmware image and setup code and data segments.
+	 *
+	 * When the trusted device is enabled and the FW code lives
+	 * in secure memory we will only setup the data segments here,
+	 * while the code segments will be loaded to secure memory
+	 * by the trusted device.
+	 */
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = RGXProcessFWImage(&psDevInfo->sLayerParams,
+								   sFWParams.pvFirmware,
+								   pvFWCodeHostAddr,
+								   pvFWDataHostAddr,
+								   pvFWCorememCodeHostAddr,
+								   pvFWCorememDataHostAddr,
+								   &uFWParams);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXProcessFWImage failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+		RGXFwSharedMemCacheOpExec(pvFWCodeHostAddr,
+		                          sizeof(psDevInfo->psRGXFWCodeMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememCodeAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememCodeHostAddr,
+			                          sizeof(psDevInfo->psRGXFWCorememCodeMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+
+		RGXFwSharedMemCacheOpExec(pvFWDataHostAddr,
+		                          sizeof(psDevInfo->psRGXFWDataMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememDataAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememDataHostAddr,
+			                          sizeof(psDevInfo->psRGXFWIfCorememDataStoreMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+#endif
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	sFWParams.uFWP = uFWParams;
+	RGXTDProcessFWImage(psDeviceNode, &sFWParams);
+#endif
+
+
+	/*
+	 * PDump Firmware allocations
+	 */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Dump firmware code image");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWCodeMemDesc,
+	                   0,
+	                   uiFWCodeAllocSize,
+	                   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware data image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWDataMemDesc,
+		                   0,
+		                   uiFWDataAllocSize,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem code image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWCorememCodeMemDesc,
+						   0,
+						   uiFWCorememCodeAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem data store image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+						   0,
+						   uiFWCorememDataAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	/*
+	 * Release Firmware allocations and clean up
+	 */
+release_fw_allocations:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+	}
+
+release_corememcode:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememCodeAllocSize != 0))
+#else
+	if (uiFWCorememCodeAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+
+release_data:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+	}
+
+release_code:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+cleanup_initfw:
+	OSUnloadFirmware(psRGXFW);
+fw_load_fail:
+
+	return eError;
+}
+
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function      InitialiseHWPerfCounters
+
+ @Description   Initialisation of hardware performance counters and dumping
+                them out to pdump, so that they can be modified at a later
+                point.
+
+ @Input         pvDevice
+ @Input         psHWPerfDataMemDesc
+ @Input         psHWPerfInitDataInt
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *pvDevice,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc,
+                                     RGXFWIF_HWPERF_CTL *psHWPerfInitDataInt)
+{
+	RGXFWIF_HWPERF_CTL_BLK *psHWPerfInitBlkData;
+#if defined(HWPERF_UNIFIED)
+	RGXFWIF_HWPERF_DA_BLK *psHWPerfInitDABlkData;
+#endif
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc;
+	IMG_UINT32 ui32BlockID, ui32BlkCfgIdx, ui32CounterIdx;
+	RGX_HWPERF_CNTBLK_RT_INFO sCntBlkRtInfo;
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter Config starts here.");
+
+	for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+	{
+		__maybe_unused IMG_UINT32 uiUnit;
+		__maybe_unused IMG_BOOL bDirect;
+
+		/* Exit early if this core does not have any of these counter blocks
+		 * due to core type/BVNC features.... */
+		psBlkTypeDesc = &asCntBlkTypeModel[ui32BlkCfgIdx];
+		if (psBlkTypeDesc->pfnIsBlkPresent(psBlkTypeDesc, pvDevice, &sCntBlkRtInfo) == IMG_FALSE)
+		{
+			continue;
+		}
+
+		/* Program all counters in one block so those already on may
+		 * be configured off and vice-versa. */
+		for (ui32BlockID = psBlkTypeDesc->ui32CntBlkIdBase;
+			 ui32BlockID < psBlkTypeDesc->ui32CntBlkIdBase+sCntBlkRtInfo.ui32NumUnits;
+			 ui32BlockID++)
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "Unit %d Block : %s",
+			                      ui32BlockID-psBlkTypeDesc->ui32CntBlkIdBase,
+			                      psBlkTypeDesc->pszBlockNameComment);
+
+			/* Get the block configure store to update from the global store of
+			 * block configuration. This is used to remember the configuration
+			 * between configurations and core power on in APM.
+			 * For HWPERF_UNIFIED layout we will have a different
+			 * structure type to decode the HWPerf block. This is indicated by
+			 * the RGX_CNTBLK_ID_DA_MASK bit being set in the block-ID value. */
+
+			bDirect = (psBlkTypeDesc->ui32IndirectReg == 0U);
+			uiUnit = ui32BlockID - psBlkTypeDesc->ui32CntBlkIdBase;
+
+#if defined(HWPERF_UNIFIED)
+			if ((ui32BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+			{
+				psHWPerfInitDABlkData = rgxfw_hwperf_get_da_block_ctl(ui32BlockID, psHWPerfInitDataInt);
+
+				PVR_ASSERT(psHWPerfInitDABlkData);
+
+				psHWPerfInitDABlkData->eBlockID = ui32BlockID;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "eBlockID: The Block ID for the layout block. See RGX_HWPERF_CNTBLK_ID for further information.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->eBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->eBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitDABlkData->uiEnabled = 0U;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->uiEnabled) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->uiEnabled,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitDABlkData->uiNumCounters = 0U;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiNumCounters (X): Specifies the number of valid counters"
+			                      " [0..%d] which follow.", RGX_CNTBLK_COUNTERS_MAX);
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->uiNumCounters) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->uiNumCounters,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				for (ui32CounterIdx = 0; ui32CounterIdx < RGX_CNTBLK_COUNTERS_MAX; ui32CounterIdx++)
+				{
+					psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx] = IMG_UINT32_C(0x00000000);
+
+					if (bDirect)
+					{
+						PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+					                      "%s_COUNTER_%d",
+					                      psBlkTypeDesc->pszBlockNameComment,
+					                      ui32CounterIdx);
+					}
+					else
+					{
+						PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+					                      "%s%d_COUNTER_%d",
+					                      psBlkTypeDesc->pszBlockNameComment,
+					                      uiUnit, ui32CounterIdx);
+					}
+
+					DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+					        psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx],
+							PDUMP_FLAGS_CONTINUOUS);
+				}
+			}
+			else
+#endif	/* defined(HWPERF_UNIFIED) */
+			{
+				psHWPerfInitBlkData = rgxfw_hwperf_get_block_ctl(ui32BlockID, psHWPerfInitDataInt);
+				/* Assert to check for HWPerf block mis-configuration */
+				PVR_ASSERT(psHWPerfInitBlkData);
+
+				psHWPerfInitBlkData->ui32Valid = IMG_TRUE;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "bValid: This specifies if the layout block is valid for the given BVNC.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->ui32Valid) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->ui32Valid,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->ui32Enabled = IMG_FALSE;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "bEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->ui32Enabled) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->ui32Enabled,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->eBlockID = ui32BlockID;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "eBlockID: The Block ID for the layout block. See RGX_HWPERF_CNTBLK_ID for further information.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->eBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->eBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->uiCounterMask = 0x00;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiCounterMask: Bitmask for selecting the counters that need to be configured. (Bit 0 - counter0, bit 1 - counter1 and so on.)");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->uiCounterMask) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->uiCounterMask,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				for (ui32CounterIdx = RGX_CNTBLK_COUNTER0_ID; ui32CounterIdx < psBlkTypeDesc->ui8NumCounters; ui32CounterIdx++)
+				{
+					psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx] = IMG_UINT64_C(0x0000000000000000);
+
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				                      "%s_COUNTER_%d", psBlkTypeDesc->pszBlockNameComment, ui32CounterIdx);
+					DevmemPDumpLoadMemValue64(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx],
+							PDUMP_FLAGS_CONTINUOUS);
+
+				}
+			}
+		}
+	}
+}
+/*!
+*******************************************************************************
+
+ @Function      InitialiseCustomCounters
+
+ @Description   Initialisation of custom counters and dumping them out to
+                pdump, so that they can be modified at a later point.
+
+ @Input         psHWPerfDataMemDesc
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseCustomCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc)
+{
+	IMG_UINT32 ui32CustomBlock, ui32CounterID;
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "ui32SelectedCountersBlockMask - The Bitmask of the custom counters that are to be selected");
+	DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+						offsetof(RGXFWIF_HWPERF_CTL, ui32SelectedCountersBlockMask),
+						0,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	for (ui32CustomBlock = 0; ui32CustomBlock < RGX_HWPERF_MAX_CUSTOM_BLKS; ui32CustomBlock++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiOffsetOfCustomBlockSelectedCounters
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_HWPERF_CTL *)0)->SelCntr[ui32CustomBlock].ui32NumSelectedCounters);
+
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "ui32NumSelectedCounters - The Number of counters selected for this Custom Block: %d",ui32CustomBlock );
+		DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+					uiOffsetOfCustomBlockSelectedCounters,
+					0,
+					PDUMP_FLAGS_CONTINUOUS);
+
+		for (ui32CounterID = 0; ui32CounterID < RGX_HWPERF_MAX_CUSTOM_CNTRS; ui32CounterID++ )
+		{
+			const IMG_DEVMEM_OFFSET_T uiOffsetOfCustomBlockSelectedCounterIDs
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_HWPERF_CTL *)0)->SelCntr[ui32CustomBlock].aui32SelectedCountersIDs[ui32CounterID]);
+
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "CUSTOMBLK_%d_COUNTERID_%d",ui32CustomBlock, ui32CounterID);
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+					uiOffsetOfCustomBlockSelectedCounterIDs,
+					0,
+					PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseAllCounters
+
+ @Description   Initialise HWPerf and custom counters
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitialiseAllCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	RGXFWIF_HWPERF_CTL *psHWPerfInitData;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc, (void **)&psHWPerfInitData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", failHWPerfCountersMemDescAqCpuVirt);
+
+	InitialiseHWPerfCounters(psDeviceNode, psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc, psHWPerfInitData);
+	InitialiseCustomCounters(psDeviceNode, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+	RGXFwSharedMemCacheOpPtr(psHWPerfInitData, FLUSH);
+
+failHWPerfCountersMemDescAqCpuVirt:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+
+	return eError;
+}
+#endif /* PDUMP */
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR RGXValidateTDHeap(PVRSRV_DEVICE_NODE *psDeviceNode,
+									  PVRSRV_PHYS_HEAP ePhysHeap,
+									  PHYS_HEAP_USAGE_FLAGS ui32RequiredFlags)
+{
+	PHYS_HEAP *psHeap = psDeviceNode->apsPhysHeap[ePhysHeap];
+	PHYS_HEAP_USAGE_FLAGS ui32HeapFlags = PhysHeapGetFlags(psHeap);
+	PHYS_HEAP_USAGE_FLAGS ui32InvalidFlags = ~(PHYS_HEAP_USAGE_FW_PRIV_DATA | PHYS_HEAP_USAGE_FW_CODE
+											   | PHYS_HEAP_USAGE_GPU_SECURE | PHYS_HEAP_USAGE_FW_PRIVATE);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32RequiredFlags) != 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap is missing required flags. flags: 0x%x / required:0x%x",
+							   ui32HeapFlags,
+							   ui32RequiredFlags);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32InvalidFlags) == 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap uses invalid flags. flags: 0x%x / invalid:0x%x",
+							   ui32HeapFlags,
+							   ui32InvalidFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXValidateTDHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_PRIV_DATA, PHYS_HEAP_USAGE_FW_PRIV_DATA);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_PRIV_DATA");
+
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_CODE, PHYS_HEAP_USAGE_FW_CODE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_CODE");
+	}
+
+	eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_GPU_SECURE, PHYS_HEAP_USAGE_GPU_SECURE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:GPU_SECURE");
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInit
+
+ @Description   RGX Initialisation
+
+ @Input         psDeviceNode
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Services initialisation parameters */
+	RGX_INIT_APPHINTS sApphints = {0};
+	IMG_UINT32 ui32FWConfigFlags, ui32FWConfigFlagsExt, ui32FwOsCfgFlags;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 1");
+
+	PDUMPCOMMENT(psDeviceNode, "Device Name: %s",
+	             psDeviceNode->psDevConfig->pszName);
+	PDUMPCOMMENT(psDeviceNode, "Device ID: %u (%d)",
+	             psDeviceNode->sDevId.ui32InternalID,
+	             psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->psDevConfig->pszVersion)
+	{
+		PDUMPCOMMENT(psDeviceNode, "Device Version: %s",
+		             psDeviceNode->psDevConfig->pszVersion);
+	}
+
+	/* pdump info about the core */
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGX Version Information (KM): %d.%d.%d.%d",
+	             psDevInfo->sDevFeatureCfg.ui32B,
+	             psDevInfo->sDevFeatureCfg.ui32V,
+	             psDevInfo->sDevFeatureCfg.ui32N,
+	             psDevInfo->sDevFeatureCfg.ui32C);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	eError = RGXValidateTDHeaps(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeaps");
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		/* The RGX_CR_MTS_DM4_INTERRUPT_ENABLE register is always set by the firmware during initialisation
+		 * and it provides a good method of determining if the firmware has been booted previously */
+		psDeviceNode->bAutoVzFwIsUp = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_DM4_INTERRUPT_ENABLE) != 0);
+
+		PVR_LOG(("AutoVz startup check: firmware is %s;",
+				(psDeviceNode->bAutoVzFwIsUp) ? "already running" : "powered down"));
+		PVR_LOG(("AutoVz allow GPU powerdown is %s:",
+				(psDeviceNode->bAutoVzAllowGPUPowerdown) ? "enabled" : "disabled"));
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest assumes the firmware is always available */
+		psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+		psDeviceNode->bAutoVzAllowGPUPowerdown = IMG_FALSE;
+	}
+	else
+#endif
+	{
+		/* Firmware does not follow the AutoVz life-cycle */
+		psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+	{
+		/* set the device power state here as the regular power
+		 * callbacks will not be executed on this driver */
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+
+	/* Set which HW Safety Events will be handled by the driver */
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER) ?
+										  RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= (RGX_DEVICE_HAS_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS)
+										   && (RGX_DEVICE_GET_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS) > 0)) ?
+										  RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_EN : 0;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Register defs revision: %d", RGX_CR_DEFS_KM_REVISION);
+#endif
+
+	/* Services initialisation parameters */
+	GetApphints(psDevInfo, &sApphints);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	if ((sApphints.bEnableTrustedDeviceAceConfig) &&
+		(RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE)))
+	{
+		SetTrustedDeviceAceEnabled(psDeviceNode->psDevConfig->hSysData);
+	}
+#endif
+#endif
+
+	eError = RGXInitCreateFWKernelMemoryContext(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create FW kernel memory context (%u)",
+		         __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitFirmware(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitFirmware failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Setup Firmware initialisation data
+	 */
+
+	GetFWConfigFlags(psDeviceNode, &sApphints, &ui32FWConfigFlags, &ui32FWConfigFlagsExt, &ui32FwOsCfgFlags);
+
+	eError = RGXInitFirmware(psDeviceNode,
+	                         &sApphints,
+	                         ui32FWConfigFlags,
+	                         ui32FWConfigFlagsExt,
+	                         ui32FwOsCfgFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitFirmware failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+#if defined(PDUMP)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitialiseAllCounters(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitialiseAllCounters failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+#endif
+
+	/*
+	 * Perform second stage of RGX initialisation
+	 */
+	eError = RGXInitDevPart2(psDeviceNode,
+	                         &sApphints);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitDevPart2 failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+
+	eError = PVRSRV_OK;
+
+cleanup:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxsrvinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxstartstop.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxstartstop.c
new file mode 100644
index 000000000000..2b93b46ff3ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxstartstop.c
@@ -0,0 +1,1245 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific start/stop routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific start/stop routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxstartstop.h"
+
+#define SOC_FEATURE_STRICT_SAME_ADDRESS_WRITE_ORDERING
+
+/*
+	RGXWriteMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAT;
+	}
+
+	eError = RGXPollReg32(hPrivate, ui32StateReg, ui32StateReadyFlag, ui32StateReadyFlag);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Write */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+		RGXWriteReg32(hPrivate, ui32DataReg, ui32RegValue);
+		(void) RGXReadReg32(hPrivate, ui32DataReg); /* Fence write */
+	}
+
+	return eError;
+}
+
+/*
+	RGXReadMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN);
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN);
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAX;
+		BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	}
+
+	/* Wait for Slave Port to be Ready */
+	eError = RGXPollReg32(hPrivate,
+						  ui32StateReg,
+						  ui32StateReadyFlag,
+						  ui32StateReadyFlag);
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Read */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+
+		/* Wait for Slave Port to be Ready */
+		eError = RGXPollReg32(hPrivate,
+				  ui32StateReg,
+				  ui32StateReadyFlag,
+				  ui32StateReadyFlag);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+#if !defined(NO_HARDWARE)
+	*ui32RegValue = RGXReadReg32(hPrivate, ui32DataReg);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32DataReg);
+	*ui32RegValue = 0xFFFFFFFF;
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXWriteMetaCoreRegThoughSP(const void *hPrivate,
+                                                IMG_UINT32 ui32CoreReg,
+                                                IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 i = 0;
+
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXDT_OFFSET, ui32Value);
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, ui32CoreReg & ~META_CR_TXUXXRXRQ_RDnWR_BIT);
+
+	do
+	{
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, &ui32Value);
+	} while (((ui32Value & META_CR_TXUXXRXRQ_DREADY_BIT) != META_CR_TXUXXRXRQ_DREADY_BIT) && (i++ < 1000));
+
+	if (i == 1000)
+	{
+		RGXCommentLog(hPrivate, "RGXWriteMetaCoreRegThoughSP: Timeout");
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXStartFirmware(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+
+	/* Give privilege to debug and slave port */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+
+	/* Point Meta to the bootloader address, global (uncached) range */
+	eError = RGXWriteMetaCoreRegThoughSP(hPrivate,
+	                                     PC_ACCESS(0),
+	                                     RGXFW_BOOTLDR_META_ADDR | META_MEM_GLOBAL_RANGE_BIT);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start failed!");
+		return eError;
+	}
+
+	/* Enable minim encoding */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXPRIVEXT, META_CR_TXPRIVEXT_MINIM_EN);
+
+	/* Enable Meta thread */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_T0ENABLE_OFFSET, META_CR_TXENABLE_ENABLE_BIT);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMetaProcWrapper
+
+ @Description   Configures the hardware wrapper of the META processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMetaProcWrapper(const void *hPrivate)
+{
+	IMG_UINT64 ui64GartenConfig;
+
+	/* Garten IDLE bit controlled by META */
+	ui64GartenConfig = RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META;
+
+#if defined(RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT)
+	/* Set the Garten Wrapper BIF Fence address */
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT;
+
+		/* Set SLC DM=META */
+		ui64GartenConfig |= ((IMG_UINT64) RGXFW_SEGMMU_META_BIFDM_ID) << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT;
+	}
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure META wrapper");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMipsProcWrapper
+
+ @Description   Configures the hardware wrapper of the MIPS processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMipsProcWrapper(const void *hPrivate)
+{
+	IMG_DEV_PHYADDR sPhyAddr;
+	IMG_UINT64 ui64RemapSettings = RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE; /* Same for all remap registers */
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure MIPS wrapper");
+
+	/*
+	 * MIPS wrapper (registers transaction ID and ISA mode) setup
+	 */
+
+	RGXCommentLog(hPrivate, "RGXStart: Write wrapper config register");
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		RGXWriteReg32(hPrivate,
+		              RGX_CR_MIPS_WRAPPER_CONFIG,
+		              (RGXMIPSFW_REGISTERS_VIRTUAL_BASE >>
+		              RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN) |
+		              RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+	else
+	{
+		RGXAcquireGPURegsAddr(hPrivate, &sPhyAddr);
+
+		RGXMIPSWrapperConfig(hPrivate,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG,
+		                     sPhyAddr.uiAddr,
+		                     RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+
+	/*
+	 * Boot remap setup
+	 */
+
+	RGXAcquireBootRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write boot remap registers");
+	RGXBootRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG1,
+	                   RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+#if defined(FIX_HW_BRN_63553_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 63553))
+	{
+		IMG_BOOL bPhysBusAbove32Bit = RGXGetDevicePhysBusWidth(hPrivate) > 32;
+		IMG_BOOL bDevicePA0IsValid  = RGXDevicePA0IsValid(hPrivate);
+
+		/* WA always required on 36 bit cores, to avoid continuous unmapped memory accesses to address 0x0 */
+		if (bPhysBusAbove32Bit || !bDevicePA0IsValid)
+		{
+			RGXCodeRemapConfig(hPrivate,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG1,
+					0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG2,
+					sPhyAddr.uiAddr,
+					~RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK,
+					ui64RemapSettings);
+		}
+	}
+#endif
+
+	/*
+	 * Data remap setup
+	 */
+
+	RGXAcquireDataRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		/* Remapped private data in secure memory */
+		ui64RemapSettings |= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN;
+	}
+	else
+	{
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+	}
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+	RGXCommentLog(hPrivate, "RGXStart: Write data remap registers");
+	RGXDataRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG1,
+	                   RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	/*
+	 * Code remap setup
+	 */
+
+	RGXAcquireCodeRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write exceptions remap registers");
+	RGXCodeRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG1,
+	                   RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) == 32)
+	{
+		/*
+		 * Trampoline remap setup
+		 */
+
+		RGXAcquireTrampolineRemapAddr(hPrivate, &sPhyAddr);
+		ui64RemapSettings = RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+		ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+		RGXCommentLog(hPrivate, "RGXStart: Write trampoline remap registers");
+		RGXTrampolineRemapConfig(hPrivate,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG1,
+		                         sPhyAddr.uiAddr | RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG2,
+		                         RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+		                         ~RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK,
+		                         ui64RemapSettings);
+	}
+
+	/* Garten IDLE bit controlled by MIPS */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to MIPS");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+
+	/* Turn on the EJTAG probe (only useful driver live) */
+	RGXWriteReg32(hPrivate, RGX_CR_MIPS_DEBUG_CONFIG, 0);
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitRiscvProcWrapper
+
+ @Description   Configures the hardware wrapper of the RISCV processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitRiscvProcWrapper(const void *hPrivate)
+{
+	IMG_UINT32 ui32BootCodeRemap = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+	IMG_UINT32 ui32BootDataRemap = RGXRISCVFW_BOOTLDR_DATA_REMAP;
+	IMG_DEV_VIRTADDR sTmp;
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure RISCV wrapper");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) < 4)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Write boot code remap");
+		RGXAcquireBootCodeAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootCodeRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN);
+
+		RGXCommentLog(hPrivate, "RGXStart: Write boot data remap");
+		RGXAcquireBootDataAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootDataRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN)
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN |
+#endif
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN);
+	}
+
+	/* Garten IDLE bit controlled by RISCV */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to RISCV");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      __RGXInitSLC
+
+ @Description   Initialise RGX SLC
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void __RGXInitSLC(const void *hPrivate)
+{
+	{
+		IMG_UINT32 ui32Reg;
+		IMG_UINT32 ui32RegVal;
+		IMG_UINT64 ui64RegVal;
+
+		/*
+		 * SLC Bypass control
+		 */
+		ui32Reg = RGX_CR_SLC_CTRL_BYPASS;
+		ui64RegVal = 0;
+
+#if defined(RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN)
+		if ((RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, SLC_SIZE_IN_KILOBYTES) == 8)  ||
+		    RGX_DEVICE_HAS_BRN(hPrivate, 61450))
+		{
+			RGXCommentLog(hPrivate, "Bypass SLC for IPF_OBJ and IPF_CPF");
+			ui64RegVal |= (IMG_UINT64) RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN |
+						(IMG_UINT64) RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_EN;
+		}
+#endif
+
+		if (ui64RegVal != 0)
+		{
+			RGXReadModifyWriteReg64(hPrivate, ui32Reg, ui64RegVal, ~ui64RegVal);
+		}
+
+		/*
+		 * SLC Misc control.
+		 *
+		 * Note: This is a 64bit register and we set only the lower 32bits leaving the top
+		 *       32bits (RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS) unchanged from the HW default.
+		 */
+		ui32Reg = RGX_CR_SLC_CTRL_MISC;
+		ui32RegVal = RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1;
+		ui32RegVal |= RGXReadReg32(hPrivate, ui32Reg) & RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+
+#if defined(FIX_HW_BRN_60084_BIT_MASK)
+		if (RGX_DEVICE_HAS_BRN(hPrivate, 60084))
+		{
+#if !defined(SOC_FEATURE_STRICT_SAME_ADDRESS_WRITE_ORDERING)
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+#else
+			if (RGX_DEVICE_HAS_ERN(hPrivate, 61389))
+			{
+				ui32RegVal |= RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+			}
+#endif
+		}
+#endif
+
+		/* Bypass burst combiner if SLC line size is smaller than 1024 bits */
+		if (RGXGetDeviceCacheLineSize(hPrivate) < 1024)
+		{
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN;
+		}
+
+		if (RGX_DEVICE_HAS_BRN(hPrivate, 71242) && !RGX_DEVICE_HAS_FEATURE(hPrivate, GPU_MULTICORE_SUPPORT))
+		{
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_EN;
+		}
+
+		RGXWriteReg32(hPrivate, ui32Reg, ui32RegVal);
+	}
+}
+
+
+static void RGXWriteKernelCatBase(const void *hPrivate, IMG_DEV_PHYADDR sPCAddr)
+{
+	{
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC64(hPrivate,
+		                      BIF_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT,
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT,
+		                      ((sPCAddr.uiAddr
+		                      >> RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT)
+		                      << RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT)
+		                      & ~RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK);
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Keep catbase registers in sync */
+			RGXWriteKernelMMUPC64(hPrivate,
+			                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+			                      ((sPCAddr.uiAddr
+			                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+			                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+			                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+		}
+
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXWriteKernelCatBase: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitBIF
+
+ @Description   Initialise RGX BIF
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitBIF(const void *hPrivate)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXInitBIF: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+	else
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+	{
+		IMG_DEV_PHYADDR sPCAddr;
+
+		/*
+		 * Acquire the address of the Kernel Page Catalogue.
+		 */
+		RGXAcquireKernelMMUPC(hPrivate, &sPCAddr);
+
+		/*
+		 * Write the kernel catalogue base.
+		 */
+		RGXCommentLog(hPrivate, "RGX firmware MMU Page Catalogue");
+
+		RGXWriteKernelCatBase(hPrivate, sPCAddr);
+	}
+}
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/**************************************************************************/ /*!
+@Function       RGXInitMMURangeRegisters
+@Description    Initialises MMU range registers for Non4K pages.
+@Input          hPrivate           Implementation specific data
+@Return         void
+ */ /**************************************************************************/
+static void RGXInitMMURangeRegisters(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psParams->psDevInfo;
+	IMG_UINT32 ui32RegAddr = RGX_CR_MMU_PAGE_SIZE_RANGE_ONE;
+	IMG_UINT32 i;
+
+	for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i, ui32RegAddr += sizeof(IMG_UINT64))
+	{
+		RGXWriteReg64(hPrivate, ui32RegAddr, psDevInfo->aui64MMUPageSizeRangeValue[i]);
+	}
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitAXIACE
+
+ @Description    Initialise AXI-ACE interface
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitAXIACE(const void *hPrivate)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	ui32RegAddr = RGX_CR_AXI_ACE_LITE_CONFIGURATION;
+
+	/* Setup AXI-ACE config. Set everything to outer cache */
+	ui64RegVal = (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT) |
+	             (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT)  |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT);
+
+#if defined(FIX_HW_BRN_42321_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 42321))
+	{
+		ui64RegVal |= (((IMG_UINT64) 1) << RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT);
+	}
+#endif
+
+#if defined(FIX_HW_BRN_68186_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 68186))
+	{
+		/* default value for reg_enable_fence_out is zero. Force to 1 to allow core_clk < mem_clk */
+		ui64RegVal |= (IMG_UINT64)1 << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT;
+	}
+#endif
+#else /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+	ui32RegAddr = RGX_CR_ACE_CTRL;
+
+	/**
+	 * The below configuration is only applicable for RGX cores supporting
+	 * ACE/ACE-lite protocol and connected to ACE coherent interconnect.
+	 */
+
+	/**
+	 * Configure AxDomain and AxCache for MMU transactions.
+	 * AxDomain set to non sharable (0x0).
+	 */
+	ui64RegVal = RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE |
+				 RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE;
+
+	/**
+	 * Configure AxCache for PM/MMU transactions.
+	 * Set to same value (i.e WBRWALLOC caching, rgxmmunit.c:RGXDerivePTEProt8)
+	 * as non-coherent PTEs
+	 */
+	ui64RegVal |= (IMG_UINT64_C(0xF)) << RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT;
+
+	/**
+	 * Configure AxDomain for non MMU transactions.
+	 */
+	ui64RegVal |= (IMG_UINT64)(RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE |
+							   RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE);
+#endif /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+
+	RGXCommentLog(hPrivate, "Init AXI-ACE interface");
+	RGXWriteReg64(hPrivate, ui32RegAddr, ui64RegVal);
+}
+
+static void RGXResetSequence(const void *hPrivate, const IMG_CHAR *pcRGXFW_PROCESSOR)
+{
+	IMG_UINT64 ui64SoftResetMask;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXCommentLog(hPrivate, "RGXStart: soft reset cpu core");
+		RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 0);
+	}
+
+	{
+		/* Set RGX in soft-reset */
+		RGXCommentLog(hPrivate, "RGXStart: soft reset everything");
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, ui64SoftResetMask);
+
+		/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+		/* Take Rascal and Dust out of reset */
+		RGXCommentLog(hPrivate, "RGXStart: Rascal and Dust out of reset");
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, ui64SoftResetMask ^ RGX_CR_SOFT_RESET_RASCALDUSTS_EN);
+
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+		/* Take everything out of reset but the FW processor */
+		RGXCommentLog(hPrivate, "RGXStart: Take everything out of reset but %s", pcRGXFW_PROCESSOR);
+
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+	}
+}
+
+static void DeassertMetaReset(const void *hPrivate)
+{
+	/* Need to wait for at least 32 cycles before taking the FW processor out of reset ... */
+	RGXWaitCycles(hPrivate, 32, 3);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, 0x0);
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* ... and afterwards */
+	RGXWaitCycles(hPrivate, 32, 3);
+}
+
+PVRSRV_ERROR RGXStart(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR *pcRGXFW_PROCESSOR;
+	IMG_BOOL bDoFWSlaveBoot = IMG_FALSE;
+	IMG_BOOL bMetaFW = IMG_FALSE;
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_RISCV;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_MIPS;
+	}
+#endif
+	else
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_META;
+		bMetaFW = IMG_TRUE;
+		bDoFWSlaveBoot = RGXDoFWSlaveBoot(hPrivate);
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SYS_BUS_SECURE_RESET))
+	{
+		/* Disable the default sys_bus_secure protection to perform minimal setup */
+		RGXCommentLog(hPrivate, "RGXStart: Disable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, 0);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+
+#if defined(RGX_FEATURE_HYPERVISOR_MMU_BIT_MASK)
+	/* Only bypass HMMU if the module is present */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, HYPERVISOR_MMU))
+	{
+		/* Always set HMMU in bypass mode */
+		RGXWriteReg32(hPrivate, RGX_CR_HMMU_BYPASS, RGX_CR_HMMU_BYPASS_MASKFULL);
+		(void) RGXReadReg32(hPrivate, RGX_CR_HMMU_BYPASS);
+	}
+#endif
+
+	/*!
+	 * Start FW init sequence
+	 */
+	RGXResetSequence(hPrivate, pcRGXFW_PROCESSOR);
+
+	/*
+	 * Initialise SLC.
+	 */
+	__RGXInitSLC(hPrivate);
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) > 0)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Enable safety events");
+		RGXWriteReg32(hPrivate, RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE,
+		                        RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL);
+	}
+
+	if (bMetaFW)
+	{
+		if (bDoFWSlaveBoot)
+		{
+			/* Configure META to Slave boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Slave boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		}
+		else
+		{
+			/* Configure META to Master boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Master boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN);
+		}
+	}
+
+	/*
+	 * Initialise Firmware wrapper
+	 */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXInitRiscvProcWrapper(hPrivate);
+	}
+	else if (bMetaFW)
+	{
+		RGXInitMetaProcWrapper(hPrivate);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		RGXInitMipsProcWrapper(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, MMU_VERSION) >= 4)
+	{
+		/* initialise the MMU range based config registers for Non4K pages */
+		RGXInitMMURangeRegisters(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, AXI_ACELITE))
+#endif
+	{
+		/* We must init the AXI-ACE interface before 1st BIF transaction */
+		RGXInitAXIACE(hPrivate);
+	}
+
+	/*
+	 * Initialise BIF.
+	 */
+	RGXInitBIF(hPrivate);
+
+	RGXSetPoweredState(hPrivate, IMG_TRUE);
+
+	RGXCommentLog(hPrivate, "RGXStart: Take %s out of reset", pcRGXFW_PROCESSOR);
+	DeassertMetaReset(hPrivate);
+
+	if (bMetaFW && bDoFWSlaveBoot)
+	{
+		eError = RGXFabricCoherencyTest(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start");
+		eError = RGXStartFirmware(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+	}
+	else
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Master boot Start");
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Bring Debug Module out of reset */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) >= 4)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN);
+			}
+			else
+#endif
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL, RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+			}
+
+			/* Boot the FW */
+			RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 1);
+			RGXWaitCycles(hPrivate, 32, 3);
+		}
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	RGXCommentLog(hPrivate, "RGXStart: Enable sys_bus_secure");
+	RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN);
+	(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStop(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+
+	RGXDeviceAckIrq(hPrivate);
+
+	/* Set FW power state OFF to disable LISR handler */
+	RGXSetPoweredState(hPrivate, IMG_FALSE);
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper
+	 * For LAYOUT_MARS = 1, SIDEKICK would have been powered down by FW
+	 */
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN),
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN));
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		/*
+		 * Wait for SLC to signal IDLE
+		 * For LAYOUT_MARS = 1, SLC would have been powered down by FW
+		 */
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SLC_IDLE,
+					RGX_CR_SLC_IDLE_MASKFULL,
+					RGX_CR_SLC_IDLE_MASKFULL);
+		}
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	/* Unset MTS DM association with threads */
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL);
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL);
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL);
+	}
+
+#if defined(PDUMP)
+	if (bMetaFW)
+	{
+		/* Disabling threads is only required for pdumps to stop the fw gracefully */
+
+		/* Disable thread 0 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T0ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Disable thread 1 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T1ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Clear down any irq raised by META (done after disabling the FW
+		 * threads to avoid a race condition).
+		 * This is only really needed for PDumps but we do it anyway driver-live.
+		 */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		/* Wait for the Slave Port to finish all the transactions */
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+		{
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+			else
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+		}
+		else
+#endif
+		{
+			RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS, 0x0);
+			(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS); /* Fence write */
+
+			eError = RGXPollReg32(hPrivate,
+								  RGX_CR_META_SP_MSLVCTRL1,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN);
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+#endif
+
+	/* Extra Idle checks */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIF_STATUS_MMU,
+	                      0,
+	                      RGX_CR_BIF_STATUS_MMU_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIFPM_STATUS_MMU,
+	                      0,
+	                      RGX_CR_BIFPM_STATUS_MMU_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	if (!RGX_DEVICE_HAS_FEATURE(hPrivate, XT_TOP_INFRASTRUCTURE))
+	{
+		eError = RGXPollReg32(hPrivate,
+		                      RGX_CR_BIF_READS_EXT_STATUS,
+		                      0,
+		                      RGX_CR_BIF_READS_EXT_STATUS_MASKFULL);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIFPM_READS_EXT_STATUS,
+	                      0,
+	                      RGX_CR_BIFPM_READS_EXT_STATUS_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS1,
+	                      0,
+		                  RGX_CR_SLC_STATUS1_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	if (4 == RGXGetDeviceSLCBanks(hPrivate))
+	{
+		eError = RGXPollReg64(hPrivate,
+		                      RGX_CR_SLC_STATUS2,
+		                      0,
+		                      RGX_CR_SLC_STATUS2_MASKFULL);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		/*
+		 * Wait for SLC to signal IDLE
+		 * For LAYOUT_MARS = 1, SLC would have been powered down by FW
+		 */
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SLC_IDLE,
+					RGX_CR_SLC_IDLE_MASKFULL,
+					RGX_CR_SLC_IDLE_MASKFULL);
+		}
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper
+	 * For LAYOUT_MARS = 1, SIDEKICK would have been powered down by FW
+	 */
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN),
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN));
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (bMetaFW)
+	{
+		IMG_UINT32 ui32RegValue;
+
+		eError = RGXReadMetaRegThroughSP(hPrivate,
+		                                 META_CR_TxVECINT_BHALT,
+		                                 &ui32RegValue);
+		if (eError != PVRSRV_OK) return eError;
+
+		if ((ui32RegValue & 0xFFFFFFFFU) == 0x0)
+		{
+			/* Wait for Sidekick/Jones to signal IDLE including
+			 * the Garten Wrapper if there is no debugger attached
+			 * (TxVECINT_BHALT = 0x0) */
+			{
+				eError = RGXPollReg32(hPrivate,
+				                      RGX_CR_SIDEKICK_IDLE,
+				                      RGX_CR_SIDEKICK_IDLE_GARTEN_EN,
+				                      RGX_CR_SIDEKICK_IDLE_GARTEN_EN);
+				if (eError != PVRSRV_OK) return eError;
+			}
+		}
+	}
+	else
+	{
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0)
+		{
+			/* As FW core has been moved from SIDEKICK to the new MARS domain, checking
+			 * idle bits for CPU & System Arbiter excluding SOCIF which will never be Idle
+			 * if Host polling on this register
+			 */
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_MARS_IDLE,
+					RGX_CR_MARS_IDLE_CPU_EN | RGX_CR_MARS_IDLE_MH_SYSARB0_EN,
+					RGX_CR_MARS_IDLE_CPU_EN | RGX_CR_MARS_IDLE_MH_SYSARB0_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+		else
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_GARTEN_EN,
+					RGX_CR_SIDEKICK_IDLE_GARTEN_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxstartstop.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.c
new file mode 100644
index 000000000000..515b988e938f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.c
@@ -0,0 +1,5785 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA/3D routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX TA/3D routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* for the offsetof macro */
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxta3d.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "ri_server.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "rgx_memallocflags.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxsyncutils.h"
+#include "htbserver.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "physmem.h"
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "process_stats.h"
+
+#if !defined(PM_INTERACTIVE_MODE)
+#include "rgxpmdefs.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+#include "rgxmmudefs_km.h"
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TA3D_UFO_DUMP	0
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static INLINE
+void _DebugSyncValues(const IMG_CHAR *pszFunction,
+					  const IMG_UINT32 *pui32UpdateValues,
+					  const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", pszFunction, i, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+
+static INLINE
+void _DebugSyncCheckpoints(const IMG_CHAR *pszFunction,
+						   const IMG_CHAR *pszDMName,
+						   const PSYNC_CHECKPOINT *apsSyncCheckpoints,
+						   const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFence%sSyncCheckpoints[%d]=<%p>", pszFunction, pszDMName, i, *(apsSyncCheckpoints + i)));
+	}
+}
+
+#else
+#define CHKPT_DBG(X)
+#endif
+
+/* define the number of commands required to be set up by the CCB helper */
+/* 1 command for the TA */
+#define CCB_CMD_HELPER_NUM_TA_COMMANDS 1
+/* Up to 3 commands for the 3D (partial render fence, partial render, and render) */
+#define CCB_CMD_HELPER_NUM_3D_COMMANDS 3
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define WORKEST_CYCLES_PREDICTION_GET(x) ((x).ui32CyclesPrediction)
+#else
+#define WORKEST_CYCLES_PREDICTION_GET(x) (NO_CYCEST)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_TA_DATA;
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_3D_DATA;
+
+struct _RGX_SERVER_RENDER_CONTEXT_ {
+	/* this lock protects usage of the render context.
+	 * it ensures only one kick is being prepared and/or submitted on
+	 * this render context at any time
+	 */
+	POS_LOCK				hLock;
+	RGX_CCB_CMD_HELPER_DATA asTACmdHelperData[CCB_CMD_HELPER_NUM_TA_COMMANDS];
+	RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[CCB_CMD_HELPER_NUM_3D_COMMANDS];
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRenderContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	RGX_SERVER_RC_TA_DATA		sTAData;
+	RGX_SERVER_RC_3D_DATA		s3DData;
+	IMG_UINT32					ui32CleanupStatus;
+#define RC_CLEANUP_TA_COMPLETE		(1 << 0)
+#define RC_CLEANUP_3D_COMPLETE		(1 << 1)
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListTAFence;
+	SYNC_ADDR_LIST				sSyncAddrListTAUpdate;
+	SYNC_ADDR_LIST				sSyncAddrList3DFence;
+	SYNC_ADDR_LIST				sSyncAddrList3DUpdate;
+	ATOMIC_T					hIntJobRef;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+
+/*
+	Static functions used by render context code
+*/
+
+static
+PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  psTAData->psServerCommonContext,
+											  RGXFWIF_DM_GEOM,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psTAData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, psTAData->psContextStateMemDesc);
+	psTAData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psContextStateMemDesc);
+	ps3DData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static void _RGXDumpPMRPageList(DLLIST_NODE *psNode)
+{
+	RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	PVRSRV_ERROR			eError;
+
+	eError = PMRDumpPageList(psPMRNode->psPMR,
+							RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Error (%s) printing pmr %p",
+				PVRSRVGetErrorString(eError),
+				psPMRNode->psPMR));
+	}
+}
+
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016" IMG_UINT64_FMTSPECx,
+				psFreeList->sFreeListFWDevVAddr.ui32Addr,
+				psFreeList->ui32FreelistID,
+				psFreeList->ui64FreelistChecksum));
+
+	/* Dump Init FreeList page list */
+	PVR_LOG(("  Initial Memory block"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	/* Dump Grow FreeList page list */
+	PVR_LOG(("  Grow Memory blocks"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	return IMG_TRUE;
+}
+
+static void _CheckFreelist(RGX_FREELIST *psFreeList,
+						   IMG_UINT32 ui32NumOfPagesToCheck,
+						   IMG_UINT64 ui64ExpectedCheckSum,
+						   IMG_UINT64 *pui64CalculatedCheckSum)
+{
+#if defined(NO_HARDWARE)
+	/* No checksum needed as we have all information in the pdumps */
+	PVR_UNREFERENCED_PARAMETER(psFreeList);
+	PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
+	PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
+	*pui64CalculatedCheckSum = 0;
+#else
+	PVRSRV_ERROR eError;
+	size_t uiNumBytes;
+	IMG_UINT8* pui8Buffer;
+	IMG_UINT32* pui32Buffer;
+	IMG_UINT32 ui32CheckSumAdd = 0;
+	IMG_UINT32 ui32CheckSumXor = 0;
+	IMG_UINT32 ui32Entry;
+	IMG_UINT32 ui32Entry2;
+	IMG_BOOL bFreelistBad = IMG_FALSE;
+
+	*pui64CalculatedCheckSum = 0;
+
+	PVR_ASSERT(ui32NumOfPagesToCheck <= (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+	/* Allocate Buffer of the size of the freelist */
+	pui8Buffer = OSAllocMem(ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+	if (pui8Buffer == NULL)
+	{
+		PVR_LOG(("%s: Failed to allocate buffer to check freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	/* Copy freelist content into Buffer */
+	eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
+					psFreeList->uiFreeListPMROffset +
+					(((psFreeList->ui32MaxFLPages -
+					   psFreeList->ui32CurrentFLPages - psFreeList->ui32ReadyFLPages) * sizeof(IMG_UINT32)) &
+					 ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1)),
+					 pui8Buffer,
+					 ui32NumOfPagesToCheck * sizeof(IMG_UINT32),
+					&uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pui8Buffer);
+		PVR_LOG(("%s: Failed to get freelist data for freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	PVR_ASSERT(uiNumBytes == ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+
+	/* Generate checksum (skipping the first page if not allocated) */
+	pui32Buffer = (IMG_UINT32 *)pui8Buffer;
+	ui32Entry = ((psFreeList->ui32GrowFLPages == 0  &&  psFreeList->ui32CurrentFLPages > 1) ? 1 : 0);
+	for (/*ui32Entry*/ ; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
+	{
+		ui32CheckSumAdd += pui32Buffer[ui32Entry];
+		ui32CheckSumXor ^= pui32Buffer[ui32Entry];
+
+		/* Check for double entries */
+		for (ui32Entry2 = ui32Entry+1; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
+		{
+			if (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2])
+			{
+				PVR_LOG(("%s: Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
+						__func__,
+						psFreeList->sFreeListFWDevVAddr.ui32Addr,
+						pui32Buffer[ui32Entry2],
+						ui32Entry,
+						ui32Entry2,
+						psFreeList->ui32CurrentFLPages));
+				bFreelistBad = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	OSFreeMem(pui8Buffer);
+
+	/* Check the calculated checksum against the expected checksum... */
+	*pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
+
+	if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
+	{
+		PVR_LOG(("%s: Checksum mismatch for freelist %p! Expected 0x%016" IMG_UINT64_FMTSPECx " calculated 0x%016" IMG_UINT64_FMTSPECx,
+				__func__, psFreeList,
+				ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
+		bFreelistBad = IMG_TRUE;
+	}
+
+	if (bFreelistBad)
+	{
+		PVR_LOG(("%s: Sleeping for ever!", __func__));
+		PVR_ASSERT(!bFreelistBad);
+	}
+#endif
+}
+
+
+/*
+ *  Function to work out the number of freelist pages to reserve for growing
+ *  within the FW without having to wait for the host to progress a grow
+ *  request.
+ *
+ *  The number of pages must be a multiple of 4 to align the PM addresses
+ *  for the initial freelist allocation and also be less than the grow size.
+ *
+ *  If the threshold or grow size means less than 4 pages, then the feature
+ *  is not used.
+ */
+static IMG_UINT32 _CalculateFreelistReadyPages(RGX_FREELIST *psFreeList,
+                                               IMG_UINT32  ui32FLPages)
+{
+	IMG_UINT32  ui32ReadyFLPages = ((ui32FLPages * psFreeList->ui32GrowThreshold) / 100) &
+	                               ~((RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE/sizeof(IMG_UINT32))-1);
+
+	if (ui32ReadyFLPages > psFreeList->ui32GrowFLPages)
+	{
+		ui32ReadyFLPages = psFreeList->ui32GrowFLPages;
+	}
+
+	return ui32ReadyFLPages;
+}
+
+
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader)
+{
+	RGX_PMR_NODE	*psPMRNode;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32  ui32MappingTable = 0;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_DEVMEM_SIZE_T uistartPage;
+	PVRSRV_ERROR eError;
+	static const IMG_CHAR szAllocName[] = "Free List";
+
+	/* Are we allowed to grow ? */
+	if (psFreeList->ui32MaxFLPages - (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) < ui32NumPages)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: grow by %u pages denied. "
+				"Max PB size reached (current pages %u+%u/%u)",
+				psFreeList,
+				ui32NumPages,
+				psFreeList->ui32CurrentFLPages,
+				psFreeList->ui32ReadyFLPages,
+				psFreeList->ui32MaxFLPages));
+		return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
+	}
+
+	/* Allocate kernel memory block structure */
+	psPMRNode = OSAllocMem(sizeof(*psPMRNode));
+	if (psPMRNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to allocate host data structure",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocHost;
+	}
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages
+	 */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+
+	/*
+	 *  The PM never takes the last page in a freelist, so if this block
+	 *  of pages is the first one and there is no ability to grow, then
+	 *  we can skip allocating one 4K page for the lowest entry.
+	 */
+	if (OSGetPageSize() > RGX_BIF_PM_PHYSICAL_PAGE_SIZE)
+	{
+		/*
+		 * Allocation size will be rounded up to the OS page size,
+		 * any attempt to change it a bit now will be invalidated later.
+		 */
+		psPMRNode->bFirstPageMissing = IMG_FALSE;
+	}
+	else
+	{
+		psPMRNode->bFirstPageMissing = (psFreeList->ui32GrowFLPages == 0  &&  ui32NumPages > 1);
+	}
+
+	psPMRNode->ui32NumPages = ui32NumPages;
+	psPMRNode->psFreeList = psFreeList;
+
+	/* Allocate Memory Block */
+	PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Allocate PB Block (Pages %08X)", ui32NumPages);
+	uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	if (psPMRNode->bFirstPageMissing)
+	{
+		uiSize -= RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	}
+
+	eError = PhysmemNewRamBackedPMR(psFreeList->psConnection,
+			psFreeList->psDevInfo->psDeviceNode,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_PRIVATE) |
+			PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP,
+			sizeof(szAllocName),
+			szAllocName,
+			psFreeList->ownerPid,
+			&psPMRNode->psPMR,
+			PDUMP_NONE,
+			NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate PB block of size: 0x%016" IMG_UINT64_FMTSPECX,
+				 __func__,
+				 (IMG_UINT64)uiSize));
+		goto ErrorBlockAlloc;
+	}
+
+	/* Zeroing physical pages pointed by the PMR */
+	if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((uistartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRNode->psPMR,
+	                                    psFreeList->ownerPid);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: call to RIWritePMREntryWithOwnerKM failed (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+	}
+
+	 /* Attach RI information */
+	eError = RIWriteMEMDESCEntryKM(psFreeList->psConnection,
+	                               psFreeList->psDevInfo->psDeviceNode,
+	                               psPMRNode->psPMR,
+	                               OSStringNLength(szAllocName, DEVMEM_ANNOTATION_MAX_LEN),
+	                               szAllocName,
+	                               0,
+	                               uiSize,
+	                               0,
+	                               &psPMRNode->hRIHandle);
+	PVR_LOG_IF_ERROR(eError, "RIWriteMEMDESCEntryKM");
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	/* write Freelist with Memory Block physical addresses */
+	eError = PMRWritePMPageList(
+						/* Target PMR, offset, and length */
+						psFreeList->psFreeListPMR,
+						(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+						(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+						/* Referenced PMR, and "page" granularity */
+						psPMRNode->psPMR,
+						RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+						&psPMRNode->psPageList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to write pages of Node %p",
+				 __func__,
+				 psPMRNode));
+		goto ErrorPopulateFreelist;
+	}
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	/* Copy freelist memory to shadow freelist */
+	{
+		const IMG_UINT32 ui32FLMaxSize = psFreeList->ui32MaxFLPages * sizeof(IMG_UINT32);
+		const IMG_UINT32 ui32MapSize = ui32FLMaxSize * 2;
+		const IMG_UINT32 ui32CopyOffset = uiOffset - psFreeList->uiFreeListPMROffset;
+		IMG_BYTE *pFLMapAddr;
+		size_t uiNumBytes;
+		PVRSRV_ERROR res;
+		IMG_HANDLE hMapHandle;
+
+		/* Check for overflow. Validate size and offset. */
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize > psFreeList->uiFreeListPMROffset,
+		                          eError, ErrorPopulateFreelist);
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize <= PMR_LogicalSize(psFreeList->psFreeListPMR),
+		                          eError, ErrorPopulateFreelist);
+
+		/* Map both the FL and the shadow FL */
+		res = PMRAcquireKernelMappingData(psFreeList->psFreeListPMR, psFreeList->uiFreeListPMROffset, ui32MapSize,
+		                                  (void**) &pFLMapAddr, &uiNumBytes, &hMapHandle);
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map freelist (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+
+		/* Copy only the newly added memory */
+		OSCachedMemCopy(pFLMapAddr + ui32FLMaxSize + ui32CopyOffset, pFLMapAddr + ui32CopyOffset , uiLength);
+		OSWriteMemoryBarrier(pFLMapAddr);
+
+#if defined(PDUMP)
+		PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Initialize shadow freelist");
+
+		/* Translate memcpy to pdump */
+		{
+			IMG_DEVMEM_OFFSET_T uiCurrOffset;
+
+			for (uiCurrOffset = uiOffset; (uiCurrOffset - uiOffset) < uiLength; uiCurrOffset += sizeof(IMG_UINT32))
+			{
+				PMRPDumpCopyMem32(psFreeList->psFreeListPMR,
+				                  uiCurrOffset + ui32FLMaxSize,
+				                  psFreeList->psFreeListPMR,
+				                  uiCurrOffset,
+				                  ":SYSMEM:$1",
+				                  PDUMP_FLAGS_CONTINUOUS);
+			}
+		}
+#endif
+
+
+		res = PMRReleaseKernelMappingData(psFreeList->psFreeListPMR, hMapHandle);
+
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to release freelist mapping (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+	}
+#endif
+
+	/* We add It must be added to the tail, otherwise the freelist population won't work */
+	dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
+
+	/* Update number of available pages */
+	psFreeList->ui32CurrentFLPages += ui32NumPages;
+
+	/* Update statistics (needs to happen before the ReadyFL calculation to also count those pages) */
+	if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
+	{
+		psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
+	}
+
+	/* Reserve a number ready pages to allow the FW to process OOM quickly and asynchronously request a grow. */
+	psFreeList->ui32ReadyFLPages    = _CalculateFreelistReadyPages(psFreeList, psFreeList->ui32CurrentFLPages);
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+
+	if (psFreeList->bCheckFreelist)
+	{
+		/*
+		 *  We can only calculate the freelist checksum when the list is full
+		 *  (e.g. at initial creation time). At other times the checksum cannot
+		 *  be calculated and has to be disabled for this freelist.
+		 */
+		if ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages)
+		{
+			_CheckFreelist(psFreeList, ui32NumPages, 0, &psFreeList->ui64FreelistChecksum);
+		}
+		else
+		{
+			psFreeList->ui64FreelistChecksum = 0;
+		}
+	}
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: %s %u pages (pages=%u+%u/%u checksum=0x%016" IMG_UINT64_FMTSPECx "%s)",
+			psFreeList,
+	        ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages ? "Create initial" : "Grow by"),
+			ui32NumPages,
+			psFreeList->ui32CurrentFLPages,
+			psFreeList->ui32ReadyFLPages,
+			psFreeList->ui32MaxFLPages,
+			psFreeList->ui64FreelistChecksum,
+			(psPMRNode->bFirstPageMissing ? " - lowest page not allocated" : "")));
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+ErrorPopulateFreelist:
+	(void) PMRUnrefPMR(psPMRNode->psPMR);
+
+ErrorBlockAlloc:
+	OSFreeMem(psPMRNode);
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+ErrorAllocHost:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
+										RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode;
+	RGX_PMR_NODE *psPMRNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32OldValue;
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages value
+	 */
+	PVR_ASSERT(pListHeader);
+	PVR_ASSERT(psFreeList);
+	PVR_ASSERT(psFreeList->psDevInfo);
+	PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+	/* Get node from head of list and remove it */
+	psNode = dllist_get_next_node(pListHeader);
+	if (psNode)
+	{
+		dllist_remove_node(psNode);
+
+		psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+		PVR_ASSERT(psPMRNode);
+		PVR_ASSERT(psPMRNode->psPMR);
+		PVR_ASSERT(psPMRNode->psFreeList);
+
+		/* remove block from freelist list */
+
+		/* Unwrite Freelist with Memory Block physical addresses */
+		eError = PMRUnwritePMPageList(psPMRNode->psPageList);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to unwrite pages of Node %p",
+					 __func__,
+					 psPMRNode));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+		if (psPMRNode->hRIHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = RIDeleteMEMDESCEntryKM(psPMRNode->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "RIDeleteMEMDESCEntryKM");
+		}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+		/* Free PMR (We should be the only one that holds a ref on the PMR) */
+		eError = PMRUnrefPMR(psPMRNode->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to free PB block %p (%s)",
+					 __func__,
+					 psPMRNode->psPMR,
+					 PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+		/* update available pages in freelist */
+		ui32OldValue = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+
+		/*
+		 * Deallocated pages should first be deducted from ReadyPages bank, once
+		 * there are no more left, start deducting them from CurrentPage bank.
+		 */
+		if (psPMRNode->ui32NumPages > psFreeList->ui32ReadyFLPages)
+		{
+			psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages - psFreeList->ui32ReadyFLPages;
+			psFreeList->ui32ReadyFLPages = 0;
+		}
+		else
+		{
+			psFreeList->ui32ReadyFLPages -= psPMRNode->ui32NumPages;
+		}
+
+		/* check underflow */
+		PVR_ASSERT(ui32OldValue > (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
+								psFreeList,
+								psPMRNode->ui32NumPages,
+								psFreeList->ui32CurrentFLPages,
+								psFreeList->ui32MaxFLPages));
+
+		OSFreeMem(psPMRNode);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
+								psFreeList,
+								psFreeList->ui32InitFLPages));
+		eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
+	}
+
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	return eError;
+}
+
+static RGX_FREELIST *FindFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_FREELIST *psFreeList = NULL;
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		if (psThisFreeList->ui32FreelistID == ui32FreelistID)
+		{
+			psFreeList = psThisFreeList;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockFreeList);
+	return psFreeList;
+}
+
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT32 ui32FreelistID)
+{
+	RGX_FREELIST *psFreeList = NULL;
+	RGXFWIF_KCCB_CMD s3DCCBCmd;
+	IMG_UINT32 ui32GrowValue;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	psFreeList = FindFreeList(psDevInfo, ui32FreelistID);
+	if (psFreeList == NULL)
+	{
+		/* Should never happen */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "FreeList Lookup for FreeList ID 0x%08x failed (Populate)",
+		         ui32FreelistID));
+		PVR_ASSERT(IMG_FALSE);
+		return;
+	}
+
+	/* Since the FW made the request, it has already consumed the ready pages, update the host struct */
+	psFreeList->ui32CurrentFLPages += psFreeList->ui32ReadyFLPages;
+	psFreeList->ui32ReadyFLPages = 0;
+
+	/* Try to grow the freelist */
+	eError = RGXGrowFreeList(psFreeList,
+	                         psFreeList->ui32GrowFLPages,
+	                         &psFreeList->sMemoryBlockHead);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Grow successful, return size of grow size */
+		ui32GrowValue = psFreeList->ui32GrowFLPages;
+
+		psFreeList->ui32NumGrowReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		/* Update Stats */
+		PVRSRVStatsUpdateFreelistStats(psDevInfo->psDeviceNode,
+		                               0,
+		                               1, /* Add 1 to the appropriate counter (Requests by FW) */
+		                               psFreeList->ui32InitFLPages,
+		                               psFreeList->ui32NumHighPages,
+		                               psFreeList->ownerPid);
+
+#endif
+
+	}
+	else
+	{
+		/* Grow failed */
+		ui32GrowValue = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Grow for FreeList %p failed (%s)",
+		         psFreeList,
+		         PVRSRVGetErrorString(eError)));
+	}
+
+	/* send feedback */
+	s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
+	s3DCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaPages = ui32GrowValue;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages = psFreeList->ui32ReadyFLPages;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: Grow pages=%u, new pages=%u, ready pages=%u, counter=%d",
+	         psFreeList,
+	         ui32GrowValue,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages,
+	         psFreeList->ui32NumGrowReqByFW));
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_3D,
+		                            &s3DCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+static void _RGXFreeListReconstruction(PDLLIST_NODE psNode)
+{
+
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	RGX_FREELIST			*psFreeList;
+	RGX_PMR_NODE			*psPMRNode;
+	PVRSRV_ERROR			eError;
+	IMG_DEVMEM_OFFSET_T		uiOffset;
+	IMG_DEVMEM_SIZE_T		uiLength;
+	IMG_UINT32				ui32StartPage;
+
+	psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	psFreeList = psPMRNode->psFreeList;
+	PVR_ASSERT(psFreeList);
+	psDevInfo = psFreeList->psDevInfo;
+	PVR_ASSERT(psDevInfo);
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((ui32StartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+	PMRUnwritePMPageList(psPMRNode->psPageList);
+	psPMRNode->psPageList = NULL;
+	eError = PMRWritePMPageList(
+			/* Target PMR, offset, and length */
+			psFreeList->psFreeListPMR,
+			(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+			(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+			/* Referenced PMR, and "page" granularity */
+			psPMRNode->psPMR,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			&psPMRNode->psPageList);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error (%s) writing FL 0x%08x",
+				__func__,
+				PVRSRVGetErrorString(eError),
+				(IMG_UINT32)psFreeList->ui32FreelistID));
+	}
+
+	/* Zeroing physical pages pointed by the reconstructed freelist */
+	if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
+}
+
+
+static PVRSRV_ERROR RGXReconstructFreeList(RGX_FREELIST *psFreeList)
+{
+	IMG_UINT32        ui32OriginalFLPages;
+	DLLIST_NODE       *psNode, *psNext;
+	RGXFWIF_FREELIST  *psFWFreeList;
+	PVRSRV_ERROR      eError;
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: Reconstructing freelist %p (ID=%u)", psFreeList, psFreeList->ui32FreelistID));
+
+	/* Do the FreeList Reconstruction */
+	ui32OriginalFLPages            = psFreeList->ui32CurrentFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+
+	/* Reconstructing Init FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Reconstructing Grow FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Ready pages are allocated but kept hidden until OOM occurs. */
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+	if (psFreeList->ui32CurrentFLPages != ui32OriginalFLPages)
+	{
+		PVR_ASSERT(psFreeList->ui32CurrentFLPages == ui32OriginalFLPages);
+		return PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED;
+	}
+
+	/* Update firmware freelist structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psFWFreeList->ui32MaxPages     = psFreeList->ui32MaxFLPages;
+	psFWFreeList->ui32GrowPages    = psFreeList->ui32GrowFLPages;
+	psFWFreeList->ui32CurrentPages = psFreeList->ui32CurrentFLPages;
+	psFWFreeList->ui32FreeListID   = psFreeList->ui32FreelistID;
+	psFWFreeList->bGrowPending     = IMG_FALSE;
+	psFWFreeList->ui32ReadyPages   = psFreeList->ui32ReadyFLPages;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psFWFreeList->ui32CurrentStackTop       = psFWFreeList->ui32CurrentPages - 1;
+	psFWFreeList->ui64CurrentDevVAddr       = (psFWFreeList->psFreeListDevVAddr.uiAddr +
+				((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+				~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+	psFWFreeList->ui32AllocatedPageCount    = 0;
+	psFWFreeList->ui32AllocatedMMUPageCount = 0;
+#else
+	psFWFreeList->bUpdatePending                   = IMG_FALSE;
+	psFWFreeList->ui32UpdateNewPages               = 0;
+	psFWFreeList->ui32UpdateNewReadyPages          = 0;
+	psFWFreeList->sFreeListLastGrowDevVAddr.uiAddr = 0;
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+#if !defined(PM_INTERACTIVE_MODE)
+	/* Reset freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER		sFLState;
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		eError = PMR_ReadBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+		/* Note: Freelist base address is packed shifted down. */
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, psFreeList->ui32CurrentFLPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+	}
+#endif
+
+	/* Check the Freelist checksum if required (as the list is fully populated) */
+	if (psFreeList->bCheckFreelist)
+	{
+		IMG_UINT64  ui64CheckSum;
+
+		_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+	}
+
+	return eError;
+}
+
+
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                              IMG_UINT32 ui32FreelistsCount,
+                                              const IMG_UINT32 *paui32Freelists)
+{
+	PVRSRV_ERROR      eError = PVRSRV_OK;
+	DLLIST_NODE       *psNode, *psNext;
+	IMG_UINT32        ui32Loop;
+	RGXFWIF_KCCB_CMD  sTACCBCmd;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE       *psNodeHWRTData, *psNextHWRTData;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+	RGXFWIF_HWRTDATA     *psHWRTData;
+#endif
+	IMG_UINT32        ui32PIDCount = 0;
+	IMG_UINT32        aui32PIDList[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+
+	PVR_ASSERT(psDevInfo != NULL);
+	PVR_ASSERT(ui32FreelistsCount <= RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT);
+	if (ui32FreelistsCount > RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT)
+	{
+		ui32FreelistsCount = RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT;
+	}
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: %u freelist(s) requested for reconstruction", ui32FreelistsCount));
+
+	/*
+	 *  Initialise the response command (in case we don't find a freelist ID).
+	 *  Also copy the list to the 'final' freelist array.
+	 */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
+	sTACCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
+
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] = paui32Freelists[ui32Loop] |
+				RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+	}
+
+	/*
+	 *  All freelists belonging to the same PID will be reconstructed.
+	 *  This simplifies tracking for AGP, since there is no longer a
+	 *  single global freelist per local freelist. The list of unique
+	 *  PIDs is calculated from the list of freelists in this first pass.
+	 */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList   = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+		{
+			IMG_UINT32 ui32PIDLoop;
+
+			if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+			{
+				for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+				{
+					if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+					{
+						break;
+					}
+				}
+
+				if (ui32PIDLoop == ui32PIDCount)
+				{
+					aui32PIDList[ui32PIDCount++] = psFreeList->ownerPid;
+				}
+			}
+		}
+	}
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList  = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		IMG_BOOL      bReconstruct = IMG_FALSE;
+		IMG_UINT32    ui32PIDLoop;
+
+		/*
+		 *  Check if this freelist needs to be reconstructed (is it in the list
+		 *  of PIDs which will have every single one of their freelists reconstructed)
+		 */
+		for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+		{
+			if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+			{
+				bReconstruct = IMG_TRUE;
+				break;
+			}
+		}
+
+		if (bReconstruct)
+		{
+			eError = RGXReconstructFreeList(psFreeList);
+			if (eError == PVRSRV_OK)
+			{
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+				/* Mark all HWRTData's of reconstructing local freelists as HWR (applies to TA/3D's not finished yet) */
+				dllist_foreach_node(&psFreeList->sNodeHWRTDataHead, psNodeHWRTData, psNextHWRTData)
+				{
+					psKMHWRTDataSet = IMG_CONTAINER_OF(psNodeHWRTData, RGX_KM_HW_RT_DATASET, sNodeHWRTData);
+					eError = DevmemAcquireCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc, (void **)&psHWRTData);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+									"Devmem AcquireCpuVirtAddr Failed during Reconstructing of FreeList, FwMemDesc(%p),psHWRTData(%p)",
+									psKMHWRTDataSet->psHWRTDataFwMemDesc,
+									psHWRTData));
+						continue;
+					}
+
+					psHWRTData->eState = RGXFWIF_RTDATA_STATE_HWR;
+					psHWRTData->ui32HWRTDataFlags &= ~HWRTDATA_HAS_LAST_TA;
+					RGXFwSharedMemCacheOpValue(psHWRTData->eState, FLUSH);
+					RGXFwSharedMemCacheOpValue(psHWRTData->ui32HWRTDataFlags, FLUSH);
+
+					DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+				}
+#endif
+
+				/* Update the response for this freelist if it was specifically requested for reconstruction. */
+				for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+				{
+					if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+					{
+						/* Reconstruction of this requested freelist was successful... */
+						sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &= ~RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+						break;
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Reconstructing of FreeList %p failed (%s)",
+						psFreeList,
+						PVRSRVGetErrorString(eError)));
+			}
+		}
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Check that all freelists were found and reconstructed... */
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		PVR_ASSERT((sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &
+		            RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG) == 0);
+	}
+
+	/* send feedback */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+/* Create a single HWRTData instance */
+static PVRSRV_ERROR RGXCreateHWRTData_aux(CONNECTION_DATA		*psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+                                          IMG_DEV_VIRTADDR		sPMDataDevVAddr, /* per-HWRTData */
+                                          RGX_FREELIST			*apsFreeLists[RGXFW_MAX_FREELISTS],
+                                          IMG_DEV_VIRTADDR		sTailPtrsDevVAddr,
+                                          IMG_DEV_VIRTADDR		sMacrotileArrayDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR		sRgnHeaderDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR		sRTCDevVAddr,
+                                          IMG_UINT16			ui16MaxRTs,
+                                          RGX_HWRTDATA_COMMON_COOKIE	*psHWRTDataCommonCookie,
+                                          RGX_KM_HW_RT_DATASET	**ppsKMHWRTDataSet) /* per-HWRTData */
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	/* KM cookie storing all the FW/HW data */
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+
+	/* local pointers for memory descriptors of FW allocations */
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc = NULL;
+
+	/* local pointer for CPU-mapped [FW]HWRTData */
+	RGXFWIF_HWRTDATA *psHWRTData = NULL;
+
+	/*
+	 * Fill in a local copy of RGXFWIF_HWRTDATA first to reduces reads and writes to
+	 * device memory. Using a byte array buffer avoids the limitation that some OSs
+	 * cannot align local variables to 64 bytes, usually needed by RGXFWIF_HWRTDATA
+	 */
+	IMG_BYTE aui8HWRTDataLocal[sizeof(RGXFWIF_HWRTDATA)] = {0};
+	RGXFWIF_HWRTDATA *psHWRTDataLocal = (RGXFWIF_HWRTDATA *)aui8HWRTDataLocal;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Prepare the HW RT DataSet struct */
+	psKMHWRTDataSet = OSAllocZMem(sizeof(*psKMHWRTDataSet));
+	if (psKMHWRTDataSet == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto AllocError;
+	}
+
+	*ppsKMHWRTDataSet = psKMHWRTDataSet;
+	psKMHWRTDataSet->psDeviceNode = psDeviceNode;
+
+	psKMHWRTDataSet->psHWRTDataCommonCookie = psHWRTDataCommonCookie;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+	 * This FW RT-Data is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency and write-combine will
+	 * suffice on the CPU side. (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHwRTData",
+	                          &psHWRTDataFwMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: DevmemAllocate for RGX_FWIF_HWRTDATA failed",
+				__func__));
+		goto FWRTDataAllocateError;
+	}
+
+	psKMHWRTDataSet->psHWRTDataFwMemDesc = psHWRTDataFwMemDesc;
+	eError = RGXSetFirmwareAddress( &psKMHWRTDataSet->sHWRTDataFwAddr,
+							psHWRTDataFwMemDesc,
+							0,
+							RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", FWRTDataFwAddrError);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataFwMemDesc,
+									  (void **)&psHWRTData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
+#endif
+
+	psHWRTDataLocal->sHWRTDataCommonFwAddr = psHWRTDataCommonCookie->sHWRTDataCommonFwAddr;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psPMMListDevVAddr = sPMDataDevVAddr;
+#else
+	psHWRTDataLocal->sPMRenderStateDevVAddr = sPMDataDevVAddr;
+#endif
+
+	psHWRTDataLocal->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
+	psHWRTDataLocal->sMacrotileArrayDevVAddr = sMacrotileArrayDevVAddr;
+	psHWRTDataLocal->sRgnHeaderDevVAddr		= sRgnHeaderDevVAddr;
+	psHWRTDataLocal->sRTCDevVAddr			= sRTCDevVAddr;
+
+	{
+		RGXFWIF_RTA_CTL *psRTACtl = &psHWRTDataLocal->sRTACtl;
+
+		psRTACtl->ui32RenderTargetIndex = 0;
+		psRTACtl->ui32ActiveRenderTargets = 0;
+		psRTACtl->sValidRenderTargets.ui32Addr = 0;
+		psRTACtl->sRTANumPartialRenders.ui32Addr = 0;
+		psRTACtl->ui32MaxRTs = (IMG_UINT32) ui16MaxRTs;
+
+		if (ui16MaxRTs > 1)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for shadow render target cache");
+			eError = DevmemFwAllocate(	psDevInfo,
+										ui16MaxRTs * sizeof(IMG_UINT32),
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwShadowRTCache",
+										&psRTArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTArryError;
+			}
+
+			psKMHWRTDataSet->psRTArrayFwMemDesc = psRTArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sValidRenderTargets,
+									psRTArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", FWAllocateRTArryFwAddrError);
+
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for tracking renders accumulation");
+			eError = DevmemFwAllocate(psDevInfo,
+			                          ui16MaxRTs * sizeof(IMG_UINT32),
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+					                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+					                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+					                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+					                  "FwRendersAccumulation",
+					                  &psRendersAccArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s) (renders accumulation)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTAccArryError;
+			}
+
+			psKMHWRTDataSet->psRendersAccArrayFwMemDesc = psRendersAccArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sRTANumPartialRenders,
+									psRendersAccArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", FWAllocRTAccArryFwAddrError);
+		}
+	}
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		psKMHWRTDataSet->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount++;
+		psHWRTDataLocal->apsFreeLists[ui32Loop].ui32Addr = psKMHWRTDataSet->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr;
+		/* Invalid initial snapshot value. The snapshot is always taken during the first
+		 * kick and hence this value gets replaced, so it's safe to set it to zero.
+		 */
+		psHWRTDataLocal->aui32FreeListHWRSnapshot[ui32Loop] = 0;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_add_to_tail(&apsFreeLists[RGXFW_LOCAL_FREELIST]->sNodeHWRTDataHead, &(psKMHWRTDataSet->sNodeHWRTData));
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	OSCachedMemCopy(psHWRTData, psHWRTDataLocal, sizeof(*psHWRTDataLocal));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTData 0x%08X", psKMHWRTDataSet->sHWRTDataFwAddr.ui32Addr);
+	DevmemPDumpLoadMem(psKMHWRTDataSet->psHWRTDataFwMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psHWRTData, FLUSH);
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	return PVRSRV_OK;
+
+FWAllocRTAccArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRendersAccArrayFwMemDesc);
+FWAllocateRTAccArryError:
+	RGXUnsetFirmwareAddress(psRTArrayFwMemDesc);
+FWAllocateRTArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRTArrayFwMemDesc);
+FWAllocateRTArryError:
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataCpuMapError:
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataAllocateError:
+	*ppsKMHWRTDataSet = NULL;
+	OSFreeMem(psKMHWRTDataSet);
+
+AllocError:
+	return eError;
+}
+
+static void RGXDestroyHWRTData_aux(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	if (psKMHWRTDataSet == NULL)
+	{
+		return;
+	}
+
+	psDevInfo = psKMHWRTDataSet->psDeviceNode->pvDevice;
+
+	if (psKMHWRTDataSet->psRTArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRTArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRTArrayFwMemDesc);
+	}
+
+	if (psKMHWRTDataSet->psRendersAccArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+	}
+
+	/* Decrease freelist refcount */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		PVR_ASSERT(psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount > 0);
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount--;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_remove_node(&psKMHWRTDataSet->sNodeHWRTData);
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Freeing the memory has to happen _after_ removing the HWRTData from the freelist
+	 * otherwise we risk traversing the freelist to find a pointer from a freed data structure */
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+
+	OSFreeMem(psKMHWRTDataSet);
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet(
+		CONNECTION_DATA			*psConnection,
+		PVRSRV_DEVICE_NODE		*psDeviceNode,
+		IMG_DEV_VIRTADDR	asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_DEV_VIRTADDR		asPMMListDevVAddr[RGXMKIF_NUM_RTDATAS],
+		RGX_FREELIST			*apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+		IMG_UINT32           ui32ScreenPixelMax,
+		IMG_UINT64           ui64MultiSampleCtl,
+		IMG_UINT64           ui64FlippedMultiSampleCtl,
+		IMG_UINT32           ui32TPCStride,
+		IMG_DEV_VIRTADDR		asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT32           ui32TPCSize,
+		IMG_UINT32           ui32TEScreen,
+		IMG_UINT32           ui32TEAA,
+		IMG_UINT32           ui32TEMTILE1,
+		IMG_UINT32           ui32TEMTILE2,
+		IMG_UINT32           ui32MTileStride,
+		IMG_UINT32                 ui32ISPMergeLowerX,
+		IMG_UINT32                 ui32ISPMergeLowerY,
+		IMG_UINT32                 ui32ISPMergeUpperX,
+		IMG_UINT32                 ui32ISPMergeUpperY,
+		IMG_UINT32                 ui32ISPMergeScaleX,
+		IMG_UINT32                 ui32ISPMergeScaleY,
+		IMG_DEV_VIRTADDR	asMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR	asRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR	asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT32			uiRgnHeaderSize,
+		IMG_UINT32			ui32ISPMtileSize,
+		IMG_UINT16			ui16MaxRTs,
+		RGX_KM_HW_RT_DATASET	*pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(asVHeapTableDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asPMMListDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(apsFreeLists);
+	PVR_UNREFERENCED_PARAMETER(ui32ScreenPixelMax);
+	PVR_UNREFERENCED_PARAMETER(ui64MultiSampleCtl);
+	PVR_UNREFERENCED_PARAMETER(ui64FlippedMultiSampleCtl);
+	PVR_UNREFERENCED_PARAMETER(ui32TPCStride);
+	PVR_UNREFERENCED_PARAMETER(asTailPtrsDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32TPCSize);
+	PVR_UNREFERENCED_PARAMETER(ui32TEScreen);
+	PVR_UNREFERENCED_PARAMETER(ui32TEAA);
+	PVR_UNREFERENCED_PARAMETER(ui32TEMTILE1);
+	PVR_UNREFERENCED_PARAMETER(ui32TEMTILE2);
+	PVR_UNREFERENCED_PARAMETER(ui32MTileStride);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeLowerX);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeLowerY);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeUpperX);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeUpperY);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeScaleX);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeScaleY);
+	PVR_UNREFERENCED_PARAMETER(asMacrotileArrayDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asRgnHeaderDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asRTCDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(uiRgnHeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMtileSize);
+	PVR_UNREFERENCED_PARAMETER(ui16MaxRTs);
+	PVR_UNREFERENCED_PARAMETER(pasKMHWRTDataSet);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet2(
+        CONNECTION_DATA         *psConnection,
+        PVRSRV_DEVICE_NODE      *psDeviceNode,
+        IMG_DEV_VIRTADDR        asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        DEVMEMINT_RESERVATION   *psPMMListsReservation,
+        RGX_FREELIST            *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+        IMG_UINT32              ui32ScreenPixelMax,
+        IMG_UINT64              ui64MultiSampleCtl,
+        IMG_UINT64              ui64FlippedMultiSampleCtl,
+        IMG_UINT32              ui32TPCStride,
+        IMG_DEV_VIRTADDR        asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        IMG_UINT32              ui32TPCSize,
+        IMG_UINT32              ui32TEScreen,
+        IMG_UINT32              ui32TEAA,
+        IMG_UINT32              ui32TEMTILE1,
+        IMG_UINT32              ui32TEMTILE2,
+        IMG_UINT32              ui32MTileStride,
+        IMG_UINT32              ui32ISPMergeLowerX,
+        IMG_UINT32              ui32ISPMergeLowerY,
+        IMG_UINT32              ui32ISPMergeUpperX,
+        IMG_UINT32              ui32ISPMergeUpperY,
+        IMG_UINT32              ui32ISPMergeScaleX,
+        IMG_UINT32              ui32ISPMergeScaleY,
+        IMG_DEV_VIRTADDR        asMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+        IMG_DEV_VIRTADDR        asRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+        IMG_DEV_VIRTADDR        asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        IMG_UINT32              uiRgnHeaderSize,
+        IMG_UINT32              ui32ISPMtileSize,
+        IMG_UINT16              ui16MaxRTs,
+        RGX_KM_HW_RT_DATASET    *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RTDataID;
+	IMG_UINT32 ui32GlobalFLMaxPages, ui32LocalFLMaxPages;
+	IMG_DEVMEM_SIZE_T ui64MListSize;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+	RGXFWIF_HWRTDATA_COMMON *psHWRTDataCommon;
+	DEVMEM_MEMDESC *psHWRTDataCommonFwMemDesc;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataCommonFwAddr;
+
+	PMR* psMListsPMR = NULL;
+	IMG_DEV_VIRTADDR sMListsDevVAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(0 < ui16MaxRTs && ui16MaxRTs <= RGX_MAX_TA_RENDER_TARGETS, "Number of TA Render targets outside the range (0, RGX_MAX_TA_RENDER_TARGETS) is unsupported");
+
+	/* Check if freelists have correct sizes */
+	eError = ValidateFreeListSizes(apsFreeLists,
+	                               &ui32LocalFLMaxPages,
+	                               &ui32GlobalFLMaxPages);
+	PVR_LOG_RETURN_IF_ERROR(eError, "Invalid freelist sizes");
+
+	ui64MListSize = RGXCalcMListSize(psDeviceNode,
+	    ui32LocalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE,
+	    ui32GlobalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMMListsReservation,
+	                                          ui64MListSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psMListsPMR,
+	                                          &sMListsDevVAddr);
+	PVR_LOG_RETURN_IF_ERROR(eError, "Failed to obtain or validate MLIST buffer");
+
+	/* Prepare KM cleanup object for HWRTDataCommon FW object */
+	psHWRTDataCommonCookie = OSAllocZMem(sizeof(*psHWRTDataCommonCookie));
+	if (psHWRTDataCommonCookie == NULL)
+	{
+	    eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	    goto err_HWRTDataCommonCookieAlloc;
+	}
+
+	/*
+	 * This FW common context is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA_COMMON),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHWRTDataCommon",
+	                          &psHWRTDataCommonFwMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+	    PVR_DPF((PVR_DBG_ERROR, "%s: DevmemAllocate for FwHWRTDataCommon failed", __func__));
+	    goto err_HWRTDataCommonAlloc;
+	}
+	eError = RGXSetFirmwareAddress(&sHWRTDataCommonFwAddr, psHWRTDataCommonFwMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", err_HWRTDataCommonFwAddr);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataCommonFwMemDesc, (void **)&psHWRTDataCommon);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", err_HWRTDataCommonVA);
+
+	psHWRTDataCommon->bTACachesNeedZeroing = IMG_FALSE;
+	psHWRTDataCommon->ui32ScreenPixelMax    = ui32ScreenPixelMax;
+	psHWRTDataCommon->ui64MultiSampleCtl = ui64MultiSampleCtl;
+	psHWRTDataCommon->ui64FlippedMultiSampleCtl = ui64FlippedMultiSampleCtl;
+	psHWRTDataCommon->ui32TPCStride         = ui32TPCStride;
+	psHWRTDataCommon->ui32TPCSize           = ui32TPCSize;
+	psHWRTDataCommon->ui32TEScreen          = ui32TEScreen;
+	psHWRTDataCommon->ui32TEAA              = ui32TEAA;
+	psHWRTDataCommon->ui32TEMTILE1          = ui32TEMTILE1;
+	psHWRTDataCommon->ui32TEMTILE2          = ui32TEMTILE2;
+	psHWRTDataCommon->ui32MTileStride       = ui32MTileStride;
+	psHWRTDataCommon->ui32ISPMergeLowerX    = ui32ISPMergeLowerX;
+	psHWRTDataCommon->ui32ISPMergeLowerY    = ui32ISPMergeLowerY;
+	psHWRTDataCommon->ui32ISPMergeUpperX    = ui32ISPMergeUpperX;
+	psHWRTDataCommon->ui32ISPMergeUpperY    = ui32ISPMergeUpperY;
+	psHWRTDataCommon->ui32ISPMergeScaleX    = ui32ISPMergeScaleX;
+	psHWRTDataCommon->ui32ISPMergeScaleY    = ui32ISPMergeScaleY;
+	psHWRTDataCommon->uiRgnHeaderSize       = uiRgnHeaderSize;
+	psHWRTDataCommon->ui32ISPMtileSize      = ui32ISPMtileSize;
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTDataCommon");
+	DevmemPDumpLoadMem(psHWRTDataCommonFwMemDesc, 0, sizeof(*psHWRTDataCommon), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psHWRTDataCommon, FLUSH);
+	DevmemReleaseCpuVirtAddr(psHWRTDataCommonFwMemDesc);
+
+	psHWRTDataCommonCookie->ui32RefCount = 0;
+	psHWRTDataCommonCookie->psHWRTDataCommonFwMemDesc = psHWRTDataCommonFwMemDesc;
+	psHWRTDataCommonCookie->sHWRTDataCommonFwAddr = sHWRTDataCommonFwAddr;
+
+	psHWRTDataCommonCookie->psPMMListsReservation = psPMMListsReservation;
+
+	/* Here we are creating a set of HWRTData(s)
+	   the number of elements in the set equals RGXMKIF_NUM_RTDATAS.
+	*/
+
+	for (ui32RTDataID = 0; ui32RTDataID < RGXMKIF_NUM_RTDATAS; ui32RTDataID++)
+	{
+		IMG_DEV_VIRTADDR sMListDevVAddr;
+		sMListDevVAddr.uiAddr = sMListsDevVAddr.uiAddr + ui32RTDataID * ui64MListSize;
+
+		eError = RGXCreateHWRTData_aux(
+		    psConnection,
+		    psDeviceNode,
+		    asVHeapTableDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+		    sMListDevVAddr,
+		    &apsFreeLists[(ui32RTDataID % RGXMKIF_NUM_GEOMDATAS) * RGXFW_MAX_FREELISTS],
+		    asTailPtrsDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+		    asMacrotileArrayDevVAddr[ui32RTDataID],
+		    asRgnHeaderDevVAddr[ui32RTDataID],
+		    asRTCDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+		    ui16MaxRTs,
+		    psHWRTDataCommonCookie,
+		    &pasKMHWRTDataSet[ui32RTDataID]);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Failed to create HWRTData [slot %u] (%s)",
+			        __func__,
+			        ui32RTDataID,
+			        PVRSRVGetErrorString(eError)));
+			goto err_HWRTDataAlloc;
+		}
+		psHWRTDataCommonCookie->ui32RefCount += 1;
+	}
+
+	return PVRSRV_OK;
+
+err_HWRTDataAlloc:
+	PVR_DPF((PVR_DBG_WARNING, "%s: err_HWRTDataAlloc %u",
+			 __func__, psHWRTDataCommonCookie->ui32RefCount));
+	if (pasKMHWRTDataSet)
+	{
+		for (ui32RTDataID = psHWRTDataCommonCookie->ui32RefCount; ui32RTDataID > 0; ui32RTDataID--)
+		{
+			if (pasKMHWRTDataSet[ui32RTDataID-1] != NULL)
+			{
+				RGXDestroyHWRTData_aux(pasKMHWRTDataSet[ui32RTDataID-1]);
+				pasKMHWRTDataSet[ui32RTDataID-1] = NULL;
+			}
+		}
+	}
+err_HWRTDataCommonVA:
+	RGXUnsetFirmwareAddress(psHWRTDataCommonFwMemDesc);
+err_HWRTDataCommonFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psHWRTDataCommonFwMemDesc);
+err_HWRTDataCommonAlloc:
+	OSFreeMem(psHWRTDataCommonCookie);
+err_HWRTDataCommonCookieAlloc:
+	UnrefAndReleaseCriticalBuffer(psPMMListsReservation);
+	return eError;
+}
+
+/* Destroy a single instance of HWRTData.
+   Additionally, destroy the HWRTDataCommon{Cookie} objects
+   when it is the last HWRTData within a corresponding set of HWRTDatas.
+*/
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_ERROR eError;
+	PRGXFWIF_HWRTDATA psHWRTData;
+	RGX_HWRTDATA_COMMON_COOKIE *psCommonCookie;
+
+	PVR_ASSERT(psKMHWRTDataSet);
+
+	psDevNode = psKMHWRTDataSet->psDeviceNode;
+
+	eError = RGXSetFirmwareAddress(&psHWRTData,
+	                               psKMHWRTDataSet->psHWRTDataFwMemDesc, 0,
+	                               RFW_FWADDR_NOREF_FLAG);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Cleanup HWRTData */
+	eError = RGXFWRequestHWRTDataCleanUp(psDevNode, psHWRTData);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDevNode,
+						   eError,
+						   RGXFWRequestHWRTDataCleanUp);
+
+	psCommonCookie = psKMHWRTDataSet->psHWRTDataCommonCookie;
+
+	RGXDestroyHWRTData_aux(psKMHWRTDataSet);
+
+	/* We've got past potential PVRSRV_ERROR_RETRY events, so we are sure
+	   that the HWRTDATA instance will be destroyed during this call.
+	   Consequently, we decrease the ref count for HWRTDataCommonCookie.
+
+	   NOTE: This ref count does not require locks or atomics.
+	   -------------------------------------------------------
+	     HWRTDatas bound into one pair are always destroyed sequentially,
+	     within a single loop on the Client side.
+	     The Common/Cookie objects always belong to only one pair of
+	     HWRTDatas, and ref count is used to ensure that the Common/Cookie
+	     objects will be destroyed after destruction of all HWRTDatas
+	     within a single pair.
+	*/
+	psCommonCookie->ui32RefCount--;
+
+	/* When ref count for HWRTDataCommonCookie hits ZERO
+	 * we have to destroy the HWRTDataCommon [FW object] and the cookie
+	 * [KM object] afterwards. */
+	if (psCommonCookie->ui32RefCount == 0)
+	{
+		RGXUnsetFirmwareAddress(psCommonCookie->psHWRTDataCommonFwMemDesc);
+
+		/* We don't need to flush the SLC before freeing.
+		 * FW RequestCleanUp has already done that for HWRTData, so we're fine
+		 * now. */
+
+		DevmemFwUnmapAndFree(psDevNode->pvDevice,
+		                     psCommonCookie->psHWRTDataCommonFwMemDesc);
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMMListsReservation);
+		OSFreeMem(psCommonCookie);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA			*psConnection,
+                               PVRSRV_DEVICE_NODE			*psDeviceNode,
+                               IMG_HANDLE				hMemCtxPrivData,
+                               IMG_UINT32				ui32MaxFLPages,
+                               IMG_UINT32				ui32InitFLPages,
+                               IMG_UINT32				ui32GrowFLPages,
+                               IMG_UINT32				ui32GrowParamThreshold,
+                               RGX_FREELIST				*psGlobalFreeList,
+                               IMG_BOOL					bCheckFreelist,
+                               DEVMEMINT_RESERVATION		*psFreeListReservation,
+                               RGX_FREELIST				**ppsFreeList)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_FREELIST   *psFWFreeList;
+	RGXFWIF_FREELIST   sFWFreeList = {{0}};
+	DEVMEM_MEMDESC     *psFWFreelistMemDesc;
+	RGX_FREELIST       *psFreeList;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR   sFreeListDevVAddr;
+	PMR*               psFreeListPMR = NULL;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psFreeListReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Failed to acquire reservation for freelist buffer",
+		        __func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	eError = DevmemIntGetReservationData(psFreeListReservation, &psFreeListPMR, &sFreeListDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from DevmemIntGetReservationData: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+
+		goto ErrorAllocHost;
+	}
+
+	/* Check if client properly allocated PMMETA_PROTECT */
+	if ((PMR_Flags(psFreeListPMR) & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must have PMMETA_PROTECT set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psFreeListPMR)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must not have OS_LINUX_MOVABLE set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+#endif
+
+	if (PMR_IsSparse(psFreeListPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Free list PMR cannot be sparse!",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto ErrorAllocHost;
+	}
+
+	/* Ref the PMR to prevent resource being destroyed before use */
+	eError = PMRRefPMR(psFreeListPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorAllocHost);
+
+	if (OSGetPageShift() > RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+	{
+		IMG_UINT32 ui32Size, ui32NewInitFLPages, ui32NewMaxFLPages, ui32NewGrowFLPages;
+
+		/* Round up number of FL pages to the next multiple of the OS page size */
+
+		ui32Size = ui32InitFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewInitFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32GrowFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewGrowFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32MaxFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewMaxFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Increased number of PB pages: Init %u -> %u, Grow %u -> %u, Max %u -> %u",
+		        __func__, ui32InitFLPages, ui32NewInitFLPages, ui32GrowFLPages, ui32NewGrowFLPages, ui32MaxFLPages, ui32NewMaxFLPages));
+
+		ui32InitFLPages = ui32NewInitFLPages;
+		ui32GrowFLPages = ui32NewGrowFLPages;
+		ui32MaxFLPages = ui32NewMaxFLPages;
+	}
+
+	/* Allocate kernel freelist struct */
+	psFreeList = OSAllocZMem(sizeof(*psFreeList));
+	if (psFreeList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate host data structure",
+		        __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRefPMR;
+	}
+
+	/*
+	 * This FW FreeList context is only mapped into kernel for initialisation
+	 * and reconstruction (at other times it is not mapped and only used by the
+	 * FW).
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(*psFWFreeList),
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+	                          PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwFreeList",
+	                          &psFWFreelistMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: DevmemAllocate for RGXFWIF_FREELIST failed",
+		        __func__));
+		goto FWFreeListAlloc;
+	}
+
+	/* Initialise host data structures */
+	psFreeList->psDevInfo = psDevInfo;
+	psFreeList->psConnection = psConnection;
+
+	psFreeList->psFreeListPMR = psFreeListPMR;
+	psFreeList->psFreeListReservation = psFreeListReservation;
+
+	psFreeList->uiFreeListPMROffset = 0U;
+
+	psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
+	eError = RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	/* psFreeList->ui32FreelistID set below with lock... */
+	psFreeList->ui32FreelistGlobalID = (psGlobalFreeList ? psGlobalFreeList->ui32FreelistID : 0);
+	psFreeList->ui32MaxFLPages = ui32MaxFLPages;
+	psFreeList->ui32InitFLPages = ui32InitFLPages;
+	psFreeList->ui32GrowFLPages = ui32GrowFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+	psFreeList->ui32ReadyFLPages = 0;
+	psFreeList->ui32GrowThreshold = ui32GrowParamThreshold;
+	psFreeList->ui64FreelistChecksum = 0;
+	psFreeList->ui32RefCount = 0;
+	psFreeList->bCheckFreelist = bCheckFreelist;
+	dllist_init(&psFreeList->sMemoryBlockHead);
+	dllist_init(&psFreeList->sMemoryBlockInitHead);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_init(&psFreeList->sNodeHWRTDataHead);
+#endif
+	psFreeList->ownerPid = OSGetCurrentClientProcessIDKM();
+
+
+	/* Add to list of freelists */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
+	dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+
+	/* Initialise FW data structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
+
+	{
+		const IMG_UINT32 ui32ReadyPages = _CalculateFreelistReadyPages(psFreeList, ui32InitFLPages);
+
+		sFWFreeList.ui32MaxPages = ui32MaxFLPages;
+		sFWFreeList.ui32CurrentPages = ui32InitFLPages - ui32ReadyPages;
+		sFWFreeList.ui32GrowPages = ui32GrowFLPages;
+		sFWFreeList.psFreeListDevVAddr = sFreeListDevVAddr;
+#if defined(PM_INTERACTIVE_MODE)
+		sFWFreeList.ui32CurrentStackTop = sFWFreeList.ui32CurrentPages - 1;
+		sFWFreeList.ui64CurrentDevVAddr = (sFreeListDevVAddr.uiAddr +
+		        ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+		                ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+#endif
+		sFWFreeList.ui32FreeListID = psFreeList->ui32FreelistID;
+		sFWFreeList.bGrowPending = IMG_FALSE;
+		sFWFreeList.ui32ReadyPages = ui32ReadyPages;
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+		/* Get the FW Memory Context address... */
+		eError = RGXSetFirmwareAddress(&sFWFreeList.psFWMemContext,
+		                               RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData),
+		                               0, RFW_FWADDR_NOREF_FLAG);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: RGXSetFirmwareAddress for RGXFWIF_FWMEMCONTEXT failed",
+			        __func__));
+			DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+			goto FWFreeListCpuMap;
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(hMemCtxPrivData);
+#endif
+	}
+
+	OSCachedMemCopy(psFWFreeList, &sFWFreeList, sizeof(sFWFreeList));
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, "
+	        "Max FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Init FL base address 0x%016" IMG_UINT64_FMTSPECx,
+	        psFreeList,
+	        ui32MaxFLPages,
+	        ui32InitFLPages,
+	        sFreeListDevVAddr.uiAddr,
+	        sFWFreeList.ui64CurrentDevVAddr));
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FW FreeList");
+	DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(PM_INTERACTIVE_MODE)
+	/*
+	 * Separate dump of the Freelist's number of Pages and stack pointer.
+	 * This allows to easily modify the PB size in the out2.txt files.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "FreeList TotalPages");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
+	                          sFWFreeList.ui32CurrentPages,
+	                          PDUMP_FLAGS_CONTINUOUS);
+	PDUMPCOMMENT(psDeviceNode, "FreeList StackPointer");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentStackTop),
+	                          sFWFreeList.ui32CurrentStackTop,
+	                          PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+	/* Add initial PB block */
+	eError = RGXGrowFreeList(psFreeList,
+	                         ui32InitFLPages,
+	                         &psFreeList->sMemoryBlockInitHead);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: failed to allocate initial memory block for free list 0x%016" IMG_UINT64_FMTSPECx " (%s)",
+		         __func__,
+		         sFreeListDevVAddr.uiAddr,
+		         PVRSRVGetErrorString(eError)));
+		goto FWFreeListCpuMap;
+	}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Update Stats */
+	PVRSRVStatsUpdateFreelistStats(psDeviceNode,
+	                               1, /* Add 1 to the appropriate counter (Requests by App)*/
+	                               0,
+	                               psFreeList->ui32InitFLPages,
+	                               psFreeList->ui32NumHighPages,
+	                               psFreeList->ownerPid);
+
+#endif
+
+	/* return values */
+	*ppsFreeList = psFreeList;
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+
+FWFreeListCpuMap:
+	/* Remove freelists from list */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+	RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
+
+ErrorSetFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psFWFreelistMemDesc);
+
+FWFreeListAlloc:
+	OSFreeMem(psFreeList);
+
+ErrorRefPMR:
+	(void) PMRUnrefPMR(psFreeListPMR);
+
+ErrorAllocHost:
+	DevmemIntReservationRelease(psFreeListReservation);
+
+ErrorReservationAcquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyFreeList
+*/
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RefCount;
+
+	PVR_ASSERT(psFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	ui32RefCount = psFreeList->ui32RefCount;
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	if (ui32RefCount != 0)
+	{
+		/* Freelist still busy */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Freelist is not in use => start firmware cleanup */
+	eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
+										 psFreeList->sFreeListFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psFreeList->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestFreeListCleanUp);
+
+	/* Device becomes unrecoverable if the firmware took too long to
+	 * handle the cleanup request, or if SLC-flushes didn't go through
+	 * (due to some GPU lockup) */
+
+	/* Remove FreeList from linked list before we destroy it... */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* Confirm all HWRTData nodes are freed before releasing freelist */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sNodeHWRTDataHead));
+#endif
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+#if defined(PM_INTERACTIVE_MODE)
+	if (psFreeList->bCheckFreelist)
+	{
+		RGXFWIF_FREELIST  *psFWFreeList;
+		IMG_UINT32        ui32CurrentStackTop;
+		IMG_UINT64        ui64CheckSum;
+
+		/* Get the current stack pointer for this free list */
+		DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+		RGXFwSharedMemCacheOpValue(psFWFreeList->ui32CurrentStackTop, INVALIDATE);
+		ui32CurrentStackTop = psFWFreeList->ui32CurrentStackTop;
+		DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+		if (ui32CurrentStackTop > psFreeList->ui32MaxFLPages)
+		{
+			PVR_LOG(("%s: FW freelist corrupted (%d)",
+					 __func__,
+					 ui32CurrentStackTop));
+		}
+		else if (ui32CurrentStackTop == psFreeList->ui32CurrentFLPages-1)
+		{
+			/* Do consistency tests (as the list is fully populated) */
+			_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+		}
+		else
+		{
+			/* Check for duplicate pages, but don't check the checksum as the list is not fully populated */
+			_CheckFreelist(psFreeList, ui32CurrentStackTop+1, 0, &ui64CheckSum);
+		}
+	}
+#endif
+
+	/* Destroy FW structures */
+	RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
+	DevmemFwUnmapAndFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);
+
+	/* Remove grow shrink blocks */
+	while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
+	{
+		eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* Remove initial PB block */
+	eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* consistency checks */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
+	PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
+
+	/* Remove reference from the PMR and reservation resources */
+	eError = PMRUnrefPMR(psFreeList->psFreeListPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	DevmemIntReservationRelease(psFreeList->psFreeListReservation);
+
+	/* free Freelist */
+	OSFreeMem(psFreeList);
+
+	return eError;
+}
+
+
+/*
+	RGXCreateZSBuffer
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_PRBUFFER			*psFWZSBuffer;
+	RGXFWIF_PRBUFFER			sFWZSBuffer = {0};
+	RGX_ZSBUFFER_DATA			*psZSBuffer;
+	DEVMEM_MEMDESC				*psFWZSBufferMemDesc;
+	IMG_BOOL					bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiMapFlags) ? IMG_TRUE : IMG_FALSE;
+
+	/* Allocate host data structure */
+	psZSBuffer = OSAllocZMem(sizeof(*psZSBuffer));
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate cleanup data structure for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocCleanup;
+	}
+
+	/* Populate Host data */
+	psZSBuffer->psDevInfo = psDevInfo;
+	psZSBuffer->psReservation = psReservation;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psZSBuffer->psReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to acquire reservation for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	psZSBuffer->psPMR = psPMR;
+	/* Obtain reference to PMR */
+	eError = PMRRefPMR(psZSBuffer->psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorRefPMR);
+
+	psZSBuffer->ui32RefCount = 0;
+	psZSBuffer->bIsBacked = IMG_FALSE;
+	psZSBuffer->bOnDemand = bOnDemand;
+
+	/* Allocate firmware memory for ZS-Buffer. */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware ZS-Buffer data structure");
+	eError = DevmemFwAllocate(psDevInfo,
+							sizeof(*psFWZSBuffer),
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+							PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+							"FwZSBuffer",
+							&psFWZSBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAllocFWZSBuffer;
+	}
+	psZSBuffer->psFWZSBufferMemDesc = psFWZSBufferMemDesc;
+
+	/* Temporarily map the firmware render context to the kernel. */
+	eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
+	                                  (void **)&psFWZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAcquireFWZSBuffer;
+	}
+
+	/* Get firmware address of ZS-Buffer. */
+	eError = RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	if (bOnDemand)
+	{
+		/* psZSBuffer->ui32ZSBufferID set below with lock... */
+		OSLockAcquire(psDevInfo->hLockZSBuffer);
+		psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
+		dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
+		OSLockRelease(psDevInfo->hLockZSBuffer);
+	}
+
+	/* Populate FW ZS-Buffer data structure */
+	sFWZSBuffer.bOnDemand = bOnDemand;
+	sFWZSBuffer.eState = (bOnDemand) ? RGXFWIF_PRBUFFER_UNBACKED : RGXFWIF_PRBUFFER_BACKED;
+	sFWZSBuffer.ui32BufferID = psZSBuffer->ui32ZSBufferID;
+
+	OSCachedMemCopy(psFWZSBuffer, &sFWZSBuffer, sizeof(sFWZSBuffer));
+	/* Dump the ZS-Buffer and the memory content */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump firmware ZS-Buffer");
+	DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWZSBuffer, FLUSH);
+
+	/* Release address acquired above. */
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+
+
+	/* define return value */
+	*ppsZSBuffer = psZSBuffer;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
+							psZSBuffer,
+							(bOnDemand) ? "On-Demand": "Up-front"));
+
+	psZSBuffer->owner=OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+
+	/* error handling */
+
+ErrorSetFwAddr:
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+ErrorAcquireFWZSBuffer:
+	DevmemFwUnmapAndFree(psDevInfo, psFWZSBufferMemDesc);
+
+ErrorAllocFWZSBuffer:
+	(void) PMRUnrefPMR(psZSBuffer->psPMR);
+ErrorRefPMR:
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+ErrorReservationAcquire:
+	OSFreeMem(psZSBuffer);
+
+ErrorAllocCleanup:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyZSBuffer
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError, eError2;
+
+	PVR_ASSERT(psZSBuffer);
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	/* Request ZS Buffer cleanup */
+	eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
+										psZSBuffer->sZSBufferFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psZSBuffer->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestZSBufferCleanUp);
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->ui32RefCount != 0)
+	{
+		/* ZS-Buffer is still referenced */
+		OSLockRelease(hLockZSBuffer);
+		PVR_DPF((PVR_DBG_WARNING, "Trying to destroy a ZS-Buffer [%p] that's still in use.", psZSBuffer));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Free the firmware render context. */
+	RGXUnsetFirmwareAddress(psZSBuffer->psFWZSBufferMemDesc);
+	DevmemFwUnmapAndFree(psZSBuffer->psDevInfo, psZSBuffer->psFWZSBufferMemDesc);
+
+	/* Remove Deferred Allocation from list */
+	if (psZSBuffer->bOnDemand)
+	{
+		PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
+		dllist_remove_node(&psZSBuffer->sNode);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] destroyed", psZSBuffer));
+
+	/* Release reference to reservation object and the PMR */
+	eError2 = PMRUnrefPMR(psZSBuffer->psPMR);
+	PVR_LOG_IF_ERROR(eError2, "PMRUnrefPMR");
+
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+
+	/* Free ZS-Buffer host data structure */
+	OSFreeMem(psZSBuffer);
+
+	OSLockRelease(hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (!psZSBuffer->bOnDemand)
+	{
+		/* Only deferred allocations can be populated */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bIsBacked == IMG_FALSE)
+	{
+		IMG_HANDLE hDevmemHeap;
+
+		/* Get Heap */
+		eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
+		if (unlikely(hDevmemHeap == (IMG_HANDLE)NULL))
+		{
+			OSLockRelease(hLockZSBuffer);
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+
+		eError = DevmemIntMapPMR(psZSBuffer->psReservation, psZSBuffer->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable populate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+
+		}
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
+		                          psZSBuffer,
+		                          psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_TRUE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+					RGX_POPULATION **ppsPopulation)
+{
+	RGX_POPULATION *psPopulation;
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount++;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	psZSBuffer->ui32NumReqByApp++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psZSBuffer->psDevInfo->psDeviceNode,
+	                               1, 0, psZSBuffer->owner);
+#endif
+
+	/* Do the backing */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		goto OnErrorBacking;
+	}
+
+	/* Create the handle to the backing */
+	psPopulation = OSAllocMem(sizeof(*psPopulation));
+	if (psPopulation == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto OnErrorAlloc;
+	}
+
+	psPopulation->psZSBuffer = psZSBuffer;
+
+	/* return value */
+	*ppsPopulation = psPopulation;
+
+	return PVRSRV_OK;
+
+OnErrorAlloc:
+	RGXUnbackingZSBuffer(psZSBuffer);
+
+OnErrorBacking:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psZSBuffer->ui32RefCount);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bOnDemand && psZSBuffer->bIsBacked == IMG_TRUE)
+	{
+		eError = DevmemIntUnmapPMR(psZSBuffer->psReservation);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
+									psZSBuffer,
+									psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_FALSE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
+{
+	PVRSRV_ERROR eError;
+	POS_LOCK hLockZSBuffer;
+	if (!psPopulation)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	hLockZSBuffer = psPopulation->psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	OSLockAcquire(hLockZSBuffer);
+	psPopulation->psZSBuffer->ui32RefCount--;
+	OSLockRelease(hLockZSBuffer);
+
+	OSFreeMem(psPopulation);
+
+	return PVRSRV_OK;
+}
+
+static RGX_ZSBUFFER_DATA *FindAndRefZSBuffer(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32ZSBufferID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_ZSBUFFER_DATA *psZSBuffer = NULL;
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+
+	dllist_foreach_node(&psDevInfo->sZSBufferHead, psNode, psNext)
+	{
+		RGX_ZSBUFFER_DATA *psThisZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
+
+		if (psThisZSBuffer->ui32ZSBufferID == ui32ZSBufferID)
+		{
+			psZSBuffer = psThisZSBuffer;
+
+			psZSBuffer->ui32RefCount++;
+			break;
+		}
+	}
+
+
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+	return psZSBuffer;
+}
+
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID)
+{
+	IMG_BOOL bBackingDone = IMG_TRUE;
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Populate ZLS */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Populating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		bBackingDone = IMG_FALSE;
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psZSBuffer->ui32NumReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psDevInfo->psDeviceNode,
+	                               0, 1, psZSBuffer->owner);
+#endif
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+
+}
+
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32 ui32ZSBufferID)
+{
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Unpopulate ZLS */
+	eError = RGXUnbackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"UnPopulating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		PVR_ASSERT(IMG_FALSE);
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+}
+
+static
+PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_DEV_VIRTADDR sVDMCallStackAddr,
+                              IMG_UINT32 ui32CallStackDepth,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_TA_DATA *psTAData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_TACTX_STATE *psContextState;
+	IMG_UINT32 uiCoreIdx;
+	PVRSRV_ERROR eError;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TA context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_TACTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwTAContextState",
+							  &psTAData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacontextsuspendalloc;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
+			(void **)&psContextState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_suspendcpuvirtacquire;
+	}
+
+	for (uiCoreIdx = 0; uiCoreIdx < RGX_NUM_GEOM_CORES; uiCoreIdx++)
+	{
+		psContextState->asGeomCore[uiCoreIdx].uTAReg_VDM_CALL_STACK_POINTER_Init =
+			sVDMCallStackAddr.uiAddr + (uiCoreIdx * ui32CallStackDepth * sizeof(IMG_UINT64));
+	}
+	RGXFwSharedMemCacheOpPtr(psContextState->asGeomCore, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TA,
+	                                 RGXFWIF_DM_GEOM,
+	                                 NULL,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 psTAData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TA_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TA_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &psTAData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init TA fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump the TA context suspend state buffer");
+	DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TACTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	psTAData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_tacommoncontext:
+fail_suspendcpuvirtacquire:
+	DevmemFwUnmapAndFree(psDevInfo, psTAData->psContextStateMemDesc);
+fail_tacontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_3D_DATA *ps3DData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	uiNumISPStoreRegs;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware 3D context suspend state");
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	uiNumISPStoreRegs = RGXFWIF_IPP_RESUME_REG_COUNT; /* default 1 register for IPP_resume */
+
+	uiNumISPStoreRegs += (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU) *
+								RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_PER_SPU) *
+								RGXFWIF_PIPE_COUNT_PER_ISP);
+
+
+	if (uiNumISPStoreRegs > (RGXFWIF_ISP_PIPE_COUNT_MAX + RGXFWIF_IPP_RESUME_REG_COUNT))
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+#else /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+	/* Size of the CS buffer */
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			(uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]));
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui3DRegISPStateStoreSize,
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "Fw3DContextState",
+							  &ps3DData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_3D,
+	                                 RGXFWIF_DM_3D,
+	                                 NULL,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init 3D fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Dump the 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_3DCTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_3dcommoncontext:
+	DevmemFwUnmapAndFree(psDevInfo, ps3DData->psContextStateMemDesc);
+fail_3dcontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVRGXCreateRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+                                            PVRSRV_DEVICE_NODE			*psDeviceNode,
+                                            IMG_INT32					i32Priority,
+                                            IMG_DEV_VIRTADDR			sVDMCallStackAddr,
+                                            IMG_UINT32					ui32CallStackDepth,
+                                            IMG_UINT32					ui32FrameworkRegisterSize,
+                                            IMG_PBYTE					pabyFrameworkRegisters,
+                                            IMG_HANDLE					hMemCtxPrivData,
+                                            IMG_UINT32					ui32StaticRenderContextStateSize,
+                                            IMG_PBYTE					pStaticRenderContextState,
+                                            IMG_UINT32					ui32PackedCCBSizeU8888,
+                                            IMG_UINT32					ui32ContextFlags,
+                                            IMG_UINT64					ui64RobustnessAddress,
+                                            IMG_UINT32					ui32MaxTADeadlineMS,
+                                            IMG_UINT32					ui32Max3DDeadlineMS,
+                                            RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RENDER_CONTEXT	*psRenderContext;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	RGXFWIF_FWRENDERCONTEXT		*psFWRenderContext;
+
+	*ppsRenderContext = NULL;
+
+	if (ui32StaticRenderContextStateSize > RGXFWIF_STATIC_RENDERCONTEXT_SIZE)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRenderContext = OSAllocZMem(sizeof(*psRenderContext));
+	if (psRenderContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psRenderContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_lock;
+	}
+
+	psRenderContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Create the FW render context, this has the TA and 3D FW common
+		contexts embedded within it
+	*/
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_FWRENDERCONTEXT),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwRenderContext",
+							  &psRenderContext->psFWRenderContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwrendercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkRegisterSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psRenderContext->psFWFrameworkMemDesc,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psRenderContext->psFWFrameworkMemDesc,
+				pabyFrameworkRegisters,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+		sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DContext(psConnection,
+	                          psDeviceNode,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32Max3DDeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->s3DData,
+	                          U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcontext;
+	}
+
+	eError = _CreateTAContext(psConnection,
+	                          psDeviceNode,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
+	                          psFWMemContextMemDesc,
+	                          sVDMCallStackAddr,
+	                          ui32CallStackDepth,
+	                          i32Priority,
+	                          ui32MaxTADeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->sTAData,
+	                          U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tacontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+			(void **)&psFWRenderContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	/* Copy the static render context data */
+	OSDeviceMemCopy(&psFWRenderContext->sStaticRenderContextState, pStaticRenderContextState, ui32StaticRenderContextStateSize);
+#if defined(SUPPORT_TRP)
+	psFWRenderContext->eTRPGeomCoreAffinity = RGXFWIF_DM_MAX;
+#endif
+	DevmemPDumpLoadMem(psRenderContext->psFWRenderContextMemDesc, 0, sizeof(RGXFWIF_FWRENDERCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpPtr(psFWRenderContext, FLUSH);
+	DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psRenderContext->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-ta3d");
+	if (IS_ERR(psRenderContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(psRenderContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	}
+
+	*ppsRenderContext = psRenderContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _DestroyTAContext(&psRenderContext->sTAData,
+		                                         psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_tacontext:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _Destroy3DContext(&psRenderContext->s3DData,
+		                                         psRenderContext->psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_3dcontext:
+fail_frameworkcopy:
+	if (psRenderContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+fail_fwrendercontext:
+	OSLockDestroy(psRenderContext->hLock);
+fail_lock:
+	OSFreeMem(psRenderContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+ * PVRSRVRGXDestroyRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psRenderContext->psDeviceNode->pvDevice;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_remove_node(&(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* Check psBufferSyncContext has not been destroyed already (by a previous
+	 * call to this function which then later returned PVRSRV_ERROR_RETRY)
+	 */
+	if (psRenderContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psRenderContext->psBufferSyncContext);
+		psRenderContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+	/* Cleanup the TA if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
+	{
+		eError = _DestroyTAContext(&psRenderContext->sTAData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+	/* Cleanup the 3D if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
+	{
+		eError = _Destroy3DContext(&psRenderContext->s3DData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+	{
+		RGXFWIF_FWRENDERCONTEXT	*psFWRenderContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+				(void **)&psFWRenderContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware render context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+		RGXFwSharedMemCacheOpValue(psFWRenderContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+
+		ui32WorkEstCCBSubmitted = psFWRenderContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRenderContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRenderContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			eError = PVRSRV_ERROR_RETRY;
+			goto e0;
+		}
+	}
+#endif
+
+	/*
+		Only if both TA and 3D contexts have been cleaned up can we
+		free the shared resources
+	*/
+	if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
+	{
+		if (psRenderContext->psFWFrameworkMemDesc)
+		{
+			/* Free the framework buffer */
+			DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+		}
+
+		/* Free the firmware render context */
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			WorkEstDeInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+		}
+#endif
+
+		OSLockDestroy(psRenderContext->hLock);
+
+		OSFreeMem(psRenderContext);
+	}
+
+	return PVRSRV_OK;
+
+e0:
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	return eError;
+}
+
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+static void DumpUfoList(IMG_UINT32 ui32ClientTAFenceCount,
+                        IMG_UINT32 ui32ClientTAUpdateCount,
+                        IMG_UINT32 ui32Client3DFenceCount,
+                        IMG_UINT32 ui32Client3DUpdateCount,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAFenceUFOAddress,
+                        IMG_UINT32 *paui32ClientTAFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAUpdateUFOAddress,
+                        IMG_UINT32 *paui32ClientTAUpdateValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DFenceUFOAddress,
+                        IMG_UINT32 *paui32Client3DFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DUpdateUFOAddress,
+                        IMG_UINT32 *paui32Client3DUpdateValue)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: ~~~ After populating sync prims ~~~",
+			 __func__));
+
+	/* Dump Fence syncs, Update syncs and PR Update syncs */
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA fence syncs:",
+	        __func__, ui32ClientTAFenceCount));
+	for (i = 0; i < ui32ClientTAFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr,
+			        *paui32ClientTAFenceValue,
+			        *paui32ClientTAFenceValue));
+			paui32ClientTAFenceValue++;
+		}
+		pauiClientTAFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA update syncs:",
+			 __func__, ui32ClientTAUpdateCount));
+	for (i = 0; i < ui32ClientTAUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr,
+			        *paui32ClientTAUpdateValue,
+			        *paui32ClientTAUpdateValue));
+			paui32ClientTAUpdateValue++;
+		}
+		pauiClientTAUpdateUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D fence syncs:",
+			 __func__, ui32Client3DFenceCount));
+	for (i = 0; i < ui32Client3DFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr,
+			        *paui32Client3DFenceValue,
+			        *paui32Client3DFenceValue));
+			paui32Client3DFenceValue++;
+		}
+		pauiClient3DFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D update syncs:",
+			 __func__, ui32Client3DUpdateCount));
+	for (i = 0; i < ui32Client3DUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr,
+			        *paui32Client3DUpdateValue,
+			        *paui32Client3DUpdateValue));
+			paui32Client3DUpdateValue++;
+		}
+		pauiClient3DUpdateUFOAddress++;
+	}
+}
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+/*
+ * PVRSRVRGXKickTA3DKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRFenceSyncPrimBlock,
+								 IMG_UINT32					ui32PRFenceSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckTAFence,
+								 PVRSRV_TIMELINE			iUpdateTATimeline,
+								 PVRSRV_FENCE				*piUpdateTAFence,
+								 IMG_CHAR					szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheck3DFence,
+								 PVRSRV_TIMELINE			iUpdate3DTimeline,
+								 PVRSRV_FENCE				*piUpdate3DFence,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus)
+{
+	/* per-context helper structures */
+	RGX_CCB_CMD_HELPER_DATA *pasTACmdHelperData = psRenderContext->asTACmdHelperData;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelperData = psRenderContext->as3DCmdHelperData;
+
+	IMG_UINT32                ui32TACmdCount=0;
+	IMG_UINT32                ui323DCmdCount=0;
+	__maybe_unused IMG_UINT32 ui32TACmdOffset=0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset=0;
+
+	RGXFWIF_UFO        sPRUFO;
+	IMG_UINT32         i;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	PVRSRV_ERROR       eError2 = PVRSRV_OK;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psRenderContext->s3DData.psServerCommonContext);
+	IMG_UINT32         ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_BOOL           bCCBStateOpen = IMG_FALSE;
+
+	IMG_UINT32         ui32ClientPRUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR  *pauiClientPRUpdateUFOAddress = NULL;
+	IMG_UINT32         *paui32ClientPRUpdateValue = NULL;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	PRGXFWIF_UFO_ADDR  *pauiClientTAFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClientTAUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  uiPRFenceUFOAddress;
+
+	IMG_UINT64         uiCheckTAFenceUID = 0;
+	IMG_UINT64         uiCheck3DFenceUID = 0;
+	IMG_UINT64         uiUpdateTAFenceUID = 0;
+	IMG_UINT64         uiUpdate3DFenceUID = 0;
+
+	IMG_BOOL bUseCombined3DAnd3DPR = bKickPR && bKick3D && !pui83DPRDMCmd;
+
+	RGXFWIF_KCCB_CMD_KICK_DATA sTACmdKickData;
+	RGXFWIF_KCCB_CMD_KICK_DATA s3DCmdKickData;
+	IMG_BOOL bUseSingleFWCommand = bKickTA && (bKickPR || bKick3D);
+
+	IMG_UINT32 ui32TACmdSizeTmp = 0, ui323DCmdSizeTmp = 0;
+
+	IMG_BOOL bTAFenceOnSyncCheckpointsOnly = IMG_FALSE;
+
+	PVRSRV_FENCE iUpdateTAFence = PVRSRV_NO_FENCE;
+	PVRSRV_FENCE iUpdate3DFence = PVRSRV_NO_FENCE;
+
+	IMG_BOOL b3DFenceOnSyncCheckpointsOnly = IMG_FALSE;
+	IMG_UINT32 ui32TAFenceTimelineUpdateValue = 0;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+
+	/*
+	 * Count of the number of TA and 3D update values (may differ from number of
+	 * TA and 3D updates later, as sync checkpoints do not need to specify a value)
+	 */
+	IMG_UINT32 ui32ClientPRUpdateValueCount = 0;
+	IMG_UINT32 ui32ClientTAUpdateValueCount = ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32Client3DUpdateValueCount = ui32Client3DUpdateCount;
+	PSYNC_CHECKPOINT *apsFenceTASyncCheckpoints = NULL;				/*!< TA fence checkpoints */
+	PSYNC_CHECKPOINT *apsFence3DSyncCheckpoints = NULL;				/*!< 3D fence checkpoints */
+	IMG_UINT32 ui32FenceTASyncCheckpointCount = 0;
+	IMG_UINT32 ui32Fence3DSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psUpdateTASyncCheckpoint = NULL;				/*!< TA update checkpoint (output) */
+	PSYNC_CHECKPOINT psUpdate3DSyncCheckpoint = NULL;				/*!< 3D update checkpoint (output) */
+	PVRSRV_CLIENT_SYNC_PRIM *psTAFenceTimelineUpdateSync = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	void *pvTAUpdateFenceFinaliseData = NULL;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+
+	RGX_SYNC_DATA sTASyncData = {NULL};		/*!< Contains internal update syncs for TA */
+	RGX_SYNC_DATA s3DSyncData = {NULL};		/*!< Contains internal update syncs for 3D */
+
+	IMG_BOOL bTestSLRAdd3DCheck = IMG_FALSE;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTA = {0};
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickData3D = {0};
+	IMG_UINT32 ui32TACommandOffset = 0;
+	IMG_UINT32 ui323DCommandOffset = 0;
+	IMG_UINT32 ui32TACmdHeaderOffset = 0;
+	IMG_UINT32 ui323DCmdHeaderOffset = 0;
+	IMG_UINT32 ui323DFullRenderCommandOffset = 0;
+	IMG_UINT32 ui32TACmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui323DCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT32 ui32TAFenceCount, ui323DFenceCount;
+	IMG_UINT32 ui32TAUpdateCount, ui323DUpdateCount;
+	IMG_UINT32 ui32PRUpdateCount;
+
+	IMG_PID uiCurrentProcess = OSGetCurrentClientProcessIDKM();
+
+	IMG_UINT32 ui32Client3DFenceCount = 0;
+
+	/* Ensure we haven't been given a null ptr to
+	 * TA fence values if we have been told we
+	 * have TA sync prim fences
+	 */
+	if (ui32ClientTAFenceCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAFenceValue != NULL,
+		                        "paui32ClientTAFenceValue NULL but "
+		                        "ui32ClientTAFenceCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * TA update values if we have been told we
+	 * have TA updates
+	 */
+	if (ui32ClientTAUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAUpdateValue != NULL,
+		                        "paui32ClientTAUpdateValue NULL but "
+		                        "ui32ClientTAUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * 3D update values if we have been told we
+	 * have 3D updates
+	 */
+	if (ui32Client3DUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32Client3DUpdateValue != NULL,
+		                        "paui32Client3DUpdateValue NULL but "
+		                        "ui32Client3DUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Write FW addresses into CMD SHARED BLOCKs */
+	{
+		CMDTA3D_SHARED *psGeomCmdShared = (CMDTA3D_SHARED *)pui8TADMCmd;
+		CMDTA3D_SHARED *ps3DCmdShared = (CMDTA3D_SHARED *)pui83DDMCmd;
+		CMDTA3D_SHARED *psPR3DCmdShared = (CMDTA3D_SHARED *)pui83DPRDMCmd;
+		RGXFWIF_DEV_VIRTADDR sNullFWAddr = {0};
+
+		if (psKMHWRTDataSet == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "KMHWRTDataSet is a null-pointer"));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Write FW address for TA CMD
+		*/
+		if (psGeomCmdShared != NULL)
+		{
+			if (ui32TACmdSize < sizeof(*psGeomCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid TACmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psGeomCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for 3D CMD
+		*/
+		if (ps3DCmdShared != NULL)
+		{
+			if (ui323DCmdSize < sizeof(*ps3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			ps3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for PR3D CMD
+		*/
+		if (psPR3DCmdShared != NULL)
+		{
+			if (ui323DPRCmdSize < sizeof(*psPR3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DPRCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psPR3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+	}
+
+	if (unlikely(iUpdateTATimeline >= 0 && !piUpdateTAFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	if (unlikely(iUpdate3DTimeline >= 0 && !piUpdate3DFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d, "
+			   "ui32Client3DFenceCount=%d, ui32Client3DUpdateCount=%d",
+			   __func__,
+			   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+			   ui32Client3DFenceCount, ui32Client3DUpdateCount));
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/* Double-check we have a PR kick if there are client fences */
+	if (unlikely(!bKickPR && ui32Client3DFenceCount != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence passed without a PR kick",
+		        __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+	szFenceName3D[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	ui32TAFenceCount = ui32ClientTAFenceCount;
+	ui323DFenceCount = ui32Client3DFenceCount;
+	ui32TAUpdateCount = ui32ClientTAUpdateCount;
+	ui323DUpdateCount = ui32Client3DUpdateCount;
+	ui32PRUpdateCount = ui32ClientPRUpdateCount;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32SyncPMRCount)
+	{
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling"
+		          " pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psRenderContext->psBufferSyncContext,
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%d)",
+				        __func__, eError));
+			}
+			OSLockRelease(psRenderContext->hLock);
+
+			return eError;
+		}
+
+#if !defined(SUPPORT_STRIP_RENDERING)
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+		{
+			ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+#else /* !defined(SUPPORT_STRIP_RENDERING) */
+		ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+
+		PVR_UNREFERENCED_PARAMETER(bTAFenceOnSyncCheckpointsOnly);
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+
+		if (psBufferUpdateSyncCheckpoint != NULL)
+		{
+			if (bKick3D)
+			{
+				ui323DUpdateCount++;
+			}
+			else
+			{
+				ui32PRUpdateCount++;
+			}
+		}
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2
+#error "Invalid value for UPDATE_FENCE_CHECKPOINT_COUNT. Must be either 1 or 2."
+#endif /* !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2 */
+
+	if (iCheckTAFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[TA]"
+		          " (iCheckFence=%d),"
+		          " psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheckTAFence,
+		          (void *) psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheckTAFence,
+		    &ui32FenceTASyncCheckpointCount,
+		    &apsFenceTASyncCheckpoints,
+		    &uiCheckTAFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_ta_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheckTAFence, ui32FenceTASyncCheckpointCount,
+		          (void *) apsFenceTASyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFenceTASyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "TA", apsFenceTASyncCheckpoints,
+			                      ui32FenceTASyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheck3DFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[3D]"
+		          " (iCheckFence=%d), "
+		          "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheck3DFence,
+		          (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheck3DFence,
+		    &ui32Fence3DSyncCheckpointCount,
+		    &apsFence3DSyncCheckpoints,
+		    &uiCheck3DFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_3d_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheck3DFence, ui32Fence3DSyncCheckpointCount,
+		          (void*)apsFence3DSyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFence3DSyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "3D", apsFence3DSyncCheckpoints,
+			                      ui32Fence3DSyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		IMG_UINT32 i;
+
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32FenceTASyncCheckpointCount;
+
+			for (i = 0; i < ui32Fence3DSyncCheckpointCount; i++)
+			{
+				if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) !=
+				    uiCurrentProcess)
+				{
+					ui32TAFenceCount++;
+				}
+			}
+		}
+
+		if (bKick3D)
+		{
+			ui323DFenceCount += ui32Fence3DSyncCheckpointCount;
+		}
+
+		ui32TAUpdateCount += iUpdateTATimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui323DUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui32PRUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE && !bKick3D ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+	}
+
+
+	if (bKickTA)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+		          __func__, ui32TAFenceCount, ui32TAUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+		    ui32TAFenceCount,
+		    ui32TAUpdateCount,
+		    ui32TACmdSize,
+		    &pPreAddr,
+		    (bKick3D ? NULL : &pPostAddr),
+		    (bKick3D ? NULL : &pRMWUFOAddr),
+		    pasTACmdHelperData
+		);
+	}
+
+	if (bKickPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+		    ui323DFenceCount,
+		    0,
+		    sizeof(sPRUFO),
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickPR && !bUseCombined3DAnd3DPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32PRUpdateCount=%d", __func__,
+		          ui32PRUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+		    0,
+		    0,
+		    ui32PRUpdateCount,
+		    /* if the client has not provided a 3DPR command, the regular 3D
+		     * command should be used instead */
+		    pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize,
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKick3D || bAbort)
+	{
+		if (!bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+		}
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+			bKickTA ? 0 : ui323DFenceCount,
+		    ui323DUpdateCount,
+		    ui323DCmdSize,
+			(bKickTA ? NULL : & pPreAddr),
+			&pPostAddr,
+			&pRMWUFOAddr,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickTA)
+	{
+		ui32TACmdSizeTmp = RGXCmdHelperGetCommandSize(1, pasTACmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext),
+		    ui32TACmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	if (ui323DCmdCount > 0)
+	{
+		ui323DCmdSizeTmp = RGXCmdHelperGetCommandSize(ui323DCmdCount, pas3DCmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext),
+		    ui323DCmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	/* need to reset the counter here */
+
+	ui323DCmdCount = 0;
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAFence, %d fences)...",
+			   __func__, ui32ClientTAFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
+										ui32ClientTAFenceCount,
+										apsClientTAFenceSyncPrimBlock,
+										paui32ClientTAFenceSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_fence;
+	}
+
+	if (ui32ClientTAFenceCount)
+	{
+		pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAUpdate, %d updates)...",
+			   __func__, ui32ClientTAUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
+										ui32ClientTAUpdateCount,
+										apsClientTAUpdateSyncPrimBlock,
+										paui32ClientTAUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_update;
+	}
+
+	if (ui32ClientTAUpdateCount)
+	{
+		pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAUpdateUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DFence, %d fences)...",
+			   __func__, ui32Client3DFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
+										ui32Client3DFenceCount,
+										NULL,
+										NULL);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_fence;
+	}
+
+	if (ui32Client3DFenceCount)
+	{
+		pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DUpdate, %d updates)...",
+			   __func__, ui32Client3DUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
+										ui32Client3DUpdateCount,
+										apsClient3DUpdateSyncPrimBlock,
+										paui32Client3DUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_update;
+	}
+
+	if (ui32Client3DUpdateCount || (iUpdate3DTimeline != PVRSRV_NO_TIMELINE && piUpdate3DFence && bKick3D))
+	{
+		pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DUpdateUFOAddress));
+
+	eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock, ui32PRFenceSyncOffset, &uiPRFenceUFOAddress);
+
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_pr_fence_address;
+	}
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	if (ui32SyncPMRCount)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+#if !defined(SUPPORT_STRIP_RENDERING)
+		/* Append buffer sync fences to TA fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0 && bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to TA Fence "
+					   "(&psRenderContext->sSyncAddrListTAFence=<%p>, "
+					   "pauiClientTAFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrListTAFence ,
+					   (void*)pauiClientTAFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClientTAFenceUFOAddress)
+			{
+				pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+			}
+			if (ui32ClientTAFenceCount == 0)
+			{
+				bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32ClientTAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+		/* Append buffer sync fences to 3D fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to 3D Fence "
+					   "(&psRenderContext->sSyncAddrList3DFence=<%p>, "
+					   "pauiClient3DFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrList3DFence,
+					   (void*)pauiClient3DFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+					ui32BufferFenceSyncCheckpointCount,
+					apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClient3DFenceUFOAddress)
+			{
+				pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+			}
+			if (ui32Client3DFenceCount == 0)
+			{
+				b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32Client3DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			/* If we have a 3D kick append update to the 3D updates else append to the PR update */
+			if (bKick3D)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   Append 1 buffer sync checkpoint<%p> to 3D Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClient3DUpdateUFOAddress=<%p>)...",
+						   __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClient3DUpdateUFOAddress));
+				/* Append buffer sync update to 3D updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClient3DUpdateUFOAddress)
+				{
+					pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32Client3DUpdateCount++;
+			}
+			else
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+				           "%s:   Append 1 buffer sync checkpoint<%p> to PR Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClientPRUpdateUFOAddress=<%p>)...",
+				           __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClientPRUpdateUFOAddress));
+				/* Attach update to the 3D (used for PR) Updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClientPRUpdateUFOAddress)
+				{
+					pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32ClientPRUpdateCount++;
+			}
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   (after buffer_sync) ui32ClientTAFenceCount=%d, "
+				   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+				   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+				   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+				   ui32ClientPRUpdateCount));
+
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Buffer sync not supported but got %u buffers",
+				 __func__, ui32SyncPMRCount));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_no_buffer_sync_invalid_params;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	/*
+	 * The hardware requires a PR to be submitted if there is a TA (otherwise
+	 * it can wedge if we run out of PB space with no PR to run)
+	 *
+	 * If we only have a TA, attach native checks to the TA and updates to the PR
+	 * If we have a TA and 3D, attach checks to TA, updates to 3D
+	 * If we only have a 3D, attach checks and updates to the 3D
+	 *
+	 * Note that 'updates' includes the cleanup syncs for 'check' fence FDs, in
+	 * addition to the update fence FD (if supplied)
+	 *
+	 * Currently, the client driver never kicks only the 3D, so we only support
+	 * that for the time being.
+	 */
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [TA] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheckTAFence, iUpdateTATimeline));
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [3D] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheck3DFence, iUpdate3DTimeline));
+
+		{
+			/* Create the output fence for TA (if required) */
+			if (iUpdateTATimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdateTAFence, iUpdateTATimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceNameTA,
+						iUpdateTATimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdateTAFence,
+						&uiUpdateTAFenceUID,
+						&pvTAUpdateFenceFinaliseData,
+						&psUpdateTASyncCheckpoint,
+						(void*)&psTAFenceTimelineUpdateSync,
+						&ui32TAFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[TA] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_ta_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdateTAFence,
+						   (void*)psTAFenceTimelineUpdateSync,
+						   ui32TAFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClientTAIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClientTAIntUpdateUFOAddress */
+					pauiClientTAIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdateTASyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[TA]->ui32Addr=0x%x",
+							   __func__, pauiClientTAIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the TA timeline (if required) */
+			if (psTAFenceTimelineUpdateSync)
+			{
+				sTASyncData.ui32ClientUpdateCount		 = ui32ClientTAUpdateCount;
+				sTASyncData.ui32ClientUpdateValueCount	 = ui32ClientTAUpdateValueCount;
+				sTASyncData.ui32ClientPRUpdateValueCount = (bKick3D) ? 0 : ui32ClientPRUpdateValueCount;
+				sTASyncData.paui32ClientUpdateValue		 = paui32ClientTAUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui32TAFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrListTAUpdate,
+											(bKick3D) ? NULL : &psRenderContext->sSyncAddrList3DUpdate,
+											psTAFenceTimelineUpdateSync,
+											&sTASyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_TA;
+				}
+
+				paui32ClientTAUpdateValue = sTASyncData.paui32ClientUpdateValue;
+				ui32ClientTAUpdateValueCount = sTASyncData.ui32ClientUpdateValueCount;
+				pauiClientTAUpdateUFOAddress = sTASyncData.pauiClientUpdateUFOAddress;
+				ui32ClientTAUpdateCount = sTASyncData.ui32ClientUpdateCount;
+			}
+
+			/* Create the output fence for 3D (if required) */
+			if (iUpdate3DTimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdate3DFence, iUpdate3DTimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceName3D,
+						iUpdate3DTimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdate3DFence,
+						&uiUpdate3DFenceUID,
+						&pv3DUpdateFenceFinaliseData,
+						&psUpdate3DSyncCheckpoint,
+						(void*)&ps3DFenceTimelineUpdateSync,
+						&ui323DFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[3D] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_3d_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdate3DFence,
+						   (void*)ps3DFenceTimelineUpdateSync,
+						   ui323DFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClient3DIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClient3DIntUpdateUFOAddress */
+					pauiClient3DIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdate3DSyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[3D]->ui32Addr=0x%x",
+							   __func__, pauiClient3DIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the 3D timeline (if required) */
+			if (ps3DFenceTimelineUpdateSync)
+			{
+				s3DSyncData.ui32ClientUpdateCount = ui32Client3DUpdateCount;
+				s3DSyncData.ui32ClientUpdateValueCount = ui32Client3DUpdateValueCount;
+				s3DSyncData.ui32ClientPRUpdateValueCount = ui32ClientPRUpdateValueCount;
+				s3DSyncData.paui32ClientUpdateValue = paui32Client3DUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui323DFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrList3DUpdate,
+											&psRenderContext->sSyncAddrList3DUpdate,	/*!< PR update: is this required? */
+											ps3DFenceTimelineUpdateSync,
+											&s3DSyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_3D;
+				}
+
+				paui32Client3DUpdateValue = s3DSyncData.paui32ClientUpdateValue;
+				ui32Client3DUpdateValueCount = s3DSyncData.ui32ClientUpdateValueCount;
+				pauiClient3DUpdateUFOAddress = s3DSyncData.pauiClientUpdateUFOAddress;
+				ui32Client3DUpdateCount = s3DSyncData.ui32ClientUpdateCount;
+
+				if (!bKick3D)
+				{
+					paui32ClientPRUpdateValue = s3DSyncData.paui32ClientPRUpdateValue;
+					ui32ClientPRUpdateValueCount = s3DSyncData.ui32ClientPRUpdateValueCount;
+					pauiClientPRUpdateUFOAddress = s3DSyncData.pauiClientPRUpdateUFOAddress;
+					ui32ClientPRUpdateCount = s3DSyncData.ui32ClientPRUpdateCount;
+				}
+			}
+
+			/*
+			 * The hardware requires a PR to be submitted if there is a TA OOM.
+			 * If we only have a TA, attach native checks and updates to the TA
+			 * and 3D updates to the PR.
+			 * If we have a TA and 3D, attach the native TA checks and updates
+			 * to the TA and similarly for the 3D.
+			 * Note that 'updates' includes the cleanup syncs for 'check' fence
+			 * FDs, in addition to the update fence FD (if supplied).
+			 * Currently, the client driver never kicks only the 3D, so we don't
+			 * support that for the time being.
+			 */
+
+			{
+				if (bKickTA)
+				{
+					/* Attach checks and updates to TA */
+
+					/* Checks (from input fence) */
+					if (ui32FenceTASyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to TA Fence (apsFenceSyncCheckpoints=<%p>)...",
+								   __func__,
+								   ui32FenceTASyncCheckpointCount,
+								   (void*)apsFenceTASyncCheckpoints));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+													  ui32FenceTASyncCheckpointCount,
+													  apsFenceTASyncCheckpoints);
+						if (!pauiClientTAFenceUFOAddress)
+						{
+							pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+								   __func__, ui32ClientTAFenceCount,
+								   ui32ClientTAFenceCount + ui32FenceTASyncCheckpointCount));
+						if (ui32ClientTAFenceCount == 0)
+						{
+							bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32ClientTAFenceCount += ui32FenceTASyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32ClientTAFenceCount now %d}",
+							   __func__, ui32ClientTAFenceCount));
+
+					if (psUpdateTASyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Update...",
+								   __func__, (void*)psUpdateTASyncCheckpoint,
+								   SyncCheckpointGetId(psUpdateTASyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAUpdate,
+													  1,
+													  &psUpdateTASyncCheckpoint);
+						if (!pauiClientTAUpdateUFOAddress)
+						{
+							pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+						}
+						ui32ClientTAUpdateCount++;
+					}
+
+					if (!bKick3D && psUpdate3DSyncCheckpoint)
+					{
+						/* Attach update to the 3D (used for PR) Updates */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D(PR) Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClientPRUpdateUFOAddress)
+						{
+							pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32ClientPRUpdateCount++;
+					}
+				}
+
+				if (bKick3D)
+				{
+					/* Attach checks and updates to the 3D */
+
+					/* Checks (from input fence) */
+					if (ui32Fence3DSyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to 3D Fence...",
+								   __func__, ui32Fence3DSyncCheckpointCount));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+													  ui32Fence3DSyncCheckpointCount,
+													  apsFence3DSyncCheckpoints);
+						if (!pauiClient3DFenceUFOAddress)
+						{
+							pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32Client3DFenceCount was %d, now %d}",
+								   __func__, ui32Client3DFenceCount,
+								   ui32Client3DFenceCount + ui32Fence3DSyncCheckpointCount));
+						if (ui32Client3DFenceCount == 0)
+						{
+							b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32Client3DFenceCount += ui32Fence3DSyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32Client3DFenceCount was %d}",
+							   __func__, ui32Client3DFenceCount));
+
+					if (psUpdate3DSyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClient3DUpdateUFOAddress)
+						{
+							pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32Client3DUpdateCount++;
+					}
+				}
+
+				/*
+				 * Relocate sync check points from the 3D fence that are
+				 * external to the current process, to the TA fence.
+				 * This avoids a sync lockup when dependent renders are
+				 * submitted out-of-order and a PR must be scheduled.
+				 */
+				if (bKickTA)
+				{
+					/* Search for external timeline dependencies */
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: Checking 3D fence for external sync points (%d)...",
+							   __func__, ui32Fence3DSyncCheckpointCount));
+
+					for (i=0; i<ui32Fence3DSyncCheckpointCount; i++)
+					{
+						/* Check to see if the checkpoint is on a TL owned by
+						 * another process.
+						 */
+						if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) != uiCurrentProcess)
+						{
+							/* 3D Sync point represents cross process
+							 * dependency, copy sync point to TA command fence. */
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Fence...",
+									   __func__, (void*)apsFence3DSyncCheckpoints[i],
+									   SyncCheckpointGetId(apsFence3DSyncCheckpoints[i])));
+
+							SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+														  1,
+														  &apsFence3DSyncCheckpoints[i]);
+
+							if (!pauiClientTAFenceUFOAddress)
+							{
+								pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+							}
+
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+									   __func__,
+									   ui32ClientTAFenceCount,
+									   ui32ClientTAFenceCount + 1));
+
+							if (ui32ClientTAFenceCount == 0)
+							{
+								bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+							}
+
+							ui32ClientTAFenceCount++;
+						}
+					}
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   (after pvr_sync) ui32ClientTAFenceCount=%d, "
+						   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+						   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+						   __func__,
+						   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+						   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+						   ui32ClientPRUpdateCount));
+			}
+		}
+
+		if (ui32ClientTAFenceCount)
+		{
+			PVR_ASSERT(pauiClientTAFenceUFOAddress);
+			if (!bTAFenceOnSyncCheckpointsOnly)
+			{
+				PVR_ASSERT(paui32ClientTAFenceValue);
+			}
+		}
+		if (ui32ClientTAUpdateCount)
+		{
+			PVR_ASSERT(pauiClientTAUpdateUFOAddress);
+			if (ui32ClientTAUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientTAUpdateValue);
+			}
+		}
+		if (ui32Client3DFenceCount)
+		{
+			PVR_ASSERT(pauiClient3DFenceUFOAddress);
+			PVR_ASSERT(b3DFenceOnSyncCheckpointsOnly);
+		}
+		if (ui32Client3DUpdateCount)
+		{
+			PVR_ASSERT(pauiClient3DUpdateUFOAddress);
+			if (ui32Client3DUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32Client3DUpdateValue);
+			}
+		}
+		if (ui32ClientPRUpdateCount)
+		{
+			PVR_ASSERT(pauiClientPRUpdateUFOAddress);
+			if (ui32ClientPRUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientPRUpdateValue);
+			}
+		}
+
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, pauiClientTAFenceUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAFenceCount,
+			   (void*)paui32ClientTAFenceValue));
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAUpdateCount=%d, pauiClientTAUpdateUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAUpdateCount,
+			   (void*)pauiClientTAUpdateUFOAddress));
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	/* Command size check */
+	if (ui32TAFenceCount != ui32ClientTAFenceCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAFenceCount, ui32ClientTAFenceCount));
+	}
+	if (ui32TAUpdateCount != ui32ClientTAUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAUpdateCount, ui32ClientTAUpdateCount));
+	}
+	if (!bTestSLRAdd3DCheck && (ui323DFenceCount != ui32Client3DFenceCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui323DFenceCount, ui32Client3DFenceCount));
+	}
+	if (ui323DUpdateCount != ui32Client3DUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui323DUpdateCount, ui32Client3DUpdateCount));
+	}
+	if (ui32PRUpdateCount != ui32ClientPRUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PR pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32PRUpdateCount, ui32ClientPRUpdateCount));
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode)) && (bKickTA || bKick3D || bAbort))
+	{
+		sWorkloadCharacteristics.sTA3D.ui32RenderTargetSize  = ui32RenderTargetSize;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfDrawCalls = ui32NumberOfDrawCalls;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfIndices   = ui32NumberOfIndices;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfMRTs      = ui32NumberOfMRTs;
+	}
+#endif
+
+	/* Init and acquire to TA command if required */
+	if (bKickTA)
+	{
+		RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sDataTA,
+					RGXFWIF_CCB_CMD_TYPE_GEOM,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTA);
+		}
+#endif
+
+		/* Init the TA command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
+		                                 ui32ClientTAFenceCount,
+		                                 pauiClientTAFenceUFOAddress,
+		                                 paui32ClientTAFenceValue,
+		                                 ui32ClientTAUpdateCount,
+		                                 pauiClientTAUpdateUFOAddress,
+		                                 paui32ClientTAUpdateValue,
+		                                 ui32TACmdSize,
+		                                 pui8TADMCmd,
+								         &pPreAddr,
+								         (bKick3D ? NULL : & pPostAddr),
+								         (bKick3D ? NULL : & pRMWUFOAddr),
+		                                 RGXFWIF_CCB_CMD_TYPE_GEOM,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickDataTA,
+#else
+		                                 NULL,
+#endif
+		                                 "TA",
+		                                 bCCBStateOpen,
+		                                 pasTACmdHelperData);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following is used to determine the offset of the command header containing
+			   the workload estimation data so that can be accessed when the KCCB is read */
+			ui32TACmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(pasTACmdHelperData);
+		}
+#endif
+
+		eError = RGXCmdHelperAcquireCmdCCB(CCB_CMD_HELPER_NUM_TA_COMMANDS, pasTACmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taacquirecmd;
+		}
+		else
+		{
+			ui32TACmdCount++;
+		}
+	}
+
+	/* Only kick the 3D if required */
+	if (bKickPR)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+
+		/*
+			The command helper doesn't know about the PR fence so create
+			the command with all the fences against it and later create
+			the PR command itself which _must_ come after the PR fence.
+		*/
+		sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
+		sPRUFO.ui32Value = ui32PRFenceValue;
+
+		/* Init the PR fence command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32Client3DFenceCount=%d",
+				   __func__, ui32Client3DFenceCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+		                                 pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 0,
+		                                 NULL,
+		                                 NULL,
+		                                 sizeof(sPRUFO),
+		                                 (IMG_UINT8*) &sPRUFO,
+		                                 NULL,
+		                                 NULL,
+		                                 NULL,
+		                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+		                                 NULL,
+		                                 "3D-PR-Fence",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+		/* Init the 3D PR command helper */
+		/*
+			Updates for Android (fence sync and Timeline sync prim) are provided in the PR-update
+			if no 3D is present. This is so the timeline update cannot happen out of order with any
+			other 3D already in flight for the same timeline (PR-updates are done in the 3D cCCB).
+			This out of order timeline sync prim update could happen if we attach it to the TA update.
+		*/
+		if (ui32ClientPRUpdateCount)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s: Line %d, ui32ClientPRUpdateCount=%d, "
+					   "pauiClientPRUpdateUFOAddress=0x%x, "
+					   "ui32ClientPRUpdateValueCount=%d, "
+					   "paui32ClientPRUpdateValue=0x%x",
+					   __func__, __LINE__, ui32ClientPRUpdateCount,
+					   pauiClientPRUpdateUFOAddress->ui32Addr,
+					   ui32ClientPRUpdateValueCount,
+					   (ui32ClientPRUpdateValueCount == 0) ? PVRSRV_SYNC_CHECKPOINT_SIGNALLED : *paui32ClientPRUpdateValue));
+		}
+
+		if (!bUseCombined3DAnd3DPR)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientPRUpdateCount=%d",
+					   __func__, ui32ClientPRUpdateCount));
+			RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+			                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+			                                 0,
+			                                 NULL,
+			                                 NULL,
+			                                 ui32ClientPRUpdateCount,
+			                                 pauiClientPRUpdateUFOAddress,
+			                                 paui32ClientPRUpdateValue,
+			                                 pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize, // If the client has not provided a 3DPR command, the regular 3D command should be used instead
+			                                 pui83DPRDMCmd ? pui83DPRDMCmd : pui83DDMCmd,
+			                                 NULL,
+			                                 NULL,
+			                                 NULL,
+			                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
+			                                 ui32ExtJobRef,
+			                                 ui32IntJobRef,
+			                                 ui32PDumpFlags,
+			                                 NULL,
+			                                 "3D-PR",
+			                                 bCCBStateOpen,
+			                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+		}
+	}
+
+	if (bKick3D || bAbort)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+		const RGXFWIF_CCB_CMD_TYPE e3DCmdType = bAbort ? RGXFWIF_CCB_CMD_TYPE_ABORT : RGXFWIF_CCB_CMD_TYPE_3D;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sData3D,
+					e3DCmdType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickData3D);
+		}
+#endif
+
+		/* Init the 3D command helper */
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 bKickTA ? 0 : ui32Client3DFenceCount,  /* For a kick with a TA, the 3D fences are added before the PR command instead */
+		                                 bKickTA ? NULL : pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 ui32Client3DUpdateCount,
+		                                 pauiClient3DUpdateUFOAddress,
+		                                 paui32Client3DUpdateValue,
+		                                 ui323DCmdSize,
+		                                 pui83DDMCmd,
+		                                 (bKickTA ? NULL : & pPreAddr),
+		                                 &pPostAddr,
+		                                 &pRMWUFOAddr,
+		                                 e3DCmdType,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickData3D,
+#else
+		                                 NULL,
+#endif
+		                                 "3D",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following are used to determine the offset of the command header containing the workload estimation
+			   data so that can be accessed when the KCCB is read */
+			ui323DCmdHeaderOffset =	RGXCmdHelperGetDMCommandHeaderOffset(&pas3DCmdHelperData[ui323DCmdCount - 1]);
+			ui323DFullRenderCommandOffset =	RGXCmdHelperGetCommandOffset(pas3DCmdHelperData, ui323DCmdCount - 1);
+		}
+#endif
+	}
+
+	/* Protect against array overflow in RGXCmdHelperAcquireCmdCCB() */
+	if (unlikely(ui323DCmdCount > CCB_CMD_HELPER_NUM_3D_COMMANDS))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+		goto fail_3dcmdinit;
+	}
+
+	if (ui323DCmdCount)
+	{
+		PVR_ASSERT(bKickPR || bKick3D);
+
+		/* Acquire space for all the 3D command(s) */
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount, pas3DCmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			/* If RGXCmdHelperAcquireCmdCCB fails we skip the scheduling
+			 * of a new TA command with the same Write offset in Kernel CCB.
+			 */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+			goto fail_3dacquirecmd;
+		}
+	}
+
+	if (ui32TACmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTACmdKickData.apsCleanupCtl,
+				&sTACmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_GEOM,
+				bKickTA,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taattachcleanupctls;
+		}
+	}
+
+	if (ui323DCmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DCmdKickData.apsCleanupCtl,
+				&s3DCmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_3D,
+				bKick3D,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_3dattachcleanupctls;
+		}
+	}
+
+	/*
+		We should acquire the space in the kernel CCB here as after this point
+		we release the commands which will take operations on server syncs
+		which can't be undone
+	*/
+
+	/*
+		Everything is ready to go now, release the commands
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui32TACmdCount)
+	{
+		ui32TACmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
+								  pasTACmdHelperData,
+								  "TA",
+								  FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui32TACmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+
+			/* This checks if the command would wrap around at the end of the CCB and therefore  would start at an
+			   offset of 0 rather than the current command offset */
+			if (ui32TACmdOffset < ui32TACmdOffsetWrapCheck)
+			{
+				ui32TACommandOffset = ui32TACmdOffset;
+			}
+			else
+			{
+				ui32TACommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  pas3DCmdHelperData,
+								  "3D",
+								  FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui323DCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+
+			if (ui323DCmdOffset < ui323DCmdOffsetWrapCheck)
+			{
+				ui323DCommandOffset = ui323DCmdOffset;
+			}
+			else
+			{
+				ui323DCommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui32TACmdCount)
+	{
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext);
+		CMDTA3D_SHARED *psGeomCmdShared = IMG_OFFSET_ADDR(pui8TADMCmd, 0);
+
+		sTACmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
+		sTACmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTACmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Add the Workload data into the KCCB kick */
+			sTACmdKickData.ui32WorkEstCmdHeaderOffset = ui32TACommandOffset + ui32TACmdHeaderOffset;
+		}
+#endif
+
+		if (psGeomCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_TA,
+					sTACmdKickData.psContext,
+					ui32TACmdOffset,
+					psGeomCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_GEOM,
+		                  iCheckTAFence,
+		                  iUpdateTAFence,
+		                  iUpdateTATimeline,
+		                  uiCheckTAFenceUID,
+		                  uiUpdateTAFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickDataTA));
+
+		if (!bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA CCB command. */
+			RGXFWIF_KCCB_CMD sTAKCCBCmd;
+			sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			sTAKCCBCmd.uCmdData.sCmdKickData = sTACmdKickData;
+
+			LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+			{
+				eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+						RGXFWIF_DM_GEOM,
+						&sTAKCCBCmd,
+						ui32PDumpFlags);
+				if (eError2 != PVRSRV_ERROR_RETRY)
+				{
+					break;
+				}
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+		}
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			/* Mark the error and bail out */
+			eError = eError2;
+			goto fail_tasubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_GEOM);
+	}
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd = { 0 };
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext);
+		CMDTA3D_SHARED *ps3DCmdShared = IMG_OFFSET_ADDR(pui83DDMCmd, 0);
+
+		s3DCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+		s3DCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		s3DCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced when the KCCB command reaches the FW */
+			s3DCmdKickData.ui32WorkEstCmdHeaderOffset = ui323DCommandOffset + ui323DCmdHeaderOffset + ui323DFullRenderCommandOffset;
+		}
+#endif
+
+		if (ps3DCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_3D,
+					s3DCmdKickData.psContext,
+					ui323DCmdOffset,
+					ps3DCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_3D,
+		                  iCheck3DFence,
+		                  iUpdate3DFence,
+		                  iUpdate3DTimeline,
+		                  uiCheck3DFenceUID,
+		                  uiUpdate3DFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickData3D));
+
+		if (bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA/3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.sTACmdKickData = sTACmdKickData;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.s3DCmdKickData = s3DCmdKickData;
+		}
+		else
+		{
+			/* Construct the kernel 3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			s3DKCCBCmd.uCmdData.sCmdKickData = s3DCmdKickData;
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+										 RGXFWIF_DM_3D,
+										 &s3DKCCBCmd,
+										 ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			if (eError == PVRSRV_OK)
+			{
+				eError = eError2;
+			}
+			goto fail_3dsubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_3D);
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (unlikely(eError != PVRSRV_OK ))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+				   __func__, eError));
+		goto fail_3dsubmitcmd;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateTASyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [TA] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateTASyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateTASyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateTASyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateTASyncCheckpoint);
+	}
+	if (psTAFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [TA] <%p> to %d",
+				   __func__, (void*)psTAFenceTimelineUpdateSync,
+				   ui32TAFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psTAFenceTimelineUpdateSync, ui32TAFenceTimelineUpdateValue);
+	}
+
+	if (psUpdate3DSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [3D] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdate3DSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdate3DSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdate3DSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdate3DSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [3D] <%p> to %d",
+				   __func__, (void*)ps3DFenceTimelineUpdateSync,
+				   ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling pvr_buffer_sync_kick_succeeded(psBufferSyncData=<%p>)...",
+				   __func__, (void*)psBufferSyncData));
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	if (piUpdateTAFence)
+	{
+		*piUpdateTAFence = iUpdateTAFence;
+	}
+	if (piUpdate3DFence)
+	{
+		*piUpdate3DFence = iUpdate3DFence;
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+
+	if (pvTAUpdateFenceFinaliseData && (iUpdateTAFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdateTAFence,
+									pvTAUpdateFenceFinaliseData,
+									psUpdateTASyncCheckpoint, szFenceNameTA);
+	}
+	if (pv3DUpdateFenceFinaliseData && (iUpdate3DFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdate3DFence,
+									pv3DUpdateFenceFinaliseData,
+									psUpdate3DSyncCheckpoint, szFenceName3D);
+	}
+
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_3dsubmitcmd:
+fail_tasubmitcmd:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_3dattachcleanupctls:
+fail_taattachcleanupctls:
+fail_3dacquirecmd:
+fail_3dcmdinit:
+fail_taacquirecmd:
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DUpdate);
+	/* Where a TA-only kick (ie no 3D) is submitted, the PR update will make use of the unused 3DUpdate list.
+	 * If this has happened, performing a rollback on pauiClientPRUpdateUFOAddress will simply repeat what
+	 * has already been done for the sSyncAddrList3DUpdate above and result in a double decrement of the
+	 * sync checkpoint's hEnqueuedCCBCount, so we need to check before rolling back the PRUpdate.
+	 */
+	if (pauiClientPRUpdateUFOAddress && (pauiClientPRUpdateUFOAddress != psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs))
+	{
+		SyncCheckpointRollbackFromUFO(psRenderContext->psDeviceNode, pauiClientPRUpdateUFOAddress->ui32Addr);
+	}
+
+fail_alloc_update_values_mem_3D:
+	if (iUpdate3DFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdate3DFence, pv3DUpdateFenceFinaliseData);
+	}
+fail_create_3d_fence:
+fail_alloc_update_values_mem_TA:
+	if (iUpdateTAFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateTAFence, pvTAUpdateFenceFinaliseData);
+	}
+fail_create_ta_fence:
+#if !defined(SUPPORT_BUFFER_SYNC)
+err_no_buffer_sync_invalid_params:
+#endif /* !defined(SUPPORT_BUFFER_SYNC) */
+err_pr_fence_address:
+err_populate_sync_addr_list_3d_update:
+err_populate_sync_addr_list_3d_fence:
+err_populate_sync_addr_list_ta_update:
+err_populate_sync_addr_list_ta_fence:
+err_not_enough_space:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		SyncAddrListDeRefCheckpoints(ui32BufferFenceSyncCheckpointCount,
+					     apsBufferFenceSyncCheckpoints);
+	}
+#endif
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+fail_resolve_input_3d_fence:
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+fail_resolve_input_ta_fence:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	PVR_ASSERT(eError != PVRSRV_OK);
+	OSLockRelease(psRenderContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD		sDisableZSStoreCmd = { 0 };
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	sDisableZSStoreCmd.eCmdType = RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableZStore = bDisableDepthStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableSStore = bDisableStencilStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.ui32ExtJobRefToDisableZSStore = ui32ExtJobRefToDisableZSStore;
+
+	if (psRenderContext->ui32CleanupStatus & (RC_CLEANUP_TA_COMPLETE | RC_CLEANUP_3D_COMPLETE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: submit disable depth and stencil store command to render context that has been cleaned up (%u)",
+				__func__, PVRSRV_ERROR_INVALID_CONTEXT));
+		return PVRSRV_ERROR_INVALID_CONTEXT;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_3D,
+									&sDisableZSStoreCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit disable depth and stencil store command (%u)",
+				__func__,
+				eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	if (psRenderContext->sTAData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_GEOM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the TA part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_tacontext;
+		}
+		psRenderContext->sTAData.i32Priority = i32Priority;
+	}
+
+	if (psRenderContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the 3D part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_3dcontext;
+		}
+		psRenderContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+fail_tacontext:
+	OSLockRelease(psRenderContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 RGX_CONTEXT_PROPERTY eContextProperty,
+                                                 IMG_UINT64 ui64Input,
+                                                 IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psRenderContext->hLock);
+			eError = FWCommonContextSetFlags(psRenderContext->sTAData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psRenderContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psRenderContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->sTAData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->s3DData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+		if (NULL != psCurrentServerRenderCtx->sTAData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext, RGX_KICK_TYPE_DM_TA) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TA;
+			}
+		}
+
+		if (NULL != psCurrentServerRenderCtx->s3DData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_3D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * RGXRenderContextStalledKM
+ */
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	RGXCheckForStalledClientContexts((PVRSRV_RGXDEV_INFO *) psRenderContext->psDeviceNode->pvDevice, IMG_TRUE);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxta3d.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.h
new file mode 100644
index 000000000000..ff8c33d90893
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.h
@@ -0,0 +1,540 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA and 3D Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX TA and 3D Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXTA3D_H
+#define RGXTA3D_H
+
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_shared.h"
+#include "rgx_fwif_resetframework.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+typedef struct _RGX_SERVER_RENDER_CONTEXT_ RGX_SERVER_RENDER_CONTEXT;
+typedef struct _RGX_FREELIST_ RGX_FREELIST;
+typedef struct _RGX_PMR_NODE_ RGX_PMR_NODE;
+
+/*****************************************************************************
+ * The Design of Data Storage System for Render Targets                      *
+ * ====================================================                      *
+ *   Relevant for                                                            *
+ *     understanding RGXCreateHWRTDataSet & RGXDestroyHWRTDataSet            *
+ *                                                                           *
+ *                                                                           *
+ *        +=========================================+                        *
+ *        |           RenderTargetDataSet           |                        *
+ *        +---------------|---------|---------------+                        *
+ *                        |         |                                        *
+ *                        V         V                                        *
+ *  +- - - - - - - - - - - - +   +- - - - - - - - - - - - +                  *
+ *  | KM_HW_RT_DATA_HANDLE_0 |   | KM_HW_RT_DATA_HANDLE_1 |                  *
+ *  +- - -|- - - - - - - - - +   +- - - - - - - - - | - - +                  *
+ *        |                                         |                        *
+ *        |                                         |           [UM]Client   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |               Bridge   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |           [KM]Server   *
+ *        |                                         |                        *
+ *        | KM-ptr                                  | KM-ptr                 *
+ *        V                                         V                        *
+ *  +====================+           +====================+                  *
+ *  |  KM_HW_RT_DATA_0   |           |   KM_HW_RT_DATA_1  |                  *
+ *  +-----|------------|-+           +-|------------|-----+                  *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            | KM-ptr        | KM-ptr     |                        *
+ *        |            V               V            |                        *
+ *        |      +==========================+       |                        *
+ *        |      | HW_RT_DATA_COMMON_COOKIE |       |                        *
+ *        |      +--------------------------+       |                        *
+ *        |                   |                     |                        *
+ *        |                   |                     |                        *
+ *  ------|-------------------|---------------------|----------------------- *
+ *        |                   |                     |         [FW]Firmware   *
+ *        |                   |                     |                        *
+ *        | FW-addr           |                     | FW-addr                *
+ *        V                   |                     V                        *
+ *  +===============+         |           +===============+                  *
+ *  | HW_RT_DATA_0  |         |           | HW_RT_DATA_1  |                  *
+ *  +------------|--+         |           +--|------------+                  *
+ *               |            |              |                               *
+ *               | FW-addr    | FW-addr      | FW-addr                       *
+ *               V            V              V                               *
+ *        +=========================================+                        *
+ *        |           HW_RT_DATA_COMMON             |                        *
+ *        +-----------------------------------------+                        *
+ *                                                                           *
+ *****************************************************************************/
+
+typedef struct _RGX_HWRTDATA_COMMON_COOKIE_
+{
+	DEVMEM_MEMDESC			*psHWRTDataCommonFwMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sHWRTDataCommonFwAddr;
+	DEVMEMINT_RESERVATION	*psPMMListsReservation;
+	IMG_UINT32				ui32RefCount;
+
+} RGX_HWRTDATA_COMMON_COOKIE;
+
+typedef struct _RGX_KM_HW_RT_DATASET_
+{
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataFwAddr;
+
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc;
+
+	RGX_FREELIST *apsFreeLists[RGXFW_MAX_FREELISTS];
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE			sNodeHWRTData;
+#endif
+
+} RGX_KM_HW_RT_DATASET;
+
+struct _RGX_FREELIST_ {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	CONNECTION_DATA			*psConnection;
+
+	/* Free list PMR */
+	PMR						*psFreeListPMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListPMROffset;
+
+	DEVMEMINT_RESERVATION* psFreeListReservation;
+
+	/* Freelist config */
+	IMG_UINT32				ui32MaxFLPages;
+	IMG_UINT32				ui32InitFLPages;
+	IMG_UINT32				ui32CurrentFLPages;
+	IMG_UINT32				ui32GrowFLPages;
+	IMG_UINT32				ui32ReadyFLPages;
+	IMG_UINT32				ui32GrowThreshold;		/* Percentage of FL memory used that should trigger a new grow request */
+	IMG_UINT32				ui32FreelistID;
+	IMG_UINT32				ui32FreelistGlobalID;	/* related global freelist for this freelist */
+	IMG_UINT64				ui64FreelistChecksum;	/* checksum over freelist content */
+	IMG_BOOL				bCheckFreelist;			/* freelist check enabled */
+	IMG_UINT32				ui32RefCount;			/* freelist reference counting */
+
+	IMG_UINT32				ui32NumGrowReqByApp;	/* Total number of grow requests by Application */
+	IMG_UINT32				ui32NumGrowReqByFW;		/* Total Number of grow requests by Firmware */
+	IMG_UINT32				ui32NumHighPages;		/* High Mark of pages in the freelist */
+
+	IMG_PID					ownerPid;				/* Pid of the owner of the list */
+
+	/* Memory Blocks */
+	DLLIST_NODE				sMemoryBlockHead;
+	DLLIST_NODE				sMemoryBlockInitHead;
+	DLLIST_NODE				sNode;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* HWRTData nodes linked to local freelist */
+	DLLIST_NODE				sNodeHWRTDataHead;
+#endif
+
+	/* FW data structures */
+	DEVMEM_MEMDESC			*psFWFreelistMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;
+};
+
+struct _RGX_PMR_NODE_ {
+	RGX_FREELIST			*psFreeList;
+	PMR						*psPMR;
+	PMR_PAGELIST			*psPageList;
+	DLLIST_NODE				sMemoryBlock;
+	IMG_UINT32				ui32NumPages;
+	IMG_BOOL				bFirstPageMissing;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RI_HANDLE				hRIHandle;
+#endif
+};
+
+typedef struct {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	DEVMEM_MEMDESC			*psFWZSBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;
+
+	DEVMEMINT_RESERVATION	*psReservation;
+	PMR						*psPMR;
+	IMG_UINT32				ui32ZSBufferID;
+	IMG_UINT32				ui32RefCount;
+	IMG_BOOL				bOnDemand;
+	IMG_BOOL				bIsBacked;
+
+	IMG_UINT32				ui32NumReqByApp;		/* Number of Backing Requests from Application */
+	IMG_UINT32				ui32NumReqByFW;			/* Number of Backing Requests from Firmware */
+
+	IMG_PID					owner;
+
+	DLLIST_NODE	sNode;
+}RGX_ZSBUFFER_DATA;
+
+typedef struct {
+	RGX_ZSBUFFER_DATA		*psZSBuffer;
+} RGX_POPULATION;
+
+/* Dump the physical pages of a freelist */
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList);
+
+
+/* Create set of HWRTData(s) */
+PVRSRV_ERROR RGXCreateHWRTDataSet(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_DEV_VIRTADDR          psPMMListDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  RGX_FREELIST	           *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_UINT32                ui32ScreenPixelMax,
+                                  IMG_UINT64                ui64MultiSampleCtl,
+                                  IMG_UINT64                ui64FlippedMultiSampleCtl,
+                                  IMG_UINT32                ui32TPCStride,
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                ui32TPCSize,
+                                  IMG_UINT32                ui32TEScreen,
+                                  IMG_UINT32                ui32TEAA,
+                                  IMG_UINT32                ui32TEMTILE1,
+                                  IMG_UINT32                ui32TEMTILE2,
+                                  IMG_UINT32                ui32MTileStride,
+                                  IMG_UINT32                ui32ISPMergeLowerX,
+                                  IMG_UINT32                ui32ISPMergeLowerY,
+                                  IMG_UINT32                ui32ISPMergeUpperX,
+                                  IMG_UINT32                ui32ISPMergeUpperY,
+                                  IMG_UINT32                ui32ISPMergeScaleX,
+                                  IMG_UINT32                ui32ISPMergeScaleY,
+                                  IMG_DEV_VIRTADDR          sMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                uiRgnHeaderSize,
+                                  IMG_UINT32                ui32ISPMtileSize,
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+PVRSRV_ERROR RGXCreateHWRTDataSet2(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE        *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  DEVMEMINT_RESERVATION     *psPMMListsReservation,
+                                  RGX_FREELIST	             *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_UINT32                ui32ScreenPixelMax,
+                                  IMG_UINT64                ui64MultiSampleCtl,
+                                  IMG_UINT64                ui64FlippedMultiSampleCtl,
+                                  IMG_UINT32                ui32TPCStride,
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                ui32TPCSize,
+                                  IMG_UINT32                ui32TEScreen,
+                                  IMG_UINT32                ui32TEAA,
+                                  IMG_UINT32                ui32TEMTILE1,
+                                  IMG_UINT32                ui32TEMTILE2,
+                                  IMG_UINT32                ui32MTileStride,
+                                  IMG_UINT32                ui32ISPMergeLowerX,
+                                  IMG_UINT32                ui32ISPMergeLowerY,
+                                  IMG_UINT32                ui32ISPMergeUpperX,
+                                  IMG_UINT32                ui32ISPMergeUpperY,
+                                  IMG_UINT32                ui32ISPMergeScaleX,
+                                  IMG_UINT32                ui32ISPMergeScaleY,
+                                  IMG_DEV_VIRTADDR          sMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                uiRgnHeaderSize,
+                                  IMG_UINT32                ui32ISPMtileSize,
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Destroy HWRTDataSet */
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet);
+
+/*
+	RGXCreateZSBufferKM
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer);
+
+/*
+	RGXDestroyZSBufferKM
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+
+/*
+ * RGXBackingZSBuffer()
+ *
+ * Backs ZS-Buffer with physical pages
+ */
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXPopulateZSBufferKM()
+ *
+ * Backs ZS-Buffer with physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+								   RGX_POPULATION **ppsPopulation);
+
+/*
+ * RGXUnbackingZSBuffer()
+ *
+ * Frees ZS-Buffer's physical pages
+ */
+PVRSRV_ERROR RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXUnpopulateZSBufferKM()
+ *
+ * Frees ZS-Buffer's physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation);
+
+/*
+	RGXProcessRequestZSBufferBacking
+*/
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXProcessRequestZSBufferUnbacking
+*/
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXGrowFreeList
+*/
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader);
+
+/* Create free list */
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA      *psConnection,
+                               PVRSRV_DEVICE_NODE	*psDeviceNode,
+                               IMG_HANDLE			hMemCtxPrivData,
+                               IMG_UINT32			ui32MaxFLPages,
+                               IMG_UINT32			ui32InitFLPages,
+                               IMG_UINT32			ui32GrowFLPages,
+                               IMG_UINT32           ui32GrowParamThreshold,
+                               RGX_FREELIST			*psGlobalFreeList,
+                               IMG_BOOL				bCheckFreelist,
+                               DEVMEMINT_RESERVATION* psFreeListReservation,
+                               RGX_FREELIST			**ppsFreeList);
+
+/* Destroy free list */
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList);
+
+/*
+	RGXProcessRequestGrow
+*/
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   IMG_UINT32 ui32FreelistID);
+
+
+/* Reconstruct free list after Hardware Recovery */
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+											  IMG_UINT32 ui32FreelistsCount,
+											  const IMG_UINT32 *paui32Freelists);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateRenderContextKM
+
+ @Description
+	Server-side implementation of RGXCreateRenderContext
+
+ @Input psConnection -
+ @Input psDeviceNode - device node
+ @Input i32Priority - context priority
+ @Input sVDMCallStackAddr - VDM call stack device virtual address
+ @Input ui32CallStackDepth - VDM call stack depth
+ @Input ui32FrameworkCommandSize - framework command size
+ @Input pabyFrameworkCommand - ptr to framework command
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32StaticRenderContextStateSize - size of fixed render state
+ @Input pStaticRenderContextState - ptr to fixed render state buffer
+ @Input ui32PackedCCBSizeU8888 :
+		ui8TACCBAllocSizeLog2 - TA CCB size
+		ui8TACCBMaxAllocSizeLog2 - maximum size to which TA CCB can grow
+		ui83DCCBAllocSizeLog2 - 3D CCB size
+		ui83DCCBMaxAllocSizeLog2 - maximum size to which 3D CCB can grow
+ @Input ui32ContextFlags - flags which specify properties of the context
+ @Output ppsRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+											PVRSRV_DEVICE_NODE			*psDeviceNode,
+											IMG_INT32					i32Priority,
+											IMG_DEV_VIRTADDR			sVDMCallStackAddr,
+											IMG_UINT32					ui32CallStackDepth,
+											IMG_UINT32					ui32FrameworkCommandSize,
+											IMG_PBYTE					pabyFrameworkCommand,
+											IMG_HANDLE					hMemCtxPrivData,
+											IMG_UINT32					ui32StaticRenderContextStateSize,
+											IMG_PBYTE					pStaticRenderContextState,
+											IMG_UINT32					ui32PackedCCBSizeU8888,
+											IMG_UINT32					ui32ContextFlags,
+											IMG_UINT64					ui64RobustnessAddress,
+											IMG_UINT32					ui32MaxTADeadlineMS,
+											IMG_UINT32					ui32Max3DDeadlineMS,
+											RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyRenderContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRenderContext
+
+ @Input psRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXKickTA3DKM
+
+ @Description
+	Server-side implementation of RGXKickTA3D
+
+ @Input psRTDataCleanup - RT data associated with the kick (or NULL)
+ @Input psZBuffer - Z-buffer associated with the kick (or NULL)
+ @Input psSBuffer - S-buffer associated with the kick (or NULL)
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRSyncPrimBlock,
+								 IMG_UINT32					ui32PRSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckFence,
+								 PVRSRV_TIMELINE			iUpdateTimeline,
+								 PVRSRV_FENCE				*piUpdateFence,
+								 IMG_CHAR					szFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheckFence3D,
+								 PVRSRV_TIMELINE			iUpdateTimeline3D,
+								 PVRSRV_FENCE				*piUpdateFence3D,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus);
+
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore);
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDevNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+												 RGX_CONTEXT_PROPERTY eContextProperty,
+												 IMG_UINT64 ui64Input,
+												 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of render contexts on this device */
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client contexts are stalled */
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+#endif /* RGXTA3D_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.c
new file mode 100644
index 000000000000..63379f57a032
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.c
@@ -0,0 +1,1990 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific transfer queue routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxtransfer.h"
+#include "rgx_tq_shared.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_memallocflags.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "htbserver.h"
+#include "rgxshader.h"
+
+#include "pdump_km.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_bvnc_defs_km.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TQ_UFO_DUMP	0
+
+//#define TRANSFER_CHECKPOINT_DEBUG 1
+
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psFWContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_3D_DATA;
+
+typedef struct {
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_2D_DATA;
+
+struct _RGX_SERVER_TQ_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC              *psFWTransferContextMemDesc;
+	IMG_UINT32					ui32Flags;
+#define RGX_SERVER_TQ_CONTEXT_FLAGS_2D		(1<<0)
+#define RGX_SERVER_TQ_CONTEXT_FLAGS_3D		(1<<1)
+	RGX_SERVER_TQ_3D_DATA		s3DData;
+	RGX_SERVER_TQ_2D_DATA		s2DData;
+	DLLIST_NODE					sListNode;
+	ATOMIC_T			hIntJobRef;
+	IMG_UINT32			ui32PDumpFlags;
+	/* per-prepare sync address lists */
+	SYNC_ADDR_LIST			asSyncAddrListFence[TQ_MAX_PREPARES_PER_SUBMIT];
+	SYNC_ADDR_LIST			asSyncAddrListUpdate[TQ_MAX_PREPARES_PER_SUBMIT];
+	POS_LOCK				hLock;
+};
+
+/*
+	Static functions used by transfer context code
+*/
+static PVRSRV_ERROR _Create3DTransferContext(CONNECTION_DATA *psConnection,
+											 PVRSRV_DEVICE_NODE *psDeviceNode,
+											 DEVMEM_MEMDESC *psAllocatedMemDesc,
+											 IMG_UINT32 ui32AllocatedOffset,
+											 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+											 IMG_INT32 i32Priority,
+											 RGX_COMMON_CONTEXT_INFO *psInfo,
+											 RGX_SERVER_TQ_3D_DATA *ps3DData,
+											 IMG_UINT32 ui32CCBAllocSizeLog2,
+											 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+											 IMG_UINT32 ui32ContextFlags,
+											 IMG_UINT64 ui64RobustnessAddress)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+	IMG_UINT	uiNumISPStoreRegs;
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TQ/3D context suspend state");
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+													RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	ps3DData->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-tq3d");
+	if (IS_ERR(ps3DData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(ps3DData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = DevmemFwAllocate(psDevInfo,
+							ui3DRegISPStateStoreSize,
+							RGX_FWCOMCTX_ALLOCFLAGS,
+							"FwTQ3DContext",
+							&ps3DData->psFWContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextswitchstate;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TQ_3D,
+	                                 RGXFWIF_DM_3D,
+									 NULL,
+									 psAllocatedMemDesc,
+									 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psFWContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TQ3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TQ3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 UINT_MAX, /* max deadline MS */
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+
+	PDUMPCOMMENT(psDeviceNode, "Dump 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psFWContextStateMemDesc, 0, sizeof(RGXFWIF_3DCTX_STATE), PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+	DevmemFwUnmapAndFree(psDevInfo, ps3DData->psFWContextStateMemDesc);
+fail_contextswitchstate:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps3DData->psBufferSyncContext);
+	ps3DData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+static PVRSRV_ERROR _Create2DTransferContext(CONNECTION_DATA *psConnection,
+											 PVRSRV_DEVICE_NODE *psDeviceNode,
+											 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+											 IMG_INT32 i32Priority,
+											 RGX_COMMON_CONTEXT_INFO *psInfo,
+											 RGX_SERVER_TQ_2D_DATA *ps2DData,
+											 IMG_UINT32 ui32CCBAllocSizeLog2,
+											 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+											 IMG_UINT32 ui32ContextFlags,
+											 IMG_UINT64 ui64RobustnessAddress)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	ps2DData->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-tqtla");
+	if (IS_ERR(ps2DData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(ps2DData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TQ_2D,
+	                                 RGXFWIF_DM_2D,
+									 NULL,
+	                                 NULL,
+	                                 0,
+	                                 psFWMemContextMemDesc,
+	                                 NULL,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TQ2D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TQ2D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 UINT_MAX, /* max deadline MS */
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps2DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	ps2DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps2DData->psBufferSyncContext);
+	ps2DData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+static PVRSRV_ERROR _Destroy2DTransferContext(RGX_SERVER_TQ_2D_DATA *ps2DData,
+											  PVRSRV_DEVICE_NODE *psDeviceNode,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps2DData->psServerCommonContext,
+											  RGXFWIF_DM_2D,
+											  ui32PDumpFlags);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps2DData->psServerCommonContext);
+	ps2DData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps2DData->psBufferSyncContext);
+	ps2DData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+#endif /* #if defined(RGX_FEATURE_TLA_BIT_MASK) */
+
+static PVRSRV_ERROR _Destroy3DTransferContext(RGX_SERVER_TQ_3D_DATA *ps3DData,
+											  PVRSRV_DEVICE_NODE *psDeviceNode,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  ui32PDumpFlags);
+	if (PVRSRVIsRetryError(eError))
+	{
+		return eError;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+	}
+
+	/* ... it has so we can free its resources */
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psFWContextStateMemDesc);
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	ps3DData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps3DData->psBufferSyncContext);
+	ps3DData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVCreateTransferContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateTransferContextKM(CONNECTION_DATA		*psConnection,
+										   PVRSRV_DEVICE_NODE		*psDeviceNode,
+										   IMG_INT32				i32Priority,
+										   IMG_UINT32				ui32FrameworkCommandSize,
+										   IMG_PBYTE				pabyFrameworkCommand,
+										   IMG_HANDLE				hMemCtxPrivData,
+										   IMG_UINT32				ui32PackedCCBSizeU8888,
+										   IMG_UINT32				ui32ContextFlags,
+										   IMG_UINT64				ui64RobustnessAddress,
+										   RGX_SERVER_TQ_CONTEXT	**ppsTransferContext)
+{
+	RGX_SERVER_TQ_CONTEXT	*psTransferContext;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC			*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO	sInfo = {NULL};
+	PVRSRV_ERROR			eError = PVRSRV_OK;
+
+	/* Allocate the server side structure */
+	*ppsTransferContext = NULL;
+	psTransferContext = OSAllocZMem(sizeof(*psTransferContext));
+	if (psTransferContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW transfer context, this has the TQ common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWTRANSFERCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwTransferContext",
+			&psTransferContext->psFWTransferContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwtransfercontext;
+	}
+
+	eError = OSLockCreate(&psTransferContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+									__func__,
+									PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	psTransferContext->psDeviceNode = psDeviceNode;
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psTransferContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate firmware GPU framework state (%s)",
+						__func__,
+						PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psTransferContext->psFWFrameworkMemDesc,
+				pabyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to populate the framework buffer (%s)",
+						__func__,
+				PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psTransferContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DTransferContext(psConnection,
+									  psDeviceNode,
+									  psTransferContext->psFWTransferContextMemDesc,
+									  offsetof(RGXFWIF_FWTRANSFERCONTEXT, sTQContext),
+									  psFWMemContextMemDesc,
+									  i32Priority,
+									  &sInfo,
+									  &psTransferContext->s3DData,
+									  U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+									  U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+									  ui32ContextFlags,
+									  ui64RobustnessAddress);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dtransfercontext;
+	}
+	psTransferContext->ui32Flags |= RGX_SERVER_TQ_CONTEXT_FLAGS_3D;
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA))
+	{
+		eError = _Create2DTransferContext(psConnection,
+										  psDeviceNode,
+										  psFWMemContextMemDesc,
+										  i32Priority,
+										  &sInfo,
+										  &psTransferContext->s2DData,
+										  U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+										  U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+										  ui32ContextFlags,
+										  ui64RobustnessAddress);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_2dtransfercontext;
+		}
+		psTransferContext->ui32Flags |= RGX_SERVER_TQ_CONTEXT_FLAGS_2D;
+	}
+#endif
+
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sTransferCtxtListHead), &(psTransferContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+		*ppsTransferContext = psTransferContext;
+	}
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+fail_2dtransfercontext:
+	_Destroy3DTransferContext(&psTransferContext->s3DData,
+							  psTransferContext->psDeviceNode,
+							  psTransferContext->ui32PDumpFlags);
+#endif
+fail_3dtransfercontext:
+fail_frameworkcopy:
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	OSLockDestroy(psTransferContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+fail_fwtransfercontext:
+	OSFreeMem(psTransferContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*ppsTransferContext = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyTransferContextKM(RGX_SERVER_TQ_CONTEXT *psTransferContext)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psTransferContext->psDeviceNode->pvDevice;
+	IMG_UINT32 i;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+	dllist_remove_node(&(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((psTransferContext->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+			(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = _Destroy2DTransferContext(&psTransferContext->s2DData,
+										   psTransferContext->psDeviceNode,
+										   PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_destroy2d;
+		}
+		/* We've freed the 2D context, don't try to free it again */
+		psTransferContext->ui32Flags &= ~RGX_SERVER_TQ_CONTEXT_FLAGS_2D;
+	}
+#endif
+
+	if (psTransferContext->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D)
+	{
+		eError = _Destroy3DTransferContext(&psTransferContext->s3DData,
+										   psTransferContext->psDeviceNode,
+										   PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_destroy3d;
+		}
+		/* We've freed the 3D context, don't try to free it again */
+		psTransferContext->ui32Flags &= ~RGX_SERVER_TQ_CONTEXT_FLAGS_3D;
+	}
+
+	/* free any resources within the per-prepare UFO address stores */
+	for (i = 0; i < TQ_MAX_PREPARES_PER_SUBMIT; i++)
+	{
+		SyncAddrListDeinit(&psTransferContext->asSyncAddrListFence[i]);
+		SyncAddrListDeinit(&psTransferContext->asSyncAddrListUpdate[i]);
+	}
+
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+
+	OSLockDestroy(psTransferContext->hLock);
+
+	OSFreeMem(psTransferContext);
+
+	return PVRSRV_OK;
+
+fail_destroy3d:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+
+fail_destroy2d:
+#endif
+	OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTransferCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+ * PVRSRVSubmitTQ3DKickKM
+ */
+/* Old bridge call for backwards compatibility. */
+PVRSRV_ERROR PVRSRVRGXSubmitTransferKM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									   IMG_UINT32				ui32PrepareCount,
+									   IMG_UINT32				*paui32ClientUpdateCount,
+									   SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									   IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									   IMG_UINT32				**papaui32ClientUpdateValue,
+									   PVRSRV_FENCE				iCheckFence,
+									   PVRSRV_TIMELINE			i2DUpdateTimeline,
+									   PVRSRV_FENCE				*pi2DUpdateFence,
+									   PVRSRV_TIMELINE			i3DUpdateTimeline,
+									   PVRSRV_FENCE				*pi3DUpdateFence,
+									   IMG_CHAR					szFenceName[32],
+									   IMG_UINT32				*paui32FWCommandSize,
+									   IMG_UINT8				**papaui8FWCommand,
+									   IMG_UINT32				*pui32TQPrepareFlags,
+									   IMG_UINT32				ui32ExtJobRef,
+									   IMG_UINT32				ui32SyncPMRCount,
+									   IMG_UINT32				*paui32SyncPMRFlags,
+									   PMR						**ppsSyncPMRs)
+{
+	return PVRSRVRGXSubmitTransfer3KM(psTransferContext,
+									  ui32PrepareCount,
+									  paui32ClientUpdateCount,
+									  papauiClientUpdateUFODevVarBlock,
+									  papaui32ClientUpdateSyncOffset,
+									  papaui32ClientUpdateValue,
+									  iCheckFence,
+									  i2DUpdateTimeline,
+									  pi2DUpdateFence,
+									  i3DUpdateTimeline,
+									  pi3DUpdateFence,
+									  szFenceName,
+									  PVRSRV_NO_FENCE,
+									  paui32FWCommandSize,
+									  papaui8FWCommand,
+									  pui32TQPrepareFlags,
+									  ui32ExtJobRef,
+									  ui32SyncPMRCount,
+									  paui32SyncPMRFlags,
+									  ppsSyncPMRs);
+}
+
+PVRSRV_ERROR PVRSRVRGXSubmitTransfer3KM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									   IMG_UINT32				ui32PrepareCount,
+									   IMG_UINT32				*paui32ClientUpdateCount,
+									   SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									   IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									   IMG_UINT32				**papaui32ClientUpdateValue,
+									   PVRSRV_FENCE				iCheckFence,
+									   PVRSRV_TIMELINE			i2DUpdateTimeline,
+									   PVRSRV_FENCE				*pi2DUpdateFence,
+									   PVRSRV_TIMELINE			i3DUpdateTimeline,
+									   PVRSRV_FENCE				*pi3DUpdateFence,
+									   IMG_CHAR					szFenceName[32],
+									   PVRSRV_FENCE				iExportFenceToSignal,
+									   IMG_UINT32				*paui32FWCommandSize,
+									   IMG_UINT8				**papaui8FWCommand,
+									   IMG_UINT32				*pui32TQPrepareFlags,
+									   IMG_UINT32				ui32ExtJobRef,
+									   IMG_UINT32				ui32SyncPMRCount,
+									   IMG_UINT32				*paui32SyncPMRFlags,
+									   PMR						**ppsSyncPMRs)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelper;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	RGX_CCB_CMD_HELPER_DATA *pas2DCmdHelper;
+#endif
+	IMG_UINT32                ui323DCmdCount = 0;
+	IMG_UINT32                ui323DCmdLast = 0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset = 0;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	IMG_UINT32                ui322DCmdCount = 0;
+	IMG_UINT32                ui322DCmdLast = 0;
+	__maybe_unused IMG_UINT32 ui322DCmdOffset = 0;
+#endif
+	IMG_UINT32 ui32PDumpFlags = PDUMP_FLAGS_NONE;
+	IMG_UINT32 i;
+	IMG_UINT64 uiCheckFenceUID = 0;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	IMG_UINT64 ui2DUpdateFenceUID = 0;
+#endif
+	IMG_UINT64 ui3DUpdateFenceUID = 0;
+
+	PSYNC_CHECKPOINT ps3DUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT ps3DExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *ppsExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui323DIntAllocatedUpdateValues = NULL;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PSYNC_CHECKPOINT ps2DUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT ps2DExportFenceSyncCheckpoint = NULL;
+	IMG_UINT32 *pui322DIntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps2DFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui322DFenceTimelineUpdateValue = 0;
+	void *pv2DUpdateFenceFinaliseData = NULL;
+#endif
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVRSRV_FENCE i2DUpdateFence = PVRSRV_NO_FENCE;
+#endif
+	PVRSRV_FENCE i3DUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT32   ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32   ui32PreparesDone = 0;
+
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+#if !defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVR_UNREFERENCED_PARAMETER(i2DUpdateTimeline);
+	PVR_UNREFERENCED_PARAMETER(pi2DUpdateFence);
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (i2DUpdateTimeline != PVRSRV_NO_TIMELINE && !pi2DUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+	if (i3DUpdateTimeline != PVRSRV_NO_TIMELINE && !pi3DUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Validate sync prim fence/update value ptrs
+	 * for each prepare.
+	 */
+	{
+		IMG_UINT32 ui32Prepare;
+		IMG_UINT32 *pui32UpdateCount = paui32ClientUpdateCount;
+		IMG_UINT32 **papui32UpdateValue = papaui32ClientUpdateValue;
+
+		/* Check that we have not been given a null ptr for
+		 * update count parameters.
+		 */
+		PVR_LOG_RETURN_IF_FALSE((paui32ClientUpdateCount != NULL),
+		                        "paui32ClientUpdateCount NULL",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+		for (ui32Prepare=0; ui32Prepare<ui32PrepareCount; ui32Prepare++)
+		{
+			/* Ensure we haven't been given a null ptr to
+			 * update values if we have been told we
+			 * have updates for this prepare
+			 */
+			if (*pui32UpdateCount > 0)
+			{
+				PVR_LOG_RETURN_IF_FALSE(*papui32UpdateValue != NULL,
+				                        "paui32ClientUpdateValue NULL but "
+				                        "ui32ClientUpdateCount > 0",
+				                        PVRSRV_ERROR_INVALID_PARAMS);
+			}
+			/* Advance local ptr to update values ptr for next prepare. */
+			papui32UpdateValue++;
+			/* Advance local ptr to update count for next prepare. */
+			pui32UpdateCount++;
+		}
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	if ((ui32PrepareCount == 0) || (ui32PrepareCount > TQ_MAX_PREPARES_PER_SUBMIT))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32SyncPMRCount != 0)
+	{
+		if (!ppsSyncPMRs)
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+#if defined(SUPPORT_BUFFER_SYNC)
+		/* PMR sync is valid only when there is no batching */
+		if ((ui32PrepareCount != 1))
+#endif
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* We can't allocate the required amount of stack space on all consumer architectures */
+	pas3DCmdHelper = OSAllocMem(sizeof(*pas3DCmdHelper) * ui32PrepareCount);
+	if (pas3DCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc3dhelper;
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	pas2DCmdHelper = OSAllocMem(sizeof(*pas2DCmdHelper) * ui32PrepareCount);
+	if (pas2DCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc2dhelper;
+	}
+#endif
+
+	if (iCheckFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psDeviceNode->hSyncCheckpointContext));
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(psDeviceNode->hSyncCheckpointContext,
+											iCheckFence,
+											&ui32FenceSyncCheckpointCount,
+											&apsFenceSyncCheckpoints,
+											&uiCheckFenceUID,
+											ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)", __func__, eError));
+			goto fail_resolve_fencesync_input_fence;
+		}
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			IMG_UINT32 ii;
+			for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+			{
+				PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint));
+			}
+		}
+#endif
+	}
+	/*
+		Ensure we do the right thing for server syncs which cross call boundaries
+	*/
+	for (i=0;i<ui32PrepareCount;i++)
+	{
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 3D))
+		{
+			ui323DCmdLast++;
+		}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		else if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			ui322DCmdLast++;
+		}
+#endif
+	}
+
+	/*
+		Init the command helper commands for all the prepares
+	*/
+	for (i=0;i<ui32PrepareCount;i++)
+	{
+		RGX_CLIENT_CCB *psClientCCB;
+		RGX_SERVER_COMMON_CONTEXT *psServerCommonCtx;
+		IMG_CHAR *pszCommandName;
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper;
+		RGXFWIF_CCB_CMD_TYPE eType;
+		PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+		PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+		SYNC_ADDR_LIST *psSyncAddrListFence = &psTransferContext->asSyncAddrListFence[i];
+		SYNC_ADDR_LIST *psSyncAddrListUpdate = &psTransferContext->asSyncAddrListUpdate[i];
+		IMG_UINT32 ui32IntClientFenceCount = 0U;
+		IMG_UINT32 ui32IntClientUpdateCount = paui32ClientUpdateCount[i];
+		IMG_UINT32 *paui32IntUpdateValue = papaui32ClientUpdateValue[i];
+#if defined(SUPPORT_BUFFER_SYNC)
+		struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+
+		PVRSRV_FENCE *piUpdateFence = NULL;
+		PVRSRV_TIMELINE	iUpdateTimeline = PVRSRV_NO_TIMELINE;
+		void **ppvUpdateFenceFinaliseData = NULL;
+		PSYNC_CHECKPOINT * ppsUpdateSyncCheckpoint = NULL;
+		PVRSRV_CLIENT_SYNC_PRIM **ppsFenceTimelineUpdateSync = NULL;
+		IMG_UINT32 *pui32FenceTimelineUpdateValue = NULL;
+		IMG_UINT32 **ppui32IntAllocatedUpdateValues = NULL;
+		IMG_BOOL bCheckFence = IMG_FALSE;
+		IMG_BOOL bUpdateFence = IMG_FALSE;
+		IMG_BOOL bExportFence = IMG_FALSE;
+		IMG_UINT64 *puiUpdateFenceUID = NULL;
+
+		IMG_BOOL bCCBStateOpen = IMG_FALSE;
+
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 3D))
+		{
+			psServerCommonCtx = psTransferContext->s3DData.psServerCommonContext;
+			psClientCCB = FWCommonContextGetClientCCB(psServerCommonCtx);
+			pszCommandName = "TQ-3D";
+			psCmdHelper = &pas3DCmdHelper[ui323DCmdCount++];
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_3D;
+#if defined(SUPPORT_BUFFER_SYNC)
+			psBufferSyncContext = psTransferContext->s3DData.psBufferSyncContext;
+#endif
+			bCheckFence = ui323DCmdCount == 1;
+			bUpdateFence = ui323DCmdCount == ui323DCmdLast
+				&& i3DUpdateTimeline != PVRSRV_NO_TIMELINE;
+			bExportFence = ui323DCmdCount == ui323DCmdLast
+				&& iExportFenceToSignal != PVRSRV_NO_FENCE;
+
+			if (bUpdateFence)
+			{
+				piUpdateFence = &i3DUpdateFence;
+				iUpdateTimeline = i3DUpdateTimeline;
+				ppvUpdateFenceFinaliseData = &pv3DUpdateFenceFinaliseData;
+				ppsUpdateSyncCheckpoint = &ps3DUpdateSyncCheckpoint;
+				ppsFenceTimelineUpdateSync = &ps3DFenceTimelineUpdateSync;
+				pui32FenceTimelineUpdateValue = &ui323DFenceTimelineUpdateValue;
+				ppui32IntAllocatedUpdateValues = &pui323DIntAllocatedUpdateValues;
+				puiUpdateFenceUID = &ui3DUpdateFenceUID;
+			}
+
+			if (bExportFence)
+			{
+				ppsExportFenceSyncCheckpoint = &ps3DExportFenceSyncCheckpoint;
+			}
+		}
+		else
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			psServerCommonCtx = psTransferContext->s2DData.psServerCommonContext;
+			psClientCCB = FWCommonContextGetClientCCB(psServerCommonCtx);
+			pszCommandName = "TQ-2D";
+			psCmdHelper = &pas2DCmdHelper[ui322DCmdCount++];
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_2D;
+#if defined(SUPPORT_BUFFER_SYNC)
+			psBufferSyncContext = psTransferContext->s2DData.psBufferSyncContext;
+#endif
+			bCheckFence = ui322DCmdCount == 1;
+			bUpdateFence = ui322DCmdCount == ui322DCmdLast
+				&& i2DUpdateTimeline != PVRSRV_NO_TIMELINE;
+			bExportFence = ui322DCmdCount == ui322DCmdLast
+				&& iExportFenceToSignal != PVRSRV_NO_FENCE;
+
+			if (bUpdateFence)
+			{
+				piUpdateFence = &i2DUpdateFence;
+				iUpdateTimeline = i2DUpdateTimeline;
+				ppvUpdateFenceFinaliseData = &pv2DUpdateFenceFinaliseData;
+				ppsUpdateSyncCheckpoint = &ps2DUpdateSyncCheckpoint;
+				ppsFenceTimelineUpdateSync = &ps2DFenceTimelineUpdateSync;
+				pui32FenceTimelineUpdateValue = &ui322DFenceTimelineUpdateValue;
+				ppui32IntAllocatedUpdateValues = &pui322DIntAllocatedUpdateValues;
+				puiUpdateFenceUID = &ui2DUpdateFenceUID;
+			}
+
+			if (bExportFence)
+			{
+				ppsExportFenceSyncCheckpoint = &ps2DExportFenceSyncCheckpoint;
+			}
+		}
+		else
+#endif
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto fail_prepare_loop;
+		}
+
+		if (i == 0)
+		{
+			ui32PDumpFlags = ((pui32TQPrepareFlags[i] & TQ_PREP_FLAGS_PDUMPCONTINUOUS) != 0) ? PDUMP_FLAGS_CONTINUOUS : PDUMP_FLAGS_NONE;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags,
+					"%s Command Server Submit on FWCtx %08x", pszCommandName, FWCommonContextGetFWAddress(psServerCommonCtx).ui32Addr);
+			psTransferContext->ui32PDumpFlags |= ui32PDumpFlags;
+		}
+		else
+		{
+			IMG_UINT32 ui32NewPDumpFlags = ((pui32TQPrepareFlags[i] & TQ_PREP_FLAGS_PDUMPCONTINUOUS) != 0) ? PDUMP_FLAGS_CONTINUOUS : PDUMP_FLAGS_NONE;
+			if (ui32NewPDumpFlags != ui32PDumpFlags)
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				PVR_DPF((PVR_DBG_ERROR, "%s: Mixing of continuous and non-continuous command in a batch is not permitted", __func__));
+				goto fail_prepare_loop;
+			}
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncAddrListPopulate(psTransferContext->sSyncAddrListFence, %d fences)", __func__, ui32IntClientFenceCount));
+		eError = SyncAddrListPopulate(psSyncAddrListFence,
+										0,
+										NULL,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_prepare_loop;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncAddrListPopulate(psTransferContext->asSyncAddrListUpdate[], %d updates)", __func__, ui32IntClientUpdateCount));
+		eError = SyncAddrListPopulate(psSyncAddrListUpdate,
+										ui32IntClientUpdateCount,
+										papauiClientUpdateUFODevVarBlock[i],
+										papaui32ClientUpdateSyncOffset[i]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_prepare_loop;
+		}
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after sync prims) ui32IntClientUpdateCount=%d", __func__, ui32IntClientUpdateCount));
+		if (ui32SyncPMRCount)
+		{
+#if defined(SUPPORT_BUFFER_SYNC)
+			int err;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling pvr_buffer_sync_resolve_and_create_fences", __func__));
+			err = pvr_buffer_sync_resolve_and_create_fences(psBufferSyncContext,
+			                                                psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                                ui32SyncPMRCount,
+			                                                ppsSyncPMRs,
+			                                                paui32SyncPMRFlags,
+			                                                &ui32BufferFenceSyncCheckpointCount,
+			                                                &apsBufferFenceSyncCheckpoints,
+			                                                &psBufferUpdateSyncCheckpoint,
+			                                                &psBufferSyncData);
+			if (err)
+			{
+				switch (err)
+				{
+					case -EINTR:
+						eError = PVRSRV_ERROR_RETRY;
+						break;
+					case -ENOMEM:
+						eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+						break;
+					default:
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						break;
+				}
+
+				if (eError != PVRSRV_ERROR_RETRY)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   pvr_buffer_sync_resolve_and_create_fences failed (%s)", __func__, PVRSRVGetErrorString(eError)));
+				}
+				goto fail_resolve_buffersync_input_fence;
+			}
+
+			/* Append buffer sync fences */
+			if (ui32BufferFenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints to TQ Fence (psSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>)...", __func__, ui32BufferFenceSyncCheckpointCount, (void*)psSyncAddrListFence , (void*)pauiIntFenceUFOAddress));
+				SyncAddrListAppendAndDeRefCheckpoints(psSyncAddrListFence,
+													  ui32BufferFenceSyncCheckpointCount,
+													  apsBufferFenceSyncCheckpoints);
+				if (!pauiIntFenceUFOAddress)
+				{
+					pauiIntFenceUFOAddress = psSyncAddrListFence->pasFWAddrs;
+				}
+				ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+			}
+
+			if (psBufferUpdateSyncCheckpoint)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 buffer sync checkpoint<%p> to TQ Update (&psTransferContext->asSyncAddrListUpdate[i]=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)psBufferUpdateSyncCheckpoint, (void*)psSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+				/* Append the update (from output fence) */
+				SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiIntUpdateUFOAddress)
+				{
+					pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+				}
+				ui32IntClientUpdateCount++;
+			}
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after buffer_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __func__, ui32SyncPMRCount));
+			PVR_DPF((PVR_DBG_ERROR, "%s:   <--EXIT(%d)", __func__, PVRSRV_ERROR_INVALID_PARAMS));
+			OSLockRelease(psTransferContext->hLock);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+		}
+
+		/* Create the output fence (if required) */
+		if (bUpdateFence)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (piUpdateFence=%p, iUpdateTimeline=%d,  psTranserContext->psDeviceNode->hSyncCheckpointContext=<%p>)", __func__, piUpdateFence, iUpdateTimeline, (void*)psDeviceNode->hSyncCheckpointContext));
+			eError = SyncCheckpointCreateFence(psDeviceNode,
+			                                   szFenceName,
+			                                   iUpdateTimeline,
+			                                   psDeviceNode->hSyncCheckpointContext,
+			                                   piUpdateFence,
+			                                   puiUpdateFenceUID,
+			                                   ppvUpdateFenceFinaliseData,
+			                                   ppsUpdateSyncCheckpoint,
+			                                   (void*)ppsFenceTimelineUpdateSync,
+			                                   pui32FenceTimelineUpdateValue,
+			                                   ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s:   SyncCheckpointCreateFence failed (%s)",
+						__func__,
+						PVRSRVGetErrorString(eError)));
+				goto fail_prepare_loop;
+			}
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: returned from SyncCheckpointCreateFence (piUpdateFence=%p)", __func__, piUpdateFence));
+
+			/* Resolve the iExportFenceToSignal (if required) */
+			if (bExportFence)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+				eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+														  psDeviceNode->hSyncCheckpointContext,
+														  ppsExportFenceSyncCheckpoint,
+														  ui32PDumpFlags);
+				if (eError != PVRSRV_OK)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), *ppsExportFenceSyncCheckpoint));
+					goto fail_prepare_loop;
+				}
+
+				/* Check that the export fence was not also included as part of the
+				 * check fence (which is an error and would lead to a stalled kick).
+				 */
+				if (ui32FenceSyncCheckpointCount > 0)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s:   Checking export fence is not part of check fence...", __func__));
+					CHKPT_DBG((PVR_DBG_ERROR, "%s:   ui32FenceSyncCheckpointCount=%d",
+							   __func__, ui32FenceSyncCheckpointCount));
+					if (ui32FenceSyncCheckpointCount > 0)
+					{
+						IMG_UINT32 iii;
+
+						for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+						{
+							CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+							if (apsFenceSyncCheckpoints[iii] == *ppsExportFenceSyncCheckpoint)
+							{
+								CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, *ppsExportFenceSyncCheckpoint));
+								eError = PVRSRV_ERROR_INVALID_PARAMS;
+								PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+								goto fail_prepare_loop;
+							}
+						}
+					}
+				}
+			}
+
+			/* Append the sync prim update for the timeline (if required) */
+			if (*ppsFenceTimelineUpdateSync)
+			{
+				IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+				/* Allocate memory to hold the list of update values (including our timeline update) */
+				*ppui32IntAllocatedUpdateValues = OSAllocMem(sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				if (!*ppui32IntAllocatedUpdateValues)
+				{
+					/* Failed to allocate memory */
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto fail_prepare_loop;
+				}
+				OSCachedMemSet(*ppui32IntAllocatedUpdateValues, 0xbb, sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+#if defined(SUPPORT_BUFFER_SYNC)
+				if (psBufferUpdateSyncCheckpoint)
+				{
+					/* Copy the update values into the new memory, then append our timeline update value */
+					OSCachedMemCopy(*ppui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount-1));
+					pui32TimelineUpdateWp = *ppui32IntAllocatedUpdateValues + (ui32IntClientUpdateCount-1);
+				}
+				else
+#endif
+				{
+					/* Copy the update values into the new memory, then append our timeline update value */
+					OSCachedMemCopy(*ppui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(**ppui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+					pui32TimelineUpdateWp = *ppui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+				}
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: Appending the additional update value 0x%x)", __func__, *pui32FenceTimelineUpdateValue));
+				/* Now set the additional update value */
+				*pui32TimelineUpdateWp = *pui32FenceTimelineUpdateValue;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+				if (ui32IntClientUpdateCount > 0)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)*ppui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: *ppui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Now append the timeline sync prim addr to the transfer context update list */
+				SyncAddrListAppendSyncPrim(psSyncAddrListUpdate,
+				                           *ppsFenceTimelineUpdateSync);
+				ui32IntClientUpdateCount++;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+				if (ui32IntClientUpdateCount > 0)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)*ppui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: *ppui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: set paui32IntUpdateValue<%p> to point to *ppui32IntAllocatedUpdateValues<%p>", __func__, (void*)paui32IntUpdateValue, (void*)*ppui32IntAllocatedUpdateValues));
+				paui32IntUpdateValue = *ppui32IntAllocatedUpdateValues;
+			}
+		}
+
+		if (bCheckFence && ui32FenceSyncCheckpointCount)
+		{
+			/* Append the checks (from input fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to TQ Fence (psSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)psSyncAddrListFence));
+			SyncAddrListAppendCheckpoints(psSyncAddrListFence,
+										  ui32FenceSyncCheckpointCount,
+										  apsFenceSyncCheckpoints);
+			if (!pauiIntFenceUFOAddress)
+			{
+				pauiIntFenceUFOAddress = psSyncAddrListFence->pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+			if (ui32IntClientFenceCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+				for (iii=0; iii<ui32IntClientFenceCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: psSyncAddrListFence->pasFWAddrs[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+		if (bUpdateFence && *ppsUpdateSyncCheckpoint)
+		{
+			/* Append the update (from output fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to TQ Update (psSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)&psTransferContext->asSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+			SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+										  1,
+										  ppsUpdateSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		if (bExportFence && *ppsExportFenceSyncCheckpoint)
+		{
+			/* Append the update (from export fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Transfer Update (psSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)psSyncAddrListUpdate , (void*)*ppsExportFenceSyncCheckpoint));
+			SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+										  1,
+										  ppsExportFenceSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_TQ_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping TQ fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TQ fence syncs (&psTransferContext->asSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psTransferContext->asSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_ASSERT(psTmpIntFenceUFOAddress->ui32Addr & 0x1);
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TQ update syncs (&psTransferContext->asSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psTransferContext->asSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+		ui32PreparesDone++;
+
+		/*
+			Create the command helper data for this command
+		*/
+		RGXCmdHelperInitCmdCCB(psDevInfo,
+		                       psClientCCB,
+		                       0,
+		                       ui32IntClientFenceCount,
+		                       pauiIntFenceUFOAddress,
+		                       NULL, /* fence value */
+		                       ui32IntClientUpdateCount,
+		                       pauiIntUpdateUFOAddress,
+		                       paui32IntUpdateValue,
+		                       paui32FWCommandSize[i],
+		                       papaui8FWCommand[i],
+							   &pPreAddr,
+							   &pPostAddr,
+							   &pRMWUFOAddr,
+		                       eType,
+		                       ui32ExtJobRef,
+		                       ui32IntJobRef,
+		                       ui32PDumpFlags,
+		                       NULL,
+		                       pszCommandName,
+		                       bCCBStateOpen,
+		                       psCmdHelper);
+
+	}
+
+	/*
+		Acquire space for all the commands in one go
+	*/
+	if (ui323DCmdCount)
+	{
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount,
+										   &pas3DCmdHelper[0]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_cmdacquire;
+		}
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = RGXCmdHelperAcquireCmdCCB(ui322DCmdCount,
+										   &pas2DCmdHelper[0]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_cmdacquire;
+		}
+	}
+#endif
+
+	/*
+		We should acquire the kernel CCB(s) space here as the schedule could fail
+		and we would have to roll back all the syncs
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  &pas3DCmdHelper[0],
+								  "TQ_3D",
+								  FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext).ui32Addr);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		ui322DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->s2DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui322DCmdCount,
+								  &pas2DCmdHelper[0],
+								  "TQ_2D",
+								  FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext).ui32Addr);
+	}
+#endif
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd;
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *ps3DTQCCB = FWCommonContextGetClientCCB(psTransferContext->s3DData.psServerCommonContext);
+
+		/* Take one of the helper data structs and extract the common cmd struct,
+		 * this is used to obtain the frame num. Each command should share the same
+		 * frame number so we can just get the first.
+		 */
+#if defined(PVRSRV_ENABLE_HTB)
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper = &pas3DCmdHelper[0];
+		CMD_COMMON *psTransferCmdCmn = IMG_OFFSET_ADDR(psCmdHelper->pui8DMCmd, 0);
+#endif
+
+		/* Construct the kernel 3D CCB command. */
+		s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		s3DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(ps3DTQCCB);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(ps3DTQCCB);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+		HTBLOGK(HTB_SF_MAIN_KICK_3D,
+				s3DKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui323DCmdOffset,
+				psTransferCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef
+				);
+#endif
+
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQ3D,
+		                  iCheckFence,
+		                  i3DUpdateFence,
+		                  i3DUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  ui3DUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psDevInfo,
+										RGXFWIF_DM_3D,
+										&s3DKCCBCmd,
+										ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			eError = eError2;
+			goto fail_cmdsubmit;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWCtx, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQ3D);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		RGXFWIF_KCCB_CMD s2DKCCBCmd;
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *ps2DTQCCB = FWCommonContextGetClientCCB(psTransferContext->s2DData.psServerCommonContext);
+
+		/* Take one of the helper data structs and extract the common cmd struct,
+		 * this is used to obtain the frame num. Each command should share the same
+		 * frame number so we can just get the first.
+		 */
+#if defined(PVRSRV_ENABLE_HTB)
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper = &pas2DCmdHelper[0];
+		CMD_COMMON *psTransferCmdCmn = IMG_OFFSET_ADDR(psCmdHelper->pui8DMCmd, 0);
+#endif
+
+		/* Construct the kernel 2D CCB command. */
+		s2DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		s2DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(ps2DTQCCB);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(ps2DTQCCB);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(PVRSRV_ENABLE_HTB)
+		HTBLOGK(HTB_SF_MAIN_KICK_2D,
+				s2DKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui322DCmdOffset,
+				psTransferCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef);
+#endif
+
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQ2D,
+		                  iCheckFence,
+		                  i2DUpdateFence,
+		                  i2DUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  ui2DUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psDevInfo,
+										RGXFWIF_DM_2D,
+										&s2DKCCBCmd,
+										ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			eError = eError2;
+			goto fail_cmdsubmit;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWCtx, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQ2D);
+	}
+#endif
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdsubmit;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (ps2DUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling TLA NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)ps2DUpdateSyncCheckpoint, SyncCheckpointGetId(ps2DUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(ps2DUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(ps2DUpdateSyncCheckpoint);
+	}
+	if (ps2DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating TLA NOHW sync prim<%p> to %d", __func__, (void*)ps2DFenceTimelineUpdateSync, ui322DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps2DFenceTimelineUpdateSync, ui322DFenceTimelineUpdateValue);
+	}
+#endif
+	if (ps3DUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling TQ3D NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)ps3DUpdateSyncCheckpoint, SyncCheckpointGetId(ps3DUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(ps3DUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(ps3DUpdateSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating TQ3D NOHW sync prim<%p> to %d", __func__, (void*)ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+	if (ppsExportFenceSyncCheckpoint && *ppsExportFenceSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)*ppsExportFenceSyncCheckpoint, SyncCheckpointGetId(*ppsExportFenceSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(*ppsExportFenceSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(*ppsExportFenceSyncCheckpoint);
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal=%d)", __func__, iExportFenceToSignal));
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pi2DUpdateFence)
+	{
+		*pi2DUpdateFence = i2DUpdateFence;
+	}
+#endif
+	if (pi3DUpdateFence)
+	{
+		*pi3DUpdateFence = i3DUpdateFence;
+	}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pv2DUpdateFenceFinaliseData && (i2DUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, i2DUpdateFence, pv2DUpdateFenceFinaliseData,
+		                            ps2DUpdateSyncCheckpoint, szFenceName);
+	}
+#endif
+	if (pv3DUpdateFenceFinaliseData && (i3DUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, i3DUpdateFence, pv3DUpdateFenceFinaliseData,
+		                            ps3DUpdateSyncCheckpoint, szFenceName);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	OSFreeMem(pas2DCmdHelper);
+#endif
+	OSFreeMem(pas3DCmdHelper);
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pui322DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui322DIntAllocatedUpdateValues);
+		pui322DIntAllocatedUpdateValues = NULL;
+	}
+#endif
+	if (pui323DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui323DIntAllocatedUpdateValues);
+		pui323DIntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+/*
+	No resources are created in this function so there is nothing to free
+	unless we had to merge syncs.
+	If we fail after the client CCB acquire there is still nothing to do
+	as only the client CCB release will modify the client CCB
+*/
+fail_cmdsubmit:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_cmdacquire:
+fail_prepare_loop:
+
+	for (i=0;i<ui32PreparesDone;i++)
+	{
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListFence[i]);
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListUpdate[i]);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32PreparesDone > 0)
+	{
+		/* Prevent duplicate rollback in case of buffer sync. */
+		psBufferUpdateSyncCheckpoint = NULL;
+	}
+#endif
+
+	/* Free memory allocated to hold the internal list of update values */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pui322DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui322DIntAllocatedUpdateValues);
+		pui322DIntAllocatedUpdateValues = NULL;
+	}
+#endif
+	if (pui323DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui323DIntAllocatedUpdateValues);
+		pui323DIntAllocatedUpdateValues = NULL;
+	}
+	if (ppsExportFenceSyncCheckpoint && *ppsExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (i2DUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(i2DUpdateFence, pv2DUpdateFenceFinaliseData);
+	}
+#endif
+	if (i3DUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(i3DUpdateFence, pv3DUpdateFenceFinaliseData);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferUpdateSyncCheckpoint)
+	{
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListUpdate[0]);
+	}
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+fail_resolve_buffersync_input_fence:
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+fail_resolve_fencesync_input_fence:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	OSFreeMem(pas2DCmdHelper);
+fail_alloc2dhelper:
+#endif
+	OSFreeMem(pas3DCmdHelper);
+fail_alloc3dhelper:
+
+	OSLockRelease(psTransferContext->hLock);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDevNode,
+                                                   RGX_SERVER_TQ_CONTEXT *psTransferContext,
+                                                   IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+#endif
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	OSLockAcquire(psTransferContext->hLock);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((psTransferContext->s2DData.i32Priority != i32Priority) &&
+			(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = ContextSetPriority(psTransferContext->s2DData.psServerCommonContext,
+									psConnection,
+									psTransferContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_2D);
+		if (eError != PVRSRV_OK)
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 2D part of the transfercontext (%s)", __func__, PVRSRVGetErrorString(eError)));
+			}
+			goto fail_2dcontext;
+		}
+		psTransferContext->s2DData.i32Priority = i32Priority;
+	}
+#endif
+
+	if (psTransferContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psTransferContext->s3DData.psServerCommonContext,
+									psConnection,
+									psTransferContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 3D part of the transfercontext (%s)", __func__, PVRSRVGetErrorString(eError)));
+			}
+			goto fail_3dcontext;
+		}
+		psTransferContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+
+fail_2dcontext:
+#endif
+	OSLockRelease(psTransferContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPropertyKM(RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   RGX_CONTEXT_PROPERTY eContextProperty,
+												   IMG_UINT64 ui64Input,
+												   IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psTransferContext->hLock);
+			eError = FWCommonContextSetFlags(psTransferContext->s2DData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psTransferContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psTransferContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hTransferCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTransferCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_CONTEXT, sListNode);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			DumpFWCommonContextInfo(psCurrentServerTransferCtx->s2DData.psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+#endif
+
+		if (psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D)
+		{
+			DumpFWCommonContextInfo(psCurrentServerTransferCtx->s3DData.psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTransferCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hTransferCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTransferCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_CONTEXT, sListNode);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+				(NULL != psCurrentServerTransferCtx->s2DData.psServerCommonContext) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerTransferCtx->s2DData.psServerCommonContext, RGX_KICK_TYPE_DM_TQ2D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TQ2D;
+			}
+		}
+#endif
+
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D) && (NULL != psCurrentServerTransferCtx->s3DData.psServerCommonContext))
+		{
+			if ((CheckStalledClientCommonContext(psCurrentServerTransferCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_TQ3D) == PVRSRV_ERROR_CCCB_STALLED))
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TQ3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTransferCtxListLock);
+	return ui32ContextBitMask;
+}
+
+PVRSRV_ERROR PVRSRVRGXTQGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTQReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************//**
+ End of file (rgxtransfer.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.h
new file mode 100644
index 000000000000..3cdc454d3de0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.h
@@ -0,0 +1,179 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Transfer queue Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Transfer queue Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTRANSFER_H)
+#define RGXTRANSFER_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "connection_server.h"
+
+typedef struct _RGX_SERVER_TQ_CONTEXT_ RGX_SERVER_TQ_CONTEXT;
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateTransferContextKM
+
+ @Description
+	Server-side implementation of RGXCreateTransferContext
+
+ @Input pvDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateTransferContextKM(CONNECTION_DATA		*psConnection,
+										   PVRSRV_DEVICE_NODE		*psDeviceNode,
+										   IMG_INT32				i32Priority,
+										   IMG_UINT32				ui32FrameworkCommandSize,
+										   IMG_PBYTE				pabyFrameworkCommand,
+										   IMG_HANDLE				hMemCtxPrivData,
+										   IMG_UINT32				ui32PackedCCBSizeU8888,
+										   IMG_UINT32				ui32ContextFlags,
+										   IMG_UINT64				ui64RobustnessAddress,
+										   RGX_SERVER_TQ_CONTEXT	**ppsTransferContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyTransferContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyTransferContext
+
+ @Input psTransferContext - Transfer context
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyTransferContextKM(RGX_SERVER_TQ_CONTEXT *psTransferContext);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVSubmitTransferKM
+
+ @Description
+	Schedules one or more 2D or 3D HW commands on the firmware
+
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSubmitTransferKM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									IMG_UINT32				ui32PrepareCount,
+									IMG_UINT32				*paui32ClientUpdateCount,
+									SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									IMG_UINT32				**papaui32ClientUpdateValue,
+									PVRSRV_FENCE			iCheckFence,
+									PVRSRV_TIMELINE			i2DUpdateTimeline,
+									PVRSRV_FENCE			*pi2DUpdateFence,
+									PVRSRV_TIMELINE			i3DUpdateTimeline,
+									PVRSRV_FENCE			*pi3DUpdateFence,
+									IMG_CHAR				szFenceName[32],
+									IMG_UINT32				*paui32FWCommandSize,
+									IMG_UINT8				**papaui8FWCommand,
+									IMG_UINT32				*pui32TQPrepareFlags,
+									IMG_UINT32				ui32ExtJobRef,
+									IMG_UINT32				ui32SyncPMRCount,
+									IMG_UINT32				*paui32SyncPMRFlags,
+									PMR						**ppsSyncPMRs);
+
+PVRSRV_ERROR PVRSRVRGXSubmitTransfer3KM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									IMG_UINT32				ui32PrepareCount,
+									IMG_UINT32				*paui32ClientUpdateCount,
+									SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									IMG_UINT32				**papaui32ClientUpdateValue,
+									PVRSRV_FENCE			iCheckFence,
+									PVRSRV_TIMELINE			i2DUpdateTimeline,
+									PVRSRV_FENCE			*pi2DUpdateFence,
+									PVRSRV_TIMELINE			i3DUpdateTimeline,
+									PVRSRV_FENCE			*pi3DUpdateFence,
+									IMG_CHAR				szFenceName[32],
+									PVRSRV_FENCE			iExportFenceToSignal,
+									IMG_UINT32				*paui32FWCommandSize,
+									IMG_UINT8				**papaui8FWCommand,
+									IMG_UINT32				*pui32TQPrepareFlags,
+									IMG_UINT32				ui32ExtJobRef,
+									IMG_UINT32				ui32SyncPMRCount,
+									IMG_UINT32				*paui32SyncPMRFlags,
+									PMR						**ppsSyncPMRs);
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDevNode,
+												   RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPropertyKM(RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   RGX_CONTEXT_PROPERTY eContextProperty,
+												   IMG_UINT64 ui64Input,
+												   IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of transfer contexts on this device */
+void DumpTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client transfer contexts are stalled */
+IMG_UINT32 CheckForStalledClientTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR PVRSRVRGXTQGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+PVRSRV_ERROR PVRSRVRGXTQReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+#endif /* RGXTRANSFER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdebug.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdebug.c
new file mode 100644
index 000000000000..61e808bb976b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdebug.c
@@ -0,0 +1,1798 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "cache_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "rgxutils.h"
+#include "tlstream.h"
+#include "rgxfwriscv.h"
+#include "pvrsrv.h"
+#include "services_km.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_utils.h"
+#include "rgx_fwif_km.h"
+#include "rgx_fwif_sf.h"
+#include "debug_common.h"
+
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxcompute.h"
+#include "rgxtdmtransfer.h"
+#include "rgxtimecorr.h"
+#include "rgx_options.h"
+#include "rgxinit.h"
+#include "rgxlayer_impl.h"
+#include "devicemem_history_server.h"
+
+#define DD_SUMMARY_INDENT  ""
+
+#define RGX_DEBUG_STR_SIZE			(150U)
+
+
+#define RGX_TEXAS_BIF0_ID				(0)
+#define RGX_TEXAS_BIF1_ID				(1)
+
+/*
+ *  The first 7 or 8 cat bases are memory contexts used for PM
+ *  or firmware. The rest are application contexts. The numbering
+ *  is zero-based.
+ */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define MAX_RESERVED_FW_MMU_CONTEXT		(7)
+#else
+#define MAX_RESERVED_FW_MMU_CONTEXT		(6)
+#endif
+
+static const IMG_CHAR *const pszPowStateName[] =
+{
+#define X(NAME)	#NAME,
+	RGXFWIF_POW_STATES
+#undef X
+};
+
+static const IMG_FLAGS2DESC asHwrState2Description[] =
+{
+	{RGXFWIF_HWR_HARDWARE_OK, " HWR OK;"},
+	{RGXFWIF_HWR_RESET_IN_PROGRESS, " Reset ongoing;"},
+	{RGXFWIF_HWR_GENERAL_LOCKUP, " General lockup;"},
+	{RGXFWIF_HWR_DM_RUNNING_OK, " DM running ok;"},
+	{RGXFWIF_HWR_DM_STALLING, " DM stalling;"},
+	{RGXFWIF_HWR_FW_FAULT, " FW fault;"},
+	{RGXFWIF_HWR_RESTART_REQUESTED, " Restart requested;"},
+};
+
+static const IMG_FLAGS2DESC asDmState2Description[] =
+{
+	{RGXFWIF_DM_STATE_READY_FOR_HWR, " ready for hwr;"},
+	{RGXFWIF_DM_STATE_NEEDS_SKIP, " needs skip;"},
+	{RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP, " needs PR cleanup;"},
+	{RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR, " needs trace clear;"},
+	{RGXFWIF_DM_STATE_GUILTY_LOCKUP, " guilty lockup;"},
+	{RGXFWIF_DM_STATE_INNOCENT_LOCKUP, " innocent lockup;"},
+	{RGXFWIF_DM_STATE_GUILTY_OVERRUNING, " guilty overrunning;"},
+	{RGXFWIF_DM_STATE_INNOCENT_OVERRUNING, " innocent overrunning;"},
+	{RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH, " hard context switching;"},
+	{RGXFWIF_DM_STATE_GPU_ECC_HWR, " GPU ECC hwr;"},
+	{RGXFWIF_DM_STATE_GPU_PARITY_HWR, " GPU PARITY hwr;"},
+	{RGXFWIF_DM_STATE_GPU_LATENT_HWR, " GPU LATENT hwr;"},
+	{RGXFWIF_DM_STATE_ICS_HWR, " ICS hwr;"},
+};
+
+static const IMG_CHAR * const apszFwOsStateName[RGXFW_CONNECTION_FW_STATE_COUNT] =
+{
+	"offline",
+	"ready",
+	"active",
+	"offloading",
+	"cooldown"
+};
+
+#if defined(PVR_ENABLE_PHR)
+static const IMG_FLAGS2DESC asPHRConfig2Description[] =
+{
+	{BIT_ULL(RGXFWIF_PHR_MODE_OFF), "off"},
+	{BIT_ULL(RGXFWIF_PHR_MODE_RD_RESET), "reset RD hardware"},
+};
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMULevel
+
+ @Description
+
+ Return the name for the MMU level that faulted.
+
+ @Input ui32MMULevel	 - MMU level
+
+ @Return   IMG_CHAR* to the string describing the MMU level that faulted.
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodeMMULevel(IMG_UINT32 ui32MMULevel)
+{
+	const IMG_CHAR* pszMMULevel = "";
+
+	switch (ui32MMULevel)
+	{
+		case 0x0: pszMMULevel = " (Page Table)"; break;
+		case 0x1: pszMMULevel = " (Page Directory)"; break;
+		case 0x2: pszMMULevel = " (Page Catalog)"; break;
+		case 0x3: pszMMULevel = " (Cat Base Reg)"; break;
+	}
+
+	return pszMMULevel;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMUReqTags
+
+ @Description
+
+ Decodes the MMU Tag ID and Sideband data fields from RGX_CR_MMU_FAULT_META_STATUS and
+ RGX_CR_MMU_FAULT_STATUS regs.
+
+ @Input psDevInfo           - RGX device info
+ @Input ui32Requester       - Requester ID value
+ @Input ui32BIFModule       - BIF module
+ @Input bWriteBack          - Write Back flag
+ @Input bFBMFault           - FBM Fault flag
+ @Output ppszRequester      - Decoded string from the Requester ID
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDecodeMMUReqTags(PVRSRV_RGXDEV_INFO  *psDevInfo,
+								 IMG_UINT32          ui32Requester,
+								 IMG_UINT32          ui32BIFModule,
+								 IMG_BOOL            bWriteBack,
+								 IMG_BOOL            bFBMFault,
+								 IMG_CHAR            **ppszRequester)
+{
+	IMG_UINT32 ui32BIFsPerSPU = 2;
+	IMG_CHAR   *pszRequester  = "-";
+
+	PVR_ASSERT(ppszRequester != NULL);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ALBIORIX_TOP_INFRASTRUCTURE))
+	{
+		ui32BIFsPerSPU = 4;
+	}
+
+	if (bFBMFault)
+	{
+		if (bWriteBack)
+		{
+			pszRequester = "FBM (Header/state cache request)";
+		}
+		else
+		{
+			pszRequester = "FBM";
+		}
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_SPU) &&
+	         ui32BIFModule <  RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU)*ui32BIFsPerSPU)
+	{
+		IMG_BOOL  bTexasBIFA = ((ui32BIFModule % 2) == 0);
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ALBIORIX_TOP_INFRASTRUCTURE))
+		{
+			switch (ui32Requester)
+			{
+				case  0: pszRequester = "MCU L1 (IP0 PDS)"; break;
+				case  1: pszRequester = "MCU L1 (IP0 Global)"; break;
+				case  2: pszRequester = "MCU L1 (IP1 PDS)"; break;
+				case  3: pszRequester = "MCU L1 (IP1 Global)"; break;
+				case  4: pszRequester = "MCU L1 (IP2 PDS)"; break;
+				case  5: pszRequester = "MCU L1 (IP2 Global)"; break;
+				case  6: pszRequester = (bTexasBIFA ? "TCU L1" : "BSC"); break;
+				case  7: pszRequester = (bTexasBIFA ? "PBE0" : "BSC"); break;
+				case  8: pszRequester = "PBE0"; break;
+				case  9: pszRequester = "IPF"; break;
+				case 10: pszRequester = "IPF"; break;
+				case 11: pszRequester = "IPF"; break;
+				case 12: pszRequester = (bTexasBIFA ? "IPF" : "TCU L1"); break;
+				case 13: pszRequester = (bTexasBIFA ? "IPF" : "TPF"); break;
+				case 14: pszRequester = (bTexasBIFA ? "IPF" : "TPF CPF"); break;
+				case 15: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 16: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 17: pszRequester = (bTexasBIFA ? "IPF" : "PDSRW cache"); break;
+				case 18: pszRequester = (bTexasBIFA ? "IPF" : "PDS"); break;
+				case 19: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 20: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 21: pszRequester = (bTexasBIFA ? "IPF" : "USC L2"); break;
+				case 22: pszRequester = (bTexasBIFA ? "IPF" : "URI"); break;
+				case 23: pszRequester = "IPF"; break;
+				case 24: pszRequester = "IPF"; break;
+				case 25: pszRequester = "IPF"; break;
+				case 26: pszRequester = "IPF"; break;
+				case 27: pszRequester = "IPF"; break;
+				case 28: pszRequester = "IPF"; break;
+				case 29: pszRequester = "IPF"; break;
+				case 30: pszRequester = "IPF"; break;
+				case 31: pszRequester = "IPF"; break;
+				case 32: pszRequester = "IPF"; break;
+				case 33: pszRequester = "IPF_CPF"; break;
+				case 34: pszRequester = "ISP0"; break;
+				case 35: pszRequester = "ISP0"; break;
+				case 36: pszRequester = "ISP2"; break;
+				case 37: pszRequester = "ISP2"; break;
+				case 38: pszRequester = "GEOM"; break;
+				case 39: pszRequester = "GEOM"; break;
+				case 40: pszRequester = "GEOM"; break;
+				case 41: pszRequester = "GEOM"; break;
+				case 42: pszRequester = "BSC"; break;
+				case 43: pszRequester = "ASC"; break;
+
+				default:
+					PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified Texas BIF Requester ID: %d", __func__, ui32Requester));
+				break;
+			}
+		}
+		else
+		{
+			switch (ui32Requester)
+			{
+				case  0: pszRequester = "MCU L1 (IP0 PDS)"; break;
+				case  1: pszRequester = "MCU L1 (IP0 Global)"; break;
+				case  2: pszRequester = "MCU L1 (IP0 BSC)"; break;
+				case  3: pszRequester = "MCU L1 (IP0 Constants)"; break;
+				case  4: pszRequester = "MCU L1 (IP1 PDS)"; break;
+				case  5: pszRequester = "MCU L1 (IP1 Global)"; break;
+				case  6: pszRequester = "MCU L1 (IP1 BSC)"; break;
+				case  7: pszRequester = "MCU L1 (IP1 Constants)"; break;
+				case  8: pszRequester = "MCU L1 (IP2 PDS)"; break;
+				case  9: pszRequester = "MCU L1 (IP2 Global)"; break;
+				case 10: pszRequester = "MCU L1 (IP2 BSC)"; break;
+				case 11: pszRequester = "MCU L1 (IP2 Constants)"; break;
+				case 12: pszRequester = "TCU L1"; break;
+				case 13: pszRequester = (bTexasBIFA ? "PBE0" : "TPF"); break;
+				case 14: pszRequester = (bTexasBIFA ? "PBE0" : "TPF CPF"); break;
+				case 15: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 16: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 17: pszRequester = (bTexasBIFA ? "IPF" : "PDSRW cache"); break;
+				case 18: pszRequester = (bTexasBIFA ? "IPF" : "PDS"); break;
+				case 19: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 20: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 21: pszRequester = (bTexasBIFA ? "IPF" : "USC L2"); break;
+				case 22: pszRequester = (bTexasBIFA ? "IPF" : "VDM L2"); break;
+				case 23: pszRequester = (bTexasBIFA ? "IPF" : "RTU FBA L2"); break;
+				case 24: pszRequester = (bTexasBIFA ? "IPF" : "RTU SHR L2"); break;
+				case 25: pszRequester = (bTexasBIFA ? "IPF" : "RTU SHG L2"); break;
+				case 26: pszRequester = (bTexasBIFA ? "IPF" : "RTU TUL L2"); break;
+				case 27: pszRequester = "IPF"; break;
+				case 28: pszRequester = "IPF"; break;
+				case 29: pszRequester = "IPF"; break;
+				case 30: pszRequester = "IPF"; break;
+				case 31: pszRequester = "IPF"; break;
+				case 32: pszRequester = "IPF"; break;
+				case 33: pszRequester = "IPF"; break;
+				case 34: pszRequester = "IPF_CPF"; break;
+				case 35: pszRequester = "PPP"; break;
+				case 36: pszRequester = "ISP0"; break;
+				case 37: pszRequester = "ISP0"; break;
+				case 38: pszRequester = "ISP2"; break;
+				case 39: pszRequester = "ISP2"; break;
+				case 40: pszRequester = "VCE RTC"; break;
+				case 41: pszRequester = "RTU RAC"; break;
+				case 42: pszRequester = "RTU RAC"; break;
+				case 43: pszRequester = "RTU RAC"; break;
+				case 44: pszRequester = "RTU RAC"; break;
+				case 45: pszRequester = "RTU RAC"; break;
+				case 46: pszRequester = "RTU RAC"; break;
+				case 47: pszRequester = "RTU RAC"; break;
+				case 48: pszRequester = "RTU RAC"; break;
+				case 49: pszRequester = "VCE AMC"; break;
+				case 50: pszRequester = "SHF"; break;
+				case 51: pszRequester = "SHF"; break;
+
+				default:
+					PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified Texas BIF Requester ID: %d", __func__, ui32Requester));
+				break;
+			}
+		}
+	}
+	else if (ui32BIFModule == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU)*ui32BIFsPerSPU)
+	{
+		/* Jones BIF */
+		switch (ui32Requester)
+		{
+			case  0: pszRequester = "IPP"; break;
+			case  1: pszRequester = "DCE"; break;
+			case  2: pszRequester = "DCE"; break;
+			case  3: pszRequester = "DCE"; break;
+			case  4: pszRequester = "DCE"; break;
+			case  5: pszRequester = "DCE"; break;
+			case  6: pszRequester = "DCE"; break;
+			case  7: pszRequester = "DCE"; break;
+			case  8: pszRequester = "DCE"; break;
+			case  9: pszRequester = "DCE"; break;
+			case 10: pszRequester = "DCE"; break;
+			case 11: pszRequester = "DCE"; break;
+			case 12: pszRequester = "DCE"; break;
+			case 13: pszRequester = "DCE"; break;
+			case 14: pszRequester = "DCE"; break;
+			case 15: pszRequester = "TDM"; break;
+			case 16: pszRequester = "TDM"; break;
+			case 17: pszRequester = "TDM"; break;
+			case 18: pszRequester = "TDM"; break;
+			case 19: pszRequester = "TDM"; break;
+			case 20: pszRequester = "PM"; break;
+			case 21: pszRequester = "CDM"; break;
+			case 22: pszRequester = "CDM"; break;
+			case 23: pszRequester = "CDM"; break;
+			case 24: pszRequester = "CDM"; break;
+			case 25: pszRequester = "CDM"; break;
+			case 26: pszRequester = "CDM"; break;
+			case 27: pszRequester = "CDM"; break;
+			case 28: pszRequester = "CDM"; break;
+			case 29: pszRequester = "CDM"; break;
+			case 30: pszRequester = "CDM"; break;
+			case 31: pszRequester = "CDM"; break;
+			case 32: pszRequester = "META"; break;
+			case 33: pszRequester = "META DMA"; break;
+			case 34: pszRequester = "TE"; break;
+			case 35: pszRequester = "TE"; break;
+			case 36: pszRequester = "TE"; break;
+			case 37: pszRequester = "TE"; break;
+			case 38: pszRequester = "TE"; break;
+			case 39: pszRequester = "TE"; break;
+			case 40: pszRequester = "TE"; break;
+			case 41: pszRequester = "TE"; break;
+			case 42: pszRequester = "TE"; break;
+			case 43: pszRequester = "TE"; break;
+			case 44: pszRequester = "TE"; break;
+			case 45: pszRequester = "TE"; break;
+			case 46: pszRequester = "TE"; break;
+			case 47: pszRequester = "TE"; break;
+			case 48: pszRequester = "TE"; break;
+			case 49: pszRequester = "TE"; break;
+			case 50: pszRequester = "RCE"; break;
+			case 51: pszRequester = "RCE"; break;
+			case 52: pszRequester = "RCE"; break;
+
+			default:
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified Jones BIF Tag ID: %d", __func__, ui32Requester));
+			break;
+		}
+	}
+	else if (bWriteBack)
+	{
+		pszRequester = "Unknown (Writeback of dirty cacheline)";
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified BIF Module: %d", __func__, ui32BIFModule));
+	}
+
+	*ppszRequester = pszRequester;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpRGXMMUFaultStatus
+
+ @Description
+
+ Dump MMU Fault status in human readable form.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo            - RGX device info
+ @Input ui64MMUStatus        - MMU Status register value
+ @Input pszMetaOrCore        - string representing call is for META or MMU core
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpRGXMMUFaultStatus(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					const IMG_UINT64 aui64MMUStatus[],
+					const IMG_PCHAR pszMetaOrCore,
+					const IMG_CHAR *pszIndent)
+{
+	if (aui64MMUStatus[0] == 0x0)
+	{
+		PVR_DUMPDEBUG_LOG("%sMMU (%s) - OK", pszIndent, pszMetaOrCore);
+	}
+	else
+	{
+		IMG_UINT32 ui32PC        = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT;
+		IMG_UINT64 ui64Addr      = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT) <<  4; /* align shift */
+		IMG_UINT32 ui32Requester = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_REQ_ID_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_REQ_ID_SHIFT;
+		IMG_UINT32 ui32MMULevel  = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_LEVEL_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_LEVEL_SHIFT;
+		IMG_BOOL bRead           = (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS1_RNW_EN) != 0;
+		IMG_BOOL bFault          = (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS1_FAULT_EN) != 0;
+		IMG_BOOL bROFault        = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK) >>
+		                            RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT) == 0x2;
+		IMG_BOOL bProtFault      = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK) >>
+		                            RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT) == 0x3;
+		IMG_UINT32 ui32BIFModule;
+		IMG_BOOL bWriteBack, bFBMFault;
+		IMG_CHAR *pszRequester = NULL;
+		const IMG_PCHAR pszMetaOrRiscv = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? "META" : "RISCV";
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ALBIORIX_TOP_INFRASTRUCTURE))
+		{
+			ui32BIFModule = (aui64MMUStatus[1] & ~RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_CLRMSK) >>
+										RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_SHIFT;
+			bWriteBack    = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_EN) != 0;
+			bFBMFault     = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_EN) != 0;
+		}
+		else
+		{
+			ui32BIFModule = (aui64MMUStatus[1] & ~RGX_CR_MMU_FAULT_STATUS2_BIF_ID_CLRMSK) >>
+										RGX_CR_MMU_FAULT_STATUS2_BIF_ID_SHIFT;
+			bWriteBack    = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_EN) != 0;
+			bFBMFault     = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_EN) != 0;
+		}
+
+		if (strcmp(pszMetaOrCore, "Core") != 0)
+		{
+			ui32PC		= (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT;
+			ui64Addr	= ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT) <<  4; /* align shift */
+			ui32Requester = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT;
+			ui32MMULevel  = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT;
+			bRead		= (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS_META_RNW_EN) != 0;
+			bFault      = (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN) != 0;
+			bROFault    = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT) == 0x2;
+			bProtFault  = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT) == 0x3;
+		}
+		else
+		{
+			_RGXDecodeMMUReqTags(psDevInfo, ui32Requester, ui32BIFModule, bWriteBack, bFBMFault, &pszRequester);
+		}
+
+		PVR_DUMPDEBUG_LOG("%sMMU (%s) - FAULT:", pszIndent, pszMetaOrCore);
+		PVR_DUMPDEBUG_LOG("%s  * MMU status (0x%016" IMG_UINT64_FMTSPECX " | 0x%08" IMG_UINT64_FMTSPECX "): PC = %d, %s 0x%010" IMG_UINT64_FMTSPECX ", %s%s%s%s%s.",
+						  pszIndent,
+						  aui64MMUStatus[0],
+						  aui64MMUStatus[1],
+						  ui32PC,
+						  (bRead)?"Reading from":"Writing to",
+						  ui64Addr,
+						  (pszRequester) ? pszRequester : pszMetaOrRiscv,
+						  (bFault)?", Fault":"",
+						  (bROFault)?", Read Only fault":"",
+						  (bProtFault)?", PM/FW core protection fault":"",
+						  _RGXDecodeMMULevel(ui32MMULevel));
+
+	}
+}
+
+static_assert((RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_REQ_ID_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_REQ_ID_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_LEVEL_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_LEVEL_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_RNW_EN == RGX_CR_MMU_FAULT_STATUS_META_RNW_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_FAULT_EN == RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+
+
+#if !defined(NO_HARDWARE)
+static inline IMG_CHAR const *_GetRISCVException(IMG_UINT32 ui32Mcause)
+{
+	switch (ui32Mcause)
+	{
+#define X(value, fatal, description) \
+		case value: \
+			if (fatal) \
+				return description; \
+			return NULL;
+
+		RGXRISCVFW_MCAUSE_TABLE
+#undef X
+
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "Invalid RISC-V FW mcause value 0x%08x", ui32Mcause));
+			return NULL;
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+static const IMG_FLAGS2DESC asHWErrorState[] =
+{
+	{RGX_HW_ERR_NA, "N/A"},
+	{RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL, "Primitive ID failure during DM kill."},
+};
+
+/*
+ *  Translate ID code to descriptive string.
+ *  Returns on the first match.
+ */
+static void _ID2Description(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, const IMG_FLAGS2DESC *psConvTable, IMG_UINT32 ui32TableSize, IMG_UINT32 ui32ID)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (ui32ID == psConvTable[ui32Idx].uiFlag)
+		{
+			OSStringSafeCopy(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+			return;
+		}
+	}
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWAssert
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psRGXFWIfTraceBufCtl - RGX FW trace buffer
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWAssert(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl)
+{
+	const IMG_CHAR *pszTraceAssertPath;
+	const IMG_CHAR *pszTraceAssertInfo;
+	IMG_INT32 ui32TraceAssertLine;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf, INVALIDATE);
+		pszTraceAssertPath = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szPath;
+		pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szInfo;
+		ui32TraceAssertLine = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.ui32LineNum;
+
+		/* print non-null assert strings */
+		if (*pszTraceAssertInfo)
+		{
+			PVR_DUMPDEBUG_LOG("FW-T%d Assert: %.*s (%.*s:%d)",
+			                  i, RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertPath, ui32TraceAssertLine);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWFaults
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psFwSysData       - RGX FW shared system data
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWFaults(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             const RGXFWIF_SYSDATA *psFwSysData)
+{
+	if (psFwSysData->ui32FWFaults > 0)
+	{
+		IMG_UINT32	ui32StartFault = psFwSysData->ui32FWFaults - RGXFWIF_FWFAULTINFO_MAX;
+		IMG_UINT32	ui32EndFault   = psFwSysData->ui32FWFaults - 1;
+		IMG_UINT32  ui32Index;
+
+		if (psFwSysData->ui32FWFaults < RGXFWIF_FWFAULTINFO_MAX)
+		{
+			ui32StartFault = 0;
+		}
+
+		for (ui32Index = ui32StartFault; ui32Index <= ui32EndFault; ui32Index++)
+		{
+			const RGX_FWFAULTINFO *psFaultInfo = &psFwSysData->sFaultInfo[ui32Index % RGXFWIF_FWFAULTINFO_MAX];
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			/* Split OS timestamp in seconds and nanoseconds */
+			RGXConvertOSTimestampToSAndNS(psFaultInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+			PVR_DUMPDEBUG_LOG("FW Fault %d: %.*s (%.*s:%d)",
+			                  ui32Index+1, RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szPath,
+			                  psFaultInfo->sFaultBuf.ui32LineNum);
+			PVR_DUMPDEBUG_LOG("            Data = 0x%016"IMG_UINT64_FMTSPECx", CRTimer = 0x%012"IMG_UINT64_FMTSPECx", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+			                  psFaultInfo->ui64Data,
+			                  psFaultInfo->ui64CRTimer,
+			                  ui64Seconds, ui64Nanoseconds);
+		}
+	}
+}
+
+static void _RGXDumpFWPoll(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_SYSDATA *psFwSysData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psFwSysData->aui32CrPollAddr[i])
+		{
+			PVR_DUMPDEBUG_LOG("T%u polling %s (reg:0x%08X mask:0x%08X)",
+			                  i,
+			                  ((psFwSysData->aui32CrPollAddr[i] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+			                  psFwSysData->aui32CrPollAddr[i] & ~RGXFW_POLL_TYPE_SET,
+			                  psFwSysData->aui32CrPollMask[i]);
+		}
+	}
+
+}
+
+static void _RGXDumpFWHWRInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							  void *pvDumpDebugFile,
+							  const RGXFWIF_SYSDATA *psFwSysData,
+							  const RGXFWIF_HWRINFOBUF *psHWRInfoBuf,
+							  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL          bAnyLocked = IMG_FALSE;
+	IMG_UINT32        dm, i;
+	IMG_UINT32        ui32LineSize;
+	IMG_CHAR          *pszLine, *pszTemp;
+	const IMG_CHAR    *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+	const IMG_CHAR    szMsgHeader[] = "Number of HWR: ";
+	const IMG_CHAR    szMsgFalse[] = "FALSE(";
+	IMG_CHAR          *pszLockupType = "";
+	const IMG_UINT32  ui32MsgHeaderCharCount = ARRAY_SIZE(szMsgHeader) - 1; /* size includes the null */
+	const IMG_UINT32  ui32MsgFalseCharCount = ARRAY_SIZE(szMsgFalse) - 1;
+	IMG_UINT32        ui32HWRRecoveryFlags;
+	IMG_UINT32        ui32ReadIndex;
+
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+	{
+		apszDmNames[RGXFWIF_DM_TDM] = "2D";
+	}
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] ||
+		    psHWRInfoBuf->aui32HwrDmOverranCount[dm])
+		{
+			bAnyLocked = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) && !bAnyLocked && (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_HARDWARE_OK))
+	{
+		/* No HWR situation, print nothing */
+		return;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_BOOL bAnyHWROccurred = IMG_FALSE;
+
+		for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+		{
+			if (psHWRInfoBuf->aui32HwrDmRecoveredCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm] !=0)
+				{
+					bAnyHWROccurred = IMG_TRUE;
+					break;
+				}
+		}
+
+		if (!bAnyHWROccurred)
+		{
+			return;
+		}
+	}
+
+/* <DM name + left parenthesis> + <UINT32 max num of digits> + <slash> + <UINT32 max num of digits> +
+   <plus> + <UINT32 max num of digits> + <right parenthesis + comma + space> */
+#define FWHWRINFO_DM_STR_SIZE (5U + 10U + 1U + 10U + 1U + 10U + 3U)
+
+	ui32LineSize = sizeof(IMG_CHAR) * (
+			ui32MsgHeaderCharCount +
+			(psDevInfo->sDevFeatureCfg.ui32MAXDMCount * FWHWRINFO_DM_STR_SIZE) +
+			ui32MsgFalseCharCount + 1 + (psDevInfo->sDevFeatureCfg.ui32MAXDMCount*6) + 1
+				/* 'FALSE(' + ')' + (UINT16 max num + comma) per DM + \0 */
+			);
+
+	pszLine = OSAllocMem(ui32LineSize);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Out of mem allocating line string (size: %d)",
+			__func__,
+			ui32LineSize));
+		return;
+	}
+
+	OSStringSafeCopy(pszLine, szMsgHeader, ui32LineSize);
+	pszTemp = pszLine + ui32MsgHeaderCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				FWHWRINFO_DM_STR_SIZE,
+				"%s(%u/%u+%u), ",
+				apszDmNames[dm],
+				psHWRInfoBuf->aui32HwrDmRecoveredCount[dm],
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm],
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm]);
+	}
+
+	OSStringLCat(pszLine, szMsgFalse, ui32LineSize);
+	pszTemp += ui32MsgFalseCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				10 + 1 + 1 /* UINT32 max num + comma + \0 */,
+				(dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1 ? "%u," : "%u)"),
+				psHWRInfoBuf->aui32HwrDmFalseDetectCount[dm]);
+	}
+
+	PVR_DUMPDEBUG_LOG("%s", pszLine);
+
+	OSFreeMem(pszLine);
+
+	/* Print out per HWR info */
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (dm == RGXFWIF_DM_GP)
+		{
+			PVR_DUMPDEBUG_LOG("DM %d (GP)", dm);
+		}
+		else
+		{
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+			{
+				IMG_UINT32 ui32HWRRecoveryFlags = psFwSysData->aui32HWRRecoveryFlags[dm];
+				IMG_CHAR sPerDmHwrDescription[RGX_DEBUG_STR_SIZE];
+				sPerDmHwrDescription[0] = '\0';
+
+				if (ui32HWRRecoveryFlags == RGXFWIF_DM_STATE_WORKING)
+				{
+					OSStringSafeCopy(sPerDmHwrDescription, " working;", RGX_DEBUG_STR_SIZE);
+				}
+				else
+				{
+					DebugCommonFlagStrings(sPerDmHwrDescription, RGX_DEBUG_STR_SIZE,
+						asDmState2Description, ARRAY_SIZE(asDmState2Description),
+						ui32HWRRecoveryFlags);
+				}
+				PVR_DUMPDEBUG_LOG("DM %d (HWRflags 0x%08x:%.*s)", dm, ui32HWRRecoveryFlags,
+								  RGX_DEBUG_STR_SIZE, sPerDmHwrDescription);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("DM %d", dm);
+			}
+		}
+
+		ui32ReadIndex = 0;
+		for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+		{
+			IMG_BOOL bPMFault = IMG_FALSE;
+			IMG_UINT32 ui32PC;
+			IMG_UINT32 ui32PageSize = 0;
+			IMG_DEV_PHYADDR sPCDevPAddr = { 0 };
+			const RGX_HWRINFO *psHWRInfo = &psHWRInfoBuf->sHWRInfo[ui32ReadIndex];
+
+			if (ui32ReadIndex >= RGXFWIF_HWINFO_MAX)
+			{
+				PVR_DUMPDEBUG_LOG("HWINFO index error: %u", ui32ReadIndex);
+				break;
+			}
+
+			if ((psHWRInfo->eDM == dm) && (psHWRInfo->ui32HWRNumber != 0))
+			{
+				IMG_CHAR aui8RecoveryNum[10+10+1];
+				IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+				IMG_BOOL bPageFault = IMG_FALSE;
+				IMG_DEV_VIRTADDR sFaultDevVAddr;
+
+				/* Split OS timestamp in seconds and nanoseconds */
+				RGXConvertOSTimestampToSAndNS(psHWRInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+				ui32HWRRecoveryFlags = psHWRInfo->ui32HWRRecoveryFlags;
+				if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_LOCKUP) { pszLockupType = ", Guilty Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_LOCKUP) { pszLockupType = ", Innocent Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_OVERRUNING) { pszLockupType = ", Guilty Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_OVERRUNING) { pszLockupType = ", Innocent Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH) { pszLockupType = ", Hard Context Switch"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_ECC_HWR) { pszLockupType = ", GPU ECC HWR"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_PARITY_HWR) { pszLockupType = ", GPU PARITY HWR"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_LATENT_HWR) { pszLockupType = ", GPU LATENT HWR"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_ICS_HWR) { pszLockupType = ", ICS HWR"; }
+
+				OSSNPrintf(aui8RecoveryNum, sizeof(aui8RecoveryNum), "Recovery %d:", psHWRInfo->ui32HWRNumber);
+				if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+				{
+					PVR_DUMPDEBUG_LOG("  %s Core = %u, PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32CoreID,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+
+				if (psHWRInfo->eHWErrorCode != RGX_HW_ERR_NA)
+				{
+					IMG_CHAR sHWDebugInfo[RGX_DEBUG_STR_SIZE] = "";
+
+					_ID2Description(sHWDebugInfo, RGX_DEBUG_STR_SIZE, asHWErrorState, ARRAY_SIZE(asHWErrorState),
+						psHWRInfo->eHWErrorCode);
+					PVR_DUMPDEBUG_LOG("  HW error code = 0x%X: %s",
+									  psHWRInfo->eHWErrorCode, sHWDebugInfo);
+				}
+
+				pszTemp = &aui8RecoveryNum[0];
+				while (*pszTemp != '\0')
+				{
+					*pszTemp++ = ' ';
+				}
+
+				/* There's currently no time correlation for the Guest OSes on the Firmware so there's no point printing OS Timestamps on Guests */
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   ui64Seconds,
+									   ui64Nanoseconds,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+
+				if (psHWRInfo->ui64CRTimeHWResetFinish != 0)
+				{
+					if (psHWRInfo->ui64CRTimeFreelistReady != 0)
+					{
+						/* If ui64CRTimeFreelistReady is less than ui64CRTimeHWResetFinish it means APM kicked in and the time is not valid. */
+						if (psHWRInfo->ui64CRTimeHWResetFinish < psHWRInfo->ui64CRTimeFreelistReady)
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalRecoveryTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimeHWResetFinish)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimer)*256);
+						}
+						else
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = <not_timed>, TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+						}
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+										   aui8RecoveryNum,
+										   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+					}
+				}
+
+#if defined(HW_ERN_65104_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 65104))
+				{
+					PVR_DUMPDEBUG_LOG("    Active PDS DM USCs = 0x%08x", psHWRInfo->ui32PDSActiveDMUSCs);
+				}
+#endif
+
+#if defined(HW_ERN_69700_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 69700))
+				{
+					PVR_DUMPDEBUG_LOG("    DMs stalled waiting on PDS Store space = 0x%08x", psHWRInfo->ui32PDSStalledDMs);
+				}
+#endif
+
+				switch (psHWRInfo->eHWRType)
+				{
+					case RGX_HWRTYPE_ECCFAULT:
+					{
+						PVR_DUMPDEBUG_LOG("    ECC fault GPU=0x%08x", psHWRInfo->uHWRData.sECCInfo.ui32FaultGPU);
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+						{
+							_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo,
+											&psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0],
+											"Core",
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr =   psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0];
+							sFaultDevVAddr.uiAddr &=  ~RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK;
+							sFaultDevVAddr.uiAddr >>= RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT;
+							sFaultDevVAddr.uiAddr <<= 4; /* align shift */
+							ui32PC  = (psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK) >>
+													   RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT;
+							bPMFault = (ui32PC <= 8);
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sMMUInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUMETAFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+						{
+							const IMG_PCHAR pszMetaOrRiscv = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? "Meta" : "RiscV";
+
+							_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo,
+											&psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0],
+											pszMetaOrRiscv,
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr =   psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0];
+							sFaultDevVAddr.uiAddr &=  ~RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK;
+							sFaultDevVAddr.uiAddr >>= RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT;
+							sFaultDevVAddr.uiAddr <<= 4; /* align shift */
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sMMUInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_POLLFAILURE:
+					{
+						PVR_DUMPDEBUG_LOG("    T%u polling %s (reg:0x%08X mask:0x%08X last:0x%08X)",
+										  psHWRInfo->uHWRData.sPollInfo.ui32ThreadNum,
+										  ((psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & ~RGXFW_POLL_TYPE_SET,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollMask,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollLastValue);
+					}
+					break;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+					case RGX_HWRTYPE_MIPSTLBFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+						{
+							IMG_UINT32 ui32EntryLo = psHWRInfo->uHWRData.sTLBInfo.ui32EntryLo;
+
+							/* This is not exactly what the MMU code does, but the result should be the same */
+							const IMG_UINT32 ui32UnmappedEntry =
+								((IMG_UINT32)(MMU_BAD_PHYS_ADDR & 0xffffffff) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) | RGXMIPSFW_ENTRYLO_UNCACHED;
+
+							PVR_DUMPDEBUG_LOG("    MIPS TLB fault: BadVA = 0x%08X, EntryLo = 0x%08X"
+											  " (page PA 0x%" IMG_UINT64_FMTSPECx", V %u)",
+											  psHWRInfo->uHWRData.sTLBInfo.ui32BadVAddr,
+											  ui32EntryLo,
+											  RGXMIPSFW_TLB_GET_PA(ui32EntryLo),
+											  ui32EntryLo & RGXMIPSFW_TLB_VALID ? 1 : 0);
+
+							if (ui32EntryLo == ui32UnmappedEntry)
+							{
+								PVR_DUMPDEBUG_LOG("    Potential use-after-free detected");
+							}
+						}
+					}
+					break;
+#endif
+
+					case RGX_HWRTYPE_OVERRUN:
+					case RGX_HWRTYPE_UNKNOWNFAILURE:
+					{
+						/* Nothing to dump */
+					}
+					break;
+
+					default:
+					{
+						PVR_DUMPDEBUG_LOG("    Unknown HWR Info type: 0x%x", psHWRInfo->eHWRType);
+					}
+					break;
+				}
+
+				if (bPageFault)
+				{
+					RGXDumpFaultInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psHWRInfo,
+					                 ui32ReadIndex, &sFaultDevVAddr, &sPCDevPAddr, bPMFault, ui32PageSize);
+				}
+
+			}
+
+			if (ui32ReadIndex == RGXFWIF_HWINFO_MAX_FIRST - 1)
+				ui32ReadIndex = psHWRInfoBuf->ui32WriteIndex;
+			else
+				ui32ReadIndex = (ui32ReadIndex + 1) - (ui32ReadIndex / RGXFWIF_HWINFO_LAST_INDEX) * RGXFWIF_HWINFO_MAX_LAST;
+		}
+	}
+}
+
+
+
+
+#if !defined(NO_HARDWARE)
+
+/*!
+*******************************************************************************
+
+ @Function	_CheckForPendingPage
+
+ @Description
+
+ Check if the MMU indicates it is blocked on a pending page
+ MMU4 does not support pending pages, so return false.
+
+ @Input psDevInfo	 - RGX device info
+
+ @Return   IMG_BOOL      - IMG_TRUE if there is a pending page
+
+******************************************************************************/
+static INLINE IMG_BOOL _CheckForPendingPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* MMU4 doesn't support pending pages */
+	return (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) < 4) &&
+		   (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY) & RGX_CR_MMU_ENTRY_PENDING_EN);
+#else
+	IMG_UINT32 ui32BIFMMUEntry;
+
+	ui32BIFMMUEntry = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY);
+
+	if (ui32BIFMMUEntry & RGX_CR_BIF_MMU_ENTRY_PENDING_EN)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetPendingPageInfo
+
+ @Description
+
+ Get information about the pending page from the MMU status registers
+
+ @Input psDevInfo	 - RGX device info
+ @Output psDevVAddr      - The device virtual address of the pending MMU address translation
+ @Output pui32CatBase    - The page catalog base
+
+ @Return   void
+
+******************************************************************************/
+static void _GetPendingPageInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_VIRTADDR *psDevVAddr,
+								IMG_UINT32 *pui32CatBase)
+{
+	IMG_UINT64 ui64BIFMMUEntryStatus;
+
+	ui64BIFMMUEntryStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY_STATUS);
+
+	psDevVAddr->uiAddr = (ui64BIFMMUEntryStatus & ~RGX_CR_MMU_ENTRY_STATUS_ADDRESS_CLRMSK);
+
+	*pui32CatBase = (ui64BIFMMUEntryStatus & ~RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_CLRMSK) >>
+								RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_SHIFT;
+}
+
+#endif
+
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON)
+{
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	IMG_UINT32 ui32DriverID;
+	const RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	/* space for the current clock speed and 3 previous */
+	RGXFWIF_TIME_CORR asTimeCorrs[4];
+	IMG_UINT32 ui32NumClockSpeedChanges;
+
+	/* Should invalidate all reads below including when passed to functions. */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, INVALIDATE);
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(bRGXPoweredON);
+#else
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+		{
+			IMG_UINT64	aui64RegValMMUStatus[2];
+			const IMG_PCHAR pszMetaOrRiscv = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? "Meta" : "RiscV";
+
+			/*
+			 *  BRN72144 prevents reading RGX_CR_MMU_FAULT_STATUS1/2
+			 *  registers while the FW is running...
+			 */
+			if (!RGX_IS_BRN_SUPPORTED(psDevInfo, 72144) ||
+			    (psRGXFWIfTraceBufCtl->sTraceBuf[0].sAssertBuf.szInfo[0] != 0))
+			{
+				aui64RegValMMUStatus[0] = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_FAULT_STATUS1);
+				aui64RegValMMUStatus[1] = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_FAULT_STATUS2);
+				_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, &aui64RegValMMUStatus[0], "Core", DD_SUMMARY_INDENT);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("%sMMU (Core) - Unknown", DD_SUMMARY_INDENT);
+			}
+
+			aui64RegValMMUStatus[0] = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_FAULT_STATUS_META);
+			_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, &aui64RegValMMUStatus[0], pszMetaOrRiscv, DD_SUMMARY_INDENT);
+		}
+
+		if (_CheckForPendingPage(psDevInfo))
+		{
+			IMG_UINT32 ui32CatBase;
+			IMG_DEV_VIRTADDR sDevVAddr;
+
+			PVR_DUMPDEBUG_LOG("MMU Pending page: Yes");
+
+			_GetPendingPageInfo(psDevInfo, &sDevVAddr, &ui32CatBase);
+
+			if (ui32CatBase <= MAX_RESERVED_FW_MMU_CONTEXT)
+			{
+				PVR_DUMPDEBUG_LOG("Cannot check address on PM cat base %u", ui32CatBase);
+			}
+			else
+			{
+				IMG_DEV_PHYADDR sPCDevPAddr;
+				MMU_FAULT_DATA sFaultData;
+				IMG_BOOL bIsValid = IMG_TRUE;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+				IMG_UINT64 ui64CBaseMapping;
+				IMG_UINT32 ui32CBaseMapCtxReg;
+
+				if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN);
+				}
+				else
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING_INVALID_EN);
+				}
+#else
+				sPCDevPAddr.uiAddr = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_CAT_BASEN(ui32CatBase));
+#endif
+
+				PVR_DUMPDEBUG_LOG("Checking device virtual address " IMG_DEV_VIRTADDR_FMTSPEC
+							" on cat base %u. PC Addr = 0x%" IMG_UINT64_FMTSPECx " is %s",
+								sDevVAddr.uiAddr,
+								ui32CatBase,
+								sPCDevPAddr.uiAddr,
+								bIsValid ? "valid":"invalid");
+				RGXCheckFaultAddress(psDevInfo, &sDevVAddr, &sPCDevPAddr, &sFaultData);
+				RGXDumpFaultAddressHostView(&sFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_SUMMARY_INDENT);
+			}
+		}
+	}
+#endif /* NO_HARDWARE */
+
+#if !defined(NO_HARDWARE)
+	/* Determine the type virtualisation support used */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+#if defined(SUPPORT_AUTOVZ)
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with HW register support");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with shared memory");
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with static Fw heap allocation");
+#endif /* defined(SUPPORT_AUTOVZ) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with dynamic Fw heap allocation");
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+	}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_CONNECTION_FW_STATE eFwState;
+		RGXFWIF_CONNECTION_OS_STATE eOsState;
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+		eFwState = KM_GET_FW_CONNECTION(psDevInfo);
+		eOsState = KM_GET_OS_CONNECTION(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation firmware connection state: %s (Fw=%s; OS=%s)",
+						  ((eFwState == RGXFW_CONNECTION_FW_ACTIVE) && (eOsState == RGXFW_CONNECTION_OS_ACTIVE)) ? ("UP") : ("DOWN"),
+						  (eFwState < RGXFW_CONNECTION_FW_STATE_COUNT) ? (apszFwOsStateName[eFwState]) : ("invalid"),
+						  (eOsState < RGXFW_CONNECTION_OS_STATE_COUNT) ? (apszFwOsStateName[eOsState]) : ("invalid"));
+
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32FwAliveTS;
+		IMG_UINT32 ui32OsAliveTS;
+
+		KM_ALIVE_TOKEN_CACHEOP(Fw, INVALIDATE);
+		KM_ALIVE_TOKEN_CACHEOP(Os, INVALIDATE);
+
+		ui32FwAliveTS = KM_GET_FW_ALIVE_TOKEN(psDevInfo);
+		ui32OsAliveTS = KM_GET_OS_ALIVE_TOKEN(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation watchdog timestamps (in GPU timer ticks): Fw=%u; OS=%u; diff(FW, OS) = %u",
+						  ui32FwAliveTS, ui32OsAliveTS, ui32FwAliveTS - ui32OsAliveTS);
+	}
+#endif
+#endif /* !defined(NO_HARDWARE) */
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_CHAR sHwrStateDescription[RGX_DEBUG_STR_SIZE];
+		IMG_BOOL bDriverIsolationEnabled = IMG_FALSE;
+		IMG_UINT32 ui32HostIsolationGroup;
+
+		sHwrStateDescription[0] = '\0';
+
+		DebugCommonFlagStrings(sHwrStateDescription, RGX_DEBUG_STR_SIZE,
+			asHwrState2Description, ARRAY_SIZE(asHwrState2Description),
+			psFwSysData->ui32HWRStateFlags);
+		PVR_DUMPDEBUG_LOG("RGX HWR State 0x%08x:%s", psFwSysData->ui32HWRStateFlags, sHwrStateDescription);
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: %s (APM %s: %d ok, %d denied, %d non-idle, %d retry, %d other, %d total. Latency: %u ms)",
+		                  (psFwSysData->ePowState < ARRAY_SIZE(pszPowStateName) ? pszPowStateName[psFwSysData->ePowState] : "???"),
+		                  (psDevInfo->pvAPMISRData)?"enabled":"disabled",
+		                  psDevInfo->ui32ActivePMReqOk - psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqDenied,
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqRetry,
+		                  psDevInfo->ui32ActivePMReqTotal -
+		                  psDevInfo->ui32ActivePMReqOk -
+		                  psDevInfo->ui32ActivePMReqDenied -
+		                  psDevInfo->ui32ActivePMReqRetry -
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqTotal,
+		                  psRuntimeCfg->ui32ActivePMLatencyms);
+
+		ui32NumClockSpeedChanges = (IMG_UINT32) OSAtomicRead(&psDevInfo->psDeviceNode->iNumClockSpeedChanges);
+		RGXGetTimeCorrData(psDevInfo->psDeviceNode, asTimeCorrs, ARRAY_SIZE(asTimeCorrs));
+
+		PVR_DUMPDEBUG_LOG("RGX DVFS: %u frequency changes. "
+		                  "Current frequency: %u.%03u MHz (sampled at %" IMG_UINT64_FMTSPEC " ns). "
+		                  "FW frequency: %u.%03u MHz.",
+		                  ui32NumClockSpeedChanges,
+		                  asTimeCorrs[0].ui32CoreClockSpeed / 1000000,
+		                  (asTimeCorrs[0].ui32CoreClockSpeed / 1000) % 1000,
+		                  asTimeCorrs[0].ui64OSTimeStamp,
+		                  psRuntimeCfg->ui32CoreClockSpeed / 1000000,
+		                  (psRuntimeCfg->ui32CoreClockSpeed / 1000) % 1000);
+		if (ui32NumClockSpeedChanges > 0)
+		{
+			PVR_DUMPDEBUG_LOG("          Previous frequencies: %u.%03u, %u.%03u, %u.%03u MHz (Sampled at "
+							"%" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ")",
+												asTimeCorrs[1].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[1].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[2].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[2].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[3].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[3].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[1].ui64OSTimeStamp,
+												asTimeCorrs[2].ui64OSTimeStamp,
+												asTimeCorrs[3].ui64OSTimeStamp);
+		}
+
+		ui32HostIsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID];
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			RGXFWIF_OS_RUNTIME_FLAGS sFwRunFlags = psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+			IMG_UINT32 ui32IsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID];
+			IMG_BOOL bMTSEnabled = IMG_FALSE;
+
+#if !defined(NO_HARDWARE)
+			if (bRGXPoweredON)
+			{
+				bMTSEnabled = (
+#if defined(FIX_HW_BRN_64502_BIT_MASK)
+								RGX_IS_BRN_SUPPORTED(psDevInfo, 64502) ||
+#endif
+								!RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_VIRTUALISATION)) ?
+								IMG_TRUE : ((OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE_ENABLE) & BIT(ui32DriverID)) != 0);
+			}
+#endif
+			PVR_DUMPDEBUG_LOG("RGX FW OS %u - State: %s; Freelists: %s%s; Priority: %u; Isolation group: %u;  Time Slice%s: %u%% (%ums); %s", ui32DriverID,
+							  apszFwOsStateName[sFwRunFlags.bfOsState],
+							  (sFwRunFlags.bfFLOk) ? "Ok" : "Not Ok",
+							  (sFwRunFlags.bfFLGrowPending) ? "; Grow Request Pending" : "",
+							  psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID],
+							  ui32IsolationGroup,
+							  (psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] != 0) ? "" : "*",
+							  psFwSysData->aui32TSMirror[ui32DriverID],
+							  (psFwSysData->aui32TSMirror[ui32DriverID] *
+							   psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs / 100),
+							  (bMTSEnabled) ? "MTS on;" : "MTS off;"
+							 );
+
+			if (ui32IsolationGroup != ui32HostIsolationGroup)
+			{
+				bDriverIsolationEnabled = IMG_TRUE;
+			}
+
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+			{
+				/* don't print guest information on native mode drivers */
+				break;
+			}
+		}
+
+#if defined(PVR_ENABLE_PHR)
+		{
+			IMG_CHAR sPHRConfigDescription[RGX_DEBUG_STR_SIZE];
+
+			sPHRConfigDescription[0] = '\0';
+			DebugCommonFlagStrings(sPHRConfigDescription, RGX_DEBUG_STR_SIZE,
+			                   asPHRConfig2Description, ARRAY_SIZE(asPHRConfig2Description),
+			                   BIT_ULL(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode));
+
+			PVR_DUMPDEBUG_LOG("RGX PHR configuration: (%d) %.*s", psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, RGX_DEBUG_STR_SIZE, sPHRConfigDescription);
+		}
+#endif
+
+		if (bDriverIsolationEnabled)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Hard Context Switch deadline: %u ms", psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+		}
+
+		_RGXDumpFWAssert(pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl);
+		_RGXDumpFWFaults(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+		_RGXDumpFWPoll(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("RGX FW State: Unavailable under Guest Mode of operation");
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: Unavailable under Guest Mode of operation");
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+	_RGXDumpFWHWRInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData, psDevInfo->psRGXFWIfHWRInfoBufCtl, psDevInfo);
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+	/* Dump all non-zero values in lines of 8... */
+	{
+		IMG_CHAR    pszLine[(9*RGXFWIF_STATS_FRAMEWORK_LINESIZE)+1];
+		const IMG_UINT32 *pui32FWStatsBuf = psFwSysData->aui32FWStatsBuf;
+		IMG_UINT32  ui32Index1, ui32Index2;
+
+		PVR_DUMPDEBUG_LOG("STATS[START]: RGXFWIF_STATS_FRAMEWORK_MAX=%d", RGXFWIF_STATS_FRAMEWORK_MAX);
+		for (ui32Index1 = 0;  ui32Index1 < RGXFWIF_STATS_FRAMEWORK_MAX;  ui32Index1 += RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+		{
+			IMG_UINT32  ui32OrOfValues = 0;
+			IMG_CHAR    *pszBuf = pszLine;
+
+			/* Print all values in this line and skip if all zero... */
+			for (ui32Index2 = 0;  ui32Index2 < RGXFWIF_STATS_FRAMEWORK_LINESIZE;  ui32Index2++)
+			{
+				ui32OrOfValues |= pui32FWStatsBuf[ui32Index1+ui32Index2];
+				OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32FWStatsBuf[ui32Index1+ui32Index2]);
+				pszBuf += 9; /* write over the '\0' */
+			}
+
+			if (ui32OrOfValues != 0)
+			{
+				PVR_DUMPDEBUG_LOG("STATS[%08x]:%s", ui32Index1, pszLine);
+			}
+		}
+		PVR_DUMPDEBUG_LOG("STATS[END]");
+	}
+#endif
+}
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvSecureRegsBaseKM;
+	RGXRISCVFW_STATE sRiscvState;
+	const IMG_CHAR *pszException;
+	PVRSRV_ERROR eError;
+
+	/* Limit dump to what is currently being used */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4);
+	}
+	else
+#endif
+	{
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG4);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG5);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG6);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG12);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG13);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG14);
+	}
+
+	PVR_DUMPDEBUG_LOG("---- [ RISC-V internal state ] ----");
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGXRiscvIsHalted(psDevInfo))
+	{
+		/* Avoid resuming the RISC-V FW as most operations
+		 * on the debug module require a halted core */
+		PVR_DUMPDEBUG_LOG("(skipping as RISC-V found halted)");
+		return PVRSRV_OK;
+	}
+#endif
+
+	eError = RGXRiscvHalt(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+#define X(name, address)												\
+	eError = RGXRiscvReadReg(psDevInfo, address, &sRiscvState.name);	\
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXRiscvReadReg", _RISCVDMError);	\
+	DDLOGVAL32(#name, sRiscvState.name);
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+
+	eError = RGXRiscvResume(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+	pszException = _GetRISCVException(sRiscvState.mcause);
+	if (pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG("RISC-V FW hit an exception: %s", pszException);
+
+		eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+		}
+	}
+
+	return PVRSRV_OK;
+
+_RISCVDMError:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to communicate with the Debug Module"));
+
+	return eError;
+}
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(CORE_ID);
+}
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(MULTICORE);
+	DDLOG32(MULTICORE_SYSTEM);
+	DDLOG32(MULTICORE_DOMAIN);
+
+#if !defined(RGX_CR_MULTICORE_AXI)
+#define RGX_CR_MULTICORE_AXI                              (0x2508U)
+#define RGX_CR_MULTICORE_AXI_ERROR                        (0x2510U)
+#endif
+	DDLOG32(MULTICORE_AXI);
+	DDLOG32(MULTICORE_AXI_ERROR);
+	DDLOG32(MULTICORE_TDM_CTRL_COMMON);
+	DDLOG32(MULTICORE_FRAGMENT_CTRL_COMMON);
+	DDLOG32(MULTICORE_COMPUTE_CTRL_COMMON);
+}
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(CLK_CTRL0);
+	DDLOG64(CLK_STATUS0);
+	DDLOG64(CLK_CTRL1);
+	DDLOG64(CLK_STATUS1);
+	DDLOG32(CLK_CTRL2);
+	DDLOG32(CLK_STATUS2);
+}
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	/* BRN72144 prevents reading RGX_CR_MMU_FAULT_STATUS1/2 while the FW is running... */
+	if (!RGX_IS_BRN_SUPPORTED(psDevInfo, 72144)  ||
+		(psDevInfo->psRGXFWIfTraceBufCtl->sTraceBuf[0].sAssertBuf.szInfo[0] != 0))
+	{
+		DDLOG64(MMU_FAULT_STATUS1);
+		DDLOG64(MMU_FAULT_STATUS2);
+	}
+	DDLOG64(MMU_FAULT_STATUS_PM);
+	DDLOG64(MMU_FAULT_STATUS_META);
+	DDLOG64(SLC_STATUS1);
+	DDLOG64(SLC_STATUS2);
+	DDLOG64(SLC_STATUS3);
+	DDLOG64(SLC_STATUS_DEBUG);
+	DDLOG64(MMU_STATUS);
+	DDLOG32(BIF_PFS);
+	DDLOG32(BIF_TEXAS0_PFS);
+	DDLOG32(BIF_TEXAS1_PFS);
+	DDLOG32(BIF_OUTSTANDING_READ);
+	DDLOG32(BIF_TEXAS0_OUTSTANDING_READ);
+	DDLOG32(BIF_TEXAS1_OUTSTANDING_READ);
+	DDLOG32(FBCDC_IDLE);
+	DDLOG32(FBCDC_STATUS);
+	DDLOG32(FBCDC_SIGNATURE_STATUS);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, POWER_ISLAND_VERSION) &&
+	    RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) < 2)
+	{
+		DDLOG32(SPU_ENABLE);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT))
+	{
+		DDLOG64(CONTEXT_MAPPING0);
+		DDLOG64(CONTEXT_MAPPING2);
+		DDLOG64(CONTEXT_MAPPING3);
+		DDLOG64(CONTEXT_MAPPING4);
+	}
+}
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(PERF_PHASE_2D);
+	DDLOG32(PERF_CYCLE_2D_TOTAL);
+	DDLOG32(PERF_PHASE_GEOM);
+	DDLOG32(PERF_CYCLE_GEOM_TOTAL);
+	DDLOG32(PERF_PHASE_FRAG);
+	DDLOG32(PERF_CYCLE_FRAG_TOTAL);
+	DDLOG32(PERF_CYCLE_GEOM_OR_FRAG_TOTAL);
+	DDLOG32(PERF_CYCLE_GEOM_AND_FRAG_TOTAL);
+	DDLOG32(PERF_PHASE_COMP);
+	DDLOG32(PERF_CYCLE_COMP_TOTAL);
+	DDLOG32(PM_PARTIAL_RENDER_ENABLE);
+
+	DDLOG32(ISP_RENDER);
+	DDLOG32(ISP_CTL);
+
+	DDLOG32(CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5);
+	DDLOG64(CDM_CONTEXT_PDS0);
+	DDLOG64(CDM_CONTEXT_PDS1);
+	DDLOG64(CDM_TERMINATE_PDS);
+	DDLOG64(CDM_TERMINATE_PDS1);
+	DDLOG64(CDM_CONTEXT_LOAD_PDS0);
+	DDLOG64(CDM_CONTEXT_LOAD_PDS1);
+}
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(SLC_IDLE);
+	DDLOG32(SLC_FAULT_STOP_STATUS);
+}
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(JONES_IDLE);
+
+	DDLOG64(SCRATCH0);
+	DDLOG64(SCRATCH1);
+	DDLOG64(SCRATCH2);
+	DDLOG64(SCRATCH3);
+	DDLOG64(SCRATCH4);
+	DDLOG64(SCRATCH5);
+	DDLOG64(SCRATCH6);
+	DDLOG64(SCRATCH7);
+	DDLOG64(SCRATCH8);
+	DDLOG64(SCRATCH9);
+	DDLOG64(SCRATCH10);
+	DDLOG64(SCRATCH11);
+	DDLOG64(SCRATCH12);
+	DDLOG64(SCRATCH13);
+	DDLOG64(SCRATCH14);
+	DDLOG64(SCRATCH15);
+	DDLOG32(IRQ_OS0_EVENT_STATUS);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#undef REG32_FMTSPEC
+#undef REG64_FMTSPEC
+#undef DDLOG32
+#undef DDLOG64
+#undef DDLOG32_DPX
+#undef DDLOG64_DPX
+#undef DDLOGVAL32
+
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32VerbLevel)
+{
+	DumpRenderCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DumpKickSyncCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		DumpComputeCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		DumpTDMTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+}
+
+/******************************************************************************
+ End of file (rgxdebug.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdevice.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdevice.h
new file mode 100644
index 000000000000..5e197ec8a0f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdevice.h
@@ -0,0 +1,953 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX device node header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX device node
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEVICE_H)
+#define RGXDEVICE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_device_types.h"
+#include "mmu_common.h"
+#include "rgx_fwif_km.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "osfunc.h"
+#include "rgxlayer_impl.h"
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "hash.h"
+#endif
+typedef struct _RGX_SERVER_COMMON_CONTEXT_ RGX_SERVER_COMMON_CONTEXT;
+
+typedef struct {
+	DEVMEM_MEMDESC		*psFWFrameworkMemDesc;
+} RGX_COMMON_CONTEXT_INFO;
+
+
+/*!
+ ******************************************************************************
+ * Device state flags
+ *****************************************************************************/
+#define RGXKM_DEVICE_STATE_ZERO_FREELIST                          (0x1)  /*!< Zeroing the physical pages of reconstructed free lists */
+#define RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN                  (0x2)  /*!< Used to disable the Devices Watchdog logging */
+#define RGXKM_DEVICE_STATE_GPU_UNITS_POWER_CHANGE_EN              (0x4)  /*!< Used for validation to inject power units state change every DM kick */
+#define RGXKM_DEVICE_STATE_CCB_GROW_EN                            (0x8)  /*!< Used to indicate CCB grow is permitted */
+#define RGXKM_DEVICE_STATE_ENABLE_SPU_UNITS_POWER_MASK_CHANGE_EN  (0x10) /*!< Used for validation to enable SPU power state mask change */
+#define RGXKM_DEVICE_STATE_MASK                                   (0x1F)
+
+/*!
+ ******************************************************************************
+ * ECC RAM Fault Validation
+ *****************************************************************************/
+#define RGXKM_ECC_ERR_INJ_DISABLE 0
+#define RGXKM_ECC_ERR_INJ_SLC     1
+#define RGXKM_ECC_ERR_INJ_USC     2
+#define RGXKM_ECC_ERR_INJ_TPU     3
+#define RGXKM_ECC_ERR_INJ_RASCAL  4
+#define RGXKM_ECC_ERR_INJ_MARS    5
+
+#define RGXKM_ECC_ERR_INJ_INTERVAL 10U
+
+/*!
+ ******************************************************************************
+ * GPU DVFS Table
+ *****************************************************************************/
+
+#define RGX_GPU_DVFS_TABLE_SIZE                      32
+#define RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US       25000     /* Time required to calibrate a clock frequency the first time */
+#define RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US  150000    /* Time required for a recalibration after a DVFS transition */
+#define RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US    10000000  /* Time before the next periodic calibration and correlation */
+
+
+/*!
+ ******************************************************************************
+ * Global flags for driver validation
+ *****************************************************************************/
+#define RGX_VAL_FBDC_SIG_CHECK_NOERR_EN           (0x2U)  /*!< Enable FBDC signature check. Signatures must match */
+#define RGX_VAL_FBDC_SIG_CHECK_ERR_EN             (0x4U)  /*!< Enable FBDC signature check. Signatures must not match */
+#define RGX_VAL_GPUSTATEPIN_EN                    (0x8U)  /*!< Enable GPU state pin check */
+#define RGX_VAL_WGP_SIG_CHECK_NOERR_EN           (0x10U)  /*!< Enable WGP signature check. Signatures must match */
+#define RGX_VAL_WGP_SIG_CHECK_ERR_EN             (0x20U)  /*!< Enable WGP signature check. Signatures must not match */
+#define RGX_VAL_TRP_SIG_CHECK_NOERR_EN           (0x40U)  /*!< Enable TRP signature check. Signatures must match */
+#define RGX_VAL_TRP_SIG_CHECK_ERR_EN             (0x80U)  /*!< Enable TRP signature check. Signatures must not match */
+
+/*!
+ ******************************************************************************
+ * HWPerf L2 Stream ID type definition.
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_HWPERF_L2_STREAM_ID;
+/* HWPerf stream for Client HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_HWPERF 0U
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+/* HWPerf stream for FTrace HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_FTRACE 1U
+#define RGX_HWPERF_L2_STREAM_LAST   2U
+#else
+#define RGX_HWPERF_L2_STREAM_LAST   1U
+#endif
+
+/*!
+ ******************************************************************************
+ * Boot-time Faults Injection
+ *****************************************************************************/
+#define RGX_NO_FAULTS_INJ                  0
+#define RGX_AXI_BUS_PARITY_INJ             1
+#define RGX_SAFETY_IRQ_AXI_BUS_PARITY_INJ  2
+#define RGX_XPU_BUS_PARITY_INJ             3
+#define RGX_AXI_MEM_BUS_PARITY_INJ         4
+#define RGX_DCLS_INJ                       5
+#define RGX_MMU_PTE_PARITY_INJ             6
+#define RGX_MMU_PTE_PARITY_INJ_KICK_NAME   "kick-tq"
+#define RGX_MMU_PTE_PARITY_INJ_PMR          "tquscpmr"
+#define RGX_REST_LATENT_ICS_INJ            7
+#define RGX_REST_LATENT_ICS_KICK_NAME      "DoKickTA"
+
+typedef struct _GPU_FREQ_TRACKING_DATA_
+{
+	/* Core clock speed estimated by the driver */
+	IMG_UINT32 ui32EstCoreClockSpeed;
+
+	/* Amount of successful calculations of the estimated core clock speed */
+	IMG_UINT32 ui32CalibrationCount;
+} GPU_FREQ_TRACKING_DATA;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+#define RGX_GPU_FREQ_TRACKING_SIZE 16
+
+typedef struct
+{
+	IMG_UINT64 ui64BeginCRTimestamp;
+	IMG_UINT64 ui64BeginOSTimestamp;
+
+	IMG_UINT64 ui64EndCRTimestamp;
+	IMG_UINT64 ui64EndOSTimestamp;
+
+	IMG_UINT32 ui32EstCoreClockSpeed;
+	IMG_UINT32 ui32CoreClockSpeed;
+} GPU_FREQ_TRACKING_HISTORY;
+#endif
+
+typedef struct _RGX_GPU_DVFS_TABLE_
+{
+	/* Beginning of current calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimestamp;
+	IMG_UINT64 ui64CalibrationOSTimestamp;
+
+	/* Calculated calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimediff;
+	IMG_UINT64 ui64CalibrationOSTimediff;
+
+	/* Current calibration period (in us) */
+	IMG_UINT32 ui32CalibrationPeriod;
+
+	/* System layer frequency table and frequency tracking data */
+	IMG_UINT32 ui32FreqIndex;
+	IMG_UINT32 aui32GPUFrequency[RGX_GPU_DVFS_TABLE_SIZE];
+	GPU_FREQ_TRACKING_DATA asTrackingData[RGX_GPU_DVFS_TABLE_SIZE];
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	IMG_UINT32 ui32HistoryIndex;
+	GPU_FREQ_TRACKING_HISTORY asTrackingHistory[RGX_GPU_FREQ_TRACKING_SIZE];
+#endif
+} RGX_GPU_DVFS_TABLE;
+
+
+/*!
+ ******************************************************************************
+ * GPU utilisation statistics
+ *****************************************************************************/
+
+typedef struct _RGXFWIF_TEMP_GPU_UTIL_STATS_
+{
+	IMG_UINT64 aaaui64DMOSTmpCounters[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 aaui64DMOSTmpLastWord[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastState[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastPeriod[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastTime[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+} RGXFWIF_TEMP_GPU_UTIL_STATS;
+
+typedef struct _RGXFWIF_GPU_UTIL_STATS_
+{
+	IMG_BOOL   bValid;                /* If TRUE, statistics are valid.
+	                                     FALSE if the driver couldn't get reliable stats. */
+	IMG_UINT64 ui64GpuStatActive;     /* GPU active statistic */
+	IMG_UINT64 ui64GpuStatBlocked;    /* GPU blocked statistic */
+	IMG_UINT64 ui64GpuStatIdle;       /* GPU idle statistic */
+	IMG_UINT64 ui64GpuStatCumulative; /* Sum of active/blocked/idle stats */
+
+	IMG_UINT64 aaui64DMOSStatInactive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];   /* Per-DM per-OS sum of idle and blocked stats */
+	IMG_UINT64 aaui64DMOSStatActive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];     /* Per-DM per-OS active statistic */
+	IMG_UINT64 aaui64DMOSStatCumulative[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED]; /* Per-DM per-OS sum of active/blocked/idle stats */
+
+	IMG_UINT64 ui64TimeStamp;         /* Timestamp of the most recent sample of the GPU stats */
+
+	RGXFWIF_TEMP_GPU_UTIL_STATS sTempGpuStats; /* Temporary data used to calculate the per-DM per-OS statistics */
+} RGXFWIF_GPU_UTIL_STATS;
+
+
+typedef struct _RGX_REG_CONFIG_
+{
+	IMG_BOOL               bEnabled;
+	RGXFWIF_REG_CFG_TYPE   eRegCfgTypeToPush;
+	IMG_UINT32             ui32NumRegRecords;
+	POS_LOCK               hLock;
+} RGX_REG_CONFIG;
+
+typedef struct _PVRSRV_STUB_PBDESC_ PVRSRV_STUB_PBDESC;
+
+
+typedef struct _PVRSRV_DEVICE_FEATURE_CONFIG_
+{
+	IMG_UINT64 ui64ErnsBrns;
+	IMG_UINT64 ui64Features;
+	IMG_UINT32 ui32B;
+	IMG_UINT32 ui32V;
+	IMG_UINT32 ui32N;
+	IMG_UINT32 ui32C;
+	IMG_UINT32 ui32FeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX];
+	IMG_UINT32 ui32MAXDMCount;
+	IMG_UINT32 ui32MAXPowUnitCount;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	IMG_UINT32 ui32MAXRACCount;
+#endif
+	IMG_UINT32 ui32SLCSizeInBytes;
+	IMG_PCHAR  pszBVNCString;
+}PVRSRV_DEVICE_FEATURE_CONFIG;
+
+/* This is used to get the value of a specific feature.
+ * Note that it will assert if the feature is disabled or value is invalid. */
+#define RGX_GET_FEATURE_VALUE(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] )
+
+/* This is used to check if the feature value (e.g. with an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] < RGX_FEATURE_VALUE_DISABLED )
+
+/* This is used to check if the Boolean feature (e.g. WITHOUT an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_SUPPORTED(psDevInfo, Feature) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64Features, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if the ERN is available for the currently running BVNC or not */
+#define RGX_IS_ERN_SUPPORTED(psDevInfo, ERN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if the BRN is available for the currently running BVNC or not */
+#define RGX_IS_BRN_SUPPORTED(psDevInfo, BRN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+/* there is a corresponding define in rgxapi.h */
+#define RGX_MAX_TIMER_QUERIES 16U
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*!
+ * The host maintains a 512-deep cache of submitted workloads per device,
+ * i.e. a global look-up table for TA, 3D and compute (depending on the RGX
+ * hardware support present)
+ */
+
+/*
+ * For the workload estimation return data array, the max amount of commands the
+ * MTS can have is 255, therefore 512 (LOG2 = 9) is large enough to account for
+ * all corner cases
+ */
+#define RETURN_DATA_ARRAY_SIZE_LOG2 (9)
+#define RETURN_DATA_ARRAY_SIZE      ((1U) << RETURN_DATA_ARRAY_SIZE_LOG2)
+#define RETURN_DATA_ARRAY_WRAP_MASK (RETURN_DATA_ARRAY_SIZE - 1)
+
+#define WORKLOAD_HASH_SIZE_LOG2		6
+#define WORKLOAD_HASH_SIZE			((1U) << WORKLOAD_HASH_SIZE_LOG2)
+#define WORKLOAD_HASH_WRAP_MASK		(WORKLOAD_HASH_SIZE - 1)
+
+/*!
+ * Workload characteristics for supported data masters.
+ * All characteristics must match for the workload estimate to be used/updated.
+ */
+typedef union _RGX_WORKLOAD_
+{
+	struct
+	{
+		IMG_UINT32				ui32RenderTargetSize;
+		IMG_UINT32				ui32NumberOfDrawCalls;
+		IMG_UINT32				ui32NumberOfIndices;
+		IMG_UINT32				ui32NumberOfMRTs;
+	} sTA3D;
+
+	struct
+	{
+		IMG_UINT32				ui32NumberOfWorkgroups;
+		IMG_UINT32				ui32NumberOfWorkitems;
+	} sCompute;
+
+	struct
+	{
+		IMG_UINT32				ui32Characteristic1;
+		IMG_UINT32				ui32Characteristic2;
+	} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	struct
+	{
+		IMG_UINT32				ui32DispatchSize;
+		IMG_UINT32				ui32AccStructSize;
+	} sRay;
+#endif
+
+} RGX_WORKLOAD;
+
+/*!
+ * Host data used to match the return data (actual cycles count) to the
+ * submitted command packet.
+ * The hash table is a per-DM circular buffer containing a key based on the
+ * workload characteristics. On job completion, the oldest workload data
+ * is evicted if the CB is full and the driver matches the characteristics
+ * to the matching data.
+ *
+ * o If the driver finds a match the existing cycle estimate is averaged with
+ *   the actual cycles used.
+ * o Otherwise a new hash entry is created with the actual cycles for this
+ *   workload.
+ *
+ * Subsequently if a match is found during command submission, the estimate
+ * is passed to the scheduler, e.g. adjust the GPU frequency if PDVFS is enabled.
+ */
+typedef struct _WORKLOAD_MATCHING_DATA_
+{
+	POS_LOCK				psHashLock;
+	HASH_TABLE				*psHashTable;		/*! existing workload cycle estimates for this DM */
+	RGX_WORKLOAD			asHashKeys[WORKLOAD_HASH_SIZE];
+	IMG_UINT64				aui64HashData[WORKLOAD_HASH_SIZE];
+	IMG_UINT32				ui32HashArrayWO;	/*! track the most recent workload estimates */
+} WORKLOAD_MATCHING_DATA;
+
+/*!
+ * A generic container for the workload matching data for GPU contexts:
+ * rendering (TA, 3D), compute, etc.
+ */
+typedef struct _WORKEST_HOST_DATA_
+{
+	union
+	{
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTA;	/*!< matching data for TA commands */
+			WORKLOAD_MATCHING_DATA	sData3D;	/*!< matching data for 3D commands */
+		} sTA3D;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataCDM;	/*!< matching data for CDM commands */
+		} sCompute;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTDM;	/*!< matching data for TDM-TQ commands */
+		} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataRDM;	/*!< matching data for RDM commands */
+		} sRay;
+#endif
+	} uWorkloadMatchingData;
+
+	/*
+	 * This is a per-context property, hence the TA and 3D share the same
+	 * per render context counter.
+	 */
+	IMG_UINT32				ui32WorkEstCCBReceived;	/*!< Used to ensure all submitted work
+														 estimation commands are received
+														 by the host before clean up. */
+} WORKEST_HOST_DATA;
+
+/*!
+ * Entries in the list of submitted workloads, used when the completed command
+ * returns data to the host.
+ *
+ * - the matching data is needed as it holds the hash data
+ * - the host data is needed for completion updates, ensuring memory is not
+ *   freed while workload estimates are in-flight.
+ * - the workload characteristic is used in the hash table look-up.
+ */
+typedef struct _WORKEST_RETURN_DATA_
+{
+	WORKEST_HOST_DATA		*psWorkEstHostData;
+	WORKLOAD_MATCHING_DATA	*psWorkloadMatchingData;
+	RGX_WORKLOAD			sWorkloadCharacteristics;
+} WORKEST_RETURN_DATA;
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+typedef struct
+{
+#if defined(PDUMP)
+	IMG_HANDLE      hPdumpPages;
+#endif
+	PG_HANDLE       sPages;
+	IMG_DEV_PHYADDR sPhysAddr;
+} RGX_MIPS_ADDRESS_TRAMPOLINE;
+#endif
+
+
+/*!
+ ******************************************************************************
+ * RGX Device error counts
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_ERROR_COUNTS_
+{
+	IMG_UINT32 ui32WGPErrorCount;		/*!< count of the number of WGP checksum errors */
+	IMG_UINT32 ui32TRPErrorCount;		/*!< count of the number of TRP checksum errors */
+} PVRSRV_RGXDEV_ERROR_COUNTS;
+
+/*!
+ ******************************************************************************
+ * RGX Debug dump firmware trace log type
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_FWT_LOGTYPE;
+#define RGX_FWT_LOGTYPE_NONE    0U
+#define RGX_FWT_LOGTYPE_BINARY  1U
+#define RGX_FWT_LOGTYPE_DECODED 2U
+#define RGX_FWT_LOGTYPE_PARTIAL 3U
+
+/*!
+ ******************************************************************************
+ * RGX Device info
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_INFO_
+{
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG	sDevFeatureCfg;
+
+	IMG_BOOL				bDevInit2Done;
+
+	IMG_BOOL				bFirmwareInitialised;
+	IMG_BOOL				bPDPEnabled;
+
+	IMG_HANDLE				hDbgReqNotify;
+
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvRegsBaseKM;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvSecureRegsBaseKM;
+#endif
+
+	IMG_HANDLE				hRegMapping;
+
+	/* System physical address of device registers */
+	IMG_CPU_PHYADDR			sRegsPhysBase;
+	/* Register region size in bytes */
+	IMG_UINT32				ui32RegSize;
+
+	PVRSRV_STUB_PBDESC		*psStubPBDescListKM;
+
+	/* Firmware memory context info */
+	DEVMEM_CONTEXT			*psKernelDevmemCtx;
+	DEVMEM_HEAP				*psFirmwareMainHeap;
+	DEVMEM_HEAP				*psFirmwareConfigHeap;
+	MMU_CONTEXT				*psKernelMMUCtx;
+
+	void					*pvDeviceMemoryHeap;
+
+	/* Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBCtlMemDesc;      /*!< memdesc for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtl;             /*!< kernel mapping for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtlLocal;        /*!< cpu local copy of Kernel CCB control */
+	DEVMEM_MEMDESC			*psKernelCCBMemDesc;         /*!< memdesc for Kernel CCB */
+	IMG_UINT8				*psKernelCCB;                /*!< kernel mapping for Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBRtnSlotsMemDesc; /*!< Return slot array for Kernel CCB commands */
+	IMG_UINT32				*pui32KernelCCBRtnSlots;     /*!< kernel mapping for return slot array */
+
+	/* Firmware CCB */
+	DEVMEM_MEMDESC			*psFirmwareCCBCtlMemDesc;   /*!< memdesc for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtl;          /*!< kernel mapping for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtlLocal;     /*!< cpu local copy of Firmware CCB control */
+	DEVMEM_MEMDESC			*psFirmwareCCBMemDesc;      /*!< memdesc for Firmware CCB */
+	IMG_UINT8				*psFirmwareCCB;             /*!< kernel mapping for Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBCtlMemDesc;   /*!< memdesc for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtl;          /*!< kernel mapping for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtlLocal;     /*!< cpu local copy of Workload Estimation Firmware CCB control */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBMemDesc;      /*!< memdesc for Workload Estimation Firmware CCB */
+	IMG_UINT8				*psWorkEstFirmwareCCB;             /*!< kernel mapping for Workload Estimation Firmware CCB */
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	/* Counter dumping */
+	DEVMEM_MEMDESC			*psCounterBufferMemDesc;      /*!< mem desc for counter dumping buffer */
+	POS_LOCK				hCounterDumpingLock;          /*!< Lock for guarding access to counter dumping buffer */
+#endif
+
+	PVRSRV_MEMALLOCFLAGS_T  uiFWPoisonOnFreeFlag;           /*!< Flag for poisoning FW allocations when freed */
+
+	IMG_BOOL				bIgnoreHWReportedBVNC;			/*!< Ignore BVNC reported by HW */
+
+    /* multicore configuration information */
+    IMG_UINT32              ui32MultiCoreNumCores;      /* total cores primary + secondaries. 0 for non-multi core */
+    IMG_UINT32              ui32MultiCorePrimaryId;     /* primary core id for this device */
+    IMG_UINT64             *pui64MultiCoreCapabilities; /* capabilities for each core */
+
+	/*
+		if we don't preallocate the pagetables we must
+		insert newly allocated page tables dynamically
+	*/
+	void					*pvMMUContextList;
+
+	IMG_UINT32				ui32ClkGateStatusReg;
+	IMG_UINT32				ui32ClkGateStatusMask;
+
+	DEVMEM_MEMDESC			*psRGXFWCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCodeDevVAddrBase;
+	IMG_UINT32			ui32FWCodeSizeInBytes;
+	DEVMEM_MEMDESC			*psRGXFWDataMemDesc;
+	IMG_DEV_VIRTADDR		sFWDataDevVAddrBase;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	RGX_MIPS_ADDRESS_TRAMPOLINE	*psTrampoline;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWCorememCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememCodeDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememCodeFWAddr;
+	IMG_UINT32			ui32FWCorememCodeSizeInBytes;
+
+	DEVMEM_MEMDESC			*psRGXFWIfCorememDataStoreMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememDataStoreDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememDataStoreFWAddr;
+
+	DEVMEM_MEMDESC			*psRGXFWAlignChecksMemDesc;
+
+#if defined(PDUMP)
+	DEVMEM_MEMDESC			*psRGXFWSigTAChecksMemDesc;
+	IMG_UINT32				ui32SigTAChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSig3DChecksMemDesc;
+	IMG_UINT32				ui32Sig3DChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigCDMChecksMemDesc;
+	IMG_UINT32				ui32SigCDMChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigTDMChecksMemDesc;
+	IMG_UINT32				ui32SigTDMChecksSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DEVMEM_MEMDESC			*psRGXFWSigRDMChecksMemDesc;
+	IMG_UINT32				ui32SigRDMChecksSize;
+#endif
+
+
+	IMG_BOOL				bDumpedKCCBCtlAlready;
+
+	POS_SPINLOCK			hSyncCheckpointSignalSpinLock;						/*!< Guards data shared between an atomic & sleepable-context */
+#endif
+
+	POS_LOCK				hRGXFWIfBufInitLock;								/*!< trace buffer lock for initialisation phase */
+
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufCtlMemDesc;						/*!< memdesc of trace buffer control structure */
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufferMemDesc[RGXFW_THREAD_NUM];		/*!< memdesc of actual FW trace (log) buffer(s) */
+	IMG_PUINT32				apui32TraceBuffer[RGXFW_THREAD_NUM];				/*!< Trace buffer address (Host address), to be used by host when reading from trace buffer */
+	IMG_UINT32				ui32TraceBufSizeInDWords;							/*!< CPU local copy of FW Trace buffer size in dwords */
+	DEVMEM_MEMDESC			*psRGXFWIfPowMonBufferMemDesc;						/*!< memdesc of FW power monitoring data */
+	RGXFWIF_TRACEBUF		*psRGXFWIfTraceBufCtl;								/*!< structure containing trace control data and actual trace buffer */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwSysDataMemDesc;							/*!< memdesc of the firmware-shared system data structure */
+	RGXFWIF_SYSDATA			*psRGXFWIfFwSysData;								/*!< structure containing km-firmware shared system data */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwOsDataMemDesc;							/*!< memdesc of the firmware-shared os structure */
+	RGXFWIF_OSDATA			*psRGXFWIfFwOsData;									/*!< structure containing km-firmware shared os data */
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	DEVMEM_MEMDESC			*psRGXFWIfTBIBufferMemDesc;							/*!< memdesc of actual FW TBI buffer */
+	RGXFWIF_DEV_VIRTADDR	sRGXFWIfTBIBuffer;									/*!< TBI buffer data */
+	IMG_UINT32				ui32FWIfTBIBufferSize;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWRInfoBufCtlMemDesc;
+	RGXFWIF_HWRINFOBUF		*psRGXFWIfHWRInfoBufCtl;
+	IMG_UINT32				ui32ClockSource;
+	IMG_UINT32				ui32LastClockSource;
+
+	DEVMEM_MEMDESC			*psRGXFWIfGpuUtilFWCtlMemDesc;
+	RGXFWIF_GPU_UTIL_FW     *psRGXFWIfGpuUtilFW;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfBufMemDesc;
+	IMG_BYTE				*psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32RGXFWIfHWPerfBufSize; /* in bytes */
+	IMG_UINT32				ui32RGXL2HWPerfBufSize;   /* in bytes */
+
+	DEVMEM_MEMDESC			*psRGXFWIfRegCfgMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfCountersMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfConnectionCtlMemDesc;
+	RGXFWIF_CONNECTION_CTL	*psRGXFWIfConnectionCtl;
+
+	DEVMEM_MEMDESC			*psRGXFWHeapGuardPageReserveMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfSysInitMemDesc;
+	RGXFWIF_SYSINIT			*psRGXFWIfSysInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfOsInitMemDesc;
+	RGXFWIF_OSINIT			*psRGXFWIfOsInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfRuntimeCfgMemDesc;
+	RGXFWIF_RUNTIME_CFG		*psRGXFWIfRuntimeCfg;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	DEVMEM_MEMDESC			*psRGXFWIfActiveContextBufDesc;
+	RGXFWIF_ACTIVE_CONTEXT_BUF_DATA		*psRGXFWIfActiveContextBuf;
+#endif
+
+	/* Premapped firmware memory context info */
+	DEVMEM_HEAP				*psPremappedFwRawHeap[RGX_NUM_DRIVERS_SUPPORTED];
+	DEVMEM_MEMDESC			*psPremappedFwRawMemDesc[RGX_NUM_DRIVERS_SUPPORTED];
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Array to store data needed for workload estimation when a workload
+	   has finished and its cycle time is returned to the host.	 */
+	WORKEST_RETURN_DATA     asReturnData[RETURN_DATA_ARRAY_SIZE];
+	IMG_UINT32              ui32ReturnDataWO;
+	POS_LOCK                hWorkEstLock;
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/**
+	 * Host memdesc and pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory on changing the core clock rate over
+	 * GPIO.
+	 * Note: Shared memory needs atomic access from Host driver and firmware,
+	 * hence size should not be greater than memory transaction granularity.
+	 * Currently it is chosen to be 32 bits.
+	 */
+	DEVMEM_MEMDESC			*psRGXFWIFCoreClkRateMemDesc;
+	volatile IMG_UINT32		*pui32RGXFWIFCoreClkRate;
+	/**
+	 * Last sampled core clk rate.
+	 */
+	volatile IMG_UINT32		ui32CoreClkRateSnapshot;
+#endif
+
+	/*
+	   HWPerf data for the RGX device
+	 */
+
+	POS_LOCK    hHWPerfLock;  /*! Critical section lock that protects HWPerf code
+	                           *  from multiple thread duplicate init/deinit
+	                           *  and loss/freeing of FW & Host resources while in
+	                           *  use in another thread e.g. MSIR. */
+
+	IMG_UINT64  ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_LAST];         /*! Event filter for FW events (settable by AppHint) */
+	IMG_HANDLE  hHWPerfStream[RGX_HWPERF_L2_STREAM_LAST];            /*! TL Stream buffer (L2) for firmware event stream */
+	IMG_UINT32  ui32L2BufMaxPacketSize[RGX_HWPERF_L2_STREAM_LAST];   /*! Max allowed packet size in FW HWPerf TL (L2) buffer */
+	IMG_BOOL    bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_LAST]; /*! Flag to indicate if copying HWPerf data is suspended */
+	IMG_UINT64  ui64HWPerfFwFilter;                                  /*! Event filter for FW events created from OR-ing ui64HWPerfFilter values. */
+	IMG_UINT32  uiHWPerfStreamCount;                                 /*! Value indicating if any of the HWPerf streams has been created */
+
+	IMG_UINT32  ui32HWPerfHostFilter;      /*! Event filter for HWPerfHost stream (settable by AppHint) */
+	POS_LOCK    hLockHWPerfHostStream;     /*! Lock guarding access to HWPerfHost stream from multiple threads */
+	IMG_HANDLE  hHWPerfHostStream;         /*! TL Stream buffer for host only event stream */
+	IMG_UINT32  ui32HWPerfHostBufSize;     /*! Host side buffer size in bytes */
+	IMG_UINT32  ui32HWPerfHostLastOrdinal; /*! Ordinal of the last packet emitted in HWPerfHost TL stream.
+	                                        *  Guarded by hLockHWPerfHostStream */
+	IMG_UINT32  ui32HWPerfHostNextOrdinal; /*! Ordinal number for HWPerfHost events. Guarded by hHWPerfHostSpinLock */
+	IMG_UINT8   *pui8DeferredEvents;       /*! List of HWPerfHost events yet to be emitted in the TL stream.
+	                                        *  Events generated from atomic context are deferred "emitted"
+											*  as the "emission" code can sleep */
+	IMG_UINT16  ui16DEReadIdx;             /*! Read index in the above deferred events buffer */
+	IMG_UINT16  ui16DEWriteIdx;            /*! Write index in the above deferred events buffer */
+	void        *pvHostHWPerfMISR;         /*! MISR to emit pending/deferred events in HWPerfHost TL stream */
+	POS_SPINLOCK hHWPerfHostSpinLock;      /*! Guards data shared between an atomic & sleepable-context */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32  ui32DEHighWatermark;       /*! High watermark of deferred events buffer usage. Protected by
+	                                        *! hHWPerfHostSpinLock */
+	/* Max number of times DeferredEmission waited for an atomic-context to "finish" packet write */
+	IMG_UINT32  ui32WaitForAtomicCtxPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an atomic-context packet loss (due to too long wait for "write" finish) */
+	IMG_BOOL    bWarnedAtomicCtxPktLost;
+	/* Max number of times DeferredEmission scheduled-out to give a chance to the right-ordinal packet to be emitted */
+	IMG_UINT32  ui32WaitForRightOrdPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an packet loss (due to too long wait for right ordinal to emit) */
+	IMG_BOOL    bWarnedPktOrdinalBroke;
+#endif
+
+	void        *pvGpuFtraceData;
+
+	/* Poll data for detecting firmware fatal errors */
+	IMG_UINT32				aui32CrLastPollCount[RGXFW_THREAD_NUM];
+	IMG_UINT32				ui32KCCBCmdsExecutedLastTime;
+	IMG_BOOL				bKCCBCmdsWaitingLastTime;
+	IMG_UINT32				ui32GEOTimeoutsLastTime;
+	IMG_UINT32				ui32InterruptCountLastTime;
+	IMG_UINT32				ui32MissingInterruptsLastTime;
+
+	/* Client stall detection */
+	IMG_UINT32				ui32StalledClientMask;
+
+	IMG_BOOL				bWorkEstEnabled;
+	IMG_BOOL				bPDVFSEnabled;
+
+	void					*pvLISRData;
+	void					*pvMISRData;
+	void					*pvAPMISRData;
+	RGX_ACTIVEPM_CONF		eActivePMConf;
+
+	volatile IMG_UINT32		aui32SampleIRQCount[RGXFW_THREAD_NUM];
+
+	DEVMEM_MEMDESC			*psRGXFaultAddressMemDesc;
+
+	DEVMEM_MEMDESC			*psSLC3FenceMemDesc;
+
+	/* If we do 10 deferred memory allocations per second, then the ID would wrap around after 13 years */
+	IMG_UINT32				ui32ZSBufferCurrID;	/*!< ID assigned to the next deferred devmem allocation */
+	IMG_UINT32				ui32FreelistCurrID;	/*!< ID assigned to the next freelist */
+
+	POS_LOCK				hLockZSBuffer;		/*!< Lock to protect simultaneous access to ZSBuffers */
+	DLLIST_NODE				sZSBufferHead;		/*!< List of on-demand ZSBuffers */
+	POS_LOCK				hLockFreeList;		/*!< Lock to protect simultaneous access to Freelists */
+	DLLIST_NODE				sFreeListHead;		/*!< List of growable Freelists */
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+	PVRSRV_CLIENT_SYNC_PRIM	*psPowSyncPrim;
+
+	IMG_UINT32				ui32ActivePMReqOk;
+	IMG_UINT32				ui32ActivePMReqDenied;
+	IMG_UINT32				ui32ActivePMReqNonIdle;
+	IMG_UINT32				ui32ActivePMReqRetry;
+	IMG_UINT32				ui32ActivePMReqTotal;
+
+	IMG_HANDLE				hProcessQueuesMISR;
+
+	IMG_UINT32				ui32DeviceFlags;		/*!< Flags to track general device state */
+
+	/* GPU DVFS Table */
+	RGX_GPU_DVFS_TABLE		*psGpuDVFSTable;
+
+	/* Pointer to function returning the GPU utilisation statistics since the last
+	 * time the function was called. Supports different users at the same time.
+	 *
+	 * psReturnStats [out]: GPU utilisation statistics (active high/active low/idle/blocked)
+	 *                      in microseconds since the last time the function was called
+	 *                      by a specific user (identified by hGpuUtilUser)
+	 *
+	 * Returns PVRSRV_OK in case the call completed without errors,
+	 * some other value otherwise.
+	 */
+	PVRSRV_ERROR (*pfnGetGpuUtilStats) (PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                    IMG_HANDLE hGpuUtilUser,
+	                                    RGXFWIF_GPU_UTIL_STATS *psReturnStats);
+
+	/* Pointer to function that checks if the physical GPU IRQ
+	 * line has been asserted and clears it if so */
+	IMG_BOOL (*pfnRGXAckIrq) (struct _PVRSRV_RGXDEV_INFO_ *psDevInfo);
+
+	POS_LOCK				hGPUUtilLock;
+
+	/* Register configuration */
+	RGX_REG_CONFIG			sRegConfig;
+
+	IMG_BOOL				bRGXPowered;
+	DLLIST_NODE				sMemoryContextList;
+
+	POSWR_LOCK				hRenderCtxListLock;
+	POSWR_LOCK				hComputeCtxListLock;
+	POSWR_LOCK				hTransferCtxListLock;
+	POSWR_LOCK				hTDMCtxListLock;
+	POSWR_LOCK				hMemoryCtxListLock;
+	POSWR_LOCK				hKickSyncCtxListLock;
+
+	/* Linked list of deferred KCCB commands due to a full KCCB.
+	 * Access to members sKCCBDeferredCommandsListHead and ui32KCCBDeferredCommandsCount
+	 * are protected by the hLockKCCBDeferredCommandsList spin lock. */
+	POS_SPINLOCK			hLockKCCBDeferredCommandsList; /*!< Protects deferred KCCB commands list */
+	DLLIST_NODE				sKCCBDeferredCommandsListHead;
+	IMG_UINT32				ui32KCCBDeferredCommandsCount; /*!< No of commands in the deferred list */
+
+	/* Linked lists of contexts on this device */
+	DLLIST_NODE				sRenderCtxtListHead;
+	DLLIST_NODE				sComputeCtxtListHead;
+	DLLIST_NODE				sTDMCtxtListHead;
+	DLLIST_NODE				sKickSyncCtxtListHead;
+
+	DLLIST_NODE				sCommonCtxtListHead;
+	POSWR_LOCK				hCommonCtxtListLock;
+	IMG_UINT32				ui32CommonCtxtCurrentID;	/*!< ID assigned to the next common context */
+
+	POS_LOCK				hDebugFaultInfoLock;	/*!< Lock to protect the debug fault info list */
+	POS_LOCK				hMMUCtxUnregLock;		/*!< Lock to protect list of unregistered MMU contexts */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	POS_LOCK				hNMILock; /*!< Lock to protect NMI operations */
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32				ui32AvailablePowUnitsMask;
+	IMG_UINT32				ui32AvailableRACMask;
+#endif
+
+	RGX_LAYER_PARAMS		sLayerParams;
+
+	RGXFWIF_DM				eBPDM;					/*!< Current breakpoint data master */
+	IMG_BOOL				bBPSet;					/*!< A Breakpoint has been set */
+	POS_LOCK				hBPLock;				/*!< Lock for break point operations */
+
+	IMG_UINT32				ui32CoherencyTestsDone;
+
+	ATOMIC_T				iCCBSubmissionOrdinal; /* Rolling count used to indicate CCB submission order (all CCBs) */
+	POS_LOCK				hCCBRecoveryLock;      /* Lock to protect pvEarliestStalledClientCCB and ui32OldestSubmissionOrdinal variables */
+	void					*pvEarliestStalledClientCCB; /* Will point to cCCB command to unblock in the event of a stall */
+	IMG_UINT32				ui32OldestSubmissionOrdinal; /* Earliest submission ordinal of CCB entry found so far */
+	IMG_UINT32				ui32SLRHoldoffCounter;   /* Decremented each time health check is called until zero. SLR only happen when zero. */
+
+	POS_LOCK				hCCBStallCheckLock; /* Lock used to guard against multiple threads simultaneously checking for stalled CCBs */
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Firmware gcov buffer */
+	DEVMEM_MEMDESC			*psFirmwareGcovBufferMemDesc;      /*!< mem desc for Firmware gcov dumping buffer */
+	IMG_UINT32				ui32FirmwareGcovSize;
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* Value to store for each page size range config register in MMU4 */
+	IMG_UINT64				aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES];
+#endif
+
+
+
+	IMG_HANDLE				hTQCLISharedMem;		/*!< TQ Client Shared Mem PMR */
+	IMG_HANDLE				hTQUSCSharedMem;		/*!< TQ USC Shared Mem PMR */
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	DEVMEM_MEMDESC			*psRGXFWIfSecureBufMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfNonSecureBufMemDesc;
+#endif
+
+	/* Timer Queries */
+	IMG_UINT32				ui32ActiveQueryId;		/*!< id of the active line */
+	IMG_BOOL				bSaveStart;				/*!< save the start time of the next kick on the device*/
+	IMG_BOOL				bSaveEnd;				/*!< save the end time of the next kick on the device*/
+
+	DEVMEM_MEMDESC			*psStartTimeMemDesc;    /*!< memdesc for Start Times */
+	IMG_UINT64				*pui64StartTimeById;    /*!< CPU mapping of the above */
+
+	DEVMEM_MEMDESC			*psEndTimeMemDesc;      /*!< memdesc for End Timer */
+	IMG_UINT64				*pui64EndTimeById;      /*!< CPU mapping of the above */
+
+	IMG_UINT32				aui32ScheduledOnId[RGX_MAX_TIMER_QUERIES];	/*!< kicks Scheduled on QueryId */
+	DEVMEM_MEMDESC			*psCompletedMemDesc;	/*!< kicks Completed on QueryId */
+	IMG_UINT32				*pui32CompletedById;	/*!< CPU mapping of the above */
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	POS_LOCK				hTimerQueryLock;		/*!< lock to protect simultaneous access to timer query members */
+#endif
+
+	PVRSRV_RGXDEV_ERROR_COUNTS sErrorCounts;		/*!< struct containing device error counts */
+
+	IMG_UINT32				ui32HostSafetyEventMask;/*!< mask of the safety events handled by the driver */
+
+	RGX_CONTEXT_RESET_REASON	eLastDeviceError;	/*!< device error reported to client */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	PMR						*psGenHeapSecMem;		/*!< An allocation of secure memory mapped to
+													  the general devmem heap. The allocation is
+													  created and mapped at driver init. It's used for
+													  various purposes. See rgx_fwif_km.h for all use cases. */
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	DEVMEM_MEMDESC			*psRGXFWScratchBufMemDesc;
+#endif
+
+	RGX_FWT_LOGTYPE			eDebugDumpFWTLogType;
+
+	RGX_FW_INFO_HEADER      sFWInfoHeader;
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	IMG_UINT32              ui32TFBCLossyGroup;     /*!< TFBCCompressionControlGroup
+													  setting for those cores which support
+													  this feature. */
+#endif
+	RGXFWIF_GPU_UTIL_STATS	sGpuUtilStats;          /*!< GPU usage statistics */
+	POS_LOCK				hGpuUtilStatsLock;
+
+
+} PVRSRV_RGXDEV_INFO;
+
+
+
+typedef struct _RGX_TIMING_INFORMATION_
+{
+	/*! GPU default core clock speed in Hz */
+	IMG_UINT32			ui32CoreClockSpeed;
+
+	/*! Default SOC clock speed in Hz */
+	IMG_UINT32			ui32SOCClockSpeed;
+
+	/*! Active Power Management: GPU actively requests the host driver to be powered off */
+	IMG_BOOL			bEnableActivePM;
+
+	/*! Enable the GPU to power off internal Power Islands independently from the host driver */
+	IMG_BOOL			bEnableRDPowIsland;
+
+	/*! Active Power Management: Delay between the GPU idle and the request to the host */
+	IMG_UINT32			ui32ActivePMLatencyms;
+
+} RGX_TIMING_INFORMATION;
+
+typedef struct _RGX_DATA_
+{
+	/*! Timing information */
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo;
+} RGX_DATA;
+
+typedef enum _RGX_QUERY_TIMESTAMP_TYPE_
+{
+	RGX_QUERY_HOST_TIMESTAMP,
+	RGX_QUERY_DEVICE_TIMESTAMP,
+} RGX_QUERY_TIMESTAMP_TYPE;
+
+
+/*
+	RGX PDUMP register bank name (prefix)
+*/
+#define RGX_PDUMPREG_NAME		"RGXREG"
+
+#endif /* RGXDEVICE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxfwutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxfwutils.c
new file mode 100644
index 000000000000..25ead9588808
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxfwutils.c
@@ -0,0 +1,6703 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rogue firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Rogue firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+#include "cache_km.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxfwriscv.h"
+#include "rgx_options.h"
+#include "rgx_fwif_alignchecks.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_pdump_panics.h"
+#include "fwtrace_string.h"
+#include "rgxheapconfig.h"
+#include "pvrsrv.h"
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "rgxccb.h"
+#include "rgxcompute.h"
+#include "rgxtdmtransfer.h"
+#include "rgxpower.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+#include "rgxmem.h"
+#include "rgxmmudefs_km.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#endif
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxutils.h"
+#include "rgxtimecorr.h"
+#include "rgxfwimageutils.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "tlstream.h"
+#include "devicemem_server_utils.h"
+#include "htbserver.h"
+#include "info_page.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#ifdef __linux__
+#include <linux/kernel.h>	/* sprintf */
+#else
+#include <stdio.h>
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF)
+#include "pvr_dvfs_common.h"
+#endif
+
+
+#include "vz_vmm_pvz.h"
+#include "rgx_heaps.h"
+
+/*!
+ ******************************************************************************
+ * HWPERF
+ *****************************************************************************/
+/* Size of the Firmware L1 HWPERF buffer in bytes (2MB). Accessed by the
+ * Firmware and host driver. */
+#define RGXFW_HWPERF_L1_SIZE_MIN        (16U)
+#define RGXFW_HWPERF_L1_SIZE_DEFAULT    PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB
+#define RGXFW_HWPERF_L1_SIZE_MAX        (12288U)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#if defined(DEBUG)
+/* Catch the use of auto-increment when meta_registers_unpacked_accesses feature is
+ * present in case we ever use it. No WA exists so it must not be used */
+#define CHECK_HWBRN_68777(v) \
+	do { \
+		PVR_ASSERT(((v) & RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN) == 0); \
+	} while (0)
+#else
+#define CHECK_HWBRN_68777(v)
+#endif
+#endif
+
+/* Firmware CCB length */
+#if defined(NO_HARDWARE) && defined(PDUMP)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (10)
+#elif defined(SUPPORT_PDVFS) || defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (8)
+#else
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (5)
+#endif
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS] = {IRQ_COUNTER_STORAGE_REGS};
+#endif
+
+/* Workload Estimation Firmware CCB length */
+#define RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2   (7)
+
+/* Size of memory buffer for firmware gcov data
+ * The actual data size is several hundred kilobytes. The buffer is an order of magnitude larger. */
+#define RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE (4*1024*1024)
+
+typedef struct
+{
+	RGXFWIF_KCCB_CMD        sKCCBcmd;
+	DLLIST_NODE             sListNode;
+	PDUMP_FLAGS_T           uiPDumpFlags;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+} RGX_DEFERRED_KCCB_CMD;
+
+typedef struct
+{
+	IMG_INT32  i32Priority;
+	IMG_UINT32 ui32IsolationGroups;
+	IMG_UINT32 ui32TSPercentage;
+}RGX_QOS_DEFAULTS;
+
+#define RGX_QOS_DEFAULTS_INIT(osid) \
+	{RGX_DRIVERID_##osid##_DEFAULT_PRIORITY,\
+	RGX_DRIVERID_##osid##_DEFAULT_ISOLATION_GROUP,\
+	RGX_DRIVERID_##osid##_DEFAULT_TIME_SLICE}
+
+#if defined(PDUMP)
+/* ensure PIDs are 32-bit because a 32-bit PDump load is generated for the
+ * PID filter example entries
+ */
+static_assert(sizeof(IMG_PID) == sizeof(IMG_UINT32),
+		"FW PID filtering assumes the IMG_PID type is 32-bits wide as it "
+		"generates WRW commands for loading the PID values");
+#endif
+
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+static_assert(RGX_DRIVER_DEFAULT_TIME_SLICES_SUM <= PVRSRV_VZ_TIME_SLICE_MAX, "Invalid driverid time slice aggregate");
+#endif
+
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo);
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+static PVRSRV_ERROR _AllocateSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo, RGXFWIF_SYSINIT* psFwSysInit)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC** ppsSLC3FenceMemDesc = &psDevInfo->psSLC3FenceMemDesc;
+	IMG_UINT32 ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(
+			RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS));
+
+	PVR_DPF_ENTERED;
+
+	eError = DevmemAllocate(psDevInfo->psFirmwareMainHeap,
+			1,
+			ui32CacheLineSize,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+			"FwSLC3FenceWA",
+			ppsSLC3FenceMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	/* We need to map it so the heap for this allocation is set */
+	eError = DevmemMapToDevice(*ppsSLC3FenceMemDesc,
+							   psDevInfo->psFirmwareMainHeap,
+							   &psFwSysInit->sSLC3FenceDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsSLC3FenceMemDesc);
+		*ppsSLC3FenceMemDesc = NULL;
+	}
+
+	PVR_DPF_RETURN_RC1(eError, *ppsSLC3FenceMemDesc);
+}
+
+static void _FreeSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	DEVMEM_MEMDESC* psSLC3FenceMemDesc = psDevInfo->psSLC3FenceMemDesc;
+
+	if (psSLC3FenceMemDesc)
+	{
+		DevmemReleaseDevVirtAddr(psSLC3FenceMemDesc);
+		DevmemFree(psSLC3FenceMemDesc);
+	}
+}
+#endif
+
+static void __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
+{
+	/* This should *NOT* happen. Try to trace what caused this and avoid a NPE
+	 * with the Write/Read at the foot of the function.
+	 */
+	PVR_ASSERT((psDevInfo != NULL));
+	if (psDevInfo == NULL)
+	{
+		return;
+	}
+
+	/* Kick MTS to wake firmware. */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
+
+	/* Uncached device/IO mapping will ensure MTS kick leaves CPU, read back
+	 * will ensure it reaches the regbank via inter-connects (AXI, PCIe etc)
+	 * before continuing.
+	 */
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          ppsMemDesc      pointer to the allocation's memory descriptor
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO*  psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL bClearByMemset;
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiAllocFlags))
+	{
+		/* Under AutoVz the ZERO_ON_ALLOC flag is avoided as it causes the memory to
+		 * be allocated from a different PMR than an allocation without the flag.
+		 * When the content of an allocation needs to be recovered from physical memory
+		 * on a later driver reboot, the memory then cannot be zeroed but the allocation
+		 * addresses must still match.
+		 * If the memory requires clearing, perform a memset after the allocation. */
+		uiAllocFlags &= ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+		bClearByMemset = IMG_TRUE;
+	}
+	else
+	{
+		bClearByMemset = IMG_FALSE;
+	}
+#endif
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate %s", pszName);
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui32Size,
+							  uiAllocFlags,
+							  pszName,
+							  ppsMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate %u bytes for %s (%u)",
+				 __func__,
+				 ui32Size,
+				 pszName,
+				 eError));
+		goto fail_alloc;
+	}
+
+	if (psFwPtr)
+	{
+		eError = RGXSetFirmwareAddress(psFwPtr, *ppsMemDesc, 0, ui32DevVAFlags);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire firmware virtual address for %s (%u)",
+					 __func__,
+					 pszName,
+					 eError));
+			goto fail_fwaddr;
+		}
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	if ((bClearByMemset) || (ppvCpuPtr))
+#else
+	if (ppvCpuPtr)
+#endif
+	{
+		void *pvTempCpuPtr;
+
+		eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, &pvTempCpuPtr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to acquire CPU virtual address for %s (%u)",
+					__func__,
+					 pszName,
+					eError));
+			goto fail_cpuva;
+		}
+
+#if defined(SUPPORT_AUTOVZ)
+		if (bClearByMemset)
+		{
+			if (PVRSRV_CHECK_CPU_WRITE_COMBINE(uiAllocFlags))
+			{
+				OSCachedMemSetWMB(pvTempCpuPtr, 0, ui32Size);
+			}
+			else
+			{
+				OSDeviceMemSet(pvTempCpuPtr, 0, ui32Size);
+			}
+		}
+		if (ppvCpuPtr)
+#endif
+		{
+			*ppvCpuPtr = pvTempCpuPtr;
+		}
+#if defined(SUPPORT_AUTOVZ)
+		else
+		{
+			DevmemReleaseCpuVirtAddr(*ppsMemDesc);
+			pvTempCpuPtr = NULL;
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s set up at Fw VA 0x%x and CPU VA 0x%p with alloc flags 0x%" IMG_UINT64_FMTSPECX,
+			 __func__, pszName,
+			 (psFwPtr)   ? (psFwPtr->ui32Addr) : (0),
+			 (ppvCpuPtr) ? (*ppvCpuPtr)        : (NULL),
+			 uiAllocFlags));
+
+	return eError;
+
+fail_cpuva:
+	if (psFwPtr)
+	{
+		RGXUnsetFirmwareAddress(*ppsMemDesc);
+	}
+fail_fwaddr:
+	DevmemFree(*ppsMemDesc);
+fail_alloc:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHwPerfBufferSize
+
+@Description    Computes the effective size of the HW Perf Buffer
+@Input          ui32HWPerfFWBufSizeKB       Device Info struct
+@Return         HwPerfBufferSize
+*/ /**************************************************************************/
+static IMG_UINT32 GetHwPerfBufferSize(IMG_UINT32 ui32HWPerfFWBufSizeKB)
+{
+	IMG_UINT32 HwPerfBufferSize;
+
+	/* HWPerf: Determine the size of the FW buffer */
+	if (ui32HWPerfFWBufSizeKB == 0 ||
+			ui32HWPerfFWBufSizeKB == RGXFW_HWPERF_L1_SIZE_DEFAULT)
+	{
+		/* Under pvrsrvctl 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. Set it to the default
+		 * size, no logging.
+		 */
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_DEFAULT<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX))
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MAX<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN))
+	{
+		/* Size specified as in AppHint HWPerfFWBufSizeInKB */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Using HWPerf FW buffer size of %u KB",
+				__func__,
+				ui32HWPerfFWBufSizeKB));
+		HwPerfBufferSize = ui32HWPerfFWBufSizeKB<<10;
+	}
+	else
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MIN<<10;
+	}
+
+	return HwPerfBufferSize;
+}
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupSignatureChecks
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
+                                              DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
+                                              IMG_UINT32          ui32SigChecksBufSize,
+                                              RGXFWIF_SIGBUF_CTL* psSigBufCtl)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+								  ui32SigChecksBufSize,
+								  "FwSignatureChecks",
+								  ppsSigChecksMemDesc,
+								  &psSigBufCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	DevmemPDumpLoadMem(	*ppsSigChecksMemDesc,
+			0,
+			ui32SigChecksBufSize,
+			PDUMP_FLAGS_CONTINUOUS);
+
+	psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
+fail:
+	return eError;
+}
+#endif
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupFirmwareGcovBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupFirmwareGcovBuffer(PVRSRV_RGXDEV_INFO*			psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32FirmwareGcovBufferSize,
+		RGXFWIF_FIRMWARE_GCOV_CTL*	psFirmwareGcovCtl,
+		const IMG_CHAR*				pszBufferName)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for gcov */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32FirmwareGcovBufferSize,
+								  pszBufferName,
+								  ppsBufferMemDesc,
+								  &psFirmwareGcovCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psFirmwareGcovCtl->ui32Size = ui32FirmwareGcovBufferSize;
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+/*!
+ ******************************************************************************
+ @Function		RGXFWSetupCounterBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+ *****************************************************************************/
+static PVRSRV_ERROR RGXFWSetupCounterBuffer(PVRSRV_RGXDEV_INFO* psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32CounterDataBufferSize,
+		RGXFWIF_COUNTER_DUMP_CTL*	psCounterDumpCtl)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32CounterDataBufferSize,
+								  "FwCounterBuffer",
+								  ppsBufferMemDesc,
+								  &psCounterDumpCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psCounterDumpCtl->ui32SizeInDwords = ui32CounterDataBufferSize >> 2;
+	RGXFwSharedMemCacheOpValue(psCounterDumpCtl->ui32SizeInDwords, FLUSH);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      RGXFWSetupAlignChecks
+ @Description   This functions allocates and fills memory needed for the
+                aligns checks of the UM and KM structures shared with the
+                firmware. The format of the data in the memory is as follows:
+                    <number of elements in the KM array>
+                    <array of KM structures' sizes and members' offsets>
+                    <number of elements in the UM array>
+                    <array of UM structures' sizes and members' offsets>
+                The UM array is passed from the user side. Now the firmware is
+                responsible for filling this part of the memory. If that
+                happens the check of the UM structures will be performed
+                by the host driver on client's connect.
+                If the macro is not defined the client driver fills the memory
+                and the firmware checks for the alignment of all structures.
+ @Input			psDeviceNode
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_DEVICE_NODE *psDeviceNode,
+								RGXFWIF_DEV_VIRTADDR	*psAlignChecksDevFW)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32			aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
+	IMG_UINT32			ui32RGXFWAlignChecksTotal;
+	IMG_UINT32*			paui32AlignChecks;
+	PVRSRV_ERROR		eError;
+
+	/* In this case we don't know the number of elements in UM array.
+	 * We have to assume something so we assume RGXFW_ALIGN_CHECKS_UM_MAX.
+	 */
+	ui32RGXFWAlignChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
+	                            + RGXFW_ALIGN_CHECKS_UM_MAX * sizeof(IMG_UINT32)
+	                            + 2 * sizeof(IMG_UINT32);
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32RGXFWAlignChecksTotal,
+								  "FwAlignmentChecks",
+								  &psDevInfo->psRGXFWAlignChecksMemDesc,
+								  psAlignChecksDevFW,
+								  (void**) &paui32AlignChecks,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Copy the values */
+		*paui32AlignChecks++ = ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+		OSCachedMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0],
+		                sizeof(aui32RGXFWAlignChecksKM));
+		paui32AlignChecks += ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+
+		*paui32AlignChecks = 0;
+
+		OSWriteMemoryBarrier(paui32AlignChecks);
+		RGXFwSharedMemCacheOpExec(paui32AlignChecks - (ARRAY_SIZE(aui32RGXFWAlignChecksKM) + 1),
+		                          ui32RGXFWAlignChecksTotal,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+
+
+	DevmemPDumpLoadMem(	psDevInfo->psRGXFWAlignChecksMemDesc,
+						0,
+						ui32RGXFWAlignChecksTotal,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	if (psDevInfo->psRGXFWAlignChecksMemDesc != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
+		psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
+	}
+}
+
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+						   DEVMEM_MEMDESC		*psSrc,
+						   IMG_UINT32			uiExtraOffset,
+						   IMG_UINT32			ui32Flags)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	psDevVirtAddr;
+	PVRSRV_DEVICE_NODE	*psDeviceNode;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+
+	psDeviceNode = (PVRSRV_DEVICE_NODE *) DevmemGetConnection(psSrc);
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		IMG_UINT32          ui32Offset;
+		IMG_BOOL            bCachedInMETA;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+		IMG_UINT32          uiGPUCacheMode;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in META memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the META cache flags */
+		bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		/* Honour the SLC cache flags */
+		eError = DevmemDeviceCacheMode(uiDevFlags, &uiGPUCacheMode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemDeviceCacheMode", failDevCacheMode);
+
+		/*
+		 * Choose Meta virtual address based on Meta and SLC cacheability.
+		 */
+		ui32Offset += RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+		if (bCachedInMETA)
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+		}
+
+		if (PVRSRV_CHECK_GPU_CACHED(uiGPUCacheMode))
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_GOTO_IF_ERROR(eError, failDevVAAcquire);
+
+		ppDest->ui32Addr = (IMG_UINT32)((psDevVirtAddr.uiAddr + uiExtraOffset) & 0xFFFFFFFF);
+	}
+#endif
+	else
+	{
+		IMG_UINT32      ui32Offset;
+		IMG_BOOL        bCachedInRISCV;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in RISCV memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the RISCV cache flags */
+		bCachedInRISCV = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		if (bCachedInRISCV)
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_CACHED_DATA_BASE;
+		}
+		else
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_UNCACHED_DATA_BASE;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+
+	if ((ppDest->ui32Addr & 0x3U) != 0)
+	{
+		IMG_CHAR *pszAnnotation;
+		/* It is expected that the annotation returned by DevmemGetAnnotation() is always valid */
+		DevmemGetAnnotation(psSrc, &pszAnnotation);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: %s @ 0x%x is not aligned to 32 bit",
+				 __func__, pszAnnotation, ppDest->ui32Addr));
+
+		return PVRSRV_ERROR_INVALID_ALIGNMENT;
+	}
+
+	if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
+	{
+		DevmemReleaseDevVirtAddr(psSrc);
+	}
+
+	return PVRSRV_OK;
+
+failDevCacheMode:
+	DevmemReleaseDevVirtAddr(psSrc);
+failDevVAAcquire:
+	return eError;
+}
+
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	sDevVirtAddr;
+
+	eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
+	psDest->psDevVirtAddr.uiAddr += uiOffset;
+	psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
+
+	DevmemReleaseDevVirtAddr(psSrcMemDesc);
+}
+
+
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
+{
+	DevmemReleaseDevVirtAddr(psSrc);
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCB
+ @Description	Free the kernel or firmware CCB
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppvCCBCtlLocal
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+******************************************************************************/
+static void RGXFreeCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtl,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+					   DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+					   IMG_UINT8			**ppui8CCB,
+					   DEVMEM_MEMDESC		**ppsCCBMemDesc)
+{
+	if (*ppsCCBMemDesc != NULL)
+	{
+		if (*ppui8CCB != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBMemDesc);
+			*ppui8CCB = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBMemDesc);
+		*ppsCCBMemDesc = NULL;
+	}
+	if (*ppsCCBCtlMemDesc != NULL)
+	{
+		if (*ppsCCBCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBCtlMemDesc);
+			*ppsCCBCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBCtlMemDesc);
+		*ppsCCBCtlMemDesc = NULL;
+	}
+	if (*ppsCCBCtlLocal != NULL)
+	{
+		OSFreeMem(*ppsCCBCtlLocal);
+		*ppsCCBCtlLocal = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCBReturnSlots
+ @Description	Free the kernel CCB's return slot array and associated mappings
+ @Input			psDevInfo              Device Info struct
+ @Input			ppui32CCBRtnSlots      CPU mapping of slot array
+ @Input			ppsCCBRtnSlotsMemDesc  Slot array's device memdesc
+******************************************************************************/
+static void RGXFreeCCBReturnSlots(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32         **ppui32CCBRtnSlots,
+								  DEVMEM_MEMDESC     **ppsCCBRtnSlotsMemDesc)
+{
+	/* Free the return slot array if allocated */
+	if (*ppsCCBRtnSlotsMemDesc != NULL)
+	{
+		/* Before freeing, ensure the CPU mapping as well is released */
+		if (*ppui32CCBRtnSlots != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBRtnSlotsMemDesc);
+			*ppui32CCBRtnSlots = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBRtnSlotsMemDesc);
+		*ppsCCBRtnSlotsMemDesc = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXSetupCCB
+ @Description	Allocate and initialise a circular command buffer
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppui8CCB
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+ @Input			ui32NumCmdsLog2
+ @Input			ui32CmdSize
+ @Input			uiCCBMemAllocFlags
+ @Input			pszName
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								RGXFWIF_CCB_CTL		**ppsCCBCtl,
+								RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+								DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+								IMG_UINT8			**ppui8CCB,
+								DEVMEM_MEMDESC		**ppsCCBMemDesc,
+								PRGXFWIF_CCB_CTL	*psCCBCtlFWAddr,
+								PRGXFWIF_CCB		*psCCBFWAddr,
+								IMG_UINT32			ui32NumCmdsLog2,
+								IMG_UINT32			ui32CmdSize,
+								PVRSRV_MEMALLOCFLAGS_T uiCCBMemAllocFlags,
+								const IMG_CHAR		*pszName)
+{
+	PVRSRV_ERROR		eError;
+	RGXFWIF_CCB_CTL		*psCCBCtl;
+	IMG_UINT32		ui32CCBSize = (1U << ui32NumCmdsLog2);
+	IMG_CHAR		szCCBCtlName[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_INT32		iStrLen;
+
+	/* Append "Control" to the name for the control struct. */
+	iStrLen = OSSNPrintf(szCCBCtlName, sizeof(szCCBCtlName), "%sControl", pszName);
+	PVR_ASSERT(iStrLen < sizeof(szCCBCtlName));
+
+	if (unlikely(iStrLen < 0))
+	{
+		OSStringSafeCopy(szCCBCtlName, "FwCCBControl", DEVMEM_ANNOTATION_MAX_LEN);
+	}
+
+	/* Allocate memory for the CCB control.*/
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_CCB_CTL),
+								  szCCBCtlName,
+								  ppsCCBCtlMemDesc,
+								  psCCBCtlFWAddr,
+								  (void**) ppsCCBCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Allocate memory for the CCB.
+	 * (this will reference further command data in non-shared CCBs)
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  uiCCBMemAllocFlags,
+								  ui32CCBSize * ui32CmdSize,
+								  pszName,
+								  ppsCCBMemDesc,
+								  psCCBFWAddr,
+								  (void**) ppui8CCB,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Initialise the CCB control.
+	 */
+	psCCBCtl = OSAllocZMem(sizeof(*psCCBCtl));
+	PVR_LOG_GOTO_IF_NOMEM(psCCBCtl, eError, fail);
+
+	psCCBCtl->ui32WrapMask = ui32CCBSize - 1;
+
+	OSDeviceMemCopy(*ppsCCBCtl, psCCBCtl, sizeof(*psCCBCtl));
+	RGXFwSharedMemCacheOpPtr(psCCBCtl, FLUSH);
+
+	*ppsCCBCtlLocal = psCCBCtl;
+
+	/* Pdump the CCB control */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Initialise %s", szCCBCtlName);
+	DevmemPDumpLoadMem(*ppsCCBCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCB_CTL),
+					   0);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeCCB(psDevInfo,
+			   ppsCCBCtl,
+			   ppsCCBCtlLocal,
+			   ppsCCBCtlMemDesc,
+			   ppui8CCB,
+			   ppsCCBMemDesc);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+#endif
+	{
+		if (psDevInfo->psRGXFaultAddressMemDesc)
+		{
+			if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR) == PVRSRV_OK)
+			{
+				PMRUnlockSysPhysAddresses(psPMR);
+			}
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+			psDevInfo->psRGXFaultAddressMemDesc = NULL;
+		}
+	}
+}
+
+static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE *psDeviceNode, RGXFWIF_SYSINIT *psFwSysInit)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			*pui32MemoryVirtAddr;
+	IMG_UINT32			i;
+	size_t				ui32PageSize = OSGetPageSize();
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PMR					*psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	/* Allocate page of memory to use for page faults on non-blocking memory transactions.
+	 * Doesn't need to be cleared as it is initialised with the 0xDEADBEEF pattern below. */
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+			ui32PageSize,
+			ui32PageSize,
+			RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS & ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+			"FwExFaultAddress",
+			&psDevInfo->psRGXFaultAddressMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate mem for fault address (%u)",
+		         __func__, eError));
+		goto failFaultAddressDescAlloc;
+	}
+
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
+										  (void **)&pui32MemoryVirtAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire mem for fault address (%u)",
+					 __func__, eError));
+			goto failFaultAddressDescAqCpuVirt;
+		}
+
+		/* fill the page with a known pattern when booting the firmware */
+		for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
+		}
+
+		OSWriteMemoryBarrier(pui32MemoryVirtAddr);
+		RGXFwSharedMemCacheOpExec(pui32MemoryVirtAddr, ui32PageSize, PVRSRV_CACHE_OP_FLUSH);
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
+	}
+
+	eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Error getting PMR for fault address (%u)",
+		         __func__, eError));
+
+		goto failFaultAddressDescGetPMR;
+	}
+	else
+	{
+		IMG_BOOL bValid;
+		IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error locking physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescLockPhys;
+		}
+
+		eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize, 1, 0, &(psFwSysInit->sFaultPhysAddr), &bValid, DEVICE_USE);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error getting physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescGetPhys;
+		}
+
+		if (!bValid)
+		{
+			psFwSysInit->sFaultPhysAddr.uiAddr = 0;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed getting physical address for fault address MemDesc - invalid page (0x%" IMG_UINT64_FMTSPECX ")",
+			         __func__, psFwSysInit->sFaultPhysAddr.uiAddr));
+
+			goto failFaultAddressDescGetPhys;
+		}
+	}
+
+	return PVRSRV_OK;
+
+failFaultAddressDescGetPhys:
+	PMRUnlockSysPhysAddresses(psPMR);
+
+failFaultAddressDescLockPhys:
+failFaultAddressDescGetPMR:
+failFaultAddressDescAqCpuVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+
+failFaultAddressDescAlloc:
+
+	return eError;
+}
+
+#if defined(PDUMP)
+/* Replace the DevPhy address with the one Pdump allocates at pdump_player run time */
+static PVRSRV_ERROR RGXPDumpFaultReadRegister(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	PMR *psFWInitPMR, *psFaultAddrPMR;
+	IMG_UINT32 ui32Dstoffset;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	psFWInitPMR = (PMR *)(psDevInfo->psRGXFWIfSysInitMemDesc->psImport->hPMR);
+	ui32Dstoffset = psDevInfo->psRGXFWIfSysInitMemDesc->uiOffset + offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr.uiAddr);
+
+	psFaultAddrPMR = (PMR *)(psDevInfo->psRGXFaultAddressMemDesc->psImport->hPMR);
+
+	eError = PDumpMemLabelToMem64(psFaultAddrPMR,
+				psFWInitPMR,
+				0,
+				ui32Dstoffset,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dump of Fault Page Phys address failed(%u)", __func__, eError));
+	}
+	return eError;
+}
+#endif
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW tbi buffer
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTBIBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTBIBufferMemDesc);
+	psDevInfo->psRGXFWIfTBIBufferMemDesc = NULL;
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = 0;
+	psDevInfo->ui32RGXL2HWPerfBufSize = 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferInitOnDemandResources
+
+@Description    Allocates the firmware TBI buffer required for reading SFs
+		strings and initialize it with SFs.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         i, ui32Len;
+	const IMG_UINT32   ui32FWTBIBufsize = g_ui32SFsCount * sizeof(RGXFW_STID_FMT);
+	RGXFW_STID_FMT     *psFW_SFs = NULL;
+
+	/* Firmware address should not be already set */
+	if (psDevInfo->sRGXFWIfTBIBuffer.ui32Addr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW address for FWTBI is already set. Resetting it with newly allocated one",
+		         __func__));
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS,
+								  ui32FWTBIBufsize,
+								  "FwTBIBuffer",
+								  &psDevInfo->psRGXFWIfTBIBufferMemDesc,
+								  &psDevInfo->sRGXFWIfTBIBuffer,
+								  (void**)&psFW_SFs,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* Copy SFs entries to FW buffer */
+	for (i = 0; i < g_ui32SFsCount; i++)
+	{
+		OSDeviceMemCopy(&psFW_SFs[i].ui32Id, &SFs[i].ui32Id, sizeof(SFs[i].ui32Id));
+		ui32Len = OSStringLength(SFs[i].psName);
+		OSDeviceMemCopy(psFW_SFs[i].sName, SFs[i].psName, MIN(ui32Len, IMG_SF_STRING_MAX_SIZE - 1));
+	}
+
+	/* flush write buffers for psFW_SFs */
+	OSWriteMemoryBarrier(psFW_SFs);
+
+	/* Set size of TBI buffer */
+	psDevInfo->ui32FWIfTBIBufferSize = ui32FWTBIBufsize;
+
+	/* release CPU mapping */
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTBIBufferMemDesc);
+
+	return PVRSRV_OK;
+fail:
+	RGXTBIBufferDeinit(psDevInfo);
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW trace buffer(s)
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTraceBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufferMemDesc[i])
+		{
+			if (psDevInfo->apui32TraceBuffer[i] != NULL)
+			{
+				DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+				psDevInfo->apui32TraceBuffer[i] = NULL;
+			}
+
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+			psDevInfo->psRGXFWIfTraceBufferMemDesc[i] = NULL;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferInitOnDemandResources
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+		info from the firmware.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo,
+												 PVRSRV_MEMALLOCFLAGS_T uiAllocFlags)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         ui32FwThreadNum;
+	IMG_UINT32         ui32DefaultTraceBufSize;
+	IMG_DEVMEM_SIZE_T  uiTraceBufSizeInBytes;
+	void               *pvAppHintState = NULL;
+	IMG_CHAR           pszBufferName[] = "FwTraceBuffer_Thread0";
+
+	/* Check AppHint value for module-param FWTraceBufSizeInDWords */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32DefaultTraceBufSize = RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,
+						 pvAppHintState,
+						 FWTraceBufSizeInDWords,
+						 &ui32DefaultTraceBufSize,
+						 &psDevInfo->ui32TraceBufSizeInDWords);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/* Write tracebuf size once to devmem */
+	psTraceBufCtl->ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (psDevInfo->ui32TraceBufSizeInDWords < RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS ||
+		psDevInfo->ui32TraceBufSizeInDWords > RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Requested trace buffer size (%u) out of its minimum (%u) & maximum (%u) range. Exiting error.",
+				  __func__,
+				  psDevInfo->ui32TraceBufSizeInDWords,
+				  RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS,
+				  RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS));
+		eError = PVRSRV_ERROR_OUT_OF_RANGE;
+		goto exit_error;
+	}
+
+	uiTraceBufSizeInBytes = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+
+	for (ui32FwThreadNum = 0; ui32FwThreadNum < RGXFW_THREAD_NUM; ui32FwThreadNum++)
+	{
+#if !defined(SUPPORT_AUTOVZ)
+		/* Ensure allocation API is only called when not already allocated */
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum] == NULL);
+		/* Firmware address should not be already set */
+		PVR_ASSERT(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer.ui32Addr == 0x0);
+#endif
+
+		/* update the firmware thread number in the Trace Buffer's name */
+		pszBufferName[sizeof(pszBufferName) - 2] += ui32FwThreadNum;
+
+		eError = RGXSetupFwAllocation(psDevInfo,
+									  uiAllocFlags,
+									  uiTraceBufSizeInBytes,
+									  pszBufferName,
+									  &psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
+									  &psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer,
+									  (void**)&psDevInfo->apui32TraceBuffer[ui32FwThreadNum],
+									  RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+	}
+
+	return PVRSRV_OK;
+
+fail:
+	RGXTraceBufferDeinit(psDevInfo);
+exit_error:
+	return eError;
+}
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       RGXPDumpLoadFWInitData
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+                info from the firmware.
+
+@Input          psDevInfo RGX device info
+ */ /*************************************************************************/
+static void RGXPDumpLoadFWInitData(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   RGX_INIT_APPHINTS  *psApphints,
+								   IMG_UINT32         ui32HWPerfCountersDataSize)
+{
+	IMG_UINT32 ui32ConfigFlags  = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+	IMG_UINT32 ui32FwOsCfgFlags = psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Dump RGXFW Init data");
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "(to enable rgxfw signatures place the following line after the RTCONF line)");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, asSigBufCtl),
+						   sizeof(RGXFWIF_SIGBUF_CTL)*(RGXFWIF_DM_MAX),
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump initial state of FW runtime configuration");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_RUNTIME_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw hwperfctl structure");
+	DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+							0,
+							ui32HWPerfCountersDataSize,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw trace control structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TRACEBUF),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware system data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware OS data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgx TBI buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTBIBufferMemDesc,
+					   0,
+					   psDevInfo->ui32FWIfTBIBufferSize,
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw register configuration buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRegCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_REG_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_USER_REGISTER_CONFIGURATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw system init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw os init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfOsInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* RGXFW Init structure needs to be loaded before we overwrite FaultPhysAddr, else this address patching won't have any effect */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Overwrite FaultPhysAddr of FwSysInit in pdump with actual physical address");
+	RGXPDumpFaultReadRegister(psDevInfo);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "RTCONF: run-time configuration");
+
+	/* Dump the config options so they can be edited. */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW system config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Rand mode:                      0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Soft Reset Enable:              0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable HWPerf:                             0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Rascal+Dust Power Island:                  0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( FBCDC Version 3.1 Enable:                  0x%08x)", RGXFWIF_INICFG_FBCDC_V3_1_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Check MList:                               0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable Auto Clock Gating:                 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+				 "( Try overlapping DM pipelines:              0x%08x)", RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable DM pipeline roadblocks:             0x%08x)", RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#endif
+				 "( Enable register configuration:             0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on TA Out-of-Memory:                0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable HWPerf counter filter:             0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable Ctx Switch profile mode: 0x%08x (none=b'000, fast=b'001, medium=b'010, slow=b'011, nodelay=b'100))", RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on HWR trigger (page fault, lockup, overrun or poll failure): 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ICS Fault Injection:                       0x%08x)", RGXFWIF_INICFG_INJECT_ICS_FAULT);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable IRQ validation:                     0x%08x)", RGXFWIF_INICFG_VALIDATE_IRQ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPU power state mask change Enable:        0x%08x)", RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN);
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Workload Estimation:                0x%08x)", RGXFWIF_INICFG_WORKEST);
+#if defined(SUPPORT_PDVFS)
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Proactive DVFS:                     0x%08x)", RGXFWIF_INICFG_PDVFS);
+#endif /* defined(SUPPORT_PDVFS) */
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ISP Scheduling Mode (v1=b'01, v2=b'10):    0x%08x)", RGXFWIF_INICFG_ISPSCHEDMODE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Validate SOC & USC timers:                 0x%08x)", RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+							offsetof(RGXFWIF_SYSDATA, ui32ConfigFlags),
+							ui32ConfigFlags,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Extended FW system config options not used.)");
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW OS config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch TDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch GEOM Enable:                    0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch 3D Enable:                      0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch CDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch RDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN);
+#endif
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  2D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  TA Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  3D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch CDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch RDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM);
+#endif
+#if defined(SUPPORT_ICS)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing TDM Enable:            0x%08x)", RGXFWIF_INICFG_OS_ICS_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing GEOM Enable:           0x%08x)", RGXFWIF_INICFG_OS_ICS_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing 3D Enable:             0x%08x)", RGXFWIF_INICFG_OS_ICS_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing CDM Enable:            0x%08x)", RGXFWIF_INICFG_OS_ICS_CDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing RDM Enable:            0x%08x)", RGXFWIF_INICFG_OS_ICS_RDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable FDTI profile mode:                  0x%08x (none=b'00, long=b'01, medium=b'10, short=b'11))", RGXFWIF_INICFG_FDTI_PROFILE_MASK);
+#endif
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+							  offsetof(RGXFWIF_OSDATA, ui32FwOsConfigFlags),
+							  ui32FwOsCfgFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#if defined(SUPPORT_ICS)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,"Set the FDTI value in usec here");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+								  offsetof(RGXFWIF_SYSINIT, ui32FDTI),
+								  psDevInfo->psRGXFWIfSysInit->ui32FDTI,
+								  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	{
+		PDUMP_FLAGS_T      ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+		IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+		IMG_BOOL           bRunTimeUpdate = IMG_FALSE;
+		IMG_UINT32         ui32DstOffset = psDevInfo->psRGXFWIfRuntimeCfgMemDesc->uiOffset + offsetof(RGXFWIF_RUNTIME_CFG, ui32PowUnitsState);
+		IMG_CHAR           aszPowUnitsMaskRegVar[] = ":SYSMEM:$1";
+		IMG_CHAR           aszPowUnitsEnable[] = "RUNTIME_POW_UNITS_MASK";
+		PMR                *psPMR = (PMR *)(psDevInfo->psRGXFWIfRuntimeCfgMemDesc->psImport->hPMR);
+
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPIF(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+		                      "Load initial value power units mask in FW runtime configuration");
+		DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  ui32DstOffset,
+								  psDevInfo->psRGXFWIfRuntimeCfg->ui32PowUnitsState,
+								  ui32PDumpFlags);
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPELSE(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+			PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags, "Read initial SPU mask value from HW registers");
+			PDumpRegRead32ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_SPU_ENABLE, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, aszPowUnitsMaskRegVar, ui32AllPowUnitsMask, ui32PDumpFlags);
+			PDumpInternalVarToMemLabel(psPMR, ui32DstOffset, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDUMPFI(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+	}
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Select one or more security tests here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW private data from non-FW contexts: 0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW code from non-FW contexts:         0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from non-secure memory:          0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from secure (non-FW) memory:     0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui32SecurityTestFlags),
+							  psDevInfo->psRGXFWIfSysInit->ui32SecurityTestFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_SECURITY_VALIDATION) */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
+				 RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+				 RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, sPIDFilter.eMode),
+							  psDevInfo->psRGXFWIfSysInit->sPIDFilter.eMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter PID/DriverID list (Up to %u entries. Terminate with a zero PID))",
+				 RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
+	{
+		IMG_UINT32 i;
+
+		/* generate a few WRWs in the pdump stream as an example */
+		for (i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
+		{
+			/*
+			 * Some compilers cannot cope with the uses of offsetof() below - the specific problem being the use of
+			 * a non-const variable in the expression, which it needs to be const. Typical compiler output is
+			 * "expression must have a constant value".
+			 */
+			const IMG_DEVMEM_OFFSET_T uiPIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].uiPID);
+
+			const IMG_DEVMEM_OFFSET_T uiDriverIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].ui32DriverID);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID and DriverID pair %u)", i);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiPIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(DriverID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiDriverIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+	/*
+	 * Dump the log config so it can be edited.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the log config here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Log Type: TRACE mode using shared memory buffer: 0x00000001)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           TBI mode via external interface or sim support: 0x00000000)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           Note: TBI mode will hang on most hardware devices!)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DMA Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DMA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MISC Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MISC);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VZ Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VZ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SAFETY Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SAFETY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VERBOSE Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VERBOSE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CUSTOMER Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CUSTOMER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+							  offsetof(RGXFWIF_TRACEBUF, ui32LogType),
+							  psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Set the HWPerf Filter config here, see \"hwperfbin2jsont -h\"");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui64HWPerfFilter),
+							  psDevInfo->psRGXFWIfSysInit->ui64HWPerfFilter,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Number of registers configurations for types(byte index): pow on(%d), dust change(%d), ta(%d), 3d(%d), cdm(%d), "
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 "tla(%d), "
+#endif
+				 "tdm(%d))",
+				 RGXFWIF_REG_CFG_TYPE_PWR_ON,
+				 RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,
+				 RGXFWIF_REG_CFG_TYPE_TA,
+				 RGXFWIF_REG_CFG_TYPE_3D,
+				 RGXFWIF_REG_CFG_TYPE_CDM,
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 RGXFWIF_REG_CFG_TYPE_TLA,
+#endif
+				 RGXFWIF_REG_CFG_TYPE_TDM);
+
+	{
+		IMG_UINT32 i;
+
+		/* Write 32 bits in each iteration as required by PDUMP WRW command */
+		for (i = 0; i < RGXFWIF_REG_CFG_TYPE_ALL; i += sizeof(IMG_UINT32))
+		{
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
+									offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[i]),
+									0,
+									PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "(Set registers here: address, mask, value)");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Mask),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
+}
+#endif /* defined(PDUMP) */
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwGuardPage
+
+ @Description Allocate a Guard Page at the start of a Guest's Main Heap
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwGuardPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)),
+								  OSGetPageSize(),
+								  "FwGuardPage",
+								  &psDevInfo->psRGXFWHeapGuardPageReserveMemDesc,
+								  NULL,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwSysData
+
+ @Description Sets up all system-wide firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwSysData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									  RGX_INIT_APPHINTS        *psApphints,
+									  IMG_UINT32               ui32ConfigFlags,
+									  IMG_UINT32               ui32ConfigFlagsExt,
+									  IMG_UINT32               ui32HWPerfCountersDataSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32         ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+	RGXFWIF_SYSINIT    *psFwSysInitScratch = NULL;
+#if defined(PDUMP)
+	IMG_UINT32         ui32SignatureChecksBufSize = psApphints->ui32SignatureChecksBufSize;
+#endif
+
+	psFwSysInitScratch = OSAllocZMem(sizeof(*psFwSysInitScratch));
+	PVR_LOG_GOTO_IF_NOMEM(psFwSysInitScratch, eError, fail);
+
+	/* Sys Fw init data */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              (RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+	                              PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSINIT),
+	                              "FwSysInitStructure",
+	                              &psDevInfo->psRGXFWIfSysInitMemDesc,
+	                              NULL,
+	                              (void**) &psDevInfo->psRGXFWIfSysInit,
+	                              RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Sys Init structure allocation", fail);
+
+	/* Setup Fault read register */
+	eError = RGXSetupFaultReadRegister(psDeviceNode, psFwSysInitScratch);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Fault read register setup", fail);
+
+#if defined(SUPPORT_AUTOVZ)
+	psFwSysInitScratch->ui32VzWdgPeriod = PVR_AUTOVZ_WDG_PERIOD_MS;
+#endif
+
+#if defined(SUPPORT_ICS)
+	psFwSysInitScratch->ui32FDTI = psApphints->ui32FDTI;
+	psFwSysInitScratch->ui32ICSThreshold = psApphints->ui32ICSThreshold;
+	psFwSysInitScratch->bTestModeOn = psApphints->bTestModeOn;
+#endif
+
+	/* RD Power Island */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
+		IMG_BOOL bEnableRDPowIsland = ((psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
+		                               (psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
+
+		ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
+	}
+
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		ui32ConfigFlags |= RGXFWIF_INICFG_WORKEST;
+#endif
+#if defined(SUPPORT_FW_OPP_TABLE)
+		{
+			RGXFWIF_OPP_INFO    *psOPPInfo;
+			IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg;
+
+			/* Pro-active DVFS depends on Workload Estimation */
+			psOPPInfo = &psFwSysInitScratch->sOPPInfo;
+			psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+#if defined(CONFIG_OF) && defined(CONFIG_PM_OPP) && !defined(NO_HARDWARE)
+			if (psDVFSDeviceCfg->bDTConfig)
+			{
+				/* OPP table configured from Device tree */
+				eError = DVFSCopyOPPTable(psDeviceNode,
+										  psOPPInfo,
+										  ARRAY_SIZE(psOPPInfo->asOPPValues));
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Unable to copy OPP table to FW init buffer (%u)", eError));
+					goto fail;
+				}
+			}
+#endif
+			if (!psDVFSDeviceCfg->bDTConfig)
+			{
+				PVR_LOG_IF_FALSE(psDVFSDeviceCfg->pasOPPTable, "RGXSetupFwSysData: Missing OPP Table");
+
+				if (psDVFSDeviceCfg->pasOPPTable != NULL)
+				{
+					if (psDVFSDeviceCfg->ui32OPPTableSize > ARRAY_SIZE(psOPPInfo->asOPPValues))
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								 "%s: OPP Table too large: Size = %u, Maximum size = %lu",
+								 __func__,
+								 psDVFSDeviceCfg->ui32OPPTableSize,
+								 (unsigned long)(ARRAY_SIZE(psOPPInfo->asOPPValues))));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto fail;
+					}
+
+					OSDeviceMemCopy(psOPPInfo->asOPPValues,
+									psDVFSDeviceCfg->pasOPPTable,
+									sizeof(psOPPInfo->asOPPValues));
+
+					psOPPInfo->ui32MaxOPPPoint = psDVFSDeviceCfg->ui32OPPTableSize - 1;
+				}
+			}
+
+#if defined(SUPPORT_PDVFS)
+			ui32ConfigFlags |= RGXFWIF_INICFG_PDVFS;
+#endif
+		}
+#endif /* defined(SUPPORT_FW_OPP_TABLE) */
+	}
+
+	/* FW trace control structure */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_TRACEBUF),
+								  "FwTraceCtlStruct",
+								  &psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								  &psFwSysInitScratch->sTraceBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfTraceBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Set initial firmware log type/group(s) */
+		if (psApphints->ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid initial log type (0x%X)",
+			         __func__, psApphints->ui32LogType));
+			goto fail;
+		}
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = psApphints->ui32LogType;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+	}
+
+	/* When PDUMP is enabled, ALWAYS allocate on-demand trace buffer resource
+	 * (irrespective of loggroup(s) enabled), given that logtype/loggroups can
+	 * be set during PDump playback in logconfig, at any point of time,
+	 * Otherwise, allocate only if required. */
+#if !defined(PDUMP)
+#if defined(SUPPORT_AUTOVZ)
+	/* always allocate trace buffer for AutoVz Host drivers to allow
+	 * deterministic addresses of all SysData structures */
+	if ((PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo)) || (RGXTraceBufferIsInitRequired(psDevInfo)))
+#else
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+#endif
+#endif
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo,
+													 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+													 RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp));
+	}
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXTraceBufferInitOnDemandResources", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSDATA),
+	                              "FwSysData",
+	                              &psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                              &psFwSysInitScratch->sFwSysData,
+	                              (void**) &psDevInfo->psRGXFWIfFwSysData,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* GPIO validation setup */
+	psFwSysInitScratch->eGPIOValidationMode = RGXFWIF_GPIO_VAL_OFF;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = RGXFWSetupCounterBuffer(psDevInfo,
+									 &psDevInfo->psCounterBufferMemDesc,
+									 OSGetPageSize(),
+									 &psFwSysInitScratch->sCounterDumpCtl);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Counter Buffer allocation", fail);
+
+	PVR_DPF((PVR_DBG_WARNING, "Counter buffer allocated at %p, size %zu Bytes.", psDevInfo->psCounterBufferMemDesc, OSGetPageSize()));
+#endif /* defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = RGXFWSetupFirmwareGcovBuffer(psDevInfo,
+	                                      &psDevInfo->psFirmwareGcovBufferMemDesc,
+	                                      RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE,
+	                                      &psFwSysInitScratch->sFirmwareGcovCtl,
+	                                      "FirmwareGcovBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware GCOV buffer allocation", fail);
+	psDevInfo->ui32FirmwareGcovSize = RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE;
+#endif /* defined(SUPPORT_FIRMWARE_GCOV) */
+
+#if defined(PDUMP)
+	/* Require a minimum amount of memory for the signature buffers */
+	if (ui32SignatureChecksBufSize < RGXFW_SIG_BUFFER_SIZE_MIN)
+	{
+		ui32SignatureChecksBufSize = RGXFW_SIG_BUFFER_SIZE_MIN;
+	}
+
+	/* Setup Signature and Checksum Buffers */
+	psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	psDevInfo->ui32SigTDMChecksSize = 0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* Buffer allocated only when feature present because all known TDM
+		 * signature registers are dependent on this feature being present */
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TDM Signature check setup", fail);
+		psDevInfo->ui32SigTDMChecksSize = ui32SignatureChecksBufSize;
+	}
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GEOM Signature check setup", fail);
+	psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "3D Signature check setup", fail);
+	psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigCDMChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CDM Signature check setup", fail);
+	psDevInfo->ui32SigCDMChecksSize = ui32SignatureChecksBufSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RDM Signature check setup", fail);
+		psDevInfo->ui32SigRDMChecksSize = ui32SignatureChecksBufSize;
+	}
+#endif
+
+
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM].sBuffer.ui32Addr = 0x0;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY].sBuffer.ui32Addr = 0x0;
+#endif
+	}
+#endif /* defined(PDUMP) */
+
+	eError = RGXFWSetupAlignChecks(psDeviceNode,
+	                               &psFwSysInitScratch->sAlignChecks);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Alignment checks setup", fail);
+
+	psFwSysInitScratch->ui32FilterFlags = psApphints->ui32FilterFlags;
+
+	/* Fill the remaining bits of fw the init data */
+	psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
+	psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
+	psFwSysInitScratch->sFBCDCStateTableBase.uiAddr = RGX_FBCDC_HEAP_BASE;
+	psFwSysInitScratch->sFBCDCLargeStateTableBase.uiAddr = RGX_FBCDC_LARGE_HEAP_BASE;
+	psFwSysInitScratch->sTextureHeapBase.uiAddr = RGX_TEXTURE_STATE_HEAP_BASE;
+
+	psFwSysInitScratch->sPDSIndirectHeapBase.uiAddr = RGX_PDS_INDIRECT_STATE_HEAP_BASE;
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/* Fill the remaining bits of fw the init data */
+		psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_BRN_65273_HEAP_BASE;
+		psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_BRN_65273_HEAP_BASE;
+	}
+#endif
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT))
+	{
+		eError = _AllocateSLC3Fence(psDevInfo, psFwSysInitScratch);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SLC3Fence memory allocation", fail);
+	}
+#endif
+#if defined(SUPPORT_PDVFS)
+	/* Core clock rate */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(IMG_UINT32),
+	                              "FwPDVFSCoreClkRate",
+	                              &psDevInfo->psRGXFWIFCoreClkRateMemDesc,
+	                              &psFwSysInitScratch->sCoreClockRate,
+	                              (void**) &psDevInfo->pui32RGXFWIFCoreClkRate,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDVFS core clock rate memory setup", fail);
+#endif
+	{
+	/* Timestamps */
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags =
+		PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+		PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
+		PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/*
+	  the timer query arrays
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate timer query arrays (FW)");
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwStartTimesArray",
+	                          & psDevInfo->psStartTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64StartTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwEndTimesArray",
+	                          & psDevInfo->psEndTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64EndTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags,
+	                          "FwCompletedOpsArray",
+	                          & psDevInfo->psCompletedMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to completed ops array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
+	                                  (void **)& psDevInfo->pui32CompletedById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map completed ops array",
+				__func__));
+		goto fail;
+	}
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	eError = OSLockCreate(&psDevInfo->hTimerQueryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate log for timer query",
+				__func__));
+		goto fail;
+	}
+#endif
+#if defined(SUPPORT_TBI_INTERFACE)
+#if !defined(PDUMP)
+	/* allocate only if required */
+	if (RGXTBIBufferIsInitRequired(psDevInfo))
+#endif /* !defined(PDUMP) */
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand TBI buffer resource
+		 * (irrespective of loggroup(s) enabled), given that logtype/loggroups
+		 * can be set during PDump playback in logconfig, at any point of time
+		 */
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXTBIBufferInitOnDemandResources", fail);
+	}
+
+	psFwSysInitScratch->sTBIBuf = psDevInfo->sRGXFWIfTBIBuffer;
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+	/* Allocate shared buffer for GPU utilisation.
+	 * Enable FIRMWARE_CACHED to reduce read latency in the FW.
+	 * The FW flushes the cache after any writes.
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_GPU_UTIL_FW),
+								  "FwGPUUtilisationBuffer",
+								  &psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc,
+								  &psFwSysInitScratch->sGpuUtilFWCtl,
+								  (void**) &psDevInfo->psRGXFWIfGpuUtilFW,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GPU Utilisation Buffer ctl allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_RUNTIME_CFG),
+								  "FwRuntimeCfg",
+								  &psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  &psFwSysInitScratch->sRuntimeCfg,
+								  (void**) &psDevInfo->psRGXFWIfRuntimeCfg,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware runtime configuration memory allocation", fail);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_REG_CFG),
+								  "FwRegisterConfigStructure",
+								  &psDevInfo->psRGXFWIfRegCfgMemDesc,
+								  &psFwSysInitScratch->sRegCfg,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware register user configuration structure allocation", fail);
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS | PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  RGXFW_SCRATCH_BUF_SIZE,
+								  "FwScratchBuf",
+								  &psDevInfo->psRGXFWScratchBufMemDesc,
+								  &psFwSysInitScratch->pbFwScratchBuf,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware scratch buffer allocation", fail);
+#endif
+
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = GetHwPerfBufferSize(psApphints->ui32HWPerfFWBufSize);
+
+	/* Host L2 HWPERF buffer size in bytes must be bigger than the L1 buffer
+	 * accessed by the FW. The MISR may try to write one packet the size of the L1
+	 * buffer in some scenarios. When logging is enabled in the MISR, it can be seen
+	 * if the L2 buffer hits a full condition. The closer in size the L2 and L1 buffers
+	 * are the more chance of this happening.
+	 * Size chosen to allow MISR to write an L1 sized packet and for the client
+	 * application/daemon to drain a L1 sized packet e.g. ~ 1.5*L1.
+	 */
+	psDevInfo->ui32RGXL2HWPerfBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize +
+	        (psDevInfo->ui32RGXFWIfHWPerfBufSize>>1);
+
+	/* Second stage initialisation or HWPerf, hHWPerfLock created in first
+	 * stage. See RGXRegisterDevice() call to RGXHWPerfInit(). */
+	if (psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF] == 0)
+	{
+		psFwSysInitScratch->ui64HWPerfFilter =
+		    RGXHWPerfFwSetEventFilter(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF,
+		                              (IMG_UINT64) psApphints->ui32HWPerfFilter0 |
+		                              ((IMG_UINT64) psApphints->ui32HWPerfFilter1 << 32));
+	}
+	else
+	{
+		/* The filter has already been modified. This can happen if
+		 * pvr/apphint/EnableFTraceGPU was enabled. */
+		psFwSysInitScratch->ui64HWPerfFilter = psDevInfo->ui64HWPerfFwFilter;
+	}
+
+#if !defined(PDUMP)
+	/* Allocate if HWPerf filter has already been set. This is possible either
+	 * by setting a proper AppHint or enabling GPU ftrace events. */
+	if (psFwSysInitScratch->ui64HWPerfFilter != 0)
+#endif
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand HWPerf resources
+		 * (irrespective of HWPerf enabled or not), given that HWPerf can be
+		 * enabled during PDump playback via RTCONF at any point of time. */
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL1Buffer", fail);
+
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL2Stream", fail);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		/* Align the hwperf allocation to the DMA block size to ensure
+		 * the shared control structure buffer isn't overflowed when
+		 * Meta DMA writes to it. */
+		ui32HWPerfCountersDataSize = PVR_ALIGN(ui32HWPerfCountersDataSize, RGX_META_DMA_BLOCK_SIZE);
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWCOMCTX_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32HWPerfCountersDataSize,
+								  "FwHWPerfControlStructure",
+								  &psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+								  &psFwSysInitScratch->sHWPerfCtl,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware HW Perf control struct allocation", fail);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		RGXSetMetaDMAAddress(&psDevInfo->psRGXFWIfRuntimeCfg->sHWPerfCtlDMABuf,
+		                     psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+		                     &psFwSysInitScratch->sHWPerfCtl, 0);
+	}
+
+	psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_INICFG_DISABLE_PDP_EN)
+							  ? IMG_FALSE : IMG_TRUE;
+
+	psFwSysInitScratch->eFirmwarePerf = psApphints->eFirmwarePerf;
+
+#if defined(PDUMP)
+	/* default: no filter */
+	psFwSysInitScratch->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
+	psFwSysInitScratch->sPIDFilter.asItems[0].uiPID = 0;
+#endif
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	{
+		PVRSRV_MEMALLOCFLAGS_T uiFlags = RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS;
+		PVRSRV_SET_PHYS_HEAP_HINT(FW_PRIV_DATA, uiFlags);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate non-secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+											"FwExNonSecureBuffer",
+											&psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Non-secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbNonSecureBuffer,
+									   psDevInfo->psRGXFWIfNonSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											uiFlags,
+											"FwExSecureBuffer",
+											&psDevInfo->psRGXFWIfSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbSecureBuffer,
+									   psDevInfo->psRGXFWIfSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail);
+	}
+#endif /* SUPPORT_SECURITY_VALIDATION */
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) || RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		psFwSysInitScratch->ui32TFBCCompressionControl =
+			(ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >> RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+	}
+#endif /* RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK */
+
+	/* Initialize FW started flag */
+	psFwSysInitScratch->bFirmwareStarted = IMG_FALSE;
+	psFwSysInitScratch->ui32MarkerVal = 1;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = RGX_VZ_CONNECTION_COOLDOWN_PERIOD;
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		IMG_UINT32 ui32DriverID;
+
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+
+		/* Required info by FW to calculate the ActivePM idle timer latency */
+		psFwSysInitScratch->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psFwSysInitScratch->ui32InitialSOCClockSpeed = psRGXData->psRGXTimingInfo->ui32SOCClockSpeed;
+#endif
+		psFwSysInitScratch->ui32InitialActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
+
+		/* Initialise variable runtime configuration to the system defaults */
+		psRuntimeCfg->ui32CoreClockSpeed = psFwSysInitScratch->ui32InitialCoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psRuntimeCfg->ui32SOCClockSpeed = psFwSysInitScratch->ui32InitialSOCClockSpeed;
+#endif
+		psRuntimeCfg->ui32ActivePMLatencyms = psFwSysInitScratch->ui32InitialActivePMLatencyms;
+		psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
+		psRuntimeCfg->ui32HCSDeadlineMS = RGX_HCS_DEFAULT_DEADLINE_MS;
+
+		if ((RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US > 0U) && (RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US < 1000U))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_LOG_GOTO_IF_ERROR(eError,
+								  "RGXSetupFwSysData: RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US must be either 0 (disabled) or greater than 1000",
+								  fail);
+		}
+		else
+		{
+			psRuntimeCfg->ui32WdgPeriodUs = RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+		{
+			psRuntimeCfg->ai32DriverPriority[RGXFW_HOST_DRIVER_ID] = 0;
+			psRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID] = RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP;
+			psRuntimeCfg->aui32TSPercentage[RGXFW_HOST_DRIVER_ID] = (IMG_UINT8)RGX_DRIVERID_0_DEFAULT_TIME_SLICE;
+		}
+		else
+		{
+			const RGX_QOS_DEFAULTS asQosDefaults[RGXFW_MAX_NUM_OSIDS] = {
+					RGX_QOS_DEFAULTS_INIT(0),
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+					RGX_QOS_DEFAULTS_INIT(1),
+#if (RGXFW_MAX_NUM_OSIDS > 2)
+					RGX_QOS_DEFAULTS_INIT(2),
+					RGX_QOS_DEFAULTS_INIT(3),
+					RGX_QOS_DEFAULTS_INIT(4),
+					RGX_QOS_DEFAULTS_INIT(5),
+					RGX_QOS_DEFAULTS_INIT(6),
+					RGX_QOS_DEFAULTS_INIT(7),
+#if (RGXFW_MAX_NUM_OSIDS > 8)
+					RGX_QOS_DEFAULTS_INIT(8),
+					RGX_QOS_DEFAULTS_INIT(9),
+					RGX_QOS_DEFAULTS_INIT(10),
+					RGX_QOS_DEFAULTS_INIT(11),
+					RGX_QOS_DEFAULTS_INIT(12),
+					RGX_QOS_DEFAULTS_INIT(13),
+					RGX_QOS_DEFAULTS_INIT(14),
+					RGX_QOS_DEFAULTS_INIT(15),
+					RGX_QOS_DEFAULTS_INIT(16),
+					RGX_QOS_DEFAULTS_INIT(17),
+					RGX_QOS_DEFAULTS_INIT(18),
+					RGX_QOS_DEFAULTS_INIT(19),
+					RGX_QOS_DEFAULTS_INIT(20),
+					RGX_QOS_DEFAULTS_INIT(21),
+					RGX_QOS_DEFAULTS_INIT(22),
+					RGX_QOS_DEFAULTS_INIT(23),
+					RGX_QOS_DEFAULTS_INIT(24),
+					RGX_QOS_DEFAULTS_INIT(25),
+					RGX_QOS_DEFAULTS_INIT(26),
+					RGX_QOS_DEFAULTS_INIT(27),
+					RGX_QOS_DEFAULTS_INIT(28),
+					RGX_QOS_DEFAULTS_INIT(29),
+					RGX_QOS_DEFAULTS_INIT(30),
+					RGX_QOS_DEFAULTS_INIT(31),
+#if (RGXFW_MAX_NUM_OSIDS > 32)
+#error "Support for more than 32 OSIDs not implemented."
+#endif
+#endif /* RGXFW_MAX_NUM_OSIDS > 8 */
+#endif /* RGXFW_MAX_NUM_OSIDS > 2 */
+
+#endif /* RGXFW_MAX_NUM_OSIDS > 1 */
+			};
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				/* Set up initial priorities between different OSes */
+				psRuntimeCfg->ai32DriverPriority[ui32DriverID] = asQosDefaults[ui32DriverID].i32Priority;
+				psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = asQosDefaults[ui32DriverID].ui32IsolationGroups;
+				psRuntimeCfg->aui32TSPercentage[ui32DriverID] = (asQosDefaults[ui32DriverID].ui32TSPercentage <=
+																				PVRSRV_VZ_TIME_SLICE_MAX) ?
+																				asQosDefaults[ui32DriverID].ui32TSPercentage:(0);
+			}
+		}
+		psRuntimeCfg->ui32TSIntervalMs = RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL;
+
+#if defined(PVR_ENABLE_PHR) && defined(PDUMP)
+		psRuntimeCfg->ui32PHRMode = RGXFWIF_PHR_MODE_RD_RESET;
+#else
+		psRuntimeCfg->ui32PHRMode = 0;
+#endif
+
+		/* Validate the power units mask and initialize to number of units to power up */
+		if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+		{
+			eError = PVRSRV_ERROR_INVALID_SPU_MASK;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s:Invalid power units mask (All=0x%X, Non Fused=0x%X). At-least one power unit must to be powered up.",
+			         __func__,
+			         ui32AllPowUnitsMask,
+			         psApphints->ui32AvailablePowUnitsMask));
+			goto fail;
+		}
+		psRuntimeCfg->ui32PowUnitsState = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+		psRuntimeCfg->ui32RACUnitsState = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+
+		/* flush write buffers for psDevInfo->psRGXFWIfRuntimeCfg */
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfRuntimeCfg);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, FLUSH);
+
+		/* Setup FW coremem data */
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psFwSysInitScratch->sCorememDataStore.pbyFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+			{
+				RGXSetMetaDMAAddress(&psFwSysInitScratch->sCorememDataStore,
+									 psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+									 &psFwSysInitScratch->sCorememDataStore.pbyFWAddr,
+									 0);
+			}
+		}
+
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags    = ui32ConfigFlags    & RGXFWIF_INICFG_ALL;
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt = ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_ALL;
+
+		/* Initialise GPU utilisation buffer */
+		{
+			IMG_UINT64 ui64LastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(), RGXFWIF_GPU_UTIL_STATE_IDLE);
+			RGXFWIF_DM eDM;
+			IMG_UINT64 ui64LastWordTimeShifted =
+			        RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64() >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT, RGXFWIF_GPU_UTIL_STATE_IDLE);
+			IMG_UINT32 ui32DriverID;
+
+			psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord = ui64LastWord;
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				RGXFWIF_GPU_STATS *psStats = &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID];
+
+				for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+				{
+					psStats->aui32DMOSLastWord[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted & IMG_UINT32_MAX);
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted >> 32);
+				}
+			}
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+		}
+
+		/* init HWPERF data */
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfUt = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfDropCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32FirstDropOrdinal = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32LastDropOrdinal = 0;
+
+		psDevInfo->psRGXFWIfFwSysData->ui32MemFaultCheck = 0;
+
+		// flush write buffers for psRGXFWIfFwSysData
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwSysData);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, FLUSH);
+
+		/*Send through the BVNC Feature Flags*/
+		eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, &psFwSysInitScratch->sBvncKmFeatureFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags", fail);
+
+		/* populate the real FwOsInit structure with the values stored in the scratch copy */
+		OSCachedMemCopyWMB(psDevInfo->psRGXFWIfSysInit, psFwSysInitScratch, sizeof(RGXFWIF_SYSINIT));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit,
+		                         FLUSH);
+	}
+
+	OSFreeMem(psFwSysInitScratch);
+
+
+	return PVRSRV_OK;
+
+fail:
+	if (psFwSysInitScratch)
+	{
+		OSFreeMem(psFwSysInitScratch);
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwOsData
+
+ @Description Sets up all os-specific firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwOsData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									 IMG_UINT32               ui32KCCBSizeLog2,
+									 IMG_UINT32               ui32HWRDebugDumpLimit,
+									 IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_OSINIT     sFwOsInitScratch;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSCachedMemSet(&sFwOsInitScratch, 0, sizeof(RGXFWIF_OSINIT));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXSetupFwGuardPage(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware heap guard pages", fail);
+	}
+
+	/* Memory tracking the connection state should be non-volatile and
+	 * is not cleared on allocation to prevent loss of pre-reset information */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS &
+								  ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+								  sizeof(RGXFWIF_CONNECTION_CTL),
+								  "FwConnectionCtl",
+								  &psDevInfo->psRGXFWIfConnectionCtlMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfConnectionCtl,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Connection Control structure allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED),
+								  sizeof(RGXFWIF_OSINIT),
+								  "FwOsInitStructure",
+								  &psDevInfo->psRGXFWIfOsInitMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfOsInit,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Os Init structure allocation", fail);
+
+	/* init HWR frame info */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_HWRINFOBUF),
+								  "FwHWRInfoBuffer",
+								  &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
+								  &sFwOsInitScratch.sRGXFWIfHWRInfoBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfHWRInfoBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HWR Info Buffer allocation", fail);
+
+	OSCachedMemSetWMB(psDevInfo->psRGXFWIfHWRInfoBufCtl, 0, sizeof(RGXFWIF_HWRINFOBUF));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, FLUSH);
+
+	/* Allocate a sync for power management */
+	eError = SyncPrimContextCreate(psDevInfo->psDeviceNode,
+	                               &psDevInfo->hSyncPrimContext);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive context allocation", fail);
+
+	eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive allocation", fail);
+
+	/* Set up kernel CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psKernelCCBCtl,
+						 &psDevInfo->psKernelCCBCtlLocal,
+						 &psDevInfo->psKernelCCBCtlMemDesc,
+						 &psDevInfo->psKernelCCB,
+						 &psDevInfo->psKernelCCBMemDesc,
+						 &sFwOsInitScratch.psKernelCCBCtl,
+						 &sFwOsInitScratch.psKernelCCB,
+						 ui32KCCBSizeLog2,
+						 sizeof(RGXFWIF_KCCB_CMD),
+						 (RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS |
+						 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+						 "FwKernelCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB allocation", fail);
+
+	/* KCCB additionally uses a return slot array for FW to be able to send back
+	 * return codes for each required command
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  (1U << ui32KCCBSizeLog2) * sizeof(IMG_UINT32),
+								  "FwKernelCCBRtnSlots",
+								  &psDevInfo->psKernelCCBRtnSlotsMemDesc,
+								  &sFwOsInitScratch.psKernelCCBRtnSlots,
+								  (void**) &psDevInfo->pui32KernelCCBRtnSlots,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB return slot array allocation", fail);
+
+	/* Set up firmware CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psFirmwareCCBCtl,
+						 &psDevInfo->psFirmwareCCBCtlLocal,
+						 &psDevInfo->psFirmwareCCBCtlMemDesc,
+						 &psDevInfo->psFirmwareCCB,
+						 &psDevInfo->psFirmwareCCBMemDesc,
+						 &sFwOsInitScratch.psFirmwareCCBCtl,
+						 &sFwOsInitScratch.psFirmwareCCB,
+						 RGXFWIF_FWCCB_NUMCMDS_LOG2,
+						 sizeof(RGXFWIF_FWCCB_CMD),
+						 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+						 "FwCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware CCB allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_OSDATA),
+								  "FwOsData",
+								  &psDevInfo->psRGXFWIfFwOsDataMemDesc,
+								  &sFwOsInitScratch.sFwOsData,
+								  (void**) &psDevInfo->psRGXFWIfFwOsData,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags = ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_ALL;
+
+	eError = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim, &psDevInfo->psRGXFWIfFwOsData->sPowerSync.ui32Addr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Get Sync Prim FW address", fail);
+
+	/* flush write buffers for psRGXFWIfFwOsData */
+	OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwOsData);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         FLUSH);
+
+	sFwOsInitScratch.ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Set up Workload Estimation firmware CCB */
+		eError = RGXSetupCCB(psDevInfo,
+							 &psDevInfo->psWorkEstFirmwareCCBCtl,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+							 &psDevInfo->psWorkEstFirmwareCCB,
+							 &psDevInfo->psWorkEstFirmwareCCBMemDesc,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCBCtl,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCB,
+							 RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2,
+							 sizeof(RGXFWIF_WORKEST_FWCCB_CMD),
+							 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+							 "FwWEstCCB");
+		PVR_LOG_GOTO_IF_ERROR(eError, "Workload Estimation Firmware CCB allocation", fail);
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise the compatibility check data */
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sFWBVNC);
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sHWBVNC);
+	}
+
+	/* populate the real FwOsInit structure with the values stored in the scratch copy */
+	OSCachedMemCopyWMB(psDevInfo->psRGXFWIfOsInit, &sFwOsInitScratch, sizeof(RGXFWIF_OSINIT));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfOsInit,
+	                         FLUSH);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeFwOsData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFirmware
+
+ @Description Sets up all firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+							  RGX_INIT_APPHINTS        *psApphints,
+							  IMG_UINT32               ui32ConfigFlags,
+							  IMG_UINT32               ui32ConfigFlagsExt,
+							  IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32NumHWPerfBlocks;
+	IMG_UINT32 ui32HWPerfCountersDataSize;
+
+	eError = RGXSetupFwOsData(psDeviceNode,
+							  psApphints->ui32KCCBSizeLog2,
+							  psApphints->ui32HWRDebugDumpLimit,
+							  ui32FwOsCfgFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware os data", fail);
+
+	ui32NumHWPerfBlocks = RGXHWPerfMaxDefinedBlks((void *)psDevInfo);
+	ui32HWPerfCountersDataSize = sizeof(RGXFWIF_HWPERF_CTL) +
+		(ui32NumHWPerfBlocks - 1) * sizeof(RGXFWIF_HWPERF_CTL_BLK);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest drivers do not configure system-wide firmware data */
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+	else
+	{
+		/* Native and Host drivers must initialise the firmware's system data */
+		eError = RGXSetupFwSysData(psDeviceNode,
+								   psApphints,
+								   ui32ConfigFlags,
+								   ui32ConfigFlagsExt,
+								   ui32HWPerfCountersDataSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware system data", fail);
+	}
+
+	psDevInfo->bFirmwareInitialised = IMG_TRUE;
+
+#if defined(PDUMP)
+	RGXPDumpLoadFWInitData(psDevInfo,
+	                       psApphints,
+	                       ui32HWPerfCountersDataSize);
+#endif /* PDUMP */
+
+fail:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwSysData
+
+ @Description Frees all system-wide firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	psDevInfo->bFirmwareInitialised = IMG_FALSE;
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc)
+	{
+		RGXFWFreeAlignChecks(psDevInfo);
+	}
+
+#if defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM) &&
+	    psDevInfo->psRGXFWSigTDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTDMChecksMemDesc);
+		psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigTAChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTAChecksMemDesc);
+		psDevInfo->psRGXFWSigTAChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSig3DChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSig3DChecksMemDesc);
+		psDevInfo->psRGXFWSig3DChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigCDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigCDMChecksMemDesc);
+		psDevInfo->psRGXFWSigCDMChecksMemDesc = NULL;
+	}
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1 &&
+	   psDevInfo->psRGXFWSigRDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigRDMChecksMemDesc);
+		psDevInfo->psRGXFWSigRDMChecksMemDesc = NULL;
+	}
+#endif
+
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	if (psDevInfo->psCounterBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCounterBufferMemDesc);
+		psDevInfo->psCounterBufferMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	if (psDevInfo->psFirmwareGcovBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psFirmwareGcovBufferMemDesc);
+		psDevInfo->psFirmwareGcovBufferMemDesc = NULL;
+	}
+#endif
+
+	RGXSetupFaultReadRegisterRollback(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfGpuUtilFW != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+			psDevInfo->psRGXFWIfGpuUtilFW = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+		psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfRuntimeCfg != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+			psDevInfo->psRGXFWIfRuntimeCfg = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+		psDevInfo->psRGXFWIfRuntimeCfgMemDesc = NULL;
+	}
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+			psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+		}
+	}
+
+	if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufCtl != NULL)
+		{
+			/* deinit/free the tracebuffer allocation */
+			RGXTraceBufferDeinit(psDevInfo);
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+			psDevInfo->psRGXFWIfTraceBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+		psDevInfo->psRGXFWIfTraceBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwSysDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwSysData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwSysDataMemDesc);
+			psDevInfo->psRGXFWIfFwSysData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwSysDataMemDesc);
+		psDevInfo->psRGXFWIfFwSysDataMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	if (psDevInfo->psRGXFWIfTBIBufferMemDesc)
+	{
+		RGXTBIBufferDeinit(psDevInfo);
+	}
+#endif
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	if (psDevInfo->psRGXFWIfRegCfgMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRegCfgMemDesc);
+		psDevInfo->psRGXFWIfRegCfgMemDesc = NULL;
+	}
+#endif
+	if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		psDevInfo->psRGXFWIfHWPerfCountersMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDevInfo->psRGXFWIfNonSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		psDevInfo->psRGXFWIfNonSecureBufMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSecureBufMemDesc);
+		psDevInfo->psRGXFWIfSecureBufMemDesc = NULL;
+	}
+#endif
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT))
+	{
+		_FreeSLC3Fence(psDevInfo);
+	}
+#endif
+#if defined(SUPPORT_PDVFS)
+	if (psDevInfo->psRGXFWIFCoreClkRateMemDesc)
+	{
+		if (psDevInfo->pui32RGXFWIFCoreClkRate != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+			psDevInfo->pui32RGXFWIFCoreClkRate = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+		psDevInfo->psRGXFWIFCoreClkRateMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+		psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwOsData
+
+ @Description Frees all os-specific firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFreeCCBReturnSlots(psDevInfo,
+	                      &psDevInfo->pui32KernelCCBRtnSlots,
+	                      &psDevInfo->psKernelCCBRtnSlotsMemDesc);
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psKernelCCBCtl,
+	           &psDevInfo->psKernelCCBCtlLocal,
+	           &psDevInfo->psKernelCCBCtlMemDesc,
+	           &psDevInfo->psKernelCCB,
+	           &psDevInfo->psKernelCCBMemDesc);
+
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psFirmwareCCBCtl,
+	           &psDevInfo->psFirmwareCCBCtlLocal,
+	           &psDevInfo->psFirmwareCCBCtlMemDesc,
+	           &psDevInfo->psFirmwareCCB,
+	           &psDevInfo->psFirmwareCCBMemDesc);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFreeCCB(psDevInfo,
+		           &psDevInfo->psWorkEstFirmwareCCBCtl,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+		           &psDevInfo->psWorkEstFirmwareCCB,
+		           &psDevInfo->psWorkEstFirmwareCCBMemDesc);
+	}
+#endif
+
+	if (psDevInfo->psPowSyncPrim != NULL)
+	{
+		SyncPrimFree(psDevInfo->psPowSyncPrim);
+		psDevInfo->psPowSyncPrim = NULL;
+	}
+
+	if (psDevInfo->hSyncPrimContext != (IMG_HANDLE) NULL)
+	{
+		SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
+		psDevInfo->hSyncPrimContext = (IMG_HANDLE) NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfHWRInfoBufCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+			psDevInfo->psRGXFWIfHWRInfoBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+		psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwOsDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwOsData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwOsDataMemDesc);
+			psDevInfo->psRGXFWIfFwOsData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwOsDataMemDesc);
+		psDevInfo->psRGXFWIfFwOsDataMemDesc = NULL;
+	}
+
+	if (psDevInfo->psCompletedMemDesc)
+	{
+		if (psDevInfo->pui32CompletedById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
+			psDevInfo->pui32CompletedById = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCompletedMemDesc);
+		psDevInfo->psCompletedMemDesc = NULL;
+	}
+	if (psDevInfo->psEndTimeMemDesc)
+	{
+		if (psDevInfo->pui64EndTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
+			psDevInfo->pui64EndTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psEndTimeMemDesc);
+		psDevInfo->psEndTimeMemDesc = NULL;
+	}
+	if (psDevInfo->psStartTimeMemDesc)
+	{
+		if (psDevInfo->pui64StartTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
+			psDevInfo->pui64StartTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psStartTimeMemDesc);
+		psDevInfo->psStartTimeMemDesc = NULL;
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	if (psDevInfo->hTimerQueryLock)
+	{
+		OSLockDestroy(psDevInfo->hTimerQueryLock);
+		psDevInfo->hTimerQueryLock = NULL;
+	}
+#endif
+
+	if (psDevInfo->psRGXFWHeapGuardPageReserveMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWHeapGuardPageReserveMemDesc);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFirmware
+
+ @Description Frees all the firmware-related allocations
+
+ @Input       psDevInfo
+******************************************************************************/
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFreeFwOsData(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfConnectionCtl)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtl = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfConnectionCtlMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInitMemDesc = NULL;
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+	if (psDevInfo->psRGXFWIfSysInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSysInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInitMemDesc = NULL;
+	}
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalKCCBRoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT32 ui32ReadOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update KCCB read offset */
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+	ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+	if (ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psKCCBCtlLocal->ui32ReadOffset = ui32ReadOffset;
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalFWCCBWoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT32 ui32WriteOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update FWCCB write offset */
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32WriteOffset, INVALIDATE);
+	ui32WriteOffset = psFWCCBCtl->ui32WriteOffset;
+
+	if (ui32WriteOffset > psFWCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psFWCCBCtlLocal->ui32WriteOffset = ui32WriteOffset;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireKernelCCBSlot
+
+ PURPOSE	: Attempts to obtain a slot in the Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32		*pui32Offset)
+{
+	IMG_UINT32 ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+#if defined(PDUMP)
+	/* Wait for sufficient CCB space to become available */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, 0,
+	                      "Wait for space to write kCCB woff=%u", psKCCBCtlLocal->ui32WriteOffset);
+	DevmemPDumpCBP(psDevInfo->psKernelCCBCtlMemDesc,
+	               offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+	               psKCCBCtlLocal->ui32WriteOffset,
+	               1,
+	               (psKCCBCtlLocal->ui32WrapMask + 1));
+#endif
+
+	if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+	}
+	*pui32Offset = ui32NextWriteOffset;
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXPollKernelCCBSlot
+
+ PURPOSE	: Poll for space in Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXPollKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32	ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+	if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+	{
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_OK;
+		}
+
+		/*
+		 * The following check doesn't impact performance, since the
+		 * CPU has to wait for the GPU anyway (full kernel CCB).
+		 */
+		if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return PVRSRV_ERROR_KERNEL_CCB_FULL;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXGetCmdMemCopySize
+
+ PURPOSE	: Calculates actual size of KCCB command getting used
+
+ PARAMETERS	: eCmdType     Type of KCCB command
+
+ RETURNS	: Returns actual size of KCCB command on success else zero
+******************************************************************************/
+static IMG_UINT32 RGXGetCmdMemCopySize(RGXFWIF_KCCB_CMD_TYPE eCmdType)
+{
+	/* First get offset of uCmdData inside the struct RGXFWIF_KCCB_CMD
+	 * This will account alignment requirement of uCmdData union
+	 *
+	 * Then add command-data size depending on command type to calculate actual
+	 * command size required to do mem copy
+	 *
+	 * NOTE: Make sure that uCmdData is the last member of RGXFWIF_KCCB_CMD struct.
+	 */
+	switch (eCmdType)
+	{
+		case RGXFWIF_KCCB_CMD_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_MMUCACHE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_MMUCACHEDATA);
+		}
+#if defined(SUPPORT_USC_BREAKPOINT)
+		case RGXFWIF_KCCB_CMD_BP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_BPDATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_SLCFLUSHINVAL:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_SLCFLUSHINVALDATA);
+		}
+		case RGXFWIF_KCCB_CMD_CLEANUP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CLEANUP_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_POW:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_POWER_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE:
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_ZSBUFFER_BACKING_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELIST_GS_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELISTS_RECONSTRUCTION_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_WRITE_OFFSET_UPDATE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FORCE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA);
+		}
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+		case RGXFWIF_KCCB_CMD_REGCONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_REGCONFIG_DATA);
+		}
+#endif
+#if defined(SUPPORT_PDVFS)
+		case RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_PDVFS_MAX_FREQ_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_OS_STATE_CHANGE_DATA);
+		}
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+		case RGXFWIF_KCCB_CMD_COUNTER_DUMP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_COUNTER_DUMP_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CORECLKSPEEDCHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL:
+		case RGXFWIF_KCCB_CMD_WDG_CFG:
+		case RGXFWIF_KCCB_CMD_HEALTH_CHECK:
+		case RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL:
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS_DIRECT:
+		{
+			/* No command specific data */
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData);
+		}
+		case RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_DEV_VIRTADDR);
+		}
+		case RGXFWIF_KCCB_CMD_CANCEL_WORK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CANCEL_WORK_DATA);
+		}
+		default:
+		{
+			/* Invalid (OR) Unused (OR) Newly added command type */
+			return 0; /* Error */
+		}
+	}
+}
+
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+	                           INVALIDATE);
+	eError = PVRSRVWaitForValueKM(
+	              (IMG_UINT32 __iomem *)&psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFwSharedMemCacheOpExecPfn);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVWaitForValueKM");
+
+#if defined(PDUMP)
+	/* PDumping conditions same as RGXSendCommandRaw for the actual command and poll command to go in harmony */
+	if (PDumpCheckFlagsWrite(psDevInfo->psDeviceNode, ui32PDumpFlags))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Poll on KCCB slot %u for value %u (mask: 0x%x)", ui32SlotNum,
+					 RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED, RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED);
+
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+										ui32SlotNum * sizeof(IMG_UINT32),
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										PDUMP_POLL_OPERATOR_EQUAL,
+										ui32PDumpFlags);
+		PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO  *psDevInfo,
+									  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+									  IMG_UINT32          uiPDumpFlags,
+									  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE	*psDeviceNode = psDevInfo->psDeviceNode;
+	RGXFWIF_CCB_CTL		*psKCCBCtl;
+	RGXFWIF_CCB_CTL		*psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT8			*pui8KCCB = psDevInfo->psKernelCCB;
+	IMG_UINT32			ui32NewWriteOffset;
+	IMG_UINT32			ui32OldWriteOffset;
+	IMG_UINT32			ui32CmdMemCopySize;
+
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+#else
+	IMG_BOOL bContCaptureOn = PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER); /* client connected or in pdump init phase */
+	IMG_BOOL bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, uiPDumpFlags); /* Are we in capture range or continuous and not in a power transition */
+
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			if (!psDevInfo->bDumpedKCCBCtlAlready)
+			{
+				/* entering capture range */
+				psDevInfo->bDumpedKCCBCtlAlready = IMG_TRUE;
+
+				/* Wait for the live FW to catch up */
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: waiting on fw to catch-up, roff: %d, woff: %d",
+						__func__,
+						psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
+				PVRSRVPollForValueKM(psDeviceNode,
+				                     (IMG_UINT32 __iomem *)&psKCCBCtl->ui32ReadOffset,
+				                     ui32OldWriteOffset, 0xFFFFFFFF,
+				                     POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+				                     NULL);
+
+				/* Dump Init state of Kernel CCB control (read and write offset) */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+						"Initial state of kernel CCB Control, roff: %d, woff: %d",
+						psKCCBCtl->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset);
+
+				DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+						0,
+						sizeof(RGXFWIF_CCB_CTL),
+						uiPDumpFlags);
+			}
+		}
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		if ((likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+			(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo)))) ||
+			(KM_FW_CONNECTION_IS(READY, psDevInfo) && KM_OS_CONNECTION_IS(READY, psDevInfo)))
+		{
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: The firmware-driver connection is invalid:"
+									"driver state = %u / firmware state = %u;"
+									"expected READY (%u/%u) or ACTIVE (%u/%u) or in transition (%u/%u);",
+									__func__, KM_GET_OS_CONNECTION(psDevInfo), KM_GET_FW_CONNECTION(psDevInfo),
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_READY,
+									RGXFW_CONNECTION_OS_ACTIVE, RGXFW_CONNECTION_FW_ACTIVE,
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_ACTIVE));
+			eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+			goto _RGXSendCommandRaw_Exit;
+		}
+	}
+#endif
+
+	if (!OSLockIsLocked(psDeviceNode->hPowerLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called without power lock held!",
+				__func__));
+		PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+	}
+
+	/* Acquire a slot in the CCB */
+	eError = RGXAcquireKernelCCBSlot(psDevInfo, &ui32NewWriteOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto _RGXSendCommandRaw_Exit;
+	}
+
+	/* Calculate actual size of command to optimize device mem copy */
+	ui32CmdMemCopySize = RGXGetCmdMemCopySize(psKCCBCmd->eCmdType);
+	PVR_LOG_RETURN_IF_FALSE(ui32CmdMemCopySize !=0, "RGXGetCmdMemCopySize failed", PVRSRV_ERROR_INVALID_CCB_COMMAND);
+
+	/* Copy the command into the CCB */
+	OSCachedMemCopy(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)],
+	                psKCCBCmd, ui32CmdMemCopySize);
+	RGXFwSharedMemCacheOpExec(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)], ui32CmdMemCopySize, PVRSRV_CACHE_OP_FLUSH);
+
+	/* If non-NULL pui32CmdKCCBSlot passed-in, return the kCCB slot in which the command was enqueued */
+	if (pui32CmdKCCBSlot)
+	{
+		*pui32CmdKCCBSlot = ui32OldWriteOffset;
+
+		/* Each such command enqueue needs to reset the slot value first. This is so that a caller
+		 * doesn't get to see stale/false value in allotted slot */
+		OSWriteDeviceMem32WithWMB(&psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                          RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                           FLUSH);
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+							  "Reset kCCB slot number %u", ui32OldWriteOffset);
+		DevmemPDumpLoadMem(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+		                   ui32OldWriteOffset * sizeof(IMG_UINT32),
+						   sizeof(IMG_UINT32),
+						   uiPDumpFlags);
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Device (%p) KCCB slot %u reset with value %u for command type %x",
+		         __func__, psDevInfo, ui32OldWriteOffset, RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE, psKCCBCmd->eCmdType));
+	}
+
+	/* Memory barrier before KCCB write offset update. */
+	OSWriteMemoryBarrier(NULL);
+
+
+	/* Move past the current command */
+	psKCCBCtlLocal->ui32WriteOffset = ui32NewWriteOffset;
+	psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
+	/* Read-back of memory before Kick MTS */
+	OSWriteMemoryBarrier(&psKCCBCtl->ui32WriteOffset);
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32WriteOffset, FLUSH);
+
+#if defined(PDUMP)
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			/* Dump new Kernel CCB content */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCB cmd woff = %d",
+					ui32OldWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBMemDesc,
+					ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD),
+					ui32CmdMemCopySize,
+					uiPDumpFlags);
+
+			/* Dump new kernel CCB write offset */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCBCtl woff: %d",
+					ui32NewWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					sizeof(IMG_UINT32),
+					uiPDumpFlags);
+
+			/* mimic the read-back of the write from above */
+			DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					ui32NewWriteOffset,
+					0xFFFFFFFF,
+					PDUMP_POLL_OPERATOR_EQUAL,
+					uiPDumpFlags);
+		}
+		/* out of capture range */
+		else
+		{
+			eError = RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", _RGXSendCommandRaw_Exit);
+		}
+	}
+#endif
+
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags, "MTS kick for kernel CCB");
+	/*
+	 * Kick the MTS to schedule the firmware.
+	 */
+	__MTSScheduleWrite(psDevInfo, RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK);
+
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE,
+	           RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK, uiPDumpFlags);
+
+#if defined(SUPPORT_AUTOVZ)
+	RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+
+#if defined(NO_HARDWARE)
+	/* keep the roff updated because fw isn't there to update it */
+	psKCCBCtl->ui32ReadOffset = psKCCBCtlLocal->ui32WriteOffset;
+#endif
+
+_RGXSendCommandRaw_Exit:
+	return eError;
+}
+
+/******************************************************************************
+ FUNCTION	: _AllocDeferredCommand
+
+ PURPOSE	: Allocate a KCCB command and add it to KCCB deferred list
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: eKCCBType		Firmware Command type
+			: psKCCBCmd		Firmware Command
+			: uiPDumpFlags	Pdump flags
+
+ RETURNS	: PVRSRV_OK	If all went good, PVRSRV_ERROR_RETRY otherwise.
+******************************************************************************/
+static PVRSRV_ERROR _AllocDeferredCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                          RGXFWIF_KCCB_CMD   *psKCCBCmd,
+                                          IMG_UINT32         uiPDumpFlags)
+{
+	RGX_DEFERRED_KCCB_CMD *psDeferredCommand;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psDeferredCommand = OSAllocMem(sizeof(*psDeferredCommand));
+
+	if (!psDeferredCommand)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Deferring a KCCB command failed: allocation failure: requesting retry"));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeferredCommand->sKCCBcmd = *psKCCBCmd;
+	psDeferredCommand->uiPDumpFlags = uiPDumpFlags;
+	psDeferredCommand->psDevInfo = psDevInfo;
+
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_add_to_tail(&(psDevInfo->sKCCBDeferredCommandsListHead), &(psDeferredCommand->sListNode));
+	psDevInfo->ui32KCCBDeferredCommandsCount++;
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: _FreeDeferredCommand
+
+ PURPOSE	: Remove from the deferred list the sent deferred KCCB command
+
+ PARAMETERS	: psNode			Node in deferred list
+			: psDeferredKCCBCmd	KCCB Command to free
+
+ RETURNS	: None
+******************************************************************************/
+static void _FreeDeferredCommand(DLLIST_NODE *psNode, RGX_DEFERRED_KCCB_CMD *psDeferredKCCBCmd)
+{
+	dllist_remove_node(psNode);
+	psDeferredKCCBCmd->psDevInfo->ui32KCCBDeferredCommandsCount--;
+	OSFreeMem(psDeferredKCCBCmd);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXSendCommandsFromDeferredList
+
+ PURPOSE	: Try send KCCB commands in deferred list to KCCB
+		  Should be called by holding PowerLock
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: bPoll		Poll for space in KCCB
+
+ RETURNS	: PVRSRV_OK	If all commands in deferred list are sent to KCCB,
+			  PVRSRV_ERROR_KERNEL_CCB_FULL otherwise.
+******************************************************************************/
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+	RGX_DEFERRED_KCCB_CMD *psTempDeferredKCCBCmd;
+	DLLIST_NODE sCommandList;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDevInfo->psDeviceNode));
+
+	/* !!! Important !!!
+	 *
+	 * The idea of moving the whole list hLockKCCBDeferredCommandsList below
+	 * to the temporary list is only valid under the principle that all of the
+	 * operations are also protected by the power lock. It must be held
+	 * so that the order of the commands doesn't get messed up while we're
+	 * performing the operations on the local list.
+	 *
+	 * The necessity of releasing the hLockKCCBDeferredCommandsList comes from
+	 * the fact that _FreeDeferredCommand() is allocating memory and it can't
+	 * be done in atomic context (inside section protected by a spin lock).
+	 *
+	 * We're using spin lock here instead of mutex to quickly perform a check
+	 * if the list is empty in MISR without a risk that the MISR is going
+	 * to sleep due to a lock.
+	 */
+
+	/* move the whole list to a local list so it can be processed without lock */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_replace_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (dllist_is_empty(&sCommandList))
+		{
+			return PVRSRV_OK;
+		}
+
+		/* For every deferred KCCB command, try to send it*/
+		dllist_foreach_node(&sCommandList, psNode, psNext)
+		{
+			psTempDeferredKCCBCmd = IMG_CONTAINER_OF(psNode, RGX_DEFERRED_KCCB_CMD, sListNode);
+			eError = RGXSendCommandRaw(psTempDeferredKCCBCmd->psDevInfo,
+			                           &psTempDeferredKCCBCmd->sKCCBcmd,
+			                           psTempDeferredKCCBCmd->uiPDumpFlags,
+			                           NULL /* We surely aren't interested in kCCB slot number of deferred command */);
+			if (eError != PVRSRV_OK)
+			{
+				if (!bPoll)
+				{
+					eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+					goto cleanup_;
+				}
+				break;
+			}
+
+			_FreeDeferredCommand(psNode, psTempDeferredKCCBCmd);
+		}
+
+		if (bPoll)
+		{
+			PVRSRV_ERROR eErrPollForKCCBSlot;
+
+			/* Don't overwrite eError because if RGXPollKernelCCBSlot returns OK and the
+			 * outer loop times-out, we'll still want to return KCCB_FULL to caller
+			 */
+			eErrPollForKCCBSlot = RGXPollKernelCCBSlot(psDevInfo);
+			if (eErrPollForKCCBSlot == PVRSRV_ERROR_KERNEL_CCB_FULL)
+			{
+				eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+				goto cleanup_;
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+cleanup_:
+	/* if the local list is not empty put it back to the deferred list head
+	 * so that the old order of commands is retained */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_insert_list_at_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+										  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+										  IMG_UINT32          uiPDumpFlags,
+										  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	IMG_BOOL     bPoll = (pui32CmdKCCBSlot != NULL);
+	PVRSRV_ERROR eError;
+
+	/*
+	 * First try to Flush all the cmds in deferred list.
+	 *
+	 * We cannot defer an incoming command if the caller is interested in
+	 * knowing the command's kCCB slot: it plans to poll/wait for a
+	 * response from the FW just after the command is enqueued, so we must
+	 * poll for space to be available.
+	 */
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, bPoll);
+	if (eError == PVRSRV_OK)
+	{
+		eError = RGXSendCommandRaw(psDevInfo,
+								   psKCCBCmd,
+								   uiPDumpFlags,
+								   pui32CmdKCCBSlot);
+	}
+
+	/*
+	 * If we don't manage to enqueue one of the deferred commands or the command
+	 * passed as argument because the KCCB is full, insert the latter into the deferred commands list.
+	 * The deferred commands will also be flushed eventually by:
+	 *  - one more KCCB command sent for any DM
+	 *  - RGX_MISRHandler_CheckFWActivePowerState
+	 */
+	if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+	{
+		if (pui32CmdKCCBSlot == NULL)
+		{
+			eError = _AllocDeferredCommand(psDevInfo, psKCCBCmd, uiPDumpFlags);
+		}
+		else
+		{
+			/* Let the caller retry. Otherwise if we deferred the command and returned OK,
+			 * the caller can end up looking in a stale CCB slot.
+			 */
+			PVR_DPF((PVR_DBG_WARNING, "%s: Couldn't flush the deferred queue for a command (Type:%d) "
+			                        "- will be retried", __func__, psKCCBCmd->eCmdType));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32         *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+
+	/* Ensure Rogue is powered up before kicking MTS */
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: failed to acquire powerlock (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVPowerLock_Exit;
+	}
+
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition Rogue to ON (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+										  psKCCBCmd,
+										  ui32PDumpFlags,
+	                                      pui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to schedule command (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+#if defined(DEBUG)
+		/* PVRSRVDebugRequest must be called without powerlock */
+		PVRSRVPowerUnlock(psDeviceNode);
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+		goto _PVRSRVPowerLock_Exit;
+#endif
+	}
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_PVRSRVPowerLock_Exit:
+	return eError;
+}
+
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGX_MISRHandler_ScheduleProcessQueues
+
+ @Description - Sends uncounted kick to all the DMs (the FW will process all
+				the queue for all the DMs)
+******************************************************************************/
+static void RGX_MISRHandler_ScheduleProcessQueues(void *pvData)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR           eError;
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_OFF;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guests are not permitted to change the device power state */
+		if ((eError != PVRSRV_OK) || (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+	else
+	{
+		if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			RGXFWIF_GPU_UTIL_FW    *psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+			IMG_BOOL               bGPUHasWorkWaiting;
+
+			/* Check whether it's worth waking up the GPU */
+			RGXFwSharedMemCacheOpValue(psUtilFW->ui64GpuLastWord, INVALIDATE);
+			bGPUHasWorkWaiting =
+			    (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
+
+			if (!bGPUHasWorkWaiting)
+			{
+				/* all queues are empty, don't wake up the GPU */
+				PVRSRVPowerUnlock(psDeviceNode);
+				return;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		/* wake up the GPU */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition device to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+
+	/* Memory barrier before Kick MTS */
+	OSWriteMemoryBarrier(NULL);
+
+	/* uncounted kick to the FW */
+	HTBLOGK(HTB_SF_MAIN_KICK_UNCOUNTED);
+	__MTSScheduleWrite(psDevInfo, (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return OSInstallMISR(phMISR,
+			RGX_MISRHandler_ScheduleProcessQueues,
+			psDeviceNode,
+			"RGX_ScheduleProcessQueues");
+}
+
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+                                              RGXFWIF_DM          eKCCBType,
+                                              RGXFWIF_KCCB_CMD    *psKCCBCmd,
+                                              IMG_UINT32          ui32PDumpFlags,
+                                              IMG_UINT32          *pui32CmdKCCBSlot,
+                                              IMG_BOOL            bCallerHasPwrLock)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMMUSyncUpdate;
+
+	/* Don't send the command/power up request if device not available. */
+	if (unlikely((psDevInfo == NULL) ||
+	             (psDevInfo->psDeviceNode == NULL) ||
+				 (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)))
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	/* Don't send the command/power up request if device in deinit phase.
+	 * The de-init thread could destroy the device whilst the power up
+	 * sequence below is accessing the HW registers.
+	 * Not yet safe to free resources. Caller should retry later.
+	 */
+	if (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	    psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+
+	if (!bCallerHasPwrLock)
+	{
+		/* PVRSRVPowerLock guarantees atomicity between commands. This is helpful
+		   in a scenario with several applications allocating resources. */
+		eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+
+			/* If system is found powered OFF, Retry scheduling the command */
+			if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+			{
+				eError = PVRSRV_ERROR_RETRY;
+			}
+
+			goto RGXScheduleCommand_exit;
+		}
+	}
+
+	if (unlikely(psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	             psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING))
+	{
+		/* If we have the power lock the device is valid but the deinit
+		 * thread could be waiting for the lock. */
+		eError = PVRSRV_ERROR_RETRY;
+		goto _PVRSRVInvalidDeviceError_Exit;
+	}
+
+	/* Ensure device is powered up before sending any commands */
+	PDUMPPOWCMDSTART(psDevInfo->psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDevInfo->psDeviceNode,
+	                                     PVRSRV_DEV_POWER_STATE_ON,
+	                                     PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+
+	eError = RGXPreKickCacheCommand(psDevInfo, eKCCBType, &uiMMUSyncUpdate);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+_PVRSRVInvalidDeviceError_Exit:
+	if (!bCallerHasPwrLock)
+	{
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+	}
+RGXScheduleCommand_exit:
+	return eError;
+}
+
+/*
+ * RGXCheckFirmwareCCB
+ */
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT8 *psFWCCB = psDevInfo->psFirmwareCCB;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo) ||
+								 (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+								  (KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))),
+								 "FW-KM connection is down");
+#endif
+
+	eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+		return;
+	}
+
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtlLocal->ui32WriteOffset)
+	{
+		/* Point to the next command */
+		const RGXFWIF_FWCCB_CMD *psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtlLocal->ui32ReadOffset;
+		RGXFwSharedMemCacheOpPtr(psFwCCBCmd, INVALIDATE);
+
+
+		HTBLOGK(HTB_SF_MAIN_FWCCB_CMD, psFwCCBCmd->eCmdType);
+		switch (psFwCCBCmd->eCmdType)
+		{
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
+				}
+				RGXProcessRequestZSBufferBacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
+				}
+				RGXProcessRequestZSBufferUnbacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELIST_GROW, "Request to grow the free list");
+				}
+				RGXProcessRequestGrow(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
+				}
+
+				if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d/%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psDevInfo->psRGXFWIfHWRInfoBufCtl->ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+
+				RGXProcessRequestFreelistsReconstruction(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION:
+			{
+				/* Notify client drivers */
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+					const RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA *psCmdFwPagefault =
+							&psFwCCBCmd->uCmdData.sCmdFWPagefault;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+						sErrorData.uErrData.sFwPFErrData.sFWFaultAddr.uiAddr = psCmdFwPagefault->sFWFaultAddr.uiAddr;
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+														  &sErrorData);
+					}
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
+			{
+				const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification =
+						&psFwCCBCmd->uCmdData.sCmdContextResetNotification;
+				IMG_UINT32 ui32ErrorPid = 0;
+
+				FWCommonContextListSetLastResetReason(psDevInfo,
+				                                      &ui32ErrorPid,
+				                                      psCmdContextResetNotification);
+
+				/* Increment error counter (if appropriate) */
+				if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32WGPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32WGPErrorCount++;
+					}
+				}
+				else if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32TRPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32TRPErrorCount++;
+					}
+				}
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = psCmdContextResetNotification->eResetReason;
+						sErrorData.pid = ui32ErrorPid;
+
+						/* Populate error data according to reset reason */
+						switch (psCmdContextResetNotification->eResetReason)
+						{
+							case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+							case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+							{
+								sErrorData.uErrData.sChecksumErrData.ui32ExtJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+								sErrorData.uErrData.sChecksumErrData.eDM = psCmdContextResetNotification->eDM;
+								break;
+							}
+							default:
+							{
+								break;
+							}
+						}
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+						                                  &sErrorData);
+					}
+				}
+
+				/* Notify if a page fault */
+				if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF)
+				{
+					DevmemIntPFNotify(psDevInfo->psDeviceNode,
+							psCmdContextResetNotification->ui64PCAddress,
+							psCmdContextResetNotification->sFaultAddress);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
+			{
+				PVRSRV_ERROR eError;
+				PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+				OSAtomicWrite(&psDevInfo->psDeviceNode->eDebugDumpRequested, PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE);
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal FW Cmd debug dump event, dumping now instead", __func__));
+					PVRSRVDebugRequest(psDevInfo->psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				IMG_PID pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
+				IMG_INT32 i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
+
+				switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
+				{
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,i32AdjustmentValue,0,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,i32AdjustmentValue,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,i32AdjustmentValue,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,i32AdjustmentValue,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,i32AdjustmentValue,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,i32AdjustmentValue,0,pidTmp);
+						break;
+					}
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,0,i32AdjustmentValue,pidTmp);
+						break;
+					}
+#endif
+				}
+#endif
+				break;
+			}
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+			case RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE:
+			{
+				RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo,
+											  psFwCCBCmd->uCmdData.sCmdCoreClkRateChange.ui32CoreClkRate);
+				break;
+			}
+#endif
+			case RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
+			{
+				RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+				if (psDevInfo->psRGXFWIfFwSysData != NULL  &&
+				    psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)
+				{
+					PVRSRV_ERROR eError;
+
+					/* Power down... */
+					eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+															 PVRSRV_SYS_POWER_STATE_OFF,
+															 PVRSRV_POWER_FLAGS_NONE);
+					if (eError == PVRSRV_OK)
+					{
+						/* Clear the FW faulted flags... */
+						psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags &= ~(RGXFWIF_HWR_FW_FAULT|RGXFWIF_HWR_RESTART_REQUESTED);
+						OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags);
+						RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags,
+						                           FLUSH);
+
+						/* Power back up again... */
+						eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+																 PVRSRV_SYS_POWER_STATE_ON,
+																 PVRSRV_POWER_FLAGS_NONE);
+
+						/* Send a dummy KCCB command to ensure the FW wakes up and checks the queues... */
+						if (eError == PVRSRV_OK)
+						{
+							LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+							{
+								eError = RGXFWHealthCheckCmd(psDevInfo);
+								if (eError != PVRSRV_ERROR_RETRY)
+								{
+									break;
+								}
+								OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+							} END_LOOP_UNTIL_TIMEOUT_US();
+						}
+					}
+
+					/* Notify client drivers and system layer of FW fault */
+					{
+						PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+						PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+						/* Client notification of device error will be achieved by
+						 * clients calling UM function RGXGetLastDeviceError() */
+						psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+
+						/* Notify system layer */
+						if (psDevConfig->pfnSysDevErrorNotify)
+						{
+							PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+							sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+							psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+							                                  &sErrorData);
+						}
+					}
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Failed firmware restart (%s)",
+								 __func__, PVRSRVGetErrorString(eError)));
+					}
+				}
+				break;
+			}
+			default:
+			{
+				/* unknown command */
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unknown Command (eCmdType=0x%08x)",
+				         __func__, psFwCCBCmd->eCmdType));
+				/* Assert on magic value corruption */
+				PVR_ASSERT((((IMG_UINT32)psFwCCBCmd->eCmdType & RGX_CMD_MAGIC_DWORD_MASK) >> RGX_CMD_MAGIC_DWORD_SHIFT) == RGX_CMD_MAGIC_DWORD);
+			}
+		}
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		OSMemoryBarrier(NULL);
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+		OSWriteMemoryBarrier(NULL);
+
+		if (psFWCCBCtlLocal->ui32ReadOffset == psFWCCBCtlLocal->ui32WriteOffset)
+		{
+			eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+				return;
+			}
+		}
+	}
+}
+
+/*
+ * PVRSRVRGXFrameworkCopyCommand
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+		DEVMEM_MEMDESC	*psFWFrameworkMemDesc,
+		IMG_PBYTE		pbyGPUFRegisterList,
+		IMG_UINT32		ui32FrameworkRegisterSize)
+{
+	PVRSRV_ERROR	eError;
+	RGXFWIF_RF_REGISTERS	*psRFReg;
+
+	eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
+			(void **)&psRFReg);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	OSDeviceMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
+	RGXFwSharedMemCacheOpPtr(psRFReg, FLUSH);
+
+	/* Release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
+
+	/*
+	 * Dump the FW framework buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FWFramework buffer");
+	DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXFrameworkCreateKM
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+		DEVMEM_MEMDESC		**ppsFWFrameworkMemDesc,
+		IMG_UINT32			ui32FrameworkCommandSize)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+		Allocate device memory for the firmware GPU framework state.
+		Sufficient info to kick one or more DMs should be contained in this buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware framework state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+			ui32FrameworkCommandSize,
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwGPUFrameworkState",
+			ppsFWFrameworkMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware framework state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem *pui32LinMemAddr,
+												IMG_UINT32			ui32Value,
+												IMG_UINT32			ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32CurrentQueueLength, ui32MaxRetries;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDevNode->pvDevice;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+	ui32CurrentQueueLength = (psKCCBCtlLocal->ui32WrapMask+1 +
+					psKCCBCtlLocal->ui32WriteOffset -
+					psKCCBCtlLocal->ui32ReadOffset) & psKCCBCtlLocal->ui32WrapMask;
+	ui32CurrentQueueLength += psDevInfo->ui32KCCBDeferredCommandsCount;
+
+	for (ui32MaxRetries = ui32CurrentQueueLength + 1;
+				ui32MaxRetries > 0;
+				ui32MaxRetries--)
+	{
+
+		/*
+		 * PVRSRVPollForValueKM flags are set to POLL_FLAG_NONE in this case so that the function
+		 * does not generate an error message. In this case, the PollForValueKM is expected to
+		 * timeout as there is work ongoing on the GPU which may take longer than the timeout period.
+		 */
+		eError = PVRSRVPollForValueKM(psDevNode, pui32LinMemAddr, ui32Value, ui32Mask, POLL_FLAG_NONE, NULL);
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			break;
+		}
+
+		RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		         __func__, PVRSRVGetErrorString(eError),
+		         pui32LinMemAddr, ui32Value));
+		OSDumpStack();
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32ConfigState,
+				IMG_BOOL bSetNotClear)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	RGXFWIF_KCCB_CMD sStateFlagCmd = { 0 };
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_SYSDATA *psFwSysData;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (NULL == psFwSysData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Fw Sys Config is not mapped into CPU space", __func__));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* apply change and ensure the new data is written to memory
+	 * before requesting the FW to read it
+	 */
+	ui32Config = ui32Config & RGXFWIF_INICFG_ALL;
+	if (bSetNotClear)
+	{
+		psFwSysData->ui32ConfigFlags |= ui32Config;
+	}
+	else
+	{
+		psFwSysData->ui32ConfigFlags &= ~ui32Config;
+	}
+	OSWriteMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+	RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags, FLUSH);
+
+	/* return current/new value to caller */
+	if (pui32ConfigState)
+	{
+		*pui32ConfigState = psFwSysData->ui32ConfigFlags;
+	}
+
+	OSMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	/* notify FW to update setting */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of the value */
+		sStateFlagCmd.eCmdType = RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sStateFlagCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock(psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the value to be updated as the FW validates
+		 * the parameters and modifies the ui32ConfigFlags
+		 * accordingly
+		 * (for completeness as registered callbacks should also
+		 *  not permit invalid transitions)
+		 */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO	*psDevInfo,
+									   RGXFWIF_DM			eDM,
+									   RGXFWIF_KCCB_CMD		*psKCCBCmd,
+									   RGXFWIF_CLEANUP_TYPE	eCleanupType,
+									   IMG_UINT32			ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32kCCBCommandSlot;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus);
+
+	PVR_LOG_RETURN_IF_FALSE((eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+							(eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT),
+							"Cleanup aborted: Device in bad state", PVRSRV_OK);
+#endif
+
+
+	/* Clean-up commands sent during frame capture intervals must be dumped even when not in capture range... */
+	ui32PDumpFlags |= PDUMP_FLAGS_INTERVAL;
+
+	psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
+	psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
+
+	/*
+		Send the cleanup request to the firmware. If the resource is still busy
+		the firmware will tell us and we'll drop out with a retry.
+	*/
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+											  eDM,
+											  psKCCBCmd,
+											  ui32PDumpFlags,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR ,"RGXScheduleCommandAndGetKCCBSlot() failed (%s) in %s()",
+			         PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		goto fail_command;
+	}
+
+	/* Wait for command kCCB slot to be updated by FW */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+						  "Wait for the firmware to reply to the cleanup command");
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot,
+									  ui32PDumpFlags);
+	/*
+		If the firmware hasn't got back to us in a timely manner
+		then bail and let the caller retry the command.
+	 */
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: RGXWaitForKCCBSlotUpdate timed out. Dump debug information.",
+		         __func__));
+
+		eError = PVRSRV_ERROR_RETRY;
+#if defined(DEBUG)
+		PVRSRVDebugRequest(psDevInfo->psDeviceNode,
+				DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+#endif
+		goto fail_poll;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		goto fail_poll;
+	}
+
+#if defined(PDUMP)
+	/*
+	 * The cleanup request to the firmware will tell us if a given resource is busy or not.
+	 * If the RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY flag is set, this means that the resource is
+	 * still in use. In this case we return a PVRSRV_ERROR_RETRY error to the client drivers
+	 * and they will re-issue the cleanup request until it succeed.
+	 *
+	 * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
+	 * that cleanup requests are only submitted if the resource is unused.
+	 * If this is not the case, the following poll will block infinitely, making sure
+	 * the issue doesn't go unnoticed.
+	 */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+					"Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
+					eDM,
+					psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
+					psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+									ui32kCCBCommandSlot * sizeof(IMG_UINT32),
+									0,
+									RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+#endif
+
+	/*
+		If the command has was run but a resource was busy, then the request
+		will need to be retried.
+	*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot],
+	                           INVALIDATE);
+
+	if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY))
+	{
+		if (psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+		eError = PVRSRV_ERROR_RETRY;
+		goto fail_requestbusy;
+	}
+
+	return PVRSRV_OK;
+
+fail_requestbusy:
+fail_poll:
+fail_command:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	RGXRequestCommonContextCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD			sRCCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+	PRGXFWIF_FWCOMMONCONTEXT	psFWCommonContextFWAddr;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	/* Force retry if this context's CCB is currently being dumped
+	 * as part of the stalled CCB debug */
+	if (psDevInfo->pvEarliestStalledClientCCB == (void*)FWCommonContextGetClientCCB(psServerCommonContext))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: [%u/%d] Forcing retry as psDevInfo->pvEarliestStalledClientCCB = psCtxClientCCB <%p>",
+		         __func__,
+		         psDeviceNode->sDevId.ui32InternalID,
+		         psDeviceNode->sDevId.i32KernelDeviceID,
+		         psDevInfo->pvEarliestStalledClientCCB));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psFWCommonContextFWAddr = FWCommonContextGetFWAddress(psServerCommonContext);
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Common ctx cleanup Request DM%d [context = 0x%08x]",
+			eDM, psFWCommonContextFWAddr.ui32Addr);
+	PDUMPCOMMENT(psDeviceNode, "Wait for CCB to be empty before common ctx cleanup");
+
+	RGXCCBPDumpDrainCCB(FWCommonContextGetClientCCB(psServerCommonContext), ui32PDumpFlags);
+#endif
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+									   eDM,
+									   &sRCCleanUpCmd,
+									   RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
+									   ui32PDumpFlags);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+/*
+ * RGXFWRequestHWRTDataCleanUp
+ */
+
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         PRGXFWIF_HWRTDATA psHWRTData)
+{
+	RGXFWIF_KCCB_CMD			sHWRTDataCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDeviceNode, "HW RTData cleanup Request [HWRTData = 0x%08x]", psHWRTData.ui32Addr);
+
+	sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
+
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+	                                   RGXFWIF_DM_GP,
+	                                   &sHWRTDataCleanUpCmd,
+	                                   RGXFWIF_CLEANUP_HWRTDATA,
+	                                   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a HWRTData cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestFreeListCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_FREELIST psFWFreeList)
+{
+	RGXFWIF_KCCB_CMD			sFLCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_GP,
+									   &sFLCleanUpCmd,
+									   RGXFWIF_CLEANUP_FREELIST,
+									   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a memory context cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestZSBufferCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer)
+{
+	RGXFWIF_KCCB_CMD			sZSBufferCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_3D,
+									   &sZSBufferCleanUpCmd,
+									   RGXFWIF_CLEANUP_ZSBUFFER,
+									   PDUMP_FLAGS_NONE);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS = ui32HCSDeadlineMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Hard Context Switching deadline inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32HCSDeadlineMS),
+							  ui32HCSDeadlineMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock)
+{
+	RGXFWIF_KCCB_CMD	sCmpKCCBCmd = { 0 };
+
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+	if (bCallerHasPwrLock)
+	{
+		return	RGXScheduleCommandWithoutPowerLock(psDevInfo,
+												   RGXFWIF_DM_GP,
+												   &sCmpKCCBCmd,
+												   PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		return	RGXScheduleCommand(psDevInfo,
+								   RGXFWIF_DM_GP,
+								   &sCmpKCCBCmd,
+								   PDUMP_FLAGS_CONTINUOUS);
+	}
+}
+
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32CBaseMapCtxReg;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+	else
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+#else
+	/*
+	 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+	 * FW crash from its memory context
+	 */
+	RGXWriteKernelMMUPC64(&psDevInfo->sLayerParams,
+				                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+				                      ((0xDEADBEEF
+				                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+				                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+				                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               IMG_UINT32 ui32DriverID,
+                               RGXFWIF_OS_STATE_CHANGE eOSOnlineState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sOSOnlineStateCmd = { 0 };
+	const RGXFWIF_SYSDATA *psFwSysData;
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	sOSOnlineStateCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.ui32DriverID = ui32DriverID;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = eOSOnlineState;
+
+#if defined(SUPPORT_AUTOVZ)
+	{
+		IMG_BOOL bConnectionDown = IMG_FALSE;
+
+		PVR_UNREFERENCED_PARAMETER(psFwSysData);
+		sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = RGXFWIF_OS_OFFLINE;
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			/* Send the offline command regardless if power lock is held or not.
+			 * Under AutoVz this is done during regular driver deinit, store-to-ram suspend
+			 * or (optionally) from a kernel panic callback. Deinit and suspend operations
+			 * take the lock in the rgx pre/post power functions as expected.
+			 * The kernel panic callback is a last resort way of letting the firmware know that
+			 * the VM is unrecoverable and the vz connection must be disabled. It cannot wait
+			 * on other kernel threads to finish and release the lock. */
+			eError = RGXSendCommand(psDevInfo,
+									&sOSOnlineStateCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Guests and Host going offline should wait for confirmation
+		 * from the Firmware of the state change. If this fails, break
+		 * the connection on the OS Driver's end as backup. */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (ui32DriverID == RGXFW_HOST_DRIVER_ID))
+		{
+			LOOP_UNTIL_TIMEOUT_US(SECONDS_TO_MICROSECONDS/2)
+			{
+				KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+				if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+				{
+					bConnectionDown = IMG_TRUE;
+					break;
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (!bConnectionDown)
+			{
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+		}
+	}
+#else
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* no reason for Guests to update their state or any other VM's.
+		 * This is the Hypervisor and Host driver's responsibility. */
+		return PVRSRV_OK;
+	}
+	else
+	{
+		const volatile RGXFWIF_OS_RUNTIME_FLAGS *psFwRunFlags;
+
+		PVR_ASSERT(psFwSysData != NULL);
+
+		psFwRunFlags = (const volatile RGXFWIF_OS_RUNTIME_FLAGS*) &psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+		/* Attempt several times until the FW manages to offload the OS */
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			IMG_UINT32 ui32kCCBCommandSlot;
+
+			/* Send request */
+			eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+													  RGXFWIF_DM_GP,
+													  &sOSOnlineStateCmd,
+													  PDUMP_FLAGS_CONTINUOUS,
+													  &ui32kCCBCommandSlot);
+			if (unlikely(eError == PVRSRV_ERROR_RETRY)) continue;
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommand", return_);
+
+			/* Wait for FW to process the cmd */
+			eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", return_);
+
+			/* read the OS state */
+			OSMemoryBarrier(NULL);
+			/* check if FW finished offloading the driver and is stopped */
+			if ((eOSOnlineState == RGXFWIF_OS_ONLINE && (psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_READY || psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_ACTIVE)) ||
+			    (eOSOnlineState == RGXFWIF_OS_OFFLINE && psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_OFFLINE))
+			{
+				eError = PVRSRV_OK;
+				break;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_TIMEOUT;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+
+return_ :
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgPHRCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE((ui32PHRMode == RGXFWIF_PHR_MODE_OFF) ||
+							(ui32PHRMode == RGXFWIF_PHR_MODE_RD_RESET),
+							"Invalid PHR Mode.", PVRSRV_ERROR_INVALID_PARAMS);
+
+	sCfgPHRCmd.eCmdType = RGXFWIF_KCCB_CMD_PHR_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode = ui32PHRMode;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Periodic Hardware Reset Mode inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32PHRMode),
+							  ui32PHRMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GP,
+		                            &sCfgPHRCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+							IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgWdgCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCfgWdgCmd.eCmdType = RGXFWIF_KCCB_CMD_WDG_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs = ui32WdgPeriodUs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the firmware watchdog period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32WdgPeriodUs),
+							  ui32WdgPeriodUs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sCfgWdgCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious)
+{
+	/* Attempt to detect and deal with any stalled client contexts.
+	 * bIgnorePrevious may be set by the caller if they know a context to be
+	 * stalled, as otherwise this function will only identify stalled
+	 * contexts which have not been previously reported.
+	 */
+
+	IMG_UINT32 ui32StalledClientMask = 0;
+
+	if (!(OSTryLockAcquire(psDevInfo->hCCBStallCheckLock)))
+	{
+		PVR_LOG(("RGXCheckForStalledClientContexts: Failed to acquire hCCBStallCheckLock, returning..."));
+		return;
+	}
+
+	ui32StalledClientMask |= CheckForStalledClientTDMTransferCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientRenderCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientComputeCtxt(psDevInfo);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	ui32StalledClientMask |= CheckForStalledClientKickSyncCtxt(psDevInfo);
+#endif
+	/* If at least one DM stalled bit is different than before */
+	if (bIgnorePrevious || (psDevInfo->ui32StalledClientMask != ui32StalledClientMask))
+	{
+		if (ui32StalledClientMask > 0)
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Print all the stalled DMs */
+			PVR_LOG(("Possible stalled client RGX contexts detected: %s%s%s%s%s%s%s",
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_GP),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TDM_2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TA),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_3D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_CDM),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ3D)));
+
+			PVR_LOG(("Trying to identify stalled context...(%s) [%d]",
+			         pszStalledAction, bIgnorePrevious));
+
+			DumpStalledContextInfo(psDevInfo);
+		}
+		else
+		{
+			if (psDevInfo->ui32StalledClientMask> 0)
+			{
+				/* Indicate there are no stalled DMs */
+				PVR_LOG(("No further stalled client contexts exist"));
+			}
+		}
+		psDevInfo->ui32StalledClientMask = ui32StalledClientMask;
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+	OSLockRelease(psDevInfo->hCCBStallCheckLock);
+}
+
+/*
+	RGXUpdateHealthStatus
+*/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed)
+{
+	const PVRSRV_DATA*           psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
+	PVRSRV_RGXDEV_INFO*          psDevInfo;
+	const RGXFWIF_TRACEBUF*      psRGXFWIfTraceBufCtl;
+	const RGXFWIF_SYSDATA*       psFwSysData;
+	const RGXFWIF_OSDATA*        psFwOsData;
+	const RGXFWIF_CCB_CTL*       psKCCBCtl;
+	RGXFWIF_CCB_CTL*             psKCCBCtlLocal;
+	IMG_UINT32                   ui32ThreadCount;
+	IMG_BOOL                     bKCCBCmdsWaiting;
+	PVRSRV_ERROR                 eError;
+
+	PVR_ASSERT(psDevNode != NULL);
+	psDevInfo = psDevNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, PVRSRV_OK);
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_OK;
+	}
+
+	psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData,
+								 INVALIDATE);
+		psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         INVALIDATE);
+	psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+
+	/* If this is a quick update, then include the last current value... */
+	if (!bCheckAfterTimePassed)
+	{
+		eNewStatus = OSAtomicRead(&psDevNode->eHealthStatus);
+		eNewReason = OSAtomicRead(&psDevNode->eHealthReason);
+	}
+
+	/* Decrement the SLR holdoff counter (if non-zero) */
+	if (psDevInfo->ui32SLRHoldoffCounter > 0)
+	{
+		psDevInfo->ui32SLRHoldoffCounter--;
+	}
+
+	/* Take power lock, retry if it's in use in another task. */
+	eError = PVRSRVPowerTryLockWaitForTimeout(psDevNode);
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		/* Skip health status update if timeout */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Power lock timeout, increase OS_POWERLOCK_TIMEOUT_US.", __func__));
+		goto _RGXUpdateHealthStatus_Exit;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerTryLockWaitForTimeout");
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+		return PVRSRV_OK;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		/* On a PCI error all reads from the PCI bar may return 0xFFFFFFFF.
+		   This value is not valid for a core ID. */
+		if (psFwSysData->ui32MemFaultCheck == RGX_PCI_ERROR_VALUE_DWORD)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: PCI error", __func__));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR;
+			PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_PCI_ERROR);
+			PVRSRVPowerUnlock(psDevNode);
+			goto _RGXUpdateHealthStatus_Exit;
+		}
+	}
+
+	/* If Rogue is not powered on, just skip ahead and check for stalled client CCBs */
+	if (PVRSRVIsDevicePowered(psDevNode))
+	{
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			/*
+			   Firmware thread checks...
+			 */
+			for (ui32ThreadCount = 0; ui32ThreadCount < RGXFW_THREAD_NUM; ui32ThreadCount++)
+			{
+				const IMG_CHAR* pszTraceAssertInfo;
+
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf, INVALIDATE);
+				pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
+
+				/*
+				Check if the FW has hit an assert...
+				*/
+				if (*pszTraceAssertInfo != '\0')
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Firmware thread %d has asserted: %.*s (%.*s:%d)",
+							__func__, ui32ThreadCount, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							pszTraceAssertInfo, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+				{
+					/*
+					   Check the threads to see if they are in the same poll locations as last time...
+					*/
+					if (bCheckAfterTimePassed)
+					{
+						if (psFwSysData->aui32CrPollAddr[ui32ThreadCount] != 0  &&
+							psFwSysData->aui32CrPollCount[ui32ThreadCount] == psDevInfo->aui32CrLastPollCount[ui32ThreadCount])
+						{
+							PVR_DPF((PVR_DBG_WARNING, "%s: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
+									__func__, ui32ThreadCount,
+									((psFwSysData->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+									psFwSysData->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
+									psFwSysData->aui32CrPollMask[ui32ThreadCount]));
+							eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+							eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
+							PVRSRVPowerUnlock(psDevNode);
+							goto _RGXUpdateHealthStatus_Exit;
+						}
+						psDevInfo->aui32CrLastPollCount[ui32ThreadCount] = psFwSysData->aui32CrPollCount[ui32ThreadCount];
+					}
+				}
+			}
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+			{
+				/*
+				Check if the FW has faulted...
+				*/
+				if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_FW_FAULT)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+							"%s: Firmware has faulted and needs to restart",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_FAULT;
+					if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_RESTART_REQUESTED)
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_RESTARTING;
+					}
+					else
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_IDLING;
+					}
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+			}
+		}
+
+		/*
+		   Event Object Timeouts check...
+		*/
+		if (!bCheckAfterTimePassed)
+		{
+			if (psDevInfo->ui32GEOTimeoutsLastTime > 1 && psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Global Event Object Timeouts have risen (from %d to %d)",
+				         __func__,
+				         psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
+			}
+			psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
+		}
+
+		/*
+		   Check the Kernel CCB pointer is valid. If any commands were waiting last time, then check
+		   that some have executed since then.
+		*/
+		bKCCBCmdsWaiting = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+		psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+		if (psKCCBCtl != NULL && psKCCBCtlLocal != NULL)
+		{
+			/* update KCCB read offset */
+			RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+			psKCCBCtlLocal->ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+			if (psKCCBCtlLocal->ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask  ||
+				psKCCBCtlLocal->ui32WriteOffset > psKCCBCtlLocal->ui32WrapMask)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: KCCB has invalid offset (ROFF=%d WOFF=%d)",
+						__func__, psKCCBCtlLocal->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+			}
+
+			if (psKCCBCtlLocal->ui32ReadOffset != psKCCBCtlLocal->ui32WriteOffset)
+			{
+				bKCCBCmdsWaiting = IMG_TRUE;
+			}
+		}
+
+		if (bCheckAfterTimePassed && psFwOsData != NULL)
+		{
+			IMG_UINT32 ui32KCCBCmdsExecuted = psFwOsData->ui32KCCBCmdsExecuted;
+
+			if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
+			{
+				/*
+				   If something was waiting last time then the Firmware has stopped processing commands.
+				*/
+				if (psDevInfo->bKCCBCmdsWaitingLastTime)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: No KCCB commands executed since check!",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+				}
+
+				/*
+				   If no commands are currently pending and nothing happened since the last poll, then
+				   schedule a dummy command to ping the firmware so we know it is alive and processing.
+				*/
+				if ((!bKCCBCmdsWaiting) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+				{
+					/* Protect the PDumpLoadMem. RGXScheduleCommand() cannot take the
+					 * PMR lock itself, because some bridge functions will take the PMR lock
+					 * before calling RGXScheduleCommand
+					 */
+					PVRSRV_ERROR eError = RGXFWHealthCheckCmdWithoutPowerLock(psDevNode->pvDevice);
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Cannot schedule Health Check command! (0x%x)",
+								__func__, eError));
+					}
+					else
+					{
+						bKCCBCmdsWaiting = IMG_TRUE;
+					}
+				}
+			}
+
+			psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
+			psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
+		}
+	}
+
+	/*
+	   Interrupt counts check...
+	*/
+	if (bCheckAfterTimePassed  &&  psFwOsData != NULL)
+	{
+		IMG_UINT32  ui32LISRCount   = 0;
+		IMG_UINT32  ui32FWCount     = 0;
+		IMG_UINT32  ui32MissingInts = 0;
+
+		/* Add up the total number of interrupts issued, sampled/received and missed... */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* Only the Host OS has a sample count, so only one counter to check. */
+		ui32LISRCount += psDevInfo->aui32SampleIRQCount[RGXFW_HOST_DRIVER_ID];
+		ui32FWCount   += OSReadHWReg32(psDevInfo->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[RGXFW_HOST_DRIVER_ID]);
+#else
+		IMG_UINT32  ui32Index;
+
+		for (ui32Index = 0;  ui32Index < RGXFW_THREAD_NUM;  ui32Index++)
+		{
+			ui32LISRCount += psDevInfo->aui32SampleIRQCount[ui32Index];
+			ui32FWCount   += psFwOsData->aui32InterruptCount[ui32Index];
+		}
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+		if (ui32LISRCount < ui32FWCount)
+		{
+			ui32MissingInts = (ui32FWCount-ui32LISRCount);
+		}
+
+		if (ui32LISRCount == psDevInfo->ui32InterruptCountLastTime  &&
+		    ui32MissingInts >= psDevInfo->ui32MissingInterruptsLastTime  &&
+		    psDevInfo->ui32MissingInterruptsLastTime > 1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: LISR has not received the last %d interrupts",
+					__func__, ui32MissingInts));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+
+			/* Schedule the MISRs to help mitigate the problems of missing interrupts. */
+			OSScheduleMISR(psDevInfo->pvMISRData);
+			if (psDevInfo->pvAPMISRData != NULL)
+			{
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+		psDevInfo->ui32InterruptCountLastTime    = ui32LISRCount;
+		psDevInfo->ui32MissingInterruptsLastTime = ui32MissingInts;
+	}
+
+	/* Release power lock before RGXCheckForStalledClientContexts */
+	PVRSRVPowerUnlock(psDevNode);
+
+	/*
+	   Stalled CCB check...
+	*/
+	if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
+	{
+		RGXCheckForStalledClientContexts(psDevInfo, IMG_FALSE);
+	}
+
+	/* Notify client driver and system layer of any eNewStatus errors */
+	if (eNewStatus > PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		/* Client notification of device error will be achieved by
+		 * clients calling UM function RGXGetLastDeviceError() */
+		psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+
+		/* Notify system layer */
+		{
+			PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+			PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+			if (psDevConfig->pfnSysDevErrorNotify)
+			{
+				PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+				sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+				sErrorData.uErrData.sHostWdgData.ui32Status = (IMG_UINT32)eNewStatus;
+				sErrorData.uErrData.sHostWdgData.ui32Reason = (IMG_UINT32)eNewReason;
+
+				psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+				                                  &sErrorData);
+			}
+		}
+	}
+
+	/*
+	   Finished, save the new status...
+	*/
+_RGXUpdateHealthStatus_Exit:
+	OSAtomicWrite(&psDevNode->eHealthStatus, eNewStatus);
+	OSAtomicWrite(&psDevNode->eHealthReason, eNewReason);
+	RGXSRV_HWPERF_DEVICE_INFO_HEALTH(psDevInfo, eNewStatus, eNewReason);
+
+	/*
+	 * Attempt to service the HWPerf buffer to regularly transport idle/periodic
+	 * packets to host buffer.
+	 */
+	if (psDevNode->pfnServiceHWPerf != NULL)
+	{
+		PVRSRV_ERROR eError = psDevNode->pfnServiceHWPerf(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: "
+			         "Error occurred when servicing HWPerf buffer (%d)",
+			         __func__, eError));
+		}
+	}
+
+	/* Attempt to refresh timer correlation data */
+	RGXTimeCorrRestartPeriodic(psDevNode);
+
+	return PVRSRV_OK;
+} /* RGXUpdateHealthStatus */
+
+#if defined(SUPPORT_AUTOVZ)
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+		(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))))
+	{
+		/* read and write back the alive token value to confirm to the
+		 * virtualisation watchdog that this connection is healthy */
+		KM_SET_OS_ALIVE_TOKEN(KM_GET_FW_ALIVE_TOKEN(psDevInfo), psDevInfo);
+		KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH);
+	}
+}
+
+/*
+	RGXUpdateAutoVzWatchdog
+*/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode)
+{
+	if (likely(psDevNode != NULL))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+
+		if (unlikely((psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || !psDevInfo->bRGXPowered ||
+					  psDevInfo->pvRegsBaseKM == NULL || psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+					  psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)))
+		{
+			/* If the firmware is not initialised, stop here */
+			return;
+		}
+		else
+		{
+			PVRSRV_ERROR eError = PVRSRVPowerLock(psDevNode);
+			PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PVRSRVPowerLock");
+
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+	}
+}
+
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DriverID;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+	                           INVALIDATE);
+
+	for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+	     ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED;
+	     ui32DriverID++)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+			psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+		if (eGuestState == RGXFW_CONNECTION_FW_ACTIVE ||
+		    eGuestState == RGXFW_CONNECTION_FW_READY)
+		{
+			PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+		}
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer)
+{
+	PVRSRV_ERROR eError;
+	PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
+
+	PVR_ASSERT((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+	PVR_RETURN_IF_INVALID_PARAM((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+
+	if (bKick)
+	{
+		if (psKMHWRTDataSet)
+		{
+			PRGXFWIF_CLEANUP_CTL psCleanupCtl;
+
+			eError = RGXSetFirmwareAddress(&psCleanupCtl, psKMHWRTDataSet->psHWRTDataFwMemDesc,
+					offsetof(RGXFWIF_HWRTDATA, sCleanupState),
+					RFW_FWADDR_NOREF_FLAG);
+			PVR_RETURN_IF_ERROR(eError);
+
+			*(psCleanupCtlWrite++) = psCleanupCtl;
+		}
+
+		if (eDM == RGXFWIF_DM_3D)
+		{
+			RGXFWIF_PRBUFFER_TYPE eBufferType;
+			RGX_ZSBUFFER_DATA *psBuffer = NULL;
+
+			for (eBufferType = RGXFWIF_PRBUFFER_START; eBufferType < RGXFWIF_PRBUFFER_MAXSUPPORTED; eBufferType++)
+			{
+				switch (eBufferType)
+				{
+				case RGXFWIF_PRBUFFER_ZSBUFFER:
+					psBuffer = psZSBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MSAABUFFER:
+					psBuffer = psMSAAScratchBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MAXSUPPORTED:
+					psBuffer = NULL;
+					break;
+				}
+				if (psBuffer)
+				{
+					(psCleanupCtlWrite++)->ui32Addr = psBuffer->sZSBufferFWDevVAddr.ui32Addr +
+									offsetof(RGXFWIF_PRBUFFER, sCleanupState);
+					psBuffer = NULL;
+				}
+			}
+		}
+	}
+
+	*pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
+	PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO       *psDevInfo;
+	RGXFWIF_HWRINFOBUF       *psHWRInfoBuf;
+	IMG_UINT32               i;
+
+	if (psDevNode->pvDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVINFO;
+	}
+	psDevInfo = psDevNode->pvDevice;
+
+	psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+	for (i = 0 ; i < psDevInfo->sDevFeatureCfg.ui32MAXDMCount ; i++)
+	{
+		/* Reset the HWR numbers */
+		psHWRInfoBuf->aui32HwrDmLockedUpCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmFalseDetectCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmRecoveredCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmOverranCount[i] = 0;
+	}
+
+	for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+	{
+		psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
+	}
+
+	psHWRInfoBuf->ui32WriteIndex = 0;
+	psHWRInfoBuf->ui32DDReqCount = 0;
+
+	OSWriteMemoryBarrier(&psHWRInfoBuf->ui32DDReqCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid)
+{
+
+	PVRSRV_ERROR eError;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+								 ui32Log2PageSize,
+								 ui32NumOfPages,
+								 ui32LogicalOffset,
+								 psPhyAddr,
+								 bValid,
+								 DEVICE_USE);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR_DevPhysAddr failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRUnLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	return eError;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_OK);
+
+	if (psDevInfo->bDumpedKCCBCtlAlready)
+	{
+		/* exiting capture range or pdump block */
+		psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+		/* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+                              PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
+                              "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
+                              psDevInfo->psKernelCCBCtl,
+                              ui32WriteOffset,
+                              ui32WriteOffset);
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+				offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+				ui32WriteOffset,
+				0xffffffff,
+				PDUMP_POLL_OPERATOR_EQUAL,
+				PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: problem pdumping POL for kCCBCtl (%d)", __func__, eError));
+		}
+	}
+
+	return eError;
+
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXClientConnectCompatCheck_ClientAgainstFW
+
+ @Description
+
+ Check compatibility of client and firmware (build options)
+ at the connection time.
+
+ @Input psDeviceNode - device node
+ @Input ui32ClientBuildOptions - build options for the client
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+******************************************************************************/
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32ClientBuildOptions)
+{
+	IMG_UINT32		ui32BuildOptionsMismatch;
+	IMG_UINT32		ui32BuildOptionsFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_FW_INFO_HEADER *psFWInfoHeader;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psDevInfo == NULL || psDevInfo->psRGXFWIfOsInitMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Cannot acquire kernel fw compatibility check info, RGXFWIF_OSINIT structure not allocated.",
+		         __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *) &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+#endif
+
+	ui32BuildOptionsFW = psFWInfoHeader->ui32Flags;
+	ui32BuildOptionsMismatch = ui32ClientBuildOptions ^ ui32BuildOptionsFW;
+
+	if (ui32BuildOptionsMismatch != 0)
+	{
+		if ((ui32ClientBuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in client: (0x%x). Please check rgx_options.h",
+					 ui32ClientBuildOptions & ui32BuildOptionsMismatch ));
+		}
+
+		if ((ui32BuildOptionsFW & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFW & ui32BuildOptionsMismatch ));
+		}
+
+		return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware and client build options match. [ OK ]", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapAllocMap
+
+ @Description Register firmware heap for the specified driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID - Guest driver
+ @Input sDevPAddr    - Heap address
+ @Input ui64DevPSize - Heap size
+
+ @Return   PVRSRV_ERROR - PVRSRV_OK if heap setup was successful.
+
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR szRegionRAName[RA_MAX_NAME_LENGTH];
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_MEMALLOCFLAGS_T uiRawFwHeapAllocFlags = (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+													PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PREMAP0 + ui32DriverID));
+	PHYS_HEAP_CONFIG *psFwHeapConfig = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+	PHYS_HEAP_CONFIG sFwHeapConfig;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psFwHeapConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW_MAIN heap config not found."));
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	OSSNPrintf(szRegionRAName, sizeof(szRegionRAName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	if (!ui64DevPSize ||
+		!sDevPAddr.uiAddr ||
+		ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED ||
+		ui64DevPSize != RGX_FIRMWARE_RAW_HEAP_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid parameters for %s", szRegionRAName));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sFwHeapConfig = *psFwHeapConfig;
+	sFwHeapConfig.eType = PHYS_HEAP_TYPE_LMA;
+	sFwHeapConfig.ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP;
+	sFwHeapConfig.uConfig.sLMA.sStartAddr.uiAddr = 0;
+	sFwHeapConfig.uConfig.sLMA.sCardBase.uiAddr = sDevPAddr.uiAddr;
+	sFwHeapConfig.uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	eError = PhysmemCreateHeapLMA(psDeviceNode,
+	                              RGXPhysHeapGetLMAPolicy(sFwHeapConfig.ui32UsageFlags, psDeviceNode),
+	                              &sFwHeapConfig,
+	                              szRegionRAName,
+	                              &psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysmemCreateHeapLMA:PREMAP [%d]", ui32DriverID);
+
+	eError = PhysHeapAcquire(psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysHeapAcquire:PREMAP [%d]", ui32DriverID);
+
+	psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID] = psDeviceNode->apsFWPremapPhysHeap[ui32DriverID];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate and map raw firmware heap for DriverID: [%d]", ui32DriverID);
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* don't clear the heap of other guests on allocation */
+	uiRawFwHeapAllocFlags &= (ui32DriverID > RGXFW_HOST_DRIVER_ID) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL);
+#endif
+
+	/* if the firmware is already powered up, consider the firmware heaps are pre-mapped. */
+	if (psDeviceNode->bAutoVzFwIsUp)
+	{
+		uiRawFwHeapAllocFlags &= RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Allocation and mapping for Firmware heaps done by TEE.", __func__));
+#else
+	eError = DevmemFwAllocate(psDevInfo,
+							  RGX_FIRMWARE_RAW_HEAP_SIZE,
+							  uiRawFwHeapAllocFlags,
+							  psDevInfo->psPremappedFwRawHeap[ui32DriverID]->pszName,
+							  &psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+#endif
+
+	/* Mark this devmem heap as premapped so allocations will not require device mapping. */
+	DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+
+	if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+	{
+		/* if the Host's raw fw heap is premapped, mark its main & config sub-heaps accordingly
+		 * No memory allocated from these sub-heaps will be individually mapped into the device's
+		 * address space so they can remain marked permanently as premapped. */
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapUnmapFree
+
+ @Description Unregister firmware heap for the specified guest driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID
+
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* remove the premap status, so the heap can be unmapped and freed */
+	if (psDevInfo->psPremappedFwRawHeap[ui32DriverID])
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_FALSE);
+	}
+
+	if (psDevInfo->psPremappedFwRawMemDesc[ui32DriverID])
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+		psDevInfo->psPremappedFwRawMemDesc[ui32DriverID] = NULL;
+	}
+}
+
+/*
+	RGXReadMETAAddr
+*/
+static PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO	*psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem  *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem  *pui8RegBase = pvRegBase;
+	IMG_UINT32 ui32PollValue;
+	IMG_UINT32 ui32PollMask;
+	IMG_UINT32 ui32PollRegOffset;
+	IMG_UINT32 ui32ReadOffset;
+	IMG_UINT32 ui32WriteOffset;
+	IMG_UINT32 ui32WriteValue;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+	{
+		ui32PollValue = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollMask = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1;
+		ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0;
+		ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN;
+		ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX;
+	}
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteUncheckedHWReg32(pvRegBase, ui32WriteOffset, ui32WriteValue);
+	(void)OSReadUncheckedHWReg32(pvRegBase, ui32WriteOffset);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadUncheckedHWReg32(pvRegBase, ui32ReadOffset);
+#else
+	IMG_UINT8 __iomem  *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteHWReg32(
+			psDevInfo->pvRegsBaseKM,
+			RGX_CR_META_SP_MSLVCTRL0,
+			ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGXWriteMETAAddr
+*/
+static PVRSRV_ERROR RGXWriteMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 ui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem *pui8RegBase = pvRegBase;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA, ui32Value);
+		}
+		else
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA, ui32Value);
+		}
+	}
+	else
+	{
+		/* Wait for Slave Port to be Ready */
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+				(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Issue the Write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0); /* Fence write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT); /* Fence write */
+	}
+#else
+	IMG_UINT8 __iomem *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Write */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 *pui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXReadMETAAddr(psDevInfo, ui32FWAddr, pui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvReadMem(psDevInfo, ui32FWAddr, pui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 ui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXWriteMETAAddr(psDevInfo, ui32FWAddr, ui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvWriteMem(psDevInfo, ui32FWAddr, ui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32FwVA,
+                             IMG_CPU_PHYADDR *psCpuPA,
+                             IMG_DEV_PHYADDR *psDevPA,
+                             IMG_UINT64 *pui64RawPTE)
+{
+	PVRSRV_ERROR eError       = PVRSRV_OK;
+	IMG_CPU_PHYADDR sCpuPA    = {0U};
+	IMG_DEV_PHYADDR sDevPA    = {0U};
+	IMG_UINT64 ui64RawPTE     = 0U;
+	MMU_FAULT_DATA sFaultData = {0U};
+	MMU_CONTEXT *psFwMMUCtx   = psDevInfo->psKernelMMUCtx;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+	IMG_UINT32 ui32DriverID   = (ui32FwVA - ui32FwHeapBase) / RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT32 ui32HeapId;
+	PHYS_HEAP *psPhysHeap;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* MIPS uses the same page size as the OS, while others default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) ?
+	                             OSGetPageSize() : BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#else
+	/* default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#endif
+
+	IMG_UINT32 ui32PageOffset = (ui32FwVA & (ui32FwPageSize - 1));
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM((ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((psCpuPA != NULL) ||
+	                               (psDevPA != NULL) ||
+	                               (pui64RawPTE != NULL)),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((ui32FwVA >= ui32FwHeapBase) &&
+	                              (ui32FwVA < ui32FwHeapEnd)),
+	                              eError, ErrorExit);
+
+	ui32HeapId = (ui32DriverID == RGXFW_HOST_DRIVER_ID) ?
+	              PVRSRV_PHYS_HEAP_FW_MAIN : (PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID);
+	psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[ui32HeapId];
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS is equipped with a dedicated MMU */
+		RGXMipsCheckFaultAddress(psFwMMUCtx, ui32FwVA, &sFaultData);
+	}
+	else
+#endif
+	{
+		IMG_UINT64 ui64FwDataBaseMask;
+		IMG_DEV_VIRTADDR sDevVAddr;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			ui64FwDataBaseMask = ~(RGXFW_SEGMMU_DATA_META_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_BASE_ADDRESS);
+		}
+#if !defined(EMULATOR)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			ui64FwDataBaseMask = ~RGXRISCVFW_REGION_MASK;
+		}
+#endif
+		else
+		{
+			PVR_LOG_GOTO_WITH_ERROR("RGXGetFwMapping", eError, PVRSRV_ERROR_NOT_IMPLEMENTED, ErrorExit);
+		}
+
+		sDevVAddr.uiAddr = (ui32FwVA & ui64FwDataBaseMask) | RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Fw CPU shares a subset of the GPU's VA space */
+		MMU_CheckFaultAddress(psFwMMUCtx, &sDevVAddr, &sFaultData);
+	}
+
+	ui64RawPTE = sFaultData.sLevelData[MMU_LEVEL_1].ui64Address;
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		IMG_BOOL bValidPage = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+		                       BITMASK_HAS(ui64RawPTE, RGXMIPSFW_TLB_VALID) :
+		                       BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#else
+		IMG_BOOL bValidPage = BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#endif
+
+		if (!bValidPage)
+		{
+			/* don't report invalid pages */
+			eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+			sDevPA.uiAddr = ui32PageOffset + ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			                RGXMIPSFW_TLB_GET_PA(ui64RawPTE) :
+			                (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK));
+#else
+			sDevPA.uiAddr = ui32PageOffset + (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK);
+#endif
+
+			/* Only the Host's Firmware heap is present in the Host's CPU IPA space */
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPA, &sDevPA);
+			}
+			else
+			{
+				sCpuPA.uiAddr = 0U;
+			}
+		}
+	}
+
+	if (psCpuPA != NULL)
+	{
+		*psCpuPA = sCpuPA;
+	}
+
+	if (psDevPA != NULL)
+	{
+		*psDevPA = sDevPA;
+	}
+
+	if (pui64RawPTE != NULL)
+	{
+		*pui64RawPTE = ui64RawPTE;
+	}
+
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = ui32VzConnectionCooldownPeriodInSec;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Vz reconnect request cooldown period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32VzConnectionCooldownPeriodInSec),
+							  ui32VzConnectionCooldownPeriodInSec,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxfwutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxhwperf.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxhwperf.c
new file mode 100644
index 000000000000..67812e3c5d0a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxhwperf.c
@@ -0,0 +1,616 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+
+#include "info_page_defs.h"
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] =
+{
+#define X(a, b, c, d, e, f, g)  {a, b, 0xFF, d, e, f, NULL}
+RGX_CNT_BLK_TYPE_MODEL_DIRECT_LIST,
+RGX_CNT_BLK_TYPE_MODEL_INDIRECT_LIST
+#undef X
+};
+
+IMG_INTERNAL IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel)
+{
+    *ppsModel = gasCntBlkTypeModel;
+    return ARRAY_SIZE(gasCntBlkTypeModel);
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXHWPerfMaxDefinedBlks
+
+ @Description Return the number of valid block-IDs for the given device node
+
+ @Input       (PVRSRV_RGXDEV_INFO *)   pvDevice    device-node to query
+
+ @Returns     (IMG_UINT32)             Number of block-IDs (RGX_CNTBLK_ID)
+                                       valid for this device.
+******************************************************************************/
+IMG_INTERNAL IMG_UINT32
+RGXHWPerfMaxDefinedBlks(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_HWPERF_CNTBLK_RT_INFO sRtInfo;
+	IMG_UINT32  uiRetVal;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *psHWPBlkConfig;
+	IMG_UINT32  uiNumArrayEls, ui;
+
+	uiRetVal = RGX_CNTBLK_ID_DIRECT_LAST;
+
+	uiNumArrayEls = RGXGetHWPerfBlockConfig(&psHWPBlkConfig);
+
+	if (psHWPBlkConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected NULL Config Block", __func__));
+		return 0;
+	}
+	PVR_ASSERT(uiNumArrayEls > 0);
+
+	/* Iterate over each block-ID and find the number of instances of each
+	 * block which are present for this device type. We only query the
+	 * Indirect blocks as their presence varies according to GPU. All direct
+	 * blocks have an entry - but they may not be physically present.
+	 */
+	for (ui = RGX_CNTBLK_ID_DIRECT_LAST; ui < uiNumArrayEls; ui++)
+	{
+		if (rgx_hwperf_blk_present(&psHWPBlkConfig[ui], (void *)psDevInfo, &sRtInfo))
+		{
+			uiRetVal += sRtInfo.uiNumUnits;
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Block %u, NumUnits %u, Total %u",
+			        __func__, ui, sRtInfo.uiNumUnits, uiRetVal));
+		}
+#ifdef DEBUG
+		else
+		{
+			if (psHWPBlkConfig[ui].uiCntBlkIdBase == RGX_CNTBLK_ID_RAC0)
+			{
+				if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevInfo->psDeviceNode,
+				    RAY_TRACING_ARCH) > 2U)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Block %u *NOT* present",
+					        __func__, ui));
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Block %u *NOT* present",
+				        __func__, ui));
+			}
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Num Units = %u", __func__, uiRetVal));
+
+	return uiRetVal;
+}
+
+static IMG_BOOL RGXServerFeatureFlagsToHWPerfFlagsAddBlock(
+	RGX_HWPERF_BVNC_BLOCK	* const psBlocks,
+	IMG_UINT16				* const pui16Count,
+	const IMG_UINT16		ui16BlockID, /* see RGX_HWPERF_CNTBLK_ID */
+	const IMG_UINT16		ui16NumCounters,
+	const IMG_UINT16		ui16NumBlocks)
+{
+	const IMG_UINT16 ui16Count = *pui16Count;
+
+	if (ui16Count < RGX_HWPERF_MAX_BVNC_BLOCK_LEN)
+	{
+		RGX_HWPERF_BVNC_BLOCK * const psBlock = &psBlocks[ui16Count];
+
+		/* If the GROUP is non-zero, convert from e.g. RGX_CNTBLK_ID_USC0 to
+		 * RGX_CNTBLK_ID_USC_ALL. The table stores the former (plus the
+		 * number of blocks and counters) but PVRScopeServices expects the
+		 * latter (plus the number of blocks and counters). The conversion
+		 * could always be moved to PVRScopeServices, but it's less code this
+		 * way.
+		 * For SLC0 we generate a single SLCBANK_ALL which has NUM_MEMBUS
+		 * instances.
+		 * This replaces the SLC0 .. SLC3 entries.
+		 */
+		if ((ui16BlockID == RGX_CNTBLK_ID_SLCBANK0) || (ui16BlockID & RGX_CNTBLK_ID_GROUP_MASK))
+		{
+			psBlock->ui16BlockID	= ui16BlockID | RGX_CNTBLK_ID_UNIT_ALL_MASK;
+		}
+		else
+		{
+			psBlock->ui16BlockID	= ui16BlockID;
+		}
+		psBlock->ui16NumCounters	= ui16NumCounters;
+		psBlock->ui16NumBlocks		= ui16NumBlocks;
+
+		*pui16Count = ui16Count + 1;
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_HWPERF_BVNC *psBVNC)
+{
+	IMG_PCHAR pszBVNC;
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDevInfo), "psDevInfo invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((pszBVNC = RGXDevBVNCString(psDevInfo)))
+	{
+		size_t uiStringLength = OSStringNLength(pszBVNC, RGX_HWPERF_MAX_BVNC_LEN - 1);
+		OSStringSafeCopy(psBVNC->aszBvncString, pszBVNC, uiStringLength + 1);
+		memset(&psBVNC->aszBvncString[uiStringLength], 0, RGX_HWPERF_MAX_BVNC_LEN - uiStringLength);
+	}
+	else
+	{
+		*psBVNC->aszBvncString = 0;
+	}
+
+	psBVNC->ui32BvncKmFeatureFlags = 0x0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERFBUS_FLAG;
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_MULTICORE_FLAG;
+	}
+
+	/* Determine if we've got the new RAY_TRACING feature supported. This is
+	 * only determined by comparing the RAY_TRACING_ARCHITECTURE value */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_RAYTRACING_FLAG;
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CATURIX_TOP_INFRASTRUCTURE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG;
+	}
+
+#ifdef SUPPORT_WORKLOAD_ESTIMATION
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Not a part of BVNC feature line and so doesn't need the feature supported check */
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION;
+	}
+#endif
+
+	/* Define the HW counter block counts. */
+	{
+		RGX_HWPERF_BVNC_BLOCK					* const psBlocks	= psBVNC->aBvncBlocks;
+		IMG_UINT16								* const pui16Count	= &psBVNC->ui16BvncBlocks;
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	*asCntBlkTypeModel;
+		const IMG_UINT32						ui32CntBlkModelLen	= RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+		IMG_UINT32								ui32BlkCfgIdx;
+		size_t									uiCount;
+		IMG_BOOL								bOk					= IMG_TRUE;
+
+		// Initialise to zero blocks
+		*pui16Count = 0;
+
+		// Add all the blocks
+		for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+		{
+			const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	* const psCntBlkInfo = &asCntBlkTypeModel[ui32BlkCfgIdx];
+			RGX_HWPERF_CNTBLK_RT_INFO				sCntBlkRtInfo;
+			/* psCntBlkInfo->uiNumUnits gives compile-time info. For BVNC agnosticism, we use this: */
+			if (psCntBlkInfo->pfnIsBlkPresent(psCntBlkInfo, psDevInfo, &sCntBlkRtInfo))
+			{
+				IMG_UINT32	uiNumUnits;
+
+				switch (psCntBlkInfo->uiCntBlkIdBase)
+				{
+					case RGX_CNTBLK_ID_SLCBANK0:
+						/* Generate the SLCBANK_ALL block for SLC0..SLC3
+						 * we have to special-case this as the iteration will
+						 * generate entries starting at SLC0 and we need to
+						 * defer until we are processing the last 'present'
+						 * entry.
+						 * The SLC_BLKID_ALL is keyed from SLC0. Need to access
+						 * the NUM_MEMBUS feature to see how many are physically
+						 * present.
+						 * For CXT_TOP_INFRASTRUCTURE systems we present a
+						 * singleton SLCBANK - this provides accumulated counts
+						 * for all registers within the physical SLCBANK instances.
+						 */
+						if (psBVNC->ui32BvncKmFeatureFlags &
+						    RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG)
+						{
+							uiNumUnits = 1U;
+						}
+						else
+						{
+							uiNumUnits = RGX_GET_FEATURE_VALUE(psDevInfo,
+							                                   NUM_MEMBUS);
+						}
+						break;
+					case RGX_CNTBLK_ID_SLCBANK1:
+					case RGX_CNTBLK_ID_SLCBANK2:
+					case RGX_CNTBLK_ID_SLCBANK3:
+						/* These are contained within SLCBANK_ALL block */
+						continue;
+					default:
+						uiNumUnits = sCntBlkRtInfo.uiNumUnits;
+						break;
+				}
+				bOk &= RGXServerFeatureFlagsToHWPerfFlagsAddBlock(psBlocks, pui16Count, psCntBlkInfo->uiCntBlkIdBase, RGX_CNTBLK_COUNTERS_MAX, uiNumUnits);
+			}
+		}
+
+		/* If this fails, consider why the static_assert didn't fail, and consider increasing RGX_HWPERF_MAX_BVNC_BLOCK_LEN */
+		PVR_ASSERT(bOk);
+
+		// Zero the remaining entries
+		uiCount = *pui16Count;
+		OSDeviceMemSet(&psBlocks[uiCount], 0, (RGX_HWPERF_MAX_BVNC_BLOCK_LEN - uiCount) * sizeof(*psBlocks));
+	}
+
+	/* The GPU core count is overwritten by the FW */
+	psBVNC->ui16BvncGPUCores = 0;
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVRGXConfigureHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+		CONNECTION_DATA          * psConnection,
+		PVRSRV_DEVICE_NODE       * psDeviceNode,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32ArrayLen,
+		RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs)
+{
+	PVRSRV_ERROR             eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD         sKccbCmd;
+	DEVMEM_MEMDESC           *psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_CNTBLK *psFwArray;
+	IMG_UINT32               ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO       *psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32CtrlWord = ui32CtrlWord;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_CNTBLK) * ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.sBlockConfigs,
+	                               psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/* Ask the FW to carry out the HWPerf configuration command. */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration. */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED",
+	         ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfConfigureCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32					ui32CtrlWord,
+		IMG_UINT32					ui32NumBlocks,
+		RGX_HWPERF_CONFIG_CNTBLK*	asBlockConfigs)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !asBlockConfigs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXConfigureHWPerfBlocksKM(NULL,
+		                                          psDevData->psRgxDevNode,
+		                                          ui32CtrlWord,
+		                                          ui32NumBlocks,
+		                                          asBlockConfigs);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXConfigureHWPerfBlocksKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) < RGX_CNTBLK_ID_LAST)
+	{
+		RGXFWIF_HWPERF_CTL_BLK *psBlock = NULL;
+		RGX_HWPERF_CONFIG_CNTBLK sBlockConfig;
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		for (i = 0; i < psHWPerfCtl->ui32NumBlocks; i++)
+		{
+			if (psHWPerfCtl->sBlkCfg[i].uiBlockID != ui32BlockID)
+			{
+				continue;
+			}
+			else if (psHWPerfCtl->sBlkCfg[i].uiEnabled == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+			}
+
+			psBlock = &psHWPerfCtl->sBlkCfg[i];
+			break;
+		}
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->uiBlockID;
+		sBlockConfig.ui16NumCounters = psBlock->uiNumCounters;
+
+		for (i = 0; i < psBlock->uiNumCounters; i++)
+		{
+			sBlockConfig.ui16Counters[i] = psBlock->aui32CounterCfg[i];
+		}
+
+		*psConfiguredCounters = sBlockConfig;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLen,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	IMG_UINT32 ui32LastIdx = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	*pui32BlockCount = 0;
+
+	if (ui32ArrayLen > 0 && pui32EnabledBlockIDs == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen is greater than 0 but pui32EnabledBlockIDs is NULL"));
+	}
+
+	PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+	for (i = 0; i < psHWPerfCtl->ui32NumBlocks; i++)
+	{
+		if (psHWPerfCtl->sBlkCfg[i].uiEnabled)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sBlkCfg[i].uiBlockID;
+			ui32LastIdx++;
+		}
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxinit.c
new file mode 100644
index 000000000000..eca671110a4f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxinit.c
@@ -0,0 +1,5603 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_bridge_init.h"
+#include "rgx_bridge_init.h"
+#include "syscommon.h"
+#include "rgx_heaps_server.h"
+#include "rgxheapconfig.h"
+#include "rgxpower.h"
+#include "tlstream.h"
+#include "pvrsrv_tlstreams.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "pvr_ricommon.h"
+#endif
+
+#include "rgxinit.h"
+#include "rgxbvnc.h"
+#include "rgxmulticore.h"
+
+#include "pdump_km.h"
+#include "handle.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxmem.h"
+#include "sync_internal.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "rgxfwdbg.h"
+#include "info_page.h"
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxfwimageutils.h"
+#endif
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+#include "rgxmmuinit.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#include "physmem.h"
+#endif
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "rgxta3d.h"
+#include "rgxtimecorr.h"
+#include "rgxshader.h"
+
+#if defined(PDUMP)
+#include "rgxstartstop.h"
+#endif
+
+#include "rgx_fwif_alignchecks.h"
+#include "vmm_pvz_client.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+
+#if defined(PDUMP) && defined(SUPPORT_SECURITY_VALIDATION)
+#include "pdump_physmem.h"
+#endif
+
+#if defined(PDUMP)
+#include "rgxpdump_common.h"
+#endif
+
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_CHAR **ppszVersionString);
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PUINT32 pui32RGXClockSpeed);
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID);
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/* Services internal heap identification used in this file only */
+#define RGX_FIRMWARE_MAIN_HEAP_IDENT   "FwMain"   /*!< RGX Main Firmware Heap identifier */
+#define RGX_FIRMWARE_CONFIG_HEAP_IDENT "FwConfig" /*!< RGX Config firmware Heap identifier */
+
+#define RGX_MMU_PAGE_SIZE_4KB   (   4 * 1024)
+#define RGX_MMU_PAGE_SIZE_16KB  (  16 * 1024)
+#define RGX_MMU_PAGE_SIZE_64KB  (  64 * 1024)
+#define RGX_MMU_PAGE_SIZE_256KB ( 256 * 1024)
+#define RGX_MMU_PAGE_SIZE_1MB   (1024 * 1024)
+#define RGX_MMU_PAGE_SIZE_2MB   (2048 * 1024)
+#define RGX_MMU_PAGE_SIZE_MIN RGX_MMU_PAGE_SIZE_4KB
+#define RGX_MMU_PAGE_SIZE_MAX RGX_MMU_PAGE_SIZE_2MB
+
+#define VAR(x) #x
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@Function       SampleIRQCount
+@Description    Utility function taking snapshots of RGX FW interrupt count.
+@Input          psDevInfo    Device Info structure
+
+@Return         IMG_BOOL     Returns IMG_TRUE if RGX FW IRQ is not equal to
+                             sampled RGX FW IRQ count for any RGX FW thread.
+ */ /**************************************************************************/
+static INLINE IMG_BOOL SampleIRQCount(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bReturnVal = IMG_FALSE;
+	volatile IMG_UINT32 *pui32SampleIrqCount = psDevInfo->aui32SampleIRQCount;
+	IMG_UINT32 ui32IrqCnt;
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bReturnVal = IMG_TRUE;
+	}
+	else
+	{
+		get_irq_cnt_val(ui32IrqCnt, RGXFW_HOST_DRIVER_ID, psDevInfo);
+
+		if (ui32IrqCnt != pui32SampleIrqCount[RGXFW_THREAD_0])
+		{
+			pui32SampleIrqCount[RGXFW_THREAD_0] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#else
+	IMG_UINT32 ui32TID;
+
+	for_each_irq_cnt(ui32TID)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32TID, psDevInfo);
+
+		/* treat unhandled interrupts here to align host count with fw count */
+		if (pui32SampleIrqCount[ui32TID] != ui32IrqCnt)
+		{
+			pui32SampleIrqCount[ui32TID] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#endif
+
+	return bReturnVal;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHostSafetyEvents
+@Description    Returns the event status masked to keep only the safety
+                events handled by the Host
+@Input          psDevInfo    Device Info structure
+@Return         IMG_UINT32   Status of Host-handled safety events
+ */ /**************************************************************************/
+static INLINE IMG_UINT32 RGXHostSafetyEvents(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (psDevInfo->ui32HostSafetyEventMask == 0))
+	{
+		return 0;
+	}
+	else
+	{
+		IMG_UINT32 ui32EventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_STATUS);
+
+		return (ui32EventStatus & psDevInfo->ui32HostSafetyEventMask);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       GetContextResetReason
+@Description    Maps a bit corresponding to an uncorrected FW fault reported in
+                RGX_CR_FAULT_FW_STATUS register into RGX_CONTEXT_RESET_REASON value
+@Input          ui32FaultFlag Offset into the bitmask of uncorrected FW faults
+                psDevInfo Device Info structure
+@Return         RGX_CONTEXT_RESET_REASON Reset Reason
+ */ /**************************************************************************/
+static inline RGX_CONTEXT_RESET_REASON GetContextResetReason(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FaultFlag)
+{
+	RGX_CONTEXT_RESET_REASON eResetReason;
+
+	if (ui32FaultFlag == RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_SHIFT)
+	{
+		IMG_UINT32 ui32ParityStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_PT_PARITY_STATUS);
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_PT_PARITY_CLEAR, ui32ParityStatus);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: MMU PTE parity errors: 0x%x", __func__, ui32ParityStatus));
+
+		eResetReason = RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR;
+	}
+	else if (ui32FaultFlag == RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_SHIFT)
+	{
+		eResetReason = RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR;
+	}
+	else if (ui32FaultFlag == RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_SHIFT)
+	{
+		eResetReason = RGX_CONTEXT_RESET_REASON_DCLS_ERR;
+	}
+	else
+	{
+		eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_ERR;
+	}
+
+	return eResetReason;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventHandler
+@Description    Handles the Safety Events that the Host is responsible for
+@Input          psDevInfo    Device Info structure
+ */ /**************************************************************************/
+static void RGXSafetyEventHandler(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	if (psDevInfo->bRGXPowered)
+	{
+		RGX_CONTEXT_RESET_REASON eResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+		IMG_UINT32 ui32HostSafetyStatus = RGXHostSafetyEvents(psDevInfo);
+		IMG_UINT32 ui32HostSafetyStatusBitPos = 0;
+
+		if (ui32HostSafetyStatus != 0)
+		{
+			/* clear the safety bus events handled by the Host */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_CLEAR, ui32HostSafetyStatus);
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_EVENT_STATUS_FAULT_FW_SHIFT))
+			{
+				IMG_UINT32 ui32FaultFlag;
+				IMG_UINT32 ui32FaultFW = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_STATUS);
+				IMG_UINT32 ui32CorrectedBitOffset = RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_SHIFT -
+													RGX_CR_FAULT_FW_STATUS_MMU_DETECT_SHIFT;
+
+				PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety fault status: 0x%X", __func__, ui32FaultFW));
+
+				for (ui32FaultFlag = 0; ui32FaultFlag < ui32CorrectedBitOffset; ui32FaultFlag++)
+				{
+					if (BIT_ISSET(ui32FaultFW, ui32FaultFlag))
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety hardware fault detected (0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+						eResetReason = GetContextResetReason(psDevInfo, ui32FaultFlag);
+					}
+					else if BIT_ISSET(ui32FaultFW, ui32FaultFlag + ui32CorrectedBitOffset)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Firmware safety hardware fault corrected.(0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+
+						/* Only report this if we haven't detected a more serious error */
+						if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR &&
+						    eResetReason != RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR &&
+							eResetReason != RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR &&
+							eResetReason != RGX_CONTEXT_RESET_REASON_DCLS_ERR)
+
+						{
+							eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_OK;
+						}
+					}
+				}
+
+				OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_CLEAR, ui32FaultFW);
+			}
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+				RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0)
+			{
+				ui32HostSafetyStatusBitPos = RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT;
+			}
+			else
+#endif
+			{
+				ui32HostSafetyStatusBitPos = RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT;
+			}
+
+			if (BIT_ISSET(ui32HostSafetyStatus, ui32HostSafetyStatusBitPos))
+			{
+				volatile RGXFWIF_POW_STATE ePowState;
+
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+				                           INVALIDATE);
+				ePowState = psDevInfo->psRGXFWIfFwSysData->ePowState;
+
+				if (ePowState != RGXFWIF_POW_OFF)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Safety Watchdog Trigger !", __func__));
+
+					/* Only report this if we haven't detected a more serious error */
+					if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR &&
+					    eResetReason != RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR)
+					{
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_WATCHDOG;
+					}
+				}
+			}
+
+			/* Notify client and system layer of any error */
+			if (eResetReason != RGX_CONTEXT_RESET_REASON_NONE)
+			{
+				PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+				PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = eResetReason;
+
+				/* Notify system layer of any error */
+				if (psDevConfig->pfnSysDevErrorNotify)
+				{
+					PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+					sErrorData.eResetReason = eResetReason;
+
+					psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+					                                  &sErrorData);
+				}
+			}
+		}
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+static IMG_BOOL _WaitForInterruptsTimeoutCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32idx;
+#endif
+
+	RGXDEBUG_PRINT_IRQ_COUNT(psDevInfo);
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVR_DPF((PVR_DBG_ERROR,
+	        "Last RGX_LISRHandler State (DevID %u): 0x%08X Clock: %" IMG_UINT64_FMTSPEC,
+			 psDeviceNode->sDevId.ui32InternalID,
+			 psDeviceNode->sLISRExecutionInfo.ui32Status,
+			 psDeviceNode->sLISRExecutionInfo.ui64Clockns));
+
+	for_each_irq_cnt(ui32idx)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				MSG_IRQ_CNT_TYPE " %u: InterruptCountSnapshot: 0x%X",
+				ui32idx, psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[ui32idx]));
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR, "No further information available. Please enable PVRSRV_DEBUG_LISR_EXECUTION"));
+#endif
+
+	return SampleIRQCount(psDevInfo);
+}
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bScheduleMISR;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bScheduleMISR = IMG_TRUE;
+	}
+	else
+	{
+		bScheduleMISR = _WaitForInterruptsTimeoutCheck(psDevInfo);
+	}
+
+	if (bScheduleMISR)
+	{
+		OSScheduleMISR(psDevInfo->pvMISRData);
+
+		if (psDevInfo->pvAPMISRData != NULL)
+		{
+			OSScheduleMISR(psDevInfo->pvAPMISRData);
+		}
+	}
+}
+
+static inline IMG_BOOL RGXAckHwIrq(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_UINT32 ui32IRQStatusReg,
+								   IMG_UINT32 ui32IRQStatusEventMsk,
+								   IMG_UINT32 ui32IRQClearReg,
+								   IMG_UINT32 ui32IRQClearMask)
+{
+	IMG_UINT32 ui32IRQStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQStatusReg);
+
+	/* clear only the pending bit of the thread that triggered this interrupt */
+	ui32IRQClearMask &= ui32IRQStatus;
+
+	if (ui32IRQStatus & ui32IRQStatusEventMsk)
+	{
+		/* acknowledge and clear the interrupt */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQClearReg, ui32IRQClearMask);
+		return IMG_TRUE;
+	}
+	else
+	{
+		/* spurious interrupt */
+		return IMG_FALSE;
+	}
+}
+
+static __maybe_unused IMG_BOOL RGXAckIrqMETA(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static IMG_BOOL RGXAckIrqMIPS(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN);
+}
+#endif
+
+static IMG_BOOL RGXAckIrqDedicated(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+		/* status & clearing registers are available on both Host and Guests
+		 * and are agnostic of the Fw CPU type. Due to the remappings done by
+		 * the 2nd stage device MMU, all drivers assume they are accessing
+		 * register bank 0 */
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_IRQ_OS0_EVENT_STATUS,
+					   ~RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK,
+					   RGX_CR_IRQ_OS0_EVENT_CLEAR,
+					   ~RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK);
+}
+
+static PVRSRV_ERROR RGXSetAckIrq(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_SINGLE_IRQ_WORKAROUND)
+	/* Register appropriate mechanism for clearing hw interrupts */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psDevInfo->pfnRGXAckIrq = NULL;
+	}
+	else
+#endif
+	{
+		psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+	}
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL RGX_LISRHandler(void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bIrqAcknowledged = IMG_FALSE;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	IMG_UINT32 ui32idx, ui32IrqCnt;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+		UPDATE_LISR_DBG_SNAPSHOT(ui32idx, ui32IrqCnt);
+	}
+
+	UPDATE_LISR_DBG_STATUS(RGX_LISR_INIT);
+	UPDATE_LISR_DBG_TIMESTAMP();
+#endif
+
+	UPDATE_LISR_DBG_COUNTER();
+
+	if (psDevInfo->bRGXPowered)
+	{
+		IMG_BOOL bSafetyEvent = (RGXHostSafetyEvents(psDevInfo) != 0);
+
+		if ((psDevInfo->pfnRGXAckIrq == NULL) || psDevInfo->pfnRGXAckIrq(psDevInfo) || bSafetyEvent)
+		{
+			bIrqAcknowledged = IMG_TRUE;
+
+			if (bSafetyEvent || SampleIRQCount(psDevInfo))
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_PROCESSED);
+				UPDATE_MISR_DBG_COUNTER();
+
+				OSScheduleMISR(psDevInfo->pvMISRData);
+
+#if defined(SUPPORT_AUTOVZ)
+				RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+				if (psDevInfo->pvAPMISRData != NULL)
+				{
+					OSScheduleMISR(psDevInfo->pvAPMISRData);
+				}
+			}
+			else
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED);
+			}
+		}
+		else
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_NOT_TRIGGERED_BY_HW);
+		}
+	}
+	else
+	{
+#if defined(SUPPORT_AUTOVZ)
+		/* AutoVz drivers rebooting while the firmware is active must acknowledge
+		 * and clear the hw IRQ line before the RGXInit() has finished. */
+		if ((psDevInfo->pfnRGXAckIrq != NULL) &&
+			 psDevInfo->pfnRGXAckIrq(psDevInfo))
+		{
+			bIrqAcknowledged = IMG_TRUE;
+		}
+		else
+#endif
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_DEVICE_NOT_POWERED);
+		}
+	}
+
+	return bIrqAcknowledged;
+}
+
+static void RGX_MISR_ProcessKCCBDeferredList(PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* First check whether there are pending commands in Deferred KCCB List */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	if (dllist_is_empty(&psDevInfo->sKCCBDeferredCommandsListHead))
+	{
+		OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+		return;
+	}
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	/* Powerlock to avoid further Power transition requests
+	   while KCCB deferred list is being processed */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Try to send deferred KCCB commands Do not Poll from here*/
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s could not flush Deferred KCCB list, KCCB is full.",
+				 __func__));
+	}
+}
+
+static void RGX_MISRHandler_CheckFWActivePowerState(void *psDevice)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevice;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_ON || psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		/* The FW is IDLE and therefore could be shut down */
+		eError = RGXActivePowerRequest(psDeviceNode);
+
+		if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED))
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+					"%s: Failed RGXActivePowerRequest call (device: %p) with %s",
+					__func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+			else
+			{
+				/* Re-schedule the power down request as it was deferred. */
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+	}
+
+}
+
+/* Shorter defines to keep the code a bit shorter */
+#define GPU_IDLE       RGXFWIF_GPU_UTIL_STATE_IDLE
+#define GPU_ACTIVE     RGXFWIF_GPU_UTIL_STATE_ACTIVE
+#define GPU_BLOCKED    RGXFWIF_GPU_UTIL_STATE_BLOCKED
+#define GPU_INACTIVE   RGXFWIF_GPU_UTIL_STATE_INACTIVE
+#define MAX_ITERATIONS 64
+#define MAX_DIFF_TIME_NS (300000ULL)
+#define MAX_DIFF_DM_TIME_NS (MAX_DIFF_TIME_NS >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT)
+
+static PVRSRV_ERROR RGXGetGpuUtilStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_HANDLE hGpuUtilUser,
+                                       RGXFWIF_GPU_UTIL_STATS *psReturnStats)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_STATS sStats;
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+	IMG_UINT64 (*paaui64DMOSTmpCounters)[RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 (*paui64DMOSTmpLastWord)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastState)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastPeriod)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastTime)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT64 ui64TimeNowShifted;
+	IMG_UINT32 ui32Attempts;
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32MaxDMCount;
+	RGXFWIF_DM eDM;
+
+	/***** (1) Initialise return stats *****/
+
+	psReturnStats->bValid = IMG_FALSE;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psAggregateStats = hGpuUtilUser;
+
+	/* decrease by 1 to account for excluding GP DM from the statics */;
+	ui32MaxDMCount =  psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1;
+
+	/* Reset temporary counters used in the attempts loop */
+	paaui64DMOSTmpCounters  = &psAggregateStats->sTempGpuStats.aaaui64DMOSTmpCounters[0];
+	paui64DMOSTmpLastWord   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastWord[0];
+	paui64DMOSTmpLastState  = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastState[0];
+	paui64DMOSTmpLastPeriod = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastPeriod[0];
+	paui64DMOSTmpLastTime   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastTime[0];
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	/* Try to acquire GPU utilisation counters and repeat if the FW is in the middle of an update */
+	for (ui32Attempts = 0; ui32Attempts < 4; ui32Attempts++)
+	{
+		IMG_UINT64 aui64GpuTmpCounters[RGXFWIF_GPU_UTIL_STATE_NUM] = {0};
+		IMG_UINT64 ui64GpuLastPeriod = 0, ui64GpuLastWord = 0, ui64GpuLastState = 0, ui64GpuLastTime = 0;
+
+		/***** (2) Get latest data from shared area *****/
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			IMG_UINT64 aui64StatsCountersNew[RGXFWIF_GPU_UTIL_STATE_NUM];
+			IMG_UINT64 ui64GpuLastWordNew;
+			RGXFWIF_GPU_STATS sStatsNew;
+			IMG_UINT32 i = 0;
+
+			ui64GpuLastWord = 0;
+			ui64GpuLastState = 0;
+
+			OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+			/* Copy data from device memory */
+			memcpy(&sStatsNew, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+			memcpy(&ui64GpuLastWordNew, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+			memcpy(aui64StatsCountersNew, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64StatsCountersNew));
+
+			/*
+			 * First attempt at detecting if the FW is in the middle of an update.
+			 * This should also help if the FW is in the middle of a 64 bit variable update.
+			 * This loop must be fast. Faster than FW updates the stats.
+			 */
+			for (i = 0; i < MAX_ITERATIONS; i++)
+			{
+				IMG_UINT32 j,k;
+				IMG_BOOL bRetry = IMG_FALSE;
+
+				if (i > 0)
+				{
+					/* On retry keep previous data */
+					ui64GpuLastWordNew = ui64GpuLastWord;
+					memcpy(aui64StatsCountersNew, aui64GpuTmpCounters, sizeof(aui64StatsCountersNew));
+					memcpy(&sStatsNew, &sStats, sizeof(sStatsNew));
+				}
+
+				/* Copy data from device memory */
+				memcpy(&sStats, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+				memcpy(&ui64GpuLastWord, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+				memcpy(aui64GpuTmpCounters, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64GpuTmpCounters));
+
+				/* Check for abnormal time difference between reads */
+				if (RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord) - RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWordNew) > MAX_DIFF_TIME_NS)
+				{
+					bRetry = IMG_TRUE;
+					continue;
+				}
+
+				for (j = 0; j < RGXFWIF_GPU_UTIL_STATE_NUM; j++)
+				{
+					/* Check for abnormal time difference between reads */
+					if (aui64GpuTmpCounters[j] - aui64StatsCountersNew[j] > MAX_DIFF_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+				}
+
+				if (bRetry)
+				{
+					continue;
+				}
+
+				/* Check for DM counters wrapped or
+				   abnormal time difference between reads.
+				   The DM time is shifted by RGXFWIF_DM_OS_TIMESTAMP_SHIFT */
+				for (j = 0; j < RGXFWIF_GPU_UTIL_DM_MAX; j++)
+				{
+					if (sStats.aui32DMOSLastWordWrap[j] != sStatsNew.aui32DMOSLastWordWrap[j] ||
+						RGXFWIF_GPU_UTIL_GET_TIME32(sStats.aui32DMOSLastWord[j]) - RGXFWIF_GPU_UTIL_GET_TIME32(sStatsNew.aui32DMOSLastWord[j]) > MAX_DIFF_DM_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+
+					for (k = 0; k < RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM; k++)
+					{
+						if (sStats.aaui32DMOSCountersWrap[j][k] != sStatsNew.aaui32DMOSCountersWrap[j][k] ||
+							sStats.aaui32DMOSStatsCounters[j][k] - sStatsNew.aaui32DMOSStatsCounters[j][k] > MAX_DIFF_DM_TIME_NS)
+						{
+							bRetry = IMG_TRUE;
+							break;
+						}
+
+					}
+
+					if (bRetry)
+					{
+						break;
+					}
+				}
+
+				if (!bRetry)
+				{
+					/* Stats are good*/
+					break;
+				}
+			}
+
+			OSLockRelease(psDevInfo->hGPUUtilLock);
+
+			ui64GpuLastState = RGXFWIF_GPU_UTIL_GET_STATE(ui64GpuLastWord);
+
+			if (i == MAX_ITERATIONS)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						 "RGXGetGpuUtilStats could not get reliable data after trying %u times", i));
+
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+			{
+				paui64DMOSTmpLastWord[eDM][ui32DriverID]  =
+					((IMG_UINT64)sStats.aui32DMOSLastWordWrap[eDM] << 32) + sStats.aui32DMOSLastWord[eDM];
+				paui64DMOSTmpLastState[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_STATE(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				if (paui64DMOSTmpLastState[eDM][ui32DriverID] != GPU_ACTIVE)
+				{
+					paui64DMOSTmpLastState[eDM][ui32DriverID] = GPU_INACTIVE;
+				}
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE] = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_INACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_INACTIVE] << 32);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE]  = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_ACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_ACTIVE] << 32);
+			}
+
+		} /* FOREACH_SUPPORTED_DRIVER(ui32DriverID) */
+
+
+		/***** (3) Compute return stats *****/
+
+		/* Update temp counters to account for the time since the last update to the shared ones */
+		OSMemoryBarrier(NULL); /* Ensure the current time is read after the loop above */
+		ui64TimeNow    = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDeviceNode));
+
+		ui64GpuLastTime   = RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord);
+		ui64GpuLastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64GpuLastTime);
+		aui64GpuTmpCounters[ui64GpuLastState] += ui64GpuLastPeriod;
+
+		/* Get statistics for a user since its last request */
+		psReturnStats->ui64GpuStatIdle = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_IDLE],
+		                                                             psAggregateStats->ui64GpuStatIdle);
+		psReturnStats->ui64GpuStatActive = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_ACTIVE],
+		                                                               psAggregateStats->ui64GpuStatActive);
+		psReturnStats->ui64GpuStatBlocked = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_BLOCKED],
+		                                                                psAggregateStats->ui64GpuStatBlocked);
+		psReturnStats->ui64GpuStatCumulative = psReturnStats->ui64GpuStatIdle +
+		                                       psReturnStats->ui64GpuStatActive +
+		                                       psReturnStats->ui64GpuStatBlocked;
+
+		/* convert time into the same units as used by fw */
+		ui64TimeNowShifted  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+		for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+		{
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				paui64DMOSTmpLastTime[eDM][ui32DriverID]   = RGXFWIF_GPU_UTIL_GET_TIME(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				paui64DMOSTmpLastPeriod[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNowShifted , paui64DMOSTmpLastTime[eDM][ui32DriverID]);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][paui64DMOSTmpLastState[eDM][ui32DriverID]] += paui64DMOSTmpLastPeriod[eDM][ui32DriverID];
+				/* Get statistics for a user since its last request */
+				psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE],
+				                                                             psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE],
+				                                                               psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatCumulative[eDM][ui32DriverID] = psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] +
+				                                       psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+			}
+		}
+
+		if (psAggregateStats->ui64TimeStamp != 0)
+		{
+			IMG_UINT64 ui64TimeSinceLastCall = ui64TimeNow - psAggregateStats->ui64TimeStamp;
+			/* We expect to return at least 75% of the time since the last call in GPU stats */
+			IMG_UINT64 ui64MinReturnedStats = ui64TimeSinceLastCall - (ui64TimeSinceLastCall / 4);
+
+			/*
+			 * If the returned stats are substantially lower than the time since
+			 * the last call, then the Host might have read a partial update from the FW.
+			 * If this happens, try sampling the shared counters again.
+			 */
+			if (psReturnStats->ui64GpuStatCumulative < ui64MinReturnedStats)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s: Return stats (%" IMG_UINT64_FMTSPEC ") too low "
+				         "(call period %" IMG_UINT64_FMTSPEC ")",
+				         __func__, psReturnStats->ui64GpuStatCumulative, ui64TimeSinceLastCall));
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Attempt #%u has failed, trying again",
+				         __func__, ui32Attempts));
+				continue;
+			}
+		}
+
+		break;
+	}
+
+	/***** (4) Update aggregate stats for the current user *****/
+
+	psAggregateStats->ui64GpuStatIdle    += psReturnStats->ui64GpuStatIdle;
+	psAggregateStats->ui64GpuStatActive  += psReturnStats->ui64GpuStatActive;
+	psAggregateStats->ui64GpuStatBlocked += psReturnStats->ui64GpuStatBlocked;
+	psAggregateStats->ui64TimeStamp       = ui64TimeNow;
+
+	for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+	{
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]    += psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID];
+			psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]  += psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+		}
+	}
+
+	/***** (5) Convert return stats to microseconds *****/
+
+	psReturnStats->ui64GpuStatIdle       = OSDivide64(psReturnStats->ui64GpuStatIdle, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatActive     = OSDivide64(psReturnStats->ui64GpuStatActive, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatBlocked    = OSDivide64(psReturnStats->ui64GpuStatBlocked, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatCumulative = OSDivide64(psReturnStats->ui64GpuStatCumulative, 1000, &ui32Remainder);
+
+	/* Check that the return stats make sense */
+	if (psReturnStats->ui64GpuStatCumulative == 0)
+	{
+		/* We can enter here only if allocating the temporary stats
+		 * buffers failed, or all the RGXFWIF_GPU_UTIL_GET_PERIOD
+		 * returned 0. The latter could happen if the GPU frequency value
+		 * is not well calibrated and the FW is updating the GPU state
+		 * while the Host is reading it.
+		 * When such an event happens frequently, timers or the aggregate
+		 * stats might not be accurate...
+		 */
+#if defined(VIRTUAL_PLATFORM)
+		/* To avoid spamming the console logging system on emulated devices,
+		 * we special-case so that we will only produce a single message per
+		 * driver invocation. This should reduce the time spent logging
+		 * information which is not relevant for very slow timers found in
+		 * VP device configurations
+		 */
+		static IMG_BOOL bFirstTime = IMG_TRUE;
+
+		if (bFirstTime)
+		{
+			bFirstTime = IMG_FALSE;
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "RGXGetGpuUtilStats could not get reliable data."));
+#if defined(VIRTUAL_PLATFORM)
+		}
+#endif	/* defined(VIRTUAL_PLATFORM) */
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psReturnStats->bValid = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psAggregateStats = OSAllocZMemNoStats(sizeof(RGXFWIF_GPU_UTIL_STATS));
+	if (psAggregateStats == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phGpuUtilUser = psAggregateStats;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAggregateStats = hGpuUtilUser;
+	OSFreeMemNoStats(psAggregateStats);
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGX MISR Handler
+*/
+static void RGX_MISRHandler_Main (void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* Give the HWPerf service a chance to transfer some data from the FW
+	 * buffer to the host driver transport layer buffer.
+	 */
+	RGXHWPerfDataStoreCB(psDeviceNode);
+
+	/* Inform other services devices that we have finished an operation */
+	PVRSRVNotifyCommandCompletion(psDeviceNode);
+
+#if defined(SUPPORT_PDVFS) && defined(RGXFW_META_SUPPORT_2ND_THREAD)
+	/* Normally, firmware CCB only exists for the primary FW thread unless PDVFS
+	   is running on the second[ary] FW thread, here we process said CCB */
+	RGXPDVFSCheckCoreClkRateChange(psDeviceNode->pvDevice);
+#endif
+
+	/* Only execute SafetyEventHandler if RGX_FEATURE_SAFETY_EVENT is on */
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, ECC_RAMS) > 0 ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, WATCHDOG_TIMER) ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, RISCV_DUAL_LOCKSTEP) ||
+		PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, FAULT_DECODE_VERSION) >= 2)
+	{
+		/* Handle Safety events if necessary */
+		RGXSafetyEventHandler(psDeviceNode->pvDevice);
+	}
+
+	/* Signal the global event object */
+	PVRSRVSignalDriverWideEO();
+
+	/* Process the Firmware CCB for pending commands */
+	RGXCheckFirmwareCCB(psDeviceNode->pvDevice);
+
+	/* Calibrate the GPU frequency and recorrelate Host and GPU timers (done every few seconds) */
+	RGXTimeCorrRestartPeriodic(psDeviceNode);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Process Workload Estimation Specific commands from the FW */
+		WorkEstCheckFirmwareCCB(psDeviceNode->pvDevice);
+	}
+#endif
+
+	if (psDevInfo->pvAPMISRData == NULL)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+static PVRSRV_ERROR RGXPDumpBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+		PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psFWDataPMR;
+	RGXMIPSFW_BOOT_DATA *psBootData;
+	IMG_DEV_PHYADDR sTmpAddr;
+	IMG_UINT32 ui32BootConfOffset, ui32ParamOffset, i;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+	ui32BootConfOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+	ui32BootConfOffset += RGXMIPSFW_BOOTLDR_CONF_OFFSET;
+
+	/* The physical addresses used by a pdump player will be different
+	 * than the ones we have put in the MIPS bootloader configuration data.
+	 * We have to tell the pdump player to replace the original values with the real ones.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Pass new boot parameters to the FW");
+
+	/* Rogue Registers physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64RegBase);
+
+	eError = PDumpRegLabelToMem64(RGX_PDUMPREG_NAME,
+			0x0,
+			psFWDataPMR,
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of Rogue registers phy address failed (%u)", eError));
+		return eError;
+	}
+
+	/* Page Table physical Address */
+	eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sTmpAddr);
+	if (eError !=  PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXBootldrDataInit: MMU_AcquireBaseAddr failed (%u)",
+		         eError));
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+									 (void **)&psBootData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire pointer to FW data (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	psBootData = IMG_OFFSET_ADDR(psBootData, ui32BootConfOffset);
+
+	for (i = 0; i < psBootData->ui32PTNumPages; i++)
+	{
+		ui32ParamOffset = ui32BootConfOffset +
+			offsetof(RGXMIPSFW_BOOT_DATA, aui64PTPhyAddr[0])
+			+ i * sizeof(psBootData->aui64PTPhyAddr[0]);
+
+		eError = PDumpPTBaseObjectToMem64(psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				psFWDataPMR,
+				0,
+				ui32ParamOffset,
+				PDUMP_FLAGS_CONTINUOUS,
+				MMU_LEVEL_1,
+				sTmpAddr.uiAddr,
+				i << psBootData->ui32PTLog2PageSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of page tables phy address failed (%u)", eError));
+			return eError;
+		}
+	}
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+
+	/* Stack physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64StackPhyAddr);
+
+	eError = PDumpMemLabelToMem64(psFWDataPMR,
+			psFWDataPMR,
+			RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of stack phy address failed (%u)", eError));
+		return eError;
+	}
+
+	return eError;
+}
+#endif /* PDUMP */
+
+static PVRSRV_ERROR RGXSetPowerParams(PVRSRV_RGXDEV_INFO   *psDevInfo,
+                                      PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR sKernelMMUCtxPCAddr;
+	IMG_BOOL bPremappedFw;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVCFG, psDevConfig, PVRSRV_OK);
+
+	/* Save information used on power transitions for later
+	 * (when RGXStart and RGXStop are executed)
+	 */
+#if defined(PDUMP)
+	psDevInfo->sLayerParams.ui32PdumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	/* Volcanic drivers with security support and premapped fw heaps
+	 * always have their fw heap premapped by the TEE */
+	bPremappedFw = IMG_TRUE;
+#else
+	/* If AutoVz firmware is up at this stage, the driver initialised it
+	 * during a previous life-cycle. The firmware's memory is already pre-mapped
+	 * and the MMU page tables reside in the predetermined memory carveout.
+	 * The Kernel MMU Context created in this life-cycle is a dummy structure
+	 * that is not used for mapping.
+	 * To program the Device's BIF with the correct PC address, use the base
+	 * address of the carveout reserved for MMU mappings as Kernel MMU PC Address */
+	bPremappedFw = psDevInfo->psDeviceNode->bAutoVzFwIsUp;
+#endif
+
+	if (bPremappedFw)
+	{
+		IMG_DEV_PHYADDR sDevPAddr;
+		PHYS_HEAP *psFwPageTableHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_RETURN_IF_FALSE((NULL != psFwPageTableHeap),
+								"Firmware Page Table heap not defined.",
+								PVRSRV_ERROR_INVALID_HEAP);
+
+		PhysHeapGetDevPAddr(psFwPageTableHeap, &sDevPAddr);
+		sKernelMMUCtxPCAddr.uiAddr = sDevPAddr.uiAddr;
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx,
+		                             &sKernelMMUCtxPCAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire Kernel MMU Ctx page catalog"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		PMR *psFWCodePMR = (PMR *)(psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR);
+		PMR *psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+		IMG_DEV_PHYADDR sPhyAddr;
+		IMG_BOOL bValid;
+
+#if defined(SUPPORT_ALT_REGBASE)
+		psDevInfo->sLayerParams.sGPURegAddr = psDevConfig->sAltRegsGpuPBase;
+#else
+		/* The physical address of the GPU registers needs to be translated
+		 * in case we are in a LMA scenario
+		 */
+		PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+				1,
+				&sPhyAddr,
+				&(psDevConfig->sRegsCpuPBase));
+
+		psDevInfo->sLayerParams.sGPURegAddr = sPhyAddr;
+#endif
+
+		/* Register bank must be aligned to 512KB (as per the core integration) to
+		 * prevent the FW accessing incorrect registers */
+		if ((psDevInfo->sLayerParams.sGPURegAddr.uiAddr & 0x7FFFFU) != 0U)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Register bank must be aligned to 512KB, but current address (0x%016"IMG_UINT64_FMTSPECX") is not",
+						psDevInfo->sLayerParams.sGPURegAddr.uiAddr));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE),
+				OSGetPageShift(), /* FW will be using the same page size as the OS */
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI code address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sBootRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWDataPMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI data address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sDataRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW exceptions address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sCodeRemapAddr = sPhyAddr;
+
+		psDevInfo->sLayerParams.sTrampolineRemapAddr.uiAddr = psDevInfo->psTrampoline->sPhysAddr.uiAddr;
+
+		psDevInfo->sLayerParams.bDevicePA0IsValid = psDevConfig->bDevicePA0IsValid;
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	/* Send information used on power transitions to the trusted device as
+	 * in this setup the driver cannot start/stop the GPU and perform resets
+	 */
+	if (psDevConfig->pfnTDSetPowerParams)
+	{
+		PVRSRV_TD_POWER_PARAMS sTDPowerParams;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+		    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			sTDPowerParams.sPCAddr = psDevInfo->sLayerParams.sPCAddr;
+		}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			sTDPowerParams.sGPURegAddr    = psDevInfo->sLayerParams.sGPURegAddr;
+			sTDPowerParams.sBootRemapAddr = psDevInfo->sLayerParams.sBootRemapAddr;
+			sTDPowerParams.sCodeRemapAddr = psDevInfo->sLayerParams.sCodeRemapAddr;
+			sTDPowerParams.sDataRemapAddr = psDevInfo->sLayerParams.sDataRemapAddr;
+		}
+#endif
+
+		eError = psDevConfig->pfnTDSetPowerParams(psDevConfig->hSysData,
+												  &sTDPowerParams);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: TDSetPowerParams not implemented!"));
+		eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	return eError;
+}
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*
+	RGXSystemGetFabricCoherency
+*/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode)
+{
+	IMG_CHAR *aszLabels[] = {"none", "acelite", "fullace", "unknown"};
+	PVRSRV_DEVICE_SNOOP_MODE eAppHintCacheSnoopingMode;
+	PVRSRV_DEVICE_SNOOP_MODE eDeviceCacheSnoopingMode;
+	IMG_UINT32 ui32AppHintFabricCoherency;
+	IMG_UINT32 ui32DeviceFabricCoherency;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+#if !defined(NO_HARDWARE)
+	void *pvRegsBaseKM;
+	IMG_BOOL bPowerDown = IMG_TRUE;
+	PVRSRV_ERROR eError;
+#endif
+
+	if (!sRegsCpuPBase.uiAddr || !ui32RegsSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Invalid RGX register base/size parameters"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(NO_HARDWARE)
+	pvRegsBaseKM = OSMapPhysToLin(sRegsCpuPBase, ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvRegsBaseKM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Failed to create RGX register mapping"));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	bPowerDown = ! PVRSRVIsSystemPowered(psDevConfig->psDevNode);
+
+	/* Power-up the device as required to read the registers */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+	}
+
+	/* AXI support within the SoC, bitfield COHERENCY_SUPPORT [1 .. 0]
+		value NO_COHERENCY        0x0 {SoC does not support any form of Coherency}
+		value ACE_LITE_COHERENCY  0x1 {SoC supports ACE-Lite or I/O Coherency}
+		value FULL_ACE_COHERENCY  0x2 {SoC supports full ACE or 2-Way Coherency} */
+	ui32DeviceFabricCoherency = OSReadHWReg32((void __iomem *)pvRegsBaseKM, RGX_CR_SOC_AXI);
+	PVR_LOG(("AXI fabric coherency (RGX_CR_SOC_AXI): 0x%x", ui32DeviceFabricCoherency));
+#if defined(DEBUG)
+	if (ui32DeviceFabricCoherency & ~((IMG_UINT32)RGX_CR_SOC_AXI_MASKFULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid RGX_CR_SOC_AXI value.", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+	ui32DeviceFabricCoherency &= ~((IMG_UINT32)RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK);
+	ui32DeviceFabricCoherency >>= RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+	}
+
+	/* UnMap Regs */
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32RegsSize);
+
+	switch (ui32DeviceFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_FULLACE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+	default:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_NONE;
+		break;
+	}
+#else /* !defined(NO_HARDWARE) */
+    *peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+    eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+    ui32DeviceFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY;
+#endif /* !defined(NO_HARDWARE) */
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, FabricCoherencyOverride,
+						 &ui32AppHintDefault, &ui32AppHintFabricCoherency);
+	OSFreeAppHintState(pvAppHintState);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	/* Temporarily disable coherency */
+	ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY;
+#endif
+
+	/* Suppress invalid AppHint value */
+	switch (ui32AppHintFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"Invalid FabricCoherencyOverride AppHint %d, ignoring",
+				ui32AppHintFabricCoherency));
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+		break;
+	}
+
+	if (ui32AppHintFabricCoherency < ui32DeviceFabricCoherency)
+	{
+		PVR_LOG(("Downgrading device fabric coherency from %s to %s",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+		eDeviceCacheSnoopingMode = eAppHintCacheSnoopingMode;
+	}
+	else if (ui32AppHintFabricCoherency > ui32DeviceFabricCoherency)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Cannot upgrade device fabric coherency from %s to %s, not supported by device!",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+
+		/* Override requested-for app-hint with actual app-hint value being used */
+		ui32AppHintFabricCoherency = ui32DeviceFabricCoherency;
+	}
+
+	*peCacheSnoopingMode = eDeviceCacheSnoopingMode;
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXSystemHasFBCDCVersion31
+*/
+static IMG_BOOL RGXSystemHasFBCDCVersion31(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(HW_ERN_66622_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 66622))
+#endif
+	{
+		{
+			if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+			{
+				return IMG_TRUE;
+			}
+		}
+	}
+#if defined(HW_ERN_66622_BIT_MASK)
+	else
+	{
+
+#if !defined(NO_HARDWARE)
+		if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: System uses FBCDC3.1 but GPU doesn't support it!",
+			         __func__));
+		}
+#endif
+	}
+#endif /* defined(HW_ERN_66622_BIT_MASK) */
+
+	return IMG_FALSE;
+}
+
+/*
+	RGXGetTFBCLossyGroup
+*/
+static IMG_UINT32 RGXGetTFBCLossyGroup(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	return psDevInfo->ui32TFBCLossyGroup;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return 0;
+#endif
+}
+
+/*
+	RGXDevMMUAttributes
+*/
+static MMU_DEVICEATTRIBS *RGXDevMMUAttributes(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              IMG_BOOL bKernelFWMemoryCtx)
+{
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = NULL;
+
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			psMMUDevAttrs = bKernelFWMemoryCtx ?
+			                psDeviceNode->psFirmwareMMUDevAttrs :
+			                psDeviceNode->psMMUDevAttrs;
+		}
+		else
+#endif
+		{
+			PVR_UNREFERENCED_PARAMETER(bKernelFWMemoryCtx);
+			psMMUDevAttrs = psDeviceNode->psMMUDevAttrs;
+		}
+	}
+
+	return psMMUDevAttrs;
+}
+
+
+/*
+	RGXDevSnoopMode
+*/
+static PVRSRV_DEVICE_SNOOP_MODE RGXDevSnoopMode(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+/*
+ * RGXInitDevPart2
+ */
+PVRSRV_ERROR RGXInitDevPart2(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                             RGX_INIT_APPHINTS	*psApphints)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEV_POWER_STATE	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32			ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32			ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	IMG_UINT32 ui32PDumpFlags = PDUMP_FLAGS_DEINIT;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 2");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		RGXPDumpBootldrData(psDeviceNode, psDevInfo);
+	}
+#endif
+#if defined(TIMING) || defined(DEBUG)
+	OSUserModeAccessToPerfCountersEn();
+#endif
+
+	/* Initialise Device Flags */
+	psDevInfo->ui32DeviceFlags = 0;
+	RGXSetDeviceFlags(psDevInfo, psApphints->ui32DeviceFlags, IMG_TRUE);
+
+	/* Allocate DVFS Table (needs to be allocated before GPU trace events
+	 *  component is initialised because there is a dependency between them) */
+	psDevInfo->psGpuDVFSTable = OSAllocZMem(sizeof(*(psDevInfo->psGpuDVFSTable)));
+	PVR_LOG_GOTO_IF_NOMEM(psDevInfo->psGpuDVFSTable, eError, ErrorExit);
+
+	if (psDevInfo->ui32HWPerfHostFilter == 0)
+	{
+		RGXHWPerfHostSetEventFilter(psDevInfo, psApphints->ui32HWPerfHostFilter);
+	}
+
+	/* If HWPerf enabled allocate all resources for the host side buffer. */
+	if (psDevInfo->ui32HWPerfHostFilter != 0)
+	{
+		if (RGXHWPerfHostInitOnDemandResources(psDevInfo) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "HWPerfHost buffer on demand"
+			        " initialisation failed."));
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise work estimation lock */
+		eError = OSLockCreate(&psDevInfo->hWorkEstLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(WorkEstLock)", ErrorExit);
+	}
+#endif
+
+	/* Initialise lists of ZSBuffers */
+	eError = OSLockCreate(&psDevInfo->hLockZSBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockZSBuffer)", ErrorExit);
+	dllist_init(&psDevInfo->sZSBufferHead);
+	psDevInfo->ui32ZSBufferCurrID = 1;
+
+	/* Initialise lists of growable Freelists */
+	eError = OSLockCreate(&psDevInfo->hLockFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockFreeList)", ErrorExit);
+	dllist_init(&psDevInfo->sFreeListHead);
+	psDevInfo->ui32FreelistCurrID = 1;
+
+	eError = OSLockCreate(&psDevInfo->hDebugFaultInfoLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(DebugFaultInfoLock)", ErrorExit);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		eError = OSLockCreate(&psDevInfo->hMMUCtxUnregLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(MMUCtxUnregLock)", ErrorExit);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = OSLockCreate(&psDevInfo->hNMILock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(NMILock)", ErrorExit);
+	}
+#endif
+
+	/* Setup GPU utilisation stats update callback */
+	eError = OSLockCreate(&psDevInfo->hGPUUtilLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(GPUUtilLock)", ErrorExit);
+#if !defined(NO_HARDWARE)
+	psDevInfo->pfnGetGpuUtilStats = RGXGetGpuUtilStats;
+#endif
+
+	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	psDevInfo->eActivePMConf = psApphints->eRGXActivePMConf;
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	/* Validate the SPU mask and initialize to number of SPUs to power up */
+	if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s:Invalid SPU mask (All=0x%X, Non Fused=0x%X). At-least one SPU must to be powered up.",
+		         __func__,
+		         ui32AllPowUnitsMask,
+		         psApphints->ui32AvailablePowUnitsMask));
+		PVR_LOG_GOTO_WITH_ERROR("ui32AvailablePowUnitsMask", eError, PVRSRV_ERROR_INVALID_SPU_MASK, ErrorExit);
+	}
+
+	psDevInfo->ui32AvailablePowUnitsMask = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+	psDevInfo->ui32AvailableRACMask = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* set-up the Active Power Mgmt callback */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableAPM = psRGXData->psRGXTimingInfo->bEnableActivePM;
+		IMG_BOOL bEnableAPM = ((psApphints->eRGXActivePMConf == RGX_ACTIVEPM_DEFAULT) && bSysEnableAPM) ||
+							   (psApphints->eRGXActivePMConf == RGX_ACTIVEPM_FORCE_ON);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1) && defined(SUPPORT_AUTOVZ)
+		/* The AutoVz driver enables a virtualisation watchdog not compatible with APM */
+		if (bEnableAPM && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Active Power Management disabled in AutoVz mode", __func__));
+			bEnableAPM = IMG_FALSE;
+		}
+
+		PVR_ASSERT(bEnableAPM == IMG_FALSE);
+#endif
+
+		if (bEnableAPM)
+		{
+			eError = OSInstallMISR(&psDevInfo->pvAPMISRData,
+					RGX_MISRHandler_CheckFWActivePowerState,
+					psDeviceNode,
+					"RGX_CheckFWActivePower");
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(APMISR)", ErrorExit);
+
+			/* Prevent the device being woken up before there is something to do. */
+			eDefaultPowerState = PVRSRV_DEV_POWER_STATE_OFF;
+		}
+	}
+#endif
+
+	psDevInfo->eDebugDumpFWTLogType = psApphints->eDebugDumpFWTLogType;
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableAPM,
+	                                    RGXQueryAPMState,
+	                                    RGXSetAPMState,
+	                                    psDeviceNode,
+	                                    NULL);
+
+	RGXTimeCorrInitAppHintCallbacks(psDeviceNode);
+
+	/* Register the device with the power manager */
+	eError = PVRSRVRegisterPowerDevice(psDeviceNode,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPrePowerState : &RGXVzPrePowerState,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPostPowerState : &RGXVzPostPowerState,
+	                                   psDevConfig->pfnPrePowerState, psDevConfig->pfnPostPowerState,
+	                                   &RGXPreClockSpeedChange, &RGXPostClockSpeedChange,
+	                                   &RGXForcedIdleRequest, &RGXCancelForcedIdleRequest, &RGXCancelForcedIdleRequestAsync,
+	                                   &RGXPowUnitsChange,
+	                                   (IMG_HANDLE)psDeviceNode,
+	                                   PVRSRV_DEV_POWER_STATE_OFF,
+	                                   eDefaultPowerState);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterPowerDevice", ErrorExit);
+
+	eError = RGXSetPowerParams(psDevInfo, psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetPowerParams", ErrorExit);
+
+
+#if defined(PDUMP)
+#if defined(NO_HARDWARE)
+	/* Capture the HWPERF buffer as the last thing we do */
+	PDUMPIF(psDeviceNode, "ENABLE_HWPERF", ui32PDumpFlags);
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+	                                 psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                                 0,
+	                                 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+	                                 "out.hwperf",
+	                                 0,
+	                                 ui32PDumpFlags);
+	PDUMPFI(psDeviceNode, "ENABLE_HWPERF", ui32PDumpFlags);
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT, "Wait for the FW to signal idle");
+
+	/* Kick the FW once, in case it still needs to detect and set the idle state */
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+			   RGX_CR_MTS_SCHEDULE,
+			   RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK,
+			   PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                                offsetof(RGXFWIF_SYSDATA, ePowState),
+	                                RGXFWIF_POW_IDLE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", ErrorExit);
+#endif
+
+	/* Run RGXStop with the correct PDump flags to feed the last-frame deinit buffer */
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT,
+	                      "RGX deinitialisation commands");
+
+	psDevInfo->sLayerParams.ui32PdumpFlags |= PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW;
+
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXStop(&psDevInfo->sLayerParams);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXStop", ErrorExit);
+	}
+
+	psDevInfo->sLayerParams.ui32PdumpFlags &= ~(PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW);
+#endif
+
+#if !defined(NO_HARDWARE)
+	eError = RGXInstallProcessQueuesMISR(&psDevInfo->hProcessQueuesMISR, psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInstallProcessQueuesMISR", ErrorExit);
+
+	/* Register RGX to receive notifies when other devices complete some work */
+	PVRSRVRegisterCmdCompleteNotify(&psDeviceNode->hCmdCompNotify, &RGXScheduleProcessQueuesKM, psDeviceNode);
+
+	/* Register the interrupt handlers */
+	eError = OSInstallMISR(&psDevInfo->pvMISRData,
+			RGX_MISRHandler_Main,
+			psDeviceNode,
+			"RGX_Main");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(MISR)", ErrorExit);
+
+	/* Register appropriate mechanism for clearing hw interrupts */
+	eError = RGXSetAckIrq(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetAckIrq", ErrorExit);
+
+	eError = SysInstallDeviceLISR(psDevConfig->hSysData,
+								  psDevConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  RGX_LISRHandler,
+								  psDeviceNode,
+								  &psDevInfo->pvLISRData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysInstallDeviceLISR", ErrorExit);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(PDUMP)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_CACHE_HIERARCHY)))
+	{
+		if (PVRSRVSystemSnoopingOfCPUCache(psDevConfig))
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				              "System has CPU cache snooping");
+		}
+		else
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "System has NO cache snooping");
+		}
+	}
+#endif
+
+	eError = PVRSRVTQLoadShaders(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVTQLoadShaders", ErrorExit);
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	eError = OSAllocateSecBuf(psDeviceNode, RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE, "SharedSecMem", &psDevInfo->psGenHeapSecMem);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSAllocateSecBuf", ErrorExit);
+#endif
+
+	psDevInfo->bDevInit2Done = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	DevPart2DeInitRGX(psDeviceNode);
+
+	return eError;
+}
+
+#define VZ_RGX_FW_FILENAME_SUFFIX ".vz"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#define RGX_64K_FW_FILENAME_SUFFIX ".64k"
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX) + sizeof(RGX_64K_FW_FILENAME_SUFFIX)))
+#else
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX)))
+#endif
+
+static void _GetFWFileName(PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_CHAR *pszFWFilenameStr,
+		IMG_CHAR *pszFWpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const IMG_CHAR * const pszFWFilenameSuffix =
+			PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) ? "" : VZ_RGX_FW_FILENAME_SUFFIX;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	const IMG_CHAR * const pszFWFilenameSuffix2 =
+			((OSGetPageSize() == RGX_MMU_PAGE_SIZE_64KB) &&
+			 RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+			? RGX_64K_FW_FILENAME_SUFFIX : "";
+#else
+	const IMG_CHAR * const pszFWFilenameSuffix2 = "";
+#endif
+
+	OSSNPrintf(pszFWFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STR_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+
+	OSSNPrintf(pszFWpFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STRP_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+}
+
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW)
+{
+	IMG_CHAR aszFWFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR aszFWpFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR *pszLoadedFwStr;
+	PVRSRV_ERROR eErr;
+
+	/* Prepare the image filenames to use in the following code */
+	_GetFWFileName(psDeviceNode, aszFWFilenameStr, aszFWpFilenameStr);
+
+	/* Get pointer to Firmware image */
+	pszLoadedFwStr = aszFWFilenameStr;
+	eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+	if (eErr == PVRSRV_ERROR_NOT_FOUND)
+	{
+		pszLoadedFwStr = aszFWpFilenameStr;
+		eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+		if (eErr == PVRSRV_ERROR_NOT_FOUND)
+		{
+			pszLoadedFwStr = RGX_FW_FILENAME;
+			eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+			if (eErr == PVRSRV_ERROR_NOT_FOUND)
+			{
+				PVR_DPF((PVR_DBG_FATAL, "All RGX Firmware image loads failed for '%s' (%s)",
+						aszFWFilenameStr, PVRSRVGetErrorString(eErr)));
+			}
+		}
+	}
+
+	if (eErr == PVRSRV_OK)
+	{
+		PVR_LOG(("RGX Firmware image '%s' loaded", pszLoadedFwStr));
+	}
+
+	return eErr;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	PVRSRV_ERROR eError;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS_DIRECT;
+
+	eError = RGXSendCommandWithPowLock(psDeviceNode->pvDevice,
+	                                   &sKccbCmd,
+	                                   PDUMP_FLAGS_CONTINUOUS);
+
+	return eError;
+}
+#endif
+
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* set up fw memory contexts */
+	PVRSRV_RGXDEV_INFO   *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	IMG_BOOL  bNativeFwUMAHeap = PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) &&
+	                             (PhysHeapGetType(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM]) == PHYS_HEAP_TYPE_UMA);
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)) && (!psDeviceNode->bAutoVzFwIsUp) && (!bNativeFwUMAHeap))
+	{
+		PHYS_HEAP *psFwPageTableHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_GOTO_IF_INVALID_PARAM((psFwPageTableHeap != NULL),
+		                              eError, failed_to_create_ctx);
+
+		/* Temporarily swap the MMU and default GPU physheap to allow the page
+		 * tables of all memory mapped by the FwKernel context to be placed
+		 * in a dedicated memory carveout. This should allow the firmware mappings to
+		 * persist after a Host kernel crash or driver reset. */
+		psDeviceNode->psMMUPhysHeap = psFwPageTableHeap;
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+	/* Set the device fabric coherency before FW context creation */
+	eError = RGXSystemGetFabricCoherency(psDevConfig,
+										 psDevConfig->sRegsCpuPBase,
+										 psDevConfig->ui32RegsSize,
+										 &psDeviceNode->eDevFabricType,
+										 &psDevConfig->eCacheSnoopingMode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed RGXSystemGetFabricCoherency (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+#endif
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	RGXFwSharedMemCheckSnoopMode(psDevConfig);
+
+	/* Create the memory context for the firmware. */
+	eError = DevmemCreateContext(psDeviceNode, DEVMEM_HEAPCFG_FORFW,
+	                             &psDevInfo->psKernelDevmemCtx);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemCreateContext (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_MAIN_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareMainHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_CONFIG_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareConfigHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+#if (defined(RGX_PREMAP_FW_HEAPS)) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_CHAR szHeapName[RA_MAX_NAME_LENGTH];
+
+			OSSNPrintf(szHeapName, sizeof(szHeapName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+			eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, szHeapName,
+										  &psDevInfo->psPremappedFwRawHeap[ui32DriverID]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "DevmemFindHeapByName", failed_to_find_heap);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && !bNativeFwUMAHeap)
+	{
+		IMG_DEV_PHYADDR sPhysHeapBase;
+		IMG_UINT32 ui32DriverID;
+		void *pvAppHintState = NULL;
+		IMG_UINT64 ui64DefaultHeapStride;
+		IMG_UINT64 ui64GuestHeapDevBaseStride;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64DefaultHeapStride = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							GuestFWHeapStride,
+							&ui64DefaultHeapStride,
+							&ui64GuestHeapDevBaseStride);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		eError = PhysHeapGetDevPAddr(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM], &sPhysHeapBase);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", failed_to_find_heap);
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_DEV_PHYADDR sRawFwHeapBase = {sPhysHeapBase.uiAddr + (ui32DriverID * ui64GuestHeapDevBaseStride)};
+
+			eError = RGXFwRawHeapAllocMap(psDeviceNode,
+										  ui32DriverID,
+										  sRawFwHeapBase,
+										  RGX_FIRMWARE_RAW_HEAP_SIZE);
+			if (eError != PVRSRV_OK)
+			{
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+				}
+				PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", failed_to_find_heap);
+			}
+		}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+		/* restore default Px setup */
+		psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* On setups with dynamically mapped Guest heaps, the Guest makes
+	 * a PVZ call to the Host to request the mapping during init. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PvzClientMapDevPhysHeap(psDevConfig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PvzClientMapDevPhysHeap", failed_to_find_heap);
+	}
+#endif /* !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+
+failed_to_find_heap:
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+	DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+	psDevInfo->psKernelDevmemCtx = NULL;
+failed_to_create_ctx:
+	return eError;
+}
+
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR       eError;
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(RGX_PREMAP_FW_HEAPS)
+		psDeviceNode->psMMUPhysHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		if (!psDeviceNode->bAutoVzFwIsUp)
+#endif
+		{
+			IMG_UINT32 ui32DriverID;
+
+			FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+			{
+				RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+			}
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		(void) PvzClientUnmapDevPhysHeap(psDeviceNode->psDevConfig);
+
+		if (psDevInfo->psFirmwareMainHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_FALSE);
+		}
+		if (psDevInfo->psFirmwareConfigHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_FALSE);
+		}
+	}
+#endif
+
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+
+	if (psDevInfo->psKernelDevmemCtx)
+	{
+		eError = DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+		PVR_LOG_IF_ERROR(eError, "DevmemDestroyContext");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+}
+
+static PVRSRV_ERROR RGXAlignmentCheck(PVRSRV_DEVICE_NODE *psDevNode,
+                                      IMG_UINT32 ui32AlignChecksSizeUM,
+                                      IMG_UINT32 aui32AlignChecksUM[])
+{
+	static const IMG_UINT32 aui32AlignChecksKM[] = {RGXFW_ALIGN_CHECKS_INIT_KM};
+	IMG_UINT32 ui32UMChecksOffset = ARRAY_SIZE(aui32AlignChecksKM) + 1;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32 i, *paui32FWAlignChecks;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Skip the alignment check if the driver is guest
+	   since there is no firmware to check against */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, eError);
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW Alignment Check Mem Descriptor is NULL",
+		         __func__));
+		return PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
+	                                  (void **) &paui32FWAlignChecks);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire kernel address for alignment checks (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	paui32FWAlignChecks += ui32UMChecksOffset;
+	/* Invalidate the size value, check the next region size (UM) and invalidate */
+	RGXFwSharedMemCacheOpPtr(paui32FWAlignChecks, INVALIDATE);
+	if (*paui32FWAlignChecks++ != ui32AlignChecksSizeUM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Mismatching sizes of RGXFW_ALIGN_CHECKS_INIT"
+		         " array between UM(%d) and FW(%d)",
+		         __func__,
+		         ui32AlignChecksSizeUM,
+		         *paui32FWAlignChecks));
+		eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		goto return_;
+	}
+
+	RGXFwSharedMemCacheOpExec(paui32FWAlignChecks,
+	                          ui32AlignChecksSizeUM * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	for (i = 0; i < ui32AlignChecksSizeUM; i++)
+	{
+		if (aui32AlignChecksUM[i] != paui32FWAlignChecks[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: size/offset mismatch in RGXFW_ALIGN_CHECKS_INIT[%d]"
+					" between UM(%d) and FW(%d)",
+					__func__, i, aui32AlignChecksUM[i], paui32FWAlignChecks[i]));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		}
+	}
+
+	if (eError == PVRSRV_ERROR_INVALID_ALIGNMENT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Check for FW/KM structure"
+				" alignment failed.", __func__));
+	}
+
+return_:
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXAllocateFWMemoryRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_DEVMEM_SIZE_T ui32Size,
+                                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                       const IMG_PCHAR pszText,
+                                       DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Align = OSGetPageShift();
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(SUPPORT_MIPS_CONTIGUOUS_FW_MEMORY)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		uiLog2Align = RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiLog2Align);
+#endif
+
+	uiMemAllocFlags = (uiMemAllocFlags |
+					   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+					   PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &
+	                   RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	uiMemAllocFlags &= PVRSRV_MEMALLOCFLAGS_TDFWMASK;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate FW %s memory", pszText);
+
+	eError = DevmemFwAllocate(psDeviceNode->pvDevice,
+	                          ui32Size,
+	                          uiMemAllocFlags,
+	                          pszText,
+	                          ppsMemDescPtr);
+
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver
+
+ @Description
+
+ Validate the FW build options against KM driver build options (KM build options only)
+
+ Following check is redundant, because next check checks the same bits.
+ Redundancy occurs because if client-server are build-compatible and client-firmware are
+ build-compatible then server-firmware are build-compatible as well.
+
+ This check is left for clarity in error messages if any incompatibility occurs.
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+
+	IMG_UINT32			ui32BuildOptions, ui32BuildOptionsFWKMPart, ui32BuildOptionsMismatch;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+	IMG_UINT8               ui8FwOsCount;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32BuildOptions = (RGX_BUILD_OPTIONS_KM & RGX_BUILD_OPTIONS_MASK_FW);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32BuildOptionsFWKMPart = psFwOsInit->sRGXCompChecks.ui32BuildOptions & RGX_BUILD_OPTIONS_MASK_FW;
+
+		ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+		if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32BuildOptionsFWKMPart = psFWInfoHeader->ui32Flags & RGX_BUILD_OPTIONS_MASK_FW;
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo) && BITMASK_HAS(psFWInfoHeader->ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psFWInfoHeader->ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+				return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+			}
+		}
+	}
+
+	/* Check if the FW is missing support for any features required by the driver */
+	if (~ui32BuildOptionsFWKMPart & ui32BuildOptions)
+	{
+		ui32BuildOptionsMismatch = ui32BuildOptions ^ ui32BuildOptionsFWKMPart;
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask non-critical options out as we do support combining them in UM & KM */
+		ui32BuildOptionsMismatch &= FW_OPTIONS_STRICT;
+#endif
+		if ((ui32BuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and KM driver build options; "
+					 "extra options present in the KM driver: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptions & ui32BuildOptionsMismatch));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+
+		if ((ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware-side and KM driver build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch ));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "RGXDevInitCompatCheck: Firmware and KM driver build options differ."));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: Firmware and KM driver build options match. [ OK ]"));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK version against driver DDK version
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32			ui32KMDDKVersion;
+	IMG_UINT32			ui32FWDDKVersion;
+	PVRSRV_ERROR		eError;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32KMDDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKVersion = psFwOsInit->sRGXCompChecks.ui32DDKVersion;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+		ui32FWDDKVersion = PVRVERSION_PACK(psFWInfoHeader->ui16PVRVersionMajor, psFWInfoHeader->ui16PVRVersionMinor);
+	}
+
+	if (ui32FWDDKVersion != ui32KMDDKVersion)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Incompatible driver DDK version (%u.%u) / Firmware DDK version (%u.%u).",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_UNPACK_MAJ(ui32FWDDKVersion),
+				 PVRVERSION_UNPACK_MIN(ui32FWDDKVersion)));
+		eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK version (%u.%u) and Firmware DDK version (%u.%u) match. [ OK ]",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK build against driver DDK build
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32KMDDKBuild;
+	IMG_UINT32			ui32FWDDKBuild;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	ui32KMDDKBuild = PVRVERSION_BUILD;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKBuild = psFwOsInit->sRGXCompChecks.ui32DDKBuild;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32FWDDKBuild = psFWInfoHeader->ui32PVRVersionBuild;
+	}
+
+	if (ui32FWDDKBuild != ui32KMDDKBuild)
+	{
+		PVR_LOG(("(WARN) RGXDevInitCompatCheck: Different driver DDK build version (%d) / Firmware DDK build version (%d).",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		eError = PVRSRV_ERROR_DDK_BUILD_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK build version (%d) and Firmware DDK build version (%d) match. [ OK ]",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+	}
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_BVNC_FWAgainstDriver
+
+ @Description
+
+ Validate FW BVNC against driver BVNC
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_BVNC_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+
+	RGX_FW_INFO_HEADER	*psFWInfoHeader;
+	IMG_UINT64			ui64KMBVNC;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+	ui64KMBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+							   psDevInfo->sDevFeatureCfg.ui32V,
+							   psDevInfo->sDevFeatureCfg.ui32N,
+							   psDevInfo->sDevFeatureCfg.ui32C);
+
+	if (ui64KMBVNC != psFWInfoHeader->ui64BVNC)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in KM driver BVNC (%u.%u.%u.%u) and Firmware BVNC (%u.%u.%u.%u)",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_ERROR_BVNC_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: KM driver BVNC (%d.%d.%d.%d) and FW BVNC (%d.%d.%d.%d) match. [ OK ]",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevInitCompatCheck
+
+ @Description
+
+ Check compatibility of host driver and firmware (DDK and build options)
+ for RGX devices at services/device initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32			ui32FwTimeout = MAX_HW_TIME_US;
+
+		LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Flush covers this instance and the reads in the functions below */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                               INVALIDATE);
+		if (!*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU Firmware not responding: failed to supply compatibility info (%u)",
+					__func__, eError));
+
+			PVR_DPF((PVR_DBG_ERROR, "%s: Potential causes: firmware not initialised or the current Guest driver's "
+									"OsConfig initialisation data was not accepted by the firmware", __func__));
+			goto chk_exit;
+		}
+	}
+#endif /* defined(NO_HARDWARE) */
+
+	eError = RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDevInitCompatCheck_BVNC_FWAgainstDriver(psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			goto chk_exit;
+		}
+	}
+
+	eError = PVRSRV_OK;
+chk_exit:
+
+	return eError;
+}
+
+static void _RGXSoftResetToggle(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT64  ui64ResetValue1,
+                                IMG_UINT64  ui64ResetValue2)
+{
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, ui64ResetValue1);
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) == 1)
+	{
+		OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET_SPU, ui64ResetValue2);
+	}
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) == 1)
+	{
+		(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET_SPU);
+	}
+}
+
+/**************************************************************************/ /*!
+@Function       RGXSoftReset
+@Description    Resets some modules of the RGX device
+@Input          psDeviceNode		Device node
+@Input          ui64ResetValue1 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET
+                                register).
+@Input          ui64ResetValue2 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET_SPU
+                                register).
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT64  ui64ResetValue1,
+                                 IMG_UINT64  ui64ResetValue2)
+{
+	PVRSRV_RGXDEV_INFO        *psDevInfo;
+	IMG_BOOL	bSoftReset = IMG_FALSE;
+	IMG_UINT64	ui64SoftResetMask = 0;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+#if defined(RGX_CR_SOFT_RESET2_MASKFULL)
+	if ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE)) &&
+			((ui64ResetValue2 & RGX_CR_SOFT_RESET2_MASKFULL) != ui64ResetValue2))
+	{
+		bSoftReset = IMG_TRUE;
+	}
+#elif defined(RGX_CR_SOFT_RESET_SPU_MASKFULL)
+	if ((ui64ResetValue2 & RGX_CR_SOFT_RESET_SPU_MASKFULL) != ui64ResetValue2)
+	{
+		bSoftReset = IMG_TRUE;
+	}
+#endif
+
+	if (((ui64ResetValue1 & ui64SoftResetMask) != ui64ResetValue1) || bSoftReset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Set in soft-reset */
+	_RGXSoftResetToggle(psDevInfo, ui64ResetValue1, ui64ResetValue2);
+
+	/* Take the modules out of reset... */
+	_RGXSoftResetToggle(psDevInfo, 0, 0);
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static const RGX_MIPS_ADDRESS_TRAMPOLINE sNullTrampoline;
+
+static void RGXFreeTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+			psDevInfo->psTrampoline->hPdumpPages,
+#endif
+			&psDevInfo->psTrampoline->sPages);
+
+	if (psDevInfo->psTrampoline != &sNullTrampoline)
+	{
+		OSFreeMem(psDevInfo->psTrampoline);
+	}
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+}
+
+#define RANGES_OVERLAP(x,y,size) (x < (y+size) && y < (x+size))
+#define TRAMPOLINE_ALLOC_MAX_RETRIES (3)
+
+static PVRSRV_ERROR RGXAllocTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT32 i, j;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_MIPS_ADDRESS_TRAMPOLINE *pasTrampoline[TRAMPOLINE_ALLOC_MAX_RETRIES];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate pages for trampoline");
+
+	/* Retry the allocation of the trampoline block (16KB), retaining any
+	 * previous allocations overlapping  with the target range until we get an
+	 * allocation that doesn't overlap with the target range.
+	 * Any allocation like this will require a maximum of 3 tries as we are
+	 * allocating a physical contiguous block of memory, not individual pages.
+	 * Free the unused allocations at the end only after the desired range
+	 * is obtained to prevent the alloc function from returning the same bad
+	 * range repeatedly.
+	 */
+	for (i = 0; i < TRAMPOLINE_ALLOC_MAX_RETRIES; i++)
+	{
+		pasTrampoline[i] = OSAllocMem(sizeof(RGX_MIPS_ADDRESS_TRAMPOLINE));
+		eError = DevPhysMemAlloc(psDeviceNode,
+				RGXMIPSFW_TRAMPOLINE_SIZE,
+				RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE,
+				0,         // (init) u8Value
+				IMG_FALSE, // bInitPage,
+#if defined(PDUMP)
+				psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				"TrampolineRegion",
+				&pasTrampoline[i]->hPdumpPages,
+#endif
+				PVR_SYS_ALLOC_PID,
+				&pasTrampoline[i]->sPages,
+				&pasTrampoline[i]->sPhysAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s failed (%u)",
+			         __func__,
+			         eError));
+			goto fail;
+		}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		/* Set the persistent uiOSid value so that we free from the correct
+		 * base arena when unloading the driver and freeing the trampoline.
+		 */
+		pasTrampoline[i]->sPages.uiOSid = 0;	/* Firmware global arena */
+#endif
+
+		if (!RANGES_OVERLAP(pasTrampoline[i]->sPhysAddr.uiAddr,
+				RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+				RGXMIPSFW_TRAMPOLINE_SIZE))
+		{
+			break;
+		}
+	}
+	if (TRAMPOLINE_ALLOC_MAX_RETRIES == i)
+	{
+		/* Failed to find a physical allocation after 3 attempts */
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s failed to allocate non-overlapping pages (%u)",
+				__func__, eError));
+		/* Fall through, clean up and return error. */
+	}
+	else
+	{
+		/* Remember the last physical block allocated, it will not be freed */
+		psDevInfo->psTrampoline = pasTrampoline[i];
+	}
+
+fail:
+	/* free all unused allocations */
+	for (j = 0; j < i; j++)
+	{
+		DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+				pasTrampoline[j]->hPdumpPages,
+#endif
+				&pasTrampoline[j]->sPages);
+		OSFreeMem(pasTrampoline[j]);
+	}
+
+	return eError;
+}
+
+#undef RANGES_OVERLAP
+#endif
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    uiFWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR        eError;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEVMEM_SIZE_T	uiDummyLen;
+	DEVMEM_MEMDESC		*psDummyMemDesc = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		eError = RGXAllocTrampoline(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Failed to allocate trampoline region (%u)",
+					eError));
+			goto failTrampolineMemDescAlloc;
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW code section
+	 */
+	uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+	                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE);
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWCodeLen,
+	                                   uiMemAllocFlags,
+	                                   "FwCodeRegion",
+	                                   &psDevInfo->psRGXFWCodeMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc,
+	                                  &psDevInfo->sFWCodeDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAqDevVirt;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) || (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))))
+#endif
+	{
+		/*
+		 * The FW code must be the first allocation in the firmware heap, otherwise
+		 * the bootloader will not work (the FW will not be able to find the bootloader).
+		 */
+		PVR_ASSERT(psDevInfo->sFWCodeDevVAddrBase.uiAddr == RGX_FIRMWARE_RAW_HEAP_BASE);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/*
+		 * Allocate Dummy Pages so that Data segment allocation gets the same
+		 * device virtual address as specified in MIPS firmware linker script
+		 */
+		uiDummyLen = RGXGetFWImageSectionMaxSize(NULL, MIPS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_EXCEPTIONS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_BOOT_CODE) -
+				uiFWCodeLen; /* code actual size */
+
+		if (uiDummyLen > 0)
+		{
+			eError = DevmemFwAllocateExportable(psDeviceNode,
+					uiDummyLen,
+					OSGetPageSize(),
+					uiMemAllocFlags,
+					"FwExDummyPages",
+					&psDummyMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "Failed to allocate fw dummy mem (%u)",
+				         eError));
+				goto failDummyMemDescAlloc;
+			}
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW data section
+	 */
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWDataLen,
+	                                   RGX_FWCODEDATA_ALLOCFLAGS |
+	                                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA),
+	                                   "FwDataRegion",
+	                                   &psDevInfo->psRGXFWDataMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+	                                  &psDevInfo->sFWDataDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAqDevVirt;
+	}
+
+	if (uiFWCorememCodeLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem code section
+		 */
+		uiMemAllocFlags = (RGX_FWCODEDATA_ALLOCFLAGS |
+		                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE)) &
+		                   ~PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE;
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+		                                   uiFWCorememCodeLen,
+		                                   uiMemAllocFlags,
+		                                   "FwCorememCodeRegion",
+		                                   &psDevInfo->psRGXFWCorememCodeMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem code mem, size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememCodeLen, uiMemAllocFlags, eError));
+			goto failFWCorememCodeMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc,
+		                                  &psDevInfo->sFWCorememCodeDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem code (%u)",
+			         eError));
+			goto failFWCorememCodeMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememCodeFWAddr,
+		                      psDevInfo->psRGXFWCorememCodeMemDesc,
+		                      0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", failFWCorememCodeMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememCodeDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememCodeFWAddr.ui32Addr = 0;
+	}
+
+	if (uiFWCorememDataLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem data section
+		 */
+		uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+		                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA);
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+				uiFWCorememDataLen,
+				uiMemAllocFlags,
+				"FwCorememDataRegion",
+				&psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem data mem, "
+			         "size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememDataLen,
+			         uiMemAllocFlags,
+			         eError));
+			goto failFWCorememDataMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				&psDevInfo->sFWCorememDataStoreDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem data (%u)",
+			         eError));
+			goto failFWCorememDataMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememDataStoreFWAddr,
+				psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", failFWCorememDataMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememDataStoreDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememDataStoreFWAddr.ui32Addr = 0;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Free Dummy Pages */
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failFWCorememDataMemDescFwAddr:
+failFWCorememDataMemDescAqDevVirt:
+	if (uiFWCorememDataLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+failFWCorememDataMemDescAlloc:
+failFWCorememCodeMemDescFwAddr:
+failFWCorememCodeMemDescAqDevVirt:
+	if (uiFWCorememCodeLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+failFWCorememCodeMemDescAlloc:
+failFWDataMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+	psDevInfo->psRGXFWDataMemDesc = NULL;
+failFWDataMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+failDummyMemDescAlloc:
+#endif
+failFWCodeMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+	psDevInfo->psRGXFWCodeMemDesc = NULL;
+failFWCodeMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		RGXFreeTrampoline(psDeviceNode);
+	}
+failTrampolineMemDescAlloc:
+#endif
+	return eError;
+}
+
+/*
+	AppHint parameter interface
+ */
+static
+PVRSRV_ERROR RGXFWTraceQueryFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	*pui32Value &= RGXFWIF_LOG_TYPE_GROUP_MASK;
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceQueryLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	if (PVRSRV_OK == eResult)
+	{
+		if (*pui32Value & RGXFWIF_LOG_TYPE_TRACE)
+		{
+			*pui32Value = 0; /* Trace */
+		}
+		else
+		{
+			*pui32Value = 1; /* TBI */
+		}
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const void *psPrivate,
+                                 IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType;
+
+	eResult = RGXFWTraceQueryLogType(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK == eResult)
+	{
+		if (0 == ui32RGXFWLogType)
+		{
+			BITMASK_SET(ui32Value, RGXFWIF_LOG_TYPE_TRACE);
+		}
+		eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32Value);
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType = ui32Value;
+
+	eResult = RGXFWTraceQueryFilter(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK != eResult)
+	{
+		return eResult;
+	}
+
+	/* 0 - trace, 1 - tbi */
+	if (0 == ui32Value)
+	{
+		BITMASK_SET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	else if (1 == ui32Value)
+	{
+		BITMASK_UNSET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid parameter %u specified to set FW log type AppHint.",
+		         __func__, ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32RGXFWLogType);
+	return eResult;
+}
+
+#if defined(DEBUG)
+static
+PVRSRV_ERROR RGXQueryFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL *pbValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pbValue = (PVRSRV_MEMALLOCFLAG_POISON_ON_FREE == psDevInfo->uiFWPoisonOnFreeFlag)
+		? IMG_TRUE
+		: IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXSetFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL bValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	psDevInfo->uiFWPoisonOnFreeFlag = bValue
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+ * RGXInitFirmware
+ */
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+#if defined(DEBUG)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bEnableFWPoisonOnFree = IMG_FALSE;
+#endif
+
+	eError = RGXSetupFirmware(psDeviceNode,
+	                          psApphints,
+	                          ui32ConfigFlags,
+	                          ui32ConfigFlagsExt,
+	                          ui32FwOsCfgFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXInitFirmwareKM: RGXSetupFirmware failed (%u)",
+		         eError));
+		goto failed_init_firmware;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableLogGroup,
+		                                    RGXFWTraceQueryFilter,
+		                                    RGXFWTraceSetFilter,
+		                                    psDeviceNode,
+		                                    NULL);
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_FirmwareLogType,
+		                                    RGXFWTraceQueryLogType,
+		                                    RGXFWTraceSetLogType,
+		                                    psDeviceNode,
+		                                    NULL);
+	}
+
+#if defined(DEBUG)
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFWPOISONONFREE;
+	OSGetAppHintBOOL(psDeviceNode,
+			pvAppHintState,
+			EnableFWPoisonOnFree,
+			&bAppHintDefault,
+			&bEnableFWPoisonOnFree);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFWPoisonOnFree,
+	                                   RGXQueryFWPoisonOnFree,
+	                                   RGXSetFWPoisonOnFree,
+	                                   psDeviceNode,
+	                                   NULL);
+
+	psDevInfo->uiFWPoisonOnFreeFlag = bEnableFWPoisonOnFree
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+#else
+	psDevInfo->uiFWPoisonOnFreeFlag = 0ULL;
+#endif
+
+	psDevInfo->ui32ClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	psDevInfo->ui32LastClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+
+	return PVRSRV_OK;
+
+failed_init_firmware:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* See device.h for function declaration */
+static PVRSRV_ERROR RGXAllocUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+									 IMG_UINT32 ui32RequestedSize,
+									 DEVMEM_MEMDESC **psMemDesc,
+									 IMG_UINT32 *puiSyncPrimVAddr,
+									 IMG_UINT32 *puiSyncPrimBlockSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
+	IMG_DEVMEM_ALIGN_T uiUFOBlockAlign = MAX(sizeof(IMG_UINT32), sizeof(SYNC_CHECKPOINT_FW_OBJ));
+	IMG_DEVMEM_SIZE_T uiUFOBlockSize = PVR_ALIGN(ui32RequestedSize, uiUFOBlockAlign);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Size and align are 'expanded' because we request an Exportalign allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiUFOBlockSize,
+	                                             &uiUFOBlockAlign);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										uiUFOBlockSize,
+										uiUFOBlockAlign,
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_UNCACHED,
+										"FwExUFOBlock",
+										psMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = RGXSetFirmwareAddress(&pFirmwareAddr, *psMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = pFirmwareAddr.ui32Addr;
+	*puiSyncPrimBlockSize = TRUNCATE_64BITS_TO_32BITS(uiUFOBlockSize);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, *psMemDesc);
+e0:
+	return eError;
+}
+
+/* See device.h for function declaration */
+static void RGXFreeUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+							DEVMEM_MEMDESC *psMemDesc)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGXUnsetFirmwareAddress(psMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psMemDesc);
+}
+
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	psDevInfo->bDevInit2Done = IMG_FALSE;
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	if (psDevInfo->psGenHeapSecMem != NULL)
+	{
+		OSFreeSecBuf(psDevInfo->psGenHeapSecMem);
+	}
+#endif
+
+
+	if ((psDevInfo->hTQUSCSharedMem != NULL) &&
+		(psDevInfo->hTQCLISharedMem != NULL))
+	{
+		PVRSRVTQUnloadShaders(psDeviceNode);
+	}
+
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvLISRData != NULL)
+	{
+		(void) SysUninstallDeviceLISR(psDevInfo->pvLISRData);
+	}
+	if (psDevInfo->pvMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvMISRData);
+	}
+	if (psDevInfo->hProcessQueuesMISR != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->hProcessQueuesMISR);
+	}
+	if (psDevInfo->pvAPMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvAPMISRData);
+	}
+	if (psDeviceNode->hCmdCompNotify != NULL)
+	{
+		/* Cancel notifications to this device */
+		PVRSRVUnregisterCmdCompleteNotify(psDeviceNode->hCmdCompNotify);
+		psDeviceNode->hCmdCompNotify = NULL;
+	}
+#endif /* !NO_HARDWARE */
+
+	/* Remove the device from the power manager */
+	PVRSRVRemovePowerDevice(psDeviceNode);
+
+	psDevInfo->pfnGetGpuUtilStats = NULL;
+	if (psDevInfo->hGPUUtilLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hGPUUtilLock);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(psDevInfo->hNMILock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hNMILock);
+	}
+#endif
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(psDevInfo->hMMUCtxUnregLock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	if (psDevInfo->hDebugFaultInfoLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hDebugFaultInfoLock);
+	}
+
+	/* De-init Freelists/ZBuffers... */
+	if (psDevInfo->hLockFreeList != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockFreeList);
+	}
+
+	if (psDevInfo->hLockZSBuffer != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockZSBuffer);
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* De-init work estimation lock */
+		if (psDevInfo->hWorkEstLock != NULL)
+		{
+			OSLockDestroy(psDevInfo->hWorkEstLock);
+		}
+	}
+#endif
+
+	/* Free DVFS Table */
+	if (psDevInfo->psGpuDVFSTable != NULL)
+	{
+		OSFreeMem(psDevInfo->psGpuDVFSTable);
+		psDevInfo->psGpuDVFSTable = NULL;
+	}
+}
+
+/*
+	DevDeInitRGX
+ */
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO		*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	PVRSRV_ERROR			eError;
+	DEVICE_MEMORY_INFO		*psDevMemoryInfo;
+
+	if (!psDevInfo)
+	{
+		/* Can happen if DevInitRGX failed */
+		PVR_DPF((PVR_DBG_ERROR, "DevDeInitRGX: Null DevInfo"));
+		return PVRSRV_OK;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	RGXUnregisterBridges(psDevInfo);
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	OSLockDestroy(psDevInfo->hCounterDumpingLock);
+#endif
+
+	/* Unregister debug request notifiers first as they could depend on anything. */
+
+	RGXDebugDeinit(psDevInfo);
+
+	/* De-initialise in reverse order, so stage 2 init is undone first. */
+	if (psDevInfo->bDevInit2Done)
+	{
+		DevPart2DeInitRGX(psDeviceNode);
+	}
+
+	/* Unregister MMU related stuff */
+	eError = RGXMMUInit_Unregister(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevDeInitRGX: Failed RGXMMUInit_Unregister (0x%x)",
+		         eError));
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* Unregister MMU related stuff */
+		eError = RGXMipsMMUInit_Unregister(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed RGXMipsMMUInit_Unregister (0x%x)",
+			         eError));
+		}
+	}
+#endif
+
+	/* UnMap Regs */
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+						 psDevInfo->ui32RegSize);
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvRegsBaseKM = NULL;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+			(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* undo the VA offset performed in RGXRegisterDevice() to allow the allocation to be unmapped */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM, RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvSecureRegsBaseKM = NULL;
+	}
+#endif
+
+#if 0 /* not required at this time */
+	if (psDevInfo->hTimer)
+	{
+		eError = OSRemoveTimer(psDevInfo->hTimer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed to remove timer"));
+			return eError;
+		}
+		psDevInfo->hTimer = NULL;
+	}
+#endif
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+
+	RGXDeInitHeaps(psDevMemoryInfo, psDeviceNode);
+
+	if (psDevInfo->psRGXFWCodeMemDesc)
+	{
+		/* Free fw code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+		psDevInfo->psRGXFWCodeMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware code memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		if (psDevInfo->psTrampoline->sPages.u.pvHandle)
+		{
+			/* Free trampoline region */
+			PDUMPCOMMENT(psDeviceNode, "Freeing trampoline memory");
+			RGXFreeTrampoline(psDeviceNode);
+		}
+	}
+#endif
+
+	if (psDevInfo->psRGXFWDataMemDesc)
+	{
+		/* Free fw data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW data memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+		psDevInfo->psRGXFWDataMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware data memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+	if (psDevInfo->psRGXFWCorememCodeMemDesc)
+	{
+		/* Free fw core mem code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+	{
+		/* Free fw core mem data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem data store memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+
+	/*
+	   Free the firmware allocations.
+	 */
+	RGXFreeFirmware(psDevInfo);
+
+
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+
+	/* De-initialise non-device specific (TL) users of RGX device memory */
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+		{
+			RGXHWPerfDeinitL2Stream(psDevInfo, i);
+		}
+
+		RGXHWPerfDeinit(psDevInfo);
+	}
+
+	RGXDeInitDestroyFWKernelMemoryContext(psDeviceNode);
+
+	RGXHWPerfHostDeInit(psDevInfo);
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+
+	/* destroy the stalled CCB locks */
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+
+	/* destroy the context list locks */
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+	OSLockDestroy(psDevInfo->hBPLock);
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+
+	/* Free device BVNC string */
+	if (NULL != psDevInfo->sDevFeatureCfg.pszBVNCString)
+	{
+		OSFreeMem(psDevInfo->sDevFeatureCfg.pszBVNCString);
+	}
+
+
+	/* DeAllocate devinfo */
+	OSFreeMem(psDevInfo);
+
+	psDeviceNode->pvDevice = NULL;
+
+	return PVRSRV_OK;
+}
+
+#if defined(PDUMP)
+static
+PVRSRV_ERROR RGXResetPDump(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/* Takes a log2 page size parameter and calculates a suitable page size
+ * for the RGX heaps. Returns 0 if parameter is wrong.*/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize)
+{
+	IMG_BOOL bFound = IMG_FALSE;
+	IMG_UINT32 ui32PageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	/* OS page shift must be at least RGX_HEAP_4KB_PAGE_SHIFT,
+	 * max RGX_HEAP_2MB_PAGE_SHIFT, non-zero and a power of two*/
+	if (uiLog2PageSize == 0U ||
+	    (uiLog2PageSize < RGX_HEAP_4KB_PAGE_SHIFT) ||
+	    (uiLog2PageSize > RGX_HEAP_2MB_PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Provided incompatible log2 page size %u",
+				__func__,
+				uiLog2PageSize));
+		PVR_ASSERT(0);
+		return 0;
+	}
+
+	do
+	{
+		if ((IMG_PAGE2BYTES32(uiLog2PageSize) & ui32PageSizeMask) == 0)
+		{
+			/* We have to fall back to a smaller device
+			 * page size than given page size because there
+			 * is no exact match for any supported size. */
+			uiLog2PageSize -= 1U;
+		}
+		else
+		{
+			/* All good, RGX page size equals given page size
+			 * => use it as default for heaps */
+			bFound = IMG_TRUE;
+		}
+	} while (!bFound);
+
+	return uiLog2PageSize;
+}
+
+/* First 16-bits define possible types */
+#define HEAP_INST_VALUE_MASK     (0xFFFF)
+#define HEAP_INST_DEFAULT_VALUE  (1U)  /* Used to show either the heap is always instantiated by default (pfn = NULL)
+	                                      OR
+	                                      that this is the default configuration of the heap with an Alternative BRN */
+#define HEAP_INST_BRN_DEP_VALUE  (2U)  /* The inclusion of this heap is dependent on the brn being present */
+#define HEAP_INST_FEAT_DEP_VALUE (3U)  /* The inclusion of this heap is dependent on the feature being present */
+#define HEAP_INST_BRN_ALT_VALUE  (4U)  /* This entry is a possible alternative to the default determined by a BRN */
+#define HEAP_INST_FEAT_ALT_VALUE (5U)  /* The entry is a possible alternative to the default determined by a Feature define */
+
+/* Latter 16-bits define other flags we may need */
+#define HEAP_INST_NON4K_FLAG     (1 << 16U) /* This is a possible NON4K Entry and we should use the device
+                                               NON4K size when instantiating */
+
+typedef struct RGX_HEAP_INFO_TAG RGX_HEAP_INFO; // Forward declaration
+typedef IMG_BOOL (*PFN_IS_PRESENT)(PVRSRV_RGXDEV_INFO*, const RGX_HEAP_INFO*);
+typedef void (*PFN_HEAP_DYNAMIC)(PVRSRV_DEVICE_NODE*, RGX_HEAP_INFO*);
+
+struct RGX_HEAP_INFO_TAG
+{
+	IMG_CHAR           *pszName;
+	IMG_UINT64         ui64HeapBase;
+	IMG_DEVMEM_SIZE_T  uiHeapLength;
+	IMG_DEVMEM_SIZE_T  uiHeapReservedRegionLength;
+	IMG_UINT32         ui32Log2ImportAlignment;
+	PFN_IS_PRESENT     pfnIsHeapPresent;
+	PFN_HEAP_DYNAMIC   pfnDynamicBaseSize; /* May modify the psHeapInfo's base & length. May be NULL.
+	                                          Only called once if the heap is present otherwise never. */
+	PFN_HEAP_INIT      pfnInit;
+	PFN_HEAP_DEINIT    pfnDeInit;
+	IMG_UINT32         ui32HeapInstanceFlags;
+};
+
+/* RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES is the total amount of reserved space, to be specified in gasRGXHeapLayoutApp[] */
+#define RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES	(RGX_HEAP_UM_GENERAL_RESERVED_SIZE + RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE)
+
+/* Private heap data struct. */
+typedef struct RGX_HEAP_DATA_TAG
+{
+	DEVMEMINT_RESERVATION  *psMemReservation;
+} RGX_HEAP_DATA;
+
+static PVRSRV_ERROR HeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_HEAP *psDevmemHeap,
+							 PMR *psPMR,
+							 IMG_DEVMEM_SIZE_T uiSize,
+							 IMG_UINT64 ui64Offset,
+							 IMG_BOOL bWriteAble,
+							 IMG_HANDLE *phPrivData)
+{
+	RGX_HEAP_DATA *psHeapData;
+	IMG_DEV_VIRTADDR sCarveOutAddr;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap, "psDevmemHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phPrivData, "phPrivData");
+
+	psHeapData = OSAllocMem(sizeof(*psHeapData));
+	PVR_LOG_RETURN_IF_NOMEM(psHeapData, "psHeapData");
+
+	/* Map the per device secure mem PMR allocation to the general devmem heap carveout. */
+	sCarveOutAddr = DevmemIntHeapGetBaseAddr(psDevmemHeap);
+	sCarveOutAddr.uiAddr += ui64Offset;
+
+	eError = DevmemIntReserveRange(NULL,
+	                               NULL,
+	                               psDevmemHeap,
+	                               sCarveOutAddr,
+	                               uiSize,
+	                               PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                               (bWriteAble ? PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE : 0),
+	                               &psHeapData->psMemReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeHeapData);
+
+	eError = DevmemIntMapPMR(psHeapData->psMemReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	*phPrivData = (IMG_HANDLE)psHeapData;
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+ErrorFreeHeapData:
+	OSFreeMem(psHeapData);
+
+	return eError;
+}
+
+/* Deinit callback function for general heap. */
+static void HeapDeInit(IMG_HANDLE hPrivData)
+{
+	RGX_HEAP_DATA *psHeapData = (RGX_HEAP_DATA*)hPrivData;
+
+	PVR_ASSERT(hPrivData);
+
+	DevmemIntUnmapPMR(psHeapData->psMemReservation);
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+
+	OSFreeMem(psHeapData);
+}
+
+/* Init callback function for general heap. */
+static PVRSRV_ERROR USCHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_FALSE;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	uiSize = PMR_PhysicalSize(psDevInfo->hTQUSCSharedMem);
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->hTQUSCSharedMem,
+					  uiSize,
+					  RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+/* Init callback function for general heap. */
+static PVRSRV_ERROR GeneralHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_TRUE;
+	PVRSRV_ERROR eError;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->psGenHeapSecMem,
+					  RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE,
+					  RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#define GeneralHeapDeInit HeapDeInit
+#else
+/* Callbacks not used */
+#define GeneralHeapInit NULL
+#define GeneralHeapDeInit NULL
+#endif
+
+static void SVMHeapDynamic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           RGX_HEAP_INFO *psHeapInfo)
+{
+	IMG_UINT64 ui64OSPageSize = OSGetPageSize();
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+
+	/* Ensures the SVM heap has the correct alignment & size for any OS page size.
+	 *
+	 * The SVM heap's base must be the smallest possible address mappable by UM.
+	 * This is 32KB unless the page size is larger than 32KB. [1]
+	 * If the page size > 32KB, raise the SVM heap base to the next page boundary.
+	 * Also reduce the length to ensure it's still page aligned and doesn't go
+	 * into another heap.
+	 *
+	 * [1]: https://chromium.googlesource.com/chromium/src/+/fe24932ee14aa93e1fe4d3e7003b9362591a54d4/docs/security/faq.md#why-aren_t-null-pointer-dereferences-considered-security-bugs
+	 */
+	IMG_UINT64 ui64Base = PVR_ALIGN(psHeapInfo->ui64HeapBase, ui64OSPageSize);
+	IMG_UINT64 ui64BaseDiff = ui64Base - psHeapInfo->ui64HeapBase;
+	psHeapInfo->ui64HeapBase = ui64Base;
+	if (psHeapInfo->uiHeapLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapLength -= ui64BaseDiff;
+	if (psHeapInfo->uiHeapReservedRegionLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapReservedRegionLength -= ui64BaseDiff;
+
+	/* The device shared-virtual-memory heap address-space size is stored on the device for
+	   faster look-up without having to walk the device heap configuration structures during
+	   client device connection  (i.e. this size is relative to a zero-based offset) */
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = 0;
+	}
+	else
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = psHeapInfo->ui64HeapBase + psHeapInfo->uiHeapLength;
+	}
+}
+
+/* Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+static IMG_BOOL BRN65273IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		return (((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_ALT_VALUE) ||
+		        ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_DEP_VALUE)) ?
+		        IMG_TRUE : IMG_FALSE;
+	}
+	else
+	{
+		return ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_DEFAULT_VALUE) ? IMG_TRUE : IMG_FALSE;
+	}
+}
+#endif
+
+#if defined(FIX_HW_BRN_63142_BIT_MASK)
+static IMG_BOOL BRN63142IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 63142))
+	{
+		PVR_ASSERT((pksHeapInfo->ui64HeapBase & IMG_UINT64_C(0x3FFFFFFFF)) +
+		            pksHeapInfo->uiHeapLength == IMG_UINT64_C(0x400000000));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+#endif
+
+/* FW Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+static IMG_BOOL FWBRN65101IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap size and base at run time VS the current table instance
+	 */
+	IMG_UINT64 ui64MainSubHeapSize;
+
+	/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)))
+	{
+		if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101;
+		}
+		else
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL;
+		}
+	}
+	else
+	{
+		ui64MainSubHeapSize = RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE;
+	}
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->uiHeapLength == ui64MainSubHeapSize &&
+	        pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ?
+	        IMG_TRUE : IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FWVZConfigPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_CONFIG_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+static IMG_BOOL FWVZMainHeapPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+static IMG_BOOL BPHandlerHeapPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+	return IMG_TRUE;
+#else
+	return IMG_FALSE;
+#endif
+}
+
+static RGX_HEAP_INFO gasRGXHeapLayoutApp[] =
+{
+	/* Name                             HeapBase                                 HeapLength                               HeapReservedRegionLength                     Log2ImportAlignment pfnIsHeapPresent      pfnDynamicBaseSize pfnInit           pfnDeInit          HeapInstanceFlags   */
+	{RGX_GENERAL_SVM_HEAP_IDENT,        RGX_GENERAL_SVM_HEAP_BASE,               RGX_GENERAL_SVM_HEAP_SIZE,               0,                                           0,                  NULL,                 SVMHeapDynamic,    NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_HEAP_BASE,                   RGX_GENERAL_HEAP_SIZE,                   RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  NULL,                 NULL,              GeneralHeapInit,  GeneralHeapDeInit, HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_HEAP_BASE,             RGX_GENERAL_NON4K_HEAP_SIZE,             0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_HEAP_BASE,               RGX_PDSCODEDATA_HEAP_SIZE,               RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_HEAP_BASE,                   RGX_USCCODE_HEAP_SIZE,                   RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  NULL,                 NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_BPH_HEAP_IDENT,        RGX_USCCODE_BPH_HEAP_BASE,               RGX_USCCODE_BPH_HEAP_SIZE,               0,                                           0,                  BPHandlerHeapPresent, NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VK_CAPT_REPLAY_HEAP_IDENT,     RGX_VK_CAPT_REPLAY_HEAP_BASE,            RGX_VK_CAPT_REPLAY_HEAP_SIZE,            0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_COMPONENT_CTRL_HEAP_IDENT,     RGX_COMPONENT_CTRL_HEAP_BASE,            RGX_COMPONENT_CTRL_HEAP_SIZE,            0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_HEAP_IDENT,              RGX_FBCDC_HEAP_BASE,                     RGX_FBCDC_HEAP_SIZE,                     0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_LARGE_HEAP_IDENT,        RGX_FBCDC_LARGE_HEAP_BASE,               RGX_FBCDC_LARGE_HEAP_SIZE,               0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_PDS_INDIRECT_STATE_HEAP_IDENT, RGX_PDS_INDIRECT_STATE_HEAP_BASE,        RGX_PDS_INDIRECT_STATE_HEAP_SIZE,        0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_MISSION_RMW_HEAP_IDENT,    RGX_CMP_MISSION_RMW_HEAP_BASE,           RGX_CMP_MISSION_RMW_HEAP_SIZE,           0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_SAFETY_RMW_HEAP_IDENT,     RGX_CMP_SAFETY_RMW_HEAP_BASE,            RGX_CMP_SAFETY_RMW_HEAP_SIZE,            0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TEXTURE_STATE_HEAP_IDENT,      RGX_TEXTURE_STATE_HEAP_BASE,             RGX_TEXTURE_STATE_HEAP_SIZE,             0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_HEAP_BASE,           RGX_VISIBILITY_TEST_HEAP_SIZE,           0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+};
+
+static RGX_HEAP_INFO gasRGXHeapLayoutFW[] =
+{
+	/* Name                          HeapBase                        HeapLength                                      HeapReservedRegionLength Log2ImportAlignment pfnIsHeapPresent     pfnDynamicBaseSize pfnInit pfnDeInit HeapInstanceFlags*/
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE,            0,                       0,                  FWVZMainHeapPresent, NULL,              NULL,   NULL,     HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_CONFIG_HEAP_IDENT, RGX_FIRMWARE_CONFIG_HEAP_BASE,  RGX_FIRMWARE_CONFIG_HEAP_SIZE,                  0,                       0,                  FWVZConfigPresent,   NULL,              NULL,   NULL,     HEAP_INST_DEFAULT_VALUE},
+};
+
+static INLINE IMG_BOOL IsFwHeapLayout(const RGX_HEAP_INFO *psHeapInfo)
+{
+	return psHeapInfo->pszName[0] == 'F' &&
+	       psHeapInfo->pszName[1] == 'w' ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE void CheckHeapAlignment(const RGX_HEAP_INFO *psHeapInfo,
+                                      const PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 uiAlignment = RGX_HEAP_BASE_SIZE_ALIGN - 1;
+
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	if (IsFwHeapLayout(psHeapInfo))
+	{
+		/*
+		 * 1) Main heap starts at 2MB offset 0xEC10000000UL
+		 * 2) Config Sub heap is created at the end of the main heap making the entire unit start and end at 2MB offset aligned,
+		 *    these 2 heaps will always have the same page size
+		 *    There are no other heaps in between these two heaps, there is no risk for another devmem heap be created between them.
+		 */
+		return;
+	}
+
+	/* General SVM Heap must be placed below 2MiB boundary so we need to adjust
+	 * the validity condition. This is because an OS might return virtual
+	 * addresses below 2MiB threshold. By default (based on testing on Linux)
+	 * this is around 32KiB. */
+	if (OSStringNCompare(psHeapInfo->pszName, RGX_GENERAL_SVM_HEAP_IDENT,
+	                     sizeof(RGX_GENERAL_SVM_HEAP_IDENT)) == 0)
+	{
+		uiAlignment = RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT - 1;
+	}
+
+	/* All UM accessible heap bases should be aligned to 2MB */
+	if (psHeapInfo->ui64HeapBase & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Base not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Base: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Base (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->ui64HeapBase,
+		         psHeapInfo->ui64HeapBase,
+		         uiAlignment + 1));
+	}
+
+	/* All UM accessible heaps should also be size aligned to 2MB */
+	if (psHeapInfo->uiHeapLength & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Size not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Size (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_SIZE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->uiHeapLength,
+		         psHeapInfo->uiHeapLength,
+		         uiAlignment + 1));
+	}
+}
+
+/* Generic counting method. */
+static void _CountRequiredHeaps(PVRSRV_RGXDEV_INFO  *psDevInfo,
+	                            const RGX_HEAP_INFO  pksHeapInfo[],
+	                            IMG_UINT32           ui32HeapListSize,
+	                            IMG_UINT32*          ui32HeapCount)
+{
+	IMG_UINT32 i;
+
+	/* Loop over rows in the heap data array using callback to decide if we
+	 * should include the heap
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo = &pksHeapInfo[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		(*ui32HeapCount)++;
+	}
+}
+/* Generic heap instantiator */
+static void _InstantiateRequiredHeaps(PVRSRV_RGXDEV_INFO     *psDevInfo,
+                                      RGX_HEAP_INFO           psHeapInfos[],
+                                      IMG_UINT32              ui32HeapListSize,
+                                      const IMG_UINT32        ui32Log2RgxDefaultPageShift,
+                                      DEVMEM_HEAP_BLUEPRINT **psDeviceMemoryHeapCursor)
+{
+	IMG_UINT32 i;
+
+#if defined DEBUG
+	IMG_UINT32 ui32heapListCnt;
+	bool bHeapPageSizeMisMatch = false;
+
+	/*
+	 * To ensure all heaps within a 2MB region have the same page sizes
+	 */
+	for (ui32heapListCnt = 0; ui32heapListCnt < (ui32HeapListSize-1); ui32heapListCnt++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo1 = &psHeapInfos[ui32heapListCnt];
+		const RGX_HEAP_INFO *psHeapInfo2 = &psHeapInfos[ui32heapListCnt+1];
+
+		if (((psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) &&
+			((psHeapInfo1->ui64HeapBase + psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) ==
+			((psHeapInfo2->ui64HeapBase) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)))
+		{
+			if (psHeapInfo1->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				if (!(psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG))
+				{
+					bHeapPageSizeMisMatch = true;
+				}
+			}
+			else if (psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				bHeapPageSizeMisMatch = true;
+			}
+		}
+		if (bHeapPageSizeMisMatch)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Two Heap with Different Page Size allocated in the same PD space(2MB)\n"
+			         "Invalid Heaps 1) \"%s\" and 2) \"%s\"",
+			         __func__,
+			         psHeapInfo1->pszName,
+			         psHeapInfo2->pszName));
+		}
+	}
+#endif
+
+	/* We now have a list of the heaps to include and so we should loop over this
+	 * list and instantiate.
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		IMG_UINT32 ui32Log2DataPageSize = 0;
+		RGX_HEAP_INFO *psHeapInfo = &psHeapInfos[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		if (psHeapInfo->pfnDynamicBaseSize != NULL)
+		{
+			psHeapInfo->pfnDynamicBaseSize(psDevInfo->psDeviceNode, psHeapInfo);
+		}
+
+		if (psHeapInfo->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+		{
+			ui32Log2DataPageSize = psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2;
+		}
+		else
+		{
+			ui32Log2DataPageSize = ui32Log2RgxDefaultPageShift;
+		}
+
+		CheckHeapAlignment(psHeapInfo, psDevInfo);
+
+		HeapCfgBlueprintInit(psHeapInfo->pszName,
+		                     psHeapInfo->ui64HeapBase,
+		                     psHeapInfo->uiHeapLength,
+		                     psHeapInfo->uiHeapReservedRegionLength,
+		                     ui32Log2DataPageSize,
+		                     psHeapInfo->ui32Log2ImportAlignment,
+		                     psHeapInfo->pfnInit,
+		                     psHeapInfo->pfnDeInit,
+		                     *psDeviceMemoryHeapCursor);
+
+		(*psDeviceMemoryHeapCursor)++;
+	}
+}
+
+static PVRSRV_ERROR RGXInitHeaps(PVRSRV_RGXDEV_INFO *psDevInfo,
+	                             DEVICE_MEMORY_INFO *psNewMemoryInfo)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor;
+
+	IMG_UINT32 ui32AppHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutApp);
+	IMG_UINT32 ui32FWHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutFW);
+	IMG_UINT32 ui32CountedHeapSize;
+
+	IMG_UINT32 ui32AppHeapCount = 0;
+	IMG_UINT32 ui32FWHeapCount = 0;
+
+	IMG_UINT32 ui32Log2DefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	if (ui32Log2DefaultPageShift == 0)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2DefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2DefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e0;
+		}
+	}
+#endif
+
+	/* Count heaps required for the app heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutApp,
+		                ui32AppHeapListSize,
+		                &ui32AppHeapCount);
+
+	/* Count heaps required for the FW heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutFW,
+		                ui32FWHeapListSize,
+		                &ui32FWHeapCount);
+
+	ui32CountedHeapSize = (ui32AppHeapCount + ui32FWHeapCount + RGX_NUM_DRIVERS_SUPPORTED);
+
+	psNewMemoryInfo->psDeviceMemoryHeap = OSAllocMem(sizeof(DEVMEM_HEAP_BLUEPRINT) * ui32CountedHeapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeap, eError, e0);
+
+	/* Initialise the heaps */
+	psDeviceMemoryHeapCursor = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	/* Instantiate App Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutApp,
+	                          ui32AppHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* Instantiate FW Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutFW,
+	                          ui32FWHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* set the heap count */
+	psNewMemoryInfo->ui32HeapCount = (IMG_UINT32)(psDeviceMemoryHeapCursor - psNewMemoryInfo->psDeviceMemoryHeap);
+
+	/* Check we have allocated the correct # of heaps, minus any VZ heaps as these
+	 * have not been created at this point
+	 */
+	PVR_ASSERT(psNewMemoryInfo->ui32HeapCount == (ui32CountedHeapSize - RGX_NUM_DRIVERS_SUPPORTED));
+
+	/*
+	   In the new heap setup, we initialise 2 configurations:
+		1 - One will be for the firmware only (index 1 in array)
+			a. This primarily has the firmware heap in it.
+			b. It also has additional guest OSID firmware heap(s)
+				- Only if the number of support firmware OSID > 1
+		2 - Others shall be for clients only (index 0 in array)
+			a. This has all the other client heaps in it.
+	 */
+	psNewMemoryInfo->uiNumHeapConfigs = 2;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray = OSAllocMem(sizeof(DEVMEM_HEAP_CONFIG) * psNewMemoryInfo->uiNumHeapConfigs);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeapConfigArray, eError, e1);
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].pszName = "Default Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].uiNumHeaps = psNewMemoryInfo->ui32HeapCount - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].psHeapBlueprintArray = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].pszName = "Firmware Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps = RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].psHeapBlueprintArray = psDeviceMemoryHeapCursor - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4)
+	{
+		IMG_UINT32 i;
+
+		/*
+		 * Initialise all MMU Page Size Range Config register to the default page size
+		 * used by the OS, leaving the address range 0;
+		 */
+		for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[i] = RGXMMUInitRangeValue(i);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		/* Create additional raw firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVINFO, psDevInfo)
+		{
+			eError = RGXInitFwRawHeap(psDevInfo, psDeviceMemoryHeapCursor, ui32DriverID);
+			if (eError != PVRSRV_OK)
+			{
+				/* if any allocation fails, free previously allocated heaps and abandon initialisation */
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+					psDeviceMemoryHeapCursor--;
+				}
+				goto e1;
+			}
+
+			/* Append additional firmware heaps to host driver firmware context heap configuration */
+			psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps += 1;
+
+			/* advance to the next heap */
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	return PVRSRV_OK;
+e1:
+	OSFreeMem(psNewMemoryInfo->psDeviceMemoryHeap);
+e0:
+	return eError;
+}
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+		DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor = psDevMemoryInfo->psDeviceMemoryHeap;
+
+		/* Delete all guest firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeapConfigArray);
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeap);
+}
+
+static PVRSRV_ERROR RGXInitSharedFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psSysHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* VZ heap validation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg != NULL,
+								"FW Main heap is required for VZ Guest.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+#endif
+
+	if (psSysHeapCfg != NULL)
+	{
+		/* Check FW_SHARED for multiple usage flags. Because FW_SHARED is divided
+		   into subheaps, shared usage with other heaps is not allowed.  */
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_SHARED,
+								"FW_SHARED phys heap config not specified with more than one usage."
+								"FW_SHARED heap must be exclusively used as FW_SHARED.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	if (psSysHeapCfg == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap not set", __func__));
+		/* Nothing to do. Default to the physheap fallback option */
+	}
+	else if (psSysHeapCfg->eType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses OS System memory (UMA)", __func__));
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psSysHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG] = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	}
+	else /* PHYS_HEAP_TYPE_LMA or PHYS_HEAP_TYPE_DMA */
+	{
+		PHYS_HEAP_CONFIG sFwMainHeapCfg, sFwCfgHeapCfg;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses local memory managed by the driver (LMA)", __func__));
+
+
+		/* Subheap layout: Main + (optional MIPS reserved range) + Config */
+		sFwMainHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		/* Reserve space for the Config heap */
+		sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+			/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)))
+			{
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+				if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101;
+				}
+				else
+#endif
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL;
+				}
+			}
+		}
+#endif
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwMainHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwMainHeapCfg,
+		                              "Fw Main subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:MAIN", ErrorDeinit);
+
+		sFwCfgHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		sFwCfgHeapCfg.uConfig.sLMA.sStartAddr.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+		sFwCfgHeapCfg.uConfig.sLMA.sCardBase.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		sFwCfgHeapCfg.uConfig.sLMA.uiSize = RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwCfgHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwCfgHeapCfg,
+		                              "Fw Cfg subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:CFG", ErrorDeinit);
+	}
+
+	/* Acquire FW heaps */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_MAIN, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_MAIN", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CONFIG, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CONFIG", ErrorDeinit);
+
+	return eError;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitPrivateFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psFwCodeHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_CODE);
+	PHYS_HEAP_CONFIG *psFwDataHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIV_DATA);
+	PHYS_HEAP_CONFIG *psFwPrivateHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIVATE);
+	PHYS_HEAP_CONFIG sFwPrivateTempCfg;
+
+	if (psFwPrivateHeapCfg != NULL)
+	{
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg == NULL) && (psFwDataHeapCfg == NULL),
+								"FW_PRIVATE and the FW_CODE & FW_PRIV_DATA usage flags "
+								"achieve the same goal and are mutually exclusive.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Fw code and data are both allocated from this unified heap */
+		sFwPrivateTempCfg = *psFwPrivateHeapCfg;
+		sFwPrivateTempCfg.ui32UsageFlags = PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+		psFwCodeHeapCfg = &sFwPrivateTempCfg;
+		psFwDataHeapCfg = &sFwPrivateTempCfg;
+	}
+
+	if ((psFwCodeHeapCfg == NULL) || (psFwDataHeapCfg == NULL))
+	{
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware code and data heaps must be either both defined
+			 * or both undefined. There is no point in isolating one but not
+			 * the other.*/
+			eError = PVRSRV_ERROR_PHYSHEAP_CONFIG;
+			PVR_LOG_GOTO_IF_ERROR(eError, "PrivateFwPhysHeap check", ErrorDeinit);
+		}
+		else
+		{
+			/* No dedicated heaps, default to the physheap fallback option */
+		}
+	}
+	else if (psFwCodeHeapCfg == psFwDataHeapCfg)
+	{
+		if (psFwCodeHeapCfg->ui32UsageFlags ==
+			(PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA))
+		{
+			/* Fw code and private data allocations come from the same system heap
+			 * Instantiate one physheap and share it between them. */
+
+			eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+												  psFwCodeHeapCfg,
+												  NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig");
+		}
+		else
+		{
+			/* Not an exclusive heap, can be used for other purposes (e.g. secure buffers).
+			 * Expect the PVR layer to have already created a heap for the other uses. */
+		}
+	}
+	else
+	{
+		/*
+		 * Separating private Firmware code and data is allowed for backwards compatibility
+		 * purposes. New platforms should use the unified FW_PRIVATE heap instead.
+		 *
+		 * Early security implementations on Rogue cores required separate FW_PRIV_DATA
+		 * and FW_CODE heaps, as access permissions to Firmware were granted differently
+		 * based on the transaction types (code or data).
+		 */
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_CODE) &&
+								(psFwDataHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PRIV_DATA),
+								"Dedicated private heaps for Fw code and "
+								"data must have one usage flag exclusively.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Dedicated Fw code heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwCodeHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		/* Dedicated Fw private data heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwDataHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+	/* When premapping distinct private and shared Firmware phys heaps
+	 * inside the same virtual devmem heap, their sizes must add up to
+	 * the fixed RGX_FIRMWARE_RAW_HEAP_SIZE for the premapping to work */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwSharedHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		PHYS_HEAP_USAGE_FW_SHARED);
+		IMG_UINT64 ui64FwCodeHeapSize = PhysHeapConfigGetSize(psFwCodeHeapCfg);
+		IMG_UINT64 ui64FwDataHeapSize = PhysHeapConfigGetSize(psFwDataHeapCfg);
+		IMG_UINT64 ui64FwSharedHeapSize = PhysHeapConfigGetSize(psFwSharedHeapCfg);
+		IMG_UINT64 ui64FwPrivateHeapSize;
+
+		PVR_LOG_GOTO_IF_FALSE((psFwCodeHeapCfg != NULL) && (psFwDataHeapCfg != NULL),
+							  "Security support requires Fw code and data memory be"
+							  " separate from the heap shared with the kernel driver.", FailDeinit);
+
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware allocations come from 2 different heaps */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize + ui64FwDataHeapSize;
+		}
+		else
+		{
+			/* Private Firmware allocations come from a single heap */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize;
+		}
+
+				PVR_LOG_GOTO_IF_FALSE((ui64FwSharedHeapSize +
+									   ui64FwPrivateHeapSize) ==
+									   RGX_FIRMWARE_RAW_HEAP_SIZE,
+									   "Invalid firmware physical heap size.", FailDeinit);
+	}
+#endif
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CODE, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CODE]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CODE", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_PRIV_DATA, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PRIV_DATA]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_DATA", ErrorDeinit);
+
+	return eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+FailDeinit:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitFwPageTableHeap(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwPageTableHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		  PHYS_HEAP_USAGE_FW_PREMAP_PT);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg != NULL),
+								"The Firmware Page Table phys heap config not found.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PREMAP_PT),
+								"The Firmware Page Table heap must be used exclusively for this purpose",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_LMA) ||
+								(psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_DMA),
+								"The Firmware Page Table heap must be LMA or DMA memory.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->uConfig.sLMA.uiSize >= RGX_FIRMWARE_MAX_PAGETABLE_SIZE),
+								"The Firmware Page Table heap must be able to hold the maximum "
+								"number of pagetables needed to cover the Firmware's VA space.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwPageTableHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig:FwPageTableHeap");
+
+		eError = PhysHeapAcquire(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquire:FwPageTableHeap");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) */
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXInitFwPageTableHeap(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitFwPageTableHeap", ErrorDeinit);
+	eError = RGXInitSharedFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitSharedFwPhysHeaps", ErrorDeinit);
+	eError = RGXInitPrivateFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitPrivateFwPhysHeaps", ErrorDeinit);
+
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDeviceFWMainHeapMemCheck
+@Description    Checks the free memory in FW Main PhysHeap of a device to ensure
+                there is enough for a connection to be made.
+
+@Input          psDeviceNode    The device of the FW Main PhysHeap to be checked.
+
+@Return         On success PVRSRV_OK, else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDeviceFWMainHeapMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP *psFWMainPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psFWMainPhysHeap = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	if (psFWMainPhysHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's FW Main PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psFWMainPhysHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		const IMG_UINT32 ui32MinMemInKBs = RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION;
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psFWMainPhysHeap,
+		                              KB2B(ui32MinMemInKBs),
+		                              &ui64FreePhysHeapMem);
+
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FW_MAIN PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"KB "
+				"Minimum required: %uKB",
+				B2KB(ui64FreePhysHeapMem),
+				ui32MinMemInKBs));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate, IMG_UINT32 *pui32Log2Non4KPgShift)
+{
+	IMG_UINT32 uiLog2OSPageShift = OSGetPageShift();
+
+	/* We support Non4K pages only on platforms with 4KB pages. On all platforms
+	 * where OS pages are larger than 4KB we must ensure the non4K device memory
+	 * heap matches the page size used in all other device memory heaps, which
+	 * is the OS page size, see RGXHeapDerivePageSize. */
+	if (uiLog2OSPageShift > RGX_HEAP_4KB_PAGE_SHIFT)
+	{
+		*pui32Log2Non4KPgShift = RGXHeapDerivePageSize(uiLog2OSPageShift);
+	}
+	else
+	{
+		IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+		IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+		IMG_UINT32 ui32GeneralNon4KValidHeapPageSize;
+		IMG_UINT32 ui32Log2Non4KPgShift;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* The TEE-DDK is not aware of REE-DDK apphints. Stick to default */
+		ui32GeneralNon4KHeapPageSize = ui32AppHintDefault;
+#else
+		void *pvAppHintState = NULL;
+		/* Get the page size for the dummy page from the NON4K heap apphint */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState,
+		                     GeneralNon4KHeapPageSize, &ui32AppHintDefault,
+		                     &ui32GeneralNon4KHeapPageSize);
+		OSFreeAppHintState(pvAppHintState);
+#endif
+		/* Validate that the given ui32GeneralNon4KHeapPageSize is one of the
+		 * supported range of values.
+		 */
+		ui32Log2Non4KPgShift = RGXHeapDerivePageSize(ExactLog2(ui32GeneralNon4KHeapPageSize));
+		ui32GeneralNon4KValidHeapPageSize = (1U << ui32Log2Non4KPgShift);
+		if (ui32GeneralNon4KValidHeapPageSize != ui32GeneralNon4KHeapPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Non4K Heap Size: Requested"
+			         " Heap size=%u, actual=%u, pageshift 0x%x",
+			         __func__, ui32GeneralNon4KHeapPageSize,
+			         ui32GeneralNon4KValidHeapPageSize, ui32Log2Non4KPgShift));
+		}
+
+		if (ui32Log2Non4KPgShift == 0U)
+		{
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+
+		*pui32Log2Non4KPgShift = ui32Log2Non4KPgShift;
+	}
+
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 71317))
+	{
+		if (*pui32Log2Non4KPgShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || *pui32Log2Non4KPgShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Page sizes of 2MB or 1MB cause page faults."));
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+#endif
+
+	/* Check the Non4k page size is at least the size of the OS page size
+	 * or larger. The Non4k page size also has to be a multiple of the OS page
+	 * size but since we have the log2 value from the apphint we know powers of 2
+	 * will always be multiples. If the Non4k page size is less than OS page size
+	 * we notify and upgrade the size.
+	 */
+	if (*pui32Log2Non4KPgShift < uiLog2OSPageShift)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "Non4K page size smaller than OS page size, upgrading to "
+		                          "match OS page size."));
+		*pui32Log2Non4KPgShift = uiLog2OSPageShift;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* RGXRegisterDevice
+ *
+ * WARNING!
+ *
+ * No PDUMP statements are allowed until device initialisation starts.
+ */
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	DEVICE_MEMORY_INFO *psDevMemoryInfo;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = HWPERF_HOST_TL_STREAM_SIZE_DEFAULT, ui32HWPerfHostBufSizeKB;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault, &ui32HWPerfHostBufSizeKB);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/*********************
+	 * Device node setup *
+	 *********************/
+	/* Setup static data and callbacks on the device agnostic device node */
+#if defined(PDUMP)
+	psDeviceNode->sDevId.pszPDumpRegName	= RGX_PDUMPREG_NAME;
+	psDeviceNode->sDevId.pszPDumpDevName	= PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+	psDeviceNode->pfnPDumpInitDevice = &RGXResetPDump;
+	psDeviceNode->ui64FBCClearColour = RGX_FBC_CC_DEFAULT;
+
+#endif /* PDUMP */
+
+	OSAtomicWrite(&psDeviceNode->eHealthStatus, PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	OSAtomicWrite(&psDeviceNode->eHealthReason, PVRSRV_DEVICE_HEALTH_REASON_NONE);
+
+	psDeviceNode->pfnInvalFBSCTable = RGXInvalidateFBSCTable;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = NULL;
+
+	/* Callback for getting the MMU device attributes */
+	psDeviceNode->pfnGetMMUDeviceAttributes = RGXDevMMUAttributes;
+	psDeviceNode->pfnGetDeviceSnoopMode = RGXDevSnoopMode;
+	psDeviceNode->pfnMMUCacheInvalidate = RGXMMUCacheInvalidate;
+	psDeviceNode->pfnMMUCacheInvalidateKick = RGXMMUCacheInvalidateKick;
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = RGXMapBRN71422TargetPhysicalAddress;
+#else
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = NULL;
+#endif
+	/* pfnMMUTweakProtFlags is set later on once BVNC features are setup */
+	psDeviceNode->pfnMMUTweakProtFlags = NULL;
+
+	psDeviceNode->pfnInitDeviceCompatCheck	= &RGXDevInitCompatCheck;
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	/* Register callbacks for Unified Fence Objects */
+	psDeviceNode->pfnAllocUFOBlock = RGXAllocUFOBlock;
+	psDeviceNode->pfnFreeUFOBlock = RGXFreeUFOBlock;
+
+	/* Register callback for checking the device's health */
+	psDeviceNode->pfnUpdateHealthStatus = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? NULL : RGXUpdateHealthStatus;
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Register callback for updating the virtualization watchdog */
+	psDeviceNode->pfnUpdateAutoVzWatchdog = RGXUpdateAutoVzWatchdog;
+#endif
+
+	/* Register method to service the FW HWPerf buffer */
+	psDeviceNode->pfnServiceHWPerf = RGXHWPerfDataStoreCB;
+
+	/* Register callback for getting the device version information string */
+	psDeviceNode->pfnDeviceVersionString = RGXDevVersionString;
+
+	/* Register callback for getting the device clock speed */
+	psDeviceNode->pfnDeviceClockSpeed = RGXDevClockSpeed;
+
+	/* Register callback for soft resetting some device modules */
+	psDeviceNode->pfnSoftReset = RGXSoftReset;
+
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnResetHWRLogs = RGXResetHWRLogs;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnVerifyBVNC = RGXVerifyBVNC;
+
+	/* Register callback for checking alignment of UM structures */
+	psDeviceNode->pfnAlignmentCheck = RGXAlignmentCheck;
+
+	/*Register callback for checking the supported features and getting the
+	 * corresponding values */
+	psDeviceNode->pfnCheckDeviceFeature = RGXBvncCheckFeatureSupported;
+	psDeviceNode->pfnGetDeviceFeatureValue = RGXBvncGetSupportedFeatureValue;
+
+	/* Callback for checking if system layer supports FBC 3.1 */
+	psDeviceNode->pfnHasFBCDCVersion31 = RGXSystemHasFBCDCVersion31;
+
+	/* Callback for getting TFBC configuration */
+	psDeviceNode->pfnGetTFBCLossyGroup = RGXGetTFBCLossyGroup;
+
+	/* Register callback for initialising device-specific physical memory heaps */
+	psDeviceNode->pfnPhysMemDeviceHeapsInit = RGXPhysMemDeviceHeapsInit;
+
+	/* Register callback for checking a device's FW Main physical heap for sufficient free memory */
+	psDeviceNode->pfnCheckForSufficientFWPhysMem = RGXDeviceFWMainHeapMemCheck;
+
+	/*********************
+	 * Device info setup *
+	 *********************/
+	/* Allocate device control block */
+	psDevInfo = OSAllocZMem(sizeof(*psDevInfo));
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevInitRGXPart1 : Failed to alloc memory for DevInfo"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* initialise the layer parameters needed for early hw feature checks */
+	psDevInfo->sLayerParams.psDevInfo = psDevInfo;
+	psDevInfo->sLayerParams.psDevConfig = psDeviceNode->psDevConfig;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Default psTrampoline to point to null struct */
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+#endif
+
+	/* create locks for the context lists stored in the DevInfo structure.
+	 * these lists are modified on context create/destroy and read by the
+	 * watchdog thread
+	 */
+
+	eError = OSWRLockCreate(&(psDevInfo->hRenderCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create render context list lock", __func__));
+		goto e0;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hComputeCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create compute context list lock", __func__));
+		goto e1;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTransferCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create transfer context list lock", __func__));
+		goto e2;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTDMCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create TDM context list lock", __func__));
+		goto e3;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hKickSyncCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create kick sync context list lock", __func__));
+		goto e4;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hMemoryCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create memory context list lock", __func__));
+		goto e5;
+	}
+
+	eError = OSSpinLockCreate(&psDevInfo->hLockKCCBDeferredCommandsList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to KCCB deferred commands list lock", __func__));
+		goto e6;
+	}
+	dllist_init(&(psDevInfo->sKCCBDeferredCommandsListHead));
+
+	dllist_init(&(psDevInfo->sRenderCtxtListHead));
+	dllist_init(&(psDevInfo->sComputeCtxtListHead));
+	dllist_init(&(psDevInfo->sTDMCtxtListHead));
+	dllist_init(&(psDevInfo->sKickSyncCtxtListHead));
+
+	dllist_init(&(psDevInfo->sCommonCtxtListHead));
+	psDevInfo->ui32CommonCtxtCurrentID = 1;
+
+
+	eError = OSWRLockCreate(&psDevInfo->hCommonCtxtListLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create common context list lock", __func__));
+		goto e7;
+	}
+
+	eError = OSLockCreate(&psDevInfo->sRegConfig.hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create register configuration lock", __func__));
+		goto e8;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hBPLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for break points", __func__));
+		goto e9;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hRGXFWIfBufInitLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for trace buffers", __func__));
+		goto e10;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hCCBStallCheckLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB checking lock", __func__));
+		goto e11;
+	}
+	eError = OSLockCreate(&psDevInfo->hCCBRecoveryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB recovery lock", __func__));
+		goto e12;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hGpuUtilStatsLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create GPU stats lock", __func__));
+		goto e13;
+	}
+
+	dllist_init(&psDevInfo->sMemoryContextList);
+
+	/* initialise ui32SLRHoldoffCounter */
+	if (RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS > DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT)
+	{
+		psDevInfo->ui32SLRHoldoffCounter = RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS / DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	}
+	else
+	{
+		psDevInfo->ui32SLRHoldoffCounter = 0;
+	}
+
+	/* Setup static data and callbacks on the device specific device info */
+	psDevInfo->psDeviceNode		= psDeviceNode;
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+	psDevInfo->pvDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
+
+	/*
+	 * Map RGX Registers
+	 */
+	psDevInfo->ui32RegSize = psDeviceNode->psDevConfig->ui32RegsSize;
+	psDevInfo->sRegsPhysBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+#if !defined(NO_HARDWARE)
+	psDevInfo->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create RGX register mapping",
+		         __func__));
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto e14;
+	}
+#endif /* !NO_HARDWARE */
+
+	psDeviceNode->pvDevice = psDevInfo;
+
+	eError = RGXBvncInitialiseConfiguration(psDeviceNode);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Unsupported HW device detected by driver",
+		         __func__));
+		goto e15;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/*
+	 * We must now setup the SECURITY mappings if supported. We cannot
+	 * check on the features until we have reached here as the BVNC is
+	 * not setup before now.
+	 */
+#if defined(NO_HARDWARE) || defined(SUPPORT_TRUSTED_DEVICE)
+	/* secure register bank not accessible or unused */
+	psDevInfo->pvSecureRegsBaseKM = 0;
+#else
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+	{
+		IMG_CPU_PHYADDR sHostSecureRegBankBase = {psDeviceNode->psDevConfig->sRegsCpuPBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET};
+
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *) OSMapPhysToLin(sHostSecureRegBankBase,
+																		RGX_HOST_SECURE_REGBANK_SIZE,
+																		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (psDevInfo->pvSecureRegsBaseKM == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "PVRSRVRGXInitDevPart2KM: Failed to create RGX secure register mapping"));
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			goto e15;
+		}
+
+		/*
+		 * The secure register bank is mapped into the CPU VA space starting from
+		 * the base of the normal register bank + an offset of RGX_HOST_SECURE_REGBAK_OFFSET.
+		 * The hardware register addresses are all indexed from the base of the regular register bank.
+		 * For the RegBankBase+RegOffset computation to still be accurate for host-secure registers,
+		 * we need to compensate for offsets of registers in the secure bank
+		 */
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM - RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+	else
+	{
+		psDevInfo->pvSecureRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	}
+#endif /* defined(NO_HARDWARE) || defined(SUPPORT_TRUSTED_DEVICE) */
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	psDeviceNode->pfnMMUTweakProtFlags = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			RGXMMUTweakProtFlags : NULL;
+#endif
+
+	eError = RGXGetNon4KHeapPageShift(&psDevInfo->sLayerParams,
+									 &psDeviceNode->ui32Non4KPageSizeLog2);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXGetNon4KHeapPageSize", e16);
+
+	/* Configure MMU specific stuff */
+	RGXMMUInit_Register(psDeviceNode);
+
+	eError = RGXInitHeaps(psDevInfo, psDevMemoryInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto e16;
+	}
+
+	eError = RGXHWPerfInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInit", e16);
+
+	eError = RGXHWPerfHostInit(psDeviceNode->pvDevice, ui32HWPerfHostBufSizeKB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfHostInit", ErrorDeInitHWPerfFw);
+
+
+	/* Register callback for dumping debug info */
+	eError = RGXDebugInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDebugInit", e17);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Register callback for fw mmu init */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDeviceNode->pfnFwMMUInit = RGXMipsMMUInit_Register;
+	}
+#endif
+
+	/* Initialise the device dependent bridges */
+	eError = RGXRegisterBridges(psDevInfo);
+	PVR_LOG_IF_ERROR(eError, "RGXRegisterBridges");
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = OSLockCreate(&psDevInfo->hCounterDumpingLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for counter sampling.", __func__));
+		goto ErrorDeInitDeviceDepBridge;
+	}
+#endif
+
+	/* Initialise error counters */
+	memset(&psDevInfo->sErrorCounts, 0, sizeof(PVRSRV_RGXDEV_ERROR_COUNTS));
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+ErrorDeInitDeviceDepBridge:
+	RGXUnregisterBridges(psDevInfo);
+#endif
+
+e17:
+	RGXHWPerfHostDeInit(psDevInfo);
+ErrorDeInitHWPerfFw:
+	RGXHWPerfDeinit(psDevInfo);
+e16:
+#if !defined(NO_HARDWARE)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+		/* Adjust pvSecureRegsBaseKM if device has SECURITY_VERSION > 1 */
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		    (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* Undo the VA offset adjustment to unmap correct VAddr */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM,
+			                 RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+	}
+#endif
+#endif /* !NO_HARDWARE */
+e15:
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+		                 psDevInfo->ui32RegSize);
+	}
+e14:
+#endif /* !NO_HARDWARE */
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+e13:
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+e12:
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+e11:
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+e10:
+	OSLockDestroy(psDevInfo->hBPLock);
+e9:
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+e8:
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+e7:
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+e6:
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+e5:
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+e4:
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+e3:
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+e2:
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+e1:
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+e0:
+	OSFreeMem(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_PCHAR psz = psDevInfo->sDevFeatureCfg.pszBVNCString;
+	if (NULL == psz)
+	{
+		IMG_CHAR pszBVNCInfo[RGX_HWPERF_MAX_BVNC_LEN];
+		size_t uiBVNCStringSize;
+		size_t uiStringLength;
+
+		uiStringLength = OSSNPrintf(pszBVNCInfo, RGX_HWPERF_MAX_BVNC_LEN, "%d.%d.%d.%d",
+				psDevInfo->sDevFeatureCfg.ui32B,
+				psDevInfo->sDevFeatureCfg.ui32V,
+				psDevInfo->sDevFeatureCfg.ui32N,
+				psDevInfo->sDevFeatureCfg.ui32C);
+		PVR_ASSERT(uiStringLength < RGX_HWPERF_MAX_BVNC_LEN);
+
+		uiBVNCStringSize = (uiStringLength + 1) * sizeof(IMG_CHAR);
+		psz = OSAllocMem(uiBVNCStringSize);
+		if (NULL != psz)
+		{
+			OSCachedMemCopy(psz, pszBVNCInfo, uiBVNCStringSize);
+			psDevInfo->sDevFeatureCfg.pszBVNCString = psz;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Allocating memory for BVNC Info string failed",
+					__func__));
+		}
+	}
+
+	return psz;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDevVersionString
+@Description    Gets the version string for the given device node and returns
+                a pointer to it in ppszVersionString. It is then the
+                responsibility of the caller to free this memory.
+@Input          psDeviceNode            Device node from which to obtain the
+                                        version string
+@Output	        ppszVersionString	Contains the version string upon return
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR **ppszVersionString)
+{
+#if defined(NO_HARDWARE) || defined(EMULATOR)
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (SW)";
+#else
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (HW)";
+#endif
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_PCHAR pszBVNC;
+	size_t uiStringLength;
+
+	if (psDeviceNode == NULL || ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	pszBVNC = RGXDevBVNCString(psDevInfo);
+
+	if (NULL == pszBVNC)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	uiStringLength = OSStringLength(pszBVNC);
+	uiStringLength += (sizeof(szFormatString) - 2); /* sizeof includes the null, -2 for "%s" */
+	*ppszVersionString = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (*ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSSNPrintf(*ppszVersionString, uiStringLength, szFormatString,
+		pszBVNC);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       RGXDevClockSpeed
+@Description    Gets the clock speed for the given device node and returns
+                it in pui32RGXClockSpeed.
+@Input          psDeviceNode		Device node
+@Output         pui32RGXClockSpeed  Variable for storing the clock speed
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_PUINT32  pui32RGXClockSpeed)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* get clock speed */
+	*pui32RGXClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/*!
+ *******************************************************************************
+
+ @Function		RGXInitFwRawHeap
+
+ @Description	Called to perform additional initialisation
+ ******************************************************************************/
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID)
+{
+	IMG_UINT32 uiStringLength;
+	IMG_UINT32 uiStringLengthMax = 32;
+
+	IMG_UINT32 ui32Log2RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	PVR_RETURN_IF_FALSE(ui32Log2RgxDefaultPageShift != 0, PVRSRV_ERROR_INVALID_PARAMS);
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2RgxDefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2RgxDefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	uiStringLength = MIN(sizeof(RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT), uiStringLengthMax + 1);
+
+	/* Start by allocating memory for this DriverID heap identification string */
+	psDevMemHeap->pszName = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (psDevMemHeap->pszName == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Append the DriverID number to the RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT string */
+	OSSNPrintf((IMG_CHAR *)psDevMemHeap->pszName, uiStringLength, RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	/* Use the common blueprint template support function to initialise the heap */
+	HeapCfgBlueprintInit(psDevMemHeap->pszName,
+		                 RGX_FIRMWARE_RAW_HEAP_BASE + (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE),
+		                 RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                 0,
+		                 ui32Log2RgxDefaultPageShift,
+		                 0,
+		                 NULL,
+		                 NULL,
+		                 psDevMemHeap);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function		RGXDeInitFwRawHeap
+
+ @Description	Called to perform additional deinitialisation
+ ******************************************************************************/
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap)
+{
+	IMG_UINT64 uiBase = RGX_FIRMWARE_RAW_HEAP_BASE + RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT64 uiSpan = uiBase + ((RGX_NUM_DRIVERS_SUPPORTED - 1) * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	/* Safe to do as the guest firmware heaps are last in the list */
+	if (psDevMemHeap->sHeapBaseAddr.uiAddr >= uiBase &&
+	    psDevMemHeap->sHeapBaseAddr.uiAddr < uiSpan)
+	{
+		void *pszName = (void*)psDevMemHeap->pszName;
+		OSFreeMem(pszName);
+	}
+}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/******************************************************************************
+ End of file (rgxinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxlayer_impl.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxlayer_impl.c
new file mode 100644
index 000000000000..f95f500dd5b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxlayer_impl.c
@@ -0,0 +1,639 @@
+/*************************************************************************/ /*!
+@File
+@Title          DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxinit.h"
+#include "rgxfwimageutils.h"
+#include "cache_km.h"
+#include "km/rgxdefs_km.h"
+#include "rgx_heaps_server.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+
+#define MAX_NUM_COHERENCY_TESTS  (10)
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	if (psDevInfo->ui32CoherencyTestsDone >= MAX_NUM_COHERENCY_TESTS)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_FALSE;
+}
+
+/*
+ * The fabric coherency test is performed when platform supports fabric coherency
+ * either in the form of ACE-lite or Full-ACE. This test is done quite early
+ * with the firmware processor quiescent and makes exclusive use of the slave
+ * port interface for reading/writing through the device memory hierarchy. The
+ * rationale for the test is to ensure that what the CPU writes to its dcache
+ * is visible to the GPU via coherency snoop miss/hit and vice-versa without
+ * any intervening cache maintenance by the writing agent.
+ */
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 *pui32FabricCohTestBufferCpuVA = NULL;
+	IMG_UINT32 *pui32FabricCohCcTestBufferCpuVA = NULL;
+	IMG_UINT32 *pui32FabricCohNcTestBufferCpuVA = NULL;
+	DEVMEM_MEMDESC *psFabricCohTestBufferMemDesc = NULL;
+	DEVMEM_MEMDESC *psFabricCohCcTestBufferMemDesc = NULL;
+	DEVMEM_MEMDESC *psFabricCohNcTestBufferMemDesc = NULL;
+	RGXFWIF_DEV_VIRTADDR sFabricCohCcTestBufferDevVA;
+	RGXFWIF_DEV_VIRTADDR sFabricCohNcTestBufferDevVA;
+	RGXFWIF_DEV_VIRTADDR *psFabricCohTestBufferDevVA = NULL;
+	IMG_DEVMEM_SIZE_T uiFabricCohTestBlockSize = sizeof(IMG_UINT64);
+	IMG_DEVMEM_ALIGN_T uiFabricCohTestBlockAlign = sizeof(IMG_UINT64);
+	IMG_UINT64 ui64SegOutAddrTopCached = 0;
+	IMG_UINT64 ui64SegOutAddrTopUncached = 0;
+	IMG_UINT32 ui32OddEven;
+	IMG_UINT32 ui32OddEvenSeed = 1;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bFullTestPassed = IMG_TRUE;
+	IMG_BOOL bExit = IMG_FALSE;
+#if defined(DEBUG)
+	IMG_BOOL bSubTestPassed = IMG_FALSE;
+#endif
+	enum TEST_TYPE {
+		CPU_WRITE_GPU_READ_SM=0, GPU_WRITE_CPU_READ_SM,
+		CPU_WRITE_GPU_READ_SH,   GPU_WRITE_CPU_READ_SH
+	} eTestType;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	PVR_LOG(("Starting fabric coherency test ....."));
+
+	/* Size and align are 'expanded' because we request an export align allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiFabricCohTestBlockSize,
+	                                             &uiFabricCohTestBlockAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemExportalignAdjustSizeAndAlign() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	/* Allocate, acquire cpu address and set firmware address for cc=1 buffer */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyCcTestBuffer",
+										&psFabricCohCcTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohCcTestBufferMemDesc, (void **) &pui32FabricCohCcTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e1;
+	}
+
+	/* Create a FW address which is uncached in the Meta DCache and in the SLC using the Meta bootloader segment.
+	   This segment is the only one configured correctly out of reset (when this test is meant to be executed) */
+	eError = RGXSetFirmwareAddress(&sFabricCohCcTestBufferDevVA,
+						  psFabricCohCcTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", e2);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohCcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohCcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohCcTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohCcTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohCcTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	/* Allocate, acquire cpu address and set firmware address for cc=0 buffer */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyNcTestBuffer",
+										&psFabricCohNcTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e3;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohNcTestBufferMemDesc, (void **) &pui32FabricCohNcTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e4;
+	}
+
+	eError = RGXSetFirmwareAddress(&sFabricCohNcTestBufferDevVA,
+						  psFabricCohNcTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", e5);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohNcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohNcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohNcTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohNcTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohNcTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	/* Obtain the META segment addresses corresponding to cached and uncached windows into SLC */
+	ui64SegOutAddrTopCached   = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(MMU_CONTEXT_MAPPING_FWIF);
+	ui64SegOutAddrTopUncached = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED(MMU_CONTEXT_MAPPING_FWIF);
+
+	/* At the top level, we perform snoop-miss (i.e. to verify slave port) & snoop-hit (i.e. to verify ACE) test.
+	   NOTE: For now, skip snoop-miss test as Services currently forces all firmware allocations to be coherent */
+	for (eTestType = CPU_WRITE_GPU_READ_SH; eTestType <= GPU_WRITE_CPU_READ_SH && bExit == IMG_FALSE; eTestType++)
+	{
+		IMG_CPU_PHYADDR sCpuPhyAddr;
+		IMG_BOOL bValid;
+		PMR *psPMR;
+
+		if (eTestType == CPU_WRITE_GPU_READ_SM)
+		{
+			/* All snoop miss test must bypass the SLC, here memory is region of coherence so
+			   configure META to use SLC bypass cache policy for the bootloader segment. Note
+			   this cannot be done on a cache-coherent (i.e. CC=1) VA, as this violates ACE
+			   standard as one cannot issue a non-coherent request into the bus fabric for
+			   an allocation's VA that is cache-coherent in SLC, so use non-coherent buffer */
+			RGXWriteMetaRegThroughSP(hPrivate, META_CR_MMCU_SEGMENTn_OUTA1(6),
+									(ui64SegOutAddrTopUncached |  RGXFW_BOOTLDR_DEVV_ADDR) >> 32);
+			pui32FabricCohTestBufferCpuVA = pui32FabricCohNcTestBufferCpuVA;
+			psFabricCohTestBufferMemDesc = psFabricCohNcTestBufferMemDesc;
+			psFabricCohTestBufferDevVA = &sFabricCohNcTestBufferDevVA;
+		}
+		else if (eTestType == CPU_WRITE_GPU_READ_SH)
+		{
+			/* All snoop hit test must obviously use SLC, here SLC is region of coherence so
+			   configure META not to bypass the SLC for the bootloader segment */
+			RGXWriteMetaRegThroughSP(hPrivate, META_CR_MMCU_SEGMENTn_OUTA1(6),
+									(ui64SegOutAddrTopCached |  RGXFW_BOOTLDR_DEVV_ADDR) >> 32);
+			pui32FabricCohTestBufferCpuVA = pui32FabricCohCcTestBufferCpuVA;
+			psFabricCohTestBufferMemDesc = psFabricCohCcTestBufferMemDesc;
+			psFabricCohTestBufferDevVA = &sFabricCohCcTestBufferDevVA;
+		}
+
+		if (eTestType == GPU_WRITE_CPU_READ_SH)
+		{
+			/* Cannot perform this test if there is no snooping of device cache */
+			continue;
+		}
+
+		/* Acquire underlying PMR CpuPA in preparation for cache maintenance */
+		(void) DevmemLocalGetImportHandle(psFabricCohTestBufferMemDesc, (void**)&psPMR);
+		eError = PMR_CpuPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sCpuPhyAddr, &bValid, CPU_USE);
+		if (eError != PVRSRV_OK || bValid == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"PMR_CpuPhysAddr error: %s, exiting",
+					PVRSRVGetErrorString(eError)));
+			bExit = IMG_TRUE;
+			continue;
+		}
+
+		/* Here we do two passes mostly to account for the effects of using a different
+		   seed (i.e. ui32OddEvenSeed) value to read and write */
+		for (ui32OddEven = 1; ui32OddEven < 3 && bExit == IMG_FALSE; ui32OddEven++)
+		{
+			IMG_UINT32 i;
+
+#if defined(DEBUG)
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+
+			/* Do multiple sub-dword cache line tests */
+			for (i = 0; i < 2 && bExit == IMG_FALSE; i++)
+			{
+				IMG_UINT32 ui32FWAddr;
+				IMG_UINT32 ui32FWValue;
+				IMG_UINT32 ui32FWValue2;
+				IMG_UINT32 ui32LastFWValue = ~0;
+				IMG_UINT32 ui32Offset = i * sizeof(IMG_UINT32);
+
+				/* Calculate next address and seed value to write/read from slave-port */
+				ui32FWAddr = psFabricCohTestBufferDevVA->ui32Addr + ui32Offset;
+				ui32OddEvenSeed += 1;
+
+				if (eTestType == GPU_WRITE_CPU_READ_SM || eTestType == GPU_WRITE_CPU_READ_SH)
+				{
+					/* Clean dcache to ensure there is no stale data in dcache that might over-write
+					   what we are about to write via slave-port here because if it drains from the CPU
+					   dcache before we read it, it would corrupt what we are going to read back via
+					   the CPU */
+					CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_FLUSH);
+
+					/* Calculate a new value to write */
+					ui32FWValue = i + ui32OddEvenSeed;
+
+					/* Write the value using the RGX slave-port interface */
+					eError = RGXWriteFWModuleAddr(psDevInfo, ui32FWAddr, ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXWriteFWModuleAddr error: %s, exiting",
+						          PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Read back value using RGX slave-port interface, this is used
+					   as a sort of memory barrier for the above write */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue2);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXReadFWModuleAddr error: %s, exiting",
+						         PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+					else if (ui32FWValue != ui32FWValue2)
+					{
+						//IMG_UINT32 ui32FWValue3;
+						//RGXReadFWModuleAddr(psDevInfo, 0xC1F00000, &ui32FWValue3);
+
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW via SlavePort failed: expected: %x, got: %x",
+								i,
+								ui32FWValue,
+								ui32FWValue2));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Invalidate dcache to ensure that any prefetched data by the CPU from this memory
+					   region is discarded before we read (i.e. next read must trigger a cache miss).
+					   Previously there was snooping of device cache, where prefetching done by the CPU
+					   would reflect the most up to date datum writing by GPU into said location,
+					   that is to say prefetching was coherent so CPU d-flush was not needed */
+					CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_INVALIDATE);
+				}
+				else
+				{
+					IMG_UINT32 ui32RAWCpuValue;
+
+					/* Ensures line is in dcache */
+					ui32FWValue = IMG_UINT32_MAX;
+
+					/* Dirty allocation in dcache */
+					ui32RAWCpuValue = i + ui32OddEvenSeed;
+					pui32FabricCohTestBufferCpuVA[i] = i + ui32OddEvenSeed;
+
+					/* Flush possible cpu store-buffer(ing) on LMA */
+					OSWriteMemoryBarrier(&pui32FabricCohTestBufferCpuVA[i]);
+
+					if (eTestType == CPU_WRITE_GPU_READ_SM)
+					{
+						/* Flush dcache to force subsequent incoming CPU-bound snoop to miss so
+						   memory is coherent before the SlavePort reads */
+						CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_FLUSH);
+					}
+
+					/* Read back value using RGX slave-port interface */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"RGXReadFWModuleAddr error: %s, exiting",
+								PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Being mostly paranoid here, verify that CPU RAW operation is valid
+					   after the above slave port read */
+					CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_INVALIDATE);
+					if (pui32FabricCohTestBufferCpuVA[i] != ui32RAWCpuValue)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW by CPU failed: expected: %x, got: %x",
+								i,
+								ui32RAWCpuValue,
+								pui32FabricCohTestBufferCpuVA[i]));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				/* Compare to see if sub-test passed */
+				if (pui32FabricCohTestBufferCpuVA[i] == ui32FWValue)
+				{
+#if defined(DEBUG)
+					bSubTestPassed = IMG_TRUE;
+#endif
+				}
+				else
+				{
+					bFullTestPassed = IMG_FALSE;
+					eError = PVRSRV_ERROR_INIT_FAILURE;
+#if defined(DEBUG)
+					bSubTestPassed = IMG_FALSE;
+#endif
+					if (ui32LastFWValue != ui32FWValue)
+					{
+#if defined(DEBUG)
+						PVR_LOG(("At Offset: %d, Expected: %x, Got: %x",
+								 i,
+								 (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i],
+								 (eTestType & 0x1) ? pui32FabricCohTestBufferCpuVA[i] : ui32FWValue));
+#endif
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"test encountered unexpected error, exiting"));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				ui32LastFWValue = (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i];
+			}
+
+#if defined(DEBUG)
+			if (bExit)
+			{
+				continue;
+			}
+
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+		}
+	}
+
+	/* Release and free NC/CC test buffers */
+	RGXUnsetFirmwareAddress(psFabricCohCcTestBufferMemDesc);
+e5:
+	DevmemReleaseCpuVirtAddr(psFabricCohCcTestBufferMemDesc);
+e4:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohCcTestBufferMemDesc);
+
+e3:
+	RGXUnsetFirmwareAddress(psFabricCohNcTestBufferMemDesc);
+e2:
+	DevmemReleaseCpuVirtAddr(psFabricCohNcTestBufferMemDesc);
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohNcTestBufferMemDesc);
+
+e0:
+	/* Restore bootloader segment settings */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_MMCU_SEGMENTn_OUTA1(6),
+	                         (ui64SegOutAddrTopCached | RGXFW_BOOTLDR_DEVV_ADDR) >> 32);
+
+	bFullTestPassed = bExit ? IMG_FALSE: bFullTestPassed;
+	if (bFullTestPassed)
+	{
+		PVR_LOG(("fabric coherency test: PASSED"));
+		psDevInfo->ui32CoherencyTestsDone = MAX_NUM_COHERENCY_TESTS + 1;
+	}
+	else
+	{
+		PVR_LOG(("fabric coherency test: FAILED"));
+		psDevInfo->ui32CoherencyTestsDone++;
+	}
+
+	return eError;
+}
+
+static IMG_UINT64 RGXMMUComputeRangeValue(IMG_UINT32 ui32DataPageShift, IMG_UINT64 ui64BaseAddress, IMG_UINT64 ui64RangeSize)
+{
+	/* end address of range is inclusive */
+	IMG_UINT64 ui64EndAddress = ui64BaseAddress + ui64RangeSize - (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT);
+	IMG_UINT64 ui64RegValue = 0;
+
+	switch (ui32DataPageShift)
+	{
+		case RGX_HEAP_16KB_PAGE_SHIFT:
+			ui64RegValue = 1;
+			break;
+		case RGX_HEAP_64KB_PAGE_SHIFT:
+			ui64RegValue = 2;
+			break;
+		case RGX_HEAP_256KB_PAGE_SHIFT:
+			ui64RegValue = 3;
+			break;
+		case RGX_HEAP_1MB_PAGE_SHIFT:
+			ui64RegValue = 4;
+			break;
+		case RGX_HEAP_2MB_PAGE_SHIFT:
+			ui64RegValue = 5;
+			break;
+		case RGX_HEAP_4KB_PAGE_SHIFT:
+			/* fall through */
+		default:
+			/* anything we don't support, use 4K */
+			break;
+	}
+
+	/* check that the range is defined by valid 40 bit virtual addresses */
+	PVR_ASSERT((ui64BaseAddress & ~((1ULL << 40) - 1)) == 0);
+	PVR_ASSERT((ui64EndAddress  & ~((1ULL << 40) - 1)) == 0);
+
+	/* the range config register addresses are in 2MB chunks so check 21 lsb are zero */
+	PVR_ASSERT((ui64BaseAddress & ((1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT) - 1)) == 0);
+	PVR_ASSERT((ui64EndAddress  & ((1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT)  - 1)) == 0);
+
+	ui64BaseAddress >>= RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT;
+	ui64EndAddress  >>= RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT;
+
+	ui64RegValue = (ui64RegValue << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT) |
+				   (ui64EndAddress  << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT) |
+				   (ui64BaseAddress << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT);
+	return ui64RegValue;
+}
+
+IMG_UINT64 RGXMMUInitRangeValue(IMG_UINT32 ui32MMURange)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32Non4KHeapPageShift;
+	IMG_UINT32 ui32RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+	PVRSRV_ERROR eError;
+
+	switch (ui32MMURange)
+	{
+		case RGX_MMU_RANGE_GLOBAL:
+			/* set the last MMU config range covering the entire virtual memory to the OS's page size */
+			ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1ULL << 40));
+			break;
+		case RGX_MMU_RANGE_NON4KHEAP:
+			/*
+			 * If the Non4K heap has a different page size than the OS's page size
+			 * (used as default for all other heaps), configure one MMU config range
+			 * for the Non4K heap
+			 */
+			eError = RGXGetNon4KHeapPageShift(NULL, &ui32Non4KHeapPageShift);
+			PVR_LOG_IF_ERROR(eError, "RGXGetNon4KHeapPageShift");
+
+			if (eError == PVRSRV_OK)
+			{
+				if (ui32Non4KHeapPageShift != ui32RgxDefaultPageShift)
+				{
+					ui64RegVal = RGXMMUComputeRangeValue(ui32Non4KHeapPageShift, RGX_GENERAL_NON4K_HEAP_BASE, RGX_GENERAL_NON4K_HEAP_SIZE);
+				}
+				else
+				{
+					ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+				}
+			}
+			else
+			{
+				/*
+				 * Error from Non4K Heap shift. Default to the normal PageShift
+				 * to attempt to continue.
+				 */
+				ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+			}
+			break;
+		default:
+			ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+			break;
+	}
+
+	return ui64RegVal;
+}
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxmulticore.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxmulticore.c
new file mode 100644
index 000000000000..0c8bc6747805
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxmulticore.c
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.c
+@Title          Functions related to multicore devices
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxdefs_km.h"
+#include "pdump_km.h"
+#include "rgxmulticore.h"
+#include "multicore_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwmemctx.h"
+
+
+
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores,
+                                        IMG_UINT64 *pui64Caps);
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Return multicore information to clients.
+ * Return not supported on cores without multicore.
+ */
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32CapsSize,
+                                 IMG_UINT32 *pui32NumCores,
+                                 IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDevInfo->ui32MultiCoreNumCores == 0)
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+	else
+	{
+		*pui32NumCores = psDevInfo->ui32MultiCoreNumCores;
+		if (ui32CapsSize > 0)
+		{
+			if (ui32CapsSize < psDevInfo->ui32MultiCoreNumCores)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore caps buffer too small"));
+				eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			}
+			else
+			{
+				IMG_UINT32 i;
+
+				for (i = 0; i < psDevInfo->ui32MultiCoreNumCores; ++i)
+				{
+					pui64Caps[i] = psDevInfo->pui64MultiCoreCapabilities[i];
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Read multicore HW registers and fill in data structure for clients.
+ * Return not supported on cores without multicore.
+ */
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		/* we only set this up once */
+		return PVRSRV_OK;
+	}
+
+	/* defaults for non-multicore devices */
+	psDevInfo->ui32MultiCoreNumCores = 0;
+	psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	psDevInfo->pui64MultiCoreCapabilities = NULL;
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		IMG_BOOL bPowerWasDown;
+		IMG_UINT32 ui32MulticoreInfo;
+		IMG_UINT32 ui32PrimaryCoreIds;
+		IMG_UINT32 ui32PrimaryId;
+		IMG_UINT32 ui32NumCores;
+		IMG_UINT32 i;
+		IMG_UINT32 ui32CoresFoundInDomain = 0;
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(XPU_MAX_REGBANKS_ADDR_WIDTH)
+		IMG_UINT32 ui32MulticoreRegBankOffset = (1 << RGX_GET_FEATURE_VALUE(psDevInfo, XPU_MAX_REGBANKS_ADDR_WIDTH));
+
+		/* Ensure the HOST_SECURITY reg bank definitions are correct */
+		if ((RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE) != ui32MulticoreRegBankOffset)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Register bank definitions for HOST_SECURITY don't match core's configuration.", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+#endif
+		bPowerWasDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			PVRSRVPowerLock(psDeviceNode);
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVSetSystemPowerState ON failed (%u)", __func__, eError));
+				PVRSRVPowerUnlock(psDeviceNode);
+				return eError;
+			}
+		}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			RGXFWIF_MULTICORE_INFO *psRGXMulticoreInfo;
+			IMG_UINT32 ui32FwTimeout = MAX_HW_TIME_US;
+
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores,
+		                               INVALIDATE);
+				if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores))
+				{
+					/* No need to wait if the FW has already updated the values */
+					break;
+				}
+				OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores) == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore info not available for guest"));
+				return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+			}
+
+			psRGXMulticoreInfo = &psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo;
+			ui32NumCores = psRGXMulticoreInfo->ui32NumCores;
+			ui32MulticoreInfo = psRGXMulticoreInfo->ui32MulticoreInfo;
+
+			PVR_LOG(("RGX Guest Device initialised with %u %s",
+					 ui32NumCores, (ui32NumCores == 1U) ? "core" : "cores"));
+		}
+		else
+#endif
+		{
+			ui32NumCores = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_DOMAIN)
+		                                            & ~RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_CLRMSK)
+		                                            >> RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_SHIFT;
+			ui32MulticoreInfo = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE);
+		}
+#if defined(NO_HARDWARE)
+		/* override to defaults if no hardware */
+		ui32NumCores = RGX_MULTICORE_MAX_NOHW_CORES;
+		ui32MulticoreInfo = 0;  /* primary id 0 with 7 secondaries */
+#endif
+		/* ID for this primary is in this register */
+		ui32PrimaryId = (ui32MulticoreInfo & ~RGX_CR_MULTICORE_ID_CLRMSK) >> RGX_CR_MULTICORE_ID_SHIFT;
+
+		/* allocate storage for capabilities */
+		psDevInfo->pui64MultiCoreCapabilities = OSAllocMem(ui32NumCores * sizeof(psDevInfo->pui64MultiCoreCapabilities[0]));
+		if (psDevInfo->pui64MultiCoreCapabilities == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to alloc memory for Multicore info", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+
+		ui32PrimaryCoreIds = (ui32MulticoreInfo & ~RGX_CR_MULTICORE_PRIMARY_CORE_ID_CLRMSK)
+		                                        >> RGX_CR_MULTICORE_PRIMARY_CORE_ID_SHIFT;
+
+		psDevInfo->ui32MultiCorePrimaryId = ui32PrimaryId;
+		psDevInfo->ui32MultiCoreNumCores = ui32NumCores;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Multicore domain has %d cores with primary id %u\n", ui32NumCores, ui32PrimaryId));
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGX Multicore domain has %d cores with primary id %u\n",
+		             ui32NumCores, ui32PrimaryId);
+		for (i = 0; i < RGX_MULTICORE_MAX_NOHW_CORES; i++)
+		{
+			if ((ui32PrimaryCoreIds & 0x7) == ui32PrimaryId)
+			{
+				if (ui32CoresFoundInDomain >= ui32NumCores)
+				{
+					/* Enough cores have already been found in the domain, but there is an additional match.
+					   This is an illegal combination. */
+					PVR_ASSERT(ui32CoresFoundInDomain < ui32NumCores);
+					break;
+				}
+
+				/* currently all cores are identical so have the same capabilities */
+				psDevInfo->pui64MultiCoreCapabilities[ui32CoresFoundInDomain] = i
+				                    | ((i == ui32PrimaryId) ? RGX_MULTICORE_CAPABILITY_PRIMARY_EN : 0)
+				                    | RGX_MULTICORE_CAPABILITY_GEOMETRY_EN
+				                    | RGX_MULTICORE_CAPABILITY_COMPUTE_EN
+				                    | RGX_MULTICORE_CAPABILITY_FRAGMENT_EN;
+				PDUMPCOMMENT(psDeviceNode, "\tCore %u has caps 0x%08x", i,
+				             (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[ui32CoresFoundInDomain]);
+				PVR_DPF((PVR_DBG_MESSAGE, "Core %u has caps 0x%08x", i, (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[ui32CoresFoundInDomain]));
+				ui32CoresFoundInDomain++;
+			}
+			ui32PrimaryCoreIds >>= 3;
+		}
+
+		PVR_ASSERT(ui32CoresFoundInDomain == ui32NumCores);
+
+        /* revert power state to what it was on entry to this function */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Register callback to return info about multicore setup to client bridge */
+		psDeviceNode->pfnGetMultiCoreInfo = RGXGetMultiCoreInfo;
+	}
+	else
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	return eError;
+}
+
+
+/*
+ * RGXDeinitMultiCoreInfo:
+ * Release resources and clear the MultiCore values in the DeviceNode.
+ */
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pui64MultiCoreCapabilities != NULL)
+	{
+		OSFreeMem(psDevInfo->pui64MultiCoreCapabilities);
+		psDevInfo->pui64MultiCoreCapabilities = NULL;
+		psDevInfo->ui32MultiCoreNumCores = 0;
+		psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	}
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxpdump.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxpdump.c
new file mode 100644
index 000000000000..a844e17e9b97
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxpdump.c
@@ -0,0 +1,768 @@
+/*************************************************************************/ /*!
+@File           rgxpdump.c
+@Title          Device specific pdump routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(PDUMP)
+#include "pvrsrv.h"
+#include "devicemem_utils.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+#include "rgxpdump.h"
+#include "rgxpdump_common.h"
+#include "pdumpdesc.h"
+
+/*
+ * There are two different set of functions one for META/RISCV and one for MIPS
+ * because the Pdump player does not implement the support for
+ * the MIPS MMU yet. So for MIPS builds we cannot use DevmemPDumpSaveToFileVirtual,
+ * we have to use DevmemPDumpSaveToFile instead.
+ */
+static PVRSRV_ERROR _FWDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DECLARE_DLLIST(sFirmwareGcovBufferValidRegions);
+#endif
+	DECLARE_DLLIST(sSigTDMChecksValidRegions);
+	DECLARE_DLLIST(sSigTAChecksValidRegions);
+	DECLARE_DLLIST(sSig3DChecksValidRegions);
+	DECLARE_DLLIST(sSigCDMChecksValidRegions);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DECLARE_DLLIST(sSigRDMChecksValidRegions);
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psFirmwareGcovBufferMemDesc,
+	                             psDevInfo->ui32FirmwareGcovSize,
+	                             &sFirmwareGcovBufferValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrReturnError);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                             psDevInfo->ui32SigTDMChecksSize,
+		                             &sSigTDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeGcovBufferRegions);
+	}
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                             psDevInfo->ui32SigTAChecksSize,
+	                             &sSigTAChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTDMChecksRegions);
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                             psDevInfo->ui32Sig3DChecksSize,
+	                             &sSig3DChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTAChecksRegions);
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSigCDMChecksMemDesc,
+	                             psDevInfo->ui32SigCDMChecksSize,
+	                             &sSigCDMChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSig3DChecksRegions);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+	    RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                             psDevInfo->ui32SigRDMChecksSize,
+		                             &sSigRDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigCDMChecksRegions);
+	}
+#endif
+
+
+	PDUMPIF(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+	PDUMPELSE(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Gcov */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Gcov Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psFirmwareGcovBufferMemDesc,
+	                       &sFirmwareGcovBufferValidRegions,
+	                       "firmware_gcov.img",
+	                       ui32PDumpFlags);
+#endif
+
+	/* TDM signatures */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                       &sSigTDMChecksValidRegions,
+		                       "out.tdmsig",
+		                       ui32PDumpFlags);
+	}
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                       &sSigTAChecksValidRegions,
+	                       "out.tasig",
+	                       ui32PDumpFlags);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                       &sSig3DChecksValidRegions,
+	                       "out.3dsig",
+	                       ui32PDumpFlags);
+
+	/* CDM signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump CDM signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSigCDMChecksMemDesc,
+	                       &sSigCDMChecksValidRegions,
+	                       "out.cdmsig",
+	                       ui32PDumpFlags);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		/* RDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump RDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                       &sSigRDMChecksValidRegions,
+		                       "out.rdmsig",
+		                       ui32PDumpFlags);
+	}
+#endif
+
+	PDUMPFI(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+ErrFreeSigCDMChecksRegions:
+#endif
+	DevmemIntPDumpFreeValidRegions(&sSigCDMChecksValidRegions);
+ErrFreeSig3DChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSig3DChecksValidRegions);
+ErrFreeSigTAChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTAChecksValidRegions);
+ErrFreeSigTDMChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTDMChecksValidRegions);
+ErrFreeGcovBufferRegions:
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DevmemIntPDumpFreeValidRegions(&sFirmwareGcovBufferValidRegions);
+ErrReturnError:
+#endif
+	return eError;
+}
+
+static PVRSRV_ERROR _FWDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32	ui32ThreadNum, ui32Size, ui32OutFileOffset;
+
+	const IMG_CHAR acOutTraceFileName[] = "out.trace";
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDUMPIF(psDeviceNode, "ENABLE_TRACEBUF", ui32PDumpFlags);
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufThreadNumOff =
+				(IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+		                                 uiTraceBufThreadNumOff,
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+		                                 offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+		                                 0, /* 0 offset in the trace buffer mem desc */
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+		                                 /* move to first element of sTraceBuf */
+		                                 offsetof(RGXFWIF_TRACEBUF, sTraceBuf)
+		                                     /* skip required number of sTraceBuf elements */
+		                                     + ui32ThreadNum * sizeof(RGXFWIF_TRACEBUF_SPACE)
+		                                     /* offset into its sAssertBuf, to be pdumped */
+		                                     + offsetof(RGXFWIF_TRACEBUF_SPACE, sAssertBuf),
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+	}
+	PDUMPFI(psDeviceNode, "ENABLE_TRACEBUF", ui32PDumpFlags);
+
+	/* FW HWPerf buffer is always allocated when PDUMP is defined, irrespective of HWPerf events being enabled/disabled */
+	PVR_ASSERT(psDevInfo->psRGXFWIfHWPerfBufMemDesc);
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _MipsDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTAChecksMemDesc,
+								 0,
+								 psDevInfo->ui32SigTAChecksSize,
+								 "out.tasig",
+								 0);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSig3DChecksMemDesc,
+								 0,
+								 psDevInfo->ui32Sig3DChecksSize,
+								 "out.3dsig",
+								 0);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* TDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTDMChecksMemDesc,
+									 0,
+									 psDevInfo->ui32SigTDMChecksSize,
+									 "out.tdmsig",
+									 0);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MipsDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32		ui32ThreadNum, ui32Size, ui32OutFileOffset;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* Same again... */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufSpaceAssertBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF_SPACE *)0)->sAssertBuf);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+								0, /* 0 offset in the trace buffer mem desc */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff + uiTraceBufSpaceAssertBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+	}
+
+	/* Dump hwperf buffer */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+								 0,
+								 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+								 "out.hwperf",
+								 0);
+
+	return PVRSRV_OK;
+}
+#endif
+
+
+/*
+ * PVRSRVPDumpSignatureBufferKM
+ */
+PVRSRV_ERROR PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_UINT32			ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpSignatureBufferKM(psConnection,
+											  psDeviceNode,
+											  ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpSignatureBufferKM(psConnection,
+											psDeviceNode,
+											ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR PVRSRVPDumpCRCSignatureCheckKM(CONNECTION_DATA * psConnection,
+                                            PVRSRV_DEVICE_NODE * psDeviceNode,
+                                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+/*
+ * PVRSRVPDumpValCheckPreCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPreCommandKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVPDumpValCheckPostCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPostCommandKM(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE * psDeviceNode,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR PVRSRVPDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputImageDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_PBYTE pbyPDumpImageHdr)
+{
+	IMG_PUINT32 pui32Word;
+	IMG_UINT32 ui32HeaderDataSize;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK) || defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK) || defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	IMG_UINT32 ui32TFBCControl = (psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >>
+																					  RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+#endif
+
+	/* Validate parameters */
+	if (((IMAGE_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((IMAGE_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	memset(pbyPDumpImageHdr, 0, IMAGE_HEADER_SIZE);
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpImageHdr, 0);
+	pui32Word[0] = (IMAGE_HEADER_TYPE << HEADER_WORD0_TYPE_SHIFT);
+	pui32Word[1] = (IMAGE_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+				   (IMAGE_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+
+	ui32HeaderDataSize = ui32DataSize;
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		ui32HeaderDataSize += ui32HeaderSize;
+	}
+	pui32Word[2] = ui32HeaderDataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+	pui32Word[3] = ui32LogicalWidth << IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT;
+	pui32Word[4] = ui32LogicalHeight << IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT;
+
+	pui32Word[5] = ePixFmt << IMAGE_HEADER_WORD5_FORMAT_SHIFT;
+
+	pui32Word[6] = ui32PhysicalWidth << IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT;
+	pui32Word[7] = ui32PhysicalHeight << IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT;
+
+	pui32Word[8] = IMAGE_HEADER_WORD8_STRIDE_POSITIVE | IMAGE_HEADER_WORD8_BIFTYPE_NONE;
+
+	switch (eMemLayout)
+	{
+	case IMG_MEMLAYOUT_STRIDED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_STRIDED;
+		break;
+	case IMG_MEMLAYOUT_TWIDDLED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_ZTWIDDLE;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported memory layout - %d", eMemLayout));
+		return PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT;
+	}
+
+	switch (eFBCompression)
+	{
+	case IMG_FB_COMPRESSION_NONE:
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_PACKED_8x8:
+	case IMG_FB_COMPRESSION_DIRECT_8x8:
+	case IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_8X8;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_16x4:
+	case IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_16x4;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_32x2:
+	case IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2:
+		/* Services Client guards against unsupported FEATURE_FB_CDC_32x2.
+		   We should never pass through the UM|KM bridge on cores lacking the feature.
+		*/
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_32x2;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported compression mode - %d", eFBCompression));
+		return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+	}
+
+	pui32Word[9] = 0;
+
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		if ((RGX_GET_FEATURE_VALUE(psDevInfo, FBCDC) == 4) || (RGX_GET_FEATURE_VALUE(psDevInfo, FBCDC) == 5))
+		{
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V4;
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_ON;
+			}
+
+			pui32Word[9] |= (eFBCSwizzle << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT) & IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK;
+		}
+		else /* 3 or 3.1 */
+		{
+			IMG_BOOL bIsFBC31 = psDevInfo->psRGXFWIfFwSysData->
+					ui32ConfigFlags & RGXFWIF_INICFG_FBCDC_V3_1_EN;
+
+			if (bIsFBC31)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT2;
+			}
+			else
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2;
+			}
+		}
+
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+	}
+
+	pui32Word[10] = paui32FBCClearColour[0];
+	pui32Word[11] = paui32FBCClearColour[1];
+	pui32Word[12] = (IMG_UINT32) (psDeviceNode->ui64FBCClearColour & 0xFFFFFFFF);
+	pui32Word[13] = (IMG_UINT32) (psDeviceNode->ui64FBCClearColour >> 32);
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP */
+		IMG_UINT32 ui32TFBCGroup  = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK;
+		switch (ui32TFBCGroup)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50;
+				break;
+		}
+	}
+	else
+#endif
+	{
+		pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+	}
+
+#if defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME */
+		IMG_UINT32 ui32TFBCScheme = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK;
+		switch (ui32TFBCScheme)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_ALL;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "Unsupported TFBC compression control scheme - %d", ui32TFBCScheme));
+				return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+		}
+	}
+	else
+#endif
+	{
+		/* Should always be set to 2 ("TFBC delta standard only") on cores without this feature */
+		pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+	}
+
+#if defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_NATIVE_YUV10))
+	{
+		IMG_UINT32 ui32TFBCOverrideYUV10 = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN);
+
+		if (ui32TFBCOverrideYUV10)
+		{
+			pui32Word[14] |= IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN;
+		}
+	}
+#endif
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, TFBC_VERSION) >= 20U)
+		{
+			IMG_UINT32 ui32TFBCOverrideLossyMinChannel = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN);
+
+			if (ui32TFBCOverrideLossyMinChannel)
+			{
+				pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+			}
+		}
+		else
+		{
+			/* Should be set on TFBC version < 2.0 cores */
+			pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputDataDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_PBYTE pbyPDumpDataHdr)
+{
+	IMG_PUINT32 pui32Word;
+
+	/* Validate parameters */
+	if (((DATA_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((DATA_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpDataHdr, 0);
+
+	if (ui32HeaderType == DATA_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (DATA_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(DATA_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+		pui32Word[3] = ui32ElementType << DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT;
+		pui32Word[4] = ui32ElementCount << DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT;
+	}
+
+	if (ui32HeaderType == IBIN_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (IBIN_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(IBIN_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/******************************************************************************
+ End of file (rgxpdump.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.c
new file mode 100644
index 000000000000..fb5c757d5d80
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.c
@@ -0,0 +1,894 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Ray routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Ray routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvkm.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxray.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_memallocflags.h"
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#include "rgxworkest_ray.h"
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_RAY_UFO_DUMP	0
+
+//#define RAY_CHECKPOINT_DEBUG 1
+
+#if defined(RAY_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+struct _RGX_SERVER_RAY_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRayContextMemDesc;
+	POS_LOCK					hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	SYNC_ADDR_LIST				sSyncAddrListFence;
+	SYNC_ADDR_LIST				sSyncAddrListUpdate;
+};
+
+PVRSRV_ERROR PVRSRVRGXCreateRayContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE		*psDeviceNode,
+											 IMG_INT32				i32Priority,
+											 IMG_HANDLE				hMemCtxPrivData,
+											 IMG_UINT32				ui32ContextFlags,
+											 IMG_UINT32				ui32StaticRayContextStateSize,
+											 IMG_PBYTE				pStaticRayContextState,
+											 IMG_UINT64				ui64RobustnessAddress,
+											 IMG_UINT32				ui32MaxDeadlineMS,
+											 RGX_SERVER_RAY_CONTEXT	**ppsRayContext)
+{
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RAY_CONTEXT		*psRayContext;
+	RGXFWIF_FWRAYCONTEXT		*psFWRayContext;
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	PVRSRV_ERROR				eError;
+
+	*ppsRayContext = NULL;
+
+	psRayContext = OSAllocZMem(sizeof(*psRayContext));
+	if (psRayContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRayContext->psDeviceNode = psDeviceNode;
+	/*
+		Create the FW ray context, this has the RDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWRAYCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwRayContext",
+			&psRayContext->psFWRayContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwraycontext;
+	}
+
+	eError = OSLockCreate(&psRayContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_RAY,
+									 RGXFWIF_DM_RAY,
+									 hMemCtxPrivData,
+									 psRayContext->psFWRayContextMemDesc,
+									 offsetof(RGXFWIF_FWRAYCONTEXT, sRDMContext),
+									 psFWMemContextMemDesc,
+									 NULL,
+									 RGX_RDM_CCB_SIZE_LOG2,
+									 RGX_RDM_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 i32Priority,
+									 ui32MaxDeadlineMS,
+									 ui64RobustnessAddress,
+									 &sInfo,
+									 &psRayContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init Ray fw common context (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_raycommoncontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRayContext->psFWRayContextMemDesc,
+			(void **)&psFWRayContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitRay(psDevInfo, &psRayContext->sWorkEstData);
+	}
+#endif
+
+	OSDeviceMemCopy(&psFWRayContext->sStaticRayContextState, pStaticRayContextState, ui32StaticRayContextStateSize);
+	DevmemPDumpLoadMem(psRayContext->psFWRayContextMemDesc, 0, sizeof(RGXFWIF_FWRAYCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	DevmemReleaseCpuVirtAddr(psRayContext->psFWRayContextMemDesc);
+
+	SyncAddrListInit(&psRayContext->sSyncAddrListFence);
+	SyncAddrListInit(&psRayContext->sSyncAddrListUpdate);
+
+	*ppsRayContext = psRayContext;
+
+	return PVRSRV_OK;
+fail_acquire_cpu_mapping:
+fail_raycommoncontext:
+	OSLockDestroy(psRayContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psRayContext->psFWRayContextMemDesc);
+fail_fwraycontext:
+	OSFreeMem(psRayContext);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyRayContextKM(RGX_SERVER_RAY_CONTEXT *psRayContext)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psRayContext->psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_FWRAYCONTEXT	*psFWRayContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRayContext->psFWRayContextMemDesc,
+				(void **)&psFWRayContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware ray context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+		ui32WorkEstCCBSubmitted = psFWRayContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRayContext->psFWRayContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRayContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRayContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psRayContext->psDeviceNode,
+											  psRayContext->psServerCommonContext,
+											  RGXFWIF_DM_RAY,
+											  PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psRayContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psRayContext->psServerCommonContext);
+
+	psRayContext->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		WorkEstDeInitRay(psDevInfo, &psRayContext->sWorkEstData);
+	}
+#endif
+
+	DevmemFwUnmapAndFree(psDevInfo, psRayContext->psFWRayContextMemDesc);
+
+	OSLockDestroy(psRayContext->hLock);
+	OSFreeMem(psRayContext);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXKickRDMKM(RGX_SERVER_RAY_CONTEXT	*psRayContext,
+								IMG_UINT32				ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK	**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32				*paui32ClientUpdateSyncOffset,
+								IMG_UINT32				*paui32ClientUpdateValue,
+								PVRSRV_FENCE			iCheckFence,
+								PVRSRV_TIMELINE			iUpdateTimeline,
+								PVRSRV_FENCE			*piUpdateFence,
+								IMG_CHAR				pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								IMG_UINT32				ui32CmdSize,
+								IMG_PBYTE				pui8DMCmd,
+								IMG_UINT32				ui32PDumpFlags,
+								IMG_UINT32				ui32ExtJobRef,
+								IMG_UINT32				ui32AccStructSizeInBytes,
+								IMG_UINT32				ui32DispatchSize,
+								IMG_UINT64				ui64DeadlineInus)
+{
+
+	RGXFWIF_KCCB_CMD		sRayKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	PVRSRV_ERROR			eError, eError2;
+	IMG_UINT32				ui32FWCtx;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+	RGX_CLIENT_CCB          *psClientCCB;
+	IMG_UINT32              ui32IntJobRef;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataRay = {0};
+	IMG_UINT32 ui32RDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32RDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32RDMCmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui32RDMCmdOffset = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_BOOL				bCCBStateOpen = IMG_FALSE;
+	IMG_UINT64 ui64FBSCEntryMask;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	psDevInfo = FWCommonContextGetRGXDevInfo(psRayContext->psServerCommonContext);
+	psClientCCB = FWCommonContextGetClientCCB(psRayContext->psServerCommonContext);
+	ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRayContext->hLock);
+
+	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	ui32IntClientUpdateCount = ui32ClientUpdateCount;
+
+	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListUpdate,
+									ui32ClientUpdateCount,
+									pauiClientUpdateUFODevVarBlock,
+									paui32ClientUpdateSyncOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount && !pauiIntUpdateUFOAddress)
+	{
+		pauiIntUpdateUFOAddress = psRayContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psRayContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psRayContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psRayContext->psDeviceNode->hSyncCheckpointContext,
+										iCheckFence,
+										&ui32FenceSyncCheckpointCount,
+										&apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)", __func__, eError));
+		goto fail_resolve_input_fence;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(RAY_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psRayContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psRayContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psRayContext->psDeviceNode,
+		                                   pcszUpdateFenceName,
+										   iUpdateTimeline,
+										   psRayContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%d)", __func__, eError));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+#if defined(RAY_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the ray context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the ray context update list */
+			SyncAddrListAppendSyncPrim(&psRayContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+#if defined(RAY_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Ray RDM Fence (&psRayContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psRayContext->sSyncAddrListFence));
+#if defined(RAY_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+		SyncAddrListAppendCheckpoints(&psRayContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psRayContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+#if defined(RAY_CHECKPOINT_DEBUG)
+	if (ui32IntClientUpdateCount > 0)
+	{
+		IMG_UINT32 iii;
+		IMG_UINT32 *pui32Tmp = (IMG_UINT32*)paui32IntUpdateValue;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Dumping %d update values (paui32IntUpdateValue=<%p>)...", __func__, ui32IntClientUpdateCount, (void*)paui32IntUpdateValue));
+		for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: paui32IntUpdateValue[%d] = <%p>", __func__, iii, (void*)pui32Tmp));
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: *paui32IntUpdateValue[%d] = 0x%x", __func__, iii, *pui32Tmp));
+			pui32Tmp++;
+		}
+	}
+#endif
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Ray RDM Update (&psRayContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psRayContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psRayContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psRayContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(RAY_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_RAY_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping Ray (RDM) fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Ray (RDM) fence syncs (&psRayContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psRayContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Ray (RDM) update syncs (&psRayContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psRayContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRayContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/*
+	* Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	* in other words, take the value and set it to zero afterwards.
+	* FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	* as it must be ready at the time of context activation.
+	*/
+	{
+		eError = RGXExtractFBSCEntryMaskFromMMUContext(psRayContext->psDeviceNode,
+														FWCommonContextGetServerMMUCtx(psRayContext->psServerCommonContext),
+														&ui64FBSCEntryMask);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+			goto fail_invalfbsc;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		sWorkloadCharacteristics.sRay.ui32AccStructSize = ui32AccStructSizeInBytes;
+		sWorkloadCharacteristics.sRay.ui32DispatchSize = ui32DispatchSize;
+
+		/* Prepare workload estimation */
+		WorkEstPrepare(psRayContext->psDeviceNode->pvDevice,
+				&psRayContext->sWorkEstData,
+				&psRayContext->sWorkEstData.uWorkloadMatchingData.sRay.sDataRDM,
+				RGXFWIF_CCB_CMD_TYPE_RAY,
+				&sWorkloadCharacteristics,
+				ui64DeadlineInus,
+				&sWorkloadKickDataRay);
+
+		if (sWorkloadKickDataRay.ui32CyclesPrediction != 0)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Dispatch size = %u, Acc struct size = %u, prediction = %u",
+					__func__,
+					sWorkloadCharacteristics.sRay.ui32DispatchSize,
+					sWorkloadCharacteristics.sRay.ui32AccStructSize,
+					sWorkloadKickDataRay.ui32CyclesPrediction));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32AccStructSizeInBytes);
+	PVR_UNREFERENCED_PARAMETER(ui32DispatchSize);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       ui64FBSCEntryMask,
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+                           &pPreAddr,
+                           &pPostAddr,
+                           &pRMWUFOAddr,
+	                       RGXFWIF_CCB_CMD_TYPE_RAY,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	                       &sWorkloadKickDataRay,
+#else
+	                       NULL,
+#endif
+	                       "Ray",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+		{
+			ui32RDMCmdOffset = RGXGetHostWriteOffsetCCB(psClientCCB);
+		}
+#endif
+		/*
+			All the required resources are ready at this point, we can't fail so
+			take the required server sync operations and commit all the resources
+		*/
+		RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "RDM", FWCommonContextGetFWAddress(psRayContext->psServerCommonContext).ui32Addr);
+	}
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* The following is used to determine the offset of the command header containing
+		 * the workload estimation data so that can be accessed when the KCCB is read */
+		ui32RDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(asCmdHelperData);
+
+		ui32RDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRayContext->psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32RDMCmdOffset < ui32RDMCmdOffsetWrapCheck)
+		{
+			ui32RDMWorkloadDataRO = ui32RDMCmdOffset;
+		}
+		else
+		{
+			ui32RDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/* Construct the kernel ray CCB command. */
+	sRayKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sRayKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRayContext->psServerCommonContext);
+	sRayKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sRayKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sRayKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Store the offset to the CCCB command header so that it can be referenced
+		 * when the KCCB command reaches the FW */
+		sRayKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32RDMWorkloadDataRO + ui32RDMCmdHeaderOffset;
+	}
+#endif
+	ui32FWCtx = FWCommonContextGetFWAddress(psRayContext->psServerCommonContext).ui32Addr;
+
+	RGXSRV_HWPERF_ENQ(psRayContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_RS,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	/*
+	 * Submit the ray command to the firmware.
+	 */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError2 = RGXScheduleCommandWithoutPowerLock(psRayContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_RAY,
+									&sRayKCCBCmd,
+									ui32PDumpFlags);
+		if (eError2 != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError2 != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError2)));
+		if (eError == PVRSRV_OK)
+		{
+			eError = eError2;
+		}
+	}
+	else
+	{
+		PVRGpuTraceEnqueueEvent(psRayContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_RS);
+	}
+	/*
+	 * Now check eError (which may have returned an error from our earlier call
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdsubmit;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__, (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif /* defined(NO_HARDWARE) */
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRayContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pcszUpdateFenceName);
+	}
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psRayContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_cmdsubmit:
+fail_acquirepowerlock:
+fail_cmdacquire:
+fail_invalfbsc:
+	SyncAddrListRollbackCheckpoints(psRayContext->psDeviceNode, &psRayContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psRayContext->psDeviceNode, &psRayContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+fail_resolve_input_fence:
+
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psRayContext->hLock);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxray.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.h
new file mode 100644
index 000000000000..ecf23db03a46
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.h
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX ray functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX compute functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXRAY_H_)
+#define RGXRAY_H_
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxfwutils.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+
+
+typedef struct _RGX_SERVER_RAY_CONTEXT_ RGX_SERVER_RAY_CONTEXT;
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXCreateRayContextKM
+
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRayContextKM(CONNECTION_DATA		*psConnection,
+										 PVRSRV_DEVICE_NODE		*psDeviceNode,
+										 IMG_INT32				i32Priority,
+										 IMG_HANDLE				hMemCtxPrivData,
+										 IMG_UINT32				ui32ContextFlags,
+										 IMG_UINT32				ui32StaticRayContextStateSize,
+										 IMG_PBYTE				pStaticRayContextState,
+										 IMG_UINT64				ui64RobustnessAddress,
+										 IMG_UINT32				ui32MaxDeadlineMS,
+										 RGX_SERVER_RAY_CONTEXT	**ppsRayContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDestroyRayContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRayContext
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRayContextKM(RGX_SERVER_RAY_CONTEXT *psRayContext);
+
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXKickRDMKM
+
+ @Description
+	Server-side implementation of RGXKickRDM
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickRDMKM(RGX_SERVER_RAY_CONTEXT	*psRayContext,
+								IMG_UINT32				ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK	**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32				*paui32ClientUpdateSyncOffset,
+								IMG_UINT32				*paui32ClientUpdateValue,
+								PVRSRV_FENCE			iCheckFence,
+								PVRSRV_TIMELINE			iUpdateTimeline,
+								PVRSRV_FENCE			*piUpdateFence,
+								IMG_CHAR				pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								IMG_UINT32				ui32CmdSize,
+								IMG_PBYTE				pui8DMCmd,
+								IMG_UINT32				ui32PDumpFlags,
+								IMG_UINT32				ui32ExtJobRef,
+								IMG_UINT32				ui32AccStructSizeInBytes,
+								IMG_UINT32				ui32DispatchSize,
+								IMG_UINT64				ui64DeadlineInus);
+
+
+#endif /* RGXRAY_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxsrvinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxsrvinit.c
new file mode 100644
index 000000000000..709cbfd1002e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxsrvinit.c
@@ -0,0 +1,1690 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvinit.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "km_apphint_defs.h"
+#include "htbuffer_types.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "rgx_compat_bvnc.h"
+
+#include "rgxdefs_km.h"
+#include "pvrsrv.h"
+
+#include "rgxinit_apphints.h"
+#include "rgxinit.h"
+#include "rgxmulticore.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "osfunc.h"
+
+#include "rgxdefs_km.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "rgx_fwif_hwperf.h"
+#include "rgxhwperf_common.h"
+
+#include "fwload.h"
+#include "rgxlayer_impl.h"
+#include "rgxfwimageutils.h"
+#include "rgxfwutils.h"
+
+#include "rgx_bvnc_defs_km.h"
+
+#include "rgxdevice.h"
+
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#include "rgxdevice.h"
+#include "pvrsrv_device.h"
+#endif
+
+#define	HW_PERF_FILTER_DEFAULT         0x00000000 /* Default to no HWPerf */
+#define HW_PERF_FILTER_DEFAULT_ALL_ON  0xFFFFFFFF /* All events */
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+#define AVAIL_POW_UNITS_MASK_DEFAULT   (PVRSRV_APPHINT_HWVALAVAILABLESPUMASK)
+#define AVAIL_RAC_MASK_DEFAULT         (PVRSRV_APPHINT_HWVALAVAILABLERACMASK)
+#endif
+
+/* Kernel CCB size */
+
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE 4
+#endif
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE 16
+#endif
+
+#if PVRSRV_APPHINT_KCCB_SIZE_LOG2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too low.
+#elif PVRSRV_APPHINT_KCCB_SIZE_LOG2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too high.
+#endif
+
+
+#include "os_apphint.h"
+
+/*
+ * _ParseHTBAppHints:
+ *
+ * Generate necessary references to the globally visible AppHints which are
+ * declared in the above #include "km_apphint_defs.h"
+ * Without these local references some compiler tool-chains will treat
+ * unreferenced declarations as fatal errors. This function duplicates the
+ * HTB_specific apphint references which are made in htbserver.c:HTBInit()
+ * However, it makes absolutely *NO* use of these hints.
+ */
+static void
+_ParseHTBAppHints(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvAppHintState)
+{
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32OpMode;
+	IMG_UINT32 ui32BufferSize;
+
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEHTBLOGGROUP;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, EnableHTBLogGroup,
+	                     &ui32AppHintDefault, &ui32LogType);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBOPERATIONMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBOperationMode,
+	                     &ui32AppHintDefault, &ui32OpMode);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBUFFERSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+	                     &ui32AppHintDefault, &ui32BufferSize);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetFilterFlags
+
+ @Description   Initialise and return filter flags
+
+ @Input         bFilteringMode : Enable new TPU filtering mode
+ @Input         ui32TruncateMode : TPU Truncate mode
+
+ @Return        IMG_UINT32 : Filter flags
+
+******************************************************************************/
+static INLINE IMG_UINT32 GetFilterFlags(IMG_BOOL bFilteringMode, IMG_UINT32 ui32TruncateMode)
+{
+	IMG_UINT32 ui32FilterFlags = 0;
+
+	ui32FilterFlags |= bFilteringMode ? RGXFWIF_FILTCFG_NEW_FILTER_MODE : 0;
+	if (ui32TruncateMode == 2)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_INT;
+	}
+	else if (ui32TruncateMode == 3)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_HALF;
+	}
+
+	return ui32FilterFlags;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      InitDeviceFlags
+
+ @Description   Initialise and return device flags
+
+ @Input         psDeviceNode     : Pointer to device node
+ @Input         pvAppHintState   : Pointer to apphint state
+ @Input         psHints          : Apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void InitDeviceFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   void *pvAppHintState,
+                                   RGX_INIT_APPHINTS *psHints)
+{
+	IMG_UINT32 ui32DeviceFlags = 0;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bZeroFreelist;
+	IMG_BOOL bDisableFEDLogging;
+
+
+	bAppHintDefault = PVRSRV_APPHINT_ZEROFREELIST;
+	OSGetAppHintBOOL(psDeviceNode,  pvAppHintState,  ZeroFreelist,
+	                   &bAppHintDefault,         &bZeroFreelist);
+	ui32DeviceFlags |= bZeroFreelist ? RGXKM_DEVICE_STATE_ZERO_FREELIST : 0;
+
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEFEDLOGGING;
+	OSGetAppHintBOOL(psDeviceNode,    pvAppHintState,    DisableFEDLogging,
+	                   &bAppHintDefault,        &bDisableFEDLogging);
+	ui32DeviceFlags |= bDisableFEDLogging ? RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN : 0;
+
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	BITMASK_SET(ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+#endif
+
+	psHints->ui32DeviceFlags = ui32DeviceFlags;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetApphints
+
+ @Description   Read init time apphints and initialise apphints structure
+
+ @Input         psHints : Pointer to apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetApphints(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_INIT_APPHINTS *psHints)
+{
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+	IMG_UINT32 ui32ParamTemp;
+#if defined(__linux__)
+	IMG_UINT64 ui64AppHintDefault;
+#endif
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESIGNATURECHECKS;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSignatureChecks,
+	                     &bAppHintDefault,    &psHints->bEnableSignatureChecks);
+	ui32AppHintDefault = PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    SignatureChecksBufSize,
+	                     &ui32AppHintDefault,    &psHints->ui32SignatureChecksBufSize);
+
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTOUTOFMEMORY;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOutOfMemory,
+	                   &bAppHintDefault,        &psHints->bAssertOnOutOfMem);
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTONHWRTRIGGER;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOnHWRTrigger,
+	                   &bAppHintDefault,        &psHints->bAssertOnHWRTrigger);
+	bAppHintDefault = PVRSRV_APPHINT_CHECKMLIST;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    CheckMList,
+	                   &bAppHintDefault,        &psHints->bCheckMlist);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLECLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableClockGating,
+	                   &bAppHintDefault,        &psHints->bDisableClockGating);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEDMOVERLAP;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableDMOverlap,
+	                   &bAppHintDefault,        &psHints->bDisableDMOverlap);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEAPM;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    EnableAPM,
+	                     &ui32AppHintDefault,      &ui32ParamTemp);
+	psHints->eRGXActivePMConf = ui32ParamTemp;
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEDMKILLINGRANDMODE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableGenericDMKillingRandMode,
+	                   &bAppHintDefault,           &psHints->bEnableDMKillRand);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableRandomContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableRandomCsw);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSoftResetContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableSoftResetCsw);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableFWContextSwitch,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableFWContextSwitch);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLERDPOWERISLAND;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableRDPowerIsland,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eRGXRDPowerIslandConf = ui32ParamTemp;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESPUCLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableSPUClockGating,
+	                   &bAppHintDefault,           &psHints->bSPUClockGating);
+#endif
+	ui32AppHintDefault = PVRSRV_APPHINT_FIRMWAREPERF;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FirmwarePerf,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eFirmwarePerf = ui32ParamTemp;
+	ui32AppHintDefault = PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FWContextSwitchProfile,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchProfile);
+
+	/*
+	 * HWPerf apphints *
+	 */
+	bAppHintDefault = PVRSRV_APPHINT_HWPERFDISABLECOUNTERFILTER;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    HWPerfDisableCounterFilter,
+	                     &bAppHintDefault,    &psHints->bHWPerfDisableCounterFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfHostBufSize);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfFWBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfFWBufSize);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_KCCB_SIZE_LOG2;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,  KernelCCBSizeLog2,
+	                     &ui32AppHintDefault,    &psHints->ui32KCCBSizeLog2);
+
+	if (psHints->ui32KCCBSizeLog2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too low, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE;
+	}
+	else if (psHints->ui32KCCBSizeLog2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too high, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE;
+	}
+
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AppHintDefault = PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ISPSchedulingLatencyMode,
+	                     &ui32AppHintDefault,  &psHints->ui32ISPSchedulingLatencyMode);
+#endif
+
+#if defined(__linux__)
+	/* name changes */
+	{
+		IMG_UINT64 ui64Tmp;
+		bAppHintDefault = PVRSRV_APPHINT_DISABLEPDUMPPANIC;
+		OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,  DisablePDumpPanic,
+		                   &bAppHintDefault,           &psHints->bDisablePDP);
+		ui64AppHintDefault = PVRSRV_APPHINT_HWPERFFWFILTER;
+		OSGetAppHintUINT64(psDevInfo->psDeviceNode,  pvAppHintState,  HWPerfFWFilter,
+		                   &ui64AppHintDefault,        &ui64Tmp);
+		psHints->ui32HWPerfFilter0 = (IMG_UINT32)(ui64Tmp & 0xffffffffllu);
+		psHints->ui32HWPerfFilter1 = (IMG_UINT32)((ui64Tmp >> 32) & 0xffffffffllu);
+	}
+#endif
+
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTFILTER;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWPerfHostFilter,
+	                     &ui32AppHintDefault,          &psHints->ui32HWPerfHostFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    SecondaryOSClockSource,
+	                         &ui32AppHintDefault,      &psHints->ui32SecondaryOSClockSource);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWRDebugDumpLimit,
+	                     &ui32AppHintDefault,          &ui32ParamTemp);
+	psHints->ui32HWRDebugDumpLimit = MIN(ui32ParamTemp, RGXFWIF_HWR_DEBUG_DUMP_ALL);
+
+	{
+		IMG_BOOL bFilteringMode = IMG_FALSE;
+		IMG_UINT32 ui32TruncateMode = 0U;
+
+#if defined(HW_ERN_42290_BIT_MASK) && defined(RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42290) && RGX_IS_FEATURE_SUPPORTED(psDevInfo, TPU_FILTERING_MODE_CONTROL))
+#endif
+		{
+			bAppHintDefault = PVRSRV_APPHINT_NEWFILTERINGMODE;
+			OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  NewFilteringMode,
+			                   &bAppHintDefault,  &bFilteringMode);
+		}
+
+#if defined(HW_ERN_42606_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42606))
+#endif
+		{
+			ui32AppHintDefault = PVRSRV_APPHINT_TRUNCATEMODE;
+			OSGetAppHintUINT32(APPHINT_NO_DEVICE,  pvAppHintState,  TruncateMode,
+			                     &ui32AppHintDefault,  &ui32TruncateMode);
+		}
+
+		psHints->ui32FilterFlags = GetFilterFlags(bFilteringMode, ui32TruncateMode);
+	}
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#endif
+	{
+		bAppHintDefault = PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG;
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  EnableTrustedDeviceAceConfig,
+		                   &bAppHintDefault,     &psHints->bEnableTrustedDeviceAceConfig);
+	}
+
+#if defined(__linux__)
+	ui32AppHintDefault = 0;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  FWContextSwitchCrossDM,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchCrossDM);
+#endif
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	ui32AppHintDefault = PVRSRV_APPHINT_PHYSMEMTESTPASSES;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  PhysMemTestPasses,
+	                     &ui32AppHintDefault,    &psHints->ui32PhysMemTestPasses);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	psHints->ui32AvailablePowUnitsMask = AVAIL_POW_UNITS_MASK_DEFAULT;
+	psHints->ui32AvailableRACMask = AVAIL_RAC_MASK_DEFAULT;
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	ui32AppHintDefault = PVRSRV_APPHINT_TFBCVERSION;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, TFBCVersionDowngrade,
+	 &ui32AppHintDefault, &psHints->ui32TFBCVersion);
+
+	if (ui32AppHintDefault != psHints->ui32TFBCVersion)
+	{
+		PVR_LOG(("TFBCVersionDowngrade set to %u", psHints->ui32TFBCVersion));
+	}
+
+	psHints->bTFBCCompressionControlLossyMinChannel = false;
+	psHints->bTFBCCompressionControlYUVFormat = false;
+	psHints->ui32TFBCCompressionControlScheme =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME;
+	psHints->ui32TFBCCompressionControlGroup =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	ui32AppHintDefault = PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  DebugDumpFWTLogType,
+	                     &ui32AppHintDefault,  &psHints->eDebugDumpFWTLogType);
+	if ((IMG_UINT32)psHints->eDebugDumpFWTLogType > RGX_FWT_LOGTYPE_PARTIAL)
+	{
+		psHints->eDebugDumpFWTLogType = RGX_FWT_LOGTYPE_NONE;
+		PVR_DPF((PVR_DBG_WARNING, "Invalid value for DebugDumpFWTLogType. Setting to 0 (disabled)."));
+	}
+#if defined(SUPPORT_ICS)
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableIdleCycleStealing,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableIdleCycleStealing);
+	ui32AppHintDefault = PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  FaultDetectionTimeInterval, &ui32AppHintDefault, &psHints->ui32FDTI);
+	ui32AppHintDefault = PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ICSTimeIntervalThreshold, &ui32AppHintDefault, &psHints->ui32ICSThreshold);
+	bAppHintDefault = IMG_FALSE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  ICSTestModeOn, &bAppHintDefault, &psHints->bTestModeOn);
+#endif
+
+	/*
+	 * FW logs apphints
+	 */
+	{
+		IMG_UINT32 ui32LogGroup, ui32TraceOrTBI;
+
+		ui32AppHintDefault = PVRSRV_APPHINT_ENABLELOGGROUP;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, EnableLogGroup, &ui32AppHintDefault, &ui32LogGroup);
+		ui32AppHintDefault = PVRSRV_APPHINT_FIRMWARELOGTYPE;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, FirmwareLogType, &ui32AppHintDefault, &ui32TraceOrTBI);
+
+		/* Defaulting to TRACE */
+		BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+		if (ui32TraceOrTBI == 1 /* TBI */)
+		{
+			if ((ui32LogGroup & RGXFWIF_LOG_TYPE_GROUP_MASK) == 0)
+			{
+				/* No groups configured - defaulting to MAIN group */
+				BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_GROUP_MAIN);
+			}
+			BITMASK_UNSET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+		}
+#endif
+		psHints->ui32LogType = ui32LogGroup;
+	}
+
+	_ParseHTBAppHints(psDevInfo->psDeviceNode, pvAppHintState);
+
+	InitDeviceFlags(psDevInfo->psDeviceNode, pvAppHintState, psHints);
+
+	OSFreeAppHintState(pvAppHintState);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      GetFWConfigFlags
+
+ @Description   Initialise and return FW config flags
+
+ @Input         psHints            : Apphints container
+ @Input         pui32FWConfigFlags : Pointer to config flags
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetFWConfigFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    RGX_INIT_APPHINTS *psHints,
+                                    IMG_UINT32 *pui32FWConfigFlags,
+                                    IMG_UINT32 *pui32FWConfigFlagsExt,
+                                    IMG_UINT32 *pui32FwOsCfgFlags)
+{
+	IMG_UINT32 ui32FWConfigFlags = 0;
+	IMG_UINT32 ui32FWConfigFlagsExt = 0;
+	IMG_UINT32 ui32FwOsCfgFlags = psHints->ui32FWContextSwitchCrossDM |
+	                              (psHints->ui32EnableFWContextSwitch & ~RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK)
+#if defined(SUPPORT_ICS)
+	                              | ((psHints->ui32EnableIdleCycleStealing << RGXFWIF_INICFG_OS_ICS_DM_ALL_SHIFT) &
+	                              ~RGXFWIF_INICFG_OS_ICS_CLRMSK)
+#endif
+;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32FWConfigFlagsSupValExt = 0;
+	IMG_UINT32 ui32TFBCVersion = 0U;
+#endif
+
+	ui32FWConfigFlags |= psHints->bAssertOnOutOfMem ? RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY : 0;
+	ui32FWConfigFlags |= psHints->bAssertOnHWRTrigger ? RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER : 0;
+	ui32FWConfigFlags |= psHints->bCheckMlist ? RGXFWIF_INICFG_CHECK_MLIST_EN : 0;
+	ui32FWConfigFlags |= psHints->bDisableClockGating ? RGXFWIF_INICFG_DISABLE_CLKGATING_EN : 0;
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0))
+	{
+		if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, ERYX_TOP_INFRASTRUCTURE))
+		{
+			if ((!RGX_IS_FEATURE_SUPPORTED(psDevInfo, DIVANO_TOP_INFRASTRUCTURE)) ||
+				(RGX_GET_FEATURE_VALUE(psDevInfo, ECC_RAMS) == 0))
+			{
+				/* Pipeline DM roadblocks are currently enabled pre-DXS. */
+				ui32FWConfigFlags |= RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN;
+			}
+		}
+	}
+#endif
+
+	ui32FWConfigFlags |= psHints->bDisableDMOverlap ? RGXFWIF_INICFG_DISABLE_DM_OVERLAP : 0;
+	ui32FWConfigFlags |= psHints->bDisablePDP ? RGXFWIF_INICFG_DISABLE_PDP_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableDMKillRand ? RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableRandomCsw ? RGXFWIF_INICFG_CTXSWITCH_MODE_RAND : 0;
+	ui32FWConfigFlags |= psHints->bEnableSoftResetCsw ? RGXFWIF_INICFG_CTXSWITCH_SRESET_EN : 0;
+	ui32FWConfigFlags |= (psHints->ui32HWPerfFilter0 != 0 || psHints->ui32HWPerfFilter1 != 0) ? RGXFWIF_INICFG_HWPERF_EN : 0;
+	ui32FWConfigFlags |= psHints->bHWPerfDisableCounterFilter ? RGXFWIF_INICFG_HWP_DISABLE_FILTER : 0;
+	ui32FWConfigFlags |= (psHints->ui32FWContextSwitchProfile << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT) & RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK;
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32FWConfigFlags |= (psHints->ui32ISPSchedulingLatencyMode << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT) & RGXFWIF_INICFG_ISPSCHEDMODE_MASK;
+#endif
+
+	{
+		ui32FWConfigFlags |= psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode) ? RGXFWIF_INICFG_FBCDC_V3_1_EN : 0;
+	}
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+
+	/* Determine if we need to present a TFBC v1.0, v1.1 or native
+	 * behaviour. For V1.0 we need to set the following features:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x1
+	 *  TFBCCompressionControlScheme = 0x2
+	 *  TFBCCompressionControlGroup = 0x0
+	 * For V1.1 we need to set the following:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x0
+	 *  TFBCCompressionControlScheme = 0x1
+	 *  TFBCCompressionControlGroup = 0 / 1 (depends on LOSSY_37_PERCENT)
+	 * The gating for these values depends on whether the GPU supports
+	 * RGX_FEATURE_TFBC_VERSION = 20U
+	 */
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		ui32TFBCVersion = RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION);
+
+		if (ui32TFBCVersion >= 20U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 11:	/* TFBC Version 1.1 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = false;
+					psHints->ui32TFBCCompressionControlScheme = 1U;
+					psHints->ui32TFBCCompressionControlGroup =
+					    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.1, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+					         " Ignoring. Using value %u instead",
+					         __func__, psHints->ui32TFBCVersion,
+					         ui32TFBCVersion));
+					break;
+			}
+
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else if (ui32TFBCVersion == 11U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+				             " Ignoring. Using value %u instead",
+				             __func__, psHints->ui32TFBCVersion,
+				             ui32TFBCVersion));
+					break;
+			}
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else	/* TFBC v1.0 */
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32FWConfigFlagsSupValExt);
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = 0;
+#endif
+			if ((psHints->ui32TFBCVersion != 0U) &&
+			    (psHints->ui32TFBCVersion != ui32TFBCVersion))
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s: Cannot specify TFBC version %u"
+				         " on a version %u GPU core", __func__,
+				         psHints->ui32TFBCVersion, ui32TFBCVersion));
+			}
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	if ((ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN) &&
+		((ui32FWConfigFlags & RGXFWIF_INICFG_ISPSCHEDMODE_MASK) == RGXFWIF_INICFG_ISPSCHEDMODE_NONE))
+	{
+		ui32FwOsCfgFlags &= ~RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN;
+		PVR_DPF((PVR_DBG_WARNING, "ISPSchedulingLatencyMode=0 implies context switching is inoperable on DM_3D.\n"
+				 "Overriding current value EnableFWContextSwitch=0x%x with new value 0x%x",
+				 psHints->ui32EnableFWContextSwitch,
+				 ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL));
+	}
+
+	*pui32FWConfigFlags    = ui32FWConfigFlags;
+	*pui32FWConfigFlagsExt = ui32FWConfigFlagsExt;
+	*pui32FwOsCfgFlags     = ui32FwOsCfgFlags;
+}
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+/*!
+*******************************************************************************
+
+ @Function      RGXTDProcessFWImage
+
+ @Description   Fetch and send data used by the trusted device to complete
+                the FW image setup
+
+ @Input         psDeviceNode : Device node
+ @Input         psFWParams : Firmware and parameters used by the FW
+
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXTDProcessFWImage(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        PVRSRV_FW_PARAMS *psFWParams)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+	PVRSRV_ERROR eError;
+
+	if (psDevConfig->pfnTDSendFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TDSendFWImage not implemented!", __func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		if (psFWParams->uFWP.sMips.ui32FWPageTableNumPages > TD_MAX_NUM_MIPS_PAGETABLE_PAGES)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Number of page table pages %u greater "
+					 "than what is allowed by the TD interface (%u), FW might "
+					 "not work properly!", __func__,
+					 psFWParams->uFWP.sMips.ui32FWPageTableNumPages,
+					 TD_MAX_NUM_MIPS_PAGETABLE_PAGES));
+		}
+	}
+#endif
+
+	eError = psDevConfig->pfnTDSendFWImage(psDevConfig->hSysData, psFWParams);
+
+	return eError;
+}
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXAcquireMipsBootldrData
+
+ @Description   Acquire MIPS bootloader data parameters
+
+ @Input         psDeviceNode : Device node
+ @Input         puFWParams   : FW boot parameters
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireMipsBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+	MMU_DEVICEATTRIBS *psFWMMUDevAttrs = psDevInfo->psDeviceNode->psFirmwareMMUDevAttrs;
+	IMG_DEV_PHYADDR sAddr;
+	IMG_UINT32 ui32PTSize, i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bValid;
+
+	/* Rogue Registers physical address */
+#if defined(SUPPORT_ALT_REGBASE)
+	puFWParams->sMips.sGPURegAddr = psDeviceNode->psDevConfig->sAltRegsGpuPBase;
+#else
+	PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+	                           1,
+	                           &puFWParams->sMips.sGPURegAddr,
+	                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+#endif
+
+	/* MIPS Page Table physical address */
+	MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sAddr);
+
+	/* MIPS Page Table allocation is contiguous. Pass one or more addresses
+	 * to the FW depending on the Page Table size and alignment. */
+
+	ui32PTSize = (psFWMMUDevAttrs->psTopLevelDevVAddrConfig->uiNumEntriesPT)
+	             << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	ui32PTSize = PVR_ALIGN(ui32PTSize, 1U << psFWMMUDevAttrs->ui32BaseAlign);
+
+	puFWParams->sMips.ui32FWPageTableLog2PageSize = psFWMMUDevAttrs->ui32BaseAlign;
+	puFWParams->sMips.ui32FWPageTableNumPages = ui32PTSize >> psFWMMUDevAttrs->ui32BaseAlign;
+
+	if (puFWParams->sMips.ui32FWPageTableNumPages > 4U)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Page table cannot be mapped by the FW "
+		         "(size 0x%x, log2 page size %u, %u pages)",
+		         __func__, ui32PTSize, puFWParams->sMips.ui32FWPageTableLog2PageSize,
+		         puFWParams->sMips.ui32FWPageTableNumPages));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Confirm page alignment fits in 64-bits */
+	if (psFWMMUDevAttrs->ui32BaseAlign > 63)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid page alignment "
+		         "(psFWMMUDevAttrs->ui32BaseAlign = %u)",
+		         __func__, psFWMMUDevAttrs->ui32BaseAlign));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	for (i = 0; i < puFWParams->sMips.ui32FWPageTableNumPages; i++)
+	{
+		puFWParams->sMips.asFWPageTableAddr[i].uiAddr =
+		    sAddr.uiAddr + i * (1ULL << psFWMMUDevAttrs->ui32BaseAlign);
+	}
+
+	/* MIPS Stack Pointer Physical Address */
+	eError = RGXGetPhyAddr(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+	                       &puFWParams->sMips.sFWStackAddr,
+	                       RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+	                       OSGetPageShift(),
+	                       1,
+	                       &bValid);
+
+	return eError;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      InitFirmware
+
+ @Description   Allocate, initialise and pdump Firmware code and data memory
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitFirmware(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	OS_FW_IMAGE       *psRGXFW = NULL;
+	PVRSRV_FW_PARAMS  sFWParams = {0};
+
+	/* FW code memory */
+	IMG_DEVMEM_SIZE_T uiFWCodeAllocSize;
+	void              *pvFWCodeHostAddr;
+
+	/* FW data memory */
+	IMG_DEVMEM_SIZE_T uiFWDataAllocSize;
+	void              *pvFWDataHostAddr;
+
+	/* FW coremem code memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememCodeAllocSize;
+	void              *pvFWCorememCodeHostAddr = NULL;
+
+	/* FW coremem data memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememDataAllocSize;
+	void              *pvFWCorememDataHostAddr = NULL;
+
+	PVRSRV_FW_BOOT_PARAMS uFWParams;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_BOOL bUseSecureFWData =
+								RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	                            (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+	                             RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32) ||
+#endif
+	                            RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR);
+#endif
+
+	/*
+	 * Get pointer to Firmware image
+	 */
+	eError = RGXLoadAndGetFWData(psDeviceNode, &psRGXFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Error or confirmation message generated in RGXLoadAndGetFWData */
+		goto fw_load_fail;
+	}
+
+	/*
+	 * Get pointer and size
+	 */
+	sFWParams.pvFirmware       = OSFirmwareData(psRGXFW);
+	sFWParams.ui32FirmwareSize = OSFirmwareSize(psRGXFW);
+
+	/*
+	 * Allow it to be pre-processed by the platform hook
+	 */
+	if (psDeviceNode->psDevConfig->pfnPrepareFWImage != NULL)
+	{
+		eError = psDeviceNode->psDevConfig->pfnPrepareFWImage(
+				psDeviceNode->psDevConfig->hSysData, &sFWParams);
+		if (eError != PVRSRV_OK)
+			goto cleanup_initfw;
+	}
+
+	/*
+	 * Allocate Firmware memory
+	 */
+
+	eError = RGXGetFWImageAllocSize(&psDevInfo->sLayerParams,
+	                                sFWParams.pvFirmware,
+	                                sFWParams.ui32FirmwareSize,
+	                                &uiFWCodeAllocSize,
+	                                &uiFWDataAllocSize,
+	                                &uiFWCorememCodeAllocSize,
+	                                &uiFWCorememDataAllocSize,
+	                                &psDevInfo->sFWInfoHeader);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXGetFWImageAllocSize failed",
+		        __func__));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Initiate FW compatibility check for Native and Host.
+	 * Guest compatibility check must be done after FW boot.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed compatibility check for device %p (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+	}
+
+	psDevInfo->ui32FWCodeSizeInBytes = uiFWCodeAllocSize;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s: META DMA not available, disabling core memory code/data",
+		        __func__));
+		uiFWCorememCodeAllocSize = 0;
+		uiFWCorememDataAllocSize = 0;
+	}
+#endif
+
+	psDevInfo->ui32FWCorememCodeSizeInBytes = uiFWCorememCodeAllocSize;
+
+	eError = RGXInitAllocFWImgMem(psDeviceNode,
+	                              uiFWCodeAllocSize,
+	                              uiFWDataAllocSize,
+	                              uiFWCorememCodeAllocSize,
+	                              uiFWCorememDataAllocSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXInitAllocFWImgMem failed (%d)",
+		        __func__,
+		        eError));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Acquire pointers to Firmware allocations
+	 */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCodeHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, &pvFWCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", cleanup_initfw);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWDataHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc, &pvFWDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_code);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCorememCodeHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, &pvFWCorememCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_data);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		pvFWCorememDataHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememDataAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc, &pvFWCorememDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_corememcode);
+	}
+
+	/*
+	 * Prepare FW boot parameters
+	 */
+	OSCachedMemSet(&uFWParams, 0, sizeof(PVRSRV_FW_BOOT_PARAMS));
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		uFWParams.sMeta.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sMeta.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeFWAddr = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sMeta.uiFWCorememCodeSize = uiFWCorememCodeAllocSize;
+		uFWParams.sMeta.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sMeta.sFWCorememDataFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+		uFWParams.sMeta.ui32NumThreads = 2;
+#else
+		uFWParams.sMeta.ui32NumThreads = 1;
+#endif
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = RGXAcquireMipsBootldrData(psDeviceNode, &uFWParams);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXAcquireMipsBootldrData failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+	}
+#endif
+	else
+	{
+		uFWParams.sRISCV.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+
+		uFWParams.sRISCV.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememCodeFWAddr   = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sRISCV.uiFWCorememCodeSize    = uiFWCorememCodeAllocSize;
+
+		uFWParams.sRISCV.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememDataFWAddr   = psDevInfo->sFWCorememDataStoreFWAddr;
+		uFWParams.sRISCV.uiFWCorememDataSize    = uiFWCorememDataAllocSize;
+	}
+
+
+	/*
+	 * On Volcanic the TEE handles the loading of all Firmware sections.
+	 */
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = RGXProcessFWImage(&psDevInfo->sLayerParams,
+								   sFWParams.pvFirmware,
+								   pvFWCodeHostAddr,
+								   pvFWDataHostAddr,
+								   pvFWCorememCodeHostAddr,
+								   pvFWCorememDataHostAddr,
+								   &uFWParams);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXProcessFWImage failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+		RGXFwSharedMemCacheOpExec(pvFWCodeHostAddr,
+		                          sizeof(psDevInfo->psRGXFWCodeMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememCodeAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememCodeHostAddr,
+			                          sizeof(psDevInfo->psRGXFWCorememCodeMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+
+		RGXFwSharedMemCacheOpExec(pvFWDataHostAddr,
+		                          sizeof(psDevInfo->psRGXFWDataMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememDataAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememDataHostAddr,
+			                          sizeof(psDevInfo->psRGXFWIfCorememDataStoreMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+	}
+#else
+	sFWParams.uFWP = uFWParams;
+	RGXTDProcessFWImage(psDeviceNode, &sFWParams);
+#endif
+
+
+	/*
+	 * PDump Firmware allocations
+	 */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Dump firmware code image");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWCodeMemDesc,
+	                   0,
+	                   uiFWCodeAllocSize,
+	                   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware data image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWDataMemDesc,
+		                   0,
+		                   uiFWDataAllocSize,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem code image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWCorememCodeMemDesc,
+						   0,
+						   uiFWCorememCodeAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem data store image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+						   0,
+						   uiFWCorememDataAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION) || defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/*
+	 * Release Firmware allocations and clean up
+	 */
+release_fw_allocations:
+#endif
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememDataAllocSize != 0))
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+	}
+
+release_corememcode:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememCodeAllocSize != 0))
+#else
+	if (uiFWCorememCodeAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+
+release_data:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+	}
+
+release_code:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+cleanup_initfw:
+	OSUnloadFirmware(psRGXFW);
+fw_load_fail:
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseHWPerfCounters
+
+ @Description   Initialisation of hardware performance counters and dumping
+                them out to pdump, so that they can be modified at a later
+                point.
+
+ @Input         pvDevice
+ @Input         psHWPerfDataMemDesc
+ @Input         psHWPerfInitDataInt
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *pvDevice,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc,
+                                     RGXFWIF_HWPERF_CTL *psHWPerfInitDataInt)
+{
+	RGXFWIF_HWPERF_CTL_BLK *psHWPerfInitBlkData;
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc;
+	IMG_UINT32 ui32BlockID, ui32BlkCfgIdx, ui32CounterIdx;
+	RGX_HWPERF_CNTBLK_RT_INFO sCntBlkRtInfo;
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: #BlockConfig entries = %d", __func__, ui32CntBlkModelLen));
+
+	/* Initialise the number of blocks in the RGXFWIF_HWPERF_CTL structure.
+	 * This allows Firmware to validate that it has been correctly configured.
+	 */
+	psHWPerfInitDataInt->ui32NumBlocks = RGXHWPerfMaxDefinedBlks(pvDevice);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	    "HWPerf Block count = %u.",
+	    psHWPerfInitDataInt->ui32NumBlocks);
+#if defined(PDUMP)
+	/* Ensure that we record the BVNC specific ui32NumBlocks in the PDUMP data
+	 * so that when we playback we have the correct value present.
+	 */
+	DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+	    (size_t)&(psHWPerfInitDataInt->ui32NumBlocks) - (size_t)(psHWPerfInitDataInt),
+	    psHWPerfInitDataInt->ui32NumBlocks, PDUMP_FLAGS_CONTINUOUS);
+#endif	/* defined(PDUMP) */
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter Config starts here.");
+
+	/* Simply iterate over all the RGXFWIW_HWPERF_CTL blocks in order */
+	psHWPerfInitBlkData = &psHWPerfInitDataInt->sBlkCfg[0];
+
+	for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen;
+	     ui32BlkCfgIdx++, psHWPerfInitBlkData++)
+	{
+		IMG_UINT32 uiUnit;
+		IMG_BOOL bDirect;
+		IMG_BOOL bSingleton;
+
+		/* Exit early if this core does not have any of these counter blocks
+		 * due to core type/BVNC features.... */
+		psBlkTypeDesc = &asCntBlkTypeModel[ui32BlkCfgIdx];
+
+		if (psBlkTypeDesc == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected NULL - Index %d / %d",
+			    __func__, ui32BlkCfgIdx, ui32CntBlkModelLen));
+			continue;
+		}
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+		        "%s: CfgIdx = %u, InitBlkData @ 0x%p, BlkTypeDesc @ 0x%p",
+		        __func__, ui32BlkCfgIdx, psHWPerfInitBlkData, psBlkTypeDesc));
+
+		if (psBlkTypeDesc->pfnIsBlkPresent(psBlkTypeDesc, pvDevice, &sCntBlkRtInfo) == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: %s [ID 0x%x] NOT present", __func__,
+			    psBlkTypeDesc->pszBlockNameComment,
+			    psBlkTypeDesc->uiCntBlkIdBase ));
+			/* Block isn't present, but has an entry in the table. Populate
+			 * the Init data so that we can track the block later.
+			 */
+			psHWPerfInitBlkData->uiBlockID = psBlkTypeDesc->uiCntBlkIdBase;
+			continue;
+		}
+#ifdef DEBUG
+		else
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: %s has %d %s", __func__,
+				psBlkTypeDesc->pszBlockNameComment, sCntBlkRtInfo.uiNumUnits,
+			    (sCntBlkRtInfo.uiNumUnits > 1) ? "units" : "unit"));
+		}
+#endif	/* DEBUG */
+
+		/* Program all counters in one block so those already on may
+		 * be configured off and vice-versa. */
+		bDirect = psBlkTypeDesc->uiIndirectReg == 0;
+
+		/* Set if there is only one instance of this block-ID present */
+		bSingleton = sCntBlkRtInfo.uiNumUnits == 1;
+
+		for (ui32BlockID = psBlkTypeDesc->uiCntBlkIdBase, uiUnit = 0;
+		     ui32BlockID < psBlkTypeDesc->uiCntBlkIdBase+sCntBlkRtInfo.uiNumUnits;
+		     ui32BlockID++, uiUnit++)
+		{
+
+			if (bDirect)
+			{
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				    "Block : %s", psBlkTypeDesc->pszBlockNameComment);
+			}
+			else
+			{
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+				    PDUMP_FLAGS_CONTINUOUS,
+				    "Unit %d Block : %s%d",
+				    ui32BlockID-psBlkTypeDesc->uiCntBlkIdBase,
+				    psBlkTypeDesc->pszBlockNameComment, uiUnit);
+			}
+
+			psHWPerfInitBlkData->uiBlockID = ui32BlockID;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			    "uiBlockID: The Block ID for the layout block. See RGX_CNTBLK_ID for further information.");
+#if defined(PDUMP)
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->uiBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->uiBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			psHWPerfInitBlkData->uiNumCounters = 0;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			    "uiNumCounters (X): Specifies the number of valid counters"
+			    " [0..%d] which follow.", RGX_CNTBLK_COUNTERS_MAX);
+#if defined(PDUMP)
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+			    (size_t)&(psHWPerfInitBlkData->uiNumCounters) - (size_t)(psHWPerfInitDataInt),
+			    psHWPerfInitBlkData->uiNumCounters,
+			    PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			psHWPerfInitBlkData->uiEnabled = 0;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			    "uiEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+#if defined(PDUMP)
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+			    (size_t)&(psHWPerfInitBlkData->uiEnabled) - (size_t)(psHWPerfInitDataInt),
+			    psHWPerfInitBlkData->uiEnabled,
+			    PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			for (ui32CounterIdx = 0; ui32CounterIdx < RGX_CNTBLK_COUNTERS_MAX; ui32CounterIdx++)
+			{
+				psHWPerfInitBlkData->aui32CounterCfg[ui32CounterIdx] = IMG_UINT32_C(0x00000000);
+
+				if (bDirect)
+				{
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					    PDUMP_FLAGS_CONTINUOUS,
+					    "%s_COUNTER_%d",
+					    psBlkTypeDesc->pszBlockNameComment, ui32CounterIdx);
+				}
+				else
+				{
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					    PDUMP_FLAGS_CONTINUOUS,
+					    "%s%d_COUNTER_%d",
+					    psBlkTypeDesc->pszBlockNameComment,
+					    uiUnit, ui32CounterIdx);
+				}
+#if defined(PDUMP)
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+				    (size_t)&(psHWPerfInitBlkData->aui32CounterCfg[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+				    psHWPerfInitBlkData->aui32CounterCfg[ui32CounterIdx],
+				    PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			}
+
+			/* Update our block reference for indirect units which have more
+			 * than a single unit present. Only increment if we have more than
+			 * one unit left to process as the external loop counter will be
+			 * incremented after final unit is processed.
+			 */
+			if (!bSingleton && (uiUnit < (sCntBlkRtInfo.uiNumUnits - 1)))
+			{
+				psHWPerfInitBlkData++;
+			}
+		}
+	}
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter config finishes here.");
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseAllCounters
+
+ @Description   Initialise HWPerf and custom counters
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitialiseAllCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	RGXFWIF_HWPERF_CTL *psHWPerfInitData;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc, (void **)&psHWPerfInitData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", failHWPerfCountersMemDescAqCpuVirt);
+
+	InitialiseHWPerfCounters(psDeviceNode, psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc, psHWPerfInitData);
+	RGXFwSharedMemCacheOpPtr(psHWPerfInitData, FLUSH);
+
+failHWPerfCountersMemDescAqCpuVirt:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+
+	return eError;
+}
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR RGXValidateTDHeap(PVRSRV_DEVICE_NODE *psDeviceNode,
+									  PVRSRV_PHYS_HEAP ePhysHeap,
+									  PHYS_HEAP_USAGE_FLAGS ui32RequiredFlags)
+{
+	PHYS_HEAP *psHeap = psDeviceNode->apsPhysHeap[ePhysHeap];
+	PHYS_HEAP_USAGE_FLAGS ui32HeapFlags = PhysHeapGetFlags(psHeap);
+	PHYS_HEAP_USAGE_FLAGS ui32InvalidFlags = ~(PHYS_HEAP_USAGE_FW_PRIV_DATA | PHYS_HEAP_USAGE_FW_CODE
+											   | PHYS_HEAP_USAGE_GPU_SECURE | PHYS_HEAP_USAGE_FW_PRIVATE);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32RequiredFlags) != 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap is missing required flags. flags: 0x%x / required:0x%x",
+							   ui32HeapFlags,
+							   ui32RequiredFlags);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32InvalidFlags) == 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap uses invalid flags. flags: 0x%x / invalid:0x%x",
+							   ui32HeapFlags,
+							   ui32InvalidFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXValidateTDHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_PRIV_DATA, PHYS_HEAP_USAGE_FW_PRIV_DATA);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_PRIV_DATA");
+
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_CODE, PHYS_HEAP_USAGE_FW_CODE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_CODE");
+	}
+
+	eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_GPU_SECURE, PHYS_HEAP_USAGE_GPU_SECURE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:GPU_SECURE");
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInit
+
+ @Description   RGX Initialisation
+
+ @Input         psDeviceNode
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Services initialisation parameters */
+	RGX_INIT_APPHINTS sApphints = {0};
+	IMG_UINT32 ui32FWConfigFlags, ui32FWConfigFlagsExt, ui32FwOsCfgFlags;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 1");
+
+	PDUMPCOMMENT(psDeviceNode, "Device Name: %s",
+	             psDeviceNode->psDevConfig->pszName);
+	PDUMPCOMMENT(psDeviceNode, "Device ID: %u (%d)",
+	             psDeviceNode->sDevId.ui32InternalID,
+	             psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->psDevConfig->pszVersion)
+	{
+		PDUMPCOMMENT(psDeviceNode, "Device Version: %s",
+		             psDeviceNode->psDevConfig->pszVersion);
+	}
+
+	/* pdump info about the core */
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGX Version Information (KM): %d.%d.%d.%d",
+	             psDevInfo->sDevFeatureCfg.ui32B,
+	             psDevInfo->sDevFeatureCfg.ui32V,
+	             psDevInfo->sDevFeatureCfg.ui32N,
+	             psDevInfo->sDevFeatureCfg.ui32C);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	eError = RGXValidateTDHeaps(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeaps");
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		/* The RGX_CR_MTS_DM4_INTERRUPT_ENABLE register is always set by the firmware during initialisation
+		 * and it provides a good method of determining if the firmware has been booted previously */
+		psDeviceNode->bAutoVzFwIsUp = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_DM4_INTERRUPT_ENABLE) != 0);
+
+		PVR_LOG(("AutoVz startup check: firmware is %s;",
+				(psDeviceNode->bAutoVzFwIsUp) ? "already running" : "powered down"));
+		PVR_LOG(("AutoVz allow GPU powerdown is %s:",
+				(psDeviceNode->bAutoVzAllowGPUPowerdown) ? "enabled" : "disabled"));
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest assumes the firmware is always available */
+		psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+		psDeviceNode->bAutoVzAllowGPUPowerdown = IMG_FALSE;
+	}
+	else
+#endif
+	{
+		/* Firmware does not follow the AutoVz life-cycle */
+		psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+	{
+		/* set the device power state here as the regular power
+		 * callbacks will not be executed on this driver */
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+
+	/* Set which HW Safety Events will be handled by the driver */
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER) ?
+										  RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= (RGX_DEVICE_HAS_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS)
+										   && (RGX_DEVICE_GET_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS) > 0)) ?
+										  RGX_CR_EVENT_STATUS_FAULT_FW_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_DUAL_LOCKSTEP) ?
+										  RGX_CR_EVENT_STATUS_FAULT_FW_EN : 0;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Register defs revision: %d", RGX_CR_DEFS_KM_REVISION);
+#endif
+
+	/* Services initialisation parameters */
+	GetApphints(psDevInfo, &sApphints);
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	if ((sApphints.bEnableTrustedDeviceAceConfig) &&
+		(RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACE)))
+	{
+		SetTrustedDeviceAceEnabled(psDeviceNode->psDevConfig->hSysData);
+	}
+#endif
+#endif
+
+	eError = RGXInitCreateFWKernelMemoryContext(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create FW kernel memory context (%u)",
+		         __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitFirmware(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitFirmware failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Setup Firmware initialisation data
+	 */
+
+	GetFWConfigFlags(psDeviceNode, &sApphints, &ui32FWConfigFlags, &ui32FWConfigFlagsExt, &ui32FwOsCfgFlags);
+
+	eError = RGXInitFirmware(psDeviceNode,
+	                         &sApphints,
+	                         ui32FWConfigFlags,
+	                         ui32FWConfigFlagsExt,
+	                         ui32FwOsCfgFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitFirmware failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitialiseAllCounters(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitialiseAllCounters failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Perform second stage of RGX initialisation
+	 */
+	eError = RGXInitDevPart2(psDeviceNode,
+	                         &sApphints);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitDevPart2 failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+
+	eError = PVRSRV_OK;
+
+cleanup:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxsrvinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxstartstop.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxstartstop.c
new file mode 100644
index 000000000000..042138268df0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxstartstop.c
@@ -0,0 +1,1331 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific start/stop routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific start/stop routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxstartstop.h"
+
+
+
+/*
+	RGXWriteMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAT;
+	}
+
+	eError = RGXPollReg32(hPrivate, ui32StateReg, ui32StateReadyFlag, ui32StateReadyFlag);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Write */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+		RGXWriteReg32(hPrivate, ui32DataReg, ui32RegValue);
+		(void) RGXReadReg32(hPrivate, ui32DataReg); /* Fence write */
+	}
+
+	return eError;
+}
+
+/*
+	RGXReadMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN);
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN);
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAX;
+		BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	}
+
+	/* Wait for Slave Port to be Ready */
+	eError = RGXPollReg32(hPrivate,
+						  ui32StateReg,
+						  ui32StateReadyFlag,
+						  ui32StateReadyFlag);
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Read */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+
+		/* Wait for Slave Port to be Ready */
+		eError = RGXPollReg32(hPrivate,
+				  ui32StateReg,
+				  ui32StateReadyFlag,
+				  ui32StateReadyFlag);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+#if !defined(NO_HARDWARE)
+	*ui32RegValue = RGXReadReg32(hPrivate, ui32DataReg);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32DataReg);
+	*ui32RegValue = 0xFFFFFFFF;
+#endif
+
+	return eError;
+}
+
+/*
+ * Specific fields for RGX_CR_IDLE must not be polled in pdumps
+ * (technical reasons)
+ */
+#define CR_IDLE_UNSELECTED_MASK ((~RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK) | \
+								 (~RGX_CR_SLC_IDLE_OWDB_CLRMSK) |		\
+								 (RGX_CR_SLC_IDLE_FBCDC_ARB_EN))
+
+static PVRSRV_ERROR RGXWriteMetaCoreRegThoughSP(const void *hPrivate,
+                                                IMG_UINT32 ui32CoreReg,
+                                                IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 i = 0;
+
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXDT_OFFSET, ui32Value);
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, ui32CoreReg & ~META_CR_TXUXXRXRQ_RDnWR_BIT);
+
+	do
+	{
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, &ui32Value);
+	} while (((ui32Value & META_CR_TXUXXRXRQ_DREADY_BIT) != META_CR_TXUXXRXRQ_DREADY_BIT) && (i++ < 1000));
+
+	if (i == 1000)
+	{
+		RGXCommentLog(hPrivate, "RGXWriteMetaCoreRegThoughSP: Timeout");
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXStartFirmware(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+
+	/* Give privilege to debug and slave port */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+
+	/* Point Meta to the bootloader address, global (uncached) range */
+	eError = RGXWriteMetaCoreRegThoughSP(hPrivate,
+	                                     PC_ACCESS(0),
+	                                     RGXFW_BOOTLDR_META_ADDR | META_MEM_GLOBAL_RANGE_BIT);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start failed!");
+		return eError;
+	}
+
+	/* Enable minim encoding */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXPRIVEXT, META_CR_TXPRIVEXT_MINIM_EN);
+
+	/* Enable Meta thread */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_T0ENABLE_OFFSET, META_CR_TXENABLE_ENABLE_BIT);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMetaProcWrapper
+
+ @Description   Configures the hardware wrapper of the META processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMetaProcWrapper(const void *hPrivate)
+{
+	IMG_UINT64 ui64GartenConfig;
+
+	/* Garten IDLE bit controlled by META */
+	ui64GartenConfig = RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META;
+
+#if defined(RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT)
+	/* Set the Garten Wrapper BIF Fence address */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, S7_TOP_INFRASTRUCTURE))
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT;
+
+	}
+	else
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT;
+
+		/* Set SLC DM=META */
+		ui64GartenConfig |= ((IMG_UINT64) RGXFW_SEGMMU_META_BIFDM_ID) << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT;
+	}
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure META wrapper");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMipsProcWrapper
+
+ @Description   Configures the hardware wrapper of the MIPS processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMipsProcWrapper(const void *hPrivate)
+{
+	IMG_DEV_PHYADDR sPhyAddr;
+	IMG_UINT64 ui64RemapSettings = RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE; /* Same for all remap registers */
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure MIPS wrapper");
+
+	/*
+	 * MIPS wrapper (registers transaction ID and ISA mode) setup
+	 */
+
+	RGXCommentLog(hPrivate, "RGXStart: Write wrapper config register");
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		RGXWriteReg32(hPrivate,
+		              RGX_CR_MIPS_WRAPPER_CONFIG,
+		              (RGXMIPSFW_REGISTERS_VIRTUAL_BASE >>
+		              RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN) |
+		              RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+	else
+	{
+		RGXAcquireGPURegsAddr(hPrivate, &sPhyAddr);
+
+		RGXMIPSWrapperConfig(hPrivate,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG,
+		                     sPhyAddr.uiAddr,
+		                     RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+
+	/*
+	 * Boot remap setup
+	 */
+
+	RGXAcquireBootRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write boot remap registers");
+	RGXBootRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG1,
+	                   RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+#if defined(FIX_HW_BRN_63553_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 63553))
+	{
+		IMG_BOOL bPhysBusAbove32Bit = RGXGetDevicePhysBusWidth(hPrivate) > 32;
+		IMG_BOOL bDevicePA0IsValid  = RGXDevicePA0IsValid(hPrivate);
+
+		/* WA always required on 36 bit cores, to avoid continuous unmapped memory accesses to address 0x0 */
+		if (bPhysBusAbove32Bit || !bDevicePA0IsValid)
+		{
+			RGXCodeRemapConfig(hPrivate,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG1,
+					0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG2,
+					sPhyAddr.uiAddr,
+					~RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK,
+					ui64RemapSettings);
+		}
+	}
+#endif
+
+	/*
+	 * Data remap setup
+	 */
+
+	RGXAcquireDataRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		/* Remapped private data in secure memory */
+		ui64RemapSettings |= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN;
+	}
+	else
+	{
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+	}
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+	RGXCommentLog(hPrivate, "RGXStart: Write data remap registers");
+	RGXDataRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG1,
+	                   RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	/*
+	 * Code remap setup
+	 */
+
+	RGXAcquireCodeRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write exceptions remap registers");
+	RGXCodeRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG1,
+	                   RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) == 32)
+	{
+		/*
+		 * Trampoline remap setup
+		 */
+
+		RGXAcquireTrampolineRemapAddr(hPrivate, &sPhyAddr);
+		ui64RemapSettings = RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+		ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+		RGXCommentLog(hPrivate, "RGXStart: Write trampoline remap registers");
+		RGXTrampolineRemapConfig(hPrivate,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG1,
+		                         sPhyAddr.uiAddr | RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG2,
+		                         RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+		                         ~RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK,
+		                         ui64RemapSettings);
+	}
+
+	/* Garten IDLE bit controlled by MIPS */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to MIPS");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+
+	/* Turn on the EJTAG probe (only useful driver live) */
+	RGXWriteReg32(hPrivate, RGX_CR_MIPS_DEBUG_CONFIG, 0);
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitRiscvProcWrapper
+
+ @Description   Configures the hardware wrapper of the RISCV processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitRiscvProcWrapper(const void *hPrivate)
+{
+	IMG_UINT32 ui32BootCodeRemap = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+	IMG_UINT32 ui32BootDataRemap = RGXRISCVFW_BOOTLDR_DATA_REMAP;
+	IMG_DEV_VIRTADDR sTmp;
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure RISCV wrapper");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) < 4)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Write boot code remap");
+		RGXAcquireBootCodeAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootCodeRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN);
+
+		RGXCommentLog(hPrivate, "RGXStart: Write boot data remap");
+		RGXAcquireBootDataAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootDataRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN)
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN |
+#endif
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN);
+	}
+
+	/* Garten IDLE bit controlled by RISCV */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to RISCV");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+}
+
+
+static void RGXWriteKernelCatBase(const void *hPrivate, IMG_DEV_PHYADDR sPCAddr)
+{
+	IMG_UINT32 uiPCAddr;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+	{
+		IMG_UINT32 ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+		uiPCAddr = (((sPCAddr.uiAddr >> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT)
+		             << RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+		            & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK);
+
+		/* Set the mapping context */
+		RGXWriteReg32(hPrivate, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(hPrivate, ui32CBaseMapCtxReg); /* Fence write */
+
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+
+#if (MMU_CONTEXT_MAPPING_FWIF != MMU_CONTEXT_MAPPING_FWPRIV)
+		/* Set-up different MMU ID mapping to the same PC used above */
+		RGXWriteReg32(hPrivate, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWIF);
+		(void)RGXReadReg32(hPrivate, ui32CBaseMapCtxReg); /* Fence write */
+
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+#endif
+	}
+#else /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+	if (!RGX_DEVICE_HAS_FEATURE(hPrivate, SLC_VIVT))
+	{
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC64(hPrivate,
+		                      BIF_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT,
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT,
+		                      ((sPCAddr.uiAddr
+		                      >> RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT)
+		                      << RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT)
+		                      & ~RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK);
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Keep catbase registers in sync */
+			RGXWriteKernelMMUPC64(hPrivate,
+			                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+			                      ((sPCAddr.uiAddr
+			                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+			                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+			                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+		}
+
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXWriteKernelCatBase: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+	else
+	{
+		uiPCAddr = (((sPCAddr.uiAddr >> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT)
+		             << RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+		            & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK);
+
+		/* Set the mapping context */
+		RGXWriteReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT); /* Fence write */
+
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+
+#if (MMU_CONTEXT_MAPPING_FWIF != MMU_CONTEXT_MAPPING_FWPRIV)
+		/* Set-up different MMU ID mapping to the same PC used above */
+		RGXWriteReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT, MMU_CONTEXT_MAPPING_FWIF);
+		(void)RGXReadReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT); /* Fence write */
+
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+#endif
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitBIF
+
+ @Description   Initialise RGX BIF
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitBIF(const void *hPrivate)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXInitBIF: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+	else
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+	{
+		IMG_DEV_PHYADDR sPCAddr;
+
+		/*
+		 * Acquire the address of the Kernel Page Catalogue.
+		 */
+		RGXAcquireKernelMMUPC(hPrivate, &sPCAddr);
+
+		/*
+		 * Write the kernel catalogue base.
+		 */
+		RGXCommentLog(hPrivate, "RGX firmware MMU Page Catalogue");
+
+		RGXWriteKernelCatBase(hPrivate, sPCAddr);
+	}
+}
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/**************************************************************************/ /*!
+@Function       RGXInitMMURangeRegisters
+@Description    Initialises MMU range registers for Non4K pages.
+@Input          hPrivate           Implementation specific data
+@Return         void
+ */ /**************************************************************************/
+static void RGXInitMMURangeRegisters(const void *hPrivate)
+{
+	IMG_UINT32 ui32RegAddr = RGX_CR_MMU_PAGE_SIZE_RANGE_ONE;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES; ++i, ui32RegAddr += sizeof(IMG_UINT64))
+	{
+		RGXWriteReg64(hPrivate, ui32RegAddr, RGXMMUInitRangeValue(i));
+	}
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitAXIACE
+
+ @Description    Initialise AXI-ACE interface
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitAXIACE(const void *hPrivate)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	ui32RegAddr = RGX_CR_AXI_ACE_LITE_CONFIGURATION;
+
+	/* Setup AXI-ACE config. Set everything to outer cache */
+	ui64RegVal = (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT) |
+	             (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT)  |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT);
+
+#if defined(FIX_HW_BRN_42321_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 42321))
+	{
+		ui64RegVal |= (((IMG_UINT64) 1) << RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT);
+	}
+#endif
+
+#if defined(FIX_HW_BRN_68186_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 68186))
+	{
+		/* default value for reg_enable_fence_out is zero. Force to 1 to allow core_clk < mem_clk */
+		ui64RegVal |= (IMG_UINT64)1 << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT;
+	}
+#endif
+#else /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+	ui32RegAddr = RGX_CR_ACE_CTRL;
+
+	/**
+	 * The below configuration is only applicable for RGX cores supporting
+	 * ACE/ACE-lite protocol and connected to ACE coherent interconnect.
+	 */
+
+	/**
+	 * Configure AxDomain and AxCache for MMU transactions.
+	 * AxDomain set to non sharable (0x0).
+	 */
+	ui64RegVal = RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE |
+				 RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE;
+
+	/**
+	 * Configure AxCache for PM/MMU transactions.
+	 * Set to same value (i.e WBRWALLOC caching, rgxmmunit.c:RGXDerivePTEProt8)
+	 * as non-coherent PTEs
+	 */
+	ui64RegVal |= (IMG_UINT64_C(0xF)) << RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT;
+
+	/**
+	 * Configure AxDomain for non MMU transactions.
+	 */
+	ui64RegVal |= (IMG_UINT64)(RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE |
+							   RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE);
+#endif /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+
+	RGXCommentLog(hPrivate, "Init AXI-ACE interface");
+	RGXWriteReg64(hPrivate, ui32RegAddr, ui64RegVal);
+}
+
+static void RGXMercerSoftResetSet(const void *hPrivate, IMG_UINT64 ui32MercerFlags)
+{
+	RGXWriteReg64(hPrivate, RGX_CR_MERCER_SOFT_RESET, ui32MercerFlags & RGX_CR_MERCER_SOFT_RESET_MASKFULL);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) RGXReadReg64(hPrivate, RGX_CR_MERCER_SOFT_RESET);
+}
+
+static void RGXSPUSoftResetAssert(const void *hPrivate)
+{
+	/* Assert Mercer0 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN);
+	/* Assert Mercer1 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN | RGX_CR_MERCER1_SOFT_RESET_SPU_EN);
+	/* Assert Mercer2 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN | RGX_CR_MERCER1_SOFT_RESET_SPU_EN | RGX_CR_MERCER2_SOFT_RESET_SPU_EN);
+
+	RGXWriteReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET, RGX_CR_SWIFT_SOFT_RESET_MASKFULL);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET);
+
+	RGXWriteReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET, RGX_CR_TEXAS_SOFT_RESET_MASKFULL);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET);
+}
+
+static void RGXSPUSoftResetDeAssert(const void *hPrivate)
+{
+	RGXWriteReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET, 0);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET);
+
+
+	RGXWriteReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET, 0);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET);
+
+	/* Deassert Mercer2 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN | RGX_CR_MERCER1_SOFT_RESET_SPU_EN);
+	/* Deassert Mercer1 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN);
+	/* Deassert Mercer0 */
+	RGXMercerSoftResetSet(hPrivate, 0);
+}
+
+static void RGXResetSequence(const void *hPrivate, const IMG_CHAR *pcRGXFW_PROCESSOR)
+{
+	/* Set RGX in soft-reset */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXCommentLog(hPrivate, "RGXStart: soft reset cpu core");
+		RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 0);
+	}
+
+	RGXCommentLog(hPrivate, "RGXStart: soft reset assert step 1");
+	RGXSPUSoftResetAssert(hPrivate);
+
+	RGXCommentLog(hPrivate, "RGXStart: soft reset assert step 2");
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_SOFT_RESET_JONES_ALL);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_SOFT_RESET_JONES_ALL | RGX_SOFT_RESET_EXTRA);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* Take everything out of reset but the FW processor */
+	RGXCommentLog(hPrivate, "RGXStart: soft reset de-assert step 1 excluding %s", pcRGXFW_PROCESSOR);
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_SOFT_RESET_EXTRA | RGX_CR_SOFT_RESET_GARTEN_EN);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	RGXCommentLog(hPrivate, "RGXStart: soft reset de-assert step 2 excluding %s", pcRGXFW_PROCESSOR);
+	RGXSPUSoftResetDeAssert(hPrivate);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+}
+
+static void DeassertMetaReset(const void *hPrivate)
+{
+	/* Need to wait for at least 32 cycles before taking the FW processor out of reset ... */
+	RGXWaitCycles(hPrivate, 32, 3);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, 0x0);
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* ... and afterwards */
+	RGXWaitCycles(hPrivate, 32, 3);
+}
+
+static PVRSRV_ERROR InitJonesECCRAM(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Value;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+	IMG_UINT32 ui32Mask;
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) == 0)
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	if (bMetaFW)
+	{
+		/* META must be taken out of reset (without booting) during Coremem initialization. */
+		RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		DeassertMetaReset(hPrivate);
+	}
+
+	/* Clocks must be set to "on" during RAMs initialization. */
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, PIPELINED_DATAMASTERS_VERSION) > 0)
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_ON);
+	}
+	else
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0_ALL_ON);
+	}
+	RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL1, RGX_CR_CLK_CTRL1_ALL_ON);
+	RGXWriteReg32(hPrivate, RGX_CR_CLK_CTRL2, RGX_CR_CLK_CTRL2_ALL_ON);
+
+	if (bMetaFW)
+	{
+		RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+		RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, META_CR_TXCLKCTRL_ALL_ON);
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, &ui32Value);
+	}
+
+	ui32Mask = bMetaFW ?
+		RGX_CR_JONES_RAM_INIT_KICK_MASKFULL
+		: RGX_CR_JONES_RAM_INIT_KICK_MASKFULL & ~RGX_CR_JONES_RAM_INIT_KICK_GARTEN_EN;
+
+	/* SLC component in CHEST either does not exist or not powered at this point */
+	ui32Mask &= RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_CLRMSK;
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, POWER_ISLAND_VERSION) >= 4)
+	{
+		/* FBCDC has components in CHEST which may not be powered at this point */
+		ui32Mask &= RGX_CR_JONES_RAM_INIT_KICK_FBCDC_CLRMSK;
+	}
+
+	RGXWriteReg64(hPrivate, RGX_CR_JONES_RAM_INIT_KICK, ui32Mask);
+	eError = RGXPollReg64(hPrivate, RGX_CR_JONES_RAM_STATUS, ui32Mask, ui32Mask);
+
+	if (bMetaFW)
+	{
+		RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, META_CR_TXCLKCTRL_ALL_AUTO);
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, &ui32Value);
+	}
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, PIPELINED_DATAMASTERS_VERSION) > 0)
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_AUTO);
+	}
+	else
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0_ALL_AUTO);
+	}
+	RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL1, RGX_CR_CLK_CTRL1_ALL_AUTO);
+	RGXWriteReg32(hPrivate, RGX_CR_CLK_CTRL2, RGX_CR_CLK_CTRL2_ALL_AUTO);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+		RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+	}
+
+	return eError;
+}
+
+
+PVRSRV_ERROR RGXStart(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR *pcRGXFW_PROCESSOR;
+	IMG_BOOL bDoFWSlaveBoot = IMG_FALSE;
+	IMG_BOOL bMetaFW = IMG_FALSE;
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_RISCV;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_MIPS;
+	}
+#endif
+	else
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_META;
+		bMetaFW = IMG_TRUE;
+		bDoFWSlaveBoot = RGXDoFWSlaveBoot(hPrivate);
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SYS_BUS_SECURE_RESET))
+	{
+		/* Disable the default sys_bus_secure protection to perform minimal setup */
+		RGXCommentLog(hPrivate, "RGXStart: Disable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, 0);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+
+#if defined(RGX_FEATURE_HYPERVISOR_MMU_BIT_MASK)
+	/* Only bypass HMMU if the module is present */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, HYPERVISOR_MMU))
+	{
+		/* Always set HMMU in bypass mode */
+		RGXWriteReg32(hPrivate, RGX_CR_HMMU_BYPASS, RGX_CR_HMMU_BYPASS_MASKFULL);
+		(void) RGXReadReg32(hPrivate, RGX_CR_HMMU_BYPASS);
+	}
+#endif
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_DUAL_LOCKSTEP))
+	{
+		/* Enable the dual LS mode */
+		RGXCommentLog(hPrivate, "RGXStart: Enable Dual Core Lock Step");
+		RGXWriteReg32(hPrivate, RGX_CR_FIRMWARE_PROCESSOR_LS, (0x3 << RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__ENABLE_SHIFT));
+		(void) RGXReadReg32(hPrivate, RGX_CR_FIRMWARE_PROCESSOR_LS);
+	}
+
+
+	/*!
+	 * Start FW init sequence
+	 */
+	RGXResetSequence(hPrivate, pcRGXFW_PROCESSOR);
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) > 0)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Init Jones ECC RAM");
+		eError = InitJonesECCRAM(hPrivate);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, GPU_MULTICORE_SUPPORT))
+		{
+			/* Set OR reduce for ECC faults to ensure faults are not missed during early boot stages */
+			RGXWriteReg32(hPrivate, RGX_CR_MULTICORE_EVENT_REDUCE, RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_EN | RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_EN);
+		}
+
+		/* Route fault events to the host */
+		RGXWriteReg32(hPrivate, RGX_CR_EVENT_ENABLE, RGX_CR_EVENT_ENABLE_FAULT_FW_EN);
+	}
+
+	if (bMetaFW)
+	{
+		if (bDoFWSlaveBoot)
+		{
+			/* Configure META to Slave boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Slave boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		}
+		else
+		{
+			/* Configure META to Master boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Master boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN);
+		}
+	}
+
+	/*
+	 * Initialise Firmware wrapper
+	 */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXInitRiscvProcWrapper(hPrivate);
+	}
+	else if (bMetaFW)
+	{
+		RGXInitMetaProcWrapper(hPrivate);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		RGXInitMipsProcWrapper(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, MMU_VERSION) >= 4)
+	{
+		/* initialise the MMU range based config registers for Non4K pages */
+		RGXInitMMURangeRegisters(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, AXI_ACELITE))
+#endif
+	{
+		/* We must init the AXI-ACE interface before 1st BIF transaction */
+		RGXInitAXIACE(hPrivate);
+	}
+
+	/*
+	 * Initialise BIF.
+	 */
+	RGXInitBIF(hPrivate);
+
+	RGXSetPoweredState(hPrivate, IMG_TRUE);
+
+	RGXCommentLog(hPrivate, "RGXStart: Take %s out of reset", pcRGXFW_PROCESSOR);
+	DeassertMetaReset(hPrivate);
+
+	if (bMetaFW && bDoFWSlaveBoot)
+	{
+		eError = RGXFabricCoherencyTest(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start");
+		eError = RGXStartFirmware(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+	}
+	else
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Master boot Start");
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Bring Debug Module out of reset */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) >= 4)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN);
+			}
+			else
+#endif
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL, RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+			}
+
+			/* Boot the FW */
+			RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 1);
+			RGXWaitCycles(hPrivate, 32, 3);
+		}
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) == 1)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Enable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStop(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+	IMG_UINT32 ui32JonesIdleMask = RGX_CR_JONES_IDLE_MASKFULL^(RGX_CR_JONES_IDLE_AXI2IMG_EN|RGX_CR_JONES_IDLE_SI_AXI2IMG_EN);
+
+	RGXDeviceAckIrq(hPrivate);
+
+	/* Set FW power state OFF to disable LISR handler */
+	RGXSetPoweredState(hPrivate, IMG_FALSE);
+
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper */
+	if (!RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, RAY_TRACING_ARCH) ||
+	    RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, RAY_TRACING_ARCH) < 2)
+	{
+		ui32JonesIdleMask ^= (RGX_CR_JONES_IDLE_ASC_EN|RGX_CR_JONES_IDLE_RCE_EN);
+	}
+
+#if defined(FIX_HW_BRN_74812_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 74812))
+	{
+		/* BRN74812 means the IPP idle signal may not be valid and is therefore ignored... */
+		ui32JonesIdleMask &= RGX_CR_JONES_IDLE_IPP_CLRMSK;
+	}
+#endif
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_JONES_IDLE,
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN),
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN));
+
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Wait for SLC to signal IDLE */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_SLC_IDLE,
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK),
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK));
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Unset MTS DM association with threads */
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL);
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL);
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL);
+	}
+
+#if defined(PDUMP)
+	if (bMetaFW)
+	{
+		/* Disabling threads is only required for pdumps to stop the fw gracefully */
+
+		/* Disable thread 0 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T0ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Disable thread 1 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T1ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Clear down any irq raised by META (done after disabling the FW
+		 * threads to avoid a race condition).
+		 * This is only really needed for PDumps but we do it anyway driver-live.
+		 */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		/* Wait for the Slave Port to finish all the transactions */
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+		{
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+			else
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+		}
+		else
+#endif
+		{
+			RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS, 0x0);
+			(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS); /* Fence write */
+
+			eError = RGXPollReg32(hPrivate,
+								  RGX_CR_META_SP_MSLVCTRL1,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN);
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+#endif
+
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS1,
+	                      0,
+	                      (~RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_WRITES_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_WRITES_CLRMSK));
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS2,
+	                      0,
+	                      (~RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_WRITES_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_WRITES_CLRMSK));
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Wait for SLC to signal IDLE */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_SLC_IDLE,
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK),
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK));
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Wait for Jones to signal IDLE except for the Garten Wrapper */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_JONES_IDLE,
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN),
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN));
+	if (eError != PVRSRV_OK) return eError;
+
+	if (bMetaFW)
+	{
+		IMG_UINT32 ui32RegValue;
+
+		eError = RGXReadMetaRegThroughSP(hPrivate,
+		                                 META_CR_TxVECINT_BHALT,
+		                                 &ui32RegValue);
+		if (eError != PVRSRV_OK) return eError;
+
+		if ((ui32RegValue & 0xFFFFFFFFU) == 0x0)
+		{
+			/* Wait for Sidekick/Jones to signal IDLE including
+			 * the Garten Wrapper if there is no debugger attached
+			 * (TxVECINT_BHALT = 0x0) */
+			eError = RGXPollReg32(hPrivate,
+			                      RGX_CR_JONES_IDLE,
+			                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN,
+			                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+	}
+	else
+	{
+		eError = RGXPollReg32(hPrivate,
+		                      RGX_CR_JONES_IDLE,
+		                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN,
+		                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.c
new file mode 100644
index 000000000000..9ebf43429e19
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.c
@@ -0,0 +1,5798 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA/3D routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX TA/3D routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* for the offsetof macro */
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxta3d.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "ri_server.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "rgx_memallocflags.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxsyncutils.h"
+#include "htbserver.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "physmem.h"
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "process_stats.h"
+
+#if !defined(PM_INTERACTIVE_MODE)
+#include "rgxpmdefs.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TA3D_UFO_DUMP	0
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static INLINE
+void _DebugSyncValues(const IMG_CHAR *pszFunction,
+					  const IMG_UINT32 *pui32UpdateValues,
+					  const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", pszFunction, i, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+
+static INLINE
+void _DebugSyncCheckpoints(const IMG_CHAR *pszFunction,
+						   const IMG_CHAR *pszDMName,
+						   const PSYNC_CHECKPOINT *apsSyncCheckpoints,
+						   const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFence%sSyncCheckpoints[%d]=<%p>", pszFunction, pszDMName, i, *(apsSyncCheckpoints + i)));
+	}
+}
+
+#else
+#define CHKPT_DBG(X)
+#endif
+
+/* define the number of commands required to be set up by the CCB helper */
+/* 1 command for the TA */
+#define CCB_CMD_HELPER_NUM_TA_COMMANDS 1
+/* Up to 3 commands for the 3D (partial render fence, partial render, and render) */
+#define CCB_CMD_HELPER_NUM_3D_COMMANDS 3
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define WORKEST_CYCLES_PREDICTION_GET(x) ((x).ui32CyclesPrediction)
+#else
+#define WORKEST_CYCLES_PREDICTION_GET(x) (NO_CYCEST)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_TA_DATA;
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_3D_DATA;
+
+struct _RGX_SERVER_RENDER_CONTEXT_ {
+	/* this lock protects usage of the render context.
+	 * it ensures only one kick is being prepared and/or submitted on
+	 * this render context at any time
+	 */
+	POS_LOCK				hLock;
+	RGX_CCB_CMD_HELPER_DATA asTACmdHelperData[CCB_CMD_HELPER_NUM_TA_COMMANDS];
+	RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[CCB_CMD_HELPER_NUM_3D_COMMANDS];
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRenderContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	RGX_SERVER_RC_TA_DATA		sTAData;
+	RGX_SERVER_RC_3D_DATA		s3DData;
+	IMG_UINT32					ui32CleanupStatus;
+#define RC_CLEANUP_TA_COMPLETE		(1 << 0)
+#define RC_CLEANUP_3D_COMPLETE		(1 << 1)
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListTAFence;
+	SYNC_ADDR_LIST				sSyncAddrListTAUpdate;
+	SYNC_ADDR_LIST				sSyncAddrList3DFence;
+	SYNC_ADDR_LIST				sSyncAddrList3DUpdate;
+	ATOMIC_T					hIntJobRef;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+
+/*
+	Static functions used by render context code
+*/
+
+static
+PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  psTAData->psServerCommonContext,
+											  RGXFWIF_DM_GEOM,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psTAData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, psTAData->psContextStateMemDesc);
+	psTAData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psContextStateMemDesc);
+	ps3DData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static void _RGXDumpPMRPageList(DLLIST_NODE *psNode)
+{
+	RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	PVRSRV_ERROR			eError;
+
+	eError = PMRDumpPageList(psPMRNode->psPMR,
+							RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Error (%s) printing pmr %p",
+				PVRSRVGetErrorString(eError),
+				psPMRNode->psPMR));
+	}
+}
+
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016" IMG_UINT64_FMTSPECx,
+				psFreeList->sFreeListFWDevVAddr.ui32Addr,
+				psFreeList->ui32FreelistID,
+				psFreeList->ui64FreelistChecksum));
+
+	/* Dump Init FreeList page list */
+	PVR_LOG(("  Initial Memory block"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	/* Dump Grow FreeList page list */
+	PVR_LOG(("  Grow Memory blocks"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	return IMG_TRUE;
+}
+
+static void _CheckFreelist(RGX_FREELIST *psFreeList,
+						   IMG_UINT32 ui32NumOfPagesToCheck,
+						   IMG_UINT64 ui64ExpectedCheckSum,
+						   IMG_UINT64 *pui64CalculatedCheckSum)
+{
+#if defined(NO_HARDWARE)
+	/* No checksum needed as we have all information in the pdumps */
+	PVR_UNREFERENCED_PARAMETER(psFreeList);
+	PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
+	PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
+	*pui64CalculatedCheckSum = 0;
+#else
+	PVRSRV_ERROR eError;
+	size_t uiNumBytes;
+	IMG_UINT8* pui8Buffer;
+	IMG_UINT32* pui32Buffer;
+	IMG_UINT32 ui32CheckSumAdd = 0;
+	IMG_UINT32 ui32CheckSumXor = 0;
+	IMG_UINT32 ui32Entry;
+	IMG_UINT32 ui32Entry2;
+	IMG_BOOL bFreelistBad = IMG_FALSE;
+
+	*pui64CalculatedCheckSum = 0;
+
+	PVR_ASSERT(ui32NumOfPagesToCheck <= (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+	/* Allocate Buffer of the size of the freelist */
+	pui8Buffer = OSAllocMem(ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+	if (pui8Buffer == NULL)
+	{
+		PVR_LOG(("%s: Failed to allocate buffer to check freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	/* Copy freelist content into Buffer */
+	eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
+					psFreeList->uiFreeListPMROffset +
+					(((psFreeList->ui32MaxFLPages -
+					   psFreeList->ui32CurrentFLPages - psFreeList->ui32ReadyFLPages) * sizeof(IMG_UINT32)) &
+					 ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1)),
+					 pui8Buffer,
+					 ui32NumOfPagesToCheck * sizeof(IMG_UINT32),
+					&uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pui8Buffer);
+		PVR_LOG(("%s: Failed to get freelist data for freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	PVR_ASSERT(uiNumBytes == ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+
+	/* Generate checksum (skipping the first page if not allocated) */
+	pui32Buffer = (IMG_UINT32 *)pui8Buffer;
+	ui32Entry = ((psFreeList->ui32GrowFLPages == 0  &&  psFreeList->ui32CurrentFLPages > 1) ? 1 : 0);
+	for (/*ui32Entry*/ ; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
+	{
+		ui32CheckSumAdd += pui32Buffer[ui32Entry];
+		ui32CheckSumXor ^= pui32Buffer[ui32Entry];
+
+		/* Check for double entries */
+		for (ui32Entry2 = ui32Entry+1; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
+		{
+			if (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2])
+			{
+				PVR_LOG(("%s: Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
+						__func__,
+						psFreeList->sFreeListFWDevVAddr.ui32Addr,
+						pui32Buffer[ui32Entry2],
+						ui32Entry,
+						ui32Entry2,
+						psFreeList->ui32CurrentFLPages));
+				bFreelistBad = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	OSFreeMem(pui8Buffer);
+
+	/* Check the calculated checksum against the expected checksum... */
+	*pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
+
+	if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
+	{
+		PVR_LOG(("%s: Checksum mismatch for freelist %p! Expected 0x%016" IMG_UINT64_FMTSPECx " calculated 0x%016" IMG_UINT64_FMTSPECx,
+				__func__, psFreeList,
+				ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
+		bFreelistBad = IMG_TRUE;
+	}
+
+	if (bFreelistBad)
+	{
+		PVR_LOG(("%s: Sleeping for ever!", __func__));
+		PVR_ASSERT(!bFreelistBad);
+	}
+#endif
+}
+
+
+/*
+ *  Function to work out the number of freelist pages to reserve for growing
+ *  within the FW without having to wait for the host to progress a grow
+ *  request.
+ *
+ *  The number of pages must be a multiple of 4 to align the PM addresses
+ *  for the initial freelist allocation and also be less than the grow size.
+ *
+ *  If the threshold or grow size means less than 4 pages, then the feature
+ *  is not used.
+ */
+static IMG_UINT32 _CalculateFreelistReadyPages(RGX_FREELIST *psFreeList,
+                                               IMG_UINT32  ui32FLPages)
+{
+	IMG_UINT32  ui32ReadyFLPages = ((ui32FLPages * psFreeList->ui32GrowThreshold) / 100) &
+	                               ~((RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE/sizeof(IMG_UINT32))-1);
+
+	if (ui32ReadyFLPages > psFreeList->ui32GrowFLPages)
+	{
+		ui32ReadyFLPages = psFreeList->ui32GrowFLPages;
+	}
+
+	return ui32ReadyFLPages;
+}
+
+
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader)
+{
+	RGX_PMR_NODE	*psPMRNode;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32  ui32MappingTable = 0;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_DEVMEM_SIZE_T uistartPage;
+	PVRSRV_ERROR eError;
+	static const IMG_CHAR szAllocName[] = "Free List";
+
+	/* Are we allowed to grow ? */
+	if (psFreeList->ui32MaxFLPages - (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) < ui32NumPages)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: grow by %u pages denied. "
+				"Max PB size reached (current pages %u+%u/%u)",
+				psFreeList,
+				ui32NumPages,
+				psFreeList->ui32CurrentFLPages,
+				psFreeList->ui32ReadyFLPages,
+				psFreeList->ui32MaxFLPages));
+		return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
+	}
+
+	/* Allocate kernel memory block structure */
+	psPMRNode = OSAllocMem(sizeof(*psPMRNode));
+	if (psPMRNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to allocate host data structure",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocHost;
+	}
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages
+	 */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+
+	/*
+	 *  The PM never takes the last page in a freelist, so if this block
+	 *  of pages is the first one and there is no ability to grow, then
+	 *  we can skip allocating one 4K page for the lowest entry.
+	 */
+	if (OSGetPageSize() > RGX_BIF_PM_PHYSICAL_PAGE_SIZE)
+	{
+		/*
+		 * Allocation size will be rounded up to the OS page size,
+		 * any attempt to change it a bit now will be invalidated later.
+		 */
+		psPMRNode->bFirstPageMissing = IMG_FALSE;
+	}
+	else
+	{
+		psPMRNode->bFirstPageMissing = (psFreeList->ui32GrowFLPages == 0  &&  ui32NumPages > 1);
+	}
+
+	psPMRNode->ui32NumPages = ui32NumPages;
+	psPMRNode->psFreeList = psFreeList;
+
+	/* Allocate Memory Block */
+	PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Allocate PB Block (Pages %08X)", ui32NumPages);
+	uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	if (psPMRNode->bFirstPageMissing)
+	{
+		uiSize -= RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	}
+
+	eError = PhysmemNewRamBackedPMR(psFreeList->psConnection,
+			psFreeList->psDevInfo->psDeviceNode,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_PRIVATE) |
+			PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP,
+			sizeof(szAllocName),
+			szAllocName,
+			psFreeList->ownerPid,
+			&psPMRNode->psPMR,
+			PDUMP_NONE,
+			NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate PB block of size: 0x%016" IMG_UINT64_FMTSPECX,
+				 __func__,
+				 (IMG_UINT64)uiSize));
+		goto ErrorBlockAlloc;
+	}
+
+	/* Zeroing physical pages pointed by the PMR */
+	if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((uistartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRNode->psPMR,
+	                                    psFreeList->ownerPid);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: call to RIWritePMREntryWithOwnerKM failed (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+	}
+
+	 /* Attach RI information */
+	eError = RIWriteMEMDESCEntryKM(psFreeList->psConnection,
+	                               psFreeList->psDevInfo->psDeviceNode,
+	                               psPMRNode->psPMR,
+	                               OSStringNLength(szAllocName, DEVMEM_ANNOTATION_MAX_LEN),
+	                               szAllocName,
+	                               0,
+	                               uiSize,
+	                               0,
+	                               &psPMRNode->hRIHandle);
+	PVR_LOG_IF_ERROR(eError, "RIWriteMEMDESCEntryKM");
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	/* write Freelist with Memory Block physical addresses */
+	eError = PMRWritePMPageList(
+						/* Target PMR, offset, and length */
+						psFreeList->psFreeListPMR,
+						(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+						(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+						/* Referenced PMR, and "page" granularity */
+						psPMRNode->psPMR,
+						RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+						&psPMRNode->psPageList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to write pages of Node %p",
+				 __func__,
+				 psPMRNode));
+		goto ErrorPopulateFreelist;
+	}
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	/* Copy freelist memory to shadow freelist */
+	{
+		const IMG_UINT32 ui32FLMaxSize = psFreeList->ui32MaxFLPages * sizeof(IMG_UINT32);
+		const IMG_UINT32 ui32MapSize = ui32FLMaxSize * 2;
+		const IMG_UINT32 ui32CopyOffset = uiOffset - psFreeList->uiFreeListPMROffset;
+		IMG_BYTE *pFLMapAddr;
+		size_t uiNumBytes;
+		PVRSRV_ERROR res;
+		IMG_HANDLE hMapHandle;
+
+		/* Check for overflow. Validate size and offset. */
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize > psFreeList->uiFreeListPMROffset,
+		                          eError, ErrorPopulateFreelist);
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize <= PMR_LogicalSize(psFreeList->psFreeListPMR),
+		                          eError, ErrorPopulateFreelist);
+
+		/* Map both the FL and the shadow FL */
+		res = PMRAcquireKernelMappingData(psFreeList->psFreeListPMR, psFreeList->uiFreeListPMROffset, ui32MapSize,
+		                                  (void**) &pFLMapAddr, &uiNumBytes, &hMapHandle);
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map freelist (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+
+		/* Copy only the newly added memory */
+		OSCachedMemCopy(pFLMapAddr + ui32FLMaxSize + ui32CopyOffset, pFLMapAddr + ui32CopyOffset , uiLength);
+		OSWriteMemoryBarrier(pFLMapAddr);
+
+#if defined(PDUMP)
+		PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Initialize shadow freelist");
+
+		/* Translate memcpy to pdump */
+		{
+			IMG_DEVMEM_OFFSET_T uiCurrOffset;
+
+			for (uiCurrOffset = uiOffset; (uiCurrOffset - uiOffset) < uiLength; uiCurrOffset += sizeof(IMG_UINT32))
+			{
+				PMRPDumpCopyMem32(psFreeList->psFreeListPMR,
+				                  uiCurrOffset + ui32FLMaxSize,
+				                  psFreeList->psFreeListPMR,
+				                  uiCurrOffset,
+				                  ":SYSMEM:$1",
+				                  PDUMP_FLAGS_CONTINUOUS);
+			}
+		}
+#endif
+
+
+		res = PMRReleaseKernelMappingData(psFreeList->psFreeListPMR, hMapHandle);
+
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to release freelist mapping (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+	}
+#endif
+
+	/* We add It must be added to the tail, otherwise the freelist population won't work */
+	dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
+
+	/* Update number of available pages */
+	psFreeList->ui32CurrentFLPages += ui32NumPages;
+
+	/* Update statistics (needs to happen before the ReadyFL calculation to also count those pages) */
+	if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
+	{
+		psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
+	}
+
+	/* Reserve a number ready pages to allow the FW to process OOM quickly and asynchronously request a grow. */
+	psFreeList->ui32ReadyFLPages    = _CalculateFreelistReadyPages(psFreeList, psFreeList->ui32CurrentFLPages);
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+
+	if (psFreeList->bCheckFreelist)
+	{
+		/*
+		 *  We can only calculate the freelist checksum when the list is full
+		 *  (e.g. at initial creation time). At other times the checksum cannot
+		 *  be calculated and has to be disabled for this freelist.
+		 */
+		if ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages)
+		{
+			_CheckFreelist(psFreeList, ui32NumPages, 0, &psFreeList->ui64FreelistChecksum);
+		}
+		else
+		{
+			psFreeList->ui64FreelistChecksum = 0;
+		}
+	}
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: %s %u pages (pages=%u+%u/%u checksum=0x%016" IMG_UINT64_FMTSPECx "%s)",
+			psFreeList,
+	        ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages ? "Create initial" : "Grow by"),
+			ui32NumPages,
+			psFreeList->ui32CurrentFLPages,
+			psFreeList->ui32ReadyFLPages,
+			psFreeList->ui32MaxFLPages,
+			psFreeList->ui64FreelistChecksum,
+			(psPMRNode->bFirstPageMissing ? " - lowest page not allocated" : "")));
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+ErrorPopulateFreelist:
+	(void) PMRUnrefPMR(psPMRNode->psPMR);
+
+ErrorBlockAlloc:
+	OSFreeMem(psPMRNode);
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+ErrorAllocHost:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
+										RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode;
+	RGX_PMR_NODE *psPMRNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32OldValue;
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages value
+	 */
+	PVR_ASSERT(pListHeader);
+	PVR_ASSERT(psFreeList);
+	PVR_ASSERT(psFreeList->psDevInfo);
+	PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+	/* Get node from head of list and remove it */
+	psNode = dllist_get_next_node(pListHeader);
+	if (psNode)
+	{
+		dllist_remove_node(psNode);
+
+		psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+		PVR_ASSERT(psPMRNode);
+		PVR_ASSERT(psPMRNode->psPMR);
+		PVR_ASSERT(psPMRNode->psFreeList);
+
+		/* remove block from freelist list */
+
+		/* Unwrite Freelist with Memory Block physical addresses */
+		eError = PMRUnwritePMPageList(psPMRNode->psPageList);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to unwrite pages of Node %p",
+					 __func__,
+					 psPMRNode));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+		if (psPMRNode->hRIHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = RIDeleteMEMDESCEntryKM(psPMRNode->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "RIDeleteMEMDESCEntryKM");
+		}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+		/* Free PMR (We should be the only one that holds a ref on the PMR) */
+		eError = PMRUnrefPMR(psPMRNode->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to free PB block %p (%s)",
+					 __func__,
+					 psPMRNode->psPMR,
+					 PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+		/* update available pages in freelist */
+		ui32OldValue = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+
+		/*
+		 * Deallocated pages should first be deducted from ReadyPages bank, once
+		 * there are no more left, start deducting them from CurrentPage bank.
+		 */
+		if (psPMRNode->ui32NumPages > psFreeList->ui32ReadyFLPages)
+		{
+			psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages - psFreeList->ui32ReadyFLPages;
+			psFreeList->ui32ReadyFLPages = 0;
+		}
+		else
+		{
+			psFreeList->ui32ReadyFLPages -= psPMRNode->ui32NumPages;
+		}
+
+		/* check underflow */
+		PVR_ASSERT(ui32OldValue > (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
+								psFreeList,
+								psPMRNode->ui32NumPages,
+								psFreeList->ui32CurrentFLPages,
+								psFreeList->ui32MaxFLPages));
+
+		OSFreeMem(psPMRNode);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
+								psFreeList,
+								psFreeList->ui32InitFLPages));
+		eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
+	}
+
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	return eError;
+}
+
+static RGX_FREELIST *FindFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_FREELIST *psFreeList = NULL;
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		if (psThisFreeList->ui32FreelistID == ui32FreelistID)
+		{
+			psFreeList = psThisFreeList;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockFreeList);
+	return psFreeList;
+}
+
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT32 ui32FreelistID)
+{
+	RGX_FREELIST *psFreeList = NULL;
+	RGXFWIF_KCCB_CMD s3DCCBCmd;
+	IMG_UINT32 ui32GrowValue;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	psFreeList = FindFreeList(psDevInfo, ui32FreelistID);
+	if (psFreeList == NULL)
+	{
+		/* Should never happen */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "FreeList Lookup for FreeList ID 0x%08x failed (Populate)",
+		         ui32FreelistID));
+		PVR_ASSERT(IMG_FALSE);
+		return;
+	}
+
+	/* Since the FW made the request, it has already consumed the ready pages, update the host struct */
+	psFreeList->ui32CurrentFLPages += psFreeList->ui32ReadyFLPages;
+	psFreeList->ui32ReadyFLPages = 0;
+
+	/* Try to grow the freelist */
+	eError = RGXGrowFreeList(psFreeList,
+	                         psFreeList->ui32GrowFLPages,
+	                         &psFreeList->sMemoryBlockHead);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Grow successful, return size of grow size */
+		ui32GrowValue = psFreeList->ui32GrowFLPages;
+
+		psFreeList->ui32NumGrowReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		/* Update Stats */
+		PVRSRVStatsUpdateFreelistStats(psDevInfo->psDeviceNode,
+		                               0,
+		                               1, /* Add 1 to the appropriate counter (Requests by FW) */
+		                               psFreeList->ui32InitFLPages,
+		                               psFreeList->ui32NumHighPages,
+		                               psFreeList->ownerPid);
+
+#endif
+
+	}
+	else
+	{
+		/* Grow failed */
+		ui32GrowValue = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Grow for FreeList %p failed (%s)",
+		         psFreeList,
+		         PVRSRVGetErrorString(eError)));
+	}
+
+	/* send feedback */
+	s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
+	s3DCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaPages = ui32GrowValue;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages = psFreeList->ui32ReadyFLPages;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: Grow pages=%u, new pages=%u, ready pages=%u, counter=%d",
+	         psFreeList,
+	         ui32GrowValue,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages,
+	         psFreeList->ui32NumGrowReqByFW));
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_3D,
+		                            &s3DCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+static void _RGXFreeListReconstruction(PDLLIST_NODE psNode)
+{
+
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	RGX_FREELIST			*psFreeList;
+	RGX_PMR_NODE			*psPMRNode;
+	PVRSRV_ERROR			eError;
+	IMG_DEVMEM_OFFSET_T		uiOffset;
+	IMG_DEVMEM_SIZE_T		uiLength;
+	IMG_UINT32				ui32StartPage;
+
+	psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	psFreeList = psPMRNode->psFreeList;
+	PVR_ASSERT(psFreeList);
+	psDevInfo = psFreeList->psDevInfo;
+	PVR_ASSERT(psDevInfo);
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((ui32StartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+	PMRUnwritePMPageList(psPMRNode->psPageList);
+	psPMRNode->psPageList = NULL;
+	eError = PMRWritePMPageList(
+			/* Target PMR, offset, and length */
+			psFreeList->psFreeListPMR,
+			(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+			(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+			/* Referenced PMR, and "page" granularity */
+			psPMRNode->psPMR,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			&psPMRNode->psPageList);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error (%s) writing FL 0x%08x",
+				__func__,
+				PVRSRVGetErrorString(eError),
+				(IMG_UINT32)psFreeList->ui32FreelistID));
+	}
+
+	/* Zeroing physical pages pointed by the reconstructed freelist */
+	if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
+}
+
+
+static PVRSRV_ERROR RGXReconstructFreeList(RGX_FREELIST *psFreeList)
+{
+	IMG_UINT32        ui32OriginalFLPages;
+	DLLIST_NODE       *psNode, *psNext;
+	RGXFWIF_FREELIST  *psFWFreeList;
+	PVRSRV_ERROR      eError;
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: Reconstructing freelist %p (ID=%u)", psFreeList, psFreeList->ui32FreelistID));
+
+	/* Do the FreeList Reconstruction */
+	ui32OriginalFLPages            = psFreeList->ui32CurrentFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+
+	/* Reconstructing Init FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Reconstructing Grow FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Ready pages are allocated but kept hidden until OOM occurs. */
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+	if (psFreeList->ui32CurrentFLPages != ui32OriginalFLPages)
+	{
+		PVR_ASSERT(psFreeList->ui32CurrentFLPages == ui32OriginalFLPages);
+		return PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED;
+	}
+
+	/* Update firmware freelist structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psFWFreeList->ui32MaxPages     = psFreeList->ui32MaxFLPages;
+	psFWFreeList->ui32GrowPages    = psFreeList->ui32GrowFLPages;
+	psFWFreeList->ui32CurrentPages = psFreeList->ui32CurrentFLPages;
+	psFWFreeList->ui32FreeListID   = psFreeList->ui32FreelistID;
+	psFWFreeList->bGrowPending     = IMG_FALSE;
+	psFWFreeList->ui32ReadyPages   = psFreeList->ui32ReadyFLPages;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psFWFreeList->ui32CurrentStackTop       = psFWFreeList->ui32CurrentPages - 1;
+	psFWFreeList->ui64CurrentDevVAddr       = (psFWFreeList->psFreeListDevVAddr.uiAddr +
+				((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+				~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+	psFWFreeList->ui32AllocatedPageCount    = 0;
+	psFWFreeList->ui32AllocatedMMUPageCount = 0;
+#else
+	psFWFreeList->bUpdatePending                   = IMG_FALSE;
+	psFWFreeList->ui32UpdateNewPages               = 0;
+	psFWFreeList->ui32UpdateNewReadyPages          = 0;
+	psFWFreeList->sFreeListLastGrowDevVAddr.uiAddr = 0;
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+#if !defined(PM_INTERACTIVE_MODE)
+	/* Reset freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER		sFLState;
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		eError = PMR_ReadBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+		/* Note: Freelist base address is packed shifted down. */
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, psFreeList->ui32CurrentFLPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+	}
+#endif
+
+	/* Check the Freelist checksum if required (as the list is fully populated) */
+	if (psFreeList->bCheckFreelist)
+	{
+		IMG_UINT64  ui64CheckSum;
+
+		_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+	}
+
+	return eError;
+}
+
+
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                              IMG_UINT32 ui32FreelistsCount,
+                                              const IMG_UINT32 *paui32Freelists)
+{
+	PVRSRV_ERROR      eError = PVRSRV_OK;
+	DLLIST_NODE       *psNode, *psNext;
+	IMG_UINT32        ui32Loop;
+	RGXFWIF_KCCB_CMD  sTACCBCmd;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE       *psNodeHWRTData, *psNextHWRTData;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+	RGXFWIF_HWRTDATA     *psHWRTData;
+#endif
+	IMG_UINT32        ui32PIDCount = 0;
+	IMG_UINT32        aui32PIDList[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+
+	PVR_ASSERT(psDevInfo != NULL);
+	PVR_ASSERT(ui32FreelistsCount <= RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT);
+	if (ui32FreelistsCount > RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT)
+	{
+		ui32FreelistsCount = RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT;
+	}
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: %u freelist(s) requested for reconstruction", ui32FreelistsCount));
+
+	/*
+	 *  Initialise the response command (in case we don't find a freelist ID).
+	 *  Also copy the list to the 'final' freelist array.
+	 */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
+	sTACCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
+
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] = paui32Freelists[ui32Loop] |
+				RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+	}
+
+	/*
+	 *  All freelists belonging to the same PID will be reconstructed.
+	 *  This simplifies tracking for AGP, since there is no longer a
+	 *  single global freelist per local freelist. The list of unique
+	 *  PIDs is calculated from the list of freelists in this first pass.
+	 */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList   = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+		{
+			IMG_UINT32 ui32PIDLoop;
+
+			if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+			{
+				for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+				{
+					if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+					{
+						break;
+					}
+				}
+
+				if (ui32PIDLoop == ui32PIDCount)
+				{
+					aui32PIDList[ui32PIDCount++] = psFreeList->ownerPid;
+				}
+			}
+		}
+	}
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList  = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		IMG_BOOL      bReconstruct = IMG_FALSE;
+		IMG_UINT32    ui32PIDLoop;
+
+		/*
+		 *  Check if this freelist needs to be reconstructed (is it in the list
+		 *  of PIDs which will have every single one of their freelists reconstructed)
+		 */
+		for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+		{
+			if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+			{
+				bReconstruct = IMG_TRUE;
+				break;
+			}
+		}
+
+		if (bReconstruct)
+		{
+			eError = RGXReconstructFreeList(psFreeList);
+			if (eError == PVRSRV_OK)
+			{
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+				/* Mark all HWRTData's of reconstructing local freelists as HWR (applies to TA/3D's not finished yet) */
+				dllist_foreach_node(&psFreeList->sNodeHWRTDataHead, psNodeHWRTData, psNextHWRTData)
+				{
+					psKMHWRTDataSet = IMG_CONTAINER_OF(psNodeHWRTData, RGX_KM_HW_RT_DATASET, sNodeHWRTData);
+					eError = DevmemAcquireCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc, (void **)&psHWRTData);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+									"Devmem AcquireCpuVirtAddr Failed during Reconstructing of FreeList, FwMemDesc(%p),psHWRTData(%p)",
+									psKMHWRTDataSet->psHWRTDataFwMemDesc,
+									psHWRTData));
+						continue;
+					}
+
+					psHWRTData->eState = RGXFWIF_RTDATA_STATE_HWR;
+					psHWRTData->ui32HWRTDataFlags &= ~HWRTDATA_HAS_LAST_TA;
+					RGXFwSharedMemCacheOpValue(psHWRTData->eState, FLUSH);
+					RGXFwSharedMemCacheOpValue(psHWRTData->ui32HWRTDataFlags, FLUSH);
+
+					DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+				}
+#endif
+
+				/* Update the response for this freelist if it was specifically requested for reconstruction. */
+				for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+				{
+					if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+					{
+						/* Reconstruction of this requested freelist was successful... */
+						sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &= ~RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+						break;
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Reconstructing of FreeList %p failed (%s)",
+						psFreeList,
+						PVRSRVGetErrorString(eError)));
+			}
+		}
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Check that all freelists were found and reconstructed... */
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		PVR_ASSERT((sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &
+		            RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG) == 0);
+	}
+
+	/* send feedback */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+/* Create a single HWRTData instance */
+static PVRSRV_ERROR RGXCreateHWRTData_aux(CONNECTION_DATA		*psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+                                          IMG_DEV_VIRTADDR		sPMDataDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR sPMSecureDataDevVAddr, /* per-HWRTData */
+                                          RGX_FREELIST			*apsFreeLists[RGXFW_MAX_FREELISTS],
+                                          IMG_DEV_VIRTADDR		sTailPtrsDevVAddr,
+                                          IMG_UINT16			ui16MaxRTs,
+                                          RGX_HWRTDATA_COMMON_COOKIE	*psHWRTDataCommonCookie,
+                                          RGX_KM_HW_RT_DATASET	**ppsKMHWRTDataSet) /* per-HWRTData */
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	/* KM cookie storing all the FW/HW data */
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+
+	/* local pointers for memory descriptors of FW allocations */
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc = NULL;
+
+	/* local pointer for CPU-mapped [FW]HWRTData */
+	RGXFWIF_HWRTDATA *psHWRTData = NULL;
+
+	/*
+	 * Fill in a local copy of RGXFWIF_HWRTDATA first to reduces reads and writes to
+	 * device memory. Using a byte array buffer avoids the limitation that some OSs
+	 * cannot align local variables to 64 bytes, usually needed by RGXFWIF_HWRTDATA
+	 */
+	IMG_BYTE aui8HWRTDataLocal[sizeof(RGXFWIF_HWRTDATA)] = {0};
+	RGXFWIF_HWRTDATA *psHWRTDataLocal = (RGXFWIF_HWRTDATA *)aui8HWRTDataLocal;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Prepare the HW RT DataSet struct */
+	psKMHWRTDataSet = OSAllocZMem(sizeof(*psKMHWRTDataSet));
+	if (psKMHWRTDataSet == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto AllocError;
+	}
+
+	*ppsKMHWRTDataSet = psKMHWRTDataSet;
+	psKMHWRTDataSet->psDeviceNode = psDeviceNode;
+	psKMHWRTDataSet->psHWRTDataCommonCookie = psHWRTDataCommonCookie;
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+	 * This FW RT-Data is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency and write-combine will
+	 * suffice on the CPU side. (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHwRTData",
+	                          &psHWRTDataFwMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: DevmemAllocate for RGX_FWIF_HWRTDATA failed",
+				__func__));
+		goto FWRTDataAllocateError;
+	}
+
+	psKMHWRTDataSet->psHWRTDataFwMemDesc = psHWRTDataFwMemDesc;
+	eError = RGXSetFirmwareAddress( &psKMHWRTDataSet->sHWRTDataFwAddr,
+							psHWRTDataFwMemDesc,
+							0,
+							RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", FWRTDataFwAddrError);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataFwMemDesc,
+									  (void **)&psHWRTData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
+#endif
+
+	psHWRTDataLocal->sPMSecureRenderStateDevVAddr = sPMSecureDataDevVAddr;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psPMMListDevVAddr = sPMDataDevVAddr;
+#else
+	psHWRTDataLocal->sPMRenderStateDevVAddr = sPMDataDevVAddr;
+#endif
+
+	psHWRTDataLocal->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
+
+	{
+		RGXFWIF_RTA_CTL *psRTACtl = &psHWRTDataLocal->sRTACtl;
+
+		psRTACtl->ui32RenderTargetIndex = 0;
+		psRTACtl->ui32ActiveRenderTargets = 0;
+		psRTACtl->sValidRenderTargets.ui32Addr = 0;
+		psRTACtl->sRTANumPartialRenders.ui32Addr = 0;
+		psRTACtl->ui32MaxRTs = (IMG_UINT32) ui16MaxRTs;
+
+		if (ui16MaxRTs > 1)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for shadow render target cache");
+			eError = DevmemFwAllocate(	psDevInfo,
+										ui16MaxRTs * sizeof(IMG_UINT32),
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwShadowRTCache",
+										&psRTArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTArryError;
+			}
+
+			psKMHWRTDataSet->psRTArrayFwMemDesc = psRTArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sValidRenderTargets,
+									psRTArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", FWAllocateRTArryFwAddrError);
+
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for tracking renders accumulation");
+			eError = DevmemFwAllocate(psDevInfo,
+			                          ui16MaxRTs * sizeof(IMG_UINT32),
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+					                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+					                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+					                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+					                  "FwRendersAccumulation",
+					                  &psRendersAccArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s) (renders accumulation)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTAccArryError;
+			}
+
+			psKMHWRTDataSet->psRendersAccArrayFwMemDesc = psRendersAccArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sRTANumPartialRenders,
+									psRendersAccArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", FWAllocRTAccArryFwAddrError);
+		}
+	}
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		psKMHWRTDataSet->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount++;
+		psHWRTDataLocal->apsFreeLists[ui32Loop].ui32Addr = psKMHWRTDataSet->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr;
+		/* Invalid initial snapshot value. The snapshot is always taken during the first
+		 * kick and hence this value gets replaced, so it's safe to set it to zero.
+		 */
+		psHWRTDataLocal->aui32FreeListHWRSnapshot[ui32Loop] = 0;
+		psHWRTDataLocal->bRenderStateNeedsReset = IMG_FALSE;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_add_to_tail(&apsFreeLists[RGXFW_LOCAL_FREELIST]->sNodeHWRTDataHead, &(psKMHWRTDataSet->sNodeHWRTData));
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	OSCachedMemCopy(psHWRTData, psHWRTDataLocal, sizeof(*psHWRTDataLocal));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTData 0x%08X", psKMHWRTDataSet->sHWRTDataFwAddr.ui32Addr);
+	DevmemPDumpLoadMem(psKMHWRTDataSet->psHWRTDataFwMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psHWRTData, FLUSH);
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	return PVRSRV_OK;
+
+FWAllocRTAccArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRendersAccArrayFwMemDesc);
+FWAllocateRTAccArryError:
+	RGXUnsetFirmwareAddress(psRTArrayFwMemDesc);
+FWAllocateRTArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRTArrayFwMemDesc);
+FWAllocateRTArryError:
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataCpuMapError:
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataAllocateError:
+	*ppsKMHWRTDataSet = NULL;
+	OSFreeMem(psKMHWRTDataSet);
+
+AllocError:
+	return eError;
+}
+
+static void RGXDestroyHWRTData_aux(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	if (psKMHWRTDataSet == NULL)
+	{
+		return;
+	}
+
+	psDevInfo = psKMHWRTDataSet->psDeviceNode->pvDevice;
+
+	if (psKMHWRTDataSet->psRTArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRTArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRTArrayFwMemDesc);
+	}
+
+	if (psKMHWRTDataSet->psRendersAccArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+	}
+
+	/* Decrease freelist refcount */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		PVR_ASSERT(psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount > 0);
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount--;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_remove_node(&psKMHWRTDataSet->sNodeHWRTData);
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Freeing the memory has to happen _after_ removing the HWRTData from the freelist
+	 * otherwise we risk traversing the freelist to find a pointer from a freed data structure */
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+
+	OSFreeMem(psKMHWRTDataSet);
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet(
+		CONNECTION_DATA			*psConnection,
+		PVRSRV_DEVICE_NODE		*psDeviceNode,
+		IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+		IMG_DEV_VIRTADDR		asPMDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR		asPMSecureDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+		RGX_FREELIST			*apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+		IMG_DEV_VIRTADDR		asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT16			ui16MaxRTs,
+		RGX_KM_HW_RT_DATASET	*pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psVHeapTableDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asPMDataDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asPMSecureDataDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(apsFreeLists);
+	PVR_UNREFERENCED_PARAMETER(asTailPtrsDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(ui16MaxRTs);
+	PVR_UNREFERENCED_PARAMETER(pasKMHWRTDataSet);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+_WritePMStateBuffer(PMR* psRenderStatesPMR,
+                    IMG_DEVMEM_OFFSET_T ui64Offset,
+                    IMG_DEV_VIRTADDR sMListDevVAddr,
+                    IMG_DEV_VIRTADDR sVHeapTableDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	size_t uiNbBytes;
+	IMG_UINT32 asRenderStateBuffer[RGX_PM_MAX_RSTATE_SIZE_DWORDS] = {0};
+
+	PVR_ASSERT(PVRSRV_IS_FEATURE_SUPPORTED(PMR_DeviceNode(psRenderStatesPMR), PM_BYTE_ALIGNED_BASE_ADDRESSES));
+	PVR_ASSERT((sMListDevVAddr.uiAddr & (RGX_PM_MLIST_BASE_ADDR_MAX_ALIGNSIZE - 1)) == 0);
+	PVR_ASSERT((sVHeapTableDevVAddr.uiAddr & (RGX_PM_VHEAP_BASE_ADDR_MAX_ALIGNSIZE - 1)) == 0);
+
+	/* Initialise the Mlist base and Vheap base */
+	_RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(asRenderStateBuffer, sMListDevVAddr.uiAddr);
+	_RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(asRenderStateBuffer, sVHeapTableDevVAddr.uiAddr);
+
+	eError = PMR_WriteBytes(psRenderStatesPMR,
+	                        ui64Offset,
+	                        (IMG_UINT8*)&asRenderStateBuffer,
+	                        sizeof(asRenderStateBuffer),
+	                        &uiNbBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from PMR_WriteBytes: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+		goto ErrorExit;
+	}
+
+	PVR_ASSERT(uiNbBytes == sizeof(asRenderStateBuffer));
+
+	PDUMPCOMMENT(PMR_DeviceNode(psRenderStatesPMR), "PMState buffer");
+	PMRPDumpLoadMem(psRenderStatesPMR, ui64Offset, sizeof(asRenderStateBuffer), PDUMP_FLAGS_CONTINUOUS, false);
+
+	return PVRSRV_OK;
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR RGXCreateHWRTDataSet2(
+        CONNECTION_DATA         *psConnection,
+        PVRSRV_DEVICE_NODE      *psDeviceNode,
+        IMG_DEV_VIRTADDR        sVHeapTableDevVAddr,
+        DEVMEMINT_RESERVATION   *psPMMListsReservation,
+        DEVMEMINT_RESERVATION   *psPMStateReservation,
+        DEVMEMINT_RESERVATION   *psPMSecureStateReservation,
+        RGX_FREELIST            *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+        IMG_DEV_VIRTADDR        asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        IMG_UINT16              ui16MaxRTs,
+        RGX_KM_HW_RT_DATASET    *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RTDataID;
+	IMG_UINT32 ui32GlobalFLMaxPages, ui32LocalFLMaxPages;
+	IMG_DEVMEM_SIZE_T ui64MListSize, ui64AlignedRenderStateSize;
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+
+	PMR *psPMStatePMR=NULL, *psPMSecureStatePMR=NULL, *psMListsPMR=NULL;
+	IMG_DEV_VIRTADDR sPMStateBaseDevVAddr, sPMSecureStateDevVAddr, sMListsDevVAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(0 < ui16MaxRTs && ui16MaxRTs <= RGX_MAX_RTA_INDICES, "Number of TA Render targets outside the range (0, RGX_MAX_RTA_INDICES) is unsupported");
+
+	ui64AlignedRenderStateSize = PVR_ALIGN(sizeof(RGX_PM_RENDERSTATE_BUFFER),
+	   MAX(RGX_PM_RENDERSTATE_BASE_ADDR_ALIGNSIZE,
+	       GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, SLC_CACHE_LINE_SIZE_BITS))));
+
+	/* Check if freelists have uniform sizes */
+	eError = ValidateFreeListSizes(apsFreeLists,
+	                               &ui32LocalFLMaxPages,
+	                               &ui32GlobalFLMaxPages);
+	PVR_LOG_RETURN_IF_ERROR(eError, "Invalid freelist sizes");
+
+	ui64MListSize = RGXCalcMListSize(psDeviceNode,
+	    ui32LocalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE,
+	    ui32GlobalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMStateReservation,
+	                                          ui64AlignedRenderStateSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psPMStatePMR,
+	                                          &sPMStateBaseDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	    "Validation failed for MListAndState reservations", err_validation_rs);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMSecureStateReservation,
+	                                          ui64AlignedRenderStateSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psPMSecureStatePMR,
+	                                          &sPMSecureStateDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	    "Validation failed for secure data reservations", err_validation_srs);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMMListsReservation,
+	                                          ui64MListSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psMListsPMR,
+	                                          &sMListsDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	    "Validation failed for secure data reservations", err_validation_mlist);
+
+	/* Prepare KM cleanup object for common data */
+	psHWRTDataCommonCookie = OSAllocZMem(sizeof(*psHWRTDataCommonCookie));
+	if (psHWRTDataCommonCookie == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_common_cookie_alloc;
+	}
+
+	psHWRTDataCommonCookie->psPMStateReservation = psPMStateReservation;
+	psHWRTDataCommonCookie->psPMSecureStateReservation = psPMSecureStateReservation;
+	psHWRTDataCommonCookie->psPMMListsReservation = psPMMListsReservation;
+
+	/* Here we are creating a set of HWRTData(s)
+	   the number of elements in the set equals RGXMKIF_NUM_RTDATAS.
+	*/
+	for (ui32RTDataID = 0; ui32RTDataID < RGXMKIF_NUM_RTDATAS; ui32RTDataID++)
+	{
+		IMG_DEV_VIRTADDR sRenderStateDevVirtAddr;
+		IMG_DEV_VIRTADDR sSecureRenderStateDevVirtAddr;
+		IMG_DEV_VIRTADDR sMListDevVirtAddr;
+
+		sRenderStateDevVirtAddr.uiAddr =
+		    sPMStateBaseDevVAddr.uiAddr + ui64AlignedRenderStateSize * ui32RTDataID;
+		sSecureRenderStateDevVirtAddr.uiAddr =
+		    sPMSecureStateDevVAddr.uiAddr + ui64AlignedRenderStateSize * ui32RTDataID;
+		sMListDevVirtAddr.uiAddr =
+		    sMListsDevVAddr.uiAddr + ui64MListSize * ui32RTDataID;
+
+		eError = _WritePMStateBuffer(psPMStatePMR,
+		                             ui64AlignedRenderStateSize * ui32RTDataID,
+		                             sMListDevVirtAddr,
+		                             sVHeapTableDevVAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError,
+		    "Failed to write PM state buffer", err_HWRTDataAlloc);
+
+		eError = RGXCreateHWRTData_aux(
+			psConnection,
+			psDeviceNode,
+			sVHeapTableDevVAddr,
+			sRenderStateDevVirtAddr,
+			sSecureRenderStateDevVirtAddr,
+			&apsFreeLists[(ui32RTDataID % RGXMKIF_NUM_GEOMDATAS) * RGXFW_MAX_FREELISTS],
+			asTailPtrsDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+			ui16MaxRTs,
+			psHWRTDataCommonCookie,
+			&pasKMHWRTDataSet[ui32RTDataID]);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to create HWRTData [slot %u] (%s)",
+					__func__,
+					ui32RTDataID,
+					PVRSRVGetErrorString(eError)));
+			goto err_HWRTDataAlloc;
+		}
+		psHWRTDataCommonCookie->ui32RefCount += 1;
+	}
+
+	return PVRSRV_OK;
+
+err_HWRTDataAlloc:
+	PVR_DPF((PVR_DBG_WARNING, "%s: err_HWRTDataAlloc", __func__));
+	if (pasKMHWRTDataSet)
+	{
+		for (; ui32RTDataID > 0; ui32RTDataID--)
+		{
+			if (pasKMHWRTDataSet[ui32RTDataID-1] != NULL)
+			{
+				RGXDestroyHWRTData_aux(pasKMHWRTDataSet[ui32RTDataID-1]);
+				pasKMHWRTDataSet[ui32RTDataID-1] = NULL;
+			}
+		}
+	}
+	OSFreeMem(psHWRTDataCommonCookie);
+err_common_cookie_alloc:
+	UnrefAndReleaseCriticalBuffer(psPMMListsReservation);
+err_validation_mlist:
+	UnrefAndReleaseCriticalBuffer(psPMSecureStateReservation);
+err_validation_srs:
+	UnrefAndReleaseCriticalBuffer(psPMStateReservation);
+err_validation_rs:
+	return eError;
+}
+
+/* Destroy a single instance of HWRTData.
+   Additionally, destroy the HWRTDataCommon{Cookie} objects
+   when it is the last HWRTData within a corresponding set of HWRTDatas.
+*/
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_ERROR eError;
+	PRGXFWIF_HWRTDATA psHWRTData;
+	RGX_HWRTDATA_COMMON_COOKIE *psCommonCookie;
+	PVR_ASSERT(psKMHWRTDataSet);
+
+	psDevNode = psKMHWRTDataSet->psDeviceNode;
+
+	eError = RGXSetFirmwareAddress(&psHWRTData,
+	                               psKMHWRTDataSet->psHWRTDataFwMemDesc, 0,
+	                               RFW_FWADDR_NOREF_FLAG);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Cleanup HWRTData */
+	eError = RGXFWRequestHWRTDataCleanUp(psDevNode, psHWRTData);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDevNode,
+						   eError,
+						   RGXFWRequestHWRTDataCleanUp);
+
+	psCommonCookie = psKMHWRTDataSet->psHWRTDataCommonCookie;
+
+	RGXDestroyHWRTData_aux(psKMHWRTDataSet);
+
+	/* We've got past potential PVRSRV_ERROR_RETRY events, so we are sure
+	   that the HWRTDATA instance will be destroyed during this call.
+	   Consequently, we decrease the ref count for HWRTDataCommonCookie.
+
+	   NOTE: This ref count does not require locks or atomics.
+	   -------------------------------------------------------
+	     HWRTDatas bound into one pair are always destroyed sequentially,
+	     within a single loop on the Client side.
+	     The Common/Cookie objects always belong to only one pair of
+	     HWRTDatas, and ref count is used to ensure that the Common/Cookie
+	     objects will be destroyed after destruction of all HWRTDatas
+	     within a single pair.
+	*/
+	psCommonCookie->ui32RefCount--;
+
+	/* When ref count for HWRTDataCommonCookie hits ZERO
+	 * we have to destroy the cookie
+	 * [KM object] afterwards. */
+	if (psCommonCookie->ui32RefCount == 0)
+	{
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMStateReservation);
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMSecureStateReservation);
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMMListsReservation);
+
+		OSFreeMem(psCommonCookie);
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA       *psConnection,
+                               PVRSRV_DEVICE_NODE     *psDeviceNode,
+                               IMG_HANDLE             hMemCtxPrivData,
+                               IMG_UINT32             ui32MaxFLPages,
+                               IMG_UINT32             ui32InitFLPages,
+                               IMG_UINT32             ui32GrowFLPages,
+                               IMG_UINT32             ui32GrowParamThreshold,
+                               RGX_FREELIST           *psGlobalFreeList,
+                               IMG_BOOL               bCheckFreelist,
+                               DEVMEMINT_RESERVATION *psFreeListAndStateReservation,
+                               RGX_FREELIST           **ppsFreeList)
+{
+	PVRSRV_ERROR        eError;
+	RGXFWIF_FREELIST    *psFWFreeList;
+	RGXFWIF_FREELIST    sFWFreeList = {{0}};
+	DEVMEM_MEMDESC      *psFWFreelistMemDesc;
+	RGX_FREELIST        *psFreeList;
+	PVRSRV_RGXDEV_INFO  *psDevInfo = psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR    sFreeListBaseDevVAddr;
+	IMG_DEV_VIRTADDR    sFreeListStateDevVAddr;
+	PMR*                psFreeListAndStatePMR = NULL;
+	IMG_UINT32          ui32ReadyPages;
+	IMG_UINT32          uiFreeListOffset = PVR_ALIGN(
+	                        sizeof(RGX_PM_FREELISTSTATE_BUFFER),
+	                        GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, SLC_CACHE_LINE_SIZE_BITS)));
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psFreeListAndStateReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Failed to acquire reservation for freelist buffer",
+		        __func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	eError = DevmemIntGetReservationData(psFreeListAndStateReservation, &psFreeListAndStatePMR, &sFreeListStateDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from DevmemIntGetReservationData: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+
+		goto ErrorAllocHost;
+	}
+
+	/* Check if client properly allocated PMMETA_PROTECT */
+	if ((PMR_Flags(psFreeListAndStatePMR) & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must have PMMETA_PROTECT set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psFreeListAndStatePMR)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must not have OS_LINUX_MOVABLE set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+#endif
+
+	/* Ref the PMR to prevent resource being destroyed before use */
+	eError = PMRRefPMR(psFreeListAndStatePMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorAllocHost);
+
+	if (PMR_IsSparse(psFreeListAndStatePMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Free list PMR cannot be sparse!",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto ErrorRefPMR;
+	}
+
+	sFreeListBaseDevVAddr.uiAddr = sFreeListStateDevVAddr.uiAddr + uiFreeListOffset;
+
+	if (OSGetPageShift() > RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+	{
+		IMG_UINT32 ui32Size, ui32NewInitFLPages, ui32NewMaxFLPages, ui32NewGrowFLPages;
+
+		/* Round up number of FL pages to the next multiple of the OS page size */
+
+		ui32Size = ui32InitFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewInitFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32GrowFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewGrowFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32MaxFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewMaxFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Increased number of PB pages: Init %u -> %u, Grow %u -> %u, Max %u -> %u",
+		         __func__, ui32InitFLPages, ui32NewInitFLPages, ui32GrowFLPages, ui32NewGrowFLPages, ui32MaxFLPages, ui32NewMaxFLPages));
+
+		ui32InitFLPages = ui32NewInitFLPages;
+		ui32GrowFLPages = ui32NewGrowFLPages;
+		ui32MaxFLPages = ui32NewMaxFLPages;
+	}
+
+	/* Allocate kernel freelist struct */
+	psFreeList = OSAllocZMem(sizeof(*psFreeList));
+	if (psFreeList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate host data structure",
+		        __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRefPMR;
+	}
+
+	/*
+	 * This FW FreeList context is only mapped into kernel for initialisation
+	 * and reconstruction (at other times it is not mapped and only used by the
+	 * FW).
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(*psFWFreeList),
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+	                          PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwFreeList",
+	                          &psFWFreelistMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: DevmemAllocate for RGXFWIF_FREELIST failed",
+		        __func__));
+		goto FWFreeListAlloc;
+	}
+
+	/* Initialise host data structures */
+	psFreeList->psDevInfo = psDevInfo;
+	psFreeList->psConnection = psConnection;
+
+	psFreeList->psFreeListStatePMR = psFreeListAndStatePMR;
+	psFreeList->uiFreeListStatePMROffset = 0;
+
+	psFreeList->psFreeListPMR = psFreeListAndStatePMR;
+	psFreeList->uiFreeListPMROffset = uiFreeListOffset;
+
+	psFreeList->psFreeListAndStateReservation = psFreeListAndStateReservation;
+
+	psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
+	psFreeList->sFreeListBaseDevVAddr = sFreeListBaseDevVAddr;
+	eError = RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	/* psFreeList->ui32FreelistID set below with lock... */
+	psFreeList->ui32FreelistGlobalID = (psGlobalFreeList ? psGlobalFreeList->ui32FreelistID : 0);
+	psFreeList->ui32MaxFLPages = ui32MaxFLPages;
+	psFreeList->ui32InitFLPages = ui32InitFLPages;
+	psFreeList->ui32GrowFLPages = ui32GrowFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+	psFreeList->ui32ReadyFLPages = 0;
+	psFreeList->ui32GrowThreshold = ui32GrowParamThreshold;
+	psFreeList->ui64FreelistChecksum = 0;
+	psFreeList->ui32RefCount = 0;
+	psFreeList->bCheckFreelist = bCheckFreelist;
+	dllist_init(&psFreeList->sMemoryBlockHead);
+	dllist_init(&psFreeList->sMemoryBlockInitHead);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_init(&psFreeList->sNodeHWRTDataHead);
+#endif
+	psFreeList->ownerPid = OSGetCurrentClientProcessIDKM();
+
+	/* Add to list of freelists */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
+	dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	ui32ReadyPages = _CalculateFreelistReadyPages(psFreeList, ui32InitFLPages);
+
+	/* Write freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER sFLState = {0};
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - ui32InitFLPages + ui32ReadyPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+
+
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, (sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT));
+
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, ui32InitFLPages - ui32ReadyPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Error from PMR_WriteBytes: %s",
+			        __func__, PVRSRVGetErrorString(eError)));
+			goto FWFreeListAlloc;
+		}
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		PDUMPCOMMENT(psDeviceNode, "FreeListState buffer");
+		PMRPDumpLoadMem(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, sizeof(sFLState), PDUMP_FLAGS_CONTINUOUS, false);
+	}
+
+	/* Initialise FW data structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
+
+	{
+		sFWFreeList.ui32MaxPages = ui32MaxFLPages;
+		sFWFreeList.ui32CurrentPages = ui32InitFLPages - ui32ReadyPages;
+		sFWFreeList.ui32GrowPages = ui32GrowFLPages;
+		sFWFreeList.bUpdatePending = IMG_FALSE;
+		sFWFreeList.ui32UpdateNewPages = 0;
+		sFWFreeList.ui32UpdateNewReadyPages = 0;
+		sFWFreeList.sFreeListBaseDevVAddr = sFreeListBaseDevVAddr;
+#if defined(PM_INTERACTIVE_MODE)
+		sFWFreeList.ui32CurrentStackTop = sFWFreeList.ui32CurrentPages - 1;
+		sFWFreeList.ui64CurrentDevVAddr = (sFreeListBaseDevVAddr.uiAddr +
+		                                     ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+		                                    ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+#endif
+		sFWFreeList.ui32FreeListID = psFreeList->ui32FreelistID;
+		sFWFreeList.bGrowPending = IMG_FALSE;
+		sFWFreeList.ui32ReadyPages = ui32ReadyPages;
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+		/* Get the FW Memory Context address... */
+		eError = RGXSetFirmwareAddress(&sFWFreeList.psFWMemContext,
+		                               RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData),
+		                               0, RFW_FWADDR_NOREF_FLAG);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: RGXSetFirmwareAddress for RGXFWIF_FWMEMCONTEXT failed",
+			        __func__));
+			DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+			goto FWFreeListCpuMap;
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(hMemCtxPrivData);
+#endif
+
+		/*
+		 * Only the PM state buffer address is needed which contains the PM
+		 * state including the freelist base address.
+		 *
+		 * Access to the physical PMR will be used to update the contents of the
+		 * PM state buffer when PB grow occurs following OOM.
+		 */
+		sFWFreeList.sFreeListLastGrowDevVAddr.uiAddr = 0;
+		sFWFreeList.sFreeListStateDevVAddr = sFreeListStateDevVAddr;
+	}
+
+	OSCachedMemCopy(psFWFreeList, &sFWFreeList, sizeof(sFWFreeList));
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, "
+	        "FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Current FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Current pages %u",
+	        psFreeList,
+	        ui32MaxFLPages,
+	        ui32InitFLPages,
+	        sFreeListBaseDevVAddr.uiAddr,
+	        (sFreeListBaseDevVAddr.uiAddr +
+	            ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+	        ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1),
+	        sFWFreeList.ui32CurrentPages - 1));
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FW FreeList");
+	DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(PM_INTERACTIVE_MODE)
+	/*
+	 * Separate dump of the Freelist's number of Pages and stack pointer.
+	 * This allows to easily modify the PB size in the out2.txt files.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "FreeList TotalPages");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
+	                          sFWFreeList.ui32CurrentPages,
+	                          PDUMP_FLAGS_CONTINUOUS);
+	PDUMPCOMMENT(psDeviceNode, "FreeList StackPointer");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentStackTop),
+	                          sFWFreeList.ui32CurrentStackTop,
+	                          PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+	/* Add initial PB block */
+	eError = RGXGrowFreeList(psFreeList,
+	                         ui32InitFLPages,
+	                         &psFreeList->sMemoryBlockInitHead);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate initial memory block for free list 0x%016" IMG_UINT64_FMTSPECx " (%s)",
+		        __func__,
+		        sFreeListBaseDevVAddr.uiAddr,
+		        PVRSRVGetErrorString(eError)));
+		goto FWFreeListCpuMap;
+	}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Update Stats */
+	PVRSRVStatsUpdateFreelistStats(psDeviceNode,
+	                               1, /* Add 1 to the appropriate counter (Requests by App)*/
+	                               0,
+	                               psFreeList->ui32InitFLPages,
+	                               psFreeList->ui32NumHighPages,
+	                               psFreeList->ownerPid);
+
+#endif
+
+	/* return values */
+	*ppsFreeList = psFreeList;
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+FWFreeListCpuMap:
+	/* Remove freelists from list */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+	RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
+
+ErrorSetFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psFWFreelistMemDesc);
+
+FWFreeListAlloc:
+	OSFreeMem(psFreeList);
+
+ErrorRefPMR:
+	(void) PMRUnrefPMR(psFreeListAndStatePMR);
+
+ErrorAllocHost:
+	DevmemIntReservationRelease(psFreeListAndStateReservation);
+
+ErrorReservationAcquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyFreeList
+*/
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RefCount;
+
+	PVR_ASSERT(psFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	ui32RefCount = psFreeList->ui32RefCount;
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	if (ui32RefCount != 0)
+	{
+		/* Freelist still busy */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Freelist is not in use => start firmware cleanup */
+	eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
+										 psFreeList->sFreeListFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psFreeList->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestFreeListCleanUp);
+
+	/* Device becomes unrecoverable if the firmware took too long to
+	 * handle the cleanup request, or if SLC-flushes didn't go through
+	 * (due to some GPU lockup) */
+
+	/* Remove FreeList from linked list before we destroy it... */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* Confirm all HWRTData nodes are freed before releasing freelist */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sNodeHWRTDataHead));
+#endif
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+#if defined(PM_INTERACTIVE_MODE)
+	if (psFreeList->bCheckFreelist)
+	{
+		RGXFWIF_FREELIST  *psFWFreeList;
+		IMG_UINT32        ui32CurrentStackTop;
+		IMG_UINT64        ui64CheckSum;
+
+		/* Get the current stack pointer for this free list */
+		DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+		RGXFwSharedMemCacheOpValue(psFWFreeList->ui32CurrentStackTop, INVALIDATE);
+		ui32CurrentStackTop = psFWFreeList->ui32CurrentStackTop;
+		DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+		if (ui32CurrentStackTop > psFreeList->ui32MaxFLPages)
+		{
+			PVR_LOG(("%s: FW freelist corrupted (%d)",
+					 __func__,
+					 ui32CurrentStackTop));
+		}
+		else if (ui32CurrentStackTop == psFreeList->ui32CurrentFLPages-1)
+		{
+			/* Do consistency tests (as the list is fully populated) */
+			_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+		}
+		else
+		{
+			/* Check for duplicate pages, but don't check the checksum as the list is not fully populated */
+			_CheckFreelist(psFreeList, ui32CurrentStackTop+1, 0, &ui64CheckSum);
+		}
+	}
+#endif
+
+	/* Destroy FW structures */
+	RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
+	DevmemFwUnmapAndFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);
+
+	/* Remove grow shrink blocks */
+	while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
+	{
+		eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* Remove initial PB block */
+	eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* consistency checks */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
+	PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
+
+	/* Remove references from the PMR resources */
+	eError = PMRUnrefPMR(psFreeList->psFreeListStatePMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	DevmemIntReservationRelease(psFreeList->psFreeListAndStateReservation);
+
+	/* free Freelist */
+	OSFreeMem(psFreeList);
+
+	return eError;
+}
+
+
+/*
+	RGXCreateZSBuffer
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_PRBUFFER			*psFWZSBuffer;
+	RGXFWIF_PRBUFFER			sFWZSBuffer = {0};
+	RGX_ZSBUFFER_DATA			*psZSBuffer;
+	DEVMEM_MEMDESC				*psFWZSBufferMemDesc;
+	IMG_BOOL					bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiMapFlags) ? IMG_TRUE : IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Allocate host data structure */
+	psZSBuffer = OSAllocZMem(sizeof(*psZSBuffer));
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate cleanup data structure for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocCleanup;
+	}
+
+	/* Populate Host data */
+	psZSBuffer->psDevInfo = psDevInfo;
+	psZSBuffer->psReservation = psReservation;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psZSBuffer->psReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to acquire reservation for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	psZSBuffer->psPMR = psPMR;
+	/* Obtain reference to PMR */
+	eError = PMRRefPMR(psZSBuffer->psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorRefPMR);
+
+	psZSBuffer->ui32RefCount = 0;
+	psZSBuffer->bIsBacked = IMG_FALSE;
+	psZSBuffer->bOnDemand = bOnDemand;
+
+	/* Allocate firmware memory for ZS-Buffer. */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware ZS-Buffer data structure");
+	eError = DevmemFwAllocate(psDevInfo,
+							sizeof(*psFWZSBuffer),
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+							PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+							"FwZSBuffer",
+							&psFWZSBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAllocFWZSBuffer;
+	}
+	psZSBuffer->psFWZSBufferMemDesc = psFWZSBufferMemDesc;
+
+	/* Temporarily map the firmware render context to the kernel. */
+	eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
+	                                  (void **)&psFWZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAcquireFWZSBuffer;
+	}
+
+	/* Get firmware address of ZS-Buffer. */
+	eError = RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	if (bOnDemand)
+	{
+		/* psZSBuffer->ui32ZSBufferID set below with lock... */
+		OSLockAcquire(psDevInfo->hLockZSBuffer);
+		psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
+		dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
+		OSLockRelease(psDevInfo->hLockZSBuffer);
+	}
+
+	/* Populate FW ZS-Buffer data structure */
+	sFWZSBuffer.bOnDemand = bOnDemand;
+	sFWZSBuffer.eState = (bOnDemand) ? RGXFWIF_PRBUFFER_UNBACKED : RGXFWIF_PRBUFFER_BACKED;
+	sFWZSBuffer.ui32BufferID = psZSBuffer->ui32ZSBufferID;
+
+	OSCachedMemCopy(psFWZSBuffer, &sFWZSBuffer, sizeof(sFWZSBuffer));
+	/* Dump the ZS-Buffer and the memory content */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump firmware ZS-Buffer");
+	DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWZSBuffer, FLUSH);
+
+	/* Release address acquired above. */
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+
+
+	/* define return value */
+	*ppsZSBuffer = psZSBuffer;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
+							psZSBuffer,
+							(bOnDemand) ? "On-Demand": "Up-front"));
+
+	psZSBuffer->owner=OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+
+	/* error handling */
+
+ErrorSetFwAddr:
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+ErrorAcquireFWZSBuffer:
+	DevmemFwUnmapAndFree(psDevInfo, psFWZSBufferMemDesc);
+
+ErrorAllocFWZSBuffer:
+	(void) PMRUnrefPMR(psZSBuffer->psPMR);
+ErrorRefPMR:
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+ErrorReservationAcquire:
+	OSFreeMem(psZSBuffer);
+
+ErrorAllocCleanup:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyZSBuffer
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError, eError2;
+
+	PVR_ASSERT(psZSBuffer);
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	/* Request ZS Buffer cleanup */
+	eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
+										psZSBuffer->sZSBufferFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psZSBuffer->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestZSBufferCleanUp);
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->ui32RefCount != 0)
+	{
+		/* ZS-Buffer is still referenced */
+		OSLockRelease(hLockZSBuffer);
+		PVR_DPF((PVR_DBG_WARNING, "Trying to destroy a ZS-Buffer [%p] that's still in use.", psZSBuffer));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Free the firmware render context. */
+	RGXUnsetFirmwareAddress(psZSBuffer->psFWZSBufferMemDesc);
+	DevmemFwUnmapAndFree(psZSBuffer->psDevInfo, psZSBuffer->psFWZSBufferMemDesc);
+
+	/* Remove Deferred Allocation from list */
+	if (psZSBuffer->bOnDemand)
+	{
+		PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
+		dllist_remove_node(&psZSBuffer->sNode);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] destroyed", psZSBuffer));
+
+	/* Release reference to reservation object and the PMR */
+	eError2 = PMRUnrefPMR(psZSBuffer->psPMR);
+	PVR_LOG_IF_ERROR(eError2, "PMRUnrefPMR");
+
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+
+	/* Free ZS-Buffer host data structure */
+	OSFreeMem(psZSBuffer);
+
+	OSLockRelease(hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (!psZSBuffer->bOnDemand)
+	{
+		/* Only deferred allocations can be populated */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bIsBacked == IMG_FALSE)
+	{
+		IMG_HANDLE hDevmemHeap;
+
+		/* Get Heap */
+		eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
+		if (unlikely(hDevmemHeap == (IMG_HANDLE)NULL))
+		{
+			OSLockRelease(hLockZSBuffer);
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+
+		eError = DevmemIntMapPMR(psZSBuffer->psReservation, psZSBuffer->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable populate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+
+		}
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
+		                          psZSBuffer,
+		                          psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_TRUE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+					RGX_POPULATION **ppsPopulation)
+{
+	RGX_POPULATION *psPopulation;
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount++;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	psZSBuffer->ui32NumReqByApp++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psZSBuffer->psDevInfo->psDeviceNode,
+	                               1, 0, psZSBuffer->owner);
+#endif
+
+	/* Do the backing */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		goto OnErrorBacking;
+	}
+
+	/* Create the handle to the backing */
+	psPopulation = OSAllocMem(sizeof(*psPopulation));
+	if (psPopulation == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto OnErrorAlloc;
+	}
+
+	psPopulation->psZSBuffer = psZSBuffer;
+
+	/* return value */
+	*ppsPopulation = psPopulation;
+
+	return PVRSRV_OK;
+
+OnErrorAlloc:
+	RGXUnbackingZSBuffer(psZSBuffer);
+
+OnErrorBacking:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bOnDemand && psZSBuffer->bIsBacked == IMG_TRUE)
+	{
+		eError = DevmemIntUnmapPMR(psZSBuffer->psReservation);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
+									psZSBuffer,
+									psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_FALSE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
+{
+	PVRSRV_ERROR eError;
+	POS_LOCK hLockZSBuffer;
+
+	if (!psPopulation)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	hLockZSBuffer = psPopulation->psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	OSLockAcquire(hLockZSBuffer);
+	psPopulation->psZSBuffer->ui32RefCount--;
+	OSLockRelease(hLockZSBuffer);
+
+	OSFreeMem(psPopulation);
+
+	return PVRSRV_OK;
+}
+
+static RGX_ZSBUFFER_DATA *FindAndRefZSBuffer(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32ZSBufferID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_ZSBUFFER_DATA *psZSBuffer = NULL;
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+
+	dllist_foreach_node(&psDevInfo->sZSBufferHead, psNode, psNext)
+	{
+		RGX_ZSBUFFER_DATA *psThisZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
+
+		if (psThisZSBuffer->ui32ZSBufferID == ui32ZSBufferID)
+		{
+			psZSBuffer = psThisZSBuffer;
+
+			psZSBuffer->ui32RefCount++;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+	return psZSBuffer;
+}
+
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID)
+{
+	IMG_BOOL bBackingDone = IMG_TRUE;
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Populate ZLS */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Populating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		bBackingDone = IMG_FALSE;
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psZSBuffer->ui32NumReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psDevInfo->psDeviceNode,
+	                               0, 1, psZSBuffer->owner);
+#endif
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+
+}
+
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32 ui32ZSBufferID)
+{
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Unpopulate ZLS */
+	eError = RGXUnbackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"UnPopulating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		PVR_ASSERT(IMG_FALSE);
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+}
+
+static
+PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              SERVER_MMU_CONTEXT *psServerMMUContext,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_TA_DATA *psTAData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TA context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_TACTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwTAContextState",
+							  &psTAData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TA,
+	                                 RGXFWIF_DM_GEOM,
+	                                 psServerMMUContext,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 psTAData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TA_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TA_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &psTAData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init TA fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump the TA context suspend state buffer");
+	DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TACTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	psTAData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_tacommoncontext:
+	DevmemFree(psTAData->psContextStateMemDesc);
+fail_tacontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              SERVER_MMU_CONTEXT *psServerMMUContext,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_3D_DATA *ps3DData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	uiNumISPStoreRegs;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware 3D context suspend state");
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	uiNumISPStoreRegs = RGXFWIF_IPP_RESUME_REG_COUNT; /* default 1 register for IPP_resume */
+
+	uiNumISPStoreRegs += (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU) *
+								RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_PER_SPU) *
+								RGXFWIF_PIPE_COUNT_PER_ISP);
+
+
+	if (uiNumISPStoreRegs > (RGXFWIF_ISP_PIPE_COUNT_MAX + RGXFWIF_IPP_RESUME_REG_COUNT))
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+#else /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+	/* Size of the CS buffer */
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			(uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]));
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui3DRegISPStateStoreSize,
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "Fw3DContextState",
+							  &ps3DData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_3D,
+	                                 RGXFWIF_DM_3D,
+	                                 psServerMMUContext,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init 3D fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Dump the 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_3DCTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_3dcommoncontext:
+	DevmemFree(ps3DData->psContextStateMemDesc);
+fail_3dcontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVRGXCreateRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+                                            PVRSRV_DEVICE_NODE			*psDeviceNode,
+                                            IMG_INT32					i32Priority,
+                                            IMG_UINT32					ui32FrameworkRegisterSize,
+                                            IMG_PBYTE					pabyFrameworkRegisters,
+                                            IMG_HANDLE					hMemCtxPrivData,
+                                            IMG_UINT32					ui32StaticRenderContextStateSize,
+                                            IMG_PBYTE					pStaticRenderContextState,
+                                            IMG_UINT32					ui32PackedCCBSizeU8888,
+                                            IMG_UINT32					ui32ContextFlags,
+                                            IMG_UINT64					ui64RobustnessAddress,
+                                            IMG_UINT32					ui32MaxTADeadlineMS,
+                                            IMG_UINT32					ui32Max3DDeadlineMS,
+                                            RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RENDER_CONTEXT	*psRenderContext;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	RGXFWIF_FWRENDERCONTEXT		*psFWRenderContext;
+
+	*ppsRenderContext = NULL;
+
+	if (ui32StaticRenderContextStateSize > RGXFWIF_STATIC_RENDERCONTEXT_SIZE)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRenderContext = OSAllocZMem(sizeof(*psRenderContext));
+	if (psRenderContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psRenderContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_lock;
+	}
+
+	psRenderContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Create the FW render context, this has the TA and 3D FW common
+		contexts embedded within it
+	*/
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_FWRENDERCONTEXT),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwRenderContext",
+							  &psRenderContext->psFWRenderContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwrendercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkRegisterSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psRenderContext->psFWFrameworkMemDesc,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psRenderContext->psFWFrameworkMemDesc,
+				pabyFrameworkRegisters,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+		sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DContext(psConnection,
+	                          psDeviceNode,
+	                          hMemCtxPrivData,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32Max3DDeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->s3DData,
+	                          U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcontext;
+	}
+
+	eError = _CreateTAContext(psConnection,
+	                          psDeviceNode,
+	                          hMemCtxPrivData,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32MaxTADeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->sTAData,
+	                          U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tacontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+			(void **)&psFWRenderContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	/* Copy the static render context data */
+	OSDeviceMemCopy(&psFWRenderContext->sStaticRenderContextState, pStaticRenderContextState, ui32StaticRenderContextStateSize);
+#if defined(SUPPORT_TRP)
+	psFWRenderContext->eTRPGeomCoreAffinity = RGXFWIF_DM_MAX;
+#endif
+	DevmemPDumpLoadMem(psRenderContext->psFWRenderContextMemDesc, 0, sizeof(RGXFWIF_FWRENDERCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpPtr(psFWRenderContext, FLUSH);
+	DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psRenderContext->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-ta3d");
+	if (IS_ERR(psRenderContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(psRenderContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	}
+
+	*ppsRenderContext = psRenderContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _DestroyTAContext(&psRenderContext->sTAData,
+		                                         psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_tacontext:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _Destroy3DContext(&psRenderContext->s3DData,
+		                                         psRenderContext->psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_3dcontext:
+fail_frameworkcopy:
+	if (psRenderContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+fail_fwrendercontext:
+	OSLockDestroy(psRenderContext->hLock);
+fail_lock:
+	OSFreeMem(psRenderContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+ * PVRSRVRGXDestroyRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psRenderContext->psDeviceNode->pvDevice;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_remove_node(&(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* Check psBufferSyncContext has not been destroyed already (by a previous
+	 * call to this function which then later returned PVRSRV_ERROR_RETRY)
+	 */
+	if (psRenderContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psRenderContext->psBufferSyncContext);
+		psRenderContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+	/* Cleanup the TA if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
+	{
+		eError = _DestroyTAContext(&psRenderContext->sTAData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+	/* Cleanup the 3D if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
+	{
+		eError = _Destroy3DContext(&psRenderContext->s3DData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+	{
+		RGXFWIF_FWRENDERCONTEXT	*psFWRenderContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+				(void **)&psFWRenderContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware render context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+		RGXFwSharedMemCacheOpValue(psFWRenderContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+
+		ui32WorkEstCCBSubmitted = psFWRenderContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRenderContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRenderContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			eError = PVRSRV_ERROR_RETRY;
+			goto e0;
+		}
+	}
+#endif
+
+	/*
+		Only if both TA and 3D contexts have been cleaned up can we
+		free the shared resources
+	*/
+	if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
+	{
+		if (psRenderContext->psFWFrameworkMemDesc)
+		{
+			/* Free the framework buffer */
+			DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+		}
+
+		/* Free the firmware render context */
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			WorkEstDeInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+		}
+#endif
+
+		OSLockDestroy(psRenderContext->hLock);
+
+		OSFreeMem(psRenderContext);
+	}
+
+	return PVRSRV_OK;
+
+e0:
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	return eError;
+}
+
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+static void DumpUfoList(IMG_UINT32 ui32ClientTAFenceCount,
+                        IMG_UINT32 ui32ClientTAUpdateCount,
+                        IMG_UINT32 ui32Client3DFenceCount,
+                        IMG_UINT32 ui32Client3DUpdateCount,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAFenceUFOAddress,
+                        IMG_UINT32 *paui32ClientTAFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAUpdateUFOAddress,
+                        IMG_UINT32 *paui32ClientTAUpdateValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DFenceUFOAddress,
+                        IMG_UINT32 *paui32Client3DFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DUpdateUFOAddress,
+                        IMG_UINT32 *paui32Client3DUpdateValue)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: ~~~ After populating sync prims ~~~",
+			 __func__));
+
+	/* Dump Fence syncs, Update syncs and PR Update syncs */
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA fence syncs:",
+	        __func__, ui32ClientTAFenceCount));
+	for (i = 0; i < ui32ClientTAFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr,
+			        *paui32ClientTAFenceValue,
+			        *paui32ClientTAFenceValue));
+			paui32ClientTAFenceValue++;
+		}
+		pauiClientTAFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA update syncs:",
+			 __func__, ui32ClientTAUpdateCount));
+	for (i = 0; i < ui32ClientTAUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr,
+			        *paui32ClientTAUpdateValue,
+			        *paui32ClientTAUpdateValue));
+			paui32ClientTAUpdateValue++;
+		}
+		pauiClientTAUpdateUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D fence syncs:",
+			 __func__, ui32Client3DFenceCount));
+	for (i = 0; i < ui32Client3DFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr,
+			        *paui32Client3DFenceValue,
+			        *paui32Client3DFenceValue));
+			paui32Client3DFenceValue++;
+		}
+		pauiClient3DFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D update syncs:",
+			 __func__, ui32Client3DUpdateCount));
+	for (i = 0; i < ui32Client3DUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr,
+			        *paui32Client3DUpdateValue,
+			        *paui32Client3DUpdateValue));
+			paui32Client3DUpdateValue++;
+		}
+		pauiClient3DUpdateUFOAddress++;
+	}
+}
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+/*
+ * PVRSRVRGXKickTA3DKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRFenceSyncPrimBlock,
+								 IMG_UINT32					ui32PRFenceSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckTAFence,
+								 PVRSRV_TIMELINE			iUpdateTATimeline,
+								 PVRSRV_FENCE				*piUpdateTAFence,
+								 IMG_CHAR					szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheck3DFence,
+								 PVRSRV_TIMELINE			iUpdate3DTimeline,
+								 PVRSRV_FENCE				*piUpdate3DFence,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus)
+{
+	/* per-context helper structures */
+	RGX_CCB_CMD_HELPER_DATA *pasTACmdHelperData = psRenderContext->asTACmdHelperData;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelperData = psRenderContext->as3DCmdHelperData;
+
+	IMG_UINT64                ui64FBSCEntryMask;
+
+	IMG_UINT32                ui32TACmdCount=0;
+	IMG_UINT32                ui323DCmdCount=0;
+	__maybe_unused IMG_UINT32 ui32TACmdOffset=0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset=0;
+
+	RGXFWIF_UFO        sPRUFO;
+	IMG_UINT32         i;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	PVRSRV_ERROR       eError2 = PVRSRV_OK;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psRenderContext->s3DData.psServerCommonContext);
+	IMG_UINT32         ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_BOOL           bCCBStateOpen = IMG_FALSE;
+
+	IMG_UINT32         ui32ClientPRUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR  *pauiClientPRUpdateUFOAddress = NULL;
+	IMG_UINT32         *paui32ClientPRUpdateValue = NULL;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	PRGXFWIF_UFO_ADDR  *pauiClientTAFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClientTAUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  uiPRFenceUFOAddress;
+
+	IMG_UINT64         uiCheckTAFenceUID = 0;
+	IMG_UINT64         uiCheck3DFenceUID = 0;
+	IMG_UINT64         uiUpdateTAFenceUID = 0;
+	IMG_UINT64         uiUpdate3DFenceUID = 0;
+
+	IMG_BOOL bUseCombined3DAnd3DPR = bKickPR && bKick3D && !pui83DPRDMCmd;
+
+	RGXFWIF_KCCB_CMD_KICK_DATA sTACmdKickData;
+	RGXFWIF_KCCB_CMD_KICK_DATA s3DCmdKickData;
+	IMG_BOOL bUseSingleFWCommand = bKickTA && (bKickPR || bKick3D);
+
+	IMG_UINT32 ui32TACmdSizeTmp = 0, ui323DCmdSizeTmp = 0;
+
+	IMG_BOOL bTAFenceOnSyncCheckpointsOnly = IMG_FALSE;
+
+	PVRSRV_FENCE iUpdateTAFence = PVRSRV_NO_FENCE;
+	PVRSRV_FENCE iUpdate3DFence = PVRSRV_NO_FENCE;
+
+	IMG_BOOL   b3DFenceOnSyncCheckpointsOnly = IMG_FALSE;
+	IMG_UINT32 ui32TAFenceTimelineUpdateValue = 0;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+
+	/*
+	 * Count of the number of TA and 3D update values (may differ from number of
+	 * TA and 3D updates later, as sync checkpoints do not need to specify a value)
+	 */
+	IMG_UINT32 ui32ClientPRUpdateValueCount = 0;
+	IMG_UINT32 ui32ClientTAUpdateValueCount = ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32Client3DUpdateValueCount = ui32Client3DUpdateCount;
+	PSYNC_CHECKPOINT *apsFenceTASyncCheckpoints = NULL;				/*!< TA fence checkpoints */
+	PSYNC_CHECKPOINT *apsFence3DSyncCheckpoints = NULL;				/*!< 3D fence checkpoints */
+	IMG_UINT32 ui32FenceTASyncCheckpointCount = 0;
+	IMG_UINT32 ui32Fence3DSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psUpdateTASyncCheckpoint = NULL;				/*!< TA update checkpoint (output) */
+	PSYNC_CHECKPOINT psUpdate3DSyncCheckpoint = NULL;				/*!< 3D update checkpoint (output) */
+	PVRSRV_CLIENT_SYNC_PRIM *psTAFenceTimelineUpdateSync = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	void *pvTAUpdateFenceFinaliseData = NULL;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+
+	RGX_SYNC_DATA sTASyncData = {NULL};		/*!< Contains internal update syncs for TA */
+	RGX_SYNC_DATA s3DSyncData = {NULL};		/*!< Contains internal update syncs for 3D */
+
+	IMG_BOOL bTestSLRAdd3DCheck = IMG_FALSE;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTA = {0};
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickData3D = {0};
+	IMG_UINT32 ui32TACommandOffset = 0;
+	IMG_UINT32 ui323DCommandOffset = 0;
+	IMG_UINT32 ui32TACmdHeaderOffset = 0;
+	IMG_UINT32 ui323DCmdHeaderOffset = 0;
+	IMG_UINT32 ui323DFullRenderCommandOffset = 0;
+	IMG_UINT32 ui32TACmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui323DCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT32 ui32TAFenceCount, ui323DFenceCount;
+	IMG_UINT32 ui32TAUpdateCount, ui323DUpdateCount;
+	IMG_UINT32 ui32PRUpdateCount;
+
+	IMG_PID uiCurrentProcess = OSGetCurrentClientProcessIDKM();
+
+	IMG_UINT32 ui32Client3DFenceCount = 0;
+
+	/* Ensure we haven't been given a null ptr to
+	 * TA fence values if we have been told we
+	 * have TA sync prim fences
+	 */
+	if (ui32ClientTAFenceCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAFenceValue != NULL,
+		                        "paui32ClientTAFenceValue NULL but "
+		                        "ui32ClientTAFenceCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * TA update values if we have been told we
+	 * have TA updates
+	 */
+	if (ui32ClientTAUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAUpdateValue != NULL,
+		                        "paui32ClientTAUpdateValue NULL but "
+		                        "ui32ClientTAUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * 3D update values if we have been told we
+	 * have 3D updates
+	 */
+	if (ui32Client3DUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32Client3DUpdateValue != NULL,
+		                        "paui32Client3DUpdateValue NULL but "
+		                        "ui32Client3DUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Write FW addresses into CMD SHARED BLOCKs */
+	{
+		CMDTA3D_SHARED *psGeomCmdShared = (CMDTA3D_SHARED *)pui8TADMCmd;
+		CMDTA3D_SHARED *ps3DCmdShared = (CMDTA3D_SHARED *)pui83DDMCmd;
+		CMDTA3D_SHARED *psPR3DCmdShared = (CMDTA3D_SHARED *)pui83DPRDMCmd;
+		RGXFWIF_DEV_VIRTADDR sNullFWAddr = {0};
+
+		if (psKMHWRTDataSet == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "KMHWRTDataSet is a null-pointer"));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Write FW address for TA CMD
+		*/
+		if (psGeomCmdShared != NULL)
+		{
+			if (ui32TACmdSize < sizeof(*psGeomCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid TACmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psGeomCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for 3D CMD
+		*/
+		if (ps3DCmdShared != NULL)
+		{
+			if (ui323DCmdSize < sizeof(*ps3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			ps3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for PR3D CMD
+		*/
+		if (psPR3DCmdShared != NULL)
+		{
+			if (ui323DPRCmdSize < sizeof(*psPR3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DPRCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psPR3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+	}
+
+	if (unlikely(iUpdateTATimeline >= 0 && !piUpdateTAFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	if (unlikely(iUpdate3DTimeline >= 0 && !piUpdate3DFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d, "
+			   "ui32Client3DFenceCount=%d, ui32Client3DUpdateCount=%d",
+			   __func__,
+			   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+			   ui32Client3DFenceCount, ui32Client3DUpdateCount));
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/* Double-check we have a PR kick if there are client fences */
+	if (unlikely(!bKickPR && ui32Client3DFenceCount != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence passed without a PR kick",
+		        __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+	szFenceName3D[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	ui32TAFenceCount = ui32ClientTAFenceCount;
+	ui323DFenceCount = ui32Client3DFenceCount;
+	ui32TAUpdateCount = ui32ClientTAUpdateCount;
+	ui323DUpdateCount = ui32Client3DUpdateCount;
+	ui32PRUpdateCount = ui32ClientPRUpdateCount;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32SyncPMRCount)
+	{
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling"
+		          " pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psRenderContext->psBufferSyncContext,
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%d)",
+				        __func__, eError));
+			}
+			OSLockRelease(psRenderContext->hLock);
+
+			return eError;
+		}
+
+#if !defined(SUPPORT_STRIP_RENDERING)
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+		{
+			ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+#else /* !defined(SUPPORT_STRIP_RENDERING) */
+		ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+
+		PVR_UNREFERENCED_PARAMETER(bTAFenceOnSyncCheckpointsOnly);
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+
+		if (psBufferUpdateSyncCheckpoint != NULL)
+		{
+			if (bKick3D)
+			{
+				ui323DUpdateCount++;
+			}
+			else
+			{
+				ui32PRUpdateCount++;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+	PVR_UNREFERENCED_PARAMETER(ppsSyncPMRs);
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2
+#error "Invalid value for UPDATE_FENCE_CHECKPOINT_COUNT. Must be either 1 or 2."
+#endif /* !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2 */
+
+	if (iCheckTAFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[TA]"
+		          " (iCheckFence=%d),"
+		          " psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheckTAFence,
+		          (void *) psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheckTAFence,
+		    &ui32FenceTASyncCheckpointCount,
+		    &apsFenceTASyncCheckpoints,
+		    &uiCheckTAFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_ta_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheckTAFence, ui32FenceTASyncCheckpointCount,
+		          (void *) apsFenceTASyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFenceTASyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "TA", apsFenceTASyncCheckpoints,
+			                      ui32FenceTASyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheck3DFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[3D]"
+		          " (iCheckFence=%d), "
+		          "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheck3DFence,
+		          (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheck3DFence,
+		    &ui32Fence3DSyncCheckpointCount,
+		    &apsFence3DSyncCheckpoints,
+		    &uiCheck3DFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_3d_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheck3DFence, ui32Fence3DSyncCheckpointCount,
+		          (void*)apsFence3DSyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFence3DSyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "3D", apsFence3DSyncCheckpoints,
+			                      ui32Fence3DSyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		IMG_UINT32 i;
+
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32FenceTASyncCheckpointCount;
+
+			for (i = 0; i < ui32Fence3DSyncCheckpointCount; i++)
+			{
+				if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) !=
+				    uiCurrentProcess)
+				{
+					ui32TAFenceCount++;
+				}
+			}
+		}
+
+		if (bKick3D)
+		{
+			ui323DFenceCount += ui32Fence3DSyncCheckpointCount;
+		}
+
+		ui32TAUpdateCount += iUpdateTATimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui323DUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui32PRUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE && !bKick3D ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+	}
+
+
+	/*
+	 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	 * in other words, take the value and set it to zero afterwards.
+	 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	 * as it must be ready at the time of context activation.
+	 *
+	 * NOTE: We use sTAData to get the ServerCommonContext giving us the ServerMMUCtx,
+	 *       should we use s3DData in some cases?
+	 *       Under assumption that sTAData and s3DData share the same psServerCommonContext,
+	 *       the answer is NO.
+	 *
+	 *       The ui64FBSCEntryMask filled by the following call gets cleared
+	 *       after the first KICK command and is ignored in the context of partial renders.
+	 */
+	eError = RGXExtractFBSCEntryMaskFromMMUContext(
+	    psRenderContext->psDeviceNode,
+	    FWCommonContextGetServerMMUCtx(psRenderContext->sTAData.psServerCommonContext),
+	    &ui64FBSCEntryMask
+	);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+		goto fail_tacmdinvalfbsc;
+	}
+
+	if (bKickTA)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+		          __func__, ui32TAFenceCount, ui32TAUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			ui64FBSCEntryMask,
+		    ui32TAFenceCount,
+		    ui32TAUpdateCount,
+		    ui32TACmdSize,
+		    &pPreAddr,
+		    (bKick3D ? NULL : &pPostAddr),
+		    (bKick3D ? NULL : &pRMWUFOAddr),
+		    pasTACmdHelperData
+		);
+
+		/* Clear the mask as we don't want to invalidate the FBSC multiple times
+		 * with the same value of ui64FBSCEntryMask.
+		 */
+		ui64FBSCEntryMask = 0;
+	}
+
+	if (bKickPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0, /* empty ui64FBSCEntryMask it is assumed that PRs should
+		        * not invalidate FBSC */
+		    ui323DFenceCount,
+		    0,
+		    sizeof(sPRUFO),
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickPR && !bUseCombined3DAnd3DPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32PRUpdateCount=%d", __func__,
+		          ui32PRUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0, /* empty ui64FBSCEntryMask it is assumed that PRs should
+		        * not invalidate FBSC */
+		    0,
+		    ui32PRUpdateCount,
+		    /* if the client has not provided a 3DPR command, the regular 3D
+		     * command should be used instead */
+		    pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize,
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKick3D || bAbort)
+	{
+		if (!bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+		}
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			ui64FBSCEntryMask, /* equals: [a] 0 if 3D is preceded by TA
+		                        *         [b] value from the MMU ctx otherwise */
+			bKickTA ? 0 : ui323DFenceCount,
+		    ui323DUpdateCount,
+		    ui323DCmdSize,
+			(bKickTA ? NULL : & pPreAddr),
+			&pPostAddr,
+			&pRMWUFOAddr,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickTA)
+	{
+		ui32TACmdSizeTmp = RGXCmdHelperGetCommandSize(1, pasTACmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext),
+		    ui32TACmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	if (ui323DCmdCount > 0)
+	{
+		ui323DCmdSizeTmp = RGXCmdHelperGetCommandSize(ui323DCmdCount, pas3DCmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext),
+		    ui323DCmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	/* need to reset the counter here */
+
+	ui323DCmdCount = 0;
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAFence, %d fences)...",
+			   __func__, ui32ClientTAFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
+										ui32ClientTAFenceCount,
+										apsClientTAFenceSyncPrimBlock,
+										paui32ClientTAFenceSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_fence;
+	}
+
+	if (ui32ClientTAFenceCount)
+	{
+		pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAUpdate, %d updates)...",
+			   __func__, ui32ClientTAUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
+										ui32ClientTAUpdateCount,
+										apsClientTAUpdateSyncPrimBlock,
+										paui32ClientTAUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_update;
+	}
+
+	if (ui32ClientTAUpdateCount)
+	{
+		pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAUpdateUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DFence, %d fences)...",
+			   __func__, ui32Client3DFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
+										ui32Client3DFenceCount,
+										NULL,
+										NULL);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_fence;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DUpdate, %d updates)...",
+			   __func__, ui32Client3DUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
+										ui32Client3DUpdateCount,
+										apsClient3DUpdateSyncPrimBlock,
+										paui32Client3DUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_update;
+	}
+
+	if (ui32Client3DUpdateCount || (iUpdate3DTimeline != PVRSRV_NO_TIMELINE && piUpdate3DFence && bKick3D))
+	{
+		pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DUpdateUFOAddress));
+
+	eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock, ui32PRFenceSyncOffset, &uiPRFenceUFOAddress);
+
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_pr_fence_address;
+	}
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	if (ui32SyncPMRCount)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+#if !defined(SUPPORT_STRIP_RENDERING)
+		/* Append buffer sync fences to TA fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0 && bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to TA Fence "
+					   "(&psRenderContext->sSyncAddrListTAFence=<%p>, "
+					   "pauiClientTAFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrListTAFence ,
+					   (void*)pauiClientTAFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClientTAFenceUFOAddress)
+			{
+				pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+			}
+			if (ui32ClientTAFenceCount == 0)
+			{
+				bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32ClientTAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+		/* Append buffer sync fences to 3D fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to 3D Fence "
+					   "(&psRenderContext->sSyncAddrList3DFence=<%p>, "
+					   "pauiClient3DFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrList3DFence,
+					   (void*)pauiClient3DFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+					ui32BufferFenceSyncCheckpointCount,
+					apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClient3DFenceUFOAddress)
+			{
+				pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+			}
+			if (ui32Client3DFenceCount == 0)
+			{
+				b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32Client3DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			/* If we have a 3D kick append update to the 3D updates else append to the PR update */
+			if (bKick3D)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   Append 1 buffer sync checkpoint<%p> to 3D Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClient3DUpdateUFOAddress=<%p>)...",
+						   __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClient3DUpdateUFOAddress));
+				/* Append buffer sync update to 3D updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClient3DUpdateUFOAddress)
+				{
+					pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32Client3DUpdateCount++;
+			}
+			else
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+				           "%s:   Append 1 buffer sync checkpoint<%p> to PR Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClientPRUpdateUFOAddress=<%p>)...",
+				           __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClientPRUpdateUFOAddress));
+				/* Attach update to the 3D (used for PR) Updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClientPRUpdateUFOAddress)
+				{
+					pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32ClientPRUpdateCount++;
+			}
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   (after buffer_sync) ui32ClientTAFenceCount=%d, "
+				   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+				   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+				   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+				   ui32ClientPRUpdateCount));
+
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Buffer sync not supported but got %u buffers",
+				 __func__, ui32SyncPMRCount));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_no_buffer_sync_invalid_params;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	/*
+	 * The hardware requires a PR to be submitted if there is a TA (otherwise
+	 * it can wedge if we run out of PB space with no PR to run)
+	 *
+	 * If we only have a TA, attach native checks to the TA and updates to the PR
+	 * If we have a TA and 3D, attach checks to TA, updates to 3D
+	 * If we only have a 3D, attach checks and updates to the 3D
+	 *
+	 * Note that 'updates' includes the cleanup syncs for 'check' fence FDs, in
+	 * addition to the update fence FD (if supplied)
+	 *
+	 * Currently, the client driver never kicks only the 3D, so we only support
+	 * that for the time being.
+	 */
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [TA] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheckTAFence, iUpdateTATimeline));
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [3D] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheck3DFence, iUpdate3DTimeline));
+
+		{
+			/* Create the output fence for TA (if required) */
+			if (iUpdateTATimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdateTAFence, iUpdateTATimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceNameTA,
+						iUpdateTATimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdateTAFence,
+						&uiUpdateTAFenceUID,
+						&pvTAUpdateFenceFinaliseData,
+						&psUpdateTASyncCheckpoint,
+						(void*)&psTAFenceTimelineUpdateSync,
+						&ui32TAFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[TA] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_ta_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdateTAFence,
+						   (void*)psTAFenceTimelineUpdateSync,
+						   ui32TAFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClientTAIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClientTAIntUpdateUFOAddress */
+					pauiClientTAIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdateTASyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[TA]->ui32Addr=0x%x",
+							   __func__, pauiClientTAIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the TA timeline (if required) */
+			if (psTAFenceTimelineUpdateSync)
+			{
+				sTASyncData.ui32ClientUpdateCount		 = ui32ClientTAUpdateCount;
+				sTASyncData.ui32ClientUpdateValueCount	 = ui32ClientTAUpdateValueCount;
+				sTASyncData.ui32ClientPRUpdateValueCount = (bKick3D) ? 0 : ui32ClientPRUpdateValueCount;
+				sTASyncData.paui32ClientUpdateValue		 = paui32ClientTAUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui32TAFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrListTAUpdate,
+											(bKick3D) ? NULL : &psRenderContext->sSyncAddrList3DUpdate,
+											psTAFenceTimelineUpdateSync,
+											&sTASyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_TA;
+				}
+
+				paui32ClientTAUpdateValue = sTASyncData.paui32ClientUpdateValue;
+				ui32ClientTAUpdateValueCount = sTASyncData.ui32ClientUpdateValueCount;
+				pauiClientTAUpdateUFOAddress = sTASyncData.pauiClientUpdateUFOAddress;
+				ui32ClientTAUpdateCount = sTASyncData.ui32ClientUpdateCount;
+			}
+
+			/* Create the output fence for 3D (if required) */
+			if (iUpdate3DTimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdate3DFence, iUpdate3DTimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceName3D,
+						iUpdate3DTimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdate3DFence,
+						&uiUpdate3DFenceUID,
+						&pv3DUpdateFenceFinaliseData,
+						&psUpdate3DSyncCheckpoint,
+						(void*)&ps3DFenceTimelineUpdateSync,
+						&ui323DFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[3D] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_3d_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdate3DFence,
+						   (void*)ps3DFenceTimelineUpdateSync,
+						   ui323DFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClient3DIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClient3DIntUpdateUFOAddress */
+					pauiClient3DIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdate3DSyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[3D]->ui32Addr=0x%x",
+							   __func__, pauiClient3DIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the 3D timeline (if required) */
+			if (ps3DFenceTimelineUpdateSync)
+			{
+				s3DSyncData.ui32ClientUpdateCount = ui32Client3DUpdateCount;
+				s3DSyncData.ui32ClientUpdateValueCount = ui32Client3DUpdateValueCount;
+				s3DSyncData.ui32ClientPRUpdateValueCount = ui32ClientPRUpdateValueCount;
+				s3DSyncData.paui32ClientUpdateValue = paui32Client3DUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui323DFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrList3DUpdate,
+											&psRenderContext->sSyncAddrList3DUpdate,	/*!< PR update: is this required? */
+											ps3DFenceTimelineUpdateSync,
+											&s3DSyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_3D;
+				}
+
+				paui32Client3DUpdateValue = s3DSyncData.paui32ClientUpdateValue;
+				ui32Client3DUpdateValueCount = s3DSyncData.ui32ClientUpdateValueCount;
+				pauiClient3DUpdateUFOAddress = s3DSyncData.pauiClientUpdateUFOAddress;
+				ui32Client3DUpdateCount = s3DSyncData.ui32ClientUpdateCount;
+
+				if (!bKick3D)
+				{
+					paui32ClientPRUpdateValue = s3DSyncData.paui32ClientPRUpdateValue;
+					ui32ClientPRUpdateValueCount = s3DSyncData.ui32ClientPRUpdateValueCount;
+					pauiClientPRUpdateUFOAddress = s3DSyncData.pauiClientPRUpdateUFOAddress;
+					ui32ClientPRUpdateCount = s3DSyncData.ui32ClientPRUpdateCount;
+				}
+			}
+
+			/*
+			 * The hardware requires a PR to be submitted if there is a TA OOM.
+			 * If we only have a TA, attach native checks and updates to the TA
+			 * and 3D updates to the PR.
+			 * If we have a TA and 3D, attach the native TA checks and updates
+			 * to the TA and similarly for the 3D.
+			 * Note that 'updates' includes the cleanup syncs for 'check' fence
+			 * FDs, in addition to the update fence FD (if supplied).
+			 * Currently, the client driver never kicks only the 3D, so we don't
+			 * support that for the time being.
+			 */
+
+			{
+				if (bKickTA)
+				{
+					/* Attach checks and updates to TA */
+
+					/* Checks (from input fence) */
+					if (ui32FenceTASyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to TA Fence (apsFenceSyncCheckpoints=<%p>)...",
+								   __func__,
+								   ui32FenceTASyncCheckpointCount,
+								   (void*)apsFenceTASyncCheckpoints));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+													  ui32FenceTASyncCheckpointCount,
+													  apsFenceTASyncCheckpoints);
+						if (!pauiClientTAFenceUFOAddress)
+						{
+							pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+								   __func__, ui32ClientTAFenceCount,
+								   ui32ClientTAFenceCount + ui32FenceTASyncCheckpointCount));
+						if (ui32ClientTAFenceCount == 0)
+						{
+							bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32ClientTAFenceCount += ui32FenceTASyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32ClientTAFenceCount now %d}",
+							   __func__, ui32ClientTAFenceCount));
+
+					if (psUpdateTASyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Update...",
+								   __func__, (void*)psUpdateTASyncCheckpoint,
+								   SyncCheckpointGetId(psUpdateTASyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAUpdate,
+													  1,
+													  &psUpdateTASyncCheckpoint);
+						if (!pauiClientTAUpdateUFOAddress)
+						{
+							pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+						}
+						ui32ClientTAUpdateCount++;
+					}
+
+					if (!bKick3D && psUpdate3DSyncCheckpoint)
+					{
+						/* Attach update to the 3D (used for PR) Updates */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D(PR) Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClientPRUpdateUFOAddress)
+						{
+							pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32ClientPRUpdateCount++;
+					}
+				}
+
+				if (bKick3D)
+				{
+					/* Attach checks and updates to the 3D */
+
+					/* Checks (from input fence) */
+					if (ui32Fence3DSyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to 3D Fence...",
+								   __func__, ui32Fence3DSyncCheckpointCount));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+													  ui32Fence3DSyncCheckpointCount,
+													  apsFence3DSyncCheckpoints);
+						if (!pauiClient3DFenceUFOAddress)
+						{
+							pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32Client3DFenceCount was %d, now %d}",
+								   __func__, ui32Client3DFenceCount,
+								   ui32Client3DFenceCount + ui32Fence3DSyncCheckpointCount));
+						if (ui32Client3DFenceCount == 0)
+						{
+							b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32Client3DFenceCount += ui32Fence3DSyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32Client3DFenceCount was %d}",
+							   __func__, ui32Client3DFenceCount));
+
+					if (psUpdate3DSyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClient3DUpdateUFOAddress)
+						{
+							pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32Client3DUpdateCount++;
+					}
+				}
+
+				/*
+				 * Relocate sync check points from the 3D fence that are
+				 * external to the current process, to the TA fence.
+				 * This avoids a sync lockup when dependent renders are
+				 * submitted out-of-order and a PR must be scheduled.
+				 */
+				if (bKickTA)
+				{
+					/* Search for external timeline dependencies */
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: Checking 3D fence for external sync points (%d)...",
+							   __func__, ui32Fence3DSyncCheckpointCount));
+
+					for (i=0; i<ui32Fence3DSyncCheckpointCount; i++)
+					{
+						/* Check to see if the checkpoint is on a TL owned by
+						 * another process.
+						 */
+						if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) != uiCurrentProcess)
+						{
+							/* 3D Sync point represents cross process
+							 * dependency, copy sync point to TA command fence. */
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Fence...",
+									   __func__, (void*)apsFence3DSyncCheckpoints[i],
+									   SyncCheckpointGetId(apsFence3DSyncCheckpoints[i])));
+
+							SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+														  1,
+														  &apsFence3DSyncCheckpoints[i]);
+
+							if (!pauiClientTAFenceUFOAddress)
+							{
+								pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+							}
+
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+									   __func__,
+									   ui32ClientTAFenceCount,
+									   ui32ClientTAFenceCount + 1));
+
+							if (ui32ClientTAFenceCount == 0)
+							{
+								bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+							}
+
+							ui32ClientTAFenceCount++;
+						}
+					}
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   (after pvr_sync) ui32ClientTAFenceCount=%d, "
+						   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+						   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+						   __func__,
+						   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+						   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+						   ui32ClientPRUpdateCount));
+			}
+		}
+
+		if (ui32ClientTAFenceCount)
+		{
+			PVR_ASSERT(pauiClientTAFenceUFOAddress);
+			if (!bTAFenceOnSyncCheckpointsOnly)
+			{
+				PVR_ASSERT(paui32ClientTAFenceValue);
+			}
+		}
+		if (ui32ClientTAUpdateCount)
+		{
+			PVR_ASSERT(pauiClientTAUpdateUFOAddress);
+			if (ui32ClientTAUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientTAUpdateValue);
+			}
+		}
+		if (ui32Client3DFenceCount)
+		{
+			PVR_ASSERT(pauiClient3DFenceUFOAddress);
+			PVR_ASSERT(b3DFenceOnSyncCheckpointsOnly);
+		}
+		if (ui32Client3DUpdateCount)
+		{
+			PVR_ASSERT(pauiClient3DUpdateUFOAddress);
+			if (ui32Client3DUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32Client3DUpdateValue);
+			}
+		}
+		if (ui32ClientPRUpdateCount)
+		{
+			PVR_ASSERT(pauiClientPRUpdateUFOAddress);
+			if (ui32ClientPRUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientPRUpdateValue);
+			}
+		}
+
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, pauiClientTAFenceUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAFenceCount,
+			   (void*)paui32ClientTAFenceValue));
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAUpdateCount=%d, pauiClientTAUpdateUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAUpdateCount,
+			   (void*)pauiClientTAUpdateUFOAddress));
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	/* Command size check */
+	if (ui32TAFenceCount != ui32ClientTAFenceCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAFenceCount, ui32ClientTAFenceCount));
+	}
+	if (ui32TAUpdateCount != ui32ClientTAUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAUpdateCount, ui32ClientTAUpdateCount));
+	}
+	if (!bTestSLRAdd3DCheck && (ui323DFenceCount != ui32Client3DFenceCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui323DFenceCount, ui32Client3DFenceCount));
+	}
+	if (ui323DUpdateCount != ui32Client3DUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui323DUpdateCount, ui32Client3DUpdateCount));
+	}
+	if (ui32PRUpdateCount != ui32ClientPRUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PR pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32PRUpdateCount, ui32ClientPRUpdateCount));
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)) && (bKickTA || bKick3D || bAbort))
+	{
+		sWorkloadCharacteristics.sTA3D.ui32RenderTargetSize  = ui32RenderTargetSize;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfDrawCalls = ui32NumberOfDrawCalls;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfIndices   = ui32NumberOfIndices;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfMRTs      = ui32NumberOfMRTs;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32RenderTargetSize);
+	PVR_UNREFERENCED_PARAMETER(ui32NumberOfDrawCalls);
+	PVR_UNREFERENCED_PARAMETER(ui32NumberOfIndices);
+	PVR_UNREFERENCED_PARAMETER(ui32NumberOfMRTs);
+#endif
+
+	/* Init and acquire to TA command if required */
+	if (bKickTA)
+	{
+		RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sDataTA,
+					RGXFWIF_CCB_CMD_TYPE_GEOM,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTA);
+		}
+#endif
+
+		/* Init the TA command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
+		                                 ui32ClientTAFenceCount,
+		                                 pauiClientTAFenceUFOAddress,
+		                                 paui32ClientTAFenceValue,
+		                                 ui32ClientTAUpdateCount,
+		                                 pauiClientTAUpdateUFOAddress,
+		                                 paui32ClientTAUpdateValue,
+		                                 ui32TACmdSize,
+		                                 pui8TADMCmd,
+								         &pPreAddr,
+								         (bKick3D ? NULL : & pPostAddr),
+								         (bKick3D ? NULL : & pRMWUFOAddr),
+		                                 RGXFWIF_CCB_CMD_TYPE_GEOM,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickDataTA,
+#else
+		                                 NULL,
+#endif
+		                                 "TA",
+		                                 bCCBStateOpen,
+		                                 pasTACmdHelperData);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following is used to determine the offset of the command header containing
+			   the workload estimation data so that can be accessed when the KCCB is read */
+			ui32TACmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(pasTACmdHelperData);
+		}
+#endif
+
+		eError = RGXCmdHelperAcquireCmdCCB(CCB_CMD_HELPER_NUM_TA_COMMANDS, pasTACmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taacquirecmd;
+		}
+		else
+		{
+			ui32TACmdCount++;
+		}
+	}
+
+	/* Only kick the 3D if required */
+	if (bKickPR)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+
+		/*
+			The command helper doesn't know about the PR fence so create
+			the command with all the fences against it and later create
+			the PR command itself which _must_ come after the PR fence.
+		*/
+		sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
+		sPRUFO.ui32Value = ui32PRFenceValue;
+
+		/* Init the PR fence command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32Client3DFenceCount=%d",
+				   __func__, ui32Client3DFenceCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+		                                 pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 0,
+		                                 NULL,
+		                                 NULL,
+		                                 sizeof(sPRUFO),
+		                                 (IMG_UINT8*) &sPRUFO,
+		                                 NULL,
+		                                 NULL,
+		                                 NULL,
+		                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+		                                 NULL,
+		                                 "3D-PR-Fence",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+		/* Init the 3D PR command helper */
+		/*
+			Updates for Android (fence sync and Timeline sync prim) are provided in the PR-update
+			if no 3D is present. This is so the timeline update cannot happen out of order with any
+			other 3D already in flight for the same timeline (PR-updates are done in the 3D cCCB).
+			This out of order timeline sync prim update could happen if we attach it to the TA update.
+		*/
+		if (ui32ClientPRUpdateCount)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s: Line %d, ui32ClientPRUpdateCount=%d, "
+					   "pauiClientPRUpdateUFOAddress=0x%x, "
+					   "ui32ClientPRUpdateValueCount=%d, "
+					   "paui32ClientPRUpdateValue=0x%x",
+					   __func__, __LINE__, ui32ClientPRUpdateCount,
+					   pauiClientPRUpdateUFOAddress->ui32Addr,
+					   ui32ClientPRUpdateValueCount,
+					   (ui32ClientPRUpdateValueCount == 0) ? PVRSRV_SYNC_CHECKPOINT_SIGNALLED : *paui32ClientPRUpdateValue));
+		}
+
+		if (!bUseCombined3DAnd3DPR)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientPRUpdateCount=%d",
+					   __func__, ui32ClientPRUpdateCount));
+			RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+			                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+			                                 0,
+			                                 NULL,
+			                                 NULL,
+			                                 ui32ClientPRUpdateCount,
+			                                 pauiClientPRUpdateUFOAddress,
+			                                 paui32ClientPRUpdateValue,
+			                                 pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize, // If the client has not provided a 3DPR command, the regular 3D command should be used instead
+			                                 pui83DPRDMCmd ? pui83DPRDMCmd : pui83DDMCmd,
+			                                 NULL,
+			                                 NULL,
+			                                 NULL,
+			                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
+			                                 ui32ExtJobRef,
+			                                 ui32IntJobRef,
+			                                 ui32PDumpFlags,
+			                                 NULL,
+			                                 "3D-PR",
+			                                 bCCBStateOpen,
+			                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+		}
+	}
+
+	if (bKick3D || bAbort)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+		const RGXFWIF_CCB_CMD_TYPE e3DCmdType = bAbort ? RGXFWIF_CCB_CMD_TYPE_ABORT : RGXFWIF_CCB_CMD_TYPE_3D;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sData3D,
+					e3DCmdType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickData3D);
+		}
+#endif
+
+		/* Init the 3D command helper */
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 bKickTA ? 0 : ui32Client3DFenceCount,  /* For a kick with a TA, the 3D fences are added before the PR command instead */
+		                                 bKickTA ? NULL : pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 ui32Client3DUpdateCount,
+		                                 pauiClient3DUpdateUFOAddress,
+		                                 paui32Client3DUpdateValue,
+		                                 ui323DCmdSize,
+		                                 pui83DDMCmd,
+		                                 (bKickTA ? NULL : & pPreAddr),
+		                                 &pPostAddr,
+		                                 &pRMWUFOAddr,
+		                                 e3DCmdType,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickData3D,
+#else
+		                                 NULL,
+#endif
+		                                 "3D",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following are used to determine the offset of the command header containing the workload estimation
+			   data so that can be accessed when the KCCB is read */
+			ui323DCmdHeaderOffset =	RGXCmdHelperGetDMCommandHeaderOffset(&pas3DCmdHelperData[ui323DCmdCount - 1]);
+			ui323DFullRenderCommandOffset =	RGXCmdHelperGetCommandOffset(pas3DCmdHelperData, ui323DCmdCount - 1);
+		}
+#endif
+	}
+
+	/* Protect against array overflow in RGXCmdHelperAcquireCmdCCB() */
+	if (unlikely(ui323DCmdCount > CCB_CMD_HELPER_NUM_3D_COMMANDS))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+		goto fail_3dcmdinit;
+	}
+
+	if (ui323DCmdCount)
+	{
+		PVR_ASSERT(bKickPR || bKick3D);
+
+		/* Acquire space for all the 3D command(s) */
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount, pas3DCmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			/* If RGXCmdHelperAcquireCmdCCB fails we skip the scheduling
+			 * of a new TA command with the same Write offset in Kernel CCB.
+			 */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+			goto fail_3dacquirecmd;
+		}
+	}
+
+	if (ui32TACmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTACmdKickData.apsCleanupCtl,
+				&sTACmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_GEOM,
+				bKickTA,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taattachcleanupctls;
+		}
+	}
+
+	if (ui323DCmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DCmdKickData.apsCleanupCtl,
+				&s3DCmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_3D,
+				bKick3D,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_3dattachcleanupctls;
+		}
+	}
+
+	/*
+		We should acquire the space in the kernel CCB here as after this point
+		we release the commands which will take operations on server syncs
+		which can't be undone
+	*/
+
+	/*
+		Everything is ready to go now, release the commands
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui32TACmdCount)
+	{
+		ui32TACmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
+								  pasTACmdHelperData,
+								  "TA",
+								  FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui32TACmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+
+			/* This checks if the command would wrap around at the end of the CCB and therefore  would start at an
+			   offset of 0 rather than the current command offset */
+			if (ui32TACmdOffset < ui32TACmdOffsetWrapCheck)
+			{
+				ui32TACommandOffset = ui32TACmdOffset;
+			}
+			else
+			{
+				ui32TACommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  pas3DCmdHelperData,
+								  "3D",
+								  FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui323DCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+
+			if (ui323DCmdOffset < ui323DCmdOffsetWrapCheck)
+			{
+				ui323DCommandOffset = ui323DCmdOffset;
+			}
+			else
+			{
+				ui323DCommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui32TACmdCount)
+	{
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext);
+		CMDTA3D_SHARED *psGeomCmdShared = IMG_OFFSET_ADDR(pui8TADMCmd, 0);
+
+		sTACmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
+		sTACmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTACmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Add the Workload data into the KCCB kick */
+			sTACmdKickData.ui32WorkEstCmdHeaderOffset = ui32TACommandOffset + ui32TACmdHeaderOffset;
+		}
+#endif
+
+		if (psGeomCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_TA,
+					sTACmdKickData.psContext,
+					ui32TACmdOffset,
+					psGeomCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_GEOM,
+		                  iCheckTAFence,
+		                  iUpdateTAFence,
+		                  iUpdateTATimeline,
+		                  uiCheckTAFenceUID,
+		                  uiUpdateTAFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickDataTA));
+
+		if (!bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA CCB command. */
+			RGXFWIF_KCCB_CMD sTAKCCBCmd;
+			sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			sTAKCCBCmd.uCmdData.sCmdKickData = sTACmdKickData;
+
+			LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+			{
+				eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+						RGXFWIF_DM_GEOM,
+						&sTAKCCBCmd,
+						ui32PDumpFlags);
+				if (eError2 != PVRSRV_ERROR_RETRY)
+				{
+					break;
+				}
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+		}
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			/* Mark the error and bail out */
+			eError = eError2;
+			goto fail_tasubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_GEOM);
+	}
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd = { 0 };
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext);
+		CMDTA3D_SHARED *ps3DCmdShared = IMG_OFFSET_ADDR(pui83DDMCmd, 0);
+
+		s3DCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+		s3DCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		s3DCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced when the KCCB command reaches the FW */
+			s3DCmdKickData.ui32WorkEstCmdHeaderOffset = ui323DCommandOffset + ui323DCmdHeaderOffset + ui323DFullRenderCommandOffset;
+		}
+#endif
+
+		if (ps3DCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_3D,
+					s3DCmdKickData.psContext,
+					ui323DCmdOffset,
+					ps3DCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_3D,
+		                  iCheck3DFence,
+		                  iUpdate3DFence,
+		                  iUpdate3DTimeline,
+		                  uiCheck3DFenceUID,
+		                  uiUpdate3DFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickData3D));
+
+		if (bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA/3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.sTACmdKickData = sTACmdKickData;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.s3DCmdKickData = s3DCmdKickData;
+		}
+		else
+		{
+			/* Construct the kernel 3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			s3DKCCBCmd.uCmdData.sCmdKickData = s3DCmdKickData;
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+										 RGXFWIF_DM_3D,
+										 &s3DKCCBCmd,
+										 ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			if (eError == PVRSRV_OK)
+			{
+				eError = eError2;
+			}
+			goto fail_3dsubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_3D);
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (unlikely(eError != PVRSRV_OK ))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+				   __func__, eError));
+		goto fail_3dsubmitcmd;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateTASyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [TA] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateTASyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateTASyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateTASyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateTASyncCheckpoint);
+	}
+	if (psTAFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [TA] <%p> to %d",
+				   __func__, (void*)psTAFenceTimelineUpdateSync,
+				   ui32TAFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psTAFenceTimelineUpdateSync, ui32TAFenceTimelineUpdateValue);
+	}
+
+	if (psUpdate3DSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [3D] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdate3DSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdate3DSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdate3DSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdate3DSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [3D] <%p> to %d",
+				   __func__, (void*)ps3DFenceTimelineUpdateSync,
+				   ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling pvr_buffer_sync_kick_succeeded(psBufferSyncData=<%p>)...",
+				   __func__, (void*)psBufferSyncData));
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	if (piUpdateTAFence)
+	{
+		*piUpdateTAFence = iUpdateTAFence;
+	}
+	if (piUpdate3DFence)
+	{
+		*piUpdate3DFence = iUpdate3DFence;
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+
+	if (pvTAUpdateFenceFinaliseData && (iUpdateTAFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdateTAFence,
+									pvTAUpdateFenceFinaliseData,
+									psUpdateTASyncCheckpoint, szFenceNameTA);
+	}
+	if (pv3DUpdateFenceFinaliseData && (iUpdate3DFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdate3DFence,
+									pv3DUpdateFenceFinaliseData,
+									psUpdate3DSyncCheckpoint, szFenceName3D);
+	}
+
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_3dsubmitcmd:
+fail_tasubmitcmd:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_3dattachcleanupctls:
+fail_taattachcleanupctls:
+fail_3dacquirecmd:
+fail_3dcmdinit:
+fail_taacquirecmd:
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DUpdate);
+	/* Where a TA-only kick (ie no 3D) is submitted, the PR update will make use of the unused 3DUpdate list.
+	 * If this has happened, performing a rollback on pauiClientPRUpdateUFOAddress will simply repeat what
+	 * has already been done for the sSyncAddrList3DUpdate above and result in a double decrement of the
+	 * sync checkpoint's hEnqueuedCCBCount, so we need to check before rolling back the PRUpdate.
+	 */
+	if (pauiClientPRUpdateUFOAddress && (pauiClientPRUpdateUFOAddress != psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs))
+	{
+		SyncCheckpointRollbackFromUFO(psRenderContext->psDeviceNode, pauiClientPRUpdateUFOAddress->ui32Addr);
+	}
+
+fail_alloc_update_values_mem_3D:
+	if (iUpdate3DFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdate3DFence, pv3DUpdateFenceFinaliseData);
+	}
+fail_create_3d_fence:
+fail_alloc_update_values_mem_TA:
+	if (iUpdateTAFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateTAFence, pvTAUpdateFenceFinaliseData);
+	}
+fail_create_ta_fence:
+#if !defined(SUPPORT_BUFFER_SYNC)
+err_no_buffer_sync_invalid_params:
+#endif /* !defined(SUPPORT_BUFFER_SYNC) */
+err_pr_fence_address:
+err_populate_sync_addr_list_3d_update:
+err_populate_sync_addr_list_3d_fence:
+err_populate_sync_addr_list_ta_update:
+err_populate_sync_addr_list_ta_fence:
+err_not_enough_space:
+fail_tacmdinvalfbsc:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		SyncAddrListDeRefCheckpoints(ui32BufferFenceSyncCheckpointCount,
+					     apsBufferFenceSyncCheckpoints);
+	}
+#endif
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+fail_resolve_input_3d_fence:
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+fail_resolve_input_ta_fence:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	PVR_ASSERT(eError != PVRSRV_OK);
+	OSLockRelease(psRenderContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD		sDisableZSStoreCmd = { 0 };
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	sDisableZSStoreCmd.eCmdType = RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableZStore = bDisableDepthStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableSStore = bDisableStencilStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.ui32ExtJobRefToDisableZSStore = ui32ExtJobRefToDisableZSStore;
+
+	if (psRenderContext->ui32CleanupStatus & (RC_CLEANUP_TA_COMPLETE | RC_CLEANUP_3D_COMPLETE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: submit disable depth and stencil store command to render context that has been cleaned up (%u)",
+				__func__, PVRSRV_ERROR_INVALID_CONTEXT));
+		return PVRSRV_ERROR_INVALID_CONTEXT;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_3D,
+									&sDisableZSStoreCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit disable depth and stencil store command (%u)",
+				__func__,
+				eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	if (psRenderContext->sTAData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_GEOM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the TA part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_tacontext;
+		}
+		psRenderContext->sTAData.i32Priority = i32Priority;
+	}
+
+	if (psRenderContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the 3D part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_3dcontext;
+		}
+		psRenderContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+fail_tacontext:
+	OSLockRelease(psRenderContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 RGX_CONTEXT_PROPERTY eContextProperty,
+                                                 IMG_UINT64 ui64Input,
+                                                 IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psRenderContext->hLock);
+			eError = FWCommonContextSetFlags(psRenderContext->sTAData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psRenderContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psRenderContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->sTAData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->s3DData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+		if (NULL != psCurrentServerRenderCtx->sTAData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext, RGX_KICK_TYPE_DM_TA) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TA;
+			}
+		}
+
+		if (NULL != psCurrentServerRenderCtx->s3DData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_3D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * RGXRenderContextStalledKM
+ */
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	RGXCheckForStalledClientContexts((PVRSRV_RGXDEV_INFO *) psRenderContext->psDeviceNode->pvDevice, IMG_TRUE);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxta3d.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.h
new file mode 100644
index 000000000000..c7cf0edfc108
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.h
@@ -0,0 +1,506 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA and 3D Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX TA and 3D Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXTA3D_H
+#define RGXTA3D_H
+
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_shared.h"
+#include "rgx_fwif_resetframework.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+typedef struct _RGX_SERVER_RENDER_CONTEXT_ RGX_SERVER_RENDER_CONTEXT;
+typedef struct _RGX_FREELIST_ RGX_FREELIST;
+typedef struct _RGX_PMR_NODE_ RGX_PMR_NODE;
+
+/*****************************************************************************
+ * The Design of Data Storage System for Render Targets                      *
+ * ====================================================                      *
+ *   Relevant for                                                            *
+ *     understanding RGXCreateHWRTDataSet & RGXDestroyHWRTDataSet            *
+ *                                                                           *
+ *                                                                           *
+ *        +=========================================+                        *
+ *        |           RenderTargetDataSet           |                        *
+ *        +---------------|---------|---------------+                        *
+ *                        |         |                                        *
+ *                        V         V                                        *
+ *  +- - - - - - - - - - - - +   +- - - - - - - - - - - - +                  *
+ *  | KM_HW_RT_DATA_HANDLE_0 |   | KM_HW_RT_DATA_HANDLE_1 |                  *
+ *  +- - -|- - - - - - - - - +   +- - - - - - - - - | - - +                  *
+ *        |                                         |                        *
+ *        |                                         |           [UM]Client   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |               Bridge   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |           [KM]Server   *
+ *        |                                         |                        *
+ *        | KM-ptr                                  | KM-ptr                 *
+ *        V                                         V                        *
+ *  +====================+           +====================+                  *
+ *  |  KM_HW_RT_DATA_0   |           |   KM_HW_RT_DATA_1  |                  *
+ *  +-----|------------|-+           +-|------------|-----+                  *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            | KM-ptr        | KM-ptr     |                        *
+ *        |            V               V            |                        *
+ *        |      +==========================+       |                        *
+ *        |      | HW_RT_DATA_COMMON_COOKIE |       |                        *
+ *        |      +--------------------------+       |                        *
+ *        |                   |                     |                        *
+ *        |                   |                     |                        *
+ *  ------|-------------------|---------------------|----------------------- *
+ *        |                   |                     |         [FW]Firmware   *
+ *        |                   |                     |                        *
+ *        | FW-addr           |                     | FW-addr                *
+ *        V                   |                     V                        *
+ *  +===============+         |           +===============+                  *
+ *  | HW_RT_DATA_0  |         |           | HW_RT_DATA_1  |                  *
+ *  +------------|--+         |           +--|------------+                  *
+ *               |            |              |                               *
+ *               | FW-addr    | FW-addr      | FW-addr                       *
+ *               V            V              V                               *
+ *        +=========================================+                        *
+ *        |           HW_RT_DATA_COMMON             |                        *
+ *        +-----------------------------------------+                        *
+ *                                                                           *
+ *****************************************************************************/
+
+typedef struct _RGX_HWRTDATA_COMMON_COOKIE_
+{
+	DEVMEMINT_RESERVATION* psPMStateReservation;
+	DEVMEMINT_RESERVATION* psPMSecureStateReservation;
+	DEVMEMINT_RESERVATION* psPMMListsReservation;
+	IMG_UINT32             ui32RefCount;
+} RGX_HWRTDATA_COMMON_COOKIE;
+
+typedef struct _RGX_KM_HW_RT_DATASET_
+{
+	/* RGX_RTDATA_CLEANUP_DATA */
+	/* RGXMKIF_NUM_RTDATAS */
+	RGX_HWRTDATA_COMMON_COOKIE* psHWRTDataCommonCookie;
+
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataFwAddr;
+
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc;
+
+	RGX_FREELIST *apsFreeLists[RGXFW_MAX_FREELISTS];
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE			sNodeHWRTData;
+#endif
+
+} RGX_KM_HW_RT_DATASET;
+
+struct _RGX_FREELIST_ {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	CONNECTION_DATA			*psConnection;
+
+	/* Free list PMR */
+	PMR						*psFreeListPMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListPMROffset;
+
+	/* Free list PM state PMR */
+	PMR						*psFreeListStatePMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListStatePMROffset;
+
+	DEVMEMINT_RESERVATION	*psFreeListAndStateReservation;
+
+	/* Freelist config */
+	IMG_UINT32				ui32MaxFLPages;
+	IMG_UINT32				ui32InitFLPages;
+	IMG_UINT32				ui32CurrentFLPages;
+	IMG_UINT32				ui32GrowFLPages;
+	IMG_UINT32				ui32ReadyFLPages;
+	IMG_UINT32				ui32GrowThreshold;		/* Percentage of FL memory used that should trigger a new grow request */
+	IMG_UINT32				ui32FreelistID;
+	IMG_UINT32				ui32FreelistGlobalID;	/* related global freelist for this freelist */
+	IMG_UINT64				ui64FreelistChecksum;	/* checksum over freelist content */
+	IMG_BOOL				bCheckFreelist;			/* freelist check enabled */
+	IMG_UINT32				ui32RefCount;			/* freelist reference counting */
+
+	IMG_UINT32				ui32NumGrowReqByApp;	/* Total number of grow requests by Application */
+	IMG_UINT32				ui32NumGrowReqByFW;		/* Total Number of grow requests by Firmware */
+	IMG_UINT32				ui32NumHighPages;		/* High Mark of pages in the freelist */
+
+	IMG_PID					ownerPid;				/* Pid of the owner of the list */
+
+	/* Memory Blocks */
+	DLLIST_NODE				sMemoryBlockHead;
+	DLLIST_NODE				sMemoryBlockInitHead;
+	DLLIST_NODE				sNode;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* HWRTData nodes linked to local freelist */
+	DLLIST_NODE				sNodeHWRTDataHead;
+#endif
+
+	/* FW data structures */
+	DEVMEM_MEMDESC			*psFWFreelistMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;
+	IMG_DEV_VIRTADDR		sFreeListBaseDevVAddr;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	HASH_TABLE*				psWorkloadHashTable;
+#endif
+};
+
+struct _RGX_PMR_NODE_ {
+	RGX_FREELIST			*psFreeList;
+	PMR						*psPMR;
+	PMR_PAGELIST			*psPageList;
+	DLLIST_NODE				sMemoryBlock;
+	IMG_UINT32				ui32NumPages;
+	IMG_BOOL				bFirstPageMissing;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RI_HANDLE				hRIHandle;
+#endif
+};
+
+typedef struct {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	DEVMEM_MEMDESC			*psFWZSBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;
+
+	DEVMEMINT_RESERVATION	*psReservation;
+	PMR						*psPMR;
+	IMG_UINT32				ui32ZSBufferID;
+	IMG_UINT32				ui32RefCount;
+	IMG_BOOL				bOnDemand;
+	IMG_BOOL				bIsBacked;
+
+	IMG_UINT32				ui32NumReqByApp;		/* Number of Backing Requests from Application */
+	IMG_UINT32				ui32NumReqByFW;			/* Number of Backing Requests from Firmware */
+
+	IMG_PID					owner;
+
+	DLLIST_NODE	sNode;
+}RGX_ZSBUFFER_DATA;
+
+typedef struct {
+	RGX_ZSBUFFER_DATA		*psZSBuffer;
+} RGX_POPULATION;
+
+/* Dump the physical pages of a freelist */
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList);
+
+PVRSRV_ERROR RGXCreateHWRTDataSet2(CONNECTION_DATA          *psConnection,
+                                   PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                   IMG_DEV_VIRTADDR         psVHeapTableDevVAddr,
+                                   DEVMEMINT_RESERVATION    *psPMMListsReservation,
+                                   DEVMEMINT_RESERVATION    *psPMStateReservation,
+                                   DEVMEMINT_RESERVATION    *psPMSecureStateReservation,
+                                   RGX_FREELIST             *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                   IMG_DEV_VIRTADDR         asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                   IMG_UINT16               ui16MaxRTs,
+                                   RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Create HWRTDataSet */
+PVRSRV_ERROR RGXCreateHWRTDataSet(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          psVHeapTableDevVAddr,
+                                  IMG_DEV_VIRTADDR          sPMDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sPMSecureDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  RGX_FREELIST	           *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Destroy HWRTDataSet */
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet);
+
+/*
+	RGXCreateZSBufferKM
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer);
+
+/*
+	RGXDestroyZSBufferKM
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+
+/*
+ * RGXBackingZSBuffer()
+ *
+ * Backs ZS-Buffer with physical pages
+ */
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXPopulateZSBufferKM()
+ *
+ * Backs ZS-Buffer with physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+								   RGX_POPULATION **ppsPopulation);
+
+/*
+ * RGXUnbackingZSBuffer()
+ *
+ * Frees ZS-Buffer's physical pages
+ */
+PVRSRV_ERROR RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXUnpopulateZSBufferKM()
+ *
+ * Frees ZS-Buffer's physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation);
+
+/*
+	RGXProcessRequestZSBufferBacking
+*/
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXProcessRequestZSBufferUnbacking
+*/
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXGrowFreeList
+*/
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader);
+
+/* Create free list */
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA       *psConnection,
+                               PVRSRV_DEVICE_NODE	     *psDeviceNode,
+                               IMG_HANDLE             hMemCtxPrivData,
+                               IMG_UINT32             ui32MaxFLPages,
+                               IMG_UINT32             ui32InitFLPages,
+                               IMG_UINT32             ui32GrowFLPages,
+                               IMG_UINT32             ui32GrowParamThreshold,
+                               RGX_FREELIST           *psGlobalFreeList,
+                               IMG_BOOL               bCheckFreelist,
+                               DEVMEMINT_RESERVATION	 *psFreeListAndStateReservation,
+                               RGX_FREELIST           **ppsFreeList);
+
+/* Destroy free list */
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList);
+
+/*
+	RGXProcessRequestGrow
+*/
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   IMG_UINT32 ui32FreelistID);
+
+
+/* Reconstruct free list after Hardware Recovery */
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+											  IMG_UINT32 ui32FreelistsCount,
+											  const IMG_UINT32 *paui32Freelists);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateRenderContextKM
+
+ @Description
+	Server-side implementation of RGXCreateRenderContext
+
+ @Input psConnection -
+ @Input psDeviceNode - device node
+ @Input i32Priority - context priority
+ @Input ui32FrameworkCommandSize - framework command size
+ @Input pabyFrameworkCommand - ptr to framework command
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32StaticRenderContextStateSize - size of fixed render state
+ @Input pStaticRenderContextState - ptr to fixed render state buffer
+ @Input ui32PackedCCBSizeU8888 :
+		ui8TACCBAllocSizeLog2 - TA CCB size
+		ui8TACCBMaxAllocSizeLog2 - maximum size to which TA CCB can grow
+		ui83DCCBAllocSizeLog2 - 3D CCB size
+		ui83DCCBMaxAllocSizeLog2 - maximum size to which 3D CCB can grow
+ @Input ui32ContextFlags - flags which specify properties of the context
+ @Output ppsRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+											PVRSRV_DEVICE_NODE			*psDeviceNode,
+											IMG_INT32					i32Priority,
+											IMG_UINT32					ui32FrameworkCommandSize,
+											IMG_PBYTE					pabyFrameworkCommand,
+											IMG_HANDLE					hMemCtxPrivData,
+											IMG_UINT32					ui32StaticRenderContextStateSize,
+											IMG_PBYTE					pStaticRenderContextState,
+											IMG_UINT32					ui32PackedCCBSizeU8888,
+											IMG_UINT32					ui32ContextFlags,
+											IMG_UINT64					ui64RobustnessAddress,
+											IMG_UINT32					ui32MaxTADeadlineMS,
+											IMG_UINT32					ui32Max3DDeadlineMS,
+											RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyRenderContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRenderContext
+
+ @Input psRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXKickTA3DKM
+
+ @Description
+	Server-side implementation of RGXKickTA3D
+
+ @Input psRTDataCleanup - RT data associated with the kick (or NULL)
+ @Input psZBuffer - Z-buffer associated with the kick (or NULL)
+ @Input psSBuffer - S-buffer associated with the kick (or NULL)
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRSyncPrimBlock,
+								 IMG_UINT32					ui32PRSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckFence,
+								 PVRSRV_TIMELINE			iUpdateTimeline,
+								 PVRSRV_FENCE				*piUpdateFence,
+								 IMG_CHAR					szFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheckFence3D,
+								 PVRSRV_TIMELINE			iUpdateTimeline3D,
+								 PVRSRV_FENCE				*piUpdateFence3D,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus);
+
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore);
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDevNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+												 RGX_CONTEXT_PROPERTY eContextProperty,
+												 IMG_UINT64 ui64Input,
+												 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of render contexts on this device */
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client contexts are stalled */
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+#endif /* RGXTA3D_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.c
new file mode 100644
index 000000000000..9397bf9d0338
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.c
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@File           rgxworkest_ray.c
+@Title          RGX Workload Estimation Functionality for ray datamaster
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality for ray datamaster.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxworkest.h"
+#include "rgxworkest_ray.h"
+#include "rgxfwutils.h"
+#include "rgxpdvfs.h"
+#include "rgx_options.h"
+#include "device.h"
+#include "hash.h"
+#include "pvr_debug.h"
+
+
+static IMG_BOOL WorkEstHashCompareRay(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sRay.ui32DispatchSize == psWorkload2->sRay.ui32DispatchSize &&
+		    psWorkload1->sRay.ui32AccStructSize == psWorkload2->sRay.ui32AccStructSize)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_UINT32 WorkEstHashFuncRay(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on transfer src/dest attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sRay.ui32DispatchSize);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sRay.ui32AccStructSize);
+
+	return ui32HashKey;
+}
+
+void WorkEstInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sRay.sDataRDM,
+		(HASH_FUNC *)WorkEstHashFuncRay,
+		(HASH_KEY_COMP *)WorkEstHashCompareRay);
+}
+
+void WorkEstDeInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sRay.sDataRDM);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.h
new file mode 100644
index 000000000000..21f3d3d8271a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.h
@@ -0,0 +1,53 @@
+/*************************************************************************/ /*!
+@File           rgxworkest_ray.h
+@Title          RGX Workload Estimation Functionality for ray datamaster
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXWORKEST_RAY_H
+#define RGXWORKEST_RAY_H
+
+#include "img_types.h"
+
+void WorkEstInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+void WorkEstDeInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+#endif /* RGXWORKEST_RAY_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..a53db2eac5bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Kbuild.mk
@@ -0,0 +1,695 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+# Define used in code when no feature MASK or IDX flag can determine the
+# PVR_ARCH build config of the KMD server. KMD Server either supports all
+# Rogue cores, or all Volcanic cores, but not both.
+ifeq ($(PVR_ARCH),rogue)
+ ccflags-y += -DPVR_ARCH_ROGUE
+else ifeq ($(PVR_ARCH),volcanic)
+ ccflags-y += -DPVR_ARCH_VOLCANIC
+endif
+
+# Window system
+ccflags-y += -DWINDOW_SYSTEM=\"$(WINDOW_SYSTEM)\"
+
+# Linux kernel headers
+ccflags-y += \
+ -Iinclude \
+ -Iinclude/drm
+
+# Compatibility BVNC
+ccflags-y += -I$(TOP)/services/shared/devices/$(PVR_ARCH_DEFS)
+
+# Errata files
+ccflags-y += -I$(HWDEFS_DIR) -I$(HWDEFS_DIR)/$(RGX_BNC)
+
+# Linux-specific headers
+ccflags-y += \
+ -I$(TOP)/include/drm \
+ -I$(TOP)/services/include/env/linux \
+ -I$(TOP)/services/server/env/linux/$(PVR_ARCH) -I$(TOP)/services/server/env/linux \
+ -I$(TOP)/kernel/drivers/staging/imgtec
+
+# System dir
+ifneq ($(wildcard $(TOP)/services/system/$(PVR_ARCH)/$(PVR_SYSTEM)/Kbuild.mk),)
+SYSTEM_DIR := $(TOP)/services/system/$(PVR_ARCH)/$(PVR_SYSTEM)
+else
+SYSTEM_DIR := $(TOP)/services/system/$(PVR_SYSTEM)
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/dkf_server.o \
+ services/server/env/linux/event.o \
+ services/server/env/linux/fwload.o \
+ services/server/env/linux/km_apphint.o \
+ services/server/env/linux/module_common.o \
+ services/server/env/linux/osmmap_stub.o \
+ services/server/env/linux/osfunc.o \
+ services/server/env/linux/allocmem.o \
+ services/server/env/linux/osconnection_server.o \
+ services/server/env/linux/physmem_osmem_linux.o \
+ services/server/env/linux/pmr_os.o \
+ services/server/env/linux/pmr_env.o \
+ services/server/env/linux/pvr_bridge_k.o \
+ services/server/env/linux/pvr_debug.o \
+ services/server/env/linux/pvr_drm.o \
+ services/server/env/linux/physmem_dmabuf.o \
+ services/server/common/devicemem_heapcfg.o \
+ services/shared/common/devicemem.o \
+ services/shared/common/devicemem_utils.o \
+ services/shared/common/hash.o \
+ services/shared/common/hash_functions.o \
+ services/shared/common/ra.o \
+ services/shared/common/sync.o \
+ services/shared/common/mem_utils.o \
+ services/server/common/devicemem_server.o \
+ services/server/common/handle.o \
+ services/server/common/lists.o \
+ services/server/common/mmu_common.o \
+ services/server/common/connection_server.o \
+ services/server/common/physheap.o \
+ services/server/common/physmem.o \
+ services/server/common/physmem_lma.o \
+ services/server/common/physmem_ramem.o \
+ services/server/common/physmem_osmem.o \
+ services/server/common/physmem_hostmem.o \
+ services/server/common/pmr.o \
+ services/server/common/power.o \
+ services/server/common/process_stats.o \
+ services/server/common/pvr_notifier.o \
+ services/server/common/pvrsrv.o \
+ services/server/common/srvcore.o \
+ services/server/common/sync_checkpoint.o \
+ services/server/common/sync_server.o \
+ services/server/common/tlintern.o \
+ services/shared/common/tlclient.o \
+ services/server/common/tlserver.o \
+ services/server/common/tlstream.o \
+ services/server/common/cache_km.o \
+ services/shared/common/uniq_key_splay_tree.o \
+ services/server/common/pvrsrv_pool.o \
+ services/server/common/pvrsrv_bridge_init.o \
+ services/server/common/info_page_km.o \
+ services/shared/common/pvrsrv_error.o \
+ services/server/common/debug_common.o \
+ services/server/common/di_server.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/$(PVR_ARCH)/common/vmm_type_$(VMM_TYPE).o
+
+ifeq ($(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/common/physmem_dlm.o \
+ services/server/common/physmem_ima.o
+endif
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+$(PVRSRV_MODNAME)-y += \
+ services/shared/common/htbuffer.o \
+ services/server/common/htbserver.o \
+ services/server/common/htb_debug.o
+endif
+
+ifeq ($(SUPPORT_DMA_TRANSFER),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/common/dma_km.o
+endif
+
+# Wrap ExtMem support
+ifeq ($(SUPPORT_WRAP_EXTMEM),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/physmem_extmem_linux.o \
+ services/server/common/physmem_extmem.o
+endif
+
+ifeq ($(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pg_walk_through.o
+endif
+
+ifeq ($(SUPPORT_PHYSMEM_TEST),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/physmem_test.o
+endif
+
+ifneq ($(PVR_LOADER),)
+ ifeq ($(KERNEL_DRIVER_DIR),)
+  $(PVRSRV_MODNAME)-y += services/server/env/linux/$(PVR_LOADER).o
+ else
+  ifneq ($(wildcard $(KERNELDIR)/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM)/$(PVR_LOADER).c),)
+    $(PVRSRV_MODNAME)-y += external/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM)/$(PVR_LOADER).o
+  else
+   ifneq ($(wildcard $(KERNELDIR)/$(KERNEL_DRIVER_DIR)/$(PVR_LOADER).c),)
+     $(PVRSRV_MODNAME)-y += external/$(KERNEL_DRIVER_DIR)/$(PVR_LOADER).o
+   else
+     $(PVRSRV_MODNAME)-y += services/server/env/linux/$(PVR_LOADER).o
+   endif
+  endif
+ endif
+else
+ $(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_platform_drv.o
+endif
+
+ifeq ($(SUPPORT_RGX),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgx_bridge_init.o \
+ services/server/devices/rgxfwdbg.o \
+ services/server/devices/rgxtimerquery.o \
+ services/server/devices/rgxccb.o \
+ services/server/devices/rgxdebug_common.o \
+ services/server/devices/$(PVR_ARCH)/rgxdebug.o \
+ services/server/devices/rgxfwtrace_strings.o \
+ services/server/devices/$(PVR_ARCH)/rgxfwutils.o \
+ services/server/devices/rgxfwcmnctx.o \
+ services/server/devices/rgxfwriscv.o \
+ services/server/devices/$(PVR_ARCH)/rgxinit.o \
+ services/server/devices/rgxbvnc.o \
+ services/server/devices/rgxlayer_impl_common.o \
+ services/server/devices/$(PVR_ARCH)/rgxlayer_impl.o \
+ services/server/devices/rgxmem.o \
+ services/server/devices/rgxmmuinit.o \
+ services/server/devices/rgxregconfig.o \
+ services/server/devices/$(PVR_ARCH)/rgxta3d.o \
+ services/server/devices/rgxsyncutils.o \
+ services/server/devices/rgxtdmtransfer.o \
+ services/server/devices/rgxutils.o \
+ services/server/devices/rgxhwperf_common.o \
+ services/server/devices/$(PVR_ARCH)/rgxhwperf.o \
+ services/server/devices/rgxpower.o \
+ services/server/devices/$(PVR_ARCH)/rgxstartstop.o \
+ services/server/devices/rgxtimecorr.o \
+ services/server/devices/rgxcompute.o \
+ services/server/devices/$(PVR_ARCH)/rgxmulticore.o \
+ services/server/devices/rgxshader.o
+
+# add pvr_gputrace.o if any of these features are enabled
+ifneq (, $(filter 1,$(PVRSRV_TRACE_ROGUE_EVENTS) $(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)))
+$(PVRSRV_MODNAME)-$(CONFIG_EVENT_TRACING) += services/server/env/linux/pvr_gputrace.o
+endif
+
+ifeq ($(PVRSRV_ANDROID_TRACE_GPU_FREQ),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_gpufreq.o
+endif
+
+ifeq ($(SUPPORT_RGXKICKSYNC_BRIDGE),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxkicksync.o
+endif
+
+ifeq ($(SUPPORT_USC_BREAKPOINT),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxbreakpoint.o
+endif
+
+ifeq ($(PVR_ARCH),volcanic)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/$(PVR_ARCH)/rgxray.o
+endif
+
+
+ifeq ($(PVR_ARCH),rogue)
+ $(PVRSRV_MODNAME)-y += \
+  services/server/devices/$(PVR_ARCH)/rgxtransfer.o \
+  services/server/devices/$(PVR_ARCH)/rgxmipsmmuinit.o
+endif
+
+ifeq ($(SUPPORT_PDVFS),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxpdvfs.o
+endif
+
+ifeq ($(SUPPORT_WORKLOAD_ESTIMATION),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxworkest.o
+ ifeq ($(PVR_ARCH),volcanic)
+  $(PVRSRV_MODNAME)-y += \
+  services/server/devices/$(PVR_ARCH)/rgxworkest_ray.o
+ endif
+endif
+
+ifeq ($(SUPPORT_VALIDATION),1)
+ifeq ($(PVR_TESTING_UTILS),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxgpumap.o
+endif
+endif
+
+ifeq ($(SUPPORT_VALIDATION),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxsoctimer.o
+endif
+endif # SUPPORT_RGX
+
+ifeq ($(SUPPORT_DISPLAY_CLASS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/common/dc_server.o \
+ services/server/common/scp.o
+endif
+
+ifeq ($(SUPPORT_SECURE_EXPORT),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/ossecure_export.o
+endif
+
+ifeq ($(PDUMP),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/common/pdump_server.o \
+ services/server/common/pdump_mmu.o \
+ services/server/common/pdump_physmem.o \
+ services/shared/common/devicemem_pdump.o \
+ services/shared/common/devicememx_pdump.o
+
+ifeq ($(SUPPORT_RGX),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/$(PVR_ARCH)/rgxpdump.o
+ifeq ($(PDUMP),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxpdump_common.o
+endif
+endif
+
+endif
+
+
+
+ifeq ($(PVRSRV_ENABLE_GPU_MEMORY_INFO),1)
+$(PVRSRV_MODNAME)-y += services/server/common/ri_server.o
+endif
+
+ifeq ($(PVR_TESTING_UTILS),1)
+$(PVRSRV_MODNAME)-y += services/server/common/tutils.o
+endif
+
+$(PVRSRV_MODNAME)-y += services/server/common/devicemem_history_server.o
+
+ifeq ($(PVRSRV_PHYSMEM_CPUMAP_HISTORY),1)
+$(PVRSRV_MODNAME)-y += services/server/common/physmem_cpumap_history.o
+endif
+
+ifeq ($(PVR_HANDLE_BACKEND),generic)
+$(PVRSRV_MODNAME)-y += services/server/common/handle_generic.o
+else
+ifeq ($(PVR_HANDLE_BACKEND),idr)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/handle_idr.o
+endif
+endif
+
+ifeq ($(PVRSRV_ENABLE_LINUX_MMAP_STATS),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/mmap_stats.o
+endif
+
+ifeq ($(SUPPORT_BUFFER_SYNC),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_buffer_sync.o \
+ services/server/env/linux/pvr_fence.o
+endif
+
+ifeq ($(SUPPORT_NATIVE_FENCE_SYNC),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_sync_ioctl_common.o
+ifeq ($(USE_PVRSYNC_DEVNODE),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_sync_ioctl_dev.o
+else
+$(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_sync_ioctl_drm.o
+endif
+ifeq ($(SUPPORT_DMA_FENCE),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_sync_file.o \
+ services/server/env/linux/pvr_counting_timeline.o \
+ services/server/env/linux/pvr_sw_fence.o \
+ services/server/env/linux/pvr_export_fence.o \
+ services/server/env/linux/pvr_fence.o
+endif
+else
+ifeq ($(SUPPORT_FALLBACK_FENCE_SYNC),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/common/sync_fallback_server.o \
+ services/server/env/linux/ossecure_export.o
+endif
+endif
+
+ifeq ($(SUPPORT_LINUX_DVFS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_dvfs_device.o \
+ services/server/env/linux/pvr_dvfs_common.o
+else
+ ifeq ($(SUPPORT_PDVFS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_dvfs_common.o
+ endif
+endif
+
+ifeq ($(PVRSRV_ENABLE_PVR_ION_STATS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_ion_stats.o
+endif
+
+$(PVRSRV_MODNAME)-$(CONFIG_X86) += services/server/env/linux/osfunc_x86.o
+$(PVRSRV_MODNAME)-$(CONFIG_ARM) += services/server/env/linux/osfunc_arm.o
+$(PVRSRV_MODNAME)-$(CONFIG_ARM64) += services/server/env/linux/osfunc_arm64.o
+$(PVRSRV_MODNAME)-$(CONFIG_METAG) += services/server/env/linux/osfunc_metag.o
+$(PVRSRV_MODNAME)-$(CONFIG_MIPS) += services/server/env/linux/osfunc_mips.o
+$(PVRSRV_MODNAME)-$(CONFIG_RISCV) += services/server/env/linux/osfunc_riscv.o
+
+ifeq ($(SUPPORT_ANDROID_PLATFORM),1)
+ ifeq ($(CONFIG_PROC_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_PROC_FS) += services/server/env/linux/pvr_procfs.o
+ else ifeq ($(CONFIG_DEBUG_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_DEBUG_FS) += services/server/env/linux/pvr_debugfs.o
+ endif
+else
+ ifeq ($(CONFIG_DEBUG_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_DEBUG_FS) += services/server/env/linux/pvr_debugfs.o
+ else ifeq ($(CONFIG_PROC_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_PROC_FS) += services/server/env/linux/pvr_procfs.o
+ endif
+endif
+
+ifeq ($(SUPPORT_DI_BRG_IMPL),1)
+$(PVRSRV_MODNAME)-y += services/server/common/di_impl_brg.o
+endif
+$(PVRSRV_MODNAME)-$(CONFIG_EVENT_TRACING) += services/server/env/linux/trace_events.o
+
+ccflags-y += -I$(OUT)/target_neutral/intermediates/firmware
+
+ifeq ($(SUPPORT_RGX),1)
+# Srvinit headers and source files
+
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/$(PVR_ARCH)/rgxsrvinit.o \
+ services/server/devices/rgxfwimageutils.o
+ifeq ($(PVR_ARCH),rogue)
+$(PVRSRV_MODNAME)-y += \
+ services/shared/devices/$(PVR_ARCH)/rgx_hwperf_table.o
+endif
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ services/system/$(PVR_ARCH)/common/env/linux/dma_support.o \
+ services/system/common/env/linux/interrupt_support.o
+
+$(PVRSRV_MODNAME)-$(CONFIG_PCI) += \
+ services/system/common/env/linux/pci_support.o
+
+ccflags-y += \
+ -I$(HWDEFS_DIR)/km
+ifeq ($(PVR_ARCH),rogue)
+ccflags-y += \
+ -I$(TOP)/include/$(PVR_ARCH_DEFS)
+endif
+ccflags-y += \
+ -I$(TOP)/include/$(PVR_ARCH) -I$(TOP)/include \
+ -I$(TOP)/include/$(PVR_ARCH)/public -I$(TOP)/include/public \
+ -I$(TOP)/services/include/$(PVR_ARCH) -I$(TOP)/services/include \
+ -I$(TOP)/services/shared/include \
+ -I$(TOP)/services/server/devices/$(PVR_ARCH) -I$(TOP)/services/server/devices \
+ -I$(TOP)/services/server/include/$(PVR_ARCH) -I$(TOP)/services/server/include \
+ -I$(TOP)/services/shared/common \
+ -I$(TOP)/services/shared/devices \
+ -I$(TOP)/services/system/include \
+ -I$(TOP)/services/system/$(PVR_ARCH)/include \
+ -I$(TOP)/services/server/common/$(PVR_ARCH) -I$(TOP)/services/server/common
+
+ifeq ($(KERNEL_DRIVER_DIR),)
+ ccflags-y += -I$(SYSTEM_DIR)
+endif
+
+# Bridge headers and source files
+
+# Keep in sync with:
+# build/linux/common/bridges.mk AND
+# services/bridge/Linux.mk
+
+ccflags-y += \
+ -I$(bridge_base)/mm_bridge \
+ -I$(bridge_base)/cmm_bridge \
+ -I$(bridge_base)/srvcore_bridge \
+ -I$(bridge_base)/sync_bridge \
+ -I$(bridge_base)/synctracking_bridge \
+ -I$(bridge_base)/pvrtl_bridge \
+ -I$(bridge_base)/cache_bridge \
+ -I$(bridge_base)/dmabuf_bridge
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+ccflags-y += \
+ -I$(bridge_base)/htbuffer_bridge
+endif
+
+ifeq ($(SUPPORT_DMA_TRANSFER),1)
+ccflags-y += \
+ -I$(bridge_base)/dma_bridge
+endif
+
+ifeq ($(SUPPORT_RGX),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxtq2_bridge \
+ -I$(bridge_base)/rgxta3d_bridge \
+ -I$(bridge_base)/rgxhwperf_bridge \
+ -I$(bridge_base)/rgxcmp_bridge \
+ -I$(bridge_base)/rgxregconfig_bridge \
+ -I$(bridge_base)/rgxtimerquery_bridge \
+ -I$(bridge_base)/rgxfwdbg_bridge
+ifeq ($(PVR_ARCH),volcanic)
+ccflags-y += \
+ -I$(bridge_base)/rgxray_bridge
+endif
+ifeq ($(PVR_ARCH),rogue)
+ccflags-y += \
+ -I$(bridge_base)/rgxtq_bridge
+endif
+ifeq ($(SUPPORT_USC_BREAKPOINT),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxbreakpoint_bridge
+endif
+ifeq ($(SUPPORT_RGXKICKSYNC_BRIDGE),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxkicksync_bridge
+endif
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/mm_bridge/server_mm_bridge.o \
+ generated/$(PVR_ARCH)/cmm_bridge/server_cmm_bridge.o \
+ generated/$(PVR_ARCH)/srvcore_bridge/server_srvcore_bridge.o \
+ generated/$(PVR_ARCH)/sync_bridge/server_sync_bridge.o \
+ generated/$(PVR_ARCH)/pvrtl_bridge/server_pvrtl_bridge.o \
+ generated/$(PVR_ARCH)/cache_bridge/server_cache_bridge.o \
+ generated/$(PVR_ARCH)/dmabuf_bridge/server_dmabuf_bridge.o
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/htbuffer_bridge/server_htbuffer_bridge.o
+endif
+
+ifeq ($(SUPPORT_DMA_TRANSFER),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/dma_bridge/server_dma_bridge.o
+endif
+
+ifeq ($(SUPPORT_RGX),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxtq2_bridge/server_rgxtq2_bridge.o \
+ generated/$(PVR_ARCH)/rgxta3d_bridge/server_rgxta3d_bridge.o \
+ generated/$(PVR_ARCH)/rgxhwperf_bridge/server_rgxhwperf_bridge.o \
+ generated/$(PVR_ARCH)/rgxcmp_bridge/server_rgxcmp_bridge.o \
+ generated/$(PVR_ARCH)/rgxregconfig_bridge/server_rgxregconfig_bridge.o \
+ generated/$(PVR_ARCH)/rgxtimerquery_bridge/server_rgxtimerquery_bridge.o \
+ generated/$(PVR_ARCH)/rgxfwdbg_bridge/server_rgxfwdbg_bridge.o
+ifeq ($(PVR_ARCH),volcanic)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxray_bridge/server_rgxray_bridge.o
+endif
+ifeq ($(PVR_ARCH),rogue)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxtq_bridge/server_rgxtq_bridge.o
+endif
+ifeq ($(SUPPORT_USC_BREAKPOINT),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.o
+endif
+ifeq ($(SUPPORT_RGXKICKSYNC_BRIDGE),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxkicksync_bridge/server_rgxkicksync_bridge.o
+endif
+endif
+
+ifeq ($(SUPPORT_WRAP_EXTMEM),1)
+ccflags-y += -I$(bridge_base)/mmextmem_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/mmextmem_bridge/server_mmextmem_bridge.o
+endif
+
+ifeq ($(SUPPORT_DISPLAY_CLASS),1)
+ccflags-y += -I$(bridge_base)/dc_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/dc_bridge/server_dc_bridge.o
+endif
+
+ifeq ($(SUPPORT_SECURE_EXPORT),1)
+ccflags-y += -I$(bridge_base)/smm_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/smm_bridge/server_smm_bridge.o
+endif
+
+ifeq ($(PDUMP),1)
+ccflags-y += \
+ -I$(bridge_base)/pdump_bridge \
+ -I$(bridge_base)/pdumpctrl_bridge \
+ -I$(bridge_base)/pdumpmm_bridge
+
+ifeq ($(SUPPORT_RGX),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxpdump_bridge
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxpdump_bridge/server_rgxpdump_bridge.o
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/pdump_bridge/server_pdump_bridge.o \
+ generated/$(PVR_ARCH)/pdumpctrl_bridge/server_pdumpctrl_bridge.o \
+ generated/$(PVR_ARCH)/pdumpmm_bridge/server_pdumpmm_bridge.o
+endif
+
+ifeq ($(PVRSRV_ENABLE_GPU_MEMORY_INFO),1)
+ccflags-y += -I$(bridge_base)/ri_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/ri_bridge/server_ri_bridge.o
+endif
+
+ifeq ($(SUPPORT_VALIDATION),1)
+ccflags-y += -I$(bridge_base)/validation_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/validation_bridge/server_validation_bridge.o
+$(PVRSRV_MODNAME)-y += services/server/common/validation.o
+$(PVRSRV_MODNAME)-y += services/server/common/validation_soc.o
+endif
+
+ifeq ($(PVR_TESTING_UTILS),1)
+ccflags-y += -I$(bridge_base)/tutils_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/tutils_bridge/server_tutils_bridge.o
+endif
+
+ccflags-y += -I$(bridge_base)/devicememhistory_bridge
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/devicememhistory_bridge/server_devicememhistory_bridge.o
+
+ccflags-y += -I$(bridge_base)/synctracking_bridge
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/synctracking_bridge/server_synctracking_bridge.o
+
+ifeq ($(SUPPORT_FALLBACK_FENCE_SYNC),1)
+ccflags-y += \
+ -I$(bridge_base)/syncfallback_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/syncfallback_bridge/server_syncfallback_bridge.o
+endif
+
+ifeq ($(SUPPORT_DI_BRG_IMPL),1)
+ccflags-y += -I$(bridge_base)/di_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/di_bridge/server_di_bridge.o
+endif
+
+
+# Direct bridges
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/mm_bridge/client_mm_direct_bridge.o \
+ generated/$(PVR_ARCH)/sync_bridge/client_sync_direct_bridge.o \
+ generated/$(PVR_ARCH)/cache_bridge/client_cache_direct_bridge.o \
+ generated/$(PVR_ARCH)/pvrtl_bridge/client_pvrtl_direct_bridge.o
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+$(PVRSRV_MODNAME)-y +=\
+ generated/$(PVR_ARCH)/htbuffer_bridge/client_htbuffer_direct_bridge.o
+endif
+
+ifeq ($(PDUMP),1)
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/pdumpmm_bridge/client_pdumpmm_direct_bridge.o
+endif
+
+ifeq ($(PVRSRV_ENABLE_GPU_MEMORY_INFO),1)
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/ri_bridge/client_ri_direct_bridge.o
+endif
+
+ifeq ($(PDUMP),1)
+ $(PVRSRV_MODNAME)-y += \
+  generated/$(PVR_ARCH)/pdump_bridge/client_pdump_direct_bridge.o \
+  generated/$(PVR_ARCH)/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.o
+
+ifeq ($(SUPPORT_RGX),1)
+ $(PVRSRV_MODNAME)-y += \
+  generated/$(PVR_ARCH)/rgxpdump_bridge/client_rgxpdump_direct_bridge.o
+endif
+
+endif
+
+# Enable -Werror for all built object files
+ifneq ($(W),1)
+$(foreach _o,$(addprefix CFLAGS_,$($(PVRSRV_MODNAME)-y)),$(eval $(_o) += -Werror))
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/devicememhistory_bridge/client_devicememhistory_direct_bridge.o
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/synctracking_bridge/client_synctracking_direct_bridge.o
+
+# Ignore address-of-packed-member warning for all bridge files
+$(foreach _o,$(addprefix CFLAGS_,$(filter generated/%.o,$($(PVRSRV_MODNAME)-y))),$(eval $(_o) += -Wno-address-of-packed-member))
+
+# With certain build configurations, e.g., ARM, Werror, we get a build
+# failure in the ftrace Linux kernel header.  So disable the relevant check.
+CFLAGS_services/server/env/linux/trace_events.o := -Wno-missing-prototypes
+
+# Make sure the mem_utils are built in 'free standing' mode, so the compiler
+# is not encouraged to call out to C library functions
+ifeq ($(CC),clang)
+ ifneq ($(SUPPORT_ANDROID_PLATFORM),1)
+  CFLAGS_services/shared/common/mem_utils.o := -ffreestanding -fforce-enable-int128
+ else
+  CFLAGS_services/shared/common/mem_utils.o := -ffreestanding
+ endif
+endif
+
+# Chrome OS kernel adds some issues
+ccflags-y += -Wno-ignored-qualifiers
+
+# Treat #warning as a warning
+ccflags-y += -Wno-error=cpp
+
+include $(SYSTEM_DIR)/Kbuild.mk
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Linux.mk
new file mode 100644
index 000000000000..caf3a077a3a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Linux.mk
@@ -0,0 +1,46 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+modules := srvkm
+
+srvkm_type := kernel_module
+srvkm_target := $(PVRSRV_MODNAME).ko
+srvkm_makefile := $(THIS_DIR)/Kbuild.mk
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/allocmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/allocmem.c
new file mode 100644
index 000000000000..817680376d79
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/allocmem.c
@@ -0,0 +1,377 @@
+/*************************************************************************/ /*!
+@File
+@Title          Host memory management implementation for Linux
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+
+#include "img_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "process_stats.h"
+#include "osfunc.h"
+
+
+/*
+ * When memory statistics are disabled, memory records are used instead.
+ * In order for these to work, the PID of the process that requested the
+ * allocation needs to be stored at the end of the kmalloc'd memory, making
+ * sure 4 extra bytes are allocated to fit the PID.
+ *
+ * There is no need for this extra allocation when memory statistics are
+ * enabled, since all allocations are tracked in DebugFS mem_area files.
+ */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+/* kmalloc guarantees a minimal alignment which is ARCH_KMALLOC_MINALIGN. This
+ * alignment is architecture specific and can be quite big, e.g. on Aarch64
+ * it can be 64 bytes. This is too much for keeping a single PID field and could
+ * lead to a lot of wasted memory. This is a reason why we're defaulting to 8
+ * bytes alignment which should be enough for any architecture.
+ */
+#define ALLOCMEM_PID_SIZE_PADDING PVR_ALIGN(sizeof(IMG_UINT32), 8)
+#else
+#define ALLOCMEM_PID_SIZE_PADDING 0UL
+#endif
+
+/* How many times kmalloc can fail before the allocation threshold is reduced */
+static const IMG_UINT32 g_ui32kmallocFailLimit = 10;
+/* How many kmalloc failures happened since the last allocation threshold change */
+static IMG_UINT32 g_ui32kmallocFailCount = 0;
+/* Current kmalloc threshold value in bytes */
+static IMG_UINT32 g_ui32kmallocThreshold = PVR_LINUX_KMALLOC_ALLOCATION_THRESHOLD;
+/* Spinlock used so that the global variables above may not be modified by more than 1 thread at a time */
+static DEFINE_SPINLOCK(kmalloc_lock);
+
+
+static inline void OSTryDecreaseKmallocThreshold(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&kmalloc_lock, flags);
+
+	g_ui32kmallocFailCount++;
+
+	if (g_ui32kmallocFailCount >= g_ui32kmallocFailLimit)
+	{
+		g_ui32kmallocFailCount = 0;
+		if (g_ui32kmallocThreshold > PAGE_SIZE)
+		{
+			g_ui32kmallocThreshold >>= 1;
+			printk(KERN_INFO "Threshold is now set to %d\n", g_ui32kmallocThreshold);
+		}
+	}
+
+	spin_unlock_irqrestore(&kmalloc_lock, flags);
+}
+
+static inline void OSResetKmallocFailCount(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&kmalloc_lock, flags);
+
+	g_ui32kmallocFailCount = 0;
+
+	spin_unlock_irqrestore(&kmalloc_lock, flags);
+}
+
+static inline void _pvr_vfree(const void* pvAddr)
+{
+#if defined(DEBUG)
+	/* Size harder to come by for vmalloc and since vmalloc allocates
+	 * a whole number of pages, poison the minimum size known to have
+	 * been allocated.
+	 */
+	OSCachedMemSet((void*)pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE,
+	               PAGE_SIZE);
+#endif
+	vfree(pvAddr);
+}
+
+static inline void _pvr_kfree(const void* pvAddr)
+{
+#if defined(DEBUG)
+	/* Poison whole memory block */
+	OSCachedMemSet((void*)pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE,
+	               ksize(pvAddr));
+#endif
+	kfree(pvAddr);
+}
+
+static inline void *_pvr_alloc_stats_add(void *pvAddr, IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+#if !defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVR_UNREFERENCED_PARAMETER(pvAddr);
+#else
+	if (!is_vmalloc_addr(pvAddr))
+	{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_KMALLOC,
+									  pvAddr,
+									  sCpuPAddr,
+									  ksize(pvAddr),
+									  OSGetCurrentClientProcessIDKM()
+									  DEBUG_MEMSTATS_ARGS);
+#else
+		/* because clang has some features that allow detection out-of-bounds
+		 * access we need to put the metadata in the beginning of the allocation */
+		*(IMG_UINT32 *) pvAddr = OSGetCurrentClientProcessIDKM();
+		PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_KMALLOC, ksize(pvAddr),
+		                            *(IMG_UINT32 *) pvAddr);
+
+		/* because metadata is kept in the beginning of the allocation we need
+		 * to return address offset by the ALLOCMEM_PID_SIZE_PADDING */
+		pvAddr = (IMG_UINT8 *) pvAddr + ALLOCMEM_PID_SIZE_PADDING;
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+	else
+	{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+									  pvAddr,
+									  sCpuPAddr,
+									  PVR_ALIGN(ui32Size, PAGE_SIZE),
+									  OSGetCurrentClientProcessIDKM()
+									  DEBUG_MEMSTATS_ARGS);
+#else
+		PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                    PVR_ALIGN(ui32Size, PAGE_SIZE),
+		                                    (IMG_UINT64)(uintptr_t) pvAddr,
+		                                    OSGetCurrentClientProcessIDKM());
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#endif /* !defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return pvAddr;
+}
+
+static inline void *_pvr_alloc_stats_remove(void *pvAddr)
+{
+#if !defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVR_UNREFERENCED_PARAMETER(pvAddr);
+#else
+	if (!is_vmalloc_addr(pvAddr))
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		/* because metadata is kept in the beginning of the allocation we need
+		 * shift address offset by the ALLOCMEM_PID_SIZE_PADDING to the original
+		 * value */
+		pvAddr = (IMG_UINT8 *) pvAddr - ALLOCMEM_PID_SIZE_PADDING;
+
+		/* first 4 bytes of the allocation are the process' PID */
+		PVRSRVStatsDecrMemKAllocStat(ksize(pvAddr), *(IMG_UINT32 *) pvAddr);
+#else
+		PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_KMALLOC,
+		                                (IMG_UINT64)(uintptr_t) pvAddr,
+		                                OSGetCurrentClientProcessIDKM());
+#endif
+	}
+	else
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                      (IMG_UINT64)(uintptr_t) pvAddr);
+#else
+		PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                (IMG_UINT64)(uintptr_t) pvAddr,
+		                                OSGetCurrentClientProcessIDKM());
+#endif
+	}
+#endif /* !defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return pvAddr;
+}
+
+void *(OSAllocMem)(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+	void *pvRet = NULL;
+
+	if ((ui32Size + ALLOCMEM_PID_SIZE_PADDING) <= g_ui32kmallocThreshold)
+	{
+		pvRet = kmalloc(ui32Size + ALLOCMEM_PID_SIZE_PADDING, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vmalloc(ui32Size);
+	}
+
+	if (pvRet != NULL)
+	{
+		pvRet = _pvr_alloc_stats_add(pvRet, ui32Size DEBUG_MEMSTATS_ARGS);
+	}
+
+	return pvRet;
+}
+
+void *(OSAllocZMem)(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+	void *pvRet = NULL;
+
+	if ((ui32Size + ALLOCMEM_PID_SIZE_PADDING) <= g_ui32kmallocThreshold)
+	{
+		pvRet = kzalloc(ui32Size + ALLOCMEM_PID_SIZE_PADDING, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vzalloc(ui32Size);
+	}
+
+	if (pvRet != NULL)
+	{
+		pvRet = _pvr_alloc_stats_add(pvRet, ui32Size DEBUG_MEMSTATS_ARGS);
+	}
+
+	return pvRet;
+}
+
+/*
+ * The parentheses around OSFreeMem prevent the macro in allocmem.h from
+ * applying, as it would break the function's definition.
+ */
+void (OSFreeMem)(void *pvMem)
+{
+	if (pvMem != NULL)
+	{
+		pvMem = _pvr_alloc_stats_remove(pvMem);
+
+		if (!is_vmalloc_addr(pvMem))
+		{
+			_pvr_kfree(pvMem);
+		}
+		else
+		{
+			_pvr_vfree(pvMem);
+		}
+	}
+}
+
+void *OSAllocMemNoStats(IMG_UINT32 ui32Size)
+{
+	void *pvRet = NULL;
+
+	if (ui32Size <= g_ui32kmallocThreshold)
+	{
+		pvRet = kmalloc(ui32Size, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vmalloc(ui32Size);
+	}
+
+	return pvRet;
+}
+
+void *OSAllocZMemNoStats(IMG_UINT32 ui32Size)
+{
+	void *pvRet = NULL;
+
+	if (ui32Size <= g_ui32kmallocThreshold)
+	{
+		pvRet = kzalloc(ui32Size, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vzalloc(ui32Size);
+	}
+
+	return pvRet;
+}
+
+/*
+ * The parentheses around OSFreeMemNoStats prevent the macro in allocmem.h from
+ * applying, as it would break the function's definition.
+ */
+void (OSFreeMemNoStats)(void *pvMem)
+{
+	if (pvMem != NULL)
+	{
+		if (!is_vmalloc_addr(pvMem))
+		{
+			_pvr_kfree(pvMem);
+		}
+		else
+		{
+			_pvr_vfree(pvMem);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.c
new file mode 100644
index 000000000000..6136b627d9ae
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.c
@@ -0,0 +1,409 @@
+/*************************************************************************/ /*!
+@File         dkf_server.c
+@Title        DRM Key Framework support routines.
+@Copyright    Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dkf_server.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvrsrv_error.h"
+#include "dkp_impl.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#if defined(SUPPORT_LINUX_FDINFO)
+typedef struct DKF_REGISTERED_DKP_TAG
+{
+	IMG_HANDLE              hPrivHandle;    /*!< Private data - can be NULL */
+	const IMG_CHAR          *pszDKPName;    /*!< DKP provider name */
+	IMG_PID                 pid;            /*!< Process-ID - can be 0 */
+	IMG_UINT32              uiReserved1;    /*!< Reserved field - padding */
+	DKP_PFN_SHOW            *psDKPShowPfn;  /*!< DKP Callback function */
+	DLLIST_NODE             sDKFEntryNode;  /*!< List of DKP entries */
+
+	DKP_CONNECTION_FLAGS    ui32Filter;     /*!< The types of connection to output to */
+} DKF_REGISTERED_DKP;
+
+/* Global sentinel to track all allocated DKP callback key/value pairs */
+typedef struct DKF_TAG
+{
+	POS_LOCK            hDKFListLock;   /*!< Lock for accessing sDKFListNode */
+	IMG_UINT32          ui32NumEntries; /*!< Number of registered DKP entries */
+	IMG_UINT32          uiReserved1;    /*!< Reserved field - padding */
+	DLLIST_NODE         sDKFListNode;   /*!< Head of the DKF_REGISTERED_DKP linked list */
+	DKF_VPRINTF_FUNC    *pfnPrint;  /*!< Default printf-style output fn */
+	void                *pvPrintArg1; /*!< First arg for pfnPrint */
+} DKF;
+
+static DKF *gpsDKF;
+
+static_assert(DKF_CONNECTION_FLAG_INVALID == DKP_CONNECTION_FLAG_INVALID, "DKF and DKP INVALID connection flags do not match");
+static_assert(DKF_CONNECTION_FLAG_SYNC == DKP_CONNECTION_FLAG_SYNC, "DKF and DKP SYNC connection flags do not match");
+static_assert(DKF_CONNECTION_FLAG_SERVICES == DKP_CONNECTION_FLAG_SERVICES, "DKF and DKP SERVICES connection flags do not match");
+
+PVRSRV_ERROR PVRDKFInit(void)
+{
+	DKF *psDKF;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	if (gpsDKF != NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: gpsDKF = %p, NULL expected",
+		        __func__, gpsDKF));
+
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s called", __func__));
+
+	psDKF = OSAllocZMemNoStats(sizeof(*psDKF));
+	PVR_GOTO_IF_NOMEM(psDKF, eError, Error);
+
+	dllist_init(&psDKF->sDKFListNode);
+	eError = OSLockCreate(&psDKF->hDKFListLock);
+	PVR_GOTO_IF_ERROR(eError, ErrorFree);
+
+	gpsDKF = psDKF;
+
+	return PVRSRV_OK;
+
+ErrorFree:
+	OSFreeMemNoStats(psDKF);
+	/* fallthrough */
+
+Error:
+	PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+	return eError;
+}
+
+
+void PVRDKFDeInit(void)
+{
+	IMG_UINT32 uiNumFreed = 0U;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s called", __func__));
+
+	if (gpsDKF == NULL)
+	{
+		return;
+	}
+
+	/* Ensure we leave no data allocated. The DKP instances should have
+	 * been cleaned up by their module deInit processing. Handle badly
+	 * behaved clients here.
+	 */
+	if (gpsDKF->ui32NumEntries > 0)
+	{
+		DLLIST_NODE *psThis, *psNext;
+
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Have %u un-freed allocations remaining", __func__,
+		         gpsDKF->ui32NumEntries));
+
+		OSLockAcquire(gpsDKF->hDKFListLock);
+
+		dllist_foreach_node(&gpsDKF->sDKFListNode, psThis, psNext)
+		{
+			DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(psThis,
+			                                             DKF_REGISTERED_DKP,
+			                                             sDKFEntryNode);
+
+			dllist_remove_node(&psEntry->sDKFEntryNode);
+
+			uiNumFreed++;
+
+			OSFreeMemNoStats(psEntry);
+		}
+
+		if (uiNumFreed != gpsDKF->ui32NumEntries)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could only free %u out of %u",
+			         uiNumFreed, gpsDKF->ui32NumEntries));
+		}
+
+		dllist_remove_node(&gpsDKF->sDKFListNode);
+
+		OSLockRelease(gpsDKF->hDKFListLock);
+	}
+
+	OSLockDestroy(gpsDKF->hDKFListLock);
+
+	OSFreeMemNoStats(gpsDKF);
+
+	gpsDKF = NULL;
+}
+
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    DKF_CONNECTION_FLAGS ui32ConnectionType)
+{
+	PDLLIST_NODE pNext, pNode;
+
+	PVR_ASSERT(gpsDKF != NULL);
+	PVR_ASSERT(pfnPrint != NULL);
+	PVR_ASSERT(ui32ConnectionType != DKF_CONNECTION_FLAG_INVALID);
+
+	OSLockAcquire(gpsDKF->hDKFListLock);
+
+	gpsDKF->pfnPrint = pfnPrint;
+	gpsDKF->pvPrintArg1 = pvArg;
+
+	if (dllist_is_empty(&gpsDKF->sDKFListNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: No DKPs registered", __func__));
+	}
+	else
+	{
+		dllist_foreach_node(&gpsDKF->sDKFListNode, pNode, pNext)
+		{
+			DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(pNode,
+			                                               DKF_REGISTERED_DKP,
+			                                               sDKFEntryNode);
+
+			if (psEntry->psDKPShowPfn != NULL &&
+			    BITMASK_ANY(psEntry->ui32Filter, ui32ConnectionType))
+			{
+				psEntry->psDKPShowPfn(psDevNode, pid,
+				                      psEntry->hPrivHandle);
+			}
+		}
+	}
+
+	OSLockRelease(gpsDKF->hDKFListLock);
+}
+
+/*
+ * Wrapper function to display data using preconfigured DKF-specific output
+ * function.
+ */
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...)
+{
+	DKF_REGISTERED_DKP *psDKFEntry = (DKF_REGISTERED_DKP *)hPrivData;
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list Arglist;
+	DLLIST_NODE *pNode, *pNext;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (psDKFEntry == NULL || gpsDKF == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: NULL DKF entry found (%p, %p)",
+		        __func__, psDKFEntry, gpsDKF));
+		return;
+	}
+
+	PVR_ASSERT(gpsDKF->pfnPrint != NULL);
+
+	/* validate that this is a legitimate output function reference */
+
+	dllist_foreach_node(&gpsDKF->sDKFListNode, pNode, pNext)
+	{
+		DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(pNode,
+		                                               DKF_REGISTERED_DKP,
+		                                               sDKFEntryNode);
+
+		if (psEntry == psDKFEntry)
+		{
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!bFound)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Handle %p not found.", __func__,
+		         hPrivData));
+		return;
+	}
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN, "%s", fmt);
+
+	va_start(Arglist, fmt);
+	gpsDKF->pfnPrint(gpsDKF->pvPrintArg1, szBuffer, &Arglist);
+	va_end(Arglist);
+}
+
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                               const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	DKF_REGISTERED_DKP *psDKFEntry; /* New entry for this DKP */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	/* Check for a NULL argument. Nothing to allocate if we are not provided
+	 * a location to store the DkpHandle reference.
+	 */
+	if (phDkpHandle == NULL || ui32Filter == DKF_CONNECTION_FLAG_INVALID)
+	{
+
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_WARNING, "%s(%p, %s, %p, %p) Called", __func__,
+		         hPrivData, pszDKPName, psShowPfn,
+		         phDkpHandle));
+#endif
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDKFEntry = OSAllocZMemNoStats(sizeof(*psDKFEntry));
+	PVR_GOTO_IF_NOMEM(psDKFEntry, eError, Error);
+
+	OSLockAcquire(gpsDKF->hDKFListLock);
+
+	psDKFEntry->hPrivHandle = hPrivData;
+	psDKFEntry->pszDKPName = pszDKPName;
+	psDKFEntry->psDKPShowPfn = psShowPfn;
+	psDKFEntry->ui32Filter = ui32Filter;
+	dllist_init(&psDKFEntry->sDKFEntryNode);
+
+	/*
+	 * Append the new entry to the end of the gpsDKF list-head. Return a
+	 * reference to the new entry to the caller.
+	 */
+	dllist_add_to_tail(&gpsDKF->sDKFListNode, &psDKFEntry->sDKFEntryNode);
+
+	gpsDKF->ui32NumEntries++;
+
+	OSLockRelease(gpsDKF->hDKFListLock);
+
+	*phDkpHandle = psDKFEntry;
+
+	return PVRSRV_OK;
+
+
+Error:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Error: '%s'", __func__,
+	         PVRSRVGetErrorString(eError)));
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData, PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	DKF_REGISTERED_DKP *psDKFEntry = (DKF_REGISTERED_DKP *)hDkpHandle;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDKFEntry)
+	{
+#if defined(DEBUG)
+		if (psDKFEntry->hPrivHandle == hPrivData)
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Matched %p private handle",
+			         __func__, hDkpHandle));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_VERBOSE,
+			         "%s: Did not find match (%p. vs %p), freeing anyway",
+			         __func__, hPrivData, psDKFEntry->hPrivHandle));
+		}
+#endif	/* DEBUG */
+
+		OSLockAcquire(gpsDKF->hDKFListLock);
+
+		dllist_remove_node(&psDKFEntry->sDKFEntryNode);
+
+		gpsDKF->ui32NumEntries--;
+
+		OSLockRelease(gpsDKF->hDKFListLock);
+
+		OSFreeMemNoStats(psDKFEntry);
+
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+#else
+
+/* Stub routines for earlier kernel versions */
+
+PVRSRV_ERROR PVRDKFInit(void)
+{
+	return PVRSRV_OK;
+}
+
+void PVRDKFDeInit(void)
+{
+}
+
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    IMG_UINT32 ui32ConnectionType)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(pid);
+}
+
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(fmt);
+}
+
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData, const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pszDKPName);
+	PVR_UNREFERENCED_PARAMETER(psShowPfn);
+	PVR_UNREFERENCED_PARAMETER(phDkpHandle);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData, PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(hDkpHandle);
+
+	return PVRSRV_OK;
+}
+#endif	/* SUPPORT_LINUX_FDINFO */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.h
new file mode 100644
index 000000000000..304d9bc71b60
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.h
@@ -0,0 +1,103 @@
+/**************************************************************************/ /*!
+@File           dkf_server.h
+@Title          Functions for supporting the DRM Key Framework
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(DKF_SERVER_H)
+#define DKF_SERVER_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(SUPPORT_LINUX_FDINFO)
+
+#include <drm/drm_print.h>
+typedef void (DKF_VPRINTF_FUNC)(struct drm_printer *p, const char *fmt, va_list *va) __printf(2, 0);
+
+#else /* !defined(SUPPORT_LINUX_FDINFO) */
+typedef void (DKF_VPRINTF_FUNC)(void *p, const char *fmt, ...) __printf(2, 3);
+
+#endif	/* defined(SUPPORT_LINUX_FDINFO) */
+struct _PVRSRV_DEVICE_NODE_;
+
+typedef IMG_UINT32 DKF_CONNECTION_FLAGS;
+
+#define DKF_CONNECTION_FLAG_SYNC        BIT(0)
+#define DKF_CONNECTION_FLAG_SERVICES    BIT(1)
+
+#define DKF_CONNECTION_FLAG_INVALID     IMG_UINT32_C(0)
+
+/*! @Function PVRDKFTraverse
+ *
+ * @Description
+ * Outputs the DKP data associated with the given device node's
+ * framework entries.
+ *
+ * @Input   pfnPrint            The print function callback to be used to output.
+ * @Input   pvArg               Print function first argument.
+ * @Input   psDevNode           Device node associated with fdinfo owner.
+ * @Input   pid                 Process ID of the process owning the FD the fdinfo
+ *                              file relates to.
+ * @Input   ui32ConnectionType  A value indicating the PVR connection type
+ *                              (sync or services).
+ */
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    DKF_CONNECTION_FLAGS ui32ConnectionType);
+
+/* @Function PVRDKFInit
+ *
+ * @Description
+ * Initialises the DKF infrastructure for subsequent usage by the PVR system.
+ *
+ * @Returns  PVRSRV_ERROR.
+ */
+PVRSRV_ERROR PVRDKFInit(void);
+
+/* @Function PVRDKFDeInit
+ *
+ * @Description
+ * Removes and frees all associated system-specific DKF meta-data.
+ */
+void PVRDKFDeInit(void);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/env_connection.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/env_connection.h
new file mode 100644
index 000000000000..d2e4f9743370
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/env_connection.h
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux specific server side connection management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(ENV_CONNECTION_H)
+#define ENV_CONNECTION_H
+
+#include <linux/version.h>
+#include <linux/list.h>
+#include <linux/types.h>
+
+#include "handle.h"
+#include "pvr_debug.h"
+#include "device.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_sys.h"
+#include "allocmem.h"
+#endif
+
+struct drm_file;
+
+typedef struct _ENV_CONNECTION_PRIVATE_DATA_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	struct drm_file *psDRMFile;
+} ENV_CONNECTION_PRIVATE_DATA;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define ION_CLIENT_NAME_SIZE	50
+
+typedef struct _ENV_ION_CONNECTION_DATA_
+{
+	IMG_CHAR azIonClientName[ION_CLIENT_NAME_SIZE];
+	struct ion_device *psIonDev;
+	struct ion_client *psIonClient;
+} ENV_ION_CONNECTION_DATA;
+#endif
+
+typedef struct _ENV_CONNECTION_DATA_
+{
+	pid_t owner;
+
+	PVRSRV_DEVICE_NODE *psDevNode;
+	struct drm_file *psDRMFile;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	void *pvPvrSyncPrivateData;
+#endif
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	ENV_ION_CONNECTION_DATA *psIonData;
+#endif
+} ENV_CONNECTION_DATA;
+
+#endif /* !defined(ENV_CONNECTION_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.c
new file mode 100644
index 000000000000..98b029c926df
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.c
@@ -0,0 +1,516 @@
+/*************************************************************************/ /*!
+@File
+@Title          Event Object
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+
+#include <linux/version.h>
+#include <linux/string.h>
+#include <linux/sched.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,11,0)
+#include <linux/sched/signal.h>
+#endif
+#include <linux/interrupt.h>
+#include <asm/hardirq.h>
+#include <linux/timer.h>
+#include <linux/capability.h>
+#include <linux/freezer.h>
+#include <linux/uaccess.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "event.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "pvr_bridge_k.h"
+
+#include "osfunc.h"
+
+/* Uncomment to enable event object stats that are useful for debugging.
+ * The stats can be gotten at any time (during lifetime of event object)
+ * using OSEventObjectDumpdebugInfo API */
+// #define LINUX_EVENT_OBJECT_STATS
+
+
+typedef struct PVRSRV_LINUX_EVENT_OBJECT_LIST_TAG
+{
+	rwlock_t sLock;
+	/* Counts how many times event object was signalled i.e. how many times
+	 * LinuxEventObjectSignal() was called on a given event object.
+	 * Used for detecting pending signals.
+	 * Note that this is in no way related to OS signals. */
+	atomic_t sEventSignalCount;
+	struct list_head sList;
+} PVRSRV_LINUX_EVENT_OBJECT_LIST;
+
+
+typedef struct PVRSRV_LINUX_EVENT_OBJECT_TAG
+{
+	IMG_UINT32 ui32EventSignalCountPrevious;
+#if defined(DEBUG)
+	IMG_UINT ui32Stats;
+#endif
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	POS_LOCK hLock;
+	IMG_UINT32 ui32ScheduleAvoided;
+	IMG_UINT32 ui32ScheduleCalled;
+	IMG_UINT32 ui32ScheduleSleptFully;
+	IMG_UINT32 ui32ScheduleSleptPartially;
+	IMG_UINT32 ui32ScheduleReturnedImmediately;
+#endif
+	wait_queue_head_t sWait;
+	struct list_head sList;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList;
+} PVRSRV_LINUX_EVENT_OBJECT;
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectListCreate
+
+ @Description
+
+ Linux wait object list creation
+
+ @Output    hOSEventKM : Pointer to the event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectListCreate(IMG_HANDLE *phEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psEvenObjectList;
+
+	psEvenObjectList = OSAllocMem(sizeof(*psEvenObjectList));
+	if (psEvenObjectList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectCreate: failed to allocate memory for event list"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_LIST_HEAD(&psEvenObjectList->sList);
+
+	rwlock_init(&psEvenObjectList->sLock);
+	atomic_set(&psEvenObjectList->sEventSignalCount, 0);
+
+	*phEventObjectList = (IMG_HANDLE *) psEvenObjectList;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectListDestroy
+
+ @Description
+
+ Linux wait object list destruction
+
+ @Input    hOSEventKM : Event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectListDestroy(IMG_HANDLE hEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psEvenObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST *) hEventObjectList;
+
+	if (psEvenObjectList)
+	{
+		if (!list_empty(&psEvenObjectList->sList))
+		{
+			 PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectListDestroy: Event List is not empty"));
+			 return PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+		}
+		OSFreeMem(psEvenObjectList);
+		/*not nulling pointer, copy on stack*/
+	}
+	return PVRSRV_OK;
+}
+
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectDelete
+
+ @Description
+
+ Linux wait object removal
+
+ @Input    hOSEventObject : Event object handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectDelete(IMG_HANDLE hOSEventObject)
+{
+	if (hOSEventObject)
+	{
+		PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)hOSEventObject;
+		PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = psLinuxEventObject->psLinuxEventObjectList;
+
+		write_lock_bh(&psLinuxEventObjectList->sLock);
+		list_del(&psLinuxEventObject->sList);
+		write_unlock_bh(&psLinuxEventObjectList->sLock);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+		OSLockDestroy(psLinuxEventObject->hLock);
+#endif
+
+#if defined(DEBUG)
+//		PVR_DPF((PVR_DBG_MESSAGE, "LinuxEventObjectDelete: Event object waits: %u", psLinuxEventObject->ui32Stats));
+#endif
+
+		OSFreeMem(psLinuxEventObject);
+		/*not nulling pointer, copy on stack*/
+
+		return PVRSRV_OK;
+	}
+	return PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectAdd
+
+ @Description
+
+ Linux wait object addition
+
+ @Input    hOSEventObjectList : Event object list handle
+ @Output   phOSEventObject : Pointer to the event object handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectAdd(IMG_HANDLE hOSEventObjectList, IMG_HANDLE *phOSEventObject)
+ {
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST*)hOSEventObjectList;
+
+	/* allocate completion variable */
+	psLinuxEventObject = OSAllocMem(sizeof(*psLinuxEventObject));
+	if (psLinuxEventObject == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectAdd: failed to allocate memory"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_LIST_HEAD(&psLinuxEventObject->sList);
+
+	/* Start with the timestamp at which event object was added to the list */
+	psLinuxEventObject->ui32EventSignalCountPrevious = atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&psLinuxEventObject->hLock), "OSLockCreate");
+	psLinuxEventObject->ui32ScheduleAvoided = 0;
+	psLinuxEventObject->ui32ScheduleCalled = 0;
+	psLinuxEventObject->ui32ScheduleSleptFully = 0;
+	psLinuxEventObject->ui32ScheduleSleptPartially = 0;
+	psLinuxEventObject->ui32ScheduleReturnedImmediately = 0;
+#endif
+
+#if defined(DEBUG)
+	psLinuxEventObject->ui32Stats = 0;
+#endif
+	init_waitqueue_head(&psLinuxEventObject->sWait);
+
+	psLinuxEventObject->psLinuxEventObjectList = psLinuxEventObjectList;
+
+	write_lock_bh(&psLinuxEventObjectList->sLock);
+	list_add(&psLinuxEventObject->sList, &psLinuxEventObjectList->sList);
+	write_unlock_bh(&psLinuxEventObjectList->sLock);
+
+	*phOSEventObject = psLinuxEventObject;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectSignal
+
+ @Description
+
+ Linux wait object signaling function
+
+ @Input    hOSEventObjectList : Event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectSignal(IMG_HANDLE hOSEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST*)hOSEventObjectList;
+	struct list_head *psListEntry, *psListEntryTemp, *psList;
+	psList = &psLinuxEventObjectList->sList;
+
+	/* Move the timestamp ahead for this call, so a potential "Wait" from any
+	 * EventObject/s doesn't wait for the signal to occur before returning. Early
+	 * setting/incrementing of timestamp reduces the window where a concurrent
+	 * "Wait" call might block while "this" Signal call is being processed */
+	atomic_inc(&psLinuxEventObjectList->sEventSignalCount);
+
+	read_lock_bh(&psLinuxEventObjectList->sLock);
+	list_for_each_safe(psListEntry, psListEntryTemp, psList)
+	{
+		psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)list_entry(psListEntry, PVRSRV_LINUX_EVENT_OBJECT, sList);
+		wake_up_interruptible(&psLinuxEventObject->sWait);
+	}
+	read_unlock_bh(&psLinuxEventObjectList->sLock);
+
+	return PVRSRV_OK;
+}
+
+static void _TryToFreeze(void)
+{
+	/* if we reach zero it means that all of the threads called try_to_freeze */
+	LinuxBridgeNumActiveKernelThreadsDecrement();
+
+	/* Returns true if the thread was frozen, should we do anything with this
+	* information? What do we return? Which one is the error case? */
+	try_to_freeze();
+
+	LinuxBridgeNumActiveKernelThreadsIncrement();
+}
+
+void LinuxEventObjectDumpDebugInfo(IMG_HANDLE hOSEventObject)
+{
+#ifdef LINUX_EVENT_OBJECT_STATS
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)hOSEventObject;
+
+	OSLockAcquire(psLinuxEventObject->hLock);
+	PVR_LOG(("%s: EvObj(%p) schedule: Avoided(%u) Called(%u) ReturnedImmediately(%u) SleptFully(%u) SleptPartially(%u)",
+	         __func__, psLinuxEventObject, psLinuxEventObject->ui32ScheduleAvoided,
+			 psLinuxEventObject->ui32ScheduleCalled, psLinuxEventObject->ui32ScheduleReturnedImmediately,
+			 psLinuxEventObject->ui32ScheduleSleptFully, psLinuxEventObject->ui32ScheduleSleptPartially));
+	OSLockRelease(psLinuxEventObject->hLock);
+#else
+	PVR_LOG(("%s: LINUX_EVENT_OBJECT_STATS disabled!", __func__));
+#endif
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectWait
+
+ @Description
+
+ Linux wait object routine
+
+ @Input    hOSEventObject : Event object handle
+
+ @Input   ui64Timeoutus : Time out value in usec
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectWait(IMG_HANDLE hOSEventObject,
+                                  IMG_UINT64 ui64Timeoutus,
+                                  IMG_BOOL bFreezable)
+{
+	IMG_UINT32 ui32EventSignalCount;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32Remainder;
+	long timeOutJiffies;
+#ifdef LINUX_EVENT_OBJECT_STATS
+	long totalTimeoutJiffies;
+	IMG_BOOL bScheduleCalled = IMG_FALSE;
+#endif
+
+	DEFINE_WAIT(sWait);
+
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT*)hOSEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = psLinuxEventObject->psLinuxEventObjectList;
+
+	PVR_ASSERT(psLinuxEventObjectList != NULL);
+
+	/* Check if the driver is good shape */
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* usecs_to_jiffies only takes an uint. So if our timeout is bigger than an
+	 * uint use the msec version. With such a long timeout we really don't need
+	 * the high resolution of usecs. */
+	if (ui64Timeoutus > 0xffffffffULL)
+		timeOutJiffies = msecs_to_jiffies(OSDivide64(ui64Timeoutus, 1000, &ui32Remainder));
+	else
+		timeOutJiffies = usecs_to_jiffies(ui64Timeoutus);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	totalTimeoutJiffies = timeOutJiffies;
+#endif
+
+	do
+	{
+		prepare_to_wait(&psLinuxEventObject->sWait, &sWait, TASK_INTERRUPTIBLE);
+		ui32EventSignalCount = (IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+		if (psLinuxEventObject->ui32EventSignalCountPrevious != ui32EventSignalCount)
+		{
+			/* There is a pending event signal i.e. LinuxEventObjectSignal()
+			 * was called on the event object since the last time we checked.
+			 * Return without waiting. */
+			break;
+		}
+
+		if (signal_pending(current))
+		{
+			/* There is an OS signal pending so return.
+			 * This allows to kill/interrupt user space processes which
+			 * are waiting on this event object. */
+			break;
+		}
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+		bScheduleCalled = IMG_TRUE;
+#endif
+		timeOutJiffies = schedule_timeout(timeOutJiffies);
+
+		if (bFreezable)
+		{
+			_TryToFreeze();
+		}
+
+#if defined(DEBUG)
+		psLinuxEventObject->ui32Stats++;
+#endif
+
+
+	} while (timeOutJiffies);
+
+	finish_wait(&psLinuxEventObject->sWait, &sWait);
+
+	psLinuxEventObject->ui32EventSignalCountPrevious = ui32EventSignalCount;
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	OSLockAcquire(psLinuxEventObject->hLock);
+	if (bScheduleCalled)
+	{
+		psLinuxEventObject->ui32ScheduleCalled++;
+		if (totalTimeoutJiffies == timeOutJiffies)
+		{
+			psLinuxEventObject->ui32ScheduleReturnedImmediately++;
+		}
+		else if (timeOutJiffies == 0)
+		{
+			psLinuxEventObject->ui32ScheduleSleptFully++;
+		}
+		else
+		{
+			psLinuxEventObject->ui32ScheduleSleptPartially++;
+		}
+	}
+	else
+	{
+		psLinuxEventObject->ui32ScheduleAvoided++;
+	}
+	OSLockRelease(psLinuxEventObject->hLock);
+#endif
+
+	if (signal_pending(current) && test_tsk_thread_flag(current, TIF_SIGPENDING))
+	{
+		return PVRSRV_ERROR_INTERRUPTED;
+	}
+	else
+	{
+		return timeOutJiffies ? PVRSRV_OK : PVRSRV_ERROR_TIMEOUT;
+	}
+}
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+PVRSRV_ERROR LinuxEventObjectWaitUntilSignalled(IMG_HANDLE hOSEventObject)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	DEFINE_WAIT(sWait);
+
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject =
+			(PVRSRV_LINUX_EVENT_OBJECT *) hOSEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList =
+			psLinuxEventObject->psLinuxEventObjectList;
+
+	/* Check if the driver is in good shape */
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	prepare_to_wait(&psLinuxEventObject->sWait, &sWait, TASK_INTERRUPTIBLE);
+
+	if (psLinuxEventObject->ui32EventSignalCountPrevious !=
+	    (IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount))
+	{
+		/* There is a pending signal, so return without waiting */
+		goto finish;
+	}
+
+	schedule();
+
+	_TryToFreeze();
+
+finish:
+	finish_wait(&psLinuxEventObject->sWait, &sWait);
+
+	psLinuxEventObject->ui32EventSignalCountPrevious =
+			(IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.h
new file mode 100644
index 000000000000..bb378cb9220f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@File
+@Title          Event Object
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR LinuxEventObjectListCreate(IMG_HANDLE *phEventObjectList);
+PVRSRV_ERROR LinuxEventObjectListDestroy(IMG_HANDLE hEventObjectList);
+PVRSRV_ERROR LinuxEventObjectAdd(IMG_HANDLE hOSEventObjectList, IMG_HANDLE *phOSEventObject);
+PVRSRV_ERROR LinuxEventObjectDelete(IMG_HANDLE hOSEventObject);
+PVRSRV_ERROR LinuxEventObjectSignal(IMG_HANDLE hOSEventObjectList);
+PVRSRV_ERROR LinuxEventObjectWait(IMG_HANDLE hOSEventObject,
+                                  IMG_UINT64 ui64Timeoutus,
+                                  IMG_BOOL bFreezable);
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+PVRSRV_ERROR LinuxEventObjectWaitUntilSignalled(IMG_HANDLE hOSEventObject);
+#endif
+void LinuxEventObjectDumpDebugInfo(IMG_HANDLE hOSEventObject);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/fwload.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/fwload.c
new file mode 100644
index 000000000000..8c597b3c2e28
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/fwload.c
@@ -0,0 +1,259 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services firmware load and access routines for Linux
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/firmware.h>
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/err.h>
+
+#include "device.h"
+#include "module_common.h"
+#include "fwload.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+
+#if defined(RGX_FW_SIGNED)
+
+#include <linux/verification.h>
+#include <linux/module.h>
+#include <crypto/public_key.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/module_signature.h>
+#else
+#define PKEY_ID_PKCS7 2
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) */
+
+#include "signfw.h"
+#endif /* RGX_FW_SIGNED */
+
+struct OS_FW_IMAGE_t
+{
+	const struct firmware *psFW;
+	size_t                 uSignatureSize;
+};
+
+#if defined(RGX_FW_SIGNED)
+
+static int OSCheckSignature(const struct FirmwareSignatureHeader *psHeader, size_t uSize)
+{
+	if (be32_to_cpu(psHeader->ui32SignatureLen) >= uSize - sizeof(*psHeader))
+	{
+		return -EBADMSG;
+	}
+
+	if (psHeader->ui8IDType != PKEY_ID_PKCS7)
+	{
+		return -ENOPKG;
+	}
+
+	if (psHeader->ui8Algo != 0 || psHeader->ui8HashAlgo != 0 ||
+	    psHeader->ui8SignerLen != 0 || psHeader->ui8KeyIDLen != 0 ||
+	    psHeader->__ui8Padding[0] != 0 || psHeader->__ui8Padding[1] != 0 ||
+	    psHeader->__ui8Padding[2] != 0)
+	{
+		return -EBADMSG;
+	}
+
+	return 0;
+}
+
+bool OSVerifyFirmware(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW        = psFWImage->psFW;
+	const u8              *pui8FWData  = psFW->data;
+	size_t                uFWSize      = psFW->size;
+	uint32_t              ui32MagicLen = sizeof(MODULE_SIG_STRING) - 1;
+	struct FirmwareSignatureHeader sHeader;
+	int                            err;
+
+	if (uFWSize <= ui32MagicLen)
+	{
+		return false;
+	}
+
+	/*
+	 * Linux Kernel's sign-file utility is primarily intended for signing
+	 * modules, and so appends the MODULE_SIG_STRING magic at the end of
+	 * the signature. Only proceed with verification if this magic is found.
+	 */
+	if (memcmp(pui8FWData + uFWSize - ui32MagicLen, MODULE_SIG_STRING, ui32MagicLen) != 0)
+	{
+		return false;
+	}
+
+	uFWSize -= ui32MagicLen;
+	if (uFWSize <= sizeof(sHeader))
+	{
+		return false;
+	}
+
+	/*
+	 * After the magic, a header is placed which informs about the digest /
+	 * crypto algorithm etc. Copy that header and ensure that it has valid
+	 * contents (We only support RSA Crypto, SHA Hash, X509 certificate and
+	 * PKCS#7 signature).
+	 */
+	memcpy(&sHeader, pui8FWData + (uFWSize - sizeof(sHeader)), sizeof(sHeader));
+	if (OSCheckSignature(&sHeader, uFWSize) != 0)
+	{
+		return false;
+	}
+
+	/*
+	 * As all information is now extracted, we can go ahead and ask PKCS
+	 * module to verify the sign.
+	 */
+	uFWSize -= be32_to_cpu(sHeader.ui32SignatureLen) + sizeof(sHeader);
+	err = verify_pkcs7_signature(pui8FWData, uFWSize, pui8FWData + uFWSize,
+				     be32_to_cpu(sHeader.ui32SignatureLen), NULL,
+				     VERIFYING_UNSPECIFIED_SIGNATURE, NULL, NULL);
+	if (err == 0)
+	{
+		psFWImage->uSignatureSize = psFW->size - uFWSize;
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware Successfully Verified",
+						__func__));
+		return true;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Verification Failed (%d)",
+					__func__, err));
+	return false;
+}
+
+#else /* defined(RGX_FW_SIGNED) */
+
+inline bool OSVerifyFirmware(OS_FW_IMAGE *psFWImage)
+{
+	return true;
+}
+
+#endif /* defined(RGX_FW_SIGNED) */
+
+PVRSRV_ERROR
+OSLoadFirmware(PVRSRV_DEVICE_NODE *psDeviceNode, const IMG_CHAR *pszBVNCString,
+               bool (*pfnVerifyFirmware)(OS_FW_IMAGE*), OS_FW_IMAGE **ppsFWImage)
+{
+	const struct firmware *psFW = NULL;
+	OS_FW_IMAGE *psFWImage;
+	IMG_INT32    res;
+	PVRSRV_ERROR eError;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0))
+	res = firmware_request_nowarn(&psFW, pszBVNCString, psDeviceNode->psDevConfig->pvOSDevice);
+#else
+	res = request_firmware(&psFW, pszBVNCString, psDeviceNode->psDevConfig->pvOSDevice);
+#endif
+	if (res != 0)
+	{
+		release_firmware(psFW);
+		if (res == -ENOENT)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: requested firmware('%s') not found (%d)",
+							__func__, pszBVNCString, res));
+			eError = PVRSRV_ERROR_NOT_FOUND;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: requested firmware('%s') not ready (%d)",
+							__func__, pszBVNCString, res));
+			eError = PVRSRV_ERROR_NOT_READY;
+		}
+		goto err_exit;
+	}
+
+	psFWImage = OSAllocZMem(sizeof(*psFWImage));
+	if (psFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: OSAllocZMem('%s') failed.",
+						__func__, pszBVNCString));
+
+		release_firmware(psFW);
+		eError =  PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_exit;
+	}
+
+	psFWImage->psFW = psFW;
+	if (pfnVerifyFirmware != NULL && !pfnVerifyFirmware(psFWImage))
+	{
+		release_firmware(psFW);
+		OSFreeMem(psFWImage);
+		eError = PVRSRV_ERROR_NOT_AUTHENTICATED;
+		goto err_exit;
+	}
+
+	*ppsFWImage = psFWImage;
+	return PVRSRV_OK;
+
+err_exit:
+	*ppsFWImage = NULL;
+	return eError;
+}
+
+void
+OSUnloadFirmware(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+
+	release_firmware(psFW);
+	OSFreeMem(psFWImage);
+}
+
+size_t
+OSFirmwareSize(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+	return psFW->size - psFWImage->uSignatureSize;
+}
+
+const void *
+OSFirmwareData(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+
+	return psFW->data;
+}
+
+/******************************************************************************
+ End of file (fwload.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/handle_idr.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/handle_idr.c
new file mode 100644
index 000000000000..48bb8f4410e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/handle_idr.c
@@ -0,0 +1,417 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Handle Manager - IDR Back-end
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide IDR based resource handle management back-end
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/gfp.h>
+#include <linux/idr.h>
+#include <linux/pid.h>
+
+#include "handle_impl.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#define ID_VALUE_MIN	1
+#define ID_VALUE_MAX	INT_MAX
+
+#define	ID_TO_HANDLE(i) ((IMG_HANDLE)(uintptr_t)(i))
+#define	HANDLE_TO_ID(h) ((IMG_INT)(uintptr_t)(h))
+
+struct _HANDLE_IMPL_BASE_
+{
+	struct idr sIdr;
+
+	IMG_UINT32 ui32MaxHandleValue;
+
+	IMG_UINT32 ui32TotalHandCount;
+};
+
+typedef struct _HANDLE_ITER_DATA_WRAPPER_
+{
+	PFN_HANDLE_ITER pfnHandleIter;
+	void *pvHandleIterData;
+} HANDLE_ITER_DATA_WRAPPER;
+
+
+static int HandleIterFuncWrapper(int id, void *data, void *iter_data)
+{
+	HANDLE_ITER_DATA_WRAPPER *psIterData = (HANDLE_ITER_DATA_WRAPPER *)iter_data;
+
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	return (int)psIterData->pfnHandleIter(ID_TO_HANDLE(id), psIterData->pvHandleIterData);
+}
+
+/*!
+******************************************************************************
+
+ @Function	AcquireHandle
+
+ @Description	Acquire a new handle
+
+ @Input		psBase - Pointer to handle base structure
+		phHandle - Points to a handle pointer
+		pvData - Pointer to resource to be associated with the handle
+
+ @Output	phHandle - Points to a handle pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR AcquireHandle(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData)
+{
+	int id;
+	int result;
+
+	PVR_ASSERT(psBase != NULL);
+	PVR_ASSERT(phHandle != NULL);
+	PVR_ASSERT(pvData != NULL);
+
+	idr_preload(GFP_KERNEL);
+	id = idr_alloc(&psBase->sIdr, pvData, ID_VALUE_MIN, psBase->ui32MaxHandleValue + 1, 0);
+	idr_preload_end();
+
+	result = id;
+
+	if (result < 0)
+	{
+		if (result == -ENOSPC)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Limit of %u handles reached",
+				 __func__, psBase->ui32MaxHandleValue));
+
+			return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		}
+
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psBase->ui32TotalHandCount++;
+
+	*phHandle = ID_TO_HANDLE(id);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	ReleaseHandle
+
+ @Description	Release a handle that is no longer needed.
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle to release
+		ppvData - Points to a void data pointer
+
+ @Output	ppvData - Points to a void data pointer
+
+ @Return	PVRSRV_OK or PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR ReleaseHandle(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void **ppvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvData;
+
+	PVR_ASSERT(psBase);
+
+	/* Get the data associated with the handle. If we get back NULL then
+	   it's an invalid handle */
+
+	pvData = idr_find(&psBase->sIdr, id);
+	if (likely(pvData))
+	{
+		idr_remove(&psBase->sIdr, id);
+		psBase->ui32TotalHandCount--;
+	}
+
+	if (unlikely(pvData == NULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle out of range (%u > %u)",
+			 __func__, id, psBase->ui32TotalHandCount));
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+
+	if (ppvData)
+	{
+		*ppvData = pvData;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	GetHandleData
+
+ @Description	Get the data associated with the given handle
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle from which data should be retrieved
+		ppvData - Points to a void data pointer
+
+ @Output	ppvData - Points to a void data pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR GetHandleData(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void **ppvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvData;
+
+	PVR_ASSERT(psBase);
+	PVR_ASSERT(ppvData);
+
+	pvData = idr_find(&psBase->sIdr, id);
+	if (likely(pvData))
+	{
+		*ppvData = pvData;
+
+		return PVRSRV_OK;
+	}
+	else
+	{
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+}
+
+/*!
+******************************************************************************
+
+ @Function	SetHandleData
+
+ @Description	Set the data associated with the given handle
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle for which data should be changed
+		pvData - Pointer to new data to be associated with the handle
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR SetHandleData(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void *pvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvOldData;
+
+	PVR_ASSERT(psBase);
+
+	pvOldData = idr_replace(&psBase->sIdr, pvData, id);
+	if (IS_ERR(pvOldData))
+	{
+		if (PTR_ERR(pvOldData) == -ENOENT)
+		{
+			return PVRSRV_ERROR_HANDLE_NOT_ALLOCATED;
+		}
+		else
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR IterateOverHandles(HANDLE_IMPL_BASE *psBase, PFN_HANDLE_ITER pfnHandleIter, void *pvHandleIterData)
+{
+	HANDLE_ITER_DATA_WRAPPER sIterData;
+
+	PVR_ASSERT(psBase);
+	PVR_ASSERT(pfnHandleIter);
+
+	sIterData.pfnHandleIter = pfnHandleIter;
+	sIterData.pvHandleIterData = pvHandleIterData;
+
+	return (PVRSRV_ERROR)idr_for_each(&psBase->sIdr, HandleIterFuncWrapper, &sIterData);
+}
+
+/*!
+******************************************************************************
+
+ @Function	EnableHandlePurging
+
+ @Description	Enable purging for a given handle base
+
+ @Input		psBase - pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR EnableHandlePurging(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+	PVR_ASSERT(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PurgeHandles
+
+ @Description	Purge handles for a given handle base
+
+ @Input		psBase - Pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR PurgeHandles(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+	PVR_ASSERT(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	CreateHandleBase
+
+ @Description	Create a handle base structure
+
+ @Input		ppsBase - pointer to handle base structure pointer
+
+ @Output	ppsBase - points to handle base structure pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR CreateHandleBase(HANDLE_IMPL_BASE **ppsBase)
+{
+	HANDLE_IMPL_BASE *psBase;
+
+	PVR_ASSERT(ppsBase);
+
+	psBase = OSAllocZMem(sizeof(*psBase));
+	if (psBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't allocate generic handle base",
+				 __func__));
+
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	idr_init(&psBase->sIdr);
+
+	psBase->ui32MaxHandleValue = ID_VALUE_MAX;
+	psBase->ui32TotalHandCount = 0;
+
+	*ppsBase = psBase;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	DestroyHandleBase
+
+ @Description	Destroy a handle base structure
+
+ @Input		psBase - pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR DestroyHandleBase(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_ASSERT(psBase);
+
+	/* Finally destroy the idr */
+	idr_destroy(&psBase->sIdr);
+
+	OSFreeMem(psBase);
+
+	return PVRSRV_OK;
+}
+
+static const HANDLE_IMPL_FUNCTAB g_sHandleFuncTab =
+{
+	.pfnAcquireHandle = AcquireHandle,
+	.pfnReleaseHandle = ReleaseHandle,
+	.pfnGetHandleData = GetHandleData,
+	.pfnSetHandleData = SetHandleData,
+	.pfnIterateOverHandles = IterateOverHandles,
+	.pfnEnableHandlePurging = EnableHandlePurging,
+	.pfnPurgeHandles = PurgeHandles,
+	.pfnCreateHandleBase = CreateHandleBase,
+	.pfnDestroyHandleBase = DestroyHandleBase,
+};
+
+PVRSRV_ERROR PVRSRVHandleGetFuncTable(HANDLE_IMPL_FUNCTAB const **ppsFuncs)
+{
+	static IMG_BOOL bAcquired = IMG_FALSE;
+
+	if (bAcquired)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Function table already acquired",
+			 __func__));
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	if (ppsFuncs == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*ppsFuncs = &g_sHandleFuncTab;
+
+	bAcquired = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.c
new file mode 100644
index 000000000000..6e9bfd6f07d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.c
@@ -0,0 +1,1767 @@
+/*************************************************************************/ /*!
+@File           km_apphint.c
+@Title          Apphint routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "di_server.h"
+#include "pvr_uaccess.h"
+#include <linux/moduleparam.h>
+#include <linux/workqueue.h>
+#include <linux/string.h>
+
+/* Common and SO layer */
+#include "img_defs.h"
+#include "sofunc_pvr.h"
+
+/* for action device access */
+#include "pvrsrv.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+#include "rgxutils.h"
+#include "rgxapi_km.h"
+
+
+/* defines for default values */
+#include "rgx_fwif_km.h"
+#include "htbuffer_types.h"
+
+#include "pvr_notifier.h"
+
+#include "km_apphint_defs.h"
+#include "km_apphint.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#include "pdump_km.h"
+#endif
+
+/* Size of temporary buffers used to read and write AppHint data.
+ * Must be large enough to contain any strings read or written but no larger
+ * than 4096: which is the buffer size for the kernel_param_ops .get
+ * function. And less than 1024 to keep the stack frame size within bounds.
+ */
+#define APPHINT_BUFFER_SIZE 512
+
+/* Apphint Debug output level */
+#define APPHINT_DPF_LEVEL PVR_DBG_VERBOSE
+
+/*
+*******************************************************************************
+ * AppHint mnemonic data type helper tables
+******************************************************************************/
+struct apphint_lookup {
+	const char *name;
+	int value;
+};
+
+static const struct apphint_lookup fwt_logtype_tbl[] = {
+	{ "trace", 0},
+	{ "none", 0}
+#if defined(SUPPORT_TBI_INTERFACE)
+	, { "tbi", 1}
+#endif
+};
+
+static const struct apphint_lookup fwt_loggroup_tbl[] = {
+	RGXFWIF_LOG_GROUP_NAME_VALUE_MAP
+};
+
+static const struct apphint_lookup htb_loggroup_tbl[] = {
+#define X(a, b) { #b, HTB_LOG_GROUP_FLAG(a) },
+	HTB_LOG_SFGROUPLIST
+#undef X
+};
+
+static const struct apphint_lookup htb_opmode_tbl[] = {
+	{ "droplatest", HTB_OPMODE_DROPLATEST},
+	{ "dropoldest", HTB_OPMODE_DROPOLDEST},
+	{ "block", HTB_OPMODE_BLOCK}
+};
+
+__maybe_unused
+static const struct apphint_lookup htb_logmode_tbl[] = {
+	{ "all", HTB_LOGMODE_ALLPID},
+	{ "restricted", HTB_LOGMODE_RESTRICTEDPID}
+};
+
+__maybe_unused
+static const struct apphint_lookup timecorr_clk_tbl[] = {
+	{ "mono", 0 },
+	{ "mono_raw", 1 },
+	{ "sched", 2 }
+};
+
+/*
+*******************************************************************************
+ Data types
+******************************************************************************/
+union apphint_value {
+	IMG_UINT64 UINT64;
+	IMG_UINT32 UINT32;
+	IMG_BOOL BOOL;
+	IMG_CHAR *STRING;
+};
+
+union apphint_query_action {
+	PVRSRV_ERROR (*UINT64)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT64 *value);
+	PVRSRV_ERROR (*UINT32)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT32 *value);
+	PVRSRV_ERROR (*BOOL)(const PVRSRV_DEVICE_NODE *device,
+	                     const void *private_data, IMG_BOOL *value);
+	PVRSRV_ERROR (*STRING)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_CHAR **value);
+};
+
+union apphint_set_action {
+	PVRSRV_ERROR (*UINT64)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT64 value);
+	PVRSRV_ERROR (*UINT32)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT32 value);
+	PVRSRV_ERROR (*BOOL)(const PVRSRV_DEVICE_NODE *device,
+	                     const void *private_data, IMG_BOOL value);
+	PVRSRV_ERROR (*STRING)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_CHAR *value);
+};
+
+struct apphint_action {
+	union apphint_query_action query; /*!< Query callbacks. */
+	union apphint_set_action set;     /*!< Set callbacks. */
+	const PVRSRV_DEVICE_NODE *device; /*!< Pointer to the device node.*/
+	const void *private_data;         /*!< Opaque data passed to `query` and
+	                                       `set` callbacks. */
+	union apphint_value stored;       /*!< Value of the AppHint. */
+	bool free;                        /*!< Flag indicating that memory has been
+	                                       allocated for this AppHint and it
+	                                       needs to be freed on deinit. */
+	bool initialised;                 /*!< Flag indicating if the AppHint has
+	                                       been already initialised. */
+};
+
+struct apphint_param {
+	IMG_UINT32 id;
+	APPHINT_DATA_TYPE data_type;
+	const void *data_type_helper;
+	IMG_UINT32 helper_size;
+};
+
+struct apphint_init_data {
+	IMG_UINT32 id;          /* index into AppHint Table */
+	APPHINT_CLASS class;
+	const IMG_CHAR *name;
+	union apphint_value default_value;
+	APPHINT_RT_CLASS guest; /* ALWAYS => present on GUEST,
+	                           NEVER => not present on GUEST */
+};
+
+struct apphint_init_data_mapping {
+	IMG_UINT32 device_apphint_id;
+	IMG_UINT32 modparam_apphint_id;
+};
+
+struct apphint_class_state {
+	APPHINT_CLASS class;
+	IMG_BOOL enabled;
+};
+
+struct apphint_work {
+	struct work_struct work;
+	union apphint_value new_value;
+	struct apphint_action *action;
+};
+
+/*
+*******************************************************************************
+ Initialization / configuration table data
+******************************************************************************/
+#define UINT32Bitfield UINT32
+#define UINT32List UINT32
+
+static const struct apphint_init_data init_data_buildvar[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+};
+
+static const struct apphint_init_data init_data_modparam[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+};
+
+static const struct apphint_init_data init_data_debuginfo[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+};
+
+static const struct apphint_init_data init_data_debuginfo_device[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+};
+
+static const struct apphint_init_data_mapping init_data_debuginfo_device_to_modparams[] = {
+#define X(a, b) \
+	{APPHINT_ID_ ## a, APPHINT_ID_ ## b},
+	APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT_COMMON
+	APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT
+#undef X
+};
+
+#undef UINT32Bitfield
+#undef UINT32List
+
+__maybe_unused static const char NO_PARAM_TABLE[] = {};
+
+static const struct apphint_param param_lookup[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_DATA_TYPE_ ## b, e, ARRAY_SIZE(e) },
+	APPHINT_LIST_ALL
+#undef X
+};
+
+static const struct apphint_class_state class_state[] = {
+#define X(a) {APPHINT_CLASS_ ## a, APPHINT_ENABLED_CLASS_ ## a},
+	APPHINT_CLASS_LIST
+#undef X
+};
+
+/*
+*******************************************************************************
+ Global state
+******************************************************************************/
+/* If the union apphint_value becomes such that it is not possible to read
+ * and write atomically, a mutex may be desirable to prevent a read returning
+ * a partially written state.
+ * This would require a statically initialized mutex outside of the
+ * struct apphint_state to prevent use of an uninitialized mutex when
+ * module_params are provided on the command line.
+ *     static DEFINE_MUTEX(apphint_mutex);
+ */
+static struct apphint_state
+{
+	struct workqueue_struct *workqueue;
+	DI_GROUP *debuginfo_device_rootdir[PVRSRV_MAX_DEVICES];
+	DI_ENTRY *debuginfo_device_entry[PVRSRV_MAX_DEVICES][APPHINT_DEBUGINFO_DEVICE_ID_MAX];
+	DI_GROUP *debuginfo_rootdir;
+	DI_ENTRY *debuginfo_entry[APPHINT_DEBUGINFO_ID_MAX];
+	DI_GROUP *buildvar_rootdir;
+	DI_ENTRY *buildvar_entry[APPHINT_BUILDVAR_ID_MAX];
+
+	unsigned int num_devices;
+	PVRSRV_DEVICE_NODE *devices[PVRSRV_MAX_DEVICES];
+	unsigned int initialized;
+
+	/* Array contains value space for 1 copy of all apphint values defined
+	 * (for device 1) and N copies of device specific apphint values for
+	 * multi-device platforms.
+	 */
+	struct apphint_action val[APPHINT_ID_MAX + ((PVRSRV_MAX_DEVICES-1)*APPHINT_DEBUGINFO_DEVICE_ID_MAX)];
+
+} apphint = {
+/* statically initialise default values to ensure that any module_params
+ * provided on the command line are not overwritten by defaults.
+ */
+	.val = {
+#define UINT32Bitfield UINT32
+#define UINT32List UINT32
+#define X(a, b, c, d, e, f) \
+	{ {NULL}, {NULL}, NULL, NULL, {.b=d}, NULL },
+	APPHINT_LIST_ALL
+#undef X
+#undef UINT32Bitfield
+#undef UINT32List
+	},
+	.initialized = 0,
+	.num_devices = 0
+};
+
+#define APPHINT_DEBUGINFO_DEVICE_ID_OFFSET (APPHINT_ID_MAX-APPHINT_DEBUGINFO_DEVICE_ID_MAX)
+
+static inline void
+get_apphint_id_from_action_addr(const struct apphint_action * const addr,
+                                APPHINT_ID * const id)
+{
+	*id = (APPHINT_ID)(addr - apphint.val);
+	if (*id >= APPHINT_ID_MAX) {
+		*id -= APPHINT_DEBUGINFO_DEVICE_ID_OFFSET;
+		*id %= APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+		*id += APPHINT_DEBUGINFO_DEVICE_ID_OFFSET;
+	}
+}
+
+static inline void
+get_value_offset_from_device(const PVRSRV_DEVICE_NODE * const device,
+                             int * const offset,
+                             APPHINT_ID id)
+{
+	int i;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	/* No device offset if not a device specific apphint */
+	if (APPHINT_OF_DRIVER_NO_DEVICE == device) {
+		*offset = 0;
+		return;
+	}
+
+	/* Check that the specified ID is a device-specific one. If not we
+	 * set the offset to 0 for the global MODPARAM / BUILDVAR etc. AppHint
+	 */
+	for (i = 0; i < ARRAY_SIZE(init_data_debuginfo_device); i++)
+	{
+		const struct apphint_init_data *device_init = &init_data_debuginfo_device[i];
+
+		if ((IMG_UINT32)id == device_init->id) {
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!bFound) {
+		*offset = 0;
+		return;
+	}
+
+	for (i = 0; device && i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i] == device)
+			break;
+	}
+	if (PVRSRV_MAX_DEVICES == i) {
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unregistered device", __func__));
+		i = 0;
+	}
+	*offset = i * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+}
+
+/**
+ * apphint_action_worker - perform an action after an AppHint update has been
+ *                    requested by a UM process
+ *                    And update the record of the current active value
+ */
+static void apphint_action_worker(struct work_struct *work)
+{
+	struct apphint_work *work_pkt = container_of(work,
+	                                             struct apphint_work,
+	                                             work);
+	struct apphint_action *a = work_pkt->action;
+	union apphint_value value = work_pkt->new_value;
+	APPHINT_ID id;
+	PVRSRV_ERROR result = PVRSRV_OK;
+
+	get_apphint_id_from_action_addr(a, &id);
+
+	if (a->set.UINT64) {
+		switch (param_lookup[id].data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			result = a->set.UINT64(a->device,
+			                       a->private_data,
+			                       value.UINT64);
+			break;
+
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+		case APPHINT_DATA_TYPE_UINT32List:
+			result = a->set.UINT32(a->device,
+			                       a->private_data,
+			                       value.UINT32);
+			break;
+
+		case APPHINT_DATA_TYPE_BOOL:
+			result = a->set.BOOL(a->device,
+			                     a->private_data,
+			                     value.BOOL);
+			break;
+
+		case APPHINT_DATA_TYPE_STRING:
+			result = a->set.STRING(a->device,
+								   a->private_data,
+								   value.STRING);
+			kfree(value.STRING);
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: unrecognised data type (%d), index (%d)",
+			         __func__, param_lookup[id].data_type, id));
+		}
+
+		if (PVRSRV_OK != result) {
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: failed (%s)",
+			         __func__, PVRSRVGetErrorString(result)));
+		}
+	} else {
+		if (a->free) {
+			kfree(a->stored.STRING);
+		}
+		a->stored = value;
+		if (param_lookup[id].data_type == APPHINT_DATA_TYPE_STRING) {
+			a->free = true;
+		}
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: AppHint value updated before handler is registered, ID(%d)",
+		         __func__, id));
+	}
+	kfree((void *)work_pkt);
+}
+
+static void apphint_action(union apphint_value new_value,
+                           struct apphint_action *action)
+{
+	struct apphint_work *work_pkt = kmalloc(sizeof(*work_pkt), GFP_KERNEL);
+
+	/* queue apphint update on a serialized workqueue to avoid races */
+	if (work_pkt) {
+		work_pkt->new_value = new_value;
+		work_pkt->action = action;
+		INIT_WORK(&work_pkt->work, apphint_action_worker);
+		if (0 == queue_work(apphint.workqueue, &work_pkt->work)) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to queue apphint change request",
+				__func__));
+			goto err_exit;
+		}
+	} else {
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: failed to alloc memory for apphint change request",
+			__func__));
+		goto err_exit;
+	}
+	return;
+err_exit:
+	kfree(new_value.STRING);
+}
+
+/**
+ * apphint_read - read the different AppHint data types
+ * return -errno or the buffer size
+ */
+static int apphint_read(char *buffer, size_t count, APPHINT_ID ue,
+			 union apphint_value *value)
+{
+	APPHINT_DATA_TYPE data_type = param_lookup[ue].data_type;
+	int result = 0;
+
+	switch (data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		if (kstrtou64(buffer, 0, &value->UINT64) < 0) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid UINT64 input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32:
+		if (kstrtou32(buffer, 0, &value->UINT32) < 0) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid UINT32 input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_BOOL:
+		switch (buffer[0]) {
+		case '0':
+		case 'n':
+		case 'N':
+		case 'f':
+		case 'F':
+			value->BOOL = IMG_FALSE;
+			break;
+		case '1':
+		case 'y':
+		case 'Y':
+		case 't':
+		case 'T':
+			value->BOOL = IMG_TRUE;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid BOOL input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32List:
+	{
+		int i;
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *)
+			param_lookup[ue].data_type_helper;
+		int size = param_lookup[ue].helper_size;
+		/* buffer may include '\n', remove it */
+		char *arg = strsep(&buffer, "\n");
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < size; i++) {
+			if (strcasecmp(lookup[i].name, arg) == 0) {
+				value->UINT32 = lookup[i].value;
+				break;
+			}
+		}
+		if (i == size) {
+			if (OSStringLength(arg) == 0) {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: No value set for AppHint",
+					__func__));
+			} else {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unrecognised AppHint value (%s)",
+					__func__, arg));
+			}
+			result = -EINVAL;
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	{
+		int i;
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *)
+			param_lookup[ue].data_type_helper;
+		int size = param_lookup[ue].helper_size;
+		/* buffer may include '\n', remove it */
+		char *string = strsep(&buffer, "\n");
+		char *token = strsep(&string, ",");
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		value->UINT32 = 0;
+		/* empty string is valid to clear the bitfield */
+		while (token && *token) {
+			for (i = 0; i < size; i++) {
+				if (strcasecmp(lookup[i].name, token) == 0) {
+					value->UINT32 |= lookup[i].value;
+					break;
+				}
+			}
+			if (i == size) {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unrecognised AppHint value (%s)",
+					__func__, token));
+				result = -EINVAL;
+				goto err_exit;
+			}
+			token = strsep(&string, ",");
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_STRING:
+	{
+		/* buffer may include '\n', remove it */
+		char *string = strsep(&buffer, "\n");
+		size_t len = OSStringLength(string);
+
+		if (!len) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		++len;
+
+		value->STRING = kmalloc(len , GFP_KERNEL);
+		if (!value->STRING) {
+			result = -ENOMEM;
+			goto err_exit;
+		}
+
+		OSStringSafeCopy(value->STRING, string, len);
+		break;
+	}
+	default:
+		result = -EINVAL;
+		goto err_exit;
+	}
+
+err_exit:
+	return (result < 0) ? result : count;
+}
+
+static PVRSRV_ERROR get_apphint_value_from_action(const struct apphint_action * const action,
+                                                  union apphint_value * const value,
+                                                  const PVRSRV_DEVICE_NODE * const psDevNode)
+{
+	APPHINT_ID id;
+	APPHINT_DATA_TYPE data_type;
+	PVRSRV_ERROR result = PVRSRV_OK;
+	const PVRSRV_DEVICE_NODE *psDevice;
+
+	get_apphint_id_from_action_addr(action, &id);
+	data_type = param_lookup[id].data_type;
+
+	/* If we've got an entry that is APPHINT_OF_DRIVER_NO_DEVICE we should use
+	 * the higher-level psDevNode value instead. This is the device-node that is
+	 * associated with the original debug_dump request.
+	 * Note: if we're called with psDevNode == APPHINT_OF_DRIVER_NO_DEVICE
+	 * we attempt to use the first registered apphint.devices[0] (if any
+	 * devices have been presented). If we have no devices hooked into the
+	 * apphint mechanism we just return the default value for the AppHint.
+	 */
+	if (psDevNode == APPHINT_OF_DRIVER_NO_DEVICE) {
+		if (action->device == APPHINT_OF_DRIVER_NO_DEVICE) {
+			if (apphint.num_devices > 0) {
+				psDevice = apphint.devices[0];
+			} else {
+				PVR_DPF((PVR_DBG_ERROR,
+				        "Uninitialised AppHint device for AppHint index (%d)",
+				        id));
+				return PVRSRV_ERROR_RETRY;
+			}
+		} else {
+			psDevice = action->device;
+		}
+	} else {
+		if (action->device == APPHINT_OF_DRIVER_NO_DEVICE) {
+			if (psDevNode != action->device) {
+				return PVRSRV_ERROR_INVALID_DEVICE;
+			}
+			psDevice = psDevNode;
+		} else {
+			psDevice = action->device;
+		}
+	}
+
+	if (action->query.UINT64) {
+		switch (data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			result = action->query.UINT64(psDevice,
+			                              action->private_data,
+			                              &value->UINT64);
+			break;
+
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+		case APPHINT_DATA_TYPE_UINT32List:
+			result = action->query.UINT32(psDevice,
+			                              action->private_data,
+			                              &value->UINT32);
+			break;
+
+		case APPHINT_DATA_TYPE_BOOL:
+			result = action->query.BOOL(psDevice,
+			                            action->private_data,
+			                            &value->BOOL);
+			break;
+
+		case APPHINT_DATA_TYPE_STRING:
+			result = action->query.STRING(psDevice,
+			                              action->private_data,
+			                              &value->STRING);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: unrecognised data type (%d), index (%d)",
+			         __func__, data_type, id));
+		}
+	} else {
+		*value = action->stored;
+	}
+
+	if (PVRSRV_OK != result) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed (%d), index (%d)", __func__, result, id));
+	}
+
+	return result;
+}
+
+/*
+ * apphint_write - write the current AppHint data to a buffer
+ *
+ * Returns length written or -errno
+ */
+static int apphint_write(char *buffer, const size_t size,
+                         const struct apphint_action *a)
+{
+	const struct apphint_param *hint;
+	int result = 0;
+	APPHINT_ID id;
+	union apphint_value value;
+
+	get_apphint_id_from_action_addr(a, &id);
+	hint = &param_lookup[id];
+
+	result = get_apphint_value_from_action(a, &value, a->device);
+
+	switch (hint->data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		result += snprintf(buffer + result, size - result,
+				"0x%016llx",
+				value.UINT64);
+		break;
+	case APPHINT_DATA_TYPE_UINT32:
+		result += snprintf(buffer + result, size - result,
+				"0x%08x",
+				value.UINT32);
+		break;
+	case APPHINT_DATA_TYPE_BOOL:
+		result += snprintf(buffer + result, size - result,
+			"%s",
+			value.BOOL ? "Y" : "N");
+		break;
+	case APPHINT_DATA_TYPE_STRING:
+		if (value.STRING) {
+			result += snprintf(buffer + result, size - result,
+				"%s",
+				*value.STRING ? value.STRING : "(none)");
+		} else {
+			result += snprintf(buffer + result, size - result,
+			"(none)");
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32List:
+	{
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *) hint->data_type_helper;
+		IMG_UINT32 i;
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < hint->helper_size; i++) {
+			if (lookup[i].value == value.UINT32) {
+				result += snprintf(buffer + result,
+						size - result,
+						"%s",
+						lookup[i].name);
+				break;
+			}
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	{
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *) hint->data_type_helper;
+		IMG_UINT32 i;
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < hint->helper_size; i++) {
+			if (lookup[i].value & value.UINT32) {
+				result += snprintf(buffer + result,
+						size - result,
+						"%s,",
+						lookup[i].name);
+			}
+		}
+		if (result) {
+			/* remove any trailing ',' */
+			--result;
+			*(buffer + result) = '\0';
+		} else {
+			result += snprintf(buffer + result,
+					size - result, "none");
+		}
+		break;
+	}
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: unrecognised data type (%d), index (%d)",
+			 __func__, hint->data_type, id));
+		result = -EINVAL;
+	}
+
+err_exit:
+	return result;
+}
+
+/*
+*******************************************************************************
+ Module parameters initialization - different from debuginfo
+******************************************************************************/
+/*
+ * apphint_kparam_set - Handle an update of a module parameter
+ *
+ * Returns 0, or -errno.  arg is in kp->arg.
+ */
+static int apphint_kparam_set(const char *val, const struct kernel_param *kp)
+{
+	char val_copy[APPHINT_BUFFER_SIZE];
+	APPHINT_ID id;
+	union apphint_value value;
+	ssize_t result = OSStringSafeCopy(val_copy, val, APPHINT_BUFFER_SIZE);
+
+	/* Document the assumption - we can safely store the result of
+	 * apphint_read() in a ssize_t variable, no need for a separate one */
+#ifndef SSIZE_MAX
+#define SSIZE_MAX ((~(size_t)0) >> 1)
+#endif
+	BUILD_BUG_ON(INT_MAX > SSIZE_MAX);
+
+	if (result < 0) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: String too long", __func__));
+		return (int)result;
+	}
+
+	get_apphint_id_from_action_addr(kp->arg, &id);
+	result = apphint_read(val_copy, result, id, &value);
+	if (result < 0) {
+		return (int)result;
+	}
+
+	((struct apphint_action *)kp->arg)->stored = value;
+	((struct apphint_action *)kp->arg)->initialised = true;
+	if (param_lookup[id].data_type == APPHINT_DATA_TYPE_STRING) {
+		((struct apphint_action *)kp->arg)->free = true;
+	}
+
+	return 0;
+}
+
+/*
+ * apphint_kparam_get - handle a read of a module parameter
+ *
+ * Returns length written or -errno.  Buffer is 4k (ie. be short!)
+ */
+static int apphint_kparam_get(char *buffer, const struct kernel_param *kp)
+{
+	return apphint_write(buffer, PAGE_SIZE, kp->arg);
+}
+
+__maybe_unused
+static const struct kernel_param_ops apphint_kparam_fops = {
+	.set = apphint_kparam_set,
+	.get = apphint_kparam_get,
+};
+
+/*
+ * call module_param_cb() for all AppHints listed in APPHINT_LIST_MODPARAM_COMMON + APPHINT_LIST_MODPARAM
+ * apphint_modparam_class_ ## resolves to apphint_modparam_enable() except for
+ * AppHint classes that have been disabled.
+ */
+
+#define apphint_modparam_enable(name, number, perm) \
+	module_param_cb(name, &apphint_kparam_fops, &apphint.val[number], perm);
+
+#define X(a, b, c, d, e, f) \
+	apphint_modparam_class_ ##c(a, APPHINT_ID_ ## a, 0444)
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+
+/*
+*******************************************************************************
+ Debug Info get (seq file) operations - supporting functions
+******************************************************************************/
+static void *apphint_di_start(OSDI_IMPL_ENTRY *s, IMG_UINT64 *pos)
+{
+	if (*pos == 0) {
+		/* We want only one entry in the sequence, one call to show() */
+		return (void *) 1;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(s);
+
+	return NULL;
+}
+
+static void apphint_di_stop(OSDI_IMPL_ENTRY *s, void *v)
+{
+	PVR_UNREFERENCED_PARAMETER(s);
+	PVR_UNREFERENCED_PARAMETER(v);
+}
+
+static void *apphint_di_next(OSDI_IMPL_ENTRY *s, void *v, IMG_UINT64 *pos)
+{
+	PVR_UNREFERENCED_PARAMETER(s);
+	PVR_UNREFERENCED_PARAMETER(v);
+	(*pos)++;
+	return NULL;
+}
+
+static int apphint_di_show(OSDI_IMPL_ENTRY *s, void *v)
+{
+	IMG_CHAR km_buffer[APPHINT_BUFFER_SIZE];
+	int result;
+	void *private = DIGetPrivData(s);
+
+	PVR_UNREFERENCED_PARAMETER(v);
+
+	result = apphint_write(km_buffer, APPHINT_BUFFER_SIZE, private);
+	if (result < 0) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: failure", __func__));
+	} else {
+		/* debuginfo requires a trailing \n, module_params don't */
+		result += snprintf(km_buffer + result,
+				APPHINT_BUFFER_SIZE - result,
+				"\n");
+		DIPuts(s, km_buffer);
+	}
+
+	/* have to return 0 to see output */
+	return (result < 0) ? result : 0;
+}
+
+/*
+*******************************************************************************
+ Debug Info supporting functions
+******************************************************************************/
+
+/*
+ * apphint_set - Handle a DI value update
+ */
+static IMG_INT64 apphint_set(const IMG_CHAR *buffer, IMG_UINT64 count,
+                             IMG_UINT64 *ppos, void *data)
+{
+	APPHINT_ID id;
+	union apphint_value value;
+	struct apphint_action *action = data;
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	int result = 0;
+
+	if (ppos == NULL)
+		return -EIO;
+
+	if (count >= APPHINT_BUFFER_SIZE) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: String too long (%" IMG_INT64_FMTSPECd ")",
+			__func__, count));
+		result = -EINVAL;
+		goto err_exit;
+	}
+
+	/* apphint_read() modifies the buffer so we need to copy it */
+	memcpy(km_buffer, buffer, count);
+	/* count is larger than real buffer by 1 because DI framework appends
+	 * a '\0' character at the end, but here we're ignoring this */
+	count -= 1;
+	km_buffer[count] = '\0';
+
+	get_apphint_id_from_action_addr(action, &id);
+	result = apphint_read(km_buffer, count, id, &value);
+	if (result >= 0)
+		apphint_action(value, action);
+
+	*ppos += count;
+err_exit:
+	return result;
+}
+
+/*
+ * apphint_debuginfo_init - Create the specified debuginfo entries
+ */
+static int apphint_debuginfo_init(const char *sub_dir,
+		unsigned int device_num,
+		unsigned int init_data_size,
+		const struct apphint_init_data *init_data,
+		DI_GROUP *parentdir,
+		DI_GROUP **rootdir,
+		DI_ENTRY *entry[])
+{
+	PVRSRV_ERROR result;
+	unsigned int i;
+	unsigned int device_value_offset = device_num * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	const DI_ITERATOR_CB iterator = {
+		.pfnStart = apphint_di_start, .pfnStop = apphint_di_stop,
+		.pfnNext  = apphint_di_next,  .pfnShow = apphint_di_show,
+		.pfnWrite = apphint_set,      .ui32WriteLenMax = APPHINT_BUFFER_SIZE
+	};
+
+	/* Determine if we're booted as a GUEST VZ OS */
+	IMG_BOOL bIsGUEST = PVRSRV_VZ_MODE_IS(GUEST, DEVID, device_num);
+
+	if (*rootdir) {
+		PVR_DPF((PVR_DBG_WARNING,
+			"AppHint DebugFS already created, skipping"));
+		result = -EEXIST;
+		goto err_exit;
+	}
+
+	result = DICreateGroup(sub_dir, parentdir, rootdir);
+	if (result != PVRSRV_OK) {
+		PVR_DPF((PVR_DBG_WARNING,
+			"Failed to create \"%s\" DebugFS directory.", sub_dir));
+		goto err_exit;
+	}
+
+	for (i = 0; i < init_data_size; i++) {
+		if (!class_state[init_data[i].class].enabled)
+			continue;
+
+		/* Check to see if this AppHint should appear in a GUEST OS.
+		 * This will have a value in the init_data[i].guest field of ALWAYS
+		 * and if we don't have this set (and we're in GUEST mode) we must
+		 * not present this AppHint to the OS.
+		 */
+		if (bIsGUEST && (init_data[i].guest != APPHINT_RT_CLASS_ALWAYS))
+			continue;
+
+		result = DICreateEntry(init_data[i].name,
+				*rootdir,
+				&iterator,
+				(void *) &apphint.val[init_data[i].id + device_value_offset],
+				DI_ENTRY_TYPE_GENERIC,
+				&entry[i]);
+		if (result != PVRSRV_OK) {
+			PVR_DPF((PVR_DBG_WARNING,
+				"Failed to create \"%s/%s\" DebugFS entry.",
+				sub_dir, init_data[i].name));
+		}
+	}
+
+	return 0;
+
+err_exit:
+	return result;
+}
+
+/*
+ * apphint_debuginfo_deinit- destroy the debuginfo entries
+ */
+static void apphint_debuginfo_deinit(unsigned int num_entries,
+		DI_GROUP **rootdir,
+		DI_ENTRY *entry[])
+{
+	unsigned int i;
+
+	for (i = 0; i < num_entries; i++) {
+		if (entry[i]) {
+			DIDestroyEntry(entry[i]);
+		}
+	}
+
+	if (*rootdir) {
+		DIDestroyGroup(*rootdir);
+		*rootdir = NULL;
+	}
+}
+
+/*
+*******************************************************************************
+ AppHint status dump implementation
+******************************************************************************/
+#if defined(PDUMP)
+static void apphint_pdump_values(void *pvDeviceNode,
+                                 const IMG_CHAR *format, ...)
+{
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS;
+	va_list ap;
+
+	va_start(ap, format);
+	(void)vsnprintf(km_buffer, APPHINT_BUFFER_SIZE, format, ap);
+	va_end(ap);
+
+	/* ui32CommentSize set to 0 here as function does not make use of the value. */
+	PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE*)pvDeviceNode, 0, km_buffer, ui32Flags);
+}
+#endif
+
+static IMG_BOOL is_apphint_value_equal(const APPHINT_DATA_TYPE data_type,
+									const union apphint_value * const left,
+									const union apphint_value * const right)
+{
+		switch (data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			return left->UINT64 == right->UINT64;
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32List:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+			return left->UINT32 == right->UINT32;
+		case APPHINT_DATA_TYPE_BOOL:
+			return left->BOOL == right->BOOL;
+		case APPHINT_DATA_TYPE_STRING:
+			return (OSStringNCompare(left->STRING, right->STRING, OSStringLength(right->STRING) + 1) == 0 ? IMG_TRUE : IMG_FALSE);
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "%s: unhandled data type (%d)", __func__, data_type));
+			return IMG_FALSE;
+		}
+}
+
+static void apphint_dump_values(const char *group_name,
+			int device_num,
+			const struct apphint_init_data *group_data,
+			int group_size,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile,
+			bool list_all,
+			PVRSRV_DEVICE_NODE *psDevNode)
+{
+	int i, result;
+	int device_value_offset = device_num * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	char count = 0;
+
+	PVR_DUMPDEBUG_LOG("  %s", group_name);
+	for (i = 0; i < group_size; i++)
+	{
+		IMG_UINT32 id = group_data[i].id;
+		APPHINT_DATA_TYPE data_type = param_lookup[id].data_type;
+		const struct apphint_action *action = &apphint.val[id + device_value_offset];
+		union apphint_value value;
+
+		result = get_apphint_value_from_action(action, &value, psDevNode);
+
+		if (PVRSRV_OK != result) {
+			continue;
+		}
+
+		/* List only apphints with non-default values */
+		if (!list_all &&
+			is_apphint_value_equal(data_type, &value, &group_data[i].default_value)) {
+			continue;
+		}
+
+		result = apphint_write(km_buffer, APPHINT_BUFFER_SIZE, action);
+		count++;
+
+		if (result <= 0) {
+			PVR_DUMPDEBUG_LOG("    %s: <Error>",
+				group_data[i].name);
+		} else {
+			PVR_DUMPDEBUG_LOG("    %s: %s",
+				group_data[i].name, km_buffer);
+		}
+	}
+
+	if (count == 0) {
+		PVR_DUMPDEBUG_LOG("    none");
+	}
+}
+
+/*
+ * Callback for debug dump
+ */
+static void apphint_dump_state(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+			IMG_UINT32 ui32VerbLevel,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	int i, result;
+	char description_buffer[50];
+	PVRSRV_DEVICE_NODE *device = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH)) {
+		PVR_DUMPDEBUG_LOG("------[ AppHint Settings ]------");
+
+		apphint_dump_values("Build Vars", 0,
+			init_data_buildvar, ARRAY_SIZE(init_data_buildvar),
+			pfnDumpDebugPrintf, pvDumpDebugFile, true, device);
+
+		apphint_dump_values("Module Params", 0,
+			init_data_modparam, ARRAY_SIZE(init_data_modparam),
+			pfnDumpDebugPrintf, pvDumpDebugFile, false, device);
+
+		apphint_dump_values("Debug Info Params", 0,
+			init_data_debuginfo, ARRAY_SIZE(init_data_debuginfo),
+			pfnDumpDebugPrintf, pvDumpDebugFile, false, device);
+
+		for (i = 0; i < PVRSRV_MAX_DEVICES; i++) {
+			if (!apphint.devices[i]
+			    || (device && device != apphint.devices[i]))
+				continue;
+
+			result = snprintf(description_buffer,
+					  sizeof(description_buffer),
+					  "Debug Info Params Device ID: %d",
+					  i);
+			if (0 > result)
+				continue;
+
+			apphint_dump_values(description_buffer, i,
+					    init_data_debuginfo_device,
+					    ARRAY_SIZE(init_data_debuginfo_device),
+					    pfnDumpDebugPrintf,
+					    pvDumpDebugFile,
+						false, device);
+		}
+	}
+}
+
+/*
+*******************************************************************************
+ Public interface
+******************************************************************************/
+int pvr_apphint_init(void)
+{
+	int result, i;
+
+	if (apphint.initialized) {
+		result = -EEXIST;
+		goto err_out;
+	}
+
+	for (i = 0; i < PVRSRV_MAX_DEVICES; i++)
+		apphint.devices[i] = NULL;
+
+	/* create workqueue with strict execution ordering to ensure no
+	 * race conditions when setting/updating apphints from different
+	 * contexts
+	 */
+	apphint.workqueue = alloc_workqueue("apphint_workqueue",
+	                                    WQ_UNBOUND | WQ_FREEZABLE, 1);
+	if (!apphint.workqueue) {
+		result = -ENOMEM;
+		goto err_out;
+	}
+
+	result = apphint_debuginfo_init("apphint", 0,
+		ARRAY_SIZE(init_data_debuginfo), init_data_debuginfo,
+		NULL,
+		&apphint.debuginfo_rootdir, apphint.debuginfo_entry);
+	if (0 != result)
+		goto err_out;
+
+	result = apphint_debuginfo_init("buildvar", 0,
+		ARRAY_SIZE(init_data_buildvar), init_data_buildvar,
+		NULL,
+		&apphint.buildvar_rootdir, apphint.buildvar_entry);
+
+	apphint.initialized = 1;
+
+err_out:
+	return result;
+}
+
+int pvr_apphint_device_register(PVRSRV_DEVICE_NODE *device)
+{
+	int result, i;
+	unsigned int device_value_offset;
+
+	if (!apphint.initialized) {
+		result = -EAGAIN;
+		goto err_out;
+	}
+
+	if (apphint.num_devices+1 > PVRSRV_MAX_DEVICES) {
+		result = -EMFILE;
+		goto err_out;
+	}
+
+	/* Set the default values for the new device */
+	device_value_offset = apphint.num_devices * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	for (i = 0; i < APPHINT_DEBUGINFO_DEVICE_ID_MAX; i++) {
+		apphint.val[init_data_debuginfo_device[i].id + device_value_offset].stored
+			= init_data_debuginfo_device[i].default_value;
+	}
+
+	/* Set value of an apphint if mapping to module param exists for it
+	 * and this module parameter has been initialised */
+	for (i = 0; i < ARRAY_SIZE(init_data_debuginfo_device_to_modparams); i++) {
+		const struct apphint_init_data_mapping *mapping =
+			&init_data_debuginfo_device_to_modparams[i];
+		const struct apphint_action *modparam_action =
+			&apphint.val[mapping->modparam_apphint_id];
+		struct apphint_action *device_action =
+			&apphint.val[mapping->device_apphint_id + device_value_offset];
+
+		/* Set only if the module parameter was explicitly set during the module
+		 * load. */
+		if (modparam_action->initialised) {
+			device_action->stored = modparam_action->stored;
+		}
+	}
+
+	result = apphint_debuginfo_init("apphint", device->sDevId.ui32InternalID,
+	                              ARRAY_SIZE(init_data_debuginfo_device),
+	                              init_data_debuginfo_device,
+	                              device->sDebugInfo.psGroup,
+	                              &apphint.debuginfo_device_rootdir[device->sDevId.ui32InternalID],
+	                              apphint.debuginfo_device_entry[device->sDevId.ui32InternalID]);
+	if (0 != result)
+		goto err_out;
+
+	apphint.devices[device->sDevId.ui32InternalID] = device;
+	apphint.num_devices++;
+
+	(void)SOPvrDbgRequestNotifyRegister(
+			&device->hAppHintDbgReqNotify,
+			device,
+			apphint_dump_state,
+			DEBUG_REQUEST_APPHINT,
+			device);
+
+err_out:
+	return result;
+}
+
+void pvr_apphint_device_unregister(PVRSRV_DEVICE_NODE *device)
+{
+	int i;
+
+	if (!apphint.initialized)
+		return;
+
+	/* find the device */
+	for (i = 0; i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i] == device)
+			break;
+	}
+
+	if (PVRSRV_MAX_DEVICES == i)
+		return;
+
+	if (device->hAppHintDbgReqNotify) {
+		(void)SOPvrDbgRequestNotifyUnregister(
+			device->hAppHintDbgReqNotify);
+		device->hAppHintDbgReqNotify = NULL;
+	}
+
+	apphint_debuginfo_deinit(APPHINT_DEBUGINFO_DEVICE_ID_MAX,
+	                       &apphint.debuginfo_device_rootdir[i],
+	                       apphint.debuginfo_device_entry[i]);
+
+	apphint.devices[i] = NULL;
+
+	WARN_ON(apphint.num_devices==0);
+	apphint.num_devices--;
+}
+
+void pvr_apphint_deinit(void)
+{
+	int i;
+
+	if (!apphint.initialized)
+		return;
+
+	/* remove any remaining device data */
+	for (i = 0; apphint.num_devices && i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i])
+			pvr_apphint_device_unregister(apphint.devices[i]);
+	}
+
+	/* free all alloc'd string apphints and set to NULL */
+	for (i = 0; i < ARRAY_SIZE(apphint.val); i++) {
+		if (apphint.val[i].free && apphint.val[i].stored.STRING) {
+			kfree(apphint.val[i].stored.STRING);
+			apphint.val[i].stored.STRING = NULL;
+			apphint.val[i].free = false;
+		}
+	}
+
+	apphint_debuginfo_deinit(APPHINT_DEBUGINFO_ID_MAX,
+			&apphint.debuginfo_rootdir, apphint.debuginfo_entry);
+	apphint_debuginfo_deinit(APPHINT_BUILDVAR_ID_MAX,
+			&apphint.buildvar_rootdir, apphint.buildvar_entry);
+
+	destroy_workqueue(apphint.workqueue);
+
+	apphint.initialized = 0;
+}
+
+void pvr_apphint_dump_state(PVRSRV_DEVICE_NODE *device)
+{
+#if defined(PDUMP)
+	/* NB. apphint_pdump_values() is the pfnDumpDebugPrintf
+	 * function used when PDUMP is defined.
+	 * apphintpdump_values() calls PDumpCommentKM(), which
+	 * requires the device but as it is only called as a
+	 * DUMPDEBUG_PRINTF_FUNC it is only passed pvDumpDebugFile
+	 * (which happens to be the 4th parameter in the call to
+	 * apphint_dump_state() below).
+	 * Hence, we also need to pass device in the 4th parameter.
+	 */
+	apphint_dump_state(device, DEBUG_REQUEST_VERBOSITY_HIGH,
+	                   apphint_pdump_values, device);
+#endif
+	apphint_dump_state(device, DEBUG_REQUEST_VERBOSITY_HIGH,
+	                   NULL, NULL);
+}
+
+
+int pvr_apphint_get_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.UINT64;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.UINT64;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.UINT32;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.UINT32;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.BOOL;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.BOOL;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX && apphint.val[ue].stored.STRING) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			if (OSStringSafeCopy(pBuffer, apphint.val[ue + device_offset].stored.STRING, size) >= 0) {
+				error = 0;
+			}
+		}
+		else
+		{
+			if (OSStringSafeCopy(pBuffer, apphint.val[ue].stored.STRING, size) >= 0) {
+				error = 0;
+			}
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_set_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_UINT64)) {
+
+		if (apphint.val[ue + device_offset].set.UINT64) {
+			apphint.val[ue + device_offset].set.UINT64(apphint.val[ue + device_offset].device,
+													 apphint.val[ue + device_offset].private_data,
+													 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.UINT64 = Val;
+		}
+		apphint.val[ue].device = device;
+		error = 0;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_UINT32)) {
+
+		if (apphint.val[ue + device_offset].set.UINT32) {
+			apphint.val[ue + device_offset].set.UINT32(apphint.val[ue + device_offset].device,
+													 apphint.val[ue + device_offset].private_data,
+													 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.UINT32 = Val;
+		}
+		apphint.val[ue].device = device;
+		error = 0;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_BOOL)) {
+
+		error = 0;
+		if (apphint.val[ue + device_offset].set.BOOL) {
+			apphint.val[ue + device_offset].set.BOOL(apphint.val[ue + device_offset].device,
+												 apphint.val[ue + device_offset].private_data,
+												 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.BOOL = Val;
+		}
+		apphint.val[ue].device = device;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		((param_lookup[ue].data_type == APPHINT_DATA_TYPE_STRING) &&
+		apphint.val[ue + device_offset].stored.STRING)) {
+
+		if (apphint.val[ue + device_offset].set.STRING) {
+			error = apphint.val[ue + device_offset].set.STRING(apphint.val[ue + device_offset].device,
+															 apphint.val[ue + device_offset].private_data,
+															 pBuffer);
+		} else if (OSStringSafeCopy(apphint.val[ue + device_offset].stored.STRING, pBuffer, size) >= 0) {
+			error = 0;
+		}
+		apphint.val[ue].device = device;
+	}
+
+	return error;
+}
+
+void pvr_apphint_register_handlers_uint64(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.UINT64 = query,
+		.set.UINT64 = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_uint32(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_UINT32:
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	case APPHINT_DATA_TYPE_UINT32List:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.UINT32 = query,
+		.set.UINT32 = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_bool(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_BOOL:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.BOOL = query,
+		.set.BOOL = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_string(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR **value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR *value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_STRING:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.STRING = query,
+		.set.STRING = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.h
new file mode 100644
index 000000000000..71e2ce94000a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File           km_apphint.h
+@Title          Apphint internal header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux kernel AppHint control
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef KM_APPHINT_H
+#define KM_APPHINT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "pvrsrv_apphint.h"
+#include "km_apphint_defs.h"
+#include "device.h"
+
+int pvr_apphint_init(void);
+void pvr_apphint_deinit(void);
+int pvr_apphint_device_register(PVRSRV_DEVICE_NODE *device);
+void pvr_apphint_device_unregister(PVRSRV_DEVICE_NODE *device);
+void pvr_apphint_dump_state(PVRSRV_DEVICE_NODE *device);
+
+int pvr_apphint_get_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 *pVal);
+int pvr_apphint_get_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 *pVal);
+int pvr_apphint_get_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL *pVal);
+int pvr_apphint_get_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size);
+
+int pvr_apphint_set_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 Val);
+int pvr_apphint_set_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 Val);
+int pvr_apphint_set_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL Val);
+int pvr_apphint_set_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size);
+
+void pvr_apphint_register_handlers_uint64(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void * private_data);
+void pvr_apphint_register_handlers_uint32(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+void pvr_apphint_register_handlers_bool(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+void pvr_apphint_register_handlers_string(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR **value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR *value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* KM_APPHINT_H */
+
+/******************************************************************************
+ End of file (km_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/linkage.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/linkage.h
new file mode 100644
index 000000000000..3f24dc68eff6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/linkage.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux specific Services code internal interfaces
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Interfaces between various parts of the Linux specific
+                Services code, that don't have any other obvious
+                header file to go into.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(LINKAGE_H)
+#define LINKAGE_H
+
+PVRSRV_ERROR PVROSFuncInit(void);
+void PVROSFuncDeInit(void);
+
+#endif /* !defined(LINKAGE_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.c
new file mode 100644
index 000000000000..8df0593db286
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.c
@@ -0,0 +1,787 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common Linux module setup
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+
+#if defined(CONFIG_DEBUG_FS)
+#include "pvr_debugfs.h"
+#endif /* defined(CONFIG_DEBUG_FS) */
+#if defined(CONFIG_PROC_FS)
+#include "pvr_procfs.h"
+#endif /* defined(CONFIG_PROC_FS) */
+#include "di_server.h"
+#include "private_data.h"
+#include "linkage.h"
+#include "power.h"
+#include "env_connection.h"
+#include "process_stats.h"
+#include "module_common.h"
+#include "pvrsrv.h"
+#include "srvcore.h"
+#if defined(SUPPORT_RGX)
+#include "rgxdevice.h"
+#endif
+#include "pvrsrv_error.h"
+#include "pvr_drv.h"
+#include "pvr_bridge_k.h"
+
+#include "pvr_fence.h"
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+#include "pvr_sync.h"
+#if !defined(USE_PVRSYNC_DEVNODE)
+#include "pvr_sync_ioctl_drm.h"
+#endif
+#endif
+
+#include "ospvr_gputrace.h"
+
+#include "km_apphint.h"
+#include "srvinit.h"
+
+#include "pvr_ion_stats.h"
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+#include "physmem_osmem_linux.h"
+#endif
+#endif
+
+#include "dkf_server.h"
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+/* Display class interface */
+#include "kerneldisplay.h"
+EXPORT_SYMBOL(DCRegisterDevice);
+EXPORT_SYMBOL(DCUnregisterDevice);
+EXPORT_SYMBOL(DCDisplayConfigurationRetired);
+EXPORT_SYMBOL(DCDisplayHasPendingCommand);
+EXPORT_SYMBOL(DCImportBufferAcquire);
+EXPORT_SYMBOL(DCImportBufferRelease);
+
+EXPORT_SYMBOL(PVRSRVGetDriverStatus);
+EXPORT_SYMBOL(PVRSRVSystemInstallDeviceLISR);
+EXPORT_SYMBOL(PVRSRVSystemUninstallDeviceLISR);
+
+#include "pvr_notifier.h"
+EXPORT_SYMBOL(PVRSRVCheckStatus);
+#endif /* defined(SUPPORT_DISPLAY_CLASS) */
+
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+/*
+ * Physmem interface.
+ * Required by LMA DC drivers, and some non-DC LMA display drivers.
+ */
+#include "physheap.h"
+EXPORT_SYMBOL(PhysHeapAcquireByID);
+EXPORT_SYMBOL(PhysHeapRelease);
+EXPORT_SYMBOL(PhysHeapGetType);
+EXPORT_SYMBOL(PhysHeapGetCpuPAddr);
+EXPORT_SYMBOL(PhysHeapGetSize);
+EXPORT_SYMBOL(PhysHeapCpuPAddrToDevPAddr);
+
+#include "pvr_debug.h"
+EXPORT_SYMBOL(PVRSRVGetErrorString);
+EXPORT_SYMBOL(PVRSRVGetDeviceInstance);
+#endif
+
+#if defined(SUPPORT_RGX)
+#include "rgxapi_km.h"
+EXPORT_SYMBOL(RGXHWPerfConnect);
+EXPORT_SYMBOL(RGXHWPerfDisconnect);
+EXPORT_SYMBOL(RGXHWPerfControl);
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+EXPORT_SYMBOL(RGXHWPerfConfigureCounters);
+#else
+EXPORT_SYMBOL(RGXHWPerfConfigMuxCounters);
+EXPORT_SYMBOL(RGXHWPerfConfigureAndEnableCustomCounters);
+#endif
+EXPORT_SYMBOL(RGXHWPerfDisableCounters);
+EXPORT_SYMBOL(RGXHWPerfAcquireEvents);
+EXPORT_SYMBOL(RGXHWPerfReleaseEvents);
+EXPORT_SYMBOL(RGXHWPerfConvertCRTimeStamp);
+#if defined(SUPPORT_KERNEL_HWPERF_TEST)
+EXPORT_SYMBOL(OSAddTimer);
+EXPORT_SYMBOL(OSEnableTimer);
+EXPORT_SYMBOL(OSDisableTimer);
+EXPORT_SYMBOL(OSRemoveTimer);
+#endif
+#endif
+
+static int PVRSRVDeviceSyncOpen(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                                struct drm_file *psDRMFile);
+
+CONNECTION_DATA *LinuxServicesConnectionFromFile(struct file *pFile)
+{
+	if (pFile)
+	{
+		struct drm_file *psDRMFile;
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+
+		psDRMFile = pFile->private_data;
+		PVR_LOG_RETURN_IF_FALSE(psDRMFile != NULL, "psDRMFile is NULL", NULL);
+
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+		PVR_LOG_RETURN_IF_FALSE(psConnectionPriv != NULL, "psConnectionPriv is NULL", NULL);
+
+		return (CONNECTION_DATA*)psConnectionPriv->pvConnectionData;
+	}
+
+	return NULL;
+}
+
+CONNECTION_DATA *LinuxSyncConnectionFromFile(struct file *pFile)
+{
+	if (pFile)
+	{
+		struct drm_file *psDRMFile = pFile->private_data;
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+		return (CONNECTION_DATA*)psConnectionPriv->pvConnectionData;
+#else
+		return (CONNECTION_DATA*)psConnectionPriv->pvSyncConnectionData;
+#endif
+	}
+
+	return NULL;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDriverInit
+@Description  Common one time driver initialisation
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDriverInit(void)
+{
+	PVRSRV_ERROR error;
+	int os_err;
+
+	error = PVROSFuncInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	error = PVRDKFInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+	error = PVRSRVCommonDriverInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	error = pvr_sync_register_functions();
+	if (error != PVRSRV_OK)
+	{
+		return -EPERM;
+	}
+
+	os_err = pvr_sync_init();
+	if (os_err != 0)
+	{
+		return os_err;
+	}
+#endif
+
+	os_err = pvr_apphint_init();
+	if (os_err != 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed AppHint setup(%d)", __func__,
+			     os_err));
+	}
+
+#if defined(SUPPORT_RGX)
+	error = PVRGpuTraceSupportInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif
+
+#if defined(ANDROID)
+#if defined(CONFIG_PROC_FS)
+	error = PVRProcFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#elif defined(CONFIG_DEBUG_FS)
+	error = PVRDebugFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif /* defined(CONFIG_PROC_FS) || defined(CONFIG_DEBUG_FS) */
+#else
+#if defined(CONFIG_DEBUG_FS)
+	error = PVRDebugFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#elif defined(CONFIG_PROC_FS)
+	error = PVRProcFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif /* defined(CONFIG_DEBUG_FS) || defined(CONFIG_PROC_FS) */
+#endif /* defined(ANDROID) */
+
+	error = PVRSRVIonStatsInitialise();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+#if defined(SUPPORT_RGX)
+	/* calling here because we need to handle input from the file even
+	 * before the devices are initialised
+	 * note: we're not passing a device node because apphint callbacks don't
+	 * need it */
+	PVRGpuTraceInitAppHintCallbacks(NULL);
+#endif
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDriverDeinit
+@Description  Common one time driver de-initialisation
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDriverDeinit(void)
+{
+	pvr_apphint_deinit();
+
+	PVRSRVIonStatsDestroy();
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	pvr_sync_deinit();
+#endif
+
+#if defined(SUPPORT_RGX)
+	PVRGpuTraceSupportDeInit();
+#endif
+
+	PVRSRVCommonDriverDeInit();
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	PVRDKFDeInit();
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+	PVROSFuncDeInit();
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceInit
+@Description  Common device related initialisation.
+@Input        psDeviceNode  The device node for which initialisation should be
+                            performed
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	{
+		PVRSRV_ERROR eError = pvr_sync_device_init(psDeviceNode->psDevConfig->pvOSDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: unable to create sync (%d)",
+					 __func__, eError));
+			return -EBUSY;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_RGX)
+	{
+		int error = PVRGpuTraceInitDevice(psDeviceNode);
+		if (error != 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+				 "%s: failed to initialise PVR GPU Tracing on device%d (%d)",
+				 __func__, psDeviceNode->sDevId.i32KernelDeviceID, error));
+		}
+	}
+#endif
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceDeinit
+@Description  Common device related de-initialisation.
+@Input        psDeviceNode  The device node for which de-initialisation should
+                            be performed
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDeviceDeinit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_RGX)
+	PVRGpuTraceDeInitDevice(psDeviceNode);
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	pvr_sync_device_deinit(psDeviceNode->psDevConfig->pvOSDevice);
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	pvr_fence_cleanup();
+#endif
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceShutdown
+@Description  Common device shutdown.
+@Input        psDev  The device node representing the device that should
+                     be shutdown
+@Return       void
+*/ /***************************************************************************/
+
+void PVRSRVDeviceShutdown(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	/* Since this is a shutdown request ignore the returned error and try to
+	 * to power off the device. This is done because there is no way of
+	 * signalling the OS that this call failed. */
+	(void) LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+
+	/* Passing PVRSRV_POWER_FLAGS_NONE as there are no special actions required
+	 * from the shutdown call beside the regular device power off. */
+	(void) PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+	                                       PVRSRV_SYS_POWER_STATE_OFF,
+	                                       PVRSRV_POWER_FLAGS_NONE);
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceSuspend
+@Description  Common device suspend.
+@Input        psDev  The device node representing the device that should
+                     be suspended
+@Return       int    0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_FALSE);
+	PVR_LOG_RETURN_IF_FALSE_VA(eError == PVRSRV_OK, -EFAULT,
+	                           "LinuxBridgeBlockClientsAccess() failed with error %u",
+	                           eError);
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceResume
+@Description  Common device resume.
+@Input        psDev  The device node representing the device that should
+                     be resumed
+@Return       int    0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_ON,
+										PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceServicesOpen
+@Description  Services device open.
+@Input        psDeviceNode  The device node representing the device being
+                            opened by a user mode process
+@Input        psDRMFile     The DRM file data that backs the file handle
+                            returned to the user mode process
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceServicesOpen(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             struct drm_file *psDRMFile)
+{
+	static DEFINE_MUTEX(sDeviceInitMutex);
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	ENV_CONNECTION_PRIVATE_DATA sPrivData;
+	PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+	PVRSRV_ERROR eError;
+	int iErr = 0;
+
+	if (!psPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device data", __func__));
+		iErr = -ENODEV;
+		goto out;
+	}
+
+	mutex_lock(&sDeviceInitMutex);
+	/*
+	 * If the first attempt already set the state to bad,
+	 * there is no point in going the second time, so get out
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_BAD)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Driver already in bad state. Device open failed.",
+				 __func__));
+		iErr = -ENODEV;
+		mutex_unlock(&sDeviceInitMutex);
+		goto out;
+	}
+
+	if (psDRMFile->driver_priv == NULL)
+	{
+		/* Allocate psConnectionPriv (stores private data and release pfn under driver_priv) */
+		psConnectionPriv = kzalloc(sizeof(*psConnectionPriv), GFP_KERNEL);
+		if (!psConnectionPriv)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate driver_priv data", __func__));
+			iErr = -ENOMEM;
+			mutex_unlock(&sDeviceInitMutex);
+			goto fail_alloc_connection_priv;
+		}
+		psConnectionPriv->ui32Type = DKF_CONNECTION_FLAG_SERVICES;
+	}
+	else
+	{
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+	}
+
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_CREATED)
+	{
+		eError = PVRSRVCommonDeviceInitialise(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise device (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			iErr = -ENODEV;
+			mutex_unlock(&sDeviceInitMutex);
+			goto fail_device_init;
+		}
+
+#if defined(SUPPORT_RGX)
+		PVRGpuTraceInitIfEnabled(psDeviceNode);
+#endif
+	}
+	mutex_unlock(&sDeviceInitMutex);
+
+	sPrivData.psDevNode = psDeviceNode;
+	sPrivData.psDRMFile = psDRMFile;
+
+	/*
+	 * Here we pass the file pointer which will passed through to our
+	 * OSConnectionPrivateDataInit function where we can save it so
+	 * we can back reference the file structure from its connection
+	 */
+	eError = PVRSRVCommonConnectionConnect(&psConnectionPriv->pvConnectionData,
+	                                       (void *)&sPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		iErr = -ENOMEM;
+		goto fail_connect;
+	}
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pfDeviceRelease = PVRSRVCommonConnectionDisconnect;
+#endif
+	psDRMFile->driver_priv = (void*)psConnectionPriv;
+
+	goto out;
+
+fail_connect:
+fail_device_init:
+	kfree(psConnectionPriv);
+fail_alloc_connection_priv:
+out:
+	return iErr;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceSyncOpen
+@Description  Sync device open.
+@Input        psDeviceNode  The device node representing the device being
+                            opened by a user mode process
+@Input        psDRMFile     The DRM file data that backs the file handle
+                            returned to the user mode process
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+static int PVRSRVDeviceSyncOpen(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                struct drm_file *psDRMFile)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	CONNECTION_DATA *psConnection = NULL;
+	ENV_CONNECTION_PRIVATE_DATA sPrivData;
+	PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+	PVRSRV_ERROR eError;
+	int iErr = 0;
+
+	if (!psPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device data", __func__));
+		iErr = -ENODEV;
+		goto out;
+	}
+
+	if (psDRMFile->driver_priv == NULL)
+	{
+		/* Allocate psConnectionPriv (stores private data and release pfn under driver_priv) */
+		psConnectionPriv = kzalloc(sizeof(*psConnectionPriv), GFP_KERNEL);
+		if (!psConnectionPriv)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate driver_priv data", __func__));
+			iErr = -ENOMEM;
+			goto out;
+		}
+	}
+	else
+	{
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+	}
+
+	/* Allocate connection data area, no stats since process not registered yet */
+	psConnection = kzalloc(sizeof(*psConnection), GFP_KERNEL);
+	if (!psConnection)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate connection data", __func__));
+		iErr = -ENOMEM;
+		goto fail_alloc_connection;
+	}
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pvConnectionData = (void*)psConnection;
+#else
+	psConnectionPriv->pvSyncConnectionData = (void*)psConnection;
+#endif
+
+	psConnectionPriv->ui32Type = DKF_CONNECTION_FLAG_SYNC;
+
+	sPrivData.psDevNode = psDeviceNode;
+	sPrivData.psDRMFile = psDRMFile;
+
+	/* Call environment specific connection data init function */
+	eError = OSConnectionPrivateDataInit(&psConnection->hOsPrivateData, &sPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSConnectionPrivateDataInit() failed (%s)",
+		        __func__, PVRSRVGetErrorString(eError)));
+		goto fail_private_data_init;
+	}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	iErr = pvr_sync_open(psConnectionPriv->pvConnectionData, psDRMFile);
+#else
+	iErr = pvr_sync_open(psConnectionPriv->pvSyncConnectionData, psDRMFile);
+#endif
+	if (iErr)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: pvr_sync_open() failed(%d)",
+				__func__, iErr));
+		goto fail_pvr_sync_open;
+	}
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pfDeviceRelease = pvr_sync_close;
+#endif
+#endif
+	psDRMFile->driver_priv = psConnectionPriv;
+	goto out;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+fail_pvr_sync_open:
+	OSConnectionPrivateDataDeInit(psConnection->hOsPrivateData);
+#endif
+fail_private_data_init:
+	kfree(psConnection);
+fail_alloc_connection:
+	kfree(psConnectionPriv);
+out:
+	return iErr;
+#else /* if defined(SUPPORT_NATIVE_FENCE_SYNC) */
+	/* Ioctl should not be being called (no DDK support for native sync) */
+	PVR_DPF((PVR_DBG_ERROR, "%s: only supported when SUPPORT_NATIVE_FENCE_SYNC=1", __func__));
+	return -ENOTTY;
+#endif /* if defined(SUPPORT_NATIVE_FENCE_SYNC) */
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceRelease
+@Description  Common device release.
+@Input        psDeviceNode  The device node for the device that the given file
+                            represents
+@Input        psDRMFile     The DRM file data that's being released
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDeviceRelease(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         struct drm_file *psDRMFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	if (psDRMFile->driver_priv)
+	{
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+
+		if (psConnectionPriv->pvConnectionData)
+		{
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+			if (psConnectionPriv->pfDeviceRelease)
+			{
+				psConnectionPriv->pfDeviceRelease(psConnectionPriv->pvConnectionData);
+			}
+#else
+			if (psConnectionPriv->pvConnectionData)
+				PVRSRVCommonConnectionDisconnect(psConnectionPriv->pvConnectionData);
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+			if (psConnectionPriv->pvSyncConnectionData)
+				pvr_sync_close(psConnectionPriv->pvSyncConnectionData);
+#endif
+#endif
+		}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+		LinuxDeregisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+
+		kfree(psDRMFile->driver_priv);
+		psDRMFile->driver_priv = NULL;
+	}
+}
+
+int
+drm_pvr_srvkm_init(struct drm_device *dev, void *arg, struct drm_file *psDRMFile)
+{
+	struct drm_pvr_srvkm_init_data *data = arg;
+	struct pvr_drm_private *priv = dev->dev_private;
+	int iErr = 0;
+
+	switch (data->init_module)
+	{
+		case PVR_SRVKM_SYNC_INIT:
+		{
+			iErr = PVRSRVDeviceSyncOpen(priv->dev_node, psDRMFile);
+			break;
+		}
+		case PVR_SRVKM_SERVICES_INIT:
+		{
+			iErr = PVRSRVDeviceServicesOpen(priv->dev_node, psDRMFile);
+			if (iErr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDeviceServicesOpen() failed(%d)",
+						__func__, iErr));
+				break;
+			}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+			iErr = LinuxRegisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+			break;
+		}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		case PVR_SRVKM_SERVICES_PAGE_MIGRATE_INIT:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+			iErr = LinuxRegisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+			break;
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid init_module (%d)",
+			        __func__, data->init_module));
+			iErr = -EINVAL;
+		}
+	}
+
+	return iErr;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.h
new file mode 100644
index 000000000000..35f33770e2e5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@File           module_common.h
+@Title          Common linux module setup header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef MODULE_COMMON_H
+#define MODULE_COMMON_H
+
+#include "pvr_drm.h"
+
+/* DRVNAME is the name we use to register our driver. */
+#define DRVNAME PVR_LDM_DRIVER_REGISTRATION_NAME
+
+struct _PVRSRV_DEVICE_NODE_;
+struct drm_file;
+struct drm_device;
+
+/* psDRMFile->driver_priv will point to a PVRSV_CONNECTION_PRIV
+ * struct, which will contain a ptr to the CONNECTION_DATA and
+ * a pfn to the release function (which will differ depending
+ * on whether the connection is to Sync or Services).
+ */
+typedef void (*PFN_PVRSRV_DEV_RELEASE)(void *pvData);
+typedef struct
+{
+	/* pvConnectionData is used to hold Services connection data
+	 * for all PVRSRV_DEVICE_INIT_MODE options.
+	 */
+	void *pvConnectionData;
+
+	/* pfDeviceRelease is used to indicate the release function
+	 * to be called when PVRSRV_DEVICE_INIT_MODE is PVRSRV_LINUX_DEV_INIT_ON_CONNECT,
+	 * as we can then have one connections made (either for Services or Sync) per
+	 * psDRMFile, and need to know which type of connection is being released
+	 * (as the ioctl release call is common for both).
+	 */
+	PFN_PVRSRV_DEV_RELEASE pfDeviceRelease;
+
+	/* pvSyncConnectionData is used to hold Sync connection data
+	 * when PVRSRV_DEVICE_INIT_MODE is not PVRSRV_LINUX_DEV_INIT_ON_CONNECT,
+	 * as we can then have two connections made (for Services and Sync) to
+	 * the same psDRMFile.
+	 */
+	void *pvSyncConnectionData;
+
+	/* An integer representing the type of connection (indicated by the
+	 * DKF_CONNECTION_FLAGS).
+	 */
+	uint32_t ui32Type;
+
+} PVRSRV_CONNECTION_PRIV;
+
+int PVRSRVDriverInit(void);
+void PVRSRVDriverDeinit(void);
+
+int PVRSRVDeviceInit(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+void PVRSRVDeviceDeinit(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+void PVRSRVDeviceShutdown(struct drm_device *psDev);
+int PVRSRVDeviceSuspend(struct drm_device *psDev);
+int PVRSRVDeviceResume(struct drm_device *psDev);
+
+int PVRSRVDeviceServicesOpen(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                             struct drm_file *psDRMFile);
+void PVRSRVDeviceRelease(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                         struct drm_file *psDRMFile);
+int drm_pvr_srvkm_init(struct drm_device *dev,
+                       void *arg, struct drm_file *psDRMFile);
+
+#endif /* MODULE_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osconnection_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osconnection_server.c
new file mode 100644
index 000000000000..543b4d8ddabf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osconnection_server.c
@@ -0,0 +1,168 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux specific per process data functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "connection_server.h"
+#include "osconnection_server.h"
+
+#include "env_connection.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+
+#include <linux/sched.h>
+
+#if defined(SUPPORT_ION)
+#include <linux/err.h>
+#include PVR_ANDROID_ION_HEADER
+
+/*
+	The ion device (the base object for all requests)
+	gets created by the system and we acquire it via
+	Linux specific functions provided by the system layer
+*/
+#include "ion_sys.h"
+#endif
+
+PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData)
+{
+	ENV_CONNECTION_PRIVATE_DATA *psPrivData = pvOSData;
+	ENV_CONNECTION_DATA *psEnvConnection;
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	ENV_ION_CONNECTION_DATA *psIonConnection;
+#endif
+
+	*phOsPrivateData = OSAllocZMem(sizeof(ENV_CONNECTION_DATA));
+
+	if (*phOsPrivateData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem failed", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psEnvConnection = (ENV_CONNECTION_DATA *)*phOsPrivateData;
+
+	psEnvConnection->owner = current->tgid;
+
+	psEnvConnection->psDevNode = psPrivData->psDevNode;
+	psEnvConnection->psDRMFile = psPrivData->psDRMFile;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	psEnvConnection->pvPvrSyncPrivateData = NULL;
+#endif
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	psIonConnection = (ENV_ION_CONNECTION_DATA *)OSAllocZMem(sizeof(ENV_ION_CONNECTION_DATA));
+	if (psIonConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem failed", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psEnvConnection->psIonData = psIonConnection;
+	/*
+		We can have more than one connection per process, so we need
+		more than the PID to have a unique name.
+	*/
+	psEnvConnection->psIonData->psIonDev = IonDevAcquire();
+	OSSNPrintf(psEnvConnection->psIonData->azIonClientName, ION_CLIENT_NAME_SIZE, "pvr_ion_client-%p-%d", *phOsPrivateData, OSGetCurrentClientProcessIDKM());
+	psEnvConnection->psIonData->psIonClient =
+		ion_client_create(psEnvConnection->psIonData->psIonDev,
+						  psEnvConnection->psIonData->azIonClientName);
+
+	if (IS_ERR_OR_NULL(psEnvConnection->psIonData->psIonClient))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSConnectionPrivateDataInit: Couldn't create "
+								"ion client for per connection data"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+#endif /* SUPPORT_ION && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData)
+{
+	if (hOsPrivateData == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	{
+		ENV_CONNECTION_DATA *psEnvConnection = hOsPrivateData;
+
+		PVR_ASSERT(psEnvConnection->psIonData != NULL);
+
+		PVR_ASSERT(psEnvConnection->psIonData->psIonClient != NULL);
+		ion_client_destroy(psEnvConnection->psIonData->psIonClient);
+
+		IonDevRelease(psEnvConnection->psIonData->psIonDev);
+		OSFreeMem(psEnvConnection->psIonData);
+	}
+#endif
+
+	OSFreeMem(hOsPrivateData);
+	/*not nulling pointer, copy on stack*/
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_DEVICE_NODE *OSGetDevNode(CONNECTION_DATA *psConnection)
+{
+	ENV_CONNECTION_DATA *psEnvConnection;
+
+	psEnvConnection = PVRSRVConnectionPrivateData(psConnection);
+	PVR_ASSERT(psEnvConnection);
+
+	return psEnvConnection->psDevNode;
+}
+
+struct drm_file *OSGetDRMFile(CONNECTION_DATA *psConnection)
+{
+	ENV_CONNECTION_DATA *psEnvConnection;
+
+	psEnvConnection = PVRSRVConnectionPrivateData(psConnection);
+	PVR_ASSERT(psEnvConnection);
+
+	return psEnvConnection->psDRMFile;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc.c
new file mode 100644
index 000000000000..cc0652febd98
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc.c
@@ -0,0 +1,2801 @@
+/*************************************************************************/ /*!
+@File
+@Title          Environment related functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <asm/page.h>
+#include <asm/div64.h>
+#include <linux/atomic.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/hugetlb.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+#include <linux/genalloc.h>
+#include <linux/string.h>
+#include <linux/freezer.h>
+#include <asm/hardirq.h>
+#include <asm/tlbflush.h>
+#include <linux/timer.h>
+#include <linux/capability.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-direction.h>
+#include <linux/kthread.h>
+#include <linux/utsname.h>
+#include <linux/scatterlist.h>
+#include <linux/interrupt.h>
+#include <linux/pfn_t.h>
+#include <linux/pfn.h>
+#include <linux/pid.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+#include <linux/sched/clock.h>
+#include <linux/sched/signal.h>
+#else
+#include <linux/sched.h>
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) */
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+#include <linux/dma-heap.h>
+#include "physmem_dmabuf.h"
+#else
+#include "physmem.h"
+#endif
+#endif
+
+#include "log2.h"
+#include "sysinfo.h"
+#include "osfunc.h"
+#include "cache_km.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "devicemem_server_utils.h"
+#include "event.h"
+#include "linkage.h"
+#include "pvr_uaccess.h"
+#include "pvr_debug.h"
+#include "pvr_bridge_k.h"
+#include "pvrsrv_memallocflags.h"
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+#include "physmem_osmem_linux.h"
+#include "dma_support.h"
+
+#include "pvrsrv_sync_server.h"
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#include "pvr_ricommon.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+#if !defined(EVENT_OBJECT_TIMEOUT_US)
+#error EVENT_OBJECT_TIMEOUT_US should be defined sysinfo.h
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+#define PVR_FOLL_LONGTERM (0x0U)
+#else
+#define PVR_FOLL_LONGTERM FOLL_LONGTERM
+#endif
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 6, 0))
+
+#define pvr_pin_user_pages_for_dma(puiAddress, num_pages, bWrite, pages)  get_user_pages_fast( \
+    (unsigned long)puiAddress, \
+    (int)num_pages, \
+    (int) (bWrite ? FOLL_WRITE : 0) | PVR_FOLL_LONGTERM, \
+    pages)
+#define pvr_unpin_user_page_for_dma(p) put_page(p)
+
+#else
+
+#define pvr_pin_user_pages_for_dma(puiAddress, num_pages, bWrite, pages) pin_user_pages_fast( \
+    (unsigned long)puiAddress, \
+    (int) num_pages, \
+    (int) (bWrite ? FOLL_WRITE : 0) | PVR_FOLL_LONGTERM, \
+    pages)
+#define pvr_unpin_user_page_for_dma(p) unpin_user_page(p)
+
+#endif
+
+#define _FREEZABLE IMG_TRUE
+#define _NON_FREEZABLE IMG_FALSE
+
+typedef struct {
+	struct task_struct *kthread;
+	PFN_THREAD pfnThread;
+	void *hData;
+	IMG_CHAR *pszThreadName;
+	IMG_BOOL   bIsThreadRunning;
+	IMG_BOOL   bIsSupportingThread;
+	PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB;
+	DLLIST_NODE sNode;
+} OSThreadData;
+
+void OSSuspendTaskInterruptible(void)
+{
+	set_current_state(TASK_INTERRUPTIBLE);
+	schedule();
+}
+
+static DLLIST_NODE gsThreadListHead;
+
+static void _ThreadListAddEntry(OSThreadData *psThreadListNode)
+{
+	dllist_add_to_tail(&gsThreadListHead, &(psThreadListNode->sNode));
+}
+
+static void _ThreadListRemoveEntry(OSThreadData *psThreadListNode)
+{
+	dllist_remove_node(&(psThreadListNode->sNode));
+}
+
+static void _ThreadSetStopped(OSThreadData *psOSThreadData)
+{
+	psOSThreadData->bIsThreadRunning = IMG_FALSE;
+}
+
+static void _OSInitThreadList(void)
+{
+	dllist_init(&gsThreadListHead);
+}
+
+void OSThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile)
+{
+	PDLLIST_NODE psNodeCurr, psNodeNext;
+	LINUX_THREAD_ACTIVITY_STATS sThreadStats;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	dllist_foreach_node(&gsThreadListHead, psNodeCurr, psNodeNext)
+	{
+		OSThreadData *psThreadListNode;
+		psThreadListNode = IMG_CONTAINER_OF(psNodeCurr, OSThreadData, sNode);
+
+		PVR_DUMPDEBUG_LOG("  %s : %s",
+				  psThreadListNode->pszThreadName,
+				  (psThreadListNode->bIsThreadRunning) ? "Running" : "Stopped");
+
+		if (psThreadListNode->pfnDebugDumpCB)
+		{
+			psThreadListNode->pfnDebugDumpCB(pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+	}
+
+	eError = LinuxGetThreadActivityStats(&sThreadStats);
+	if (eError == PVRSRV_OK)
+	{
+		PVR_DUMPDEBUG_LOG("Active Threads (UM/KM): %d / %d",
+		                  sThreadStats.i32DriverThreadCount,
+		                  sThreadStats.i32KernelThreadCount);
+
+		PVR_DUMPDEBUG_LOG("Suspended Threads: %d",
+		                  sThreadStats.i32SuspendedThreadCount);
+	}
+	else
+	{
+		PVR_LOG_ERROR(eError, "LinuxGetThreadActivityStats");
+	}
+}
+
+PVRSRV_ERROR OSPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+							PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+							IMG_PID uiPid)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	IMG_CPU_PHYADDR sCpuPAddr;
+	struct page *psPage;
+	IMG_UINT32	ui32Order=0;
+	gfp_t gfp_flags;
+
+	PVR_ASSERT(uiSize != 0);
+	/*Align the size to the page granularity */
+	uiSize = PAGE_ALIGN(uiSize);
+
+	/*Get the order to be used with the allocation */
+	ui32Order = get_order(uiSize);
+
+	gfp_flags = GFP_KERNEL;
+
+#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+	if (psDev)
+	{
+		if (*psDev->dma_mask == DMA_BIT_MASK(32))
+		{
+			/* Limit to 32 bit.
+			 * Achieved by setting __GFP_DMA32 for 64 bit systems */
+			gfp_flags |= __GFP_DMA32;
+		}
+		else if (*psDev->dma_mask < DMA_BIT_MASK(32))
+		{
+			/* Limit to whatever the size of DMA zone is. */
+			gfp_flags |= __GFP_DMA;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDev);
+#endif
+
+	/*allocate the pages */
+	psPage = alloc_pages_node(dev_to_node(psDev), gfp_flags, ui32Order);
+	if (psPage == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	uiSize = (1 << ui32Order) * PAGE_SIZE;
+
+	psMemHandle->u.pvHandle = psPage;
+	psMemHandle->uiOrder = ui32Order;
+	sCpuPAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(psPage));
+
+	/*
+	 * Even when more pages are allocated as base MMU object we still need one single physical address because
+	 * they are physically contiguous.
+	 */
+	PhysHeapCpuPAddrToDevPAddr(psPhysHeap, 1, psDevPAddr, &sCpuPAddr);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                    uiSize,
+	                                    (IMG_UINT64)(uintptr_t) psPage,
+	                                    uiPid);
+#else
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                             psPage,
+	                             sCpuPAddr,
+	                             uiSize,
+	                             uiPid
+	                             DEBUG_MEMSTATS_VALUES);
+#endif
+#else
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif
+
+	return PVRSRV_OK;
+}
+
+void OSPhyContigPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle)
+{
+	struct page *psPage = (struct page*) psMemHandle->u.pvHandle;
+	IMG_UINT32 ui32Order;
+
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+	ui32Order = psMemHandle->uiOrder;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                      (IMG_UINT64)(uintptr_t) psPage);
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                (IMG_UINT64)(uintptr_t) psPage,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	__free_pages(psPage, ui32Order);
+	psMemHandle->uiOrder = 0;
+}
+
+PVRSRV_ERROR OSPhyContigPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle,
+						size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+						void **pvPtr)
+{
+	size_t actualSize = 1 << (PAGE_SHIFT + psMemHandle->uiOrder);
+	#if (!MAP_UNCACHED_MEM)
+	*pvPtr = kmap((struct page*)psMemHandle->u.pvHandle);
+	#else
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	struct page *page = (struct page*)psMemHandle->u.pvHandle;
+	unsigned long i, num_pages = PAGE_ALIGN(actualSize) / PAGE_SIZE;
+	struct page **pages = kmalloc_node(sizeof(struct page *) * num_pages, GFP_KERNEL, dev_to_node(psDev));
+	struct page **tmp = pages;
+	pgprot_t prot = PAGE_KERNEL;
+	void *vaddr;
+
+	#ifdef CONFIG_NUMA
+	if (page_to_nid(page) != dev_to_node(psDev)) {
+		pr_warn("%s: mem node %d(phys 0x%llx size 0x%lx) and dev node %d are not on the same die\n", __func__,
+			page_to_nid(page), page_to_phys(page) ,uiSize, dev_to_node(psDev));
+	}
+	#endif
+
+	for (i = 0; i < num_pages; ++i)
+		*tmp++ = page++;
+
+	prot = pgprot_dmacoherent(PAGE_KERNEL);
+	vaddr = vmap(pages, num_pages, VM_MAP, prot);
+	kfree(pages);
+
+	if (!vaddr)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	*pvPtr = vaddr;
+	#endif
+	PVR_UNREFERENCED_PARAMETER(psDevPAddr);
+
+	PVR_UNREFERENCED_PARAMETER(actualSize); /* If we don't take an #ifdef path */
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA, actualSize, OSGetCurrentClientProcessIDKM());
+#else
+	{
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+									 *pvPtr,
+									 sCpuPAddr,
+									 actualSize,
+									 OSGetCurrentClientProcessIDKM()
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+}
+
+void OSPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	/* Mapping is done a page at a time */
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+	                            (1 << (PAGE_SHIFT + psMemHandle->uiOrder)),
+	                            OSGetCurrentClientProcessIDKM());
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+	                                (IMG_UINT64)(uintptr_t)pvPtr,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+	PVR_UNREFERENCED_PARAMETER(pvPtr);
+
+	#if (!MAP_UNCACHED_MEM)
+	kunmap((struct page*) psMemHandle->u.pvHandle);
+	#else
+	vunmap(pvPtr);
+	#endif
+}
+
+PVRSRV_ERROR OSPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_UINT32 uiOffset,
+                                   IMG_UINT32 uiLength)
+{
+	#if (!MAP_UNCACHED_MEM)
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct page* psPage = (struct page*) psMemHandle->u.pvHandle;
+
+	void* pvVirtAddrStart = kmap(psPage) + uiOffset;
+	IMG_CPU_PHYADDR sPhysStart, sPhysEnd;
+
+	IMG_UINT32 ui32Order;
+
+	if (uiLength == 0)
+	{
+		goto e0;
+	}
+
+	ui32Order = psMemHandle->uiOrder;
+	if ((uiOffset + uiLength) > ((1 << ui32Order) * PAGE_SIZE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid size params, uiOffset %u, uiLength %u",
+				__func__,
+				uiOffset,
+				uiLength));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	sPhysStart.uiAddr = page_to_phys(psPage) + uiOffset;
+	sPhysEnd.uiAddr = sPhysStart.uiAddr + uiLength;
+
+	CacheOpExec(psDevNode,
+				pvVirtAddrStart,
+				pvVirtAddrStart + uiLength,
+				sPhysStart,
+				sPhysEnd,
+				PVRSRV_CACHE_OP_CLEAN);
+
+e0:
+	kunmap(psPage);
+
+	return eError;
+	#else
+	return PVRSRV_OK;
+	#endif
+
+}
+
+#if defined(__GNUC__)
+#define PVRSRV_MEM_ALIGN __attribute__ ((aligned (0x8)))
+#define PVRSRV_MEM_ALIGN_MASK (0x7)
+#else
+#error "PVRSRV Alignment macros need to be defined for this compiler"
+#endif
+
+IMG_UINT32 OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE eCacheAttribute)
+{
+	IMG_UINT32 uiSize = 0;
+
+	switch (eCacheAttribute)
+	{
+		case OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE:
+			uiSize = cache_line_size();
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid cache attribute type %d",
+					__func__, (IMG_UINT32)eCacheAttribute));
+			PVR_ASSERT(0);
+			break;
+	}
+
+	return uiSize;
+}
+
+IMG_INT OSMemCmp(void *pvBufA, void *pvBufB, size_t uiLen)
+{
+	return (IMG_INT)memcmp(pvBufA, pvBufB, uiLen);
+}
+
+size_t OSStringLCat(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDstSize)
+{
+	/*
+	 * Let strlcat handle any truncation cases correctly.
+	 * We will definitely get a NUL-terminated string set in pszDest
+	 */
+	size_t uSrcSize = strlcat(pszDest, pszSrc, uDstSize);
+
+#if defined(PVR_DEBUG_STRLCPY) && defined(DEBUG)
+	/* Handle truncation by dumping calling stack if debug allows */
+	if (uSrcSize >= uDstSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+			"%s: String truncated Src = '<%s>' %ld bytes, Dest = '%s'",
+			__func__, pszSrc, (long)uDstSize, pszDest));
+		OSDumpStack();
+	}
+#endif /* defined(PVR_DEBUG_STRLCPY) && defined(DEBUG) */
+
+	return uSrcSize;
+}
+
+IMG_INT32 OSSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR *pszFormat, ...)
+{
+	va_list argList;
+	IMG_INT32 iCount;
+
+	va_start(argList, pszFormat);
+	iCount = vsnprintf(pStr, (size_t)ui32Size, pszFormat, argList);
+	va_end(argList);
+
+	return iCount;
+}
+
+IMG_INT32 OSVSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR* pszFormat, va_list vaArgs)
+{
+	return vsnprintf(pStr, ui32Size, pszFormat, vaArgs);
+}
+
+size_t OSStringLength(const IMG_CHAR *pStr)
+{
+	return strlen(pStr);
+}
+
+size_t OSStringNLength(const IMG_CHAR *pStr, size_t uiCount)
+{
+	return strnlen(pStr, uiCount);
+}
+
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                          size_t uiSize)
+{
+#if defined(DEBUG)
+	/* Double-check that we are not passing NULL parameters in. If we are we
+	 * return -1 (for arg1 == NULL, arg2 != NULL)
+	 * 0 (for arg1 == NULL, arg2 == NULL
+	 * +1 (for arg1 != NULL, arg2 == NULL)
+	 * strncmp(arg1, arg2, size) otherwise
+	 */
+	if (pStr1 == NULL)
+	{
+		if (pStr2 == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): Both args NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return 0;	/* Both NULL */
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): arg1 NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return -1;	/* NULL < non-NULL */
+		}
+	}
+	else
+	{
+		if (pStr2 == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): arg2 NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return +1;	/* non-NULL > NULL */
+		}
+		else
+		{
+			return strncmp(pStr1, pStr2, uiSize);
+		}
+	}
+#else
+	return strncmp(pStr1, pStr2, uiSize);
+#endif
+}
+
+PVRSRV_ERROR OSStringToUINT32(const IMG_CHAR *pStr, IMG_UINT32 ui32Base,
+                              IMG_UINT32 *ui32Result)
+{
+	if (kstrtou32(pStr, ui32Base, ui32Result) != 0)
+		return PVRSRV_ERROR_CONVERSION_FAILED;
+
+	return PVRSRV_OK;
+}
+
+IMG_UINT32 OSStringUINT32ToStr(IMG_CHAR *pszBuf, size_t uSize,
+							IMG_UINT32 ui32Num)
+{
+	IMG_UINT32 ui32i, ui32Len = 0, ui32NumCopy = ui32Num;
+
+	/* calculate string length required to hold the number string */
+	do
+	{
+		ui32Len++;
+		ui32NumCopy /= 10;
+	} while (ui32NumCopy != 0);
+
+	if (unlikely(ui32Len >= uSize))
+	{
+		/* insufficient buffer */
+		return 0;
+	}
+
+	for (ui32i = 0; ui32i < ui32Len; ui32i++)
+	{
+		pszBuf[ui32Len - (ui32i + 1)] = '0' + ui32Num % 10;
+		ui32Num = ui32Num / 10;
+	}
+
+	pszBuf[ui32Len] = '\0';
+	return ui32Len;
+}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+static struct workqueue_struct *gpFenceStatusWq;
+static struct workqueue_struct *gpFenceCtxDestroyWq;
+
+static PVRSRV_ERROR _NativeSyncInit(void)
+{
+	gpFenceStatusWq = create_freezable_workqueue("pvr_fence_status");
+	if (!gpFenceStatusWq)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create foreign fence status workqueue",
+				 __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	gpFenceCtxDestroyWq = create_freezable_workqueue("pvr_fence_context_destroy");
+	if (!gpFenceCtxDestroyWq)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create foreign fence context destruction workqueue",
+				 __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void _NativeSyncDeinit(void)
+{
+	destroy_workqueue(gpFenceStatusWq);
+	destroy_workqueue(gpFenceCtxDestroyWq);
+}
+
+struct workqueue_struct *NativeSyncGetFenceStatusWq(void)
+{
+	if (!gpFenceStatusWq)
+	{
+#if defined(DEBUG)
+		PVR_ASSERT(gpFenceStatusWq);
+#endif
+		return NULL;
+	}
+
+	return gpFenceStatusWq;
+}
+
+struct workqueue_struct *NativeSyncGetFenceCtxDestroyWq(void)
+{
+	if (!gpFenceCtxDestroyWq)
+	{
+#if defined(DEBUG)
+		PVR_ASSERT(gpFenceCtxDestroyWq);
+#endif
+		return NULL;
+	}
+
+	return gpFenceCtxDestroyWq;
+}
+#endif
+
+PVRSRV_ERROR OSInitEnvData(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = LinuxInitPhysmem();
+	PVR_GOTO_IF_ERROR(eError, error_out);
+
+	_OSInitThreadList();
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	eError = _NativeSyncInit();
+#endif
+
+error_out:
+	return eError;
+}
+
+void OSDeInitEnvData(void)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	_NativeSyncDeinit();
+#endif
+
+	LinuxDeinitPhysmem();
+}
+
+void OSReleaseThreadQuanta(void)
+{
+	schedule();
+}
+
+void OSMemoryBarrier(volatile void *hReadback)
+{
+	mb();
+
+	if (hReadback)
+	{
+		/* Force a read-back to memory to avoid posted writes on certain buses
+		 * e.g. PCI(E)
+		 */
+		(void) OSReadDeviceMem32(hReadback);
+	}
+}
+
+void OSWriteMemoryBarrier(volatile void *hReadback)
+{
+	wmb();
+
+	if (hReadback)
+	{
+		/* Force a read-back to memory to avoid posted writes on certain buses
+		 * e.g. PCI(E)
+		 */
+		(void) OSReadDeviceMem32(hReadback);
+	}
+}
+
+/* Not matching/aligning this API to the Clockus() API above to avoid necessary
+ * multiplication/division operations in calling code.
+ */
+static inline IMG_UINT64 Clockns64(void)
+{
+	IMG_UINT64 timenow;
+
+	/* Kernel thread preempt protection. Some architecture implementations
+	 * (ARM) of sched_clock are not preempt safe when the kernel is configured
+	 * as such e.g. CONFIG_PREEMPT and others.
+	 */
+	preempt_disable();
+
+	/* Using sched_clock instead of ktime_get since we need a time stamp that
+	 * correlates with that shown in kernel logs and trace data not one that
+	 * is a bit behind. */
+	timenow = sched_clock();
+
+	preempt_enable();
+
+	return timenow;
+}
+
+IMG_UINT64 OSClockns64(void)
+{
+	return Clockns64();
+}
+
+IMG_UINT64 OSClockus64(void)
+{
+	IMG_UINT64 timenow = Clockns64();
+	IMG_UINT32 remainder;
+
+	return OSDivide64r64(timenow, 1000, &remainder);
+}
+
+IMG_UINT32 OSClockus(void)
+{
+	return (IMG_UINT32) OSClockus64();
+}
+
+IMG_UINT32 OSClockms(void)
+{
+	IMG_UINT64 timenow = Clockns64();
+	IMG_UINT32 remainder;
+
+	return OSDivide64(timenow, 1000000, &remainder);
+}
+
+static inline IMG_UINT64 KClockns64(void)
+{
+	ktime_t sTime = ktime_get();
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return sTime;
+#else
+	return sTime.tv64;
+#endif
+}
+
+PVRSRV_ERROR OSClockMonotonicns64(IMG_UINT64 *pui64Time)
+{
+	*pui64Time = KClockns64();
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSClockMonotonicus64(IMG_UINT64 *pui64Time)
+{
+	IMG_UINT64 timenow = KClockns64();
+	IMG_UINT32 remainder;
+
+	*pui64Time = OSDivide64r64(timenow, 1000, &remainder);
+	return PVRSRV_OK;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+IMG_UINT64 OSClockMonotonicRawns64(void)
+{
+	struct timespec64 ts;
+
+	ktime_get_raw_ts64(&ts);
+	return ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+#else
+IMG_UINT64 OSClockMonotonicRawns64(void)
+{
+	struct timespec ts;
+
+	getrawmonotonic(&ts);
+	return (IMG_UINT64) ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+#endif
+
+IMG_UINT64 OSClockMonotonicRawus64(void)
+{
+	IMG_UINT32 rem;
+	return OSDivide64r64(OSClockMonotonicRawns64(), 1000, &rem);
+}
+
+/*
+	OSWaitus
+*/
+void OSWaitus(IMG_UINT32 ui32Timeus)
+{
+	udelay(ui32Timeus);
+}
+
+
+/*
+	OSSleepms
+*/
+void OSSleepms(IMG_UINT32 ui32Timems)
+{
+	msleep(ui32Timems);
+}
+
+
+INLINE IMG_UINT64 OSGetCurrentProcessVASpaceSize(void)
+{
+	return (IMG_UINT64)TASK_SIZE;
+}
+
+INLINE IMG_PID OSGetCurrentProcessID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return (IMG_PID)task_tgid_nr(current);
+}
+
+INLINE IMG_PID OSGetCurrentVirtualProcessID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return (IMG_PID)task_tgid_vnr(current);
+}
+
+INLINE IMG_CHAR *OSGetCurrentProcessName(void)
+{
+	return current->comm;
+}
+
+INLINE uintptr_t OSGetCurrentThreadID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return current->pid;
+}
+
+IMG_PID OSGetCurrentClientProcessIDKM(void)
+{
+	return OSGetCurrentProcessID();
+}
+
+IMG_CHAR *OSGetCurrentClientProcessNameKM(void)
+{
+	return OSGetCurrentProcessName();
+}
+
+uintptr_t OSAcquireCurrentPPIDResourceRefKM(void)
+{
+	struct pid *psPPIDResource = find_pid_ns(OSGetCurrentClientProcessIDKM(), &init_pid_ns);
+
+	PVR_ASSERT(psPPIDResource != NULL);
+	/* Take ref on pPid */
+	get_pid(psPPIDResource);
+	return (uintptr_t)psPPIDResource;
+}
+
+void OSReleasePPIDResourceRefKM(uintptr_t psPPIDResource)
+{
+	/* Drop ref on uiProc */
+	put_pid((struct pid*)psPPIDResource);
+}
+
+uintptr_t OSGetCurrentClientThreadIDKM(void)
+{
+	return OSGetCurrentThreadID();
+}
+
+size_t OSGetPageSize(void)
+{
+	return PAGE_SIZE;
+}
+
+size_t OSGetPageShift(void)
+{
+	return PAGE_SHIFT;
+}
+
+size_t OSGetPageMask(void)
+{
+	return (OSGetPageSize()-1);
+}
+
+size_t OSGetOrder(size_t uSize)
+{
+	return get_order(PAGE_ALIGN(uSize));
+}
+
+IMG_UINT64 OSGetRAMSize(void)
+{
+	struct sysinfo SI;
+	si_meminfo(&SI);
+
+	return (PAGE_SIZE * SI.totalram);
+}
+
+typedef struct
+{
+	int os_error;
+	PVRSRV_ERROR pvr_error;
+} error_map_t;
+
+#define PVRSRV_ERROR_TO_OS_ERROR \
+	X(PVRSRV_OK, 0) \
+	X(PVRSRV_ERROR_BRIDGE_EFAULT, EFAULT) \
+	X(PVRSRV_ERROR_BRIDGE_EINVAL, EINVAL) \
+	X(PVRSRV_ERROR_BRIDGE_ENOMEM, ENOMEM) \
+	X(PVRSRV_ERROR_BRIDGE_ERANGE, ERANGE) \
+	X(PVRSRV_ERROR_BRIDGE_EPERM, EPERM) \
+	X(PVRSRV_ERROR_BRIDGE_ENOTTY, ENOTTY) \
+	X(PVRSRV_ERROR_BRIDGE_CALL_FAILED, ENOTTY) \
+	X(PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL, ERANGE) \
+	X(PVRSRV_ERROR_OUT_OF_MEMORY, ENOMEM) \
+	X(PVRSRV_ERROR_PMR_NOT_PERMITTED, EACCES) \
+	X(PVRSRV_ERROR_INVALID_PARAMS, EINVAL) \
+	X(PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING, EPERM) \
+	X(PVRSRV_ERROR_NOT_IMPLEMENTED, ENOSYS) \
+	X(PVRSRV_ERROR_BAD_MAPPING, EINVAL)
+
+/* return -ve versions of POSIX errors as they are used in this form */
+int PVRSRVToNativeError(PVRSRV_ERROR eError)
+{
+	switch (eError)
+	{
+#define X(_PVRSRV_ERROR, _OS_ERROR) \
+	case (_PVRSRV_ERROR): return -(_OS_ERROR);
+
+		PVRSRV_ERROR_TO_OS_ERROR
+
+#undef X
+		default:
+			return -EFAULT;
+	}
+}
+
+typedef struct  _MISR_DATA_ {
+	struct workqueue_struct *psWorkQueue;
+	struct work_struct sMISRWork;
+	const IMG_CHAR* pszName;
+	PFN_MISR pfnMISR;
+	void *hData;
+} MISR_DATA;
+
+/*
+	MISRWrapper
+*/
+static void MISRWrapper(struct work_struct *data)
+{
+	MISR_DATA *psMISRData = container_of(data, MISR_DATA, sMISRWork);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Waking up '%s' MISR %p", psMISRData->pszName, psMISRData));
+
+	psMISRData->pfnMISR(psMISRData->hData);
+}
+
+/*
+	OSInstallMISR
+*/
+PVRSRV_ERROR OSInstallMISR(IMG_HANDLE *hMISRData, PFN_MISR pfnMISR,
+							void *hData, const IMG_CHAR *pszMisrName)
+{
+	MISR_DATA *psMISRData;
+
+	psMISRData = OSAllocMem(sizeof(*psMISRData));
+	PVR_LOG_RETURN_IF_NOMEM(psMISRData, "psMISRData");
+
+	psMISRData->hData = hData;
+	psMISRData->pfnMISR = pfnMISR;
+	psMISRData->pszName = pszMisrName;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Installing MISR with cookie %p", psMISRData));
+
+	psMISRData->psWorkQueue = create_singlethread_workqueue("pvr_misr");
+
+	if (psMISRData->psWorkQueue == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSInstallMISR: create_singlethreaded_workqueue failed"));
+		OSFreeMem(psMISRData);
+		return PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD;
+	}
+
+	INIT_WORK(&psMISRData->sMISRWork, MISRWrapper);
+
+	*hMISRData = (IMG_HANDLE) psMISRData;
+
+	return PVRSRV_OK;
+}
+
+/*
+	OSUninstallMISR
+*/
+PVRSRV_ERROR OSUninstallMISR(IMG_HANDLE hMISRData)
+{
+	MISR_DATA *psMISRData = (MISR_DATA *) hMISRData;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Uninstalling MISR with cookie %p", psMISRData));
+
+	destroy_workqueue(psMISRData->psWorkQueue);
+	OSFreeMem(psMISRData);
+
+	return PVRSRV_OK;
+}
+
+/*
+	OSScheduleMISR
+*/
+PVRSRV_ERROR OSScheduleMISR(IMG_HANDLE hMISRData)
+{
+	MISR_DATA *psMISRData = (MISR_DATA *) hMISRData;
+
+	/*
+		Note:
+
+		In the case of NO_HARDWARE we want the driver to be synchronous so
+		that we don't have to worry about waiting for previous operations
+		to complete
+	*/
+#if defined(NO_HARDWARE)
+	psMISRData->pfnMISR(psMISRData->hData);
+	return PVRSRV_OK;
+#else
+	{
+		bool rc = queue_work(psMISRData->psWorkQueue, &psMISRData->sMISRWork);
+		return rc ? PVRSRV_OK : PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+#endif
+}
+
+void OSSyncIRQ(IMG_UINT32 ui32IRQ)
+{
+	synchronize_irq(ui32IRQ);
+}
+
+/* OS specific values for thread priority */
+static const IMG_INT32 ai32OSPriorityValues[OS_THREAD_LAST_PRIORITY] =
+{
+	  0, /* OS_THREAD_NOSET_PRIORITY */
+	-20, /* OS_THREAD_HIGHEST_PRIORITY */
+	-10, /* OS_THREAD_HIGH_PRIORITY */
+	  0, /* OS_THREAD_NORMAL_PRIORITY */
+	  9, /* OS_THREAD_LOW_PRIORITY */
+	 19, /* OS_THREAD_LOWEST_PRIORITY */
+};
+
+static int OSThreadRun(void *data)
+{
+	OSThreadData *psOSThreadData = data;
+
+	/* count freezable threads */
+	LinuxBridgeNumActiveKernelThreadsIncrement();
+
+	/* Returns true if the thread was frozen, should we do anything with this
+	 * information? What do we return? Which one is the error case? */
+	set_freezable();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Starting Thread '%s'...", psOSThreadData->pszThreadName));
+
+	/* Call the client's kernel thread with the client's data pointer */
+	psOSThreadData->pfnThread(psOSThreadData->hData);
+
+	if (psOSThreadData->bIsSupportingThread)
+	{
+		_ThreadSetStopped(psOSThreadData);
+	}
+
+	/* Wait for OSThreadDestroy() to call kthread_stop() */
+	while (!kthread_freezable_should_stop(NULL))
+	{
+		schedule();
+	}
+
+	LinuxBridgeNumActiveKernelThreadsDecrement();
+
+	return 0;
+}
+
+PVRSRV_ERROR OSThreadCreate(IMG_HANDLE *phThread,
+                            IMG_CHAR *pszThreadName,
+                            PFN_THREAD pfnThread,
+                            PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                            IMG_BOOL bIsSupportingThread,
+                            void *hData)
+{
+	return OSThreadCreatePriority(phThread, pszThreadName, pfnThread,
+	                              pfnDebugDumpCB, bIsSupportingThread, hData,
+	                              OS_THREAD_NOSET_PRIORITY);
+}
+
+PVRSRV_ERROR OSThreadCreatePriority(IMG_HANDLE *phThread,
+                                    IMG_CHAR *pszThreadName,
+                                    PFN_THREAD pfnThread,
+                                    PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                                    IMG_BOOL bIsSupportingThread,
+                                    void *hData,
+                                    OS_THREAD_LEVEL eThreadPriority)
+{
+	OSThreadData *psOSThreadData;
+	PVRSRV_ERROR eError;
+
+	psOSThreadData = OSAllocZMem(sizeof(*psOSThreadData));
+	PVR_LOG_GOTO_IF_NOMEM(psOSThreadData, eError, fail_alloc);
+
+	psOSThreadData->pfnThread = pfnThread;
+	psOSThreadData->hData = hData;
+	psOSThreadData->pszThreadName = pszThreadName;
+	psOSThreadData->kthread = kthread_run(OSThreadRun, psOSThreadData, "%s", pszThreadName);
+
+	if (IS_ERR(psOSThreadData->kthread))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_kthread;
+	}
+
+	if (bIsSupportingThread)
+	{
+		psOSThreadData->pfnDebugDumpCB = pfnDebugDumpCB;
+		psOSThreadData->bIsThreadRunning = IMG_TRUE;
+		psOSThreadData->bIsSupportingThread = IMG_TRUE;
+
+		_ThreadListAddEntry(psOSThreadData);
+	}
+
+	if (eThreadPriority != OS_THREAD_NOSET_PRIORITY &&
+	    eThreadPriority < OS_THREAD_LAST_PRIORITY)
+	{
+		set_user_nice(psOSThreadData->kthread,
+		              ai32OSPriorityValues[eThreadPriority]);
+	}
+
+	*phThread = psOSThreadData;
+
+	return PVRSRV_OK;
+
+fail_kthread:
+	OSFreeMem(psOSThreadData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR OSThreadDestroy(IMG_HANDLE hThread)
+{
+	OSThreadData *psOSThreadData = hThread;
+	int ret;
+
+	/* Let the thread know we are ready for it to end and wait for it. */
+	ret = kthread_stop(psOSThreadData->kthread);
+	if (0 != ret)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "kthread_stop failed(%d)", ret));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	if (psOSThreadData->bIsSupportingThread)
+	{
+		_ThreadListRemoveEntry(psOSThreadData);
+	}
+
+	OSFreeMem(psOSThreadData);
+
+	return PVRSRV_OK;
+}
+
+void OSPanic(void)
+{
+	BUG();
+
+#if defined(__KLOCWORK__)
+	/* Klocwork does not understand that BUG is terminal... */
+	abort();
+#endif
+}
+
+IMG_BOOL OSIsMapPhysNonContigSupported(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) && !defined(CONFIG_VMAP_PFN)
+	return IMG_FALSE;
+#else
+	return IMG_TRUE;
+#endif
+}
+
+void OSUnMapPhysArrayToLin(void *pvLinAddr, void *pvPrivData)
+{
+	if (is_vmalloc_addr(pvLinAddr))
+	{
+#if defined(CONFIG_VMAP_PFN)
+		PVR_UNREFERENCED_PARAMETER(pvPrivData);
+		vunmap(pvLinAddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+		unmap_kernel_range((unsigned long) (uintptr_t) pvLinAddr,
+                           get_vm_area_size(pvPrivData));
+		free_vm_area(pvPrivData);
+#else
+		PVR_DPF((PVR_DBG_ERROR,"%s: Cannot map into kernel, no method supported.", __func__));
+		PVR_ASSERT(0);
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Given kernel address is not a vmalloc addr", __func__));
+	}
+}
+
+#define PagePAToPFN(PageAddr) (PageAddr >> PAGE_SHIFT)
+
+PVRSRV_ERROR OSMapPhysArrayToLin(IMG_CPU_PHYADDR pPagePA[],
+                                 IMG_UINT32 uiPagesCount,
+                                 void **ppvLinAddr,
+                                 void **ppvPrivData)
+{
+	if (ppvLinAddr == NULL || ppvPrivData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if defined(CONFIG_VMAP_PFN)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < uiPagesCount; i++)
+		{
+			pPagePA[i].uiAddr = PagePAToPFN(pPagePA[i].uiAddr);
+		}
+
+		*ppvLinAddr = vmap_pfn((unsigned long *)pPagePA,
+							   (unsigned int)uiPagesCount,
+							   pgprot_device(PAGE_KERNEL));
+		if (NULL == *ppvLinAddr)
+		{
+			return PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+		}
+		*ppvPrivData = NULL;
+		return PVRSRV_OK;
+	}
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	{
+		pte_t *pte[32], **pte_array;
+		struct vm_struct *psVMA;
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_UINT32 i = 0;
+
+		pte_array = &pte[0];
+		if (sizeof(pte) < (sizeof(pte[0]) * uiPagesCount))
+		{
+			pte_array = kzalloc(uiPagesCount * sizeof(*pte_array), GFP_KERNEL);
+			if (NULL == pte_array)
+			{
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+		}
+
+		psVMA = alloc_vm_area((size_t)(uiPagesCount << PAGE_SHIFT), pte_array);
+		if (NULL == psVMA)
+		{
+			eError = PVRSRV_ERROR_FAILED_TO_ALLOC_VIRT_MEMORY;
+			goto FreePTEArray;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Allocated VM: %s VMA: %p Addr: %p Size: 0x%lx count: %d", __func__,
+				psVMA, psVMA->addr, psVMA->size, psVMA->nr_pages));
+
+		for (i = 0; i < uiPagesCount; i++)
+		{
+			*(pte_array[i]) = pte_mkspecial(pfn_pte((unsigned long) PagePAToPFN(pPagePA[i].uiAddr),
+													pgprot_device(PAGE_KERNEL)));
+		}
+
+		OSWriteMemoryBarrier(psVMA->addr);
+
+		*ppvLinAddr = psVMA->addr;
+		*ppvPrivData = psVMA;
+
+FreePTEArray:
+		if (pte_array != pte)
+		{
+			kfree(pte_array);
+		}
+
+		return eError;
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR,"%s: Cannot map into kernel, no method supported.", __func__));
+	PVR_ASSERT(0);
+	*ppvLinAddr = NULL;
+	return PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+#endif
+}
+
+void *
+OSMapPhysToLin(IMG_CPU_PHYADDR BasePAddr,
+			   size_t ui32Bytes,
+			   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags)
+{
+	void __iomem *pvLinAddr;
+
+	if (uiMappingFlags & ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Found non-cpu cache mode flag when mapping to the cpu"));
+		return NULL;
+	}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/*
+	  This is required to support DMA physheaps for GPU virtualization.
+	  Unfortunately, if a region of kernel managed memory is turned into
+	  a DMA buffer, conflicting mappings can come about easily on Linux
+	  as the original memory is mapped by the kernel as normal cached
+	  memory whilst DMA buffers are mapped mostly as uncached device or
+	  cache-coherent device memory. In both cases the system will have
+	  two conflicting mappings for the same memory region and will have
+	  "undefined behaviour" for most processors notably ARMv6 onwards
+	  and some x86 micro-architectures. As a result, perform ioremapping
+	  manually for DMA physheap allocations by translating from CPU/VA
+	  to BUS/PA thereby preventing the creation of conflicting mappings.
+	*/
+	pvLinAddr = (void __iomem *) SysDmaDevPAddrToCpuVAddr(BasePAddr.uiAddr, ui32Bytes);
+	if (pvLinAddr != NULL)
+	{
+		return (void __force *) pvLinAddr;
+	}
+#endif
+
+	switch (uiMappingFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+#if defined(CONFIG_X86) || defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+			pvLinAddr = (void __iomem *)ioremap_wc(BasePAddr.uiAddr, ui32Bytes);
+#else
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+#endif
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+#if defined(CONFIG_X86) || defined(CONFIG_ARM)
+			pvLinAddr = (void __iomem *)ioremap_cache(BasePAddr.uiAddr, ui32Bytes);
+#else
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+#endif
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT:
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT:
+			PVR_DPF((PVR_DBG_ERROR, "Unexpected cpu cache mode"));
+			pvLinAddr = NULL;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Unsupported cpu cache mode"));
+			pvLinAddr = NULL;
+			break;
+	}
+
+	return (void __force *) pvLinAddr;
+}
+
+
+IMG_BOOL
+OSUnMapPhysToLin(void *pvLinAddr, size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32Bytes);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (SysDmaCpuVAddrToDevPAddr(pvLinAddr))
+	{
+		return IMG_TRUE;
+	}
+#endif
+
+	iounmap((void __iomem *) pvLinAddr);
+
+	return IMG_TRUE;
+}
+
+#define OS_MAX_TIMERS	8
+
+/* Timer callback structure used by OSAddTimer */
+typedef struct TIMER_CALLBACK_DATA_TAG
+{
+	IMG_BOOL			bInUse;
+	PFN_TIMER_FUNC		pfnTimerFunc;
+	void				*pvData;
+	struct timer_list	sTimer;
+	IMG_UINT32			ui32Delay;
+	IMG_BOOL			bActive;
+	struct work_struct	sWork;
+}TIMER_CALLBACK_DATA;
+
+static struct workqueue_struct *psTimerWorkQueue = NULL;
+
+static TIMER_CALLBACK_DATA sTimers[OS_MAX_TIMERS];
+
+static DEFINE_MUTEX(sTimerStructLock);
+
+static void OSTimerCallbackBody(TIMER_CALLBACK_DATA *psTimerCBData)
+{
+	if (!psTimerCBData->bActive)
+		return;
+
+	/* call timer callback */
+	psTimerCBData->pfnTimerFunc(psTimerCBData->pvData);
+
+	/* reset timer */
+	mod_timer(&psTimerCBData->sTimer, psTimerCBData->sTimer.expires + psTimerCBData->ui32Delay);
+}
+
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+/*************************************************************************/ /*!
+@Function       OSTimerCallbackWrapper
+@Description    OS specific timer callback wrapper function
+@Input          psTimer    Timer list structure
+*/ /**************************************************************************/
+static void OSTimerCallbackWrapper(struct timer_list *psTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = from_timer(psTimerCBData, psTimer, sTimer);
+#else
+/*************************************************************************/ /*!
+@Function       OSTimerCallbackWrapper
+@Description    OS specific timer callback wrapper function
+@Input          uData    Timer callback data
+*/ /**************************************************************************/
+static void OSTimerCallbackWrapper(uintptr_t uData)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = (TIMER_CALLBACK_DATA*)uData;
+#endif
+	int res;
+
+	res = queue_work(psTimerWorkQueue, &psTimerCBData->sWork);
+	if (res == 0)
+	{
+		PVR_LOG(("OSTimerCallbackWrapper: work already queued"));
+	}
+}
+
+
+static void OSTimerWorkQueueCallBack(struct work_struct *psWork)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = container_of(psWork, TIMER_CALLBACK_DATA, sWork);
+
+	OSTimerCallbackBody(psTimerCBData);
+}
+
+IMG_HANDLE OSAddTimer(PFN_TIMER_FUNC pfnTimerFunc, void *pvData, IMG_UINT32 ui32MsTimeout)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData;
+	IMG_UINT32		ui32i;
+
+	/* check callback */
+	if (!pfnTimerFunc)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAddTimer: passed invalid callback"));
+		return NULL;
+	}
+
+	/* Allocate timer callback data structure */
+	mutex_lock(&sTimerStructLock);
+	for (ui32i = 0; ui32i < OS_MAX_TIMERS; ui32i++)
+	{
+		psTimerCBData = &sTimers[ui32i];
+		if (!psTimerCBData->bInUse)
+		{
+			psTimerCBData->bInUse = IMG_TRUE;
+			break;
+		}
+	}
+	mutex_unlock(&sTimerStructLock);
+	if (ui32i >= OS_MAX_TIMERS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAddTimer: all timers are in use"));
+		return NULL;
+	}
+
+	psTimerCBData->pfnTimerFunc = pfnTimerFunc;
+	psTimerCBData->pvData = pvData;
+	psTimerCBData->bActive = IMG_FALSE;
+
+	/*
+		HZ = ticks per second
+		ui32MsTimeout = required ms delay
+		ticks = (Hz * ui32MsTimeout) / 1000
+	*/
+	psTimerCBData->ui32Delay = ((HZ * ui32MsTimeout) < 1000)
+								?	1
+								:	((HZ * ui32MsTimeout) / 1000);
+
+	/* initialise object */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+	timer_setup(&psTimerCBData->sTimer, OSTimerCallbackWrapper, 0);
+#else
+	init_timer(&psTimerCBData->sTimer);
+
+	/* setup timer object */
+	psTimerCBData->sTimer.function = (void *)OSTimerCallbackWrapper;
+	psTimerCBData->sTimer.data = (uintptr_t)psTimerCBData;
+#endif
+
+	return (IMG_HANDLE)(uintptr_t)(ui32i + 1);
+}
+
+
+static inline TIMER_CALLBACK_DATA *GetTimerStructure(IMG_HANDLE hTimer)
+{
+	IMG_UINT32 ui32i = (IMG_UINT32)((uintptr_t)hTimer) - 1;
+
+	PVR_ASSERT(ui32i < OS_MAX_TIMERS);
+
+	return &sTimers[ui32i];
+}
+
+PVRSRV_ERROR OSRemoveTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(!psTimerCBData->bActive);
+
+	/* free timer callback data struct */
+	psTimerCBData->bInUse = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSEnableTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(!psTimerCBData->bActive);
+
+	/* Start timer arming */
+	psTimerCBData->bActive = IMG_TRUE;
+
+	/* set the expire time */
+	psTimerCBData->sTimer.expires = psTimerCBData->ui32Delay + jiffies;
+
+	/* Add the timer to the list */
+	add_timer(&psTimerCBData->sTimer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR OSDisableTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(psTimerCBData->bActive);
+
+	/* Stop timer from arming */
+	psTimerCBData->bActive = IMG_FALSE;
+	smp_mb();
+
+	flush_workqueue(psTimerWorkQueue);
+
+	/* remove timer */
+	del_timer_sync(&psTimerCBData->sTimer);
+
+	/*
+	 * This second flush is to catch the case where the timer ran
+	 * before we managed to delete it, in which case, it will have
+	 * queued more work for the workqueue. Since the bActive flag
+	 * has been cleared, this second flush won't result in the
+	 * timer being rearmed.
+	 */
+	flush_workqueue(psTimerWorkQueue);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR OSEventObjectCreate(const IMG_CHAR *pszName, IMG_HANDLE *hEventObject)
+{
+	PVR_UNREFERENCED_PARAMETER(pszName);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectListCreate(hEventObject);
+}
+
+
+PVRSRV_ERROR OSEventObjectDestroy(IMG_HANDLE hEventObject)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectListDestroy(hEventObject);
+}
+
+PVRSRV_ERROR OSEventObjectWaitTimeout(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus)
+{
+	PVRSRV_ERROR eError;
+
+	if (hOSEventKM && uiTimeoutus > 0)
+	{
+		eError = LinuxEventObjectWait(hOSEventKM, uiTimeoutus, _NON_FREEZABLE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSEventObjectWait: invalid arguments %p, %lld", hOSEventKM, uiTimeoutus));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR OSEventObjectWait(IMG_HANDLE hOSEventKM)
+{
+	return OSEventObjectWaitTimeout(hOSEventKM, EVENT_OBJECT_TIMEOUT_US);
+}
+
+PVRSRV_ERROR OSEventObjectWaitKernel(IMG_HANDLE hOSEventKM,
+                                     IMG_UINT64 uiTimeoutus)
+{
+	PVRSRV_ERROR eError;
+
+	if (hOSEventKM != NULL && uiTimeoutus > 0)
+	{
+		eError = LinuxEventObjectWait(hOSEventKM, uiTimeoutus, _FREEZABLE);
+	}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	else if (hOSEventKM != NULL)
+	{
+		eError = LinuxEventObjectWaitUntilSignalled(hOSEventKM);
+	}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSEventObjectWaitKernel: invalid arguments %p",
+		        hOSEventKM));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+
+void OSEventObjectDumpDebugInfo(IMG_HANDLE hOSEventKM)
+{
+	LinuxEventObjectDumpDebugInfo(hOSEventKM);
+}
+
+PVRSRV_ERROR OSEventObjectOpen(IMG_HANDLE hEventObject, IMG_HANDLE *phOSEvent)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phOSEvent, "phOSEvent");
+	PVR_LOG_GOTO_IF_INVALID_PARAM(hEventObject, eError, error);
+
+	eError = LinuxEventObjectAdd(hEventObject, phOSEvent);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LinuxEventObjectAdd", error);
+
+	return PVRSRV_OK;
+
+error:
+	*phOSEvent = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR OSEventObjectClose(IMG_HANDLE hOSEventKM)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hOSEventKM, "hOSEventKM");
+
+	return LinuxEventObjectDelete(hOSEventKM);
+}
+
+PVRSRV_ERROR OSEventObjectSignal(IMG_HANDLE hEventObject)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectSignal(hEventObject);
+}
+
+PVRSRV_ERROR OSCopyToUser(void *pvProcess,
+						  void __user *pvDest,
+						  const void *pvSrc,
+						  size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(pvProcess);
+
+	if (pvr_copy_to_user(pvDest, pvSrc, ui32Bytes)==0)
+		return PVRSRV_OK;
+	else
+		return PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY;
+}
+
+PVRSRV_ERROR OSCopyFromUser(void *pvProcess,
+							void *pvDest,
+							const void __user *pvSrc,
+							size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(pvProcess);
+
+	if (likely(pvr_copy_from_user(pvDest, pvSrc, ui32Bytes)==0))
+		return PVRSRV_OK;
+	else
+		return PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY;
+}
+
+IMG_UINT64 OSDivide64r64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder)
+{
+	*pui32Remainder = do_div(ui64Divident, ui32Divisor);
+
+	return ui64Divident;
+}
+
+IMG_UINT32 OSDivide64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder)
+{
+	*pui32Remainder = do_div(ui64Divident, ui32Divisor);
+
+	return (IMG_UINT32) ui64Divident;
+}
+
+/* One time osfunc initialisation */
+PVRSRV_ERROR PVROSFuncInit(void)
+{
+	{
+		PVR_ASSERT(!psTimerWorkQueue);
+
+		psTimerWorkQueue = create_freezable_workqueue("pvr_timer");
+		if (psTimerWorkQueue == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: couldn't create timer workqueue",
+					 __func__));
+			return PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD;
+		}
+	}
+
+	{
+		IMG_UINT32 ui32i;
+
+		for (ui32i = 0; ui32i < OS_MAX_TIMERS; ui32i++)
+		{
+			TIMER_CALLBACK_DATA *psTimerCBData = &sTimers[ui32i];
+
+			INIT_WORK(&psTimerCBData->sWork, OSTimerWorkQueueCallBack);
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*
+ * Osfunc deinitialisation.
+ * Note that PVROSFuncInit may not have been called
+ */
+void PVROSFuncDeInit(void)
+{
+	if (psTimerWorkQueue != NULL)
+	{
+		destroy_workqueue(psTimerWorkQueue);
+		psTimerWorkQueue = NULL;
+	}
+}
+
+void OSDumpStack(void)
+{
+	dump_stack();
+}
+
+/*************************************************************************/ /*!
+@Function       OSDebugSignalPID
+@Description    Sends a SIGTRAP signal to a specific PID in user mode for
+                debugging purposes. The user mode process can register a handler
+                against this signal.
+                This is necessary to support the Rogue debugger. If the Rogue
+                debugger is not used then this function may be implemented as
+                a stub.
+@Input          ui32PID    The PID for the signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDebugSignalPID(IMG_UINT32 ui32PID)
+{
+	int err;
+	struct pid *psPID;
+
+	psPID = find_vpid(ui32PID);
+	if (psPID == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get PID struct.", __func__));
+		return PVRSRV_ERROR_NOT_FOUND;
+	}
+
+	err = kill_pid(psPID, SIGTRAP, 0);
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Signal Failure %d", __func__, err));
+		return PVRSRV_ERROR_SIGNAL_FAILED;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSIsKernelThread
+@Description    This API determines if the current running thread is a kernel
+                thread (i.e. one not associated with any userland process,
+                typically an MISR handler.)
+@Return         IMG_TRUE if it is a kernel thread, otherwise IMG_FALSE.
+*/ /**************************************************************************/
+IMG_BOOL OSIsKernelThread(void)
+{
+	/*
+	 * Kernel threads have a NULL memory descriptor.
+	 *
+	 * See https://www.kernel.org/doc/Documentation/vm/active_mm.txt
+	 */
+	return current->mm == NULL;
+}
+
+void OSDumpVersionInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					   void *pvDumpDebugFile)
+{
+	PVR_DUMPDEBUG_LOG("OS kernel info: %s %s %s %s",
+					utsname()->sysname,
+					utsname()->release,
+					utsname()->version,
+					utsname()->machine);
+}
+#if defined(SUPPORT_DMA_TRANSFER)
+
+typedef struct _OS_CLEANUP_DATA_
+{
+	IMG_BOOL bSucceed;
+	IMG_BOOL bAdvanceTimeline;
+	IMG_UINT uiRefCount;
+	IMG_UINT uiNumDMA;
+	IMG_UINT uiCount;
+
+	struct dma_async_tx_descriptor** ppsDescriptors;
+
+
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PFN_SERVER_CLEANUP pfnServerCleanup;
+	void* pvServerCleanupData;
+
+	enum dma_data_direction eDirection;
+	struct sg_table **ppsSg;
+	struct page ***pages;
+	IMG_UINT32* puiNumPages;
+	spinlock_t spinlock;
+
+	struct completion start_cleanup;
+	struct completion *sync_completion;
+
+	/* Sparse PMR transfer information */
+	IMG_BOOL *pbIsSparse;
+	IMG_UINT *uiNumValidPages;
+	struct sg_table ***ppsSgSparse;
+	struct dma_async_tx_descriptor*** ppsDescriptorsSparse;
+
+} OS_CLEANUP_DATA;
+
+static int cleanup_thread(void *pvData)
+{
+	IMG_UINT32 i, j;
+	struct completion *sync_completion = NULL;
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvData;
+	IMG_BOOL bSucceed = psOSCleanup->bSucceed;
+
+	sync_completion = psOSCleanup->sync_completion;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Cleanup thread waiting (%p) on completion", pvData));
+#endif
+
+	wait_for_completion(&psOSCleanup->start_cleanup);
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Cleanup thread notified (%p)", pvData));
+#endif
+	/* Free resources */
+	for (i=0; i<psOSCleanup->uiCount; i++)
+	{
+		if (!psOSCleanup->pbIsSparse[i])
+		{
+			dma_sync_sg_for_cpu(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+								psOSCleanup->ppsSg[i]->sgl,
+								psOSCleanup->ppsSg[i]->nents,
+								psOSCleanup->eDirection);
+
+			dma_unmap_sg(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+						 psOSCleanup->ppsSg[i]->sgl,
+						 psOSCleanup->ppsSg[i]->nents,
+						 psOSCleanup->eDirection);
+
+			sg_free_table(psOSCleanup->ppsSg[i]);
+
+			OSFreeMem(psOSCleanup->ppsSg[i]);
+
+			/* Unpin pages */
+			for (j=0; j<psOSCleanup->puiNumPages[i]; j++)
+			{
+				/*
+				 * using DMA_FROM_DEVICE from enum dma_data_direction instead of DMA_DEV_TO_MEM
+				 * from enum dma_transfer_direction to avoid casting explicitly
+				 */
+				if (psOSCleanup->eDirection == DMA_FROM_DEVICE)
+				{
+					set_page_dirty_lock(psOSCleanup->pages[i][j]);
+				}
+
+				pvr_unpin_user_page_for_dma(psOSCleanup->pages[i][j]);
+			}
+		}
+		else
+		{
+			for (j = 0; j < psOSCleanup->puiNumPages[i]; j++)
+			{
+				if (psOSCleanup->ppsSgSparse[i][j]) {
+					dma_sync_sg_for_cpu(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+										psOSCleanup->ppsSgSparse[i][j]->sgl,
+										psOSCleanup->ppsSgSparse[i][j]->nents,
+										psOSCleanup->eDirection);
+
+
+					dma_unmap_sg(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+								psOSCleanup->ppsSgSparse[i][j]->sgl,
+								psOSCleanup->ppsSgSparse[i][j]->nents,
+								psOSCleanup->eDirection);
+
+					sg_free_table(psOSCleanup->ppsSgSparse[i][j]);
+
+					OSFreeMem(psOSCleanup->ppsSgSparse[i][j]);
+
+				}
+			}
+
+			OSFreeMem(psOSCleanup->ppsSgSparse[i]);
+			OSFreeMem(psOSCleanup->ppsDescriptorsSparse[i]);
+
+			/* Unpin pages */
+			for (j=0; j<psOSCleanup->puiNumPages[i]*2; j++)
+			{
+				/*
+				 * Some pages might've been pinned twice
+				 * Others may have not been pinned at all
+				 */
+				if (psOSCleanup->pages[i][j])
+				{
+					/*
+					 * using DMA_FROM_DEVICE from enum dma_data_direction instead of DMA_DEV_TO_MEM
+					 * from enum dma_transfer_direction to avoid casting explicitly
+					 */
+					if (psOSCleanup->eDirection == DMA_FROM_DEVICE)
+					{
+						set_page_dirty_lock(psOSCleanup->pages[i][j]);
+					}
+
+					pvr_unpin_user_page_for_dma(psOSCleanup->pages[i][j]);
+				}
+			}
+		}
+
+		OSFreeMem(psOSCleanup->pages[i]);
+	}
+
+	psOSCleanup->pfnServerCleanup(psOSCleanup->pvServerCleanupData,
+								  psOSCleanup->bAdvanceTimeline);
+
+	OSFreeMem(psOSCleanup->ppsSg);
+	OSFreeMem(psOSCleanup->pages);
+	OSFreeMem(psOSCleanup->puiNumPages);
+	OSFreeMem(psOSCleanup->ppsSgSparse);
+	OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+	OSFreeMem(psOSCleanup->ppsDescriptors);
+	OSFreeMem(psOSCleanup->pbIsSparse);
+	OSFreeMem(psOSCleanup->uiNumValidPages);
+	OSFreeMem(psOSCleanup);
+
+	if (bSucceed)
+	{
+		kthread_complete_and_exit(sync_completion, 0);
+	}
+	else
+	{
+		kthread_complete_and_exit(NULL, 0);
+	}
+	return 0;
+}
+
+static void dma_callback(void *pvOSCleanup)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvOSCleanup;
+	unsigned long flags;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "dma_callback (%p) refcount decreased to %d", psOSCleanup, psOSCleanup->uiRefCount - 1));
+#endif
+	spin_lock_irqsave(&psOSCleanup->spinlock, flags);
+
+	psOSCleanup->uiRefCount--;
+
+	if (psOSCleanup->uiRefCount==0)
+	{
+		/* Notify the cleanup thread */
+		spin_unlock_irqrestore(&psOSCleanup->spinlock, flags);
+		complete(&psOSCleanup->start_cleanup);
+		return;
+	}
+
+	spin_unlock_irqrestore(&psOSCleanup->spinlock, flags);
+}
+
+#if defined(PVRSRV_DEBUG_DMA)
+static void
+DMADumpPhysicalAddresses(struct page **ppsHostMemPages,
+						 IMG_UINT32 uiNumPages,
+						 IMG_DMA_ADDR *sDmaAddr,
+						 IMG_UINT64 ui64Offset)
+{
+	IMG_CPU_PHYADDR sPagePhysAddr;
+	IMG_UINT32 uiIdx;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "DMA Transfer Address Dump"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Hostmem phys addresses:"));
+
+	for (uiIdx = 0; uiIdx < uiNumPages; uiIdx++)
+	{
+		sPagePhysAddr.uiAddr = page_to_phys(ppsHostMemPages[uiIdx]);
+		if (uiIdx == 0)
+		{
+			sPagePhysAddr.uiAddr += ui64Offset;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "\tHost mem start at 0x%" IMG_UINT64_FMTSPECx,
+					 (IMG_UINT64)sPagePhysAddr.uiAddr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "\tHost Mem Page %d at 0x%" IMG_UINT64_FMTSPECx,
+					 uiIdx, (IMG_UINT64)sPagePhysAddr.uiAddr));
+		}
+	}
+	PVR_DPF((PVR_DBG_MESSAGE,
+			 "Devmem CPU phys address: 0x%" IMG_UINT64_FMTSPECx,
+			 (IMG_UINT64)sDmaAddr->uiAddr));
+}
+#endif
+
+PVRSRV_ERROR OSDmaSubmitTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *pvOSData,
+				    void *pvChan, IMG_BOOL bSynchronous)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvOSData;
+	struct completion* sync_completion = NULL;
+
+	psOSCleanup->bSucceed = IMG_TRUE;
+	psOSCleanup->bAdvanceTimeline = IMG_TRUE;
+
+	if (bSynchronous)
+	{
+		sync_completion = OSAllocZMem(sizeof(struct completion));
+		init_completion(sync_completion);
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	/* Wait only on number of ops scheduled. This might be different to NumDMAs
+	in certain error conditions */
+	psOSCleanup->uiRefCount = psOSCleanup->uiCount;
+	psOSCleanup->sync_completion = sync_completion;
+
+	{
+		IMG_UINT32 i,j;
+		for (i=0; i<psOSCleanup->uiCount; i++)
+		{
+			if (psOSCleanup->pbIsSparse[i])
+			{
+				for (j=0; j<psOSCleanup->puiNumPages[i]; j++)
+				{
+					if (psOSCleanup->ppsDescriptorsSparse[i][j])
+						dmaengine_submit(psOSCleanup->ppsDescriptorsSparse[i][j]);
+				}
+			}
+			else
+			{
+				dmaengine_submit(psOSCleanup->ppsDescriptors[i]);
+			}
+		}
+	}
+
+	dma_async_issue_pending(pvChan);
+
+	if (bSynchronous)
+	{
+		wait_for_completion(sync_completion);
+		OSFreeMem(sync_completion);
+	}
+
+	return PVRSRV_OK;
+}
+
+void OSDmaForceCleanup(PVRSRV_DEVICE_NODE *psDevNode, void *pvChan,
+					   void *pvOSData, void *pvServerCleanupParam,
+					   PFN_SERVER_CLEANUP pfnServerCleanup)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA *)pvOSData;
+	IMG_UINT ui32Retries;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	psOSCleanup->bSucceed = IMG_FALSE;
+	psOSCleanup->bAdvanceTimeline = IMG_TRUE;
+
+	/* Need to wait for outstanding DMA Engine ops before advancing the
+	   user-supplied timeline in case of error. dmaengine_terminate_sync
+	   cannot be called from within atomic context, so cannot invoke it
+	   from inside the cleanup kernel thread. */
+	for (ui32Retries = 0; ui32Retries < DMA_ERROR_SYNC_RETRIES; ui32Retries++)
+	{
+		if (dmaengine_terminate_sync(pvChan) == 0)
+		{
+			break;
+		}
+	}
+	if (ui32Retries == DMA_ERROR_SYNC_RETRIES)
+	{
+		/* We cannot guarantee all outstanding DMAs were terminated
+		 * so we let the UM fence time out as a fallback mechanism */
+		psOSCleanup->bAdvanceTimeline = IMG_FALSE;
+	}
+
+	if (psOSCleanup->uiCount > 0)
+	{
+		complete(&psOSCleanup->start_cleanup);
+	}
+	else
+	{
+		/* Cleanup kthread never run, need to manually wind down */
+		pfnServerCleanup(pvServerCleanupParam, psOSCleanup->bAdvanceTimeline);
+
+		OSFreeMem(psOSCleanup->ppsSg);
+		OSFreeMem(psOSCleanup->pages);
+		OSFreeMem(psOSCleanup->puiNumPages);
+		OSFreeMem(psOSCleanup->ppsSgSparse);
+		OSFreeMem(psOSCleanup->pbIsSparse);
+		OSFreeMem(psOSCleanup->uiNumValidPages);
+		OSFreeMem(psOSCleanup->ppsDescriptors);
+		OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+
+		OSFreeMem(psOSCleanup);
+	}
+}
+
+PVRSRV_ERROR OSDmaAllocData(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 uiNumDMA, void **pvOutData)
+{
+	PVRSRV_ERROR eError;
+	OS_CLEANUP_DATA *psOSCleanup = OSAllocZMem(sizeof(OS_CLEANUP_DATA));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup, eError, e0);
+
+	psOSCleanup->uiNumDMA = uiNumDMA;
+	psOSCleanup->psDevNode = psDevNode;
+
+	spin_lock_init(&psOSCleanup->spinlock);
+
+	init_completion(&psOSCleanup->start_cleanup);
+
+	psOSCleanup->ppsDescriptors = OSAllocZMem(uiNumDMA * sizeof(struct dma_async_tx_descriptor*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsDescriptors, eError, e0);
+
+	psOSCleanup->ppsDescriptorsSparse = OSAllocZMem(uiNumDMA * sizeof(struct dma_async_tx_descriptor*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsDescriptorsSparse, eError, e11);
+
+	psOSCleanup->ppsSg = OSAllocZMem(uiNumDMA * sizeof(struct sg_table*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsSg, eError, e1);
+
+	psOSCleanup->ppsSgSparse = OSAllocZMem(uiNumDMA * sizeof(struct sg_table*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsSgSparse, eError, e12);
+
+	psOSCleanup->pbIsSparse = OSAllocZMem(uiNumDMA * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->pbIsSparse, eError, e13);
+
+	psOSCleanup->uiNumValidPages = OSAllocZMem(uiNumDMA * sizeof(IMG_UINT));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->uiNumValidPages, eError, e14);
+
+	psOSCleanup->pages = OSAllocZMem(uiNumDMA * sizeof(struct page **));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->pages, eError, e2);
+
+	psOSCleanup->puiNumPages = OSAllocZMem(uiNumDMA * sizeof(IMG_UINT32));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->puiNumPages, eError, e3);
+
+	*pvOutData = psOSCleanup;
+
+	return PVRSRV_OK;
+
+e3:
+	OSFreeMem(psOSCleanup->pages);
+e2:
+	OSFreeMem(psOSCleanup->uiNumValidPages);
+e14:
+	OSFreeMem(psOSCleanup->pbIsSparse);
+e13:
+	OSFreeMem(psOSCleanup->ppsSgSparse);
+e12:
+	OSFreeMem(psOSCleanup->ppsSg);
+e1:
+	OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+e11:
+	OSFreeMem(psOSCleanup->ppsDescriptors);
+e0:
+	OSFreeMem(psOSCleanup);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       OSDmaTransfer
+@Description    This API is used to ask OS to perform a DMA transfer operation
+@Return
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDmaPrepareTransfer(PVRSRV_DEVICE_NODE *psDevNode,
+							   void* pvChan,
+							   IMG_DMA_ADDR* psDmaAddr, IMG_UINT64* puiAddress,
+							   IMG_UINT64 uiSize, IMG_BOOL bMemToDev,
+							   void* pvOSData,
+							   void* pvServerCleanupParam, PFN_SERVER_CLEANUP pfnServerCleanup, IMG_BOOL bFirst)
+{
+
+	IMG_INT iRet;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+	OS_CLEANUP_DATA* psOSCleanupData = pvOSData;
+
+	enum dma_data_direction eDataDirection = bMemToDev ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct dma_slave_config sConfig = {0};
+	struct dma_async_tx_descriptor *psDesc;
+
+	unsigned long offset = (unsigned long)puiAddress & ((1 << PAGE_SHIFT) - 1);
+	unsigned int num_pages = (uiSize + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	int num_pinned_pages = 0;
+
+	struct sg_table *psSg = OSAllocZMem(sizeof(struct sg_table));
+	PVR_LOG_GOTO_IF_NOMEM(psSg, eError, e0);
+
+	psOSCleanupData->pages[psOSCleanupData->uiCount] = OSAllocZMem(num_pages * sizeof(struct page *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->pages[psOSCleanupData->uiCount], eError, e1);
+
+	num_pinned_pages = pvr_pin_user_pages_for_dma(
+		puiAddress,
+		num_pages,
+		!bMemToDev,
+		psOSCleanupData->pages[psOSCleanupData->uiCount]);
+
+	psOSCleanupData->puiNumPages[psOSCleanupData->uiCount] = num_pinned_pages;
+	if (num_pinned_pages != num_pages)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "get_user_pages_fast failed: (%d - %u)", num_pinned_pages, num_pages));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e2; /* Unpin what was pinned and return error */
+	}
+
+#if defined(PVRSRV_DEBUG_DMA)
+	DMADumpPhysicalAddresses(psOSCleanupData->pages[psOSCleanupData->uiCount],
+							 num_pages, psDmaAddr, offset);
+#endif
+
+	if (sg_alloc_table_from_pages(psSg, psOSCleanupData->pages[psOSCleanupData->uiCount], num_pages, offset, uiSize, GFP_KERNEL) != 0)
+	{
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table_from_pages failed"));
+		goto e2;
+	}
+
+	if (bMemToDev)
+	{
+		sConfig.direction = DMA_MEM_TO_DEV;
+		sConfig.src_addr = 0;
+		sConfig.dst_addr = psDmaAddr->uiAddr;
+	}
+	else
+	{
+		sConfig.direction = DMA_DEV_TO_MEM;
+		sConfig.src_addr = psDmaAddr->uiAddr;
+		sConfig.dst_addr = 0;
+	}
+	dmaengine_slave_config(pvChan, &sConfig);
+
+	iRet = dma_map_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+	if (!iRet)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error mapping SG list", __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e3;
+	}
+
+	dma_sync_sg_for_device(psDevConfig->pvOSDevice, psSg->sgl,(unsigned int)iRet, eDataDirection);
+
+	psDesc = dmaengine_prep_slave_sg(pvChan, psSg->sgl, (unsigned int)iRet, sConfig.direction, 0);
+	if (!psDesc)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: dmaengine_prep_slave_sg failed", __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e4;
+	}
+
+	psOSCleanupData->eDirection = eDataDirection;
+	psOSCleanupData->ppsSg[psOSCleanupData->uiCount] = psSg;
+	psOSCleanupData->pfnServerCleanup = pfnServerCleanup;
+	psOSCleanupData->pvServerCleanupData = pvServerCleanupParam;
+
+	psDesc->callback_param = psOSCleanupData;
+	psDesc->callback = dma_callback;
+
+	if (bFirst)
+		kthread_run(cleanup_thread, psOSCleanupData, "dma-cleanup-thread");
+	psOSCleanupData->ppsDescriptors[psOSCleanupData->uiCount] = psDesc;
+
+	psOSCleanupData->uiCount++;
+
+	return PVRSRV_OK;
+
+e4:
+	dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+e3:
+	sg_free_table(psSg);
+e2:
+	{
+		IMG_UINT32 i;
+		/* Unpin pages */
+		for (i=0; i<psOSCleanupData->puiNumPages[psOSCleanupData->uiCount]; i++)
+		{
+			pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][i]);
+		}
+	}
+	OSFreeMem(psOSCleanupData->pages[psOSCleanupData->uiCount]);
+e1:
+	OSFreeMem(psSg);
+e0:
+	return eError;
+}
+
+static IMG_UINT32
+CalculateValidPages(IMG_BOOL *pbValid,
+					IMG_UINT32 ui32SizeInPages)
+{
+	IMG_UINT32 ui32nValid;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0, ui32nValid = 0; ui32Idx < ui32SizeInPages; ui32Idx++)
+	{
+		ui32nValid += pbValid[ui32Idx] ? 1 : 0;
+	}
+
+	return ui32nValid;
+}
+
+PVRSRV_ERROR OSDmaPrepareTransferSparse(PVRSRV_DEVICE_NODE *psDevNode,
+										void* pvChan,
+										IMG_DMA_ADDR* psDmaAddr,
+										IMG_BOOL *pbValid,
+										IMG_UINT64* puiAddress,
+										IMG_UINT64 uiSize,
+										IMG_UINT32 uiOffsetInFirstPMRPage,
+										IMG_UINT32 ui32SizeInPages,
+										IMG_BOOL bMemToDev,
+										void* pvOSData,
+										void* pvServerCleanupParam,
+										PFN_SERVER_CLEANUP pfnServerCleanup,
+										IMG_BOOL bFirst)
+{
+
+	IMG_INT iRet;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+	OS_CLEANUP_DATA* psOSCleanupData = pvOSData;
+	IMG_UINT32 ui32PageSize = OSGetPageSize();
+	void *pvNextAddress = puiAddress;
+	IMG_UINT32 ui32Idx;
+	IMG_INT32 i32Rwd;
+
+	enum dma_data_direction eDataDirection = bMemToDev ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct dma_slave_config sConfig = {0};
+	struct dma_async_tx_descriptor *psDesc;
+
+	unsigned long offset = (unsigned long)puiAddress & ((1 << PAGE_SHIFT) - 1);
+	unsigned int num_pages = (uiSize + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	unsigned int num_valid_pages = CalculateValidPages(pbValid, ui32SizeInPages);
+	unsigned int num_pinned_pages = 0;
+	unsigned int valid_idx;
+	size_t transfer_size;
+	struct page ** next_pages;
+	struct sg_table *psSg;
+
+	psOSCleanupData->uiNumValidPages[psOSCleanupData->uiCount] = num_valid_pages;
+	psOSCleanupData->pbIsSparse[psOSCleanupData->uiCount] = IMG_TRUE;
+
+	/*
+	 * If an SG transfer from virtual memory to card memory goes over a page boundary in
+	 * main memory, it'll span two different pages - therefore, total number of pages to
+	 * keep track of should be twice as many as for a simple transfer. This twice-as-big
+	 * allocation is also necessary because the same virtual memory page might be present
+	 * in more than one SG DMA transfer, because of differences in first-page offset between
+	 * the sparse device PMR and the virtual memory buffer.
+	 */
+	psOSCleanupData->pages[psOSCleanupData->uiCount] = OSAllocZMem(2*num_valid_pages * sizeof(struct page *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->pages[psOSCleanupData->uiCount], eError, e0);
+
+	psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount] = OSAllocZMem(num_valid_pages * sizeof(struct sg_table *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount], eError, e1);
+
+	psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount] = OSAllocZMem(num_valid_pages * sizeof(struct dma_async_tx_descriptor *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount], eError, e11);
+
+	for (ui32Idx = 0, valid_idx = 0; ui32Idx < ui32SizeInPages; ui32Idx++)
+	{
+		if (valid_idx == num_valid_pages)
+		{
+			break;
+		}
+		if (!pbValid[ui32Idx])
+		{
+			pvNextAddress += (ui32Idx == 0) ? ui32PageSize - uiOffsetInFirstPMRPage : ui32PageSize;
+			continue;
+		}
+
+		/* Pick transfer size */
+		if (ui32Idx == 0)
+		{
+			if (uiOffsetInFirstPMRPage + uiSize <= ui32PageSize)
+			{
+				PVR_ASSERT(num_valid_pages == 1);
+				transfer_size = uiSize;
+			}
+			else
+			{
+				transfer_size = ui32PageSize - uiOffsetInFirstPMRPage;
+			}
+		}
+		else
+		{
+			/* Last valid LMA page */
+			if (valid_idx == num_valid_pages - 1)
+			{
+				transfer_size = ((uiOffsetInFirstPMRPage + uiSize - 1) % ui32PageSize) + 1;
+			}
+			else
+			{
+				transfer_size = ui32PageSize;
+			}
+		}
+
+		if (((unsigned long long)pvNextAddress & (ui32PageSize - 1)) + transfer_size > ui32PageSize)
+		{
+			num_pages = 2;
+		}
+		else
+		{
+			num_pages = 1;
+		}
+
+		next_pages = psOSCleanupData->pages[psOSCleanupData->uiCount] + (valid_idx * 2);
+
+		num_pinned_pages = pvr_pin_user_pages_for_dma(
+			pvNextAddress,
+			num_pages,
+			!bMemToDev,
+			next_pages);
+
+		if (num_pinned_pages != num_pages)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "get_user_pages_fast for sparse failed: (%d - %u)", num_pinned_pages, num_pages));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e2;
+		}
+
+#if defined(PVRSRV_DEBUG_DMA)
+		DMADumpPhysicalAddresses(next_pages, num_pages,
+								 &psDmaAddr[ui32Idx],
+								 (unsigned long)pvNextAddress & (ui32PageSize - 1));
+#endif
+
+		psSg = OSAllocZMem(sizeof(struct sg_table));
+		PVR_LOG_GOTO_IF_NOMEM(psSg, eError, e3);
+
+		if (sg_alloc_table_from_pages(psSg, next_pages, num_pages,
+									  (unsigned long)pvNextAddress & (ui32PageSize - 1),
+									  transfer_size,
+									  GFP_KERNEL) != 0)
+		{
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table_from_pages failed"));
+			goto e4;
+		}
+
+		pvNextAddress += transfer_size;
+
+		if (bMemToDev)
+		{
+			sConfig.direction = DMA_MEM_TO_DEV;
+			sConfig.src_addr = 0;
+			sConfig.dst_addr = psDmaAddr[ui32Idx].uiAddr;
+		}
+		else
+		{
+			sConfig.direction = DMA_DEV_TO_MEM;
+			sConfig.src_addr = psDmaAddr[ui32Idx].uiAddr;
+			sConfig.dst_addr = 0;
+		}
+		dmaengine_slave_config(pvChan, &sConfig);
+
+		iRet = dma_map_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+		if (!iRet)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Error mapping SG list", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e5;
+		}
+		dma_sync_sg_for_device(psDevConfig->pvOSDevice, psSg->sgl,(unsigned int)iRet, eDataDirection);
+
+		psDesc = dmaengine_prep_slave_sg(pvChan, psSg->sgl, (unsigned int)iRet, sConfig.direction, 0);
+		if (!psDesc)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: dmaengine_prep_slave_sg failed", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto e6;
+		}
+
+		psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount][valid_idx] = psSg;
+		psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount][valid_idx] = psDesc;
+		psOSCleanupData->puiNumPages[psOSCleanupData->uiCount] = ++valid_idx;
+
+		if (valid_idx == num_valid_pages)
+		{
+			psDesc->callback_param = psOSCleanupData;
+			psDesc->callback = dma_callback;
+
+			if (bFirst)
+			{
+				psOSCleanupData->eDirection = eDataDirection;
+				psOSCleanupData->pfnServerCleanup = pfnServerCleanup;
+				psOSCleanupData->pvServerCleanupData = pvServerCleanupParam;
+
+				kthread_run(cleanup_thread, psOSCleanupData, "dma-cleanup-thread");
+			}
+
+			psOSCleanupData->uiCount++;
+		}
+
+	}
+
+	return PVRSRV_OK;
+
+e6:
+	dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+e5:
+	sg_free_table(psSg);
+e4:
+	OSFreeMem(psSg);
+e3:
+	/* Unpin last */
+	pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx]);
+
+	if (psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx+1])
+	{
+		pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx+1]);
+	}
+e2:
+	/* rewind */
+	for (i32Rwd=valid_idx-1; i32Rwd >= 0; i32Rwd--)
+	{
+		IMG_UINT32 i;
+
+		psSg = psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount][i32Rwd];
+		dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+		sg_free_table(psSg);
+
+		/* Unpin pages */
+		for (i=0; i < psOSCleanupData->puiNumPages[psOSCleanupData->uiCount]*2; i++)
+		{
+			if (psOSCleanupData->pages[psOSCleanupData->uiCount][i])
+			{
+				pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][i]);
+			}
+		}
+	}
+	OSFreeMem(psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount]);
+e11:
+	OSFreeMem(psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount]);
+e1:
+	OSFreeMem(psOSCleanupData->pages[psOSCleanupData->uiCount]);
+e0:
+	return eError;
+}
+
+#endif /* SUPPORT_DMA_TRANSFER */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+IMG_INTERNAL PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR)
+{
+	struct dma_heap *heap;
+	struct dma_buf *buf;
+	struct device *dev;
+	struct dma_buf_attachment *buf_attachment;
+
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_ERROR eError;
+	IMG_CHAR *pszHeapName;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode->psDevConfig->pszSecureDMAHeapName, "pszSecureDMAHeapName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM((OSStringLength(psDeviceNode->psDevConfig->pszSecureDMAHeapName) > 0), "pszSecureDMAHeapName length");
+
+	pszHeapName = psDeviceNode->psDevConfig->pszSecureDMAHeapName;
+	dev = (struct device*)psDeviceNode->psDevConfig->pvOSDevice;
+
+	heap = dma_heap_find(pszHeapName);
+	PVR_LOG_GOTO_IF_NOMEM(heap, eError, ErrorExit);
+
+	buf = dma_heap_buffer_alloc(heap, uiSize, 0, 0);
+	if (IS_ERR_OR_NULL(buf))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("dma_heap_buffer_alloc", eError,
+		                        PVRSRV_ERROR_OUT_OF_MEMORY, ErrorHeapPut);
+	}
+
+	if (buf->size < uiSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: buffer size (%ld) is less than requested (%lld).",
+				 __func__, buf->size, uiSize));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorBufFree;
+	}
+
+	buf_attachment = dma_buf_attach(buf, dev);
+	if (IS_ERR_OR_NULL(buf_attachment))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("dma_buf_attach", eError,
+		                        PVRSRV_ERROR_OUT_OF_MEMORY, ErrorBufFree);
+	}
+
+	eError = PhysmemCreateNewDmaBufBackedPMR(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_EXTERNAL],
+											 buf_attachment,
+											 NULL,
+											 PVRSRV_MEMALLOCFLAG_GPU_READABLE
+											 | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
+											 OSGetCurrentClientProcessIDKM(),
+											 buf->size,
+											 1,
+											 1,
+											 &ui32MappingTable,
+											 OSStringLength(pszName),
+											 pszName,
+											 ppsPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateNewDmaBufBackedPMR", ErrorBufDetach);
+
+	PhysmemSetDmaHeap(psPMR, heap);
+
+	return PVRSRV_OK;
+
+ErrorBufDetach:
+	dma_buf_detach(buf, buf_attachment);
+ErrorBufFree:
+	dma_heap_buffer_free(buf);
+ErrorHeapPut:
+	dma_heap_put(heap);
+ErrorExit:
+	return eError;
+}
+
+IMG_INTERNAL void
+OSFreeSecBuf(PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	struct dma_buf *buf = PhysmemGetDmaBuf(psPMR);
+	struct dma_heap *heap = PhysmemGetDmaHeap(psPMR);
+
+	/* calls dma_buf_put() */
+	dma_heap_buffer_free(buf);
+
+	if (heap != NULL)
+	{
+		PhysmemSetDmaHeap(psPMR, NULL);
+		dma_heap_put(heap);
+	}
+
+	eError = PMRUnrefPMR(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+#else /* PVR_ANDROID_HAS_DMA_HEAP_FIND */
+IMG_INTERNAL PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR)
+{
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_ERROR eError;
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiSize,
+									1,
+									1,
+									&ui32MappingTable,
+									ExactLog2(OSGetPageSize()),
+									PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_SECURE)
+									| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
+									OSStringLength(pszName),
+									pszName,
+									OSGetCurrentClientProcessIDKM(),
+									ppsPMR,
+									PDUMP_NONE,
+									NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemNewRamBackedPMR", ErrorExit);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(*ppsPMR, PVR_SYS_ALLOC_PID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RIWritePMREntryWithOwnerKM", ErrorUnrefPMR);
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+ErrorUnrefPMR:
+	(void) PMRUnrefPMR(*ppsPMR);
+#endif
+ErrorExit:
+	return eError;
+}
+
+IMG_INTERNAL void
+OSFreeSecBuf(PMR *psPMR)
+{
+	PVRSRV_ERROR eError = PMRUnrefPMR(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+#endif
+#endif /* SUPPORT_SECURE_ALLOC_KM */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm.c
new file mode 100644
index 000000000000..d3d3ab9f83fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm.c
@@ -0,0 +1,155 @@
+/*************************************************************************/ /*!
+@File
+@Title          arm specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+ #include <linux/dma-map-ops.h>
+#else
+ #include <linux/dma-mapping.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+#include <asm/cacheflush.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+
+static inline size_t pvr_dmac_range_len(const void *pvStart, const void *pvEnd)
+{
+	return (size_t)((char *)pvEnd - (char *)pvStart);
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+		                           sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                           DMA_TO_DEVICE);
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+		                        sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                        DMA_FROM_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_device(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_TO_DEVICE);
+	arm_dma_ops.sync_single_for_cpu(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_FROM_DEVICE);
+#endif
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+		                           sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                           DMA_TO_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_device(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_TO_DEVICE);
+#endif
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+		                        sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                        DMA_FROM_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_cpu(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_FROM_DEVICE);
+#endif
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+}
+
+/* User Enable Register */
+#define PMUSERENR_EN      0x00000001 /* enable user access to the counters */
+
+static void per_cpu_perf_counter_user_access_en(void *data)
+{
+	PVR_UNREFERENCED_PARAMETER(data);
+	/* Enable user-mode access to counters. */
+	asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(PMUSERENR_EN));
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+	on_each_cpu(per_cpu_perf_counter_user_access_en, NULL, 1);
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	/*
+	 * The kernel looks to have always used normal memory under ARM32.
+	 * See osfunc_arm64.c implementation for more details.
+	 */
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm64.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm64.c
new file mode 100644
index 000000000000..5d0419a00d48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm64.c
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File
+@Title          arm64 specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#include <linux/cpumask.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+ #include <linux/dma-map-ops.h>
+#else
+ #include <linux/dma-mapping.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+#include <asm/cacheflush.h>
+#include <linux/uaccess.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#include "kernel_compatibility.h"
+
+#if defined(CONFIG_OUTER_CACHE)
+  /* If you encounter a 64-bit ARM system with an outer cache, you'll need
+   * to add the necessary code to manage that cache. See osfunc_arm.c
+   * for an example of how to do so.
+   */
+	#error "CONFIG_OUTER_CACHE not supported on arm64."
+#endif
+
+static inline void begin_user_mode_access(void)
+{
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	uaccess_enable_privileged();
+#endif
+}
+
+static inline void end_user_mode_access(void)
+{
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	uaccess_disable_privileged();
+#endif
+}
+
+static inline void FlushRange(void *pvRangeAddrStart,
+							  void *pvRangeAddrEnd,
+							  PVRSRV_CACHE_OP eCacheOp)
+{
+	IMG_UINT32 ui32CacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+	IMG_BYTE *pbStart = pvRangeAddrStart;
+	IMG_BYTE *pbEnd = pvRangeAddrEnd;
+	IMG_BYTE *pbBase;
+
+	/*
+	  On arm64, the TRM states in D5.8.1 (data and unified caches) that if cache
+	  maintenance is performed on a memory location using a VA, the effect of
+	  that cache maintenance is visible to all VA aliases of the physical memory
+	  location. So here it's quicker to issue the machine cache maintenance
+	  instruction directly without going via the Linux kernel DMA framework as
+	  this is sufficient to maintain the CPU d-caches on arm64.
+	 */
+
+	begin_user_mode_access();
+
+	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)ui32CacheLineSize);
+
+	/* Memory-barrier */
+	asm volatile("dsb sy" : : : "memory");
+
+	switch (eCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			for (pbBase = pbStart; pbBase < pbEnd; pbBase += ui32CacheLineSize)
+			{
+				asm volatile ("dc cvac, %0" :: "r" (pbBase));
+			}
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+		case PVRSRV_CACHE_OP_FLUSH:
+			for (pbBase = pbStart; pbBase < pbEnd; pbBase += ui32CacheLineSize)
+			{
+				asm volatile ("dc civac, %0" :: "r" (pbBase));
+			}
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Cache maintenance operation type %d is invalid",
+					__func__, eCacheOp));
+			break;
+	}
+
+	/* Memory-barrier */
+	asm volatile("dsb sy" : : : "memory");
+
+	end_user_mode_access();
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	if (pvVirtStart == NULL)
+	{
+		/*
+		 * Converting the physical addresses to virtual addresses allows us to
+		 * utilize the assembly instruction that makes the Flush + Invalidate
+		 * cache operation atomic.
+		 * There is no state in-between the flush and the invalidate operation.
+		 */
+		pvVirtStart = phys_to_virt(sCPUPhysStart.uiAddr);
+		pvVirtEnd = phys_to_virt(sCPUPhysEnd.uiAddr);
+	}
+
+	FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_FLUSH);
+	return;
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+
+	if (pvVirtStart)
+	{
+		FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_CLEAN);
+		return;
+	}
+
+	dev = psDevNode->psDevConfig->pvOSDevice;
+
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+								   sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								   DMA_TO_DEVICE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cache operation cannot be completed!"));
+	}
+
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+								 void *pvVirtStart,
+								 void *pvVirtEnd,
+								 IMG_CPU_PHYADDR sCPUPhysStart,
+								 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+
+	if (pvVirtStart)
+	{
+		FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_INVALIDATE);
+		return;
+	}
+
+	dev = psDevNode->psDevConfig->pvOSDevice;
+
+	if (dev)
+	{
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+								sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								DMA_FROM_DEVICE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cache operation cannot be completed!"));
+	}
+}
+
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	if (!psDevNode->psDevConfig->pvOSDevice)
+	{
+		/* Host Mem device node doesn't have an associated Linux dev ptr.
+		   Use virtual addr ops instead of asking kernel to do physical
+		   maintenance */
+		return OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+	}
+
+	return OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	/*
+	 * Under ARM64 there is the concept of 'device' [0] and 'normal' [1] memory.
+	 * Unaligned access on device memory is explicitly disallowed [2]:
+	 *
+	 * 'Further, unaligned accesses are only allowed to regions marked as Normal
+	 *  memory type.
+	 *  ...
+	 *  Attempts to perform unaligned accesses when not allowed will cause an
+	 *  alignment fault (data abort).'
+	 *
+	 * Write-combine on ARM64 can be implemented as either normal non-cached
+	 * memory (NORMAL_NC) or as device memory with gathering enabled
+	 * (DEVICE_GRE.) Kernel 3.13 changed this from the latter to the former.
+	 *
+	 * [0]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/CHDBDIDF.html
+	 * [1]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch13s01s01.html
+	 * [2]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
+	 */
+
+	pgprot_t pgprot = pgprot_writecombine(PAGE_KERNEL);
+
+	return (pgprot_val(pgprot) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_riscv.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_riscv.c
new file mode 100644
index 000000000000..2d2ad2360433
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_riscv.c
@@ -0,0 +1,173 @@
+/*************************************************************************/ /*!
+@File
+@Title          RISC-V specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "cache_ops.h"
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_FLUSH,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: System doesn't implement cache maintenance. Skipping!",
+		         __func__));
+	}
+#endif
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_CLEAN,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: System doesn't implement cache maintenance. Skipping!",
+		         __func__));
+	}
+#endif
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+								 void *pvVirtStart,
+								 void *pvVirtEnd,
+								 IMG_CPU_PHYADDR sCPUPhysStart,
+								 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_INVALIDATE,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: System doesn't implement cache maintenance. Skipping!",
+		         __func__));
+	}
+#endif
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	/*
+	 * Need to obtain psDevNode here and do the following:
+	 *
+	 * OS_CACHE_OP_ADDR_TYPE eOpAddrType =
+	 *	psDevNode->psDevConfig->bHasPhysicalCacheMaintenance ?
+	 *		OS_CACHE_OP_ADDR_TYPE_PHYSICAL : OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+	 *
+	 * Return BOTH for now on.
+	 *
+	 */
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_BOTH;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+#if !defined(NO_HARDWARE)
+	PVR_DPF((PVR_DBG_WARNING, "%s: Not implemented!", __func__));
+	PVR_ASSERT(0);
+#endif
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+    // EIC770X support unaligned access.
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_x86.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_x86.c
new file mode 100644
index 000000000000..2c5c145e268f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_x86.c
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File
+@Title          x86 specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/smp.h>
+#include <linux/uaccess.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+static void x86_flush_cache_range(const void *pvStart, const void *pvEnd)
+{
+	IMG_BYTE *pbStart = (IMG_BYTE *)pvStart;
+	IMG_BYTE *pbEnd = (IMG_BYTE *)pvEnd;
+	IMG_BYTE *pbBase;
+
+	pbEnd = (IMG_BYTE *)PVR_ALIGN((uintptr_t)pbEnd,
+	                              (uintptr_t)boot_cpu_data.x86_clflush_size);
+
+	mb();
+
+	__uaccess_begin();
+
+	for (pbBase = pbStart; pbBase < pbEnd; pbBase += boot_cpu_data.x86_clflush_size)
+	{
+		clflush(pbBase);
+	}
+
+	__uaccess_end();
+
+	mb();
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	/* No clean feature on x86 */
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	/* No invalidate-only support */
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+	/* Not applicable to x86 architecture. */
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osmmap_stub.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osmmap_stub.c
new file mode 100644
index 000000000000..74bad7073c58
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osmmap_stub.c
@@ -0,0 +1,146 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS abstraction for the mmap2 interface for mapping PMRs into
+                User Mode memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* our exported API */
+#include "osmmap.h"
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+/* services/include/ */
+
+/* services/include/srvhelper/ */
+#include "ra.h"
+
+#include "pmr.h"
+
+PVRSRV_ERROR
+OSMMapPMR(IMG_HANDLE hBridge,
+          IMG_HANDLE hPMR,
+          IMG_DEVMEM_SIZE_T uiPMRSize,
+          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+          IMG_HANDLE *phOSMMapPrivDataOut,
+          void **ppvMappingAddressOut,
+          size_t *puiMappingLengthOut)
+{
+    PVRSRV_ERROR eError;
+    PMR *psPMR;
+    void *pvKernelAddress;
+    size_t uiLength;
+    IMG_HANDLE hPriv;
+
+    PVR_UNREFERENCED_PARAMETER(hBridge);
+    PVR_UNREFERENCED_PARAMETER(uiFlags);
+
+    /*
+      Normally this function would mmap a PMR into the memory space of
+      user process, but in this case we're taking a PMR and mapping it
+      into kernel virtual space.  We keep the same function name for
+      symmetry as this allows the higher layers of the software stack
+      to not care whether they are user mode or kernel
+    */
+
+    psPMR = hPMR;
+
+    if (PMR_IsSparse(psPMR))
+    {
+        eError = PMRAcquireSparseKernelMappingData(psPMR,
+                                            0,
+                                            0,
+                                            &pvKernelAddress,
+                                            &uiLength,
+                                            &hPriv);
+    }
+    else
+    {
+        eError = PMRAcquireKernelMappingData(psPMR,
+                                            0,
+                                            0,
+                                            &pvKernelAddress,
+                                            &uiLength,
+                                            &hPriv);
+    }
+    if (eError != PVRSRV_OK)
+    {
+        goto e0;
+    }
+
+    *phOSMMapPrivDataOut = hPriv;
+    *ppvMappingAddressOut = pvKernelAddress;
+    *puiMappingLengthOut = uiLength;
+
+    /* MappingLength might be rounded up to page size */
+    PVR_ASSERT(*puiMappingLengthOut >= uiPMRSize);
+
+    return PVRSRV_OK;
+
+    /*
+      error exit paths follow
+    */
+
+e0:
+    PVR_ASSERT(eError != PVRSRV_OK);
+    return eError;
+}
+
+void
+OSMUnmapPMR(IMG_HANDLE hBridge,
+            IMG_HANDLE hPMR,
+            IMG_HANDLE hOSMMapPrivData,
+            void *pvMappingAddress,
+            size_t uiMappingLength)
+{
+    PMR *psPMR;
+
+    PVR_UNREFERENCED_PARAMETER(hBridge);
+    PVR_UNREFERENCED_PARAMETER(pvMappingAddress);
+    PVR_UNREFERENCED_PARAMETER(uiMappingLength);
+
+    psPMR = hPMR;
+    PMRReleaseKernelMappingData(psPMR,
+                                hOSMMapPrivData);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/ossecure_export.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/ossecure_export.c
new file mode 100644
index 000000000000..c1b4a7083bf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/ossecure_export.c
@@ -0,0 +1,155 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/anon_inodes.h>
+#include <linux/dcache.h>
+#include <linux/mount.h>
+#include <linux/sched.h>
+#include <linux/cred.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "ossecure_export.h"
+#include "private_data.h"
+#include "pvr_debug.h"
+
+#include "kernel_compatibility.h"
+
+typedef struct
+{
+	PVRSRV_ERROR (*pfnReleaseFunc)(void *);
+	void *pvData;
+} OSSecureFileData;
+
+static IMG_INT _OSSecureFileReleaseFunc(struct inode *psInode,
+                                        struct file *psFile)
+{
+	OSSecureFileData *psSecureFileData = (OSSecureFileData *)psFile->private_data;
+	psSecureFileData->pfnReleaseFunc(psSecureFileData->pvData);
+
+	OSFreeMem(psSecureFileData);
+	PVR_UNREFERENCED_PARAMETER(psInode);
+
+	return 0;
+}
+
+static struct file_operations secure_file_fops = {
+	.release	= _OSSecureFileReleaseFunc,
+};
+
+PVRSRV_ERROR OSSecureExport(const IMG_CHAR *pszName,
+                            PVRSRV_ERROR (*pfnReleaseFunc)(void *),
+                            void *pvData,
+                            IMG_SECURE_TYPE *phSecure)
+{
+	struct file *secure_file;
+	int secure_fd;
+	PVRSRV_ERROR eError;
+	OSSecureFileData *psSecureFileData;
+
+	PVR_ASSERT(pfnReleaseFunc != NULL || pvData != NULL);
+
+	psSecureFileData = OSAllocMem(sizeof(*psSecureFileData));
+	if (psSecureFileData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	psSecureFileData->pvData = pvData;
+	psSecureFileData->pfnReleaseFunc = pfnReleaseFunc;
+
+	/* Allocate a fd number */
+	secure_fd = get_unused_fd_flags(0);
+	if (secure_fd < 0)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	/* Create a file with provided name, fops and flags,
+	 * also store the private data in the file */
+	secure_file = anon_inode_getfile(pszName, &secure_file_fops, psSecureFileData, 0);
+	if (IS_ERR(secure_file))
+	{
+		put_unused_fd(secure_fd);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	/* Bind our struct file with it's fd number */
+	fd_install(secure_fd, secure_file);
+
+	*phSecure = secure_fd;
+	return PVRSRV_OK;
+
+e0:
+	OSFreeMem(psSecureFileData);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR OSSecureImport(IMG_SECURE_TYPE hSecure, void **ppvData)
+{
+	struct file *secure_file;
+	PVRSRV_ERROR eError;
+	OSSecureFileData *psSecureFileData;
+
+	secure_file = fget(hSecure);
+	if (!secure_file)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	psSecureFileData = (OSSecureFileData *)secure_file->private_data;
+	*ppvData = psSecureFileData->pvData;
+
+	fput(secure_file);
+	return PVRSRV_OK;
+
+err_out:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf.c
new file mode 100644
index 000000000000..e82b0d5cebe6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf.c
@@ -0,0 +1,2396 @@
+/*************************************************************************/ /*!
+@File           physmem_dmabuf.c
+@Title          dmabuf memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for dmabuf memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "physmem_dmabuf.h"
+#include "physmem_dmabuf_internal.h"
+#include "physmem.h"
+#include "pvrsrv.h"
+#include "pmr.h"
+
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_prime.h>
+#else
+#include <drm/drmP.h>
+#endif
+#include <drm/drm_gem.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pmr_impl.h"
+#include "pmr_env.h"
+#include "hash.h"
+#include "private_data.h"
+#include "module_common.h"
+#include "pvr_ion_stats.h"
+#include "cache_km.h"
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+typedef struct _PMR_DMA_BUF_WRAPPER_
+{
+	PMR *psPMR;
+
+	/*
+	 * Set if the PMR has been exported via GEM. This field should only
+	 * be accessed using smp_store_release and smp_load_acquire.
+	 * In general, the field is not safe to access, as the wrapper doesn't
+	 * hold a reference on the GEM object.
+	 */
+	struct drm_gem_object *psObj;
+
+	struct dma_resv sDmaResv;
+
+	/* kernel mapping */
+	IMG_UINT32 uiKernelMappingRefCnt;
+	IMG_HANDLE hKernelMappingHandle;
+	void *pvKernelMappingAddr;
+	PMR_SIZE_T uiKernelMappingLen;
+	POS_LOCK hKernelMappingLock;
+
+	/* device mapping */
+	IMG_UINT32 uiDeviceMappingRefCnt;
+	POS_LOCK hDeviceMappingLock;
+	struct sg_table *psTable;
+} PMR_DMA_BUF_WRAPPER;
+
+typedef struct _PMR_DMA_BUF_GEM_OBJ {
+	struct drm_gem_object sBase;
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+} PMR_DMA_BUF_GEM_OBJ;
+
+#define TO_PMR_DMA_BUF_GEM_OBJ(psObj) IMG_CONTAINER_OF((psObj), PMR_DMA_BUF_GEM_OBJ, sBase)
+
+static PVRSRV_ERROR _PMRKMap(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                             size_t uiSize,
+                             void **pvAddr)
+{
+	IMG_HANDLE hKernelMapping;
+	void *pvAddrOut = NULL;
+	size_t uiLengthOut = 0;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt++ == 0)
+	{
+		PVR_ASSERT(psPMRWrapper->hKernelMappingHandle == NULL);
+		PVR_ASSERT(psPMRWrapper->pvKernelMappingAddr == NULL);
+
+		if (PMR_IsSparse(psPMRWrapper->psPMR))
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMRWrapper->psPMR,
+			                                           0,
+			                                           uiSize,
+			                                           &pvAddrOut,
+			                                           &uiLengthOut,
+			                                           &hKernelMapping);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData",
+			                      ErrReleaseLock);
+		}
+		else
+		{
+			eError = PMRAcquireKernelMappingData(psPMRWrapper->psPMR,
+			                                     0,
+			                                     uiSize,
+			                                     &pvAddrOut,
+			                                     &uiLengthOut,
+			                                     &hKernelMapping);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData",
+			                      ErrReleaseLock);
+		}
+
+		psPMRWrapper->hKernelMappingHandle = hKernelMapping;
+		psPMRWrapper->pvKernelMappingAddr = pvAddrOut;
+		psPMRWrapper->uiKernelMappingLen = uiLengthOut;
+	}
+
+	*pvAddr = psPMRWrapper->pvKernelMappingAddr;
+	goto ExitUnlock;
+
+ErrReleaseLock:
+	psPMRWrapper->uiKernelMappingRefCnt--;
+
+ExitUnlock:
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+	return eError;
+}
+
+static void _PMRKUnmap(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (--psPMRWrapper->uiKernelMappingRefCnt == 0)
+	{
+		PVRSRV_ERROR eError;
+
+		PVR_ASSERT(psPMRWrapper->hKernelMappingHandle != NULL);
+		PVR_ASSERT(psPMRWrapper->pvKernelMappingAddr != NULL);
+
+		eError = PMRReleaseKernelMappingData(psPMRWrapper->psPMR,
+		                                     psPMRWrapper->hKernelMappingHandle);
+		PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+		psPMRWrapper->hKernelMappingHandle = NULL;
+		psPMRWrapper->pvKernelMappingAddr = NULL;
+		psPMRWrapper->uiKernelMappingLen = 0;
+	}
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+}
+
+static PVRSRV_ERROR _PMRInvalidateCache(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt == 0) {
+		OSLockRelease(psPMRWrapper->hKernelMappingLock);
+		return PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+	}
+
+	eError = CacheOpValExec(psPMRWrapper->psPMR,
+	                        (IMG_UINT64) (uintptr_t) psPMRWrapper->pvKernelMappingAddr,
+	                        0,
+	                        psPMRWrapper->uiKernelMappingLen,
+	                        PVRSRV_CACHE_OP_INVALIDATE);
+	PVR_LOG_IF_ERROR(eError, "CacheOpValExec");
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _PMRCleanCache(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt == 0) {
+		OSLockRelease(psPMRWrapper->hKernelMappingLock);
+		return PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+	}
+
+	eError = CacheOpValExec(psPMRWrapper->psPMR,
+	                        (IMG_UINT64) (uintptr_t) psPMRWrapper->pvKernelMappingAddr,
+	                        0,
+	                        psPMRWrapper->uiKernelMappingLen,
+	                        PVRSRV_CACHE_OP_FLUSH);
+	PVR_LOG_IF_ERROR(eError, "CacheOpValExec");
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+
+	return eError;
+}
+
+/*
+ * dma_buf_ops common code
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ * The following common functions are used by both the dma_buf and GEM
+ * callbacks.
+ */
+
+static int PVRDmaBufOpsAttachCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+							  struct dma_buf_attachment *psAttachment)
+{
+	PMR *psPMR = psPMRWrapper->psPMR;
+
+	if (PMR_GetType(psPMR) == PMR_TYPE_DMABUF)
+	{
+		// don't support exporting PMRs that are itself created from imported
+		// DmaBufs
+		PVR_DPF((PVR_DBG_ERROR, "exporting PMRs of type DMABUF not supported"));
+
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static struct sg_table *PVRDmaBufOpsMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                              struct dma_buf_attachment *psAttachment,
+                                              enum dma_data_direction eDirection)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMR = psPMRWrapper->psPMR;
+	IMG_UINT uiNents;
+	IMG_DEVMEM_SIZE_T uiPhysSize, uiVirtSize;
+	IMG_UINT32 uiNumVirtPages;
+	void *pvPAddrData = NULL;
+	IMG_DEV_PHYADDR asPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC], *psPAddr = asPAddr;
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC], *pbValid = abValid;
+	IMG_UINT32 i;
+	IMG_DEV_PHYADDR sPAddrPrev, sPAddrCurr;
+	struct sg_table *psTable;
+	struct scatterlist *psSg;
+	int iRet = 0;
+	IMG_UINT32 uiDevPageShift, uiDevPageSize;
+
+	OSLockAcquire(psPMRWrapper->hDeviceMappingLock);
+
+	psPMRWrapper->uiDeviceMappingRefCnt++;
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt > 1)
+	{
+		goto OkUnlock;
+	}
+
+	PVR_ASSERT(psPMRWrapper->psTable == NULL);
+
+	uiDevPageShift = PMR_GetLog2Contiguity(psPMR);
+	uiDevPageSize = 1u << uiDevPageShift;
+	uiVirtSize = PMR_LogicalSize(psPMR);
+	uiPhysSize = PMR_PhysicalSize(psPMR);
+	if (uiPhysSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "invalid PMR size"));
+		iRet = PVRSRVToNativeError(PVRSRV_ERROR_BAD_MAPPING);
+		goto ErrUnlockMapping;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): mapping pmr: 0x%p@0x%" IMG_UINT64_FMTSPECx
+	         " from heap: \"%s\" with page size: 0x%x", __func__, psPMR,
+	         uiPhysSize, PhysHeapName(PMR_PhysHeap(psPMR)), uiDevPageSize));
+
+	uiNumVirtPages = uiVirtSize >> uiDevPageShift;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMRLockSysPhysAddresses");
+		iRet = PVRSRVToNativeError(eError);
+		goto ErrUnlockMapping;
+	}
+
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	psTable = OSAllocZMem(sizeof(*psTable));
+	if (psTable == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAllocMem.1() failed"));
+		iRet = -ENOMEM;
+		goto ErrUnlockPhysAddresses;
+	}
+
+	if (uiNumVirtPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		pvPAddrData = OSAllocMem(uiNumVirtPages * (sizeof(*psPAddr) + sizeof(*pbValid)));
+		if (pvPAddrData == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "OSAllocMem.2() failed"));
+			iRet = -ENOMEM;
+			goto ErrFreeTable;
+		}
+
+		psPAddr = IMG_OFFSET_ADDR(pvPAddrData, 0);
+		pbValid = IMG_OFFSET_ADDR(pvPAddrData, uiNumVirtPages * sizeof(*psPAddr));
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+	                         uiDevPageShift,
+	                         uiNumVirtPages,
+	                         0,
+	                         psPAddr,
+	                         pbValid,
+	                         DEVICE_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "PMR_DevPhysAddr");
+		iRet = PVRSRVToNativeError(eError);
+		goto ErrFreePAddrData;
+	}
+
+	/* Calculate how many contiguous regions there are in the PMR. This
+	 * value will be used to allocate one scatter list for every region. */
+
+	/* Find first valid physical address. */
+	for (i = 0; i < uiNumVirtPages && !pbValid[i]; i++);
+
+	sPAddrPrev = psPAddr[i];
+	uiNents = 1;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): %03u paddr: 0x%" IMG_UINT64_FMTSPECx,
+	         __func__, i, sPAddrPrev.uiAddr));
+
+	/* Find the rest of the addresses. */
+	for (i = i + 1; i < uiNumVirtPages; i++)
+	{
+		if (!pbValid[i])
+		{
+			continue;
+		}
+
+		sPAddrCurr = psPAddr[i];
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s(): %03u paddr: 0x%" IMG_UINT64_FMTSPECx
+		         ", pprev: 0x%" IMG_UINT64_FMTSPECx ", valid: %u", __func__,
+		         i, sPAddrCurr.uiAddr, sPAddrPrev.uiAddr, pbValid[i]));
+
+		if (sPAddrCurr.uiAddr != (sPAddrPrev.uiAddr + uiDevPageSize))
+		{
+			uiNents++;
+		}
+
+		sPAddrPrev = sPAddrCurr;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): found %u contiguous regions", __func__,
+	         uiNents));
+
+	/* Allocate scatter lists from all of the contiguous regions. */
+
+	iRet = sg_alloc_table(psTable, uiNents, GFP_KERNEL);
+	if (iRet != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table() failed with error %d", iRet));
+		goto ErrFreePAddrData;
+	}
+
+	/* Fill in all of the physical addresses and sizes for all of the
+	 * contiguous regions that were calculated. */
+
+	for (i = 0; i < uiNumVirtPages && !pbValid[i]; i++);
+
+	psSg = psTable->sgl;
+	sPAddrPrev = psPAddr[i];
+
+	sg_dma_address(psSg) = sPAddrPrev.uiAddr;
+	sg_dma_len(psSg) = uiDevPageSize;
+
+	for (i = i + 1; i < uiNumVirtPages; i++)
+	{
+		if (!pbValid[i])
+		{
+			continue;
+		}
+
+		sPAddrCurr = psPAddr[i];
+
+		if (sPAddrCurr.uiAddr != (sPAddrPrev.uiAddr + uiDevPageSize))
+		{
+			psSg = sg_next(psSg);
+			PVR_ASSERT(psSg != NULL);
+
+			sg_dma_address(psSg) = sPAddrCurr.uiAddr;
+			sg_dma_len(psSg) = uiDevPageSize;
+		}
+		else
+		{
+			sg_dma_len(psSg) += uiDevPageSize;
+		}
+
+		sPAddrPrev = sPAddrCurr;
+	}
+
+	if (pvPAddrData != NULL)
+	{
+		OSFreeMem(pvPAddrData);
+	}
+
+	psPMRWrapper->psTable = psTable;
+
+OkUnlock:
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+
+	return psPMRWrapper->psTable;
+
+ErrFreePAddrData:
+	if (pvPAddrData != NULL)
+	{
+		OSFreeMem(pvPAddrData);
+	}
+ErrFreeTable:
+	OSFreeMem(psTable);
+ErrUnlockPhysAddresses:
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+	}
+ErrUnlockMapping:
+	psPMRWrapper->uiDeviceMappingRefCnt--;
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+
+	return ERR_PTR(iRet);
+}
+
+static void PVRDmaBufOpsUnmapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                              struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hDeviceMappingLock);
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "reference count on mapping already 0"));
+		goto ErrUnlock;
+	}
+
+	psPMRWrapper->uiDeviceMappingRefCnt--;
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt > 0)
+	{
+		goto ErrUnlock;
+	}
+
+	dma_unmap_sg(psAttachment->dev, psTable->sgl, psTable->nents, eDirection);
+	sg_free_table(psTable);
+
+	eError = PMRUnlockSysPhysAddresses(psPMRWrapper->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+
+	OSFreeMem(psPMRWrapper->psTable);
+	psPMRWrapper->psTable = NULL;
+
+ErrUnlock:
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+}
+
+static int PVRDmaBufOpsBeginCpuAccessCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                      enum dma_data_direction eDirection)
+{
+	if (PVRSRV_CHECK_CPU_CACHED(PMR_Flags(psPMRWrapper->psPMR)))
+	{
+		PVRSRV_ERROR eError = _PMRInvalidateCache(psPMRWrapper);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PMRInvalidateCache");
+			return OSPVRSRVToNativeError(eError);
+		}
+	}
+
+	return 0;
+}
+
+static int PVRDmaBufOpsEndCpuAccessCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                    enum dma_data_direction eDirection)
+{
+	if (PVRSRV_CHECK_CPU_CACHED(PMR_Flags(psPMRWrapper->psPMR)))
+	{
+		PVRSRV_ERROR eError = _PMRCleanCache(psPMRWrapper);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PMRCleanCache");
+			return OSPVRSRVToNativeError(eError);
+		}
+	}
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, unsigned long uiPageNum)
+{
+	return NULL;
+}
+
+static void PVRDmaBufOpsKUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, unsigned long uiPageNum, void *pvMem)
+{
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	void *pvAddrOut = NULL;
+	PVRSRV_ERROR eError;
+
+	eError = _PMRKMap(psPMRWrapper, 0, &pvAddrOut);
+
+	return eError == PVRSRV_OK ? pvAddrOut : NULL;
+}
+#else
+static int PVRDmaBufOpsVMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct iosys_map *psMap)
+{
+	void *pvAddrOut = NULL;
+	PVRSRV_ERROR eError;
+
+	eError = _PMRKMap(psPMRWrapper, 0, &pvAddrOut);
+
+	if (eError != PVRSRV_OK)
+	{
+		return OSPVRSRVToNativeError(eError);
+	}
+
+	if (PhysHeapGetType(PMR_PhysHeap(psPMRWrapper->psPMR)) == PHYS_HEAP_TYPE_UMA)
+	{
+		iosys_map_set_vaddr(psMap, pvAddrOut);
+	}
+	else
+	{
+		iosys_map_set_vaddr_iomem(psMap, pvAddrOut);
+	}
+
+	return 0;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, void *pvAddr)
+{
+	_PMRKUnmap(psPMRWrapper);
+}
+#else
+static void PVRDmaBufOpsVUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct iosys_map *psMap)
+{
+	_PMRKUnmap(psPMRWrapper);
+
+	iosys_map_clear(psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct vm_area_struct *psVMA)
+{
+	PMR *psPMR = psPMRWrapper->psPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiProtFlags =
+	    (BITMASK_HAS(psVMA->vm_flags, VM_READ) ? PVRSRV_MEMALLOCFLAG_CPU_READABLE : 0) |
+	    (BITMASK_HAS(psVMA->vm_flags, VM_WRITE) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0);
+	PVRSRV_ERROR eError;
+
+	/* Forcibly clear the VM_MAYWRITE flag as this is inherited from the
+	 * kernel mmap code and we do not want to produce a potentially writable
+	 * mapping from a read-only mapping.
+	 */
+	if (!BITMASK_HAS(psVMA->vm_flags, VM_WRITE))
+	{
+		pvr_vm_flags_clear(psVMA, VM_MAYWRITE);
+	}
+
+	eError = PMRMMapPMR(psPMR, psVMA, uiProtFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMRMMapPMR");
+		return OSPVRSRVToNativeError(eError);
+	}
+
+	return 0;
+}
+
+/* end of dma_buf_ops common code*/
+
+/*
+ * dma_buf_ops (non-GEM)
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ */
+
+static int PVRDmaBufOpsAttach(struct dma_buf *psDmaBuf,
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+							  struct device *psDev,
+#endif
+							  struct dma_buf_attachment *psAttachment)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsAttachCommon(psPMRWrapper, psAttachment);
+}
+
+static struct sg_table *PVRDmaBufOpsMap(struct dma_buf_attachment *psAttachment,
+                                        enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psAttachment->dmabuf->priv;
+
+	return PVRDmaBufOpsMapCommon(psPMRWrapper, psAttachment, eDirection);
+}
+
+static void PVRDmaBufOpsUnmap(struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psAttachment->dmabuf->priv;
+
+	PVRDmaBufOpsUnmapCommon(psPMRWrapper, psAttachment, psTable, eDirection);
+}
+
+static void PVRDmaBufOpsRelease(struct dma_buf *psDmaBuf)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+	PMR *psPMR = psPMRWrapper->psPMR;
+
+	PVRSRV_ERROR eError = PMRUnrefPMR(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+}
+
+static int PVRDmaBufOpsBeginCpuAccess(struct dma_buf *psDmaBuf,
+                                      enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsBeginCpuAccessCommon(psPMRWrapper, eDirection);
+}
+
+static int PVRDmaBufOpsEndCpuAccess(struct dma_buf *psDmaBuf,
+                                    enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+	int iErr;
+
+	iErr = PVRDmaBufOpsEndCpuAccessCommon(psPMRWrapper, eDirection);
+
+	return iErr;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMap(struct dma_buf *psDmaBuf, unsigned long uiPageNum)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsKMapCommon(psPMRWrapper, uiPageNum);
+}
+
+static void PVRDmaBufOpsKUnMap(struct dma_buf *psDmaBuf, unsigned long uiPageNum, void *pvMem)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsKUnMapCommon(psPMRWrapper, uiPageNum, pvMem);
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMap(struct dma_buf *psDmaBuf)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper);
+}
+#else
+static int PVRDmaBufOpsVMap(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMap(struct dma_buf *psDmaBuf, void *pvAddr)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, pvAddr);
+}
+#else
+static void PVRDmaBufOpsVUnMap(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMap(struct dma_buf *psDmaBuf, struct vm_area_struct *psVMA)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): psDmaBuf = %px, psPMR = %px", __func__,
+	         psDmaBuf, psPMRWrapper->psPMR));
+
+	return PVRDmaBufOpsMMapCommon(psPMRWrapper, psVMA);
+}
+
+static const struct dma_buf_ops sPVRDmaBufOps =
+{
+	.attach        = PVRDmaBufOpsAttach,
+	.map_dma_buf   = PVRDmaBufOpsMap,
+	.unmap_dma_buf = PVRDmaBufOpsUnmap,
+	.release       = PVRDmaBufOpsRelease,
+	.begin_cpu_access = PVRDmaBufOpsBeginCpuAccess,
+	.end_cpu_access   = PVRDmaBufOpsEndCpuAccess,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+	.map_atomic    = PVRDmaBufOpsKMap,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map           = PVRDmaBufOpsKMap,
+	.unmap         = PVRDmaBufOpsKUnMap,
+#endif
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.kmap_atomic   = PVRDmaBufOpsKMap,
+	.kmap          = PVRDmaBufOpsKMap,
+	.kunmap        = PVRDmaBufOpsKUnMap,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.mmap          = PVRDmaBufOpsMMap,
+
+	.vmap          = PVRDmaBufOpsVMap,
+	.vunmap        = PVRDmaBufOpsVUnMap,
+};
+
+/* end of dma_buf_ops (non-GEM) */
+
+/*
+ * dma_buf_ops (GEM)
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ */
+
+static int PVRDmaBufOpsAttachGEM(struct dma_buf *psDmaBuf,
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+							  struct device *psDev,
+#endif
+							  struct dma_buf_attachment *psAttachment)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsAttachCommon(psPMRWrapper, psAttachment);
+}
+
+static struct sg_table *PVRDmaBufOpsMapGEM(struct dma_buf_attachment *psAttachment,
+                                        enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psAttachment->dmabuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsMapCommon(psPMRWrapper, psAttachment, eDirection);
+}
+
+static void PVRDmaBufOpsUnmapGEM(struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psAttachment->dmabuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsUnmapCommon(psPMRWrapper, psAttachment, psTable, eDirection);
+}
+
+static int PVRDmaBufOpsBeginCpuAccessGEM(struct dma_buf *psDmaBuf,
+                                      enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsBeginCpuAccessCommon(psPMRWrapper, eDirection);
+}
+
+static int PVRDmaBufOpsEndCpuAccessGEM(struct dma_buf *psDmaBuf,
+                                    enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	int iErr;
+
+	iErr = PVRDmaBufOpsEndCpuAccessCommon(psPMRWrapper, eDirection);
+
+	return iErr;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMapGEM(struct dma_buf *psDmaBuf, unsigned long uiPageNum)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsKMapCommon(psPMRWrapper, uiPageNum);
+}
+
+static void PVRDmaBufOpsKUnMapGEM(struct dma_buf *psDmaBuf, unsigned long uiPageNum, void *pvMem)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsKUnMapCommon(psPMRWrapper, uiPageNum, pvMem);
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMapGEM(struct dma_buf *psDmaBuf)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper);
+}
+#else
+static int PVRDmaBufOpsVMapGEM(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMapGEM(struct dma_buf *psDmaBuf, void *pvAddr)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, pvAddr);
+}
+#else
+static void PVRDmaBufOpsVUnMapGEM(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMapGEM(struct dma_buf *psDmaBuf, struct vm_area_struct *psVMA)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): psDmaBuf = %px, psPMR = %px", __func__,
+	         psDmaBuf, psPMRWrapper->psPMR));
+
+	return PVRDmaBufOpsMMapCommon(psPMRWrapper, psVMA);
+}
+
+static const struct dma_buf_ops sPVRDmaBufOpsGEM =
+{
+	.attach        = PVRDmaBufOpsAttachGEM,
+	.map_dma_buf   = PVRDmaBufOpsMapGEM,
+	.unmap_dma_buf = PVRDmaBufOpsUnmapGEM,
+	.release       = drm_gem_dmabuf_release,
+	.begin_cpu_access = PVRDmaBufOpsBeginCpuAccessGEM,
+	.end_cpu_access   = PVRDmaBufOpsEndCpuAccessGEM,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+	.map_atomic    = PVRDmaBufOpsKMapGEM,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map           = PVRDmaBufOpsKMapGEM,
+	.unmap         = PVRDmaBufOpsKUnMapGEM,
+#endif
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.kmap_atomic   = PVRDmaBufOpsKMapGEM,
+	.kmap          = PVRDmaBufOpsKMapGEM,
+	.kunmap        = PVRDmaBufOpsKUnMapGEM,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.mmap          = PVRDmaBufOpsMMapGEM,
+
+	.vmap          = PVRDmaBufOpsVMapGEM,
+	.vunmap        = PVRDmaBufOpsVUnMapGEM,
+};
+
+/* end of dma_buf_ops (GEM) */
+
+
+typedef struct _PMR_DMA_BUF_DATA_
+{
+	/* Filled in at PMR create time */
+	PHYS_HEAP *psPhysHeap;
+	struct dma_buf_attachment *psAttachment;
+#if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+	struct dma_heap *psDmaHeap;
+#endif
+	PFN_DESTROY_DMABUF_PMR pfnDestroy;
+	IMG_BOOL bPoisonOnFree;
+	IMG_PID uiOriginPID;
+
+	/* Mapping information. */
+	struct iosys_map sMap;
+
+	/* Modified by PMR lock/unlock */
+	struct sg_table *psSgTable;
+	IMG_DEV_PHYADDR *pasDevPhysAddr;
+	IMG_UINT32 ui32PhysPageCount;
+	IMG_UINT32 ui32VirtPageCount;
+
+	IMG_BOOL bZombie;
+} PMR_DMA_BUF_DATA;
+
+/* Start size of the g_psDmaBufHash hash table */
+#define DMA_BUF_HASH_SIZE 20
+
+static DEFINE_MUTEX(g_FactoryLock);
+
+static HASH_TABLE *g_psDmaBufHash;
+static IMG_UINT32 g_ui32HashRefCount;
+
+#if defined(PVR_ANDROID_ION_USE_SG_LENGTH)
+#define pvr_sg_length(sg) ((sg)->length)
+#else
+#define pvr_sg_length(sg) sg_dma_len(sg)
+#endif
+
+/*
+ * _pvr_dma_resv_lock
+ *
+ * Locks a dma_buf's reservation object (dma_resv). This is required when calling
+ * certain functions from the dma_buf interface (for more info see:
+ * https://docs.kernel.org/driver-api/dma-buf.html#dma-buf-locking-convention).
+ *
+ * This is a wrapper for the kernel's own locking functions which have changed
+ * over a number of versions.
+ *
+ * The dma_resv uses a ww_mutex which provides additional behaviours to prevent
+ * deadlocks. However, this function expects that only a single lock will be
+ * locked in a function at a time making the error code returned is redundant.
+ *
+ * If this changes a ww_acquire_ctx should be used.
+ * For more information about ww_mutex and ww_acquire_ctx see:
+ * https://www.kernel.org/doc/Documentation/locking/ww-mutex-design.txt
+ *
+ * Unlock the dma_resv lock with _pvr_dma_resv_unlock
+ */
+static inline void _pvr_dma_resv_lock(const struct dma_buf *psDMABuf)
+{
+	int iErr = 0;
+
+	PVR_ASSERT(psDMABuf != NULL);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	iErr = dma_resv_lock(psDMABuf->resv, NULL);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	iErr = reservation_object_lock(psDMABuf->resv, NULL);
+#else
+	iErr = ww_mutex_lock(&psDMABuf->resv->lock, NULL);
+#endif
+
+	/* This is not expected to occur, but if it does we are notified. */
+	PVR_LOG_IF_FALSE_VA(PVR_DBG_ERROR,
+	                    iErr == 0,
+	                    "Attempt to lock dma_resv resulted in %d",
+	                    iErr);
+}
+
+/*
+ * _pvr_dma_resv_unlock
+ *
+ * Unlocks a dma_buf's dma_resv struct, previously locked with _pvr_dma_resv_lock.
+ */
+static inline void _pvr_dma_resv_unlock(const struct dma_buf *psDMABuf)
+{
+	PVR_ASSERT(psDMABuf != NULL);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	dma_resv_unlock(psDMABuf->resv);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	reservation_object_unlock(psDMABuf->resv);
+#else
+	ww_mutex_unlock(&psDMABuf->resv->lock);
+#endif
+}
+
+static inline PVRSRV_ERROR _pvr_dma_buf_map_attachment(struct dma_buf_attachment *psAttachment,
+                                                       enum dma_data_direction eDirection,
+                                                       struct sg_table **ppsTable)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0))
+	/* DMA buf reservation locks will be taken internally */
+	*ppsTable = dma_buf_map_attachment_unlocked(psAttachment, eDirection);
+#else
+	_pvr_dma_resv_lock(psAttachment->dmabuf);
+	*ppsTable = dma_buf_map_attachment(psAttachment, eDirection);
+	_pvr_dma_resv_unlock(psAttachment->dmabuf);
+#endif
+
+	if (IS_ERR_OR_NULL(*ppsTable))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map attachment. ppsTable = %p",
+		         __func__,
+		         *ppsTable));
+		eError = PVRSRV_ERROR_DMABUF_ATTACHMENT_MAPPING;
+	}
+
+	return eError;
+}
+
+static inline void _pvr_dma_buf_unmap_attachment(struct dma_buf_attachment *psAttachment,
+                                                 struct sg_table *psTable,
+                                                 enum dma_data_direction eDirection)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0))
+	/* DMA buf reservation locks will be taken internally */
+	dma_buf_unmap_attachment_unlocked(psAttachment, psTable, eDirection);
+#else
+	_pvr_dma_resv_lock(psAttachment->dmabuf);
+	dma_buf_unmap_attachment(psAttachment, psTable, eDirection);
+	_pvr_dma_resv_unlock(psAttachment->dmabuf);
+#endif
+}
+
+static int
+DmaBufSetValue(struct dma_buf *psDmaBuf, int iValue, const char *szFunc)
+{
+	struct iosys_map sMap;
+	int err, err_end_access;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	int i;
+#endif
+
+	err = dma_buf_begin_cpu_access(psDmaBuf, DMA_FROM_DEVICE);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to begin cpu access (err=%d)",
+		                        szFunc, err));
+		goto err_out;
+	}
+
+	_pvr_dma_resv_lock(psDmaBuf);
+	err = dma_buf_vmap(psDmaBuf, &sMap);
+	if (err)
+	{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map page (err=%d)",
+		                        szFunc, err));
+		goto exit_end_access;
+#else
+		for (i = 0; i < psDmaBuf->size / PAGE_SIZE; i++)
+		{
+			void *pvKernAddr;
+
+			pvKernAddr = dma_buf_kmap(psDmaBuf, i);
+			if (IS_ERR_OR_NULL(pvKernAddr))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map page (err=%ld)",
+							szFunc,
+							pvKernAddr ? PTR_ERR(pvKernAddr) : -ENOMEM));
+				err = !pvKernAddr ? -ENOMEM : -EINVAL;
+
+				goto exit_end_access;
+			}
+
+			memset(pvKernAddr, iValue, PAGE_SIZE);
+
+			dma_buf_kunmap(psDmaBuf, i, pvKernAddr);
+		}
+#endif
+	}
+	else
+	{
+		memset(sMap.vaddr, iValue, psDmaBuf->size);
+
+		dma_buf_vunmap(psDmaBuf, &sMap);
+	}
+
+	err = 0;
+
+exit_end_access:
+	_pvr_dma_resv_unlock(psDmaBuf);
+
+	do {
+		err_end_access = dma_buf_end_cpu_access(psDmaBuf, DMA_TO_DEVICE);
+	} while (err_end_access == -EAGAIN || err_end_access == -EINTR);
+
+	if (err_end_access)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to end cpu access (err=%d)",
+		                        szFunc, err_end_access));
+		if (!err)
+		{
+			err = err_end_access;
+		}
+	}
+
+err_out:
+	return err;
+}
+
+/*****************************************************************************
+ *                          PMR callback functions                           *
+ *****************************************************************************/
+
+/* This function is protected by the pfn(Get/Release)PMRFactoryLock() lock
+ * acquired/released in _UnrefAndMaybeDestroy() in pmr.c. */
+static void PMRFinalizeDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+	struct sg_table *psSgTable = psPrivData->psSgTable;
+
+	if (psDmaBuf->ops != &sPVRDmaBufOps && psDmaBuf->ops != &sPVRDmaBufOpsGEM)
+	{
+		if (g_psDmaBufHash)
+		{
+			/* We have a hash table so check if we've seen this dmabuf before */
+			if (HASH_Remove(g_psDmaBufHash, (uintptr_t) psDmaBuf) != 0U)
+			{
+				g_ui32HashRefCount--;
+
+				if (g_ui32HashRefCount == 0)
+				{
+					HASH_Delete(g_psDmaBufHash);
+					g_psDmaBufHash = NULL;
+				}
+			}
+
+			PVRSRVIonRemoveMemAllocRecord(psDmaBuf);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (psPrivData->bZombie)
+	{
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+		                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+		                            psPrivData->uiOriginPID);
+	}
+	else
+#endif
+	{
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+		                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+		                            psPrivData->uiOriginPID);
+	}
+#endif
+
+	psPrivData->ui32PhysPageCount = 0;
+
+	_pvr_dma_buf_unmap_attachment(psAttachment, psSgTable, DMA_BIDIRECTIONAL);
+
+	if (psPrivData->bPoisonOnFree)
+	{
+		int err = DmaBufSetValue(psDmaBuf, PVRSRV_POISON_ON_FREE_VALUE,
+		                         __func__);
+		PVR_LOG_IF_FALSE(err != 0, "Failed to poison allocation before free");
+
+		PVR_ASSERT(err != 0);
+	}
+
+	if (psPrivData->pfnDestroy)
+	{
+		psPrivData->pfnDestroy(psPrivData->psPhysHeap, psPrivData->psAttachment);
+	}
+
+	OSFreeMem(psPrivData->pasDevPhysAddr);
+	OSFreeMem(psPrivData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static PVRSRV_ERROR PMRZombifyDmaBufMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	psPrivData->bZombie = IMG_TRUE;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+#endif
+
+	PVRSRVIonZombifyMemAllocRecord(psDmaBuf);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static PVRSRV_ERROR PMRLockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+{
+	/* The imported memory is assumed to be backed by a permanent allocation.
+	 * The PMR does not need to be (un)locked */
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+                                                 PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	/* The imported memory is assumed to be backed by a permanent allocation.
+	 * The PMR does not need to be (un)locked */
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+	return PVRSRV_OK;
+}
+
+static void PMRFactoryLock(void)
+{
+	mutex_lock(&g_FactoryLock);
+}
+
+static void PMRFactoryUnlock(void)
+{
+	mutex_unlock(&g_FactoryLock);
+}
+
+static PVRSRV_ERROR PMRDevPhysAddrDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+					 IMG_UINT32 ui32Log2PageSize,
+					 IMG_UINT32 ui32NumOfPages,
+					 IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					 IMG_UINT64 ui64IPAPolicyValue,
+					 IMG_UINT64 ui64IPAClearMask,
+#endif
+					 IMG_BOOL *pbValid,
+					 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	IMG_UINT32 ui32PageIndex;
+	IMG_UINT32 idx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (ui32Log2PageSize != PAGE_SHIFT)
+	{
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (idx=0; idx < ui32NumOfPages; idx++)
+	{
+		if (pbValid[idx])
+		{
+			IMG_UINT32 ui32InPageOffset;
+
+			ui32PageIndex = puiOffset[idx] >> PAGE_SHIFT;
+			ui32InPageOffset = puiOffset[idx] - ((IMG_DEVMEM_OFFSET_T)ui32PageIndex << PAGE_SHIFT);
+
+			PVR_LOG_RETURN_IF_FALSE(ui32PageIndex < psPrivData->ui32VirtPageCount,
+			                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+			PVR_ASSERT(ui32InPageOffset < PAGE_SIZE);
+			psDevPAddr[idx].uiAddr = psPrivData->pasDevPhysAddr[ui32PageIndex].uiAddr + ui32InPageOffset;
+#if defined(SUPPORT_STATIC_IPA)
+			/* Modify the physical address with the associated IPA values */
+			psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+			psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+		}
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+				  size_t uiOffset,
+				  size_t uiSize,
+				  void **ppvKernelAddressOut,
+				  IMG_HANDLE *phHandleOut,
+				  PMR_FLAGS_T ulFlags)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if (psPrivData->ui32PhysPageCount != psPrivData->ui32VirtPageCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Kernel mappings for sparse DMABufs "
+				"are not allowed!", __func__));
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail;
+	}
+
+	err = dma_buf_begin_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	if (err)
+	{
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail;
+	}
+
+	_pvr_dma_resv_lock(psDmaBuf);
+	err = dma_buf_vmap(psDmaBuf, &psPrivData->sMap);
+	_pvr_dma_resv_unlock(psDmaBuf);
+
+	if (err != 0 || psPrivData->sMap.vaddr == NULL)
+	{
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail_kmap;
+	}
+
+	*ppvKernelAddressOut = psPrivData->sMap.vaddr + uiOffset;
+	*phHandleOut = psPrivData->sMap.vaddr;
+
+	return PVRSRV_OK;
+
+fail_kmap:
+	do {
+		err = dma_buf_end_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	} while (err == -EAGAIN || err == -EINTR);
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+					      IMG_HANDLE hHandle)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	int err;
+
+	_pvr_dma_resv_lock(psDmaBuf);
+	dma_buf_vunmap(psDmaBuf, &psPrivData->sMap);
+	_pvr_dma_resv_unlock(psDmaBuf);
+
+	do {
+		err = dma_buf_end_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	} while (err == -EAGAIN || err == -EINTR);
+}
+
+static PVRSRV_ERROR PMRMMapDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+                                  PMR *psPMR,
+                                  PMR_MMAP_DATA pOSMMapData)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	struct vm_area_struct *psVma = pOSMMapData;
+	int err;
+
+	if (psPrivData->ui32PhysPageCount != psPrivData->ui32VirtPageCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Not possible to MMAP sparse DMABufs",
+				__func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	err = dma_buf_mmap(psDmaBuf, psVma, 0);
+	if (err)
+	{
+		return (err == -EINVAL) ? PVRSRV_ERROR_NOT_SUPPORTED : PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	MMapStatsAddOrUpdatePMR(psPMR, psVma->vm_end - psVma->vm_start);
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRDmaBufFuncTab =
+{
+	.pfnLockPhysAddresses = PMRLockPhysAddressesDmaBuf,
+	.pfnUnlockPhysAddresses = PMRUnlockPhysAddressesDmaBuf,
+	.pfnDevPhysAddr = PMRDevPhysAddrDmaBuf,
+	.pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataDmaBuf,
+	.pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataDmaBuf,
+	.pfnMMap = PMRMMapDmaBuf,
+	.pfnFinalize = PMRFinalizeDmaBuf,
+	.pfnGetPMRFactoryLock = PMRFactoryLock,
+	.pfnReleasePMRFactoryLock = PMRFactoryUnlock,
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = PMRZombifyDmaBufMem,
+#endif
+};
+
+/*****************************************************************************
+ *                          Public facing interface                          *
+ *****************************************************************************/
+
+PVRSRV_ERROR
+PhysmemCreateNewDmaBufBackedPMR(PHYS_HEAP *psHeap,
+                                struct dma_buf_attachment *psAttachment,
+                                PFN_DESTROY_DMABUF_PMR pfnDestroy,
+                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                IMG_PID uiPid,
+                                IMG_DEVMEM_SIZE_T uiChunkSize,
+                                IMG_UINT32 ui32NumPhysChunks,
+                                IMG_UINT32 ui32NumVirtChunks,
+                                IMG_UINT32 *pui32MappingTable,
+                                IMG_UINT32 ui32NameSize,
+                                const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                                PMR **ppsPMRPtr)
+{
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+	PMR_DMA_BUF_DATA *psPrivData;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_BOOL bZeroOnAlloc;
+	IMG_BOOL bPoisonOnAlloc;
+	IMG_BOOL bPoisonOnFree;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, j;
+	IMG_UINT32 uiPagesPerChunk = uiChunkSize >> PAGE_SHIFT;
+	IMG_UINT32 ui32PageCount = 0;
+	struct scatterlist *sg;
+	struct sg_table *table;
+	IMG_UINT32 uiSglOffset;
+	IMG_CHAR pszAnnotation[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_UINT32 ui32ActualDmaBufPageCount;
+
+	bZeroOnAlloc = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
+	bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags);
+#if defined(DEBUG)
+	bPoisonOnFree = PVRSRV_CHECK_POISON_ON_FREE(uiFlags);
+#else
+	bPoisonOnFree = IMG_FALSE;
+#endif
+	if (bZeroOnAlloc && bPoisonOnFree)
+	{
+		/* Zero on Alloc and Poison on Alloc are mutually exclusive */
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errReturn;
+	}
+
+	if (!PMRValidateSize((IMG_UINT64) ui32NumVirtChunks * uiChunkSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				 "PMR size exceeds limit #Chunks: %u ChunkSz %"IMG_UINT64_FMTSPECX"",
+				 ui32NumVirtChunks,
+				 uiChunkSize);
+		eError = PVRSRV_ERROR_PMR_TOO_LARGE;
+		goto errReturn;
+	}
+
+	psPrivData = OSAllocZMem(sizeof(*psPrivData));
+	if (psPrivData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto errReturn;
+	}
+
+	psPrivData->psPhysHeap = psHeap;
+	psPrivData->psAttachment = psAttachment;
+	psPrivData->pfnDestroy = pfnDestroy;
+	psPrivData->bPoisonOnFree = bPoisonOnFree;
+	psPrivData->uiOriginPID = uiPid;
+	psPrivData->ui32VirtPageCount =
+			(ui32NumVirtChunks * uiChunkSize) >> PAGE_SHIFT;
+
+	psPrivData->pasDevPhysAddr =
+			OSAllocZMem(sizeof(*(psPrivData->pasDevPhysAddr)) *
+			            psPrivData->ui32VirtPageCount);
+	if (!psPrivData->pasDevPhysAddr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate buffer for physical addresses (oom)",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto errFreePrivData;
+	}
+
+	if (bZeroOnAlloc || bPoisonOnAlloc)
+	{
+		int iValue = bZeroOnAlloc ? 0 : PVRSRV_POISON_ON_ALLOC_VALUE;
+		int err;
+
+		err = DmaBufSetValue(psDmaBuf, iValue, __func__);
+		if (err)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map buffer for %s",
+			                        __func__,
+			                        bZeroOnAlloc ? "zeroing" : "poisoning"));
+
+			eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+			goto errFreePhysAddr;
+		}
+	}
+
+	eError = _pvr_dma_buf_map_attachment(psAttachment, DMA_BIDIRECTIONAL, &table);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_pvr_dma_buf_map_attachment", errFreePhysAddr);
+
+	/*
+	 * We do a two pass process: first work out how many pages there
+	 * are and second, fill in the data.
+	 */
+	for_each_sg(table->sgl, sg, table->nents, i)
+	{
+		ui32PageCount += PAGE_ALIGN(pvr_sg_length(sg)) / PAGE_SIZE;
+	}
+
+	if (WARN_ON(!ui32PageCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of phys. pages must not be zero",
+				 __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errUnmap;
+	}
+
+	/* Obtain actual page count of dma buf */
+	ui32ActualDmaBufPageCount = psAttachment->dmabuf->size / PAGE_SIZE;
+
+	if (WARN_ON(ui32ActualDmaBufPageCount < ui32NumPhysChunks * uiPagesPerChunk))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Requested physical chunks greater than "
+				"number of physical dma buf pages",
+				 __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errUnmap;
+	}
+
+	psPrivData->ui32PhysPageCount = ui32ActualDmaBufPageCount;
+	psPrivData->psSgTable = table;
+	ui32PageCount = 0;
+	sg = table->sgl;
+	uiSglOffset = 0;
+
+
+	/* Fill physical address array */
+	for (i = 0; i < ui32NumPhysChunks; i++)
+	{
+		for (j = 0; j < uiPagesPerChunk; j++)
+		{
+			IMG_UINT32 uiIdx = pui32MappingTable[i] * uiPagesPerChunk + j;
+
+			psPrivData->pasDevPhysAddr[uiIdx].uiAddr =
+					sg_dma_address(sg) + uiSglOffset;
+
+			/* Get the next offset for the current sgl or the next sgl */
+			uiSglOffset += PAGE_SIZE;
+			if (uiSglOffset >= pvr_sg_length(sg))
+			{
+				sg = sg_next(sg);
+				uiSglOffset = 0;
+
+				/* Check that we haven't looped */
+				if (WARN_ON(sg == table->sgl))
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to fill phys. address "
+							"array",
+							 __func__));
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					goto errUnmap;
+				}
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+#endif
+
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+
+	/*
+	 * Check no significant bits were lost in cast due to different
+	 * bit widths for flags
+	 */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (OSSNPrintf((IMG_CHAR *)pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN, "ImpDmaBuf:%s", (IMG_CHAR *)pszName) < 0)
+	{
+		pszAnnotation[0] = '\0';
+	}
+	else
+	{
+		pszAnnotation[DEVMEM_ANNOTATION_MAX_LEN-1] = '\0';
+	}
+
+	eError = PMRCreatePMR(psHeap,
+			      ui32NumVirtChunks * uiChunkSize,
+			      ui32NumPhysChunks,
+			      ui32NumVirtChunks,
+			      pui32MappingTable,
+			      PAGE_SHIFT,
+			      uiPMRFlags,
+			      pszAnnotation,
+			      &_sPMRDmaBufFuncTab,
+			      psPrivData,
+			      PMR_TYPE_DMABUF,
+			      ppsPMRPtr,
+			      PDUMP_NONE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create PMR (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		goto errFreePhysAddr;
+	}
+
+	return PVRSRV_OK;
+
+errUnmap:
+	_pvr_dma_buf_unmap_attachment(psAttachment, table, DMA_BIDIRECTIONAL);
+errFreePhysAddr:
+	OSFreeMem(psPrivData->pasDevPhysAddr);
+errFreePrivData:
+	OSFreeMem(psPrivData);
+errReturn:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PhysmemDestroyDmaBuf(PHYS_HEAP *psHeap,
+                                 struct dma_buf_attachment *psAttachment)
+{
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+
+	PVR_UNREFERENCED_PARAMETER(psHeap);
+
+	/* PMRUnlockSysPhysAddresses(psPMR) is redundant.
+	 * See PMRUnlockPhysAddressesDmaBuf */
+
+	dma_buf_detach(psDmaBuf, psAttachment);
+	dma_buf_put(psDmaBuf);
+}
+
+struct dma_buf *
+PhysmemGetDmaBuf(PMR *psPMR)
+{
+	PMR_DMA_BUF_DATA *psPrivData;
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData)
+	{
+		return psPrivData->psAttachment->dmabuf;
+	}
+
+	return NULL;
+}
+
+struct dma_resv *
+PhysmemGetDmaResv(PMR *psPMR)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	if (psDmaBuf)
+	{
+		return psDmaBuf->resv;
+	}
+	else
+	{
+		PMR_DMA_BUF_WRAPPER *psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+
+		if (psPMRWrapper)
+		{
+			return &psPMRWrapper->sDmaResv;
+
+		}
+
+		return NULL;
+	}
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+struct dma_heap *
+PhysmemGetDmaHeap(PMR *psPMR)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData != NULL)
+	{
+		return psPrivData->psDmaHeap;
+	}
+
+	return NULL;
+}
+
+void
+PhysmemSetDmaHeap(PMR *psPMR, struct dma_heap *psDmaHeap)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData != NULL)
+	{
+		psPrivData->psDmaHeap = psDmaHeap;
+	}
+}
+#endif /* #if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND) */
+
+static void
+PhysmemDestroyPMRWrapper(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	dma_resv_fini(&psPMRWrapper->sDmaResv);
+
+	OSLockDestroy(psPMRWrapper->hKernelMappingLock);
+	OSLockDestroy(psPMRWrapper->hDeviceMappingLock);
+	OSFreeMem(psPMRWrapper);
+}
+
+void
+PhysmemDmaBufExportFinalize(void *pvDmaBufExportData)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = pvDmaBufExportData;
+
+	PhysmemDestroyPMRWrapper(psPMRWrapper);
+}
+
+static PVRSRV_ERROR
+PhysmemCreatePMRWrapper(PMR *psPMR, PMR_DMA_BUF_WRAPPER **ppsPMRWrapper)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	psPMRWrapper = OSAllocZMem(sizeof(*psPMRWrapper));
+	PVR_LOG_GOTO_IF_NOMEM(psPMRWrapper, eError, fail_alloc_mem);
+
+	psPMRWrapper->psPMR = psPMR;
+
+	eError = OSLockCreate(&psPMRWrapper->hKernelMappingLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.1",
+	                      fail_kernel_mapping_lock_create);
+
+	eError = OSLockCreate(&psPMRWrapper->hDeviceMappingLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.2",
+	                      fail_device_mapping_lock_create);
+
+	dma_resv_init(&psPMRWrapper->sDmaResv);
+
+	*ppsPMRWrapper = psPMRWrapper;
+
+	return PVRSRV_OK;
+fail_device_mapping_lock_create:
+	OSLockDestroy(psPMRWrapper->hKernelMappingLock);
+fail_kernel_mapping_lock_create:
+	OSFreeMem(psPMRWrapper);
+fail_alloc_mem:
+	return eError;
+}
+
+static PVRSRV_ERROR
+PhysmemGetOrCreatePMRWrapper(PMR *psPMR, PMR_DMA_BUF_WRAPPER **ppsPMRWrapper)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+	if (!psPMRWrapper)
+	{
+		PMRFactoryLock();
+
+		/* Check again with the factory lock held */
+		psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+		if (!psPMRWrapper)
+		{
+			eError = PhysmemCreatePMRWrapper(psPMR, &psPMRWrapper);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreatePMRWrapper", fail_create_pmr_wrapper);
+
+			/*
+			 * A PMR memory layout can't change once exported.
+			 * This makes sure the exported and imported parties
+			 * see the same layout of the memory.
+			 */
+			PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+			PMREnvDmaBufSetExportData(psPMR, psPMRWrapper);
+		}
+
+		PMRFactoryUnlock();
+	}
+
+	*ppsPMRWrapper = psPMRWrapper;
+
+	return PVRSRV_OK;
+
+fail_create_pmr_wrapper:
+	PMRFactoryUnlock();
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemExportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    PMR *psPMR,
+                    IMG_INT *piFd)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+
+	eError = PhysmemGetOrCreatePMRWrapper(psPMR, &psPMRWrapper);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemExportDmaBuf", fail_get_pmr_wrapper);
+
+	eError = PMRRefPMR(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", fail_ref_pmr);
+
+	{
+		DEFINE_DMA_BUF_EXPORT_INFO(sDmaBufExportInfo);
+
+		sDmaBufExportInfo.priv  = psPMRWrapper;
+		sDmaBufExportInfo.ops   = &sPVRDmaBufOps;
+		sDmaBufExportInfo.size  = PMR_LogicalSize(psPMR);
+		sDmaBufExportInfo.flags = O_RDWR;
+		sDmaBufExportInfo.resv  = &psPMRWrapper->sDmaResv;
+
+		psDmaBuf = dma_buf_export(&sDmaBufExportInfo);
+	}
+
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to export buffer (err=%ld)",
+		         __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_export;
+	}
+
+	iFd = dma_buf_fd(psDmaBuf, O_RDWR);
+	if (iFd < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf fd (err=%d)",
+		         __func__, iFd));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_dma_buf;
+	}
+
+	*piFd = iFd;
+
+	return PVRSRV_OK;
+
+fail_dma_buf:
+	dma_buf_put(psDmaBuf);
+	return eError;
+
+fail_export:
+	(void) PMRUnrefPMR(psPMR);
+
+fail_ref_pmr:
+	/* No need to destroy psPMRWrapper here. It will be (or already was)
+	 * destroyed alongside the PMR. */
+fail_get_pmr_wrapper:
+	return eError;
+}
+
+struct dma_buf *
+PhysmemGEMPrimeExport(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+			struct drm_device *psDev,
+#endif
+
+			struct drm_gem_object *psObj,
+			int iFlags)
+{
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	struct dma_buf *psDmaBuf = NULL;
+	DEFINE_DMA_BUF_EXPORT_INFO(sDmaBufExportInfo);
+
+
+	sDmaBufExportInfo.priv  = psObj;
+	sDmaBufExportInfo.ops   = &sPVRDmaBufOpsGEM;
+	sDmaBufExportInfo.size  = PMR_LogicalSize(psPMRWrapper->psPMR);
+	sDmaBufExportInfo.flags = iFlags;
+	sDmaBufExportInfo.resv  = &psPMRWrapper->sDmaResv;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	psDmaBuf = drm_gem_dmabuf_export(psObj->dev, &sDmaBufExportInfo);
+#else
+	psDmaBuf = drm_gem_dmabuf_export(psDev, &sDmaBufExportInfo);
+#endif
+	if (IS_ERR(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to export buffer (err=%ld)",
+		         __func__, PTR_ERR(psDmaBuf)));
+	}
+
+	return psDmaBuf;
+}
+
+void
+PhysmemGEMObjectFree(struct drm_gem_object *psObj)
+{
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	drm_gem_object_release(psObj);
+
+	if (smp_load_acquire(&psPMRWrapper->psObj) == psObj)
+	{
+		smp_store_release(&psPMRWrapper->psObj, NULL);
+	}
+
+	eError = PMRUnrefPMR(psPMRWrapper->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+	OSFreeMem(psGEMObj);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static const struct drm_gem_object_funcs sPhysmemGEMObjFuncs = {
+	.export = PhysmemGEMPrimeExport,
+	.free = PhysmemGEMObjectFree,
+};
+#endif
+
+PVRSRV_ERROR
+PhysmemExportGemHandle(CONNECTION_DATA *psConnection,
+		       PVRSRV_DEVICE_NODE *psDevNode,
+		       PMR *psPMR,
+		       IMG_UINT32 *puHandle)
+{
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	struct drm_device *psDRMDev = dev_get_drvdata(psDev);
+	struct drm_file *psDRMFile = OSGetDRMFile(psConnection);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj;
+	bool bAlreadyExported;
+	PVRSRV_ERROR eError;
+	int iErr;
+
+	eError = PhysmemGetOrCreatePMRWrapper(psPMR, &psPMRWrapper);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemExportGemHandle", fail_get_pmr_wrapper);
+
+	psGEMObj = OSAllocZMem(sizeof(*psGEMObj));
+	PVR_LOG_GOTO_IF_NOMEM(psGEMObj, eError, fail_alloc_mem);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	psGEMObj->sBase.funcs = &sPhysmemGEMObjFuncs;
+#endif
+	psGEMObj->psPMRWrapper = psPMRWrapper;
+
+	eError = PMRRefPMR(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", fail_ref_pmr);
+
+	drm_gem_private_object_init(psDRMDev, &psGEMObj->sBase,
+	                            PMR_LogicalSize(psPMR));
+
+	PMRFactoryLock();
+	bAlreadyExported = smp_load_acquire(&psPMRWrapper->psObj) != NULL;
+	if (!bAlreadyExported)
+	{
+		smp_store_release(&psPMRWrapper->psObj, &psGEMObj->sBase);
+	}
+	PMRFactoryUnlock();
+
+	if (bAlreadyExported)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_STILL_REFERENCED,
+		                    fail_export_check);
+	}
+
+	iErr = drm_gem_handle_create(psDRMFile, &psGEMObj->sBase, puHandle);
+	if (iErr)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY,
+		                    fail_handle_create);
+	}
+
+	/* The handle holds a reference on the object, so drop ours */
+	drm_gem_object_put(&psGEMObj->sBase);
+
+	return PVRSRV_OK;
+
+fail_export_check:
+fail_handle_create:
+	/* Decrement reference on the GEM object and call .free()
+	 * (PhysmemGEMObjectFree()). This will unreference the PMR and call
+	 * OSMemFree() on psGEMObj. */
+	drm_gem_object_put(&psGEMObj->sBase);
+	return eError;
+
+fail_ref_pmr:
+	OSFreeMem(psGEMObj);
+fail_alloc_mem:
+fail_get_pmr_wrapper:
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_INT fd,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    IMG_UINT32 ui32NameSize,
+                    const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                    PMR **ppsPMRPtr,
+                    IMG_DEVMEM_SIZE_T *puiSize,
+                    IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 ui32MappingTable = 0;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+
+	/* Get the buffer handle */
+	psDmaBuf = dma_buf_get(fd);
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf from fd (err=%ld)",
+				 __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		return PVRSRV_ERROR_BAD_MAPPING;
+
+	}
+
+	uiSize = psDmaBuf->size;
+
+	eError = PhysmemImportSparseDmaBuf(psConnection,
+	                                 psDevNode,
+	                                 fd,
+	                                 uiFlags,
+	                                 uiSize,
+	                                 1,
+	                                 1,
+	                                 &ui32MappingTable,
+	                                 ui32NameSize,
+	                                 pszName,
+	                                 ppsPMRPtr,
+	                                 puiSize,
+	                                 puiAlign);
+
+	dma_buf_put(psDmaBuf);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemImportSparseDmaBuf(CONNECTION_DATA *psConnection,
+                          PVRSRV_DEVICE_NODE *psDevNode,
+                          IMG_INT fd,
+                          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                          IMG_DEVMEM_SIZE_T uiChunkSize,
+                          IMG_UINT32 ui32NumPhysChunks,
+                          IMG_UINT32 ui32NumVirtChunks,
+                          IMG_UINT32 *pui32MappingTable,
+                          IMG_UINT32 ui32NameSize,
+                          const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                          PMR **ppsPMRPtr,
+                          IMG_DEVMEM_SIZE_T *puiSize,
+                          IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PMR *psPMR = NULL;
+	struct dma_buf_attachment *psAttachment;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bHashTableCreated = IMG_FALSE;
+	IMG_PID uiPid = OSGetCurrentClientProcessIDKM();
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, errReturn);
+
+	/* Terminate string from bridge to prevent corrupt annotations in RI */
+	if (pszName != NULL)
+	{
+		IMG_CHAR* pszName0 = (IMG_CHAR*) pszName;
+		pszName0[ui32NameSize-1] = '\0';
+	}
+
+	PMRFactoryLock();
+
+	/* Get the buffer handle */
+	psDmaBuf = dma_buf_get(fd);
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf from fd (err=%ld)",
+		        __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, errUnlockReturn);
+	}
+
+	if (psDmaBuf->ops == &sPVRDmaBufOps || psDmaBuf->ops == &sPVRDmaBufOpsGEM)
+	{
+		/* We exported this dma_buf, so we can just get its PMR. */
+		if (psDmaBuf->ops == &sPVRDmaBufOps)
+		{
+			PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+			psPMR = psPMRWrapper->psPMR;
+		}
+		else
+		{
+			struct drm_gem_object *psObj = psDmaBuf->priv;
+			PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+			PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+			psPMR = psPMRWrapper->psPMR;
+		}
+
+		/* However, we can't import it if it belongs to a different device. */
+		if (PMR_DeviceNode(psPMR) != psDevNode)
+		{
+#if defined(PVRSRV_ENABLE_XD_MEM)
+			/* and if it's not in a shareable physical address space. */
+			if (PhysHeapSpasWithDevice(PMR_PhysHeap(psPMR), psDevNode) != PVRSRV_OK)
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device",
+				        __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PERMITTED,
+				                    errUnlockAndDMAPut);
+			}
+		}
+	}
+	else if (g_psDmaBufHash != NULL)
+	{
+		/* We have a hash table so check if we've seen this dmabuf
+		 * before. */
+		psPMR = (PMR *) HASH_Retrieve(g_psDmaBufHash, (uintptr_t) psDmaBuf);
+	}
+	else
+	{
+		/* As different processes may import the same dmabuf we need to
+		 * create a hash table so we don't generate a duplicate PMR but
+		 * rather just take a reference on an existing one. */
+		g_psDmaBufHash = HASH_Create(DMA_BUF_HASH_SIZE);
+		PVR_GOTO_IF_NOMEM(g_psDmaBufHash, eError, errUnlockAndDMAPut);
+
+		bHashTableCreated = IMG_TRUE;
+	}
+
+	if (psPMR != NULL)
+	{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		if (PMR_IsZombie(psPMR))
+		{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			PMR_DMA_BUF_DATA *psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+#endif
+
+			PMRReviveZombieAndRef(psPMR);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			if (psPrivData != NULL)
+			{
+				PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+				                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+				                            psPrivData->uiOriginPID);
+				PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+				                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+				                            psPrivData->uiOriginPID);
+			}
+#endif
+
+			PVRSRVIonReviveMemAllocRecord(psDmaBuf);
+		}
+		else
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+		{
+			/* Reuse the PMR we already created */
+			eError = PMRRefPMR(psPMR);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", errUnlockAndDMAPut);
+		}
+
+		/* If an existing PMR is found, the table wasn't created by this func
+		 * call, so the error path can be safely ignored allowing for
+		 * the factory lock can be dropped. */
+		PVR_ASSERT(bHashTableCreated == IMG_FALSE);
+		dma_buf_put(psDmaBuf);
+		PMRFactoryUnlock();
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+		/* The device import can only be registered on an alive & healthy PMR
+		 * therefore we wait for the potential zombie to be dequeued first. */
+		eError = PMR_RegisterDeviceImport(psPMR, psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			/* The factory lock might be taken in PMRUnrefPMR. */
+			(void) PMRUnrefPMR(psPMR);
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to register PMR with device: %u",
+			        __func__, psDevNode->sDevId.ui32InternalID));
+			goto errReturn;
+		}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+		*ppsPMRPtr = psPMR;
+		*puiSize = PMR_LogicalSize(psPMR);
+		*puiAlign = PAGE_SIZE;
+
+		/* We expect a PMR to be immutable at this point.
+		 * But its explicitly set here to cover a corner case
+		 * where a PMR created through non-DMA interface could be
+		 * imported back again through DMA interface. */
+		PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+		return PVRSRV_OK;
+	}
+
+	/* Do we want this to be a sparse PMR? */
+	if (ui32NumVirtChunks > 1)
+	{
+		/* Parameter validation */
+		if (psDmaBuf->size < (uiChunkSize * ui32NumPhysChunks) ||
+		    uiChunkSize != PAGE_SIZE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Requesting sparse buffer: "
+					"uiChunkSize ("IMG_DEVMEM_SIZE_FMTSPEC") must be equal to "
+					"OS page size (%lu). uiChunkSize * ui32NumPhysChunks "
+					"("IMG_DEVMEM_SIZE_FMTSPEC") must"
+					" not be greater than the buffer size ("IMG_SIZE_FMTSPEC").",
+					 __func__,
+					uiChunkSize,
+					PAGE_SIZE,
+					uiChunkSize * ui32NumPhysChunks,
+					psDmaBuf->size));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS,
+			                    errUnlockAndDMAPut);
+		}
+	}
+	else
+	{
+		/* if ui32NumPhysChunks == 0 then pui32MappingTable == NULL
+		 * this is handled by the generated bridge code.
+		 * Because ui32NumPhysChunks is set to 1 below, we don't allow NULL array */
+		if (pui32MappingTable == NULL)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS,
+			                    errUnlockAndDMAPut);
+		}
+
+		/* Make sure parameters are valid for non-sparse allocations as well */
+		uiChunkSize = psDmaBuf->size;
+		ui32NumPhysChunks = 1;
+		ui32NumVirtChunks = 1;
+	}
+
+	{
+		IMG_DEVMEM_SIZE_T uiSize = ui32NumVirtChunks * uiChunkSize;
+		IMG_UINT32 uiLog2PageSize = PAGE_SHIFT; /* log2(uiChunkSize) */
+		eError = PhysMemValidateParams(psDevNode,
+		                               ui32NumPhysChunks,
+		                               ui32NumVirtChunks,
+		                               pui32MappingTable,
+		                               uiFlags,
+		                               uiPid,
+		                               &uiLog2PageSize,
+		                               &uiSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysMemValidateParams", errUnlockAndDMAPut);
+	}
+
+	psAttachment = dma_buf_attach(psDmaBuf, psDevNode->psDevConfig->pvOSDevice);
+	if (IS_ERR_OR_NULL(psAttachment))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to attach to dma-buf (err=%ld)",
+				 __func__, psAttachment? PTR_ERR(psAttachment) : -ENOMEM));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DMABUF_ATTACH,
+		                    errUnlockAndDMAPut);
+	}
+
+	/*
+	 * Note:
+	 * While we have no way to determine the type of the buffer we just
+	 * assume that all dmabufs are from the same physical heap.
+	 */
+	eError = PhysmemCreateNewDmaBufBackedPMR(psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_EXTERNAL],
+	                                         psAttachment,
+	                                         PhysmemDestroyDmaBuf,
+	                                         uiFlags,
+	                                         uiPid,
+	                                         uiChunkSize,
+	                                         ui32NumPhysChunks,
+	                                         ui32NumVirtChunks,
+	                                         pui32MappingTable,
+	                                         ui32NameSize,
+	                                         pszName,
+	                                         &psPMR);
+	PVR_GOTO_IF_ERROR(eError, errDMADetach);
+
+	/* The imported memory is assumed to be backed by a permanent allocation. */
+	/* PMRLockSysPhysAddresses(psPMR) is redundant.
+	 * See PMRLockPhysAddressesDmaBuf */
+
+	/* First time we've seen this dmabuf so store it in the hash table */
+	HASH_Insert(g_psDmaBufHash, (uintptr_t) psDmaBuf, (uintptr_t) psPMR);
+	g_ui32HashRefCount++;
+
+	PMRFactoryUnlock();
+
+	PVRSRVIonAddMemAllocRecord(psDmaBuf);
+
+	*ppsPMRPtr = psPMR;
+	*puiSize = ui32NumVirtChunks * uiChunkSize;
+	*puiAlign = PAGE_SIZE;
+
+	/* The memory that's just imported is owned by some other entity.
+	 * Hence the memory layout cannot be changed through our API */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	return PVRSRV_OK;
+
+errDMADetach:
+	dma_buf_detach(psDmaBuf, psAttachment);
+
+errUnlockAndDMAPut:
+	if (bHashTableCreated)
+	{
+		HASH_Delete(g_psDmaBufHash);
+		g_psDmaBufHash = NULL;
+	}
+	dma_buf_put(psDmaBuf);
+
+errUnlockReturn:
+	PMRFactoryUnlock();
+
+errReturn:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf_internal.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf_internal.h
new file mode 100644
index 000000000000..50a8ab37f442
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf_internal.h
@@ -0,0 +1,69 @@
+/**************************************************************************/ /*!
+@File           physmem_dmabuf_internal.h
+@Title          Internal header for dmabuf PMR factory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks importing Ion allocations
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PHYSMEM_DMABUF_INTERNAL_H)
+#define PHYSMEM_DMABUF_INTERNAL_H
+
+#include <linux/version.h>
+
+struct dma_buf;
+struct drm_device;
+struct drm_gem_object;
+
+struct dma_buf *
+PhysmemGEMPrimeExport(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+			struct drm_device *psDev,
+#endif
+
+			struct drm_gem_object *psObj,
+			int iFlags);
+
+void
+PhysmemGEMObjectFree(struct drm_gem_object *psObj);
+
+void
+PhysmemDmaBufExportFinalize(void *pvDmaBufExportData);
+
+#endif /* !defined(PHYSMEM_DMABUF_INTERNAL_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_linux.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_linux.c
new file mode 100644
index 000000000000..994f675329b8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_linux.c
@@ -0,0 +1,1077 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR functions to wrap non-services allocated
+                memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                implementing the function callbacks for physical memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/uaccess.h>
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "devicemem_server_utils.h"
+
+#include "physmem_extmem.h"
+#include "physmem_extmem_wrap.h"
+#include "pdump_physmem.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "physmem.h"
+#include "cache_km.h"
+
+#include "kernel_compatibility.h"
+
+typedef struct _WRAP_KERNEL_MAP_DATA_
+{
+	void *pvKernLinAddr;
+	IMG_BOOL bVMAP;
+} WRAP_KERNEL_MAP_DATA;
+
+/* Free the PMR private data */
+static void _FreeWrapData(PMR_WRAP_DATA *psPrivData)
+{
+	OSFreeMem(psPrivData->ppsPageArray);
+	OSFreeMem(psPrivData->ppvPhysAddr);
+	psPrivData->psVMArea = NULL;
+	OSFreeMem(psPrivData);
+}
+
+
+/* Allocate the PMR private data */
+static PVRSRV_ERROR _AllocWrapData(PMR_WRAP_DATA **ppsPrivData,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_CPU_VIRTADDR pvCpuVAddr,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_WRAP_DATA *psPrivData;
+	struct vm_area_struct *psVMArea;
+	IMG_UINT32 ui32CPUCacheMode;
+
+	/* Obtain a reader lock on the process mmap lock while we are using
+	 * the psVMArea.
+	 */
+
+	mmap_read_lock(current->mm);
+
+	/* Find the VMA */
+	psVMArea = find_vma(current->mm, (uintptr_t)pvCpuVAddr);
+	if (psVMArea == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Couldn't find memory region containing start address %p",
+				__func__,
+				(void*) pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto eUnlockReturn;
+	}
+
+	/* If requested size is larger than actual allocation
+	 * return error. Can never request more memory to be imported
+	 * than its original allocation */
+	/* Now check the end address is in range */
+	if (((uintptr_t)pvCpuVAddr + uiSize) > psVMArea->vm_end)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: End address %p is outside of the region returned by find_vma",
+				__func__,
+				(void*) (uintptr_t)((uintptr_t)pvCpuVAddr + uiSize)));
+		eError = PVRSRV_ERROR_BAD_PARAM_SIZE;
+		goto eUnlockReturn;
+	}
+
+	/* Find_vma locates a region with an end point past a given
+	 * virtual address. So check the address is actually in the region. */
+	if ((uintptr_t)pvCpuVAddr < psVMArea->vm_start)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Start address %p is outside of the region returned by find_vma",
+				__func__,
+				(void*) pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto eUnlockReturn;
+	}
+
+	eError = DevmemCPUCacheMode(uiFlags, &ui32CPUCacheMode);
+	if (eError != PVRSRV_OK)
+	{
+		goto eUnlockReturn;
+	}
+
+	/* Allocate and initialise private factory data */
+	psPrivData = OSAllocZMem(sizeof(*psPrivData));
+	if (psPrivData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	psPrivData->ui32CPUCacheFlags = ui32CPUCacheMode;
+
+	/* Track the VMA area structure so that it can be checked later */
+	psPrivData->psVMArea = psVMArea;
+
+	psPrivData->psDevNode = psDevNode;
+	psPrivData->uiTotalNumPages = uiSize >> PAGE_SHIFT;
+
+	/* Allocate page and phys address arrays */
+	psPrivData->ppsPageArray = OSAllocZMem(sizeof(*(psPrivData->ppsPageArray)) * psPrivData->uiTotalNumPages);
+	if (psPrivData == NULL)
+	{
+		OSFreeMem(psPrivData);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	psPrivData->ppvPhysAddr = OSAllocZMem(sizeof(*(psPrivData->ppvPhysAddr)) * psPrivData->uiTotalNumPages);
+	if (psPrivData->ppvPhysAddr == NULL)
+	{
+		OSFreeMem(psPrivData->ppsPageArray);
+		OSFreeMem(psPrivData);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	if (uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+					PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED))
+	{
+		psPrivData->bWrite = IMG_TRUE;
+	}
+
+	*ppsPrivData = psPrivData;
+
+	eError = PVRSRV_OK;
+eUnlockReturn:
+	mmap_read_unlock(current->mm);
+
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+/* Free the pages we got via _TryFindVMA() */
+static void _FreeFindVMAPages(PMR_WRAP_DATA *psPrivData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i] != NULL)
+		{
+			/* Release the page */
+			put_page(psPrivData->ppsPageArray[i]);
+		}
+	}
+}
+#endif
+
+/* Release pages got via get_user_pages().
+ * As the function comment for GUP states we have
+ * to set the page dirty if it has been written to and
+ * we have to put the page to decrease its refcount. */
+static void _FreeGetUserPages(PMR_WRAP_DATA *psPrivData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i])
+		{
+			if (IMG_TRUE == psPrivData->bWrite)
+			{
+				/* Write data back to fs if necessary */
+				set_page_dirty_lock(psPrivData->ppsPageArray[i]);
+			}
+
+			/* Release the page */
+			put_page(psPrivData->ppsPageArray[i]);
+		}
+	}
+}
+
+/* Get the page structures and physical addresses mapped to
+ * a CPU virtual range via get_user_pages() */
+static PVRSRV_ERROR _TryGetUserPages(PVRSRV_DEVICE_NODE *psDevNode,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_CPU_VIRTADDR pvCpuVAddr,
+                                PMR_WRAP_DATA *psPrivData)
+{
+	IMG_INT32 iMappedPages, i;
+
+	IMG_UINT64 ui64DmaMask = dma_get_mask(psDevNode->psDevConfig->pvOSDevice);
+
+	/* Do the actual call */
+	iMappedPages = get_user_pages_fast((uintptr_t) pvCpuVAddr, uiSize >> PAGE_SHIFT,
+	                                    psPrivData->bWrite, psPrivData->ppsPageArray);
+	if (iMappedPages < 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "get_user_pages_fast() failed, got back %d, expected num pages %d",
+		         iMappedPages,
+		         psPrivData->uiTotalNumPages));
+
+		return PVRSRV_ERROR_FAILED_TO_ACQUIRE_PAGES;
+	}
+
+	/* Fill the physical address array */
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i])
+		{
+			psPrivData->ppvPhysAddr[i].uiAddr = page_to_phys(psPrivData->ppsPageArray[i]);
+			psPrivData->uiNumBackedPages += 1;
+		}
+		else
+		{
+			psPrivData->ppvPhysAddr[i].uiAddr = 0;
+		}
+
+
+		/* Check the data transfer capability of the DMA.
+		 *
+		 * For instance:
+		 * APOLLO test chips TCF5 or ES2 can only access 4G maximum memory from the card.
+		 * Hence pages with a physical address beyond 4G range cannot be accessed by the
+		 * device. An error is reported in such a case. */
+		if (psPrivData->ppvPhysAddr[i].uiAddr & ~ui64DmaMask)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Backed page @pos:%d Physical Address: %pa exceeds GPU "
+					"accessible range(mask): 0x%0llx",
+					i, &psPrivData->ppvPhysAddr[i].uiAddr, ui64DmaMask));
+
+			psPrivData->bWrite = IMG_FALSE;
+
+			/* Free the acquired pages */
+			_FreeGetUserPages(psPrivData);
+
+			return PVRSRV_ERROR_INVALID_PHYS_ADDR;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Release all the pages we got via _TryGetUserPages or _TryFindVMA */
+static PVRSRV_ERROR _WrapExtMemReleasePages(PMR_WRAP_DATA *psPrivData)
+{
+	switch (psPrivData->eWrapType)
+	{
+		case WRAP_TYPE_GET_USER_PAGES:
+			_FreeGetUserPages(psPrivData);
+			break;
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+		case WRAP_TYPE_FIND_VMA:
+			_FreeFindVMAPages(psPrivData);
+			break;
+#endif
+		case WRAP_TYPE_NULL:
+			/* fall through */
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Wrong wrap type, cannot release pages",
+					__func__));
+			return PVRSRV_ERROR_INVALID_WRAP_TYPE;
+	}
+
+	_FreeWrapData(psPrivData);
+
+	return PVRSRV_OK;
+}
+
+/* Try to get pages and physical addresses for a CPU virtual address.
+ * Will try both methods:
+ * get_user_pages or find_vma + page table walk if the first one fails */
+static PVRSRV_ERROR _WrapExtMemAcquirePages(PVRSRV_DEVICE_NODE *psDevNode,
+                                            IMG_DEVMEM_SIZE_T uiSize,
+                                            IMG_CPU_VIRTADDR pvCpuVAddr,
+                                            PMR_WRAP_DATA *psPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	eError = _TryGetUserPages(psDevNode,
+	                            uiSize,
+	                            pvCpuVAddr,
+	                            psPrivData);
+	if (eError == PVRSRV_OK)
+	{
+		psPrivData->eWrapType = WRAP_TYPE_GET_USER_PAGES;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"%s: Used GetUserPages",
+				__func__));
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+	if (PVRSRV_ERROR_INVALID_PHYS_ADDR != eError)
+	{
+		PVRSRV_ERROR eError2;
+		eError2 = _TryFindVMA(uiSize,
+		                      (uintptr_t) pvCpuVAddr,
+		                      psPrivData);
+		if (eError2 == PVRSRV_OK)
+		{
+			psPrivData->eWrapType = WRAP_TYPE_FIND_VMA;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Used FindVMA",
+					__func__));
+			return PVRSRV_OK;
+		}
+
+		PVR_WARN_IF_ERROR(eError, "_TryGetUserPages");
+		PVR_LOG_ERROR(eError2, "_TryFindVMA");
+	}
+	else
+	{
+		PVR_LOG_ERROR(eError, "_TryGetUserPages");
+	}
+#else
+	PVR_LOG_ERROR(eError, "_TryGetUserPages");
+#endif
+
+	psPrivData->eWrapType = WRAP_TYPE_NULL;
+	return eError;
+}
+
+static PVRSRV_ERROR
+PMRSysPhysAddrExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                     IMG_UINT32 ui32Log2PageSize,
+                     IMG_UINT32 ui32NumOfPages,
+                     IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+                     IMG_UINT64 ui64IPAPolicyValue,
+                     IMG_UINT64 ui64IPAClearMask,
+#endif
+                     IMG_BOOL *pbValid,
+                     IMG_DEV_PHYADDR *psDevPAddr)
+{
+	const PMR_WRAP_DATA *psWrapData = pvPriv;
+	IMG_UINT32 uiPageSize = 1U << PAGE_SHIFT;
+	IMG_UINT32 uiInPageOffset;
+	IMG_UINT32 uiPageIndex;
+	IMG_UINT32 uiIdx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+
+	if (PAGE_SHIFT != ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Requested ui32Log2PageSize %u is different from "
+				"OS page shift %u. Not supported.",
+				__func__,
+				ui32Log2PageSize,
+				PAGE_SHIFT));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
+	{
+		uiPageIndex = puiOffset[uiIdx] >> PAGE_SHIFT;
+		uiInPageOffset = puiOffset[uiIdx] - ((IMG_DEVMEM_OFFSET_T)uiPageIndex << PAGE_SHIFT);
+
+		PVR_LOG_RETURN_IF_FALSE(uiPageIndex < psWrapData->uiTotalNumPages,
+		                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+		PVR_ASSERT(uiInPageOffset < uiPageSize);
+
+		/* The ExtMem is only enabled in UMA mode, in that mode, the device physical
+		 * address translation will be handled by the PRM factory after this call.
+		 * Here we just copy the physical address like other callback implementations. */
+		psDevPAddr[uiIdx].uiAddr = psWrapData->ppvPhysAddr[uiPageIndex].uiAddr;
+
+		pbValid[uiIdx] = (psDevPAddr[uiIdx].uiAddr)? IMG_TRUE:IMG_FALSE;
+
+		psDevPAddr[uiIdx].uiAddr += uiInPageOffset;
+#if defined(SUPPORT_STATIC_IPA)
+		psDevPAddr[uiIdx].uiAddr &= ~ui64IPAClearMask;
+		psDevPAddr[uiIdx].uiAddr |= ui64IPAPolicyValue;
+#endif	/* SUPPORT_STATIC_IPA */
+	}
+
+	return PVRSRV_OK;
+}
+
+
+static void
+PMRFinalizeExtMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PMR_WRAP_DATA *psWrapData = pvPriv;
+
+	PVRSRV_ERROR eError = _WrapExtMemReleasePages(psWrapData);
+	PVR_LOG_IF_ERROR(eError, "_WrapExtMemReleasePages");
+}
+
+
+static void _UnmapPage(PMR_WRAP_DATA *psWrapData,
+                       WRAP_KERNEL_MAP_DATA *psMapData)
+{
+	IMG_BOOL bSuccess;
+
+	if (psMapData->bVMAP)
+	{
+		vunmap(psMapData->pvKernLinAddr);
+	}
+	else
+	{
+		if (psWrapData->ui32CPUCacheFlags & ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unable to unmap wrapped extmem "
+			        "page - wrong cached mode flags passed. This may leak "
+			        "memory.", __func__));
+			PVR_DPF((PVR_DBG_ERROR, "Found non-cpu cache mode flag when unmapping from "
+					 "the cpu"));
+		}
+		else
+		{
+			bSuccess = OSUnMapPhysToLin(psMapData->pvKernLinAddr, PAGE_SIZE);
+			if (!bSuccess)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Unable to unmap wrapped extmem "
+				        "page. This may leak memory.", __func__));
+			}
+		}
+	}
+}
+
+static void _MapPage(PMR_WRAP_DATA *psWrapData,
+                     IMG_UINT32 uiPageIdx,
+                     WRAP_KERNEL_MAP_DATA *psMapData)
+{
+	IMG_UINT8 *puiLinCPUAddr;
+
+	if (psWrapData->ppsPageArray[uiPageIdx])
+	{
+		pgprot_t prot = PAGE_KERNEL;
+
+		switch (PVRSRV_CPU_CACHE_MODE(psWrapData->ui32CPUCacheFlags))
+		{
+			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				prot = pgprot_noncached(prot);
+				break;
+
+			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				prot = pgprot_writecombine(prot);
+				break;
+
+			case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+				break;
+
+			default:
+				break;
+		}
+
+		puiLinCPUAddr = vmap(&psWrapData->ppsPageArray[uiPageIdx], 1, VM_MAP, prot);
+
+		psMapData->bVMAP = IMG_TRUE;
+	}
+	else
+	{
+		puiLinCPUAddr = (IMG_UINT8*) OSMapPhysToLin(psWrapData->ppvPhysAddr[uiPageIdx],
+		                                              PAGE_SIZE,
+		                                              psWrapData->ui32CPUCacheFlags);
+
+		psMapData->bVMAP = IMG_FALSE;
+	}
+
+	psMapData->pvKernLinAddr = puiLinCPUAddr;
+
+}
+
+static PVRSRV_ERROR _CopyBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                     IMG_DEVMEM_OFFSET_T uiOffset,
+                                     IMG_UINT8 *pcBuffer,
+                                     size_t uiBufSz,
+                                     size_t *puiNumBytes,
+                                     IMG_BOOL bWrite)
+{
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	size_t uiBytesToCopy = uiBufSz;
+	IMG_UINT32 uiPageIdx = uiOffset >> PAGE_SHIFT;
+	IMG_BOOL bFirst = IMG_TRUE;
+	WRAP_KERNEL_MAP_DATA sKernMapData;
+
+
+	if ((uiBufSz + uiOffset) > (psWrapData->uiTotalNumPages << PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Trying to read out of bounds of PMR.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Copy the pages */
+	while (uiBytesToCopy != 0)
+	{
+		size_t uiBytesPerLoop;
+		IMG_DEVMEM_OFFSET_T uiCopyOffset;
+
+		if (bFirst)
+		{
+			bFirst = IMG_FALSE;
+			uiCopyOffset = (uiOffset & ~PAGE_MASK);
+			uiBytesPerLoop = (uiBufSz >= (PAGE_SIZE - uiCopyOffset)) ?
+					PAGE_SIZE - uiCopyOffset : uiBufSz;
+		}
+		else
+		{
+			uiCopyOffset = 0;
+			uiBytesPerLoop = (uiBytesToCopy > PAGE_SIZE) ? PAGE_SIZE : uiBytesToCopy;
+		}
+
+		_MapPage(psWrapData,
+		         uiPageIdx,
+		         &sKernMapData);
+
+		if (sKernMapData.pvKernLinAddr == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unable to map wrapped extmem page.",
+					__func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+
+		/* Use 'DeviceMemCopy' because we need to be conservative. We can't
+		 * know whether the wrapped memory was originally imported as cached.
+		 */
+
+		if (bWrite)
+		{
+			OSDeviceMemCopy(sKernMapData.pvKernLinAddr + uiCopyOffset,
+			          pcBuffer,
+			          uiBytesPerLoop);
+		}
+		else
+		{
+			OSDeviceMemCopy(pcBuffer,
+			          sKernMapData.pvKernLinAddr + uiCopyOffset,
+			          uiBytesPerLoop);
+		}
+
+		_UnmapPage(psWrapData,
+		           &sKernMapData);
+
+		uiBytesToCopy -= uiBytesPerLoop;
+		uiPageIdx++;
+	}
+
+	*puiNumBytes = uiBufSz;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRReadBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_UINT8 *pcBuffer,
+                   size_t uiBufSz,
+                   size_t *puiNumBytes)
+{
+
+	return _CopyBytesExtMem(pvPriv,
+	                        uiOffset,
+	                        pcBuffer,
+	                        uiBufSz,
+	                        puiNumBytes,
+	                        IMG_FALSE);
+}
+
+static PVRSRV_ERROR
+PMRWriteBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_UINT8 *pcBuffer,
+                    size_t uiBufSz,
+                    size_t *puiNumBytes)
+{
+	return _CopyBytesExtMem(pvPriv,
+	                        uiOffset,
+	                        pcBuffer,
+	                        uiBufSz,
+	                        puiNumBytes,
+	                        IMG_TRUE);
+}
+
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                  size_t uiOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  IMG_HANDLE *phHandleOut,
+                                  PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	WRAP_KERNEL_MAP_DATA *psKernMapData;
+	IMG_UINT32 ui32PageIndex = uiOffset >> PAGE_SHIFT;
+	IMG_UINT32 ui32PageOffset = (IMG_UINT32)(uiOffset & ~PAGE_MASK);
+
+	/* Offset was out of bounds */
+	if (ui32PageIndex > psWrapData->uiTotalNumPages)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, offset out of PMR bounds.",
+				__func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	/* We can not map in more than one page with ioremap.
+	 * Only possible with physically contiguous pages */
+	if ((ui32PageOffset + uiSize) > PAGE_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, cannot map more than one page for wrapped extmem.",
+				__func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	psKernMapData = OSAllocMem(sizeof(*psKernMapData));
+	if (psKernMapData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, unable to allocate memory for kernel mapping data.",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	_MapPage(psWrapData,
+	         ui32PageIndex,
+	         psKernMapData);
+
+	if (psKernMapData->pvKernLinAddr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Unable to map wrapped extmem page.",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	*ppvKernelAddressOut = ((IMG_CHAR *) psKernMapData->pvKernLinAddr) + ui32PageOffset;
+	*phHandleOut = psKernMapData;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+e1:
+	OSFreeMem(psKernMapData);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                              IMG_HANDLE hHandle)
+{
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	WRAP_KERNEL_MAP_DATA *psKernMapData = (void*) hHandle;
+
+	_UnmapPage(psWrapData,
+	           psKernMapData);
+
+	OSFreeMem(psKernMapData);
+}
+
+static inline void begin_user_mode_access(IMG_UINT *uiState)
+{
+#if defined(CONFIG_ARM) && defined(CONFIG_CPU_SW_DOMAIN_PAN)
+	*uiState = uaccess_save_and_enable();
+#elif defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	uaccess_enable_privileged();
+#elif defined(CONFIG_X86)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	__uaccess_begin();
+#else
+	PVR_UNREFERENCED_PARAMETER(uiState);
+#endif
+
+}
+
+static inline void end_user_mode_access(IMG_UINT uiState)
+{
+#if defined(CONFIG_ARM) && defined(CONFIG_CPU_SW_DOMAIN_PAN)
+	uaccess_restore(uiState);
+#elif defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	uaccess_disable_privileged();
+#elif defined(CONFIG_X86)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	__uaccess_end();
+#else
+	PVR_UNREFERENCED_PARAMETER(uiState);
+#endif
+}
+
+static PVRSRV_ERROR _FlushUMVirtualRange(PVRSRV_DEVICE_NODE *psDevNode,
+							PMR_WRAP_DATA *psPrivData,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_CPU_VIRTADDR pvCpuVAddr)
+{
+	struct vm_area_struct *psVMArea;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT	uiUserAccessState=0;
+
+	mmap_read_lock(current->mm);
+
+	/* Check that the recorded psVMArea matches the one associated with
+	 * this request. If not, fail the request.
+	 */
+	psVMArea = find_vma(current->mm, (uintptr_t)pvCpuVAddr);
+	if ((psVMArea != psPrivData->psVMArea) || (psVMArea == NULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Couldn't find memory region containing start address %p",
+		         __func__, (void *)pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto UMFlushUnlockReturn;
+	}
+
+
+	/*
+	 * Latest kernels enable "Privileged access never" feature in the kernel
+	 * This features leads to a privilege access failure fault (oops) whenever
+	 * the kernel tries to access any user mode address.
+	 * .
+	 * This is an additional security feature that allows kernel
+	 * to be protected against possible software attacks.
+	 * In this case, as we need to flush the cache lines associated
+	 * with the user mode virtual address, we need to explicitly disable
+	 * the feature for the duration of access and re-enable it once done.
+	 * */
+	begin_user_mode_access(&uiUserAccessState);
+	{
+		if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart = {0};
+
+			eError = CacheOpExec(psDevNode,
+								pvCpuVAddr,
+								((IMG_UINT8 *)pvCpuVAddr + uiSize),
+								sCPUPhysStart,
+								sCPUPhysStart,
+								PVRSRV_CACHE_OP_FLUSH);
+		}
+		else if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+			IMG_UINT i = 0;
+
+			for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+			{
+				if (NULL != psPrivData->ppsPageArray[i])
+				{
+					sCPUPhysStart.uiAddr = psPrivData->ppvPhysAddr[i].uiAddr;
+					sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+
+					eError = CacheOpExec(psDevNode,
+										NULL,
+										NULL,
+										sCPUPhysStart,
+										sCPUPhysEnd,
+										PVRSRV_CACHE_OP_FLUSH);
+					if (eError != PVRSRV_OK)
+					{
+						break;
+					}
+				}
+			}
+		}
+		else if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_BOTH)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+			void *pvVirtStart, *pvVirtEnd;
+			IMG_UINT i = 0;
+
+			for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+			{
+				if (NULL != psPrivData->ppsPageArray[i])
+				{
+					pvVirtStart = pvCpuVAddr + (i * PAGE_SIZE);
+					pvVirtEnd = pvVirtStart + PAGE_SIZE;
+					sCPUPhysStart.uiAddr = psPrivData->ppvPhysAddr[i].uiAddr;
+					sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+
+					eError = CacheOpExec(psDevNode,
+										pvVirtStart,
+										pvVirtEnd,
+										sCPUPhysStart,
+										sCPUPhysEnd,
+										PVRSRV_CACHE_OP_FLUSH);
+					if (eError != PVRSRV_OK)
+					{
+						break;
+					}
+				}
+			}
+		}
+
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to clean the virtual region cache %p",
+					__func__,
+					pvCpuVAddr));
+			goto UMFlushFailed;
+		}
+	}
+
+UMFlushFailed:
+	end_user_mode_access(uiUserAccessState);
+
+UMFlushUnlockReturn:
+	mmap_read_unlock(current->mm);
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRWrapPFuncTab = {
+    .pfnLockPhysAddresses = NULL,
+    .pfnUnlockPhysAddresses = NULL,
+    .pfnDevPhysAddr = &PMRSysPhysAddrExtMem,
+    .pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataExtMem,
+    .pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataExtMem,
+    .pfnReadBytes = PMRReadBytesExtMem,
+    .pfnWriteBytes = PMRWriteBytesExtMem,
+    .pfnChangeSparseMem = NULL,
+    .pfnFinalize = &PMRFinalizeExtMem,
+};
+
+static inline PVRSRV_ERROR PhysmemValidateParam( IMG_DEVMEM_SIZE_T uiSize,
+                                                 IMG_CPU_VIRTADDR pvCpuVAddr,
+                                                 PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (!access_ok(pvCpuVAddr, uiSize))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid User mode CPU virtual address"));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* Fail if requesting coherency on one side but uncached on the other */
+	if ((PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
+			(PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
+				"Please use GPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if ((PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+			(PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
+				"Please use CPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+#if !defined(PVRSRV_WRAP_EXTMEM_WRITE_ATTRIB_ENABLE)
+	if (uiFlags & (PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED |
+			PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED |
+			PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+			PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC
+#if defined(DEBUG)
+			| PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+#endif
+		))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Write Attribute not supported. Passed Flags: 0x%"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+				__func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+#endif
+
+	/* Check address and size alignment */
+	if (uiSize & (PAGE_SIZE - 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Given size %llu is not multiple of OS page size (%lu)",
+				__func__,
+				uiSize,
+				PAGE_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (((uintptr_t) pvCpuVAddr) & (PAGE_SIZE - 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Given address %p is not aligned to OS page size (%lu)",
+				__func__,
+				pvCpuVAddr,
+				PAGE_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PhysmemWrapExtMemOS(CONNECTION_DATA * psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    IMG_CPU_VIRTADDR pvCpuVAddr,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    PMR **ppsPMRPtr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32	*pui32MappingTable = NULL;
+	PMR_WRAP_DATA *psPrivData;
+	PMR *psPMR;
+	IMG_UINT uiTotalNumPages = (uiSize >> PAGE_SHIFT);
+	IMG_BOOL bIsPMRDestroyed = IMG_FALSE;
+	IMG_UINT i = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0))
+	/* Ignore the most significant byte. */
+	pvCpuVAddr = (IMG_CPU_VIRTADDR)untagged_addr((uintptr_t)pvCpuVAddr);
+#endif
+
+	eError = PhysmemValidateParam(uiSize,
+	                              pvCpuVAddr,
+	                              uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Allocate private factory data */
+	eError = _AllocWrapData(&psPrivData,
+	                        psDevNode,
+	                        uiSize,
+	                        pvCpuVAddr,
+	                        uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Actually find and acquire the pages and physical addresses */
+	eError = _WrapExtMemAcquirePages(psDevNode,
+	                                 uiSize,
+	                                 pvCpuVAddr,
+	                                 psPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		_FreeWrapData(psPrivData);
+		goto e0;
+	}
+
+
+	pui32MappingTable = OSAllocMem(sizeof(*pui32MappingTable) * uiTotalNumPages);
+	if (pui32MappingTable == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	for (i = 0; i < uiTotalNumPages; i++)
+	{
+		pui32MappingTable[i] = i;
+	}
+
+	/* Avoid creating scratch page or zero page when the entire
+	 * allocation is backed and pinned */
+	if (psPrivData->uiNumBackedPages == psPrivData->uiTotalNumPages)
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING;
+	}
+
+	/* Create a suitable PMR */
+	eError = PMRCreatePMR(psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL],
+	                      uiSize,    /* PMR_SIZE_T uiLogicalSize                  */
+	                      uiTotalNumPages,    /* IMG_UINT32 ui32NumPhysChunks              */
+	                      uiTotalNumPages,    /* IMG_UINT32 ui32NumVirtChunks              */
+	                      pui32MappingTable,
+	                      PAGE_SHIFT,           /* PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee */
+	                      (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK), /* PMR_FLAGS_T uiFlags */
+	                      "WrappedExtMem",      /* const IMG_CHAR *pszAnnotation             */
+	                      &_sPMRWrapPFuncTab,   /* const PMR_IMPL_FUNCTAB *psFuncTab         */
+	                      psPrivData,           /* PMR_IMPL_PRIVDATA pvPrivData              */
+	                      PMR_TYPE_EXTMEM,
+	                      &psPMR,               /* PMR **ppsPMRPtr                           */
+	                      PDUMP_NONE);          /* IMG_UINT32 ui32PDumpFlags                 */
+	if (eError != PVRSRV_OK)
+	{
+		goto e2;
+	}
+
+	if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags))
+	{
+		eError = _FlushUMVirtualRange(psDevNode,
+									psPrivData,
+									uiSize,
+									pvCpuVAddr);
+		if (eError != PVRSRV_OK)
+		{
+			goto e3;
+		}
+	}
+
+	/* Mark the PMR such that no layout changes can happen.
+	 * The memory is allocated in the CPU domain and hence
+	 * no changes can be made through any of our API */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	*ppsPMRPtr = psPMR;
+
+	OSFreeMem(pui32MappingTable);
+
+	return PVRSRV_OK;
+e3:
+	(void) PMRUnrefPMR(psPMR);
+	bIsPMRDestroyed = IMG_TRUE;
+e2:
+	OSFreeMem(pui32MappingTable);
+e1:
+	if (!bIsPMRDestroyed)
+	{
+		(void)_WrapExtMemReleasePages(psPrivData);
+	}
+e0:
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_wrap.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_wrap.h
new file mode 100644
index 000000000000..015360736d03
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_wrap.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File           physmem_extmem_wrap.h
+@Title          Header for wrapping non-services allocated memory PMR factory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the support for wrapping non services allocated
+                memory into GPU space.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_EXTEMEM_WRAP_H
+#define SRVSRV_PHYSMEM_EXTEMEM_WRAP_H
+
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/mm_types.h>
+#include <linux/vmalloc.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/atomic.h>
+
+#include "img_types.h"
+#include "device.h"
+
+typedef enum _WRAP_EXT_MEM_TYPE_
+{
+	WRAP_TYPE_NULL = 0,
+	WRAP_TYPE_GET_USER_PAGES,
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+	WRAP_TYPE_FIND_VMA
+#endif
+} WRAP_EXT_MEM_TYPE;
+
+typedef struct _PMR_WRAP_DATA_
+{
+	/* Device for which this allocation has been made */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* Total Number of pages in the allocation */
+	IMG_UINT32 uiTotalNumPages;
+
+	/* Total number of actual physically backed pages */
+	IMG_UINT32 uiNumBackedPages;
+
+	/* This is only filled if we have page mappings,
+	 * for pfn mappings this stays empty */
+	struct page **ppsPageArray;
+
+	/* VM Area structure reference */
+	struct vm_area_struct *psVMArea;
+
+	/* This should always be filled and hold the physical addresses */
+	IMG_CPU_PHYADDR *ppvPhysAddr;
+
+	/* Did we find the pages via get_user_pages()
+	 * or via FindVMA and a page table walk? */
+	WRAP_EXT_MEM_TYPE eWrapType;
+
+	/* CPU caching modes for the PMR mapping */
+	IMG_UINT32 ui32CPUCacheFlags;
+
+	/* Write permitted ?*/
+	IMG_BOOL bWrite;
+
+} PMR_WRAP_DATA;
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+/* Find the VMA to a given CPU virtual address and do a
+ * page table walk to find the corresponding pfns */
+PVRSRV_ERROR _TryFindVMA(IMG_DEVMEM_SIZE_T uiSize,
+                            uintptr_t pvCpuVAddr,
+                            PMR_WRAP_DATA *psPrivData);
+#endif
+
+#endif /* SRVSRV_PHYSMEM_EXTEMEM_WRAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.c
new file mode 100644
index 000000000000..50d7c7e77731
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.c
@@ -0,0 +1,4790 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR functions for OS managed memory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                implementing the function callbacks for physical memory borrowed
+                from that normally managed by the operating system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/mm_types.h>
+#include <linux/vmalloc.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/atomic.h>
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#include <linux/migrate.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+#include <drm/drm_file.h>
+#endif
+#endif
+
+#if defined(CONFIG_X86)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
+#include <asm/set_memory.h>
+#else
+#include <asm/cacheflush.h>
+#endif
+#endif
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "rgx_pdump_panics.h"
+/* services/server/include/ */
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "pmr_os.h"
+#include "cache_km.h"
+#include "devicemem_server_utils.h"
+#include "pvr_vmap.h"
+#include "physheap.h"
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+#include "physmem_cpumap_history.h"
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+/* services/server/env/linux */
+#include "env_connection.h"
+#endif
+#endif
+
+/* ourselves */
+#include "physmem_osmem.h"
+#include "physmem_osmem_linux.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+#include "hash.h"
+#endif
+#endif
+
+#include "kernel_compatibility.h"
+
+
+static IMG_UINT32 g_uiMaxOrder = PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM;
+
+
+/* Defines how many pages should be mapped at once to the kernel */
+#define PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES 1024 /* 4 MB */
+
+/*
+	These are used to get/set/mask lower-order bits in a dma_addr_t
+	to provide side-band information associated with that address.
+	These includes whether the address was obtained via alloc_page
+	or dma_alloc and if address came allocated pre-aligned or an
+	adjustment was made manually to aligned it. They can also indicate
+	a page entry in the array has been generated by us for ease of mapping
+	and is part of a higher order page, this is known as a CMA Ghost.
+*/
+#define DMA_SET_CMA_GHOST(x)			((x) | ((dma_addr_t)0x03))
+#define DMA_IS_CMA_GHOST(x)				(((x) & ((dma_addr_t)0x03)) == 0x03)
+#define DMA_SET_ADJUSTED_ADDR(x)		((x) | ((dma_addr_t)0x02))
+#define DMA_IS_ADDR_ADJUSTED(x)			(((x) & ((dma_addr_t)0x02)) == 0x02)
+#define DMA_SET_ALLOCPG_ADDR(x)			((x) | ((dma_addr_t)0x01))
+#define DMA_IS_ALLOCPG_ADDR(x)			(((x) & ((dma_addr_t)0x01)) == 0x01)
+#define DMA_GET_ALIGN_ADJUSTMENT(x)		((x>>2) & ((dma_addr_t)0x3ff))
+#define DMA_SET_ALIGN_ADJUSTMENT(x,y)	((x) | (((dma_addr_t)y)<<0x02))
+#define DMA_GET_ADDR(x)					(((dma_addr_t)x) & ((dma_addr_t)~0xfff))
+#define DMA_VADDR_NOT_IN_USE			0xCAFEF00DDEADBEEFULL
+
+#define PVRSRV_ZERO_VALUE 0
+
+typedef struct _PMR_OSPAGEARRAY_DATA_ {
+	/* Device for which this allocation has been made */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+	/* DRM Node associated with connection that made the allocation */
+	struct drm_file *psDRMFile;
+#endif
+	/* Protects migration in progress count and action of idling the device */
+	POSWR_LOCK hPagesUnderMigrationRWLock;
+	/* How many pages in the PMR are currently in process of migration,
+	 * Must be protected by above RW lock.
+	 */
+	IMG_INT32 iNumOSPagesUnderMigration;
+
+	/*
+	 * CPU mappings associated with PMR.
+	 * Must be protected with PMR lock.
+	 */
+	DLLIST_NODE sCpuMappingListHead;
+#endif
+
+	/* The pid that made this allocation */
+	IMG_PID uiPid;
+
+	/*
+	 * iNumOSPagesAllocated:
+	 * Number of pages allocated in this PMR so far.
+	 * This allows for up to (2^31 - 1) pages. With 4KB pages, that's 8TB of memory for each PMR.
+	 */
+	IMG_INT32 iNumOSPagesAllocated;
+
+	/*
+	 * uiTotalNumOSPages:
+	 * Total number of pages supported by this PMR. (Fixed as of now due the fixed Page table array size)
+	 *  number of "pages" (a.k.a. macro pages, compound pages, higher order pages, etc...)
+	 */
+	IMG_UINT32 uiTotalNumOSPages;
+
+	/*
+	  uiLog2DevPageSize;
+
+	  size of each requested device "page" -- this would normally be the same as
+	  PAGE_SHIFT, but we support the idea that we may allocate pages
+	  in larger chunks for better contiguity, using order>0 in the
+	  call to alloc_pages()
+	*/
+	IMG_UINT32 uiLog2DevPageSize;
+
+	/*
+	  ui64DmaMask;
+	*/
+	IMG_UINT64 ui64DmaMask;
+
+	/*
+	  For non DMA/CMA allocation, pagearray references the pages
+	  thus allocated; one entry per compound page when compound
+	  pages are used. In addition, for DMA/CMA allocations, we
+	  track the returned cpu virtual and device bus address.
+	*/
+	struct page **pagearray;
+	dma_addr_t *dmaphysarray;
+	void **dmavirtarray;
+
+
+#define FLAG_ZERO              (0U)
+#define FLAG_POISON_ON_FREE    (1U)
+#define FLAG_POISON_ON_ALLOC   (2U)
+#define FLAG_ONDEMAND          (3U)
+
+/* IS CMA ensures the allocation will go through the dma_alloc_coherent path */
+#define FLAG_DMA_CMA           (5U)
+#define FLAG_UNSET_MEMORY_TYPE (6U)
+
+#define FLAG_IS_ZOMBIE         (7U)
+/* Pref CMA appends a flag (if exists) to hint alloc_pages should source from CMA region */
+#define FLAG_PREF_CMA          (8U)
+#define FLAG_IS_MOVABLE        (9U)
+
+	/*
+	 * Allocation flags related to the pages:
+	 * Zero              - Should we Zero memory on alloc
+	 * Poison on free    - Should we Poison the memory on free.
+	 * Poison on alloc   - Should we Poison the memory on alloc.
+	 * On demand         - Is the allocation on Demand i.e Do we defer allocation to time of use.
+	 * CMA               - Is CMA memory allocated via DMA framework
+	 * Unset Memory Type - Upon free do we need to revert the cache type before return to OS
+	 * Zombie            - Pages are part of a zombie PMR
+	 * */
+	IMG_UINT32 ui32AllocFlags;
+
+	/*
+	  The cache mode of the PMR. Additionally carrying the CPU-Cache-Clean
+	  flag, advising us to do cache maintenance on behalf of the caller.
+	  Boolean used to track if we need to revert the cache attributes
+	  of the pages used in this allocation. Depends on OS/architecture.
+	*/
+	IMG_UINT32 ui32CPUCacheFlags;
+	/*
+	 * In CMA allocation path, algorithm can allocate double the size of
+	 * requested allocation size to satisfy the alignment. In this case
+	 * the additional pages allocated are tracked through this additional
+	 * variable and are accounted for in the memory statistics */
+	IMG_UINT32 ui32CMAAdjustedPageCount;
+
+	/*
+	  Handle on the parent PMR
+	 */
+	PMR *hPMR;
+
+} PMR_OSPAGEARRAY_DATA;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+typedef struct _OSMEM_PAGE_PRIVDATA_
+{
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	/* Store index of entry to speed up migration */
+	IMG_UINT32 uiPMRArrIdx;
+} OSMEM_PAGE_PRIVDATA;
+#endif
+
+/***********************************
+ * Page pooling for uncached pages *
+ ***********************************/
+
+static INLINE void
+_FreeOSPage_CMA(struct device *dev,
+				size_t alloc_size,
+				IMG_UINT32 uiOrder,
+				void *virt_addr,
+				dma_addr_t dev_addr,
+				struct page *psPage);
+
+static void
+_FreeOSPage(IMG_UINT32 uiOrder,
+			IMG_BOOL bUnsetMemoryType,
+			PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			struct page *psPage);
+
+static PVRSRV_ERROR
+_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			IMG_UINT32 *pai32FreeIndices,
+			IMG_UINT32 ui32FreePageCount);
+
+static PVRSRV_ERROR
+_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree,
+						   IMG_UINT32 *puiPagesFreed);
+
+static inline PVRSRV_ERROR
+_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages,
+					   IMG_BOOL bFlush,
+					   IMG_UINT32 ui32CPUCacheFlags);
+
+/* A struct for our page pool holding an array of zeroed (!) pages.
+ * We always put units of page arrays to the pool but are
+ * able to take individual pages */
+typedef	struct
+{
+	/* Linkage for page pool LRU list */
+	struct list_head sPagePoolItem;
+
+	/* How many items are still in the page array */
+	IMG_UINT32 uiItemsRemaining;
+	/* Array of the actual pages */
+	struct page **ppsPageArray;
+
+} LinuxPagePoolEntry;
+
+/* CleanupThread structure to put allocation in page pool */
+typedef struct
+{
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupWork;
+	IMG_UINT32 ui32CPUCacheMode;
+	LinuxPagePoolEntry *psPoolEntry;
+} LinuxCleanupData;
+
+
+/* Caches to hold page pool and page array structures */
+static struct kmem_cache *g_psLinuxPagePoolCache;
+static struct kmem_cache *g_psLinuxPageArray;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static struct kmem_cache *g_psLinuxPagePrivateData;
+#endif
+
+/* Track what is live, all protected by pool lock.
+ * x86 needs two page pools because we have to change the memory attributes
+ * of the pages which is expensive due to an implicit flush.
+ * See set_pages_array_uc/wc/wb. */
+static IMG_UINT32 g_ui32PagePoolUCCount;
+#if defined(CONFIG_X86)
+static IMG_UINT32 g_ui32PagePoolWCCount;
+#endif
+/* Tracks asynchronous tasks currently accessing the page pool.
+ * It is incremented if a defer free task
+ * is created. Both will decrement the value when they finished the work.
+ * The atomic prevents piling up of deferred work in case the deferred thread
+ * cannot keep up with the application.*/
+static ATOMIC_T g_iPoolCleanTasks;
+/* We don't want too many asynchronous threads trying to access the page pool
+ * at the same time */
+#define PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS 128
+
+/* Defines how many pages the page cache should hold. */
+#if defined(PVR_LINUX_PHYSMEM_MAX_POOL_PAGES)
+static const IMG_UINT32 g_ui32PagePoolMaxEntries = PVR_LINUX_PHYSMEM_MAX_POOL_PAGES;
+#else
+static const IMG_UINT32 g_ui32PagePoolMaxEntries;
+#endif
+
+/*	We double check if we would exceed this limit if we are below MAX_POOL_PAGES
+	and want to add an allocation to the pool.
+	This prevents big allocations being given back to the OS just because they
+	exceed the MAX_POOL_PAGES limit even though the pool is currently empty. */
+#if defined(PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES)
+static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries = PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES;
+#else
+static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries;
+#endif
+
+#if defined(CONFIG_X86)
+#define PHYSMEM_OSMEM_NUM_OF_POOLS 2
+static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = {
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC
+};
+#else
+#define PHYSMEM_OSMEM_NUM_OF_POOLS 1
+static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = {
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED
+};
+#endif
+
+/* Global structures we use to manage the page pool */
+static DEFINE_MUTEX(g_sPagePoolMutex);
+
+/* List holding the page array pointers: */
+static LIST_HEAD(g_sPagePoolList_WC);
+static LIST_HEAD(g_sPagePoolList_UC);
+
+
+static IMG_BOOL g_bInitialisedOnAlloc = IMG_FALSE;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+static bool OSMemPageIsolate(struct page *psPage, isolate_mode_t uIsolateMode);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static int OSMemPageMigrate(struct address_space *psAddrSpace,
+		struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode);
+#else
+static int OSMemPageMigrate(struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode);
+#endif
+static void OSMemPagePutback(struct page *psPage);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION_DEBUG)
+#define MIGRATE_DBG_LOG(x) PVR_DPF(x);
+#else
+#define MIGRATE_DBG_LOG(...)
+#endif
+
+static INLINE OSMEM_PAGE_PRIVDATA* GetPrivateDataFromPage(struct page* psPage)
+{
+	return (OSMEM_PAGE_PRIVDATA*) page_private(psPage);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static const struct address_space_operations movable_callbacks =
+{
+	.isolate_page = OSMemPageIsolate,
+	.migratepage = OSMemPageMigrate,
+	.putback_page = OSMemPagePutback
+
+};
+#else
+static const struct movable_operations movable_callbacks =
+{
+	.isolate_page = OSMemPageIsolate,
+	.migrate_page = OSMemPageMigrate,
+	.putback_page = OSMemPagePutback
+};
+#endif
+
+static INLINE void OSMemSetMovablePageAttr(struct page* psPage,
+                                           PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                           IMG_UINT32 uiPageIndex)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	struct drm_file *psDRMFile = psPageArrayData->psDRMFile;
+#endif
+	OSMEM_PAGE_PRIVDATA *psPrivData;
+
+	if (!BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		return;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	/* Allocations marked with movable may occur before we have a valid connection
+	 * and thus DRM file. In this case ignore the request. An example of this would be
+	 * a kernel originated request which wouldn't have a connection associated.
+	 */
+	if (!psDRMFile)
+	{
+		PVR_ASSERT(!"Attempt to make allocation movable without connection associated, "
+		            "possible server origin");
+		return;
+	}
+#endif
+
+	psPrivData = kmem_cache_alloc(g_psLinuxPagePrivateData, GFP_KERNEL);
+	if (psPrivData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: OS refused the memory allocation for the private page data. "
+		         "Page will not be marked movable. PMR UID:%llu",
+				 __func__,
+				 PMRInternalGetUID(psPrivData->psPMRData->hPMR)));
+		return;
+	}
+
+	psPrivData->psPMRData = psPageArrayData;
+	psPrivData->uiPMRArrIdx = uiPageIndex;
+
+	SetPagePrivate(psPage);
+	set_page_private(psPage, (unsigned long)psPrivData);
+
+	/* Take page lock here while we set movable as __SetPageMovable asserts it */
+	lock_page(psPage);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	__SetPageMovable(psPage, psDRMFile->filp->f_inode->i_mapping);
+#else
+	__SetPageMovable(psPage, &movable_callbacks);
+#endif
+	unlock_page(psPage);
+}
+
+/* Page should be locked before calling this function */
+static INLINE void OSMemSetPagePrivate(struct page* psPage)
+{
+	__ClearPageMovable(psPage);
+	set_page_private(psPage, 0);
+	ClearPagePrivate(psPage);
+}
+
+static INLINE void OSMemUnsetMovablePageAttr(struct page* psPage, PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData;
+
+	if (!BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		return;
+	}
+
+	/* Private data should only be set if page is movable */
+	psPrivData = GetPrivateDataFromPage(psPage);
+	if (!psPrivData)
+	{
+		return;
+	}
+
+	lock_page(psPage);
+	OSMemSetPagePrivate(psPage);
+	unlock_page(psPage);
+
+	kmem_cache_free(g_psLinuxPagePrivateData, psPrivData);
+}
+static IMG_BOOL _MigrateGPUIdleGet(PMR_OSPAGEARRAY_DATA *psPMRData)
+{
+	PVRSRV_ERROR eError;
+
+	/* Try take a ref on PMR, this ensures destruction will not occur while migrating */
+	eError = PMRTryRefPMR(psPMRData->hPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorOut);
+
+	OSWRLockAcquireWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+	/* If we just started migrating pages */
+	if ((++psPMRData->iNumOSPagesUnderMigration) == 1)
+	{
+		eError = PVRSRVDeviceIdleLatchedGetKM(psPMRData->psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleLatchedGetKM", ErrorOutDecr);
+
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+		PMRLockPMR(psPMRData->hPMR);
+		PMRNotifyMigrateInProgress(psPMRData->hPMR);
+		PMRUnlockPMR(psPMRData->hPMR);
+	}
+	else
+	{
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	}
+
+	return IMG_TRUE;
+
+ErrorOutDecr:
+	psPMRData->iNumOSPagesUnderMigration--;
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING, "%s: Failed to Idle GPU", __func__));
+#endif
+	OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	(void) PMRUnrefPMR(psPMRData->hPMR);
+ErrorOut:
+	return IMG_FALSE;
+}
+
+static void _MigrateGPUIdlePut(PMR_OSPAGEARRAY_DATA *psPMRData)
+{
+	PVRSRV_ERROR eError;
+
+	OSWRLockAcquireWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+	/* If we just finished migrating pages */
+	if ((--psPMRData->iNumOSPagesUnderMigration) == 0)
+	{
+		eError = PVRSRVDeviceIdleLatchedPutAsyncKM(psPMRData->psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleLatchedPutAsyncKM", ErrorOut);
+
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+		PMRLockPMR(psPMRData->hPMR);
+		PMRNotifyMigrateComplete(psPMRData->hPMR);
+		PMRUnlockPMR(psPMRData->hPMR);
+	}
+	else
+	{
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	}
+
+
+	/* Unref parent PMR to allow for subsequent destruction,
+	 * if PMR resource no longer required and the bridge has
+	 * dropped its handles to it, destruction will follow.
+	 */
+	(void) PMRUnrefPMR(psPMRData->hPMR);
+
+	return;
+
+ErrorOut:
+	psPMRData->iNumOSPagesUnderMigration++;
+	OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	PVR_DPF((PVR_DBG_FATAL,"Request to Unidle GPU failed, unrecoverable state!"));
+}
+
+static bool OSMemPageIsolate(struct page *psPage, isolate_mode_t uIsolateMode)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData = GetPrivateDataFromPage(psPage);
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	IMG_BOOL bSuccess;
+
+
+	PVR_GOTO_IF_FALSE(psPrivData != NULL, condition_failed);
+	psPMRData = psPrivData->psPMRData;
+
+	/* Attempt to migrate pages before PMR wrapper has been setup,
+	 * avoid migration at this time.
+	 */
+	PVR_GOTO_IF_FALSE(psPMRData->hPMR, condition_failed);
+
+	bSuccess = _MigrateGPUIdleGet(psPMRData);
+	PVR_GOTO_IF_FALSE(bSuccess, condition_failed);
+
+	PVR_GOTO_IF_FALSE(!PMR_IsMemLayoutFixed(psPMRData->hPMR), clear_movable);
+	PVR_GOTO_IF_FALSE(!PMR_IsGpuMultiMapped(psPMRData->hPMR), clear_movable);
+	PVR_GOTO_IF_FALSE(!PMR_IsKernelCpuMapped(psPMRData->hPMR), condition_failed_decr);
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVR_GOTO_IF_FALSE(!PMR_IsZombie(psPMRData->hPMR), condition_failed_decr);
+#endif
+
+
+	psPrivData->psPMRData->pagearray[psPrivData->uiPMRArrIdx] = NULL;
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Isolated: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, PageAddr 0x%llx",
+	                                PMRInternalGetUID(psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psPage));
+
+	/* Movable */
+	return true;
+
+clear_movable:
+	OSMemSetPagePrivate(psPage);
+	kmem_cache_free(g_psLinuxPagePrivateData, psPrivData);
+
+condition_failed_decr:
+	_MigrateGPUIdlePut(psPMRData);
+
+condition_failed:
+	/* Not movable */
+	return false;
+}
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static int OSMemPageMigrate(struct address_space *psAddrSpace,
+		struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode)
+#else
+static int OSMemPageMigrate(struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode)
+#endif
+{
+	PVRSRV_ERROR eError;
+	int iLinuxError;
+	void* vpDst;
+	void* vpSrc;
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	OSMEM_PAGE_PRIVDATA *psSrcPagePrivData = GetPrivateDataFromPage(psSrcPage);
+	/* Align phys to logical */
+	IMG_UINT32 uiLogicalAlignedOffset;
+
+	PVR_UNREFERENCED_PARAMETER(eMigrateMode);
+
+	if (!psSrcPagePrivData)
+	{
+		/* Shouldn't hit this case */
+		PVR_DPF((PVR_DBG_ERROR, "Migrate invalid, potentially leaked PMR"));
+		return -EINVAL;
+	}
+	psPMRData = psSrcPagePrivData->psPMRData;
+
+	/* Calc logical aligned index offset, dev page aligned down. */
+	uiLogicalAlignedOffset = (
+	                          IMG_PAGES2BYTES64(psSrcPagePrivData->uiPMRArrIdx, PAGE_SHIFT) &
+	                          ~(psPMRData->uiLog2DevPageSize - 1)
+	                         ) >> PAGE_SHIFT;
+
+	/* If layout has been fixed after isolation, cancel the migration */
+	PVR_RETURN_IF_FALSE(!PMR_IsMemLayoutFixed(psPMRData->hPMR), -EPERM);
+
+	/* Unmap the page from any CPU mappings on this PMR. The mapping will
+	 * be repaired by the fault handler installed on the vma that
+	 * represents the CPU mapping.
+	 */
+	OSLinuxPMRUnmapPageInPMR(psPMRData->hPMR,
+	                         &psPMRData->sCpuMappingListHead,
+	                         uiLogicalAlignedOffset);
+
+	/* Set the new page in place, if any errors occur with migration the
+	 * putback callback will reset the original page.
+	 */
+	psPMRData->pagearray[psSrcPagePrivData->uiPMRArrIdx] = psDstPage;
+
+	/* Apply caching reqs to page we receive from OS */
+	eError = _ApplyOSPagesAttribute(psPMRData->psDevNode,
+	                                &psPMRData->pagearray[psSrcPagePrivData->uiPMRArrIdx],
+	                                1,
+	                                IMG_FALSE,
+	                                psPMRData->ui32CPUCacheFlags);
+	PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "_ApplyOSPagesAttribute", -EPERM);
+
+	/* Trigger remapping */
+	eError = PMRRemapGPUPMR(psPMRData->hPMR, uiLogicalAlignedOffset);
+	if (eError != PVRSRV_OK)
+	{
+		iLinuxError = -EPERM;
+		if (eError != PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PMRRemapGPUPMR() failed to remap page.", __func__));
+		}
+		goto error_reset_page_attrs;
+	}
+
+	vpDst = kmap(psDstPage);
+	vpSrc = kmap(psSrcPage);
+
+
+	OSDeviceMemCopy(vpDst, vpSrc, PAGE_SIZE);
+
+	kunmap(vpDst);
+	kunmap(vpSrc);
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Migrated: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, SrcPageAddr 0x%llx, DstPageAddr 0x%llx",
+	                                PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psSrcPagePrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psSrcPage,
+	                                (unsigned long long) psDstPage));
+
+	get_page(psDstPage);
+	SetPagePrivate(psDstPage);
+	set_page_private(psDstPage, (unsigned long)psSrcPagePrivData);
+
+	/* DstPage already locked prior to callback */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	__SetPageMovable(psDstPage, psPMRData->psDRMFile->filp->f_inode->i_mapping);
+#else
+	__SetPageMovable(psDstPage, &movable_callbacks);
+#endif
+
+	/* Migration for this page no longer in progress */
+	_MigrateGPUIdlePut(psSrcPagePrivData->psPMRData);
+
+#if defined(CONFIG_X86)
+	/* Reset caching attrs of page we give back to OS */
+	if (!set_pages_array_wb(&psSrcPage, 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute for "
+		                        " page given back to kernel. PMR UID:%llu",
+		         __func__,
+		         (unsigned long long)PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR)));
+	}
+#endif
+
+	set_page_private(psSrcPage, 0);
+	ClearPagePrivate(psSrcPage);
+	put_page(psSrcPage);
+	__ClearPageMovable(psSrcPage);
+
+	return MIGRATEPAGE_SUCCESS;
+
+error_reset_page_attrs:
+#if defined(CONFIG_X86)
+	/* Reset caching attrs of page we give back to OS */
+	if (!set_pages_array_wb(&psDstPage, 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute for "
+		                        " page given from kernel. PMR UID:%llu",
+		         __func__,
+		         (unsigned long long)PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR)));
+	}
+#endif
+	return iLinuxError;
+}
+
+static void OSMemPagePutback(struct page *psPage)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData = GetPrivateDataFromPage(psPage);
+	if (!psPrivData)
+	{
+		/* We shouldn't hit this case */
+		PVR_DPF((PVR_DBG_ERROR, "Putback invalid, potentially leaked PMR"));
+		return;
+	}
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Putback: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, PageAddr 0x%llx",
+	                                PMRInternalGetUID(psPrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psPrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psPage));
+
+
+	/* Restore the page to our structures */
+	psPrivData->psPMRData->pagearray[psPrivData->uiPMRArrIdx] = psPage;
+
+	/* Migration for this page no longer in progress */
+	_MigrateGPUIdlePut(psPrivData->psPMRData);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+int LinuxRegisterMigrateCallbacks(struct file* filp)
+{
+	if (filp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "filp NULL, cannot set movable callbacks, PID %u:%s",
+		                        OSGetCurrentClientProcessIDKM(),
+		                        OSGetCurrentClientProcessNameKM()));
+		return -EINVAL;
+	}
+
+	filp->f_inode->i_mapping->a_ops = &movable_callbacks;
+
+	return 0;
+}
+
+void LinuxDeregisterMigrateCallbacks(struct file* filp)
+{
+	if (filp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "filp NULL, cannot unset movable callbacks"));
+		return;
+	}
+
+	filp->f_inode->i_mapping->a_ops = NULL;
+}
+#endif
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+static inline IMG_BOOL
+_ShouldInitMem(IMG_UINT32 ui32AllocFlags)
+{
+	return BIT_ISSET(ui32AllocFlags, FLAG_ZERO) && !g_bInitialisedOnAlloc;
+}
+
+static inline IMG_UINT32
+_PagesInPoolUnlocked(void)
+{
+	IMG_UINT32 uiCnt = g_ui32PagePoolUCCount;
+#if defined(CONFIG_X86)
+	uiCnt += g_ui32PagePoolWCCount;
+#endif
+	return uiCnt;
+}
+
+static inline void
+_PagePoolLock(void)
+{
+	mutex_lock(&g_sPagePoolMutex);
+}
+
+static inline int
+_PagePoolTrylock(void)
+{
+	return mutex_trylock(&g_sPagePoolMutex);
+}
+
+static inline void
+_PagePoolUnlock(void)
+{
+	mutex_unlock(&g_sPagePoolMutex);
+}
+
+static inline IMG_BOOL
+_GetPoolListHead(IMG_UINT32 ui32CPUCacheFlags,
+				 struct list_head **ppsPoolHead,
+				 IMG_UINT32 **ppuiCounter)
+{
+	switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+#if defined(CONFIG_X86)
+		/*
+			For x86 we need to keep different lists for uncached
+			and write-combined as we must always honour the PAT
+			setting which cares about this difference.
+		*/
+
+			*ppsPoolHead = &g_sPagePoolList_WC;
+			*ppuiCounter = &g_ui32PagePoolWCCount;
+			break;
+#endif
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			*ppsPoolHead = &g_sPagePoolList_UC;
+			*ppuiCounter = &g_ui32PagePoolUCCount;
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unknown CPU caching mode. "
+					 "Using default UC pool.",
+					 __func__));
+			*ppsPoolHead = &g_sPagePoolList_UC;
+			*ppuiCounter = &g_ui32PagePoolUCCount;
+			PVR_ASSERT(0);
+			return IMG_FALSE;
+	}
+	return IMG_TRUE;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+static struct shrinker g_sShrinker;
+static struct shrinker *g_psShrinker = &g_sShrinker;
+#else
+static struct shrinker *g_psShrinker;
+#endif
+
+/* Returning the number of pages that still reside in the page pool. */
+static unsigned long
+_GetNumberOfPagesInPoolUnlocked(void)
+{
+	return _PagesInPoolUnlocked();
+}
+
+/* Linux shrinker function that informs the OS about how many pages we are caching and
+ * it is able to reclaim. */
+static unsigned long
+_CountObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl)
+{
+	int remain;
+
+	PVR_ASSERT(psShrinker == g_psShrinker);
+	(void)psShrinker;
+	(void)psShrinkControl;
+
+	/* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */
+	if (_PagePoolTrylock() == 0)
+		return 0;
+	remain = _GetNumberOfPagesInPoolUnlocked();
+	_PagePoolUnlock();
+
+	return remain;
+}
+
+/* Linux shrinker function to reclaim the pages from our page pool */
+static unsigned long
+_ScanObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl)
+{
+	unsigned long uNumToScan = psShrinkControl->nr_to_scan;
+	IMG_UINT32 uiPagesFreed;
+
+	PVR_ASSERT(psShrinker == g_psShrinker);
+	(void)psShrinker;
+
+	/* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */
+	if (_PagePoolTrylock() == 0)
+		return SHRINK_STOP;
+
+	_FreePagesFromPoolUnlocked(uNumToScan,
+							   &uiPagesFreed);
+	uNumToScan -= uiPagesFreed;
+
+	/* Returning the number of pages freed during the scan */
+	_PagePoolUnlock();
+	return psShrinkControl->nr_to_scan - uNumToScan;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+static struct shrinker g_sShrinker =
+{
+	.count_objects = _CountObjectsInPagePool,
+	.scan_objects = _ScanObjectsInPagePool,
+	.seeks = DEFAULT_SEEKS
+};
+#endif
+
+/* Register the shrinker so Linux can reclaim cached pages */
+PVRSRV_ERROR LinuxInitPhysmem(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	g_psLinuxPagePrivateData = kmem_cache_create("pvr-ppd", sizeof(OSMEM_PAGE_PRIVDATA), 0, 0, NULL);
+	if (!g_psLinuxPagePrivateData)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPagePrivateData",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e1);
+	}
+#endif
+
+	g_psLinuxPageArray = kmem_cache_create("pvr-pa", sizeof(PMR_OSPAGEARRAY_DATA), 0, 0, NULL);
+	if (!g_psLinuxPageArray)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPageArray",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e2);
+	}
+
+	g_psLinuxPagePoolCache = kmem_cache_create("pvr-pp", sizeof(LinuxPagePoolEntry), 0, 0, NULL);
+	if (g_psLinuxPagePoolCache)
+	{
+		/* Only create the shrinker if we created the cache OK */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+		register_shrinker(&g_sShrinker, "pvr-pp");
+#else
+		g_psShrinker = shrinker_alloc(0, "pvr-pp");
+		if (g_psShrinker)
+		{
+			g_psShrinker->count_objects = _CountObjectsInPagePool;
+			g_psShrinker->scan_objects = _ScanObjectsInPagePool;
+			g_psShrinker->seeks = DEFAULT_SEEKS;
+			shrinker_register(g_psShrinker);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Unable to allocate a shrinker for page pool cache"));
+		}
+#endif
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPagePoolCache",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e3);
+	}
+
+	OSAtomicWrite(&g_iPoolCleanTasks, 0);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,3,0))
+/* Check both config and modparam setting */
+#if PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC == 1
+	g_bInitialisedOnAlloc = want_init_on_alloc(0x0);
+
+/* Assume modparam setting not in use on system */
+#elif PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC == 2
+#   if defined(CONFIG_INIT_ON_ALLOC_DEFAULT_ON)
+	g_bInitialisedOnAlloc = IMG_TRUE;
+#   else
+	g_bInitialisedOnAlloc = IMG_FALSE;
+#   endif
+
+/* Ignore both config and modparam settings */
+#else
+	g_bInitialisedOnAlloc = IMG_FALSE;
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+e3:
+	/* Free the page array */
+	kmem_cache_destroy(g_psLinuxPageArray);
+e2:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Free the page private data */
+	kmem_cache_destroy(g_psLinuxPagePrivateData);
+e1:
+#endif
+
+	return eError;
+}
+
+/* Unregister the shrinker and remove all pages from the pool that are still left */
+void LinuxDeinitPhysmem(void)
+{
+	IMG_UINT32 uiPagesFreed;
+
+	if (OSAtomicRead(&g_iPoolCleanTasks) > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Still deferred cleanup tasks running "
+				"while deinitialising memory subsystem."));
+	}
+
+	_PagePoolLock();
+	if (_FreePagesFromPoolUnlocked(IMG_UINT32_MAX, &uiPagesFreed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Unable to free all pages from page pool when "
+				"deinitialising memory subsystem."));
+		PVR_ASSERT(0);
+	}
+
+	PVR_ASSERT(_PagesInPoolUnlocked() == 0);
+
+	/* Free the page cache */
+	if (g_psLinuxPagePoolCache)
+		kmem_cache_destroy(g_psLinuxPagePoolCache);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+	unregister_shrinker(&g_sShrinker);
+#else
+	shrinker_free(g_psShrinker);
+#endif
+	_PagePoolUnlock();
+
+	if (g_psLinuxPageArray)
+		kmem_cache_destroy(g_psLinuxPageArray);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (g_psLinuxPagePrivateData)
+		kmem_cache_destroy(g_psLinuxPagePrivateData);
+#endif
+}
+
+static void EnableOOMKiller(void)
+{
+	current->flags &= ~PF_DUMPCORE;
+}
+
+static void DisableOOMKiller(void)
+{
+	/* PF_DUMPCORE is treated by the VM as if the OOM killer was disabled.
+	 *
+	 * As oom_killer_disable() is an inline, non-exported function, we
+	 * can't use it from a modular driver. Furthermore, the OOM killer
+	 * API doesn't look thread safe, which 'current' is.
+	 */
+	WARN_ON(current->flags & PF_DUMPCORE);
+	current->flags |= PF_DUMPCORE;
+}
+
+/* Prints out the addresses in a page array for debugging purposes
+ * Define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY locally to activate: */
+/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY 1 */
+static inline void
+_DumpPageArray(struct page **pagearray, IMG_UINT32 uiPagesToPrint)
+{
+#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY)
+	IMG_UINT32 i;
+	if (pagearray)
+	{
+		printk("Array %p:\n", pagearray);
+		for (i = 0; i < uiPagesToPrint; i++)
+		{
+			printk("%p | ", (pagearray)[i]);
+		}
+		printk("\n");
+	}
+	else
+	{
+		printk("Array is NULL:\n");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(pagearray);
+	PVR_UNREFERENCED_PARAMETER(uiPagesToPrint);
+#endif
+}
+
+/* Debugging function that dumps out the number of pages for every
+ * page array that is currently in the page pool.
+ * Not defined by default. Define locally to activate feature: */
+/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL 1 */
+static void
+_DumpPoolStructure(void)
+{
+#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL)
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 j;
+	IMG_UINT32 *puiCounter;
+
+	printk("\n");
+	/* Empty all pools */
+	for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++)
+	{
+
+		printk("pool = %u\n", j);
+
+		/* Get the correct list for this caching mode */
+		if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter))
+		{
+			break;
+		}
+
+		list_for_each_entry_safe(psPagePoolEntry,
+								 psTempPoolEntry,
+								 psPoolHead,
+								 sPagePoolItem)
+		{
+			printk("%u | ", psPagePoolEntry->uiItemsRemaining);
+		}
+		printk("\n");
+	}
+#endif
+}
+
+/* Free a certain number of pages from the page pool.
+ * Mainly used in error paths or at deinitialisation to
+ * empty the whole pool. */
+static PVRSRV_ERROR
+_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree,
+						   IMG_UINT32 *puiPagesFreed)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 i, j;
+	IMG_UINT32 *puiCounter;
+
+	*puiPagesFreed = uiMaxPagesToFree;
+
+	/* Empty all pools */
+	for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++)
+	{
+
+		/* Get the correct list for this caching mode */
+		if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter))
+		{
+			break;
+		}
+
+		/* Free the pages and remove page arrays from the pool if they are exhausted */
+		list_for_each_entry_safe(psPagePoolEntry,
+								 psTempPoolEntry,
+								 psPoolHead,
+								 sPagePoolItem)
+		{
+			IMG_UINT32 uiItemsToFree;
+			struct page **ppsPageArray;
+
+			/* Check if we are going to free the whole page array or just parts */
+			if (psPagePoolEntry->uiItemsRemaining <= uiMaxPagesToFree)
+			{
+				uiItemsToFree = psPagePoolEntry->uiItemsRemaining;
+				ppsPageArray = psPagePoolEntry->ppsPageArray;
+			}
+			else
+			{
+				uiItemsToFree = uiMaxPagesToFree;
+				ppsPageArray = &(psPagePoolEntry->ppsPageArray[psPagePoolEntry->uiItemsRemaining - uiItemsToFree]);
+			}
+
+#if defined(CONFIG_X86)
+			/* Set the correct page caching attributes on x86 */
+			if (!PVRSRV_CHECK_CPU_CACHED(g_aui32CPUCacheFlags[j]))
+			{
+				int ret;
+				ret = set_pages_array_wb(ppsPageArray, uiItemsToFree);
+				if (ret)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s: Failed to reset page attributes",
+							 __func__));
+					eError = PVRSRV_ERROR_FAILED_TO_FREE_PAGES;
+					goto e_exit;
+				}
+			}
+#endif
+
+			/* Free the actual pages */
+			for (i = 0; i < uiItemsToFree; i++)
+			{
+				__free_pages(ppsPageArray[i], 0);
+				ppsPageArray[i] = NULL;
+			}
+
+			/* Reduce counters */
+			uiMaxPagesToFree -= uiItemsToFree;
+			*puiCounter -= uiItemsToFree;
+			psPagePoolEntry->uiItemsRemaining -= uiItemsToFree;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			/*
+			 * MemStats usually relies on having the bridge lock held, however
+			 * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and
+			 * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so
+			 * the page pool lock is used to ensure these calls are mutually
+			 * exclusive
+			 */
+			PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * uiItemsToFree);
+#endif
+
+			/* Is this pool entry exhausted, delete it */
+			if (psPagePoolEntry->uiItemsRemaining == 0)
+			{
+				OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+				list_del(&psPagePoolEntry->sPagePoolItem);
+				kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+			}
+
+			/* Return if we have all our pages */
+			if (uiMaxPagesToFree == 0)
+			{
+				goto e_exit;
+			}
+		}
+	}
+
+e_exit:
+	*puiPagesFreed -= uiMaxPagesToFree;
+	_DumpPoolStructure();
+	return eError;
+}
+
+/* Get a certain number of pages from the page pool and
+ * copy them directly into a given page array. */
+static void
+_GetPagesFromPoolUnlocked(IMG_UINT32 ui32CPUCacheFlags,
+						  IMG_UINT32 uiMaxNumPages,
+						  struct page **ppsPageArray,
+						  IMG_UINT32 *puiNumReceivedPages)
+{
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 i;
+	IMG_UINT32 *puiCounter;
+
+	*puiNumReceivedPages = 0;
+
+	/* Get the correct list for this caching mode */
+	if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter))
+	{
+		return;
+	}
+
+	/* Check if there are actually items in the list */
+	if (list_empty(psPoolHead))
+	{
+		return;
+	}
+
+	PVR_ASSERT(*puiCounter > 0);
+
+	/* Receive pages from the pool */
+	list_for_each_entry_safe(psPagePoolEntry,
+							 psTempPoolEntry,
+							 psPoolHead,
+							 sPagePoolItem)
+	{
+		/* Get the pages from this pool entry */
+		for (i = psPagePoolEntry->uiItemsRemaining; i != 0 && *puiNumReceivedPages < uiMaxNumPages; i--)
+		{
+			ppsPageArray[*puiNumReceivedPages] = psPagePoolEntry->ppsPageArray[i-1];
+			(*puiNumReceivedPages)++;
+			psPagePoolEntry->uiItemsRemaining--;
+		}
+
+		/* Is this pool entry exhausted, delete it */
+		if (psPagePoolEntry->uiItemsRemaining == 0)
+		{
+			OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+			list_del(&psPagePoolEntry->sPagePoolItem);
+			kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+		}
+
+		/* Return if we have all our pages */
+		if (*puiNumReceivedPages == uiMaxNumPages)
+		{
+			goto exit_ok;
+		}
+	}
+
+exit_ok:
+
+	/* Update counters */
+	*puiCounter -= *puiNumReceivedPages;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* MemStats usually relies on having the bridge lock held, however
+	 * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and
+	 * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so
+	 * the page pool lock is used to ensure these calls are mutually
+	 * exclusive
+	 */
+	PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * (*puiNumReceivedPages));
+#endif
+
+	_DumpPoolStructure();
+}
+
+/* Same as _GetPagesFromPoolUnlocked but handles locking and
+ * checks first whether pages from the pool are a valid option. */
+static inline void
+_GetPagesFromPoolLocked(PVRSRV_DEVICE_NODE *psDevNode,
+						IMG_UINT32 ui32CPUCacheFlags,
+						IMG_UINT32 ui32AllocFlags,
+						IMG_UINT32 uiPagesToAlloc,
+						IMG_UINT32 uiOrder,
+						struct page **ppsPageArray,
+						IMG_UINT32 *puiPagesFromPool)
+{
+#if !defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	/* Don't get pages from pool as it doesn't provide zeroed pages */
+	if (BIT_ISSET(ui32AllocFlags, FLAG_ZERO))
+	{
+		return;
+	}
+#endif
+
+	/* The page pool stores only order 0 pages. If we need zeroed memory we
+	 * directly allocate from the OS because it is faster than
+	 * doing it within the driver. CMA is naturally more commonly used for higher
+	 * order pages and so we reject that also.
+	 */
+	if (uiOrder == 0 &&
+	    !PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags) &&
+	    !BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA) &&
+	    !BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+
+		_PagePoolLock();
+		_GetPagesFromPoolUnlocked(ui32CPUCacheFlags,
+								  uiPagesToAlloc,
+								  ppsPageArray,
+								  puiPagesFromPool);
+		_PagePoolUnlock();
+	}
+}
+
+/* Takes a page array and maps it into the kernel to write zeros */
+static PVRSRV_ERROR
+_MemsetPageArray(IMG_UINT32 uiNumToClean,
+                 struct page **ppsCleanArray,
+                 pgprot_t pgprot,
+                 IMG_UINT8 ui8Pattern)
+{
+	IMG_CPU_VIRTADDR pvAddr;
+	IMG_UINT32 uiMaxPagesToMap = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES,
+	                                 uiNumToClean);
+
+	/* Map and fill the pages with zeros.
+	 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+	 * at a time. */
+	while (uiNumToClean != 0)
+	{
+		IMG_UINT32 uiToClean = MIN(uiNumToClean, uiMaxPagesToMap);
+
+		pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_MAP, pgprot);
+		if (!pvAddr)
+		{
+			if (uiMaxPagesToMap <= 1)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				        "%s: Out of vmalloc memory, unable to map pages for %s.",
+				        __func__,
+				        ui8Pattern == PVRSRV_ZERO_VALUE ? "zeroing" : "poisoning"));
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+			else
+			{
+				/* Halve the pages to map at once and try again. */
+				uiMaxPagesToMap = uiMaxPagesToMap >> 1;
+				continue;
+			}
+		}
+
+		if (pgprot_val(pgprot) == pgprot_val(pgprot_noncached(PAGE_KERNEL)))
+		{
+			/* this is most likely unnecessary as all pages must be 8-bytes
+			 * aligned so there unaligned access is impossible */
+			OSDeviceMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		else if (pgprot_val(pgprot) == pgprot_val(pgprot_writecombine(PAGE_KERNEL)))
+		{
+			OSCachedMemSetWMB(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		else
+		{
+			OSCachedMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		pvr_vunmap(pvAddr, uiToClean, pgprot);
+		ppsCleanArray = &(ppsCleanArray[uiToClean]);
+		uiNumToClean -= uiToClean;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_CleanupThread_CleanPages(void *pvData)
+{
+	LinuxCleanupData *psCleanupData = (LinuxCleanupData*) pvData;
+	LinuxPagePoolEntry *psPagePoolEntry = psCleanupData->psPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 *puiCounter = NULL;
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	PVRSRV_ERROR eError;
+	pgprot_t pgprot;
+	IMG_UINT32 i;
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+
+	/* Get the correct pool for this caching mode. */
+	_GetPoolListHead(psCleanupData->ui32CPUCacheMode , &psPoolHead, &puiCounter);
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	switch (PVRSRV_CPU_CACHE_MODE(psCleanupData->ui32CPUCacheMode))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+#if defined(CONFIG_X86)
+			/* For x86 we can only map with the same attributes
+			 * as in the PAT settings*/
+			pgprot = pgprot_noncached(PAGE_KERNEL);
+			break;
+#endif
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			pgprot = pgprot_writecombine(PAGE_KERNEL);
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unknown caching mode to set page protection flags.",
+					__func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto eExit;
+	}
+
+	/* Map and fill the pages with zeros.
+	 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+	 * at a time. */
+	eError = _MemsetPageArray(psPagePoolEntry->uiItemsRemaining,
+	                          psPagePoolEntry->ppsPageArray,
+	                          pgprot, PVRSRV_ZERO_VALUE);
+	if (eError != PVRSRV_OK)
+	{
+		goto eExit;
+	}
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+
+	/* Lock down pool and add item */
+	_PagePoolLock();
+
+	/* Pool counters were already updated so don't do it here again*/
+
+	/* The pages are all zeroed so return them to the pool. */
+	list_add_tail(&psPagePoolEntry->sPagePoolItem, psPoolHead);
+
+	_DumpPoolStructure();
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Calling PVRSRVStatsIncrMemAllocPoolStat and PVRSRVStatsDecrMemAllocPoolStat
+	 * inside page pool lock ensures that the stat reflects the state of the pool. */
+	PVRSRVStatsIncrMemAllocPoolStat(PAGE_SIZE * psPagePoolEntry->uiItemsRemaining);
+#endif
+
+	_PagePoolUnlock();
+
+	OSFreeMem(pvData);
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+
+	return PVRSRV_OK;
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+eExit:
+	/* we failed to zero the pages so return the error so we can
+	 * retry during the next spin */
+	if ((psCleanupData->sCleanupWork.ui32RetryCount - 1) > 0)
+	{
+		return eError;
+	}
+
+	/* this was the last retry, give up and free pages to OS */
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Deferred task error, freeing pages to OS.",
+			__func__));
+	_PagePoolLock();
+
+	*puiCounter -= psPagePoolEntry->uiItemsRemaining;
+
+	_PagePoolUnlock();
+
+	for (i = 0; i < psCleanupData->psPoolEntry->uiItemsRemaining; i++)
+	{
+		_FreeOSPage(0, IMG_TRUE, NULL, psPagePoolEntry->ppsPageArray[i]);
+	}
+	OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+	kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+	OSFreeMem(psCleanupData);
+
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+
+	return PVRSRV_OK;
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+}
+
+static bool _PagesHaveOtherRefs(struct page **ppsPageArray, IMG_UINT32 uiNumPages)
+{
+	IMG_UINT32 ui32pageIndex;
+
+	PVR_DPF_ENTERED;
+
+	if (!ppsPageArray)
+	{
+		PVR_DPF_RETURN_RC(false);
+	}
+
+	/* Iterate pages in ppsPageArray and return
+	   false if any are found to have page_count() > 1
+	*/
+	for (ui32pageIndex = 0; ui32pageIndex < uiNumPages; ui32pageIndex++)
+	{
+		struct page *psPage = ppsPageArray[ui32pageIndex];
+
+		if (page_count(psPage) > 1)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: page %d in page array found with page_count() of %d",
+			         __func__, ui32pageIndex, page_count(psPage)));
+			PVR_DPF_RETURN_RC(true);
+		}
+	}
+
+	PVR_DPF_RETURN_RC(false);
+}
+
+/* Put page array to the page pool.
+ * Handles locking and checks whether the pages are
+ * suitable to be stored in the pool. */
+static inline IMG_BOOL
+_PutPagesToPoolLocked(PVRSRV_DEVICE_NODE *psDevNode,
+                      IMG_UINT32 ui32CPUCacheFlags,
+                      IMG_UINT32 ui32AllocFlags,
+                      struct page **ppsPageArray,
+                      IMG_UINT32 uiOrder,
+                      IMG_UINT32 uiNumPages)
+{
+	LinuxCleanupData *psCleanupData;
+	PVRSRV_CLEANUP_THREAD_WORK *psCleanupThreadFn;
+	IMG_UINT32 uiEntries;
+	IMG_UINT32 *puiCounter;
+	struct list_head *psPoolHead;
+#if defined(SUPPORT_PHYSMEM_TEST)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+#endif
+
+	PVR_DPF_ENTERED;
+
+	/* The page pool stores only order 0 pages. CMA is naturally more
+	 * commonly used for higher order pages and so we reject that also.
+	 */
+	if (uiOrder != 0 ||
+		PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags) ||
+		BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA) ||
+		BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		goto eExitFalse;
+	}
+
+	if (_PagesHaveOtherRefs(ppsPageArray, uiNumPages))
+	{
+		/* Pages still have other references, so
+		   must not be put into our pool */
+		goto eExitFalse;
+	}
+
+	_PagePoolLock();
+
+	uiEntries = _PagesInPoolUnlocked();
+
+	/* Check for number of current page pool entries */
+	if ( (uiEntries >= g_ui32PagePoolMaxEntries) ||
+		 ((uiEntries + uiNumPages) >=
+		  (g_ui32PagePoolMaxEntries + g_ui32PagePoolMaxExcessEntries) ))
+	{
+		goto eUnlock;
+	}
+
+	/* Check for whether we have other asynchronous tasks in-flight */
+	if (OSAtomicIncrement(&g_iPoolCleanTasks) >
+			PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS)
+	{
+		goto eDecrement;
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST)
+	if (!psPVRSRVData->hCleanupThread)
+	{
+		goto eDecrement;
+	}
+#endif
+
+	psCleanupData = OSAllocMem(sizeof(*psCleanupData));
+
+	if (!psCleanupData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get memory for deferred page pool cleanup. "
+				 "Trying to free pages immediately",
+				 __func__));
+		goto eDecrement;
+	}
+
+	psCleanupThreadFn = &psCleanupData->sCleanupWork;
+	psCleanupData->ui32CPUCacheMode = ui32CPUCacheFlags;
+	psCleanupData->psPoolEntry = kmem_cache_alloc(g_psLinuxPagePoolCache, GFP_KERNEL);
+
+	if (!psCleanupData->psPoolEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get memory for deferred page pool cleanup. "
+				 "Trying to free pages immediately",
+				 __func__));
+		goto eFreeCleanupData;
+	}
+
+	if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get correct page pool",
+				 __func__));
+		goto eFreePoolEntry;
+	}
+
+	/* Increase counter here to avoid deferred cleanup tasks piling up */
+	*puiCounter = *puiCounter + uiNumPages;
+
+	psCleanupData->psPoolEntry->ppsPageArray = ppsPageArray;
+	psCleanupData->psPoolEntry->uiItemsRemaining = uiNumPages;
+
+	psCleanupThreadFn->pfnFree = _CleanupThread_CleanPages;
+	psCleanupThreadFn->pvData = psCleanupData;
+	psCleanupThreadFn->bDependsOnHW = IMG_FALSE;
+	psCleanupThreadFn->eCleanupType = PVRSRV_CLEANUP_TYPE_OSMEM;
+	CLEANUP_THREAD_SET_RETRY_COUNT(psCleanupThreadFn,
+								   CLEANUP_THREAD_RETRY_COUNT_DEFAULT);
+
+	/* We must not hold the pool lock when calling AddWork because it might call us back to
+	 * free pooled pages directly when unloading the driver	 */
+	_PagePoolUnlock();
+
+	PVRSRVCleanupThreadAddWork(psDevNode, psCleanupThreadFn);
+
+	PVR_DPF_RETURN_RC(IMG_TRUE);
+
+eFreePoolEntry:
+	OSFreeMem(psCleanupData->psPoolEntry);
+eFreeCleanupData:
+	OSFreeMem(psCleanupData);
+eDecrement:
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+eUnlock:
+	_PagePoolUnlock();
+eExitFalse:
+	PVR_DPF_RETURN_RC(IMG_FALSE);
+}
+
+/* Get the GFP flags that we pass to the page allocator */
+static inline gfp_t
+_GetGFPFlags(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	/* If higher order pages then potentially ask the kernel to zero the allocation
+	 * for us. If zero order we can do it ourselves later in a batch operation
+	 */
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bZero = uiOrder ? _ShouldInitMem(psPageArrayData->ui32AllocFlags) : IMG_FALSE;
+	struct device *psDev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+
+	gfp_t gfp_flags = __GFP_NOWARN | __GFP_NOMEMALLOC;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		gfp_flags |= GFP_HIGHUSER_MOVABLE;
+	}
+	else
+#else
+	{
+		gfp_flags |= GFP_USER;
+	}
+#endif
+
+#if defined(ANDROID)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)) && \
+	(LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	/* Only Android has CMA suggestion flag, base Linux kernel uses heuristics based on memory
+	 * pressure / availability to achieve similar goal.
+	 */
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_PREF_CMA))
+	{
+		gfp_flags |= __GFP_CMA;
+	}
+#endif
+#endif
+
+#if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+	/* Force use of HIGHMEM */
+	gfp_flags |= __GFP_HIGHMEM;
+
+	PVR_UNREFERENCED_PARAMETER(psDev);
+#else
+	if (psDev)
+	{
+#if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE)
+		if (*psDev->dma_mask > DMA_BIT_MASK(32))
+		{
+			/* If our system is able to handle large addresses use highmem */
+			gfp_flags |= __GFP_HIGHMEM;
+		}
+		else if (*psDev->dma_mask == DMA_BIT_MASK(32))
+		{
+			/* Limit to 32 bit.
+			 * Achieved by setting __GFP_DMA32 for 64 bit systems */
+			gfp_flags |= __GFP_DMA32;
+		}
+		else
+		{
+			/* Limit to size of DMA zone. */
+			gfp_flags |= __GFP_DMA;
+		}
+#else
+		if (*psDev->dma_mask < DMA_BIT_MASK(32))
+		{
+			gfp_flags |= __GFP_DMA;
+		}
+		else
+		{
+			gfp_flags |= __GFP_HIGHMEM;
+		}
+#endif /* if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE) */
+	}
+
+#endif /* if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY) */
+
+	if (bZero)
+	{
+		gfp_flags |= __GFP_ZERO;
+	}
+
+	return gfp_flags;
+}
+
+/*
+ * @Function _PoisonDevicePage
+ *
+ * @Description  Poisons a device page. In normal case the device page has the
+ *               same size as the OS page and so the ui32DevPageOrder will be
+ *               equal to 0 and page argument will point to one OS page
+ *               structure. In case of Non4K pages the order will be greater
+ *               than 0 and page argument will point to an array of OS
+ *               allocated pages.
+ *
+ * @Input psDevNode          pointer to the device object
+ * @Input page               array of the pages allocated by from the OS
+ * @Input ui32DevPageOrder   order of the page (same as the one used to allocate
+ *                           the page array by alloc_pages())
+ * @Input ui32CPUCacheFlags  CPU cache flags applied to the page
+ * @Input ui8PoisonValue     value used to poison the page
+ */
+static void
+_PoisonDevicePage(PVRSRV_DEVICE_NODE *psDevNode,
+                  struct page *page,
+                  IMG_UINT32 ui32DevPageOrder,
+                  IMG_UINT32 ui32CPUCacheFlags,
+                  IMG_BYTE ui8PoisonValue)
+{
+	IMG_UINT32 ui32OsPageIdx;
+
+	for (ui32OsPageIdx = 0;
+	     ui32OsPageIdx < (1U << ui32DevPageOrder);
+	     ui32OsPageIdx++)
+	{
+		struct page *current_page = page + ui32OsPageIdx;
+		IMG_CPU_PHYADDR sCPUPhysAddrStart = {page_to_phys(current_page)};
+		IMG_CPU_PHYADDR sCPUPhysAddrEnd = {sCPUPhysAddrStart.uiAddr + PAGE_SIZE};
+
+		void *kvaddr = kmap_atomic(current_page);
+
+		/* kmap_atomic maps pages as cached so it's safe to use OSCachedMemSet
+		 * here (also pages are always 8 bytes aligned anyway) */
+		OSCachedMemSet(kvaddr, ui8PoisonValue, PAGE_SIZE);
+
+		OSCPUCacheFlushRangeKM(psDevNode, kvaddr, kvaddr + PAGE_SIZE,
+		                       sCPUPhysAddrStart, sCPUPhysAddrEnd);
+
+		kunmap_atomic(kvaddr);
+	}
+}
+
+/* Allocate and initialise the structure to hold the metadata of the allocation */
+static PVRSRV_ERROR
+_AllocOSPageArray(PVRSRV_DEVICE_NODE *psDevNode,
+				  PMR_SIZE_T uiSize,
+				  IMG_UINT32 uiLog2DevPageSize,
+				  IMG_UINT32 ui32AllocFlags,
+				  IMG_UINT32 ui32CPUCacheFlags,
+				  IMG_PID uiPid,
+				  PMR_OSPAGEARRAY_DATA **ppsPageArrayDataPtr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiNumOSPageSizeVirtPages;
+	PMR_OSPAGEARRAY_DATA *psPageArrayData;
+	IMG_UINT64 ui64DmaMask = 0;
+
+	/* Use of cast below is justified by the assertion that follows to
+	 * prove that no significant bits have been truncated */
+	uiNumOSPageSizeVirtPages = (IMG_UINT32) (((uiSize - 1) >> PAGE_SHIFT) + 1);
+	PVR_ASSERT(((PMR_SIZE_T) uiNumOSPageSizeVirtPages << PAGE_SHIFT) == uiSize);
+
+	/* Allocate the struct to hold the metadata */
+	psPageArrayData = kmem_cache_alloc(g_psLinuxPageArray, GFP_KERNEL);
+	if (psPageArrayData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: OS refused the memory allocation for the private data.",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e_freed_none;
+	}
+
+	/*
+	 * Allocate the page array
+	 *
+	 * We avoid tracking this memory because this structure might go into the page pool.
+	 * The OS can drain the pool asynchronously and when doing that we have to avoid
+	 * any potential deadlocks.
+	 *
+	 * In one scenario the process stats vmalloc hash table lock is held and then
+	 * the oom-killer softirq is trying to call _ScanObjectsInPagePool(), it must not
+	 * try to acquire the vmalloc hash table lock again.
+	 */
+	psPageArrayData->pagearray = OSAllocZMemNoStats(sizeof(struct page *) * uiNumOSPageSizeVirtPages);
+	if (psPageArrayData->pagearray == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e_free_kmem_cache;
+	}
+
+	if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		/* Allocate additional DMA/CMA cpu kernel virtual address & device bus address array state */
+		psPageArrayData->dmavirtarray = OSAllocZMemNoStats(sizeof(void*) * uiNumOSPageSizeVirtPages);
+		if (psPageArrayData->dmavirtarray == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e_free_pagearray;
+		}
+
+		psPageArrayData->dmaphysarray = OSAllocZMemNoStats(sizeof(dma_addr_t) * uiNumOSPageSizeVirtPages);
+		if (psPageArrayData->dmaphysarray == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e_free_cpuvirtaddrarray;
+		}
+	}
+	else
+	{
+		psPageArrayData->dmavirtarray = NULL;
+		psPageArrayData->dmaphysarray = NULL;
+	}
+
+
+	if (psDevNode->psDevConfig && psDevNode->psDevConfig->pvOSDevice)
+	{
+		struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+		ui64DmaMask = *psDev->dma_mask;
+	}
+
+	/* Init metadata */
+	psPageArrayData->hPMR = NULL;
+	psPageArrayData->psDevNode = psDevNode;
+	psPageArrayData->uiPid = uiPid;
+	psPageArrayData->iNumOSPagesAllocated = 0;
+	psPageArrayData->uiTotalNumOSPages = uiNumOSPageSizeVirtPages;
+	psPageArrayData->uiLog2DevPageSize = uiLog2DevPageSize;
+	psPageArrayData->ui64DmaMask = ui64DmaMask;
+	psPageArrayData->ui32AllocFlags = ui32AllocFlags;
+	psPageArrayData->ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psPageArrayData->ui32CMAAdjustedPageCount = 0;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	eError = OSWRLockCreate(&psPageArrayData->hPagesUnderMigrationRWLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSWRLockCreate", e_free_cpuvirtaddrarray);
+	psPageArrayData->iNumOSPagesUnderMigration = 0;
+	dllist_init(&psPageArrayData->sCpuMappingListHead);
+#endif
+
+	*ppsPageArrayDataPtr = psPageArrayData;
+	return PVRSRV_OK;
+
+/* Error path */
+e_free_cpuvirtaddrarray:
+	OSFreeMemNoStats(psPageArrayData->dmavirtarray);
+
+e_free_pagearray:
+	OSFreeMemNoStats(psPageArrayData->pagearray);
+
+e_free_kmem_cache:
+	kmem_cache_free(g_psLinuxPageArray, psPageArrayData);
+	PVR_DPF((PVR_DBG_ERROR,
+			 "%s: OS refused the memory allocation for the page pointer table. "
+			 "Did you ask for too much?",
+			 __func__));
+
+e_freed_none:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static inline void
+_ApplyCacheMaintenance(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages)
+{
+	void * pvAddr;
+
+	if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+	{
+		pgprot_t pgprot = PAGE_KERNEL;
+
+		IMG_UINT32 uiNumToClean = uiNumPages;
+		struct page **ppsCleanArray = ppsPage;
+
+		/* Map and flush page.
+		 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+		 * at a time. */
+		while (uiNumToClean != 0)
+		{
+			IMG_UINT32 uiToClean = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES,
+			                           uiNumToClean);
+			IMG_CPU_PHYADDR sUnused =
+				{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
+
+			pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_MAP, pgprot);
+			if (!pvAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Unable to flush page cache for new allocation, skipping flush."));
+				return;
+			}
+
+			CacheOpExec(psDevNode,
+						pvAddr,
+						pvAddr + PAGE_SIZE,
+						sUnused,
+						sUnused,
+						PVRSRV_CACHE_OP_FLUSH);
+
+			pvr_vunmap(pvAddr, uiToClean, pgprot);
+			ppsCleanArray = &(ppsCleanArray[uiToClean]);
+			uiNumToClean -= uiToClean;
+		}
+	}
+	else
+	{
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 0; ui32Idx < uiNumPages; ++ui32Idx)
+		{
+			IMG_CPU_PHYADDR sCPUPhysAddrStart, sCPUPhysAddrEnd;
+
+			pvAddr = kmap(ppsPage[ui32Idx]);
+			sCPUPhysAddrStart.uiAddr = page_to_phys(ppsPage[ui32Idx]);
+			sCPUPhysAddrEnd.uiAddr = sCPUPhysAddrStart.uiAddr + PAGE_SIZE;
+
+			/* If we're zeroing, we need to make sure the cleared memory is pushed out
+			 * of the cache before the cache lines are invalidated */
+			CacheOpExec(psDevNode,
+						pvAddr,
+						pvAddr + PAGE_SIZE,
+						sCPUPhysAddrStart,
+						sCPUPhysAddrEnd,
+						PVRSRV_CACHE_OP_FLUSH);
+
+			kunmap(ppsPage[ui32Idx]);
+		}
+	}
+}
+
+/* Change the caching attribute of pages on x86 systems and takes care of
+ * cache maintenance. This function is supposed to be called once for pages that
+ * came from alloc_pages(). It expects an array of OS page sized pages!
+ *
+ * Flush/Invalidate pages in case the allocation is not cached. Necessary to
+ * remove pages from the cache that might be flushed later and corrupt memory. */
+static inline PVRSRV_ERROR
+_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages,
+					   IMG_BOOL bFlush,
+					   IMG_UINT32 ui32CPUCacheFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bCPUCached = PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags);
+	IMG_BOOL bCPUUncached = PVRSRV_CHECK_CPU_UNCACHED(ui32CPUCacheFlags);
+	IMG_BOOL bCPUWriteCombine = PVRSRV_CHECK_CPU_WRITE_COMBINE(ui32CPUCacheFlags);
+
+	if (ppsPage != NULL && uiNumPages != 0)
+	{
+#if defined(CONFIG_X86)
+		/* On x86 we have to set page cache attributes for non-cached pages.
+		 * The call is implicitly taking care of all flushing/invalidating
+		 * and therefore we can skip the usual cache maintenance after this. */
+		if (bCPUUncached || bCPUWriteCombine)
+		{
+			/* On x86 if we already have a mapping (e.g. low memory) we need to change the mode of
+				current mapping before we map it ourselves */
+			int ret = IMG_FALSE;
+
+			switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags))
+			{
+				case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+					ret = set_pages_array_uc(ppsPage, uiNumPages);
+					if (ret)
+					{
+						eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE;
+						PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to UC failed, returned %d", ret));
+					}
+					break;
+
+				case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+					ret = set_pages_array_wc(ppsPage, uiNumPages);
+					if (ret)
+					{
+						eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE;
+						PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to WC failed, returned %d", ret));
+					}
+					break;
+
+				case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+					break;
+
+				default:
+					break;
+			}
+		}
+		else
+#endif
+		{
+			if ( bFlush ||
+				 bCPUUncached || bCPUWriteCombine ||
+				 (bCPUCached && PVRSRV_CHECK_CPU_CACHE_CLEAN(ui32CPUCacheFlags)) )
+			{
+				/*  We can be given pages which still remain in the cache.
+					In order to make sure that the data we write through our mappings
+					doesn't get overwritten by later cache evictions we invalidate the
+					pages that are given to us.
+
+					Note:
+					This still seems to be true if we request cold pages, it's just less
+					likely to be in the cache. */
+				_ApplyCacheMaintenance(psDevNode,
+									   ppsPage,
+									   uiNumPages);
+			}
+		}
+	}
+
+	return eError;
+}
+
+/* Same as _AllocOSPage except it uses DMA framework to perform allocation.
+ * uiPageIndex is expected to be the pagearray index where to store the higher order page. */
+static PVRSRV_ERROR
+_AllocOSPage_CMA(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+				gfp_t gfp_flags,
+				IMG_UINT32 ui32AllocOrder,
+				IMG_UINT32 ui32MinOrder,
+				IMG_UINT32 uiPageIndex)
+{
+	void *virt_addr;
+	struct page *page;
+	dma_addr_t bus_addr;
+	IMG_UINT32 uiAllocIsMisaligned;
+	size_t alloc_size = PAGE_SIZE << ui32AllocOrder;
+	struct device *dev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+
+	do
+	{
+		DisableOOMKiller();
+#if defined(PVR_LINUX_PHYSMEM_SUPPRESS_DMA_AC)
+		virt_addr = NULL;
+#else
+		virt_addr = dma_alloc_coherent(dev, alloc_size, &bus_addr, gfp_flags);
+#endif
+		if (virt_addr == NULL)
+		{
+			/* The idea here is primarily to support some older kernels with
+			   broken or non-functioning DMA/CMA implementations (< Linux-3.4)
+			   and to also handle DMA/CMA allocation failures by attempting a
+			   normal page allocation though we expect dma_alloc_coherent()
+			   already attempts this internally also before failing but
+			   nonetheless it does no harm to retry the allocation ourselves */
+			page = alloc_pages(gfp_flags, ui32AllocOrder);
+			if (page)
+			{
+				/* Taint bus_addr as alloc_page, needed when freeing;
+				   also acquire the low memory page address only, this
+				   prevents mapping possible high memory pages into
+				   kernel virtual address space which might exhaust
+				   the VMALLOC address space */
+				bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page));
+				virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE;
+			}
+			else
+			{
+				EnableOOMKiller();
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+		}
+		else
+		{
+#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+			page = pfn_to_page(bus_addr >> PAGE_SHIFT);
+#else
+			/* Assumes bus address space is identical to physical address space */
+			page = phys_to_page(bus_addr);
+#endif
+		}
+		EnableOOMKiller();
+
+		/* Physical allocation alignment works/hidden behind the scene transparently,
+		   we do this here if the allocated buffer address does not meet its alignment
+		   requirement by over-allocating using the next power-2 order and reporting
+		   aligned-adjusted values back to meet the requested alignment constraint.
+		   Evidently we waste memory by doing this so should only do so if we do not
+		   initially meet the alignment constraint. */
+		uiAllocIsMisaligned = DMA_GET_ADDR(bus_addr) & ((PAGE_SIZE<<ui32MinOrder)-1);
+		if (uiAllocIsMisaligned || ui32AllocOrder > ui32MinOrder)
+		{
+			IMG_BOOL bUsedAllocPages = DMA_IS_ALLOCPG_ADDR(bus_addr);
+			if (ui32AllocOrder == ui32MinOrder)
+			{
+				if (bUsedAllocPages)
+				{
+					__free_pages(page, ui32AllocOrder);
+				}
+				else
+				{
+					dma_free_coherent(dev, alloc_size, virt_addr, bus_addr);
+				}
+
+				ui32AllocOrder = ui32AllocOrder + 1;
+				alloc_size = PAGE_SIZE << ui32AllocOrder;
+
+				PVR_ASSERT(uiAllocIsMisaligned != 0);
+			}
+			else
+			{
+				size_t align_adjust = PAGE_SIZE << ui32MinOrder;
+
+				/* Adjust virtual/bus addresses to meet alignment */
+				bus_addr = bUsedAllocPages ? page_to_phys(page) : bus_addr;
+				align_adjust = PVR_ALIGN((size_t)bus_addr, align_adjust);
+				align_adjust -= (size_t)bus_addr;
+
+				if (align_adjust)
+				{
+					if (bUsedAllocPages)
+					{
+						page += align_adjust >> PAGE_SHIFT;
+						bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page));
+						virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE;
+					}
+					else
+					{
+						bus_addr += align_adjust;
+						virt_addr += align_adjust;
+#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+						page = pfn_to_page(bus_addr >> PAGE_SHIFT);
+#else
+						/* Assumes bus address space is identical to physical address space */
+						page = phys_to_page(bus_addr);
+#endif
+					}
+
+					/* Store adjustments in PAGE_SIZE counts */
+					align_adjust = align_adjust >> PAGE_SHIFT;
+					bus_addr = DMA_SET_ALIGN_ADJUSTMENT(bus_addr, align_adjust);
+
+					/* Taint bus_addr due to over-allocation, allows us to free
+					 * memory correctly */
+					bus_addr = DMA_SET_ADJUSTED_ADDR(bus_addr);
+				}
+				uiAllocIsMisaligned = 0;
+			}
+		}
+	} while (uiAllocIsMisaligned);
+
+	/* Convert OSPageSize-based index into DevicePageSize-based index */
+	psPageArrayData->ui32CMAAdjustedPageCount += (alloc_size - (PAGE_SIZE << ui32AllocOrder));
+	psPageArrayData->dmavirtarray[uiPageIndex] = virt_addr;
+	psPageArrayData->dmaphysarray[uiPageIndex] = bus_addr;
+	psPageArrayData->pagearray[uiPageIndex] = page;
+
+	return PVRSRV_OK;
+}
+
+/* Allocate a page of order uiAllocOrder and stores it in the page array ppsPage at
+ * position uiPageIndex.
+ *
+ * If the order is higher than 0, it splits the page into multiples and
+ * stores them at position uiPageIndex to uiPageIndex+(1<<uiAllocOrder).
+ *
+ * This function is supposed to be used for uiMinOrder == 0 only! */
+static PVRSRV_ERROR
+_AllocOSPage(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			gfp_t gfp_flags,
+			IMG_UINT32 uiAllocOrder,
+			IMG_UINT32 uiMinOrder,
+			IMG_UINT32 uiPageIndex)
+{
+	struct page *psPage;
+	IMG_UINT32 ui32Count;
+	struct device *psDev = NULL;
+
+	/* Parameter check. If it fails we write into the wrong places in the array. */
+	PVR_ASSERT(uiMinOrder == 0);
+
+	if (psPageArrayData->psDevNode && psPageArrayData->psDevNode->psDevConfig)
+		psDev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+	/* Allocate the page */
+	DisableOOMKiller();
+	psPage = alloc_pages_node(psDev ? dev_to_node(psDev) : NUMA_NO_NODE,
+				  gfp_flags, uiAllocOrder);
+	EnableOOMKiller();
+
+	if (psPage == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* In case we need to, split the higher order page;
+	   this should only be used for order-0 allocations
+	   as higher order allocations should use DMA/CMA */
+	if (uiAllocOrder != 0)
+	{
+		split_page(psPage, uiAllocOrder);
+	}
+
+	/* Store the page (or multiple split pages) in the page array */
+	for (ui32Count = 0; ui32Count < (1 << uiAllocOrder); ui32Count++)
+	{
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		OSMemSetMovablePageAttr(&psPage[ui32Count], psPageArrayData, uiPageIndex + ui32Count);
+#endif
+		psPageArrayData->pagearray[uiPageIndex + ui32Count] = &(psPage[ui32Count]);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+
+static inline void _AddMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                               const struct page *psPage,
+                                               IMG_UINT32 uiOrder)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) };
+	IMG_UINT32 uiPageSize = 1 << psPageArrayData->uiLog2DevPageSize;
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                             NULL, sCPUPhysAddr,
+	                             uiPageSize << uiOrder,
+	                             psPageArrayData->uiPid
+	                             DEBUG_MEMSTATS_VALUES);
+}
+
+static inline void _RemoveMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                                  const struct page *psPage)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	IMG_UINT32 uiStat = BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE)
+		? PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES
+		: PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#else
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#endif
+
+	PVRSRVStatsRemoveMemAllocRecord(uiStat, (IMG_UINT64) page_to_phys(psPage),
+	                                psPageArrayData->uiPid);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static inline void _TransferToMemZombieRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                                       const struct page *psPage)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) };
+
+	PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                                  PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,
+	                                  (IMG_UINT64) sCPUPhysAddr.uiAddr,
+	                                  psPageArrayData->uiPid
+	                                  DEBUG_MEMSTATS_VALUES);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+
+static inline void _IncrMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid)
+{
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                            uiSize, uiPid);
+}
+
+static inline void _DecrMemAllocStat_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                              size_t uiSize, IMG_PID uiPid)
+{
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES;
+	}
+#endif
+
+	PVRSRVStatsDecrMemAllocStat(uiStat, uiSize, uiPid);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static inline void _ZombifyMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid)
+{
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                            uiSize, uiPid);
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,
+	                            uiSize, uiPid);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+/* Allocation of OS pages: We may allocate 2^N order pages at a time for two reasons.
+ *
+ * Firstly to support device pages which are larger than OS. By asking the OS for 2^N
+ * order OS pages at a time we guarantee the device page is contiguous.
+ *
+ * Secondly for performance where we may ask for 2^N order pages to reduce the number
+ * of calls to alloc_pages, and thus reduce time for huge allocations.
+ *
+ * Regardless of page order requested, we need to break them down to track _OS pages.
+ * The maximum order requested is increased if all max order allocations were successful.
+ * If any request fails we reduce the max order.
+ */
+static PVRSRV_ERROR
+_AllocOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiArrayIndex = 0;
+	/* Order of the allocation to request, can be upgraded for large allocations
+	 * to aid efficient allocation. */
+	IMG_UINT32 ui32Order;
+	/* Min order the requested allocation can be. We express this in terms of the device
+	 * page size to the OS page size. The OS page size is the minimum page size we can request
+	 * else we cannot map it. Requesters may request further size / contiguity constraints in form
+	 * of the log2AllocPageSize */
+	IMG_UINT32 ui32MinOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bIncreaseMaxOrder = IMG_TRUE;
+
+	IMG_UINT32 ui32NumPageReq;
+	IMG_UINT32 uiOSPagesToAlloc;
+	IMG_UINT32 uiDevPagesFromPool = 0;
+
+	gfp_t gfp_flags = _GetGFPFlags(psPageArrayData);
+	gfp_t ui32GfpFlags;
+	gfp_t ui32HighOrderGfpFlags = ((gfp_flags & ~__GFP_RECLAIM) | __GFP_NORETRY);
+
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	struct page **ppsPageAttributeArray = NULL;
+
+	uiOSPagesToAlloc = psPageArrayData->uiTotalNumOSPages;
+
+	/* Try to get pages from the pool since it is faster;
+	   the page pool currently only supports zero-order pages
+	   thus currently excludes all DMA/CMA allocated memory.
+	   _ShouldInitMem() must not be used for bZero argument since it only
+	   applies to new pages allocated from the kernel.  */
+	_GetPagesFromPoolLocked(psPageArrayData->psDevNode,
+							psPageArrayData->ui32CPUCacheFlags,
+							psPageArrayData->ui32AllocFlags,
+							uiOSPagesToAlloc,
+							ui32MinOrder,
+							ppsPageArray,
+							&uiDevPagesFromPool);
+
+	uiArrayIndex = uiDevPagesFromPool;
+
+	if ((uiOSPagesToAlloc - uiDevPagesFromPool) < PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD &&
+	    !BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{	/* Small allocations: ask for one device page at a time */
+		ui32Order = ui32MinOrder;
+		bIncreaseMaxOrder = IMG_FALSE;
+	}
+	else
+	{
+		/* Large zero-order or none zero-order allocations, ask for
+		   MAX(max-order, min-order) order pages at a time; alloc
+		   failures throttles this down to ZeroOrder allocations */
+		ui32Order = MAX(g_uiMaxOrder, ui32MinOrder);
+	}
+
+	/* Only if asking for more contiguity than we actually need, let it fail */
+	ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+	ui32NumPageReq = (1 << ui32Order);
+
+	while (uiArrayIndex < uiOSPagesToAlloc)
+	{
+		IMG_UINT32 ui32PageRemain = uiOSPagesToAlloc - uiArrayIndex;
+
+		while (ui32NumPageReq > ui32PageRemain)
+		{
+			/* Pages to request is larger than that remaining
+			   so ask for less so never over allocate */
+			ui32Order = MAX(ui32Order >> 1, ui32MinOrder);
+			ui32NumPageReq = (1 << ui32Order);
+			ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+		}
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			/* As the DMA/CMA framework rounds-up request to the
+			   next power-of-two, we request multiple uiMinOrder
+			   pages to satisfy allocation request in order to
+			   minimise wasting memory */
+			eError = _AllocOSPage_CMA(psPageArrayData,
+									  gfp_flags,
+									  ui32Order,
+									  ui32MinOrder,
+									  uiArrayIndex);
+		}
+		else
+		{
+			/* Allocate uiOrder pages at uiArrayIndex */
+			eError = _AllocOSPage(psPageArrayData,
+								  ui32GfpFlags,
+								  ui32Order,
+								  ui32MinOrder,
+								  uiArrayIndex);
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+			if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+			{
+				/* We encode non Ghost entries for CMA so we can record in order chunks */
+				_AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[uiArrayIndex], ui32Order);
+			}
+			else
+			{
+				/* Higher order pages get split using split_page() meaning we can't create a higher
+				 * order record in case the logic below reduces the order request part way through
+				 */
+				IMG_UINT32 i;
+				for (i = 0; i < (1 << ui32Order); i++)
+				{
+					_AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[uiArrayIndex + i], 0);
+				}
+			}
+#endif
+#endif
+			/* Successful request. Move onto next. */
+			uiArrayIndex += ui32NumPageReq;
+		}
+		else
+		{
+			if (ui32Order > ui32MinOrder)
+			{
+				/* Last request failed. Let's ask for less next time */
+				ui32Order = MAX(ui32Order >> 1, ui32MinOrder);
+				bIncreaseMaxOrder = IMG_FALSE;
+				ui32NumPageReq = (1 << ui32Order);
+				ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+				g_uiMaxOrder = ui32Order;
+			}
+			else
+			{
+				/* Failed to alloc pages at required contiguity. Failed allocation */
+				PVR_DPF((PVR_DBG_ERROR, "%s: %s failed to honour request at %u of %u, flags = %x, order = %u (%s)",
+								__func__,
+								BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA) ? "dma_alloc_coherent" : "alloc_pages",
+								uiArrayIndex,
+								uiOSPagesToAlloc,
+								ui32GfpFlags,
+								ui32Order,
+								PVRSRVGetErrorString(eError)));
+				eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+				goto e_free_pages;
+			}
+		}
+	}
+
+	if (bIncreaseMaxOrder && (g_uiMaxOrder < PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM))
+	{	/* All successful allocations on max order. Let's ask for more next time */
+		g_uiMaxOrder++;
+	}
+
+	/* Construct table of page pointers to apply attributes */
+	ppsPageAttributeArray = &ppsPageArray[uiDevPagesFromPool];
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT32 i;
+		dma_addr_t *psBusAddrArray = &psPageArrayData->dmaphysarray[uiDevPagesFromPool];
+
+		/* Iterate over page array generating ghost CMA pages */
+		for (i = 0; i < uiOSPagesToAlloc;)
+		{
+			IMG_UINT32 j;
+			IMG_UINT32 ui32NumPagesPerOrder = 1;
+
+			/* Iterate and populate until we find the next real page */
+			for (j = i + 1;
+			     j < uiOSPagesToAlloc &&
+			     ppsPageAttributeArray[j] == NULL;
+			     j++, ui32NumPagesPerOrder++)
+			{
+				/* Generate ghost page */
+				ppsPageAttributeArray[j] = ppsPageAttributeArray[i];
+				ppsPageAttributeArray[j] += ui32NumPagesPerOrder;
+				/* Mark DMA addr as ghost */
+				psBusAddrArray[j] = DMA_SET_CMA_GHOST(psBusAddrArray[i]);
+			}
+
+			i += ui32NumPagesPerOrder;
+		}
+	}
+
+	if (_ShouldInitMem(psPageArrayData->ui32AllocFlags) && ui32MinOrder == 0)
+	{
+		eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool,
+		                          ppsPageAttributeArray, PAGE_KERNEL,
+		                          PVRSRV_ZERO_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to zero pages (fast)"));
+			goto e_free_pages;
+		}
+	}
+	else if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC))
+	{
+		/* need to call twice because ppsPageArray and ppsPageAttributeArray
+		 * can point to different allocations: first for pages obtained from
+		 * the pool and then the remaining pages */
+		eError = _MemsetPageArray(uiDevPagesFromPool, ppsPageArray, PAGE_KERNEL,
+		                          PVRSRV_POISON_ON_ALLOC_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)"));
+		}
+		eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool,
+		                          ppsPageAttributeArray, PAGE_KERNEL,
+		                          PVRSRV_POISON_ON_ALLOC_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)"));
+		}
+
+		/* for poisoning need to also flush the pool pages as the 0s have
+		 * been overwritten */
+		_ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsPageArray,
+		                       uiDevPagesFromPool);
+	}
+
+	/* Do the cache management as required */
+	eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode,
+									ppsPageAttributeArray,
+									uiOSPagesToAlloc - uiDevPagesFromPool,
+									BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO) ||
+									BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC),
+									psPageArrayData->ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes"));
+		goto e_free_pages;
+	}
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated = psPageArrayData->uiTotalNumOSPages;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		_IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)),
+		                           psPageArrayData->uiPid);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return PVRSRV_OK;
+
+/* Error path */
+e_free_pages:
+	{
+		IMG_UINT32 ui32PageToFree;
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			IMG_UINT32 uiDevArrayIndex = uiArrayIndex >> ui32Order;
+			IMG_UINT32 uiDevPageSize = PAGE_SIZE << ui32Order;
+			PVR_ASSERT(ui32Order == ui32MinOrder);
+
+			for (ui32PageToFree = 0; ui32PageToFree < uiDevArrayIndex; ui32PageToFree++)
+			{
+				if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[ui32PageToFree]))
+				{
+					_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+									uiDevPageSize,
+									ui32MinOrder,
+									psPageArrayData->dmavirtarray[ui32PageToFree],
+									psPageArrayData->dmaphysarray[ui32PageToFree],
+									ppsPageArray[ui32PageToFree]);
+				}
+				psPageArrayData->dmaphysarray[ui32PageToFree]= (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[ui32PageToFree] = NULL;
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+		}
+		else
+		{
+			/* Free the pages we got from the pool */
+			for (ui32PageToFree = 0; ui32PageToFree < uiDevPagesFromPool; ui32PageToFree++)
+			{
+				/* Pages added to pool are always order 0 */
+				_FreeOSPage(0,
+							BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE),
+							psPageArrayData,
+							ppsPageArray[ui32PageToFree]);
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+
+			for (ui32PageToFree = uiDevPagesFromPool; ui32PageToFree < uiArrayIndex; ui32PageToFree++)
+			{
+				/* Higher order pages from _AllocOSPage are always split into order 0 using split_page */
+				_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsPageArray[ui32PageToFree]);
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+		}
+
+		return eError;
+	}
+}
+
+static INLINE PVRSRV_ERROR
+_CheckIfIndexInRange(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 ui32Limit)
+{
+	if (pui32Indices[ui32Index] >= ui32Limit)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Given alloc index %u at %u is larger than page array %u.",
+		        __func__, pui32Indices[ui32Index], ui32Index, ui32Limit));
+		return PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+_CheckIfPageNotAllocated(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 uiOrder, struct page **ppsPageArray)
+{
+	if (ppsPageArray[pui32Indices[ui32Index] << uiOrder] != NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Mapping Idx Dev:%u|OS:%u at page array index %u already exists. "
+		        "Page struct %p", __func__,
+		        pui32Indices[ui32Index],
+		        pui32Indices[ui32Index] << uiOrder,
+		        ui32Index,
+		        ppsPageArray[pui32Indices[ui32Index] << uiOrder]));
+		return PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Allocation of OS pages: This function is used for sparse allocations.
+ *
+ * Sparse allocations provide only a proportion of sparse physical backing within the total
+ * virtual range. */
+static PVRSRV_ERROR
+_AllocOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+					 IMG_UINT32 *pauiDevPageAllocIndices,
+					 IMG_UINT32 uiDevPagesToAlloc)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_UINT32 uiDevPagesFromPool = 0;
+	IMG_UINT32 uiOSPagesToAlloc = uiDevPagesToAlloc * (1 << uiOrder);
+	IMG_UINT32 uiDevPagesAllocated = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+	const IMG_UINT32 ui32AllocFlags = psPageArrayData->ui32AllocFlags;
+	gfp_t ui32GfpFlags = _GetGFPFlags(psPageArrayData);
+
+	/* We use this page array to receive pages from the pool and then reuse it afterwards to
+	 * store pages that need their cache attribute changed on x86 */
+	struct page **ppsTempPageArray;
+	IMG_UINT32 uiTempPageArrayIndex = 0;
+
+	/* Allocate the temporary page array that we need here to receive pages
+	 * from the pool and to store pages that need their caching attributes changed.
+	 * Allocate number of OS pages to be able to use the attribute function later. */
+	ppsTempPageArray = OSAllocMem(sizeof(struct page*) * uiOSPagesToAlloc);
+	PVR_LOG_GOTO_IF_NOMEM(ppsTempPageArray, eError, e_exit);
+
+	/* Check the requested number of pages if they fit in the page array */
+	if (uiDevPagesAllocated <
+	        ((psPageArrayData->iNumOSPagesAllocated >> uiOrder) + uiDevPagesToAlloc))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Trying to allocate more pages (Order %u) than this buffer can handle, "
+				 "Request + Allocated < Max! Request %u, Allocated %u, Max %u.",
+				 __func__,
+				 uiOrder,
+				 uiDevPagesToAlloc,
+				 psPageArrayData->iNumOSPagesAllocated >> uiOrder,
+				 uiDevPagesAllocated));
+		eError = PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+		goto e_free_temp_array;
+	}
+
+	/* Try to get pages from the pool since it is faster. The pages from pool are going to be
+	 * allocated only if:
+	 * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 1 && uiOrder == 0
+	 * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 0 && uiOrder == 0 &&
+	 *   !(BIT_ISSET(ui32AllocFlags, FLAG_ZERO))
+	 * - !BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA)
+	 * _ShouldInitMem() must not be used for bZero argument since it only
+	 * applies to new pages allocated from the kernel.  */
+	_GetPagesFromPoolLocked(psPageArrayData->psDevNode,
+							psPageArrayData->ui32CPUCacheFlags,
+							psPageArrayData->ui32AllocFlags,
+							uiDevPagesToAlloc,
+							uiOrder,
+							ppsTempPageArray,
+							&uiDevPagesFromPool);
+
+	/* In general device pages can have higher order than 0 but page pool always provides only 0
+	 * order pages so they can be assigned to the OS pages values (in other words if we're
+	 * allocating non-4k pages uiDevPagesFromPool will always be 0) */
+	uiTempPageArrayIndex = uiDevPagesFromPool;
+
+	/* Move pages we got from the pool to the array. */
+	for (i = 0; i < uiDevPagesFromPool; i++)
+	{
+		eError = _CheckIfIndexInRange(i, pauiDevPageAllocIndices, uiDevPagesAllocated);
+		PVR_GOTO_IF_ERROR(eError, e_free_pool_pages);
+		eError = _CheckIfPageNotAllocated(i, pauiDevPageAllocIndices, uiOrder, ppsPageArray);
+		PVR_GOTO_IF_ERROR(eError, e_free_pool_pages);
+
+		/* In this case the allocation order is the same for device and OS page size, we can use
+		 * DevPageAllocIndices here without having to scale up from device to OS page indices.
+		 */
+		PVR_ASSERT(uiOrder == 0);
+		ppsPageArray[pauiDevPageAllocIndices[i]] = ppsTempPageArray[i];
+	}
+
+	/* Allocate pages from the OS */
+	for (i = uiDevPagesFromPool; i < uiDevPagesToAlloc; i++)
+	{
+		IMG_UINT32 uiOSPageAllocIndex = pauiDevPageAllocIndices[i] << uiOrder;
+		/* We can perform range checks in terms of device pages, then we scale the indices
+		 * to OS page size for the storage array.
+		 */
+		eError = _CheckIfIndexInRange(i, pauiDevPageAllocIndices, uiDevPagesAllocated);
+		PVR_GOTO_IF_ERROR(eError, e_free_pages);
+		eError = _CheckIfPageNotAllocated(i, pauiDevPageAllocIndices, uiOrder, ppsPageArray);
+		PVR_GOTO_IF_ERROR(eError, e_free_pages);
+
+		/* Allocated pages and assign them the array. */
+		if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			IMG_UINT32 uiSubPageInOrder;
+			dma_addr_t *psBusAddrArray = &psPageArrayData->dmaphysarray[uiDevPagesFromPool];
+
+			/* As the DMA/CMA framework rounds-up request to the
+			   next power-of-two, we request multiple uiMinOrder
+			   pages to satisfy allocation request in order to
+			   minimise wasting memory */
+			eError = _AllocOSPage_CMA(psPageArrayData,
+									  ui32GfpFlags,
+									  uiOrder,
+									  uiOrder,
+									  uiOSPageAllocIndex);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages"));
+				goto e_free_pages;
+			}
+
+			for (uiSubPageInOrder = 1;
+			     uiSubPageInOrder < (1 << uiOrder);
+			     uiSubPageInOrder++)
+			{
+				/* Generate ghost page */
+				ppsPageArray[uiOSPageAllocIndex + uiSubPageInOrder] = ppsPageArray[uiOSPageAllocIndex];
+				ppsPageArray[uiOSPageAllocIndex + uiSubPageInOrder] += uiSubPageInOrder;
+				/* Mark DMA addr as ghost */
+				psBusAddrArray[uiOSPageAllocIndex + uiSubPageInOrder] = DMA_SET_CMA_GHOST(psBusAddrArray[uiOSPageAllocIndex]);
+			}
+		}
+		else
+		{
+			eError = _AllocOSPage(psPageArrayData,
+								  ui32GfpFlags,
+								  uiOrder,
+								  uiOrder,
+								  uiOSPageAllocIndex);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages"));
+				goto e_free_pages;
+			}
+		}
+
+		if (ppsPageArray[uiOSPageAllocIndex] != NULL)
+		{
+			/* Append pages to the temporary array so it's easier to process
+			 * them later on. */
+
+			if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+			{
+				IMG_UINT32 idx;
+				struct page *psPageAddr;
+
+				psPageAddr = ppsPageArray[uiOSPageAllocIndex];
+
+				/* "divide" CMA pages into OS pages if they have higher order */
+				for (idx = 0; idx < (1 << uiOrder); idx++)
+				{
+					ppsTempPageArray[uiTempPageArrayIndex + idx] = psPageAddr;
+					psPageAddr++;
+				}
+				uiTempPageArrayIndex += (1 << uiOrder);
+			}
+			else
+			{
+				ppsTempPageArray[uiTempPageArrayIndex] = ppsPageArray[uiOSPageAllocIndex];
+				uiTempPageArrayIndex++;
+			}
+		}
+		else
+		{
+			/* Failed to alloc pages at required contiguity. Failed allocation */
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: alloc_pages failed to honour request at %u of %u, flags = %x, order = %u",
+			        __func__, i, uiDevPagesToAlloc, ui32GfpFlags, uiOrder));
+			eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+			goto e_free_pages;
+		}
+	}
+
+	if (_ShouldInitMem(ui32AllocFlags) && uiOrder == 0)
+	{
+		/* At this point this array contains pages allocated from the page pool at its start
+		 * and pages allocated from the OS after that.
+		 * If there are pages from the pool here they must be zeroed already hence we don't have
+		 * to do it again. This is because if PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES is enabled pool pages
+		 * are zeroed in the cleanup thread. If it's disabled they aren't, and in that case we never
+		 * allocate pages with FLAG_ZERO from the pool. This is why those pages need to be zeroed
+		 * here.
+		 * All of the above is true for the 0 order pages. For higher order we never allocated from
+		 * the pool and those pages are allocated already zeroed from the OS.
+		 * Long story short we can always skip pages allocated from the pool because they are either
+		 * zeroed or we didn't allocate any of them. */
+		eError = _MemsetPageArray(uiTempPageArrayIndex - uiDevPagesFromPool,
+		                          &ppsTempPageArray[uiDevPagesFromPool],
+		                          PAGE_KERNEL, PVRSRV_ZERO_VALUE);
+		PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to zero pages (sparse)", e_free_pages);
+	}
+	else if (BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC))
+	{
+		/* Here we need to poison all of the pages regardless if they were
+		 * allocated from the pool or from the system. */
+		eError = _MemsetPageArray(uiTempPageArrayIndex, ppsTempPageArray,
+		                          PAGE_KERNEL, PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "failed to poison pages (sparse)");
+
+		/* We need to flush the cache for the poisoned pool pages here. The flush for the pages
+		 * allocated from the system is done below because we also need to add appropriate cache
+		 * attributes to them. Pages allocated from the pool already come with correct caching
+		 * mode. */
+		_ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsTempPageArray, uiDevPagesFromPool);
+	}
+
+	/* Do the cache management as required */
+	eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode,
+	                                &ppsTempPageArray[uiDevPagesFromPool],
+	                                uiTempPageArrayIndex - uiDevPagesFromPool,
+	                                BIT_ISSET(ui32AllocFlags, FLAG_ZERO) ||
+	                                BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC),
+	                                psPageArrayData->ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes"));
+		goto e_free_pages;
+	}
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated += uiOSPagesToAlloc;
+
+	/* Free temporary page array */
+	OSFreeMem(ppsTempPageArray);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiDevPagesToAlloc; i++)
+	{
+		_AddMemAllocRecord_UmaPages(psPageArrayData,
+		                            ppsPageArray[pauiDevPageAllocIndices[i] << uiOrder],
+		                            uiOrder);
+	}
+#else
+	_IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)),
+	                           psPageArrayData->uiPid);
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+e_free_pages:
+	if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder;
+
+		/* Free the pages we just allocated from the CMA */
+		for (; i > uiDevPagesFromPool; i--)
+		{
+			IMG_UINT32 idx = (i - 1) << uiOrder;
+			IMG_UINT32 uiSubPageInOrder;
+			_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+			                uiDevPageSize,
+			                uiOrder,
+			                psPageArrayData->dmavirtarray[pauiDevPageAllocIndices[idx]],
+			                psPageArrayData->dmaphysarray[pauiDevPageAllocIndices[idx]],
+			                ppsPageArray[pauiDevPageAllocIndices[idx]]);
+			psPageArrayData->dmaphysarray[pauiDevPageAllocIndices[idx]]= (dma_addr_t) 0;
+			psPageArrayData->dmavirtarray[pauiDevPageAllocIndices[idx]] = NULL;
+			ppsPageArray[pauiDevPageAllocIndices[idx]] = NULL;
+
+			for (uiSubPageInOrder = 1;
+				 uiSubPageInOrder < (1 << uiOrder);
+				 uiSubPageInOrder++)
+			{
+				ppsPageArray[idx + uiSubPageInOrder] = NULL;
+				psPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+			}
+		}
+
+	}
+	else
+	{
+		/* Free the pages we just allocated from the OS */
+		for (; i > uiDevPagesFromPool; i--)
+		{
+			_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsPageArray[pauiDevPageAllocIndices[i-1]]);
+			ppsPageArray[pauiDevPageAllocIndices[i-1]] = NULL;
+		}
+	}
+
+e_free_pool_pages:
+	/* And now free all of the pages we allocated from the pool. */
+	for (i = 0; i < uiDevPagesFromPool; i++)
+	{
+		_FreeOSPage(0, BIT_ISSET(ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE),
+		            psPageArrayData,
+		            ppsTempPageArray[i]);
+
+		/* not using _CheckIfIndexInRange() to not print error message */
+		if (pauiDevPageAllocIndices[i] < uiDevPagesAllocated)
+		{
+			ppsPageArray[pauiDevPageAllocIndices[i]] = NULL;
+		}
+	}
+
+e_free_temp_array:
+	OSFreeMem(ppsTempPageArray);
+
+e_exit:
+	return eError;
+}
+
+/* Allocate pages for a given page array.
+ *
+ * The executed allocation path depends whether an array with allocation
+ * indices has been passed or not */
+static PVRSRV_ERROR
+_AllocOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			  IMG_UINT32 *puiAllocIndices,
+			  IMG_UINT32 uiPagesToAlloc)
+{
+	PVRSRV_ERROR eError;
+	struct page **ppsPageArray;
+
+	/* Parameter checks */
+	PVR_ASSERT(NULL != psPageArrayData);
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		PVR_ASSERT(psPageArrayData->dmaphysarray != NULL);
+		PVR_ASSERT(psPageArrayData->dmavirtarray != NULL);
+	}
+	PVR_ASSERT(psPageArrayData->pagearray != NULL);
+	PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated);
+
+	ppsPageArray = psPageArrayData->pagearray;
+
+	/* Go the sparse alloc path if we have an array with alloc indices.*/
+	if (puiAllocIndices != NULL)
+	{
+		eError = _AllocOSPages_Sparse(psPageArrayData,
+									  puiAllocIndices,
+									  uiPagesToAlloc);
+	}
+	else
+	{
+		eError = _AllocOSPages_Fast(psPageArrayData);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e_exit;
+	}
+
+	_DumpPageArray(ppsPageArray,
+	               psPageArrayData->uiTotalNumOSPages >>
+	               (psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT) );
+
+	PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: allocated OS memory for PMR @0x%p", psPageArrayData));
+	return PVRSRV_OK;
+
+e_exit:
+	return eError;
+}
+
+/* Same as _FreeOSPage except free memory using DMA framework */
+static INLINE void
+_FreeOSPage_CMA(struct device *dev,
+				size_t alloc_size,
+				IMG_UINT32 uiOrder,
+				void *virt_addr,
+				dma_addr_t dev_addr,
+				struct page *psPage)
+{
+	if (DMA_IS_ALLOCPG_ADDR(dev_addr))
+	{
+#if defined(CONFIG_X86)
+		void *pvPageVAddr = page_address(psPage);
+		if (pvPageVAddr)
+		{
+			int ret = set_memory_wb((unsigned long)pvPageVAddr, 1);
+			if (ret)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to reset page attribute",
+						__func__));
+			}
+		}
+#endif
+
+		if (DMA_IS_ADDR_ADJUSTED(dev_addr))
+		{
+			psPage -= DMA_GET_ALIGN_ADJUSTMENT(dev_addr);
+			uiOrder += 1;
+		}
+
+		__free_pages(psPage, uiOrder);
+	}
+	else
+	{
+		if (DMA_IS_ADDR_ADJUSTED(dev_addr))
+		{
+			size_t align_adjust;
+
+			align_adjust = DMA_GET_ALIGN_ADJUSTMENT(dev_addr);
+			alloc_size = alloc_size << 1;
+
+			dev_addr = DMA_GET_ADDR(dev_addr);
+			dev_addr -= align_adjust << PAGE_SHIFT;
+			virt_addr -= align_adjust << PAGE_SHIFT;
+		}
+
+#if defined(DEBUG)
+		/* We should never attempt to free a CMA Ghost as it is handled in layer above */
+		if (DMA_IS_CMA_GHOST(dev_addr))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Attempt to free CMA Ghost entry!"));
+			return;
+		}
+#endif
+
+		dma_free_coherent(dev, alloc_size, virt_addr, DMA_GET_ADDR(dev_addr));
+	}
+}
+
+/* Free a single page back to the OS.
+ * Make sure the cache type is set back to the default value.
+ *
+ * Note:
+ * We must _only_ check bUnsetMemoryType in the case where we need to free
+ * the page back to the OS since we may have to revert the cache properties
+ * of the page to the default as given by the OS when it was allocated. */
+static void
+_FreeOSPage(IMG_UINT32 uiOrder,
+			IMG_BOOL bUnsetMemoryType,
+			PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			struct page *psPage)
+{
+
+#if defined(CONFIG_X86)
+	void *pvPageVAddr;
+	pvPageVAddr = page_address(psPage);
+
+	if (pvPageVAddr && bUnsetMemoryType)
+	{
+		int ret;
+
+		ret = set_memory_wb((unsigned long)pvPageVAddr, 1);
+		if (ret)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute",
+					 __func__));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(bUnsetMemoryType);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* May call _FreeOSPage from clean pages for pool path, it has no
+	 * association with PMR_OSPAGEARRAY_DATA at this point
+	 */
+	if (psPageArrayData)
+	{
+		OSMemUnsetMovablePageAttr(psPage, psPageArrayData);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psPageArrayData);
+#endif
+
+	__free_pages(psPage, uiOrder);
+}
+
+/* Free the struct holding the metadata */
+static void
+_FreeOSPagesArray(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: freed OS memory for PMR @0x%p", psPageArrayData));
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (!dllist_is_empty(&psPageArrayData->sCpuMappingListHead))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Attempt to free linux cpu mapping data while still in use."));
+		OSWarnOn(1);
+	}
+#endif
+
+	psPageArrayData->hPMR = NULL;
+
+	/* Check if the page array actually still exists.
+	 * It might be the case that has been moved to the page pool */
+	if (psPageArrayData->pagearray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->pagearray);
+	}
+
+	/* Check if we need to free additional DMA/CMA cpu kernel virtual address & device bus address arrays */
+	if (psPageArrayData->dmaphysarray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->dmaphysarray);
+	}
+	if (psPageArrayData->dmavirtarray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->dmavirtarray);
+	}
+
+	kmem_cache_free(g_psLinuxPageArray, psPageArrayData);
+}
+
+/* Free all or some pages from a sparse page array */
+static PVRSRV_ERROR
+_FreeOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+					IMG_UINT32 *paui32DevPageFreeIndices,
+					IMG_UINT32 uiDevPagesToFree)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	IMG_UINT32 uiNumDevPages;
+	struct page **ppsOSPagesToFree;
+	IMG_UINT32 *puiDevIndicesToFree;
+	IMG_UINT32 uiOSPageCount;
+	IMG_UINT32 uiTransferedDevPageCount;
+	IMG_UINT32 i;
+
+	/* We really should have something to free before we call this */
+	PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0);
+
+	if (paui32DevPageFreeIndices == NULL)
+	{
+		uiNumDevPages = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+		uiOSPageCount = psPageArrayData->iNumOSPagesAllocated;
+	}
+	else
+	{
+		uiNumDevPages = uiDevPagesToFree;
+		uiOSPageCount = uiNumDevPages << uiOrder;
+	}
+
+	/* Construct the array of pages before any free operation so error recovery is possible.
+	 * OSAllocMemNoStats required because this code may be run without the bridge lock held */
+	ppsOSPagesToFree = OSAllocMemNoStats(uiOSPageCount * sizeof(*ppsOSPagesToFree));
+	PVR_LOG_GOTO_IF_NOMEM(ppsOSPagesToFree, eError, err_array);
+	puiDevIndicesToFree = OSAllocMemNoStats(uiNumDevPages * sizeof(*puiDevIndicesToFree));
+	PVR_LOG_GOTO_IF_NOMEM(puiDevIndicesToFree, eError, err_indices);
+
+	/* Count how many allocated dev pages have been found. In case we need to unwind */
+	uiTransferedDevPageCount = 0;
+	for (i = 0; i < uiNumDevPages; i++)
+	{
+		IMG_UINT32 uiSubPageInOrder;
+		IMG_UINT32 idx = paui32DevPageFreeIndices ? paui32DevPageFreeIndices[i] : i;
+		/* Scale Device page index to OS page Idx */
+		idx <<= uiOrder;
+
+		if (ppsPageArray[idx] != NULL)
+		{
+			IMG_UINT32 uiOSFreeArrayIdx = uiTransferedDevPageCount << uiOrder;
+			puiDevIndicesToFree[uiTransferedDevPageCount] = idx;
+
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				ppsOSPagesToFree[uiOSFreeArrayIdx + uiSubPageInOrder] = ppsPageArray[idx + uiSubPageInOrder];
+				ppsPageArray[idx + uiSubPageInOrder] = NULL;
+			}
+			uiTransferedDevPageCount++;
+		}
+		else if (paui32DevPageFreeIndices != NULL)
+		{
+			IMG_UINT32 j;
+			PVR_DPF((PVR_DBG_ERROR, "Invalid free index (%u), possible duplicate", idx));
+			/* Restore the ppsPageArray to its initial state.
+			 * Reverse the loop and swap the ppsTempPages back into ppsPageArray. */
+			for (j = 0; j < uiTransferedDevPageCount; j++)
+			{
+				IMG_UINT32 uiSubPageInOrder;
+
+				idx = puiDevIndicesToFree[j];
+				ppsPageArray[idx] = ppsOSPagesToFree[j];
+
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					ppsPageArray[idx + uiSubPageInOrder] = ppsOSPagesToFree[j + uiSubPageInOrder];
+				}
+			}
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK, err_indices);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiTransferedDevPageCount; i++)
+	{
+		/* Scale Device page index to OS page Idx */
+		IMG_UINT32 idx = i << uiOrder;
+
+		_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsOSPagesToFree[idx]);
+	}
+#endif /* PVRSRV_ENABLE_PROCESS_STATS && PVRSRV_ENABLE_MEMORY_STATS */
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE))
+	{
+		for (i = 0; i < uiTransferedDevPageCount; i++)
+		{
+			/* Scale Device page index to OS page Idx */
+			IMG_UINT32 idx = i << uiOrder;
+
+			_PoisonDevicePage(psPageArrayData->psDevNode,
+			                  ppsOSPagesToFree[idx],
+			                  uiOrder,
+			                  psPageArrayData->ui32CPUCacheFlags,
+			                  PVRSRV_POISON_ON_FREE_VALUE);
+		}
+	}
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT64 ui64DevPageSize = IMG_PAGE2BYTES64(psPageArrayData->uiLog2DevPageSize);
+		for (i = 0; i < uiTransferedDevPageCount; i++)
+		{
+			IMG_UINT32 uiSubPageInOrder;
+			/* Scale Device page index to OS page Idx */
+			IMG_UINT32 uiFreeIdx = i << uiOrder;
+			/* Get the page's original index in ppsPageArray, already scaled to dev pages */
+			IMG_UINT32 idx = puiDevIndicesToFree[i];
+			_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+			                ui64DevPageSize,
+			                uiOrder,
+			                psPageArrayData->dmavirtarray[idx],
+			                psPageArrayData->dmaphysarray[idx],
+			                ppsOSPagesToFree[uiFreeIdx]);
+			for (uiSubPageInOrder = 0;
+			     uiSubPageInOrder < (1 << uiOrder);
+			     uiSubPageInOrder++)
+			{
+				psPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+			}
+		}
+	}
+	else
+	{
+		IMG_BOOL bPagesInPool;
+
+		/* Try to move the temp page array to the pool */
+		bPagesInPool = _PutPagesToPoolLocked(psPageArrayData->psDevNode,
+		                                     psPageArrayData->ui32CPUCacheFlags,
+		                                     psPageArrayData->ui32AllocFlags,
+		                                     ppsOSPagesToFree,
+		                                     0,
+		                                     uiOSPageCount);
+		if (bPagesInPool)
+		{
+			/* Do NOT free ppsPagesToFree if it moved to the pool */
+			ppsOSPagesToFree = NULL;
+		}
+		else
+		{
+			/* Free pages and reset page caching attributes on x86 */
+#if defined(CONFIG_X86)
+			if (uiOSPageCount != 0 && BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE))
+			{
+				int iError;
+				iError = set_pages_array_wb(ppsOSPagesToFree, uiOSPageCount);
+
+				if (iError)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes", __func__));
+				}
+			}
+#endif
+
+			/* Free the pages */
+			for (i = 0; i < uiOSPageCount; i++)
+			{
+				_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsOSPagesToFree[i]);
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	_DecrMemAllocStat_UmaPages(psPageArrayData,
+	                           (uiOSPageCount * PAGE_SIZE) - psPageArrayData->ui32CMAAdjustedPageCount,
+	                           psPageArrayData->uiPid);
+#endif
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated -= uiOSPageCount;
+	PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated);
+
+err_indices:
+	OSFreeMemNoStats(puiDevIndicesToFree);
+err_array:
+	if (ppsOSPagesToFree)
+	{
+		OSFreeMemNoStats(ppsOSPagesToFree);
+	}
+	return eError;
+}
+
+/* Free all the pages in a page array */
+static PVRSRV_ERROR
+_FreeOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	IMG_BOOL bSuccess;
+	IMG_UINT32 i;
+	IMG_UINT32 uiOSNumPages = psPageArrayData->uiTotalNumOSPages;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_UINT32 uiDevNumPages = uiOSNumPages >> uiOrder;
+	IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+
+	/* We really should have something to free before we call this */
+	PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiOSNumPages; i++)
+	{
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+			{
+				_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+			}
+		}
+		else
+		{
+			_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+		}
+	}
+#else
+	_DecrMemAllocStat_UmaPages(psPageArrayData, uiOSNumPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+	                           psPageArrayData->uiPid);
+#endif
+#endif
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE))
+	{
+		for (i = 0; i < uiDevNumPages; i++)
+		{
+			/* Scale Device page index to OS page idx */
+			IMG_UINT32 idx = i << uiOrder;
+
+			_PoisonDevicePage(psPageArrayData->psDevNode,
+			                  ppsPageArray[idx],
+			                  uiOrder,
+			                  psPageArrayData->ui32CPUCacheFlags,
+			                  PVRSRV_POISON_ON_FREE_VALUE);
+		}
+	}
+
+	/* Try to move the page array to the pool */
+	bSuccess = _PutPagesToPoolLocked(psPageArrayData->psDevNode,
+	                                 psPageArrayData->ui32CPUCacheFlags,
+	                                 psPageArrayData->ui32AllocFlags,
+	                                 ppsPageArray,
+	                                 uiOrder,
+	                                 uiOSNumPages);
+	if (bSuccess)
+	{
+		psPageArrayData->pagearray = NULL;
+		goto exit_ok;
+	}
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		for (i = 0; i < uiOSNumPages; i++)
+		{
+			if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+			{
+				_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+								uiDevPageSize,
+								uiOrder,
+								psPageArrayData->dmavirtarray[i],
+								psPageArrayData->dmaphysarray[i],
+								ppsPageArray[i]);
+
+			}
+			psPageArrayData->dmaphysarray[i] = (dma_addr_t)0;
+			psPageArrayData->dmavirtarray[i] = NULL;
+			ppsPageArray[i] = NULL;
+		}
+	}
+	else
+	{
+#if defined(CONFIG_X86)
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE))
+		{
+			int ret;
+
+			ret = set_pages_array_wb(ppsPageArray, uiOSNumPages);
+			if (ret)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes",
+						 __func__));
+			}
+		}
+#endif
+
+		for (i = 0; i < uiOSNumPages; i++)
+		{
+			_FreeOSPage(uiOrder, IMG_FALSE, psPageArrayData, ppsPageArray[i]);
+			ppsPageArray[i] = NULL;
+		}
+	}
+
+exit_ok:
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated = 0;
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRFreeZombiePagesOSMem(PMR_IMPL_ZOMBIEPAGES pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psZombiePageArray = pvPriv;
+
+	eError = _FreeOSPages(psZombiePageArray,
+	                      NULL,
+	                      0 /* Unused */);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	_FreeOSPagesArray(psZombiePageArray);
+
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+
+/* Extracts ui32ExtractPageCount of pages referenced in pai32ExtractIndices from the psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted pages information.
+ */
+static PVRSRV_ERROR
+_ExtractPages(PMR_OSPAGEARRAY_DATA *psSrcPageArrayData,
+			  IMG_UINT32 *pai32ExtractIndices,
+			  IMG_UINT32 ui32ExtractPageCount,
+			  PMR_OSPAGEARRAY_DATA** psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, uiSubPageInOrder, uiOrder;
+	PMR_OSPAGEARRAY_DATA* psDstPageArrayData;
+
+	/* Alloc PMR_OSPAGEARRAY_DATA for the extracted pages */
+	eError = _AllocOSPageArray(psSrcPageArrayData->psDevNode,
+	                           (IMG_UINT64)ui32ExtractPageCount << psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->ui32AllocFlags,
+	                           psSrcPageArrayData->ui32CPUCacheFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_AllocOSPageArray failed in _ExtractPages"));
+		return eError;
+	}
+
+	uiOrder = psSrcPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+
+	/* Transfer from src pagearray to dst pagearray */
+	for (i = 0; i < ui32ExtractPageCount; i++)
+	{
+		IMG_UINT32 idxSrc = pai32ExtractIndices[i] << uiOrder;
+		IMG_UINT32 idxExtracted = i << uiOrder;
+
+		if (psSrcPageArrayData->pagearray[idxSrc] != NULL)
+		{
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				psDstPageArrayData->pagearray[idxExtracted + uiSubPageInOrder] =
+				    psSrcPageArrayData->pagearray[idxSrc + uiSubPageInOrder];
+
+				psSrcPageArrayData->pagearray[idxSrc + uiSubPageInOrder] = NULL;
+			}
+		}
+	}
+
+	/* Do the same for dmaphysarray and dmavirtarray if allocated with CMA */
+	if (BIT_ISSET(psSrcPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		for (i = 0; i < ui32ExtractPageCount; i++)
+		{
+			IMG_UINT32 idxSrc = pai32ExtractIndices[i] << uiOrder;
+			IMG_UINT32 idxDst = i << uiOrder;
+
+			if (psSrcPageArrayData->dmaphysarray[idxSrc] != (dma_addr_t)0 ||
+			    psSrcPageArrayData->dmavirtarray[idxSrc] != NULL)
+			{
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					psDstPageArrayData->dmaphysarray[idxDst + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmaphysarray[idxSrc + uiSubPageInOrder];
+					psDstPageArrayData->dmavirtarray[idxDst + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmavirtarray[idxSrc + uiSubPageInOrder];
+
+					psSrcPageArrayData->dmaphysarray[idxSrc + uiSubPageInOrder] = (dma_addr_t)0;
+					psSrcPageArrayData->dmavirtarray[idxSrc + uiSubPageInOrder] = NULL;
+				}
+			}
+		}
+	}
+
+	/* Update page counts */
+	psSrcPageArrayData->iNumOSPagesAllocated -= ui32ExtractPageCount << uiOrder;
+	psDstPageArrayData->iNumOSPagesAllocated += ui32ExtractPageCount << uiOrder;
+
+	*psOutPageArrayData = psDstPageArrayData;
+	return PVRSRV_OK;
+}
+
+/* Extracts all allocated pages referenced psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted
+ * pages information.
+ */
+static PVRSRV_ERROR
+_ExtractAllPages(PMR_OSPAGEARRAY_DATA *psSrcPageArrayData,
+				 PMR_OSPAGEARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	PMR_OSPAGEARRAY_DATA* psDstPageArrayData;
+	IMG_UINT32 uiPagesCopied = 0;
+
+	/* Alloc PMR_OSPAGEARRAY_DATA for the extracted pages */
+	eError = _AllocOSPageArray(psSrcPageArrayData->psDevNode,
+	                           (IMG_UINT64)psSrcPageArrayData->iNumOSPagesAllocated << psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->ui32AllocFlags,
+	                           psSrcPageArrayData->ui32CPUCacheFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_AllocOSPageArray failed in _ExtractPages"));
+		return eError;
+	}
+
+	/* Transfer from src pagearray to dst pagearray */
+	/* Iterate through all pages in psSrcPageArrayData but stop once
+	 * we have copied psSrcPageArrayData->iNumOSPagesAllocated pages to
+	 * psDstPageArrayData.
+	 */
+
+	for (i = 0; ((i < psSrcPageArrayData->uiTotalNumOSPages) &&
+	             (uiPagesCopied < psSrcPageArrayData->iNumOSPagesAllocated)); i++)
+	{
+		if (psSrcPageArrayData->pagearray[i] != NULL)
+		{
+			psDstPageArrayData->pagearray[uiPagesCopied] =
+			        psSrcPageArrayData->pagearray[i];
+
+			psSrcPageArrayData->pagearray[i] = NULL;
+
+			if (BIT_ISSET(psSrcPageArrayData->ui32AllocFlags, FLAG_DMA_CMA) &&
+			    (psSrcPageArrayData->dmaphysarray[i] != (dma_addr_t)0 ||
+			     psSrcPageArrayData->dmavirtarray[i] != NULL))
+			{
+				psDstPageArrayData->dmaphysarray[uiPagesCopied] =
+				    psSrcPageArrayData->dmaphysarray[i];
+				psDstPageArrayData->dmavirtarray[uiPagesCopied] =
+				    psSrcPageArrayData->dmavirtarray[i];
+
+				psSrcPageArrayData->dmaphysarray[i] = (dma_addr_t)0;
+				psSrcPageArrayData->dmavirtarray[i] = NULL;
+			}
+			uiPagesCopied++;
+		}
+	}
+	/* Update page counts */
+	psDstPageArrayData->iNumOSPagesAllocated = psSrcPageArrayData->iNumOSPagesAllocated;
+	psSrcPageArrayData->iNumOSPagesAllocated = 0;
+
+	*psOutPageArrayData = psDstPageArrayData;
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+/* Free pages from a page array.
+ * Takes care of mem stats and chooses correct free path depending on parameters. */
+static PVRSRV_ERROR
+_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			 IMG_UINT32 *pai32FreeIndices,
+			 IMG_UINT32 ui32FreePageCount)
+{
+	PVRSRV_ERROR eError;
+
+	/* Go the sparse or non-sparse path */
+	if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages
+		|| pai32FreeIndices != NULL)
+	{
+		eError = _FreeOSPages_Sparse(psPageArrayData,
+									 pai32FreeIndices,
+									 ui32FreePageCount);
+	}
+	else
+	{
+		eError = _FreeOSPages_Fast(psPageArrayData);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_FreeOSPages_FreePages failed"));
+	}
+
+	_DumpPageArray(psPageArrayData->pagearray,
+	               psPageArrayData->uiTotalNumOSPages >>
+	              (psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT) );
+
+	return eError;
+}
+
+/*
+ *
+ * Implementation of callback functions
+ *
+ */
+
+/* Destruction function is called after last reference disappears,
+ * but before PMR itself is freed.
+ */
+static void
+PMRFinalizeOSMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+
+	/* We can't free pages until now. */
+	if (psOSPageArrayData->iNumOSPagesAllocated != 0)
+	{
+
+		eError = _FreeOSPages(psOSPageArrayData, NULL, 0);
+		PVR_LOG_IF_ERROR(eError, "_FreeOSPages");
+	}
+
+	_FreeOSPagesArray(psOSPageArrayData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static PVRSRV_ERROR PMRZombifyOSMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PMR_OSPAGEARRAY_DATA *psPageArrayData = pvPriv;
+	IMG_UINT32 uiNumOSPages = psPageArrayData->uiTotalNumOSPages;
+
+	BIT_SET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE);
+
+	/* no need to check for free indices as it's always all the memory we're
+	 * freeing */
+	if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages)
+	{
+		/* _FreeOSPages_Sparse() path */
+
+		struct page *const *const ppsPageArray = psPageArrayData->pagearray;
+		const IMG_UINT32 uiFlags = psPageArrayData->ui32AllocFlags;
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_UINT32 i;
+
+		if (BIT_ISSET(uiFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (ppsPageArray[i] != NULL && !DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+		else
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (ppsPageArray[i] != NULL)
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+		const IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+		IMG_UINT32 uiNumDevPages = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+		IMG_UINT32 i, j, uiAllocatedNumPages = 0;
+
+		if (BIT_ISSET(uiFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumDevPages; i++)
+			{
+				IMG_UINT32 idx = i << uiOrder;
+				if (ppsPageArray[idx] != NULL)
+				{
+					uiAllocatedNumPages++;
+				}
+			}
+			uiAllocatedNumPages <<= uiOrder;
+		}
+		else
+		{
+			for (i = 0; i < uiNumDevPages; i++)
+			{
+				IMG_UINT32 idx = i << uiOrder;
+				if (ppsPageArray[idx] != NULL)
+				{
+					for (j = 0; j < (1<<uiOrder); j++)
+					{
+						uiAllocatedNumPages++;
+					}
+				}
+			}
+		}
+
+		_ZombifyMemAllocStat_UmaPages(
+		    uiAllocatedNumPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+		    psPageArrayData->uiPid
+		);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+	else
+	{
+		/* _FreeOSPages_Fast() path */
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		struct page *const *const ppsPageArray = psPageArrayData->pagearray;
+		IMG_UINT32 i;
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+		else
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+			}
+		}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+		_ZombifyMemAllocStat_UmaPages(
+		    uiNumOSPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+		    psPageArrayData->uiPid
+		);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#else /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Callback function for locking the system physical page addresses.
+ * This function must be called before the lookup address func. */
+static PVRSRV_ERROR
+PMRLockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+
+	if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		/* Allocate Memory for deferred allocation */
+		eError = _AllocOSPages(psOSPageArrayData, NULL, psOSPageArrayData->uiTotalNumOSPages);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	}
+
+	eError = PVRSRV_OK;
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv,
+                               PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	/* Just drops the refcount. */
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_OSPAGEARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+	*ppvZombiePages = NULL;
+#endif
+
+	if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND))
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (psOSPageArrayData->iNumOSPagesAllocated == 0)
+		{
+			*ppvZombiePages = NULL;
+			return PVRSRV_OK;
+		}
+
+		eError = _ExtractAllPages(psOSPageArrayData,
+		                          &psExtractedPagesPageArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractAllPages", e0);
+
+		if (psExtractedPagesPageArray)
+		{
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyOSMem(psExtractedPagesPageArray, NULL);
+			PVR_WARN_IF_ERROR(eError, "PMRZombifyOSMem");
+		}
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		/* Free Memory for deferred allocation */
+		eError = _FreeOSPages(psOSPageArrayData,
+							  NULL,
+							  0);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+#endif
+	}
+
+	return eError;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+#endif
+}
+
+static INLINE IMG_BOOL IsOffsetValid(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData,
+                                     IMG_UINT32 ui32Offset)
+{
+	return (ui32Offset >> psOSPageArrayData->uiLog2DevPageSize) <
+	    psOSPageArrayData->uiTotalNumOSPages;
+}
+
+/* Determine PA for specified offset into page array. */
+static IMG_DEV_PHYADDR GetOffsetPA(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData,
+                                   IMG_UINT32 ui32Offset)
+{
+	IMG_UINT32 ui32PageIndex = ui32Offset >> PAGE_SHIFT;
+	IMG_UINT32 ui32InPageOffset = ui32Offset - (ui32PageIndex << PAGE_SHIFT);
+	IMG_DEV_PHYADDR sPA;
+
+	PVR_ASSERT(ui32InPageOffset < (1U << PAGE_SHIFT));
+
+	sPA.uiAddr = page_to_phys(psOSPageArrayData->pagearray[ui32PageIndex]);
+	sPA.uiAddr += ui32InPageOffset;
+
+	return sPA;
+}
+
+/* N.B. It is assumed that PMRLockSysPhysAddressesOSMem() is called _before_ this function! */
+static PVRSRV_ERROR
+PMRSysPhysAddrOSMem(PMR_IMPL_PRIVDATA pvPriv,
+					IMG_UINT32 ui32Log2PageSize,
+					IMG_UINT32 ui32NumOfPages,
+					IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					IMG_UINT64 ui64IPAPolicyValue,
+					IMG_UINT64 ui64IPAClearMask,
+#endif
+					IMG_BOOL *pbValid,
+					IMG_DEV_PHYADDR *psDevPAddr)
+{
+	const PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+	IMG_UINT32 uiIdx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (psOSPageArrayData->uiLog2DevPageSize < ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "for incompatible contiguity %u!",
+		         __func__,
+		         ui32Log2PageSize));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
+	{
+		if (pbValid[uiIdx])
+		{
+			PVR_LOG_RETURN_IF_FALSE(IsOffsetValid(psOSPageArrayData, puiOffset[uiIdx]),
+			                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+			psDevPAddr[uiIdx] = GetOffsetPA(psOSPageArrayData, puiOffset[uiIdx]);
+
+#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+			/* this is just a precaution, normally this should be always
+			 * available */
+			if (psOSPageArrayData->ui64DmaMask)
+			{
+				if (psDevPAddr[uiIdx].uiAddr > psOSPageArrayData->ui64DmaMask)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: physical address"
+							" (%" IMG_UINT64_FMTSPECX ") out of allowable range"
+							" [0; %" IMG_UINT64_FMTSPECX "]", __func__,
+							psDevPAddr[uiIdx].uiAddr,
+							psOSPageArrayData->ui64DmaMask));
+					BUG();
+				}
+			}
+#endif
+#if defined(SUPPORT_STATIC_IPA)
+			/* Modify the physical address with the associated IPA values */
+			psDevPAddr[uiIdx].uiAddr &= ~ui64IPAClearMask;
+			psDevPAddr[uiIdx].uiAddr |= ui64IPAPolicyValue;
+#endif
+
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+typedef struct _PMR_OSPAGEARRAY_KERNMAP_DATA_ {
+	void *pvBase;
+	IMG_UINT32 ui32PageCount;
+	pgprot_t PageProps;
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	IMG_UINT32 ui32CpuCacheFlags;
+#endif
+} PMR_OSPAGEARRAY_KERNMAP_DATA;
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv,
+								 size_t uiOffset,
+								 size_t uiSize,
+								 void **ppvKernelAddressOut,
+								 IMG_HANDLE *phHandleOut,
+								 PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+	void *pvAddress;
+	pgprot_t prot = PAGE_KERNEL;
+	IMG_UINT32 ui32PageOffset=0;
+	size_t uiMapOffset=0;
+	IMG_UINT32 ui32PageCount = 0;
+	IMG_UINT32 uiLog2DevPageSize = psOSPageArrayData->uiLog2DevPageSize;
+	struct page **pagearray;
+	PMR_OSPAGEARRAY_KERNMAP_DATA *psData;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	OSWRLockAcquireRead(psOSPageArrayData->hPagesUnderMigrationRWLock);
+	if (psOSPageArrayData->iNumOSPagesUnderMigration != 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: Requested kernel mapping from PMR "
+		         "in process of page migration, retry.",
+		         __func__));
+		OSWRLockReleaseRead(psOSPageArrayData->hPagesUnderMigrationRWLock);
+		return PVRSRV_ERROR_RETRY;
+	}
+	OSWRLockReleaseRead(psOSPageArrayData->hPagesUnderMigrationRWLock);
+#endif
+
+	/*
+		Zero offset and size as a special meaning which means map in the
+		whole of the PMR, this is due to fact that the places that call
+		this callback might not have access to be able to determine the
+		physical size
+	*/
+	if ((uiOffset == 0) && (uiSize == 0))
+	{
+		ui32PageOffset = 0;
+		uiMapOffset = 0;
+		/* Page count = amount of OS pages */
+		ui32PageCount = psOSPageArrayData->iNumOSPagesAllocated;
+	}
+	else
+	{
+		size_t uiEndoffset;
+
+		ui32PageOffset = uiOffset >> uiLog2DevPageSize;
+		uiMapOffset = uiOffset - (ui32PageOffset << uiLog2DevPageSize);
+		uiEndoffset = uiOffset + uiSize - 1;
+		/* Add one as we want the count, not the offset */
+		/* Page count = amount of device pages (note uiLog2DevPageSize being used) */
+		ui32PageCount = (uiEndoffset >> uiLog2DevPageSize) + 1;
+		ui32PageCount -= ui32PageOffset;
+	}
+
+	switch (PVRSRV_CPU_CACHE_MODE(psOSPageArrayData->ui32CPUCacheFlags))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				prot = pgprot_noncached(prot);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				prot = pgprot_writecombine(prot);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+				break;
+
+		default:
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto e0;
+	}
+
+	pagearray = &psOSPageArrayData->pagearray[ui32PageOffset];
+
+	psData = OSAllocMem(sizeof(*psData));
+	if (psData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	pvAddress = pvr_vmap(pagearray, ui32PageCount, VM_MAP, prot);
+	if (pvAddress == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	*ppvKernelAddressOut = pvAddress + uiMapOffset;
+	psData->pvBase = pvAddress;
+	psData->ui32PageCount = ui32PageCount;
+	psData->PageProps = prot;
+	*phHandleOut = psData;
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	{
+		IMG_CPU_PHYADDR pvAddrPhy;
+		pvAddrPhy.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(*pagearray));
+		InsertMappingRecord(PMR_GetAnnotation(psOSPageArrayData->hPMR),
+							psOSPageArrayData->uiPid,
+							pvAddress,
+							pvAddrPhy,
+							psOSPageArrayData->ui32CPUCacheFlags,
+							uiMapOffset,
+							ui32PageCount);
+
+		psData->ui32CpuCacheFlags = psOSPageArrayData->ui32CPUCacheFlags;
+	}
+#endif
+
+	return PVRSRV_OK;
+	/*
+	  error exit paths follow
+	*/
+e1:
+	OSFreeMem(psData);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv,
+											 IMG_HANDLE hHandle)
+{
+	PMR_OSPAGEARRAY_KERNMAP_DATA *psData = hHandle;
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	{
+		IMG_CPU_PHYADDR pvAddrPhy;
+		pvAddrPhy.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(vmalloc_to_page(psData->pvBase)));
+		InsertUnMappingRecord(psData->pvBase,
+				pvAddrPhy,
+				psData->ui32CpuCacheFlags,
+				psData->ui32PageCount);
+	}
+#endif
+
+	pvr_vunmap(psData->pvBase, psData->ui32PageCount, psData->PageProps);
+	OSFreeMem(psData);
+}
+
+/*************************************************************************/ /*!
+@Function       PMRChangeSparseMemOSMem
+@Description    This function Changes the sparse mapping by allocating and
+                freeing of pages. It changes the GPU and CPU maps accordingly.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PMRChangeSparseMemOSMem(PMR_IMPL_PRIVDATA pPriv,
+						const PMR *psPMR,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 *pai32AllocIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+						PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+						IMG_UINT32 uiFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR);
+	PMR_OSPAGEARRAY_DATA *psPMRPageArrayData = (PMR_OSPAGEARRAY_DATA *)pPriv;
+	struct page **psPageArray = psPMRPageArrayData->pagearray;
+	void **psDMAVirtArray = psPMRPageArrayData->dmavirtarray;
+	dma_addr_t *psDMAPhysArray = psPMRPageArrayData->dmaphysarray;
+	IMG_UINT32 ui32AdtnlAllocPages = 0; /*<! Number of pages to alloc from the OS */
+	IMG_UINT32 ui32AdtnlFreePages = 0; /*<! Number of pages to free back to the OS */
+	IMG_UINT32 ui32CommonRequestCount = 0; /*<! Number of pages to move position in the page array */
+	IMG_UINT32 ui32Loop = 0;
+	IMG_UINT32 ui32Index = 0;
+	IMG_UINT32 uiAllocpgidx;
+	IMG_UINT32 uiFreepgidx;
+	IMG_UINT32 uiOrder = psPMRPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bCMA = BIT_ISSET(psPMRPageArrayData->ui32AllocFlags, FLAG_DMA_CMA);
+
+
+	/* Check SPARSE flags and calculate pages to allocate and free */
+	if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
+	{
+		ui32CommonRequestCount = (ui32AllocPageCount > ui32FreePageCount) ?
+				ui32FreePageCount : ui32AllocPageCount;
+
+		PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free pages not supported");
+	}
+
+	if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
+	{
+		ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequestCount;
+	}
+	else
+	{
+		ui32AllocPageCount = 0;
+	}
+
+	if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
+	{
+		ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequestCount;
+	}
+	else
+	{
+		ui32FreePageCount = 0;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+
+	if (0 == (ui32CommonRequestCount || ui32AdtnlAllocPages || ui32AdtnlFreePages))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Missing parameters for number of pages to alloc/free",
+		         __func__));
+		return eError;
+	}
+
+	/* The incoming request is classified into two operations independent of
+	 * each other: alloc & free pages.
+	 * These operations can be combined with two mapping operations as well
+	 * which are GPU & CPU space mappings.
+	 *
+	 * From the alloc and free page requests, the net amount of pages to be
+	 * allocated or freed is computed. Pages that were requested to be freed
+	 * will be reused to fulfil alloc requests.
+	 *
+	 * The order of operations is:
+	 * 1. Allocate new pages from the OS
+	 * 2. Move the free pages from free request to alloc positions.
+	 * 3. Free the rest of the pages not used for alloc
+	 *
+	 * Alloc parameters are validated at the time of allocation
+	 * and any error will be handled then. */
+
+	/* Validate the free indices */
+	if (ui32FreePageCount)
+	{
+		if (NULL != pai32FreeIndices){
+
+			for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
+			{
+				uiFreepgidx = pai32FreeIndices[ui32Loop];
+
+				if (uiFreepgidx >= (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder))
+				{
+					eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+					goto e0;
+				}
+
+				if (NULL == (psPageArray[uiFreepgidx << uiOrder]))
+				{
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					PVR_DPF((PVR_DBG_ERROR,
+					         "%s: Trying to free non-allocated page",
+					         __func__));
+					goto e0;
+				}
+			}
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Given non-zero free count but missing indices array",
+			         __func__));
+			return eError;
+		}
+	}
+
+	/* Validate the alloc indices */
+	for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
+	{
+		uiAllocpgidx = pai32AllocIndices[ui32Loop];
+
+		if (uiAllocpgidx >= (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder))
+		{
+			eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+			goto e0;
+		}
+
+		if ((NULL != psPageArray[uiAllocpgidx << uiOrder]) ||
+		    (TRANSLATION_INVALID != psPMRMapTable->aui32Translation[uiAllocpgidx]))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to allocate already allocated page again",
+			         __func__));
+			goto e0;
+		}
+	}
+
+	ui32Loop = 0;
+
+	/* Allocate new pages from the OS */
+	if (0 != ui32AdtnlAllocPages)
+	{
+		eError = _AllocOSPages(psPMRPageArrayData, pai32AllocIndices, ui32AdtnlAllocPages);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "%s: New Addtl Allocation of pages failed",
+					 __func__));
+			goto e0;
+		}
+
+		psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages;
+		/*Mark the corresponding pages of translation table as valid */
+		for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
+		{
+			psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
+		}
+	}
+
+
+	ui32Index = ui32Loop;
+
+	/* Move the corresponding free pages to alloc request */
+	for (ui32Loop = 0; ui32Loop < ui32CommonRequestCount; ui32Loop++, ui32Index++)
+	{
+		IMG_UINT32 uiOrderIdx;
+		uiAllocpgidx = pai32AllocIndices[ui32Index] << uiOrder;
+		uiFreepgidx  = pai32FreeIndices[ui32Loop] << uiOrder;
+
+		for (uiOrderIdx = 0; uiOrderIdx < (1 << uiOrder); uiOrderIdx++)
+		{
+			psPageArray[uiAllocpgidx + uiOrderIdx] = psPageArray[uiFreepgidx + uiOrderIdx];
+			if (bCMA)
+			{
+				psDMAVirtArray[uiAllocpgidx + uiOrderIdx] = psDMAVirtArray[uiFreepgidx + uiOrderIdx];
+				psDMAPhysArray[uiAllocpgidx + uiOrderIdx] = psDMAPhysArray[uiFreepgidx + uiOrderIdx];
+				psDMAVirtArray[uiFreepgidx + uiOrderIdx] = NULL;
+				psDMAPhysArray[uiFreepgidx + uiOrderIdx] = (dma_addr_t)0;
+			}
+			psPageArray[uiFreepgidx + uiOrderIdx] = NULL;
+		}
+
+		psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
+		psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
+	}
+
+	/* Free or zombie the additional free pages */
+	if (0 != ui32AdtnlFreePages)
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		PMR_OSPAGEARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+		eError = _ExtractPages(psPMRPageArrayData,
+		                       &pai32FreeIndices[ui32Loop],
+		                       ui32AdtnlFreePages,
+		                       &psExtractedPagesPageArray);
+		if (eError != PVRSRV_OK)
+		{
+			goto e0;
+		}
+
+		/* Zombify pages to get proper stats */
+		eError = PMRZombifyOSMem(psExtractedPagesPageArray, NULL);
+		PVR_LOG_IF_ERROR(eError, "psExtractedPagesPageArray");
+
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		eError = _FreeOSPages(psPMRPageArrayData,
+		                      &pai32FreeIndices[ui32Loop],
+		                      ui32AdtnlFreePages);
+		if (eError != PVRSRV_OK)
+		{
+			goto e0;
+		}
+#endif /* SUPPORT_PMR_PAGES_DEFERRED_FREE */
+		psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages;
+		while (ui32Loop < ui32FreePageCount)
+		{
+			psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Loop]] = TRANSLATION_INVALID;
+			ui32Loop++;
+		}
+	}
+
+	eError = PVRSRV_OK;
+
+e0:
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMROSPFuncTab = {
+	.pfnLockPhysAddresses = &PMRLockSysPhysAddressesOSMem,
+	.pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesOSMem,
+	.pfnDevPhysAddr = &PMRSysPhysAddrOSMem,
+	.pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataOSMem,
+	.pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataOSMem,
+	.pfnReadBytes = NULL,
+	.pfnWriteBytes = NULL,
+	.pfnChangeSparseMem = &PMRChangeSparseMemOSMem,
+	.pfnFinalize = &PMRFinalizeOSMem,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	.pfnFreeZombiePages = &PMRFreeZombiePagesOSMem,
+#endif
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = &PMRZombifyOSMem,
+#endif
+};
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+DLLIST_NODE *
+LinuxOSGetCPUMappingPrivateDataList(PMR *psPMR)
+{
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = PMRGetPrivateData(psPMR, &_sPMROSPFuncTab);
+	PVR_ASSERT(psOSPageArrayData);
+
+	return &psOSPageArrayData->sCpuMappingListHead;
+}
+#endif
+
+/* Wrapper around OS page allocation. */
+static PVRSRV_ERROR
+DoPageAlloc(PMR_OSPAGEARRAY_DATA *psPrivData,
+            IMG_UINT32 *puiAllocIndices,
+            IMG_UINT32 ui32NumPhysChunks,
+            IMG_UINT32 ui32NumLogicalChunks,
+            IMG_UINT32 ui32Log2AllocPageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Do we fill the whole page array or just parts (sparse)? */
+	if (ui32NumPhysChunks == ui32NumLogicalChunks)
+	{
+		/* Allocate the physical pages */
+		eError = _AllocOSPages(psPrivData,
+		                       NULL,
+		                       psPrivData->uiTotalNumOSPages >>
+		                       (ui32Log2AllocPageSize - PAGE_SHIFT));
+	}
+	else if (ui32NumPhysChunks != 0)
+	{
+		/* Allocate the physical pages */
+		eError = _AllocOSPages(psPrivData, puiAllocIndices,
+		                       ui32NumPhysChunks);
+	}
+
+	return eError;
+}
+
+static void _EncodeAllocationFlags(IMG_UINT32 uiLog2DevPageSize,
+	                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+	                               IMG_UINT32* ui32AllocFlags)
+{
+
+	if (PVRSRV_CHECK_OS_LINUX_PREFER_CMA(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_PREF_CMA);
+	}
+	/*
+	 * Use CMA framework if order is greater than OS page size; please note
+	 * that OSMMapPMRGeneric() has the same expectation as well.
+	 */
+	else if (uiLog2DevPageSize > PAGE_SHIFT)
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_DMA_CMA);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_IS_MOVABLE);
+	}
+#endif
+
+	/* OnDemand? */
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_ONDEMAND);
+	}
+
+	/* Zero? */
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_ZERO);
+	}
+
+	/* Poison on alloc? */
+	if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_ALLOC);
+	}
+
+#if defined(DEBUG)
+	/* Poison on free? */
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_FREE);
+	}
+#endif
+
+	/* Indicate whether this is an allocation with default caching attribute (i.e cached) or not */
+	if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) ||
+		PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE);
+	}
+
+}
+
+void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                             IMG_UINT64 *pui64TotalSize,
+                             IMG_UINT64 *pui64FreeSize)
+{
+	struct sysinfo sMeminfo;
+	si_meminfo(&sMeminfo);
+
+	PVR_UNREFERENCED_PARAMETER(pvImplData);
+
+	*pui64TotalSize = sMeminfo.totalram * sMeminfo.mem_unit;
+	*pui64FreeSize = sMeminfo.freeram * sMeminfo.mem_unit;
+
+}
+
+PVRSRV_ERROR
+PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap,
+						 CONNECTION_DATA *psConnection,
+						 IMG_DEVMEM_SIZE_T uiSize,
+						 IMG_UINT32 ui32NumPhysChunks,
+						 IMG_UINT32 ui32NumLogicalChunks,
+						 IMG_UINT32 *puiAllocIndices,
+						 IMG_UINT32 uiLog2DevPageSize,
+						 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						 const IMG_CHAR *pszAnnotation,
+						 IMG_PID uiPid,
+						 PMR **ppsPMRPtr,
+						 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	PMR *psPMR;
+	struct _PMR_OSPAGEARRAY_DATA_ *psPrivData;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_UINT32 ui32CPUCacheFlags;
+	IMG_UINT32 ui32AllocFlags = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	IMG_UINT32 ui32ValidPageSizeMask = psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask;
+	IMG_UINT32 ui32PageSize = IMG_PAGE2BYTES32(uiLog2DevPageSize);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32PageSize & ui32ValidPageSizeMask, eError, errorOnParam);
+
+	/*
+	 * The host driver (but not guest) can still use this factory for firmware
+	 * allocations
+	 */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode) && PVRSRV_CHECK_FW_MAIN(uiFlags))
+	{
+		PVR_ASSERT(0);
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errorOnParam;
+	}
+
+	/* Select correct caching mode */
+	eError = DevmemCPUCacheMode(uiFlags, &ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnParam;
+	}
+
+	if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags))
+	{
+		ui32CPUCacheFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN;
+	}
+
+	_EncodeAllocationFlags(uiLog2DevPageSize, uiFlags, &ui32AllocFlags);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Reject sparse and non4k PMR marked for migrate */
+	if (BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE) &&
+	   ((ui32NumPhysChunks != ui32NumLogicalChunks|| ui32NumLogicalChunks > 1) ||
+	    uiLog2DevPageSize != PAGE_SHIFT)
+	   )
+	{
+		PVR_LOG_GOTO_WITH_ERROR("PhysmemNewOSRamBackedPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, errorOnParam);
+	}
+#endif
+
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	/* Overwrite flags and always zero pages that could go back to UM */
+	BIT_SET(ui32AllocFlags, FLAG_ZERO);
+	BIT_UNSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC);
+#endif
+
+	/* Physical allocation alignment is generally not supported except under
+	   very restrictive conditions, also there is a maximum alignment value
+	   which must not exceed the largest device page-size. If these are not
+	   met then fail the aligned-requested allocation */
+	if (uiLog2DevPageSize > PAGE_SHIFT)
+	{
+		IMG_UINT64 ui64InvalidSizeMask = ui32PageSize - 1;
+		IMG_UINT32 ui32PageSizeMask = psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask;
+		IMG_UINT32 ui32MaxPageSize = 1;
+
+		while (ui32PageSizeMask > 1)
+		{
+			ui32PageSizeMask >>= 1;
+			ui32MaxPageSize <<= 1;
+		}
+
+		if ((uiSize & ui64InvalidSizeMask) || (ui32PageSize > ui32MaxPageSize))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid PA alignment: size 0x%llx, align 0x%x",
+					__func__, uiSize, ui32PageSize));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+			goto errorOnParam;
+		}
+	}
+
+	/* Create Array structure that hold the physical pages */
+	eError = _AllocOSPageArray(psDevNode,
+							   uiSize,
+							   uiLog2DevPageSize,
+							   ui32AllocFlags,
+							   ui32CPUCacheFlags,
+							   uiPid,
+							   &psPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnAllocPageArray;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	if (psConnection)
+	{
+		psPrivData->psDRMFile = ((ENV_CONNECTION_DATA*)psConnection->hOsPrivateData)->psDRMFile;
+	}
+	else
+	{
+		psPrivData->psDRMFile = NULL;
+	}
+#endif
+#endif
+
+	if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		eError = DoPageAlloc(psPrivData, puiAllocIndices, ui32NumPhysChunks,
+		                     ui32NumLogicalChunks, uiLog2DevPageSize);
+		if (eError != PVRSRV_OK)
+		{
+			goto errorOnAllocPages;
+		}
+	}
+
+	/*
+	 * In this instance, we simply pass flags straight through.
+	 *
+	 * Generically, uiFlags can include things that control the PMR factory, but
+	 * we don't need any such thing (at the time of writing!), and our caller
+	 * specifies all PMR flags so we don't need to meddle with what was given to
+	 * us.
+	 */
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+
+	/*
+	 * Check no significant bits were lost in cast due to different bit widths
+	 * for flags
+	 */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (UMA)");
+	}
+
+	eError = PMRCreatePMR(psPhysHeap,
+						  uiSize,
+						  ui32NumPhysChunks,
+						  ui32NumLogicalChunks,
+						  puiAllocIndices,
+						  uiLog2DevPageSize,
+						  uiPMRFlags,
+						  pszAnnotation,
+						  &_sPMROSPFuncTab,
+						  psPrivData,
+						  PMR_TYPE_OSMEM,
+						  &psPMR,
+						  ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnCreate;
+	}
+
+	psPrivData->hPMR = psPMR;
+	*ppsPMRPtr = psPMR;
+
+	return PVRSRV_OK;
+
+errorOnCreate:
+	if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		eError2 = _FreeOSPages(psPrivData, NULL, 0);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+errorOnAllocPages:
+	_FreeOSPagesArray(psPrivData);
+
+errorOnAllocPageArray:
+errorOnParam:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.h
new file mode 100644
index 000000000000..2581cceeb219
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux OS physmem implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_OSMEM_LINUX_H
+#define PHYSMEM_OSMEM_LINUX_H
+
+PVRSRV_ERROR LinuxInitPhysmem(void);
+void LinuxDeinitPhysmem(void);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+DLLIST_NODE *
+LinuxOSGetCPUMappingPrivateDataList(PMR *psPMR);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+int LinuxRegisterMigrateCallbacks(struct file* filp);
+void LinuxDeregisterMigrateCallbacks(struct file* filp);
+#endif
+#endif
+
+#endif /* PHYSMEM_OSMEM_LINUX_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.c
new file mode 100644
index 000000000000..00c02f16d3e2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.c
@@ -0,0 +1,992 @@
+/*************************************************************************/ /*!
+@Title          Physmem_test
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Single entry point for testing of page factories
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "physmem_test.h"
+#include "device.h"
+#include "syscommon.h"
+#include "pmr.h"
+#include "osfunc.h"
+#include "physmem.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+#include "pvrsrv.h"
+
+#define PHYSMEM_TEST_PAGES        2     /* Mem test pages */
+#define PHYSMEM_TEST_PASSES_MAX   1000  /* Limit number of passes to some reasonable value */
+
+
+/* Test patterns for mem test */
+
+static const IMG_UINT64 gui64Patterns[] = {
+	0,
+	0xffffffffffffffffULL,
+	0x5555555555555555ULL,
+	0xaaaaaaaaaaaaaaaaULL,
+	0x1111111111111111ULL,
+	0x2222222222222222ULL,
+	0x4444444444444444ULL,
+	0x8888888888888888ULL,
+	0x3333333333333333ULL,
+	0x6666666666666666ULL,
+	0x9999999999999999ULL,
+	0xccccccccccccccccULL,
+	0x7777777777777777ULL,
+	0xbbbbbbbbbbbbbbbbULL,
+	0xddddddddddddddddULL,
+	0xeeeeeeeeeeeeeeeeULL,
+	0x7a6c7258554e494cULL,
+};
+
+static const IMG_UINT32 gui32Patterns[] = {
+	0,
+	0xffffffffU,
+	0x55555555U,
+	0xaaaaaaaaU,
+	0x11111111U,
+	0x22222222U,
+	0x44444444U,
+	0x88888888U,
+	0x33333333U,
+	0x66666666U,
+	0x99999999U,
+	0xccccccccU,
+	0x77777777U,
+	0xbbbbbbbbU,
+	0xddddddddU,
+	0xeeeeeeeeU,
+	0x7a6c725cU,
+};
+
+static const IMG_UINT16 gui16Patterns[] = {
+	0,
+	0xffffU,
+	0x5555U,
+	0xaaaaU,
+	0x1111U,
+	0x2222U,
+	0x4444U,
+	0x8888U,
+	0x3333U,
+	0x6666U,
+	0x9999U,
+	0xccccU,
+	0x7777U,
+	0xbbbbU,
+	0xddddU,
+	0xeeeeU,
+	0x7a6cU,
+};
+
+static const IMG_UINT8 gui8Patterns[] = {
+	0,
+	0xffU,
+	0x55U,
+	0xaaU,
+	0x11U,
+	0x22U,
+	0x44U,
+	0x88U,
+	0x33U,
+	0x66U,
+	0x99U,
+	0xccU,
+	0x77U,
+	0xbbU,
+	0xddU,
+	0xeeU,
+	0x6cU,
+};
+
+static PVRSRV_ERROR
+PMRContiguousSparseMappingTest(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError, eError1;
+	PHYS_HEAP *psHeap;
+	PHYS_HEAP_POLICY psHeapPolicy;
+
+	PMR *psPMR = NULL;
+	PMR *psSpacingPMR = NULL, *psSecondSpacingPMR = NULL;
+	IMG_UINT32 aui32MappingTableFirstAlloc[4] = {0,1,2,3};
+	IMG_UINT32 aui32MappingTableSecondAlloc[8] = {4,5,6,7,8,9,10,11};
+	IMG_UINT32 aui32MappingTableThirdAlloc[4] = {12,13,14,15};
+	IMG_UINT32 ui32NoMappingTable = 0;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_BOOL *pbValid;
+	IMG_DEV_PHYADDR *apsDevPAddr;
+	IMG_UINT32 ui32NumOfPages = 16;
+	size_t uiMappedSize, uiPageSize;
+	IMG_UINT32 i, uiAttempts;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	eError = PhysHeapAcquireByID(PVRSRV_GET_PHYS_HEAP_HINT(uiFlags),
+	                             psDeviceNode,
+	                             &psHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorReturn);
+
+	psHeapPolicy = PhysHeapGetPolicy(psHeap);
+
+	PhysHeapRelease(psHeap);
+
+	/* If this is the case then it's not supported and so don't attempt the test */
+	if (psHeapPolicy != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		return PVRSRV_OK;
+	}
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate OS memory for PMR page list */
+	apsDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+	PVR_LOG_RETURN_IF_NOMEM(apsDevPAddr, "OSAllocMem");
+
+	/* Allocate OS memory for PMR page state */
+	pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrorFreePMRPageListMem);
+
+	/* Allocate OS memory for write buffer */
+	pcWriteBuffer = OSAllocMem(uiPageSize * ui32NumOfPages);
+	PVR_LOG_GOTO_IF_NOMEM(pcWriteBuffer, eError, ErrorFreePMRPageStateMem);
+	OSCachedMemSet(pcWriteBuffer, 0xF, uiPageSize);
+
+	/* Allocate OS memory for read buffer */
+	pcReadBuffer = OSAllocMem(uiPageSize * ui32NumOfPages);
+	PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+	/* Allocate Sparse PMR with SPARSE | READ | WRITE | UNCACHED_WC attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/*
+	 * Construct a sparse PMR attempting to ensure the allocations
+	 * are physically non contiguous but sequentially placed in the mapping
+	 * table.
+	 */
+	for (uiAttempts = 3; uiAttempts > 0; uiAttempts--)
+	{
+		/* Allocate a sparse PMR from given physical heap - CPU/GPU/FW */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										4,
+										ui32NumOfPages,
+										aui32MappingTableFirstAlloc,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorFreeReadBuffer;
+		}
+
+		/* Allocate some memory from the same physheap so that we can ensure
+		 * the allocations aren't linear
+		 */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										1,
+										1,
+										&ui32NoMappingTable,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psSpacingPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorUnrefPMR;
+		}
+
+		/* Allocate 8 more physical pages on the Sparse PMR */
+		eError = PMR_ChangeSparseMem(psPMR,
+									 8,
+									 aui32MappingTableSecondAlloc,
+									 0,
+									 NULL,
+									 uiFlags | SPARSE_RESIZE_ALLOC);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_ChangeSparseMem", ErrorUnrefSpacingPMR);
+
+		/* Allocate some more memory from the same physheap so that we can ensure
+		 * the allocations aren't linear
+		 */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										1,
+										1,
+										&ui32NoMappingTable,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psSecondSpacingPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorUnrefSpacingPMR;
+		}
+
+		/* Allocate final 4 physical pages on the Sparse PMR */
+		eError = PMR_ChangeSparseMem(psPMR,
+									 4,
+									 aui32MappingTableThirdAlloc,
+									 0,
+									 NULL,
+									 uiFlags | SPARSE_RESIZE_ALLOC);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_ChangeSparseMem", ErrorUnrefSecondSpacingPMR);
+
+		/*
+		 * Check we have in fact managed to obtain a PMR with non contiguous
+		 * physical pages.
+		 */
+		eError = PMRLockSysPhysAddresses(psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to lock PMR"));
+			goto ErrorUnrefSecondSpacingPMR;
+		}
+
+		/* Get the Device physical addresses of the pages */
+		eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), ui32NumOfPages, 0, apsDevPAddr, pbValid, CPU_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses"));
+			goto ErrorUnlockPhysAddresses;
+		}
+
+		{
+			IMG_BOOL bPhysicallyContiguous = IMG_TRUE;
+			IMG_DEV_PHYADDR sPrevDevPAddr = apsDevPAddr[0];
+			for (i = 1; i < ui32NumOfPages && bPhysicallyContiguous; i++)
+			{
+				if (apsDevPAddr[i].uiAddr != sPrevDevPAddr.uiAddr + uiPageSize)
+				{
+					bPhysicallyContiguous = IMG_FALSE;
+				}
+				sPrevDevPAddr = apsDevPAddr[i];
+			}
+
+			if (bPhysicallyContiguous)
+			{
+				/* We haven't yet managed to create the mapping scenario we
+				 * require: unwind and attempt again.
+				 */
+				eError1 = PMRUnlockSysPhysAddresses(psPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+				}
+				eError1 = PMRUnrefPMR(psPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+				}
+				eError1 = PMRUnrefPMR(psSpacingPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free Spacing PMR"));
+				}
+				eError1 = PMRUnrefPMR(psSecondSpacingPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free Second Spacing PMR"));
+				}
+			} else {
+				/* We have the scenario, break out of the attempt loop */
+				break;
+			}
+		}
+	}
+
+	if (uiAttempts == 0)
+	{
+		/* We can't create the scenario, very unlikely this would happen */
+		PVR_LOG_GOTO_IF_ERROR(PVRSRV_ERROR_MEMORY_TEST_FAILED,
+		                      "Unable to create Non Contiguous PMR scenario",
+		                      ErrorFreeReadBuffer);
+	}
+
+	/* We have the PMR scenario to test, now attempt to map the whole PMR,
+	 * write and then read from it
+	 */
+	eError = PMRAcquireSparseKernelMappingData(psPMR, 0, ui32NumOfPages * uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	OSCachedMemCopyWMB(pvKernAddr, pcWriteBuffer, ui32NumOfPages * uiPageSize);
+
+	eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+	PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+	/*
+	 * Release and reacquire the mapping to exercise the mapping paths
+	 */
+	eError = PMRAcquireSparseKernelMappingData(psPMR, 0, ui32NumOfPages * uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	OSCachedMemSetWMB(pcReadBuffer, 0x0, ui32NumOfPages * uiPageSize);
+	OSCachedMemCopyWMB(pcReadBuffer, pvKernAddr, ui32NumOfPages * uiPageSize);
+
+	eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+	PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+	for (i = 0; i < ui32NumOfPages * uiPageSize; i++)
+	{
+		if (pcReadBuffer[i] != pcWriteBuffer[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)! @ %u",
+			         __func__, pcReadBuffer[i], pcWriteBuffer[i], i));
+			eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+			goto ErrorUnlockPhysAddresses;
+		}
+	}
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it */
+	eError1 = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+	}
+
+ErrorUnrefSecondSpacingPMR:
+	eError1 = PMRUnrefPMR(psSecondSpacingPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free Second Spacing PMR"));
+	}
+ErrorUnrefSpacingPMR:
+	eError1 = PMRUnrefPMR(psSpacingPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free Spacing PMR"));
+	}
+ErrorUnrefPMR:
+	eError1 = PMRUnrefPMR(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+	}
+
+ErrorFreeReadBuffer:
+	OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+	OSFreeMem(pcWriteBuffer);
+ErrorFreePMRPageStateMem:
+	OSFreeMem(pbValid);
+ErrorFreePMRPageListMem:
+	OSFreeMem(apsDevPAddr);
+ErrorReturn:
+	return eError;
+}
+
+/* Test for PMR factory validation */
+static PVRSRV_ERROR
+PMRValidationTest(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError, eError1;
+	IMG_UINT32 i = 0, j = 0, ui32Index = 0;
+	IMG_UINT32 *pui32MappingTable = NULL;
+	PMR *psPMR = NULL;
+	IMG_BOOL *pbValid;
+	IMG_DEV_PHYADDR *apsDevPAddr;
+	IMG_UINT32 ui32NumOfPages = 10, ui32NumOfPhysPages = 5;
+	size_t uiMappedSize, uiPageSize;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate OS memory for PMR page list */
+	apsDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+	PVR_LOG_RETURN_IF_NOMEM(apsDevPAddr, "OSAllocMem");
+
+	/* Allocate OS memory for PMR page state */
+	pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrorFreePMRPageListMem);
+
+	/* Allocate OS memory for write buffer */
+	pcWriteBuffer = OSAllocMem(uiPageSize);
+	PVR_LOG_GOTO_IF_NOMEM(pcWriteBuffer, eError, ErrorFreePMRPageStateMem);
+	OSCachedMemSet(pcWriteBuffer, 0xF, uiPageSize);
+
+	/* Allocate OS memory for read buffer */
+	pcReadBuffer = OSAllocMem(uiPageSize);
+	PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+	/* Allocate OS memory for mapping table */
+	pui32MappingTable = (IMG_UINT32 *)OSAllocMem(ui32NumOfPhysPages * sizeof(*pui32MappingTable));
+	PVR_LOG_GOTO_IF_NOMEM(pui32MappingTable, eError, ErrorFreeReadBuffer);
+
+	/* Pages having even index will have physical backing in PMR */
+	for (ui32Index=0; ui32Index < ui32NumOfPages; ui32Index+=2)
+	{
+		pui32MappingTable[i++] = ui32Index;
+	}
+
+	/* Allocate Sparse PMR with SPARSE | READ | WRITE | UNCACHED_WC attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/* Allocate a sparse PMR from given physical heap - CPU/GPU/FW */
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									ui32NumOfPages * uiPageSize,
+									ui32NumOfPhysPages,
+									ui32NumOfPages,
+									pui32MappingTable,
+									OSGetPageShift(),
+									uiFlags,
+									sizeof("PMR ValidationTest"),
+									"PMR ValidationTest",
+									OSGetCurrentClientProcessIDKM(),
+									&psPMR,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+		goto ErrorFreeMappingTable;
+	}
+
+	/* Check whether allocated PMR can be locked and obtain physical addresses
+	 * of underlying memory pages.
+	 */
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to lock PMR"));
+		goto ErrorUnrefPMR;
+	}
+
+	/* Get the Device physical addresses of the pages */
+	eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), ui32NumOfPages, 0, apsDevPAddr, pbValid, CPU_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	/* Check whether device address of each physical page is OS PAGE_SIZE aligned */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			if ((apsDevPAddr[i].uiAddr & OSGetPageMask()) != 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Physical memory of PMR is not page aligned"));
+				eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+				goto ErrorUnlockPhysAddresses;
+			}
+		}
+	}
+
+	/* Acquire kernel virtual address of each physical page and write to it
+	 * and then release it.
+	 */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMR, (i * uiPageSize), uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+				goto ErrorUnlockPhysAddresses;
+			}
+			OSCachedMemCopyWMB(pvKernAddr, pcWriteBuffer, OSGetPageSize());
+
+			eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+			PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+		}
+	}
+
+	/* Acquire kernel virtual address of each physical page and read
+	 * from it and check where contents are intact.
+	 */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMR, (i * uiPageSize), uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+				goto ErrorUnlockPhysAddresses;
+			}
+			OSCachedMemSetWMB(pcReadBuffer, 0x0, uiPageSize);
+			OSCachedMemCopyWMB(pcReadBuffer, pvKernAddr, uiMappedSize);
+
+			eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+			PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+			for (j = 0; j < uiPageSize; j++)
+			{
+				if (pcReadBuffer[j] != pcWriteBuffer[j])
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+					         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!",
+					         __func__, pcReadBuffer[j], pcWriteBuffer[j]));
+					eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+					goto ErrorUnlockPhysAddresses;
+				}
+			}
+		}
+	}
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it */
+	eError1 = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+	}
+
+ErrorUnrefPMR:
+	eError1 = PMRUnrefPMR(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+	}
+ErrorFreeMappingTable:
+	OSFreeMem(pui32MappingTable);
+ErrorFreeReadBuffer:
+	OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+	OSFreeMem(pcWriteBuffer);
+ErrorFreePMRPageStateMem:
+	OSFreeMem(pbValid);
+ErrorFreePMRPageListMem:
+	OSFreeMem(apsDevPAddr);
+
+	return eError;
+}
+
+#define DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, Patterns, NumOfPatterns, Error, ptr, i) \
+	for (i = 0; i < NumOfPatterns; i++) \
+	{ \
+		/* Write pattern */ \
+		for (ptr = StartAddr; ptr < EndAddr; ptr++) \
+		{ \
+			*ptr = Patterns[i]; \
+		} \
+		\
+		/* Read back and validate pattern */ \
+		for (ptr = StartAddr; ptr < EndAddr ; ptr++) \
+		{ \
+			if (*ptr != Patterns[i]) \
+			{ \
+				Error = PVRSRV_ERROR_MEMORY_TEST_FAILED; \
+				break; \
+			} \
+		} \
+		\
+		if (Error != PVRSRV_OK) \
+		{ \
+			break; \
+		} \
+	}
+
+static PVRSRV_ERROR
+TestPatternU8(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT8 *StartAddr = (IMG_UINT8 *) pvKernAddr;
+	IMG_UINT8 *EndAddr = ((IMG_UINT8 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT8));
+	IMG_UINT8 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT8)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui8Patterns, sizeof(gui8Patterns)/sizeof(IMG_UINT8), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!",
+		         __func__, *p, gui8Patterns[i]));
+	}
+
+	return eError;
+}
+
+
+static PVRSRV_ERROR
+TestPatternU16(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT16 *StartAddr = (IMG_UINT16 *) pvKernAddr;
+	IMG_UINT16 *EndAddr = ((IMG_UINT16 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT16));
+	IMG_UINT16 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT16)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui16Patterns, sizeof(gui16Patterns)/sizeof(IMG_UINT16), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%hx), expected (0x%hx)!",
+		         __func__, *p, gui16Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestPatternU32(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT32 *StartAddr = (IMG_UINT32 *) pvKernAddr;
+	IMG_UINT32 *EndAddr = ((IMG_UINT32 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT32));
+	IMG_UINT32 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT32)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui32Patterns, sizeof(gui32Patterns)/sizeof(IMG_UINT32), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%x), expected (0x%x)!",
+		         __func__, *p, gui32Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestPatternU64(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT64 *StartAddr = (IMG_UINT64 *) pvKernAddr;
+	IMG_UINT64 *EndAddr = ((IMG_UINT64 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT64));
+	IMG_UINT64 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT64)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui64Patterns, sizeof(gui64Patterns)/sizeof(IMG_UINT64), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%llx), expected (0x%llx)!",
+		         __func__, *p, gui64Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestSplitCacheline(void *pvKernAddr, size_t uiMappedSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	size_t uiCacheLineSize;
+	size_t uiBlockSize;
+	size_t j;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_UINT8 *StartAddr = (IMG_UINT8 *) pvKernAddr;
+	IMG_UINT8 *EndAddr, *p;
+
+	uiCacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+
+	if (uiCacheLineSize > 0)
+	{
+		uiBlockSize = (uiCacheLineSize * 2)/3; /* split cacheline */
+
+		pcWriteBuffer = OSAllocMem(uiBlockSize);
+		PVR_LOG_RETURN_IF_NOMEM(pcWriteBuffer, "OSAllocMem");
+
+		/* Fill the write buffer with test data, 0xAB*/
+		OSCachedMemSet(pcWriteBuffer, 0xAB, uiBlockSize);
+
+		pcReadBuffer = OSAllocMem(uiBlockSize);
+		PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+		/* Fit only complete blocks in uiMappedSize, ignore leftover bytes */
+		EndAddr = StartAddr + (uiBlockSize * (uiMappedSize / uiBlockSize));
+
+		/* Write blocks into the memory */
+		for (p = StartAddr; p < EndAddr; p += uiBlockSize)
+		{
+			OSCachedMemCopy(p, pcWriteBuffer, uiBlockSize);
+		}
+
+		/* Read back blocks and check */
+		for (p = StartAddr; p < EndAddr; p += uiBlockSize)
+		{
+			OSCachedMemCopy(pcReadBuffer, p, uiBlockSize);
+
+			for (j = 0; j < uiBlockSize; j++)
+			{
+				if (pcReadBuffer[j] != pcWriteBuffer[j])
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!", __func__, pcReadBuffer[j], pcWriteBuffer[j]));
+					eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+					goto ErrorMemTestFailed;
+				}
+			}
+		}
+
+ErrorMemTestFailed:
+		OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+		OSFreeMem(pcWriteBuffer);
+	}
+
+	return eError;
+}
+
+/* Memory test - writes and reads back different patterns to memory and validate the same */
+static PVRSRV_ERROR
+MemTestPatterns(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32MappingTable = 0;
+	PMR *psPMR = NULL;
+	size_t uiMappedSize, uiPageSize;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate PMR with READ | WRITE | WRITE_COMBINE attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+			   PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+			   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/*Allocate a PMR from given physical heap */
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiPageSize * PHYSMEM_TEST_PAGES,
+									1,
+									1,
+									&ui32MappingTable,
+									OSGetPageShift(),
+									uiFlags,
+									sizeof("PMR PhysMemTest"),
+									"PMR PhysMemTest",
+									OSGetCurrentClientProcessIDKM(),
+									&psPMR,
+									PDUMP_NONE,
+									NULL);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewRamBackedPMR");
+
+	/* Check whether allocated PMR can be locked and obtain physical
+	 * addresses of underlying memory pages.
+	 */
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", ErrorUnrefPMR);
+
+	/* Map the physical page(s) into kernel space, acquire kernel mapping
+	 * for PMR.
+	 */
+	eError = PMRAcquireKernelMappingData(psPMR, 0, uiPageSize * PHYSMEM_TEST_PAGES, &pvKernAddr, &uiMappedSize, &hPrivData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", ErrorUnlockPhysAddresses);
+
+	PVR_ASSERT((uiPageSize * PHYSMEM_TEST_PAGES) == uiMappedSize);
+
+	/* Test various patterns */
+	eError = TestPatternU64(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU32(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU16(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU8(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	/* Test split cachelines */
+	eError = TestSplitCacheline(pvKernAddr, uiMappedSize);
+
+ErrorReleaseKernelMappingData:
+	(void) PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it, ignore returned value */
+	(void) PMRUnlockSysPhysAddresses(psPMR);
+ErrorUnrefPMR:
+	(void) PMRUnrefPMR(psPMR);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+PhysMemTestRun(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags, IMG_UINT32 ui32Passes)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	/* PMR validation test */
+	eError = PMRValidationTest(psDeviceNode, uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR Contiguous PhysHeap self test failed! %"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         __func__,
+		         uiFlags));
+		return eError;
+	}
+
+	eError = PMRContiguousSparseMappingTest(psDeviceNode, uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR Non-contiguous PhysHeap self test failed! %"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         __func__,
+		         uiFlags));
+		return eError;
+	}
+
+
+	for (i = 0; i < ui32Passes; i++)
+	{
+		/* Mem test */
+		eError = MemTestPatterns(psDeviceNode, uiFlags);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: [Pass#%u] MemTestPatterns failed!",
+			         __func__, i));
+			break;
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysMemTest(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvDevConfig, IMG_UINT32 ui32MemTestPasses)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = pvDevConfig;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* validate memtest passes requested */
+	ui32MemTestPasses = (ui32MemTestPasses > PHYSMEM_TEST_PASSES_MAX)? PHYSMEM_TEST_PASSES_MAX : ui32MemTestPasses;
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		PHYS_HEAP_CONFIG *psHeapConfig = &psDevConfig->pasPhysHeaps[i];
+
+		if (psHeapConfig->ui32UsageFlags & PHYS_HEAP_USAGE_GPU_LOCAL)
+		{
+			/* GPU local mem (should be only up to 1 heap) */
+			eError = PhysMemTestRun(psDeviceNode, PHYS_HEAP_USAGE_GPU_LOCAL, ui32MemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "GPU local memory test failed!", ErrorPhysMemTestEnd);
+		}
+
+		if (psHeapConfig->ui32UsageFlags & PHYS_HEAP_USAGE_CPU_LOCAL)
+		{
+			/* CPU local mem (should be only up to 1 heap) */
+			eError = PhysMemTestRun(psDeviceNode, PHYS_HEAP_USAGE_CPU_LOCAL, ui32MemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "CPU local memory test failed!", ErrorPhysMemTestEnd);
+		}
+	}
+
+
+ErrorPhysMemTestEnd:
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PhysMemTest: Failed."));
+	}
+	else
+	{
+		PVR_LOG(("PhysMemTest: Passed."));
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.h
new file mode 100644
index 000000000000..75795afe9cab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@Title          Physmem test header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for single entry point for testing of page factories
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_TEST_H
+#define SRVSRV_PHYSMEM_TEST_H
+
+#include "device.h"
+
+/*
+ * PhysMemTest
+ */
+PVRSRV_ERROR
+PhysMemTest(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvDevConfig, IMG_UINT32 ui32MemTestPasses);
+
+#endif /* SRVSRV_PHYSMEM_TEST_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.c
new file mode 100644
index 000000000000..e0c74b4b508d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.c
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux environment PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/string.h>
+
+#include "img_defs.h"
+
+#include "pmr_env.h"
+#include "physmem_dmabuf_internal.h"
+
+void
+PMREnvInitialize(PMR_ENV *psPMREnv)
+{
+	memset(psPMREnv, 0, sizeof(*psPMREnv));
+}
+
+void
+PMREnvFinalize(PMR_ENV *psPMREnv)
+{
+	void *pvDmaBufExportData;
+
+	pvDmaBufExportData = smp_load_acquire(&psPMREnv->pvDmaBufExportData);
+	if (pvDmaBufExportData)
+		PhysmemDmaBufExportFinalize(pvDmaBufExportData);
+}
+
+void
+PMREnvDmaBufSetExportData(PMR *psPMR, void *pvData)
+{
+	PMR_ENV *psPMREnv = PMREnvGetData(psPMR);
+
+	smp_store_release(&psPMREnv->pvDmaBufExportData, pvData);
+}
+
+void *
+PMREnvDmaBufGetExportData(PMR *psPMR)
+{
+	PMR_ENV *psPMREnv = PMREnvGetData(psPMR);
+	void *pvDmaBufExportData;
+
+	pvDmaBufExportData = smp_load_acquire(&psPMREnv->pvDmaBufExportData);
+
+	return pvDmaBufExportData;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.h
new file mode 100644
index 000000000000..52a172e80b82
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          Environment PMR functions and data
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Environment specific PMR functions and data
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PMR_ENV_H)
+#define PMR_ENV_H
+
+#include "pvrsrv_memallocflags.h"
+#include "powervr/mem_types.h"
+#include "pmr_impl.h"
+
+/* Structures, functions and definitions shared with the generic PMR code */
+struct _PMR_ENV_
+{
+	/*
+	 * The smp_store_release and smp_load_acquire functions are used
+	 * to access this field, in order to avoid taking locks, where
+	 * possible.
+	 */
+	void *pvDmaBufExportData;
+};
+
+typedef struct _PMR_ENV_ PMR_ENV;
+
+#define	DECLARE_PMR_ENV_DATA(name) PMR_ENV name;
+
+/* The generic PMR code calls these functions, via the macros that follow */
+void PMREnvInitialize(PMR_ENV *psPMREnv);
+void PMREnvFinalize(PMR_ENV *psPMREnv);
+
+#define	PMR_ENV_INITIALIZE(psPMR, sEnvData) PMREnvInitialize(&psPMR->sEnvData)
+#define	PMR_ENV_FINALIZE(psPMR, sEnvData) PMREnvFinalize(&psPMR->sEnvData)
+
+/* The generic PMR code provides this function, using the macro that follows */
+PMR_ENV *PMREnvGetData(PMR *psPMR);
+
+#define	DEFINE_PMR_ENV_GET_DATA(psPMR, sEnvData) \
+	PMR_ENV *PMREnvGetData(PMR *psPMR) { return &psPMR->sEnvData; }
+
+/*
+ * Structures, functions and definitions used by the environment specific
+ * PMR code.
+ */
+
+void
+PMREnvDmaBufSetExportData(PMR *psPMR, void *pvData);
+
+void *
+PMREnvDmaBufGetExportData(PMR *psPMR);
+
+#endif /* !defined(PMR_ENV_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_os.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_os.c
new file mode 100644
index 000000000000..63645c930dba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_os.c
@@ -0,0 +1,815 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux OS PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/version.h>
+#include <linux/pfn_t.h>
+#include <linux/pfn.h>
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "devicemem_server_utils.h"
+#include "pmr.h"
+#include "pmr_os.h"
+#include "physmem_osmem_linux.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION_DEBUG)
+#define MIGRATE_DBG_LOG(x) PVR_DPF(x);
+#else
+#define MIGRATE_DBG_LOG(...)
+#endif
+
+typedef struct _PMR_OS_CPU_MAPPING_
+{
+	DLLIST_NODE sListNode;
+	struct vm_area_struct *ps_vma;
+} PMR_OS_CPU_MAPPING;
+
+
+static PVRSRV_ERROR
+PMROSLinkCPUMapping(PMR *psPMR, struct vm_area_struct *psVMArea)
+{
+	PMR_OS_CPU_MAPPING *psCPUMapping;
+	DLLIST_NODE *psListHead;
+
+	PMRLockHeldAssert(psPMR);
+
+	psCPUMapping = OSAllocMem(sizeof(*psCPUMapping));
+	PVR_RETURN_IF_NOMEM(psCPUMapping);
+
+	psCPUMapping->ps_vma = psVMArea;
+
+	psListHead = LinuxOSGetCPUMappingPrivateDataList(psPMR);
+	PVR_ASSERT(psListHead);
+
+	dllist_add_to_head(psListHead, &psCPUMapping->sListNode);
+
+	return PVRSRV_OK;
+}
+
+static void
+PMROSUnlinkCPUMapping(PMR *psPMR, struct vm_area_struct *psTargetVMArea)
+{
+	PDLLIST_NODE pNext, pNode;
+	DLLIST_NODE *psListHead;
+	PMR_OS_CPU_MAPPING *psExtractedMapping = NULL;
+
+	PMRLockHeldAssert(psPMR);
+
+	psListHead = LinuxOSGetCPUMappingPrivateDataList(psPMR);
+	PVR_ASSERT(psListHead);
+
+	dllist_foreach_node(psListHead, pNode, pNext)
+	{
+		PMR_OS_CPU_MAPPING *psCheckMapping = IMG_CONTAINER_OF(pNode,
+		                                                      PMR_OS_CPU_MAPPING,
+		                                                      sListNode);
+		if (psTargetVMArea == psCheckMapping->ps_vma)
+		{
+			psExtractedMapping = psCheckMapping;
+			break;
+		}
+	}
+
+	PVR_ASSERT(psExtractedMapping != NULL);
+
+	if (psExtractedMapping)
+	{
+		dllist_remove_node(&psExtractedMapping->sListNode);
+		OSFreeMem(psExtractedMapping);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unable to find vm_area_struct", __func__));
+	}
+}
+#endif
+
+/*
+ * x86_32:
+ * Use vm_insert_page because remap_pfn_range has issues when mapping HIGHMEM
+ * pages with default memory attributes; these HIGHMEM pages are skipped in
+ * set_pages_array_[uc,wc] during allocation; see reserve_pfn_range().
+ * Also vm_insert_page is faster.
+ *
+ * x86_64:
+ * Use vm_insert_page because it is faster.
+ *
+ * Other platforms:
+ * Use remap_pfn_range by default because it does not issue a cache flush.
+ * It is known that ARM32 benefits from this. When other platforms become
+ * available it has to be investigated if this assumption holds for them as well.
+ *
+ * Since vm_insert_page does more precise memory accounting we have the build
+ * flag PVR_MMAP_USE_VM_INSERT that forces its use. This is useful as a debug
+ * feature.
+ *
+ */
+#if defined(CONFIG_X86) || defined(PVR_MMAP_USE_VM_INSERT)
+#define PMR_OS_USE_VM_INSERT_PAGE 1
+#endif
+
+static void MMapPMROpen(struct vm_area_struct *ps_vma)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	PVRSRV_ERROR eError;
+
+	/* VM_DONTCOPY should ensure this function never gets called */
+	PVR_DPF((PVR_DBG_WARNING,
+			 "%s: Unexpected mmap open call, this is probably an application bug.",
+			 __func__));
+	PVR_DPF((PVR_DBG_WARNING,
+			 "%s: vma struct: 0x%p, vAddr: %#lX, length: %#lX, PMR pointer: 0x%p",
+			 __func__,
+			 ps_vma,
+			 ps_vma->vm_start,
+			 ps_vma->vm_end - ps_vma->vm_start,
+			 psPMR));
+
+	/* Should this entry-point be called for one of our PMRs we must increase
+	 * the refcount and lock down the physical addresses.
+	 */
+	eError = PMRRefPMR(psPMR);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PMRRefPMR");
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", ErrUnref);
+
+	/* MMapPMROpen() is called when a process is forked, but only if
+	 * mappings are to be inherited - so increment mapping count of the
+	 * PMR to prevent its layout being changed (if sparse).
+	 */
+	PMRClientCpuMapCountIncr(psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRLockPMR(psPMR);
+	eError = PMROSLinkCPUMapping(psPMR, ps_vma);
+	PMRUnlockPMR(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMROSLinkCPUMapping", ErrUnlock);
+#endif
+
+	return;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrUnlock:
+	PMRUnlockSysPhysAddresses(psPMR);
+#endif
+ErrUnref:
+	(void) PMRUnrefPMR(psPMR);
+}
+
+static void MMapPMRClose(struct vm_area_struct *ps_vma)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	PVRSRV_ERROR eError;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		uintptr_t vAddr = ps_vma->vm_start;
+
+		while (vAddr < ps_vma->vm_end)
+		{
+			/* USER MAPPING */
+			PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+			                                (IMG_UINT64)vAddr,
+			                                OSGetCurrentClientProcessIDKM());
+			vAddr += PAGE_SIZE;
+		}
+	}
+#else
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+	                            ps_vma->vm_end - ps_vma->vm_start,
+	                            OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRLockPMR(psPMR);
+	PMROSUnlinkCPUMapping(psPMR, ps_vma);
+	PMRUnlockPMR(psPMR);
+#endif
+
+	PMRUnlockSysPhysAddresses(psPMR);
+	/* Decrement the mapping count before Unref of PMR (as Unref could destroy the PMR) */
+	PMRClientCpuMapCountDecr(psPMR);
+	eError = PMRUnrefPMR(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+}
+
+/*
+ * This vma operation is used to read data from mmap regions. It is called
+ * by access_process_vm, which is called to handle PTRACE_PEEKDATA ptrace
+ * requests and reads from /proc/<pid>/mem.
+ */
+static int MMapVAccess(struct vm_area_struct *ps_vma, unsigned long addr,
+		       void *buf, int len, int write)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	unsigned long ulOffset = addr - ps_vma->vm_start;
+	size_t uiBytesCopied;
+	PVRSRV_ERROR eError;
+	int iRetVal = -EINVAL;
+
+	if (write)
+	{
+		eError = PMR_WriteBytes(psPMR,
+					(IMG_DEVMEM_OFFSET_T) ulOffset,
+					buf,
+					len,
+					&uiBytesCopied);
+	}
+	else
+	{
+		eError = PMR_ReadBytes(psPMR,
+				       (IMG_DEVMEM_OFFSET_T) ulOffset,
+				       buf,
+				       len,
+				       &uiBytesCopied);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error from %s (%d)",
+			 __func__,
+			 write ? "PMR_WriteBytes" : "PMR_ReadBytes",
+			 eError));
+	}
+	else
+	{
+		iRetVal = uiBytesCopied;
+	}
+
+	return iRetVal;
+}
+
+static INLINE int _OSMMapPMR(PVRSRV_DEVICE_NODE *psDevNode,
+							struct vm_area_struct *ps_vma,
+							IMG_DEVMEM_OFFSET_T uiOffset,
+							IMG_CPU_PHYADDR *psCpuPAddr,
+							IMG_UINT32 uiLog2PageSize,
+							IMG_BOOL bUseVMInsertPage,
+							IMG_BOOL bUseMixedMap)
+{
+	IMG_INT32 iStatus;
+	pfn_t sPFN;
+
+	sPFN = phys_to_pfn_t(psCpuPAddr->uiAddr, 0);
+
+	/*
+	 * vm_insert_page() allows insertion of individual pages into user
+	 * VMA space _only_ if page is a order-zero allocated page
+	 */
+	if (bUseVMInsertPage)
+	{
+		if (bUseMixedMap)
+		{
+			/*
+			 * This path is just for debugging. It should be
+			 * equivalent to the remap_pfn_range() path.
+			 */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
+			vm_fault_t vmf;
+
+			vmf = vmf_insert_mixed(ps_vma,
+									ps_vma->vm_start + uiOffset,
+									sPFN);
+			if (vmf & VM_FAULT_ERROR)
+			{
+				iStatus = vm_fault_to_errno(vmf, 0);
+			}
+			else
+			{
+				iStatus = 0;
+			}
+#else
+			iStatus = vm_insert_mixed(ps_vma,
+									  ps_vma->vm_start + uiOffset,
+									  sPFN);
+#endif
+		}
+		else
+		{
+			/* Since kernel 3.7 this sets VM_MIXEDMAP internally */
+			iStatus = vm_insert_page(ps_vma,
+									 ps_vma->vm_start + uiOffset,
+									 pfn_t_to_page(sPFN));
+		}
+	}
+	else
+	{
+		/*
+		   NOTE: Regarding absence of dma_mmap_coherent() in _OSMMapPMR()
+
+		   The current services mmap model maps in a PMR's full-length size
+		   into the user VMA & applies any user specified offset to the kernel
+		   returned zero-offset based VA in services client; this essentially
+		   means services server ignores ps_vma->vm_pgoff (this houses hPMR)
+		   during a mmap call.
+
+		   Furthermore, during a DMA/CMA memory allocation, multiple order-n
+		   pages are used to satisfy an allocation request due to DMA/CMA
+		   framework rounding-up allocation size to next power-of-two which
+		   can lead to wasted memory (so we don't allocate using single call).
+
+		   The combination of the above two issues mean that we cannot use the
+		   dma_mmap_coherent() for a number of reasons outlined below:
+
+		     - Services mmap semantics does not fit with dma_mmap_coherent()
+		       which requires proper ps_vma->vm_pgoff; seeing this houses a
+		       hPMR handle value, calls into dma_mmap_coherent() fails. This
+		       could be avoided by forcing ps_vma->vm_pgoff to zero but the
+		       ps_vma->vm_pgoff is applied to DMA bus address PFN and not
+		       user VMA which is always mapped at ps_vma->vm_start.
+
+		     - As multiple order-n pages are used for DMA/CMA allocations, a
+		       single dma_mmap_coherent() call with a vma->vm_pgoff set to
+		       zero cannot (maybe) be used because there is no guarantee that
+		       all of the multiple order-n pages in the PMR are physically
+		       contiguous from the first entry to the last. Whilst this is
+		       highly likely to be the case, there is no guarantee that it
+		       will be so we cannot depend on this being the case.
+
+		   The solution is to manually mmap DMA/CMA pages into user VMA
+		   using remap_pfn_range() directly. Furthermore, accounting is
+		   always compromised for DMA/CMA allocations.
+		*/
+		size_t uiNumContiguousBytes = 1ULL << uiLog2PageSize;
+
+		iStatus = remap_pfn_range(ps_vma,
+								  ps_vma->vm_start + uiOffset,
+								  pfn_t_to_pfn(sPFN),
+								  uiNumContiguousBytes,
+								  ps_vma->vm_page_prot);
+	}
+
+	return iStatus;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static vm_fault_t MMapPMRFault(struct vm_fault *ps_vmf)
+{
+	PVRSRV_ERROR eError;
+	vm_fault_t iFault;
+	PMR *psPMR = ps_vmf->vma->vm_private_data;
+
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_BOOL bValid;
+	IMG_UINT64 uiAddrOffsetBytes = ps_vmf->address - ps_vmf->vma->vm_start;
+	unsigned long ulPFN;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	if (!PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psPMR)))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Attempt to fault non movable PMR."));
+		return VM_FAULT_SIGBUS;
+	}
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Fault offset: %llu, addr 0x%llx, vm_start 0x%llx, addr offset b 0x%llx, PMR 0x%llx",
+	                (unsigned long long) ps_vmf->pgoff,
+	                (unsigned long long) ps_vmf->address,
+	                (unsigned long long) ps_vmf->address - ps_vmf->vma->vm_start,
+	                (unsigned long long) uiAddrOffsetBytes,
+	                (unsigned long long) psPMR
+	                ));
+
+	PMRLockPMR(psPMR);
+
+	/* Obtain map range pfns */
+	eError = PMR_CpuPhysAddr(psPMR,
+	                         PAGE_SHIFT,
+	                         1,
+	                         uiAddrOffsetBytes,
+	                         &sCpuPAddr,
+	                         &bValid,
+	                         MAPPING_USE | CPU_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_CpuPhysAddr", ErrSigFaultReturn);
+	PVR_LOG_GOTO_IF_FALSE(bValid, "Attempted remap with invalid PMR offset", ErrSigFaultReturn);
+
+	ulPFN = pfn_t_to_pfn(phys_to_pfn_t(sCpuPAddr.uiAddr, 0));
+
+	iFault = vmf_insert_pfn_prot(ps_vmf->vma,
+	                             ps_vmf->address,
+	                             ulPFN,
+	                             ps_vmf->vma->vm_page_prot);
+
+	if (iFault != VM_FAULT_NOPAGE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "vmf_insert_pfn_prot failed in MMapPMRFault"));
+		goto ErrReturn;
+	}
+
+
+	PMRUnlockPMR(psPMR);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+								(void*)(uintptr_t)(ps_vmf->vma->vm_start + uiAddrOffsetBytes),
+								sCpuPAddr,
+								PAGE_SIZE,
+								OSGetCurrentClientProcessIDKM()
+								DEBUG_MEMSTATS_VALUES);
+#endif
+
+	return iFault;
+
+ErrSigFaultReturn:
+	iFault = VM_FAULT_SIGBUS;
+ErrReturn:
+	PMRUnlockPMR(psPMR);
+	return iFault;
+}
+#endif
+
+static const struct vm_operations_struct gsMMapOps =
+{
+	.open = &MMapPMROpen,
+	.close = &MMapPMRClose,
+	.access = &MMapVAccess,
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	.fault = &MMapPMRFault
+#endif
+};
+
+PVRSRV_ERROR
+OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData)
+{
+	struct vm_area_struct *ps_vma = pOSMMapData;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+	PVRSRV_ERROR eError;
+	size_t uiLength;
+	IMG_INT32 iStatus;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_UINT32 ui32CPUCacheFlags;
+	pgprot_t sPageProt;
+	IMG_CPU_PHYADDR asCpuPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 uiOffsetIdx;
+	IMG_UINT32 uiNumOfPFNs;
+	IMG_UINT32 uiLog2PageSize;
+	IMG_CPU_PHYADDR *psCpuPAddr;
+	IMG_BOOL *pbValid;
+	IMG_BOOL bUseMixedMap = IMG_FALSE;
+	IMG_BOOL bUseVMInsertPage = IMG_FALSE;
+	vm_flags_t uVMFlags = ps_vma->vm_flags;
+
+	/* if writeable but not shared mapping is requested then fail */
+	PVR_RETURN_IF_INVALID_PARAM(((uVMFlags & VM_WRITE) == 0) ||
+	                            ((uVMFlags & VM_SHARED) != 0));
+
+	uiLength = ps_vma->vm_end - ps_vma->vm_start;
+
+	/* Check early if the requested mapping size doesn't exceed the virtual
+	 * PMR size. */
+	if (PMR_LogicalSize(psPMR) < uiLength)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, ErrReturn);
+	}
+
+	uiLog2PageSize = PMR_GetLog2Contiguity(psPMR);
+
+	/* Check the number of PFNs to be mapped is valid. */
+	uiNumOfPFNs = uiLength >> uiLog2PageSize;
+	if (uiNumOfPFNs == 0)
+	{
+		/* print as 64-bit value to avoid Smatch warning */
+		PVR_LOG_VA(PVR_DBG_ERROR,
+		           "uiLength is invalid. Must be >= %" IMG_UINT64_FMTSPEC ".",
+		           IMG_UINT64_C(1) << uiLog2PageSize);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, ErrReturn);
+	}
+
+	/*
+	 * Take a reference on the PMR so that it can't be freed while mapped
+	 * into the user process.
+	 */
+	eError = PMRRefPMR(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrReturn);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrUnrefPMR);
+
+	/* Increment mapping count of the PMR so that its layout cannot be
+	 * changed (if sparse).
+	 */
+	PMRLockPMR(psPMR);
+
+	PMRClientCpuMapCountIncr(psPMR);
+	sPageProt = vm_get_page_prot(uVMFlags);
+
+	eError = DevmemCPUCacheMode(PMR_Flags(psPMR), &ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrUnlockPhysAddr;
+	}
+
+	switch (ui32CPUCacheFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				sPageProt = pgprot_noncached(sPageProt);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				sPageProt = pgprot_writecombine(sPageProt);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+		{
+/* Do not set to write-combine for plato */
+#if !defined(PLATO_MEMORY_CONFIG)
+				PHYS_HEAP *psPhysHeap = PMR_PhysHeap(psPMR);
+
+				if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA)
+					sPageProt = pgprot_writecombine(sPageProt);
+#endif
+				break;
+		}
+
+		default:
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto ErrUnlockPhysAddr;
+	}
+	ps_vma->vm_page_prot = sPageProt;
+
+	uVMFlags |= VM_IO;
+
+	/* Don't include the mapping in core dumps */
+	uVMFlags |= VM_DONTDUMP;
+
+	/*
+	 * Disable mremap because our nopage handler assumes all
+	 * page requests have already been validated.
+	 */
+	uVMFlags |= VM_DONTEXPAND;
+
+	/* Don't allow mapping to be inherited across a process fork */
+	uVMFlags |= VM_DONTCOPY;
+
+#if defined(PMR_OS_USE_VM_INSERT_PAGE)
+	/* Is this mmap targeting non order-zero pages or does it use pfn mappings?
+	 * If yes, don't use vm_insert_page */
+	bUseVMInsertPage = (uiLog2PageSize == PAGE_SHIFT) && (PMR_GetType(psPMR) != PMR_TYPE_EXTMEM);
+#endif
+
+	/* Can we use stack allocations */
+	if (uiNumOfPFNs > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psCpuPAddr = OSAllocMem(uiNumOfPFNs * sizeof(*psCpuPAddr));
+		PVR_LOG_GOTO_IF_NOMEM(psCpuPAddr, eError, ErrUnlockPhysAddr);
+
+		/* Should allocation fail, clean-up here before exiting */
+		pbValid = OSAllocMem(uiNumOfPFNs * sizeof(*pbValid));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrFreeCpuPAddr);
+	}
+	else
+	{
+		psCpuPAddr = asCpuPAddr;
+		pbValid = abValid;
+	}
+
+	/* Obtain map range pfns */
+	eError = PMR_CpuPhysAddr(psPMR,
+				 uiLog2PageSize,
+				 uiNumOfPFNs,
+				 0,
+				 psCpuPAddr,
+				 pbValid,
+				 CPU_USE | MAPPING_USE);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrFreeValid;
+	}
+
+	/*
+	 * Scan the map range for pfns without struct page* handling. If
+	 * we find one, this is a mixed map, and we can't use vm_insert_page()
+	 * NOTE: vm_insert_page() allows insertion of individual pages into user
+	 * VMA space _only_ if said page is an order-zero allocated page.
+	 */
+	if (bUseVMInsertPage)
+	{
+		pfn_t sPFN;
+
+		for (uiOffsetIdx = 0; uiOffsetIdx < uiNumOfPFNs; ++uiOffsetIdx)
+		{
+			if (pbValid[uiOffsetIdx])
+			{
+				sPFN = phys_to_pfn_t(psCpuPAddr[uiOffsetIdx].uiAddr, 0);
+
+				if (!pfn_t_valid(sPFN) || page_count(pfn_t_to_page(sPFN)) == 0)
+				{
+					bUseMixedMap = IMG_TRUE;
+					break;
+				}
+			}
+		}
+
+		if (bUseMixedMap)
+		{
+			uVMFlags |= VM_MIXEDMAP;
+		}
+	}
+	else
+	{
+		uVMFlags |= VM_PFNMAP;
+	}
+
+	/* Actually initialise the flags */
+	pvr_vm_flags_init(ps_vma, uVMFlags);
+
+	/* For each PMR page-size contiguous bytes, map page(s) into user VMA */
+	for (uiOffset = 0; uiOffset < uiLength; uiOffset += 1ULL<<uiLog2PageSize)
+	{
+		uiOffsetIdx = uiOffset >> uiLog2PageSize;
+		/*
+		 * Only map in pages that are valid, any that aren't will be
+		 * picked up by the nopage handler which will return a zeroed
+		 * page for us.
+		 */
+		if (pbValid[uiOffsetIdx])
+		{
+			iStatus = _OSMMapPMR(psDevNode,
+								 ps_vma,
+								 uiOffset,
+								 &psCpuPAddr[uiOffsetIdx],
+								 uiLog2PageSize,
+								 bUseVMInsertPage,
+								 bUseMixedMap);
+			if (iStatus)
+			{
+				/* Failure error code doesn't get propagated */
+				eError = PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+				PVR_ASSERT(0);
+				goto ErrFreeValid;
+			}
+		}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+#define PMR_OS_BAD_CPUADDR 0x0BAD0BAD
+		{
+			IMG_CPU_PHYADDR sPAddr;
+			sPAddr.uiAddr = pbValid[uiOffsetIdx] ?
+					psCpuPAddr[uiOffsetIdx].uiAddr :
+					IMG_CAST_TO_CPUPHYADDR_UINT(PMR_OS_BAD_CPUADDR);
+
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+										(void*)(uintptr_t)(ps_vma->vm_start + uiOffset),
+										sPAddr,
+										1<<uiLog2PageSize,
+										OSGetCurrentClientProcessIDKM()
+										DEBUG_MEMSTATS_VALUES);
+		}
+#undef PMR_OS_BAD_CPUADDR
+#endif
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, uiNumOfPFNs * PAGE_SIZE, OSGetCurrentClientProcessIDKM());
+#endif
+
+	if (psCpuPAddr != asCpuPAddr)
+	{
+		OSFreeMem(psCpuPAddr);
+		OSFreeMem(pbValid);
+	}
+
+	/* let us see the PMR so we can unlock it later */
+	ps_vma->vm_private_data = psPMR;
+
+	/* Install open and close handlers for ref-counting */
+	ps_vma->vm_ops = &gsMMapOps;
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	/* record the stats */
+	MMapStatsAddOrUpdatePMR(psPMR, uiLength);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	eError = PMROSLinkCPUMapping(psPMR, ps_vma);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLinkCpuMapping", ErrUnlockPhysAddr);
+#endif
+
+	PMRUnlockPMR(psPMR);
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+ErrFreeValid:
+	if (pbValid != abValid)
+	{
+		OSFreeMem(pbValid);
+	}
+ErrFreeCpuPAddr:
+	if (psCpuPAddr != asCpuPAddr)
+	{
+		OSFreeMem(psCpuPAddr);
+	}
+ErrUnlockPhysAddr:
+	PMRClientCpuMapCountDecr(psPMR);
+	PMRUnlockPMR(psPMR);
+	PMRUnlockSysPhysAddresses(psPMR);
+ErrUnrefPMR:
+	(void) PMRUnrefPMR(psPMR);
+ErrReturn:
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static void
+OSLinuxUnmapPageInCPUMapping(PMR *psPMR, struct vm_area_struct *ps_vma, IMG_UINT32 ui32LogicalPgOffset)
+{
+	IMG_UINT64 ui64OffsetBytes = IMG_PAGES2BYTES64(ui32LogicalPgOffset, PAGE_SHIFT);
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "CPU PTE Zap, PMR 0x%llx, Offset %u, OffsetB 0x%llx, vm_start 0x%llx, vm_addr 0x%llx",
+	                (unsigned long long) psPMR,
+	                ui32LogicalPgOffset,
+	                (unsigned long long) ui64OffsetBytes,
+	                (unsigned long long) ps_vma->vm_start,
+	                (unsigned long long) ps_vma->vm_start + ui64OffsetBytes));
+
+	zap_vma_ptes(ps_vma, ps_vma->vm_start + ui64OffsetBytes, PAGE_SIZE);
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+	                                (IMG_UINT64)ps_vma->vm_start + ui64OffsetBytes,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+}
+
+void
+OSLinuxPMRUnmapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset)
+{
+	DLLIST_NODE *psNext, *psNode;
+
+	PMRLockPMR(psPMR);
+
+	dllist_foreach_node(psMappingListHead, psNode, psNext)
+	{
+		PMR_OS_CPU_MAPPING *psCPUMapping = IMG_CONTAINER_OF(psNode,
+		                                                 PMR_OS_CPU_MAPPING,
+		                                                 sListNode);
+
+		OSLinuxUnmapPageInCPUMapping(psPMR, psCPUMapping->ps_vma, ui32LogicalPgOffset);
+	}
+
+	PMRUnlockPMR(psPMR);
+
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/private_data.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/private_data.h
new file mode 100644
index 000000000000..60a1fac0b970
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/private_data.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux private data structure
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(INCLUDED_PRIVATE_DATA_H)
+#define INCLUDED_PRIVATE_DATA_H
+
+#include <linux/fs.h>
+
+#include "connection_server.h"
+#include "pvr_drm.h"
+
+#define PVR_SRVKM_PRIV_DATA_IDX 0
+#define PVR_SYNC_PRIV_DATA_IDX  1
+
+#define PVR_NUM_PRIV_DATA_IDXS  2
+
+CONNECTION_DATA *LinuxServicesConnectionFromFile(struct file *pFile);
+CONNECTION_DATA *LinuxSyncConnectionFromFile(struct file *pFile);
+
+#endif /* !defined(INCLUDED_PRIVATE_DATA_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.c
new file mode 100644
index 000000000000..77739189b0aa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.c
@@ -0,0 +1,643 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Receives calls from the user portion of services and
+                despatches them to functions in the kernel portion.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include <linux/mm_types.h>
+
+#include "img_defs.h"
+#include "pvr_bridge.h"
+#include "pvr_bridge_k.h"
+#include "connection_server.h"
+#include "syscommon.h"
+#include "pvr_debug.h"
+#include "di_server.h"
+#include "private_data.h"
+#include "linkage.h"
+#include "pmr.h"
+#include "rgx_bvnc_defs_km.h"
+#include "pvrsrv_bridge_init.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_print.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "pvr_drm.h"
+#include "pvr_drv.h"
+
+#include "env_connection.h"
+#include <linux/sched.h>
+#include <linux/freezer.h>
+
+/* RGX: */
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+
+#include "srvcore.h"
+#include "common_srvcore_bridge.h"
+#include "kernel_compatibility.h"
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+#if defined(MODULE_TEST)
+/************************************************************************/
+// additional includes for services testing
+/************************************************************************/
+#include "pvr_test_bridge.h"
+#include "kern_test.h"
+/************************************************************************/
+// end of additional includes
+/************************************************************************/
+#endif
+
+/* The mmap code has its own mutex, to prevent possible re-entrant issues
+ * when the same PMR is mapped from two different connections/processes.
+ */
+static DEFINE_MUTEX(g_sMMapMutex);
+
+#define _SUSPENDED 1
+#define _NOT_SUSPENDED 0
+static ATOMIC_T g_iDriverSuspendCount;
+static ATOMIC_T g_iNumActiveDriverThreads;
+static ATOMIC_T g_iNumActiveKernelThreads;
+static IMG_HANDLE g_hDriverThreadEventObject;
+
+
+#if defined(DEBUG_BRIDGE_KM)
+static DI_ENTRY *gpsDIBridgeStatsEntry;
+
+static void *BridgeStatsDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psDispatchTable = DIGetPrivData(psEntry);
+
+	if (psDispatchTable == NULL || *pui64Pos > BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		return NULL;
+	}
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	return &(psDispatchTable[*pui64Pos - 1]);
+}
+
+static void BridgeStatsDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+}
+
+static void *BridgeStatsDINext(OSDI_IMPL_ENTRY *psEntry, void *pvData,
+                               IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psDispatchTable = DIGetPrivData(psEntry);
+	IMG_UINT64 uiItemAskedFor = *pui64Pos; /* pui64Pos on entry is the index to return */
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	/* Is the item asked for (starts at 0) a valid table index? */
+	if (uiItemAskedFor < BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		(*pui64Pos)++; /* on exit it is the next DI index to ask for */
+		return &(psDispatchTable[uiItemAskedFor]);
+	}
+
+	/* Now passed the end of the table to indicate stop */
+	return NULL;
+}
+
+static int BridgeStatsDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	if (pvData == DI_START_TOKEN)
+	{
+		BridgeGlobalStatsLock();
+		DIPrintf(psEntry,
+		         "Total ioctl call count = %u\n"
+		         "Total number of bytes copied via copy_from_user = %u\n"
+		         "Total number of bytes copied via copy_to_user = %u\n"
+		         "Total number of bytes copied via copy_*_user = %u\n\n"
+		         "%3s: %-60s | %-48s | %10s | %20s | %20s | %20s | %20s\n",
+		         g_BridgeGlobalStats.ui32IOCTLCount,
+		         g_BridgeGlobalStats.ui32TotalCopyFromUserBytes,
+		         g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		         g_BridgeGlobalStats.ui32TotalCopyFromUserBytes +
+		             g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		         "#",
+		         "Bridge Name",
+		         "Wrapper Function",
+		         "Call Count",
+		         "copy_from_user (B)",
+		         "copy_to_user (B)",
+		         "Total Time (us)",
+		         "Max Time (us)");
+		BridgeGlobalStatsUnlock();
+	}
+	else if (pvData != NULL)
+	{
+		PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psTableEntry = pvData;
+		IMG_UINT32 ui32Remainder;
+
+		BridgeGlobalStatsLock();
+		DIPrintf(psEntry,
+		         "%3d: %-60s   %-48s   %-10u   %-20u   %-20u   %-20" IMG_UINT64_FMTSPEC "   %-20" IMG_UINT64_FMTSPEC "\n",
+		         (IMG_UINT32)(((size_t)psTableEntry-(size_t)g_BridgeDispatchTable)/sizeof(*g_BridgeDispatchTable)),
+		         psTableEntry->pszIOCName,
+		         (psTableEntry->pfFunction != NULL) ? psTableEntry->pszFunctionName : "(null)",
+		         psTableEntry->ui32CallCount,
+		         psTableEntry->ui32CopyFromUserTotalBytes,
+		         psTableEntry->ui32CopyToUserTotalBytes,
+		         OSDivide64r64(psTableEntry->ui64TotalTimeNS, 1000, &ui32Remainder),
+		         OSDivide64r64(psTableEntry->ui64MaxTimeNS, 1000, &ui32Remainder));
+		BridgeGlobalStatsUnlock();
+	}
+
+	return 0;
+}
+
+static IMG_INT64 BridgeStatsWrite(const IMG_CHAR *pcBuffer,
+                                  IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                                  void *pvData)
+{
+	IMG_UINT32 i;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[0] == '0', -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	/* Reset stats. */
+
+	BridgeGlobalStatsLock();
+
+	g_BridgeGlobalStats.ui32IOCTLCount = 0;
+	g_BridgeGlobalStats.ui32TotalCopyFromUserBytes = 0;
+	g_BridgeGlobalStats.ui32TotalCopyToUserBytes = 0;
+
+	for (i = 0; i < ARRAY_SIZE(g_BridgeDispatchTable); i++)
+	{
+		g_BridgeDispatchTable[i].ui32CallCount = 0;
+		g_BridgeDispatchTable[i].ui32CopyFromUserTotalBytes = 0;
+		g_BridgeDispatchTable[i].ui32CopyToUserTotalBytes = 0;
+		g_BridgeDispatchTable[i].ui64TotalTimeNS = 0;
+		g_BridgeDispatchTable[i].ui64MaxTimeNS = 0;
+	}
+
+	BridgeGlobalStatsUnlock();
+
+	return ui64Count;
+}
+
+#endif /* defined(DEBUG_BRIDGE_KM) */
+
+PVRSRV_ERROR LinuxGetThreadActivityStats(LINUX_THREAD_ACTIVITY_STATS *psThreadStats)
+{
+	PVR_RETURN_IF_FALSE(psThreadStats != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psThreadStats->i32KernelThreadCount = OSAtomicRead(&g_iNumActiveKernelThreads);
+	psThreadStats->i32DriverThreadCount = OSAtomicRead(&g_iNumActiveDriverThreads);
+	psThreadStats->i32SuspendedThreadCount = OSAtomicRead(&g_iDriverSuspendCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSPlatformBridgeInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = InitDMABUFBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDMABUFBridge");
+
+	OSAtomicWrite(&g_iDriverSuspendCount, 0);
+	OSAtomicWrite(&g_iNumActiveDriverThreads, 0);
+	OSAtomicWrite(&g_iNumActiveKernelThreads, 0);
+
+	eError = OSEventObjectCreate("Global driver thread event object",
+	                             &g_hDriverThreadEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", EventCreateError);
+
+#if defined(DEBUG_BRIDGE_KM)
+	{
+		DI_ITERATOR_CB sIter = {
+			.pfnStart = BridgeStatsDIStart,
+			.pfnStop = BridgeStatsDIStop,
+			.pfnNext = BridgeStatsDINext,
+			.pfnShow = BridgeStatsDIShow,
+			.pfnWrite = BridgeStatsWrite,
+
+			//Expects '0' + Null terminator
+			.ui32WriteLenMax = ((1U)+1U)
+		};
+
+		eError = DICreateEntry("bridge_stats", NULL, &sIter,
+		                       &g_BridgeDispatchTable[0],
+		                       DI_ENTRY_TYPE_GENERIC,
+		                       &gpsDIBridgeStatsEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", DIEntryCreateError);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(DEBUG_BRIDGE_KM)
+DIEntryCreateError:
+#endif
+EventCreateError:
+	OSEventObjectDestroy(g_hDriverThreadEventObject);
+	g_hDriverThreadEventObject = NULL;
+
+	return eError;
+}
+
+void OSPlatformBridgeDeInit(void)
+{
+#if defined(DEBUG_BRIDGE_KM)
+	DIDestroyEntry(gpsDIBridgeStatsEntry);
+#endif
+
+	OSEventObjectDestroy(g_hDriverThreadEventObject);
+	g_hDriverThreadEventObject = NULL;
+
+	DeinitDMABUFBridge();
+}
+
+PVRSRV_ERROR LinuxBridgeBlockClientsAccess(struct pvr_drm_private *psDevPriv,
+                                           IMG_BOOL bShutdown)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	__maybe_unused IMG_INT iSuspendCount;
+
+	eError = OSEventObjectOpen(g_hDriverThreadEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return eError;
+	}
+
+	iSuspendCount = OSAtomicIncrement(&g_iDriverSuspendCount);
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver suspended %d times.", __func__,
+	         iSuspendCount));
+
+	if (OSAtomicCompareExchange(&psDevPriv->suspended, _NOT_SUSPENDED,
+	                            _SUSPENDED) == _SUSPENDED)
+	{
+		OSAtomicDecrement(&g_iDriverSuspendCount);
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p already suspended", __func__,
+		        psDevPriv->dev_node));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto CloseEventObject;
+	}
+
+	/* now wait for any threads currently in the server to exit */
+	while (OSAtomicRead(&g_iNumActiveDriverThreads) != 0 ||
+		   (OSAtomicRead(&g_iNumActiveKernelThreads) != 0 && !bShutdown))
+	{
+		if (OSAtomicRead(&g_iNumActiveDriverThreads) != 0)
+		{
+			PVR_LOG(("%s: waiting for user threads (%d)", __func__,
+			        OSAtomicRead(&g_iNumActiveDriverThreads)));
+		}
+		if (OSAtomicRead(&g_iNumActiveKernelThreads) != 0)
+		{
+			PVR_LOG(("%s: waiting for kernel threads (%d)", __func__,
+			        OSAtomicRead(&g_iNumActiveKernelThreads)));
+		}
+		/* Regular wait is called here (and not OSEventObjectWaitKernel) because
+		 * this code is executed by the caller of .suspend/.shutdown callbacks
+		 * which is most likely PM (or other actor responsible for suspend
+		 * process). Because of that this thread shouldn't and most likely
+		 * event cannot be frozen. */
+		OSEventObjectWait(hEvent);
+	}
+
+CloseEventObject:
+	OSEventObjectClose(hEvent);
+
+	return eError;
+}
+
+PVRSRV_ERROR LinuxBridgeUnblockClientsAccess(struct pvr_drm_private *psDevPriv)
+{
+	PVRSRV_ERROR eError;
+	__maybe_unused IMG_INT iSuspendCount;
+
+	/* resume the driver and then signal so any waiting threads wake up */
+	if (OSAtomicCompareExchange(&psDevPriv->suspended, _SUSPENDED,
+	                            _NOT_SUSPENDED) == _NOT_SUSPENDED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device is not suspended", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iSuspendCount = OSAtomicDecrement(&g_iDriverSuspendCount);
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver suspended %d times.", __func__,
+	         iSuspendCount));
+
+	eError = OSEventObjectSignal(g_hDriverThreadEventObject);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSEventObjectSignal failed: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR LinuxBridgeSignalIfSuspended(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (OSAtomicRead(&g_iDriverSuspendCount) > 0)
+	{
+		PVRSRV_ERROR eError = OSEventObjectSignal(g_hDriverThreadEventObject);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal driver thread event"
+			        " object: %s", __func__, PVRSRVGetErrorString(eError)));
+		}
+	}
+
+	return eError;
+}
+
+void LinuxBridgeNumActiveKernelThreadsIncrement(void)
+{
+	OSAtomicIncrement(&g_iNumActiveKernelThreads);
+}
+
+void LinuxBridgeNumActiveKernelThreadsDecrement(void)
+{
+	OSAtomicDecrement(&g_iNumActiveKernelThreads);
+	PVR_ASSERT(OSAtomicRead(&g_iNumActiveKernelThreads) >= 0);
+
+	/* Signal on every decrement in case LinuxBridgeBlockClientsAccess() is
+	 * waiting for the threads to freeze.
+	 * (error is logged in called function so ignore, we can't do much with
+	 * it anyway) */
+	(void) LinuxBridgeSignalIfSuspended();
+}
+
+static PVRSRV_ERROR _WaitForDriverUnsuspend(void)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+
+	eError = OSEventObjectOpen(g_hDriverThreadEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return eError;
+	}
+
+	while (OSAtomicRead(&g_iDriverSuspendCount) == 0)
+	{
+		/* we should be able to use normal (not kernel) wait here since
+		 * we were just unfrozen and most likely we're not going to
+		 * be frozen again (?) */
+		OSEventObjectWait(hEvent);
+	}
+
+	OSEventObjectClose(hEvent);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVDriverThreadEnter(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	CONNECTION_DATA *psConnection = (CONNECTION_DATA *)pvData;
+
+	/* Block if the associated device has been placed into a FROZEN state.
+	 * In this case we must await a PVRSRVDeviceThaw() completion request.
+	 * Device is obtained from the incoming psConnection if pvData is non-NULL.
+	 */
+	if (likely(pvData != NULL))
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+		PVRSRVBlockIfFrozen(psDevNode);
+		OSAtomicIncrement(&psDevNode->iThreadsActive);
+	}
+
+	/* increment first so there is no race between this value and
+	 * g_iDriverSuspendCount in LinuxBridgeBlockClientsAccess() */
+	OSAtomicIncrement(&g_iNumActiveDriverThreads);
+
+	if (OSAtomicRead(&g_iDriverSuspendCount) > 0)
+	{
+		/* decrement here because the driver is going to be suspended and
+		 * this thread is going to be frozen so we don't want to wait for
+		 * it in LinuxBridgeBlockClientsAccess() */
+		OSAtomicDecrement(&g_iNumActiveDriverThreads);
+
+		/* during suspend procedure this will put the current thread to
+		 * the freezer but during shutdown this will just return */
+		try_to_freeze();
+
+		/* if the thread was unfrozen but the number of suspends is non-0 wait
+		 * for it
+		 * in case this is a shutdown the thread was not frozen so we'll
+		 * wait here indefinitely but this is ok (and this is in fact what
+		 * we want) because no thread should be entering the driver in such
+		 * case */
+		eError = _WaitForDriverUnsuspend();
+
+		/* increment here because that means that the thread entered the
+		 * driver */
+		OSAtomicIncrement(&g_iNumActiveDriverThreads);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to wait for driver"
+			        " unsuspend: %s", __func__,
+			        PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVDriverThreadExit(void *pvData)
+{
+	CONNECTION_DATA *psConnection = (CONNECTION_DATA *)pvData;
+	OSAtomicDecrement(&g_iNumActiveDriverThreads);
+
+	/* Decrement the number of threads active on this device if the
+	 * connection is known.
+	 */
+	if (psConnection != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+		OSAtomicDecrement(&psDevNode->iThreadsActive);
+	}
+	/* if the driver is being suspended then we need to signal the
+	 * event object as the thread suspending the driver is waiting
+	 * for active threads to exit
+	 * error is logged in called function so ignore returned error
+	 */
+	(void) LinuxBridgeSignalIfSuspended();
+}
+
+int
+PVRSRV_BridgeDispatchKM(struct drm_device __maybe_unused *dev, void *arg, struct drm_file *pDRMFile)
+{
+	struct drm_pvr_srvkm_cmd *psSrvkmCmd = (struct drm_pvr_srvkm_cmd *) arg;
+	PVRSRV_BRIDGE_PACKAGE sBridgePackageKM = { 0 };
+	CONNECTION_DATA *psConnection = LinuxServicesConnectionFromFile(pDRMFile->filp);
+	PVRSRV_ERROR error;
+
+	if (psConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid connection data"));
+		return -EFAULT;
+	}
+
+	PVR_ASSERT(psSrvkmCmd != NULL);
+
+	DRM_DEBUG("tgid=%d, tgid_connection=%d, bridge_id=%d, func_id=%d",
+			  task_tgid_nr(current),
+			  ((ENV_CONNECTION_DATA *)PVRSRVConnectionPrivateData(psConnection))->owner,
+			  psSrvkmCmd->bridge_id,
+			  psSrvkmCmd->bridge_func_id);
+
+	error = PVRSRVDriverThreadEnter(psConnection);
+	PVR_LOG_GOTO_IF_ERROR(error, "PVRSRVDriverThreadEnter", e0);
+
+	sBridgePackageKM.ui32BridgeID = psSrvkmCmd->bridge_id;
+	sBridgePackageKM.ui32FunctionID = psSrvkmCmd->bridge_func_id;
+	sBridgePackageKM.ui32Size = sizeof(sBridgePackageKM);
+	sBridgePackageKM.pvParamIn = (void __user *)(uintptr_t)psSrvkmCmd->in_data_ptr;
+	sBridgePackageKM.ui32InBufferSize = psSrvkmCmd->in_data_size;
+	sBridgePackageKM.pvParamOut = (void __user *)(uintptr_t)psSrvkmCmd->out_data_ptr;
+	sBridgePackageKM.ui32OutBufferSize = psSrvkmCmd->out_data_size;
+
+	error = BridgedDispatchKM(psConnection, &sBridgePackageKM);
+
+e0:
+	PVRSRVDriverThreadExit(psConnection);
+
+	return OSPVRSRVToNativeError(error);
+}
+
+int
+PVRSRV_MMap(struct file *pFile, struct vm_area_struct *ps_vma)
+{
+	CONNECTION_DATA *psConnection = LinuxServicesConnectionFromFile(pFile);
+	IMG_HANDLE hSecurePMRHandle = (IMG_HANDLE)((uintptr_t)ps_vma->vm_pgoff);
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiProtFlags =
+	    (BITMASK_HAS(ps_vma->vm_flags, VM_READ) ? PVRSRV_MEMALLOCFLAG_CPU_READABLE : 0) |
+	    (BITMASK_HAS(ps_vma->vm_flags, VM_WRITE) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0);
+
+	if (psConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid connection data"));
+		return -ENOENT;
+	}
+
+	eError = PVRSRVDriverThreadEnter(psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDriverThreadEnter", e0);
+
+	/*
+	 * The bridge lock used here to protect PVRSRVLookupHandle is replaced
+	 * by a specific lock considering that the handle functions have now
+	 * their own lock. This change was necessary to solve the lockdep issues
+	 * related with the PVRSRV_MMap.
+	 */
+
+	eError = PVRSRVLookupHandle(psConnection->psHandleBase,
+								(void **)&psPMR,
+								hSecurePMRHandle,
+								PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								IMG_TRUE);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	mutex_lock(&g_sMMapMutex);
+
+	/* Forcibly clear the VM_MAYWRITE flag as this is inherited from the
+	 * kernel mmap code and we do not want to produce a potentially writable
+	 * mapping from a read-only mapping.
+	 */
+	if (!BITMASK_HAS(ps_vma->vm_flags, VM_WRITE))
+	{
+		pvr_vm_flags_clear(ps_vma, VM_MAYWRITE);
+	}
+
+	/* Note: PMRMMapPMR will take a reference on the PMR.
+	 * Unref the handle immediately, because we have now done
+	 * the required operation on the PMR (whether it succeeded or not)
+	 */
+	eError = PMRMMapPMR(psPMR, ps_vma, uiProtFlags);
+	mutex_unlock(&g_sMMapMutex);
+	PVRSRVReleaseHandle(psConnection->psHandleBase, hSecurePMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMRMMapPMR failed (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	PVRSRVDriverThreadExit(psConnection);
+
+	return 0;
+
+e0:
+	PVRSRVDriverThreadExit(psConnection);
+
+	PVR_DPF((PVR_DBG_ERROR, "Failed with error: %s", PVRSRVGetErrorString(eError)));
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return OSPVRSRVToNativeError(eError);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.h
new file mode 100644
index 000000000000..2885dd465eaf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Receives calls from the user portion of services and
+                despatches them to functions in the kernel portion.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BRIDGE_K_H
+#define PVR_BRIDGE_K_H
+
+#include "pvrsrv_error.h"
+#include "pvr_drv.h"
+
+typedef struct LINUX_THREAD_ACTIVITY_STATS
+{
+	IMG_INT32 i32KernelThreadCount;
+	IMG_INT32 i32DriverThreadCount;
+	IMG_INT32 i32SuspendedThreadCount;
+} LINUX_THREAD_ACTIVITY_STATS;
+
+/*!
+******************************************************************************
+ @Function      LinuxBridgeBlockClientsAccess
+ @Description   This function will wait for any existing threads in the Server
+                to exit and then disable access to the driver. New threads will
+                not be allowed to enter the Server until the driver is
+                unsuspended (see LinuxBridgeUnblockClientsAccess).
+ @Input         psDevPriv pointer to devices OS specific data
+ @Input         bShutdown this flag indicates that the function was called
+                          from a shutdown callback and therefore it will
+                          not wait for the kernel threads to get frozen
+                          (because this doesn't happen during shutdown
+                          procedure)
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR LinuxBridgeBlockClientsAccess(struct pvr_drm_private *psDevPriv,
+                                           IMG_BOOL bShutdown);
+
+/*!
+******************************************************************************
+ @Function      LinuxBridgeUnblockClientsAccess
+ @Description   This function will re-enable the bridge and allow any threads
+                waiting to enter the Server to continue.
+ @Input         psDevPriv pointer to devices OS specific data
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR LinuxBridgeUnblockClientsAccess(struct pvr_drm_private *psDevPriv);
+
+void LinuxBridgeNumActiveKernelThreadsIncrement(void);
+void LinuxBridgeNumActiveKernelThreadsDecrement(void);
+
+/*************************************************************************/ /*!
+ @Function      LinuxGetThreadActivityStats
+ @Description   Getter for active and suspended thread stats.
+
+ @Output        psThreadStats   Struct to be populated with thread activity
+                                stats.
+
+ @Return        PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR LinuxGetThreadActivityStats(LINUX_THREAD_ACTIVITY_STATS *psThreadStats);
+
+/*!
+******************************************************************************
+ @Function      PVRSRVDriverThreadEnter
+ @Description   Increments number of client threads currently operating
+                in the driver's context.
+                If the driver is currently being suspended this function
+                will call try_to_freeze() on behalf of the client thread.
+                When the driver is resumed the function will exit and allow
+                the thread into the driver.
+ @Input         Reference to Connection data. NULL if no associated
+                 connection / device.
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDriverThreadEnter(void *pvData);
+
+/*!
+******************************************************************************
+ @Function      PVRSRVDriverThreadExit
+ @Description   Decrements the number of client threads currently operating
+                in the driver's context to match the call to
+                PVRSRVDriverThreadEnter().
+                The function also signals the driver that a thread left the
+                driver context so if it's waiting to suspend it knows that
+                the number of threads decreased.
+ @Input         Reference to Connection data. NULL if no associated
+                 connection / device.
+******************************************************************************/
+void PVRSRVDriverThreadExit(void *pvData);
+
+#endif /* PVR_BRIDGE_K_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.c
new file mode 100644
index 000000000000..529cafd2bcf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.c
@@ -0,0 +1,733 @@
+/*
+ * @File
+ * @Title       Linux buffer sync interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/dma-buf.h>
+
+#include "services_kernel_client.h"
+#include "pvr_dma_resv.h"
+#include "pvr_buffer_sync.h"
+#include "pvr_buffer_sync_shared.h"
+#include "pvr_drv.h"
+#include "pvr_fence.h"
+
+struct pvr_buffer_sync_context {
+	struct mutex ctx_lock;
+	struct pvr_fence_context *fence_ctx;
+	struct ww_acquire_ctx acquire_ctx;
+};
+
+struct pvr_buffer_sync_check_data {
+	struct dma_fence_cb base;
+
+	u32 nr_fences;
+	struct pvr_fence **fences;
+};
+
+struct pvr_buffer_sync_append_data {
+	struct pvr_buffer_sync_context *ctx;
+
+	u32 nr_pmrs;
+	struct _PMR_ **pmrs;
+	u32 *pmr_flags;
+
+	struct pvr_fence *update_fence;
+	struct pvr_buffer_sync_check_data *check_data;
+};
+
+static struct dma_resv *
+pmr_reservation_object_get(struct _PMR_ *pmr)
+{
+	return PhysmemGetDmaResv(pmr);
+}
+
+static int
+pvr_buffer_sync_pmrs_lock(struct pvr_buffer_sync_context *ctx,
+			  u32 nr_pmrs,
+			  struct _PMR_ **pmrs)
+{
+	struct dma_resv *resv, *cresv = NULL, *lresv = NULL;
+	int i, err;
+	struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx;
+
+	mutex_lock(&ctx->ctx_lock);
+
+	ww_acquire_init(acquire_ctx, &reservation_ww_class);
+retry:
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (!resv) {
+			pr_err("%s: Failed to get reservation object from pmr %p\n",
+			       __func__, pmrs[i]);
+			err = -EINVAL;
+			goto fail;
+		}
+
+		if (resv != lresv) {
+			err = ww_mutex_lock_interruptible(&resv->lock,
+							  acquire_ctx);
+			if (err) {
+				cresv = (err == -EDEADLK) ? resv : NULL;
+				goto fail;
+			}
+		} else {
+			lresv = NULL;
+		}
+	}
+
+	ww_acquire_done(acquire_ctx);
+
+	return 0;
+
+fail:
+	while (i--) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+		ww_mutex_unlock(&resv->lock);
+	}
+
+	if (lresv)
+		ww_mutex_unlock(&lresv->lock);
+
+	if (cresv) {
+		err = ww_mutex_lock_slow_interruptible(&cresv->lock,
+						       acquire_ctx);
+		if (!err) {
+			lresv = cresv;
+			cresv = NULL;
+			goto retry;
+		}
+	}
+
+	ww_acquire_fini(acquire_ctx);
+
+	mutex_unlock(&ctx->ctx_lock);
+	return err;
+}
+
+static void
+pvr_buffer_sync_pmrs_unlock(struct pvr_buffer_sync_context *ctx,
+			    u32 nr_pmrs,
+			    struct _PMR_ **pmrs)
+{
+	struct dma_resv *resv;
+	int i;
+	struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+		ww_mutex_unlock(&resv->lock);
+	}
+
+	ww_acquire_fini(acquire_ctx);
+
+	mutex_unlock(&ctx->ctx_lock);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+
+static void
+dma_resv_count_fences(struct dma_resv *resv, u32 *read_fence_count_out, u32 *write_fence_count_out)
+{
+	struct dma_resv_iter cursor;
+	u32 write_fence_count = 0;
+	u32 read_fence_count = 0;
+	struct dma_fence *fence;
+
+	dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_READ);
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
+		if (dma_resv_iter_is_restarted(&cursor)) {
+			read_fence_count = 0;
+			write_fence_count = 0;
+		}
+		if (dma_resv_iter_usage(&cursor) == DMA_RESV_USAGE_READ)
+			read_fence_count++;
+		else if (dma_resv_iter_usage(&cursor) == DMA_RESV_USAGE_WRITE)
+			write_fence_count++;
+	}
+
+	*read_fence_count_out = read_fence_count;
+	*write_fence_count_out = write_fence_count;
+}
+
+static u32
+pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs,
+				 u32 *pmr_flags)
+{
+	struct dma_resv *resv;
+	u32 fence_count = 0;
+	bool exclusive;
+	int i;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		u32 write_fence_count = 0;
+		u32 read_fence_count = 0;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		dma_resv_count_fences(resv, &read_fence_count, &write_fence_count);
+
+		if (!exclusive || !read_fence_count)
+			fence_count += write_fence_count;
+		if (exclusive)
+			fence_count += read_fence_count;
+	}
+
+	return fence_count;
+}
+
+static struct pvr_buffer_sync_check_data *
+pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx,
+				    PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+				    u32 nr_pmrs,
+				    struct _PMR_ **pmrs,
+				    u32 *pmr_flags)
+{
+	struct pvr_buffer_sync_check_data *data;
+	struct dma_resv *resv;
+	struct dma_fence *fence;
+	u32 fence_count;
+	bool exclusive;
+	int i;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs,
+						       pmr_flags);
+	if (fence_count) {
+		data->fences = kcalloc(fence_count, sizeof(*data->fences),
+				       GFP_KERNEL);
+		if (!data->fences)
+			goto err_check_data_free;
+	}
+
+	for (i = 0; i < nr_pmrs; i++) {
+		struct dma_resv_iter cursor;
+		bool include_write_fences;
+		bool include_read_fences;
+		u32 write_fence_count = 0;
+		u32 read_fence_count = 0;
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		dma_resv_count_fences(resv, &read_fence_count, &write_fence_count);
+
+		include_write_fences = (!exclusive || !read_fence_count);
+		include_read_fences = exclusive;
+
+		dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_READ);
+		dma_resv_for_each_fence_unlocked(&cursor, fence) {
+			enum dma_resv_usage usage = dma_resv_iter_usage(&cursor);
+
+			if ((!include_write_fences && usage == DMA_RESV_USAGE_WRITE) ||
+			    (!include_read_fences && usage == DMA_RESV_USAGE_READ))
+				continue;
+
+			data->fences[data->nr_fences++] =
+				pvr_fence_create_from_fence(fence_ctx,
+							    sync_checkpoint_ctx,
+							    fence,
+							    PVRSRV_NO_FENCE,
+							    (usage == DMA_RESV_USAGE_WRITE) ?
+							    "write check fence" :
+							    "read check fence");
+			if (!data->fences[data->nr_fences - 1]) {
+				data->nr_fences--;
+				PVR_FENCE_TRACE(fence,
+						(usage == DMA_RESV_USAGE_WRITE) ?
+						"waiting on write fence" :
+						"waiting on read fence\n");
+				WARN_ON(dma_fence_wait(fence, true) <= 0);
+			}
+		}
+	}
+
+	WARN_ON(i != nr_pmrs);
+
+	return data;
+
+err_check_data_free:
+	kfree(data);
+	return NULL;
+}
+
+#else
+
+static u32
+pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs,
+				 u32 *pmr_flags)
+{
+	struct dma_resv *resv;
+	struct dma_resv_list *resv_list;
+	struct dma_fence *fence;
+	u32 fence_count = 0;
+	bool exclusive;
+	int i;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		resv_list = dma_resv_shared_list(resv);
+		fence = dma_resv_excl_fence(resv);
+
+		if (fence &&
+		    (!exclusive || !resv_list || !resv_list->shared_count))
+			fence_count++;
+
+		if (exclusive && resv_list)
+			fence_count += resv_list->shared_count;
+	}
+
+	return fence_count;
+}
+
+static struct pvr_buffer_sync_check_data *
+pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx,
+				    PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+				    u32 nr_pmrs,
+				    struct _PMR_ **pmrs,
+				    u32 *pmr_flags)
+{
+	struct pvr_buffer_sync_check_data *data;
+	struct dma_resv *resv;
+	struct dma_resv_list *resv_list;
+	struct dma_fence *fence;
+	u32 fence_count;
+	bool exclusive;
+	int i, j;
+	int err;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs,
+						       pmr_flags);
+	if (fence_count) {
+		data->fences = kcalloc(fence_count, sizeof(*data->fences),
+				       GFP_KERNEL);
+		if (!data->fences)
+			goto err_check_data_free;
+	}
+
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+		if (!exclusive) {
+			err = dma_resv_reserve_shared(resv
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+						      , 1
+#endif
+				);
+			if (err)
+				goto err_destroy_fences;
+		}
+
+		resv_list = dma_resv_shared_list(resv);
+		fence = dma_resv_excl_fence(resv);
+
+		if (fence &&
+		    (!exclusive || !resv_list || !resv_list->shared_count)) {
+			data->fences[data->nr_fences++] =
+				pvr_fence_create_from_fence(fence_ctx,
+							    sync_checkpoint_ctx,
+							    fence,
+							    PVRSRV_NO_FENCE,
+							    "exclusive check fence");
+			if (!data->fences[data->nr_fences - 1]) {
+				data->nr_fences--;
+				PVR_FENCE_TRACE(fence,
+						"waiting on exclusive fence\n");
+				WARN_ON(dma_fence_wait(fence, true) <= 0);
+			}
+		}
+
+		if (exclusive && resv_list) {
+			for (j = 0; j < resv_list->shared_count; j++) {
+				fence = rcu_dereference_protected(resv_list->shared[j],
+								  dma_resv_held(resv));
+				data->fences[data->nr_fences++] =
+					pvr_fence_create_from_fence(fence_ctx,
+								    sync_checkpoint_ctx,
+								    fence,
+								    PVRSRV_NO_FENCE,
+								    "check fence");
+				if (!data->fences[data->nr_fences - 1]) {
+					data->nr_fences--;
+					PVR_FENCE_TRACE(fence,
+							"waiting on non-exclusive fence\n");
+					WARN_ON(dma_fence_wait(fence, true) <= 0);
+				}
+			}
+		}
+	}
+
+	WARN_ON((i != nr_pmrs) || (data->nr_fences != fence_count));
+
+	return data;
+
+err_destroy_fences:
+	for (i = 0; i < data->nr_fences; i++)
+		pvr_fence_destroy(data->fences[i]);
+	kfree(data->fences);
+err_check_data_free:
+	kfree(data);
+	return NULL;
+}
+
+#endif
+
+static void
+pvr_buffer_sync_check_fences_destroy(struct pvr_buffer_sync_check_data *data)
+{
+	int i;
+
+	for (i = 0; i < data->nr_fences; i++)
+		pvr_fence_destroy(data->fences[i]);
+
+	kfree(data->fences);
+	kfree(data);
+}
+
+struct pvr_buffer_sync_context *
+pvr_buffer_sync_context_create(struct device *dev, const char *name)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+	struct pvr_buffer_sync_context *ctx;
+	int err;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	ctx->fence_ctx = pvr_fence_context_create(priv->dev_node,
+						  NativeSyncGetFenceStatusWq(),
+						  name);
+	if (!ctx->fence_ctx) {
+		err = -ENOMEM;
+		goto err_free_ctx;
+	}
+
+	mutex_init(&ctx->ctx_lock);
+
+	return ctx;
+
+err_free_ctx:
+	kfree(ctx);
+err_exit:
+	return ERR_PTR(err);
+}
+
+void
+pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx)
+{
+	pvr_fence_context_destroy(ctx->fence_ctx);
+	kfree(ctx);
+}
+
+int
+pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx,
+					  PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+					  u32 nr_pmrs,
+					  struct _PMR_ **pmrs,
+					  u32 *pmr_flags,
+					  u32 *nr_fence_checkpoints_out,
+					  PSYNC_CHECKPOINT **fence_checkpoints_out,
+					  PSYNC_CHECKPOINT *update_checkpoints_out,
+					  struct pvr_buffer_sync_append_data **data_out)
+{
+	struct pvr_buffer_sync_append_data *data;
+	PSYNC_CHECKPOINT *fence_checkpoints;
+	const size_t data_size = sizeof(*data);
+	const size_t pmrs_size = sizeof(*pmrs) * nr_pmrs;
+	const size_t pmr_flags_size = sizeof(*pmr_flags) * nr_pmrs;
+	int i;
+	int j;
+	int err;
+
+	if (unlikely((nr_pmrs && !(pmrs && pmr_flags)) ||
+	    !nr_fence_checkpoints_out || !fence_checkpoints_out ||
+	    !update_checkpoints_out))
+		return -EINVAL;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		if (unlikely(!(pmr_flags[i] & PVR_BUFFER_FLAG_MASK))) {
+			pr_err("%s: Invalid flags %#08x for pmr %p\n",
+			       __func__, pmr_flags[i], pmrs[i]);
+			return -EINVAL;
+		}
+	}
+
+#if defined(NO_HARDWARE)
+	/*
+	 * For NO_HARDWARE there's no checking or updating of sync checkpoints
+	 * which means SW waits on our fences will cause a deadlock (since they
+	 * will never be signalled). Avoid this by not creating any fences.
+	 */
+	nr_pmrs = 0;
+#endif
+
+	if (!nr_pmrs) {
+		*nr_fence_checkpoints_out = 0;
+		*fence_checkpoints_out = NULL;
+		*update_checkpoints_out = NULL;
+		*data_out = NULL;
+
+		return 0;
+	}
+
+	data = kzalloc(data_size + pmrs_size + pmr_flags_size, GFP_KERNEL);
+	if (unlikely(!data))
+		return -ENOMEM;
+
+	data->ctx = ctx;
+	data->pmrs = (struct _PMR_ **)(void *)(data + 1);
+	data->pmr_flags = (u32 *)(void *)(data->pmrs + nr_pmrs);
+
+	/*
+	 * It's expected that user space will provide a set of unique PMRs
+	 * but, as a PMR can have multiple handles, it's still possible to
+	 * end up here with duplicates. Take this opportunity to filter out
+	 * any remaining duplicates (updating flags when necessary) before
+	 * trying to process them further.
+	 */
+	for (i = 0; i < nr_pmrs; i++) {
+		for (j = 0; j < data->nr_pmrs; j++) {
+			if (data->pmrs[j] == pmrs[i]) {
+				data->pmr_flags[j] |= pmr_flags[i];
+				break;
+			}
+		}
+
+		if (j == data->nr_pmrs) {
+			data->pmrs[j] = pmrs[i];
+			data->pmr_flags[j] = pmr_flags[i];
+			data->nr_pmrs++;
+		}
+	}
+
+	err = pvr_buffer_sync_pmrs_lock(ctx, data->nr_pmrs, data->pmrs);
+	if (unlikely(err)) {
+		/*
+		 * -EINTR is returned if a signal arrives while trying to acquire a PMR
+		 * lock. In this case the operation should be retried after the signal
+		 * has been serviced. As this is expected behaviour, don't print an
+		 * error in this case.
+		 */
+		if (err != -EINTR) {
+			pr_err("%s: failed to lock pmrs (errno=%d)\n",
+			       __func__, err);
+		}
+		goto err_free_data;
+	}
+
+	/* create the check data */
+	data->check_data = pvr_buffer_sync_check_fences_create(ctx->fence_ctx,
+							 sync_checkpoint_ctx,
+							 data->nr_pmrs,
+							 data->pmrs,
+							 data->pmr_flags);
+	if (unlikely(!data->check_data)) {
+		err = -ENOMEM;
+		goto err_pmrs_unlock;
+	}
+
+	fence_checkpoints = kcalloc(data->check_data->nr_fences,
+				    sizeof(*fence_checkpoints),
+				    GFP_KERNEL);
+	if (fence_checkpoints) {
+		pvr_fence_get_checkpoints(data->check_data->fences,
+					  data->check_data->nr_fences,
+					  fence_checkpoints);
+	} else {
+		if (unlikely(data->check_data->nr_fences)) {
+			err = -ENOMEM;
+			goto err_free_check_data;
+		}
+	}
+
+	/* create the update fence */
+	data->update_fence = pvr_fence_create(ctx->fence_ctx,
+			sync_checkpoint_ctx,
+			SYNC_CHECKPOINT_FOREIGN_CHECKPOINT, "update fence");
+	if (unlikely(!data->update_fence)) {
+		err = -ENOMEM;
+		goto err_free_fence_checkpoints;
+	}
+
+	/*
+	 * We need to clean up the fences once the HW has finished with them.
+	 * We can do this using fence callbacks. However, instead of adding a
+	 * callback to every fence, which would result in more work, we can
+	 * simply add one to the update fence since this will be the last fence
+	 * to be signalled. This callback can do all the necessary clean up.
+	 *
+	 * Note: we take an additional reference on the update fence in case
+	 * it signals before we can add it to a reservation object.
+	 */
+	PVR_FENCE_TRACE(&data->update_fence->base,
+			"create fence calling dma_fence_get\n");
+	dma_fence_get(&data->update_fence->base);
+
+	*nr_fence_checkpoints_out = data->check_data->nr_fences;
+	*fence_checkpoints_out = fence_checkpoints;
+	*update_checkpoints_out = pvr_fence_get_checkpoint(data->update_fence);
+	*data_out = data;
+
+	return 0;
+
+err_free_fence_checkpoints:
+	kfree(fence_checkpoints);
+err_free_check_data:
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+err_pmrs_unlock:
+	pvr_buffer_sync_pmrs_unlock(ctx, data->nr_pmrs, data->pmrs);
+err_free_data:
+	kfree(data);
+	return err;
+}
+
+void
+pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data)
+{
+	struct dma_resv *resv;
+	int i;
+
+	dma_fence_enable_sw_signaling(&data->update_fence->base);
+
+	for (i = 0; i < data->nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(data->pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+		dma_resv_reserve_fences(resv, 1);
+#endif
+		if (data->pmr_flags[i] & PVR_BUFFER_FLAG_WRITE) {
+			PVR_FENCE_TRACE(&data->update_fence->base,
+					"added exclusive fence (%s) to resv %p\n",
+					data->update_fence->name, resv);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+			dma_resv_add_fence(resv,
+					   &data->update_fence->base,
+					   DMA_RESV_USAGE_WRITE);
+#else
+			dma_resv_add_excl_fence(resv,
+						&data->update_fence->base);
+#endif
+		} else if (data->pmr_flags[i] & PVR_BUFFER_FLAG_READ) {
+			PVR_FENCE_TRACE(&data->update_fence->base,
+					"added non-exclusive fence (%s) to resv %p\n",
+					data->update_fence->name, resv);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+			dma_resv_add_fence(resv,
+					   &data->update_fence->base,
+					   DMA_RESV_USAGE_READ);
+#else
+			dma_resv_add_shared_fence(resv,
+						  &data->update_fence->base);
+#endif
+		}
+	}
+
+	/*
+	 * Now that the fence has been added to the necessary
+	 * reservation objects we can safely drop the extra reference
+	 * we took in pvr_buffer_sync_resolve_and_create_fences().
+	 */
+	dma_fence_put(&data->update_fence->base);
+	pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs,
+					data->pmrs);
+
+	/* destroy the check fences */
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+	/* destroy the update fence */
+	pvr_fence_destroy(data->update_fence);
+
+	/* free the append data */
+	kfree(data);
+}
+
+void
+pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data)
+{
+
+	/* drop the extra reference we took on the update fence in
+	 * pvr_buffer_sync_resolve_and_create_fences().
+	 */
+	dma_fence_put(&data->update_fence->base);
+
+	if (data->nr_pmrs > 0)
+		pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs,
+					    data->pmrs);
+
+	/* destroy the check fences */
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+	/* destroy the update fence */
+	pvr_fence_destroy(data->update_fence);
+
+	/* free the append data */
+	kfree(data);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.h
new file mode 100644
index 000000000000..b6aadf940479
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.h
@@ -0,0 +1,125 @@
+/*
+ * @File        pvr_buffer_sync.h
+ * @Title       PowerVR Linux buffer sync interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_BUFFER_SYNC_H
+#define PVR_BUFFER_SYNC_H
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/types.h>
+
+struct _PMR_;
+struct pvr_buffer_sync_context;
+struct pvr_buffer_sync_append_data;
+
+/**
+ * pvr_buffer_sync_context_create - creates a buffer sync context
+ * @dev: Linux device
+ * @name: context name (used for debugging)
+ *
+ * pvr_buffer_sync_context_destroy() should be used to clean up the buffer
+ * sync context.
+ *
+ * Return: A buffer sync context or NULL if it fails for any reason.
+ */
+struct pvr_buffer_sync_context *
+pvr_buffer_sync_context_create(struct device *dev, const char *name);
+
+/**
+ * pvr_buffer_sync_context_destroy() - frees a buffer sync context
+ * @ctx: buffer sync context
+ */
+void
+pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx);
+
+/**
+ * pvr_buffer_sync_resolve_and_create_fences() - create checkpoints from
+ *                                               buffers
+ * @ctx: buffer sync context
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @nr_pmrs: number of buffer objects (PMRs)
+ * @pmrs: buffer array
+ * @pmr_flags: internal flags
+ * @nr_fence_checkpoints_out: returned number of fence sync checkpoints
+ * @fence_checkpoints_out: returned array of fence sync checkpoints
+ * @update_checkpoint_out: returned update sync checkpoint
+ * @data_out: returned buffer sync data
+ *
+ * After this call, either pvr_buffer_sync_kick_succeeded() or
+ * pvr_buffer_sync_kick_failed() must be called.
+ *
+ * Return: 0 on success or an error code otherwise.
+ */
+int
+pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx,
+					  PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+					  u32 nr_pmrs,
+					  struct _PMR_ **pmrs,
+					  u32 *pmr_flags,
+					  u32 *nr_fence_checkpoints_out,
+					  PSYNC_CHECKPOINT **fence_checkpoints_out,
+					  PSYNC_CHECKPOINT *update_checkpoint_out,
+					  struct pvr_buffer_sync_append_data **data_out);
+
+/**
+ * pvr_buffer_sync_kick_succeeded() - cleans up after a successful kick
+ *                                    operation
+ * @data: buffer sync data returned by
+ *        pvr_buffer_sync_resolve_and_create_fences()
+ *
+ * Should only be called following pvr_buffer_sync_resolve_and_create_fences().
+ */
+void
+pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data);
+
+/**
+ * pvr_buffer_sync_kick_failed() - cleans up after a failed kick operation
+ * @data: buffer sync data returned by
+ *        pvr_buffer_sync_resolve_and_create_fences()
+ *
+ * Should only be called following pvr_buffer_sync_resolve_and_create_fences().
+ */
+void
+pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data);
+
+#endif /* PVR_BUFFER_SYNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.c
new file mode 100644
index 000000000000..240872e0b0fd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.c
@@ -0,0 +1,308 @@
+/*
+ * @File
+ * @Title       PowerVR Linux software "counting" timeline fence implementation
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/kref.h>
+
+#include "services_kernel_client.h"
+#include "pvr_counting_timeline.h"
+#include "pvr_sw_fence.h"
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+struct pvr_counting_fence_timeline {
+	struct pvr_sw_fence_context *context;
+
+	void *dbg_request_handle;
+
+	spinlock_t active_fences_lock;
+	u64 current_value; /* guarded by active_fences_lock */
+	u64 next_value; /* guarded by active_fences_lock */
+	struct list_head active_fences;
+
+	struct kref kref;
+};
+
+struct pvr_counting_fence {
+	u64 value;
+	struct dma_fence *fence;
+	struct list_head active_list_entry;
+};
+
+void pvr_counting_fence_timeline_dump_timeline(
+	void *data,
+	DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+	void *dump_debug_file)
+{
+
+	struct pvr_counting_fence_timeline *timeline =
+		(struct pvr_counting_fence_timeline *) data;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "TL:%s SeqNum: %llu/%llu",
+					  pvr_sw_fence_context_name(
+							  timeline->context),
+					  timeline->current_value,
+					  timeline->next_value);
+
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+}
+
+static void
+pvr_counting_fence_timeline_debug_request(void *data, u32 verbosity,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	struct pvr_counting_fence_timeline *timeline =
+		(struct pvr_counting_fence_timeline *)data;
+	struct pvr_counting_fence *obj;
+	unsigned long flags;
+	char value[128];
+
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM)) {
+		spin_lock_irqsave(&timeline->active_fences_lock, flags);
+		pvr_sw_fence_context_value_str(timeline->context, value,
+					       sizeof(value));
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  "sw: %s @%s cur=%llu",
+				  pvr_sw_fence_context_name(timeline->context),
+				  value, timeline->current_value);
+		list_for_each_entry(obj, &timeline->active_fences,
+				    active_list_entry) {
+			obj->fence->ops->fence_value_str(obj->fence,
+							 value, sizeof(value));
+			PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+					  " @%s: val=%llu", value, obj->value);
+		}
+		spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+	}
+}
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_create(
+	const char *name)
+{
+	PVRSRV_ERROR srv_err;
+	struct pvr_counting_fence_timeline *timeline =
+		kzalloc(sizeof(*timeline), GFP_KERNEL);
+
+	if (!timeline)
+		goto err_out;
+
+	timeline->context = pvr_sw_fence_context_create(name,
+							"pvr_sw_sync");
+	if (!timeline->context)
+		goto err_free_timeline;
+
+	timeline->current_value = 0;
+	timeline->next_value = 1;
+	kref_init(&timeline->kref);
+	spin_lock_init(&timeline->active_fences_lock);
+	INIT_LIST_HEAD(&timeline->active_fences);
+
+	srv_err = PVRSRVRegisterDriverDbgRequestNotify(
+				&timeline->dbg_request_handle,
+				pvr_counting_fence_timeline_debug_request,
+				DEBUG_REQUEST_LINUXFENCE,
+				timeline);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+			   __func__, PVRSRVGetErrorString(srv_err));
+		goto err_free_timeline_ctx;
+	}
+
+err_out:
+	return timeline;
+
+err_free_timeline_ctx:
+	pvr_sw_fence_context_destroy(timeline->context);
+
+err_free_timeline:
+	kfree(timeline);
+	timeline = NULL;
+	goto err_out;
+}
+
+void pvr_counting_fence_timeline_force_complete(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	struct list_head *entry, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+#if defined(DEBUG) && !defined(SUPPORT_AUTOVZ)
+	/* This is just a safety measure. Normally we should never see any
+	 * unsignaled sw fences when we come here. Warn if we still do!
+	 */
+	WARN_ON(!list_empty(&timeline->active_fences));
+#endif
+
+	list_for_each_safe(entry, tmp, &timeline->active_fences) {
+		struct pvr_counting_fence *fence =
+			list_entry(entry, struct pvr_counting_fence,
+			active_list_entry);
+		dma_fence_signal(fence->fence);
+		dma_fence_put(fence->fence);
+		fence->fence = NULL;
+		list_del(&fence->active_list_entry);
+		kfree(fence);
+	}
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+}
+
+static void pvr_counting_fence_timeline_destroy(
+	struct kref *kref)
+{
+	struct pvr_counting_fence_timeline *timeline =
+		container_of(kref, struct pvr_counting_fence_timeline, kref);
+
+	WARN_ON(!list_empty(&timeline->active_fences));
+
+	PVRSRVUnregisterDriverDbgRequestNotify(timeline->dbg_request_handle);
+
+	pvr_sw_fence_context_destroy(timeline->context);
+	kfree(timeline);
+}
+
+void pvr_counting_fence_timeline_put(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	kref_put(&timeline->kref, pvr_counting_fence_timeline_destroy);
+}
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_get(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	if (!timeline)
+		return NULL;
+	kref_get(&timeline->kref);
+	return timeline;
+}
+
+struct dma_fence *pvr_counting_fence_create(
+	struct pvr_counting_fence_timeline *timeline, u64 *sync_pt_idx)
+{
+	unsigned long flags;
+	struct dma_fence *sw_fence;
+	struct pvr_counting_fence *fence = kmalloc(sizeof(*fence), GFP_KERNEL);
+
+	if (!fence)
+		return NULL;
+
+	sw_fence = pvr_sw_fence_create(timeline->context);
+	if (!sw_fence)
+		goto err_free_fence;
+
+	fence->fence = dma_fence_get(sw_fence);
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	fence->value = timeline->next_value++;
+	if (sync_pt_idx)
+		*sync_pt_idx = fence->value;
+
+	list_add_tail(&fence->active_list_entry, &timeline->active_fences);
+
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+
+	/* Counting fences can be signalled any time after creation */
+	dma_fence_enable_sw_signaling(sw_fence);
+
+	return sw_fence;
+
+err_free_fence:
+	kfree(fence);
+	return NULL;
+}
+
+bool pvr_counting_fence_timeline_inc(
+	struct pvr_counting_fence_timeline *timeline, u64 *sync_pt_idx)
+{
+	struct list_head *entry, *tmp;
+	unsigned long flags;
+	bool res;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	if (timeline->current_value == timeline->next_value-1) {
+		res = false;
+		goto exit_unlock;
+	}
+
+	timeline->current_value++;
+
+	if (sync_pt_idx)
+		*sync_pt_idx = timeline->current_value;
+
+	list_for_each_safe(entry, tmp, &timeline->active_fences) {
+		struct pvr_counting_fence *fence =
+			list_entry(entry, struct pvr_counting_fence,
+			active_list_entry);
+		if (fence->value <= timeline->current_value) {
+			dma_fence_signal(fence->fence);
+			dma_fence_put(fence->fence);
+			fence->fence = NULL;
+			list_del(&fence->active_list_entry);
+			kfree(fence);
+		}
+	}
+
+	res = true;
+
+exit_unlock:
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+
+	return res;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.h
new file mode 100644
index 000000000000..2cb8db1e5c50
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.h
@@ -0,0 +1,68 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_COUNTING_TIMELINE_H__)
+#define __PVR_COUNTING_TIMELINE_H__
+
+#include "pvr_linux_fence.h"
+
+struct pvr_counting_fence_timeline;
+
+void pvr_counting_fence_timeline_dump_timeline(
+	void *data,
+	DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+	void *dump_debug_file);
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_create(
+	const char *name);
+void pvr_counting_fence_timeline_put(
+	struct pvr_counting_fence_timeline *fence_timeline);
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_get(
+	struct pvr_counting_fence_timeline *fence_timeline);
+struct dma_fence *pvr_counting_fence_create(
+	struct pvr_counting_fence_timeline *fence_timeline, u64 *sync_pt_idx);
+bool pvr_counting_fence_timeline_inc(
+	struct pvr_counting_fence_timeline *fence_timeline, u64 *sync_pt_idx);
+void pvr_counting_fence_timeline_force_complete(
+	struct pvr_counting_fence_timeline *fence_timeline);
+
+#endif /* !defined(__PVR_COUNTING_TIMELINE_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debug.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debug.c
new file mode 100644
index 000000000000..cf59a2d5aa3e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debug.c
@@ -0,0 +1,487 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides kernel side Debug Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/sched.h>
+#include <linux/pid.h>
+#include <linux/moduleparam.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "linkage.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "di_server.h"
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/******** BUFFERED LOG MESSAGES ********/
+
+/* Because we don't want to have to handle CCB wrapping, each buffered
+ * message is rounded up to PVRSRV_DEBUG_CCB_MESG_MAX bytes. This means
+ * there is the same fixed number of messages that can be stored,
+ * regardless of message length.
+ */
+
+#if defined(PVRSRV_DEBUG_CCB_MAX)
+
+#define PVRSRV_DEBUG_CCB_MESG_MAX	PVR_MAX_DEBUG_MESSAGE_LEN
+
+typedef struct
+{
+	const IMG_CHAR *pszFile;
+	IMG_INT iLine;
+	IMG_UINT32 ui32TID;
+	IMG_UINT32 ui32PID;
+	IMG_CHAR pcMesg[PVRSRV_DEBUG_CCB_MESG_MAX];
+	struct timeval sTimeVal;
+}
+PVRSRV_DEBUG_CCB;
+
+static PVRSRV_DEBUG_CCB gsDebugCCB[PVRSRV_DEBUG_CCB_MAX];
+
+static IMG_UINT giOffset;
+
+/* protects access to gsDebugCCB */
+static DEFINE_SPINLOCK(gsDebugCCBLock);
+
+static void
+AddToBufferCCB(const IMG_CHAR *pszFileName, IMG_UINT32 ui32Line,
+			   const IMG_CHAR *szBuffer)
+{
+	unsigned long uiFlags;
+
+	spin_lock_irqsave(&gsDebugCCBLock, uiFlags);
+
+	gsDebugCCB[giOffset].pszFile = pszFileName;
+	gsDebugCCB[giOffset].iLine   = ui32Line;
+	gsDebugCCB[giOffset].ui32TID = current->pid;
+	gsDebugCCB[giOffset].ui32PID = current->tgid;
+
+	do_gettimeofday(&gsDebugCCB[giOffset].sTimeVal);
+
+	OSStringSafeCopy(gsDebugCCB[giOffset].pcMesg, szBuffer,
+	              PVRSRV_DEBUG_CCB_MESG_MAX);
+
+	giOffset = (giOffset + 1) % PVRSRV_DEBUG_CCB_MAX;
+
+	spin_unlock_irqrestore(&gsDebugCCBLock, uiFlags);
+}
+
+void PVRSRVDebugPrintfDumpCCB(void)
+{
+	int i;
+	unsigned long uiFlags;
+
+	spin_lock_irqsave(&gsDebugCCBLock, uiFlags);
+
+	for (i = 0; i < PVRSRV_DEBUG_CCB_MAX; i++)
+	{
+		PVRSRV_DEBUG_CCB *psDebugCCBEntry =
+			&gsDebugCCB[(giOffset + i) % PVRSRV_DEBUG_CCB_MAX];
+
+		/* Early on, we won't have PVRSRV_DEBUG_CCB_MAX messages */
+		if (!psDebugCCBEntry->pszFile)
+		{
+			continue;
+		}
+
+		printk(KERN_ERR "%s:%d: (%ld.%ld, tid=%u, pid=%u) %s\n",
+			   psDebugCCBEntry->pszFile,
+			   psDebugCCBEntry->iLine,
+			   (long)psDebugCCBEntry->sTimeVal.tv_sec,
+			   (long)psDebugCCBEntry->sTimeVal.tv_usec,
+			   psDebugCCBEntry->ui32TID,
+			   psDebugCCBEntry->ui32PID,
+			   psDebugCCBEntry->pcMesg);
+
+		/* Clear this entry so it doesn't get printed the next time again. */
+		psDebugCCBEntry->pszFile = NULL;
+	}
+
+	spin_unlock_irqrestore(&gsDebugCCBLock, uiFlags);
+}
+
+#else /* defined(PVRSRV_DEBUG_CCB_MAX) */
+
+static INLINE void
+AddToBufferCCB(const IMG_CHAR *pszFileName, IMG_UINT32 ui32Line,
+			   const IMG_CHAR *szBuffer)
+{
+	(void)pszFileName;
+	(void)szBuffer;
+	(void)ui32Line;
+}
+
+void PVRSRVDebugPrintfDumpCCB(void)
+{
+	/* Not available */
+}
+
+#endif /* defined(PVRSRV_DEBUG_CCB_MAX) */
+
+static IMG_UINT32 PVRDebugLevel =
+	(
+	 DBGPRIV_FATAL | DBGPRIV_ERROR | DBGPRIV_WARNING
+#if defined(PVRSRV_DEBUG_CCB_MAX)
+	 | DBGPRIV_BUFFERED
+#endif /* defined(PVRSRV_DEBUG_CCB_MAX) */
+#if defined(PVR_DPF_ADHOC_DEBUG_ON)
+	 | DBGPRIV_DEBUG
+#endif /* defined(PVR_DPF_ADHOC_DEBUG_ON) */
+	);
+
+module_param(PVRDebugLevel, uint, 0644);
+MODULE_PARM_DESC(PVRDebugLevel,
+                 "Sets the level of debug output (default 0x7)");
+
+IMG_UINT32 OSDebugLevel(void)
+{
+	return PVRDebugLevel;
+}
+
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVRDebugLevel = ui32DebugLevel;
+}
+
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	return (PVRDebugLevel & ui32DebugLevel) != 0;
+}
+
+#else /* defined(PVRSRV_NEED_PVR_DPF) */
+
+IMG_UINT32 OSDebugLevel(void)
+{
+	return 0;
+}
+
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32DebugLevel);
+}
+
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32DebugLevel);
+	return IMG_FALSE;
+}
+
+#endif /* defined(PVRSRV_NEED_PVR_DPF) */
+
+#define	PVR_MAX_MSG_LEN PVR_MAX_DEBUG_MESSAGE_LEN
+
+/* Message buffer for messages */
+static IMG_CHAR gszBuffer[PVR_MAX_MSG_LEN + 1];
+
+/* The lock is used to control access to gszBuffer */
+static DEFINE_SPINLOCK(gsDebugLock);
+
+/*
+ * Append a string to a buffer using formatted conversion.
+ * The function takes a variable number of arguments, pointed
+ * to by the var args list.
+ */
+__printf(3, 0)
+static IMG_BOOL VBAppend(IMG_CHAR *pszBuf, IMG_UINT32 ui32BufSiz, const IMG_CHAR *pszFormat, va_list VArgs)
+{
+	IMG_UINT32 ui32Used;
+	IMG_UINT32 ui32Space;
+	IMG_INT32 i32Len;
+
+	ui32Used = OSStringLength(pszBuf);
+	BUG_ON(ui32Used >= ui32BufSiz);
+	ui32Space = ui32BufSiz - ui32Used;
+
+	i32Len = vsnprintf(&pszBuf[ui32Used], ui32Space, pszFormat, VArgs);
+	pszBuf[ui32BufSiz - 1] = 0;
+
+	/* Return true if string was truncated */
+	return i32Len < 0 || i32Len >= (IMG_INT32)ui32Space;
+}
+
+void PVRSRVReleasePrintfVArgs(const IMG_CHAR *pszFormat, va_list vaArgs)
+{
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+	IMG_INT32  result;
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	result = snprintf(pszBuf, (ui32BufSiz - 2), "PVR_K:  %u: ", current->pid);
+	PVR_ASSERT(result>0);
+	ui32BufSiz -= result;
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, vaArgs))
+	{
+		printk(KERN_INFO "%s (truncated)\n", pszBuf);
+	}
+	else
+	{
+		printk(KERN_INFO "%s\n", pszBuf);
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVReleasePrintf
+@Description    To output an important message to the user in release builds
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVReleasePrintf(const IMG_CHAR *pszFormat, ...)
+{
+	va_list vaArgs;
+
+	va_start(vaArgs, pszFormat);
+	PVRSRVReleasePrintfVArgs(pszFormat, vaArgs);
+	va_end(vaArgs);
+}
+
+#if defined(PVRSRV_NEED_PVR_TRACE)
+
+/*************************************************************************/ /*!
+@Function       PVRTrace
+@Description    To output a debug message to the user
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVTrace(const IMG_CHAR *pszFormat, ...)
+{
+	va_list VArgs;
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+	IMG_INT32  result;
+
+	va_start(VArgs, pszFormat);
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	result = snprintf(pszBuf, (ui32BufSiz - 2), "PVR: %u: ", current->pid);
+	PVR_ASSERT(result>0);
+	ui32BufSiz -= result;
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, VArgs))
+	{
+		printk(KERN_ERR "PVR_K:(Message Truncated): %s\n", pszBuf);
+	}
+	else
+	{
+		printk(KERN_ERR "%s\n", pszBuf);
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+
+	va_end(VArgs);
+}
+
+#endif /* defined(PVRSRV_NEED_PVR_TRACE) */
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/*
+ * Append a string to a buffer using formatted conversion.
+ * The function takes a variable number of arguments, calling
+ * VBAppend to do the actual work.
+ */
+__printf(3, 4)
+static IMG_BOOL BAppend(IMG_CHAR *pszBuf, IMG_UINT32 ui32BufSiz, const IMG_CHAR *pszFormat, ...)
+{
+	va_list VArgs;
+	IMG_BOOL bTrunc;
+
+	va_start (VArgs, pszFormat);
+
+	bTrunc = VBAppend(pszBuf, ui32BufSiz, pszFormat, VArgs);
+
+	va_end (VArgs);
+
+	return bTrunc;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintf
+@Description    To output a debug message to the user
+@Input          uDebugLevel The current debug level
+@Input          pszFile     The source file generating the message
+@Input          uLine       The line of the source file
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVDebugPrintf(IMG_UINT32 ui32DebugLevel,
+			   const IMG_CHAR *pszFullFileName,
+			   IMG_UINT32 ui32Line,
+			   const IMG_CHAR *pszFormat,
+			   ...)
+{
+	const IMG_CHAR *pszFileName = pszFullFileName;
+	IMG_CHAR *pszLeafName;
+	va_list vaArgs;
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+
+	if (!(PVRDebugLevel & ui32DebugLevel))
+	{
+		return;
+	}
+
+	va_start(vaArgs, pszFormat);
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	switch (ui32DebugLevel)
+	{
+		case DBGPRIV_FATAL:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Fatal): ", ui32BufSiz);
+			PVRSRV_REPORT_ERROR();
+			break;
+		}
+		case DBGPRIV_ERROR:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Error): ", ui32BufSiz);
+			PVRSRV_REPORT_ERROR();
+			break;
+		}
+		case DBGPRIV_WARNING:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Warn):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_MESSAGE:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Mesg):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_VERBOSE:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Verb):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_DEBUG:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Debug): ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_CALLTRACE:
+		case DBGPRIV_ALLOC:
+		case DBGPRIV_BUFFERED:
+		default:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K: ", ui32BufSiz);
+			break;
+		}
+	}
+
+	if (current->pid == task_tgid_nr(current))
+	{
+		(void) BAppend(pszBuf, ui32BufSiz, "%5u: ", current->pid);
+	}
+	else
+	{
+		(void) BAppend(pszBuf, ui32BufSiz, "%5u-%5u: ", task_tgid_nr(current) /* pid id of group*/, current->pid /* task id */);
+	}
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, vaArgs))
+	{
+		printk(KERN_ERR "%s (truncated)\n", pszBuf);
+	}
+	else
+	{
+		IMG_BOOL bTruncated = IMG_FALSE;
+
+#if !defined(__sh__)
+		pszLeafName = (IMG_CHAR *)strrchr (pszFileName, '/');
+
+		if (pszLeafName)
+		{
+			pszFileName = pszLeafName+1;
+		}
+#endif /* __sh__ */
+
+#if defined(DEBUG)
+		{
+			static const IMG_CHAR *lastFile;
+
+			if (lastFile == pszFileName)
+			{
+				bTruncated = BAppend(pszBuf, ui32BufSiz, " [%u]", ui32Line);
+			}
+			else
+			{
+				bTruncated = BAppend(pszBuf, ui32BufSiz, " [%s:%u]", pszFileName, ui32Line);
+				lastFile = pszFileName;
+			}
+		}
+#else
+		bTruncated = BAppend(pszBuf, ui32BufSiz, " [%u]", ui32Line);
+#endif
+
+		if (bTruncated)
+		{
+			printk(KERN_ERR "%s (truncated)\n", pszBuf);
+		}
+		else
+		{
+			if (ui32DebugLevel & DBGPRIV_BUFFERED)
+			{
+				AddToBufferCCB(pszFileName, ui32Line, pszBuf);
+			}
+			else
+			{
+				printk(KERN_ERR "%s\n", pszBuf);
+			}
+		}
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+
+	va_end (vaArgs);
+}
+
+#endif /* PVRSRV_NEED_PVR_DPF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.c
new file mode 100644
index 000000000000..63c557bc8459
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.c
@@ -0,0 +1,611 @@
+/*************************************************************************/ /*!
+@File
+@Title          DebugFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via DebugFS.
+
+                Note about locking in DebugFS module.
+
+                Access to DebugFS is protected against the race where any
+                file could be removed while being accessed or accessed while
+                being removed. Any calls to debugfs_remove() will block
+                until all operations are finished.
+
+                See implementation of proxy file operations (FULL_PROXY_FUNC)
+                and implementation of debugfs_file_[get|put]() in
+                fs/debugfs/file.c in Linux kernel sources for more details.
+
+                Not about locking for sequential files.
+
+                The seq_file objects have a mutex that protects access
+                to all of the file operations hence all of the sequential
+                *read* operations are protected.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvr_debugfs.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_bridge_k.h"
+#include "pvr_uaccess.h"
+#include "osdi_impl.h"
+
+#define _DRIVER_THREAD_ENTER() \
+	do { \
+		PVRSRV_ERROR eLocalError = PVRSRVDriverThreadEnter(NULL); \
+		if (eLocalError != PVRSRV_OK) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDriverThreadEnter failed: %s", \
+				__func__, PVRSRVGetErrorString(eLocalError))); \
+			return OSPVRSRVToNativeError(eLocalError); \
+		} \
+	} while (0)
+
+#define _DRIVER_THREAD_EXIT() \
+	PVRSRVDriverThreadExit(NULL)
+
+#define PVR_DEBUGFS_PVR_DPF_LEVEL PVR_DBG_ERROR
+
+typedef struct DFS_DIR
+{
+	struct dentry *psDirEntry;
+	struct DFS_DIR *psParentDir;
+} DFS_DIR;
+
+typedef struct DFS_ENTRY
+{
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+} DFS_ENTRY;
+
+typedef struct DFS_FILE
+{
+	struct dentry *psFileEntry;
+	struct DFS_DIR *psParentDir;
+	const struct seq_operations *psSeqOps;
+	struct DFS_ENTRY sEntry;
+	DI_ENTRY_TYPE eType;
+} DFS_FILE;
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	seq_write(pvNativeHandle, pvData, uiSize);
+}
+
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	seq_vprintf(pvNativeHandle, pszFmt, pArgs);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	seq_puts(pvNativeHandle, pszStr);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	struct seq_file *psSeqFile = pvNativeHandle;
+	return seq_has_overflowed(psSeqFile);
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- sequential file operations ----------------------------------------- */
+
+static void *_Start(struct seq_file *psSeqFile, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	void *pvRet = psEntry->sIterCb.pfnStart(&psEntry->sImplEntry, puiPos);
+
+	if (pvRet == DI_START_TOKEN)
+	{
+		return SEQ_START_TOKEN;
+	}
+
+	return pvRet;
+}
+
+static void _Stop(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	psEntry->sIterCb.pfnStop(&psEntry->sImplEntry, pvPriv);
+}
+
+static void *_Next(struct seq_file *psSeqFile, void *pvPriv, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	return psEntry->sIterCb.pfnNext(&psEntry->sImplEntry, pvPriv, puiPos);
+}
+
+static int _Show(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	if (pvPriv == SEQ_START_TOKEN)
+	{
+		pvPriv = DI_START_TOKEN;
+	}
+
+	return psEntry->sIterCb.pfnShow(&psEntry->sImplEntry, pvPriv);
+}
+
+static struct seq_operations _g_sSeqOps = {
+	.start = _Start,
+	.stop = _Stop,
+	.next = _Next,
+	.show = _Show
+};
+
+/* ----- file operations ---------------------------------------------------- */
+
+static int _Open(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psINode != NULL && psINode->i_private != NULL,
+	                        "psDFSFile is NULL", -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	psDFSFile = psINode->i_private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_open(psFile, psDFSFile->psSeqOps);
+	}
+	else
+	{
+		/* private data is NULL as it's going to be set below */
+		iRes = single_open(psFile, _Show, NULL);
+	}
+
+	if (iRes == 0)
+	{
+		struct seq_file *psSeqFile = psFile->private_data;
+
+		DFS_ENTRY *psEntry = OSAllocMem(sizeof(*psEntry));
+		if (psEntry == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem() failed", __func__));
+			iRes = -ENOMEM;
+			goto return_;
+		}
+
+		*psEntry = psDFSFile->sEntry;
+		psSeqFile->private = psEntry;
+		psEntry->sImplEntry.pvNative = psSeqFile;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to seq_open psFile, returning %d",
+		        __func__, iRes));
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static int _Close(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = psINode->i_private;
+	DFS_ENTRY *psEntry;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* save pointer to DFS_ENTRY */
+	psEntry = ((struct seq_file *) psFile->private_data)->private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_release(psINode, psFile);
+	}
+	else
+	{
+		iRes = single_release(psINode, psFile);
+	}
+
+	/* free DFS_ENTRY allocated in _Open */
+	OSFreeMem(psEntry);
+
+	/* Validation check as seq_release (and single_release which calls it)
+	 * never fail */
+	if (iRes != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release psFile, returning %d",
+		        __func__, iRes));
+	}
+
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Read(struct file *psFile, char __user *pcBuffer,
+                     size_t uiCount, loff_t *puiPos)
+{
+	DFS_FILE *psDFSFile = psFile->f_path.dentry->d_inode->i_private;
+	ssize_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_read(psFile, pcBuffer, uiCount, puiPos);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file, pfnRead() "
+			        "returned %zd", __func__, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Count = uiCount;
+
+		IMG_CHAR *pcLocalBuffer = OSAllocMem(uiCount);
+		PVR_GOTO_IF_FALSE(pcLocalBuffer != NULL, return_);
+
+		iRes = psEntry->sIterCb.pfnRead(pcLocalBuffer, ui64Count, puiPos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file, pfnRead() "
+			        "returned %zd", __func__, iRes));
+			OSFreeMem(pcLocalBuffer);
+			goto return_;
+		}
+
+		if (pvr_copy_to_user(pcBuffer, pcLocalBuffer, iRes) != 0)
+		{
+			iRes = -1;
+		}
+
+		OSFreeMem(pcLocalBuffer);
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static loff_t _LSeek(struct file *psFile, loff_t iOffset, int iOrigin)
+{
+	DFS_FILE *psDFSFile = psFile->f_path.dentry->d_inode->i_private;
+	loff_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_lseek(psFile, iOffset, iOrigin);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position in psFile<%p> to offset "
+			        "%lld, iOrigin %d, seq_lseek() returned %lld (dentry='%s')", __func__,
+			        psFile, iOffset, iOrigin, iRes, psFile->f_path.dentry->d_name.name));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Pos;
+
+		switch (iOrigin)
+		{
+			case SEEK_SET:
+				ui64Pos = psFile->f_pos + iOffset;
+				break;
+			case SEEK_CUR:
+				ui64Pos = iOffset;
+				break;
+			case SEEK_END:
+				/* not supported as we don't know the file size here */
+				/* fall through */
+			default:
+				return -1;
+		}
+
+		/* only pass the absolute position to the callback, it's up to the
+		 * implementer to determine if the position is valid */
+
+		iRes = psEntry->sIterCb.pfnSeek(ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to offset "
+			        "%lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+
+		psFile->f_pos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Write(struct file *psFile, const char __user *pszBuffer,
+                      size_t uiCount, loff_t *puiPos)
+{
+	struct inode *psINode = psFile->f_path.dentry->d_inode;
+	DFS_FILE *psDFSFile = psINode->i_private;
+	DI_ITERATOR_CB *psIter = &psDFSFile->sEntry.sIterCb;
+	IMG_CHAR *pcLocalBuffer;
+	IMG_UINT64 ui64Count;
+	IMG_INT64 i64Res = -EIO;
+	IMG_UINT64 ui64Pos = *puiPos;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+	PVR_LOG_RETURN_IF_FALSE(psIter->pfnWrite != NULL, "pfnWrite is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* Make sure we allocate the smallest amount of needed memory*/
+	ui64Count = psIter->ui32WriteLenMax;
+	PVR_LOG_GOTO_IF_FALSE(uiCount <= ui64Count, "uiCount too long", return_);
+	ui64Count = MIN(uiCount + 1, ui64Count);
+
+	/* allocate buffer with one additional byte for NUL character */
+	pcLocalBuffer = OSAllocMem(ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(pcLocalBuffer != NULL, "OSAllocMem() failed",
+	                      return_);
+
+	i64Res = pvr_copy_from_user(pcLocalBuffer, pszBuffer, ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(i64Res == 0, "pvr_copy_from_user() failed",
+	                      free_local_buffer_);
+
+	/* ensure that the framework user gets a NUL terminated buffer */
+	pcLocalBuffer[ui64Count - 1] = '\0';
+
+	i64Res = psIter->pfnWrite(pcLocalBuffer, ui64Count, &ui64Pos,
+	                          psDFSFile->sEntry.sImplEntry.pvPrivData);
+	PVR_LOG_GOTO_IF_FALSE(i64Res >= 0, "pfnWrite failed", free_local_buffer_);
+
+	*puiPos = ui64Pos;
+
+free_local_buffer_:
+	OSFreeMem(pcLocalBuffer);
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return i64Res;
+}
+
+static const struct file_operations _g_psFileOpsGen = {
+	.owner = THIS_MODULE,
+	.open = _Open,
+	.release = _Close,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+static const struct file_operations _g_psFileOpsRndAcc = {
+	.owner = THIS_MODULE,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	return PVRSRV_OK;
+}
+
+static void _DeInit(void)
+{
+}
+
+static PVRSRV_ERROR _CreateFile(const IMG_CHAR *pszName,
+                                DI_ENTRY_TYPE eType,
+                                const DI_ITERATOR_CB *psIterCb,
+                                void *pvPrivData,
+                                void *pvParentDir,
+                                void **pvFile)
+{
+	DFS_DIR *psParentDir = pvParentDir;
+	DFS_FILE *psFile;
+	umode_t uiMode = S_IFREG;
+	struct dentry *psEntry;
+	const struct file_operations *psFileOps = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvFile != NULL, "pvFile");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentDir != NULL, "pvParentDir");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			psFileOps = &_g_psFileOpsGen;
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			psFileOps = &_g_psFileOpsRndAcc;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto return_;
+	}
+
+	psFile = OSAllocMem(sizeof(*psFile));
+	PVR_LOG_GOTO_IF_NOMEM(psFile, eError, return_);
+
+	uiMode |= psIterCb->pfnShow != NULL || psIterCb->pfnRead != NULL ?
+	        S_IRUGO : 0;
+	uiMode |= psIterCb->pfnWrite != NULL ? S_IWUSR : 0;
+
+	psEntry = debugfs_create_file(pszName, uiMode, psParentDir->psDirEntry,
+	                              psFile, psFileOps);
+	if (IS_ERR_OR_NULL(psEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create debugfs '%s' file",
+		        __func__, pszName));
+
+		eError = psEntry == NULL ?
+		        PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_ERROR_INVALID_DEVICE;
+		goto free_file_;
+	}
+
+	psFile->eType = eType;
+	psFile->psSeqOps = &_g_sSeqOps;
+	psFile->sEntry.sIterCb = *psIterCb;
+	psFile->sEntry.sImplEntry.pvPrivData = pvPrivData;
+	psFile->sEntry.sImplEntry.pvNative = NULL;
+	psFile->sEntry.sImplEntry.psCb = &_g_sEntryCallbacks;
+	psFile->psParentDir = psParentDir;
+	psFile->psFileEntry = psEntry;
+
+	*pvFile = psFile;
+
+	return PVRSRV_OK;
+
+free_file_:
+	OSFreeMem(psFile);
+
+return_:
+	return eError;
+}
+
+static void _DestroyFile(void *pvFile)
+{
+	DFS_FILE *psFile = pvFile;
+
+	PVR_ASSERT(psFile != NULL);
+
+	psFile->psFileEntry->d_inode->i_private = NULL;
+
+	debugfs_remove(psFile->psFileEntry);
+	OSFreeMem(psFile);
+}
+
+static PVRSRV_ERROR _CreateDir(const IMG_CHAR *pszName,
+                               void *pvParentDir,
+                               void **ppvDir)
+{
+	DFS_DIR *psNewDir;
+	struct dentry *psDirEntry, *psParentDir = NULL;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvDir != NULL, "ppvDir");
+
+	psNewDir = OSAllocMem(sizeof(*psNewDir));
+	PVR_LOG_RETURN_IF_NOMEM(psNewDir, "OSAllocMem");
+
+	psNewDir->psParentDir = pvParentDir;
+
+	if (pvParentDir != NULL)
+	{
+		psParentDir = psNewDir->psParentDir->psDirEntry;
+	}
+
+	psDirEntry = debugfs_create_dir(pszName, psParentDir);
+	if (IS_ERR_OR_NULL(psDirEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create '%s' debugfs directory",
+		        __func__, pszName));
+		OSFreeMem(psNewDir);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psNewDir->psDirEntry = psDirEntry;
+	*ppvDir = psNewDir;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyDir(void *pvDir)
+{
+	DFS_DIR *psDir = pvDir;
+
+	PVR_ASSERT(psDir != NULL);
+
+	debugfs_remove(psDir->psDirEntry);
+	OSFreeMem(psDir);
+}
+
+PVRSRV_ERROR PVRDebugFsRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateFile,
+		.pfnDestroyEntry = _DestroyFile,
+		.pfnCreateGroup = _CreateDir,
+		.pfnDestroyGroup = _DestroyDir
+	};
+
+	return DIRegisterImplementation("debugfs", &sImplCb);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.h
new file mode 100644
index 000000000000..23ae55b12069
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Title          DebugFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DEBUGFS_H
+#define PVR_DEBUGFS_H
+
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR PVRDebugFsRegister(void);
+
+#endif /* PVR_DEBUGFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drm.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drm.c
new file mode 100644
index 000000000000..f75542490b78
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drm.c
@@ -0,0 +1,551 @@
+/*
+ * @File
+ * @Title       PowerVR DRM driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#include <drm/drm.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_print.h>
+#include <linux/dma-mapping.h>
+#else
+#include <drm/drmP.h> /* include before drm_crtc.h for kernels older than 3.9 */
+#endif
+
+#include <drm/drm_crtc.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+
+#include "module_common.h"
+#include "pvr_drm.h"
+#include "pvr_drv.h"
+#include "pvrversion.h"
+#include "services_kernel_client.h"
+#include "pvr_sync_ioctl_drm.h"
+#include "physmem_dmabuf_internal.h"
+
+#include "kernel_compatibility.h"
+
+#include "dkf_server.h"
+#include "dkp_impl.h"
+
+#define PVR_DRM_DRIVER_NAME PVR_DRM_NAME
+#define PVR_DRM_DRIVER_DESC "Imagination Technologies PVR DRM"
+#define	PVR_DRM_DRIVER_DATE "20170530"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#define	PVR_DRM_DRIVER_PRIME 0
+#else
+#define	PVR_DRM_DRIVER_PRIME DRIVER_PRIME
+#endif
+
+/*
+ * Protects global PVRSRV_DATA on a multi device system. i.e. this is used to
+ * protect the PVRSRVCommonDeviceXXXX() APIs in the Server common layer which
+ * are not re-entrant for device creation and initialisation.
+ */
+static DEFINE_MUTEX(g_device_mutex);
+
+/* Executed before sleep/suspend-to-RAM/S3. During this phase the content
+ * of the video memory is preserved (copied to system RAM). This step is
+ * necessary because the device can be powered off and the content of the
+ * video memory lost.
+ */
+static int pvr_pm_suspend(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("device %p\n", dev);
+
+	return PVRSRVDeviceSuspend(ddev);
+}
+
+/* Executed after the system is woken up from sleep/suspend-to-RAM/S3. This
+ * phase restores the content of the video memory from the system RAM.
+ */
+static int pvr_pm_resume(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("device %p\n", dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+/* Executed before the hibernation image is created. This callback allows to
+ * preserve the content of the video RAM into the system RAM which in turn
+ * is then stored into a disk.
+ */
+static int pvr_pm_freeze(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceSuspend(ddev);
+}
+
+/* Executed after the hibernation image is created or if the creation of the
+ * image has failed. This callback should undo whatever was done in
+ * pvr_pm_freeze to allow the device to operate in the same way as before the
+ * call to pvr_pm_freeze.
+ */
+static int pvr_pm_thaw(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+/* Executed after the hibernation image is created. This callback should not
+ * preserve the content of the video memory since this was already done
+ * in pvr_pm_freeze.
+ *
+ * Note: from the tests performed on a TestChip this callback is not executed
+ *       and driver's pvr_shutdown() is executed instead.
+ */
+static int pvr_pm_poweroff(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	PVRSRVDeviceShutdown(ddev);
+
+	return 0;
+}
+
+/* Executed after the content of the system memory is restored from the
+ * hibernation image. This callback restored video RAM from the system RAM
+ * and performs any necessary device setup required for the device to operate
+ * properly.
+ */
+static int pvr_pm_restore(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+const struct dev_pm_ops pvr_pm_ops = {
+	/* Sleep (suspend-to-RAM/S3) callbacks.
+	 * This mode saves the content of the video RAM to the system RAM and
+	 * powers off the device to reduce the power consumption. Because the
+	 * video RAM can be powered off, it needs to be preserved beforehand.
+	 */
+	.suspend = pvr_pm_suspend,
+	.resume = pvr_pm_resume,
+
+	/* Hibernation (suspend-to-disk/S4) callbacks.
+	 * This mode saves the content of the video RAM to the system RAM and then
+	 * dumps the system RAM to disk (swap partition or swap file). The system
+	 * then powers off. After power on the system RAM content is loaded from
+	 * the disk and then video RAM is restored from the system RAM.
+	 *
+	 * The procedure is executed in following order
+	 *
+	 * - Suspend-to-disk is triggered
+	 *   At this point the OS goes through the list of all registered devices and
+	 *   calls provided callbacks.
+	 * -- pvr_pm_freeze() is called
+	 *         The GPU is powered of and submitting new work is blocked.
+	 *         The content of the video RAM is saved to the system RAM, and
+	 *         other actions required to suspend the device are performed.
+	 * -- system RAM image is created and saved on the disk
+	 *         The disk now contains a snapshot for the DDK Driver for the
+	 *         moment when pvr_pm_freeze() was called.
+	 * -- pvr_pm_thaw() is called
+	 *         All actions taken in pvr_pm_freeze() are undone. The memory
+	 *         allocated for the video RAM is freed and all actions necessary
+	 *         to bring the device to operational state are taken.
+	 *         This makes sure that regardless if image was created successfully
+	 *         or not the device remains operational.
+	 *
+	 * - System is powered off
+	 * -- pvr_shutdown() is called
+	 *         No actions are required beside powering off the GPU.
+	 *
+	 * - System is powered up
+	 * -- system RAM image is read from the disk
+	 *         This restores the snapshot of the DDK driver along with the saved
+	 *         video RAM buffer.
+	 * -- pvr_pm_restore() is called
+	 *         Video RAM is restored from the buffer located in the system RAM.
+	 *         Actions to reset the device and bring it back to working state
+	 *         are taken. Video RAM buffer is freed.
+	 *         In summary the same procedure as in the case of pvr_pm_thaw() is
+	 *         performed.
+	 */
+	.freeze = pvr_pm_freeze,
+	.thaw = pvr_pm_thaw,
+	.poweroff = pvr_pm_poweroff,
+	.restore = pvr_pm_restore,
+};
+
+#if defined(SUPPORT_LINUX_FDINFO)
+static void pvr_drm_show_drm_info(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	int pid, void *hPrivHandle)
+{
+	struct pvr_drm_private *priv;
+	struct drm_device *pdev;
+
+	pdev = (struct drm_device *)hPrivHandle;
+
+	/* The only field possibly valid in 'priv' is the dev_node field */
+	priv = (struct pvr_drm_private *)pdev->dev_private;
+
+	if (priv->dev_node == psDevNode) {
+
+		/* For kernels post 6.5.0 the mandatory driver fields are produced
+		 * by the 'drm_show_fdinfo' routine in the kernel.
+		 * Avoid duplicating this information here.
+		 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0))
+		PVRDKPOutput(priv->hDeviceDKPRef,
+		             "drm-driver:\t%s\n",
+		             PVR_DRM_DRIVER_NAME);
+#if defined(CONFIG_PCI)
+		if (dev_is_pci(pdev->dev)) {
+			struct pci_dev *pcidev = to_pci_dev(pdev->dev);
+
+			PVRDKPOutput(priv->hDeviceDKPRef,
+			             "drm-pdev:\t%04x:%02x:%02x.%d\n",
+			             pci_domain_nr(pcidev->bus),
+			             pcidev->bus->number,
+			             PCI_SLOT(pcidev->devfn),
+			             PCI_FUNC(pcidev->devfn));
+		}
+#endif /* defined(CONFIG_PCI) */
+#endif	/* LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0) */
+	}
+}
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+int pvr_drm_load(struct drm_device *ddev, unsigned long flags)
+{
+	struct pvr_drm_private *priv;
+	enum PVRSRV_ERROR_TAG srv_err;
+	int err, deviceId;
+
+	DRM_DEBUG_DRIVER("device %p\n", ddev->dev);
+
+	dev_set_drvdata(ddev->dev, ddev);
+
+	if (ddev->render)
+		deviceId = ddev->render->index;
+	else /* when render node is NULL, fallback to primary node */
+		deviceId = ddev->primary->index;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+	ddev->dev_private = priv;
+
+	if (!ddev->dev->dma_parms)
+		ddev->dev->dma_parms = &priv->dma_parms;
+	dma_set_max_seg_size(ddev->dev, DMA_BIT_MASK(32));
+
+	mutex_lock(&g_device_mutex);
+
+	srv_err = PVRSRVCommonDeviceCreate(ddev->dev, deviceId, &priv->dev_node);
+	if (srv_err != PVRSRV_OK) {
+		DRM_ERROR("failed to create device node for device %p (%s)\n",
+			  ddev->dev, PVRSRVGetErrorString(srv_err));
+		if (srv_err == PVRSRV_ERROR_PROBE_DEFER)
+			err = -EPROBE_DEFER;
+		else
+			err = -ENODEV;
+		goto err_unset_dma_parms;
+	}
+
+	err = PVRSRVDeviceInit(priv->dev_node);
+	if (err) {
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+			  ddev->dev, err);
+		goto err_device_destroy;
+	}
+
+	drm_mode_config_init(ddev);
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+	srv_err = PVRSRVCommonDeviceInitialise(priv->dev_node);
+	if (srv_err != PVRSRV_OK) {
+		err = -ENODEV;
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+			  ddev->dev, err);
+		goto err_device_deinit;
+	}
+#endif
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	srv_err = PVRSRVRegisterDKP(ddev,
+	                            "drm-pvr-drm",
+	                            pvr_drm_show_drm_info,
+	                            DKP_CONNECTION_FLAG_ALL,
+	                            &priv->hDeviceDKPRef);
+
+	if (srv_err != PVRSRV_OK) {
+		err = -ENODEV;
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+				  ddev->dev, err);
+		goto err_device_deinit;
+	}
+#endif
+
+	mutex_unlock(&g_device_mutex);
+
+	return 0;
+
+#if defined(SUPPORT_LINUX_FDINFO) || (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+err_device_deinit:
+#endif
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+	drm_mode_config_cleanup(ddev);
+	PVRSRVDeviceDeinit(priv->dev_node);
+#endif
+err_device_destroy:
+	PVRSRVCommonDeviceDestroy(priv->dev_node);
+err_unset_dma_parms:
+	mutex_unlock(&g_device_mutex);
+	if (ddev->dev->dma_parms == &priv->dma_parms)
+		ddev->dev->dma_parms = NULL;
+	kfree(priv);
+err_exit:
+	return err;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+int pvr_drm_unload(struct drm_device *ddev)
+#else
+void pvr_drm_unload(struct drm_device *ddev)
+#endif
+{
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	DRM_DEBUG_DRIVER("device %p\n", ddev->dev);
+
+	drm_mode_config_cleanup(ddev);
+
+	PVRSRVDeviceDeinit(priv->dev_node);
+
+	mutex_lock(&g_device_mutex);
+	PVRSRVCommonDeviceDestroy(priv->dev_node);
+	mutex_unlock(&g_device_mutex);
+
+	if (ddev->dev->dma_parms == &priv->dma_parms)
+		ddev->dev->dma_parms = NULL;
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	PVRSRVUnRegisterDKP(ddev, priv->hDeviceDKPRef);
+#endif
+
+	kfree(priv);
+	ddev->dev_private = NULL;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	return 0;
+#endif
+}
+
+static int pvr_drm_open(struct drm_device *ddev, struct drm_file *dfile)
+{
+#if (PVRSRV_DEVICE_INIT_MODE != PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	struct pvr_drm_private *priv = ddev->dev_private;
+	int err;
+#endif
+
+	if (!try_module_get(THIS_MODULE)) {
+		DRM_ERROR("failed to get module reference\n");
+		return -ENOENT;
+	}
+
+#if (PVRSRV_DEVICE_INIT_MODE != PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	err = PVRSRVDeviceServicesOpen(priv->dev_node, dfile);
+	if (err)
+		module_put(THIS_MODULE);
+
+	return err;
+#else
+	return 0;
+#endif
+}
+
+static void pvr_drm_release(struct drm_device *ddev, struct drm_file *dfile)
+{
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	PVRSRVDeviceRelease(priv->dev_node, dfile);
+
+	module_put(THIS_MODULE);
+}
+
+static struct drm_ioctl_desc pvr_drm_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(PVR_SRVKM_CMD, PVRSRV_BridgeDispatchKM,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SRVKM_INIT, drm_pvr_srvkm_init,
+			  DRM_RENDER_ALLOW),
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_RENAME_CMD, pvr_sync_rename_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_FORCE_SW_ONLY_CMD, pvr_sync_force_sw_only_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SW_SYNC_CREATE_FENCE_CMD, pvr_sw_sync_create_fence_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SW_SYNC_INC_CMD, pvr_sw_sync_inc_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_EXP_FENCE_SYNC_FORCE_CMD, pvr_sync_ioctl_force_exp_only,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_CREATE_EXPORT_FENCE_CMD,
+			  pvr_export_fence_sync_create_fence_ioctl,
+			  DRM_RENDER_ALLOW),
+#endif
+};
+
+#if defined(CONFIG_COMPAT)
+static long pvr_compat_ioctl(struct file *file, unsigned int cmd,
+			     unsigned long arg)
+{
+	unsigned int nr = DRM_IOCTL_NR(cmd);
+
+	if (nr < DRM_COMMAND_BASE)
+		return drm_compat_ioctl(file, cmd, arg);
+
+	return drm_ioctl(file, cmd, arg);
+}
+#endif /* defined(CONFIG_COMPAT) */
+
+#if defined(SUPPORT_LINUX_FDINFO)
+/*
+ * Produce the PVR specific fdinfo (utilization figures etc.) when queried.
+ *
+ * For kernels post 6.5 there is a helper function called 'drm_show_fdinfo'
+ * which will generate the mandatory keys (drm-driver, drm-pdev, drm-client-id)
+ * so we do not need to generate these if running on a later kernel etc.
+ *
+ */
+static void pvr_show_fdinfo(struct seq_file *seq_file, struct file *file)
+{
+	struct drm_file *dfile = file->private_data;
+	struct drm_device *dev = dfile->minor->dev;
+	struct drm_printer p = drm_seq_file_printer(seq_file);
+	PVRSRV_CONNECTION_PRIV *pvr_connection = dfile->driver_priv;
+	struct pvr_drm_private *priv;
+	int my_pid;
+
+	/* Grab the PID from the associated drm_file->pid->numbers[0].nr */
+	my_pid = dfile->pid->numbers[0].nr;
+
+	priv = (struct pvr_drm_private *)dev->dev_private;
+
+	/* Generate driver-specific keys */
+	PVRDKFTraverse((DKF_VPRINTF_FUNC*)drm_vprintf,
+	               &p,
+	               priv->dev_node,
+	               my_pid,
+	               pvr_connection->ui32Type);
+
+	/* Call into OS-specific drm_show_fdinfo if it is supported */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0))
+	drm_show_fdinfo(seq_file, file);
+#endif	/* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0) */
+}
+#endif /* SUPPORT_LINUX_FDINFO */
+
+const struct file_operations pvr_drm_fops = {
+	.owner			= THIS_MODULE,
+	.open			= drm_open,
+	.release		= drm_release,
+	.unlocked_ioctl		= drm_ioctl,
+#if defined(CONFIG_COMPAT)
+	.compat_ioctl		= pvr_compat_ioctl,
+#endif
+	.mmap			= PVRSRV_MMap,
+	.poll			= drm_poll,
+	.read			= drm_read,
+#if defined(SUPPORT_LINUX_FDINFO)
+	.show_fdinfo	= pvr_show_fdinfo,
+#endif /* SUPPORT_LINUX_FDINFO */
+};
+
+const struct drm_driver pvr_drm_generic_driver = {
+	.driver_features	= DRIVER_RENDER |
+				  DRIVER_GEM | PVR_DRM_DRIVER_PRIME,
+
+	.load			= NULL,
+	.unload			= NULL,
+	.open			= pvr_drm_open,
+	.postclose		= pvr_drm_release,
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	/* prime_fd_to_handle is not supported */
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_prime_export	= PhysmemGEMPrimeExport,
+	.gem_free_object	= PhysmemGEMObjectFree,
+#endif
+	.ioctls			= pvr_drm_ioctls,
+	.num_ioctls		= ARRAY_SIZE(pvr_drm_ioctls),
+	.fops			= &pvr_drm_fops,
+
+	.name			= PVR_DRM_DRIVER_NAME,
+	.desc			= PVR_DRM_DRIVER_DESC,
+	.date			= PVR_DRM_DRIVER_DATE,
+	.major			= PVRVERSION_MAJ,
+	.minor			= PVRVERSION_MIN,
+	.patchlevel		= PVRVERSION_BUILD,
+};
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drv.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drv.h
new file mode 100644
index 000000000000..827145d60c50
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drv.h
@@ -0,0 +1,116 @@
+/*
+ * @File
+ * @Title       PowerVR DRM driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_DRV_H__)
+#define __PVR_DRV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#include <linux/device.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <linux/pm.h>
+
+struct file;
+struct _PVRSRV_DEVICE_NODE_;
+struct workqueue_struct;
+struct vm_area_struct;
+
+/* This structure is used to store Linux specific per-device information. */
+struct pvr_drm_private {
+	struct _PVRSRV_DEVICE_NODE_ *dev_node;
+
+	/*
+	 * This is needed for devices that don't already have their own dma
+	 * parameters structure, e.g. platform devices, and, if necessary, will
+	 * be assigned to the 'struct device' during device initialisation. It
+	 * should therefore never be accessed directly via this structure as
+	 * this may not be the version of dma parameters in use.
+	 */
+	struct device_dma_parameters dma_parms;
+
+	/* PVR Sync debug notify handle */
+	void *sync_debug_notify_handle;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	/* Only used in fence sync as sync_debug_notify_handle is used
+	 * to print a header only. Content is registered separately.
+	 * Used to print foreign sync debug
+	 */
+	void *sync_foreign_debug_notify_handle;
+#endif
+
+	/* Flag stating if the device was suspended/resumed. If this is 0 then
+	 * the device was either resumed or no suspend was called but if 1 then
+	 * the OS called suspend on this device.
+	 */
+	atomic_t suspended;
+
+	/* Padding for 8-byte alignment */
+	unsigned int uiPad1;
+
+	/* Handle associated with DKP for this DRM device */
+	void *hDeviceDKPRef;
+};
+
+extern const struct dev_pm_ops pvr_pm_ops;
+extern const struct drm_driver pvr_drm_generic_driver;
+extern const struct file_operations pvr_drm_fops;
+
+int pvr_drm_load(struct drm_device *ddev, unsigned long flags);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+int pvr_drm_unload(struct drm_device *ddev);
+#else
+void pvr_drm_unload(struct drm_device *ddev);
+#endif
+
+int PVRSRV_BridgeDispatchKM(struct drm_device *dev, void *arg,
+			    struct drm_file *file);
+int PVRSRV_MMap(struct file *file, struct vm_area_struct *ps_vma);
+
+#endif /* !defined(__PVR_DRV_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.c
new file mode 100644
index 000000000000..30640d542b0e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.c
@@ -0,0 +1,363 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_common.c
+@Title          PowerVR devfreq device common utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux DVFS and PDVFS shared code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(NO_HARDWARE)
+
+#include <linux/version.h>
+#include <linux/device.h>
+#include <drm/drm.h>
+#if defined(CONFIG_PM_OPP)
+#include <linux/pm_opp.h>
+#endif
+
+#include "pvrsrv.h"
+
+/*
+ * Common DVFS support code shared between SUPPORT_LINUX_DVFS and
+ * SUPPORT_PDVFS, primarily for OPP table support.
+ *
+ * Note that PDVFS implements the Linux/OS devfreq module in
+ * the firmware, so no devfreq API calls should be used here.
+ */
+#include "pvr_dvfs.h"
+#include "pvr_dvfs_common.h"
+
+#include "kernel_compatibility.h"
+
+
+/*************************************************************************/ /*!
+@Function       GetOPPValues
+
+@Description    Common code to store the OPP points in the freq_table,
+                for use with the IMG DVFS/devfreq implementation, or with
+                the Proactive DVFS.
+                Requires CONFIG_PM_OPP support in the kernel.
+
+@Input          dev        OS Device node
+@Output         min_freq   Min clock freq (Hz)
+@Output         min_volt   Min voltage (V)
+@Output         max_freq   Max clock freq (Hz)
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(CONFIG_PM_OPP)
+int GetOPPValues(struct device *dev,
+                 unsigned long *min_freq,
+                 unsigned long *min_volt,
+                 unsigned long *max_freq,
+                 struct pvr_opp_freq_table *pvr_freq_table)
+{
+	struct dev_pm_opp *opp;
+	int count, i, err = 0;
+	unsigned long freq;
+
+	unsigned long *freq_table;
+
+	count = dev_pm_opp_get_opp_count(dev);
+	if (count <= 0)
+	{
+		dev_err(dev, "Could not fetch OPP count, %d\n", count);
+		return -EINVAL;
+	}
+
+	dev_info(dev, "Found %d OPP points.\n", count);
+
+	freq_table = devm_kcalloc(dev, count, sizeof(*freq_table), GFP_ATOMIC);
+	if (! freq_table)
+	{
+		return -ENOMEM;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	/* Start RCU read-side critical section to map frequency to OPP */
+	rcu_read_lock();
+#endif
+
+	/* Iterate over OPP table; Iteration 0 finds "opp w/ freq >= 0 Hz". */
+	freq = 0;
+	opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+	if (IS_ERR(opp))
+	{
+		err = PTR_ERR(opp);
+		dev_err(dev, "Couldn't find lowest frequency, %d\n", err);
+		goto exit;
+	}
+
+	*min_volt = dev_pm_opp_get_voltage(opp);
+	*max_freq = *min_freq = freq_table[0] = freq;
+	dev_info(dev, "opp[%d/%d]: (%lu Hz, %lu uV)\n", 1, count, freq, *min_volt);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	dev_pm_opp_put(opp);
+#endif
+
+	/* Iteration i > 0 finds "opp w/ freq >= (opp[i-1].freq + 1)". */
+	for (i = 1; i < count; i++)
+	{
+		freq++;
+		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+		if (IS_ERR(opp))
+		{
+			err = PTR_ERR(opp);
+			dev_err(dev, "Couldn't find %dth frequency, %d\n", i, err);
+			goto exit;
+		}
+
+		freq_table[i] = freq;
+		*max_freq = freq;
+		dev_info(dev,
+				 "opp[%d/%d]: (%lu Hz, %lu uV)\n",
+				  i + 1,
+				  count,
+				  freq,
+				  dev_pm_opp_get_voltage(opp));
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+		dev_pm_opp_put(opp);
+#endif
+	}
+
+exit:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#endif
+
+	if (!err)
+	{
+		pvr_freq_table->freq_table = freq_table;
+		pvr_freq_table->num_levels = count;
+	}
+	else
+	{
+		devm_kfree(dev, freq_table);
+	}
+
+	return err;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       DVFSCopyOPPTable
+
+@Description    Copies the OPP points (voltage/frequency) to the target
+                firmware structure RGXFWIF_OPP_INFO which must be pre-allocated.
+                The source values are expected to be read into the kernel
+                Power Manager from the platform's Device Tree.
+                Requires CONFIG_OF and CONFIG_PM_OPP support in the kernel.
+
+@Input          psDeviceNode       Device node
+@Input          psOPPInfo          Target OPP data buffer
+@Input          ui32MaxOPPLevels   Maximum number of OPP levels allowed in buffer.
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+PVRSRV_ERROR DVFSCopyOPPTable(PPVRSRV_DEVICE_NODE psDeviceNode,
+							  RGXFWIF_OPP_INFO   *psOPPInfo,
+							  IMG_UINT32          ui32MaxOPPLevels)
+{
+	PVRSRV_ERROR            eError = PVRSRV_OK;
+	struct device          *psDev = NULL;
+	OPP_LEVEL              *psOPPValue;
+	struct dev_pm_opp      *opp;
+	struct pvr_opp_freq_table pvr_freq_table = {0};
+	unsigned long           min_freq = 0, max_freq = 0, min_volt = 0;
+	unsigned int            i, err;
+	unsigned long *freq_table;
+
+	if (!psDeviceNode || !psOPPInfo)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid device or argument", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+	err = GetOPPValues(psDev, &min_freq, &min_volt, &max_freq, &pvr_freq_table);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: DVFS OPP table not initialised.", __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	freq_table = pvr_freq_table.freq_table;
+
+	if (pvr_freq_table.num_levels > ui32MaxOPPLevels)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Too many OPP levels (%u), max (%u).", __func__,
+		         pvr_freq_table.num_levels, ui32MaxOPPLevels));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto exit;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	/* Start RCU read-side critical section to map frequency to OPP */
+	rcu_read_lock();
+#endif
+
+	/* Loop over the OPP/frequency levels */
+	psOPPValue = &psOPPInfo->asOPPValues[0];
+	for (i=0; i<pvr_freq_table.num_levels; i++)
+	{
+		psOPPValue->ui32Freq = freq_table[i];
+		opp = dev_pm_opp_find_freq_exact(psDev, freq_table[i], IMG_TRUE);
+		if (IS_ERR(opp))
+		{
+			err = PTR_ERR(opp);
+			dev_err(psDev, "Couldn't find %dth frequency, %d\n", i, err);
+			eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+			goto exit;
+		}
+		psOPPValue->ui32Volt = dev_pm_opp_get_voltage(opp);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+		dev_pm_opp_put(opp);
+#endif
+		psOPPValue++;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Copied %u OPP points to the FW processor table.", __func__,
+	         pvr_freq_table.num_levels));
+	psOPPInfo->ui32MaxOPPPoint = pvr_freq_table.num_levels - 1;
+exit:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#endif
+
+	/* Free memory allocated in GetOPPValues */
+	devm_kfree(psDev, freq_table);
+
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+                Prepares the OPP table from the devicetree, if enabled.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_PDVFS)
+PVRSRV_ERROR InitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if !(defined(CONFIG_PM_OPP) && defined(CONFIG_OF))
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_OK;
+#else
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	struct device          *psDev;
+	int                     err;
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psDeviceNode->psDevConfig);
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	/* Setup the OPP table from the device tree for Proactive DVFS. */
+	err = dev_pm_opp_of_add_table(psDev);
+	if (err == 0)
+	{
+		psDVFSDeviceCfg->bDTConfig = IMG_TRUE;
+	}
+	else
+	{
+		/*
+		 * If there are no device tree or system layer provided operating points
+		 * then return an error
+		 */
+		if (psDVFSDeviceCfg->pasOPPTable)
+		{
+			psDVFSDeviceCfg->bDTConfig = IMG_FALSE;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "No system or device tree opp points found, %d", err));
+			return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		}
+	}
+	return PVRSRV_OK;
+#endif
+}
+
+/*************************************************************************/ /*!
+@Function       DeinitPDVFS
+
+@Description    De-Initialise the device for Proactive DVFS support.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+void DeinitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if !(defined(CONFIG_PM_OPP) && defined(CONFIG_OF))
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#else
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = NULL;
+	struct device *psDev = NULL;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	if (psDVFSDeviceCfg->bDTConfig)
+	{
+		/*
+		 * Remove OPP entries for this device; only static entries from
+		 * the device tree are present.
+		 */
+		dev_pm_opp_of_remove_table(psDev);
+	}
+#endif
+}
+#endif /* SUPPORT_PDVFS */
+
+#endif /* !NO_HARDWARE */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.h
new file mode 100644
index 000000000000..87eccce5762e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.h
@@ -0,0 +1,123 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_common.h
+@Title          System level interface for DVFS and PDVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_COMMON_H
+#define PVR_DVFS_COMMON_H
+
+#include "opaque_types.h"
+#include "pvrsrv_error.h"
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+#include "rgx_fwif_km.h"
+#endif
+
+struct pvr_opp_freq_table
+{
+	unsigned long *freq_table;
+	int num_levels;
+};
+
+/*************************************************************************/ /*!
+@Function       GetOPPValues
+
+@Description    Common code to store the OPP points in the pvr_freq_table,
+                for use with the IMG DVFS/devfreq implementation, or with
+                the Proactive DVFS.
+                Requires CONFIG_PM_OPP support in the kernel.
+
+@Input          dev        OS Device node
+@Output         min_freq   Min clock freq (Hz)
+@Output         min_volt   Min voltage (V)
+@Output         max_freq   Max clock freq (Hz)
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(CONFIG_PM_OPP)
+int GetOPPValues(struct device *dev,
+                 unsigned long *min_freq,
+                 unsigned long *min_volt,
+                 unsigned long *max_freq,
+                 struct pvr_opp_freq_table *pvr_freq_table);
+#endif
+
+/*************************************************************************/ /*!
+@Function       DVFSCopyOPPTable
+
+@Description    Copies the OPP points (voltage/frequency) to the target
+                firmware structure RGXFWIF_OPP_INFO which must be pre-allocated.
+                The source values are expected to be read into the kernel
+                Power Manager from the platform's Device Tree.
+                Requires CONFIG_OF and CONFIG_PM_OPP support in the kernel.
+
+@Input          psDeviceNode       Device node
+@Input          psOPPInfo          Target OPP data buffer
+@Input          ui32MaxOPPLevels   Maximum number of OPP levels allowed in buffer.
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+PVRSRV_ERROR DVFSCopyOPPTable(PPVRSRV_DEVICE_NODE psDeviceNode,
+							  RGXFWIF_OPP_INFO   *psOPPInfo,
+							  IMG_UINT32          ui32MaxOPPLevels);
+#endif
+
+#if defined(SUPPORT_PDVFS)
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+                Prepares the OPP table from the devicetree, if enabled.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR InitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+void DeinitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+#endif /* PVR_DVFS_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.c
new file mode 100644
index 000000000000..f5e1a057a0ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.c
@@ -0,0 +1,1412 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_device.c
+@Title          PowerVR devfreq device implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux DVFS module setup
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(NO_HARDWARE)
+
+#include <linux/devfreq.h>
+#if defined(CONFIG_DEVFREQ_THERMAL)
+#include <linux/devfreq_cooling.h>
+#endif
+#include <linux/version.h>
+#include <linux/device.h>
+#include <drm/drm.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "power.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+
+#include "rgxdevice.h"
+#include "rgxinit.h"
+#include "sofunc_rgx.h"
+
+#include "syscommon.h"
+
+#include "pvr_dvfs.h"
+#include "pvr_dvfs_device.h"
+#include "pvr_dvfs_common.h"
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+
+#if !defined(CONFIG_PM_DEVFREQ)
+#error "PVR DVFS governor requires kernel support for devfreq (CONFIG_PM_DEVFREQ = 1)"
+#endif
+#include "linux/governor.h"
+#include "pvr_dvfs_governor.h"
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+#include "linux/devfreq-event.h"
+#include "pvr_dvfs_events.h"
+#endif
+#endif /* SUPPORT_PVR_DVFS_GOVERNOR */
+
+#include "kernel_compatibility.h"
+
+
+/* Default constants for PVR-Balanced (PVR) governor */
+#define PVR_UPTHRESHOLD			(90)
+#define PVR_DOWNDIFFERENTIAL	(5)
+
+static int _device_get_devid(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	int deviceId;
+
+	if (ddev->render)
+		deviceId = ddev->render->index;
+	else /* when render node is NULL, fallback to primary node */
+		deviceId = ddev->primary->index;
+
+	return deviceId;
+}
+
+static IMG_INT32 devfreq_target(struct device *dev, unsigned long *requested_freq, IMG_UINT32 flags)
+{
+	int deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	RGX_DATA		*psRGXData = NULL;
+	IMG_DVFS_DEVICE		*psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG	*psDVFSDeviceCfg = NULL;
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo = NULL;
+	IMG_UINT32		ui32Freq, ui32CurFreq, ui32Volt;
+	struct dev_pm_opp *opp;
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	/* Check the RGX device is initialised */
+	if (!psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+	if (psDVFSDevice->eState != PVR_DVFS_STATE_READY)
+	{
+		*requested_freq = psRGXTimingInfo->ui32CoreClockSpeed;
+		return 0;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_lock();
+#endif
+
+	opp = devfreq_recommended_opp(dev, requested_freq, flags);
+	if (IS_ERR(opp)) {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+		rcu_read_unlock();
+#endif
+		PVR_DPF((PVR_DBG_ERROR, "Invalid OPP"));
+		return PTR_ERR(opp);
+	}
+
+	ui32Freq = dev_pm_opp_get_freq(opp);
+	ui32Volt = dev_pm_opp_get_voltage(opp);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#else
+	dev_pm_opp_put(opp);
+#endif
+
+	ui32CurFreq = psRGXTimingInfo->ui32CoreClockSpeed;
+
+	if (ui32CurFreq == ui32Freq)
+	{
+		return 0;
+	}
+
+	if (PVRSRV_OK != PVRSRVDevicePreClockSpeedChange(psDeviceNode,
+													 psDVFSDeviceCfg->bIdleReq,
+													 NULL))
+	{
+		dev_err(dev, "PVRSRVDevicePreClockSpeedChange failed\n");
+		return -EPERM;
+	}
+
+	/* Increasing frequency, change voltage first */
+	if (ui32Freq > ui32CurFreq)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, ui32Volt);
+	}
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDeviceNode->psDevConfig->hSysData, ui32Freq);
+
+	/* Decreasing frequency, change frequency first */
+	if (ui32Freq < ui32CurFreq)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, ui32Volt);
+	}
+
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32Freq;
+
+	PVRSRVDevicePostClockSpeedChange(psDeviceNode, psDVFSDeviceCfg->bIdleReq,
+									 NULL);
+
+	return 0;
+}
+
+static int devfreq_get_dev_status(struct device *dev, struct devfreq_dev_status *stat)
+{
+	int                      deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE      *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	PVRSRV_RGXDEV_INFO      *psDevInfo = NULL;
+	IMG_DVFS_DEVICE         *psDVFSDevice = NULL;
+	RGX_DATA                *psRGXData = NULL;
+	RGX_TIMING_INFORMATION  *psRGXTimingInfo = NULL;
+	RGXFWIF_GPU_UTIL_STATS  *psGpuUtilStats = NULL;
+	PVRSRV_ERROR             eError;
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	struct pvr_profiling_dev_status *pvr_stat = stat->private_data;
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* Check the RGX device is initialised */
+	if (!psDevInfo || !psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+	stat->current_frequency = psRGXTimingInfo->ui32CoreClockSpeed;
+
+	if (psDevInfo->pfnGetGpuUtilStats == NULL)
+	{
+		/* Not yet ready. So set times to something sensible. */
+		stat->busy_time = 0;
+		stat->total_time = 0;
+		return 0;
+	}
+
+	psGpuUtilStats = kzalloc(sizeof(*psGpuUtilStats), GFP_KERNEL);
+
+	if (!psGpuUtilStats)
+	{
+		return -ENOMEM;
+	}
+
+	eError = psDevInfo->pfnGetGpuUtilStats(psDeviceNode,
+						psDVFSDevice->hGpuUtilUserDVFS,
+						psGpuUtilStats);
+
+	if (eError != PVRSRV_OK)
+	{
+		kfree(psGpuUtilStats);
+		return -EAGAIN;
+	}
+
+	stat->busy_time = psGpuUtilStats->ui64GpuStatActive;
+	stat->total_time = psGpuUtilStats->ui64GpuStatCumulative;
+
+	kfree(psGpuUtilStats);
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	err = pvr_get_dev_status_get_events(psDVFSDevice->psProfilingDevice, pvr_stat);
+	if (err)
+		return err;
+#endif
+
+	return 0;
+}
+
+static IMG_INT32 devfreq_cur_freq(struct device *dev, unsigned long *freq)
+{
+	int deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	RGX_DATA *psRGXData = NULL;
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* Check the RGX device is initialised */
+	if (!psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	*freq = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return 0;
+}
+
+static struct devfreq_dev_profile img_devfreq_dev_profile =
+{
+	.polling_ms	        = 10,
+	.target             = devfreq_target,
+	.get_dev_status     = devfreq_get_dev_status,
+	.get_cur_freq       = devfreq_cur_freq,
+};
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+
+/* DEVFREQ governor name */
+#define DEVFREQ_GOV_PVR_BALANCED	"pvr_balanced"
+
+static unsigned long _get_max_freq(struct devfreq *devfreq_dev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+	return devfreq_dev->scaling_max_freq;
+#else
+	return devfreq_dev->max_freq;
+#endif
+}
+
+/*
+ * Calculate the utilisation percentage of X cycles as a proportion
+ * of Y cycles
+ */
+static inline
+unsigned long UTILISATION_PC(unsigned long X, unsigned long Y)
+{
+	PVR_ASSERT(Y > 0);
+
+/*
+	if (X < UINT_MAX / 100UL)
+		return min(100UL, X * 100UL / Y);
+	else
+		return min(100UL, X / (Y / 100UL));
+*/
+	/* Prevent overflow */
+	if (X < UINT32_MAX / 100U)
+		return (X * 100U / Y);
+	else
+		return (X / (Y / 100U));
+}
+
+static int pvr_governor_get_target(struct devfreq *devfreq_dev,
+								   unsigned long *freq)
+{
+	struct devfreq_dev_status *stat;
+	struct pvr_profiling_dev_status *pvr_stat;
+	IMG_DVFS_GOVERNOR_CFG *data = devfreq_dev->data;
+	unsigned long long a, b;
+	unsigned int pvr_upthreshold = PVR_UPTHRESHOLD;
+	unsigned int pvr_downdifferential = PVR_DOWNDIFFERENTIAL;
+	unsigned long ui32Util, ui32UtilDM, ui32UtilBus;
+	int iGeomEventID, iFragEventID, iCompEventID, iSLCReadEventID, iSLCWriteEventID;
+	int iSLCReadEventID2, iSLCWriteEventID2;
+	int err;
+
+	if (!devfreq_dev)
+	{
+		return -ENODEV;
+	}
+
+	/*
+	 *	Governor main function for setting target frequency, based on
+	 *	metrics for
+		- utilisation
+		- memory bus bandwidth
+		- ALU counters etc.
+	 */
+
+	err = devfreq_update_stats(devfreq_dev);
+	if (err)
+	{
+		dev_warn(&devfreq_dev->dev, "get_dev_status: error (%d)", err);
+		return err;
+	}
+
+	stat = &devfreq_dev->last_status;
+	pvr_stat = stat->private_data;
+	if (!pvr_stat)
+	{
+		return -EINVAL;
+	}
+
+	iGeomEventID = pvr_find_event_by_name("geom-cycle");
+	iFragEventID = pvr_find_event_by_name("3d-cycle");
+	iCompEventID = pvr_find_event_by_name("comp-cycle");
+	if (iGeomEventID < 0 || iFragEventID < 0 || iCompEventID < 0)
+	{
+		return -EINVAL;
+	}
+
+	if (pvr_stat->event_data[iGeomEventID].total_count == 0 ||
+	    pvr_stat->event_data[iFragEventID].total_count == 0 ||
+	    pvr_stat->event_data[iCompEventID].total_count == 0)
+	{
+		/* no error to avoid verbose kernel logs before the firmware/GPU
+		 * is available. */
+		return 0;
+	}
+
+	/* GEOM utilisation */
+	if (iGeomEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iGeomEventID].load_count,
+						   pvr_stat->event_data[iGeomEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	/* 3D utilisation */
+	if (iFragEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iFragEventID].load_count,
+						   pvr_stat->event_data[iFragEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	/* Compute utilisation */
+	if (iCompEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iCompEventID].load_count,
+						   pvr_stat->event_data[iCompEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	if (ui32UtilDM > 1000U)
+	{
+		/* Illegal value, keep current frequency */
+		*freq = stat->current_frequency;
+		return 0;
+	}
+
+	iSLCReadEventID =   pvr_find_event_by_name("slc-read");
+	iSLCWriteEventID =  pvr_find_event_by_name("slc-write");
+	iSLCReadEventID2 =  pvr_find_event_by_name("slc-read-2");
+	iSLCWriteEventID2 = pvr_find_event_by_name("slc-write-2");
+	if (iSLCReadEventID2 >= 0 && iSLCWriteEventID2 >= 0)
+	{
+		/* Use XE-series memory bus counters */
+
+		ui32UtilBus = 0;
+
+		while (iSLCWriteEventID < ARRAY_SIZE(g_pvr_governor_events))
+		{
+			ui32Util =
+				UTILISATION_PC(pvr_stat->event_data[iSLCReadEventID].load_count,
+							   pvr_stat->event_data[iSLCReadEventID].total_count);
+			ui32UtilBus = ui32UtilBus + ui32Util;
+
+			ui32Util =
+				UTILISATION_PC(pvr_stat->event_data[iSLCWriteEventID].load_count,
+							   pvr_stat->event_data[iSLCWriteEventID].total_count);
+			ui32UtilBus = ui32UtilBus + ui32Util;
+			iSLCReadEventID += 2;
+			iSLCWriteEventID += 2;
+		}
+	}
+	else if (iSLCReadEventID >= 0 && iSLCWriteEventID >= 0)
+	{
+		/* Use DXT-series memory bus counters */
+
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iSLCReadEventID].load_count / data->uiNumMembus,
+						   pvr_stat->event_data[iSLCReadEventID].total_count);
+		ui32UtilBus = ui32Util;
+
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iSLCWriteEventID].load_count / data->uiNumMembus,
+						   pvr_stat->event_data[iSLCWriteEventID].total_count);
+		ui32UtilBus = ui32UtilBus + ui32Util;
+	}
+
+	if (ui32UtilBus > 1000U)
+	{
+		/* Illegal value, use current frequency */
+		*freq = stat->current_frequency;
+		return 0;
+	}
+
+#if defined(DEBUG)
+	iSLCReadEventID =  pvr_find_event_by_name("slc-read");
+	iSLCWriteEventID = pvr_find_event_by_name("slc-write");
+	dev_info(&devfreq_dev->dev, "Geom: %10lu, 3D: %10lu, Comp: %10lu, SLC: %10lu cycles. DM %lu pc, Bus %lu pc.",
+			pvr_stat->event_data[iGeomEventID].load_count,
+			pvr_stat->event_data[iFragEventID].load_count,
+			pvr_stat->event_data[iCompEventID].load_count,
+			pvr_stat->event_data[iSLCReadEventID].load_count + pvr_stat->event_data[iSLCWriteEventID].load_count,
+			ui32UtilDM,
+			ui32UtilBus);
+#endif
+
+	if (ui32UtilDM > 2 * ui32UtilBus) {
+		// DM pipeline limited
+		ui32Util = ui32UtilDM;
+	}
+	else if (ui32UtilBus > 2 * ui32UtilDM) {
+		// Mem bandwidth limited
+		ui32Util = ui32UtilBus;
+	}
+	else {
+		// Average utilisation to smooth random fluctuations
+		ui32Util = (ui32UtilDM + ui32UtilBus) / 2;
+	}
+
+	/* Set MAX if it's busy enough */
+	if (ui32Util > pvr_upthreshold)
+	{
+		*freq = _get_max_freq(devfreq_dev);
+		return 0;
+	}
+
+	/* Set MAX if we do not know the initial frequency */
+	if (stat->current_frequency == 0) {
+		*freq = _get_max_freq(devfreq_dev);
+		return 0;
+	}
+
+	a = (unsigned long long) ui32Util * stat->current_frequency;
+	b = div_u64(a, (pvr_upthreshold - pvr_downdifferential / 2));
+	*freq = (unsigned long) b;
+	return 0;
+}
+
+static void pvr_governor_suspend(struct devfreq *devfreq_dev)
+{
+	/* Nothing to do */
+}
+
+static void pvr_governor_resume(struct devfreq *devfreq_dev)
+{
+	int deviceId;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_DVFS_DEVICE		*psDVFSDevice = NULL;
+	struct pvr_profiling_device *psProfilingDevice;
+
+	if (!devfreq_dev || !dev_get_drvdata(&devfreq_dev->dev))
+		return;
+
+	deviceId = _device_get_devid(&devfreq_dev->dev);
+	psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psProfilingDevice = psDVFSDevice->psProfilingDevice;
+
+	/* Reset the profiling events */
+	pvr_governor_reset_events(psProfilingDevice);
+}
+
+static int pvr_governor_event_handler(struct devfreq *devfreq_dev,
+									  unsigned int event, void *data)
+{
+	if (!devfreq_dev)
+	{
+		pr_err("%s: devfreq_dev not ready.\n", __func__);
+		return -ENODEV;
+	}
+
+	/*
+	 * We cannot take the deviceId here, as the DRM device
+	 * may not be initialised. Null pointer in
+	 * struct drm_device *ddev = dev_get_drvdata(dev)
+	 */
+
+	switch (event) {
+	case DEVFREQ_GOV_START:
+		dev_info(&devfreq_dev->dev,"GOV_START event.\n");
+		devfreq_monitor_start(devfreq_dev);
+		break;
+
+	case DEVFREQ_GOV_STOP:
+		dev_info(&devfreq_dev->dev,"GOV_STOP event.\n");
+		devfreq_monitor_stop(devfreq_dev);
+		break;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	case DEVFREQ_GOV_UPDATE_INTERVAL:
+		dev_info(&devfreq_dev->dev,"GOV_UPDATE_INTERVAL event.\n");
+		devfreq_update_interval(devfreq_dev, (unsigned int *)data);
+		break;
+
+#else
+	case DEVFREQ_GOV_INTERVAL:
+		dev_info(&devfreq_dev->dev,"GOV_INTERVAL event.\n");
+		devfreq_interval_update(devfreq_dev, (unsigned int *)data);
+		break;
+#endif
+
+	case DEVFREQ_GOV_SUSPEND:
+		dev_info(&devfreq_dev->dev,"GOV_SUSPEND event.\n");
+		pvr_governor_suspend(devfreq_dev);
+		devfreq_monitor_suspend(devfreq_dev);
+		break;
+
+	case DEVFREQ_GOV_RESUME:
+		dev_info(&devfreq_dev->dev,"GOV_RESUME event.\n");
+		devfreq_monitor_resume(devfreq_dev);
+		pvr_governor_resume(devfreq_dev);
+		break;
+
+	default:
+		printk("Unknown event.\n");
+		break;
+	}
+
+	return 0;
+}
+
+static struct devfreq_governor pvr_balanced_governor = {
+	.name = DEVFREQ_GOV_PVR_BALANCED,
+	.get_target_freq = pvr_governor_get_target,
+	.event_handler   = pvr_governor_event_handler,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	.attrs = DEVFREQ_GOV_ATTR_POLLING_INTERVAL
+		| DEVFREQ_GOV_ATTR_TIMER,
+#else
+	.immutable = true,
+#endif
+};
+
+int pvr_governor_init(void)
+{
+	int ret;
+
+	ret = devfreq_add_governor(&pvr_balanced_governor);
+	if (ret)
+	{
+		pr_err("%s: failed to install governor %d\n", __func__, ret);
+	}
+
+	return ret;
+}
+
+void pvr_governor_exit(void)
+{
+	int ret;
+
+	ret = devfreq_remove_governor(&pvr_balanced_governor);
+	if (ret)
+	{
+		pr_err("Failed to remove governor (%u)\n", ret);
+	}
+}
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+
+static int pvr_get_dev_status_get_events(struct pvr_profiling_device *pvr_prof_dev,
+										 struct pvr_profiling_dev_status *pvr_stat)
+{
+	struct devfreq_event_data event_data;
+	int i, ret = 0;
+
+	for (i = 0; i < pvr_prof_dev->num_events; i++)
+	{
+		if (!pvr_prof_dev->edev[i])
+			continue;
+
+		ret = devfreq_event_get_event(pvr_prof_dev->edev[i], &event_data);
+		if (ret < 0)
+			return ret;
+
+		pvr_stat->event_data[i].load_count = event_data.load_count;
+		pvr_stat->event_data[i].total_count = event_data.total_count;
+	}
+
+	return ret;
+}
+
+static int pvr_governor_reset_events(struct pvr_profiling_device *pvr_prof_dev)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < pvr_prof_dev->num_events; i++)
+	{
+		if (!pvr_prof_dev->edev[i])
+			continue;
+
+		ret = devfreq_event_set_event(pvr_prof_dev->edev[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int pvr_events_enable(struct devfreq_event_dev *edev)
+{
+	/* Enable counters */
+
+	dev_dbg(&edev->dev, "%s (Enable)\n", edev->desc->name);
+
+	return 0;
+}
+
+static int pvr_events_disable(struct devfreq_event_dev *edev)
+{
+	/* Disable counters */
+
+	dev_dbg(&edev->dev, "%s (Disable)\n", edev->desc->name);
+
+	return 0;
+}
+
+static int pvr_find_event_by_name(const char *name)
+{
+	int idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(g_pvr_governor_events); idx++)
+		if (strcmp(name, g_pvr_governor_events[idx].name) == 0)
+			return idx;
+
+	return -EINVAL;
+}
+
+static int pvr_governor_set_event(struct devfreq_event_dev *edev)
+{
+	int id = pvr_find_event_by_name(edev->desc->name);
+	if (id < 0)
+		return -EINVAL;
+
+	/* clear the last value */
+	g_pvr_governor_events[id].last_value = 0;
+
+	return 0;
+}
+
+static int pvr_governor_get_event(struct devfreq_event_dev *edev,
+				struct devfreq_event_data *edata)
+{
+	struct pvr_profiling_device *info = devfreq_event_get_drvdata(edev);
+	volatile unsigned long perf_ctr;
+	int id = pvr_find_event_by_name(edev->desc->name);
+	if (id < 0)
+		return -EINVAL;
+
+	/*
+	 * Read performance counter
+	 */
+	perf_ctr = OSReadHWReg32(info->pvRegsBaseKM, g_pvr_governor_events[id].cntr);
+
+	/* Calculate deltas */
+	if (perf_ctr >= g_pvr_governor_events[id].last_value)
+	{
+		edata->load_count = perf_ctr - g_pvr_governor_events[id].last_value;
+	}
+	else
+	{
+		edata->load_count = UINT32_MAX - g_pvr_governor_events[id].last_value + perf_ctr + 1;
+	}
+
+	g_pvr_governor_events[id].last_value = perf_ctr;
+
+	/*
+	 * Calc the utilisation as a fraction of the total cycles
+	 */
+	if (edev->desc->event_type & PVR_EVENT_OP_PERF_CNTR)
+	{
+		edata->total_count = info->stat->event_data[0].load_count;
+
+		/*
+		 * Timer has 256-cycle granularity
+		 * Cycle counters have 1-cycle or 256-cycle granularity depending on GPU variant
+		 * SLC read/write counters have 1 event granularity
+		 */
+		if ((g_pvr_governor_events[0].shift > 0) && (g_pvr_governor_events[id].shift == 0))
+		{
+			int timer_shift = g_pvr_governor_events[0].shift;
+			if (edata->total_count < (UINT32_MAX >> timer_shift))
+			{
+				edata->total_count <<= timer_shift;	/*!< convert total to cycles */
+			}
+			else
+			{
+				/* avoid overflow but some loss of precision will occur */
+				edata->load_count >>= timer_shift;	/*!< convert load to ticks */
+			}
+		}
+	}
+	else
+	{
+		/* timer runs continuously, effective load is 100% */
+		edata->total_count = edata->load_count;
+	}
+
+	return 0;
+}
+
+static const struct devfreq_event_ops pvr_governor_event_ops = {
+	.enable = pvr_events_enable,
+	.disable = pvr_events_disable,
+	.set_event = pvr_governor_set_event,
+	.get_event = pvr_governor_get_event,
+};
+
+int pvr_events_register(struct device *dev, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice;
+	struct pvr_profiling_device *pvr_prof_dev;
+	struct devfreq_event_dev **edev;
+	struct devfreq_event_desc *desc;
+	int size, err;
+	int idx;
+
+	if (!psDeviceNode)
+		return -ENODEV;
+
+	pvr_prof_dev = devm_kzalloc(dev, sizeof(*pvr_prof_dev), GFP_KERNEL);
+	if (!pvr_prof_dev)
+		return -ENOMEM;
+
+	pvr_prof_dev->dev = dev;
+	pvr_prof_dev->num_events = ARRAY_SIZE(g_pvr_governor_events);
+
+	/* Alloc the event descriptor table */
+	desc = devm_kcalloc(dev, pvr_prof_dev->num_events, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	for (idx = 0; idx < pvr_prof_dev->num_events; idx++)
+	{
+		if (!g_pvr_governor_events[idx].name)
+			continue;
+
+		desc[idx].name = g_pvr_governor_events[idx].name;
+		desc[idx].event_type = (idx < PERF_CNTR_OFF) ? PVR_EVENT_OP_TIMER : PVR_EVENT_OP_PERF_CNTR;
+		desc[idx].event_type |= PVR_EVENT_OP_PERF_VER;
+		desc[idx].driver_data = pvr_prof_dev;
+		desc[idx].ops = &pvr_governor_event_ops;
+	}
+	pvr_prof_dev->desc = desc;
+
+	size = sizeof(struct devfreq_event_dev *) * pvr_prof_dev->num_events;
+	pvr_prof_dev->edev = devm_kzalloc(dev, size, GFP_KERNEL);
+	if (!pvr_prof_dev->edev)
+		return -ENOMEM;
+
+	/* Populate the events */
+	for (idx = 0, edev = pvr_prof_dev->edev; idx < pvr_prof_dev->num_events; idx++)
+	{
+		edev[idx] = devm_devfreq_event_add_edev(dev, &desc[idx]);
+		if (IS_ERR(edev[idx]))
+		{
+			dev_err(dev, "failed to add devfreq-event device (idx=%d)\n", idx);
+			return PTR_ERR(edev[idx]);
+		}
+
+		/* enable */
+		err = devfreq_event_enable_edev(edev[idx]);
+		if (err)
+		{
+			dev_err(dev, "failed to enable devfreq-event device (idx=%d)\n", idx);
+			return PTR_ERR(edev[idx]);
+		}
+
+		dev_info(dev, "%s: new PVR devfreq-event device registered %s (%s)\n",
+			__func__, dev_name(dev), desc[idx].name);
+	}
+
+	/* Allocate the profiling stats, passed to governor */
+	pvr_prof_dev->stat = devm_kzalloc(dev, sizeof(struct pvr_profiling_dev_status), GFP_KERNEL);
+	if (!pvr_prof_dev->stat)
+		return -ENOMEM;
+
+	/* Map the reg bank */
+	pvr_prof_dev->reg_size = psDeviceNode->psDevConfig->ui32RegsSize;
+	pvr_prof_dev->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvr_prof_dev->pvRegsBaseKM)
+	{
+		dev_err(dev, "failed to map register bank.");
+		return -ENODEV;
+	}
+
+	dev_info(dev, "Mapped regbank at %p, size 0x%x Bytes.\n", pvr_prof_dev->pvRegsBaseKM,
+			pvr_prof_dev->reg_size);
+
+	/* Update the DVFS device */
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDevice->psProfilingDevice = pvr_prof_dev;
+
+	return 0;
+}
+
+void pvr_events_unregister(struct device *dev, IMG_DVFS_DEVICE *psDVFSDevice)
+{
+	struct pvr_profiling_device *pvr_prof_dev = psDVFSDevice->psProfilingDevice;
+	struct devfreq_event_dev **edev;
+	int idx, err;
+
+	/* Remove the events */
+	for (idx = 0, edev = pvr_prof_dev->edev; idx < pvr_prof_dev->num_events; idx++)
+	{
+		err = devfreq_event_disable_edev(edev[idx]);
+		if (err)
+		{
+			dev_warn(dev, "failed to disable devfreq-event device (idx=%d)\n", idx);
+		}
+		devm_devfreq_event_remove_edev(dev, edev[idx]);
+	}
+
+	/* Unmap the register bank */
+	if (pvr_prof_dev != NULL && pvr_prof_dev->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) pvr_prof_dev->pvRegsBaseKM,
+		                 pvr_prof_dev->reg_size);
+	}
+
+	/* devfreq_event resources are managed by the kernel */
+}
+#endif /* CONFIG_PM_DEVFREQ_EVENT */
+
+#endif /* SUPPORT_PVR_DVFS_GOVERNOR */
+
+static int FillOPPTable(struct device *dev, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	const IMG_OPP *iopp;
+	int i, err = 0;
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = NULL;
+
+	/* Check the device exists */
+	if (!dev || !psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	if (!psDVFSDeviceCfg->pasOPPTable)
+	{
+		dev_err(dev, "No DVFS OPP table provided in system layer and no device tree support.");
+		return -ENODATA;
+	}
+
+	for (i = 0, iopp = psDVFSDeviceCfg->pasOPPTable;
+	     i < psDVFSDeviceCfg->ui32OPPTableSize;
+	     i++, iopp++)
+	{
+		err = dev_pm_opp_add(dev, iopp->ui32Freq, iopp->ui32Volt);
+		if (err) {
+			dev_err(dev, "Could not add OPP entry, %d\n", err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+static int RegisterCoolingDevice(struct device *dev,
+								 IMG_DVFS_DEVICE *psDVFSDevice,
+								 struct devfreq_cooling_power *powerOps)
+{
+	struct device_node *of_node;
+	int err = 0;
+
+	if (!psDVFSDevice)
+	{
+		return -EINVAL;
+	}
+
+	if (!powerOps)
+	{
+		dev_info(dev, "Cooling: power ops not registered, not enabling cooling");
+		return 0;
+	}
+
+	of_node = of_node_get(dev->of_node);
+
+	psDVFSDevice->psDevfreqCoolingDevice = of_devfreq_cooling_register_power(
+		of_node, psDVFSDevice->psDevFreq, powerOps);
+
+	if (IS_ERR(psDVFSDevice->psDevfreqCoolingDevice))
+	{
+		err = PTR_ERR(psDVFSDevice->psDevfreqCoolingDevice);
+		dev_err(dev, "Failed to register as devfreq cooling device %d", err);
+	}
+
+	of_node_put(of_node);
+
+	return err;
+}
+#endif
+
+#define TO_IMG_ERR(err) ((err == -EPROBE_DEFER) ? PVRSRV_ERROR_PROBE_DEFER : PVRSRV_ERROR_INIT_FAILURE)
+
+PVRSRV_ERROR InitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE        *psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	struct device          *psDev;
+	PVRSRV_ERROR            eError;
+	int                     err;
+
+#if !defined(CONFIG_PM_OPP)
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	PVR_ASSERT(psDeviceNode->psDevConfig);
+
+	if (psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState == PVR_DVFS_STATE_INIT_PENDING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "DVFS initialise pending for device node %p",
+				 psDeviceNode));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState = PVR_DVFS_STATE_INIT_PENDING;
+
+#if defined(SUPPORT_SOC_TIMER)
+	if (! psDeviceNode->psDevConfig->pfnSoCTimerRead)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "System layer SoC timer callback not implemented"));
+		//return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	eError = SORgxGpuUtilStatsRegister(&psDVFSDevice->hGpuUtilUserDVFS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to register to the GPU utilisation stats, %d", eError));
+		return eError;
+	}
+
+	err = dev_pm_opp_of_add_table(psDev);
+	if (err == 0)
+	{
+		psDVFSDeviceCfg->bDTConfig = IMG_TRUE;
+	}
+	else
+	{
+		/*
+		 * If there are no device tree or system layer provided operating points
+		 * then return an error
+		 */
+		if ((err == -ENOTSUPP || err == -ENODEV) && psDVFSDeviceCfg->pasOPPTable)
+		{
+			err = FillOPPTable(psDev, psDeviceNode);
+			if (err != 0 && err != -ENODATA)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to fill OPP table with data, %d", err));
+				eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+				goto err_exit;
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to init opp table from devicetree, %d", err));
+			eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+			goto err_exit;
+		}
+	}
+
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+	err = pvr_events_register(psDev, psDeviceNode);
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to init PVR profiling events, %d", err));
+		goto err_exit;
+	}
+#endif
+
+	err = pvr_governor_init();
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to init PVR governor, %d", err));
+		goto err_exit;
+	}
+	psDVFSDevice->bGovernorReady = true;
+#endif
+
+	PVR_ASSERT(psDev);
+	PVR_ASSERT(psDeviceNode);
+
+	PVR_TRACE(("PVR DVFS init pending: dev = %p, PVR device = %p",
+			   psDev, psDeviceNode));
+
+	return PVRSRV_OK;
+
+err_exit:
+	DeinitDVFS(psDeviceNode);
+	return eError;
+}
+
+PVRSRV_ERROR RegisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE        *psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	IMG_DVFS_GOVERNOR_CFG  *psDVFSGovernorCfg = NULL;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = NULL;
+	struct device          *psDev;
+	struct pvr_opp_freq_table	pvr_freq_table = {0};
+	unsigned long           min_freq = 0, max_freq = 0, min_volt = 0;
+	PVRSRV_ERROR            eError;
+	int                     err;
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState != PVR_DVFS_STATE_INIT_PENDING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "DVFS initialise not yet pending for device node %p",
+				 psDeviceNode));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	psDVFSGovernorCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSGovernorCfg;
+	psRGXTimingInfo = ((RGX_DATA *)psDeviceNode->psDevConfig->hDevData)->psRGXTimingInfo;
+	psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState = PVR_DVFS_STATE_READY;
+
+	err = GetOPPValues(psDev, &min_freq, &min_volt, &max_freq, &pvr_freq_table);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to read OPP points, %d", err));
+		eError = TO_IMG_ERR(err);
+		goto err_exit;
+	}
+
+	img_devfreq_dev_profile.freq_table = pvr_freq_table.freq_table;
+	img_devfreq_dev_profile.max_state = pvr_freq_table.num_levels;
+	img_devfreq_dev_profile.initial_freq = min_freq;
+	img_devfreq_dev_profile.polling_ms = psDVFSDeviceCfg->ui32PollMs;
+
+	psRGXTimingInfo->ui32CoreClockSpeed = min_freq;
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDeviceNode->psDevConfig->hSysData, min_freq);
+	psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, min_volt);
+
+#if !defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* Use the Linux 'simple_ondemand' governor */
+	psDVFSDevice->data.upthreshold = psDVFSGovernorCfg->ui32UpThreshold;
+	psDVFSDevice->data.downdifferential = psDVFSGovernorCfg->ui32DownDifferential;
+#endif
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	psDVFSDevice->data.ui32UpThreshold = psDVFSGovernorCfg->ui32UpThreshold;
+	psDVFSDevice->data.ui32DownDifferential = psDVFSGovernorCfg->ui32DownDifferential;
+
+#if defined(SUPPORT_RGX)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_MEMBUS))
+		{
+			psDVFSDevice->data.uiNumMembus = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_MEMBUS);
+		}
+		else
+		{
+			psDVFSDevice->data.uiNumMembus = 1;
+		}
+	}
+#endif
+#endif
+
+	psDVFSDevice->psDevFreq = devm_devfreq_add_device(psDev,
+													  &img_devfreq_dev_profile,
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+													  "pvr_balanced",
+													  &psDVFSDevice->data);
+#else
+													  "simple_ondemand",
+													  &psDVFSDevice->data);
+#endif
+
+	if (IS_ERR(psDVFSDevice->psDevFreq))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "Failed to add as devfreq device %p, %ld",
+				 psDVFSDevice->psDevFreq,
+				 PTR_ERR(psDVFSDevice->psDevFreq)));
+		eError = TO_IMG_ERR(PTR_ERR(psDVFSDevice->psDevFreq));
+		goto err_exit;
+	}
+#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 50)))
+	/* Handle Linux kernel bug where a NULL return can occur. */
+	if (psDVFSDevice->psDevFreq == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "Failed to add as devfreq device %p, NULL return",
+				 psDVFSDevice->psDevFreq));
+		eError = TO_IMG_ERR(-EINVAL);
+		goto err_exit;
+	}
+#endif
+
+	eError = SuspendDVFS(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVInit: Failed to suspend DVFS"));
+		goto err_exit;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+	psDVFSDevice->psDevFreq->scaling_min_freq = min_freq;
+	psDVFSDevice->psDevFreq->scaling_max_freq = max_freq;
+#else
+	psDVFSDevice->psDevFreq->min_freq = min_freq;
+	psDVFSDevice->psDevFreq->max_freq = max_freq;
+#endif
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	psDVFSDevice->psDevFreq->last_status.private_data =
+		(void *)psDVFSDevice->psProfilingDevice->stat;
+#endif
+
+	err = devfreq_register_opp_notifier(psDev, psDVFSDevice->psDevFreq);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to register opp notifier, %d", err));
+		eError = TO_IMG_ERR(err);
+		goto err_exit;
+	}
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		err = RegisterCoolingDevice(psDev, psDVFSDevice, psDVFSDeviceCfg->psPowerOps);
+		if (err)
+		{
+			eError = TO_IMG_ERR(err);
+			goto err_exit;
+		}
+	}
+#endif
+
+	PVR_TRACE(("PVR DVFS activated: %lu-%lu Hz, Period: %ums",
+			   min_freq,
+			   max_freq,
+			   psDVFSDeviceCfg->ui32PollMs));
+
+	return PVRSRV_OK;
+
+err_exit:
+	UnregisterDVFSDevice(psDeviceNode);
+	return eError;
+}
+
+void UnregisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice = NULL;
+	struct device *psDev = NULL;
+	IMG_INT32 i32Error;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+	if (! psDVFSDevice)
+	{
+		return;
+	}
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	if (!IS_ERR_OR_NULL(psDVFSDevice->psDevfreqCoolingDevice))
+	{
+		devfreq_cooling_unregister(psDVFSDevice->psDevfreqCoolingDevice);
+		psDVFSDevice->psDevfreqCoolingDevice = NULL;
+	}
+#endif
+
+	if (psDVFSDevice->psDevFreq)
+	{
+		i32Error = devfreq_unregister_opp_notifier(psDev, psDVFSDevice->psDevFreq);
+		if (i32Error < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to unregister OPP notifier"));
+		}
+
+		devm_devfreq_remove_device(psDev, psDVFSDevice->psDevFreq);
+		psDVFSDevice->psDevFreq = NULL;
+	}
+
+	psDVFSDevice->eState = PVR_DVFS_STATE_DEINIT;
+}
+
+void DeinitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice = NULL;
+	struct device *psDev = NULL;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	if (psDVFSDevice->psProfilingDevice)
+	{
+		pvr_events_unregister(psDev, psDVFSDevice);
+	}
+	if (psDVFSDevice->bGovernorReady)
+	{
+		pvr_governor_exit();
+		psDVFSDevice->bGovernorReady = false;
+	}
+#endif
+
+	/*
+	 * Remove OPP entries for this device; both static entries from
+	 * the device tree and dynamic entries.
+	 */
+	dev_pm_opp_remove_table(psDev);
+
+	SORgxGpuUtilStatsUnregister(psDVFSDevice->hGpuUtilUserDVFS);
+	psDVFSDevice->hGpuUtilUserDVFS = NULL;
+	psDVFSDevice->eState = PVR_DVFS_STATE_NONE;
+}
+
+PVRSRV_ERROR SuspendDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE	*psDVFSDevice = NULL;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	if (psDVFSDevice->eState == PVR_DVFS_STATE_DEINIT)
+	{
+		/* Device is shutting down, nothing to do. */
+		return PVRSRV_OK;
+	}
+	psDVFSDevice->eState = PVR_DVFS_STATE_OFF;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* Communicate power suspend to devfreq framework */
+	err = devfreq_suspend_device(psDVFSDevice->psDevFreq);
+	if (err < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Failed to suspend DVFS (%d)", err));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR ResumeDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE	*psDVFSDevice = NULL;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+
+	/* Not supported in GuestOS drivers */
+	psDVFSDevice->eState = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? PVR_DVFS_STATE_NONE : PVR_DVFS_STATE_READY;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Communicate power resume to devfreq framework */
+		err = devfreq_resume_device(psDVFSDevice->psDevFreq);
+		if (err < 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Failed to resume DVFS (%d)", err));
+			return PVRSRV_ERROR_INVALID_DEVICE;
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+#endif /* !NO_HARDWARE */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.h
new file mode 100644
index 000000000000..a246b240d9ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.h
@@ -0,0 +1,62 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs.c
+@Title          System level interface for DVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_DEVICE_H
+#define PVR_DVFS_DEVICE_H
+
+#include "opaque_types.h"
+#include "pvrsrv_error.h"
+
+
+PVRSRV_ERROR InitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+void DeinitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR RegisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+void UnregisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR SuspendDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR ResumeDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+#endif /* PVR_DVFS_DEVICE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.c
new file mode 100644
index 000000000000..9a192b1b6fd1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.c
@@ -0,0 +1,472 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock_types.h>
+#include <linux/atomic.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+#include "pvr_export_fence.h"
+#include "osfunc_common.h"
+
+struct pvr_exp_fence_context {
+	struct kref kref;
+	unsigned int context;
+	char context_name[32];
+	char driver_name[32];
+	atomic_t seqno;
+	atomic_t fence_count;
+
+	void *cmd_complete_handle;
+
+	/* lock for signal and fence lists */
+	spinlock_t list_lock;
+	struct list_head signal_list;
+	struct list_head fence_list;
+};
+
+struct pvr_exp_fence {
+	struct dma_fence base;
+	struct pvr_exp_fence_context *fence_context;
+	PSYNC_CHECKPOINT checkpoint_handle;
+	/* dma_fence fd (needed for hwperf) */
+	int fd;
+	/* Lock for the dma fence */
+	spinlock_t lock;
+	/* fence will point to the dma_fence in base */
+	struct dma_fence *fence;
+	struct list_head fence_head;
+	struct list_head signal_head;
+};
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+static inline bool
+pvr_exp_fence_sync_is_signaled(struct pvr_exp_fence *exp_fence, u32 fence_sync_flags)
+{
+	if (exp_fence->checkpoint_handle) {
+		return SyncCheckpointIsSignalled(exp_fence->checkpoint_handle,
+						 fence_sync_flags);
+	}
+	return false;
+}
+
+const char *pvr_exp_fence_context_name(struct pvr_exp_fence_context *fctx)
+{
+	return fctx->context_name;
+}
+
+void pvr_exp_fence_context_value_str(struct pvr_exp_fence_context *fctx,
+				    char *str, int size)
+{
+	snprintf(str, size, "%d", atomic_read(&fctx->seqno));
+}
+
+static void
+pvr_exp_fence_context_fences_dump(struct pvr_exp_fence_context *fctx,
+				  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				  void *pvDumpDebugFile)
+{
+	unsigned long flags;
+	char value[128];
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	pvr_exp_fence_context_value_str(fctx, value, sizeof(value));
+	PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+			 "exp_fence_ctx: @%s", value);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+}
+
+static inline unsigned
+pvr_exp_fence_context_seqno_next(struct pvr_exp_fence_context *fence_context)
+{
+	if (fence_context)
+		return atomic_inc_return(&fence_context->seqno) - 1;
+	else
+		return 0xfeedface;
+}
+
+static void
+pvr_exp_fence_context_signal_fences(void *data)
+{
+	struct pvr_exp_fence_context *fctx = (struct pvr_exp_fence_context *)data;
+	struct pvr_exp_fence *pvr_exp_fence, *tmp;
+	unsigned long flags1;
+	int chkpt_ct = 0;
+	int chkpt_sig_ct = 0;
+
+	LIST_HEAD(signal_list);
+
+	/*
+	 * We can't call fence_signal while holding the lock as we can end up
+	 * in a situation whereby pvr_fence_foreign_signal_sync, which also
+	 * takes the list lock, ends up being called as a result of the
+	 * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
+	 *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
+	 * the foreign_fence callback is pvr_fence_foreign_signal_sync.
+	 *
+	 * So extract the items we intend to signal and add them to their own
+	 * queue.
+	 */
+	spin_lock_irqsave(&fctx->list_lock, flags1);
+	list_for_each_entry_safe(pvr_exp_fence, tmp, &fctx->signal_list, signal_head) {
+		chkpt_ct++;
+		if (pvr_exp_fence_sync_is_signaled(pvr_exp_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT)) {
+			chkpt_sig_ct++;
+			list_move_tail(&pvr_exp_fence->signal_head, &signal_list);
+		}
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags1);
+
+	list_for_each_entry_safe(pvr_exp_fence, tmp, &signal_list, signal_head) {
+		spin_lock_irqsave(&pvr_exp_fence->fence_context->list_lock, flags1);
+		list_del(&pvr_exp_fence->signal_head);
+		spin_unlock_irqrestore(&pvr_exp_fence->fence_context->list_lock, flags1);
+		dma_fence_signal(pvr_exp_fence->fence);
+		dma_fence_put(pvr_exp_fence->fence);
+	}
+}
+
+static const char *pvr_exp_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		return pvr_exp_fence->fence_context->driver_name;
+	else
+		return "***NO_DRIVER***";
+}
+
+static const char *pvr_exp_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		return pvr_exp_fence_context_name(pvr_exp_fence->fence_context);
+	else
+		return "***NO_TIMELINE***";
+}
+
+static void pvr_exp_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%llu", (u64) fence->seqno);
+}
+
+static void pvr_exp_fence_timeline_value_str(struct dma_fence *fence,
+					    char *str, int size)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		pvr_exp_fence_context_value_str(pvr_exp_fence->fence_context, str, size);
+}
+
+static bool pvr_exp_fence_enable_signaling(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *exp_fence = to_pvr_exp_fence(fence);
+	unsigned long flags;
+
+	if (!exp_fence)
+		return false;
+
+	if (pvr_exp_fence_sync_is_signaled(exp_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return false;
+
+	dma_fence_get(&exp_fence->base);
+
+	spin_lock_irqsave(&exp_fence->fence_context->list_lock, flags);
+	list_add_tail(&exp_fence->signal_head, &exp_fence->fence_context->signal_list);
+	spin_unlock_irqrestore(&exp_fence->fence_context->list_lock, flags);
+
+	return true;
+}
+
+static void pvr_exp_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_exp_fence_context *fence_context =
+		container_of(kref, struct pvr_exp_fence_context, kref);
+	unsigned int fence_count;
+
+	if (WARN_ON(!list_empty_careful(&fence_context->fence_list)))
+		pvr_exp_fence_context_fences_dump(fence_context, NULL, NULL);
+
+	PVRSRVUnregisterCmdCompleteNotify(fence_context->cmd_complete_handle);
+
+	fence_count = atomic_read(&fence_context->fence_count);
+	if (WARN_ON(fence_count))
+		pr_debug("%s context has %u fence(s) remaining\n",
+			 fence_context->context_name, fence_count);
+
+	kfree(fence_context);
+}
+
+static void pvr_exp_fence_release(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+	unsigned long flags;
+
+	if (pvr_exp_fence) {
+		if (pvr_exp_fence->fence_context) {
+			spin_lock_irqsave(&pvr_exp_fence->fence_context->list_lock, flags);
+			list_del(&pvr_exp_fence->fence_head);
+			atomic_dec(&pvr_exp_fence->fence_context->fence_count);
+			spin_unlock_irqrestore(&pvr_exp_fence->fence_context->list_lock, flags);
+
+			kref_put(&pvr_exp_fence->fence_context->kref,
+				 pvr_exp_fence_context_destroy_kref);
+		}
+
+		if (pvr_exp_fence->checkpoint_handle) {
+			SyncCheckpointFree(pvr_exp_fence->checkpoint_handle);
+			pvr_exp_fence->checkpoint_handle = NULL;
+		}
+
+		kfree(pvr_exp_fence);
+	}
+}
+
+static const struct dma_fence_ops pvr_exp_fence_ops = {
+	.get_driver_name = pvr_exp_fence_get_driver_name,
+	.get_timeline_name = pvr_exp_fence_get_timeline_name,
+	.fence_value_str = pvr_exp_fence_value_str,
+	.timeline_value_str = pvr_exp_fence_timeline_value_str,
+	.enable_signaling = pvr_exp_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = pvr_exp_fence_release,
+};
+
+struct pvr_exp_fence_context *
+pvr_exp_fence_context_create(const char *context_name, const char *driver_name)
+{
+	struct pvr_exp_fence_context *fence_context;
+	PVRSRV_ERROR srv_err;
+
+	fence_context = kzalloc(sizeof(*fence_context), GFP_KERNEL);
+	if (!fence_context)
+		return NULL;
+
+	fence_context->context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fence_context->context_name, context_name,
+		sizeof(fence_context->context_name));
+	OSStringSafeCopy(fence_context->driver_name, driver_name,
+		sizeof(fence_context->driver_name));
+	atomic_set(&fence_context->seqno, 0);
+	atomic_set(&fence_context->fence_count, 0);
+	kref_init(&fence_context->kref);
+
+	spin_lock_init(&fence_context->list_lock);
+	INIT_LIST_HEAD(&fence_context->signal_list);
+	INIT_LIST_HEAD(&fence_context->fence_list);
+
+	srv_err = PVRSRVRegisterCmdCompleteNotify(&fence_context->cmd_complete_handle,
+				pvr_exp_fence_context_signal_fences,
+				fence_context);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register command complete callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+		kfree(fence_context);
+		return NULL;
+	}
+
+	return fence_context;
+}
+
+void pvr_exp_fence_context_destroy(struct pvr_exp_fence_context *fence_context)
+{
+	if (fence_context) {
+		kref_put(&fence_context->kref, pvr_exp_fence_context_destroy_kref);
+	}
+}
+
+struct dma_fence *
+pvr_exp_fence_create(struct pvr_exp_fence_context *fence_context, int fd, u64 *sync_pt_idx)
+{
+	struct pvr_exp_fence *pvr_exp_fence;
+	unsigned long flags;
+	unsigned int seqno;
+	struct pvr_exp_fence_context *pfence_context = fence_context;
+
+	if (WARN_ON(!fence_context))
+		return NULL;
+
+	pvr_exp_fence = kzalloc(sizeof(*pvr_exp_fence), GFP_KERNEL);
+	if (WARN_ON(!pvr_exp_fence))
+		return NULL;
+
+	spin_lock_init(&pvr_exp_fence->lock);
+
+	INIT_LIST_HEAD(&pvr_exp_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_exp_fence->signal_head);
+
+	pvr_exp_fence->fence_context = pfence_context;
+
+	/* No sync checkpoint is assigned until attached to a kick */
+	pvr_exp_fence->checkpoint_handle = NULL;
+
+	seqno = pvr_exp_fence_context_seqno_next(pfence_context);
+
+	pvr_exp_fence->fence = &pvr_exp_fence->base;
+
+	dma_fence_init(&pvr_exp_fence->base, &pvr_exp_fence_ops,
+		       &pvr_exp_fence->lock, pfence_context->context, seqno);
+
+	atomic_inc(&pfence_context->fence_count);
+
+	spin_lock_irqsave(&fence_context->list_lock, flags);
+	list_add_tail(&pvr_exp_fence->fence_head, &fence_context->fence_list);
+	spin_unlock_irqrestore(&fence_context->list_lock, flags);
+
+	kref_get(&pfence_context->kref);
+
+	pvr_exp_fence->fd = fd;
+
+	*sync_pt_idx = pvr_exp_fence_context_seqno_next(pfence_context);
+
+	return &pvr_exp_fence->base;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_exp_fence_assign_checkpoint(PVRSRV_FENCE fence_to_resolve,
+				struct dma_fence *fence,
+				PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+				PSYNC_CHECKPOINT *assigned_checkpoint)
+{
+	struct SYNC_CHECKPOINT_TAG *new_sync_checkpoint;
+	struct pvr_exp_fence *pvr_exp_fence;
+	PVRSRV_FENCE export_fence_fd = fence_to_resolve;
+	PVRSRV_ERROR err;
+
+	pvr_exp_fence = to_pvr_exp_fence(fence);
+	if (!pvr_exp_fence)	{
+		pr_err("%s: Invalid fence_to_resolve\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (pvr_exp_fence->checkpoint_handle) {
+		/* export fence already has a sync checkpoint assigned */
+		*assigned_checkpoint = pvr_exp_fence->checkpoint_handle;
+		return PVRSRV_OK;
+	}
+
+	/* Ensure fd assigned to export fence when it was created is passed
+	 * to SyncCheckpointAlloc() (so correct fd is used in HWPerf event)
+	 * as the export fence may only be part of the check fence.
+	 */
+	if (pvr_exp_fence->fd != PVRSRV_NO_FENCE) {
+		export_fence_fd = pvr_exp_fence->fd;
+	}
+
+	err = SyncCheckpointAlloc(checkpoint_context,
+				  PVRSRV_NO_TIMELINE, export_fence_fd,
+				  fence->ops->get_timeline_name(fence), &new_sync_checkpoint);
+	if (unlikely(err != PVRSRV_OK)) {
+		pr_err("%s: SyncCheckpointAlloc() failed (err%d)\n",
+		       __func__, err);
+		*assigned_checkpoint = NULL;
+		goto err_out;
+	}
+
+	pvr_exp_fence->checkpoint_handle = new_sync_checkpoint;
+	*assigned_checkpoint = new_sync_checkpoint;
+	return PVRSRV_OK;
+
+err_out:
+	return err;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_exp_fence_rollback(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence;
+	PVRSRV_ERROR err;
+
+	pvr_exp_fence = to_pvr_exp_fence(fence);
+	if (!pvr_exp_fence) {
+		pr_err("%s: Invalid fence\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (pvr_exp_fence->checkpoint_handle) {
+		/* Free the assigned sync checkpoint */
+		SyncCheckpointFree(pvr_exp_fence->checkpoint_handle);
+		pvr_exp_fence->checkpoint_handle = NULL;
+	}
+
+	return PVRSRV_OK;
+
+err_out:
+	return err;
+}
+
+bool pvr_is_exp_fence(struct dma_fence *fence)
+{
+	return (fence->ops == &pvr_exp_fence_ops);
+}
+
+struct pvr_exp_fence *to_pvr_exp_fence(struct dma_fence *fence)
+{
+	if (pvr_is_exp_fence(fence))
+		return container_of(fence, struct pvr_exp_fence, base);
+
+	return NULL;
+}
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_exp_fence_get_checkpoint(struct pvr_exp_fence *export_fence)
+{
+	if (export_fence) {
+		return export_fence->checkpoint_handle;
+	}
+	else
+		return NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.h
new file mode 100644
index 000000000000..0ffe8a11ce40
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.h
@@ -0,0 +1,77 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_EXPORT_FENCES_H__)
+#define __PVR_EXPORT_FENCES_H__
+
+#include "pvr_linux_fence.h"
+#include "services_kernel_client.h"
+
+struct pvr_exp_fence_context;
+struct pvr_exp_fence;
+
+struct pvr_exp_fence_context *pvr_exp_fence_context_create(const char *name,
+				const char *driver_name);
+void pvr_exp_fence_context_destroy(struct pvr_exp_fence_context *fence_context);
+struct dma_fence *pvr_exp_fence_create(struct pvr_exp_fence_context *fence_context,
+				       int fd,
+				       u64 *sync_pt_idx);
+
+const char *pvr_exp_fence_context_name(struct pvr_exp_fence_context *fctx);
+void pvr_exp_fence_context_value_str(struct pvr_exp_fence_context *fctx,
+				    char *str, int size);
+
+enum PVRSRV_ERROR_TAG pvr_exp_fence_assign_checkpoint(PVRSRV_FENCE fence_to_resolve,
+						      struct dma_fence *fence,
+						      PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+						      PSYNC_CHECKPOINT *assigned_checkpoint);
+
+enum PVRSRV_ERROR_TAG pvr_exp_fence_rollback(struct dma_fence *fence);
+
+bool pvr_is_exp_fence(struct dma_fence *fence);
+
+struct pvr_exp_fence *to_pvr_exp_fence(struct dma_fence *fence);
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_exp_fence_get_checkpoint(struct pvr_exp_fence *export_fence);
+
+#endif /* !defined(__PVR_EXPORT_FENCES_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.c
new file mode 100644
index 000000000000..130c68d825fc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.c
@@ -0,0 +1,1166 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include "pvr_fence.h"
+#include "services_kernel_client.h"
+#include "sync_checkpoint_external.h"
+#include "osfunc_common.h"
+
+#define CREATE_TRACE_POINTS
+#include "pvr_fence_trace.h"
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+/* Global kmem_cache for pvr_fence object allocations */
+static struct kmem_cache *pvr_fence_cache;
+static DEFINE_MUTEX(pvr_fence_cache_mutex);
+static u32 pvr_fence_cache_refcount;
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+static inline void
+pvr_fence_sync_signal(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
+{
+	SyncCheckpointSignal(pvr_fence->sync_checkpoint, fence_sync_flags);
+}
+
+static inline bool
+pvr_fence_sync_is_signaled(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
+{
+	return SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
+					 fence_sync_flags);
+}
+
+static inline u32
+pvr_fence_sync_value(struct pvr_fence *pvr_fence)
+{
+	if (SyncCheckpointIsErrored(pvr_fence->sync_checkpoint,
+				    PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return PVRSRV_SYNC_CHECKPOINT_ERRORED;
+	else if (SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
+					   PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+	else
+		return PVRSRV_SYNC_CHECKPOINT_ACTIVE;
+}
+
+static void
+pvr_fence_context_check_status(struct work_struct *data)
+{
+	PVRSRVCheckStatus(NULL);
+}
+
+void
+pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size)
+{
+	snprintf(str, size,
+		 "%u ctx=%llu refs=%u",
+		 atomic_read(&fctx->fence_seqno),
+		 fctx->fence_context,
+		 refcount_read(&fctx->kref.refcount));
+}
+
+static void
+pvr_fence_context_fences_dump(struct pvr_fence_context *fctx,
+			      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			      void *pvDumpDebugFile)
+{
+	struct pvr_fence *pvr_fence;
+	unsigned long flags;
+	char value[128];
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	pvr_context_value_str(fctx, value, sizeof(value));
+	PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+			 "%s: @%s", fctx->name, value);
+	list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
+		struct dma_fence *fence = pvr_fence->fence;
+		const char *timeline_value_str = "unknown timeline value";
+		const char *fence_value_str = "unknown fence value";
+
+		pvr_fence->base.ops->fence_value_str(&pvr_fence->base, value,
+						     sizeof(value));
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " @%s", value);
+
+		if (is_pvr_fence(fence))
+			continue;
+
+		if (fence->ops->timeline_value_str) {
+			fence->ops->timeline_value_str(fence, value,
+						       sizeof(value));
+			timeline_value_str = value;
+		}
+
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " | %s: %s (driver: %s)",
+				  fence->ops->get_timeline_name(fence),
+				  timeline_value_str,
+				  fence->ops->get_driver_name(fence));
+
+		if (fence->ops->fence_value_str) {
+			fence->ops->fence_value_str(fence, value,
+						    sizeof(value));
+			fence_value_str = value;
+		}
+
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " |  @%s (foreign)", fence_value_str);
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+}
+
+static inline unsigned int
+pvr_fence_context_seqno_next(struct pvr_fence_context *fctx)
+{
+	return atomic_inc_return(&fctx->fence_seqno) - 1;
+}
+
+/* This function prepends seqno to fence name */
+static inline void
+pvr_fence_prepare_name(char *fence_name, size_t fence_name_size,
+		const char *name, unsigned int seqno)
+{
+	unsigned int len;
+
+	len = OSStringUINT32ToStr(fence_name, fence_name_size, seqno);
+	if (likely((len > 0) && (fence_name_size >= (len + 1)))) {
+		fence_name[len] = '-';
+		fence_name[len + 1] = '\0';
+	}
+	strlcat(fence_name, name, fence_name_size);
+}
+
+static void
+pvr_fence_sched_free(struct rcu_head *rcu)
+{
+	struct pvr_fence *pvr_fence = container_of(rcu, struct pvr_fence, rcu);
+
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+}
+
+static inline void
+pvr_fence_context_free_deferred(struct pvr_fence_context *fctx)
+{
+	struct pvr_fence *pvr_fence, *tmp;
+	LIST_HEAD(deferred_free_list);
+	unsigned long flags;
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_for_each_entry_safe(pvr_fence, tmp,
+				 &fctx->deferred_free_list,
+				 fence_head)
+		list_move(&pvr_fence->fence_head, &deferred_free_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+	list_for_each_entry_safe(pvr_fence, tmp,
+				 &deferred_free_list,
+				 fence_head) {
+		list_del(&pvr_fence->fence_head);
+		SyncCheckpointFree(pvr_fence->sync_checkpoint);
+		call_rcu(&pvr_fence->rcu, pvr_fence_sched_free);
+		module_put(THIS_MODULE);
+	}
+}
+
+void
+pvr_fence_context_free_deferred_callback(void *data)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+
+	/*
+	 * Free up any fence objects we have deferred freeing.
+	 */
+	pvr_fence_context_free_deferred(fctx);
+}
+
+static void
+pvr_fence_context_signal_fences(void *data)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+	struct pvr_fence *pvr_fence, *tmp;
+	unsigned long flags1;
+
+	LIST_HEAD(signal_list);
+
+	/*
+	 * We can't call fence_signal while holding the lock as we can end up
+	 * in a situation whereby pvr_fence_foreign_signal_sync, which also
+	 * takes the list lock, ends up being called as a result of the
+	 * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
+	 *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
+	 * the foreign_fence callback is pvr_fence_foreign_signal_sync.
+	 *
+	 * So extract the items we intend to signal and add them to their own
+	 * queue.
+	 */
+	spin_lock_irqsave(&fctx->list_lock, flags1);
+	list_for_each_entry_safe(pvr_fence, tmp, &fctx->signal_list, signal_head) {
+		if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+			list_move_tail(&pvr_fence->signal_head, &signal_list);
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags1);
+
+	list_for_each_entry_safe(pvr_fence, tmp, &signal_list, signal_head) {
+
+		PVR_FENCE_TRACE(&pvr_fence->base, "signalled fence (%s)\n",
+				pvr_fence->name);
+		trace_pvr_fence_signal_fence(pvr_fence);
+		spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags1);
+		list_del(&pvr_fence->signal_head);
+		spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags1);
+		dma_fence_signal(pvr_fence->fence);
+		dma_fence_put(pvr_fence->fence);
+	}
+
+	/*
+	 * Take this opportunity to free up any fence objects we
+	 * have deferred freeing.
+	 */
+	pvr_fence_context_free_deferred(fctx);
+}
+
+void
+pvr_fence_context_signal_fences_nohw(void *data)
+{
+	pvr_fence_context_signal_fences(data);
+}
+
+static void
+pvr_fence_context_destroy_internal(struct pvr_fence_context *fctx)
+{
+	pvr_fence_context_free_deferred(fctx);
+
+	if (WARN_ON(!list_empty_careful(&fctx->fence_list)))
+		pvr_fence_context_fences_dump(fctx, NULL, NULL);
+
+	PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
+
+	// wait for all fences to be freed before kmem_cache_destroy() is called
+	rcu_barrier();
+
+	/* Destroy pvr_fence object cache, if no one is using it */
+	WARN_ON(pvr_fence_cache == NULL);
+	mutex_lock(&pvr_fence_cache_mutex);
+	if (--pvr_fence_cache_refcount == 0)
+		kmem_cache_destroy(pvr_fence_cache);
+	mutex_unlock(&pvr_fence_cache_mutex);
+
+	kfree(fctx);
+}
+
+static void
+pvr_fence_context_unregister_dbg(void *dbg_request_handle)
+{
+	PVRSRVUnregisterDeviceDbgRequestNotify(dbg_request_handle);
+}
+
+static void
+pvr_fence_foreign_context_destroy_work(struct work_struct *data)
+{
+	struct pvr_fence_context *fctx =
+		container_of(data, struct pvr_fence_context, destroy_work);
+
+	pvr_fence_context_destroy_internal(fctx);
+}
+
+static void
+pvr_fence_context_destroy_work(struct work_struct *data)
+{
+	struct pvr_fence_context *fctx =
+		container_of(data, struct pvr_fence_context, destroy_work);
+
+	pvr_fence_context_unregister_dbg(fctx->dbg_request_handle);
+	pvr_fence_context_destroy_internal(fctx);
+}
+
+static void
+pvr_fence_context_debug_request(void *data, u32 verbosity,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+		pvr_fence_context_fences_dump(fctx, pfnDumpDebugPrintf,
+					      pvDumpDebugFile);
+}
+
+static struct pvr_fence_context *
+pvr_fence_context_create_internal(struct workqueue_struct *fence_status_wq,
+			const char *name,
+			work_func_t destroy_callback)
+{
+	struct pvr_fence_context *fctx;
+	PVRSRV_ERROR srv_err;
+
+	fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	if (!fctx)
+		return NULL;
+
+	spin_lock_init(&fctx->lock);
+	atomic_set(&fctx->fence_seqno, 0);
+	INIT_WORK(&fctx->check_status_work, pvr_fence_context_check_status);
+	INIT_WORK(&fctx->destroy_work, destroy_callback);
+	spin_lock_init(&fctx->list_lock);
+	INIT_LIST_HEAD(&fctx->signal_list);
+	INIT_LIST_HEAD(&fctx->fence_list);
+	INIT_LIST_HEAD(&fctx->deferred_free_list);
+
+	fctx->fence_wq = fence_status_wq;
+
+	fctx->fence_context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fctx->name, name, sizeof(fctx->name));
+
+	srv_err = PVRSRVRegisterCmdCompleteNotify(&fctx->cmd_complete_handle,
+				pvr_fence_context_signal_fences,
+				fctx);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register command complete callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+		goto err_free_fctx;
+	}
+
+	/* Create pvr_fence object cache, if not already created */
+	mutex_lock(&pvr_fence_cache_mutex);
+	if (pvr_fence_cache_refcount == 0) {
+		pvr_fence_cache = KMEM_CACHE(pvr_fence, 0);
+		if (!pvr_fence_cache) {
+			pr_err("%s: failed to allocate pvr_fence cache\n",
+					__func__);
+			mutex_unlock(&pvr_fence_cache_mutex);
+			goto err_unregister_cmd_complete_notify;
+		}
+	}
+	pvr_fence_cache_refcount++;
+	mutex_unlock(&pvr_fence_cache_mutex);
+
+	kref_init(&fctx->kref);
+
+	PVR_FENCE_CTX_TRACE(fctx, "created fence context (%s)\n", name);
+	trace_pvr_fence_context_create(fctx);
+
+	return fctx;
+
+err_unregister_cmd_complete_notify:
+	PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
+err_free_fctx:
+	kfree(fctx);
+	return NULL;
+}
+
+/**
+ * pvr_fence_context_register_dbg - registers the debug handler for a
+ * fence context
+ *
+ * @dbg_request_handle: handle used to keep a reference for deregister
+ * @dev: device to attach the debug notifier.
+ * @pvr_fence_context: context used as data to the callback for debug
+ *
+ * Registers a debug notifier for a given context for a given device.
+ *
+ * Returns PVRSRV_OK if successful.
+ */
+PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
+				void *dev,
+				struct pvr_fence_context *fctx)
+{
+	PVRSRV_ERROR srv_err;
+
+	srv_err = PVRSRVRegisterDeviceDbgRequestNotify(dbg_request_handle,
+				dev,
+				pvr_fence_context_debug_request,
+				DEBUG_REQUEST_LINUXFENCE,
+				fctx);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+	}
+
+	return srv_err;
+}
+
+/**
+ * pvr_fence_foreign_context_create - creates a PVR fence context
+ * @fence_status_wq: linux workqueue used to signal foreign fences
+ * @name: context name (used for debugging)
+ *
+ * Creates a PVR foreign fence context that can be used to create PVR fences
+ * or to create PVR fences from an existing fence.
+ *
+ * pvr_fence_context_destroy should be called to clean up the fence context.
+ *
+ * Returns NULL if a context cannot be created.
+ */
+struct pvr_fence_context *
+pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
+		const char *name)
+{
+	return pvr_fence_context_create_internal(fence_status_wq, name,
+							pvr_fence_foreign_context_destroy_work);
+}
+
+/**
+ * pvr_fence_context_create - creates a PVR fence context
+ * @dev_cookie: services device cookie
+ * @fence_status_wq: Status workqueue to queue fence update CBs.
+ * @name: context name (used for debugging)
+ *
+ * Creates a PVR fence context that can be used to create PVR fences or to
+ * create PVR fences from an existing fence.
+ *
+ * pvr_fence_context_destroy should be called to clean up the fence context.
+ *
+ * Returns NULL if a context cannot be created.
+ */
+struct pvr_fence_context *
+pvr_fence_context_create(void *dev_cookie,
+			 struct workqueue_struct *fence_status_wq,
+			 const char *name)
+{
+	struct pvr_fence_context *fctx;
+	PVRSRV_ERROR eError;
+
+	fctx = pvr_fence_context_create_internal(fence_status_wq, name,
+					pvr_fence_context_destroy_work);
+	if (fctx == NULL) {
+		pr_err("%s: failed to create fence context", __func__);
+		goto err_out;
+	}
+
+	fctx->dev_cookie = dev_cookie;
+
+	eError = pvr_fence_context_register_dbg(&fctx->dbg_request_handle,
+					dev_cookie,
+					fctx);
+	if (eError != PVRSRV_OK) {
+		pr_err("%s: failed to register fence context debug (%s)\n",
+		       __func__, PVRSRVGetErrorString(eError));
+		goto err_destroy_ctx;
+	}
+
+	PVR_FENCE_CTX_TRACE(fctx,
+						"%s: Created fence context (%s)[%p], dev_cookie %p\n",
+						__func__, fctx->name, fctx, dev_cookie);
+
+	return fctx;
+
+err_destroy_ctx:
+	pvr_fence_context_destroy(fctx);
+err_out:
+	return NULL;
+}
+
+static void pvr_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_fence_context *fctx =
+		container_of(kref, struct pvr_fence_context, kref);
+
+	PVR_FENCE_CTX_TRACE(fctx, "destroyed fence context (%s)\n", fctx->name);
+
+	trace_pvr_fence_context_destroy_kref(fctx);
+
+	queue_work(NativeSyncGetFenceCtxDestroyWq(), &fctx->destroy_work);
+}
+
+/**
+ * pvr_fence_context_destroy - destroys a context
+ * @fctx: PVR fence context to destroy
+ *
+ * Destroys a PVR fence context with the expectation that all fences have been
+ * destroyed.
+ */
+void
+pvr_fence_context_destroy(struct pvr_fence_context *fctx)
+{
+	trace_pvr_fence_context_destroy(fctx);
+
+	kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+}
+
+static const char *
+pvr_fence_get_driver_name(struct dma_fence *fence)
+{
+	return PVR_LDM_DRIVER_REGISTRATION_NAME;
+}
+
+static const char *
+pvr_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		return pvr_fence->fctx->name;
+	return NULL;
+}
+
+static
+void pvr_fence_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (!pvr_fence)
+		return;
+
+	snprintf(str, size,
+		 "%llu: (%s%s) refs=%u fwaddr=%#08x enqueue=%u status=%-9s %s%s",
+		 (u64) pvr_fence->fence->seqno,
+		 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 refcount_read(&pvr_fence->fence->refcount.refcount),
+		 SyncCheckpointGetFirmwareAddr(
+			 pvr_fence->sync_checkpoint),
+		 SyncCheckpointGetEnqueuedCount(pvr_fence->sync_checkpoint),
+		 SyncCheckpointGetStateString(pvr_fence->sync_checkpoint),
+		 pvr_fence->name,
+		 (&pvr_fence->base != pvr_fence->fence) ?
+		 "(foreign)" : "");
+}
+
+static
+void pvr_fence_timeline_value_str(struct dma_fence *fence, char *str, int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		pvr_context_value_str(pvr_fence->fctx, str, size);
+}
+
+static bool
+pvr_fence_enable_signaling(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (!pvr_fence)
+		return false;
+
+	WARN_ON_SMP(!spin_is_locked(&pvr_fence->fctx->lock));
+
+	if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return false;
+
+	dma_fence_get(&pvr_fence->base);
+
+	spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->signal_head, &pvr_fence->fctx->signal_list);
+	spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "signalling enabled (%s)\n",
+			pvr_fence->name);
+	trace_pvr_fence_enable_signaling(pvr_fence);
+
+	return true;
+}
+
+static bool
+pvr_fence_is_signaled(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		return pvr_fence_sync_is_signaled(pvr_fence,
+						  PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT);
+	return false;
+}
+
+static void
+pvr_fence_release(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (pvr_fence) {
+		struct pvr_fence_context *fctx = pvr_fence->fctx;
+
+		PVR_FENCE_TRACE(&pvr_fence->base, "released fence (%s)\n",
+				pvr_fence->name);
+		trace_pvr_fence_release(pvr_fence);
+
+		spin_lock_irqsave(&fctx->list_lock, flags);
+		list_move(&pvr_fence->fence_head,
+			  &fctx->deferred_free_list);
+		spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+		kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+	}
+}
+
+const struct dma_fence_ops pvr_fence_ops = {
+	.get_driver_name = pvr_fence_get_driver_name,
+	.get_timeline_name = pvr_fence_get_timeline_name,
+	.fence_value_str = pvr_fence_fence_value_str,
+	.timeline_value_str = pvr_fence_timeline_value_str,
+	.enable_signaling = pvr_fence_enable_signaling,
+	.signaled = pvr_fence_is_signaled,
+	.wait = dma_fence_default_wait,
+	.release = pvr_fence_release,
+};
+
+/**
+ * pvr_fence_create - creates a PVR fence
+ * @fctx: PVR fence context on which the PVR fence should be created
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @timeline_fd: timeline on which the PVR fence should be created
+ * @name: PVR fence name (used for debugging)
+ *
+ * Creates a PVR fence.
+ *
+ * Once the fence is finished with, pvr_fence_destroy should be called.
+ *
+ * Returns NULL if a PVR fence cannot be created.
+ */
+struct pvr_fence *
+pvr_fence_create(struct pvr_fence_context *fctx,
+		struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+		int timeline_fd, const char *name)
+{
+	struct pvr_fence *pvr_fence;
+	unsigned int seqno;
+	unsigned long flags;
+	PVRSRV_ERROR srv_err;
+
+	if (!try_module_get(THIS_MODULE))
+		goto err_exit;
+
+	/* Note: As kmem_cache is used to allocate pvr_fence objects,
+	 * make sure that all members of pvr_fence struct are initialized
+	 * here
+	 */
+	pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
+	if (unlikely(!pvr_fence))
+		goto err_module_put;
+
+	srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
+				      (PVRSRV_TIMELINE) timeline_fd, PVRSRV_NO_FENCE,
+				      name, &pvr_fence->sync_checkpoint);
+	if (unlikely(srv_err != PVRSRV_OK))
+		goto err_free_fence;
+
+	INIT_LIST_HEAD(&pvr_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_fence->signal_head);
+	pvr_fence->fctx = fctx;
+	seqno = pvr_fence_context_seqno_next(fctx);
+	/* Add the seqno to the fence name for easier debugging */
+	pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
+			name, seqno);
+
+	/* Reset cb to zero */
+	memset(&pvr_fence->cb, 0, sizeof(pvr_fence->cb));
+	pvr_fence->fence = &pvr_fence->base;
+
+	dma_fence_init(&pvr_fence->base, &pvr_fence_ops, &fctx->lock,
+		       fctx->fence_context, seqno);
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+	kref_get(&fctx->kref);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "created fence (%s)\n", name);
+	trace_pvr_fence_create(pvr_fence);
+
+	return pvr_fence;
+
+err_free_fence:
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+err_module_put:
+	module_put(THIS_MODULE);
+err_exit:
+	return NULL;
+}
+
+static const char *
+pvr_fence_foreign_get_driver_name(struct dma_fence *fence)
+{
+	return PVR_LDM_DRIVER_REGISTRATION_NAME;
+}
+
+static const char *
+pvr_fence_foreign_get_timeline_name(struct dma_fence *fence)
+{
+	return "foreign";
+}
+
+static
+void pvr_fence_foreign_fence_value_str(struct dma_fence *fence, char *str,
+				       int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	u32 sync_addr = 0;
+	u32 sync_value_next;
+
+	if (WARN_ON(!pvr_fence))
+		return;
+
+	sync_addr = SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint);
+	sync_value_next = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+
+	/*
+	 * Include the fence flag bits from the foreign fence instead of our
+	 * shadow copy. This is done as the shadow fence flag bits aren't used.
+	 */
+	snprintf(str, size,
+		 "%llu: (%s%s) refs=%u fwaddr=%#08x cur=%#08x nxt=%#08x %s",
+		 (u64) fence->seqno,
+		 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 refcount_read(&fence->refcount.refcount),
+		 sync_addr,
+		 pvr_fence_sync_value(pvr_fence),
+		 sync_value_next,
+		 pvr_fence->name);
+}
+
+static
+void pvr_fence_foreign_timeline_value_str(struct dma_fence *fence, char *str,
+					  int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		pvr_context_value_str(pvr_fence->fctx, str, size);
+}
+
+static bool
+pvr_fence_foreign_enable_signaling(struct dma_fence *fence)
+{
+	WARN_ON("cannot enable signalling on foreign fence");
+	return false;
+}
+
+static signed long
+pvr_fence_foreign_wait(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	WARN_ON("cannot wait on foreign fence");
+	return 0;
+}
+
+static void
+pvr_fence_foreign_release(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (pvr_fence) {
+		struct pvr_fence_context *fctx = pvr_fence->fctx;
+		struct dma_fence *foreign_fence = pvr_fence->fence;
+
+		PVR_FENCE_TRACE(&pvr_fence->base,
+				"released fence for foreign fence %llu#%d (%s)\n",
+				(u64) pvr_fence->fence->context,
+				(int)pvr_fence->fence->seqno, pvr_fence->name);
+		trace_pvr_fence_foreign_release(pvr_fence);
+
+		spin_lock_irqsave(&fctx->list_lock, flags);
+		list_move(&pvr_fence->fence_head,
+			  &fctx->deferred_free_list);
+		spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+		dma_fence_put(foreign_fence);
+
+		kref_put(&fctx->kref,
+			 pvr_fence_context_destroy_kref);
+	}
+}
+
+const struct dma_fence_ops pvr_fence_foreign_ops = {
+	.get_driver_name = pvr_fence_foreign_get_driver_name,
+	.get_timeline_name = pvr_fence_foreign_get_timeline_name,
+	.fence_value_str = pvr_fence_foreign_fence_value_str,
+	.timeline_value_str = pvr_fence_foreign_timeline_value_str,
+	.enable_signaling = pvr_fence_foreign_enable_signaling,
+	.wait = pvr_fence_foreign_wait,
+	.release = pvr_fence_foreign_release,
+};
+
+static void
+pvr_fence_foreign_signal_sync(struct dma_fence *fence, struct dma_fence_cb *cb)
+{
+	struct pvr_fence *pvr_fence = container_of(cb, struct pvr_fence, cb);
+	struct pvr_fence_context *fctx = pvr_fence->fctx;
+
+	/* Callback registered by dma_fence_add_callback can be called from an atomic ctx */
+	pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_CTX_ATOMIC);
+
+	trace_pvr_fence_foreign_signal(pvr_fence);
+
+	queue_work(fctx->fence_wq, &fctx->check_status_work);
+
+	PVR_FENCE_TRACE(&pvr_fence->base,
+			"foreign fence %llu#%d signalled (%s)\n",
+			(u64) pvr_fence->fence->context,
+			(int)pvr_fence->fence->seqno, pvr_fence->name);
+
+	/* Drop the reference on the base fence */
+	dma_fence_put(&pvr_fence->base);
+}
+
+/**
+ * pvr_fence_create_from_fence - creates a PVR fence from a fence
+ * @fctx: PVR fence context on which the PVR fence should be created
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @fence: fence from which the PVR fence should be created
+ * @fence_fd: fd for the sync file to which the fence belongs. If it doesn't
+ *            belong to a sync file then PVRSRV_NO_FENCE should be given
+ *            instead.
+ * @name: PVR fence name (used for debugging)
+ *
+ * Creates a PVR fence from an existing fence. If the fence is a foreign fence,
+ * i.e. one that doesn't originate from a PVR fence context, then a new PVR
+ * fence will be created using the specified sync_checkpoint_context.
+ * Otherwise, a reference will be taken on the underlying fence and the PVR
+ * fence will be returned.
+ *
+ * Once the fence is finished with, pvr_fence_destroy should be called.
+ *
+ * Returns NULL if a PVR fence cannot be created.
+ */
+
+struct pvr_fence *
+pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
+			    struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+			    struct dma_fence *fence,
+			    PVRSRV_FENCE fence_fd,
+			    const char *name)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned int seqno;
+	unsigned long flags;
+	PVRSRV_ERROR srv_err;
+	int err;
+	bool mirror_other_dev_fence = false;
+	char tempString[40] = {0};
+
+	PVR_FENCE_TRACE(fence, "%s: fence @%p, pvr_fence = %p fctx = %p, dev %p\n",
+			__func__, fence, pvr_fence, fctx, (fctx ? fctx->dev_cookie : NULL));
+
+	if (pvr_fence) {
+		if ((SyncCheckpointCommonDeviceIDs(sync_checkpoint_ctx,
+						pvr_fence->fctx->dev_cookie)) ||
+		    (fctx->dev_cookie == pvr_fence->fctx->dev_cookie)) {
+			if (WARN_ON(fence->ops == &pvr_fence_foreign_ops))
+				return NULL;
+			dma_fence_get(fence);
+
+			PVR_FENCE_TRACE(fence, "created fence from PVR fence (%s)\n",
+					name);
+			return pvr_fence;
+		} else {
+			snprintf(tempString, sizeof(tempString), "Mirror(FWAddr0x%x)",
+				 SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint));
+			mirror_other_dev_fence = true;
+
+			PVR_FENCE_TRACE(fence, "MIRRORED FENCE '%s'", tempString);
+		}
+	}
+
+	if (!try_module_get(THIS_MODULE))
+		goto err_exit;
+
+	/* Note: As kmem_cache is used to allocate pvr_fence objects,
+	 * make sure that all members of pvr_fence struct are initialized
+	 * here
+	 */
+	pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
+	if (!pvr_fence) {
+		pr_err("%s: kmem_cache_alloc() failed",
+			   __func__);
+		goto err_module_put;
+	}
+
+	srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
+					  (mirror_other_dev_fence ?
+					  SYNC_CHECKPOINT_MIRRORED_CHECKPOINT :
+					  SYNC_CHECKPOINT_FOREIGN_CHECKPOINT),
+					  fence_fd,
+					  mirror_other_dev_fence ? tempString : name,
+					  &pvr_fence->sync_checkpoint);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: SyncCheckpointAlloc() failed (srv_err=%d)",
+			   __func__, srv_err);
+		goto err_free_pvr_fence;
+	}
+
+	INIT_LIST_HEAD(&pvr_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_fence->signal_head);
+	pvr_fence->fctx = fctx;
+	pvr_fence->fence = dma_fence_get(fence);
+	seqno = pvr_fence_context_seqno_next(fctx);
+	/* Add the seqno to the fence name for easier debugging */
+	pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
+			name, seqno);
+
+	/*
+	 * We use the base fence to refcount the PVR fence and to do the
+	 * necessary clean up once the refcount drops to 0.
+	 */
+	dma_fence_init(&pvr_fence->base, &pvr_fence_foreign_ops, &fctx->lock,
+		       fctx->fence_context, seqno);
+
+	/*
+	 * Take an extra reference on the base fence that gets dropped when the
+	 * foreign fence is signalled.
+	 */
+	dma_fence_get(&pvr_fence->base);
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	kref_get(&fctx->kref);
+
+	PVR_FENCE_TRACE(&pvr_fence->base,
+			"created fence from foreign fence %llu#%d (%s)\n",
+			(u64) pvr_fence->fence->context,
+			(int)pvr_fence->fence->seqno, name);
+
+	err = dma_fence_add_callback(fence, &pvr_fence->cb,
+				     pvr_fence_foreign_signal_sync);
+	if (err) {
+		if (err != -ENOENT) {
+			pr_err("%s: failed to add fence callback (err=%d)",
+			       __func__, err);
+			goto err_put_ref;
+		}
+
+
+		/*
+		 * The fence has already signalled so set the sync as signalled.
+		 * The "signalled" hwperf packet should be emitted because the
+		 * callback won't be called for already signalled fence hence,
+		 * PVRSRV_FENCE_FLAG_NONE flag.
+		 */
+		pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
+		PVR_FENCE_TRACE(&pvr_fence->base,
+				"foreign fence %llu#%d already signaled (%s)\n",
+				(u64) pvr_fence->fence->context,
+				(int)pvr_fence->fence->seqno,
+				name);
+		dma_fence_put(&pvr_fence->base);
+	}
+
+	trace_pvr_fence_foreign_create(pvr_fence);
+
+	return pvr_fence;
+
+err_put_ref:
+	kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_del(&pvr_fence->fence_head);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	SyncCheckpointFree(pvr_fence->sync_checkpoint);
+err_free_pvr_fence:
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+err_module_put:
+	module_put(THIS_MODULE);
+err_exit:
+	return NULL;
+}
+
+/**
+ * pvr_fence_destroy - destroys a PVR fence
+ * @pvr_fence: PVR fence to destroy
+ *
+ * Destroys a PVR fence. Upon return, the PVR fence may still exist if something
+ * else still references the underlying fence, e.g. a reservation object, or if
+ * software signalling has been enabled and the fence hasn't yet been signalled.
+ */
+void
+pvr_fence_destroy(struct pvr_fence *pvr_fence)
+{
+	PVR_FENCE_TRACE(&pvr_fence->base, "destroyed fence (%s)\n",
+			pvr_fence->name);
+
+	dma_fence_put(&pvr_fence->base);
+}
+
+/**
+ * pvr_fence_sw_signal - signals a PVR fence sync
+ * @pvr_fence: PVR fence to signal
+ *
+ * Sets the PVR fence sync value to signalled.
+ *
+ * Returns -EINVAL if the PVR fence represents a foreign fence.
+ */
+int
+pvr_fence_sw_signal(struct pvr_fence *pvr_fence)
+{
+	if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
+		return -EINVAL;
+
+	pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
+
+	queue_work(pvr_fence->fctx->fence_wq,
+		   &pvr_fence->fctx->check_status_work);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync signalled (%s)\n",
+			pvr_fence->name);
+
+	return 0;
+}
+
+/**
+ * pvr_fence_sw_error - errors the sync checkpoint backing a PVR fence
+ * @pvr_fence: PVR fence to error
+ *
+ * Sets the PVR fence sync checkpoint value to errored.
+ *
+ * Returns -EINVAL if the PVR fence represents a foreign fence.
+ */
+int
+pvr_fence_sw_error(struct pvr_fence *pvr_fence)
+{
+	if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
+		return -EINVAL;
+
+	SyncCheckpointError(pvr_fence->sync_checkpoint, PVRSRV_FENCE_FLAG_NONE);
+	PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync errored (%s)\n",
+			pvr_fence->name);
+
+	return 0;
+}
+
+int
+pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
+			  struct SYNC_CHECKPOINT_TAG **fence_checkpoints)
+{
+	struct SYNC_CHECKPOINT_TAG **next_fence_checkpoint = fence_checkpoints;
+	struct pvr_fence **next_pvr_fence = pvr_fences;
+	int fence_checkpoint_idx;
+
+	if (nr_fences > 0) {
+
+		for (fence_checkpoint_idx = 0; fence_checkpoint_idx < nr_fences;
+		     fence_checkpoint_idx++) {
+			struct pvr_fence *next_fence = *next_pvr_fence++;
+			*next_fence_checkpoint++ = next_fence->sync_checkpoint;
+			/* Take reference on sync checkpoint (will be dropped
+			 * later by kick code)
+			 */
+			SyncCheckpointTakeRef(next_fence->sync_checkpoint);
+		}
+	}
+
+	return 0;
+}
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_fence_get_checkpoint(struct pvr_fence *update_fence)
+{
+	return update_fence->sync_checkpoint;
+}
+
+/**
+ * pvr_fence_dump_info_on_stalled_ufos - displays debug
+ * information on a native fence associated with any of
+ * the ufos provided. This function will be called from
+ * pvr_sync_file.c if the driver determines any GPU work
+ * is stuck waiting for a sync checkpoint representing a
+ * foreign sync to be signalled.
+ * @fctx:    fence context
+ * @nr_ufos: number of ufos in vaddrs
+ * @vaddrs:  array of FW addresses of UFOs which the
+ *           driver is waiting on.
+ *
+ * Output debug information to kernel log on linux fences
+ * which would be responsible for signalling the sync
+ * checkpoints indicated by the ufo vaddresses.
+ *
+ * Returns the number of ufos in the array which were found
+ * to be associated with foreign syncs.
+ */
+u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
+					u32 nr_ufos, u32 *vaddrs)
+{
+	int our_ufo_ct = 0;
+	struct pvr_fence *pvr_fence;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	/* dump info on any ufos in our active list */
+	list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
+		u32 *this_ufo_vaddr = vaddrs;
+		int ufo_num;
+		DUMPDEBUG_PRINTF_FUNC *pfnDummy = NULL;
+
+		for (ufo_num = 0; ufo_num < nr_ufos; ufo_num++) {
+			struct SYNC_CHECKPOINT_TAG *checkpoint =
+				pvr_fence->sync_checkpoint;
+			const u32 fence_ufo_addr =
+				SyncCheckpointGetFirmwareAddr(checkpoint);
+
+			if (fence_ufo_addr != this_ufo_vaddr[ufo_num])
+				continue;
+
+			/* Dump sync info */
+			PVR_DUMPDEBUG_LOG(pfnDummy, NULL,
+					  "\tSyncID = %d, FWAddr = 0x%08x: TLID = %d (Foreign Fence - [%p] %s)",
+					  SyncCheckpointGetId(checkpoint),
+					  fence_ufo_addr,
+					  SyncCheckpointGetTimeline(checkpoint),
+					  pvr_fence->fence,
+					  pvr_fence->name);
+			our_ufo_ct++;
+		}
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	return our_ufo_ct;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.h
new file mode 100644
index 000000000000..4e2b17096669
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.h
@@ -0,0 +1,234 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_FENCE_H__)
+#define __PVR_FENCE_H__
+
+#include "services_kernel_client.h"
+#include "pvr_linux_fence.h"
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+struct SYNC_CHECKPOINT_CONTEXT_TAG;
+struct SYNC_CHECKPOINT_TAG;
+
+/**
+ * pvr_fence_context - PVR fence context used to create and manage PVR fences
+ * @lock: protects the context and fences created on the context
+ * @name: fence context name (used for debugging)
+ * @dbg_request_handle: handle for callback used to dump debug data
+ * @fence_context: fence context with which to associate fences
+ * @fence_seqno: sequence number to use for the next fence
+ * @fence_wq: work queue for signalled fence work
+ * @check_status_work: work item used to inform services when a foreign fence
+ * has signalled
+ * @cmd_complete_handle: handle for callback used to signal fences when fence
+ * syncs are met
+ * @list_lock: protects the active and active foreign lists
+ * @signal_list: list of fences waiting to be signalled
+ * @fence_list: list of fences (used for debugging)
+ * @deferred_free_list: list of fences that we will free when we are no longer
+ * holding spinlocks.  The frees get implemented when an update fence is
+ * signalled or the context is freed.
+ */
+struct pvr_fence_context {
+	spinlock_t lock;
+	char name[32];
+	void *dbg_request_handle;
+	u64 fence_context;
+	atomic_t fence_seqno;
+
+	struct workqueue_struct *fence_wq;
+	struct work_struct check_status_work;
+
+	void *cmd_complete_handle;
+
+	spinlock_t list_lock;
+	struct list_head signal_list;
+	struct list_head fence_list;
+	struct list_head deferred_free_list;
+
+	struct kref kref;
+	struct work_struct destroy_work;
+	void *dev_cookie;
+};
+
+/**
+ * pvr_fence - PVR fence that represents both native and foreign fences
+ * @base: fence structure
+ * @fctx: fence context on which this fence was created
+ * @name: fence name (used for debugging)
+ * @fence: pointer to base fence structure or foreign fence
+ * @sync_checkpoint: services sync checkpoint used by hardware
+ * @fence_head: entry on the context fence and deferred free list
+ * @signal_head: entry on the context signal list
+ * @cb: foreign fence callback to set the sync to signalled
+ */
+struct pvr_fence {
+	struct dma_fence base;
+	struct pvr_fence_context *fctx;
+	char name[32];
+
+	struct dma_fence *fence;
+	struct SYNC_CHECKPOINT_TAG *sync_checkpoint;
+
+	struct list_head fence_head;
+	struct list_head signal_head;
+	struct dma_fence_cb cb;
+	struct rcu_head rcu;
+};
+
+extern const struct dma_fence_ops pvr_fence_ops;
+extern const struct dma_fence_ops pvr_fence_foreign_ops;
+
+static inline bool is_our_fence(struct pvr_fence_context *fctx,
+				struct dma_fence *fence)
+{
+	return (fence->context == fctx->fence_context);
+}
+
+static inline bool is_pvr_fence(struct dma_fence *fence)
+{
+	return ((fence->ops == &pvr_fence_ops) ||
+		(fence->ops == &pvr_fence_foreign_ops));
+}
+
+static inline struct pvr_fence *to_pvr_fence(struct dma_fence *fence)
+{
+	if (is_pvr_fence(fence))
+		return container_of(fence, struct pvr_fence, base);
+
+	return NULL;
+}
+
+PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
+				void *dev,
+				struct pvr_fence_context *fctx);
+struct pvr_fence_context *
+pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
+		const char *name);
+struct pvr_fence_context *
+pvr_fence_context_create(void *dev_cookie,
+			 struct workqueue_struct *fence_status_wq,
+			 const char *name);
+void pvr_fence_context_destroy(struct pvr_fence_context *fctx);
+void pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size);
+
+struct pvr_fence *
+pvr_fence_create(struct pvr_fence_context *fctx,
+		 struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+		 int timeline_fd, const char *name);
+struct pvr_fence *
+pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
+			    struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+			    struct dma_fence *fence,
+			    PVRSRV_FENCE fence_fd,
+			    const char *name);
+void pvr_fence_destroy(struct pvr_fence *pvr_fence);
+int pvr_fence_sw_signal(struct pvr_fence *pvr_fence);
+int pvr_fence_sw_error(struct pvr_fence *pvr_fence);
+
+int pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
+			      struct SYNC_CHECKPOINT_TAG **fence_checkpoints);
+struct SYNC_CHECKPOINT_TAG *
+pvr_fence_get_checkpoint(struct pvr_fence *update_fence);
+
+void pvr_fence_context_signal_fences_nohw(void *data);
+
+void pvr_fence_context_free_deferred_callback(void *data);
+
+u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
+					u32 nr_ufos,
+					u32 *vaddrs);
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+static inline void pvr_fence_cleanup(void)
+{
+	/*
+	 * Ensure all PVR fence contexts have been destroyed, by flushing
+	 * the context destruction workqueue.
+	 */
+	flush_workqueue(NativeSyncGetFenceCtxDestroyWq());
+}
+#endif
+
+#if defined(PVR_FENCE_DEBUG)
+#define PVR_FENCE_CTX_TRACE(c, fmt, ...)                                   \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_err("c %llu: (PVR) " fmt, (u64) __fctx->fence_context,  \
+		       ## __VA_ARGS__);                                    \
+	} while (0)
+#else
+#define PVR_FENCE_CTX_TRACE(c, fmt, ...)
+#endif
+
+#define PVR_FENCE_CTX_WARN(c, fmt, ...)                                    \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_warn("c %llu: (PVR) " fmt, (u64) __fctx->fence_context, \
+			## __VA_ARGS__);                                   \
+	} while (0)
+
+#define PVR_FENCE_CTX_ERR(c, fmt, ...)                                     \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_err("c %llu: (PVR) " fmt, (u64) __fctx->fence_context,  \
+		       ## __VA_ARGS__);                                    \
+	} while (0)
+
+#if defined(PVR_FENCE_DEBUG)
+#define PVR_FENCE_TRACE(f, fmt, ...)                                       \
+	DMA_FENCE_ERR(f, "(PVR) " fmt, ## __VA_ARGS__)
+#else
+#define PVR_FENCE_TRACE(f, fmt, ...)
+#endif
+
+#define PVR_FENCE_WARN(f, fmt, ...)                                        \
+	DMA_FENCE_WARN(f, "(PVR) " fmt, ## __VA_ARGS__)
+
+#define PVR_FENCE_ERR(f, fmt, ...)                                         \
+	DMA_FENCE_ERR(f, "(PVR) " fmt, ## __VA_ARGS__)
+
+#endif /* !defined(__PVR_FENCE_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence_trace.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence_trace.h
new file mode 100644
index 000000000000..95b2a0dc6c7c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence_trace.h
@@ -0,0 +1,225 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM pvr_fence
+
+#if !defined(_TRACE_PVR_FENCE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_PVR_FENCE_H
+
+#include <linux/tracepoint.h>
+
+struct pvr_fence;
+struct pvr_fence_context;
+
+DECLARE_EVENT_CLASS(pvr_fence_context,
+
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx),
+
+	TP_STRUCT__entry(
+		__string(name, fctx->name)
+		__array(char, val, 128)
+	),
+
+	TP_fast_assign(
+		__assign_str(name, fctx->name)
+		pvr_context_value_str(fctx, __entry->val,
+			sizeof(__entry->val));
+	),
+
+	TP_printk("name=%s val=%s",
+		  __get_str(name),
+		  __entry->val
+	)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_create,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_destroy,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_destroy_kref,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_signal_fences,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DECLARE_EVENT_CLASS(pvr_fence,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__string(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		__array(char, val, 128)
+		__field(u64, context)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__assign_str(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		fence->base.ops->fence_value_str(&fence->base,
+			__entry->val, sizeof(__entry->val));
+		__entry->context = fence->base.context;
+	),
+
+	TP_printk("driver=%s timeline=%s ctx=%llu val=%s",
+		  __get_str(driver), __get_str(timeline),
+		  __entry->context, __entry->val
+	)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_create,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_release,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_enable_signaling,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_signal_fence,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DECLARE_EVENT_CLASS(pvr_fence_foreign,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__string(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		__array(char, val, 128)
+		__field(u64, context)
+		__string(foreign_driver,
+			fence->fence->ops->get_driver_name ?
+			fence->fence->ops->get_driver_name(fence->fence) :
+			"unknown")
+		__string(foreign_timeline,
+			fence->fence->ops->get_timeline_name ?
+			fence->fence->ops->get_timeline_name(fence->fence) :
+			"unknown")
+		__array(char, foreign_val, 128)
+		__field(u64, foreign_context)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__assign_str(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		fence->base.ops->fence_value_str(&fence->base, __entry->val,
+			sizeof(__entry->val));
+		__entry->context = fence->base.context;
+		__assign_str(foreign_driver,
+			fence->fence->ops->get_driver_name ?
+			fence->fence->ops->get_driver_name(fence->fence) :
+			"unknown")
+		__assign_str(foreign_timeline,
+			fence->fence->ops->get_timeline_name ?
+			fence->fence->ops->get_timeline_name(fence->fence) :
+			"unknown")
+		fence->fence->ops->fence_value_str ?
+			fence->fence->ops->fence_value_str(
+				fence->fence, __entry->foreign_val,
+				sizeof(__entry->foreign_val)) :
+			(void) OSStringSafeCopy(__entry->foreign_val,
+				"unknown", sizeof(__entry->foreign_val));
+		__entry->foreign_context = fence->fence->context;
+	),
+
+	TP_printk("driver=%s timeline=%s ctx=%llu val=%s foreign: driver=%s timeline=%s ctx=%llu val=%s",
+		  __get_str(driver), __get_str(timeline), __entry->context,
+		  __entry->val, __get_str(foreign_driver),
+		  __get_str(foreign_timeline), __entry->foreign_context,
+		  __entry->foreign_val
+	)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_create,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_release,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_signal,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+#endif /* _TRACE_PVR_FENCE_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+
+/* This is needed because the name of this file doesn't match TRACE_SYSTEM. */
+#define TRACE_INCLUDE_FILE pvr_fence_trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_gputrace.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_gputrace.c
new file mode 100644
index 000000000000..bbfa3504495a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_gputrace.c
@@ -0,0 +1,1680 @@
+/*************************************************************************/ /*!
+@File           pvr_gputrace.c
+@Title          PVR GPU Trace module Linux implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/fs.h>
+#include <linux/trace_events.h>
+
+#include "pvrsrv_error.h"
+#include "pvrsrv_apphint.h"
+#include "pvr_debug.h"
+#include "ospvr_gputrace.h"
+#include "rgxhwperf.h"
+#include "rgxtimecorr.h"
+#include "device.h"
+#include "trace_events.h"
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "tlclient.h"
+#include "tlstream.h"
+#include "pvr_debug.h"
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#define CREATE_TRACE_POINTS
+#include "rogue_trace_events.h"
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#define CREATE_TRACE_POINTS
+#include "gpu_work.h"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#if defined(SUPPORT_RGX)
+
+#define TRACE_CLK     RGXTIMECORR_CLOCK_SCHED
+#define TRACE_CLK_STR "local\n"
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#define TRACE_FS_CLK "/sys/kernel/tracing/trace_clock"
+#else /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)*/
+#define TRACE_FS_CLK "/sys/kernel/debug/tracing/trace_clock"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+
+#endif /* defined(SUPPORT_RGX) */
+
+/******************************************************************************
+ Module internal implementation
+******************************************************************************/
+
+typedef enum {
+	PVR_GPUTRACE_SWITCH_TYPE_UNDEF = 0,
+
+	PVR_GPUTRACE_SWITCH_TYPE_BEGIN = 1,
+	PVR_GPUTRACE_SWITCH_TYPE_END = 2,
+	PVR_GPUTRACE_SWITCH_TYPE_SINGLE = 3
+} PVR_GPUTRACE_SWITCH_TYPE;
+
+typedef struct RGX_HWPERF_FTRACE_DATA {
+	/* This lock ensures the HWPerf TL stream reading resources are not destroyed
+	 * by one thread disabling it while another is reading from it. Keeps the
+	 * state and resource create/destroy atomic and consistent. */
+	POS_LOCK    hFTraceResourceLock;
+
+	IMG_HANDLE  hGPUTraceCmdCompleteHandle;
+	IMG_HANDLE  hGPUFTraceTLStream;
+	IMG_UINT64  ui64LastSampledTimeCorrOSTimeStamp;
+	IMG_UINT32  ui32FTraceLastOrdinal;
+	IMG_BOOL    bTrackOrdinals;
+} RGX_HWPERF_FTRACE_DATA;
+
+/* This lock ensures state change of GPU_TRACING on/off is done atomically */
+static POS_LOCK ghGPUTraceStateLock;
+static IMG_BOOL gbFTraceGPUEventsEnabled = PVRSRV_APPHINT_ENABLEFTRACEGPU;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+/* This is the FTrace clock source prior to driver initialisation */
+static IMG_CHAR gszLastClockSource[32] = {0};
+#endif
+
+/* This lock ensures that the reference counting operation on the FTrace UFO
+ * events and enable/disable operation on firmware event are performed as
+ * one atomic operation. This should ensure that there are no race conditions
+ * between reference counting and firmware event state change.
+ * See below comment for guiUfoEventRef.
+ */
+static POS_LOCK ghLockFTraceEventLock;
+
+/* Multiple FTrace UFO events are reflected in the firmware as only one event. When
+ * we enable FTrace UFO event we want to also at the same time enable it in
+ * the firmware. Since there is a multiple-to-one relation between those events
+ * we count how many FTrace UFO events is enabled. If at least one event is
+ * enabled we enabled the firmware event. When all FTrace UFO events are disabled
+ * we disable firmware event. */
+static IMG_UINT guiUfoEventRef;
+
+/******************************************************************************
+ Module In-bound API
+******************************************************************************/
+
+static PVRSRV_ERROR _GpuTraceDisable(
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+	IMG_BOOL bDeInit);
+
+static void _GpuTraceCmdCompleteNotify(PVRSRV_CMDCOMP_HANDLE);
+static void _GpuTraceProcessPackets(PVRSRV_RGXDEV_INFO *psDevInfo, void *pBuffer,
+                                    IMG_UINT32 ui32ReadLen);
+
+static void _FTrace_FWOnReaderOpenCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_FTRACE] = IMG_FALSE;
+}
+
+static void _FTrace_FWOnReaderCloseCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_FTRACE] = IMG_TRUE;
+}
+
+/* Currently supported by default */
+#if defined(SUPPORT_TL_PRODUCER_CALLBACK)
+static PVRSRV_ERROR GPUTraceTLCB(IMG_HANDLE hStream,
+                                  IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser)
+{
+	/* IN DEV: Not required as the streams goal is to be a copy of HWPerf */
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(hStream);
+	PVR_UNREFERENCED_PARAMETER(ui32ReqOp);
+	PVR_UNREFERENCED_PARAMETER(ui32Resp);
+	PVR_UNREFERENCED_PARAMETER(pvUser);
+
+	return eError;
+}
+#endif /* defined(SUPPORT_TL_PRODUCER_CALLBACK) */
+
+#if defined(SUPPORT_RGX)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+/* Configure the FTrace clock source to use the DDK apphint clock source */
+static void PVRGpuTraceInitFTraceClockSource(void)
+{
+	int ret, i, j;
+	bool bFound = false;
+	loff_t pos = 0;
+	char str[64];
+
+	/*  Just force the value to be what the DDK says it is
+	    Note for filp_open, the mode is only used for O_CREAT
+		Hence its value doesn't matter in this context
+	 */
+	struct file *filep = filp_open(TRACE_FS_CLK, O_RDWR, 0);
+	PVR_LOG_RETURN_VOID_IF_FALSE(!IS_ERR(filep), "TraceFS not found");
+
+	PVR_LOG_VA(PVR_DBG_MESSAGE,
+	        "Writing %s to %s to enable parallel HWPerf and FTrace support",
+	        TRACE_CLK_STR, TRACE_FS_CLK);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_read(filep, str, sizeof(str)-1, &pos);
+#else
+	ret = kernel_read(filep, pos, str, sizeof(str)-1);
+#endif
+	PVR_LOG_RETURN_VOID_IF_FALSE((ret > 0), "TraceFS Read failed");
+	str[ret] = 0;
+	pos = 0;
+
+	/* Determine clock value. trace_clock value is stored like [<clock_string>]
+	   File example: [local] global counter mono mono_raw x86-tsc
+	*/
+	for (i = 0, j = 0; i < sizeof(str) && j < sizeof(gszLastClockSource); i++)
+	{
+		if (str[i] == ']')
+		{
+			break;
+		}
+		else if (str[i] == '[')
+		{
+			bFound = true;
+		}
+		else if (bFound)
+		{
+			gszLastClockSource[j] = str[i];
+			j++;
+		}
+	}
+	PVR_LOG_VA(PVR_DBG_MESSAGE, "Got %s from FTraceFS", gszLastClockSource);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_write(filep, TRACE_CLK_STR, sizeof(TRACE_CLK_STR), &pos);
+#else
+	ret = kernel_write(filep, TRACE_CLK_STR, sizeof(TRACE_CLK_STR), pos);
+#endif
+	PVR_LOG_IF_FALSE((ret > 0), "Setting FTrace clock source failed");
+
+	filp_close(filep, NULL);
+}
+
+/* Reset the FTrace clock source to the original clock source */
+static void PVRGpuTraceDeinitFTraceClockSource(void)
+{
+	int ret;
+	loff_t pos = 0;
+
+	/* Return the original value of the file */
+	struct file *filep = filp_open(TRACE_FS_CLK, O_WRONLY, 0);
+	PVR_LOG_RETURN_VOID_IF_FALSE(!IS_ERR(filep), "TraceFS not found");
+
+	/* FTraceFS write will ignore any writes to it that don't match a clock source */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_write(filep, gszLastClockSource, sizeof(gszLastClockSource), &pos);
+#else
+	ret = kernel_write(filep, gszLastClockSource, sizeof(gszLastClockSource), pos);
+#endif
+	PVR_LOG_IF_FALSE((ret >= 0), "Setting FTrace clock source failed");
+
+	filp_close(filep, NULL);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* defined(SUPPORT_RGX) */
+
+PVRSRV_ERROR PVRGpuTraceSupportInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	if (ghLockFTraceEventLock != NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FTrace Support is already initialized"));
+		return PVRSRV_OK;
+	}
+
+	/* common module params initialization */
+	eError = OSLockCreate(&ghLockFTraceEventLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	eError = OSLockCreate(&ghGPUTraceStateLock);
+	PVR_LOG_RETURN_IF_ERROR (eError, "OSLockCreate");
+
+#if defined(SUPPORT_RGX)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+	PVRGpuTraceInitFTraceClockSource();
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* defined(SUPPORT_RGX) */
+
+	return PVRSRV_OK;
+}
+
+void PVRGpuTraceSupportDeInit(void)
+{
+#if defined(SUPPORT_RGX)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+	PVRGpuTraceDeinitFTraceClockSource();
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* defined(SUPPORT_RGX) */
+
+	if (ghGPUTraceStateLock)
+	{
+		OSLockDestroy(ghGPUTraceStateLock);
+	}
+
+	if (ghLockFTraceEventLock)
+	{
+		OSLockDestroy(ghLockFTraceEventLock);
+		ghLockFTraceEventLock = NULL;
+	}
+}
+
+/* Called from RGXHWPerfInitOnDemandL2Stream() which is alway called from
+ * a critical section protected by hHWPerfLock. */
+PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszFTraceStreamName[sizeof(PVRSRV_TL_FTRACE_RGX_FW_STREAM) + 4];
+	/* + 4 is used to allow names up to "ftrace_999", which is enough */
+
+	IMG_HANDLE hStream = NULL;
+
+	/* form the FTrace stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszFTraceStreamName, sizeof(pszFTraceStreamName), "%s%d",
+				   PVRSRV_TL_FTRACE_RGX_FW_STREAM,
+				   psDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to form FTrace stream name for device %d",
+				__func__,
+				psDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = TLStreamCreate(&hStream,
+							pszFTraceStreamName,
+							psDevInfo->ui32RGXL2HWPerfBufSize,
+							TL_OPMODE_DROP_NEWER | TL_FLAG_NO_SIGNAL_ON_COMMIT,
+							_FTrace_FWOnReaderOpenCB, psDevInfo,
+							_FTrace_FWOnReaderCloseCB, psDevInfo,
+#if !defined(SUPPORT_TL_PRODUCER_CALLBACK)
+							NULL, NULL
+#else
+							/* Not enabled by default */
+							GPUTraceTLCB, psDevInfo
+#endif
+							);
+	PVR_RETURN_IF_ERROR(eError);
+
+	psDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_FTRACE] = hStream;
+	psDevInfo->uiHWPerfStreamCount++;
+	PVR_ASSERT(psDevInfo->uiHWPerfStreamCount <= RGX_HWPERF_L2_STREAM_LAST);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRGpuTraceInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	RGX_HWPERF_FTRACE_DATA *psData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psData = OSAllocZMem(sizeof(RGX_HWPERF_FTRACE_DATA));
+	psDevInfo->pvGpuFtraceData = psData;
+	PVR_LOG_GOTO_IF_NOMEM(psData, eError, e0);
+
+	/* We initialise it only once because we want to track if any
+	 * packets were dropped. */
+	psData->ui32FTraceLastOrdinal = IMG_UINT32_MAX - 1;
+
+	eError = OSLockCreate(&psData->hFTraceResourceLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	return PVRSRV_OK;
+
+e0:
+	PVRGpuTraceDeInitDevice(psDeviceNode);
+	return eError;
+}
+
+void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_HWPERF_FTRACE_DATA *psData = psDevInfo->pvGpuFtraceData;
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+	if (psData)
+	{
+		/* first disable the tracing, to free up TL resources */
+		if (psData->hFTraceResourceLock)
+		{
+			OSLockAcquire(psData->hFTraceResourceLock);
+			_GpuTraceDisable(psDeviceNode->pvDevice, IMG_TRUE);
+			OSLockRelease(psData->hFTraceResourceLock);
+
+			/* now free all the FTrace resources */
+			OSLockDestroy(psData->hFTraceResourceLock);
+		}
+		OSFreeMem(psData);
+		psDevInfo->pvGpuFtraceData = NULL;
+	}
+}
+
+static INLINE IMG_BOOL PVRGpuTraceIsEnabled(void)
+{
+	return gbFTraceGPUEventsEnabled;
+}
+
+void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PVRGpuTraceIsEnabled())
+	{
+		IMG_BOOL bEnable = IMG_FALSE;
+
+		PVRSRV_ERROR eError = PVRGpuTraceSetEnabled(psDeviceNode, IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to initialise GPU event tracing"
+					" (%s)", PVRSRVGetErrorString(eError)));
+		}
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		/* below functions will enable FTrace events which in turn will
+		 * execute HWPerf callbacks that set appropriate filter values
+		 * note: unfortunately the functions don't allow to pass private
+		 *       data so they enable events for all of the devices
+		 *       at once, which means that this can happen more than once
+		 *       if there is more than one device */
+
+		/* single events can be enabled by calling trace_set_clr_event()
+		 * with the event name, e.g.:
+		 * trace_set_clr_event("rogue", "rogue_ufo_update", 1) */
+		if (trace_set_clr_event("rogue", NULL, 1) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to enable \"rogue\" event"
+			        " group"));
+		}
+		else
+		{
+			PVR_LOG(("FTrace events from \"rogue\" group enabled"));
+			bEnable = IMG_TRUE;
+		}
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+		if (trace_set_clr_event("power", "gpu_work_period", 1) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to enable \"gpu_work_period\" event"));
+		}
+		else
+		{
+			PVR_LOG(("FTrace event from \"gpu_work_period\" enabled"));
+			bEnable = IMG_TRUE;
+		}
+#endif
+
+		if (bEnable)
+		{
+			/* this enables FTrace globally (if not enabled nothing will appear
+			 * in the FTrace buffer) */
+			tracing_on();
+		}
+	}
+}
+
+/* Caller must now hold hFTraceResourceLock before calling this method.
+ */
+static PVRSRV_ERROR _GpuTraceEnable(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+	PVRSRV_DEVICE_NODE *psRgxDevNode = psRgxDevInfo->psDeviceNode;
+	IMG_CHAR pszFTraceStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 4];
+	IMG_UINT64 uiFilter;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	PVR_ASSERT(OSLockIsLocked(psFtraceData->hFTraceResourceLock));
+
+	/* return if already enabled */
+	if (psFtraceData->hGPUFTraceTLStream != NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	uiFilter =
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	    RGX_HWPERF_EVENT_MASK_HW_KICKFINISH | RGX_HWPERF_EVENT_MASK_FW_SED | RGX_HWPERF_EVENT_MASK_FW_UFO |
+#endif
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD) |
+#endif
+	    0;
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	/* Signal FW to enable event generation */
+	if (psRgxDevInfo->bFirmwareInitialised)
+	{
+		eError = PVRSRVRGXCtrlHWPerfFW(psRgxDevNode,
+		                               RGX_HWPERF_L2_STREAM_FTRACE,
+		                               uiFilter, HWPERF_FILTER_OPERATION_SET);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfFW", err_out);
+	}
+	else
+#endif
+	{
+		/* only set filter and exit */
+		(void) RGXHWPerfFwSetEventFilter(psRgxDevInfo, RGX_HWPERF_L2_STREAM_FTRACE, uiFilter);
+
+		return PVRSRV_OK;
+	}
+
+	/* form the FTrace stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszFTraceStreamName, sizeof(pszFTraceStreamName), "%s%d",
+					PVRSRV_TL_FTRACE_RGX_FW_STREAM, psRgxDevNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to form FTrace stream name for device %d",
+		         __func__,
+		         psRgxDevNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* In case there was any data copied from the L1 buffer to the FTrace L2
+	 * buffer in the narrow window between disabling the FTrace and MISR
+	 * running, drop this data to make the ordinal tracking correct. */
+	TLStreamReset(psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_FTRACE]);
+
+	/* Open the TL Stream for FTrace data consumption */
+	eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+								pszFTraceStreamName,
+								PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING,
+								&psFtraceData->hGPUFTraceTLStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLClientOpenStream", err_out);
+
+#if defined(SUPPORT_RGX)
+	if (RGXTimeCorrGetClockSource(psRgxDevNode) != TRACE_CLK)
+	{
+		/* Set clock source for timer correlation data to hwperf clock */
+		eError = RGXTimeCorrSetClockSource(psRgxDevNode, TRACE_CLK);
+		PVR_LOG_IF_ERROR(eError, "RGXTimeCorrSetClockSource");
+	}
+#endif
+
+	/* Reset the ordinal tracking flag. We should skip checking for gaps in
+	 * ordinal on the first run after FTrace is enabled in case another HWPerf
+	 * consumer was connected while FTrace was disabled. */
+	psFtraceData->bTrackOrdinals = IMG_FALSE;
+
+	/* Reset the OS timestamp coming from the timer correlation data
+	 * associated with the latest HWPerf event we processed.
+	 */
+	psFtraceData->ui64LastSampledTimeCorrOSTimeStamp = 0;
+
+	/* Register a notifier to collect HWPerf data whenever the HW completes
+	 * an operation.
+	 */
+	eError = PVRSRVRegisterCmdCompleteNotify(
+		&psFtraceData->hGPUTraceCmdCompleteHandle,
+		&_GpuTraceCmdCompleteNotify,
+		psRgxDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterCmdCompleteNotify", err_close_stream);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+err_close_stream:
+	TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+						psFtraceData->hGPUFTraceTLStream);
+	psFtraceData->hGPUFTraceTLStream = NULL;
+	goto err_out;
+}
+
+/* Caller must now hold hFTraceResourceLock before calling this method.
+ */
+static PVRSRV_ERROR _GpuTraceDisable(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_BOOL bDeInit)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_NODE *psRgxDevNode = psRgxDevInfo->psDeviceNode;
+#endif
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	PVR_ASSERT(OSLockIsLocked(psFtraceData->hFTraceResourceLock));
+
+	/* if FW is not yet initialised, just set filter and exit */
+	if (!psRgxDevInfo->bFirmwareInitialised)
+	{
+		(void) RGXHWPerfFwSetEventFilter(psRgxDevInfo, RGX_HWPERF_L2_STREAM_FTRACE,
+		                                 RGX_HWPERF_EVENT_MASK_NONE);
+		return PVRSRV_OK;
+	}
+
+	if (psFtraceData->hGPUFTraceTLStream == NULL)
+	{
+		/* Tracing already disabled, just return */
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_RGX)
+	if (!bDeInit)
+	{
+		eError = PVRSRVRGXCtrlHWPerfFW(psRgxDevNode,
+		                               RGX_HWPERF_L2_STREAM_FTRACE,
+		                               RGX_HWPERF_EVENT_MASK_NONE,
+		                               HWPERF_FILTER_OPERATION_SET);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfFW");
+	}
+#endif
+
+	if (psFtraceData->hGPUTraceCmdCompleteHandle)
+	{
+		/* Tracing is being turned off. Unregister the notifier. */
+		eError = PVRSRVUnregisterCmdCompleteNotify(
+				psFtraceData->hGPUTraceCmdCompleteHandle);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVUnregisterCmdCompleteNotify");
+		psFtraceData->hGPUTraceCmdCompleteHandle = NULL;
+	}
+
+	/* Let close stream perform the release data on the outstanding acquired
+	 * data */
+	eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+	                             psFtraceData->hGPUFTraceTLStream);
+	PVR_LOG_IF_ERROR(eError, "TLClientCloseStream");
+
+	psFtraceData->hGPUFTraceTLStream = NULL;
+
+#if defined(SUPPORT_RGX)
+	if (psRgxDevInfo->ui32LastClockSource != TRACE_CLK)
+	{
+		RGXTimeCorrSetClockSource(psRgxDevNode, psRgxDevInfo->ui32LastClockSource);
+	}
+#endif
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                        IMG_BOOL bNewValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	/* About to create/destroy FTrace resources, lock critical section
+	 * to avoid HWPerf MISR thread contention.
+	 */
+	OSLockAcquire(psFtraceData->hFTraceResourceLock);
+
+	eError = (bNewValue ? _GpuTraceEnable(psRgxDevInfo)
+					   : _GpuTraceDisable(psRgxDevInfo, IMG_FALSE));
+
+	OSLockRelease(psFtraceData->hFTraceResourceLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabledForAllDevices(IMG_BOOL bNewValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	/* enable/disable GPU trace on all devices */
+	while (psDeviceNode)
+	{
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			eError = _GpuTraceSetEnabled(psRgxDevInfo, bNewValue);
+			if (eError != PVRSRV_OK)
+				break;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR PVRGpuTraceSetEnabled(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_BOOL bNewValue)
+{
+	return _GpuTraceSetEnabled(psDeviceNode->pvDevice, bNewValue);
+}
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+/* ----- HWPerf to FTrace packet processing and events injection ------------ */
+
+static const IMG_CHAR *_HWPerfKickTypeToStr(RGX_HWPERF_KICK_TYPE eKickType)
+{
+	static const IMG_CHAR *aszKickType[RGX_HWPERF_KICK_TYPE2_LAST+1] = {
+		"TA3D", /* Deprecated */
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+		/* Volcanic deprecated kick types */
+		"CDM", "RS", "SHG", "TQTDM", "SYNC", "TA", "3D", "LAST",
+
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+#else
+		/* Rogue deprecated kick types */
+		"TQ2D", "TQ3D", "CDM", "RS", "VRDM", "TQTDM", "SYNC", "TA", "3D", "LAST",
+
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+		"<UNKNOWN>",
+#endif
+		"TQ2D", "TQ3D", "TQTDM", "CDM", "GEOM", "3D", "SYNC", "RS", "LAST"
+	};
+
+	/* cast in case of negative value */
+	if (((IMG_UINT32) eKickType) >= RGX_HWPERF_KICK_TYPE2_LAST)
+	{
+		return "<UNKNOWN>";
+	}
+
+	return aszKickType[eKickType];
+}
+void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExtJobRef,
+		IMG_UINT32 ui32IntJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType)
+{
+	const IMG_CHAR *pszKickType = _HWPerfKickTypeToStr(eKickType);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "PVRGpuTraceEnqueueEvent(%s): contextId %u, "
+	        "jobId %u", pszKickType, ui32FirmwareCtx, ui32IntJobRef));
+
+	if (PVRGpuTraceIsEnabled())
+	{
+		trace_rogue_job_enqueue(psDevNode->sDevId.ui32InternalID, ui32FirmwareCtx,
+		                        ui32IntJobRef, ui32ExtJobRef, pszKickType);
+	}
+}
+
+static void _GpuTraceWorkSwitch(
+		IMG_UINT64 ui64HWTimestampInOSTime,
+		IMG_UINT32 ui32GpuId,
+		IMG_UINT32 ui32CtxId,
+		IMG_UINT32 ui32CtxPriority,
+		IMG_UINT32 ui32ExtJobRef,
+		IMG_UINT32 ui32IntJobRef,
+		const IMG_CHAR* pszWorkType,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	PVR_ASSERT(pszWorkType);
+	trace_rogue_sched_switch(pszWorkType, eSwType, ui64HWTimestampInOSTime,
+	                         ui32GpuId, ui32CtxId, 2-ui32CtxPriority, ui32IntJobRef,
+	                         ui32ExtJobRef);
+}
+
+static void _GpuTraceUfo(
+		IMG_UINT64 ui64OSTimestamp,
+		const RGX_HWPERF_UFO_EV eEvType,
+		const IMG_UINT32 ui32GpuId,
+		const IMG_UINT32 ui32CtxId,
+		const IMG_UINT32 ui32ExtJobRef,
+		const IMG_UINT32 ui32IntJobRef,
+		const IMG_UINT32 ui32UFOCount,
+		const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	switch (eEvType) {
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			trace_rogue_ufo_updates(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, ui32UFOCount, puData);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+			trace_rogue_ufo_checks_success(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_FALSE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			trace_rogue_ufo_checks_success(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_TRUE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+			trace_rogue_ufo_checks_fail(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_FALSE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			trace_rogue_ufo_checks_fail(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_TRUE, ui32UFOCount,
+					puData);
+			break;
+		default:
+			break;
+	}
+}
+
+static void _GpuTraceFirmware(
+		IMG_UINT64 ui64HWTimestampInOSTime,
+		IMG_UINT32 ui32GpuId,
+		const IMG_CHAR* pszWorkType,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	trace_rogue_firmware_activity(ui64HWTimestampInOSTime, ui32GpuId, pszWorkType, eSwType);
+}
+
+static void _GpuTraceEventsLost(
+		const RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT32 ui32GpuId,
+		const IMG_UINT32 ui32LastOrdinal,
+		const IMG_UINT32 ui32CurrOrdinal)
+{
+	trace_rogue_events_lost(eStreamId, ui32GpuId, ui32LastOrdinal, ui32CurrOrdinal);
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+/* Calculate the OS timestamp given an RGX timestamp in the HWPerf event. */
+static uint64_t CalculateEventTimestamp(
+	PVRSRV_RGXDEV_INFO *psDevInfo,
+	uint32_t ui32TimeCorrIndex,
+	uint64_t ui64EventTimestamp)
+{
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData = psDevInfo->pvGpuFtraceData;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	uint64_t ui64CRTimeStamp;
+	uint64_t ui64OSTimeStamp;
+	uint64_t ui64CRDeltaToOSDeltaKNs;
+	uint64_t ui64EventOSTimestamp, deltaRgxTimer, delta_ns;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32TimeCorrIndex], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32TimeCorrIndex];
+	ui64CRTimeStamp = psTimeCorr->ui64CRTimeStamp;
+	/* This is configurable using the AppHint 'SecondaryTimerCorrOSClockSource'
+	 * and can be: sched, mono, mono_raw. */
+	ui64OSTimeStamp = psTimeCorr->ui64OSTimeStamp;
+	ui64CRDeltaToOSDeltaKNs = psTimeCorr->ui64CRDeltaToOSDeltaKNs;
+
+	if (psFtraceData->ui64LastSampledTimeCorrOSTimeStamp > ui64OSTimeStamp)
+	{
+		/* The previous packet had a time reference (time correlation data) more
+		 * recent than the one in the current packet, it means the timer
+		 * correlation array wrapped too quickly (buffer too small) and in the
+		 * previous call to _GpuTraceUfoEvent we read one of the
+		 * newest timer correlations rather than one of the oldest ones.
+		 */
+		PVR_DPF((PVR_DBG_ERROR, "%s: The timestamps computed so far could be "
+				 "wrong! The time correlation array size should be increased "
+				 "to avoid this.", __func__));
+	}
+
+	psFtraceData->ui64LastSampledTimeCorrOSTimeStamp = ui64OSTimeStamp;
+
+	/* RGX CR timer ticks delta */
+	deltaRgxTimer = ui64EventTimestamp - ui64CRTimeStamp;
+	/* RGX time delta in nanoseconds */
+	delta_ns = RGXFWIF_GET_DELTA_OSTIME_NS(deltaRgxTimer, ui64CRDeltaToOSDeltaKNs);
+	/* Calculate OS time of HWPerf event */
+	ui64EventOSTimestamp = ui64OSTimeStamp + delta_ns;
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: psCurrentDvfs RGX %llu, OS %llu, DVFSCLK %u",
+			 __func__, ui64CRTimeStamp, ui64OSTimeStamp,
+			 psTimeCorr->ui32CoreClockSpeed));
+
+	return ui64EventOSTimestamp;
+}
+
+static void _GpuTraceSwitchEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt, const IMG_CHAR* pszWorkName,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	IMG_UINT64 ui64Timestamp;
+	RGX_HWPERF_HW_DATA* psHWPerfPktData;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psHWPerfPkt);
+	PVR_ASSERT(pszWorkName);
+
+	psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceSwitchEvent: %s ui32ExtJobRef=%d, ui32IntJobRef=%d, eSwType=%d",
+			 pszWorkName != NULL ? pszWorkName : "?", psHWPerfPktData->ui32DMContext,
+			 psHWPerfPktData->ui32IntJobRef, eSwType));
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	_GpuTraceWorkSwitch(ui64Timestamp,
+	                    psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+	                    psHWPerfPktData->ui32DMContext,
+	                    psHWPerfPktData->ui32CtxPriority,
+	                    psHWPerfPktData->ui32ExtJobRef,
+	                    psHWPerfPktData->ui32IntJobRef,
+	                    pszWorkName,
+	                    eSwType);
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+	PVR_DPF_RETURN;
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+static void _GpuTraceUfoEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt)
+{
+	IMG_UINT64 ui64Timestamp;
+	RGX_HWPERF_UFO_DATA *psHWPerfPktData;
+	IMG_UINT32 ui32UFOCount;
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData;
+
+	psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui32UFOCount = RGX_HWPERF_GET_UFO_STREAMSIZE(psHWPerfPktData->ui32StreamInfo);
+	puData = (RGX_HWPERF_UFO_DATA_ELEMENT *) IMG_OFFSET_ADDR(psHWPerfPktData, RGX_HWPERF_GET_UFO_STREAMOFFSET(psHWPerfPktData->ui32StreamInfo));
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceUfoEvent: ui32ExtJobRef=%d, "
+	        "ui32IntJobRef=%d", psHWPerfPktData->ui32ExtJobRef,
+	        psHWPerfPktData->ui32IntJobRef));
+
+	_GpuTraceUfo(ui64Timestamp, psHWPerfPktData->eEvType,
+	             psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+	             psHWPerfPktData->ui32DMContext, psHWPerfPktData->ui32ExtJobRef,
+	             psHWPerfPktData->ui32IntJobRef, ui32UFOCount, puData);
+}
+
+static void _GpuTraceFirmwareEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt, const IMG_CHAR* pszWorkName,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+
+{
+	uint64_t ui64Timestamp;
+	RGX_HWPERF_FW_DATA *psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	_GpuTraceFirmware(ui64Timestamp, psDevInfo->psDeviceNode->sDevId.ui32InternalID, pszWorkName,
+	                  eSwType);
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+static IMG_BOOL ValidAndEmitFTraceEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt)
+{
+	RGX_HWPERF_EVENT_TYPE eType;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData = psDevInfo->pvGpuFtraceData;
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	IMG_UINT32 ui32HwEventTypeIndex;
+	static const struct {
+		IMG_CHAR* pszName;
+		PVR_GPUTRACE_SWITCH_TYPE eSwType;
+	} aszHwEventTypeMap[] = {
+#define _T(T) PVR_GPUTRACE_SWITCH_TYPE_##T
+		{ "BG",             _T(BEGIN)  }, /* RGX_HWPERF_FW_BGSTART */
+		{ "BG",             _T(END)    }, /* RGX_HWPERF_FW_BGEND */
+		{ "IRQ",            _T(BEGIN)  }, /* RGX_HWPERF_FW_IRQSTART */
+		{ "IRQ",            _T(END)    }, /* RGX_HWPERF_FW_IRQEND */
+		{ "DBG",            _T(BEGIN)  }, /* RGX_HWPERF_FW_DBGSTART */
+		{ "DBG",            _T(END)    }, /* RGX_HWPERF_FW_DBGEND */
+		{ "PMOOM_TAPAUSE",  _T(END)    }, /* RGX_HWPERF_HW_PMOOM_TAPAUSE */
+		{ "TA",             _T(BEGIN)  }, /* RGX_HWPERF_HW_TAKICK */
+		{ "TA",             _T(END)    }, /* RGX_HWPERF_HW_TAFINISHED */
+		{ "TQ3D",           _T(BEGIN)  }, /* RGX_HWPERF_HW_3DTQKICK */
+		{ "3D",             _T(BEGIN)  }, /* RGX_HWPERF_HW_3DKICK */
+		{ "3D",             _T(END)    }, /* RGX_HWPERF_HW_3DFINISHED */
+		{ "CDM",            _T(BEGIN)  }, /* RGX_HWPERF_HW_CDMKICK */
+		{ "CDM",            _T(END)    }, /* RGX_HWPERF_HW_CDMFINISHED */
+		{ "TQ2D",           _T(BEGIN)  }, /* RGX_HWPERF_HW_TLAKICK */
+		{ "TQ2D",           _T(END)    }, /* RGX_HWPERF_HW_TLAFINISHED */
+		{ "3DSPM",          _T(BEGIN)  }, /* RGX_HWPERF_HW_3DSPMKICK */
+		{ NULL,             0          }, /* RGX_HWPERF_HW_PERIODIC (unsupported) */
+		{ "RTU",            _T(BEGIN)  }, /* RGX_HWPERF_HW_RTUKICK */
+		{ "RTU",            _T(END)    }, /* RGX_HWPERF_HW_RTUFINISHED */
+		{ "SHG",            _T(BEGIN)  }, /* RGX_HWPERF_HW_SHGKICK */
+		{ "SHG",            _T(END)    }, /* RGX_HWPERF_HW_SHGFINISHED */
+		{ "TQ3D",           _T(END)    }, /* RGX_HWPERF_HW_3DTQFINISHED */
+		{ "3DSPM",          _T(END)    }, /* RGX_HWPERF_HW_3DSPMFINISHED */
+		{ "PMOOM_TARESUME", _T(BEGIN)  }, /* RGX_HWPERF_HW_PMOOM_TARESUME */
+		{ "TDM",            _T(BEGIN)  }, /* RGX_HWPERF_HW_TDMKICK */
+		{ "TDM",            _T(END)    }, /* RGX_HWPERF_HW_TDMFINISHED */
+		{ "NULL",           _T(SINGLE) }, /* RGX_HWPERF_HW_NULLKICK */
+		{ "GPU_WORK_PERIOD",_T(SINGLE) }, /* RGX_HWPERF_HW_GPU_WORK_PERIOD */
+#undef _T
+	};
+	static_assert(RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE == RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE + 1,
+				  "FW and HW events are not contiguous in RGX_HWPERF_EVENT_TYPE");
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+	PVR_ASSERT(psHWPerfPkt);
+	eType = RGX_HWPERF_GET_TYPE(psHWPerfPkt);
+
+	if (psFtraceData->bTrackOrdinals)
+	{
+		if (psFtraceData->ui32FTraceLastOrdinal != psHWPerfPkt->ui32Ordinal - 1)
+		{
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+			_GpuTraceEventsLost(RGX_HWPERF_GET_STREAM_ID(psHWPerfPkt),
+			                    psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+			                    psFtraceData->ui32FTraceLastOrdinal,
+			                    psHWPerfPkt->ui32Ordinal);
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+			PVR_DPF((PVR_DBG_WARNING, "FTrace events lost (stream_id = %u, ordinal: last = %u, current = %u)",
+			         RGX_HWPERF_GET_STREAM_ID(psHWPerfPkt), psFtraceData->ui32FTraceLastOrdinal,
+			         psHWPerfPkt->ui32Ordinal));
+		}
+	}
+	else
+	{
+		psFtraceData->bTrackOrdinals = IMG_TRUE;
+	}
+
+	psFtraceData->ui32FTraceLastOrdinal = psHWPerfPkt->ui32Ordinal;
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	/* Process UFO packets */
+	if (eType == RGX_HWPERF_UFO)
+	{
+		_GpuTraceUfoEvent(psDevInfo, psHWPerfPkt);
+		return IMG_TRUE;
+	}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	if (eType <= RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE)
+	{
+		/* this ID belongs to range 0, so index directly in range 0 */
+		ui32HwEventTypeIndex = eType - RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+	}
+	else
+	{
+		/* this ID belongs to range 1, so first index in range 1 and skip number of slots used up for range 0 */
+		ui32HwEventTypeIndex = (eType - RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE) +
+		                       (RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE - RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE + 1);
+	}
+
+	if (ui32HwEventTypeIndex >= ARRAY_SIZE(aszHwEventTypeMap))
+	{
+		goto err_unsupported;
+	}
+
+	if (aszHwEventTypeMap[ui32HwEventTypeIndex].pszName == NULL)
+	{
+		/* Not supported map entry, ignore event */
+		goto err_unsupported;
+	}
+
+	if (HWPERF_PACKET_IS_HW_TYPE(eType))
+	{
+		if (aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType == PVR_GPUTRACE_SWITCH_TYPE_SINGLE)
+		{
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     PVR_GPUTRACE_SWITCH_TYPE_BEGIN);
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     PVR_GPUTRACE_SWITCH_TYPE_END);
+		}
+		else
+		{
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType);
+		}
+
+		return IMG_TRUE;
+	}
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	else if (psDevInfo->psDeviceNode->bGPUWorkPeriodFTraceEnabled &&
+		 eType == RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD &&
+		 RGX_HWPERF_GET_OSID(psHWPerfPkt) == RGXFW_HOST_DRIVER_ID)
+	{
+		RGX_HWPERF_GPU_WORK_PERIOD_DATA *psHWPerfPktData;
+		IMG_UINT64 ui64StartTimestamp, ui64EndTimestamp;
+
+		psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+		ui64StartTimestamp = CalculateEventTimestamp(psDevInfo,
+					psHWPerfPktData->ui32StartTimeCorrIndex,
+					psHWPerfPktData->ui64GPUWorkPeriodStartTime);
+		ui64EndTimestamp = CalculateEventTimestamp(psDevInfo,
+					psHWPerfPktData->ui32TimeCorrIndex,
+					psHWPerfPkt->ui64Timestamp);
+
+		PVR_ASSERT(ui64EndTimestamp > ui64StartTimestamp);
+
+		trace_gpu_work_period(psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+				psHWPerfPktData->ui32UID,
+				ui64StartTimestamp,
+				ui64EndTimestamp,
+				ui64EndTimestamp - ui64StartTimestamp);
+
+		return IMG_TRUE;
+	}
+#endif
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	if (HWPERF_PACKET_IS_FW_TYPE(eType))
+	{
+		_GpuTraceFirmwareEvent(psDevInfo, psHWPerfPkt,
+										aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+										aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType);
+
+		return IMG_TRUE;
+	}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+err_unsupported:
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Unsupported event type %d", __func__, eType));
+	return IMG_FALSE;
+}
+
+static void _GpuTraceProcessPackets(PVRSRV_RGXDEV_INFO *psDevInfo, void *pBuffer,
+                                    IMG_UINT32 ui32ReadLen)
+{
+	IMG_UINT32			ui32TlPackets = 0;
+	IMG_UINT32			ui32HWPerfPackets = 0;
+	IMG_UINT32			ui32HWPerfPacketsSent = 0;
+	void				*pBufferEnd;
+	PVRSRVTL_PPACKETHDR psHDRptr;
+	PVRSRVTL_PACKETTYPE ui16TlType;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDevInfo);
+	PVR_ASSERT(pBuffer);
+	PVR_ASSERT(ui32ReadLen);
+
+	/* Process the TL Packets
+	 */
+	pBufferEnd = IMG_OFFSET_ADDR(pBuffer, ui32ReadLen);
+	psHDRptr = GET_PACKET_HDR(pBuffer);
+	while ( psHDRptr < (PVRSRVTL_PPACKETHDR)pBufferEnd )
+	{
+		ui16TlType = GET_PACKET_TYPE(psHDRptr);
+		if (ui16TlType == PVRSRVTL_PACKETTYPE_DATA)
+		{
+			IMG_UINT16 ui16DataLen = GET_PACKET_DATA_LEN(psHDRptr);
+			if (0 == ui16DataLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "_GpuTraceProcessPackets: ZERO Data in TL data packet: %p", psHDRptr));
+			}
+			else
+			{
+				RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt;
+				RGX_HWPERF_V2_PACKET_HDR* psHWPerfEnd;
+
+				/* Check for lost hwperf data packets */
+				psHWPerfEnd = RGX_HWPERF_GET_PACKET(GET_PACKET_DATA_PTR(psHDRptr)+ui16DataLen);
+				psHWPerfPkt = RGX_HWPERF_GET_PACKET(GET_PACKET_DATA_PTR(psHDRptr));
+				do
+				{
+					if (ValidAndEmitFTraceEvent(psDevInfo, psHWPerfPkt))
+					{
+						ui32HWPerfPacketsSent++;
+					}
+					ui32HWPerfPackets++;
+					psHWPerfPkt = RGX_HWPERF_GET_NEXT_PACKET(psHWPerfPkt);
+				}
+				while (psHWPerfPkt < psHWPerfEnd);
+			}
+		}
+		else if (ui16TlType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "_GpuTraceProcessPackets: Indication that the transport buffer was full"));
+		}
+		else
+		{
+			/* else Ignore padding packet type and others */
+			PVR_DPF((PVR_DBG_MESSAGE, "_GpuTraceProcessPackets: Ignoring TL packet, type %d", ui16TlType ));
+		}
+
+		psHDRptr = GET_NEXT_PACKET_ADDR(psHDRptr);
+		ui32TlPackets++;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceProcessPackets: TL "
+			"Packets processed %03d, HWPerf packets %03d, sent %03d",
+			ui32TlPackets, ui32HWPerfPackets, ui32HWPerfPacketsSent));
+
+	PVR_DPF_RETURN;
+}
+
+static void _GpuTraceCmdCompleteNotify(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_RGXDEV_INFO *psDeviceInfo = hCmdCompHandle;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+	PVRSRV_ERROR eError;
+	IMG_PBYTE pBuffer;
+	IMG_UINT32 ui32ReadLen;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDeviceInfo != NULL);
+
+	psFtraceData = psDeviceInfo->pvGpuFtraceData;
+
+	/* Command-complete notifiers can run concurrently. If this is happening,
+	 * just bail out and let the previous call finish.
+	 * This is ok because we can process the queued packets on the next call.
+	 */
+	if (!OSTryLockAcquire(psFtraceData->hFTraceResourceLock))
+	{
+		PVR_DPF_RETURN;
+	}
+
+	/* If this notifier is called, it means the TL resources will be valid
+	 * at-least until the end of this call, since the DeInit function will wait
+	 * on the hFTraceResourceLock to clean-up the TL resources and un-register
+	 * the notifier, so just assert here.
+	 */
+	PVR_ASSERT(psFtraceData->hGPUFTraceTLStream != NULL);
+
+	/* If we have a valid stream attempt to acquire some data */
+	eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+	                             psFtraceData->hGPUFTraceTLStream,
+	                             &pBuffer, &ui32ReadLen);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_LOG_ERROR(eError, "TLClientAcquireData");
+		}
+
+		goto unlock;
+	}
+
+	/* Process the HWPerf packets and release the data */
+	if (ui32ReadLen > 0)
+	{
+		PVR_DPF((PVR_DBG_VERBOSE, "%s: DATA AVAILABLE offset=%p, length=%d",
+		        __func__, pBuffer, ui32ReadLen));
+
+		/* Process the transport layer data for HWPerf packets... */
+		_GpuTraceProcessPackets(psDeviceInfo, pBuffer, ui32ReadLen);
+
+		eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE,
+		                             psFtraceData->hGPUFTraceTLStream);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "TLClientReleaseData");
+
+			/* Serious error, disable FTrace GPU events */
+			_GpuTraceDisable(psDeviceInfo, IMG_FALSE);
+		}
+	}
+
+unlock:
+	OSLockRelease(psFtraceData->hFTraceResourceLock);
+
+	PVR_DPF_RETURN;
+}
+
+/* ----- AppHint interface -------------------------------------------------- */
+
+static PVRSRV_ERROR _GpuTraceIsEnabledCallback(
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data,
+	IMG_BOOL *value)
+{
+	PVR_UNREFERENCED_PARAMETER(private_data);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (device && PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, device))
+	{
+		*value = IMG_FALSE;
+
+		return PVRSRV_OK;
+	}
+	else
+#endif
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+		PVRSRV_DEVICE_NODE *psDeviceNode = psPVRSRVData ? psPVRSRVData->psDeviceNodeList : NULL;
+
+		while (psDeviceNode)
+		{
+			PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+			{
+				*value = gbFTraceGPUEventsEnabled;
+				return PVRSRV_OK;
+			}
+
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		*value = IMG_FALSE;
+
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#else
+		*value = gbFTraceGPUEventsEnabled;
+
+		return PVRSRV_OK;
+#endif
+	}
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabledCallback(
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data,
+	IMG_BOOL value)
+{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (device && PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, device))
+	{
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	/* Lock down the state to avoid concurrent writes */
+	OSLockAcquire(ghGPUTraceStateLock);
+
+	if (value != gbFTraceGPUEventsEnabled)
+	{
+#if defined(PVRSRV_NEED_PVR_DPF)
+		const IMG_CHAR *pszOperation = value ? "enable" : "disable";
+		/* in case MESSAGE level is compiled out */
+		PVR_UNREFERENCED_PARAMETER(pszOperation);
+#endif
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		if (trace_set_clr_event("rogue", NULL, (int) value) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s GPU FTrace event group",
+			        pszOperation));
+			goto err_restore_state;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "FTrace events from \"rogue\" group %sd",
+			        pszOperation));
+		}
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+		if (trace_set_clr_event("power", "gpu_work_period", (int) value) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s gpu_work_period GPU FTrace event",
+			        pszOperation));
+			goto err_restore_state;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "FTrace event from \"gpu_work_period\" %sd",
+			        pszOperation));
+		}
+#endif
+
+		if (value)
+		{
+			/* this enables FTrace globally (if not enabled nothing will appear
+			 * in the FTrace buffer) */
+			tracing_on();
+		}
+
+		/*  The HWPerf supplier is activated here,
+		    The FTrace consumer is activated above,
+		    The consumer should be active before the supplier    */
+		if (_GpuTraceSetEnabledForAllDevices(value) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s GPU FTrace for all devices",
+			        pszOperation));
+			goto err_restore_state;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s GPU FTrace", value ? "ENABLED" : "DISABLED"));
+		gbFTraceGPUEventsEnabled = value;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "GPU FTrace already %s!",
+		        value ? "enabled" : "disabled"));
+	}
+
+	OSLockRelease(ghGPUTraceStateLock);
+
+	return PVRSRV_OK;
+
+err_restore_state:
+	/* On failure, partial enable/disable might have resulted. Try best to
+	 * restore to previous state. Ignore error */
+	(void) _GpuTraceSetEnabledForAllDevices(gbFTraceGPUEventsEnabled);
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	(void) trace_set_clr_event("rogue", NULL,
+	                           (int) gbFTraceGPUEventsEnabled);
+#endif
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	(void) trace_set_clr_event("power", "gpu_work_period",
+	                           (int) gbFTraceGPUEventsEnabled);
+#endif
+
+	OSLockRelease(ghGPUTraceStateLock);
+
+	return PVRSRV_ERROR_UNABLE_TO_ENABLE_EVENT;
+}
+
+void PVRGpuTraceInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFTraceGPU,
+	                                  _GpuTraceIsEnabledCallback,
+	                                  _GpuTraceSetEnabledCallback,
+	                                  psDeviceNode, NULL);
+}
+
+/* ----- FTrace event callbacks -------------------------------------------- */
+
+void PVRGpuTraceEnableUfoCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* Lock down events state, for consistent value of guiUfoEventRef */
+	OSLockAcquire(ghLockFTraceEventLock);
+	if (guiUfoEventRef++ == 0)
+	{
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+		/* make sure UFO events are enabled on all rogue devices */
+		while (psDeviceNode)
+		{
+#if defined(SUPPORT_RGX)
+			PVRSRV_ERROR eError;
+
+			/* Small chance exists that ui64HWPerfFilter can be changed here and
+			 * the newest filter value will be changed to the old one + UFO event.
+			 * This is not a critical problem. */
+			eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+			                               RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO),
+			                               HWPERF_FILTER_OPERATION_BIT_OR);
+			if (eError == PVRSRV_ERROR_NOT_INITIALISED)
+			{
+				/* If we land here that means that the FW is not initialised yet.
+				 * We stored the filter and it will be passed to the firmware
+				 * during its initialisation phase. So ignore. */
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Could not enable UFO HWPerf events on device %d", psDeviceNode->sDevId.i32KernelDeviceID));
+			}
+#endif
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+	OSLockRelease(ghLockFTraceEventLock);
+}
+
+void PVRGpuTraceDisableUfoCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* We have to check if lock is valid because on driver unload
+	 * PVRGpuTraceSupportDeInit is called before kernel disables the ftrace
+	 * events. This means that the lock will be destroyed before this callback
+	 * is called.
+	 * We can safely return if that situation happens because driver will be
+	 * unloaded so we don't care about HWPerf state anymore. */
+	if (ghLockFTraceEventLock == NULL)
+		return;
+
+	/* Lock down events state, for consistent value of guiUfoEventRef */
+	OSLockAcquire(ghLockFTraceEventLock);
+	if (--guiUfoEventRef == 0)
+	{
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+		/* make sure UFO events are disabled on all rogue devices */
+		while (psDeviceNode)
+		{
+#if defined(SUPPORT_RGX)
+			PVRSRV_ERROR eError;
+
+			/* Small chance exists that ui64HWPerfFilter can be changed here and
+			 * the newest filter value will be changed to the old one + UFO event.
+			 * This is not a critical problem. */
+			eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+			                               RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO),
+			                               HWPERF_FILTER_OPERATION_BIT_CLR);
+			if (eError == PVRSRV_ERROR_NOT_INITIALISED)
+			{
+				/* If we land here that means that the FW is not initialised yet.
+				 * We stored the filter and it will be passed to the firmware
+				 * during its initialisation phase. So ignore. */
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Could not disable UFO HWPerf events on device %d",
+				        psDeviceNode->sDevId.i32KernelDeviceID));
+			}
+#endif
+
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+	OSLockRelease(ghLockFTraceEventLock);
+}
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_UINT64 uiFilter = 0;
+	int i;
+
+	for (i = RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+		 i <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE; i++)
+	{
+		uiFilter |= RGX_HWPERF_EVENT_MASK_VALUE(i);
+	}
+
+	OSLockAcquire(ghLockFTraceEventLock);
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	/* Enable all FW events on all the devices */
+	while (psDeviceNode)
+	{
+		PVRSRV_ERROR eError;
+
+		eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+		                               uiFilter, HWPERF_FILTER_OPERATION_BIT_OR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could not enable HWPerf event for firmware"
+			        " task timings (%s).", PVRSRVGetErrorString(eError)));
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	OSLockRelease(ghLockFTraceEventLock);
+#endif /* defined(SUPPORT_RGX) */
+}
+
+void PVRGpuTraceDisableFirmwareActivityCallback(void)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_UINT64 uiFilter = 0;
+	int i;
+
+	/* We have to check if lock is valid because on driver unload
+	 * PVRGpuTraceSupportDeInit is called before kernel disables the ftrace
+	 * events. This means that the lock will be destroyed before this callback
+	 * is called.
+	 * We can safely return if that situation happens because driver will be
+	 * unloaded so we don't care about HWPerf state anymore. */
+	if (ghLockFTraceEventLock == NULL)
+		return;
+
+	OSLockAcquire(ghLockFTraceEventLock);
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	for (i = RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+		 i <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE; i++)
+	{
+		uiFilter |= RGX_HWPERF_EVENT_MASK_VALUE(i);
+	}
+
+	/* Disable all FW events on all the devices */
+	while (psDeviceNode)
+	{
+		if (PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+		                          uiFilter, HWPERF_FILTER_OPERATION_BIT_CLR) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could not disable HWPerf event for firmware task timings."));
+		}
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	OSLockRelease(ghLockFTraceEventLock);
+#endif /* defined(SUPPORT_RGX) */
+}
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableWorkPeriodCallback(void)
+#else
+void PVRGpuTraceEnableWorkPeriodCallback(void)
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	/* enable/disable GPU trace on all devices */
+	while (psDeviceNode)
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			psDeviceNode->bGPUWorkPeriodFTraceEnabled = true;
+			eError = PVRSRV_OK;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	if (eError != PVRSRV_OK)
+		return -ENODEV;
+	return 0;
+#else
+	return;
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+}
+
+void PVRGpuTraceDisableWorkPeriodCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	if (!psPVRSRVData)
+		return;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	while (psDeviceNode)
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			psDeviceNode->bGPUWorkPeriodFTraceEnabled = false;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+}
+#endif
+/******************************************************************************
+ End of file (pvr_gputrace.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_ion_stats.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_ion_stats.h
new file mode 100644
index 000000000000..c4748138ccd8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_ion_stats.h
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for recording ION memory stats.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_ION_STATS_H
+#define PVR_ION_STATS_H
+
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+struct dma_buf;
+
+#if defined(PVRSRV_ENABLE_PVR_ION_STATS)
+PVRSRV_ERROR PVRSRVIonStatsInitialise(void);
+
+void PVRSRVIonStatsDestroy(void);
+
+void PVRSRVIonAddMemAllocRecord(struct dma_buf *psDmaBuf);
+
+void PVRSRVIonRemoveMemAllocRecord(struct dma_buf *psDmaBuf);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void PVRSRVIonZombifyMemAllocRecord(const struct dma_buf *psDmaBuf);
+void PVRSRVIonReviveMemAllocRecord(const struct dma_buf *psDmaBuf);
+#endif
+#else
+static INLINE PVRSRV_ERROR PVRSRVIonStatsInitialise(void)
+{
+	return PVRSRV_OK;
+}
+
+static INLINE void PVRSRVIonStatsDestroy(void)
+{
+}
+
+static INLINE void PVRSRVIonAddMemAllocRecord(struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+
+static INLINE void PVRSRVIonRemoveMemAllocRecord(struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static INLINE void PVRSRVIonZombifyMemAllocRecord(const struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+static INLINE void PVRSRVIonReviveMemAllocRecord(const struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+#endif
+#endif /* defined(PVRSRV_ENABLE_PVR_ION_STATS) */
+
+#endif /* PVR_ION_STATS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_platform_drv.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_platform_drv.c
new file mode 100644
index 000000000000..9bffd02fc353
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_platform_drv.c
@@ -0,0 +1,378 @@
+/*
+ * @File
+ * @Title       PowerVR DRM platform driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_print.h>
+#include <linux/mod_devicetable.h>
+#include <linux/dma-mapping.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "module_common.h"
+#include "pvr_drv.h"
+#include "pvrmodule.h"
+#include "sysinfo.h"
+#include <linux/devfreq.h>
+#include <linux/pm_opp.h>
+
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+unsigned int _corefreq_div = 800000000;
+module_param_named(corefreq_div, _corefreq_div, uint, 0644);
+MODULE_PARM_DESC(corefreq_div, "core frequency div from 1600M, default 800M");
+MODULE_IMPORT_NS(DMA_BUF);
+
+static struct drm_driver pvr_drm_platform_driver;
+
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+/*
+ * This is an arbitrary value. If it's changed then the 'num_devices' module
+ * parameter description should also be updated to match.
+ */
+#define MAX_DEVICES 16
+static unsigned int pvr_num_devices = 1;
+static struct platform_device **pvr_devices;
+
+#if defined(NO_HARDWARE)
+static int pvr_num_devices_set(const char *val,
+			       const struct kernel_param *param)
+{
+	int err;
+
+	err = param_set_uint(val, param);
+	if (err)
+		return err;
+
+	if (pvr_num_devices == 0 || pvr_num_devices > PVRSRV_MAX_DEVICES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct kernel_param_ops pvr_num_devices_ops = {
+	.set = pvr_num_devices_set,
+	.get = param_get_uint,
+};
+
+#define STR(s) #s
+#define STRINGIFY(s) STR(s)
+
+module_param_cb(num_devices, &pvr_num_devices_ops, &pvr_num_devices, 0444);
+MODULE_PARM_DESC(num_devices,
+		 "Number of platform devices to register (default: 1 - max: "
+		 STRINGIFY(PVRSRV_MAX_DEVICES) ")");
+#endif /* defined(NO_HARDWARE) */
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+
+static int pvr_devices_register(void)
+{
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+	struct platform_device_info pvr_dev_info = {
+		.name = SYS_RGX_DEV_NAME,
+		.id = -2,
+#if defined(NO_HARDWARE)
+		/* Not all cores have 40 bit physical support, but this
+		 * will work unless > 32 bit address is returned on those cores.
+		 * In the future this will be fixed more correctly.
+		 */
+		.dma_mask = DMA_BIT_MASK(40),
+#else
+		.dma_mask = DMA_BIT_MASK(32),
+#endif
+	};
+	unsigned int i;
+
+	BUG_ON(pvr_num_devices == 0 || pvr_num_devices > PVRSRV_MAX_DEVICES);
+
+	pvr_devices = kmalloc_array(pvr_num_devices, sizeof(*pvr_devices),
+				    GFP_KERNEL);
+	if (!pvr_devices)
+		return -ENOMEM;
+
+	for (i = 0; i < pvr_num_devices; i++) {
+		pvr_devices[i] = platform_device_register_full(&pvr_dev_info);
+		if (IS_ERR(pvr_devices[i])) {
+			DRM_ERROR("unable to register device %u (err=%ld)\n",
+				  i, PTR_ERR(pvr_devices[i]));
+			pvr_devices[i] = NULL;
+			return -ENODEV;
+		}
+	}
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+
+	return 0;
+}
+
+static void pvr_devices_unregister(void)
+{
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+	unsigned int i;
+
+	BUG_ON(!pvr_devices);
+
+	for (i = 0; i < pvr_num_devices && pvr_devices[i]; i++)
+		platform_device_unregister(pvr_devices[i]);
+
+	kfree(pvr_devices);
+	pvr_devices = NULL;
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+}
+
+#if defined(CONFIG_PM_DEVFREQ)
+
+static void eswin_exit(struct device *dev)
+{
+	;
+}
+
+static int eswin_get_dev_status(struct device *dev,
+				     struct devfreq_dev_status *stat)
+{
+	stat->busy_time = 1024;	
+	stat->total_time = 1024;
+	igpu_devfreq_get_cur_freq(dev,&stat->current_frequency);
+
+	return 0;
+}
+
+/** devfreq profile */
+static struct devfreq_dev_profile igpu_devfreq_profile = {
+	.initial_freq = 800000000,
+	.timer = DEVFREQ_TIMER_DELAYED,
+	.polling_ms = 100, /** Poll every 1000ms to monitor load */
+	.target = igpu_devfreq_target,
+	.get_cur_freq = igpu_devfreq_get_cur_freq,
+	.get_dev_status = eswin_get_dev_status,
+	.exit = eswin_exit,
+	.is_cooling_device = true,
+};
+static struct devfreq_simple_ondemand_data ondemand_data =
+{
+	.upthreshold =80,
+	.downdifferential=10,
+};
+#endif
+
+static int pvr_probe(struct platform_device *pdev)
+{
+	struct drm_device *ddev;
+	int ret;
+#if defined(CONFIG_PM_DEVFREQ)
+	struct devfreq *df;
+#endif
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	ddev = drm_dev_alloc(&pvr_drm_platform_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+#if defined(CONFIG_PM_DEVFREQ)
+	/* Add OPP table from device tree */
+	ret = dev_pm_opp_of_add_table(&pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev,"%s, %d, Failed to add OPP table\n", __func__, __LINE__);
+		goto err_drm_dev_put;
+	}
+
+	df = devm_devfreq_add_device(&pdev->dev, &igpu_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND, &ondemand_data);
+	if (IS_ERR(df)) {
+		dev_err(&pdev->dev,"%s, %d, add devfreq failed\n", __func__, __LINE__);
+		ret = PTR_ERR(df);
+		goto err_drm_dev_put;
+	};
+
+	/* Register opp_notifier to catch the change of OPP  ????*/
+	ret = devm_devfreq_register_opp_notifier(&pdev->dev, df);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to register opp notifier\n");
+		return ret;
+	}
+#endif
+	/*
+	 * The load callback, called from drm_dev_register, is deprecated,
+	 * because of potential race conditions. Calling the function here,
+	 * before calling drm_dev_register, avoids those potential races.
+	 */
+	BUG_ON(pvr_drm_platform_driver.load != NULL);
+	ret = pvr_drm_load(ddev, 0);
+	if (ret)
+		goto err_drm_dev_put;
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto err_drm_dev_unload;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+		pvr_drm_platform_driver.name,
+		pvr_drm_platform_driver.major,
+		pvr_drm_platform_driver.minor,
+		pvr_drm_platform_driver.patchlevel,
+		pvr_drm_platform_driver.date,
+		ddev->primary->index);
+#endif
+	return 0;
+
+err_drm_dev_unload:
+	pvr_drm_unload(ddev);
+err_drm_dev_put:
+	drm_dev_put(ddev);
+	return	ret;
+}
+
+static int pvr_remove(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	drm_dev_unregister(ddev);
+
+	/* The unload callback, called from drm_dev_unregister, is
+	 * deprecated. Call the unload function directly.
+	 */
+	BUG_ON(pvr_drm_platform_driver.unload != NULL);
+	pvr_drm_unload(ddev);
+
+	drm_dev_put(ddev);
+	return 0;
+}
+
+static void pvr_shutdown(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	PVRSRVDeviceShutdown(ddev);
+}
+
+static const struct of_device_id pvr_of_ids[] = {
+#if defined(SYS_RGX_OF_COMPATIBLE)
+	{ .compatible = SYS_RGX_OF_COMPATIBLE, },
+#endif
+	{},
+};
+
+#if !defined(CHROMIUMOS_KERNEL)
+MODULE_DEVICE_TABLE(of, pvr_of_ids);
+#endif
+
+static struct platform_device_id pvr_platform_ids[] = {
+#if defined(SYS_RGX_DEV_NAME)
+	{ SYS_RGX_DEV_NAME, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_0)
+	{ SYS_RGX_DEV_NAME_0, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_1)
+	{ SYS_RGX_DEV_NAME_1, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_2)
+	{ SYS_RGX_DEV_NAME_2, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_3)
+	{ SYS_RGX_DEV_NAME_3, 0 },
+#endif
+	{ }
+};
+
+#if !defined(CHROMIUMOS_KERNEL)
+MODULE_DEVICE_TABLE(platform, pvr_platform_ids);
+#endif
+
+static struct platform_driver pvr_platform_driver = {
+	.driver = {
+		.name		= DRVNAME,
+		.of_match_table	= of_match_ptr(pvr_of_ids),
+		.pm		= &pvr_pm_ops,
+	},
+	.id_table		= pvr_platform_ids,
+	.probe			= pvr_probe,
+	.remove			= pvr_remove,
+	.shutdown		= pvr_shutdown,
+};
+
+static int __init pvr_init(void)
+{
+	int err;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	pvr_drm_platform_driver = pvr_drm_generic_driver;
+
+	err = PVRSRVDriverInit();
+	if (err)
+		return err;
+
+	err = platform_driver_register(&pvr_platform_driver);
+	if (err)
+		return err;
+
+	return pvr_devices_register();
+}
+
+static void __exit pvr_exit(void)
+{
+	DRM_DEBUG_DRIVER("\n");
+
+	pvr_devices_unregister();
+	platform_driver_unregister(&pvr_platform_driver);
+	PVRSRVDriverDeinit();
+
+	DRM_DEBUG_DRIVER("done\n");
+}
+
+module_init(pvr_init);
+module_exit(pvr_exit);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.c
new file mode 100644
index 000000000000..34408fc64875
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.c
@@ -0,0 +1,633 @@
+/*************************************************************************/ /*!
+@File
+@Title          ProcFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via ProcFS.
+
+                Note about locking in ProcFS module.
+
+                Access to ProcFS is protected against the race where any
+                file could be removed while being accessed or accessed while
+                being removed. Any calls to proc_remove() will block until all
+                operations are finished.
+
+                See implementation of file operations (proc_reg_*()) and
+                implementation of (un)use_pde() and proc_entry_rundown() in
+                source/fs/proc/inode.c in Linux kernel sources for more details.
+
+                Not about locking for sequential files.
+
+                The seq_file objects have a mutex that protects access
+                to all of the file operations hence all of the sequential
+                *read* operations are protected.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvr_procfs.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_bridge_k.h"
+#include "pvr_uaccess.h"
+#include "osdi_impl.h"
+#include "kernel_compatibility.h"
+
+#define _DRIVER_THREAD_ENTER() \
+	do { \
+		PVRSRV_ERROR eLocalError = PVRSRVDriverThreadEnter(NULL); \
+		if (eLocalError != PVRSRV_OK) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDriverThreadEnter failed: %s", \
+				__func__, PVRSRVGetErrorString(eLocalError))); \
+			return OSPVRSRVToNativeError(eLocalError); \
+		} \
+	} while (0)
+
+#define _DRIVER_THREAD_EXIT() \
+	PVRSRVDriverThreadExit(NULL)
+
+#define PVR_DEBUGFS_PVR_DPF_LEVEL PVR_DBG_ERROR
+
+typedef struct DFS_DIR
+{
+	struct proc_dir_entry *psDirEntry;
+	struct DFS_DIR *psParentDir;
+} DFS_DIR;
+
+typedef struct DFS_ENTRY
+{
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+} DFS_ENTRY;
+
+typedef struct DFS_FILE
+{
+	struct proc_dir_entry *psFileEntry;
+	struct DFS_DIR *psParentDir;
+	const struct seq_operations *psSeqOps;
+	struct DFS_ENTRY sEntry;
+	DI_ENTRY_TYPE eType;
+} DFS_FILE;
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	seq_write(pvNativeHandle, pvData, uiSize);
+}
+
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	seq_vprintf(pvNativeHandle, pszFmt, pArgs);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	seq_puts(pvNativeHandle, pszStr);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	struct seq_file *psSeqFile = pvNativeHandle;
+	return seq_has_overflowed(psSeqFile);
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- sequential file operations ----------------------------------------- */
+
+static void *_Start(struct seq_file *psSeqFile, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	void *pvRet = psEntry->sIterCb.pfnStart(&psEntry->sImplEntry, puiPos);
+
+	if (pvRet == DI_START_TOKEN)
+	{
+		return SEQ_START_TOKEN;
+	}
+
+	return pvRet;
+}
+
+static void _Stop(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	psEntry->sIterCb.pfnStop(&psEntry->sImplEntry, pvPriv);
+}
+
+static void *_Next(struct seq_file *psSeqFile, void *pvPriv, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	return psEntry->sIterCb.pfnNext(&psEntry->sImplEntry, pvPriv, puiPos);
+}
+
+static int _Show(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	if (pvPriv == SEQ_START_TOKEN)
+	{
+		pvPriv = DI_START_TOKEN;
+	}
+
+	return psEntry->sIterCb.pfnShow(&psEntry->sImplEntry, pvPriv);
+}
+
+static struct seq_operations _g_sSeqOps = {
+	.start = _Start,
+	.stop = _Stop,
+	.next = _Next,
+	.show = _Show
+};
+
+/* ----- file operations ---------------------------------------------------- */
+
+static int _Open(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = pde_data(psINode);
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL", -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_open(psFile, psDFSFile->psSeqOps);
+	}
+	else
+	{
+		/* private data is NULL as it's going to be set below */
+		iRes = single_open(psFile, _Show, NULL);
+	}
+
+	if (iRes == 0)
+	{
+		struct seq_file *psSeqFile = psFile->private_data;
+
+		DFS_ENTRY *psEntry = OSAllocMem(sizeof(*psEntry));
+		if (psEntry == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem() failed", __func__));
+			iRes = -ENOMEM;
+			goto return_;
+		}
+
+		*psEntry = psDFSFile->sEntry;
+		psSeqFile->private = psEntry;
+		psEntry->sImplEntry.pvNative = psSeqFile;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to seq_open psFile, returning %d",
+		        __func__, iRes));
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static int _Close(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = pde_data(psINode);
+	DFS_ENTRY *psEntry;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* save pointer to DFS_ENTRY */
+	psEntry = ((struct seq_file *) psFile->private_data)->private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_release(psINode, psFile);
+	}
+	else
+	{
+		iRes = single_release(psINode, psFile);
+	}
+
+	/* free DFS_ENTRY allocated in _Open */
+	OSFreeMem(psEntry);
+
+	/* Validation check as seq_release (and single_release which calls it)
+	 * never fail */
+	if (iRes != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release psFile, returning %d",
+		        __func__, iRes));
+	}
+
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Read(struct file *psFile, char __user *pcBuffer,
+                     size_t uiCount, loff_t *puiPos)
+{
+	DFS_FILE *psDFSFile = pde_data(psFile->f_path.dentry->d_inode);
+	ssize_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_read(psFile, pcBuffer, uiCount, puiPos);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file pfnRead() "
+			        "returned %zd", __func__, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Count = uiCount, ui64Pos;
+
+		IMG_CHAR *pcLocalBuffer = OSAllocMem(uiCount);
+		PVR_GOTO_IF_FALSE(pcLocalBuffer != NULL, return_);
+
+		iRes = psEntry->sIterCb.pfnRead(pcLocalBuffer, ui64Count, &ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file pfnRead() "
+			        "returned %zd", __func__, iRes));
+			OSFreeMem(pcLocalBuffer);
+			goto return_;
+		}
+
+		if (pvr_copy_to_user(pcBuffer, pcLocalBuffer, iRes) != 0)
+		{
+			iRes = -1;
+		}
+
+		OSFreeMem(pcLocalBuffer);
+
+		*puiPos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static loff_t _LSeek(struct file *psFile, loff_t iOffset, int iOrigin)
+{
+	DFS_FILE *psDFSFile = pde_data(psFile->f_path.dentry->d_inode);
+	loff_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_lseek(psFile, iOffset, iOrigin);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to "
+			        "offset %lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Pos;
+
+		switch (iOrigin)
+		{
+			case SEEK_SET:
+				ui64Pos = psFile->f_pos + iOffset;
+				break;
+			case SEEK_CUR:
+				ui64Pos = iOffset;
+				break;
+			case SEEK_END:
+				/* not supported as we don't know the file size here */
+				/* fall through */
+			default:
+				return -1;
+		}
+
+		/* only pass the absolute position to the callback, it's up to the
+		 * implementer to determine if the position is valid */
+
+		iRes = psEntry->sIterCb.pfnSeek(ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to "
+			        "offset %lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+
+		psFile->f_pos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Write(struct file *psFile, const char __user *pszBuffer,
+                      size_t uiCount, loff_t *puiPos)
+{
+	struct inode *psINode = psFile->f_path.dentry->d_inode;
+	DFS_FILE *psDFSFile = pde_data(psINode);
+	DI_ITERATOR_CB *psIter = &psDFSFile->sEntry.sIterCb;
+	IMG_CHAR *pcLocalBuffer;
+	IMG_UINT64 ui64Count;
+	IMG_INT64 i64Res = -EIO;
+	IMG_UINT64 ui64Pos = *puiPos;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+	PVR_LOG_RETURN_IF_FALSE(psIter->pfnWrite != NULL, "pfnWrite is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* Make sure we allocate the smallest amount of needed memory*/
+	ui64Count = psIter->ui32WriteLenMax;
+	PVR_LOG_GOTO_IF_FALSE(uiCount <= ui64Count, "uiCount too long", return_);
+	ui64Count = MIN(uiCount+1, ui64Count);
+
+	/* allocate buffer with one additional byte for NUL character */
+	pcLocalBuffer = OSAllocMem(ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(pcLocalBuffer != NULL, "OSAllocMem() failed",
+	                      return_);
+
+	i64Res = pvr_copy_from_user(pcLocalBuffer, pszBuffer, ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(i64Res == 0, "pvr_copy_from_user() failed",
+	                      free_local_buffer_);
+
+	/* ensure that the framework user gets a NUL terminated buffer */
+	pcLocalBuffer[ui64Count - 1] = '\0';
+
+	i64Res = psIter->pfnWrite(pcLocalBuffer, ui64Count, &ui64Pos,
+	                          psDFSFile->sEntry.sImplEntry.pvPrivData);
+	PVR_LOG_GOTO_IF_FALSE(i64Res >= 0, "pfnWrite failed", free_local_buffer_);
+
+	*puiPos = ui64Pos;
+
+free_local_buffer_:
+	OSFreeMem(pcLocalBuffer);
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return i64Res;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+
+static const struct file_operations _g_psFileOpsGen = {
+	.owner = THIS_MODULE,
+	.open = _Open,
+	.release = _Close,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+static const struct file_operations _g_psFileOpsRndAcc = {
+	.owner = THIS_MODULE,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+#else
+
+static const struct proc_ops _g_psFileOpsGen = {
+	.proc_open    = _Open,
+	.proc_read    = _Read,
+	.proc_write   = _Write,
+	.proc_lseek   = _LSeek,
+	.proc_release = _Close,
+};
+
+static const struct proc_ops _g_psFileOpsRndAcc = {
+	.proc_read  = _Read,
+	.proc_write = _Write,
+	.proc_lseek = _LSeek,
+};
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)) */
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	return PVRSRV_OK;
+}
+
+static void _DeInit(void)
+{
+}
+
+static PVRSRV_ERROR _CreateFile(const IMG_CHAR *pszName,
+                                DI_ENTRY_TYPE eType,
+                                const DI_ITERATOR_CB *psIterCb,
+                                void *pvPrivData,
+                                void *pvParentDir,
+                                void **pvFile)
+{
+	DFS_DIR *psParentDir = pvParentDir;
+	DFS_FILE *psFile;
+	umode_t uiMode = S_IFREG;
+	struct proc_dir_entry *psEntry;
+	PVRSRV_ERROR eError;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	const struct file_operations *psProcOps = NULL;
+#else
+	const struct proc_ops *psProcOps = NULL;
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)) */
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvFile != NULL, "pvFile");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentDir != NULL, "pvParentDir");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			psProcOps = &_g_psFileOpsGen;
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			psProcOps = &_g_psFileOpsRndAcc;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto return_;
+	}
+
+	psFile = OSAllocMem(sizeof(*psFile));
+	PVR_LOG_GOTO_IF_NOMEM(psFile, eError, return_);
+
+	uiMode |= psIterCb->pfnShow != NULL || psIterCb->pfnRead != NULL ?
+	        S_IRUGO : 0;
+	uiMode |= psIterCb->pfnWrite != NULL ? S_IWUSR : 0;
+
+	psEntry = proc_create_data(pszName, uiMode, psParentDir->psDirEntry,
+	                           psProcOps, psFile);
+	if (IS_ERR_OR_NULL(psEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create debugfs '%s' file",
+		        __func__, pszName));
+
+		eError = psEntry == NULL ?
+		        PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_ERROR_INVALID_DEVICE;
+		goto free_file_;
+	}
+
+	psFile->eType = eType;
+	psFile->psSeqOps = &_g_sSeqOps;
+	psFile->sEntry.sIterCb = *psIterCb;
+	psFile->sEntry.sImplEntry.pvPrivData = pvPrivData;
+	psFile->sEntry.sImplEntry.pvNative = NULL;
+	psFile->sEntry.sImplEntry.psCb = &_g_sEntryCallbacks;
+	psFile->psParentDir = psParentDir;
+	psFile->psFileEntry = psEntry;
+
+	*pvFile = psFile;
+
+	return PVRSRV_OK;
+
+free_file_:
+	OSFreeMem(psFile);
+
+return_:
+	return eError;
+}
+
+static void _DestroyFile(void *pvFile)
+{
+	DFS_FILE *psFile = pvFile;
+
+	PVR_ASSERT(psFile != NULL);
+
+	proc_remove(psFile->psFileEntry);
+	OSFreeMem(psFile);
+}
+
+static PVRSRV_ERROR _CreateDir(const IMG_CHAR *pszName,
+                               void *pvParentDir,
+                               void **ppvDir)
+{
+	DFS_DIR *psNewDir;
+	struct proc_dir_entry *psDirEntry, *psParentDir = NULL;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvDir != NULL, "ppvDir");
+
+	psNewDir = OSAllocMem(sizeof(*psNewDir));
+	PVR_LOG_RETURN_IF_NOMEM(psNewDir, "OSAllocMem");
+
+	psNewDir->psParentDir = pvParentDir;
+
+	if (pvParentDir != NULL)
+	{
+		psParentDir = psNewDir->psParentDir->psDirEntry;
+	}
+
+	psDirEntry = proc_mkdir(pszName, psParentDir);
+	if (IS_ERR_OR_NULL(psDirEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create '%s' debugfs directory",
+		        __func__, pszName));
+		OSFreeMem(psNewDir);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psNewDir->psDirEntry = psDirEntry;
+	*ppvDir = psNewDir;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyDir(void *pvDir)
+{
+	DFS_DIR *psDir = pvDir;
+
+	PVR_ASSERT(psDir != NULL);
+
+	proc_remove(psDir->psDirEntry);
+	OSFreeMem(psDir);
+}
+
+PVRSRV_ERROR PVRProcFsRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateFile,
+		.pfnDestroyEntry = _DestroyFile,
+		.pfnCreateGroup = _CreateDir,
+		.pfnDestroyGroup = _DestroyDir
+	};
+
+	return DIRegisterImplementation("procfs", &sImplCb);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.h
new file mode 100644
index 000000000000..61a1f0ee28d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Title          ProcFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_PROCFS_H
+#define PVR_PROCFS_H
+
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR PVRProcFsRegister(void);
+
+#endif /* PVR_PROCFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.c
new file mode 100644
index 000000000000..cccc4a64e321
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.c
@@ -0,0 +1,200 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock_types.h>
+#include <linux/atomic.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+#include "pvr_sw_fence.h"
+#include "osfunc_common.h"
+
+struct pvr_sw_fence_context {
+	struct kref kref;
+	unsigned int context;
+	char context_name[32];
+	char driver_name[32];
+	atomic_t seqno;
+	atomic_t fence_count;
+};
+
+struct pvr_sw_fence {
+	struct dma_fence base;
+	struct pvr_sw_fence_context *fence_context;
+	spinlock_t lock;
+};
+
+#define to_pvr_sw_fence(fence) container_of(fence, struct pvr_sw_fence, base)
+
+const char *pvr_sw_fence_context_name(struct pvr_sw_fence_context *fctx)
+{
+	return fctx->context_name;
+}
+
+void pvr_sw_fence_context_value_str(struct pvr_sw_fence_context *fctx,
+				    char *str, int size)
+{
+	snprintf(str, size, "%d", atomic_read(&fctx->seqno));
+}
+
+static inline unsigned
+pvr_sw_fence_context_seqno_next(struct pvr_sw_fence_context *fence_context)
+{
+	return atomic_inc_return(&fence_context->seqno) - 1;
+}
+
+static const char *pvr_sw_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	return pvr_sw_fence->fence_context->driver_name;
+}
+
+static const char *pvr_sw_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	return pvr_sw_fence_context_name(pvr_sw_fence->fence_context);
+}
+
+static void pvr_sw_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%llu", (u64) fence->seqno);
+}
+
+static void pvr_sw_fence_timeline_value_str(struct dma_fence *fence,
+					    char *str, int size)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	pvr_sw_fence_context_value_str(pvr_sw_fence->fence_context, str, size);
+}
+
+static bool pvr_sw_fence_enable_signaling(struct dma_fence *fence)
+{
+	return true;
+}
+
+static void pvr_sw_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_sw_fence_context *fence_context =
+		container_of(kref, struct pvr_sw_fence_context, kref);
+	unsigned int fence_count;
+
+	fence_count = atomic_read(&fence_context->fence_count);
+	if (WARN_ON(fence_count))
+		pr_debug("%s context has %u fence(s) remaining\n",
+			 fence_context->context_name, fence_count);
+
+	kfree(fence_context);
+}
+
+static void pvr_sw_fence_release(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	atomic_dec(&pvr_sw_fence->fence_context->fence_count);
+	kref_put(&pvr_sw_fence->fence_context->kref,
+		pvr_sw_fence_context_destroy_kref);
+	kfree(pvr_sw_fence);
+}
+
+static const struct dma_fence_ops pvr_sw_fence_ops = {
+	.get_driver_name = pvr_sw_fence_get_driver_name,
+	.get_timeline_name = pvr_sw_fence_get_timeline_name,
+	.fence_value_str = pvr_sw_fence_value_str,
+	.timeline_value_str = pvr_sw_fence_timeline_value_str,
+	.enable_signaling = pvr_sw_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = pvr_sw_fence_release,
+};
+
+struct pvr_sw_fence_context *
+pvr_sw_fence_context_create(const char *context_name, const char *driver_name)
+{
+	struct pvr_sw_fence_context *fence_context;
+
+	fence_context = kmalloc(sizeof(*fence_context), GFP_KERNEL);
+	if (!fence_context)
+		return NULL;
+
+	fence_context->context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fence_context->context_name, context_name,
+		sizeof(fence_context->context_name));
+	OSStringSafeCopy(fence_context->driver_name, driver_name,
+		sizeof(fence_context->driver_name));
+	atomic_set(&fence_context->seqno, 0);
+	atomic_set(&fence_context->fence_count, 0);
+	kref_init(&fence_context->kref);
+
+	return fence_context;
+}
+
+void pvr_sw_fence_context_destroy(struct pvr_sw_fence_context *fence_context)
+{
+	kref_put(&fence_context->kref, pvr_sw_fence_context_destroy_kref);
+}
+
+struct dma_fence *
+pvr_sw_fence_create(struct pvr_sw_fence_context *fence_context)
+{
+	struct pvr_sw_fence *pvr_sw_fence;
+	unsigned int seqno;
+
+	pvr_sw_fence = kmalloc(sizeof(*pvr_sw_fence), GFP_KERNEL);
+	if (!pvr_sw_fence)
+		return NULL;
+
+	spin_lock_init(&pvr_sw_fence->lock);
+	pvr_sw_fence->fence_context = fence_context;
+
+	seqno = pvr_sw_fence_context_seqno_next(fence_context);
+	dma_fence_init(&pvr_sw_fence->base, &pvr_sw_fence_ops,
+		       &pvr_sw_fence->lock, fence_context->context, seqno);
+
+	atomic_inc(&fence_context->fence_count);
+	kref_get(&fence_context->kref);
+
+	return &pvr_sw_fence->base;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.h
new file mode 100644
index 000000000000..bebbcb75e23b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.h
@@ -0,0 +1,60 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_SW_FENCES_H__)
+#define __PVR_SW_FENCES_H__
+
+#include "pvr_linux_fence.h"
+
+struct pvr_sw_fence_context;
+
+struct pvr_sw_fence_context *pvr_sw_fence_context_create(const char *name,
+				const char *driver_name);
+void pvr_sw_fence_context_destroy(struct pvr_sw_fence_context *fence_context);
+struct dma_fence *pvr_sw_fence_create(struct pvr_sw_fence_context *
+				      fence_context);
+
+const char *pvr_sw_fence_context_name(struct pvr_sw_fence_context *fctx);
+void pvr_sw_fence_context_value_str(struct pvr_sw_fence_context *fctx,
+				    char *str, int size);
+
+#endif /* !defined(__PVR_SW_FENCES_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync.h
new file mode 100644
index 000000000000..a8ecd8b2caa0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync.h
@@ -0,0 +1,120 @@
+/*
+ * @File        pvr_sync.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_H
+#define _PVR_SYNC_H
+
+#include <linux/device.h>
+
+#include "pvr_fd_sync_kernel.h"
+#include "services_kernel_client.h"
+
+
+/* Services internal interface */
+
+/**
+ * pvr_sync_register_functions()
+ *
+ * Return: PVRSRV_OK on success.
+ */
+enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void);
+
+/**
+ *  pvr_sync_init - register the pvr_sync misc device
+ *
+ *  Return: error code, 0 on success.
+ */
+int pvr_sync_init(void);
+
+/**
+ * pvr_sync_deinit - unregister the pvr_sync misc device
+ */
+void pvr_sync_deinit(void);
+
+/**
+ * pvr_sync_device_init() - create an internal sync context
+ * @dev: Linux device
+ *
+ * Return: PVRSRV_OK on success.
+ */
+enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev);
+
+/**
+ * pvr_sync_device_deinit() - destroy an internal sync context
+ *
+ * Drains any work items with outstanding sync fence updates/dependencies.
+ */
+void pvr_sync_device_deinit(struct device *dev);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out);
+
+enum PVRSRV_ERROR_TAG
+pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
+				  int timeline_fd,
+				  const char *fence_name,
+				  int *fence_fd_out,
+				  u64 *sync_pt_idx);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline,
+					       u64 *sync_pt_idx);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
+					   void **timeline_out);
+
+enum PVRSRV_ERROR_TAG
+sync_dump_fence(void *sw_fence_obj,
+		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		void *dump_debug_file);
+
+enum PVRSRV_ERROR_TAG
+sync_sw_dump_timeline(void *sw_timeline_obj,
+		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		      void *dump_debug_file);
+
+#endif /* _PVR_SYNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_api.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_api.h
new file mode 100644
index 000000000000..97dd746439b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_api.h
@@ -0,0 +1,65 @@
+/*
+ * @File        pvr_sync_api.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_API_H
+#define _PVR_SYNC_API_H
+
+#include "img_types.h"
+
+int pvr_sync_api_init(void *file_handle, void **api_priv);
+int pvr_sync_api_deinit(void *api_priv, bool is_sw);
+int pvr_sync_api_rename(void *api_priv, void *user_data);
+int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new);
+int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data);
+int pvr_sync_api_sw_inc(void *api_priv, void *user_data);
+int pvr_sync_api_force_exp_only(void *api_priv, void *api_data);
+int pvr_sync_api_create_export_fence(void *api_priv, void *user_data);
+
+struct file;
+
+int pvr_sync_ioctl_init(void);
+void pvr_sync_ioctl_deinit(void);
+void *pvr_sync_get_api_priv(struct file *file);
+struct file *pvr_sync_get_file_struct(void *file_handle);
+
+#endif /* _PVR_SYNC_API_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_file.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_file.c
new file mode 100644
index 000000000000..c6cec6d9dd0d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_file.c
@@ -0,0 +1,1309 @@
+/*
+ * @File        pvr_sync_file.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "services_kernel_client.h"
+#include "pvr_drv.h"
+#include "pvr_sync.h"
+#include "pvr_fence.h"
+#include "pvr_counting_timeline.h"
+#include "pvr_export_fence.h"
+
+#include "linux_sw_sync.h"
+
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/sync_file.h>
+#include <linux/file.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+
+#include "pvr_sync_api.h"
+#include "osfunc_common.h"
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)) && !defined(CHROMIUMOS_KERNEL)
+#define sync_file_user_name(s)	((s)->name)
+#else
+#define sync_file_user_name(s)	((s)->user_name)
+#endif
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+#define	FILE_NAME "pvr_sync_file"
+
+struct sw_sync_create_fence_data {
+	__u32 value;
+	char name[32];
+	__s32 fence;
+};
+#define SW_SYNC_IOC_MAGIC 'W'
+#define SW_SYNC_IOC_CREATE_FENCE \
+	(_IOWR(SW_SYNC_IOC_MAGIC, 0, struct sw_sync_create_fence_data))
+#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
+
+/* Global data for the sync driver */
+static struct {
+	struct pvr_fence_context *foreign_fence_context;
+	PFN_SYNC_CHECKPOINT_STRUCT sync_checkpoint_ops;
+} pvr_sync_data;
+
+#if defined(NO_HARDWARE)
+static DEFINE_MUTEX(pvr_timeline_active_list_lock);
+static struct list_head pvr_timeline_active_list;
+#endif
+
+/* This is the actual timeline metadata. We might keep this around after the
+ * base sync driver has destroyed the pvr_sync_timeline_wrapper object.
+ */
+struct pvr_sync_timeline {
+	char name[32];
+	void *file_handle;
+	bool is_sw;
+	bool is_export;
+	/* Fence context used for hw fences */
+	struct pvr_fence_context *hw_fence_context;
+	/* Timeline and context for sw fences */
+	union {
+		struct pvr_counting_fence_timeline *sw_fence_timeline;
+		struct pvr_exp_fence_context *exp_fence_context;
+	};
+#if defined(NO_HARDWARE)
+	/* List of all timelines (used to advance all timelines in nohw builds) */
+	struct list_head list;
+#endif
+};
+
+static
+void pvr_sync_free_checkpoint_list_mem(void *mem_ptr)
+{
+	kfree(mem_ptr);
+}
+
+#if defined(NO_HARDWARE)
+/* function used to signal pvr fence in nohw builds */
+static
+void pvr_sync_nohw_signal_fence(void *fence_data_to_signal)
+{
+	struct pvr_sync_timeline *this_timeline;
+
+	mutex_lock(&pvr_timeline_active_list_lock);
+	list_for_each_entry(this_timeline, &pvr_timeline_active_list, list) {
+		pvr_fence_context_signal_fences_nohw(this_timeline->hw_fence_context);
+	}
+	mutex_unlock(&pvr_timeline_active_list_lock);
+}
+static
+void pvr_sync_nohw_signal_exp_fence(PVRSRV_FENCE fence_to_signal)
+{
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_signal);
+	if (fence && pvr_is_exp_fence(fence))
+		dma_fence_signal(fence);
+
+	dma_fence_put(fence);
+}
+#endif
+
+static struct pvr_sync_timeline *pvr_sync_timeline_fget(int fd)
+{
+	struct file *file = fget(fd);
+	struct pvr_sync_timeline *timeline;
+
+	if (!file)
+		return NULL;
+
+	timeline = pvr_sync_get_api_priv(file);
+	if (!timeline)
+		fput(file);
+
+	return timeline;
+}
+
+static void pvr_sync_timeline_fput(struct pvr_sync_timeline *timeline)
+{
+	struct file *file = pvr_sync_get_file_struct(timeline->file_handle);
+
+	if (file)
+		fput(file);
+	else
+		pr_err(FILE_NAME ": %s: Timeline incomplete\n", __func__);
+}
+
+/* ioctl and fops handling */
+
+int pvr_sync_api_init(void *file_handle, void **api_priv)
+{
+	struct pvr_sync_timeline *timeline;
+	char task_comm[TASK_COMM_LEN];
+
+	get_task_comm(task_comm, current);
+
+	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
+	if (!timeline)
+		return -ENOMEM;
+
+	OSStringSafeCopy(timeline->name, task_comm, sizeof(timeline->name));
+	timeline->file_handle = file_handle;
+	timeline->is_sw = false;
+	timeline->is_export = false;
+
+	*api_priv = (void *)timeline;
+
+	return 0;
+}
+
+int pvr_sync_api_deinit(void *api_priv, bool is_sw)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	if (!timeline)
+		return 0;
+
+	if (timeline->is_sw && timeline->sw_fence_timeline) {
+		/* This makes sure any outstanding SW syncs are marked as
+		 * complete at timeline close time. Otherwise it'll leak the
+		 * timeline (as outstanding fences hold a ref) and possibly
+		 * wedge the system if something is waiting on one of those
+		 * fences
+		 */
+		pvr_counting_fence_timeline_force_complete(
+			timeline->sw_fence_timeline);
+		pvr_counting_fence_timeline_put(timeline->sw_fence_timeline);
+	} else if (timeline->is_export && timeline->exp_fence_context) {
+		pvr_exp_fence_context_destroy(timeline->exp_fence_context);
+	}
+
+	if (timeline->hw_fence_context) {
+#if defined(NO_HARDWARE)
+		mutex_lock(&pvr_timeline_active_list_lock);
+		list_del(&timeline->list);
+		mutex_unlock(&pvr_timeline_active_list_lock);
+#endif
+		pvr_fence_context_destroy(timeline->hw_fence_context);
+	}
+
+	kfree(timeline);
+
+	return 0;
+}
+
+/*
+ * This is the function that kick code will call in order to 'finalise' a
+ * created output fence just prior to returning from the kick function.
+ * The OS native sync code needs to implement a function meeting this
+ * specification - the implementation may be a nop if the OS does not need
+ * to perform any actions at this point.
+ *
+ * Input: fence_fd            The PVRSRV_FENCE to be 'finalised'. This value
+ *                            will have been returned by an earlier call to
+ *                            pvr_sync_create_fence().
+ * Input: finalise_data       The finalise data returned by an earlier call
+ *                            to pvr_sync_create_fence().
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_finalise_fence(PVRSRV_FENCE fence_fd, void *finalise_data)
+{
+	struct sync_file *sync_file = finalise_data;
+	struct pvr_fence *pvr_fence;
+
+	if (!sync_file || (fence_fd < 0)) {
+		pr_err(FILE_NAME ": %s: Invalid input fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pvr_fence = to_pvr_fence(sync_file->fence);
+
+	if (!pvr_fence) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* pvr fences can be signalled any time after creation */
+	dma_fence_enable_sw_signaling(&pvr_fence->base);
+
+	fd_install(fence_fd, sync_file->file);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain a new
+ * PVRSRV_FENCE from the OS native sync code and the PSYNC_CHECKPOINT used
+ * in that fence. The OS native sync code needs to implement a function
+ * meeting this specification.
+ *
+ * Input: device                   Device node to use in creating a hw_fence_ctx
+ * Input: fence_name               A string to annotate the fence with (for
+ *                                 debug).
+ * Input: timeline                 The timeline on which the new fence is to be
+ *                                 created.
+ * Output: new_fence               The new PVRSRV_FENCE to be returned by the
+ *                                 kick call.
+ * Output: fence_uid               Unique ID of the update fence.
+ * Output: fence_finalise_data     Pointer to data needed to finalise the fence.
+ * Output: new_checkpoint_handle   The PSYNC_CHECKPOINT used by the new fence.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_create_fence(
+		      struct _PVRSRV_DEVICE_NODE_ *device,
+		      const char *fence_name,
+		      PVRSRV_TIMELINE new_fence_timeline,
+		      PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		      PVRSRV_FENCE *new_fence, u64 *fence_uid,
+		      void **fence_finalise_data,
+		      PSYNC_CHECKPOINT *new_checkpoint_handle,
+		      void **timeline_update_sync,
+		      __u32 *timeline_update_value)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	PVRSRV_FENCE new_fence_fd = -1;
+	struct pvr_sync_timeline *timeline;
+	struct pvr_fence *pvr_fence;
+	PSYNC_CHECKPOINT checkpoint;
+	struct sync_file *sync_file;
+
+	if (new_fence_timeline < 0 || !new_fence || !new_checkpoint_handle
+		|| !fence_finalise_data) {
+		pr_err(FILE_NAME ": %s: Invalid input params\n", __func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	/* We reserve the new fence FD before taking any operations
+	 * as we do not want to fail (e.g. run out of FDs)
+	 */
+	new_fence_fd = get_unused_fd_flags(O_CLOEXEC);
+	if (new_fence_fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to get fd\n", __func__);
+		err = PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		goto err_out;
+	}
+
+	timeline = pvr_sync_timeline_fget(new_fence_timeline);
+	if (!timeline) {
+		pr_err(FILE_NAME ": %s: Failed to open supplied timeline fd (%d)\n",
+			__func__, new_fence_timeline);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_fd;
+	}
+
+	if (timeline->is_sw) {
+		/* This should never happen! */
+		pr_err(FILE_NAME ": %s: Request to create a pvr fence on sw timeline (%d)\n",
+			__func__, new_fence_timeline);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_timeline;
+	}
+
+	if (!timeline->hw_fence_context) {
+		/* First time we use this timeline, so create a context. */
+		timeline->hw_fence_context =
+			pvr_fence_context_create(
+				device,
+				NativeSyncGetFenceStatusWq(),
+				timeline->name);
+		if (!timeline->hw_fence_context) {
+			pr_err(FILE_NAME ": %s: Failed to create fence context (%d)\n",
+			       __func__, new_fence_timeline);
+			err = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto err_put_timeline;
+		}
+#if defined(NO_HARDWARE)
+		/* Add timeline to active list */
+		INIT_LIST_HEAD(&timeline->list);
+		mutex_lock(&pvr_timeline_active_list_lock);
+		list_add_tail(&timeline->list, &pvr_timeline_active_list);
+		mutex_unlock(&pvr_timeline_active_list_lock);
+#endif
+	}
+
+	pvr_fence = pvr_fence_create(timeline->hw_fence_context,
+								 psSyncCheckpointContext,
+								 new_fence_timeline,
+								 fence_name);
+	if (!pvr_fence) {
+		pr_err(FILE_NAME ": %s: Failed to create new pvr_fence\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_timeline;
+	}
+
+	checkpoint = pvr_fence_get_checkpoint(pvr_fence);
+	if (!checkpoint) {
+		pr_err(FILE_NAME ": %s: Failed to get fence checkpoint\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_destroy_fence;
+	}
+
+	sync_file = sync_file_create(&pvr_fence->base);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create sync_file\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_destroy_fence;
+	}
+	OSStringSafeCopy(sync_file_user_name(sync_file),
+		pvr_fence->name,
+		sizeof(sync_file_user_name(sync_file)));
+	dma_fence_put(&pvr_fence->base);
+
+	*new_fence = new_fence_fd;
+	*fence_finalise_data = sync_file;
+	*new_checkpoint_handle = checkpoint;
+	*fence_uid = OSGetCurrentClientProcessIDKM();
+	*fence_uid = (*fence_uid << 32) | (new_fence_fd & U32_MAX);
+	/* not used but don't want to return dangling pointers */
+	*timeline_update_sync = NULL;
+	*timeline_update_value = 0;
+
+	pvr_sync_timeline_fput(timeline);
+err_out:
+	return err;
+
+err_destroy_fence:
+	pvr_fence_destroy(pvr_fence);
+err_put_timeline:
+	pvr_sync_timeline_fput(timeline);
+err_put_fd:
+	put_unused_fd(new_fence_fd);
+	*fence_uid = PVRSRV_NO_FENCE;
+	goto err_out;
+}
+
+/*
+ * This is the function that kick code will call in order to 'rollback' a
+ * created output fence should an error occur when submitting the kick.
+ * The OS native sync code needs to implement a function meeting this
+ * specification.
+ *
+ * Input: fence_to_rollback The PVRSRV_FENCE to be 'rolled back'. The fence
+ *                          should be destroyed and any actions taken due to
+ *                          its creation that need to be undone should be
+ *                          reverted.
+ * Input: finalise_data     The finalise data for the fence to be 'rolled back'.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_rollback_fence_data(PVRSRV_FENCE fence_to_rollback,
+			     void *fence_data_to_rollback)
+{
+	struct sync_file *sync_file = fence_data_to_rollback;
+	struct pvr_fence *pvr_fence;
+
+	if (!sync_file || fence_to_rollback < 0) {
+		pr_err(FILE_NAME ": %s: Invalid fence (%d)\n", __func__,
+			fence_to_rollback);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pvr_fence = to_pvr_fence(sync_file->fence);
+	if (!pvr_fence) {
+		pr_err(FILE_NAME
+			": %s: Non-PVR fence (%p)\n",
+			__func__, sync_file->fence);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	fput(sync_file->file);
+
+	put_unused_fd(fence_to_rollback);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain a list of
+ * the PSYNC_CHECKPOINTs for a given PVRSRV_FENCE passed to a kick function.
+ * The OS native sync code will allocate the memory to hold the returned list
+ * of PSYNC_CHECKPOINT ptrs. The caller will free this memory once it has
+ * finished referencing it.
+ *
+ * Input: fence                     The input (check) fence
+ * Output: nr_checkpoints           The number of PVRSRV_SYNC_CHECKPOINT ptrs
+ *                                  returned in the checkpoint_handles
+ *                                  parameter.
+ * Output: fence_uid                Unique ID of the check fence
+ * Input/Output: checkpoint_handles The returned list of PVRSRV_SYNC_CHECKPOINTs.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_resolve_fence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		       PVRSRV_FENCE fence_to_resolve, u32 *nr_checkpoints,
+		       PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid)
+{
+	PSYNC_CHECKPOINT *checkpoints = NULL;
+	unsigned int i, num_fences = 0, num_used_fences = 0;
+	struct dma_fence **fences = NULL;
+	struct dma_fence *fence;
+	PVRSRV_ERROR err = PVRSRV_OK;
+
+	if (!nr_checkpoints || !checkpoint_handles || !fence_uid) {
+		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
+			__func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	*nr_checkpoints = 0;
+	*checkpoint_handles = NULL;
+	*fence_uid = 0;
+
+	if (fence_to_resolve < 0)
+		goto err_out;
+
+	fence = sync_file_get_fence(fence_to_resolve);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_resolve);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+
+		if (array) {
+			fences = array->fences;
+			num_fences = array->num_fences;
+		}
+	} else {
+		fences = &fence;
+		num_fences = 1;
+	}
+
+	checkpoints = kmalloc_array(num_fences, sizeof(PSYNC_CHECKPOINT),
+			      GFP_KERNEL);
+	if (!checkpoints) {
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fence;
+	}
+	for (i = 0; i < num_fences; i++) {
+		/*
+		 * Only return the checkpoint if the fence is still active.
+		 * Don't checked for signalled on PDUMP drivers as we need
+		 * to make sure that all fences make it to the pdump.
+		 */
+#if !defined(PDUMP)
+		if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			      &fences[i]->flags))
+#endif
+		{
+			struct pvr_fence *pvr_fence;
+
+			/* Check if fences[i] is an export fence */
+			if (pvr_is_exp_fence(fences[i])) {
+				/* Assign export fence a sync checkpoint if it does not already
+				 * have one.
+				 */
+				err = pvr_exp_fence_assign_checkpoint(PVRSRV_NO_FENCE,
+				                                      fences[i],
+				                                      psSyncCheckpointContext,
+				                                      &checkpoints[num_used_fences]);
+				SyncCheckpointTakeRef(checkpoints[num_used_fences]);
+				++num_used_fences;
+			}
+			else {
+				pvr_fence = pvr_fence_create_from_fence(
+				            pvr_sync_data.foreign_fence_context,
+			            psSyncCheckpointContext,
+			            fences[i],
+			            fence_to_resolve,
+			            "foreign");
+				if (!pvr_fence) {
+					pr_err(FILE_NAME ": %s: Failed to create fence\n",
+					       __func__);
+					err = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto err_free_checkpoints;
+				}
+				checkpoints[num_used_fences] =
+				        pvr_fence_get_checkpoint(pvr_fence);
+				SyncCheckpointTakeRef(checkpoints[num_used_fences]);
+				++num_used_fences;
+				dma_fence_put(&pvr_fence->base);
+			}
+		}
+	}
+	/* If we don't return any checkpoints, delete the array because
+	 * the caller will not.
+	 */
+	if (num_used_fences == 0) {
+		kfree(checkpoints);
+		checkpoints = NULL;
+	}
+
+	*checkpoint_handles = checkpoints;
+	*nr_checkpoints = num_used_fences;
+	*fence_uid = OSGetCurrentClientProcessIDKM();
+	*fence_uid = (*fence_uid << 32) | (fence_to_resolve & U32_MAX);
+
+err_put_fence:
+	dma_fence_put(fence);
+err_out:
+	return err;
+
+err_free_checkpoints:
+	for (i = 0; i < num_used_fences; i++) {
+		if (checkpoints[i])
+			SyncCheckpointDropRef(checkpoints[i]);
+	}
+	kfree(checkpoints);
+	goto err_put_fence;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain the
+ * PSYNC_CHECKPOINT for a given export fence (PVRSRV_FENCE) passed to a kick
+ * function.
+ * If export_fence is not a valid export fence, (ie fops != &pvr_exp_fence_ops)
+ * this function will return (error).
+ *
+ * Input:  export_fence              The export fence to resolve
+ * Input:  checkpoint_context        The context in which to create the new
+ *                                   sync checkpoint for the export fence
+ * Output: checkpoint_handle         The returned PVRSRV_SYNC_CHECKPOINT.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_resolve_export_fence(PVRSRV_FENCE fence_to_resolve,
+			      PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+			      PSYNC_CHECKPOINT *checkpoint_handle)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_resolve);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_resolve);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (!pvr_is_exp_fence(fence)) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr export fence\n", __func__);
+		dma_fence_put(fence);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	err = pvr_exp_fence_assign_checkpoint(fence_to_resolve,
+					      fence,
+					      checkpoint_context,
+					      checkpoint_handle);
+	if (err != PVRSRV_OK) {
+		pr_err(FILE_NAME ": %s: Failed to assign export fence a sync checkpoint\n",
+		       __func__);
+	}
+
+	dma_fence_put(fence);
+
+err_out:
+	return err;
+}
+
+/*
+ * This is the function that kick code will call in order to rollback the
+ * PSYNC_CHECKPOINT assigned to an export fence (PVRSRV_FENCE) passed to a kick
+ * function.
+ * If export_fence is not a valid export fence, (ie fops != &pvr_exp_fence_ops)
+ * this function will return (error).
+ *
+ * Input:  fence_to_rollback              The export fence to rollback
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_rollback_export_fence(PVRSRV_FENCE fence_to_rollback)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_rollback);
+	if (!fence) {
+		pr_err("%s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_rollback);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_get_fence;
+	}
+
+	if (!pvr_is_exp_fence(fence)) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr export fence\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_is_exp_fence;
+	}
+
+	err = pvr_exp_fence_rollback(fence);
+	if (err != PVRSRV_OK) {
+		pr_err("%s: Failed to rollback export fence\n",
+		       __func__);
+	}
+
+err_is_exp_fence:
+	dma_fence_put(fence);
+
+err_get_fence:
+	return err;
+}
+
+/*
+ * This is the function that driver code will call in order to request the
+ * sync implementation to output debug information relating to any sync
+ * checkpoints it may have created which appear in the provided array of
+ * FW addresses of Unified Fence Objects (UFOs).
+ *
+ * Input: nr_ufos             The number of FW addresses provided in the
+ *                            vaddrs parameter.
+ * Input: vaddrs              The array of FW addresses of UFOs. The sync
+ *                            implementation should check each of these to
+ *                            see if any relate to sync checkpoints it has
+ *                            created and where they do output debug information
+ *                            pertaining to the native/fallback sync with
+ *                            which it is associated.
+ */
+static u32
+pvr_sync_dump_info_on_stalled_ufos(u32 nr_ufos, u32 *vaddrs)
+{
+	return pvr_fence_dump_info_on_stalled_ufos(pvr_sync_data.foreign_fence_context,
+						   nr_ufos,
+						   vaddrs);
+}
+
+#if defined(PDUMP)
+static enum PVRSRV_ERROR_TAG
+pvr_sync_fence_get_checkpoints(PVRSRV_FENCE fence_to_pdump, u32 *nr_checkpoints,
+				struct SYNC_CHECKPOINT_TAG ***checkpoint_handles)
+{
+	struct dma_fence **fences = NULL;
+	struct dma_fence *fence;
+	struct pvr_fence *pvr_fence;
+	struct SYNC_CHECKPOINT_TAG **checkpoints = NULL;
+	unsigned int i, num_fences, num_used_fences = 0;
+	enum PVRSRV_ERROR_TAG err;
+
+	if (fence_to_pdump < 0) {
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (!nr_checkpoints || !checkpoint_handles) {
+		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
+			__func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	fence = sync_file_get_fence(fence_to_pdump);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_pdump);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+
+		fences = array->fences;
+		num_fences = array->num_fences;
+	} else {
+		fences = &fence;
+		num_fences = 1;
+	}
+
+	checkpoints = kmalloc_array(num_fences, sizeof(*checkpoints),
+			      GFP_KERNEL);
+	if (!checkpoints) {
+		pr_err("pvr_sync_file: %s: Failed to alloc memory for returned list of sync checkpoints\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fence;
+	}
+
+	for (i = 0; i < num_fences; i++) {
+		if (is_pvr_fence(fences[i])) {
+			pvr_fence = to_pvr_fence(fences[i]);
+			if (pvr_fence) {
+				checkpoints[num_used_fences] = pvr_fence_get_checkpoint(pvr_fence);
+				++num_used_fences;
+			}
+		} else if (pvr_is_exp_fence(fences[i])) {
+			struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fences[i]);
+
+			checkpoints[num_used_fences] = pvr_exp_fence_get_checkpoint(pvr_exp_fence);
+			if (checkpoints[num_used_fences])
+				++num_used_fences;
+		} else
+			continue;
+	}
+
+	*checkpoint_handles = checkpoints;
+	*nr_checkpoints = num_used_fences;
+	err =  PVRSRV_OK;
+
+err_put_fence:
+	dma_fence_put(fence);
+err_out:
+	return err;
+}
+#endif
+
+int pvr_sync_api_rename(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sync_rename_ioctl_data *data = user_data;
+
+	data->szName[sizeof(data->szName) - 1] = '\0';
+	OSStringSafeCopy(timeline->name, data->szName, sizeof(timeline->name));
+	if (timeline->hw_fence_context)
+		OSStringSafeCopy(timeline->hw_fence_context->name, data->szName,
+			sizeof(timeline->hw_fence_context->name));
+
+	return 0;
+}
+
+int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	/* Already in SW mode? */
+	if (timeline->sw_fence_timeline)
+		return 0;
+
+	/* Create a sw_sync timeline with the old GPU timeline's name */
+	timeline->sw_fence_timeline = pvr_counting_fence_timeline_create(
+		timeline->name);
+	if (!timeline->sw_fence_timeline)
+		return -ENOMEM;
+
+	timeline->is_sw = true;
+
+	*api_priv_new = (void *)timeline;
+
+	return 0;
+}
+
+/* We simply treat an export-fence as a SW fence and tweak
+ * the timeline structure to flag it as an 'is_export' type.
+ */
+int pvr_sync_api_force_exp_only(void *api_priv, void *api_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	if (timeline->is_export) {
+		pr_err(FILE_NAME ": %s: Already marked export timeline\n", __func__);
+		return 0;
+	}
+
+	timeline->exp_fence_context = pvr_exp_fence_context_create("pvr_exp_fence_ctx", "pvr_sync");
+	if (!timeline->exp_fence_context)
+		return -ENOMEM;
+
+	timeline->is_export = true;
+
+	return 0;
+}
+
+int pvr_sync_api_create_export_fence(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_exp_fence_context *exp_fence_context = timeline->exp_fence_context;
+	pvr_exp_sync_create_fence_data_t *data = user_data;
+	struct sync_file *sync_file;
+	int fd;
+	struct dma_fence *fence;
+	int err;
+
+	if (data == NULL) {
+		pr_err(FILE_NAME ": %s: Unexpected NULL user_data\n", __func__);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
+		       __func__, fd);
+		err = -EMFILE;
+		goto err_out;
+	}
+
+	fence = pvr_exp_fence_create(exp_fence_context, fd, &data->sync_pt_idx);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to create a dma fence (%d)\n",
+		       __func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync_file (%d)\n",
+			__func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	data->fence = fd;
+
+	fd_install(fd, sync_file->file);
+
+	return 0;
+
+err_put_fd:
+	put_unused_fd(fd);
+err_out:
+	return err;
+}
+
+int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sw_sync_create_fence_data *data = user_data;
+	struct sync_file *sync_file;
+	int fd = get_unused_fd_flags(O_CLOEXEC);
+	struct dma_fence *fence;
+	int err;
+
+	if (fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
+		       __func__, fd);
+		err = -EMFILE;
+		goto err_out;
+	}
+
+	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, &data->sync_pt_idx);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
+		       __func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
+			__func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	data->fence = fd;
+
+	fd_install(fd, sync_file->file);
+
+	return 0;
+
+err_put_fd:
+	put_unused_fd(fd);
+err_out:
+	return err;
+}
+
+int pvr_sync_api_sw_inc(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sw_timeline_advance_data *data = user_data;
+	bool res;
+
+	res = pvr_counting_fence_timeline_inc(timeline->sw_fence_timeline, &data->sync_pt_idx);
+
+	/* pvr_counting_fence_timeline_inc won't allow sw timeline to be
+	 * advanced beyond the last defined point
+	 */
+	if (!res) {
+		pr_err("pvr_sync_file: attempt to advance SW timeline beyond last defined point\n");
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static void
+pvr_sync_debug_request_heading(void *data, u32 verbosity,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  "------[ Native Fence Sync: timelines ]------");
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void)
+{
+	/* Register the resolve fence and create fence functions with
+	 * sync_checkpoint.c
+	 * The pvr_fence context registers its own EventObject callback to
+	 * update sync status
+	 */
+	/* Initialise struct and register with sync_checkpoint.c */
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceResolve = pvr_sync_resolve_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceCreate = pvr_sync_create_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceDataRollback = pvr_sync_rollback_fence_data;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceFinalise = pvr_sync_finalise_fence;
+#if defined(NO_HARDWARE)
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = pvr_sync_nohw_signal_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWSignalExpFence = pvr_sync_nohw_signal_exp_fence;
+#else
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = NULL;
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWSignalExpFence = NULL;
+#endif
+	pvr_sync_data.sync_checkpoint_ops.pfnFreeCheckpointListMem =
+		pvr_sync_free_checkpoint_list_mem;
+	pvr_sync_data.sync_checkpoint_ops.pfnDumpInfoOnStalledUFOs =
+		pvr_sync_dump_info_on_stalled_ufos;
+	OSStringSafeCopy(pvr_sync_data.sync_checkpoint_ops.pszImplName, "pvr_sync_file",
+		SYNC_CHECKPOINT_IMPL_MAX_STRLEN);
+#if defined(PDUMP)
+	pvr_sync_data.sync_checkpoint_ops.pfnSyncFenceGetCheckpoints =
+		pvr_sync_fence_get_checkpoints;
+#endif
+	pvr_sync_data.sync_checkpoint_ops.pfnExportFenceResolve = pvr_sync_resolve_export_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnExportFenceRollback = pvr_sync_rollback_export_fence;
+
+	return SyncCheckpointRegisterFunctions(&pvr_sync_data.sync_checkpoint_ops);
+}
+
+int pvr_sync_init(void)
+{
+	int err;
+
+	pvr_sync_data.foreign_fence_context =
+			pvr_fence_foreign_context_create(
+					NativeSyncGetFenceStatusWq(),
+					"foreign_sync");
+	if (!pvr_sync_data.foreign_fence_context) {
+		pr_err(FILE_NAME ": %s: Failed to create foreign sync context\n",
+			__func__);
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+#if defined(NO_HARDWARE)
+	INIT_LIST_HEAD(&pvr_timeline_active_list);
+#endif
+
+	err = pvr_sync_ioctl_init();
+	if (err) {
+		pr_err(FILE_NAME ": %s: Failed to register pvr_sync device (%d)\n",
+		       __func__, err);
+		goto err_ioctl_init;
+	}
+
+	return 0;
+
+err_ioctl_init:
+	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
+	pvr_fence_cleanup();
+err_out:
+	return err;
+}
+
+void pvr_sync_deinit(void)
+{
+	pvr_sync_ioctl_deinit();
+	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
+	pvr_fence_cleanup();
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+	enum PVRSRV_ERROR_TAG error;
+
+	error = PVRSRVRegisterDeviceDbgRequestNotify(
+				&priv->sync_debug_notify_handle,
+				priv->dev_node,
+				pvr_sync_debug_request_heading,
+				DEBUG_REQUEST_LINUXFENCE,
+				NULL);
+	if (error != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(error));
+		goto err_out;
+	}
+
+	/* Register the foreign sync context debug notifier on each device */
+	error = pvr_fence_context_register_dbg(
+				&priv->sync_foreign_debug_notify_handle,
+				priv->dev_node,
+				pvr_sync_data.foreign_fence_context);
+	if (error != PVRSRV_OK) {
+		pr_err("%s: failed to register fence debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(error));
+		goto err_context_regdbg;
+	}
+
+#if defined(NO_HARDWARE)
+	INIT_LIST_HEAD(&pvr_timeline_active_list);
+#endif
+
+	return PVRSRV_OK;
+
+err_context_regdbg:
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
+err_out:
+	return error;
+}
+
+void pvr_sync_device_deinit(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_foreign_debug_notify_handle);
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms)
+{
+	long timeout = msecs_to_jiffies(timeout_in_ms);
+	int err;
+
+	err = dma_fence_wait_timeout(fence, true, timeout);
+	/*
+	 * dma_fence_wait_timeout returns:
+	 * - the remaining timeout on success
+	 * - 0 on timeout
+	 * - -ERESTARTSYS if interrupted
+	 */
+	if (err > 0)
+		return PVRSRV_OK;
+	else if (err == 0)
+		return PVRSRV_ERROR_TIMEOUT;
+
+	return PVRSRV_ERROR_FAILED_DEPENDENCIES;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence)
+{
+	dma_fence_put(fence);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out)
+{
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_fd);
+	if (fence == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	*fence_out = fence;
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
+				  int timeline_fd,
+				  const char *fence_name,
+				  int *fence_fd_out,
+				  u64 *sync_pt_idx)
+{
+	enum PVRSRV_ERROR_TAG srv_err;
+	struct pvr_sync_timeline *timeline;
+	struct dma_fence *fence = NULL;
+	struct sync_file *sync_file = NULL;
+	int fd;
+
+	(void)(pvrsrv_dev_node);
+
+	fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fd < 0)
+		return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+
+	timeline = pvr_sync_timeline_fget(timeline_fd);
+	if (!timeline) {
+		/* unrecognised timeline */
+		srv_err = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		goto err_put_fd;
+	}
+	if (!timeline->is_sw) {
+		pvr_sync_timeline_fput(timeline);
+		srv_err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_fd;
+	}
+
+	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, sync_pt_idx);
+	pvr_sync_timeline_fput(timeline);
+	if (!fence) {
+		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fd;
+	}
+
+	fd_install(fd, sync_file->file);
+
+	*fence_fd_out = fd;
+
+	return PVRSRV_OK;
+
+err_put_fd:
+	put_unused_fd(fd);
+	return srv_err;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline, u64 *sync_pt_idx)
+{
+	if (timeline == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	pvr_counting_fence_timeline_inc(timeline, sync_pt_idx);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline)
+{
+	if (timeline == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	pvr_counting_fence_timeline_put(timeline);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
+					   void **timeline_out)
+{
+	struct pvr_counting_fence_timeline *sw_timeline;
+	struct pvr_sync_timeline *timeline;
+
+	timeline = pvr_sync_timeline_fget(timeline_fd);
+	if (!timeline)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	sw_timeline =
+		pvr_counting_fence_timeline_get(timeline->sw_fence_timeline);
+	pvr_sync_timeline_fput(timeline);
+	if (!sw_timeline)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	*timeline_out = sw_timeline;
+
+	return PVRSRV_OK;
+}
+static void _dump_sync_point(struct dma_fence *fence,
+							  DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+							  void *dump_debug_file)
+{
+	const struct dma_fence_ops *fence_ops = fence->ops;
+	bool signaled = dma_fence_is_signaled(fence);
+	char time[16] = { '\0' };
+
+	fence_ops->timeline_value_str(fence, time, sizeof(time));
+
+	PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "<%p> Seq#=%llu TS=%s State=%s TLN=%s",
+					  fence,
+					  (u64) fence->seqno,
+					  time,
+					  (signaled) ? "Signalled" : "Active",
+					  fence_ops->get_timeline_name(fence));
+}
+
+static void _dump_fence(struct dma_fence *fence,
+			DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+			void *dump_debug_file)
+{
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *fence_array = to_dma_fence_array(fence);
+		int i;
+
+		if (fence_array) {
+			PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "Fence: [%p] Sync Points:\n",
+					  fence_array);
+
+			for (i = 0; i < fence_array->num_fences; i++)
+				_dump_sync_point(fence_array->fences[i],
+						 dump_debug_printf,
+						 dump_debug_file);
+		}
+
+	} else {
+		_dump_sync_point(fence, dump_debug_printf, dump_debug_file);
+	}
+}
+
+enum PVRSRV_ERROR_TAG
+sync_dump_fence(void *sw_fence_obj,
+		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		void *dump_debug_file)
+{
+	struct dma_fence *fence = (struct dma_fence *) sw_fence_obj;
+
+	_dump_fence(fence, dump_debug_printf, dump_debug_file);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG
+sync_sw_dump_timeline(void *sw_timeline_obj,
+		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		      void *dump_debug_file)
+{
+	pvr_counting_fence_timeline_dump_timeline(sw_timeline_obj,
+						  dump_debug_printf,
+						  dump_debug_file);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.c
new file mode 100644
index 000000000000..0e1facde86cb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.c
@@ -0,0 +1,405 @@
+/*
+ * @File        pvr_sync_ioctl_common.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/slab.h>
+
+#include "pvr_drm.h"
+#include "pvr_sync_api.h"
+#include "pvr_sync_ioctl_common.h"
+
+#include "pvr_debug.h"
+#include "pvr_uaccess.h"
+
+/*
+ * The PVR Sync API is unusual in that some operations configure the
+ * timeline for use, and are no longer allowed once the timeline is
+ * in use. A locking mechanism, such as a read/write semaphore, would
+ * be one method of helping to ensure the API rules are followed, but
+ * this would add unnecessary overhead once the timeline has been
+ * configured, as read locks would continue to have to be taken after
+ * the timeline is in use. To avoid locks, two atomic variables are used,
+ * together with memory barriers. The in_setup variable indicates a "rename"
+ * or "force software only" ioctl is in progress. At most one of these two
+ * configuration ioctls can be in progress at any one time, and they can't
+ * overlap with any other Sync ioctl. The in_use variable indicates one
+ * of the other Sync ioctls has started. Once set, in_use stays set, and
+ * prevents any further configuration ioctls. Non-configuration ioctls
+ * are allowed to overlap.
+ * It is possible for a configuration and non-configuration ioctl to race,
+ * but at most one will be allowed to proceed, and perhaps neither.
+ * Given the intended usage of the API in user space, where the timeline
+ * is fully configured before being used, the race behaviour won't be
+ * an issue.
+ */
+
+struct pvr_sync_file_data {
+	atomic_t in_setup;
+	atomic_t in_use;
+	void *api_private;
+	bool is_sw;
+	bool is_export;
+};
+
+static bool pvr_sync_set_in_use(struct pvr_sync_file_data *fdata)
+{
+	if (atomic_read(&fdata->in_use) < 2) {
+		atomic_set(&fdata->in_use, 1);
+		/* Ensure in_use change is visible before in_setup is read */
+		smp_mb();
+		if (atomic_read(&fdata->in_setup) != 0)
+			return false;
+
+		atomic_set(&fdata->in_use, 2);
+	} else {
+		/* Ensure stale private data isn't read */
+		smp_rmb();
+	}
+
+	return true;
+}
+
+static bool pvr_sync_set_in_setup(struct pvr_sync_file_data *fdata)
+{
+	int in_setup;
+
+	in_setup = atomic_inc_return(&fdata->in_setup);
+	if (in_setup > 1 || atomic_read(&fdata->in_use) != 0) {
+		atomic_dec(&fdata->in_setup);
+		return false;
+	}
+
+	return true;
+}
+
+static inline void pvr_sync_reset_in_setup(struct pvr_sync_file_data *fdata)
+{
+	/*
+	 * Ensure setup changes are visible before allowing other
+	 * operations to proceed.
+	 */
+	smp_mb__before_atomic();
+	atomic_dec(&fdata->in_setup);
+}
+
+void *pvr_sync_get_api_priv_common(struct file *file)
+{
+	if (file != NULL && pvr_sync_is_timeline(file)) {
+		struct pvr_sync_file_data *fdata = pvr_sync_get_private_data(file);
+
+		if (fdata != NULL && pvr_sync_set_in_use(fdata))
+			return fdata->api_private;
+	}
+
+	return NULL;
+}
+
+int pvr_sync_open_common(void *connection_data, void *file_handle)
+{
+	void *data = NULL;
+	struct pvr_sync_file_data *fdata;
+	int err;
+
+	fdata = kzalloc(sizeof(*fdata), GFP_KERNEL);
+	if (!fdata)
+		return -ENOMEM;
+
+	atomic_set(&fdata->in_setup, 0);
+	atomic_set(&fdata->in_use, 0);
+
+	if (!pvr_sync_set_private_data(connection_data, fdata)) {
+		kfree(fdata);
+		return -EINVAL;
+	}
+
+	err = pvr_sync_api_init(file_handle, &data);
+	if (err)
+		kfree(fdata);
+	else
+		fdata->api_private = data;
+
+	return err;
+}
+
+int pvr_sync_close_common(void *connection_data)
+{
+	struct pvr_sync_file_data *fdata;
+
+	fdata = pvr_sync_connection_private_data(connection_data);
+	if (fdata) {
+		int err;
+
+		err = pvr_sync_api_deinit(fdata->api_private, fdata->is_sw);
+
+		kfree(fdata);
+
+		return err;
+	}
+
+	return 0;
+}
+
+#define PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(name, type) \
+static inline int pvr_sync_ioctl_common_internal_ ## name(struct pvr_sync_file_data *fdata, type data)
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(rename, struct pvr_sync_rename_ioctl_data *)
+{
+	return pvr_sync_api_rename(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(force_sw_only, void*)
+{
+	void *new_api_private = fdata->api_private;
+	int err;
+
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	err = pvr_sync_api_force_sw_only(fdata->api_private, &new_api_private);
+	if (!err) {
+		if (new_api_private != fdata->api_private)
+			fdata->api_private = new_api_private;
+
+		fdata->is_sw = true;
+	}
+
+	return err;
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(force_exp_only, void*)
+{
+	int err;
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	err = pvr_sync_api_force_exp_only(fdata->api_private, data);
+	if (!err)
+		fdata->is_export = true;
+
+	return err;
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(create_export_fence, void*)
+{
+	return pvr_sync_api_create_export_fence(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(sw_create_fence, struct pvr_sw_sync_create_fence_data *)
+{
+	return pvr_sync_api_sw_create_fence(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(sw_inc, struct pvr_sw_timeline_advance_data *)
+{
+	return pvr_sync_api_sw_inc(fdata->api_private, data);
+}
+
+/*
+ * enum pvr_sync_ioctl_dispatch_type
+ * @pvr_sync_ioctl_dispatch_type_setup: The command may only be used during the setup phase
+ * @pvr_sync_ioctl_dispatch_type_export: The command may only be used with an export only timeline.
+ * @pvr_sync_ioctl_dispatch_type_software: The command may only be used with a software only timeline.
+ */
+enum pvr_sync_ioctl_dispatch_type {
+	pvr_sync_ioctl_dispatch_type_setup,
+	pvr_sync_ioctl_dispatch_type_export,
+	pvr_sync_ioctl_dispatch_type_software,
+};
+
+/* Generates a function `from` which performs validation on the data before passing it to `into` */
+#define PVR_SYNC_IOCTL_DISPATCH_VALIDATE(name, structure, type)                                                                        \
+PVR_SYNC_IOCTL_DISPATCH_DECL(name)                                                                                                     \
+{                                                                                                                                      \
+	int err = -ENOTTY;                                                                                                                 \
+	structure server_data;                                                                                                             \
+                                                                                                                                       \
+	struct pvr_sync_file_data *fdata = pvr_sync_get_private_data(file);                                                                \
+	if (unlikely(!fdata))                                                                                                              \
+		return -EINVAL;                                                                                                                \
+                                                                                                                                       \
+	/* Check if the device is busy and the operation is valid for the timelines current state */                                       \
+	if (type == pvr_sync_ioctl_dispatch_type_setup)                                                                                    \
+	{                                                                                                                                  \
+		if (!pvr_sync_set_in_setup(fdata))                                                                                             \
+			return -EBUSY;                                                                                                             \
+		if (fdata->is_sw || fdata->is_export)                                                                                          \
+			goto return_;                                                                                                              \
+	}                                                                                                                                  \
+	else                                                                                                                               \
+	{                                                                                                                                  \
+		if (!pvr_sync_set_in_use(fdata))                                                                                               \
+			return -EBUSY;                                                                                                             \
+                                                                                                                                       \
+		switch (type)                                                                                                                  \
+		{                                                                                                                              \
+		case pvr_sync_ioctl_dispatch_type_software: {                                                                                  \
+			if (!fdata->is_sw) /* Not a software timeline, software operations cannot be used */                                       \
+				goto return_;                                                                                                          \
+		}                                                                                                                              \
+		break;                                                                                                                         \
+		case pvr_sync_ioctl_dispatch_type_export: {                                                                                    \
+			if (!fdata->is_export) /* Not an export timeline, export operations cannot be used */                                      \
+				goto return_;                                                                                                          \
+		}                                                                                                                              \
+		break;                                                                                                                         \
+		default:                                                                                                                       \
+			goto return_; /* Invalid Type */                                                                                           \
+		}                                                                                                                              \
+	}                                                                                                                                  \
+                                                                                                                                       \
+	/* copy_from_user */                                                                                                               \
+	err = pvr_sync_ioctl_dispatch_copy_in__##name((structure __user *)user_data, &server_data);                                        \
+	if (unlikely(err))                                                                                                                 \
+		goto return_;                                                                                                                  \
+                                                                                                                                       \
+	/* Continue into api */                                                                                                            \
+	err = pvr_sync_ioctl_common_internal_ ## name(fdata, (structure __force *) PVR_SYNC_IOCTL_DISPATCH_DATA(user_data, &server_data)); \
+                                                                                                                                       \
+	if (likely(!err))                                                                                                                  \
+	{                                                                                                                                  \
+		/* copy_to_user */                                                                                                             \
+		err = pvr_sync_ioctl_dispatch_copy_out__##name((structure __user *)user_data, &server_data);                                   \
+	}                                                                                                                                  \
+                                                                                                                                       \
+return_:                                                                                                                               \
+	if (type == pvr_sync_ioctl_dispatch_type_setup)                                                                                    \
+		pvr_sync_reset_in_setup(fdata);                                                                                                \
+	return err;                                                                                                                        \
+}
+
+#if !defined(USE_PVRSYNC_DEVNODE)
+/* drm_ioctl() already copies the data over, see comment on drm_ioctl_t */
+#define PVR_SYNC_IOCTL_DISPATCH_DATA(pUM, pKM) pUM
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(dir, name, structure, copy) \
+INLINE static int pvr_sync_ioctl_dispatch_copy_ ## dir ## __ ## name (structure __user *pUM, structure *pKM) \
+{ return 0; }
+#else /* !defined(USE_PVRSYNC_DEVNODE) */
+/* Generates a function to copy over the arguments to/from user-mode */
+#define PVR_SYNC_IOCTL_DISPATCH_DATA(pUM, pKM) pKM
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(dir, name, structure, copy)                                     \
+INLINE static int pvr_sync_ioctl_dispatch_copy_ ## dir ## __ ## name (structure __user *pUM, structure *pKM) \
+{                                                                                                            \
+	/* May be unused if there are no in/out args */                                                          \
+	PVR_UNREFERENCED_PARAMETER(pUM);                                                                         \
+	PVR_UNREFERENCED_PARAMETER(pKM);                                                                         \
+	/* Copy over the data */                                                                                 \
+	{ copy }                                                                                                 \
+	return 0;                                                                                                \
+}
+#endif /* !defined(USE_PVRSYNC_DEVNODE) */
+
+/*************************************************************************/ /*!
+@Function       PVR_SYNC_IOCTL_DISPATCH_COPY
+@Description    Generates the code to copy from/to user mode depending on @dir.
+@Input          dir  Either `from` or `to` corresponding to `copy_from_user`
+                    and `copy_to_user` respectively.
+@Input          to   Pointer for the dest.
+@Input          from Pointer for the src.
+*/ /**************************************************************************/
+#define PVR_SYNC_IOCTL_DISPATCH_COPY(dir, to, from)        \
+if (pvr_copy_##dir##_user(to, from, sizeof(*pKM)))         \
+{                                                          \
+	PVR_DPF((PVR_DBG_ERROR, "Failed copy " #dir " user")); \
+	return -EFAULT;                                        \
+}
+
+/* Copy data from user */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_IN \
+PVR_SYNC_IOCTL_DISPATCH_COPY(from, pKM, pUM)
+
+/* Copy data to user */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_OUT \
+PVR_SYNC_IOCTL_DISPATCH_COPY(to, pUM, pKM)
+
+/* Copy no data */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_NONE
+
+/*************************************************************************/ /*!
+@Function       PVR_SYNC_IOCTL_DISPATCH_FUNCTION
+@Description    Generates a dispatch function which validates the ioctl command
+                and dispatches it to the internal an function.
+                Generates a dispatch function
+                `pvr_sync_ioctl_common_##name(struct file*, void*)`
+                which validates the ioctl command and dispatches it to
+                an internal function
+                `pvr_sync_ioctl_common_internal_##name(struct pvr_sync_file_data*, structure*)`
+@Input          name      A name for the dispatch functions, must be unique.
+@Input          structure The type of the user_data.
+@Input          type      The type specifies when the ioctls are allowed or
+                          if the ioctl is allowed on the current
+                          timeline configuration.
+@Input          copy_in   Either `COPY_IN` or `COPY_NONE`
+@Input          copy_out  Either `COPY_OUT` or `COPY_NONE`
+*/ /**************************************************************************/
+#define PVR_SYNC_IOCTL_DISPATCH_FUNCTION(name, structure, type, copy_in, copy_out) \
+	PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(in, name, structure, PVR_SYNC_IOCTL_DISPATCH_ ## copy_in) \
+	PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(out, name, structure, PVR_SYNC_IOCTL_DISPATCH_ ## copy_out) \
+	PVR_SYNC_IOCTL_DISPATCH_VALIDATE(name, structure, pvr_sync_ioctl_dispatch_type_ ## type)
+
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(rename, struct pvr_sync_rename_ioctl_data,
+                                 setup,
+                                 COPY_IN,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(force_sw_only, void*,
+                                 setup,
+                                 COPY_NONE,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(force_exp_only, void*,
+                                 setup,
+                                 COPY_NONE,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(sw_create_fence, struct pvr_sw_sync_create_fence_data,
+                                 software,
+                                 COPY_IN,
+                                 COPY_OUT);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(create_export_fence, pvr_exp_sync_create_fence_data_t,
+                                 export,
+                                 COPY_IN,
+                                 COPY_OUT);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(sw_inc, struct pvr_sw_timeline_advance_data,
+                                 software,
+                                 COPY_NONE,
+                                 COPY_OUT);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.h
new file mode 100644
index 000000000000..0959e7673164
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.h
@@ -0,0 +1,85 @@
+/*
+ * @File        pvr_sync_ioctl_common.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_IOCTL_COMMON_H
+#define _PVR_SYNC_IOCTL_COMMON_H
+
+struct file;
+
+/* Functions provided by pvr_sync_ioctl_common.c */
+
+int pvr_sync_open_common(void *connection_data, void *file_handle);
+int pvr_sync_close_common(void *connection_data);
+void *pvr_sync_get_api_priv_common(struct file *file);
+
+#if defined(USE_PVRSYNC_DEVNODE)
+#define PVR_SYNC_IOCTL_DISPATCH_DECL(name) \
+	int pvr_sync_ioctl_common_ ## name(struct file *file, void __user *user_data)
+#else /* defined(USE_PVRSYNC_DEVNODE) */
+#define PVR_SYNC_IOCTL_DISPATCH_DECL(name) \
+	int pvr_sync_ioctl_common_ ## name(struct file *file, void *user_data)
+#endif /* defined(USE_PVRSYNC_DEVNODE) */
+
+PVR_SYNC_IOCTL_DISPATCH_DECL(rename);
+PVR_SYNC_IOCTL_DISPATCH_DECL(force_sw_only);
+PVR_SYNC_IOCTL_DISPATCH_DECL(force_exp_only);
+PVR_SYNC_IOCTL_DISPATCH_DECL(sw_create_fence);
+PVR_SYNC_IOCTL_DISPATCH_DECL(create_export_fence);
+PVR_SYNC_IOCTL_DISPATCH_DECL(sw_inc);
+
+struct pvr_sync_file_data;
+
+/* Functions required by pvr_sync_ioctl_common.c */
+
+bool pvr_sync_set_private_data(void *connection_data,
+			       struct pvr_sync_file_data *fdata);
+
+struct pvr_sync_file_data *
+pvr_sync_connection_private_data(void *connection_data);
+
+struct pvr_sync_file_data *
+pvr_sync_get_private_data(struct file *file);
+
+bool pvr_sync_is_timeline(struct file *file);
+
+#endif /* _PVR_SYNC_IOCTL_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.c
new file mode 100644
index 000000000000..313393d09c1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.c
@@ -0,0 +1,184 @@
+/*
+ * @File        pvr_sync_ioctl_drm.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pvr_drv.h"
+#include "pvr_drm.h"
+#include "private_data.h"
+#include "env_connection.h"
+#include "pvr_sync_api.h"
+#include "pvr_sync_ioctl_common.h"
+#include "pvr_sync_ioctl_drm.h"
+
+bool pvr_sync_set_private_data(void *connection_data,
+			       struct pvr_sync_file_data *fdata)
+{
+	if (connection_data) {
+		ENV_CONNECTION_DATA *env_data;
+
+		env_data = PVRSRVConnectionPrivateData(connection_data);
+		if (env_data) {
+			env_data->pvPvrSyncPrivateData = fdata;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+struct pvr_sync_file_data *
+pvr_sync_connection_private_data(void *connection_data)
+{
+	if (connection_data) {
+		ENV_CONNECTION_DATA *env_data;
+
+		env_data = PVRSRVConnectionPrivateData(connection_data);
+
+		if (env_data)
+			return env_data->pvPvrSyncPrivateData;
+	}
+
+	return NULL;
+}
+
+struct pvr_sync_file_data *
+pvr_sync_get_private_data(struct file *file)
+{
+	CONNECTION_DATA *connection_data = LinuxSyncConnectionFromFile(file);
+
+	return pvr_sync_connection_private_data(connection_data);
+}
+
+bool pvr_sync_is_timeline(struct file *file)
+{
+	return file->f_op == &pvr_drm_fops;
+}
+
+void *pvr_sync_get_api_priv(struct file *file)
+{
+	return pvr_sync_get_api_priv_common(file);
+}
+
+struct file *pvr_sync_get_file_struct(void *file_handle)
+{
+	if (file_handle) {
+		struct drm_file *file = file_handle;
+
+		return file->filp;
+	}
+
+	return NULL;
+}
+
+int pvr_sync_open(void *connection_data, struct drm_file *file)
+{
+	/*
+	 * The file structure pointer (file->filp) may not have been
+	 * initialised at this point, so pass down a pointer to the
+	 * drm_file structure instead.
+	 */
+	return pvr_sync_open_common(connection_data, file);
+}
+
+void pvr_sync_close(void *connection_data)
+{
+	int iErr = pvr_sync_close_common(connection_data);
+
+	if (iErr < 0)
+		pr_err("%s: ERROR (%d) returned by pvr_sync_close_common()\n",
+		       __func__, iErr);
+
+	iErr = OSConnectionPrivateDataDeInit(PVRSRVConnectionPrivateData(connection_data));
+	if (iErr != 0)
+		pr_err("%s: ERROR (%d) returned by OSConnectionPrivateDataDeInit()\n",
+		       __func__, iErr);
+
+	if (connection_data)
+		kfree(connection_data);
+}
+
+
+int pvr_sync_rename_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_rename(file->filp, arg);
+}
+
+int pvr_sync_force_sw_only_ioctl(struct drm_device __maybe_unused *dev,
+				 void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_force_sw_only(file->filp, arg);
+}
+
+int pvr_sw_sync_create_fence_ioctl(struct drm_device __maybe_unused *dev,
+				   void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_sw_create_fence(file->filp, arg);
+}
+
+int pvr_sw_sync_inc_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_sw_inc(file->filp, arg);
+}
+
+int pvr_sync_ioctl_force_exp_only(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_force_exp_only(file->filp, arg);
+}
+
+int pvr_export_fence_sync_create_fence_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_create_export_fence(file->filp, arg);
+}
+
+int pvr_sync_ioctl_init(void)
+{
+	return 0;
+}
+
+void pvr_sync_ioctl_deinit(void)
+{
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.h
new file mode 100644
index 000000000000..93182205ca0b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.h
@@ -0,0 +1,66 @@
+/*
+ * @File        pvr_sync_ioctl_drm.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_IOCTL_DRM_H
+#define _PVR_SYNC_IOCTL_DRM_H
+
+struct drm_device;
+struct drm_file;
+
+int pvr_sync_open(void *connection_data, struct drm_file *file);
+void pvr_sync_close(void *connection_data);
+
+int pvr_sync_rename_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_sync_force_sw_only_ioctl(struct drm_device *dev, void *arg,
+				 struct drm_file *file);
+int pvr_sw_sync_create_fence_ioctl(struct drm_device *dev, void *arg,
+				   struct drm_file *file);
+int pvr_sw_sync_inc_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_sync_ioctl_force_exp_only(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_export_fence_sync_create_fence_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+
+#endif /* _PVR_SYNC_IOCTL_DRM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_uaccess.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_uaccess.h
new file mode 100644
index 000000000000..13864eab4c7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_uaccess.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          Utility functions for user space access
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_UACCESS_H
+#define PVR_UACCESS_H
+
+#include <linux/uaccess.h>
+#include <linux/version.h>
+
+static inline unsigned long pvr_copy_to_user(void __user *pvTo, const void *pvFrom, unsigned long ulBytes)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+	if (access_ok(VERIFY_WRITE, pvTo, ulBytes))
+#else
+	if (access_ok(pvTo, ulBytes))
+#endif
+	{
+		return __copy_to_user(pvTo, pvFrom, ulBytes);
+	}
+
+	return ulBytes;
+}
+
+
+#if defined(__KLOCWORK__)
+	/* this part is only to tell Klocwork not to report false positive because
+	   it doesn't understand that pvr_copy_from_user will initialise the memory
+	   pointed to by pvTo */
+#include <linux/string.h> /* get the memset prototype */
+static inline unsigned long pvr_copy_from_user(void *pvTo, const void __user *pvFrom, unsigned long ulBytes)
+{
+	if (pvTo != NULL)
+	{
+		memset(pvTo, 0xAA, ulBytes);
+		return 0;
+	}
+	return 1;
+}
+
+#else /* real implementation */
+
+static inline unsigned long pvr_copy_from_user(void *pvTo, const void __user *pvFrom, unsigned long ulBytes)
+{
+	/*
+	 * The compile time correctness checking introduced for copy_from_user in
+	 * Linux 2.6.33 isn't fully compatible with our usage of the function.
+	 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+	if (access_ok(VERIFY_READ, pvFrom, ulBytes))
+#else
+	if (access_ok(pvFrom, ulBytes))
+#endif
+	{
+		return __copy_from_user(pvTo, pvFrom, ulBytes);
+	}
+
+	return ulBytes;
+}
+#endif /* klocworks */
+
+#endif /* PVR_UACCESS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/rogue_trace_events.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/rogue_trace_events.h
new file mode 100644
index 000000000000..8952bec0f102
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/rogue_trace_events.h
@@ -0,0 +1,592 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rogue
+
+#if !defined(ROGUE_TRACE_EVENTS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define ROGUE_TRACE_EVENTS_H
+
+#include <linux/version.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+#include <linux/time.h>
+
+#define show_secs_from_ns(ns) \
+	({ \
+		u64 t = ns + (NSEC_PER_USEC / 2); \
+		do_div(t, NSEC_PER_SEC); \
+		t; \
+	})
+
+#define show_usecs_from_ns(ns) \
+	({ \
+		u64 t = ns + (NSEC_PER_USEC / 2); \
+		do_div(t, NSEC_PER_USEC); \
+		do_div(t, USEC_PER_SEC); \
+	})
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_update_enabled_callback(void);
+#else
+void trace_fence_update_enabled_callback(void);
+#endif
+void trace_fence_update_disabled_callback(void);
+
+TRACE_EVENT_FN(rogue_fence_update,
+
+	TP_PROTO(const char *comm, const char *cmd, const char *dm, u32 gpu_id, u32 ctx_id, u32 offset,
+		u32 sync_fwaddr, u32 sync_value),
+
+	TP_ARGS(comm, cmd, dm, gpu_id, ctx_id, offset, sync_fwaddr, sync_value),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       cmd,            cmd             )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+		__field(        u32,            offset          )
+		__field(        u32,            sync_fwaddr     )
+		__field(        u32,            sync_value      )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm, comm);
+		__assign_str(cmd, cmd);
+		__assign_str(dm, dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->offset = offset;
+		__entry->sync_fwaddr = sync_fwaddr;
+		__entry->sync_value = sync_value;
+	),
+
+	TP_printk("comm=%s cmd=%s dm=%s gpu=%lu ctx_id=%lu offset=%lu sync_fwaddr=%#lx sync_value=%#lx",
+		__get_str(comm),
+		__get_str(cmd),
+		__get_str(dm),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->offset,
+		(unsigned long) __entry->sync_fwaddr,
+		(unsigned long) __entry->sync_value
+	),
+
+	trace_fence_update_enabled_callback,
+	trace_fence_update_disabled_callback
+);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_check_enabled_callback(void);
+#else
+void trace_fence_check_enabled_callback(void);
+#endif
+void trace_fence_check_disabled_callback(void);
+
+TRACE_EVENT_FN(rogue_fence_check,
+
+	TP_PROTO(const char *comm, const char *cmd, const char *dm, u32 gpu_id, u32 ctx_id, u32 offset,
+		u32 sync_fwaddr, u32 sync_value),
+
+	TP_ARGS(comm, cmd, dm, gpu_id, ctx_id, offset, sync_fwaddr, sync_value),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       cmd,            cmd             )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+		__field(        u32,            offset          )
+		__field(        u32,            sync_fwaddr     )
+		__field(        u32,            sync_value      )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm, comm);
+		__assign_str(cmd, cmd);
+		__assign_str(dm, dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->offset = offset;
+		__entry->sync_fwaddr = sync_fwaddr;
+		__entry->sync_value = sync_value;
+	),
+
+	TP_printk("comm=%s cmd=%s dm=%s gpu=%lu ctx_id=%lu offset=%lu sync_fwaddr=%#lx sync_value=%#lx",
+		__get_str(comm),
+		__get_str(cmd),
+		__get_str(dm),
+		(unsigned long)__entry->gpu_id,
+		(unsigned long)__entry->ctx_id,
+		(unsigned long)__entry->offset,
+		(unsigned long)__entry->sync_fwaddr,
+		(unsigned long)__entry->sync_value
+	),
+
+	trace_fence_check_enabled_callback,
+	trace_fence_check_disabled_callback
+);
+
+TRACE_EVENT(rogue_job_enqueue,
+
+	TP_PROTO(u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         const char *kick_type),
+
+	TP_ARGS(gpu_id, ctx_id, int_id, ext_id, kick_type),
+
+	TP_STRUCT__entry(
+		__field(u32, gpu_id)
+		__field(u32, ctx_id)
+		__field(u32, int_id)
+		__field(u32, ext_id)
+		__string(kick_type, kick_type)
+	),
+
+	TP_fast_assign(
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__assign_str(kick_type, kick_type);
+	),
+
+	TP_printk("gpu=%lu, ctx_id=%lu int_id=%lu ext_id=%lu kick_type=%s",
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		__get_str(kick_type)
+	)
+);
+
+TRACE_EVENT(rogue_sched_switch,
+
+	TP_PROTO(const char *work_type, u32 switch_type, u64 timestamp, u32 gpu_id, u32 next_ctx_id,
+	         u32 next_prio, u32 next_int_id, u32 next_ext_id),
+
+	TP_ARGS(work_type, switch_type, timestamp, gpu_id, next_ctx_id, next_prio, next_int_id,
+	        next_ext_id),
+
+	TP_STRUCT__entry(
+		__string(work_type, work_type)
+		__field(u32, switch_type)
+		__field(u64, timestamp)
+		__field(u32, gpu_id)
+		__field(u32, next_ctx_id)
+		__field(u32, next_prio)
+		__field(u32, next_int_id)
+		__field(u32, next_ext_id)
+	),
+
+	TP_fast_assign(
+		__assign_str(work_type, work_type);
+		__entry->switch_type = switch_type;
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->next_ctx_id = next_ctx_id;
+		__entry->next_prio = next_prio;
+		__entry->next_int_id = next_int_id;
+		__entry->next_ext_id = next_ext_id;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu next_ctx_id=%lu next_int_id=%lu next_ext_id=%lu"
+		" next_prio=%lu work_type=%s switch_type=%s",
+		(unsigned long long) show_secs_from_ns(__entry->timestamp),
+		(unsigned long) show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->next_ctx_id,
+		(unsigned long) __entry->next_int_id,
+		(unsigned long) __entry->next_ext_id,
+		(unsigned long) __entry->next_prio,
+		__get_str(work_type),
+		__print_symbolic(__entry->switch_type,
+			/* These values are from ospvr_gputrace.h. */
+			{ 1, "begin" },
+			{ 2, "end" })
+	)
+);
+
+TRACE_EVENT(rogue_create_fw_context,
+
+	TP_PROTO(const char *comm, const char *dm, u32 gpu_id, u32 ctx_id),
+
+	TP_ARGS(comm, dm, gpu_id, ctx_id),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm, comm);
+		__assign_str(dm, dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+	),
+
+	TP_printk("comm=%s dm=%s gpu=%lu ctx_id=%lu",
+		__get_str(comm),
+		__get_str(dm),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id
+	)
+);
+
+void PVRGpuTraceEnableUfoCallback(void);
+void PVRGpuTraceDisableUfoCallback(void);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableUfoCallbackWrapper(void);
+#else
+#define PVRGpuTraceEnableUfoCallbackWrapper \
+		PVRGpuTraceEnableUfoCallback
+#endif
+
+TRACE_EVENT_FN(rogue_ufo_update,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 old_value, u32 new_value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, old_value,
+	        new_value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            old_value   )
+		__field(        u32,            new_value   )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->old_value = old_value;
+		__entry->new_value = new_value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx old_value=%#lx new_value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->old_value,
+		(unsigned long) __entry->new_value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_check_fail,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value, u32 required),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value, required),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+		__field(        u32,            required    )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+		__entry->required = required;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx required=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value,
+		(unsigned long) __entry->required
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_pr_check_fail,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value, u32 required),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value, required),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+		__field(        u32,            required    )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+		__entry->required = required;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx required=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value,
+		(unsigned long) __entry->required
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_check_success,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_pr_check_success,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT(rogue_events_lost,
+
+	TP_PROTO(u32 event_source, u32 gpu_id, u32 last_ordinal, u32 curr_ordinal),
+
+	TP_ARGS(event_source, gpu_id, last_ordinal, curr_ordinal),
+
+	TP_STRUCT__entry(
+		__field(        u32,            event_source     )
+		__field(        u32,            gpu_id           )
+		__field(        u32,            last_ordinal     )
+		__field(        u32,            curr_ordinal     )
+	),
+
+	TP_fast_assign(
+		__entry->event_source = event_source;
+		__entry->gpu_id = gpu_id;
+		__entry->last_ordinal = last_ordinal;
+		__entry->curr_ordinal = curr_ordinal;
+	),
+
+	TP_printk("event_source=%s gpu=%u last_ordinal=%u curr_ordinal=%u",
+		__print_symbolic(__entry->event_source, {0, "GPU"}, {1, "Host"}),
+		__entry->gpu_id,
+		__entry->last_ordinal,
+		__entry->curr_ordinal)
+);
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void);
+void PVRGpuTraceDisableFirmwareActivityCallback(void);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableFirmwareActivityCallbackWrapper(void);
+#else
+#define PVRGpuTraceEnableFirmwareActivityCallbackWrapper \
+		PVRGpuTraceEnableFirmwareActivityCallback
+#endif
+
+TRACE_EVENT_FN(rogue_firmware_activity,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, const char *task, u32 fw_event),
+
+	TP_ARGS(timestamp, gpu_id, task, fw_event),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp       )
+		__field(        u32,            gpu_id             )
+		__string(       task,           task            )
+		__field(        u32,            fw_event        )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__assign_str(task, task);
+		__entry->fw_event = fw_event;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu task=%s event=%s",
+		(unsigned long long) show_secs_from_ns(__entry->timestamp),
+		(unsigned long) show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		__get_str(task),
+		__print_symbolic(__entry->fw_event,
+			/* These values are from ospvr_gputrace.h. */
+			{ 1, "begin" },
+			{ 2, "end" })),
+
+	PVRGpuTraceEnableFirmwareActivityCallbackWrapper,
+	PVRGpuTraceDisableFirmwareActivityCallback
+);
+
+#undef show_secs_from_ns
+#undef show_usecs_from_ns
+
+#endif /* ROGUE_TRACE_EVENTS_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+
+/* This is needed because the name of this file doesn't match TRACE_SYSTEM. */
+#define TRACE_INCLUDE_FILE rogue_trace_events
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/services_kernel_client.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/services_kernel_client.h
new file mode 100644
index 000000000000..4ec5356abbf0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/services_kernel_client.h
@@ -0,0 +1,308 @@
+/*************************************************************************/ /*!
+@File           services_kernel_client.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* This file contains a partial redefinition of the PowerVR Services 5
+ * interface for use by components which are checkpatch clean. This
+ * header is included by the unrefined, non-checkpatch clean headers
+ * to ensure that prototype/typedef/macro changes break the build.
+ */
+
+#ifndef __SERVICES_KERNEL_CLIENT__
+#define __SERVICES_KERNEL_CLIENT__
+
+#include "pvrsrv_error.h"
+
+#include <linux/types.h>
+
+#include "pvrsrv_sync_km.h"
+#include "sync_checkpoint_external.h"
+
+/* included for the define PVRSRV_LINUX_DEV_INIT_ON_PROBE */
+#include "pvr_drm.h"
+
+#include "pvr_dma_resv.h"
+
+#ifndef __pvrsrv_defined_struct_enum__
+
+/* sync_external.h */
+
+struct PVRSRV_CLIENT_SYNC_PRIM_TAG {
+	volatile __u32 *pui32LinAddr;
+};
+
+struct PVRSRV_CLIENT_SYNC_PRIM_OP {
+	__u32 ui32Flags;
+	struct pvrsrv_sync_prim *psSync;
+	__u32 ui32FenceValue;
+	__u32 ui32UpdateValue;
+};
+
+#else /* __pvrsrv_defined_struct_enum__ */
+
+struct PVRSRV_CLIENT_SYNC_PRIM_TAG;
+struct PVRSRV_CLIENT_SYNC_PRIM_OP;
+
+#endif /* __pvrsrv_defined_struct_enum__ */
+
+struct _PMR_;
+struct _PVRSRV_DEVICE_NODE_;
+struct dma_buf;
+struct dma_resv;
+struct SYNC_PRIM_CONTEXT_TAG;
+
+/* pvr_notifier.h */
+
+#ifndef CMDCOMPNOTIFY_PFN
+typedef void (*PFN_CMDCOMP_NOTIFY)(void *hCmdCompHandle);
+#define CMDCOMPNOTIFY_PFN
+#endif
+enum PVRSRV_ERROR_TAG PVRSRVRegisterCmdCompleteNotify(void **phNotify,
+	PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify, void *hPrivData);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterCmdCompleteNotify(void *hNotify);
+void PVRSRVCheckStatus(void *hCmdCompCallerHandle);
+
+#define DEBUG_REQUEST_DC               0
+#define DEBUG_REQUEST_SYNCTRACKING     1
+#define DEBUG_REQUEST_SRV              2
+#define DEBUG_REQUEST_SYS              3
+#define DEBUG_REQUEST_RGX              4
+#define DEBUG_REQUEST_ANDROIDSYNC      5
+#define DEBUG_REQUEST_LINUXFENCE       6
+#define DEBUG_REQUEST_SYNCCHECKPOINT   7
+#define DEBUG_REQUEST_HTB              8
+#define DEBUG_REQUEST_APPHINT          9
+#define DEBUG_REQUEST_FALLBACKSYNC     10
+
+#define DEBUG_REQUEST_VERBOSITY_LOW    0
+#define DEBUG_REQUEST_VERBOSITY_MEDIUM 1
+#define DEBUG_REQUEST_VERBOSITY_HIGH   2
+#define DEBUG_REQUEST_VERBOSITY_MAX    DEBUG_REQUEST_VERBOSITY_HIGH
+
+#define DD_VERB_LVL_ENABLED(_verbLvl, _verbLvlChk) ((_verbLvl) >= (_verbLvlChk))
+
+#ifndef DBGNOTIFY_PFNS
+typedef void (DUMPDEBUG_PRINTF_FUNC)(void *pvDumpDebugFile,
+	const char *fmt, ...) __printf(2, 3);
+typedef void (*PFN_DBGREQ_NOTIFY) (void *hDebugRequestHandle,
+	__u32 ui32VerbLevel,
+	DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	void *pvDumpDebugFile);
+#define DBGNOTIFY_PFNS
+#endif
+enum PVRSRV_ERROR_TAG PVRSRVRegisterDeviceDbgRequestNotify(void **phNotify,
+	struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	__u32 ui32RequesterID,
+	void *hDbgRequestHandle);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterDeviceDbgRequestNotify(void *hNotify);
+enum PVRSRV_ERROR_TAG PVRSRVRegisterDriverDbgRequestNotify(void **phNotify,
+	PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	__u32 ui32RequesterID,
+	void *hDbgRequestHandle);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterDriverDbgRequestNotify(void *hNotify);
+
+/* physmem_dmabuf.h */
+
+struct dma_buf *PhysmemGetDmaBuf(struct _PMR_ *psPMR);
+
+struct dma_resv *PhysmemGetDmaResv(struct _PMR_ *psPMR);
+
+/* pvrsrv.h */
+
+enum PVRSRV_ERROR_TAG PVRSRVAcquireGlobalEventObjectKM(void **phGlobalEventObject);
+enum PVRSRV_ERROR_TAG PVRSRVReleaseGlobalEventObjectKM(void *hGlobalEventObject);
+
+/* sync.h */
+
+enum PVRSRV_ERROR_TAG SyncPrimContextCreate(
+	struct _PVRSRV_DEVICE_NODE_ *psDevConnection,
+	struct SYNC_PRIM_CONTEXT_TAG **phSyncPrimContext);
+void SyncPrimContextDestroy(struct SYNC_PRIM_CONTEXT_TAG *hSyncPrimContext);
+
+enum PVRSRV_ERROR_TAG SyncPrimAlloc(struct SYNC_PRIM_CONTEXT_TAG *hSyncPrimContext,
+	struct PVRSRV_CLIENT_SYNC_PRIM_TAG **ppsSync, const char *pszClassName);
+enum PVRSRV_ERROR_TAG SyncPrimFree(struct PVRSRV_CLIENT_SYNC_PRIM_TAG *psSync);
+enum PVRSRV_ERROR_TAG SyncPrimGetFirmwareAddr(
+	struct PVRSRV_CLIENT_SYNC_PRIM_TAG *psSync,
+	__u32 *sync_addr);
+
+/* osfunc.h */
+enum PVRSRV_ERROR_TAG OSEventObjectWait(void *hOSEventKM);
+enum PVRSRV_ERROR_TAG OSEventObjectOpen(void *hEventObject, void **phOSEventKM);
+enum PVRSRV_ERROR_TAG OSEventObjectClose(void *hOSEventKM);
+__u32 OSGetCurrentClientProcessIDKM(void);
+__u32 OSStringUINT32ToStr(char *pszBuf, size_t uSize, __u32 ui32Num);
+
+/* srvkm.h */
+
+enum PVRSRV_ERROR_TAG PVRSRVCommonDeviceCreate(void *pvOSDevice,
+	int i32KernelDeviceID,
+	struct _PVRSRV_DEVICE_NODE_ **ppsDeviceNode);
+void PVRSRVCommonDeviceDestroy(
+	struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+const char *PVRSRVGetErrorString(enum PVRSRV_ERROR_TAG eError);
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+enum PVRSRV_ERROR_TAG PVRSRVCommonDeviceInitialise(
+	struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN)(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext, PVRSRV_FENCE fence, u32 *nr_checkpoints, PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid);
+
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN)(PVRSRV_FENCE export_fence, PSYNC_CHECKPOINT_CONTEXT checkpoint_context, PSYNC_CHECKPOINT *checkpoint_handle);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN)(PVRSRV_FENCE export_fence);
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN)(
+		struct _PVRSRV_DEVICE_NODE_ *device,
+		const char *fence_name,
+		PVRSRV_TIMELINE timeline,
+		PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		PVRSRV_FENCE *new_fence,
+		u64 *fence_uid,
+		void **fence_finalise_data,
+		PSYNC_CHECKPOINT *new_checkpoint_handle,
+		void **timeline_update_sync,
+		__u32 *timeline_update_value);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN)(PVRSRV_FENCE fence_to_rollback, void *finalise_data);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN)(PVRSRV_FENCE fence_to_finalise, void *finalise_data);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef __u32 (*PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN)(__u32 num_ufos, __u32 *vaddrs);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef bool (*PFN_SYNC_CHECKPOINT_UFO_HAS_SIGNALLED_FN)(
+	__u32 ui32FwAddr, __u32 ui32Value);
+typedef enum PVRSRV_ERROR_TAG (*PFN_SYNC_CHECKPOINT_SIGNAL_WAITERS_FN)(void);
+typedef void(*PFN_SYNC_CHECKPOINT_CHECK_STATE_FN)(void);
+#if defined(PDUMP)
+typedef PVRSRV_ERROR(*PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN)(PVRSRV_FENCE iFence,
+						IMG_UINT32 *puiNumCheckpoints,
+		                                PSYNC_CHECKPOINT **papsCheckpoints);
+#endif
+#endif
+
+/* This is the function that kick code will call in a NO_HARDWARE build only after
+ * sync checkpoints have been manually signalled, to allow the OS native sync
+ * implementation to update its timelines (as the usual callback notification
+ * of signalled checkpoints is not supported for NO_HARDWARE).
+ */
+#ifndef CHECKPOINT_PFNS
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN)(void *private_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN)(PVRSRV_FENCE fence_to_signal);
+typedef void (*PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN)(void *mem_ptr);
+
+#define SYNC_CHECKPOINT_IMPL_MAX_STRLEN 20
+
+typedef struct {
+	PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN pfnFenceResolve;
+	PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN pfnFenceCreate;
+	PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN pfnFenceDataRollback;
+	PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN pfnFenceFinalise;
+	PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN pfnNoHWUpdateTimelines;
+	PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN pfnNoHWSignalExpFence;
+	PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN pfnFreeCheckpointListMem;
+	PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN pfnDumpInfoOnStalledUFOs;
+	char pszImplName[SYNC_CHECKPOINT_IMPL_MAX_STRLEN];
+#if defined(PDUMP)
+	PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN pfnSyncFenceGetCheckpoints;
+#endif
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN pfnExportFenceResolve;
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN pfnExportFenceRollback;
+} PFN_SYNC_CHECKPOINT_STRUCT;
+
+enum PVRSRV_ERROR_TAG SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns);
+
+#define CHECKPOINT_PFNS
+#endif
+
+/* sync_checkpoint.h */
+enum PVRSRV_ERROR_TAG SyncCheckpointContextCreate(struct _PVRSRV_DEVICE_NODE_ *psDevConnection, PSYNC_CHECKPOINT_CONTEXT *phSyncCheckpointContext);
+enum PVRSRV_ERROR_TAG SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext);
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext);
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext);
+enum PVRSRV_ERROR_TAG SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext, PVRSRV_TIMELINE timeline, PVRSRV_FENCE fence, const char *pszCheckpointName, PSYNC_CHECKPOINT *ppsSyncCheckpoint);
+void SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+void SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+bool SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+bool SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+enum PVRSRV_ERROR_TAG SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+enum PVRSRV_ERROR_TAG SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+void SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+void SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+PVRSRV_TIMELINE SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint);
+const char *SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint);
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+struct _PVRSRV_DEVICE_NODE_ *SyncCheckpointGetAssociatedDevice(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+#endif
+IMG_BOOL SyncCheckpointCommonDeviceIDs(PSYNC_CHECKPOINT_CONTEXT psSyncContext, IMG_HANDLE hDevRef);
+enum PVRSRV_ERROR_TAG SyncCheckpointGetCounters(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 *puiInUse, IMG_UINT32 *puiMax, IMG_UINT32 *puiXDInUse, IMG_UINT32*puiXDMax);
+
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+/*************************************************************************/ /*!
+@Function       NativeSyncGetFenceStatusWq
+@Description    Called to get the Foreign Fence status workqueue used in
+                Fence sync and Buffer sync.
+@Return         struct workqueue_struct ptr on success, NULL otherwise.
+*/ /**************************************************************************/
+struct workqueue_struct *NativeSyncGetFenceStatusWq(void);
+/*************************************************************************/ /*!
+@Function       NativeSyncGetFenceCtxDestroyWq
+@Return         struct workqueue_struct ptr on success, NULL otherwise.
+*/ /**************************************************************************/
+struct workqueue_struct *NativeSyncGetFenceCtxDestroyWq(void);
+#endif
+
+#endif /* __SERVICES_KERNEL_CLIENT__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.c
new file mode 100644
index 000000000000..9fdd8598f2ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.c
@@ -0,0 +1,269 @@
+/*************************************************************************/ /*!
+@Title          Linux trace event helper functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/sched.h>
+
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+#if !defined(CONFIG_TRACE_GPU_MEM)
+#define CREATE_TRACE_POINTS
+#include <trace/events/gpu_mem.h>
+#undef CREATE_TRACE_POINTS
+#else /* !defined(CONFIG_TRACE_GPU_MEM) */
+#include <trace/events/gpu_mem.h>
+#endif /* !defined(CONFIG_TRACE_GPU_MEM) */
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+
+#include "img_types.h"
+#include "trace_events.h"
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#include "rogue_trace_events.h"
+#endif
+#include "sync_checkpoint_external.h"
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+static bool fence_update_event_enabled, fence_check_event_enabled;
+
+bool trace_rogue_are_fence_updates_traced(void)
+{
+	return fence_update_event_enabled;
+}
+
+bool trace_rogue_are_fence_checks_traced(void)
+{
+	return fence_check_event_enabled;
+}
+
+/*
+ * Call backs referenced from rogue_trace_events.h. Note that these are not
+ * thread-safe, however, since running trace code when tracing is not enabled is
+ * simply a no-op, there is no harm in it.
+ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_update_enabled_callback(void)
+#else
+void trace_fence_update_enabled_callback(void)
+#endif
+{
+	fence_update_event_enabled = true;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return 0;
+#endif
+}
+
+void trace_fence_update_disabled_callback(void)
+{
+	fence_update_event_enabled = false;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_check_enabled_callback(void)
+#else
+void trace_fence_check_enabled_callback(void)
+#endif
+{
+	fence_check_event_enabled = true;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return 0;
+#endif
+}
+
+void trace_fence_check_disabled_callback(void)
+{
+	fence_check_event_enabled = false;
+}
+
+#if defined(SUPPORT_RGX)
+/* This is a helper that calls trace_rogue_fence_update for each fence in an
+ * array.
+ */
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values)
+{
+	IMG_UINT i;
+	for (i = 0; i < uCount; i++)
+	{
+		trace_rogue_fence_update(current->comm, cmd, dm, ui32GpuId, ui32FWContext, ui32Offset,
+								 pauiAddresses[i].ui32Addr, PVRSRV_SYNC_CHECKPOINT_SIGNALLED);
+	}
+}
+
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values)
+{
+	IMG_UINT i;
+	for (i = 0; i < uCount; i++)
+	{
+		trace_rogue_fence_check(current->comm, cmd, dm, ui32GpuId, ui32FWContext, ui32Offset,
+							  pauiAddresses[i].ui32Addr, PVRSRV_SYNC_CHECKPOINT_SIGNALLED);
+	}
+}
+
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		trace_rogue_ufo_update(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+				ui32IntJobRef,
+				ui32ExtJobRef,
+				puData->sUpdate.ui32FWAddr,
+				puData->sUpdate.ui32OldValue,
+				puData->sUpdate.ui32NewValue);
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sUpdate));
+	}
+}
+
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		if (bPrEvent)
+		{
+			trace_rogue_ufo_pr_check_success(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckSuccess.ui32FWAddr,
+					puData->sCheckSuccess.ui32Value);
+		}
+		else
+		{
+			trace_rogue_ufo_check_success(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckSuccess.ui32FWAddr,
+					puData->sCheckSuccess.ui32Value);
+		}
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sCheckSuccess));
+	}
+}
+
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		if (bPrEvent)
+		{
+			trace_rogue_ufo_pr_check_fail(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckFail.ui32FWAddr,
+					puData->sCheckFail.ui32Value,
+					puData->sCheckFail.ui32Required);
+		}
+		else
+		{
+			trace_rogue_ufo_check_fail(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckFail.ui32FWAddr,
+					puData->sCheckFail.ui32Value,
+					puData->sCheckFail.ui32Required);
+		}
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sCheckFail));
+	}
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+
+int PVRGpuTraceEnableUfoCallbackWrapper(void)
+{
+	PVRGpuTraceEnableUfoCallback();
+	return 0;
+}
+
+int PVRGpuTraceEnableFirmwareActivityCallbackWrapper(void)
+{
+	PVRGpuTraceEnableFirmwareActivityCallback();
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+#endif /* defined(SUPPORT_RGX) */
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size)
+{
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+	trace_gpu_mem_total(ui8GPUId, 0, ui64Size);
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+}
+
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size)
+{
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+	trace_gpu_mem_total(ui8GPUId, ui32Pid, ui64Size);
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.h
new file mode 100644
index 000000000000..5a4d0567688b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.h
@@ -0,0 +1,196 @@
+/*************************************************************************/ /*!
+@Title          Linux trace events and event helper functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TRACE_EVENTS_H)
+#define TRACE_EVENTS_H
+
+#include "rgx_fwif_km.h"
+#include "rgx_hwperf.h"
+
+/* We need to make these functions do nothing if CONFIG_EVENT_TRACING isn't
+ * enabled, just like the actual trace event functions that the kernel
+ * defines for us.
+ */
+
+#if defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+bool trace_rogue_are_fence_checks_traced(void);
+
+bool trace_rogue_are_fence_updates_traced(void);
+
+#if defined(SUPPORT_RGX)
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values);
+
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values);
+
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+#endif /* if defined(SUPPORT_RGX) */
+#else /* defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+static inline
+bool trace_rogue_are_fence_checks_traced(void)
+{
+	return false;
+}
+
+static inline
+bool trace_rogue_are_fence_updates_traced(void)
+{
+	return false;
+}
+
+#if defined(SUPPORT_RGX)
+static inline
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values)
+{
+}
+
+static inline
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values)
+{
+}
+
+static inline
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+
+static inline
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+
+static inline
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+#endif /* if defined(SUPPORT_RGX)*/
+#endif /* defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(CONFIG_EVENT_TRACING)
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size);
+
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size);
+#else
+static inline
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size)
+{
+}
+
+static inline
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size)
+{
+}
+#endif /* defined(CONFIG_EVENT_TRACING) */
+
+#endif /* TRACE_EVENTS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/cache_km.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/cache_km.h
new file mode 100644
index 000000000000..282ff5bc5f0f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/cache_km.h
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File           cache_km.h
+@Title          CPU cache management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef CACHE_KM_H
+#define CACHE_KM_H
+
+#if defined(__linux__)
+#include <linux/version.h>
+#else
+#define KERNEL_VERSION
+#endif
+
+#include "pvrsrv_error.h"
+#include "os_cpu_cache.h"
+#include "img_types.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "pmr.h"
+
+typedef IMG_UINT32 PVRSRV_CACHE_OP_ADDR_TYPE;	/*!< Represents CPU address type required for CPU d-cache maintenance */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL	0x1	/*!< Operation requires CPU virtual address only */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_PHYSICAL	0x2	/*!< Operation requires CPU physical address only */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_BOTH		0x3	/*!< Operation requires both CPU virtual & physical addresses */
+
+#include "connection_server.h"
+
+/*
+ * CacheOpInit() & CacheOpDeInit()
+ *
+ * This must be called to initialise the KM cache maintenance framework.
+ * This is called early during the driver/module (un)loading phase.
+ */
+PVRSRV_ERROR CacheOpInit(void);
+void CacheOpDeInit(void);
+
+/*
+ * CacheOpInit2() & CacheOpDeInit2()
+ *
+ * This must be called to initialise the UM cache maintenance framework.
+ * This is called when the driver is loaded/unloaded from the kernel.
+ */
+PVRSRV_ERROR CacheOpInit2(void);
+void CacheOpDeInit2(void);
+
+/*
+ * CacheOpExec()
+ *
+ * This is the primary CPU data-cache maintenance interface and it is
+ * always guaranteed to be synchronous; the arguments supplied must be
+ * pre-validated for performance reasons else the d-cache maintenance
+ * operation might cause the underlying OS kernel to fault.
+ */
+PVRSRV_ERROR CacheOpExec(PPVRSRV_DEVICE_NODE psDevNode,
+						void *pvVirtStart,
+						void *pvVirtEnd,
+						IMG_CPU_PHYADDR sCPUPhysStart,
+						IMG_CPU_PHYADDR sCPUPhysEnd,
+						PVRSRV_CACHE_OP uiCacheOp);
+
+/*
+ * CacheOpValExec()
+ *
+ * Same as CacheOpExec(), except arguments are _Validated_ before being
+ * presented to the underlying OS kernel for CPU data-cache maintenance.
+ * The uiAddress is the start CPU virtual address for the to-be d-cache
+ * maintained PMR, it can be NULL in which case a remap will be performed
+ * internally, if required for cache maintenance. This is primarily used
+ * as the services client bridge call handler for synchronous user-mode
+ * cache maintenance requests.
+ */
+PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
+							IMG_UINT64 uiAddress,
+							IMG_DEVMEM_OFFSET_T uiOffset,
+							IMG_DEVMEM_SIZE_T uiSize,
+							PVRSRV_CACHE_OP uiCacheOp);
+
+/*
+ * CacheOpQueue()
+ *
+ * This is the secondary cache maintenance interface and it is not
+ * guaranteed to be synchronous in that requests could be deferred
+ * and executed asynchronously. This interface is primarily meant
+ * as services client bridge call handler. Both uiInfoPgGFSeqNum
+ * and ui32[Current,Next]FenceSeqNum implements an internal client
+ * server queueing protocol so making use of this interface outside
+ * of services client is not recommended and should not be done.
+ */
+PVRSRV_ERROR CacheOpQueue(CONNECTION_DATA *psConnection,
+						PPVRSRV_DEVICE_NODE psDevNode,
+						IMG_UINT32 ui32OpCount,
+						PMR **ppsPMR,
+						IMG_UINT64 *puiAddress,
+						IMG_DEVMEM_OFFSET_T *puiOffset,
+						IMG_DEVMEM_SIZE_T *puiSize,
+						PVRSRV_CACHE_OP *puiCacheOp,
+						IMG_UINT32 ui32OpTimeline);
+
+/*
+ * CacheOpLog()
+ *
+ * This is used for logging client cache maintenance operations that
+ * was executed in user-space.
+ */
+PVRSRV_ERROR CacheOpLog(PMR *psPMR,
+						IMG_UINT64 uiAddress,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT64 ui64StartTime,
+						IMG_UINT64 ui64EndTime,
+						PVRSRV_CACHE_OP uiCacheOp);
+
+#endif	/* CACHE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/connection_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/connection_server.h
new file mode 100644
index 000000000000..673c37c02a3c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/connection_server.h
@@ -0,0 +1,143 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    API for server side connection management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(CONNECTION_SERVER_H)
+#define CONNECTION_SERVER_H
+
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "handle.h"
+#include "pvrsrv_cleanup.h"
+
+/* Variable used to hold in memory the timeout for the current time slice*/
+extern IMG_UINT64 gui64TimesliceLimit;
+/* Counter number of handle data freed during the current time slice */
+extern IMG_UINT32 gui32HandleDataFreeCounter;
+/* Set the maximum time the freeing of the resources can keep the lock */
+#define CONNECTION_DEFERRED_CLEANUP_TIMESLICE_NS (3000 * 1000) /* 3ms */
+
+typedef struct _CONNECTION_DATA_
+{
+	PVRSRV_HANDLE_BASE		*psHandleBase;
+	PROCESS_HANDLE_BASE		*psProcessHandleBase;
+	struct _SYNC_CONNECTION_DATA_	*psSyncConnectionData;
+	struct _PDUMP_CONNECTION_DATA_	*psPDumpConnectionData;
+
+	/* Holds the client flags supplied at connection time */
+	IMG_UINT32          ui32ClientFlags;
+
+	/*
+	 * OS specific data can be stored via this handle.
+	 * See osconnection_server.h for a generic mechanism
+	 * for initialising this field.
+	 */
+	IMG_HANDLE          hOsPrivateData;
+
+#define PVRSRV_CONNECTION_PROCESS_NAME_LEN (16)
+	IMG_PID             pid;
+	IMG_PID				vpid;
+	IMG_UINT32			tid;
+	IMG_CHAR            pszProcName[PVRSRV_CONNECTION_PROCESS_NAME_LEN];
+
+	IMG_HANDLE          hProcessStats;
+
+	IMG_HANDLE          hClientTLStream;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/*
+	 * Connection-based values per application which can be modified by the
+	 * AppHint settings 'OSid, OSidReg, bOSidAxiProtReg' for each application.
+	 * These control where the connection's memory allocation is sourced from.
+	 * ui32OSid, ui32OSidReg range from 0..(GPUVIRT_VALIDATION_NUM_OS - 1).
+	 */
+	IMG_UINT32          ui32OSid;
+	IMG_UINT32          ui32OSidReg;
+	IMG_BOOL            bOSidAxiProtReg;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	IMG_BOOL            bAcceptDmaRequests;
+	ATOMIC_T            ui32NumDmaTransfersInFlight;
+	IMG_HANDLE          hDmaEventObject;
+#endif
+	/* Structure which is hooked into the cleanup thread work list */
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+
+	DLLIST_NODE         sConnectionListNode;
+
+	/* List navigation for deferred freeing of connection data */
+	struct _CONNECTION_DATA_	**ppsThis;
+	struct _CONNECTION_DATA_	*psNext;
+} CONNECTION_DATA;
+
+#include "osconnection_server.h"
+
+PVRSRV_ERROR PVRSRVCommonConnectionConnect(void **ppvPrivData, void *pvOSData);
+void PVRSRVCommonConnectionDisconnect(void *pvPrivData);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVGetPurgeConnectionPid
+
+@Description    Returns PID associated with Connection currently being purged by
+                Cleanup Thread. If no Connection is purged 0 is returned.
+
+@Return         PID associated with currently purged connection or 0 if no
+                connection is being purged
+*/ /***************************************************************************/
+IMG_PID PVRSRVGetPurgeConnectionPid(void);
+
+void PVRSRVConnectionDebugNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile);
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVConnectionPrivateData)
+#endif
+static INLINE
+IMG_HANDLE PVRSRVConnectionPrivateData(CONNECTION_DATA *psConnection)
+{
+	return (psConnection != NULL) ? psConnection->hOsPrivateData : NULL;
+}
+
+#endif /* !defined(CONNECTION_SERVER_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/debug_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/debug_common.h
new file mode 100644
index 000000000000..aba576d273a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/debug_common.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common debug definitions and functions.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEBUG_COMMON_H
+#define DEBUG_COMMON_H
+
+#include "pvrsrv_error.h"
+#include "device.h"
+
+PVRSRV_ERROR DebugCommonInitDriver(void);
+void DebugCommonDeInitDriver(void);
+
+const IMG_CHAR *PVRSRVGetDebugDevStateString(PVRSRV_DEVICE_STATE eDevState);
+const IMG_CHAR *PVRSRVGetDebugHealthStatusString(PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus);
+const IMG_CHAR *PVRSRVGetDebugHealthReasonString(PVRSRV_DEVICE_HEALTH_REASON eHealthReason);
+
+PVRSRV_ERROR DebugCommonInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+void DebugCommonDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+typedef struct _IMG_FLAGS2DESC_
+{
+	IMG_UINT32		uiFlag;
+	const IMG_CHAR	*pszLabel;
+} IMG_FLAGS2DESC;
+
+void DebugCommonFlagStrings(IMG_CHAR *psDesc,
+							IMG_UINT32 ui32DescSize,
+							const IMG_FLAGS2DESC *psConvTable,
+							IMG_UINT32 ui32TableSize,
+							IMG_UINT32 ui32Flags);
+
+#endif /* DEBUG_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/device.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/device.h
new file mode 100644
index 000000000000..e301ef1738e8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/device.h
@@ -0,0 +1,681 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common Device header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device related function templates and defines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DEVICE_H
+#define DEVICE_H
+
+#include "devicemem_heapcfg.h"
+#include "mmu_common.h"
+#include "ra.h"			/* RA_ARENA */
+#include "pvrsrv_device.h"
+#include "sync_checkpoint.h"
+#include "srvkm.h"
+#include "physheap.h"
+#include "sync_internal.h"
+#include "dllist.h"
+
+#include "rgx_bvnc_defs_km.h"
+
+#include "lock.h"
+
+#include "power.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+typedef struct _PVRSRV_POWER_DEV_TAG_ *PPVRSRV_POWER_DEV;
+
+struct SYNC_RECORD;
+
+struct _CONNECTION_DATA_;
+
+/*************************************************************************/ /*!
+ @Function      AllocUFOBlockCallback
+ @Description   Device specific callback for allocation of a UFO block
+
+ @Input         psDeviceNode          Pointer to device node to allocate
+                                      the UFO for.
+ @Input         ui32RequestedSize     Minimum size of allocation requested
+ @Output        ppsMemDesc            Pointer to pointer for the memdesc of
+                                      the allocation
+ @Output        pui32SyncAddr         FW Base address of the UFO block
+ @Output        puiSyncPrimBlockSize  Size of the UFO block
+
+ @Return        PVRSRV_OK if allocation was successful
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*AllocUFOBlockCallback)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+											  IMG_UINT32 ui32RequestedSize,
+											  DEVMEM_MEMDESC **ppsMemDesc,
+											  IMG_UINT32 *pui32SyncAddr,
+											  IMG_UINT32 *puiSyncPrimBlockSize);
+
+/*************************************************************************/ /*!
+ @Function      FreeUFOBlockCallback
+ @Description   Device specific callback for freeing of a UFO
+
+ @Input         psDeviceNode    Pointer to device node that the UFO block was
+                                allocated from.
+ @Input         psMemDesc       Pointer to pointer for the memdesc of the UFO
+                                block to free.
+*/ /**************************************************************************/
+typedef void (*FreeUFOBlockCallback)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+									 DEVMEM_MEMDESC *psMemDesc);
+
+typedef struct _PVRSRV_DEVICE_IDENTIFIER_
+{
+	/* Pdump memory and register bank names */
+	IMG_CHAR				*pszPDumpDevName;
+	IMG_CHAR				*pszPDumpRegName;
+
+	/* Under Linux, this is the minor number of RenderNode corresponding to this Device */
+	IMG_INT32				i32KernelDeviceID;
+	/* Services layer enumeration of the device used in pvrdebug */
+	IMG_UINT32				ui32InternalID;
+} PVRSRV_DEVICE_IDENTIFIER;
+
+typedef struct _DEVICE_MEMORY_INFO_
+{
+	/* Heap count. Doesn't include additional heaps from PVRSRVCreateDeviceMemHeap */
+	IMG_UINT32				ui32HeapCount;
+
+	/* Blueprints for creating new device memory contexts */
+	IMG_UINT32              uiNumHeapConfigs;
+	DEVMEM_HEAP_CONFIG      *psDeviceMemoryHeapConfigArray;
+	DEVMEM_HEAP_BLUEPRINT   *psDeviceMemoryHeap;
+} DEVICE_MEMORY_INFO;
+
+#define MMU_BAD_PHYS_ADDR (0xbadbad00badULL)
+
+typedef struct __DEFAULT_PAGE__
+{
+	/*Page handle for the page allocated (UMA/LMA)*/
+	PG_HANDLE	sPageHandle;
+	POS_LOCK	psPgLock;
+	/*Default page size in terms of log2 */
+	IMG_UINT32	ui32Log2PgSize;
+	IMG_UINT64	ui64PgPhysAddr;
+#if defined(PDUMP)
+	IMG_HANDLE hPdumpPg;
+#endif
+} PVRSRV_DEF_PAGE;
+
+#define PVRSRV_DEVICE_STATE_LIST \
+	X(UNDEFINED)                 \
+	X(CREATING)                  \
+	X(CREATED)                   \
+	X(ACTIVE)                    \
+	X(FROZEN)                    \
+	X(DEINIT)                    \
+	X(DESTRUCTING)               \
+	X(BAD)                       \
+	X(PCI_ERROR)                 \
+	X(LAST)                      \
+
+typedef enum _PVRSRV_DEVICE_STATE_
+{
+#define X(_name) PVRSRV_DEVICE_STATE_ ## _name,
+	PVRSRV_DEVICE_STATE_LIST
+#undef X
+
+} PVRSRV_DEVICE_STATE;
+
+#define PVRSRV_DEVICE_HEALTH_STATUS_LIST \
+	X(UNDEFINED)                         \
+	X(OK)                                \
+	X(NOT_RESPONDING)                    \
+	X(DEAD)                              \
+	X(FAULT)                             \
+	X(LAST)                              \
+
+typedef enum _PVRSRV_DEVICE_HEALTH_STATUS_
+{
+#define X(_name) PVRSRV_DEVICE_HEALTH_STATUS_ ## _name,
+	PVRSRV_DEVICE_HEALTH_STATUS_LIST
+#undef X
+
+} PVRSRV_DEVICE_HEALTH_STATUS;
+
+#define PVRSRV_DEVICE_HEALTH_REASON_LIST \
+	X(NONE)                              \
+	X(ASSERTED)                          \
+	X(POLL_FAILING)                      \
+	X(TIMEOUTS)                          \
+	X(QUEUE_CORRUPT)                     \
+	X(QUEUE_STALLED)                     \
+	X(IDLING)                            \
+	X(RESTARTING)                        \
+	X(MISSING_INTERRUPTS)                \
+	X(PCI_ERROR)                         \
+	X(LAST)                              \
+
+typedef enum _PVRSRV_DEVICE_HEALTH_REASON_
+{
+#define X(_name) PVRSRV_DEVICE_HEALTH_REASON_ ## _name,
+	PVRSRV_DEVICE_HEALTH_REASON_LIST
+#undef X
+
+} PVRSRV_DEVICE_HEALTH_REASON;
+
+typedef enum _PVRSRV_DEVICE_DEBUG_DUMP_STATUS_
+{
+	PVRSRV_DEVICE_DEBUG_DUMP_NONE = 0,
+	PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE
+} PVRSRV_DEVICE_DEBUG_DUMP_STATUS;
+
+#ifndef DI_GROUP_DEFINED
+#define DI_GROUP_DEFINED
+typedef struct DI_GROUP DI_GROUP;
+#endif
+#ifndef DI_ENTRY_DEFINED
+#define DI_ENTRY_DEFINED
+typedef struct DI_ENTRY DI_ENTRY;
+#endif
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#ifndef DI_VZ_DATA_DEFINED
+#define DI_VZ_DATA_DEFINED
+typedef struct DI_VZ_DATA DI_VZ_DATA;
+#endif
+#endif
+
+typedef struct _PVRSRV_DEVICE_DEBUG_INFO_
+{
+	DI_GROUP *psGroup;
+	DI_ENTRY *psDumpDebugEntry;
+#ifdef SUPPORT_RGX
+	DI_ENTRY *psFWTraceEntry;
+#ifdef SUPPORT_FIRMWARE_GCOV
+	DI_ENTRY *psFWGCOVEntry;
+#endif
+	DI_ENTRY *psFWMappingsEntry;
+#if  defined(SUPPORT_RISCV_GDB)
+	DI_ENTRY *psRiscvDmiDIEntry;
+	IMG_UINT64 ui64RiscvDmi;
+#endif
+	DI_ENTRY *psDevMemEntry;
+	IMG_HANDLE hGpuUtilUserDebugFS;
+#endif /* SUPPORT_RGX */
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	DI_ENTRY *psPowerDataEntry;
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	DI_ENTRY *psPowerTimingStatsEntry;
+#endif
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	DI_GROUP *psVZGroup;
+	DI_GROUP *apsVZDriverGroups[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *apsVZDriverPriorityDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *apsVZDriverTimeSliceDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *psVZDriverTimeSliceIntervalDIEntry;
+	DI_ENTRY *apsVZDriverIsolationGroupDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_VZ_DATA *apsVZDriverData[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *psVZDriverConnectionCooldownPeriodDIEntry;
+#endif
+} PVRSRV_DEVICE_DEBUG_INFO;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+#define RGX_LISR_INIT							(0U)
+#define RGX_LISR_DEVICE_NOT_POWERED				(1U)
+#define RGX_LISR_NOT_TRIGGERED_BY_HW			(2U)
+#define RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED		(3U)
+#define RGX_LISR_PROCESSED						(4U)
+
+typedef IMG_UINT32 LISR_STATUS;
+
+typedef struct _LISR_EXECUTION_INFO_
+{
+	/* status of last LISR invocation */
+	LISR_STATUS ui32Status;
+
+	/* snapshot from the last LISR invocation */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	IMG_UINT32 aui32InterruptCountSnapshot[RGX_NUM_DRIVERS_SUPPORTED];
+#else
+	IMG_UINT32 aui32InterruptCountSnapshot[RGXFW_THREAD_NUM];
+#endif
+
+	/* time of the last LISR invocation */
+	IMG_UINT64 ui64Clockns;
+} LISR_EXECUTION_INFO;
+
+#if !defined(__CHECKER__)
+#define UPDATE_LISR_DBG_STATUS(status)		\
+	do { \
+		psDeviceNode->sLISRExecutionInfo.ui32Status = (status); \
+		if ((status > RGX_LISR_INIT) && (status < RGX_LISR_PROCESSED)) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: IRQ %" IMG_UINT64_FMTSPEC " rejected: %s", __func__, psDeviceNode->ui64nLISR, #status)); \
+		} \
+	} while (0)
+
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)	psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[idx] = (val)
+#define UPDATE_LISR_DBG_TIMESTAMP()			psDeviceNode->sLISRExecutionInfo.ui64Clockns = OSClockns64()
+#define UPDATE_LISR_DBG_COUNTER()			psDeviceNode->ui64nLISR++
+#define UPDATE_MISR_DBG_COUNTER()			psDeviceNode->ui64nMISR++
+#else /*!defined(__CHECKER__) */
+#define UPDATE_LISR_DBG_STATUS(status)
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)
+#define UPDATE_LISR_DBG_TIMESTAMP()
+#define UPDATE_LISR_DBG_COUNTER()
+#define UPDATE_MISR_DBG_COUNTER()
+#endif /* !defined(__CHECKER__) */
+#else /* defined(PVRSRV_DEBUG_LISR_EXECUTION) */
+#define UPDATE_LISR_DBG_STATUS(status)
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)
+#define UPDATE_LISR_DBG_TIMESTAMP()
+#define UPDATE_LISR_DBG_COUNTER()
+#define UPDATE_MISR_DBG_COUNTER()
+#endif /* defined(PVRSRV_DEBUG_LISR_EXECUTION) */
+
+typedef struct _PVRSRV_DEVICE_NODE_
+{
+	PVRSRV_DEVICE_IDENTIFIER	sDevId;
+	PVRSRV_DEVICE_STATE			eDevState; /* Set using PVRSRVDeviceSetState, not directly. */
+	PVRSRV_DEVICE_FABRIC_TYPE	eDevFabricType;
+
+	ATOMIC_T					eHealthStatus; /* Holds values from PVRSRV_DEVICE_HEALTH_STATUS */
+	ATOMIC_T					eHealthReason; /* Holds values from PVRSRV_DEVICE_HEALTH_REASON */
+	ATOMIC_T					eDebugDumpRequested; /* Holds values from PVRSRV_DEVICE_DEBUG_DUMP_STATUS */
+
+	IMG_HANDLE					*hDebugTable;
+
+	/* device specific MMU attributes */
+	MMU_DEVICEATTRIBS      *psMMUDevAttrs;
+	/* Device specific MMU firmware attributes, used only in some devices */
+	MMU_DEVICEATTRIBS      *psFirmwareMMUDevAttrs;
+	/* Physical Heap where MMU PT pages are allocated from, normally the
+	 * system's default physical heap but can be different for AutoVz driver. */
+	PHYS_HEAP              *psMMUPhysHeap;
+
+	/* lock for power state transitions */
+	POS_LOCK				hPowerLock;
+	IMG_PID                 uiPwrLockOwnerPID; /* Only valid between lock and corresponding unlock
+	                                              operations of hPowerLock */
+#if defined(DEBUG)
+	struct
+	{
+		const char        *pszFile;			    /* Power lock acquired location (File) */
+		IMG_UINT32         ui32LineNum;		    /* Power lock acquired location (Line number) */
+		IMG_UINT64         ui64Timestamp;       /* Power lock acquired timestamp */
+	} sPowerLockOwner;
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+	IMG_UINT32              uiPowerOffCounter; /* Counts how many times the device has been powered
+	                                              off. Incremented in PVRSRVSetDeviceCurrentPowerState().*/
+	IMG_UINT32              uiPowerOffCounterNext; /* Value of next update to uiPowerOffCounter. */
+#endif
+
+	/* current system device power state */
+	PVRSRV_SYS_POWER_STATE	eCurrentSysPowerState;
+	PPVRSRV_POWER_DEV	psPowerDev;
+
+	/*
+		callbacks the device must support:
+	*/
+
+	PVRSRV_ERROR (*pfnInvalFBSCTable)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+									  MMU_CONTEXT *psMMUContext,
+									  IMG_UINT64 ui64FBSCEntries);
+
+	PVRSRV_ERROR (*pfnValidateOrTweakPhysAddrs)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												MMU_DEVICEATTRIBS *psDevAttrs,
+												IMG_UINT64 *pui64Addr);
+
+	void (*pfnMMUCacheInvalidate)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+								  MMU_CONTEXT *psMMUContext,
+								  MMU_LEVEL eLevel,
+								  IMG_BOOL bUnmap);
+
+	PVRSRV_ERROR (*pfnMMUCacheInvalidateKick)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	                                          IMG_UINT32 *pui32NextMMUInvalidateUpdate);
+
+	IMG_UINT32 (*pfnMMUCacheGetInvalidateCounter)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	/* Callback pfnMMUTopLevelPxWorkarounds may be NULL if not required */
+	void (*pfnMMUTopLevelPxWorkarounds)(struct _CONNECTION_DATA_ *psConnection,
+			                            struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+			                            IMG_DEV_PHYADDR sPhysAddrL1Px,
+			                            void *pxL1PxCpuVAddr);
+
+	/* Callback pfnMMUTweakProtFlags may be NULL if not required */
+	void (*pfnMMUTweakProtFlags)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+			                     MMU_DEVICEATTRIBS *psDevAttrs,
+			                     PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+			                     MMU_PROTFLAGS_T *uiMMUProtFlags);
+
+
+	void (*pfnDumpDebugInfo)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnUpdateHealthStatus)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	                                      IMG_BOOL bIsTimerPoll);
+
+#if defined(SUPPORT_AUTOVZ)
+	void (*pfnUpdateAutoVzWatchdog)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+#endif
+
+	PVRSRV_ERROR (*pfnValidationGPUUnitsPowerChange)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32NewState);
+
+	PVRSRV_ERROR (*pfnResetHWRLogs)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnVerifyBVNC)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask);
+
+	/* Method to drain device HWPerf packets from firmware buffer to host buffer */
+	PVRSRV_ERROR (*pfnServiceHWPerf)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnDeviceVersionString)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_CHAR **ppszVersionString);
+
+	PVRSRV_ERROR (*pfnDeviceClockSpeed)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_PUINT32 pui32RGXClockSpeed);
+
+	PVRSRV_ERROR (*pfnSoftReset)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+
+	PVRSRV_ERROR (*pfnAlignmentCheck)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32FWAlignChecksSize, IMG_UINT32 aui32FWAlignChecks[]);
+	IMG_BOOL	(*pfnCheckDeviceFeature)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64FeatureMask);
+
+	IMG_INT32	(*pfnGetDeviceFeatureValue)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, enum _RGX_FEATURE_WITH_VALUE_INDEX_ eFeatureIndex);
+
+    PVRSRV_ERROR (*pfnGetMultiCoreInfo)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores, IMG_UINT64 *pui64Caps);
+
+	IMG_BOOL (*pfnHasFBCDCVersion31)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	IMG_UINT32 (*pfnGetTFBCLossyGroup)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	MMU_DEVICEATTRIBS* (*pfnGetMMUDeviceAttributes)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_BOOL bKernelMemoryCtx);
+
+	PVRSRV_DEVICE_SNOOP_MODE (*pfnGetDeviceSnoopMode)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_DEVICE_CONFIG	*psDevConfig;
+
+	/* device post-finalise compatibility check */
+	PVRSRV_ERROR			(*pfnInitDeviceCompatCheck) (struct _PVRSRV_DEVICE_NODE_*);
+
+	/* initialise device-specific physheaps */
+	PVRSRV_ERROR			(*pfnPhysMemDeviceHeapsInit) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* determining the appropriate LMA allocation policy */
+	PHYS_HEAP_POLICY		(*pfnPhysHeapGetLMAPolicy) (PHYS_HEAP_USAGE_FLAGS, struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	/* initialise fw mmu, if FW not using GPU mmu, NULL otherwise. */
+	PVRSRV_ERROR			(*pfnFwMMUInit) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* Check device's FW Main physheap free memory */
+	PVRSRV_ERROR			(*pfnCheckForSufficientFWPhysMem) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* information about the device's address space and heaps */
+	DEVICE_MEMORY_INFO		sDevMemoryInfo;
+
+	/* device's shared-virtual-memory heap max virtual address */
+	IMG_UINT64				ui64GeneralSVMHeapTopVA;
+
+	ATOMIC_T				iNumClockSpeedChanges;
+
+	/* private device information */
+	void					*pvDevice;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	RA_ARENA				*psOSSharedArena;
+	RA_ARENA				*psOSidSubArena[GPUVIRT_VALIDATION_NUM_OS];
+#endif
+
+	/* When virtualisation support is enabled the Firmware heaps of virtualised
+	 * drivers can be entirely premapped into the Fw's VA space, during init
+	 * or during runtime on explicit request from Guest drivers. */
+	PHYS_HEAP               *apsFWPremapPhysHeap[RGX_NUM_DRIVERS_SUPPORTED];
+
+	/* Head of the physical heap list. Tracks PhysHeap objects created from
+	 * the PHYS_HEAP_CONFIG definitions supplied by the system layer at
+	 * device creation time. There could be 1 or more and varies from system
+	 * to system.
+	 */
+	struct _PHYS_HEAP_		*psPhysHeapList;
+	POS_LOCK				hPhysHeapLock;
+
+	/* The apsPhysHeap array is a mapping table to the system's, often fewer,
+	 * physical memory heaps defined for this device. It contains
+	 * PVRSRV_PHYS_HEAP_LAST entries, one for each possible physical
+	 * heaps allowed in the design. Each PhysHeap in the design is acquired
+	 * and stored in the mapping table during device create. Fall-back logic
+	 * is employed to ensure a valid heap is always found from the set defined
+	 * in the system layer for the device. Responsibility for this is shared
+	 * between the common layer (PhysHeapInitDeviceHeaps) and sub-device
+	 * layer (pfnPhysMemDeviceHeapsInit).
+	 * It is used in the PhysMem module to create PMRs from a given PhysHeap
+	 * of memory. See PhysHeapCreatePMR()
+	 */
+	PHYS_HEAP				*apsPhysHeap[PVRSRV_PHYS_HEAP_LAST];
+	IMG_UINT32				ui32UserAllocHeapCount;
+
+	/* Flag indicating if the firmware has been initialised during the
+	 * 1st boot of the Host driver according to the AutoVz life-cycle. */
+	IMG_BOOL				bAutoVzFwIsUp;
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL				bAutoVzAllowGPUPowerdown;
+#endif
+
+	/* Flags indicating VM state and if PVZ calls originating from it are valid */
+	IMG_UINT32				ui32VmState;
+
+	struct _PVRSRV_DEVICE_NODE_	*psNext;
+	struct _PVRSRV_DEVICE_NODE_	**ppsThis;
+
+	/* Functions for notification about memory contexts */
+	PVRSRV_ERROR			(*pfnRegisterMemoryContext)(struct _PVRSRV_DEVICE_NODE_	*psDeviceNode,
+														MMU_CONTEXT					*psMMUContext,
+														IMG_HANDLE					*hPrivData);
+	void					(*pfnUnregisterMemoryContext)(IMG_HANDLE hPrivData);
+
+	/* Functions for allocation/freeing of UFOs */
+	AllocUFOBlockCallback	pfnAllocUFOBlock;	/*!< Callback for allocation of a block of UFO memory */
+	FreeUFOBlockCallback	pfnFreeUFOBlock;	/*!< Callback for freeing of a block of UFO memory */
+
+	IMG_HANDLE				hSyncServerRecordNotify;
+	POS_LOCK				hSyncServerRecordLock;
+	IMG_UINT32				ui32SyncServerRecordCount;
+	IMG_UINT32				ui32SyncServerRecordCountHighWatermark;
+	DLLIST_NODE				sSyncServerRecordList;
+	struct SYNC_RECORD		*apsSyncServerRecordsFreed[PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN];
+	IMG_UINT32				uiSyncServerRecordFreeIdx;
+
+	IMG_HANDLE				hSyncCheckpointRecordNotify;
+	POS_LOCK				hSyncCheckpointRecordLock;
+	IMG_UINT32				ui32SyncCheckpointRecordCount;
+	IMG_UINT32				ui32SyncCheckpointRecordCountHighWatermark;
+	DLLIST_NODE				sSyncCheckpointRecordList;
+	struct SYNC_CHECKPOINT_RECORD	*apsSyncCheckpointRecordsFreed[PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN];
+	IMG_UINT32				uiSyncCheckpointRecordFreeIdx;
+
+	IMG_HANDLE				hSyncCheckpointNotify;
+	POS_SPINLOCK			hSyncCheckpointListLock; /*!< Protects sSyncCheckpointSyncsList */
+	DLLIST_NODE				sSyncCheckpointSyncsList;
+
+	PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext;
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+
+	/* With this sync-prim we make sure the MMU cache is flushed
+	 * before we free the page table memory */
+	PVRSRV_CLIENT_SYNC_PRIM	*psMMUCacheSyncPrim;
+	IMG_UINT32				ui32NextMMUInvalidateUpdate;
+
+	IMG_HANDLE				hCmdCompNotify;
+	IMG_HANDLE				hDbgReqNotify;
+	IMG_HANDLE				hAppHintDbgReqNotify;
+	IMG_HANDLE				hPhysHeapDbgReqNotify;
+
+	/* Device MMU common module can support one other larger page size
+	 * (e.g. 16KB) in addition to the default OS page size (often 4KB).
+	 * If supported, device code will store the size here as a log2 value
+	 * during device creation/registration. A 0 value will imply it is
+	 * not supported.
+	 */
+	IMG_UINT32              ui32Non4KPageSizeLog2;
+
+	PVRSRV_DEF_PAGE			sScratchPage;
+	PVRSRV_DEF_PAGE			sDevZeroPage;
+
+	/* Lock protects access to sMemoryContextPageFaultNotifyListHead and
+	 * per memory context DEVMEMINT_CTX::sProcessNotifyListHead lists. */
+	POSWR_LOCK				hPageFaultNotifyLock;
+	DLLIST_NODE				sMemoryContextPageFaultNotifyListHead;
+
+	/* System DMA channels */
+	IMG_UINT32				ui32RefCountDMA;
+	IMG_HANDLE				hDmaTxChan;
+	IMG_HANDLE				hDmaRxChan;
+	POS_LOCK				hDmaTxLock;
+	POS_LOCK				hDmaRxLock;
+
+#if defined(PDUMP)
+	/*
+	 * FBC clear color register default value to use.
+	 */
+	IMG_UINT64				ui64FBCClearColour;
+
+	/* Device-level callback which is called when pdump.exe starts.
+	 * Should be implemented in device-specific init code, e.g. rgxinit.c
+	 */
+	PVRSRV_ERROR			(*pfnPDumpInitDevice)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+	/* device-level callback to return pdump ID associated to a memory context */
+	IMG_UINT32				(*pfnMMUGetContextID)(IMG_HANDLE hDevMemContext);
+
+	IMG_UINT8			*pui8DeferredSyncCPSignal;	/*! Deferred fence events buffer */
+
+	IMG_UINT16			ui16SyncCPReadIdx;		/*! Read index in the above deferred fence events buffer */
+
+	IMG_UINT16			ui16SyncCPWriteIdx;		/*! Write index in the above deferred fence events buffer */
+
+	POS_LOCK			hSyncCheckpointSignalLock;	/*! Guards data shared between an sleepable-contexts */
+
+	void				*pvSyncCPMISR;			/*! MISR to emit pending/deferred fence signals */
+
+	void				*hTransition;			/*!< SyncCheckpoint PdumpTransition Cookie */
+
+	DLLIST_NODE			sSyncCheckpointContextListHead;	/*!< List head for the sync chkpt contexts */
+
+	POS_LOCK			hSyncCheckpointContextListLock;	/*! lock for accessing sync chkpt contexts list */
+
+#endif
+
+
+	/* Members for linking which connections are open on this device */
+	POS_LOCK                hConnectionsLock;    /*!< Lock protecting sConnections */
+	DLLIST_NODE             sConnections;        /*!< The list of currently active connection objects for this device node */
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	LISR_EXECUTION_INFO     sLISRExecutionInfo;  /*!< Information about the last execution of the LISR */
+	IMG_UINT64              ui64nLISR;           /*!< Number of LISR calls seen */
+	IMG_UINT64              ui64nMISR;           /*!< Number of MISR calls made */
+#endif
+
+	PVRSRV_DEVICE_DEBUG_INFO sDebugInfo;
+	IMG_BOOL                bEnablePFDebug;      /*!< EnablePageFaultDebug AppHint setting for device */
+
+	DLLIST_NODE             sCleanupThreadWorkList; /*!< List of work for the cleanup thread associated with the device */
+	ATOMIC_T                i32NumCleanupItems;   /*!< Number of cleanup thread work items. Includes items being freed. */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Data for the deferred freeing of a PMR physical pages for a given device */
+	DLLIST_NODE             sPMRZombieList;       /*!< List of PMRs to free */
+	POS_LOCK                hPMRZombieListLock;   /*!< List lock */
+	IMG_UINT32              uiPMRZombieCount;     /*!< Number of elements in the list */
+	IMG_UINT32              uiPMRZombieCountInCleanup; /*!< Number of elements in cleanup items */
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+	ATOMIC_T                eFrozen;              /*< Frozen / Unfrozen indicator */
+	IMG_HANDLE              hDeviceThreadEvObj;   /*< Event Object for Freeze indicator */
+	IMG_HANDLE              hDeviceFreezeThaw;    /*< Event handle for Freeze/Thaw */
+	POS_LOCK                hFreezeThawLock;      /*< Freeze/Thaw lock */
+	ATOMIC_T                iFreezeCount;         /*< Number of blocked on frozen tasks */
+	ATOMIC_T                iTotalFreezes;        /*< Total number of times device frozen */
+	ATOMIC_T                iThreadsActive;       /*< Number of threads active on this device */
+	IMG_UINT64              ui64LastDeviceOffTimestamp; /* Last device power off timestamp */
+	IMG_UINT64              ui64LastDeviceOffHostTimestampNs; /* Last device power off host timestamp */
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	IMG_BOOL bGPUWorkPeriodFTraceEnabled;
+#endif
+} PVRSRV_DEVICE_NODE;
+
+/*
+ * Macros to be used instead of calling directly the pfns since these macros
+ * will expand the feature passed as argument into the bitmask/index to work
+ * with the macros defined in rgx_bvnc_defs_km.h
+ */
+#define PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, Feature) \
+		psDevNode->pfnCheckDeviceFeature(psDevNode, RGX_FEATURE_##Feature##_BIT_MASK)
+#define PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, Feature) \
+		psDevNode->pfnGetDeviceFeatureValue(psDevNode, RGX_FEATURE_##Feature##_IDX)
+
+PVRSRV_ERROR PVRSRVDeviceFinalise(PVRSRV_DEVICE_NODE *psDeviceNode,
+											   IMG_BOOL bInitSuccessful);
+
+PVRSRV_ERROR PVRSRVDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE * psDeviceNode, IMG_UINT32 ui32ClientBuildOptions);
+
+void PVRSRVDeviceSetState(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_STATE eNewDevState);
+
+#define PVRSRVIsStatusRecoverable(eStatus) \
+	(((eStatus == PVRSRV_DEVICE_HEALTH_STATUS_DEAD)) ? \
+	 IMG_FALSE : IMG_TRUE)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+/* Determines if a 32-bit `uiCurrent` counter advanced to or beyond
+ * `uiRequired` value. The function takes into consideration that the
+ * counter could have wrapped around. */
+static INLINE IMG_BOOL PVRSRVHasCounter32Advanced(IMG_UINT32 uiCurrent,
+                                                  IMG_UINT32 uiRequired)
+{
+	return uiCurrent >= uiRequired ?
+	    /* ... with the counter wrapped around ...
+	     * There can't be ~4 billion transactions completed, so consider wrapped */
+	    (((uiCurrent - uiRequired) > 0xF0000000UL) ? IMG_FALSE : IMG_TRUE) :
+	    /* There can't be ~4 billion transactions pending, so consider wrapped */
+	    (((uiRequired - uiCurrent) > 0xF0000000UL) ? IMG_TRUE : IMG_FALSE);
+}
+#endif
+
+#endif /* DEVICE_H */
+
+/******************************************************************************
+ End of file (device.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_heapcfg.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_heapcfg.h
new file mode 100644
index 000000000000..3b8c0aa08921
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_heapcfg.h
@@ -0,0 +1,213 @@
+/**************************************************************************/ /*!
+@File
+@Title          Device Heap Configuration Helper Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DEVICEMEMHEAPCFG_H
+#define DEVICEMEMHEAPCFG_H
+
+#include <powervr/mem_types.h>
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+struct _PVRSRV_DEVICE_NODE_;
+struct _CONNECTION_DATA_;
+struct _DEVMEMINT_HEAP_;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_HEAP_INIT
+@Description    Device heap initialisation function. Called in server devmem
+                heap create if the callback pointer in RGX_HEAP_INFO is
+                not NULL.
+@Input          psDeviceNode       The device node.
+@Input          psDevmemHeap       Server internal devmem heap.
+@Output         phPrivData         Private data handle. Allocated resources
+                                   can be freed in PFN_HEAP_DEINIT.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_HEAP_INIT)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                                      struct _DEVMEMINT_HEAP_ *psDevmemHeap,
+                                      IMG_HANDLE *phPrivData);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_HEAP_DEINIT
+@Description    Device heap deinit function. Called in server devmem
+                heap create if the callback pointer in RGX_HEAP_INFO is
+                not NULL.
+@Input          hPrivData          Private data handle. To free any resources.
+*/ /**************************************************************************/
+typedef void (*PFN_HEAP_DEINIT)(IMG_HANDLE hPrivData);
+
+/*
+  A "heap config" is a blueprint to be used for initial setting up of heaps
+  when a device memory context is created.
+
+  We define a data structure to define this, but it's really down to the
+  caller to populate it. This is all expected to be in-kernel. We provide an
+  API that client code can use to enquire about the blueprint, such that it may
+  do the heap set-up during the context creation call on behalf of the user.
+*/
+
+/* Blueprint for a single heap */
+typedef struct _DEVMEM_HEAP_BLUEPRINT_
+{
+	/* Name of this heap - for debug purposes, and perhaps for lookup
+	by name */
+	const IMG_CHAR *pszName;
+
+	/* Virtual address of the beginning of the heap.  This _must_ be a
+	multiple of the data page size for the heap.  It is
+	_recommended_ that it be coarser than that - especially, it
+	should begin on a boundary appropriate to the MMU for the
+	device.  For Rogue, this is a Page Directory boundary, or 1GB
+	(virtual address a multiple of 0x0040000000). */
+	IMG_DEV_VIRTADDR sHeapBaseAddr;
+
+	/* Length of the heap.  Given that the END address of the heap has
+	a similar restriction to that of the _beginning_ of the heap.
+	That is the heap length _must_ be a whole number of data pages.
+	Again, the recommendation is that it ends on a 1GB boundary.
+	Again, this is not essential, but we do know that (at the time
+	of writing) the current implementation of mmu_common.c is such
+	that no two heaps may share a page directory, thus the
+	remaining virtual space would be wasted if the length were not
+	a multiple of 1GB */
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+
+	/* VA space starting sHeapBaseAddr to uiReservedRegionLength-1 are reserved
+	for statically defined addresses (shared/known between clients and FW).
+	Services never maps allocations into this reserved address space _unless_
+	explicitly requested via PVRSRVMapToDeviceAddress by passing sDevVirtAddr
+	which falls within this reserved range. Since this range is completely for
+	clients to manage (where allocations are page granular), it _must_ again be
+	a whole number of data pages. Additionally, another constraint enforces this
+	to be a multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY (which evaluates to
+	max page size supported) to support varied pages sizes */
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+
+	/* Data page size.  This is the page size that is going to get
+	programmed into the MMU, so it needs to be a valid one for the
+	device.  Importantly, the start address and length _must_ be
+	multiples of this page size.  Note that the page size is
+	specified as the log 2 relative to 1 byte (e.g. 12 indicates
+	4kB) */
+	IMG_UINT32 uiLog2DataPageSize;
+
+	/* Import alignment.  Force imports to this heap to be
+	aligned to at least this value */
+	IMG_UINT32 uiLog2ImportAlignment;
+
+	/* Callback function for device specific heap init. */
+	PFN_HEAP_INIT pfnInit;
+
+	/* Callback function for device specific heap deinit. */
+	PFN_HEAP_DEINIT pfnDeInit;
+
+} DEVMEM_HEAP_BLUEPRINT;
+
+void HeapCfgBlueprintInit(const IMG_CHAR        *pszName,
+	                      IMG_UINT64             ui64HeapBaseAddr,
+	                      IMG_DEVMEM_SIZE_T      uiHeapLength,
+	                      IMG_DEVMEM_SIZE_T      uiReservedRegionLength,
+	                      IMG_UINT32             ui32Log2DataPageSize,
+	                      IMG_UINT32             uiLog2ImportAlignment,
+	                      PFN_HEAP_INIT          pfnInit,
+	                      PFN_HEAP_DEINIT        pfnDeInit,
+	                      DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint);
+
+/* Entire named heap config */
+typedef struct _DEVMEM_HEAP_CONFIG_
+{
+    /* Name of this heap config - for debug and maybe lookup */
+    const IMG_CHAR *pszName;
+
+    /* Number of heaps in this config */
+    IMG_UINT32 uiNumHeaps;
+
+    /* Array of individual heap blueprints as defined above */
+    DEVMEM_HEAP_BLUEPRINT *psHeapBlueprintArray;
+} DEVMEM_HEAP_CONFIG;
+
+
+PVRSRV_ERROR
+HeapCfgHeapConfigCount(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 *puiNumHeapConfigsOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapCount(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 *puiNumHeapsOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapConfigName(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 uiHeapConfigNameBufSz,
+    IMG_CHAR *pszHeapConfigNameOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapDetails(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 uiHeapIndex,
+    IMG_UINT32 uiHeapNameBufSz,
+    IMG_CHAR *pszHeapNameOut,
+    IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+    IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+    IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+    IMG_UINT32 *puiLog2DataPageSizeOut,
+    IMG_UINT32 *puiLog2ImportAlignmentOut
+);
+
+PVRSRV_ERROR
+HeapCfgGetCallbacks(const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    PFN_HEAP_INIT *ppfnInit,
+                    PFN_HEAP_DEINIT *ppfnDeinit);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_history_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_history_server.h
new file mode 100644
index 000000000000..81fe702371ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_history_server.h
@@ -0,0 +1,167 @@
+/*************************************************************************/ /*!
+@File           devicemem_history_server.h
+@Title          Resource Information abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Devicemem History functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_HISTORY_SERVER_H
+#define DEVICEMEM_HISTORY_SERVER_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxmem.h"
+#include "devicemem_utils.h"
+#include "connection_server.h"
+
+PVRSRV_ERROR DevicememHistoryInitKM(void);
+
+void DevicememHistoryDeInitKM(void);
+
+PVRSRV_ERROR DevicememHistoryDeviceInit(PVRSRV_DEVICE_NODE *psDevNode);
+PVRSRV_ERROR DevicememHistoryDeviceCreate(PVRSRV_DEVICE_NODE *psDevNode);
+void DevicememHistoryDeviceDestroy(PVRSRV_DEVICE_NODE *psDevNode);
+
+PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
+							PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							IMG_DEVMEM_SIZE_T uiAllocSize,
+							const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *ui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
+							PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							IMG_DEVMEM_SIZE_T uiAllocSize,
+							const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *ui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocPageCount,
+							IMG_UINT32 *paui32AllocPageIndices,
+							IMG_UINT32 ui32FreePageCount,
+							IMG_UINT32 *pauiFreePageIndices,
+							IMG_UINT32 AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+/* used when the PID does not matter */
+#define DEVICEMEM_HISTORY_PID_ANY 0xFFFFFFFE
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_IN_
+{
+	IMG_PID uiPID;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_DEVICE_NODE *psDevNode;
+} DEVICEMEM_HISTORY_QUERY_IN;
+
+/* Store up to 4 results for a lookup. In the case of the faulting page being
+ * re-mapped between the page fault occurring on HW and the page fault analysis
+ * being done, the second result entry will show the allocation being unmapped.
+ * A further 2 entries are added to cater for multiple buffers in the same page.
+ */
+#define DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS 4
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_OUT_RESULT_
+{
+	IMG_CHAR szString[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_BOOL bMap;
+	IMG_BOOL bRange;
+	IMG_BOOL bAll;
+	IMG_UINT64 ui64When;
+	IMG_UINT64 ui64Age;
+	/* info for sparse map/unmap operations (i.e. bRange=IMG_TRUE) */
+	IMG_UINT32 ui32StartPage;
+	IMG_UINT32 ui32PageCount;
+	IMG_DEV_VIRTADDR sMapStartAddr;
+	IMG_DEV_VIRTADDR sMapEndAddr;
+	RGXMEM_PROCESS_INFO sProcessInfo;
+} DEVICEMEM_HISTORY_QUERY_OUT_RESULT;
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_OUT_
+{
+	IMG_UINT32 ui32NumResults;
+	IMG_UINT64 ui64SearchCount;
+	/* result 0 is the newest */
+	DEVICEMEM_HISTORY_QUERY_OUT_RESULT sResults[DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS];
+} DEVICEMEM_HISTORY_QUERY_OUT;
+
+void DevicememHistoryDumpRecordStats(PVRSRV_DEVICE_NODE *psDevNode,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile);
+
+IMG_BOOL
+DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
+                      DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+                      IMG_UINT32 ui32PageSizeBytes,
+                      IMG_BOOL bMatchAnyAllocInPage);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server.h
new file mode 100644
index 000000000000..8e000c663970
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server.h
@@ -0,0 +1,786 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server side component for device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_SERVER_H
+#define DEVICEMEM_SERVER_H
+
+#include "device.h" /* For device node */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "connection_server.h"
+#include "pmr.h"
+
+typedef struct _DEVMEMINT_CTX_ DEVMEMINT_CTX;
+typedef struct _DEVMEMINT_CTX_EXPORT_ DEVMEMINT_CTX_EXPORT;
+typedef struct _DEVMEMINT_HEAP_ DEVMEMINT_HEAP;
+
+typedef struct _DEVMEMINT_RESERVATION_ DEVMEMINT_RESERVATION;
+typedef struct _DEVMEMXINT_RESERVATION_ DEVMEMXINT_RESERVATION;
+typedef struct _DEVMEMINT_PF_NOTIFY_ DEVMEMINT_PF_NOTIFY;
+
+/*
+ * DevmemServerGetImportHandle()
+ *
+ * For given exportable memory descriptor returns PMR handle
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+                            IMG_HANDLE *phImport);
+
+/*
+ * DevmemServerGetHeapHandle()
+ *
+ * For given reservation returns the Heap handle
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetHeapHandle(DEVMEMINT_RESERVATION *psReservation,
+                          IMG_HANDLE *phHeap);
+
+/*
+ * DevmemServerGetContext()
+ *
+ * For given heap returns the context.
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetContext(DEVMEMINT_HEAP *psDevmemHeap,
+                       DEVMEMINT_CTX **ppsDevmemCtxPtr);
+
+/*
+ * DevmemServerGetPrivData()
+ *
+ * For given context returns the private data handle.
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetPrivData(DEVMEMINT_CTX *psDevmemCtx,
+                        IMG_HANDLE *phPrivData);
+
+/*
+ * DevmemIntCtxCreate()
+ *
+ * Create a Server-side Device Memory Context. This is usually the counterpart
+ * of the client side memory context, and indeed is usually created at the
+ * same time.
+ *
+ * You must have one of these before creating any heaps.
+ *
+ * All heaps must have been destroyed before calling
+ * DevmemIntCtxDestroy()
+ *
+ * If you call DevmemIntCtxCreate() (and it succeeds) you are promising to
+ * later call DevmemIntCtxDestroy()
+ *
+ * Note that this call will cause the device MMU code to do some work for
+ * creating the device memory context, but it does not guarantee that a page
+ * catalogue will have been created, as this may be deferred until the first
+ * allocation.
+ *
+ * Caller to provide storage for a pointer to the DEVMEM_CTX object that will
+ * be created by this call.
+ */
+PVRSRV_ERROR
+DevmemIntCtxCreate(CONNECTION_DATA *psConnection,
+                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                   /* devnode / perproc etc */
+                   IMG_BOOL bKernelMemoryCtx,
+                   DEVMEMINT_CTX **ppsDevmemCtxPtr,
+                   IMG_HANDLE *hPrivData,
+                   IMG_UINT32 *pui32CPUCacheLineSize);
+/*
+ * DevmemIntCtxDestroy()
+ *
+ * Undoes a prior DevmemIntCtxCreate or DevmemIntCtxImport.
+ */
+PVRSRV_ERROR
+DevmemIntCtxDestroy(DEVMEMINT_CTX *psDevmemCtx);
+
+/*
+ * DevmemIntHeapCreate()
+ *
+ * Creates a new heap in this device memory context.  This will cause a call
+ * into the MMU code to allocate various data structures for managing this
+ * heap. It will not necessarily cause any page tables to be set up, as this
+ * can be deferred until first allocation. (i.e. we shouldn't care - it's up
+ * to the MMU code)
+ *
+ * Note that the data page size must be specified (as log 2). The data page
+ * size as specified here will be communicated to the mmu module, and thus may
+ * determine the page size configured in page directory entries for subsequent
+ * allocations from this heap. It is essential that the page size here is less
+ * than or equal to the "minimum contiguity guarantee" of any PMR that you
+ * subsequently attempt to map to this heap.
+ *
+ * If you call DevmemIntHeapCreate() (and the call succeeds) you are promising
+ * that you shall subsequently call DevmemIntHeapDestroy()
+ *
+ * Caller to provide storage for a pointer to the DEVMEM_HEAP object that will
+ * be created by this call.
+ */
+PVRSRV_ERROR
+DevmemIntHeapCreate(DEVMEMINT_CTX *psDevmemCtx,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    DEVMEMINT_HEAP **ppsDevmemHeapPtr);
+/*
+ * DevmemIntHeapDestroy()
+ *
+ * Destroys a heap previously created with DevmemIntHeapCreate()
+ *
+ * All allocations from his heap must have been freed before this
+ * call.
+ */
+PVRSRV_ERROR
+DevmemIntHeapDestroy(DEVMEMINT_HEAP *psDevmemHeap);
+
+/* DevmemIntHeapGetBaseAddr()
+ *
+ * Get heap base address pre carveouts.
+ */
+IMG_DEV_VIRTADDR
+DevmemIntHeapGetBaseAddr(DEVMEMINT_HEAP *psDevmemHeap);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntReserveRange()
+ * @Description Reserves a number of virtual addresses starting sReservationVAddr
+ *              and continuing until sReservationVAddr + uiVirtualSize - 1.
+ *
+ *              If you call DevmemIntReserveRange() (and the call succeeds)
+ *              then you are promising that you shall later call DevmemIntUnreserveRange()
+ *
+ * @Input       psConnectionData    The connection data from the bridge. Used
+ *                                  to determine where the call to this function
+ *                                  originated from.
+ * @Input       psDeviceNode        The device node (unused).
+ * @Input       psDevmemHeap        The virtual heap the DevVAddr is within.
+ * @Input       sReservationVAddr   The first virtual address of the range.
+ * @Input       uiVirtualSize       The number of bytes in the virtual range.
+ * @Input       uiFlags             Mem alloc flags
+ * @Output      ppsReservationPtr   A pointer to the created reservation.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntReserveRange(CONNECTION_DATA *psConnectionData,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      DEVMEMINT_HEAP *psDevmemHeap,
+                      IMG_DEV_VIRTADDR sReservationVAddr,
+                      IMG_DEVMEM_SIZE_T uiVirtualSize,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      DEVMEMINT_RESERVATION **ppsReservationPtr);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntUnreserveRange()
+ * @Description Unreserves the specified virtual range. In the case that the
+ *              virtual range has not been unmapped, it will be unmapped.
+ *              If any references are held on the reservation PVRSRV_ERROR_RETRY
+ *              will be returned.
+ *
+ * @Input       psDevmemReservation The reservation to unreserve
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntUnreserveRange(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntMapPMR
+ *
+ * @Description Maps the given PMR to the virtual range previously reserved with
+ *              DevmemIntReserveRange(). When calling this function, the reservation
+ *              must be valid, and not mapped. Additionally, the PMRs logical
+ *              size and the reservations virtual size must be equal.
+ *
+ *              If appropriate, the PMR must have had its physical backing
+ *              committed, as this call will call into the MMU code to set
+ *              up the page tables for this allocation, which shall in turn
+ *              request the physical addresses from the PMR. Alternatively,
+ *              the PMR implementation can choose to do so off the back of
+ *              the "lock" callback, which it will receive as a result
+ *              (indirectly) of this call.
+ *
+ *              If you call DevmemIntMapPMR() (and the call succeeds) then you
+ *              are promising that you shall later call DevmemIntUnmapPMR()
+ *
+ * @Input       psReservation    The reservation the PMR will be mapped into.
+ * @Input       psPMR            The PMR to be mapped.
+ *
+ * @Return      PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntMapPMR(DEVMEMINT_RESERVATION *psReservation, PMR *psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntRemapPageInPMR
+ *
+ * @Description Distributes calls to the MMU module to remap a given PMR
+ *              page offset into all associated mappings.
+ *
+ * @Input       psPMR                The PMR to be mapped.
+ * @Input       psMappingListHead    The mapping node list head where nodes are
+ *                                   associated with the PMR via calls to
+ *                                   PMRLinkGPUMapping.
+ *                                   Expected type:
+ *                                   DLLIST_NODE list head from the PMR
+ *                                   (sGpuMappingListHead)
+ * @Input       ui32LogicalPgOffset  The logical page offset into the
+ *                                   PMR and reservation.
+ *
+ * @Return      PVRSRV_ERROR failure code.
+ *              PVRSRV_ERROR_DEVICEMEM_REJECT_REMAP_REQUEST can be returned
+ *              if remap is not possible on the given page offset.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntRemapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntUnmapPMR()
+ *
+ * @Description Unmaps a previously mapped virtual range.
+ *
+ * @Input       psReservation   The virtual range to unmap.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntUnmapPMR(DEVMEMINT_RESERVATION *psReservation);
+
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntReserveRangeAndMapPMR()
+ *
+ * @Description Reserve (with DevmemIntReserveRange), and map a virtual range
+ *              to a PMR (with DevmemIntMapPMR).
+ *
+ * @Input       psConnectionData    The connection data from the bridge. Used
+ *                                  to determine where the call to this function
+ *                                  originated from.
+ * @Input       psDeviceNode        The device node.
+ * @Input       psDevmemHeap        The virtual heap DevVAddr is within.
+ * @Input       sReservationVAddr   The first virtual address of the range.
+ * @Input       uiVirtualSize       The number of bytes in the virtual range.
+ * @Input       psPMR               The PMR to be mapped.
+ * @Input       uiFlags             Mem alloc flags
+ * @Output      ppsReservation      A pointer to the created reservation.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntReserveRangeAndMapPMR(CONNECTION_DATA *psConnectionData,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               DEVMEMINT_HEAP *psDevmemHeap,
+                               IMG_DEV_VIRTADDR sReservationVAddr,
+                               IMG_DEVMEM_SIZE_T uiVirtualSize,
+                               PMR *psPMR,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               DEVMEMINT_RESERVATION **ppsReservation);
+
+/*************************************************************************/ /*!
+ * @Function       DevmemIntChangeSparse
+ * @Description    Changes the sparse allocations of a PMR by allocating and freeing
+ *                 pages and changing their corresponding GPU mapping.
+ *
+ *                 Prior to calling this function DevmemIntMapPMR
+ *                 or DevmemIntReserveRangeAndMapPMR must be used.
+ *
+ * @Input          psReservation         The reservation that the PMR is mapped to.
+ * @Input          ui32AllocPageCount    Number of pages to allocate
+ * @Input          pai32AllocIndices     The logical PMR indices where pages will
+ *                                       be allocated. May be NULL.
+ * @Input          ui32FreePageCount     Number of pages to free
+ * @Input          pai32FreeIndices      The logical PMR indices where pages will
+ *                                       be freed. May be NULL.
+ * @Input          uiSparseFlags         Flags passed in to determine which kind
+ *                                       of sparse change the user wanted.
+ *                                       See devicemem_typedefs.h for details.
+ * @Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntChangeSparse(IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiSparseFlags,
+                      DEVMEMINT_RESERVATION *psReservation);
+
+PVRSRV_ERROR
+DevmemIntGetReservationData(DEVMEMINT_RESERVATION* psReservation, PMR** ppsPMR, IMG_DEV_VIRTADDR* psDevVAddr);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntReserveRange()
+ * @Description Indicates that the specified range should be reserved from the
+ *              given heap.
+ *
+ *              In turn causes the page tables to be allocated to cover the
+ *              specified range.
+ *
+ *              If you call DevmemIntReserveRange() (and the call succeeds)
+ *              then you are promising that you shall later call
+ *              DevmemIntUnreserveRange().
+ *
+ * @Input       psDevmemHeap        Pointer to the heap the reservation is made
+ *                                  on
+ * @Input       sReservationVAddr   Virtual address of the reservation
+ * @Input       uiVirtualSize       Size of the reservation (in bytes)
+ * @Input       ppsRsrv             Return pointer to the reservation object
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                       IMG_DEV_VIRTADDR sReservationVAddr,
+                       IMG_DEVMEM_SIZE_T uiVirtualSize,
+                       DEVMEMXINT_RESERVATION **ppsRsrv);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntUnreserveRange()
+ * @Description Undoes the state change caused by DevmemXIntReserveRage()
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntUnreserveRange(DEVMEMXINT_RESERVATION *psRsrv);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationAcquire
+@Description    Acquire a reference to the provided device memory reservation.
+                Prevents releasing of the reservation if external device
+                resource components still require it.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+IMG_BOOL
+DevmemIntReservationAcquire(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationRelease
+@Description    Release the reference to the provided device memory reservation.
+                Once these references have been released the
+                reservation is allowed to be released from UM.
+@Return         None.
+*/ /**************************************************************************/
+void
+DevmemIntReservationRelease(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntMapPages()
+ * @Description Maps an arbitrary amount of pages from a PMR to a reserved range
+ *              and takes references on the PMR.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       psPMR              PMR that is mapped
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 mapped
+ * @Input       uiPhysPageOffset   Logical offset in the PMR (measured in pages)
+ * @Input       uiFlags            Mapping flags
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntMapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                   PMR *psPMR,
+                   IMG_UINT32 uiPageCount,
+                   IMG_UINT32 uiPhysPageOffset,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   IMG_UINT32 uiVirtPageOffset);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntUnmapPages()
+ * @Description Unmaps an arbitrary amount of pages from a reserved range and
+ *              releases references on associated PMRs.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 unmapped
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntUnmapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                     IMG_UINT32 uiVirtPageOffset,
+                     IMG_UINT32 uiPageCount);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntMapVRangeToBackingPage()
+ * @Description Maps a kernel internal backing page to a reserved range.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 mapped
+ * @Input       uiFlags            Mapping flags
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntMapVRangeToBackingPage(DEVMEMXINT_RESERVATION *psRsrv,
+                                 IMG_UINT32 uiPageCount,
+                                 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                 IMG_UINT32 uiVirtPageOffset);
+
+/*
+ * DevmemIntInvalidateFBSCTable()
+ *
+ * Invalidate selected FBSC table indices.
+ *
+ */
+PVRSRV_ERROR
+DevmemIntInvalidateFBSCTable(DEVMEMINT_CTX *psDevmemCtx,
+                             IMG_UINT64 ui64FBSCEntryMask);
+
+PVRSRV_ERROR
+DevmemIntIsVDevAddrValid(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDevNode,
+                         DEVMEMINT_CTX *psDevMemContext,
+                         IMG_DEV_VIRTADDR sDevAddr);
+
+PVRSRV_ERROR
+DevmemIntGetFaultAddress(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDevNode,
+                         DEVMEMINT_CTX *psDevMemContext,
+                         IMG_DEV_VIRTADDR *psFaultAddress);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntRegisterPFNotifyKM
+@Description    Registers a PID to be notified when a page fault occurs on a
+                specific device memory context.
+@Input          psDevmemCtx    The context to be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntRegisterPFNotifyKM(DEVMEMINT_CTX *psDevmemCtx,
+                            IMG_BOOL      bRegister);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntPFNotify
+@Description    Notifies any processes that have registered themselves to be
+                notified when a page fault happens on a specific device memory
+                context.
+@Input          *psDevNode           The device node.
+@Input          ui64FaultedPCAddress The page catalogue address that faulted.
+@Input          sFaultAddress        The address that triggered the fault.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntPFNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                               IMG_UINT64         ui64FaultedPCAddress,
+                               IMG_DEV_VIRTADDR   sFaultAddress);
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize,
+                              DLLIST_NODE *psValidRegionsList);
+
+void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psValidRegionsList);
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * DevmemIntPDumpSaveToFileVirtual()
+ *
+ * Writes out PDump "SAB" commands with the data found in memory at
+ * the given virtual address.
+ */
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 uiArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * DevmemIntPDumpSaveToFileVirtualNoValidate()
+ *
+ * Writes out PDump "SAB" commands with the data found in memory at
+ * the given virtual address. Doesn't perform address validation.
+ */
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          DEVMEMINT_CTX *psDevmemCtx,
+                                          IMG_DEV_VIRTADDR sDevAddrStart,
+                                          IMG_DEVMEM_SIZE_T uiSize,
+                                          const IMG_CHAR *pszFilename,
+                                          IMG_UINT32 ui32FileOffset,
+                                          IMG_UINT32 ui32PDumpFlags);
+
+IMG_UINT32
+DevmemIntMMUContextID(DEVMEMINT_CTX *psDevMemContext);
+
+PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32LogicalWidth,
+                              IMG_UINT32 ui32LogicalHeight,
+                              IMG_UINT32 ui32PhysicalWidth,
+                              IMG_UINT32 ui32PhysicalHeight,
+                              PDUMP_PIXEL_FORMAT ePixFmt,
+                              IMG_MEMLAYOUT eMemLayout,
+                              IMG_FB_COMPRESSION eFBCompression,
+                              const IMG_UINT32 *paui32FBCClearColour,
+                              PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                              IMG_DEV_VIRTADDR sHeader,
+                              IMG_UINT32 ui32HeaderSize,
+                              IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             DEVMEMINT_CTX *psDevMemContext,
+                             IMG_UINT32 ui32Size,
+                             const IMG_CHAR *pszFileName,
+                             IMG_DEV_VIRTADDR sData,
+                             IMG_UINT32 ui32DataSize,
+                             IMG_UINT32 ui32HeaderType,
+                             IMG_UINT32 ui32ElementType,
+                             IMG_UINT32 ui32ElementCount,
+                             IMG_UINT32 ui32PDumpFlags);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpGetValidRegions)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(sDevAddrStart);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpFreeValidRegions)
+#endif
+static INLINE void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psDevAddrRegions)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevAddrRegions);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpSaveFromRegionListToFileVirtual)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(psDevAddrRegions);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpSaveToFileVirtual)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 uiArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(sDevAddrStart);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpImageDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32LogicalWidth,
+                              IMG_UINT32 ui32LogicalHeight,
+                              IMG_UINT32 ui32PhysicalWidth,
+                              IMG_UINT32 ui32PhysicalHeight,
+                              PDUMP_PIXEL_FORMAT ePixFmt,
+                              IMG_MEMLAYOUT eMemLayout,
+                              IMG_FB_COMPRESSION eFBCompression,
+                              const IMG_UINT32 *paui32FBCClearColour,
+                              PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                              IMG_DEV_VIRTADDR sHeader,
+                              IMG_UINT32 ui32HeaderSize,
+                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevMemContext);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	PVR_UNREFERENCED_PARAMETER(pszFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpDataDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32ElementType,
+                              IMG_UINT32 ui32ElementCount,
+                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevMemContext);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	PVR_UNREFERENCED_PARAMETER(pszFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#endif /* PDUMP */
+
+PVRSRV_ERROR
+DevmemIntInit(void);
+
+PVRSRV_ERROR
+DevmemIntDeInit(void);
+
+PVRSRV_ERROR
+DevmemIntExportCtx(DEVMEMINT_CTX *psContext,
+                   PMR *psPMR,
+                   DEVMEMINT_CTX_EXPORT **ppsContextExport);
+
+PVRSRV_ERROR
+DevmemIntUnexportCtx(DEVMEMINT_CTX_EXPORT *psContextExport);
+
+PVRSRV_ERROR
+DevmemIntAcquireRemoteCtx(PMR *psPMR,
+                          DEVMEMINT_CTX **ppsContext,
+                          IMG_HANDLE *phPrivData);
+
+#endif /* DEVICEMEM_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server_utils.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server_utils.h
new file mode 100644
index 000000000000..923bfc9a8f18
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server_utils.h
@@ -0,0 +1,179 @@
+/**************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header file utilities that are specific to device memory functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv.h"
+
+static INLINE PVRSRV_ERROR DevmemCPUCacheMode(PVRSRV_MEMALLOCFLAGS_T ulFlags,
+											  IMG_UINT32 *pui32Ret)
+{
+	IMG_UINT32 ui32CPUCacheMode = PVRSRV_CPU_CACHE_MODE(ulFlags);
+	IMG_UINT32 ui32Ret;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32CPUCacheMode == PVRSRV_CPU_CACHE_MODE(ulFlags));
+
+	switch (ui32CPUCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_CACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT:
+
+			/*
+			 * If system has no coherency but coherency has been requested for CPU
+			 * and GPU we currently fall back to write-combine.
+			 * This avoids errors on arm64 when uncached is turned into ordered device memory
+			 * and suffers from problems with unaligned access.
+			 */
+			if (PVRSRV_GPU_CACHE_MODE(ulFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+			}
+			else
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_CACHED;
+			}
+			break;
+
+		default:
+			PVR_LOG(("DevmemCPUCacheMode: Unknown CPU cache mode 0x%08x", ui32CPUCacheMode));
+			PVR_ASSERT(0);
+			/*
+				We should never get here, but if we do then setting the mode
+				to uncached is the safest thing to do.
+			*/
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED;
+			eError = PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+			break;
+	}
+
+	*pui32Ret = ui32Ret;
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR DevmemDeviceCacheMode(PVRSRV_MEMALLOCFLAGS_T ulFlags,
+												 IMG_UINT32 *pui32Ret)
+{
+	IMG_UINT32 ui32DeviceCacheMode = PVRSRV_GPU_CACHE_MODE(ulFlags);
+	IMG_UINT32 ui32Ret;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32DeviceCacheMode == PVRSRV_GPU_CACHE_MODE(ulFlags));
+
+	switch (ui32DeviceCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_CACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT:
+
+			/*
+			 * If system has no coherency but coherency has been requested for CPU
+			 * and GPU we currently fall back to write-combine.
+			 * This avoids errors on arm64 when uncached is turned into ordered device memory
+			 * and suffers from problems with unaligned access.
+			 */
+			if (PVRSRV_CPU_CACHE_MODE(ulFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC;
+			}
+			else
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_CACHED;
+			}
+			break;
+
+		default:
+			PVR_LOG(("DevmemDeviceCacheMode: Unknown device cache mode 0x%08x", ui32DeviceCacheMode));
+			PVR_ASSERT(0);
+			/*
+				We should never get here, but if we do then setting the mode
+				to uncached is the safest thing to do.
+			*/
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED;
+			eError = PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+			break;
+	}
+
+	*pui32Ret = ui32Ret;
+
+	return eError;
+}
+
+static INLINE IMG_BOOL DevmemDeviceCacheCoherency(PVRSRV_DEVICE_NODE *psDeviceNode,
+												  PVRSRV_MEMALLOCFLAGS_T ulFlags)
+{
+	IMG_UINT32 ui32DeviceCacheMode = PVRSRV_GPU_CACHE_MODE(ulFlags);
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_ASSERT(ui32DeviceCacheMode == PVRSRV_GPU_CACHE_MODE(ulFlags));
+
+	if (ui32DeviceCacheMode == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+	{
+		bRet = PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig);
+	}
+	return bRet;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/di_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_common.h
new file mode 100644
index 000000000000..a10178708a75
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_common.h
@@ -0,0 +1,236 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common types for Debug Info framework.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DI_COMMON_H
+#define DI_COMMON_H
+
+#include "img_types.h"
+
+/* Token that signals that a header should be printed. */
+#define DI_START_TOKEN ((void *) 1)
+
+/* This is a public handle to an entry. */
+#ifndef DI_GROUP_DEFINED
+#define DI_GROUP_DEFINED
+typedef struct DI_GROUP DI_GROUP;
+#endif
+#ifndef DI_ENTRY_DEFINED
+#define DI_ENTRY_DEFINED
+typedef struct DI_ENTRY DI_ENTRY;
+#endif
+typedef struct OSDI_IMPL_ENTRY OSDI_IMPL_ENTRY;
+
+/*! Debug Info entries types. */
+typedef enum DI_ENTRY_TYPE
+{
+    DI_ENTRY_TYPE_GENERIC,          /*!< generic entry type, implements
+                                         start/stop/next/show iterator
+                                         interface */
+    DI_ENTRY_TYPE_RANDOM_ACCESS,    /*!< random access entry, implements
+                                         seek/read iterator interface */
+} DI_ENTRY_TYPE;
+
+/*! @Function DI_PFN_START
+ *
+ * @Description
+ * Start operation returns first entry and passes it to Show operation.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @InOut pui64Pos current data position in the entry
+ *
+ * @Return pointer to data that will be passed to the other iterator
+ *         functions in pvData argument
+ */
+typedef void *(*DI_PFN_START)(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos);
+
+/*! @Function DI_PFN_STOP
+ *
+ * @Description
+ * Stop operations is called after iterator reaches end of data.
+ *
+ * If pvData was allocated in pfnStart it should be freed here.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ */
+typedef void (*DI_PFN_STOP)(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*! @Function DI_PFN_NEXT
+ *
+ * @Description
+ * Next returns next data entry and passes it to Show operation.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ * @InOut pui64Pos current data position in the entry
+ */
+typedef void *(*DI_PFN_NEXT)(OSDI_IMPL_ENTRY *psEntry, void *pvData,
+                             IMG_UINT64 *pui64Pos);
+
+/*! @Function DI_PFN_SHOW
+ *
+ * @Description
+ * Outputs the data element.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ */
+typedef int (*DI_PFN_SHOW)(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*! @Function DI_PFN_SEEK
+ *
+ * @Description
+ * Changes position of the entry data pointer
+ *
+ * @Input uiOffset new entry offset (absolute)
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_SEEK)(IMG_UINT64 ui64Offset, void *pvData);
+
+/*! @Function DI_PFN_READ
+ *
+ * @Description
+ * Retrieves data from the entry from position previously set by Seek.
+ *
+ * @Input pszBuffer output buffer
+ * @Input ui64Count length of the output buffer
+ * @InOut pui64Pos pointer to the current position in the entry
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_READ)(IMG_CHAR *pszBuffer, IMG_UINT64 ui64Count,
+                                 IMG_UINT64 *pui64Pos, void *pvData);
+
+/*! @Function DI_PFN_WRITE
+ *
+ * @Description
+ * Handle writes operation to the entry.
+ *
+ * @Input pszBuffer NUL-terminated buffer containing written data
+ * @Input ui64Count length of the data in pszBuffer (length of the buffer)
+ * @InOut pui64Pos pointer to the current position in the entry
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_WRITE)(const IMG_CHAR *pszBuffer,
+                                  IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                                  void *pvData);
+
+/*! Debug info entry iterator.
+ *
+ * This covers all entry types: GENERIC and RANDOM_ACCESS.
+ *
+ * The GENERIC entry type
+ *
+ * The GENERIC type should implement either a full set of following callbacks:
+ * pfnStart, pfnStop, pfnNext and pfnShow, or pfnShow only. If only pfnShow
+ * callback is given the framework will use default handlers in place of the
+ * other ones.
+ *
+ * e.g. for generic entry:
+ *
+ *   struct sIter = {
+ *     .pfnStart = StartCb, .pfnStop = StopCb, pfnNext = NextCb,
+ *     .pfnShow = ShowCb
+ *   };
+ *
+ * The use case for implementing pfnShow only is if the data for the given
+ * entry is short and can be printed in one go because the pfnShow callback
+ * will be called only once.
+ *
+ * e.g. for one-shot print generic entry:
+ *
+ *   struct sIter = {
+ *     .pfnShow = SingleShowCb
+ *   };
+ *
+ * The DICreateEntry() function will return error if DI_ENTRY_TYPE_GENERIC
+ * type is used and invalid combination of callbacks is given.
+ *
+ * The RANDOM_ACCESS entry
+ *
+ * The RANDOM_ACCESS type should implement either both pfnSeek and pfnRead
+ * or pfnRead only callbacks.
+ *
+ * e.g. of seekable and readable random access entry:
+ *
+ *   struct sIter = {
+ *     .pfnSeek = SeekCb, .pfnRead = ReadCb
+ *   };
+ *
+ * The DICreateEntry() function will return error if DI_ENTRY_TYPE_RANDOM_ACCESS
+ * type is used and invalid combination of callbacks is given.
+ *
+ * Writing to file (optional)
+ *
+ * The iterator allows also to pass a pfnWrite callback that allows implementing
+ * write operation on the entry. The write operation is entry type agnostic
+ * which means that it can be defined for both GENERIC and RANDOM_ACCESS
+ * entries.
+ *
+ * e.g. for writable one-shot print generic entry
+ *
+ *   struct sIter = {
+ *     .pfnShow = SingleShowCb, .pfnWrite = WriteCb
+ *   };
+ */
+typedef struct DI_ITERATOR_CB
+{
+    /* Generic entry interface. */
+
+    DI_PFN_START pfnStart; /*!< Starts iteration and returns first element
+                                of entry's data. */
+    DI_PFN_STOP pfnStop;   /*!< Stops iteration. */
+    DI_PFN_NEXT pfnNext;   /*!< Returns next element of entry's data. */
+    DI_PFN_SHOW pfnShow;   /*!< Shows current data element of an entry. */
+
+    /* Optional random access entry interface. */
+
+    DI_PFN_SEEK pfnSeek;   /*!< Sets data pointer in an entry. */
+    DI_PFN_READ pfnRead;   /*!< Reads data from an entry. */
+
+    /* Optional writing to entry interface. Null terminated. */
+
+    DI_PFN_WRITE pfnWrite; /*!< Performs write operation on an entry. */
+    IMG_UINT32   ui32WriteLenMax; /*!< Maximum char length of entry
+                                       accepted for write. Includes \0 */
+} DI_ITERATOR_CB;
+
+#endif /* DI_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/di_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_server.h
new file mode 100644
index 000000000000..a797890c069a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_server.h
@@ -0,0 +1,220 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for creating Debug Info groups and entries.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DI_SERVER_H
+#define DI_SERVER_H
+
+#if defined(__KERNEL__) && defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __KERNEL__ && __linux__ */
+
+#include "di_common.h"
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+/*! @Function DIInit
+ *
+ * @Description
+ * Initialises Debug Info framework. This function will create common resources
+ * for the framework.
+ *
+ * Note: This function must be called before first call to
+ *       DIRegisterImplementation() all of the implementations.
+ */
+PVRSRV_ERROR DIInit(void);
+
+/*! @Function DIDeInit
+ *
+ * @Description
+ * De-initialises Debug Info framework. This function will call pfnDeInit()
+ * on each implementation and clean up common resources.
+ *
+ * In case some of the entries and groups have not been cleaned up this function
+ * will also perform recursive sweep and remove all entries and group for
+ * all implementations.
+ */
+void DIDeInit(void);
+
+/*! @Function DICreateEntry
+ *
+ * @Description
+ * Creates debug info entry. Depending on different implementations the entry
+ * might be for example a DebugFS file or something totally different.
+ *
+ * The entry will belong to a parent group if provided or to the root group
+ * if not.
+ *
+ * @Input pszName: name of the new entry
+ * @Input psDiGroup: parent group, if NULL entry will belong to the root group
+ * @Input psIterCb: implementation of the iterator for the entry
+ * @Input psPriv: private data that will be passed to the iterator operations
+ * @Input eType: type of the entry
+ *
+ * @Output ppsEntry: handle to the newly created entry
+ *
+ * @Return   PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DICreateEntry(const IMG_CHAR *pszName,
+                           DI_GROUP *psGroup,
+                           const DI_ITERATOR_CB *psIterCb,
+                           void *psPriv,
+                           DI_ENTRY_TYPE eType,
+                           DI_ENTRY **ppsEntry);
+
+/*! @Function DIDestroyEntry
+ *
+ * @Description
+ * Destroys debug info entry.
+ *
+ * @Input psEntry: handle to the entry
+ */
+void DIDestroyEntry(DI_ENTRY *psEntry);
+
+/*! @Function DICreateGroup
+ *
+ * @Description
+ * Creates debug info group. Depending on different implementations the group
+ * might be for example a DebugFS directory or something totally different.
+ *
+ * The group will belong to a parent group if provided or to the root group
+ * if not.
+ *
+ * @Input pszName: name of the new entry
+ * @Input psParent: parent group, if NULL entry will belong to the root group
+ *
+ * @Output ppsGroup: handle to the newly created entry
+ *
+ * @Return   PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DICreateGroup(const IMG_CHAR *pszName,
+                           DI_GROUP *psParent,
+                           DI_GROUP **ppsGroup);
+
+/*! @Function DIDestroyGroup
+ *
+ * @Description
+ * Destroys debug info group.
+ *
+ * @Input psGroup: handle to the group
+ */
+void DIDestroyGroup(DI_GROUP *psGroup);
+
+/*! @Function DIGetPrivData
+ *
+ * @Description
+ * Retrieves private data from psEntry. The data is either passed during
+ * entry creation via psPriv parameter of DICreateEntry() function
+ * or by explicitly setting it with DIGetPrivData() function.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ *
+ * @Returns pointer to the private data (can be NULL if private data
+ *          has not been specified)
+ */
+void *DIGetPrivData(const OSDI_IMPL_ENTRY *psEntry);
+
+/*! @Function DIWrite
+ *
+ * @Description
+ * Writes the binary data of the DI entry to the output sync, whatever that may
+ * be for the DI implementation.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pvData data
+ * @Input uiSize pvData length
+ */
+void DIWrite(const OSDI_IMPL_ENTRY *psEntry, const void *pvData,
+             IMG_UINT32 uiSize);
+
+/*! @Function DIPrintf
+ *
+ * @Description
+ * Prints formatted string to the DI entry.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated format string
+ */
+__printf(2, 3)
+void DIPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt, ...);
+
+/*! @Function DIVPrintf
+ *
+ * @Description
+ * Prints formatted string to the DI entry. Equivalent to DIPrintf but takes
+ * va_list instead of a variable number of arguments.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated format string
+ * @Input pArgs vs_list object
+ */
+__printf(2, 0)
+void DIVPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt,
+               va_list pArgs);
+
+/*! @Function DIPrintf
+ *
+ * @Description
+ * Prints a string to the DI entry.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated string
+ */
+void DIPuts(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszStr);
+
+/*! @Function DIHasOverflowed
+ *
+ * @Description
+ * Checks if the DI buffer has overflowed.
+ *
+ * @Return IMG_TRUE if buffer overflowed
+ */
+IMG_BOOL DIHasOverflowed(const OSDI_IMPL_ENTRY *psEntry);
+
+#endif /* DI_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/dkp_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/dkp_impl.h
new file mode 100644
index 000000000000..b5f47f8910a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/dkp_impl.h
@@ -0,0 +1,163 @@
+/**************************************************************************/ /*!
+@File           dkp_impl.h
+@Title          Functions for supporting the DRM Key Provider
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DKP_IMPL_H
+#define DKP_IMPL_H
+
+typedef IMG_HANDLE PVRDKF_DKP_HANDLE;
+typedef PVRDKF_DKP_HANDLE * PPVRDKF_DKP_HANDLE;
+
+/*! @Function DKP_PFN_SHOW
+ *
+ * @Description
+ *
+ * Describes the function called by the DKF infrastructure to request the DKP
+ * outputs all the mandatory and optional keys it supports. Each
+ * key / value pair should be output one per line using the provided
+ * PVRDPFOutput routine.
+ *
+ * The function should check that the passed 'psDevNode' references the
+ * same device as that associated with the module-specific 'hPrivData' handle.
+ * If it doesn't, no data should be displayed.
+ *
+ * @Input psDevNode   Pointer to device node for which data must be shown.
+ * @Input pid         Process-ID for which data must be shown.
+ * @Input hPrivData   Private data passed to DKF from DKP_Register. The DKP
+ *                     must use this within calls to the DKP_PFN_SHOW to limit
+ *                     the data key/value pairs to only those that are relevant
+ *                     to this instance.
+ */
+typedef void (DKP_PFN_SHOW)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, int pid,
+                            IMG_HANDLE hPrivData);
+
+typedef IMG_UINT32 DKP_CONNECTION_FLAGS;
+
+#define DKP_CONNECTION_FLAG_SYNC        BIT(0)
+#define DKP_CONNECTION_FLAG_SERVICES    BIT(1)
+
+#define DKP_CONNECTION_FLAG_INVALID     IMG_UINT32_C(0)
+#define DKP_CONNECTION_FLAG_ALL (DKP_CONNECTION_FLAG_SYNC | DKP_CONNECTION_FLAG_SERVICES)
+
+/* @Function PVRSRVRegisterDKP
+ *
+ * @Description
+ * Registers a DKP with the specified module's DKF entry.
+ *
+ * @Input   hPrivData               DKP Private data handle that the DKF will pass into
+ *                                  the ShowPFN when called. Can be NULL if there is no
+ *                                  DKP instance private data.
+ * @Input   pszDKPName              Provider name associated with the caller.
+ * @Input   psShowPfn               Function to be used when the fdinfo statistics
+ *                                  are queried.
+ * @Input   ui32Filter              The connection types this DKP should be output
+ *                                  on.
+ * @Output  phDkpHandle             Location to store generated handle created by this
+ *                                  function.
+ *
+ * @Returns  PVRSRV_ERROR
+ */
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                               const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle);
+
+/* @Function PVRSRVUnRegisterDKP
+ *
+ * @Description
+ * Removes a previously registered key from the device's DKF entry.
+ *
+ * @Input  hPrivData      Private data handle.
+ * @Input  hDkpHandle     Handle to the DKP's registered entry.
+ *                         Obtained from earlier PVRSRV_DKP_Register call.
+ *
+ * @Returns  PVRSRV_ERROR
+ */
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData,
+                                 PVRDKF_DKP_HANDLE hDkpHandle);
+
+
+/* @Function PVRDKPOutput
+ * Wrapper function which passes the printf-style arguments to the registered
+ * DKF output function associated with the registered DKP handle.
+ *
+ * @Input   hPrivData   Handle for associated DKP instance producing the output.
+ * @Input   fmt         printf-style format string
+ *
+ */
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...) __printf(2, 3);
+
+
+#if !defined(__linux__) || defined(INTEGRITY_OS) || defined(__QNXNTO__)
+
+/* Stub routines follow */
+static inline PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                                             const char *pszDKPName,
+                                             DKP_PFN_SHOW *psShowPfn,
+                                             DKP_CONNECTION_FLAGS ui32Filter,
+                                             PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pszDKPName);
+	PVR_UNREFERENCED_PARAMETER(psShowPfn);
+	PVR_UNREFERENCED_PARAMETER(phDkpHandle);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+static inline PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData,
+                                               PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(hDkpHandle);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+static inline void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...) __printf(2, 3)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(fmt);
+}
+#endif /* !__linux__ || INTEGRITY_OS || __QNXNTO__ */
+
+#endif /* DKP_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/dma_km.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/dma_km.h
new file mode 100644
index 000000000000..9bf633dfd98f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/dma_km.h
@@ -0,0 +1,85 @@
+/*************************************************************************/ /*!
+@File           dma_km.h
+@Title          DMA transfer module header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_KM_H
+#define DMA_KM_H
+
+#if defined(__linux__)
+#include <linux/version.h>
+#else
+#define KERNEL_VERSION
+#endif
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "pmr.h"
+#include "pvrsrv_sync_km.h"
+#include "connection_server.h"
+
+PVRSRV_ERROR DmaDeviceParams(CONNECTION_DATA *psConnection,
+							 PVRSRV_DEVICE_NODE *psDevNode,
+							 IMG_UINT32 *ui32DmaBuffAlign,
+							 IMG_UINT32 *ui32DmaTransferMult);
+
+PVRSRV_ERROR DmaSparseMappingTable(PMR *psPMR,
+								   IMG_DEVMEM_OFFSET_T uiOffset,
+								   IMG_UINT32 ui32SizeInPages,
+								   IMG_BOOL *pbTable);
+
+PVRSRV_ERROR DmaTransfer(CONNECTION_DATA *psConnection,
+		    PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 uiNumDMAs,
+			PMR** ppsPMR,
+			IMG_UINT64 *puiAddress,
+			IMG_DEVMEM_OFFSET_T *puiOffset,
+			IMG_DEVMEM_SIZE_T *puiSize,
+			IMG_UINT32 uiFlags,
+			PVRSRV_TIMELINE iUpdateTimeline);
+
+PVRSRV_ERROR PVRSRVInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode,
+								 CONNECTION_DATA *psConnectionData);
+void PVRSRVDeInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   CONNECTION_DATA *psConnectionData);
+
+#endif /* DMA_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/fwload.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwload.h
new file mode 100644
index 000000000000..08e7f533b666
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwload.h
@@ -0,0 +1,158 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services RGX OS Interface for loading the firmware
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines the OS interface through which the RGX
+                device initialisation code in the kernel/server will obtain
+                the RGX firmware binary image. The API is used during the
+                initialisation of an RGX device via the
+                PVRSRVCommonDeviceInitialise()
+                call sequence.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef FWLOAD_H
+#define FWLOAD_H
+
+#include "img_defs.h"
+#include "device_connection.h"
+#include "device.h"
+
+/*! Opaque type handle defined and known to the OS layer implementation of this
+ * fwload.h OS API. This private data is allocated in the implementation of
+ * OSLoadFirmware() and contains whatever data and information needed to be
+ * able to acquire and return the firmware binary image to the Services
+ * kernel/server during initialisation.
+ * It is no longer required and may be freed when OSUnloadFirmware() is called.
+ */
+typedef struct OS_FW_IMAGE_t OS_FW_IMAGE;
+
+#if defined(__linux__)
+
+bool OSVerifyFirmware(OS_FW_IMAGE* psFWImage);
+
+#endif
+
+/*************************************************************************/ /*!
+@Function     OSLoadFirmware
+@Description  The OS implementation must load or acquire the firmware (FW)
+              image binary needed by the driver stack.
+              A handle to the common layer device node is given to identify
+              which device instance in the system is being initialised. The
+              BVNC string is also supplied so that the implementation knows
+              which FW image to retrieve since each FW image only supports one
+              GPU type/revision.
+              The calling server code supports multiple GPU types and revisions
+              and will detect the specific GPU type and revision before calling
+              this API. It will also have runtime configuration of the VZ mode,
+              hence this API must be able to retrieve different FW binary
+              images based on the pszBVNCString given. The purpose of the end
+              platform/system is key to understand which FW images must be
+              available to the kernel server.
+              On exit the implementation must return a pointer to some private
+              data it uses to hold the FW image information and data. It will
+              be passed onto later API calls by the kernel server code.
+              NULL should be returned if the FW image could not be retrieved.
+              The format of the BVNC string is as follows ([x] denotes
+              optional field):
+                "rgx.fw[.signed].B.V[p].N.C[.vz]"
+              The implementation must first try to load the FW identified
+              by the pszBVpNCString parameter. If this is not available then it
+              should drop back to retrieving the FW identified by the
+              pszBVNCString parameter. The fields in the string are:
+                B, V, N, C are all unsigned integer identifying type/revision.
+                [.signed] is present when RGX_FW_SIGNED=1 is defined in the
+                  server build.
+                [p] denotes a provisional (pre-silicon) GPU configuration.
+                [.vz] is present when the kernel server is loaded on the HOST
+                  of a virtualised platform. See the DriverMode server
+                  AppHint for details.
+
+@Input        psDeviceNode       Device instance identifier.
+@Input        pszBVNCString      Identifier string of the FW image to
+                                 be loaded/acquired in production driver.
+@Input        pfnVerifyFirmware  Callback which checks validity of FW image.
+@Output       ppsFWImage         Ptr to private data on success,
+                                 NULL otherwise.
+@Return       PVRSRV_ERROR       PVRSRV_OK on success,
+                                 PVRSRV_ERROR_NOT_READY if filesystem is not
+                                                        ready/initialised,
+                                 PVRSRV_ERROR_NOT_FOUND if no suitable FW
+                                                        image could be found
+                                 PVRSRV_ERROR_OUT_OF_MEMORY if unable to alloc
+                                                        memory for FW image
+                                 PVRSRV_ERROR_NOT_AUTHENTICATED if FW image
+                                                        cannot be verified.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSLoadFirmware(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            const IMG_CHAR *pszBVNCString,
+                            bool (*pfnVerifyFirmware)(OS_FW_IMAGE*),
+                            OS_FW_IMAGE **ppsFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSFirmwareData
+@Description  This function returns a pointer to the start of the FW image
+              binary data held in memory. It must remain valid until
+              OSUnloadFirmware() is called.
+@Input        psFWImage  Private data opaque handle
+@Return       void*      Ptr to FW binary image to start on GPU.
+*/ /**************************************************************************/
+const void* OSFirmwareData(OS_FW_IMAGE *psFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSFirmwareSize
+@Description  This function returns the size of the FW image binary data.
+@Input        psFWImage  Private data opaque handle
+@Return       size_t     Size in bytes of the firmware binary image
+*/ /**************************************************************************/
+size_t OSFirmwareSize(OS_FW_IMAGE *psFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSUnloadFirmware
+@Description  This is called when the server has completed firmware
+              initialisation and no longer needs the private data, possibly
+              allocated by OSLoadFirmware().
+@Input        psFWImage  Private data opaque handle
+*/ /**************************************************************************/
+void OSUnloadFirmware(OS_FW_IMAGE *psFWImage);
+
+#endif /* FWLOAD_H */
+
+/******************************************************************************
+ End of file (fwload.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/fwtrace_string.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwtrace_string.h
new file mode 100644
index 000000000000..a2ab95c3a815
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwtrace_string.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File           fwtrace_string.h
+@Title          RGX Firmware trace strings for KM
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       Generic
+@Description    This file defines SFs tuple.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef KM_TRACE_STRING_H
+#define KM_TRACE_STRING_H
+
+#include "rgx_fwif_sf.h"
+
+extern const RGXKM_STID_FMT SFs[];
+extern const IMG_UINT32 g_ui32SFsCount;
+
+#endif /* KM_TRACE_STRING_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/handle.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle.h
new file mode 100644
index 000000000000..2c132901be1e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle.h
@@ -0,0 +1,211 @@
+/**************************************************************************/ /*!
+@File
+@Title          Handle Manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(HANDLE_API_H)
+#define HANDLE_API_H
+
+#include "lock_types.h"
+
+/*
+ * Handle API
+ * ----------
+ * The handle API is intended to provide handles for kernel resources, which
+ * can then be passed back to user space processes.
+ *
+ * The following functions comprise the API. Each function takes a pointer to
+ * a PVRSRV_HANDLE_BASE structure, one of which is allocated for each process,
+ * and stored in the per-process data area. Use KERNEL_HANDLE_BASE for handles
+ * not allocated for a particular process, or for handles that need to be
+ * allocated before the PVRSRV_HANDLE_BASE structure for the process is
+ * available.
+ *
+ * PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType,
+ *      PVRSRV_HANDLE_ALLOC_FLAG eFlag);
+ *
+ * Allocate a handle phHandle, for the resource of type eType pointed to by
+ * pvData.
+ *
+ * For handles that have a definite lifetime, where the corresponding resource
+ * is explicitly created and destroyed, eFlag should be zero.
+ *
+ * If a particular resource may be referenced multiple times by a given
+ * process, setting eFlag to PVRSRV_HANDLE_ALLOC_FLAG_MULTI will allow multiple
+ * handles to be allocated for the resource. Such handles cannot be found with
+ * PVRSRVFindHandle.
+ *
+ * PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType,
+ *      PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+ *
+ * This function is similar to PVRSRVAllocHandle, except that the allocated
+ * handles are associated with a parent handle, hParent, that has been
+ * allocated previously. Subhandles are automatically deallocated when their
+ * parent handle is deallocated.
+ * Subhandles can be treated as ordinary handles. For example, they may have
+ * subhandles of their own, and may be explicitly deallocated using
+ * PVRSRVReleaseHandle (see below).
+ *
+ * PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Find the handle previously allocated for the resource pointed to by pvData,
+ * of type eType. Handles allocated with the flag
+ * PVRSRV_HANDLE_ALLOC_FLAG_MULTI cannot be found using this function.
+ *
+ * PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Given a handle for a resource of type eType, return the pointer to the
+ * resource.
+ *
+ * PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType,
+ *      IMH_HANDLE hAncestor);
+ *
+ * Similar to PVRSRVLookupHandle, but checks the handle is a descendant
+ * of hAncestor.
+ *
+ * PVRSRV_ERROR PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Deallocate a handle of given type.
+ *
+ * Return the parent of a handle in *phParent, or NULL if the handle has
+ * no parent.
+ */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "hash.h"
+
+typedef enum
+{
+	#define HANDLETYPE(x) PVRSRV_HANDLE_TYPE_##x,
+	#include "handle_types.h"
+	#undef HANDLETYPE
+} PVRSRV_HANDLE_TYPE;
+
+static_assert(PVRSRV_HANDLE_TYPE_NONE == 0, "PVRSRV_HANDLE_TYPE_NONE must be zero");
+
+typedef enum
+{
+	PVRSRV_HANDLE_BASE_TYPE_CONNECTION,
+	PVRSRV_HANDLE_BASE_TYPE_PROCESS,
+	PVRSRV_HANDLE_BASE_TYPE_GLOBAL
+} PVRSRV_HANDLE_BASE_TYPE;
+
+
+typedef enum
+{
+	/* No flags */
+	PVRSRV_HANDLE_ALLOC_FLAG_NONE =    0,
+	/* Multiple handles can point at the given data pointer */
+	PVRSRV_HANDLE_ALLOC_FLAG_MULTI =   0x01,
+	/* Subhandles are allocated in a private handle space */
+	PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE = 0x02
+} PVRSRV_HANDLE_ALLOC_FLAG;
+
+typedef struct _HANDLE_BASE_ PVRSRV_HANDLE_BASE;
+
+typedef struct _PROCESS_HANDLE_BASE_
+{
+	PVRSRV_HANDLE_BASE *psHandleBase;
+	ATOMIC_T iRefCount;
+	uintptr_t uiHashKey;
+} PROCESS_HANDLE_BASE;
+
+extern PVRSRV_HANDLE_BASE *gpsKernelHandleBase;
+#define	KERNEL_HANDLE_BASE (gpsKernelHandleBase)
+
+#define HANDLE_DEBUG_LISTING_MAX_NUM 20
+
+typedef PVRSRV_ERROR (*PFN_HANDLE_RELEASE)(void *pvData);
+
+PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, PFN_HANDLE_RELEASE pfnReleaseData);
+PVRSRV_ERROR PVRSRVAllocHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, PFN_HANDLE_RELEASE pfnReleaseData);
+
+PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+PVRSRV_ERROR PVRSRVAllocSubHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+
+PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVFindHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_BOOL bRef);
+PVRSRV_ERROR PVRSRVLookupHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_BOOL bRef);
+
+PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_HANDLE hAncestor);
+
+void PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+void PVRSRVReleaseHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVDestroyHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVDestroyHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVDestroyHandleStagedUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVPurgeHandles(PVRSRV_HANDLE_BASE *psBase);
+
+PVRSRV_ERROR PVRSRVAllocHandleBase(PVRSRV_HANDLE_BASE **ppsBase,
+                                   PVRSRV_HANDLE_BASE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVFreeHandleBase(PVRSRV_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime);
+
+PVRSRV_ERROR PVRSRVFreeKernelHandles(PVRSRV_HANDLE_BASE *psBase);
+
+PVRSRV_ERROR PVRSRVHandleInit(void);
+
+PVRSRV_ERROR PVRSRVHandleDeInit(void);
+
+/* Only called from sync_fallback_server.c */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+PVRSRV_HANDLE_BASE *PVRSRVRetrieveProcessHandleBase(void);
+#endif
+
+PVRSRV_ERROR PVRSRVAcquireProcessHandleBase(PROCESS_HANDLE_BASE **ppsBase);
+PVRSRV_ERROR PVRSRVReleaseProcessHandleBase(PROCESS_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime);
+
+void LockHandle(PVRSRV_HANDLE_BASE *psBase);
+void UnlockHandle(PVRSRV_HANDLE_BASE *psBase);
+
+
+#endif /* !defined(HANDLE_API_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_impl.h
new file mode 100644
index 000000000000..e965e536fb5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_impl.h
@@ -0,0 +1,90 @@
+/**************************************************************************/ /*!
+@File
+@Title          Implementation Callbacks for Handle Manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the handle manager API. This file is for declarations
+                and definitions that are private/internal to the handle manager
+                API but need to be shared between the generic handle manager
+                code and the various handle manager backends, i.e. the code that
+                implements the various callbacks.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(HANDLE_IMPL_H)
+#define HANDLE_IMPL_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+typedef struct _HANDLE_IMPL_BASE_ HANDLE_IMPL_BASE;
+
+typedef PVRSRV_ERROR (*PFN_HANDLE_ITER)(IMG_HANDLE hHandle, void *pvData);
+
+typedef struct _HANDLE_IMPL_FUNCTAB_
+{
+	/* Acquire a new handle which is associated with the given data */
+	PVRSRV_ERROR (*pfnAcquireHandle)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE *phHandle, void *pvData);
+
+	/* Release the given handle (optionally returning the data associated with it) */
+	PVRSRV_ERROR (*pfnReleaseHandle)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void **ppvData);
+
+	/* Get the data associated with the given handle */
+	PVRSRV_ERROR (*pfnGetHandleData)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void **ppvData);
+
+	/* Set the data associated with the given handle */
+	PVRSRV_ERROR (*pfnSetHandleData)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void *pvData);
+
+	PVRSRV_ERROR (*pfnIterateOverHandles)(HANDLE_IMPL_BASE *psHandleBase, PFN_HANDLE_ITER pfnHandleIter, void *pvHandleIterData);
+
+	/* Enable handle purging on the given handle base */
+	PVRSRV_ERROR (*pfnEnableHandlePurging)(HANDLE_IMPL_BASE *psHandleBase);
+
+	/* Purge handles on the given handle base */
+	PVRSRV_ERROR (*pfnPurgeHandles)(HANDLE_IMPL_BASE *psHandleBase);
+
+	/* Create handle base */
+	PVRSRV_ERROR (*pfnCreateHandleBase)(HANDLE_IMPL_BASE **psHandleBase);
+
+	/* Destroy handle base */
+	PVRSRV_ERROR (*pfnDestroyHandleBase)(HANDLE_IMPL_BASE *psHandleBase);
+
+} HANDLE_IMPL_FUNCTAB;
+
+PVRSRV_ERROR PVRSRVHandleGetFuncTable(HANDLE_IMPL_FUNCTAB const **ppsFuncs);
+
+#endif /* !defined(HANDLE_IMPL_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_types.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_types.h
new file mode 100644
index 000000000000..776ab86cc097
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_types.h
@@ -0,0 +1,87 @@
+/**************************************************************************/ /*!
+@File
+@Title          Handle Manager handle types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+/* NOTE: Do not add include guards to this file */
+
+HANDLETYPE(NONE)
+HANDLETYPE(SHARED_EVENT_OBJECT)
+HANDLETYPE(EVENT_OBJECT_CONNECT)
+HANDLETYPE(PMR_LOCAL_EXPORT_HANDLE)
+HANDLETYPE(PHYSMEM_PMR)
+HANDLETYPE(PHYSMEM_PMR_EXPORT)
+HANDLETYPE(PHYSMEM_PMR_SECURE_EXPORT)
+HANDLETYPE(DEVMEMINT_CTX)
+HANDLETYPE(DEVMEMINT_CTX_EXPORT)
+HANDLETYPE(DEVMEMINT_HEAP)
+HANDLETYPE(DEVMEMINT_RESERVATION)
+HANDLETYPE(DEVMEMXINT_RESERVATION)
+HANDLETYPE(DEVMEMINT_MAPPING)
+HANDLETYPE(RGX_FW_MEMDESC)
+HANDLETYPE(RGX_FREELIST)
+HANDLETYPE(RGX_MEMORY_BLOCK)
+HANDLETYPE(RGX_SERVER_RENDER_CONTEXT)
+HANDLETYPE(RGX_SERVER_TQ_CONTEXT)
+HANDLETYPE(RGX_SERVER_TQ_TDM_CONTEXT)
+HANDLETYPE(RGX_SERVER_COMPUTE_CONTEXT)
+HANDLETYPE(RGX_SERVER_RAY_CONTEXT)
+HANDLETYPE(RGX_SERVER_KICKSYNC_CONTEXT)
+HANDLETYPE(SYNC_PRIMITIVE_BLOCK)
+HANDLETYPE(SYNC_RECORD_HANDLE)
+HANDLETYPE(PVRSRV_TIMELINE_SERVER)
+HANDLETYPE(PVRSRV_FENCE_SERVER)
+HANDLETYPE(PVRSRV_FENCE_EXPORT)
+HANDLETYPE(RGX_KM_HW_RT_DATASET)
+HANDLETYPE(RGX_FWIF_ZSBUFFER)
+HANDLETYPE(RGX_POPULATION)
+HANDLETYPE(DC_DEVICE)
+HANDLETYPE(DC_DISPLAY_CONTEXT)
+HANDLETYPE(DC_BUFFER)
+HANDLETYPE(DC_PIN_HANDLE)
+HANDLETYPE(DEVMEM_MEM_IMPORT)
+HANDLETYPE(PHYSMEM_PMR_PAGELIST)
+HANDLETYPE(PVR_TL_SD)
+HANDLETYPE(RI_HANDLE)
+HANDLETYPE(DEV_PRIV_DATA)
+HANDLETYPE(MM_PLAT_CLEANUP)
+HANDLETYPE(WORKEST_RETURN_DATA)
+HANDLETYPE(DI_CONTEXT)
+HANDLETYPE(PVR_HWPERF_SD)
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/htbserver.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/htbserver.h
new file mode 100644
index 000000000000..4aaa3dd35e08
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/htbserver.h
@@ -0,0 +1,240 @@
+/*************************************************************************/ /*!
+@File           htbserver.h
+@Title          Host Trace Buffer server implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+
+                A Host Trace can be merged with a corresponding Firmware Trace.
+                This is achieved by inserting synchronisation data into both
+                traces and post processing to merge them.
+
+                The FW Trace will contain a "Sync Partition Marker". This is
+                updated every time the RGX is brought out of reset (RGX clock
+                timestamps reset at this point) and is repeated when the FW
+                Trace buffer wraps to ensure there is always at least 1
+                partition marker in the Firmware Trace buffer whenever it is
+                read.
+
+                The Host Trace will contain corresponding "Sync Partition
+                Markers" - #HTBSyncPartitionMarker(). Each partition is then
+                subdivided into "Sync Scale" sections - #HTBSyncScale(). The
+                "Sync Scale" data allows the timestamps from the two traces to
+                be correlated. The "Sync Scale" data is updated as part of the
+                standard RGX time correlation code (rgxtimecorr.c) and is
+                updated periodically including on power and clock changes.
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HTBSERVER_H
+#define HTBSERVER_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "htbuffer_types.h"
+
+#define HTBLOGK(SF, args...) do { if (HTB_GROUP_ENABLED(SF)) { (void)HTBLogSimple(SF, ## args); } } while (0)
+
+/* macros to cast 64 or 32-bit pointers into 32-bit integer components for Host Trace */
+#define HTBLOG_PTR_BITS_HIGH(p) ((IMG_UINT32)((((IMG_UINT64)((uintptr_t)p))>>32)&0xffffffff))
+#define HTBLOG_PTR_BITS_LOW(p)  ((IMG_UINT32)(((IMG_UINT64)((uintptr_t)p))&0xffffffff))
+
+/* macros to cast 64-bit integers into 32-bit integer components for Host Trace */
+#define HTBLOG_U64_BITS_HIGH(u) ((IMG_UINT32)((u>>32)&0xffffffff))
+#define HTBLOG_U64_BITS_LOW(u)  ((IMG_UINT32)(u&0xffffffff))
+
+/* Host Trace Buffer name */
+#define HTB_STREAM_NAME	"PVRHTBuffer"
+
+/************************************************************************/ /*!
+ @Function      HTBInit
+ @Description   Initialise the Host Trace Buffer and allocate all resources
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBInit_Impl(void);
+
+/************************************************************************/ /*!
+ @Function      HTBDeInit
+ @Description   Close the Host Trace Buffer and free all resources
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBDeInit_Impl(void);
+
+/*************************************************************************/ /*!
+ @Function      HTBControlKM
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control the behaviour of the data buffer
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBControlKM_Impl(const IMG_UINT32 ui32NumFlagGroups,
+			 const IMG_UINT32 *aui32GroupEnable,
+			 const IMG_UINT32 ui32LogLevel,
+			 const IMG_UINT32 ui32EnablePID,
+			 const HTB_LOGMODE_CTRL eLogMode,
+			 const HTB_OPMODE_CTRL eOpMode);
+
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarker
+ @Description   Write an HTB sync partition marker to the HTB log
+
+ @Input         ui32Marker      Marker value
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarker_Impl(const IMG_UINT32 ui32Marker);
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarkerRpt
+ @Description   Write a HTB sync partition marker to the HTB log, given
+                the previous values to repeat.
+
+ @Input         ui32Marker      Marker value
+ @Input         ui64SyncOSTS    previous OSTS
+ @Input         ui64SyncCRTS    previous CRTS
+ @Input         ui32ClkSpeed    previous Clockspeed
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarkerRepeat_Impl(const IMG_UINT32 ui32Marker,
+							 const IMG_UINT64 ui64SyncOSTS,
+							 const IMG_UINT64 ui64SyncCRTS,
+							 const IMG_UINT32 ui32ClkSpeed);
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncScale
+ @Description   Write FW-Host synchronisation data to the HTB log when clocks
+                change or are re-calibrated
+
+ @Input         bLogValues      IMG_TRUE if value should be immediately written
+                                out to the log
+
+ @Input         ui64OSTS        OS Timestamp
+
+ @Input         ui64CRTS        Rogue timestamp
+
+ @Input         ui32CalcClkSpd  Calculated clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncScale_Impl(const IMG_BOOL bLogValues, const IMG_UINT64 ui64OSTS,
+			 const IMG_UINT64 ui64CRTS, const IMG_UINT32 ui32CalcClkSpd);
+
+/*************************************************************************/ /*!
+ @Function      HTBLogSimple
+ @Description   Record a Host Trace Buffer log event with implicit PID and Timestamp
+
+ @Input         SF              The log event ID
+
+ @Input         ...             Log parameters
+
+ @Return        PVRSRV_OK       Success.
+
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBLogSimple_Impl(IMG_UINT32 SF, ...);
+
+/*  DEBUG log group enable */
+#if !defined(HTB_DEBUG_LOG_GROUP)
+#undef HTB_LOG_TYPE_DBG    /* No trace statements in this log group should be checked in */
+#define HTB_LOG_TYPE_DBG    __BUILDERROR__
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+/*************************************************************************/ /*!
+ @Function      HTBIsConfigured
+ @Description   Determine if HTB stream has been configured
+
+ @Input         none
+
+ @Return        IMG_FALSE       Stream has not been configured
+                IMG_TRUE        Stream has been configured
+
+*/ /**************************************************************************/
+IMG_BOOL
+HTBIsConfigured_Impl(void);
+
+#define HTBIsConfigured HTBIsConfigured_Impl
+#define HTBLogSimple HTBLogSimple_Impl
+#define HTBSyncScale(bLogValues, ui64OSTS, ui64CRTS, ui32CalcClkSpd) \
+  HTBSyncScale_Impl((bLogValues), (ui64OSTS), (ui64CRTS), (ui32CalcClkSpd))
+#define HTBSyncPartitionMarkerRepeat(ui32Marker, ui64SyncOSTS, ui64SyncCRTS, ui32ClkSpeed) \
+  HTBSyncPartitionMarkerRepeat_Impl((ui32Marker), (ui64SyncOSTS), (ui64SyncCRTS), (ui32ClkSpeed))
+#define HTBSyncPartitionMarker(a) HTBSyncPartitionMarker_Impl((a))
+#define HTBControlKM(ui32NumFlagGroups, aui32GroupEnable, ui32LogLevel, ui32EnablePID, eLogMode, eOpMode) \
+  HTBControlKM_Impl((ui32NumFlagGroups), (aui32GroupEnable), (ui32LogLevel), (ui32EnablePID), (eLogMode), (eOpMode))
+#define HTBInit() HTBInit_Impl()
+#define HTBDeInit() HTBDeInit_Impl()
+#else	/* !PVRSRV_ENABLE_HTB) */
+#define HTBIsConfigured()  IMG_FALSE
+#define HTBLogSimple(SF, args...)  PVRSRV_OK
+#define HTBSyncScale(a, b, c, d)
+#define HTBSyncPartitionMarkerRepeat(a, b, c, d)
+#define HTBSyncPartitionMarker(a)
+#define HTBControlKM(a, b, c, d, e, f) PVRSRV_OK
+#define HTBDeInit() PVRSRV_OK
+#define HTBInit() PVRSRV_OK
+#endif	/* PVRSRV_ENABLE_HTB */
+#endif /* HTBSERVER_H */
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/info_page.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/info_page.h
new file mode 100644
index 000000000000..5816125d742f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/info_page.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose memory shared between kernel driver and user
+                mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_KM_H
+#define INFO_PAGE_KM_H
+
+#include "pvrsrv_error.h"
+
+#include "pmr.h"
+#include "pvrsrv.h"
+#include "info_page_defs.h"
+
+/**
+ * @Function InfoPageCreate
+ * @Description Allocates resources for global information page.
+ * @Input psData pointer to PVRSRV data
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+PVRSRV_ERROR InfoPageCreate(PVRSRV_DATA *psData);
+
+/**
+ * @Function InfoPageDestroy
+ * @Description Frees all of the resource of global information page.
+ * @Input psData pointer to PVRSRV data
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+void InfoPageDestroy(PVRSRV_DATA *psData);
+
+/**
+ * @Function PVRSRVAcquireInfoPageKM()
+ * @Description This interface is used for obtaining the global information page
+ *              which acts as a general purpose shared memory between KM and UM.
+ *              The use of this information page outside of services is _not_
+ *              recommended.
+ * @Output ppsPMR handle to exported PMR
+ * @Return
+ */
+PVRSRV_ERROR PVRSRVAcquireInfoPageKM(PMR **ppsPMR);
+
+/**
+ * @Function PVRSRVReleaseInfoPageKM()
+ * @Description This function matches PVRSRVAcquireInfoPageKM().
+ * @Input psPMR handle to exported PMR
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+PVRSRV_ERROR PVRSRVReleaseInfoPageKM(PMR *psPMR);
+
+/**
+ * @Function GetInfoPageDebugFlagsKM()
+ * @Description Return info page debug flags
+ * @Return info page debug flags
+ */
+static INLINE IMG_UINT32 GetInfoPageDebugFlagsKM(void)
+{
+	return (PVRSRVGetPVRSRVData())->pui32InfoPage[DEBUG_FEATURE_FLAGS];
+}
+
+#endif /* INFO_PAGE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/lists.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/lists.h
new file mode 100644
index 000000000000..2e2c29a0b7b0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/lists.h
@@ -0,0 +1,367 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linked list shared functions templates.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Definition of the linked list function templates.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LISTS_UTILS_H
+#define LISTS_UTILS_H
+
+/* instruct QAC to ignore warnings about the following custom formatted macros */
+/* PRQA S 0881,3410 ++ */
+
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "img_types.h"
+#include "device.h"
+#include "power.h"
+
+/*
+ - USAGE -
+
+ The list functions work with any structure that provides the fields psNext and
+ ppsThis. In order to make a function available for a given type, it is required
+ to use the function template macro that creates the actual code.
+
+ There are 5 main types of functions:
+ - INSERT      : given a pointer to the head pointer of the list and a pointer
+                 to the node, inserts it as the new head.
+ - INSERT TAIL : given a pointer to the head pointer of the list and a pointer
+                 to the node, inserts the node at the tail of the list.
+ - REMOVE      : given a pointer to a node, removes it from its list.
+ - FOR EACH    : apply a function over all the elements of a list.
+ - ANY         : apply a function over the elements of a list, until one of them
+                 return a non null value, and then returns it.
+
+ The two last functions can have a variable argument form, with allows to pass
+ additional parameters to the callback function. In order to do this, the
+ callback function must take two arguments, the first is the current node and
+ the second is a list of variable arguments (va_list).
+
+ The ANY functions have also another for which specifies the return type of the
+ callback function and the default value returned by the callback function.
+
+*/
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_ForEach
+@Description    Apply a callback function to all the elements of a list.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+*/ /**************************************************************************/
+#define DECLARE_LIST_FOR_EACH(TYPE) \
+void List_##TYPE##_ForEach(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_FOR_EACH(TYPE) \
+void List_##TYPE##_ForEach(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))\
+{\
+	while (psHead)\
+	{\
+		pfnCallBack(psHead);\
+		psHead = psHead->psNext;\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_ForEachSafe
+@Description    Apply a callback function to all the elements of a list. Do it
+                in a safe way that handles the fact that a node might remove
+                itself from the list during the iteration.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+*/ /**************************************************************************/
+#define DECLARE_LIST_FOR_EACH_SAFE(TYPE) \
+void List_##TYPE##_ForEachSafe(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_FOR_EACH_SAFE(TYPE) \
+void List_##TYPE##_ForEachSafe(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))\
+{\
+	TYPE *psNext;\
+\
+	while (psHead)\
+	{\
+		psNext = psHead->psNext; \
+		pfnCallBack(psHead);\
+		psHead = psNext;\
+	}\
+}
+
+
+#define DECLARE_LIST_FOR_EACH_VA(TYPE) \
+void List_##TYPE##_ForEach_va(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_FOR_EACH_VA(TYPE) \
+void List_##TYPE##_ForEach_va(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode, va_list va), ...) \
+{\
+	va_list ap;\
+	while (psHead)\
+	{\
+		va_start(ap, pfnCallBack);\
+		pfnCallBack(psHead, ap);\
+		psHead = psHead->psNext;\
+		va_end(ap);\
+	}\
+}
+
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Any
+@Description    Applies a callback function to the elements of a list until
+                the function returns a non null value, then returns it.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+@Return         The first non null value returned by the callback function.
+*/ /**************************************************************************/
+#define DECLARE_LIST_ANY(TYPE) \
+void* List_##TYPE##_Any(TYPE *psHead, void* (*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_ANY(TYPE) \
+void* List_##TYPE##_Any(TYPE *psHead, void* (*pfnCallBack)(TYPE* psNode))\
+{ \
+	void *pResult;\
+	TYPE *psNextNode;\
+	pResult = NULL;\
+	psNextNode = psHead;\
+	while (psHead && !pResult)\
+	{\
+		psNextNode = psNextNode->psNext;\
+		pResult = pfnCallBack(psHead);\
+		psHead = psNextNode;\
+	}\
+	return pResult;\
+}
+
+
+/*with variable arguments, that will be passed as a va_list to the callback function*/
+
+#define DECLARE_LIST_ANY_VA(TYPE) \
+void* List_##TYPE##_Any_va(TYPE *psHead, void*(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_ANY_VA(TYPE) \
+void* List_##TYPE##_Any_va(TYPE *psHead, void*(*pfnCallBack)(TYPE* psNode, va_list va), ...)\
+{\
+	va_list ap;\
+	TYPE *psNextNode;\
+	void* pResult = NULL;\
+	while (psHead && !pResult)\
+	{\
+		psNextNode = psHead->psNext;\
+		va_start(ap, pfnCallBack);\
+		pResult = pfnCallBack(psHead, ap);\
+		va_end(ap);\
+		psHead = psNextNode;\
+	}\
+	return pResult;\
+}
+
+/*those ones are for extra type safety, so there's no need to use castings for the results*/
+
+#define DECLARE_LIST_ANY_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any(TYPE *psHead, RTYPE (*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_ANY_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any(TYPE *psHead, RTYPE (*pfnCallBack)(TYPE* psNode))\
+{ \
+	RTYPE result;\
+	TYPE *psNextNode;\
+	result = CONTINUE;\
+	psNextNode = psHead;\
+	while (psHead && result == CONTINUE)\
+	{\
+		psNextNode = psNextNode->psNext;\
+		result = pfnCallBack(psHead);\
+		psHead = psNextNode;\
+	}\
+	return result;\
+}
+
+
+#define DECLARE_LIST_ANY_VA_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any_va(TYPE *psHead, RTYPE(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_ANY_VA_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any_va(TYPE *psHead, RTYPE(*pfnCallBack)(TYPE* psNode, va_list va), ...)\
+{\
+	va_list ap;\
+	TYPE *psNextNode;\
+	RTYPE result = CONTINUE;\
+	while (psHead && result == CONTINUE)\
+	{\
+		psNextNode = psHead->psNext;\
+		va_start(ap, pfnCallBack);\
+		result = pfnCallBack(psHead, ap);\
+		va_end(ap);\
+		psHead = psNextNode;\
+	}\
+	return result;\
+}
+
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Remove
+@Description    Removes a given node from the list.
+@Input          psNode      The pointer to the node to be removed.
+*/ /**************************************************************************/
+#define DECLARE_LIST_REMOVE(TYPE) \
+void List_##TYPE##_Remove(TYPE *psNode)
+
+#define IMPLEMENT_LIST_REMOVE(TYPE) \
+void List_##TYPE##_Remove(TYPE *psNode)\
+{\
+	(*psNode->ppsThis)=psNode->psNext;\
+	if (psNode->psNext)\
+	{\
+		psNode->psNext->ppsThis = psNode->ppsThis;\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Insert
+@Description    Inserts a given node at the beginning of the list.
+@Input          psHead   The pointer to the pointer to the head node.
+@Input          psNode   The pointer to the node to be inserted.
+*/ /**************************************************************************/
+#define DECLARE_LIST_INSERT(TYPE) \
+void List_##TYPE##_Insert(TYPE **ppsHead, TYPE *psNewNode)
+
+#define IMPLEMENT_LIST_INSERT(TYPE) \
+void List_##TYPE##_Insert(TYPE **ppsHead, TYPE *psNewNode)\
+{\
+	psNewNode->ppsThis = ppsHead;\
+	psNewNode->psNext = *ppsHead;\
+	*ppsHead = psNewNode;\
+	if (psNewNode->psNext)\
+	{\
+		psNewNode->psNext->ppsThis = &(psNewNode->psNext);\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_InsertTail
+@Description    Inserts a given node at the end of the list.
+@Input          psHead   The pointer to the pointer to the head node.
+@Input          psNode   The pointer to the node to be inserted.
+*/ /**************************************************************************/
+#define DECLARE_LIST_INSERT_TAIL(TYPE) \
+void List_##TYPE##_InsertTail(TYPE **ppsHead, TYPE *psNewNode)
+
+#define IMPLEMENT_LIST_INSERT_TAIL(TYPE) \
+void List_##TYPE##_InsertTail(TYPE **ppsHead, TYPE *psNewNode)\
+{\
+	TYPE *psTempNode = *ppsHead;\
+	if (psTempNode != NULL)\
+	{\
+		while (psTempNode->psNext)\
+			psTempNode = psTempNode->psNext;\
+		ppsHead = &psTempNode->psNext;\
+	}\
+	psNewNode->ppsThis = ppsHead;\
+	psNewNode->psNext = NULL;\
+	*ppsHead = psNewNode;\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Reverse
+@Description    Reverse a list in place
+@Input          ppsHead    The pointer to the pointer to the head node.
+*/ /**************************************************************************/
+#define DECLARE_LIST_REVERSE(TYPE) \
+void List_##TYPE##_Reverse(TYPE **ppsHead)
+
+#define IMPLEMENT_LIST_REVERSE(TYPE) \
+void List_##TYPE##_Reverse(TYPE **ppsHead)\
+{\
+	TYPE *psTmpNode1; \
+	TYPE *psTmpNode2; \
+	TYPE *psCurNode; \
+	psTmpNode1 = NULL; \
+	psCurNode = *ppsHead; \
+	while (psCurNode) { \
+		psTmpNode2 = psCurNode->psNext; \
+		psCurNode->psNext = psTmpNode1; \
+		psTmpNode1 = psCurNode; \
+		psCurNode = psTmpNode2; \
+		if (psCurNode) \
+		{ \
+			psTmpNode1->ppsThis = &(psCurNode->psNext); \
+		} \
+		else \
+		{ \
+			psTmpNode1->ppsThis = ppsHead; \
+		} \
+	} \
+	*ppsHead = psTmpNode1; \
+}
+
+#define IS_LAST_ELEMENT(x) ((x)->psNext == NULL)
+
+
+DECLARE_LIST_ANY(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_ANY_2(PVRSRV_DEVICE_NODE, IMG_BOOL, IMG_FALSE);
+DECLARE_LIST_ANY_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK);
+DECLARE_LIST_ANY_VA(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_ANY_VA_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK);
+DECLARE_LIST_FOR_EACH(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_FOR_EACH_VA(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_INSERT_TAIL(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_REMOVE(PVRSRV_DEVICE_NODE);
+
+#undef DECLARE_LIST_ANY_2
+#undef DECLARE_LIST_ANY_VA
+#undef DECLARE_LIST_ANY_VA_2
+#undef DECLARE_LIST_FOR_EACH
+#undef DECLARE_LIST_FOR_EACH_VA
+#undef DECLARE_LIST_INSERT
+#undef DECLARE_LIST_REMOVE
+
+#endif
+
+/* re-enable warnings */
+/* PRQA S 0881,3410 -- */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/mmu_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/mmu_common.h
new file mode 100644
index 000000000000..d0c5e9f03aed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/mmu_common.h
@@ -0,0 +1,875 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef MMU_COMMON_H
+#define MMU_COMMON_H
+
+/*
+	The Memory Management Unit (MMU) performs device virtual to physical
+	translation.
+
+	Terminology:
+	 - page catalogue, PC	(optional, 3 tier MMU)
+	 - page directory, PD
+	 - page table, PT (can be variable sized)
+	 - data page, DP (can be variable sized)
+	Note: PD and PC are fixed size and can't be larger than the native
+	      physical (CPU) page size
+	Shifts and AlignShift variables:
+	 - 'xxxShift' represent the number of bits a bitfield is shifted left from bit0
+	 - 'xxxAlignShift' is used to convert a bitfield (based at bit0) into byte units
+	   by applying a bit shift left by 'xxxAlignShift' bits
+*/
+
+/*
+	Device Virtual Address Config:
+
+	Incoming Device Virtual Address is deconstructed into up to 4
+	fields, where the virtual address is up to 64bits:
+	MSB-----------------------------------------------LSB
+	| PC Index:   | PD Index:  | PT Index: | DP offset: |
+	| d bits      | c bits     | b-v bits  |  a+v bits  |
+	-----------------------------------------------------
+	where v is the variable page table modifier, e.g.
+			v == 0 -> 4KB DP
+			v == 2 -> 16KB DP
+			v == 4 -> 64KB DP
+			v == 6 -> 256KB DP
+			v == 8 -> 1MB DP
+			v == 10 -> 4MB DP
+*/
+
+/* services/server/include/ */
+#include "pmr.h"
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv_error.h"
+#include "servicesext.h"
+#include "sync_prim_internal.h"
+
+/*!
+	The level of the MMU
+*/
+typedef enum
+{
+	MMU_LEVEL_0 = 0,    /* Level 0 = Page */
+
+	MMU_LEVEL_1,        /* Level 1 = PT */
+	MMU_LEVEL_2,        /* Level 2 = PD */
+	MMU_LEVEL_3,        /* Level 3 = PC */
+	MMU_LEVEL_LAST
+} MMU_LEVEL;
+
+/* moved after declaration of MMU_LEVEL, as pdump_mmu.h references it */
+#include "pdump_mmu.h"
+
+#define MMU_MAX_LEVEL 3
+
+
+typedef struct _MMU_CONTEXT_ MMU_CONTEXT;
+
+
+/*
+	P(C/D/T) Entry Config:
+
+	MSB-----------------------------------------------LSB
+	| PT Addr:   | variable PT ctrl | protection flags: |
+	| bits c+v   | b bits           | a bits            |
+	-----------------------------------------------------
+	where v is the variable page table modifier and is optional
+*/
+/*!
+	Generic MMU entry description. This is used to describe PC, PD and PT entries.
+*/
+typedef struct _MMU_PxE_CONFIG_
+{
+	MMU_LEVEL		ePxLevel;        /*! MMU Level this config describes */
+	const IMG_CHAR	*pszPxLevelStr;  /*! Px string for this level */
+	IMG_UINT8		uiBytesPerEntry; /*! Size of an entry in bytes */
+
+	IMG_UINT64	uiAddrMask;      /*! Physical address mask */
+	IMG_UINT8	uiAddrShift;     /*! Physical address shift */
+	IMG_UINT8	uiAddrLog2Align; /*! Physical address Log 2 alignment */
+
+	IMG_UINT64	uiVarCtrlMask;   /*! Variable control mask */
+	IMG_UINT8	uiVarCtrlShift;  /*! Variable control shift */
+
+	IMG_UINT64	uiProtMask;      /*! Protection flags mask */
+	IMG_UINT8	uiProtShift;     /*! Protection flags shift */
+	IMG_UINT64	uiPendingEnMask; /*! Entry pending bit mask */
+	IMG_UINT64	uiValidEnMask;   /*! Entry valid bit mask */
+	IMG_UINT8	uiValidEnShift;  /*! Entry valid bit shift */
+	IMG_UINT64	uiParityBitMask;   /*! Entry parity bit mask */
+	IMG_UINT8	uiParityBitShift;  /*! Entry parity bit shift */
+} MMU_PxE_CONFIG;
+
+/*!
+	MMU virtual address split
+*/
+typedef struct _MMU_DEVVADDR_CONFIG_
+{
+	/*! Page catalogue index mask */
+	IMG_UINT64	uiPCIndexMask;
+	/*! Page catalogue index shift */
+	IMG_UINT8	uiPCIndexShift;
+	/*! Total number of PC entries */
+	IMG_UINT32	uiNumEntriesPC;
+
+	/*! Page directory mask */
+	IMG_UINT64	uiPDIndexMask;
+	/*! Page directory shift */
+	IMG_UINT8	uiPDIndexShift;
+	/*! Total number of PD entries */
+	IMG_UINT32	uiNumEntriesPD;
+
+	/*! Page table mask */
+	IMG_UINT64	uiPTIndexMask;
+	/*! Page index shift */
+	IMG_UINT8	uiPTIndexShift;
+	/*! Total number of PT entries */
+	IMG_UINT32	uiNumEntriesPT;
+
+	/*! Page offset mask */
+	IMG_UINT64	uiPageOffsetMask;
+	/*! Page offset shift */
+	IMG_UINT8	uiPageOffsetShift;
+
+	/*! First virtual address mappable for this config */
+	IMG_UINT64	uiOffsetInBytes;
+
+} MMU_DEVVADDR_CONFIG;
+
+/*!
+	MMU device attributes. This structure is the interface between the generic
+	MMU code and the device specific MMU code.
+*/
+typedef struct _MMU_DEVICEATTRIBS_
+{
+	/*! Page and address type */
+	PDUMP_MMU_TYPE eMMUType;
+
+	/*! Name string of the PDUMP memory space to use */
+	IMG_CHAR *pszMMUPxPDumpMemSpaceName;
+
+	/*! Alignment requirement of the top/base object */
+	IMG_UINT32 ui32BaseAlign;
+
+	/*! HW config of the top/base object */
+	struct _MMU_PxE_CONFIG_ *psBaseConfig;
+
+	/*! Address split for the base object */
+	const struct _MMU_DEVVADDR_CONFIG_ *psTopLevelDevVAddrConfig;
+
+	/*! Supported page sizes validation mask */
+	IMG_UINT32 ui32ValidPageSizeMask;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_UINT64* pui64PrecomputedAllocParity[2];
+#endif
+
+	/*! Callback for creating protection bits for the page catalogue entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePCEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page catalogue entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePCEProt4)(IMG_UINT32 uiProtFlags);
+	/*! Callback for creating protection bits for the page directory entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePDEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page directory entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePDEProt4)(IMG_UINT32 uiProtFlags);
+	/*! Callback for creating protection bits for the page table entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePTEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page table entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePTEProt4)(IMG_UINT32 uiProtFlags);
+
+	/*! Callback for getting the MMU configuration based on the specified page size */
+	PVRSRV_ERROR (*pfnGetPageSizeConfiguration)(IMG_UINT32 ui32DataPageSize,
+												const struct _MMU_PxE_CONFIG_ **ppsMMUPDEConfig,
+												const struct _MMU_PxE_CONFIG_ **ppsMMUPTEConfig,
+												const struct _MMU_DEVVADDR_CONFIG_ **ppsMMUDevVAddrConfig,
+												IMG_HANDLE *phPriv2);
+	/*! Callback for putting the MMU configuration obtained from pfnGetPageSizeConfiguration */
+	PVRSRV_ERROR (*pfnPutPageSizeConfiguration)(IMG_HANDLE hPriv);
+
+	/*! Callback for getting the page size from the PDE for the page table entry with 4 byte entry */
+	PVRSRV_ERROR (*pfnGetPageSizeFromPDE4)(IMG_UINT32, IMG_UINT32 *);
+	/*! Callback for getting the page size from the PDE for the page table entry with 8 byte entry */
+	PVRSRV_ERROR (*pfnGetPageSizeFromPDE8)(IMG_UINT64, IMG_UINT32 *);
+	/*! Callback for getting the page size directly from the address. Supported on MMU4 */
+	PVRSRV_ERROR (*pfnGetPageSizeFromVirtAddr)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_DEV_VIRTADDR, IMG_UINT32 *);
+
+	/*! Private data handle */
+	IMG_HANDLE hGetPageSizeFnPriv;
+} MMU_DEVICEATTRIBS;
+
+
+/* MMU Protection flags */
+
+/* These are specified generically and in a h/w independent way, and
+   are interpreted at each level (PC/PD/PT) separately. */
+
+/* The following flags are for internal use only, and should not
+   traverse the API */
+#define MMU_PROTFLAGS_INVALID 0x80000000U
+
+typedef IMG_UINT32 MMU_PROTFLAGS_T;
+
+/* The following flags should be supplied by the caller: */
+#define MMU_PROTFLAGS_READABLE					(1U<<0)
+#define MMU_PROTFLAGS_WRITEABLE					(1U<<1)
+#define MMU_PROTFLAGS_CACHE_COHERENT			(1U<<2)
+#define MMU_PROTFLAGS_CACHED					(1U<<3)
+
+/* Device specific flags*/
+#define MMU_PROTFLAGS_DEVICE_OFFSET		16
+#define MMU_PROTFLAGS_DEVICE_MASK		0x000f0000UL
+#define MMU_PROTFLAGS_DEVICE(n)	\
+			(((n) << MMU_PROTFLAGS_DEVICE_OFFSET) & \
+			MMU_PROTFLAGS_DEVICE_MASK)
+
+
+struct _PVRSRV_DEVICE_NODE_;
+
+struct _CONNECTION_DATA_;
+
+typedef struct _MMU_PAGESIZECONFIG_
+{
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT32 uiRefCount;
+	IMG_UINT32 uiMaxRefCount;
+} MMU_PAGESIZECONFIG;
+
+/*************************************************************************/ /*!
+@Function       MMU_InitDevice
+
+@Description    Creates MMU device specific resources.
+
+@Input          psDevNode               Device node of the device to create the
+                                        MMU context for
+
+@Return         PVRSRV_OK if the initialisation process was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_InitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       MMU_DeInitDevice
+
+@Description    Clean-up MMU device specific resources.
+
+@Input          psDevNode               Device node of the device
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_DeInitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextCreate
+
+@Description    Create a new MMU context
+
+@Input          psConnection            Connection requesting the MMU context
+                                        creation. Can be NULL for kernel/FW
+                                        memory context.
+@Input          psDevNode               Device node of the device to create the
+                                        MMU context for
+@Output         ppsMMUContext           The created MMU context
+
+@Return         PVRSRV_OK if the MMU context was successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ContextCreate(struct _CONNECTION_DATA_ *psConnection,
+                  struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                  MMU_CONTEXT **ppsMMUContext,
+                  MMU_DEVICEATTRIBS *psDevAttrs);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextDestroy
+
+@Description    Destroy a MMU context
+
+@Input          psMMUContext            MMU context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+MMU_ContextDestroy(MMU_CONTEXT *psMMUContext);
+
+/*************************************************************************/ /*!
+@Function       MMU_Alloc
+
+@Description    Allocate the page tables required for the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uSize                   The size of the allocation
+
+@Input          uiProtFlags             Generic MMU protection flags
+
+@Input          uDevVAddrAlignment      Alignment requirement of the virtual
+                                        allocation
+
+@Input          psDevVAddr              Virtual address to start the allocation
+                                        from
+
+@Return         PVRSRV_OK if the allocation of the page tables was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_Alloc(MMU_CONTEXT *psMMUContext,
+          IMG_DEVMEM_SIZE_T uSize,
+          IMG_UINT32 uiProtFlags,
+          IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
+          IMG_DEV_VIRTADDR *psDevVAddr,
+          IMG_UINT32 uiLog2PageSize);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_Free
+
+@Description    Free the page tables of the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddr               Virtual address to start the free
+                                        from
+
+@Input          uiSize                  The size of the allocation
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+MMU_Free(MMU_CONTEXT *psMMUContext,
+         IMG_DEV_VIRTADDR sDevVAddr,
+         IMG_DEVMEM_SIZE_T uiSize,
+         IMG_UINT32 uiLog2DataPageSize);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_MapPages
+
+@Description    Map pages to the MMU.
+                Two modes of operation: One requires a list of physical page
+                indices that are going to be mapped, the other just takes
+                the PMR and a possible offset to map parts of it.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+
+@Input          psPMR                   PMR to map
+
+@Input          ui32PhysPgOffset        Physical offset into the PMR
+
+@Input          ui32MapPageCount        Number of pages to map
+
+@Input          paui32MapIndices        List of page indices to map,
+                                         can be NULL
+
+@Input          uiLog2PageSize          Log2 page size of the pages to map
+
+@Return         PVRSRV_OK if the mapping was successful
+                PVRSRV_ERROR_RETRY if SUPPORT_LINUX_OSPAGE_MIGRATION is
+                enabled and migrate is in progress. Requests to MMU_MapPages
+                may return retry if the target PMR to map is
+                in migrate state. This is because in order for migrate
+                to complete higher locking primitives are required to give
+                way to the migrate path. Expect PVRSRV_ERROR_RETRY return
+                from this function if this give way is required.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_MapPages(MMU_CONTEXT *psMMUContext,
+             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+             IMG_DEV_VIRTADDR sDevVAddrBase,
+             PMR *psPMR,
+             IMG_UINT32 ui32PhysPgOffset,
+             IMG_UINT32 ui32MapPageCount,
+             IMG_UINT32 *paui32MapIndices,
+             IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_UnmapPages
+
+@Description    Unmap pages from the MMU.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddr               Device virtual address of the 1st page
+
+@Input          ui32PageCount           Number of pages to unmap
+
+@Input          pai32UnmapIndicies      Array of page indices to be unmapped
+
+@Input          uiLog2PageSize          log2 size of the page
+
+
+@Input          uiMemAllocFlags         Indicates if the unmapped regions need
+                                        to be backed by dummy or zero page
+
+@Return         PVRSRV_OK if the unmap operation was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_UnmapPages(MMU_CONTEXT *psMMUContext,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_DEV_VIRTADDR sDevVAddr,
+               IMG_UINT32 ui32PageCount,
+               IMG_UINT32 *pai32UnmapIndicies,
+               IMG_UINT32 uiLog2PageSize,
+               PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags);
+
+/*************************************************************************/ /*!
+@Function       MMUX_MapVRangeToBackingPage
+
+@Description    Map virtual range to a backing page in the MMU.
+                Used in DevmemX calls which don't tie a PMR to a virtual
+                range implicitly.
+
+@Input          psMMUContext            MMU context to operate on
+@Input          uiMappingFlags          Memalloc flags for the mapping
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+@Input          ui32MapPageCount        Number of pages to map
+@Input          uiLog2PageSize          Log2 page size of the pages to map
+
+@Return         PVRSRV_OK if the mapping was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMUX_MapVRangeToBackingPage(MMU_CONTEXT *psMMUContext,
+                            PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                            IMG_DEV_VIRTADDR sDevVAddrBase,
+                            IMG_UINT32 ui32MapPageCount,
+                            IMG_UINT32 uiLog2HeapPageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_MapPMRFast
+
+@Description    Map a PMR into the MMU. Must be not sparse.
+                This is supposed to cover most mappings and, as the name suggests,
+                should be as fast as possible.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddr               Device virtual address to map the PMR
+                                        into
+
+@Input          psPMR                   PMR to map
+
+@Input          uiSizeBytes             Size in bytes to map
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Return         PVRSRV_OK if the PMR was successfully mapped.
+                PVRSRV_ERROR_RETRY if SUPPORT_LINUX_OSPAGE_MIGRATION is
+                enabled and migrate is in progress. Requests to MMU_MapPages
+                may return retry if the target PMR to map is
+                in migrate state. This is because in order for migrate
+                to complete higher locking primitives are required to give
+                way to the migrate path. Expect PVRSRV_ERROR_RETRY return
+                from this function if this give way is required.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_MapPMRFast(MMU_CONTEXT *psMMUContext,
+               IMG_DEV_VIRTADDR sDevVAddr,
+               PMR *psPMR,
+               IMG_DEVMEM_SIZE_T uiSizeBytes,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_UnmapPMRFast
+
+@Description    Unmap pages from the MMU as fast as possible.
+                PMR must be non-sparse!
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+
+@Input          ui32PageCount           Number of pages to unmap
+
+@Input          uiLog2PageSize          log2 size of the page
+
+@Return         PVRSRV_OK if the PMR was successfully unmapped
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrBase,
+                 IMG_UINT32 ui32PageCount,
+                 IMG_UINT32 uiLog2PageSize);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*************************************************************************/ /*!
+@Function       MMU_RemapPage
+
+@Description    Remap a single page from a PMR in place.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddr               Device virtual address of the page
+
+@Input          uiLog2HeapPageSize      log2 size of the page
+
+@Input          psOriginPMR             PMR to remap
+
+@Input          ui32LogicalPgOffset     Page offset into the PMR of the page
+                                        to remap.
+
+@Return         PVRSRV_OK if the PMR was successfully re-mapped
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_RemapPage(MMU_CONTEXT *psMMUContext,
+              PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+              IMG_DEV_VIRTADDR sDevVAddr,
+              IMG_UINT32 uiLog2HeapPageSize,
+              PMR *psOriginPMR,
+              IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquireBaseAddr
+
+@Description    Acquire the device physical address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         psPhysAddr              Device physical address of the base level
+                                        MMU object
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquireCPUBaseAddr
+
+@Description    Acquire the CPU Virtual Address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         ppvCPUVAddr             CPU Virtual Address of the base level
+                                        MMU object
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquireCPUBaseAddr(MMU_CONTEXT *psMMUContext, void **ppvCPUVAddr);
+
+/*************************************************************************/ /*!
+@Function       MMU_ReleaseBaseAddr
+
+@Description    Release the device physical address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+void
+MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext);
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+/***********************************************************************************/ /*!
+@Function       MMU_SetOSid
+
+@Description    Set the OSid associated with the application (and the MMU Context)
+
+@Input          psMMUContext            MMU context to store the OSid on
+
+@Input          ui32OSid                the OSid in question
+
+@Input          ui32OSidReg             The value that the firmware will assign to the
+                                        registers.
+
+@Input          bOSidAxiProt            Toggles whether the AXI prot bit will be set or
+                                        not.
+@Return None
+*/
+/***********************************************************************************/
+
+void MMU_SetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32OSid,
+                  IMG_UINT32 ui32OSidReg, IMG_BOOL bOSidAxiProt);
+
+/***********************************************************************************/ /*!
+@Function       MMU_GetOSid
+
+@Description    Retrieve the OSid associated with the MMU context.
+
+@Input          psMMUContext            MMU context in which the OSid is stored
+
+@Output         pui32OSid               The OSid in question
+
+@Output         pui32OSidReg            The OSid that the firmware will assign to the
+                                        registers.
+
+@Output         pbOSidAxiProt           Toggles whether the AXI prot bit will be set or
+                                        not.
+@Return None
+*/
+/***********************************************************************************/
+
+void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 * pui32OSid,
+                  IMG_UINT32 * pui32OSidReg, IMG_BOOL *pbOSidAxiProt);
+#endif
+
+/*************************************************************************/ /*!
+@Function       MMU_AppendCacheFlags
+
+@Description    Set the cache flags to the bitwise or of themselves and the
+                specified input flags, i.e. ui32CacheFlags |= ui32NewCacheFlags,
+                atomically.
+
+@Input          psMMUContext            MMU context
+
+@Input          ui32NewCacheFlags       Cache flags to append.
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_AppendCacheFlags(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32NewCacheFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_GetAndResetCacheFlags
+
+@Description    Clears MMU context flags, atomically.
+
+@Input          psMMUContext            MMU context
+
+@Return         Previous MMU context cache flags.
+*/
+/*****************************************************************************/
+IMG_UINT32 MMU_GetAndResetCacheFlags(MMU_CONTEXT *psMMUContext);
+
+typedef struct _MMU_LEVEL_DATA_
+{
+	IMG_UINT32	ui32Index;
+	IMG_UINT32	ui32NumOfEntries;
+	const IMG_CHAR *psDebugStr;
+	IMG_UINT8	uiBytesPerEntry;
+	IMG_UINT64	ui64Address;
+} MMU_LEVEL_DATA;
+
+typedef enum _MMU_FAULT_TYPE_
+{
+	MMU_FAULT_TYPE_UNKNOWN = 0, /* If fault is not analysed by Host */
+	MMU_FAULT_TYPE_PM,
+	MMU_FAULT_TYPE_NON_PM,
+} MMU_FAULT_TYPE;
+
+typedef struct _MMU_FAULT_DATA_
+{
+	MMU_LEVEL	eTopLevel;
+	MMU_FAULT_TYPE	eType;
+	MMU_LEVEL_DATA	sLevelData[MMU_LEVEL_LAST];
+} MMU_FAULT_DATA;
+
+/*************************************************************************/ /*!
+@Function       MMU_CheckFaultAddress
+
+@Description    Check the specified MMU context to see if the provided address
+                should be valid
+
+@Input          psMMUContext            MMU context to store the data on
+
+@Input          psDevVAddr              Address to check
+
+@Output          psOutFaultData          To store fault details after checking
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
+                           IMG_DEV_VIRTADDR *psDevVAddr,
+                           MMU_FAULT_DATA *psOutFaultData);
+
+/*************************************************************************/ /*!
+@Function       MMU_IsVDevAddrValid
+@Description    Checks if given address is valid.
+@Input          psMMUContext MMU context to store the data on
+@Input          uiLog2PageSize page size
+@Input          sDevVAddr Address to check
+@Return         IMG_TRUE of address is valid
+*/ /**************************************************************************/
+IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
+                             IMG_UINT32 uiLog2PageSize,
+                             IMG_DEV_VIRTADDR sDevVAddr);
+
+#if defined(PDUMP)
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextDerivePCPDumpSymAddr
+
+@Description    Derives a PDump Symbolic address for the top level MMU object
+
+@Input          psMMUContext                    MMU context to operate on
+
+@Input          pszPDumpSymbolicNameBuffer      Buffer to write the PDump symbolic
+                                                address to
+
+@Input          uiPDumpSymbolicNameBufferSize   Size of the buffer
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
+                                IMG_CHAR *pszPDumpSymbolicNameBuffer,
+                                size_t uiPDumpSymbolicNameBufferSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_PDumpWritePageCatBase
+
+@Description    PDump write of the top level MMU object to a device register
+
+@Input          psMMUContext        MMU context to operate on
+
+@Input          pszSpaceName        PDump name of the mem/reg space
+
+@Input          uiOffset            Offset to write the address to
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                                       const IMG_CHAR *pszSpaceName,
+                                       IMG_DEVMEM_OFFSET_T uiOffset,
+                                       IMG_UINT32 ui32WordSize,
+                                       IMG_UINT32 ui32AlignShift,
+                                       IMG_UINT32 ui32Shift,
+                                       PDUMP_FLAGS_T uiPdumpFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquirePDumpMMUContext
+
+@Description    Acquire a reference to the PDump MMU context for this MMU
+                context
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         pui32PDumpMMUContextID  PDump MMU context ID
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                           IMG_UINT32 *pui32PDumpMMUContextID,
+                           IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_ReleasePDumpMMUContext
+
+@Description    Release a reference to the PDump MMU context for this MMU context
+
+@Input          psMMUContext            MMU context to operate on
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                           IMG_UINT32 ui32PDumpFlags);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(MMU_PDumpWritePageCatBase)
+#endif
+static INLINE void
+MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                          const IMG_CHAR *pszSpaceName,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32WordSize,
+                          IMG_UINT32 ui32AlignShift,
+                          IMG_UINT32 ui32Shift,
+                          PDUMP_FLAGS_T uiPdumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMMUContext);
+	PVR_UNREFERENCED_PARAMETER(pszSpaceName);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32WordSize);
+	PVR_UNREFERENCED_PARAMETER(ui32AlignShift);
+	PVR_UNREFERENCED_PARAMETER(ui32Shift);
+	PVR_UNREFERENCED_PARAMETER(uiPdumpFlags);
+}
+#endif /* PDUMP */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*************************************************************************/ /*!
+@Function       MMU_CacheInvalidateKick
+
+@Description    Kicks the Firmware to invalidate caches
+
+@Input          psDeviceNode            Pointer to the device node
+@Output         puiRequiredSyncValue    Value the associated sync prim will be
+                                        updated to after the kick is finished
+                                        (parameter ignored if NULL)
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_CacheInvalidateKick(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                     IMG_UINT32 *puiRequiredSyncValue);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#endif /* #ifdef MMU_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/opaque_types.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/opaque_types.h
new file mode 100644
index 000000000000..766bc22ea418
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/opaque_types.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          Opaque Types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines opaque types for various services types
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef SERVICES_OPAQUE_TYPES_H
+#define SERVICES_OPAQUE_TYPES_H
+
+#include "img_defs.h"
+#include "img_types.h"
+
+typedef struct _PVRSRV_DEVICE_NODE_ *PPVRSRV_DEVICE_NODE;
+typedef const struct _PVRSRV_DEVICE_NODE_ *PCPVRSRV_DEVICE_NODE;
+
+#endif /* SERVICES_OPAQUE_TYPES_H */
+
+/******************************************************************************
+ End of file (opaque_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/os_apphint.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/os_apphint.h
new file mode 100644
index 000000000000..93c9272498e5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/os_apphint.h
@@ -0,0 +1,186 @@
+/*************************************************************************/ /*!
+@File           os_apphint.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS-independent interface for retrieving apphints
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_defs.h"
+#if defined(__linux__)
+#include "km_apphint.h"
+#include "device.h"
+#else
+#include "services_client_porting.h"
+#endif
+#if !defined(OS_APPHINT_H)
+#define OS_APPHINT_H
+
+/*! Supplied to os_get_apphint_XXX() functions when the param/AppHint is
+ * applicable to all devices and not a specific device. Typically used
+ * for server-wide build and module AppHints.
+ */
+#define APPHINT_NO_DEVICE (NULL)
+
+#if defined(__linux__) && !defined(DOXYGEN)
+static INLINE IMG_UINT os_get_apphint_UINT32(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT32 *pAppHintDefault, IMG_UINT32 *pVal) {
+	return !pvr_apphint_get_uint32(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_UINT64(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT64 *pAppHintDefault, IMG_UINT64 *pVal) {
+	return !pvr_apphint_get_uint64(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_BOOL(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_BOOL *pAppHintDefault, IMG_BOOL *pVal) {
+	return !pvr_apphint_get_bool(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_STRING(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_CHAR *pAppHintDefault, IMG_CHAR *buffer, size_t size) {
+	return !pvr_apphint_get_string(device, id, buffer, size);
+}
+
+#define OSGetAppHintUINT32(device, state, name, appHintDefault, value) \
+	os_get_apphint_UINT32(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintUINT64(device, state, name, appHintDefault, value) \
+	os_get_apphint_UINT64(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintBOOL(device, state, name, appHintDefault, value) \
+	os_get_apphint_BOOL(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	os_get_apphint_STRING(device, state, APPHINT_ID_ ## name, appHintDefault, buffer, size)
+
+
+#define OSCreateAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#define OSFreeAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#else /* defined(__linux__) && !defined(DOXYGEN) */
+
+/**************************************************************************/ /*!
+@def OSGetAppHintUINT32(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint32 server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintUINT32(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintUINT64(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint64 server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintUINT64(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintBOOL(state, name, appHintDefault, value)
+@Description    Interface for retrieval of IMG_BOOL server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintBOOL(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_BOOL_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintSTRING(state, name, appHintDefault, buffer, size)
+@Description    Interface for retrieval of string server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         buffer            Buffer used to return app hint string.
+@Input			size			  Size of the buffer.
+ */ /**************************************************************************/
+#define OSGetAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	(PVR_UNREFERENCED_PARAMETER(size), PVRSRVGetAppHint(state, # name, IMG_STRING_TYPE, appHintDefault, buffer))
+
+/**************************************************************************/ /*!
+@def OSCreateAppHintState(state)
+@Description    Creates the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSCreateAppHintState(state) \
+	PVRSRVCreateAppHintState(IMG_SRV_UM, 0, state)
+
+/**************************************************************************/ /*!
+@def OSFreeAppHintState
+@Description    Free the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSFreeAppHintState(state) \
+	PVRSRVFreeAppHintState(IMG_SRV_UM, state)
+
+#endif /* defined(__linux__) */
+
+#endif /* OS_APPHINT_H */
+
+/******************************************************************************
+ End of file (os_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/osconnection_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/osconnection_server.h
new file mode 100644
index 000000000000..88f7aab29dca
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/osconnection_server.h
@@ -0,0 +1,137 @@
+/**************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    API for OS specific callbacks from server side connection
+                management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#ifndef OSCONNECTION_SERVER_H
+#define OSCONNECTION_SERVER_H
+
+#include "handle.h"
+#include "osfunc.h"
+
+/*! Function not implemented definition. */
+#define OSCONNECTION_SERVER_NOT_IMPLEMENTED 0
+/*! Assert used for OSCONNECTION_SERVER_NOT_IMPLEMENTED. */
+#define OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT() PVR_ASSERT(OSCONNECTION_SERVER_NOT_IMPLEMENTED)
+
+#if defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData);
+PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData);
+
+PVRSRV_ERROR OSConnectionSetHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase);
+
+PVRSRV_DEVICE_NODE* OSGetDevNode(CONNECTION_DATA *psConnection);
+
+struct drm_file;
+
+struct drm_file* OSGetDRMFile(CONNECTION_DATA *psConnection);
+
+#else	/* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionPrivateDataInit)
+#endif
+/*************************************************************************/ /*!
+@Function       OSConnectionPrivateDataInit
+@Description    Allocates and initialises any OS-specific private data
+                relating to a connection.
+                Called from PVRSRVCommonConnectionConnect().
+@Input          pvOSData            pointer to any OS private data
+@Output         phOsPrivateData     handle to the created connection
+                                    private data
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData)
+{
+	PVR_UNREFERENCED_PARAMETER(phOsPrivateData);
+	PVR_UNREFERENCED_PARAMETER(pvOSData);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionPrivateDataDeInit)
+#endif
+/*************************************************************************/ /*!
+@Function       OSConnectionPrivateDataDeInit
+@Description    Frees previously allocated OS-specific private data
+                relating to a connection.
+@Input          hOsPrivateData      handle to the connection private data
+                                    to be freed
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData)
+{
+	PVR_UNREFERENCED_PARAMETER(hOsPrivateData);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionSetHandleOptions)
+#endif
+static INLINE PVRSRV_ERROR OSConnectionSetHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psHandleBase);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSGetDevNode)
+#endif
+static INLINE PVRSRV_DEVICE_NODE* OSGetDevNode(CONNECTION_DATA *psConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return NULL;
+}
+#endif	/* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+
+
+#endif /* OSCONNECTION_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/osdi_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/osdi_impl.h
new file mode 100644
index 000000000000..5be78d6c2b62
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/osdi_impl.h
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions and types for creating Debug Info implementations.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSDI_IMPL_H
+#define OSDI_IMPL_H
+
+#if defined(__KERNEL__) && defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __KERNEL__ && __linux__ */
+
+#include "di_common.h"
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+/*! Implementation callbacks. Those operations are performed on native
+ * implementation handles. */
+typedef struct OSDI_IMPL_ENTRY_CB
+{
+    /*! @Function pfnWrite
+     *
+     * @Description
+     * Writes the binary data of the DI entry to the output sync, whatever that
+     * may be for the DI implementation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pvData data
+     * @Input uiSize pvData length
+     */
+    void (*pfnWrite)(void  *pvNativeHandle, const void *pvData,
+                     IMG_UINT32 uiSize);
+
+    /*! @Function pfnVPrintf
+     *
+     * @Description
+     * Implementation of the 'vprintf' operation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pszFmt NUL-terminated format string
+     * @Input va_list variable length argument list
+     */
+    __printf(2, 0)
+    void (*pfnVPrintf)(void *pvNativeHandle, const IMG_CHAR *pszFmt, va_list pArgs);
+
+    /*! @Function pfnPuts
+     *
+     * @Description
+     * Implementation of the 'puts' operation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pszStr NUL-terminated string
+     */
+    void (*pfnPuts)(void *pvNativeHandle, const IMG_CHAR *pszStr);
+
+    /*! @Function pfnHasOverflowed
+     *
+     * @Description
+     * Checks if the native implementation's buffer has overflowed.
+     *
+     * @Input pvNativeHandle native implementation handle
+     */
+    IMG_BOOL (*pfnHasOverflowed)(void *pvNativeHandle);
+} OSDI_IMPL_ENTRY_CB;
+
+/*! Debug Info entry specialisation. */
+struct OSDI_IMPL_ENTRY
+{
+    /*! Pointer to the private data. The data originates from DICreateEntry()
+     *  function. */
+    void *pvPrivData;
+    /*! Pointer to the implementation native handle. */
+    void *pvNative;
+    /*! Implementation entry callbacks. */
+    OSDI_IMPL_ENTRY_CB *psCb;
+}; /* OSDI_IMPL_ENTRY is already typedef-ed in di_common.h */
+
+/*! Debug Info implementation callbacks. */
+typedef struct OSDI_IMPL_CB
+{
+    /*! Initialise implementation callback.
+     */
+    PVRSRV_ERROR (*pfnInit)(void);
+
+    /*! De-initialise implementation callback.
+     */
+    void (*pfnDeInit)(void);
+
+    /*! @Function pfnCreateEntry
+     *
+     * @Description
+     * Creates entry of eType type with pszName in the pvNativeGroup parent
+     * group. The entry is an abstract term which depends on the implementation,
+     * e.g.: a file in DebugFS.
+     *
+     * @Input pszName: name of the entry
+     * @Input eType: type of the entry
+     * @Input psIterCb: iterator implementation for the entry
+     * @Input pvPrivData: data that will be passed to the iterator callbacks
+     *                    in OSDI_IMPL_ENTRY - it can be retrieved by calling
+     *                    DIGetPrivData() function
+     * @Input pvNativeGroup: implementation specific handle to the parent group
+     *
+     * @Output pvNativeEntry: implementation specific handle to the entry
+     *
+     * return PVRSRV_ERROR error code
+     */
+    PVRSRV_ERROR (*pfnCreateEntry)(const IMG_CHAR *pszName,
+                                   DI_ENTRY_TYPE eType,
+                                   const DI_ITERATOR_CB *psIterCb,
+                                   void *pvPrivData,
+                                   void *pvNativeGroup,
+                                   void **pvNativeEntry);
+
+    /*! @Function pfnDestroyEntry
+     *
+     * @Description
+     * Destroys native entry.
+     *
+     * @Input psNativeEntry: handle to the entry
+     */
+    void (*pfnDestroyEntry)(void *psNativeEntry);
+
+    /*! @Function pfnCreateGroup
+     *
+     * @Description
+     * Creates group with pszName in the psNativeParentGroup parent group.
+     * The group is an abstract term which depends on the implementation,
+     * e.g.: a directory in DebugFS.
+     *
+     * @Input pszName: name of the entry
+     * @Input psNativeParentGroup: implementation specific handle to the parent
+     *                             group
+     *
+     * @Output psNativeGroup: implementation specific handle to the group
+     *
+     * return PVRSRV_ERROR error code
+     */
+    PVRSRV_ERROR (*pfnCreateGroup)(const IMG_CHAR *pszName,
+                                   void *psNativeParentGroup,
+                                   void **psNativeGroup);
+
+    /*! @Function pfnDestroyGroup
+     *
+     * @Description
+     * Destroys native group.
+     *
+     * @Input psNativeGroup: handle to the group
+     */
+    void (*pfnDestroyGroup)(void *psNativeGroup);
+} OSDI_IMPL_CB;
+
+/*! @Function DIRegisterImplementation
+ *
+ * @Description
+ * Registers Debug Info implementations with the framework. The framework takes
+ * the ownership of the implementation and will clean up the resources when
+ * it's de-initialised.
+ *
+ * @Input pszName: name of the implementation
+ * @Input psImplCb: implementation callbacks
+ *
+ * @Return PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DIRegisterImplementation(const IMG_CHAR *pszName,
+                                      const OSDI_IMPL_CB *psImplCb);
+
+#endif /* OSDI_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/osfunc.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/osfunc.h
new file mode 100644
index 000000000000..2e051fcda35c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/osfunc.h
@@ -0,0 +1,1773 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS functions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifdef DEBUG_RELEASE_BUILD
+#pragma optimize( "", off )
+#define DEBUG		1
+#endif
+
+#ifndef OSFUNC_H
+/*! @cond Doxygen_Suppress */
+#define OSFUNC_H
+/*! @endcond */
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include "kernel_nospec.h"
+#if !defined(NO_HARDWARE)
+#include <linux/io.h>
+
+#endif
+#endif
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#if defined(__QNXNTO__)
+#include <stdio.h>
+#include <string.h>
+#endif
+
+#if defined(INTEGRITY_OS)
+#include <stdio.h>
+#include <string.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "device.h"
+#include "pvrsrv_device.h"
+#include "cache_ops.h"
+#include "osfunc_common.h"
+#if defined(SUPPORT_DMA_TRANSFER)
+#include "dma_km.h"
+#include "pmr.h"
+#endif
+
+#include <linux/dma-map-ops.h>
+#define MAP_UNCACHED_MEM 1
+/******************************************************************************
+ * Static defines
+ *****************************************************************************/
+/*!
+ * Returned by OSGetCurrentProcessID() and OSGetCurrentThreadID() if the OS
+ * is currently operating in the interrupt context.
+ */
+#define KERNEL_ID			0xffffffffL
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define OSConfineArrayIndexNoSpeculation(index, size) array_index_nospec((index), (size))
+#elif defined(__QNXNTO__)
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#elif defined(INTEGRITY_OS)
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#else
+/*************************************************************************/ /*!
+@Function       OSConfineArrayIndexNoSpeculation
+@Description    This macro aims to avoid code exposure to Cache Timing
+                Side-Channel Mechanisms which rely on speculative code
+                execution (Variant 1). It does so by ensuring a value to be
+                used as an array index will be set to zero if outside of the
+                bounds of the array, meaning any speculative execution of code
+                which uses this suitably adjusted index value will not then
+                attempt to load data from memory outside of the array bounds.
+                Code calling this macro must still first verify that the
+                original unmodified index value is within the bounds of the
+                array, and should then only use the modified value returned
+                by this function when accessing the array itself.
+                NB. If no OS-specific implementation of this macro is
+                defined, the original index is returned unmodified and no
+                protection against the potential exploit is provided.
+@Input          index    The original array index value that would be used to
+                         access the array.
+@Input          size     The number of elements in the array being accessed.
+@Return         The value to use for the array index, modified so that it
+                remains within array bounds.
+*/ /**************************************************************************/
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#if !defined(DOXYGEN)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#endif
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSClockns64
+@Description    This function returns the number of ticks since system boot
+                expressed in nanoseconds. Unlike OSClockns, OSClockns64 has
+                a near 64-bit range.
+@Return         The 64-bit clock value, in nanoseconds.
+*/ /**************************************************************************/
+IMG_UINT64 OSClockns64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockus64
+@Description    This function returns the number of ticks since system boot
+                expressed in microseconds. Unlike OSClockus, OSClockus64 has
+                a near 64-bit range.
+@Return         The 64-bit clock value, in microseconds.
+*/ /**************************************************************************/
+IMG_UINT64 OSClockus64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockus
+@Description    This function returns the number of ticks since system boot
+                in microseconds.
+@Return         The 32-bit clock value, in microseconds.
+*/ /**************************************************************************/
+IMG_UINT32 OSClockus(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockms
+@Description    This function returns the number of ticks since system boot
+                in milliseconds.
+@Return         The 32-bit clock value, in milliseconds.
+*/ /**************************************************************************/
+IMG_UINT32 OSClockms(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicns64
+@Description    This function returns a clock value based on the system
+                monotonic clock.
+@Output         pui64Time     The 64-bit clock value, in nanoseconds.
+@Return         Error Code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSClockMonotonicns64(IMG_UINT64 *pui64Time);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicus64
+@Description    This function returns a clock value based on the system
+                monotonic clock.
+@Output         pui64Time     The 64-bit clock value, in microseconds.
+@Return         Error Code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSClockMonotonicus64(IMG_UINT64 *pui64Time);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicRawns64
+@Description    This function returns a clock value based on the system
+                monotonic raw clock.
+@Return         64bit ns timestamp
+*/ /**************************************************************************/
+IMG_UINT64 OSClockMonotonicRawns64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicRawus64
+@Description    This function returns a clock value based on the system
+                monotonic raw clock.
+@Return         64bit us timestamp
+*/ /**************************************************************************/
+IMG_UINT64 OSClockMonotonicRawus64(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageSize
+@Description    This function returns the page size.
+                If the OS is not using memory mappings it should return a
+                default value of 4096.
+@Return         The size of a page, in bytes.
+*/ /**************************************************************************/
+size_t OSGetPageSize(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageShift
+@Description    This function returns the page size expressed as a power of
+                two. A number of pages, left-shifted by this value, gives the
+                equivalent size in bytes.
+                If the OS is not using memory mappings it should return a
+                default value of 12.
+@Return         The page size expressed as a power of two.
+*/ /**************************************************************************/
+size_t OSGetPageShift(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageMask
+@Description    This function returns a bitmask that may be applied to an
+                address to mask off the least-significant bits so as to
+                leave the start address of the page containing that address.
+@Return         The page mask.
+*/ /**************************************************************************/
+size_t OSGetPageMask(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetOrder
+@Description    This function returns the order of power of two for a given
+                size. Eg. for a uSize of 4096 bytes the function would
+                return 12 (4096 = 2^12).
+@Input          uSize     The size in bytes.
+@Return         The order of power of two.
+*/ /**************************************************************************/
+size_t OSGetOrder(size_t uSize);
+
+/*************************************************************************/ /*!
+@Function       OSGetRAMSize
+@Description    This function returns the total amount of GPU-addressable
+                memory provided by the system. In other words, after loading
+                the driver this would be the largest allocation an
+                application would reasonably expect to be able to make.
+                Note that this is function is not expected to return the
+                current available memory but the amount which would be
+                available on startup.
+@Return         Total GPU-addressable memory size, in bytes.
+*/ /**************************************************************************/
+IMG_UINT64 OSGetRAMSize(void);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a Mid-level Interrupt Service Routine (MISR).
+@Input  pvData  Pointer to MISR specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_MISR)(void *pvData);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a thread entry point function.
+@Input  pvData  Pointer to thread specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_THREAD)(void *pvData);
+
+/*************************************************************************/ /*!
+@Function       OSInstallMISR
+@Description    Installs a Mid-level Interrupt Service Routine (MISR)
+                which handles higher-level processing of interrupts from
+                the device (GPU).
+                An MISR runs outside of interrupt context, and so may be
+                descheduled. This means it can contain code that would
+                not be permitted in the LISR.
+                An MISR is invoked when OSScheduleMISR() is called. This
+                call should be made by installed LISR once it has completed
+                its interrupt processing.
+                Multiple MISRs may be installed by the driver to handle
+                different causes of interrupt.
+@Input          pfnMISR       pointer to the function to be installed
+                              as the MISR
+@Input          hData         private data provided to the MISR
+@Input          pszMisrName   Name describing purpose of MISR worker thread
+                              (Must be a string literal).
+@Output         hMISRData     handle to the installed MISR (to be used
+                              for a subsequent uninstall)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInstallMISR(IMG_HANDLE *hMISRData,
+							PFN_MISR pfnMISR,
+							void *hData,
+							const IMG_CHAR *pszMisrName);
+
+/*************************************************************************/ /*!
+@Function       OSUninstallMISR
+@Description    Uninstalls a Mid-level Interrupt Service Routine (MISR).
+@Input          hMISRData     handle to the installed MISR
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSUninstallMISR(IMG_HANDLE hMISRData);
+
+/*************************************************************************/ /*!
+@Function       OSScheduleMISR
+@Description    Schedules a Mid-level Interrupt Service Routine (MISR) to be
+                executed. An MISR should be executed outside of interrupt
+                context, for example in a work queue.
+@Input          hMISRData     handle to the installed MISR
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSScheduleMISR(IMG_HANDLE hMISRData);
+
+/*************************************************************************/ /*!
+@Function       OSSyncIRQ
+@Description    Wait for LISR to complete. If you use this function while
+                holding a resource which the IRQ handler also requires,
+                you will deadlock.
+@Input          ui32IRQ     IRQ number
+*/ /**************************************************************************/
+void OSSyncIRQ(IMG_UINT32 ui32IRQ);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a function implementing debug dump of thread-specific
+                data.
+@Input          pfnDumpDebugPrintf      Used to specify the print function used
+                                        to dump any debug information. If this
+                                        argument is NULL then a default print
+                                        function will be used.
+@Input          pvDumpDebugFile         File identifier to be passed to the
+                                        print function if specified.
+*/ /**************************************************************************/
+
+typedef void (*PFN_THREAD_DEBUG_DUMP)(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                                      void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSThreadCreate
+@Description    Creates a kernel thread and starts it running. The caller
+                is responsible for informing the thread that it must finish
+                and return from the pfnThread function. It is not possible
+                to kill or terminate it. The new thread runs with the default
+                priority provided by the Operating System.
+                Note: Kernel threads are freezable which means that they
+                can be frozen by the kernel on for example driver suspend.
+                Because of that only OSEventObjectWaitKernel() function should
+                be used to put kernel threads in waiting state.
+@Output         phThread            Returned handle to the thread.
+@Input          pszThreadName       Name to assign to the thread.
+@Input          pfnThread           Thread entry point function.
+@Input          pfnDebugDumpCB      Used to dump info of the created thread
+@Input          bIsSupportingThread Set, if summary of this thread needs to
+                                    be dumped in debug_dump
+@Input          hData               Thread specific data pointer for pfnThread().
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR OSThreadCreate(IMG_HANDLE *phThread,
+                            IMG_CHAR *pszThreadName,
+                            PFN_THREAD pfnThread,
+                            PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                            IMG_BOOL bIsSupportingThread,
+                            void *hData);
+
+/*! Available priority levels for the creation of a new Kernel Thread. */
+typedef enum priority_levels
+{
+	OS_THREAD_NOSET_PRIORITY = 0,   /* With this option the priority level is the default for the given OS */
+	OS_THREAD_HIGHEST_PRIORITY,
+	OS_THREAD_HIGH_PRIORITY,
+	OS_THREAD_NORMAL_PRIORITY,
+	OS_THREAD_LOW_PRIORITY,
+	OS_THREAD_LOWEST_PRIORITY,
+	OS_THREAD_LAST_PRIORITY     /* This must be always the last entry */
+} OS_THREAD_LEVEL;
+
+/*************************************************************************/ /*!
+@Function       OSThreadCreatePriority
+@Description    As OSThreadCreate, this function creates a kernel thread and
+                starts it running. The difference is that with this function
+                is possible to specify the priority used to schedule the new
+                thread.
+
+@Output         phThread            Returned handle to the thread.
+@Input          pszThreadName       Name to assign to the thread.
+@Input          pfnThread           Thread entry point function.
+@Input          pfnDebugDumpCB      Used to dump info of the created thread
+@Input          bIsSupportingThread Set, if summary of this thread needs to
+                                    be dumped in debug_dump
+@Input          hData               Thread specific data pointer for pfnThread().
+@Input          eThreadPriority     Priority level to assign to the new thread.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSThreadCreatePriority(IMG_HANDLE *phThread,
+                                    IMG_CHAR *pszThreadName,
+                                    PFN_THREAD pfnThread,
+                                    PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                                    IMG_BOOL bIsSupportingThread,
+                                    void *hData,
+                                    OS_THREAD_LEVEL eThreadPriority);
+
+/*************************************************************************/ /*!
+@Function       OSThreadDestroy
+@Description    Waits for the thread to end and then destroys the thread
+                handle memory. This function will block and wait for the
+                thread to finish successfully, thereby providing a sync point
+                for the thread completing its work. No attempt is made to kill
+                or otherwise terminate the thread.
+@Input          hThread   The thread handle returned by OSThreadCreate().
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSThreadDestroy(IMG_HANDLE hThread);
+
+/*************************************************************************/ /*!
+@Function       OSIsMapPhysNonContigSupported
+@Description    Determine if the kernel mapping of physically non-contiguous
+                pages is supported in the OS layer
+
+                Note: For this function to return IMG_TRUE a full implementation
+                of the following functions is required:
+                OSMapPhysArrayToLin
+                OSUnMapPhysArrayToLin
+
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+IMG_BOOL OSIsMapPhysNonContigSupported(void);
+
+/*************************************************************************/ /*!
+@Function       OSUnMapPhysArrayToLin
+@Description    UnMap a kernel virtual address that was produced by mapping
+                a number of Pages in OSMapPhysArrayToLin.
+
+                Note: This function is only required if the Non contiguous
+                allocation feature is required, in this case
+                OSIsMapPhysNonContigSupported should return IMG_TRUE.
+                If not required this function should return
+                PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED
+
+@Input          pvLinAddr       The linear mapping to be unmapped
+@Input          pvPrivData      Optional implementation specific data.
+
+@Return         None
+*/ /**************************************************************************/
+void OSUnMapPhysArrayToLin(void *pvLinAddr, void *pvPrivData);
+
+/*************************************************************************/ /*!
+@Function       OSMapPhysArrayToLin
+@Description    Given an array of OS page physical addresses and a count
+                of said Pages, this function will map those pages into a
+                virtually contiguous range, this allows for non physically
+                contiguous allocations to be mapped into the kernel.
+                Page size is assumed to be OS page size.
+
+                Note: This function is only required if the Non contiguous
+                allocation feature is required, in this case
+                OSIsMapPhysNonContigSupported should return IMG_TRUE.
+                If not required this function should return
+                PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED
+
+@Input          pPagePA          Array of Pages
+@Input          uiPageCount      Page count of pulPages
+@Output         ppvLinAddr       Pointer to a virtual kernel address of the
+                                 mapped Pages.
+@Output         ppvPrivData      Optional implementation specific data.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSMapPhysArrayToLin(IMG_CPU_PHYADDR pPagePA[],
+                                 IMG_UINT32 uiPageCount,
+                                 void **ppvLinAddr,
+                                 void **ppvPrivData);
+
+/*************************************************************************/ /*!
+@Function       OSMapPhysToLin
+@Description    Maps physical memory into a linear address range.
+@Input          BasePAddr    physical CPU address
+@Input          ui32Bytes    number of bytes to be mapped
+@Input          uiFlags      flags denoting the caching mode to be employed
+                             for the mapping (uncached/write-combined,
+                             cached coherent or cached incoherent).
+                             See pvrsrv_memallocflags.h for full flag bit
+                             definitions.
+@Return         Pointer to the new mapping if successful, NULL otherwise.
+*/ /**************************************************************************/
+void *OSMapPhysToLin(IMG_CPU_PHYADDR BasePAddr, size_t ui32Bytes, PVRSRV_MEMALLOCFLAGS_T uiFlags);
+
+/*************************************************************************/ /*!
+@Function       OSUnMapPhysToLin
+@Description    Unmaps physical memory previously mapped by OSMapPhysToLin().
+@Input          pvLinAddr    the linear mapping to be unmapped
+@Input          ui32Bytes    number of bytes to be unmapped
+@Return         IMG_TRUE if unmapping was successful, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSUnMapPhysToLin(void *pvLinAddr, size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheFlushRangeKM
+@Description    Clean and invalidate the CPU cache for the specified
+                address range.
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              flushed
+@Input          pvVirtEnd     virtual end address of the range to be
+                              flushed
+@Input          sCPUPhysStart physical start address of the range to be
+                              flushed
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              flushed
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheCleanRangeKM
+@Description    Clean the CPU cache for the specified address range.
+                This writes out the contents of the cache and clears the
+                'dirty' bit (which indicates the physical memory is
+                consistent with the cache contents).
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              cleaned
+@Input          pvVirtEnd     virtual end address of the range to be
+                              cleaned
+@Input          sCPUPhysStart physical start address of the range to be
+                              cleaned
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              cleaned
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheInvalidateRangeKM
+@Description    Invalidate the CPU cache for the specified address range.
+                The cache must reload data from those addresses if they
+                are accessed.
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              invalidated
+@Input          pvVirtEnd     virtual end address of the range to be
+                              invalidated
+@Input          sCPUPhysStart physical start address of the range to be
+                              invalidated
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              invalidated
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*! CPU Cache operations address domain type */
+typedef enum
+{
+	OS_CACHE_OP_ADDR_TYPE_VIRTUAL,    /*!< Operation requires CPU virtual address only */
+	OS_CACHE_OP_ADDR_TYPE_PHYSICAL,   /*!< Operation requires CPU physical address only */
+	OS_CACHE_OP_ADDR_TYPE_BOTH        /*!< Operation requires both CPU virtual & physical addresses */
+} OS_CACHE_OP_ADDR_TYPE;
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheOpAddressType
+@Description    Returns the address type (i.e. virtual/physical/both) the CPU
+                architecture performs cache maintenance operations under.
+                This is used to infer whether the virtual or physical address
+                supplied to the OSCPUCacheXXXRangeKM functions can be omitted
+                when called.
+@Input          psDevNode   device on which the allocation was made
+@Return         OS_CACHE_OP_ADDR_TYPE
+*/ /**************************************************************************/
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*! CPU Cache attributes available for retrieval, DCache unless specified */
+typedef enum _OS_CPU_CACHE_ATTRIBUTE_
+{
+	OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE, /*!< The cache line size */
+	OS_CPU_CACHE_ATTRIBUTE_COUNT      /*!< The number of attributes (must be last) */
+} OS_CPU_CACHE_ATTRIBUTE;
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheAttributeSize
+@Description    Returns the size of a given cache attribute.
+                Typically this function is used to return the cache line
+                size, but may be extended to return the size of other
+                cache attributes.
+@Input          eCacheAttribute   the cache attribute whose size should
+                                  be returned.
+@Return         The size of the specified cache attribute, in bytes.
+*/ /**************************************************************************/
+IMG_UINT32 OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE eCacheAttribute);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessID
+@Description    Returns ID of current process (thread group)
+@Return         ID of current process
+*****************************************************************************/
+IMG_PID OSGetCurrentProcessID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentVirtualProcessID
+@Description    Returns ID of current process (thread group of current
+                PID namespace)
+@Return         ID of current process in PID namespace
+*****************************************************************************/
+IMG_PID OSGetCurrentVirtualProcessID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessName
+@Description    Gets the name of current process
+@Return         Process name
+*****************************************************************************/
+IMG_CHAR *OSGetCurrentProcessName(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessVASpaceSize
+@Description    Returns the CPU virtual address space size of current process
+@Return         Process VA space size
+*/ /**************************************************************************/
+IMG_UINT64 OSGetCurrentProcessVASpaceSize(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentThreadID
+@Description    Returns ID for current thread
+@Return         ID of current thread
+*****************************************************************************/
+uintptr_t OSGetCurrentThreadID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientProcessIDKM
+@Description    Returns ID of current client process (thread group) which
+                has made a bridge call into the server.
+                For some operating systems, this may simply be the current
+                process id. For others, it may be that a dedicated thread
+                is used to handle the processing of bridge calls and that
+                some additional processing is required to obtain the ID of
+                the client process making the bridge call.
+@Return         ID of current client process
+*****************************************************************************/
+IMG_PID OSGetCurrentClientProcessIDKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientProcessNameKM
+@Description    Gets the name of current client process
+@Return         Client process name
+*****************************************************************************/
+IMG_CHAR *OSGetCurrentClientProcessNameKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSAcquireCurrentPPIDResourceRefKM
+@Description    Returns a unique process identifier for the current client
+                parent process (thread group) and takes a reference on it
+                (if required) to prevent it being freed/re-allocated.
+                This value may then be used as a unique reference to the
+                process rather than using the PID value which might be
+                reallocated to represent a further process on process
+                destruction.
+                Note that the value to be returned is an address relating to
+                the parent process (thread group) and not to just one thread.
+                It is the caller's responsibility to ensure the reference is
+                subsequently dropped (by calling OSReleasePPIDResourceRefKM())
+                to allow it to be freed when no longer required.
+@Return         Address of a kernel resource allocated for the current client
+                parent process (thread group)
+*****************************************************************************/
+uintptr_t OSAcquireCurrentPPIDResourceRefKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSReleasePPIDResourceRefKM
+@Description    Drops a reference on the unique process identifier provided.
+@Return         None
+*****************************************************************************/
+void OSReleasePPIDResourceRefKM(uintptr_t psPPIDResource);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientThreadIDKM
+@Description    Returns ID for current client thread
+                For some operating systems, this may simply be the current
+                thread id. For others, it may be that a dedicated thread
+                is used to handle the processing of bridge calls and that
+                some additional processing is require to obtain the ID of
+                the client thread making the bridge call.
+@Return         ID of current client thread
+*****************************************************************************/
+uintptr_t OSGetCurrentClientThreadIDKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSMemCmp
+@Description    Compares two blocks of memory for equality.
+@Input          pvBufA      Pointer to the first block of memory
+@Input          pvBufB      Pointer to the second block of memory
+@Input          uiLen       The number of bytes to be compared
+@Return         Value < 0 if pvBufA is less than pvBufB.
+                Value > 0 if pvBufB is less than pvBufA.
+                Value = 0 if pvBufA is equal to pvBufB.
+*****************************************************************************/
+IMG_INT OSMemCmp(void *pvBufA, void *pvBufB, size_t uiLen);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesAlloc
+@Description    Allocates a number of contiguous physical pages.
+                If allocations made by this function are CPU cached then
+                OSPhyContigPagesClean has to be implemented to write the
+                cached data to memory.
+@Input          psPhysHeap    the heap from which to allocate
+@Input          uiSize        the size of the required allocation (in bytes)
+@Output         psMemHandle   a returned handle to be used to refer to this
+                              allocation
+@Output         psDevPAddr    the physical address of the allocation
+@Input          uiPid         the process ID that this allocation should
+                              be associated with
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+							PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+							IMG_PID uiPid);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesFree
+@Description    Frees a previous allocation of contiguous physical pages
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be freed
+@Return         None.
+*****************************************************************************/
+void OSPhyContigPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesMap
+@Description    Maps the specified allocation of contiguous physical pages
+                to a kernel virtual address
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be mapped
+@Input          uiSize        the size of the allocation (in bytes)
+@Input          psDevPAddr    the physical address of the allocation
+@Output         pvPtr         the virtual kernel address to which the
+                              allocation is now mapped
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle,
+						size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+						void **pvPtr);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesUnmap
+@Description    Unmaps the kernel mapping for the specified allocation of
+                contiguous physical pages
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be unmapped
+@Input          pvPtr         the virtual kernel address to which the
+                              allocation is currently mapped
+@Return         None.
+*****************************************************************************/
+void OSPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesClean
+@Description    Write the content of the specified allocation from CPU cache to
+                memory from (start + uiOffset) to (start + uiOffset + uiLength)
+                It is expected to be implemented as a cache clean operation but
+                it is allowed to fall back to a cache clean + invalidate
+                (i.e. flush).
+                If allocations returned by OSPhyContigPagesAlloc are always
+                uncached this can be implemented as nop.
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be flushed
+@Input          uiOffset      the offset in bytes from the start of the
+                              allocation from where to start flushing
+@Input          uiLength      the amount to flush from the offset in bytes
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_UINT32 uiOffset,
+                                   IMG_UINT32 uiLength);
+
+
+/*************************************************************************/ /*!
+@Function       OSInitEnvData
+@Description    Called to initialise any environment-specific data. This
+                could include initialising the bridge calling infrastructure
+                or device memory management infrastructure.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInitEnvData(void);
+
+/*************************************************************************/ /*!
+@Function       OSDeInitEnvData
+@Description    The counterpart to OSInitEnvData(). Called to free any
+                resources which may have been allocated by OSInitEnvData().
+@Return         None.
+*/ /**************************************************************************/
+void OSDeInitEnvData(void);
+
+/*************************************************************************/ /*!
+@Function       OSStringLCat
+@Description    OS function to support the BSD C strlcat() function.
+*/ /**************************************************************************/
+size_t OSStringLCat(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDstSize);
+
+/*************************************************************************/ /*!
+@Function       OSSNPrintf
+@Description    OS function to support the standard C snprintf() function.
+@Output         pStr        char array to print into
+@Input          ui32Size    maximum size of data to write (chars)
+@Input          pszFormat   format string
+*/ /**************************************************************************/
+IMG_INT32 OSSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR *pszFormat, ...) __printf(3, 4);
+
+/*************************************************************************/ /*!
+@Function       OSVSNPrintf
+@Description    Printf to IMG string using variable args (see stdarg.h).
+                This is necessary because the '...' notation does not
+                support nested function calls.
+@Input          ui32Size           maximum size of data to write (chars)
+@Input          pszFormat          format string
+@Input          vaArgs             variable args structure (from stdarg.h)
+@Output         pStr               char array to print into
+@Return         Number of character written in buffer if successful other wise -1 on error
+*/ /**************************************************************************/
+IMG_INT32 OSVSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR* pszFormat, va_list vaArgs) __printf(3, 0);
+
+/*************************************************************************/ /*!
+@Function       OSStringLength
+@Description    OS function to support the standard C strlen() function.
+*/ /**************************************************************************/
+size_t OSStringLength(const IMG_CHAR *pStr);
+
+/*************************************************************************/ /*!
+@Function       OSStringNLength
+@Description    Return the length of a string, excluding the terminating null
+                byte ('\0'), but return at most 'uiCount' bytes. Only the first
+                'uiCount' bytes of 'pStr' are interrogated.
+@Input          pStr     pointer to the string
+@Input          uiCount  the maximum length to return
+@Return         Length of the string if less than 'uiCount' bytes, otherwise
+                'uiCount'.
+*/ /**************************************************************************/
+size_t OSStringNLength(const IMG_CHAR *pStr, size_t uiCount);
+
+/*************************************************************************/ /*!
+@Function       OSStringNCompare
+@Description    OS function to support the standard C strncmp() function.
+*/ /**************************************************************************/
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                           size_t uiSize);
+
+/*************************************************************************/ /*!
+@Function       OSStringToUINT32
+@Description    Changes string to IMG_UINT32.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSStringToUINT32(const IMG_CHAR *pStr, IMG_UINT32 ui32Base,
+                              IMG_UINT32 *ui32Result);
+
+/*************************************************************************/ /*!
+@Function       OSStringUINT32ToStr
+@Description    Changes IMG_UINT32 to string
+@Input          pszBuf         Buffer to write output number string
+@Input          uSize          Size of buffer provided, i.e. size of pszBuf
+@Input          ui32Num        Number to convert to string
+@Return         Returns 0 if buffer is not sufficient to hold the number string,
+                else returns length of number string
+*/ /**************************************************************************/
+IMG_UINT32 OSStringUINT32ToStr(IMG_CHAR *pszBuf, size_t uSize, IMG_UINT32 ui32Num);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectCreate
+@Description    Create an event object.
+@Input          pszName         name to assign to the new event object.
+@Output         EventObject     the created event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectCreate(const IMG_CHAR *pszName,
+								 IMG_HANDLE *EventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectDestroy
+@Description    Destroy an event object.
+@Input          hEventObject    the event object to destroy.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectDestroy(IMG_HANDLE hEventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectSignal
+@Description    Signal an event object. Any thread waiting on that event
+                object will be woken.
+@Input          hEventObject    the event object to signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectSignal(IMG_HANDLE hEventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWait
+@Description    Wait for an event object to signal. The function is passed
+                an OS event object handle (which allows the OS to have the
+                calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after a default timeout
+                period (defined in EVENT_OBJECT_TIMEOUT_MS), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+
+
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectWait(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWaitKernel
+@Description    Wait for an event object to signal. The function is passed
+                an OS event object handle (which allows the OS to have the
+                calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after a default timeout
+                period (defined in EVENT_OBJECT_TIMEOUT_MS), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+
+                Note: This function should be used only by kernel thread.
+                This is because all kernel threads are freezable and
+                this function allows the kernel to freeze the threads
+                when waiting.
+
+                See OSEventObjectWait() for more details.
+
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__)
+PVRSRV_ERROR OSEventObjectWaitKernel(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus);
+#else
+#define OSEventObjectWaitKernel OSEventObjectWaitTimeout
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSSuspendTaskInterruptible
+@Description    Suspend the current task into interruptible state.
+@Return         none.
+*/ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__)
+void OSSuspendTaskInterruptible(void);
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWaitTimeout
+@Description    Wait for an event object to signal or timeout. The function
+                is passed an OS event object handle (which allows the OS to
+                have the calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after the specified
+                timeout period (passed in 'uiTimeoutus'), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Input          uiTimeoutus   the timeout period (in usecs)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectWaitTimeout(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectDumpDebugInfo
+@Description    Emits debug counters/stats related to the event object passed
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         None.
+*/ /**************************************************************************/
+void OSEventObjectDumpDebugInfo(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectOpen
+@Description    Open an OS handle on the specified event object.
+                This OS handle may then be used to make a thread wait for
+                that event object to signal.
+@Input          hEventObject    Event object handle.
+@Output         phOSEvent       OS handle to the returned event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectOpen(IMG_HANDLE hEventObject,
+											IMG_HANDLE *phOSEvent);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectClose
+@Description    Close an OS handle previously opened for an event object.
+@Input          hOSEventKM      OS event object handle to close.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectClose(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function      OSWaitus
+@Description   Implements a busy wait of the specified number of microseconds.
+               This function does NOT release thread quanta.
+@Input         ui32Timeus     The duration of the wait period (in us)
+@Return        None.
+*/ /**************************************************************************/
+void OSWaitus(IMG_UINT32 ui32Timeus);
+
+/*************************************************************************/ /*!
+@Function       OSSleepms
+@Description    Implements a sleep of the specified number of milliseconds.
+                This function may allow pre-emption, meaning the thread
+                may potentially not be rescheduled for a longer period.
+@Input          ui32Timems    The duration of the sleep (in ms)
+@Return         None.
+*/ /**************************************************************************/
+void OSSleepms(IMG_UINT32 ui32Timems);
+
+/*************************************************************************/ /*!
+@Function       OSReleaseThreadQuanta
+@Description    Relinquishes the current thread's execution time-slice,
+                permitting the OS scheduler to schedule another thread.
+@Return         None.
+*/ /**************************************************************************/
+void OSReleaseThreadQuanta(void);
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define OSReadMemoryBarrier() rmb()
+#else
+/*************************************************************************/ /*!
+@Function       OSReadMemoryBarrier
+@Description    Insert a read memory barrier.
+                The read memory barrier guarantees that all load (read)
+                operations specified before the barrier will appear to happen
+                before all of the load operations specified after the barrier.
+*/ /**************************************************************************/
+void OSReadMemoryBarrier(void);
+#endif
+/*************************************************************************/ /*!
+@Function       OSMemoryBarrier
+@Description    Insert a read/write memory barrier.
+                The read and write memory barrier guarantees that all load
+                (read) and all store (write) operations specified before the
+                barrier will appear to happen before all of the load/store
+                operations specified after the barrier.
+@Input          hReadback     Optional pointer to memory to read back, can be
+                useful for flushing queues in bus interconnects to RAM before
+                device (GPU) access the shared memory.
+@Return         None.
+*/ /**************************************************************************/
+void OSMemoryBarrier(volatile void *hReadback);
+/*************************************************************************/ /*!
+@Function       OSWriteMemoryBarrier
+@Description    Insert a write memory barrier.
+                The write memory barrier guarantees that all store operations
+                (writes) specified before the barrier will appear to happen
+                before all of the store operations specified after the barrier.
+@Input          hReadback     Optional pointer to memory to read back, can be
+                useful for flushing queues in bus interconnects to RAM before
+                device (GPU) access the shared memory.
+@Return         None.
+*/ /**************************************************************************/
+void OSWriteMemoryBarrier(volatile void *hReadback);
+
+/*************************************************************************/ /*!
+*/ /**************************************************************************/
+
+/* The access method is dependent on the location of the physical memory that
+ * makes up the PhyHeaps defined for the system and the CPU architecture. These
+ * macros may change in future to accommodate different access requirements.
+ */
+/*! Performs a 32 bit word read from the device memory. */
+#define OSReadDeviceMem32(addr)        (*((volatile IMG_UINT32 __force *)((void*)addr)))
+/*! Performs a 32 bit word write to the device memory. */
+#define OSWriteDeviceMem32(addr, val)  (*((volatile IMG_UINT32 __force *)((void*)addr)) = (IMG_UINT32)(val))
+/*! Performs a 32 bit word write to the device memory and issues a write memory barrier */
+#define OSWriteDeviceMem32WithWMB(addr, val) \
+	do { \
+		*((volatile IMG_UINT32 __force *)((void*)addr)) = (IMG_UINT32)(val); \
+		OSWriteMemoryBarrier(addr); \
+	} while (0)
+
+#if defined(NO_HARDWARE)
+	/* OSReadHWReg and OSWriteHWReg operations are skipped to no-op in nohw builds */
+	#define OSReadUncheckedHWReg32(addr, off) ((void)(addr), (void)(off), 0x30f73a4eU)
+#if defined(__QNXNTO__) && __SIZEOF_LONG__ == 8
+	/* This is needed for 64-bit QNX builds where the size of a long is 64 bits */
+	#define OSReadUncheckedHWReg64(addr, off) ((void)(addr), (void)(off), 0x5b376c9d30f73a4eUL)
+#else
+	#define OSReadUncheckedHWReg64(addr, off) ((void)(addr), (void)(off), 0x5b376c9d30f73a4eULL)
+#endif
+
+	#define OSWriteUncheckedHWReg32(addr, off, val) ((void)(addr), (void)(off), (void)(val))
+	#define OSWriteUncheckedHWReg64(addr, off, val) ((void)(addr), (void)(off), (void)(val))
+
+	#define OSReadHWReg32(addr, off) OSReadUncheckedHWReg32(addr, off)
+	#define OSReadHWReg64(addr, off) OSReadUncheckedHWReg64(addr, off)
+
+	#define OSWriteHWReg32(addr, off, val) OSWriteUncheckedHWReg32(addr, off, val)
+	#define OSWriteHWReg64(addr, off, val) OSWriteUncheckedHWReg64(addr, off, val)
+
+#else
+
+#if defined(__linux__) && defined(__KERNEL__)
+	#define OSReadUncheckedHWReg32(addr, off) ((IMG_UINT32)readl((IMG_BYTE __iomem *)(addr) + (off)))
+
+	/* Little endian support only */
+	#define OSReadUncheckedHWReg64(addr, off) \
+			({ \
+				__typeof__(addr) _addr = addr; \
+				__typeof__(off) _off = off; \
+				(IMG_UINT64) \
+				( \
+					( (IMG_UINT64)(readl((IMG_BYTE __iomem *)(_addr) + (_off) + 4)) << 32) \
+					| readl((IMG_BYTE __iomem *)(_addr) + (_off)) \
+				); \
+			})
+
+	#define OSWriteUncheckedHWReg32(addr, off, val) writel((IMG_UINT32)(val), (IMG_BYTE __iomem *)(addr) + (off))
+	/* Little endian support only */
+	#define OSWriteUncheckedHWReg64(addr, off, val) do \
+			{ \
+				__typeof__(addr) _addr = addr; \
+				__typeof__(off) _off = off; \
+				__typeof__(val) _val = val; \
+				writel((IMG_UINT32)((_val) & 0xffffffff), (IMG_BYTE __iomem *)(_addr) + (_off)); \
+				writel((IMG_UINT32)(((IMG_UINT64)(_val) >> 32) & 0xffffffff), (IMG_BYTE __iomem *)(_addr) + (_off) + 4); \
+			} while (0)
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+/*************************************************************************/ /*!
+@Function       OSReadUncheckedHWReg32
+@Description    Read from a 32-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to read from a location
+                but instead returns a constant value.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be read.
+@Return         The long word read.
+*/ /**************************************************************************/
+	IMG_UINT32 OSReadUncheckedHWReg32(volatile void *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset);
+
+/*************************************************************************/ /*!
+@Function       OSReadUncheckedHWReg64
+@Description    Read from a 64-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to read from a location
+                but instead returns a constant value.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be read.
+@Return         The long long word read.
+*/ /**************************************************************************/
+	IMG_UINT64 OSReadUncheckedHWReg64(volatile void *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset);
+/*************************************************************************/ /*!
+@Function       OSWriteUncheckedHWReg32
+@Description    Write to a 32-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to write to a location.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be written to.
+@Input          ui32Value          The long word to be written to the register.
+@Return         None.
+*/ /**************************************************************************/
+	void OSWriteUncheckedHWReg32(volatile void *pvLinRegBaseAddr,
+								 IMG_UINT32 ui32Offset,
+								 IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+@Function       OSWriteUncheckedHWReg64
+@Description    Write to a 64-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to write to a location.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be written to.
+@Input          ui64Value          The long long word to be written to the
+                                   register.
+@Return         None.
+*/ /**************************************************************************/
+	void OSWriteUncheckedHWReg64(volatile void *pvLinRegBaseAddr,
+								 IMG_UINT32 ui32Offset,
+								 IMG_UINT64 ui64Value);
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#if !defined(DOXYGEN)
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(DEBUG)
+static INLINE bool _NonSecureRegister(IMG_UINT32 ui32Offset)
+{
+	const IMG_UINT32 ui32PerCoreRegBankSize = RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE;
+	const IMG_UINT32 ui32RegOffsetInCoreBank = ui32Offset % ui32PerCoreRegBankSize;
+
+	if (ui32RegOffsetInCoreBank < RGX_HOST_SECURE_REGBANK_OFFSET)
+	{
+		return true;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Secure register (0x%X) accessed incorrectly. "
+								"Call OS<Read/Write>UncheckedHWReg<Width> instead with "
+								"psDevInfo->pvSecureRegsBaseKM as a register base.",
+								ui32RegOffsetInCoreBank));
+		return false;
+	}
+
+}
+#else
+#define _NonSecureRegister(ui32Offset) (true)
+#endif
+
+	/* systems using real hardware must check that regular register
+	 * operations don't attempt to access secure registers */
+	static INLINE IMG_UINT32 OSReadHWReg32(volatile void __iomem *pvLinRegBaseAddr,
+										   IMG_UINT32 ui32Offset)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		return OSReadUncheckedHWReg32(pvLinRegBaseAddr, ui32Offset);
+	}
+
+	static INLINE IMG_UINT64 OSReadHWReg64(volatile void __iomem *pvLinRegBaseAddr,
+										   IMG_UINT32 ui32Offset)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		return OSReadUncheckedHWReg64(pvLinRegBaseAddr, ui32Offset);
+	}
+
+	static INLINE void OSWriteHWReg32(volatile void __iomem *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset,
+									  IMG_UINT32 ui32Value)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		OSWriteUncheckedHWReg32(pvLinRegBaseAddr, ui32Offset, ui32Value);
+	}
+
+	static INLINE void OSWriteHWReg64(volatile void __iomem *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset,
+									  IMG_UINT64 ui64Value)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		OSWriteUncheckedHWReg64(pvLinRegBaseAddr, ui32Offset, ui64Value);
+	}
+
+#endif /* !defined(DOXYGEN) */
+#endif /* defined(NO_HARDWARE) */
+
+/*************************************************************************/ /*!
+@Description    Pointer to a timer callback function.
+@Input          pvData  Pointer to timer specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_TIMER_FUNC)(void* pvData);
+
+/*************************************************************************/ /*!
+@Function       OSAddTimer
+@Description    OS specific function to install a timer callback. The
+                timer will then need to be enabled, as it is disabled by
+                default.
+                When enabled, the callback will be invoked once the specified
+                timeout has elapsed.
+@Input          pfnTimerFunc    Timer callback
+@Input          *pvData         Callback data
+@Input          ui32MsTimeout   Callback period
+@Return         Valid handle on success, NULL if a failure
+*/ /**************************************************************************/
+IMG_HANDLE OSAddTimer(PFN_TIMER_FUNC pfnTimerFunc, void *pvData, IMG_UINT32 ui32MsTimeout);
+
+/*************************************************************************/ /*!
+@Function       OSRemoveTimer
+@Description    Removes the specified timer. The handle becomes invalid and
+                should no longer be used.
+@Input          hTimer          handle of the timer to be removed
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSRemoveTimer(IMG_HANDLE hTimer);
+
+/*************************************************************************/ /*!
+@Function       OSEnableTimer
+@Description    Enable the specified timer. after enabling, the timer will
+                invoke the associated callback at an interval determined by
+                the configured timeout period until disabled.
+@Input          hTimer          handle of the timer to be enabled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEnableTimer(IMG_HANDLE hTimer);
+
+/*************************************************************************/ /*!
+@Function       OSDisableTimer
+@Description    Disable the specified timer
+@Input          hTimer          handle of the timer to be disabled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDisableTimer(IMG_HANDLE hTimer);
+
+
+/*************************************************************************/ /*!
+ @Function      OSPanic
+ @Description   Take action in response to an unrecoverable driver error
+ @Return        None
+*/ /**************************************************************************/
+void __noreturn OSPanic(void);
+
+/*************************************************************************/ /*!
+@Function       OSCopyToUser
+@Description    Copy data to user-addressable memory from kernel-addressable
+                memory.
+                Note that pvDest may be an invalid address or NULL and the
+                function should return an error in this case.
+                For operating systems that do not have a user/kernel space
+                distinction, this function should be implemented as a stub
+                which simply returns PVRSRV_ERROR_NOT_SUPPORTED.
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination User memory
+@Input          pvSrc            pointer to the source Kernel memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSCopyToUser(void *pvProcess, void __user *pvDest, const void *pvSrc, size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSCopyFromUser
+@Description    Copy data from user-addressable memory to kernel-addressable
+                memory.
+                Note that pvSrc may be an invalid address or NULL and the
+                function should return an error in this case.
+                For operating systems that do not have a user/kernel space
+                distinction, this function should be implemented as a stub
+                which simply returns PVRSRV_ERROR_NOT_SUPPORTED.
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination Kernel memory
+@Input          pvSrc            pointer to the source User memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSCopyFromUser(void *pvProcess, void *pvDest, const void __user *pvSrc, size_t ui32Bytes);
+
+#if defined(__linux__) || defined(INTEGRITY_OS)
+#define OSBridgeCopyFromUser OSCopyFromUser
+#define OSBridgeCopyToUser OSCopyToUser
+#else
+/*************************************************************************/ /*!
+@Function       OSBridgeCopyFromUser
+@Description    Copy data from user-addressable memory into kernel-addressable
+                memory as part of a bridge call operation.
+                For operating systems that do not have a user/kernel space
+                distinction, this function will require whatever implementation
+                is needed to pass data for making the bridge function call.
+                For operating systems which do have a user/kernel space
+                distinction (such as Linux) this function may be defined so
+                as to equate to a call to OSCopyFromUser().
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination Kernel memory
+@Input          pvSrc            pointer to the source User memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSBridgeCopyFromUser (void *pvProcess,
+						void *pvDest,
+						const void *pvSrc,
+						size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSBridgeCopyToUser
+@Description    Copy data to user-addressable memory from kernel-addressable
+                memory as part of a bridge call operation.
+                For operating systems that do not have a user/kernel space
+                distinction, this function will require whatever implementation
+                is needed to pass data for making the bridge function call.
+                For operating systems which do have a user/kernel space
+                distinction (such as Linux) this function may be defined so
+                as to equate to a call to OSCopyToUser().
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination User memory
+@Input          pvSrc            pointer to the source Kernel memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSBridgeCopyToUser (void *pvProcess,
+						void *pvDest,
+						const void *pvSrc,
+						size_t ui32Bytes);
+#endif
+
+/* To be increased if required in future */
+#define PVRSRV_MAX_BRIDGE_IN_SIZE      0x2000    /*!< Size of the memory block used to hold data passed in to a bridge call */
+#define PVRSRV_MAX_BRIDGE_OUT_SIZE     0x1000    /*!< Size of the memory block used to hold data returned from a bridge call */
+
+/*************************************************************************/ /*!
+@Function       OSPlatformBridgeInit
+@Description    Called during device creation to allow the OS port to register
+                other bridge modules and related resources that it requires.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPlatformBridgeInit(void);
+
+/*************************************************************************/ /*!
+@Function       OSPlatformBridgeDeInit
+@Description    Called during device destruction to allow the OS port to
+                deregister its OS specific bridges and clean up other
+                related resources.
+*/ /**************************************************************************/
+void OSPlatformBridgeDeInit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVToNativeError
+@Description    Returns the OS-specific equivalent error number/code for
+                the specified PVRSRV_ERROR value.
+                If there is no equivalent, or the PVRSRV_ERROR value is
+                PVRSRV_OK (no error), 0 is returned.
+@Return         The OS equivalent error code.
+*/ /**************************************************************************/
+int PVRSRVToNativeError(PVRSRV_ERROR e);
+/** See PVRSRVToNativeError(). */
+#define OSPVRSRVToNativeError(e) ( (PVRSRV_OK == e)? 0: PVRSRVToNativeError(e) )
+
+
+#if defined(__linux__) && defined(__KERNEL__)
+
+/* Provide LockDep friendly definitions for Services RW locks */
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include "allocmem.h"
+
+#define OSWRLockCreate(ppsLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(ppsLock) = OSAllocMem(sizeof(struct rw_semaphore)); \
+	if (*(ppsLock)) { init_rwsem(*(ppsLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSWRLockDestroy(psLock) ({OSFreeMem(psLock); PVRSRV_OK;})
+
+#define OSWRLockAcquireRead(psLock) ({down_read(psLock); PVRSRV_OK;})
+#define OSWRLockAcquireReadNested(psLock, subclass) ({down_read_nested((psLock), (subclass)); PVRSRV_OK;})
+#define OSWRLockReleaseRead(psLock) ({up_read(psLock); PVRSRV_OK;})
+#define OSWRLockAcquireWrite(psLock) ({down_write(psLock); PVRSRV_OK;})
+#define OSWRLockReleaseWrite(psLock) ({up_write(psLock); PVRSRV_OK;})
+
+#elif defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+/* User-mode unit tests use these definitions on Linux */
+
+PVRSRV_ERROR OSWRLockCreate(POSWR_LOCK *ppsLock);
+void OSWRLockDestroy(POSWR_LOCK psLock);
+void OSWRLockAcquireRead(POSWR_LOCK psLock);
+#define OSWRLockAcquireReadNested(psLock, subclass) OSWRLockAcquireRead((psLock))
+void OSWRLockReleaseRead(POSWR_LOCK psLock);
+void OSWRLockAcquireWrite(POSWR_LOCK psLock);
+void OSWRLockReleaseWrite(POSWR_LOCK psLock);
+
+#else
+
+#if !defined(DOXYGEN)
+#define OSFUNC_NOT_IMPLEMENTED 0
+#define OSFUNC_NOT_IMPLEMENTED_ASSERT() PVR_ASSERT(OSFUNC_NOT_IMPLEMENTED)
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSWRLockCreate
+@Description    Create a writer/reader lock.
+                This type of lock allows multiple concurrent readers but
+                only a single writer, allowing for optimized performance.
+@Output         ppsLock     A handle to the created WR lock.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSWRLockCreate(POSWR_LOCK *ppsLock)
+{
+	PVR_UNREFERENCED_PARAMETER(ppsLock);
+
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockDestroy
+@Description    Destroys a writer/reader lock.
+@Input          psLock     The handle of the WR lock to be destroyed.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockDestroy(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireRead
+@Description    Acquire a writer/reader read lock.
+                If the write lock is already acquired, the caller will
+                block until it is released.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           reading.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireRead(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireReadNested
+@Description    Acquire a nested writer/reader read lock.
+                If the write lock is already acquired, the caller will
+                block until it is released.
+                For operating systems other than Linux, this equates to an
+                OSWRLockAcquireRead() call. On Linux, this function wraps a call
+                to down_read_nested(). This recognises the scenario where
+                there may be multiple subclasses within a particular class
+                of lock. In such cases, the order in which the locks belonging
+                these various subclasses are acquired is important and must be
+                validated.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           reading.
+@Input          iSubclass  The subclass of the lock.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireReadNested(POSWR_LOCK psLock, IMG_INT iSubclass)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	PVR_UNREFERENCED_PARAMETER(iSubclass);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockReleaseRead
+@Description    Release a writer/reader read lock.
+@Input          psLock     The handle of the WR lock whose read lock is to
+                           be released.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockReleaseRead(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireWrite
+@Description    Acquire a writer/reader write lock.
+                If the write lock or any read lock are already acquired,
+                the caller will block until all are released.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           writing.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireWrite(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockReleaseWrite
+@Description    Release a writer/reader write lock.
+@Input          psLock     The handle of the WR lock whose write lock is to
+                           be released.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void OSWRLockReleaseWrite(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSDivide64r64
+@Description    Divide a 64-bit value by a 32-bit value. Return the 64-bit
+                quotient.
+                The remainder is also returned in 'pui32Remainder'.
+@Input          ui64Divident        The number to be divided.
+@Input          ui32Divisor         The 32-bit value 'ui64Divident' is to
+                                    be divided by.
+@Output         pui32Remainder      The remainder of the division.
+@Return         The 64-bit quotient (result of the division).
+*/ /**************************************************************************/
+IMG_UINT64 OSDivide64r64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder);
+
+/*************************************************************************/ /*!
+@Function       OSDivide64
+@Description    Divide a 64-bit value by a 32-bit value. Return a 32-bit
+                quotient.
+                The remainder is also returned in 'pui32Remainder'.
+                This function allows for a more optimal implementation
+                of a 64-bit division when the result is known to be
+                representable in 32-bits.
+@Input          ui64Divident        The number to be divided.
+@Input          ui32Divisor         The 32-bit value 'ui64Divident' is to
+                                    be divided by.
+@Output         pui32Remainder      The remainder of the division.
+@Return         The 32-bit quotient (result of the division).
+*/ /**************************************************************************/
+IMG_UINT32 OSDivide64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder);
+
+/*************************************************************************/ /*!
+@Function       OSDumpStack
+@Description    Dump the current task information and its stack trace.
+@Return         None
+*/ /**************************************************************************/
+void OSDumpStack(void);
+
+/*************************************************************************/ /*!
+@Function       OSUserModeAccessToPerfCountersEn
+@Description    Permit User-mode access to CPU performance counter
+                registers.
+                This function is called during device initialisation.
+                Certain CPU architectures may need to explicitly permit
+                User mode access to performance counters - if this is
+                required, the necessary code should be implemented inside
+                this function.
+@Return         None.
+*/ /**************************************************************************/
+void OSUserModeAccessToPerfCountersEn(void);
+
+/*************************************************************************/ /*!
+@Function       OSDebugSignalPID
+@Description    Sends a SIGTRAP signal to a specific PID in user mode for
+                debugging purposes. The user mode process can register a handler
+                against this signal.
+                This is necessary to support the Rogue debugger. If the Rogue
+                debugger is not used then this function may be implemented as
+                a stub.
+@Input          ui32PID    The PID for the signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDebugSignalPID(IMG_UINT32 ui32PID);
+
+#if defined(__linux__) && defined(__KERNEL__) && !defined(DOXYGEN)
+#define OSWarnOn(a) WARN_ON(a)
+#else
+/*************************************************************************/ /*!
+@Function       OSWarnOn
+@Description    This API allows the driver to emit a special token and stack
+                dump to the server log when an issue is detected that needs the
+                OS to be notified. The token or call may be used to trigger
+                log collection by the OS environment.
+                PVR_DPF log messages will have been emitted prior to this call.
+@Input          a    Expression to evaluate, if true trigger Warn signal
+@Return         None
+*/ /**************************************************************************/
+#define OSWarnOn(a) do { if ((a)) { OSDumpStack(); } } while (0)
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSIsKernelThread
+@Description    This API determines if the current running thread is a kernel
+                thread (i.e. one not associated with any userland process,
+                typically an MISR handler.)
+@Return         IMG_TRUE if it is a kernel thread, otherwise IMG_FALSE.
+*/ /**************************************************************************/
+IMG_BOOL OSIsKernelThread(void);
+
+/*************************************************************************/ /*!
+@Function       OSThreadDumpInfo
+@Description    Traverse the thread list and call each of the stored
+                callbacks to dump the info in debug_dump.
+@Input          pfnDumpDebugPrintf  The 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+*/ /**************************************************************************/
+void OSThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSDumpVersionInfo
+@Description    Store OS version information in debug dump.
+@Input          pfnDumpDebugPrintf  The 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+*/ /**************************************************************************/
+void OSDumpVersionInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                       void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSIsWriteCombineUnalignedSafe
+@Description    Determine if unaligned accesses to write-combine memory are
+                safe to perform, i.e. whether we are safe from a CPU fault
+                occurring. This test is specifically aimed at ARM64 platforms
+                which cannot provide this guarantee if the memory is 'device'
+                memory rather than 'normal' under the ARM memory architecture.
+@Return         IMG_TRUE if safe, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void);
+
+/*************************************************************************/ /*!
+@Function       OSDebugLevel
+@Description    Returns current value of the debug level.
+@Return         Debug level.
+*/ /**************************************************************************/
+IMG_UINT32 OSDebugLevel(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVSetDebugLevel
+@Description    Sets the current value of the debug level to ui32DebugLevel.
+@Input          ui32DebugLevel New debug level value.
+*/ /**************************************************************************/
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVIsDebugLevel
+@Description    Tests if a given debug level is enabled.
+@Input          ui32DebugLevel IMG_TRUE if debug level is enabled
+                and IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel);
+
+#if defined(SUPPORT_DMA_TRANSFER)
+
+typedef void (*PFN_SERVER_CLEANUP)(void *pvData, IMG_BOOL bAdvanceTimeline);
+
+#define DMA_COMPLETION_TIMEOUT_MS 60000
+#define DMA_ERROR_SYNC_RETRIES 100
+
+PVRSRV_ERROR OSDmaPrepareTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *psChan,
+							   IMG_DMA_ADDR* psDmaAddr, IMG_UINT64* puiAddress,
+							   IMG_UINT64 uiSize, IMG_BOOL bMemToDev,
+							   IMG_HANDLE pvOSData,
+							   IMG_HANDLE pvServerCleanupParam,PFN_SERVER_CLEANUP pfnServerCleanup, IMG_BOOL bFirst);
+
+PVRSRV_ERROR OSDmaPrepareTransferSparse(PVRSRV_DEVICE_NODE *psDevNode, IMG_HANDLE pvChan,
+										IMG_DMA_ADDR* psDmaAddr, IMG_BOOL *pbValid,
+										IMG_UINT64* puiAddress, IMG_UINT64 uiSize,
+										IMG_UINT32 uiOffsetInPage,
+										IMG_UINT32 ui32SizeInPages,
+										IMG_BOOL bMemToDev,
+										IMG_HANDLE pvOSData,
+										IMG_HANDLE pvServerCleanupParam, PFN_SERVER_CLEANUP pfnServerCleanup,
+										IMG_BOOL bFirst);
+
+PVRSRV_ERROR OSDmaAllocData(PVRSRV_DEVICE_NODE *psDevNode,IMG_UINT32 uiNumDMA, void **pvAllocedData);
+PVRSRV_ERROR OSDmaSubmitTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *pvOSData, void *psChan, IMG_BOOL bSynchronous);
+void OSDmaForceCleanup(PVRSRV_DEVICE_NODE *psDevNode, void *pvChan,
+					   void *pvOSData, IMG_HANDLE pvServerCleanupParam,
+					   PFN_SERVER_CLEANUP pfnServerCleanup);
+#endif
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR);
+
+void
+OSFreeSecBuf(PMR *psPMR);
+#endif
+#endif /* OSFUNC_H */
+
+/******************************************************************************
+ End of file (osfunc.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/ospvr_gputrace.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/ospvr_gputrace.h
new file mode 100644
index 000000000000..6bafc4d6042f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/ospvr_gputrace.h
@@ -0,0 +1,185 @@
+/*************************************************************************/ /*!
+@File           ospvr_gputrace.h
+@Title          PVR GPU Trace module common environment interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_GPUTRACE_H_
+#define PVR_GPUTRACE_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_hwperf.h"
+#include "rgxdevice.h"
+#include "device.h"
+
+#if defined(__linux__) && defined(CONFIG_EVENT_TRACING) && \
+	(defined(PVRSRV_TRACE_ROGUE_EVENTS) || \
+	 defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD))
+
+/* Early initialisation of GPU Trace events logic.
+ * This function is called on *driver* initialisation. */
+PVRSRV_ERROR PVRGpuTraceSupportInit(void);
+
+/* GPU Trace resources final cleanup.
+ * This function is called on driver de-initialisation. */
+void PVRGpuTraceSupportDeInit(void);
+
+/* Initialisation for AppHints callbacks.
+ * This function is called during the late stage of driver initialisation but
+ * before the device initialisation but after the debugfs sub-system has been
+ * initialised. */
+void PVRGpuTraceInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* Per-device initialisation of the GPU Trace resources */
+PVRSRV_ERROR PVRGpuTraceInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* TL Initialisation of FTrace */
+PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/* Per-device cleanup for the GPU Trace resources. */
+void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* Enables the gpu trace sub-system for a given device. */
+PVRSRV_ERROR PVRGpuTraceSetEnabled(
+		PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_BOOL bNewValue);
+
+/* Performs some initialisation steps if the feature was enabled
+ * on driver startup. */
+void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* FTrace events callbacks interface */
+
+void PVRGpuTraceEnableUfoCallback(void);
+void PVRGpuTraceDisableUfoCallback(void);
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void);
+void PVRGpuTraceDisableFirmwareActivityCallback(void);
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+PVRSRV_ERROR
+PVRSRVGpuTraceWorkPeriodEventStatsRegister(IMG_HANDLE*
+		phGpuWorkPeriodEventStats);
+void PVRSRVGpuTraceWorkPeriodEventStatsUnregister(
+		IMG_HANDLE hGpuWorkPeriodEventStats);
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#else /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) */
+
+static inline PVRSRV_ERROR PVRGpuTraceSupportInit(void) {
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceSupportDeInit(void) {}
+
+static inline void PVRGpuTraceInitAppHintCallbacks(
+		const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceInitDevice(
+		PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceSetEnabled(
+		PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_BOOL bNewValue)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(bNewValue);
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline void PVRGpuTraceEnableUfoCallback(void) {}
+static inline void PVRGpuTraceDisableUfoCallback(void) {}
+
+static inline void PVRGpuTraceEnableFirmwareActivityCallback(void) {}
+static inline void PVRGpuTraceDisableFirmwareActivityCallback(void) {}
+
+#endif /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) */
+
+#if defined(__linux__) && defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExternalJobRef,
+		IMG_UINT32 ui32InternalJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType);
+
+#else
+
+static inline void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExternalJobRef,
+		IMG_UINT32 ui32InternalJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ui32FirmwareCtx);
+	PVR_UNREFERENCED_PARAMETER(ui32ExternalJobRef);
+	PVR_UNREFERENCED_PARAMETER(ui32InternalJobRef);
+	PVR_UNREFERENCED_PARAMETER(eKickType);
+}
+
+#endif /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#endif /* PVR_GPUTRACE_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/ossecure_export.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/ossecure_export.h
new file mode 100644
index 000000000000..020f5f57f45e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/ossecure_export.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       OSSecureExport
+@Description    Assigns an OS-specific 'token' to allow a resource
+                to be securely referenced by another process.
+                A process wishing to reference the exported resource
+                should call OSSecureImport(), passing its OS-specific
+                reference to the same resource.
+                For the export/import to be deemed 'secure', the
+                implementation should ensure that the OS-specific
+                reference can only be meaningfully used by a process
+                which is permitted to do so.
+@Input          pszName        name of the "class" of new secure file
+@Input          pfnReleaseFunc  pointer to the function to be called
+                               while closing secure file
+@Input          pvData         pointer to the actual resource that
+                               is being exported
+@Output         phSecure       the returned secure token
+@Return         PVRSRV_OK      on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSSecureExport(const IMG_CHAR *pszName,
+                            PVRSRV_ERROR (*pfnReleaseFunc)(void *),
+                            void *pvData,
+                            IMG_SECURE_TYPE *phSecure);
+
+/*************************************************************************/ /*!
+@Function       OSSecureImport
+@Description    Imports an OS-specific 'token' that allows a resource
+                allocated by another process to be securely referenced by
+                the current process.
+@Input          hSecure             the secure token for the resource to
+                                    be imported
+@Output         ppvData             pointer to the actual resource that
+                                    is being referenced
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSSecureImport(IMG_SECURE_TYPE hSecure, void **ppvData);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_km.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_km.h
new file mode 100644
index 000000000000..58713cdfa17d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_km.h
@@ -0,0 +1,1159 @@
+/*************************************************************************/ /*!
+@File
+@Title          pdump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PDUMP_KM_H
+#define PDUMP_KM_H
+
+#if defined(PDUMP)
+ #if defined(__linux__)
+  #include <linux/version.h>
+  #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+   #include <linux/stdarg.h>
+  #else
+   #include <stdarg.h>
+  #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+ #else
+  #include <stdarg.h>
+ #endif /* __linux__ */
+#endif /* PDUMP */
+
+/* services/srvkm/include/ */
+#include "device.h"
+
+/* include/ */
+#include "pvrsrv_error.h"
+
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#include "connection_server.h"
+/* Pull in pdump flags from services include */
+#include "pdump.h"
+#include "pdumpdefs.h"
+
+/* Define this to enable the PDUMP_HERE trace in the server */
+#undef PDUMP_TRACE
+
+#if defined(PDUMP_TRACE)
+#define PDUMP_HERE_VAR  __maybe_unused IMG_UINT32 here = 0;
+#define PDUMP_HERE(a)	{ here = (a); if (ui32Flags & PDUMP_FLAGS_DEBUG) PVR_DPF((PVR_DBG_WARNING, "HERE %d", (a))); }
+#define PDUMP_HEREA(a)	{ here = (a); PVR_DPF((PVR_DBG_WARNING, "HERE ALWAYS %d", (a))); }
+#else
+#define PDUMP_HERE_VAR  __maybe_unused IMG_UINT32 here = 0;
+#define PDUMP_HERE(a)	here = (a);
+#define PDUMP_HEREA(a)	here = (a);
+#endif
+
+#define PDUMP_PD_UNIQUETAG	(IMG_HANDLE)0
+#define PDUMP_PT_UNIQUETAG	(IMG_HANDLE)0
+
+/* Defines for CMD:SetParity's state word */
+#define PDUMP_SET_PARITY_STATE_WORD_VERSION_SHIFT		(0U)
+#define PDUMP_SET_PARITY_STATE_WORD_VERSION_MASK		(0x00000007U)
+
+#define PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_SHIFT	(3U)
+#define PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_MASK	(0x000001F8U)
+
+#define PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_SHIFT		(9U)
+#define PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_MASK		(0x00000200U)
+
+/* Invalid value for PDump block number */
+#define PDUMP_BLOCKNUM_INVALID      IMG_UINT32_MAX
+
+typedef struct _PDUMP_CONNECTION_DATA_ PDUMP_CONNECTION_DATA;
+
+/* PDump transition events */
+typedef enum _PDUMP_TRANSITION_EVENT_
+{
+	PDUMP_TRANSITION_EVENT_NONE,              /* No event */
+	PDUMP_TRANSITION_EVENT_BLOCK_FINISHED,    /* Block mode event, current PDump-block has finished */
+	PDUMP_TRANSITION_EVENT_BLOCK_STARTED,     /* Block mode event, new PDump-block has started */
+	PDUMP_TRANSITION_EVENT_RANGE_ENTERED,     /* Transition into capture range */
+	PDUMP_TRANSITION_EVENT_RANGE_APPEND,      /* Append to an already entered capture range */
+	PDUMP_TRANSITION_EVENT_RANGE_EXITED,      /* Transition out of capture range */
+} PDUMP_TRANSITION_EVENT;
+
+typedef PVRSRV_ERROR (*PFN_PDUMP_TRANSITION)(void *pvData, void *pvDevice, PDUMP_TRANSITION_EVENT eEvent, IMG_UINT32 ui32PDumpFlags);
+typedef void (*PFN_PDUMP_SYNCBLOCKS)(PVRSRV_DEVICE_NODE *psDevNode, void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+
+typedef PVRSRV_ERROR (*PFN_PDUMP_TRANSITION_FENCE_SYNC)(void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+
+#ifdef PDUMP
+
+/*! Macro used to record a panic in the PDump script stream */
+#define PDUMP_PANIC(_dev, _id, _msg) do \
+		{ PVRSRV_ERROR _eE;\
+			_eE = PDumpPanic((_dev), ((RGX_PDUMP_PANIC_ ## _id) & 0xFFFF), _msg, __func__, __LINE__);	\
+			PVR_LOG_IF_ERROR(_eE, "PDumpPanic");\
+		MSC_SUPPRESS_4127\
+		} while (0)
+
+/*! Macro used to record a driver error in the PDump script stream to invalidate the capture */
+#define PDUMP_ERROR(_dev, _err, _msg) \
+	(void)PDumpCaptureError((_dev), _err, _msg, __func__, __LINE__)
+
+#define SZ_MSG_SIZE_MAX			PVRSRV_PDUMP_MAX_COMMENT_SIZE
+#define SZ_SCRIPT_SIZE_MAX		PVRSRV_PDUMP_MAX_COMMENT_SIZE
+#define SZ_FILENAME_SIZE_MAX	(PVRSRV_PDUMP_MAX_FILENAME_SIZE+sizeof(PDUMP_PARAM_N_FILE_NAME))
+
+#define PDUMP_GET_SCRIPT_STRING()																			\
+	IMG_HANDLE hScript;																						\
+	void *pvScriptAlloc;																					\
+	IMG_UINT32 ui32MaxLen = SZ_SCRIPT_SIZE_MAX-1;															\
+	pvScriptAlloc = OSAllocMem( SZ_SCRIPT_SIZE_MAX );														\
+	if (!pvScriptAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_STRING() failed to allocate memory for script buffer"));	\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+																											\
+	hScript = (IMG_HANDLE) pvScriptAlloc;
+
+#define PDUMP_GET_MSG_STRING()																				\
+	IMG_CHAR *pszMsg;																						\
+	void *pvMsgAlloc;																						\
+	IMG_UINT32 ui32MaxLen = SZ_MSG_SIZE_MAX-1;																\
+	pvMsgAlloc = OSAllocMem( SZ_MSG_SIZE_MAX );																\
+	if (!pvMsgAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_MSG_STRING() failed to allocate memory for message buffer"));	\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+	pszMsg = (IMG_CHAR *)pvMsgAlloc;
+
+#define PDUMP_GET_SCRIPT_AND_FILE_STRING()																	\
+	IMG_HANDLE hScript;																						\
+	IMG_CHAR *pszFileName;																					\
+	IMG_UINT32 ui32MaxLenScript = SZ_SCRIPT_SIZE_MAX-1;														\
+	void *pvScriptAlloc;																					\
+	void *pvFileAlloc;																						\
+	pvScriptAlloc = OSAllocMem( SZ_SCRIPT_SIZE_MAX );														\
+	if (!pvScriptAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_AND_FILE_STRING() failed to allocate memory for script buffer"));		\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+																											\
+	hScript = (IMG_HANDLE) pvScriptAlloc;																	\
+	pvFileAlloc = OSAllocMem( SZ_FILENAME_SIZE_MAX );														\
+	if (!pvFileAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_AND_FILE_STRING() failed to allocate memory for filename buffer"));	\
+		OSFreeMem(pvScriptAlloc);																			\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+	pszFileName = (IMG_CHAR *)pvFileAlloc;
+
+#define PDUMP_RELEASE_SCRIPT_STRING()																		\
+	if (pvScriptAlloc)																						\
+	{																										\
+		OSFreeMem(pvScriptAlloc);																			\
+		pvScriptAlloc = NULL;																				\
+	}
+
+#define PDUMP_RELEASE_MSG_STRING()																			\
+	if (pvMsgAlloc)																							\
+	{																										\
+		OSFreeMem(pvMsgAlloc);																				\
+		pvMsgAlloc = NULL;																					\
+	}
+
+#define PDUMP_RELEASE_FILE_STRING()																			\
+	if (pvFileAlloc)																						\
+	{																										\
+		OSFreeMem(pvFileAlloc);																				\
+		pvFileAlloc = NULL;																					\
+	}
+
+#define PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()																\
+	if (pvScriptAlloc)																						\
+	{																										\
+		OSFreeMem(pvScriptAlloc);																			\
+		pvScriptAlloc = NULL;																				\
+	}																										\
+	if (pvFileAlloc)																						\
+	{																										\
+		OSFreeMem(pvFileAlloc);																				\
+		pvFileAlloc = NULL;																					\
+	}
+
+
+/* Shared across pdump_x files */
+PVRSRV_ERROR PDumpInitCommon(void);
+void PDumpDeInitCommon(void);
+PVRSRV_ERROR PDumpValidateUMFlags(PDUMP_FLAGS_T uiFlags);
+PVRSRV_ERROR PDumpReady(void);
+void PDumpGetParameterZeroPageInfo(PDUMP_FILEOFFSET_T *puiZeroPageOffset,
+                                   size_t *puiZeroPageSize,
+                                   const IMG_CHAR **ppszZeroPageFilename);
+
+void PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode);
+void PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32Frame);
+PVRSRV_ERROR PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 *pui32Frame);
+PVRSRV_ERROR PDumpCommentKM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT32 ui32CommentSize,
+                            IMG_CHAR *pszComment,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_UINT32 ui32Mode,
+                                            IMG_UINT32 ui32Start,
+                                            IMG_UINT32 ui32End,
+                                            IMG_UINT32 ui32Interval,
+                                            IMG_UINT32 ui32MaxParamFileSize,
+                                            IMG_UINT32 ui32AutoTermTimeout);
+
+
+PVRSRV_ERROR PDumpReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_CHAR *pszPDumpRegName,
+                        IMG_UINT32 ui32RegAddr,
+                        IMG_UINT32 ui32RegValue,
+                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_CHAR *pszPDumpRegName,
+                        IMG_UINT32 ui32RegAddr,
+                        IMG_UINT64 ui64RegValue,
+                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32RegDst,
+                                  IMG_UINT32 ui32RegSrc,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpPhysHandleToInternalVar64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_CHAR *pszInternalVar,
+                                            IMG_HANDLE hPdumpPages,
+                                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToInternalVar64(IMG_CHAR *pszInternalVar,
+                                          PMR *psPMR,
+                                          IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                          IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpInternalVarToMemLabel(PMR *psPMR,
+                                        IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpWriteVarORValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const IMG_CHAR *pszInternalVariable,
+                                    const IMG_UINT64 ui64Value,
+                                    const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarANDValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarSHRValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarORVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const IMG_CHAR *pszInternalVar,
+                                  const IMG_CHAR *pszInternalVar2,
+                                  const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarANDVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const IMG_CHAR *pszInternalVar,
+                                   const IMG_CHAR *pszInternalVar2,
+                                   const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpInternalVarToReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpInternalVarToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToMem32(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToMem64(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToMem32(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToMem64(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR *pszPDumpRegName,
+                                        IMG_UINT32 ui32Reg,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpSAW(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR      *pszDevSpaceName,
+                      IMG_UINT32    ui32HPOffsetBytes,
+                      IMG_UINT32    ui32NumSaveBytes,
+                      IMG_CHAR      *pszOutfileName,
+                      IMG_UINT32    ui32OutfileOffsetByte,
+                      PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR PDumpRegPolKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           IMG_CHAR            *pszPDumpRegName,
+                           IMG_UINT32          ui32RegAddr,
+                           IMG_UINT32          ui32RegValue,
+                           IMG_UINT32          ui32Mask,
+                           IMG_UINT32          ui32Flags,
+                           PDUMP_POLL_OPERATOR eOperator);
+
+
+/**************************************************************************/ /*!
+@Function       PDumpImageDescriptor
+@Description    PDumps image data out as an IMGBv2 data section
+@Input          psDeviceNode         Pointer to device node.
+@Input          ui32MMUContextID     PDUMP MMU context ID.
+@Input          pszSABFileName       Pointer to string containing file name of
+                                     Image being SABed
+@Input          sData                GPU virtual address of this surface.
+@Input          ui32DataSize         Image data size
+@Input          ui32LogicalWidth     Image logical width
+@Input          ui32LogicalHeight    Image logical height
+@Input          ui32PhysicalWidth    Image physical width
+@Input          ui32PhysicalHeight   Image physical height
+@Input          ePixFmt              Image pixel format
+@Input          eFBCompression       FB compression mode
+@Input          paui32FBCClearColour FB clear colour (Only applicable to FBC surfaces)
+@Input          eFBCSwizzle          FBC channel swizzle (Only applicable to FBC surfaces)
+@Input          sHeader              GPU virtual address of the headers of this
+                                     surface (Only applicable to FBC surfaces)
+@Input          ui32HeaderSize       Header size (Only applicable to FBC surfaces)
+@Input          ui32PDumpFlags       PDUMP flags
+@Return         PVRSRV_ERROR:        PVRSRV_OK on success. Otherwise, a PVRSRV_
+                                     error code
+*/ /***************************************************************************/
+PVRSRV_ERROR PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32MMUContextID,
+                                  IMG_CHAR *pszSABFileName,
+                                  IMG_DEV_VIRTADDR sData,
+                                  IMG_UINT32 ui32DataSize,
+                                  IMG_UINT32 ui32LogicalWidth,
+                                  IMG_UINT32 ui32LogicalHeight,
+                                  IMG_UINT32 ui32PhysicalWidth,
+                                  IMG_UINT32 ui32PhysicalHeight,
+                                  PDUMP_PIXEL_FORMAT ePixFmt,
+                                  IMG_MEMLAYOUT eMemLayout,
+                                  IMG_FB_COMPRESSION eFBCompression,
+                                  const IMG_UINT32 *paui32FBCClearColour,
+                                  PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                                  IMG_DEV_VIRTADDR sHeader,
+                                  IMG_UINT32 ui32HeaderSize,
+                                  IMG_UINT32 ui32PDumpFlags);
+
+/**************************************************************************/ /*!
+@Function       PDumpDataDescriptor
+@Description    PDumps non-image data out as an IMGCv1 data section
+@Input          psDeviceNode         Pointer to device node.
+@Input          ui32MMUContextID     PDUMP MMU context ID.
+@Input          pszSABFileName       Pointer to string containing file name of
+                                     Data being SABed
+@Input          sData                GPU virtual address of this data.
+@Input          ui32DataSize         Data size
+@Input          ui32HeaderType       Header type
+@Input          ui32ElementType      Data element type
+@Input          ui32ElementCount     Number of data elements
+@Input          ui32PDumpFlags       PDUMP flags
+@Return         PVRSRV_ERROR:        PVRSRV_OK on success. Otherwise, a PVRSRV_
+                                     error code
+*/ /***************************************************************************/
+PVRSRV_ERROR PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32MMUContextID,
+                                 IMG_CHAR *pszSABFileName,
+                                 IMG_DEV_VIRTADDR sData,
+                                 IMG_UINT32 ui32DataSize,
+                                 IMG_UINT32 ui32HeaderType,
+                                 IMG_UINT32 ui32ElementType,
+                                 IMG_UINT32 ui32ElementCount,
+                                 IMG_UINT32 ui32PDumpFlags);
+
+
+PVRSRV_ERROR PDumpReadRegKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszFileName,
+                            IMG_UINT32 ui32FileOffset,
+                            IMG_UINT32 ui32Address,
+                            IMG_UINT32 ui32Size,
+                            IMG_UINT32 ui32PDumpFlags);
+
+__printf(3, 4)
+PVRSRV_ERROR PDumpCommentWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_UINT32 ui32Flags,
+                                   IMG_CHAR* pszFormat,
+                                   ...);
+
+PVRSRV_ERROR PDumpCommentWithFlagsVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Flags,
+                                     const IMG_CHAR * pszFormat,
+                                     va_list args);
+
+PVRSRV_ERROR PDumpPanic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT32      ui32PanicNo,
+                        IMG_CHAR*       pszPanicMsg,
+                        const IMG_CHAR* pszPPFunc,
+                        IMG_UINT32      ui32PPline);
+
+PVRSRV_ERROR PDumpCaptureError(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PVRSRV_ERROR    ui32ErrorNo,
+                               IMG_CHAR*       pszErrorMsg,
+                               const IMG_CHAR* pszPPFunc,
+                               IMG_UINT32      ui32PPline);
+
+PVRSRV_ERROR PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame);
+
+PVRSRV_ERROR PDumpGetStateKM(IMG_UINT64 *ui64State);
+
+PVRSRV_ERROR PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PDumpRegRead32ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_CHAR *pszPDumpRegName,
+                                         IMG_UINT32 ui32RegOffset,
+                                         IMG_CHAR *pszInternalVar,
+                                         IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            const IMG_UINT32 dwRegOffset,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            const IMG_UINT32 dwRegOffset,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead64ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_CHAR *pszPDumpRegName,
+                                         IMG_CHAR *pszInternalVar,
+                                         const IMG_UINT32 dwRegOffset,
+                                         IMG_UINT32	ui32Flags);
+
+PVRSRV_ERROR PDumpIDLWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Clocks,
+                               IMG_UINT32 ui32Flags);
+PVRSRV_ERROR PDumpIDL(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32Clocks);
+
+PVRSRV_ERROR PDumpRegBasedCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_CHAR   *pszPDumpRegName,
+                              IMG_UINT32 ui32RegOffset,
+                              IMG_UINT32 ui32WPosVal,
+                              IMG_UINT32 ui32PacketSize,
+                              IMG_UINT32 ui32BufferSize,
+                              IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpTRG(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR *pszMemSpace,
+                      IMG_UINT32 ui32MMUCtxID,
+                      IMG_UINT32 ui32RegionID,
+                      IMG_BOOL bEnable,
+                      IMG_UINT64 ui64VAddr,
+                      IMG_UINT64 ui64LenBytes,
+                      IMG_UINT32 ui32XStride,
+                      IMG_UINT32 ui32Flags);
+
+void PDumpLock(void);
+void PDumpUnlock(void);
+
+PVRSRV_ERROR PDumpRegCondStr(IMG_CHAR            **ppszPDumpCond,
+                             IMG_CHAR            *pszPDumpRegName,
+                             IMG_UINT32          ui32RegAddr,
+                             IMG_UINT32          ui32RegValue,
+                             IMG_UINT32          ui32Mask,
+                             IMG_UINT32          ui32Flags,
+                             PDUMP_POLL_OPERATOR eOperator);
+
+PVRSRV_ERROR PDumpInternalValCondStr(IMG_CHAR            **ppszPDumpCond,
+                                     IMG_CHAR            *pszInternalVar,
+                                     IMG_UINT32          ui32RegValue,
+                                     IMG_UINT32          ui32Mask,
+                                     IMG_UINT32          ui32Flags,
+                                     PDUMP_POLL_OPERATOR eOperator);
+
+PVRSRV_ERROR PDumpIfKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpElseKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpFiKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpStartDoLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpEndDoWhileLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_CHAR *pszPDumpWhileCond,
+                                   IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpCOMCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags,
+                             const IMG_CHAR *pszPDump);
+
+void PDumpPowerTransitionStart(PVRSRV_DEVICE_NODE *psDeviceNode);
+void PDumpPowerTransitionEnd(PVRSRV_DEVICE_NODE *psDeviceNode);
+IMG_BOOL PDumpCheckFlagsWrite(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_UINT32 ui32Flags);
+
+/*!
+ * @name	PDumpWriteParameter
+ * @brief	General function for writing to PDump stream. Used
+ *          mainly for memory dumps to parameter stream.
+ *          Usually more convenient to use PDumpWriteScript below
+ *          for the script stream.
+ * @param	psDeviceNode - device PDump pertains to
+ * @param	psui8Data - data to write
+ * @param	ui32Size - size of write
+ * @param	ui32Flags - PDump flags
+ * @param   pui32FileOffset - on return contains the file offset to
+ *                            the start of the parameter data
+ * @param   aszFilenameStr - pointer to at least a 20 char buffer to
+ *                           return the parameter filename
+ * @return	error
+ */
+PVRSRV_ERROR PDumpWriteParameter(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT8 *psui8Data, IMG_UINT32 ui32Size,
+                                 IMG_UINT32 ui32Flags, IMG_UINT32* pui32FileOffset,
+                                 IMG_CHAR* aszFilenameStr);
+
+/*!
+ * @name	PDumpWriteScript
+ * @brief	Write an PDumpOS created string to the "script" output stream
+ * @param	psDeviceNode - device PDump pertains to
+ * @param	hString - PDump OS layer handle of string buffer to write
+ * @param	ui32Flags - PDump flags
+ * @return	IMG_TRUE on success.
+ */
+IMG_BOOL PDumpWriteScript(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          IMG_HANDLE hString, IMG_UINT32 ui32Flags);
+
+/**************************************************************************/ /*!
+@Function       PDumpSNPrintf
+@Description    Printf to OS-specific PDump state buffer. This function is
+                only called if PDUMP is defined.
+@Input          hBuf               handle of buffer to write into
+@Input          ui32ScriptSizeMax  maximum size of data to write (chars)
+@Input          pszFormat          format string
+@Return         None
+*/ /**************************************************************************/
+__printf(3, 4)
+PVRSRV_ERROR PDumpSNPrintf(IMG_HANDLE hBuf, IMG_UINT32 ui32ScriptSizeMax, IMG_CHAR* pszFormat, ...);
+
+
+/*
+	PDumpWriteShiftedMaskedValue():
+
+	loads the "reference" address into an internal PDump register,
+	optionally shifts it right,
+	optionally shifts it left,
+	optionally masks it
+	then finally writes the computed value to the given destination address
+
+	i.e. it emits pdump language equivalent to this expression:
+
+	 dest = ((&ref) >> SHRamount << SHLamount) & MASK
+*/
+PVRSRV_ERROR
+PDumpWriteShiftedMaskedValue(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const IMG_CHAR *pszDestRegspaceName,
+                             const IMG_CHAR *pszDestSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiDestOffset,
+                             const IMG_CHAR *pszRefRegspaceName,
+                             const IMG_CHAR *pszRefSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiRefOffset,
+                             IMG_UINT32 uiSHRAmount,
+                             IMG_UINT32 uiSHLAmount,
+                             IMG_UINT32 uiMask,
+                             IMG_DEVMEM_SIZE_T uiWordSize,
+                             IMG_UINT32 uiPDumpFlags);
+
+/*
+	PDumpWriteSymbAddress():
+	writes the address of the "reference" to the offset given
+*/
+PVRSRV_ERROR
+PDumpWriteSymbAddress(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      const IMG_CHAR *pszDestSpaceName,
+                      IMG_DEVMEM_OFFSET_T uiDestOffset,
+                      const IMG_CHAR *pszRefSymbolicName,
+                      IMG_DEVMEM_OFFSET_T uiRefOffset,
+                      const IMG_CHAR *pszPDumpDevName,
+                      IMG_UINT32 ui32WordSize,
+                      IMG_UINT32 ui32AlignShift,
+                      IMG_UINT32 ui32Shift,
+                      IMG_UINT32 uiPDumpFlags);
+
+/* Register the connection with the PDump subsystem */
+PVRSRV_ERROR
+PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        void *hSyncPrivData,
+                        PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                        PDUMP_CONNECTION_DATA **ppsPDumpConnectionData);
+
+/* Unregister the connection with the PDump subsystem */
+void
+PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          PDUMP_CONNECTION_DATA *psPDumpConnectionData);
+
+/* Register for notification of PDump Transition into/out of capture range */
+PVRSRV_ERROR
+PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                PFN_PDUMP_TRANSITION pfnCallback,
+                                void *hPrivData,
+                                void *pvDevice,
+                                void **ppvHandle);
+
+/* Unregister notification of PDump Transition */
+void
+PDumpUnregisterTransitionCallback(void *pvHandle);
+
+PVRSRV_ERROR
+PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+                                         PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback,
+                                         void **ppvHandle);
+
+void
+PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle);
+
+/* Notify PDump of a Transition into/out of capture range */
+PVRSRV_ERROR
+PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                PDUMP_TRANSITION_EVENT eEvent,
+                IMG_UINT32 ui32PDumpFlags);
+
+/* Check if writing to a PDump file is permitted for the given device */
+IMG_BOOL PDumpIsDevicePermitted(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_LOCK(_ui32PDumpFlags) do \
+	{ if ((_ui32PDumpFlags & PDUMP_FLAGS_PDUMP_LOCK_HELD) == 0)\
+		{\
+			PDumpLock();\
+		}\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_UNLOCK(_ui32PDumpFlags) do \
+	{ if ((_ui32PDumpFlags & PDUMP_FLAGS_PDUMP_LOCK_HELD) == 0)\
+		{\
+			PDumpUnlock();\
+		}\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#define PDUMPINIT				PDumpInitCommon
+#define PDUMPDEINIT				PDumpDeInitCommon
+#define PDUMPREG32				PDumpReg32
+#define PDUMPREG64				PDumpReg64
+#define PDUMPREGREAD32			PDumpRegRead32
+#define PDUMPREGREAD64			PDumpRegRead64
+#define PDUMPCOMMENT(d, ...)	PDumpCommentWithFlags(d, PDUMP_FLAGS_CONTINUOUS, __VA_ARGS__)
+#define PDUMPCOMMENTWITHFLAGS	PDumpCommentWithFlags
+#define PDUMPREGPOL				PDumpRegPolKM
+#define PDUMPREGBASEDCBP		PDumpRegBasedCBP
+#define PDUMPENDINITPHASE		PDumpStopInitPhase
+#define PDUMPIDLWITHFLAGS		PDumpIDLWithFlags
+#define PDUMPIDL				PDumpIDL
+#define PDUMPPOWCMDSTART		PDumpPowerTransitionStart
+#define PDUMPPOWCMDEND			PDumpPowerTransitionEnd
+#define PDUMPCOM				PDumpCOMCommand
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_BLKSTART(_ui32PDumpFlags) do \
+	{ PDUMP_LOCK(_ui32PDumpFlags);\
+	_ui32PDumpFlags |= PDUMP_FLAGS_PDUMP_LOCK_HELD;\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_BLKEND(_ui32PDumpFlags) do \
+	{ _ui32PDumpFlags &= ~PDUMP_FLAGS_PDUMP_LOCK_HELD;\
+	PDUMP_UNLOCK(_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMPIF(_dev,_msg,_ui32PDumpFlags) do \
+	{PDUMP_BLKSTART(_ui32PDumpFlags);\
+	PDumpIfKM(_dev,_msg,_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#define PDUMPELSE				PDumpElseKM
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMPFI(_dev,_msg,_ui32PDumpFlags) do \
+	{ PDumpFiKM(_dev,_msg,_ui32PDumpFlags);\
+	PDUMP_BLKEND(_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#else
+/*
+	We should be clearer about which functions can be called
+	across the bridge as this looks rather unbalanced
+*/
+
+/*! Macro used to record a panic in the PDump script stream */
+#define PDUMP_PANIC(_dev, _id, _msg)  ((void)0)
+
+/*! Macro used to record a driver error in the PDump script stream to invalidate the capture */
+#define PDUMP_ERROR(_dev, _err, _msg) ((void)0)
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpInitCommon)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpInitCommon(void)
+{
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpConnectionNotify)
+#endif
+static INLINE void
+PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpDisconnectionNotify)
+#endif
+static INLINE void
+PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpLock)
+#endif
+static INLINE void
+PDumpLock(void)
+{
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnlock)
+#endif
+static INLINE void
+PDumpUnlock(void)
+{
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpStopInitPhase)
+#endif
+static INLINE void
+PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpSetFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE *psDeviceNode,
+                IMG_UINT32 ui32Frame)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32Frame);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE *psDeviceNode,
+                IMG_UINT32 *pui32Frame)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pui32Frame);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpCommentKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpCommentKM(CONNECTION_DATA *psConnection,
+               PVRSRV_DEVICE_NODE *psDeviceNode,
+               IMG_UINT32 ui32CommentSize,
+               IMG_CHAR *pszComment,
+               IMG_UINT32 ui32Flags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+    PVR_UNREFERENCED_PARAMETER(ui32CommentSize);
+	PVR_UNREFERENCED_PARAMETER(pszComment);
+	PVR_UNREFERENCED_PARAMETER(ui32Flags);
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpSetDefaultCaptureParamsKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Mode,
+                               IMG_UINT32 ui32Start,
+                               IMG_UINT32 ui32End,
+                               IMG_UINT32 ui32Interval,
+                               IMG_UINT32 ui32MaxParamFileSize,
+                               IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32Mode);
+	PVR_UNREFERENCED_PARAMETER(ui32Start);
+	PVR_UNREFERENCED_PARAMETER(ui32End);
+	PVR_UNREFERENCED_PARAMETER(ui32Interval);
+	PVR_UNREFERENCED_PARAMETER(ui32MaxParamFileSize);
+	PVR_UNREFERENCED_PARAMETER(ui32AutoTermTimeout);
+
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpPanic)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpPanic(IMG_UINT32      ui32PanicNo,
+           IMG_CHAR*       pszPanicMsg,
+           const IMG_CHAR* pszPPFunc,
+           IMG_UINT32      ui32PPline)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PanicNo);
+	PVR_UNREFERENCED_PARAMETER(pszPanicMsg);
+	PVR_UNREFERENCED_PARAMETER(pszPPFunc);
+	PVR_UNREFERENCED_PARAMETER(ui32PPline);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpCaptureError)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpCaptureError(PVRSRV_ERROR    ui32ErrorNo,
+                  IMG_CHAR*       pszErrorMsg,
+                  const IMG_CHAR* pszPPFunc,
+                  IMG_UINT32      ui32PPline)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32ErrorNo);
+	PVR_UNREFERENCED_PARAMETER(pszErrorMsg);
+	PVR_UNREFERENCED_PARAMETER(pszPPFunc);
+	PVR_UNREFERENCED_PARAMETER(ui32PPline);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpIsLastCaptureFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame)
+{
+	*pbIsLastCaptureFrame = IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetStateKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetStateKM(IMG_UINT64 *ui64State)
+{
+	*ui64State = 0;
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpForceCaptureStopKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                        PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpImageDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT32 ui32MMUContextID,
+                     IMG_CHAR *pszSABFileName,
+                     IMG_DEV_VIRTADDR sData,
+                     IMG_UINT32 ui32DataSize,
+                     IMG_UINT32 ui32LogicalWidth,
+                     IMG_UINT32 ui32LogicalHeight,
+                     IMG_UINT32 ui32PhysicalWidth,
+                     IMG_UINT32 ui32PhysicalHeight,
+                     PDUMP_PIXEL_FORMAT ePixFmt,
+                     IMG_MEMLAYOUT eMemLayout,
+                     IMG_FB_COMPRESSION eFBCompression,
+                     const IMG_UINT32 *paui32FBCClearColour,
+                     PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                     IMG_DEV_VIRTADDR sHeader,
+                     IMG_UINT32 ui32HeaderSize,
+                     IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpDataDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    IMG_UINT32 ui32MMUContextID,
+                    IMG_CHAR *pszSABFileName,
+                    IMG_DEV_VIRTADDR sData,
+                    IMG_UINT32 ui32DataSize,
+                    IMG_UINT32 ui32ElementType,
+                    IMG_UINT32 ui32ElementCount,
+                    IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterConnection)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        void *hSyncPrivData,
+                        PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                        PDUMP_CONNECTION_DATA **ppsPDumpConnectionData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hSyncPrivData);
+	PVR_UNREFERENCED_PARAMETER(pfnPDumpSyncBlocks);
+	PVR_UNREFERENCED_PARAMETER(ppsPDumpConnectionData);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterConnection)
+#endif
+static INLINE void
+PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterTransitionCallback)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                PFN_PDUMP_TRANSITION pfnCallback,
+                                void *hPrivData,
+                                void *pvDevice,
+                                void **ppvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+	PVR_UNREFERENCED_PARAMETER(pfnCallback);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pvDevice);
+	PVR_UNREFERENCED_PARAMETER(ppvHandle);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterTransitionCallback)
+#endif
+static INLINE void
+PDumpUnregisterTransitionCallback(void *pvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvHandle);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterTransitionCallback)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+                                         PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback,
+                                         void **ppvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pfnCallback);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(ppvHandle);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterTransitionCallbackFenceSync)
+#endif
+static INLINE void
+PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvHandle);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpTransition)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                PDUMP_TRANSITION_EVENT eEvent,
+                IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+	PVR_UNREFERENCED_PARAMETER(eEvent);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#if defined(__linux__) || defined(GCC_IA32) || defined(GCC_ARM) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+	#define PDUMPINIT	PDumpInitCommon
+	#define PDUMPDEINIT(args...)
+	#define PDUMPREG32(args...)
+	#define PDUMPREG64(args...)
+	#define PDUMPREGREAD32(args...)
+	#define PDUMPREGREAD64(args...)
+	#define PDUMPCOMMENT(args...)
+	#define PDUMPREGPOL(args...)
+	#define PDUMPSYNC(args...)
+	#define PDUMPCOPYTOMEM(args...)
+	#define PDUMPWRITE(args...)
+	#define PDUMPREGBASEDCBP(args...)
+	#define PDUMPCOMMENTWITHFLAGS(args...)
+	#define PDUMPENDINITPHASE(args...)
+	#define PDUMPIDLWITHFLAGS(args...)
+	#define PDUMPIDL(args...)
+	#define PDUMPPOWCMDSTART(args...)
+	#define PDUMPPOWCMDEND(args...)
+	#define PDUMP_LOCK(args...)
+	#define PDUMP_UNLOCK(args...)
+	#define PDUMPIF(args...)
+	#define PDUMPFI(args...)
+	#define PDUMPCOM(args...)
+#else
+	#error Compiler not specified
+#endif
+
+#endif /* PDUMP */
+
+#endif /* PDUMP_KM_H */
+
+/******************************************************************************
+ End of file (pdump_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_mmu.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_mmu.h
new file mode 100644
index 000000000000..5d03e430febd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_mmu.h
@@ -0,0 +1,150 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_PDUMP_MMU_H
+#define SRVKM_PDUMP_MMU_H
+
+/* services/server/include/ */
+#include "pdump_symbolicaddr.h"
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "mmu_common.h"
+
+#include "opaque_types.h"
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PDumpMMUMalloc(PPVRSRV_DEVICE_NODE psDeviceNode,
+               const IMG_CHAR *pszPDumpDevName,
+               MMU_LEVEL eMMULevel,
+               IMG_DEV_PHYADDR *psDevPAddr,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32Align,
+               PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpMMUFree(PPVRSRV_DEVICE_NODE psDeviceNode,
+             const IMG_CHAR *pszPDumpDevName,
+             MMU_LEVEL eMMULevel,
+             IMG_DEV_PHYADDR *psDevPAddr,
+             PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpPTBaseObjectToMem64(const IMG_CHAR *pszPDumpDevName,
+                         PMR *psPMRDest,
+                         IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                         IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                         IMG_UINT32 ui32Flags,
+                         MMU_LEVEL eMMULevel,
+                         IMG_UINT64 ui64PxSymAddr,
+                         IMG_UINT64 ui64PxOffset);
+
+PVRSRV_ERROR
+PDumpMMUDumpPxEntries(PPVRSRV_DEVICE_NODE psDeviceNode,
+                      MMU_LEVEL eMMULevel,
+                      const IMG_CHAR *pszPDumpDevName,
+                      void *pvPxMem,
+                      IMG_DEV_PHYADDR sPxDevPAddr,
+                      IMG_UINT32 uiFirstEntry,
+                      IMG_UINT32 uiNumEntries,
+                      const IMG_CHAR *pszMemspaceName,
+                      const IMG_CHAR *pszSymbolicAddr,
+                      IMG_UINT64 uiSymbolicAddrOffset,
+                      IMG_UINT32 uiBytesPerEntry,
+                      IMG_UINT32 uiLog2Align,
+                      IMG_UINT32 uiAddrShift,
+                      IMG_UINT64 uiAddrMask,
+                      IMG_UINT64 uiPxEProtMask,
+                      IMG_UINT64 uiDataValidEnable,
+                      IMG_UINT32 ui32Flags,
+                      IMG_UINT32 ui32VAParity,
+                      IMG_UINT32 ui32ParityShift,
+                      IMG_UINT64 ui64ParityMask,
+                      PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpMMUAllocMMUContext(PPVRSRV_DEVICE_NODE psDeviceNode,
+                        const IMG_CHAR *pszPDumpMemSpaceName,
+                        IMG_DEV_PHYADDR sPCDevPAddr,
+                        PDUMP_MMU_TYPE eMMUType,
+                        IMG_UINT32 *pui32MMUContextID,
+                        IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMMUFreeMMUContext(PPVRSRV_DEVICE_NODE psDeviceNode,
+                       const IMG_CHAR *pszPDumpMemSpaceName,
+                       IMG_UINT32 ui32MMUContextID,
+                       IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMMUSAB(PPVRSRV_DEVICE_NODE psDeviceNode,
+            const IMG_CHAR *pszPDumpMemNamespace,
+            IMG_UINT32 uiPDumpMMUCtx,
+            IMG_DEV_VIRTADDR sDevAddrStart,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            IMG_UINT32 ui32PDumpFlags);
+
+#define PDUMP_MMU_ALLOC_MMUCONTEXT(psDevNode, pszPDumpMemDevName, sPCDevPAddr, eMMUType, puiPDumpCtxID, ui32PDumpFlags) \
+        PDumpMMUAllocMMUContext(psDevNode,                              \
+                                pszPDumpMemDevName,                     \
+                                sPCDevPAddr,                            \
+                                eMMUType,                               \
+                                puiPDumpCtxID,                          \
+                                ui32PDumpFlags)
+
+#define PDUMP_MMU_FREE_MMUCONTEXT(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags) \
+        PDumpMMUFreeMMUContext(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags)
+#else /* PDUMP */
+
+#define PDUMP_MMU_ALLOC_MMUCONTEXT(pszPDumpMemDevName, sPCDevPAddr, eMMUType, puiPDumpCtxID, ui32PDumpFlags) \
+        ((void)0)
+#define PDUMP_MMU_FREE_MMUCONTEXT(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags) \
+        ((void)0)
+
+#endif /* PDUMP */
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_physmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_physmem.h
new file mode 100644
index 000000000000..bf38df94f3fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_physmem.h
@@ -0,0 +1,300 @@
+/**************************************************************************/ /*!
+@File
+@Title          pdump functions to assist with physmem allocations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PDUMP_PHYSMEM_H
+#define SRVSRV_PDUMP_PHYSMEM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pmr.h"
+#include "device.h" /* For device node */
+
+#define PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH 40
+#define PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH 60
+#define PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH (PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH + PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+
+typedef struct _PDUMP_PHYSMEM_INFO_T_ PDUMP_PHYSMEM_INFO_T;
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                     IMG_CHAR **ppszSymbolicAddress);
+
+PVRSRV_ERROR
+PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicAddress,
+            IMG_UINT64 ui64Size,
+            /* alignment is alignment of start of buffer _and_
+               minimum contiguity - i.e. smallest allowable
+               page-size. */
+            IMG_DEVMEM_ALIGN_T uiAlign,
+            IMG_BOOL bInitialise,
+            IMG_UINT8 ui8InitValue,
+            IMG_HANDLE *phHandlePtr,
+            IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    const IMG_CHAR *pszDevSpace,
+                    const IMG_CHAR *pszSymbolicAddress,
+                    IMG_UINT64 ui64Size,
+                    /* alignment is alignment of start of buffer _and_
+                       minimum contiguity - i.e. smallest allowable
+                       page-size. */
+                    IMG_DEVMEM_ALIGN_T uiAlign,
+                    IMG_BOOL bInitialise,
+                    IMG_UINT8 ui8InitValue,
+                    IMG_HANDLE *phHandlePtr,
+                    IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+          IMG_HANDLE hPDumpAllocationInfoHandle);
+
+PVRSRV_ERROR
+PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                  IMG_HANDLE hPDumpAllocationInfoHandle);
+
+void
+PDumpMakeStringValid(IMG_CHAR *pszString,
+                     IMG_UINT32 ui32StrLen);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetSymbolicAddr)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                     IMG_CHAR **ppszSymbolicAddress)
+{
+	PVR_UNREFERENCED_PARAMETER(hPhysmemPDumpHandle);
+	PVR_UNREFERENCED_PARAMETER(ppszSymbolicAddress);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicAddress,
+            IMG_UINT64 ui64Size,
+            IMG_DEVMEM_ALIGN_T uiAlign,
+            IMG_BOOL bInitialise,
+            IMG_UINT8 ui8InitValue,
+            IMG_HANDLE *phHandlePtr,
+            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pszDevSpace);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(uiAlign);
+	PVR_UNREFERENCED_PARAMETER(bInitialise);
+	PVR_UNREFERENCED_PARAMETER(ui8InitValue);
+	PVR_UNREFERENCED_PARAMETER(phHandlePtr);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    const IMG_CHAR *pszDevSpace,
+                    const IMG_CHAR *pszSymbolicAddress,
+                    IMG_UINT64 ui64Size,
+                    IMG_DEVMEM_ALIGN_T uiAlign,
+                    IMG_BOOL bInitialise,
+                    IMG_UINT8 ui8InitValue,
+                    IMG_HANDLE *phHandlePtr,
+                    IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pszDevSpace);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(uiAlign);
+	PVR_UNREFERENCED_PARAMETER(bInitialise);
+	PVR_UNREFERENCED_PARAMETER(ui8InitValue);
+	PVR_UNREFERENCED_PARAMETER(phHandlePtr);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+          IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hPDumpAllocationInfoHandle);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                  IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hPDumpAllocationInfoHandle);
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+#define PMR_DEFAULT_PREFIX "PMR"
+#define PMR_SPARSE_PREFIX "SPMR"
+#define PMR_SYMBOLICADDR_FMTSPEC "%s%"IMG_UINT64_FMTSPEC"_%"IMG_UINT64_FMTSPEC"_%s"
+#define PMR_MEMSPACE_FMTSPEC "%s"
+#define PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC "CC_%s"
+
+PVRSRV_ERROR
+PDumpPMRWRW32(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT32 ui32Value,
+              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW32InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRRDW32MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW64(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT64 ui64Value,
+              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW64InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRRDW64MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRLDB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFileName,
+            IMG_UINT32 uiFileOffset);
+
+/*
+  PDumpPMRPOL()
+
+  Emits a POL to the PDUMP.
+*/
+PVRSRV_ERROR
+PDumpPMRPOL(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMempaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_UINT32 ui32Value,
+            IMG_UINT32 ui32Mask,
+            PDUMP_POLL_OPERATOR eOperator,
+            IMG_UINT32 uiCount,
+            IMG_UINT32 uiDelay,
+            PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize);
+
+/*
+ * PDumpWriteParameterBlob()
+ *
+ * Writes a binary blob to the pdump param stream containing the current
+ * contents of the memory, and returns the filename and offset of where
+ * that blob is located (for use in a subsequent LDB, for example).
+ *
+ * Caller to provide buffer to receive filename, and declare the size of
+ * that buffer.
+ */
+PVRSRV_ERROR
+PDumpWriteParameterBlob(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiNumBytes,
+                        PDUMP_FLAGS_T uiPDumpFlags,
+                        IMG_CHAR *pszFilenameOut,
+                        size_t uiFilenameBufSz,
+                        PDUMP_FILEOFFSET_T *puiOffsetOut);
+
+#endif /* #ifndef SRVSRV_PDUMP_PHYSMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_symbolicaddr.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_symbolicaddr.h
new file mode 100644
index 000000000000..ed912a5096c9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_symbolicaddr.h
@@ -0,0 +1,55 @@
+/**************************************************************************/ /*!
+@File
+@Title          Abstraction of PDUMP symbolic address derivation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Allows pdump functions to derive symbolic addresses on-the-fly
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_PDUMP_SYMBOLICADDR_H
+#define SRVKM_PDUMP_SYMBOLICADDR_H
+
+#include "img_types.h"
+
+#include "pvrsrv_error.h"
+
+/* pdump symbolic addresses are generated on-the-fly with a callback */
+
+typedef PVRSRV_ERROR (*PVRSRV_SYMADDRFUNCPTR)(IMG_HANDLE hPriv, IMG_UINT32 uiOffset, IMG_CHAR *pszSymbolicAddr, IMG_UINT32 ui32SymbolicAddrLen, IMG_UINT32 *pui32NewOffset);
+
+#endif /* #ifndef SRVKM_PDUMP_SYMBOLICADDR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem.h
new file mode 100644
index 000000000000..6e2469f73e65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem.h
@@ -0,0 +1,303 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for common entry point for creation of RAM backed PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_H
+#define SRVSRV_PHYSMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/* Valid values for TC_MEMORY_CONFIG configuration option */
+#define TC_MEMORY_LOCAL			(1)
+#define TC_MEMORY_HOST			(2)
+#define TC_MEMORY_HYBRID		(3)
+
+/* Valid values for the PLATO_MEMORY_CONFIG configuration option */
+#define PLATO_MEMORY_LOCAL		(1)
+#define PLATO_MEMORY_HOST		(2)
+#define PLATO_MEMORY_HYBRID		(3)
+
+/*************************************************************************/ /*!
+@Function       DevPhysMemAlloc
+@Description    Allocate memory from device specific heaps directly.
+@Input          psDevNode             device node to operate on
+@Input          ui32MemSize           Size of the memory to be allocated
+@Input          u8Value               Value to be initialised to.
+@Input          bInitPage             Flag to control initialisation
+@Input          pszDevSpace           PDUMP memory space in which the
+                                        allocation is to be done
+@Input          pszSymbolicAddress    Symbolic name of the allocation
+@Input          phHandlePtr           PDUMP handle to the allocation
+@Input          uiPid                 PID of the process owning the allocation
+                                        (or PVR_SYS_ALLOC_PID if the allocation
+                                        belongs to the driver)
+@Output         hMemHandle            Handle to the allocated memory
+@Output         psDevPhysAddr         Device Physical address of allocated
+                                        page
+@Return         PVRSRV_OK if the allocation is successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevPhysMemAlloc(PVRSRV_DEVICE_NODE *psDevNode,
+                IMG_UINT32 ui32MemSize,
+                IMG_UINT32 ui32Log2Align,
+                const IMG_UINT8 u8Value,
+                IMG_BOOL bInitPage,
+#if defined(PDUMP)
+                const IMG_CHAR *pszDevSpace,
+                const IMG_CHAR *pszSymbolicAddress,
+                IMG_HANDLE *phHandlePtr,
+#endif
+                IMG_PID uiPid,
+                IMG_HANDLE hMemHandle,
+                IMG_DEV_PHYADDR *psDevPhysAddr);
+
+/*************************************************************************/ /*!
+@Function       DevPhysMemFree
+@Description    Free memory to device specific heaps directly.
+@Input          psDevNode             device node to operate on
+@Input          hPDUMPMemHandle       Pdump handle to allocated memory
+@Input          hMemHandle            Devmem handle to allocated memory
+@Return         None
+*/ /**************************************************************************/
+void
+DevPhysMemFree(PVRSRV_DEVICE_NODE *psDevNode,
+#if defined(PDUMP)
+               IMG_HANDLE hPDUMPMemHandle,
+#endif
+               IMG_HANDLE hMemHandle);
+
+/*
+ * PhysmemNewRamBackedPMR
+ *
+ * This function will create a RAM backed PMR using the device specific
+ * callback, this allows control at a per-devicenode level to select the
+ * memory source thus supporting mixed UMA/LMA systems.
+ *
+ * The size must be a multiple of page size. The page size is specified in
+ * log2. It should be regarded as a minimum contiguity of which the
+ * resulting memory must be a multiple. It may be that this should be a fixed
+ * number. It may be that the allocation size needs to be a multiple of some
+ * coarser "page size" than that specified in the page size argument.
+ * For example, take an OS whose page granularity is a fixed 16kB, but the
+ * caller requests memory in page sizes of 4kB. The request can be satisfied
+ * if and only if the SIZE requested is a multiple of 16kB. If the arguments
+ * supplied are such that this OS cannot grant the request,
+ * PVRSRV_ERROR_INVALID_PARAMS will be returned.
+ *
+ * The caller should supply storage of a pointer. Upon successful return a
+ * PMR object will have been created and a pointer to it returned in the
+ * PMROut argument.
+ *
+ * A PMR successfully created should be destroyed with PhysmemUnrefPMR.
+ *
+ * Note that this function may cause memory allocations and on some operating
+ * systems this may cause scheduling events, so it is important that this
+ * function be called with interrupts enabled and in a context where
+ * scheduling events and memory allocations are permitted.
+ *
+ * The flags may be used by the implementation to change its behaviour if
+ * required. The flags will also be stored in the PMR as immutable metadata
+ * and returned to mmu_common when it asks for it.
+ *
+ * The PID specified is used to tie this allocation to the process context
+ * that the allocation is made on behalf of.
+ */
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR(CONNECTION_DATA * psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMROut,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags);
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR_direct(CONNECTION_DATA * psConnection,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  IMG_DEVMEM_SIZE_T uiSize,
+							  IMG_UINT32 ui32NumPhysChunks,
+							  IMG_UINT32 ui32NumVirtChunks,
+							  IMG_UINT32 *pui32MappingTable,
+							  IMG_UINT32 uiLog2PageSize,
+							  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							  IMG_UINT32 uiAnnotationLength,
+							  const IMG_CHAR *pszAnnotation,
+							  IMG_PID uiPid,
+							  PMR **ppsPMROut,
+							  IMG_UINT32 ui32PDumpFlags,
+							  PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysmemImportPMR
+@Description    Import PMR a previously exported PMR
+@Input          psPMRExport           The exported PMR token
+@Input          uiPassword            Authorisation password
+                                      for the PMR being imported
+@Input          uiSize                Size of the PMR being imported
+                                      (for verification)
+@Input          uiLog2Contig          Log2 continuity of the PMR being
+                                      imported (for verification)
+@Output         ppsPMR                The imported PMR
+@Return         PVRSRV_ERROR_PMR_NOT_PERMITTED if not for the same device
+                PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR if password incorrect
+                PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES if size or contiguity incorrect
+                PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemImportPMR(CONNECTION_DATA *psConnection,
+                 PVRSRV_DEVICE_NODE *psDevNode,
+                 PMR_EXPORT *psPMRExport,
+                 PMR_PASSWORD_T uiPassword,
+                 PMR_SIZE_T uiSize,
+                 PMR_LOG2ALIGN_T uiLog2Contig,
+                 PMR **ppsPMR);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDefaultPhysicalHeapKM
+@Description    For the specified device, get the physical heap used for
+                allocations when the PVRSRV_PHYS_HEAP_DEFAULT
+                physical heap hint is set in memalloc flags.
+@Output         peHeap                 Default Heap return value
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVGetDefaultPhysicalHeapKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDevNode,
+                               PVRSRV_PHYS_HEAP *peHeap);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVPhysHeapGetMemInfoKM
+@Description    Get the memory usage statistics for a given physical heap ID
+@Input          ui32PhysHeapCount      Physical Heap count
+@Input          paePhysHeapID          Array of Physical Heap ID's
+@Output         paPhysHeapMemStats     Buffer to hold the memory statistics
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfoKM(CONNECTION_DATA *psConnection,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           IMG_UINT32 ui32PhysHeapCount,
+                           PVRSRV_PHYS_HEAP *paePhysHeapID,
+                           PHYS_HEAP_MEM_STATS_V1 *paPhysHeapMemStats);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVPhysHeapGetMemInfoKM
+@Description    Get the memory usage statistics for a given physical heap ID
+@Input          ui32PhysHeapCount      Physical Heap count
+@Input          paePhysHeapID          Array of Physical Heap ID's
+@Output         paPhysHeapMemStats     Buffer to hold the memory statistics
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfo2KM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                            IMG_UINT32 ui32PhysHeapCount,
+                            PVRSRV_PHYS_HEAP *paePhysHeapID,
+                            PHYS_HEAP_MEM_STATS_V2 *paPhysHeapMemStats);
+
+/*************************************************************************/ /*!
+@Function       PhysMemValidateMappingTable
+@Description    Checks the PMR mapping table provided is valid (ie has
+                no entries with an index value outside the valid range
+                of indices for the allocation and has no repeated indices)
+
+@Input          ui32TotalNumVirtChunks Total number of virtual chunks
+                                       the allocation has.
+@Input          ui32IndexCount         Number of entries in the mapping
+                                       table.
+@Input          pui32MappingTable      Mapping Table.
+@Return         PVRSRV_OK              if parameters are valid.
+                PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY if mapping table
+                                       contains an index out of range or a
+                                       repeated index
+                PVRSRV_ERROR_OUT_OF_MEMORY if unable to allocate memory
+                                       for the index tracking table
+                                       (used internally in this function)
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysMemValidateMappingTable(IMG_UINT32 ui32TotalNumVirtChunks,
+                            IMG_UINT32 ui32IndexCount,
+                            const IMG_UINT32 *pui32MappingTable);
+
+/*************************************************************************/ /*!
+@Function       PhysMemValidateParams
+@Description    Checks the PMR creation parameters and adjusts them
+                if possible and necessary
+
+@Input          psDevNode              The associated device node.
+@Input          ui32NumPhysChunks      Number of physical chunks.
+@Input          ui32NumVirtChunks      Number of virtual chunks.
+@Input          pui32MappingTable      Mapping Table.
+@Input          uiFlags                Allocation flags.
+@Input          uiPid                  PID of current process.
+@Inout          puiLog2AllocPageSize   Log2 of allocation page size.
+                                       May be adjusted.
+@Inout          puiSize                Size of the allocation.
+                                       May be adjusted.
+@Return         PVRSRV_OK if parameters are valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysMemValidateParams(PVRSRV_DEVICE_NODE *psDevNode,
+                      IMG_UINT32 ui32NumPhysChunks,
+                      IMG_UINT32 ui32NumVirtChunks,
+                      IMG_UINT32 *pui32MappingTable,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      IMG_PID uiPid,
+                      IMG_UINT32 *puiLog2AllocPageSize,
+                      IMG_DEVMEM_SIZE_T *puiSize);
+
+#endif /* SRVSRV_PHYSMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_cpumap_history.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_cpumap_history.h
new file mode 100644
index 000000000000..0f78191d9746
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_cpumap_history.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File
+@Title          Definitions of PMR Mapping History for OS managed memory
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PHYSMEM_CPUMAP_HISTORY_H_
+#define PHYSMEM_CPUMAP_HISTORY_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/mem_types.h>
+
+/*!
+*******************************************************************************
+ @Function	  CPUMappingHistoryInit
+ @Description CPU Mapping history initialisation
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR CPUMappingHistoryInit(void);
+
+/*!
+*******************************************************************************
+ @Function	  CPUMappingHistoryDeInit
+ @Description CPU Mapping history De-initialisation
+ @Return      None
+******************************************************************************/
+void CPUMappingHistoryDeInit(void);
+
+/*!
+*******************************************************************************
+ @Function	  InsertMappingRecord
+ @Description CPU Mapping history Map Record insertion.
+              This function given relevant mapping information, inserts a
+              mapping record to the history buffer.
+ @Input       pszAnnotation - The annotation related to the allocation to be
+                              mapped.
+ @Input       uiPID         - The PID of the process mapping the allocation.
+ @Input       pvAddress     - The CPU virtual address of the newly mapped
+                              allocation.
+ @Input       sCPUPhyAddr       - The CPU Physical address of the newly mapped
+                                  allocation.
+ @Input       ui32CPUCacheFlags - The CPU Caching flags associated with the
+                                  mapping.
+ @Input       uiMapOffset       - The offset into the PMR at which the mapping
+                                  resides.
+ @input       ui32PageCount     - The number of pages mapped.
+ @Return      None
+******************************************************************************/
+void InsertMappingRecord(const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPID,
+                         IMG_CPU_VIRTADDR pvAddress,
+                         IMG_CPU_PHYADDR sCpuPhyAddr,
+                         IMG_UINT32 ui32CPUCacheFlags,
+                         size_t uiMapOffset,
+                         IMG_UINT32 ui32PageCount);
+
+/*!
+*******************************************************************************
+ @Function	  InsertUnMappingRecord
+ @Description CPU Mapping history UnMap Record insertion.
+              This function given relevant mapping information, inserts an
+              un-mapping record to the history buffer.
+ @Input       pvAddress     - The CPU virtual address of the un-mapped
+                              allocation.
+ @Input       sCPUPhyAddr       - The CPU Physical address of the un-mapped
+                                  allocation.
+ @Input       ui32CPUCacheFlags - The CPU Caching flags associated with the
+                                  mapping.
+ @input       ui32PageCount     - The number of pages un-mapped.
+ @Return      None
+******************************************************************************/
+void InsertUnMappingRecord(IMG_CPU_VIRTADDR pvAddress,
+                           IMG_CPU_PHYADDR sCpuPhyAddr,
+                           IMG_UINT32 ui32CPUCacheFlags,
+                           IMG_UINT32 ui32PageCount);
+
+
+#endif /* PHYSMEM_CPUMAP_HISTORY_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dlm.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dlm.h
new file mode 100644
index 000000000000..7d18fd781f2f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dlm.h
@@ -0,0 +1,88 @@
+/*************************************************************************/ /*!
+@File           physmem_dlm.h
+@Title          Dedicated Local Memory allocator for Physical Memory Blocks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for dedicated local memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_DLM_H
+#define PHYSMEM_DLM_H
+
+#include "img_types.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+
+/* PMBDestroy
+ *
+ * Destroys a given PMB used to represent a block of memory
+ * obtained from a DLM heap.
+ */
+void
+PMBDestroy(PMB *psPMB);
+
+/* PMBGetAnnotation
+ *
+ * Obtain the annotation of the allocation represented
+ * by this PMB
+ */
+const IMG_CHAR *
+PMBGetAnnotation(PMB *psPMB);
+
+/* DLM */
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapDLM
+@Description    Create and register new DLM heap.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Input          pszLabel     Debug identifier label
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapDLM(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* #ifndef PHYSMEM_DLM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dmabuf.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dmabuf.h
new file mode 100644
index 000000000000..1896204d3f6e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dmabuf.h
@@ -0,0 +1,133 @@
+/**************************************************************************/ /*!
+@File           physmem_dmabuf.h
+@Title          Header for dmabuf PMR factory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks importing Ion allocations
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PHYSMEM_DMABUF_H)
+#define PHYSMEM_DMABUF_H
+
+#include <linux/dma-buf.h>
+
+#include "pvr_dma_resv.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+#include "pmr.h"
+
+typedef void (*PFN_DESTROY_DMABUF_PMR)(PHYS_HEAP *psHeap,
+                                       struct dma_buf_attachment *psAttachment);
+
+PVRSRV_ERROR
+PhysmemCreateNewDmaBufBackedPMR(PHYS_HEAP *psHeap,
+                                struct dma_buf_attachment *psAttachment,
+                                PFN_DESTROY_DMABUF_PMR pfnDestroy,
+                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                IMG_PID uiPid,
+                                IMG_DEVMEM_SIZE_T uiChunkSize,
+                                IMG_UINT32 ui32NumPhysChunks,
+                                IMG_UINT32 ui32NumVirtChunks,
+                                IMG_UINT32 *pui32MappingTable,
+		                        IMG_UINT32 ui32NameSize,
+		                        const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                                PMR **ppsPMRPtr);
+
+struct dma_buf *
+PhysmemGetDmaBuf(PMR *psPMR);
+
+struct dma_resv *
+PhysmemGetDmaResv(PMR *psPMR);
+
+#if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+struct dma_heap *
+PhysmemGetDmaHeap(PMR *psPMR);
+
+void
+PhysmemSetDmaHeap(PMR *psPMR, struct dma_heap *psDmaHeap);
+#endif /* #if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND) */
+
+PVRSRV_ERROR
+PhysmemExportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    PMR *psPMR,
+                    IMG_INT *piFd);
+
+PVRSRV_ERROR
+PhysmemExportGemHandle(CONNECTION_DATA *psConnection,
+		       PVRSRV_DEVICE_NODE *psDevNode,
+		       PMR *psPMR,
+		       IMG_UINT32 *puHandle);
+
+PVRSRV_ERROR
+PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_INT fd,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    IMG_UINT32 ui32NameSize,
+                    const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                    PMR **ppsPMRPtr,
+                    IMG_DEVMEM_SIZE_T *puiSize,
+                    IMG_DEVMEM_ALIGN_T *puiAlign);
+
+PVRSRV_ERROR
+PhysmemImportSparseDmaBuf(CONNECTION_DATA *psConnection,
+                          PVRSRV_DEVICE_NODE *psDevNode,
+                          IMG_INT fd,
+                          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                          IMG_DEVMEM_SIZE_T uiChunkSize,
+                          IMG_UINT32 ui32NumPhysChunks,
+                          IMG_UINT32 ui32NumVirtChunks,
+                          IMG_UINT32 *pui32MappingTable,
+                          IMG_UINT32 ui32NameSize,
+                          const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                          PMR **ppsPMRPtr,
+                          IMG_DEVMEM_SIZE_T *puiSize,
+                          IMG_DEVMEM_ALIGN_T *puiAlign);
+
+#endif /* !defined(PHYSMEM_DMABUF_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_extmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_extmem.h
new file mode 100644
index 000000000000..cc7d23297d43
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_extmem.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem_extmem header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for common entry point for creation of wrapped memory PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_EXTMEM_H
+#define SRVSRV_PHYSMEM_EXTMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/*
+ * Based on PhysmemNewRamBackedPMR but uses a passed in virtual address.
+ */
+PVRSRV_ERROR
+PhysmemWrapExtMemOS(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    IMG_CPU_VIRTADDR pvCpuVAddr,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    PMR **ppsPMRPtr);
+
+PVRSRV_ERROR
+PhysmemWrapExtMem(CONNECTION_DATA *psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT64 pvCpuVAddr,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  PMR **ppsPMROut);
+
+#endif /* SRVSRV_PHYSMEM_EXTMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_hostmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_hostmem.h
new file mode 100644
index 000000000000..cfa453de343a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_hostmem.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File           physmem_hostmem.h
+@Title          Host memory device node header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PHYSMEM_HOSTMEM_H)
+#define PHYSMEM_HOSTMEM_H
+
+#include "pvrsrv_device.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       HostMemDeviceCreate
+@Description    Allocate memory for and create host memory device node.
+@Output         ppsDeviceNode Pointer to device node pointer.
+@Return         PVRSRV_ERROR  PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR HostMemDeviceCreate(PVRSRV_DEVICE_NODE **ppsDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       HostMemDeviceDestroy
+@Description    Destroy host memory device node.
+@Input          psDeviceNode  Pointer to device node.
+*/ /**************************************************************************/
+void HostMemDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* !defined(PHYSMEM_HOSTMEM_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ima.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ima.h
new file mode 100644
index 000000000000..4744dacacb31
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ima.h
@@ -0,0 +1,77 @@
+/**************************************************************************/ /*!
+@File
+@Title          Header import memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of memory management. This module is responsible for
+                implementing the function callbacks for local card memory when
+                used under a shared heap system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PHYSMEM_IMA_H
+#define PHYSMEM_IMA_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapIMA
+@Description    Create and register new IMA heap with IMA specific details and
+                a DLM heap backing.
+@Input          psDevNode     Pointer to device node struct.
+@Input          uiPolicy      Heap allocation policy flags
+@Input          psConfig      Heap configuration.
+@Input          pszLabel      Debug identifier label
+@Input          psDLMHeap     DLM heap backing this LMA heap.
+@Input          uiLog2PMBSize Log 2 of PMB Size in bytes supported by DLM heap.
+@Output         ppsPhysHeap   Pointer to the created heap.
+@Return         PVRSRV_ERROR  PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapIMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP *psDLMHeap,
+                     IMG_UINT32 uiLog2PMBSize,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* PHYSMEM_IMA_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_lma.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_lma.h
new file mode 100644
index 000000000000..af42d8826dac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_lma.h
@@ -0,0 +1,75 @@
+/**************************************************************************/ /*!
+@File
+@Title          Header for local card memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for local card memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_LMA_H
+#define SRVSRV_PHYSMEM_LMA_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "device.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapLMA
+@Description    Create and register new LMA heap with LMA specific details.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Input          pszLabel     Debug identifier label
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapLMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* #ifndef SRVSRV_PHYSMEM_LMA_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_osmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_osmem.h
new file mode 100644
index 000000000000..5e76513477b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_osmem.h
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File           physmem_osmem.h
+@Title          OS memory PMR factory API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                OS memory PMR factory API that must be defined so that the
+                common & device layer code in the Services Server can allocate
+                new PMRs back with pages from the OS page allocator. Applicable
+                for UMA based platforms, such platforms must implement this API
+                in the OS Porting layer, in the "env" directory for that
+                system.
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_OSMEM_H
+#define PHYSMEM_OSMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "connection_server.h"
+#include "physheap.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapOSMEM
+@Description    Create and register new OSMEM heap with OSMEM specific details.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapOSMEM(PVRSRV_DEVICE_NODE *psDevNode,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       PHYS_HEAP_CONFIG *psConfig,
+                       PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysmemNewOSRamBackedPMR
+@Description    Rogue Services will call this function to allocate GPU device
+                memory from the PMR factory supported by the OS DDK port. This
+                factory typically obtains physical memory from the kernel/OS
+                API that allocates memory from the default heap of shared
+                system memory available on the platform. The allocated memory
+                must be page-aligned and be a whole number of pages.
+                After allocating the required memory, the implementation must
+                then call PMRCreatePMR() to obtain the PMR structure that
+                describes this allocation to the upper layers of the Services.
+                memory management sub-system.
+                NB. Implementation of this function is mandatory. If shared
+                system memory is not to be used in the OS port then the
+                implementation must return PVRSRV_ERROR_NOT_SUPPORTED.
+
+@Input          psPhysHeap       The phys heap
+@Input          psConnection     The connection to the originator process
+@Input          uiSize           The size of the allocation
+                                 (must be a multiple of page size)
+@Input          ui32NumPhysChunks     When sparse allocations are requested,
+                                      this is the number of physical chunks
+                                      to be allocated.
+                                      For regular allocations, this will be 1.
+@Input          ui32NumLogicalChunks  When sparse allocations are requested,
+                                      this is the number of logical chunks
+                                      covering the sparse allocation.
+                                      For regular allocations, this will be 1.
+@Input          pui32MappingTable  When sparse allocations are requested,
+                                   this is the list of the indices of
+                                   each physically-backed virtual chunk
+                                   For regular allocations, this will
+                                   be NULL.
+@Input          uiLog2DevPageSize  The physical dev pagesize in log2(bytes).
+@Input          uiFlags            The allocation flags.
+@Input          pszAnnotation      String describing the PMR (for debug).
+                                   This should be passed into the function
+                                   PMRCreatePMR().
+@Input          uiPid              The process ID that this allocation should
+                                   be associated with.
+@Output         ppsPMROut          Pointer to the PMR created for the
+                                   new allocation
+@Input          ui32PDumpFlags     The pdump flags.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                         CONNECTION_DATA *psConnection,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         IMG_UINT32 ui32NumPhysChunks,
+                         IMG_UINT32 ui32NumLogicalChunks,
+                         IMG_UINT32 *pui32MappingTable,
+                         IMG_UINT32 uiLog2DevPageSize,
+                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                         const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPid,
+                         PMR **ppsPMROut,
+                         IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysmemGetOSRamMemStats
+@Description    Function that gets the OS memory usage statistics
+@Input          pvImplData     Physical heap private data.
+@Output         pui64TotalSize Buffer that holds the total OS memory size
+@Output         pui64FreeSize  Buffer that holds the free OS memory size
+@Return         None.
+*/ /**************************************************************************/
+void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                            IMG_UINT64 *pui64TotalSize,
+                            IMG_UINT64 *pui64FreeSize);
+
+#endif /* PHYSMEM_OSMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ramem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ramem.h
new file mode 100644
index 000000000000..9dbf668e21b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ramem.h
@@ -0,0 +1,109 @@
+/*************************************************************************/ /*!
+@File           physmem_ramem.h
+@Title          Resource allocator managed PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                RA managed memory PMR factory API that is shared between local
+                physheap implementations (LMA & IMA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_RAMEM_H
+#define PHYSMEM_RAMEM_H
+
+#include "img_types.h"
+#include "pvrsrv_memallocflags.h"
+#include "physheap.h"
+#include "ra.h"
+#include "connection_server.h"
+#include "pmr_impl.h"
+
+IMG_UINT32
+RAMemGetPageShift(void);
+
+PVRSRV_ERROR
+RAMemDoPhyContigPagesAlloc(RA_ARENA *pArena,
+                           size_t uiSize,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_PID uiPid);
+
+void
+RAMemDoPhyContigPagesFree(RA_ARENA *pArena,
+                          PG_HANDLE *psMemHandle);
+
+PVRSRV_ERROR
+RAMemPhyContigPagesMap(PHYS_HEAP *psPhysHeap,
+                  PG_HANDLE *psMemHandle,
+                  size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                  void **pvPtr);
+
+void
+RAMemPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap,
+                    PG_HANDLE *psMemHandle,
+                    void *pvPtr);
+
+PVRSRV_ERROR
+RAMemPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                    PG_HANDLE *psMemHandle,
+                    IMG_UINT32 uiOffset,
+                    IMG_UINT32 uiLength);
+
+/*
+ * PhysmemNewRAMemRamBackedPMR
+ *
+ * This function will create a PMR using the memory managed by
+ * the PMR factory and is OS agnostic.
+ */
+PVRSRV_ERROR
+PhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                            RA_ARENA *pArena,
+                            CONNECTION_DATA *psConnection,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_UINT32 ui32NumPhysChunks,
+                            IMG_UINT32 ui32NumVirtChunks,
+                            IMG_UINT32 *pui32MappingTable,
+                            IMG_UINT32 uiLog2PageSize,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                            const IMG_CHAR *pszAnnotation,
+                            IMG_PID uiPid,
+                            PMR **ppsPMRPtr,
+                            IMG_UINT32 ui32PDumpFlags);
+
+#endif /* PHYSMEM_RAMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr.h
new file mode 100644
index 000000000000..c932f983ba5c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr.h
@@ -0,0 +1,1309 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                the "PMR" abstraction.  A PMR (Physical Memory Resource)
+                represents some unit of physical memory which is
+                allocated/freed/mapped/unmapped as an indivisible unit
+                (higher software levels provide an abstraction above that
+                to deal with dividing this down into smaller manageable units).
+                Importantly, this module knows nothing of virtual memory, or
+                of MMUs etc., with one excusable exception.  We have the
+                concept of a "page size", which really means nothing in
+                physical memory, but represents a "contiguity quantum" such
+                that the higher level modules which map this memory are able
+                to verify that it matches the needs of the page size for the
+                virtual realm into which it is being mapped.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PMR_H
+#define SRVSRV_PMR_H
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"			/* Required for export DEVMEM_EXPORTCOOKIE */
+
+/* services/include */
+#include "pdump.h"
+#include "physheap.h"
+
+/* services/server/include/ */
+#include "pmr_impl.h"
+#include "opaque_types.h"
+
+#define PMR_MAX_TRANSLATION_STACK_ALLOC				(32)
+
+/* Maximum size PMR can have is 8G of memory */
+#define PMR_MAX_SUPPORTED_SIZE IMG_UINT64_C(0x200000000)
+/* Max number of pages in a PMR at 4k page size */
+#define PMR_MAX_SUPPORTED_4K_PAGE_COUNT (PMR_MAX_SUPPORTED_SIZE >> 12ULL)
+
+typedef IMG_UINT64 PMR_BASE_T;
+typedef IMG_UINT64 PMR_SIZE_T;
+#define PMR_SIZE_FMTSPEC "0x%010"IMG_UINT64_FMTSPECX
+#define PMR_VALUE32_FMTSPEC "0x%08X"
+#define PMR_VALUE64_FMTSPEC "0x%016"IMG_UINT64_FMTSPECX
+typedef IMG_UINT32 PMR_LOG2ALIGN_T;
+typedef IMG_UINT64 PMR_PASSWORD_T;
+
+struct _PMR_MAPPING_TABLE_
+{
+	PMR_SIZE_T uiChunkSize;            /*!< Size of a "chunk" */
+	IMG_UINT32 ui32NumPhysChunks;      /*!< Number of physical chunks that are valid */
+	IMG_UINT32 ui32NumLogicalChunks;   /*!< Number of logical chunks in the mapping */
+	/* Must be last */
+	IMG_UINT32 aui32Translation[IMG_FLEX_ARRAY_MEMBER];    /*!< Translation mapping for "logical" to physical */
+};
+
+#define TRANSLATION_INVALID 0xFFFFFFFFUL
+
+typedef struct _PMR_EXPORT_ PMR_EXPORT;
+
+typedef struct _PMR_PAGELIST_ PMR_PAGELIST;
+
+IMG_INT32 PMRGetLiveCount(void);
+
+/*
+ * PMRValidateSize
+ *
+ * Given a size value, check the value against the max supported
+ * PMR size of 1GB. Return IMG_FALSE if size exceeds max, IMG_TRUE
+ * otherwise.
+ */
+static inline IMG_BOOL PMRValidateSize(IMG_UINT64 uiSize)
+{
+	return (uiSize > PMR_MAX_SUPPORTED_SIZE) ? IMG_FALSE : IMG_TRUE;
+}
+
+/*
+ * PMRCreatePMR
+ *
+ * Not to be called directly, only via implementations of PMR
+ * factories, e.g. in physmem_osmem.c, deviceclass.c, etc.
+ *
+ * Creates a PMR object, with callbacks and private data as per the
+ * FuncTab/PrivData args.
+ *
+ * Note that at creation time the PMR must set in stone the "logical
+ * size" and the "contiguity guarantee"
+ *
+ * Flags are also set at this time.  (T.B.D.  flags also immutable for
+ * the life of the PMR?)
+ *
+ * Logical size is the amount of Virtual space this allocation would
+ * take up when mapped.  Note that this does not have to be the same
+ * as the actual physical size of the memory.  For example, consider
+ * the sparsely allocated non-power-of-2 texture case.  In this
+ * instance, the "logical size" would be the virtual size of the
+ * rounded-up power-of-2 texture.  That some pages of physical memory
+ * may not exist does not affect the logical size calculation.
+ *
+ * The PMR must also supply the "contiguity guarantee" which is the
+ * finest granularity of alignment and size of physical pages that the
+ * PMR will provide after LockSysPhysAddresses is called.  Note that
+ * the calling code may choose to call PMRSysPhysAddr with a finer
+ * granularity than this, for example if it were to map into a device
+ * MMU with a smaller page size, and it's also OK for the PMR to
+ * supply physical memory in larger chunks than this.  But
+ * importantly, never the other way around.
+ *
+ * More precisely, the following inequality must be maintained
+ * whenever mappings and/or physical addresses exist:
+ *
+ *       (device MMU page size) <= 2**(uiLog2ContiguityGuarantee) <= (actual contiguity of physical memory)
+ *
+ * The function table will contain the following callbacks which may
+ * be overridden by the PMR implementation:
+ *
+ * pfnLockPhysAddresses
+ *
+ *      Called when someone locks requests that Physical pages are to
+ *      be locked down via the PMRLockSysPhysAddresses() API.  Note
+ *      that if physical pages are prefaulted at PMR creation time and
+ *      therefore static, it would not be necessary to override this
+ *      function, in which case NULL may be supplied.
+ *
+ * pfnUnlockPhysAddresses
+ *
+ *      The reverse of pfnLockPhysAddresses.  Note that this should be
+ *      NULL if and only if pfnLockPhysAddresses is NULL
+ *
+ * pfnSysPhysAddr
+ *
+ *      This function is mandatory.  This is the one which returns the
+ *      system physical address for a given offset into this PMR.  The
+ *      "lock" function will have been called, if overridden, before
+ *      this function, thus the implementation should not increase any
+ *      refcount when answering this call.  Refcounting, if necessary,
+ *      should be done in the lock/unlock calls.  Refcounting would
+ *      not be necessary in the prefaulted/static scenario, as the
+ *      pmr.c abstraction will handle the refcounting for the whole
+ *      PMR.
+ *
+ * pfnFinalize
+ *
+ *      Called when the PMR's refcount reaches zero and it gets
+ *      destroyed.  This allows the implementation to free up any
+ *      resource acquired during creation time.
+ *
+ */
+PVRSRV_ERROR
+PMRCreatePMR(PHYS_HEAP *psPhysHeap,
+             PMR_SIZE_T uiLogicalSize,
+             IMG_UINT32 ui32NumPhysChunks,
+             IMG_UINT32 ui32NumLogicalChunks,
+             IMG_UINT32 *pui32MappingTable,
+             PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+             PMR_FLAGS_T uiFlags,
+             const IMG_CHAR *pszAnnotation,
+             const PMR_IMPL_FUNCTAB *psFuncTab,
+             PMR_IMPL_PRIVDATA pvPrivData,
+             PMR_IMPL_TYPE eType,
+             PMR **ppsPMRPtr,
+             IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * PMRLockSysPhysAddresses()
+ *
+ * Calls the relevant callback to lock down the system physical addresses of
+ * the memory that makes up the whole PMR.
+ *
+ * Before this call, it is not valid to use any of the information
+ * getting APIs: PMR_Flags(), PMR_SysPhysAddr(),
+ * [ see note below about lock/unlock semantics ]
+ *
+ * The caller of this function does not have to care about how the PMR
+ * is implemented.  He only has to know that he is allowed access to
+ * the physical addresses _after_ calling this function and _until_
+ * calling PMRUnlockSysPhysAddresses().
+ *
+ *
+ * Notes to callback implementers (authors of PMR Factories):
+ *
+ * Some PMR implementations will be such that the physical memory exists for
+ * the lifetime of the PMR, with a static address, (and normally flags and
+ * symbolic address are static too) and so it is legal for a PMR
+ * implementation to not provide an implementation for the lock callback.
+ *
+ * Some PMR implementation may wish to page memory in from secondary storage
+ * on demand. The lock/unlock callbacks _may_ be the place to do this.
+ * (More likely, there would be a separate API for doing this, but this API
+ * provides a useful place to assert that it has been done)
+ */
+
+PVRSRV_ERROR
+PMRLockSysPhysAddresses(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount);
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesNested(PMR *psPMR,
+                           IMG_UINT32 uiLockCount,
+                           IMG_UINT32 ui32NestingLevel);
+
+/*
+ * PMRUnlockSysPhysAddresses()
+ *
+ * the reverse of PMRLockSysPhysAddresses()
+ */
+PVRSRV_ERROR
+PMRUnlockSysPhysAddresses(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount);
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesNested(PMR *psPMR,
+                             IMG_UINT32 uiLockCount,
+                             IMG_UINT32 ui32NestingLevel);
+
+/*
+ * PhysmemPMRExport()
+ *
+ * Given a PMR, creates a PMR "Export", which is a handle that
+ * provides sufficient data to be able to "import" this PMR elsewhere.
+ * The PMR Export is an object in its own right, whose existence
+ * implies a reference on the PMR, thus the PMR cannot be destroyed
+ * while the PMR Export exists.  The intention is that the PMR Export
+ * will be wrapped in the devicemem layer by a cross process handle,
+ * and some IPC by which to communicate the handle value and password
+ * to other processes.  The receiving process is able to unwrap this
+ * to gain access to the same PMR Export in this layer, and, via
+ * PhysmemPMRImport(), obtain a reference to the original PMR.
+ *
+ * The caller receives, along with the PMR Export object, information
+ * about the size and contiguity guarantee for the PMR, and also the
+ * PMRs secret password, in order to authenticate the subsequent
+ * import.
+ *
+ * N.B.  If you call PMRExportPMR() (and it succeeds), you are
+ * promising to later call PMRUnexportPMR()
+ */
+PVRSRV_ERROR
+PMRExportPMR(PMR *psPMR,
+             PMR_EXPORT **ppsPMRExport,
+             PMR_SIZE_T *puiSize,
+             PMR_LOG2ALIGN_T *puiLog2Contig,
+             PMR_PASSWORD_T *puiPassword);
+
+/*!
+*******************************************************************************
+
+ @Function      PMRMakeLocalImportHandle
+
+ @Description
+
+ Transform a general handle type into one that we are able to import.
+ Takes a PMR reference.
+
+ @Input   psPMR     The input PMR.
+ @Output  ppsPMR    The output PMR that is going to be transformed to the
+                    correct handle type.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PMRMakeLocalImportHandle(PMR *psPMR,
+                         PMR **ppsPMR);
+
+/*!
+*******************************************************************************
+
+ @Function      PMRUnmakeLocalImportHandle
+
+ @Description
+
+ Take a PMR, destroy the handle and release a reference.
+ Counterpart to PMRMakeServerExportClientExport().
+
+ @Input   psPMR       PMR to destroy.
+                      Created by PMRMakeLocalImportHandle().
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PMRUnmakeLocalImportHandle(PMR *psPMR);
+
+/*
+ * PMRUnexportPMR()
+ *
+ * The reverse of PMRExportPMR().  This causes the PMR to no longer be
+ * exported.  If the PMR has already been imported, the imported PMR
+ * reference will still be valid, but no further imports will be possible.
+ */
+PVRSRV_ERROR
+PMRUnexportPMR(PMR_EXPORT *psPMRExport);
+
+/*
+ * PMRImportPMR()
+ *
+ * Takes a PMR Export object, as obtained by PMRExportPMR(), and
+ * obtains a reference to the original PMR.
+ *
+ * The password must match, and is assumed to have been (by whatever
+ * means, IPC etc.) preserved intact from the former call to
+ * PMRExportPMR()
+ *
+ * The size and contiguity arguments are entirely irrelevant for the
+ * import, however they are verified in order to trap bugs.
+ *
+ * N.B.  If you call PhysmemPMRImport() (and it succeeds), you are
+ * promising to later call PhysmemPMRUnimport()
+ */
+PVRSRV_ERROR
+PMRImportPMR(PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR);
+
+/* Function that alters the mutability property
+ * of the PMR
+ * Setting it to TRUE makes sure the PMR memory layout
+ * can't be changed through future calls */
+void
+PMR_SetLayoutFixed(PMR *psPMR, IMG_BOOL bFlag);
+
+IMG_BOOL PMR_IsMemLayoutFixed(PMR *psPMR);
+
+/*
+ * PMRUnimportPMR()
+ *
+ * releases the reference on the PMR as obtained by PMRImportPMR()
+ */
+PVRSRV_ERROR
+PMRUnimportPMR(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRLocalImportPMR(PMR *psPMR,
+                  PMR **ppsPMR,
+                  IMG_DEVMEM_SIZE_T *puiSize,
+                  IMG_DEVMEM_ALIGN_T *puiAlign);
+
+/*
+ * Equivalent mapping functions when in kernel mode.
+ */
+PVRSRV_ERROR
+PMRAcquireKernelMappingData(PMR *psPMR,
+                            size_t uiLogicalOffset,
+                            size_t uiSize,
+                            void **ppvKernelAddressOut,
+                            size_t *puiLengthOut,
+                            IMG_HANDLE *phPrivOut);
+
+PVRSRV_ERROR
+PMRAcquireSparseKernelMappingData(PMR *psPMR,
+                                  size_t uiLogicalOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  size_t *puiLengthOut,
+                                  IMG_HANDLE *phPrivOut);
+
+PVRSRV_ERROR
+PMRReleaseKernelMappingData(PMR *psPMR,
+                            IMG_HANDLE hPriv);
+
+/*
+ * PMR_ReadBytes()
+ *
+ * calls into the PMR implementation to read up to uiBufSz bytes,
+ * returning the actual number read in *puiNumBytes
+ *
+ * this will read up to the end of the PMR, or the next symbolic name
+ * boundary, or until the requested number of bytes is read, whichever
+ * comes first
+ *
+ * In the case of sparse PMR's the caller doesn't know what offsets are
+ * valid and which ones aren't so we will just write 0 to invalid offsets
+ */
+PVRSRV_ERROR
+PMR_ReadBytes(PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT8 *pcBuffer,
+              size_t uiBufSz,
+              size_t *puiNumBytes);
+
+/*
+ * PMR_WriteBytes()
+ *
+ * calls into the PMR implementation to write up to uiBufSz bytes,
+ * returning the actual number read in *puiNumBytes
+ *
+ * this will write up to the end of the PMR, or the next symbolic name
+ * boundary, or until the requested number of bytes is written, whichever
+ * comes first
+ *
+ * In the case of sparse PMR's the caller doesn't know what offsets are
+ * valid and which ones aren't so we will just ignore data at invalid offsets
+ */
+PVRSRV_ERROR
+PMR_WriteBytes(PMR *psPMR,
+               IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+               IMG_UINT8 *pcBuffer,
+               size_t uiBufSz,
+               size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Function       PMRMMapPMR
+@Description    Performs the necessary steps to map the PMR into a user process
+                address space. The caller does not need to call
+                PMRLockSysPhysAddresses before calling this function.
+
+@Input          psPMR            PMR to map.
+
+@Input          pOSMMapData      OS specific data needed to create a mapping.
+
+@Input          uiCpuAccessFlags Flags to indicate if the mapping request
+                                 requires read, write or both access.
+
+@Return         PVRSRV_ERROR:   PVRSRV_OK on success or an error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PMRMMapPMR(PMR *psPMR,
+           PMR_MMAP_DATA pOSMMapData,
+           PVRSRV_MEMALLOCFLAGS_T uiCpuAccessFlags);
+
+/*
+ * PMRRefPMR()
+ *
+ * Take a reference on the passed in PMR
+ */
+PVRSRV_ERROR
+PMRRefPMR(PMR *psPMR);
+
+/*
+ * PMRUnrefPMR()
+ *
+ * This undoes a call to any of the PhysmemNew* family of APIs
+ * (i.e. any PMR factory "constructor")
+ *
+ * This relinquishes a reference to the PMR, and, where the refcount
+ * reaches 0, causes the PMR to be destroyed (calling the finalizer
+ * callback on the PMR, if there is one)
+ */
+PVRSRV_ERROR
+PMRUnrefPMR(PMR *psPMR);
+
+/*
+ * PMRRefPMRN()
+ *
+ * Take a reference N-times on the passed in PMR.
+ *
+ * This function does not perform address locking as opposed to PMRRefPMR().
+ */
+PVRSRV_ERROR
+PMRRefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount);
+
+/*
+ * PMRUnrefPMRN()
+ *
+ * This undoes a call to any of the PhysmemNew* family of APIs
+ * (i.e. any PMR factory "constructor").
+ *
+ * This relinquishes N references to the PMR, and, where the refcount
+ * reaches 0, causes the PMR to be destroyed (calling the finalizer
+ * callback on the PMR, if there is one)
+ */
+PVRSRV_ERROR
+PMRUnrefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*
+ * PMRTryRefPMR()
+ *
+ * This attempts to take a reference on the PMR but only succeeds if
+ * the PMR is not at refcount 0. Other Ref functions would class this
+ * attempt as a logical error. This function is free to attempt and return
+ * an error if PMR is in a free in progress state.
+ */
+PVRSRV_ERROR
+PMRTryRefPMR(PMR *psPMR);
+
+/*
+ * PMRKernelCpuMapCountIncr()
+ *
+ * Increment count of the number of current kernel CPU mappings of the PMR.
+ */
+void
+PMRKernelCpuMapCountIncr(PMR *psPMR);
+
+/*
+ * PMRKernelCpuMapCountDecr()
+ *
+ * Decrement count of the number of current kernel CPU mappings of the PMR.
+ */
+void
+PMRKernelCpuMapCountDecr(PMR *psPMR);
+
+IMG_BOOL
+PMR_IsKernelCpuMapped(PMR *psPMR);
+#endif /* #if defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+/*
+ * PMRClientCpuMapCountIncr()
+ *
+ * Increment count of the number of current client CPU mappings of the PMR.
+ */
+void
+PMRClientCpuMapCountIncr(PMR *psPMR);
+
+/*
+ * PMRClientCpuMapCountDecr()
+ *
+ * Decrement count of the number of current client CPU mappings of the PMR.
+ */
+void
+PMRClientCpuMapCountDecr(PMR *psPMR);
+
+IMG_BOOL
+PMR_IsClientCpuMapped(PMR *psPMR);
+
+
+/*
+ * PMRLinkGPUMapping()
+ *
+ * Link a GPU mapping with a PMR creating an association.
+ * Must be protected by PMR lock.
+ */
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRLinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode);
+#else
+void
+PMRLinkGPUMapping(PMR *psPMR);
+#endif
+
+/*
+ * PMRUnlinkGPUMapping()
+ *
+ * Unlink a GPU mapping with a PMR destroying an association.
+ * Must be protected by PMR lock.
+ */
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRUnlinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode);
+#else
+void
+PMRUnlinkGPUMapping(PMR *psPMR);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*
+ * PMRNotifyMigrateInProgress()
+ *
+ * Used to notify the PMR that the pages backing the PMR are
+ * in the process of migration.
+ */
+void
+PMRNotifyMigrateInProgress(PMR *psPMR);
+
+/*
+ * PMRNotifyMigrateComplete()
+ *
+ * Used to notify the PMR that migration of backing
+ * pages has completed.
+ */
+void
+PMRNotifyMigrateComplete(PMR *psPMR);
+
+/*
+ * PMRRemapGPUPMR()
+ *
+ * Trigger Remap of a PMR page with each associated mapping.
+ * This is called for UMA page migration and will overwrite
+ * an existing page mapping with the new one at logical
+ * pg offset in the PMR.
+ * PVRSRV_ERROR_DEVICEMEM_REJECT_REMAP_REQUEST can be returned
+ * if remap is not possible for the given page offset.
+ */
+PVRSRV_ERROR
+PMRRemapGPUPMR(PMR *psPMR, IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+/*
+ * PMR_IsGpuMultiMapped()
+ *
+ * Must be protected by PMR lock.
+ *
+ */
+IMG_BOOL
+PMR_IsGpuMultiMapped(PMR *psPMR);
+
+PPVRSRV_DEVICE_NODE
+PMR_DeviceNode(const PMR *psPMR);
+
+/*
+ * PMR_Flags()
+ *
+ * Flags are static and guaranteed for the life of the PMR.  Thus this
+ * function is idempotent and acquire/release semantics is not required.
+ *
+ * Returns the flags as specified on the PMR.  The flags are to be
+ * interpreted as mapping permissions
+ */
+PMR_FLAGS_T
+PMR_Flags(const PMR *psPMR);
+
+IMG_BOOL
+PMR_IsSparse(const PMR *psPMR);
+
+IMG_DEVMEM_SIZE_T
+PMR_PhysicalSize(const PMR *psPMR);
+
+/*
+ * PMR_IsOffsetValid()
+ *
+ * Returns if an address offset inside a PMR has a valid
+ * physical backing.
+ */
+PVRSRV_ERROR
+PMR_IsOffsetValid(const PMR *psPMR,
+				IMG_UINT32 ui32Log2PageSize,
+				IMG_UINT32 ui32NumOfPages,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_BOOL *pbValid);
+
+PHYS_HEAP *
+PMR_PhysHeap(const PMR *psPMR);
+
+PMR_MAPPING_TABLE *
+PMR_GetMappingTable(const PMR *psPMR);
+
+IMG_UINT32
+PMR_GetLog2Contiguity(const PMR *psPMR);
+
+IMG_DEVMEM_SIZE_T
+PMR_LogicalSize(const PMR *psPMR);
+
+/*
+ * PMR_GetLogicalChunkCount
+ *
+ * Retrieve the maximum number of chunks supported by the PMR.
+ * This property is fixed at creation time.
+ */
+IMG_UINT32 PMR_GetLogicalChunkCount(const PMR *psPMR);
+
+const IMG_CHAR *
+PMR_GetAnnotation(const PMR *psPMR);
+
+PMR_IMPL_TYPE
+PMR_GetType(const PMR *psPMR);
+
+IMG_CHAR *
+PMR_GetTypeStr(const PMR *psPMR);
+
+IMG_INT32
+PMR_GetRefCount(const PMR *psPMR);
+
+/* PMR usage type for callers of PMR_DevPhysAddr() */
+typedef IMG_UINT32 PMR_PHYSADDRMODE_TYPE;
+
+#define CPU_USE BIT(0) /* CPU use, disable IPA policy */
+#define DEVICE_USE BIT(1) /* Device use, enable IPA policy */
+#define MAPPING_USE BIT(2) /* Mapping use, dev phys addrs obtained in a mapping path */
+
+/*
+ * PMR_DevPhysAddr()
+ *
+ * A note regarding Lock/Unlock semantics
+ * ======================================
+ *
+ * PMR_DevPhysAddr may only be called after PMRLockSysPhysAddresses()
+ * has been called.  The data returned may be used only until
+ * PMRUnlockSysPhysAddresses() is called after which time the licence
+ * to use the data is revoked and the information may be invalid.
+ *
+ * Given an offset, this function returns the device physical address of the
+ * corresponding page in the PMR.  It may be called multiple times
+ * until the address of all relevant pages has been determined.
+ *
+ * If caller only wants one physical address it is sufficient to pass in:
+ * ui32Log2PageSize==0 and ui32NumOfPages==1
+ *
+ * Returns PVRSRV_OK if successful.
+ * PVRSRV_ERROR_RETRY if SUPPORT_LINUX_OSPAGE_MIGRATION is
+ * enabled and migrate is in progress. If we support page migration this
+ * call may return PVRSRV_ERROR_RETRY back to the requester. This is
+ * because the function is used to create device mappings and these mappings
+ * need to be synchronised across both reservations and PMRs. To do this we
+ * must take both locks in any mapping path. In order for migrate to complete,
+ * a UM requested Device mapping must temporarily back off the locks.
+ * Retry signifies this should happen.
+ */
+PVRSRV_ERROR
+PMR_DevPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEV_PHYADDR *psDevAddr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage);
+
+/*
+ * PMR_CpuPhysAddr()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * Given an offset, this function returns the CPU physical address of the
+ * corresponding page in the PMR.  It may be called multiple times
+ * until the address of all relevant pages has been determined.
+ *
+ */
+PVRSRV_ERROR
+PMR_CpuPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_CPU_PHYADDR *psCpuAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage);
+
+/* PMRGetUID()
+ *
+ * Used for bridge calls that expect a PVRSRV_ERROR returned
+ * */
+PVRSRV_ERROR
+PMRGetUID(PMR *psPMR,
+          IMG_UINT64 *pui64UID);
+
+IMG_UINT64
+PMRInternalGetUID(PMR *psPMR);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*
+ * PMR_IsZombie()
+ *
+ * Indicates if a PMR is a "zombie" PMR. This function **must** be called
+ * inside a PMR factory lock.
+ */
+IMG_BOOL
+PMR_IsZombie(const PMR *psPMR);
+
+/*
+ * PMRMarkForDeferFree
+ *
+ * Sets sync value required for this PMR to be freed.
+ */
+void
+PMRMarkForDeferFree(PMR *psPMR);
+
+/*
+ * PMRQueueZombiesForCleanup
+ *
+ * Defers cleanup of all zombie PMRs to the CleanupThread.
+ *
+ * Returns IMG_TRUE if any PMRs were queued for free and IMG_FALSE if no PMRs
+ * were queued.
+ */
+IMG_BOOL
+PMRQueueZombiesForCleanup(PPVRSRV_DEVICE_NODE psDevNode);
+
+/*
+ * PMRReviveZombieAndRef
+ *
+ * Removed the PMR either form zombie list or cleanup item's list
+ * and references it.
+ */
+void
+PMRReviveZombieAndRef(PMR *psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/*
+ * PMR_ChangeSparseMemUnlocked()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * This function alters the memory map of the given PMR in device space by
+ * adding/deleting the pages as requested. PMR lock must be taken
+ * before calling this function.
+ *
+ */
+PVRSRV_ERROR PMR_ChangeSparseMemUnlocked(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags);
+/*
+ * PMR_ChangeSparseMem()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * This function alters the memory map of the given PMR in device space by
+ *  adding/deleting the pages as requested.
+ *
+ */
+PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32	uiSparseFlags);
+
+#if defined(PDUMP)
+
+/*
+ * PMR_PDumpSymbolicAddr()
+ *
+ * Given an offset, returns the pdump memspace name and symbolic
+ * address of the corresponding page in the PMR.
+ *
+ * Note that PDump memspace names and symbolic addresses are static
+ * and valid for the lifetime of the PMR, therefore we don't require
+ * acquire/release semantics here.
+ *
+ * Note that it is expected that the pdump "mapping" code will call
+ * this function multiple times as each page is mapped in turn
+ *
+ * Note that NextSymName is the offset from the base of the PMR to the
+ * next pdump symbolic address (or the end of the PMR if the PMR only
+ * had one PDUMPMALLOC
+ */
+PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32NamespaceNameLen,
+                      IMG_CHAR *pszNamespaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+		      IMG_DEVMEM_OFFSET_T *puiNextSymName
+                      );
+
+/*
+ * PMRPDumpLoadMemValue32()
+ *
+ * writes the current contents of a dword in PMR memory to the pdump
+ * script stream. Useful for patching a buffer by simply editing the
+ * script output file in ASCII plain text.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT32 ui32Value,
+                     PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpCopyMem32
+ *
+ * Adds in the pdump script stream a copy of a dword in one PMR memory
+ * location to another PMR memory location.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem32(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpLoadMemValue64()
+ *
+ * writes the current contents of a dword in PMR memory to the pdump
+ * script stream. Useful for patching a buffer by simply editing the
+ * script output file in ASCII plain text.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT64 ui64Value,
+                     PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpCopyMem64
+ *
+ * Adds in the pdump script stream a copy of a quadword in one PMR memory
+ * location to another PMR memory location.
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem64(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpLoadMem()
+ *
+ * Writes the current contents of the PMR memory to the pdump PRM stream,
+ * and emits some PDump code to the script stream to LDB said bytes from
+ * said file. If bZero is IMG_TRUE then the PDump zero page is used as the
+ * source for the LDB.
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero);
+
+/*
+ * PMRPDumpSaveToFile()
+ *
+ * Emits some PDump that does an SAB (save bytes) using the PDump symbolic
+ * address of the PMR. Note that this is generally not the preferred way to
+ * dump the buffer contents. There is an equivalent function in
+ * devicemem_server.h which also emits SAB but using the virtual address,
+ * which is the "right" way to dump the buffer contents to a file.
+ * This function exists just to aid testing by providing a means to dump
+ * the PMR directly by symbolic address also.
+ */
+PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMR_PDumpSymbolicAddr)
+#endif
+static INLINE PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32NamespaceNameLen,
+                      IMG_CHAR *pszNamespaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                      IMG_DEVMEM_OFFSET_T *puiNextSymName)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32NamespaceNameLen);
+	PVR_UNREFERENCED_PARAMETER(pszNamespaceName);
+	PVR_UNREFERENCED_PARAMETER(ui32SymbolicAddrLen);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddr);
+	PVR_UNREFERENCED_PARAMETER(puiNewOffset);
+	PVR_UNREFERENCED_PARAMETER(puiNextSymName);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMemValue32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT32 ui32Value,
+                     PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMemValue64)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT64 ui64Value,
+                     PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui64Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMem)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(bZero);
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpSaveToFile)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(uiFileOffset);
+	return PVRSRV_OK;
+}
+
+#endif	/* PDUMP */
+
+/* This function returns the private data that a pmr subtype embedded in
+ * here. We use the function table pointer as "authorisation" that this
+ * function is being called by the pmr subtype implementation. We can
+ * assume (assert) that. It would be a bug in the implementation of the
+ * pmr subtype if this assertion ever fails.
+ */
+void *
+PMRGetPrivateData(const PMR *psPMR,
+                  const PMR_IMPL_FUNCTAB *psFuncTab);
+
+PVRSRV_ERROR
+PMRZeroingPMR(PMR *psPMR,
+				IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize);
+
+PVRSRV_ERROR
+PMRDumpPageList(PMR *psReferencePMR,
+					IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize);
+
+PVRSRV_ERROR
+PMRWritePMPageList(/* Target PMR, offset, and length */
+                   PMR *psPageListPMR,
+                   IMG_DEVMEM_OFFSET_T uiTableOffset,
+                   IMG_DEVMEM_SIZE_T  uiTableLength,
+                   /* Referenced PMR, and "page" granularity */
+                   PMR *psReferencePMR,
+                   IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+                   PMR_PAGELIST **ppsPageList);
+
+/* Doesn't actually erase the page list - just releases
+ * the appropriate refcounts
+ */
+PVRSRV_ERROR // should be void, surely
+PMRUnwritePMPageList(PMR_PAGELIST *psPageList);
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiFlags);
+
+PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize);
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpPol32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpCheck32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpCBP)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif
+
+PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR);
+
+/*
+ * PMRInit()
+ *
+ * To be called once and only once to initialise the internal data in
+ * the PMR module (mutexes and such)
+ *
+ * Not for general use.  Only PVRSRVInit(); should be calling this.
+ */
+PVRSRV_ERROR
+PMRInit(void);
+
+/*
+ * PMRDeInit()
+ *
+ * To be called once and only once to deinitialise the internal data in
+ * the PMR module (mutexes and such) and for debug checks
+ *
+ * Not for general use.  Only PVRSRVDeInit(); should be calling this.
+ */
+PVRSRV_ERROR
+PMRDeInit(void);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*
+ * PMRInitDevice()
+ *
+ * Initialised device specific PMR data.
+ */
+PVRSRV_ERROR
+PMRInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRFreeZombies()
+ *
+ * Free deferred PMRs.
+ */
+void
+PMRFreeZombies(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRFreeZombies()
+ *
+ * Print all zombies to the log.
+ */
+void
+PMRDumpZombies(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRDeInitDevice()
+ *
+ * Cleans up device specific PMR data.
+ */
+void
+PMRDeInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR
+PMRStoreRIHandle(PMR *psPMR, void *hRIHandle);
+#endif
+
+#if defined(DEBUG)
+void PMRLockHeldAssert(const PMR *psPMR);
+#else
+#define PMRLockHeldAssert(x)
+#endif
+
+/*
+ * PMRLockPMR()
+ *
+ * To be called when the PMR must not be modified by any other call-stack.
+ * Acquires the mutex on the passed in PMR.
+ */
+void
+PMRLockPMR(const PMR *psPMR);
+
+/*
+ * PMRUnlockPMR()
+ *
+ * To be called when the PMR is no longer being modified.
+ * Releases the per-PMR mutex.
+ */
+void
+PMRUnlockPMR(const PMR *psPMR);
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRModifyIPAPolicy(PMR *psPMR, IMG_UINT8 ui8NewIPAPolicy);
+
+PVRSRV_ERROR
+PMRGetIPAPolicy(PMR *psPMR, IMG_UINT8 *pui8IPAPolicy);
+
+PVRSRV_ERROR
+PMRGetIPAInfo(PMR *psPMR, IMG_UINT32 *pui32IPAPolicy, IMG_UINT32 *pui32IPAShift,
+              IMG_UINT32 *pui32IPAMask, IMG_UINT32 *pui32IPAFlagsValue);
+#endif
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+/*
+ * PMR_ImportedDevicesMask()
+ *
+ * Returns a bitmask of devices this PMR is imported to.
+ * Explicitly does NOT include the PMR_DeviceNode's device.
+ * For each device registered with the PMR: 1 << psDevNode->sDevId.ui32InternalID
+ */
+IMG_UINT64
+PMR_ImportedDevicesMask(const PMR* psPMR);
+static_assert((sizeof(IMG_UINT64) * 8) >= PVRSRV_MAX_DEVICES, "PMR_ImportedDevicesMask() needs to be updated");
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+/*
+ *
+ * PMR_RegisterDeviceImport()
+ *
+ * Register the PMR with the device node.
+ * This is required for the PMR to be marked as a XD PMR.
+ * Silently handles if the PMR is already registered with the device.
+ */
+PVRSRV_ERROR
+PMR_RegisterDeviceImport(PMR* psPMR, PPVRSRV_DEVICE_NODE psDevNode);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+#endif /* #ifdef SRVSRV_PMR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_impl.h
new file mode 100644
index 000000000000..3c290ca3a6e7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_impl.h
@@ -0,0 +1,605 @@
+/**************************************************************************/ /*!
+@File
+@Title          Implementation Callbacks for Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This file is for definitions
+                that are private to the world of PMRs, but that need to be
+                shared between pmr.c itself and the modules that implement the
+                callbacks for the PMR.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PMR_IMPL_H
+#define SRVSRV_PMR_IMPL_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+/*! Physical Memory Resource type.
+ */
+typedef struct _PMR_ PMR;
+
+/*! Per-flavour callbacks need to be shared with generic implementation
+ * (pmr.c).
+ */
+typedef void *PMR_IMPL_PRIVDATA;
+
+/*! Type for holding flags passed to the PMR factory.
+ */
+typedef PVRSRV_MEMALLOCFLAGS_T PMR_FLAGS_T;
+
+/*! Mapping table for the allocation.
+ *
+ * PMR's can be sparse in which case not all the logical addresses in it are
+ * valid. The mapping table translates logical offsets into physical offsets.
+ *
+ * This table is always passed to the PMR factory regardless if the memory is
+ * sparse or not. In case of non-sparse memory all virtual offsets are mapped
+ * to physical offsets.
+ */
+typedef struct _PMR_MAPPING_TABLE_ PMR_MAPPING_TABLE;
+
+/*! Private data passed to the ::PFN_MMAP_FN function.
+ */
+typedef void *PMR_MMAP_DATA;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+typedef void *PMR_IMPL_ZOMBIEPAGES;
+#endif
+
+#define PMR_IMPL_TYPES \
+	X(NONE), \
+	X(OSMEM), \
+	X(LMA), \
+	X(DMABUF), \
+	X(EXTMEM), \
+	X(DC), \
+	X(TDFWMEM), \
+	X(TDSECBUF), \
+	X(LAST)
+
+/*! PMR factory type.
+ */
+typedef enum _PMR_IMPL_TYPE_
+{
+#define X(type) PMR_TYPE_##type
+	PMR_IMPL_TYPES
+#undef X
+} PMR_IMPL_TYPE;
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_LOCK_PHYS_ADDRESSES_FN
+
+@Description    Called to lock down the physical addresses for all pages
+                allocated for a PMR.
+                The default implementation is to simply increment a
+                lock-count for debugging purposes.
+                If overridden, the PFN_LOCK_PHYS_ADDRESSES_FN function will
+                be called when someone first requires a physical address,
+                and the PFN_UNLOCK_PHYS_ADDRESSES_FN counterpart will be
+                called when the last such reference is released.
+                The PMR implementation may assume that physical addresses
+                will have been "locked" in this manner before any call is
+                made to the pfnDevPhysAddr() callback
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_LOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_UNLOCK_PHYS_ADDRESSES_FN
+
+@Description    Called to release the lock taken on the physical addresses
+                for all pages allocated for a PMR.
+                The default implementation is to simply decrement a
+                lock-count for debugging purposes.
+                If overridden, the PFN_UNLOCK_PHYS_ADDRESSES_FN will be
+                called when the last reference taken on the PMR is
+                released.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Output         ppvZombiePages        Zombie pages object. If non-null is returned
+                                      caller is obligated to call pfnFreeZombiePages
+                                      at an appropriate time to prevent memory leaks.
+                                      If support for deferred freeing of pages is not
+                                      provided, the implementation must set
+                                      *ppvZombiePages to NULL.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+typedef PVRSRV_ERROR (*PFN_UNLOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                                                     PMR_IMPL_ZOMBIEPAGES *ppvZombiePages);
+#else
+typedef PVRSRV_ERROR (*PFN_UNLOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv);
+#endif
+
+#if defined(SUPPORT_STATIC_IPA)
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_DEV_PHYS_ADDR_FN
+
+@Description    Called to obtain one or more physical addresses for given
+                offsets within a PMR.
+
+                The PFN_LOCK_PHYS_ADDRESSES_FN callback (if overridden) is
+                guaranteed to have been called prior to calling the
+                PFN_DEV_PHYS_ADDR_FN callback and the caller promises not to
+                rely on the physical address thus obtained after the
+                PFN_UNLOCK_PHYS_ADDRESSES_FN callback is called.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          ui32Log2PageSize      The log2 page size.
+@Input          ui32NumOfAddr         The number of addresses to be returned
+@Input          puiOffset             The offset from the start of the PMR
+                                      (in bytes) for which the physical
+                                      address is required. Where multiple
+                                      addresses are requested, this will
+                                      contain a list of offsets.
+@Input          ui64IPAPolicyValue    The Intermediate Physical Address (IPA)
+                                      Policy value to be applied to
+                                      the physical address
+@Input          ui64IPAClearMask      The IPA Clear mask to be applied to
+                                      the physical address when setting policy.
+@Output         pbValid               List of boolean flags indicating which
+                                      addresses in the returned list
+                                      (psDevAddrPtr) are valid (for sparse
+                                      allocations, not all pages may have a
+                                      physical backing)
+@Output         psDevAddrPtr          Returned list of physical addresses
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#else
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_DEV_PHYS_ADDR_FN
+
+@Description    Called to obtain one or more physical addresses for given
+                offsets within a PMR.
+
+                The PFN_LOCK_PHYS_ADDRESSES_FN callback (if overridden) is
+                guaranteed to have been called prior to calling the
+                PFN_DEV_PHYS_ADDR_FN callback and the caller promises not to
+                rely on the physical address thus obtained after the
+                PFN_UNLOCK_PHYS_ADDRESSES_FN callback is called.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          ui32Log2PageSize      The log2 page size.
+@Input          ui32NumOfAddr         The number of addresses to be returned
+@Input          puiOffset             The offset from the start of the PMR
+                                      (in bytes) for which the physical
+                                      address is required. Where multiple
+                                      addresses are requested, this will
+                                      contain a list of offsets.
+@Output         pbValid               List of boolean flags indicating which
+                                      addresses in the returned list
+                                      (psDevAddrPtr) are valid (for sparse
+                                      allocations, not all pages may have a
+                                      physical backing)
+@Output         psDevAddrPtr          Returned list of physical addresses
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#endif
+typedef PVRSRV_ERROR (*PFN_DEV_PHYS_ADDR_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_UINT32 ui32Log2PageSize,
+                      IMG_UINT32 ui32NumOfAddr,
+                      IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+                      IMG_UINT64 ui64IPAPolicyValue,
+                      IMG_UINT64 ui64IPAClearMask,
+#endif
+                      IMG_BOOL *pbValid,
+                      IMG_DEV_PHYADDR *psDevAddrPtr);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN
+
+@Description    Called to obtain a kernel-accessible address (mapped to a
+                virtual address if required) for the PMR for use internally
+                in Services.
+
+    Implementation of this function for the (default) PMR factory providing
+    OS-allocations is mandatory (the driver will expect to be able to call
+    this function for OS-provided allocations).
+    For other PMR factories, implementation of this function is only necessary
+    where an MMU mapping is required for the Kernel to be able to access the
+    allocated memory.
+    If no mapping is needed, this function can remain unimplemented and the
+    pfn may be set to NULL.
+@Input          pvPriv                Private data (which was generated by
+                                      the PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR
+                                      at which mapping is to start
+@Input          uiSize                Size of mapping (in bytes)
+@Output         ppvKernelAddressOut   Mapped kernel address
+@Output         phHandleOut           Returned handle of the new mapping
+@Input          ulFlags               Mapping flags
+
+@Return         PVRSRV_OK if the mapping was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      size_t uiOffset,
+                      size_t uiSize,
+                      void **ppvKernelAddressOut,
+                      IMG_HANDLE *phHandleOut,
+                      PMR_FLAGS_T ulFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_RELEASE_KERNEL_MAPPING_DATA_FN
+
+@Description    Called to release a mapped kernel virtual address
+
+   Implementation of this callback is mandatory if
+   PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN is provided for the PMR factory,
+   otherwise this function can remain unimplemented and the pfn may be set
+   to NULL.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          hHandle               Handle of the mapping to be released
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_RELEASE_KERNEL_MAPPING_DATA_FN)(PMR_IMPL_PRIVDATA pvPriv,
+              IMG_HANDLE hHandle);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_READ_BYTES_FN
+
+@Description    Called to read bytes from an unmapped allocation
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will use PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN to map the entire
+   PMR (if an MMU mapping is required for the Kernel to be able to access the
+   allocated memory).
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR at
+                                      which to begin reading
+@Output         pcBuffer              Buffer in which to return the read data
+@Input          uiBufSz               Number of bytes to be read
+@Output         puiNumBytes           Number of bytes actually read (may be
+                                      less than uiBufSz)
+
+@Return         PVRSRV_OK if the read was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_READ_BYTES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_UINT8 *pcBuffer,
+                      size_t uiBufSz,
+                      size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_WRITE_BYTES_FN
+
+@Description    Called to write bytes into an unmapped allocation
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will use PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN to map the entire
+   PMR (if an MMU mapping is required for the Kernel to be able to access the
+   allocated memory).
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR at
+                                      which to begin writing
+@Input          pcBuffer              Buffer containing the data to be written
+@Input          uiBufSz               Number of bytes to be written
+@Output         puiNumBytes           Number of bytes actually written (may be
+                                      less than uiBufSz)
+
+@Return         PVRSRV_OK if the write was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_WRITE_BYTES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_UINT8 *pcBuffer,
+                      size_t uiBufSz,
+                      size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_CHANGE_SPARSE_MEM_FN
+
+@Description    Called to modify the physical backing for a given sparse
+                allocation.
+                The caller provides a list of the pages within the sparse
+                allocation which should be backed with a physical allocation
+                and a list of the pages which do not require backing.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the sparse allocation to be
+                                      modified
+@Input          ui32AllocPageCount    The number of pages specified in
+                                      pai32AllocIndices
+@Input          pai32AllocIndices     The list of pages in the sparse
+                                      allocation that should be backed with a
+                                      physical allocation. Pages are
+                                      referenced by their index within the
+                                      sparse allocation (e.g. in a 10 page
+                                      allocation, pages are denoted by
+                                      indices 0 to 9)
+@Input          ui32FreePageCount     The number of pages specified in
+                                      pai32FreeIndices
+@Input          pai32FreeIndices      The list of pages in the sparse
+                                      allocation that do not require
+                                      a physical allocation.
+@Input          ui32Flags             Allocation flags
+@Output         ppvZombiePages        Zombie pages object. If non-null is returned
+                                      caller is obligated to call pfnFreeZombiePages
+                                      at an appropriate time to prevent memory leaks
+
+@Return         PVRSRV_OK if the sparse allocation physical backing was updated
+                successfully, an error code otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CHANGE_SPARSE_MEM_FN)(PMR_IMPL_PRIVDATA pPriv,
+                      const PMR *psPMR,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+                      PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+                      IMG_UINT32 uiFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_CHANGE_SPARSE_MEM_CPU_MAP_FN
+
+@Description    Called to modify which pages are mapped for a given sparse
+                allocation.
+                The caller provides a list of the pages within the sparse
+                allocation which should be given a CPU mapping and a list
+                of the pages which do not require a CPU mapping.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the sparse allocation to be
+                                      modified
+@Input          sCpuVAddrBase         The virtual base address of the sparse
+                                      allocation
+@Input          ui32AllocPageCount    The number of pages specified in
+                                      pai32AllocIndices
+@Input          pai32AllocIndices     The list of pages in the sparse
+                                      allocation that should be given a CPU
+                                      mapping. Pages are referenced by their
+                                      index within the sparse allocation (e.g.
+                                      in a 10 page allocation, pages are
+                                      denoted by indices 0 to 9)
+@Input          ui32FreePageCount     The number of pages specified in
+                                      pai32FreeIndices
+@Input          pai32FreeIndices      The list of pages in the sparse
+                                      allocation that do not require a CPU
+                                      mapping.
+
+@Return         PVRSRV_OK if the page mappings were updated successfully, an
+                error code otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CHANGE_SPARSE_MEM_CPU_MAP_FN)(PMR_IMPL_PRIVDATA pPriv,
+                      const PMR *psPMR,
+                      IMG_UINT64 sCpuVAddrBase,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_MMAP_FN
+
+@Description    Called to map pages in the specified PMR.
+
+   Implementation of this callback is optional.
+   Where it is provided, it will be used in place of OSMMapPMRGeneric().
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the allocation to be mapped
+@Input          pMMapData             OS-specific data to describe how mapping
+                                      should be performed
+
+@Return         PVRSRV_OK if the mapping was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_MMAP_FN)(PMR_IMPL_PRIVDATA pPriv,
+                                    PMR *psPMR,
+                                    PMR_MMAP_DATA pMMapData);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_FINALIZE_FN
+
+@Description    Called to destroy the PMR.
+                This callback will be called only when all references to
+                the PMR have been dropped.
+                The PMR was created via a call to PhysmemNewRamBackedPMR()
+                and is destroyed via this callback.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_FINALIZE_FN)(PMR_IMPL_PRIVDATA pvPriv);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ACQUIRE_PMR_FACTORY_LOCK_FN
+
+@Description    Called to acquire the PMR factory's global lock, if it has one,
+                hence callback optional. Factories which support entry points
+                in addition to the normal bridge calls, for example, from the
+                native OS that manipulate the PMR reference count should
+                create a factory lock and implementations for these call backs.
+
+   Implementation of this callback is optional.
+
+@Return         None
+*/
+/*****************************************************************************/
+typedef void (*PFN_ACQUIRE_PMR_FACTORY_LOCK_FN)(void);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_RELEASE_PMR_FACTORY_LOCK_FN
+
+@Description    Called to release the PMR factory's global lock acquired by calling
+                pfn_acquire_pmr_factory_lock callback.
+
+   Implementation of this callback is optional.
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_RELEASE_PMR_FACTORY_LOCK_FN)(void);
+
+#ifdef SUPPORT_PMR_DEFERRED_FREE
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ZOMBIFY_FN
+
+@Description    Called to perform factory actions necessary when PMR becomes
+                a zombie PMR.
+
+                This function should at least adjust the driver/process memory
+                stats to reflect the amount of memory is occupied by the zombie
+                PMRs and at the same time subtract the memory from the main
+                memory stat the pages are accounted under.
+
+   Implementation of this callback is required when SUPPORT_PMR_DEFERRED_FREE=1.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_ZOMBIFY_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                                       PMR *psPMR);
+#endif
+
+#ifdef SUPPORT_PMR_PAGES_DEFERRED_FREE
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_FREE_ZOMBIE_PAGES_FN
+
+@Description    Called to perform factory actions to free zombie pages object
+                previously returned by PFN_CHANGE_SPARSE_MEM_FN.
+
+                This function should free the pages described in the
+                pvZombiePages parameter and do any associated actions related
+                to freeing such as poisoning or returning to the page pool.
+
+                Implementation of this callback is required when
+                SUPPORT_PMR_PAGES_DEFERRED_FREE=1.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise. If error is returned, the PMR layer might retry.
+                On error, factory implementations should modify the contents
+                of the PMR_IMPL_ZOMBIEPAGES object reflecting any changes in
+                underlying memory as a result of the initial (failed) call.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_FREE_ZOMBIE_PAGES_FN)(PMR_IMPL_ZOMBIEPAGES pvZombiePages);
+#endif
+
+/*! PMR factory callback table.
+ */
+struct _PMR_IMPL_FUNCTAB_ {
+    /*! Callback function pointer, see ::PFN_LOCK_PHYS_ADDRESSES_FN */
+    PFN_LOCK_PHYS_ADDRESSES_FN pfnLockPhysAddresses;
+    /*! Callback function pointer, see ::PFN_UNLOCK_PHYS_ADDRESSES_FN */
+    PFN_UNLOCK_PHYS_ADDRESSES_FN pfnUnlockPhysAddresses;
+
+    /*! Callback function pointer, see ::PFN_DEV_PHYS_ADDR_FN */
+    PFN_DEV_PHYS_ADDR_FN pfnDevPhysAddr;
+
+    /*! Callback function pointer, see ::PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN */
+    PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN pfnAcquireKernelMappingData;
+    /*! Callback function pointer, see ::PFN_RELEASE_KERNEL_MAPPING_DATA_FN */
+    PFN_RELEASE_KERNEL_MAPPING_DATA_FN pfnReleaseKernelMappingData;
+
+    /*! Callback function pointer, see ::PFN_READ_BYTES_FN */
+    PFN_READ_BYTES_FN pfnReadBytes;
+    /*! Callback function pointer, see ::PFN_WRITE_BYTES_FN */
+    PFN_WRITE_BYTES_FN pfnWriteBytes;
+
+    /*! Callback function pointer, see ::PFN_CHANGE_SPARSE_MEM_FN */
+    PFN_CHANGE_SPARSE_MEM_FN pfnChangeSparseMem;
+
+#ifdef SUPPORT_PMR_PAGES_DEFERRED_FREE
+    /*! Callback function pointer, see ::PFN_FREE_ZOMBIE_PAGES_FN */
+    PFN_FREE_ZOMBIE_PAGES_FN pfnFreeZombiePages;
+#endif
+
+    /*! Callback function pointer, see ::PFN_MMAP_FN */
+    PFN_MMAP_FN pfnMMap;
+
+    /*! Callback function pointer, see ::PFN_FINALIZE_FN */
+    PFN_FINALIZE_FN pfnFinalize;
+
+    /*! Callback function pointer, see ::PFN_ACQUIRE_PMR_FACTORY_LOCK_FN */
+    PFN_ACQUIRE_PMR_FACTORY_LOCK_FN pfnGetPMRFactoryLock;
+
+    /*! Callback function pointer, see ::PFN_RELEASE_PMR_FACTORY_LOCK_FN */
+    PFN_RELEASE_PMR_FACTORY_LOCK_FN pfnReleasePMRFactoryLock;
+
+#ifdef SUPPORT_PMR_DEFERRED_FREE
+    /*! Callback function pointer, see ::PFN_ZOMBIFY_FN */
+    PFN_ZOMBIFY_FN pfnZombify;
+#endif
+};
+
+/*! PMR factory callback table.
+ */
+typedef struct _PMR_IMPL_FUNCTAB_ PMR_IMPL_FUNCTAB;
+
+#endif /* SRVSRV_PMR_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_os.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_os.h
new file mode 100644
index 000000000000..84a3a6c400ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_os.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific PMR functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PMR_OS_H)
+#define PMR_OS_H
+
+#include "pmr_impl.h"
+
+/*************************************************************************/ /*!
+@Function       OSMMapPMRGeneric
+@Description    Implements a generic PMR mapping function, which is used
+                to CPU map a PMR where the PMR does not have a mapping
+                function defined by the creating PMR factory.
+@Input          psPMR               the PMR to be mapped
+@Output         pOSMMapData         pointer to any private data
+                                    needed by the generic mapping function
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/*************************************************************************/ /*!
+ * @Function    OSLinuxPMRUnmapPageInPMR
+ *
+ * @Description Distributes calls in the CPU mmap module to unmap a given PMR
+ *              page offset into all associated mappings.
+ *
+ * @Input       psPMR                The PMR to be mapped.
+ * @Input       psMappingListHead    List of Linux mappings to adjust.
+ * @Input       ui32LogicalPgOffset  The logical page offset into the
+ *                                   PMR.
+ *
+ * @Return      None
+*/ /**************************************************************************/
+void
+OSLinuxPMRUnmapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+#endif /* !defined(PMR_OS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/power.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/power.h
new file mode 100644
index 000000000000..159fa17781a8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/power.h
@@ -0,0 +1,611 @@
+/*************************************************************************/ /*!
+@File
+@Title          Power Management Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for power management functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef POWER_H
+#define POWER_H
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_error.h"
+#include "servicesext.h"
+#include "opaque_types.h"
+#include "di_common.h"
+
+/*!
+ *****************************************************************************
+ *	Power management
+ *****************************************************************************/
+
+typedef struct _PVRSRV_POWER_DEV_TAG_ PVRSRV_POWER_DEV;
+
+typedef IMG_BOOL (*PFN_SYS_DEV_IS_DEFAULT_STATE_OFF)(PVRSRV_POWER_DEV *psPowerDevice);
+
+/* Power transition handler prototypes */
+
+/*!
+  Typedef for a pointer to a Function that will be called before a transition
+  from one power state to another. See also PFN_POST_POWER.
+ */
+typedef PVRSRV_ERROR (*PFN_PRE_POWER)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      PVRSRV_DEV_POWER_STATE eNewPowerState,
+                                      PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                                      PVRSRV_POWER_FLAGS ePwrFlags);
+/*!
+  Typedef for a pointer to a Function that will be called after a transition
+  from one power state to another. See also PFN_PRE_POWER.
+ */
+typedef PVRSRV_ERROR (*PFN_POST_POWER)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                       PVRSRV_DEV_POWER_STATE eNewPowerState,
+                                       PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                                       PVRSRV_POWER_FLAGS ePwrFlags);
+
+/* Clock speed handler prototypes */
+
+/*!
+  Typedef for a pointer to a Function that will be called before a transition
+  from one clock speed to another. See also PFN_POST_CLOCKSPEED_CHANGE.
+ */
+typedef PVRSRV_ERROR (*PFN_PRE_CLOCKSPEED_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                  PVRSRV_DEV_POWER_STATE eCurrentPowerState);
+
+/*!
+  Typedef for a pointer to a Function that will be called after a transition
+  from one clock speed to another. See also PFN_PRE_CLOCKSPEED_CHANGE.
+ */
+typedef PVRSRV_ERROR (*PFN_POST_CLOCKSPEED_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                   PVRSRV_DEV_POWER_STATE eCurrentPowerState);
+
+/*!
+  Typedef for a pointer to a function that will be called to transition the
+  device to a forced idle state. Used in unison with (forced) power requests,
+  DVFS and cluster count changes.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_REQUEST)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                IMG_BOOL bDeviceOffPermitted);
+
+/*!
+  Typedef for a pointer to a function that will be called to cancel a forced
+  idle state and return the firmware back to a state where the hardware can be
+  scheduled.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_CANCEL_REQUEST)(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+  Typedef for a pointer to a function that will be called to cancel a forced
+  idle state and return the firmware back to a state where the hardware can be
+  scheduled. This function does not wait for a response from the FW.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC)(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+
+typedef PVRSRV_ERROR (*PFN_GPU_UNITS_POWER_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                   IMG_UINT32 ui32SESPowerState);
+
+const char *PVRSRVSysPowerStateToString(PVRSRV_SYS_POWER_STATE eState);
+const char *PVRSRVDevPowerStateToString(PVRSRV_DEV_POWER_STATE eState);
+
+PVRSRV_ERROR PVRSRVPowerLockInit(PPVRSRV_DEVICE_NODE psDeviceNode);
+void PVRSRVPowerLockDeInit(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerLock
+
+ @Description	Obtain the mutex for power transitions. Only allowed when
+                system power is on.
+
+ @Return	PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or PVRSRV_OK
+
+******************************************************************************/
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                   const char *pszFile, const unsigned int ui32LineNum);
+#define PVRSRVPowerLock(DEV_NODE)	PVRSRVPowerLock_Debug(DEV_NODE, __FILE__, __LINE__)
+#else
+PVRSRV_ERROR PVRSRVPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerUnlock
+
+ @Description	Release the mutex for power transitions
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+void PVRSRVPowerUnlock(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerTryLock
+
+ @Description	Try to obtain the mutex for power transitions. Only allowed when
+		system power is on.
+
+ @Return	PVRSRV_ERROR_RETRY or PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or
+		PVRSRV_OK
+
+******************************************************************************/
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerTryLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      const char *pszFile, const unsigned int ui32LineNum);
+#define PVRSRVPowerTryLock(DEV_NODE)	PVRSRVPowerTryLock_Debug(DEV_NODE, __FILE__, __LINE__)
+#else
+PVRSRV_ERROR PVRSRVPowerTryLock(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerTryLockWaitForTimeout
+
+ @Description	Try to obtain the mutex for power transitions. Only allowed when
+		system power is on. The call blocks until either the lock is acquired,
+		or the timeout is reached.
+
+		*** Debug only. DO NOT use in core GPU functions which cannot fail. ***
+		If the power lock cannot be taken the device may be powered down at
+		any time in another worker thread.
+
+ @Return	PVRSRV_ERROR_RETRY or PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or
+		PVRSRV_OK
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPowerTryLockWaitForTimeout(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function     PVRSRVPwrLockIsLockedByMe
+
+ @Description  Determine if the calling context is holding the device power-lock
+
+ @Return       IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVPwrLockIsLockedByMe(PCPVRSRV_DEVICE_NODE psDeviceNode);
+IMG_BOOL PVRSRVDeviceIsDefaultStateOFF(PVRSRV_POWER_DEV *psPowerDevice);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVSetDevicePowerStateKM
+
+ @Description	Set the Device into a new state
+
+ @Input		psDeviceNode : Device node
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDevicePowerStateKM(PPVRSRV_DEVICE_NODE	psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+										 PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVSetDeviceSystemPowerState
+@Description  Set the device into a new power state based on the systems power
+              state
+@Input        psDeviceNode       Device node
+@Input        eNewSysPowerState  New system power state
+@Input        ePwrFlags          Power state change flags
+@Return       PVRSRV_ERROR       PVRSRV_OK on success or an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState,
+											 PVRSRV_POWER_FLAGS ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetDeviceDefaultPowerState
+
+ @Description   Set the default device power state to eNewPowerState
+
+ @Input         psDeviceNode : Device node
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceDefaultPowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+					PVRSRV_DEV_POWER_STATE eNewPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetDeviceCurrentPowerState
+
+ @Description   Set the current device power state to eNewPowerState
+
+ @Input         psPowerDevice : Power device
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceCurrentPowerState(PVRSRV_POWER_DEV *psPowerDevice,
+					PVRSRV_DEV_POWER_STATE eNewPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetSystemPowerState
+
+ @Description   Set the system power state to eNewPowerState
+
+ @Input         psDeviceConfig : Device config
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetSystemPowerState(PVRSRV_DEVICE_CONFIG * psDeviceConfig,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetPowerCallbacks
+
+ @Description   Initialise the Power Device's function pointers
+                to the appropriate callbacks depending on driver mode and
+                system setup.
+
+ @Input         psDeviceNode : Device node
+ @Input         psPowerDevice : Power device
+ @Input         pfnDevicePrePower : regular device pre power callback
+ @Input         pfnDevicePostPower : regular device post power callback
+ @Input         pfnSystemPrePower : regular system pre power callback
+ @Input         pfnSystemPostPower : regular device pre power callback
+ @Input         pfnForcedIdleRequest : forced idle request callback
+ @Input         pfnForcedIdleCancelRequest : forced idle request cancel callback
+ @Input         pfnForcedIdleCancelRequestAsync : forced idle request cancel callback,
+                                                  doesn't wait for response.
+
+******************************************************************************/
+void PVRSRVSetPowerCallbacks(PPVRSRV_DEVICE_NODE				psDeviceNode,
+							 PVRSRV_POWER_DEV					*psPowerDevice,
+							 PFN_PRE_POWER						pfnDevicePrePower,
+							 PFN_POST_POWER						pfnDevicePostPower,
+							 PFN_SYS_PRE_POWER					pfnSystemPrePower,
+							 PFN_SYS_POST_POWER					pfnSystemPostPower,
+							 PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST		pfnForcedIdleCancelRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync);
+
+/* Type PFN_DC_REGISTER_POWER */
+PVRSRV_ERROR PVRSRVRegisterPowerDevice(PPVRSRV_DEVICE_NODE				psDeviceNode,
+									   PFN_PRE_POWER					pfnDevicePrePower,
+									   PFN_POST_POWER					pfnDevicePostPower,
+									   PFN_SYS_PRE_POWER			    pfnSystemPrePower,
+									   PFN_SYS_POST_POWER			    pfnSystemPostPower,
+									   PFN_PRE_CLOCKSPEED_CHANGE		pfnPreClockSpeedChange,
+									   PFN_POST_CLOCKSPEED_CHANGE		pfnPostClockSpeedChange,
+									   PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync,
+									   PFN_GPU_UNITS_POWER_CHANGE		pfnGPUUnitsPowerChange,
+									   IMG_HANDLE						hDevCookie,
+									   PVRSRV_DEV_POWER_STATE			eCurrentPowerState,
+									   PVRSRV_DEV_POWER_STATE			eDefaultPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVRemovePowerDevice
+
+ @Description
+
+ Removes device from power management register. Device is located by Device Index
+
+ @Input		psDeviceNode : Device node
+
+******************************************************************************/
+void PVRSRVRemovePowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVGetDevicePowerState
+
+ @Description
+
+	Return the device power state
+
+ @Input		psDeviceNode : Device node
+ @Output	pePowerState : Current power state
+
+ @Return	PVRSRV_ERROR_UNKNOWN_POWER_STATE if device could not be found.
+            PVRSRV_OK otherwise.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVGetDevicePowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+									   PPVRSRV_DEV_POWER_STATE pePowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVIsDevicePowered
+
+ @Description
+
+	Whether the device is powered, for the purposes of lockup detection.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVIsDevicePowered(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVGetSystemPowerState
+
+ @Description
+
+	Return the system power state
+
+ @Input		psDeviceNode : Device node
+ @Output	peCurrentSysPowerState : Current power state
+
+ @Return	PVRSRV_ERROR_UNKNOWN_POWER_STATE if device could not be found.
+            PVRSRV_OK otherwise.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVGetSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+	                                   PPVRSRV_SYS_POWER_STATE peCurrentSysPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVIsSystemPowered
+
+ @Description
+
+	Whether the system layer is powered, for ensuring the RGX regbank is powered
+	during initial GPU driver configuration.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVIsSystemPowered(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVDevicePreClockSpeedChange
+
+@Description    This function is called before a voltage/frequency change is
+                made to the GPU HW. It informs the host driver of the intention
+                to make a DVFS change. If allows the host driver to idle
+                the GPU and begin a hold off period from starting new work
+                on the GPU.
+                When this call succeeds the caller *must* call
+                PVRSRVDevicePostClockSpeedChange() to end the hold off period
+                to allow new work to be submitted to the GPU.
+
+                Called from system layer or OS layer implementation that
+                is responsible for triggering a GPU DVFS transition.
+
+@Input          psDeviceNode pointer to the device affected by DVFS transition.
+@Input          bIdleDevice  when True, the driver will wait for the GPU to
+                             reach an idle state before the call returns.
+@Input          pvInfo       unused
+
+@Return         PVRSRV_OK    on success, power lock acquired and held on exit,
+                             GPU idle.
+                PVRSRV_ERROR on failure, power lock not held on exit, do not
+                             call PVRSRVDevicePostClockSpeedChange().
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDevicePreClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 IMG_BOOL	bIdleDevice,
+											 void	*pvInfo);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVDevicePostClockSpeedChange
+
+@Description    This function is called after a voltage/frequency change has
+                been made to the GPU HW following a call to
+                PVRSRVDevicePreClockSpeedChange().
+                Before calling this function, the caller must ensure the system
+                data RGX_DATA->RGX_TIMING_INFORMATION->ui32CoreClockSpeed has
+                been updated with the new frequency set, measured in Hz.
+                The function informs the host driver that the DVFS change has
+                completed. The driver will end the work hold off period, cancel
+                the device idle period and update its time data records.
+                When this call returns work submissions are unblocked and
+                are submitted to the GPU as normal.
+                This function *must* not be called if the preceding call to
+                PVRSRVDevicePreClockSpeedChange() failed.
+
+                Called from system layer or OS layer implementation that
+                is responsible for triggering a GPU DVFS transition.
+
+@Input          psDeviceNode pointer to the device affected by DVFS transition.
+@Input          bIdleDevice  when True, the driver will cancel the GPU
+                             device idle state before the call returns. Value
+                             given must match that used in the call to
+                             PVRSRVDevicePreClockSpeedChange() otherwise
+                             undefined behaviour will result.
+@Input          pvInfo       unused
+
+@Return         void         power lock released, no longer held on exit.
+*/ /**************************************************************************/
+void PVRSRVDevicePostClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+									  IMG_BOOL		bIdleDevice,
+									  void		*pvInfo);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVDeviceIdleRequestKM
+
+ @Description Perform device-specific processing required to force the device
+              idle. The device power-lock might be temporarily released (and
+              again re-acquired) during the course of this call, hence to
+              maintain lock-ordering power-lock should be the last acquired
+              lock before calling this function
+
+ @Input       psDeviceNode         : Device node
+
+ @Input       pfnIsDefaultStateOff : When specified, the idle request is only
+                                     processed if this function passes.
+
+ @Input       bDeviceOffPermitted  : IMG_TRUE if the transition should not fail
+                                       if device off
+                                     IMG_FALSE if the transition should fail if
+                                       device off
+
+ @Return      PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED
+                                     When re-acquisition of power-lock failed.
+                                     This error NEEDS EXPLICIT HANDLING at call
+                                     site as it signifies the caller needs to
+                                     AVOID calling PVRSRVPowerUnlock, since
+                                     power-lock is no longer "possessed" by
+                                     this context.
+
+              PVRSRV_OK              When idle request succeeded.
+              PVRSRV_ERROR           Other system errors.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF	pfnIsDefaultStateOff,
+					IMG_BOOL				bDeviceOffPermitted);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceIdleCancelRequestKM
+
+ @Description Perform device-specific processing required to cancel the forced idle state
+              on the device, returning to normal operation.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleCancelRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVDeviceIdleLatchedGetKM
+
+ @Description Perform device-specific processing required to force the device
+              idle. The device power-lock might be temporarily released (and
+              again re-acquired) during the course of this call, hence to
+              maintain lock-ordering power-lock should be the last acquired
+              lock before calling this function. Latched based on power
+              device internal refcount. Only actions idle on initial reference
+              increment via calling this function.
+
+ @Input       psDeviceNode         : Device node
+
+ @Return      PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED
+                                     When re-acquisition of power-lock failed.
+                                     Handled internally but still represents error
+                                     occurred.
+
+              PVRSRV_OK              When idle request succeeded.
+              PVRSRV_ERROR           Other system errors.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedGetKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceIdleLatchedPutAsyncKM
+
+ @Description Perform device-specific processing required to cancel the forced idle state
+              on the device, returning to normal operation. Does not wait for FW response.
+              Latched based on power device internal refcount. Only actions idle cancel
+              once final reference is dropped via calling this function.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedPutAsyncKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+@Function       PVRSRVDeviceGPUUnitsPowerChange
+@Description    Request from system layer for changing power state of GPU
+                units
+@Input          psDeviceNode            RGX Device Node.
+@Input          ui32NewValue            Value indicating the new power state
+                                        of GPU units. how this is interpreted
+                                        depends upon the device-specific
+                                        function subsequently called by the
+                                        server via a pfn.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceGPUUnitsPowerChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+					IMG_UINT32	ui32NewValue);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+void PVRSRVSetFirmwareStartTime(PVRSRV_POWER_DEV *psPowerDevice, IMG_UINT32 ui32TimeStamp);
+
+void PVRSRVSetFirmwareHandshakeIdleTime(PVRSRV_POWER_DEV *psPowerDevice, IMG_UINT64 ui64Duration);
+
+int PVRSRVPowerStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+#endif
+
+#endif /* POWER_H */
+
+/******************************************************************************
+ End of file (power.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/process_stats.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/process_stats.h
new file mode 100644
index 000000000000..c410736e1749
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/process_stats.h
@@ -0,0 +1,220 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for creating and reading proc filesystem entries.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PROCESS_STATS_H
+#define PROCESS_STATS_H
+
+#include <powervr/mem_types.h>
+
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "connection_server.h"
+
+/*
+ * The publishing of Process Stats is controlled by the
+ * PVRSRV_ENABLE_PROCESS_STATS build option. The recording of all Memory
+ * allocations is controlled by the PVRSRV_ENABLE_MEMORY_STATS build option.
+ *
+ * Note: There will be a performance degradation with memory allocation
+ *       recording enabled!
+ */
+
+
+/*
+ * Memory types which can be tracked...
+ */
+typedef enum {
+	PVRSRV_MEM_ALLOC_TYPE_KMALLOC,				/* memory allocated by kmalloc() */
+	PVRSRV_MEM_ALLOC_TYPE_VMALLOC,				/* memory allocated by vmalloc() */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,	/* pages allocated from UMA to hold page table information */
+	PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,			/* ALLOC_PAGES_PT_UMA mapped to kernel address space */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,	/* pages allocated from LMA to hold page table information */
+	PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,		/* ALLOC_PAGES_PT_LMA mapped to kernel address space */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,		/* pages allocated from LMA */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,		/* pages allocated from UMA */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES,		/* zombie pages allocated from LMA */
+	PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,		/* zombie pages allocated from UMA */
+#endif
+	PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,	/* mapped UMA/LMA pages */
+	PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES,		/* pages in the page pool */
+	PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,       /* dma-buf imports */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,       /* dma-buf zombie */
+#endif
+
+	/* Must be the last enum...*/
+	PVRSRV_MEM_ALLOC_TYPE_COUNT
+} PVRSRV_MEM_ALLOC_TYPE;
+
+/*
+ * Functions for managing the processes recorded...
+ */
+PVRSRV_ERROR PVRSRVStatsInitialise(void);
+void PVRSRVStatsDestroy(void);
+void PVRSRVStatsDestroyDI(void);
+
+PVRSRV_ERROR PVRSRVStatsRegisterProcess(IMG_HANDLE* phProcessStats);
+
+void PVRSRVStatsDeregisterProcess(IMG_HANDLE hProcessStats);
+
+PVRSRV_ERROR PVRSRVStatsDeviceConnect(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void PVRSRVStatsDeviceDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*
+ * Functions for recording the statistics...
+ */
+
+void PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  void *pvCpuVAddr,
+								  IMG_CPU_PHYADDR sCpuPAddr,
+								  size_t uiBytes,
+								  IMG_PID uiPid
+								  DEBUG_MEMSTATS_PARAMS);
+
+void PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									 IMG_UINT64 ui64Key,
+									 IMG_PID uiPid);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eCurrentType,
+                                       PVRSRV_MEM_ALLOC_TYPE eTargetType,
+                                       IMG_UINT64 ui64Key,
+                                       IMG_PID currentPid
+                                       DEBUG_MEMSTATS_PARAMS);
+#endif
+
+void PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								 size_t uiBytes,
+								 IMG_PID uiPid);
+
+/*
+ * Increases the memory stat for eAllocType. Tracks the allocation size value
+ * by inserting a value into a hash table with uiCpuVAddr as key.
+ * Pair with PVRSRVStatsDecrMemAllocStatAndUntrack().
+ */
+void PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+										 size_t uiBytes,
+										 IMG_UINT64 uiCpuVAddr,
+										 IMG_PID uiPid);
+
+void PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								 size_t uiBytes,
+								 IMG_PID uiPid);
+
+void PVRSRVStatsDecrMemKAllocStat(size_t uiBytes,
+								  IMG_PID decrPID);
+
+/*
+ * Decrease the memory stat for eAllocType. Takes the allocation size value
+ * from the hash table with uiCpuVAddr as key.
+ * Pair with PVRSRVStatsIncrMemAllocStatAndTrack().
+ */
+void PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									IMG_UINT64 uiCpuVAddr);
+
+void
+PVRSRVStatsIncrMemAllocPoolStat(size_t uiBytes);
+
+void
+PVRSRVStatsDecrMemAllocPoolStat(size_t uiBytes);
+
+PVRSRV_ERROR
+PVRSRVStatsUpdateOOMStat(CONNECTION_DATA *psConnection,
+						  PVRSRV_DEVICE_NODE *psDeviceNode,
+						  IMG_UINT32 ui32OOMStatType,
+						  IMG_PID pidOwner);
+
+void PVRSRVStatsUpdateRenderContextStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+										 IMG_UINT32 ui32TotalNumPartialRenders,
+										 IMG_UINT32 ui32TotalNumOutOfMemory,
+										 IMG_UINT32 ui32TotalTAStores,
+										 IMG_UINT32 ui32Total3DStores,
+										 IMG_UINT32 ui32TotalCDMStores,
+										 IMG_UINT32 ui32TotalTDMStores,
+										 IMG_UINT32 ui32NumRayStores,
+										 IMG_PID owner);
+
+void PVRSRVStatsUpdateZSBufferStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32NumReqByApp,
+									IMG_UINT32 ui32NumReqByFW,
+									IMG_PID owner);
+
+void PVRSRVStatsUpdateFreelistStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32NumGrowReqByApp,
+									IMG_UINT32 ui32NumGrowReqByFW,
+									IMG_UINT32 ui32InitFLPages,
+									IMG_UINT32 ui32NumHighPages,
+									IMG_PID    ownerPid);
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void PVRSRVStatsUpdateCacheOpStats(PVRSRV_CACHE_OP uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+								   IMG_DEV_VIRTADDR sDevVAddr,
+								   IMG_DEV_PHYADDR sDevPAddr,
+#endif
+								   IMG_DEVMEM_SIZE_T uiOffset,
+								   IMG_DEVMEM_SIZE_T uiSize,
+								   IMG_UINT64 ui64ExecuteTimeMs,
+								   IMG_BOOL bUserModeFlush,
+								   IMG_PID ownerPid);
+#endif
+
+/* Functions used for calculating the memory usage statistics of a process */
+PVRSRV_ERROR PVRSRVFindProcessMemStats(IMG_PID pid,
+                                       IMG_UINT32 ui32ArrSize,
+                                       IMG_BOOL bAllProcessStats,
+                                       IMG_UINT64 *pui64MemoryStats);
+
+typedef struct {
+	IMG_UINT32 ui32Pid;
+	IMG_UINT64 ui64KernelMemUsage;
+	IMG_UINT64 ui64GraphicsMemUsage;
+} PVRSRV_PER_PROCESS_MEM_USAGE;
+
+PVRSRV_ERROR PVRSRVGetProcessMemUsage(IMG_UINT64 *pui64TotalMem,
+                                      IMG_UINT32 *pui32NumberOfLivePids,
+                                      PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsageData);
+
+#endif /* PROCESS_STATS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_dvfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_dvfs.h
new file mode 100644
index 000000000000..0529b6d20a10
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_dvfs.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs.h
+@Title          System level interface for DVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_H
+#define PVR_DVFS_H
+
+#if defined(SUPPORT_LINUX_DVFS)
+ #include <linux/devfreq.h>
+ #include <linux/thermal.h>
+
+ #if defined(CONFIG_DEVFREQ_THERMAL)
+  #include <linux/devfreq_cooling.h>
+ #endif
+
+ #include <linux/pm_opp.h>
+#endif
+
+#include "img_types.h"
+
+typedef void (*PFN_SYS_DEV_DVFS_SET_FREQUENCY)(IMG_HANDLE hSysData, IMG_UINT32 ui32Freq);
+typedef void (*PFN_SYS_DEV_DVFS_SET_VOLTAGE)(IMG_HANDLE hSysData, IMG_UINT32 ui32Volt);
+
+typedef struct _IMG_OPP_
+{
+	IMG_UINT32			ui32Volt;
+	/*
+	 * Unit of frequency in Hz.
+	 */
+	IMG_UINT32			ui32Freq;
+} IMG_OPP;
+
+typedef struct _IMG_DVFS_DEVICE_CFG_
+{
+	const IMG_OPP  *pasOPPTable;
+	IMG_UINT32      ui32OPPTableSize;
+#if defined(SUPPORT_LINUX_DVFS)
+	IMG_UINT32      ui32PollMs;
+#endif
+	IMG_BOOL        bIdleReq;
+	IMG_BOOL        bDTConfig;
+	PFN_SYS_DEV_DVFS_SET_FREQUENCY  pfnSetFrequency;
+	PFN_SYS_DEV_DVFS_SET_VOLTAGE    pfnSetVoltage;
+
+#if defined(CONFIG_DEVFREQ_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+	struct devfreq_cooling_power *psPowerOps;
+#endif
+} IMG_DVFS_DEVICE_CFG;
+
+#if defined(SUPPORT_LINUX_DVFS)
+typedef struct _IMG_DVFS_GOVERNOR_
+{
+	IMG_BOOL			bEnabled;
+} IMG_DVFS_GOVERNOR;
+
+typedef struct _IMG_DVFS_GOVERNOR_CFG_
+{
+	IMG_UINT32			ui32UpThreshold;
+	IMG_UINT32			ui32DownDifferential;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* custom thresholds */
+	IMG_UINT32			uiNumMembus;
+#endif
+} IMG_DVFS_GOVERNOR_CFG;
+#endif
+
+#if defined(__linux__)
+#if defined(SUPPORT_LINUX_DVFS)
+typedef enum
+{
+	PVR_DVFS_STATE_NONE	= 0,
+	PVR_DVFS_STATE_INIT_PENDING,
+	PVR_DVFS_STATE_READY,
+	PVR_DVFS_STATE_OFF,
+	PVR_DVFS_STATE_DEINIT
+} PVR_DVFS_STATE;
+
+typedef struct _IMG_DVFS_DEVICE_
+{
+	struct dev_pm_opp		*psOPP;
+	struct devfreq			*psDevFreq;
+	PVR_DVFS_STATE		eState;
+	IMG_HANDLE			hGpuUtilUserDVFS;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	IMG_DVFS_GOVERNOR_CFG data;
+	IMG_BOOL			bGovernorReady;
+#else
+	struct devfreq_simple_ondemand_data data;
+#endif
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	struct thermal_cooling_device	*psDevfreqCoolingDevice;
+#endif
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	struct pvr_profiling_device *psProfilingDevice;
+#endif
+} IMG_DVFS_DEVICE;
+#endif
+
+typedef struct _IMG_DVFS_
+{
+#if defined(SUPPORT_LINUX_DVFS)
+	IMG_DVFS_DEVICE			sDVFSDevice;
+	IMG_DVFS_GOVERNOR		sDVFSGovernor;
+	IMG_DVFS_GOVERNOR_CFG	sDVFSGovernorCfg;
+#endif
+	IMG_DVFS_DEVICE_CFG		sDVFSDeviceCfg;
+} PVRSRV_DVFS;
+#endif/* (__linux__) */
+
+#endif /* PVR_DVFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_notifier.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_notifier.h
new file mode 100644
index 000000000000..b3757494e4b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_notifier.h
@@ -0,0 +1,326 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR notifier interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVR_NOTIFIER_H)
+#define PVR_NOTIFIER_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+
+
+/*************************************************************************/ /*!
+Command Complete Notifier Interface
+*/ /**************************************************************************/
+
+typedef IMG_HANDLE PVRSRV_CMDCOMP_HANDLE;
+#ifndef CMDCOMPNOTIFY_PFN
+typedef void (*PFN_CMDCOMP_NOTIFY)(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle);
+#define CMDCOMPNOTIFY_PFN
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCmdCompleteInit
+@Description    Performs initialisation of the command complete notifier
+                interface.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVCmdCompleteInit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCmdCompleteDeinit
+@Description    Performs cleanup for the command complete notifier interface.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+void
+PVRSRVCmdCompleteDeinit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterCmdCompleteNotify
+@Description    Register a callback function that is called when some device
+                finishes some work, which is signalled via a call to
+                PVRSRVCheckStatus.
+@Output         phNotify             On success, points to command complete
+                                     notifier handle
+@Input          pfnCmdCompleteNotify Function callback
+@Input          hPrivData            Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterCmdCompleteNotify(IMG_HANDLE *phNotify,
+                                PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify,
+                                PVRSRV_CMDCOMP_HANDLE hPrivData);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterCmdCompleteNotify
+@Description    Unregister a previously registered callback function.
+@Input          hNotify              Command complete notifier handle
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterCmdCompleteNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCheckStatus
+@Description    Calls PVRSRVNotifyCommandCompletion() to notify registered
+                command complete handlers of work completion and then calls
+                PVRSRVSignalGlobalEO() to signal the driver wide event objects.
+@Input          hCmdCompCallerHandle Used to prevent a handler from being
+                                     notified. A NULL value results in all
+                                     handlers being notified.
+*/ /**************************************************************************/
+void
+PVRSRVCheckStatus(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVNotifyCommandCompletion
+@Description    Notify any registered command complete handlers that some work
+                has been finished (unless hCmdCompCallerHandle matches a
+                handler's hPrivData).
+@Input          hCmdCompCallerHandle Used to prevent a handler from being
+                                     notified. A NULL value results in all
+                                     handlers being notified.
+*/ /**************************************************************************/
+void
+PVRSRVNotifyCommandCompletion(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVSignalDriverWideEO
+@Description    Signals the driver wide event objects.
+*/ /**************************************************************************/
+void
+PVRSRVSignalDriverWideEO(void);
+
+
+/*************************************************************************/ /*!
+Debug Notifier Interface
+*/ /**************************************************************************/
+
+#define DEBUG_REQUEST_DC                0
+#define DEBUG_REQUEST_SYNCTRACKING      1
+#define DEBUG_REQUEST_SRV               2
+#define DEBUG_REQUEST_SYS               3
+#define DEBUG_REQUEST_RGX               4
+#define DEBUG_REQUEST_ANDROIDSYNC       5
+#define DEBUG_REQUEST_LINUXFENCE        6
+#define DEBUG_REQUEST_SYNCCHECKPOINT    7
+#define DEBUG_REQUEST_HTB               8
+#define DEBUG_REQUEST_APPHINT           9
+#define DEBUG_REQUEST_FALLBACKSYNC      10
+
+#define DEBUG_REQUEST_VERBOSITY_LOW     0
+#define DEBUG_REQUEST_VERBOSITY_MEDIUM  1
+#define DEBUG_REQUEST_VERBOSITY_HIGH    2
+#define DEBUG_REQUEST_VERBOSITY_MAX     DEBUG_REQUEST_VERBOSITY_HIGH
+
+#define DD_VERB_LVL_ENABLED(_verbLvl, _verbLvlChk) ((_verbLvl) >= (_verbLvlChk))
+
+/*
+ * Macro used within debug dump functions to send output either to PVR_LOG or
+ * a custom function. The custom function should be stored as a function
+ * pointer in a local variable called 'pfnDumpDebugPrintf'. 'pvDumpDebugFile'
+ * is also required as a local variable to serve as a file identifier for the
+ * printf function if required.
+ */
+#define PVR_DUMPDEBUG_LOG(...)                                \
+	do                                                        \
+	{                                                         \
+		if (pfnDumpDebugPrintf)                               \
+			pfnDumpDebugPrintf(pvDumpDebugFile, __VA_ARGS__); \
+		else                                                  \
+			PVR_LOG((__VA_ARGS__));                           \
+	} while (0)
+
+struct _PVRSRV_DEVICE_NODE_;
+
+typedef IMG_HANDLE PVRSRV_DBGREQ_HANDLE;
+#ifndef DBGNOTIFY_PFNS
+typedef void (DUMPDEBUG_PRINTF_FUNC)(void *pvDumpDebugFile,
+					const IMG_CHAR *pszFormat, ...);
+typedef void (*PFN_DBGREQ_NOTIFY)(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile);
+#define DBGNOTIFY_PFNS
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDeviceDbgTable
+@Description    Registers a debug requester table for the given device. The
+                order in which the debug requester IDs appear in the
+                table determine the order in which a set of notifier callbacks
+                will be called. In other words, the requester ID that appears
+                first will have all of its associated debug notifier callbacks
+                called first. This will then be followed by all the callbacks
+                associated with the next requester ID in the table and so on.
+                The order table is handled internally.
+@Input          psDevNode     Device node to register requester table with
+@Return         PVRSRV_ERROR  PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgTable(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDriverDbgTable
+@Description    Registers a debug requester table for the driver. The
+                order in which the debug requester IDs appear in the
+                table determine the order in which a set of notifier callbacks
+                will be called. In other words, the requester ID that appears
+                first will have all of its associated debug notifier callbacks
+                called first. This will then be followed by all the callbacks
+                associated with the next requester ID in the table and so on.
+                The order table is handled internally.
+@Return         PVRSRV_ERROR  PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgTable(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDeviceDbgTable
+@Description    Unregisters a debug requester table.
+@Input          psDevNode     Device node for which the requester table should
+                              be unregistered
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVUnregisterDeviceDbgTable(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDriverDbgTable
+@Description    Unregisters the driver debug requester table.
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVUnregisterDriverDbgTable(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDeviceDbgRequestNotify
+@Description    Register a callback function on a given device that is called
+                when a debug request is made via a call PVRSRVDebugRequest.
+                There are a number of verbosity levels ranging from
+                DEBUG_REQUEST_VERBOSITY_LOW up to
+                DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+                for each level up to the highest level specified to
+                PVRSRVDebugRequest.
+@Output         phNotify             Points to debug notifier handle on success
+@Input          psDevNode            Device node for which the debug callback
+                                     should be registered
+@Input          pfnDbgRequestNotify  Function callback
+@Input          ui32RequesterID      Requester ID. This is used to determine
+                                     the order in which callbacks are called
+@Input          hDbgReqeustHandle    Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgRequestNotify(IMG_HANDLE *phNotify,
+                                     struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                     PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+                                     IMG_UINT32 ui32RequesterID,
+                                     PVRSRV_DBGREQ_HANDLE hDbgReqeustHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDriverDbgRequestNotify
+@Description    Register a callback function that is called when a debug request
+                is made via a call PVRSRVDebugRequest. There are a number of
+                verbosity levels ranging from DEBUG_REQUEST_VERBOSITY_LOW up to
+                DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+                for each level up to the highest level specified to
+                PVRSRVDebugRequest.
+@Output         phNotify             Points to debug notifier handle on success
+@Input          pfnDbgRequestNotify  Function callback
+@Input          ui32RequesterID      Requester ID. This is used to determine
+                                     the order in which callbacks are called
+@Input          hDbgReqeustHandle    Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgRequestNotify(IMG_HANDLE *phNotify,
+									 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+									 IMG_UINT32 ui32RequesterID,
+									 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDeviceDbgRequestNotify
+@Description    Unregister a previously registered (device context) callback
+                function.
+@Input          hNotify              Debug notifier handle.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterDeviceDbgRequestNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDriverDbgRequestNotify
+@Description    Unregister a previously registered (driver context) callback
+                function.
+@Input          hNotify              Debug notifier handle.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterDriverDbgRequestNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugRequest
+@Description    Notify any registered debug request handlers that a debug
+                request has been made and at what level.
+@Input          psDevNode           Device node for which the debug request
+                                    has been made
+@Input          ui32VerbLevel       The maximum verbosity level to dump
+@Input          pfnDumpDebugPrintf  Used to specify the print function that
+                                    should be used to dump any debug
+                                    information. If this argument is NULL then
+                                    PVR_LOG() will be used as the default
+                                    print function.
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVDebugRequest(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                   IMG_UINT32 ui32VerbLevel,
+                   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                   void *pvDumpDebugFile);
+
+#endif /* !defined(PVR_NOTIFIER_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv.h
new file mode 100644
index 000000000000..e9a87ba8ad06
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv.h
@@ -0,0 +1,521 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR services server header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_H
+#define PVRSRV_H
+
+#include "connection_server.h"
+#include "pvrsrv_pool.h"
+#include "device.h"
+#include "power.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+#include "physheap.h"
+#include "cache_ops.h"
+#include "pvr_notifier.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "dma_support.h"
+#include "vz_vmm_pvz.h"
+
+/*!
+ * For OSThreadDestroy(), which may require a retry
+ * Try for 100 ms to destroy an OS thread before failing
+ */
+#define OS_THREAD_DESTROY_TIMEOUT_US 100000ULL
+#define OS_THREAD_DESTROY_RETRY_COUNT 10
+
+typedef enum _POLL_FLAGS_
+{
+	POLL_FLAG_NONE = 0, /* No message or dump is printed on poll timeout */
+	POLL_FLAG_LOG_ERROR = 1, /* Log error on poll timeout */
+	POLL_FLAG_DEBUG_DUMP = 2 /* Print debug dump on poll timeout */
+} POLL_FLAGS;
+
+typedef struct _BUILD_INFO_
+{
+	IMG_UINT32	ui32BuildOptions;
+	IMG_UINT32	ui32BuildVersion;
+	IMG_UINT32	ui32BuildRevision;
+	IMG_UINT32	ui32BuildType;
+#define BUILD_TYPE_DEBUG	0
+#define BUILD_TYPE_RELEASE	1
+	/* The above fields are self explanatory */
+	/* B.V.N.C can be added later if required */
+} BUILD_INFO;
+
+typedef struct _DRIVER_INFO_
+{
+	BUILD_INFO	sUMBuildInfo;
+	BUILD_INFO	sKMBuildInfo;
+	IMG_UINT8	ui8UMSupportedArch;
+	IMG_UINT8	ui8KMBitArch;
+
+#define	BUILD_ARCH_64BIT			(1 << 0)
+#define	BUILD_ARCH_32BIT			(1 << 1)
+#define	BUILD_ARCH_BOTH		(BUILD_ARCH_32BIT | BUILD_ARCH_64BIT)
+	IMG_BOOL	bIsNoMatch;
+}DRIVER_INFO;
+
+
+typedef struct PVRSRV_DATA_TAG
+{
+	DRIVER_INFO           sDriverInfo;
+	IMG_UINT32            ui32DPFErrorCount;                 /*!< Number of Fatal/Error DPFs */
+
+	POSWR_LOCK            hDeviceNodeListLock;            /*!< Read-Write lock to protect the list of devices */
+	PVRSRV_DEVICE_NODE    *psDeviceNodeList;              /*!< List head of device nodes */
+	IMG_UINT32            ui32RegisteredDevices;
+	PVRSRV_DEVICE_NODE    *psHostMemDeviceNode;           /*!< DeviceNode to be used for device independent
+	                                                        host based memory allocations where the DevMem
+	                                                        framework is to be used e.g. TL */
+	PVRSRV_SERVICES_STATE eServicesState;                 /*!< global driver state */
+
+	IMG_HANDLE            hGlobalEventObject;             /*!< OS Global Event Object */
+	IMG_UINT32            ui32GEOConsecutiveTimeouts;     /*!< OS Global Event Object Timeouts */
+
+	IMG_HANDLE            hCleanupThread;                 /*!< Cleanup thread */
+	IMG_HANDLE            hCleanupEventObject;            /*!< Event object to drive cleanup thread */
+	POS_SPINLOCK          hCleanupThreadWorkListLock;     /*!< Lock protecting the cleanup thread work list */
+	IMG_PID               cleanupThreadPid;               /*!< Cleanup thread process id */
+	uintptr_t             cleanupThreadTid;               /*!< Cleanup thread id */
+	ATOMIC_T              i32NumCleanupItemsQueued;       /*!< Number of items in cleanup thread work list */
+	ATOMIC_T              i32NumCleanupItemsNotCompleted; /*!< Number of items dropped from cleanup thread work list
+	                                                           after retry limit reached */
+	ATOMIC_T              i32CleanupItemTypes[PVRSRV_CLEANUP_TYPE_LAST];         /*!< Array containing the counts for different cleanup item types. */
+
+	IMG_HANDLE            hDevicesWatchdogThread;         /*!< Devices watchdog thread */
+	IMG_HANDLE            hDevicesWatchdogEvObj;          /*! Event object to drive devices watchdog thread */
+	volatile IMG_UINT32   ui32DevicesWatchdogPwrTrans;    /*! Number of off -> on power state transitions */
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	volatile IMG_UINT32   ui32DevicesWatchdogTimeout;     /*! Timeout for the Devices watchdog Thread */
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_HANDLE            hAutoVzWatchdogThread;          /*!< Devices watchdog thread */
+	IMG_HANDLE            hAutoVzWatchdogEvObj;           /*! Event object to drive devices watchdog thread */
+#endif
+
+	POS_LOCK              hHWPerfHostPeriodicThread_Lock; /*!< Lock for the HWPerf Host periodic thread */
+	IMG_HANDLE            hHWPerfHostPeriodicThread;      /*!< HWPerf Host periodic thread */
+	IMG_HANDLE            hHWPerfHostPeriodicEvObj;       /*! Event object to drive HWPerf thread */
+	volatile IMG_BOOL     bHWPerfHostThreadStop;
+	IMG_UINT32            ui32HWPerfHostThreadTimeout;
+
+	POS_LOCK              hClientStreamTableLock;         /*!< Lock for the client stream hash */
+	HASH_TABLE            *psClientStreamTable;           /*!< Hash table for client streams */
+
+	IMG_HANDLE            hPvzConnection;                 /*!< PVZ connection used for cross-VM hyper-calls */
+	POS_LOCK              hPvzConnectionLock;             /*!< Lock protecting PVZ connection */
+
+	IMG_BOOL              bUnload;                        /*!< Driver unload is in progress */
+
+	IMG_HANDLE            hTLCtrlStream;                  /*! Control plane for TL streams */
+
+	IMG_HANDLE            hDriverThreadEventObject;       /*! Event object relating to multi-threading in the Server */
+	IMG_BOOL              bDriverSuspended;               /*! if TRUE, the driver is suspended and new threads should not enter */
+	ATOMIC_T              iNumActiveDriverThreads;        /*! Number of threads active in the Server */
+
+	PMR                   *psInfoPagePMR;                 /*! Handle to exportable PMR of the information page. */
+	IMG_UINT32            *pui32InfoPage;                 /*! CPU memory mapping for information page. */
+	DEVMEM_MEMDESC        *psInfoPageMemDesc;             /*! Memory descriptor of the information page. */
+	POS_LOCK              hInfoPageLock;                  /*! Lock guarding access to information page. */
+
+	IMG_HANDLE            hThreadsDbgReqNotify;
+
+
+	IMG_UINT32            ui32PDumpBoundDevice;           /*!< PDump is bound to the device first connected to */
+	ATOMIC_T              iNumDriverTasksActive;          /*!< Number of device-agnostic tasks active in the server */
+	PVRSRV_DRIVER_MODE    aeModuleParamDriverMode[PVRSRV_MAX_DEVICES]; /*!< Driver Mode for each device requested at launch */
+} PVRSRV_DATA;
+
+/* Function pointer used to invalidate cache between loops in wait/poll for value functions */
+typedef PVRSRV_ERROR (*PFN_INVALIDATE_CACHEFUNC)(const volatile void*, IMG_UINT64, PVRSRV_CACHE_OP);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVGetPVRSRVData
+
+ @Description	Get a pointer to the global data
+
+ @Return   PVRSRV_DATA *
+******************************************************************************/
+PVRSRV_DATA *PVRSRVGetPVRSRVData(void);
+PVRSRV_DRIVER_MODE PVRSRVGetVzModeByDevNum(IMG_UINT32 ui32DevNum);
+
+#define PVRSRV_KM_ERRORS                     ( PVRSRVGetPVRSRVData() ? PVRSRVGetPVRSRVData()->ui32DPFErrorCount : IMG_UINT32_MAX)
+#define PVRSRV_ERROR_LIMIT_REACHED                (PVRSRV_KM_ERRORS == IMG_UINT32_MAX)
+#define PVRSRV_REPORT_ERROR()                do { if (!(PVRSRV_ERROR_LIMIT_REACHED)) { PVRSRVGetPVRSRVData()->ui32DPFErrorCount++; } } while (0)
+
+#define PVRSRV_VZ_MODE_FROM_DEVNODE(pnode)     (pnode->psDevConfig->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVINFO(pdevinfo)  (pdevinfo->psDeviceNode->psDevConfig->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVCFG(pdevcfg)    (pdevcfg->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVID(devid)       (PVRSRVGetVzModeByDevNum(devid))
+
+#define PVRSRV_VZ_MODE_IS(_expr, _struct, dev) (DRIVER_MODE_##_expr == PVRSRV_VZ_MODE_FROM_##_struct(dev))
+
+#define PVRSRV_VZ_RETN_IF_MODE(_expr, _struct, dev)         do { if (  PVRSRV_VZ_MODE_IS(_expr, _struct, dev)) { return; } } while (0)
+#define PVRSRV_VZ_RET_IF_MODE(_expr, _struct, dev, _rc)     do { if (  PVRSRV_VZ_MODE_IS(_expr, _struct, dev)) { return (_rc); } } while (0)
+
+#define PVRSRV_VZ_TIME_SLICE_MAX	(100UL)
+
+typedef struct _PHYS_HEAP_ITERATOR_ PHYS_HEAP_ITERATOR;
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorCreate
+
+ @Description
+ Creates iterator for traversing physical heap requested by ePhysHeap. The
+ iterator will go through all of the segments (a segment is physically
+ contiguous) of the physical heap and return their CPU physical address and
+ size.
+
+ @Input psDevNode: Pointer to device node struct.
+ @Input ePhysHeap: Find the matching heap.
+ @Output ppsIter: Pointer to the iterator object.
+
+ @Return PVRSRV_OK upon success and PVRSRV_ERROR otherwise.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorCreate(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_PHYS_HEAP ePhysHeap,
+                                    PHYS_HEAP_ITERATOR **ppsIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorDestroy
+
+ @Description
+ Frees the iterator object created with LMA_HeapIteratorCreate.
+
+ @Input psIter: Pointer to the iterator object.
+******************************************************************************/
+void LMA_HeapIteratorDestroy(PHYS_HEAP_ITERATOR *psIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorReset
+
+ @Description
+ Resets the iterator the first segment of the physical heap.
+
+ @Input psIter: Pointer to the iterator object.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorReset(PHYS_HEAP_ITERATOR *psIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorNext
+
+ @Description
+ Retrieves current segment's physical device address and size and moves the
+ iterator to the next element (if exists). If the iterator reached an end of
+ the heap and no segment was retrieved, this function returns IMG_FALSE.
+
+ @Input psIter: Pointer to the iterator object.
+ @Output psDevPAddr: Device physical address of the current segment.
+ @Output puiSize: Size of the current segment.
+
+ @Return IMG TRUE if a segment was found and retrieved, IMG_FALSE otherwise.
+******************************************************************************/
+IMG_BOOL LMA_HeapIteratorNext(PHYS_HEAP_ITERATOR *psIter,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              IMG_UINT64 *puiSize);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorGetHeapStats
+
+ @Description
+ Retrieves phys heap's usage statistics.
+
+ @Input psPhysHeap: Pointer to the physical heap object.
+ @Output puiTotalSize: Total size of the physical heap.
+ @Output puiInUseSize: Used space in the physical heap.
+
+ @Return PVRSRV_OK upon success and PVRSRV_otherwise.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorGetHeapStats(PHYS_HEAP_ITERATOR *psIter,
+                                          IMG_UINT64 *puiTotalSize,
+                                          IMG_UINT64 *puiInUseSize);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVPollForValueKM
+
+ @Description
+ Polls for a value to match a masked read
+
+ @Input psDevNode : Pointer to device node struct
+ @Input pui32LinMemAddr : CPU linear address to poll
+ @Input ui32Value : required value
+ @Input ui32Mask : Mask
+ @Input bDebugDumpOnFailure : Whether poll failure should result into a debug
+        dump. CAUTION: When calling this function from code paths which are
+        also used by debug-dumping code, this argument MUST be IMG_FALSE
+        otherwise, we might end up requesting debug-dump in recursion and
+        eventually blow-up call stack.
+ @Input pfnFwInvalidate : Function pointer to invalidation function used
+        each loop / poll. This is only used for FWmemctx allocations.
+
+ @Return   PVRSRV_ERROR :
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPollForValueKM(PVRSRV_DEVICE_NODE *psDevNode,
+		volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+		IMG_UINT32                   ui32Value,
+		IMG_UINT32                   ui32Mask,
+		POLL_FLAGS                   ePollFlags,
+		PFN_INVALIDATE_CACHEFUNC     pfnFwInvalidate);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVWaitForValueKM
+
+ @Description
+ Waits (using EventObjects) for a value to match a masked read
+
+ @Input  pui32LinMemAddr       : CPU linear address to poll
+ @Input  ui32Value             : Required value
+ @Input  ui32Mask              : Mask to be applied before checking against
+                                 ui32Value
+ @Input  pfnFwInvalidate       : Function pointer to invalidation function used
+                                 each loop / wait. This is only used for
+                                 FWmemctx allocations.
+ @Return PVRSRV_ERROR          :
+******************************************************************************/
+PVRSRV_ERROR
+PVRSRVWaitForValueKM(volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+                     IMG_UINT32                  ui32Value,
+                     IMG_UINT32                  ui32Mask,
+                     PFN_INVALIDATE_CACHEFUNC    pfnFwInvalidate);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemHasCacheSnooping
+
+ @Description	: Returns whether the system has cache snooping
+
+ @Return : IMG_TRUE if the system has cache snooping
+******************************************************************************/
+IMG_BOOL PVRSRVSystemHasCacheSnooping(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemSnoopingIsEmulated
+
+ @Description : Returns whether system cache snooping support is emulated
+
+ @Return : IMG_TRUE if the system cache snooping is emulated in software
+******************************************************************************/
+IMG_BOOL PVRSRVSystemSnoopingIsEmulated(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemSnoopingOfCPUCache
+
+ @Description	: Returns whether the system supports snooping of the CPU cache
+
+ @Return : IMG_TRUE if the system has CPU cache snooping
+******************************************************************************/
+IMG_BOOL PVRSRVSystemSnoopingOfCPUCache(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemHasNonMappableLocalMemory
+
+ @Description	: Returns whether the device has non-mappable part of local memory
+
+ @Return : IMG_TRUE if the device has non-mappable part of local memory
+******************************************************************************/
+IMG_BOOL PVRSRVSystemHasNonMappableLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemWaitCycles
+
+ @Description	: Waits for at least ui32Cycles of the Device clk.
+******************************************************************************/
+void PVRSRVSystemWaitCycles(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT32 ui32Cycles);
+
+PVRSRV_ERROR PVRSRVSystemInstallDeviceLISR(void *pvOSDevice,
+										   IMG_UINT32 ui32IRQ,
+										   const IMG_CHAR *pszName,
+										   PFN_LISR pfnLISR,
+										   void *pvData,
+										   IMG_HANDLE *phLISRData);
+
+PVRSRV_ERROR PVRSRVSystemUninstallDeviceLISR(IMG_HANDLE hLISRData);
+
+int PVRSRVGetDriverStatus(void);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVIsBridgeEnabled
+
+ @Description	: Returns whether the given bridge group is enabled
+
+ @Return : IMG_TRUE if the given bridge group is enabled
+******************************************************************************/
+static inline IMG_BOOL PVRSRVIsBridgeEnabled(IMG_HANDLE hServices, IMG_UINT32 ui32BridgeGroup)
+{
+	IMG_UINT32 ui32Bridges;
+	IMG_UINT32 ui32Offset;
+
+	PVR_UNREFERENCED_PARAMETER(hServices);
+
+#if defined(SUPPORT_RGX)
+	if (ui32BridgeGroup >= PVRSRV_BRIDGE_RGX_FIRST)
+	{
+		ui32Bridges = gui32RGXBridges;
+		ui32Offset = PVRSRV_BRIDGE_RGX_FIRST;
+	}
+	else
+#endif /* SUPPORT_RGX */
+	{
+		ui32Bridges = gui32PVRBridges;
+		ui32Offset = PVRSRV_BRIDGE_FIRST;
+	}
+
+	return (IMG_BOOL)(((1U << (ui32BridgeGroup - ui32Offset)) & ui32Bridges) != 0);
+}
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	void SetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState);
+	void SetTrustedDeviceAceEnabled(IMG_HANDLE hSysData);
+#endif
+#endif
+
+/*!
+******************************************************************************
+ @Function			: PVRSRVCreateHWPerfHostThread
+
+ @Description		: Creates HWPerf event object and thread unless already created
+
+ @Input ui32Timeout	: Initial timeout (ms) between updates on the HWPerf thread
+
+ @Return			: PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+******************************************************************************/
+PVRSRV_ERROR PVRSRVCreateHWPerfHostThread(IMG_UINT32 ui32Timeout);
+
+/*!
+******************************************************************************
+ @Function			: PVRSRVDestroyHWPerfHostThread
+
+ @Description		: Destroys HWPerf event object and thread if created
+
+ @Return			: PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHWPerfHostThread(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVFindPhysHeapConfig
+@Description    Find Phys Heap Config from Device Config.
+@Input          psDevConfig  Pointer to device config.
+@Input          ui32Flags    Find heap that matches flags.
+@Return         PHYS_HEAP_CONFIG*  Return a config, or NULL if not found.
+*/ /**************************************************************************/
+PHYS_HEAP_CONFIG* PVRSRVFindPhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										   PHYS_HEAP_USAGE_FLAGS ui32Flags);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDeviceInstance
+@Description    Return the specified device instance from Device node list.
+@Input          ui32Instance         Device instance to find
+@Return         PVRSRV_DEVICE_NODE*  Return a device node, or NULL if not found.
+*/ /**************************************************************************/
+PVRSRV_DEVICE_NODE* PVRSRVGetDeviceInstance(IMG_UINT32 ui32Instance);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDeviceInstanceByKernelDevID
+@Description    Return the specified device instance by OS Id.
+@Input          i32OSInstance        OS device Id to find
+@Return         PVRSRV_DEVICE_NODE*  Return a device node, or NULL if not found.
+*/ /**************************************************************************/
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstanceByKernelDevID(IMG_INT32 i32OSInstance);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVAcquireInternalID
+@Description    Returns the lowest free device ID.
+@Output         pui32InternalID  The device ID
+@Return         PVRSRV_ERROR     PVRSRV_OK or an error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVAcquireInternalID(IMG_UINT32 *pui32InternalID);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDeviceFreeze
+@Description    Stops further processing from occurring on the device after the
+                 current work ends and blocks user-mode tasks on the specified
+                 device. Device node is put into POWER-DOWN state.
+@Input          PVRSRV_DEVICE_NODE*     Device node
+@Return         PVRSRV_ERROR            PVRSRV_OK on success otherwise a
+                                        PVRSRV_ERR_ code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceFreeze(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDeviceThaw
+@Description    Unfreezes a previously frozen device. Restarts work on the
+                 device, if work queues are non-empty, and unblocks any
+                 user-mode tasks on the specified device.
+@Input          PVRSRV_DEVICE_NODE*     Device node
+@Return         PVRSRV_ERROR            PVRSRV_OK on success otherwise a
+                                        PVRSRV_ERR_ code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceThaw(PVRSRV_DEVICE_NODE *psDevNode);
+
+#endif /* PVRSRV_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_apphint.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_apphint.h
new file mode 100644
index 000000000000..e35426680795
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_apphint.h
@@ -0,0 +1,71 @@
+/**************************************************************************/ /*!
+@File
+@Title          PowerVR AppHint generic interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PVRSRV_APPHINT_H)
+#define PVRSRV_APPHINT_H
+
+/* Supplied to PVRSRVAppHintRegisterHandlers*() functions when the apphint
+ * is a global driver apphint, i.e. apphints not present in
+ * APPHINT_DEBUGFS_DEVICE_ID, i.e. not per device.
+ */
+#define APPHINT_OF_DRIVER_NO_DEVICE ((void*)-1U)
+
+#if defined(__linux__)
+
+#include "km_apphint.h"
+#define PVRSRVAppHintDumpState(d) pvr_apphint_dump_state(d)
+#define PVRSRVAppHintRegisterHandlersUINT64(i,q,s,d,p) pvr_apphint_register_handlers_uint64(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersUINT32(i,q,s,d,p) pvr_apphint_register_handlers_uint32(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersBOOL(i,q,s,d,p) pvr_apphint_register_handlers_bool(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersSTRING(i,q,s,d,p) pvr_apphint_register_handlers_string(i,q,s,d,p)
+
+#else
+
+#define PVRSRVAppHintDumpState(d)
+#define PVRSRVAppHintRegisterHandlersUINT64(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersUINT32(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersBOOL(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersSTRING(i,q,s,d,p)
+
+#endif
+
+#endif /* PVRSRV_APPHINT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_bridge_init.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_bridge_init.h
new file mode 100644
index 000000000000..b225da6d368e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_bridge_init.h
@@ -0,0 +1,53 @@
+/**************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the common PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_BRIDGE_INIT_H
+#define PVRSRV_BRIDGE_INIT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR ServerBridgeInit(void);
+void ServerBridgeDeInit(void);
+
+#endif /* PVRSRV_BRIDGE_INIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_cleanup.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_cleanup.h
new file mode 100644
index 000000000000..a197ca9ec908
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_cleanup.h
@@ -0,0 +1,256 @@
+/**************************************************************************/ /*!
+@File
+@Title          PowerVR SrvKM cleanup thread deferred work interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_CLEANUP_H
+#define PVRSRV_CLEANUP_H
+
+#include "dllist.h"
+#include "device.h"
+
+/**************************************************************************/ /*!
+@Brief          CLEANUP_THREAD_FN
+
+@Description    This is the function prototype for the pfnFree member found in
+                the structure PVRSRV_CLEANUP_THREAD_WORK. The function is
+                responsible for carrying out the clean up work and if successful
+                freeing the memory originally supplied to the call
+                PVRSRVCleanupThreadAddWork().
+
+@Input          pvParam  This is private data originally supplied by the caller
+                         to PVRSRVCleanupThreadAddWork() when registering the
+                         clean up work item, psDAta->pvData. Itr can be cast
+                         to a relevant type within the using module.
+
+@Return         PVRSRV_OK if the cleanup operation was successful and the
+                callback has freed the PVRSRV_CLEANUP_THREAD_WORK* work item
+                memory original supplied to PVRSRVCleanupThreadAddWork()
+                Any other error code will lead to the work item
+                being re-queued and hence the original
+                PVRSRV_CLEANUP_THREAD_WORK* must not be freed.
+*/ /***************************************************************************/
+
+typedef PVRSRV_ERROR (*CLEANUP_THREAD_FN)(void *pvParam);
+
+
+/* Typical number of times a caller should want the work to be retried in case
+ * of the callback function (pfnFree) returning an error.
+ * Callers to PVRSRVCleanupThreadAddWork should provide this value as the retry
+ * count (ui32RetryCount) unless there are special requirements.
+ * A value of 200 corresponds to around ~20s (200 * 100ms). If it is not
+ * successful by then give up as an unrecoverable problem has occurred.
+ */
+#define CLEANUP_THREAD_RETRY_COUNT_DEFAULT 200u
+/* Like for CLEANUP_THREAD_RETRY_COUNT_DEFAULT but call will wait for
+ * a specified amount of time rather than number of retries.
+ */
+#define CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT 20000u /* 20s */
+
+/* Use to set retry count on a cleanup item.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * _count - retry count
+ */
+#define CLEANUP_THREAD_SET_RETRY_COUNT(_item,_count) \
+	do { \
+		(_item)->ui32RetryCount = (_count); \
+		(_item)->ui32TimeStart = 0; \
+		(_item)->ui32TimeEnd = 0; \
+	} while (0)
+
+/* Use to set timeout deadline on a cleanup item.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * _timeout - timeout in milliseconds, if 0
+ *            CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT is used
+ */
+#define CLEANUP_THREAD_SET_RETRY_TIMEOUT(_item,_timeout) \
+	do { \
+		(_item)->ui32RetryCount = 0; \
+		(_item)->ui32TimeStart = OSClockms(); \
+		(_item)->ui32TimeEnd = (_item)->ui32TimeStart + ((_timeout) > 0 ? \
+				(_timeout) : CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT); \
+	} while (0)
+
+/* Indicates if the timeout on a given item has been reached.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ */
+#define CLEANUP_THREAD_RETRY_TIMEOUT_NOT_REACHED(_item) \
+	((_item)->ui32TimeEnd - (_item)->ui32TimeStart >= \
+			OSClockms() - (_item)->ui32TimeStart)
+
+/* Indicates if the current item is waiting on timeout or retry count.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * */
+#define CLEANUP_THREAD_IS_RETRY_TIMEOUT(_item) \
+	((_item)->ui32TimeStart != (_item->ui32TimeEnd))
+
+#define CLEANUP_TYPE_LIST \
+	X(UNDEF)      /**/ \
+	X(CONNECTION)     /**/ \
+	X(MMU)     /**/ \
+	X(OSMEM)   /**/ \
+	X(PMR)       /**/ \
+	X(LAST)      /**/ \
+
+#define CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE 11     /* CONNECTION\0 */
+#define CLEANUP_TYPE_ITEM_DPF " %1.11s : %1.5d"
+#define CLEANUP_TYPE_ITEM_DPF_MAX_SIZE CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE+sizeof(" : ")+5+1
+
+typedef enum _PVRSRV_CLEANUP_TYPE_
+{
+#define X(_name) PVRSRV_CLEANUP_TYPE_ ## _name,
+	CLEANUP_TYPE_LIST
+#undef X
+
+} PVRSRV_CLEANUP_TYPE;
+
+#if defined(CLEANUP_TYPE_STRINGS)
+
+static const char *const _pszCleanupStrings[] = {
+#define X(_name) #_name,
+	CLEANUP_TYPE_LIST
+#undef X
+};
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetCleanupName
+@Description    Returns the name of a Cleanup Type.
+
+@Input          eCleanupType   The enum value of the cleanup type.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetCleanupName(PVRSRV_CLEANUP_TYPE eCleanupType)
+{
+	if (eCleanupType < 0 || eCleanupType > PVRSRV_CLEANUP_TYPE_LAST)
+	{
+		return "Undefined";
+	}
+
+	PVR_ASSERT(sizeof(_pszCleanupStrings[eCleanupType]) < CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE);
+
+	return _pszCleanupStrings[eCleanupType];
+}
+
+#endif /* CLEANUP_TYPE_STRINGS */
+
+/* Clean up work item specifics so that the task can be managed by the
+* pvr_defer_free cleanup thread in the Server.
+*/
+typedef struct _PVRSRV_CLEANUP_THREAD_WORK_
+{
+	DLLIST_NODE sNode;               /*!< List node used internally by the cleanup
+	                                    thread */
+	CLEANUP_THREAD_FN pfnFree;       /*!< Pointer to the function to be called to
+	                                    carry out the deferred cleanup */
+	void *pvData;                    /*!< private data for pfnFree, usually a way back
+	                                    to the original PVRSRV_CLEANUP_THREAD_WORK*
+	                                    pointer supplied in the call to
+	                                    PVRSRVCleanupThreadAddWork(). */
+	IMG_UINT32 ui32TimeStart;        /*!< Timestamp in ms of the moment when
+	                                    cleanup item has been created. */
+	IMG_UINT32 ui32TimeEnd;          /*!< Time in ms after which no further retry
+	                                    attempts will be made, item discard and
+	                                    error logged when this is reached. */
+	IMG_UINT32 ui32RetryCount;       /*!< Number of times the callback should be
+	                                    re-tried when it returns error. */
+	IMG_BOOL bDependsOnHW;           /*!< Don't drop the cleanup task if retry limit
+	                                      is reached, we could depend on event from
+	                                      device to continue. */
+	PVRSRV_CLEANUP_TYPE eCleanupType;/*!< Type of work item added to queue */
+#if defined(DEBUG)
+	const char *pszFun;
+	unsigned int ui32LineNum;
+#endif
+} PVRSRV_CLEANUP_THREAD_WORK;
+
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadAddWork
+
+@Description    Add a work item to be called from the cleanup thread
+
+@Input          psDevNode : Pointer to the device node
+@Input          psData :    The function pointer and private data for the
+                            callback
+
+@Return         None
+*/ /***************************************************************************/
+#if defined(DEBUG)
+#define PVRSRVCleanupThreadAddWork(DEV, DATA)		PVRSRVCleanupThreadAddWork_Debug(DEV, DATA, __FILE__, __LINE__)
+void PVRSRVCleanupThreadAddWork_Debug(PVRSRV_DEVICE_NODE *psDevNode,
+                                      PVRSRV_CLEANUP_THREAD_WORK *psData,
+                                      const char *pszFun, const unsigned int ui32LineNum);
+#else
+#define PVRSRVCleanupThreadAddWork					PVRSRVCleanupThreadAddWork_Int
+void PVRSRVCleanupThreadAddWork_Int(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_CLEANUP_THREAD_WORK *psData);
+#endif
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadWaitForDevice
+
+@Description    Blocking wait for all of the device's items to be cleaned.
+
+@Input          psDevNode : Pointer to the device node
+
+@Return         None
+*/ /***************************************************************************/
+void PVRSRVCleanupThreadWaitForDevice(PVRSRV_DEVICE_NODE *psDevNode);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadGetPid
+
+@Description    Returns Cleanup Thread's PID.
+
+@Return         PID of the Cleanup Thread
+*/ /***************************************************************************/
+IMG_PID PVRSRVCleanupThreadGetPid(void);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadGetTid
+
+@Description    Returns Cleanup Thread's TID.
+
+@Return         TID of the Cleanup Thread
+*/ /***************************************************************************/
+uintptr_t PVRSRVCleanupThreadGetTid(void);
+
+#endif /* PVRSRV_CLEANUP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_device.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_device.h
new file mode 100644
index 000000000000..6e484a2c625b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_device.h
@@ -0,0 +1,434 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_DEVICE_H
+#define PVRSRV_DEVICE_H
+
+#include "img_types.h"
+#include "physheap_config.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "rgx_firmware_boot.h"
+#include "rgx_fwif_km.h"
+#include "servicesext.h"
+#include "cache_ops.h"
+#include "opaque_types.h"
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+#include "pvr_dvfs.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+typedef struct _PVRSRV_DEVICE_CONFIG_ PVRSRV_DEVICE_CONFIG;
+typedef enum _DRIVER_MODE_
+{
+/* Do not use these enumerations directly, to query the
+   current driver mode, use the PVRSRV_VZ_MODE_IS()
+   macro */
+	DRIVER_MODE_NATIVE	= 0,
+	DRIVER_MODE_HOST	= 1,
+	DRIVER_MODE_GUEST	= 2,
+	DRIVER_MODE_DEFAULT	= 3,
+} PVRSRV_DRIVER_MODE;
+
+typedef enum
+{
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_MAPPABLE = 0,
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_NON_MAPPABLE = 1,
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_LAST
+} PVRSRV_DEVICE_LOCAL_MEMORY_ARENA;
+
+typedef enum _PVRSRV_DEVICE_SNOOP_MODE_
+{
+	PVRSRV_DEVICE_SNOOP_NONE = 0,
+	PVRSRV_DEVICE_SNOOP_CPU_ONLY,
+	PVRSRV_DEVICE_SNOOP_DEVICE_ONLY,
+	PVRSRV_DEVICE_SNOOP_CROSS,
+	PVRSRV_DEVICE_SNOOP_EMULATED,
+} PVRSRV_DEVICE_SNOOP_MODE;
+
+#if defined(SUPPORT_SOC_TIMER)
+typedef IMG_UINT64
+(*PFN_SYS_DEV_SOC_TIMER_READ)(IMG_HANDLE hSysData);
+#endif
+
+typedef enum _PVRSRV_DEVICE_FABRIC_TYPE_
+{
+	PVRSRV_DEVICE_FABRIC_NONE = 0,
+	PVRSRV_DEVICE_FABRIC_ACELITE,
+	PVRSRV_DEVICE_FABRIC_FULLACE,
+} PVRSRV_DEVICE_FABRIC_TYPE;
+
+typedef IMG_UINT32
+(*PFN_SYS_DEV_CLK_FREQ_GET)(IMG_HANDLE hSysData);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_PRE_POWER)(IMG_HANDLE hSysData,
+						 PVRSRV_SYS_POWER_STATE eNewPowerState,
+						 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+						 PVRSRV_POWER_FLAGS ePwrFlags);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_POST_POWER)(IMG_HANDLE hSysData,
+						  PVRSRV_SYS_POWER_STATE eNewPowerState,
+						  PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+						  PVRSRV_POWER_FLAGS ePwrFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_SYS_GET_POWER
+
+@Description    This function queries the SoC power registers to determine
+                if the power domain on which the GPU resides is powered on.
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will assume the domain power state depending on driver type:
+   regular drivers assume it is unpowered at startup, while drivers with
+   AutoVz support expect the GPU domain to be powered on initially. The power
+   state will be then tracked internally according to the pfnPrePowerState
+   and pfnPostPowerState calls using a fallback function.
+
+@Input          psDevNode                  Pointer to node struct of the
+                                           device being initialised
+
+@Return         PVRSRV_SYS_POWER_STATE_ON  if the respective device's hardware
+                                           domain is powered on
+                PVRSRV_SYS_POWER_STATE_OFF if the domain is powered off
+*/ /**************************************************************************/
+typedef PVRSRV_SYS_POWER_STATE
+(*PFN_SYS_GET_POWER)(PPVRSRV_DEVICE_NODE psDevNode);
+
+typedef void
+(*PFN_SYS_DEV_INTERRUPT_HANDLED)(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_DEV_CHECK_MEM_ALLOC_SIZE)(IMG_HANDLE hSysData,
+									IMG_UINT64 ui64MemSize);
+
+typedef void (*PFN_SYS_DEV_FEAT_DEP_INIT)(PVRSRV_DEVICE_CONFIG *, IMG_UINT64);
+
+typedef void
+(*PFN_SYS_DEV_HOST_CACHE_MAINTENANCE)(IMG_HANDLE hSysData,
+									PVRSRV_CACHE_OP eRequestType,
+									void *pvVirtStart,
+									void *pvVirtEnd,
+									IMG_CPU_PHYADDR sCPUPhysStart,
+									IMG_CPU_PHYADDR sCPUPhysEnd);
+
+typedef void*
+(*PFN_SLAVE_DMA_CHAN)(PVRSRV_DEVICE_CONFIG*, char*);
+
+typedef void
+(*PFN_SLAVE_DMA_FREE)(PVRSRV_DEVICE_CONFIG*,
+					  void*);
+
+typedef void
+(*PFN_DEV_PHY_ADDR_2_DMA_ADDR)(PVRSRV_DEVICE_CONFIG *,
+							   IMG_DMA_ADDR *,
+							   IMG_DEV_PHYADDR *,
+							   IMG_BOOL *,
+							   IMG_UINT32,
+							   IMG_BOOL);
+
+
+typedef struct _PVRSRV_FW_PARAMS_
+{
+	const void *pvFirmware;
+	IMG_UINT32 ui32FirmwareSize;
+	const void *pvSignature;
+	IMG_UINT32 ui32SignatureSize;
+	PVRSRV_FW_BOOT_PARAMS uFWP;
+} PVRSRV_FW_PARAMS;
+
+typedef PVRSRV_ERROR
+(*PFN_PREPARE_FW_IMAGE)(IMG_HANDLE hSysData,
+						PVRSRV_FW_PARAMS *psFWParams);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+
+typedef PVRSRV_ERROR
+(*PFN_TD_SEND_FW_IMAGE)(IMG_HANDLE hSysData,
+						PVRSRV_FW_PARAMS *psTDFWParams);
+
+typedef struct _PVRSRV_TD_POWER_PARAMS_
+{
+	IMG_DEV_PHYADDR sPCAddr;
+
+	/* MIPS-only fields */
+	IMG_DEV_PHYADDR sGPURegAddr;
+	IMG_DEV_PHYADDR sBootRemapAddr;
+	IMG_DEV_PHYADDR sCodeRemapAddr;
+	IMG_DEV_PHYADDR sDataRemapAddr;
+} PVRSRV_TD_POWER_PARAMS;
+
+typedef PVRSRV_ERROR
+(*PFN_TD_SET_POWER_PARAMS)(IMG_HANDLE hSysData,
+						   PVRSRV_TD_POWER_PARAMS *psTDPowerParams);
+
+typedef PVRSRV_ERROR
+(*PFN_TD_RGXSTART)(IMG_HANDLE hSysData);
+
+typedef PVRSRV_ERROR
+(*PFN_TD_RGXSTOP)(IMG_HANDLE hSysData);
+
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef void (*PFN_SYS_INIT_FIREWALL)(IMG_HANDLE hSysData,
+                                      IMG_UINT64[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                                      IMG_UINT64[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG_
+{
+	IMG_UINT32 ui32Status;     /*!< FW status */
+	IMG_UINT32 ui32Reason;     /*!< Reason for FW status */
+} PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG;
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF_
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;     /*!< FW page fault address */
+} PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF;
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM_
+{
+	IMG_UINT32 ui32ExtJobRef;     /*!< External Job Reference of any affected GPU work */
+	RGXFWIF_DM eDM;               /*!< Data Master which was running any affected GPU work */
+} PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM;
+
+typedef struct _PVRSRV_ROBUSTNESS_NOTIFY_DATA_
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reason for error/reset */
+	IMG_PID                  pid;          /*!< Pid of process which created the errored context */
+	union
+	{
+		PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM sChecksumErrData; /*!< Data returned for checksum errors */
+		PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF    sFwPFErrData;     /*!< Data returned for FW page faults */
+		PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG sHostWdgData;     /*!< Data returned for Host Wdg FW faults */
+	} uErrData;
+} PVRSRV_ROBUSTNESS_NOTIFY_DATA;
+
+typedef void
+(*PFN_SYS_DEV_ERROR_NOTIFY)(IMG_HANDLE hSysData,
+						    PVRSRV_ROBUSTNESS_NOTIFY_DATA *psRobustnessErrorData);
+
+struct _PVRSRV_DEVICE_CONFIG_
+{
+	/*! OS device passed to SysDevInit (linux: 'struct device') */
+	void *pvOSDevice;
+
+	/*!
+	 *! Service representation of pvOSDevice. Should be set to NULL when the
+	 *! config is created in SysDevInit. Set by Services once a device node has
+	 *! been created for this config and unset before SysDevDeInit is called.
+	 */
+	struct _PVRSRV_DEVICE_NODE_ *psDevNode;
+
+	/*! Name of the device */
+	IMG_CHAR *pszName;
+
+	/*! Version of the device (optional) */
+	IMG_CHAR *pszVersion;
+
+	/*! Register bank address */
+	IMG_CPU_PHYADDR sRegsCpuPBase;
+	/*! Register bank size */
+	IMG_UINT32 ui32RegsSize;
+	/*! Device interrupt number */
+	IMG_UINT32 ui32IRQ;
+
+	PVRSRV_DEVICE_SNOOP_MODE eCacheSnoopingMode;
+
+	/*! Device specific data handle */
+	IMG_HANDLE hDevData;
+
+	/*! System specific data that gets passed into system callback functions. */
+	IMG_HANDLE hSysData;
+
+	IMG_BOOL bHasNonMappableLocalMemory;
+
+	/*! Indicates if system supports FBCDC v3.1 */
+	IMG_BOOL bHasFBCDCVersion31;
+
+	/*! Physical Heap definitions for this device.
+	 * eDefaultHeap must be set to GPU_LOCAL or CPU_LOCAL. Specifying any other value
+	 *    (e.g. DEFAULT) will lead to an error at device discovery.
+	 * pasPhysHeap array must contain at least one PhysHeap, the declared default heap.
+	 */
+	PVRSRV_PHYS_HEAP  eDefaultHeap;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32PhysHeapCount;
+
+	/*!
+	 *! Callbacks to change system device power state at the beginning and end
+	 *! of a power state change (optional).
+	 */
+	PFN_SYS_PRE_POWER pfnPrePowerState;
+	PFN_SYS_POST_POWER pfnPostPowerState;
+	PFN_SYS_GET_POWER  pfnGpuDomainPower;
+
+	/*! Callback to obtain the clock frequency from the device (optional). */
+	PFN_SYS_DEV_CLK_FREQ_GET pfnClockFreqGet;
+
+#if defined(SUPPORT_SOC_TIMER)
+	/*! Callback to read SoC timer register value (mandatory). */
+	PFN_SYS_DEV_SOC_TIMER_READ	pfnSoCTimerRead;
+#endif
+
+	/*!
+	 *! Callback to perform host CPU cache maintenance. Might be needed for
+	 *! architectures which allow extensions such as RISC-V (optional).
+	 */
+	PFN_SYS_DEV_HOST_CACHE_MAINTENANCE pfnHostCacheMaintenance;
+	IMG_BOOL bHasPhysicalCacheMaintenance;
+
+	/*!
+	 *! Callback to prepare FW image after it has been loaded.  This may
+	 *! be used to separate a signature/header from the firmware proper,
+	 *! potentially modifying pvFirmware and ui32FirmwareSize to point to the
+	 *! actual firmware to be loaded.
+	 */
+	PFN_PREPARE_FW_IMAGE pfnPrepareFWImage;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/*!
+	 *! Callback to send FW image and FW boot time parameters to the trusted
+	 *! device.
+	 */
+	PFN_TD_SEND_FW_IMAGE pfnTDSendFWImage;
+
+	/*!
+	 *! Callback to send parameters needed in a power transition to the trusted
+	 *! device.
+	 */
+	PFN_TD_SET_POWER_PARAMS pfnTDSetPowerParams;
+
+	/*! Callbacks to ping the trusted device to securely run RGXStart/Stop() */
+	PFN_TD_RGXSTART pfnTDRGXStart;
+	PFN_TD_RGXSTOP pfnTDRGXStop;
+
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+	/*! Name of DMA heap to allocate secure memory from. Used with dma_heap_find. */
+	IMG_CHAR *pszSecureDMAHeapName;
+#endif
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+	/*! Function that does device feature specific system layer initialisation */
+	PFN_SYS_DEV_FEAT_DEP_INIT	pfnSysDevFeatureDepInit;
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	PVRSRV_DVFS sDVFS;
+#endif
+
+#if defined(SUPPORT_ALT_REGBASE)
+	IMG_DEV_PHYADDR sAltRegsGpuPBase;
+#endif
+
+	/*!
+	 *! Indicates if device physical address 0x0 might be used as GPU memory
+	 *! (e.g. LMA system or UMA system with CPU PA 0x0 reserved by the OS,
+	 *!  but CPU PA != device PA and device PA 0x0 available for the GPU)
+	 */
+	IMG_BOOL bDevicePA0IsValid;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/*!
+	 *! System-specific function to initialise firewall mechanism needed to
+	 *! validate the correctness of memory accesses tagged with OSIDs.
+	 *! Used for hardware validation of virtualization features only.
+	 */
+	PFN_SYS_INIT_FIREWALL		pfnSysInitFirewall;
+#endif
+
+	/*!
+	 *! Callback to notify system layer of device errors.
+	 *! NB. implementers should ensure that the minimal amount of work is
+	 *! done in the callback function, as it will be executed in the main
+	 *! RGX MISR. (e.g. any blocking or lengthy work should be performed by
+	 *! a worker queue/thread instead.)
+	 */
+	PFN_SYS_DEV_ERROR_NOTIFY	pfnSysDevErrorNotify;
+
+	/*!
+	 *!  Slave DMA channel request callbacks
+	 */
+	PFN_SLAVE_DMA_CHAN pfnSlaveDMAGetChan;
+	PFN_SLAVE_DMA_FREE pfnSlaveDMAFreeChan;
+	/*!
+	 *!  Conversion of device memory to DMA addresses
+	 */
+	PFN_DEV_PHY_ADDR_2_DMA_ADDR pfnDevPhysAddr2DmaAddr;
+	/*!
+	 *!  DMA channel names
+	 */
+	IMG_CHAR *pszDmaTxChanName;
+	IMG_CHAR *pszDmaRxChanName;
+	/*!
+	 *!  DMA device transfer restrictions
+	 */
+	IMG_UINT32 ui32DmaAlignment;
+	IMG_UINT32 ui32DmaTransferUnit;
+	/*!
+	 *!  System-wide presence of DMA capabilities
+	 */
+	IMG_BOOL bHasDma;
+
+	/*!
+	 *!  ES770x clk&reset required
+	 */
+    struct clk *aclk;
+    struct clk *gray_clk;
+    struct clk *cfg_clk;
+    struct reset_control    *rstc_axi;
+    struct reset_control    *rstc_cfg;
+    struct reset_control    *rstc_gray;
+    struct reset_control    *rstc_jones;
+    struct reset_control    *rstc_spu;
+
+	/*!
+	 *!  DriverMode required
+	 */
+	PVRSRV_DRIVER_MODE eDriverMode;
+};
+
+#endif /* PVRSRV_DEVICE_H*/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_pool.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_pool.h
new file mode 100644
index 000000000000..2272fc50ce6e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_pool.h
@@ -0,0 +1,135 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services pool implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a generic pool implementation.
+                The pool allows to dynamically retrieve and return entries from
+                it using functions pair PVRSRVPoolGet/PVRSRVPoolPut. The entries
+                are created in lazy manner which means not until first usage.
+                The pool API allows to pass and allocation/free functions
+                pair that will allocate entry's private data and return it
+                to the caller on every entry 'Get'.
+                The pool will keep up to ui32MaxEntries entries allocated.
+                Every entry that exceeds this number and is 'Put' back to the
+                pool will be freed on the spot instead being returned to the
+                pool.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PVRSRVPOOL_H)
+#define PVRSRVPOOL_H
+
+/**************************************************************************/ /*!
+ @Description  Callback function called during creation of the new element. This
+               function allocates an object that will be stored in the pool.
+               The object can be retrieved from the pool by calling
+               PVRSRVPoolGet.
+ @Input        pvPrivData      Private data passed to the alloc function.
+ @Output       pvOut           Allocated object.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+typedef PVRSRV_ERROR (PVRSRV_POOL_ALLOC_FUNC)(void *pvPrivData, void **pvOut);
+
+/**************************************************************************/ /*!
+ @Description  Callback function called to free the object allocated by
+               the counterpart alloc function.
+ @Input        pvPrivData      Private data passed to the free function.
+ @Output       pvFreeData      Object allocated by PVRSRV_POOL_ALLOC_FUNC.
+*/ /***************************************************************************/
+typedef void (PVRSRV_POOL_FREE_FUNC)(void *pvPrivData, void *pvFreeData);
+
+typedef IMG_HANDLE PVRSRV_POOL_TOKEN;
+
+typedef struct _PVRSRV_POOL_ PVRSRV_POOL;
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolCreate
+ @Description  Creates new buffer pool.
+ @Input        pfnAlloc        Allocation function pointer. Function is used
+                               to allocate new pool entries' data.
+ @Input        pfnFree         Free function pointer. Function is used to
+                               free memory allocated by pfnAlloc function.
+ @Input        ui32MaxEntries  Total maximum number of entries in the pool.
+ @Input        pszName         Name of the pool. String has to be NULL
+                               terminated.
+ @Input        pvPrivData      Private data that will be passed to pfnAlloc and
+                               pfnFree functions.
+ @Output       ppsPool         New buffer pool object.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolCreate(PVRSRV_POOL_ALLOC_FUNC *pfnAlloc,
+					PVRSRV_POOL_FREE_FUNC *pfnFree,
+					IMG_UINT32 ui32MaxEntries,
+					const IMG_CHAR *pszName,
+					void *pvPrivData,
+					PVRSRV_POOL **ppsPool);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolDestroy
+ @Description  Destroys pool created by PVRSRVPoolCreate.
+ @Input        psPool          Buffer pool object meant to be destroyed.
+*/ /***************************************************************************/
+void PVRSRVPoolDestroy(PVRSRV_POOL *psPool);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolGet
+ @Description  Retrieves an entry from a pool. If no free elements are
+               available new entry will be allocated.
+ @Input        psPool          Pointer to the pool.
+ @Output       hToken          Pointer to the entry handle.
+ @Output       ppvDataOut      Pointer to data stored in the entry (the data
+                               allocated by the pfnAlloc function).
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolGet(PVRSRV_POOL *psPool,
+						PVRSRV_POOL_TOKEN *hToken,
+						void **ppvDataOut);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolPut
+ @Description  Returns entry to the pool. If number of entries is greater
+               than ui32MaxEntries set during pool creation the entry will
+               be freed instead.
+ @Input        psPool          Pointer to the pool.
+ @Input        hToken          Entry handle.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolPut(PVRSRV_POOL *psPool,
+						PVRSRV_POOL_TOKEN hToken);
+
+#endif /* PVRSRVPOOL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_sync_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_sync_server.h
new file mode 100644
index 000000000000..5d1a10cd0a9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_sync_server.h
@@ -0,0 +1,278 @@
+/**************************************************************************/ /*!
+@File
+@Title          Fence sync server interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_SYNC_SERVER_H
+#define PVRSRV_SYNC_SERVER_H
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#include "sync_fallback_server.h"
+#include "pvr_notifier.h"
+#include "img_types.h"
+#include "pvrsrv_sync_km.h"
+#elif defined(SUPPORT_NATIVE_FENCE_SYNC)
+#include "pvr_sync.h"
+#endif
+
+#include "rgxhwperf.h"
+
+#define SYNC_SW_TIMELINE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+#define SYNC_SW_FENCE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+
+typedef struct _SYNC_TIMELINE_OBJ_
+{
+	void *pvTlObj; /* Implementation specific timeline object */
+
+	PVRSRV_TIMELINE hTimeline; /* Reference to implementation-independent timeline object */
+} SYNC_TIMELINE_OBJ;
+
+typedef struct _SYNC_FENCE_OBJ_
+{
+	void *pvFenceObj; /* Implementation specific fence object */
+
+	PVRSRV_FENCE hFence; /* Reference to implementation-independent fence object */
+} SYNC_FENCE_OBJ;
+
+static inline void SyncClearTimelineObj(SYNC_TIMELINE_OBJ *psSTO)
+{
+	psSTO->pvTlObj = NULL;
+	psSTO->hTimeline = PVRSRV_NO_TIMELINE;
+}
+
+static inline IMG_BOOL SyncIsTimelineObjValid(const SYNC_TIMELINE_OBJ *psSTO)
+{
+	return (IMG_BOOL)(psSTO->pvTlObj != NULL);
+}
+
+static inline void SyncClearFenceObj(SYNC_FENCE_OBJ *psSFO)
+{
+	psSFO->pvFenceObj = NULL;
+	psSFO->hFence = PVRSRV_NO_FENCE;
+}
+
+static inline IMG_BOOL SyncIsFenceObjValid(const SYNC_FENCE_OBJ *psSFO)
+{
+	return (IMG_BOOL)(psSFO->pvFenceObj != NULL);
+}
+
+
+/* Mapping of each required function to its appropriate sync-implementation function */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	#define SyncFenceWaitKM_                SyncFbFenceWait
+	#define SyncGetFenceObj_                SyncFbGetFenceObj
+	#define SyncFenceReleaseKM_             SyncFbFenceReleaseKM
+	#define SyncSWTimelineFenceCreateKM_    SyncFbSWTimelineFenceCreateKM
+	#define SyncSWTimelineAdvanceKM_        SyncFbSWTimelineAdvanceKM
+	#define SyncSWGetTimelineObj_           SyncFbSWGetTimelineObj
+	#define SyncSWTimelineReleaseKM_        SyncFbTimelineRelease
+	#define SyncDumpFence_                  SyncFbDumpFenceKM
+	#define SyncSWDumpTimeline_             SyncFbSWDumpTimelineKM
+#elif defined(SUPPORT_NATIVE_FENCE_SYNC)
+	#define SyncFenceWaitKM_                pvr_sync_fence_wait
+	#define SyncGetFenceObj_                pvr_sync_fence_get
+	#define SyncFenceReleaseKM_             pvr_sync_fence_release
+	#define SyncSWTimelineFenceCreateKM_    pvr_sync_sw_timeline_fence_create
+	#define SyncSWTimelineAdvanceKM_        pvr_sync_sw_timeline_advance
+	#define SyncSWGetTimelineObj_           pvr_sync_sw_timeline_get
+	#define SyncSWTimelineReleaseKM_        pvr_sync_sw_timeline_release
+	#define SyncDumpFence_                  sync_dump_fence
+	#define SyncSWDumpTimeline_             sync_sw_dump_timeline
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncFenceWaitKM
+
+@Description    Wait for all the sync points in the fence to be signalled.
+
+@Input          psFenceObj          Fence to wait on
+
+@Input          ui32TimeoutInMs     Maximum time to wait (in milliseconds)
+
+@Return         PVRSRV_OK               once the fence has been passed (all
+                                        containing check points have either
+                                        signalled or errored)
+                PVRSRV_ERROR_TIMEOUT    if the poll has exceeded the timeout
+                PVRSRV_ERROR_FAILED_DEPENDENCIES Other sync-impl specific error
+*/ /**************************************************************************/
+static inline PVRSRV_ERROR
+SyncFenceWaitKM(PVRSRV_DEVICE_NODE *psDevNode,
+                const SYNC_FENCE_OBJ *psFenceObj,
+                IMG_UINT32 ui32TimeoutInMs)
+{
+	PVRSRV_ERROR eError;
+
+	RGXSRV_HWPERF_SYNC_FENCE_WAIT(psDevNode->pvDevice,
+								  BEGIN,
+								  OSGetCurrentProcessID(),
+								  psFenceObj->hFence,
+								  ui32TimeoutInMs);
+
+	eError = SyncFenceWaitKM_(psFenceObj->pvFenceObj, ui32TimeoutInMs);
+
+	RGXSRV_HWPERF_SYNC_FENCE_WAIT(psDevNode->pvDevice,
+								  END,
+								  OSGetCurrentProcessID(),
+								  psFenceObj->hFence,
+								  ((eError == PVRSRV_OK) ?
+									  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_PASSED :
+									  ((eError == PVRSRV_ERROR_TIMEOUT) ?
+										  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_TIMEOUT :
+										  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_ERROR)));
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       SyncGetFenceObj
+
+@Description    Get the implementation specific server fence object from
+                opaque implementation independent PVRSRV_FENCE type.
+                When successful, this function gets a reference on the base
+                fence, which needs to be dropped using SyncFenceReleaseKM,
+                when fence object is no longer in use.
+
+@Input          iFence        Input opaque fence object
+
+@Output         psFenceObj    Pointer to implementation specific fence object
+
+@Return         PVRSRV_ERROR  PVRSRV_OK, on success
+*/ /**************************************************************************/
+static inline PVRSRV_ERROR
+SyncGetFenceObj(PVRSRV_FENCE iFence,
+                SYNC_FENCE_OBJ *psFenceObj)
+{
+	psFenceObj->hFence = iFence;
+	return SyncGetFenceObj_(iFence, &psFenceObj->pvFenceObj);
+}
+
+/*************************************************************************/ /*!
+@Function       SyncFenceReleaseKM
+
+@Description    Release reference on this fence.
+
+@Input          psFenceObj     Fence to be released
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static inline
+PVRSRV_ERROR SyncFenceReleaseKM(const SYNC_FENCE_OBJ *psFenceObj)
+{
+	return SyncFenceReleaseKM_(psFenceObj->pvFenceObj);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                      SW TIMELINE SPECIFIC FUNCTIONS                       */
+/*                                                                           */
+/*****************************************************************************/
+
+static inline PVRSRV_ERROR
+SyncSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            PVRSRV_TIMELINE hSWTimeline,
+                            const IMG_CHAR *pszFenceName,
+                            PVRSRV_FENCE *phOutFence)
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+	eError = SyncSWTimelineFenceCreateKM_(psDevNode,
+	                                      hSWTimeline,
+	                                      pszFenceName,
+	                                      phOutFence,
+	                                      &ui64SyncPtIdx);
+	if (eError == PVRSRV_OK)
+	{
+		RGXSRV_HWPERF_ALLOC_SW_FENCE(psDevNode, OSGetCurrentProcessID(),
+		                             *phOutFence, hSWTimeline, ui64SyncPtIdx,
+		                             pszFenceName, OSStringLength(pszFenceName));
+	}
+	return eError;
+}
+
+static inline PVRSRV_ERROR
+SyncSWTimelineAdvanceKM(PVRSRV_DEVICE_NODE *psDevNode,
+                        const SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+	eError = SyncSWTimelineAdvanceKM_(psSWTimelineObj->pvTlObj,
+	                                  &ui64SyncPtIdx);
+
+	if (eError == PVRSRV_OK)
+	{
+		RGXSRV_HWPERF_SYNC_SW_TL_ADV(psDevNode->pvDevice,
+		                             OSGetCurrentProcessID(),
+		                             psSWTimelineObj->hTimeline,
+		                             ui64SyncPtIdx);
+	}
+	return eError;
+}
+
+static inline PVRSRV_ERROR
+SyncSWGetTimelineObj(PVRSRV_TIMELINE hSWTimeline,
+                     SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	psSWTimelineObj->hTimeline = hSWTimeline;
+	return SyncSWGetTimelineObj_(hSWTimeline, &psSWTimelineObj->pvTlObj);
+}
+
+static inline PVRSRV_ERROR
+SyncSWTimelineReleaseKM(const SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	return SyncSWTimelineReleaseKM_(psSWTimelineObj->pvTlObj);
+}
+
+static inline PVRSRV_ERROR
+SyncDumpFence(const SYNC_FENCE_OBJ *psFenceObj,
+              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+              void *pvDumpDebugFile)
+{
+	return SyncDumpFence_(psFenceObj->pvFenceObj, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+static inline PVRSRV_ERROR
+SyncSWDumpTimeline(const SYNC_TIMELINE_OBJ *psSWTimelineObj,
+                   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                   void *pvDumpDebugFile)
+{
+	return SyncSWDumpTimeline_(psSWTimelineObj->pvTlObj, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+
+#endif /* PVRSRV_SYNC_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/rgx_firmware_boot.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/rgx_firmware_boot.h
new file mode 100644
index 000000000000..b0a14683e1a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/rgx_firmware_boot.h
@@ -0,0 +1,91 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef RGX_FIRMWARE_BOOT_H
+#define RGX_FIRMWARE_BOOT_H
+
+#include "img_types.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+
+#define TD_MAX_NUM_MIPS_PAGETABLE_PAGES (4U)
+
+typedef union _PVRSRV_FW_BOOT_PARAMS_
+{
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_UINT32 ui32NumThreads;
+	} sMeta;
+
+	struct
+	{
+		IMG_DEV_PHYADDR sGPURegAddr;
+		IMG_DEV_PHYADDR asFWPageTableAddr[TD_MAX_NUM_MIPS_PAGETABLE_PAGES];
+		IMG_DEV_PHYADDR sFWStackAddr;
+		IMG_UINT32 ui32FWPageTableLog2PageSize;
+		IMG_UINT32 ui32FWPageTableNumPages;
+	} sMips;
+
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememDataSize;
+	} sRISCV;
+
+} PVRSRV_FW_BOOT_PARAMS;
+
+
+#endif /* RGX_FIRMWARE_BOOT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/ri_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/ri_server.h
new file mode 100644
index 000000000000..a8bcd9f32039
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/ri_server.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File           ri_server.h
+@Title          Resource Information abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RI_SERVER_H
+#define RI_SERVER_H
+
+#include "img_defs.h"
+#include "ri_typedefs.h"
+#include "pmr.h"
+#include "pvrsrv_error.h"
+#include "physheap.h"
+#include "connection_server.h"
+#include "device.h"
+
+PVRSRV_ERROR RIInitKM(void);
+void RIDeInitKM(void);
+
+void RILockAcquireKM(void);
+void RILockReleaseKM(void);
+
+PVRSRV_ERROR RIWritePMREntryKM(PMR *psPMR);
+
+PVRSRV_ERROR RIWritePMREntryWithOwnerKM(PMR *psPMR,
+                                        IMG_PID ui32Owner);
+
+PVRSRV_ERROR RIWriteMEMDESCEntryKM(void* psConnection,
+                                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   PMR *psPMR,
+                                   IMG_UINT32 ui32TextBSize,
+                                   const IMG_CHAR *psz8TextB,
+                                   IMG_UINT64 uiOffset,
+                                   IMG_UINT64 uiSize,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   RI_HANDLE *phRIHandle);
+
+PVRSRV_ERROR RIWriteProcListEntryKM(void* psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32TextBSize,
+                                    const IMG_CHAR *psz8TextB,
+                                    IMG_UINT64 ui64Size,
+                                    IMG_UINT64 ui64DevVAddr,
+                                    RI_HANDLE *phRIHandle);
+
+PVRSRV_ERROR RIUpdateMEMDESCAddrKM(RI_HANDLE hRIHandle,
+                                   IMG_DEV_VIRTADDR sVAddr);
+
+PVRSRV_ERROR RIDeletePMREntryKM(RI_HANDLE hRIHandle);
+PVRSRV_ERROR RIDeleteMEMDESCEntryKM(RI_HANDLE hRIHandle);
+
+PVRSRV_ERROR RIDeleteListKM(void);
+
+PVRSRV_ERROR RIDumpListKM(PMR *psPMR);
+
+PVRSRV_ERROR RIDumpAllKM(void);
+
+PVRSRV_ERROR RIDumpProcessKM(IMG_PID pid);
+
+#if defined(DEBUG)
+PVRSRV_ERROR RIDumpProcessListKM(PMR *psPMR,
+                                 IMG_PID pid,
+                                 IMG_UINT64 ui64Offset,
+                                 IMG_DEV_VIRTADDR *psDevVAddr);
+#endif
+
+void RIConnectionClosed(void* psConnection);
+
+PVRSRV_ERROR RIDeleteEntriesForPID(IMG_PID pid);
+
+IMG_BOOL RIGetListEntryKM(IMG_PID pid,
+                          IMG_HANDLE **ppHandle,
+                          IMG_CHAR **ppszEntryString);
+
+IMG_INT32 RITotalAllocProcessUnlocked(IMG_PID pid, PHYS_HEAP_TYPE ePhysHeapType);
+
+#endif /* RI_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_pvr.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_pvr.h
new file mode 100644
index 000000000000..48bc94d8ef0b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_pvr.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          SO Interface header file for common PVR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains SO interface functions. These functions are defined in
+                the common layer and are called from the env layer OS specific
+                implementation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SOFUNC_PVR_H_)
+#define SOFUNC_PVR_H_
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "device.h"
+#include "pvr_notifier.h"
+
+
+/**************************************************************************/ /*!
+ @Function     SOPvrDbgRequestNotifyRegister
+ @Description  SO Interface function called from the OS layer implementation.
+               Register a callback function that is called when a debug request
+               is made via a call PVRSRVDebugRequest. There are a number of
+               verbosity levels ranging from DEBUG_REQUEST_VERBOSITY_LOW up to
+               DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+               for each level up to the highest level specified to
+               PVRSRVDebugRequest.
+@Output        phNotify             On success, points to debug notifier handle
+@Input         psDevNode            Device node for which the debug callback
+                                    should be registered
+@Input         pfnDbgRequestNotify  Function callback
+@Input         ui32RequesterID      Requester ID. This is used to determine
+                                    the order in which callbacks are called,
+                                    see DEBUG_REQUEST_*
+@Input         hDbgReqeustHandle    Data to be passed back to the caller via
+                                    the callback function
+@Return        PVRSRV_ERROR         PVRSRV_OK on success and an error otherwise
+*/ /******************************************************************** ******/
+PVRSRV_ERROR SOPvrDbgRequestNotifyRegister(IMG_HANDLE *phNotify,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+							  IMG_UINT32 ui32RequesterID,
+							  PVRSRV_DBGREQ_HANDLE hDbgRequestHandle);
+
+/**************************************************************************/ /*!
+ @Function     SOPvrDbgRequestNotifyUnregister
+ @Description  SO Interface function called from the OS layer implementation.
+               Remove and clean up the specified notifier registration so that
+               it does not receive any further callbacks.
+ @Input        hNotify     Handle returned to caller from
+                           SOPvrDbgRequestNotifyRegister().
+ @Return       PVRSRV_ERROR
+*/ /***************************************************************************/
+PVRSRV_ERROR SOPvrDbgRequestNotifyUnregister(IMG_HANDLE hNotify);
+
+
+#endif /* SOFUNC_PVR_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_rgx.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_rgx.h
new file mode 100644
index 000000000000..be9594d95242
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_rgx.h
@@ -0,0 +1,95 @@
+/*************************************************************************/ /*!
+@File
+@Title          SO Interface header file for devices/RGX functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains SO interface functions. These functions are defined in
+                the common devices layer and are called from the env layer OS
+                specific implementation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SOFUNC_RGX_H_)
+#define SOFUNC_RGX_H_
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+
+#if !defined(NO_HARDWARE)
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsRegister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Initialise data used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument). This function must be called only once for each
+               different user/handle.
+
+ @Input        phGpuUtilUser - Pointer to handle used to identify a user of
+                               RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser);
+
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsUnregister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Free data previously used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument).
+
+ @Input        hGpuUtilUser - Handle used to identify a user of
+                              RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser);
+#endif /* !defined(NO_HARDWARE) */
+
+
+
+#endif /* SOFUNC_RGX_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/srvcore.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvcore.h
new file mode 100644
index 000000000000..8d1385f618d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvcore.h
@@ -0,0 +1,274 @@
+/**************************************************************************/ /*!
+@File
+@Title          PVR Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the PVR Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVCORE_H
+#define SRVCORE_H
+
+#include "lock_types.h"
+#include "connection_server.h"
+#include "pvr_debug.h"
+
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+                    IMG_UINT32 ui32DispatchTableEntry,
+                    void *pvDest,
+                    void __user *pvSrc,
+                    IMG_UINT32 ui32Size);
+PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+                  IMG_UINT32 ui32DispatchTableEntry,
+                  void __user *pvDest,
+                  void *pvSrc,
+                  IMG_UINT32 ui32Size);
+#else
+FORCE_INLINE PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+					IMG_UINT32 ui32DispatchTableEntry,
+					void *pvDest,
+					void __user *pvSrc,
+					IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER (ui32DispatchTableEntry);
+	return OSBridgeCopyFromUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+FORCE_INLINE PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+				  IMG_UINT32 ui32DispatchTableEntry,
+				  void __user *pvDest,
+				  void *pvSrc,
+				  IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER (ui32DispatchTableEntry);
+	return OSBridgeCopyToUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+#endif
+
+IMG_INT
+DummyBW(IMG_UINT32 ui32DispatchTableEntry,
+        IMG_UINT8 *psBridgeIn,
+        IMG_UINT8 *psBridgeOut,
+        CONNECTION_DATA *psConnection);
+
+typedef PVRSRV_ERROR (*ServerResourceDestroyFunction)(IMG_HANDLE, IMG_HANDLE);
+
+typedef IMG_INT (*BridgeWrapperFunction)(IMG_UINT32 ui32DispatchTableEntry,
+									 IMG_UINT8 *psBridgeIn,
+									 IMG_UINT8 *psBridgeOut,
+									 CONNECTION_DATA *psConnection);
+
+typedef struct _PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY
+{
+	BridgeWrapperFunction pfFunction; /*!< The wrapper function that validates the ioctl
+	                                    arguments before calling into srvkm proper */
+	POS_LOCK hBridgeLock; /*!< The bridge lock which needs to be acquired
+                                   before calling the above wrapper */
+
+	IMG_UINT32 ui32InBufferSize; /*!< The expected size of the in buffer given by the client */
+	IMG_UINT32 ui32OutBufferSize; /*!< The expected size of the out buffer given by the client */
+
+#if defined(DEBUG_BRIDGE_KM)
+	const IMG_CHAR *pszIOCName; /*!< Name of the ioctl: e.g. "PVRSRV_BRIDGE_CONNECT_SERVICES" */
+	const IMG_CHAR *pszFunctionName; /*!< Name of the wrapper function: e.g. "PVRSRVConnectBW" */
+	const IMG_CHAR *pszBridgeLockName; /*!< Name of bridge lock which will be acquired */
+	IMG_UINT32 ui32CallCount; /*!< The total number of times the ioctl has been called */
+	IMG_UINT32 ui32CopyFromUserTotalBytes; /*!< The total number of bytes copied from
+	                                         userspace within this ioctl */
+	IMG_UINT32 ui32CopyToUserTotalBytes; /*!< The total number of bytes copied from
+	                                       userspace within this ioctl */
+	IMG_UINT64 ui64TotalTimeNS; /*!< The total amount of time spent in this bridge function */
+	IMG_UINT64 ui64MaxTimeNS; /*!< The maximum amount of time for a single call to this bridge function */
+#endif
+}PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY;
+
+#if defined(SUPPORT_RGX)
+	#define BRIDGE_DISPATCH_TABLE_ENTRY_COUNT  (PVRSRV_BRIDGE_RGX_DISPATCH_LAST+1)
+	#define BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT  (PVRSRV_BRIDGE_RGX_LAST+1)
+#else
+	#define BRIDGE_DISPATCH_TABLE_ENTRY_COUNT  (PVRSRV_BRIDGE_DISPATCH_LAST+1)
+	#define BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT  (PVRSRV_BRIDGE_LAST+1)
+#endif
+
+extern PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY g_BridgeDispatchTable[BRIDGE_DISPATCH_TABLE_ENTRY_COUNT];
+
+void BridgeDispatchTableStartOffsetsInit(void);
+
+void
+_SetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+                       IMG_UINT32 ui32Index,
+                       const IMG_CHAR *pszIOCName,
+                       BridgeWrapperFunction pfFunction,
+                       const IMG_CHAR *pszFunctionName,
+                       POS_LOCK hBridgeLock,
+                       const IMG_CHAR* pszBridgeLockName,
+                       IMG_UINT32 ui32InBufferSize,
+                       IMG_UINT32 ui32OutBufferSize);
+void
+UnsetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+                        IMG_UINT32 ui32Index);
+
+
+/* PRQA S 0884,3410 2*/ /* macro relies on the lack of brackets */
+#define SetDispatchTableEntry(ui32BridgeGroup, ui32Index, pfFunction,\
+                              hBridgeLock, ui32InBufferSize, ui32OutBufferSize) \
+	do \
+	{ \
+		static_assert((ui32InBufferSize) <= PVRSRV_MAX_BRIDGE_IN_SIZE, "Bridge input buffer is too small for bridge function: " #pfFunction); \
+		static_assert((ui32OutBufferSize) <= PVRSRV_MAX_BRIDGE_OUT_SIZE, "Bridge output buffer is too small for bridge function: " #pfFunction); \
+		_SetDispatchTableEntry(ui32BridgeGroup, ui32Index, #ui32Index, (BridgeWrapperFunction)pfFunction, #pfFunction,\
+		                       (POS_LOCK)hBridgeLock, #hBridgeLock, ui32InBufferSize, ui32OutBufferSize); \
+	} while (0)
+
+#define DISPATCH_TABLE_GAP_THRESHOLD 5
+
+
+#if defined(DEBUG_BRIDGE_KM)
+typedef struct _PVRSRV_BRIDGE_GLOBAL_STATS
+{
+	IMG_UINT32 ui32IOCTLCount;
+	IMG_UINT32 ui32TotalCopyFromUserBytes;
+	IMG_UINT32 ui32TotalCopyToUserBytes;
+} PVRSRV_BRIDGE_GLOBAL_STATS;
+
+void BridgeGlobalStatsLock(void);
+void BridgeGlobalStatsUnlock(void);
+
+/* OS specific code may want to report the stats held here and within the
+ * BRIDGE_DISPATCH_TABLE_ENTRYs (E.g. on Linux we report these via a
+ * debugfs entry /(sys/kernel/debug|proc)/pvr/bridge_stats) */
+extern PVRSRV_BRIDGE_GLOBAL_STATS g_BridgeGlobalStats;
+#endif
+
+PVRSRV_ERROR BridgeDispatcherInit(void);
+void BridgeDispatcherDeinit(void);
+
+PVRSRV_ERROR
+BridgedDispatchKM(CONNECTION_DATA * psConnection,
+                  PVRSRV_BRIDGE_PACKAGE   * psBridgePackageKM);
+
+PVRSRV_ERROR
+PVRSRVConnectKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE * psDeviceNode,
+                IMG_UINT32 ui32Flags,
+                IMG_UINT32 ui32ClientBuildOptions,
+                IMG_UINT32 ui32ClientDDKVersion,
+                IMG_UINT32 ui32ClientDDKBuild,
+                IMG_UINT8  *pui8KernelArch,
+                IMG_UINT32 *ui32CapabilityFlags,
+                IMG_UINT64 *ui64PackedBvnc);
+
+PVRSRV_ERROR
+PVRSRVDisconnectKM(void);
+
+PVRSRV_ERROR
+PVRSRVAcquireGlobalEventObjectKM(IMG_HANDLE *phGlobalEventObject);
+
+PVRSRV_ERROR
+PVRSRVReleaseGlobalEventObjectKM(IMG_HANDLE hGlobalEventObject);
+
+PVRSRV_ERROR
+PVRSRVDumpDebugInfoKM(CONNECTION_DATA *psConnection,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32VerbLevel);
+
+PVRSRV_ERROR
+PVRSRVGetDevClockSpeedKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_PUINT32  pui32RGXClockSpeed);
+
+PVRSRV_ERROR
+PVRSRVHWOpTimeoutKM(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PVRSRVAlignmentCheckKM(CONNECTION_DATA *psConnection,
+                                    PVRSRV_DEVICE_NODE * psDeviceNode,
+                                    IMG_UINT32 ui32FWAlignChecksSize,
+                                    IMG_UINT32 aui32FWAlignChecks[]);
+
+PVRSRV_ERROR PVRSRVGetDeviceStatusKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 *pui32DeviceStatus);
+
+PVRSRV_ERROR PVRSRVGetMultiCoreInfoKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32CapsSize,
+                                     IMG_UINT32 *pui32NumCores,
+                                     IMG_UINT64 *pui64Caps);
+
+PVRSRV_ERROR PVRSRVFindProcessMemStatsKM(IMG_PID pid,
+                                         IMG_UINT32 ui32ArrSize,
+                                         IMG_BOOL bAllProcessStats,
+                                         IMG_UINT64 *pui64MemoryStats);
+
+static INLINE
+PVRSRV_ERROR DestroyServerResource(const SHARED_DEV_CONNECTION hConnection,
+                                   IMG_HANDLE hEvent,
+                                   ServerResourceDestroyFunction pfnDestroyCall,
+                                   IMG_HANDLE hResource)
+{
+    PVR_UNREFERENCED_PARAMETER(hEvent);
+
+    return pfnDestroyCall(GetBridgeHandle(hConnection), hResource);
+}
+
+/*************************************************************************/ /*!
+@Function      PVRSRVBlockIfFrozen
+@Description   Puts caller into a blocking wait (using event-objects) if the
+                specified device is in a FROZEN state. Routine completes and
+                returns control to the caller once the underlying device state
+                clears its FROZEN state.
+@Input         psDeviceNode     Device Node reference
+@Return        Nothing.         Execution blocks if device is frozen.
+*/ /**************************************************************************/
+void PVRSRVBlockIfFrozen(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* SRVCORE_H */
+
+/******************************************************************************
+ End of file (srvcore.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/srvinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvinit.h
new file mode 100644
index 000000000000..48e6863eae1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvinit.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          Initialisation server internal header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the connections between the various parts of the
+                initialisation server.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVINIT_H
+#define SRVINIT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "device_connection.h"
+#include "device.h"
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* SRVINIT_H */
+
+/******************************************************************************
+ End of file (srvinit.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/srvkm.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvkm.h
new file mode 100644
index 000000000000..29418764daa5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvkm.h
@@ -0,0 +1,144 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services kernel module internal header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_H
+#define SRVKM_H
+
+#include "servicesext.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+struct _PVRSRV_DEVICE_NODE_;
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDriverInit
+@Description  Performs one time driver initialisation of Services Common and
+              Device layers.
+@Return       PVRSRV_ERROR   PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVCommonDriverInit(void);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDriverInit
+@Description  Performs one time driver de-initialisation of Services.
+@Return       void
+*/ /**************************************************************************/
+void PVRSRVCommonDriverDeInit(void);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceCreate
+@Description  Creates and initialises a common layer Services device node
+              for an OS native device. First stage device discovery.
+@Input        pvOSDevice      OS native device
+@Input        i32KernelDeviceID A unique identifier which helps recognise this
+                              Device in the UM space provided by the OS.
+@Output       ppsDeviceNode   Points to the new device node on success
+@Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVCommonDeviceCreate(void *pvOSDevice, IMG_INT32 i32KernelDeviceID,
+				   struct _PVRSRV_DEVICE_NODE_ **ppsDeviceNode);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceInitialise
+@Description  Initialises the device layer specifics (e.g. boot FW etc)
+              for the supplied device node, created previously by
+              PVRSRVCommonDeviceCreate. The device is ready for use when this
+              second stage device initialisation returns successfully.
+@Input        psDeviceNode  Device node of the device to be initialised
+@Return       PVRSRV_ERROR  PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVCommonDeviceInitialise(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceDestroy
+@Description  Destroys a PVR Services device node.
+@Input        psDeviceNode  Device node to destroy
+*/ /**************************************************************************/
+void
+PVRSRVCommonDeviceDestroy(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+/******************
+HIGHER LEVEL MACROS
+*******************/
+
+/*----------------------------------------------------------------------------
+Repeats the body of the loop for a certain minimum time, or until the body
+exits by its own means (break, return, goto, etc.)
+
+Example of usage:
+
+LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+{
+	if (psQueueInfo->ui32ReadOffset == psQueueInfo->ui32WriteOffset)
+	{
+		bTimeout = IMG_FALSE;
+		break;
+	}
+
+	OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+} END_LOOP_UNTIL_TIMEOUT_US();
+
+-----------------------------------------------------------------------------*/
+
+/* uiNotLastLoop will remain at 1 until the timeout has expired, at which time
+ * it will be decremented and the loop executed one final time. This is
+ * necessary when preemption is enabled.
+ */
+/* PRQA S 3411,3431 12 */ /* critical format, leave alone */
+#define LOOP_UNTIL_TIMEOUT_US(TIMEOUT_US) \
+{\
+	IMG_UINT32 uiOffset, uiStart, uiCurrent; \
+	IMG_INT32 iNotLastLoop;					 \
+	for (uiOffset = 0, uiStart = OSClockus(), uiCurrent = uiStart + 1, iNotLastLoop = 1;\
+		((uiCurrent - uiStart + uiOffset) < (TIMEOUT_US)) || iNotLastLoop--;				\
+		uiCurrent = OSClockus(),													\
+		uiOffset = uiCurrent < uiStart ? IMG_UINT32_MAX - uiStart : uiOffset,		\
+		uiStart = uiCurrent < uiStart ? 0 : uiStart)
+
+#define END_LOOP_UNTIL_TIMEOUT_US() \
+}
+
+#endif /* SRVKM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint.h
new file mode 100644
index 000000000000..3c3846d7f0cd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint.h
@@ -0,0 +1,783 @@
+/*************************************************************************/ /*!
+@File
+@Title          Synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_H
+#define SYNC_CHECKPOINT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_sync_km.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+#include "dllist.h"
+#include "pvr_debug.h"
+#include "device_connection.h"
+#include "opaque_types.h"
+
+#ifndef CHECKPOINT_TYPES
+#define CHECKPOINT_TYPES
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG *PSYNC_CHECKPOINT_CONTEXT;
+
+typedef struct SYNC_CHECKPOINT_TAG *PSYNC_CHECKPOINT;
+#endif
+
+/* definitions for functions to be implemented by OS-specific sync - the OS-specific sync code
+   will call SyncCheckpointRegisterFunctions() when initialised, in order to register functions
+   we can then call */
+#ifndef CHECKPOINT_PFNS
+#define CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN)(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                                             PVRSRV_FENCE fence,
+                                                             IMG_UINT32 *nr_checkpoints,
+                                                             PSYNC_CHECKPOINT **checkpoint_handles,
+                                                             IMG_UINT64 *pui64FenceUID);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN)(PPVRSRV_DEVICE_NODE device,
+                                                            const IMG_CHAR *fence_name,
+                                                            PVRSRV_TIMELINE timeline,
+                                                            PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                                            PVRSRV_FENCE *new_fence,
+                                                            IMG_UINT64 *pui64FenceUID,
+                                                            void **ppvFenceFinaliseData,
+                                                            PSYNC_CHECKPOINT *new_checkpoint_handle,
+                                                            IMG_HANDLE *timeline_update_sync,
+                                                            IMG_UINT32 *timeline_update_value);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN)(PVRSRV_FENCE fence_to_rollback, void *finalise_data);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN)(PVRSRV_FENCE fence_to_finalise, void *finalise_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN)(void *private_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN)(PVRSRV_FENCE fence_to_signal);
+typedef void (*PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN)(void *mem_ptr);
+typedef IMG_UINT32 (*PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN)(IMG_UINT32 num_ufos, IMG_UINT32 *vaddrs);
+#if defined(PDUMP)
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN)(PVRSRV_FENCE iFence,
+									IMG_UINT32 *puiNumCheckpoints,
+									PSYNC_CHECKPOINT **papsCheckpoints);
+#endif
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN)(PVRSRV_FENCE iExportFence,
+                                                             PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+                                                             PSYNC_CHECKPOINT *checkpoint_handle);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN)(PVRSRV_FENCE iExportFence);
+
+#define SYNC_CHECKPOINT_IMPL_MAX_STRLEN 20
+
+typedef struct
+{
+	PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN pfnFenceResolve;
+	PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN pfnFenceCreate;
+	PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN pfnFenceDataRollback;
+	PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN pfnFenceFinalise;
+	PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN pfnNoHWUpdateTimelines;
+	PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN pfnNoHWSignalExpFence;
+	PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN pfnFreeCheckpointListMem;
+	PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN pfnDumpInfoOnStalledUFOs;
+	IMG_CHAR pszImplName[SYNC_CHECKPOINT_IMPL_MAX_STRLEN];
+#if defined(PDUMP)
+	PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN pfnSyncFenceGetCheckpoints;
+#endif
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN pfnExportFenceResolve;
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN pfnExportFenceRollback;
+} PFN_SYNC_CHECKPOINT_STRUCT;
+
+PVRSRV_ERROR SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns);
+
+#endif /* ifndef CHECKPOINT_PFNS */
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextCreate
+
+@Description    Create a new synchronisation checkpoint context
+
+@Input          psDevNode                 Device node
+
+@Output         ppsSyncCheckpointContext  Handle to the created synchronisation
+                                          checkpoint context
+
+@Return         PVRSRV_OK if the synchronisation checkpoint context was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointContextCreate(PPVRSRV_DEVICE_NODE psDevNode,
+                            PSYNC_CHECKPOINT_CONTEXT *ppsSyncCheckpointContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextDestroy
+
+@Description    Destroy a synchronisation checkpoint context
+
+@Input          psSyncCheckpointContext  Handle to the synchronisation
+                                         checkpoint context to destroy
+
+@Return         PVRSRV_OK if the synchronisation checkpoint context was
+                successfully destroyed.
+                PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT if the context still
+                has sync checkpoints defined
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextRef
+
+@Description    Takes a reference on a synchronisation checkpoint context
+
+@Input          psContext  Handle to the synchronisation checkpoint context
+                           on which a ref is to be taken
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextUnref
+
+@Description    Drops a reference taken on a synchronisation checkpoint
+                context
+
+@Input          psContext  Handle to the synchronisation checkpoint context
+                           on which the ref is to be dropped
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointAlloc
+
+@Description    Allocate a new synchronisation checkpoint on the specified
+                synchronisation checkpoint context
+
+@Input          hSyncCheckpointContext  Handle to the synchronisation
+                                        checkpoint context
+
+@Input          hTimeline               Timeline on which this sync
+                                        checkpoint is being created
+
+@Input          hFence                  Fence as passed into pfnFenceResolve
+                                        API, when the API encounters a non-PVR
+                                        fence as part of its input fence. From
+                                        all other places this argument must be
+                                        PVRSRV_NO_FENCE.
+
+@Input          pszClassName            Sync checkpoint source annotation
+                                        (will be truncated to at most
+                                         PVRSRV_SYNC_NAME_LENGTH chars)
+
+@Output         ppsSyncCheckpoint       Created synchronisation checkpoint
+
+@Return         PVRSRV_OK if the synchronisation checkpoint was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                    PVRSRV_TIMELINE hTimeline,
+                    PVRSRV_FENCE hFence,
+                    const IMG_CHAR *pszCheckpointName,
+                    PSYNC_CHECKPOINT *ppsSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFree
+
+@Description    Free a synchronisation checkpoint
+                The reference count held for the synchronisation checkpoint
+                is decremented - if it has becomes zero, it is also freed.
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to free
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointSignal
+
+@Description    Signal the synchronisation checkpoint
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to signal
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointSignalNoHW
+
+@Description    Signal the synchronisation checkpoint in NO_HARDWARE build
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to signal
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointSignalNoHW(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointError
+
+@Description    Error the synchronisation checkpoint
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to error
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointStateFromUFO
+
+@Description    Returns the current state of the synchronisation checkpoint
+                which has the given UFO firmware address
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint
+
+@Return         The current state (32-bit value) of the sync checkpoint
+*/
+/*****************************************************************************/
+IMG_UINT32 SyncCheckpointStateFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointErrorFromUFO
+
+@Description    Error the synchronisation checkpoint which has the
+                given UFO firmware address
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to be errored
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to be errored
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointErrorFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackFromUFO
+
+@Description    Drop the enqueued count reference taken on the synchronisation
+                checkpoint on behalf of the firmware.
+                Called in the event of a DM Kick failing.
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to be rolled back
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to be rolled back
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointRollbackFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointIsSignalled
+
+@Description    Returns IMG_TRUE if the synchronisation checkpoint is
+                signalled or errored
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to test
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_BOOL
+SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint,
+                          IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointIsErrored
+
+@Description    Returns IMG_TRUE if the synchronisation checkpoint is
+                errored
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to test
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_BOOL
+SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint,
+                        IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointTakeRef
+
+@Description    Take a reference on a synchronisation checkpoint
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to take a
+                                        reference on
+
+@Return         PVRSRV_OK if a reference was taken on the synchronisation
+                primitive
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDropRef
+
+@Description    Drop a reference on a synchronisation checkpoint
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to drop a
+                                        reference on
+
+@Return         PVRSRV_OK if a reference was dropped on the synchronisation
+                primitive
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointResolveFence
+
+@Description    Resolve a fence, returning a list of the sync checkpoints
+                that fence contains.
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          psSyncCheckpointContext The sync checkpoint context
+                                        on which checkpoints should be
+                                        created (in the event of the fence
+                                        having a native sync pt with no
+                                        associated sync checkpoint)
+
+@Input          hFence                  The fence to be resolved
+
+@Output         pui32NumSyncCheckpoints The number of sync checkpoints the
+                                        fence contains. Can return 0 if
+                                        passed a null (-1) fence.
+
+@Output         papsSyncCheckpoints     List of sync checkpoints the fence
+                                        contains
+
+@Output         puiFenceUID             Unique ID of the resolved fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointResolveFence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PVRSRV_FENCE hFence,
+                           IMG_UINT32 *pui32NumSyncCheckpoints,
+                           PSYNC_CHECKPOINT **papsSyncCheckpoints,
+                           IMG_UINT64 *puiFenceUID,
+                           PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointResolveExportFence
+
+@Description    Resolve an export fence, returning the sync checkpoint
+                that fence contains.
+                This function in turn calls a function provided by the
+                sync implementation.
+
+@Input          hExportFence            The export fence to be resolved
+
+@Input          psSyncCheckpointContext The context in which to create the
+                                        new sync checkpoint for the export fence
+
+@Output         ppsSyncCheckpoint       The sync checkpoint the fence
+                                        contains
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointResolveExportFence(PVRSRV_FENCE hExportFence,
+                           PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PSYNC_CHECKPOINT *ppsSyncCheckpoint,
+                           PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackExportFence
+
+@Description    Rollback an export fence, freeing the sync checkpoint
+                that fence was assigned.
+                This function in turn calls a function provided by the
+                sync implementation.
+
+@Input          hExportFence            The export fence to be rolled back
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointRollbackExportFence(PVRSRV_FENCE hExportFence);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCreateFence
+
+@Description    Create a fence containing a single sync checkpoint.
+                Return the fence and a ptr to sync checkpoint it contains.
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          pszFenceName            String to assign to the new fence
+                                        (for debugging purposes)
+
+@Input          hTimeline               Timeline on which the new fence is
+                                        to be created
+
+@Input          psSyncCheckpointContext Sync checkpoint context to be used
+                                        when creating the new fence
+
+@Output         phNewFence              The newly created fence
+
+@Output         pui64FenceUID           Unique ID of the created fence
+
+@Output         ppvFenceFinaliseData    Any data needed to finalise the fence
+                                        in a later call to the function
+                                        SyncCheckpointFinaliseFence()
+
+@Output         psNewSyncCheckpoint     The sync checkpoint contained in
+                                        the new fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointCreateFence(PPVRSRV_DEVICE_NODE psDeviceNode,
+                          const IMG_CHAR *pszFenceName,
+                          PVRSRV_TIMELINE hTimeline,
+                          PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                          PVRSRV_FENCE *phNewFence,
+                          IMG_UINT64 *pui64FenceUID,
+                          void **ppvFenceFinaliseData,
+                          PSYNC_CHECKPOINT *psNewSyncCheckpoint,
+                          void **ppvTimelineUpdateSyncPrim,
+                          IMG_UINT32 *pui32TimelineUpdateValue,
+                          PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackFenceData
+
+@Description    'Rolls back' the fence specified (destroys the fence and
+                takes any other required actions to undo the fence
+                creation (eg if the implementation wishes to revert the
+                incrementing of the fence's timeline, etc).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          hFence                  Fence to be 'rolled back'
+
+@Input          pvFinaliseData          Data needed to finalise the
+                                        fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointRollbackFenceData(PVRSRV_FENCE hFence, void *pvFinaliseData);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFinaliseFence
+
+@Description    'Finalise' the fence specified (performs any actions the
+                underlying implementation may need to perform just prior
+                to the fence being returned to the client.
+                This function in turn calls a function provided by the
+                OS native sync implementation - if the native sync
+                implementation does not need to perform any actions at
+                this time, this function does not need to be registered.
+
+@Input          psDevNode               Device node
+
+@Input          hFence                  Fence to be 'finalised'
+
+@Input          pvFinaliseData          Data needed to finalise the fence
+
+@Input          psSyncCheckpoint        Base sync checkpoint that this fence
+                                        is formed of
+
+@Input          pszName                 Fence annotation
+
+@Return         PVRSRV_OK if a valid fence and finalise data were provided.
+                PVRSRV_ERROR_INVALID_PARAMS if an invalid fence or finalise
+                data were provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function (permitted).
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointFinaliseFence(PPVRSRV_DEVICE_NODE psDevNode,
+                            PVRSRV_FENCE hFence,
+                            void *pvFinaliseData,
+                            PSYNC_CHECKPOINT psSyncCheckpoint,
+                            const IMG_CHAR *pszName);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFreeCheckpointListMem
+
+@Description    Free memory the memory which was allocated by the sync
+                implementation and used to return the list of sync
+                checkpoints when resolving a fence.
+                to the fence being returned to the client.
+                This function in turn calls a free function registered by
+                the sync implementation (if a function has been registered).
+
+@Input          pvCheckpointListMem     Pointer to the memory to be freed
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointFreeCheckpointListMem(void *pvCheckpointListMem);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointNoHWUpdateTimelines
+
+@Description    Called by the DDK in a NO_HARDWARE build only.
+                After syncs have been manually signalled by the DDK, this
+                function is called to allow the OS native sync implementation
+                to update its timelines (as the usual callback notification
+                of signalled checkpoints is not supported for NO_HARDWARE).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          pvPrivateData            Any data the OS native sync
+                                         implementation might require.
+
+@Return         PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function, otherwise
+                PVRSRV_OK.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointNoHWUpdateTimelines(void *pvPrivateData);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointNoHWSignalExportFence
+
+@Description    Called by the DDK in a NO_HARDWARE build only.
+                After syncs have been manually signalled by the DDK, this
+                function is called to allow the OS native sync implementation
+                to signal the export fence (as the usual callback notification
+                of signalled checkpoints is not supported for NO_HARDWARE).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          iExportFenceToSignal     The export fence to signal.
+
+@Return         PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function, otherwise
+                PVRSRV_OK.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointNoHWSignalExportFence(PVRSRV_FENCE iExportFenceToSignal);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDumpInfoOnStalledUFOs
+
+@Description    Called by the DDK in the event of the health check watchdog
+                examining the CCBs and determining that one has failed to
+                progress after 10 second when the GPU is idle due to waiting
+                on one or more UFO fences.
+                The DDK will pass a list of UFOs on which the CCB is waiting
+                and the sync implementation will check them to see if any
+                relate to sync points it has created. If so, the
+                implementation should dump debug information on those sync
+                points to the kernel log or other suitable output (which will
+                allow the unsignalled syncs to be identified).
+                The function shall return the number of syncs in the provided
+                array that were syncs which it had created.
+
+@Input          ui32NumUFOs           The number of UFOs in the array passed
+                                      in the pui32VAddrs parameter.
+                pui32Vaddr            The array of UFOs the CCB is waiting on.
+
+@Output         pui32NumSyncOwnedUFOs The number of UFOs in pui32Vaddr which
+                                      relate to syncs created by the sync
+                                      implementation.
+
+@Return         PVRSRV_OK if a valid pointer is provided in pui32NumSyncOwnedUFOs.
+                PVRSRV_ERROR_INVALID_PARAMS if a NULL value is provided in
+                pui32NumSyncOwnedUFOs.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointDumpInfoOnStalledUFOs(IMG_UINT32 ui32NumUFOs,
+                                    IMG_UINT32 *pui32Vaddrs,
+                                    IMG_UINT32 *pui32NumSyncOwnedUFOs);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetStateString
+
+@Description    Called to get a string representing the current state of a
+                sync checkpoint.
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get the
+                                        state for.
+
+@Return         The string representing the current state of this checkpoint
+*/
+/*****************************************************************************/
+const IMG_CHAR *
+SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRecordLookup
+
+@Description    Returns a debug string with information about the
+                sync checkpoint.
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to lookup
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to lookup
+
+@Input          pszSyncInfo             Character array to write to
+
+@Input          len                     Len of the character array
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointRecordLookup(PPVRSRV_DEVICE_NODE psDevNode,
+                           IMG_UINT32 ui32FwAddr,
+                           IMG_CHAR * pszSyncInfo, size_t len);
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       PVRSRVSyncCheckpointFencePDumpPolKM
+
+@Description    Called to insert a poll into the PDump script on a given
+                Fence being signalled or errored.
+
+@Input          hFence        Fence for PDump to poll on
+
+@Return         PVRSRV_OK if a valid sync checkpoint was provided.
+*/
+/*****************************************************************************/
+
+PVRSRV_ERROR PVRSRVSyncCheckpointSignalledPDumpPolKM(PVRSRV_FENCE hFence);
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCommonDeviceIDs
+
+@Description    Determine if the DeviceIDs referenced by the checkpoint context
+                and device reference are to the same physical device.
+
+@Input          psSyncContext  Handle to the synchronisation checkpoint context
+
+@Input          hDevRef        Handle to the device reference to check against
+
+@Return         IMG_TRUE if devices refer to the same device, or if one of the
+                parameters is NULL.
+                IMG_FALSE if the two references are to different physical
+                devices.
+*/
+/*****************************************************************************/
+IMG_BOOL SyncCheckpointCommonDeviceIDs(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                                       IMG_HANDLE hDevRef);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetCounters
+
+@Description    Return the Current In Use and Max In Use Sync Checkpoint
+                counters for the specified sync context.
+
+@Input          psDevNode      Handle to the device-node
+
+@Output         puiInUse       Number of Sync CPs currently in-use
+
+@Output         puiMax         Maximum number of Sync CPs allocated
+
+@Output         puiXDInUse     Number of X-D Sync CPs currently in-use
+
+@Output         puiXDMax       Maximum number of X-D Sync CPs used
+
+@Return         PVRSRV_OK if a valid sync checkpoint context is passed.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR SyncCheckpointGetCounters(PPVRSRV_DEVICE_NODE psDevNode,
+                                       IMG_UINT32 *puiInUse,
+                                       IMG_UINT32 *puiMax,
+                                       IMG_UINT32 *puiXDInUse,
+                                       IMG_UINT32 *puiXDMax);
+#endif /* SYNC_CHECKPOINT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint_init.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint_init.h
new file mode 100644
index 000000000000..94f2e000ded9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint_init.h
@@ -0,0 +1,82 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation checkpoint initialisation interface
+                header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines synchronisation checkpoint structures that are visible
+                internally and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_INIT_H
+#define SYNC_CHECKPOINT_INIT_H
+
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointInit
+
+@Description    Initialise the sync checkpoint driver by giving it the
+                device node (needed to determine the pfnUFOAlloc function
+                to call in order to allocate sync block memory).
+
+@Input          psDevNode               Device for which sync checkpoints
+                                        are being initialised
+
+@Return         PVRSRV_OK               initialised successfully,
+                PVRSRV_ERROR_<error>    otherwise
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointInit(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDeinit
+
+@Description    Deinitialise the sync checkpoint driver.
+                Frees resources allocated during initialisation.
+
+@Input          psDevNode               Device for which sync checkpoints
+                                        are being de-initialised
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+#endif /* SYNC_CHECKPOINT_INIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_fallback_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_fallback_server.h
new file mode 100644
index 000000000000..ac6bd4755b0e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_fallback_server.h
@@ -0,0 +1,204 @@
+/**************************************************************************/ /*!
+@File
+@Title          Fallback sync interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SYNC_FALLBACK_SERVER_H
+#define SYNC_FALLBACK_SERVER_H
+
+#include "img_types.h"
+#include "sync_checkpoint.h"
+#include "device.h"
+#include "connection_server.h"
+
+
+typedef struct _PVRSRV_TIMELINE_SERVER_ PVRSRV_TIMELINE_SERVER;
+typedef struct _PVRSRV_FENCE_SERVER_ PVRSRV_FENCE_SERVER;
+typedef struct _PVRSRV_FENCE_EXPORT_ PVRSRV_FENCE_EXPORT;
+
+typedef struct _PVRSRV_SYNC_PT_ PVRSRV_SYNC_PT;
+
+#define SYNC_FB_TIMELINE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+#define SYNC_FB_FENCE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         SW SPECIFIC FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbTimelineCreateSW(IMG_UINT32 uiTimelineNameSize,
+                                    const IMG_CHAR *pszTimelineName,
+                                    PVRSRV_TIMELINE_SERVER **ppsTimeline);
+
+PVRSRV_ERROR SyncFbFenceCreateSW(CONNECTION_DATA *psConnection,
+                                 PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 PVRSRV_TIMELINE_SERVER *psTimeline,
+                                 IMG_UINT32 uiFenceNameSize,
+                                 const IMG_CHAR *pszFenceName,
+                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                 IMG_UINT64 *pui64SyncPtIdx);
+PVRSRV_ERROR SyncFbSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           PVRSRV_TIMELINE iSWTimeline,
+                                           const IMG_CHAR *pszFenceName,
+                                           PVRSRV_FENCE *piOutputFence,
+                                           IMG_UINT64* pui64SyncPtIdx);
+
+PVRSRV_ERROR SyncFbTimelineAdvanceSW(PVRSRV_TIMELINE_SERVER *psTimeline,
+                                     IMG_UINT64 *pui64SyncPtIdx);
+PVRSRV_ERROR SyncFbSWTimelineAdvanceKM(void *pvSWTimelineObj,
+                                       IMG_UINT64* pui64SyncPtIdx);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         PVR SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbTimelineCreatePVR(IMG_UINT32 uiTimelineNameSize,
+                                     const IMG_CHAR *pszTimelineName,
+                                     PVRSRV_TIMELINE_SERVER **ppsTimeline);
+
+PVRSRV_ERROR SyncFbFenceCreatePVR(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const IMG_CHAR *pszName,
+                                  PVRSRV_TIMELINE iTl,
+                                  PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext,
+                                  PVRSRV_FENCE *piOutFence,
+                                  IMG_UINT64 *puiFenceUID,
+                                  void **ppvFenceFinaliseData,
+                                  PSYNC_CHECKPOINT *ppsOutCheckpoint,
+                                  void **ppvTimelineUpdateSync,
+                                  IMG_UINT32 *puiTimelineUpdateValue);
+
+PVRSRV_ERROR SyncFbFenceResolvePVR(PSYNC_CHECKPOINT_CONTEXT psContext,
+                                   PVRSRV_FENCE iFence,
+                                   IMG_UINT32 *puiNumCheckpoints,
+                                   PSYNC_CHECKPOINT **papsCheckpoints,
+                                   IMG_UINT64 *puiFenceUID);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         GENERIC FUNCTIONS                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbGetFenceObj(PVRSRV_FENCE iFence,
+                               void **ppvFenceObj);
+
+PVRSRV_ERROR SyncFbSWGetTimelineObj(PVRSRV_TIMELINE iSWTimeline,
+                                    void **ppvSWTimelineObj);
+
+PVRSRV_ERROR SyncFbTimelineRelease(PVRSRV_TIMELINE_SERVER *psTl);
+
+PVRSRV_ERROR SyncFbFenceRelease(PVRSRV_FENCE_SERVER *psFence);
+PVRSRV_ERROR SyncFbFenceReleaseKM(void *pvFenceObj);
+
+PVRSRV_ERROR SyncFbFenceDup(PVRSRV_FENCE_SERVER *psInFence,
+                            PVRSRV_FENCE_SERVER **ppsOutFence);
+
+PVRSRV_ERROR SyncFbFenceMerge(PVRSRV_FENCE_SERVER *psInFence1,
+                              PVRSRV_FENCE_SERVER *psInFence2,
+                              IMG_UINT32 uiFenceNameSize,
+                              const IMG_CHAR *pszFenceName,
+                              PVRSRV_FENCE_SERVER **ppsOutFence);
+
+PVRSRV_ERROR SyncFbFenceWait(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiTimeout);
+
+PVRSRV_ERROR SyncFbFenceDump(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiLine,
+                             IMG_UINT32 uiFileNameLength,
+                             const IMG_CHAR *pszFile,
+                             IMG_UINT32 uiModuleLength,
+                             const IMG_CHAR *pszModule,
+                             IMG_UINT32 uiDescLength,
+                             const IMG_CHAR *pszDesc);
+
+PVRSRV_ERROR SyncFbDumpFenceKM(void *pvSWFenceObj,
+	                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	                           void *pvDumpDebugFile);
+
+PVRSRV_ERROR SyncFbSWDumpTimelineKM(void *pvSWTimelineObj,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile);
+
+PVRSRV_ERROR SyncFbRegisterSyncFunctions(void);
+
+PVRSRV_ERROR SyncFbRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR SyncFbDeregisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+IMG_UINT32 SyncFbDumpInfoOnStalledUFOs(IMG_UINT32 nr_ufos, IMG_UINT32 *vaddrs);
+
+IMG_BOOL SyncFbCheckpointHasSignalled(IMG_UINT32 ui32FwAddr, IMG_UINT32 ui32Value);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       IMPORT/EXPORT FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR SyncFbFenceExportInsecure(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppExport);
+
+PVRSRV_ERROR SyncFbFenceExportDestroyInsecure(PVRSRV_FENCE_EXPORT *psExport);
+
+PVRSRV_ERROR SyncFbFenceImportInsecure(CONNECTION_DATA *psConnection,
+                                       PVRSRV_DEVICE_NODE *psDevice,
+                                       PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFence);
+#endif /* defined(SUPPORT_INSECURE_EXPORT) */
+
+PVRSRV_ERROR SyncFbFenceExportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE * psDevNode,
+                                     PVRSRV_FENCE_SERVER *psFence,
+                                     IMG_SECURE_TYPE *phSecure,
+                                     PVRSRV_FENCE_EXPORT **ppsExport,
+                                     CONNECTION_DATA **ppsSecureConnection);
+
+PVRSRV_ERROR SyncFbFenceExportDestroySecure(PVRSRV_FENCE_EXPORT *psExport);
+
+PVRSRV_ERROR SyncFbFenceImportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDevice,
+                                     IMG_SECURE_TYPE hSecure,
+                                     PVRSRV_FENCE_SERVER **psFence);
+
+#endif /* SYNC_FALLBACK_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_server.h
new file mode 100644
index 000000000000..e35682970af6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_server.h
@@ -0,0 +1,249 @@
+/**************************************************************************/ /*!
+@File
+@Title          Server side synchronisation interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Describes the server side synchronisation functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv.h"
+#include "device.h"
+#include "devicemem.h"
+#include "pdump.h"
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+#include "pdump_km.h"
+
+#ifndef SYNC_SERVER_H
+#define SYNC_SERVER_H
+
+typedef struct _SYNC_PRIMITIVE_BLOCK_ SYNC_PRIMITIVE_BLOCK;
+typedef struct _SYNC_CONNECTION_DATA_ SYNC_CONNECTION_DATA;
+typedef struct SYNC_RECORD* SYNC_RECORD_HANDLE;
+
+typedef struct _SYNC_ADDR_LIST_
+{
+	IMG_UINT32 ui32NumSyncs;
+	PRGXFWIF_UFO_ADDR *pasFWAddrs;
+} SYNC_ADDR_LIST;
+
+PVRSRV_ERROR
+SyncPrimitiveBlockToFWAddr(SYNC_PRIMITIVE_BLOCK *psSyncPrimBlock,
+						IMG_UINT32 ui32Offset,
+						PRGXFWIF_UFO_ADDR *psAddrOut);
+
+void
+SyncAddrListInit(SYNC_ADDR_LIST *psList);
+
+void
+SyncAddrListDeinit(SYNC_ADDR_LIST *psList);
+
+PVRSRV_ERROR
+SyncAddrListPopulate(SYNC_ADDR_LIST *psList,
+						IMG_UINT32 ui32NumSyncs,
+						SYNC_PRIMITIVE_BLOCK **apsSyncPrimBlock,
+						IMG_UINT32 *paui32SyncOffset);
+
+PVRSRV_ERROR
+SyncAddrListAppendSyncPrim(SYNC_ADDR_LIST          *psList,
+						   PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim);
+PVRSRV_ERROR
+SyncAddrListAppendCheckpoints(SYNC_ADDR_LIST *psList,
+								IMG_UINT32 ui32NumCheckpoints,
+								PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+PVRSRV_ERROR
+SyncAddrListAppendAndDeRefCheckpoints(SYNC_ADDR_LIST *psList,
+									  IMG_UINT32 ui32NumCheckpoints,
+									  PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+void
+SyncAddrListDeRefCheckpoints(IMG_UINT32 ui32NumCheckpoints,
+							 PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+PVRSRV_ERROR
+SyncAddrListRollbackCheckpoints(PVRSRV_DEVICE_NODE *psDevNode, SYNC_ADDR_LIST *psList);
+
+PVRSRV_ERROR
+PVRSRVAllocSyncPrimitiveBlockKM(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+								SYNC_PRIMITIVE_BLOCK **ppsSyncBlk,
+								IMG_UINT32 *puiSyncPrimVAddr,
+								IMG_UINT32 *puiSyncPrimBlockSize,
+								PMR        **ppsSyncPMR);
+
+PVRSRV_ERROR
+PVRSRVExportSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+								 DEVMEM_EXPORTCOOKIE **psExportCookie);
+
+PVRSRV_ERROR
+PVRSRVUnexportSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk);
+
+PVRSRV_ERROR
+PVRSRVFreeSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *ppsSyncBlk);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimSetKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Index,
+					IMG_UINT32 ui32Value);
+
+PVRSRV_ERROR
+PVRSRVSyncAllocEventKM(CONNECTION_DATA *psConnection,
+					   PVRSRV_DEVICE_NODE *psDevNode,
+					   IMG_BOOL bServerSync,
+                       IMG_UINT32 ui32FWAddr,
+                       IMG_UINT32 ui32ClassNameSize,
+                       const IMG_CHAR *pszClassName);
+
+PVRSRV_ERROR
+PVRSRVSyncFreeEventKM(CONNECTION_DATA *psConnection,
+					   PVRSRV_DEVICE_NODE *psDevNode,
+					   IMG_UINT32 ui32FWAddr);
+
+PVRSRV_ERROR
+PVRSRVSyncRecordAddKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDevNode,
+					  SYNC_RECORD_HANDLE *phRecord,
+					  SYNC_PRIMITIVE_BLOCK *hServerSyncPrimBlock,
+					  IMG_UINT32 ui32FwBlockAddr,
+					  IMG_UINT32 ui32SyncOffset,
+					  IMG_BOOL bServerSync,
+					  IMG_UINT32 ui32ClassNameSize,
+					  const IMG_CHAR *pszClassName);
+
+PVRSRV_ERROR
+PVRSRVSyncRecordRemoveByHandleKM(
+			SYNC_RECORD_HANDLE hRecord);
+void SyncRecordLookup(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 ui32FwAddr,
+					  IMG_CHAR * pszSyncInfo, size_t len);
+
+PVRSRV_ERROR SyncRegisterConnection(SYNC_CONNECTION_DATA **ppsSyncConnectionData);
+void SyncUnregisterConnection(SYNC_CONNECTION_DATA *ppsSyncConnectionData);
+void SyncConnectionPDumpSyncBlocks(PVRSRV_DEVICE_NODE *psDevNode, void *hSyncPrivData, PDUMP_TRANSITION_EVENT eEvent);
+
+/*!
+******************************************************************************
+@Function      SyncServerInit
+
+@Description   Per-device initialisation for the ServerSync module
+******************************************************************************/
+PVRSRV_ERROR SyncServerInit(PVRSRV_DEVICE_NODE *psDevNode);
+void SyncServerDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+							IMG_UINT32 ui32Value);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiDumpFlags);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize);
+
+#else	/* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpValueKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+							IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpPolKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpCBPKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif	/* PDUMP */
+#endif	/*SYNC_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/tlintern.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlintern.h
new file mode 100644
index 000000000000..114f90f98548
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlintern.h
@@ -0,0 +1,347 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer internals
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer header used by TL internally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TLINTERN_H
+#define TLINTERN_H
+
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+#include "lock.h"
+#include "tlstream.h"
+
+/* Forward declarations */
+typedef struct _TL_SNODE_* PTL_SNODE;
+
+/* To debug buffer utilisation enable this macro here and define
+ * PVRSRV_NEED_PVR_TRACE in the server pvr_debug.c and in tutils.c
+ * before the inclusion of pvr_debug.h.
+ * Issue pvrtutils 6 on target to see stream buffer utilisation. */
+//#define TL_BUFFER_STATS 1
+
+/*! TL stream structure container.
+ *    pbyBuffer   holds the circular buffer.
+ *    ui32Read    points to the beginning of the buffer, ie to where data to
+ *                  Read begin.
+ *    ui32Write   points to the end of data that have been committed, ie this is
+ *                  where new data will be written.
+ *    ui32Pending number of bytes reserved in last reserve call which have not
+ *                  yet been submitted. Therefore these data are not ready to
+ *                  be transported.
+ *    hStreamWLock - provides atomic protection for the ui32Pending & ui32Write
+ *                   members of the structure for when they are checked and/or
+ *                   updated in the context of a stream writer (producer)
+ *                   calling DoTLStreamReserve() & TLStreamCommit().
+ *                 - Reader context is not multi-threaded, only one client per
+ *                   stream is allowed. Also note the read context may be in an
+ *                   ISR which prevents a design where locks can be held in the
+ *                   AcquireData/ReleaseData() calls. Thus this lock only
+ *                   protects the stream members from simultaneous writers.
+ *
+ *      ui32Read < ui32Write <= ui32Pending
+ *        where < and <= operators are overloaded to make sense in a circular way.
+ */
+typedef struct _TL_STREAM_
+{
+	IMG_CHAR                szName[PRVSRVTL_MAX_STREAM_NAME_SIZE];  /*!< String name identifier */
+	TL_OPMODE               eOpMode;                                /*!< Mode of Operation of TL Buffer */
+
+	IMG_BOOL                bWaitForEmptyOnDestroy;                 /*!< Flag: On destroying a non-empty stream block until
+                                                                         *         stream is drained. */
+	IMG_BOOL                bNoSignalOnCommit;                      /*!< Flag: Used to avoid the TL signalling waiting consumers
+                                                                         *         that new data is available on every commit. Producers
+                                                                         *         using this flag will need to manually signal when
+                                                                         *         appropriate using the TLStreamSync() API */
+
+	void                    (*pfOnReaderOpenCallback)(void *pvArg); /*!< Optional on reader connect callback */
+	void                    *pvOnReaderOpenUserData;                /*!< On reader connect user data */
+	void                    (*pfOnReaderCloseCallback)(void *pvArg);/*!< Optional on reader disconnect callback */
+	void                    *pvOnReaderCloseUserData;               /*!< On reader disconnect user data */
+	void                    (*pfProducerCallback)(void);            /*!< Optional producer callback of type TL_STREAM_SOURCECB */
+	void                    *pvProducerUserData;                    /*!< Producer callback user data */
+
+	struct _TL_STREAM_      *psNotifStream;                         /*!< Pointer to the stream to which notification will be sent */
+
+	volatile IMG_UINT32     ui32Read;                               /*!< Pointer to the beginning of available data */
+	volatile IMG_UINT32     ui32Write;                              /*!< Pointer to already committed data which are ready to be
+                                                                         *   copied to user space */
+	IMG_UINT32              ui32Pending;                            /*!< Count pending bytes reserved in buffer */
+	IMG_UINT32              ui32Size;                               /*!< Buffer size */
+	IMG_UINT32              ui32ThresholdUsageForSignal;            /*!< Buffer usage threshold at which a TL writer signals a blocked/
+	                                                                     *    waiting reader when transitioning from empty->non-empty */
+	IMG_UINT32              ui32MaxPacketSize;                      /*! Max TL packet size */
+	IMG_BYTE                *pbyBuffer;                             /*!< Actual data buffer */
+
+	PTL_SNODE               psNode;                                 /*!< Ptr to parent stream node */
+	DEVMEM_MEMDESC          *psStreamMemDesc;                       /*!< MemDescriptor used to allocate buffer space through PMR */
+
+	IMG_HANDLE              hProducerEvent;	                        /*!< Handle to wait on if there is not enough space */
+	IMG_HANDLE              hProducerEventObj;                      /*!< Handle to signal blocked reserve calls */
+	IMG_BOOL                bSignalPending;                         /*!< Tracks if a "signal" is pending to be sent to a blocked/
+	                                                                     *    waiting reader */
+
+	POS_LOCK                hStreamWLock;                           /*!< Writers Lock for ui32Pending & ui32Write*/
+	POS_LOCK                hReadLock;                              /*!< Readers Lock for bReadPending & ui32Read*/
+	IMG_BOOL                bReadPending;                           /*!< Tracks if a read operation is pending or not*/
+	IMG_BOOL                bNoWrapPermanent;                       /*!< Flag: Prevents buffer wrap and subsequent data loss
+	                                                                     *    as well as resetting the read position on close. */
+
+#if defined(TL_BUFFER_STATS)
+	IMG_UINT32              ui32CntReadFails;                       /*!< Tracks how many times reader failed to acquire read lock */
+	IMG_UINT32              ui32CntReadSuccesses;                   /*!< Tracks how many times reader acquires read lock successfully */
+	IMG_UINT32              ui32CntWriteSuccesses;                  /*!< Tracks how many times writer acquires read lock successfully */
+	IMG_UINT32              ui32CntWriteWaits;                      /*!< Tracks how many times writer had to wait to acquire read lock */
+	IMG_UINT32              ui32CntNumWriteSuccess;	                /*!< Tracks how many write operations were successful*/
+	IMG_UINT32              ui32BufferUt;                           /*!< Buffer utilisation high watermark, see TL_BUFFER_STATS above */
+	IMG_UINT32              ui32MaxReserveWatermark;                /*!< Max stream reserve size that was ever requested by a writer */
+	IMG_UINT32              ui32SignalsSent;                        /*!< Number of signals that were actually sent by the write API */
+	ATOMIC_T                bNoReaderSinceFirstReserve;             /*!< Tracks if a read has been done since the buffer was last found empty */
+	IMG_UINT32              ui32TimeStart;                          /*!< Time at which a write (Reserve call) was done into an empty buffer.
+	                                                                     *    Guarded by hStreamWLock. */
+	IMG_UINT32              ui32MinTimeToFullInUs;                  /*!< Minimum time taken to (nearly) fully fill an empty buffer. Guarded
+	                                                                     *    by hStreamWLock. */
+	/* Behaviour counters, protected by hStreamLock in case of
+	 * multi-threaded access */
+	IMG_UINT32              ui32NumCommits;     /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32SignalNotSent;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32ManSyncs;       /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32ProducerByteCount; /*!< Counters used to analysing stream performance, see ++ loc */
+
+	/* Not protected by the lock, inc in the reader thread which is currently singular */
+	IMG_UINT32              ui32AcquireRead1;   /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32AcquireRead2;   /*!< Counters used to analysing stream performance, see ++ loc */
+#endif
+
+} TL_STREAM, *PTL_STREAM;
+
+/* there need to be enough space reserved in the buffer for 2 minimal packets
+ * and it needs to be aligned the same way the buffer is or there will be a
+ * compile error.*/
+#define BUFFER_RESERVED_SPACE (2 * PVRSRVTL_PACKET_ALIGNMENT)
+
+/* ensure the space reserved follows the buffer's alignment */
+static_assert(!(BUFFER_RESERVED_SPACE&(PVRSRVTL_PACKET_ALIGNMENT-1)),
+			  "BUFFER_RESERVED_SPACE must be a multiple of PVRSRVTL_PACKET_ALIGNMENT");
+
+/* Define the largest value that a uint that matches the
+ * PVRSRVTL_PACKET_ALIGNMENT size can hold */
+#define MAX_UINT 0xffffFFFF
+
+/*! Defines the value used for TL_STREAM.ui32Pending when no reserve is
+ * outstanding on the stream. */
+#define NOTHING_PENDING IMG_UINT32_MAX
+
+
+/*
+ * Transport Layer Stream Descriptor types/defs
+ */
+typedef struct _TL_STREAM_DESC_
+{
+	PTL_SNODE   psNode;         /*!< Ptr to parent stream node */
+	IMG_UINT32  ui32Flags;      /*!< Flags supplied by client on stream open */
+	IMG_HANDLE  hReadEvent;     /*!< For wait call (only used/set in reader descriptors) */
+	IMG_INT     uiRefCount;     /*!< Reference count to the SD */
+
+#if defined(TL_BUFFER_STATS)
+	/* Behaviour counters, no multi-threading protection need as they are
+	 * incremented in a single thread due to only supporting one reader
+	 * at present */
+	IMG_UINT32  ui32AcquireCount;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32NoData;        /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32NoDataSleep;   /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32Signalled;     /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32TimeoutEmpty;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32TimeoutData;   /*!< Counters used to analysing stream performance, see ++ loc */
+#endif
+	IMG_UINT32  ui32ReadLimit;     /*!< Limit buffer reads to data present in the
+                                        buffer at the time of stream open. */
+	IMG_UINT32  ui32ReadLen;       /*!< Size of data returned by initial Acquire */
+} TL_STREAM_DESC, *PTL_STREAM_DESC;
+
+PTL_STREAM_DESC TLMakeStreamDesc(PTL_SNODE f1, IMG_UINT32 f2, IMG_HANDLE f3);
+
+#define TL_STREAM_KM_FLAG_MASK	0xFFFF0000
+#define TL_STREAM_FLAG_TEST		0x10000000
+#define TL_STREAM_FLAG_WRAPREAD	0x00010000
+
+#define TL_STREAM_UM_FLAG_MASK	0x0000FFFF
+
+#if defined(TL_BUFFER_STATS)
+#	define TL_COUNTER_INC(a)    ((a)++)
+#	define TL_COUNTER_ADD(a,b)  ((a) += (b))
+#else
+#	define TL_COUNTER_INC(a)    (void)(0)
+#	define TL_COUNTER_ADD(a,b)  (void)(0)
+#endif
+/*
+ * Transport Layer stream list node
+ */
+typedef struct _TL_SNODE_
+{
+	struct _TL_SNODE_*	psNext;				/*!< Linked list next element */
+	IMG_HANDLE			hReadEventObj;		/*!< Readers 'wait for data' event */
+	PTL_STREAM			psStream;			/*!< TL Stream object */
+	IMG_INT				uiWRefCount;		/*!< Stream writer reference count */
+	PTL_STREAM_DESC		psRDesc;			/*!< Stream reader 0 or ptr only */
+	PTL_STREAM_DESC		psWDesc;			/*!< Stream writer 0 or ptr only */
+} TL_SNODE;
+
+PTL_SNODE TLMakeSNode(IMG_HANDLE f2, TL_STREAM *f3, TL_STREAM_DESC *f4);
+
+/*
+ * Transport Layer global top types and variables
+ * Use access function to obtain pointer.
+ *
+ * hTLGDLock - provides atomicity over read/check/write operations and
+ *             sequence of operations on uiClientCnt, psHead list of SNODEs and
+ *             the immediate members in a list element SNODE structure.
+ *           - This larger scope of responsibility for this lock helps avoid
+ *             the need for a lock in the SNODE structure.
+ *           - Lock held in the client (reader) context when streams are
+ *             opened/closed and in the server (writer) context when streams
+ *             are created/open/closed.
+ */
+typedef struct _TL_GDATA_
+{
+	IMG_HANDLE hTLEventObj;        /* Global TL signal object, new streams, etc */
+
+	IMG_UINT   uiClientCnt;        /* Counter to track the number of client stream connections. */
+	PTL_SNODE  psHead;             /* List of TL streams and associated client handle */
+
+	POS_LOCK   hTLGDLock;          /* Lock for structure AND psHead SNODE list */
+} TL_GLOBAL_DATA, *PTL_GLOBAL_DATA;
+
+/*
+ * Transport Layer Internal Kernel-Mode Server API
+ */
+TL_GLOBAL_DATA* TLGGD(void);		/* TLGetGlobalData() */
+
+PVRSRV_ERROR TLInit(void);
+void TLDeInit(void);
+
+void TLAddStreamNode(PTL_SNODE psAdd);
+PTL_SNODE TLFindStreamNodeByName(const IMG_CHAR *pszName);
+PTL_SNODE TLFindStreamNodeByDesc(PTL_STREAM_DESC psDesc);
+IMG_UINT32 TLDiscoverStreamNodes(const IMG_CHAR *pszNamePattern,
+                          IMG_CHAR aaszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+                          IMG_UINT32 ui32Max);
+PTL_SNODE TLFindAndGetStreamNodeByDesc(PTL_STREAM_DESC psDesc);
+void TLReturnStreamNode(PTL_SNODE psNode);
+
+/******************************************************************************
+ Function Name	: TLTryRemoveStreamAndFreeStreamNode
+
+ Inputs		: PTL_SNODE	Pointer to the TL_SNODE whose stream is requested
+			to be removed from TL_GLOBAL_DATA's list
+
+ Return Value	: IMG_TRUE	-	If the stream was made NULL and this
+					TL_SNODE was removed from the
+					TL_GLOBAL_DATA's list
+
+		  IMG_FALSE	-	If the stream wasn't made NULL as there
+					is a client connected to this stream
+
+ Description	: If there is no client currently connected to this stream then,
+			This function removes this TL_SNODE from the
+			TL_GLOBAL_DATA's list. The caller is responsible for the
+			cleanup of the TL_STREAM whose TL_SNODE may be removed
+
+		  Otherwise, this function does nothing
+******************************************************************************/
+IMG_BOOL  TLTryRemoveStreamAndFreeStreamNode(PTL_SNODE psRemove);
+
+/******************************************************************************
+ Function Name	: TLUnrefDescAndTryFreeStreamNode
+
+ Inputs		: PTL_SNODE	Pointer to the TL_SNODE whose descriptor is
+			requested to be removed
+			: PTL_STREAM_DESC	Pointer to the STREAM_DESC
+
+ Return Value	: IMG_TRUE	-	If this	TL_SNODE was removed from the
+					TL_GLOBAL_DATA's list
+
+		  IMG_FALSE	-	Otherwise
+
+ Description	: This function removes the stream descriptor from this TL_SNODE
+			and, if there is no writer (producer context) currently bound to this
+			stream, this function removes this TL_SNODE from the TL_GLOBAL_DATA's
+			list. The caller is responsible for the cleanup of the TL_STREAM
+			whose TL_SNODE may be removed
+******************************************************************************/
+IMG_BOOL TLUnrefDescAndTryFreeStreamNode(PTL_SNODE psRemove, PTL_STREAM_DESC psSD);
+
+/*
+ * Transport Layer stream interface to server part declared here to avoid
+ * circular dependency.
+ */
+IMG_UINT32 TLStreamAcquireReadPos(PTL_STREAM psStream,
+                                  IMG_BOOL bDisableCallback,
+                                  IMG_UINT32* puiReadOffset);
+PVRSRV_ERROR TLStreamAdvanceReadPos(PTL_STREAM psStream,
+                                    IMG_UINT32 uiReadLen,
+                                    IMG_UINT32 uiOrigReadLen);
+void TLStreamResetReadPos(PTL_STREAM psStream);
+
+DEVMEM_MEMDESC* TLStreamGetBufferPointer(PTL_STREAM psStream);
+IMG_BOOL TLStreamOutOfData(IMG_HANDLE psStream);
+
+/******************************************************************************
+ Function Name	: TLStreamDestroy
+
+ Inputs		: PTL_STREAM	Pointer to the TL_STREAM to be destroyed
+
+ Description	: This function performs all the clean-up operations required for
+			destruction of this stream
+******************************************************************************/
+void TLStreamDestroy(PTL_STREAM psStream);
+
+/*
+ * Test related functions
+ */
+PVRSRV_ERROR TUtilsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR TUtilsDeinit(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+
+#endif /* TLINTERN_H */
+/******************************************************************************
+ End of file (tlintern.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/tlserver.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlserver.h
new file mode 100644
index 000000000000..7ac2958eac1e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlserver.h
@@ -0,0 +1,97 @@
+/*************************************************************************/ /*!
+@File
+@Title          KM server Transport Layer implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main bridge APIs for Transport Layer client functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef TLSERVER_H
+#define TLSERVER_H
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+
+#include "tlintern.h"
+
+/*
+ * Transport Layer Client API Kernel-Mode bridge implementation
+ */
+
+PVRSRV_ERROR TLServerConnectKM(CONNECTION_DATA *psConnection);
+PVRSRV_ERROR TLServerDisconnectKM(CONNECTION_DATA *psConnection);
+
+PVRSRV_ERROR TLServerOpenStreamKM(const IMG_CHAR* pszName,
+			   IMG_UINT32 ui32Mode,
+			   PTL_STREAM_DESC* ppsSD,
+			   PMR** ppsTLPMR);
+
+PVRSRV_ERROR TLServerCloseStreamKM(PTL_STREAM_DESC psSD);
+
+PVRSRV_ERROR TLServerDiscoverStreamsKM(const IMG_CHAR *pszNamePattern,
+                          IMG_UINT32 ui32Max,
+                          IMG_CHAR *pszStreams,
+                          IMG_UINT32 *pui32NumFound);
+
+PVRSRV_ERROR TLServerReserveStreamKM(PTL_STREAM_DESC psSD,
+                                     IMG_UINT32* ui32BufferOffset,
+                                     IMG_UINT32 ui32Size,
+                                     IMG_UINT32 ui32SizeMin,
+                                     IMG_UINT32* pui32Available);
+
+PVRSRV_ERROR TLServerCommitStreamKM(PTL_STREAM_DESC psSD,
+                                    IMG_UINT32 ui32Size);
+
+PVRSRV_ERROR TLServerAcquireDataKM(PTL_STREAM_DESC psSD,
+			   IMG_UINT32* puiReadOffset,
+			   IMG_UINT32* puiReadLen);
+
+PVRSRV_ERROR TLServerReleaseDataKM(PTL_STREAM_DESC psSD,
+				 IMG_UINT32 uiReadOffset,
+				 IMG_UINT32 uiReadLen);
+
+PVRSRV_ERROR TLServerWriteDataKM(PTL_STREAM_DESC psSD,
+                                 IMG_UINT32 ui32Size,
+                                 IMG_BYTE *pui8Data);
+
+#endif /* TLSERVER_H */
+
+/******************************************************************************
+ End of file (tlserver.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/tlstream.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlstream.h
new file mode 100644
index 000000000000..aa237bba4d79
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlstream.h
@@ -0,0 +1,619 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    TL provides driver components with a way to copy data from kernel
+                space to user space (e.g. screen/file).
+
+                Data can be passed to the Transport Layer through the
+                TL Stream (kernel space) API interface.
+
+                The buffer provided to every stream is a modified version of a
+                circular buffer. Which CB version is created is specified by
+                relevant flags when creating a stream. Currently two types
+                of buffer are available:
+                - TL_OPMODE_DROP_NEWER:
+                  When the buffer is full, incoming data are dropped
+                  (instead of overwriting older data) and a marker is set
+                  to let the user know that data have been lost.
+                - TL_OPMODE_BLOCK:
+                  When the circular buffer is full, reserve/write calls block
+                  until enough space is freed.
+                - TL_OPMODE_DROP_OLDEST:
+                  When the circular buffer is full, the oldest packets in the
+                  buffer are dropped and a flag is set in header of next packet
+                  to let the user know that data have been lost.
+
+                All size/space requests are in bytes. However, the actual
+                implementation uses native word sizes (i.e. 4 byte aligned).
+
+                The user does not need to provide space for the stream buffer
+                as the TL handles memory allocations and usage.
+
+                Inserting data to a stream's buffer can be done either:
+                - by using TLReserve/TLCommit: User is provided with a buffer
+                                                 to write data to.
+                - or by using TLWrite:         User provides a buffer with
+                                                 data to be committed. The TL
+                                                 copies the data from the
+                                                 buffer into the stream buffer
+                                                 and returns.
+                Users should be aware that there are implementation overheads
+                associated with every stream buffer. If you find that less
+                data are captured than expected then try increasing the
+                stream buffer size or use TLInfo to obtain buffer parameters
+                and calculate optimum required values at run time.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TLSTREAM_H
+#define TLSTREAM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_tlcommon.h"
+#include "device.h"
+
+/*! Extract TL stream opmode from the given stream create flags.
+ * Last 3 bits of streamFlag is used for storing opmode, hence
+ * opmode mask is set as following. */
+#define TL_OPMODE_MASK 0x7
+
+/*
+ * NOTE: This enum is used to directly access the HTB_OPMODE_xxx values
+ * within htbserver.c.
+ * As such we *MUST* keep the values matching in order of declaration.
+ */
+/*! Opmode specifying circular buffer behaviour */
+typedef enum
+{
+	/*! Undefined operation mode */
+	TL_OPMODE_UNDEF = 0,
+
+	/*! Reject new data if the buffer is full, producer may then decide to
+	 *    drop the data or retry after some time. */
+	TL_OPMODE_DROP_NEWER,
+
+	/*! When buffer is full, advance the tail/read position to accept the new
+	 * reserve call (size permitting), effectively overwriting the oldest
+	 * data in the circular buffer. Not supported yet. */
+	TL_OPMODE_DROP_OLDEST,
+
+	/*! Block Reserve (subsequently Write) calls if there is not enough space
+	 *    until some space is freed via a client read operation. */
+	TL_OPMODE_BLOCK,
+
+	/*!< For error checking */
+	TL_OPMODE_LAST
+
+} TL_OPMODE;
+
+typedef enum {
+	/* Enum to be used in conjunction with new Flags feature */
+
+	/* Flag set when Drop Oldest is set and packets have been dropped */
+	TL_FLAG_OVERWRITE_DETECTED = (1 << 0),
+	/* Prevents DoTLStreamReserve() from adding from injecting
+	 * PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED */
+	TL_FLAG_NO_WRITE_FAILED = (1 << 1),
+} TL_Flags;
+
+static_assert(TL_OPMODE_LAST <= TL_OPMODE_MASK,
+	      "TL_OPMODE_LAST must not exceed TL_OPMODE_MASK");
+
+/*! Flags specifying stream behaviour */
+/*! Do not destroy stream if there still are data that have not been
+ *     copied in user space. Block until the stream is emptied. */
+#define TL_FLAG_FORCE_FLUSH            (1U<<8)
+/*! Do not signal consumers on commit automatically when the stream buffer
+ * transitions from empty to non-empty. Producer responsible for signal when
+ * it chooses. */
+#define TL_FLAG_NO_SIGNAL_ON_COMMIT    (1U<<9)
+
+/*! When a stream has this property it never wraps around and
+ * overwrites existing data, hence it is a fixed size persistent
+ * buffer, data written is permanent. Producers need to ensure
+ * the buffer is big enough for their needs.
+ * When a stream is opened for reading the client will always
+ * find the read position at the start of the buffer/data. */
+#define TL_FLAG_PERMANENT_NO_WRAP      (1U<<10)
+
+/*! Defer allocation of stream's shared memory until first open. */
+#define TL_FLAG_ALLOCATE_ON_FIRST_OPEN (1U<<11)
+
+/*! Structure used to pass internal TL stream sizes information to users.*/
+typedef struct _TL_STREAM_INFO_
+{
+    IMG_UINT32 headerSize;          /*!< Packet header size in bytes */
+    IMG_UINT32 minReservationSize;  /*!< Minimum data size reserved in bytes */
+    IMG_UINT32 pageSize;            /*!< Page size in bytes */
+    IMG_UINT32 pageAlign;           /*!< Page alignment in bytes */
+    IMG_UINT32 maxTLpacketSize;     /*! Max allowed TL packet size*/
+} TL_STREAM_INFO, *PTL_STREAM_INFO;
+
+/*! Callback operations or notifications that a stream producer may handle
+ * when requested by the Transport Layer.
+ */
+#define TL_SOURCECB_OP_CLIENT_EOS 0x01  /*!< Client has reached end of stream,
+                                         * can anymore data be supplied?
+                                         * ui32Resp ignored in this operation */
+
+/*! Function pointer type for the callback handler into the "producer" code
+ * that writes data to the TL stream.  Producer should handle the notification
+ * or operation supplied in ui32ReqOp on stream hStream. The
+ * Operations and notifications are defined above in TL_SOURCECB_OP */
+typedef PVRSRV_ERROR (*TL_STREAM_SOURCECB)(IMG_HANDLE hStream,
+		IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser);
+
+typedef void (*TL_STREAM_ONREADEROPENCB)(void *pvArg);
+typedef void (*TL_STREAM_ONREADERCLOSECB)(void *pvArg);
+
+/*************************************************************************/ /*!
+ @Function      TLAllocSharedMemIfNull
+ @Description   Allocates shared memory for the stream.
+ @Input         hStream     Stream handle.
+ @Return        eError      Internal services call returned eError error
+                            number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLAllocSharedMemIfNull(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLFreeSharedMem
+ @Description   Frees stream's shared memory.
+ @Input         phStream    Stream handle.
+*/ /**************************************************************************/
+void
+TLFreeSharedMem(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamCreate
+ @Description   Request the creation of a new stream and open a handle.
+				If creating a stream which should continue to exist after the
+				current context is finished, then TLStreamCreate must be
+				followed by a TLStreamOpen call. On any case, the number of
+				create/open calls must balance with the number of close calls
+				used. This ensures the resources of a stream are released when
+				it is no longer required.
+ @Output        phStream        Pointer to handle to store the new stream.
+ @Input         szStreamName    Name of stream, maximum length:
+                                PRVSRVTL_MAX_STREAM_NAME_SIZE.
+                                If a longer string is provided,creation fails.
+ @Input         ui32Size        Desired buffer size in bytes.
+ @Input         ui32StreamFlags Used to configure buffer behaviour. See above.
+ @Input         pfOnReaderOpenCB    Optional callback called when a client
+                                    opens this stream, may be null.
+ @Input         pvOnReaderOpenUD    Optional user data for pfOnReaderOpenCB,
+                                    may be null.
+ @Input         pfOnReaderCloseCB   Optional callback called when a client
+                                    closes this stream, may be null.
+ @Input         pvOnReaderCloseUD   Optional user data for pfOnReaderCloseCB,
+                                    may be null.
+ @Input         pfProducerCB    Optional callback, may be null.
+ @Input         pvProducerUD    Optional user data for callback, may be null.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle or string name
+                                             exceeded MAX_STREAM_NAME_SIZE
+ @Return        PVRSRV_ERROR_OUT_OF_MEMORY   Failed to allocate space for
+                                             stream handle.
+ @Return        PVRSRV_ERROR_DUPLICATE_VALUE There already exists a stream with
+                                             the same stream name string.
+ @Return        eError                       Internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamCreate(IMG_HANDLE *phStream,
+               const IMG_CHAR *szStreamName,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32StreamFlags,
+               TL_STREAM_ONREADEROPENCB pfOnReaderOpenCB,
+               void *pvOnReaderOpenUD,
+               TL_STREAM_ONREADERCLOSECB pfOnReaderCloseCB,
+               void *pvOnReaderCloseUD,
+               TL_STREAM_SOURCECB pfProducerCB,
+               void *pvProducerUD);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamOpen
+ @Description   Attach to existing stream that has already been created by a
+                  TLStreamCreate call. A handle is returned to the stream.
+ @Output        phStream        Pointer to handle to store the stream.
+ @Input         szStreamName    Name of stream, should match an already
+                                  existing stream name
+ @Return        PVRSRV_ERROR_NOT_FOUND       None of the streams matched the
+                                             requested stream name.
+				PVRSRV_ERROR_INVALID_PARAMS  Non-NULL pointer to stream
+                                             handler is required.
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamOpen(IMG_HANDLE     *phStream,
+             const IMG_CHAR *szStreamName);
+
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReset
+ @Description   Resets read and write pointers and pending flag.
+ @Output        phStream Pointer to stream's handle
+*/ /**************************************************************************/
+void TLStreamReset(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamSetNotifStream
+ @Description   Registers a "notification stream" which will be used to
+                publish information about state change of the "hStream"
+                stream. Notification can inform about events such as stream
+                open/close, etc.
+ @Input         hStream         Handle to stream to update.
+ @Input         hNotifStream    Handle to the stream which will be used for
+                                publishing notifications.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  If either of the parameters is
+                                             NULL
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamSetNotifStream(IMG_HANDLE hStream, IMG_HANDLE hNotifStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReconfigure
+ @Description   Request the stream flags controlling buffer behaviour to
+                be updated.
+                In the case where TL_OPMODE_BLOCK is to be used,
+                TLStreamCreate should be called without that flag and this
+                function used to change the stream mode once a consumer process
+                has been started. This avoids a deadlock scenario where the
+                TLStreaWrite/TLStreamReserve call will hold the Bridge Lock
+                while blocking if the TL buffer is full.
+                The TL_OPMODE_BLOCK should never drop the Bridge Lock
+                as this leads to another deadlock scenario where the caller to
+                TLStreamWrite/TLStreamReserve has already acquired another lock
+                (e.g. gHandleLock) which is not dropped. This then leads to that
+                thread acquiring locks out of order.
+ @Input         hStream         Handle to stream to update.
+ @Input         ui32StreamFlags Flags that configure buffer behaviour. See above.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle or inconsistent
+                                             stream flags.
+ @Return        PVRSRV_ERROR_NOT_READY       Stream is currently being written to
+                                             try again later.
+ @Return        eError                       Internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReconfigure(IMG_HANDLE hStream,
+                    IMG_UINT32 ui32StreamFlags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamClose
+ @Description   Detach from the stream associated with the given handle. If
+                  the current handle is the last one accessing the stream
+				  (i.e. the number of TLStreamCreate+TLStreamOpen calls matches
+				  the number of TLStreamClose calls) then the stream is also
+				  deleted.
+				On return the handle is no longer valid.
+ @Input         hStream     Handle to stream that will be closed.
+ @Return        None.
+*/ /**************************************************************************/
+void
+TLStreamClose(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserve
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Number of bytes to reserve in buffer.
+ @Return        PVRSRV_INVALID_PARAMS       NULL stream handler.
+ @Return        PVRSRV_ERROR_NOT_READY      There are data previously reserved
+                                              that are pending to be committed.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE  Misusing the stream by trying to
+                                              reserve more space than the
+                                              buffer size.
+ @Return        PVRSRV_ERROR_STREAM_FULL    The reserve size requested
+                                            is larger than the free
+                                            space.
+ @Return         PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED  The reserve size
+                                                            requested is larger
+                                                            than max TL packet size
+ @Return        PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE Permanent stream buffer
+                                                     does not have enough space
+                                                     for the reserve.
+ @Return        PVRSRV_OK                   Success, output arguments valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserve(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserve2
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Ideal number of bytes to reserve in buffer.
+ @Input         ui32SizeMin     Minimum number of bytes to reserve in buffer.
+ @Input         pui32Available  Optional, but when present and the
+                                  RESERVE_TOO_BIG error is returned, a size
+                                  suggestion is returned in this argument which
+                                  the caller can attempt to reserve again for a
+                                  successful allocation.
+ @Output        pbIsReaderConnected Let writing clients know if reader is
+                                    connected or not, in case of error.
+ @Return        PVRSRV_INVALID_PARAMS        NULL stream handler.
+ @Return        PVRSRV_ERROR_NOT_READY       There are data previously reserved
+                                             that are pending to be committed.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE   Misusing the stream by trying to
+                                             reserve more space than the
+                                             buffer size.
+ @Return        PVRSRV_ERROR_STREAM_FULL     The reserve size requested
+                                             is larger than the free
+                                             space.
+                                             Check the pui32Available
+                                             value for the correct
+                                             reserve size to use.
+ @Return         PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED   The reserve size
+                                                             requested is larger
+                                                             than max TL packet size
+ @Return        PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE Permanent stream buffer
+                                                     does not have enough space
+                                                     for the reserve.
+ @Return        PVRSRV_OK                   Success, output arguments valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserve2(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size,
+                IMG_UINT32 ui32SizeMin,
+                IMG_UINT32* pui32Available,
+                IMG_BOOL* pbIsReaderConnected);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserveReturnFlags
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Ideal number of bytes to reserve in buffer.
+ @Output        pui32Flags      Output parameter to return flags generated within
+                                the reserve function.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserveReturnFlags(IMG_HANDLE hStream,
+        IMG_UINT8  **ppui8Data,
+        IMG_UINT32 ui32Size,
+		IMG_UINT32* pui32Flags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamGetUT
+ @Description   Returns the current stream utilisation in bytes
+ @Input         hStream     Stream handle.
+ @Return        IMG_UINT32  Stream utilisation
+*/ /**************************************************************************/
+IMG_UINT32 TLStreamGetUT(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamCommit
+ @Description   Notify TL that data have been written in the stream buffer.
+                  Should always follow and match TLStreamReserve call.
+ @Input         hStream         Stream handle.
+ @Input         ui32Size        Number of bytes that have been added to the
+                                  stream.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE   Commit results in more data
+                                             committed than the buffer size,
+                                             the stream is misused.
+ @Return        eError                       Commit was successful but
+                                             internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamCommit(IMG_HANDLE hStream,
+               IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamWrite
+ @Description   Combined Reserve/Commit call. This function Reserves space in
+                  the specified stream buffer, copies ui32Size bytes of data
+                  from the array pui8Src points to and Commits in an "atomic"
+                  style operation.
+ @Input         hStream         Stream handle.
+ @Input         pui8Src         Source to read data from.
+ @Input         ui32Size        Number of bytes to copy and commit.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                               Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamWrite(IMG_HANDLE hStream,
+              IMG_UINT8  *pui8Src,
+              IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamWriteRetFlags
+ @Description   Combined Reserve/Commit call. This function Reserves space in
+                  the specified stream buffer, copies ui32Size bytes of data
+                  from the array pui8Src points to and Commits in an "atomic"
+                  style operation. Also accepts a pointer to a bit flag value
+                  for returning write status flags.
+ @Input         hStream         Stream handle.
+ @Input         pui8Src         Source to read data from.
+ @Input         ui32Size        Number of bytes to copy and commit.
+ @Output        pui32Flags      Output parameter for write status info
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                               Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamWriteRetFlags(IMG_HANDLE hStream,
+                      IMG_UINT8 *pui8Src,
+					  IMG_UINT32 ui32Size,
+					  IMG_UINT32 *pui32Flags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamSync
+ @Description   Signal the consumer to start acquiring data from the stream
+                buffer. Called by producers that use the flag
+                TL_FLAG_NO_SIGNAL_ON_COMMIT to manually control when
+                consumers starting reading the stream.
+                Used when multiple small writes need to be batched.
+ @Input         hStream         Stream handle.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle.
+ @Return        eError                       Error codes returned by either
+                                             Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamSync(IMG_HANDLE hStream);
+
+
+/*************************************************************************/ /*!
+ @Function      TLStreamMarkEOS
+ @Description   Insert a EOS marker packet in the given stream.
+ @Input         hStream         Stream handle.
+ @Input         bRemoveOld      if TRUE, remove old stream record file before
+                                splitting to new file.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                             Reserve or Commit.
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkEOS(IMG_HANDLE hStream, IMG_BOOL bRemoveOld);
+
+/*************************************************************************/ /*!
+@Function       TLStreamMarkStreamOpen
+@Description    Puts *open* stream packet into hStream's notification stream,
+                if set, error otherwise."
+@Input          hStream Stream handle.
+@Return         PVRSRV_OK on success and error code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkStreamOpen(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+@Function       TLStreamMarkStreamClose
+@Description    Puts *close* stream packet into hStream's notification stream,
+                if set, error otherwise."
+@Input          hStream Stream handle.
+@Return         PVRSRV_OK on success and error code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkStreamClose(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamInfo
+ @Description   Run time information about buffer elemental sizes.
+                It sets psInfo members accordingly. Users can use those values
+                to calculate the parameters they use in TLStreamCreate and
+                TLStreamReserve.
+ @Output        psInfo          pointer to stream info structure.
+ @Return        None.
+*/ /**************************************************************************/
+void
+TLStreamInfo(IMG_HANDLE hStream, PTL_STREAM_INFO psInfo);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamIsOpenForReading
+ @Description   Query if a stream has any readers connected.
+ @Input         hStream         Stream handle.
+ @Return        IMG_BOOL        True if at least one reader is connected,
+                                false otherwise
+*/ /**************************************************************************/
+IMG_BOOL
+TLStreamIsOpenForReading(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamOutOfData
+ @Description   Query if the stream is empty (no data waiting to be read).
+ @Input         hStream         Stream handle.
+ @Return        IMG_BOOL        True if read==write, no data waiting,
+                                false otherwise
+*/ /**************************************************************************/
+IMG_BOOL TLStreamOutOfData(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamResetProducerByteCount
+ @Description   Reset the producer byte counter on the specified stream.
+ @Input         hStream         Stream handle.
+ @Input         ui32Value       Value to reset counter to, often 0.
+ @Return        PVRSRV_OK                   Success.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE  Success but the read and write
+                                            positions did not match,
+                                            stream not empty.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR
+TLStreamResetProducerByteCount(IMG_HANDLE hStream, IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamGetMaxTransfer
+ @Description   Obtain the maximum number of bytes that can be submitted to
+                 a given stream.
+ @Input         uiXferSize         Requested transfer size from producer.
+ @Input         hConsumerStream    Stream handle for consumer.
+ @Return        IMG_UINT32         Amount of data that can be submitted to
+                                    the consumer stream.
+*/ /**************************************************************************/
+IMG_UINT32
+TLStreamGetMaxTransfer(IMG_UINT32 uiXferSize, IMG_HANDLE hConsumerStream);
+
+#endif /* TLSTREAM_H */
+/*****************************************************************************
+ End of file (tlstream.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_impl.h
new file mode 100644
index 000000000000..928a3f6bc9ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_impl.h
@@ -0,0 +1,206 @@
+/*************************************************************************/ /*!
+@File           vmm_impl.h
+@Title          Common VM manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides common VM manager definitions that need to
+                be shared by system virtualization layer itself and modules that
+                implement the actual VM manager types.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_IMPL_H
+#define VMM_IMPL_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+
+typedef enum _VMM_CONF_PARAM_
+{
+	VMM_CONF_PRIO_DRV0 = 0,
+	VMM_CONF_PRIO_DRV1 = 1,
+	VMM_CONF_PRIO_DRV2 = 2,
+	VMM_CONF_PRIO_DRV3 = 3,
+	VMM_CONF_PRIO_DRV4 = 4,
+	VMM_CONF_PRIO_DRV5 = 5,
+	VMM_CONF_PRIO_DRV6 = 6,
+	VMM_CONF_PRIO_DRV7 = 7,
+	VMM_CONF_HCS_DEADLINE = 8,
+	VMM_CONF_ISOLATION_GROUP_DRV0 = 9,
+	VMM_CONF_ISOLATION_GROUP_DRV1 = 10,
+	VMM_CONF_ISOLATION_GROUP_DRV2 = 11,
+	VMM_CONF_ISOLATION_GROUP_DRV3 = 12,
+	VMM_CONF_ISOLATION_GROUP_DRV4 = 13,
+	VMM_CONF_ISOLATION_GROUP_DRV5 = 14,
+	VMM_CONF_ISOLATION_GROUP_DRV6 = 15,
+	VMM_CONF_ISOLATION_GROUP_DRV7 = 16,
+	VMM_CONF_TIME_SLICE_DRV0 = 17,
+	VMM_CONF_TIME_SLICE_DRV1 = 18,
+	VMM_CONF_TIME_SLICE_DRV2 = 19,
+	VMM_CONF_TIME_SLICE_DRV3 = 20,
+	VMM_CONF_TIME_SLICE_DRV4 = 21,
+	VMM_CONF_TIME_SLICE_DRV5 = 22,
+	VMM_CONF_TIME_SLICE_DRV6 = 23,
+	VMM_CONF_TIME_SLICE_DRV7 = 24,
+	VMM_CONF_TIME_SLICE_INTERVAL = 25,
+	VMM_CONF_VZ_CONNECTION_COOLDOWN_PERIOD = 26,
+} VMM_CONF_PARAM;
+
+/*
+	Virtual machine manager (hypervisor) para-virtualization (PVZ) connection:
+		- Type is implemented by host and guest drivers
+			- Assumes synchronous function call semantics
+			- Unidirectional semantics
+				- For Host  (vmm -> host)
+				- For Guest (guest -> vmm)
+			- Parameters can be IN/OUT/INOUT
+
+		- Host pvz entries are pre-implemented by IMG
+			- For host implementation, see vmm_pvz_server.c
+			- Called by host side hypercall handler or VMM
+
+		- Guest pvz entries are supplied by 3rd-party
+			- These are specific to hypervisor (VMM) type
+			- These implement the actual hypercalls mechanism
+
+	Para-virtualization (PVZ) call runtime sequence:
+		1 - Guest driver in guest VM calls PVZ function
+		1.1 - Guest PVZ connection calls
+		1.2 - Guest VM Manager type which
+		1.2.1 - Performs any pre-processing like parameter packing, etc.
+		1.2.2 - Issues hypercall (blocking synchronous call)
+
+		2 - VM Manager (hypervisor) receives hypercall
+		2.1 - Hypercall handler:
+		2.1.1 - Performs any pre-processing
+		2.1.2 - If call terminates in VM Manager: perform action and return from hypercall
+		2.1.3 - Otherwise forward to host driver (implementation specific call)
+
+		3 - Host driver receives call from VM Manager
+		3.1 - Host VM manager type:
+		3.1.1 - Performs any pre-processing like parameter unpacking, etc.
+		3.1.2 - Acquires host driver PVZ handler and calls the appropriate entry
+		3.2 - Host PVZ connection calls corresponding host system virtualisation layer
+		3.3 - Host driver system virtualisation layer:
+		3.3.1 - Perform action requested by guest driver
+		3.3.2 - Return to host VM Manager type
+		3.4 - Host VM Manager type:
+		3.4.1 - Prepare to return from hypercall
+		3.4.2 - Perform any post-processing like result packing, etc.
+		3.4.3 - Issue return from hypercall
+
+		4 - VM Manager (hypervisor)
+		4.1 - Perform any post-processing
+		4.2 - Return control to guest driver
+
+		5 - Guest driver in guest VM
+		5.1 - Perform any post-processing like parameter unpacking, etc.
+		5.2 - Continue execution in guest VM
+ */
+typedef struct _VMM_PVZ_CONNECTION_
+{
+	struct {
+		/*
+		   This pair must be implemented if the guest is responsible
+		   for allocating the physical heap that backs its firmware
+		   allocations, this is the default configuration. The physical
+		   heap is allocated within the guest VM IPA space and this
+		   IPA Addr/Size must be translated into the host's IPA space
+		   by the VM manager before forwarding request to host.
+		   If not implemented, return PVRSRV_ERROR_NOT_IMPLEMENTED.
+		 */
+		PVRSRV_ERROR (*pfnMapDevPhysHeap)(IMG_UINT64 ui64Size,
+										  IMG_UINT64 ui64PAddr);
+
+		PVRSRV_ERROR (*pfnUnmapDevPhysHeap)(void);
+	} sClientFuncTab;
+
+	struct {
+		/*
+			Corresponding server side entries to handle guest PVZ calls
+			NOTE:
+				 - Additional PVZ function ui32DriverID parameter
+					 - Driver ID determination is responsibility of VM manager
+					 - Actual Driver ID value must be supplied by VM manager
+						- This can be done either in client/VMM/host side
+					 - Must be done before host pvz function(s) are called
+					 - Host pvz function validates incoming Driver ID values
+		 */
+		PVRSRV_ERROR (*pfnMapDevPhysHeap)(IMG_UINT32 ui32DriverID,
+										  IMG_UINT32 ui32DevID,
+										  IMG_UINT64 ui64Size,
+										  IMG_UINT64 ui64PAddr);
+
+		PVRSRV_ERROR (*pfnUnmapDevPhysHeap)(IMG_UINT32 ui32DriverID,
+											IMG_UINT32 ui32DevID);
+	} sServerFuncTab;
+
+	struct {
+		/*
+		   This is used by the VM manager to report pertinent runtime guest VM
+		   information to the host; these events may in turn be forwarded to
+		   the firmware
+		 */
+		PVRSRV_ERROR (*pfnOnVmOnline)(IMG_UINT32 ui32DriverID,
+									  IMG_UINT32 ui32DevID);
+
+		PVRSRV_ERROR (*pfnOnVmOffline)(IMG_UINT32 ui32DriverID,
+									   IMG_UINT32 ui32DevID);
+
+		PVRSRV_ERROR (*pfnVMMConfigure)(VMM_CONF_PARAM eVMMParamType,
+										IMG_UINT32 ui32ParamValue,
+										IMG_UINT32 ui32DevID);
+
+	} sVmmFuncTab;
+} VMM_PVZ_CONNECTION;
+
+/*!
+*******************************************************************************
+ @Function      VMMCreatePvzConnection() and VMMDestroyPvzConnection()
+ @Description   Both the guest and VM manager call this in order to obtain a
+                PVZ connection to the VM and host respectively; that is, guest
+                calls it to obtain connection to VM, VM calls it to obtain a
+                connection to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig);
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+#endif /* VMM_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_client.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_client.h
new file mode 100644
index 000000000000..205dc4ae93b5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_client.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           vmm_pvz_client.h
+@Title          Guest VM manager client para-virtualization routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides guest VMM client para-virtualization APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_PVZ_CLIENT_H
+#define VMM_PVZ_CLIENT_H
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "vmm_impl.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzClientMapDevPhysHeap
+ @Description   The guest front-end to initiate a pfnMapDevPhysHeap PVZ call
+                to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzClientMapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+*******************************************************************************
+ @Function      PvzClientUnmapDevPhysHeap
+ @Description   The guest front-end to initiate a pfnUnmapDevPhysHeap PVZ call
+                to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzClientUnmapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+#endif /* VMM_PVZ_CLIENT_H */
+
+/******************************************************************************
+ End of file (vmm_pvz_client.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_server.h
new file mode 100644
index 000000000000..02e2683c0c19
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_server.h
@@ -0,0 +1,121 @@
+/*************************************************************************/ /*!
+@File           vmm_pvz_server.h
+@Title          VM manager para-virtualization interface helper routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides API(s) available to VM manager, this must be
+                called to close the loop during guest para-virtualization calls.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_PVZ_SERVER_H
+#define VMM_PVZ_SERVER_H
+
+#include "vmm_impl.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzServerMapDevPhysHeap
+ @Description   The VM manager calls this in response to guest PVZ interface
+                call pfnMapDevPhysHeap.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerMapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						IMG_UINT32 ui32DevID,
+						IMG_UINT64 ui64Size,
+						IMG_UINT64 ui64PAddr);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerUnmapDevPhysHeap
+ @Description   The VM manager calls this in response to guest PVZ interface
+                call pfnUnmapDevPhysHeap.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerUnmapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						  IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerOnVmOnline
+ @Description   The VM manager calls this when guest VM machine comes online.
+                The host driver will initialize the FW if it has not done so
+                already.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerOnVmOnline(IMG_UINT32 ui32DriverID,
+					IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerOnVmOffline
+ @Description   The VM manager calls this when a guest VM machine is about to
+                go offline. The VM manager might have unmapped the GPU kick
+                register for such VM but not the GPU memory until the call
+                returns. Once the function returns, the FW does not hold any
+                reference for such VM and no workloads from it are running in
+                the GPU and it is safe to remove the memory for such VM.
+ @Return        PVRSRV_OK on success. PVRSRV_ERROR_TIMEOUT if for some reason
+                the FW is taking too long to clean-up the resources of the
+                DriverID. Otherwise, a PVRSRV_ERROR code.
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerOnVmOffline(IMG_UINT32 ui32DriverID,
+					 IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerVMMConfigure
+ @Description   The VM manager calls this to configure several parameters like
+                HCS or isolation.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+					  IMG_UINT32 ui32ParamValue,
+					  IMG_UINT32 ui32DevID);
+
+#endif /* VMM_PVZ_SERVER_H */
+
+/******************************************************************************
+ End of file (vmm_pvz_server.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vm.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vm.h
new file mode 100644
index 000000000000..ee4a7fb5b3a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vm.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File			vz_vm.h
+@Title          System virtualization VM support APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides VM management support APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VZ_VM_H
+#define VZ_VM_H
+
+#include "vmm_impl.h"
+
+bool IsVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzOnVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzOnVmOffline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+							 IMG_UINT32 ui32ParamValue,
+							 IMG_UINT32 ui32DevID);
+
+#endif /* VZ_VM_H */
+
+/*****************************************************************************
+ End of file (vz_vm.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vmm_pvz.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vmm_pvz.h
new file mode 100644
index 000000000000..03d5525936b8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vmm_pvz.h
@@ -0,0 +1,79 @@
+/*************************************************************************/ /*!
+@File           vz_vmm_pvz.h
+@Title          System virtualization VM manager management APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides VM manager para-virtz management APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VZ_VMM_PVZ_H
+#define VZ_VMM_PVZ_H
+
+#include "img_types.h"
+#include "vmm_impl.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzConnectionInit() and PvzConnectionDeInit()
+ @Description   PvzConnectionInit initializes the VM manager para-virt
+                which is used subsequently for communication between guest and
+                host; depending on the underlying VM setup, this could either
+                be a hyper-call or cross-VM call
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR PvzConnectionInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+void PvzConnectionDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+*******************************************************************************
+ @Function      PvzConnectionAcquire() and PvzConnectionRelease()
+ @Description   These are to acquire/release a handle to the VM manager
+                para-virtz connection to make a pvz call; on the client, use it
+                to make the actual pvz call and on the server handler /
+                VM manager, use it to complete the processing for the pvz call
+                or make a VM manager to host pvzbridge call
+@Return         VMM_PVZ_CONNECTION* on success. Otherwise NULL
+******************************************************************************/
+VMM_PVZ_CONNECTION* PvzConnectionAcquire(void);
+void PvzConnectionRelease(VMM_PVZ_CONNECTION *psPvzConnection);
+
+#endif /* VZ_VMM_PVZ_H */
+
+/******************************************************************************
+ End of file (vz_vmm_pvz.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem.c
new file mode 100644
index 000000000000..c032094ff0b3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem.c
@@ -0,0 +1,2754 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Front End (nominally Client side part, but now invokable
+                from server too) of device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "devicemem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "osfunc.h"
+#include "osmmap.h"
+#include "devicemem_utils.h"
+#include "client_mm_bridge.h"
+#include "client_cache_bridge.h"
+#include "services_km.h"
+
+#if defined(PDUMP)
+#if defined(__KERNEL__)
+#include "pdump_km.h"
+#else
+#include "pdump_um.h"
+#endif
+#include "devicemem_pdump.h"
+#endif
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "client_ri_bridge.h"
+#endif
+#include "client_devicememhistory_bridge.h"
+#include "info_page_client.h"
+
+#include "rgx_heaps.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "pvr_ricommon.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "srvcore.h"
+#if defined(__linux__)
+#include "linux/kernel.h"
+#endif
+#else
+#include "srvcore_intern.h"
+#include "rgxdefs.h"
+#endif
+
+#if defined(__KERNEL__) && defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+extern PVRSRV_ERROR RIDumpAllKM(void);
+#endif
+
+#if defined(__KERNEL__)
+#define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError)
+#else
+#define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError)
+#endif
+
+#if defined(__KERNEL__)
+/* Derive the virtual from the hPMR */
+static
+IMG_UINT64 _GetPremappedVA(PMR *psPMR, PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+
+	IMG_DEV_PHYADDR sDevAddr;
+	IMG_BOOL bValid;
+	PHYS_HEAP *psPhysHeap = psDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	IMG_DEV_PHYADDR sHeapAddr;
+
+	eError = PhysHeapGetDevPAddr(psPhysHeap, &sHeapAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", fail);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+{
+	if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_UMA ||
+			PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_DMA)
+	{
+		IMG_DEV_PHYADDR sDevPAddrCorrected;
+
+		PhysHeapCpuPAddrToDevPAddr(psPhysHeap, 1, &sDevPAddrCorrected, (IMG_CPU_PHYADDR *)&sHeapAddr);
+		sHeapAddr.uiAddr = sDevPAddrCorrected.uiAddr;
+	}
+}
+#endif
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddr", fail);
+
+	eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sDevAddr, &bValid, DEVICE_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMR_DevPhysAddr");
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddr");
+		goto fail;
+	}
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddr");
+
+	ui64OptionalMapAddress = RGX_FIRMWARE_RAW_HEAP_BASE | (sDevAddr.uiAddr - sHeapAddr.uiAddr);
+
+	PVR_DPF((PVR_DBG_ALLOC, "%s: sDevAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" sHeapAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" => ui64OptionalMapAddress = 0x%"IMG_UINT64_FMTSPECx,
+	         __func__, sDevAddr.uiAddr, sHeapAddr.uiAddr, ui64OptionalMapAddress));
+fail:
+	return ui64OptionalMapAddress;
+}
+#endif
+
+/*****************************************************************************
+ *                    Sub allocation internals                               *
+ *****************************************************************************/
+static INLINE PVRSRV_MEMALLOCFLAGS_T
+DevmemOverrideFlagsOrPassThrough(SHARED_DEV_CONNECTION hDevConnection, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+#if defined(__KERNEL__) && defined(RGX_FEATURE_GPU_CPU_COHERENCY_BIT_MASK)
+	/*
+	 *  Override the requested memory flags of FW allocations only,
+	 *  non-FW allocations pass-through unmodified.
+	 *
+	 *  On ace-lite platforms:
+	 *    - If the allocation is not CPU cached, then there is nothing
+	 *      for the GPU to snoop regardless of the GPU cache setting.
+	 *    - If the allocation is not GPU cached, then the SLC will not
+	 *      be used and will not snoop the CPU even if it is CPU cached.
+	 *    - Therefore only the GPU setting can be upgraded to coherent
+	 *      if it is already GPU cached incoherent and the CPU is cached.
+	 *
+	 *  All other platforms:
+	 *    - Do not modify the allocation flags.
+	 */
+
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDevConnection;
+
+	if (psDevNode->pvDevice != NULL && PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, GPU_CPU_COHERENCY))
+	{
+		if (PVRSRV_CHECK_FW_MAIN(uiFlags))
+		{
+			if ((PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) || PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)) &&
+			         (PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)) &&
+			         PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+			         psDevNode->eDevFabricType == PVRSRV_DEVICE_FABRIC_ACELITE)
+			{
+				/* Upgrade the allocation from GPU cached incoherent to GPU cached coherent. */
+				uiFlags &= ~PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK;
+				uiFlags |= PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hDevConnection);
+#endif
+
+	return uiFlags;
+}
+
+static INLINE void
+CheckAnnotationLength(const IMG_CHAR *pszAnnotation)
+{
+	IMG_UINT32 length = OSStringLength(pszAnnotation);
+
+	if (length >= DEVMEM_ANNOTATION_MAX_LEN)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Annotation \"%s\" has been truncated to %d characters from %d characters",
+				__func__, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1, length));
+	}
+}
+
+static PVRSRV_ERROR
+AllocateDeviceMemory(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiLog2Quantum,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_UINT32 ui32NumPhysChunks,
+		IMG_UINT32 ui32NumVirtChunks,
+		IMG_UINT32 *pui32MappingTable,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		IMG_BOOL bExportable,
+		const IMG_CHAR *pszAnnotation,
+		DEVMEM_IMPORT **ppsImport)
+{
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failAlloc);
+
+	/* check if shift value is not too big (sizeof(1ULL)) */
+	PVR_ASSERT(uiLog2Quantum < sizeof(unsigned long long) * 8);
+	/* Check the size is a multiple of the quantum */
+	PVR_ASSERT((uiSize & ((1ULL<<uiLog2Quantum)-1)) == 0);
+
+	CheckAnnotationLength(pszAnnotation);
+
+	/* Pass only the PMR flags down */
+	uiOutFlags = uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK;
+	eError = BridgePhysmemNewRamBackedPMR(GetBridgeHandle(hDevConnection),
+			uiSize,
+			ui32NumPhysChunks,
+			ui32NumVirtChunks,
+			pui32MappingTable,
+			uiLog2Quantum,
+			uiOutFlags,
+			OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1) + 1,
+			pszAnnotation,
+			OSGetCurrentProcessID(),
+			&hPMR,
+			PDUMP_NONE,
+			&uiOutFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate memory for %s (%s)",
+				__func__,
+				pszAnnotation,
+				PVRSRVGETERRORSTRING(eError)));
+		goto failPMR;
+	}
+
+	uiFlags &= ~PVRSRV_PHYS_HEAP_HINT_MASK;
+	uiFlags |= (uiOutFlags & PVRSRV_PHYS_HEAP_HINT_MASK);
+
+	DevmemImportStructInit(psImport,
+			uiSize,
+			uiAlign,
+			uiFlags,
+			hPMR,
+			bExportable ? DEVMEM_PROPERTIES_EXPORTABLE : 0);
+
+	*ppsImport = psImport;
+	return PVRSRV_OK;
+
+failPMR:
+	DevmemImportDiscard(psImport);
+failAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*****************************************************************************
+ *                    Sub allocation internals                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
+		IMG_UINT32 ui32AllocPageCount,
+		IMG_UINT32 *paui32AllocPageIndices,
+		IMG_UINT32 ui32FreePageCount,
+		IMG_UINT32 *pauiFreePageIndices,
+		SPARSE_MEM_RESIZE_FLAGS uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_IMPORT *psImport;
+	SHARED_DEV_CONNECTION hDevConnection;
+	IMG_HANDLE hPMR;
+	DEVMEM_HEAP *psHeap;
+	IMG_HANDLE hReservation;
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	DEVMEM_PROPERTIES_T uiProperties;
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(psMemDesc->psImport != NULL);
+
+	psImport = psMemDesc->psImport;
+	hDevConnection = psImport->hDevConnection;
+	hPMR = psImport->hPMR;
+	psHeap = psImport->sDeviceImport.psHeap;
+	hReservation = psImport->sDeviceImport.hReservation;
+
+	PVR_ASSERT(hDevConnection != NULL);
+	PVR_ASSERT(hPMR != NULL);
+	PVR_ASSERT(psHeap != NULL);
+	PVR_ASSERT(BITMASK_ANY(uiSparseFlags, SPARSE_RESIZE_BOTH));
+	PVR_ASSERT(hReservation != LACK_OF_RESERVATION_POISON);
+
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	PVR_ASSERT(!BITMASK_HAS(uiProperties, DEVMEM_PROPERTIES_SECURE));
+	PVR_ASSERT(!BITMASK_HAS(uiProperties, DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE));
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount == 0);
+
+	OSLockAcquire(psImport->hLock);
+
+	eError = BridgeChangeSparseMem(GetBridgeHandle(hDevConnection),
+	                               ui32AllocPageCount,
+	                               paui32AllocPageIndices,
+	                               ui32FreePageCount,
+	                               pauiFreePageIndices,
+	                               uiSparseFlags,
+	                               hReservation);
+
+	OSLockRelease(psImport->hLock);
+
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (GetInfoPageDebugFlags(hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistorySparseChange(GetBridgeHandle(hDevConnection),
+		                                   hPMR,
+		                                   psMemDesc->uiOffset,
+		                                   psMemDesc->sDeviceMemDesc.sDevVAddr,
+		                                   psMemDesc->uiAllocSize,
+		                                   psMemDesc->szText,
+		                                   DevmemGetHeapLog2PageSize(psHeap),
+		                                   ui32AllocPageCount,
+		                                   paui32AllocPageIndices,
+		                                   ui32FreePageCount,
+		                                   pauiFreePageIndices,
+		                                   psMemDesc->ui32AllocationIndex,
+		                                   &psMemDesc->ui32AllocationIndex);
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+FreeDeviceMemory(DEVMEM_IMPORT *psImport)
+{
+	DevmemImportStructRelease(psImport);
+}
+
+static PVRSRV_ERROR
+SubAllocImportAlloc(RA_PERARENA_HANDLE hArena,
+		RA_LENGTH_T uiSize,
+		RA_FLAGS_T _flags,
+		RA_LENGTH_T uBaseAlignment,
+		const IMG_CHAR *pszAnnotation,
+		/* returned data */
+		RA_IMPORT *psRAImport)
+{
+	/* When suballocations need a new lump of memory, the RA calls
+	   back here.  Later, in the kernel, we must construct a new PMR
+	   and a pairing between the new lump of virtual memory and the
+	   PMR (whether or not such PMR is backed by physical memory) */
+	DEVMEM_HEAP *psHeap;
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags = (PVRSRV_MEMALLOCFLAGS_T) _flags;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+
+	/* Per-arena private handle is, for us, the heap */
+	psHeap = hArena;
+
+	/* The RA should not have invoked us with a size that is not a
+	   multiple of the quantum anyway */
+	PVR_ASSERT((uiSize & ((1ULL<<psHeap->uiLog2Quantum)-1)) == 0);
+
+	eError = AllocateDeviceMemory(psHeap->psCtx->hDevConnection,
+			psHeap->uiLog2Quantum,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			uBaseAlignment,
+			uiFlags,
+			IMG_FALSE,
+			"PMR sub-allocated",
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failAlloc);
+
+#if defined(PDUMP) && defined(DEBUG)
+#if defined(__KERNEL__)
+	PDUMPCOMMENTWITHFLAGS(PMR_DeviceNode((PMR*)psImport->hPMR), PDUMP_CONT,
+			"Created PMR for sub-allocations with handle ID: 0x%p Annotation: \"%s\" (PID %u)",
+			psImport->hPMR, pszAnnotation, OSGetCurrentProcessID());
+#else
+	PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS,
+			"Created PMR for sub-allocations with handle ID: %p Annotation: \"%s\" (PID %u)",
+			psImport->hPMR, pszAnnotation, OSGetCurrentProcessID());
+#endif
+#else
+	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+#if defined(__KERNEL__)
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice;
+
+		PVR_ASSERT(PVRSRV_CHECK_FW_MAIN(uiFlags));
+
+		/* If allocation is made by the Kernel from the firmware heap, account for it
+		 * under the PVR_SYS_ALLOC_PID.
+		 */
+		if ((psHeap == psDevInfo->psFirmwareMainHeap) || (psHeap == psDevInfo->psFirmwareConfigHeap))
+		{
+			eError = BridgeRIWritePMREntryWithOwner (GetBridgeHandle(psImport->hDevConnection),
+					psImport->hPMR,
+					PVR_SYS_ALLOC_PID);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntryWithOwner");
+		}
+		else
+#endif
+		{
+			eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+					psImport->hPMR);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+		}
+	}
+#endif
+
+#if defined(__KERNEL__)
+	if (psHeap->bPremapped)
+	{
+		ui64OptionalMapAddress = _GetPremappedVA(psImport->hPMR, psHeap->psCtx->hDevConnection);
+	}
+#endif
+
+	/*
+		Suballocations always get mapped into the device was we need to
+		key the RA off something and as we can't export suballocations
+		there is no valid reason to request an allocation an not map it
+	 */
+	eError = DevmemImportStructDevMap(psHeap,
+			IMG_TRUE,
+			psImport,
+			ui64OptionalMapAddress);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	OSLockAcquire(psImport->hLock);
+	/* Mark this import struct as zeroed so we can save some PDump LDBs
+	 * and do not have to CPU map + mem set()*/
+	if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC)
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_ZEROED;
+	}
+	else if (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC)
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_POISONED;
+	}
+	psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_CLEAN;
+	OSLockRelease(psImport->hLock);
+
+	psRAImport->base = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	psRAImport->uSize = uiSize;
+	psRAImport->hPriv = psImport;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failMap:
+	FreeDeviceMemory(psImport);
+failAlloc:
+
+	return eError;
+}
+
+static void
+SubAllocImportFree(RA_PERARENA_HANDLE hArena,
+		RA_BASE_T uiBase,
+		RA_PERISPAN_HANDLE hImport)
+{
+	DEVMEM_IMPORT *psImport = hImport;
+#if !defined(PVRSRV_NEED_PVR_ASSERT)
+	PVR_UNREFERENCED_PARAMETER(hArena);
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+#endif
+
+	PVR_ASSERT(psImport != NULL);
+	PVR_ASSERT(hArena == psImport->sDeviceImport.psHeap);
+	PVR_ASSERT(uiBase == psImport->sDeviceImport.sDevVAddr.uiAddr);
+
+	(void) DevmemImportStructDevUnmap(psImport);
+	(void) DevmemImportStructRelease(psImport);
+}
+
+/*****************************************************************************
+ *                    Devmem context internals                               *
+ *****************************************************************************/
+
+static PVRSRV_ERROR
+PopulateContextFromBlueprint(struct DEVMEM_CONTEXT_TAG *psCtx,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	struct DEVMEM_HEAP_TAG **ppsHeapArray;
+	IMG_UINT32 uiNumHeaps;
+	IMG_UINT32 uiHeapsToUnwindOnError;
+	IMG_UINT32 uiHeapIndex;
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_CHAR aszHeapName[DEVMEM_HEAPNAME_MAXLENGTH];
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2DataPageSize;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2ImportAlignment;
+
+	eError = DevmemHeapCount(psCtx->hDevConnection,
+			uiHeapBlueprintID,
+			&uiNumHeaps);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (uiNumHeaps == 0)
+	{
+		ppsHeapArray = NULL;
+	}
+	else
+	{
+		ppsHeapArray = OSAllocMem(sizeof(*ppsHeapArray) * uiNumHeaps);
+		PVR_GOTO_IF_NOMEM(ppsHeapArray, eError, e0);
+	}
+
+	uiHeapsToUnwindOnError = 0;
+
+	for (uiHeapIndex = 0; uiHeapIndex < uiNumHeaps; uiHeapIndex++)
+	{
+		eError = DevmemHeapDetails(psCtx->hDevConnection,
+				uiHeapBlueprintID,
+				uiHeapIndex,
+				&aszHeapName[0],
+				sizeof(aszHeapName),
+				&sDevVAddrBase,
+				&uiHeapLength,
+				&uiReservedRegionLength,
+				&uiLog2DataPageSize,
+				&uiLog2ImportAlignment);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		eError = DevmemCreateHeap(psCtx,
+				sDevVAddrBase,
+				uiHeapLength,
+				uiReservedRegionLength,
+				uiLog2DataPageSize,
+				uiLog2ImportAlignment,
+				aszHeapName,
+				uiHeapBlueprintID,
+				uiHeapIndex,
+				&ppsHeapArray[uiHeapIndex]);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		uiHeapsToUnwindOnError = uiHeapIndex + 1;
+	}
+
+	psCtx->uiAutoHeapCount = uiNumHeaps;
+	psCtx->ppsAutoHeapArray = ppsHeapArray;
+
+	PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount);
+	PVR_ASSERT(psCtx->uiAutoHeapCount == uiNumHeaps);
+
+	return PVRSRV_OK;
+
+	/* error exit paths */
+e1:
+	for (uiHeapIndex = 0; uiHeapIndex < uiHeapsToUnwindOnError; uiHeapIndex++)
+	{
+		eError2 = DevmemDestroyHeap(ppsHeapArray[uiHeapIndex]);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+	if (uiNumHeaps != 0)
+	{
+		OSFreeMem(ppsHeapArray);
+	}
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+UnpopulateContextFromBlueprint(struct DEVMEM_CONTEXT_TAG *psCtx)
+{
+	PVRSRV_ERROR eReturn = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+	IMG_UINT32 uiHeapIndex;
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+
+	for (uiHeapIndex = 0; uiHeapIndex < psCtx->uiAutoHeapCount; uiHeapIndex++)
+	{
+		if (!psCtx->ppsAutoHeapArray[uiHeapIndex])
+		{
+			continue;
+		}
+
+		eError2 = DevmemDestroyHeap(psCtx->ppsAutoHeapArray[uiHeapIndex]);
+		if (eError2 != PVRSRV_OK)
+		{
+			eReturn = eError2;
+		}
+		else
+		{
+			psCtx->ppsAutoHeapArray[uiHeapIndex] = NULL;
+		}
+	}
+
+	if ((!bDoCheck || (eReturn == PVRSRV_OK)) && psCtx->ppsAutoHeapArray)
+	{
+		OSFreeMem(psCtx->ppsAutoHeapArray);
+		psCtx->ppsAutoHeapArray = NULL;
+		psCtx->uiAutoHeapCount = 0;
+	}
+
+	return eReturn;
+}
+
+/*****************************************************************************
+ *                    Devmem context functions                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID,
+		DEVMEM_CONTEXT **ppsCtxPtr)
+{
+	PVRSRV_ERROR		eError;
+	DEVMEM_CONTEXT		*psCtx;
+	/* handle to the server-side counterpart of the device memory
+	   context (specifically, for handling mapping to device MMU) */
+	IMG_HANDLE			hDevMemServerContext;
+	IMG_HANDLE			hPrivData;
+	IMG_BOOL			bHeapCfgFWId = (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW);
+
+	PVR_GOTO_IF_NOMEM(ppsCtxPtr, eError, e0);
+
+	psCtx = OSAllocMem(sizeof(*psCtx));
+	PVR_GOTO_IF_NOMEM(psCtx, eError, e0);
+
+	psCtx->uiNumHeaps = 0;
+
+	psCtx->hDevConnection = hDevConnection;
+
+	/* Create (server-side) Device Memory context */
+	eError = BridgeDevmemIntCtxCreate(GetBridgeHandle(psCtx->hDevConnection),
+			bHeapCfgFWId,
+			&hDevMemServerContext,
+			&hPrivData,
+			&psCtx->ui32CPUCacheLineSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	psCtx->hDevMemServerContext = hDevMemServerContext;
+	psCtx->hPrivData = hPrivData;
+
+	/* automagic heap creation */
+	psCtx->uiAutoHeapCount = 0;
+
+	eError = PopulateContextFromBlueprint(psCtx, uiHeapBlueprintID);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	*ppsCtxPtr = psCtx;
+
+	PVR_ASSERT(psCtx->uiNumHeaps == psCtx->uiAutoHeapCount);
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+e2:
+	PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
+	PVR_ASSERT(psCtx->uiNumHeaps == 0);
+	BridgeDevmemIntCtxDestroy(GetBridgeHandle(psCtx->hDevConnection), hDevMemServerContext);
+
+e1:
+	OSFreeMem(psCtx);
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
+		IMG_HANDLE *hPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(psCtx, eError, e0);
+	PVR_GOTO_IF_INVALID_PARAM(hPrivData, eError, e0);
+
+	*hPrivData = psCtx->hPrivData;
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(psCtx, eError, e0);
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemFindHeapByName(const struct DEVMEM_CONTEXT_TAG *psCtx,
+		const IMG_CHAR *pszHeapName,
+		struct DEVMEM_HEAP_TAG **ppsHeapRet)
+{
+	IMG_UINT32 uiHeapIndex;
+
+	/* N.B.  This func is only useful for finding "automagic" heaps by name */
+	for (uiHeapIndex = 0;
+			uiHeapIndex < psCtx->uiAutoHeapCount;
+			uiHeapIndex++)
+	{
+		if (!OSStringNCompare(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName, pszHeapName, OSStringLength(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName) + 1))
+		{
+			*ppsHeapRet = psCtx->ppsAutoHeapArray[uiHeapIndex];
+			return PVRSRV_OK;
+		}
+	}
+
+	return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemDestroyContext(DEVMEM_CONTEXT *psCtx)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bDoCheck = IMG_TRUE;
+
+#if defined(__KERNEL__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+
+	PVR_RETURN_IF_INVALID_PARAM(psCtx);
+
+	eError = UnpopulateContextFromBlueprint(psCtx);
+	if (bDoCheck && eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: UnpopulateContextFromBlueprint failed (%d) leaving %d heaps",
+				__func__, eError, psCtx->uiNumHeaps));
+		goto e1;
+	}
+
+	eError = DestroyServerResource(psCtx->hDevConnection,
+	                               NULL,
+	                               BridgeDevmemIntCtxDestroy,
+	                               psCtx->hDevMemServerContext);
+	if (bDoCheck)
+	{
+		PVR_LOG_GOTO_IF_ERROR(eError, "BridgeDevMemIntCtxDestroy", e1);
+
+		/* should be no more heaps left */
+		if (psCtx->uiNumHeaps)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Additional heaps remain in DEVMEM_CONTEXT",
+					__func__));
+			eError = PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT;
+			goto e1;
+		}
+	}
+
+	OSCachedMemSet(psCtx, 0, sizeof(*psCtx));
+	OSFreeMem(psCtx);
+
+e1:
+	return eError;
+}
+
+/*****************************************************************************
+ *                 Devmem heap query functions                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 *puiNumHeapConfigsOut)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapConfigCount(GetBridgeHandle(hDevConnection),
+			puiNumHeapConfigsOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_UINT32 *puiNumHeapsOut)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapCount(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			puiNumHeapsOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapConfigName(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_CHAR *pszConfigNameOut,
+		IMG_UINT32 uiConfigNameBufSz)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapConfigName(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			uiConfigNameBufSz,
+			pszConfigNameOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_UINT32 uiHeapIndex,
+		IMG_CHAR *pszHeapNameOut,
+		IMG_UINT32 uiHeapNameBufSz,
+		IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+		IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+		IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+		IMG_UINT32 *puiLog2DataPageSizeOut,
+		IMG_UINT32 *puiLog2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgeHeapCfgHeapDetails(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			uiHeapIndex,
+			uiHeapNameBufSz,
+			pszHeapNameOut,
+			psDevVAddrBaseOut,
+			puiHeapLengthOut,
+			puiReservedRegionLengthOut,
+			puiLog2DataPageSizeOut,
+			puiLog2ImportAlignmentOut);
+
+	VG_MARK_INITIALIZED(pszHeapNameOut, uiHeapNameBufSz);
+
+	return eError;
+}
+
+/*****************************************************************************
+ *                    Devmem heap functions                                  *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
+		IMG_HANDLE *phDevmemHeap)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+	*phDevmemHeap = psHeap->hDevMemServerHeap;
+	return PVRSRV_OK;
+}
+
+/* See devicemem.h for important notes regarding the arguments
+   to this function */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemCreateHeap(DEVMEM_CONTEXT *psCtx,
+		IMG_DEV_VIRTADDR sBaseAddress,
+		IMG_DEVMEM_SIZE_T uiLength,
+		IMG_DEVMEM_SIZE_T uiReservedRegionLength,
+		IMG_UINT32 ui32Log2Quantum,
+		IMG_UINT32 ui32Log2ImportAlignment,
+		const IMG_CHAR *pszName,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID,
+		IMG_UINT32 uiHeapIndex,
+		DEVMEM_HEAP **ppsHeapPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+	DEVMEM_HEAP *psHeap;
+	/* handle to the server-side counterpart of the device memory heap
+	   (specifically, for handling mapping to device MMU) */
+	IMG_HANDLE hDevMemServerHeap;
+	IMG_UINT32 ui32Policy = RA_POLICY_DEFAULT, ui32PolicyVMRA;
+
+	IMG_CHAR aszBuf[100];
+	IMG_CHAR *pszStr;
+	IMG_UINT32 ui32pszStrSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsHeapPtr, "ppsHeapPtr");
+
+	/* Reserved VA space of a heap must always be multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY.
+	 * Granularity has been chosen to support the max possible practically used OS page size. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uiReservedRegionLength % DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY) == 0, "uiReservedRegionLength");
+
+	ui32PolicyVMRA = RA_POLICY_DEFAULT;
+
+	PVR_ASSERT(uiReservedRegionLength + DEVMEM_HEAP_MINIMUM_SIZE <= uiLength);
+
+	psHeap = OSAllocMem(sizeof(*psHeap));
+	PVR_GOTO_IF_NOMEM(psHeap, eError, HeapAllocError);
+
+	/* Need to keep local copy of heap name, so caller may free theirs */
+	ui32pszStrSize = OSStringLength(pszName) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	PVR_GOTO_IF_NOMEM(pszStr, eError, SubAllocRANameCopyError);
+	OSStringSafeCopy(pszStr, pszName, ui32pszStrSize);
+	psHeap->pszName = pszStr;
+
+	psHeap->uiSize = uiLength;
+	psHeap->uiReservedRegionSize = uiReservedRegionLength;
+	psHeap->sBaseAddress = sBaseAddress;
+	psHeap->bPremapped = IMG_FALSE;
+	OSAtomicWrite(&psHeap->hImportCount, 0);
+
+	OSSNPrintf(aszBuf, sizeof(aszBuf),
+			"NDM heap '%s' (suballocs) ctx:%p",
+			pszName, psCtx);
+	ui32pszStrSize = OSStringLength(aszBuf) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	PVR_GOTO_IF_NOMEM(pszStr, eError, SubAllocRANameError);
+	OSStringSafeCopy(pszStr, aszBuf, ui32pszStrSize);
+	psHeap->pszSubAllocRAName = pszStr;
+
+#if defined(__KERNEL__)
+	if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW)
+	{
+		void *pvAppHintState = NULL;
+		IMG_UINT32 ui32FirmwarePolicydefault = PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY;
+		IMG_UINT32 ui32FirmwarePolicy = PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY;
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, DevMemFWHeapPolicy,
+						&ui32FirmwarePolicydefault, &ui32FirmwarePolicy);
+		ui32PolicyVMRA = ui32Policy = ui32FirmwarePolicy;
+		OSFreeAppHintState(pvAppHintState);
+
+		/* Flag the change from default setting */
+		if (ui32FirmwarePolicy != ui32FirmwarePolicydefault)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: %s: DevMemFWHeapPolicy set to %u, default %u",
+			         __func__, pszStr,
+			         ui32FirmwarePolicy, ui32FirmwarePolicydefault));
+		}
+	}
+#endif
+
+#if defined(PDUMP)
+	/* The META heap is shared globally so a single physical memory import
+	 * may be used to satisfy allocations of different processes.
+	 * This is problematic when PDumping because the physical memory
+	 * import used to satisfy a new allocation may actually have been
+	 * imported (and thus the PDump MALLOC generated) before the PDump
+	 * client was started, leading to the MALLOC being missing.
+	 *
+	 * This is solved by disabling splitting of imports for the META physmem
+	 * RA, meaning that every firmware allocation gets its own import, thus
+	 * ensuring the MALLOC is present for every allocation made within the
+	 * pdump capture range
+	 */
+	if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW)
+	{
+		ui32Policy |= RA_POLICY_NO_SPLIT;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiHeapBlueprintID);
+#endif
+
+	psHeap->psSubAllocRA = RA_Create(psHeap->pszSubAllocRAName,
+			/* Subsequent imports: */
+			ui32Log2Quantum,
+			RA_LOCKCLASS_3,
+			SubAllocImportAlloc,
+			SubAllocImportFree,
+			(RA_PERARENA_HANDLE) psHeap,
+			ui32Policy);
+	if (psHeap->psSubAllocRA == NULL)
+	{
+		eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
+		goto SubAllocRACreateError;
+	}
+
+	psHeap->uiLog2ImportAlignment = ui32Log2ImportAlignment;
+	psHeap->uiLog2Quantum = ui32Log2Quantum;
+
+	if (!OSStringNCompare(pszName, RGX_GENERAL_SVM_HEAP_IDENT, sizeof(RGX_GENERAL_SVM_HEAP_IDENT)))
+	{
+		/* The SVM heap normally starts out as this type though
+		   it may transition to DEVMEM_HEAP_MANAGER_USER
+		   on platforms with more processor virtual address
+		   bits than device virtual address bits */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_KERNEL;
+	}
+	else if (uiReservedRegionLength != 0)
+	{
+		/* Heaps which specify reserved VA space range are dual managed:
+		 * - sBaseAddress to (sBaseAddress+uiReservedRegionLength-1): User managed
+		 * - (sBaseAddress+uiReservedRegionLength) to uiLength: RA managed
+		 */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_DUAL_USER_RA;
+	}
+	else
+	{
+		/* Otherwise, heap manager is decided (USER or RA) at first map */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_UNKNOWN;
+	}
+
+	/* Mark the heap to be managed by RA */
+	if (!OSStringNCompare(pszName, RGX_VK_CAPT_REPLAY_HEAP_IDENT,
+	                      sizeof(RGX_VK_CAPT_REPLAY_HEAP_IDENT)))
+	{
+		psHeap->ui32HeapManagerFlags |= DEVMEM_HEAP_MANAGER_RA;
+	}
+
+	OSSNPrintf(aszBuf, sizeof(aszBuf),
+			"NDM heap '%s' (QVM) ctx:%p",
+			pszName, psCtx);
+	ui32pszStrSize = OSStringLength(aszBuf) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	if (pszStr == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto VMRANameError;
+	}
+	OSStringSafeCopy(pszStr, aszBuf, ui32pszStrSize);
+	psHeap->pszQuantizedVMRAName = pszStr;
+
+	psHeap->psQuantizedVMRA = RA_Create(psHeap->pszQuantizedVMRAName,
+			/* Subsequent import: */
+			0, RA_LOCKCLASS_1, NULL, NULL,
+			(RA_PERARENA_HANDLE) psHeap,
+			ui32PolicyVMRA);
+	if (psHeap->psQuantizedVMRA == NULL)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA, VMRACreateError);
+	}
+
+	if (!RA_Add(psHeap->psQuantizedVMRA,
+			/* Make sure the VMRA doesn't allocate from reserved VAs */
+			(RA_BASE_T)sBaseAddress.uiAddr + uiReservedRegionLength,
+			(RA_LENGTH_T)uiLength,
+			(RA_FLAGS_T)0, /* This RA doesn't use or need flags */
+			NULL /* per ispan handle */))
+	{
+		RA_Delete(psHeap->psQuantizedVMRA);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA, VMRACreateError);
+	}
+
+	psHeap->psCtx = psCtx;
+
+
+	/* Create server-side counterpart of Device Memory heap */
+	eError = BridgeDevmemIntHeapCreate(GetBridgeHandle(psCtx->hDevConnection),
+	                                   psCtx->hDevMemServerContext,
+	                                   uiHeapBlueprintID,
+	                                   uiHeapIndex,
+	                                   &hDevMemServerHeap);
+	PVR_GOTO_IF_ERROR(eError, ServerHeapCreateError);
+
+	psHeap->hDevMemServerHeap = hDevMemServerHeap;
+
+	eError = OSLockCreate(&psHeap->hLock);
+	PVR_GOTO_IF_ERROR(eError, LockCreateErr);
+
+	psHeap->psCtx->uiNumHeaps++;
+	*ppsHeapPtr = psHeap;
+
+	return PVRSRV_OK;
+
+	/* error exit paths */
+LockCreateErr:
+	eError2 = BridgeDevmemIntHeapDestroy(GetBridgeHandle(psCtx->hDevConnection),
+			psHeap->hDevMemServerHeap);
+	PVR_ASSERT (eError2 == PVRSRV_OK);
+ServerHeapCreateError:
+	RA_Delete(psHeap->psQuantizedVMRA);
+VMRACreateError:
+	OSFreeMem(psHeap->pszQuantizedVMRAName);
+VMRANameError:
+	RA_Delete(psHeap->psSubAllocRA);
+SubAllocRACreateError:
+	OSFreeMem(psHeap->pszSubAllocRAName);
+SubAllocRANameError:
+	OSFreeMem(psHeap->pszName);
+SubAllocRANameCopyError:
+	OSFreeMem(psHeap);
+HeapAllocError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapBaseDevVAddr(struct DEVMEM_HEAP_TAG *psHeap,
+		IMG_DEV_VIRTADDR *pDevVAddr)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+
+	*pDevVAddr = psHeap->sBaseAddress;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL DEVMEM_SIZE_T
+DevmemGetHeapSize(struct DEVMEM_HEAP_TAG *psHeap)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap != NULL);
+
+	return psHeap->uiSize;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
+		IMG_DEVMEM_SIZE_T *puiSize,
+		IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	IMG_DEVMEM_SIZE_T uiSize = *puiSize;
+	IMG_DEVMEM_ALIGN_T uiAlign = *puiAlign;
+
+	/* Just in case someone changes definition of IMG_DEVMEM_ALIGN_T. */
+	static_assert(sizeof(unsigned long long) == sizeof(uiAlign),
+	              "invalid uiAlign size");
+	/* This value is used for shifting so it cannot be greater than number
+	 * of bits in unsigned long long (sizeof(1ULL)). Using greater value is
+	 * undefined behaviour. */
+	if (uiLog2Quantum >= sizeof(unsigned long long) * 8)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((1ULL << uiLog2Quantum) > uiAlign)
+	{
+		uiAlign = 1ULL << uiLog2Quantum;
+	}
+	uiSize = PVR_ALIGN(uiSize, uiAlign);
+
+	*puiSize = uiSize;
+	*puiAlign = uiAlign;
+
+	return PVRSRV_OK;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemDestroyHeap(DEVMEM_HEAP *psHeap)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT uiImportCount;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__)
+	if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+#endif
+
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+
+	uiImportCount = OSAtomicRead(&psHeap->hImportCount);
+	if (uiImportCount > 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%d(%s) leaks remain", uiImportCount, psHeap->pszName));
+#if defined(__KERNEL__)
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		PVR_DPF((PVR_DBG_ERROR, "Details of remaining allocated device memory (for all processes):"));
+		RIDumpAllKM();
+#else
+		PVR_DPF((PVR_DBG_ERROR, "Compile with PVRSRV_ENABLE_GPU_MEMORY_INFO=1 to get a full "
+				"list of all driver allocations."));
+#endif
+#endif
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (bDoCheck)
+#endif
+		{
+			return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP;
+		}
+	}
+
+	eError = DestroyServerResource(psHeap->psCtx->hDevConnection,
+	                               NULL,
+	                               BridgeDevmemIntHeapDestroy,
+	                               psHeap->hDevMemServerHeap);
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (bDoCheck)
+#endif
+	{
+		PVR_LOG_RETURN_IF_ERROR(eError, "BridgeDevmemIntHeapDestroy");
+	}
+
+	PVR_ASSERT(psHeap->psCtx->uiNumHeaps > 0);
+	psHeap->psCtx->uiNumHeaps--;
+
+	OSLockDestroy(psHeap->hLock);
+
+	if (psHeap->psQuantizedVMRA)
+	{
+		RA_Delete(psHeap->psQuantizedVMRA);
+	}
+	if (psHeap->pszQuantizedVMRAName)
+	{
+		OSFreeMem(psHeap->pszQuantizedVMRAName);
+	}
+
+	RA_Delete(psHeap->psSubAllocRA);
+	OSFreeMem(psHeap->pszSubAllocRAName);
+
+	OSFreeMem(psHeap->pszName);
+
+	OSCachedMemSet(psHeap, 0, sizeof(*psHeap));
+	OSFreeMem(psHeap);
+
+	return PVRSRV_OK;
+}
+
+/*****************************************************************************
+ *                Devmem allocation/free functions                           *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemSubAllocateAndMap(IMG_UINT8 uiPreAllocMultiplier,
+			DEVMEM_HEAP *psHeap,
+			IMG_DEVMEM_SIZE_T uiSize,
+			IMG_DEVMEM_ALIGN_T uiAlign,
+			PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			const IMG_CHAR *pszText,
+			DEVMEM_MEMDESC **ppsMemDescPtr,
+			IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemSubAllocate(uiPreAllocMultiplier,
+				psHeap,
+				uiSize,
+				uiAlign,
+				uiFlags,
+				pszText,
+				ppsMemDescPtr);
+	PVR_GOTO_IF_ERROR(eError, fail_alloc);
+
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+				psHeap,
+				psDevVirtAddr);
+	PVR_GOTO_IF_ERROR(eError, fail_map);
+
+	return PVRSRV_OK;
+
+fail_map:
+	DevmemFree(*ppsMemDescPtr);
+fail_alloc:
+	*ppsMemDescPtr = NULL;
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	RA_BASE_T uiAllocatedAddr = 0;
+	RA_LENGTH_T uiAllocatedSize;
+	RA_PERISPAN_HANDLE hImport; /* the "import" from which this sub-allocation came */
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	IMG_DEVMEM_OFFSET_T uiOffset = 0;
+	DEVMEM_IMPORT *psImport;
+	IMG_UINT32 ui32CPUCacheLineSize;
+	void *pvAddr = NULL;
+
+	IMG_BOOL bImportClean;
+	IMG_BOOL bCPUCleanFlag = PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags);
+	IMG_BOOL bZero = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
+	IMG_BOOL bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags);
+	IMG_BOOL bCPUCached = (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+			PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags));
+	IMG_BOOL bGPUCached = (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) ||
+			PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags));
+	IMG_BOOL bAlign = ! (PVRSRV_CHECK_NO_CACHE_LINE_ALIGN(uiFlags));
+	PVRSRV_CACHE_OP eOp = PVRSRV_CACHE_OP_INVALIDATE;
+	IMG_UINT32	ui32CacheLineSize = 0;
+	DEVMEM_PROPERTIES_T uiProperties;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+		bPoisonOnAlloc = IMG_TRUE;
+	}
+#endif
+
+	if (uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		/* Deferred Allocation not supported on SubAllocs*/
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failParams);
+	}
+
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap->psCtx, eError, failParams);
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(psHeap->psCtx->hDevConnection, uiFlags);
+
+#if defined(__KERNEL__)
+	{
+		/* The hDevConnection holds two different types of pointers depending on the
+		 * address space in which it is used.
+		 * In this instance the variable points to the device node in server */
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
+		ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, SLC_CACHE_LINE_SIZE_BITS));
+	}
+#else
+	ui32CacheLineSize = ROGUE_CACHE_LINE_SIZE;
+#endif
+
+	/* The following logic makes sure that any cached memory is aligned to both the CPU and GPU.
+	 * To be aligned on both you have to take the Lowest Common Multiple (LCM) of the cache line sizes of each.
+	 * As the possibilities are all powers of 2 then simply the largest number can be picked as the LCM.
+	 * Therefore this algorithm just picks the highest from the CPU, GPU and given alignments.
+	 */
+	ui32CPUCacheLineSize = psHeap->psCtx->ui32CPUCacheLineSize;
+	/* If the CPU cache line size is larger than the alignment given then it is the lowest common multiple
+	 * Also checking if the allocation is going to be cached on the CPU
+	 * Currently there is no check for the validity of the cache coherent option.
+	 * In this case, the alignment could be applied but the mode could still fall back to uncached.
+	 */
+	if (bAlign && ui32CPUCacheLineSize > uiAlign && bCPUCached)
+	{
+		uiAlign = ui32CPUCacheLineSize;
+	}
+
+	/* If the GPU cache line size is larger than the alignment given then it is the lowest common multiple
+	 * Also checking if the allocation is going to be cached on the GPU via checking for any of the cached options.
+	 * Currently there is no check for the validity of the cache coherent option.
+	 * In this case, the alignment could be applied but the mode could still fall back to uncached.
+	 */
+	if (bAlign && ui32CacheLineSize > uiAlign && bGPUCached)
+	{
+		uiAlign = ui32CacheLineSize;
+	}
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	/* No request for exportable memory so use the RA */
+	eError = RA_Alloc(psHeap->psSubAllocRA,
+			uiSize,
+			uiPreAllocMultiplier,
+			uiFlags,
+			uiAlign,
+			pszText,
+			&uiAllocatedAddr,
+			&uiAllocatedSize,
+			&hImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	psImport = hImport;
+
+	/* This assignment is assuming the RA returns an hImport where suballocations
+	 * can be made from if uiSize is NOT a page multiple of the passed heap.
+	 *
+	 * So we check if uiSize is a page multiple and mark it as exportable
+	 * if it is not.
+	 * */
+	OSLockAcquire(psImport->hLock);
+	if (!(uiSize & (IMG_PAGE2BYTES64(psHeap->uiLog2Quantum) - 1)) &&
+	    (uiPreAllocMultiplier == RA_NO_IMPORT_MULTIPLIER))
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_EXPORTABLE;
+	}
+	psImport->uiProperties |= DEVMEM_PROPERTIES_SUBALLOCATABLE;
+	uiProperties = psImport->uiProperties;
+	OSLockRelease(psImport->hLock);
+
+	uiOffset = uiAllocatedAddr - psImport->sDeviceImport.sDevVAddr.uiAddr;
+
+#if defined(PDUMP) && defined(DEBUG)
+#if defined(__KERNEL__)
+	PDUMPCOMMENTWITHFLAGS(PMR_DeviceNode((PMR*)psImport->hPMR), PDUMP_CONT,
+			"Suballocated %u Byte for \"%s\" from PMR with handle ID: 0x%p (PID %u)",
+			(IMG_UINT32) uiSize, pszText, psImport->hPMR, OSGetCurrentProcessID());
+#else
+	PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS,
+			"Suballocated %u Byte for \"%s\" from PMR with handle ID: %p (PID %u)",
+			(IMG_UINT32) uiSize,
+			pszText,
+			psImport->hPMR,
+			OSGetCurrentProcessID());
+#endif
+#endif
+
+	DevmemMemDescInit(psMemDesc,
+			uiOffset,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	bImportClean = ((uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_CLEAN) != 0);
+
+	/* Zero the memory */
+	if (bZero)
+	{
+		/* Has the import been zeroed on allocation and were no suballocations returned to it so far? */
+		bImportClean = bImportClean && ((uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_ZEROED) != 0);
+
+		if (!bImportClean)
+		{
+			eOp = PVRSRV_CACHE_OP_FLUSH;
+
+			eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
+			PVR_GOTO_IF_ERROR(eError, failMaintenance);
+
+			/* uiSize is a 64-bit quantity whereas the 3rd argument
+			 * to OSDeviceMemSet is a 32-bit quantity on 32-bit systems
+			 * hence a compiler warning of implicit cast and loss of data.
+			 * Added explicit cast and assert to remove warning.
+			 */
+			PVR_ASSERT(uiSize < IMG_UINT32_MAX);
+
+			DevmemCPUMemSet(pvAddr, 0, uiSize, uiFlags);
+
+#if defined(PDUMP)
+			DevmemPDumpLoadZeroMem(psMemDesc, 0, uiSize, PDUMP_FLAGS_CONTINUOUS);
+#endif
+		}
+	}
+	else if (bPoisonOnAlloc)
+	{
+		/* Has the import been poisoned on allocation and were no suballocations returned to it so far? */
+		bPoisonOnAlloc = (uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_POISONED) != 0;
+
+		if (!bPoisonOnAlloc)
+		{
+			eOp = PVRSRV_CACHE_OP_FLUSH;
+
+			eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
+			PVR_GOTO_IF_ERROR(eError, failMaintenance);
+
+			DevmemCPUMemSet(pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE, uiSize, uiFlags);
+
+			bPoisonOnAlloc = IMG_TRUE;
+		}
+	}
+
+	/* Flush or invalidate */
+	if (bCPUCached && !bImportClean && (bZero || bCPUCleanFlag || bPoisonOnAlloc))
+	{
+		eError = BridgeCacheOpExec (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				(IMG_UINT64)(uintptr_t)
+				pvAddr - psMemDesc->uiOffset,
+				psMemDesc->uiOffset,
+				psMemDesc->uiAllocSize,
+				eOp);
+		PVR_GOTO_IF_ERROR(eError, failMaintenance);
+	}
+
+	if (pvAddr)
+	{
+		DevmemReleaseCpuVirtAddr(psMemDesc);
+		pvAddr = NULL;
+	}
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				OSStringNLength(psMemDesc->szText, DEVMEM_ANNOTATION_MAX_LEN),
+				psMemDesc->szText,
+				psMemDesc->uiOffset,
+				uiAllocatedSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_SUBALLOC,
+				&(psMemDesc->hRIHandle));
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+		}
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	*ppsMemDescPtr = psMemDesc;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failMaintenance:
+	if (pvAddr)
+	{
+		DevmemReleaseCpuVirtAddr(psMemDesc);
+		pvAddr = NULL;
+	}
+	DevmemMemDescRelease(psMemDesc);
+	psMemDesc = NULL;	/* Make sure we don't do a discard after the release */
+failDeviceMemAlloc:
+	if (psMemDesc)
+	{
+		DevmemMemDescDiscard(psMemDesc);
+	}
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		IMG_UINT32 uiLog2HeapPageSize,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_UINT32 ui32MappingTable = 0;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+	}
+#endif
+
+	eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(hDevConnection, uiFlags);
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = AllocateDeviceMemory(hDevConnection,
+			uiLog2HeapPageSize,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			uiAlign,
+			uiFlags,
+			IMG_TRUE,
+			pszText,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	*ppsMemDescPtr = psMemDesc;
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				uiSize,
+				uiFlags,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failDeviceMemAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_UINT32 ui32NumPhysChunks,
+		IMG_UINT32 ui32NumVirtChunks,
+		IMG_UINT32 *pui32MappingTable,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		IMG_UINT32 uiLog2HeapPageSize,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+	}
+#endif
+
+	eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(hDevConnection, uiFlags);
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = AllocateDeviceMemory(hDevConnection,
+			uiLog2HeapPageSize,
+			uiSize,
+			ui32NumPhysChunks,
+			ui32NumVirtChunks,
+			pui32MappingTable,
+			uiAlign,
+			uiFlags,
+			IMG_TRUE,
+			pszText,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				uiSize,
+				uiFlags,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	*ppsMemDescPtr = psMemDesc;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failDeviceMemAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hServerHandle,
+		IMG_HANDLE *hLocalImportHandle)
+{
+	return BridgePMRMakeLocalImportHandle(GetBridgeHandle(hDevConnection),
+			hServerHandle,
+			hLocalImportHandle);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hLocalImportHandle)
+{
+	return DestroyServerResource(hDevConnection,
+	                             NULL,
+	                             BridgePMRUnmakeLocalImportHandle,
+	                             hLocalImportHandle);
+}
+
+/*****************************************************************************
+ *                Devmem unsecure export functions                           *
+ *****************************************************************************/
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static PVRSRV_ERROR
+_Mapping_Export(DEVMEM_IMPORT *psImport,
+		DEVMEM_EXPORTHANDLE *phPMRExportHandlePtr,
+		DEVMEM_EXPORTKEY *puiExportKeyPtr,
+		DEVMEM_SIZE_T *puiSize,
+		DEVMEM_LOG2ALIGN_T *puiLog2Contig)
+{
+	/* Gets an export handle and key for the PMR used for this mapping */
+	/* Can only be done if there are no suballocations for this mapping */
+
+	PVRSRV_ERROR eError;
+	DEVMEM_EXPORTHANDLE hPMRExportHandle;
+	DEVMEM_EXPORTKEY uiExportKey;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig;
+
+	PVR_GOTO_IF_INVALID_PARAM(psImport, eError, failParams);
+
+	if ((GetImportProperties(psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION, failParams);
+	}
+
+	eError = BridgePMRExportPMR(GetBridgeHandle(psImport->hDevConnection),
+			psImport->hPMR,
+			&hPMRExportHandle,
+			&uiSize,
+			&uiLog2Contig,
+			&uiExportKey);
+	PVR_GOTO_IF_ERROR(eError, failExport);
+
+	PVR_ASSERT(uiSize == psImport->uiSize);
+
+	*phPMRExportHandlePtr = hPMRExportHandle;
+	*puiExportKeyPtr = uiExportKey;
+	*puiSize = uiSize;
+	*puiLog2Contig = uiLog2Contig;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failExport:
+failParams:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static void
+_Mapping_Unexport(DEVMEM_IMPORT *psImport,
+		DEVMEM_EXPORTHANDLE hPMRExportHandle)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT (psImport != NULL);
+
+	eError = DestroyServerResource(psImport->hDevConnection,
+	                               NULL,
+	                               BridgePMRUnexportPMR,
+	                               hPMRExportHandle);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExport(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_EXPORTCOOKIE *psExportCookie)
+{
+	/* Caller to provide storage for export cookie struct */
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hPMRExportHandle = 0;
+	IMG_UINT64 uiPMRExportPassword = 0;
+	IMG_DEVMEM_SIZE_T uiSize = 0;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig = 0;
+
+	PVR_GOTO_IF_INVALID_PARAM(psMemDesc, eError, e0);
+	PVR_GOTO_IF_INVALID_PARAM(psExportCookie, eError, e0);
+
+	if (DEVMEM_PROPERTIES_EXPORTABLE !=
+			(GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This Memory (0x%p) cannot be exported!...",
+				__func__, psMemDesc));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_REQUEST, e0);
+	}
+
+	eError = _Mapping_Export(psMemDesc->psImport,
+			&hPMRExportHandle,
+			&uiPMRExportPassword,
+			&uiSize,
+			&uiLog2Contig);
+	if (eError != PVRSRV_OK)
+	{
+		psExportCookie->uiSize = 0;
+		goto e0;
+	}
+
+	psExportCookie->hPMRExportHandle = hPMRExportHandle;
+	psExportCookie->uiPMRExportPassword = uiPMRExportPassword;
+	psExportCookie->uiSize = uiSize;
+	psExportCookie->uiLog2ContiguityGuarantee = uiLog2Contig;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_EXPORTCOOKIE *psExportCookie)
+{
+	_Mapping_Unexport(psMemDesc->psImport,
+			psExportCookie->hPMRExportHandle);
+
+	psExportCookie->uiSize = 0;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_EXPORTCOOKIE *psCookie,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, failImportAlloc);
+	}
+
+	/* Get a handle to the PMR (inc refcount) */
+	eError = BridgePMRImportPMR(GetBridgeHandle(hDevConnection),
+			psCookie->hPMRExportHandle,
+			psCookie->uiPMRExportPassword,
+			psCookie->uiSize, /* not trusted - just for validation */
+			psCookie->uiLog2ContiguityGuarantee, /* not trusted - just for validation */
+			&hPMR);
+	PVR_GOTO_IF_ERROR(eError, failImport);
+
+	DevmemImportStructInit(psImport,
+			psCookie->uiSize,
+			1ULL << psCookie->uiLog2ContiguityGuarantee,
+			uiFlags,
+			hPMR,
+			DEVMEM_PROPERTIES_IMPORTED |
+			DEVMEM_PROPERTIES_EXPORTABLE);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			psImport->uiSize);
+
+	*ppsMemDescPtr = psMemDesc;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				psMemDesc->psImport->uiSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_IMPORT | PVRSRV_MEMALLOCFLAG_RI_SUBALLOC,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failImport:
+	DevmemImportDiscard(psImport);
+failImportAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+/*****************************************************************************
+ *                   Common MemDesc functions                                *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetSize(DEVMEM_MEMDESC *psMemDesc, IMG_DEVMEM_SIZE_T* puiSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	*puiSize = psMemDesc->uiAllocSize;
+
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc, IMG_CHAR **pszAnnotation)
+{
+	/*
+	 * It is expected that psMemDesc->szText is a valid NUL-terminated string,
+	 * since DevmemMemDescAlloc uses OSAllocZMem to create the memdesc.
+	 */
+	*pszAnnotation = psMemDesc->szText;
+}
+
+/*
+	This function is called for freeing any class of memory
+ */
+IMG_INTERNAL IMG_BOOL
+DevmemFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_SECURE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Please use methods dedicated to secure buffers.",
+				__func__));
+		return IMG_FALSE;
+	}
+
+	return DevmemMemDescRelease(psMemDesc);
+}
+
+IMG_INTERNAL IMG_BOOL
+DevmemReleaseDevAddrAndFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	DevmemReleaseDevVirtAddr(psMemDesc);
+	return DevmemFree(psMemDesc);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	DEVMEM_IMPORT *psImport;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bMap = IMG_TRUE;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED, failCheck);
+	}
+
+	/* Don't map memory for deferred allocations */
+	if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		PVR_ASSERT(uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
+		bMap = IMG_FALSE;
+	}
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	psImport = psMemDesc->psImport;
+	DevmemMemDescAcquire(psMemDesc);
+
+#if defined(__KERNEL__)
+	if (psHeap->bPremapped)
+	{
+		ui64OptionalMapAddress = _GetPremappedVA(psImport->hPMR, psHeap->psCtx->hDevConnection);
+	}
+#endif
+
+	eError = DevmemImportStructDevMap(psHeap,
+			bMap,
+			psImport,
+			ui64OptionalMapAddress);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	sDevVAddr.uiAddr += psMemDesc->uiOffset;
+	psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
+
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				psMemDesc->uiOffset,
+				psMemDesc->sDeviceMemDesc.sDevVAddr,
+				psMemDesc->uiAllocSize,
+				psMemDesc->szText,
+				DevmemGetHeapLog2PageSize(psHeap),
+				psMemDesc->ui32AllocationIndex,
+				&psMemDesc->ui32AllocationIndex);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		if (psMemDesc->hRIHandle)
+		{
+			eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection),
+					psMemDesc->hRIHandle,
+					psImport->sDeviceImport.sDevVAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIUpdateMEMDESCAddr");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failMap:
+	bDestroyed = DevmemMemDescRelease(psMemDesc);
+failCheck:
+failParams:
+	if (!bDestroyed)
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEV_VIRTADDR sDevVirtAddr)
+{
+	DEVMEM_IMPORT *psImport;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bMap = IMG_TRUE;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED, failCheck);
+	}
+
+	/* Don't map memory for deferred allocations */
+	if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		PVR_ASSERT(uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
+		bMap = IMG_FALSE;
+	}
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	psImport = psMemDesc->psImport;
+	DevmemMemDescAcquire(psMemDesc);
+
+	eError = DevmemImportStructDevMap(psHeap,
+			bMap,
+			psImport,
+			sDevVirtAddr.uiAddr);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	sDevVAddr.uiAddr += psMemDesc->uiOffset;
+	psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				psMemDesc->uiOffset,
+				psMemDesc->sDeviceMemDesc.sDevVAddr,
+				psMemDesc->uiAllocSize,
+				psMemDesc->szText,
+				DevmemGetHeapLog2PageSize(psHeap),
+				psMemDesc->ui32AllocationIndex,
+				&psMemDesc->ui32AllocationIndex);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		if (psMemDesc->hRIHandle)
+		{
+			eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection),
+					psMemDesc->hRIHandle,
+					psImport->sDeviceImport.sDevVAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIUpdateMEMDESCAddr");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failMap:
+	bDestroyed = DevmemMemDescRelease(psMemDesc);
+failCheck:
+failParams:
+	if (!bDestroyed)
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL IMG_DEV_VIRTADDR
+DevmemGetDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_DEVICEMEM_NO_MAPPING, "DevmemGetDevVirtAddr");
+	}
+
+	PVR_ASSERT(psMemDesc->sDeviceMemDesc.sDevVAddr.uiAddr !=0 );
+
+	return psMemDesc->sDeviceMemDesc.sDevVAddr;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_NO_MAPPING, failRelease);
+	}
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	return PVRSRV_OK;
+
+failRelease:
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount-1);
+
+	PVR_ASSERT(psMemDesc->sDeviceMemDesc.ui32RefCount != 0);
+
+	if (--psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+		{
+			BridgeDevicememHistoryUnmap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+					psMemDesc->psImport->hPMR,
+					psMemDesc->uiOffset,
+					psMemDesc->sDeviceMemDesc.sDevVAddr,
+					psMemDesc->uiAllocSize,
+					psMemDesc->szText,
+					DevmemGetHeapLog2PageSize(psMemDesc->psImport->sDeviceImport.psHeap),
+					psMemDesc->ui32AllocationIndex,
+					&psMemDesc->ui32AllocationIndex);
+		}
+
+		/* When device mapping destroyed, zero Dev VA so DevmemGetDevVirtAddr()
+		 * returns 0 */
+		if (DevmemImportStructDevUnmap(psMemDesc->psImport) == IMG_TRUE)
+		{
+			psMemDesc->sDeviceMemDesc.sDevVAddr.uiAddr = 0;
+		}
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+		DevmemMemDescRelease(psMemDesc);
+	}
+	else
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		void **ppvCpuVirtAddr)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(ppvCpuVirtAddr != NULL);
+
+	eError = DevmemCPUMapCheckImportProperties(psMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemCPUMapCheckImportProperties");
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount+1);
+
+	if (psMemDesc->sCPUMemDesc.ui32RefCount++ == 0)
+	{
+		DEVMEM_IMPORT *psImport = psMemDesc->psImport;
+		IMG_UINT8 *pui8CPUVAddr;
+
+		DevmemMemDescAcquire(psMemDesc);
+		eError = DevmemImportStructCPUMap(psImport);
+		PVR_GOTO_IF_ERROR(eError, failMap);
+
+		pui8CPUVAddr = psImport->sCPUImport.pvCPUVAddr;
+		pui8CPUVAddr += psMemDesc->uiOffset;
+		psMemDesc->sCPUMemDesc.pvCPUVAddr = pui8CPUVAddr;
+	}
+	*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
+
+	VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
+
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+
+	return PVRSRV_OK;
+
+failMap:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	psMemDesc->sCPUMemDesc.ui32RefCount--;
+
+	if (!DevmemMemDescRelease(psMemDesc))
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	}
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		void **ppvCpuVirtAddr)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(ppvCpuVirtAddr != NULL);
+
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU UnMapping is not possible on this allocation!",
+				__func__));
+		return;
+	}
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount+1);
+
+	*ppvCpuVirtAddr = NULL;
+	if (psMemDesc->sCPUMemDesc.ui32RefCount)
+	{
+		*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
+		psMemDesc->sCPUMemDesc.ui32RefCount += 1;
+	}
+
+	VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+}
+
+IMG_INTERNAL void
+DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU UnMapping is not possible on this allocation!",
+				__func__));
+		return;
+	}
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount-1);
+
+	PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount != 0);
+
+	if (--psMemDesc->sCPUMemDesc.ui32RefCount == 0)
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+		DevmemImportStructCPUUnmap(psMemDesc->psImport);
+		DevmemMemDescRelease(psMemDesc);
+	}
+	else
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	}
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *phImport)
+{
+	if ((GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
+	}
+
+	*phImport = psMemDesc->psImport->hPMR;
+
+	return PVRSRV_OK;
+}
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
+		IMG_UINT64 *pui64UID)
+{
+	DEVMEM_IMPORT *psImport = psMemDesc->psImport;
+	PVRSRV_ERROR eError;
+
+	if (!(GetImportProperties(psImport) & (DEVMEM_PROPERTIES_IMPORTED |
+		                        DEVMEM_PROPERTIES_EXPORTABLE)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This Memory (0x%p) doesn't support the functionality requested...",
+				__func__, psMemDesc));
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+
+	eError = BridgePMRGetUID(GetBridgeHandle(psImport->hDevConnection),
+			psImport->hPMR,
+			pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *hReservation)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*hReservation = psImport->sDeviceImport.hReservation;
+
+	return PVRSRV_OK;
+}
+
+#endif /* !__KERNEL__ */
+
+/* Kernel usage of this function will only work with
+ * memdescs of buffers allocated in the FW memory context
+ * that is created in the Server
+ */
+void
+DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *phPMR,
+		IMG_DEVMEM_OFFSET_T *puiPMROffset)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	*puiPMROffset = psMemDesc->uiOffset;
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*phPMR = psImport->hPMR;
+}
+
+#if defined(__KERNEL__)
+IMG_INTERNAL void
+DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
+		PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*puiFlags = psImport->uiFlags;
+}
+
+IMG_INTERNAL SHARED_DEV_CONNECTION
+DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc)
+{
+	return psMemDesc->psImport->hDevConnection;
+}
+#endif /* __KERNEL__ */
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hExtHandle,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		DEVMEM_MEMDESC **ppsMemDescPtr,
+		IMG_DEVMEM_SIZE_T *puiSizePtr,
+		const IMG_CHAR *pszAnnotation)
+{
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, failImportAlloc);
+	}
+
+	/* Get the PMR handle and its size from the server */
+	eError = BridgePMRLocalImportPMR(GetBridgeHandle(hDevConnection),
+			hExtHandle,
+			&hPMR,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failImport);
+
+	DevmemImportStructInit(psImport,
+			uiSize,
+			uiAlign,
+			uiFlags,
+			hPMR,
+			DEVMEM_PROPERTIES_IMPORTED |
+			DEVMEM_PROPERTIES_EXPORTABLE);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+	*ppsMemDescPtr = psMemDesc;
+	if (puiSizePtr)
+		*puiSizePtr = uiSize;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information.
+		 * Set backed size to 0 since this allocation has been allocated
+		 * by the same process and has been accounted for. */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				psMemDesc->psImport->uiSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_IMPORT,
+				&(psMemDesc->hRIHandle));
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+
+	/* Copy the allocation descriptive name and size so it can be passed
+	 * to DevicememHistory when the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszAnnotation);
+	OSStringSafeCopy(psMemDesc->szText, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN);
+
+	return PVRSRV_OK;
+
+failImport:
+	DevmemImportDiscard(psImport);
+failImportAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
+		IMG_DEV_VIRTADDR sDevVAddr)
+{
+	return BridgeDevmemIsVDevAddrValid(GetBridgeHandle(psContext->hDevConnection),
+	                                   psContext->hDevMemServerContext,
+	                                   sDevVAddr);
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext,
+		IMG_DEV_VIRTADDR *psFaultAddress)
+{
+	return BridgeDevmemGetFaultAddress(GetBridgeHandle(psContext->hDevConnection),
+	                                   psContext->hDevMemServerContext,
+	                                   psFaultAddress);
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemInvalidateFBSCTable(DEVMEM_CONTEXT *psContext,
+                          IMG_UINT64 ui64FBSCEntries)
+{
+	return BridgeDevmemInvalidateFBSCTable(GetBridgeHandle(psContext->hDevConnection),
+	                                       psContext->hDevMemServerContext,
+	                                       ui64FBSCEntries);
+}
+#endif
+
+#endif /* !__KERNEL__ */
+
+IMG_INTERNAL IMG_UINT32
+DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap)
+{
+	return psHeap->uiLog2Quantum;
+}
+
+IMG_INTERNAL PVRSRV_MEMALLOCFLAGS_T
+DevmemGetMemAllocFlags(DEVMEM_MEMDESC *psMemDesc)
+{
+	return psMemDesc->psImport->uiFlags;
+}
+
+IMG_INTERNAL IMG_DEVMEM_SIZE_T
+DevmemGetHeapReservedSize(DEVMEM_HEAP *psHeap)
+{
+	return psHeap->uiReservedRegionSize;
+}
+
+#if !defined(__KERNEL__)
+/**************************************************************************/ /*!
+@Function       RegisterDevMemPFNotify
+@Description    Registers that the application wants to be signaled when a page
+                fault occurs.
+
+@Input          psContext      Memory context the process that would like to
+                               be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR:  PVRSRV_OK on success. Otherwise, a PVRSRV_
+                               error code
+ */ /***************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
+		IMG_BOOL       bRegister)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgeDevmemIntRegisterPFNotifyKM(GetBridgeHandle(psContext->hDevConnection),
+			psContext->hDevMemServerContext,
+			bRegister);
+	if (eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
+	{
+		PVR_LOG_ERROR(eError, "BridgeDevmemIntRegisterPFNotifyKM");
+	}
+
+	return eError;
+}
+#endif /* !__KERNEL__ */
+
+IMG_INTERNAL void
+DevmemHeapSetPremapStatus(DEVMEM_HEAP *psHeap, IMG_BOOL IsPremapped)
+{
+	psHeap->bPremapped = IsPremapped;
+}
+
+/******************************************************************************
+ End of file (devicemem.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_pdump.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_pdump.c
new file mode 100644
index 000000000000..c86441217b2a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_pdump.c
@@ -0,0 +1,364 @@
+/*************************************************************************/ /*!
+@File
+@Title          Shared device memory management PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common (client & server) PDump functions for the
+                memory management code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#if defined(PDUMP)
+
+#include "allocmem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pdump.h"
+#include "devicemem.h"
+#include "devicemem_utils.h"
+#include "devicemem_pdump.h"
+#include "client_pdump_bridge.h"
+#include "client_pdumpmm_bridge.h"
+#if defined(__linux__) && !defined(__KERNEL__)
+#include <stdio.h>
+#endif
+
+IMG_INTERNAL void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiOffset + uiSize <= psMemDesc->psImport->uiSize);
+
+	eError = BridgePMRPDumpLoadMem(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                               psMemDesc->psImport->hPMR,
+	                               psMemDesc->uiOffset + uiOffset,
+	                               uiSize,
+	                               uiPDumpFlags,
+	                               IMG_FALSE);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMem");
+}
+
+IMG_INTERNAL void
+DevmemPDumpLoadZeroMem(DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiOffset + uiSize <= psMemDesc->psImport->uiSize);
+
+	eError = BridgePMRPDumpLoadMem(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                               psMemDesc->psImport->hPMR,
+	                               psMemDesc->uiOffset + uiOffset,
+	                               uiSize,
+	                               uiPDumpFlags,
+	                               IMG_TRUE);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMem");
+}
+
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgePMRPDumpLoadMemValue32(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+                                        psMemDesc->psImport->hPMR,
+                                        psMemDesc->uiOffset + uiOffset,
+                                        ui32Value,
+                                        uiPDumpFlags);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMemValue32");
+}
+
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgePMRPDumpLoadMemValue64(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                      psMemDesc->psImport->hPMR,
+	                                      psMemDesc->uiOffset + uiOffset,
+	                                      ui64Value,
+	                                      uiPDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMemValue64");
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC		*psMemDesc,
+                              IMG_DEVMEM_OFFSET_T	*puiMemOffset,
+                              IMG_CHAR				*pszName,
+                              IMG_UINT32			ui32Size)
+{
+	PVRSRV_ERROR		eError;
+	IMG_CHAR			aszMemspaceName[100];
+	IMG_CHAR			aszSymbolicName[100];
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+
+	*puiMemOffset += psMemDesc->uiOffset;
+
+	eError = BridgePMRPDumpSymbolicAddr(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                    psMemDesc->psImport->hPMR,
+	                                    *puiMemOffset,
+	                                    sizeof(aszMemspaceName),
+	                                    &aszMemspaceName[0],
+	                                    sizeof(aszSymbolicName),
+	                                    &aszSymbolicName[0],
+	                                    puiMemOffset,
+	                                    &uiNextSymName);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpSymbolicAddr");
+
+	OSSNPrintf(pszName, ui32Size, "%s:%s", &aszMemspaceName[0], &aszSymbolicName[0]);
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgePMRPDumpSaveToFile(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                  psMemDesc->psImport->hPMR,
+	                                  psMemDesc->uiOffset + uiOffset,
+	                                  uiSize,
+	                                  OSStringLength(pszFilename) + 1,
+	                                  pszFilename,
+	                                  uiFileOffset);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpSaveToFile");
+}
+
+IMG_INTERNAL void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevAddrStart;
+
+	sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset;
+	sDevAddrStart.uiAddr += uiOffset;
+
+	eError = BridgeDevmemIntPDumpSaveToFileVirtual(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                               psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+	                                               sDevAddrStart,
+	                                               uiSize,
+	                                               OSStringLength(pszFilename) + 1,
+	                                               pszFilename,
+	                                               ui32FileOffset,
+	                                               ui32PdumpFlags);
+
+	PVR_LOG_IF_ERROR(eError, "BridgeDevmemIntPDumpSaveToFileVirtual");
+}
+
+IMG_INTERNAL void
+DevmemPDumpDataDescriptor(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_DEVMEM_SIZE_T uiSize,
+                          const IMG_CHAR *pszFilename,
+                          IMG_UINT32 ui32HeaderType,
+                          IMG_UINT32 ui32ElementType,
+                          IMG_UINT32 ui32ElementCount,
+                          IMG_UINT32 ui32PdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevAddrStart;
+
+	sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset;
+	sDevAddrStart.uiAddr += uiOffset;
+
+	eError = BridgePDumpDataDescriptor(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                   psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+                                       OSStringLength(pszFilename) + 1,
+	                                   pszFilename,
+	                                   sDevAddrStart,
+	                                   uiSize,
+	                                   ui32HeaderType,
+	                                   ui32ElementType,
+	                                   ui32ElementCount,
+	                                   ui32PdumpFlags);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePDumpDataDescriptor");
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiNumBytes;
+
+	uiNumBytes = 4;
+
+	if (psMemDesc->uiOffset + uiOffset + uiNumBytes > psMemDesc->psImport->uiSize)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+	}
+
+	eError = BridgePMRPDumpPol32(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                             psMemDesc->psImport->hPMR,
+	                             psMemDesc->uiOffset + uiOffset,
+	                             ui32Value,
+	                             ui32Mask,
+	                             eOperator,
+	                             ui32PDumpFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow
+	 */
+
+e0:
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError != PVRSRV_OK);
+	}
+	return eError;
+}
+
+#if defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpDevmemCheck32(const DEVMEM_MEMDESC *psMemDesc,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_UINT32 ui32Value,
+                         IMG_UINT32 ui32Mask,
+                         PDUMP_POLL_OPERATOR eOperator,
+                         PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiNumBytes;
+
+	uiNumBytes = 4;
+
+	if (psMemDesc->uiOffset + uiOffset + uiNumBytes >= psMemDesc->psImport->uiSize)
+	{
+		eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+		goto e0;
+	}
+
+	eError = BridgePMRPDumpCheck32(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                             psMemDesc->psImport->hPMR,
+	                             psMemDesc->uiOffset + uiOffset,
+	                             ui32Value,
+	                             ui32Mask,
+	                             eOperator,
+	                             ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow
+	 */
+
+e0:
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError != PVRSRV_OK);
+	}
+	return eError;
+}
+#endif /* defined(__KERNEL__) */
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+
+	if ((psMemDesc->uiOffset + uiReadOffset) > psMemDesc->psImport->uiSize)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+	}
+
+	eError = BridgePMRPDumpCBP(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                           psMemDesc->psImport->hPMR,
+	                           psMemDesc->uiOffset + uiReadOffset,
+	                           uiWriteOffset,
+	                           uiPacketSize,
+	                           uiBufferSize);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+e0:
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError != PVRSRV_OK);
+	}
+	return eError;
+}
+
+#endif /* PDUMP */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_utils.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_utils.c
new file mode 100644
index 000000000000..52bb9c838b52
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_utils.c
@@ -0,0 +1,1245 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management internal utility functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally by device memory management
+                code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "allocmem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "ra.h"
+#include "devicemem_utils.h"
+#include "client_mm_bridge.h"
+#include "client_cache_bridge.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "client_ri_bridge.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#else
+#include "pvr_bridge_client.h"
+#endif
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "proc_stats.h"
+#endif
+
+#if defined(__KERNEL__)
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+/*
+	SVM heap management support functions for CPU (un)mapping
+ */
+#define DEVMEM_MAP_SVM_USER_MANAGED_RETRY				2
+
+static inline PVRSRV_ERROR
+DevmemCPUMapSVMKernelManaged(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT64 *ui64MapAddress)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64SvmMapAddr;
+	IMG_UINT64 ui64SvmMapAddrEnd;
+	IMG_UINT64 ui64SvmHeapAddrEnd;
+
+	/* SVM heap management always has XXX_MANAGER_KERNEL unless we
+	   have triggered the fall back code-path in which case we
+	   should not be calling into this code-path */
+	PVR_ASSERT(psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_KERNEL);
+
+	/* By acquiring the CPU virtual address here, it essentially
+	   means we lock-down the virtual address for the duration
+	   of the life-cycle of the allocation until a de-allocation
+	   request comes in. Thus the allocation is guaranteed not to
+	   change its virtual address on the CPU during its life-time.
+	   NOTE: Import might have already been CPU Mapped before now,
+	   normally this is not a problem, see fall back */
+	eError = DevmemImportStructCPUMap(psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "DevmemImportStructCPUMap");
+		eError = PVRSRV_ERROR_DEVICEMEM_MAP_FAILED;
+		goto failSVM;
+	}
+
+	/* Supplied kernel mmap virtual address is also device virtual address;
+	   calculate the heap & kernel supplied mmap virtual address limits */
+	ui64SvmMapAddr = (IMG_UINT64)(uintptr_t)psImport->sCPUImport.pvCPUVAddr;
+	ui64SvmHeapAddrEnd = psHeap->sBaseAddress.uiAddr + psHeap->uiSize;
+	ui64SvmMapAddrEnd = ui64SvmMapAddr + psImport->uiSize;
+	PVR_ASSERT(ui64SvmMapAddr != (IMG_UINT64)0);
+
+	/* SVM limit test may fail if processor has more virtual address bits than device */
+	if ((ui64SvmMapAddr >= ui64SvmHeapAddrEnd || ui64SvmMapAddrEnd > ui64SvmHeapAddrEnd) ||
+		(ui64SvmMapAddr & ~(ui64SvmHeapAddrEnd - 1)))
+	{
+		/* Unmap incompatible SVM virtual address, this
+		   may not release address if it was elsewhere
+		   CPU Mapped before call into this function */
+		DevmemImportStructCPUUnmap(psImport);
+
+		/* Flag incompatible SVM mapping */
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto failSVM;
+	}
+
+	*ui64MapAddress = ui64SvmMapAddr;
+failSVM:
+	/* either OK, MAP_FAILED or BAD_MAPPING */
+	return eError;
+}
+
+static inline void
+DevmemCPUUnmapSVMKernelManaged(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	PVR_UNREFERENCED_PARAMETER(psHeap);
+	DevmemImportStructCPUUnmap(psImport);
+}
+
+static inline PVRSRV_ERROR
+DevmemCPUMapSVMUserManaged(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT uiAlign,
+		IMG_UINT64 *ui64MapAddress)
+{
+	RA_LENGTH_T uiAllocatedSize;
+	RA_BASE_T uiAllocatedAddr;
+	IMG_UINT64 ui64SvmMapAddr;
+	IMG_UINT uiRetry = 0;
+	PVRSRV_ERROR eError;
+
+	/* If SVM heap management has transitioned to XXX_MANAGER_USER,
+	   this is essentially a fall back approach that ensures we
+	   continue to satisfy SVM alloc. This approach is not without
+	   hazards in that we may specify a virtual address that is
+	   already in use by the user process */
+	PVR_ASSERT(psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER);
+
+	/* Normally, for SVM heap allocations, CPUMap _must_ be done
+	   before DevMap; ideally the initial CPUMap should be done by
+	   SVM functions though this is not a hard requirement as long
+	   as the prior elsewhere obtained CPUMap virtual address meets
+	   SVM address requirements. This is a fall-back code-pathway
+	   so we have to test that this assumption holds before we
+	   progress any further */
+	OSLockAcquire(psImport->sCPUImport.hLock);
+
+	if (psImport->sCPUImport.ui32RefCount)
+	{
+		/* Already CPU Mapped SVM heap allocation, this prior elsewhere
+		   obtained virtual address is responsible for the above
+		   XXX_MANAGER_KERNEL failure. As we are not responsible for
+		   this, we cannot progress any further so need to fail */
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Previously obtained CPU map address not SVM compatible"
+				, __func__));
+
+		/* Revert SVM heap to DEVMEM_HEAP_MANAGER_KERNEL */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_KERNEL;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"%s: Reverting SVM heap back to kernel managed",
+				__func__));
+
+		OSLockRelease(psImport->sCPUImport.hLock);
+
+		/* Do we need a more specific error code here */
+		eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED;
+		goto failSVM;
+	}
+
+	OSLockRelease(psImport->sCPUImport.hLock);
+
+	do
+	{
+		/* Next we proceed to instruct the kernel to use the RA_Alloc supplied
+		   virtual address to map-in this SVM import suballocation; there is no
+		   guarantee that this RA_Alloc virtual address may not collide with an
+		   already in-use VMA range in the process */
+		eError = RA_Alloc(psHeap->psQuantizedVMRA,
+				psImport->uiSize,
+				RA_NO_IMPORT_MULTIPLIER,
+				0, /* flags: this RA doesn't use flags*/
+				uiAlign,
+				"SVM_Virtual_Alloc",
+				&uiAllocatedAddr,
+				&uiAllocatedSize,
+				NULL /* don't care about per-import priv data */);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "RA_Alloc");
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			if (eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+			{
+				PVRSRV_ERROR eErr;
+				eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+								  PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT,
+								  OSGetCurrentProcessID());
+				PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+			}
+#endif
+			goto failSVM;
+		}
+
+		/* No reason for allocated virtual size to be different from
+		   the PMR's size */
+		psImport->sCPUImport.pvCPUVAddr = (void*)(uintptr_t)uiAllocatedAddr;
+		PVR_ASSERT(uiAllocatedSize == psImport->uiSize);
+
+		/* Map the import or allocation using the RA_Alloc virtual address;
+		   the kernel may fail the request if the supplied virtual address
+		   is already in-use in which case we re-try using another virtual
+		   address obtained from the RA_Alloc */
+		eError = DevmemImportStructCPUMap(psImport);
+		if (eError != PVRSRV_OK)
+		{
+			/* For now we simply discard failed RA_Alloc() obtained virtual
+			   address (i.e. plenty of virtual space), this prevents us from
+			   re-using these and furthermore essentially blacklists these
+			   addresses from future SVM consideration; We exit fall-back
+			   attempt if retry exceeds the fall-back retry limit */
+			if (uiRetry++ > DEVMEM_MAP_SVM_USER_MANAGED_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Cannot find SVM compatible address, bad mapping",
+						__func__));
+				eError = PVRSRV_ERROR_BAD_MAPPING;
+				goto failSVM;
+			}
+		}
+		else
+		{
+			/* Found compatible SVM virtual address, set as device virtual address */
+			ui64SvmMapAddr = (IMG_UINT64)(uintptr_t)psImport->sCPUImport.pvCPUVAddr;
+		}
+	} while (eError != PVRSRV_OK);
+
+	*ui64MapAddress = ui64SvmMapAddr;
+failSVM:
+	return eError;
+}
+
+static inline void
+DevmemCPUUnmapSVMUserManaged(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	RA_BASE_T uiAllocatedAddr;
+
+	/* We only free SVM compatible addresses, all addresses in
+	   the blacklist are essentially excluded from future RA_Alloc */
+	uiAllocatedAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	RA_Free(psHeap->psQuantizedVMRA, uiAllocatedAddr);
+
+	DevmemImportStructCPUUnmap(psImport);
+}
+
+static inline PVRSRV_ERROR
+DevmemImportStructDevMapSVM(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT uiAlign,
+		IMG_UINT64 *ui64MapAddress)
+{
+	PVRSRV_ERROR eError;
+
+	switch (psHeap->ui32HeapManagerFlags)
+	{
+	case DEVMEM_HEAP_MANAGER_KERNEL:
+		eError = DevmemCPUMapSVMKernelManaged(psHeap,
+				psImport,
+				ui64MapAddress);
+		if (eError == PVRSRV_ERROR_BAD_MAPPING)
+		{
+			/* If the SVM map address is outside of SVM heap limits,
+				   change heap type to DEVMEM_HEAP_MANAGER_USER */
+			psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_USER;
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Kernel managed SVM heap is now user managed",
+					__func__));
+
+			/* Retry using user managed fall-back approach */
+			eError = DevmemCPUMapSVMUserManaged(psHeap,
+					psImport,
+					uiAlign,
+					ui64MapAddress);
+		}
+		break;
+
+	case DEVMEM_HEAP_MANAGER_USER:
+		eError = DevmemCPUMapSVMUserManaged(psHeap,
+				psImport,
+				uiAlign,
+				ui64MapAddress);
+		break;
+
+	default:
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		break;
+	}
+
+	return eError;
+}
+
+static inline void
+DevmemImportStructDevUnmapSVM(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	switch (psHeap->ui32HeapManagerFlags)
+	{
+	case DEVMEM_HEAP_MANAGER_KERNEL:
+		DevmemCPUUnmapSVMKernelManaged(psHeap, psImport);
+		break;
+
+	case DEVMEM_HEAP_MANAGER_USER:
+		DevmemCPUUnmapSVMUserManaged(psHeap, psImport);
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+	The Devmem import structure is the structure we use
+	to manage memory that is "imported" (which is page
+	granular) from the server into our process, this
+	includes allocations.
+
+	This allows memory to be imported without requiring
+	any CPU or device mapping. Memory can then be mapped
+	into the device or CPU on demand, but neither is
+	required.
+ */
+
+IMG_INTERNAL
+void DevmemImportStructAcquire(DEVMEM_IMPORT *psImport)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psImport->hRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+	PVR_ASSERT(iRefCount != 1);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			iRefCount-1,
+			iRefCount);
+}
+
+IMG_INTERNAL
+IMG_BOOL DevmemImportStructRelease(DEVMEM_IMPORT *psImport)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psImport->hRefCount);
+	PVR_ASSERT(iRefCount >= 0);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			iRefCount+1,
+			iRefCount);
+
+	if (iRefCount == 0)
+	{
+		PVRSRV_ERROR eError = DestroyServerResource(psImport->hDevConnection,
+		                                            NULL,
+		                                            BridgePMRUnrefPMR,
+		                                            psImport->hPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		OSLockDestroy(psImport->sCPUImport.hLock);
+		OSLockDestroy(psImport->sDeviceImport.hLock);
+		OSLockDestroy(psImport->hLock);
+		OSFreeMem(psImport);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_INTERNAL
+void DevmemImportDiscard(DEVMEM_IMPORT *psImport)
+{
+	PVR_ASSERT(OSAtomicRead(&psImport->hRefCount) == 0);
+	OSLockDestroy(psImport->sCPUImport.hLock);
+	OSLockDestroy(psImport->sDeviceImport.hLock);
+	OSLockDestroy(psImport->hLock);
+	OSFreeMem(psImport);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc)
+{
+	DEVMEM_MEMDESC *psMemDesc;
+	PVRSRV_ERROR eError;
+
+	/* Must be zeroed in case it needs to be freed before it is initialised */
+	psMemDesc = OSAllocZMem(sizeof(DEVMEM_MEMDESC));
+	PVR_GOTO_IF_NOMEM(psMemDesc, eError, failAlloc);
+
+	eError = OSLockCreate(&psMemDesc->hLock);
+	PVR_GOTO_IF_ERROR(eError, failMDLock);
+
+	eError = OSLockCreate(&psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_ERROR(eError, failDMDLock);
+
+	eError = OSLockCreate(&psMemDesc->sCPUMemDesc.hLock);
+	PVR_GOTO_IF_ERROR(eError, failCMDLock);
+
+	OSAtomicWrite(&psMemDesc->hRefCount, 0);
+
+	*ppsMemDesc = psMemDesc;
+
+	return PVRSRV_OK;
+
+failCMDLock:
+	OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+failDMDLock:
+	OSLockDestroy(psMemDesc->hLock);
+failMDLock:
+	OSFreeMem(psMemDesc);
+failAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	Init the MemDesc structure
+ */
+IMG_INTERNAL
+void DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
+		IMG_DEVMEM_OFFSET_T uiOffset,
+		DEVMEM_IMPORT *psImport,
+		IMG_DEVMEM_SIZE_T uiSize)
+{
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			0,
+			1);
+
+	psMemDesc->psImport = psImport;
+	psMemDesc->uiOffset = uiOffset;
+
+	psMemDesc->sDeviceMemDesc.ui32RefCount = 0;
+	psMemDesc->sCPUMemDesc.ui32RefCount = 0;
+	psMemDesc->uiAllocSize = uiSize;
+	psMemDesc->hPrivData = NULL;
+	psMemDesc->ui32AllocationIndex = DEVICEMEM_HISTORY_ALLOC_INDEX_NONE;
+
+#if defined(DEBUG)
+	psMemDesc->bPoisonOnFree = IMG_FALSE;
+#endif
+
+	OSAtomicWrite(&psMemDesc->hRefCount, 1);
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+void DevmemMemDescSetPoF(DEVMEM_MEMDESC *psMemDesc, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		psMemDesc->bPoisonOnFree = IMG_TRUE;
+	}
+}
+#endif
+
+IMG_INTERNAL
+void DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc)
+{
+	IMG_INT iRefCount = 0;
+
+	iRefCount = OSAtomicIncrement(&psMemDesc->hRefCount);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			iRefCount-1,
+			iRefCount);
+
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+#if defined(DEBUG)
+static void _DevmemPoisonOnFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	void *pvAddr = NULL;
+	IMG_UINT8 *pui8CPUVAddr;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemCPUMapCheckImportProperties(psMemDesc);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "DevmemCPUMapCheckImportProperties");
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	eError = DevmemImportStructCPUMap(psMemDesc->psImport);
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "DevmemImportStructCPUMap");
+
+	pui8CPUVAddr = psMemDesc->psImport->sCPUImport.pvCPUVAddr;
+	pui8CPUVAddr += psMemDesc->uiOffset;
+	pvAddr = pui8CPUVAddr;
+
+	DevmemCPUMemSet(pvAddr,
+	                PVRSRV_POISON_ON_FREE_VALUE,
+	                psMemDesc->uiAllocSize,
+	                psMemDesc->psImport->uiFlags);
+
+	if (PVRSRV_CHECK_CPU_CACHE_COHERENT(psMemDesc->psImport->uiFlags) ||
+	    PVRSRV_CHECK_CPU_CACHE_INCOHERENT(psMemDesc->psImport->uiFlags))
+	{
+		eError = BridgeCacheOpExec(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+								   psMemDesc->psImport->hPMR,
+								   (IMG_UINT64) (uintptr_t)
+								   pvAddr - psMemDesc->uiOffset,
+								   psMemDesc->uiOffset,
+								   psMemDesc->uiAllocSize,
+								   PVRSRV_CACHE_OP_FLUSH);
+		PVR_LOG_IF_ERROR(eError, "BridgeCacheOpExec");
+	}
+
+	DevmemImportStructCPUUnmap(psMemDesc->psImport);
+	pvAddr = NULL;
+}
+#endif
+
+IMG_INTERNAL
+IMG_BOOL DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc)
+{
+	IMG_INT iRefCount;
+	PVR_ASSERT(psMemDesc != NULL);
+
+	iRefCount = OSAtomicDecrement(&psMemDesc->hRefCount);
+	PVR_ASSERT(iRefCount >= 0);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			iRefCount+1,
+			iRefCount);
+
+	if (iRefCount == 0)
+	{
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI) &&
+		    (psMemDesc->hRIHandle))
+		{
+			PVRSRV_ERROR eError;
+
+			eError = DestroyServerResource(psMemDesc->psImport->hDevConnection,
+			                               NULL,
+			                               BridgeRIDeleteMEMDESCEntry,
+			                               psMemDesc->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIDeleteMEMDESCEntry");
+		}
+#endif
+
+		OSLockAcquire(psMemDesc->psImport->hLock);
+		if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SUBALLOCATABLE)
+		{
+			/* As soon as the first sub-allocation on the psImport is freed
+			 * we might get dirty memory when reusing it.
+			 * We have to delete the ZEROED, CLEAN & POISONED flag */
+			psMemDesc->psImport->uiProperties &=
+					~(DEVMEM_PROPERTIES_IMPORT_IS_ZEROED |
+							DEVMEM_PROPERTIES_IMPORT_IS_CLEAN |
+							DEVMEM_PROPERTIES_IMPORT_IS_POISONED);
+
+			OSLockRelease(psMemDesc->psImport->hLock);
+
+#if defined(DEBUG)
+			if (psMemDesc->bPoisonOnFree)
+			{
+				_DevmemPoisonOnFree(psMemDesc);
+			}
+#endif
+
+			RA_Free(psMemDesc->psImport->sDeviceImport.psHeap->psSubAllocRA,
+					psMemDesc->psImport->sDeviceImport.sDevVAddr.uiAddr +
+					psMemDesc->uiOffset);
+		}
+		else
+		{
+			OSLockRelease(psMemDesc->psImport->hLock);
+			DevmemImportStructRelease(psMemDesc->psImport);
+		}
+
+		OSLockDestroy(psMemDesc->sCPUMemDesc.hLock);
+		OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+		OSLockDestroy(psMemDesc->hLock);
+		OSFreeMem(psMemDesc);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_INTERNAL
+void DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(OSAtomicRead(&psMemDesc->hRefCount) == 0);
+
+	OSLockDestroy(psMemDesc->sCPUMemDesc.hLock);
+	OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+	OSLockDestroy(psMemDesc->hLock);
+	OSFreeMem(psMemDesc);
+}
+
+
+IMG_INTERNAL
+PVRSRV_ERROR DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &&
+			(*puiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Zero on Alloc and Poison on Alloc are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiAlign & (uiAlign-1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: The requested alignment is not a power of two.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Please request a non-zero size value.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* If zero or poison flags are set we have to have write access to the page. */
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(*puiFlags) ||
+	    PVRSRV_CHECK_POISON_ON_ALLOC(*puiFlags) ||
+#if defined(DEBUG)
+	    PVRSRV_CHECK_POISON_ON_FREE(*puiFlags) ||
+#endif
+	    PVRSRV_CHECK_CPU_WRITEABLE(*puiFlags))
+	{
+		(*puiFlags) |= PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	Allocate and init an import structure
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_IMPORT **ppsImport)
+{
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_ERROR eError;
+
+	psImport = OSAllocMem(sizeof(*psImport));
+	PVR_RETURN_IF_FALSE(psImport != NULL, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	/* Setup some known bad values for things we don't have yet */
+	psImport->sDeviceImport.hReservation = LACK_OF_RESERVATION_POISON;
+	psImport->sDeviceImport.hMapping = LACK_OF_MAPPING_POISON;
+	psImport->sDeviceImport.psHeap = NULL;
+	psImport->sDeviceImport.bMapped = IMG_FALSE;
+
+	eError = OSLockCreate(&psImport->sDeviceImport.hLock);
+	PVR_GOTO_IF_ERROR(eError, failDIOSLockCreate);
+
+	psImport->sCPUImport.hOSMMapData = NULL;
+	psImport->sCPUImport.pvCPUVAddr = NULL;
+
+	eError = OSLockCreate(&psImport->sCPUImport.hLock);
+	PVR_GOTO_IF_ERROR(eError, failCIOSLockCreate);
+
+	/* Set up common elements */
+	psImport->hDevConnection = hDevConnection;
+
+	/* Setup properties */
+	psImport->uiProperties = 0;
+
+	/* Setup refcounts */
+	psImport->sDeviceImport.ui32RefCount = 0;
+	psImport->sCPUImport.ui32RefCount = 0;
+	OSAtomicWrite(&psImport->hRefCount, 0);
+
+	/* Create the lock */
+	eError = OSLockCreate(&psImport->hLock);
+	PVR_GOTO_IF_ERROR(eError, failILockAlloc);
+
+	*ppsImport = psImport;
+
+	return PVRSRV_OK;
+
+failILockAlloc:
+	OSLockDestroy(psImport->sCPUImport.hLock);
+failCIOSLockCreate:
+	OSLockDestroy(psImport->sDeviceImport.hLock);
+failDIOSLockCreate:
+	OSFreeMem(psImport);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	Initialise the import structure
+ */
+IMG_INTERNAL
+void DevmemImportStructInit(DEVMEM_IMPORT *psImport,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		IMG_HANDLE hPMR,
+		DEVMEM_PROPERTIES_T uiProperties)
+{
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			0,
+			1);
+
+	psImport->uiSize = uiSize;
+	psImport->uiAlign = uiAlign;
+	psImport->uiFlags = uiFlags;
+	psImport->hPMR = hPMR;
+	psImport->uiProperties = uiProperties;
+	OSAtomicWrite(&psImport->hRefCount, 1);
+}
+
+/* Allocate the requested device virtual address region
+ * from the heap */
+static PVRSRV_ERROR DevmemReserveVARange(DEVMEM_HEAP *psHeap,
+                                         DEVMEM_SIZE_T uiSize,
+                                         IMG_UINT uiAlign,
+                                         RA_LENGTH_T *puiAllocatedSize,
+                                         IMG_UINT64 ui64OptionalMapAddress)
+{
+	PVRSRV_ERROR eError;
+
+	/* Allocate space in the VM */
+	eError = RA_Alloc_Range(psHeap->psQuantizedVMRA,
+							uiSize,
+							0,
+							uiAlign,
+							ui64OptionalMapAddress,
+							puiAllocatedSize);
+
+	if (PVRSRV_OK != eError)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		if ((eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL) ||
+				(eError == PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL))
+		{
+			PVRSRV_ERROR eErr;
+			eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+											PVRSRV_DEVICE_STAT_TYPE_INVALID_VIRTMEM,
+											OSGetCurrentProcessID());
+			PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+		}
+#endif
+		return eError;
+	}
+
+	/* No reason for the allocated virtual size to be different from
+					   the PMR's size */
+	PVR_ASSERT(*puiAllocatedSize == uiSize);
+
+	return PVRSRV_OK;
+}
+
+/*
+	Map an import to the device
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
+		IMG_BOOL bMap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT64 ui64OptionalMapAddress)
+{
+	DEVMEM_DEVICE_IMPORT *psDeviceImport;
+	RA_BASE_T uiAllocatedAddr;
+	RA_LENGTH_T uiAllocatedSize;
+	IMG_DEV_VIRTADDR sBase;
+	PVRSRV_ERROR eError;
+	IMG_UINT uiAlign;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+
+	/* Round the provided import alignment to the configured heap alignment */
+	uiAlign = 1ULL << psHeap->uiLog2ImportAlignment;
+	uiAlign = PVR_ALIGN(psImport->uiAlign, uiAlign);
+
+	psDeviceImport = &psImport->sDeviceImport;
+
+	OSLockAcquire(psDeviceImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psDeviceImport->ui32RefCount,
+			psDeviceImport->ui32RefCount+1);
+
+	if (psDeviceImport->ui32RefCount++ == 0)
+	{
+		DevmemImportStructAcquire(psImport);
+
+		OSAtomicIncrement(&psHeap->hImportCount);
+
+		if (PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+		{
+			/*  SVM (shared virtual memory) imports or allocations always
+				need to acquire CPU virtual address first as address is
+				used to map the allocation into the device virtual address
+				space; i.e. the virtual address of the allocation for both
+				the CPU/GPU must be identical. */
+			eError = DevmemImportStructDevMapSVM(psHeap,
+					psImport,
+					uiAlign,
+					&ui64OptionalMapAddress);
+			PVR_GOTO_IF_ERROR(eError, failVMRAAlloc);
+		}
+
+		if (ui64OptionalMapAddress == 0)
+		{
+			/* If heap is _completely_ managed by USER or KERNEL, we shouldn't
+			 * be here, as this is RA manager code-path */
+			if (psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ||
+				psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_KERNEL)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ?
+						"%s: Heap is user managed, please use PVRSRVMapToDeviceAddress().":
+						"%s: Heap is kernel managed, use right allocation flags (e.g. SVM).",
+						__func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			if (psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_UNKNOWN)
+			{
+				/* Only set the heap manager (to RA) at first map when heap manager
+				 * is unknown. It might be a dual heap (both, user and RA managed),
+				 * in which case heap manager is set at creation time */
+				psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_RA;
+			}
+
+			/* Allocate space in the VM */
+			eError = RA_Alloc(psHeap->psQuantizedVMRA,
+					psImport->uiSize,
+					RA_NO_IMPORT_MULTIPLIER,
+					0, /* flags: this RA doesn't use flags*/
+					uiAlign,
+					"Virtual_Alloc",
+					&uiAllocatedAddr,
+					&uiAllocatedSize,
+					NULL /* don't care about per-import priv data */
+			);
+			if (PVRSRV_OK != eError)
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				if (eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+				{
+					PVRSRV_ERROR eErr;
+					eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+									  PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT,
+									  OSGetCurrentProcessID());
+					PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+				}
+#endif
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_DEVICE_VM, failVMRAAlloc);
+			}
+
+			/* No reason for the allocated virtual size to be different from
+			   the PMR's size */
+			PVR_ASSERT(uiAllocatedSize == psImport->uiSize);
+
+			sBase.uiAddr = uiAllocatedAddr;
+
+		}
+		else
+		{
+			IMG_UINT64 ui64ValidEndAddr;
+
+			/* Ensure supplied ui64OptionalMapAddress is within heap range */
+			ui64ValidEndAddr = psHeap->sBaseAddress.uiAddr + psHeap->uiSize;
+			if ((ui64OptionalMapAddress + psImport->uiSize > ui64ValidEndAddr) ||
+					(ui64OptionalMapAddress < psHeap->sBaseAddress.uiAddr))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: ui64OptionalMapAddress %p is outside of heap limits <%p:%p>."
+						, __func__
+						, (void*)(uintptr_t)ui64OptionalMapAddress
+						, (void*)(uintptr_t)psHeap->sBaseAddress.uiAddr
+						, (void*)(uintptr_t)ui64ValidEndAddr));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			switch (psHeap->ui32HeapManagerFlags)
+			{
+				case DEVMEM_HEAP_MANAGER_UNKNOWN:
+					/* DEVMEM_HEAP_MANAGER_USER can apply to _any_ heap and can only
+					 * be determined here. This heap type transitions from
+					 * DEVMEM_HEAP_MANAGER_UNKNOWN to DEVMEM_HEAP_MANAGER_USER on
+					 * 1st alloc. */
+					psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_USER;
+					break;
+
+				case DEVMEM_HEAP_MANAGER_USER:
+				case DEVMEM_HEAP_MANAGER_KERNEL:
+					if (! psHeap->uiSize)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ?
+										"%s: Heap DEVMEM_HEAP_MANAGER_USER is disabled.":
+										"%s: Heap DEVMEM_HEAP_MANAGER_KERNEL is disabled."
+										, __func__));
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAP, failVMRAAlloc);
+					}
+					break;
+
+				case DEVMEM_HEAP_MANAGER_DUAL_USER_RA:
+					/* When the heap is dual managed, ensure supplied ui64OptionalMapAddress
+					 * and import size are within heap address space range */
+					if (ui64OptionalMapAddress + psImport->uiSize <=
+							psHeap->sBaseAddress.uiAddr + psHeap->uiReservedRegionSize)
+					{
+						break;
+					}
+					else
+					{
+						/* Allocate requested VM range */
+						eError = DevmemReserveVARange(psHeap,
+													psImport->uiSize,
+													uiAlign,
+													&uiAllocatedSize,
+													ui64OptionalMapAddress);
+						if (eError != PVRSRV_OK)
+						{
+							PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED, failVMRAAlloc);
+						}
+
+					}
+					break;
+				case DEVMEM_HEAP_MANAGER_RA:
+					/* Allocate requested VM range */
+					eError = DevmemReserveVARange(psHeap,
+												psImport->uiSize,
+												uiAlign,
+												&uiAllocatedSize,
+												ui64OptionalMapAddress);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED, failVMRAAlloc);
+					}
+					break;
+
+				default:
+					break;
+			}
+
+			if (ui64OptionalMapAddress & ((1 << psHeap->uiLog2Quantum) - 1))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid address to map to. Please provide an "
+						"address aligned to a page multiple of the heap."
+						, __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			if (psImport->uiSize & ((1 << psHeap->uiLog2Quantum) - 1))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid heap to map to. "
+						"Please choose a heap that can handle smaller page sizes."
+						, __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			uiAllocatedAddr = ui64OptionalMapAddress;
+			uiAllocatedSize = psImport->uiSize;
+			sBase.uiAddr = uiAllocatedAddr;
+		}
+
+		if (psHeap->bPremapped)
+		{
+			/* No virtual address reservation and mapping are required for
+			 * memory that is already pre-mapped e.g. FW heaps in VZ configs */
+			psDeviceImport->hReservation = LACK_OF_RESERVATION_POISON;
+			psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+		}
+		else
+		{
+			PVRSRV_MEMALLOCFLAGS_T uiFlags;
+			uiFlags = psImport->uiFlags & PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK;
+			if (bMap)
+			{
+				eError = BridgeDevmemIntReserveRangeAndMapPMR(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+				                                              psHeap->hDevMemServerHeap,
+				                                              sBase,
+				                                              uiAllocatedSize,
+				                                              psImport->hPMR,
+				                                              uiFlags,
+				                                              &psDeviceImport->hReservation);
+
+				psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+				PVR_GOTO_IF_ERROR(eError, failReserve);
+
+				psDeviceImport->bMapped = IMG_TRUE;
+			}
+			else
+			{
+				/* Setup page tables for the allocated VM space */
+				eError = BridgeDevmemIntReserveRange(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+				                                     psHeap->hDevMemServerHeap,
+				                                     sBase,
+				                                     uiAllocatedSize,
+				                                     uiFlags,
+				                                     &psDeviceImport->hReservation);
+				PVR_GOTO_IF_ERROR(eError, failReserve);
+			}
+		}
+
+		/* Setup device mapping specific parts of the mapping info */
+		psDeviceImport->sDevVAddr.uiAddr = uiAllocatedAddr;
+		psDeviceImport->psHeap = psHeap;
+	}
+	else
+	{
+		/*
+			Check that we've been asked to map it into the
+			same heap 2nd time around
+		 */
+		if (psHeap != psDeviceImport->psHeap)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAP, failParams);
+		}
+	}
+	OSLockRelease(psDeviceImport->hLock);
+
+	return PVRSRV_OK;
+
+failReserve:
+	if (ui64OptionalMapAddress == 0)
+	{
+		RA_Free(psHeap->psQuantizedVMRA,
+				uiAllocatedAddr);
+	}
+failVMRAAlloc:
+	if ((ui64OptionalMapAddress) && PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+	{
+		DevmemImportStructDevUnmapSVM(psHeap, psImport);
+	}
+	bDestroyed = DevmemImportStructRelease(psImport);
+	OSAtomicDecrement(&psHeap->hImportCount);
+failParams:
+	if (!bDestroyed)
+	{
+		psDeviceImport->ui32RefCount--;
+		OSLockRelease(psDeviceImport->hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+	Unmap an import from the Device
+ */
+IMG_INTERNAL
+IMG_BOOL DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_DEVICE_IMPORT *psDeviceImport;
+
+	psDeviceImport = &psImport->sDeviceImport;
+
+	OSLockAcquire(psDeviceImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psDeviceImport->ui32RefCount,
+			psDeviceImport->ui32RefCount-1);
+
+	if (--psDeviceImport->ui32RefCount == 0)
+	{
+		DEVMEM_HEAP *psHeap = psDeviceImport->psHeap;
+
+		if (!psHeap->bPremapped)
+		{
+			eError = DestroyServerResource(psImport->hDevConnection,
+			                               NULL,
+			                               BridgeDevmemIntUnreserveRange,
+			                               psDeviceImport->hReservation);
+			PVR_ASSERT(eError == PVRSRV_OK);
+		}
+
+		psDeviceImport->bMapped = IMG_FALSE;
+		psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+		psDeviceImport->hReservation = LACK_OF_RESERVATION_POISON;
+
+		/* DEVMEM_HEAP_MANAGER_RA can also come from a dual managed heap in which case,
+		   we need to check if the allocated VA falls within RA managed range */
+		if ((psHeap->ui32HeapManagerFlags & DEVMEM_HEAP_MANAGER_RA) &&
+		    psDeviceImport->sDevVAddr.uiAddr >= (psHeap->sBaseAddress.uiAddr + psHeap->uiReservedRegionSize) &&
+		    psDeviceImport->sDevVAddr.uiAddr < (psHeap->sBaseAddress.uiAddr + psHeap->uiSize))
+		{
+			RA_Free(psHeap->psQuantizedVMRA, psDeviceImport->sDevVAddr.uiAddr);
+		}
+
+		if (PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+		{
+			DevmemImportStructDevUnmapSVM(psHeap, psImport);
+		}
+
+		OSLockRelease(psDeviceImport->hLock);
+
+		DevmemImportStructRelease(psImport);
+
+		OSAtomicDecrement(&psHeap->hImportCount);
+
+		return IMG_TRUE;
+	}
+	else
+	{
+		OSLockRelease(psDeviceImport->hLock);
+		return IMG_FALSE;
+	}
+}
+
+/*
+	Map an import into the CPU
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_CPU_IMPORT *psCPUImport;
+	size_t uiMappingLength;
+
+	psCPUImport = &psImport->sCPUImport;
+
+	OSLockAcquire(psCPUImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psCPUImport->ui32RefCount,
+			psCPUImport->ui32RefCount+1);
+
+	if (psCPUImport->ui32RefCount++ == 0)
+	{
+		DevmemImportStructAcquire(psImport);
+
+		eError = OSMMapPMR(GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				psImport->uiSize,
+				psImport->uiFlags,
+				&psCPUImport->hOSMMapData,
+				&psCPUImport->pvCPUVAddr,
+				&uiMappingLength);
+		PVR_GOTO_IF_ERROR(eError, failMap);
+
+		/* MappingLength might be rounded up to page size */
+		PVR_ASSERT(uiMappingLength >= psImport->uiSize);
+	}
+	OSLockRelease(psCPUImport->hLock);
+
+	return PVRSRV_OK;
+
+failMap:
+	psCPUImport->ui32RefCount--;
+	if (!DevmemImportStructRelease(psImport))
+	{
+		OSLockRelease(psCPUImport->hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+	Unmap an import from the CPU
+ */
+IMG_INTERNAL
+void DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport)
+{
+	DEVMEM_CPU_IMPORT *psCPUImport;
+
+	psCPUImport = &psImport->sCPUImport;
+
+	OSLockAcquire(psCPUImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psCPUImport->ui32RefCount,
+			psCPUImport->ui32RefCount-1);
+
+	if (--psCPUImport->ui32RefCount == 0)
+	{
+		/* psImport->uiSize is a 64-bit quantity whereas the 5th
+		 * argument to OSUnmapPMR is a 32-bit quantity on 32-bit systems
+		 * hence a compiler warning of implicit cast and loss of data.
+		 * Added explicit cast and assert to remove warning.
+		 */
+#if defined(__linux__) && defined(__i386__)
+		PVR_ASSERT(psImport->uiSize<IMG_UINT32_MAX);
+#endif
+		OSMUnmapPMR(GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				psCPUImport->hOSMMapData,
+				psCPUImport->pvCPUVAddr,
+				(size_t)psImport->uiSize);
+
+		psCPUImport->hOSMMapData = NULL;
+		psCPUImport->pvCPUVAddr = NULL;
+
+		OSLockRelease(psCPUImport->hLock);
+
+		DevmemImportStructRelease(psImport);
+	}
+	else
+	{
+		OSLockRelease(psCPUImport->hLock);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicememx_pdump.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicememx_pdump.c
new file mode 100644
index 000000000000..7b74db5f2cca
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicememx_pdump.c
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@File
+@Title          Shared X device memory management PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common (client & server) PDump functions for the
+                memory management code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#if defined(PDUMP)
+
+#include "devicememx_pdump.h"
+#include "pdump.h"
+#include "client_pdumpmm_bridge.h"
+#include "devicemem_utils.h"
+
+IMG_INTERNAL void
+DevmemXPDumpLoadMem(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiSize != 0);
+	PVR_ASSERT(uiOffset + uiSize <= (psMemDescPhys->uiNumPages <<
+			psMemDescPhys->uiLog2PageSize));
+
+	eError = BridgePMRPDumpLoadMem(GetBridgeHandle(psMemDescPhys->hConnection),
+	                               psMemDescPhys->hPMR,
+	                               uiOffset,
+	                               uiSize,
+	                               uiPDumpFlags,
+	                               IMG_FALSE);
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMem");
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash.c
new file mode 100644
index 000000000000..434c6e967bc4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash.c
@@ -0,0 +1,747 @@
+/*************************************************************************/ /*!
+@File
+@Title          Self scaling hash tables.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+   Implements simple self scaling hash tables. Hash collisions are handled by
+   chaining entries together. Hash tables are increased in size when they
+   become more than (50%?) full and decreased in size when less than (25%?)
+   full. Hash tables are never decreased below their initial size.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* include/ */
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+/* services/shared/include/ */
+#include "hash.h"
+
+/* services/client/include/ or services/server/include/ */
+#include "osfunc_common.h"
+#include "allocmem.h"
+
+//#define PERF_DBG_RESIZE
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+#include <sys/time.h>
+#endif
+
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#endif
+
+#define	KEY_TO_INDEX(pHash, key, uSize) \
+	((pHash)->pfnHashFunc((pHash)->uKeySize, (key), (uSize)) % (uSize))
+
+#define	KEY_COMPARE(pHash, pKey1, pKey2) \
+	((pHash)->pfnKeyComp((pHash)->uKeySize, (pKey1), (pKey2)))
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define _AllocMem OSAllocMemNoStats
+#define _AllocZMem OSAllocZMemNoStats
+#define _FreeMem OSFreeMemNoStats
+#else
+#define _AllocMem OSAllocMem
+#define _AllocZMem OSAllocZMem
+#define _FreeMem OSFreeMem
+#endif
+
+#define NO_SHRINK 0
+
+/* Each entry in a hash table is placed into a bucket */
+typedef struct _BUCKET_
+{
+	struct _BUCKET_ *pNext; /*!< the next bucket on the same chain */
+	uintptr_t v;            /*!< entry value */
+	uintptr_t k[];          /* PRQA S 0642 */
+	                        /* override dynamic array declaration warning */
+} BUCKET;
+
+struct _HASH_TABLE_
+{
+	IMG_UINT32 uSize;            /*!< current size of the hash table */
+	IMG_UINT32 uCount;           /*!< number of entries currently in the hash table */
+	IMG_UINT32 uMinimumSize;     /*!< the minimum size that the hash table should be re-sized to */
+	IMG_UINT32 uKeySize;         /*!< size of key in bytes */
+	IMG_UINT32 uShrinkThreshold; /*!< The threshold at which to trigger a shrink */
+	IMG_UINT32 uGrowThreshold;   /*!< The threshold at which to trigger a grow */
+	HASH_FUNC*     pfnHashFunc;  /*!< hash function */
+	HASH_KEY_COMP* pfnKeyComp;   /*!< key comparison function */
+	BUCKET**   ppBucketTable;    /*!< the hash table array */
+#if defined(DEBUG)
+	const char*      pszFile;
+	unsigned int     ui32LineNum;
+#endif
+};
+
+/*************************************************************************/ /*!
+@Function       HASH_Func_Default
+@Description    Hash function intended for hashing keys composed of uintptr_t
+                arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey         A pointer to the key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_UINT32
+HASH_Func_Default(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	uintptr_t *p = (uintptr_t *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(uintptr_t);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	PVR_ASSERT((uKeySize % sizeof(uintptr_t)) == 0);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = (IMG_UINT32)*p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares keys composed of uintptr_t arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey1        Pointer to first hash key to compare.
+@Input          pKey2        Pointer to second hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Key_Comp_Default(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	uintptr_t *p1 = (uintptr_t *)pKey1;
+	uintptr_t *p2 = (uintptr_t *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(uintptr_t);
+	IMG_UINT32 ui;
+
+	PVR_ASSERT((uKeySize % sizeof(uintptr_t)) == 0);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _ChainInsert
+@Description    Insert a bucket into the appropriate hash table chain.
+@Input          pBucket       The bucket
+@Input          ppBucketTable The hash table
+@Input          uSize         The size of the hash table
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static void
+_ChainInsert(HASH_TABLE *pHash, BUCKET *pBucket, BUCKET **ppBucketTable, IMG_UINT32 uSize)
+{
+	IMG_UINT32 uIndex;
+
+	/* We assume that all parameters passed by the caller are valid. */
+	PVR_ASSERT(pBucket != NULL);
+	PVR_ASSERT(ppBucketTable != NULL);
+	PVR_ASSERT(uSize != 0);
+
+	uIndex = KEY_TO_INDEX(pHash, pBucket->k, uSize);	/* PRQA S 0432,0541 */ /* ignore dynamic array warning */
+	pBucket->pNext = ppBucketTable[uIndex];
+	ppBucketTable[uIndex] = pBucket;
+}
+
+/*************************************************************************/ /*!
+@Function       _Rehash
+@Description    Iterate over every entry in an old hash table and rehash into
+                the new table.
+@Input          ppOldTable   The old hash table
+@Input          uOldSize     The size of the old hash table
+@Input          ppNewTable   The new hash table
+@Input          uNewSize     The size of the new hash table
+@Return         None
+*/ /**************************************************************************/
+static void
+_Rehash(HASH_TABLE *pHash,
+		 BUCKET **ppOldTable, IMG_UINT32 uOldSize,
+		 BUCKET **ppNewTable, IMG_UINT32 uNewSize)
+{
+	IMG_UINT32 uIndex;
+	for (uIndex=0; uIndex< uOldSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		pBucket = ppOldTable[uIndex];
+		while (pBucket != NULL)
+		{
+			BUCKET *pNextBucket = pBucket->pNext;
+			_ChainInsert(pHash, pBucket, ppNewTable, uNewSize);
+			pBucket = pNextBucket;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       _Resize
+@Description    Attempt to resize a hash table, failure to allocate a new
+                larger hash table is not considered a hard failure. We simply
+                continue and allow the table to fill up, the effect is to
+                allow hash chains to become longer.
+@Input          pHash        Hash table to resize.
+@Input          uNewSize     Required table size.
+@Return         IMG_TRUE Success
+                IMG_FALSE Failed
+*/ /**************************************************************************/
+static IMG_BOOL
+_Resize(HASH_TABLE *pHash, IMG_UINT32 uNewSize)
+{
+	BUCKET **ppNewTable;
+	IMG_UINT32 uiThreshold = uNewSize >> 2;
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	struct timeval start, end;
+#endif
+
+	if (uNewSize == pHash->uSize)
+	{
+		return IMG_TRUE;
+	}
+
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	gettimeofday(&start, NULL);
+#endif
+
+	ppNewTable = _AllocZMem(sizeof(BUCKET *) * uNewSize);
+	if (ppNewTable == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	_Rehash(pHash, pHash->ppBucketTable, pHash->uSize, ppNewTable, uNewSize);
+
+	_FreeMem(pHash->ppBucketTable);
+
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	gettimeofday(&end, NULL);
+	if (start.tv_usec > end.tv_usec)
+	{
+		end.tv_usec = 1000000 - start.tv_usec + end.tv_usec;
+	}
+	else
+	{
+		end.tv_usec -= start.tv_usec;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: H:%p O:%d N:%d C:%d G:%d S:%d T:%06luus", __func__, pHash, pHash->uSize, uNewSize, pHash->uCount, pHash->uGrowThreshold, pHash->uShrinkThreshold, end.tv_usec));
+#endif
+
+	/*not nulling pointer, being reassigned just below*/
+	pHash->ppBucketTable = ppNewTable;
+	pHash->uSize = uNewSize;
+
+	pHash->uGrowThreshold = uiThreshold * 3;
+	pHash->uShrinkThreshold = (uNewSize <= pHash->uMinimumSize) ? NO_SHRINK : uiThreshold;
+
+	return IMG_TRUE;
+}
+
+
+/*************************************************************************/ /*!
+@Function       HASH_Create_Extended
+@Description    Create a self scaling hash table, using the supplied key size,
+                and the supplied hash and key comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Input          uKeySize     The size of the key, in bytes.
+@Input          pfnHashFunc  Pointer to hash function.
+@Input          pfnKeyComp   Pointer to key comparison function.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Extended_Int (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp)
+{
+	HASH_TABLE *pHash;
+
+	if (uInitialLen == 0 || uKeySize == 0 || pfnHashFunc == NULL || pfnKeyComp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid input parameters", __func__));
+		return NULL;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: InitialSize=0x%x", __func__, uInitialLen));
+
+	pHash = _AllocMem(sizeof(HASH_TABLE));
+	if (pHash == NULL)
+	{
+		return NULL;
+	}
+
+	pHash->uCount = 0;
+	pHash->uSize = uInitialLen;
+	pHash->uMinimumSize = uInitialLen;
+	pHash->uKeySize = uKeySize;
+	pHash->uGrowThreshold = (uInitialLen >> 2) * 3;
+	pHash->uShrinkThreshold = NO_SHRINK;
+	pHash->pfnHashFunc = pfnHashFunc;
+	pHash->pfnKeyComp = pfnKeyComp;
+
+	pHash->ppBucketTable = _AllocZMem(sizeof(BUCKET *) * pHash->uSize);
+	if (pHash->ppBucketTable == NULL)
+	{
+		_FreeMem(pHash);
+		/*not nulling pointer, out of scope*/
+		return NULL;
+	}
+
+	return pHash;
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Extended_Debug (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp,
+										 const char *file, const unsigned int line)
+{
+	HASH_TABLE *hash;
+	hash = HASH_Create_Extended_Int(uInitialLen, uKeySize,
+									pfnHashFunc, pfnKeyComp);
+	if (hash)
+	{
+		hash->pszFile = file;
+		hash->ui32LineNum = line;
+	}
+	return hash;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Create
+@Description    Create a self scaling hash table with a key consisting of a
+                single uintptr_t, and using the default hash and key
+                comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Int (IMG_UINT32 uInitialLen)
+{
+	return HASH_Create_Extended_Int(uInitialLen, sizeof(uintptr_t),
+									&HASH_Func_Default, &HASH_Key_Comp_Default);
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Debug(IMG_UINT32 uInitialLen, const char *file, const unsigned int line)
+{
+	HASH_TABLE *hash;
+	hash = HASH_Create_Extended_Int(uInitialLen, sizeof(uintptr_t),
+									&HASH_Func_Default, &HASH_Key_Comp_Default);
+	if (hash)
+	{
+		hash->pszFile = file;
+		hash->ui32LineNum = line;
+	}
+	return hash;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete_Extended
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table should have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Input          bWarn        Set false to suppress warnings in the case of
+                             deletion with active entries.
+*/ /**************************************************************************/
+IMG_INTERNAL void
+HASH_Delete_Extended(HASH_TABLE *pHash, IMG_BOOL bWarn)
+{
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__) && !defined(__QNXNTO__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData != NULL)
+	{
+		if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			bDoCheck = IMG_FALSE;
+		}
+	}
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	else
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+#endif
+	if (pHash != NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HASH_Delete"));
+
+		if (bDoCheck)
+		{
+			PVR_ASSERT(pHash->uCount==0);
+		}
+		if (pHash->uCount != 0)
+		{
+			IMG_UINT32 i;
+			if (bWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Leak detected in hash table!", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Removing remaining %u hash entries.", __func__, pHash->uCount));
+#if defined(DEBUG)
+				PVR_DPF ((PVR_DBG_ERROR, "%s: Hash %p created at %s:%u.", __func__, (uintptr_t*)pHash, pHash->pszFile, pHash->ui32LineNum));
+#endif
+			}
+
+			for (i = 0; i < pHash->uSize; i++)
+			{
+				BUCKET *pBucket = pHash->ppBucketTable[i];
+				while (pBucket != NULL)
+				{
+					BUCKET *pNextBucket = pBucket->pNext;
+					_FreeMem(pBucket);
+					pBucket = pNextBucket;
+				}
+			}
+
+		}
+		_FreeMem(pHash->ppBucketTable);
+		pHash->ppBucketTable = NULL;
+		_FreeMem(pHash);
+		/*not nulling pointer, copy on stack*/
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table must have been removed
+                before calling this function.
+@Input          pHash        Hash table
+*/ /**************************************************************************/
+IMG_INTERNAL void
+HASH_Delete(HASH_TABLE *pHash)
+{
+	HASH_Delete_Extended(pHash, IMG_TRUE);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert_Extended
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to the key.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Insert_Extended(HASH_TABLE *pHash, void *pKey, uintptr_t v)
+{
+	BUCKET *pBucket;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter", __func__));
+		return IMG_FALSE;
+	}
+
+	pBucket = _AllocMem(sizeof(BUCKET) + pHash->uKeySize);
+	if (pBucket == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pBucket->v = v;
+	/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k (linux)*/
+	OSCachedMemCopy(pBucket->k, pKey, pHash->uKeySize);
+
+	_ChainInsert(pHash, pBucket, pHash->ppBucketTable, pHash->uSize);
+
+	pHash->uCount++;
+
+	/* check if we need to think about re-balancing */
+	if (pHash->uCount > pHash->uGrowThreshold)
+	{
+		/* Ignore the return code from _Resize because the hash table is
+		   still in a valid state and although not ideally sized, it is still
+		   functional */
+		_Resize(pHash, pHash->uSize << 1);
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Insert(HASH_TABLE *pHash, uintptr_t k, uintptr_t v)
+{
+	return HASH_Insert_Extended(pHash, &k, v);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove_Extended
+@Description    Remove a key from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Remove_Extended(HASH_TABLE *pHash, void *pKey)
+{
+	BUCKET **ppBucket;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Null hash table", __func__));
+		return 0;
+	}
+
+	uIndex = KEY_TO_INDEX(pHash, pKey, pHash->uSize);
+
+	for (ppBucket = &(pHash->ppBucketTable[uIndex]); *ppBucket != NULL; ppBucket = &((*ppBucket)->pNext))
+	{
+		/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k */
+		if (KEY_COMPARE(pHash, (*ppBucket)->k, pKey))
+		{
+			BUCKET *pBucket = *ppBucket;
+			uintptr_t v = pBucket->v;
+			(*ppBucket) = pBucket->pNext;
+
+			_FreeMem(pBucket);
+			/*not nulling original pointer, already overwritten*/
+
+			pHash->uCount--;
+
+			/* check if we need to think about re-balancing, when the shrink
+			 * threshold is 0 we are at the minimum size, no further shrink */
+			if (pHash->uCount < pHash->uShrinkThreshold)
+			{
+				/* Ignore the return code from _Resize because the
+				   hash table is still in a valid state and although
+				   not ideally sized, it is still functional */
+				_Resize(pHash, MAX(pHash->uSize >> 1, pHash->uMinimumSize));
+			}
+
+			return v;
+		}
+	}
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove
+@Description    Remove a key value pair from a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Remove(HASH_TABLE *pHash, uintptr_t k)
+{
+	return HASH_Remove_Extended(pHash, &k);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve_Extended
+@Description    Retrieve a value from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Retrieve_Extended(HASH_TABLE *pHash, void *pKey)
+{
+	BUCKET **ppBucket;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Null hash table", __func__));
+		return 0;
+	}
+
+	uIndex = KEY_TO_INDEX(pHash, pKey, pHash->uSize);
+
+	for (ppBucket = &(pHash->ppBucketTable[uIndex]); *ppBucket != NULL; ppBucket = &((*ppBucket)->pNext))
+	{
+		/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k */
+		if (KEY_COMPARE(pHash, (*ppBucket)->k, pKey))
+		{
+			BUCKET *pBucket = *ppBucket;
+			uintptr_t v = pBucket->v;
+
+			return v;
+		}
+	}
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve
+@Description    Retrieve a value from a hash table created with HASH_Create.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Retrieve(HASH_TABLE *pHash, uintptr_t k)
+{
+	return HASH_Retrieve_Extended(pHash, &k);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Iterate
+@Description    Iterate over every entry in the hash table.
+@Input          pHash        Hash table to iterate.
+@Input          pfnCallback  Callback to call with the key and data for each
+.                            entry in the hash table
+@Return         Callback error if any, otherwise PVRSRV_OK
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HASH_Iterate(HASH_TABLE *pHash, HASH_pfnCallback pfnCallback, void* args)
+{
+	IMG_UINT32 uIndex;
+	for (uIndex=0; uIndex < pHash->uSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		pBucket = pHash->ppBucketTable[uIndex];
+		while (pBucket != NULL)
+		{
+			PVRSRV_ERROR eError;
+			BUCKET *pNextBucket = pBucket->pNext;
+
+			eError = pfnCallback((uintptr_t) ((void *) *(pBucket->k)), pBucket->v, args);
+
+			/* The callback might want us to break out early */
+			if (eError != PVRSRV_OK)
+				return eError;
+
+			pBucket = pNextBucket;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Count
+@Description    Retrieve the number of entries in the hash table.
+@Input          pHash        The hash table.
+@Return         The number of entries.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_UINT32
+HASH_Count(HASH_TABLE *pHash)
+{
+	return pHash->uCount;
+}
+
+
+#ifdef HASH_TRACE
+/*************************************************************************/ /*!
+@Function       HASH_Dump
+@Description    Dump out some information about a hash table.
+@Input          pHash         The hash table.
+*/ /**************************************************************************/
+void
+HASH_Dump(HASH_TABLE *pHash)
+{
+	IMG_UINT32 uIndex;
+	IMG_UINT32 uMaxLength=0;
+	IMG_UINT32 uEmptyCount=0;
+
+	PVR_ASSERT(pHash != NULL);
+	for (uIndex=0; uIndex<pHash->uSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		IMG_UINT32 uLength = 0;
+		if (pHash->ppBucketTable[uIndex] == NULL)
+		{
+			uEmptyCount++;
+		}
+		for (pBucket=pHash->ppBucketTable[uIndex];
+				pBucket != NULL;
+				pBucket = pBucket->pNext)
+		{
+			uLength++;
+		}
+		uMaxLength = MAX(uMaxLength, uLength);
+	}
+
+	PVR_TRACE(("hash table: uMinimumSize=%d  size=%d  count=%d",
+			   pHash->uMinimumSize, pHash->uSize, pHash->uCount));
+	PVR_TRACE(("  empty=%d  max=%d", uEmptyCount, uMaxLength));
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash_functions.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash_functions.c
new file mode 100644
index 000000000000..517dae297ff8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash_functions.c
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          Reusable hash functions for hash.c.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common hash functions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "hash_functions.h"
+#include "img_defs.h"
+
+/* Declaring function here to avoid dependencies that are introduced by
+ * including osfunc.h. */
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                           size_t uiSize);
+
+IMG_UINT32 HASH_Djb2_Hash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_CHAR *pszStr = pKey;
+	IMG_UINT32 ui32Hash = 5381, ui32Char;
+
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	while ((ui32Char = *pszStr++) != '\0')
+	{
+		ui32Hash = ((ui32Hash << 5) + ui32Hash) + ui32Char;
+	}
+
+	return ui32Hash;
+}
+
+IMG_BOOL HASH_Djb2_Compare(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_CHAR *pszKey1 = pKey1, *pszKey2 = pKey2;
+
+	return OSStringNCompare(pszKey1, pszKey2, uKeySize) == 0;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/htbuffer.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/htbuffer.c
new file mode 100644
index 000000000000..5f83127a5462
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/htbuffer.c
@@ -0,0 +1,106 @@
+/*************************************************************************/ /*!
+@File           htbuffer.c
+@Title          Host Trace Buffer shared API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "htbuffer.h"
+#include "osfunc.h"
+#include "client_htbuffer_bridge.h"
+
+/* The group flags array of ints large enough to store all the group flags
+ * NB: This will only work while all logging is in the kernel
+ */
+IMG_INTERNAL HTB_FLAG_EL_T g_auiHTBGroupEnable[HTB_FLAG_NUM_EL] = {0};
+
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+ @Input         hSrvHandle      Server Handle
+ @Input         ui32NumFlagGroups Number of group enable flags words
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+ @Input         ui32LogLevel    Log level to record
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+ @Input         eLogPidMode     Enable logging for all or specific processes,
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogPidMode,
+	HTB_OPMODE_CTRL eOpMode
+)
+{
+	return BridgeHTBControl(
+			hSrvHandle,
+			ui32NumFlagGroups,
+			aui32GroupEnable,
+			ui32LogLevel,
+			ui32EnablePID,
+			eLogPidMode,
+			eOpMode
+			);
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/mem_utils.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/mem_utils.c
new file mode 100644
index 000000000000..59f808cfcb20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/mem_utils.c
@@ -0,0 +1,445 @@
+/*************************************************************************/ /*!
+@File
+@Title          Memory manipulation functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory related functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "osfunc_common.h"
+#include "img_defs.h"
+
+/* This workaround is only *required* on ARM64. Avoid building or including
+ * it by default on other architectures, unless the 'safe memcpy' test flag
+ * is enabled. (The code should work on other architectures.)
+ */
+
+
+
+/* NOTE: This C file is compiled with -ffreestanding to avoid pattern matching
+ *       by the compiler to stdlib functions, and it must only use the below
+ *       headers. Do not include any IMG or services headers in this file.
+ */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/* The attribute "vector_size" will generate floating point instructions
+ * and use FPU registers. In kernel OS, the FPU registers might be corrupted
+ * when CPU is doing context switch because FPU registers are not expected to
+ * be stored.
+ * GCC enables compiler option, -mgeneral-regs-only, by default.
+ * This option restricts the generated code to use general registers only
+ * so that we don't have issues on that.
+ */
+#if defined(__KERNEL__) && defined(__clang__)
+
+#define DEVICE_MEMSETCPY_NON_VECTOR_KM
+/* Loading or storing 16 or 32 bytes is only supported on 64-bit machines. */
+#if DEVICE_MEMSETCPY_ALIGN_IN_BYTES > 8
+typedef __uint128_t uint128_t;
+
+typedef struct
+{
+	uint128_t ui128DataFields[2];
+}
+uint256_t;
+#endif
+
+#endif
+
+/* This file is only intended to be used on platforms which use GCC or Clang,
+ * due to its requirement on __attribute__((vector_size(n))), typeof() and
+ * __SIZEOF__ macros.
+ */
+
+#if defined(__GNUC__)
+
+#ifndef MIN
+#define MIN(a, b) \
+ ({__typeof(a) _a = (a); __typeof(b) _b = (b); _a > _b ? _b : _a;})
+#endif
+
+#if !defined(DEVICE_MEMSETCPY_ALIGN_IN_BYTES)
+#define DEVICE_MEMSETCPY_ALIGN_IN_BYTES __SIZEOF_LONG__
+#endif
+#if (DEVICE_MEMSETCPY_ALIGN_IN_BYTES & (DEVICE_MEMSETCPY_ALIGN_IN_BYTES - 1)) != 0
+#error "DEVICE_MEMSETCPY_ALIGN_IN_BYTES must be a power of 2"
+#endif
+#if DEVICE_MEMSETCPY_ALIGN_IN_BYTES < 4
+#error "DEVICE_MEMSETCPY_ALIGN_IN_BYTES must be equal or greater than 4"
+#endif
+
+#if __SIZEOF_POINTER__ != __SIZEOF_LONG__
+#error No support for architectures where void* and long are sized differently
+#endif
+
+#if __SIZEOF_LONG__ > DEVICE_MEMSETCPY_ALIGN_IN_BYTES
+/* Meaningless, and harder to do correctly */
+# error Cannot handle DEVICE_MEMSETCPY_ALIGN_IN_BYTES < sizeof(long)
+typedef unsigned long block_t;
+#elif __SIZEOF_LONG__ <= DEVICE_MEMSETCPY_ALIGN_IN_BYTES
+# if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+#  if DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+    typedef uint64_t block_t;
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+    typedef uint128_t block_t;
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+    typedef uint256_t block_t;
+#  endif
+# else
+typedef unsigned int block_t
+	__attribute__((vector_size(DEVICE_MEMSETCPY_ALIGN_IN_BYTES)));
+# endif
+# if defined(__arm64__) || defined(__aarch64__)
+#  if   DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+#   define DEVICE_MEMSETCPY_ARM64
+#   define REGSZ "w"
+#   define REGCL "w"
+#   define BVCLB "r"
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+#   define DEVICE_MEMSETCPY_ARM64
+#   define REGSZ "x"
+#   define REGCL "x"
+#   define BVCLB "r"
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+#   if defined(__ARM_NEON_FP)
+#    define DEVICE_MEMSETCPY_ARM64
+#    define REGSZ "q"
+#    define REGCL "v"
+#    define BVCLB "w"
+#   endif
+#  endif
+#  if defined(DEVICE_MEMSETCPY_ARM64)
+#   if defined(DEVICE_MEMSETCPY_ARM64_NON_TEMPORAL)
+#    define NSHLD() __asm__ ("dmb nshld")
+#    define NSHST() __asm__ ("dmb nshst")
+#    define LDP "ldnp"
+#    define STP "stnp"
+#   else
+#    define NSHLD()
+#    define NSHST()
+#    define LDP "ldp"
+#    define STP "stp"
+#   endif
+#   if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+#    if DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+typedef uint32_t block_half_t;
+#    elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+typedef uint64_t block_half_t;
+#    elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+typedef uint128_t block_half_t;
+#    endif
+#   else
+ typedef unsigned int block_half_t
+	__attribute__((vector_size(DEVICE_MEMSETCPY_ALIGN_IN_BYTES / 2)));
+#   endif
+#  endif
+# endif
+#endif
+
+__attribute__((visibility("hidden")))
+void DeviceMemCopy(void *pvDst, const void *pvSrc, size_t uSize)
+{
+	volatile const char *pcSrc = pvSrc;
+	volatile char *pcDst = pvDst;
+	size_t uPreambleBytes;
+	int bBlockCopy = 0;
+
+	size_t uSrcUnaligned = (size_t)pcSrc % sizeof(block_t);
+	size_t uDstUnaligned = (size_t)pcDst % sizeof(block_t);
+
+	if (!uSrcUnaligned && !uDstUnaligned)
+	{
+		/* Neither pointer is unaligned. Optimal case. */
+		bBlockCopy = 1;
+	}
+	else
+	{
+		if (uSrcUnaligned == uDstUnaligned)
+		{
+			/* Neither pointer is usefully aligned, but they are misaligned in
+			 * the same way, so we can copy a preamble in a slow way, then
+			 * optimize the rest.
+			 */
+			uPreambleBytes = MIN(sizeof(block_t) - uDstUnaligned, uSize);
+			uSize -= uPreambleBytes;
+			while (uPreambleBytes)
+			{
+				*pcDst++ = *pcSrc++;
+				uPreambleBytes--;
+			}
+
+			bBlockCopy = 1;
+		}
+		else if ((uSrcUnaligned | uDstUnaligned) % sizeof(int) == 0)
+		{
+			/* Both pointers are at least 32-bit aligned, and we assume that
+			 * the processor must handle all kinds of 32-bit load-stores.
+			 * NOTE: Could we optimize this with a non-temporal version?
+			 */
+			if (uSize >= sizeof(int))
+			{
+				volatile int *piSrc = (int *)((void *)pcSrc);
+				volatile int *piDst = (int *)((void *)pcDst);
+
+				while (uSize >= sizeof(int))
+				{
+					*piDst++ = *piSrc++;
+					uSize -= sizeof(int);
+				}
+
+				pcSrc = (char *)((void *)piSrc);
+				pcDst = (char *)((void *)piDst);
+			}
+		}
+	}
+
+	if (bBlockCopy && uSize >= sizeof(block_t))
+	{
+		volatile block_t *pSrc = (block_t *)((void *)pcSrc);
+		volatile block_t *pDst = (block_t *)((void *)pcDst);
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHLD();
+#endif
+
+		while (uSize >= sizeof(block_t))
+		{
+#if defined(DEVICE_MEMSETCPY_ARM64)
+			__asm__ (LDP " " REGSZ "0, " REGSZ "1, [%[pSrc]]\n\t"
+			         STP " " REGSZ "0, " REGSZ "1, [%[pDst]]"
+						:
+						: [pSrc] "r" (pSrc), [pDst] "r" (pDst)
+						: "memory", REGCL "0", REGCL "1");
+#else
+			*pDst = *pSrc;
+#endif
+			pDst++;
+			pSrc++;
+			uSize -= sizeof(block_t);
+		}
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHST();
+#endif
+
+		pcSrc = (char *)((void *)pSrc);
+		pcDst = (char *)((void *)pDst);
+	}
+
+	while (uSize)
+	{
+		*pcDst++ = *pcSrc++;
+		uSize--;
+	}
+}
+
+__attribute__((visibility("hidden")))
+void DeviceMemSet(void *pvDst, unsigned char ui8Value, size_t uSize)
+{
+	volatile char *pcDst = pvDst;
+	size_t uPreambleBytes;
+
+	size_t uDstUnaligned = (size_t)pcDst % sizeof(block_t);
+
+	if (uDstUnaligned)
+	{
+		uPreambleBytes = MIN(sizeof(block_t) - uDstUnaligned, uSize);
+		uSize -= uPreambleBytes;
+		while (uPreambleBytes)
+		{
+			*pcDst++ = ui8Value;
+			uPreambleBytes--;
+		}
+	}
+
+	if (uSize >= sizeof(block_t))
+	{
+		volatile block_t *pDst = (block_t *)((void *)pcDst);
+		size_t i, uBlockSize;
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		typedef block_half_t BLK_t;
+#else
+		typedef block_t BLK_t;
+#endif /* defined(DEVICE_MEMSETCPY_ARM64) */
+
+#if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+		BLK_t bValue = 0;
+
+		uBlockSize = sizeof(BLK_t) / sizeof(ui8Value);
+
+		for (i = 0; i < uBlockSize; i++)
+		{
+			bValue |= (BLK_t)ui8Value << ((uBlockSize - i - 1) * BITS_PER_BYTE);
+		}
+#else
+		BLK_t bValue = {0};
+
+		uBlockSize = sizeof(bValue) / sizeof(unsigned int);
+		for (i = 0; i < uBlockSize; i++)
+			bValue[i] = ui8Value << 24U |
+			            ui8Value << 16U |
+			            ui8Value <<  8U |
+			            ui8Value;
+#endif /* defined(DEVICE_MEMSETCPY_NON_VECTOR_KM) */
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHLD();
+#endif
+
+		while (uSize >= sizeof(block_t))
+		{
+#if defined(DEVICE_MEMSETCPY_ARM64)
+			__asm__ (STP " %" REGSZ "[bValue], %" REGSZ "[bValue], [%[pDst]]"
+						:
+						: [bValue] BVCLB (bValue), [pDst] "r" (pDst)
+						: "memory");
+#else
+			*pDst = bValue;
+#endif
+			pDst++;
+			uSize -= sizeof(block_t);
+		}
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHST();
+#endif
+
+		pcDst = (char *)((void *)pDst);
+	}
+
+	while (uSize)
+	{
+		*pcDst++ = ui8Value;
+		uSize--;
+	}
+}
+
+#endif /* defined(__GNUC__) */
+
+/* Potentially very slow (but safe) fallbacks for non-GNU C compilers */
+IMG_INTERNAL
+void DeviceMemCopyBytes(void *pvDst, const void *pvSrc, size_t uSize)
+{
+	volatile const char *pcSrc = pvSrc;
+	volatile char *pcDst = pvDst;
+
+	while (uSize)
+	{
+		*pcDst++ = *pcSrc++;
+		uSize--;
+	}
+}
+
+IMG_INTERNAL
+void DeviceMemSetBytes(void *pvDst, unsigned char ui8Value, size_t uSize)
+{
+	volatile char *pcDst = pvDst;
+
+	while (uSize)
+	{
+		*pcDst++ = ui8Value;
+		uSize--;
+	}
+}
+
+#if defined(__linux__) && defined(__KERNEL__)
+#if defined(DEBUG)
+/*
+ * In case of Linux kernel-mode in a debug build, choose the variant of
+ * OSStringSafeCopy that uses strscpy and logs truncation via a stack dump. For
+ * Linux kernel-mode in a release build, strscpy alone is used.
+ */
+IMG_INTERNAL
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	/*
+	 * Let strscpy handle any truncation cases correctly.
+	 * We will definitely get a NUL-terminated string set in pszDest
+	 */
+	ssize_t sCopiedCnt = strscpy(pszDest, pszSrc, uDataSize);
+
+#if defined(PVR_DEBUG_STRLCPY)
+	/* Handle truncation by dumping calling stack if debug allows */
+	if (sCopiedCnt < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+			"%s: String truncated Src = '<%s>' %zu bytes, Dest = '%s'",
+			__func__, pszSrc, uDataSize, pszDest));
+		OSDumpStack();
+	}
+#endif /* defined(PVR_DEBUG_STRLCPY) && defined(DEBUG) */
+
+	return sCopiedCnt;
+}
+#endif /* defined(DEBUG) */
+#else /* defined(__linux__) && defined(__KERNEL__) */
+/*
+ * For every other platform, make use of the strnlen and strncpy implementation
+ * of OSStringSafeCopy.
+ *
+ * NOTE: It is crucial to avoid memcpy as this has a hidden side-effect of
+ * dragging in whatever the build-environment flavour of GLIBC is which can
+ * cause unexpected failures for host-side command execution.
+ */
+IMG_INTERNAL
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	/* Match the specification of Linux strscpy - if destination size is 0,
+	 * return negative value (-E2BIG to be precise, but it's probably best
+	 * not to use Linux-specific return values, so -1 will do). */
+	if (uDataSize && (uDataSize <= SSIZE_MAX))
+	{
+		size_t uSrcSize = strnlen(pszSrc, uDataSize);
+
+		(void)strncpy(pszDest, pszSrc, MIN(uSrcSize + 1, uDataSize));
+		if (uSrcSize == uDataSize)
+		{
+			pszDest[uSrcSize-1] = '\0';
+			return -1;
+		}
+
+		return (ssize_t)uSrcSize;
+	}
+
+	return -1;
+}
+#endif /* defined(__linux__) && defined(__KERNEL__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/pvrsrv_error.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/pvrsrv_error.c
new file mode 100644
index 000000000000..5cd02a28d5a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/pvrsrv_error.c
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services error support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+IMG_EXPORT
+const IMG_CHAR *PVRSRVGetErrorString(PVRSRV_ERROR eError)
+{
+	switch (eError)
+	{
+		case PVRSRV_OK:
+			return "PVRSRV_OK";
+#define PVRE(x) \
+		case x: \
+			return #x;
+#include "pvrsrv_errors.h"
+#undef PVRE
+		default:
+			return "Unknown PVRSRV error number";
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/ra.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/ra.c
new file mode 100644
index 000000000000..d1646b309f6d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/ra.c
@@ -0,0 +1,4040 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Description
+ Implements generic resource allocation. The resource allocator was originally
+ intended to manage address spaces. In practice the resource allocator is
+ generic and can manage arbitrary sets of integers.
+
+ Resources are allocated from arenas. Arenas can be created with an initial
+ span of resources. Further resources spans can be added to arenas. A
+ callback mechanism allows an arena to request further resource spans on
+ demand.
+
+ Each arena maintains an ordered list of resource segments each described by a
+ boundary tag. Each boundary tag describes a segment of resources which are
+ either 'free', available for allocation, or 'busy' currently allocated.
+ Adjacent 'free' segments are always coalesced to avoid fragmentation.
+
+ For allocation, all 'free' segments are kept on lists of 'free' segments in
+ a table index by pvr_log2(segment size) i.e., each table index n holds 'free'
+ segments in the size range 2^n -> 2^(n+1) - 1.
+
+ Allocation policy is based on an *almost* good fit strategy.
+
+ Allocated segments are inserted into a self-scaling hash table which maps
+ the base resource of the span to the relevant boundary tag. This allows the
+ code to get back to the boundary tag without exporting explicit boundary tag
+ references through the API.
+
+ Each arena has an associated quantum size, all allocations from the arena are
+ made in multiples of the basic quantum.
+
+ On resource exhaustion in an arena, a callback if provided will be used to
+ request further resources. Resource spans allocated by the callback mechanism
+ will be returned when freed (through one of the two callbacks).
+*/ /**************************************************************************/
+
+/* Issues:
+ * - flags, flags are passed into the resource allocator but are not currently used.
+ * - determination, of import size, is currently braindead.
+ * - debug code should be moved out to own module and #ifdef'd
+ */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "uniq_key_splay_tree.h"
+
+#include "hash.h"
+#include "ra.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "osfunc.h"
+#include "allocmem.h"
+#include "lock.h"
+#include "pvr_intrinsics.h"
+
+/* The initial, and minimum size of the live address -> boundary tag structure
+ * hash table. The value 64 is a fairly arbitrary choice. The hash table
+ * resizes on demand so the value chosen is not critical.
+ */
+#define MINIMUM_HASH_SIZE (64)
+
+/* #define RA_VALIDATE */
+
+#if defined(__KLOCWORK__)
+	/* Make sure Klocwork analyses all the code (including the debug one) */
+	#if !defined(RA_VALIDATE)
+		#define RA_VALIDATE
+	#endif
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_ASSERT) || !defined(RA_VALIDATE)
+/* Disable the asserts unless explicitly told otherwise.
+ * They slow the driver too much for other people
+ */
+
+#undef PVR_ASSERT
+/* Use a macro that really do not do anything when compiling in release
+ * mode!
+ */
+#define PVR_ASSERT(x)
+#endif
+
+/* boundary tags, used to describe a resource segment */
+struct _BT_
+{
+	enum bt_type
+	{
+		btt_free,		/* free resource segment */
+		btt_live		/* allocated resource segment */
+	} type;
+
+	unsigned int is_leftmost;
+	unsigned int is_rightmost;
+	unsigned int free_import;
+
+	/* The base resource and extent of this segment */
+	RA_BASE_T base;
+	RA_LENGTH_T uSize;
+
+	/* doubly linked ordered list of all segments within the arena */
+	struct _BT_ *pNextSegment;
+	struct _BT_ *pPrevSegment;
+
+	/* doubly linked un-ordered list of free segments with the same flags. */
+	struct _BT_ *next_free;
+	struct _BT_ *prev_free;
+
+	/* A user reference associated with this span, user references are
+	 * currently only provided in the callback mechanism
+	 */
+	IMG_HANDLE hPriv;
+
+	/* Flags to match on this span */
+	RA_FLAGS_T uFlags;
+
+};
+typedef struct _BT_ BT;
+
+
+/* resource allocation arena */
+struct _RA_ARENA_
+{
+	/* arena name for diagnostics output */
+	IMG_CHAR name[RA_MAX_NAME_LENGTH];
+
+	/* Spans / Imports within this arena are at least quantum sized
+	 * and are a multiple of the uQuantum. This also has the effect of
+	 * aligning these Spans to the uQuantum.
+	 */
+	RA_LENGTH_T uQuantum;
+
+	/* import interface, if provided */
+	union {
+		void* pImportAlloc;
+		PFN_RA_IMPORT_ALLOC_SINGLE pImportAllocSingle;
+		PFN_RA_IMPORT_ALLOC_MULTI pImportAllocMulti;
+	};
+	enum RA_IMPORT_ALLOC_METHOD {
+		RA_IMPORT_ALLOC_METHOD_SINGLE,
+		RA_IMPORT_ALLOC_METHOD_MULTI,
+	} eImportAllocMethod;
+
+	PFN_RA_IMPORT_FREE pImportFree;
+
+	/* Arbitrary handle provided by arena owner to be passed into the
+	 * import alloc and free hooks
+	 */
+	void *pImportHandle;
+
+	IMG_PSPLAY_TREE per_flags_buckets;
+
+	/* resource segment list */
+	BT *pHeadSegment;
+
+	/* segment address to boundary tag hash table */
+	HASH_TABLE *pSegmentHash;
+
+	/* Lock for this arena */
+	POS_LOCK hLock;
+
+	/* Policies that govern the resource area */
+	RA_POLICY_T ui32PolicyFlags;
+
+	/* LockClass of this arena. This is used within lockdep to decide if a
+	 * recursive call sequence with the same lock class is allowed or not.
+	 */
+	IMG_UINT32 ui32LockClass;
+
+	/* Total Size of the Arena */
+	IMG_UINT64	ui64TotalArenaSize;
+
+	/* Size available for allocation in the arena */
+	IMG_UINT64	ui64FreeArenaSize;
+
+};
+
+static_assert(sizeof(void*) == sizeof(PFN_RA_IMPORT_ALLOC_SINGLE), "pImportAlloc == pImportAllocSingle. Import callbacks must be pointers of the same size.");
+static_assert(sizeof(void*) == sizeof(PFN_RA_IMPORT_ALLOC_MULTI), "pImportAlloc == pImportAllocMulti. Import callbacks must be pointers of the same size.");
+
+struct _RA_ARENA_ITERATOR_
+{
+	RA_ARENA *pArena;
+	BT *pCurrent;
+	IMG_BOOL bIncludeFreeSegments;
+};
+
+/* Enum for selecting behaviour of _ConvertAndFree* functions*/
+typedef enum {
+	CONVERT_AND_FREE = 0,
+	CONVERT_DONT_FREE = 1
+} RA_CONVERT_AND_FREE_BEHAVIOUR;
+
+static PVRSRV_ERROR _RA_FreeMultiUnlocked(RA_ARENA *pArena,
+                                   RA_BASE_ARRAY_T aBaseArray,
+                                   RA_BASE_ARRAY_SIZE_T uiBaseArraySize);
+static PVRSRV_ERROR
+_RA_FreeMultiUnlockedSparse(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 *puiFreeIndices,
+                             IMG_UINT32 *puiFreeCount);
+
+/*************************************************************************/ /*!
+@Function       _RequestAllocFail
+@Description    Default callback allocator used if no callback is specified,
+                always fails to allocate further resources to the arena.
+@Input          _h - callback handle
+@Input          _uSize - requested allocation size
+@Input          _uflags - allocation flags
+@Input          _uBaseAlignment - Alignment for the returned allocated base
+
+
+@Output         _psImport - The base and size of the span to insert.
+@Return         PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL, this function always fails
+                to allocate.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_RequestAllocFail(RA_PERARENA_HANDLE _h,
+                  RA_LENGTH_T _uSize,
+                  RA_FLAGS_T _uFlags,
+                  RA_LENGTH_T _uBaseAlignment,
+                  const IMG_CHAR *_pszAnnotation,
+                  RA_IMPORT *_psImport)
+{
+	PVR_UNREFERENCED_PARAMETER(_h);
+	PVR_UNREFERENCED_PARAMETER(_uSize);
+	PVR_UNREFERENCED_PARAMETER(_uFlags);
+	PVR_UNREFERENCED_PARAMETER(_uBaseAlignment);
+	PVR_UNREFERENCED_PARAMETER(_pszAnnotation);
+	PVR_UNREFERENCED_PARAMETER(_psImport);
+
+	return PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL;
+}
+
+
+#if defined(PVR_CTZLL)
+	/* Make sure to trigger an error if someone change the buckets or the bHasEltsMapping size
+	   the bHasEltsMapping is used to quickly determine the smallest bucket containing elements.
+	   therefore it must have at least as many bits has the buckets array have buckets. The RA
+	   implementation actually uses one more bit. */
+	static_assert(ARRAY_SIZE(((IMG_PSPLAY_TREE)0)->buckets)
+				  < 8 * sizeof(((IMG_PSPLAY_TREE) 0)->bHasEltsMapping),
+				  "Too many buckets for bHasEltsMapping bitmap");
+#endif
+
+
+/*************************************************************************/ /*!
+@Function       pvr_log2
+@Description    Computes the floor of the log base 2 of a unsigned integer
+@Input          n       Unsigned integer
+@Return         Floor(Log2(n))
+*/ /**************************************************************************/
+#if defined(PVR_CLZLL)
+/* make sure to trigger a problem if someone changes the RA_LENGTH_T type
+   indeed the __builtin_clzll is for unsigned long long variables.
+
+   if someone changes RA_LENGTH to unsigned long, then use __builtin_clzl
+   if it changes to unsigned int, use __builtin_clz
+
+   if it changes for something bigger than unsigned long long,
+   then revert the pvr_log2 to the classic implementation */
+static_assert(sizeof(RA_LENGTH_T) == sizeof(unsigned long long),
+			  "RA log routines not tuned for sizeof(RA_LENGTH_T)");
+
+static inline IMG_UINT32 pvr_log2(RA_LENGTH_T n)
+{
+	PVR_ASSERT(n != 0); /* Log2 is not defined on 0 */
+
+	return (8 * sizeof(RA_LENGTH_T)) - 1 - PVR_CLZLL(n);
+}
+#else
+static IMG_UINT32
+pvr_log2(RA_LENGTH_T n)
+{
+	IMG_UINT32 l = 0;
+
+	PVR_ASSERT(n != 0); /* Log2 is not defined on 0 */
+
+	n >>= 1;
+	while (n > 0)
+	{
+		n >>= 1;
+		l++;
+	}
+	return l;
+}
+#endif
+
+static INLINE void _FreeTableLimitBoundsCheck(IMG_UINT32 *uiIndex)
+{
+	if (*uiIndex >= FREE_TABLE_LIMIT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Index exceeds FREE_TABLE_LIMIT (1TB), "
+		                        "Clamping Index to FREE_TABLE_LIMIT"));
+		*uiIndex = FREE_TABLE_LIMIT - 1;
+	}
+}
+
+
+#if defined(RA_VALIDATE)
+/*************************************************************************/ /*!
+@Function       _IsInSegmentList
+@Description    Tests if a BT is in the segment list.
+@Input          pArena           The arena.
+@Input          pBT              The boundary tag to look for.
+@Return         IMG_FALSE  BT was not in the arena's segment list.
+                IMG_TRUE   BT was in the arena's segment list.
+*/ /**************************************************************************/
+static IMG_BOOL
+_IsInSegmentList(RA_ARENA *pArena, BT *pBT)
+{
+	BT* pBTScan;
+
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(pBT != NULL);
+
+	/* Walk the segment list until we see the BT pointer... */
+	pBTScan = pArena->pHeadSegment;
+	while (pBTScan != NULL  &&  pBTScan != pBT)
+	{
+		pBTScan = pBTScan->pNextSegment;
+	}
+
+	/* Test if we found it and then return */
+	return (pBTScan == pBT);
+}
+
+/*************************************************************************/ /*!
+@Function       _IsInFreeList
+@Description    Tests if a BT is in the free list.
+@Input          pArena           The arena.
+@Input          pBT              The boundary tag to look for.
+@Return         IMG_FALSE  BT was not in the arena's free list.
+                IMG_TRUE   BT was in the arena's free list.
+*/ /**************************************************************************/
+static IMG_BOOL
+_IsInFreeList(RA_ARENA *pArena, BT *pBT)
+{
+	BT* pBTScan;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(pBT != NULL);
+
+	/* Look for the free list that holds BTs of this size... */
+	uIndex = pvr_log2(pBT->uSize);
+	PVR_ASSERT(uIndex < FREE_TABLE_LIMIT);
+
+	pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != pBT->uFlags))
+	{
+		return 0;
+	}
+	else
+	{
+		pBTScan = pArena->per_flags_buckets->buckets[uIndex];
+		while (pBTScan != NULL  &&  pBTScan != pBT)
+		{
+			pBTScan = pBTScan->next_free;
+		}
+
+		/* Test if we found it and then return */
+		return (pBTScan == pBT);
+	}
+}
+
+/* is_arena_valid should only be used in debug mode.
+ * It checks that some properties an arena must have are verified
+ */
+static int is_arena_valid(struct _RA_ARENA_ *arena)
+{
+	struct _BT_ *chunk;
+#if defined(PVR_CTZLL)
+	unsigned int i;
+#endif
+
+	for (chunk = arena->pHeadSegment; chunk != NULL; chunk = chunk->pNextSegment)
+	{
+		/* if next segment is NULL, then it must be a rightmost */
+		PVR_ASSERT((chunk->pNextSegment != NULL) || (chunk->is_rightmost));
+		/* if prev segment is NULL, then it must be a leftmost */
+		PVR_ASSERT((chunk->pPrevSegment != NULL) || (chunk->is_leftmost));
+
+		if (chunk->type == btt_free)
+		{
+			/* checks the correctness of the type field */
+			PVR_ASSERT(_IsInFreeList(arena, chunk));
+
+			/* check that there can't be two consecutive free chunks.
+			   Indeed, instead of having two consecutive free chunks,
+			   there should be only one that span the size of the two. */
+			PVR_ASSERT((chunk->is_leftmost) || (chunk->pPrevSegment->type != btt_free));
+			PVR_ASSERT((chunk->is_rightmost) || (chunk->pNextSegment->type != btt_free));
+		}
+		else
+		{
+			/* checks the correctness of the type field */
+			PVR_ASSERT(!_IsInFreeList(arena, chunk));
+		}
+
+		PVR_ASSERT((chunk->is_leftmost) || (chunk->pPrevSegment->base + chunk->pPrevSegment->uSize == chunk->base));
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->base + chunk->uSize == chunk->pNextSegment->base));
+
+		/* all segments of the same imports must have the same flags ... */
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->uFlags == chunk->pNextSegment->uFlags));
+		/* ... and the same import handle */
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->hPriv == chunk->pNextSegment->hPriv));
+
+
+		/* if a free chunk spans a whole import, then it must be an 'not to free import'.
+		   Otherwise it should have been freed. */
+		PVR_ASSERT((!chunk->is_leftmost) || (!chunk->is_rightmost) || (chunk->type == btt_live) || (!chunk->free_import));
+	}
+
+#if defined(PVR_CTZLL)
+	if (arena->per_flags_buckets != NULL)
+	{
+		for (i = 0; i < FREE_TABLE_LIMIT; ++i)
+		{
+			/* verify that the bHasEltsMapping is correct for this flags bucket */
+			PVR_ASSERT(
+				((arena->per_flags_buckets->buckets[i] == NULL) &&
+				 (((arena->per_flags_buckets->bHasEltsMapping & ((IMG_ELTS_MAPPINGS) 1 << i)) == 0)))
+				||
+				((arena->per_flags_buckets->buckets[i] != NULL) &&
+				 (((arena->per_flags_buckets->bHasEltsMapping & ((IMG_ELTS_MAPPINGS) 1 << i)) != 0)))
+				);
+		}
+	}
+#endif
+
+	/* if arena was not valid, an earlier assert should have triggered */
+	return 1;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _SegmentListInsertAfter
+@Description    Insert a boundary tag into an arena segment list after a
+                specified boundary tag.
+@Input          pInsertionPoint  The insertion point.
+@Input          pBT              The boundary tag to insert.
+*/ /**************************************************************************/
+static INLINE void
+_SegmentListInsertAfter(BT *pInsertionPoint,
+                        BT *pBT)
+{
+	PVR_ASSERT(pBT != NULL);
+	PVR_ASSERT(pInsertionPoint != NULL);
+
+	pBT->pNextSegment = pInsertionPoint->pNextSegment;
+	pBT->pPrevSegment = pInsertionPoint;
+	if (pInsertionPoint->pNextSegment != NULL)
+	{
+		pInsertionPoint->pNextSegment->pPrevSegment = pBT;
+	}
+	pInsertionPoint->pNextSegment = pBT;
+}
+
+/*************************************************************************/ /*!
+@Function       _SegmentListInsert
+@Description    Insert a boundary tag into an arena segment list
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag to insert.
+*/ /**************************************************************************/
+static INLINE void
+_SegmentListInsert(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(!_IsInSegmentList(pArena, pBT));
+
+	/* insert into the segment chain */
+	pBT->pNextSegment = pArena->pHeadSegment;
+	pArena->pHeadSegment = pBT;
+	if (pBT->pNextSegment != NULL)
+	{
+		pBT->pNextSegment->pPrevSegment = pBT;
+	}
+
+	pBT->pPrevSegment = NULL;
+}
+
+/*************************************************************************/ /*!
+@Function       _SegmentListRemove
+@Description    Remove a boundary tag from an arena segment list.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag to remove.
+*/ /**************************************************************************/
+static void
+_SegmentListRemove(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(_IsInSegmentList(pArena, pBT));
+
+	if (pBT->pPrevSegment == NULL)
+		pArena->pHeadSegment = pBT->pNextSegment;
+	else
+		pBT->pPrevSegment->pNextSegment = pBT->pNextSegment;
+
+	if (pBT->pNextSegment != NULL)
+		pBT->pNextSegment->pPrevSegment = pBT->pPrevSegment;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _BuildBT
+@Description    Construct a boundary tag for a free segment.
+@Input          base     The base of the resource segment.
+@Input          uSize    The extent of the resource segment.
+@Input          uFlags   The flags to give to the boundary tag
+@Return         Boundary tag or NULL
+*/ /**************************************************************************/
+static BT *
+_BuildBT(RA_BASE_T base, RA_LENGTH_T uSize, RA_FLAGS_T uFlags)
+{
+	BT *pBT;
+
+	pBT = OSAllocZMem(sizeof(BT));
+	if (pBT == NULL)
+	{
+		return NULL;
+	}
+
+	pBT->is_leftmost = 1;
+	pBT->is_rightmost = 1;
+	/* pBT->free_import = 0; */
+	pBT->type = btt_live;
+	pBT->base = base;
+	pBT->uSize = uSize;
+	pBT->uFlags = uFlags;
+
+	return pBT;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _SegmentSplit
+@Description    Split a segment into two, maintain the arena segment list. The
+                boundary tag should not be in the free table. Neither the
+                original or the new neighbour boundary tag will be in the free
+                table.
+@Input          pBT       The boundary tag to split.
+@Input          uSize     The required segment size of boundary tag after
+                          splitting.
+@Return         New neighbour boundary tag or NULL.
+*/ /**************************************************************************/
+static BT *
+_SegmentSplit(BT *pBT, RA_LENGTH_T uSize)
+{
+	BT *pNeighbour;
+
+	pNeighbour = _BuildBT(pBT->base + uSize, pBT->uSize - uSize, pBT->uFlags);
+	if (pNeighbour == NULL)
+	{
+		return NULL;
+	}
+
+	_SegmentListInsertAfter(pBT, pNeighbour);
+
+	pNeighbour->is_leftmost = 0;
+	pNeighbour->is_rightmost = pBT->is_rightmost;
+	pNeighbour->free_import = pBT->free_import;
+	pBT->is_rightmost = 0;
+	pNeighbour->hPriv = pBT->hPriv;
+	pBT->uSize = uSize;
+	pNeighbour->uFlags = pBT->uFlags;
+
+	return pNeighbour;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeListInsert
+@Description    Insert a boundary tag into an arena free table.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag.
+*/ /**************************************************************************/
+static void
+_FreeListInsert(RA_ARENA *pArena, BT *pBT)
+{
+	IMG_UINT32 uIndex;
+	BT *pBTTemp = NULL;
+	uIndex = pvr_log2(pBT->uSize);
+
+	_FreeTableLimitBoundsCheck(&uIndex);
+
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+
+	pBT->type = btt_free;
+
+	pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+	/* the flags item in the splay tree must have been created before-hand by
+	   _InsertResource */
+	PVR_ASSERT(pArena->per_flags_buckets != NULL);
+
+	/* Handle NULL values for RELEASE builds and/or disabled ASSERT DEBUG builds */
+	if (unlikely(pArena->per_flags_buckets == NULL))
+	{
+		return;
+	}
+
+	/* Get the first node in the bucket */
+	pBTTemp = pArena->per_flags_buckets->buckets[uIndex];
+
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_ALLOC_NODE_SELECT_MASK) == RA_POLICY_ALLOC_OPTIMAL))
+	{
+		/* Add the node to the start if the bucket is empty */
+		if (NULL == pBTTemp)
+		{
+			pArena->per_flags_buckets->buckets[uIndex] = pBT;
+			pBT->next_free = NULL;
+			pBT->prev_free = NULL;
+
+		}
+		else
+		{
+			BT *pBTPrev = NULL;
+			/* Traverse the list and identify the appropriate
+			 * place based on the size of the Boundary being inserted */
+			while (pBTTemp && (pBTTemp->uSize < pBT->uSize))
+			{
+				pBTPrev = pBTTemp;
+				pBTTemp = pBTTemp->next_free;
+			}
+			/* point the new node to the first higher size element */
+			pBT->next_free = pBTTemp;
+			pBT->prev_free = pBTPrev;
+
+			if (pBTPrev)
+			{
+				/* Set the lower size element in the
+				 * chain to point new node */
+				pBTPrev->next_free = pBT;
+			}
+			else
+			{
+				/* Assign the new node to the start of the bucket
+				 * if the bucket is empty */
+				pArena->per_flags_buckets->buckets[uIndex] = pBT;
+			}
+			/* Make sure the higher size element in the chain points back
+			 * to the new node to be introduced */
+			if (pBTTemp)
+			{
+				pBTTemp->prev_free = pBT;
+			}
+		}
+	}
+	else
+	{
+		pBT->next_free =  pBTTemp;
+		if (pBT->next_free != NULL)
+		{
+			pBT->next_free->prev_free = pBT;
+		}
+		pBT->prev_free = NULL;
+		pArena->per_flags_buckets->buckets[uIndex] = pBT;
+	}
+
+#if defined(PVR_CTZLL)
+	/* tells that bucket[index] now contains elements */
+	pArena->per_flags_buckets->bHasEltsMapping |= ((IMG_ELTS_MAPPINGS) 1 << uIndex);
+#endif
+
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeListRemove
+@Description    Remove a boundary tag from an arena free table.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag.
+*/ /**************************************************************************/
+static void
+_FreeListRemove(RA_ARENA *pArena, BT *pBT)
+{
+	IMG_UINT32 uIndex;
+	uIndex = pvr_log2(pBT->uSize);
+
+	_FreeTableLimitBoundsCheck(&uIndex);
+
+	PVR_ASSERT(_IsInFreeList(pArena, pBT));
+
+	if (pBT->next_free != NULL)
+	{
+		pBT->next_free->prev_free = pBT->prev_free;
+	}
+
+	if (pBT->prev_free != NULL)
+	{
+		pBT->prev_free->next_free = pBT->next_free;
+	}
+	else
+	{
+		pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+		/* the flags item in the splay tree must have already been created
+		   (otherwise how could there be a segment with these flags */
+		PVR_ASSERT(pArena->per_flags_buckets != NULL);
+
+		/* Handle unlikely NULL values for RELEASE or ASSERT-disabled builds */
+		if (unlikely(pArena->per_flags_buckets == NULL))
+		{
+			pBT->type = btt_live;
+			return;
+		}
+
+		pArena->per_flags_buckets->buckets[uIndex] = pBT->next_free;
+#if defined(PVR_CTZLL)
+		if (pArena->per_flags_buckets->buckets[uIndex] == NULL)
+		{
+			/* there is no more elements in this bucket. Update the mapping. */
+			pArena->per_flags_buckets->bHasEltsMapping &= ~((IMG_ELTS_MAPPINGS) 1 << uIndex);
+		}
+#endif
+	}
+
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+	pBT->type = btt_live;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _InsertResource
+@Description    Add a free resource segment to an arena.
+@Input          pArena    The arena.
+@Input          base      The base of the resource segment.
+@Input          uSize     The extent of the resource segment.
+@Input          uFlags    The flags of the new resources.
+@Return         New bucket pointer
+                NULL on failure
+*/ /**************************************************************************/
+static BT *
+_InsertResource(RA_ARENA *pArena, RA_BASE_T base, RA_LENGTH_T uSize,
+                RA_FLAGS_T uFlags)
+{
+	BT *pBT;
+	PVR_ASSERT(pArena!=NULL);
+
+	pBT = _BuildBT(base, uSize, uFlags);
+
+	if (pBT != NULL)
+	{
+		IMG_PSPLAY_TREE tmp = PVRSRVInsert(pBT->uFlags, pArena->per_flags_buckets);
+		if (tmp == NULL)
+		{
+			OSFreeMem(pBT);
+			return NULL;
+		}
+
+		pArena->per_flags_buckets = tmp;
+		_SegmentListInsert(pArena, pBT);
+		_FreeListInsert(pArena, pBT);
+	}
+	return pBT;
+}
+
+/*************************************************************************/ /*!
+@Function       _InsertResourceSpan
+@Description    Add a free resource span to an arena, marked for free_import.
+@Input          pArena    The arena.
+@Input          base      The base of the resource segment.
+@Input          uSize     The extent of the resource segment.
+@Return         The boundary tag representing the free resource segment,
+                or NULL on failure.
+*/ /**************************************************************************/
+static INLINE BT *
+_InsertResourceSpan(RA_ARENA *pArena,
+                    RA_BASE_T base,
+                    RA_LENGTH_T uSize,
+                    RA_FLAGS_T uFlags)
+{
+	BT *pBT = _InsertResource(pArena, base, uSize, uFlags);
+	if (pBT != NULL)
+	{
+		pBT->free_import = 1;
+	}
+	return pBT;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _RemoveResourceSpan
+@Description    Frees a resource span from an arena, returning the imported
+                span via the callback.
+@Input          pArena     The arena.
+@Input          pBT        The boundary tag to free.
+@Return         IMG_FALSE failure - span was still in use
+                IMG_TRUE  success - span was removed and returned
+*/ /**************************************************************************/
+static INLINE IMG_BOOL
+_RemoveResourceSpan(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(pBT!=NULL);
+
+	if (pBT->free_import &&
+		pBT->is_leftmost &&
+		pBT->is_rightmost)
+	{
+		_SegmentListRemove(pArena, pBT);
+		pArena->ui64TotalArenaSize -= pBT->uSize;
+		pArena->ui64FreeArenaSize -= pBT->uSize;
+		pArena->pImportFree(pArena->pImportHandle, pBT->base, pBT->hPriv);
+		OSFreeMem(pBT);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeBT
+@Description    Free a boundary tag taking care of the segment list and the
+                boundary tag free table.
+@Input          pArena     The arena.
+@Input          pBT        The boundary tag to free.
+*/ /**************************************************************************/
+static void
+_FreeBT(RA_ARENA *pArena, BT *pBT)
+{
+	BT *pNeighbour;
+
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(pBT!=NULL);
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+
+	/* try and coalesce with left neighbour */
+	pNeighbour = pBT->pPrevSegment;
+	if ((!pBT->is_leftmost)	&& (pNeighbour->type == btt_free))
+	{
+		/* Verify list correctness */
+		PVR_ASSERT(pNeighbour->base + pNeighbour->uSize == pBT->base);
+
+		_FreeListRemove(pArena, pNeighbour);
+		_SegmentListRemove(pArena, pNeighbour);
+		pBT->base = pNeighbour->base;
+
+		pBT->uSize += pNeighbour->uSize;
+		pBT->is_leftmost = pNeighbour->is_leftmost;
+		OSFreeMem(pNeighbour);
+	}
+
+	/* try to coalesce with right neighbour */
+	pNeighbour = pBT->pNextSegment;
+	if ((!pBT->is_rightmost) && (pNeighbour->type == btt_free))
+	{
+		/* Verify list correctness */
+		PVR_ASSERT(pBT->base + pBT->uSize == pNeighbour->base);
+
+		_FreeListRemove(pArena, pNeighbour);
+		_SegmentListRemove(pArena, pNeighbour);
+		pBT->uSize += pNeighbour->uSize;
+		pBT->is_rightmost = pNeighbour->is_rightmost;
+		OSFreeMem(pNeighbour);
+	}
+
+	if (_RemoveResourceSpan(pArena, pBT) == IMG_FALSE)
+	{
+		_FreeListInsert(pArena, pBT);
+		PVR_ASSERT((!pBT->is_rightmost) || (!pBT->is_leftmost) || (!pBT->free_import));
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+}
+
+
+/*
+  This function returns the first element in a bucket that can be split
+  in a way that one of the sub-segments can meet the size and alignment
+  criteria.
+
+  The first_elt is the bucket to look into. Remember that a bucket is
+  implemented as a pointer to the first element of the linked list.
+
+  nb_max_try is used to limit the number of elements considered.
+  This is used to only consider the first nb_max_try elements in the
+  free-list. The special value ~0 is used to say unlimited i.e. consider
+  all elements in the free list
+ */
+static INLINE
+struct _BT_ *find_chunk_in_bucket(struct _BT_ * first_elt,
+                                  RA_LENGTH_T uSize,
+                                  RA_LENGTH_T uAlignment,
+                                  unsigned int nb_max_try)
+{
+	struct _BT_ *walker;
+
+	for (walker = first_elt; (walker != NULL) && (nb_max_try != 0); walker = walker->next_free)
+	{
+		const RA_BASE_T aligned_base = (uAlignment > 1) ?
+			PVR_ALIGN(walker->base, uAlignment)
+			: walker->base;
+
+		if (walker->base + walker->uSize >= aligned_base + uSize)
+		{
+			return walker;
+		}
+
+		/* 0xFFFF...FFFF is used has nb_max_try = infinity. */
+		if (nb_max_try != (unsigned int) ~0)
+		{
+			nb_max_try--;
+		}
+	}
+
+	return NULL;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FreeMultiBaseArray
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 free the region given as the array and size. This function
+ *                 should be used only when it is known that multiple Real
+ *                 bases will be freed from the array.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_FreeMultiBaseArray(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	IMG_UINT32 i;
+	for (i = 0; i < uiBaseArraySize; i++)
+	{
+		if (RA_BASE_IS_REAL(aBaseArray[i]))
+		{
+			BT *pBT;
+			pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &aBaseArray[i]);
+
+			if (pBT)
+			{
+				pArena->ui64FreeArenaSize += pBT->uSize;
+
+				PVR_ASSERT(pBT->base == aBaseArray[i]);
+				_FreeBT(pArena, pBT);
+				aBaseArray[i] = INVALID_BASE_ADDR;
+			}
+			else
+			{
+				/* Did we attempt to remove a ghost page?
+				 * Essentially the base was marked real but was actually a ghost.
+				 */
+				PVR_ASSERT(!"Attempt to free non-existing real base!");
+				return PVRSRV_ERROR_INVALID_REQUEST;
+			}
+		}
+#if defined(DEBUG)
+		else
+		{
+			aBaseArray[i] = INVALID_BASE_ADDR;
+		}
+#endif
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FreeSingleBaseArray
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 free the region given as the array and size. This function
+ *                 should be used only when it is known that a single Real
+ *                 base will be freed from the array. All Bases will be
+ *                 sanitised after the real has been freed.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from, entry 0 should be a
+ *                       Real base
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_FreeSingleBaseArray(RA_ARENA *pArena,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	BT *pBT;
+	PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[0]));
+
+	pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &aBaseArray[0]);
+
+	if (pBT)
+	{
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		PVR_ASSERT(pBT->base == aBaseArray[0]);
+		_FreeBT(pArena, pBT);
+	}
+	else
+	{
+		/* Did we attempt to remove a ghost page?
+		 * Essentially the base was marked real but was actually ghost.
+		 */
+		PVR_ASSERT(!"Attempt to free non-existing real base!");
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+
+	/* Set all entries to INVALID_BASE_ADDR */
+	OSCachedMemSet(aBaseArray, 0xFF, uiBaseArraySize * sizeof(RA_BASE_T));
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _GenerateGhostBases
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 generate Ghost bases for the allocation and size.
+ *
+ *  @Input  uiBase - The Real base to generate Ghost Bases from.
+ *  @Input  uiBaseSize - The size of the Real Base
+ *  @Input  uiChunkSize - The Base chunk size used to generate Ghost
+ *                           bases on specific boundaries.
+ *  @Input  aBaseArray - The array to add the Ghost bases to.
+ *
+ *  @Return array index of element past last Ghost base of given array.
+*/ /**************************************************************************/
+static IMG_UINT32
+_GenerateGhostBases(RA_BASE_T uiRealBase,
+                    RA_LENGTH_T uiBaseSize,
+                    RA_LENGTH_T uiChunkSize,
+                    RA_BASE_ARRAY_T aBaseArray)
+{
+	IMG_UINT32 ui32Index = 0;
+	RA_LENGTH_T uiRemaining = uiBaseSize - uiChunkSize;
+	RA_LENGTH_T uiCurrentBase = uiRealBase + uiChunkSize;
+	aBaseArray[ui32Index] = uiRealBase;
+
+	for (ui32Index = 1; uiRemaining != 0; ui32Index++)
+	{
+		aBaseArray[ui32Index] = RA_BASE_SET_GHOST_BIT(uiCurrentBase);
+		uiCurrentBase += uiChunkSize;
+		uiRemaining -= uiChunkSize;
+	}
+
+	return ui32Index;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FindRealBaseFromGhost
+ *
+ *  @Description   Given an array and an index into that array for the Ghost Base
+ *                 find the Real Base hosting the Ghost base in the RA.
+ *  @Input aBaseArray - The array the Ghost and Real base reside on.
+ *  @Input ui32GhostBaseIndex - The index into the given array for the Ghost Base.
+ *  @Output pRealBase - The Real Base hosting the Ghost base.
+ *  @Output pui32RealBaseIndex -  The index of the Real Base found in the array.
+ *
+ *  @Return None.
+*/ /**************************************************************************/
+static void
+_FindRealBaseFromGhost(RA_BASE_ARRAY_T aBaseArray,
+                       IMG_UINT32 ui32GhostBaseIndex,
+                       RA_BASE_T *pRealBase,
+                       IMG_UINT32 *pui32RealBaseIndex)
+{
+	IMG_UINT32 ui32Index = ui32GhostBaseIndex;
+
+	PVR_ASSERT(RA_BASE_IS_GHOST(aBaseArray[ui32GhostBaseIndex]));
+
+	while (ui32Index != 0 &&
+	       RA_BASE_IS_GHOST(aBaseArray[ui32Index]))
+	{
+		ui32Index--;
+	}
+
+	*pRealBase = aBaseArray[ui32Index];
+	*pui32RealBaseIndex = ui32Index;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertGhostBaseToReal
+ *
+ *  @Description   Convert the given Ghost Base to a Real Base in the
+ *                 RA. This is mainly used in free paths so we can be
+ *                 agile with memory regions.
+ *  @Input pArena - The RA Arena to convert the base on.
+ *  @Input aBaseArray - The Base array to convert the base on.
+ *  @Input uiRealBase - The Base hosting the Ghost base to convert.
+ *  @Input ui32RealBaseArrayIndex - The index in the array of the Real Base.
+ *  @Input ui32GhostBaseArrayIndex - The index in the array of the Ghost Base.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost bases on.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertGhostBaseToReal(RA_ARENA *pArena,
+                        RA_BASE_ARRAY_T aBaseArray,
+                        RA_BASE_T uiRealBase,
+                        IMG_UINT32 ui32RealBaseArrayIndex,
+                        IMG_UINT32 ui32GhostBaseArrayIndex,
+                        RA_LENGTH_T uiChunkSize)
+{
+	BT *pOrigRealBT;
+	BT *pNewRealBT;
+
+	pOrigRealBT = (BT *) HASH_Retrieve_Extended(pArena->pSegmentHash, &uiRealBase);
+	pNewRealBT = _SegmentSplit(pOrigRealBT,
+	                           uiChunkSize *
+	                            (ui32GhostBaseArrayIndex - ui32RealBaseArrayIndex));
+	PVR_LOG_RETURN_IF_FALSE(pNewRealBT != NULL,
+	                        "Unable to split BT, no memory available to allocate new BT",
+	                        PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	if (!HASH_Insert_Extended(pArena->pSegmentHash, &pNewRealBT->base, (uintptr_t) pNewRealBT))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_UNABLE_TO_INSERT_HASH_VALUE, "HASH_Insert_Extended");
+	}
+
+	aBaseArray[ui32GhostBaseArrayIndex] = pNewRealBT->base;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeStartFromGhost
+ *
+ *  @Description   Given a ghost base and size, convert and free contiguous
+ *                 ghost bases. This has the effect of shrinking the size of
+ *                 the real base holding the range to free. If ghost pages
+ *                 remain after the free region, a new Real base will be
+ *                 created to host them.
+ *
+ *  @Input pArena - The RA Arena to free the Ghost Bases from.
+ *  @Input aBaseArray - The array to remove bases from
+ *  @Input uiBaseArraySize - The size of the Base array to free from.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost Bases.
+ *  @Input ui32GhostBaseIndex - The index into the array of the initial Ghost base to free
+ *  @Input ui32FreeCount - The number of Ghost bases to free from the Real base.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeStartFromGhost(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                              RA_LENGTH_T uiChunkSize,
+                              IMG_UINT32 ui32GhostBaseIndex,
+                              IMG_UINT32 ui32FreeCount,
+                              RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiRealBase;
+	IMG_UINT32 ui32RealBaseIndex;
+	IMG_UINT32 ui32FreeEndIndex;
+
+	_FindRealBaseFromGhost(aBaseArray,
+	                       ui32GhostBaseIndex,
+	                       &uiRealBase,
+	                       &ui32RealBaseIndex);
+
+	/* Make the first Ghost Base to free, real. */
+	eError = _ConvertGhostBaseToReal(pArena,
+	                                 aBaseArray,
+	                                 uiRealBase,
+	                                 ui32RealBaseIndex,
+	                                 ui32GhostBaseIndex,
+	                                 uiChunkSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+
+	/* Calculate the Base after the last to free. */
+	ui32FreeEndIndex = ui32GhostBaseIndex + ui32FreeCount;
+
+	/*
+	 * If the end of the free region is a Ghost base then we need to
+	 * make it a real base so that we can free the intended middle region.
+	 */
+	if (ui32FreeEndIndex != uiBaseArraySize &&
+	    RA_BASE_IS_GHOST(aBaseArray[ui32FreeEndIndex]))
+	{
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 aBaseArray[ui32GhostBaseIndex],
+		                                 ui32GhostBaseIndex,
+		                                 ui32FreeEndIndex,
+		                                 uiChunkSize);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	if (eBehaviour == CONVERT_AND_FREE)
+	{
+		/* Free the region calculated */
+		eError = _FreeSingleBaseArray(pArena,
+		                              &aBaseArray[ui32GhostBaseIndex],
+		                              ui32FreeCount);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeStartFromReal
+ *
+ *  @Description   Used in the case that we want to keep some indices that are ghost pages
+ *                 but the indices to free start with the real base. In this case we can
+ *                 convert the keep point to a real base, then free the original real base
+ *                 along with all ghost bases prior to the new real.
+ *
+ *  @Input pArena - The RA Arena to free the bases from.
+ *  @Input aBaseArray - The Base array to free from.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost bases.
+ *  @Input uiGhostBaseIndex - The index into the array of the Ghost base to convert.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeStartFromReal(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 uiRealBaseIndex,
+                             IMG_UINT32 uiGhostBaseIndex,
+                             RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiRealBase = aBaseArray[uiRealBaseIndex];
+
+	eError = _ConvertGhostBaseToReal(pArena,
+	                                 aBaseArray,
+	                                 uiRealBase,
+	                                 uiRealBaseIndex,
+	                                 uiGhostBaseIndex,
+	                                 uiChunkSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+
+	if (eBehaviour == CONVERT_AND_FREE)
+	{
+		eError = _FreeSingleBaseArray(pArena,
+		                              &aBaseArray[uiRealBaseIndex],
+		                              uiGhostBaseIndex - uiRealBaseIndex);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_FreeBaseArray");
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeBaseArraySlice
+ *
+ *  @Description   Convert and maybe free Bases in an Array Slice.
+ *                 This function might convert some ghosts into real bases.
+ *                 This function assumes that the slice is within a single Real base alloc.
+ *                 i.e the uiFreeStartIndex and uiFreeCount remain fully within a single real
+ *                 base alloc and do not cross into another Real base region.
+ *
+ *  @Input pArena - The RA Arena to free bases from.
+ *  @Input aBaseArray - The Base array to free from.
+ *  @Input uiBaseArraySize - The size of the Base array to free from.
+ *  @Input uiChunkSize - The base chunk size used to generate the Ghost bases.
+ *  @Input uiFreeStartIndex - The index in the array to start freeing from
+ *  @Input uiFreeCount - The number of bases to free.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeBaseArraySlice(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 uiFreeStartIndex,
+                             IMG_UINT32 uiFreeCount,
+                             RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	/*3 cases:
+	 * Key: () = Region to Free
+	 *      [R] = Newly Real
+	 *      R = Real Base
+	 *      G = Ghost Base
+	 * 1. We free the whole Realbase (inc all Ghost bases)
+	 *    e.g. (RGGGGG)
+	 *    e.g. RGGG(R)RGG
+	 * 2 .We free the Real base but not all the Ghost bases meaning the first
+	 *    ghost base after the last freed will become a real base.
+	 *    e.g. RGGGG(RGGGG)[R]GGG
+	 *    e.g. (RGGGG)[R]GGGG
+	 * 3. We free some ghost bases from the real base
+	 *    e.g. RGGG(GGG)
+	 *    e.g. RGGG(GGG)[R]GGG
+	 *
+	 * Invalid Scenarios:
+	 * 1. RGG(GR)GGGRG
+	 * 2. RGG(GRG)GGRG
+	 * Higher levels should prevent these situations by ensuring that the free
+	 * index and count always focus on a single real base.
+	 * Scenario 1 & 2, correctly handled, would be a case 3. followed by a case 2.
+	 */
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(uiBaseArraySize >= uiFreeStartIndex &&
+	                        uiBaseArraySize >= uiFreeStartIndex + (uiFreeCount - 1),
+	                        "Free Index given out of array bounds",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Find which case we have */
+
+	/* Case 1 or 2 */
+	if (RA_BASE_IS_REAL(aBaseArray[uiFreeStartIndex]))
+	{
+		/* Case 1 */
+		if (uiFreeStartIndex + uiFreeCount == uiBaseArraySize ||
+		    RA_BASE_IS_REAL(aBaseArray[uiFreeStartIndex + uiFreeCount]) ||
+		    RA_BASE_IS_INVALID(aBaseArray[uiFreeStartIndex + uiFreeCount]))
+		{
+			if (eBehaviour == CONVERT_AND_FREE)
+			{
+				eError = _FreeSingleBaseArray(pArena,
+				                              &aBaseArray[uiFreeStartIndex],
+				                              uiFreeCount);
+				PVR_LOG_RETURN_IF_ERROR(eError, "_FreeBaseArray");
+			}
+		}
+		/* Case 2*/
+		else
+		{
+			eError = _ConvertAndFreeStartFromReal(pArena,
+			                                      aBaseArray,
+			                                      uiChunkSize,
+			                                      uiFreeStartIndex,
+			                                      uiFreeStartIndex + uiFreeCount,
+			                                      eBehaviour);
+			PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeStartFromReal");
+		}
+	}
+	/* Case 3 */
+	else if (RA_BASE_IS_GHOST(aBaseArray[uiFreeStartIndex]))
+	{
+		eError = _ConvertAndFreeStartFromGhost(pArena,
+		                                       aBaseArray,
+		                                       uiBaseArraySize,
+		                                       uiChunkSize,
+		                                       uiFreeStartIndex,
+		                                       uiFreeCount,
+		                                       eBehaviour);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeStartFromGhost");
+	}
+	/* Attempt to free an invalid base, this could be a duplicated
+	 * value in the free sparse index array */
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Attempt to free already free base Index %u", uiFreeStartIndex));
+		PVR_ASSERT(!"Attempted double free.");
+		return PVRSRV_ERROR_RA_FREE_INVALID_CHUNK;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _AllocAlignSplit
+@Description    Given a valid BT, trim the start and end of the BT according
+                to alignment and size requirements. Also add the resulting
+                BT to the live hash table.
+@Input          pArena       The arena.
+@Input          pBT          The BT to trim and add to live hash table
+@Input          uSize        The requested allocation size.
+@Input          uAlignment   The alignment requirements of the allocation
+                             Required uAlignment, or 0.
+                             Must be a power of 2 if not 0
+@Output         pBase        Allocated, corrected, resource base
+                             (non-optional, must not be NULL)
+@Output         phPriv       The user references associated with
+                             the imported segment. (optional)
+@Return         IMG_FALSE failure
+                IMG_TRUE success
+*/ /**************************************************************************/
+static IMG_BOOL
+_AllocAlignSplit(RA_ARENA *pArena,
+                 BT *pBT,
+                 RA_LENGTH_T uSize,
+                 RA_LENGTH_T uAlignment,
+                 RA_BASE_T *pBase,
+                 RA_PERISPAN_HANDLE *phPriv)
+{
+	RA_BASE_T aligned_base;
+
+	aligned_base = (uAlignment > 1) ? PVR_ALIGN(pBT->base, uAlignment) : pBT->base;
+
+	_FreeListRemove(pArena, pBT);
+
+	if ((pArena->ui32PolicyFlags & RA_POLICY_NO_SPLIT_MASK) == RA_POLICY_NO_SPLIT)
+	{
+		goto nosplit;
+	}
+
+	/* with uAlignment we might need to discard the front of this segment */
+	if (aligned_base > pBT->base)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, (RA_LENGTH_T)(aligned_base - pBT->base));
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Front split failed", __func__));
+			/* Put pBT back in the list */
+			_FreeListInsert(pArena, pBT);
+			return IMG_FALSE;
+		}
+
+		_FreeListInsert(pArena, pBT);
+		pBT = pNeighbour;
+	}
+
+	/* the segment might be too big, if so, discard the back of the segment */
+	if (pBT->uSize > uSize)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, uSize);
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Back split failed", __func__));
+			/* Put pBT back in the list */
+			_FreeListInsert(pArena, pBT);
+			return IMG_FALSE;
+		}
+
+		_FreeListInsert(pArena, pNeighbour);
+	}
+nosplit:
+	pBT->type = btt_live;
+
+	if (!HASH_Insert_Extended(pArena->pSegmentHash, &aligned_base, (uintptr_t)pBT))
+	{
+		_FreeBT(pArena, pBT);
+		return IMG_FALSE;
+	}
+
+	if (phPriv != NULL)
+		*phPriv = pBT->hPriv;
+
+	*pBase = aligned_base;
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptAllocAligned
+@Description    Attempt an allocation from an arena.
+@Input          pArena       The arena.
+@Input          uSize        The requested allocation size.
+@Input          uFlags       Allocation flags
+@Output         phPriv       The user references associated with
+                             the imported segment. (optional)
+@Input          uAlignment   Required uAlignment, or 0.
+                             Must be a power of 2 if not 0
+@Output         base         Allocated resource base (non-optional, must not
+                             be NULL)
+@Return         IMG_FALSE failure
+                IMG_TRUE success
+*/ /**************************************************************************/
+static IMG_BOOL
+_AttemptAllocAligned(RA_ARENA *pArena,
+                     RA_LENGTH_T uSize,
+                     RA_FLAGS_T uFlags,
+                     RA_LENGTH_T uAlignment,
+                     RA_BASE_T *base,
+                     RA_PERISPAN_HANDLE *phPriv) /* this is the "per-import" private data */
+{
+
+	IMG_UINT32 index_low;
+	IMG_UINT32 index_high;
+	IMG_UINT32 i;
+	struct _BT_ *pBT = NULL;
+
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(base != NULL);
+
+	pArena->per_flags_buckets = PVRSRVSplay(uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != uFlags))
+	{
+		/* no chunks with these flags. */
+		return IMG_FALSE;
+	}
+
+	index_low = pvr_log2(uSize);
+	if (uAlignment)
+	{
+		index_high = pvr_log2(uSize + uAlignment - 1);
+	}
+	else
+	{
+		index_high = index_low;
+	}
+
+	_FreeTableLimitBoundsCheck(&index_high);
+	_FreeTableLimitBoundsCheck(&index_low);
+
+	PVR_ASSERT(index_low <= index_high);
+
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_BUCKET_MASK) == RA_POLICY_BUCKET_BEST_FIT))
+	{
+		/* This policy ensures the selection of the first lowest size bucket that
+		 * satisfies the request size is selected */
+#if defined(PVR_CTZLL)
+		i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_low )) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+		i = index_low;
+#endif
+		for ( ; (i < FREE_TABLE_LIMIT) && (pBT == NULL); ++i)
+		{
+			if (pArena->per_flags_buckets->buckets[i])
+			{
+				pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, (unsigned int) ~0);
+			}
+		}
+	}
+	else
+	{
+#if defined(PVR_CTZLL)
+		i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_high + 1)) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+		for (i = index_high + 1; (i < FREE_TABLE_LIMIT) && (pArena->per_flags_buckets->buckets[i] == NULL); ++i)
+		{
+		}
+#endif
+		PVR_ASSERT(i <= FREE_TABLE_LIMIT);
+
+		if (i != FREE_TABLE_LIMIT)
+		{
+			/* since we start at index_high + 1, we are guaranteed to exit */
+			pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, 1);
+		}
+		else
+		{
+			for (i = index_high; (i != index_low - 1) && (pBT == NULL); --i)
+			{
+				pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, (unsigned int) ~0);
+			}
+		}
+	}
+
+	if (pBT == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	return _AllocAlignSplit(pArena, pBT, uSize, uAlignment, base, phPriv);
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptAllocAlignedAssured
+@Description    Attempt an allocation from an arena. If the arena allows
+                non-contiguous allocations, the allocation is guaranteed
+                given there is enough memory to satisfy the full allocation.
+@Input          pArena              The arena.
+@Input          uSize               The requested allocation size.
+@Input          uLog2MinContigSize  The Log2 minimum contiguity of the bases returned.
+@Input          uFlags              Allocation flags
+@Input          uAlignment          Required uAlignment, or 0.
+                                    Must be a power of 2 if not 0
+@Input          aBaseArray          Array to allocate bases to.
+@Input          bSparseAlloc        Is the allocation we are making sparse.
+@Output         bPhysContig         Is the allocation we made physically contiguous
+                                    or did we use the scoop logic
+@Return         Success: PVRSRV_OK
+                Fail:    PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptAllocAlignedAssured(RA_ARENA *pArena,
+                            RA_LENGTH_T uSize,
+                            IMG_UINT32 uLog2MinContigSize,
+                            RA_FLAGS_T uFlags,
+                            RA_LENGTH_T uAlignment,
+                            RA_BASE_ARRAY_T aBaseArray,
+                            IMG_BOOL bSparseAlloc,
+                            IMG_BOOL *bPhysContig)
+{
+	IMG_UINT32 index_low;  /* log2 Lowest contiguity required */
+	IMG_UINT32 index_high; /* log2 Size of full alloc */
+	IMG_UINT32 i;
+	struct _BT_ *pBT = NULL;
+	RA_PERISPAN_HANDLE phPriv;
+	RA_LENGTH_T uiRemaining = uSize;
+	RA_BASE_T uiBase;
+	IMG_UINT32 uiCurrentArrayIndex = 0;
+
+	PVR_ASSERT(pArena != NULL);
+
+	pArena->per_flags_buckets = PVRSRVSplay(uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != uFlags))
+	{
+		/* no chunks with these flags. */
+		return PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS;
+	}
+
+	if (pArena->ui64FreeArenaSize < uSize)
+	{
+		/* Not enough memory to accommodate kick back for a chance to import more */
+		return PVRSRV_ERROR_RA_OUT_OF_RESOURCE;
+	}
+
+	if (uLog2MinContigSize && uAlignment)
+	{
+		index_low = uLog2MinContigSize;
+		index_high = pvr_log2(uSize);
+	}
+	else if (uLog2MinContigSize)
+	{
+		index_low = uLog2MinContigSize;
+		index_high = pvr_log2(uSize);
+	}
+	else if (uAlignment)
+	{
+		index_low = 0;
+		index_high = pvr_log2(uSize + uAlignment - 1);
+	}
+	else
+	{
+		index_low = 0;
+		index_high = pvr_log2(uSize);
+	}
+
+	PVR_ASSERT(index_low < FREE_TABLE_LIMIT);
+	PVR_ASSERT(index_high < FREE_TABLE_LIMIT);
+	PVR_ASSERT(index_low <= index_high);
+
+	/* Start at index_high + 1 as then we can check all buckets larger than the desired alloc
+	 * If we don't find one larger then we could still find one of requested size in index_high and
+	 * shortcut the non-contiguous allocation path. We check index_high + 1 first as it is
+	 * guaranteed to have a free region of the requested size if the bucket has entries. Whereas
+	 * index_high is not guaranteed to have an allocation that meets the size requested due to it
+	 * representing all free regions of size 2^bucket index to 2^bucket index +1. e.g we could have
+	 * a request for 19*4k Pages which would be represented by bucket 16, bucket 16 represents free
+	 * entries from 16*4k pages to 31*4k Pages in size, if this bucket only had free entries of
+	 * 17*4k pages the search would fail, hence not guaranteed at index_high.
+	 */
+#if defined(PVR_CTZLL)
+	i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_high + 1)) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+	for (i = index_high + 1; (i < FREE_TABLE_LIMIT) && (pArena->per_flags_buckets->buckets[i] == NULL); i++)
+	{
+	}
+#endif
+
+	PVR_ASSERT(i <= FREE_TABLE_LIMIT);
+
+	if (i != FREE_TABLE_LIMIT)
+	{
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, 1);
+	}
+	else
+	{
+		/* In this case we have searched all buckets index_high + 1 to FREE_TABLE_LIMIT and not found an
+		 * available bucket with the required allocation size.
+		 * Because we haven't found an allocation of the requested size in index_high + 1 there is still a chance
+		 * that we can find an allocation of correct size in index_high, when index_high references the bucket
+		 * containing the largest free chunks in the RA Arena. i.e All buckets > index_high == NULL.
+		 * We do a final search in that bucket here before we attempt to scoop memory or return NULL.
+		 */
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[index_high], uSize, uAlignment, 1);
+	}
+
+	/* We managed to find a contiguous allocation block of sufficient size */
+	if (pBT != NULL)
+	{
+		IMG_BOOL bResult;
+		bResult =  _AllocAlignSplit(pArena, pBT, uSize, uAlignment, &uiBase, &phPriv);
+		if (bResult)
+		{
+			if (!bSparseAlloc)
+			{
+				aBaseArray[0] = uiBase;
+			}
+			else
+			{
+				_GenerateGhostBases(uiBase, uSize, 1ULL << uLog2MinContigSize, aBaseArray);
+			}
+		}
+		else
+		{
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+		*bPhysContig = IMG_TRUE;
+
+		return PVRSRV_OK;
+	}
+
+	/*
+	 * If this arena doesn't have the non-contiguous allocation functionality enabled, then
+	 * don't attempt to scoop for non physically contiguous allocations. Sparse allocations
+	 * are still able to use the scoop functionality as they map in a chunk at a time in the
+	 * worst case.
+	 */
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == 0) &&
+	    !bSparseAlloc)
+	{
+		return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+	}
+
+	/* Attempt to Scoop memory from non-contiguous blocks */
+	for (i = index_high; i >= index_low && uiRemaining != 0; i--)
+	{
+		/* While we have chunks of at least our contig size in the bucket to use */
+		for (
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], 1ULL << uLog2MinContigSize, uAlignment,(unsigned int) ~0);
+		pBT != NULL && uiRemaining != 0;
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], 1ULL << uLog2MinContigSize, uAlignment,(unsigned int) ~0))//~0 Try all elements in bucket
+		{
+			/* Grab largest chunk possible that is a multiple of our min contiguity size
+			 * N.B: C always rounds towards 0 so this effectively floors for us */
+			IMG_BOOL bResult;
+			RA_BASE_T uiAlignedBase =
+			(uAlignment > 1) ? PVR_ALIGN(pBT->base, uAlignment) : pBT->base;
+			RA_LENGTH_T uiMaxSizeAvailable = (pBT->uSize - (uiAlignedBase - pBT->base));
+			RA_LENGTH_T uiMaxMultipleOfContig = (uiMaxSizeAvailable >> uLog2MinContigSize) << uLog2MinContigSize;
+
+			/*
+			 * If the size of the BT is larger than the remaining memory to allocate
+			 * then just allocate what we need. The rest will be trimmed and put back
+			 * into the pool in _AllocAlignSplit
+			 */
+			if (uiMaxMultipleOfContig > uiRemaining)
+			{
+				uiMaxMultipleOfContig = uiRemaining;
+			}
+
+			bResult = _AllocAlignSplit(pArena, pBT, uiMaxMultipleOfContig, uAlignment, &uiBase, &phPriv);
+			if (!bResult)
+			{
+				/* Something went wrong with splitting or adding to hash,
+				 * We can try find another chunk, although this should
+				 * never occur.
+				 */
+				PVR_ASSERT(!"_AllocAlignSplit issue.");
+				continue;
+			}
+
+			uiRemaining -= uiMaxMultipleOfContig;
+
+			uiCurrentArrayIndex += _GenerateGhostBases(uiBase,
+			                                           uiMaxMultipleOfContig,
+			                                           1ULL << uLog2MinContigSize,
+			                                           &aBaseArray[uiCurrentArrayIndex]);
+		}
+	}
+
+	/* If we didn't manage to scoop enough memory then we need to unwind the allocations we just made */
+	if (uiRemaining != 0)
+	{
+		goto error_unwind;
+	}
+	*bPhysContig = IMG_FALSE;
+
+	return PVRSRV_OK;
+
+error_unwind:
+	_RA_FreeMultiUnlocked(pArena,
+	                       aBaseArray,
+	                       uiCurrentArrayIndex);
+	return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+}
+
+/* Used as an inout parameter to reduce required memory allocations.
+ * The import callbacks return one or more RA_IMPORT which is then converted into a BT*.
+ * Once a RA_IMPORT is converted to a BT*, it is no longer needed. Therefore,
+ * the space taken up by the RA_IMPORT can be reused. */
+typedef union {
+	RA_IMPORT import;
+	BT *pBT;
+} RA_IMPORT_INTERNAL;
+
+
+static_assert(sizeof(RA_IMPORT_INTERNAL) == sizeof(RA_IMPORT),
+              "An RA_IMPORT[] array must be equivalent to an RA_IMPORT_INTERNAL[] array. "
+              "This allows RA_IMPORT[] returned by callbacks to be reused internally "
+              "as BT*[] without leaking scope to the user API.");
+
+static_assert(offsetof(RA_IMPORT_INTERNAL, pBT) == 0,
+              "The pBT member must be at offset 0 to match a BT*[].");
+
+/* Support importing 16 spans at once before allocating more memory.
+ * Almost all imports will only require a single span.
+ * However, if importing multiple spans, shouldn't always require memory allocations.
+ * Nominal use of multi imports shouldn't require many spans. */
+#define RA_DEFAULT_IMPORTS_BUFFER_SIZE (16 * (sizeof(RA_IMPORT) / sizeof(BT*)))
+
+static_assert((RA_DEFAULT_IMPORTS_BUFFER_SIZE * sizeof(BT*)) >= sizeof(RA_IMPORT),
+              "The default must be able to hold at least a single RA_IMPORT.");
+
+typedef struct {
+	BT **apsBTs;
+	IMG_UINT32 uiBTCount;
+} BT_PTR_ARRAY;
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAllocSingle
+@Description    Attempt to import more memory and create a new span.
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form a new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation.
+@Output         psImportInternal  The pBT member must be set to the new span.
+                                  Unless an error is returned.
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAllocSingle(RA_ARENA *pArena,
+                              RA_LENGTH_T uRequestSize,
+                              RA_FLAGS_T uImportFlags,
+                              RA_LENGTH_T uAlignment,
+                              const IMG_CHAR *pszAnnotation,
+                              RA_IMPORT_INTERNAL *psImportInternal)
+{
+	PVRSRV_ERROR eError;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+	RA_IMPORT *psImport = &psImportInternal->import;
+	BT *pBT;
+
+	eError = pArena->pImportAllocSingle(pArena->pImportHandle,
+	                                    uRequestSize, uImportFlags,
+	                                    uAlignment,
+	                                    pszAnnotation,
+	                                    psImport);
+	if (PVRSRV_OK != eError)
+	{
+		return eError;
+	}
+
+	/* If we successfully import more resource, create a span to
+	 * represent it else free the resource we imported.
+	 */
+	pBT = _InsertResourceSpan(pArena, psImport->base, psImport->uSize, uFlags);
+	if (pBT == NULL)
+	{
+		/* insufficient resources to insert the newly acquired span,
+		   so free it back again */
+		pArena->pImportFree(pArena->pImportHandle, psImport->base, psImport->hPriv);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+		        "size=0x%llx failed!", __func__, pArena->name,
+		        (unsigned long long)uRequestSize));
+		/* RA_Dump (arena); */
+
+		return PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	}
+
+	pBT->hPriv = psImport->hPriv;
+	/* Replace the RA_IMPORT with a BT*.
+	 * To free an import we only need the BT so we can reuse the elements slot in ppsImports. */
+	psImportInternal->pBT = pBT;
+
+	pArena->ui64TotalArenaSize += pBT->uSize;
+	pArena->ui64FreeArenaSize += pBT->uSize;
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAllocMulti
+@Description    Attempt to import more memory and create new span(s).
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form at least one new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation.
+@Inout          puiBTCount        The number of elements in *ppsImports. The
+                                  count will be modified to reflect how many
+                                  BT*s have been created.
+@Inout          ppsImports        Pointer to an array of RA_IMPORT_INTERNAL.
+                                  If the must be array resized, this will point
+                                  to the new array. It will be freed higher up
+                                  the callstack if necessary. If successful,
+                                  the pBT member must be set to a new span for
+                                  every element in the array of length puiBTCount.
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAllocMulti(RA_ARENA *pArena,
+                             RA_LENGTH_T uRequestSize,
+                             RA_FLAGS_T uImportFlags,
+                             RA_LENGTH_T uAlignment,
+                             const IMG_CHAR *pszAnnotation,
+                             IMG_UINT32 *puiImportCount,
+                             RA_IMPORT_INTERNAL **ppsImports)
+{
+	PVRSRV_ERROR eError;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+	IMG_UINT32 i;
+
+	eError = pArena->pImportAllocMulti(pArena->pImportHandle,
+	                                   uRequestSize, uImportFlags,
+	                                   uAlignment,
+	                                   pszAnnotation,
+	                                   puiImportCount,
+	                                   (RA_IMPORT**) ppsImports);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* An error should be returned by the callback if the count == 0. */
+	PVR_ASSERT(*puiImportCount >= 1);
+	/* An error should be returned by the callback if *ppsImports == NULL. */
+	PVR_ASSERT(*ppsImports != NULL);
+
+	/* If we successfully import more resource, create spans to represent it. */
+	for (i = 0; i < *puiImportCount; i++)
+	{
+		BT *pBT;
+		RA_IMPORT *psImport = &(*ppsImports)[i].import;
+
+		pBT = _InsertResourceSpan(pArena, psImport->base, psImport->uSize, uFlags);
+		PVR_GOTO_IF_FALSE(pBT != NULL, release_bts);
+
+		pBT->hPriv = psImport->hPriv;
+		pArena->ui64TotalArenaSize += pBT->uSize;
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		/* Replace the RA_IMPORT with a BT*.
+		 * To free an import we only need the BT so we can reuse the elements slot in ppsImports. */
+		(*ppsImports)[i].pBT = pBT;
+	}
+
+	return PVRSRV_OK;
+
+release_bts:
+	{
+		/* Free the just imported span that failed to allocate a BT. */
+		IMG_UINT32 j;
+		RA_IMPORT *psImport = &(*ppsImports)[i].import;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+		        "request=0x%llx size=0x%llx failed!", __func__, pArena->name,
+		        (unsigned long long)uRequestSize, (unsigned long long)psImport->uSize));
+
+		/* Insufficient resources to insert the newly acquired span so free it back again. */
+		pArena->pImportFree(pArena->pImportHandle, psImport->base, psImport->hPriv);
+
+		/* Remove the previously inserted BTs. */
+		for (j = 0; j < i; j++)
+		{
+			BT *pBT = (*ppsImports)[j].pBT;
+			pArena->ui64TotalArenaSize -= pBT->uSize;
+			pArena->ui64FreeArenaSize -= pBT->uSize;
+			_FreeBT(pArena, pBT);
+		}
+
+		return PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAlloc
+@Description    Attempt to import more memory and create new span(s).
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form a new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportMultiplier Import x-times more for future requests if
+                                  we have to import new memory.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation
+@Inout          psBTArray         A container to a BT*[] and it's length.
+                                  If the array is not large enough, a new array
+                                  will be returned and must be freed with OSFreeMem
+                                  regardless of the return error code.
+                                  On error, the array may be NULL!
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAlloc(RA_ARENA *pArena,
+                        RA_LENGTH_T uRequestSize,
+                        IMG_UINT8 uImportMultiplier,
+                        RA_FLAGS_T uImportFlags,
+                        RA_LENGTH_T uAlignment,
+                        const IMG_CHAR *pszAnnotation,
+                        BT_PTR_ARRAY *psBTArray)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	RA_IMPORT_INTERNAL *psImports;
+
+	/* Interpret the BT*[] as a RA_IMPORT_INTERNAL[].
+	 * Any extra space is ignored.
+	 * Example:
+	 * |---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|
+	 * |----RA_IMPORT_INTERNAL----|----RA_IMPORT_INTERNAL----|........|
+	 *
+	 * It must be able to hold at least 1 RA_IMPORT_INTERNAL.
+	 * This should never fail if RA_DEFAULT_IMPORTS_BUFFER_SIZE is used. */
+	PVR_ASSERT(psBTArray->uiBTCount * sizeof(BT*) >= sizeof(RA_IMPORT_INTERNAL));
+	PVR_ASSERT(psBTArray != NULL && psBTArray->apsBTs != NULL);
+	psImports = (RA_IMPORT_INTERNAL*) psBTArray->apsBTs;
+
+	/* apply over-allocation multiplier after all alignment adjustments */
+	uRequestSize *= uImportMultiplier;
+	/* ensure that we import according to the quanta of this arena */
+	uRequestSize = PVR_ALIGN(uRequestSize, pArena->uQuantum);
+
+	switch (pArena->eImportAllocMethod)
+	{
+	/* Exactly 1 BT will be created. */
+	case RA_IMPORT_ALLOC_METHOD_SINGLE:
+	{
+		eError = _AttemptImportSpanAllocSingle(pArena, uRequestSize, uImportFlags, uAlignment, pszAnnotation,  &psImports[0]);
+		PVR_GOTO_IF_ERROR(eError, err_);
+		psBTArray->uiBTCount = 1;
+
+		/* Reinterpret the RA_IMPORT_INTERNAL[] back into a BT*[].
+		 * We only care about psBTArray->apsBTs[0] being valid as we return a count of 1. */
+		psBTArray->apsBTs = (BT**) psImports;
+		break;
+	}
+	/* 1 or more BTs will be created. */
+	case RA_IMPORT_ALLOC_METHOD_MULTI:
+	{
+		/* The number of RA_IMPORT_INTERNAL that can be stored using the same space. */
+		IMG_UINT32 uiImportCount = (psBTArray->uiBTCount) / (sizeof(RA_IMPORT_INTERNAL) / sizeof(BT*));
+
+		eError = _AttemptImportSpanAllocMulti(pArena, uRequestSize, uImportFlags, uAlignment, pszAnnotation, &uiImportCount, &psImports);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_AttemptImportSpanAllocMulti");
+
+		/* Reinterpret the RA_IMPORT_INTERNAL[] back into a BT*[]. */
+		psBTArray->apsBTs = (BT**) psImports;
+
+		/* If a BT* is smaller than an RA_IMPORT_INTERNAL, the array of BT*s will not be tightly packed.
+		 * Therefore, it is not a valid BT*[] and the elements need to be compressed.
+		 * Example: Given the array below
+		 * |----RA_IMPORT_INTERNAL-0--|----RA_IMPORT_INTERNAL-1--|........|
+		 * |--BT*-0-|.................|--BT*-1-|..........................|
+		 * It must be packed into this:
+		 * |--BT*-0-|--BT*-1-|............................................| */
+		if (sizeof(BT*) < sizeof(RA_IMPORT_INTERNAL))
+		{
+			IMG_UINT32 i;
+			/* The first element will always be in the correct place. Skip redundant move. */
+			for (i = 1; i < uiImportCount; i++)
+			{
+				psBTArray->apsBTs[i] = psImports[i].pBT;
+			}
+		}
+		psBTArray->uiBTCount = uiImportCount;
+		break;
+	}
+	default:
+		PVR_ASSERT(!"unreachable");
+		break;
+	}
+
+	return eError;
+
+err_:
+	if ((void*) psImports != (void*) psBTArray->apsBTs && psImports != NULL)
+	{
+		OSFreeMem(psImports);
+	}
+	return eError;
+}
+
+static RA_ARENA*
+RA_CreateCommon(IMG_CHAR *name,
+                 RA_LOG2QUANTUM_T uLog2Quantum,
+                 IMG_UINT32 ui32LockClass,
+                 enum RA_IMPORT_ALLOC_METHOD eImportAllocMethod,
+                 void* imp_alloc,
+                 PFN_RA_IMPORT_FREE imp_free,
+                 RA_PERARENA_HANDLE arena_handle,
+                 RA_POLICY_T ui32PolicyFlags)
+{
+	RA_ARENA *pArena;
+	PVRSRV_ERROR eError;
+
+	if (name == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter 'name' (NULL not accepted)", __func__));
+		return NULL;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s'", __func__, name));
+
+	pArena = OSAllocMem(sizeof(*pArena));
+	if (pArena == NULL)
+	{
+		goto arena_fail;
+	}
+
+	eError = OSLockCreate(&pArena->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		goto lock_fail;
+	}
+
+	pArena->pSegmentHash = HASH_Create_Extended(MINIMUM_HASH_SIZE, sizeof(RA_BASE_T), HASH_Func_Default, HASH_Key_Comp_Default);
+
+	if (pArena->pSegmentHash==NULL)
+	{
+		goto hash_fail;
+	}
+
+	if (imp_alloc == NULL) {
+		eImportAllocMethod = RA_IMPORT_ALLOC_METHOD_SINGLE;
+		imp_alloc = &_RequestAllocFail;
+	}
+
+	OSStringSafeCopy(pArena->name, name, RA_MAX_NAME_LENGTH);
+	pArena->eImportAllocMethod = eImportAllocMethod;
+	pArena->pImportAlloc = imp_alloc; /* import callback union */
+	pArena->pImportFree = imp_free;
+	pArena->pImportHandle = arena_handle;
+	pArena->pHeadSegment = NULL;
+	pArena->uQuantum = IMG_PAGE2BYTES64(uLog2Quantum);
+	pArena->per_flags_buckets = NULL;
+	pArena->ui32LockClass = ui32LockClass;
+	pArena->ui32PolicyFlags = ui32PolicyFlags;
+	pArena->ui64TotalArenaSize = 0;
+	pArena->ui64FreeArenaSize = 0;
+
+	PVR_ASSERT(is_arena_valid(pArena));
+	return pArena;
+
+hash_fail:
+	OSLockDestroy(pArena->hLock);
+lock_fail:
+	OSFreeMem(pArena);
+	/* not nulling pointer, out of scope */
+arena_fail:
+	return NULL;
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_Create(IMG_CHAR *name,
+          RA_LOG2QUANTUM_T uLog2Quantum,
+          IMG_UINT32 ui32LockClass,
+          PFN_RA_IMPORT_ALLOC_SINGLE imp_alloc,
+          PFN_RA_IMPORT_FREE imp_free,
+          RA_PERARENA_HANDLE arena_handle,
+          RA_POLICY_T ui32PolicyFlags)
+{
+	return RA_CreateCommon(name,
+	                       uLog2Quantum,
+	                       ui32LockClass,
+	                       RA_IMPORT_ALLOC_METHOD_SINGLE,
+	                       imp_alloc,
+	                       imp_free,
+	                       arena_handle,
+	                       ui32PolicyFlags
+	);
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_CreateMulti(IMG_CHAR *name,
+               RA_LOG2QUANTUM_T uLog2Quantum,
+               IMG_UINT32 ui32LockClass,
+               PFN_RA_IMPORT_ALLOC_MULTI imp_alloc,
+               PFN_RA_IMPORT_FREE imp_free,
+               RA_PERARENA_HANDLE arena_handle,
+               RA_POLICY_T ui32PolicyFlags)
+{
+	return RA_CreateCommon(name,
+	                       uLog2Quantum,
+	                       ui32LockClass,
+	                       RA_IMPORT_ALLOC_METHOD_MULTI,
+	                       imp_alloc,
+	                       imp_free,
+	                       arena_handle,
+	                       ui32PolicyFlags
+	);
+}
+
+static void _LogRegionCreation(const char *pszMemType,
+                               IMG_UINT64 ui64CpuPA,
+                               IMG_UINT64 ui64DevPA,
+                               IMG_UINT64 ui64Size)
+{
+#if !defined(DEBUG)
+	PVR_UNREFERENCED_PARAMETER(pszMemType);
+	PVR_UNREFERENCED_PARAMETER(ui64CpuPA);
+	PVR_UNREFERENCED_PARAMETER(ui64DevPA);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+#else
+	if ((ui64CpuPA != 0) && (ui64DevPA != 0) && (ui64CpuPA != ui64DevPA))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		        "Creating RA for \"%s\" memory"
+		        " - Cpu PA 0x%016" IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx
+		        " - Dev PA 0x%016" IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx,
+		        pszMemType,
+		        ui64CpuPA, ui64CpuPA + ui64Size,
+		        ui64DevPA, ui64DevPA + ui64Size));
+	}
+	else
+	{
+		__maybe_unused IMG_UINT64 ui64PA =
+			ui64CpuPA != 0 ? ui64CpuPA : ui64DevPA;
+		__maybe_unused const IMG_CHAR *pszAddrType =
+			ui64CpuPA == ui64DevPA ? "Cpu/Dev" : (ui64CpuPA != 0 ? "Cpu" : "Dev");
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+		        "Creating RA for \"%s\" memory - %s PA 0x%016"
+		        IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx,
+		        pszMemType, pszAddrType,
+		        ui64PA, ui64PA + ui64Size));
+	}
+#endif
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_Create_With_Span(IMG_CHAR *name,
+                    RA_LOG2QUANTUM_T uLog2Quantum,
+                    IMG_UINT64 ui64CpuBase,
+                    IMG_UINT64 ui64SpanDevBase,
+                    IMG_UINT64 ui64SpanSize,
+                    RA_POLICY_T ui32PolicyFlags)
+{
+	RA_ARENA *psRA;
+	IMG_BOOL bSuccess;
+
+	psRA = RA_Create(name,
+	                 uLog2Quantum,       /* Use OS page size, keeps things simple */
+	                 RA_LOCKCLASS_0,     /* This arena doesn't use any other arenas. */
+	                 NULL,               /* No Import */
+	                 NULL,               /* No free import */
+	                 NULL,               /* No import handle */
+	                 ui32PolicyFlags);   /* No restriction on import splitting */
+	PVR_LOG_GOTO_IF_FALSE(psRA != NULL, "RA_Create() failed", return_);
+
+	bSuccess = RA_Add(psRA, (RA_BASE_T) ui64SpanDevBase, (RA_LENGTH_T) ui64SpanSize, 0, NULL);
+	PVR_LOG_GOTO_IF_FALSE(bSuccess, "RA_Add() failed", cleanup_);
+
+	_LogRegionCreation(name, ui64CpuBase, ui64SpanDevBase, ui64SpanSize);
+
+	return psRA;
+
+cleanup_:
+	RA_Delete(psRA);
+return_:
+	return NULL;
+}
+
+IMG_INTERNAL void
+RA_Delete(RA_ARENA *pArena)
+{
+	IMG_UINT32 uIndex;
+	IMG_BOOL bPrintWarn = IMG_TRUE;
+
+	PVR_ASSERT(pArena != NULL);
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return;
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			  "%s: name='%s'", __func__, pArena->name));
+
+	while (pArena->pHeadSegment != NULL)
+	{
+		BT *pBT = pArena->pHeadSegment;
+
+		if (pBT->type != btt_free)
+		{
+#if !defined(DEBUG)
+			if (bPrintWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Allocations still exist in the arena that is being destroyed. Arena Name: %s", __func__, pArena->name));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: base = 0x%llx size=0x%llx", __func__,
+					  (unsigned long long)pBT->base, (unsigned long long)pBT->uSize));
+				PVR_DPF((PVR_DBG_ERROR, "%s: This warning will be issued only once for the first allocation found!", __func__));
+				bPrintWarn = IMG_FALSE;
+			}
+#else
+			if (bPrintWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Allocations still exist in the arena that is being destroyed. Arena Name: %s", __func__, pArena->name));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				bPrintWarn = IMG_FALSE;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: base = 0x%llx size=0x%llx flags=0x%llx", __func__,
+				  (unsigned long long)pBT->base, (unsigned long long)pBT->uSize,
+				  (unsigned long long)pBT->uFlags));
+#endif
+		}
+		else
+		{
+			_FreeListRemove(pArena, pBT);
+		}
+
+		_SegmentListRemove(pArena, pBT);
+		OSFreeMem(pBT);
+		/* not nulling original pointer, it has changed */
+	}
+
+	while (pArena->per_flags_buckets != NULL)
+	{
+		for (uIndex=0; uIndex<FREE_TABLE_LIMIT; uIndex++)
+		{
+			PVR_ASSERT(pArena->per_flags_buckets->buckets[uIndex] == NULL);
+		}
+
+		pArena->per_flags_buckets = PVRSRVDelete(pArena->per_flags_buckets->uiFlags, pArena->per_flags_buckets);
+	}
+
+	HASH_Delete(pArena->pSegmentHash);
+	OSLockDestroy(pArena->hLock);
+	OSFreeMem(pArena);
+	/* not nulling pointer, copy on stack */
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_Add(RA_ARENA *pArena,
+       RA_BASE_T base,
+       RA_LENGTH_T uSize,
+       RA_FLAGS_T uFlags,
+       RA_PERISPAN_HANDLE hPriv)
+{
+	struct _BT_* bt;
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(uSize != 0);
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return IMG_FALSE;
+	}
+
+	if (uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid size 0 added to arena %s", __func__, pArena->name));
+		return IMG_FALSE;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+			 "base=0x%llx, size=0x%llx", __func__, pArena->name,
+			 (unsigned long long)base, (unsigned long long)uSize));
+
+	uSize = PVR_ALIGN(uSize, pArena->uQuantum);
+	bt = _InsertResource(pArena, base, uSize, uFlags);
+	if (bt != NULL)
+	{
+		bt->hPriv = hPriv;
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64TotalArenaSize += uSize;
+	pArena->ui64FreeArenaSize += uSize;
+	OSLockRelease(pArena->hLock);
+
+	return bt != NULL;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_Alloc(RA_ARENA *pArena,
+         RA_LENGTH_T uRequestSize,
+         IMG_UINT8 uImportMultiplier,
+         RA_FLAGS_T uImportFlags,
+         RA_LENGTH_T uAlignment,
+         const IMG_CHAR *pszAnnotation,
+         RA_BASE_T *base,
+         RA_LENGTH_T *pActualSize,
+         RA_PERISPAN_HANDLE *phPriv)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bResult;
+	RA_LENGTH_T uSize = uRequestSize;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+
+	if (pArena == NULL || uImportMultiplier == 0 || uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: One of the necessary parameters is 0", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	if (pActualSize != NULL)
+	{
+		*pActualSize = uSize;
+	}
+
+	/* Must be a power of 2 or 0 */
+	PVR_ASSERT((uAlignment == 0) || (uAlignment & (uAlignment - 1)) == 0);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: arena='%s', size=0x%llx(0x%llx), "
+	        "alignment=0x%llx", __func__, pArena->name,
+	        (unsigned long long)uSize, (unsigned long long)uRequestSize,
+	        (unsigned long long)uAlignment));
+
+	/* if allocation failed then we might have an import source which
+	   can provide more resource, else we will have to fail the
+	   allocation to the caller. */
+	bResult = _AttemptAllocAligned(pArena, uSize, uFlags, uAlignment, base, phPriv);
+	if (!bResult)
+	{
+		BT *apsBTsStatic[RA_DEFAULT_IMPORTS_BUFFER_SIZE];
+		BT_PTR_ARRAY sBTs = {
+			.uiBTCount = ARRAY_SIZE(apsBTsStatic),
+			.apsBTs = apsBTsStatic,
+		};
+
+		eError = _AttemptImportSpanAlloc(pArena,
+		                                 uSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 uAlignment,
+		                                 pszAnnotation,
+		                                 &sBTs);
+		if (eError != PVRSRV_OK)
+		{
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			OSLockRelease(pArena->hLock);
+			return eError;
+		}
+
+		bResult = _AttemptAllocAligned(pArena, uSize, uFlags, uAlignment, base, phPriv);
+		if (!bResult)
+		{
+			IMG_UINT32 i;
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: name='%s' second alloc failed!",
+			        __func__, pArena->name));
+
+			/*
+			  On failure of _AttemptAllocAligned() depending on the exact point
+			  of failure, the imported segment may have been used and freed, or
+			  left untouched. If the later, we need to return it.
+			*/
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				_FreeBT(pArena, sBTs.apsBTs[i]);
+			}
+
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			OSLockRelease(pArena->hLock);
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+		else
+		{
+			IMG_UINT32 i;
+			IMG_BOOL bBaseInNewSpans = IMG_FALSE;
+
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				BT *pBT = sBTs.apsBTs[i];
+				/* Check if the new allocation was in a span we just added... */
+				if (*base >= pBT->base && *base < (pBT->base + pBT->uSize))
+				{
+					bBaseInNewSpans = IMG_TRUE;
+					break;
+				}
+			}
+			if (!bBaseInNewSpans)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: name='%s' alloc did not occur in the imported span!",
+				         __func__, pArena->name));
+				/* Remove the imported spans which should not be in use (if it is then that is okay).
+				 * No spans should exist and not be used unless added through the import multiplier. */
+				for (i = 0; i < sBTs.uiBTCount; i++)
+				{
+					_FreeBT(pArena, sBTs.apsBTs[i]);
+				}
+			}
+		}
+		if (sBTs.apsBTs != apsBTsStatic)
+		{
+			OSFreeMem(sBTs.apsBTs);
+		}
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', size=0x%llx, "
+	        "*base=0x%llx = %d", __func__, pArena->name, (unsigned long long)uSize,
+	        (unsigned long long)*base, bResult));
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+	OSLockRelease(pArena->hLock);
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_RA_AllocMultiUnlocked(RA_ARENA *pArena,
+                       RA_LENGTH_T uRequestSize,
+                       IMG_UINT32 uiLog2ChunkSize,
+                       IMG_UINT8 uImportMultiplier,
+                       RA_FLAGS_T uImportFlags,
+                       const IMG_CHAR *pszAnnotation,
+                       RA_BASE_ARRAY_T aBaseArray,
+                       RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                       IMG_BOOL bSparseAlloc,
+                       IMG_BOOL *bPhysContig)
+{
+	PVRSRV_ERROR eError;
+	RA_LENGTH_T uSize = uRequestSize;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+
+	PVR_LOG_RETURN_IF_FALSE(pArena != NULL && uImportMultiplier != 0 && uSize != 0,
+	                        "One of the necessary parameters is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT((uRequestSize & (IMG_PAGE2BYTES64(uiLog2ChunkSize) - 1)) == 0);
+	PVR_LOG_RETURN_IF_FALSE((uRequestSize & (IMG_PAGE2BYTES64(uiLog2ChunkSize) - 1)) == 0,
+	                        "Require uiLog2ChunkSize pow 2 & multiple of uRequestSize",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Enforce these constraints so we can use those bits to handle Ghost bases. */
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure Base Array is large enough for intended allocation */
+	PVR_LOG_RETURN_IF_FALSE(((RA_LENGTH_T)uiBaseArraySize * (IMG_PAGE2BYTES64(uiLog2ChunkSize)) >= uRequestSize),
+		                    "Not enough array space to store alloc bases",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: arena='%s', size=0x%llx(0x%llx), "
+	        "log2ChunkSize=0x%llx", __func__, pArena->name,
+	        (unsigned long long)uSize, (unsigned long long)uRequestSize,
+	        (unsigned long long)uiLog2ChunkSize));
+
+	/* if allocation failed then we might have an import source which
+	   can provide more resource, else we will have to fail the
+	   allocation to the caller. */
+	eError = _AttemptAllocAlignedAssured(pArena,
+	                                      uSize,
+	                                      uiLog2ChunkSize,
+	                                      uFlags,
+	                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                      aBaseArray,
+	                                      bSparseAlloc,
+	                                      bPhysContig);
+	if (eError)
+	{
+		BT *apsBTsStatic[RA_DEFAULT_IMPORTS_BUFFER_SIZE];
+		BT_PTR_ARRAY sBTs = {
+			.uiBTCount = ARRAY_SIZE(apsBTsStatic),
+			.apsBTs = apsBTsStatic,
+		};
+
+		if (eError == PVRSRV_ERROR_RA_OUT_OF_RESOURCE)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA out of resource, attempt to import more if possible:"
+			                         " uSize:0x%llx"
+			                         " uFlags:0x%llx",
+			                         (unsigned long long) uSize,
+			                         (unsigned long long) uFlags));
+		}
+		else if (eError == PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA no resource for flags, attempt to import some if possible:"
+			                         " uSize:0x%llx"
+			                         " uFlags:0x%llx",
+			                         (unsigned long long) uSize,
+			                         (unsigned long long) uFlags));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA Failed to Allocate, could be fragmented, attempt to import"
+			                         " more resource if possible."));
+		}
+
+		eError = _AttemptImportSpanAlloc(pArena,
+		                                 uSize,
+		                                 uImportMultiplier,
+		                                 uFlags,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                 pszAnnotation,
+		                                 &sBTs);
+		if (eError != PVRSRV_OK)
+		{
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			return eError;
+		}
+
+		eError = _AttemptAllocAlignedAssured(pArena,
+		                                      uSize,
+		                                      uiLog2ChunkSize,
+		                                      uFlags,
+		                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                      aBaseArray,
+		                                      bSparseAlloc,
+		                                      bPhysContig);
+		if (eError)
+		{
+			IMG_UINT32 i;
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: name='%s' second alloc failed!",
+					 __func__, pArena->name));
+			/*
+			  On failure of _AttemptAllocAligned() depending on the exact point
+			  of failure, the imported segment may have been used and freed, or
+			  left untouched. If the later, we need to return it.
+			*/
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				_FreeBT(pArena, sBTs.apsBTs[i]);
+			}
+
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+#if defined(DEBUG)
+		/*
+		 * This block of code checks to see if the extra memory we just imported was
+		 * used for the second allocation. If we imported memory but did not use it,
+		 * it indicates there is a bug in the allocation logic. We can still recover by
+		 * freeing the imported span(s) but we emit an error to signal that there is an
+		 * issue.
+		 * */
+		else
+		{
+			IMG_UINT32 bt_index;
+			IMG_UINT32 base_index;
+			IMG_BOOL bBasesInNewSpan = IMG_FALSE;
+
+			for (bt_index = 0; bt_index < sBTs.uiBTCount && !bBasesInNewSpan; bt_index++)
+			{
+				BT *pBT = sBTs.apsBTs[bt_index];
+				for (base_index = 0; base_index < uiBaseArraySize; base_index++)
+				{
+					RA_BASE_T uiBase = RA_BASE_STRIP_GHOST_BIT(aBaseArray[base_index]);
+
+					/* If the base hasn't been allocated then skip it */
+					if (aBaseArray[base_index] == INVALID_BASE_ADDR)
+					{
+						continue;
+					}
+
+					if (uiBase >= pBT->base &&
+					    uiBase <= pBT->base + pBT->uSize)
+					{
+						bBasesInNewSpan = IMG_TRUE;
+						break;
+					}
+				}
+			}
+			if (!bBasesInNewSpan)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: name='%s' alloc did not occur in the imported span!",
+						__func__, pArena->name));
+				/* Remove the imported spans which should not be in use (if it is then that is okay).
+				 * No spans should exist and not be used unless added through the import multiplier. */
+				for (bt_index = 0; bt_index < sBTs.uiBTCount; bt_index++)
+				{
+					_FreeBT(pArena, sBTs.apsBTs[bt_index]);
+				}
+			}
+		}
+#endif
+		if (sBTs.apsBTs != apsBTsStatic)
+		{
+			OSFreeMem(sBTs.apsBTs);
+		}
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_AllocMulti(RA_ARENA *pArena,
+              RA_LENGTH_T uRequestSize,
+              IMG_UINT32 uiLog2ChunkSize,
+              IMG_UINT8 uImportMultiplier,
+              RA_FLAGS_T uImportFlags,
+              const IMG_CHAR *pszAnnotation,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+              IMG_BOOL *bPhysContig)
+{
+	PVRSRV_ERROR eError;
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_AllocMultiUnlocked(pArena,
+	                                 uRequestSize,
+	                                 uiLog2ChunkSize,
+	                                 uImportMultiplier,
+	                                 uImportFlags,
+	                                 pszAnnotation,
+	                                 aBaseArray,
+	                                 uiBaseArraySize,
+	                                 IMG_FALSE, /* Sparse alloc */
+	                                 bPhysContig);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_AllocMultiSparse(RA_ARENA *pArena,
+                     IMG_UINT32 uiLog2ChunkSize,
+                     IMG_UINT8 uImportMultiplier,
+                     RA_FLAGS_T uImportFlags,
+                     const IMG_CHAR *pszAnnotation,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                     IMG_UINT32 *puiAllocIndices,
+                     IMG_UINT32 uiAllocCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bPhysContig;
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+
+	/*
+	 * In this case the arguments given show the allocation is
+	 * sparse but has no specific indices, this indicates
+	 * we want to populate the full aBaseArray
+	 */
+	if (puiAllocIndices == NULL)
+	{
+		RA_LENGTH_T uRequestSize = (RA_LENGTH_T) uiAllocCount << uiLog2ChunkSize;
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 uRequestSize,
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 aBaseArray,
+		                                 uiBaseArraySize,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_IF_ERROR(eError, "RA_AllocMulti");
+		OSLockRelease(pArena->hLock);
+		return eError;
+	}
+
+	/*
+	 * This case is optimised for single allocations as we can skip
+	 * some of the iteration logic in the full allocation path.
+	 */
+	if (uiAllocCount == 1)
+	{
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 &aBaseArray[puiAllocIndices[0]],
+		                                 1,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_IF_ERROR(eError, "RA_AllocMulti");
+		OSLockRelease(pArena->hLock);
+		return eError;
+	}
+
+	/*
+	 * By consolidating / grouping the indices given we can perform sparse allocations
+	 * in blocks, this has the effect of reducing fragmentation and creating optimal free
+	 * scenarios. Free can be performed in blocks rather than a chunk at a time, this reduces
+	 * the amount of BT merging cycles we perform.
+	 */
+	for (i = 0; i < uiAllocCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		for (j = i;
+		     j + 1 != uiAllocCount &&
+		     puiAllocIndices[j + 1] == puiAllocIndices[j] + 1;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 IMG_PAGES2BYTES64(uiConsolidate, uiLog2ChunkSize),
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 &aBaseArray[puiAllocIndices[i]],
+		                                 uiConsolidate,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RA_AllocMulti", unwind_alloc);
+		i += uiConsolidate;
+	}
+
+	OSLockRelease(pArena->hLock);
+	return PVRSRV_OK;
+
+unwind_alloc:
+	if (i != 0)
+	{
+		PVRSRV_ERROR eFreeError;
+		eFreeError = _RA_FreeMultiUnlockedSparse(pArena,
+		                                          aBaseArray,
+		                                          uiBaseArraySize,
+		                                          IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                          puiAllocIndices,
+		                                          &i);
+		PVR_LOG_IF_ERROR(eFreeError, "_RA_FreeMultiUnlockedSparse");
+	}
+
+	OSLockRelease(pArena->hLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RA_Find_BT_VARange
+@Description    To find the boundary tag associated with the given device
+                                  virtual address.
+@Input          pArena            The arena
+@input          base              Allocated base resource
+@Input          uRequestSize      The size of resource segment requested.
+@Input          uImportFlags            Flags influencing allocation policy.
+@Return         Boundary Tag - success, NULL on failure
+*/ /**************************************************************************/
+static BT *RA_Find_BT_VARange(RA_ARENA *pArena,
+                              RA_BASE_T base,
+                              RA_LENGTH_T uRequestSize,
+                              RA_FLAGS_T uImportFlags)
+{
+	IMG_PSPLAY_TREE psSplaynode;
+	IMG_UINT32 uIndex;
+
+	/* Find the splay node associated with these import flags */
+	psSplaynode = PVRSRVFindNode(uImportFlags, pArena->per_flags_buckets);
+
+	if (psSplaynode == NULL)
+	{
+		return NULL;
+	}
+
+	uIndex = pvr_log2(uRequestSize);
+
+	/* Find the free Boundary Tag from the bucket that holds the requested range */
+	while (uIndex < FREE_TABLE_LIMIT)
+	{
+		BT *pBT = psSplaynode->buckets[uIndex];
+
+		while (pBT)
+		{
+			if ((pBT->base <= base) && ((pBT->base + pBT->uSize) >= (base + uRequestSize)))
+			{
+				if (pBT->type == btt_free)
+				{
+					return pBT;
+				}
+				else
+				{
+					PVR_ASSERT(pBT->type == btt_free);
+				}
+			}
+			else{
+				pBT = pBT->next_free;
+			}
+		}
+
+#if defined(PVR_CTZLL)
+		/* This could further be optimised to get the next valid bucket */
+		while (!(psSplaynode->bHasEltsMapping & (1ULL << ++uIndex)));
+#else
+		uIndex++;
+#endif
+	}
+
+	return NULL;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_Alloc_Range(RA_ARENA *pArena,
+               RA_LENGTH_T uRequestSize,
+               RA_FLAGS_T uImportFlags,
+               RA_LENGTH_T uAlignment,
+               RA_BASE_T base,
+               RA_LENGTH_T *pActualSize)
+{
+	RA_LENGTH_T uSize = uRequestSize;
+	BT *pBT = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (pArena == NULL || uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: One of the necessary parameters is 0", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	/* Align the requested size to the Arena Quantum */
+	uSize = PVR_ALIGN(uSize, pArena->uQuantum);
+
+	/* Must be a power of 2 or 0 */
+	PVR_ASSERT((uAlignment == 0) || (uAlignment & (uAlignment - 1)) == 0);
+
+	if (uAlignment > 1)
+	{
+		if (base != PVR_ALIGN(base, uAlignment))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, unlock_);
+		}
+	}
+
+	/* Find if the segment in the range exists and is free
+	 * Check if the segment can be split
+	 * Find the bucket that points to this segment
+	 * Find the free segment is in the free list
+	 * remove the free segment
+	 * split the segment into three segments one prior free, alloc range,
+	 *     free segment after the range.
+	 * remove the allocated range segment from the free list
+	 * hook up the prior and after segments back to free list
+	 * For each free, find the bucket the segment should go to
+	 */
+
+	pBT = RA_Find_BT_VARange(pArena, base, uSize, uImportFlags);
+
+	if (pBT == NULL)
+	{
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL,
+		                    unlock_);
+	}
+
+	/* Remove the boundary tag from the free list */
+	_FreeListRemove (pArena, pBT);
+
+	/* if requested VA start in the middle of the BT, split the BT accordingly */
+	if (base > pBT->base)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit (pBT, (RA_LENGTH_T)(base - pBT->base));
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			/* Put pBT back in the list */
+			_FreeListInsert (pArena, pBT);
+			PVR_LOG_GOTO_WITH_ERROR("_SegmentSplit (1)", eError,
+			                        PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL,
+			                        unlock_);
+		}
+
+		/* Insert back the free BT to the free list */
+		_FreeListInsert(pArena, pBT);
+		pBT = pNeighbour;
+	}
+
+	/* the segment might be too big, if so, discard the back of the segment */
+	if (pBT->uSize > uSize)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, uSize);
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			/* Put pBT back in the list */
+			_FreeListInsert (pArena, pBT);
+			PVR_LOG_GOTO_WITH_ERROR("_SegmentSplit (2)", eError,
+			                        PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL,
+			                        unlock_);
+		}
+
+		/* Insert back the free BT to the free list */
+		_FreeListInsert (pArena, pNeighbour);
+	}
+
+	pBT->type = btt_live;
+
+	if (!HASH_Insert_Extended (pArena->pSegmentHash, &base, (uintptr_t)pBT))
+	{
+		_FreeBT (pArena, pBT);
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED,
+		                    unlock_);
+	}
+
+	if (pActualSize != NULL)
+	{
+		*pActualSize = uSize;
+	}
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+unlock_:
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+IMG_INTERNAL void
+RA_Free(RA_ARENA *pArena, RA_BASE_T base)
+{
+	BT *pBT;
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', base=0x%llx", __func__, pArena->name,
+	        (unsigned long long)base));
+
+	pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &base);
+	PVR_ASSERT(pBT != NULL);
+
+	if (pBT)
+	{
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		PVR_ASSERT(pBT->base == base);
+		_FreeBT(pArena, pBT);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: no resource span found for given base (0x%llX) in arena %s",
+		         __func__, (unsigned long long) base, pArena->name));
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+	OSLockRelease(pArena->hLock);
+}
+
+static PVRSRV_ERROR
+_RA_FreeMultiUnlocked(RA_ARENA *pArena,
+                       RA_BASE_ARRAY_T aBaseArray,
+                       RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	PVRSRV_ERROR eError;
+
+	/* Free the whole array */
+	if (uiBaseArraySize == 1)
+	{
+		eError = _FreeSingleBaseArray(pArena, aBaseArray, uiBaseArraySize);
+		PVR_LOG_IF_ERROR(eError, "_FreeSingleBaseArray");
+	}
+	else
+	{
+		eError = _FreeMultiBaseArray(pArena, aBaseArray, uiBaseArraySize);
+		PVR_LOG_IF_ERROR(eError, "_FreeMultiBaseArray");
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_FreeMulti(RA_ARENA *pArena,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	PVRSRV_ERROR eError;
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_FreeMultiUnlocked(pArena,
+	                                aBaseArray,
+	                                uiBaseArraySize);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_RA_FreeMultiUnlockedSparse(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 *puiFreeIndices,
+                             IMG_UINT32 *puiFreeCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiFreeCount = *puiFreeCount;
+	*puiFreeCount = 0;
+
+	/* Handle case where we only have 1 base to free. */
+	if (uiFreeCount == 1)
+	{
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiFreeIndices[0],
+		                                      1,
+		                                      CONVERT_AND_FREE);
+		PVR_LOG_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+		if (eError == PVRSRV_OK)
+		{
+			*puiFreeCount = uiFreeCount;
+		}
+		return eError;
+	}
+
+	for (i = 0; i < uiFreeCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[i]));
+
+		for (j = i;
+		     puiFreeIndices[j + 1] == puiFreeIndices[j] + 1 &&
+		     RA_BASE_IS_GHOST(aBaseArray[puiFreeIndices[j + 1]]) &&
+		     j + 1 < uiFreeCount;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiFreeIndices[i],
+		                                      uiConsolidate,
+		                                      CONVERT_AND_FREE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+
+		i += uiConsolidate;
+		*puiFreeCount += uiConsolidate;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_FreeMultiSparse(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                    IMG_UINT32 uiLog2ChunkSize,
+                    IMG_UINT32 *puiFreeIndices,
+                    IMG_UINT32 *puiFreeCount)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(puiFreeCount != NULL,
+		                        "puiFreeCount Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure Base Array is large enough for intended free */
+	PVR_LOG_RETURN_IF_FALSE(uiBaseArraySize >= *puiFreeCount,
+		                        "Attempt to free more bases than array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(puiFreeIndices != NULL,
+		                        "puiFreeIndices Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_FreeMultiUnlockedSparse(pArena,
+	                                      aBaseArray,
+	                                      uiBaseArraySize,
+	                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                      puiFreeIndices,
+	                                      puiFreeCount);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/*************************************************************************/ /*!
+ *  @Function   _RA_RealiseMultiSparseIndicesUnlocked
+ *
+ *  @Description   Given an array of indices to extract, prepares the base
+ *                 array so that the indices in puiExtractIndices can be
+ *                 moved to another base array.
+ *                 Called when some pages of the base array need to be
+ *                 transferred to another base array. As a result of this call,
+ *                 some ghost addresses in aBaseArray might be converted to
+ *                 real addresses.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *  @Input  uiChunkSize -  The Base chunk size used to generate ghost entries
+ *  @Input  puiExtractIndices - Array of indices to extract
+ *  @Input  puiExtractCount - Number of indices to extract
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_RA_RealiseMultiSparseIndicesUnlocked(RA_ARENA *pArena,
+                                      RA_BASE_ARRAY_T aBaseArray,
+                                      RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                                      RA_LENGTH_T uiChunkSize,
+                                      IMG_UINT32 *puiExtractIndices,
+                                      IMG_UINT32 *puiExtractCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiExtractCount = *puiExtractCount;
+	*puiExtractCount = 0;
+
+	/* Handle case where we only have 1 base to extract. */
+	if (uiExtractCount == 1)
+	{
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiExtractIndices[0],
+		                                      1,
+		                                      CONVERT_DONT_FREE);
+		PVR_LOG_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+		if (eError == PVRSRV_OK)
+		{
+			*puiExtractCount = uiExtractCount;
+		}
+		return eError;
+	}
+
+	for (i = 0; i < uiExtractCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[i]));
+
+		for (j = i;
+		     puiExtractIndices[j + 1] == puiExtractIndices[j] + 1 &&
+		     RA_BASE_IS_GHOST(aBaseArray[puiExtractIndices[j + 1]]) &&
+		     j + 1 < uiExtractCount;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiExtractIndices[i],
+		                                      uiConsolidate,
+		                                      CONVERT_DONT_FREE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+
+		i += uiConsolidate;
+		*puiExtractCount += uiConsolidate;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_TransferMultiSparseIndices(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aSrcBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiSrcBaseArraySize,
+                              RA_BASE_ARRAY_T aDstBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiDstBaseArraySize,
+                              IMG_UINT32 uiLog2ChunkSize,
+                              IMG_UINT32 *puiTransferIndices,
+                              IMG_UINT32 *puiTransferCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_FALSE(puiTransferCount != NULL,
+		                        "puiTransferCount Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure source Base Array is large enough for intended extract */
+	PVR_LOG_RETURN_IF_FALSE(uiSrcBaseArraySize >= *puiTransferCount,
+		                        "Attempt to transfer more bases than src array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure dst Base Array is large enough for intended extract */
+	PVR_LOG_RETURN_IF_FALSE(uiDstBaseArraySize <= *puiTransferCount,
+		                        "Attempt to transfer more bases than dst array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(puiTransferIndices != NULL,
+		                        "puiTransferIndices Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	/* First prepare the base array for subsequent transfer */
+	eError = _RA_RealiseMultiSparseIndicesUnlocked(pArena,
+	                                               aSrcBaseArray,
+	                                               uiSrcBaseArraySize,
+	                                               IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                               puiTransferIndices,
+	                                               puiTransferCount);
+	OSLockRelease(pArena->hLock);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	/* Now do the transfer */
+	for (i=0; i<*puiTransferCount; i++)
+	{
+		IMG_UINT32 idxSrc = puiTransferIndices[i];
+		aDstBaseArray[i] = aSrcBaseArray[idxSrc];
+		aSrcBaseArray[idxSrc] = INVALID_BASE_ADDR;
+	}
+
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_TrimBlockMakeReal(RA_ARENA *pArena,
+                   RA_BASE_ARRAY_T aBaseArray,
+                   RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                   IMG_UINT32 uiLog2ChunkSize,
+                   IMG_UINT32 uiStartIndex,
+                   IMG_UINT32 uiEndIndex)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RA_BASE_T sRealBase;
+	IMG_UINT32 uiRealBaseIndex;
+
+	/* Note: Error return paths in this function do not require unwinding.
+	 * Free logic is performed based upon indices and detection of Real base regions,
+	 * performance wise it would be more costly to unwind the conversion here than to
+	 * just free a smaller Real base region.
+	 */
+
+	/* Check Start index is real, if not make it real */
+	if (RA_BASE_IS_GHOST(aBaseArray[uiStartIndex]))
+	{
+		_FindRealBaseFromGhost(aBaseArray,
+		                       uiStartIndex,
+		                       &sRealBase,
+		                       &uiRealBaseIndex);
+
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 sRealBase,
+		                                 uiRealBaseIndex,
+		                                 uiStartIndex,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize));
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	/* Check end +1 is real or end of array , if ghost make real */
+	if (uiEndIndex + 1 != uiBaseArraySize &&
+	    RA_BASE_IS_GHOST(aBaseArray[uiEndIndex + 1]))
+	{
+		_FindRealBaseFromGhost(aBaseArray,
+		                       uiEndIndex + 1,
+		                       &sRealBase,
+		                       &uiRealBaseIndex);
+
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 sRealBase,
+		                                 uiRealBaseIndex,
+		                                 uiEndIndex + 1,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize));
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_SwapSparseMem(RA_ARENA *pArena,
+                  RA_BASE_ARRAY_T aBaseArray,
+                  RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                  IMG_UINT32 uiLog2ChunkSize,
+                  IMG_UINT32 *puiXIndices,
+                  IMG_UINT32 *puiYIndices,
+                  IMG_UINT32 uiSwapCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiSwapped = 0;
+	IMG_UINT32 uiStartIndex;
+	/* Consolidation values counting the bases after the start index*/
+	IMG_UINT32 uiXConsol;
+	IMG_UINT32 uiYConsol;
+	/* Consolidation limit, the smallest consecutive indices between the
+	 * two inputs
+	 */
+	IMG_UINT32 uiConsolidateLimit;
+	IMG_UINT32 uiTotalSwapCount;
+	IMG_UINT32 i;
+
+	/*
+	 * The algorithm below aims to swap the desired indices whilst also
+	 * maintaining a maximum contiguity of allocation blocks where possible.
+	 * It does this by:
+	 * Consolidating the contiguous indices of X and Y.
+	 * Selecting the smallest of these consolidations as a range to swap in a block.
+	 * Trim both block ranges using the indices range to ensure that Real bases are
+	 * created to represent regions that have been split due to the indices.
+	 * Perform the swap and update the swapped count ready for the next iteration.
+	 * Note: Maintaining contiguity improves performance of free logic for sparse
+	 * allocations because we can free in regions rather than chunks.
+	 */
+	while (uiSwapped != uiSwapCount)
+	{
+		IMG_UINT32 x, y;
+		uiTotalSwapCount = 1;
+		uiStartIndex = uiSwapped;
+		uiXConsol = 0;
+		uiYConsol = 0;
+
+		/* Calculate contiguous indices at X */
+		for (x = uiStartIndex;
+		     x < uiSwapCount &&
+		     puiXIndices[x] + 1 == puiXIndices[x + 1];
+		     x++)
+		{
+			uiXConsol++;
+		}
+
+		/* Calculate contiguous indices at Y */
+		for (y = uiStartIndex;
+		     y < uiSwapCount &&
+		     puiYIndices[y] + 1 == puiYIndices[y + 1];
+		     y++)
+		{
+			uiYConsol++;
+		}
+
+		/* Find lowest consolidation value */
+		uiConsolidateLimit = (uiXConsol < uiYConsol) ? uiXConsol : uiYConsol;
+
+		/* Perform RealBase translation where required */
+		eError = _TrimBlockMakeReal(pArena,
+		                            aBaseArray,
+		                            uiBaseArraySize,
+		                            uiLog2ChunkSize,
+		                            puiXIndices[uiStartIndex],
+		                            puiXIndices[uiStartIndex + uiConsolidateLimit]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_TrimBlockMakeReal", unwind);
+
+		eError = _TrimBlockMakeReal(pArena,
+		                            aBaseArray,
+		                            uiBaseArraySize,
+		                            uiLog2ChunkSize,
+		                            puiYIndices[uiStartIndex],
+		                            puiYIndices[uiStartIndex + uiConsolidateLimit]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_TrimBlockMakeReal", unwind);
+
+		uiTotalSwapCount += uiConsolidateLimit;
+		uiSwapped += uiTotalSwapCount;
+		i = uiStartIndex;
+
+		do
+		{
+			SWAP(aBaseArray[puiXIndices[i]], aBaseArray[puiYIndices[i]]);
+			uiTotalSwapCount--;
+			i++;
+		}
+		while (uiTotalSwapCount != 0);
+	}
+
+	return PVRSRV_OK;
+
+unwind:
+	/* If we hit an error when Trimming we should revert the swapping
+	 * that has already been performed.
+	 */
+	for (i = 0; i < uiSwapped; i++)
+	{
+		SWAP(aBaseArray[puiXIndices[i]], aBaseArray[puiYIndices[i]]);
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL void
+RA_Get_Usage_Stats(RA_ARENA *pArena, PRA_USAGE_STATS psRAStats)
+{
+	psRAStats->ui64TotalArenaSize = pArena->ui64TotalArenaSize;
+	psRAStats->ui64FreeArenaSize = pArena->ui64FreeArenaSize;
+}
+
+IMG_INTERNAL IMG_CHAR *
+RA_GetArenaName(RA_ARENA *pArena)
+{
+	return pArena->name;
+}
+
+/* #define _DBG(...) PVR_LOG((__VA_ARGS__)) */
+#define _DBG(...)
+
+IMG_INTERNAL RA_ARENA_ITERATOR *
+RA_IteratorAcquire(RA_ARENA *pArena, IMG_BOOL bIncludeFreeSegments)
+{
+	RA_ARENA_ITERATOR *pIter = OSAllocMem(sizeof(*pIter));
+	PVR_LOG_RETURN_IF_FALSE(pIter != NULL, "OSAllocMem", NULL);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+
+	pIter->pArena = pArena;
+	pIter->bIncludeFreeSegments = bIncludeFreeSegments;
+
+	RA_IteratorReset(pIter);
+
+	return pIter;
+}
+
+IMG_INTERNAL void
+RA_IteratorRelease(RA_ARENA_ITERATOR *pIter)
+{
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		return;
+	}
+
+	OSLockRelease(pIter->pArena->hLock);
+
+	OSFreeMem(pIter);
+}
+
+IMG_INTERNAL void
+RA_IteratorReset(RA_ARENA_ITERATOR *pIter)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	pNext = pIter->pArena->pHeadSegment;
+
+	/* find next element if we're not including the free ones */
+	if (!pIter->bIncludeFreeSegments)
+	{
+		while (pNext != NULL && pNext->type != btt_live)
+		{
+			_DBG("(%s()) skipping segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+			     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+			     pNext->type);
+			pNext = pNext->pNextSegment;
+		}
+	}
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	/* if bIncludeFreeSegments then pNext here is either a valid pointer to
+	 * "live" segment or NULL and if !bIncludeFreeSegments then it's either
+	 * a valid pointer to any next segment or NULL */
+	pIter->pCurrent = pNext;
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNext(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	if (pIter->pCurrent == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pNext = pIter->pCurrent;
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+	     pNext->type);
+
+	pData->uiAddr = pIter->pCurrent->base;
+	pData->uiSize = pIter->pCurrent->uSize;
+	pData->bFree = pIter->pCurrent->type == btt_free;
+
+	/* combine contiguous segments */
+	while ((pNext = pNext->pNextSegment) != NULL &&
+	        pNext->type == pNext->pPrevSegment->type &&
+	        pNext->type == btt_live &&
+	        pNext->base == pData->uiAddr + pData->uiSize)
+	{
+		_DBG("(%s()) combining segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+		     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+		     pNext->type);
+		pData->uiSize += pNext->uSize;
+	}
+
+	/* advance to next */
+	if (!pIter->bIncludeFreeSegments)
+	{
+		while (pNext != NULL && pNext->type != btt_live)
+		{
+			_DBG("(%s()) skipping segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+			     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+			     pNext->type);
+			pNext = pNext->pNextSegment;
+		}
+	}
+
+	_DBG("(%s()) next segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	/* if bIncludeFreeSegments then pNext here is either a valid pointer to
+	 * "live" segment or NULL and if !bIncludeFreeSegments then it's either
+	 * a valid pointer to any next segment or NULL */
+	pIter->pCurrent = pNext;
+
+	return IMG_TRUE;
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNextSpan(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	/* We have no more iterations to perform */
+	if (pIter->pCurrent == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pNext = pIter->pCurrent;
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+	     pNext->type);
+
+	/* Extract details of current span */
+	pData->uiAddr = pIter->pCurrent->base;
+	pData->uiSize = pIter->pCurrent->uSize;
+	pData->bFree = pIter->pCurrent->free_import;
+
+	/* If this BT is a full span, can occur if we have no allocations
+	 * on a span.
+	 */
+	if (pNext->is_leftmost == 1 &&
+	    pNext->is_rightmost == 1)
+	{
+		pNext = pNext->pNextSegment;
+	}
+	else
+	{
+		/* combine contiguous segments of span, these are allocations
+		 * and free segments in a span.
+		 */
+		while ((pNext = pNext->pNextSegment) != NULL &&
+		        pNext->is_leftmost == 0)
+		{
+			_DBG("(%s()) combining segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+				 "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+				 pNext->type);
+			pData->uiSize += pNext->uSize;
+		}
+	}
+
+	_DBG("(%s()) next segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	pIter->pCurrent = pNext;
+
+	return IMG_TRUE;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_BlockDump(RA_ARENA *pArena, void (*pfnLogDump)(void*, IMG_CHAR*, ...), void *pPrivData)
+{
+	RA_ARENA_ITERATOR *pIter = NULL;
+	RA_ITERATOR_DATA sIterData;
+	const IMG_UINT32 uiLineWidth = 64;
+
+	IMG_UINT32 **papRegionArray = NULL;
+	IMG_UINT32 uiRegionCount = 0;
+
+	const IMG_UINT32 uiChunkSize = 32; /* 32-bit chunks */
+	const IMG_UINT32 uiChunkCount = (uiLineWidth / uiChunkSize) * 2; /* This should equal 2 or a multiple of 2 */
+	const IMG_UINT32 uiRegionSize = uiChunkSize * uiChunkCount;
+
+	IMG_UINT32 uiRecognisedQuantum = 0;
+
+	IMG_UINT64 uiLastBase = 0;
+	IMG_UINT64 uiLastSize = 0;
+
+	IMG_UINT32 i;
+	IMG_UINT32 uiRemainder;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	IMG_UINT64 uiLargestFreeSegmentSize = 0;
+	IMG_UINT32 uiFragPercentage = 0;
+
+	/* -- papRegionArray Structure --
+	 *  papRegionArray Indexes
+	 *  |         Chunk 0      Chunk 1      Chunk 2      Chunk 3
+	 *  v     |------------|------------|------------|------------|
+	 * [0] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 | -- |
+	 * [1] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [2] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [3] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    | Regions
+	 * [4] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [5] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [6] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 | -- |
+	 * ...
+	 */
+
+	if (pArena == NULL || pfnLogDump == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pIter = RA_IteratorAcquire(pArena, IMG_TRUE);
+	PVR_LOG_RETURN_IF_NOMEM(pIter, "RA_IteratorAcquire");
+
+	uiRecognisedQuantum = pArena->uQuantum > 0 ? pArena->uQuantum : 4096;
+
+	while (RA_IteratorNext(pIter, &sIterData))
+	{
+		if (!sIterData.bFree && sIterData.uiAddr >= uiLastBase)
+		{
+			uiLastBase = sIterData.uiAddr;
+			uiLastSize = sIterData.uiSize;
+		}
+	}
+
+	uiRegionCount = OSDivide64(uiLastBase + uiLastSize, uiRecognisedQuantum,
+	                           &uiRemainder);
+	uiRegionCount = OSDivide64(uiRegionCount, uiRegionSize, &uiRemainder);
+	if (uiRemainder != 0 || uiRegionCount == 0)
+	{
+		uiRegionCount += 1;
+	}
+
+	papRegionArray = OSAllocZMem(sizeof(IMG_UINT32*) * uiRegionCount);
+	PVR_LOG_GOTO_IF_NOMEM(papRegionArray, eError, cleanup_array);
+
+	RA_IteratorReset(pIter);
+
+	while (RA_IteratorNext(pIter, &sIterData))
+	{
+		IMG_UINT64 uiDataDivRecQuant;
+
+		IMG_UINT32 uiAddrRegionIdx = 0;
+		IMG_UINT32 uiAddrRegionOffset = 0;
+		IMG_UINT32 uiAddrChunkIdx = 0;
+		IMG_UINT32 uiAddrChunkOffset = 0;
+		IMG_UINT32 uiAddrChunkShift; /* The bit-shift needed to fill the chunk */
+
+		IMG_UINT32 uiQuantisedSize;
+		IMG_UINT32 uiQuantisedSizeMod;
+		IMG_UINT32 uiAllocLastRegionIdx = 0; /* The last region that this alloc appears in */
+		IMG_UINT32 uiAllocChunkSize = 0; /* The number of chunks this alloc spans */
+
+		IMG_INT32 iBitSetCount = 0;
+		IMG_INT32 iOverflowCheck = 0;
+		IMG_INT32 iOverflow = 0;
+		IMG_UINT32 uiRegionIdx = 0;
+		IMG_UINT32 uiChunkIdx = 0;
+
+		/* If the current data is for a free block, use it to track largest
+		 * contiguous free segment size.
+		 */
+		if (sIterData.bFree && sIterData.uiSize > uiLargestFreeSegmentSize)
+		{
+			uiLargestFreeSegmentSize = sIterData.uiSize;
+			continue;
+		}
+
+		uiDataDivRecQuant = OSDivide64(sIterData.uiAddr, uiRecognisedQuantum,
+		                               &uiRemainder);
+		uiAddrRegionIdx = OSDivide64(uiDataDivRecQuant, uiRegionSize,
+		                             &uiAddrRegionOffset);
+		uiQuantisedSize = OSDivide64(sIterData.uiSize, uiRecognisedQuantum,
+		                             &uiQuantisedSizeMod);
+
+		uiAddrChunkIdx = uiAddrRegionOffset / uiChunkSize;
+		uiAddrChunkOffset = uiAddrRegionOffset % uiChunkSize;
+		uiAddrChunkShift = uiChunkSize - uiAddrChunkOffset;
+		uiRegionIdx = uiAddrRegionIdx;
+		uiChunkIdx = uiAddrChunkIdx;
+
+		if ((uiQuantisedSize == 0) || (uiQuantisedSizeMod != 0))
+		{
+			uiQuantisedSize += 1;
+		}
+
+		uiAllocLastRegionIdx = OSDivide64(uiDataDivRecQuant + uiQuantisedSize - 1,
+		                                  uiRegionSize, &uiRemainder);
+		uiAllocChunkSize = (uiAddrChunkOffset + uiQuantisedSize) / uiChunkSize;
+
+		if ((uiAddrChunkOffset + uiQuantisedSize) % uiChunkSize > 0)
+		{
+			uiAllocChunkSize += 1;
+		}
+
+		iBitSetCount = uiQuantisedSize;
+		iOverflowCheck = uiQuantisedSize - uiAddrChunkShift;
+
+		if (iOverflowCheck > 0)
+		{
+			iOverflow = iOverflowCheck;
+			iBitSetCount = uiQuantisedSize - iOverflow;
+		}
+
+		/**
+		 * Allocate memory to represent the chunks for each region the allocation
+		 * spans. If one was already allocated before don't do it again.
+		 */
+		for (i = 0; uiAddrRegionIdx + i <= uiAllocLastRegionIdx; i++)
+		{
+			if (papRegionArray[uiAddrRegionIdx + i] == NULL)
+			{
+				papRegionArray[uiAddrRegionIdx + i] = OSAllocZMem(sizeof(IMG_UINT32) * uiChunkCount);
+				PVR_LOG_GOTO_IF_NOMEM(papRegionArray[uiAddrRegionIdx + i], eError, cleanup_regions);
+			}
+		}
+
+		for (i = 0; i < uiAllocChunkSize; i++)
+		{
+			if (uiChunkIdx >= uiChunkCount)
+			{
+				uiRegionIdx++;
+				uiChunkIdx = 0;
+			}
+
+			if ((IMG_UINT32)iBitSetCount != uiChunkSize)
+			{
+				IMG_UINT32 uiBitMask = 0;
+
+				uiBitMask = (1U << iBitSetCount) - 1;
+				uiBitMask <<= (uiAddrChunkShift - iBitSetCount);
+
+				papRegionArray[uiRegionIdx][uiChunkIdx] |= uiBitMask;
+			}
+			else
+			{
+				papRegionArray[uiRegionIdx][uiChunkIdx] |= 0xFFFFFFFF;
+			}
+
+			uiChunkIdx++;
+			iOverflow -= uiChunkSize;
+			iBitSetCount = iOverflow >= 0 ? uiChunkSize : uiChunkSize + iOverflow;
+			if (iOverflow < 0)
+			{
+				uiAddrChunkShift = 32;
+			}
+		}
+	}
+	if (pArena->ui64FreeArenaSize && uiLargestFreeSegmentSize)
+	{
+		/* N.B This can look strange in a dual RA when comparing to the dump visualisation
+		 * as spans that are freed are not included in the segment list, regardless it is
+		 * an accurate representation for the spans in the Arena.
+		 */
+		uiFragPercentage = OSDivide64(100 * pArena->ui64FreeArenaSize,
+		                              pArena->ui64FreeArenaSize + uiLargestFreeSegmentSize,
+		                              &uiRemainder);
+	}
+
+	pfnLogDump(pPrivData, "~~~ '%s' Resource Arena Block Dump", pArena->name);
+	pfnLogDump(pPrivData, "    Block Size: %uB", uiRecognisedQuantum);
+	pfnLogDump(pPrivData,
+	           "    Span Memory Usage: %"IMG_UINT64_FMTSPEC"B"
+	           "    Free Span Memory: %"IMG_UINT64_FMTSPEC"B"
+	           "    Largest Free Region Size: %"IMG_UINT64_FMTSPEC"B"
+	           "    Percent Fragmented %u%%",
+	           pArena->ui64TotalArenaSize,
+	           pArena->ui64FreeArenaSize,
+	           uiLargestFreeSegmentSize,
+	           uiFragPercentage);
+	pfnLogDump(pPrivData,
+	           "===============================================================================");
+
+	for (i = 0; i < uiRegionCount; i++)
+	{
+		static IMG_BOOL bEmptyRegion = IMG_FALSE;
+		if (papRegionArray[i] != NULL)
+		{
+			IMG_CHAR pszLine[65];
+			IMG_UINT32 j;
+
+			bEmptyRegion = IMG_FALSE;
+			pszLine[64] = '\0';
+
+			for (j = 0; j < uiChunkCount; j+=2)
+			{
+				IMG_UINT8 uiBit = 0;
+				IMG_UINT32 k;
+				IMG_UINT64 uiLineAddress =
+				    (i * uiRegionSize + (j >> 1) * uiLineWidth) * uiRecognisedQuantum;
+
+				/**
+				 * Move through each of the 32 bits in the chunk and check their
+				 * value. If it is 1 we set the corresponding character to '#',
+				 * otherwise it is set to '.' representing empty space
+				 */
+				for (k = 1 << 31; k != 0; k >>= 1)
+				{
+					pszLine[uiBit] = papRegionArray[i][j] & k ? '#' : '.';
+					pszLine[32 + uiBit] = papRegionArray[i][j+1] & k ? '#' : '.';
+					uiBit++;
+				}
+
+				pfnLogDump(pPrivData,
+				           "| 0x%08"IMG_UINT64_FMTSPECx" | %s",
+				           uiLineAddress,
+				           pszLine);
+			}
+			OSFreeMem(papRegionArray[i]);
+		}
+		else
+		{
+			/* We only print this once per gap of n regions */
+			if (!bEmptyRegion)
+			{
+				pfnLogDump(pPrivData, "     ....");
+				bEmptyRegion = IMG_TRUE;
+			}
+		}
+	}
+
+	RA_IteratorRelease(pIter);
+
+	OSFreeMem(papRegionArray);
+	return eError;
+
+cleanup_regions:
+	for (i = 0; i < uiRegionCount; i++)
+	{
+		if (papRegionArray[i] != NULL)
+		{
+			OSFreeMem(papRegionArray[i]);
+		}
+	}
+
+cleanup_array:
+	OSFreeMem(papRegionArray);
+	RA_IteratorRelease(pIter);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/sync.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/sync.c
new file mode 100644
index 000000000000..3b6e8914d446
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/sync.c
@@ -0,0 +1,822 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements client side code for services synchronisation
+                interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "client_sync_bridge.h"
+#include "client_synctracking_bridge.h"
+#include "info_page_client.h"
+#include "pvr_bridge.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pvr_debug.h"
+#include "dllist.h"
+#include "sync.h"
+#include "sync_internal.h"
+#include "lock.h"
+#include "log2.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+
+#define SYNC_BLOCK_LIST_CHUNCK_SIZE	10
+
+/*
+	This defines the maximum amount of synchronisation memory
+	that can be allocated per SyncPrim context.
+	In reality this number is meaningless as we would run out
+	of synchronisation memory before we reach this limit, but
+	we need to provide a size to the span RA.
+ */
+#define MAX_SYNC_MEM				(4 * 1024 * 1024)
+
+/* forward declaration */
+static PVRSRV_ERROR
+_SyncPrimSetValue(SYNC_PRIM *psSyncInt, IMG_UINT32 ui32Value);
+
+/*
+	Internal interfaces for management of SYNC_PRIM_CONTEXT
+ */
+static void
+_SyncPrimContextUnref(SYNC_PRIM_CONTEXT *psContext)
+{
+	if (!OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: context already freed", __func__));
+	}
+	else if (0 == OSAtomicDecrement(&psContext->hRefCount))
+	{
+		/* SyncPrimContextDestroy only when no longer referenced */
+		RA_Delete(psContext->psSpanRA);
+		RA_Delete(psContext->psSubAllocRA);
+		OSFreeMem(psContext);
+	}
+}
+
+static void
+_SyncPrimContextRef(SYNC_PRIM_CONTEXT *psContext)
+{
+	if (!OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: context use after free", __func__));
+	}
+	else
+	{
+		OSAtomicIncrement(&psContext->hRefCount);
+	}
+}
+
+/*
+	Internal interfaces for management of synchronisation block memory
+ */
+static PVRSRV_ERROR
+AllocSyncPrimitiveBlock(SYNC_PRIM_CONTEXT *psContext,
+                        SYNC_PRIM_BLOCK **ppsSyncBlock)
+{
+	SYNC_PRIM_BLOCK *psSyncBlk;
+	IMG_HANDLE hSyncPMR;
+	IMG_HANDLE hSyncImportHandle;
+	IMG_DEVMEM_SIZE_T uiImportSize;
+	PVRSRV_ERROR eError;
+
+	psSyncBlk = OSAllocMem(sizeof(SYNC_PRIM_BLOCK));
+	PVR_GOTO_IF_NOMEM(psSyncBlk, eError, fail_alloc);
+
+	psSyncBlk->psContext = psContext;
+
+	/* Allocate sync prim block */
+	eError = BridgeAllocSyncPrimitiveBlock(GetBridgeHandle(psContext->hDevConnection),
+	                                       &psSyncBlk->hServerSyncPrimBlock,
+	                                       &psSyncBlk->ui32FirmwareAddr,
+	                                       &psSyncBlk->ui32SyncBlockSize,
+	                                       &hSyncPMR);
+	PVR_GOTO_IF_ERROR(eError, fail_blockalloc);
+
+	/* Make it mappable by the client */
+	eError = DevmemMakeLocalImportHandle(psContext->hDevConnection,
+	                                     hSyncPMR,
+	                                     &hSyncImportHandle);
+	PVR_GOTO_IF_ERROR(eError, fail_export);
+
+	/* Get CPU mapping of the memory block */
+	eError = DevmemLocalImport(psContext->hDevConnection,
+	                           hSyncImportHandle,
+	                           PVRSRV_MEMALLOCFLAG_CPU_READABLE,
+	                           &psSyncBlk->hMemDesc,
+	                           &uiImportSize,
+	                           "SyncPrimitiveBlock");
+
+	/*
+		Regardless of success or failure we "undo" the export
+	 */
+	DevmemUnmakeLocalImportHandle(psContext->hDevConnection,
+	                              hSyncImportHandle);
+
+	PVR_GOTO_IF_ERROR(eError, fail_import);
+
+	eError = DevmemAcquireCpuVirtAddr(psSyncBlk->hMemDesc,
+	                                  (void **) &psSyncBlk->pui32LinAddr);
+	PVR_GOTO_IF_ERROR(eError, fail_cpuvaddr);
+
+	*ppsSyncBlock = psSyncBlk;
+	return PVRSRV_OK;
+
+fail_cpuvaddr:
+	DevmemFree(psSyncBlk->hMemDesc);
+fail_import:
+fail_export:
+	BridgeFreeSyncPrimitiveBlock(GetBridgeHandle(psContext->hDevConnection),
+	                             psSyncBlk->hServerSyncPrimBlock);
+fail_blockalloc:
+	OSFreeMem(psSyncBlk);
+fail_alloc:
+	return eError;
+}
+
+static void
+FreeSyncPrimitiveBlock(SYNC_PRIM_BLOCK *psSyncBlk)
+{
+	SYNC_PRIM_CONTEXT *psContext = psSyncBlk->psContext;
+
+	DevmemReleaseCpuVirtAddr(psSyncBlk->hMemDesc);
+	DevmemFree(psSyncBlk->hMemDesc);
+	(void) DestroyServerResource(psContext->hDevConnection,
+	                             NULL,
+	                             BridgeFreeSyncPrimitiveBlock,
+	                             psSyncBlk->hServerSyncPrimBlock);
+	OSFreeMem(psSyncBlk);
+}
+
+static PVRSRV_ERROR
+SyncPrimBlockImport(RA_PERARENA_HANDLE hArena,
+                    RA_LENGTH_T uSize,
+                    RA_FLAGS_T uFlags,
+                    RA_LENGTH_T uBaseAlignment,
+                    const IMG_CHAR *pszAnnotation,
+                    RA_IMPORT *psImport)
+{
+	SYNC_PRIM_CONTEXT *psContext = hArena;
+	SYNC_PRIM_BLOCK *psSyncBlock = NULL;
+	RA_LENGTH_T uiSpanSize;
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(uFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	/* Check we've not been called with an unexpected size */
+	PVR_LOG_GOTO_IF_INVALID_PARAM(hArena, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(uSize == sizeof(IMG_UINT32), eError, e0);
+
+	/*
+		Ensure the synprim context doesn't go away while we have sync blocks
+		attached to it
+	 */
+	_SyncPrimContextRef(psContext);
+
+	/* Allocate the block of memory */
+	eError = AllocSyncPrimitiveBlock(psContext, &psSyncBlock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "AllocSyncPrimitiveBlock", fail_syncblockalloc);
+
+	/* Allocate a span for it */
+	eError = RA_Alloc(psContext->psSpanRA,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  pszAnnotation,
+	                  &psSyncBlock->uiSpanBase,
+	                  &uiSpanSize,
+	                  NULL);
+	PVR_GOTO_IF_ERROR(eError, fail_spanalloc);
+
+	/*
+		There is no reason the span RA should return an allocation larger
+		then we request
+	 */
+	PVR_ASSERT(uiSpanSize == psSyncBlock->ui32SyncBlockSize);
+
+	psImport->base = psSyncBlock->uiSpanBase;
+	psImport->uSize = psSyncBlock->ui32SyncBlockSize;
+	psImport->hPriv = psSyncBlock;
+	return PVRSRV_OK;
+
+fail_spanalloc:
+	FreeSyncPrimitiveBlock(psSyncBlock);
+fail_syncblockalloc:
+	_SyncPrimContextUnref(psContext);
+e0:
+	return eError;
+}
+
+static void
+SyncPrimBlockUnimport(RA_PERARENA_HANDLE hArena,
+                      RA_BASE_T uiBase,
+                      RA_PERISPAN_HANDLE hImport)
+{
+	SYNC_PRIM_CONTEXT *psContext = hArena;
+	SYNC_PRIM_BLOCK *psSyncBlock = hImport;
+
+	if (!psContext || !psSyncBlock || uiBase != psSyncBlock->uiSpanBase)
+	{
+		/* Invalid input params */
+		return;
+	}
+
+	/* Free the span this import is using */
+	RA_Free(psContext->psSpanRA, uiBase);
+
+	/* Free the syncpim block */
+	FreeSyncPrimitiveBlock(psSyncBlock);
+
+	/*	Drop our reference to the syncprim context */
+	_SyncPrimContextUnref(psContext);
+}
+
+static INLINE IMG_UINT32 SyncPrimGetOffset(SYNC_PRIM *psSyncInt)
+{
+	IMG_UINT64 ui64Temp = psSyncInt->uiSpanAddr - psSyncInt->psSyncBlock->uiSpanBase;
+
+	PVR_ASSERT(ui64Temp<IMG_UINT32_MAX);
+
+	return TRUNCATE_64BITS_TO_32BITS(ui64Temp);
+}
+
+static void SyncPrimGetCPULinAddr(SYNC_PRIM *psSyncInt)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock = psSyncInt->psSyncBlock;
+
+	psSyncInt->sCommon.pui32LinAddr = psSyncBlock->pui32LinAddr +
+			(SyncPrimGetOffset(psSyncInt)/sizeof(IMG_UINT32));
+}
+
+static void SyncPrimLocalFree(SYNC_PRIM *psSyncInt, IMG_BOOL bFreeFirstSyncPrim)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+#if !defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	PVR_UNREFERENCED_PARAMETER(bFreeFirstSyncPrim);
+#else
+	/* Defer freeing the first allocated sync prim in the sync context */
+	if (psSyncInt != psContext->hFirstSyncPrim || bFreeFirstSyncPrim)
+#endif
+	{
+		PVRSRV_ERROR eError;
+		SHARED_DEV_CONNECTION hDevConnection =
+			psSyncInt->psSyncBlock->psContext->hDevConnection;
+
+		if (GetInfoPageDebugFlags(hDevConnection) & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+		{
+			if (psSyncInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = DestroyServerResource(hDevConnection,
+				                               NULL,
+				                               BridgeSyncRecordRemoveByHandle,
+				                               psSyncInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "BridgeSyncRecordRemoveByHandle");
+			}
+		}
+		else
+		{
+			IMG_UINT32 ui32FWAddr = psSyncBlock->ui32FirmwareAddr +
+					SyncPrimGetOffset(psSyncInt);
+
+			eError = BridgeSyncFreeEvent(GetBridgeHandle(hDevConnection), ui32FWAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeSyncFreeEvent");
+		}
+#if defined(PVRSRV_ENABLE_SYNC_POISONING)
+		(void) _SyncPrimSetValue(psSyncInt, LOCAL_SYNC_PRIM_POISON_VALUE);
+#else
+		/* reset the sync prim value as it is freed.
+		 * this guarantees the client sync allocated to the client will
+		 * have a value of zero and the client does not need to
+		 * explicitly initialise the sync value to zero.
+		 * the allocation of the backing memory for the sync prim block
+		 * is done with ZERO_ON_ALLOC so the memory is initially all zero.
+		 */
+		(void) _SyncPrimSetValue(psSyncInt, LOCAL_SYNC_PRIM_RESET_VALUE);
+#endif
+
+		RA_Free(psContext->psSubAllocRA, psSyncInt->uiSpanAddr);
+		OSFreeMem(psSyncInt);
+		_SyncPrimContextUnref(psContext);
+	}
+}
+
+static void SyncPrimLocalUnref(SYNC_PRIM *psSyncInt)
+{
+	if (!OSAtomicRead(&psSyncInt->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncPrimLocalUnref sync already freed"));
+	}
+	else if (0 == OSAtomicDecrement(&psSyncInt->hRefCount))
+	{
+		SyncPrimLocalFree(psSyncInt, IMG_FALSE);
+	}
+}
+
+static IMG_UINT32 SyncPrimGetFirmwareAddrLocal(SYNC_PRIM *psSyncInt)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	return psSyncBlock->ui32FirmwareAddr + SyncPrimGetOffset(psSyncInt);
+}
+
+static INLINE IMG_UINT32 _Log2(IMG_UINT32 ui32Align)
+{
+	PVR_ASSERT(IsPower2(ui32Align));
+	return ExactLog2(ui32Align);
+}
+
+/*
+	External interfaces
+ */
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimContextCreate(SHARED_DEV_CONNECTION hDevConnection,
+                      PSYNC_PRIM_CONTEXT *phSyncPrimContext)
+{
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	psContext = OSAllocMem(sizeof(SYNC_PRIM_CONTEXT));
+	PVR_GOTO_IF_NOMEM(psContext, eError, fail_alloc);
+
+	psContext->hDevConnection = hDevConnection;
+
+	OSSNPrintf(psContext->azName, SYNC_PRIM_NAME_SIZE, "Sync Prim RA-%p", psContext);
+	OSSNPrintf(psContext->azSpanName, SYNC_PRIM_NAME_SIZE, "Sync Prim span RA-%p", psContext);
+
+	/*
+		Create the RA for sub-allocations of the SynPrim's
+
+		Note:
+		The import size doesn't matter here as the server will pass
+		back the blocksize when does the import which overrides
+		what we specify here.
+	 */
+
+	psContext->psSubAllocRA = RA_Create(psContext->azName,
+	                                    /* Params for imports */
+	                                    _Log2(sizeof(IMG_UINT32)),
+	                                    RA_LOCKCLASS_3,
+	                                    SyncPrimBlockImport,
+	                                    SyncPrimBlockUnimport,
+	                                    psContext,
+	                                    RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_NOMEM(psContext->psSubAllocRA, eError, fail_suballoc);
+
+	/*
+		Create the span-management RA
+
+		The RA requires that we work with linear spans. For our use
+		here we don't require this behaviour as we're always working
+		within offsets of blocks (imports). However, we need to keep
+		the RA happy so we create the "span" management RA which
+		ensures that all are imports are added to the RA in a linear
+		fashion
+	 */
+	psContext->psSpanRA = RA_Create(psContext->azSpanName,
+	                                /* Params for imports */
+	                                0,
+	                                RA_LOCKCLASS_0,
+	                                NULL,
+	                                NULL,
+	                                NULL,
+	                                RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_NOMEM(psContext->psSpanRA, eError, fail_span);
+
+	if (!RA_Add(psContext->psSpanRA, 0, MAX_SYNC_MEM, 0, NULL))
+	{
+		RA_Delete(psContext->psSpanRA);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, fail_span);
+	}
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	psContext->hFirstSyncPrim = NULL;
+#endif
+
+	OSAtomicWrite(&psContext->hRefCount, 1);
+
+	*phSyncPrimContext = psContext;
+	return PVRSRV_OK;
+fail_span:
+	RA_Delete(psContext->psSubAllocRA);
+fail_suballoc:
+	OSFreeMem(psContext);
+fail_alloc:
+	return eError;
+}
+
+IMG_INTERNAL void SyncPrimContextDestroy(PSYNC_PRIM_CONTEXT hSyncPrimContext)
+{
+	SYNC_PRIM_CONTEXT *psContext = hSyncPrimContext;
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	/* Free the first sync prim that was allocated as part of this context */
+	if (psContext->hFirstSyncPrim)
+	{
+		SyncPrimLocalFree((SYNC_PRIM *)psContext->hFirstSyncPrim, IMG_TRUE);
+		psContext->hFirstSyncPrim = NULL;
+	}
+#endif
+
+	if (1 != OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s attempted with active references, may be the result of a race", __func__));
+	}
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+#if defined(__KERNEL__)
+	if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Forcing context destruction due to bad driver state", __func__));
+		OSAtomicWrite(&psContext->hRefCount, 1);
+	}
+#endif
+#endif
+	_SyncPrimContextUnref(psContext);
+}
+
+static PVRSRV_ERROR _SyncPrimAlloc(PSYNC_PRIM_CONTEXT hSyncPrimContext,
+                                   PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+                                   const IMG_CHAR *pszClassName,
+                                   IMG_BOOL bServerSync)
+{
+	SYNC_PRIM_CONTEXT *psContext = hSyncPrimContext;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM *psNewSync;
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiSpanAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hSyncPrimContext, "hSyncPrimeContext");
+
+	psNewSync = OSAllocMem(sizeof(SYNC_PRIM));
+	PVR_GOTO_IF_NOMEM(psNewSync, eError, fail_alloc);
+
+	eError = RA_Alloc(psContext->psSubAllocRA,
+	                  sizeof(IMG_UINT32),
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  sizeof(IMG_UINT32),
+	                  "Sync_Prim",
+	                  &uiSpanAddr,
+	                  NULL,
+	                  (RA_PERISPAN_HANDLE *) &psSyncBlock);
+	PVR_GOTO_IF_ERROR(eError, fail_raalloc);
+
+	OSAtomicWrite(&psNewSync->hRefCount, 1);
+	psNewSync->uiSpanAddr = uiSpanAddr;
+	psNewSync->psSyncBlock = psSyncBlock;
+	SyncPrimGetCPULinAddr(psNewSync);
+	*ppsSync = &psNewSync->sCommon;
+	_SyncPrimContextRef(psContext);
+#if defined(PVRSRV_ENABLE_SYNC_POISONING)
+	(void) _SyncPrimSetValue(psNewSync, LOCAL_SYNC_PRIM_RESET_VALUE);
+#endif
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	/* If this is the first sync prim allocated in the context, keep a handle to it */
+	if (psSyncBlock->uiSpanBase == 0 && psNewSync->uiSpanAddr == 0)
+	{
+		psContext->hFirstSyncPrim = psNewSync;
+	}
+#endif
+
+	if (GetInfoPageDebugFlags(psSyncBlock->psContext->hDevConnection) & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		IMG_CHAR szClassName[PVRSRV_SYNC_NAME_LENGTH];
+		size_t uiSize;
+
+		if (pszClassName)
+		{
+			uiSize = OSStringNLength(pszClassName, PVRSRV_SYNC_NAME_LENGTH);
+			/* Copy the class name annotation into a fixed-size array */
+			OSCachedMemCopy(szClassName, pszClassName, uiSize);
+			if (uiSize == PVRSRV_SYNC_NAME_LENGTH)
+				szClassName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+			else
+				szClassName[uiSize++] = '\0';
+		}
+		else
+		{
+			/* No class name annotation */
+			uiSize = 0;
+			szClassName[0] = '\0';
+		}
+
+		/* record this sync */
+		eError = BridgeSyncRecordAdd(
+				GetBridgeHandle(psSyncBlock->psContext->hDevConnection),
+				&psNewSync->hRecord,
+				psSyncBlock->hServerSyncPrimBlock,
+				psSyncBlock->ui32FirmwareAddr,
+				SyncPrimGetOffset(psNewSync),
+				bServerSync,
+				uiSize,
+				szClassName);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to add SyncRecord \"%s\" (%s)",
+					__func__,
+					szClassName,
+					PVRSRVGETERRORSTRING(eError)));
+			psNewSync->hRecord = NULL;
+		}
+	}
+	else
+	{
+		size_t	uiSize;
+
+		uiSize = OSStringNLength(pszClassName, PVRSRV_SYNC_NAME_LENGTH);
+
+		if (uiSize < PVRSRV_SYNC_NAME_LENGTH)
+			uiSize++;
+		/* uiSize now reflects size used for pszClassName + NUL byte */
+
+		eError = BridgeSyncAllocEvent(GetBridgeHandle(hSyncPrimContext->hDevConnection),
+		                              bServerSync,
+		                              psSyncBlock->ui32FirmwareAddr + SyncPrimGetOffset(psNewSync),
+		                              uiSize,
+		                              pszClassName);
+		PVR_LOG_IF_ERROR(eError, "BridgeSyncAllocEvent");
+	}
+
+	return PVRSRV_OK;
+
+fail_raalloc:
+	OSFreeMem(psNewSync);
+fail_alloc:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimAlloc(PSYNC_PRIM_CONTEXT hSyncPrimContext,
+                                        PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+                                        const IMG_CHAR *pszClassName)
+{
+	return _SyncPrimAlloc(hSyncPrimContext,
+	                      ppsSync,
+	                      pszClassName,
+	                      IMG_FALSE);
+}
+
+static PVRSRV_ERROR
+_SyncPrimSetValue(SYNC_PRIM *psSyncInt, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimSet(GetBridgeHandle(psContext->hDevConnection),
+								psSyncBlock->hServerSyncPrimBlock,
+								SyncPrimGetOffset(psSyncInt)/sizeof(IMG_UINT32),
+								ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimFree(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	SyncPrimLocalUnref(psSyncInt);
+
+
+err_out:
+	return eError;
+}
+
+#if defined(NO_HARDWARE)
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimNoHwUpdate(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	eError =  _SyncPrimSetValue(psSyncInt, ui32Value);
+
+err_out:
+	return eError;
+}
+#endif
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimSet(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	eError = _SyncPrimSetValue(psSyncInt, ui32Value);
+
+#if defined(PDUMP)
+	SyncPrimPDump(psSync);
+#endif
+err_out:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimLocalGetHandleAndOffset(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                                          IMG_HANDLE *phBlock,
+                                                          IMG_UINT32 *pui32Offset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(phBlock, eError, err_out);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pui32Offset, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	*phBlock = psSyncInt->psSyncBlock->hServerSyncPrimBlock;
+	*pui32Offset = psSyncInt->uiSpanAddr - psSyncInt->psSyncBlock->uiSpanBase;
+
+err_out:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimGetFirmwareAddr(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 *pui32FwAddr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	*pui32FwAddr = 0;
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+	*pui32FwAddr = SyncPrimGetFirmwareAddrLocal(psSyncInt);
+
+err_out:
+	return eError;
+}
+
+#if defined(PDUMP)
+IMG_INTERNAL void SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDump(GetBridgeHandle(psContext->hDevConnection),
+	                             psSyncBlock->hServerSyncPrimBlock,
+	                             SyncPrimGetOffset(psSyncInt));
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDump");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDumpValue(GetBridgeHandle(psContext->hDevConnection),
+	                                  psSyncBlock->hServerSyncPrimBlock,
+	                                  SyncPrimGetOffset(psSyncInt),
+	                                  ui32Value);
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpValue");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                   IMG_UINT32 ui32Value,
+                                   IMG_UINT32 ui32Mask,
+                                   PDUMP_POLL_OPERATOR eOperator,
+                                   IMG_UINT32 ui32PDumpFlags)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDumpPol(GetBridgeHandle(psContext->hDevConnection),
+	                                psSyncBlock->hServerSyncPrimBlock,
+	                                SyncPrimGetOffset(psSyncInt),
+	                                ui32Value,
+	                                ui32Mask,
+	                                eOperator,
+	                                ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpPol");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                   IMG_UINT64 uiWriteOffset,
+                                   IMG_UINT64 uiPacketSize,
+                                   IMG_UINT64 uiBufferSize)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+#if defined(__linux__) && defined(__i386__)
+	PVR_ASSERT(uiWriteOffset<IMG_UINT32_MAX);
+	PVR_ASSERT(uiPacketSize<IMG_UINT32_MAX);
+	PVR_ASSERT(uiBufferSize<IMG_UINT32_MAX);
+#endif
+	eError = BridgeSyncPrimPDumpCBP(GetBridgeHandle(psContext->hDevConnection),
+	                                psSyncBlock->hServerSyncPrimBlock,
+	                                SyncPrimGetOffset(psSyncInt),
+	                                TRUNCATE_64BITS_TO_32BITS(uiWriteOffset),
+	                                TRUNCATE_64BITS_TO_32BITS(uiPacketSize),
+	                                TRUNCATE_64BITS_TO_32BITS(uiBufferSize));
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpCBP");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/tlclient.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/tlclient.c
new file mode 100644
index 000000000000..a72f8118483b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/tlclient.c
@@ -0,0 +1,499 @@
+/*************************************************************************/ /*!
+@File           tlclient.c
+@Title          Services Transport Layer shared API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common API used in both clients and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* DESIGN NOTE
+ * This transport layer consumer-role API was created as a shared API when a
+ * client wanted to read the data of a TL stream from within the KM server
+ * driver. This was in addition to the existing clients supported externally
+ * by the UM client library component via PVR API layer.
+ * This shared API is thus used by the PVR TL API in the client library and
+ * by clients internal to the server driver module. It depends on
+ * client entry points of the TL and DEVMEM bridge modules. These entry points
+ * encapsulate from the TL shared API whether a direct bridge or an indirect
+ * (ioctl) bridge is used.
+ * One reason for needing this layer centres around the fact that some of the
+ * API functions make multiple bridge calls and the logic that glues these
+ * together is common regardless of client location. Further this layer has
+ * allowed the defensive coding that checks parameters to move into the PVR
+ * API layer where untrusted clients enter giving a more efficient KM code path.
+ */
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+
+#include "allocmem.h"
+#include "devicemem.h"
+
+#include "tlclient.h"
+#include "pvrsrv_tlcommon.h"
+#include "client_pvrtl_bridge.h"
+
+#if defined(__KERNEL__)
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+/* Defines/Constants
+ */
+
+#define NO_ACQUIRE             0xffffffffU
+
+/* User-side stream descriptor structure.
+ */
+typedef struct _TL_STREAM_DESC_
+{
+	/* Handle on kernel-side stream descriptor*/
+	IMG_HANDLE		hServerSD;
+
+	/* Stream data buffer variables */
+	DEVMEM_MEMDESC*			psUMmemDesc;
+	IMG_PBYTE				pBaseAddr;
+
+	/* Offset in bytes into the circular buffer and valid only after
+	 * an Acquire call and undefined after a release. */
+	IMG_UINT32	uiReadOffset;
+
+	/* Always a positive integer when the Acquire call returns and a release
+	 * is outstanding. Undefined at all other times. */
+	IMG_UINT32	uiReadLen;
+
+	/* Counter indicating how many writes to a stream failed.
+	 * It's used to reduce number of errors in output log. */
+	IMG_UINT32 ui32WritesFailed;
+
+	/* Name of the stream. */
+	IMG_CHAR szName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+} TL_STREAM_DESC, *PTL_STREAM_DESC;
+
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientOpenStream(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR* pszName,
+		IMG_UINT32   ui32Mode,
+		IMG_HANDLE*  phSD)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	TL_STREAM_DESC *psSD = NULL;
+	IMG_HANDLE hTLPMR;
+	IMG_HANDLE hTLImportHandle;
+	IMG_DEVMEM_SIZE_T uiImportSize;
+	PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(pszName);
+	PVR_ASSERT(phSD);
+	*phSD = NULL;
+
+	/* Allocate memory for the stream descriptor object, initialise with
+	 * "no data read" yet. */
+	psSD = OSAllocZMem(sizeof(TL_STREAM_DESC));
+	PVR_LOG_GOTO_IF_NOMEM(psSD, eError, e0);
+	psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+
+	/* Send open stream request to kernel server to get stream handle and
+	 * buffer cookie so we can get access to the buffer in this process. */
+	eError = BridgeTLOpenStream(GetBridgeHandle(hDevConnection), pszName,
+			ui32Mode, &psSD->hServerSD, &hTLPMR);
+	if (eError != PVRSRV_OK)
+	{
+		if ((ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WAIT) &&
+			(eError == PVRSRV_ERROR_TIMEOUT))
+		{
+			goto e1;
+		}
+		PVR_LOG_GOTO_IF_ERROR(eError, "BridgeTLOpenStream", e1);
+	}
+
+	/* Convert server export cookie into a cookie for use by this client */
+	eError = DevmemMakeLocalImportHandle(hDevConnection,
+			hTLPMR, &hTLImportHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemMakeLocalImportHandle", e2);
+
+	uiMemFlags |= ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	        PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0ULL;
+	/* Now convert client cookie into a client handle on the buffer's
+	 * physical memory region */
+	eError = DevmemLocalImport(hDevConnection,
+	                           hTLImportHandle,
+	                           uiMemFlags,
+	                           &psSD->psUMmemDesc,
+	                           &uiImportSize,
+	                           "TLBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemImport", e3);
+
+	/* Now map the memory into the virtual address space of this process. */
+	eError = DevmemAcquireCpuVirtAddr(psSD->psUMmemDesc, (void **)
+															&psSD->pBaseAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", e4);
+
+	/* Ignore error, not much that can be done */
+	(void) DevmemUnmakeLocalImportHandle(hDevConnection,
+			hTLImportHandle);
+
+	/* Copy stream name */
+	OSStringSafeCopy(psSD->szName, pszName, PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	/* Return client descriptor handle to caller */
+	*phSD = psSD;
+	return PVRSRV_OK;
+
+/* Clean up post buffer setup */
+e4:
+	DevmemFree(psSD->psUMmemDesc);
+e3:
+	(void) DevmemUnmakeLocalImportHandle(hDevConnection,
+				&hTLImportHandle);
+/* Clean up post stream open */
+e2:
+	BridgeTLCloseStream(GetBridgeHandle(hDevConnection), psSD->hServerSD);
+
+/* Clean up post allocation of the descriptor object */
+e1:
+	OSFreeMem(psSD);
+
+e0:
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCloseStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	/* Check the caller provided connection is valid */
+	if (!psSD->hServerSD)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: descriptor already "
+				"closed/not open", __func__));
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	/* Check if acquire is outstanding, perform release if it is, ignore result
+	 * as there is not much we can do if it is an error other than close */
+	if (psSD->uiReadLen != NO_ACQUIRE)
+	{
+		(void) BridgeTLReleaseData(GetBridgeHandle(hDevConnection),
+				psSD->hServerSD, psSD->uiReadOffset, psSD->uiReadLen);
+		psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+	}
+
+	/* Clean up DevMem resources used for this stream in this client */
+	DevmemReleaseCpuVirtAddr(psSD->psUMmemDesc);
+
+	DevmemFree(psSD->psUMmemDesc);
+
+	/* Send close to server to clean up kernel mode resources for this
+	 * handle and release the memory. */
+	eError = DestroyServerResource(hDevConnection,
+	                               NULL,
+	                               BridgeTLCloseStream,
+	                               psSD->hServerSD);
+	PVR_LOG_IF_ERROR(eError, "BridgeTLCloseStream");
+
+	if (psSD->ui32WritesFailed != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s() %u writes failed to stream %s (%c)",
+		        __func__, psSD->ui32WritesFailed, psSD->szName,
+		        psSD->ui32WritesFailed == IMG_UINT32_MAX ? 'T' : 'F'));
+	}
+
+	OSCachedMemSet(psSD, 0x00, sizeof(TL_STREAM_DESC));
+	OSFreeMem(psSD);
+
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientDiscoverStreams(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR *pszNamePattern,
+		IMG_CHAR aszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+		IMG_UINT32 *pui32NumFound)
+{
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(pszNamePattern);
+	PVR_ASSERT(pui32NumFound);
+
+	return BridgeTLDiscoverStreams(GetBridgeHandle(hDevConnection),
+			pszNamePattern,
+			/* we need to treat this as one dimensional array */
+			*pui32NumFound * PRVSRVTL_MAX_STREAM_NAME_SIZE,
+			(IMG_CHAR *) aszStreams,
+			pui32NumFound);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+	IMG_UINT32 ui32BufferOffset, ui32Unused;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppui8Data);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLReserveStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &ui32BufferOffset, ui32Size, ui32Size, &ui32Unused);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppui8Data = psSD->pBaseAddr + ui32BufferOffset;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream2(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size,
+		IMG_UINT32 ui32SizeMin,
+		IMG_UINT32 *pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+	IMG_UINT32 ui32BufferOffset;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppui8Data);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLReserveStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &ui32BufferOffset, ui32Size, ui32SizeMin,
+			pui32Available);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppui8Data = psSD->pBaseAddr + ui32BufferOffset;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCommitStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLCommitStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, ui32Size);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientAcquireData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE  hSD,
+		IMG_PBYTE*  ppPacketBuf,
+		IMG_UINT32* pui32BufLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppPacketBuf);
+	PVR_ASSERT(pui32BufLen);
+
+	/* In case of non-blocking acquires, which can return no data, and
+	 * error paths ensure we clear the output parameters first. */
+	*ppPacketBuf = NULL;
+	*pui32BufLen = 0;
+
+	/* Check Acquire has not been called twice in a row without a release */
+	if (psSD->uiReadOffset != NO_ACQUIRE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: acquire already "
+				"outstanding, ReadOffset(%d), ReadLength(%d)",
+				__func__, psSD->uiReadOffset, psSD->uiReadLen));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Ask the kernel server for the next chunk of data to read */
+	eError = BridgeTLAcquireData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &psSD->uiReadOffset, &psSD->uiReadLen);
+	if (eError != PVRSRV_OK)
+	{
+		/* Mask reporting of the errors seen under normal operation */
+		if ((eError != PVRSRV_ERROR_TIMEOUT) &&
+			(eError != PVRSRV_ERROR_STREAM_READLIMIT_REACHED))
+		{
+			PVR_LOG_ERROR(eError, "BridgeTLAcquireData");
+		}
+		psSD->uiReadOffset = psSD->uiReadLen = NO_ACQUIRE;
+		return eError;
+	}
+	/* else PVRSRV_OK */
+
+	/* Return the data offset and length to the caller if bytes are available
+	 * to be read. Could be zero for non-blocking mode so pass back cleared
+	 * values above */
+	if (psSD->uiReadLen)
+	{
+		*ppPacketBuf = psSD->pBaseAddr + psSD->uiReadOffset;
+		*pui32BufLen = psSD->uiReadLen;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _TLClientReleaseDataLen(
+		SHARED_DEV_CONNECTION hDevConnection,
+		TL_STREAM_DESC* psSD,
+		IMG_UINT32 uiReadLen)
+{
+	PVRSRV_ERROR eError;
+
+	/* the previous acquire did not return any data, this is a no-operation */
+	if (psSD->uiReadLen == 0)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Check release has not been called twice in a row without an acquire */
+	if (psSD->uiReadOffset == NO_ACQUIRE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: no acquire to release", __func__));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Inform the kernel to release the data from the buffer */
+	eError = BridgeTLReleaseData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD,
+			psSD->uiReadOffset, uiReadLen);
+	PVR_LOG_IF_ERROR(eError, "BridgeTLReleaseData");
+
+	/* Reset state to indicate no outstanding acquire */
+	psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD)
+{
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	return _TLClientReleaseDataLen(hDevConnection, psSD, psSD->uiReadLen);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseDataLess(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD, IMG_UINT32 uiActualReadLen)
+{
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	/* Check the specified size is within the size returned by Acquire */
+	if (uiActualReadLen > psSD->uiReadLen)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: no acquire to release", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return _TLClientReleaseDataLen(hDevConnection, psSD, uiActualReadLen);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientWriteData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size,
+		IMG_BYTE *pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ui32Size);
+	PVR_ASSERT(pui8Data);
+
+	eError = BridgeTLWriteData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, ui32Size, pui8Data);
+
+	if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		if (psSD->ui32WritesFailed == 0)
+		{
+			PVR_LOG_ERROR(eError, "BridgeTLWriteData");
+		}
+		if (psSD->ui32WritesFailed != IMG_UINT32_MAX)
+		{
+			psSD->ui32WritesFailed++;
+		}
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "BridgeTLWriteData");
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (tlclient.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.c
new file mode 100644
index 000000000000..2d4f2fa8ce66
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.c
@@ -0,0 +1,280 @@
+/*************************************************************************/ /*!
+@File
+@Title          Provides splay-trees.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implementation of splay-trees.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "allocmem.h" /* for OSMemAlloc / OSMemFree */
+#include "osfunc.h" /* for OSMemFree */
+#include "pvr_debug.h"
+#include "uniq_key_splay_tree.h"
+
+/**
+ * PVRSRVSplay - perform a simple top down splay
+ * @ui32Flags: flags that must splayed to the root (if possible)
+ * @psTree:    psTree The tree to splay.
+ *
+ * Return the resulting tree after the splay operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVSplay (IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_SPLAY_TREE sTmp1;
+	IMG_PSPLAY_TREE psLeft;
+	IMG_PSPLAY_TREE psRight;
+	IMG_PSPLAY_TREE psTmp2;
+
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	sTmp1.psLeft = NULL;
+	sTmp1.psRight = NULL;
+
+	psLeft = &sTmp1;
+	psRight = &sTmp1;
+
+	for (;;)
+	{
+		if (uiFlags < psTree->uiFlags)
+		{
+			if (psTree->psLeft == NULL)
+			{
+				break;
+			}
+
+			if (uiFlags < psTree->psLeft->uiFlags)
+			{
+				/* if we get to this point, we need to rotate right the tree */
+				psTmp2 = psTree->psLeft;
+				psTree->psLeft = psTmp2->psRight;
+				psTmp2->psRight = psTree;
+				psTree = psTmp2;
+				if (psTree->psLeft == NULL)
+				{
+					break;
+				}
+			}
+
+			/* if we get to this point, we need to link right */
+			psRight->psLeft = psTree;
+			psRight = psTree;
+			psTree = psTree->psLeft;
+		}
+		else
+		{
+			if (uiFlags > psTree->uiFlags)
+			{
+				if (psTree->psRight == NULL)
+				{
+					break;
+				}
+
+				if (uiFlags > psTree->psRight->uiFlags)
+				{
+					/* if we get to this point, we need to rotate left the tree */
+					psTmp2 = psTree->psRight;
+					psTree->psRight = psTmp2->psLeft;
+					psTmp2->psLeft = psTree;
+					psTree = psTmp2;
+					if (psTree->psRight == NULL)
+					{
+						break;
+					}
+				}
+
+				/* if we get to this point, we need to link left */
+				psLeft->psRight = psTree;
+				psLeft = psTree;
+				psTree = psTree->psRight;
+			}
+			else
+			{
+				break;
+			}
+		}
+	}
+
+	/* at this point re-assemble the tree */
+	psLeft->psRight = psTree->psLeft;
+	psRight->psLeft = psTree->psRight;
+	psTree->psLeft = sTmp1.psRight;
+	psTree->psRight = sTmp1.psLeft;
+	return psTree;
+}
+
+
+/**
+ * PVRSRVInsert - insert a node into the Tree (unless it is already present, in
+ * which case it is equivalent to performing only a splay operation
+ * @ui32Flags: the key of the new node
+ * @psTree:    tree into which one wants to add a new node
+ *
+ * Return the resulting tree after the splay operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVInsert(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_PSPLAY_TREE psNew;
+
+	if (psTree != NULL)
+	{
+		psTree = PVRSRVSplay(uiFlags, psTree);
+		if (psTree->uiFlags == uiFlags)
+		{
+			return psTree;
+		}
+	}
+
+	psNew = (IMG_PSPLAY_TREE) OSAllocMem(sizeof(IMG_SPLAY_TREE));
+	if (psNew == NULL)
+	{
+		PVR_DPF ((PVR_DBG_ERROR, "Error: failed to allocate memory to add a node to the splay tree."));
+		return NULL;
+	}
+
+	psNew->uiFlags = uiFlags;
+	OSCachedMemSet(&(psNew->buckets[0]), 0, sizeof(psNew->buckets));
+
+#if defined(PVR_CTZLL)
+	psNew->bHasEltsMapping = ~(((IMG_ELTS_MAPPINGS) 1 << (sizeof(psNew->buckets) / (sizeof(psNew->buckets[0])))) - 1);
+#endif
+
+	if (psTree == NULL)
+	{
+		psNew->psLeft  = NULL;
+		psNew->psRight = NULL;
+		return psNew;
+	}
+
+	if (uiFlags < psTree->uiFlags)
+	{
+		psNew->psLeft  = psTree->psLeft;
+		psNew->psRight = psTree;
+		psTree->psLeft = NULL;
+	}
+	else
+	{
+		psNew->psRight  = psTree->psRight;
+		psNew->psLeft   = psTree;
+		psTree->psRight = NULL;
+	}
+
+	return psNew;
+}
+
+
+/**
+ * PVRSRVDelete - delete a node from the tree (unless it is not there, in which
+ * case it is equivalent to a splay operation)
+ * @ui32Flags: value of the node to remove
+ * @psTree:    tree into which the node must be removed
+ *
+ * Return the resulting tree.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVDelete(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_PSPLAY_TREE psTmp;
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	psTree = PVRSRVSplay(uiFlags, psTree);
+	if (uiFlags == psTree->uiFlags)
+	{
+		/* The value was present in the tree */
+		if (psTree->psLeft == NULL)
+		{
+			psTmp = psTree->psRight;
+		}
+		else
+		{
+			psTmp = PVRSRVSplay(uiFlags, psTree->psLeft);
+			psTmp->psRight = psTree->psRight;
+		}
+		OSFreeMem(psTree);
+		return psTmp;
+	}
+
+	/* The value was not present in the tree, so just return it as is
+	 * (after the splay) */
+	return psTree;
+}
+
+/**
+ * PVRSRVFindNode - pick up the appropriate node for the given flags
+ * @ui32Flags: flags that must associated with the node
+ * @psTree:    current splay tree node
+ *
+ * Return the resulting tree node after the search operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVFindNode(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	while (psTree)
+	{
+		if (uiFlags == psTree->uiFlags)
+		{
+			return psTree;
+		}
+
+		if (uiFlags < psTree->uiFlags)
+		{
+			psTree = psTree->psLeft;
+			continue;
+		}
+
+		if (uiFlags > psTree->uiFlags)
+		{
+			psTree = psTree->psRight;
+			continue;
+		}
+	}
+
+	return NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.h b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.h
new file mode 100644
index 000000000000..75ec9297e9c2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          Splay trees interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides debug functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef UNIQ_KEY_SPLAY_TREE_H_
+#define UNIQ_KEY_SPLAY_TREE_H_
+
+#include "img_types.h"
+#include "pvr_intrinsics.h"
+
+#if defined(PVR_CTZLL)
+  /* map the is_bucket_n_free to an int.
+   * This way, the driver can find the first non empty without loop
+   */
+  typedef IMG_UINT64 IMG_ELTS_MAPPINGS;
+#endif
+
+typedef IMG_UINT64 IMG_PSPLAY_FLAGS_T;
+
+/* head of list of free boundary tags for indexed by pvr_log2 of the
+   boundary tag size */
+
+#define FREE_TABLE_LIMIT 40
+
+struct _BT_;
+
+typedef struct img_splay_tree
+{
+	/* left child/subtree */
+    struct img_splay_tree * psLeft;
+
+	/* right child/subtree */
+    struct img_splay_tree * psRight;
+
+    /* Flags to match on this span, used as the key. */
+    IMG_PSPLAY_FLAGS_T uiFlags;
+#if defined(PVR_CTZLL)
+	/* each bit of this int is a boolean telling if the corresponding
+	   bucket is empty or not */
+    IMG_ELTS_MAPPINGS bHasEltsMapping;
+#endif
+	struct _BT_ * buckets[FREE_TABLE_LIMIT];
+} IMG_SPLAY_TREE, *IMG_PSPLAY_TREE;
+
+IMG_PSPLAY_TREE PVRSRVSplay (IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVInsert(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVDelete(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVFindNode(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+
+
+#endif /* !UNIQ_KEY_SPLAY_TREE_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.c b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.c
new file mode 100644
index 000000000000..78d731c98eb6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.c
@@ -0,0 +1,629 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance counter table
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance counters table
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(__KERNEL__)
+#include "rgxdefs_km.h"
+#else
+#include "rgxdefs.h"
+#endif
+#include "rgx_hwperf_table.h"
+
+/* Includes needed for PVRSRVKM (Server) context */
+#	include "rgx_bvnc_defs_km.h"
+#	if defined(__KERNEL__)
+#		include "rgxdevice.h"
+#	endif
+
+/* Shared compile-time context ASSERT macro */
+#if defined(RGX_FIRMWARE)
+#	include "rgxfw_utils.h"
+/* firmware context */
+#	define DBG_ASSERT(_c) RGXFW_ASSERT((_c))
+#else
+#	include "pvr_debug.h"
+/* host client/server context */
+#	define DBG_ASSERT(_c) PVR_ASSERT((_c))
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered()
+
+ Referenced in gasCntBlkTypeModel[] table below and only called from
+ RGX_FIRMWARE run-time context. Therefore compile time configuration is used.
+ *****************************************************************************/
+
+#if defined(RGX_FIRMWARE) && defined(RGX_FEATURE_PERFBUS)
+#	include "rgxfw_pow.h"
+#	include "rgxfw_utils.h"
+
+static bool rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(eBlkType);
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+	/* S6XT: TA, TORNADO */
+	return true;
+#else
+	/* S6  : TA, HUB, RASTER (RASCAL) */
+	return rgxfw_pow_is_rd_on();
+#endif
+}
+
+/* Only use conditional compilation when counter blocks appear in different
+ * islands for different Rogue families.
+ */
+static bool rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	IMG_UINT32 ui32NumDustsEnabled = rgxfw_pow_get_enabled_units();
+
+	if (rgxfw_pow_is_rd_on() && (ui32NumDustsEnabled > 0U))
+	{
+#if defined(RGX_FEATURE_DYNAMIC_DUST_POWER)
+		IMG_UINT32 ui32NumUscEnabled = ui32NumDustsEnabled*2U;
+
+		switch (eBlkType)
+		{
+		case RGX_CNTBLK_ID_TPU_MCU0:                   /* S6 and S6XT */
+			if (ui8UnitId >= ui32NumDustsEnabled)
+			{
+				return false;
+			}
+			break;
+		case RGX_CNTBLK_ID_USC0:                       /* S6, S6XT, S7 */
+		case RGX_CNTBLK_ID_PBE0:                       /* S7, PBE2_IN_XE */
+			/* Handle single cluster cores */
+			if (ui8UnitId >= ((ui32NumUscEnabled > RGX_FEATURE_NUM_CLUSTERS) ? RGX_FEATURE_NUM_CLUSTERS : ui32NumUscEnabled))
+			{
+				return false;
+			}
+			break;
+		case RGX_CNTBLK_ID_BLACKPEARL0:                /* S7 */
+		case RGX_CNTBLK_ID_RASTER0:                    /* S6XT */
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_TEXAS0:                     /* S6XT */
+#endif
+			if (ui8UnitId >= (RGX_REQ_NUM_PHANTOMS(ui32NumUscEnabled)))
+			{
+				return false;
+			}
+			break;
+		default:
+			RGXFW_ASSERT(false); /* should never get here, table error */
+			break;
+		}
+#else
+		/* Always true, no fused DUSTs, all powered so do not check unit */
+		PVR_UNREFERENCED_PARAMETER(eBlkType);
+		PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+#endif
+	}
+	else
+	{
+		return false;
+	}
+	return true;
+}
+
+#else /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+# define rgxfw_hwperf_pow_st_direct   ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+# define rgxfw_hwperf_pow_st_indirect ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+
+#endif /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() start
+
+ Referenced in gasCntBlkTypeModel[] table below and called from all build
+ contexts:
+ RGX_FIRMWARE, PVRSRVCTL (UM) and PVRSRVKM (Server).
+
+ Therefore each function has two implementations, one for compile time and one
+ run time configuration depending on the context. The functions will inform the
+ caller whether this block is valid for this particular RGX device. Other
+ run-time dependent data is returned in psRtInfo for the caller to use.
+ *****************************************************************************/
+
+/* Used for block types: USC */
+static IMG_BOOL rgx_hwperf_blk_present_perfbus(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_USC0);
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+		{
+			psRtInfo->ui32NumUnits = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) ? RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) : 0;
+			psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+			return IMG_TRUE;
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+# if defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: Direct RASTERISATION, HUB */
+static IMG_BOOL rgx_hwperf_blk_present_not_clustergrouping(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_HUB));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if ((!RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING)) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS)))
+		{
+			psRtInfo->ui32NumUnits = 1;
+			psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+			return IMG_TRUE;
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+#if defined(__KERNEL__) /* Server context */
+static IMG_UINT32 rgx_units_indirect_by_phantom(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_INDIRECT_BY_PHANTOM */
+	return ((psFeatCfg->ui64Features & RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK) == 0) ? 1
+			: (psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX]+3)/4;
+}
+
+static IMG_UINT32 rgx_units_phantom_indirect_by_dust(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST */
+	return MAX((psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX]>>1),1);
+}
+
+static IMG_UINT32 rgx_units_phantom_indirect_by_cluster(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER */
+	return psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX];
+}
+#endif /* defined(__KERNEL__) */
+
+/* Used for block types: TORNADO, TEXAS, Indirect RASTERISATION */
+static IMG_BOOL rgx_hwperf_blk_present_xttop(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TORNADO) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER0));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TORNADO)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TEXAS_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: JONES, TPU_MCU, TEXAS, BLACKPEARL, PBE */
+static IMG_BOOL rgx_hwperf_blk_present_s7top(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_JONES) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_BLACKPEARL0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0));
+
+#if defined(__KERNEL__) /* Server context */
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE_IDX)
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TPU_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TEXAS3_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_BLACKPEARL0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_cluster(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_JONES)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+		}
+	}
+#endif
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return IMG_TRUE;
+# else
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: TA, TPU_MCU. Also PBE when PBE2_IN_XE is present */
+static IMG_BOOL rgx_hwperf_blk_present_not_s7top(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TA) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TA)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+			{
+				if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+				{
+					/* PBE counters are not present on this config */
+					return IMG_FALSE;
+				}
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_cluster(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TPU_MCU_L0_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS)
+#  if !defined(RGX_FEATURE_PBE2_IN_XE)
+	if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+	{
+		/* No support for PBE counters without PBE2_IN_XE */
+		return IMG_FALSE;
+	}
+#  endif
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+static IMG_BOOL rgx_hwperf_blk_present_check_s7top_or_not(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+#if defined(__KERNEL__)
+	return (rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo)
+	     || rgx_hwperf_blk_present_not_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo));
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+
+#elif defined(RGX_FEATURE_PBE2_IN_XE) || defined(RGX_FEATURE_PERFBUS)
+	return rgx_hwperf_blk_present_not_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	return IMG_FALSE;
+#endif
+}
+
+static IMG_BOOL rgx_hwperf_blk_present_check_s7top_or_xttop(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+#if defined(__KERNEL__)
+	return (rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo)
+	     || rgx_hwperf_blk_present_xttop(psBlkTypeDesc, pvDev_km, pvRtInfo));
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+
+#elif defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_xttop(psBlkTypeDesc, pvDev_km, pvRtInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	return IMG_FALSE;
+#endif
+}
+
+#if !defined(__KERNEL__) /* Firmware or User-mode context */
+static IMG_BOOL rgx_hwperf_blk_present_false(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+
+	/* Some functions not used on some BVNCs, silence compiler warnings */
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_perfbus);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_not_clustergrouping);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_xttop);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_s7top);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_not_s7top);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_check_s7top_or_not);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_check_s7top_or_xttop);
+
+	return IMG_FALSE;
+}
+
+/* Used to instantiate a null row in the block type model table below where the
+ * block is not supported for a given build BVNC in firmware/user mode context.
+ * This is needed as the blockid to block type lookup uses the table as well
+ * and clients may try to access blocks not in the hardware. */
+#define RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(_blkid) {_blkid, 0, 0, 0, 0, 0, 0, 0, 0, #_blkid, NULL, rgx_hwperf_blk_present_false}
+
+#endif
+
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() end
+ *****************************************************************************/
+
+#if defined(__KERNEL__) /* Values will be calculated at run-time */
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+#define RGX_INDIRECT_REG_TEXAS 0xFFFFFFFF
+#define RGX_INDIRECT_REG_TPU 0xFFFFFFFF
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST
+#define RGX_INDIRECT_REG_TEXAS RGX_CR_TEXAS3_PERF_INDIRECT
+#define RGX_INDIRECT_REG_TPU RGX_CR_TPU_PERF_INDIRECT
+
+#else
+
+#if defined(RGX_FEATURE_PERFBUS)
+#define RGX_INDIRECT_REG_TPU RGX_CR_TPU_MCU_L0_PERF_INDIRECT
+#endif
+
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_INDIRECT_BY_PHANTOM
+#define RGX_INDIRECT_REG_TEXAS RGX_CR_TEXAS_PERF_INDIRECT
+#endif
+
+#endif
+
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] table
+
+ This table holds the entries for the performance counter block type model.
+ Where the block is not present on an RGX device in question the
+ pfnIsBlkPresent() returns false, if valid and present it returns true.
+ Columns in the table with a ** indicate the value is a default and the
+ value returned in RGX_HWPERF_CNTBLK_RT_INFO when calling pfnIsBlkPresent()
+ should be used at runtime by the caller. These columns are only valid for
+ compile time BVNC configured contexts.
+
+ Order of table rows must match order of counter block IDs in the enumeration
+ RGX_HWPERF_CNTBLK_ID.
+ *****************************************************************************/
+
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] =
+{
+		/*   ui32CntBlkIdBase,         ui32IndirectReg,                  ui32PerfReg,                  ui32Select0BaseReg,                    ui32Counter0BaseReg                   ui8NumCounters,  ui32NumUnits**,                  ui8SelectRegModeShift, ui8SelectRegOffsetShift,            pfnIsBlkPowered               pfnIsBlkPresent
+		 *                                                                                                                                                                                                                                                   pszBlockNameComment,  */
+		/*RGX_CNTBLK_ID_TA*/
+#if !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_TA,       0, /* direct */                RGX_CR_TA_PERF,             RGX_CR_TA_PERF_SELECT0,              RGX_CR_TA_PERF_COUNTER_0,             4,              1,                              21,                  3,  "RGX_CR_TA_PERF",              rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TA),
+#endif
+
+		/*RGX_CNTBLK_ID_RASTER*/
+#if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_RASTER,   0, /* direct */                RGX_CR_RASTERISATION_PERF,  RGX_CR_RASTERISATION_PERF_SELECT0,   RGX_CR_RASTERISATION_PERF_COUNTER_0,  4,              1,                              21,                  3,  "RGX_CR_RASTERISATION_PERF",   rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_clustergrouping },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_RASTER),
+#endif
+
+		/*RGX_CNTBLK_ID_HUB*/
+#if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_HUB,      0, /* direct */                RGX_CR_HUB_BIFPMCACHE_PERF, RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0,  RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0, 4,              1,                              21,                  3,  "RGX_CR_HUB_BIFPMCACHE_PERF",  rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_clustergrouping },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_HUB),
+#endif
+
+		/*RGX_CNTBLK_ID_TORNADO*/
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_TORNADO,  0, /* direct */                RGX_CR_TORNADO_PERF,        RGX_CR_TORNADO_PERF_SELECT0,         RGX_CR_TORNADO_PERF_COUNTER_0,        4,              1,                              21,                  4,  "RGX_CR_TORNADO_PERF",         rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TORNADO),
+#endif
+
+		/*RGX_CNTBLK_ID_JONES*/
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_JONES,   0, /* direct */                 RGX_CR_JONES_PERF,          RGX_CR_JONES_PERF_SELECT0,           RGX_CR_JONES_PERF_COUNTER_0,          4,              1,                              21,                  3,  "RGX_CR_JONES_PERF",           rgxfw_hwperf_pow_st_direct,    rgx_hwperf_blk_present_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_JONES),
+#endif
+
+		/*RGX_CNTBLK_ID_TPU_MCU0*/
+#if defined(__KERNEL__) || (defined(RGX_FEATURE_PERFBUS) && !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+		{RGX_CNTBLK_ID_TPU_MCU0, RGX_INDIRECT_REG_TPU,      RGX_CR_TPU_MCU_L0_PERF,   RGX_CR_TPU_MCU_L0_PERF_SELECT0,     RGX_CR_TPU_MCU_L0_PERF_COUNTER_0,     4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST,    21,          3,  "RGX_CR_TPU_MCU_L0_PERF",      rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_not },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TPU_MCU0),
+#endif
+
+		/*RGX_CNTBLK_ID_USC0*/
+#if defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_USC0,    RGX_CR_USC_PERF_INDIRECT,       RGX_CR_USC_PERF,            RGX_CR_USC_PERF_SELECT0,            RGX_CR_USC_PERF_COUNTER_0,            4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER, 21,          3,  "RGX_CR_USC_PERF",             rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_perfbus },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_USC0),
+#endif
+
+		/*RGX_CNTBLK_ID_TEXAS0*/
+#if defined(__KERNEL__) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+		{RGX_CNTBLK_ID_TEXAS0,  RGX_INDIRECT_REG_TEXAS,      RGX_CR_TEXAS_PERF,          RGX_CR_TEXAS_PERF_SELECT0,          RGX_CR_TEXAS_PERF_COUNTER_0,          6,              RGX_HWPERF_NUM_BLOCK_UNITS,             31,          3,  "RGX_CR_TEXAS_PERF",           rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TEXAS0),
+#endif
+
+		/*RGX_CNTBLK_ID_RASTER0*/
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_RASTER0, RGX_CR_RASTERISATION_PERF_INDIRECT, RGX_CR_RASTERISATION_PERF, RGX_CR_RASTERISATION_PERF_SELECT0, RGX_CR_RASTERISATION_PERF_COUNTER_0,  4,            RGX_HWPERF_INDIRECT_BY_PHANTOM,         21,          3,  "RGX_CR_RASTERISATION_PERF",   rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_RASTER0),
+#endif
+
+		/*RGX_CNTBLK_ID_BLACKPEARL0*/
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_BLACKPEARL0, RGX_CR_BLACKPEARL_PERF_INDIRECT, RGX_CR_BLACKPEARL_PERF, RGX_CR_BLACKPEARL_PERF_SELECT0,    RGX_CR_BLACKPEARL_PERF_COUNTER_0,     6,              RGX_HWPERF_INDIRECT_BY_PHANTOM,         21,          3,  "RGX_CR_BLACKPEARL_PERF",      rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_BLACKPEARL0),
+#endif
+
+		/*RGX_CNTBLK_ID_PBE0*/
+#if defined(__KERNEL__) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(RGX_FEATURE_PBE2_IN_XE)
+		{RGX_CNTBLK_ID_PBE0,    RGX_CR_PBE_PERF_INDIRECT,        RGX_CR_PBE_PERF,            RGX_CR_PBE_PERF_SELECT0,            RGX_CR_PBE_PERF_COUNTER_0,            4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER, 21,          3,  "RGX_CR_PBE_PERF",             rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_not },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_PBE0),
+#endif
+};
+
+
+IMG_INTERNAL IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel)
+{
+	*ppsModel = gasCntBlkTypeModel;
+	return ARRAY_SIZE(gasCntBlkTypeModel);
+}
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.h b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.h
new file mode 100644
index 000000000000..449885cdcb64
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.h
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File
+@Title          HWPerf counter table header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_HWPERF_TABLE_H
+#define RGX_HWPERF_TABLE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(__KERNEL__)
+#include "rgxdevice.h"
+#endif
+/*****************************************************************************/
+
+/* Forward declaration */
+typedef struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_ RGXFW_HWPERF_CNTBLK_TYPE_MODEL;
+
+/* Function pointer type for functions to check dynamic power state of
+ * counter block instance. Used only in firmware. */
+typedef bool (*PFN_RGXFW_HWPERF_CNTBLK_POWERED)(
+		RGX_HWPERF_CNTBLK_ID eBlkType,
+		IMG_UINT8 ui8UnitId);
+
+#if defined(__KERNEL__)
+/* Counter block run-time info */
+typedef struct
+{
+	IMG_UINT32 ui32IndirectReg;          /* 0 if direct type otherwise the indirect control register to select indirect unit */
+	IMG_UINT32 ui32NumUnits;             /* Number of instances of this block type in the core */
+} RGX_HWPERF_CNTBLK_RT_INFO;
+#endif
+
+/* Function pointer type for functions to check block is valid and present
+ * on that RGX Device at runtime. It may have compile logic or run-time
+ * logic depending on where the code executes: server, srvinit or firmware.
+ * Values in the psRtInfo output parameter are only valid if true returned.
+ */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_PRESENT)(
+		const struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_* psBlkTypeDesc,
+		const void *pvDev_km,
+		void *pvRtInfo);
+
+/* This structure encodes properties of a type of performance counter block.
+ * The structure is sometimes referred to as a block type descriptor. These
+ * properties contained in this structure represent the columns in the block
+ * type model table variable below. These values vary depending on the build
+ * BVNC and core type.
+ * Each direct block has a unique type descriptor and each indirect group has
+ * a type descriptor.
+ */
+struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_
+{
+	/* Could use RGXFW_ALIGN_DCACHEL here but then we would waste 40% of the cache line? */
+	IMG_UINT32 ui32CntBlkIdBase;         /* The starting block id for this block type */
+	IMG_UINT32 ui32IndirectReg;          /* 0 if direct type otherwise the indirect control register to select indirect unit */
+	IMG_UINT32 ui32PerfReg;              /* RGX_CR_*_PERF register for this block type */
+	IMG_UINT32 ui32Select0BaseReg;       /* RGX_CR_*_PERF_SELECT0 register for this block type */
+	IMG_UINT32 ui32Counter0BaseReg;      /* RGX_CR_*_PERF_COUNTER_0 register for this block type */
+	IMG_UINT8  ui8NumCounters;          /* Number of counters in this block type */
+	IMG_UINT8  ui8NumUnits;             /* Number of instances of this block type in the core */
+	IMG_UINT8  ui8SelectRegModeShift;   /* Mode field shift value of select registers */
+	IMG_UINT8  ui8SelectRegOffsetShift; /* Interval between select registers, either 8 bytes or 16, hence << 3 or << 4 */
+	const IMG_CHAR *pszBlockNameComment;             /* Name of the PERF register. Used while dumping the perf counters to pdumps */
+	PFN_RGXFW_HWPERF_CNTBLK_POWERED pfnIsBlkPowered; /* A function to determine dynamic power state for the block type */
+	PFN_RGXFW_HWPERF_CNTBLK_PRESENT pfnIsBlkPresent; /* A function to determine presence on RGX Device at run-time */
+};
+
+/*****************************************************************************/
+
+IMG_INTERNAL IMG_UINT32 RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel);
+
+#endif /* RGX_HWPERF_TABLE_H */
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/allocmem.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/allocmem.h
new file mode 100644
index 000000000000..3de9e6781fbd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/allocmem.h
@@ -0,0 +1,224 @@
+/*************************************************************************/ /*!
+@File           allocmem.h
+@Title          memory allocation header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory-Allocation API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef ALLOCMEM_H
+#define ALLOCMEM_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * PVRSRV_ENABLE_PROCESS_STATS enables process statistics regarding events,
+ *     resources and memory across all processes
+ * PVRSRV_ENABLE_MEMORY_STATS enables recording of Linux kernel memory
+ *     allocations, provided that PVRSRV_ENABLE_PROCESS_STATS is enabled
+ *   - Output can be found in:
+ *     /(sys/kernel/debug|proc)/pvr/proc_stats/[live|retired]_pids_stats/mem_area
+ * PVRSRV_DEBUG_LINUX_MEMORY_STATS provides more details about memory
+ *     statistics in conjunction with PVRSRV_ENABLE_MEMORY_STATS
+ * PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON is defined to encompass both memory
+ *     allocation statistics functionalities described above in a single macro
+ */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS) && defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) && defined(DEBUG)
+#define PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+#endif
+
+/*
+ * When using detailed memory allocation statistics, the line number and
+ * file name where the allocation happened are also provided.
+ * When this feature is not used, these parameters are not needed.
+ */
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+#define DEBUG_MEMSTATS_PARAMS ,void *pvAllocFromFile, IMG_UINT32 ui32AllocFromLine
+#define DEBUG_MEMSTATS_ARGS   ,pvAllocFromFile, ui32AllocFromLine
+#define DEBUG_MEMSTATS_UNREF  (void)pvAllocFromFile; (void)ui32AllocFromLine;
+#define DEBUG_MEMSTATS_VALUES ,__FILE__, __LINE__
+#else
+#define DEBUG_MEMSTATS_PARAMS /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_ARGS   /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_UNREF  /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_VALUES  /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#endif
+
+
+/**************************************************************************/ /*!
+@Function       OSAllocMem
+@Description    Allocates CPU memory. Contents are uninitialized.
+                If passed a size of zero, function should not assert,
+                but just return a NULL pointer.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+#if defined(DOXYGEN)
+void *OSAllocMem(IMG_UINT32 ui32Size);
+#else
+void *OSAllocMem(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS);
+#define OSAllocMem(_size)	(OSAllocMem)((_size) DEBUG_MEMSTATS_VALUES)
+#endif
+
+/**************************************************************************/ /*!
+@Function       OSAllocZMem
+@Description    Allocates CPU memory and initializes the contents to zero.
+                If passed a size of zero, function should not assert,
+                but just return a NULL pointer.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+#if defined(DOXYGEN)
+void *OSAllocZMem(IMG_UINT32 ui32Size);
+#else
+void *OSAllocZMem(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS);
+#define OSAllocZMem(_size)	(OSAllocZMem)((_size) DEBUG_MEMSTATS_VALUES)
+#endif
+
+
+/**************************************************************************/ /*!
+@Function       OSAllocMemNoStats
+@Description    Allocates CPU memory. Contents are uninitialized.
+                 If passed a size of zero, function should not assert,
+                 but just return a NULL pointer.
+                 The allocated memory is not accounted for by process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSAllocMem() and OSAllocMemNoStats() equate to
+                 the same operation.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+void *OSAllocMemNoStats(IMG_UINT32 ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSAllocZMemNoStats
+@Description    Allocates CPU memory and initializes the contents to zero.
+                 If passed a size of zero, function should not assert,
+                 but just return a NULL pointer.
+                 The allocated memory is not accounted for by process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSAllocZMem() and OSAllocZMemNoStats() equate to
+                 the same operation.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+void *OSAllocZMemNoStats(IMG_UINT32 ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSFreeMem
+@Description    Frees previously allocated CPU memory.
+@Input          pvCpuVAddr       Pointer to the memory to be freed.
+@Return         None.
+ */ /**************************************************************************/
+void OSFreeMem(void *pvCpuVAddr);
+
+/**************************************************************************/ /*!
+@Function       OSFreeMemNoStats
+@Description    Frees previously allocated CPU memory.
+                 The freed memory does not update the figures in process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSFreeMem() and OSFreeMemNoStats() equate to the
+                 same operation.
+@Input          pvCpuVAddr       Pointer to the memory to be freed.
+@Return         None.
+ */ /**************************************************************************/
+void OSFreeMemNoStats(void *pvCpuVAddr);
+
+/*
+ * These macros allow us to catch double-free bugs on DEBUG builds and
+ * prevent crashes on RELEASE builds.
+ */
+
+/*! @cond Doxygen_Suppress */
+#if defined(DEBUG)
+#define double_free_sentinel ((void *)&OSFreeMem)
+#define ALLOCMEM_ASSERT(exp) PVR_ASSERT(exp)
+#else
+#define double_free_sentinel NULL
+#define ALLOCMEM_ASSERT(exp) do {} while (0)
+#endif
+/*! @endcond */
+
+/*! Frees memory allocated by OSAllocMem(). */
+#define OSFreeMem(_ptr) do { \
+		ALLOCMEM_ASSERT((_ptr) != double_free_sentinel); \
+		(OSFreeMem)(_ptr); \
+		(_ptr) = double_free_sentinel; \
+		MSC_SUPPRESS_4127 \
+	} while (0)
+
+/*! Frees memory allocated by OSAllocMemNoStats(). */
+#define OSFreeMemNoStats(_ptr) do { \
+		ALLOCMEM_ASSERT((_ptr) != double_free_sentinel); \
+		(OSFreeMemNoStats)(_ptr); \
+		(_ptr) = double_free_sentinel; \
+		MSC_SUPPRESS_4127 \
+	} while (0)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* ALLOCMEM_H */
+
+/******************************************************************************
+ End of file (allocmem.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/device_connection.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/device_connection.h
new file mode 100644
index 000000000000..b80983c2d87d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/device_connection.h
@@ -0,0 +1,130 @@
+/*************************************************************************/ /*!
+@File           device_connection.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DEVICE_CONNECTION_H)
+#define DEVICE_CONNECTION_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(__KERNEL__)
+typedef struct _PVRSRV_DEVICE_NODE_ *SHARED_DEV_CONNECTION;
+#else
+#include "connection.h"
+typedef const struct PVRSRV_DEV_CONNECTION_TAG *SHARED_DEV_CONNECTION;
+#endif
+
+/******************************************************************************
+ * Device capability flags and masks
+ *
+ * Following bitmask shows allocated ranges and values for our device
+ * capability settings:
+ *
+ * 31 27  23  19  15  11   7   3  0
+ * |...|...|...|...|...|...|...|...
+ *                               ** CACHE_COHERENT                   [0x1..0x2]
+ *                                x  PVRSRV_CACHE_COHERENT_DEVICE_FLAG
+ *                               x.  PVRSRV_CACHE_COHERENT_CPU_FLAG
+ *                             *... NONMAPPABLE_MEMORY                    [0x8]
+ *                             x...  PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG
+ *                            *.... PDUMP_IS_RECORDING                   [0x10]
+ *                            x....  PVRSRV_PDUMP_IS_RECORDING
+ *                      ***........ DEVMEM_SVM_ALLOC             [0x100..0x400]
+ *                        x........  PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED
+ *                       x.........  PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED
+ *                      x..........  PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL
+ *                     *........... FBCDC_V3_1             [0x800]
+ *                     x...........  FBCDC_V3_1_USED
+ *                    *............ PVRSRV_SYSTEM_DMA
+ *                    x............  PVRSRV_SYSTEM_DMA_USED
+ *                   *............. TFBC_LOSSY_GROUP
+ *                   x.............  TFBC_LOSSY_GROUP_1
+ * |...|...|...|...|...|...|...|...
+ *****************************************************************************/
+
+/* Flag to be passed over the bridge during connection stating whether CPU cache coherent is available*/
+#define PVRSRV_CACHE_COHERENT_SHIFT (0)
+#define	PVRSRV_CACHE_COHERENT_DEVICE_FLAG (1U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define	PVRSRV_CACHE_COHERENT_CPU_FLAG (2U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define	PVRSRV_CACHE_COHERENT_EMULATE_FLAG (4U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define PVRSRV_CACHE_COHERENT_MASK (7U << PVRSRV_CACHE_COHERENT_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether CPU non-mappable memory is present */
+#define PVRSRV_NONMAPPABLE_MEMORY_PRESENT_SHIFT (7)
+#define PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG (1U << PVRSRV_NONMAPPABLE_MEMORY_PRESENT_SHIFT)
+
+/* Flag to be passed over the bridge to indicate PDump activity */
+#define PVRSRV_PDUMP_IS_RECORDING_SHIFT (4)
+#define PVRSRV_PDUMP_IS_RECORDING (1U << PVRSRV_PDUMP_IS_RECORDING_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating SVM allocation availability */
+#define PVRSRV_DEVMEM_SVM_ALLOC_SHIFT (8)
+#define PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED (1U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+#define PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED (2U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+#define PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL (4U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether GPU uses FBCDC v3.1 */
+#define PVRSRV_FBCDC_V3_1_USED_SHIFT (11)
+#define PVRSRV_FBCDC_V3_1_USED (1U << PVRSRV_FBCDC_V3_1_USED_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether System has
+   DMA transfer capability to and from device memory */
+#define PVRSRV_SYSTEM_DMA_SHIFT (12)
+#define PVRSRV_SYSTEM_DMA_USED (1U << PVRSRV_SYSTEM_DMA_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether GPU supports TFBC and is
+   configured to use lossy compression control group 1 (25% / 37.5% / 50%) */
+#define PVRSRV_TFBC_LOSSY_GROUP_SHIFT (13)
+#define PVRSRV_TFBC_LOSSY_GROUP_1 (1U << PVRSRV_TFBC_LOSSY_GROUP_SHIFT)
+
+static INLINE IMG_HANDLE GetBridgeHandle(SHARED_DEV_CONNECTION hDevConnection)
+{
+#if defined(__KERNEL__)
+    return hDevConnection;
+#else
+    return hDevConnection->hServices;
+#endif
+}
+
+
+#endif /* !defined(DEVICE_CONNECTION_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem.h
new file mode 100644
index 000000000000..0a7d550bf194
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem.h
@@ -0,0 +1,694 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management core internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to core device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVCLIENT_DEVICEMEM_H
+#define SRVCLIENT_DEVICEMEM_H
+
+/******************************************************************************
+ *                                                                            *
+ *  +------------+   +------------+      +--------------+   +--------------+  *
+ *  | a   sub-   |   | a   sub-   |      |  an          |   | allocation   |  *
+ *  | allocation |   | allocation |      |  allocation  |   | also mapped  |  *
+ *  |            |   |            |      |  in proc 1   |   | into proc 2  |  *
+ *  +------------+   +------------+      +--------------+   +--------------+  *
+ *            |         |                       |                  |          *
+ *         +--------------+              +--------------+   +--------------+  *
+ *         | page   gran- |              | page   gran- |   | page   gran- |  *
+ *         | ular mapping |              | ular mapping |   | ular mapping |  *
+ *         +--------------+              +--------------+   +--------------+  *
+ *                |                                   |       |               *
+ *                |                                   |       |               *
+ *                |                                   |       |               *
+ *         +--------------+                       +--------------+            *
+ *         |              |                       |              |            *
+ *         | A  "P.M.R."  |                       | A  "P.M.R."  |            *
+ *         |              |                       |              |            *
+ *         +--------------+                       +--------------+            *
+ *                                                                            *
+ ******************************************************************************/
+
+/*
+    All device memory allocations are ultimately a view upon (not
+    necessarily the whole of) a "PMR".
+
+    A PMR is a "Physical Memory Resource", which may be a
+    "pre-faulted" lump of physical memory, or it may be a
+    representation of some physical memory that will be instantiated
+    at some future time.
+
+    PMRs always represent multiple of some power-of-2 "contiguity"
+    promised by the PMR, which will allow them to be mapped in whole
+    pages into the device MMU.  As memory allocations may be smaller
+    than a page, these mappings may be suballocated and thus shared
+    between multiple allocations in one process.  A PMR may also be
+    mapped simultaneously into multiple device memory contexts
+    (cross-process scenario), however, for security reasons, it is not
+    legal to share a PMR "both ways" at once, that is, mapped into
+    multiple processes and divided up amongst several suballocations.
+
+    This PMR terminology is introduced here for background
+    information, but is generally of little concern to the caller of
+    this API.  This API handles suballocations and mappings, and the
+    caller thus deals primarily with MEMORY DESCRIPTORS representing
+    an allocation or suballocation, HEAPS representing ranges of
+    virtual addresses in a CONTEXT.
+*/
+
+/*
+   |<---------------------------context------------------------------>|
+   |<-------heap------->|   |<-------heap------->|<-------heap------->|
+   |<-alloc->|          |   |<-alloc->|<-alloc->||   |<-alloc->|      |
+*/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "pdump.h"
+
+#include "device_connection.h"
+
+
+typedef IMG_UINT32 DEVMEM_HEAPCFGID;
+#define DEVMEM_HEAPCFG_FORCLIENTS 0
+#define DEVMEM_HEAPCFG_FORFW 1
+
+
+/*
+  In order to call the server side functions, we need a bridge handle.
+  We abstract that here, as we may wish to change its form.
+ */
+
+typedef IMG_HANDLE DEVMEM_BRIDGE_HANDLE;
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
+				 IMG_HANDLE *phDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetSize(DEVMEM_MEMDESC *psMemDesc,
+			  IMG_DEVMEM_SIZE_T* puiSize);
+
+IMG_INTERNAL void
+DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc,
+			        IMG_CHAR **pszAnnotation);
+
+/*
+ * DevmemCreateContext()
+ *
+ * Create a device memory context
+ *
+ * This must be called before any heap is created in this context
+ *
+ * Caller to provide bridge handle which will be recorded internally and used
+ * for all future operations on items from this memory context.  Caller also
+ * to provide devicenode handle, as this is used for MMU configuration and
+ * also to determine the heap configuration for the auto-instantiated heaps.
+ *
+ * Note that when compiled in services/server, the hBridge is not used and
+ * is thrown away by the "fake" direct bridge.  (This may change. It is
+ * recommended that NULL be passed for the handle for now.)
+ *
+ * hDeviceNode and uiHeapBlueprintID shall together dictate which heap-config
+ * to use.
+ *
+ * This will cause the server side counterpart to be created also.
+ *
+ * If you call DevmemCreateContext() (and the call succeeds) you are promising
+ * that you will later call Devmem_ContextDestroy(), except for abnormal
+ * process termination in which case it is expected it will be destroyed as
+ * part of handle clean up.
+ *
+ * Caller to provide storage for the pointer to the newly created
+ * NEWDEVMEM_CONTEXT object.
+ */
+PVRSRV_ERROR
+DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
+                    DEVMEM_HEAPCFGID uiHeapBlueprintID,
+                    DEVMEM_CONTEXT **ppsCtxPtr);
+
+/*
+ * DevmemAcquireDevPrivData()
+ *
+ * Acquire the device private data for this memory context
+ */
+PVRSRV_ERROR
+DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
+                         IMG_HANDLE *hPrivData);
+
+/*
+ * DevmemReleaseDevPrivData()
+ *
+ * Release the device private data for this memory context
+ */
+PVRSRV_ERROR
+DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx);
+
+/*
+ * DevmemDestroyContext()
+ *
+ * Undoes that done by DevmemCreateContext()
+ */
+PVRSRV_ERROR
+DevmemDestroyContext(DEVMEM_CONTEXT *psCtx);
+
+/*
+ * DevmemCreateHeap()
+ *
+ * Create a heap in the given context.
+ *
+ * N.B.  Not intended to be called directly, though it can be.
+ * Normally, heaps are instantiated at context creation time according
+ * to the specified blueprint.  See DevmemCreateContext() for details.
+ *
+ * This will cause MMU code to set up data structures for the heap,
+ * but may not cause page tables to be modified until allocations are
+ * made from the heap.
+ *
+ * uiReservedRegionLength Reserved address space for static VAs shared
+ * between clients and firmware
+ *
+ * The "Quantum" is both the device MMU page size to be configured for
+ * this heap, and the unit multiples of which "quantized" allocations
+ * are made (allocations smaller than this, known as "suballocations"
+ * will be made from a "sub alloc RA" and will "import" chunks
+ * according to this quantum)
+ *
+ * Where imported PMRs (or, for example, PMRs created by device class
+ * buffers) are mapped into this heap, it is important that the
+ * physical contiguity guarantee offered by the PMR is greater than or
+ * equal to the quantum size specified here, otherwise the attempt to
+ * map it will fail.  "Normal" allocations via Devmem_Allocate
+ * shall automatically meet this requirement, as each "import" will
+ * trigger the creation of a PMR with the desired contiguity.  The
+ * supported quantum sizes in that case shall be dictated by the OS
+ * specific implementation of PhysmemNewOSRamBackedPMR() (see)
+ */
+PVRSRV_ERROR
+DevmemCreateHeap(DEVMEM_CONTEXT *psCtxPtr,
+                 /* base and length of heap */
+                 IMG_DEV_VIRTADDR sBaseAddress,
+                 IMG_DEVMEM_SIZE_T uiLength,
+                 IMG_DEVMEM_SIZE_T uiReservedRegionLength,
+                 /* log2 of allocation quantum, i.e. "page" size.
+                    All allocations (that go to server side) are
+                    multiples of this.  We use a client-side RA to
+                    make sub-allocations from this */
+                 IMG_UINT32 ui32Log2Quantum,
+                 /* The minimum import alignment for this heap */
+                 IMG_UINT32 ui32Log2ImportAlignment,
+                 /* Name of heap for debug */
+                 /* N.B.  Okay to exist on caller's stack - this
+                    func takes a copy if it needs it. */
+                 const IMG_CHAR *pszName,
+                 DEVMEM_HEAPCFGID uiHeapBlueprintID,
+                 IMG_UINT32 uiHeapIndex,
+                 DEVMEM_HEAP **ppsHeapPtr);
+/*
+ * DevmemDestroyHeap()
+ *
+ * Reverses DevmemCreateHeap()
+ *
+ * N.B. All allocations must have been freed and all mappings must
+ * have been unmapped before invoking this call
+ */
+PVRSRV_ERROR
+DevmemDestroyHeap(DEVMEM_HEAP *psHeap);
+
+/*
+ * DevmemExportalignAdjustSizeAndAlign()
+ * Compute the Size and Align passed to avoid suballocations
+ * (used when allocation with PVRSRV_MEMALLOCFLAG_EXPORTALIGN).
+ *
+ * Returns PVRSRV_ERROR_INVALID_PARAMS if uiLog2Quantum has invalid value.
+ */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
+                                    IMG_DEVMEM_SIZE_T *puiSize,
+                                    IMG_DEVMEM_ALIGN_T *puiAlign);
+
+/*
+ * DevmemSubAllocate()
+ *
+ * Makes an allocation (possibly a "suballocation", as described
+ * below) of device virtual memory from this heap.
+ *
+ * The size and alignment of the allocation will be honoured by the RA
+ * that allocates the "suballocation".  The resulting allocation will
+ * be mapped into GPU virtual memory and the physical memory to back
+ * it will exist, by the time this call successfully completes.
+ *
+ * The size must be a positive integer multiple of the alignment.
+ * (i.e. the alignment specifies the alignment of both the start and
+ * the end of the resulting allocation.)
+ *
+ * Allocations made via this API are routed through a "suballocation
+ * RA" which is responsible for ensuring that small allocations can be
+ * made without wasting physical memory in the server.  Furthermore,
+ * such suballocations can be made entirely client side without
+ * needing to go to the server unless the allocation spills into a new
+ * page.
+ *
+ * Such suballocations cause many allocations to share the same "PMR".
+ * This happens only when the flags match exactly.
+ *
+ */
+
+PVRSRV_ERROR
+DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
+                  DEVMEM_HEAP *psHeap,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_DEVMEM_ALIGN_T uiAlign,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  const IMG_CHAR *pszText,
+                  DEVMEM_MEMDESC **ppsMemDescPtr);
+
+#define DevmemAllocate(...) \
+    DevmemSubAllocate(DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER, __VA_ARGS__)
+
+PVRSRV_ERROR
+DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         IMG_DEVMEM_ALIGN_T uiAlign,
+                         IMG_UINT32 uiLog2HeapPageSize,
+                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                         const IMG_CHAR *pszText,
+                         DEVMEM_MEMDESC **ppsMemDescPtr);
+
+PVRSRV_ERROR
+DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *paui32AllocPageIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pauiFreePageIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiFlags);
+
+PVRSRV_ERROR
+DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
+                     IMG_DEVMEM_SIZE_T uiSize,
+                     IMG_UINT32 ui32NumPhysChunks,
+                     IMG_UINT32 ui32NumVirtChunks,
+                     IMG_UINT32 *pui32MappingTable,
+                     IMG_DEVMEM_ALIGN_T uiAlign,
+                     IMG_UINT32 uiLog2HeapPageSize,
+                     PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                     const IMG_CHAR *pszText,
+                     DEVMEM_MEMDESC **ppsMemDescPtr);
+
+PVRSRV_ERROR
+DevmemSubAllocateAndMap(IMG_UINT8 uiPreAllocMultiplier,
+			DEVMEM_HEAP *psHeap,
+			IMG_DEVMEM_SIZE_T uiSize,
+			IMG_DEVMEM_ALIGN_T uiAlign,
+			PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			const IMG_CHAR *pszText,
+			DEVMEM_MEMDESC **ppsMemDescPtr,
+			IMG_DEV_VIRTADDR *psDevVirtAddr);
+
+#define DevmemAllocateAndMap(...) \
+	DevmemSubAllocateAndMap(DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER, __VA_ARGS__)
+
+/*
+ * DevmemFree()
+ *
+ * Reverses that done by DevmemSubAllocate() N.B.  The underlying
+ * mapping and server side allocation _may_ not be torn down, for
+ * example, if the allocation has been exported, or if multiple
+ * allocations were suballocated from the same mapping, but this is
+ * properly refcounted, so the caller does not have to care.
+ */
+
+IMG_BOOL
+DevmemFree(DEVMEM_MEMDESC *psMemDesc);
+
+IMG_BOOL
+DevmemReleaseDevAddrAndFree(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+	DevmemMapToDevice:
+
+	Map an allocation to the device it was allocated from.
+	This function _must_ be called before any call to
+	DevmemAcquireDevVirtAddr is made as it binds the allocation
+	to the heap.
+	DevmemReleaseDevVirtAddr is used to release the reference
+	to the device mapping this function created, but it doesn't
+	mean that the memory will actually be unmapped from the
+	device as other references to the mapping obtained via
+	DevmemAcquireDevVirtAddr could still be active.
+*/
+PVRSRV_ERROR DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
+							   DEVMEM_HEAP *psHeap,
+							   IMG_DEV_VIRTADDR *psDevVirtAddr);
+
+/*
+	DevmemMapToDeviceAddress:
+
+	Same as DevmemMapToDevice but the caller chooses the address
+	to map to.
+*/
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
+                         DEVMEM_HEAP *psHeap,
+                         IMG_DEV_VIRTADDR sDevVirtAddr);
+
+/*
+	DevmemGetDevVirtAddr
+
+	Obtain the MemDesc's device virtual address.
+	This function _must_ be called after DevmemMapToDevice(Address)
+	and is expected to be used be functions which didn't allocate
+	the MemDesc but need to know it's address.
+	It will PVR_ASSERT if no device mapping exists and 0 is returned.
+ */
+IMG_DEV_VIRTADDR
+DevmemGetDevVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+	DevmemAcquireDevVirtAddr
+
+	Acquire the MemDesc's device virtual address.
+	This function _must_ be called after DevmemMapToDevice
+	and is expected to be used be functions which didn't allocate
+	the MemDesc but need to know it's address
+ */
+PVRSRV_ERROR DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEV_VIRTADDR *psDevVirtAddrRet);
+
+/*
+ * DevmemReleaseDevVirtAddr()
+ *
+ * give up the licence to use the device virtual address that was
+ * acquired by "Acquire" or "MapToDevice"
+ */
+void
+DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+ * DevmemAcquireCpuVirtAddr()
+ *
+ * Acquires a license to use the cpu virtual address of this mapping.
+ * Note that the memory may not have been mapped into cpu virtual
+ * memory prior to this call.  On first "acquire" the memory will be
+ * mapped in (if it wasn't statically mapped in) and on last put it
+ * _may_ become unmapped.  Later calling "Acquire" again, _may_ cause
+ * the memory to be mapped at a different address.
+ */
+PVRSRV_ERROR DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                      void **ppvCpuVirtAddr);
+
+/*
+ * DevmemReacquireCpuVirtAddr()
+ *
+ * (Re)acquires license to use the cpu virtual address of this mapping
+ * if (and only if) there is already a pre-existing license to use the
+ * cpu virtual address for the mapping, returns NULL otherwise.
+ */
+void DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                void **ppvCpuVirtAddr);
+
+/*
+ * DevmemReleaseDevVirtAddr()
+ *
+ * give up the licence to use the cpu virtual address that was granted
+ * with the "Get" call.
+ */
+void
+DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+/*
+ * DevmemExport()
+ *
+ * Given a memory allocation allocated with DevmemAllocateExportable()
+ * create a "cookie" that can be passed intact by the caller's own choice
+ * of secure IPC to another process and used as the argument to "map"
+ * to map this memory into a heap in the target processes.  N.B.  This can
+ * also be used to map into multiple heaps in one process, though that's not
+ * the intention.
+ *
+ * Note, the caller must later call Unexport before freeing the
+ * memory.
+ */
+PVRSRV_ERROR DevmemExport(DEVMEM_MEMDESC *psMemDesc,
+                          DEVMEM_EXPORTCOOKIE *psExportCookie);
+
+
+void DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
+					DEVMEM_EXPORTCOOKIE *psExportCookie);
+
+PVRSRV_ERROR
+DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
+			 DEVMEM_EXPORTCOOKIE *psCookie,
+			 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			 DEVMEM_MEMDESC **ppsMemDescPtr);
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+/*
+ * DevmemMakeLocalImportHandle()
+ *
+ * This is a "special case" function for making a server export cookie
+ * which went through the direct bridge into an export cookie that can
+ * be passed through the client bridge.
+ */
+PVRSRV_ERROR
+DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+                            IMG_HANDLE hServerExport,
+                            IMG_HANDLE *hClientExport);
+
+/*
+ * DevmemUnmakeLocalImportHandle()
+ *
+ * Free any resource associated with the Make operation
+ */
+PVRSRV_ERROR
+DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+                              IMG_HANDLE hClientExport);
+
+/*
+ *
+ * The following set of functions is specific to the heap "blueprint"
+ * stuff, for automatic creation of heaps when a context is created
+ *
+ */
+
+
+/* Devmem_HeapConfigCount: returns the number of heap configs that
+   this device has.  Note that there is no acquire/release semantics
+   required, as this data is guaranteed to be constant for the
+   lifetime of the device node */
+PVRSRV_ERROR
+DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
+                      IMG_UINT32 *puiNumHeapConfigsOut);
+
+/* Devmem_HeapCount: returns the number of heaps that a given heap
+   config on this device has.  Note that there is no acquire/release
+   semantics required, as this data is guaranteed to be constant for
+   the lifetime of the device node */
+PVRSRV_ERROR
+DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
+                IMG_UINT32 uiHeapConfigIndex,
+                IMG_UINT32 *puiNumHeapsOut);
+/* Devmem_HeapConfigName: return the name of the given heap config.
+   The caller is to provide the storage for the returned string and
+   indicate the number of bytes (including null terminator) for such
+   string in the BufSz arg.  Note that there is no acquire/release
+   semantics required, as this data is guaranteed to be constant for
+   the lifetime of the device node.
+ */
+PVRSRV_ERROR
+DevmemHeapConfigName(SHARED_DEV_CONNECTION hsDevConnection,
+                     IMG_UINT32 uiHeapConfigIndex,
+                     IMG_CHAR *pszConfigNameOut,
+                     IMG_UINT32 uiConfigNameBufSz);
+
+/* Devmem_HeapDetails: fetches all the metadata that is recorded in
+   this heap "blueprint".  Namely: heap name (caller to provide
+   storage, and indicate buffer size (including null terminator) in
+   BufSz arg), device virtual address and length, log2 of data page
+   size (will be one of 12, 14, 16, 18, 20, 21, at time of writing).
+   Note that there is no acquire/release semantics required, as this
+   data is guaranteed to be constant for the lifetime of the device
+   node. */
+PVRSRV_ERROR
+DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
+                  IMG_UINT32 uiHeapConfigIndex,
+                  IMG_UINT32 uiHeapIndex,
+                  IMG_CHAR *pszHeapNameOut,
+                  IMG_UINT32 uiHeapNameBufSz,
+                  IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+                  IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+                  IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+                  IMG_UINT32 *puiLog2DataPageSize,
+                  IMG_UINT32 *puiLog2ImportAlignmentOut);
+
+/*
+ * Devmem_FindHeapByName()
+ *
+ * returns the heap handle for the named _automagic_ heap in this
+ * context.  "automagic" heaps are those that are born with the
+ * context from a blueprint
+ */
+PVRSRV_ERROR
+DevmemFindHeapByName(const DEVMEM_CONTEXT *psCtx,
+                     const IMG_CHAR *pszHeapName,
+                     DEVMEM_HEAP **ppsHeapRet);
+
+/*
+ * DevmemGetHeapBaseDevVAddr()
+ *
+ * returns the device virtual address of the base of the heap.
+ */
+PVRSRV_ERROR
+DevmemGetHeapBaseDevVAddr(DEVMEM_HEAP *psHeap,
+			  IMG_DEV_VIRTADDR *pDevVAddr);
+
+/*
+ * DevmemGetHeapSize()
+ *
+ * returns the size of the heap.
+ */
+IMG_INTERNAL DEVMEM_SIZE_T
+DevmemGetHeapSize(struct DEVMEM_HEAP_TAG *psHeap);
+
+PVRSRV_ERROR
+DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+			   IMG_HANDLE *phImport);
+
+PVRSRV_ERROR
+DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
+		   IMG_UINT64 *pui64UID);
+
+PVRSRV_ERROR
+DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
+		     IMG_HANDLE *hReservation);
+
+IMG_INTERNAL void
+DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *hPMR,
+		IMG_DEVMEM_OFFSET_T *puiPMROffset);
+
+IMG_INTERNAL void
+DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
+				PVRSRV_MEMALLOCFLAGS_T *puiFlags);
+
+IMG_INTERNAL SHARED_DEV_CONNECTION
+DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc);
+
+PVRSRV_ERROR
+DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection,
+				  IMG_HANDLE hExtHandle,
+				  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+				  DEVMEM_MEMDESC **ppsMemDescPtr,
+				  IMG_DEVMEM_SIZE_T *puiSizePtr,
+				  const IMG_CHAR *pszAnnotation);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
+                         IMG_DEV_VIRTADDR sDevVAddr);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext,
+                      IMG_DEV_VIRTADDR *psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemInvalidateFBSCTable(DEVMEM_CONTEXT *psContext,
+                          IMG_UINT64 ui64FBSCEntries);
+
+/* DevmemGetHeapLog2PageSize()
+ *
+ * Get the page size used for a certain heap.
+ */
+IMG_UINT32
+DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap);
+
+/* DevmemGetMemFlags()
+ *
+ * Get the memalloc flags for a certain memdesc.
+ */
+PVRSRV_MEMALLOCFLAGS_T
+DevmemGetMemAllocFlags(DEVMEM_MEMDESC *psMemDesc);
+
+/* DevmemGetHeapReservedSize()
+ *
+ * Get the reserved size used for a certain heap.
+ */
+IMG_DEVMEM_SIZE_T
+DevmemGetHeapReservedSize(DEVMEM_HEAP *psHeap);
+
+/*************************************************************************/ /*!
+@Function       RegisterDevMemPFNotify
+@Description    Registers that the application wants to be signaled when a page
+                fault occurs.
+
+@Input          psContext      Memory context the process that would like to
+                               be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR:  PVRSRV_OK on success. Otherwise, a PVRSRV_
+                               error code
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
+                       IMG_BOOL       bRegister);
+
+/*************************************************************************/ /*!
+@Function       DevmemHeapSetPremapStatus
+@Description    In some special cases like virtualisation, a device memory heap
+			    must be entirely backed by physical memory and mapped into the
+				device's virtual address space. This is done at context creation.
+			    When objects are allocated from such a heap, the mapping part
+			    must be skipped. The 'bPremapped' flag dictates if allocations
+			    are to be mapped or not.
+
+@Input          psHeap            Device memory heap to be updated
+@Input          IsPremapped       The premapping status to be set
+*/ /**************************************************************************/
+IMG_INTERNAL void
+DevmemHeapSetPremapStatus(DEVMEM_HEAP *psHeap, IMG_BOOL IsPremapped);
+
+#endif /* #ifndef SRVCLIENT_DEVICEMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_pdump.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_pdump.h
new file mode 100644
index 000000000000..09b28afe7e51
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_pdump.h
@@ -0,0 +1,363 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management PDump internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to PDump device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_PDUMP_H
+#define DEVICEMEM_PDUMP_H
+
+#include "devicemem.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+
+#if defined(PDUMP)
+/*
+ * DevmemPDumpLoadMem()
+ *
+ * takes a memory descriptor, offset, and size, and takes the current contents
+ * of the memory at that location and writes it to the prm pdump file, and
+ * emits a pdump LDB to load the data from that file.  The intention here is
+ * that the contents of the simulated buffer upon pdump playback will be made
+ * to be the same as they are when this command is run, enabling pdump of
+ * cases where the memory has been modified externally, i.e. by the host cpu
+ * or by a third party.
+ */
+void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpLoadZeroMem()
+ *
+ * As DevmemPDumpLoadMem() but the PDump allocation will be populated with
+ * zeros from the zero page in the parameter stream
+ */
+void
+DevmemPDumpLoadZeroMem(DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpLoadMemValue32()
+ *
+ * As above but dumps the value at a dword-aligned address in plain text to
+ * the pdump script2 file. Useful for patching a buffer at pdump playback by
+ * simply editing the script output file.
+ *
+ * (The same functionality can be achieved by the above function but the
+ *  binary PARAM file must be patched in that case.)
+ */
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpMemValue64()
+ *
+ * As above but dumps the 64bit-value at a dword-aligned address in plain text
+ * to the pdump script2 file. Useful for patching a buffer at pdump playback by
+ * simply editing the script output file.
+ *
+ * (The same functionality can be achieved by the above function but the
+ *  binary PARAM file must be patched in that case.)
+ */
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpPageCatBaseToSAddr()
+ *
+ * Returns the symbolic address of a piece of memory represented by an offset
+ * into the mem descriptor.
+ */
+PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC *psMemDesc,
+                              IMG_DEVMEM_OFFSET_T *puiMemOffset,
+                              IMG_CHAR *pszName,
+                              IMG_UINT32 ui32Size);
+
+/*
+ * DevmemPDumpSaveToFile()
+ *
+ * Emits a pdump SAB to cause the current contents of the memory to be written
+ * to the given file during playback
+ */
+void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset);
+
+/*
+ * DevmemPDumpSaveToFileVirtual()
+ *
+ * Emits a pdump SAB, just like DevmemPDumpSaveToFile(), but uses the virtual
+ * address and device MMU context to cause the pdump player to traverse the
+ * MMU page tables itself.
+ */
+void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags);
+
+/*
+ * DevmemPDumpDataDescriptor()
+ *
+ * Emits a pdump CMD:OutputData, using the virtual address and device MMU
+ * context. Provides more flexibility than a pdump SAB because metadata can
+ * be passed to an external pdump player library via the command header.
+ */
+void
+DevmemPDumpDataDescriptor(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_DEVMEM_SIZE_T uiSize,
+                          const IMG_CHAR *pszFilename,
+                          IMG_UINT32 ui32HeaderType,
+                          IMG_UINT32 ui32ElementType,
+                          IMG_UINT32 ui32ElementCount,
+                          IMG_UINT32 ui32PdumpFlags);
+
+
+/*
+ *
+ * DevmemPDumpDevmemPol32()
+ *
+ * Writes a PDump 'POL' command to wait for a masked 32-bit memory location to
+ * become the specified value.
+ */
+PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags);
+
+#if defined(__KERNEL__)
+/*
+ *
+ * DevmemPDumpDevmemCheck32()
+ *
+ * Writes a PDump 'POL' command to run a single-shot check for a masked
+ * 32-bit memory location to match the specified value.
+ */
+PVRSRV_ERROR
+DevmemPDumpDevmemCheck32(const DEVMEM_MEMDESC *psMemDesc,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_UINT32 ui32Value,
+                         IMG_UINT32 ui32Mask,
+                         PDUMP_POLL_OPERATOR eOperator,
+                         PDUMP_FLAGS_T ui32PDumpFlags);
+#endif
+
+/*
+ * DevmemPDumpCBP()
+ *
+ * Polls for space in circular buffer. Reads the read offset from memory and
+ * waits until there is enough space to write the packet.
+ *
+ * psMemDesc     - MemDesc which contains the read offset
+ * uiReadOffset  - Offset into MemDesc to the read offset
+ * uiWriteOffset - Current write offset
+ * uiPacketSize  - Size of packet to write
+ * uiBufferSize  - Size of circular buffer
+ */
+PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize);
+
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMem)
+#endif
+static INLINE void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMemValue32)
+#endif
+static INLINE void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMemValue64)
+#endif
+static INLINE void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui64Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpPageCatBaseToSAddr)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC *psMemDesc,
+                              IMG_DEVMEM_OFFSET_T *puiMemOffset,
+                              IMG_CHAR *pszName,
+                              IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(puiMemOffset);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpSaveToFile)
+#endif
+static INLINE void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(uiFileOffset);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpSaveToFileVirtual)
+#endif
+static INLINE void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PdumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpDevmemPol32)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpCBP)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+#endif /* DEVICEMEM_PDUMP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_utils.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_utils.h
new file mode 100644
index 000000000000..9e24c1dceca7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_utils.h
@@ -0,0 +1,582 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management internal utility functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally by device memory management
+                code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_UTILS_H
+#define DEVICEMEM_UTILS_H
+
+#include "devicemem.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "osmmap.h"
+#include "pvrsrv_memallocflags_internal.h"
+
+#define DEVMEM_HEAPNAME_MAXLENGTH 160
+
+/*
+ * VA heap size should be at least OS page size. This check is validated in the DDK.
+ */
+#define DEVMEM_HEAP_MINIMUM_SIZE                     0x10000 /* 64KB is MAX anticipated OS page size */
+
+#if defined(DEVMEM_DEBUG) && defined(REFCOUNT_DEBUG)
+#define DEVMEM_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_ERROR, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define DEVMEM_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+/* If we need a "hMapping" but we don't have a server-side mapping, we poison
+ * the entry with this value so that it's easily recognised in the debugger.
+ * Note that this is potentially a valid handle, but then so is NULL, which is
+ * no better, indeed worse, as it's not obvious in the debugger. The value
+ * doesn't matter. We _never_ use it (and because it's valid, we never assert
+ * it isn't this) but it's nice to have a value in the source code that we can
+ * grep for if things go wrong.
+ */
+#define LACK_OF_MAPPING_POISON ((IMG_HANDLE)0x6116dead)
+#define LACK_OF_RESERVATION_POISON ((IMG_HANDLE)0x7117dead)
+
+#define DEVICEMEM_HISTORY_ALLOC_INDEX_NONE 0xFFFFFFFF
+
+struct DEVMEM_CONTEXT_TAG
+{
+
+	SHARED_DEV_CONNECTION hDevConnection;
+
+	/* Number of heaps that have been created in this context
+	 * (regardless of whether they have allocations)
+	 */
+	IMG_UINT32 uiNumHeaps;
+
+	/* Each "DEVMEM_CONTEXT" has a counterpart in the server, which
+	 * is responsible for handling the mapping into device MMU.
+	 * We have a handle to that here.
+	 */
+	IMG_HANDLE hDevMemServerContext;
+
+	/* Number of automagically created heaps in this context,
+	 *  i.e. those that are born at context creation time from the
+	 * chosen "heap config" or "blueprint"
+	 */
+	IMG_UINT32 uiAutoHeapCount;
+
+	/* Pointer to array of such heaps */
+	struct DEVMEM_HEAP_TAG **ppsAutoHeapArray;
+
+	/* The cache line size for use when allocating memory,
+	 * as it is not queryable on the client side
+	 */
+	IMG_UINT32 ui32CPUCacheLineSize;
+
+	/* Private data handle for device specific data */
+	IMG_HANDLE hPrivData;
+};
+
+/* Flags that record how a heaps virtual address space is managed. */
+#define DEVMEM_HEAP_MANAGER_UNKNOWN      0
+/* Heap VAs assigned by the client of Services APIs, heap's RA not used at all. */
+#define DEVMEM_HEAP_MANAGER_USER         (1U << 0)
+/* Heap VAs managed by the OSs kernel, VA from CPU mapping call used */
+#define DEVMEM_HEAP_MANAGER_KERNEL       (1U << 1)
+/* Heap VAs managed by the heap's own RA */
+#define DEVMEM_HEAP_MANAGER_RA           (1U << 2)
+/* Heap VAs managed jointly by Services and the client of Services.
+ * The reserved region of the heap is managed explicitly by the client of Services
+ * The non-reserved region of the heap is managed by the heap's own RA */
+#define DEVMEM_HEAP_MANAGER_DUAL_USER_RA (DEVMEM_HEAP_MANAGER_USER | DEVMEM_HEAP_MANAGER_RA)
+
+struct DEVMEM_HEAP_TAG
+{
+	/* Name of heap - for debug and lookup purposes. */
+	IMG_CHAR *pszName;
+
+	/* Number of live imports in the heap */
+	ATOMIC_T hImportCount;
+
+	/* Base address and size of heap, required by clients due to some
+	 * requesters not being full range
+	 */
+	IMG_DEV_VIRTADDR sBaseAddress;
+	DEVMEM_SIZE_T uiSize;
+
+	DEVMEM_SIZE_T uiReservedRegionSize; /* uiReservedRegionLength in DEVMEM_HEAP_BLUEPRINT */
+
+	/* The heap manager, describing if the space is managed by the user, an RA,
+	 * kernel or combination */
+	IMG_UINT32 ui32HeapManagerFlags;
+
+	/* This RA is for managing sub-allocations within the imports (PMRs)
+	 * within the heap's virtual space. RA only used in DevmemSubAllocate()
+	 * to track sub-allocated buffers.
+	 *
+	 * Resource Span - a PMR import added when the RA calls the
+	 *                 imp_alloc CB (SubAllocImportAlloc) which returns the
+	 *                 PMR import and size (span length).
+	 * Resource - an allocation/buffer i.e. a MemDesc. Resource size represents
+	 *            the size of the sub-allocation.
+	 */
+	RA_ARENA *psSubAllocRA;
+	IMG_CHAR *pszSubAllocRAName;
+
+	/* The psQuantizedVMRA is for the coarse allocation (PMRs) of virtual
+	 * space from the heap.
+	 *
+	 * Resource Span - the heap's VM space from base to base+length,
+	 *                 only one is added at heap creation.
+	 * Resource - a PMR import associated with the heap. Dynamic number
+	 *            as memory is allocated/freed from or mapped/unmapped to
+	 *            the heap. Resource size follows PMR logical size.
+	 */
+	RA_ARENA *psQuantizedVMRA;
+	IMG_CHAR *pszQuantizedVMRAName;
+
+	/* We also need to store a copy of the quantum size in order to feed
+	 * this down to the server.
+	 */
+	IMG_UINT32 uiLog2Quantum;
+
+	/* Store a copy of the minimum import alignment */
+	IMG_UINT32 uiLog2ImportAlignment;
+
+	/* The parent memory context for this heap */
+	struct DEVMEM_CONTEXT_TAG *psCtx;
+
+	/* Lock to protect this structure */
+	POS_LOCK hLock;
+
+	/* Each "DEVMEM_HEAP" has a counterpart in the server, which is
+	 * responsible for handling the mapping into device MMU.
+	 * We have a handle to that here.
+	 */
+	IMG_HANDLE hDevMemServerHeap;
+
+	/* This heap is fully allocated and premapped into the device address space.
+	 * Used in virtualisation for firmware heaps of Guest and optionally Host
+	 * drivers. */
+	IMG_BOOL bPremapped;
+};
+
+typedef IMG_UINT32 DEVMEM_PROPERTIES_T;                  /*!< Typedef for Devicemem properties */
+#define DEVMEM_PROPERTIES_EXPORTABLE         (1UL<<0)    /*!< Is it exportable? */
+#define DEVMEM_PROPERTIES_IMPORTED           (1UL<<1)    /*!< Is it imported from another process? */
+#define DEVMEM_PROPERTIES_SUBALLOCATABLE     (1UL<<2)    /*!< Is it suballocatable? */
+#define DEVMEM_PROPERTIES_IMPORT_IS_ZEROED   (1UL<<4)    /*!< Is the memory fully zeroed? */
+#define DEVMEM_PROPERTIES_IMPORT_IS_CLEAN    (1UL<<5)    /*!< Is the memory clean, i.e. not been used before? */
+#define DEVMEM_PROPERTIES_SECURE             (1UL<<6)    /*!< Is it a special secure buffer? No CPU maps allowed! */
+#define DEVMEM_PROPERTIES_IMPORT_IS_POISONED (1UL<<7)    /*!< Is the memory fully poisoned? */
+#define DEVMEM_PROPERTIES_NO_CPU_MAPPING     (1UL<<8)    /* No CPU Mapping is allowed, RW attributes
+                                                            are further derived from allocation memory flags */
+#define DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE	 (1UL<<9)    /* No sparse resizing allowed, once a memory
+                                                            layout is chosen, no change allowed later */
+
+
+typedef struct DEVMEM_DEVICE_IMPORT_TAG
+{
+	DEVMEM_HEAP *psHeap;            /*!< Heap this import is bound to */
+	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the import */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
+	IMG_HANDLE hReservation;        /*!< Device memory reservation handle */
+	IMG_HANDLE hMapping;            /*!< Device mapping handle */
+	IMG_BOOL bMapped;               /*!< This is import mapped? */
+	POS_LOCK hLock;                 /*!< Lock to protect the device import */
+} DEVMEM_DEVICE_IMPORT;
+
+typedef struct DEVMEM_CPU_IMPORT_TAG
+{
+	void *pvCPUVAddr;               /*!< CPU virtual address of the import */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the CPU virtual address */
+	IMG_HANDLE hOSMMapData;         /*!< CPU mapping handle */
+	POS_LOCK hLock;                 /*!< Lock to protect the CPU import */
+} DEVMEM_CPU_IMPORT;
+
+typedef struct DEVMEM_IMPORT_TAG
+{
+	SHARED_DEV_CONNECTION hDevConnection;
+	IMG_DEVMEM_ALIGN_T uiAlign;         /*!< Alignment of the PMR */
+	DEVMEM_SIZE_T uiSize;               /*!< Size of import */
+	ATOMIC_T hRefCount;                 /*!< Refcount for this import */
+	DEVMEM_PROPERTIES_T uiProperties;   /*!< Stores properties of an import like if
+	                                         it is exportable, pinned or suballocatable */
+	IMG_HANDLE hPMR;                    /*!< Handle to the PMR */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;     /*!< Flags for this import */
+	POS_LOCK hLock;                     /*!< Lock to protect the import */
+
+	DEVMEM_DEVICE_IMPORT sDeviceImport; /*!< Device specifics of the import */
+	DEVMEM_CPU_IMPORT sCPUImport;       /*!< CPU specifics of the import */
+} DEVMEM_IMPORT;
+
+typedef struct DEVMEM_DEVICE_MEMDESC_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the allocation */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
+	POS_LOCK hLock;                 /*!< Lock to protect device memdesc */
+} DEVMEM_DEVICE_MEMDESC;
+
+typedef struct DEVMEM_CPU_MEMDESC_TAG
+{
+	void *pvCPUVAddr;           /*!< CPU virtual address of the import */
+	IMG_UINT32 ui32RefCount;    /*!< Refcount of the device CPU address */
+	POS_LOCK hLock;             /*!< Lock to protect CPU memdesc */
+} DEVMEM_CPU_MEMDESC;
+
+struct DEVMEM_MEMDESC_TAG
+{
+	DEVMEM_IMPORT *psImport;                /*!< Import this memdesc is on */
+	IMG_DEVMEM_OFFSET_T uiOffset;           /*!< Offset into import where our allocation starts */
+	IMG_DEVMEM_SIZE_T uiAllocSize;          /*!< Size of the allocation */
+	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
+	POS_LOCK hLock;                         /*!< Lock to protect memdesc */
+	IMG_HANDLE hPrivData;
+
+	DEVMEM_DEVICE_MEMDESC sDeviceMemDesc;   /*!< Device specifics of the memdesc */
+	DEVMEM_CPU_MEMDESC sCPUMemDesc;         /*!< CPU specifics of the memdesc */
+
+	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this memdesc */
+
+	IMG_UINT32 ui32AllocationIndex;
+
+#if defined(DEBUG)
+	IMG_BOOL bPoisonOnFree;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
+#endif
+};
+
+/* The physical descriptor used to store handles and information of device
+ * physical allocations.
+ */
+struct DEVMEMX_PHYS_MEMDESC_TAG
+{
+	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
+	IMG_UINT32 uiLog2PageSize;              /*!< Page size */
+	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;         /*!< Flags for this import */
+	IMG_HANDLE hPMR;                        /*!< Handle to the PMR */
+	DEVMEM_CPU_IMPORT sCPUImport;           /*!< CPU specifics of the memdesc */
+	SHARED_DEV_CONNECTION hConnection;      /*!< Services connection for the server */
+	void *pvUserData;						/*!< User data */
+};
+
+/* The virtual descriptor used to store handles and information of a device
+ * virtual range and the mappings to it.
+ */
+struct DEVMEMX_VIRT_MEMDESC_TAG
+{
+	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;         /*!< Flags for this import */
+	DEVMEMX_PHYSDESC **apsPhysDescTable;    /*!< Table to store links to physical descs */
+	DEVMEM_DEVICE_IMPORT sDeviceImport;     /*!< Device specifics of the memdesc */
+
+	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this virt memdesc */
+	IMG_UINT32 ui32AllocationIndex;         /*!< To track mappings in this range */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
+#endif
+};
+
+#define DEVICEMEM_UTILS_NO_ADDRESS 0
+
+/******************************************************************************
+@Function       DevmemValidateParams
+@Description    Check if flags are conflicting and if align is a size multiple.
+
+@Input          uiSize      Size of the import.
+@Input          uiAlign     Alignment of the import.
+@Input          puiFlags    Pointer to the flags for the import.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
+                                   IMG_DEVMEM_ALIGN_T uiAlign,
+                                   PVRSRV_MEMALLOCFLAGS_T *puiFlags);
+
+/******************************************************************************
+@Function       DevmemImportStructAlloc
+@Description    Allocates memory for an import struct. Does not allocate a PMR!
+                Create locks for CPU and Devmem mappings.
+
+@Input          hDevConnection  Connection to use for calls from the import.
+@Input          ppsImport       The import to allocate.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
+                                      DEVMEM_IMPORT **ppsImport);
+
+/******************************************************************************
+@Function       DevmemImportStructInit
+@Description    Initialises the import struct with the given parameters.
+                Set it's refcount to 1!
+
+@Input          psImport     The import to initialise.
+@Input          uiSize       Size of the import.
+@Input          uiAlign      Alignment of allocations in the import.
+@Input          uiMapFlags
+@Input          hPMR         Reference to the PMR of this import struct.
+@Input          uiProperties Properties of the import. Is it exportable,
+                              imported, suballocatable?
+******************************************************************************/
+void DevmemImportStructInit(DEVMEM_IMPORT *psImport,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             IMG_DEVMEM_ALIGN_T uiAlign,
+                             PVRSRV_MEMALLOCFLAGS_T uiMapFlags,
+                             IMG_HANDLE hPMR,
+                             DEVMEM_PROPERTIES_T uiProperties);
+
+/******************************************************************************
+@Function       DevmemImportStructDevMap
+@Description    NEVER call after the last DevmemMemDescRelease()
+                Maps the PMR referenced by the import struct to the device's
+                virtual address space.
+                Does nothing but increase the cpu mapping refcount if the
+                import struct was already mapped.
+
+@Input          psHeap    The heap to map to.
+@Input          bMap      Caller can choose if the import should be really
+                          mapped in the page tables or if just a virtual range
+                          should be reserved and the refcounts increased.
+@Input          psImport  The import we want to map.
+@Input          uiOptionalMapAddress  An optional address to map to.
+                                      Pass DEVICEMEM_UTILS_NOADDRESS if not used.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
+                                       IMG_BOOL bMap,
+                                       DEVMEM_IMPORT *psImport,
+                                       IMG_UINT64 uiOptionalMapAddress);
+
+/******************************************************************************
+@Function       DevmemImportStructDevUnmap
+@Description    Unmaps the PMR referenced by the import struct from the
+                device's virtual address space.
+                If this was not the last remaining CPU mapping on the import
+                struct only the cpu mapping refcount is decreased.
+@return         A boolean to signify if the import was unmapped.
+******************************************************************************/
+IMG_BOOL DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructCPUMap
+@Description    NEVER call after the last DevmemMemDescRelease()
+                Maps the PMR referenced by the import struct to the CPU's
+                virtual address space.
+                Does nothing but increase the cpu mapping refcount if the
+                import struct was already mapped.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructCPUUnmap
+@Description    Unmaps the PMR referenced by the import struct from the CPU's
+                virtual address space.
+                If this was not the last remaining CPU mapping on the import
+                struct only the cpu mapping refcount is decreased.
+******************************************************************************/
+void DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport);
+
+
+/******************************************************************************
+@Function       DevmemImportStructAcquire
+@Description    Acquire an import struct by increasing it's refcount.
+******************************************************************************/
+void DevmemImportStructAcquire(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructRelease
+@Description    Reduces the refcount of the import struct.
+                Destroys the import in the case it was the last reference.
+                Destroys underlying PMR if this import was the last reference
+                to it.
+@return         A boolean to signal if the import was destroyed. True = yes.
+******************************************************************************/
+IMG_BOOL DevmemImportStructRelease(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportDiscard
+@Description    Discard a created, but uninitialised import structure.
+                This must only be called before DevmemImportStructInit
+                after which DevmemImportStructRelease must be used to
+                "free" the import structure.
+******************************************************************************/
+void DevmemImportDiscard(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemMemDescAlloc
+@Description    Allocates a MemDesc and create it's various locks.
+                Zero the allocated memory.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc);
+
+#if defined(DEBUG)
+/******************************************************************************
+@Function       DevmemMemDescSetPoF
+@Description    Sets the Poison on Free flag to true for this MemDesc if the
+                given MemAllocFlags have the Poison on Free bit set.
+                Poison on Free is a debug only feature.
+******************************************************************************/
+void DevmemMemDescSetPoF(DEVMEM_MEMDESC *psMemDesc, PVRSRV_MEMALLOCFLAGS_T uiFlags);
+#endif
+
+/******************************************************************************
+@Function       DevmemMemDescInit
+@Description    Sets the given offset and import struct fields in the MemDesc.
+                Initialises refcount to 1 and other values to 0.
+
+@Input          psMemDesc    MemDesc to initialise.
+@Input          uiOffset     Offset in the import structure.
+@Input          psImport     Import the MemDesc is on.
+@Input          uiAllocSize  Size of the allocation
+******************************************************************************/
+void DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						DEVMEM_IMPORT *psImport,
+						IMG_DEVMEM_SIZE_T uiAllocSize);
+
+/******************************************************************************
+@Function       DevmemMemDescAcquire
+@Description    Acquires the MemDesc by increasing it's refcount.
+******************************************************************************/
+void DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc);
+
+/******************************************************************************
+@Function       DevmemMemDescRelease
+@Description    Releases the MemDesc by reducing it's refcount.
+                Destroy the MemDesc if it's recount is 0.
+                Destroy the import struct the MemDesc is on if that was the
+                last MemDesc on the import, probably following the destruction
+                of the underlying PMR.
+@return         A boolean to signal if the MemDesc was destroyed. True = yes.
+******************************************************************************/
+IMG_BOOL DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc);
+
+/******************************************************************************
+@Function       DevmemMemDescDiscard
+@Description    Discard a created, but uninitialised MemDesc structure.
+                This must only be called before DevmemMemDescInit after
+                which DevmemMemDescRelease must be used to "free" the
+                MemDesc structure.
+******************************************************************************/
+void DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc);
+
+
+/******************************************************************************
+@Function       GetImportProperties
+@Description    Atomically read psImport->uiProperties
+                It's possible that another thread modifies uiProperties
+                immediately after this function returns, making its result
+                stale. So, it's recommended to use this function only to
+                check if certain non-volatile flags were set.
+******************************************************************************/
+static INLINE DEVMEM_PROPERTIES_T GetImportProperties(DEVMEM_IMPORT *psImport)
+{
+	DEVMEM_PROPERTIES_T uiProperties;
+
+	OSLockAcquire(psImport->hLock);
+	uiProperties = psImport->uiProperties;
+	OSLockRelease(psImport->hLock);
+	return uiProperties;
+}
+
+/******************************************************************************
+@Function       DevmemCPUMemSet
+@Description    Given a CPU Mapped Devmem address, set the memory at that
+                range (address, address + size) to the uiPattern provided.
+                Flags determine the OS abstracted MemSet method to use.
+******************************************************************************/
+static INLINE void DevmemCPUMemSet(void *pvMem,
+                                   IMG_UINT8 uiPattern,
+                                   IMG_DEVMEM_SIZE_T uiSize,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags))
+	{
+		OSDeviceMemSet(pvMem, uiPattern, uiSize);
+	}
+	else
+	{
+		/* it's safe to use OSCachedMemSet() for cached and wc memory */
+		OSCachedMemSet(pvMem, uiPattern, uiSize);
+	}
+}
+
+/******************************************************************************
+@Function       DevmemCPUMapCheckImportProperties
+@Description    Given a MemDesc check that the import properties are correct
+                to allow for mapping the MemDesc to the CPU.
+                Returns PVRSRV_OK on success.
+******************************************************************************/
+static INLINE PVRSRV_ERROR DevmemCPUMapCheckImportProperties(DEVMEM_MEMDESC *psMemDesc)
+{
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiProperties & DEVMEM_PROPERTIES_SECURE)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+					"%s: Allocation is a secure buffer. "
+					"It should not be possible to map to CPU, but for security "
+					"validation this will be allowed for testing purposes, "
+					"as long as the buffer is pinned.",
+					__func__));
+	}
+#endif
+
+	if (uiProperties & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU Mapping is not possible on this allocation!",
+				__func__));
+		return PVRSRV_ERROR_INVALID_MAP_REQUEST;
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* DEVICEMEM_UTILS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx.h
new file mode 100644
index 000000000000..86049c4a705c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx.h
@@ -0,0 +1,221 @@
+/*************************************************************************/ /*!
+@File
+@Title          X Device Memory Management core internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Internal interface for extended device memory management.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEMX_H
+#define DEVICEMEMX_H
+
+#include "img_types.h"
+#include "devicemem_typedefs.h"
+#include "devicemem_utils.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "osfunc.h"
+
+/* DevmemXAllocPhysical()
+ *
+ * Allocate physical device memory and return a physical
+ * descriptor for it.
+ */
+PVRSRV_ERROR
+DevmemXAllocPhysical(DEVMEM_CONTEXT *psCtx,
+                    IMG_UINT32 uiNumPages,
+                    IMG_UINT32 uiLog2PageSize,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    const IMG_CHAR *pszText,
+                    DEVMEMX_PHYSDESC **ppsPhysDesc);
+
+/* DevmemXReleasePhysical()
+ *
+ * Removes a physical device allocation if all references
+ * to it are dropped, otherwise just decreases the refcount.
+ */
+void
+DevmemXReleasePhysical(DEVMEMX_PHYSDESC *psPhysDesc);
+
+/* DevmemAllocVirtualAddr()
+ *
+ * Reserve a requested device virtual range and return
+ * a virtual descriptor for it.
+ */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemXAllocVirtualAddr(DEVMEM_HEAP* hHeap,
+                   IMG_UINT32 uiNumPages,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   const IMG_CHAR *pszText,
+                   IMG_DEV_VIRTADDR sVirtAddr,
+                   DEVMEMX_VIRTDESC **ppsVirtDesc);
+
+/* DevmemAllocVirtual()
+ *
+ * Allocate and reserve a device virtual range and return
+ * a virtual descriptor for it.
+ */
+PVRSRV_ERROR
+DevmemXAllocVirtual(DEVMEM_HEAP* hHeap,
+                   IMG_UINT32 uiNumPages,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   const IMG_CHAR *pszText,
+                   DEVMEMX_VIRTDESC **ppsVirtDesc,
+                   IMG_DEV_VIRTADDR *psVirtAddr);
+
+/* DevmemXFreeVirtual()
+ *
+ * Removes a device virtual range if all mappings on it
+ * have been removed.
+ */
+PVRSRV_ERROR
+DevmemXFreeVirtual(DEVMEMX_VIRTDESC *psVirtDesc);
+
+/* DevmemXMapVirtualRange()
+ *
+ * Map memory from a physical descriptor into a virtual range.
+ */
+PVRSRV_ERROR
+DevmemXMapVirtualRange(IMG_UINT32 ui32PageCount,
+                      DEVMEMX_PHYSDESC *psPhysDesc,
+                      IMG_UINT32 ui32PhysOffset,
+                      DEVMEMX_VIRTDESC *psVirtDesc,
+                      IMG_UINT32 ui32VirtOffset);
+
+/* DevmemXUnmapVirtualRange()
+ *
+ * Unmap pages from a device virtual range.
+ */
+PVRSRV_ERROR
+DevmemXUnmapVirtualRange(IMG_UINT32 ui32PageCount,
+                        DEVMEMX_VIRTDESC *psVirtDesc,
+                        IMG_UINT32 ui32VirtPgOffset);
+
+/* DevmemXMapVirtualRangeToBackingPage()
+ *
+ * Map a virtual range to kernel backing page.
+ */
+PVRSRV_ERROR
+DevmemXMapVirtualRangeToBackingPage(DEVMEMX_VIRTDESC *psVirtDesc,
+                                    PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                    IMG_UINT32 ui32VirtPgOffset,
+                                    IMG_UINT32 ui32PageCount);
+
+/* DevmemXMapPhysicalToCPU()
+ *
+ * Map a full physical descriptor to CPU space.
+ */
+PVRSRV_ERROR
+DevmemXMapPhysicalToCPU(DEVMEMX_PHYSDESC *psMemAllocPhys,
+                       IMG_CPU_VIRTADDR *psVirtAddr);
+
+/* DevmemXUnmapPhysicalToCPU()
+ *
+ * Remove the CPU mapping from the descriptor.
+ */
+PVRSRV_ERROR
+DevmemXUnmapPhysicalToCPU(DEVMEMX_PHYSDESC *psMemAllocPhys);
+
+/* DevmemXReacquireCpuVirtAddr()
+ *
+ * Reacquire the CPU mapping by incrementing the refcount.
+ */
+void
+DevmemXReacquireCpuVirtAddr(DEVMEMX_PHYSDESC *psPhysDesc,
+                            void **ppvCpuVirtAddr);
+
+/* DevmemXReleaseCpuVirtAddr()
+ *
+ * Release CPU mapping by decrementing the refcount.
+ */
+void
+DevmemXReleaseCpuVirtAddr(DEVMEMX_PHYSDESC *psPhysDesc);
+
+/* DevmemXCreateDevmemMemDesc()
+ *
+ * Create a devmem memdesc from a physical and
+ * virtual descriptor.
+ * Always destroy with DevmemXFreeDevmemMemDesc().
+ */
+
+PVRSRV_ERROR
+DevmemXCreateDevmemMemDesc(DEVMEMX_PHYSDESC *psPhysDesc,
+                           DEVMEMX_VIRTDESC *psVirtDesc,
+                           DEVMEM_MEMDESC **ppsMemDesc);
+
+/* DevmemXFreeDevmemMemDesc()
+ *
+ * Free the memdesc again. Has no impact on the underlying
+ * physical and virtual descriptors.
+ */
+PVRSRV_ERROR
+DevmemXFreeDevmemMemDesc(DEVMEM_MEMDESC *psMemDesc);
+
+PVRSRV_ERROR
+DevmemXFlagCompatibilityCheck(PVRSRV_MEMALLOCFLAGS_T uiPhysFlags,
+                              PVRSRV_MEMALLOCFLAGS_T uiVirtFlags);
+
+PVRSRV_ERROR
+DevmemXPhysDescAlloc(DEVMEMX_PHYSDESC **ppsPhysDesc);
+
+void
+DevmemXPhysDescInit(DEVMEMX_PHYSDESC *psPhysDesc,
+                    IMG_HANDLE hPMR,
+                    IMG_UINT32 uiNumPages,
+                    IMG_UINT32 uiLog2PageSize,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    SHARED_DEV_CONNECTION hConnection);
+
+void
+DevmemXPhysDescFree(DEVMEMX_PHYSDESC *psPhysDesc);
+
+void
+DevmemXPhysDescAcquire(DEVMEMX_PHYSDESC *psPhysDesc,
+                       IMG_UINT32 uiAcquireCount);
+void
+DevmemXPhysDescRelease(DEVMEMX_PHYSDESC *psPhysDesc,
+                       IMG_UINT32 uiReleaseCount);
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemXGetImportUID(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_UINT64       *pui64UID);
+#endif
+
+#endif /* DEVICEMEMX_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx_pdump.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx_pdump.h
new file mode 100644
index 000000000000..b6e99f7eca66
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx_pdump.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File
+@Title          X Device Memory Management PDump internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to PDump device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEMX_PDUMP_H
+#define DEVICEMEMX_PDUMP_H
+
+#include "devicememx.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+
+#if defined(PDUMP)
+/*
+ * DevmemXPDumpLoadMem()
+ *
+ * Same as DevmemPDumpLoadMem().
+ */
+IMG_INTERNAL void
+DevmemXPDumpLoadMem(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    PDUMP_FLAGS_T uiPDumpFlags);
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemXPDumpLoadMem)
+#endif
+
+static INLINE void
+DevmemXPDumpLoadMem(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDescPhys);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+#endif /* PDUMP */
+#endif /* DEVICEMEMX_PDUMP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash.h
new file mode 100644
index 000000000000..feb879c1489a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash.h
@@ -0,0 +1,255 @@
+/*************************************************************************/ /*!
+@File
+@Title          Self scaling hash tables
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements simple self scaling hash tables.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HASH_H
+#define HASH_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * Keys passed to the comparison function are only guaranteed to be aligned on
+ * an uintptr_t boundary.
+ */
+typedef IMG_UINT32 HASH_FUNC(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+typedef IMG_BOOL HASH_KEY_COMP(size_t uKeySize, void *pKey1, void *pKey2);
+
+typedef struct _HASH_TABLE_ HASH_TABLE;
+
+typedef PVRSRV_ERROR (*HASH_pfnCallback) (
+	uintptr_t k,
+	uintptr_t v,
+	void* pvPriv
+);
+
+#if defined(DEBUG)
+#else
+#define HASH_CREATE(LEN)		HASH_Create(LEN)
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Func_Default
+@Description    Hash function intended for hashing keys composed of uintptr_t
+                arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey         A pointer to the key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Func_Default(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares keys composed of uintptr_t arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey1        Pointer to first hash key to compare.
+@Input          pKey2        Pointer to second hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Key_Comp_Default(size_t uKeySize, void *pKey1, void *pKey2);
+
+/*************************************************************************/ /*!
+@Function       HASH_Create_Extended
+@Description    Create a self scaling hash table, using the supplied key size,
+                and the supplied hash and key comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Input          uKeySize     The size of the key, in bytes.
+@Input          pfnHashFunc  Pointer to hash function.
+@Input          pfnKeyComp   Pointer to key comparison function.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+HASH_TABLE * HASH_Create_Extended_Int(IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp);
+#if defined(DEBUG)
+#define HASH_Create_Extended(LEN, KS, FUN, CMP)		HASH_Create_Extended_Debug(LEN, KS, FUN, CMP, __FILE__, __LINE__)
+HASH_TABLE * HASH_Create_Extended_Debug (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp,
+										 const char *file, const unsigned int line);
+#else
+#define HASH_Create_Extended	HASH_Create_Extended_Int
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Create
+@Description    Create a self scaling hash table with a key consisting of a
+                single uintptr_t, and using the default hash and key
+                comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+HASH_TABLE * HASH_Create_Int(IMG_UINT32 uInitialLen);
+#if defined(DEBUG)
+#define HASH_Create(LEN)		HASH_Create_Debug(LEN, __FILE__, __LINE__)
+HASH_TABLE * HASH_Create_Debug (IMG_UINT32 uInitialLen, const char *file, const unsigned int line);
+#else
+#define HASH_Create				HASH_Create_Int
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete_Extended
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table should have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Input          bWarn        Set false to suppress warnings in the case of
+                             deletion with active entries.
+@Return         None
+*/ /**************************************************************************/
+void HASH_Delete_Extended(HASH_TABLE *pHash, IMG_BOOL bWarn);
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table must have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Return         None
+*/ /**************************************************************************/
+void HASH_Delete(HASH_TABLE *pHash);
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert_Extended
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to the key.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Insert_Extended(HASH_TABLE *pHash, void *pKey, uintptr_t v);
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Insert(HASH_TABLE *pHash, uintptr_t k, uintptr_t v);
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove_Extended
+@Description    Remove a key from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Remove_Extended(HASH_TABLE *pHash, void *pKey);
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove
+@Description    Remove a key value pair from a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Remove(HASH_TABLE *pHash, uintptr_t k);
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve_Extended
+@Description    Retrieve a value from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Retrieve_Extended(HASH_TABLE *pHash, void *pKey);
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve
+@Description    Retrieve a value from a hash table created with HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Retrieve(HASH_TABLE *pHash, uintptr_t k);
+
+/*************************************************************************/ /*!
+@Function       HASH_Iterate
+@Description    Iterate over every entry in the hash table.
+@Input          pHash        Hash table to iterate.
+@Input          pfnCallback  Callback to call with the key and data for each
+.                            entry in the hash table
+@Return         Callback error if any, otherwise PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR HASH_Iterate(HASH_TABLE *pHash, HASH_pfnCallback pfnCallback, void* args);
+
+/*************************************************************************/ /*!
+@Function       HASH_Count
+@Description    Retrieve the number of entries in the hash table.
+@Input          pHash        The hash table.
+@Return         The number of entries.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Count(HASH_TABLE *pHash);
+
+#ifdef HASH_TRACE
+/*************************************************************************/ /*!
+@Function       HASH_Dump
+@Description    Dump out some information about a hash table.
+@Input          pHash         The hash table.
+*/ /**************************************************************************/
+void HASH_Dump(HASH_TABLE *pHash);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HASH_H */
+
+/******************************************************************************
+ End of file (hash.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash_functions.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash_functions.h
new file mode 100644
index 000000000000..154770848be4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash_functions.h
@@ -0,0 +1,66 @@
+/*************************************************************************/ /*!
+@File
+@Title          Reusable hash functions for hash.c.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common hash functions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+
+/*************************************************************************/ /*!
+@Function       HASH_Djb2_Hash
+@Description    Hash function intended for hashing string keys. This function
+                implements DJB2 algorithm.
+@Input          uKeySize     The size of the string hash key, in bytes.
+@Input          pKey         A pointer to the string key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Djb2_Hash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares string keys.
+@Input          uKeySize     The size of the string key.
+@Input          pKey1        Pointer to first string hash key to compare.
+@Input          pKey2        Pointer to second string hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Djb2_Compare(size_t uKeySize, void *pKey1, void *pKey2);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer.h
new file mode 100644
index 000000000000..96daa9727c47
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           htbuffer.h
+@Title          Host Trace Buffer shared API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_H
+#define HTBUFFER_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "htbuffer_sf.h"
+#include "htbuffer_types.h"
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+ @Input         hSrvHandle      Server Handle
+ @Input         ui32NumFlagGroups Number of group enable flags words
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+ @Input         ui32LogLevel    Log level to record
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+ @Input         eLogPidMode     Enable logging for all or specific processes,
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogPidMode,
+	HTB_OPMODE_CTRL eOpMode);
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_H */
+/*****************************************************************************
+ End of file (htbuffer.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer_init.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer_init.h
new file mode 100644
index 000000000000..d114579964b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer_init.h
@@ -0,0 +1,114 @@
+/*************************************************************************/ /*!
+@File           htbuffer_init.h
+@Title          Host Trace Buffer functions needed for Services initialisation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_INIT_H
+#define HTBUFFER_INIT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+/*************************************************************************/ /*!
+ @Function      HTBConfigure
+ @Description   Configure the Host Trace Buffer.
+                Once these parameters are set they may not be changed
+
+ @Input         hSrvHandle      Server Handle
+
+ @Input         pszBufferName   Name to use for the TL buffer, this will be
+                                required to request trace data from the TL
+
+ @Input         ui32BufferSize  Requested TL buffer size in bytes
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBConfigure(
+	IMG_HANDLE hSrvHandle,
+	IMG_CHAR * pszBufferName,
+	IMG_UINT32 ui32BufferSize
+);
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         hSrvHandle      Server Handle
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogMode,
+	HTB_OPMODE_CTRL eOpMode
+);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_INIT_H */
+/*****************************************************************************
+ End of file (htbuffer_init.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/lock.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/lock.h
new file mode 100644
index 000000000000..386e3b2f88a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/lock.h
@@ -0,0 +1,527 @@
+/*************************************************************************/ /*!
+@File           lock.h
+@Title          Locking interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal locking interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOCK_H
+#define LOCK_H
+
+/* In Linux kernel mode we are using the kernel mutex implementation directly
+ * with macros. This allows us to use the kernel lockdep feature for lock
+ * debugging. */
+#include "lock_types.h"
+#include "pvr_debug.h"
+
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include "allocmem.h"
+#include <linux/atomic.h>
+#include <linux/version.h>
+
+#define OSLockCreateNoStats(phLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(phLock) = OSAllocMemNoStats(sizeof(struct mutex)); \
+	if (*(phLock)) { mutex_init(*(phLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSLockCreate(phLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(phLock) = OSAllocMem(sizeof(struct mutex)); \
+	if (*(phLock)) { mutex_init(*(phLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSLockDestroy(hLock) ({mutex_destroy((hLock)); OSFreeMem((hLock));})
+#define OSLockDestroyNoStats(hLock) ({mutex_destroy((hLock)); OSFreeMemNoStats((hLock));})
+
+#define OSLockAcquire(hLock) ({mutex_lock((hLock));})
+#define OSLockAcquireNested(hLock, subclass) ({mutex_lock_nested((hLock), (subclass));})
+#define OSLockRelease(hLock) ({mutex_unlock((hLock));})
+
+#define OSLockIsLocked(hLock) ((mutex_is_locked((hLock)) == 1) ? IMG_TRUE : IMG_FALSE)
+#define OSTryLockAcquire(hLock) ((mutex_trylock(hLock) == 1) ? IMG_TRUE : IMG_FALSE)
+
+#if defined(DEBUG)
+#if defined(CONFIG_LOCKDEP)
+#define OSLockHeldAssert(hLock) ({ lockdep_assert_held((hLock)); })
+#else
+#define OSLockHeldAssert(hLock) ({ PVR_ASSERT(OSLockIsLocked((hLock)) == IMG_TRUE); })
+#endif
+#else
+#define OSLockHeldAssert(hLock)
+#endif
+
+#define OSSpinLockCreate(_ppsLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(_ppsLock) = OSAllocMem(sizeof(spinlock_t)); \
+	if (*(_ppsLock)) {spin_lock_init(*(_ppsLock)); e = PVRSRV_OK;} \
+	e;})
+#define OSSpinLockDestroy(_psLock) ({OSFreeMem(_psLock);})
+
+typedef unsigned long OS_SPINLOCK_FLAGS;
+#define OSSpinLockAcquire(_pLock, _flags) spin_lock_irqsave(_pLock, _flags)
+#define OSSpinLockRelease(_pLock, _flags) spin_unlock_irqrestore(_pLock, _flags)
+
+/* These _may_ be reordered or optimized away entirely by the compiler/hw */
+#define OSAtomicRead(pCounter)	atomic_read(pCounter)
+#define OSAtomicWrite(pCounter, i)	atomic_set(pCounter, i)
+
+/* The following atomic operations, in addition to being SMP-safe, also
+   imply a memory barrier around the operation */
+#define OSAtomicIncrement(pCounter) atomic_inc_return(pCounter)
+#define OSAtomicDecrement(pCounter) atomic_dec_return(pCounter)
+#define OSAtomicCompareExchange(pCounter, oldv, newv) atomic_cmpxchg(pCounter,oldv,newv)
+#define OSAtomicExchange(pCounter, iNewVal) atomic_xchg(pCounter, iNewVal)
+
+static inline IMG_INT OSAtomicOr(ATOMIC_T *pCounter, IMG_INT iVal)
+{
+	IMG_INT iOldVal, iLastVal, iNewVal;
+
+	iLastVal = OSAtomicRead(pCounter);
+	do
+	{
+		iOldVal = iLastVal;
+		iNewVal = iOldVal | iVal;
+
+		iLastVal = OSAtomicCompareExchange(pCounter, iOldVal, iNewVal);
+	}
+	while (iOldVal != iLastVal);
+
+	return iNewVal;
+}
+
+#define OSAtomicAdd(pCounter, incr) atomic_add_return(incr,pCounter)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
+#define OSAtomicAddUnless(pCounter, incr, test) atomic_fetch_add_unless(pCounter, (incr), (test))
+#else
+#define OSAtomicAddUnless(pCounter, incr, test) __atomic_add_unless(pCounter,incr,test)
+#endif
+
+#define OSAtomicSubtract(pCounter, incr) atomic_add_return(-(incr),pCounter)
+#define OSAtomicSubtractUnless(pCounter, incr, test) OSAtomicAddUnless(pCounter, -(incr), (test))
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+/**************************************************************************/ /*!
+@Function       OSLockCreate
+@Description    Creates an operating system lock object.
+@Output         phLock           The created lock.
+@Return         PVRSRV_OK on success. PVRSRV_ERROR_OUT_OF_MEMORY if the driver
+                cannot allocate CPU memory needed for the lock.
+                PVRSRV_ERROR_INIT_FAILURE if the Operating System fails to
+                allocate the lock.
+ */ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR OSLockCreate(POS_LOCK *phLock);
+#if defined(INTEGRITY_OS) || defined(__QNXNTO__)
+#define OSLockCreateNoStats OSLockCreate
+#endif
+
+/**************************************************************************/ /*!
+@Function       OSLockDestroy
+@Description    Destroys an operating system lock object.
+@Input          hLock            The lock to be destroyed.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockDestroy(POS_LOCK hLock);
+
+#if defined(INTEGRITY_OS) || defined(__QNXNTO__)
+#define OSLockDestroyNoStats OSLockDestroy
+#endif
+/**************************************************************************/ /*!
+@Function       OSLockAcquire
+@Description    Acquires an operating system lock.
+                NB. This function must not return until the lock is acquired
+                (meaning the implementation should not timeout or return with
+                an error, as the caller will assume they have the lock).
+@Input          hLock            The lock to be acquired.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockAcquire(POS_LOCK hLock);
+
+/**************************************************************************/ /*!
+@Function       OSTryLockAcquire
+@Description    Try to acquire an operating system lock.
+                NB. If lock is acquired successfully in the first attempt,
+                then the function returns true and else it will return false.
+@Input          hLock            The lock to be acquired.
+@Return         IMG_TRUE if lock acquired successfully,
+                IMG_FALSE otherwise.
+ */ /**************************************************************************/
+IMG_INTERNAL
+IMG_BOOL OSTryLockAcquire(POS_LOCK hLock);
+
+/* Nested notation isn't used in UM or other OS's */
+/**************************************************************************/ /*!
+@Function       OSLockAcquireNested
+@Description    For operating systems other than Linux, this equates to an
+                OSLockAcquire() call. On Linux, this function wraps a call
+                to mutex_lock_nested(). This recognises the scenario where
+                there may be multiple subclasses within a particular class
+                of lock. In such cases, the order in which the locks belonging
+                these various subclasses are acquired is important and must be
+                validated.
+@Input          hLock            The lock to be acquired.
+@Input          subclass         The subclass of the lock.
+@Return         None.
+ */ /**************************************************************************/
+#define OSLockAcquireNested(hLock, subclass) OSLockAcquire((hLock))
+
+/**************************************************************************/ /*!
+@Function       OSLockRelease
+@Description    Releases an operating system lock.
+@Input          hLock            The lock to be released.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockRelease(POS_LOCK hLock);
+
+/**************************************************************************/ /*!
+@Function       OSLockIsLocked
+@Description    Tests whether or not an operating system lock is currently
+                locked.
+@Input          hLock            The lock to be tested.
+@Return         IMG_TRUE if locked, IMG_FALSE if not locked.
+ */ /**************************************************************************/
+IMG_INTERNAL
+IMG_BOOL OSLockIsLocked(POS_LOCK hLock);
+
+#define OSLockHeldAssert(hLock) PVR_ASSERT(OSLockIsLocked((hLock)) == IMG_TRUE)
+
+#if defined(__linux__)
+
+/* Use GCC intrinsics (read/write semantics consistent with kernel-side implementation) */
+#define OSAtomicRead(pCounter) (*(volatile IMG_INT32 *)&(pCounter)->counter)
+#define OSAtomicWrite(pCounter, i) ((pCounter)->counter = (IMG_INT32) i)
+#define OSAtomicIncrement(pCounter) __sync_add_and_fetch((&(pCounter)->counter), 1)
+#define OSAtomicDecrement(pCounter) __sync_sub_and_fetch((&(pCounter)->counter), 1)
+#define OSAtomicCompareExchange(pCounter, oldv, newv) \
+	__sync_val_compare_and_swap((&(pCounter)->counter), oldv, newv)
+#define OSAtomicOr(pCounter, iVal) __sync_or_and_fetch((&(pCounter)->counter), iVal)
+
+static inline IMG_UINT32 OSAtomicExchange(ATOMIC_T *pCounter, IMG_UINT32 iNewVal)
+{
+	IMG_UINT32 iOldVal;
+	IMG_UINT32 iLastVal;
+
+	iLastVal = OSAtomicRead(pCounter);
+	do
+	{
+		iOldVal = iLastVal;
+		iLastVal = OSAtomicCompareExchange(pCounter, iOldVal, iNewVal);
+	}
+	while (iOldVal != iLastVal);
+
+	return iOldVal;
+}
+
+#define OSAtomicAdd(pCounter, incr) __sync_add_and_fetch((&(pCounter)->counter), incr)
+#define OSAtomicAddUnless(pCounter, incr, test) ({ \
+	IMG_INT32 c; IMG_INT32 old; \
+	c = OSAtomicRead(pCounter); \
+	while (1) { \
+		if (c == (test)) break; \
+		old = OSAtomicCompareExchange(pCounter, c, c+(incr)); \
+		if (old == c) break; \
+		c = old; \
+	} c; })
+
+#define OSAtomicSubtract(pCounter, incr) OSAtomicAdd(pCounter, -(incr))
+#define OSAtomicSubtractUnless(pCounter, incr, test) OSAtomicAddUnless(pCounter, -(incr), test)
+
+#else
+
+/*************************************************************************/ /*!
+@Function       OSAtomicRead
+@Description    Read the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to read
+@Return         The value of the atomic variable
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicRead(const ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicWrite
+@Description    Write the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be written to
+@Input          v               The value to write
+@Return         None
+*/ /**************************************************************************/
+IMG_INTERNAL
+void OSAtomicWrite(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/* For the following atomic operations, in addition to being SMP-safe,
+   should also  have a memory barrier around each operation */
+/*************************************************************************/ /*!
+@Function       OSAtomicIncrement
+@Description    Increment the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be incremented
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicIncrement(ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicDecrement
+@Description    Decrement the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be decremented
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicDecrement(ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicAdd
+@Description    Add a specified value to a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to add the value to
+@Input          v               The value to be added
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicAdd(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicAddUnless
+@Description    Add a specified value to a variable atomically unless it
+                already equals a particular value.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to add the value to
+@Input          v               The value to be added to 'pCounter'
+@Input          t               The test value. If 'pCounter' equals this,
+                                its value will not be adjusted
+@Return         The old value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicAddUnless(ATOMIC_T *pCounter, IMG_INT32 v, IMG_INT32 t);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicSubtract
+@Description    Subtract a specified value to a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to subtract the value from
+@Input          v               The value to be subtracted
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicSubtract(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicSubtractUnless
+@Description    Subtract a specified value from a variable atomically unless
+                it already equals a particular value.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to subtract the value from
+@Input          v               The value to be subtracted from 'pCounter'
+@Input          t               The test value. If 'pCounter' equals this,
+                                its value will not be adjusted
+@Return         The old value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicSubtractUnless(ATOMIC_T *pCounter, IMG_INT32 v, IMG_INT32 t);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicCompareExchange
+@Description    Set a variable to a given value only if it is currently
+                equal to a specified value. The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be checked and
+                                possibly updated
+@Input          oldv            The value the atomic variable must have in
+                                order to be modified
+@Input          newv            The value to write to the atomic variable if
+                                it equals 'oldv'
+@Return         The old value of *pCounter
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicCompareExchange(ATOMIC_T *pCounter, IMG_INT32 oldv, IMG_INT32 newv);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicExchange
+@Description    Set a variable to a given value and retrieve previous value.
+                The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be updated
+@Input          iNewVal         The value to write to the atomic variable
+@Return         The previous value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicExchange(ATOMIC_T *pCounter, IMG_INT32 iNewVal);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicOr
+@Description    Set a variable to the bitwise or of its current value and the
+                specified value. Equivalent to *pCounter |= iVal.
+                The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be updated
+@Input          iVal            The value to bitwise or against
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicOr(ATOMIC_T *pCounter, IMG_INT32 iVal);
+
+/* For now, spin-locks are required on Linux only, so other platforms fake
+ * spinlocks with normal mutex locks */
+/*! Type definitions for OS_SPINLOCK accessor and creation / deletion */
+typedef unsigned long OS_SPINLOCK_FLAGS;
+/*! Pointer to an OS Spinlock */
+#define POS_SPINLOCK POS_LOCK
+/*! Wrapper for OSLockCreate() */
+#define OSSpinLockCreate(ppLock) OSLockCreate(ppLock)
+/*! Wrapper for OSLockDestroy() */
+#define OSSpinLockDestroy(pLock) OSLockDestroy(pLock)
+/*! Wrapper for OSLockAcquire() */
+#define OSSpinLockAcquire(pLock, flags) {flags = 0; OSLockAcquire(pLock);}
+/*! Wrapper for OSLockRelease() */
+#define OSSpinLockRelease(pLock, flags) {flags = 0; OSLockRelease(pLock);}
+
+#endif /* defined(__linux__) */
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+static inline PVRSRV_ERROR OSAtomicAddUnlessOverflow(ATOMIC_T *pCounter, IMG_INT iAdd, IMG_INT iMax)
+{
+	IMG_INT iCurrent;
+	IMG_INT iOld, iNew;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Should be using subtract function */
+	if (iAdd <= 0)
+	{
+		PVR_ASSERT(!"Attempt to add negative number, rejecting.");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iCurrent = OSAtomicRead(pCounter);
+
+	do
+	{
+		iOld = iCurrent;
+#if defined(__GNUC__) || defined(__clang__)
+		if (iAdd > (iMax - iCurrent) || __builtin_sadd_overflow(iCurrent, iAdd, &iNew))
+		{
+			/* We would overflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+		}
+#else
+		if (iAdd > (iMax - iCurrent) || iAdd > (IMG_INT_MAX - iCurrent))
+		{
+			/* We would overflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+		}
+		iNew = iCurrent + iAdd;
+#endif
+		iCurrent = OSAtomicCompareExchange(pCounter, iOld, iNew);
+	}
+	while (iOld != iCurrent);
+
+	return eError;
+}
+
+static inline PVRSRV_ERROR OSAtomicSubtractUnlessUnderflow(ATOMIC_T *pCounter, IMG_INT iSub, IMG_INT iMin)
+{
+	IMG_INT iCurrent;
+	IMG_INT iOld, iNew;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Should be using add function */
+	if (iSub <= 0)
+	{
+		PVR_ASSERT(!"Attempt to subtract negative number, rejecting.");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iCurrent = OSAtomicRead(pCounter);
+
+	do
+	{
+		iOld = iCurrent;
+#if defined(__GNUC__) || defined(__clang__)
+		if (iSub < (iMin + iCurrent) || __builtin_ssub_overflow(iCurrent, iSub, &iNew))
+		{
+			/* We would underflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_UNDERFLOW;
+		}
+#else
+		if (iCurrent < (iMin + iSub) || iCurrent < (IMG_INT_MIN + iSub))
+		{
+			/* We would underflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_UNDERFLOW;
+		}
+		iNew = iCurrent - iSub;
+#endif
+		iCurrent = OSAtomicCompareExchange(pCounter, iOld, iNew);
+	}
+	while (iOld != iCurrent);
+
+	return eError;
+}
+
+#endif /* LOCK_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/osmmap.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/osmmap.h
new file mode 100644
index 000000000000..40a509d19440
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/osmmap.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS Interface for mapping PMRs into CPU space.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS abstraction for the mmap2 interface for mapping PMRs into
+                User Mode memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSMMAP_H
+#define OSMMAP_H
+
+#include <powervr/mem_types.h>
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+/*************************************************************************/ /*!
+@Function       OSMMapPMR
+@Description    Maps the specified PMR into CPU memory so that it may be
+                accessed by the user process.
+                Whether the memory is mapped read only, read/write, or not at
+                all, is dependent on the PMR itself.
+                The PMR handle is opaque to the user, and lower levels of this
+                stack ensure that the handle is private to this process, such
+                that this API cannot be abused to gain access to other people's
+                PMRs. The OS implementation of this function should return the
+                virtual address and length for the User to use. The "PrivData"
+                is to be stored opaquely by the caller (N.B. he should make no
+                assumptions, in particular, NULL is a valid handle) and given
+                back to the call to OSMUnmapPMR.
+                The OS implementation is free to use the PrivData handle for
+                any purpose it sees fit.
+@Input          hBridge              The bridge handle.
+@Input          hPMR                 The handle of the PMR to be mapped.
+@Input          uiPMRLength          The size of the PMR.
+@Input          uiFlags              Flags indicating how the mapping should
+                                     be done (read-only, etc). These may not
+                                     be honoured if the PMR does not permit
+                                     them.
+@Output         phOSMMapPrivDataOut  Returned private data.
+@Output         ppvMappingAddressOut The returned mapping.
+@Output         puiMappingLengthOut  The size of the returned mapping.
+@Return         PVRSRV_OK on success, failure code otherwise.
+ */ /*************************************************************************/
+PVRSRV_ERROR
+OSMMapPMR(IMG_HANDLE hBridge,
+          IMG_HANDLE hPMR,
+          IMG_DEVMEM_SIZE_T uiPMRLength,
+          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+          IMG_HANDLE *phOSMMapPrivDataOut,
+          void **ppvMappingAddressOut,
+          size_t *puiMappingLengthOut);
+
+/*************************************************************************/ /*!
+@Function       OSMUnmapPMR
+@Description    Unmaps the specified PMR from CPU memory.
+                This function is the counterpart to OSMMapPMR.
+                The caller is required to pass the PMR handle back in along
+                with the same 3-tuple of information that was returned by the
+                call to OSMMapPMR in phOSMMapPrivDataOut.
+                It is possible to unmap only part of the original mapping
+                with this call, by specifying only the address range to be
+                unmapped in pvMappingAddress and uiMappingLength.
+@Input          hBridge              The bridge handle.
+@Input          hPMR                 The handle of the PMR to be unmapped.
+@Input          hOSMMapPrivData      The OS private data of the mapping.
+@Input          pvMappingAddress     The address to be unmapped.
+@Input          uiMappingLength      The size to be unmapped.
+@Return         PVRSRV_OK on success, failure code otherwise.
+ */ /*************************************************************************/
+void
+OSMUnmapPMR(IMG_HANDLE hBridge,
+            IMG_HANDLE hPMR,
+            IMG_HANDLE hOSMMapPrivData,
+            void *pvMappingAddress,
+            size_t uiMappingLength);
+
+#endif /* OSMMAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/proc_stats.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/proc_stats.h
new file mode 100644
index 000000000000..72c14c5f8e4d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/proc_stats.h
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File
+@Title          Process and driver statistic definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PROC_STATS_H
+#define PROC_STATS_H
+
+#define DKP_HIDDEN "hidden"
+
+#define PROCESS_STAT_KMALLOC                   X(PVRSRV_PROCESS_STAT_TYPE_KMALLOC,                "MemoryUsageKMalloc",                "host_kmalloc")
+#define PROCESS_STAT_KMALLOC_MAX               X(PVRSRV_PROCESS_STAT_TYPE_KMALLOC_MAX,            "MemoryUsageKMallocMax",             DKP_HIDDEN)
+
+#define PROCESS_STAT_VMALLOC                   X(PVRSRV_PROCESS_STAT_TYPE_VMALLOC,                "MemoryUsageVMalloc",                "host_vmalloc")
+#define PROCESS_STAT_VMALLOC_MAX               X(PVRSRV_PROCESS_STAT_TYPE_VMALLOC_MAX,            "MemoryUsageVMallocMax",             DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_PAGES_PT_UMA        X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA,     "MemoryUsageAllocPTMemoryUMA",       "host_mem_dev_pt")
+#define PROCESS_STAT_ALLOC_PAGES_PT_UMA_MAX    X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA_MAX, "MemoryUsageAllocPTMemoryUMAMax",    DKP_HIDDEN)
+
+#define PROCESS_STAT_VMAP_PT_UMA               X(PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA,            "MemoryUsageVMapPTUMA",              DKP_HIDDEN)
+#define PROCESS_STAT_VMAP_PT_UMA_MAX           X(PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA_MAX,        "MemoryUsageVMapPTUMAMax",           DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_PAGES_PT_LMA        X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA,     "MemoryUsageAllocPTMemoryLMA",       "local_mem_dev_pt")
+#define PROCESS_STAT_ALLOC_PAGES_PT_LMA_MAX    X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA_MAX, "MemoryUsageAllocPTMemoryLMAMax",    DKP_HIDDEN)
+
+#define PROCESS_STAT_IOREMAP_PT_LMA            X(PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA,         "MemoryUsageIORemapPTLMA",           DKP_HIDDEN)
+#define PROCESS_STAT_IOREMAP_PT_LMA_MAX        X(PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA_MAX,     "MemoryUsageIORemapPTLMAMax",        DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_LMA_PAGES           X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES,        "MemoryUsageAllocGPUMemLMA",         "local_mem_dev_buf")
+#define PROCESS_STAT_ALLOC_LMA_PAGES_MAX       X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES_MAX,    "MemoryUsageAllocGPUMemLMAMax",      DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES          X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES,       "MemoryUsageZombieGPUMemLMA",        "local_mem_dev_buf_purgeable")
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX      X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES_MAX,   "MemoryUsageZombieGPUMemLMAMax",     DKP_HIDDEN)
+#else
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX
+#endif
+
+#define PROCESS_STAT_ALLOC_UMA_PAGES           X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES,        "MemoryUsageAllocGPUMemUMA",         "host_mem_dev_buf")
+#define PROCESS_STAT_ALLOC_UMA_PAGES_MAX       X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES_MAX,    "MemoryUsageAllocGPUMemUMAMax",      DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES          X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES,       "MemoryUsageZombieGPUMemUMA",        "host_mem_dev_buf_purgeable")
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX      X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES_MAX,   "MemoryUsageZombieGPUMemUMAMax",     DKP_HIDDEN)
+#else
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX
+#endif
+
+#define PROCESS_STAT_MAP_UMA_LMA_PAGES         X(PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES,      "MemoryUsageMappedGPUMemUMA/LMA",    DKP_HIDDEN)
+#define PROCESS_STAT_MAP_UMA_LMA_PAGES_MAX     X(PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES_MAX,  "MemoryUsageMappedGPUMemUMA/LMAMax", DKP_HIDDEN)
+
+#define PROCESS_STAT_DMA_BUF_IMPORT            X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT,         "MemoryUsageDmaBufImport",           "dma_buf_import")
+#define PROCESS_STAT_DMA_BUF_IMPORT_MAX        X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT_MAX,     "MemoryUsageDmaBufImportMax",        DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_DMA_BUF_ZOMBIE            X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE,         "MemoryUsageDmaBufZombie",           "dma_buf_purgeable")
+#define PROCESS_STAT_DMA_BUF_ZOMBIE_MAX        X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE_MAX,     "MemoryUsageDmaBufZombieMax",        DKP_HIDDEN)
+#else
+#define PROCESS_STAT_DMA_BUF_ZOMBIE
+#define PROCESS_STAT_DMA_BUF_ZOMBIE_MAX
+#endif
+
+#define PROCESS_STAT_TOTAL                     X(PVRSRV_PROCESS_STAT_TYPE_TOTAL,                  "MemoryUsageTotal",                  DKP_HIDDEN)
+#define PROCESS_STAT_TOTAL_MAX                 X(PVRSRV_PROCESS_STAT_TYPE_TOTAL_MAX,              "MemoryUsageTotalMax",               DKP_HIDDEN)
+
+/* Process stat keys */
+#define PVRSRV_PROCESS_STAT_KEY \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_KMALLOC_MAX \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_VMALLOC_MAX \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA_MAX \
+	PROCESS_STAT_VMAP_PT_UMA \
+	PROCESS_STAT_VMAP_PT_UMA_MAX \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA_MAX \
+	PROCESS_STAT_IOREMAP_PT_LMA \
+	PROCESS_STAT_IOREMAP_PT_LMA_MAX \
+	PROCESS_STAT_ALLOC_LMA_PAGES \
+	PROCESS_STAT_ALLOC_LMA_PAGES_MAX \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ALLOC_UMA_PAGES_MAX \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX \
+	PROCESS_STAT_MAP_UMA_LMA_PAGES \
+	PROCESS_STAT_MAP_UMA_LMA_PAGES_MAX \
+	PROCESS_STAT_DMA_BUF_IMPORT \
+	PROCESS_STAT_DMA_BUF_IMPORT_MAX \
+	PROCESS_STAT_DMA_BUF_ZOMBIE \
+	PROCESS_STAT_DMA_BUF_ZOMBIE_MAX \
+	PROCESS_STAT_TOTAL \
+	PROCESS_STAT_TOTAL_MAX
+
+#if defined(SUPPORT_LINUX_FDINFO)
+/* DKP process stats within the drm-memory-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_MEMORY \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES
+
+/* DKP process stats within the drm-shared-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_SHARED \
+	PROCESS_STAT_DMA_BUF_IMPORT \
+	PROCESS_STAT_DMA_BUF_ZOMBIE
+
+/* DKP process stats within the drm-total-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_TOTAL \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_LMA_PAGES
+
+/* DKP process stats within the drm-resident-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_RESIDENT \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+
+/* DKP process stats within the drm-purgeable-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_PURGEABLE \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_DMA_BUF_ZOMBIE
+
+/* DKP process stats within the drm-active-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_ACTIVE
+
+#endif
+
+/* X-Macro for Device stat keys */
+#define PVRSRV_DEVICE_STAT_KEY \
+	X(PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS, "Connections") \
+	X(PVRSRV_DEVICE_STAT_TYPE_MAX_CONNECTIONS, "ConnectionsMax") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_OOMS, "RenderContextOutOfMemoryEvents") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_PRS, "RenderContextPartialRenders") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_GROWS, "RenderContextGrows") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_PUSH_GROWS, "RenderContextPushGrows") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_TA_STORES, "RenderContextTAStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_3D_STORES, "RenderContext3DStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_CDM_STORES, "RenderContextCDMStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_TDM_STORES, "RenderContextTDMStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_RAY_STORES, "RenderContextRayStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_APP, "ZSBufferRequestsByApp") \
+	X(PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_FW, "ZSBufferRequestsByFirmware") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_APP, "FreeListGrowRequestsByApp") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_FW, "FreeListGrowRequestsByFirmware") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_PAGES_INIT, "FreeListInitialPages") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_MAX_PAGES, "FreeListMaxPages") \
+	X(PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT, "MemoryOOMCountDeviceVirtual") \
+	X(PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT, "MemoryOOMCountPhysicalHeap") \
+	X(PVRSRV_DEVICE_STAT_TYPE_INVALID_VIRTMEM, "MemoryOOMCountDeviceVirtualAtAddr")
+
+/* X-Macro for Driver stat keys */
+#define PVRSRV_DRIVER_STAT_KEY \
+	X(PVRSRV_DRIVER_STAT_TYPE_KMALLOC, "MemoryUsageKMalloc") \
+	X(PVRSRV_DRIVER_STAT_TYPE_KMALLOC_MAX, "MemoryUsageKMallocMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMALLOC, "MemoryUsageVMalloc") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMALLOC_MAX, "MemoryUsageVMallocMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, "MemoryUsageAllocPTMemoryUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA_MAX, "MemoryUsageAllocPTMemoryUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, "MemoryUsageVMapPTUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA_MAX, "MemoryUsageVMapPTUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, "MemoryUsageAllocPTMemoryLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA_MAX, "MemoryUsageAllocPTMemoryLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, "MemoryUsageIORemapPTLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA_MAX, "MemoryUsageIORemapPTLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, "MemoryUsageAllocGPUMemLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA_MAX, "MemoryUsageAllocGPUMemLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, "MemoryUsageZombieGPUMemLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA_MAX, "MemoryUsageZombieGPUMemLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, "MemoryUsageAllocGPUMemUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_MAX, "MemoryUsageAllocGPUMemUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, "MemoryUsageZombieGPUMemUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA_MAX, "MemoryUsageZombieGPUMemUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, "MemoryUsageAllocGPUMemUMAPool") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL_MAX, "MemoryUsageAllocGPUMemUMAPoolMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, "MemoryUsageMappedGPUMemUMA/LMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA_MAX, "MemoryUsageMappedGPUMemUMA/LMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, "MemoryUsageDmaBufImport") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT_MAX, "MemoryUsageDmaBufImportMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, "MemoryUsageDmaBufZombie") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE_MAX, "MemoryUsageDmaBufZombieMax")
+
+
+typedef enum {
+#define X(stat_type, stat_str, drm_str) stat_type,
+	PVRSRV_PROCESS_STAT_KEY
+#undef X
+	PVRSRV_PROCESS_STAT_TYPE_COUNT
+}PVRSRV_PROCESS_STAT_TYPE;
+
+typedef enum {
+#define X(stat_type, stat_str) stat_type,
+	PVRSRV_DEVICE_STAT_KEY
+#undef X
+	PVRSRV_DEVICE_STAT_TYPE_COUNT
+}PVRSRV_DEVICE_STAT_TYPE;
+
+typedef enum {
+#define X(stat_type, stat_str) stat_type,
+	PVRSRV_DRIVER_STAT_KEY
+#undef X
+	PVRSRV_DRIVER_STAT_TYPE_COUNT
+}PVRSRV_DRIVER_STAT_TYPE;
+
+#endif // PROC_STATS_H
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/ra.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/ra.h
new file mode 100644
index 000000000000..ee4492afa1ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/ra.h
@@ -0,0 +1,763 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Allocator API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RA_H
+#define RA_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define RA_MAX_NAME_LENGTH 20
+
+/** Resource arena.
+ *  struct _RA_ARENA_ deliberately opaque
+ */
+typedef struct _RA_ARENA_ RA_ARENA;			//PRQA S 3313
+
+/** Resource arena's iterator.
+ *  struct _RA_ARENA_ITERATOR_ deliberately opaque
+ */
+typedef struct _RA_ARENA_ITERATOR_ RA_ARENA_ITERATOR;
+
+typedef struct _RA_ITERATOR_DATA_ {
+	IMG_UINT64 uiAddr;
+	IMG_UINT64 uiSize;
+	/* Standard Iteration: is the segment free
+	 * Span Iteration: is the span a free import
+	 */
+	IMG_BOOL bFree;
+} RA_ITERATOR_DATA;
+
+/** Resource arena usage statistics.
+ *  struct _RA_USAGE_STATS
+ */
+typedef struct _RA_USAGE_STATS {
+	IMG_UINT64	ui64TotalArenaSize;
+	IMG_UINT64	ui64FreeArenaSize;
+} RA_USAGE_STATS, *PRA_USAGE_STATS;
+
+/*
+ * Per-Arena handle - this is private data for the caller of the RA.
+ * The RA knows nothing about this data. It is given it in RA_Create, and
+ * promises to pass it to calls to the ImportAlloc and ImportFree callbacks
+ */
+typedef IMG_HANDLE RA_PERARENA_HANDLE;
+/*
+ * Per-Import handle - this is private data for the caller of the RA.
+ * The RA knows nothing about this data. It is given it on a per-import basis,
+ * basis, either the "initial" import at RA_Create time, or further imports
+ * via the ImportAlloc callback. It sends it back via the ImportFree callback,
+ * and also provides it in answer to any RA_Alloc request to signify from
+ * which "import" the allocation came.
+ */
+typedef IMG_HANDLE RA_PERISPAN_HANDLE;
+
+typedef IMG_UINT64 RA_BASE_T;
+typedef IMG_UINT32 RA_LOG2QUANTUM_T;
+typedef IMG_UINT64 RA_LENGTH_T;
+typedef IMG_UINT32 RA_POLICY_T;
+
+typedef struct _RA_BASE_MULTI_ RA_BASE_MULTI_T;
+
+typedef IMG_UINT32 RA_BASE_ARRAY_SIZE_T;
+
+
+/*
+ * RA_BASE_ARRAY can represent a number of bases of which are packed,
+ * that is, they can be one of two types, a Real Base or a Ghost base.
+ * A Real Base is a base that has been created by the RA and is used to
+ * represent an allocated region, it has an entry in the RA Hash table and
+ * as such has a BT associated with it.
+ * A Ghost base is a fabricated base address generated at chunk boundaries
+ * given by the caller. These are used to divide a RealBase into
+ * arbitrary regions that the caller requires e.g. 4k pages. Ghost bases don't
+ * exist from the RA memory tracking point of view but they do exist and are treated
+ * as base addresses from the PoV of the caller. This allows the RA to allocate in
+ * largest possible lengths meaning fewer alloc calls whilst allowing the chunk
+ * flexibility for callers. Ghost refers to the concept that they
+ * don't exist in this RA internals context but do in the callers (LMA) context i.e.
+ * they appear Real from another perspective but we the RA know they are a ghost of the
+ * Real Base.
+ * */
+typedef RA_BASE_T RA_BASE_ARRAY_T[IMG_FLEX_ARRAY_MEMBER];
+
+/* Since 0x0 is a valid BaseAddr, we rely on max 64-bit value to be an invalid
+ * page address.
+ */
+#define INVALID_BASE_ADDR (IMG_UINT64_MAX)
+/* Used to check for duplicated alloc indices in sparse alloc path
+ * prior to attempting allocations */
+#define RA_BASE_SPARSE_PREP_ALLOC_ADDR (IMG_UINT64_MAX - 1)
+#define RA_BASE_FLAGS_MASK 0xFFF /* 12 Bits 4k alignment. */
+#define RA_BASE_FLAGS_LOG2 12
+#define RA_BASE_CHUNK_LOG2_MAX 64
+#define RA_BASE_GHOST_BIT (1ULL << 0)
+#define RA_BASE_STRIP_GHOST_BIT(uiBase) ((uiBase) & ~(RA_BASE_GHOST_BIT))
+#define RA_BASE_SET_GHOST_BIT(uiBase)   ((uiBase) |= RA_BASE_GHOST_BIT)
+#define RA_BASE_IS_GHOST(uiBase) (BITMASK_HAS((uiBase), RA_BASE_GHOST_BIT) && (uiBase) != INVALID_BASE_ADDR)
+#define RA_BASE_IS_REAL(uiBase) (!BITMASK_HAS((uiBase), RA_BASE_GHOST_BIT))
+#define RA_BASE_IS_SPARSE_PREP(uiBase) ((uiBase) == RA_BASE_SPARSE_PREP_ALLOC_ADDR)
+#define RA_BASE_IS_INVALID(uiBase) ((uiBase) == INVALID_BASE_ADDR)
+
+typedef struct _RA_MULTIBASE_ITERATOR_ RA_MULTIBASE_ITERATOR;
+
+/* Lock classes: describes the level of nesting between different arenas.
+ * A RA with lockclass x is permitted to obtain locks of a lockclass less than x.
+ * A RA with lockclass 0 cannot obtain the lock of any other RA. */
+#define RA_LOCKCLASS_0 0
+#define RA_LOCKCLASS_1 1
+#define RA_LOCKCLASS_2 2
+#define RA_LOCKCLASS_3 3
+
+#define RA_NO_IMPORT_MULTIPLIER 1
+
+/*
+ * Allocation Policies that govern the resource areas.
+ * */
+
+/* --- Resource allocation policy definitions ---
+* | 31.........5|.......4....|......3....|........2.............|1...................0|
+* | Reserved    | Non-Contig | No split  | Area bucket selection| Alloc node selection|
+*/
+
+/*
+ * Fast allocation policy allows to pick the first node
+ * that satisfies the request.
+ * It is the default policy for all arenas.
+ *  */
+#define RA_POLICY_ALLOC_FAST			(0U)
+/*
+ * Optimal allocation policy allows to pick the lowest size node
+ * that satisfies the request. This picking policy helps in reducing the fragmentation.
+ * This minimises the necessity to split the nodes more often as the optimal
+ * ones are picked.
+ * As a result any future higher size allocation requests are likely to succeed
+ */
+#define RA_POLICY_ALLOC_OPTIMAL		(1U)
+#define RA_POLICY_ALLOC_NODE_SELECT_MASK			(3U)
+
+/*
+ * Bucket selection policies
+ * */
+/* Assured bucket policy makes sure the selected bucket is guaranteed
+ * to satisfy the given request. Generally Nodes picked up from such a
+ * bucket need to be further split. However picking node that belongs to this
+ * bucket is likely to succeed and thus promises better response times */
+#define RA_POLICY_BUCKET_ASSURED_FIT		(0U)
+/*
+ * Best fit bucket policy selects a bucket with free nodes that are likely
+ * to satisfy the request and nodes that are close to the requested size.
+ * Nodes picked up from this bucket may likely to satisfy the request but not
+ * guaranteed. Failing to satisfy the request from this bucket mean further
+ * higher size buckets are selected in the later iterations till the request
+ * is satisfied.
+ *
+ * Hence response times may vary depending on availability of free nodes
+ * that satisfy the request.
+ * */
+#define RA_POLICY_BUCKET_BEST_FIT		(4U)
+#define RA_POLICY_BUCKET_MASK			(4U)
+
+/* This flag ensures the imports will not be split up and Allocations will always get
+ * their own import
+ */
+#define RA_POLICY_NO_SPLIT			(8U)
+#define RA_POLICY_NO_SPLIT_MASK		(8U)
+
+/* This flag is used in physmem_lma only. it is used to decide if we should
+ * activate the non-contiguous allocation feature of RA MultiAlloc.
+ * Requirements for activation are that the OS implements the
+ * OSMapPageArrayToKernelVA function in osfunc which allows for mapping
+ * physically sparse pages as a virtually contiguous range.
+ * */
+#define RA_POLICY_ALLOC_ALLOW_NONCONTIG      (16U)
+#define RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK (16U)
+
+/*
+ * Default Arena Policy
+ * */
+#define RA_POLICY_DEFAULT			(RA_POLICY_ALLOC_FAST | RA_POLICY_BUCKET_ASSURED_FIT)
+
+/*
+ * Flags in an "import" must match the flags for an allocation
+ */
+typedef IMG_UINT64 RA_FLAGS_T;
+
+typedef struct _RA_IMPORT_ {
+	RA_BASE_T base;           /* Allocation base */
+	RA_LENGTH_T uSize;        /* Allocation size */
+	RA_PERISPAN_HANDLE hPriv; /* Per import private data */
+} RA_IMPORT;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_ALLOC_SINGLE
+@Description    RA import allocate function create a single span when
+                requesting extra resource for the arena.
+@Input          RA_PERARENA_HANDLE RA handle
+@Input          RA_LENGTH_T        Request size
+@Input          RA_FLAGS_T         RA flags
+@Input          RA_LENGTH_T        Base Alignment
+@Input          IMG_CHAR           Annotation
+@Output         RA_IMPORT          Attributes of an import.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_RA_IMPORT_ALLOC_SINGLE)(RA_PERARENA_HANDLE,
+                                                   RA_LENGTH_T,
+                                                   RA_FLAGS_T,
+                                                   RA_LENGTH_T,
+                                                   const IMG_CHAR*,
+                                                   RA_IMPORT*);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_ALLOC_MULTI
+@Description    RA import allocate function is able to create multiple spans
+                for a single import request.
+                The pointer to the array of inputs will be provided but
+                can be set to a larger array via OSAllocMem
+                if the count is not large enough.
+                The pointer will be freed by OSFreeMem after the callback.
+                It is permissible for the callback to set:
+                count == 0 or array pointer == NULL
+                but the callback MUST return an error in these cases.
+@Input          RA_PERARENA_HANDLE RA handle
+@Input          RA_LENGTH_T        Request size
+@Input          RA_FLAGS_T         RA flags
+@Input          RA_LENGTH_T        Base Alignment
+@Input          IMG_CHAR           Annotation
+@Inout          IMG_UINT32         Num of imports = Num of elements in array below.
+                                   Will always be >= 1. Must be set to the number of
+                                   imports returned.
+@Inout          RA_IMPORT          Pointer to array of imports. Will always point
+                                   to an array of equal length as Num of Imports.
+                                   Will be freed by OSFreeMem.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_RA_IMPORT_ALLOC_MULTI)(RA_PERARENA_HANDLE,
+                                                  RA_LENGTH_T,
+                                                  RA_FLAGS_T,
+                                                  RA_LENGTH_T,
+                                                  const IMG_CHAR*,
+                                                  IMG_UINT32*,
+                                                  RA_IMPORT**);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_FREE
+@Description    RA free imported allocation
+@Input          RA_PERARENA_HANDLE   RA handle
+@Input          RA_BASE_T            Allocation base
+@Input          RA_PERISPAN_HANDLE   Per import private data
+*/ /**************************************************************************/
+typedef void (*PFN_RA_IMPORT_FREE)(RA_PERARENA_HANDLE,
+                                   RA_BASE_T,
+                                   RA_PERISPAN_HANDLE);
+
+/**
+ *  @Function   RA_Create
+ *
+ *  @Description    To create a resource arena.
+ *
+ *  @Input name - the name of the arena for diagnostic purposes.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui32LockClass - the lock class level this arena uses.
+ *  @Input imp_alloc - a resource allocation callback or 0.
+ *  @Input imp_free - a resource de-allocation callback or 0.
+ *  @Input per_arena_handle - private handle passed to alloc and free or 0.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+ */
+RA_ARENA *
+RA_Create(IMG_CHAR *name,
+          /* subsequent imports: */
+          RA_LOG2QUANTUM_T uLog2Quantum,
+          IMG_UINT32 ui32LockClass,
+          PFN_RA_IMPORT_ALLOC_SINGLE imp_alloc,
+          PFN_RA_IMPORT_FREE imp_free,
+          RA_PERARENA_HANDLE per_arena_handle,
+          RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_CreateMulti
+ *
+ *  @Description    To create a resource arena using the
+ *                  ALLOC_MULTI callback.
+ *
+ *  @Input name - the name of the arena for diagnostic purposes.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui32LockClass - the lock class level this arena uses.
+ *  @Input imp_alloc - a resource allocation callback or 0.
+ *  @Input imp_free - a resource de-allocation callback or 0.
+ *  @Input arena_handle - private handle passed to alloc and free or 0.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+ */
+RA_ARENA *
+RA_CreateMulti(IMG_CHAR *name,
+               RA_LOG2QUANTUM_T uLog2Quantum,
+               IMG_UINT32 ui32LockClass,
+               PFN_RA_IMPORT_ALLOC_MULTI imp_alloc,
+               PFN_RA_IMPORT_FREE imp_free,
+               RA_PERARENA_HANDLE arena_handle,
+               RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_Create_With_Span
+ *
+ *  @Description
+ *
+ *  Create a resource arena and initialises it, with a given resource span.
+ *
+ *  @Input name - String briefly describing the RA's purpose.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui64CpuBase - CPU Physical Base Address of the RA.
+ *  @Input ui64SpanDevBase - Device Physical Base Address of the RA.
+ *  @Input ui64SpanSize - Size of the span to add to the created RA.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+*/
+RA_ARENA *
+RA_Create_With_Span(IMG_CHAR *name,
+                    RA_LOG2QUANTUM_T uLog2Quantum,
+                    IMG_UINT64 ui64CpuBase,
+                    IMG_UINT64 ui64SpanDevBase,
+                    IMG_UINT64 ui64SpanSize,
+                    RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_Delete
+ *
+ *  @Description
+ *
+ *  To delete a resource arena. All resources allocated from the arena
+ *  must be freed before deleting the arena.
+ *
+ *  @Input  pArena - the arena to delete.
+ *  @Return None
+ */
+void
+RA_Delete(RA_ARENA *pArena);
+
+/**
+ *  @Function   RA_Add
+ *
+ *  @Description
+ *
+ *  To add a resource span to an arena. The span must not overlap with
+ *  any span previously added to the arena.
+ *
+ *  @Input pArena - the arena to add a span into.
+ *  @Input base - the base of the span.
+ *  @Input uSize - the extent of the span.
+ *  @Input hPriv - handle associated to the span (reserved for user uses)
+ *  @Return IMG_TRUE - success, IMG_FALSE - failure
+ */
+IMG_BOOL
+RA_Add(RA_ARENA *pArena,
+       RA_BASE_T base,
+       RA_LENGTH_T uSize,
+       RA_FLAGS_T uFlags,
+       RA_PERISPAN_HANDLE hPriv);
+
+/**
+ *  @Function   RA_Alloc
+ *
+ *  @Description    To allocate resource from an arena.
+ *
+ *  @Input  pArena - the arena
+ *  @Input  uRequestSize - the size of resource segment requested.
+ *  @Input  uImportMultiplier - Import x-times of the uRequestSize
+ *          for future RA_Alloc calls.
+ *          Use RA_NO_IMPORT_MULTIPLIER to import the exact size.
+ *  @Input  uImportFlags - flags influencing allocation policy.
+ *  @Input  uAlignment - the alignment constraint required for the
+ *          allocated segment, use 0 if alignment not required.
+ *  @Input  pszAnnotation - a string to describe the allocation
+ *  @Output base - allocated base resource
+ *  @Output pActualSize - the actual_size of resource segment allocated,
+ *          typically rounded up by quantum.
+ *  @Output phPriv - the user reference associated with allocated
+ *          resource span.
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_Alloc(RA_ARENA *pArena,
+         RA_LENGTH_T uRequestSize,
+         IMG_UINT8 uImportMultiplier,
+         RA_FLAGS_T uImportFlags,
+         RA_LENGTH_T uAlignment,
+         const IMG_CHAR *pszAnnotation,
+         RA_BASE_T *base,
+         RA_LENGTH_T *pActualSize,
+         RA_PERISPAN_HANDLE *phPriv);
+
+/*************************************************************************/ /*!
+@Function       RA_AllocMulti
+@Description    To allocate resource from an arena.
+                This method of allocation can be used to guarantee that if there
+                is enough space in the RA and the contiguity given is the
+                greatest common divisor of the contiguities used on this RA
+                the allocation can be made.
+                Allocations with contiguity less than the current GCD
+                (Greatest Common Divisor) abiding to pow2 are also guaranteed to
+                succeed. See scenario 4.
+                Allocations are not guaranteed but still reduce fragmentation
+                using this method when multiple contiguities are used e.g.
+                4k & 16k and the current allocation has a contiguity higher than
+                the greatest common divisor used.
+                Scenarios with Log 2 contiguity examples:
+                1. All allocations have contiguity of 4k. Allocations can be
+                guaranteed given enough RA space since the GCD is always used.
+                2. Allocations of 4k and 16k contiguity have been previously
+                made on this RA. A new allocation of 4k contiguity is guaranteed
+                to succeed given enough RA space since the contiguity is the GCD.
+                3. Allocations of 4k and 16k contiguity have been previously made
+                on this RA. A new allocation of 16k contiguity is not guaranteed
+                to succeed since it is not the GCD of all contiguities used.
+                4. Contiguity 16k and 64k already exist, a 4k contiguity
+                allocation would be guaranteed to succeed but would now be the
+                new GCD. So further allocations would be required to match this
+                GCD to guarantee success.
+                This method does not suffer the same fragmentation pitfalls
+                as RA_Alloc as it constructs the allocation size from many
+                smaller constituent allocations, these are represented and returned
+                in the given array. In addition, Ghost bases are generated in
+                array entries conforming to the chunk size, this allows for
+                representing chunks of any size that work as page addrs
+                in upper levels.
+                The aforementioned array must be at least of size
+                uRequestsize / uiChunkSize, this ensures there is at least one
+                array entry per chunk required.
+                This function must have a uiChunkSize value of
+                at least 4096, this is to ensure space for the base type encoding.
+@Input          pArena            The arena
+@Input          uRequestSize      The size of resource requested.
+@Input          uiLog2ChunkSize   The log2 contiguity multiple of the bases i.e all
+                                  Real bases must be a multiple in size of this
+                                  size, also used to generate Ghost bases.
+                                  Allocations will also be aligned to this value.
+@Input          uImportMultiplier Import x-times more for future requests if
+                                  we have to import new resource.
+@Input          uImportFlags      Flags influencing allocation policy.
+                                  required, otherwise must be a power of 2.
+@Input          pszAnnotation     String to describe the allocation
+@InOut          aBaseArray        Array of bases to populate.
+@Input          uiBaseArraySize   Size of the array to populate.
+@Output         bPhysContig       Are the allocations made in the RA physically
+                                  contiguous.
+@Return         PVRSRV_OK - success
+*/ /**************************************************************************/
+PVRSRV_ERROR
+RA_AllocMulti(RA_ARENA *pArena,
+               RA_LENGTH_T uRequestSize,
+               IMG_UINT32 uiLog2ChunkSize,
+               IMG_UINT8 uImportMultiplier,
+               RA_FLAGS_T uImportFlags,
+               const IMG_CHAR *pszAnnotation,
+               RA_BASE_ARRAY_T aBaseArray,
+               RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+               IMG_BOOL *bPhysContig);
+
+/**
+ * @Function   RA_AllocMultiSparse
+ *
+ * @Description    To Alloc resource from an RA arena at the specified indices.
+ *                 This function follows the same conditions and functionality as
+ *                 RA_AllocMulti although with the added aspect of specifying the
+ *                 indices to allocate in the Base Array. This means we can still
+ *                 attempt to maintain contiguity where possible with the aim of
+ *                 reducing fragmentation and increasing occurrence of optimal free
+ *                 scenarios.
+ * @Input          pArena            The Arena
+ * @Input          uiLog2ChunkSize   The log2 contiguity multiple of the bases i.e all
+ *                                   Real bases must be a multiple in size of this
+ *                                   size, also used to generate Ghost bases.
+ *                                   Allocations will also be aligned to this value.
+ * @Input          uImportMultiplier Import x-times more for future requests if
+ *                                   we have to import new resource.
+ * @Input          uImportFlags      Flags influencing allocation policy.
+ *                                   required, otherwise must be a power of 2.
+ * @Input          pszAnnotation     String to describe the allocation
+ * @InOut          aBaseArray        Array of bases to populate.
+ * @Input          uiBaseArraySize   Size of the array to populate.
+ * @Input          puiAllocIndices   The indices into the array to alloc, if indices are NULL
+ *                                   then we will allocate uiAllocCount chunks sequentially.
+ * @InOut          uiAllocCount      The number of bases to alloc from the array.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_AllocMultiSparse(RA_ARENA *pArena,
+                     IMG_UINT32 uiLog2ChunkSize,
+                     IMG_UINT8 uImportMultiplier,
+                     RA_FLAGS_T uImportFlags,
+                     const IMG_CHAR *pszAnnotation,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                     IMG_UINT32 *puiAllocIndices,
+                     IMG_UINT32 uiAllocCount);
+/**
+ *  @Function   RA_FreeMulti
+ *
+ *  @Description    To free a multi-base resource constructed using
+ *                  a call to RA_AllocMulti.
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aBaseArray - The array to free bases from.
+ *  @Input  uiBaseArraysize - Size of the array to free bases from.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_FreeMulti(RA_ARENA *pArena,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize);
+
+/**
+ *  @Function   RA_FreeMultiSparse
+ *
+ *  @Description    To free part of a multi-base resource constructed using
+ *                  a call to RA_AllocMulti.
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aBaseArray - The array to free bases from.
+ *  @Input  uiBaseArraysize - Size of the array to free bases from.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases.
+ *  @Input  puiFreeIndices - The indices into the array to free.
+ *  @InOut  puiFreeCount - The number of bases to free from the array, becomes the number
+ *                         of bases actually free'd. The in value may differ from the out
+ *                         value in cases of error when freeing. The out value can then be
+ *                         used in upper levels to keep any mem tracking structures consistent
+ *                         with what was actually freed before the error occurred.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_FreeMultiSparse(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                    IMG_UINT32 uiLog2ChunkSize,
+                    IMG_UINT32 *puiFreeIndices,
+                    IMG_UINT32 *puiFreeCount);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/**
+ *  @Function   RA_TransferMultiSparseIndices
+ *
+ *  @Description   Transfers a set of indices specified in puiTransferIndices from
+ *                 aSrcBaseArray to aDstBaseArray.
+ *                 Called when some pages of the base array need to be
+ *                 transfered to another base array. As a result of this call,
+ *                 some ghost addresses in aBaseArray might be converted to
+ *                 real addresses before being transferred..
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aSrcBaseArray - The array to transfer bases from.
+ *  @Input  uiSrcBaseArraySize - Size of the array to transfer bases from.
+ *  @Input  aDstBaseArray - The array to transfer bases to.
+ *  @Input  uiDstBaseArraySize - Size of the array to transfer bases to.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases.
+ *  @Input  puiTransferIndices - The indices into the array to be extracted.
+ *  @InOut  puiTransferCount - The number of bases to prepare for extraction.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_TransferMultiSparseIndices(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aSrcBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiSrcBaseArraySize,
+                              RA_BASE_ARRAY_T aDstBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiDstBaseArraySize,
+                              IMG_UINT32 uiLog2ChunkSize,
+                              IMG_UINT32 *puiTransferIndices,
+                              IMG_UINT32 *puiTransferCount);
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+/**
+ *  @Function   RA_Alloc_Range
+ *
+ *  @Description
+ *
+ *  To allocate a resource at a specified base from an arena.
+ *
+ *  @Input  pArena - the arena
+ *  @Input  uRequestSize - the size of resource segment requested.
+ *  @Input  uImportFlags - flags influencing allocation policy.
+ *  @Input  uAlignment - the alignment constraint required for the
+ *          allocated segment, use 0 if alignment not required.
+ *  @Input  base - allocated base resource
+ *  @Output pActualSize - the actual_size of resource segment allocated,
+ *          typically rounded up by quantum.
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_Alloc_Range(RA_ARENA *pArena,
+		  RA_LENGTH_T uRequestSize,
+		  RA_FLAGS_T uImportFlags,
+		  RA_LENGTH_T uAlignment,
+		  RA_BASE_T base,
+		  RA_LENGTH_T *pActualSize);
+
+/**
+ *  @Function   RA_Free
+ *
+ *  @Description    To free a resource segment.
+ *
+ *  @Input  pArena - the arena the segment was originally allocated from.
+ *  @Input  base - the base of the resource span to free.
+ *
+ *  @Return None
+ */
+void
+RA_Free(RA_ARENA *pArena, RA_BASE_T base);
+
+/**
+ *  @Function   RA_SwapSparseMem
+ *
+ *  @Description    Swaps chunk sized allocations at X<->Y indices.
+ *                  The function is most optimal when Indices are provided
+ *                  in ascending order, this allows the internals to optimally
+ *                  swap based on contiguity and reduces the amount of ghost to
+ *                  real conversion performed. Note this function can also be used
+ *                  to move pages, in this case, we effectively swap real allocations
+ *                  with invalid marked bases.
+ *  @Input  pArena     - The arena.
+ *  @InOut  aBaseArray - The array to Swap bases in.
+ *  @Input  uiBaseArraysize - Size of the array to Swap bases in.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases
+ *                            and size the Real chunks.
+ *  @Input  puiXIndices - Set of X indices to swap with parallel indices in Y.
+ *  @Input  puiYIndices - Set of Y indices to swap with parallel indices in X.
+ *  @Input  uiSwapCount - Number of indices to swap.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_SwapSparseMem(RA_ARENA *pArena,
+                  RA_BASE_ARRAY_T aBaseArray,
+                  RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                  IMG_UINT32 uiLog2ChunkSize,
+                  IMG_UINT32 *puiXIndices,
+                  IMG_UINT32 *puiYIndices,
+                  IMG_UINT32 uiSwapCount);
+
+/**
+ *  @Function   RA_Get_Usage_Stats
+ *
+ *  @Description    To collect the arena usage statistics.
+ *
+ *  @Input  pArena - the arena to acquire usage statistics from.
+ *  @Input  psRAStats - the buffer to hold the usage statistics of the arena.
+ *
+ *  @Return None
+ */
+IMG_INTERNAL void
+RA_Get_Usage_Stats(RA_ARENA *pArena, PRA_USAGE_STATS psRAStats);
+
+/**
+ *  @Function   RA_GetArenaName
+ *
+ *  @Description    To obtain the arena name.
+ *
+ *  @Input  pArena - the arena to acquire the name from.
+ *
+ *  @Return IMG_CHAR* Arena name.
+ */
+IMG_INTERNAL IMG_CHAR *
+RA_GetArenaName(RA_ARENA *pArena);
+
+IMG_INTERNAL RA_ARENA_ITERATOR *
+RA_IteratorAcquire(RA_ARENA *pArena, IMG_BOOL bIncludeFreeSegments);
+
+IMG_INTERNAL void
+RA_IteratorReset(RA_ARENA_ITERATOR *pIter);
+
+IMG_INTERNAL void
+RA_IteratorRelease(RA_ARENA_ITERATOR *pIter);
+
+/*************************************************************************/ /*!
+@Function       RA_IteratorNext
+@Description    Function used to iterate over segments in the arena. Segments
+                can either be allocated or free. Iteration can be configured
+                to include free segments when creating the iterator via
+                RA_IteratorAcquire.
+
+@Input          pIter        Iteration handle used to keep state between calls
+@InOut          pData        Data about current block iteration, base and size
+                             etc...
+@Return         IMG_BOOL     Boolean value signalling if next element exists
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNext(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData);
+
+/*************************************************************************/ /*
+@Function       RA_IteratorNextSpan
+@Description    Function used to iterate over spans in the arena. Spans
+                can either be allocated or free. Iteration can be configured
+                to include free spans when creating the iterator via
+                RA_IteratorAcquire. Spans are not allocations on the arena but
+                instead the regions of resource added that allocations can
+                be made from.
+                See RA_Add().
+
+@Input          pIter        Iteration handle used to keep state between calls
+@InOut          pData        Data about current block iteration, base and size
+                             etc...
+@Return         IMG_BOOL     Boolean value signalling if next element exists
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNextSpan(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData);
+
+/*************************************************************************/ /*!
+@Function       RA_BlockDump
+@Description    Debug dump of all memory allocations within the RA and the space
+                between. A '#' represents a block of memory (the arena's quantum
+                in size) that has been allocated whereas a '.' represents a free
+                block.
+@Input          pArena        The arena to dump.
+@Input          pfnLogDump    The dumping method.
+@Input          pPrivData     Data to be passed into the pfnLogDump method.
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RA_BlockDump(RA_ARENA *pArena,
+             __printf(2, 3) void (*pfnLogDump)(void*, IMG_CHAR*, ...),
+             void *pPrivData);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync.h
new file mode 100644
index 000000000000..f126915060f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync.h
@@ -0,0 +1,292 @@
+/*************************************************************************/ /*!
+@File
+@Title          Synchronisation interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_H
+#define SYNC_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "sync_prim_internal.h"
+#include "pdumpdefs.h"
+#include "dllist.h"
+#include "pvr_debug.h"
+
+#include "device_connection.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncPrimContextCreate
+
+@Description    Create a new synchronisation context
+
+@Input          hBridge                 Bridge handle
+
+@Input          hDeviceNode             Device node handle
+
+@Output         hSyncPrimContext        Handle to the created synchronisation
+                                        primitive context
+
+@Return         PVRSRV_OK if the synchronisation primitive context was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimContextCreate(SHARED_DEV_CONNECTION hDevConnection,
+                      PSYNC_PRIM_CONTEXT    *hSyncPrimContext);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimContextDestroy
+
+@Description    Destroy a synchronisation context
+
+@Input          hSyncPrimContext        Handle to the synchronisation
+                                        primitive context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimContextDestroy(PSYNC_PRIM_CONTEXT hSyncPrimContext);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimAlloc
+
+@Description    Allocate a new synchronisation primitive on the specified
+                synchronisation context
+
+@Input          hSyncPrimContext        Handle to the synchronisation
+                                        primitive context
+
+@Output         ppsSync                 Created synchronisation primitive
+
+@Input          pszClassName            Sync source annotation
+
+@Return         PVRSRV_OK if the synchronisation primitive was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimAlloc(PSYNC_PRIM_CONTEXT      hSyncPrimContext,
+              PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+              const IMG_CHAR          *pszClassName);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimFree
+
+@Description    Free a synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to free
+
+@Return         PVRSRV_OK if the synchronisation primitive was
+                successfully freed
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimFree(PVRSRV_CLIENT_SYNC_PRIM *psSync);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimSet
+
+@Description    Set the synchronisation primitive to a value
+
+@Input          psSync                  The synchronisation primitive to set
+
+@Input          ui32Value               Value to set it to
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimSet(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+
+#if defined(NO_HARDWARE)
+
+/*************************************************************************/ /*!
+@Function       SyncPrimNoHwUpdate
+
+@Description    Updates the synchronisation primitive value (in NoHardware drivers)
+
+@Input          psSync                  The synchronisation primitive to update
+
+@Input          ui32Value               Value to update it to
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimNoHwUpdate(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+#endif
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       SyncPrimPDump
+
+@Description    PDump the current value of the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpValue
+
+@Description    PDump the ui32Value as the value of the synchronisation
+                primitive (regardless of the current value).
+
+@Input          psSync          The synchronisation primitive to PDump
+@Input          ui32Value       Value to give to the sync prim on the pdump
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpPol
+
+@Description    Do a PDump poll of the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Input          ui32Value               Value to poll for
+
+@Input          ui32Mask                PDump mask operator
+
+@Input          ui32PDumpFlags          PDump flags
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT32 ui32Value,
+				 IMG_UINT32 ui32Mask,
+				 PDUMP_POLL_OPERATOR eOperator,
+				 IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpCBP
+
+@Description    Do a PDump CB poll using the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Input          uiWriteOffset           Current write offset of buffer
+
+@Input          uiPacketSize            Size of the packet to write into CB
+
+@Input          uiBufferSize            Size of the CB
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT64 uiWriteOffset,
+				 IMG_UINT64 uiPacketSize,
+				 IMG_UINT64 uiBufferSize);
+
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpValue)
+#endif
+static INLINE void
+SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDump)
+#endif
+static INLINE void
+SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpPol)
+#endif
+static INLINE void
+SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT32 ui32Value,
+				 IMG_UINT32 ui32Mask,
+				 PDUMP_POLL_OPERATOR eOperator,
+				 IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpCBP)
+#endif
+static INLINE void
+SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT64 uiWriteOffset,
+				 IMG_UINT64 uiPacketSize,
+				 IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+}
+#endif /* PDUMP */
+#endif /* SYNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync_internal.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync_internal.h
new file mode 100644
index 000000000000..929b66e0f37c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync_internal.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the internal client side interface for services
+                synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_INTERNAL
+#define SYNC_INTERNAL
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "ra.h"
+#include "dllist.h"
+#include "lock.h"
+#include "devicemem.h"
+#include "sync_prim_internal.h"
+
+#define LOCAL_SYNC_PRIM_RESET_VALUE 0
+#define LOCAL_SYNC_PRIM_POISON_VALUE 0xa5a5a5a5u
+
+/*
+	Debug feature to protect against GP DM page faults when
+	sync prims are freed by client before work is completed.
+*/
+#define LOCAL_SYNC_BLOCK_RETAIN_FIRST
+
+/*
+	Private structure's
+*/
+#define SYNC_PRIM_NAME_SIZE		50
+typedef struct SYNC_PRIM_CONTEXT_TAG
+{
+	SHARED_DEV_CONNECTION       hDevConnection;
+	IMG_CHAR					azName[SYNC_PRIM_NAME_SIZE];	/*!< Name of the RA */
+	RA_ARENA					*psSubAllocRA;					/*!< RA context */
+	IMG_CHAR					azSpanName[SYNC_PRIM_NAME_SIZE];/*!< Name of the span RA */
+	RA_ARENA					*psSpanRA;						/*!< RA used for span management of SubAllocRA */
+	ATOMIC_T				hRefCount;	/*!< Ref count for this context */
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	IMG_HANDLE					hFirstSyncPrim; /*!< Handle to the first allocated sync prim */
+#endif
+} SYNC_PRIM_CONTEXT;
+
+typedef struct SYNC_PRIM_BLOCK_TAG
+{
+	SYNC_PRIM_CONTEXT	*psContext;				/*!< Our copy of the services connection */
+	IMG_HANDLE			hServerSyncPrimBlock;	/*!< Server handle for this block */
+	IMG_UINT32			ui32SyncBlockSize;		/*!< Size of the sync prim block */
+	IMG_UINT32			ui32FirmwareAddr;		/*!< Firmware address */
+	DEVMEM_MEMDESC		*hMemDesc;				/*!< Host mapping handle */
+	IMG_UINT32 __iomem	*pui32LinAddr;			/*!< User CPU mapping */
+	IMG_UINT64			uiSpanBase;				/*!< Base of this import in the span RA */
+	DLLIST_NODE			sListNode;				/*!< List node for the sync block list */
+} SYNC_PRIM_BLOCK;
+
+typedef struct SYNC_PRIM_TAG
+{
+	PVRSRV_CLIENT_SYNC_PRIM	sCommon;		/*!< Client visible part of the sync prim */
+	ATOMIC_T				hRefCount;		/*!< Ref count for this sync */
+	SYNC_PRIM_BLOCK			*psSyncBlock;	/*!< Synchronisation block this primitive is allocated on */
+	IMG_UINT64				uiSpanAddr;		/*!< Span address of the sync */
+	IMG_HANDLE				hRecord;		/*!< Sync record handle */
+} SYNC_PRIM;
+
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimGetFirmwareAddr(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 *pui32FwAddr);
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimLocalGetHandleAndOffset(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+							IMG_HANDLE *phBlock,
+							IMG_UINT32 *pui32Offset);
+
+
+#endif	/* SYNC_INTERNAL */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/tlclient.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tlclient.h
new file mode 100644
index 000000000000..00f7aa8fc043
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tlclient.h
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File           tlclient.h
+@Title          Services Transport Layer shared API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common API used in both clients and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef TLCLIENT_H
+#define TLCLIENT_H
+
+
+#include "img_defs.h"
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_error.h"
+
+
+/* This value is used for the hSrvHandle argument in the client API when
+ * called directly from the kernel which will lead to a direct bridge access.
+ */
+#define DIRECT_BRIDGE_HANDLE	((IMG_HANDLE)0xDEADBEEFU)
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientOpenStream
+ @Description   Open a descriptor onto an existing kernel transport stream.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         pszName         Address of the stream name string, no longer
+                                than PRVSRVTL_MAX_STREAM_NAME_SIZE.
+ @Input         ui32Mode        Unused
+ @Output        phSD            Address of a pointer to an stream object
+ @Return        PVRSRV_ERROR_NOT_FOUND          when named stream not found
+ @Return        PVRSRV_ERROR_ALREADY_OPEN       stream already open by another
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_TIMEOUT            timed out, stream not found
+ @Return        PVRSRV_ERROR                    for other system codes
+*/ /**************************************************************************/
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientOpenStream(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR* pszName,
+		IMG_UINT32   ui32Mode,
+		IMG_HANDLE*  phSD);
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientCloseStream
+ @Description   Close and release the stream connection to Services kernel
+                server transport layer. Any outstanding Acquire will be
+                released.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle is not known
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCloseStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD);
+
+/*************************************************************************/ /*!
+ @Function      TLClientDiscoverStreams
+ @Description   Finds all streams that's name starts with pszNamePattern and
+                ends with a number.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         pszNamePattern  Name pattern. Must be beginning of a string.
+ @Output        aszStreams      Array of numbers from end of the discovered
+                names.
+ @inOut         pui32NumFound   When input, max number that can fit into
+                                pui32Streams. When output, number of
+                                discovered streams.
+ @Return        PVRSRV_ERROR    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientDiscoverStreams(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR *pszNamePattern,
+		IMG_CHAR aszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+		IMG_UINT32 *pui32NumFound);
+
+/*************************************************************************/ /*!
+ @Function      TLClientReserveStream
+ @Description   Reserves a region with given size in the stream. If the stream
+                is already reserved the function will return an error.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Output        ppui8Data       pointer to the buffer
+ @Input         ui32Size        size of the data
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLClientStreamReserve2
+ @Description   Reserves a region with given size in the stream. If the stream
+                is already reserved the function will return an error.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Output        ppui8Data       pointer to the buffer
+ @Input         ui32Size        size of the data
+ @Input         ui32SizeMin     minimum size of the data
+ @Input         ui32Available   available space in buffer
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream2(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size,
+		IMG_UINT32 ui32SizeMin,
+		IMG_UINT32 *pui32Available);
+
+/*************************************************************************/ /*!
+ @Function      TLClientStreamCommit
+ @Description   Commits previously reserved region in the stream and therefore
+                allows next reserves.
+                This function call has to be preceded by the call to
+                TLClientReserveStream or TLClientReserveStream2.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Input         ui32Size        Size of the data
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCommitStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLClientAcquireData
+ @Description   When there is data available in the stream buffer this call
+                returns with the address and length of the data buffer the
+                client can safely read. This buffer may contain one or more
+                packets of data.
+                If no data is available then this call blocks until it becomes
+                available. However if the stream has been destroyed while
+                waiting then a resource unavailable error will be returned to
+                the caller. Clients must pair this call with a ReleaseData
+                call.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Output        ppPacketBuf     Address of a pointer to an byte buffer. On exit
+                                pointer contains address of buffer to read from
+ @Output        puiBufLen       Pointer to an integer. On exit it is the size
+                                of the data to read from the packet buffer
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              release not called beforehand
+ @Return        PVRSRV_ERROR_TIMEOUT            block timed out, no data
+ @Return        PVRSRV_ERROR                    for other system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientAcquireData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE  hSD,
+		IMG_PBYTE*  ppPacketBuf,
+		IMG_UINT32* puiBufLen);
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientReleaseData
+ @Description   Called after client has read the stream data out of the buffer
+                The data is subsequently flushed from the stream buffer to make
+                room for more data packets from the stream source.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known to TL
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              acquire not called beforehand
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD);
+
+/*************************************************************************/ /*!
+ @Function      TLClientReleaseDataLess
+ @Description   Called after client has read only some data out of the buffer
+                and wishes to complete the read early i.e. does not want to
+                read the full data that the acquire call returned e.g read just
+                one packet from the stream.
+                The data is subsequently flushed from the stream buffer to make
+                room for more data packets from the stream source.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Input         uiActualReadLen Size of data read, in bytes. Must be on a TL
+                                packet boundary.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS     when read length too big
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known to TL
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              acquire not called beforehand
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseDataLess(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD, IMG_UINT32 uiActualReadLen);
+
+/*************************************************************************/ /*!
+ @Function      TLClientWriteData
+ @Description   Writes data to the stream.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Input         ui32Size        Size of the data
+ @Input         pui8Data        Pointer to data
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientWriteData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size,
+		IMG_BYTE *pui8Data);
+
+
+#endif /* TLCLIENT_H */
+
+/******************************************************************************
+ End of file (tlclient.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/tutilsdefs.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tutilsdefs.h
new file mode 100644
index 000000000000..b89e4a4bbb1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tutilsdefs.h
@@ -0,0 +1,230 @@
+/*************************************************************************/ /*!
+@File           tutilsdefs.h
+@Title          Testing utils bridge defines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared structures and constants between client and server sides
+                of tutils bridge
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TUTILSDEFS_H
+#define TUTILSDEFS_H
+
+
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_sync_km.h"
+
+/******************************************************************************
+ *
+ * TEST Related definitions and constants
+ */
+#define PVR_TL_TEST_STREAM_BRIDGE_NAME "TLBRIDGE_TEST"
+#define PVR_TL_TEST_UMBASE             0x00202000
+#define PVR_TL_TEST_OFFSET             0x0008
+#define PVR_TL_TEST_LEN                0x0010
+
+#define PVR_TL_TEST_STREAM2_NAME	"TLSTREAM2_TEST"
+#define PVR_TL_TEST_STREAM2_SIZE	2
+
+#define PVR_TL_TEST_STREAM3_NAME	"TLSTREAM3_TEST"
+#define PVR_TL_TEST_STREAM3_SIZE	256
+
+// This constant, when used as a parameter in StreamCreate, lessens the size of
+// the buffer that is created for a stream, to avoid going over a page boundary.
+#define PVR_TL_TEST_STREAM_BUFFER_REDUCTION 32
+
+#define PVR_TL_TEST_CMD_SOURCE_START			10
+typedef struct _PVR_TL_TEST_CMD_SOURCE_START_IN_
+{
+	/* Stream name must always be first in struct */
+	IMG_CHAR    pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_UINT16  uiStreamSizeInPages;  /* # of 4Kb pages */
+	IMG_UINT16  uiInterval;           /* in milliseconds */
+	IMG_UINT16  uiCallbackKicks;      /* 0 for no limit of timer call backs */
+	IMG_UINT16  uiEOSMarkerKicks;     /* Insert EOS Marker every N Kicks, 0 for none */
+	IMG_UINT16  uiPacketSizeInBytes;  /* 0 for random size between 1..255 size in bytes */
+	IMG_UINT32  uiStreamCreateFlags;  /* See TLStreamCreate() */
+	IMG_UINT16  uiStartDelay;         /* 0 for normal uiInterval delay, one off delay in ms */
+	IMG_BOOL    bDoNotDeleteStream;   /* When true the stream is not deleted on self
+	                                   * cleanup sources only the timers and other resources are */
+	IMG_BOOL    bDelayStreamCreate;   /* When true the stream used in the source is created
+	                                   * in the first kick. False for normal behaviour where
+	                                   * the stream is created in the bridge source start context */
+} PVR_TL_TEST_CMD_SOURCE_START_IN;
+
+
+#define PVR_TL_TEST_CMD_SOURCE_STOP		11
+typedef struct _PVR_TL_TEST_CMD_SOURCE_STOP_IN_
+{
+	/* Stream name must always be first in struct */
+	IMG_CHAR  pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_BOOL  bDoNotDeleteStream;
+} PVR_TL_TEST_CMD_SOURCE_STOP_IN;
+
+#define PVR_TL_TEST_CMD_SOURCE_START2	12	/* Uses two stage data submit */
+typedef PVR_TL_TEST_CMD_SOURCE_START_IN PVR_TL_TEST_CMD_SOURCE_START2_IN;
+
+#define PVR_TL_TEST_CMD_DEBUG_LEVEL	13
+/* No typedef, uses integer uiIn1 in union */
+
+#define PVR_TL_TEST_CMD_DUMP_TL_STATE	14
+/* No typedef, uses integer uiIn1 in union */
+
+#define PVR_TL_TEST_CMD_STREAM_CREATE	15
+typedef struct _PVR_TL_TEST_CMD_STREAM_CREATE_IN_
+{
+	/* Stream name must always be first in struct */
+	IMG_CHAR    pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_UINT16  uiStreamSizeInPages;
+	IMG_UINT32  uiStreamCreateFlags;
+	IMG_BOOL    bWithOpenCallback;
+} PVR_TL_TEST_CMD_STREAM_CREATE_IN;
+
+#define PVR_TL_TEST_CMD_STREAM_CLOSE	16
+typedef struct _PVR_TL_TEST_CMD_STREAM_NAME_IN_
+{
+	IMG_CHAR  pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+} PVR_TL_TEST_CMD_STREAM_NAME_IN;
+
+#define PVR_TL_TEST_CMD_STREAM_OPEN 17
+
+#define PVR_TL_TEST_CMD_DUMP_HWPERF_STATE 18
+
+#define PVR_TL_TEST_CMD_FLUSH_HWPERF_FWBUF 19
+
+#define PVR_TL_TEST_CMD_DUMP_PDUMP_STATE 21
+
+typedef union _PVR_TL_TEST_CMD_IN_
+{
+	PVR_TL_TEST_CMD_SOURCE_START_IN sStart;
+	PVR_TL_TEST_CMD_SOURCE_STOP_IN  sStop;
+/*	PVR_TL_TEST_CMD_SOURCE_START_IN sStart2;  Used by #12, use sStart instead */
+	IMG_UINT32	uiIn1;						 /* Used by #13, #14 */
+	PVR_TL_TEST_CMD_STREAM_CREATE_IN  sCreate;
+	PVR_TL_TEST_CMD_STREAM_NAME_IN sName;
+	IMG_UINT32 uiParams[6];
+} PVR_TL_TEST_CMD_IN;
+
+/* Has to be the largest test IN structure */
+#define PVR_TL_TEST_PARAM_MAX_SIZE  (sizeof(PVR_TL_TEST_CMD_IN)+4)
+
+#define PVR_TL_TEST_CMD_SET_PWR_STATE              22
+#define PVR_TL_TEST_CMD_GET_PWR_STATE              23
+#define PVR_TL_TEST_CMD_SET_DWT_PWR_CHANGE_COUNTER 24
+#define PVR_TL_TEST_CMD_GET_DWT_PWR_CHANGE_COUNTER 25
+
+#define PVR_TL_TEST_PWR_STATE_ON  1
+#define PVR_TL_TEST_PWR_STATE_OFF 0
+
+/****************************************************************************
+ * PowMonTestThread IOCTL calls and constants
+ */
+
+#define PVR_POWMON_CMD_GET_ESTIMATES	1
+#define PVR_POWMON_CMD_SET_THREAD_LATENCY	2
+#define PVR_POWMON_CMD_TEST_THREAD_UPDATE_STATE	3
+
+#define PVR_POWMON_TEST_THREAD_RESUME	1
+#define PVR_POWMON_TEST_THREAD_PAUSE	0
+
+/****************************************************************************
+ * PowerTestThread IOCTL calls and constants
+ */
+
+#define PVR_POWER_TEST_CMD_DVFS                 1
+#define PVR_POWER_TEST_CMD_FORCED_IDLE          2
+#define PVR_POWER_TEST_CMD_CANCEL_FORCED_IDLE   3
+#define PVR_POWER_TEST_CMD_POWER_ON             4
+#define PVR_POWER_TEST_CMD_POWER_OFF            5
+#define PVR_POWER_TEST_CMD_APM_LATENCY          6
+#define PVR_POWER_TEST_CMD_INVALID              7
+
+#define PVR_POWER_TEST_NON_FORCED 0
+#define PVR_POWER_TEST_FORCED     1
+
+/****************************************************************************
+ * SyncCheckpointTest IOCTL types
+ */
+
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CONTEXT_CREATE		26
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CONTEXT_DESTROY		27
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_REGISTER_FUNCS		28
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CREATE				29
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CREATE_NULL_CTXT	30
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CREATE_NULL_RTRN	31
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_DESTROY				32
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_SIGNAL				33
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_ERROR				34
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_IS_SIGNALLED		35
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_IS_ERRORED			36
+
+typedef struct _PVR_TL_TEST_CMD_CHECKPOINT_CREATE_IN_
+{
+	/* Checkpoint name must always be first in struct */
+	IMG_CHAR    pszCheckpointName[PVRSRV_SYNC_NAME_LENGTH];
+	IMG_UINT16  uiStreamSizeInPages;
+	IMG_UINT32  uiStreamCreateFlags;
+} PVR_TL_TEST_CMD_CHECKPOINT_CREATE_IN;
+
+#define PVR_TL_TEST_CMD_SET_STREAM_OPEN_COUNTER 37
+#define PVR_TL_TEST_CMD_GET_STREAM_OPEN_COUNTER 38
+
+typedef struct _PVR_TL_TEST_CMD_STREAM_OPEN_COUNTER_IN_
+{
+	IMG_CHAR    pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_UINT32  ui32Counter;
+} PVR_TL_TEST_CMD_STREAM_OPEN_COUNTER_IN;
+
+/****************************************************************************
+ * KmallocThreshold IOCTL types
+ */
+
+#define PVR_TL_TEST_CMD_KMALLOC 39
+
+typedef struct _PVR_TL_TEST_CMD_KMALLOC_IN_
+{
+	IMG_UINT32 uiAllocCount;
+	IMG_UINT32 uiAllocSize;
+	IMG_UINT32 uiFailedAllocThreshold;
+	IMG_UINT32 uiFailedAllocFrequency;
+} PVR_TL_TEST_CMD_KMALLOC_IN;
+
+#endif /* TUTILSDEFS_H */
+
+/******************************************************************************
+ End of file (tutilsdefs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/interrupt_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/interrupt_support.c
new file mode 100644
index 000000000000..c67d45352050
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/interrupt_support.c
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/interrupt.h>
+
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "interrupt_support.h"
+
+typedef struct LISR_DATA_TAG
+{
+	IMG_UINT32	ui32IRQ;
+	PFN_SYS_LISR	pfnLISR;
+	void		*pvData;
+} LISR_DATA;
+
+static irqreturn_t SystemISRWrapper(int irq, void *dev_id)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *)dev_id;
+
+	PVR_UNREFERENCED_PARAMETER(irq);
+
+	if (psLISRData)
+	{
+		if (psLISRData->pfnLISR(psLISRData->pvData))
+		{
+			return IRQ_HANDLED;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Missing interrupt data", __func__));
+	}
+
+	return IRQ_NONE;
+}
+
+PVRSRV_ERROR OSInstallSystemLISR(IMG_HANDLE *phLISR,
+				 IMG_UINT32 ui32IRQ,
+				 const IMG_CHAR *pszDevName,
+				 PFN_SYS_LISR pfnLISR,
+				 void *pvData,
+				 IMG_UINT32 ui32Flags)
+{
+	LISR_DATA *psLISRData;
+	unsigned long ulIRQFlags = 0;
+
+	if (pfnLISR == NULL || pvData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32Flags & ~SYS_IRQ_FLAG_MASK)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (ui32Flags & SYS_IRQ_FLAG_TRIGGER_MASK)
+	{
+		case SYS_IRQ_FLAG_TRIGGER_DEFAULT:
+			break;
+		case SYS_IRQ_FLAG_TRIGGER_LOW:
+			ulIRQFlags |= IRQF_TRIGGER_LOW;
+			break;
+		case SYS_IRQ_FLAG_TRIGGER_HIGH:
+			ulIRQFlags |= IRQF_TRIGGER_HIGH;
+			break;
+		default:
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32Flags & SYS_IRQ_FLAG_SHARED)
+	{
+		ulIRQFlags |= IRQF_SHARED;
+	}
+
+	psLISRData = OSAllocMem(sizeof(*psLISRData));
+	if (psLISRData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psLISRData->ui32IRQ = ui32IRQ;
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+
+	if (request_irq(ui32IRQ, SystemISRWrapper, ulIRQFlags, pszDevName, psLISRData))
+	{
+		OSFreeMem(psLISRData);
+
+		return PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER;
+	}
+
+	*phLISR = (IMG_HANDLE)psLISRData;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSUninstallSystemLISR(IMG_HANDLE hLISR)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *)hLISR;
+
+	if (psLISRData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	free_irq(psLISRData->ui32IRQ, psLISRData);
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/pci_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/pci_support.c
new file mode 100644
index 000000000000..a8321a493e3e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/pci_support.c
@@ -0,0 +1,643 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/pci.h>
+
+#if defined(CONFIG_MTRR)
+#include <asm/mtrr.h>
+#endif
+
+#include "pci_support.h"
+#include "allocmem.h"
+
+typedef	struct _PVR_PCI_DEV_TAG
+{
+	struct pci_dev		*psPCIDev;
+	HOST_PCI_INIT_FLAGS	ePCIFlags;
+	IMG_BOOL		abPCIResourceInUse[DEVICE_COUNT_RESOURCE];
+	int			iMTRR[DEVICE_COUNT_RESOURCE];
+} PVR_PCI_DEV;
+
+/*************************************************************************/ /*!
+@Function       OSPCISetDev
+@Description    Set a PCI device for subsequent use.
+@Input          pvPCICookie             Pointer to OS specific PCI structure
+@Input          eFlags                  Flags
+@Return		PVRSRV_PCI_DEV_HANDLE   Pointer to PCI device handle
+*/ /**************************************************************************/
+PVRSRV_PCI_DEV_HANDLE OSPCISetDev(void *pvPCICookie, HOST_PCI_INIT_FLAGS eFlags)
+{
+	int err;
+	IMG_UINT32 i;
+	PVR_PCI_DEV *psPVRPCI;
+
+	psPVRPCI = OSAllocMem(sizeof(*psPVRPCI));
+	if (psPVRPCI == NULL)
+	{
+		printk(KERN_ERR "OSPCISetDev: Couldn't allocate PVR PCI structure\n");
+		return NULL;
+	}
+
+	psPVRPCI->psPCIDev = (struct pci_dev *)pvPCICookie;
+	psPVRPCI->ePCIFlags = eFlags;
+
+	err = pci_enable_device(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCISetDev: Couldn't enable device (%d)\n", err);
+		OSFreeMem(psPVRPCI);
+		return NULL;
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_set_master(psPVRPCI->psPCIDev);
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_MSI)		/* PRQA S 3358 */ /* misuse of enums */
+	{
+#if defined(CONFIG_PCI_MSI)
+		err = pci_enable_msi(psPVRPCI->psPCIDev);
+		if (err != 0)
+		{
+			printk(KERN_ERR "OSPCISetDev: Couldn't enable MSI (%d)", err);
+			psPVRPCI->ePCIFlags &= ~HOST_PCI_INIT_FLAG_MSI;	/* PRQA S 1474,3358,4130 */ /* misuse of enums */
+		}
+#else
+		printk(KERN_ERR "OSPCISetDev: MSI support not enabled in the kernel");
+#endif
+	}
+
+	/* Initialise the PCI resource and MTRR tracking array */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		psPVRPCI->abPCIResourceInUse[i] = IMG_FALSE;
+		psPVRPCI->iMTRR[i] = -1;
+	}
+
+	return (PVRSRV_PCI_DEV_HANDLE)psPVRPCI;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAcquireDev
+@Description    Acquire a PCI device for subsequent use.
+@Input          ui16VendorID            Vendor PCI ID
+@Input          ui16DeviceID            Device PCI ID
+@Input          eFlags                  Flags
+@Return		PVRSRV_PCI_DEV_HANDLE   Pointer to PCI device handle
+*/ /**************************************************************************/
+PVRSRV_PCI_DEV_HANDLE OSPCIAcquireDev(IMG_UINT16 ui16VendorID,
+				      IMG_UINT16 ui16DeviceID,
+				      HOST_PCI_INIT_FLAGS eFlags)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(ui16VendorID, ui16DeviceID, NULL);
+	if (psPCIDev == NULL)
+	{
+		return NULL;
+	}
+
+	return OSPCISetDev((void *)psPCIDev, eFlags);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIIRQ
+@Description    Get the interrupt number for the device.
+@Input          hPVRPCI                 PCI device handle
+@Output         pui16DeviceID           Pointer to where the interrupt number
+                                        should be returned
+@Return		PVRSRV_ERROR            Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIIRQ(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 *pui32IRQ)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (pui32IRQ == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui32IRQ = psPVRPCI->psPCIDev->irq;
+
+	return PVRSRV_OK;
+}
+
+/* Functions supported by OSPCIAddrRangeFunc */
+enum HOST_PCI_ADDR_RANGE_FUNC
+{
+	HOST_PCI_ADDR_RANGE_FUNC_LEN,
+	HOST_PCI_ADDR_RANGE_FUNC_START,
+	HOST_PCI_ADDR_RANGE_FUNC_END,
+	HOST_PCI_ADDR_RANGE_FUNC_REQUEST,
+	HOST_PCI_ADDR_RANGE_FUNC_RELEASE
+};
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeFunc
+@Description    Internal support function for various address range related
+                functions
+@Input          eFunc                   Function to perform
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Function dependent value
+*/ /**************************************************************************/
+static IMG_UINT64 OSPCIAddrRangeFunc(enum HOST_PCI_ADDR_RANGE_FUNC eFunc,
+										 PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+										 IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (ui32Index >= DEVICE_COUNT_RESOURCE)
+	{
+		printk(KERN_ERR "OSPCIAddrRangeFunc: Index out of range");
+		return 0;
+	}
+
+	switch (eFunc)
+	{
+		case HOST_PCI_ADDR_RANGE_FUNC_LEN:
+		{
+			return pci_resource_len(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_START:
+		{
+			return pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_END:
+		{
+			return pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_REQUEST:
+		{
+			int err = pci_request_region(psPVRPCI->psPCIDev, (IMG_INT)ui32Index, PVRSRV_MODNAME);
+			if (err != 0)
+			{
+				printk(KERN_ERR "OSPCIAddrRangeFunc: pci_request_region_failed (%d)", err);
+				return 0;
+			}
+			psPVRPCI->abPCIResourceInUse[ui32Index] = IMG_TRUE;
+			return 1;
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_RELEASE:
+		{
+			if (psPVRPCI->abPCIResourceInUse[ui32Index])
+			{
+				pci_release_region(psPVRPCI->psPCIDev, (IMG_INT)ui32Index);
+				psPVRPCI->abPCIResourceInUse[ui32Index] = IMG_FALSE;
+			}
+			return 1;
+		}
+		default:
+		{
+			printk(KERN_ERR "OSPCIAddrRangeFunc: Unknown function");
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeLen
+@Description    Returns length of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Length of address range or 0 if no
+                                        such range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeLen(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_LEN, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeStart
+@Description    Returns the start of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Start of address range or 0 if no
+                                        such range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeStart(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_START, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeEnd
+@Description    Returns the end of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              End of address range or 0 if no such
+                                        range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeEnd(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_END, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIRequestAddrRange
+@Description    Request a given address range index for subsequent use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIRequestAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+								   IMG_UINT32 ui32Index)
+{
+	if (OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_REQUEST, hPVRPCI, ui32Index) == 0)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseAddrRange
+@Description    Release a given address range that is no longer being used
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	if (OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_RELEASE, hPVRPCI, ui32Index) == 0)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIRequestAddrRegion
+@Description    Request a given region from an address range for subsequent use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Input          uiOffset              Offset into the address range that forms
+                                        the start of the region
+@Input          uiLength              Length of the region
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIRequestAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+									IMG_UINT32 ui32Index,
+									IMG_UINT64 uiOffset,
+									IMG_UINT64 uiLength)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start;
+	resource_size_t end;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+
+	/* Check that the requested region is valid */
+	if ((start + uiOffset + uiLength - 1) > end)
+	{
+		return PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH;
+	}
+
+	if (pci_resource_flags(psPVRPCI->psPCIDev, ui32Index) & IORESOURCE_IO)
+	{
+		if (request_region(start + uiOffset, uiLength, PVRSRV_MODNAME) == NULL)
+		{
+			return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		}
+	}
+	else
+	{
+		if (request_mem_region(start + uiOffset, uiLength, PVRSRV_MODNAME) == NULL)
+		{
+			return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseAddrRegion
+@Description    Release a given region, from an address range, that is no
+                longer in use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Input          ui32Offset              Offset into the address range that forms
+                                        the start of the region
+@Input          ui32Length              Length of the region
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+									IMG_UINT32 ui32Index,
+									IMG_UINT64 uiOffset,
+									IMG_UINT64 uiLength)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start;
+	resource_size_t end;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+
+	/* Check that the region is valid */
+	if ((start + uiOffset + uiLength - 1) > end)
+	{
+		return PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH;
+	}
+
+	if (pci_resource_flags(psPVRPCI->psPCIDev, ui32Index) & IORESOURCE_IO)
+	{
+		release_region(start + uiOffset, uiLength);
+	}
+	else
+	{
+		release_mem_region(start + uiOffset, uiLength);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseDev
+@Description    Release a PCI device that is no longer being used
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int i;
+
+	/* Release all PCI regions that are currently in use */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			pci_release_region(psPVRPCI->psPCIDev, i);
+			psPVRPCI->abPCIResourceInUse[i] = IMG_FALSE;
+		}
+	}
+
+#if defined(CONFIG_PCI_MSI)
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_MSI)		/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_disable_msi(psPVRPCI->psPCIDev);
+	}
+#endif
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_clear_master(psPVRPCI->psPCIDev);
+	}
+
+	pci_disable_device(psPVRPCI->psPCIDev);
+
+	OSFreeMem(psPVRPCI);
+	/*not nulling pointer, copy on stack*/
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCISuspendDev
+@Description    Prepare PCI device to be turned off by power management
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCISuspendDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int i;
+	int err;
+
+	/* Release all PCI regions that are currently in use */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			pci_release_region(psPVRPCI->psPCIDev, i);
+		}
+	}
+
+	err = pci_save_state(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCISuspendDev: pci_save_state_failed (%d)", err);
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+
+	pci_disable_device(psPVRPCI->psPCIDev);
+
+	err = pci_set_power_state(psPVRPCI->psPCIDev, pci_choose_state(psPVRPCI->psPCIDev, PMSG_SUSPEND));
+	switch (err)
+	{
+		case 0:
+			break;
+		case -EIO:
+			printk(KERN_ERR "OSPCISuspendDev: device doesn't support PCI PM");
+			break;
+		case -EINVAL:
+			printk(KERN_ERR "OSPCISuspendDev: can't enter requested power state");
+			break;
+		default:
+			printk(KERN_ERR "OSPCISuspendDev: pci_set_power_state failed (%d)", err);
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIResumeDev
+@Description    Prepare a PCI device to be resumed by power management
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIResumeDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int err;
+	int i;
+
+	err = pci_set_power_state(psPVRPCI->psPCIDev, pci_choose_state(psPVRPCI->psPCIDev, PMSG_ON));
+	switch (err)
+	{
+		case 0:
+			break;
+		case -EIO:
+			printk(KERN_ERR "OSPCIResumeDev: device doesn't support PCI PM");
+			break;
+		case -EINVAL:
+			printk(KERN_ERR "OSPCIResumeDev: can't enter requested power state");
+			return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+		default:
+			printk(KERN_ERR "OSPCIResumeDev: pci_set_power_state failed (%d)", err);
+			return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	pci_restore_state(psPVRPCI->psPCIDev);
+
+	err = pci_enable_device(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCIResumeDev: Couldn't enable device (%d)", err);
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+		pci_set_master(psPVRPCI->psPCIDev);
+
+	/* Restore the PCI resource tracking array */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			err = pci_request_region(psPVRPCI->psPCIDev, i, PVRSRV_MODNAME);
+			if (err != 0)
+			{
+				printk(KERN_ERR "OSPCIResumeDev: pci_request_region_failed (region %d, error %d)", i, err);
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIGetVendorDeviceIDs
+@Description    Retrieve PCI vendor ID and device ID.
+@Input          hPVRPCI                 PCI device handle
+@Output         pui16VendorID           Vendor ID
+@Output         pui16DeviceID           Device ID
+@Return         PVRSRV_ERROR            Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIGetVendorDeviceIDs(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+                                     IMG_UINT16 *pui16VendorID,
+                                     IMG_UINT16 *pui16DeviceID)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	struct pci_dev *psPCIDev;
+
+	if (psPVRPCI == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psPCIDev = psPVRPCI->psPCIDev;
+	if (psPCIDev == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui16VendorID = psPCIDev->vendor;
+	*pui16DeviceID = psPCIDev->device;
+
+	return PVRSRV_OK;
+}
+
+#if defined(CONFIG_MTRR)
+
+/*************************************************************************/ /*!
+@Function       OSPCIClearResourceMTRRs
+@Description    Clear any BIOS-configured MTRRs for a PCI memory region
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start, end;
+	int res;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index) + 1;
+
+	res = arch_io_reserve_memtype_wc(start, end - start);
+	if (res)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	res = arch_phys_wc_add(start, end - start);
+	if (res < 0)
+	{
+		arch_io_free_memtype_wc(start, end - start);
+
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	psPVRPCI->iMTRR[ui32Index] = res;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseResourceMTRRs
+@Description    Release resources allocated by OSPCIClearResourceMTRRs
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+*/ /**************************************************************************/
+void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (psPVRPCI->iMTRR[ui32Index] >= 0)
+	{
+		arch_phys_wc_del(psPVRPCI->iMTRR[ui32Index]);
+		psPVRPCI->iMTRR[ui32Index] = -1;
+
+		{
+			resource_size_t start, end;
+
+			start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+			end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index) + 1;
+
+			arch_io_free_memtype_wc(start, end - start);
+		}
+	}
+}
+#endif /* defined(CONFIG_MTRR) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/sysconfig_cmn.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/sysconfig_cmn.c
new file mode 100644
index 000000000000..d8a09182dc04
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/sysconfig_cmn.c
@@ -0,0 +1,167 @@
+/*************************************************************************/ /*!
+@File
+@Title          Sysconfig layer common to all platforms
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements system layer functions common to all platforms
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "pvr_debug.h"
+#include "os_apphint.h"
+//#include "physmem.h"
+
+void SysRGXErrorNotify(IMG_HANDLE hSysData,
+                       PVRSRV_ROBUSTNESS_NOTIFY_DATA *psErrorData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+	{
+		IMG_UINT32 ui32DgbLvl;
+
+		switch (psErrorData->eResetReason)
+		{
+			case RGX_CONTEXT_RESET_REASON_NONE:
+			case RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP:
+			case RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP:
+			case RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING:
+			case RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING:
+			case RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH:
+			case RGX_CONTEXT_RESET_REASON_GPU_ECC_OK:
+			case RGX_CONTEXT_RESET_REASON_FW_ECC_OK:
+			{
+				ui32DgbLvl = PVR_DBG_MESSAGE;
+				break;
+			}
+			case RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR:
+			case RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR:
+			case RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR:
+			case RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR:
+			{
+				ui32DgbLvl = PVR_DBG_WARNING;
+				break;
+			}
+			case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_FW_ECC_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_WATCHDOG:
+			case RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT:
+			case RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR:
+			case RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH:
+			case RGX_CONTEXT_RESET_REASON_DCLS_ERR:
+			{
+				ui32DgbLvl = PVR_DBG_ERROR;
+				break;
+			}
+			default:
+			{
+				PVR_ASSERT(false && "Unhandled reset reason");
+				ui32DgbLvl = PVR_DBG_ERROR;
+				break;
+			}
+		}
+
+		if (psErrorData->pid > 0)
+		{
+			PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, " PID %d experienced error %d",
+					 psErrorData->pid, psErrorData->eResetReason);
+		}
+		else
+		{
+			PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, " Device experienced error %d",
+					 psErrorData->eResetReason);
+		}
+
+		switch (psErrorData->eResetReason)
+		{
+			case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+			{
+				PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, "   ExtJobRef 0x%x, DM %d",
+						 psErrorData->uErrData.sChecksumErrData.ui32ExtJobRef,
+						 psErrorData->uErrData.sChecksumErrData.eDM);
+			break;
+			}
+			default:
+			{
+				break;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psErrorData);
+#endif /* PVRSRV_NEED_PVR_DPF */
+}
+
+IMG_UINT64 SysRestrictGpuLocalPhysheap(IMG_UINT64 uiHeapSize)
+{
+	return uiHeapSize;
+}
+
+IMG_BOOL SysRestrictGpuLocalAddPrivateHeap(void)
+{
+	return IMG_FALSE;
+}
+
+IMG_BOOL SysDefaultToCpuLocalHeap(void)
+{
+//#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault = IMG_FALSE;
+	IMG_BOOL bSetToCPULocal = IMG_FALSE;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState,
+			PhysHeapHybridDefault2CpuLocal, &bAppHintDefault, &bSetToCPULocal);
+	OSFreeAppHintState(pvAppHintState);
+
+	return bSetToCPULocal;
+//#else
+//	return IMG_FALSE;
+//#endif
+}
+/******************************************************************************
+ End of file (sysconfig_cmn.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/uma_heap_fns.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/uma_heap_fns.c
new file mode 100644
index 000000000000..77e481a74e42
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/uma_heap_fns.c
@@ -0,0 +1,88 @@
+/*****************************************************************************
+@File
+@Title          Boilerplate heap functions for UMA systems
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements generic nop conversions for physical addresses in
+                UMA systems.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#include "uma_heap_fns.h"
+
+/*
+ * CPU to Device physical address translation
+ */
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				   IMG_UINT32 ui32NumOfAddr,
+				   IMG_DEV_PHYADDR *psDevPAddr,
+				   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+	}
+}
+/*
+ * Device to CPU physical address translation
+ */
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				   IMG_UINT32 ui32NumOfAddr,
+				   IMG_CPU_PHYADDR *psCpuPAddr,
+				   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+	}
+}
+
+PHYS_HEAP_FUNCTIONS g_sUmaHeapFns = {
+	.pfnCpuPAddrToDevPAddr = UMAPhysHeapCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = UMAPhysHeapDevPAddrToCpuPAddr,
+};
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/Kbuild.mk
new file mode 100644
index 000000000000..6bd9041535b3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/Kbuild.mk
@@ -0,0 +1,58 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += \
+	services/system/$(PVR_SYSTEM)/sysconfig.o \
+	services/system/common/sysconfig_cmn.o \
+	services/server/common/vmm_pvz_client.o \
+	services/server/common/vmm_pvz_server.o \
+	services/server/common/vz_vmm_pvz.o \
+	services/server/common/vz_vmm_vm.o \
+	services/system/eswin_cpu/common/vmm_type_$(VMM_TYPE).o
+
+ccflags-y += \
+	-I$(TOP)/services/system/eswin_cpu/common/env/linux \
+	-I$(TOP)/services/system/common/env/linux \
+	-I$(TOP)/services/include/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc \
+	-I$(TOP)/include/system/$(PVR_SYSTEM)
+	
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/env/linux/dma_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/env/linux/dma_support.c
new file mode 100644
index 000000000000..977f9c85d037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/env/linux/dma_support.c
@@ -0,0 +1,335 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#endif
+
+#include "allocmem.h"
+#include "dma_support.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 8
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+static IMG_UINT32 gsDmaIoRemapRef[DMA_MAX_IOREMAP_ENTRIES] = {0};
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDmaAlloc != NULL && psDmaAlloc->pvOSDevice != NULL)
+	{
+#if defined(__linux__)
+		psDmaAlloc->pvVirtAddr =
+				dma_alloc_coherent((struct device *)psDmaAlloc->pvOSDevice,
+								   (size_t) psDmaAlloc->ui64Size,
+								   (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr,
+								   GFP_KERNEL);
+		PVR_LOG_RETURN_IF_FALSE((NULL != psDmaAlloc->pvVirtAddr), "dma_alloc_coherent() failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+#else
+		#error "Provide OS implementation of DMA allocation";
+#endif
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	if (psDmaAlloc && psDmaAlloc->pvVirtAddr)
+	{
+#if defined(__linux__)
+		dma_free_coherent((struct device *)psDmaAlloc->pvOSDevice,
+						  (size_t) psDmaAlloc->ui64Size,
+						  psDmaAlloc->pvVirtAddr,
+						  (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+#else
+		#error "Provide OS implementation of DMA deallocation";
+#endif
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (psDmaAlloc->pvVirtAddr == NULL)
+	{
+		/* Check if an I/O remap entry already exists for this request */
+		for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+		{
+			if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+			{
+				PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr);
+				break;
+			}
+		}
+
+		if (ui32Idx < DMA_MAX_IOREMAP_ENTRIES)
+		{
+			IMG_UINT64 ui64Offset;
+			ui64Offset = psDmaAlloc->sBusAddr.uiAddr - gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr;
+			psDmaAlloc->pvVirtAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr + (uintptr_t)ui64Offset;
+			gsDmaIoRemapRef[ui32Idx] += 1;
+			return PVRSRV_OK;
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check if there is a free I/O remap table entry for this request */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		return eError;
+	}
+
+	gsDmaIoRemapArray[ui32Idx].ui64Size = psDmaAlloc->ui64Size;
+	gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+	gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+	gsDmaIoRemapRef[ui32Idx] += 1;
+
+	PVR_LOG(("DMA: register I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+			psDmaAlloc->pvVirtAddr,
+			psDmaAlloc->sBusAddr.uiAddr,
+			psDmaAlloc->ui64Size));
+
+	gbEnableDmaIoRemapping = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return;
+	}
+
+	/* Remove specified entry from the list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+		{
+			if (! --gsDmaIoRemapRef[ui32Idx])
+			{
+				PVR_LOG(("DMA: deregister I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+						gsDmaIoRemapArray[ui32Idx].pvVirtAddr,
+						gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr,
+						gsDmaIoRemapArray[ui32Idx].ui64Size));
+
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+				gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+				gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+			}
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/vmm_type_stub.c b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/vmm_type_stub.c
new file mode 100644
index 000000000000..0f52c3447e88
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/vmm_type_stub.c
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+                                    PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+    PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_ERROR, "Using a stub VM manager type, no runtime VZ support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+                             PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+    PVR_UNREFERENCED_PARAMETER(psDevConfig);
+    PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/include/dma_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/include/dma_support.h
new file mode 100644
index 000000000000..cc1e4ebc31ab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/include/dma_support.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysconfig.c
new file mode 100644
index 000000000000..2f250b946a7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysconfig.c
@@ -0,0 +1,691 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN  driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Limei<limei@eswin.com>
+*/
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+
+#include "physheap.h"
+
+#include <linux/dma-mapping.h>
+
+#include "rgxdevice.h"
+#include "interrupt_support.h"
+#include "osfunc.h"
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include <linux/platform_device.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/dma-map-ops.h>
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include "rgxdevice.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#include <asm/dma-noncoherent.h>
+
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+#include "validation_soc.h"
+#endif
+#define ES_MEM_THRESH_OF_FLUSH_CACHE_ALL 0x100000
+
+extern unsigned int _corefreq_div;
+
+IMG_UINT64 *cpu_cache_flush_addr = NULL;
+// static int cache_reg_map = 0;
+#if 0
+extern void eswin_l2_flush64(phys_addr_t addr, size_t size);
+#else
+void eswin_l2_flush64(phys_addr_t addr, size_t size) {
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	EIC770X_LOGICAL_MEM_NODE_E nid;
+	int cpuid;
+	eic770x_memory_type_t mem_type;
+
+	cpuid = smp_processor_id();
+	arch_get_mem_node_and_type(phys_to_pfn(addr), &nid, &mem_type);
+	if (likely(mem_type == FLAT_DDR_MEM)) {
+		if (cpu_to_node(cpuid) != nid) {
+			pr_debug("%s, pid %d addr 0x%llx, size 0x%lx mem_nid %d cur_cpu %d \n",
+				__func__, task_pid_nr(current), addr, size, nid, smp_processor_id());
+			sched_setaffinity(current->pid, cpumask_of_node(nid));
+		}
+	}
+
+	if (unlikely(size > ES_MEM_THRESH_OF_FLUSH_CACHE_ALL))
+		arch_sync_cache_all(addr, size);
+	else
+#endif
+		arch_sync_dma_for_device(addr, size, DMA_TO_DEVICE);
+};
+#endif
+void riscv_invalidate_addr(phys_addr_t addr, size_t size,IMG_BOOL virtual) {
+
+ printk(KERN_ALERT "eswin:%s not implement now start 0x%llx size=0x%lx\n",__func__,addr,size);
+}
+void riscv_flush_addr(IMG_UINT64 cpuaddr,IMG_UINT64 bytes_size, IMG_BOOL virtual)
+{
+		// printk(KERN_ALERT "eswin print %s: ----------riscv_flush_addr--run-----------\n", __func__);
+    IMG_UINT64 bStart = 0;
+    IMG_UINT64 bEnd = 0;
+    // IMG_UINT64 bBase;
+    IMG_UINT64 line_size = (IMG_UINT64)cache_line_size();
+
+    if (virtual)//if cpuaddr is virtual address, here need to convert it to physical address
+        bStart =  page_to_phys(virt_to_page(cpuaddr)) & RGX_ESWIN_CPU_ADDR_MASK;
+    else
+        bStart =  cpuaddr & RGX_ESWIN_CPU_ADDR_MASK;;
+
+    bEnd = bStart + bytes_size;
+
+    bEnd = PVR_ALIGN((IMG_UINT64)bEnd, line_size);
+
+    #if 1
+    eswin_l2_flush64((phys_addr_t)bStart, (size_t)(bEnd-bStart));
+    #else
+    if (cache_reg_map == 0){
+        cache_reg_map = 1;
+	    cpu_cache_flush_addr = (IMG_UINT64 __iomem *)ioremap(RGX_ESWIN_CPU_CACHE_FLUSH_ADDR, 0x10);
+    }
+    if (cpu_cache_flush_addr == NULL){
+        printk(KERN_ALERT "NULL pointer file %s line %d\n", __FILE__, __LINE__);
+        return;
+    }
+    mb();
+    for (bBase = bStart; bBase < bEnd; bBase += line_size)
+    {
+        *(IMG_UINT64 *)cpu_cache_flush_addr = bBase;
+    }
+    mb();
+    #endif
+}
+
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+    PVRSRV_ERROR eError;
+    PVR_UNREFERENCED_PARAMETER(hSysData);
+
+    #ifndef NO_HARDWARE
+    eError = OSInstallSystemLISR(phLISRData, ui32IRQ, pszName, pfnLISR, pvData, SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+    if (eError != PVRSRV_OK)
+		PVR_DPF((PVR_DBG_ERROR, "%s: install error %d != PVRSRV_OK", __func__, eError));
+    #endif
+    return eError;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return OSUninstallSystemLISR(hLISRData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+    IMG_UINT32 ui32Idx;
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+    IMG_UINT32 ui32Idx;
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+	}
+}
+
+
+static PHYS_HEAP_FUNCTIONS gsPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	UMAPhysHeapCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	UMAPhysHeapDevPAddrToCpuPAddr,
+	/* pfnGetRegionId */
+	// NULL,
+};
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut,
+                                    PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32NextHeapID = 0;
+	IMG_UINT32 uiHeapCount = 1;
+
+
+	uiHeapCount += !PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig) ? 1:0;
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+    //eswin fixme: not sure if following states are okay!
+	pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+	ui32NextHeapID++;
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+        //eswin fixme: not sure if following states are okay!
+        pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+        pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+		ui32NextHeapID++;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+        int ret;
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+
+	/* eswin, assert the reset */
+	if (psDevConfig->rstc_spu)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_spu);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_spu);
+        }
+	if (psDevConfig->rstc_jones)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_jones);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_jones);
+        }
+	if (psDevConfig->rstc_gray)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_gray);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_gray);
+        }
+	if (psDevConfig->rstc_cfg)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_cfg);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_cfg);
+        }
+	if (psDevConfig->rstc_axi)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_axi);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_axi);
+        }
+        printk(KERN_DEBUG "eswin print %s: gpu reset assert ok\n", __func__);
+
+	/* eswin, close the clk */
+        if (psDevConfig->aclk)
+        {
+		clk_disable_unprepare(psDevConfig->aclk);
+		clk_put(psDevConfig->aclk);
+        }
+        if (psDevConfig->cfg_clk)
+        {
+            clk_disable_unprepare(psDevConfig->cfg_clk);
+	    clk_put(psDevConfig->cfg_clk);
+        }
+        if (psDevConfig->gray_clk)
+        {
+            clk_disable_unprepare(psDevConfig->gray_clk);
+	    clk_put(psDevConfig->gray_clk);
+        }
+        printk(KERN_DEBUG "eswin print %s: gpu clk disable ok\n", __func__);
+
+	OSFreeMem(psDevConfig);
+}
+
+void riscv_flush_cache_range(IMG_HANDLE hSysData,
+                                        PVRSRV_CACHE_OP eRequestType,
+                                        void *pvVirtStart,
+                                        void *pvVirtEnd,
+                                        IMG_CPU_PHYADDR sCPUPhysStart,
+                                        IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	IMG_UINT64 bStart = sCPUPhysStart.uiAddr;
+	IMG_UINT64 bEnd = sCPUPhysEnd.uiAddr;
+
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	// PVR_UNREFERENCED_PARAMETER(eRequestType);
+	PVR_UNREFERENCED_PARAMETER(pvVirtStart);
+	PVR_UNREFERENCED_PARAMETER(pvVirtEnd);
+	switch (eRequestType)
+	{
+	case PVRSRV_CACHE_OP_FLUSH:
+		riscv_flush_addr(bStart, (bEnd - bStart), IMG_FALSE);
+		break;
+	case PVRSRV_CACHE_OP_INVALIDATE:
+		riscv_flush_addr(bStart, (bEnd - bStart), IMG_FALSE);
+		break;
+	case PVRSRV_CACHE_OP_CLEAN:
+		riscv_flush_addr(bStart, (bEnd - bStart), IMG_FALSE);
+		break;
+	default:
+		printk(KERN_ALERT "%s: unhandled eRequestType val=0x%x \n", __func__, eRequestType);
+	}
+}
+PVRSRV_DEVICE_CONFIG *IGPUGetDevConfigByDevNum(IMG_UINT32 ui32DevNum)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DRIVER_MODE eRetMode = DRIVER_MODE_NATIVE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+    PVRSRV_DEVICE_CONFIG *psDevConfig = NULL;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Iterate over all devices. */
+	for (psDevNode = psPVRSRVData->psDeviceNodeList;
+		 psDevNode != NULL;
+		 psDevNode = psDevNode->psNext)
+	{
+		if (psDevNode->sDevId.ui32InternalID == ui32DevNum)
+		{
+			psDevConfig = psDevNode->psDevConfig;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDevConfig;
+}
+#if defined(CONFIG_PM_DEVFREQ)
+int igpu_devfreq_target(struct device *dev, unsigned long *freq, u32 flags)
+{
+	int ret;
+	IMG_UINT32 rgx_freq=0;
+    PVRSRV_DEVICE_CONFIG *psDevConfig = IGPUGetDevConfigByDevNum(0);
+
+	if(IS_ERR_OR_NULL(psDevConfig))
+	{
+		return 0;
+	}
+	rgx_freq= clk_round_rate(psDevConfig->aclk, *freq);//24M -> 800M
+	if (rgx_freq > 0) {
+		ret = clk_set_rate(psDevConfig->aclk, rgx_freq);
+		if (ret) {
+			dev_err(dev, "failed to set aclk: %d\n", ret);
+			return ret;
+		}
+	}
+	rgx_freq = clk_get_rate(psDevConfig->aclk);
+
+	return 0;
+}
+
+int igpu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+    PVRSRV_DEVICE_CONFIG *psDevConfig = IGPUGetDevConfigByDevNum(0);
+	if(IS_ERR_OR_NULL(psDevConfig))
+	{
+		*freq = 800000000;
+	}
+	else
+	{
+		*freq = clk_get_rate(psDevConfig->aclk);
+	}
+
+	return 0;
+}
+#endif
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+#ifndef NO_HARDWARE
+	IMG_UINT32 rgx_freq=0;
+	struct resource res;
+	struct device_node *np=NULL;
+	int ret;
+	struct platform_device *pdev = to_platform_device((struct device *)pvOSDevice);
+	struct device *dev = (struct device *)pvOSDevice;
+#endif //CONFIG_SPARSE_IRQ
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+    psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount, psDevConfig);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+#ifndef NO_HARDWARE
+	np = dev->of_node;
+
+	if (!np)
+	{
+		pr_err("%s:%d can not find node img,gpu\n", __func__, __LINE__);
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto ErrorFreeDevConfig;
+	}
+	// gpu clk enable
+	psDevConfig->aclk = of_clk_get_by_name(np, "aclk");
+	if (IS_ERR(psDevConfig->aclk))
+	{
+		ret = PTR_ERR(psDevConfig->aclk);
+		dev_err(&pdev->dev, "failed to get aclk: %d\n", ret);
+		printk(KERN_ALERT "eswin print %s: failed to get aclk\n", __func__);
+		return ret;
+	}
+
+	// get gpu clk, it is supposed to be 800MHz
+	rgx_freq = clk_get_rate(psDevConfig->aclk);
+
+	ret = clk_prepare_enable(psDevConfig->aclk);
+	if (ret)
+	{
+		dev_err(&pdev->dev, "failed to enable aclk: %d\n", ret);
+		printk(KERN_ALERT "eswin print %s: failed to enable aclk\n", __func__);
+		return ret;
+	}
+
+	psDevConfig->cfg_clk = of_clk_get_by_name(np, "cfg_clk");
+	if (IS_ERR(psDevConfig->cfg_clk))
+	{
+		ret = PTR_ERR(psDevConfig->cfg_clk);
+		dev_err(&pdev->dev, "failed to get cfg_clk: %d\n", ret);
+		printk(KERN_ALERT "eswin print %s: failed to get cfg_clk\n", __func__);
+		return ret;
+	}
+	ret = clk_prepare_enable(psDevConfig->cfg_clk);
+	if (ret)
+	{
+		dev_err(&pdev->dev, "failed to enable cfg_clk: %d\n", ret);
+		printk(KERN_ALERT "eswin print %s: failed to enable cfg_clk\n", __func__);
+		return ret;
+	}
+
+	psDevConfig->gray_clk = of_clk_get_by_name(np, "gray_clk");
+	if (IS_ERR(psDevConfig->gray_clk))
+	{
+		ret = PTR_ERR(psDevConfig->gray_clk);
+		dev_err(&pdev->dev, "failed to get gray_clk: %d\n", ret);
+		printk(KERN_ALERT "eswin print %s: failed to get gray_clk\n", __func__);
+		return ret;
+	}
+	ret = clk_prepare_enable(psDevConfig->gray_clk);
+	if (ret)
+	{
+		dev_err(&pdev->dev, "failed to enable gray_clk: %d\n", ret);
+		printk(KERN_ALERT "eswin print %s: failed to enable gray_clk\n", __func__);
+		return ret;
+	}
+	printk(KERN_ALERT "esgpu 20250625 %s: gpu clk enable ok\n", __func__);
+	clk_disable_unprepare(psDevConfig->aclk);
+	clk_disable_unprepare(psDevConfig->cfg_clk);
+	clk_disable_unprepare(psDevConfig->gray_clk);
+	printk(KERN_ALERT "eswin print %s: gpu clk disable ok\n", __func__);
+
+	if (_corefreq_div > 800000000)
+		_corefreq_div = 800000000;
+
+	rgx_freq= clk_round_rate(psDevConfig->aclk, _corefreq_div);//24M -> 800M
+	if (rgx_freq > 0) {
+		ret = clk_set_rate(psDevConfig->aclk, rgx_freq);
+		if (ret) {
+			dev_err(&pdev->dev, "failed to set aclk: %d\n", ret);
+			return ret;
+		}
+		printk(KERN_ALERT "eswin print : set aclk to %dHZ \n", rgx_freq);
+	}
+	rgx_freq = clk_get_rate(psDevConfig->aclk);
+	printk(KERN_ALERT "eswin print : read back aclk  %dHZ \n", rgx_freq);
+	psRGXTimingInfo->ui32CoreClockSpeed = rgx_freq;
+
+	clk_prepare_enable(psDevConfig->aclk);
+	clk_prepare_enable(psDevConfig->cfg_clk);
+	clk_prepare_enable(psDevConfig->gray_clk);
+
+	printk(KERN_ALERT "eswin print %s: gpu clk enable ok\n", __func__);
+
+	// reset gpu
+	psDevConfig->rstc_axi = of_reset_control_get_optional_exclusive(np, "axi");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_axi))
+	{
+		dev_err(&pdev->dev, "Failed to get axi reset handle\n");
+		printk(KERN_ALERT "eswin print %s: Failed to get axi reset handle\n", __func__);
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_axi);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_cfg = of_reset_control_get_optional_exclusive(np, "cfg");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_cfg))
+	{
+		dev_err(&pdev->dev, "Failed to get cfg reset handle\n");
+		printk(KERN_ALERT "eswin print %s: Failed to get cfg reset handle\n", __func__);
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_cfg);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_gray = of_reset_control_get_optional_exclusive(np, "gray");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_gray))
+	{
+		dev_err(&pdev->dev, "Failed to get gray reset handle\n");
+		printk(KERN_ALERT "eswin print %s: Failed to get gray reset handle\n", __func__);
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_gray);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_jones = of_reset_control_get_optional_exclusive(np, "jones");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_jones))
+	{
+		dev_err(&pdev->dev, "Failed to get jones reset handle\n");
+		printk(KERN_ALERT "eswin print %s: Failed to get jones reset handle\n", __func__);
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_jones);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_spu = of_reset_control_get_optional_exclusive(np, "spu");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_spu))
+	{
+		dev_err(&pdev->dev, "Failed to get spu reset handle\n");
+		printk(KERN_ALERT "eswin print %s: Failed to get spu reset handle\n", __func__);
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_spu);
+	WARN_ON(0 != ret);
+	printk(KERN_ALERT "eswin print %s: gpu clk reset ok\n", __func__);
+#else
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+#endif //CONFIG_SPARSE_IRQ
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszVersion             = NULL;
+#ifndef NO_HARDWARE
+	psDevConfig->pszName = pdev->name;
+	printk(KERN_ALERT "%s: --------------->dev_name=%s\n", __func__, psDevConfig->pszName);
+
+	if (of_address_to_resource(np, 0, &res))
+	{
+		printk(KERN_ALERT "%s: failed to get resource of rgx register\n", __func__);
+		eError = PVRSRV_ERROR_INVALID_MEMINFO;
+		goto ErrorFreeDevConfig;
+	}
+	psDevConfig->sRegsCpuPBase.uiAddr = res.start;
+	psDevConfig->ui32RegsSize = resource_size(&res);
+	pr_err("%s: --------------->reg_base=%llx, reg_size=%llx\n", __func__, res.start, resource_size(&res));
+
+	psDevConfig->ui32IRQ = irq_of_parse_and_map(np, 0);
+	pr_err("%s: --------------->ui32IRQ=%d\n", __func__, psDevConfig->ui32IRQ);
+	if (!psDevConfig->ui32IRQ)
+	{
+		printk(KERN_ALERT "%s: failed to map GPU interrupt\n", __func__);
+		eError = PVRSRV_ERROR_MAPPING_NOT_FOUND;
+		goto ErrorFreeDevConfig;
+	}
+#else
+	/* Device setup information */
+	psDevConfig->pszName                = SYS_RGX_DEV_NAME;
+	psDevConfig->sRegsCpuPBase.uiAddr   = RGX_ESWIN_GPU_REG_BASE;
+	psDevConfig->ui32RegsSize           = RGX_ESWIN_GPU_REG_SIZE;
+	psDevConfig->ui32IRQ                = RGX_ESWIN_IRQ_ID;
+#endif
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+    //eswin fixme: DRIVER_MODE_NATIVE okay?
+    psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+	if (! PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		/* Virtualization support services needs to know which heap ID corresponds to FW */
+		// psDevConfig->aui32PhysHeapID[PVRSRV_DEVICE_PHYS_HEAP_FW_LOCAL] = PHYS_HEAP_IDX_VIRTFW;
+	}
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+	psDevConfig->hSysData               = NULL;
+	psDevConfig->bDevicePA0IsValid       = IMG_FALSE;
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	/* Pdump validation system registers */
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+	PVRSRVConfigureSysCtrl(NULL, PDUMP_FLAGS_CONTINUOUS);
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PVRSRVConfigureTrustedDevice(NULL, PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_ERROR eError;
+    dma_set_mask(pvOSDevice, DMA_BIT_MASK(40)); //?why is '40', maybe need ask the hw engineer
+
+    eError = DeviceConfigCreate(pvOSDevice, ppsDevConfig);
+
+    (*ppsDevConfig)->pfnHostCacheMaintenance = riscv_flush_cache_range;
+    (*ppsDevConfig)->bHasPhysicalCacheMaintenance = OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+	return eError;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysinfo.h
new file mode 100644
index 000000000000..89c8663ab623
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysinfo.h
@@ -0,0 +1,84 @@
+/*************************************************************************/ /*!
+ ########################################################################### ###
+ #@File
+ #@Copyright ESWIN
+ #@Auther: Limei<limei@eswin.com>
+ #@Date:2020-04-03
+ #@History:
+ #  ChenShuo 2020-08-21 adapt for zhimo-kernel
+ ### ###########################################################################
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+#if 0
+#include "config_kernel.h"
+#endif
+//#define RGX_ESWIN_PROJECT_V91
+#include "pvrsrv_device.h"
+#include "pvrsrv.h"
+
+#define SYS_RGX_DEV_NAME    "pvrsrvkm"
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#define RGX_ESWIN_DISP_NUM_BUFS 2 //pay attention to this val
+
+#ifndef RGX_ESWIN_PROJECT_U84
+#define RGX_ESWIN_PROJECT_U84
+#endif
+
+#ifdef RGX_ESWIN_PROJECT_U84
+    #define RGX_ESWIN_CPU_ADDR_W        41/*this is based on hw, Notes:eswin support 41bits width addrees, 3D-GPU support 40bits(1TB)*/
+    #define RGX_ESWIN_CPU_ADDR_MASK     ((1UL<<RGX_ESWIN_CPU_ADDR_W)-1)/*this is based on hw*/
+    #define RGX_ESWIN_IRQ_ID           14 
+    /* display infor resolution and rate */
+    #define RGX_ESWIN_DISP_DEF_W        1280
+    #define RGX_ESWIN_DISP_DEF_H        720
+    #define RGX_ESWIN_DISP_DEF_R        60
+    /* ddr address from the view of CPU*/
+    #define RGX_ESWIN_MEM_BASE_CPU          (0x80000000UL) //0x8,000,0000, ddr mem address from CPU view
+
+    #ifdef  RGX_ESWIN_DISP_OFFSCREEN    /* fix address used by disp/vby1 for offscreen test */
+    #define RGX_ESWIN_DISP_MEM_START    (0x81B00000UL) //(0x9B000000)
+    #define RGX_ESWIN_MEM_SIZE          (RGX_ESWIN_DISP_MEM_START - RGX_ESWIN_MEM_BASE_CPU)
+    #else
+    #define RGX_ESWIN_MEM_SIZE          (0x20000000UL)
+    #endif
+    /* the cache flush reg address of CPU */
+    #define RGX_ESWIN_CPU_CACHE_FLUSH_ADDR  0x02010200
+    /* GPU reg address and size*/
+    #ifndef NO_HARDWARE
+	#define RGX_ESWIN_GPU_REG_BASE      0x51400000 //0x51400000 on haps
+    #else
+    #define RGX_ESWIN_GPU_REG_BASE      0x21400000 
+    #endif
+    #define RGX_ESWIN_GPU_REG_SIZE      0xFFFFF
+    /* GPU clk */
+    #define RGX_NOHW_CORE_CLOCK_SPEED   5000000//850000000// 850MHz GPU clk //100000000 //e.g. 100 MHz
+
+    #ifdef LMA  /* mem address offset from the view of GPU */
+        #define RGX_ESWIN_MEM_BASE_GPU   0x80000000UL //0x4,0000,0000
+    #endif
+#else
+    #error "please update the proper Project infor"
+    #define RGX_ESWIN_DISP_DEF_W        4096
+    #define RGX_ESWIN_DISP_DEF_H        2160
+    #define RGX_NOHW_CORE_CLOCK_SPEED   100000000 //e.g. 100 MHz
+#endif
+
+#define RGX_ESWIN_DISP_DEF_TOTAL_SIZE   (RGX_ESWIN_DISP_DEF_W*RGX_ESWIN_DISP_DEF_H*4*RGX_ESWIN_DISP_NUM_BUFS)
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000) 
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#define SYS_RGX_OF_COMPATIBLE "img,gpu"
+int igpu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq);
+int igpu_devfreq_target(struct device *dev, unsigned long *freq, u32 flags);
+#endif	/* !defined(__SYSINFO_H__) */
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/interrupt_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/interrupt_support.h
new file mode 100644
index 000000000000..b87772d223cb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/interrupt_support.h
@@ -0,0 +1,103 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(INTERRUPT_SUPPORT_H)
+#define INTERRUPT_SUPPORT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+
+/*! Default trigger type for the interrupt line. */
+#define SYS_IRQ_FLAG_TRIGGER_DEFAULT (0x0 << 0)
+/*! Interrupt triggered when interrupt line is low. */
+#define SYS_IRQ_FLAG_TRIGGER_LOW     (0x1 << 0)
+/*! Interrupt triggered when interrupt line is high. */
+#define SYS_IRQ_FLAG_TRIGGER_HIGH    (0x2 << 0)
+/*! Interrupt trigger mask. */
+#define SYS_IRQ_FLAG_TRIGGER_MASK    (SYS_IRQ_FLAG_TRIGGER_DEFAULT | \
+                                      SYS_IRQ_FLAG_TRIGGER_LOW | \
+                                      SYS_IRQ_FLAG_TRIGGER_HIGH)
+/*! The irq is allowed to be shared among several devices. */
+#define SYS_IRQ_FLAG_SHARED          (0x1 << 8)
+
+/*! Interrupt flags mask. */
+#define SYS_IRQ_FLAG_MASK            (SYS_IRQ_FLAG_TRIGGER_MASK | \
+                                      SYS_IRQ_FLAG_SHARED)
+
+/*************************************************************************/ /*!
+@Description    Pointer to a system Low-level Interrupt Service Routine (LISR).
+@Input  pvData  Private data provided to the LISR.
+@Return         IMG_TRUE if interrupt handled, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_SYS_LISR)(void *pvData);
+
+/*************************************************************************/ /*!
+@Function       OSInstallSystemLISR
+@Description    Installs a system low-level interrupt handler
+@Output         phLISR                  On return, contains a handle to the
+                                        installed LISR
+@Input          ui32IRQ                 The IRQ number for which the
+                                        interrupt handler should be installed
+@Input          pszDevName              Name of the device for which the handler
+                                        is being installed
+@Input          pfnLISR                 A pointer to an interrupt handler
+                                        function
+@Input          pvData                  A pointer to data that should be passed
+                                        to pfnLISR when it is called
+@Input          ui32Flags               Interrupt flags
+@Return         PVRSRV_OK on success, a failure code otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInstallSystemLISR(IMG_HANDLE *phLISR,
+				 IMG_UINT32 ui32IRQ,
+				 const IMG_CHAR *pszDevName,
+				 PFN_SYS_LISR pfnLISR,
+				 void *pvData,
+				 IMG_UINT32 ui32Flags);
+
+/*************************************************************************/ /*!
+@Function       OSUninstallSystemLISR
+@Description    Uninstalls a system low-level interrupt handler
+@Input          hLISRData              The handle to the LISR to uninstall
+@Return         PVRSRV_OK on success, a failure code otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR OSUninstallSystemLISR(IMG_HANDLE hLISRData);
+#endif /* !defined(INTERRUPT_SUPPORT_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/pci_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/pci_support.h
new file mode 100644
index 000000000000..29746c672047
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/pci_support.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PCI_SUPPORT_H
+#define PCI_SUPPORT_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(__linux__)
+#include <linux/pci.h>
+#define TO_PCI_COOKIE(dev) to_pci_dev((struct device *)(dev))
+#else
+#define TO_PCI_COOKIE(dev) (dev)
+#endif
+
+typedef enum _HOST_PCI_INIT_FLAGS_
+{
+	HOST_PCI_INIT_FLAG_BUS_MASTER	= 0x00000001,
+	HOST_PCI_INIT_FLAG_MSI		= 0x00000002,
+	HOST_PCI_INIT_FLAG_FORCE_I32	= 0x7fffffff
+} HOST_PCI_INIT_FLAGS;
+
+struct _PVRSRV_PCI_DEV_OPAQUE_STRUCT_;
+typedef struct _PVRSRV_PCI_DEV_OPAQUE_STRUCT_ *PVRSRV_PCI_DEV_HANDLE;
+
+PVRSRV_PCI_DEV_HANDLE OSPCIAcquireDev(IMG_UINT16 ui16VendorID, IMG_UINT16 ui16DeviceID, HOST_PCI_INIT_FLAGS eFlags);
+PVRSRV_PCI_DEV_HANDLE OSPCISetDev(void *pvPCICookie, HOST_PCI_INIT_FLAGS eFlags);
+PVRSRV_ERROR OSPCIReleaseDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIIRQ(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 *pui32IRQ);
+IMG_UINT64 OSPCIAddrRangeLen(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+IMG_UINT64 OSPCIAddrRangeStart(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+IMG_UINT64 OSPCIAddrRangeEnd(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIRequestAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIReleaseAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIRequestAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index, IMG_UINT64 uiOffset, IMG_UINT64 uiLength);
+PVRSRV_ERROR OSPCIReleaseAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index, IMG_UINT64 uiOffset, IMG_UINT64 uiLength);
+PVRSRV_ERROR OSPCISuspendDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIResumeDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIGetVendorDeviceIDs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT16 *pui16VendorID, IMG_UINT16 *pui16DeviceID);
+
+#if defined(CONFIG_MTRR)
+PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+#else
+static inline PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRPCI);
+	PVR_UNREFERENCED_PARAMETER(ui32Index);
+	return PVRSRV_OK;
+}
+
+static inline void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRPCI);
+	PVR_UNREFERENCED_PARAMETER(ui32Index);
+}
+#endif
+
+#endif /* PCI_SUPPORT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/syscommon.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/syscommon.h
new file mode 100644
index 000000000000..b7babda19cab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/syscommon.h
@@ -0,0 +1,175 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common System APIs and structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides common system-specific declarations and
+                macros that are supported by all systems
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(SYSCOMMON_H)
+#define SYSCOMMON_H
+
+#include "img_types.h"
+#include "pvr_notifier.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_error.h"
+
+/*************************************************************************/ /*!
+@Description    Pointer to a Low-level Interrupt Service Routine (LISR).
+@Input  pvData  Private data provided to the LISR.
+@Return         True if interrupt handled, false otherwise.
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_LISR)(void *pvData);
+
+/**************************************************************************/ /*!
+@Function       SysDevInit
+@Description    System specific device initialisation function.
+@Input          pvOSDevice          pointer to the OS device reference
+@Input          ppsDevConfig        returned device configuration info
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig);
+
+/**************************************************************************/ /*!
+@Function       SysDevDeInit
+@Description    System specific device deinitialisation function.
+@Input          psDevConfig        device configuration info of the device to be
+                                   deinitialised
+@Return         None.
+*/ /***************************************************************************/
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/**************************************************************************/ /*!
+@Function       SysDebugInfo
+@Description    Dump system specific device debug information.
+@Input          psDevConfig         pointer to device configuration info
+@Input          pfnDumpDebugPrintf  the 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     optional file identifier to be passed to
+                                    the 'printf' function if required
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile);
+
+/**************************************************************************/ /*!
+@Function       SysInstallDeviceLISR
+@Description    Installs the system Low-level Interrupt Service Routine (LISR)
+                which handles low-level processing of interrupts from the device
+                (GPU).
+                The LISR will be invoked when the device raises an interrupt. An
+                LISR may not be descheduled, so code which needs to do so should
+                be placed in an MISR.
+                The installed LISR will schedule any MISRs once it has completed
+                its interrupt processing, by calling OSScheduleMISR().
+@Input          hSysData      pointer to the system data of the device
+@Input          ui32IRQ       the IRQ on which the LISR is to be installed
+@Input          pszName       name of the module installing the LISR
+@Input          pfnLISR       pointer to the function to be installed as the
+                              LISR
+@Input          pvData        private data provided to the LISR
+@Output         phLISRData    handle to the installed LISR (to be used for a
+                              subsequent uninstall)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData);
+
+/**************************************************************************/ /*!
+@Function       SysUninstallDeviceLISR
+@Description    Uninstalls the system Low-level Interrupt Service Routine (LISR)
+                which handles low-level processing of interrupts from the device
+                (GPU).
+@Input          hLISRData     handle of the LISR to be uninstalled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData);
+
+/**************************************************************************/ /*!
+@Function       SysRGXErrorNotify
+@Description    Error reporting callback function, registered as the
+                pfnSysDevErrorNotify member of the PVRSRV_DEVICE_CONFIG
+                struct. System layer will be notified of device errors and
+                resets via this callback.
+                NB. implementers should ensure that the minimal amount of
+                work is done in this callback function, as it will be
+                executed in the main RGX MISR. (e.g. any blocking or lengthy
+                work should be performed by a worker queue/thread instead).
+@Input          hSysData      pointer to the system data of the device
+@Output         psErrorData   structure containing details of the reported error
+@Return         None.
+*/ /***************************************************************************/
+void SysRGXErrorNotify(IMG_HANDLE hSysData,
+                       PVRSRV_ROBUSTNESS_NOTIFY_DATA *psErrorData);
+
+/**************************************************************************/ /*!
+@Function       SysRestrictGpuLocalPhysheap
+@Description    If the Restriction apphint has been set, validate the
+                restriction value and return the new GPU_LOCAL heap size.
+
+@Input          uiHeapSize      Current syslayer detected GPU_LOCAL heap size.
+@Return         IMG_UINT64      New GPU_LOCAL heap size in bytes.
+*/ /***************************************************************************/
+IMG_UINT64 SysRestrictGpuLocalPhysheap(IMG_UINT64 uiHeapSize);
+
+/**************************************************************************/ /*!
+@Function       SysRestrictGpuLocalAddPrivateHeap
+@Description    Determine if the restriction apphint has been set.
+
+@Return         IMG_BOOL        IMG_TRUE if the restriction apphint has been
+                                set.
+*/ /***************************************************************************/
+IMG_BOOL SysRestrictGpuLocalAddPrivateHeap(void);
+
+/**************************************************************************/ /*!
+@Function       SysDefaultToCpuLocalHeap
+@Description    Determine if the Default Heap should be CPU_LOCAL
+                Can only be used on TC_MEMORY_HYBRID systems.
+
+@Return         IMG_BOOL        IMG_TRUE if the Default heap apphint has been
+                                set.
+*/ /***************************************************************************/
+IMG_BOOL SysDefaultToCpuLocalHeap(void);
+
+#endif /* !defined(SYSCOMMON_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/sysvalidation.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/sysvalidation.h
new file mode 100644
index 000000000000..e3e731e8b810
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/sysvalidation.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Validation System APIs and structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+                needed for hardware validation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSVALIDATION_H)
+#define SYSVALIDATION_H
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "img_types.h"
+#include "rgxdefs_km.h"
+#include "virt_validation_defs.h"
+
+void SysInitFirewall(IMG_HANDLE hSysData,
+                     IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                     IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+
+#if defined(EMULATOR)
+void SysSetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState);
+void SysSetTrustedDeviceAceEnabled(IMG_HANDLE hSysData);
+#endif
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+#endif /* !defined(SYSVALIDATION_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/uma_heap_fns.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/uma_heap_fns.h
new file mode 100644
index 000000000000..bfef1e520880
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/uma_heap_fns.h
@@ -0,0 +1,53 @@
+/*****************************************************************************
+@File
+@Title          Boilerplate heap functions for UMA systems
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares generic nop conversions for physical addresses in UMA
+                systems.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef UMA_HEAP_FNS_H
+#define UMA_HEAP_FNS_H
+
+#include "physheap_config.h"
+
+extern PHYS_HEAP_FUNCTIONS g_sUmaHeapFns;
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/plato/Kbuild.mk
new file mode 100644
index 000000000000..84b92c971b0a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/Kbuild.mk
@@ -0,0 +1,53 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += \
+	services/system/$(PVR_SYSTEM)/sysconfig.o \
+	services/server/common/vmm_pvz_client.o \
+	services/server/common/vmm_pvz_server.o \
+	services/server/common/vz_vmm_pvz.o \
+	services/server/common/vz_vmm_vm.o \
+	services/system/rogue/common/vmm_type_$(VMM_TYPE).o
+
+ccflags-y += \
+	-I$(TOP)/services/system/rogue/common/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec/plato
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/plato.h b/drivers/gpu/drm/img/img-volcanic/services/system/plato/plato.h
new file mode 100644
index 000000000000..1aaf14d94c62
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/plato.h
@@ -0,0 +1,337 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PLATO_H)
+#define PLATO_H
+
+#define PLATO_SYSTEM_NAME			"Plato"
+
+/* Valid values for the PLATO_MEMORY_CONFIG configuration option */
+#define PLATO_MEMORY_LOCAL			(1)
+#define PLATO_MEMORY_HOST			(2)
+#define PLATO_MEMORY_HYBRID			(3)
+
+/* Interrupt defines */
+typedef enum _PLATO_IRQ_
+{
+    PLATO_IRQ_GPU = 0,
+    PLATO_IRQ_PDP,
+    PLATO_IRQ_HDMI,
+    PLATO_IRQ_MAX,
+}PLATO_IRQ;
+
+#define PLATO_GPU_INT_SHIFT				(0)
+#define PLATO_PDP_INT_SHIFT				(8)
+#define PLATO_HDMI_INT_SHIFT			(9)
+#define PLATO_HDMI_WAKEUP_INT_SHIFT		(11)
+#define PLATO_TEMP_A_INT_SHIFT			(12)
+
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#define PHYS_HEAP_ID_CPU_LOCAL 0
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_CPU_LOCAL 1
+#define PHYS_HEAP_ID_PDP_LOCAL 2
+#define PHYS_HEAP_ID_NON_MAPPABLE 3
+#define PHYS_HEAP_ID_FW_LOCAL 4
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_PDP_LOCAL 1
+#define PHYS_HEAP_ID_NON_MAPPABLE 2
+#endif
+
+#define DCPDP_PHYS_HEAP_ID PHYS_HEAP_ID_PDP_LOCAL
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+#define RGX_PLATO_RESERVE_DC_MEM_SIZE (PLATO_DC_MEM_SIZE_MB * 1024 * 1024)
+#endif
+
+#define SYS_PLATO_REG_DMA_BASENUM	(0)
+#define SYS_PLATO_REG_DMA_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_PCI_BASENUM	(1)
+#define SYS_PLATO_REG_REGION_SIZE	(4 * 1024 * 1024)
+
+#define SYS_PLATO_SRAM_BASENUM		(2)
+#define SYS_PLATO_SRAM_SIZE			(2 * 1024 * 1024)
+/*
+ * Give system region a whole span of the reg space including
+ * RGX registers. That's because there are sys register segments
+ * both before and after the RGX segment.
+ */
+#define SYS_PLATO_REG_SYS_OFFSET			(0x0)
+#define SYS_PLATO_REG_SYS_SIZE				(4 * 1024 * 1024)
+
+/* Entire Peripheral region */
+#define SYS_PLATO_REG_PERIP_OFFSET			(0x20000)
+#define SYS_PLATO_REG_PERIP_SIZE			(164 * 1024)
+
+/* Chip level registers */
+#define SYS_PLATO_REG_CHIP_LEVEL_OFFSET		(SYS_PLATO_REG_PERIP_OFFSET)
+#define SYS_PLATO_REG_CHIP_LEVEL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_TEMPA_OFFSET			(0x80000)
+#define SYS_PLATO_REG_TEMPA_SIZE			(64 * 1024)
+
+/* USB not included */
+
+#define SYS_PLATO_REG_DDR_A_CTRL_OFFSET		(0x120000)
+#define SYS_PLATO_REG_DDR_A_CTRL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_B_CTRL_OFFSET		(0x130000)
+#define SYS_PLATO_REG_DDR_B_CTRL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_A_PUBL_OFFSET		(0x140000)
+#define SYS_PLATO_REG_DDR_A_PUBL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_B_PUBL_OFFSET		(0x150000)
+#define SYS_PLATO_REG_DDR_B_PUBL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_NOC_OFFSET			(0x160000)
+#define SYS_PLATO_REG_NOC_SIZE				(64 * 1024)
+
+/* Debug NOC registers */
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET (0x1500)
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET (0x1580)
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET (0x1600)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET (0x1680)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET (0x1700)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET (0x1780)
+#define SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET  (0x1800)
+#define SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET   (0x1900)
+#define SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET   (0x1980)
+#define SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET      (0x1A00)
+#define SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET    (0x1A80)
+#define SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET    (0x1B00)
+#define SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET      (0x1B80)
+#define SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET   (0x1c00)
+#define SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET    (0x1D00)
+#define SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET    (0x1E00)
+
+#define SYS_PLATO_REG_RGX_OFFSET			(0x170000)
+#define SYS_PLATO_REG_RGX_SIZE				(64 * 1024)
+
+#define SYS_PLATO_REG_AON_OFFSET			(0x180000)
+#define SYS_PLATO_REG_AON_SIZE				(64 * 1024)
+
+#define SYS_PLATO_REG_PDP_OFFSET			(0x200000)
+#define SYS_PLATO_REG_PDP_SIZE				(128 * 1024)
+
+#define SYS_PLATO_REG_PDP_BIF_OFFSET        (SYS_PLATO_REG_PDP_OFFSET + SYS_PLATO_REG_PDP_SIZE)
+#define SYS_PLATO_REG_PDP_BIF_SIZE          (0x200)
+
+#define SYS_PLATO_REG_HDMI_OFFSET           (SYS_PLATO_REG_PDP_OFFSET + 0x20000)
+#define SYS_PLATO_REG_HDMI_SIZE             (128 * 1024)
+
+/* Device memory (including HP mapping) on base register 4 */
+#define SYS_DEV_MEM_PCI_BASENUM		(4)
+
+/* Device memory size */
+#define ONE_GB_IN_BYTES				(0x40000000ULL)
+#define SYS_DEV_MEM_REGION_SIZE		(PLATO_MEMORY_SIZE_GIGABYTES * ONE_GB_IN_BYTES)
+
+/* Plato DDR offset in device memory map at 32GB */
+#define PLATO_DDR_DEV_PHYSICAL_BASE	(0x800000000)
+
+/* DRAM is split at 48GB */
+#define PLATO_DRAM_SPLIT_ADDR (0xc00000000)
+
+/* Plato DDR region is aliased if less than 32GB memory is present.
+ * This defines memory base closest to the DRAM split point.
+ * If 32GB is present this is equal to PLATO_DDR_DEV_PHYSICAL_BASE */
+#define PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE	(PLATO_DRAM_SPLIT_ADDR - (SYS_DEV_MEM_REGION_SIZE >> 1))
+
+#define PLATO_DDR_ALIASED_DEV_PHYSICAL_END	(PLATO_DRAM_SPLIT_ADDR + (SYS_DEV_MEM_REGION_SIZE >> 1))
+
+#define PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE	((32ULL / PLATO_MEMORY_SIZE_GIGABYTES) * ONE_GB_IN_BYTES)
+
+/* Plato Host memory offset in device memory map at 512GB */
+#define PLATO_HOSTRAM_DEV_PHYSICAL_BASE (0x8000000000)
+
+/* Plato PLL, DDR/GPU and PDP clocks */
+#define PLATO_PLL_REF_CLOCK_SPEED		(19200000)
+
+/* 600 MHz */
+#define PLATO_MEM_CLOCK_SPEED			(600000000)
+#define PLATO_MIN_MEM_CLOCK_SPEED		(600000000)
+#define PLATO_MAX_MEM_CLOCK_SPEED		(800000000)
+
+/* 396 MHz (~400 MHz) on HW, around 1MHz on the emulator */
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+#define	PLATO_RGX_CORE_CLOCK_SPEED		(1000000)
+#define	PLATO_RGX_MIN_CORE_CLOCK_SPEED		(1000000)
+#define	PLATO_RGX_MAX_CORE_CLOCK_SPEED		(1000000)
+#else
+
+#define	PLATO_RGX_CORE_CLOCK_SPEED		(396000000)
+#define	PLATO_RGX_MIN_CORE_CLOCK_SPEED	(396000000)
+#define	PLATO_RGX_MAX_CORE_CLOCK_SPEED	(742500000)
+#endif
+
+#define PLATO_MIN_PDP_CLOCK_SPEED				(165000000)
+#define PLATO_TARGET_HDMI_SFR_CLOCK_SPEED		(27000000)
+#define PLATO_TARGET_HDMI_CEC_CLOCK_SPEED		(32768)
+
+#define REG_TO_CELSIUS(reg)			(((reg) * 352/4096) - 109)
+#define CELSIUS_TO_REG(temp)		((((temp) + 109) * 4096) / 352)
+#define PLATO_MAX_TEMP_CELSIUS		(100)
+
+#define PLATO_LMA_HEAP_REGION_MAPPABLE			0
+#define PLATO_LMA_HEAP_REGION_NONMAPPABLE		1
+
+#if defined(ENABLE_PLATO_HDMI)
+
+#if defined(HDMI_PDUMP)
+/* Hard coded video formats for pdump type run only */
+#define VIDEO_FORMAT_1280_720p          0
+#define VIDEO_FORMAT_1920_1080p         1
+#define DC_DEFAULT_VIDEO_FORMAT     (VIDEO_FORMAT_1920_1080p)
+#endif
+
+#define DEFAULT_BORDER_WIDTH_PIXELS     (0)
+#define HDMI_DTD_MAX                    (20) // Reference driver has 37...seems like a lot
+
+typedef enum
+{
+	ENCODING_RGB = 0,
+	ENCODING_YCC444,
+	ENCODING_YCC422,
+	ENCODING_YCC420
+} ENCODING_T;
+
+typedef enum
+{
+	ITU601 = 1,
+	ITU709,
+	EXTENDED_COLORIMETRY
+} COLORIMETRY_T;
+
+typedef struct
+{
+	/** VIC code */
+	IMG_UINT32 mCode;
+	/** Identifies modes that ONLY can be displayed in YCC 4:2:0 */
+	IMG_UINT8 mLimitedToYcc420;
+	/** Identifies modes that can also be displayed in YCC 4:2:0 */
+	IMG_UINT8 mYcc420;
+	IMG_UINT16 mPixelRepetitionInput;
+	/** in units of 10KHz */
+	IMG_UINT16 mPixelClock;
+	/** 1 for interlaced, 0 progressive */
+	IMG_UINT8 mInterlaced;
+	IMG_UINT16 mHActive;
+	IMG_UINT16 mHBlanking;
+	IMG_UINT16 mHBorder;
+	IMG_UINT16 mHImageSize;
+	IMG_UINT16 mHFrontPorchWidth;
+	IMG_UINT16 mHBackPorchWidth;
+	IMG_UINT16 mHSyncPulseWidth;
+	/** 0 for Active low, 1 active high */
+	IMG_UINT8 mHSyncPolarity;
+	IMG_UINT16 mVActive;
+	IMG_UINT16 mVBlanking;
+	IMG_UINT16 mVBorder;
+	IMG_UINT16 mVImageSize;
+	IMG_UINT16 mVFrontPorchWidth;
+	IMG_UINT16 mVBackPorchWidth;
+	IMG_UINT16 mVSyncPulseWidth;
+	/** 0 for Active low, 1 active high */
+	IMG_UINT8 mVSyncPolarity;
+
+    IMG_UINT8 mValid;
+} DTD;
+
+/* Monitor range limits comes from Display Descriptor (EDID) */
+typedef struct _MONITOR_RANGE_LIMITS
+{
+	IMG_UINT8 vMinRate;
+	IMG_UINT8 vMaxRate;
+	IMG_UINT8 hMinRate;
+	IMG_UINT8 hMaxRate;
+	IMG_UINT8 maxPixelClock;
+	IMG_UINT8 valid;
+} MONITOR_RANGE_LIMITS;
+
+typedef struct
+{
+    IMG_UINT8               mHdmi;
+    ENCODING_T              mEncodingOut;
+    ENCODING_T              mEncodingIn;
+    IMG_UINT8               mColorResolution;
+    IMG_UINT8               mPixelRepetitionFactor;
+    DTD                     mDtdList[HDMI_DTD_MAX];
+    IMG_UINT8               mDtdIndex;
+    IMG_UINT8               mDtdActiveIndex;
+    IMG_UINT8               mRgbQuantizationRange;
+    IMG_UINT8               mPixelPackingDefaultPhase;
+    IMG_UINT8               mColorimetry;
+    IMG_UINT8               mScanInfo;
+    IMG_UINT8               mActiveFormatAspectRatio;
+    IMG_UINT8               mNonUniformScaling;
+    IMG_UINT8               mExtColorimetry;
+    IMG_UINT8               mItContent;
+    IMG_UINT16              mEndTopBar;
+    IMG_UINT16              mStartBottomBar;
+    IMG_UINT16              mEndLeftBar;
+    IMG_UINT16              mStartRightBar;
+    IMG_UINT16              mCscFilter;
+    IMG_UINT16              mCscA[4];
+    IMG_UINT16              mCscC[4];
+    IMG_UINT16              mCscB[4];
+    IMG_UINT16              mCscScale;
+    IMG_UINT8               mHdmiVideoFormat;
+    IMG_UINT8               m3dStructure;
+    IMG_UINT8               m3dExtData;
+    IMG_UINT8               mHdmiVic;
+    IMG_UINT8               mDataEnablePolarity;
+    MONITOR_RANGE_LIMITS    mRangeLimits;
+    IMG_UINT8               mPreferredTimingIncluded;
+} VIDEO_PARAMS;
+
+/* Callback for HDMI->PDP */
+PVRSRV_ERROR PDPInitialize(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance);
+
+#endif /* ENABLE_PLATO_HDMI */
+
+#endif /* if !defined(PLATO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.c
new file mode 100644
index 000000000000..488d8caf4e27
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.c
@@ -0,0 +1,2869 @@
+/*************************************************************************/ /*!
+@File
+@Title			System Configuration
+@Copyright		Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description	System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_memallocflags.h"
+#include "syscommon.h"
+#include "power.h"
+#include "sysinfo.h"
+#include "plato.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pci_support.h"
+#include "interrupt_support.h"
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+
+#include "plato_aon_regs.h"
+#include "plato_top_regs.h"
+#include "plato_ddr_ctrl_regs.h"
+#include "plato_ddr_publ_regs.h"
+
+/* BUP flags */
+#define PLATO_DDR_TRAINING_RETRY_MAX 5
+#define PLATO_DDR_KINGSTON 1
+
+#if defined(PLATO_LOG_CHECKPOINTS)
+#define PLATO_CHECKPOINT PVR_LOG(("- %s: %d", __func__, __LINE__))
+#else
+#define PLATO_CHECKPOINT
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define	HOST_PCI_INIT_FLAGS	0
+#else
+#define	HOST_PCI_INIT_FLAGS	HOST_PCI_INIT_FLAG_BUS_MASTER
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoLocalCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoLocalDevPAddrToCpuPAddr,
+};
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoSystemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoSystemDevPAddrToCpuPAddr,
+};
+#endif
+
+/* Clock speed module parameters */
+static IMG_UINT32 ui32MemClockSpeed  = PLATO_MEM_CLOCK_SPEED;
+static IMG_UINT32 ui32CoreClockSpeed = PLATO_RGX_CORE_CLOCK_SPEED;
+
+/* Helpers for getting DDR/GPU/PLL clock speed */
+IMG_UINT32 SysGetPlatoMemClockSpeed(void)
+{
+	IMG_UINT32 ret = ui32MemClockSpeed;
+
+	(ui32MemClockSpeed > PLATO_MAX_MEM_CLOCK_SPEED) ? ret = PLATO_MAX_MEM_CLOCK_SPEED : 0;
+	(ui32MemClockSpeed < PLATO_MIN_MEM_CLOCK_SPEED) ? ret = PLATO_MIN_MEM_CLOCK_SPEED : 0;
+
+	return ret;
+}
+
+IMG_UINT32 SysGetPlatoCoreClockSpeed(void)
+{
+	IMG_UINT32 ret = ui32CoreClockSpeed;
+
+	(ui32CoreClockSpeed > PLATO_RGX_MAX_CORE_CLOCK_SPEED) ? ret = PLATO_RGX_MAX_CORE_CLOCK_SPEED : 0;
+	(ui32CoreClockSpeed < PLATO_RGX_MIN_CORE_CLOCK_SPEED) ? ret = PLATO_RGX_MIN_CORE_CLOCK_SPEED : 0;
+
+	return ret;
+}
+
+/*
+ * This function returns a PLL clock value which can be easily reduced
+ * (using integer dividers) to get 148.5 MHz HDMI pixel clock.
+ *
+ * NOTE: Accurate values are returned only in the cases
+ *       which are known by the function.
+ */
+IMG_UINT32 SysGetPlatoPLLClockSpeed(IMG_UINT32 ui32ClockSpeed)
+{
+    /*
+     * Force the lowest possible PLL clock in case when the requested
+     * clock speed is higher than the largest value known by the function.
+     */
+    IMG_UINT32 ui32PLLClockSpeed = ui32ClockSpeed;
+    IMG_UINT32 ui32AccClockSpeed = ui32ClockSpeed;
+
+    /* Tweak the values if the requested clock speed is a supported one */
+    (ui32ClockSpeed <= 742500000) ? (ui32PLLClockSpeed = 742500000,  ui32AccClockSpeed = 742500000) : 0;
+    (ui32ClockSpeed <= 668250000) ? (ui32PLLClockSpeed = 1336500000, ui32AccClockSpeed = 668250000) : 0;
+    (ui32ClockSpeed <= 594000000) ? (ui32PLLClockSpeed = 1188000000, ui32AccClockSpeed = 594000000) : 0;
+    (ui32ClockSpeed <= 544500000) ? (ui32PLLClockSpeed = 1633500000, ui32AccClockSpeed = 544500000) : 0;
+    (ui32ClockSpeed <= 519750000) ? (ui32PLLClockSpeed = 1039500000, ui32AccClockSpeed = 519750000) : 0;
+    (ui32ClockSpeed <= 495000000) ? (ui32PLLClockSpeed = 1485000000, ui32AccClockSpeed = 495000000) : 0;
+    (ui32ClockSpeed <= 445500000) ? (ui32PLLClockSpeed = 891000000,  ui32AccClockSpeed = 445500000) : 0;
+    (ui32ClockSpeed <= 408375000) ? (ui32PLLClockSpeed = 1633500000, ui32AccClockSpeed = 408375000) : 0;
+    (ui32ClockSpeed <= 396000000) ? (ui32PLLClockSpeed = 1188000000, ui32AccClockSpeed = 396000000) : 0;
+
+    /*
+     * If the requested core clock speed matches the expected one, there is
+     * no need to do any additional adjustments of the PLL clock speed.
+     */
+    if (ui32ClockSpeed == ui32AccClockSpeed)
+    {
+        return ui32PLLClockSpeed;
+    }
+
+    /* Do fine grained adjustment if the requested clock speed is different than expected */
+    return ((((IMG_UINT64)ui32PLLClockSpeed << 32) / ui32AccClockSpeed) >> 32) * ui32ClockSpeed;
+}
+
+#if defined(__linux__)
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+module_param_named(sys_mem_clk_speed,  ui32MemClockSpeed,  uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(sys_mem_clk_speed, "Plato memory clock speed in Hz (600000000 - 800000000)");
+module_param_named(sys_core_clk_speed, ui32CoreClockSpeed, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(sys_core_clk_speed, "Plato core clock speed in Hz (396000000 - 742500000)");
+
+EXPORT_SYMBOL(SysGetPlatoMemClockSpeed);
+EXPORT_SYMBOL(SysGetPlatoCoreClockSpeed);
+EXPORT_SYMBOL(SysGetPlatoPLLClockSpeed);
+#endif
+
+#define PLATO_HAS_NON_MAPPABLE(dev) (dev->ui64MappedMemSize < SYS_DEV_MEM_REGION_SIZE)
+
+static IMG_BOOL PollPr(IMG_UINT32* base, IMG_UINT32 reg, IMG_UINT32 val, IMG_UINT32 msk, IMG_UINT32 cnt, IMG_UINT32 intrvl)
+{
+	IMG_UINT32 polnum;
+
+	for (polnum = 0; polnum < cnt; polnum++)
+	{
+		if ((OSReadHWReg32(base, reg) & msk) == val)
+		{
+			break;
+		}
+		OSSleepms(intrvl);
+	}
+	if (polnum == cnt)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Poll failed for register: 0x%08X", (unsigned int)reg));
+		return false;
+	}
+
+	return true;
+}
+
+#define Poll(base,reg,val,msk) PollPr(base,reg,val,msk,10,10)
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	return NULL;
+}
+
+static
+IMG_BOOL SystemISRHandler(void *pvData)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)pvData;
+	IMG_UINT32 ui32InterruptStatus;
+	IMG_UINT32 i;
+	IMG_BOOL bInterruptServiced = IMG_FALSE;
+
+#if defined(VIRTUAL_PLATFORM)
+	/* On virtual platform all interrupt handlers need to be called */
+	ui32InterruptStatus = (0x1 << PLATO_GPU_INT_SHIFT) | (0x1 << PLATO_PDP_INT_SHIFT);
+#else
+	{
+		void* pvSystemDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+		ui32InterruptStatus = OSReadHWReg32(pvSystemDbgPeripRegs , PLATO_TOP_CR_INT_STATUS);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s:%d Interrupt Status (%u)",
+				 __func__, __LINE__, ui32InterruptStatus));
+	}
+#endif
+
+	for (i = 0; i < ARRAY_SIZE(psDevData->sInterruptData); i++)
+	{
+		if ((ui32InterruptStatus & psDevData->sInterruptData[i].ui32InterruptFlag) != 0)
+		{
+			bInterruptServiced = psDevData->sInterruptData[i].pfnLISR(psDevData->sInterruptData[i].pvData);
+		}
+	}
+
+	return bInterruptServiced;
+}
+
+static void EnableInterrupt(SYS_DATA *psDevData, IMG_UINT32 ui32InterruptFlag)
+{
+#if !defined(VIRTUAL_PLATFORM)
+	IMG_UINT32 ui32Value;
+
+	void* pvSystemDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+
+	ui32Value = OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	ui32Value |= ui32InterruptFlag;
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK, ui32Value);
+
+	(void) OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	OSWaitus(10);
+#endif
+}
+
+static void DisableInterrupt(SYS_DATA *psDevData, IMG_UINT32 ui32InterruptFlag)
+{
+#if !defined(VIRTUAL_PLATFORM)
+	IMG_UINT32 ui32Value;
+
+	void* pvSystemDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+
+	ui32Value = OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	ui32Value &= ~ui32InterruptFlag;
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK, ui32Value);
+
+	(void) OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	OSWaitus(10);
+#endif
+}
+
+PVRSRV_ERROR SysGetPlatoGPUTemp(SYS_DATA * psDevData, IMG_UINT32 *ui32Temp);
+
+static IMG_UINT32 HashNOCRegisterAddress(IMG_UINT32 ui32Addr)
+{
+	IMG_UINT32 ui32Hash_bits = (ui32Addr & 0x4000 ? 1 : 0);
+	if (ui32Addr & 0x2000)
+		++ui32Hash_bits;
+	if (ui32Addr & 0x1000)
+		++ui32Hash_bits;
+	if (ui32Addr & 0x0800)
+		++ui32Hash_bits;
+	if ((ui32Hash_bits == 1) || (ui32Hash_bits == 3))
+		ui32Addr ^= 0x100;
+
+	return ui32Addr;
+}
+
+
+typedef struct PLATO_NOC_DBG_REG
+{
+	IMG_UINT32 ui32Offset;
+	IMG_CHAR *pszName;
+}PLATO_NOC_DBG_REGS;
+
+static PLATO_NOC_DBG_REGS asPlatoNOCDbgRegs[] = {
+	{SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET, "DDR A Ctrl"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET, "DDR A Data"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET, "DDR A Publ"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET, "DDR B Ctrl"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET, "DDR B Data"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET, "DDR B Publ"},
+
+	{SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET, "Display S"},
+	{SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET,  "GPIO 0 S"},
+	{SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET,  "GPIO 1 S"},
+	{SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET,	 "GPU S"},
+	{SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET,	 "PCI PHY"},
+	{SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET,	 "PCI Reg"},
+	{SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET,	 "PCI S"},
+	{SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET,  "Periph S"},
+	{SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET,	 "Ret Reg"},
+	{SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET,	 "Service"}
+};
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)psDevConfig->hSysData;
+	void *pvDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void *pvNocRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_NOC_OFFSET;
+	void *pvAonRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	IMG_UINT32 ui32Val;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32GPUTemp;
+
+	PVR_DUMPDEBUG_LOG("------[ System Debug - Device ID:%u ]------", psDevConfig->psDevNode->sDevId.ui32InternalID);
+
+	if ((eError = SysGetPlatoGPUTemp(psDevData, &ui32GPUTemp)) != PVRSRV_OK)
+	{
+		return eError;
+	}
+	PVR_DUMPDEBUG_LOG("GPU Temp: %u °C", ui32GPUTemp);
+	PVR_DUMPDEBUG_LOG("PCI_PHY_STATUS: 0x%x",
+					   OSReadHWReg32(pvDbgPeripRegs, PLATO_TOP_CR_PCI_PHY_STATUS));
+	for (i = 0; i <= 0x1C; i += 4)
+	{
+		PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", i,
+						   OSReadHWReg32(pvNocRegs, i));
+	}
+	PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", 0x50,
+					   OSReadHWReg32(pvNocRegs, 0x50));
+	PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", 0x54,
+					   OSReadHWReg32(pvNocRegs, 0x54));
+	PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", 0x58,
+					   OSReadHWReg32(pvNocRegs, 0x58));
+	PVR_DUMPDEBUG_LOG("AON Offset 0x%x: 0x%x", PLATO_AON_CR_BM_STATUS,
+					   OSReadHWReg32(pvAonRegs, PLATO_AON_CR_BM_STATUS));
+	PVR_DUMPDEBUG_LOG("AON Offset 0x%x: 0x%x", PLATO_AON_CR_PLL_STATUS,
+					   OSReadHWReg32(pvAonRegs, PLATO_AON_CR_PLL_STATUS));
+
+	PVR_DUMPDEBUG_LOG("NOC Error Status: ");
+	for (i = 0; i < ARRAY_SIZE(asPlatoNOCDbgRegs); i++)
+	{
+		IMG_UINT32 ui32Offset = asPlatoNOCDbgRegs[i].ui32Offset + 0xC;
+		IMG_UINT32 ui32HashedOffset = HashNOCRegisterAddress(ui32Offset);
+
+		ui32Val = OSReadHWReg32(pvNocRegs, ui32HashedOffset);
+		if (ui32Val == 0)
+		{
+			PVR_DUMPDEBUG_LOG("%s ok", asPlatoNOCDbgRegs[i].pszName);
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, ui32Val, ui32Offset);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x8), ui32Offset + 0x8);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0xC), ui32Offset + 0xC);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x14), ui32Offset + 0x14);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x18), ui32Offset + 0x18);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x1C), ui32Offset + 0x1C);
+		}
+
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PlatoMapRegisters(SYS_DATA *psDevData, IMG_UINT32 ui32BaseNum, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Size, void ** pvCpuVAddr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CPU_PHYADDR	sCpuPBase;
+
+	sCpuPBase.uiAddr = OSPCIAddrRangeStart(psDevData->hRGXPCI, ui32BaseNum) + ui32Offset;
+
+#if !defined(VIRTUAL_PLATFORM)
+	/* Reserve all register banks up to HDMI bank */
+	eError = OSPCIRequestAddrRegion(psDevData->hRGXPCI, ui32BaseNum, ui32Offset, ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the system address region(BAR %d)",
+				 __func__, ui32BaseNum));
+		goto ErrorReturn;
+	}
+
+	*pvCpuVAddr = OSMapPhysToLin(sCpuPBase, ui32Size, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (*pvCpuVAddr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers (BAR %d)",
+				 __func__, ui32BaseNum));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+#endif
+
+	return eError;
+
+#if !defined(VIRTUAL_PLATFORM)
+ErrorSystemRegReleaseAddrRegion:
+	OSPCIReleaseAddrRegion(psDevData->hRGXPCI, SYS_PLATO_REG_PCI_BASENUM, SYS_PLATO_REG_SYS_OFFSET, SYS_PLATO_REG_PDP_OFFSET - SYS_PLATO_REG_SYS_OFFSET);
+
+ErrorReturn:
+	return eError;
+#endif
+}
+
+void PlatoUnmapRegisters(SYS_DATA * psDevData, void * pvBase, IMG_UINT32 ui32BaseNum, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Size)
+{
+	OSUnMapPhysToLin(pvBase, ui32Size);
+	OSPCIReleaseAddrRegion(psDevData->hRGXPCI, ui32BaseNum, ui32Offset, ui32Size);
+}
+
+
+static PVRSRV_ERROR ConfigPlatoDram(void * pvPublRegs, void * pvCtrlRegs, void * pvAonRegs,
+									IMG_UINT32 pvBLDRData[PLATO_DDR_PUBL_DATX_LANE_COUNT][PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE],
+									IMG_UINT32 ui32ResetFlags)
+{
+	/*
+
+	Phase 1: Program the DWC_ddr_umctl2 registers
+
+	*/
+
+	/* Single rank only for Kingston DDRs*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_MSTR, 0x41040001);
+
+	OSWaitus(100);
+
+	/*refresh timings*/
+#if defined(PLATO_DDR_KINGSTON)
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x0081008B);
+#else
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x006100BB);
+#endif
+
+	// Train DDR sequence
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT0, 0x00020100);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT1, 0x00010000);
+
+#if defined(PLATO_DDR_KINGSTON)
+	/*write recovery */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT3, 0x01700000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00280000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT5, 0x0012000c);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0f0e2112);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x00040618);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x0506040B);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x06020307);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x090e0403);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x0d0e000e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a05);
+
+	/*impedance registers */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x30ab002b);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+
+	/*refresh control*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e01020);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x0078007e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0057000e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+	/*DFI Timings*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x02878208);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x00f000ff);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x80100010);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x088105c3);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000000);
+#else
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT3, 0x0d700000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00180000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT5, 0x00090009);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0c101a0e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x000a0313);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x04050509);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x06020306);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x070c0202);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x0d0e000e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a07);
+	/*impedance registers */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x10800020);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+
+	/*refresh control*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e06010);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x00600031);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0004002a);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+
+	/*DFI Timings*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x02878206);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFILPCFG0, 0x07111031);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFILPCFG1, 0x00000050);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x006a006f);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x0d0b02b6);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x00100010);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+#endif
+
+	/* Single rank only on Kingston */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP0, 0x00001F1F);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP1, 0x00070707);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP2, 0x00000000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP3, 0x0F000000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP4, 0x00000F0F);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP5, 0x06060606);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP6, 0x06060606);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP7, 0x00000F0F);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP8, 0x00000707);
+
+#if defined(PLATO_DDR_KINGSTON)
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x06000604);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTMAP, 0x99c5b050);
+#else
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x0d0f0740);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTMAP, 0x99c5b050);
+#endif
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PERFHPR1, 0x9f008f23);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_SCHED, 0x00003f00);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PERFLPR1, 0x18000064);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PERFWR1, 0x18000096);
+
+	/* Setup the virtual channels */
+	OSWriteHWReg32(pvCtrlRegs, 0x00000410, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000414, 0x00000000);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000418, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000041C, 0x00000001);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000420, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000424, 0x00000002);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000428, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000042C, 0x00000003);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000430, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000434, 0x00000004);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000438, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000043C, 0x00000005);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000440, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000444, 0x00000006);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000448, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000044C, 0x00000007);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000450, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000454, 0x00000008);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000458, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000045C, 0x00000009);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000460, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000464, 0x0000000A);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000468, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000046C, 0x0000000B);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000470, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000474, 0x0000000C);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000478, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000047C, 0x0000000D);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000480, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000484, 0x0000000E);
+
+	Poll(pvCtrlRegs, 0x484, 0x0000000E, 0x0000000F);
+
+	OSWaitus(1000);
+
+	PLATO_CHECKPOINT;
+
+	/*
+
+	Phase 2: Deassert soft reset signal core_ddrc_rstn
+
+	*/
+
+	/* Now getting DRAM controller out of reset */
+	OSWriteHWReg32(pvAonRegs, PLATO_AON_CR_RESET_CTRL, ui32ResetFlags | OSReadHWReg32(pvAonRegs, PLATO_AON_CR_RESET_CTRL));
+
+	OSWaitus(1000);
+
+	/* ECC disable */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DBG1, 0x00000000);
+	/* power related */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PWRCTL, 0x00000000);
+	/* Enabling AXI input port (Port control) */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PCTRL, 0x00000001);
+
+	PLATO_CHECKPOINT;
+	/*
+
+	Phase 7: Set DFIMISC.dfi_init_complete_en to 1
+
+	(skipped on emu?)
+	*/
+
+	/*
+
+	Phase 3: Start PHY initialization by accessing relevant PUB registers
+
+	*/
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DCR_OFFSET, 0x0000040B);
+
+#if defined(PLATO_DDR_KINGSTON)
+	/* IF DDR_RISE_RISE */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0064641F);
+
+		/*MR Registers */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0x170);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000400);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000228);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x06220308);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x281b1004);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00060120);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x01200807);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x00300c08);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR2_OFFSET, 0x00F09088);
+#else
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PTR0_OFFSET, 0x10000010);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PTR1_OFFSET, 0x271012c0);
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PTR2_OFFSET, 0x00080421);
+
+	/* IF DDR_RISE_RISE */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0064641F);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0xd70);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000000);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000018);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x061c0B06);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x28200400);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00040005);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x01180805);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x00250B06);
+	/* IF TRAIN_DDR */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR2_OFFSET, 0x00F09088);
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+#endif
+
+	/* DISABLE VT COMPENSATION */
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR6_OFFSET, 0x00013001);
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR4_OFFSET, 0x40000000);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXCCR_OFFSET, 0x20C01884);
+
+	/* VREF CHANGE */
+	/*
+	OSWriteHWReg32(pvPublRegs, 0x0710, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0810, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0910, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0A10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0B10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0C10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0D10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0E10, 0x0E00083C);
+	*/
+
+	PLATO_CHECKPOINT;
+	/*
+
+	Phase 4: Trigger PHY initialization: Impedance, PLL, and DDL; assert PHY reset
+
+	*/
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00000073);
+
+	/*
+
+	Phase 5: Monitor PHY initialization status by polling the PUB register PGSR0
+
+	*/
+	Poll(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET, 0xF, 0xF);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000001);
+
+	PLATO_CHECKPOINT;
+
+	/*
+
+	Phase 6: Indicate to the PUB that the controller will perform SDRAM
+	initialization by setting PIR.INIT and PIR.CTLDINIT, and
+	poll PGSR0.IDONE
+
+	*/
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00040001);
+	Poll(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET, 0x11, 0x11);
+
+	/*
+
+	Phase 8: Wait for DWC_ddr_umctl2 to move to "normal" operating
+	mode by monitoring STAT.operating_mode signal
+
+	*/
+	PollPr(pvCtrlRegs, PLATO_DDR_CTRL_STAT, 0x01, 0x01, 10, 100);
+
+	OSWaitus(100);
+
+	/* IF TRAIN_DDR */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTCR0_OFFSET, 0x8000B1C7);
+	/*single rank only for Kingston */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTCR1_OFFSET, 0x00010237);
+
+#if defined(PLATO_DDR_BDLR_TRAINING)
+	/* Bit delay line register training */
+	{
+		IMG_UINT8 ui8Lane = 0;
+		IMG_UINT8 ui8Reg = 0;
+		for (ui8Lane = 0; ui8Lane < PLATO_DDR_PUBL_DATX_LANE_COUNT; ui8Lane++)
+		{
+			for (ui8Reg = 0; ui8Reg < PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE; ui8Reg++)
+			{
+				// PVR_LOG(("- %s: BDLR Lane %d Register %d: 0x%08x", __func__, ui8Lane, ui8Reg, pvBLDRData[ui8Lane][ui8Reg]));
+				OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXnBDLR_OFFSET(ui8Lane, ui8Reg), pvBLDRData[ui8Lane][ui8Reg]);
+			}
+		}
+	}
+
+	/* Poll on general status register 2 for each lane */
+	{
+		IMG_UINT8 ui8Lane;
+		for (ui8Lane = 0; ui8Lane < PLATO_DDR_PUBL_DATX_LANE_COUNT; ui8Lane++)
+		{
+			Poll(pvPublRegs, PLATO_DDR_PUBL_DXnGSR_OFFSET(ui8Lane, 2), 0, 0x001FFFFF);
+		}
+	}
+#endif
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x0000ff72);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x0000ff73);
+	Poll(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET, 0x80000fff, 0xfff80fff);
+	Poll(pvCtrlRegs, PLATO_DDR_CTRL_STAT, 0x1, 0x1);
+
+	/* Setting the Anti Glitch OFF (?), Disabling On Die pullup/pulldowns */
+#if defined(PLATO_DDR_KINGSTON)
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXCCR_OFFSET, 0x02401884);
+#else
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXCCR_OFFSET, 0x02400004);
+#endif
+
+	if (OSReadHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET) != 0x80000fff)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: DDR Training failed", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	OSWaitus(100);
+
+#if defined(DEBUG)
+	{
+		IMG_UINT8 lane = 0;
+		IMG_UINT8 reg = 0;
+		for (lane = 0; lane < PLATO_DDR_PUBL_DATX_LANE_COUNT; lane++)
+		{
+			for (reg = 0; reg < PLATO_DDR_PUBL_DXGSR_REGS_PER_LANE; reg++)
+			{
+				PVR_LOG(("DX%dGSR%d: 0x%08x", lane, reg,
+					OSReadHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXnGSR_OFFSET(lane, reg))));
+			}
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+#if defined(EMULATOR)
+static PVRSRV_ERROR ConfigPlatoEmu(SYS_DATA *psDevData)
+{
+	IMG_BOOL bPollPassed;
+	/* To be obtained differently */
+	void* pvSystemDbgPeripRegs		= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void* pvSystemDbgAonRegs		= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	void* pvSystemDbgDdrACtrlRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_CTRL_OFFSET;
+	void* pvSystemDbgDdrAPublRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_PUBL_OFFSET;
+	void* pvSystemDbgDdrBCtrlRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_CTRL_OFFSET;
+	void* pvSystemDbgDdrBPublRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_PUBL_OFFSET;
+	void* pvSystemDbgNocRegs		= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_NOC_OFFSET;
+
+#if defined(ENABLE_PLATO_HDMI)
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, 0x3370A03);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, 0x3370A03, 0x3370A03);
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1, 0x1);
+#endif
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1, 0x1);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, 0x01101037);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, 0x01101037, 0x01101037);
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, 0x00780000);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, 0x00780000, 0x00780000);
+
+	/* Waiting for DDR PLL getting locked */
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x2, 0x2, -1 , 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01, 0x01);
+
+	OSWaitus(100);
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x011);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x011, 0x011);
+#else
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x01);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x01, 0x01);
+#endif
+	/* PLL Lock is done */
+
+	OSWaitus(1000);
+
+	/* Enabling PDP gated clock output - 198 MHz */
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, 0x00001210);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, 0x00001210, 0x00001210);
+
+	OSWaitus(100);
+
+	/* PDP needs HDMI clocks on for framegrabber, start them here */
+	/* Enabling HDMI gated clock output - 148.5 MHz */
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, 0x00001310);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, 0x00001310, 0x00001310);
+
+	OSWaitus(100);
+	PVR_LOG(("%s: Enabled PDP and HDMI clocks", __func__));
+
+	/* GPU PLL configuration */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, 0x0110103D);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, 0x0110103D, 0x0110103D);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, 0x00E00000);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, 0x00E00000, 0x00E00000);
+	/* Waiting for GPU PLL getting locked */
+
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x3, 0x3, -1, 10);
+	/* GPU and DDR PLL Locked */
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x1, 0x1);
+
+	OSWaitus(100);
+
+	/* Enabling gated clock output */
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, 0x011);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, 0x011, 0x011);
+
+	OSWaitus(1000);
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	/* Setting dual memory interleaved mode */
+	OSWriteHWReg32(pvSystemDbgNocRegs, 0x00000050, 0x01);
+	PollPr(pvSystemDbgNocRegs, 0x00000058, 0x01, 0x1, 1, 10);
+
+	/* Now putting DRAM controller out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0F30);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0F30, 0x0F30);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	OSWaitus(10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0A30);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0A30, 0xA30);
+
+#else
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0330);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0330, 0x0330);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	OSWaitus(10);
+
+	/* Now putting DRAM controller into reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0230);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0230, 0x0230);
+
+#endif
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/*
+
+	Phase 1: Program the DWC_ddr_umctl2 registers
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001);
+	Poll(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001, 0x8F040001);
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001);
+	Poll(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001, 0x8F040001);
+#endif
+
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x007f0056);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x007f0056);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT3, 0x01140000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT3, 0x01140000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT1, 0x00010000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT1, 0x00010000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT0, 0x00020001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT0, 0x00020001);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00280000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00280000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT5, 0x000C000C);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT5, 0x000C000C);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0f132312);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0f132312);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x00080419);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x00080419);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x0507050b);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x0507050b);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x07020407);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x07020407);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x090e0403);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x090e0403);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x020e000e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x020e000e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a05);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a05);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x30ab002b);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x30ab002b);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e01020);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e01020);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x0078007e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x0078007e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0057000e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0057000e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x028A8208);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x028A8208);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x00F000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x00F000FF);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x80100010);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x80100010);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x00100010);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x00100010);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP0, 0x00001414);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP1, 0x00070707);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP2, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP3, 0x0F000000);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP4, 0x00000F0F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP5, 0x06060606);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP6, 0x0F0F0606);
+
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP0, 0x00001414);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP1, 0x00070707);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP2, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP3, 0x0F000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP4, 0x00000F0F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP5, 0x06060606);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP6, 0x0F0F0606);
+
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x04000400);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x04000400);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_SCHED, 0x00003F00);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_SCHED, 0x00003F00);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PERFHPR1, 0x0F0000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PERFHPR1, 0x0F0000FF);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PERFLPR1, 0x0F0000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PERFLPR1, 0x0F0000FF);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PERFWR1, 0x0F0000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PERFWR1, 0x0F0000FF);
+
+	/* Setup the virtual channels */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000410, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000414, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000418, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000041C, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000420, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000424, 0x00000002);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000428, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000042C, 0x00000003);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000430, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000434, 0x00000004);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000438, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000043C, 0x00000005);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000440, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000444, 0x00000006);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000448, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000044C, 0x00000007);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000450, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000454, 0x00000008);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000458, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000045C, 0x00000009);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000460, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000464, 0x0000000A);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000468, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000046C, 0x0000000B);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000470, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000474, 0x0000000C);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000478, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000047C, 0x0000000D);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000480, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000484, 0x0000000E);
+
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000410, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000414, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000418, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000041C, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000420, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000424, 0x00000002);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000428, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000042C, 0x00000003);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000430, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000434, 0x00000004);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000438, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000043C, 0x00000005);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000440, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000444, 0x00000006);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000448, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000044C, 0x00000007);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000450, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000454, 0x00000008);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000458, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000045C, 0x00000009);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000460, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000464, 0x0000000A);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000468, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000046C, 0x0000000B);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000470, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000474, 0x0000000C);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000478, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000047C, 0x0000000D);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000480, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000484, 0x0000000E);
+
+	/* DRAM controller registers configuration done */
+
+	OSWaitus(100);
+
+	/*
+
+	Phase 2: Deassert soft reset signal core_ddrc_rstn
+
+	*/
+
+	/* Now getting DRAM controller out of reset */
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F30);
+#else
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000330);
+#endif
+
+	OSWaitus(100);
+
+	/* ECC disable */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DBG1, 0x00000000);
+	/* power related */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PWRCTL, 0x00000000);
+	/* Enabling AXI input port (Port control) */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PCTRL, 0x00000001);
+
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DBG1, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PWRCTL, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PCTRL, 0x00000001);
+
+	/*
+
+	Phase 7: Set DFIMISC.dfi_init_complete_en to 1
+
+	(skipped on emu?)
+	*/
+
+	/*
+
+	Phase 3: Start PHY initialization by accessing relevant PUB registers
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DCR_OFFSET, 0x0000040B);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DCR_OFFSET, 0x0000040B);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PTR0_OFFSET, 0x10000010);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PTR0_OFFSET, 0x10000010);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PTR1_OFFSET, 0x00800080);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PTR1_OFFSET, 0x00800080);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PTR2_OFFSET, 0x00080421);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PTR2_OFFSET, 0x00080421);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0020641F);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0020641F);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0x00000114);
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000028);
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0x00000114);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000028);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x040F0406);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x040F0406);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x28110402);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x28110402);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00030002);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00030002);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x00190602);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x00190602);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x0018040B);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x0018040B);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+
+	/*
+
+	Phase 4: Trigger PHY initialization: Impedance, PLL, and DDL; assert PHY reset
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00000073);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00000073);
+
+	/*
+
+	Phase 5: Monitor PHY initialization status by polling the PUB register PGSR0
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000001);
+
+	/*
+
+	Phase 6: Indicate to the PUB that the controller will perform SDRAM
+	initialization by setting PIR.INIT and PIR.CTLDINIT, and
+	poll PGSR0.IDONE
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00040001);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00040001);
+
+	/*
+
+	Phase 8: Wait for DWC_ddr_umctl2 to move to "normal" operating
+	mode by monitoring STAT.operating_mode signal
+
+	*/
+	PollPr(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_STAT, 0x01, 0x01, 10, 100);
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	PollPr(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_STAT, 0x01, 0x01, 10, 100);
+#endif
+	OSWaitus(100);
+
+	/* Getting GPU And DDR A out of reset */
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12, 0x00000F12, -1, 100);
+#else
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312, 0x00000312, -1, 100);
+#endif
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	/* setting CR_ISO_CTRL:CR_GPU_CLK_E */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_ISO_CTRL, 0x000001F);
+
+	return PVRSRV_OK;
+}
+
+#else // HW
+
+static PVRSRV_ERROR CheckPDPAccessibility(SYS_DATA *psDevData)
+{
+	#define PLATO_PDP_REGISTER_CORE_ID_OFFSET (0x04e0)
+	#define VALID_PDP_CORE_ID (0x7010003)
+
+	void* pvPDPRegs;
+	IMG_UINT32 ui32CoreId;
+
+	if (PlatoMapRegisters(psDevData, SYS_PLATO_REG_PCI_BASENUM, SYS_PLATO_REG_PDP_OFFSET,
+		SYS_PLATO_REG_PDP_SIZE, &pvPDPRegs) != PVRSRV_OK || pvPDPRegs == NULL)
+	{
+		return PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE;
+	}
+
+	ui32CoreId = OSReadHWReg32(pvPDPRegs, PLATO_PDP_REGISTER_CORE_ID_OFFSET);
+	if (ui32CoreId != VALID_PDP_CORE_ID)
+	{
+		PVR_LOG(("PDP Access failed, expected 0x%x but got 0x%x", VALID_PDP_CORE_ID, ui32CoreId));
+		PlatoUnmapRegisters(psDevData, pvPDPRegs, SYS_PLATO_REG_PCI_BASENUM,
+			SYS_PLATO_REG_PDP_OFFSET, SYS_PLATO_REG_PDP_SIZE);
+		return PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE;
+	}
+
+	PlatoUnmapRegisters(psDevData, pvPDPRegs, SYS_PLATO_REG_PCI_BASENUM,
+		SYS_PLATO_REG_PDP_OFFSET, SYS_PLATO_REG_PDP_SIZE);
+	return PVRSRV_OK;
+}
+
+/*
+ * Helpers for getting integer and fractional values which are needed when
+ * programming PLATO_AON_CR_{GPU|DDR}_PLL_CTRL_{0|1} registers for DDR/GPU PLLs.
+ *
+ * PLL_CLOCK_SPEED = (PLL_REF_CLOCK_SPEED * REFDIV) *
+ *                   (PLL_INT + (PLL_FRAC / 2^24)) / POSTDIV1 / POSTDIV2
+ *
+ * NOTE: It's assumed that REFDIV, POSTDIV1 and POSTDIV2 are '1' in all cases.
+ */
+static IMG_UINT32 GetPlatoPLLInt(IMG_UINT32 ui32PLLClock)
+{
+	return ui32PLLClock / PLATO_PLL_REF_CLOCK_SPEED;
+}
+
+static IMG_UINT32 GetPlatoPLLFrac(IMG_UINT32 ui32PLLClock)
+{
+	/* Shift to get 24 bits for fractional part after div */
+	IMG_UINT64 shift = (IMG_UINT64)ui32PLLClock << 24;
+
+	/* Div and return only fractional part of the result */
+	return (shift / PLATO_PLL_REF_CLOCK_SPEED) & ((1 << 24) - 1);
+}
+
+/*
+ * Helper for getting value of integer divider for GPU clock.
+ */
+static IMG_UINT32 GetPlatoGPUVDiv0(IMG_UINT32 ui32PLLClock, IMG_UINT32 ui32CoreClockSpeed)
+{
+	IMG_UINT32 div, ret;
+
+	div = ui32PLLClock / ui32CoreClockSpeed;
+
+	/* Bias the result by (-1) with saturation, then clip it */
+	ret = (div - (div > 0)) & (PLATO_CR_GPUV_DIV_0_MASK >> PLATO_CR_GPUV_DIV_0_SHIFT);
+
+	/* Check for lost result after clipping, saturate if so */
+	return (div > 1) && (ret != (div - (div > 0))) ?
+			(PLATO_CR_GPUV_DIV_0_MASK >> PLATO_CR_GPUV_DIV_0_SHIFT) : ret;
+}
+
+/*
+ * Helpers for getting values of integer dividers for PDP clock.
+ *
+ * NOTE: Use only if PLL clock speed > ~350 MHz.
+ */
+static IMG_UINT32 GetPlatoPDPV0Div0(IMG_UINT32 ui32PLLClock)
+{
+	IMG_UINT32 div, ret;
+
+	div = ui32PLLClock / PLATO_MIN_PDP_CLOCK_SPEED;
+
+	/* Bias the result by (-1) with saturation, then clip it */
+	ret = (div - (div > 0)) & (PLATO_CR_PDPV0_DIV_0_MASK >> PLATO_CR_PDPV0_DIV_0_SHIFT);
+
+	/* Check for lost result after clipping, saturate if so */
+	return (div > 1) && (ret != (div - (div > 0))) ?
+			(PLATO_CR_PDPV0_DIV_0_MASK >> PLATO_CR_PDPV0_DIV_0_SHIFT) : ret;
+}
+
+static IMG_UINT32 GetPlatoPDPV1Div0(IMG_UINT32 ui32PLLClock)
+{
+	IMG_UINT32 div, ret;
+
+	div = (ui32PLLClock / (GetPlatoPDPV0Div0(ui32PLLClock) + 1)) / PLATO_MIN_PDP_CLOCK_SPEED;
+
+	/* Bias the result by (-1) with saturation, then clip it */
+	ret = (div - (div > 0)) & (PLATO_CR_PDPV1_DIV_0_MASK >> PLATO_CR_PDPV1_DIV_0_SHIFT);
+
+	/* Check for lost result after clipping, saturate if so */
+	return (div > 1) && (ret != (div - (div > 0))) ?
+			(PLATO_CR_PDPV1_DIV_0_MASK >> PLATO_CR_PDPV1_DIV_0_SHIFT) : ret;
+}
+
+#if defined(ENABLE_PLATO_HDMI)
+/*
+ * Helpers for getting values of integer dividers for HDMICEC clocks.
+ *
+ * NOTE: They strive to get clock speed of HDMI-SFR as close to
+ *       27 MHz as possible.
+ */
+static IMG_UINT32 GetPlatoHDMICECV0Div0(IMG_UINT32 ui32PLLClock)
+{
+    IMG_UINT32 ui32HDMICECV0Div0, ui32HDMICECV0Div0Limit;
+    IMG_UINT32 ui32HDMICECV1Div0, ui32HDMICECV1Div0Limit;
+    IMG_UINT32 ui32HDMISFRClockSpeed;
+
+    ui32HDMICECV0Div0Limit = PLATO_CR_HDMICECV0_DIV_0_MASK >> PLATO_CR_HDMICECV0_DIV_0_SHIFT;
+    ui32HDMICECV1Div0Limit = PLATO_CR_HDMICECV1_DIV_0_MASK >> PLATO_CR_HDMICECV1_DIV_0_SHIFT;
+
+    ui32HDMICECV0Div0 = 0;
+    while (ui32HDMICECV0Div0 < ui32HDMICECV0Div0Limit)
+    {
+        ui32HDMICECV1Div0 = 0;
+        while (ui32HDMICECV1Div0 < ui32HDMICECV1Div0Limit)
+        {
+            ui32HDMISFRClockSpeed = ui32PLLClock / (ui32HDMICECV0Div0 + 1) / (ui32HDMICECV1Div0 + 1);
+
+            if (ui32HDMISFRClockSpeed <= PLATO_TARGET_HDMI_SFR_CLOCK_SPEED)
+            {
+                /* Done, value of the divider found */
+                return ui32HDMICECV0Div0;
+            }
+
+            ui32HDMICECV1Div0++;
+        }
+
+        ui32HDMICECV0Div0++;
+    }
+
+    /* Here the function returns the highest possible value of the divider */
+    return ui32HDMICECV0Div0;
+}
+
+static IMG_UINT32 GetPlatoHDMICECV1Div0(IMG_UINT32 ui32PLLClock)
+{
+    IMG_UINT64 div, ret;
+
+    /* Calculate the divider using 32.32 fixed point math */
+    div = (IMG_UINT64)ui32PLLClock << 32;
+    div /= GetPlatoHDMICECV0Div0(ui32PLLClock) + 1;
+    div /= PLATO_TARGET_HDMI_SFR_CLOCK_SPEED;
+
+    /* Round up if the fractional part is present */
+    div = (div >> 32) + ((div & 0xFFFFFFFF) > 0);
+
+    /* Bias the result by (-1) with saturation, then clip it */
+    ret = (div - (div > 0)) & (PLATO_CR_HDMICECV1_DIV_0_MASK >> PLATO_CR_HDMICECV1_DIV_0_SHIFT);
+
+    /* Check for lost result after clipping, saturate if so */
+    return (div > 1) && (ret != (div - (div > 0))) ?
+            (PLATO_CR_HDMICECV1_DIV_0_MASK >> PLATO_CR_HDMICECV1_DIV_0_SHIFT) : ret;
+}
+
+static IMG_UINT32 GetPlatoHDMICECV2Div0(IMG_UINT32 ui32PLLClock)
+{
+    IMG_UINT64 div, ret;
+
+    /* Calculate the divider using 32.32 fixed point math */
+    div = (IMG_UINT64)ui32PLLClock << 32;
+    div /= GetPlatoHDMICECV0Div0(ui32PLLClock) + 1;
+    div /= GetPlatoHDMICECV1Div0(ui32PLLClock) + 1;
+    div /= PLATO_TARGET_HDMI_CEC_CLOCK_SPEED;
+
+    /* Round up if the fractional part is present */
+    div = (div >> 32) + ((div & 0xFFFFFFFF) > 0);
+
+    /* Bias the result by (-1) with saturation, then clip it */
+    ret = (div - (div > 0)) & (PLATO_CR_HDMICECV2_DIV_0_MASK >> PLATO_CR_HDMICECV2_DIV_0_SHIFT);
+
+    /* Check for lost result after clipping, saturate if so */
+    return (div > 1) && (ret != (div - (div > 0))) ?
+            (PLATO_CR_HDMICECV2_DIV_0_MASK >> PLATO_CR_HDMICECV2_DIV_0_SHIFT) : ret;
+}
+#endif
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+
+static PVRSRV_ERROR ConfigPlatoDualChannel(SYS_DATA *psDevData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bPollPassed;
+
+	void* pvSystemDbgPeripRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void* pvSystemDbgAonRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	void* pvSystemDbgDdrACtrlRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_CTRL_OFFSET;
+	void* pvSystemDbgDdrAPublRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_PUBL_OFFSET;
+	void* pvSystemDbgDdrBCtrlRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_CTRL_OFFSET;
+	void* pvSystemDbgDdrBPublRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_PUBL_OFFSET;
+	void* pvSystemDbgNocRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_NOC_OFFSET;
+
+	IMG_UINT32 ui32BDLRSetupDddrA[PLATO_DDR_PUBL_DATX_LANE_COUNT][PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE] =
+		{{0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F}};
+	IMG_UINT32 ui32BDLRSetupDddrB[PLATO_DDR_PUBL_DATX_LANE_COUNT][PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE] =
+		{{0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F}};
+
+	IMG_UINT32 ui32MemClockSpeed, ui32MemPLLClockSpeed;
+	IMG_UINT32 ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl1, ui32MemClockCtrl;
+
+	IMG_UINT32 ui32CoreClockSpeed, ui32CorePLLClockSpeed;
+	IMG_UINT32 ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl1, ui32CoreClockCtrl;
+
+#if defined(ENABLE_PLATO_HDMI)
+	IMG_UINT32 ui32HDMICECClockCtrl;
+#endif
+
+	IMG_UINT32 ui32PDPClockCtrl, ui32HDMIClockCtrl;
+
+	/* Plato Soft reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x40);
+	OSSleepms(100);
+
+	/* Temporary fix for non-32GB PCI BAR Plato boards that seem to not signal getting out reset */
+	if (!PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x030, 0xF0);
+		if (!bPollPassed)
+		{
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+	}
+
+	/* On non-32GB PCI BAR Plato boards, bring display subsystem out of reset with PLL bypassed. */
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x0);
+		OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, PLATO_CR_DISPLAY_RESET_MASK);
+		OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x1);
+	}
+
+	/* Setting dual memory interleaved mode */
+	OSWriteHWReg32(pvSystemDbgNocRegs, 0x00000050, 0x01);
+	PollPr(pvSystemDbgNocRegs, 0x00000058, 0x01, 0x1, 1, 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1, 0x1);
+
+	/* Setup DDR PLL's */
+	ui32MemClockSpeed = SysGetPlatoMemClockSpeed();
+	ui32MemPLLClockSpeed = ui32MemClockSpeed;
+
+	ui32MemClockPLLCtrl0 = (GetPlatoPLLInt(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FBDIV_SHIFT);
+	ui32MemClockPLLCtrl0 &= PLATO_CR_DDR_PLL_FBDIV_MASK;
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_REFDIV_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV1_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV2_SHIFT);
+
+	ui32MemClockPLLCtrl1 = (GetPlatoPLLFrac(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FRAC_SHIFT);
+	ui32MemClockPLLCtrl1 &= PLATO_CR_DDR_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1, ui32MemClockPLLCtrl1);
+
+	PVR_LOG(("- %s: DDR clock: %u Hz (%u MHz)",
+			 __func__, ui32MemClockSpeed, ui32MemClockSpeed/1000000));
+
+	/* Setup GPU PLL's */
+	ui32CoreClockSpeed = SysGetPlatoCoreClockSpeed();
+	ui32CorePLLClockSpeed = SysGetPlatoPLLClockSpeed(ui32CoreClockSpeed);
+
+	ui32CoreClockPLLCtrl0 = (GetPlatoPLLInt(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FBDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 &= PLATO_CR_GPU_PLL_FBDIV_MASK;
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_REFDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV1_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV2_SHIFT);
+
+	ui32CoreClockPLLCtrl1 = (GetPlatoPLLFrac(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FRAC_SHIFT);
+	ui32CoreClockPLLCtrl1 &= PLATO_CR_GPU_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1, ui32CoreClockPLLCtrl1);
+
+	PVR_LOG(("- %s: GPU clock: %u Hz (%u MHz)",
+			 __func__, ui32CoreClockSpeed,
+			 ui32CoreClockSpeed / 1000000));
+	PVR_LOG(("- %s: GPU-PLL clock: %u Hz (%u MHz)",
+			 __func__, ui32CorePLLClockSpeed,
+			 ui32CorePLLClockSpeed / 1000000));
+
+#if defined(ENABLE_PLATO_HDMI)
+	{
+		IMG_UINT32 ui32HdmiCecV0 = GetPlatoHDMICECV0Div0(ui32CorePLLClockSpeed);
+		IMG_UINT32 ui32HdmiCecV1 = GetPlatoHDMICECV1Div0(ui32CorePLLClockSpeed);
+		IMG_UINT32 ui32HdmiCecV2 = GetPlatoHDMICECV2Div0(ui32CorePLLClockSpeed);
+
+		PVR_LOG(("- %s: Setting HDMI CEC dividers:", __func__));
+		PVR_LOG(("- %s:		Hdmi CEC div0: %u(+1)", __func__, ui32HdmiCecV0));
+		PVR_LOG(("- %s:		Hdmi CEC div1: %u(+1)", __func__, ui32HdmiCecV1));
+		PVR_LOG(("- %s:		Hdmi CEC div2: %u(+1)", __func__, ui32HdmiCecV2));
+
+		/* Setup HDMI CEC clock outputs */
+		ui32HDMICECClockCtrl = 0;
+		ui32HDMICECClockCtrl |= (ui32HdmiCecV0 << PLATO_CR_HDMICECV0_DIV_0_SHIFT);
+		ui32HDMICECClockCtrl |= (ui32HdmiCecV1 << PLATO_CR_HDMICECV1_DIV_0_SHIFT);
+		ui32HDMICECClockCtrl |= (ui32HdmiCecV2 << PLATO_CR_HDMICECV2_DIV_0_SHIFT);
+
+		OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl);
+		Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl, ui32HDMICECClockCtrl);
+
+		OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1);
+		Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1, 0x1);
+	}
+#endif
+
+	PLATO_CHECKPOINT;
+
+	/* Waiting for DDR and GPU PLL's to lock */
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x3, 0x3, -1 , 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01, 0x01);
+	/* PLL Lock is done */
+
+	OSWaitus(1000);
+
+	/* Enabling gated clock output for DDR A/B */
+	ui32MemClockCtrl = (1 << PLATO_CR_DDRAG_GATE_EN_SHIFT) | (1 << PLATO_CR_DDRBG_GATE_EN_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl, ui32MemClockCtrl);
+
+	/* Enabling gated clock output for GPU and dividing the clock */
+	ui32CoreClockCtrl = (1 << PLATO_CR_GPUG_GATE_EN_SHIFT);
+	ui32CoreClockCtrl |= (GetPlatoGPUVDiv0(ui32CorePLLClockSpeed, ui32CoreClockSpeed) << PLATO_CR_GPUV_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl, ui32CoreClockCtrl);
+
+	PLATO_CHECKPOINT;
+
+	OSWaitus(100);
+
+	/* Enabling PDP gated clock output >= 165 MHz for <= 1080p */
+	ui32PDPClockCtrl = (1 << PLATO_CR_PDPG_GATE_EN_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV0Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV0_DIV_0_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV1Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV1_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl, ui32PDPClockCtrl);
+
+	OSWaitus(100);
+
+	/*
+	 * Enabling HDMI gated clock output. PDP needs HDMI clocks on for framegrabber.
+	 *
+	 * NOTE: The dividers will be reconfigured in video.c,
+	 *       for now they are set to their highest values.
+	 */
+	ui32HDMIClockCtrl = (1 << PLATO_CR_HDMIG_GATE_EN_SHIFT);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV0_DIV_0_MASK);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV1_DIV_0_MASK);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl, ui32HDMIClockCtrl);
+
+	OSWaitus(100);
+
+	PVR_LOG(("%s: Enabled PDP and HDMI clocks", __func__));
+	PLATO_CHECKPOINT;
+
+	/* Now putting DRAM controller out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL,
+		PLATO_CR_DDR_A_DATA_RESET_N_MASK | PLATO_CR_DDR_A_CTRL_RESET_N_MASK |
+		PLATO_CR_DDR_B_DATA_RESET_N_MASK | PLATO_CR_DDR_B_CTRL_RESET_N_MASK);
+	OSWaitus(100);
+
+	/* Now putting DRAM controller into reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, PLATO_CR_DDR_A_CTRL_RESET_N_MASK | PLATO_CR_DDR_B_CTRL_RESET_N_MASK);
+
+	/* Always configure DDR A */
+	eError = ConfigPlatoDram(pvSystemDbgDdrAPublRegs, pvSystemDbgDdrACtrlRegs, pvSystemDbgAonRegs,
+		ui32BDLRSetupDddrA, PLATO_CR_DDR_A_DATA_RESET_N_MASK | PLATO_CR_DDR_A_CTRL_RESET_N_MASK);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("DDR Bank A setup failed. Init cannot proceed."));
+		return eError;
+	}
+
+	PVR_LOG(("- %s: Finished DDR A Setup", __func__));
+
+	/* Configure DDR B */
+	eError = ConfigPlatoDram(pvSystemDbgDdrBPublRegs, pvSystemDbgDdrBCtrlRegs, pvSystemDbgAonRegs,
+		ui32BDLRSetupDddrB, PLATO_CR_DDR_B_DATA_RESET_N_MASK | PLATO_CR_DDR_B_CTRL_RESET_N_MASK);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("DDR Bank B setup failed. Init cannot proceed."));
+		return eError;
+	}
+
+	PVR_LOG(("- %s: Finished DDR B Setup", __func__));
+
+	/* Getting GPU And DDR A/B out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12, 0x00000F12, -1, 100);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* setting CR_ISO_CTRL:CR_GPU_CLK_E */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_ISO_CTRL, 0x000001F);
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		eError = CheckPDPAccessibility(psDevData);
+	}
+
+	return eError;
+
+}
+#else
+
+static PVRSRV_ERROR ConfigPlatoSingleChannel(SYS_DATA *psDevData)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bPollPassed;
+
+	void* pvSystemDbgPeripRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void* pvSystemDbgAonRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	void* pvSystemDbgDdrACtrlRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_CTRL_OFFSET;
+	void* pvSystemDbgDdrAPublRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_PUBL_OFFSET;
+
+	IMG_UINT32 ui32MemClockSpeed, ui32MemPLLClockSpeed;
+	IMG_UINT32 ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl1, ui32MemClockCtrl;
+
+	IMG_UINT32 ui32CoreClockSpeed;
+	IMG_UINT32 ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl1, ui32CoreClockCtrl;
+
+#if defined(ENABLE_PLATO_HDMI)
+	IMG_UINT32 ui32HDMICECClockCtrl;
+#endif
+
+	IMG_UINT32 ui32PDPClockCtrl, ui32HDMIClockCtrl;
+
+	/* Plato Soft reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x40);
+	OSSleepms(1000);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x030, 0xF0);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	PLATO_CHECKPOINT;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1, 0x1);
+
+	/* Setup DDR PLL's */
+	ui32MemClockSpeed = SysGetPlatoMemClockSpeed();
+	ui32MemPLLClockSpeed = ui32MemClockSpeed;
+
+	ui32MemClockPLLCtrl0 = (GetPlatoPLLInt(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FBDIV_SHIFT);
+	ui32MemClockPLLCtrl0 &= PLATO_CR_DDR_PLL_FBDIV_MASK;
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_REFDIV_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV1_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV2_SHIFT);
+
+	ui32MemClockPLLCtrl1 = (GetPlatoPLLFrac(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FRAC_SHIFT);
+	ui32MemClockPLLCtrl1 &= PLATO_CR_DDR_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1, ui32MemClockPLLCtrl1);
+
+	PVR_LOG(("- %s: DDR clock: %u", __func__, ui32MemClockSpeed));
+
+	/* Setup GPU PLL's */
+	ui32CoreClockSpeed = SysGetPlatoCoreClockSpeed();
+	ui32CorePLLClockSpeed = SysGetPlatoPLLClockSpeed(ui32CoreClockSpeed);
+
+	ui32CoreClockPLLCtrl0 = (GetPlatoPLLInt(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FBDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 &= PLATO_CR_GPU_PLL_FBDIV_MASK;
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_REFDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV1_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV2_SHIFT);
+
+	ui32CoreClockPLLCtrl1 = (GetPlatoPLLFrac(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FRAC_SHIFT);
+	ui32CoreClockPLLCtrl1 &= PLATO_CR_GPU_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1, ui32CoreClockPLLCtrl1);
+
+	PVR_LOG(("- %s: GPU clock: %u", __func__, ui32CoreClockSpeed));
+
+#if defined(ENABLE_PLATO_HDMI)
+	/* Setup HDMI CEC clock outputs */
+	ui32HDMICECClockCtrl = 0;
+	ui32HDMICECClockCtrl |= (GetPlatoHDMICECV0Div0(ui32CorePLLClockSpeed) << PLATO_CR_HDMICECV0_DIV_0_SHIFT);
+	ui32HDMICECClockCtrl |= (GetPlatoHDMICECV1Div0(ui32CorePLLClockSpeed) << PLATO_CR_HDMICECV1_DIV_0_SHIFT);
+	ui32HDMICECClockCtrl |= (GetPlatoHDMICECV2Div0(ui32CorePLLClockSpeed) << PLATO_CR_HDMICECV2_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl, ui32HDMICECClockCtrl);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1, 0x1);
+#endif
+
+	PLATO_CHECKPOINT;
+
+	/* Waiting for DDR and GPU PLL's to lock */
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x3, 0x3, -1 , 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01, 0x01);
+	/* PLL Lock is done */
+
+	OSWaitus(100);
+
+	/* Enabling gated clock output for DDR A/B */
+	ui32MemClockCtrl = (1 << PLATO_CR_DDRAG_GATE_EN_SHIFT) | (1 << PLATO_CR_DDRBG_GATE_EN_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl, ui32MemClockCtrl);
+
+	/* Enabling gated clock output for GPU and dividing the clock */
+	ui32CoreClockCtrl = (1 << PLATO_CR_GPUG_GATE_EN_SHIFT);
+	ui32CoreClockCtrl |= (GetPlatoGPUVDiv0(ui32CorePLLClockSpeed, ui32CoreClockSpeed) << PLATO_CR_GPUV_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl, ui32CoreClockCtrl);
+
+	PLATO_CHECKPOINT;
+
+	OSWaitus(1000);
+
+	/* Enabling PDP gated clock output >= 165 MHz for <= 1080p */
+	ui32PDPClockCtrl = (1 << PLATO_CR_PDPG_GATE_EN_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV0Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV0_DIV_0_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV1Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV1_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl, ui32PDPClockCtrl);
+
+	OSWaitus(100);
+
+	/*
+	 * Enabling HDMI gated clock output. PDP needs HDMI clocks on for framegrabber.
+	 *
+	 * NOTE: The dividers will be reconfigured in video.c,
+	 *       for now they are set to their highest values.
+	 */
+	ui32HDMIClockCtrl = (1 << PLATO_CR_HDMIG_GATE_EN_SHIFT);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV0_DIV_0_MASK);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV1_DIV_0_MASK);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl, ui32HDMIClockCtrl);
+
+	OSWaitus(100);
+
+	PVR_LOG(("%s: Enabled PDP and HDMI clocks", __func__));
+
+	PLATO_CHECKPOINT;
+
+	/* Now putting DRAM A controller out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0300);
+	OSWaitus(100);
+
+	/* Now putting DRAM A controller into reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0200);
+
+	/* Configure DRAM A control and publ regs */
+	eError = ConfigPlatoDram(pvSystemDbgDdrAPublRegs, pvSystemDbgDdrACtrlRegs, pvSystemDbgAonRegs, PLATO_CR_DDR_A_DATA_RESET_N_MASK | PLATO_CR_DDR_A_CTRL_RESET_N_MASK);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("DDR Bank setup failed. Init cannot proceed."));
+		return eError;
+	}
+	PVR_LOG(("- %s: Finished DDR A Setup", __func__));
+
+	/* Getting GPU and DDR A out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312, 0x00000312, -1, 100);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	/* setting CR_ISO_CTRL:CR_GPU_CLK_E */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_ISO_CTRL, 0x000001F);
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		eError = CheckPDPAccessibility(psDevData);
+	}
+
+	return eError;
+}
+
+#endif
+
+static PVRSRV_ERROR ConfigPlatoHw(SYS_DATA *psDevData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32MaxRestartCount = PLATO_PDP_RELOADS_MAX;
+	IMG_UINT32 ui32RestartCount = 0;
+
+	PLATO_CHECKPOINT;
+
+	/* Config Plato until PDP registers become accessible */
+	do
+	{
+		#if defined(PLATO_DUAL_CHANNEL_DDR)
+		eError = ConfigPlatoDualChannel(psDevData);
+		#else
+		eError = ConfigPlatoSingleChannel(psDevData);
+		#endif
+
+		ui32RestartCount++;
+
+		if (eError == PVRSRV_OK)
+		{
+			break;
+		}
+
+		PVR_LOG(("PDP registers inaccessible after %d out of %d tries.",
+					ui32RestartCount, ui32MaxRestartCount));
+
+	} while (ui32RestartCount < ui32MaxRestartCount &&
+			 eError == PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE);
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData) && eError == PVRSRV_OK)
+	{
+		PVR_LOG(("Got correct PDP register values after %d tries!",
+					ui32RestartCount));
+	}
+
+	if (ui32RestartCount == ui32MaxRestartCount && eError == PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE)
+	{
+		PVR_LOG(("Failed to access PDP registers after %d tries!",
+					ui32RestartCount));
+	}
+
+	return eError;
+}
+
+#endif
+
+static PVRSRV_ERROR ConfigPlato(SYS_DATA *psDevData)
+{
+	#if defined(EMULATOR)
+	return ConfigPlatoEmu(psDevData);
+	#elif defined(VIRTUAL_PLATFORM)
+	return PVRSRV_OK;
+	#else
+	return ConfigPlatoHw(psDevData);
+	#endif
+}
+
+static PVRSRV_ERROR PlatoLocalMemoryTest(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT64 i, j = 0;
+	IMG_UINT32 tmp = 0;
+	IMG_UINT32 chunk = sizeof(IMG_UINT32) * 10;
+
+	IMG_UINT64 ui64TestMemoryBase = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr;
+	IMG_UINT64 ui64TestMemorySize = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+
+	PVR_LOG(("- %s: Starting Local memory test from "
+			 "0x%llx to 0x%llx (in CPU space)",
+			 __func__, ui64TestMemoryBase,
+			 ui64TestMemoryBase + ui64TestMemorySize));
+
+	while (j < ui64TestMemorySize)
+	{
+		IMG_CPU_PHYADDR myPaddr;
+		IMG_UINT32* pui32Virt;
+
+		myPaddr.uiAddr = ui64TestMemoryBase + j;
+		pui32Virt = OSMapPhysToLin(myPaddr, chunk, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		for (i = 0; i < chunk/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32Virt + i) = 0xdeadbeef;
+			OSWriteMemoryBarrier(pui32Virt);
+			tmp = *(pui32Virt + i);
+			if (tmp != 0xdeadbeef)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Local memory read-write test failed at address=0x%llx: written 0x%x, read 0x%x",
+						ui64TestMemoryBase + ((i * sizeof(IMG_UINT32)) + j) , (IMG_UINT32) 0xdeadbeef, tmp));
+
+				OSUnMapPhysToLin(pui32Virt, chunk);
+				return PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL;
+			}
+		}
+
+		OSUnMapPhysToLin(pui32Virt, chunk);
+
+		j+= (1024 * 1024 * 500);
+	}
+
+	PVR_LOG(("Local memory read-write test passed!"));
+
+	return PVRSRV_OK;
+}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+/* Queries PCI BAR register, reads its base address and returns the reported size */
+static PVRSRV_ERROR AcquireLocalMappableMemory(IMG_HANDLE hPCI,
+											   IMG_PUINT64 pui64MemCpuPAddr,
+											   IMG_PUINT64 pui64MemSize)
+{
+	IMG_UINT16 uiVendorID, uiDevID;
+	IMG_UINT64 uiBarSize;
+
+	PVRSRV_ERROR eError;
+
+	OSPCIGetVendorDeviceIDs(hPCI, &uiVendorID, &uiDevID);
+	if (uiDevID != SYS_RGX_DEV_DEVICE_ID)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected device ID 0x%X",
+				 __func__, uiDevID));
+
+		return PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+	}
+
+	uiBarSize = OSPCIAddrRangeLen(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+
+	/* Reserve the address region for whole BAR */
+	eError = OSPCIRequestAddrRegion(hPCI, SYS_DEV_MEM_PCI_BASENUM, 0, uiBarSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device memory region not available",
+				 __func__));
+
+		return eError;
+	}
+
+	/* Clear any BIOS-configured MTRRs */
+	eError = OSPCIClearResourceMTRRs(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Failed to clear BIOS MTRRs (Error %d)",
+				 __func__, eError));
+		/* Soft-fail, the driver can limp along. */
+	}
+
+	*pui64MemCpuPAddr = OSPCIAddrRangeStart(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+	*pui64MemSize = uiBarSize;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void ReleaseLocalMappableMemory(IMG_HANDLE hPCI,
+											  IMG_UINT64 ui64MemCpuPAddr,
+											  IMG_UINT64 ui64MemSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64MemCpuPAddr);
+
+	OSPCIReleaseAddrRegion(hPCI, SYS_DEV_MEM_PCI_BASENUM, 0, ui64MemSize);
+
+	OSPCIReleaseResourceMTRRs(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+
+static PVRSRV_ERROR InitLocalMemory(SYS_DATA *psDevData,
+									PHYS_HEAP_CONFIG *pasPhysHeaps,
+									IMG_UINT32 uiPhysHeapCount,
+									IMG_HANDLE hPhysHeapPrivData)
+{
+	IMG_UINT64 ui64MappedMemSize = psDevData->ui64MappedMemSize;
+	IMG_UINT64 ui64MappedMemCpuPAddr = psDevData->ui64MappedMemCpuPAddr;
+	IMG_UINT64 ui64MappedMemDevPAddr;
+	PHYS_HEAP_CONFIG *psPhysHeap;
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	IMG_UINT64 uiFwCarveoutSize;
+#if defined(SUPPORT_AUTOVZ)
+	/* Carveout out enough LMA memory to hold the heaps of
+	 * all supported OSIDs and the FW page tables */
+	uiFwCarveoutSize = (RGX_NUM_OS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE) +
+						RGX_FIRMWARE_MAX_PAGETABLE_SIZE;
+#elif defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* Carveout out enough LMA memory to hold the heaps of all supported OSIDs */
+	uiFwCarveoutSize = (RGX_NUM_OS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+#else
+	/* Create a memory carveout just for the Host's Firmware heap.
+	 * Guests will allocate their own physical memory. */
+	uiFwCarveoutSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+#endif
+#endif /* (RGX_NUM_OS_SUPPORTED > 1) */
+
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->pszPDumpMemspaceName = "LMA";
+	psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+
+	/* Configure mappable heap region */
+	psPhysHeap->sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+	if (!PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		/*
+		 * With a BAR size that's greater than the actual memory size,
+		 * move base CPU address of the heap region to use last
+		 * aliased part of odd region and first aliased part of even region.
+		 *
+		 * This allows to use full available memory in one contiguous heap region.
+		 */
+		ui64MappedMemCpuPAddr +=
+			PLATO_DRAM_SPLIT_ADDR - (SYS_DEV_MEM_REGION_SIZE >> 1) - PLATO_DDR_DEV_PHYSICAL_BASE;
+
+		/*
+		 * As we can't read the actual memory size from the BAR we limit it to
+		 * a value set during build time.
+		 */
+		ui64MappedMemSize = SYS_DEV_MEM_REGION_SIZE;
+
+		ui64MappedMemDevPAddr = PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE;
+	}
+	else
+	{
+		ui64MappedMemDevPAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+		/*
+		 * With a BAR size less than the actual memory size (8GB),
+		 * we need to dynamically calculate the device base address
+		 * that the PCI memory window is pointing to. The address depends on
+		 * what address in host memory space the memory BAR was assigned by the BIOS.
+		 *
+		 * Bits x to 34 in host CPU base address decide on where
+		 * within the DRAM region the BAR points to, where x is the shift calculated
+		 * below based on the mapped memory size (BAR).
+		 */
+		{
+			IMG_UINT32 ui32Shift = 0;
+			IMG_UINT64 ui64Mask;
+			ui32Shift = __builtin_ffsll(psDevData->ui64MappedMemSize)-1;
+			PVR_ASSERT(ui32Shift < 35U);
+			ui64Mask = (1ULL << (35U-ui32Shift)) - 1ULL;
+			PVR_ASSERT(ui64Mask != 0);
+
+			ui64MappedMemDevPAddr += ui64MappedMemCpuPAddr & (ui64Mask << ui32Shift);
+		}
+
+		/*
+		 * Our aliased address can point anywhere in the 32GB - 64GB range,
+		 * we now need to determine the closest aliased address to the split point (48GB).
+		 * This is done by first finding the offset from the previous segment (mod below)
+		 * and adding it to either the DRAM split point or the start of the aliased
+		 * region that's closest to the split point.
+		 */
+		if (ui64MappedMemDevPAddr >= PLATO_DRAM_SPLIT_ADDR)
+		{
+			ui64MappedMemDevPAddr = PLATO_DRAM_SPLIT_ADDR + (ui64MappedMemDevPAddr % PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE);
+		}
+		else
+		{
+			ui64MappedMemDevPAddr = PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE + (ui64MappedMemDevPAddr % PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE);
+		}
+	}
+
+	/* Confidence check on mapped device physical address */
+	PVR_ASSERT(ui64MappedMemDevPAddr >= PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE && ui64MappedMemDevPAddr < PLATO_DDR_ALIASED_DEV_PHYSICAL_END);
+
+	psPhysHeap->sStartAddr.uiAddr = ui64MappedMemCpuPAddr;
+	psPhysHeap->sCardBase.uiAddr = ui64MappedMemDevPAddr;
+	psPhysHeap->uiSize = ui64MappedMemSize;
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	psPhysHeap->uiSize -= uiFwCarveoutSize;
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+	psPhysHeap->uiSize -= RGX_PLATO_RESERVE_DC_MEM_SIZE;
+#endif
+
+	/* Setup non-mappable region if BAR size is less than actual memory size (8GB) */
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		IMG_UINT64 ui64PrecedingRegionBase = 0;
+		IMG_UINT64 ui64PrecedingRegionSize = 0;
+		IMG_UINT64 ui64FollowingRegionBase = 0;
+		IMG_UINT64 ui64FollowingRegionSize = 0;
+
+		psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_NON_MAPPABLE];
+		psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeap->pszPDumpMemspaceName = "LMA";
+		psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+		psPhysHeap->hPrivData = hPhysHeapPrivData;
+		psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_PRIVATE;
+		psPhysHeap->sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+		/*
+		 * If mapped region is not at the base of memory,
+		 * then it is preceded by a non-mappable region
+		 */
+		 ui64PrecedingRegionBase = PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE;
+		 ui64PrecedingRegionSize = ui64MappedMemDevPAddr - ui64PrecedingRegionBase;
+
+		/*
+		 * If mapped region is not at the end of memory,
+		 * then it is followed by a non-mappable region
+		 */
+		ui64FollowingRegionBase = ui64MappedMemDevPAddr + psDevData->ui64MappedMemSize;
+		ui64FollowingRegionSize =
+			PLATO_DDR_ALIASED_DEV_PHYSICAL_END - (ui64MappedMemDevPAddr + psDevData->ui64MappedMemSize);
+
+		/* Use only bigger region for now */
+		if (ui64FollowingRegionSize > ui64PrecedingRegionSize)
+		{
+			psPhysHeap->sCardBase.uiAddr = ui64FollowingRegionBase;
+			psPhysHeap->uiSize = ui64FollowingRegionSize;
+		}
+		else
+		{
+			psPhysHeap->sCardBase.uiAddr = ui64PrecedingRegionBase;
+			psPhysHeap->uiSize = ui64PrecedingRegionSize;
+		}
+
+		psPhysHeap->sStartAddr.uiAddr = 0;
+
+		PVR_LOG(("Added non-mappable local memory region. Base = 0x%016llx, Size=0x%016llx",
+					psPhysHeap->sCardBase.uiAddr,
+					psPhysHeap->uiSize));
+
+		PVR_ASSERT(psPhysHeap->uiSize < SYS_DEV_MEM_REGION_SIZE);
+	}
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_FW_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->pszPDumpMemspaceName = "LMA";
+	psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+
+	psPhysHeap->sCardBase.uiAddr =
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+
+	psPhysHeap->sStartAddr.uiAddr =
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+	psPhysHeap->uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->pszPDumpMemspaceName = "LMA";
+	psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_DISPLAY;
+
+	psPhysHeap->sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+	psPhysHeap->sStartAddr.uiAddr =
+#if (RGX_NUM_OS_SUPPORTED > 1)
+		pasPhysHeaps[PHYS_HEAP_ID_FW_LOCAL].sStartAddr.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_FW_LOCAL].uiSize;
+#else
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+#endif
+	psPhysHeap->uiSize = RGX_PLATO_RESERVE_DC_MEM_SIZE;
+#endif
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHostMemory(SYS_DATA *psDevData,
+								   PHYS_HEAP_CONFIG *pasPhysHeaps,
+								   IMG_UINT32 uiPhysHeapCount,
+								   IMG_HANDLE hPhysHeapPrivData)
+{
+	PHYS_HEAP_CONFIG *psPhysHeap;
+
+	PVR_ASSERT(uiPhysHeapCount == 1);
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_CPU_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeap->pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeap->psMemFuncs = &gsHostPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+	PVR_DPF((PVR_DBG_WARNING, "Initialising CPU_LOCAL UMA Host PhysHeaps"));
+#endif
+
+	psPhysHeap->sCardBase.uiAddr = PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHybridMemory(SYS_DATA *psDevData,
+									 PHYS_HEAP_CONFIG *pasPhysHeaps,
+									 IMG_UINT32 uiPhysHeapCount,
+									 IMG_HANDLE hPhysHeapPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiPhysHeapCount >= PHYS_HEAP_ID_PDP_LOCAL);
+
+	eError = InitHostMemory(psDevData, pasPhysHeaps,
+							1, hPhysHeapPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/*
+	 * InitLocalMemory should set up the correct heaps regardless of whether the
+	 * memory configuration is 'local' or 'hybrid'.
+	 */
+	eError = InitLocalMemory(psDevData, pasPhysHeaps,
+							 uiPhysHeapCount, hPhysHeapPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Adjust the pdump memory space names */
+	pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].pszPDumpMemspaceName = "LMA";
+	pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL].pszPDumpMemspaceName = "LMA";
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR InitMemory(SYS_DATA *psDevData,
+							   PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+							   IMG_UINT32 *puiPhysHeapCountOut)
+{
+	IMG_UINT32 uiHeapCount = 1;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	uiHeapCount++;
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	uiHeapCount++;
+#endif
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+#if (PLATO_MEMORY_CONFIG != PLATO_MEMORY_HYBRID)
+#error "VZ support implemented only hybrid memory configuration (PLATO_MEMORY_HYBRID)"
+#endif
+	uiHeapCount++;
+#endif
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		uiHeapCount++;
+	}
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+	eError = InitLocalMemory(psDevData, pasPhysHeaps,
+							 uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	eError = InitHostMemory(psDevData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	eError = InitHybridMemory(psDevData, pasPhysHeaps,
+							  uiHeapCount, psDevConfig);
+#endif
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void DeInitMemory(PHYS_HEAP_CONFIG *pasPhysHeaps,
+								IMG_UINT32 uiPhysHeapCount)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr;
+
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr;
+		}
+	}
+}
+
+void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr)
+{
+	IMG_BOOL bInvalidAddress=IMG_FALSE;
+	IMG_DEV_PHYADDR sMappableRegionCardBase;
+	IMG_UINT64	uiMaxSize;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	sMappableRegionCardBase = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase;
+	uiMaxSize = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	uiMaxSize += RGX_PLATO_RESERVE_DC_MEM_SIZE;
+#endif
+
+	if (psDevPAddr[0].uiAddr > (sMappableRegionCardBase.uiAddr + uiMaxSize) ||
+	    psDevPAddr[0].uiAddr < sMappableRegionCardBase.uiAddr)
+	{
+		bInvalidAddress = IMG_TRUE;
+	}
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr;
+
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			if (psDevPAddr[ui32Idx].uiAddr > (sMappableRegionCardBase.uiAddr + uiMaxSize) ||
+			    psDevPAddr[ui32Idx].uiAddr < sMappableRegionCardBase.uiAddr)
+			{
+				bInvalidAddress = IMG_TRUE;
+			}
+
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr;
+		}
+	}
+
+	if (bInvalidAddress)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"/*****************************WARNING**********************************************/"));
+		PVR_DPF((PVR_DBG_ERROR,"%s: Given Device Physical Address belongs to non-mappable region",__func__));
+		PVR_DPF((PVR_DBG_ERROR,"Accessing the buffer may lead to unexpected behaviour"));
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_ERROR,"Input Device Physical Address count: %d", ui32NumOfAddr));
+		{
+			IMG_UINT32 ui32Idx;
+			for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			{
+				if (psDevPAddr[ui32Idx].uiAddr > (sMappableRegionCardBase.uiAddr + uiMaxSize) ||
+				    psDevPAddr[ui32Idx].uiAddr < sMappableRegionCardBase.uiAddr)
+				{
+					PVR_DPF((PVR_DBG_ERROR,"Device Physical Address : 0x%llx", psDevPAddr[ui32Idx].uiAddr));
+				}
+			}
+		}
+		dump_stack();
+#endif
+		PVR_DPF((PVR_DBG_ERROR,"/**********************************************************************************/"));
+	}
+
+}
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+		}
+	}
+}
+
+void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+		}
+	}
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   SYS_DATA *psDevData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PVRSRV_ERROR eError;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	/* Set up the RGX timing information */
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed;
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Set up the device config */
+	eError = InitMemory(psDevData, psDevConfig, &psDevConfig->pasPhysHeaps,
+						&psDevConfig->ui32PhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psDevConfig);
+
+		return eError;
+	}
+
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevConfig->pszName = PLATO_SYSTEM_NAME;
+	psDevConfig->pszVersion = GetDeviceVersionString(psDevData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr =
+		OSPCIAddrRangeStart(psDevData->hRGXPCI, SYS_PLATO_REG_PCI_BASENUM);
+	psDevConfig->sRegsCpuPBase.uiAddr += SYS_PLATO_REG_RGX_OFFSET;
+
+	psDevConfig->ui32RegsSize = SYS_PLATO_REG_RGX_SIZE;
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+	psDevConfig->bHasNonMappableLocalMemory = PLATO_HAS_NON_MAPPABLE(psDevData);
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psDevData;
+
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	/* device error notify callback function */
+	psDevConfig->pfnSysDevErrorNotify = NULL;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	DeInitMemory(psDevConfig->pasPhysHeaps, psDevConfig->ui32PhysHeapCount);
+
+	/*
+	 * The device config, RGX data and RGX timing info are part of the same
+	 * allocation so do only one free.
+	 */
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psDevData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pvOSDevice);
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevData = OSAllocZMem(sizeof(*psDevData));
+	if (psDevData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psDevData->hRGXPCI = OSPCISetDev(TO_PCI_COOKIE(pvOSDevice), HOST_PCI_INIT_FLAGS);
+	if (!psDevData->hRGXPCI)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire PCI device",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+		goto ErrorFreeDevData;
+	}
+
+	/*
+		Reserve and map system registers (does not include DMA) up to PDP
+	 */
+	psDevData->uiSystemRegSize = SYS_PLATO_REG_PDP_OFFSET - SYS_PLATO_REG_SYS_OFFSET;
+
+	eError = PlatoMapRegisters(psDevData, SYS_PLATO_REG_PCI_BASENUM, SYS_PLATO_REG_SYS_OFFSET,
+		psDevData->uiSystemRegSize, &psDevData->pvSystemRegCpuVBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers",
+				 __func__));
+		goto ErrorPCIReleaseDevice;
+	}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	eError = AcquireLocalMappableMemory(psDevData->hRGXPCI,
+										&psDevData->ui64MappedMemCpuPAddr,
+										&psDevData->ui64MappedMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorPlatoRegMap;
+	}
+
+	PVR_LOG(("Detected Plato mappable memory size: 0x%llx",
+			 psDevData->ui64MappedMemSize));
+#endif
+
+#if !defined(VIRTUAL_PLATFORM)
+	eError = ConfigPlato(psDevData);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseLocalMappableMemory;
+	}
+#endif
+
+	eError = OSPCIIRQ(psDevData->hRGXPCI, &psDevData->ui32IRQ);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't get IRQ", __func__));
+
+		goto ErrorReleaseLocalMappableMemory;
+	}
+
+	/* Register our handler */
+	eError = OSInstallSystemLISR(&psDevData->hLISR,
+								 psDevData->ui32IRQ,
+								 PLATO_SYSTEM_NAME,
+								 SystemISRHandler,
+								 psDevData,
+								 SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to install the system device interrupt handler",
+				 __func__));
+
+		goto ErrorReleaseLocalMappableMemory;
+	}
+
+	eError = DeviceConfigCreate(pvOSDevice, psDevData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorOSUninstallSystemLISR;
+	}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	/*
+	 * When using local memory perform a read-write test
+	 * to confirm that the memory has been configured correctly
+	 */
+	eError = PlatoLocalMemoryTest(psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+
+ErrorOSUninstallSystemLISR:
+	OSUninstallSystemLISR(psDevData->hLISR);
+
+ErrorReleaseLocalMappableMemory:
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	ReleaseLocalMappableMemory(psDevData->hRGXPCI,
+							   psDevData->ui64MappedMemCpuPAddr,
+							   psDevData->ui64MappedMemSize);
+
+ErrorPlatoRegMap:
+#endif
+#if !defined(VIRTUAL_PLATFORM)
+	PlatoUnmapRegisters(psDevData, psDevData->pvSystemRegCpuVBase, SYS_PLATO_REG_PCI_BASENUM,
+		SYS_PLATO_REG_SYS_OFFSET, psDevData->uiSystemRegSize);
+#endif
+
+ErrorPCIReleaseDevice:
+	OSPCIReleaseDev(psDevData->hRGXPCI);
+
+ErrorFreeDevData:
+	OSFreeMem(psDevData);
+
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)psDevConfig->hSysData;
+
+	DeviceConfigDestroy(psDevConfig);
+
+	OSUninstallSystemLISR(psDevData->hLISR);
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	ReleaseLocalMappableMemory(psDevData->hRGXPCI,
+							   psDevData->ui64MappedMemCpuPAddr,
+							   psDevData->ui64MappedMemSize);
+#endif
+
+#if !defined(VIRTUAL_PLATFORM)
+	PlatoUnmapRegisters(psDevData, psDevData->pvSystemRegCpuVBase, SYS_PLATO_REG_PCI_BASENUM,
+		SYS_PLATO_REG_SYS_OFFSET, psDevData->uiSystemRegSize);
+#endif
+
+	OSPCIReleaseDev(psDevData->hRGXPCI);
+	OSFreeMem(psDevData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)hSysData;
+	IMG_UINT32 ui32InterruptFlag;
+
+	switch (ui32IRQ)
+	{
+		case PLATO_IRQ_GPU:
+			ui32InterruptFlag = (0x1 << PLATO_GPU_INT_SHIFT);
+			break;
+		case PLATO_IRQ_PDP:
+			ui32InterruptFlag = (0x1 << PLATO_PDP_INT_SHIFT);
+			break;
+		case PLATO_IRQ_HDMI:
+			ui32InterruptFlag = (0x1 << PLATO_HDMI_INT_SHIFT);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d",
+					 __func__, ui32IRQ));
+			return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	if (psDevData->sInterruptData[ui32IRQ].pfnLISR)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ISR for %s already installed!",
+				 __func__, pszName));
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+
+	psDevData->sInterruptData[ui32IRQ].psSysData		= psDevData;
+	psDevData->sInterruptData[ui32IRQ].pszName		= pszName;
+	psDevData->sInterruptData[ui32IRQ].pfnLISR		= pfnLISR;
+	psDevData->sInterruptData[ui32IRQ].pvData		= pvData;
+	psDevData->sInterruptData[ui32IRQ].ui32InterruptFlag	= ui32InterruptFlag;
+
+	*phLISRData = &psDevData->sInterruptData[ui32IRQ];
+
+	EnableInterrupt(psDevData, psDevData->sInterruptData[ui32IRQ].ui32InterruptFlag);
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, psDevData->ui32IRQ));
+
+	return PVRSRV_OK;
+
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+
+	SYS_INTERRUPT_DATA *psInterruptData = (SYS_INTERRUPT_DATA *)hLISRData;
+
+	PVR_ASSERT(psInterruptData);
+
+	PVR_LOG(("Uninstalling device LISR %s on IRQ %d", psInterruptData->pszName, ((SYS_DATA *)psInterruptData->psSysData)->ui32IRQ));
+
+	/* Disable interrupts for this device */
+#if !defined(VIRTUAL_PLATFORM)
+	DisableInterrupt(psInterruptData->psSysData, psInterruptData->ui32InterruptFlag);
+#endif
+
+	/* Reset the interrupt data */
+	psInterruptData->pszName = NULL;
+	psInterruptData->psSysData = NULL;
+	psInterruptData->pfnLISR = NULL;
+	psInterruptData->pvData = NULL;
+	psInterruptData->ui32InterruptFlag = 0;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysGetPlatoGPUTemp(SYS_DATA * psDevData, IMG_UINT32 *ui32Temp)
+{
+	IMG_UINT32 ui32Value = 0;
+#if defined(PLATO_TEMPA_SENSOR_IMPLEMENTED)
+	void* pvSystemTempRegs;
+
+	/* Temp sensor A starts at 0x80000 within peripheral regs */
+	pvSystemTempRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_TEMPA_OFFSET;
+
+	OSWriteHWReg32(pvSystemTempRegs, 0x04, 0x3);
+	Poll(pvSystemTempRegs, 0x10, (1 << 16), (1 << 16));
+	ui32Value = OSReadHWReg32(pvSystemTempRegs, 0x10) & 0xFFF;
+
+	ui32Value = ((ui32Value * 352) / 4096) - 109;
+#endif  // PLATO_TEMPA_SENSOR_IMPLEMENTED
+
+	*ui32Temp = ui32Value;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.h
new file mode 100644
index 000000000000..479f9f990135
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.h
@@ -0,0 +1,135 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSCONFIG_H)
+#define SYSCONFIG_H
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "interrupt_support.h"
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+#define MAX_SYSTEMS 32
+
+typedef struct _SYS_INTERRUPT_DATA_
+{
+	void			*psSysData;
+	const IMG_CHAR	*pszName;
+	PFN_SYS_LISR	pfnLISR;
+	void			*pvData;
+	IMG_UINT32		ui32InterruptFlag;
+	IMG_UINT32		ui32IRQ;
+} SYS_INTERRUPT_DATA;
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	IMG_HANDLE					hRGXPCI;
+
+	void						*pvSystemRegCpuVBase;
+	size_t						uiSystemRegSize;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	IMG_UINT64					ui64MappedMemCpuPAddr;
+	IMG_UINT64					ui64MappedMemSize;
+#endif
+
+	IMG_HANDLE					hLISR;
+	IMG_UINT32					ui32IRQ;
+
+	/* Rogue, PDP and HDMI */
+	SYS_INTERRUPT_DATA			sInterruptData[PLATO_IRQ_MAX];
+};
+
+/* Helpers for getting DDR/GPU/PLL clock speed */
+IMG_UINT32 SysGetPlatoMemClockSpeed(void);
+IMG_UINT32 SysGetPlatoCoreClockSpeed(void);
+IMG_UINT32 SysGetPlatoPLLClockSpeed(IMG_UINT32 ui32ClockSpeed);
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr);
+#endif
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr);
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+PVRSRV_ERROR PlatoMapRegisters(SYS_DATA *psDevData,
+	IMG_UINT32 ui32BaseNum,
+	IMG_UINT32 ui32Offset,
+	IMG_UINT32 ui32Size,
+	void ** pvCpuVAddr);
+
+void PlatoUnmapRegisters(SYS_DATA *psDevData,
+	void * pvBase,
+	IMG_UINT32 ui32BaseNum,
+	IMG_UINT32 ui32Offset,
+	IMG_UINT32 ui32Size);
+
+#if (PLATO_MEMORY_CONFIG != PLATO_MEMORY_LOCAL) && \
+	(PLATO_MEMORY_CONFIG != PLATO_MEMORY_HYBRID) && \
+	(PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#error "PLATO_MEMORY_CONFIG not valid"
+#endif
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* !defined(SYSCONFIG_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysinfo.h
new file mode 100644
index 000000000000..a7d2857d3c7c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysinfo.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+#define SYS_RGX_DEV_VENDOR_ID	(0x1AEE)
+#define SYS_RGX_DEV_DEVICE_ID	(0x0003)
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(EMULATOR)
+/* Emulator clock ~600 times slower than HW */
+#define MAX_HW_TIME_US                           (300000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (100000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#endif
+
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#endif /* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/Kbuild.mk
new file mode 100644
index 000000000000..14b67465936f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/Kbuild.mk
@@ -0,0 +1,45 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+	services/system/$(PVR_SYSTEM)/sysconfig.o
+
+ccflags-y += \
+	-I$(TOP)/services/system/rogue/common/env/linux
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.c
new file mode 100644
index 000000000000..df4613df5fe7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.c
@@ -0,0 +1,746 @@
+/*************************************************************************/ /*!
+@File
+@Title			System Configuration
+@Copyright		Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description	System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_memallocflags.h"
+#include "syscommon.h"
+#include "power.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pci_support.h"
+#include "interrupt_support.h"
+#include "plato_drv.h"
+#include <linux/dma-mapping.h>
+
+#define PLATO_HAS_NON_MAPPABLE(sys) (sys->pdata->has_nonmappable == true)
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#define PHYS_HEAP_ID_CPU_LOCAL 0
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_FW_SHARED 1
+#define PHYS_HEAP_ID_CPU_LOCAL 2
+#if defined(SUPPORT_PLATO_DISPLAY)
+#define PHYS_HEAP_ID_PDP_LOCAL 3
+#define PHYS_HEAP_ID_NON_MAPPABLE 4
+#else
+#define PHYS_HEAP_ID_NON_MAPPABLE 3
+#endif
+
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_FW_SHARED 1
+#if defined(SUPPORT_PLATO_DISPLAY)
+#define PHYS_HEAP_ID_PDP_LOCAL 2
+#define PHYS_HEAP_ID_NON_MAPPABLE 3
+#else
+#define PHYS_HEAP_ID_NON_MAPPABLE 2
+#endif
+#endif
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+static_assert(PHYS_HEAP_ID_PDP_LOCAL == PVRSRV_PHYS_HEAP_CONFIG_PDP_LOCAL_ID, "PDP heap ID mismatch.");
+#endif
+
+static struct plato_debug_register plato_noc_regs[] = {
+	{"NOC Offset 0x00", 0x00, 0},
+	{"NOC Offset 0x04", 0x04, 0},
+	{"NOC Offset 0x08", 0x08, 0},
+	{"NOC Offset 0x0C", 0x0C, 0},
+	{"NOC Offset 0x10", 0x10, 0},
+	{"NOC Offset 0x14", 0x14, 0},
+	{"NOC Offset 0x18", 0x18, 0},
+	{"NOC Offset 0x1C", 0x1C, 0},
+	{"NOC Offset 0x50", 0x50, 0},
+	{"NOC Offset 0x54", 0x54, 0},
+	{"NOC Offset 0x58", 0x58, 0},
+	{"DDR A Ctrl", SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET, 0},
+	{"DDR A Data", SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET, 0},
+	{"DDR A Publ", SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET, 0},
+	{"DDR B Ctrl", SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET, 0},
+	{"DDR B Data", SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET, 0},
+	{"DDR B Publ", SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET, 0},
+	{"Display S", SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET, 0},
+	{"GPIO 0 S", SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET, 0},
+	{"GPIO 1 S", SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET, 0},
+	{"GPU S", SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET, 0},
+	{"PCI PHY", SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET, 0},
+	{"PCI Reg", SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET, 0},
+	{"PCI S", SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET, 0},
+	{"Periph S", SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET, 0},
+	{"Ret Reg", SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET, 0},
+	{"Service", SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET, 0},
+};
+
+static struct plato_debug_register plato_aon_regs[] = {
+	{"AON Offset 0x0000", 0x0000, 0},
+	{"AON Offset 0x0070", 0x0070, 0},
+};
+
+typedef struct _SYS_DATA_ {
+	struct platform_device *pdev;
+	struct resource *registers;
+	struct plato_rogue_platform_data *pdata;
+} SYS_DATA;
+
+typedef struct {
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	return NULL;
+}
+
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 i = 0;
+
+	PVR_DUMPDEBUG_LOG("------[ Plato System Debug ]------");
+
+	if (plato_debug_info(psSysData->pdev->dev.parent, &plato_noc_regs[0], &plato_aon_regs[0]))
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	for (i = 0; i < ARRAY_SIZE(plato_noc_regs); i++)
+		PVR_DUMPDEBUG_LOG("%s: 0x%x", plato_noc_regs[i].description, plato_noc_regs[i].value);
+
+	for (i = 0; i < ARRAY_SIZE(plato_aon_regs); i++)
+		PVR_DUMPDEBUG_LOG("%s: 0x%x", plato_aon_regs[i].description, plato_aon_regs[i].value);
+
+	return PVRSRV_OK;
+}
+
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+
+static PVRSRV_ERROR InitLocalHeaps(SYS_DATA *psSysData,
+									PHYS_HEAP_CONFIG *pasPhysHeaps,
+									IMG_UINT32 uiPhysHeapCount,
+									IMG_HANDLE hPhysHeapPrivData)
+{
+	PHYS_HEAP_CONFIG *psPhysHeap;
+	IMG_UINT64 ui64FwCarveoutSize = (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = "lma_gpu_local";
+	psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+	/* Configure mappable heap region */
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = psSysData->pdata->rogue_heap_mappable.base;
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = psSysData->pdata->rogue_heap_dev_addr;
+	psPhysHeap->uConfig.sLMA.uiSize = psSysData->pdata->rogue_heap_mappable.size - ui64FwCarveoutSize;
+
+	PVR_LOG(("Added mappable local memory heap. Base = 0x%016llx, Size=0x%016llx",
+			psPhysHeap->uConfig.sLMA.sCardBase.uiAddr,
+			psPhysHeap->uConfig.sLMA.uiSize));
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_FW_SHARED];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = "lma_fw_shared";
+	psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+	/* Configure mappable heap region */
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = psSysData->pdata->rogue_heap_mappable.base +
+	                                             pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.uiSize;
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = psSysData->pdata->rogue_heap_dev_addr +
+	                                            pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.uiSize;
+	psPhysHeap->uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	PVR_LOG(("Added mappable Firmware heap. Base = 0x%016llx, Size=0x%016llx",
+			psPhysHeap->uConfig.sLMA.sCardBase.uiAddr,
+			psPhysHeap->uConfig.sLMA.uiSize));
+
+	/* Setup non-mappable region if BAR size is less than actual memory size (8GB) */
+	if (PLATO_HAS_NON_MAPPABLE(psSysData))
+	{
+		psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_NON_MAPPABLE];
+		psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_PRIVATE;
+		psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+		psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+		psPhysHeap->uConfig.sLMA.pszHeapName = "lma_gpu_private";
+		psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+		psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = psSysData->pdata->rogue_heap_nonmappable.base;
+		psPhysHeap->uConfig.sLMA.uiSize = psSysData->pdata->rogue_heap_nonmappable.size;
+		psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = 0;
+
+		PVR_LOG(("Added non-mappable local memory heap. Base = 0x%016llx, Size=0x%016llx",
+					psPhysHeap->uConfig.sLMA.sCardBase.uiAddr,
+					psPhysHeap->uConfig.sLMA.uiSize));
+
+		PVR_ASSERT(psPhysHeap->uConfig.sLMA.uiSize < SYS_DEV_MEM_REGION_SIZE);
+	}
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_EXTERNAL;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = "lma_pdp_local";
+	psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = psSysData->pdata->pdp_heap.base;
+	psPhysHeap->uConfig.sLMA.uiSize = psSysData->pdata->pdp_heap.size;
+
+	PVR_LOG(("Added PDP heap. Base = 0x%016llx, Size=0x%016llx",
+			psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr,
+			psPhysHeap->uConfig.sLMA.uiSize));
+#endif
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHostHeaps(SYS_DATA *psSysData,
+								PHYS_HEAP_CONFIG *pasPhysHeaps,
+								IMG_UINT32 uiPhysHeapCount,
+								IMG_HANDLE hPhysHeapPrivData)
+{
+	PHYS_HEAP_CONFIG *psPhysHeap;
+
+	PVR_ASSERT(uiPhysHeapCount == 1);
+
+	psPhysHeap = &pasPhysHeaps[0];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeap->uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeap->uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeap->uConfig.sUMA.pszHeapName = "uma_gpu_local";
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+	psPhysHeap->uConfig.sUMA.pszHeapName = "uma_cpu_local";
+	PVR_DPF((PVR_DBG_WARNING, "Initialising CPU_LOCAL UMA Host PhysHeaps"));
+#if !defined(SUPPORT_PLATO_DISPLAY)
+	psPhysHeap->ui32UsageFlags |= PHYS_HEAP_USAGE_EXTERNAL;
+#endif
+#endif
+	psPhysHeap->uConfig.sUMA.sCardBase.uiAddr = PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	psPhysHeap->uConfig.sUMA.hPrivData = hPhysHeapPrivData;
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHybridHeaps(SYS_DATA *psSysData,
+									PHYS_HEAP_CONFIG *pasPhysHeaps,
+									IMG_UINT32 uiPhysHeapCount,
+									IMG_HANDLE hPhysHeapPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiPhysHeapCount >= PHYS_HEAP_ID_NON_MAPPABLE);
+
+	eError = InitHostHeaps(psSysData,
+							&pasPhysHeaps[PHYS_HEAP_ID_CPU_LOCAL], 1,
+							hPhysHeapPrivData);
+	if (eError != PVRSRV_OK)
+		return eError;
+
+	/*
+	 * InitLocalHeaps should set up the correct heaps regardless of whether the
+	 * memory configuration is 'local' or 'hybrid'.
+	 */
+	eError = InitLocalHeaps(psSysData, pasPhysHeaps,
+							 uiPhysHeapCount, hPhysHeapPrivData);
+	if (eError != PVRSRV_OK) {
+		return eError;
+	}
+
+	/* Adjust the pdump memory space names */
+	pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.pszPDumpMemspaceName = "LMA0";
+	pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.pszHeapName = "lma0_gpu_local";
+	pasPhysHeaps[PHYS_HEAP_ID_FW_SHARED].uConfig.sLMA.pszPDumpMemspaceName = "LMA0";
+	pasPhysHeaps[PHYS_HEAP_ID_FW_SHARED].uConfig.sLMA.pszHeapName = "lma0_fw_shared";
+#if defined(SUPPORT_PLATO_DISPLAY)
+	pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL].uConfig.sLMA.pszPDumpMemspaceName = "LMA1";
+	pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL].uConfig.sLMA.pszHeapName = "lma1_pdp_local";
+#endif
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR PhysHeapsCreate(SYS_DATA *psSysData,
+									PVRSRV_DEVICE_CONFIG *psDevConfig,
+									PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	IMG_UINT32 uiHeapCount = 2; /* GPU_LOCAL + FW_SHARED*/
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	uiHeapCount++;
+#endif
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+	uiHeapCount++;
+#endif
+
+	if (PLATO_HAS_NON_MAPPABLE(psSysData))
+	{
+		uiHeapCount++;
+	}
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+	eError = InitLocalHeaps(psSysData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	eError = InitHostHeaps(psSysData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	eError = InitHybridHeaps(psSysData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#endif
+
+	if (eError != PVRSRV_OK) {
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr;
+
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx) {
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr;
+		}
+	}
+}
+
+static void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr,
+					IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr;
+
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx) {
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr;
+		}
+	}
+}
+
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	}
+}
+
+static void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+	}
+}
+
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PVRSRV_ERROR eError;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	/* Set up the RGX timing information */
+	psRGXTimingInfo->ui32CoreClockSpeed = plato_core_clock_speed(&psSysData->pdev->dev);
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Initialize heaps */
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &psDevConfig->pasPhysHeaps,
+						&psDevConfig->ui32PhysHeapCount);
+	if (eError != PVRSRV_OK) {
+		OSFreeMem(psDevConfig);
+		return eError;
+	}
+
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevConfig->pszName = PLATO_SYSTEM_NAME;
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = SYS_PLATO_REG_RGX_SIZE;
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+	psDevConfig->bHasNonMappableLocalMemory = PLATO_HAS_NON_MAPPABLE(psSysData);
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	psDevConfig->ui32IRQ = PLATO_INTERRUPT_GPU;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+		OSFreeMem(psDevConfig->pszVersion);
+
+	if (psDevConfig->pasPhysHeaps)
+		OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	/*
+	 * The device config, RGX data and RGX timing info are part of the same
+	 * allocation so do only one free.
+	 */
+	OSFreeMem(psDevConfig);
+}
+
+static PVRSRV_ERROR PlatoLocalMemoryTest(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT64 i, j = 0;
+	IMG_UINT32 tmp = 0;
+	IMG_UINT32 chunk = sizeof(IMG_UINT32) * 10;
+
+	IMG_UINT64 ui64TestMemoryBase = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr;
+	IMG_UINT64 ui64TestMemorySize = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.uiSize;
+
+	PVR_LOG(("%s: Starting Local memory test from 0x%llx to 0x%llx (in CPU space)",
+			 __func__, ui64TestMemoryBase, ui64TestMemoryBase + ui64TestMemorySize));
+
+	while (j < ui64TestMemorySize) {
+		IMG_CPU_PHYADDR myPaddr;
+		IMG_UINT32 *pui32Virt;
+
+		myPaddr.uiAddr = ui64TestMemoryBase + j;
+		pui32Virt = OSMapPhysToLin(myPaddr, chunk, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		for (i = 0; i < chunk/sizeof(IMG_UINT32); i++) {
+			*(pui32Virt + i) = 0xdeadbeef;
+			OSWriteMemoryBarrier(pui32Virt);
+			tmp = *(pui32Virt + i);
+			if (tmp != 0xdeadbeef) {
+				PVR_DPF((PVR_DBG_ERROR,
+						"Local memory read-write test failed at address=0x%llx: written 0x%x, read 0x%x",
+						ui64TestMemoryBase + ((i * sizeof(IMG_UINT32)) + j), (IMG_UINT32) 0xdeadbeef, tmp));
+
+				OSUnMapPhysToLin(pui32Virt, chunk);
+				return PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL;
+			}
+		}
+
+		OSUnMapPhysToLin(pui32Virt, chunk);
+
+		j += (1024 * 1024 * 500);
+	}
+
+	PVR_LOG(("Local memory read-write test passed!"));
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	IMG_UINT32 uiRegistersSize;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+
+	/* Retrieve platform device and data */
+	psSysData->pdev = to_platform_device((struct device *) pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/* Enable plato PCI */
+	if (plato_enable(psSysData->pdev->dev.parent)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device", __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev, IORESOURCE_MEM, PLATO_ROGUE_RESOURCE_REGS);
+	if (!psSysData->registers) {
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+	if (uiRegistersSize < SYS_PLATO_REG_RGX_SIZE) {
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %08X, required 0x%08x)",
+				 __func__, uiRegistersSize, SYS_PLATO_REG_RGX_SIZE));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+#if !defined(VIRTUAL_PLATFORM)
+	/* Reserve the rogue registers address range */
+	if (!request_mem_region(psSysData->registers->start,
+							uiRegistersSize,
+							PVRSRV_MODNAME)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Rogue register memory region not available", __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrorDevDisable;
+	}
+#endif
+
+	eError = DeviceConfigCreate(pvOSDevice, psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create device config", __func__));
+		goto ErrorReleaseMemRegion;
+	}
+
+	PlatoLocalMemoryTest(psDevConfig);
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	plato_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	plato_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+static void PlatoInterruptHandler(void *pvData)
+{
+	LISR_DATA *psLISRData = pvData;
+
+	psLISRData->pfnLISR(psLISRData->pvData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+
+	/* Should only accept GPU interrupts through this API */
+	if (ui32IRQ != PLATO_INTERRUPT_GPU) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	if (plato_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, PlatoInterruptHandler, psLISRData)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_set_interrupt_handler() failed", __func__));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	if (plato_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_enable_interrupt() failed", __func__));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, ui32IRQ));
+
+	return PVRSRV_OK;
+
+err_unset_interrupt_handler:
+	plato_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	return eError;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = plato_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_enable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = plato_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " from irq %u", psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.h
new file mode 100644
index 000000000000..5761da3a1bf6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.h
@@ -0,0 +1,105 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSCONFIG_H)
+#define SYSCONFIG_H
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "plato_drv.h"
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+#define MAX_SYSTEMS 32
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_DEV_PHYADDR *psDevPAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr,
+					IMG_DEV_PHYADDR *psDevPAddr);
+#endif
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_DEV_PHYADDR *psDevPAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr,
+					IMG_DEV_PHYADDR *psDevPAddr);
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs = {
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoLocalCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoLocalDevPAddrToCpuPAddr,
+};
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs = {
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoSystemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoSystemDevPAddrToCpuPAddr,
+};
+#endif
+
+#if (PLATO_MEMORY_CONFIG != PLATO_MEMORY_LOCAL) && \
+	(PLATO_MEMORY_CONFIG != PLATO_MEMORY_HYBRID) && \
+	(PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#error "PLATO_MEMORY_CONFIG not valid"
+#endif
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* !defined(SYSCONFIG_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysinfo.h
new file mode 100644
index 000000000000..fd796a75797f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysinfo.h
@@ -0,0 +1,86 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+#if defined(__KERNEL__)
+#include "plato_drv.h"
+#endif
+
+#define SYS_RGX_DEV_VENDOR_ID	(0x1AEE)
+#define SYS_RGX_DEV_DEVICE_ID	(0x0003)
+
+#if defined(__KERNEL__)
+#if defined(PLATO_MULTI_DEVICE)
+#define SYS_RGX_DEV_NAME_0	PLATO_MAKE_DEVICE_NAME_ROGUE(0)
+#define SYS_RGX_DEV_NAME_1	PLATO_MAKE_DEVICE_NAME_ROGUE(1)
+#define SYS_RGX_DEV_NAME_2	PLATO_MAKE_DEVICE_NAME_ROGUE(2)
+#define SYS_RGX_DEV_NAME_3	PLATO_MAKE_DEVICE_NAME_ROGUE(3)
+#else
+#define SYS_RGX_DEV_NAME	PLATO_DEVICE_NAME_ROGUE
+#endif
+#endif
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(EMULATOR)
+/* Emulator clock ~600 times slower than HW */
+#define MAX_HW_TIME_US                           (300000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1000000)
+
+#if defined(VIRTUAL_PLATFORM)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#elif defined(EMULATOR)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#endif
+
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(100000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#endif /* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/Kbuild.mk
new file mode 100644
index 000000000000..41b532383198
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/Kbuild.mk
@@ -0,0 +1,52 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+ services/system/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/system/common/uma_heap_fns.o
+
+ifeq ($(SUPPORT_ION),1)
+$(PVRSRV_MODNAME)-y += services/system/common/env/linux/ion_support_generic.o
+endif
+
+ifeq ($(PVR_ARCH),volcanic)
+ ccflags-y += -DVOLCANIC_FBCDCV31
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.c
new file mode 100644
index 000000000000..4543f8e7d25f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.c
@@ -0,0 +1,330 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#include "uma_heap_fns.h"
+
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+
+
+/*
+ * In systems that support trusted device address protection, there are three
+ * physical heaps from which pages should be allocated:
+ * - one heap for normal allocations
+ * - one heap for allocations holding META code memory
+ * - one heap for allocations holding secured DRM data
+ */
+
+#define PHYS_HEAP_IDX_GENERAL     0
+#define PHYS_HEAP_IDX_FW          1
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define PHYS_HEAP_IDX_TDFWMEM     2
+#define PHYS_HEAP_IDX_TDSECUREBUF 3
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+#define PHYS_HEAP_IDX_FW_MEMORY   2
+#endif
+
+/* Change to test CPU_LOCAL sys layers*/
+#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_GPU_LOCAL
+//#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_CPU_LOCAL
+
+#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_GPU_LOCAL
+//#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_CPU_LOCAL
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+static DECLARE_PHYS_HEAP_SPAS_REGION(sSpasSystemMemory);
+#endif
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	/*
+	 * This function is called during device initialisation, which on Linux,
+	 * means it won't be called concurrently. As such, there's no need to
+	 * protect it with a lock or use an atomic variable.
+	 */
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiHeapCount = 2;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	uiHeapCount += 2;
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	uiHeapCount += 1;
+#endif
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].ui32UsageFlags = UMA_HEAP_USAGE_FLAG;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.pszHeapName = "uma_local";
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.psSpasRegion = &sSpasSystemMemory;
+#endif
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	/* Provide a specific set of Intermediate Physical Address modifiers
+	 * for the default heap. Also name this as "SYSMEM_IPA" to aid PDump
+	 * processing.
+	 * Modifiers are:
+	 *  IPAPolicyDefault = 3
+	 *  IPAPolicyMask = 7
+	 *  IPAPolicyShift = 52
+	 * This will result in the PhysAddr values having bits 53..55 set with '3'
+	 * by default.
+	 */
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM_IPA";
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].sIPAConfig.ui8IPAPolicyDefault = 3;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].sIPAConfig.ui8IPAPolicyMask = 7;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].sIPAConfig.ui8IPAPolicyShift = 52;
+#endif
+
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].eType = PHYS_HEAP_TYPE_UMA;
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM_FW";
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].uConfig.sUMA.pszHeapName = "uma_fw_shared";
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].uConfig.sUMA.pszPDumpMemspaceName = "TDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].uConfig.sUMA.pszHeapName = "uma_tdfwmem";
+
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_SECURE;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].uConfig.sUMA.pszPDumpMemspaceName = "TDSECBUFMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].uConfig.sUMA.pszHeapName = "uma_tdsecbufmem";
+
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].uConfig.sUMA.pszPDumpMemspaceName = "DEDICATEDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].uConfig.sUMA.pszHeapName = "uma_dedicatedfwmem";
+#endif
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+#if defined(SUPPORT_AXI_ACE_TEST)
+		if ( ui64Features & RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		}else
+#endif
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_NONE;
+		}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszName                = "nohw";
+	psDevConfig->pszVersion             = NULL;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr   = 0x00f00000;
+	psDevConfig->ui32RegsSize           = 0x100000;
+	psDevConfig->ui32IRQ                = 0x00000bad;
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+	psDevConfig->eDefaultHeap = UMA_DEFAULT_HEAP;
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+
+	psDevConfig->bDevicePA0IsValid       = IMG_FALSE;
+	psDevConfig->pfnSysDevFeatureDepInit = SysDevFeatureDepInit;
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Pdump validation system registers */
+
+#if !defined(HW_ERN_66622) && !defined(VOLCANIC_FBCDCV31)
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+#else
+	psDevConfig->bHasFBCDCVersion31 = IMG_TRUE;
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	PVR_UNREFERENCED_PARAMETER(ui32IRQ);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(pfnLISR);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(phLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.h
new file mode 100644
index 000000000000..acbadc22e6a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(SYSCCONFIG_H)
+#define SYSCCONFIG_H
+
+
+#define RGX_NOHW_CORE_CLOCK_SPEED 100000000
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* SYSCCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysinfo.h
new file mode 100644
index 000000000000..709b54c3bd0b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysinfo.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME    "rgxnohw"
+#endif
+
+#endif	/* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/Kbuild.mk
new file mode 100644
index 000000000000..9d2ca9476931
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/Kbuild.mk
@@ -0,0 +1,48 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += services/system/$(PVR_SYSTEM)/sysconfig.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/$(PVR_ARCH)/common/vmm_type_$(VMM_TYPE).o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.c
new file mode 100644
index 000000000000..99b8366c11a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.c
@@ -0,0 +1,310 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the system layer for QEMU vexpress virtual-platform
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#endif
+
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "interrupt_support.h"
+#include "vz_vmm_pvz.h"
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+static RGX_TIMING_INFORMATION	gsRGXTimingInfo;
+static RGX_DATA					gsRGXData;
+static PVRSRV_DEVICE_CONFIG		gsDevices[1];
+static PHYS_HEAP_FUNCTIONS		gsPhysHeapFuncs;
+static PHYS_HEAP_CONFIG			gsPhysHeapConfig[2];
+
+#if defined(SUPPORT_PDVFS)
+static const IMG_OPP asOPPTable[] =
+{
+	{ 824,  240000000},
+	{ 856,  280000000},
+	{ 935,  380000000},
+	{ 982,  440000000},
+	{ 1061, 540000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(asOPPTable[0]))
+
+static void SetFrequency(IMG_UINT32 ui32Frequency) {}
+
+static void SetVoltage(IMG_UINT32 ui32Volt) {}
+#endif
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+#if defined(SUPPORT_AXI_ACE_TEST)
+		if ( ui64Features & RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+		{
+			gsDevices[0].eCacheSnoopingMode     = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		}
+		else
+#endif
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_EMULATED;
+		}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	IMG_UINT32 ui32NextPhysHeapID = 0;
+#if defined(__linux__)
+	int iIrq;
+	struct resource *psDevMemRes = NULL;
+	struct platform_device *psDev;
+
+	psDev = to_platform_device((struct device *)pvOSDevice);
+#endif
+
+	if (gsDevices[0].pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+#endif
+
+	/*
+	 * Setup information about physical memory heap(s) we have
+	 */
+	gsPhysHeapFuncs.pfnCpuPAddrToDevPAddr = UMAPhysHeapCpuPAddrToDevPAddr;
+	gsPhysHeapFuncs.pfnDevPAddrToCpuPAddr = UMAPhysHeapDevPAddrToCpuPAddr;
+
+	gsPhysHeapConfig[ui32NextPhysHeapID].pszPDumpMemspaceName = "SYSMEM";
+	gsPhysHeapConfig[ui32NextPhysHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	gsPhysHeapConfig[ui32NextPhysHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	gsPhysHeapConfig[ui32NextPhysHeapID].hPrivData = NULL;
+	gsPhysHeapConfig[ui32NextPhysHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	ui32NextPhysHeapID += 1;
+
+	/*
+	 * Setup RGX specific timing data
+	 */
+	gsRGXTimingInfo.ui32CoreClockSpeed        = RGX_VEXPRESS_CORE_CLOCK_SPEED;
+	gsRGXTimingInfo.bEnableActivePM           = IMG_FALSE;
+	gsRGXTimingInfo.bEnableRDPowIsland        = IMG_FALSE;
+	gsRGXTimingInfo.ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/*
+	 *Setup RGX specific data
+	 */
+	gsRGXData.psRGXTimingInfo = &gsRGXTimingInfo;
+
+	/*
+	 * Setup device
+	 */
+	gsDevices[0].pvOSDevice				= pvOSDevice;
+	gsDevices[0].pszName                = "vexpress";
+	gsDevices[0].pszVersion             = NULL;
+
+	/* Device setup information */
+#if defined(__linux__)
+	psDevMemRes = platform_get_resource(psDev, IORESOURCE_MEM, 0);
+	if (psDevMemRes)
+	{
+		gsDevices[0].sRegsCpuPBase.uiAddr = psDevMemRes->start;
+		gsDevices[0].ui32RegsSize         = (unsigned int)(psDevMemRes->end - psDevMemRes->start);
+	}
+	else
+#endif
+	{
+#if defined(__linux__)
+		PVR_LOG(("%s: platform_get_resource() failed, using mmio/sz 0x%x/0x%x",
+				__func__,
+				VEXPRESS_GPU_PBASE,
+				VEXPRESS_GPU_SIZE));
+#endif
+		gsDevices[0].sRegsCpuPBase.uiAddr   = VEXPRESS_GPU_PBASE;
+		gsDevices[0].ui32RegsSize           = VEXPRESS_GPU_SIZE;
+	}
+
+#if defined(__linux__)
+	iIrq = platform_get_irq(psDev, 0);
+	if (iIrq >= 0)
+	{
+		gsDevices[0].ui32IRQ = (IMG_UINT32) iIrq;
+	}
+	else
+#endif
+	{
+#if defined(__linux__)
+		PVR_LOG(("%s: platform_get_irq() failed, using irq %d",
+				__func__,
+				VEXPRESS_IRQ_GPU));
+#endif
+		gsDevices[0].ui32IRQ = VEXPRESS_IRQ_GPU;
+	}
+
+	/* Device's physical heaps */
+	gsDevices[0].pasPhysHeaps = gsPhysHeapConfig;
+	gsDevices[0].ui32PhysHeapCount = ARRAY_SIZE(gsPhysHeapConfig);
+	gsDevices[0].eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	/* No power management on VIRTUAL_PLATFORM system */
+	gsDevices[0].pfnPrePowerState       = NULL;
+	gsDevices[0].pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	gsDevices[0].pfnClockFreqGet        = NULL;
+
+	gsDevices[0].hDevData               = &gsRGXData;
+
+	gsDevices[0].pfnSysDevFeatureDepInit = &SysDevFeatureDepInit;
+
+	gsDevices[0].bHasFBCDCVersion31	= IMG_FALSE;
+	gsDevices[0].bDevicePA0IsValid	= IMG_FALSE;
+
+	/* device error notify callback function */
+	gsDevices[0].pfnSysDevErrorNotify = NULL;
+
+	/* Virtualization support services needs to know which heap ID corresponds to FW */
+	PVR_ASSERT(ui32NextPhysHeapID < ARRAY_SIZE(gsPhysHeapConfig));
+	gsPhysHeapConfig[ui32NextPhysHeapID].pszPDumpMemspaceName = "SYSMEM";
+	gsPhysHeapConfig[ui32NextPhysHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	gsPhysHeapConfig[ui32NextPhysHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	gsPhysHeapConfig[ui32NextPhysHeapID].hPrivData = NULL;
+	gsPhysHeapConfig[ui32NextPhysHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+
+#if defined(SUPPORT_PDVFS)
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+
+	*ppsDevConfig = &gsDevices[0];
+
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	psDevConfig->pvOSDevice = NULL;
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	return OSInstallSystemLISR(phLISRData, ui32IRQ, pszName, pfnLISR, pvData,
+								SYS_IRQ_FLAG_TRIGGER_DEFAULT);
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return OSUninstallSystemLISR(hLISRData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.h
new file mode 100644
index 000000000000..db7625c698ea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(__SYSCCONFIG_H__)
+#define __SYSCCONFIG_H__
+
+#define VEXPRESS_IRQ_GPU 63
+#define VEXPRESS_GPU_SIZE 0x00010000
+#define VEXPRESS_GPU_PBASE 0x021B0000
+
+#define RGX_VEXPRESS_CORE_CLOCK_SPEED (320 * 1000 * 1000)
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (100)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* __SYSCCONFIG_H__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysinfo.h
new file mode 100644
index 000000000000..66acc666273b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysinfo.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM)
+#define MAX_HW_TIME_US								(5000000)
+#else
+#define MAX_HW_TIME_US								(500000)
+#endif
+
+#define FATAL_ERROR_DETECTION_POLL_MS				(10000)
+#if defined(VIRTUAL_PLATFORM)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT		(10000)
+#else
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT		(1500)//(10000)
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT	(3600000)
+#define WAIT_TRY_COUNT								(10000)
+
+
+#if defined(VIRTUAL_PLATFORM)
+#define SYS_RGX_OF_COMPATIBLE "img,gpu"
+#endif
+
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME "rgx_vexpress"
+#endif
+
+#endif	/* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/env/linux/dma_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/env/linux/dma_support.c
new file mode 100644
index 000000000000..c561d0622d5f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/env/linux/dma_support.c
@@ -0,0 +1,523 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/mm.h>
+#include <asm/page.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/vmalloc.h>
+#include <linux/dma-mapping.h>
+#include <asm-generic/getorder.h>
+
+#include "allocmem.h"
+#include "dma_support.h"
+#include "pvr_vmap.h"
+#include "kernel_compatibility.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 2
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+
+static void*
+SysDmaAcquireKernelAddress(struct page *psPage, IMG_UINT64 ui64Size, DMA_ALLOC *psDmaAlloc)
+{
+	IMG_BOOL bPageByPage = IMG_TRUE;
+	IMG_UINT32 uiIdx;
+	void *pvVirtAddr = NULL;
+	IMG_UINT32 ui32PgCount = (IMG_UINT32)(ui64Size >> OSGetPageShift());
+	PVRSRV_DEVICE_NODE *psDevNode = OSAllocZMemNoStats(sizeof(*psDevNode));
+	PVRSRV_DEVICE_CONFIG *psDevConfig = OSAllocZMemNoStats(sizeof(*psDevConfig));
+	struct page **pagearray = OSAllocZMemNoStats(ui32PgCount * sizeof(struct page *));
+	void *pvOSDevice = psDmaAlloc->pvOSDevice;
+#if defined(CONFIG_ARM64)
+	pgprot_t prot = pgprot_writecombine(PAGE_KERNEL);
+#else
+	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
+#endif
+
+	/* Validate all required dynamic tmp buffer allocations */
+	if (psDevNode == NULL || psDevConfig == NULL || pagearray == NULL)
+	{
+		if (psDevNode)
+		{
+			OSFreeMem(psDevNode);
+		}
+
+		if (psDevConfig)
+		{
+			OSFreeMem(psDevConfig);
+		}
+
+		if (pagearray)
+		{
+			OSFreeMem(pagearray);
+		}
+
+		goto e0;
+	}
+
+	/* Fake psDevNode->psDevConfig->pvOSDevice */
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevNode->psDevConfig = psDevConfig;
+
+	/* Evict any page data contents from d-cache */
+	for (uiIdx = 0; uiIdx < ui32PgCount; uiIdx++)
+	{
+		void *pvVirtStart, *pvVirtEnd;
+		IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+
+		/* Prepare array required for vmap */
+		pagearray[uiIdx] = &psPage[uiIdx];
+
+		if (bPageByPage)
+		{
+#if defined(CONFIG_64BIT)
+			bPageByPage = IMG_FALSE;
+
+			pvVirtStart = kmap(&psPage[uiIdx]);
+			pvVirtEnd = pvVirtStart + ui64Size;
+
+			sCPUPhysStart.uiAddr = page_to_phys(&psPage[uiIdx]);
+			sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + ui64Size;
+			/* all pages have a kernel linear address, flush entire range */
+#else
+			pvVirtStart = kmap(&psPage[uiIdx]);
+			pvVirtEnd = pvVirtStart + PAGE_SIZE;
+
+			sCPUPhysStart.uiAddr = page_to_phys(&psPage[uiIdx]);
+			sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+			/* pages might be from HIGHMEM, need to kmap/flush per page */
+#endif
+
+			/* Fallback to range-based d-cache flush */
+			OSCPUCacheInvalidateRangeKM(psDevNode,
+										pvVirtStart, pvVirtEnd,
+										sCPUPhysStart, sCPUPhysEnd);
+
+			kunmap(&psPage[uiIdx]);
+		}
+	}
+
+	/* Remap pages into VMALLOC space */
+	pvVirtAddr = pvr_vmap(pagearray, ui32PgCount, VM_MAP, prot);
+	psDmaAlloc->PageProps = prot;
+
+	/* Clean-up tmp buffers */
+	OSFreeMem(psDevConfig);
+	OSFreeMem(psDevNode);
+	OSFreeMem(pagearray);
+
+e0:
+	return pvVirtAddr;
+}
+
+static void SysDmaReleaseKernelAddress(void *pvVirtAddr, IMG_UINT64 ui64Size, pgprot_t pgprot)
+{
+	pvr_vunmap(pvVirtAddr, ui64Size >> OSGetPageShift(), pgprot);
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct device *psDev;
+	struct page *psPage;
+	size_t uiSize;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->hHandle ||
+		psDmaAlloc->pvVirtAddr ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr ||
+		psDmaAlloc->pvOSDevice == NULL)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+	psDev = (struct device *)psDmaAlloc->pvOSDevice;
+
+	psDmaAlloc->hHandle = dma_alloc_coherent(psDev, uiSize, (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr, GFP_KERNEL);
+
+	if (psDmaAlloc->hHandle)
+	{
+		psDmaAlloc->pvVirtAddr = psDmaAlloc->hHandle;
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"Allocated DMA buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
+				psDmaAlloc->pvVirtAddr,
+				psDmaAlloc->sBusAddr.uiAddr,
+				uiSize));
+	}
+	else if ((psPage = alloc_pages(GFP_KERNEL, get_order(uiSize))))
+	{
+		psDmaAlloc->sBusAddr.uiAddr = dma_map_page(psDev, psPage, 0, uiSize, DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(psDev, psDmaAlloc->sBusAddr.uiAddr))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"dma_map_page() failed, page 0x%p order %d",
+					psPage,
+					get_order(uiSize)));
+			__free_pages(psPage, get_order(uiSize));
+			goto e0;
+		}
+		psDmaAlloc->psPage = psPage;
+
+		psDmaAlloc->pvVirtAddr = SysDmaAcquireKernelAddress(psPage, uiSize, psDmaAlloc);
+		if (! psDmaAlloc->pvVirtAddr)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"SysDmaAcquireKernelAddress() failed, page 0x%p order %d",
+					psPage,
+					get_order(uiSize)));
+			dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
+			__free_pages(psPage, get_order(uiSize));
+			goto e0;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"Allocated contiguous buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
+				psDmaAlloc->pvVirtAddr,
+				psDmaAlloc->sBusAddr.uiAddr,
+				uiSize));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,	"Unable to allocate contiguous buffer, size: 0x"IMG_SIZE_FMTSPECX, uiSize));
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+	}
+
+e0:
+	PVR_LOG_RETURN_IF_FALSE((psDmaAlloc->pvVirtAddr), "DMA/CMA allocation failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	size_t uiSize;
+	struct device *psDev;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+	psDev = (struct device *)psDmaAlloc->pvOSDevice;
+
+	if (psDmaAlloc->pvVirtAddr != psDmaAlloc->hHandle)
+	{
+		SysDmaReleaseKernelAddress(psDmaAlloc->pvVirtAddr, uiSize, psDmaAlloc->PageProps);
+	}
+
+	if (! psDmaAlloc->hHandle)
+	{
+		struct page *psPage;
+		dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
+		psPage = psDmaAlloc->psPage;
+		__free_pages(psPage, get_order(uiSize));
+		return;
+	}
+
+	dma_free_coherent(psDev, uiSize, psDmaAlloc->hHandle, (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	size_t uiSize;
+	IMG_UINT32 ui32Idx;
+	IMG_BOOL bTabEntryFound = IMG_TRUE;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		/* Check if an I/O remap entry exists for remapping */
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		bTabEntryFound = IMG_FALSE;
+	}
+
+	if (bTabEntryFound)
+	{
+		IMG_BOOL bSameVAddr, bSamePAddr, bSameSize;
+
+		bSamePAddr = gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == psDmaAlloc->sBusAddr.uiAddr;
+		bSameVAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr;
+		bSameSize = gsDmaIoRemapArray[ui32Idx].ui64Size == uiSize;
+
+		if (bSameVAddr)
+		{
+			if (bSamePAddr && bSameSize)
+			{
+				eError = PVRSRV_OK;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_ALREADY_EXISTS;
+			}
+		}
+		else
+		{
+			PVR_ASSERT(bSamePAddr == IMG_FALSE);
+
+			gsDmaIoRemapArray[ui32Idx].ui64Size = uiSize;
+			gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: register I/O remap: "
+					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
+					psDmaAlloc->pvVirtAddr,
+					psDmaAlloc->sBusAddr.uiAddr,
+					uiSize));
+
+			gbEnableDmaIoRemapping = IMG_TRUE;
+			eError = PVRSRV_OK;
+		}
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	__maybe_unused size_t uiSize;
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+
+	/* Remove specified entries from list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr)
+		{
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+			gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: deregister I/O remap: "
+					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
+					psDmaAlloc->pvVirtAddr,
+					psDmaAlloc->sBusAddr.uiAddr,
+					uiSize));
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_stub.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_stub.c
new file mode 100644
index 000000000000..a2237281d703
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_stub.c
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_MESSAGE, "Using a stub VM manager type, no hypercall support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_vzfpga.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_vzfpga.c
new file mode 100644
index 000000000000..d379fa061cc1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_vzfpga.c
@@ -0,0 +1,170 @@
+/*************************************************************************/ /*!
+@File           vmm_type_vzfpga.c
+@Title          VZFPGA manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager implementation to support vzfpga platform
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+#include "rgxdevice.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+GetDriverIDFromHeapBase(IMG_UINT64 ui64Addr, IMG_UINT32 *pui32DriverID)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_DEV_PHYADDR sHostFwHeapBaseDevPA = {0};
+	PHYS_HEAP *psHostFwHeap = NULL;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetDeviceInstance(0);
+
+	PVR_LOG_RETURN_IF_FALSE((psHostDevNode != NULL),
+							"Host Device Node not initialised.",
+							PVRSRV_ERROR_NO_DEVICENODE_FOUND);
+
+	psHostFwHeap = psHostDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	PVR_LOG_RETURN_IF_FALSE((psHostFwHeap != NULL),
+							"Host Fw heap not initialised.",
+							PVRSRV_ERROR_PHYSHEAP_ID_INVALID);
+
+	eErr = PhysHeapGetDevPAddr(psHostFwHeap, &sHostFwHeapBaseDevPA);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PhysHeapGetDevPAddr");
+
+	*pui32DriverID = (ui64Addr - sHostFwHeapBaseDevPA.uiAddr) / PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+	PVR_LOG_RETURN_IF_FALSE((*pui32DriverID >= RGXFW_GUEST_DRIVER_ID_START) &&
+							(*pui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+							"Invalid Guest DriverID",
+							PVRSRV_ERROR_INVALID_PVZ_OSID);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+VZFPGAMapDevPhysHeap(IMG_UINT64 ui64Size,
+					 IMG_UINT64 ui64Addr)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT32 ui32GuestDriverID;
+
+	eErr = GetDriverIDFromHeapBase(ui64Addr, &ui32GuestDriverID);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "GetDriverIDFromHeapBase");
+
+	eErr = PvzServerOnVmOnline(ui32GuestDriverID, 0);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerOnVmOnline");
+
+	eErr = PvzServerMapDevPhysHeap(ui32GuestDriverID, 0, ui64Size, ui64Addr);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerMapDevPhysHeap");
+
+	return eErr;
+}
+
+static PVRSRV_ERROR
+VZFPGAUnmapDevPhysHeap(void)
+{
+	IMG_UINT32 ui32ID;
+
+	/* During shutdown, the Guests will be deinitialised in reverse order */
+	for (ui32ID=(RGX_NUM_DRIVERS_SUPPORTED-1);
+		 ui32ID >= RGXFW_HOST_DRIVER_ID; ui32ID--)
+	{
+		if (IsVmOnline(ui32ID, 0))
+		{
+			PvzServerUnmapDevPhysHeap(ui32ID, 0);
+			PvzServerOnVmOffline(ui32ID, 0);
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static VMM_PVZ_CONNECTION gsVZFPGAPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&VZFPGAMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&VZFPGAUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsVZFPGAPvz;
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_vzfpga.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/include/dma_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/include/dma_support.h
new file mode 100644
index 000000000000..c1d22bddf523
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/include/dma_support.h
@@ -0,0 +1,117 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	IMG_HANDLE       hHandle;
+#if defined(__linux__)
+	struct page      *psPage;
+	pgprot_t         PageProps;
+#endif
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/Kbuild.mk
new file mode 100644
index 000000000000..e5594c7feabe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/Kbuild.mk
@@ -0,0 +1,55 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+ifeq ($(KERNEL_DRIVER_DIR),)
+ SYSTEM_BASEDIR := services/system/rogue/$(PVR_SYSTEM)
+else
+ SYSTEM_BASEDIR := external/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM)
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ $(SYSTEM_BASEDIR)/mt8173_mfgsys.o \
+ $(SYSTEM_BASEDIR)/mt8173_sysconfig.o \
+ services/system/common/uma_heap_fns.o
+
+ifeq ($(SUPPORT_ION),1)
+ $(PVRSRV_MODNAME)-y += \
+  services/system/common/env/linux/ion_support_generic.o
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.c
new file mode 100644
index 000000000000..8aace9dae30a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.c
@@ -0,0 +1,329 @@
+/*
+* Copyright (c) 2014 MediaTek Inc.
+* Author: Chiawen Lee <chiawen.lee@mediatek.com>
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*/
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/thermal.h>
+
+#include "mt8173_mfgsys.h"
+
+static const char * const top_mfg_clk_name[] = {
+	"mfg_mem_in_sel",
+	"mfg_axi_in_sel",
+	"top_axi",
+	"top_mem",
+};
+
+#define MAX_TOP_MFG_CLK ARRAY_SIZE(top_mfg_clk_name)
+
+#define REG_MFG_AXI BIT(0)
+#define REG_MFG_MEM BIT(1)
+#define REG_MFG_G3D BIT(2)
+#define REG_MFG_26M BIT(3)
+#define REG_MFG_ALL (REG_MFG_AXI | REG_MFG_MEM | REG_MFG_G3D | REG_MFG_26M)
+
+#define REG_MFG_CG_STA 0x00
+#define REG_MFG_CG_SET 0x04
+#define REG_MFG_CG_CLR 0x08
+
+static void mtk_mfg_clr_clock_gating(void __iomem *reg)
+{
+	writel(REG_MFG_ALL, reg + REG_MFG_CG_CLR);
+}
+
+static int mtk_mfg_prepare_clock(struct mtk_mfg *mfg)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < MAX_TOP_MFG_CLK; i++) {
+		ret = clk_prepare(mfg->top_clk[i]);
+		if (ret)
+			goto unwind;
+	}
+	ret = clk_prepare(mfg->top_mfg);
+	if (ret)
+		goto unwind;
+
+	return 0;
+unwind:
+	while (i--)
+		clk_unprepare(mfg->top_clk[i]);
+
+	return ret;
+}
+
+static void mtk_mfg_unprepare_clock(struct mtk_mfg *mfg)
+{
+	int i;
+
+	clk_unprepare(mfg->top_mfg);
+	for (i = MAX_TOP_MFG_CLK - 1; i >= 0; i--)
+		clk_unprepare(mfg->top_clk[i]);
+}
+
+static int mtk_mfg_enable_clock(struct mtk_mfg *mfg)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < MAX_TOP_MFG_CLK; i++) {
+		ret = clk_enable(mfg->top_clk[i]);
+		if (ret)
+			goto unwind;
+	}
+	ret = clk_enable(mfg->top_mfg);
+	if (ret)
+		goto unwind;
+	mtk_mfg_clr_clock_gating(mfg->reg_base);
+
+	return 0;
+unwind:
+	while (i--)
+		clk_disable(mfg->top_clk[i]);
+
+	return ret;
+}
+
+static void mtk_mfg_disable_clock(struct mtk_mfg *mfg)
+{
+	int i;
+
+	clk_disable(mfg->top_mfg);
+	for (i = MAX_TOP_MFG_CLK - 1; i >= 0; i--)
+		clk_disable(mfg->top_clk[i]);
+}
+
+static void mtk_mfg_enable_hw_apm(struct mtk_mfg *mfg)
+{
+	writel(0x003c3d4d, mfg->reg_base + 0x24);
+	writel(0x4d45440b, mfg->reg_base + 0x28);
+	writel(0x7a710184, mfg->reg_base + 0xe0);
+	writel(0x835f6856, mfg->reg_base + 0xe4);
+	writel(0x002b0234, mfg->reg_base + 0xe8);
+	writel(0x80000000, mfg->reg_base + 0xec);
+	writel(0x08000000, mfg->reg_base + 0xa0);
+}
+
+int mtk_mfg_enable(struct mtk_mfg *mfg)
+{
+	int ret;
+
+	ret = pm_runtime_get_sync(mfg->dev);
+	if (ret < 0)
+		return ret;
+
+	ret = mtk_mfg_enable_clock(mfg);
+	if (ret)
+		goto err_pm_runtime_put;
+
+	mtk_mfg_enable_hw_apm(mfg);
+
+	return 0;
+
+err_pm_runtime_put:
+	pm_runtime_put_sync(mfg->dev);
+	return ret;
+}
+
+static void mtk_mfg_disable_hw_apm(struct mtk_mfg *mfg)
+{
+	writel(0x00, mfg->reg_base + 0xec);
+}
+
+void mtk_mfg_disable(struct mtk_mfg *mfg)
+{
+	mtk_mfg_disable_hw_apm(mfg);
+
+	mtk_mfg_disable_clock(mfg);
+	pm_runtime_put_sync(mfg->dev);
+}
+
+int mtk_mfg_freq_set(struct mtk_mfg *mfg, unsigned long freq)
+{
+	int ret;
+
+	ret = clk_prepare_enable(mfg->top_mfg);
+	if (ret) {
+		dev_err(mfg->dev, "enable and prepare top_mfg failed, %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_set_parent(mfg->top_mfg, mfg->clk26m);
+	if (ret) {
+		dev_err(mfg->dev, "Set clk parent to clk26m failed, %d\n", ret);
+		goto unprepare_top_mfg;
+	}
+
+	ret = clk_set_rate(mfg->mmpll, freq);
+	if (ret)
+		dev_err(mfg->dev, "Set freq to %lu Hz failed, %d\n", freq, ret);
+
+	ret = clk_set_parent(mfg->top_mfg, mfg->top_mmpll);
+	if (ret)
+		dev_err(mfg->dev, "Set clk parent to top_mmpll failed, %d\n", ret);
+
+unprepare_top_mfg:
+	clk_disable_unprepare(mfg->top_mfg);
+
+	if (!ret)
+		dev_dbg(mfg->dev, "Freq set to %lu Hz\n", freq);
+
+	return ret;
+}
+
+int mtk_mfg_volt_set(struct mtk_mfg *mfg, int volt)
+{
+	int ret;
+
+	ret = regulator_set_voltage(mfg->vgpu, volt, volt);
+	if (ret != 0) {
+		dev_err(mfg->dev, "Set voltage to %u uV failed, %d\n",
+			volt, ret);
+		return ret;
+	}
+
+	dev_dbg(mfg->dev, "Voltage set to %d uV\n", volt);
+
+	return 0;
+}
+
+static int mtk_mfg_bind_device_resource(struct mtk_mfg *mfg)
+{
+	struct device *dev = mfg->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	int i;
+	struct resource *res;
+
+	mfg->top_clk = devm_kcalloc(dev, MAX_TOP_MFG_CLK,
+				    sizeof(*mfg->top_clk), GFP_KERNEL);
+	if (!mfg->top_clk)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+	mfg->rgx_start = res->start;
+	mfg->rgx_size = resource_size(res);
+
+	mfg->rgx_irq = platform_get_irq_byname(pdev, "RGX");
+	if (mfg->rgx_irq < 0)
+		return mfg->rgx_irq;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	mfg->reg_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(mfg->reg_base))
+		return PTR_ERR(mfg->reg_base);
+
+	mfg->mmpll = devm_clk_get(dev, "mmpll_clk");
+	if (IS_ERR(mfg->mmpll)) {
+		dev_err(dev, "devm_clk_get mmpll_clk failed !!!\n");
+		return PTR_ERR(mfg->mmpll);
+	}
+
+	for (i = 0; i < MAX_TOP_MFG_CLK; i++) {
+		mfg->top_clk[i] = devm_clk_get(dev, top_mfg_clk_name[i]);
+		if (IS_ERR(mfg->top_clk[i])) {
+			dev_err(dev, "devm_clk_get %s failed !!!\n",
+				top_mfg_clk_name[i]);
+			return PTR_ERR(mfg->top_clk[i]);
+		}
+	}
+
+	mfg->top_mfg = devm_clk_get(dev, "top_mfg");
+	if (IS_ERR(mfg->top_mfg)) {
+		dev_err(dev, "devm_clk_get top_mfg failed !!!\n");
+		return PTR_ERR(mfg->top_mfg);
+	}
+
+	mfg->top_mmpll = devm_clk_get(dev, "top_mmpll");
+	if (IS_ERR(mfg->top_mmpll)) {
+		dev_err(dev, "devm_clk_get top_mmpll failed !!!\n");
+		return PTR_ERR(mfg->top_mmpll);
+	}
+
+	mfg->clk26m = devm_clk_get(dev, "clk26m");
+	if (IS_ERR(mfg->clk26m)) {
+		dev_err(dev, "devm_clk_get clk26m failed !!!\n");
+		return PTR_ERR(mfg->clk26m);
+	}
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	mfg->tz = thermal_zone_get_zone_by_name("cpu_thermal");
+	if (IS_ERR(mfg->tz)) {
+		dev_warn(dev, "Failed to get cpu_thermal zone\n");
+	}
+#endif
+
+	mfg->vgpu = devm_regulator_get(dev, "mfgsys-power");
+	if (IS_ERR(mfg->vgpu))
+		return PTR_ERR(mfg->vgpu);
+
+	pm_runtime_enable(dev);
+
+	return 0;
+}
+
+static void mtk_mfg_unbind_device_resource(struct mtk_mfg *mfg)
+{
+	struct device *dev = mfg->dev;
+
+	pm_runtime_disable(dev);
+}
+
+struct mtk_mfg *mtk_mfg_create(struct device *dev)
+{
+	int err;
+	struct mtk_mfg *mfg;
+
+	mtk_mfg_debug("mtk_mfg_create Begin\n");
+
+	mfg = devm_kzalloc(dev, sizeof(*mfg), GFP_KERNEL);
+	if (!mfg)
+		return ERR_PTR(-ENOMEM);
+	mfg->dev = dev;
+
+	err = mtk_mfg_bind_device_resource(mfg);
+	if (err != 0)
+		return ERR_PTR(err);
+
+	mutex_init(&mfg->set_power_state);
+
+	err = mtk_mfg_prepare_clock(mfg);
+	if (err)
+		goto err_unbind_resource;
+
+	mtk_mfg_debug("mtk_mfg_create End\n");
+
+	return mfg;
+err_unbind_resource:
+	mtk_mfg_unbind_device_resource(mfg);
+
+	return ERR_PTR(err);
+}
+
+void mtk_mfg_destroy(struct mtk_mfg *mfg)
+{
+	mtk_mfg_unprepare_clock(mfg);
+
+	mtk_mfg_unbind_device_resource(mfg);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.h
new file mode 100644
index 000000000000..c8f07e6f7a26
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.h
@@ -0,0 +1,63 @@
+/*
+* Copyright (c) 2014 MediaTek Inc.
+* Author: Chiawen Lee <chiawen.lee@mediatek.com>
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*/
+
+#ifndef MT8173_MFGSYS_H
+#define MT8173_MFGSYS_H
+
+#include <linux/device.h>
+
+/* unit ms, timeout interval for DVFS detection */
+#define MTK_DVFS_SWITCH_INTERVAL  300
+
+#define ENABLE_MTK_MFG_DEBUG 0
+
+#if ENABLE_MTK_MFG_DEBUG
+#define mtk_mfg_debug(fmt, args...) pr_info("[MFG]" fmt, ##args)
+#else
+#define mtk_mfg_debug(fmt, args...) do { } while (0)
+#endif
+
+struct mtk_mfg {
+	struct device *dev;
+
+	struct clk **top_clk;
+	void __iomem *reg_base;
+
+	resource_size_t rgx_start;
+	resource_size_t rgx_size;
+	int rgx_irq;
+
+	/* mutex protect for set power state */
+	struct mutex set_power_state;
+
+	/* for gpu device freq/volt update */
+	struct regulator *vgpu;
+	struct clk *mmpll;
+	struct clk *top_mfg;
+	struct clk *top_mmpll;
+	struct clk *clk26m;
+
+	struct thermal_zone_device *tz;
+};
+
+struct mtk_mfg *mtk_mfg_create(struct device *dev);
+void mtk_mfg_destroy(struct mtk_mfg *mfg);
+
+int mtk_mfg_enable(struct mtk_mfg *mfg);
+void mtk_mfg_disable(struct mtk_mfg *mfg);
+
+int mtk_mfg_freq_set(struct mtk_mfg *mfg, unsigned long freq);
+int mtk_mfg_volt_set(struct mtk_mfg *mfg, int volt);
+
+#endif /* MT8173_MFGSYS_H*/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_sysconfig.c
new file mode 100644
index 000000000000..58e4543670d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_sysconfig.c
@@ -0,0 +1,544 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/thermal.h>
+#if defined(CONFIG_DEVFREQ_THERMAL)
+#include <linux/devfreq_cooling.h>
+#endif
+
+#include "physheap.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#include "uma_heap_fns.h"
+
+#include "mt8173_mfgsys.h"
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS 10
+#define RGX_HW_CORE_CLOCK_SPEED 395000000
+#define MT8173_SYSTEM_NAME "mt8173"
+
+static IMG_HANDLE ghSysData;
+
+typedef struct
+{
+	IMG_UINT32 ui32IRQ;
+	PFN_LISR pfnLISR;
+	void *pvLISRData;
+} LISR_WRAPPER_DATA;
+
+static irqreturn_t MTKLISRWrapper(int iIrq, void *pvData)
+{
+	LISR_WRAPPER_DATA *psWrapperData = pvData;
+
+	if (psWrapperData->pfnLISR(psWrapperData->pvLISRData))
+	{
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static PVRSRV_ERROR MTKSysDevPrePowerState(
+		IMG_HANDLE hSysData,
+		PVRSRV_SYS_POWER_STATE eNewPowerState,
+		PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+		PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	struct mtk_mfg *mfg = hSysData;
+
+	mtk_mfg_debug("MTKSysDevPrePowerState (%d->%d), bPwrFlags = 0x%08x\n",
+		      eCurrentPowerState, eNewPowerState, ePwrFlags);
+
+	mutex_lock(&mfg->set_power_state);
+
+	if ((PVRSRV_SYS_POWER_STATE_OFF == eNewPowerState) &&
+	    (PVRSRV_SYS_POWER_STATE_ON == eCurrentPowerState))
+		mtk_mfg_disable(mfg);
+
+	mutex_unlock(&mfg->set_power_state);
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR MTKSysDevPostPowerState(
+		IMG_HANDLE hSysData,
+		PVRSRV_SYS_POWER_STATE eNewPowerState,
+		PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+		PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	struct mtk_mfg *mfg = hSysData;
+	PVRSRV_ERROR ret;
+
+	mtk_mfg_debug("MTKSysDevPostPowerState (%d->%d)\n",
+		      eCurrentPowerState, eNewPowerState);
+
+	mutex_lock(&mfg->set_power_state);
+
+	if ((PVRSRV_SYS_POWER_STATE_ON == eNewPowerState) &&
+	    (PVRSRV_SYS_POWER_STATE_OFF == eCurrentPowerState)) {
+		if (mtk_mfg_enable(mfg)) {
+			ret = PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
+			goto done;
+		}
+	}
+
+	ret = PVRSRV_OK;
+done:
+	mutex_unlock(&mfg->set_power_state);
+
+	return ret;
+}
+
+#ifdef SUPPORT_LINUX_DVFS
+#if defined(CONFIG_DEVFREQ_THERMAL)
+
+#define FALLBACK_STATIC_TEMPERATURE 65000
+
+/* Temperatures on power over-temp-and-voltage curve (C) */
+static const int vt_temperatures[] = { 25, 45, 65, 85, 105 };
+
+/* Voltages on power over-temp-and-voltage curve (mV) */
+static const int vt_voltages[] = { 900, 1000, 1130 };
+
+#define POWER_TABLE_NUM_TEMP ARRAY_SIZE(vt_temperatures)
+#define POWER_TABLE_NUM_VOLT ARRAY_SIZE(vt_voltages)
+
+static const unsigned int
+power_table[POWER_TABLE_NUM_VOLT][POWER_TABLE_NUM_TEMP] = {
+	/*   25     45      65      85     105 */
+	{ 14540, 35490,  60420, 120690, 230000 },  /*  900 mV */
+	{ 21570, 41910,  82380, 159140, 298620 },  /* 1000 mV */
+	{ 32320, 72950, 111320, 209290, 382700 },  /* 1130 mV */
+};
+
+/** Frequency and Power in Khz and mW respectively */
+static const int f_range[] = {253500, 299000, 396500, 455000, 494000, 598000};
+static const IMG_UINT32 max_dynamic_power[] = {612, 722, 957, 1100, 1194, 1445};
+
+static u32 interpolate(int value, const int *x, const unsigned int *y, int len)
+{
+	u64 tmp64;
+	u32 dx;
+	u32 dy;
+	int i, ret;
+
+	if (value <= x[0])
+		return y[0];
+	if (value >= x[len - 1])
+		return y[len - 1];
+
+	for (i = 1; i < len - 1; i++) {
+		/* If value is identical, no need to interpolate */
+		if (value == x[i])
+			return y[i];
+		if (value < x[i])
+			break;
+	}
+
+	/* Linear interpolation between the two (x,y) points */
+	dy = y[i] - y[i - 1];
+	dx = x[i] - x[i - 1];
+
+	tmp64 = value - x[i - 1];
+	tmp64 *= dy;
+	do_div(tmp64, dx);
+	ret = y[i - 1] + tmp64;
+
+	return ret;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+static unsigned long mtk_mfg_get_static_power(struct devfreq *df,
+					      unsigned long voltage)
+#else
+static unsigned long mtk_mfg_get_static_power(unsigned long voltage)
+#endif
+{
+	struct mtk_mfg *mfg = ghSysData;
+	struct thermal_zone_device *tz = mfg->tz;
+	unsigned long power;
+	int temperature = FALLBACK_STATIC_TEMPERATURE;
+	int low_idx = 0, high_idx = POWER_TABLE_NUM_VOLT - 1;
+	int i;
+
+	if (!tz)
+		return 0;
+
+	if (tz->ops->get_temp(tz, &temperature))
+		dev_warn(mfg->dev, "Failed to read temperature\n");
+	do_div(temperature, 1000);
+
+	for (i = 0; i < POWER_TABLE_NUM_VOLT; i++) {
+		if (voltage <= vt_voltages[POWER_TABLE_NUM_VOLT - 1 - i])
+			high_idx = POWER_TABLE_NUM_VOLT - 1 - i;
+
+		if (voltage >= vt_voltages[i])
+			low_idx = i;
+	}
+
+	if (low_idx == high_idx) {
+		power = interpolate(temperature,
+				    vt_temperatures,
+				    &power_table[low_idx][0],
+				    POWER_TABLE_NUM_TEMP);
+	} else {
+		unsigned long dvt =
+				vt_voltages[high_idx] - vt_voltages[low_idx];
+		unsigned long power1, power2;
+
+		power1 = interpolate(temperature,
+				     vt_temperatures,
+				     &power_table[high_idx][0],
+				     POWER_TABLE_NUM_TEMP);
+
+		power2 = interpolate(temperature,
+				     vt_temperatures,
+				     &power_table[low_idx][0],
+				     POWER_TABLE_NUM_TEMP);
+
+		power = (power1 - power2) * (voltage - vt_voltages[low_idx]);
+		do_div(power, dvt);
+		power += power2;
+	}
+
+	/* convert to mw */
+	do_div(power, 1000);
+
+	mtk_mfg_debug("mtk_mfg_get_static_power: %lu at Temperature %d\n",
+		      power, temperature);
+	return power;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+static unsigned long mtk_mfg_get_dynamic_power(struct devfreq *df,
+					       unsigned long freq,
+					       unsigned long voltage)
+#else
+static unsigned long mtk_mfg_get_dynamic_power(unsigned long freq,
+					       unsigned long voltage)
+#endif
+{
+	#define NUM_RANGE  ARRAY_SIZE(f_range)
+	/** Frequency and Power in Khz and mW respectively */
+	IMG_INT32 i, low_idx = 0, high_idx = NUM_RANGE - 1;
+	IMG_UINT32 power;
+
+	for (i = 0; i < NUM_RANGE; i++) {
+		if (freq <= f_range[NUM_RANGE - 1 - i])
+			high_idx = NUM_RANGE - 1 - i;
+
+		if (freq >= f_range[i])
+			low_idx = i;
+	}
+
+	if (low_idx == high_idx) {
+		power = max_dynamic_power[low_idx];
+	} else {
+		IMG_UINT32 f_interval = f_range[high_idx] - f_range[low_idx];
+		IMG_UINT32 p_interval = max_dynamic_power[high_idx] -
+				max_dynamic_power[low_idx];
+
+		power = p_interval * (freq - f_range[low_idx]);
+		do_div(power, f_interval);
+		power += max_dynamic_power[low_idx];
+	}
+
+	power = (IMG_UINT32)div_u64((IMG_UINT64)power * voltage * voltage,
+				    1000000UL);
+
+	return power;
+	#undef NUM_RANGE
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0))
+static int mtk_mfg_get_real_power(struct devfreq *df,
+						   u32 *power,
+					       unsigned long freq,
+					       unsigned long voltage)
+{
+	if (!df || !power)
+		return -EINVAL;
+
+	*power = mtk_mfg_get_static_power(df, voltage) +
+		     mtk_mfg_get_dynamic_power(df, freq, voltage);
+
+	return 0;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) */
+
+static struct devfreq_cooling_power sPowerOps = {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	.get_static_power = mtk_mfg_get_static_power,
+	.get_dynamic_power = mtk_mfg_get_dynamic_power,
+#else
+	.get_real_power = mtk_mfg_get_real_power,
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+};
+#endif
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 freq)
+{
+	struct mtk_mfg *mfg = hSysData;
+
+	/* freq is in Hz */
+	mtk_mfg_freq_set(mfg, freq);
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 volt)
+{
+	struct mtk_mfg *mfg = hSysData;
+
+	mtk_mfg_volt_set(mfg, volt);
+}
+#endif
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   struct mtk_mfg *mfg,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *psPhysHeapConfig;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo) +
+							  sizeof(*psPhysHeapConfig));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+	psPhysHeapConfig = (PHYS_HEAP_CONFIG *)((IMG_CHAR *)psRGXTimingInfo + sizeof(*psRGXTimingInfo));
+
+	/* Set up the RGX timing information */
+	psRGXTimingInfo->ui32CoreClockSpeed = RGX_HW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM = IMG_TRUE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_TRUE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	psPhysHeapConfig->eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeapConfig->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeapConfig->uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeapConfig->uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	psPhysHeapConfig->uConfig.sUMA.pszHeapName = "uma_gpu_local";
+	psPhysHeapConfig->uConfig.sUMA.hPrivData = NULL;
+
+	psDevConfig->pasPhysHeaps = psPhysHeapConfig;
+	psDevConfig->ui32PhysHeapCount = 1U;
+
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevConfig->pszName = MT8173_SYSTEM_NAME;
+	psDevConfig->pszVersion = NULL;
+
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+	psDevConfig->bDevicePA0IsValid = IMG_FALSE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = (IMG_HANDLE) mfg;
+	ghSysData = psDevConfig->hSysData;
+
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	psDevConfig->ui32IRQ = mfg->rgx_irq;
+
+	psDevConfig->sRegsCpuPBase.uiAddr = mfg->rgx_start;
+	psDevConfig->ui32RegsSize = mfg->rgx_size;
+
+#ifdef SUPPORT_LINUX_DVFS
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = MTK_DVFS_SWITCH_INTERVAL;
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.psPowerOps = &sPowerOps;
+#endif
+
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	/* power management on HW system */
+	psDevConfig->pfnPrePowerState = MTKSysDevPrePowerState;
+	psDevConfig->pfnPostPowerState = MTKSysDevPostPowerState;
+
+	/* clock frequency */
+	psDevConfig->pfnClockFreqGet = NULL;
+
+	/* device error notify callback function */
+	psDevConfig->pfnSysDevErrorNotify = NULL;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	/*
+	 * The device config, RGX data and RGX timing info are part of the same
+	 * allocation so do only one free.
+	 */
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	struct device *dev = pvOSDevice;
+	struct mtk_mfg *mfg;
+	PVRSRV_ERROR eError;
+
+	mfg = mtk_mfg_create(dev);
+	if (IS_ERR(mfg)) {
+		if (PTR_ERR(mfg) == -EPROBE_DEFER)
+			return PVRSRV_ERROR_PROBE_DEFER;
+		else
+			return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	dma_set_mask(dev, DMA_BIT_MASK(33));
+
+	eError = DeviceConfigCreate(pvOSDevice, mfg, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		mtk_mfg_destroy(mfg);
+
+		return eError;
+	}
+
+	*ppsDevConfig = psDevConfig;
+
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	struct mtk_mfg *mfg = psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	mtk_mfg_destroy(mfg);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	LISR_WRAPPER_DATA *psWrapperData;
+
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	psWrapperData = OSAllocMem(sizeof(*psWrapperData));
+	if (!psWrapperData)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psWrapperData->ui32IRQ = ui32IRQ;
+	psWrapperData->pfnLISR = pfnLISR;
+	psWrapperData->pvLISRData = pvData;
+
+	if (request_irq(ui32IRQ, MTKLISRWrapper, IRQF_TRIGGER_LOW, pszName,
+					psWrapperData))
+	{
+		OSFreeMem(psWrapperData);
+
+		return PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER;
+	}
+
+	*phLISRData = (IMG_HANDLE) psWrapperData;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_WRAPPER_DATA *psWrapperData = hLISRData;
+
+	free_irq(psWrapperData->ui32IRQ, psWrapperData);
+
+	OSFreeMem(psWrapperData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+			  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			  void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/sysinfo.h
new file mode 100644
index 000000000000..bd7bc38e3412
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/sysinfo.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (1000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (20000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#define SYS_RGX_OF_COMPATIBLE "mediatek,mt8173-gpu"
+
+#endif	/* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/Kbuild.mk
new file mode 100644
index 000000000000..55d0369bcc6d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/Kbuild.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+	services/system/rogue/$(PVR_SYSTEM)/sysconfig.o \
+	services/system/common/sysconfig_cmn.o
+
+ccflags-y += \
+	-I$(TOP)/services/system/rogue/common/env/linux -I$(TOP)/services/system/rogue/common/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc \
+	-I$(TOP)/include/rogue/system/rgx_tc -I$(TOP)/include/system/rgx_tc
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysconfig.c
new file mode 100644
index 000000000000..16191d7de281
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysconfig.c
@@ -0,0 +1,1740 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwutils.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_support.h"
+#include "ion_sys.h"
+#endif
+
+#include "vmm_pvz_server.h"
+#include "pvr_bridge_k.h"
+#include "pvr_drv.h"
+#include "tc_drv.h"
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+
+#define SECURE_FW_MEM_SIZE (0x400000)  /*  4MB */
+#define SECURE_MEM_SIZE    (0x4000000) /* 64MB */
+
+typedef struct
+{
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;
+	IMG_UINT64 uiSize;
+	IMG_BOOL bUsed;
+} CARD_PHYS_HEAP_CONFIG_SPEC;
+
+#define HEAP_SPEC_IDX_GPU_PRIVATE (0U)
+#define HEAP_SPEC_IDX_GPU_LOCAL   (1U)
+
+static const CARD_PHYS_HEAP_CONFIG_SPEC gasCardHeapTemplate[] =
+{
+	{
+	 PHYS_HEAP_USAGE_GPU_PRIVATE,
+	 0,					/* determined at runtime by apphints */
+	 false				/* determined at runtime by apphints */
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_LOCAL,
+	 0,					/* determined at runtime */
+	 true
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_SECURE,
+	 SECURE_MEM_SIZE,
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	 true
+#else
+	 false
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PRIVATE,
+	 SECURE_FW_MEM_SIZE,
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	 true
+#else
+	 false
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_SHARED,
+#if defined(SUPPORT_SECURITY_VALIDATION) && defined(RGX_PREMAP_FW_HEAPS)
+	 /* simultaneous virtualisation and security support requires premapped heaps,
+	  * i.e. FW_PRIVATE and FW_SHARED must fit contiguously into Fw's VA heap (RGX_FIRMWARE_RAW_HEAP_SIZE) */
+	 RGX_FIRMWARE_RAW_HEAP_SIZE - SECURE_FW_MEM_SIZE,
+#else
+	 RGX_FIRMWARE_RAW_HEAP_SIZE,
+#endif
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	true				/* VZ drivers need dedicated Fw heaps */
+#else
+	false				/* Native drivers can fallback on GPU_LOCAL for Fw mem */
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PREMAP_PT,
+	 RGX_FIRMWARE_MAX_PAGETABLE_SIZE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 true
+#else
+	 false
+#endif
+	}
+};
+
+#define ODIN_MEMORY_HYBRID_DEVICE_BASE 0x400000000
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+#define UI64_TOPWORD_IS_ZERO(ui64) ((ui64 >> 32) == 0)
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Voltage)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+
+#endif
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCLocalDevPAddrToCpuPAddr,
+};
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHostCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHostDevPAddrToCpuPAddr,
+};
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHybridCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHybridDevPAddrToCpuPAddr
+};
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	struct platform_device *pdev;
+
+	struct tc_rogue_platform_data *pdata;
+
+	struct resource *registers;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_client *ion_client;
+	struct ion_handle *ion_rogue_allocation;
+#endif
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	void *pvS3Buffer;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+};
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	int err;
+	char str_fpga_rev[FPGA_REV_MAX_LEN]={0};
+	char str_tcf_core_rev[TCF_CORE_REV_MAX_LEN]={0};
+	char str_tcf_core_target_build_id[TCF_CORE_CFG_MAX_LEN]={0};
+	char str_pci_ver[PCI_VERSION_MAX_LEN]={0};
+	char str_macro_ver[MACRO_VERSION_MAX_LEN]={0};
+
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+
+	err = tc_sys_strings(psSysData->pdev->dev.parent,
+							 str_fpga_rev, sizeof(str_fpga_rev),
+							 str_tcf_core_rev, sizeof(str_tcf_core_rev),
+							 str_tcf_core_target_build_id, sizeof(str_tcf_core_target_build_id),
+							 str_pci_ver, sizeof(str_pci_ver),
+							 str_macro_ver, sizeof(str_macro_ver));
+	if (err)
+	{
+		return NULL;
+	}
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(str_fpga_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_target_build_id);
+	ui32StringLength += OSStringLength(str_pci_ver);
+	ui32StringLength += OSStringLength(str_macro_ver);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+				   SYSTEM_INFO_FORMAT_STRING,
+				   str_fpga_rev,
+				   str_tcf_core_rev,
+				   str_tcf_core_target_build_id,
+				   str_pci_ver,
+				   str_macro_ver);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+static SYS_DATA *gpsIonPrivateData;
+
+PVRSRV_ERROR IonInit(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYS_DATA *psSysData = pvPrivateData;
+	gpsIonPrivateData = psSysData;
+
+	psSysData->ion_client = ion_client_create(psSysData->pdata->ion_device, SYS_RGX_DEV_NAME);
+	if (IS_ERR(psSysData->ion_client))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create ION client (%ld)", __func__, PTR_ERR(psSysData->ion_client)));
+		eError = PVRSRV_ERROR_ION_NO_CLIENT;
+		goto err_out;
+	}
+	/* Allocate the whole rogue ion heap and pass that to services to manage */
+	psSysData->ion_rogue_allocation = ion_alloc(psSysData->ion_client, psSysData->pdata->rogue_heap_memory_size, 4096, (1 << psSysData->pdata->ion_heap_id), 0);
+	if (IS_ERR(psSysData->ion_rogue_allocation))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate ION rogue buffer (%ld)", __func__, PTR_ERR(psSysData->ion_rogue_allocation)));
+		eError = PVRSRV_ERROR_ION_FAILED_TO_ALLOC;
+		goto err_destroy_client;
+
+	}
+
+	return PVRSRV_OK;
+err_destroy_client:
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+err_out:
+	return eError;
+}
+
+void IonDeinit(void)
+{
+	SYS_DATA *psSysData = gpsIonPrivateData;
+	ion_free(psSysData->ion_client, psSysData->ion_rogue_allocation);
+	psSysData->ion_rogue_allocation = NULL;
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+}
+
+struct ion_device *IonDevAcquire(void)
+{
+	return gpsIonPrivateData->pdata->ion_device;
+}
+
+void IonDevRelease(struct ion_device *ion_device)
+{
+	PVR_ASSERT(ion_device == gpsIonPrivateData->pdata->ion_device);
+}
+#endif /* defined(SUPPORT_ION) */
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+			psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+			psDevPAddr[ui32Idx].uiAddr + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	if (sizeof(*psDevPAddr) == sizeof(*psCpuPAddr))
+	{
+		OSCachedMemCopy(psDevPAddr, psCpuPAddr, uiNumOfAddr * sizeof(*psDevPAddr));
+		return;
+	}
+
+	/* In this case we may have a 32bit host, so we can't do a memcpy */
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (uiNumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < uiNumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	if (sizeof(*psCpuPAddr) == sizeof(*psDevPAddr))
+	{
+		OSCachedMemCopy(psCpuPAddr, psDevPAddr, uiNumOfAddr * sizeof(*psCpuPAddr));
+		return;
+	}
+
+	/* In this case we may have a 32bit host, so we can't do a memcpy.
+	 * Check we are not dropping any data from the 64bit dev addr */
+	PVR_ASSERT(UI64_TOPWORD_IS_ZERO(psDevPAddr[0].uiAddr));
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (uiNumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < uiNumOfAddr; ++ui32Idx)
+		{
+			PVR_ASSERT(UI64_TOPWORD_IS_ZERO(psDevPAddr[ui32Idx].uiAddr));
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+
+}
+
+static inline
+IMG_CHAR* GetHeapName(PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_LOCAL))    return "lma_gpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_SECURE))   return "lma_gpu_secure";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_PRIVATE))  return "lma_gpu_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PRIVATE))   return "lma_fw_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_SHARED))    return "lma_fw_shared";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PREMAP_PT)) return "lma_fw_pagetables";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_CPU_LOCAL))    return "lma_cpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_DISPLAY))      return "lma_gpu_display";
+	else                                                     return "Unexpected Heap";
+}
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    (psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base) +
+		    ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    (psDevPAddr[ui32Idx].uiAddr - ODIN_MEMORY_HYBRID_DEVICE_BASE) +
+		    psSysData->pdata->tc_memory_base;
+	}
+}
+
+static PVRSRV_ERROR
+InitLocalHeap(PHYS_HEAP_CONFIG *psPhysHeap,
+			  IMG_UINT64 uiBaseAddr, IMG_UINT64 uiStartAddr,
+			  IMG_UINT64 uiSize, PHYS_HEAP_FUNCTIONS *psFuncs,
+			  PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = ui32Flags;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = psFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = GetHeapName(ui32Flags);
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = uiBaseAddr;
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(uiStartAddr);
+	psPhysHeap->uConfig.sLMA.uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardGPUHeaps(const SYS_DATA *psSysData,
+				   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+				   PHYS_HEAP_CONFIG *pasPhysHeaps,
+				   PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				   IMG_UINT32 *pui32HeapIdx,
+				   IMG_UINT64 ui64CardAddr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->rogue_heap_memory_base;
+	IMG_UINT32 ui32SpecIdx;
+
+	for (ui32SpecIdx = 0; ui32SpecIdx < ARRAY_SIZE(gasCardHeapTemplate); ui32SpecIdx++)
+	{
+		if (pasCardHeapSpec[ui32SpecIdx].bUsed)
+		{
+			IMG_UINT64 ui64HeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+
+			eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+								   ui64CardAddr,
+								   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+								   ui64HeapSize,
+								   psHeapFuncs,
+								   pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			ui64CardAddr  += pasCardHeapSpec[ui32SpecIdx].uiSize;
+			ui64StartAddr += pasCardHeapSpec[ui32SpecIdx].uiSize;
+			(*pui32HeapIdx)++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+static PVRSRV_ERROR
+CreateCardEXTHeap(const SYS_DATA *psSysData,
+				  PHYS_HEAP_CONFIG *pasPhysHeaps,
+				  PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				  IMG_UINT32 *pui32HeapIdx,
+				  IMG_UINT64 ui64CardBase)
+{
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->pdp_heap_memory_base;
+	IMG_UINT64 ui64Size = psSysData->pdata->pdp_heap_memory_size;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+						   ui64CardBase + psSysData->pdata->rogue_heap_memory_size,
+						   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+						   ui64Size, psHeapFuncs,
+						   PHYS_HEAP_USAGE_EXTERNAL | PHYS_HEAP_USAGE_DISPLAY);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	(*pui32HeapIdx)++;
+
+	return PVRSRV_OK;
+}
+#endif
+
+static PVRSRV_ERROR
+InitLocalHeaps(const SYS_DATA *psSysData,
+			   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			   PHYS_HEAP_CONFIG *pasPhysHeaps,
+			   IMG_UINT32 *pui32HeapIdx)
+{
+	PHYS_HEAP_FUNCTIONS *psHeapFuncs;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64CardBase;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHybridPhysHeapFuncs;
+		ui64CardBase = ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+	else if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHostPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->tc_memory_base;
+	}
+	else
+	{
+		psHeapFuncs = &gsLocalPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->rogue_heap_memory_base - psSysData->pdata->tc_memory_base;
+	}
+
+	eError = CreateCardGPUHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+	eError = CreateCardEXTHeap(psSysData, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitHostHeaps(const SYS_DATA *psSysData, PHYS_HEAP_CONFIG *pasPhysHeaps, IMG_UINT32 *pui32HeapIdx)
+{
+	if (psSysData->pdata->mem_mode != TC_MEMORY_LOCAL)
+	{
+		pasPhysHeaps[*pui32HeapIdx].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[*pui32HeapIdx].ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszHeapName = "uma_cpu_local";
+
+		(*pui32HeapIdx)++;
+
+		PVR_DPF((PVR_DBG_WARNING,
+				 "Initialising CPU_LOCAL UMA Host PhysHeaps with memory mode: %d",
+				 psSysData->pdata->mem_mode));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsInit(const SYS_DATA *psSysData,
+			  CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			  PHYS_HEAP_CONFIG *pasPhysHeaps,
+			  void *pvPrivData, IMG_UINT32 ui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32HeapCounter = 0;
+
+	eError = InitLocalHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = InitHostHeaps(psSysData, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((ui32HeapCounter == ui32NumHeaps),
+							"Number of PhysHeapConfigs set up doesn't match the initial requirement.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	/* Initialise fields that don't change between memory modes.
+	 * Fix up heap IDs. This is needed for multi-testchip systems to
+	 * ensure the heap IDs are unique as this is what Services expects.
+	 */
+	for (i = 0; i < ui32NumHeaps; i++)
+	{
+		switch (pasPhysHeaps[i].eType)
+		{
+		case PHYS_HEAP_TYPE_UMA:
+			pasPhysHeaps[i].uConfig.sUMA.hPrivData = pvPrivData;
+			break;
+		case PHYS_HEAP_TYPE_LMA:
+			pasPhysHeaps[i].uConfig.sLMA.hPrivData = pvPrivData;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Invalid PHYS_HEAP_TYPE: %u in %s",
+			                        pasPhysHeaps[i].eType,
+			                        __func__));
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapSetRequirements(const SYS_DATA *psSysData,
+						CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+						IMG_UINT32 *pui32CardPhysHeapCfgCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64FreeCardMemory = psSysData->pdata->rogue_heap_memory_size;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_PRIVATE) &&
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_LOCAL),
+		"PhysHeapConfigs not set correctly in the system layer.", PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < ARRAY_SIZE(gasCardHeapTemplate); i++)
+	{
+		if (pasCardHeapSpec[i].bUsed)
+		{
+			/* Determine the memory requirements of heaps with a fixed size */
+			ui64FreeCardMemory -= pasCardHeapSpec[i].uiSize;
+
+			/* Count card physheap configs used by the system */
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	if (SysRestrictGpuLocalAddPrivateHeap())
+	{
+		IMG_UINT64 ui64GpuSharedMem = SysRestrictGpuLocalPhysheap(ui64FreeCardMemory);
+
+		if (ui64GpuSharedMem == ui64FreeCardMemory)
+		{
+			/* No memory reserved for GPU private use, special heap not needed */
+		}
+		else
+		{
+			/* Set up the GPU private heap */
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed = true;
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].uiSize = ui64FreeCardMemory - ui64GpuSharedMem;
+			ui64FreeCardMemory = ui64GpuSharedMem;
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	/* all remaining memory card memory goes to GPU_LOCAL */
+	pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].uiSize = ui64FreeCardMemory;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsCreate(const SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevConfig,
+				PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+				IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps = 0;
+	CARD_PHYS_HEAP_CONFIG_SPEC asCardHeapSpec[ARRAY_SIZE(gasCardHeapTemplate)];
+
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->mem_mode == TC_MEMORY_LOCAL) ||
+							(psSysData->pdata->mem_mode == TC_MEMORY_HYBRID),
+							"Unsupported memory mode", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	/* Initialise the local heap specs with the build-time template */
+	memcpy(asCardHeapSpec, gasCardHeapTemplate, sizeof(gasCardHeapTemplate));
+
+	eError = PhysHeapSetRequirements(psSysData, asCardHeapSpec, &ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psDevConfig->bHasNonMappableLocalMemory = asCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed;
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* CPU_LOCAL heap */
+		ui32NumHeaps++;
+	}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL ||
+		psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* EXTERNAL / DISPLAY heap */
+		ui32NumHeaps++;
+	}
+#endif
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * ui32NumHeaps);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = PhysHeapsInit(psSysData, asCardHeapSpec, pasPhysHeaps, psDevConfig, ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = ui32NumHeaps;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	OSFreeMem(psDevConfig);
+}
+
+static void odinTCDevPhysAddr2DmaAddr(PVRSRV_DEVICE_CONFIG *psDevConfig,
+									  IMG_DMA_ADDR *psDmaAddr,
+									  IMG_DEV_PHYADDR *psDevPAddr,
+									  IMG_BOOL *pbValid,
+									  IMG_UINT32 ui32NumAddr,
+									  IMG_BOOL bSparseAlloc)
+{
+	IMG_CPU_PHYADDR sCpuPAddr = {0};
+	IMG_UINT32 ui32Idx;
+
+	/* Fast path */
+	if (!bSparseAlloc)
+	{
+		/* In Odin, DMA address space is the same as host CPU */
+		TCLocalDevPAddrToCpuPAddr(psDevConfig,
+								  1,
+								  &sCpuPAddr,
+								  psDevPAddr);
+		psDmaAddr->uiAddr = sCpuPAddr.uiAddr;
+	}
+	else
+	{
+		for (ui32Idx = 0; ui32Idx < ui32NumAddr; ui32Idx++)
+		{
+			if (pbValid[ui32Idx])
+			{
+				TCLocalDevPAddrToCpuPAddr(psDevConfig,
+										  1,
+										  &sCpuPAddr,
+										  &psDevPAddr[ui32Idx]);
+				psDmaAddr[ui32Idx].uiAddr = sCpuPAddr.uiAddr;
+			}
+			else
+			{
+				/* Invalid DMA address marker */
+				psDmaAddr[ui32Idx].uiAddr = ~((IMG_UINT64)0x0);
+			}
+		}
+	}
+}
+
+static void * odinTCgetCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig, char *name)
+{
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	return tc_dma_chan(psDev->parent, name);
+}
+
+static void odinTCFreeCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   void* channel)
+{
+
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	struct dma_chan *chan = (struct dma_chan*) channel;
+
+	tc_dma_chan_free(psDev->parent, chan);
+}
+
+static void GetDriverMode(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DeviceID;
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	if (PVRSRVAcquireInternalID(&ui32DeviceID) != PVRSRV_OK)
+	{
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		return;
+	}
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		void __iomem *pvRegBase;
+
+		pvRegBase = (void __iomem *) OSMapPhysToLin(psDevConfig->sRegsCpuPBase, psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (pvRegBase == NULL)
+		{
+			/* failed to map register bank, default to native mode */
+			psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		}
+		else
+		{
+			IMG_UINT64 ui64ClkCtrl;
+
+			/* the CLK_CTRL register is valid only in the Os 0 (Host) register bank
+			 * if it reads 0 then we can conclude this Os is set up to run as Guest */
+#if defined(RGX_CR_CLK_CTRL)
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL);
+#else
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL1);
+#endif
+			OSUnMapPhysToLin((void __force *) pvRegBase, psDevConfig->ui32RegsSize);
+
+			psDevConfig->eDriverMode = (ui64ClkCtrl != 0) ? (DRIVER_MODE_HOST) : (DRIVER_MODE_GUEST);
+		}
+#else
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+							 psSysData->pdata->mem_mode == TC_MEMORY_LOCAL),
+							"Multidevice virtualization setup supported only on Odin device with TC_MEMORY_LOCAL",
+							PVRSRV_ERROR_INVALID_DEVICE);
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *) IMG_OFFSET_ADDR(psDevConfig, sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *) IMG_OFFSET_ADDR(psRGXData, sizeof(*psRGXData));
+
+	/* Setup RGX specific timing data */
+#if defined(TC_APOLLO_BONNIE)
+	/* For BonnieTC there seems to be an additional 5x multiplier that occurs to the clock as measured speed is 540Mhz not 108Mhz. */
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent)  * 6 * 5;
+#elif defined(TC_APOLLO_ES2)
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent)  * 6;
+#else
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent) /
+											tc_core_clock_multiplex(psSysData->pdev->dev.parent);
+#endif
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice = &psSysData->pdev->dev;
+	psDevConfig->pszName = "tc";
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = resource_size(psSysData->registers);
+
+	PVRSRVAcquireInternalID(&ui32DeviceID);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* Rogue FPGA images are correctly routing the OSID interrupts
+	 * for cores with the IRQ_PER_OS feature */
+	psDevConfig->ui32IRQ = TC_INTERRUPT_OSID0 + ui32DeviceID;
+#else
+	psDevConfig->ui32IRQ = TC_INTERRUPT_EXT;
+#endif
+
+	GetDriverMode(psDevConfig);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* If there is device running in native mode, prevent any attempts at
+	 * creating any Guest devices, as there will be no Host to support them.
+	 * Currently the VZFPGA supports only one physical GPU. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+		PVRSRV_DEVICE_NODE *psDN;
+
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		for (psDN = psPVRSRVData->psDeviceNodeList; psDN != NULL; psDN = psDN->psNext)
+		{
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDN))
+			{
+				OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+				PVR_DPF((PVR_DBG_ERROR, "%s() Device %u is already running in native mode, no other Guests supported in the system.",  __func__, psDN->sDevId.ui32InternalID));
+				eError = PVRSRV_ERROR_INVALID_DEVICE;
+				goto ErrorFreeDevConfig;
+			}
+		}
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+#endif
+
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &pasPhysHeaps, &uiPhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psDevConfig->eDefaultHeap = SysDefaultToCpuLocalHeap() ?
+		    PVRSRV_PHYS_HEAP_CPU_LOCAL : PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+	else
+	{
+		psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+
+	psDevConfig->pasPhysHeaps = pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount = uiPhysHeapCount;
+
+	/* Only required for LMA but having this always set shouldn't be a problem */
+	psDevConfig->bDevicePA0IsValid = IMG_TRUE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+#if defined(SUPPORT_ALT_REGBASE)
+	if (psSysData->pdata->mem_mode != TC_MEMORY_LOCAL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: alternative GPU register base is "
+					"supported only in LMA mode", __func__));
+		goto ErrorFreeDevConfig;
+	}
+	else
+	{
+		/* Using display memory base as the alternative GPU register base,
+		 * since the display memory range is not used by the firmware. */
+		IMG_CPU_PHYADDR  sDisplayMemAddr;
+
+		sDisplayMemAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psSysData->pdata->pdp_heap_memory_base);
+		TCLocalCpuPAddrToDevPAddr(psDevConfig, 1,
+		                          &psDevConfig->sAltRegsGpuPBase,
+		                          &sDisplayMemAddr);
+	}
+#endif
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	/* DMA channel config */
+	psDevConfig->pfnSlaveDMAGetChan = odinTCgetCDMAChan;
+	psDevConfig->pfnSlaveDMAFreeChan = odinTCFreeCDMAChan;
+	psDevConfig->pfnDevPhysAddr2DmaAddr = odinTCDevPhysAddr2DmaAddr;
+	psDevConfig->pszDmaTxChanName = psSysData->pdata->tc_dma_tx_chan_name;
+	psDevConfig->pszDmaRxChanName = psSysData->pdata->tc_dma_rx_chan_name;
+	psDevConfig->bHasDma = IMG_TRUE;
+	/* Following two values are expressed in number of bytes */
+	psDevConfig->ui32DmaTransferUnit = 1;
+	psDevConfig->ui32DmaAlignment = 1;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+/* #define _DBG(...) PVR_LOG((__VA_ARGS__)) */
+#define _DBG(...)
+
+static PVRSRV_ERROR PrePower(IMG_HANDLE hSysData,
+                             PVRSRV_SYS_POWER_STATE eNewPowerState,
+                             PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                             PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize;
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags: 0x%08x", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags);
+
+	/* The transition might be both from ON or OFF states to OFF state so check
+	 * only for the *new* state. Also this is only valid for suspend requests. */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = LMA_HeapIteratorCreate(psSysData->psDevConfig->psDevNode,
+	                                PVRSRV_PHYS_HEAP_GPU_LOCAL,
+	                                &psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorCreate", return_error);
+
+	eError = LMA_HeapIteratorGetHeapStats(psSysData->psHeapIter, &uiHeapTotalSize,
+	                                      &uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorGetHeapStats",
+	                      return_error);
+	uiHeapFreeSize = uiHeapTotalSize - uiHeapUsedSize;
+
+	_DBG("(%s()) heap stats: total=0x%" IMG_UINT64_FMTSPECx ", "
+	     "used=0x%" IMG_UINT64_FMTSPECx ", free=0x%" IMG_UINT64_FMTSPECx,
+	     __func__, uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize);
+
+	psSysData->pvS3Buffer = OSAllocMem(uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_NOMEM(psSysData->pvS3Buffer, eError, destroy_iterator);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		if (uiOffset + uiSize > uiHeapUsedSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "uiOffset = %" IMG_UINT64_FMTSPECx ", "
+			        "uiSize = %" IMG_UINT64_FMTSPECx, uiOffset, uiSize));
+
+			PVR_LOG_GOTO_WITH_ERROR("LMA_HeapIteratorNext", eError,
+			                        PVRSRV_ERROR_INVALID_OFFSET,
+			                        free_buffer);
+		}
+
+		TCLocalDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                          &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy((IMG_BYTE *) psSysData->pvS3Buffer + uiOffset, pvCpuVAddr,
+		       uiSize);
+		/* and now poison it */
+		memset(pvCpuVAddr, 0x9b, uiSize);
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+destroy_iterator:
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+return_error:
+	return eError;
+}
+
+static PVRSRV_ERROR PostPower(IMG_HANDLE hSysData,
+                              PVRSRV_SYS_POWER_STATE eNewPowerState,
+                              PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags=0x%08x; buffer=%px", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags,
+	     psSysData->pvS3Buffer);
+
+	/* The transition might be both to ON or OFF states from OFF state so check
+	 * only for the *current* state. Also this is only valid for resume requests. */
+	if (eCurrentPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ) ||
+	    psSysData->pvS3Buffer == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = LMA_HeapIteratorReset(psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorReset", free_buffer);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		TCLocalDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                          &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy(pvCpuVAddr, (IMG_BYTE *) psSysData->pvS3Buffer + uiOffset,
+		       uiSize);
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+
+	return eError;
+}
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	resource_size_t uiRegistersSize;
+	IMG_UINT32 ui32MinRegBankSize;
+	PVRSRV_ERROR eError;
+	int err = 0;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->pdev = to_platform_device((struct device *)pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/*
+	 * The device cannot address system memory, so there is no DMA
+	 * limitation.
+	 */
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL)
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(64));
+	}
+	else
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+	}
+
+	err = tc_enable(psSysData->pdev->dev.parent);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device (%d)", __func__, err));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev,
+														IORESOURCE_MEM,
+														"rogue-regs");
+	if (!psSysData->registers)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* each GPU instance gets the minimum 64kb register range */
+	ui32MinRegBankSize = RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE;
+#else
+	/* the GPU gets the entire 64MB IO range */
+	ui32MinRegBankSize = SYS_RGX_REG_REGION_SIZE;
+#endif
+
+	if (uiRegistersSize < ui32MinRegBankSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %pa, required 0x%08x)",
+				 __func__, &uiRegistersSize, ui32MinRegBankSize));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+	/* Reserve the address range */
+	if (!request_mem_region(psSysData->registers->start,
+							resource_size(psSysData->registers),
+							SYS_RGX_DEV_NAME))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+
+		goto ErrorDevDisable;
+	}
+
+	eError = DeviceConfigCreate(psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	eError = IonInit(psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise ION", __func__));
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	/* power functions */
+	psDevConfig->pfnPrePowerState = PrePower;
+	psDevConfig->pfnPostPowerState = PostPower;
+
+	psSysData->psDevConfig = psDevConfig;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	tc_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	tc_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	u32 tmp = 0;
+	u32 pll;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	if (tc_sys_info(psSysData->pdev->dev.parent, &tmp, &pll))
+		goto err_out;
+
+	if (tmp > 0)
+		PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", tmp);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", pll);
+
+err_out:
+	return eError;
+#endif
+}
+
+typedef struct
+{
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static void TCInterruptHandler(void* pvData)
+{
+	LISR_DATA *psLISRData = pvData;
+	psLISRData->pfnLISR(psLISRData->pvData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if ((ui32IRQ != TC_INTERRUPT_EXT) &&
+		(ui32IRQ < TC_INTERRUPT_OSID0) && (ui32IRQ > TC_INTERRUPT_OSID7))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, TCInterruptHandler, psLISRData);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	err = tc_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_enable_interrupt() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+	eError = PVRSRV_OK;
+
+	PVR_TRACE(("Installed device LISR " IMG_PFN_FMTSPEC " with tc module to ID %u",
+			   pfnLISR, ui32IRQ));
+
+err_out:
+	return eError;
+err_unset_interrupt_handler:
+	tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	goto err_out;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = tc_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_disable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " with tc module from ID %u",
+			   psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
+
+/****************************************************************************************************/
+/****                                   VM migration test code                                   ****/
+/****************************************************************************************************/
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode);
+static void PreMigrationDeviceSuspend(struct drm_device *psDev);
+static void PostMigrationDeviceResume(struct drm_device *psDev);
+
+void PVRVMMigration(unsigned int src, unsigned int dest);
+EXPORT_SYMBOL(PVRVMMigration);
+
+#define SWAP_REGSBASE_PTR(a, b) do \
+	{ \
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		b = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+	} while (0)
+
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psHostNode = PVRSRVGetDeviceInstance(0);
+	PVRSRV_RGXDEV_INFO *psSrcInfo = psSrcNode->pvDevice;
+	PVRSRV_RGXDEV_INFO *psDestInfo = psDestNode->pvDevice;
+	PVRSRV_DEVICE_CONFIG *psSrcConfig = psSrcNode->psDevConfig;
+	PVRSRV_DEVICE_CONFIG *psDestConfig = psDestNode->psDevConfig;
+	LISR_DATA *psSrcLISRData = (LISR_DATA *) psSrcInfo->pvLISRData;
+	void *pfnLISR = psSrcLISRData->pfnLISR;
+	IMG_UINT32 ui32SrcHyperLane, ui32DestHyperLane;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psHostNode != NULL) &&
+								  (psHostNode->psDevConfig != NULL)),
+								 "Device 0 (expected Host) not initialised.");
+
+	/* Determine the HyperLane ID used by a Guest Device from the Register Bank Base address used */
+	ui32SrcHyperLane = (psSrcConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psSrcConfig->ui32RegsSize;
+	ui32DestHyperLane = (psDestConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psDestConfig->ui32RegsSize;
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Swapping hyperlanes between Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32SrcHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32DestHyperLane));
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resulting configuration:    Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32DestHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32SrcHyperLane));
+
+	/* swap the register bank details */
+	SWAP_REGSBASE_PTR(psSrcInfo->pvRegsBaseKM, psDestInfo->pvRegsBaseKM);
+	SWAP(psSrcConfig->sRegsCpuPBase.uiAddr, psDestConfig->sRegsCpuPBase.uiAddr);
+	/* DevConfig->ui32RegsSize remains the same */
+
+	/* Swap interrupt lines between devices */
+	eError = SysUninstallDeviceLISR(psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+	eError = SysUninstallDeviceLISR(psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	SWAP(psSrcConfig->ui32IRQ, psDestConfig->ui32IRQ);
+
+	eError = SysInstallDeviceLISR(psSrcConfig->hSysData,
+								  psSrcConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psSrcNode,
+								  &psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+
+	eError = SysInstallDeviceLISR(psDestConfig->hSysData,
+								  psDestConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psDestNode,
+								  &psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	/* Swap contents of LMA carveouts between virtual devices */
+	{
+		/* Guest Raw Fw Heap mapping is done using the Host Devices */
+		PHYS_HEAP *psSrcHeap = NULL;
+		PHYS_HEAP *psDestHeap = NULL;
+		IMG_DEV_PHYADDR sSrcHeapBase, sDestHeapBase;
+
+		psSrcHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32SrcHyperLane];
+		psDestHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DestHyperLane];
+
+		PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcHeap != NULL) &&
+									  (psDestHeap != NULL)),
+									 "Guest firmware heaps not premapped by the Host Device.");
+
+		eError = PhysHeapGetDevPAddr(psSrcHeap, &sSrcHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(src fw heap)");
+		eError = PhysHeapGetDevPAddr(psDestHeap, &sDestHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(dest fw heap)");
+
+		eError = PvzServerUnmapDevPhysHeap(ui32SrcHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(src fw heap)");
+		eError = PvzServerUnmapDevPhysHeap(ui32DestHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(dest fw heap)");
+
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32SrcHyperLane]);
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32DestHyperLane]);
+
+		/* create new heaps with new base addresses */
+		eError = PvzServerMapDevPhysHeap(ui32SrcHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sDestHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(src fw heap)");
+		eError = PvzServerMapDevPhysHeap(ui32DestHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sSrcHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(dest fw heap)");
+	}
+}
+
+static void PreMigrationDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(eError == PVRSRV_OK,
+	                           "LinuxBridgeBlockClientsAccess()");
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * be declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+	}
+}
+
+static void PostMigrationDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+									PVRSRV_SYS_POWER_STATE_ON,
+									PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ);
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+}
+
+void PVRVMMigration(unsigned int src, unsigned int dest)
+{
+	PVRSRV_DEVICE_NODE *psSrcNode = PVRSRVGetDeviceInstance(src);
+	PVRSRV_DEVICE_NODE *psDestNode = PVRSRVGetDeviceInstance(dest);
+	struct device *psSrcDev, *psDestDev;
+	struct drm_device *psSrcDrmDev, *psDestDrmDev;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode != NULL) && (psDestNode != NULL) && (psSrcNode != psDestNode)),
+								 "Invalid Device IDs requested for migration.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE) &&
+								  (psDestNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)),
+								 "Devices not fully initialised.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig != NULL) &&
+								  (psDestNode->psDevConfig != NULL)),
+								 "Device config structure is NULL.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig->pvOSDevice != NULL) &&
+								  (psDestNode->psDevConfig->pvOSDevice != NULL)),
+								 "Linux kernel device pointer is NULL.");
+
+	psSrcDev = psSrcNode->psDevConfig->pvOSDevice;
+	psDestDev = psDestNode->psDevConfig->pvOSDevice;
+	psSrcDrmDev = dev_get_drvdata(psSrcDev);
+	psDestDrmDev = dev_get_drvdata(psDestDev);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcDrmDev != NULL) &&
+								  (psDestDrmDev != NULL)),
+								 "Linux kernel drm_device pointer is NULL.");
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psSrcNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psSrcDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psDestNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psDestDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Migrating vGPU resources (regbank, irq, osid)", __func__));
+	SwapHyperlanes(psSrcNode, psDestNode);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psSrcNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psSrcDrmDev);
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psDestNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psDestDrmDev);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysinfo.h
new file mode 100644
index 000000000000..6355a98482c3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysinfo.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(FPGA)
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(VIRTUAL_PLATFORM)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#elif defined(TC_APOLLO_TCF5)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#else
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#define SYS_RGX_DEV_NAME "tc_rogue"
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/Kbuild.mk
new file mode 100644
index 000000000000..0873e072163f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/Kbuild.mk
@@ -0,0 +1,53 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += services/system/rogue/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/rogue/common/vmm_type_$(VMM_TYPE).o
+
+ifeq ($(SUPPORT_ION),1)
+$(PVRSRVKM_NAME)-y += services/system/common/env/linux/ion_support_generic.o
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.c
new file mode 100644
index 000000000000..a3a34ed37e8a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.c
@@ -0,0 +1,344 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+#include "rgx_bvnc_defs_km.h"
+/*
+ * In systems that support trusted device address protection, there are three
+ * physical heaps from which pages should be allocated:
+ * - one heap for normal allocations
+ * - one heap for allocations holding META code memory
+ * - one heap for allocations holding secured DRM data
+ */
+
+#define PHYS_HEAP_IDX_GENERAL     0
+#define PHYS_HEAP_IDX_FW          1
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define PHYS_HEAP_IDX_TDFWMEM     2
+#define PHYS_HEAP_IDX_TDSECUREBUF 3
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+#define PHYS_HEAP_IDX_FW_MEMORY   2
+#endif
+
+/* Change to test CPU_LOCAL sys layers*/
+#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_GPU_LOCAL
+//#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_CPU_LOCAL
+
+#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_GPU_LOCAL
+//#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_CPU_LOCAL
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static PHYS_HEAP_FUNCTIONS gsPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	UMAPhysHeapCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	UMAPhysHeapDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	/*
+	 * This function is called during device initialisation, which on Linux,
+	 * means it won't be called concurrently. As such, there's no need to
+	 * protect it with a lock or use an atomic variable.
+	 */
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiHeapCount = 2;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	uiHeapCount += 2;
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	uiHeapCount += 1;
+#endif
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].ui32UsageFlags = UMA_HEAP_USAGE_FLAG;
+
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].pszPDumpMemspaceName = "SYSMEM_FW";
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].pszPDumpMemspaceName = "TDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].pszPDumpMemspaceName = "TDSECBUFMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].ui32UsageFlags =
+		PHYS_HEAP_USAGE_GPU_SECURE;
+
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].pszPDumpMemspaceName = "DEDICATEDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+#endif
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+#if defined(SUPPORT_AXI_ACE_TEST)
+		if ( ui64Features & RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		}else
+#endif
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_NONE;
+		}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszName                = "nohw";
+	psDevConfig->pszVersion             = NULL;
+	psDevConfig->pfnSysDevFeatureDepInit = SysDevFeatureDepInit;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr   = 0x00f00000;
+	psDevConfig->ui32RegsSize           = 0x4000;
+	psDevConfig->ui32IRQ                = 0x00000bad;
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+	psDevConfig->eDefaultHeap = UMA_DEFAULT_HEAP;
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	psDevConfig->bHasFBCDCVersion31      = IMG_FALSE;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	PVR_UNREFERENCED_PARAMETER(ui32IRQ);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(pfnLISR);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(phLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.h
new file mode 100644
index 000000000000..acbadc22e6a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(SYSCCONFIG_H)
+#define SYSCCONFIG_H
+
+
+#define RGX_NOHW_CORE_CLOCK_SPEED 100000000
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* SYSCCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysinfo.h
new file mode 100644
index 000000000000..566fc4bb7256
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysinfo.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME    "rgxnohw"
+#endif
+
+#endif	/* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/Kbuild.mk
new file mode 100644
index 000000000000..485b59bfa0e6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/Kbuild.mk
@@ -0,0 +1,55 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += \
+ services/system/rogue/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/rogue/common/vmm_type_$(VMM_TYPE).o
+
+ccflags-y += \
+ -I$(TOP)/services/system/rogue/common/env/linux -I$(TOP)/services/system/rogue/common/env/linux \
+ -I$(TOP)/include/rogue/system/$(PVR_SYSTEM) -I$(TOP)/include/system/$(PVR_SYSTEM)
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.c
new file mode 100644
index 000000000000..e02bd6823920
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.c
@@ -0,0 +1,2721 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "power.h"
+#include "sysinfo.h"
+#include "apollo_regs.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pci_support.h"
+#include "interrupt_support.h"
+#include "tcf_clk_ctrl.h"
+#include "tcf_pll.h"
+#include "sysdata.h"
+#include "dc_server.h"
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+/*
+Notes on device virtualization support on PCI systems:
+	Host/Guest driver expectations in VM domain:
+		- RGX discovery available on vPCI/bus.
+		- RGX PCI/BAR available on vPCI/bus.
+	Guest driver limitations in VM domain:
+		- Does not perform device management.
+		- Does not power RGX up/down.
+ */
+#include "vz_vmm_pvz.h"
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+#define	HOST_PCI_INIT_FLAGS	0
+#else
+#define	HOST_PCI_INIT_FLAGS	HOST_PCI_INIT_FLAG_BUS_MASTER
+#endif
+
+/* The VM information page size which is allocated from LMA memory, this
+   page allows VZ drivers to communicate information without going via
+   the hypervisor. This is used only by VZ drivers to synchronise
+   cross-VM interrupt handling */
+#define VM_INFOPG_SIZE          0x1000
+
+/* This is used only by guest drivers during interrupt handling */
+#define IRQ_ACK_RETRY_CLEAR_MAX 2
+
+/* Interrupt defines */
+typedef enum _APOLLO_IRQ_
+{
+	APOLLO_IRQ_GPU = 0,
+	APOLLO_IRQ_PDP,
+	APOLLO_IRQ_MAX,
+} APOLLO_IRQ;
+
+/* Clock speed module parameters */
+static IMG_UINT32 ui32MemClockSpeed  = RGX_TC_MEM_CLOCK_SPEED;
+static IMG_UINT32 ui32CoreClockSpeed = RGX_TC_CORE_CLOCK_SPEED;
+#if !defined(TC_APOLLO_ES2)
+static IMG_UINT32 ui32SysClockSpeed = RGX_TC_SYS_CLOCK_SPEED;
+#endif
+#if defined(TC_APOLLO_TCF5)
+static IMG_UINT32 ui32MemReadLatency = 0;
+static IMG_UINT32 ui32MemWriteResponseLatency = 0;
+#endif
+
+#if defined(__linux__)
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+module_param_named(sys_mem_clk_speed,  ui32MemClockSpeed,  uint, S_IRUGO | S_IWUSR);
+module_param_named(sys_core_clk_speed, ui32CoreClockSpeed, uint, S_IRUGO | S_IWUSR);
+#if !defined(TC_APOLLO_ES2)
+module_param_named(sys_sysif_clk_speed, ui32SysClockSpeed, uint, S_IRUGO | S_IWUSR);
+#endif
+#if defined(TC_APOLLO_TCF5)
+module_param_named(sys_mem_latency, ui32MemReadLatency, uint, S_IRUGO | S_IWUSR);
+module_param_named(sys_wresp_latency, ui32MemWriteResponseLatency, uint, S_IRUGO | S_IWUSR);
+#endif
+#endif
+
+/* Specifies the maximum number of ApolloHardReset() attempts */
+#define APOLLO_RESET_MAX_RETRIES	10
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+#define HEX2DEC(hexIntVal)    ((((hexIntVal) >> 4) * 10) + ((hexIntVal) & 0x0F))
+
+static PVRSRV_ERROR InitMemory(PVRSRV_DEVICE_CONFIG *psSysConfig, SYS_DATA *psSysData);
+static INLINE void DeInitMemory(PVRSRV_DEVICE_CONFIG *psSysConfig, SYS_DATA *psSysData);
+
+size_t GetMemorySize(IMG_CPU_PHYADDR sMemCpuPhyAddr, IMG_UINT64 pciBarSize);
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	IMG_CHAR apszFPGARev[FPGA_REV_MAX_LEN];
+	IMG_CHAR apszCoreRev[TCF_CORE_REV_MAX_LEN];
+	IMG_CHAR apszConfigRev[TCF_CORE_CFG_MAX_LEN];
+	IMG_CHAR apszPCIVer[PCI_VERSION_MAX_LEN];
+	IMG_CHAR apszMacroVer[MACRO_VERSION_MAX_LEN];
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+	IMG_UINT32 ui32Value;
+	IMG_CPU_PHYADDR	sHostFPGARegCpuPBase;
+	void *pvHostFPGARegCpuVBase;
+
+	/* To get some of the version information we need to read from a register that we don't normally have
+	   mapped. Map it temporarily (without trying to reserve it) to get the information we need. */
+	sHostFPGARegCpuPBase.uiAddr	= OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM) + 0x40F0;
+	pvHostFPGARegCpuVBase		= OSMapPhysToLin(sHostFPGARegCpuPBase, 0x04, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvHostFPGARegCpuVBase == NULL)
+	{
+		return NULL;
+	}
+
+	/* Create the components of the PCI and macro versions */
+	ui32Value = OSReadHWReg32(pvHostFPGARegCpuVBase, 0);
+	OSSNPrintf(apszPCIVer, sizeof(apszPCIVer), "%d",
+	           HEX2DEC((ui32Value & 0x00FF0000) >> 16));
+	OSSNPrintf(apszMacroVer, sizeof(apszMacroVer), "%d.%d",
+	           ((ui32Value & 0x00000F00) >> 8),
+	           HEX2DEC((ui32Value & 0x000000FF) >> 0));
+
+	/* Unmap the register now that we no longer need it */
+	OSUnMapPhysToLin(pvHostFPGARegCpuVBase, 0x04);
+
+	/*
+	 * Check bits 7:0 of register 0x28 (TCF_CORE_REV_REG or SW_IF_VERSION
+	 * depending on its own value) to find out how the driver should
+	 * generate the strings for FPGA and core revision.
+	 */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_SW_IF_VERSION);
+	ui32Value = (ui32Value & VERSION_MASK) >> VERSION_SHIFT;
+
+	if (ui32Value == 0)
+	{
+		/* Create the components of the TCF core revision number */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TCF_CORE_REV_REG);
+		OSSNPrintf(apszCoreRev, sizeof(apszCoreRev), "%d.%d.%d",
+		           HEX2DEC((ui32Value & TCF_CORE_REV_REG_MAJOR_MASK) >> TCF_CORE_REV_REG_MAJOR_SHIFT),
+		           HEX2DEC((ui32Value & TCF_CORE_REV_REG_MINOR_MASK) >> TCF_CORE_REV_REG_MINOR_SHIFT),
+		           HEX2DEC((ui32Value & TCF_CORE_REV_REG_MAINT_MASK) >> TCF_CORE_REV_REG_MAINT_SHIFT));
+
+		/* Create the components of the FPGA revision number */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_FPGA_REV_REG);
+		OSSNPrintf(apszFPGARev, sizeof(apszFPGARev), "%d.%d.%d",
+		           HEX2DEC((ui32Value & FPGA_REV_REG_MAJOR_MASK) >> FPGA_REV_REG_MAJOR_SHIFT),
+		           HEX2DEC((ui32Value & FPGA_REV_REG_MINOR_MASK) >> FPGA_REV_REG_MINOR_SHIFT),
+		           HEX2DEC((ui32Value & FPGA_REV_REG_MAINT_MASK) >> FPGA_REV_REG_MAINT_SHIFT));
+	}
+	else if (ui32Value == 1)
+	{
+		/* Create the components of the TCF core revision number */
+		OSSNPrintf(apszCoreRev, sizeof(apszCoreRev), "%d", ui32Value);
+
+		/* Create the components of the FPGA revision number */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_REL);
+		OSSNPrintf(apszFPGARev, sizeof(apszFPGARev), "%d.%d",
+		           HEX2DEC((ui32Value & MAJOR_MASK) >> MAJOR_SHIFT),
+		           HEX2DEC((ui32Value & MINOR_MASK) >> MINOR_SHIFT));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: unrecognised SW_IF_VERSION 0x%08x", __func__, ui32Value));
+
+		/* Create the components of the TCF core revision number */
+		OSSNPrintf(apszCoreRev, sizeof(apszCoreRev), "%d", ui32Value);
+
+		/* Create the components of the FPGA revision number */
+		OSSNPrintf(apszFPGARev, sizeof(apszFPGARev), "N/A");
+	}
+
+	/* Create the component of the TCF core target build ID */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG);
+	OSSNPrintf(apszConfigRev, sizeof(apszConfigRev), "%d",
+	           (ui32Value & TCF_CORE_TARGET_BUILD_ID_MASK) >> TCF_CORE_TARGET_BUILD_ID_SHIFT);
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(apszFPGARev);
+	ui32StringLength += OSStringLength(apszCoreRev);
+	ui32StringLength += OSStringLength(apszConfigRev);
+	ui32StringLength += OSStringLength(apszPCIVer);
+	ui32StringLength += OSStringLength(apszMacroVer);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+		           SYSTEM_INFO_FORMAT_STRING,
+		           apszFPGARev,
+		           apszCoreRev,
+		           apszConfigRev,
+		           apszPCIVer,
+		           apszMacroVer);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+static IMG_BOOL SystemISRHandler(void *pvData)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 ui32InterruptStatus;
+	IMG_UINT32 ui32InterruptClear = 0;
+	IMG_UINT32 ui32InterruptClear2 = 0;
+	SYS_DATA *psSysData = (SYS_DATA *)pvData;
+
+	/* Read device interrupt status register on entry, for VZ multiple drivers might sample same value */
+	ui32InterruptStatus = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_STATUS);
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		/*
+		  Multi-VM PCI device sharing/virtualization
+
+		  PCI device sharing is problematic, almost all hypervisors do not support it; for
+		  those that do (i.e. Fiasco/L4), it's not possible to correctly handle clearing
+		  device interrupt register concurrently from multiple VM/drivers without the aid
+		  of an external interrupt routing logic that forwards vIRQ to VM, also this logic
+		  would have to be implemented per-hypervisor environment. An alternative approach
+		  used here is to always perform the clearing of the device interrupt register in
+		  the host driver and only do so in the guest as a last resort mostly so that the
+		  guest can still function on a VM shared interrupt line with same or other active
+		  VM device drivers. NOTE: This assumes underlying hypervisor or vm manager walks
+		  and delivers vIRQ into each VM vPCI/bus for every RGX interrupt it receives thus
+		  ensuring interrupt events are not lost, it's OK if pending interrupts are merged.
+		*/
+		if ((ui32InterruptStatus & psSysData->sInterruptData[0].ui32InterruptFlag) ||
+			(ui32InterruptStatus & psSysData->sInterruptData[1].ui32InterruptFlag))
+		{
+			/* Set RGX/PDP interrupt status, we cannot assume what's interrupting */
+			ui32InterruptClear = (0x1 << EXT_INT_SHIFT) | (0x1 << PDP1_INT_SHIFT);
+			ui32InterruptClear2 = ui32InterruptClear;
+
+			if (PVRSRV_VZ_MODE_IS(HOST))
+			{
+				/*
+				   Assigning a fixed driver (i.e. the host) the responsibility of clearing
+				   the RGX interrupt register does not always work because the hypervisor
+				   might schedule another VM and run it long enough for that VM OS/kernel
+				   to detect that RGX interrupt line has not been cleared after multiple
+				   successive back-to-back calls to the guest interrupt handler. To help
+				   mitigate against this, the host driver updates this counter for valid
+				   RGX interrupt events only and the guest driver samples it to know how
+				   to respond/acknowledge interrupt events to its OS/kernel.
+				 */
+				(void) OSAtomicIncrement((void*)psSysData->pui32IRQCount);
+			}
+		}
+
+		/*
+		  The idea here is to always assume the driver is being called due to RGX IRQ event
+		  because multiple devices may be sharing same interrupt line both at BIOS hypervisor
+		  level and/or VM level and due to possibility of race between host/guest clearing
+		  the interrupt register. Doing this means drivers never miss any IRQ event & all
+		  LISR routine is guaranteed to be called for every HW irq event, the down side is
+		  that the LISR routine is called in all driver VM for every IRQ event on line.
+		 */
+		ui32InterruptStatus = 0xFFFFFFFF;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		/* Should this array grow, ensure to unroll the loop for the VZ condition above */
+		if ((ui32InterruptStatus & psSysData->sInterruptData[i].ui32InterruptFlag) != 0)
+		{
+			psSysData->sInterruptData[i].pfnLISR(psSysData->sInterruptData[i].pvData);
+			ui32InterruptClear |= psSysData->sInterruptData[i].ui32InterruptFlag;
+		}
+	}
+
+	if (PVRSRV_VZ_MODE_IS(HOST))
+	{
+		/* Only clear interrupt register if device has actually interrupted */
+		ui32InterruptClear = ui32InterruptClear2 ? ui32InterruptClear : 0;
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST))
+	{
+		/* If neither of two conditions below are met, then RGX/PDP is not interrupting */
+		IMG_UINT32 ui32CurrentIrqCount = OSAtomicRead((void *)psSysData->pui32IRQCount);
+		static IMG_UINT32 ui32IrqAckRetryClearCount = 0;
+		static IMG_UINT32 ui32LastIrqCount = 0;
+		ui32InterruptClear = 0;
+
+		if (ui32CurrentIrqCount != ui32LastIrqCount)
+		{
+			/*
+			   One complication is knowing if the RGX device is the only device on its PCI
+			   interrupt line either within VM or down at the BIOS/hypervisor level and
+			   if RGX device is also sharing line with another VM PCI device driver. As
+			   it is impossible to determine this, guest driver requires a way of filtering
+			   RGX IRQ events such that it can positively respond back to the OS/kernel
+			   that it has handled interrupts due to it else the OS/kernel might disable
+			   the RGX interrupt line which on some hypervisor environments will impact
+			   interrupt delivery to all the VMs sharing the PCI/RGX device.
+			*/
+			ui32LastIrqCount = ui32CurrentIrqCount;
+			return IMG_TRUE;
+		}
+		else if (ui32InterruptClear2)
+		{
+			/*
+			   The guest driver has been called with the RGX/PDP interrupt flag bits still
+			   pending so we need to determine if this is a back-to-back call. If not we
+			   give priority to the host driver to clear it else guest drivers does it.
+			   This can be due to guest VM/driver running ahead of host VM/driver for this
+			   interrupt event and/or the hypervisor scheduler has parked momentarily the
+			   host VM/driver whilst running the guest VM/driver across many cycles right
+			   up to this point.
+			*/
+			if (ui32IrqAckRetryClearCount < IRQ_ACK_RETRY_CLEAR_MAX)
+			{
+				/* NOTE: This assumes OS/kernel calls-back into this ISR a couple of times
+				   before giving-up and disabling the interrupt line if said interrupt
+				   line is still asserted after each ISR invocation */
+				ui32IrqAckRetryClearCount = ui32IrqAckRetryClearCount + 1;
+				return IMG_TRUE;
+			}
+
+			/*
+			   Now the race between the host/guest(s) driver here can result in two outcomes.
+			   First of which is that the loser(s) will clear the interrupt register after it
+			   has been cleared by the winner and before the device has raised another IRQ
+			   event - the effects of this should be benign. Second is that this write will
+			   happen after the device has raised another IRQ event - the effects of this is
+			   should be OK so long as the hypervisor delivers the corresponding vIRQ.
+			 */
+			ui32InterruptClear = ui32InterruptClear2;
+			ui32IrqAckRetryClearCount = 0;
+		}
+	}
+
+	if (ui32InterruptClear)
+	{
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_CLEAR, ui32InterruptClear);
+
+		/*
+		   CPU-PCI-WRITE-BUFFER
+
+		   On PC platforms, this BIOS performance enhancing feature controls the chipset's
+		   CPU-to-PCI write buffer used to store PCI writes from the CPU before being written
+		   onto the PCI bus. By reading from the register, this CPU-to-PCI write buffer is
+		   flushed. Without this read, at times the kernel reports unhandled IRQs due to
+		   above interrupt clear not being reflected on the IRQ line after exit from the
+		   interrupt handler.
+		 */
+		(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_CLEAR);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+#if defined(TC_APOLLO_ES2) || defined(TC_APOLLO_BONNIE)
+
+static PVRSRV_ERROR SPI_Read(void *pvLinRegBaseAddr,
+							 IMG_UINT32 ui32Offset,
+							 IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32Count = 0;
+	*pui32Value = 0;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	OSWriteHWReg32(pvLinRegBaseAddr,
+				   TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR,
+				   TCF_SPI_MST_RDNWR_MASK | ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(100);
+
+	while (((OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_STATUS)) != 0x08) &&
+			(ui32Count < 10000))
+	{
+		ui32Count++;
+	}
+
+	if (ui32Count < 10000)
+	{
+		*pui32Value = OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_RDATA);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SPI_Read: Time out reading SPI register (0x%x)", ui32Offset));
+		return PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void SPI_Write(void *pvLinRegBaseAddr,
+					  IMG_UINT32 ui32Offset,
+					  IMG_UINT32 ui32Value)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR, ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_WDATA, ui32Value);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(1000);
+}
+
+static IMG_BOOL IsInterfaceAligned(IMG_UINT32 ui32Eyes,
+								   IMG_UINT32 ui32ClkTaps,
+								   IMG_UINT32 ui32TrainAck)
+{
+	IMG_UINT32	ui32MaxEyeStart = ui32Eyes >> 16;
+	IMG_UINT32	ui32MinEyeEnd = ui32Eyes & 0xffff;
+
+	IMG_BOOL	bTrainingComplete = (ui32ClkTaps & 0x10000);
+
+	IMG_BOOL	bTrainAckComplete = (ui32TrainAck & 0x100);
+	IMG_UINT32	ui32FailedAcks = (ui32TrainAck & 0xf0) >> 4;
+
+
+	//If either the training or training ack failed, we haven't aligned
+	if (!bTrainingComplete || !bTrainAckComplete)
+	{
+		return IMG_FALSE;
+	}
+
+	//If the max eye >= min eye it means the readings are nonsense
+	if (ui32MaxEyeStart >= ui32MinEyeEnd)
+	{
+		return IMG_FALSE;
+	}
+
+	//If we failed the ack pattern more than 4 times
+	if (ui32FailedAcks > 4)
+	{
+		return IMG_FALSE;
+	}
+
+	//If there is less than 7 taps
+	//(240ps @40ps/tap, this number should be lower for the fpga, since its taps are bigger
+	//We should really calculate the "7" based on the interface clock speed.
+	if ((ui32MinEyeEnd - ui32MaxEyeStart) < 7)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static IMG_UINT32 SAI_Read(void *base, IMG_UINT32 addr)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, 0);
+	OSWriteHWReg32(base, 0x300, 0x200 | addr);
+	OSWriteHWReg32(base, 0x318, 1);
+	return OSReadHWReg32(base, 0x310);
+}
+
+static PVRSRV_ERROR ApolloHardReset(SYS_DATA *psSysData)
+{
+	IMG_BOOL bAlignmentOK = IMG_FALSE;
+	IMG_INT resetAttempts = 0;
+	IMG_UINT32 ui32ClkResetN;
+#if !defined(TC_APOLLO_BONNIE)
+	IMG_UINT32 ui32DUTCtrl;
+#endif
+	//This is required for SPI reset which is not yet implemented.
+	//IMG_UINT32 ui32AuxResetsN;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+#if !defined(TC_APOLLO_BONNIE)
+	//power down
+	ui32DUTCtrl = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1);
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_0V9EN;
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_1V8EN;
+	ui32DUTCtrl |= DUT_CTRL_VCC_IO_INH;
+	ui32DUTCtrl |= DUT_CTRL_VCC_CORE_INH;
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1, ui32DUTCtrl);
+#endif
+
+	OSSleepms(500);
+
+	//set clock speed here, before reset.
+#if defined(TC_APOLLO_BONNIE)
+	//The ES2 FPGA builds do not like their core clocks being set (it takes apollo down).
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_CORE_CLK0, ui32CoreClockSpeed / 1000000);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_CORE_DRP_GO, 1);
+#endif
+
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_MEMIF_CLK0, ui32MemClockSpeed / 1000000);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_MEM_DRP_GO, 1);
+
+	PVR_LOG(("Setting clocks to %uMHz/%uMHz", ui32CoreClockSpeed / 1000000, ui32MemClockSpeed / 1000000));
+
+	OSWaitus(400);
+
+	//Put DCM, DUT, DDR, PDP1, and PDP2 into reset
+	ui32ClkResetN = (0x1 << GLB_CLKG_EN_SHIFT);
+	ui32ClkResetN |= (0x1 << SCB_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+
+	OSSleepms(100);
+
+#if !defined(TC_APOLLO_BONNIE)
+	//Enable the voltage control regulators on DUT
+	ui32DUTCtrl = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1);
+	ui32DUTCtrl |= DUT_CTRL_VCC_0V9EN;
+	ui32DUTCtrl |= DUT_CTRL_VCC_1V8EN;
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_IO_INH;
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_CORE_INH;
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1, ui32DUTCtrl);
+
+	OSSleepms(300);
+#endif
+
+	//Take DCM, DDR, PDP1, and PDP2 out of reset
+	ui32ClkResetN |= (0x1 << DDR_RESETN_SHIFT);
+	ui32ClkResetN |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	ui32ClkResetN |= (0x1 << PDP1_RESETN_SHIFT);
+	ui32ClkResetN |= (0x1 << PDP2_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+
+#if !defined(TC_APOLLO_BONNIE)
+	//Set ODT to a specific value that seems to provide the most stable signals.
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0x11, 0x413130);
+#endif
+
+	//Take DUT out of reset
+	ui32ClkResetN |= (0x1 << DUT_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+	OSSleepms(100);
+
+	//try to enable the core clock PLL
+	{
+		IMG_UINT32 ui32DUTResets;
+		IMG_UINT32 ui32DUTGPIO1;
+		IMG_UINT32 ui32PLLStatus = 0;
+
+		/* Un-bypass the PLL on the DUT */
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0x1, 0x0);
+		ui32DUTResets = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, 0x320);
+		ui32DUTResets |= 0x1;
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x320, ui32DUTResets);
+		ui32DUTResets &= 0xfffffffe;
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x320, ui32DUTResets);
+		OSSleepms(1000);
+
+		if (SPI_Read(psSysData->pvSystemRegCpuVBase, 0x2, &ui32PLLStatus) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Unable to read PLL status"));
+		}
+
+		if (ui32PLLStatus != 0x1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PLL has failed to lock, status = %x", ui32PLLStatus));
+		}
+		else
+		{
+			/* Select DUT PLL as core clock */
+			ui32DUTGPIO1 = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, 0x108);
+			ui32DUTGPIO1 &= 0xfffffff7;
+			OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x108, ui32DUTGPIO1);
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "PLL has been set to 6x"));
+	}
+
+	while (!bAlignmentOK && resetAttempts < 10)
+	{
+		IMG_UINT32 ui32Eyes     = 0;
+		IMG_UINT32 ui32ClkTaps  = 0;
+		IMG_UINT32 ui32TrainAck = 0;
+
+		IMG_INT bank;
+
+
+		resetAttempts += 1;
+
+		//reset the DUT to allow the SAI to retrain
+		ui32ClkResetN &= ~(0x1 << DUT_RESETN_SHIFT);
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+		OSWaitus(100);
+		ui32ClkResetN |= (0x1 << DUT_RESETN_SHIFT);
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+		OSWaitus(100);
+
+		//Assume alignment passed, if any bank fails on either DUT or FPGA
+		//we will set this to false and try again for a max of 10 times.
+		bAlignmentOK = IMG_TRUE;
+
+		//for each of the banks
+		for (bank = 0; bank < 10; bank++)
+		{
+
+			//check alignment on the DUT
+			IMG_UINT32 ui32BankBase = 0x7000 + (0x1000 * bank);
+
+			SPI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x4, &ui32Eyes);
+			SPI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x3, &ui32ClkTaps);
+			SPI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x6, &ui32TrainAck);
+
+			if (!IsInterfaceAligned(ui32Eyes, ui32ClkTaps, ui32TrainAck))
+			{
+				bAlignmentOK = IMG_FALSE;
+				break;
+			}
+
+			//check alignment on the FPGA
+			ui32BankBase = 0xb0 + (0x10 * bank);
+
+			ui32Eyes = SAI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x4);
+			ui32ClkTaps = SAI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x3);
+			ui32TrainAck = SAI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x6);
+
+			if (!IsInterfaceAligned(ui32Eyes, ui32ClkTaps, ui32TrainAck))
+			{
+				bAlignmentOK = IMG_FALSE;
+				break;
+			}
+		}
+	}
+
+	if (!bAlignmentOK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Unable to initialise the testchip (interface alignment failure), please restart the system."));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	if (resetAttempts > 1)
+	{
+		PVR_LOG(("Note: The testchip required more than one reset to find a good interface alignment!"));
+		PVR_LOG(("      This should be harmless, but if you do suspect foul play, please reset the machine."));
+		PVR_LOG(("      If you continue to see this message you may want to report it to PowerVR Verification Platforms."));
+	}
+
+#if !defined(TC_APOLLO_BONNIE)
+	/* Enable the temperature sensor */
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xc, 0); //power up
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xc, 2); //reset
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xc, 6); //init & run
+#endif
+
+	{
+		IMG_UINT32 ui32HoodCtrl;
+		SPI_Read(psSysData->pvSystemRegCpuVBase, 0xF, &ui32HoodCtrl);
+		ui32HoodCtrl |= 0x1;
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0xF, ui32HoodCtrl);
+	}
+
+	{
+		IMG_UINT32 ui32RevID = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, 0x10);
+
+		IMG_UINT32 ui32BuildInc = (ui32RevID >> 12) & 0xff;
+
+		if (ui32BuildInc)
+		{
+			PVR_LOG(("BE WARNED: You are not running a tagged release of the FPGA image!"));
+			PVR_LOG(("Owner: 0x%01x, Inc: 0x%02x", (ui32RevID >> 20) & 0xf, ui32BuildInc));
+		}
+
+		PVR_LOG(("FPGA Release: %u.%02u", ui32RevID >> 8 & 0xf, ui32RevID & 0xff));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HardResetDevice(SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevice)
+{
+	RGX_TIMING_INFORMATION *psTimingInfo;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiValue;
+
+	for (uiValue = 0; uiValue < APOLLO_RESET_MAX_RETRIES; uiValue++)
+	{
+		eError = ApolloHardReset(psSysData);
+		if (eError == PVRSRV_OK)
+		{
+			break;
+		}
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "ApolloHardReset");
+
+	psTimingInfo = ((RGX_DATA *) psDevice->hDevData)->psRGXTimingInfo;
+
+	/* Make the core clock speed value available to the driver (matching the
+	 * one set by ApolloHardReset()) */
+#if defined(TC_APOLLO_BONNIE)
+	/* For BonnieTC there seems to be an additional 5x multiplier that occurs to
+	 * the clock as measured speed is 540Mhz not 108Mhz. */
+	psTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed * 6 * 5;
+#else /* defined(TC_APOLLO_BONNIE) */
+	psTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed * 6;
+#endif /* defined(TC_APOLLO_BONNIE) */
+
+#if defined(TC_APOLLO_BONNIE)
+	/* enable ASTC via SPI */
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, 0xf, &uiValue);
+	PVR_LOG_IF_ERROR(eError, "SPI_Read");
+
+	uiValue |= 0x1 << 4;
+
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xf, uiValue);
+#endif /* defined(TC_APOLLO_BONNIE) */
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PCIInitDev(PVRSRV_DEVICE_CONFIG *psDevice, void *pvOSDevice)
+{
+	SYS_DATA *psSysData;
+	IMG_CPU_PHYADDR	sApolloRegCpuPBase;
+	IMG_UINT32 uiApolloRegSize;
+	PVRSRV_ERROR eError;
+#if defined(TC_APOLLO_BONNIE)
+	IMG_UINT32 ui32Value;
+#endif
+
+	PVR_ASSERT(pvOSDevice);
+
+	if (psDevice->pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+#endif
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (!psSysData)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->hRGXPCI = OSPCISetDev(TO_PCI_COOKIE(pvOSDevice), HOST_PCI_INIT_FLAGS);
+	if (!psSysData->hRGXPCI)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire PCI device", __func__));
+		eError = PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+		goto ErrorFreeSysData;
+	}
+
+	/* Get Apollo register information */
+	sApolloRegCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM);
+	uiApolloRegSize	 = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM));
+
+	/* Check the address range is large enough. */
+	if (uiApolloRegSize < SYS_APOLLO_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Apollo register region isn't big enough (was 0x%08X, required 0x%08X)",
+			 __func__, uiApolloRegSize, SYS_APOLLO_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorPCIReleaseDevice;
+	}
+
+	/* The Apollo register range contains several register regions. Request only the system control register region */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the system register region",
+				 __func__));
+		goto ErrorPCIReleaseDevice;
+	}
+	psSysData->sSystemRegCpuPBase.uiAddr = sApolloRegCpuPBase.uiAddr + SYS_APOLLO_REG_SYS_OFFSET;
+	psSysData->uiSystemRegSize = SYS_APOLLO_REG_REGION_SIZE;
+
+	/* Setup Rogue register information */
+	psDevice->sRegsCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->ui32RegsSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM));
+
+	/* Save data for this device */
+	psDevice->hSysData = (IMG_HANDLE)psSysData;
+
+	/* Check the address range is large enough. */
+	if (psDevice->ui32RegsSize < SYS_RGX_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough "
+				 "(was 0x%08x, required 0x%08x)",
+				 __func__, psDevice->ui32RegsSize, SYS_RGX_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/* Reserve the address range */
+	eError = OSPCIRequestAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/*
+	   Map in system registers so we can:
+	   - Configure the memory mode (LMA, UMA or LMA/UMA hybrid)
+	   - Hard reset Apollo
+	   - Clear interrupts
+	*/
+	psSysData->pvSystemRegCpuVBase = OSMapPhysToLin(psSysData->sSystemRegCpuPBase,
+													psSysData->uiSystemRegSize,
+													PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (psSysData->pvSystemRegCpuVBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRGXRegReleaseAddrRange;
+	}
+
+	eError = HardResetDevice(psSysData, psDevice);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnMapAddrRange);
+
+	eError = InitMemory(psDevice, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorUnMapAddrRange;
+	}
+
+	psSysData->pszVersion = GetDeviceVersionString(psSysData);
+	if (psSysData->pszVersion)
+	{
+		psDevice->pszVersion = psSysData->pszVersion;
+	}
+
+	eError = OSPCIIRQ(psSysData->hRGXPCI, &psSysData->ui32IRQ);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't get IRQ", __func__));
+		goto ErrorDeInitMemory;
+	}
+
+	/* Register our handler */
+	eError = OSInstallSystemLISR(&psSysData->hLISR,
+								psSysData->ui32IRQ,
+								psDevice->pszName,
+								SystemISRHandler,
+								psSysData,
+								SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to install the system device interrupt handler",
+				 __func__));
+
+		goto ErrorDeInitMemory;
+	}
+
+	psDevice->pvOSDevice = pvOSDevice;
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	psSysData->psDevConfig = psDevice;
+#endif
+
+	return PVRSRV_OK;
+
+ErrorDeInitMemory:
+	DeInitMemory(psDevice, psSysData);
+
+ErrorUnMapAddrRange:
+	OSUnMapPhysToLin(psSysData->pvSystemRegCpuVBase, psSysData->uiSystemRegSize);
+	psSysData->pvSystemRegCpuVBase = NULL;
+
+ErrorRGXRegReleaseAddrRange:
+	OSPCIReleaseAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->sRegsCpuPBase.uiAddr	= 0;
+	psDevice->ui32RegsSize		= 0;
+
+ErrorSystemRegReleaseAddrRegion:
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	psSysData->sSystemRegCpuPBase.uiAddr	= 0;
+	psSysData->uiSystemRegSize		= 0;
+
+ErrorPCIReleaseDevice:
+	OSPCIReleaseDev(psSysData->hRGXPCI);
+	psSysData->hRGXPCI = NULL;
+
+ErrorFreeSysData:
+	OSFreeMem(psSysData);
+	psDevice->hSysData = NULL;
+
+	return eError;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevice,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVRSRV_ERROR	eError;
+	SYS_DATA		*psSysData = psDevice->hSysData;
+	IMG_UINT32		ui32RegVal;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+#if !defined(TC_APOLLO_BONNIE)
+	/* Read the temperature */
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, TCF_TEMP_SENSOR_SPI_OFFSET, &ui32RegVal);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SysDebugInfo: SPI_Read failed for register 0x%x (0x%x)", TCF_TEMP_SENSOR_SPI_OFFSET, eError));
+		goto SysDebugInfo_exit;
+	}
+
+	PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", TCF_TEMP_SENSOR_TO_C(ui32RegVal));
+#endif
+
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, 0x2, &ui32RegVal);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", ui32RegVal);
+
+#if !defined(TC_APOLLO_BONNIE)
+SysDebugInfo_exit:
+#endif
+	return eError;
+}
+
+#else /* if defined(TC_APOLLO_ES2) else TC_APOLLO_ES1 or TC_APOLLO_TCF5 */
+
+#define pol(base,reg,val,msk) \
+	do { \
+		int polnum; \
+		for (polnum = 0; polnum < 500; polnum++) \
+		{ \
+			if ((OSReadHWReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			OSSleepms(1); \
+		} \
+		if (polnum == 500) \
+		{ \
+			PVR_DPF((PVR_DBG_WARNING, "Poll failed for register: 0x%08X", (unsigned int)reg)); \
+		} \
+	} while (0)
+
+#define polrgx(reg,val,msk) pol(pvRegsBaseKM, reg, val, msk)
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_UINT32 ui32Frequency)
+{
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_UINT32 ui32Voltage)
+{
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+#endif
+
+static void SPI_Write(void *pvLinRegBaseAddr,
+					  IMG_UINT32 ui32Offset,
+					  IMG_UINT32 ui32Value)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR, ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_WDATA, ui32Value);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(1000);
+
+}
+
+static PVRSRV_ERROR SPI_Read(void *pvLinRegBaseAddr,
+							 IMG_UINT32 ui32Offset,
+							 IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32Count = 0;
+	*pui32Value = ui32Count;
+	PVRSRV_VZ_RET_IF_MODE(NATIVE, PVRSRV_OK);
+
+	OSWriteHWReg32(pvLinRegBaseAddr,
+				   TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR,
+				   TCF_SPI_MST_RDNWR_MASK | ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(1000);
+
+	while (((OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_STATUS)) != 0x08) &&
+			(ui32Count < 10000))
+	{
+		ui32Count++;
+	}
+
+	if (ui32Count < 10000)
+	{
+		*pui32Value = OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_RDATA);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SPI_Read: Time out reading SPI register (0x%x)", ui32Offset));
+		return PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE;
+	}
+
+	return PVRSRV_OK;
+}
+
+#if !defined(TC_APOLLO_TCF5)
+#if !defined(VIRTUAL_PLATFORM)
+/* This is a quick RGX bootup (built-in-self) test */
+static void RGXInitBistery(IMG_CPU_PHYADDR sRegisters, IMG_UINT32 ui32Size)
+{
+	void *pvRegsBaseKM;
+	IMG_UINT instance;
+	IMG_UINT i;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	pvRegsBaseKM = OSMapPhysToLin(sRegisters, ui32Size, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	/* Force clocks on */
+	OSWriteHWReg32(pvRegsBaseKM, 0, 0x55555555);
+	OSWriteHWReg32(pvRegsBaseKM, 4, 0x55555555);
+
+	polrgx(0xa18, 0x05000000, 0x05000000);
+	OSWriteHWReg32(pvRegsBaseKM, 0xa10, 0x048000b0);
+	OSWriteHWReg32(pvRegsBaseKM, 0xa08, 0x55111111);
+	polrgx(0xa18, 0x05000000, 0x05000000);
+
+	/* Clear PDS CSRM and USRM to prevent ERRORs at end of test */
+	OSWriteHWReg32(pvRegsBaseKM, 0x630, 0x1);
+	OSWriteHWReg32(pvRegsBaseKM, 0x648, 0x1);
+	OSWriteHWReg32(pvRegsBaseKM, 0x608, 0x1);
+
+	/* Run BIST for SLC (43) */
+	/* Reset BIST */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0x8);
+	OSWaitus(100);
+
+	/* Clear BIST controller */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0x10);
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		/* Start BIST */
+		OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0x4);
+
+		OSWaitus(100);
+
+		/* Wait for pause */
+		polrgx(0x7000, ui32Pol, ui32Pol);
+	}
+	OSWaitus(100);
+
+	/* Check results for 43 RAMs */
+	polrgx(0x7010, 0xffffffff, 0xffffffff);
+	polrgx(0x7014, 0x7, 0x7);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 8);
+	OSWriteHWReg32(pvRegsBaseKM, 0x7008, 0);
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 6);
+	polrgx(0x7000, 0x00010000, 0x00010000);
+	OSWaitus(100);
+
+	polrgx(0x75B0, 0, ~0U);
+	polrgx(0x75B4, 0, ~0U);
+	polrgx(0x75B8, 0, ~0U);
+	polrgx(0x75BC, 0, ~0U);
+	polrgx(0x75C0, 0, ~0U);
+	polrgx(0x75C4, 0, ~0U);
+	polrgx(0x75C8, 0, ~0U);
+	polrgx(0x75CC, 0, ~0U);
+
+	/* Sidekick */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7040, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7040, 0x10);
+	//OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7040, 4);
+		OSWaitus(100);
+		polrgx(0x7040, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7050, 0xffffffff, 0xffffffff);
+	polrgx(0x7054, 0xffffffff, 0xffffffff);
+	polrgx(0x7058, 0x1, 0x1);
+
+	/* USC */
+	for (instance = 0; instance < 4; instance++)
+	{
+		OSWriteHWReg32(pvRegsBaseKM, 0x8010, instance);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7088, 8);
+		OSWaitus(100);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7088, 0x10);
+		OSWaitus(100);
+
+		for (i = 0; i < 3; i++)
+		{
+			IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+			OSWriteHWReg32(pvRegsBaseKM, 0x7088, 4);
+			OSWaitus(100);
+			polrgx(0x7088, ui32Pol, ui32Pol);
+		}
+
+		OSWaitus(100);
+		polrgx(0x7098, 0xffffffff, 0xffffffff);
+		polrgx(0x709c, 0xffffffff, 0xffffffff);
+		polrgx(0x70a0, 0x3f, 0x3f);
+	}
+
+	/* tpumcul0 DustA and DustB */
+	for (instance = 0; instance < 2; instance++)
+	{
+		OSWriteHWReg32(pvRegsBaseKM, 0x8018, instance);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7380, 8);
+		OSWaitus(100);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7380, 0x10);
+		OSWaitus(100);
+
+		for (i = 0; i < 3; i++)
+		{
+			IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+			OSWriteHWReg32(pvRegsBaseKM, 0x7380, 4);
+			OSWaitus(100);
+			polrgx(0x7380, ui32Pol, ui32Pol);
+		}
+
+		OSWaitus(100);
+		polrgx(0x7390, 0x1fff, 0x1fff);
+	}
+
+	/* TA */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7500, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7500, 0x10);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7500, 4);
+		OSWaitus(100);
+		polrgx(0x7500, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7510, 0x1fffffff, 0x1fffffff);
+
+	/* Rasterisation */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7540, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7540, 0x10);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7540, 4);
+		OSWaitus(100);
+		polrgx(0x7540, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7550, 0xffffffff, 0xffffffff);
+	polrgx(0x7554, 0xffffffff, 0xffffffff);
+	polrgx(0x7558, 0xf, 0xf);
+
+	/* hub_bifpmache */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7588, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7588, 0x10);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7588, 4);
+		OSWaitus(100);
+		polrgx(0x7588, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7598, 0xffffffff, 0xffffffff);
+	polrgx(0x759c, 0xffffffff, 0xffffffff);
+	polrgx(0x75a0, 0x1111111f, 0x1111111f);
+
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32Size);
+}
+#endif /* !defined(VIRTUAL_PLATFORM) */
+#endif /* !defined(TC_APOLLO_TCF5) */
+
+#if defined(TC_APOLLO_TCF5)
+static void ApolloFPGAUpdateDUTClockFreq(SYS_DATA   *psSysData,
+                                         IMG_UINT32 *pui32CoreClockSpeed,
+                                         IMG_UINT32 *pui32MemClockSpeed)
+{
+#if defined(SUPPORT_FPGA_DUT_CLK_INFO)
+	IMG_UINT32 ui32Reg;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* DUT_CLK_INFO available only if SW_IF_VERSION >= 1 */
+	ui32Reg = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_SW_IF_VERSION);
+	ui32Reg = (ui32Reg & VERSION_MASK) >> VERSION_SHIFT;
+
+	if (ui32Reg >= 1)
+	{
+		ui32Reg = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CLK_INFO);
+
+		if ((ui32Reg != 0) && (ui32Reg != 0xbaadface) && (ui32Reg != 0xffffffff))
+		{
+			PVR_LOG(("TCF_CLK_CTRL_DUT_CLK_INFO = %08x", ui32Reg));
+			PVR_LOG(("Overriding provided DUT clock values: core %u, mem %u",
+			         *pui32CoreClockSpeed, *pui32MemClockSpeed));
+
+			*pui32CoreClockSpeed = ((ui32Reg & CORE_MASK) >> CORE_SHIFT) * 1000000;
+			*pui32MemClockSpeed = ((ui32Reg & MEM_MASK) >> MEM_SHIFT) * 1000000;
+		}
+	}
+#endif
+
+	PVR_LOG(("DUT clock values: core %u, mem %u",
+	         *pui32CoreClockSpeed, *pui32MemClockSpeed));
+}
+#endif
+
+static PVRSRV_ERROR SetClocks(SYS_DATA *psSysData,
+							  IMG_UINT32 ui32CoreClock,
+							  IMG_UINT32 ui32MemClock,
+							  IMG_UINT32 ui32SysClock)
+{
+	IMG_CPU_PHYADDR	sPLLRegCpuPBase;
+	void *pvPLLRegCpuVBase;
+	IMG_UINT32 ui32Value;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* Reserve the PLL register region and map the registers in */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_PLL_OFFSET, SYS_APOLLO_REG_PLL_SIZE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the PLL register region (%d)",
+				 __func__, eError));
+		return eError;
+	}
+	sPLLRegCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM) + SYS_APOLLO_REG_PLL_OFFSET;
+
+	pvPLLRegCpuVBase = OSMapPhysToLin(sPLLRegCpuPBase, SYS_APOLLO_REG_PLL_SIZE, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvPLLRegCpuVBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map PLL registers", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Modify the core clock */
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_CORE_CLK0, (ui32CoreClock / 1000000));
+
+	ui32Value = 0x1 << PLL_CORE_DRP_GO_SHIFT;
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_CORE_DRP_GO, ui32Value);
+
+	OSSleepms(600);
+
+	/* Modify the memory clock */
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_MEMIF_CLK0, (ui32MemClock / 1000000));
+
+	ui32Value = 0x1 << PLL_MEM_DRP_GO_SHIFT;
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_MEM_DRP_GO, ui32Value);
+
+	OSSleepms(600);
+#if defined(TC_APOLLO_TCF5)
+	/* Modify the sys clock */
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_SYSIF_CLK0, (ui32SysClock / 1000000));
+
+	ui32Value = 0x1 << PLL_MEM_DRP_GO_SHIFT;
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_SYS_DRP_GO, ui32Value);
+
+	OSSleepms(600);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32SysClock);
+#endif
+
+	/* Unmap and release the PLL registers since we no longer need access to them */
+	OSUnMapPhysToLin(pvPLLRegCpuVBase, SYS_APOLLO_REG_PLL_SIZE);
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_PLL_OFFSET, SYS_APOLLO_REG_PLL_SIZE);
+
+	return PVRSRV_OK;
+}
+
+static void ApolloHardReset(SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	IMG_UINT32 ui32PolValue;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+#if defined(TC_APOLLO_TCF5)
+#if defined(TC_APOLLO_TCF5_BVNC_NOT_SUPPORTED)
+	PVR_DPF((PVR_DBG_WARNING,"\n********************************************************************************"));
+	PVR_DPF((PVR_DBG_WARNING, "%s: You are using a core which is not supported on TCF5.", __func__));
+	PVR_DPF((PVR_DBG_WARNING, "A set of fallback clock frequencies will be used. Depending on the core, you"));
+	PVR_DPF((PVR_DBG_WARNING, "could experience some problems (driver running slower, failing after some time"));
+	PVR_DPF((PVR_DBG_WARNING, "or unable to start at all)."));
+	PVR_DPF((PVR_DBG_WARNING, "********************************************************************************\n"));
+#endif
+
+	/* Put everything under reset */
+	ui32Value = (0x1 << GLB_CLKG_EN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	OSSleepms(4);
+
+	ApolloFPGAUpdateDUTClockFreq(psSysData, &ui32CoreClockSpeed, &ui32MemClockSpeed);
+
+	if (SetClocks(psSysData, ui32CoreClockSpeed, ui32MemClockSpeed, ui32SysClockSpeed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to set the core/system/memory clocks",
+				 __func__));
+	}
+
+	/* Remove units from reset */
+	ui32Value |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << SCB_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << PDP2_RESETN_SHIFT);
+	ui32Value |= (0x1 << PDP1_RESETN_SHIFT);
+	ui32Value |= (0x1 << DDR_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DUT_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32PolValue = 0x4;
+#else
+	ui32Value = (0x1 << GLB_CLKG_EN_SHIFT);
+	ui32Value |= (0x1 << SCB_RESETN_SHIFT);
+	ui32Value |= (0x1 << PDP2_RESETN_SHIFT);
+	ui32Value |= (0x1 << PDP1_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DDR_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DUT_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32PolValue = 0x7;
+#endif
+
+	OSSleepms(4);
+
+	pol(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DCM_LOCK_STATUS, ui32PolValue, DCM_LOCK_STATUS_MASK);
+#undef pol
+#undef polrgx
+}
+
+#if defined(TC_APOLLO_TCF5)
+static void SetMemoryLatency(SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Latency = 0, ui32RegVal = 0;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	if (ui32MemReadLatency <= 4)
+	{
+		/* The total memory read latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a read.
+		 * Set the memory read latency to 0 cycles.
+		 */
+		ui32MemReadLatency = 0;
+	}
+	else
+	{
+		ui32MemReadLatency -= 4;
+
+		PVR_LOG(("%s: Setting memory read latency to %u cycles",
+		         __func__, ui32MemReadLatency));
+	}
+
+	if (ui32MemWriteResponseLatency <= 2)
+	{
+		/* The total memory write latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a write.
+		 * Set the memory write latency to 0 cycles.
+		 */
+		ui32MemWriteResponseLatency = 0;
+	}
+	else
+	{
+		ui32MemWriteResponseLatency -= 2;
+
+		PVR_LOG(("%s: Setting memory write response latency to %u cycles",
+		         __func__, ui32MemWriteResponseLatency));
+	}
+
+	ui32Latency = (ui32MemWriteResponseLatency << 16) | ui32MemReadLatency;
+
+	/* Configure latency register */
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0x1009, ui32Latency);
+
+	/* Read back latency register */
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, 0x1009, &ui32RegVal);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: SPI_Read failed (%u)", __func__, eError));
+	}
+
+	if (ui32RegVal != ui32Latency)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Register value mismatch (actual: 0x%08x, expected: 0x%08x)",
+		         __func__, ui32RegVal, ui32Latency));
+	}
+}
+#endif
+
+static PVRSRV_ERROR PCIInitDev(PVRSRV_DEVICE_CONFIG *psDevice, void *pvOSDevice)
+{
+	SYS_DATA *psSysData;
+	IMG_CPU_PHYADDR	sApolloRegCpuPBase;
+	IMG_UINT32 uiApolloRegSize;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevice);
+
+	if (psDevice->pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+#endif
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (!psSysData)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->hRGXPCI = OSPCISetDev(TO_PCI_COOKIE(pvOSDevice), HOST_PCI_INIT_FLAGS);
+	if (!psSysData->hRGXPCI)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire PCI device", __func__));
+		eError = PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+		goto ErrorFreeSysData;
+	}
+
+	/* Get Apollo register information */
+	sApolloRegCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM);
+	uiApolloRegSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM));
+
+	/* Check the address range is large enough. */
+	if (uiApolloRegSize < SYS_APOLLO_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Apollo register region isn't big enough "
+				 "(was 0x%08X, required 0x%08X)",
+				 __func__, uiApolloRegSize, SYS_APOLLO_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorPCIReleaseDevice;
+	}
+
+	/* The Apollo register range contains several register regions. Request only the system control register region */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the system register region",
+				 __func__));
+		goto ErrorPCIReleaseDevice;
+	}
+	psSysData->sSystemRegCpuPBase.uiAddr = sApolloRegCpuPBase.uiAddr + SYS_APOLLO_REG_SYS_OFFSET;
+	psSysData->uiSystemRegSize = SYS_APOLLO_REG_REGION_SIZE;
+
+	/* Setup Rogue register information */
+	psDevice->sRegsCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->ui32RegsSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM));
+
+	/* Save data for this device */
+	psDevice->hSysData = (IMG_HANDLE)psSysData;
+
+	/* Check the address range is large enough. */
+	if (psDevice->ui32RegsSize < SYS_RGX_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough "
+				 "(was 0x%08x, required 0x%08x)",
+				 __func__, psDevice->ui32RegsSize, SYS_RGX_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/* Reserve the address range */
+	eError = OSPCIRequestAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/*
+	   Map in system registers so we can:
+	   - Configure the memory mode (LMA, UMA or LMA/UMA hybrid)
+	   - Hard reset Apollo
+	   - Run BIST
+	   - Clear interrupts
+	*/
+	psSysData->pvSystemRegCpuVBase = OSMapPhysToLin(psSysData->sSystemRegCpuPBase,
+													psSysData->uiSystemRegSize,
+													PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (psSysData->pvSystemRegCpuVBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRGXRegReleaseAddrRange;
+	}
+
+	ApolloHardReset(psSysData);
+
+#if defined(TC_APOLLO_TCF5)
+	SetMemoryLatency(psSysData);
+#endif
+
+#if !defined(TC_APOLLO_TCF5)
+#if !defined(VIRTUAL_PLATFORM)
+	RGXInitBistery(psDevice->sRegsCpuPBase, psDevice->ui32RegsSize);
+	ApolloHardReset(psSysData);
+#endif
+	if (SetClocks(psSysData, ui32CoreClockSpeed, ui32MemClockSpeed, ui32SysClockSpeed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to set the core/system/memory clocks",
+				 __func__));
+	}
+#endif
+
+	eError = InitMemory(psDevice, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorUnMapAddrRange;
+	}
+
+#if defined(TC_APOLLO_TCF5)
+	((RGX_DATA *)psDevice->hDevData)->psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed;
+	OSSleepms(600);
+#else
+	if (! PVRSRV_VZ_MODE_IS(GUEST))
+	{
+		IMG_UINT32 ui32Value;
+		/* Enable the rogue PLL (defaults to 3x), giving a Rogue clock of 3 x RGX_TC_CORE_CLOCK_SPEED */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1);
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1, ui32Value & 0xFFFFFFFB);
+
+		((RGX_DATA *)psDevice->hDevData)->psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed * 3;
+
+		OSSleepms(600);
+
+		/* Enable the temperature sensor */
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0x3, 0x46);
+	}
+
+	{
+		IMG_UINT32 ui32HoodCtrl;
+		SPI_Read(psSysData->pvSystemRegCpuVBase, 0x9, &ui32HoodCtrl);
+		ui32HoodCtrl |= 0x1;
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0x9, ui32HoodCtrl);
+	}
+#endif
+
+	psSysData->pszVersion = GetDeviceVersionString(psSysData);
+	if (psSysData->pszVersion)
+	{
+		psDevice->pszVersion = psSysData->pszVersion;
+	}
+
+	eError = OSPCIIRQ(psSysData->hRGXPCI, &psSysData->ui32IRQ);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't get IRQ", __func__));
+		goto ErrorDeInitMemory;
+	}
+
+	/* Register our handler */
+	eError = OSInstallSystemLISR(&psSysData->hLISR,
+								psSysData->ui32IRQ,
+								psDevice->pszName,
+								SystemISRHandler,
+								psSysData,
+								SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to install the system device interrupt handler",
+				 __func__));
+		goto ErrorDeInitMemory;
+	}
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevice->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevice->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevice->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevice->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevice->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevice->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevice->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevice->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	psDevice->pvOSDevice = pvOSDevice;
+
+	return PVRSRV_OK;
+
+ErrorDeInitMemory:
+	DeInitMemory(psDevice, psSysData);
+
+ErrorUnMapAddrRange:
+	OSUnMapPhysToLin(psSysData->pvSystemRegCpuVBase, psSysData->uiSystemRegSize);
+	psSysData->pvSystemRegCpuVBase = NULL;
+
+ErrorRGXRegReleaseAddrRange:
+	OSPCIReleaseAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->sRegsCpuPBase.uiAddr	= 0;
+	psDevice->ui32RegsSize		= 0;
+
+ErrorSystemRegReleaseAddrRegion:
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	psSysData->sSystemRegCpuPBase.uiAddr	= 0;
+	psSysData->uiSystemRegSize		= 0;
+
+ErrorPCIReleaseDevice:
+	OSPCIReleaseDev(psSysData->hRGXPCI);
+	psSysData->hRGXPCI = NULL;
+
+ErrorFreeSysData:
+	OSFreeMem(psSysData);
+	psDevice->hSysData = NULL;
+
+	return eError;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevice,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	PVRSRV_ERROR	eError;
+	SYS_DATA		*psSysData = psDevice->hSysData;
+	IMG_UINT32		ui32RegOffset;
+	IMG_UINT32		ui32RegVal;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	/* Read the temperature */
+	ui32RegOffset = 0x5;
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, ui32RegOffset, &ui32RegVal);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SysDebugInfo: SPI_Read failed for register 0x%x (0x%x)", ui32RegOffset, eError));
+		goto SysDebugInfo_exit;
+	}
+
+	PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", (ui32RegVal * 233 / 4096) - 66);
+
+SysDebugInfo_exit:
+	return eError;
+#endif
+}
+#endif /* if defined(TC_APOLLO_ES2) else TC_APOLLO_TCF5 */
+
+/*
+ * GetMemorySize:
+ * For a given PCI phys addr and size, go through the memory to check the actual size.
+ * Do this by writing to address zero, and then checking positions in the memory to see if there
+ * is any aliasing. e.g. some devices may advertise 1GB of memory but in reality only have half that.
+ */
+size_t GetMemorySize(IMG_CPU_PHYADDR sMemCpuPhyAddr, IMG_UINT64 pciBarSize)
+{
+#define MEM_PROBE_VALUE 0xa5a5a5a5
+	void *pvHostMapped;
+	volatile IMG_UINT32* pMemZero;
+	size_t mem_size = 0;
+
+	PVR_ASSERT((pciBarSize & (pciBarSize-1)) == 0);   // PCI BARs are always a power of 2
+	pvHostMapped = OSMapPhysToLin(sMemCpuPhyAddr, pciBarSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvHostMapped == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot map PCI memory at phys 0x%"\
+				IMG_UINT64_FMTSPECx" for size 0x%"IMG_UINT64_FMTSPECx,\
+				__func__, (IMG_UINT64) sMemCpuPhyAddr.uiAddr, pciBarSize));
+	}
+	else
+	{
+		pMemZero = (IMG_UINT32*)pvHostMapped;
+		mem_size = 1024*1024;    // assume at least 1MB of memory
+		while (mem_size < pciBarSize)
+		{
+			volatile IMG_UINT32* pMemProbe = pMemZero + mem_size/4;
+
+			// check that the memory at offset actually exists and is not aliased
+			*pMemProbe = MEM_PROBE_VALUE;
+			*pMemZero = 0;
+			if (*pMemProbe != MEM_PROBE_VALUE)
+				break;
+			*pMemProbe = 0;     // reset value
+			mem_size <<= 1;     // double the offset we probe each time.
+		}
+		OSUnMapPhysToLin(pvHostMapped, pciBarSize);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Local memory size 0x" IMG_SIZE_FMTSPECX " (PCI BAR size 0x%llx)", __func__, mem_size, pciBarSize));
+	}
+
+	return mem_size;
+}
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL) || (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static PVRSRV_ERROR AcquireLocalMemory(SYS_DATA *psSysData, IMG_CPU_PHYADDR *psMemCpuPAddr, size_t *puiMemSize)
+{
+	IMG_UINT16 uiVendorID, uiDevID;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMemSize;
+	IMG_UINT32 ui32Value;
+	IMG_UINT32 uiMemLimit;
+	IMG_UINT32 ui32PCIVersion;
+	void *pvHostFPGARegCpuVBase;
+	IMG_CPU_PHYADDR	sHostFPGARegCpuPBase;
+
+	OSPCIGetVendorDeviceIDs(psSysData->hRGXPCI, &uiVendorID, &uiDevID);
+	if (uiDevID != SYS_RGX_DEV_DEVICE_ID)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected device ID 0x%X",
+				 __func__, uiDevID));
+		return PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+	}
+
+	/* To get some of the version information we need to read from a register that we don't normally have
+	   mapped. Map it temporarily (without trying to reserve it) to get the information we need. */
+	sHostFPGARegCpuPBase.uiAddr	= OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM) + 0x40F0;
+	pvHostFPGARegCpuVBase = OSMapPhysToLin(sHostFPGARegCpuPBase, 0x04, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	ui32Value = OSReadHWReg32(pvHostFPGARegCpuVBase, 0);
+
+	/* Unmap the register now that we no longer need it */
+	OSUnMapPhysToLin(pvHostFPGARegCpuVBase, 0x04);
+
+	ui32PCIVersion = HEX2DEC((ui32Value & 0x00FF0000) >> 16);
+
+	if (ui32PCIVersion < 18)
+	{
+		PVR_DPF((PVR_DBG_WARNING,"\n********************************************************************************"));
+		PVR_DPF((PVR_DBG_WARNING, "%s: You have an outdated test chip fpga image.", __func__));
+		PVR_DPF((PVR_DBG_WARNING, "Restricting available graphics memory to 512mb as a workaround."));
+		PVR_DPF((PVR_DBG_WARNING, "This restriction may cause test failures for those tests"));
+		PVR_DPF((PVR_DBG_WARNING, "that require a large amount of graphics memory."));
+		PVR_DPF((PVR_DBG_WARNING, "Please speak to your customer support representative about"));
+		PVR_DPF((PVR_DBG_WARNING, "upgrading your test chip fpga image."));
+		PVR_DPF((PVR_DBG_WARNING, "********************************************************************************\n"));
+
+		/* limit to 512mb */
+		uiMemLimit = 512 * 1024 * 1024;
+		uiMemSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+	}
+	else
+	{
+		IMG_CPU_PHYADDR sMemCpuPhyAddr;
+		IMG_UINT64 pciBarSize = OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM);
+		sMemCpuPhyAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+
+		uiMemSize = TRUNCATE_64BITS_TO_32BITS(GetMemorySize(sMemCpuPhyAddr, pciBarSize) - SYS_DEV_MEM_BROKEN_BYTES);
+		uiMemLimit = SYS_DEV_MEM_REGION_SIZE;
+	}
+
+	if (uiMemLimit > uiMemSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device memory region smaller than requested "
+				 "(got 0x%08X, requested 0x%08X)",
+				 __func__, uiMemSize, uiMemLimit));
+	}
+	else if (uiMemLimit < uiMemSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Limiting device memory region size to 0x%08X from 0x%08X",
+				 __func__, uiMemLimit, uiMemSize));
+		uiMemSize = uiMemLimit;
+	}
+
+	/* Reserve the address region */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM, 0, uiMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device memory region not available",
+				 __func__));
+		return eError;
+	}
+
+	/* Clear any BIOS-configured MTRRs */
+	eError = OSPCIClearResourceMTRRs(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Failed to clear BIOS MTRRs",
+				 __func__));
+		/* Soft-fail, the driver can limp along. */
+	}
+
+	psMemCpuPAddr->uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+	*puiMemSize = (size_t)uiMemSize;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void ReleaseLocalMemory(SYS_DATA *psSysData, IMG_CPU_PHYADDR *psMemCpuPAddr, size_t uiMemSize)
+{
+	IMG_CPU_PHYADDR sMemCpuPBaseAddr;
+	IMG_UINT32 uiOffset;
+
+	sMemCpuPBaseAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+
+	PVR_ASSERT(psMemCpuPAddr->uiAddr >= sMemCpuPBaseAddr.uiAddr);
+
+	uiOffset = (IMG_UINT32)(uintptr_t)(psMemCpuPAddr->uiAddr - sMemCpuPBaseAddr.uiAddr);
+
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM, uiOffset, (IMG_UINT32)uiMemSize);
+
+	OSPCIReleaseResourceMTRRs(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM);
+}
+
+static INLINE PVRSRV_ERROR InitLocalMemoryHeaps(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT64 ui64CardBase = 0;
+	IMG_UINT32 ui32HeapNum = 0;
+	PVRSRV_ERROR eError;
+
+	eError = AcquireLocalMemory(psSysData, &psSysData->sLocalMemCpuPBase, &psSysData->uiLocalMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		/* In VZ mode, allocate LMA space for the shared cross-VM information page */
+		psSysData->pui32IRQCount = (void*) OSMapPhysToLin(psSysData->sLocalMemCpuPBase,
+		                                                  (IMG_UINT64)VM_INFOPG_SIZE,
+		                                                  PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+		if (psSysData->pui32IRQCount == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map local memory page",
+					__func__));
+			return PVRSRV_ERROR_MAPPING_NOT_FOUND;
+		}
+
+		/* Initialise the only cross-VM information page entry to zero */
+		(void) OSAtomicWrite((void*)psSysData->pui32IRQCount, 0);
+
+		/* Update LMA memory range spec. to reflect above deduction */
+		psSysData->uiLocalMemSize -= (IMG_UINT64)VM_INFOPG_SIZE;
+		ui64CardBase = (IMG_UINT64)VM_INFOPG_SIZE;
+	}
+
+	/* Setup the Rogue LMA heap */
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr =
+			psSysData->sLocalMemCpuPBase.uiAddr;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr =
+#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+			psSysData->sLocalMemCpuPBase.uiAddr +
+#endif
+			ui64CardBase;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize =
+			psSysData->uiLocalMemSize
+#if defined(SUPPORT_DISPLAY_CLASS)
+			- RGX_TC_RESERVE_DC_MEM_SIZE
+#endif
+			;
+	ui32HeapNum++;
+
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	/* Setup the DC heap */
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr =
+		psDevConfig->pasPhysHeaps[ui32HeapNum - 1].sStartAddr.uiAddr +
+		ui64CardBase + psDevConfig->pasPhysHeaps[ui32HeapNum - 1].uiSize;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr =
+#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+		psSysData->sLocalMemCpuPBase.uiAddr +
+#endif
+		ui64CardBase + psDevConfig->pasPhysHeaps[ui32HeapNum - 1].uiSize;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize = RGX_TC_RESERVE_DC_MEM_SIZE;
+	ui32HeapNum++;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static INLINE void DeInitLocalMemoryHeaps(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32HeapNum = 0;
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		OSUnMapPhysToLin(psSysData->pui32IRQCount, VM_INFOPG_SIZE);
+	}
+
+	ReleaseLocalMemory(psSysData, &psSysData->sLocalMemCpuPBase, psSysData->uiLocalMemSize);
+
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize = 0;
+	ui32HeapNum++;
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize = 0;
+	ui32HeapNum++;
+#endif
+}
+#endif /* (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL) || (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID) */
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static PVRSRV_ERROR InitLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalMemoryHeaps(psDevConfig, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* No additional setup needed by guests drivers */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* Configure Apollo for regression compatibility (i.e. local memory) mode */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	ui32Value &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK | HOST_PHY_MODE_MASK);
+	ui32Value |= (0x1 << ADDRESS_FORCE_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+
+	return PVRSRV_OK;
+}
+
+static void DeInitLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	DeInitLocalMemoryHeaps(psDevConfig, psSysData);
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set the register back to the default value */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, 0x1 << ADDRESS_FORCE_SHIFT);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+}
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static PVRSRV_ERROR InitHostMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* Configure Apollo for host only mode */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	ui32Value &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK | HOST_PHY_MODE_MASK);
+	ui32Value |= (0x1 << HOST_ONLY_MODE_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+
+	return PVRSRV_OK;
+}
+
+static void DeInitHostMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set the register back to the default value */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, 0x1 << ADDRESS_FORCE_SHIFT);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+}
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHybridMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalMemoryHeaps(psDevConfig, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* No additional setup needed for guest driver(s) */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* No additional setup needed for the Rogue UMA heap */
+
+	/* Configure Apollo for host physical (i.e. hybrid) mode */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	ui32Value &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK | HOST_PHY_MODE_MASK);
+	ui32Value |= ((0x1 << HOST_ONLY_MODE_SHIFT) | (0x1 << HOST_PHY_MODE_SHIFT));
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+
+	/* Setup the start address of the 1GB window used by the _GPU_ to access local memory.
+	 * The value used here is the start address of the 1GB window used by the _CPU_
+	 * to access the local memory on the TC. This means that both CPU and GPU will use
+	 * the same range of physical addresses to access local memory, and any access
+	 * out of that range will be directed to system memory instead.
+	 * In the hybrid config case the conversion cpuPaddr <-> devPaddr is an assignment.
+	 */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase,
+					TCF_CLK_CTRL_HOST_PHY_OFFSET,
+					psSysData->sLocalMemCpuPBase.uiAddr);
+	OSWaitus(10);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_HOST_PHY_OFFSET);
+	OSWaitus(10);
+
+	return PVRSRV_OK;
+}
+
+static void DeInitHybridMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	DeInitLocalMemoryHeaps(psDevConfig, psSysData);
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set the register back to the default value */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, 0x1 << ADDRESS_FORCE_SHIFT);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+}
+#endif
+
+static PVRSRV_ERROR InitMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+	eError = InitLocalMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+	eError = InitHostMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	eError = InitHybridMemory(psDevConfig, psSysData);
+#endif
+
+	return eError;
+}
+
+static INLINE void DeInitMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+	DeInitLocalMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+	DeInitHostMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	DeInitHybridMemory(psDevConfig, psSysData);
+#endif
+}
+
+static void EnableInterrupt(SYS_DATA *psSysData, IMG_UINT32 ui32InterruptFlag)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set sense to active high */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_OP_CFG);
+	ui32Value &= ~(INT_SENSE_MASK);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_OP_CFG, ui32Value);
+	OSWaitus(1000);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_OP_CFG);
+
+	/* Enable Rogue and PDP1 interrupts */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	ui32Value |= ui32InterruptFlag;
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	OSWaitus(10);
+}
+
+static void DisableInterrupt(SYS_DATA *psSysData, IMG_UINT32 ui32InterruptFlag)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	ui32Value &= ~(ui32InterruptFlag);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	OSWaitus(10);
+}
+
+static void PCIDeInitDev(PVRSRV_DEVICE_CONFIG *psDevice)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevice->hSysData;
+
+	psDevice->hSysData = NULL;
+	psDevice->pszVersion = NULL;
+
+	OSUninstallSystemLISR(psSysData->hLISR);
+
+	if (psSysData->pszVersion)
+	{
+		OSFreeMem(psSysData->pszVersion);
+		psSysData->pszVersion = NULL;
+	}
+
+	DeInitMemory(psDevice, psSysData);
+
+	OSUnMapPhysToLin(psSysData->pvSystemRegCpuVBase, psSysData->uiSystemRegSize);
+	OSPCIReleaseAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+
+	OSPCIReleaseDev(psSysData->hRGXPCI);
+
+	if (psSysData->pszVersion)
+	{
+		OSFreeMem(psSysData->pszVersion);
+	}
+
+	OSFreeMem(psSysData);
+
+	psDevice->pvOSDevice = NULL;
+}
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr - psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr - psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+		}
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr + psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr + psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+		}
+	}
+}
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static void TCSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void TCSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+}
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* The CPU and GPU physical addresses are always the same in Hybrid model
+	 * However on 64 bit platforms, RAM mapped above 4G of processor physical address space
+	 * cannot be accessed by this version of the test chip due to its 32bit interface.
+	 *
+	 * In fact its possible to further optimise this function by eliminating this
+	 * unnecessary copy
+	*/
+#if defined(CONFIG_64BIT)
+	PVR_ASSERT(sizeof(IMG_CPU_PHYADDR) == sizeof(IMG_DEV_PHYADDR));
+	OSCachedMemCopy(psDevPAddr, psCpuPAddr, (ui32NumOfAddr * sizeof(*psCpuPAddr)));
+#else
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+#endif
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* The CPU and GPU physical addresses are always the same in Hybrid model
+	 * However on 64 bit platforms, RAM mapped above 4G of processor physical address space
+	 * cannot be accessed by this version of the test chip due to its 32bit interface.
+	 *
+	 * In fact its possible to further optimise this function by eliminating this
+	 * unnecessary copy
+	*/
+
+#if defined(CONFIG_64BIT)
+	PVR_ASSERT(sizeof(IMG_CPU_PHYADDR) == sizeof(IMG_DEV_PHYADDR));
+	OSCachedMemCopy(psCpuPAddr, psDevPAddr, (ui32NumOfAddr * sizeof(*psCpuPAddr)));
+#else
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+#endif
+
+}
+
+#endif /* (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID) */
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+// #define _DBG(...) PVR_LOG((__VA_ARGS__))
+#define _DBG(...)
+
+static PVRSRV_ERROR PrePower(IMG_HANDLE hSysData,
+                             PVRSRV_SYS_POWER_STATE eNewPowerState,
+                             PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                             PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize;
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags: 0x%08x", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags);
+
+	/* The transition might be both from ON or OFF states to OFF state so check
+	 * only for the *new* state. Also this is only valid for suspend requests. */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_SUSPEND_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* disable interrupts */
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		IMG_UINT32 uiIntFlag = psSysData->sInterruptData[i].ui32InterruptFlag;
+		if (uiIntFlag != 0)
+		{
+			DisableInterrupt(psSysData, uiIntFlag);
+		}
+	}
+
+	/* dump video RAM content to system RAM */
+
+	eError = LMA_HeapIteratorCreate(psSysData->psDevConfig->psDevNode,
+	                                PHYS_HEAP_USAGE_GPU_LOCAL,
+	                                &psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorCreate", enable_interrupts);
+
+	eError = LMA_HeapIteratorGetHeapStats(psSysData->psHeapIter, &uiHeapTotalSize,
+	                                      &uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorGetHeapStats",
+	                      enable_interrupts);
+	uiHeapFreeSize = uiHeapTotalSize - uiHeapUsedSize;
+
+	_DBG("(%s()) heap stats: total=0x%" IMG_UINT64_FMTSPECx ", "
+	     "used=0x%" IMG_UINT64_FMTSPECx ", free=0x%" IMG_UINT64_FMTSPECx,
+	     __func__, uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize);
+
+	psSysData->pvSuspendBuffer = OSAllocMem(uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_NOMEM(psSysData->pvSuspendBuffer, eError, destroy_iterator);
+
+	_DBG("(%s()) buffer=%px", __func__, psSysData->pvSuspendBuffer);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		if (uiOffset + uiSize > uiHeapUsedSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "uiOffset = %" IMG_UINT64_FMTSPECx ", "
+			        "uiSize = %" IMG_UINT64_FMTSPECx, uiOffset, uiSize));
+
+			PVR_LOG_GOTO_WITH_ERROR("LMA_HeapIteratorNext", eError,
+			                        PVRSRV_ERROR_INVALID_OFFSET,
+			                        free_buffer);
+		}
+
+		TCHybridDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                           &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy((IMG_BYTE *) psSysData->pvSuspendBuffer + uiOffset, pvCpuVAddr,
+		       uiSize);
+#ifdef DEBUG
+		/* and now poison it - for debugging purposes so we know that the
+		 * resume procedure works */
+		memset(pvCpuVAddr, 0x9b, uiSize);
+#endif
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	/* suspend the PCI device */
+	OSPCISuspendDev(psSysData->hRGXPCI);
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvSuspendBuffer);
+	psSysData->pvSuspendBuffer = NULL;
+destroy_iterator:
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+enable_interrupts:
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		IMG_UINT32 uiIntFlag = psSysData->sInterruptData[i].ui32InterruptFlag;
+		if (uiIntFlag != 0)
+		{
+			DisableInterrupt(psSysData, uiIntFlag);
+		}
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR PostPower(IMG_HANDLE hSysData,
+                              PVRSRV_SYS_POWER_STATE eNewPowerState,
+                              PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags=0x%08x; buffer=%px", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags,
+	     psSysData->pvSuspendBuffer);
+
+	/* restore the video RAM */
+
+	/* The transition might be both to ON or OFF states from OFF state so check
+	 * only for the *current* state. Also this is only valid for resume requests. */
+	if ((eCurrentPowerState != eNewPowerState && eCurrentPowerState != PVRSRV_SYS_POWER_STATE_OFF) ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_RESUME_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* resume the PCI device */
+	OSPCIResumeDev(psSysData->hRGXPCI);
+
+	if (psSysData->pvSuspendBuffer != NULL)
+	{
+		eError = LMA_HeapIteratorReset(psSysData->psHeapIter);
+		PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorReset", free_buffer);
+
+		while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+		{
+			void *pvCpuVAddr;
+			IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+			TCHybridDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+			                           &sDevPAddr);
+
+			pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+			                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+			_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+			     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+			     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+			     pvCpuVAddr, uiSize);
+
+			/* copy memory */
+			memcpy(pvCpuVAddr, (IMG_BYTE *) psSysData->pvSuspendBuffer + uiOffset,
+			       uiSize);
+
+			uiOffset += uiSize;
+
+			OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+		}
+
+		OSFreeMem(psSysData->pvSuspendBuffer);
+		psSysData->pvSuspendBuffer = NULL;
+	}
+
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+
+	/* reset the TestChip */
+
+	eError = HardResetDevice(psSysData, psSysData->psDevConfig);
+	PVR_GOTO_IF_ERROR(eError, free_buffer);
+
+	/* enable interrupts */
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		IMG_UINT32 uiIntFlag = psSysData->sInterruptData[i].ui32InterruptFlag;
+		if (uiIntFlag != 0)
+		{
+			EnableInterrupt(psSysData, uiIntFlag);
+		}
+	}
+
+	/* reset DC devices */
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = psSysData->psDevConfig->psDevNode;
+		IMG_UINT32 uiDeviceCount = 0, auiDeviceIndex[16];
+
+		eError = DCDevicesEnumerate(NULL, psDevNode, ARRAY_SIZE(auiDeviceIndex),
+		                            &uiDeviceCount, auiDeviceIndex);
+		PVR_LOG_IF_ERROR(eError, "DCDevicesEnumerate");
+
+		for (i = 0; i < uiDeviceCount; i++)
+		{
+			DC_DEVICE *psDCDevice;
+			eError = DCDeviceAcquire(NULL, psDevNode, auiDeviceIndex[i],
+			                         &psDCDevice);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "DCDeviceAcquire");
+				continue;
+			}
+
+			eError = DCResetDevice(psDCDevice);
+			PVR_LOG_IF_ERROR(eError, "DCResetDevice");
+
+			eError = DCDeviceRelease(psDCDevice);
+			PVR_LOG_IF_ERROR(eError, "DCDeviceRelease");
+		}
+	}
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvSuspendBuffer);
+	psSysData->pvSuspendBuffer = NULL;
+
+	return eError;
+}
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = &gsDevices[0];
+	PVRSRV_ERROR eError;
+
+	eError = PCIInitDev(psDevConfig, pvOSDevice);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	/* power functions */
+	psDevConfig->pfnPrePowerState = PrePower;
+	psDevConfig->pfnPostPowerState = PostPower;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PCIDeInitDev(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32InterruptFlag;
+
+	switch (ui32IRQ)
+	{
+		case APOLLO_IRQ_GPU:
+			ui32InterruptFlag = (0x1 << EXT_INT_SHIFT);
+			break;
+		case APOLLO_IRQ_PDP:
+			ui32InterruptFlag = (0x1 << PDP1_INT_SHIFT);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d",
+					 __func__, ui32IRQ));
+			return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	if (psSysData->sInterruptData[ui32IRQ].pfnLISR)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ISR for %s already installed!",
+				 __func__, pszName));
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+	else if (ui32IRQ > ARRAY_SIZE(psSysData->sInterruptData))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Invalid IRQ %d, exceeds max. IRQ " IMG_SIZE_FMTSPEC "!",
+				 __func__, ui32IRQ, ARRAY_SIZE(psSysData->sInterruptData)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psSysData->sInterruptData[ui32IRQ].psSysData	= psSysData;
+	psSysData->sInterruptData[ui32IRQ].pszName		= pszName;
+	psSysData->sInterruptData[ui32IRQ].pfnLISR		= pfnLISR;
+	psSysData->sInterruptData[ui32IRQ].pvData		= pvData;
+	psSysData->sInterruptData[ui32IRQ].ui32InterruptFlag = ui32InterruptFlag;
+
+	*phLISRData = &psSysData->sInterruptData[ui32IRQ];
+
+	EnableInterrupt(psSysData, psSysData->sInterruptData[ui32IRQ].ui32InterruptFlag);
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, psSysData->ui32IRQ));
+
+	return eError;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	SYS_INTERRUPT_DATA *psInterruptData = (SYS_INTERRUPT_DATA *)hLISRData;
+	SYS_DATA *psSysData = psInterruptData->psSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psInterruptData);
+
+	PVR_LOG(("Uninstalling device LISR %s on IRQ %d", psInterruptData->pszName, psSysData->ui32IRQ));
+
+	/* Disable interrupts for this device */
+	DisableInterrupt(psInterruptData->psSysData, psInterruptData->ui32InterruptFlag);
+
+	/* Reset the interrupt data */
+	psInterruptData->pszName = NULL;
+	psInterruptData->psSysData = NULL;
+	psInterruptData->pfnLISR = NULL;
+	psInterruptData->pvData = NULL;
+	psInterruptData->ui32InterruptFlag = 0;
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.h
new file mode 100644
index 000000000000..7f4172ad6d9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.h
@@ -0,0 +1,318 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSCONFIG_H__)
+#define __SYSCONFIG_H__
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+/* Valid values for TC_MEMORY_CONFIG configuration option */
+#define TC_MEMORY_LOCAL			(1)
+#define TC_MEMORY_HOST			(2)
+#define TC_MEMORY_HYBRID		(3)
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+/* Memory reserved for use by the PDP DC. */
+#define RGX_TC_RESERVE_DC_MEM_SIZE	((TC_DISPLAY_MEM_SIZE) * 1024 * 1024)
+#endif
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+static PVRSRV_DEVICE_CONFIG gsDevices[];
+
+static RGX_TIMING_INFORMATION gsRGXTimingInfo =
+{
+	/* ui32CoreClockSpeed */
+	0,	/* Initialize to 0, real value will be set in PCIInitDev() */
+	/* bEnableActivePM */
+	IMG_TRUE,
+	/* bEnableRDPowIsland */
+	IMG_FALSE,
+	/* ui32ActivePMLatencyms */
+	SYS_RGX_ACTIVE_POWER_LATENCY_MS
+};
+
+static RGX_DATA gsRGXData =
+{
+	/* psRGXTimingInfo */
+	&gsRGXTimingInfo
+};
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr);
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static void TCSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr);
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr);
+
+#endif /* (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID) */
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	TCLocalCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	TCLocalDevPAddrToCpuPAddr,
+};
+
+static PHYS_HEAP_CONFIG	gsPhysHeapConfig[] =
+{
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsLocalPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_GPU_LOCAL,
+	},
+#if defined(SUPPORT_DISPLAY_CLASS)
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsLocalPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_DISPLAY,
+	},
+#endif
+};
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static PHYS_HEAP_FUNCTIONS gsSystemPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	TCSystemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	TCSystemDevPAddrToCpuPAddr,
+};
+
+static PHYS_HEAP_CONFIG	gsPhysHeapConfig[] =
+{
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_UMA,
+		/* pszPDumpMemspaceName */
+		"SYSMEM",
+		/* psMemFuncs */
+		&gsSystemPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_GPU_LOCAL,
+	}
+};
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	TCHybridCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	TCHybridDevPAddrToCpuPAddr,
+};
+
+static PHYS_HEAP_CONFIG	gsPhysHeapConfig[] =
+{
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsHybridPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_GPU_LOCAL,
+	},
+#if defined(SUPPORT_DISPLAY_CLASS)
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsHybridPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_DISPLAY,
+	},
+#endif
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_UMA,
+		/* pszPDumpMemspaceName */
+		"SYSMEM",
+		/* psMemFuncs */
+		&gsHybridPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_CPU_LOCAL,
+	}
+};
+#else
+#error "TC_MEMORY_CONFIG not valid"
+#endif
+
+static PVRSRV_DEVICE_CONFIG gsDevices[] =
+{
+	{
+		.pvOSDevice		= NULL,
+		.psDevNode		= NULL,
+		.pszName		= "apollo",
+		.pszVersion		= NULL,
+
+		/* Device setup information */
+		.sRegsCpuPBase		= { 0 },
+		.ui32RegsSize		= 0,
+		.ui32IRQ			= 0,
+
+		.eCacheSnoopingMode	= PVRSRV_DEVICE_SNOOP_NONE,
+
+		.hDevData			= &gsRGXData,
+		.hSysData			= NULL,
+
+		.bHasNonMappableLocalMemory	= IMG_FALSE,
+
+		/* Physical memory heaps */
+		.pasPhysHeaps		= &gsPhysHeapConfig[0],
+		.ui32PhysHeapCount	= ARRAY_SIZE(gsPhysHeapConfig),
+
+		.pfnPrePowerState	= NULL,
+		.pfnPostPowerState	= NULL,
+		.bHasFBCDCVersion31	= IMG_FALSE,
+
+		/* Only required for LMA but having this always set shouldn't be a problem */
+		.bDevicePA0IsValid	= IMG_TRUE,
+
+		.pfnClockFreqGet	= NULL,
+
+		.pfnCheckMemAllocSize = NULL,
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		.pfnTDSendFWImage		= NULL,
+		.pfnTDSetPowerParams	= NULL,
+		.pfnTDRGXStart			= NULL,
+		.pfnTDRGXStop			= NULL,
+#endif
+		.pfnSysDevFeatureDepInit	= NULL
+	}
+};
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* !defined(__SYSCONFIG_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysdata.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysdata.h
new file mode 100644
index 000000000000..0a0fed814ae9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysdata.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Data Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv_device.h"
+#include "interrupt_support.h"
+
+#if !defined(__SYSDATA_H__)
+#define __SYSDATA_H__
+
+typedef struct _SYS_INTERRUPT_DATA_
+{
+	void			*psSysData;
+	const IMG_CHAR	*pszName;
+	PFN_SYS_LISR	pfnLISR;
+	void			*pvData;
+	IMG_UINT32		ui32InterruptFlag;
+	IMG_UINT32		ui32IRQ;
+} SYS_INTERRUPT_DATA;
+
+typedef struct _SYS_DATA_
+{
+	IMG_HANDLE			hRGXPCI;
+
+	IMG_CHAR			*pszVersion;
+
+	IMG_CPU_PHYADDR		sSystemRegCpuPBase;
+	void				*pvSystemRegCpuVBase;
+	size_t				uiSystemRegSize;
+
+	IMG_CPU_PHYADDR		sLocalMemCpuPBase;
+	size_t				uiLocalMemSize;
+
+	IMG_HANDLE			hLISR;
+	IMG_UINT32			ui32IRQ;
+
+	/* Used by VZ host/guest(s) drivers */
+	IMG_UINT32			*pui32IRQCount;
+
+	/* Rogue and PDP */
+	SYS_INTERRUPT_DATA	sInterruptData[2];
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	void *pvSuspendBuffer;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+} SYS_DATA;
+
+#endif /* !defined(__SYSDATA_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysinfo.h
new file mode 100644
index 000000000000..042b4e8ceb5a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysinfo.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+#define SYS_RGX_DEV_VENDOR_ID	(0x1010)
+#define SYS_RGX_DEV_DEVICE_ID	(0x1CF2)
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM)
+#define MAX_HW_TIME_US                           (5000000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#endif
+
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/env/linux/dma_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/env/linux/dma_support.c
new file mode 100644
index 000000000000..977f9c85d037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/env/linux/dma_support.c
@@ -0,0 +1,335 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#endif
+
+#include "allocmem.h"
+#include "dma_support.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 8
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+static IMG_UINT32 gsDmaIoRemapRef[DMA_MAX_IOREMAP_ENTRIES] = {0};
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDmaAlloc != NULL && psDmaAlloc->pvOSDevice != NULL)
+	{
+#if defined(__linux__)
+		psDmaAlloc->pvVirtAddr =
+				dma_alloc_coherent((struct device *)psDmaAlloc->pvOSDevice,
+								   (size_t) psDmaAlloc->ui64Size,
+								   (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr,
+								   GFP_KERNEL);
+		PVR_LOG_RETURN_IF_FALSE((NULL != psDmaAlloc->pvVirtAddr), "dma_alloc_coherent() failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+#else
+		#error "Provide OS implementation of DMA allocation";
+#endif
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	if (psDmaAlloc && psDmaAlloc->pvVirtAddr)
+	{
+#if defined(__linux__)
+		dma_free_coherent((struct device *)psDmaAlloc->pvOSDevice,
+						  (size_t) psDmaAlloc->ui64Size,
+						  psDmaAlloc->pvVirtAddr,
+						  (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+#else
+		#error "Provide OS implementation of DMA deallocation";
+#endif
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (psDmaAlloc->pvVirtAddr == NULL)
+	{
+		/* Check if an I/O remap entry already exists for this request */
+		for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+		{
+			if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+			{
+				PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr);
+				break;
+			}
+		}
+
+		if (ui32Idx < DMA_MAX_IOREMAP_ENTRIES)
+		{
+			IMG_UINT64 ui64Offset;
+			ui64Offset = psDmaAlloc->sBusAddr.uiAddr - gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr;
+			psDmaAlloc->pvVirtAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr + (uintptr_t)ui64Offset;
+			gsDmaIoRemapRef[ui32Idx] += 1;
+			return PVRSRV_OK;
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check if there is a free I/O remap table entry for this request */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		return eError;
+	}
+
+	gsDmaIoRemapArray[ui32Idx].ui64Size = psDmaAlloc->ui64Size;
+	gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+	gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+	gsDmaIoRemapRef[ui32Idx] += 1;
+
+	PVR_LOG(("DMA: register I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+			psDmaAlloc->pvVirtAddr,
+			psDmaAlloc->sBusAddr.uiAddr,
+			psDmaAlloc->ui64Size));
+
+	gbEnableDmaIoRemapping = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return;
+	}
+
+	/* Remove specified entry from the list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+		{
+			if (! --gsDmaIoRemapRef[ui32Idx])
+			{
+				PVR_LOG(("DMA: deregister I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+						gsDmaIoRemapArray[ui32Idx].pvVirtAddr,
+						gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr,
+						gsDmaIoRemapArray[ui32Idx].ui64Size));
+
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+				gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+				gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+			}
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_stub.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_stub.c
new file mode 100644
index 000000000000..2dc3fe1e92d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_stub.c
@@ -0,0 +1,119 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+#if 0
+//eswin: conflict with eswin version!
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_MESSAGE, "Using a stub VM manager type, no hypercall support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+#endif
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_vzfpga.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_vzfpga.c
new file mode 100644
index 000000000000..71f85d715f17
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_vzfpga.c
@@ -0,0 +1,169 @@
+/*************************************************************************/ /*!
+@File           vmm_type_vzfpga.c
+@Title          VZFPGA manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager implementation to support vzfpga platform
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+GetDriverIDFromHeapBase(IMG_UINT64 ui64Addr, IMG_UINT32 *pui32DriverID)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_DEV_PHYADDR sHostFwHeapBaseDevPA = {0};
+	PHYS_HEAP *psHostFwHeap = NULL;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetDeviceInstance(0);
+
+	PVR_LOG_RETURN_IF_FALSE((psHostDevNode != NULL),
+							"Host Device Node not initialised.",
+							PVRSRV_ERROR_NO_DEVICENODE_FOUND);
+
+	psHostFwHeap = psHostDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	PVR_LOG_RETURN_IF_FALSE((psHostFwHeap != NULL),
+							"Host Fw heap not initialised.",
+							PVRSRV_ERROR_PHYSHEAP_ID_INVALID);
+
+	eErr = PhysHeapGetDevPAddr(psHostFwHeap, &sHostFwHeapBaseDevPA);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PhysHeapGetDevPAddr");
+
+	*pui32DriverID = (ui64Addr - sHostFwHeapBaseDevPA.uiAddr) / PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+	PVR_LOG_RETURN_IF_FALSE((*pui32DriverID >= RGXFW_GUEST_DRIVER_ID_START) &&
+							(*pui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+							"Invalid Guest DriverID",
+							PVRSRV_ERROR_INVALID_PVZ_OSID);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+VZFPGAMapDevPhysHeap(IMG_UINT64 ui64Size,
+					 IMG_UINT64 ui64Addr)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT32 ui32GuestDriverID;
+
+	eErr = GetDriverIDFromHeapBase(ui64Addr, &ui32GuestDriverID);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "GetDriverIDFromHeapBase");
+
+	eErr = PvzServerOnVmOnline(ui32GuestDriverID, 0);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerOnVmOnline");
+
+	eErr = PvzServerMapDevPhysHeap(ui32GuestDriverID, 0, ui64Size, ui64Addr);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerMapDevPhysHeap");
+
+	return eErr;
+}
+
+static PVRSRV_ERROR
+VZFPGAUnmapDevPhysHeap(void)
+{
+	IMG_UINT32 ui32ID;
+
+	/* During shutdown, the Guests will be deinitialised in reverse order */
+	for (ui32ID=(RGX_NUM_DRIVERS_SUPPORTED-1);
+		 ui32ID >= RGXFW_HOST_DRIVER_ID; ui32ID--)
+	{
+		if (IsVmOnline(ui32ID, 0))
+		{
+			PvzServerUnmapDevPhysHeap(ui32ID, 0);
+			PvzServerOnVmOffline(ui32ID, 0);
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static VMM_PVZ_CONNECTION gsVZFPGAPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&VZFPGAMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&VZFPGAUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsVZFPGAPvz;
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_vzfpga.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/include/dma_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/include/dma_support.h
new file mode 100644
index 000000000000..97445e8cf9ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/include/dma_support.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/Kbuild.mk
new file mode 100644
index 000000000000..3a0b79b13c7c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/Kbuild.mk
@@ -0,0 +1,52 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+	services/system/volcanic/$(PVR_SYSTEM)/sysconfig.o \
+	services/system/common/sysconfig_cmn.o
+
+$(PVRSRV_MODNAME)-y += \
+	services/system/volcanic/$(PVR_SYSTEM)/fpga.o
+
+ccflags-y += \
+	-I$(TOP)/services/system/volcanic/common/env/linux \
+	-I$(TOP)/services/include/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc \
+	-I$(TOP)/include/volcanic/system/rgx_tc -I$(TOP)/include/system/rgx_tc
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.c
new file mode 100644
index 000000000000..2698c12bf534
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.c
@@ -0,0 +1,302 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#include <linux/module.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pci_support.h"
+#include "os_apphint.h"
+#include "pdump_km.h"
+#include "rgxtbdefs_km.h"
+
+
+#include "fpga.h"
+
+/*!
+ ******************************************************************************
+ * System FPGA parameters
+ *****************************************************************************/
+static IMG_UINT32 gui32SysTBBWDropN                 = 0;
+static IMG_UINT32 gui32SysTBBWPeriod                = 0;
+static IMG_UINT32 gui32SysTBQOS0Min                 = 0;
+static IMG_UINT32 gui32SysTBQOS0Max                 = 0;
+static IMG_UINT32 gui32SysTBQOS15Min                = 0;
+static IMG_UINT32 gui32SysTBQOS15Max                = 0;
+static IMG_UINT32 gui32SysTBQOSDist                 = 0;
+static IMG_UINT32 gui32SysTBMemArbiter              = 0;
+static IMG_UINT32 gui32SysTBMaxIdOutRW              = 0;
+static IMG_UINT32 gui32SysTBMaxIdOutWr              = 0;
+static IMG_UINT32 gui32SysTBMaxIdOutRd              = 0;
+/* these allow raw writes to RGX_TB_QOS_RD_LATENCY and RGX_TB_QOS_WR_LATENCY */
+static IMG_UINT64 gui64SysTBQOSLatencyRd            = 0;
+static IMG_UINT64 gui64SysTBQOSLatencyWr            = 0;
+
+#if defined(__linux__)
+#include <linux/module.h>
+#include <linux/dma-mapping.h>
+module_param_named(sys_tb_bandwidth_drop,            gui32SysTBBWDropN,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_bandwidth_period,          gui32SysTBBWPeriod,                uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos0_min,                  gui32SysTBQOS0Min,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos0_max,                  gui32SysTBQOS0Max,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos15_min,                 gui32SysTBQOS15Min,                uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos15_max,                 gui32SysTBQOS15Max,                uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos_dist,                  gui32SysTBQOSDist,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_mem_arbiter,               gui32SysTBMemArbiter,              uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_max_id_outstanding_rdwr,   gui32SysTBMaxIdOutRW,              uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_max_id_outstanding_wr,     gui32SysTBMaxIdOutWr,              uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_max_id_outstanding_rd,     gui32SysTBMaxIdOutRd,              uint,   S_IRUGO | S_IWUSR);
+
+module_param_named(sys_tb_qos_latency_rd,            gui64SysTBQOSLatencyRd,            ullong,  S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos_latency_wr,            gui64SysTBQOSLatencyWr,            ullong,  S_IRUGO | S_IWUSR);
+#endif
+
+
+/*
+ * work out the length in bits for a register field from its ~CLRMSK.
+ * Assumes the relevant bits are all contiguous.
+ */
+static IMG_UINT32 FieldLengthBits(IMG_UINT32 mask)
+{
+	IMG_UINT32 count = 0;
+
+	while (mask != 0)
+	{
+		count += (mask & 1);
+		mask >>= 1;
+	}
+	return count;
+}
+
+
+static IMG_UINT32 TBBandwidthLimiterGet(void)
+{
+	IMG_UINT32 ui32BandwidthLimiter = 0;
+
+	/* create bandwidth limiter reg value */
+	if (gui32SysTBBWDropN != 0  ||  gui32SysTBBWPeriod != 0)
+	{
+		IMG_UINT32 ui31DropN_ext;
+		IMG_UINT32 ui31Period_ext;
+
+		/* get EXT bits from the apphint values */
+		ui31DropN_ext = gui32SysTBBWDropN >> FieldLengthBits(~RGX_TB_BW_LIMITER_DROPN_CLRMSK);
+		ui31Period_ext = gui32SysTBBWPeriod >> FieldLengthBits(~RGX_TB_BW_LIMITER_PERIOD_CLRMSK);
+
+		ui32BandwidthLimiter = (RGX_TB_BW_LIMITER_ENABLE_EN << RGX_TB_BW_LIMITER_ENABLE_SHIFT) |
+							   ((gui32SysTBBWDropN << RGX_TB_BW_LIMITER_DROPN_SHIFT) & ~RGX_TB_BW_LIMITER_DROPN_CLRMSK) |
+							   ((gui32SysTBBWPeriod << RGX_TB_BW_LIMITER_PERIOD_SHIFT) & ~RGX_TB_BW_LIMITER_PERIOD_CLRMSK) |
+							   ((ui31DropN_ext << RGX_TB_BW_LIMITER_DROPN_EXT_SHIFT) & ~RGX_TB_BW_LIMITER_DROPN_EXT_CLRMSK) |
+							   ((ui31Period_ext << RGX_TB_BW_LIMITER_PERIOD_EXT_SHIFT) & ~RGX_TB_BW_LIMITER_PERIOD_EXT_CLRMSK);
+	}
+
+	return ui32BandwidthLimiter;
+}
+
+
+/*
+ * These latencies can be specified in total using gui64SysTBQOSLatencyRd/Wr or in individual fields.
+ * If full register is specified then the individual fields are ignored.
+ * Individual fields only allow same values to be set for read and write.
+ */
+static IMG_UINT64 TBQOSReadLatencyGet(void)
+{
+	if (gui64SysTBQOSLatencyRd != 0)
+	{
+		return gui64SysTBQOSLatencyRd;
+	}
+
+	return ((((IMG_UINT64)gui32SysTBQOS15Max) << RGX_TB_QOS_RD_LATENCY_MAX_15_SHIFT) & ~RGX_TB_QOS_RD_LATENCY_MAX_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS15Min) << RGX_TB_QOS_RD_LATENCY_MIN_15_SHIFT) & ~RGX_TB_QOS_RD_LATENCY_MIN_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Max)  << RGX_TB_QOS_RD_LATENCY_MAX_0_SHIFT)  & ~RGX_TB_QOS_RD_LATENCY_MAX_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Min)  << RGX_TB_QOS_RD_LATENCY_MIN_0_SHIFT)  & ~RGX_TB_QOS_RD_LATENCY_MIN_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOSDist)  << RGX_TB_QOS_RD_LATENCY_DIST_SHIFT)  & ~RGX_TB_QOS_RD_LATENCY_DIST_CLRMSK);
+}
+
+static IMG_UINT64 TBQOSWriteLatencyGet(void)
+{
+	if (gui64SysTBQOSLatencyWr != 0)
+	{
+		return gui64SysTBQOSLatencyWr;
+	}
+
+	return ((((IMG_UINT64)gui32SysTBQOS15Max) << RGX_TB_QOS_WR_LATENCY_MAX_15_SHIFT) & ~RGX_TB_QOS_WR_LATENCY_MAX_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS15Min) << RGX_TB_QOS_WR_LATENCY_MIN_15_SHIFT) & ~RGX_TB_QOS_WR_LATENCY_MIN_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Max)  << RGX_TB_QOS_WR_LATENCY_MAX_0_SHIFT)  & ~RGX_TB_QOS_WR_LATENCY_MAX_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Min)  << RGX_TB_QOS_WR_LATENCY_MIN_0_SHIFT)  & ~RGX_TB_QOS_WR_LATENCY_MIN_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOSDist)  << RGX_TB_QOS_WR_LATENCY_DIST_SHIFT)  & ~RGX_TB_QOS_WR_LATENCY_DIST_CLRMSK);
+}
+
+static IMG_UINT32 TBMemArbiterGet(void)
+{
+	return gui32SysTBMemArbiter;
+}
+
+static IMG_UINT64 TBQOSMaxIdOutstandingGet(void)
+{
+	return ((((IMG_UINT64)gui32SysTBMaxIdOutRW) << RGX_TB_MAX_ID_OUTSTANDING_RD_WR_SHIFT) & ~RGX_TB_MAX_ID_OUTSTANDING_RD_WR_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBMaxIdOutWr) << RGX_TB_MAX_ID_OUTSTANDING_WRITE_SHIFT) & ~RGX_TB_MAX_ID_OUTSTANDING_WRITE_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBMaxIdOutRd) << RGX_TB_MAX_ID_OUTSTANDING_READ_SHIFT) & ~RGX_TB_MAX_ID_OUTSTANDING_READ_CLRMSK);
+}
+
+
+PVRSRV_ERROR FPGA_Reset(struct resource *registers, IMG_BOOL bFullReset)
+{
+	IMG_CPU_PHYADDR	sWrapperRegsCpuPBase;
+	void			*pvWrapperRegs;
+	IMG_UINT32		ui32BandwidthLimiter;
+	IMG_UINT64		ui64ReadQOSLatency;
+	IMG_UINT64		ui64WriteQOSLatency;
+	IMG_UINT32		ui32MemArbiter;
+	IMG_UINT64		ui64MaxIdOutstanding;
+
+	sWrapperRegsCpuPBase.uiAddr = registers->start + FPGA_RGX_TB_REG_WRAPPER_OFFSET;
+
+	/*
+		Create a temporary mapping of the FPGA wrapper registers in order to reset
+		required registers.
+	*/
+	pvWrapperRegs = OSMapPhysToLin(sWrapperRegsCpuPBase,
+								   FPGA_RGX_TB_REG_WRAPPER_SIZE,
+								   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvWrapperRegs == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"%s: Failed to create wrapper register mapping", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	/*
+		Set the bandwidth limiter if required.
+	*/
+	ui32BandwidthLimiter = TBBandwidthLimiterGet();
+	if (ui32BandwidthLimiter != 0)
+	{
+		OSWriteHWReg32(pvWrapperRegs, RGX_TB_BW_LIMITER, ui32BandwidthLimiter);
+		(void) OSReadHWReg32(pvWrapperRegs, RGX_TB_BW_LIMITER);
+		PVR_LOG(("%s: Bandwidth limiter = 0x%08X", __func__, OSReadHWReg32(pvWrapperRegs, RGX_TB_BW_LIMITER)));
+	}
+
+	/*
+		Set the read/write QoS latency values if required.
+	*/
+	ui64ReadQOSLatency = TBQOSReadLatencyGet();
+	if (ui64ReadQOSLatency != 0)
+	{
+		OSWriteHWReg64(pvWrapperRegs, RGX_TB_QOS_RD_LATENCY, ui64ReadQOSLatency);
+		(void) OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_RD_LATENCY);
+		PVR_LOG(("%s: QOS Read latency = 0x%016llX", __func__, OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_RD_LATENCY)));
+	}
+
+	ui64WriteQOSLatency = TBQOSWriteLatencyGet();
+	if (ui64WriteQOSLatency != 0)
+	{
+		OSWriteHWReg64(pvWrapperRegs, RGX_TB_QOS_WR_LATENCY, ui64WriteQOSLatency);
+		(void) OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_WR_LATENCY);
+		PVR_LOG(("%s: QOS Write latency = 0x%016llX", __func__, OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_WR_LATENCY)));
+	}
+
+	ui32MemArbiter = TBMemArbiterGet();
+	if (ui32MemArbiter != 0)
+	{
+		OSWriteHWReg32(pvWrapperRegs, RGX_TB_MEM_ARBITER, ui32MemArbiter);
+		(void) OSReadHWReg32(pvWrapperRegs, RGX_TB_MEM_ARBITER);
+		PVR_LOG(("%s: Mem arbiter = 0x%08X", __func__, OSReadHWReg32(pvWrapperRegs, RGX_TB_MEM_ARBITER)));
+	}
+
+	ui64MaxIdOutstanding = TBQOSMaxIdOutstandingGet();
+	if (ui64MaxIdOutstanding != 0)
+	{
+		OSWriteHWReg64(pvWrapperRegs, RGX_TB_MAX_ID_OUTSTANDING, ui64MaxIdOutstanding);
+		(void) OSReadHWReg64(pvWrapperRegs, RGX_TB_MAX_ID_OUTSTANDING);
+		PVR_LOG(("%s: Max Id Outstanding = 0x%016llX", __func__, OSReadHWReg64(pvWrapperRegs, RGX_TB_MAX_ID_OUTSTANDING)));
+	}
+
+	/*
+		Remove the temporary register mapping.
+	*/
+	OSUnMapPhysToLin(pvWrapperRegs, FPGA_RGX_TB_REG_WRAPPER_SIZE);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR FPGA_SysDebugInfo(struct resource *registers,
+                               DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                               void *pvDumpDebugFile)
+{
+	IMG_CPU_PHYADDR	sWrapperRegsCpuPBase;
+	void			*pvWrapperRegs;
+
+	sWrapperRegsCpuPBase.uiAddr = registers->start + FPGA_RGX_TB_REG_WRAPPER_OFFSET;
+
+	/*
+		Create a temporary mapping of the FPGA wrapper registers.
+	*/
+	pvWrapperRegs = OSMapPhysToLin(sWrapperRegsCpuPBase,
+								   FPGA_RGX_TB_REG_WRAPPER_SIZE,
+								   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvWrapperRegs == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"%s: Failed to create wrapper register mapping", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+#define SYS_FPGA_DBG_R32(R)	PVR_DUMPDEBUG_LOG("%-29s : 0x%08X", #R, (IMG_UINT32)OSReadHWReg32(pvWrapperRegs, RGX_TB_##R))
+#define SYS_FPGA_DBG_R64(R)	PVR_DUMPDEBUG_LOG("%-29s : 0x%016llX", #R, (IMG_UINT64)OSReadHWReg64(pvWrapperRegs, RGX_TB_##R))
+
+	SYS_FPGA_DBG_R32(BW_LIMITER);
+
+	SYS_FPGA_DBG_R64(QOS_RD_LATENCY);
+	SYS_FPGA_DBG_R64(QOS_WR_LATENCY);
+
+	SYS_FPGA_DBG_R32(MEM_ARBITER);
+	SYS_FPGA_DBG_R64(MAX_ID_OUTSTANDING);
+
+	/*
+		Remove the temporary register mapping.
+	*/
+	OSUnMapPhysToLin(pvWrapperRegs, FPGA_RGX_TB_REG_WRAPPER_SIZE);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.h
new file mode 100644
index 000000000000..d810efc45d19
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.h
@@ -0,0 +1,67 @@
+/*************************************************************************/ /*!
+@File
+@Title          FPGA PCI header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines FPGA PCI registers
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*
+ * Defines for FPGA System Bus Decode.
+ */
+
+#define FPGA_RGX_TB_REG_PCI_BASENUM    2
+#define FPGA_RGX_TB_REG_SIZE           (16 * 1024)
+#define FPGA_RGX_TB_REG_WRAPPER_OFFSET 0x1000000
+#define FPGA_RGX_TB_REG_WRAPPER_SIZE   0x8000
+
+
+/*
+ * Interface to the Reset function
+ */
+PVRSRV_ERROR FPGA_Reset(struct resource *registers, IMG_BOOL bFullReset);
+
+#if 1
+#define DevReset(psSysData, flags)  FPGA_Reset((psSysData)->registers, flags)
+#else
+#define DevReset(psSysData, flags)  PVRSRV_OK
+#endif
+
+PVRSRV_ERROR FPGA_SysDebugInfo(struct resource *registers,
+                               DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                               void *pvDumpDebugFile);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysconfig.c
new file mode 100644
index 000000000000..0dda8636bf8e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysconfig.c
@@ -0,0 +1,1772 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#include "os_apphint.h"
+#include "rgxfwutils.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_support.h"
+#include "ion_sys.h"
+#endif
+
+#include "vmm_pvz_server.h"
+#include "pvr_bridge_k.h"
+#include "pvr_drv.h"
+#include "tc_drv.h"
+#include "fpga.h"
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/delay.h>
+
+#define SECURE_FW_MEM_SIZE   (0x80000) /* 512Kb */
+#define SECURE_MEM_SIZE    (0x4000000) /*  64MB */
+
+typedef struct
+{
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;
+	IMG_UINT64 uiSize;
+	IMG_BOOL bUsed;
+	IMG_UINT32 ui32DriverModeMask;
+} CARD_PHYS_HEAP_CONFIG_SPEC;
+
+#define PHYSHEAP_ALL_DRIVERS	(1 << DRIVER_MODE_NATIVE | \
+								 1 << DRIVER_MODE_HOST   | \
+								 1 << DRIVER_MODE_GUEST)
+
+#define PHYSHEAP_NO_GUESTS	(1 << DRIVER_MODE_NATIVE | \
+							 1 << DRIVER_MODE_HOST)
+
+#define PHYSHEAP_ONLY_VZ	(1 << DRIVER_MODE_GUEST | \
+							 1 << DRIVER_MODE_HOST)
+
+#define HEAP_SPEC_IDX_GPU_PRIVATE (0U)
+#define HEAP_SPEC_IDX_GPU_LOCAL   (1U)
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(FPGA) && !defined(SUPPORT_SECURITY_VALIDATION)
+static PVRSRV_ERROR TEE_LoadFirmwareWrapper(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams);
+extern PVRSRV_ERROR TEE_LoadFirmware(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams);
+extern PVRSRV_ERROR TEE_SetPowerParams(IMG_HANDLE hSysData, PVRSRV_TD_POWER_PARAMS *psTDPowerParams);
+extern PVRSRV_ERROR TEE_RGXStart(IMG_HANDLE hSysData);
+extern PVRSRV_ERROR TEE_RGXStop(IMG_HANDLE hSysData);
+#endif
+
+/*
+	 -----------------------------------------------------------------------
+	|                      Phys Heap definition matrix                      |
+	 -----------------------------------------------------------------------
+	|   Phys    |    GPU    |    GPU    |    Fw     |    Fw     |    Fw     |
+	|   Heap    |  Private  |   Secure  |  Private  |  Shared   |  PageTab  |
+	 -----------------------------------------------------------------------
+	|  VzMode   |   N H G   |   N H G   |   N H G   |   N H G   |   N H G   |
+	 -----------------------------------------------------------------------
+	|  TEE      |   1 1 1   |   1 1 1   |   1 1 0   |   1 1 1   |   1 1 0   |
+	|  PREMAP   |   1 1 1   |   0 0 0   |   0 0 0   |   1 1 1   |   1 1 0   |
+	|  DEFAULT  |   1 1 1   |   0 0 0   |   0 0 0   |   0 0 1   |   0 0 0   |
+	 -----------------------------------------------------------------------
+*/
+static const CARD_PHYS_HEAP_CONFIG_SPEC gasCardHeapTemplate[] =
+{
+	{
+	 PHYS_HEAP_USAGE_GPU_PRIVATE,
+	 0,					/* determined at runtime by apphints */
+	 false,				/* determined at runtime by apphints */
+	 PHYSHEAP_ALL_DRIVERS
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_LOCAL,
+	 0,					/* determined at runtime */
+	 true,
+	 PHYSHEAP_ALL_DRIVERS
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_SECURE,
+	 SECURE_MEM_SIZE,
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	 true,
+#else
+	 false,
+#endif
+	 PHYSHEAP_ALL_DRIVERS
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PREMAP_PT,
+	 RGX_FIRMWARE_MAX_PAGETABLE_SIZE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 true,
+#else
+	 false,
+#endif
+	 PHYSHEAP_NO_GUESTS
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PRIVATE,
+	 SECURE_FW_MEM_SIZE,
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	 true,
+#else
+	 false,
+#endif
+	 PHYSHEAP_NO_GUESTS
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_SHARED,
+	 RGX_FIRMWARE_RAW_HEAP_SIZE,
+	 true,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 PHYSHEAP_ALL_DRIVERS
+#else
+	 PHYSHEAP_ONLY_VZ
+#endif
+	}
+};
+
+#define ODIN_MEMORY_HYBRID_DEVICE_BASE 0x400000000
+
+#define VALI_MEMORY_DEVICE_BASE (0x800000000U)
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_DEVICE_CONFIG *apsDevCfgs[PVRSRV_MAX_DEVICES] = {0};
+#endif
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Voltage)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+
+#endif
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCLocalDevPAddrToCpuPAddr,
+};
+
+static void TCValiLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCValiLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsValiLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCValiLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCValiLocalDevPAddrToCpuPAddr,
+};
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHostCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHostDevPAddrToCpuPAddr,
+};
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHybridCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHybridDevPAddrToCpuPAddr
+};
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	IMG_UINT32 ui32SysDataSize;
+	IMG_UINT64 ui64GpuRegisterBase;
+	IMG_UINT64 ui64FwHeapCpuBase;
+	IMG_UINT64 ui64FwHeapGpuBase;
+	IMG_UINT64 ui64FwPrivateHeapSize;
+	IMG_UINT64 ui64FwTotalHeapSize;
+	IMG_UINT64 ui64FwHeapStride;
+	IMG_UINT64 ui64FwPageTableHeapCpuBase;
+	IMG_UINT64 ui64FwPageTableHeapGpuBase;
+	IMG_UINT64 ui64FwPageTableHeapSize;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG sDevFeatureCfg;
+
+	struct platform_device *pdev;
+
+	struct tc_rogue_platform_data *pdata;
+
+	struct resource *registers;
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_client *ion_client;
+	struct ion_handle *ion_rogue_allocation;
+#endif
+};
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	int err;
+	char str_fpga_rev[FPGA_REV_MAX_LEN]={0};
+	char str_tcf_core_rev[TCF_CORE_REV_MAX_LEN]={0};
+	char str_tcf_core_target_build_id[TCF_CORE_CFG_MAX_LEN]={0};
+	char str_pci_ver[PCI_VERSION_MAX_LEN]={0};
+	char str_macro_ver[MACRO_VERSION_MAX_LEN]={0};
+
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+
+	err = tc_sys_strings(psSysData->pdev->dev.parent,
+							 str_fpga_rev, sizeof(str_fpga_rev),
+							 str_tcf_core_rev, sizeof(str_tcf_core_rev),
+							 str_tcf_core_target_build_id, sizeof(str_tcf_core_target_build_id),
+							 str_pci_ver, sizeof(str_pci_ver),
+							 str_macro_ver, sizeof(str_macro_ver));
+	if (err)
+	{
+		return NULL;
+	}
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(str_fpga_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_target_build_id);
+	ui32StringLength += OSStringLength(str_pci_ver);
+	ui32StringLength += OSStringLength(str_macro_ver);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+				   SYSTEM_INFO_FORMAT_STRING,
+				   str_fpga_rev,
+				   str_tcf_core_rev,
+				   str_tcf_core_target_build_id,
+				   str_pci_ver,
+				   str_macro_ver);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+static SYS_DATA *gpsIonPrivateData;
+
+PVRSRV_ERROR IonInit(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYS_DATA *psSysData = pvPrivateData;
+	gpsIonPrivateData = psSysData;
+
+	psSysData->ion_client = ion_client_create(psSysData->pdata->ion_device, SYS_RGX_DEV_NAME);
+	if (IS_ERR(psSysData->ion_client))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create ION client (%ld)", __func__, PTR_ERR(psSysData->ion_client)));
+		eError = PVRSRV_ERROR_ION_NO_CLIENT;
+		goto err_out;
+	}
+	/* Allocate the whole rogue ion heap and pass that to services to manage */
+	psSysData->ion_rogue_allocation = ion_alloc(psSysData->ion_client, psSysData->pdata->rogue_heap_memory_size, 4096, (1 << psSysData->pdata->ion_heap_id), 0);
+	if (IS_ERR(psSysData->ion_rogue_allocation))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate ION rogue buffer (%ld)", __func__, PTR_ERR(psSysData->ion_rogue_allocation)));
+		eError = PVRSRV_ERROR_ION_FAILED_TO_ALLOC;
+		goto err_destroy_client;
+
+	}
+
+	return PVRSRV_OK;
+err_destroy_client:
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+err_out:
+	return eError;
+}
+
+void IonDeinit(void)
+{
+	SYS_DATA *psSysData = gpsIonPrivateData;
+	ion_free(psSysData->ion_client, psSysData->ion_rogue_allocation);
+	psSysData->ion_rogue_allocation = NULL;
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+}
+
+struct ion_device *IonDevAcquire(void)
+{
+	return gpsIonPrivateData->pdata->ion_device;
+}
+
+void IonDevRelease(struct ion_device *ion_device)
+{
+	PVR_ASSERT(ion_device == gpsIonPrivateData->pdata->ion_device);
+}
+#endif /* defined(SUPPORT_ION) */
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+			psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+			psDevPAddr[ui32Idx].uiAddr + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCValiLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base + VALI_MEMORY_DEVICE_BASE;
+	}
+}
+
+static void TCValiLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    psDevPAddr[ui32Idx].uiAddr - VALI_MEMORY_DEVICE_BASE + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_ASSERT(sizeof(*psDevPAddr) == sizeof(*psCpuPAddr));
+	OSCachedMemCopy(psDevPAddr, psCpuPAddr, uiNumOfAddr * sizeof(*psDevPAddr));
+}
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_ASSERT(sizeof(*psCpuPAddr) == sizeof(*psDevPAddr));
+	OSCachedMemCopy(psCpuPAddr, psDevPAddr, uiNumOfAddr * sizeof(*psCpuPAddr));
+}
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    (psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base) +
+		    ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    (psDevPAddr[ui32Idx].uiAddr - ODIN_MEMORY_HYBRID_DEVICE_BASE) +
+		    psSysData->pdata->tc_memory_base;
+	}
+}
+
+static inline
+IMG_CHAR* GetHeapName(PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_LOCAL))    return "lma_gpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_SECURE))   return "lma_gpu_secure";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_PRIVATE))  return "lma_gpu_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PRIVATE))   return "lma_fw_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_SHARED))    return "lma_fw_shared";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PREMAP_PT)) return "lma_fw_pagetables";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_CPU_LOCAL))    return "lma_cpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_DISPLAY))      return "lma_gpu_display";
+	else                                                     return "Unexpected Heap";
+}
+
+static PVRSRV_ERROR
+InitLocalHeap(PHYS_HEAP_CONFIG *psPhysHeap,
+			  IMG_UINT64 uiBaseAddr, IMG_UINT64 uiStartAddr,
+			  IMG_UINT64 uiSize, PHYS_HEAP_FUNCTIONS *psFuncs,
+			  PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = ui32Flags;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = psFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = GetHeapName(ui32Flags);
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(uiStartAddr);
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = uiBaseAddr;
+	psPhysHeap->uConfig.sLMA.uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardGPUHeaps(SYS_DATA *psSysData,
+				   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+				   PHYS_HEAP_CONFIG *pasPhysHeaps,
+				   PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				   IMG_UINT32 *pui32HeapIdx,
+				   IMG_UINT64 ui64CardAddr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->rogue_heap_memory_base;
+	IMG_UINT32 ui32SpecIdx;
+
+	for (ui32SpecIdx = 0; ui32SpecIdx < ARRAY_SIZE(gasCardHeapTemplate); ui32SpecIdx++)
+	{
+		if (pasCardHeapSpec[ui32SpecIdx].bUsed)
+		{
+			IMG_UINT64 ui64HeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+
+			eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+								   ui64CardAddr,
+								   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+								   ui64HeapSize,
+								   psHeapFuncs,
+								   pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+			/* save the heap details in a structure passed to the tee_ddk module */
+			if (BITMASK_HAS(pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags, PHYS_HEAP_USAGE_FW_PREMAP_PT))
+			{
+				psSysData->ui64FwPageTableHeapCpuBase = ui64StartAddr;
+				psSysData->ui64FwPageTableHeapGpuBase = ui64CardAddr;
+				psSysData->ui64FwPageTableHeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+			}
+			else if (BITMASK_HAS(pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags, PHYS_HEAP_USAGE_FW_PRIVATE))
+			{
+				psSysData->ui64FwPrivateHeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+				psSysData->ui64FwHeapCpuBase = ui64StartAddr;
+				psSysData->ui64FwHeapGpuBase = ui64CardAddr;
+			}
+			else if (BITMASK_HAS(pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags, PHYS_HEAP_USAGE_FW_SHARED))
+			{
+				psSysData->ui64FwTotalHeapSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+			}
+#endif
+
+			ui64CardAddr  += ui64HeapSize;
+			ui64StartAddr += ui64HeapSize;
+			(*pui32HeapIdx)++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardEXTHeap(const SYS_DATA *psSysData,
+				  PHYS_HEAP_CONFIG *pasPhysHeaps,
+				  PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				  IMG_UINT32 *pui32HeapIdx,
+				  IMG_UINT64 ui64CardBase)
+{
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->pdp_heap_memory_base;
+	IMG_UINT64 ui64Size = psSysData->pdata->pdp_heap_memory_size;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+						   ui64CardBase + psSysData->pdata->rogue_heap_memory_size,
+						   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+						   ui64Size, psHeapFuncs,
+						   PHYS_HEAP_USAGE_EXTERNAL | PHYS_HEAP_USAGE_DISPLAY);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	(*pui32HeapIdx)++;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitLocalHeaps(SYS_DATA *psSysData,
+			   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			   PHYS_HEAP_CONFIG *pasPhysHeaps,
+			   IMG_UINT32 *pui32HeapIdx)
+{
+	PHYS_HEAP_FUNCTIONS *psHeapFuncs;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64CardBase;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHybridPhysHeapFuncs;
+		ui64CardBase = ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+	else if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHostPhysHeapFuncs;
+		ui64CardBase = 0;
+	}
+	else if (psSysData->pdata->baseboard == TC_BASEBOARD_VALI)
+	{
+		psHeapFuncs = &gsValiLocalPhysHeapFuncs;
+		ui64CardBase = VALI_MEMORY_DEVICE_BASE;
+	}
+	else
+	{
+		psHeapFuncs = &gsLocalPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->rogue_heap_memory_base - psSysData->pdata->tc_memory_base;
+	}
+
+	eError = CreateCardGPUHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = CreateCardEXTHeap(psSysData, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitHostHeaps(const SYS_DATA *psSysData, PHYS_HEAP_CONFIG *pasPhysHeaps, IMG_UINT32 *pui32HeapIdx)
+{
+	PVR_UNREFERENCED_PARAMETER(psSysData);
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		pasPhysHeaps[*pui32HeapIdx].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[*pui32HeapIdx].ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszHeapName = "uma_cpu_local";
+
+		(*pui32HeapIdx)++;
+
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Initialising CPU_LOCAL UMA Host PhysHeaps with memory mode: %d",
+		         psSysData->pdata->mem_mode));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsInit(SYS_DATA *psSysData,
+			  CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			  PHYS_HEAP_CONFIG *pasPhysHeaps,
+			  void *pvPrivData, IMG_UINT32 ui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32HeapCounter = 0;
+
+	eError = InitLocalHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = InitHostHeaps(psSysData, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((ui32HeapCounter == ui32NumHeaps),
+							"Number of PhysHeapConfigs set up doesn't match the initial requirement.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	/* Initialise fields that don't change between memory modes.
+	 * Fix up heap IDs. This is needed for multi-testchip systems to
+	 * ensure the heap IDs are unique as this is what Services expects.
+	 */
+	for (i = 0; i < ui32NumHeaps; i++)
+	{
+		switch (pasPhysHeaps[i].eType)
+		{
+		case PHYS_HEAP_TYPE_LMA:
+			pasPhysHeaps[i].uConfig.sLMA.hPrivData = (void*) pvPrivData;
+			break;
+		case PHYS_HEAP_TYPE_UMA:
+			pasPhysHeaps[i].uConfig.sUMA.hPrivData = (void*) pvPrivData;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Invalid PHYS_HEAP_TYPE: %u in %s",
+			                        pasPhysHeaps[i].eType,
+			                        __func__));
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapSetRequirements(const SYS_DATA *psSysData,
+						PVRSRV_DEVICE_CONFIG *psDevConfig,
+						CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+						IMG_UINT32 *pui32CardPhysHeapCfgCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64FreeCardMemory = psSysData->pdata->rogue_heap_memory_size;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_PRIVATE) &&
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_LOCAL),
+		"PhysHeapConfigs not set correctly in the system layer.", PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < ARRAY_SIZE(gasCardHeapTemplate); i++)
+	{
+		if (pasCardHeapSpec[i].bUsed)
+		{
+			if (BITMASK_HAS(pasCardHeapSpec[i].ui32DriverModeMask,
+							BIT(psDevConfig->eDriverMode)))
+			{
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+				IMG_UINT32 ui32UsageFlags = pasCardHeapSpec[i].ui32UsageFlags;
+
+				if (BITMASK_HAS(ui32UsageFlags, PHYS_HEAP_USAGE_FW_SHARED) &&
+					(psDevConfig->eDriverMode != DRIVER_MODE_GUEST))
+				{
+					/* The Firmware private heap in which the TEE loads the Fw binary
+					 * carves some memory out of the Host/Native driver's Fw shared heap.
+					 * Both Fw Private and Fw Shared heaps are premapped by the TEE as
+					 * one contiguous range of RGX_FIRMWARE_RAW_HEAP_SIZE */
+					pasCardHeapSpec[i].uiSize -= SECURE_FW_MEM_SIZE;
+				}
+#endif
+
+				/* Determine the memory requirements of heaps with a fixed size */
+				ui64FreeCardMemory -= pasCardHeapSpec[i].uiSize;
+
+				/* Count card physheap configs used by the system */
+				(*pui32CardPhysHeapCfgCount)++;
+			}
+			else
+			{
+				/* Heap not used in this driver mode */
+				pasCardHeapSpec[i].bUsed = false;
+			}
+		}
+	}
+
+	if (SysRestrictGpuLocalAddPrivateHeap())
+	{
+		IMG_UINT64 ui64GpuSharedMem = SysRestrictGpuLocalPhysheap(ui64FreeCardMemory);
+
+		if (ui64GpuSharedMem == ui64FreeCardMemory)
+		{
+			/* No memory reserved for GPU private use, special heap not needed */
+		}
+		else
+		{
+			/* Set up the GPU private heap */
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed = true;
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].uiSize = ui64FreeCardMemory - ui64GpuSharedMem;
+			ui64FreeCardMemory = ui64GpuSharedMem;
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	/* all remaining memory card memory goes to GPU_LOCAL */
+	pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].uiSize = ui64FreeCardMemory;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsCreate(SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevConfig,
+				PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+				IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps = 0;
+	CARD_PHYS_HEAP_CONFIG_SPEC asCardHeapSpec[ARRAY_SIZE(gasCardHeapTemplate)];
+
+	/* Initialise the local heap specs with the build-time template */
+	memcpy(asCardHeapSpec, gasCardHeapTemplate, sizeof(gasCardHeapTemplate));
+
+	eError = PhysHeapSetRequirements(psSysData, psDevConfig, asCardHeapSpec, &ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psDevConfig->bHasNonMappableLocalMemory = asCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed;
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* CPU_LOCAL heap also required */
+		ui32NumHeaps++;
+	}
+
+	/* DISPLAY heap is always present */
+	ui32NumHeaps++;
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * ui32NumHeaps);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = PhysHeapsInit(psSysData, asCardHeapSpec, pasPhysHeaps, psDevConfig, ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = ui32NumHeaps;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	IMG_UINT32 ui32DeviceID;
+
+	for (ui32DeviceID=0; ui32DeviceID < PVRSRV_MAX_DEVICES; ui32DeviceID++)
+	{
+		PVRSRV_DEVICE_CONFIG *psDC = apsDevCfgs[ui32DeviceID];
+		if (psDC == psDevConfig)
+		{
+			apsDevCfgs[ui32DeviceID] = NULL;
+		}
+	}
+#endif
+
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	OSFreeMem(psDevConfig);
+}
+
+static void odinTCDevPhysAddr2DmaAddr(PVRSRV_DEVICE_CONFIG *psDevConfig,
+									  IMG_DMA_ADDR *psDmaAddr,
+									  IMG_DEV_PHYADDR *psDevPAddr,
+									  IMG_BOOL *pbValid,
+									  IMG_UINT32 ui32NumAddr,
+									  IMG_BOOL bSparseAlloc)
+{
+	IMG_CPU_PHYADDR sCpuPAddr = {0};
+	IMG_UINT32 ui32Idx;
+
+	/* Fast path */
+	if (!bSparseAlloc)
+	{
+		/* In Odin, DMA address space is the same as host CPU */
+		TCLocalDevPAddrToCpuPAddr(psDevConfig,
+								  1,
+								  &sCpuPAddr,
+								  psDevPAddr);
+		psDmaAddr->uiAddr = sCpuPAddr.uiAddr;
+	}
+	else
+	{
+		for (ui32Idx = 0; ui32Idx < ui32NumAddr; ui32Idx++)
+		{
+			if (pbValid[ui32Idx])
+			{
+				TCLocalDevPAddrToCpuPAddr(psDevConfig,
+										  1,
+										  &sCpuPAddr,
+										  &psDevPAddr[ui32Idx]);
+				psDmaAddr[ui32Idx].uiAddr = sCpuPAddr.uiAddr;
+			}
+			else
+			{
+				/* Invalid DMA address marker */
+				psDmaAddr[ui32Idx].uiAddr = ~((IMG_UINT64)0x0);
+			}
+		}
+	}
+}
+
+static void * odinTCgetCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig, char *name)
+{
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	return tc_dma_chan(psDev->parent, name);
+}
+
+static void odinTCFreeCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   void* channel)
+{
+
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	struct dma_chan *chan = (struct dma_chan*) channel;
+
+	tc_dma_chan_free(psDev->parent, chan);
+}
+
+static void GetDriverMode(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DeviceID;
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	if (PVRSRVAcquireInternalID(&ui32DeviceID) != PVRSRV_OK)
+	{
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		return;
+	}
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		void __iomem *pvRegBase;
+
+		pvRegBase = (void __iomem *) OSMapPhysToLin(psDevConfig->sRegsCpuPBase, psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (pvRegBase == NULL)
+		{
+			/* failed to map register bank, default to native mode */
+			psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		}
+		else
+		{
+			IMG_UINT64 ui64ClkCtrl;
+
+			/* the CLK_CTRL register is valid only in the Os 0 (Host) register bank
+			 * if it reads 0 then we can conclude this Os is set up to run as Guest */
+#if defined(RGX_CR_CLK_CTRL)
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL);
+#else
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL1);
+#endif
+			OSUnMapPhysToLin((void __force *) pvRegBase, psDevConfig->ui32RegsSize);
+
+			psDevConfig->eDriverMode = (ui64ClkCtrl != 0) ? (DRIVER_MODE_HOST) : (DRIVER_MODE_GUEST);
+		}
+#else
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+}
+
+static PVRSRV_ERROR PrepareFWImage(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams);
+
+static PVRSRV_ERROR PrepareFWImage(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams)
+{
+	if ((psTDFWParams != NULL) &&
+		(psTDFWParams->pvFirmware != 0) &&
+		(psTDFWParams->ui32FirmwareSize > 0))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware image size = %u;",
+						__func__, psTDFWParams->ui32FirmwareSize));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,  "%s: Invalid Firmware image.", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Firmware binary not signed on this platform */
+	psTDFWParams->pvSignature = NULL;
+	psTDFWParams->ui32SignatureSize = 0;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+							 psSysData->pdata->mem_mode == TC_MEMORY_LOCAL),
+							"Multidevice virtualization setup supported only on Odin device with TC_MEMORY_LOCAL",
+							PVRSRV_ERROR_INVALID_DEVICE);
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = IMG_OFFSET_ADDR(psDevConfig, sizeof(*psDevConfig));
+	psRGXTimingInfo = IMG_OFFSET_ADDR(psRGXData, sizeof(*psRGXData));
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent) /
+											tc_core_clock_multiplex(psSysData->pdev->dev.parent);
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice = &psSysData->pdev->dev;
+	psDevConfig->pszName = "tc";
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = resource_size(psSysData->registers);
+
+	PVRSRVAcquireInternalID(&ui32DeviceID);
+	/*
+	 * Current FPGA images route don't route the GPU's OSID IRQs to their proper card signals.
+	 * All OSID IRQs are ORed together and output on the legacy DUT_IRQ pin.
+	 * To work around this, we pass the legacy DUT IRQ to the Host driver and leave the floating
+	 * OSID IRQs to Guest Drivers. When an IRQ is received via the DUT line, the system layer's
+	 * IRQ handler can query the Firmware's state register to find out which device/OSID emitted
+	 * an interrupt and call the appropriate handler for each device.
+	 */
+	psDevConfig->ui32IRQ = (ui32DeviceID == 0) ? TC_INTERRUPT_EXT : (TC_INTERRUPT_OSID0 + ui32DeviceID);
+
+	GetDriverMode(psDevConfig);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* If there is device running in native mode, prevent any attempts at
+	 * creating any Guest devices, as there will be no Host to support them.
+	 * Currently the VZFPGA supports only one physical GPU. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		IMG_UINT32 i;
+
+		for (i=0; i < PVRSRV_MAX_DEVICES; i++)
+		{
+			PVRSRV_DEVICE_CONFIG *psDC = apsDevCfgs[i];
+			if ((psDC != NULL) && (PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDC)))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s() Device %u is already running in native mode, no other Guests supported in the system.",  __func__, psDC->psDevNode->sDevId.ui32InternalID));
+				eError = PVRSRV_ERROR_INVALID_DEVICE;
+				goto ErrorFreeDevConfig;
+			}
+		}
+	}
+
+	apsDevCfgs[ui32DeviceID] = psDevConfig;
+#endif
+
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &pasPhysHeaps, &uiPhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	psDevConfig->pasPhysHeaps = pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount = uiPhysHeapCount;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psDevConfig->eDefaultHeap = SysDefaultToCpuLocalHeap() ?
+		    PVRSRV_PHYS_HEAP_CPU_LOCAL : PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+	else
+	{
+		psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+
+	/* Only required for LMA but having this always set shouldn't be a problem */
+	psDevConfig->bDevicePA0IsValid = IMG_TRUE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	/* DMA channel config */
+	psDevConfig->pfnSlaveDMAGetChan = odinTCgetCDMAChan;
+	psDevConfig->pfnSlaveDMAFreeChan = odinTCFreeCDMAChan;
+	psDevConfig->pfnDevPhysAddr2DmaAddr = odinTCDevPhysAddr2DmaAddr;
+	psDevConfig->pszDmaTxChanName = psSysData->pdata->tc_dma_tx_chan_name;
+	psDevConfig->pszDmaRxChanName = psSysData->pdata->tc_dma_rx_chan_name;
+	psDevConfig->bHasDma = IMG_TRUE;
+	/* Following two values are expressed in number of bytes */
+	psDevConfig->ui32DmaTransferUnit = 1;
+	psDevConfig->ui32DmaAlignment = 1;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(FPGA) && !defined(SUPPORT_SECURITY_VALIDATION)
+	psDevConfig->pfnTDSendFWImage = TEE_LoadFirmwareWrapper;
+	psDevConfig->pfnTDSetPowerParams = TEE_SetPowerParams;
+	psDevConfig->pfnTDRGXStart = TEE_RGXStart;
+	psDevConfig->pfnTDRGXStop = TEE_RGXStop;
+#endif
+	psDevConfig->pfnPrepareFWImage = PrepareFWImage;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	resource_size_t uiRegistersSize;
+	IMG_UINT32 ui32MinRegBankSize;
+	PVRSRV_ERROR eError;
+	int err = 0;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->pdev = to_platform_device((struct device *)pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/*
+	 * The device cannot address system memory, so there is no DMA
+	 * limitation.
+	 */
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL)
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(64));
+	}
+	else
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+	}
+
+	err = tc_enable(psSysData->pdev->dev.parent);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device (%d)", __func__, err));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev,
+														IORESOURCE_MEM,
+														"rogue-regs");
+	if (!psSysData->registers)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	psSysData->ui64FwPrivateHeapSize = SECURE_FW_MEM_SIZE;
+	psSysData->ui64GpuRegisterBase = psSysData->registers->start;
+	psSysData->ui32SysDataSize = sizeof(SYS_DATA);
+
+	{
+		void       *pvAppHintState = NULL;
+		IMG_UINT64 ui64AppHintDefault;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64AppHintDefault = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE, pvAppHintState, GuestFWHeapStride,
+		                     &ui64AppHintDefault, &psSysData->ui64FwHeapStride);
+		OSFreeAppHintState(pvAppHintState);
+	}
+#else
+	psSysData->ui64FwPrivateHeapSize = 0;
+#endif
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* each GPU instance gets the minimum 64kb register range */
+	ui32MinRegBankSize = RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE;
+#else
+	/* the GPU gets the entire 64MB IO range */
+	ui32MinRegBankSize = SYS_RGX_REG_REGION_SIZE;
+#endif
+
+	if (uiRegistersSize < ui32MinRegBankSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %pa, required 0x%08x)",
+				 __func__, &uiRegistersSize, ui32MinRegBankSize));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+	/* Reserve the address range */
+	if (!request_mem_region(psSysData->registers->start,
+							resource_size(psSysData->registers),
+							SYS_RGX_DEV_NAME))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+
+		goto ErrorDevDisable;
+	}
+
+	if (psSysData->pdata->baseboard != TC_BASEBOARD_VALI)
+	{
+		/*
+		 * Reset the device as required.
+		 */
+		eError = DevReset(psSysData, IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't reset device", __func__));
+			goto ErrorDevDisable;
+		}
+	}
+
+	eError = DeviceConfigCreate(psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	eError = IonInit(psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise ION", __func__));
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	tc_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	tc_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	u32 tmp = 0;
+	u32 pll;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	if (tc_sys_info(psSysData->pdev->dev.parent, &tmp, &pll))
+		goto err_out;
+
+	if (tmp > 0)
+		PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", tmp);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", pll);
+
+    eError = FPGA_SysDebugInfo(psSysData->registers,
+                               pfnDumpDebugPrintf,
+                               pvDumpDebugFile);
+    if (eError != PVRSRV_OK)
+    {
+        PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't dump registers from device", __func__));
+        goto err_out;
+    }
+
+err_out:
+	return eError;
+#endif
+}
+
+typedef struct
+{
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static void TCInterruptHandler(void* pvData)
+{
+#if (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	LISR_DATA *psLISRData = pvData;
+	psLISRData->pfnLISR(psLISRData->pvData);
+#else
+	IMG_UINT32 ui32DeviceID;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	for (ui32DeviceID=0; ui32DeviceID < PVRSRV_MAX_DEVICES; ui32DeviceID++)
+	{
+		PVRSRV_DEVICE_CONFIG *psDC = apsDevCfgs[ui32DeviceID];
+
+		if (psDC != NULL)
+		{
+			PVRSRV_RGXDEV_INFO *psDI = (PVRSRV_RGXDEV_INFO *) psDC->psDevNode->pvDevice;
+
+			if ((psDI != NULL) && (psDI->pvRegsBaseKM != NULL))
+			{
+				IMG_UINT32 ui32Status = OSReadHWReg32(psDI->pvRegsBaseKM, RGX_CR_IRQ_OS0_EVENT_STATUS);
+
+				if (ui32Status)
+				{
+					LISR_DATA *psLISRData = psDI->pvLISRData;
+					psLISRData->pfnLISR(psLISRData->pvData);
+				}
+			}
+		}
+	}
+#endif
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if ((ui32IRQ != TC_INTERRUPT_EXT) &&
+		((ui32IRQ < TC_INTERRUPT_OSID0) || (ui32IRQ > TC_INTERRUPT_OSID7)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, TCInterruptHandler, psLISRData);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	err = tc_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_enable_interrupt() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+	eError = PVRSRV_OK;
+
+	PVR_TRACE(("Installed device LISR " IMG_PFN_FMTSPEC " to irq %u", pfnLISR, ui32IRQ));
+
+err_out:
+	return eError;
+err_unset_interrupt_handler:
+	tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	goto err_out;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = tc_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_disable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " from irq %u", psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(FPGA) && !defined(SUPPORT_SECURITY_VALIDATION)
+/* The TEE needs to query the device's hardware feature capabilities and ERNs/BRNs.
+ * On first entry to the TEE, supply the config structure embedded in SYS_DATA */
+static PVRSRV_ERROR TEE_LoadFirmwareWrapper(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	psDevInfo = psDeviceNode->pvDevice;
+	psSysData->sDevFeatureCfg = psDevInfo->sDevFeatureCfg;
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return TEE_LoadFirmware(hSysData, psTDFWParams);
+}
+#endif
+
+/****************************************************************************************************/
+/****                                   VM migration test code                                   ****/
+/****************************************************************************************************/
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode);
+static void PreMigrationDeviceSuspend(struct drm_device *psDev);
+static void PostMigrationDeviceResume(struct drm_device *psDev);
+
+void PVRVMMigration(unsigned int src, unsigned int dest);
+EXPORT_SYMBOL(PVRVMMigration);
+
+#define SWAP_REGSBASE_PTR(a, b) do \
+	{ \
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		b = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+	} while (0)
+
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psHostNode = PVRSRVGetDeviceInstance(0);
+	PVRSRV_RGXDEV_INFO *psSrcInfo = psSrcNode->pvDevice;
+	PVRSRV_RGXDEV_INFO *psDestInfo = psDestNode->pvDevice;
+	PVRSRV_DEVICE_CONFIG *psSrcConfig = psSrcNode->psDevConfig;
+	PVRSRV_DEVICE_CONFIG *psDestConfig = psDestNode->psDevConfig;
+	LISR_DATA *psSrcLISRData = (LISR_DATA *) psSrcInfo->pvLISRData;
+	void *pfnLISR = psSrcLISRData->pfnLISR;
+	IMG_UINT32 ui32SrcHyperLane, ui32DestHyperLane;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psHostNode != NULL) &&
+								  (psHostNode->psDevConfig != NULL)),
+								 "Device 0 (expected Host) not initialised.");
+
+	/* Determine the HyperLane ID used by a Guest Device from the Register Bank Base address used */
+	ui32SrcHyperLane = (psSrcConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psSrcConfig->ui32RegsSize;
+	ui32DestHyperLane = (psDestConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psDestConfig->ui32RegsSize;
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Swapping hyperlanes between Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32SrcHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32DestHyperLane));
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resulting configuration:    Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32DestHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32SrcHyperLane));
+
+	/* swap the register bank details */
+	SWAP_REGSBASE_PTR(psSrcInfo->pvRegsBaseKM, psDestInfo->pvRegsBaseKM);
+	SWAP(psSrcConfig->sRegsCpuPBase.uiAddr, psDestConfig->sRegsCpuPBase.uiAddr);
+	/* DevConfig->ui32RegsSize remains the same */
+
+	/* Swap interrupt lines between devices */
+	eError = SysUninstallDeviceLISR(psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+	eError = SysUninstallDeviceLISR(psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	SWAP(psSrcConfig->ui32IRQ, psDestConfig->ui32IRQ);
+
+	eError = SysInstallDeviceLISR(psSrcConfig->hSysData,
+								  psSrcConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psSrcNode,
+								  &psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+
+	eError = SysInstallDeviceLISR(psDestConfig->hSysData,
+								  psDestConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psDestNode,
+								  &psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	/* Swap contents of LMA carveouts between virtual devices */
+	{
+		/* Guest Raw Fw Heap mapping is done using the Host Devices */
+		PHYS_HEAP *psSrcHeap = NULL;
+		PHYS_HEAP *psDestHeap = NULL;
+		IMG_DEV_PHYADDR sSrcHeapBase, sDestHeapBase;
+
+		psSrcHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32SrcHyperLane];
+		psDestHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DestHyperLane];
+
+		PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcHeap != NULL) &&
+									  (psDestHeap != NULL)),
+									 "Guest firmware heaps not premapped by the Host Device.");
+
+		eError = PhysHeapGetDevPAddr(psSrcHeap, &sSrcHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(src fw heap)");
+		eError = PhysHeapGetDevPAddr(psDestHeap, &sDestHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(dest fw heap)");
+
+		eError = PvzServerUnmapDevPhysHeap(ui32SrcHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(src fw heap)");
+		eError = PvzServerUnmapDevPhysHeap(ui32DestHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(dest fw heap)");
+
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32SrcHyperLane]);
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32DestHyperLane]);
+
+		/* create new heaps with new base addresses */
+		eError = PvzServerMapDevPhysHeap(ui32SrcHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sDestHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(src fw heap)");
+		eError = PvzServerMapDevPhysHeap(ui32DestHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sSrcHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(dest fw heap)");
+	}
+}
+
+static void PreMigrationDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(eError == PVRSRV_OK,
+	                           "LinuxBridgeBlockClientsAccess()");
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * be declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+	}
+}
+
+static void PostMigrationDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+									PVRSRV_SYS_POWER_STATE_ON,
+									PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ);
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+}
+
+void PVRVMMigration(unsigned int src, unsigned int dest)
+{
+	PVRSRV_DEVICE_NODE *psSrcNode = PVRSRVGetDeviceInstance(src);
+	PVRSRV_DEVICE_NODE *psDestNode = PVRSRVGetDeviceInstance(dest);
+	struct device *psSrcDev, *psDestDev;
+	struct drm_device *psSrcDrmDev, *psDestDrmDev;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode != NULL) && (psDestNode != NULL) && (psSrcNode != psDestNode)),
+								 "Invalid Device IDs requested for migration.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE) &&
+								  (psDestNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)),
+								 "Devices not fully initialised.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig != NULL) &&
+								  (psDestNode->psDevConfig != NULL)),
+								 "Device config structure is NULL.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig->pvOSDevice != NULL) &&
+								  (psDestNode->psDevConfig->pvOSDevice != NULL)),
+								 "Linux kernel device pointer is NULL.");
+
+	psSrcDev = psSrcNode->psDevConfig->pvOSDevice;
+	psDestDev = psDestNode->psDevConfig->pvOSDevice;
+	psSrcDrmDev = dev_get_drvdata(psSrcDev);
+	psDestDrmDev = dev_get_drvdata(psDestDev);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcDrmDev != NULL) &&
+								  (psDestDrmDev != NULL)),
+								 "Linux kernel drm_device pointer is NULL.");
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psSrcNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psSrcDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psDestNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psDestDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Migrating vGPU resources (regbank, irq, osid)", __func__));
+	SwapHyperlanes(psSrcNode, psDestNode);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psSrcNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psSrcDrmDev);
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psDestNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psDestDrmDev);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysinfo.h
new file mode 100644
index 000000000000..91b41bc4b66d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysinfo.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM)
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#else
+#define MAX_HW_TIME_US                           (20000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (10000) // (1500)
+
+#if defined(TC_APOLLO_TCF5)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#else
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+
+
+#define SYS_RGX_DEV_NAME "tc_rogue"
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/Kbuild.mk
new file mode 100644
index 000000000000..21e77fc75786
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/Kbuild.mk
@@ -0,0 +1,53 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += services/system/volcanic/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/volcanic/common/vmm_type_$(VMM_TYPE).o
+
+ifeq ($(SUPPORT_ION),1)
+$(PVRSRVKM_NAME)-y += services/system/common/env/linux/ion_support_generic.o
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.c
new file mode 100644
index 000000000000..89b8ff3ba0f3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.c
@@ -0,0 +1,352 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+/*
+ * In systems that support trusted device address protection, there are three
+ * physical heaps from which pages should be allocated:
+ * - one heap for normal allocations
+ * - one heap for allocations holding META code memory
+ * - one heap for allocations holding secured DRM data
+ */
+
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+#include "validation_soc.h"
+#endif
+
+/* Change to test CPU_LOCAL sys layers*/
+#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_GPU_LOCAL
+//#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_CPU_LOCAL
+#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_GPU_LOCAL
+//#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_CPU_LOCAL
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static PHYS_HEAP_FUNCTIONS gsPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	UMAPhysHeapCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	UMAPhysHeapDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32NextHeapID = 0;
+	IMG_UINT32 uiHeapCount = 1;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	uiHeapCount += 2;
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	uiHeapCount += 1;
+#endif
+	uiHeapCount += !PVRSRV_VZ_MODE_IS(NATIVE) ? 1:0;
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = UMA_HEAP_USAGE_FLAG;
+	ui32NextHeapID++;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "TDFWMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	ui32NextHeapID++;
+
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "TDSECBUFMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags =
+		PHYS_HEAP_USAGE_GPU_SECURE;
+	ui32NextHeapID++;
+
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "DEDICATEDFWMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	ui32NextHeapID++;
+#endif
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+		pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+		ui32NextHeapID++;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszName                = "nohw";
+	psDevConfig->pszVersion             = NULL;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr   = 0x00f00baa;
+	psDevConfig->ui32RegsSize           = 0x4000;
+	psDevConfig->ui32IRQ                = 0x00000bad;
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+
+	psDevConfig->eDefaultHeap = UMA_DEFAULT_HEAP;
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+
+	psDevConfig->bDevicePA0IsValid       = IMG_FALSE;
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Pdump validation system registers */
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+	{
+		PVRSRV_DEVICE_NODE* psDeviceNode;
+
+		/* Pretend we are calling for device 0 for now
+		   (sDeviceNode.sDevId.ui32InternalID is 0 above)
+		   until this code can be moved elsewhere.
+		 */
+		psDeviceNode = OSAllocZMem(sizeof(*psDeviceNode));
+		PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "OSAllocZMem");
+
+		PVRSRVConfigureSysCtrl(psDeviceNode, NULL, PDUMP_FLAGS_CONTINUOUS);
+#if defined(SUPPORT_SECURITY_VALIDATION)
+		PVRSRVConfigureTrustedDevice(psDeviceNode, NULL, PDUMP_FLAGS_CONTINUOUS);
+#endif
+		OSFreeMem(psDeviceNode);
+	}
+#endif
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_TRUE;
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	PVR_UNREFERENCED_PARAMETER(ui32IRQ);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(pfnLISR);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(phLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.h
new file mode 100644
index 000000000000..acbadc22e6a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(SYSCCONFIG_H)
+#define SYSCCONFIG_H
+
+
+#define RGX_NOHW_CORE_CLOCK_SPEED 100000000
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* SYSCCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysinfo.h
new file mode 100644
index 000000000000..566fc4bb7256
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysinfo.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME    "rgxnohw"
+#endif
+
+#endif	/* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/Kbuild.mk
new file mode 100644
index 000000000000..fe58742c81a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/Kbuild.mk
@@ -0,0 +1,42 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+$(PVRSRV_MODNAME)-y += services/system/$(PVR_SYSTEM)/sysconfig.o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.c
new file mode 100644
index 000000000000..6b65db8c40fd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.c
@@ -0,0 +1,664 @@
+/*************************************************************************/ /*!
+@File           sysconfig.c
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#include "img_defs.h"
+#include "physheap.h"
+#include "pvrsrv.h"
+#include "rgxdevice.h"
+#include "interrupt_support.h"
+#include "sysconfig.h"
+
+#define VZ_EXAMPLE_SYSTEM_NAME "vz_example"
+
+static void SysCpuPAToDevPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_DEV_PHYADDR *psDevPA, IMG_CPU_PHYADDR *psCpuPA);
+static void SysDevPAToCpuPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_CPU_PHYADDR *psCpuPA, IMG_DEV_PHYADDR *psDevPA);
+
+typedef struct _SYS_DATA_
+{
+	IMG_HANDLE hSysLISRData;
+	PFN_LISR pfnDeviceLISR;
+	void *pvDeviceLISRData;
+	PVRSRV_DRIVER_MODE eDriverMode;
+} SYS_DATA;
+
+typedef enum _PHYS_HEAP_IDX_
+{
+	PHYS_HEAP_IDX_SYSMEM,
+	PHYS_HEAP_IDX_FIRMWARE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP_IDX_FW_PT,
+#endif
+	PHYS_HEAP_IDX_COUNT,
+} PHYS_HEAP_IDX;
+
+static PHYS_HEAP_FUNCTIONS      gsPhysHeapFuncs = {
+	SysCpuPAToDevPA,        /* pfnCpuPAddrToDevPAddr */
+	SysDevPAToCpuPA,        /* pfnDevPAddrToCpuPAddr */
+};
+
+/*
+ * CPU Physical to Device Physical address translation:
+ * Template implementation below assumes CPU and GPU views of physical memory are identical
+ */
+static void SysCpuPAToDevPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_DEV_PHYADDR *psDevPA, IMG_CPU_PHYADDR *psCpuPA)
+{
+	/* Optimise common case */
+	psDevPA[0].uiAddr = psCpuPA[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPA[ui32Idx].uiAddr = psCpuPA[ui32Idx].uiAddr;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+}
+
+/*
+ * Device Physical to CPU Physical address translation:
+ * Template implementation below assumes CPU and GPU views of physical memory are identical
+ */
+static void SysDevPAToCpuPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_CPU_PHYADDR *psCpuPA, IMG_DEV_PHYADDR *psDevPA)
+{
+	/* Optimise common case */
+	psCpuPA[0].uiAddr = psDevPA[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPA[ui32Idx].uiAddr = psDevPA[ui32Idx].uiAddr;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+}
+
+static IMG_BOOL SystemISRHandler(void *pvData)
+{
+	SYS_DATA *psSysData = pvData;
+	IMG_BOOL bHandled;
+
+	/* Any special system interrupt handling goes here */
+
+#if defined(RGX_SINGLE_IRQ_WORKAROUND)
+	if (psSysData->eDriverMode == DRIVER_MODE_HOST)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = psSysData->pvDeviceLISRData;
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		IMG_UINT32 ui32DriverID;
+
+		/* When hardware support for multiple IRQs is not implemented, the virtualized GPU
+		 * cannot notify drivers of events using dedicated interrupt lines. The single line
+		 * available is routed to Host driver, which in turn broadcasts the IRQ to all other
+		 * active Drivers currently connected to the Firmware. */
+		FOREACH_ACTIVE_DRIVER(psDevInfo, ui32DriverID)
+		{
+			/* the customer must implement a method which allows the Host driver to trigger
+			 * an interrupt signal into each operating system running a Guest GPU driver.
+			 * This method is platform specific. */
+
+			/* INJECT_IRQ_INTO_VM(ui32DriverID); */
+		}
+		END_FOREACH_ACTIVE_DRIVER
+	}
+#endif
+
+	bHandled = psSysData->pfnDeviceLISR(psSysData->pvDeviceLISRData);
+	return bHandled;
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+                                  IMG_UINT32 ui32IRQ,
+                                  const IMG_CHAR *pszName,
+                                  PFN_LISR pfnLISR,
+                                  void *pvData,
+                                  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	PVRSRV_ERROR eError;
+
+	if (psSysData->hSysLISRData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ISR for %s already installed!", __func__, pszName));
+		return PVRSRV_ERROR_CANT_REGISTER_CALLBACK;
+	}
+
+	/* Wrap the device LISR */
+	psSysData->pfnDeviceLISR = pfnLISR;
+	psSysData->pvDeviceLISRData = pvData;
+
+	eError = OSInstallSystemLISR(&psSysData->hSysLISRData, ui32IRQ, pszName,
+	                             SystemISRHandler, psSysData,
+	                             SYS_IRQ_FLAG_TRIGGER_DEFAULT);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	*phLISRData = psSysData;
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, ui32IRQ));
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hLISRData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSysData);
+
+	eError = OSUninstallSystemLISR(psSysData->hSysLISRData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* clear interrupt data */
+	psSysData->pfnDeviceLISR    = NULL;
+	psSysData->pvDeviceLISRData = NULL;
+	psSysData->hSysLISRData     = NULL;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR SysPrePower(IMG_HANDLE hSysData,
+                                PVRSRV_SYS_POWER_STATE eNewPowerState,
+                                PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                                PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+
+	if (psSysData->eDriverMode == DRIVER_MODE_GUEST)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", PVRSRV_OK);
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	/* on powering down */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_ON)
+	{
+		IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+
+		void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+		                                 SOC_REGBANK_SIZE,
+		                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		OSWriteHWReg32(pvSocRegs, POW_DOMAIN_DISABLE_REG, POW_DOMAIN_GPU);
+		OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR SysPostPower(IMG_HANDLE hSysData,
+                                 PVRSRV_SYS_POWER_STATE eNewPowerState,
+                                 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                                 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+
+	if (psSysData->eDriverMode == DRIVER_MODE_GUEST)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", PVRSRV_OK);
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	/* on powering up */
+	if (eCurrentPowerState != PVRSRV_SYS_POWER_STATE_ON)
+	{
+		IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+
+		void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+		                                 SOC_REGBANK_SIZE,
+		                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		OSWriteHWReg32(pvSocRegs, POW_DOMAIN_ENABLE_REG, POW_DOMAIN_GPU);
+		OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_SYS_POWER_STATE RGXGpuDomainPower(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+
+	void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+	                                 SOC_REGBANK_SIZE,
+	                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	IMG_UINT32 ui32SocDomainPower = OSReadHWReg32(pvSocRegs, POW_DOMAIN_STATUS_REG);
+
+	bool bGpuDomainIsPowered = BITMASK_HAS(ui32SocDomainPower, POW_DOMAIN_GPU);
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+
+	return (bGpuDomainIsPowered) ? PVRSRV_SYS_POWER_STATE_ON : PVRSRV_SYS_POWER_STATE_OFF;
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Features);
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+static PVRSRV_DRIVER_MODE GetDriverMode(struct platform_device *psDev)
+{
+	PVRSRV_DRIVER_MODE eDriverMode;
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (of_property_read_u32(psDev->dev.of_node, "vz-mode", (IMG_UINT32*) &eDriverMode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Cannot retrieve driver mode from Device Tree. "
+								  "Default to native mode.", __func__));
+		eDriverMode = DRIVER_MODE_NATIVE;
+	}
+#else
+	eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+
+	return eDriverMode;
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct platform_device *psDev;
+	struct resource *dev_res = NULL;
+	int dev_irq;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	SYS_DATA *psSysData;
+	PHYS_HEAP_CONFIG *psPhysHeapConfig;
+	IMG_UINT32 ui32DeviceID;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo) +
+							  sizeof(*psSysData) +
+							  sizeof(*psPhysHeapConfig) * PHYS_HEAP_IDX_COUNT);
+
+	PVR_LOG_GOTO_IF_NOMEM(psDevConfig, eError, err_sys);
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+	psSysData = (SYS_DATA *)((IMG_CHAR *)psRGXTimingInfo + sizeof(*psRGXTimingInfo));
+	psPhysHeapConfig = (PHYS_HEAP_CONFIG *)((IMG_CHAR *)psSysData + sizeof(*psSysData));
+
+	psDev = to_platform_device((struct device *)pvOSDevice);
+
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	eError = PVRSRVAcquireInternalID(&ui32DeviceID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireInternalID", err_sys);
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+		psDevConfig->eDriverMode = GetDriverMode(psDev);
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+
+	dev_irq = platform_get_irq(psDev, 0);
+	if (dev_irq < 0)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("platform_get_irq", eError, PVRSRV_ERROR_INVALID_DEVICE, err_sys);
+	}
+
+	dev_res = platform_get_resource(psDev, IORESOURCE_MEM, 0);
+	if (dev_res == NULL)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("platform_get_resource", eError, PVRSRV_ERROR_INVALID_DEVICE, err_sys);
+	}
+
+	/* Device Setup */
+	psDevConfig->pvOSDevice              = pvOSDevice;
+	psDevConfig->pszName                 = VZ_EXAMPLE_SYSTEM_NAME;
+	psDevConfig->pszVersion              = NULL;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr    = dev_res->start;
+	psDevConfig->ui32IRQ                 = dev_irq;
+
+	psDevConfig->ui32RegsSize            = (unsigned int)(dev_res->end - dev_res->start);
+
+	PVR_ASSERT(psDevConfig->ui32RegsSize >= RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+
+	/* Power management */
+	psDevConfig->pfnPrePowerState        = SysPrePower;
+	psDevConfig->pfnPostPowerState       = SysPostPower;
+	psDevConfig->pfnGpuDomainPower       = PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) ? NULL : RGXGpuDomainPower;
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = DEFAULT_CLOCK_RATE;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = 0;
+
+	/* Setup RGX specific data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Minimal configuration */
+	psDevConfig->pfnClockFreqGet         = NULL;
+	psDevConfig->hDevData                = psRGXData;
+	psDevConfig->hSysData                = psSysData;
+	psDevConfig->pfnSysDevFeatureDepInit = SysDevFeatureDepInit;
+	psDevConfig->bHasFBCDCVersion31      = IMG_FALSE;
+
+	psDevConfig->pasPhysHeaps = psPhysHeapConfig;
+
+	return eError;
+
+err_sys:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+/* Obtain the IPA of the carveout range reserved for this VM */
+static IMG_UINT64 GetFwCarveoutBase(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	struct platform_device *psDev = to_platform_device((struct device *)psDevConfig->pvOSDevice);
+	IMG_UINT64 ui64BaseAddress;
+
+	if (of_property_read_u64(psDev->dev.of_node, "fw-carveout", &ui64BaseAddress))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Cannot retrieve firmware carveout address from Device Tree."
+								  " Using default Base Address: 0x%llX",
+								  __func__, FW_CARVEOUT_IPA_BASE));
+		ui64BaseAddress = FW_CARVEOUT_IPA_BASE;
+	}
+
+	return ui64BaseAddress;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+static IMG_UINT64 GetFwPageTableCarveoutBase(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	struct platform_device *psDev = to_platform_device((struct device *)psDevConfig->pvOSDevice);
+	IMG_UINT64 ui64BaseAddress;
+
+	if (of_property_read_u64(psDev->dev.of_node, "fw-pt-carveout", &ui64BaseAddress))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Cannot retrieve firmware page table carveout address"
+								  " from Device Tree. Using default Base Address: 0x%llX",
+								  __func__, FW_PT_CARVEOUT_IPA_BASE));
+		ui64BaseAddress = FW_PT_CARVEOUT_IPA_BASE;
+	}
+
+	return ui64BaseAddress;
+}
+#endif
+#endif
+
+static PVRSRV_ERROR PhysHeapCfgCreate(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_CPU_PHYADDR sCpuBase;
+	IMG_DEV_PHYADDR sDeviceBase;
+	PHYS_HEAP_CONFIG *psPhysHeapConfig = psDevConfig->pasPhysHeaps;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Virtualization driver */
+	/* Heap configuration for general use */
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.pszHeapName = "uma_gpu_local";
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.hPrivData = NULL;
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	{
+		/*
+		 * In a static memory VZ setup, the size and base addresses of
+		 * all Host and Guest drivers' Firmware heaps are laid out
+		 * consecutively in a physically contiguous memory range known
+		 * in advance by the Host driver.
+		 *
+		 * During the Host driver initialisation, it maps the entire range
+		 * into the Firmware's virtual address space. No other mapping
+		 * operations into the Firmware's VA space are needed after this.
+		 * Guest driver must know only the base address of the range
+		 * assign to it during system planning stage.
+		 *
+		 * The system integrator must ensure that:
+		 *  - physically contiguous RAM region used as a Firmware heap
+		 *    is not managed by any OS or Hypervisor (a carveout)
+		 *  - Host driver must come online before any Guest drivers
+		 *  - Host driver sets up the Firmware before Guests submits work
+		 */
+
+		sCpuBase.uiAddr = GetFwCarveoutBase(psDevConfig);
+		SysCpuPAToDevPA(NULL, 1, &sDeviceBase, &sCpuBase);
+
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.pszPDumpMemspaceName = "SYSMEM_FW";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.psMemFuncs = &gsPhysHeapFuncs;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.pszHeapName = "lma_fw_shared";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.sStartAddr = sCpuBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.sCardBase = sDeviceBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.hPrivData = NULL;
+	}
+#else
+	/* Dynamic Firmware heap allocation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		/*
+		 * Guest drivers must provide a physically contiguous memory
+		 * range to the Host via a PVZ call to have it mapped into
+		 * the Firmware's address space. Guest drivers use the OS
+		 * kernel's DMA/CMA allocator to obtain a DMA buffer to be
+		 * used as a firmware heap. This memory will be managed
+		 * internally by the Guest driver after the heap is created.
+		 */
+		DMA_ALLOC *psDmaAlloc = OSAllocZMem(sizeof(DMA_ALLOC));
+
+		eError = (psDmaAlloc == NULL) ? PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_OK;
+		if (eError == PVRSRV_OK)
+		{
+			psDmaAlloc->pvOSDevice = psDevConfig->pvOSDevice;
+			psDmaAlloc->ui64Size = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+			eError = SysDmaAllocMem(psDmaAlloc);
+			if (eError == PVRSRV_OK)
+			{
+				eError = SysDmaRegisterForIoRemapping(psDmaAlloc);
+
+				if (eError == PVRSRV_OK)
+				{
+					sCpuBase.uiAddr = psDmaAlloc->sBusAddr.uiAddr;
+					SysCpuPAToDevPA(NULL, 1, &sDeviceBase, &sCpuBase);
+
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].eType = PHYS_HEAP_TYPE_DMA;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.pszPDumpMemspaceName = "SYSMEM_FW";
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.psMemFuncs = &gsPhysHeapFuncs;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.pszHeapName = "dma_fw_shared";
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.sStartAddr = sCpuBase;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.sCardBase = sDeviceBase;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.hPrivData = psDmaAlloc;
+				}
+				else
+				{
+					OSFreeMem(psDmaAlloc);
+					SysDmaFreeMem(psDmaAlloc);
+				}
+			}
+			else
+			{
+				OSFreeMem(psDmaAlloc);
+			}
+		}
+	}
+	else
+	{
+		/*
+		 * The Host or Native driver uses memory managed by
+		 * the kernel on a page granularity and creates on-demand
+		 * mappings into the Firmware's address space.
+		 */
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].eType = PHYS_HEAP_TYPE_UMA;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM_FW";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.pszHeapName = "uma_fw_shared";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.hPrivData = NULL;
+	}
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		sCpuBase.uiAddr = GetFwPageTableCarveoutBase(psDevConfig);
+		SysCpuPAToDevPA(NULL, 1, &sDeviceBase, &sCpuBase);
+
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP_PT;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.pszPDumpMemspaceName = "SYSMEM_FW_PT";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.psMemFuncs = &gsPhysHeapFuncs;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.pszHeapName = "lma_fw_pt";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.sStartAddr = sCpuBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.sCardBase = sDeviceBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.uiSize = RGX_FIRMWARE_MAX_PAGETABLE_SIZE;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.hPrivData = NULL;
+	}
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	/* Device's physical heaps */
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	psDevConfig->ui32PhysHeapCount = PHYS_HEAP_IDX_COUNT;
+
+	return eError;
+}
+
+static void PhysHeapCfgDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		DMA_ALLOC *psDmaAlloc = psDevConfig->pasPhysHeaps[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.hPrivData;
+
+		SysDmaDeregisterForIoRemapping(psDmaAlloc);
+		SysDmaFreeMem(psDmaAlloc);
+		OSFreeMem(psDmaAlloc);
+	}
+#endif
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	PVRSRV_ERROR eError;
+
+	eError = DeviceConfigCreate(pvOSDevice, &psDevConfig);
+	if (eError == PVRSRV_OK)
+	{
+		eError = PhysHeapCfgCreate(psDevConfig);
+		if (eError != PVRSRV_OK)
+		{
+			OSFreeMem(psDevConfig);
+		}
+
+		*ppsDevConfig = psDevConfig;
+	}
+
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PhysHeapCfgDestroy(psDevConfig);
+
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+						  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						  void *pvDumpDebugFile)
+{
+	/* print here any system information useful for debug dumps */
+
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.h
new file mode 100644
index 000000000000..0670c7dde015
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File           sysconfig.h
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(__SYSCCONFIG_H__)
+#define __SYSCCONFIG_H__
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS 100
+
+#define DEFAULT_CLOCK_RATE			IMG_UINT64_C(600000000)
+
+/* fixed IPA Base of the memory carveout reserved for the GPU Firmware Heaps */
+#define FW_CARVEOUT_IPA_BASE		IMG_UINT64_C(0x7E000000)
+
+/* fixed IPA Base of the memory carveout reserved for the Firmware's Page Tables */
+#define FW_PT_CARVEOUT_IPA_BASE		IMG_UINT64_C(0x8F000000)
+
+/* mock SoC registers */
+#define SOC_REGBANK_BASE			IMG_UINT64_C(0xF0000000)
+#define SOC_REGBANK_SIZE			IMG_UINT32_C(0x10000)
+#define POW_DOMAIN_ENABLE_REG		IMG_UINT32_C(0xA000)
+#define POW_DOMAIN_DISABLE_REG		IMG_UINT32_C(0xA008)
+#define POW_DOMAIN_STATUS_REG		IMG_UINT32_C(0xA010)
+
+#define POW_DOMAIN_GPU				IMG_UINT32_C(0x1)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* __SYSCCONFIG_H__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysinfo.h
new file mode 100644
index 000000000000..0ab175e0692e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysinfo.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME                         "powervr-vz-example"
+#endif
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.c b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.c
new file mode 100644
index 000000000000..94a7629c1f51
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.c
@@ -0,0 +1,240 @@
+/*************************************************************************/ /*!
+@File           vz_validation.c
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "sysconfig.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "vz_validation.h"
+#include "virt_validation_defs.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef struct _VALIDATION_CTRL_
+{
+	IMG_BOOL	bUnload;
+	IMG_HANDLE	hMPUWatchdogThread;
+	IMG_HANDLE	hSystemWatchdogEvObj;
+} VALIDATION_CTRL;
+
+static VALIDATION_CTRL gsValidCtrl = {false, NULL, NULL};
+
+static void SysPrintAndResetFaultStatusRegister(void)
+{
+	IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+	IMG_UINT32 ui32MPUEventStatus;
+
+	void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+	                                 SOC_REGBANK_SIZE,
+	                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	ui32MPUEventStatus = OSReadHWReg32(pvSocRegs, MPU_EVENT_STATUS_REG);
+
+	if (ui32MPUEventStatus & MPU_GPU_BUS_REQUESTER)
+	{
+		IMG_UINT32 ui32MPUEventOSID      = OSReadHWReg32(pvSocRegs, MPU_EVENT_OSID_REG);
+		IMG_UINT32 ui32MPUEventAddress   = OSReadHWReg32(pvSocRegs, MPU_EVENT_ADDRESS_REG);
+		IMG_UINT32 ui32MPUEventDirection = OSReadHWReg32(pvSocRegs, MPU_EVENT_DIRECTION_REG);
+
+		PVR_DPF((PVR_DBG_ERROR, "MPU fault event: GPU attempted an illegal %s access to address 0x%08X while emitting OSID %u",
+		                        (ui32MPUEventDirection == MPU_WRITE_ACCESS) ? "WRITE" : "READ",
+		                        ui32MPUEventAddress,
+		                        ui32MPUEventOSID));
+
+		OSWriteHWReg32(pvSocRegs, MPU_EVENT_CLEAR_REG, 1);
+	}
+
+	OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+}
+
+void SysInitValidation(IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                       IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS])
+{
+	IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+	IMG_UINT32 ui32OSID, ui32Region;
+
+	void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+	                                 SOC_REGBANK_SIZE,
+	                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	IMG_UINT64 aui64RegMin[GPUVIRT_VALIDATION_MAX_OS][GPUVIRT_VALIDATION_NUM_REGIONS] =
+	{
+		{MPU_PROTECTED_RANGE0_START_REG, MPU_PROTECTED_RANGE8_START_REG},
+		{MPU_PROTECTED_RANGE1_START_REG, MPU_PROTECTED_RANGE9_START_REG},
+		{MPU_PROTECTED_RANGE2_START_REG, MPU_PROTECTED_RANGE10_START_REG},
+		{MPU_PROTECTED_RANGE3_START_REG, MPU_PROTECTED_RANGE11_START_REG},
+		{MPU_PROTECTED_RANGE4_START_REG, MPU_PROTECTED_RANGE12_START_REG},
+		{MPU_PROTECTED_RANGE5_START_REG, MPU_PROTECTED_RANGE13_START_REG},
+		{MPU_PROTECTED_RANGE6_START_REG, MPU_PROTECTED_RANGE14_START_REG},
+		{MPU_PROTECTED_RANGE7_START_REG, MPU_PROTECTED_RANGE15_START_REG}
+	};
+
+	IMG_UINT64 aui64RegMax[GPUVIRT_VALIDATION_MAX_OS][GPUVIRT_VALIDATION_NUM_REGIONS] =
+	{
+		{MPU_PROTECTED_RANGE0_END_REG, MPU_PROTECTED_RANGE8_END_REG},
+		{MPU_PROTECTED_RANGE1_END_REG, MPU_PROTECTED_RANGE9_END_REG},
+		{MPU_PROTECTED_RANGE2_END_REG, MPU_PROTECTED_RANGE10_END_REG},
+		{MPU_PROTECTED_RANGE3_END_REG, MPU_PROTECTED_RANGE11_END_REG},
+		{MPU_PROTECTED_RANGE4_END_REG, MPU_PROTECTED_RANGE12_END_REG},
+		{MPU_PROTECTED_RANGE5_END_REG, MPU_PROTECTED_RANGE13_END_REG},
+		{MPU_PROTECTED_RANGE6_END_REG, MPU_PROTECTED_RANGE14_END_REG},
+		{MPU_PROTECTED_RANGE7_END_REG, MPU_PROTECTED_RANGE15_END_REG}
+	};
+
+	IMG_UINT64 aui64RangeOSIDAccessReg[] =
+	{
+		MPU_PROTECTED_RANGE0_OSID_REG,
+		MPU_PROTECTED_RANGE1_OSID_REG,
+		MPU_PROTECTED_RANGE2_OSID_REG,
+		MPU_PROTECTED_RANGE3_OSID_REG,
+		MPU_PROTECTED_RANGE4_OSID_REG,
+		MPU_PROTECTED_RANGE5_OSID_REG,
+		MPU_PROTECTED_RANGE6_OSID_REG,
+		MPU_PROTECTED_RANGE7_OSID_REG,
+		MPU_PROTECTED_RANGE8_OSID_REG,
+		MPU_PROTECTED_RANGE9_OSID_REG,
+		MPU_PROTECTED_RANGE10_OSID_REG,
+		MPU_PROTECTED_RANGE11_OSID_REG,
+		MPU_PROTECTED_RANGE12_OSID_REG,
+		MPU_PROTECTED_RANGE13_OSID_REG,
+		MPU_PROTECTED_RANGE14_OSID_REG,
+		MPU_PROTECTED_RANGE15_OSID_REG
+	};
+
+	for (ui32OSID = 0; ui32OSID < GPUVIRT_VALIDATION_NUM_OS ; ui32OSID++)
+	{
+		/* every OSID gets access to 2 ranges: one secure range and one shared with all the other OSIDs
+		 * e.g. OSID 0: secure RANGE 0 & shared RANGE 8 */
+		OSWriteHWReg64(pvSocRegs, aui64RangeOSIDAccessReg[ui32OSID], ui32OSID);
+		OSWriteHWReg64(pvSocRegs, aui64RangeOSIDAccessReg[ui32OSID+GPUVIRT_VALIDATION_MAX_OS], ui32OSID);
+
+		for (ui32Region=0; ui32Region < GPUVIRT_VALIDATION_NUM_REGIONS; ui32Region++)
+		{
+			OSWriteHWReg64(pvSocRegs, aui64RegMin[ui32OSID][ui32Region], aui64OSidMin[ui32Region][ui32OSID]);
+			OSWriteHWReg64(pvSocRegs, aui64RegMax[ui32OSID][ui32Region], aui64OSidMax[ui32Region][ui32OSID]);
+		}
+	}
+
+	OSWriteHWReg32(pvSocRegs, MPU_PROTECTION_ENABLE_REG, 1);
+
+	OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+}
+
+static void MPUWatchdogThread(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hOSEvent;
+	IMG_UINT32 ui32Timeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+
+	/* Open an event on the system watchdog event object so we can listen on it
+	   and abort the system watchdog thread. */
+	eError = OSEventObjectOpen(gsValidCtrl.hSystemWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+	while (!gsValidCtrl.bUnload)
+	{
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+		if (eError == PVRSRV_OK)
+		{
+			if (gsValidCtrl.bUnload)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Shutdown event received.", __func__));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Power state change event received.", __func__));
+			}
+		}
+		else if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			/* If timeout do nothing otherwise print warning message. */
+			PVR_DPF((PVR_DBG_ERROR, "%s: "
+					"Error (%d) when waiting for event!", __func__, eError));
+		}
+
+		SysPrintAndResetFaultStatusRegister();
+	}
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+void CreateMPUWatchdogThread(void)
+{
+	/* Create the SysWatchdog Event Object */
+	OSEventObjectCreate("PVRSRV_SYSWDG_EVENTOBJECT", &gsValidCtrl.hSystemWatchdogEvObj);
+
+	/* Create a thread which is used to detect fatal errors */
+	OSThreadCreatePriority(&gsValidCtrl.hMPUWatchdogThread,
+	                       "emu_check_fault_wdg",
+	                       MPUWatchdogThread,
+	                       NULL,
+	                       IMG_TRUE,
+	                       &gsValidCtrl,
+	                       0);
+}
+
+void DestroyMPUWatchdogThread(void)
+{
+	gsValidCtrl.bUnload = IMG_TRUE;
+
+	if (gsValidCtrl.hMPUWatchdogThread)
+	{
+		if (gsValidCtrl.hSystemWatchdogEvObj)
+		{
+			OSEventObjectSignal(gsValidCtrl.hSystemWatchdogEvObj);
+		}
+		LOOP_UNTIL_TIMEOUT(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			OSThreadDestroy(gsValidCtrl.hMPUWatchdogThread);
+			gsValidCtrl.hMPUWatchdogThread = NULL;
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT();
+	}
+
+	if (gsValidCtrl.hSystemWatchdogEvObj)
+	{
+		OSEventObjectDestroy(gsValidCtrl.hSystemWatchdogEvObj);
+		gsValidCtrl.hSystemWatchdogEvObj = NULL;
+	}
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.h b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.h
new file mode 100644
index 000000000000..f68facb42d04
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.h
@@ -0,0 +1,55 @@
+/*************************************************************************/ /*!
+@File           vz_validation.h
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__VZ_VALIDATION_H__)
+#define __VZ_VALIDATION_H__
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+void CreateMPUWatchdogThread(void);
+void DestroyMPUWatchdogThread(void);
+
+void SysInitValidation(IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                       IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+#endif
+
+#endif /* __VZ_VALIDATION_H__ */
diff --git a/drivers/gpu/drm/img/kernel_compatibility.h b/drivers/gpu/drm/img/kernel_compatibility.h
new file mode 100644
index 000000000000..048f4449ae68
--- /dev/null
+++ b/drivers/gpu/drm/img/kernel_compatibility.h
@@ -0,0 +1,404 @@
+/*************************************************************************/ /*!
+@Title          Kernel versions compatibility macros
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_COMPATIBILITY_H__
+#define __KERNEL_COMPATIBILITY_H__
+
+#include <linux/version.h>
+#include <linux/compiler.h>
+
+/* Explicitly error out if DDK is built against out-of-support Linux kernel */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
+#error Linux kernels older than 4.9.0 are not supported
+#endif
+
+/*
+ * Stop supporting an old kernel? Remove the top block.
+ * New incompatible kernel?       Append a new block at the bottom.
+ *
+ * Please write your version test as `VERSION < X.Y`, and use the earliest
+ * possible version :)
+ *
+ * If including this header file in other files, this should always be the
+ * last file included, as it can affect definitions/declarations in files
+ * included after it.
+ */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+#define refcount_read(r) atomic_read(r)
+#define drm_mm_insert_node(mm, node, size) drm_mm_insert_node(mm, node, size, 0, DRM_MM_SEARCH_DEFAULT)
+
+#define drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd) drm_helper_mode_fill_fb_struct(fb, mode_cmd)
+
+/*
+ * In Linux Kernels >= 4.12 for x86 another level of page tables has been
+ * added. The added level (p4d) sits between pgd and pud, so when it
+ * doesn`t exist, pud_offset function takes pgd as a parameter instead
+ * of p4d.
+ */
+#define p4d_t pgd_t
+#define p4d_offset(pgd, address) (pgd)
+#define p4d_none(p4d) (0)
+#define p4d_bad(p4d) (0)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+
+#define drm_mode_object_get(obj)          drm_mode_object_reference(obj)
+#define drm_mode_object_put(obj)          drm_mode_object_unreference(obj)
+#define drm_connector_get(obj)            drm_connector_reference(obj)
+#define drm_connector_put(obj)            drm_connector_unreference(obj)
+#define drm_framebuffer_get(obj)          drm_framebuffer_reference(obj)
+#define drm_framebuffer_put(obj)          drm_framebuffer_unreference(obj)
+#define drm_gem_object_get(obj)           drm_gem_object_reference(obj)
+#define drm_gem_object_put_locked(obj)    drm_gem_object_unreference(obj)
+#define __drm_gem_object_put(obj)         __drm_gem_object_unreference(obj)
+#define drm_property_blob_get(obj)        drm_property_reference_blob(obj)
+#define drm_property_blob_put(obj)        drm_property_unreference_blob(obj)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+
+#define drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, format_modifiers, type, name, ...) \
+	({ (void) format_modifiers; drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, type, name, ##__VA_ARGS__); })
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0))
+
+#define drm_dev_put(dev) drm_dev_unref(dev)
+
+#define drm_mode_object_find(dev, file_priv, id, type) drm_mode_object_find(dev, id, type)
+#define drm_encoder_find(dev, file_priv, id) drm_encoder_find(dev, id)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_plane_helper_check_state(plane_state, \
+												 &__clip, \
+												 min_scale, max_scale, \
+												 can_position, \
+												 can_update_disabled); \
+		__ret; \
+	})
+
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_atomic_helper_check_plane_state(plane_state, \
+														crtc_state, \
+														&__clip, \
+														min_scale, max_scale, \
+														can_position, \
+														can_update_disabled); \
+		__ret; \
+	})
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+
+#define drm_connector_attach_encoder(connector, encoder) \
+	drm_mode_connector_attach_encoder(connector, encoder)
+
+#define drm_connector_update_edid_property(connector, edid) \
+	drm_mode_connector_update_edid_property(connector, edid)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+
+/*
+ * Work around architectures, e.g. MIPS, that define copy_from_user and
+ * copy_to_user as macros that call access_ok, as this gets redefined below.
+ * As of kernel 4.12, these functions are no longer defined per-architecture
+ * so this work around isn't needed.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#if defined(copy_from_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_from_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	return copy_from_user(to, from, n);
+}
+
+#undef copy_from_user
+#define copy_from_user(to, from, n) __copy_from_user(to, from, n)
+#endif
+
+#if defined(copy_to_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_to_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	return copy_to_user(to, from, n);
+}
+
+#undef copy_to_user
+#define copy_to_user(to, from, n) __copy_to_user(to, from, n)
+#endif
+#endif
+
+/*
+ * Linux 5.0 dropped the type argument.
+ *
+ * This is unused in at least Linux 3.4 and above for all architectures other
+ * than 'um' (User Mode Linux), which stopped using it in 4.2.
+ */
+#if defined(access_ok)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around access_ok being defined as a macro.
+  */
+static inline int
+__pvr_access_ok_compat(int type, const void __user * addr, unsigned long size)
+{
+	return access_ok(type, addr, size);
+}
+
+#undef access_ok
+#define access_ok(addr, size) __pvr_access_ok_compat(0, addr, size)
+#else
+#define access_ok(addr, size) access_ok(0, addr, size)
+#endif
+
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+#define MODULE_IMPORT_NS(ns)
+#endif
+
+/*
+ * Before v5.8, the "struct mm" has a semaphore named "mmap_sem" which is
+ * renamed to "mmap_lock" in v5.8. Moreover, new APIs are provided to
+ * access this lock starting from v5.8.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+
+#define mmap_write_lock(mm)   down_write(&mm->mmap_sem)
+#define mmap_write_unlock(mm) up_write(&mm->mmap_sem)
+
+#define mmap_read_lock(mm)    down_read(&mm->mmap_sem)
+#define mmap_read_unlock(mm)  up_read(&mm->mmap_sem)
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define drm_gem_object_put(obj) drm_gem_object_unreference_unlocked(obj)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+#define drm_gem_object_put(obj) drm_gem_object_put_unlocked(obj)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+
+#define drm_prime_pages_to_sg(dev, pages, nr_pages) \
+	drm_prime_pages_to_sg(pages, nr_pages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+
+struct dma_buf_map {
+	void *vaddr;
+};
+
+#define dma_buf_vmap(dmabuf, map) \
+	({ \
+		(map)->vaddr = dma_buf_vmap(dmabuf); \
+		(map)->vaddr ? 0 : ((dmabuf) && (dmabuf)->ops->vmap) ? -ENOMEM : -EINVAL; \
+	})
+
+#define dma_buf_vunmap(dmabuf, map) \
+	({ \
+		dma_buf_vunmap(dmabuf, (map)->vaddr); \
+		(map)->vaddr = NULL; \
+	})
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0))
+
+#define drm_prime_sg_to_page_array(sgt, pages, npages) \
+	drm_prime_sg_to_page_addr_arrays(sgt, pages, NULL, npages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0))
+
+#define drm_gem_plane_helper_prepare_fb drm_gem_fb_prepare_fb
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0)) */
+
+/*
+ * Linux 5.11 renames the privileged uaccess routines for arm64 and Android
+ * kernel v5.10 merges the change as well. These routines are only used for
+ * arm64 so CONFIG_ARM64 testing can be ignored.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) || \
+	((LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) && !defined(ANDROID))
+#define uaccess_enable_privileged() uaccess_enable()
+#define uaccess_disable_privileged() uaccess_disable()
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
+#define pde_data PDE_DATA
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+#define kthread_complete_and_exit(comp, ret) complete_and_exit(comp, ret);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0))
+#define iosys_map dma_buf_map
+#define iosys_map_set_vaddr dma_buf_map_set_vaddr
+#define iosys_map_set_vaddr_iomem dma_buf_map_set_vaddr_iomem
+#define iosys_map_clear dma_buf_map_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+#define register_shrinker(shrinker, name) \
+	register_shrinker(shrinker)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
+#define DRM_PLANE_NO_SCALING DRM_PLANE_HELPER_NO_SCALING
+#define drm_plane_helper_destroy drm_primary_helper_destroy
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0))
+#define genl_split_ops genl_ops
+#define COMPAT_FB_INFO fbdev
+#define drm_fb_helper_alloc_info drm_fb_helper_alloc_fbi
+#define drm_fb_helper_unregister_info drm_fb_helper_unregister_fbi
+#else
+#define COMPAT_FB_INFO info
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) || \
+        ((LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) && !defined(ANDROID))
+static inline void pvr_vm_flags_set(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags |= flags;
+}
+static inline void pvr_vm_flags_init(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags = flags;
+}
+static inline void pvr_vm_flags_clear(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags &= ~flags;
+}
+#else
+#define pvr_vm_flags_set  vm_flags_set
+#define pvr_vm_flags_init vm_flags_init
+#define pvr_vm_flags_clear vm_flags_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0))
+#define pvr_class_create(name) class_create(THIS_MODULE, name)
+#else
+#define pvr_class_create(name) class_create(name)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+#define thermal_tripless_zone_device_register(type, devdata, ops, tzp) \
+	thermal_zone_device_register((type), 0, 0, (devdata), (ops), (tzp), 0, 0)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0)) */
+
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+#if !defined(__fallthrough)
+	#if GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+#endif
+
+#endif /* __KERNEL_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/img/kernel_config_compatibility.h b/drivers/gpu/drm/img/kernel_config_compatibility.h
new file mode 100644
index 000000000000..63effd65ecf9
--- /dev/null
+++ b/drivers/gpu/drm/img/kernel_config_compatibility.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@Title          Kernel config compatibility define options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file is exclusively for Linux config kernel options.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_CONFIG_COMPATIBILITY_H__
+#define __KERNEL_CONFIG_COMPATIBILITY_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0))
+#ifdef SUPPORT_DRM_FBDEV_EMULATION
+#define CONFIG_DRM_FBDEV_EMULATION
+#endif
+#endif
+
+#endif /* __KERNEL_CONFIG_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/img/kernel_nospec.h b/drivers/gpu/drm/img/kernel_nospec.h
new file mode 100644
index 000000000000..ef6fee368327
--- /dev/null
+++ b/drivers/gpu/drm/img/kernel_nospec.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@Title          Macro to limit CPU speculative execution in kernel code
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_NOSPEC_H__
+#define __KERNEL_NOSPEC_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 2) ||			\
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) &&		\
+	 LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 18)) ||		\
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) &&		\
+	 LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 81)))
+#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <linux/nospec.h>
+#else
+#define array_index_nospec(index, size) (index)
+#endif
+
+/*
+ * For Ubuntu kernels, the features available for a given Linux version code
+ * may not match those in upstream kernels. This is the case for the
+ * availability of the array_index_nospec macro.
+ */
+#if !defined(array_index_nospec)
+#define array_index_nospec(index, size) (index)
+#endif
+
+#endif /* __KERNEL_NOSPEC_H__ */
diff --git a/drivers/gpu/drm/img/pvr_dma_resv.h b/drivers/gpu/drm/img/pvr_dma_resv.h
new file mode 100644
index 000000000000..a51c9de84ada
--- /dev/null
+++ b/drivers/gpu/drm/img/pvr_dma_resv.h
@@ -0,0 +1,80 @@
+/*************************************************************************/ /*!
+@Title          Kernel reservation object compatibility header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __PVR_DMA_RESV_H__
+#define __PVR_DMA_RESV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/dma-resv.h>
+#else
+#include <linux/reservation.h>
+
+/* Reservation object types */
+#define dma_resv			reservation_object
+#define dma_resv_list			reservation_object_list
+
+/* Reservation object functions */
+#define dma_resv_add_excl_fence		reservation_object_add_excl_fence
+#define dma_resv_add_shared_fence	reservation_object_add_shared_fence
+#define dma_resv_fini			reservation_object_fini
+#define dma_resv_get_excl		reservation_object_get_excl
+#define dma_resv_get_list		reservation_object_get_list
+#define dma_resv_held			reservation_object_held
+#define dma_resv_init			reservation_object_init
+#define dma_resv_reserve_shared		reservation_object_reserve_shared
+#define dma_resv_test_signaled_rcu	reservation_object_test_signaled_rcu
+#define dma_resv_wait_timeout_rcu	reservation_object_wait_timeout_rcu
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
+
+#define dma_resv_shared_list   dma_resv_get_list
+#define dma_resv_excl_fence    dma_resv_get_excl
+#define dma_resv_wait_timeout  dma_resv_wait_timeout_rcu
+#define dma_resv_test_signaled dma_resv_test_signaled_rcu
+#define dma_resv_get_fences    dma_resv_get_fences_rcu
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)) */
+
+#endif /* __PVR_DMA_RESV_H__ */
diff --git a/drivers/gpu/drm/img/pvr_linux_fence.h b/drivers/gpu/drm/img/pvr_linux_fence.h
new file mode 100644
index 000000000000..b9c542a26a65
--- /dev/null
+++ b/drivers/gpu/drm/img/pvr_linux_fence.h
@@ -0,0 +1,103 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence compatibility header
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_LINUX_FENCE_H__)
+#define __PVR_LINUX_FENCE_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+#include <linux/fence.h>
+#else
+#include <linux/dma-fence.h>
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+/* Structures */
+#define	dma_fence fence
+#define dma_fence_array fence_array
+#define	dma_fence_cb fence_cb
+#define	dma_fence_ops fence_ops
+
+/* Defines and Enums */
+#define DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT FENCE_FLAG_ENABLE_SIGNAL_BIT
+#define DMA_FENCE_FLAG_SIGNALED_BIT FENCE_FLAG_SIGNALED_BIT
+#define DMA_FENCE_FLAG_USER_BITS FENCE_FLAG_USER_BITS
+
+#define DMA_FENCE_ERR FENCE_ERR
+#define	DMA_FENCE_TRACE FENCE_TRACE
+#define DMA_FENCE_WARN FENCE_WARN
+
+/* Functions */
+#define dma_fence_add_callback fence_add_callback
+#define dma_fence_context_alloc fence_context_alloc
+#define dma_fence_default_wait fence_default_wait
+#define dma_fence_is_signaled fence_is_signaled
+#define dma_fence_enable_sw_signaling fence_enable_sw_signaling
+#define dma_fence_free fence_free
+#define dma_fence_get fence_get
+#define dma_fence_get_rcu fence_get_rcu
+#define dma_fence_init fence_init
+#define dma_fence_is_array fence_is_array
+#define dma_fence_put fence_put
+#define dma_fence_signal fence_signal
+#define dma_fence_wait fence_wait
+#define to_dma_fence_array to_fence_array
+
+static inline signed long
+dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	signed long lret;
+
+	lret = fence_wait_timeout(fence, intr, timeout);
+	if (lret || timeout)
+		return lret;
+
+	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1 : 0;
+}
+
+#endif
+
+#endif /* !defined(__PVR_LINUX_FENCE_H__) */
diff --git a/drivers/gpu/drm/img/pvr_vmap.h b/drivers/gpu/drm/img/pvr_vmap.h
new file mode 100644
index 000000000000..6424b20e764c
--- /dev/null
+++ b/drivers/gpu/drm/img/pvr_vmap.h
@@ -0,0 +1,83 @@
+/*
+ * @File        pvr_vmap.h
+ * @Title       Utility functions for virtual memory mapping
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_VMAP_H
+#define PVR_VMAP_H
+
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+
+static inline void *pvr_vmap(struct page **pages,
+			     unsigned int count,
+			     __maybe_unused unsigned long flags,
+			     pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	return vmap(pages, count, flags, prot);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	return vm_map_ram(pages, count, -1, prot);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		return vm_map_ram(pages, count, -1);
+	else
+		return vmap(pages, count, flags, prot);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+static inline void pvr_vunmap(const void *vaddr,
+			      __maybe_unused unsigned int count,
+			      __maybe_unused pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	vunmap(vaddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	vm_unmap_ram(vaddr, count);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		vm_unmap_ram(vaddr, count);
+	else
+		vunmap(vaddr);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+#endif /* PVR_VMAP_H */
diff --git a/drivers/gpu/drm/img/pvrversion.h b/drivers/gpu/drm/img/pvrversion.h
new file mode 100644
index 000000000000..4d17f6e55594
--- /dev/null
+++ b/drivers/gpu/drm/img/pvrversion.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File           pvrversion.h
+@Title          PowerVR version numbers and strings.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Version numbers and strings for PowerVR components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRVERSION_H
+#define PVRVERSION_H
+
+#define PVRVERSION_MAJ               24U
+#define PVRVERSION_MIN               2U
+
+#define PVRVERSION_FAMILY           "rogueddk"
+#define PVRVERSION_BRANCHNAME       "24.2"
+#define PVRVERSION_BUILD             6643903
+#define PVRVERSION_BSCONTROL        "Rogue_DDK_Linux_WS"
+
+#define PVRVERSION_STRING           "Rogue_DDK_Linux_WS rogueddk 24.2@6643903"
+#define PVRVERSION_STRING_SHORT     "24.2@6643903"
+
+#define COPYRIGHT_TXT               "Copyright (c) Imagination Technologies Ltd. All Rights Reserved."
+
+#define PVRVERSION_BUILD_HI          664
+#define PVRVERSION_BUILD_LO          3903
+#define PVRVERSION_STRING_NUMERIC   "24.2.664.3903"
+
+#define PVRVERSION_PACK(MAJOR,MINOR) (((IMG_UINT32)((IMG_UINT32)(MAJOR) & 0xFFFFU) << 16U) | (((MINOR) & 0xFFFFU) << 0U))
+#define PVRVERSION_UNPACK_MAJ(VERSION) (((VERSION) >> 16U) & 0xFFFFU)
+#define PVRVERSION_UNPACK_MIN(VERSION) (((VERSION) >> 0U) & 0xFFFFU)
+
+#endif /* PVRVERSION_H */
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 0d69098eddd9..9a35ee8d5810 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -3377,6 +3377,31 @@ static const struct panel_desc rocktech_rk043fn48h = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
+static const struct drm_display_mode raspberrypi_7inch_mode = {
+	.clock = 30000,
+	.hdisplay = 800,
+	.hsync_start = 800 + 131,
+	.hsync_end = 800 + 131 + 2,
+	.htotal = 800 + 131 + 2 + 45,
+	.vdisplay = 480,
+	.vsync_start = 480 + 7,
+	.vsync_end = 480 + 7 + 2,
+	.vtotal = 480 + 7 + 2 + 22,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc raspberrypi_7inch = {
+	.modes = &raspberrypi_7inch_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 154,
+		.height = 86,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.connector_type = DRM_MODE_CONNECTOR_DSI,
+};
+
 static const struct display_timing rocktech_rk070er9427_timing = {
 	.pixelclock = { 26400000, 33300000, 46800000 },
 	.hactive = { 800, 800, 800 },
@@ -4452,6 +4477,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "rocktech,rk043fn48h",
 		.data = &rocktech_rk043fn48h,
+	}, {
+		.compatible = "raspberrypi,7inch-dsi",
+		.data = &raspberrypi_7inch,
 	}, {
 		.compatible = "rocktech,rk070er9427",
 		.data = &rocktech_rk070er9427,
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 64606104551c..8392393b7bf5 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -355,6 +355,7 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 	if (ret)
 		return ret;
 
+#if 0
 	if (num_pages == 1 && ttm->caching == ttm_cached &&
 	    !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) {
 		/*
@@ -365,7 +366,9 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 		map->bo_kmap_type = ttm_bo_map_kmap;
 		map->page = ttm->pages[start_page];
 		map->virtual = kmap(map->page);
-	} else {
+	} else
+#endif
+	{
 		/*
 		 * We need to use vmap to get the desired page protection
 		 * or to make the buffer object look contiguous.
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index b3fffe7b5062..aa137ead5cc5 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -74,7 +74,8 @@ pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp)
 #endif /* CONFIG_UML */
 #endif /* __i386__ || __x86_64__ */
 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
-	defined(__powerpc__) || defined(__mips__) || defined(__loongarch__)
+	defined(__powerpc__) || defined(__mips__) || defined(__loongarch__) || \
+	defined(__riscv)
 	if (caching == ttm_write_combined)
 		tmp = pgprot_writecombine(tmp);
 	else
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 46ff9c75bb12..63a9b8d41b94 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -187,7 +187,7 @@ void ttm_resource_init(struct ttm_buffer_object *bo,
 	res->bus.addr = NULL;
 	res->bus.offset = 0;
 	res->bus.is_iomem = false;
-	res->bus.caching = ttm_cached;
+	res->bus.caching = ttm_write_combined;
 	res->bo = bo;
 
 	man = ttm_manager_type(bo->bdev, place->mem_type);
@@ -670,17 +670,18 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 	} else {
 		iter_io->needs_unmap = true;
 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
-		if (mem->bus.caching == ttm_write_combined)
+		if (mem->bus.caching == ttm_write_combined  || mem->bus.caching == ttm_cached)
 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
 						  ioremap_wc(mem->bus.offset,
 							     mem->size));
+#if 0
 		else if (mem->bus.caching == ttm_cached)
 			iosys_map_set_vaddr(&iter_io->dmap,
 					    memremap(mem->bus.offset, mem->size,
 						     MEMREMAP_WB |
 						     MEMREMAP_WT |
 						     MEMREMAP_WC));
-
+#endif
 		/* If uncached requested or if mapping cached or wc failed */
 		if (iosys_map_is_null(&iter_io->dmap))
 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index bf9601351fa3..af3ab03200c0 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -154,7 +154,7 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm,
 	ttm->dma_address = NULL;
 	ttm->swap_storage = NULL;
 	ttm->sg = bo->sg;
-	ttm->caching = caching;
+	ttm->caching = ttm_write_combined;
 }
 
 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo,
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index a4c361b6619c..88735efccb69 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -312,6 +312,15 @@ config SENSORS_AXI_FAN_CONTROL
 	  This driver can also be built as a module. If so, the module
 	  will be called axi-fan-control
 
+config SENSORS_ESWIN_FAN_CONTROL
+	tristate "ESWIN FAN Control Core driver"
+	help
+	  If you say yes here you get support for the Analog Devices
+	  ESWIN FAN monitoring core.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called eswin-fan-control
+
 config SENSORS_K8TEMP
 	tristate "AMD Athlon64/FX or Opteron temperature sensor"
 	depends on X86 && PCI
@@ -559,6 +568,18 @@ config SENSORS_DA9055
 	  This driver can also be built as a module. If so, the module
 	  will be called da9055-hwmon.
 
+config SENSORS_ESWIN_DUMMY_PVT
+		tristate "Eswin Process, Voltage, Temperature sensor driver"
+		depends on HWMON
+		help
+		  If you say yes here you get support for Eswin PVT sensor.
+
+config SENSORS_ESWIN_PVT
+        tristate "Eswin Process, Voltage, Temperature sensor driver"
+        depends on HWMON
+        help
+          If you say yes here you get support for Eswin PVT sensor.
+
 config SENSORS_I5K_AMB
 	tristate "FB-DIMM AMB temperature sensor on Intel 5000 series chipsets"
 	depends on PCI
@@ -2371,6 +2392,12 @@ config SENSORS_INTEL_M10_BMC_HWMON
 	  sensors monitor various telemetry data of different components on the
 	  card, e.g. board temperature, FPGA core temperature/voltage/current.
 
+config SENSORS_PAC1934
+	tristate "microchip PAC193X  Hardware Monitoring"
+	depends on I2C
+	help
+	  If you say yes here you get support for microchip PAC193X
+	  Single/Multi-Channel DC Power/Energy 	Monitor	with Accumulator.
 if ACPI
 
 comment "ACPI drivers"
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 4ac9452b5430..e74a753a77de 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -220,7 +220,10 @@ obj-$(CONFIG_SENSORS_W83L786NG)	+= w83l786ng.o
 obj-$(CONFIG_SENSORS_WM831X)	+= wm831x-hwmon.o
 obj-$(CONFIG_SENSORS_WM8350)	+= wm8350-hwmon.o
 obj-$(CONFIG_SENSORS_XGENE)	+= xgene-hwmon.o
-
+obj-$(CONFIG_SENSORS_PAC1934)	+= pac193x.o
+obj-$(CONFIG_SENSORS_ESWIN_FAN_CONTROL) += eswin-fan-control.o
+obj-$(CONFIG_SENSORS_ESWIN_DUMMY_PVT) += eswin_dummy_pvt.o
+obj-$(CONFIG_SENSORS_ESWIN_PVT) += eswin_pvt.o
 obj-$(CONFIG_SENSORS_OCC)	+= occ/
 obj-$(CONFIG_SENSORS_PECI)	+= peci/
 obj-$(CONFIG_PMBUS)		+= pmbus/
diff --git a/drivers/hwmon/eswin-fan-control.c b/drivers/hwmon/eswin-fan-control.c
new file mode 100644
index 000000000000..9cdaf32f1ea8
--- /dev/null
+++ b/drivers/hwmon/eswin-fan-control.c
@@ -0,0 +1,746 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Fan Control CORE driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Han Min <hanmin@eswincomputing.com>
+ */
+
+
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/wait.h>
+#include <linux/delay.h>
+
+#define FAN_PWM_DUTY			0x0
+#define FAN_PWM_PERIOD			0x1
+#define FAN_PWM_FREE			0x2
+#define DDR_TRAINING_TEMP		0x3
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+#define PMIX_LOWEST_TEMP		0x4
+#define D2D_RECOVERY			0x5
+#endif
+
+#define FAN_RPM_MAX_VALUE			(100000)
+#define FAN_RPM_MAX_READ_CNT		(100)
+#define FAN_RPM_RETRY_INTERVAL		(10)
+#define FAN_RPM_SAMPLE_CNT			(10)//sampling times
+#define FAM_RPM_TOLERANCE_RATIO 	(5)
+
+/* register map */
+#define REG_FAN_INT				0x0
+#define REG_FAN_RPM				0x4
+/* test register map */
+#define SYS_CON_TESTREG0		(0x0) //0x51810668 and 0x71810668
+#define SYS_CON_TESTREG1		(0x4) //0x5181066C and 0x7181066C
+#define SYS_CON_TESTREG2		(0x8) //0x51810670 and 0x71810670
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+#define D2D_RECOVERY_MASK		BIT(31) //d2d recovery bit
+#define PMIX_LOWEST_TEMP_MASK	GENMASK(20, 0) //pmix lowest temperature
+#endif
+#define SYS_CON_TESTREG3		(0xC) //0x51810674 and 0x71810674
+
+
+/* wait for 50 times pwm period to trigger read interrupt */
+#define TIMEOUT(period)        nsecs_to_jiffies(50*(period))
+
+struct eswin_fan_control_data {
+	struct reset_control *fan_rst;
+	struct clk *clk;
+	void __iomem *base;
+	void __iomem *cur_die_test_reg_base;
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+	void __iomem *other_die_test_reg_base;
+#endif
+	struct device *hdev;
+	unsigned long clk_rate;
+	int pwm_id;
+	struct pwm_device *pwm;
+	wait_queue_head_t wq;
+	bool wait_flag;
+	int irq;
+	/* pwm minimum period */
+	u32 min_period;
+	/* pulses per revolution */
+	u32 ppr;
+	/* revolutions per minute */
+	u32 rpm;
+	/* last revolutions per minute */
+	u32 last_rpm;
+	u8 pwm_inverted;
+	/* for getting rpm and setting pwm */
+	struct mutex fan_lock;
+};
+
+static inline void fan_iowrite(const u32 val, const u32 reg,
+			       const struct eswin_fan_control_data *ctl)
+{
+	iowrite32(val, ctl->base + reg);
+}
+
+static inline u32 fan_ioread(const u32 reg,
+			     const struct eswin_fan_control_data *ctl)
+{
+	return ioread32(ctl->base + reg);
+}
+
+static ssize_t eswin_fan_pwm_ctl_show(struct device *dev, struct device_attribute *da, char *buf)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	long temp = 0;
+	long period = 0;
+	if (FAN_PWM_DUTY == attr->index) {
+		temp = pwm_get_duty_cycle(ctl->pwm);
+		if(1 ==  ctl->pwm_inverted)
+		{
+			period = pwm_get_period(ctl->pwm);
+			temp =  period- temp;
+		}
+	}
+	else if (FAN_PWM_PERIOD == attr->index) {
+		temp = pwm_get_period(ctl->pwm);
+	}
+	else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	return sprintf(buf, "%lu\n", temp);
+}
+
+static ssize_t eswin_fan_pwm_ctl_store(struct device *dev, struct device_attribute *da,
+				     const char *buf, size_t count)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	struct pwm_state state;
+	int ret;
+
+	pwm_get_state(ctl->pwm, &state);
+
+	if (FAN_PWM_DUTY == attr->index) {
+		long val = 0;
+		ret = kstrtoul(buf, 10, &val);
+		if (ret)
+			return ret;
+		if(1 ==  ctl->pwm_inverted)
+		{
+			state.duty_cycle =  state.period - val;
+		}
+		else
+		{
+			state.duty_cycle = val;
+		}
+	}
+	else if (FAN_PWM_PERIOD == attr->index) {
+		long val = 0;
+		ret = kstrtoul(buf, 10, &val);
+		if (ret)
+			return ret;
+		if (val >= ctl->min_period)
+			state.period = val;
+		else
+			dev_err(dev, "invalid pwm period!\n");
+	}
+	else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	mutex_lock(&ctl->fan_lock);
+	pwm_apply_might_sleep(ctl->pwm, &state);
+	mutex_unlock(&ctl->fan_lock);
+
+	return count;
+}
+
+static ssize_t eswin_fan_pwm_free_store(struct device *dev, struct device_attribute *da,
+				     const char *buf, size_t count)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	long val;
+	int ret;
+	ret = kstrtoul(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val) {
+		mutex_lock(&ctl->fan_lock);
+		pwm_put(ctl->pwm);
+		mutex_unlock(&ctl->fan_lock);
+	}
+
+	return count;
+}
+
+static ssize_t eswin_ddr_training_temp_show(struct device *dev, struct device_attribute *da, char *buf)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	int temp = 0;
+
+	if (DDR_TRAINING_TEMP == attr->index) {
+		temp = ioread32(ctl->cur_die_test_reg_base + SYS_CON_TESTREG3);
+	} else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	return sprintf(buf, "%d\n", temp);
+}
+
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+static ssize_t eswin_pmix_lowest_temp_show(struct device *dev, struct device_attribute *da, char *buf)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	int temp = 0;
+
+	if (PMIX_LOWEST_TEMP == attr->index) {
+		temp = ioread32(ctl->cur_die_test_reg_base + SYS_CON_TESTREG2) & PMIX_LOWEST_TEMP_MASK;
+	} else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	return sprintf(buf, "%d\n", temp);
+}
+
+static ssize_t eswin_d2d_recovery_show(struct device *dev, struct device_attribute *da, char *buf)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	int recovery = 0;
+	int cur_die_val = 0;
+	int other_die_val = 0;
+
+	if (D2D_RECOVERY == attr->index) {
+		cur_die_val = ioread32(ctl->cur_die_test_reg_base + SYS_CON_TESTREG2) & D2D_RECOVERY_MASK;
+		other_die_val = ioread32(ctl->other_die_test_reg_base + SYS_CON_TESTREG2) & D2D_RECOVERY_MASK;
+		if (cur_die_val || other_die_val) {
+			recovery = 1;
+		}
+	} else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	return sprintf(buf, "%d\n", recovery);
+}
+#endif
+
+static long eswin_fan_control_get_pwm_duty(const struct eswin_fan_control_data *ctl)
+{
+	struct pwm_state state;
+	int duty;
+
+	pwm_get_state(ctl->pwm, &state);
+	duty = pwm_get_relative_duty_cycle(&state, 100);
+
+	return duty;
+}
+
+static long es_fan_rpm_filter_and_average(
+	struct device *dev, u32 samples[], u32 count) {
+	u32 idx = 0;
+	u32 min = 0;
+	u32 sum = 0;
+	u32 cnt = 0;
+
+	if ((samples == NULL) || (count <= 0) || (count > FAN_RPM_SAMPLE_CNT)) {
+		dev_err(dev, "param error with count=%d\n", count);
+		return 0;
+	}
+
+	min = samples[0];
+	for (idx = 1; idx < count; idx++) {
+		if (samples[idx] < min) {
+			min = samples[idx];
+		}
+	}
+	for (idx = 0; idx < count; idx++) {
+		if ((samples[idx] - min) <= (min * FAM_RPM_TOLERANCE_RATIO / 100)) {
+			sum += samples[idx];
+			cnt++;
+		}
+	}
+
+	return cnt ? (sum / cnt) : 0;
+}
+
+static long eswin_fan_control_get_fan_rpm(struct eswin_fan_control_data *ctl)
+{
+	unsigned int val;
+	long period, timeout;
+	int ret;
+
+	ctl->wait_flag = false;
+	period = pwm_get_period(ctl->pwm);
+	timeout = msecs_to_jiffies(1500);
+
+	val = fan_ioread(REG_FAN_INT, ctl);
+	val = val | 0x1;
+	fan_iowrite(val, REG_FAN_INT, ctl);
+
+	/* wair read interrupt */
+	ret = wait_event_interruptible_timeout(ctl->wq,
+                                        ctl->wait_flag,
+                                        timeout);
+
+	if (!ret){
+		/* timeout, set rpm to 0 */
+		ctl->rpm = 0;
+	}
+	if(ctl->rpm)
+		ctl->rpm = DIV_ROUND_CLOSEST(60 * ctl->clk_rate, ctl->ppr * ctl->rpm);
+
+	return ret;
+}
+
+static int eswin_fan_control_read_fan(struct device *dev, u32 attr, long *val)
+{
+	long ret = 0;
+	int retry = 0;
+	u32 samples[FAN_RPM_SAMPLE_CNT] = {0};
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+
+	switch (attr) {
+	case hwmon_fan_input:
+		mutex_lock(&ctl->fan_lock);
+		for (int32_t idx = 0; idx < FAN_RPM_SAMPLE_CNT; idx++) {
+			retry = 0;
+			while (retry < FAN_RPM_MAX_READ_CNT) {
+				ret = eswin_fan_control_get_fan_rpm(ctl);
+				if (ret == 0) {
+					/* timeout case */
+					*val = 0;
+					mutex_unlock(&ctl->fan_lock);
+					return 0;
+				} else if (ret < 0) {
+					if (ret == -ERESTARTSYS) {
+						/* cancel case */
+						mutex_unlock(&ctl->fan_lock);
+						return -EINTR;
+					}
+					dev_err(dev, "wait read interrupt fail, ret=%ld\n", ret);
+					retry++;
+					continue;
+				}
+				if (ctl->rpm > FAN_RPM_MAX_VALUE) {
+					msleep(FAN_RPM_RETRY_INTERVAL);
+					retry++;
+					continue;
+				} else {
+					break;
+				}
+			}
+			if (retry == FAN_RPM_MAX_READ_CNT) {
+				samples[idx] = ctl->last_rpm;
+			} else {
+				samples[idx] = ctl->rpm;
+			}
+			msleep(FAN_RPM_RETRY_INTERVAL);
+		}
+		*val = es_fan_rpm_filter_and_average(dev, samples, FAN_RPM_SAMPLE_CNT);
+		ctl->last_rpm = *val;
+		mutex_unlock(&ctl->fan_lock);
+		return 0;
+
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_read_pwm(struct device *dev, u32 attr, long *val)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+
+	switch (attr) {
+	case hwmon_pwm_input:
+		*val = eswin_fan_control_get_pwm_duty(ctl);
+		if(1 ==  ctl->pwm_inverted)
+		{
+			*val = 100 - *val;
+		}
+		return 0;
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_set_pwm_duty(const long val, struct eswin_fan_control_data *ctl)
+{
+	struct pwm_state state;
+
+	mutex_lock(&ctl->fan_lock);
+	pwm_get_state(ctl->pwm, &state);
+	pwm_set_relative_duty_cycle(&state, val, 100);
+	pwm_apply_might_sleep(ctl->pwm, &state);
+	mutex_unlock(&ctl->fan_lock);
+
+	return 0;
+}
+
+static int eswin_fan_control_write_pwm(struct device *dev, u32 attr, long val)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	switch (attr) {
+		case hwmon_pwm_input:
+	if((val < 10) || (val > 99))
+	{
+		dev_err(dev,"pwm range is form 10 to 99\n");
+		return -EINVAL;
+	}
+	else
+	{
+		if(1 ==  ctl->pwm_inverted)
+		{
+			val = 100 - val;
+		}
+		return eswin_fan_control_set_pwm_duty(val, ctl);
+	}
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int eswin_fan_control_read_labels(struct device *dev,
+					enum hwmon_sensor_types type,
+					u32 attr, int channel, const char **str)
+{
+	switch (type) {
+	case hwmon_fan:
+		*str = "FAN";
+		return 0;
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_read(struct device *dev,
+				enum hwmon_sensor_types type,
+				u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_fan:
+		return eswin_fan_control_read_fan(dev, attr, val);
+	case hwmon_pwm:
+		return eswin_fan_control_read_pwm(dev, attr, val);
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_write(struct device *dev,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int channel, long val)
+{
+	switch (type) {
+	case hwmon_pwm:
+		return eswin_fan_control_write_pwm(dev, attr, val);
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static umode_t eswin_fan_control_fan_is_visible(const u32 attr)
+{
+	switch (attr) {
+	case hwmon_fan_input:
+	case hwmon_fan_label:
+		return 0444;
+	default:
+		return 0;
+	}
+}
+
+static umode_t eswin_fan_control_pwm_is_visible(const u32 attr)
+{
+	switch (attr) {
+	case hwmon_pwm_input:
+		return 0644;
+	default:
+		return 0;
+	}
+}
+
+static umode_t eswin_fan_control_is_visible(const void *data,
+					enum hwmon_sensor_types type,
+					u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_fan:
+		return eswin_fan_control_fan_is_visible(attr);
+	case hwmon_pwm:
+		return eswin_fan_control_pwm_is_visible(attr);
+	default:
+		return 0;
+	}
+}
+
+static irqreturn_t eswin_fan_control_irq_handler(int irq, void *data)
+{
+	struct eswin_fan_control_data *ctl = (struct eswin_fan_control_data *)data;
+	u32 status = 0;
+
+	status = fan_ioread(REG_FAN_INT, ctl);
+	if (0x3 == (status & 0x3)){
+		ctl->rpm = fan_ioread(REG_FAN_RPM, ctl);
+
+		/* clear interrupt */
+		fan_iowrite(0x5, REG_FAN_INT, ctl);
+
+		/* When the fan does not support obtaining speed, bit3 will be set to 1 */
+		if (0x0 == (status & (0x1 << 3))) {
+			/* wake up fan_rpm read */
+			ctl->wait_flag = true;
+			wake_up_interruptible(&ctl->wq);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int eswin_fan_control_init(struct eswin_fan_control_data *ctl,
+				const struct device_node *np)
+{
+	int ret;
+	/* get fan pulses per revolution */
+	ret = of_property_read_u32(np, "pulses-per-revolution", &ctl->ppr);
+	if (ret)
+		return ret;
+
+	/* 1, 2 and 4 are the typical and accepted values */
+	if (ctl->ppr != 1 && ctl->ppr != 2 && ctl->ppr != 4)
+		return -EINVAL;
+
+	/* get pwm minimum period */
+	ret = of_property_read_u32(np, "pwm-minimum-period", &ctl->min_period);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static void eswin_fan_control_remove(void *data)
+{
+	int ret;
+	struct eswin_fan_control_data *ctl = data;
+	pwm_put(ctl->pwm);
+	ret = reset_control_assert(ctl->fan_rst);
+	WARN_ON(0 != ret);
+	clk_disable_unprepare(ctl->clk);
+}
+
+static const struct hwmon_channel_info *eswin_fan_control_info[] = {
+	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT),
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_LABEL),
+	NULL
+};
+
+static const struct hwmon_ops eswin_fan_control_hwmon_ops = {
+	.is_visible = eswin_fan_control_is_visible,
+	.read = eswin_fan_control_read,
+	.write = eswin_fan_control_write,
+	.read_string = eswin_fan_control_read_labels,
+};
+
+static const struct hwmon_chip_info eswin_chip_info = {
+	.ops = &eswin_fan_control_hwmon_ops,
+	.info = eswin_fan_control_info,
+};
+
+static SENSOR_DEVICE_ATTR_RW(fan_pwm_duty, eswin_fan_pwm_ctl, FAN_PWM_DUTY);
+static SENSOR_DEVICE_ATTR_RW(fan_pwm_period, eswin_fan_pwm_ctl, FAN_PWM_PERIOD);
+static SENSOR_DEVICE_ATTR_WO(fan_pwm_free, eswin_fan_pwm_free, FAN_PWM_FREE);
+static SENSOR_DEVICE_ATTR_RO(ddr_training_temp, eswin_ddr_training_temp, DDR_TRAINING_TEMP);
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+static SENSOR_DEVICE_ATTR_RO(pmix_lowest_temp, eswin_pmix_lowest_temp, PMIX_LOWEST_TEMP);
+static SENSOR_DEVICE_ATTR_RO(d2d_recovery, eswin_d2d_recovery, D2D_RECOVERY);
+#endif
+
+static struct attribute *eswin_fan_control_attrs[] = {
+	&sensor_dev_attr_fan_pwm_duty.dev_attr.attr,
+	&sensor_dev_attr_fan_pwm_period.dev_attr.attr,
+	&sensor_dev_attr_fan_pwm_free.dev_attr.attr,
+	&sensor_dev_attr_ddr_training_temp.dev_attr.attr,
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+	&sensor_dev_attr_pmix_lowest_temp.dev_attr.attr,
+	&sensor_dev_attr_d2d_recovery.dev_attr.attr,
+#endif
+	NULL,
+};
+ATTRIBUTE_GROUPS(eswin_fan_control);
+
+static const struct of_device_id eswin_fan_control_of_match[] = {
+	{ .compatible = "eswin-fan-control"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, eswin_fan_control_of_match);
+
+static int eswin_fan_control_probe(struct platform_device *pdev)
+{
+	struct eswin_fan_control_data *ctl;
+	const struct of_device_id *id;
+	const char *name = "eswin_fan_control";
+	struct pwm_state state;
+	struct pwm_args pwm_args;
+	struct resource *res;
+	int ret;
+
+	id = of_match_node(eswin_fan_control_of_match, pdev->dev.of_node);
+	if (!id)
+		return -EINVAL;
+
+	ctl = devm_kzalloc(&pdev->dev, sizeof(*ctl), GFP_KERNEL);
+	if (!ctl)
+		return -ENOMEM;
+
+	ctl->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ctl->base))
+		return PTR_ERR(ctl->base);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (!res)
+		return -ENODEV;
+	ctl->cur_die_test_reg_base = ioremap(res->start, res->end - res->start + 1);
+	if (IS_ERR_OR_NULL(ctl->cur_die_test_reg_base)) {
+		return PTR_ERR(ctl->cur_die_test_reg_base);
+	}
+#ifdef CONFIG_ARCH_ESWIN_EIC7702_SOC
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+	if (!res)
+		return -ENODEV;
+	ctl->other_die_test_reg_base = ioremap(res->start, res->end - res->start + 1);
+	if (IS_ERR_OR_NULL(ctl->other_die_test_reg_base)) {
+		return PTR_ERR(ctl->other_die_test_reg_base);
+	}
+#endif
+
+	ctl->clk = devm_clk_get(&pdev->dev, "pclk");
+	if (IS_ERR(ctl->clk)) {
+		dev_err(&pdev->dev, "Couldn't get the clock for fan-controller\n");
+		return -ENODEV;
+	}
+
+	ret = clk_prepare_enable(ctl->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to enable clock for fan-controller\n");
+		return ret;
+	}
+
+	ctl->clk_rate = clk_get_rate(ctl->clk);
+	if (!ctl->clk_rate)
+		return -EINVAL;
+
+	ctl->fan_rst = devm_reset_control_get_optional(&pdev->dev, "fan_rst");
+	if (IS_ERR_OR_NULL(ctl->fan_rst)) {
+		dev_err(&pdev->dev, "Failed to get fan_rst reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(ctl->fan_rst);
+	WARN_ON(0 != ret);
+	ctl->pwm_inverted = of_property_read_bool(pdev->dev.of_node, "eswin,pwm_inverted");
+	init_waitqueue_head(&ctl->wq);
+
+	ctl->irq = platform_get_irq(pdev, 0);
+	if (ctl->irq < 0)
+		return ctl->irq;
+
+	ret = devm_request_threaded_irq(&pdev->dev, ctl->irq,
+					eswin_fan_control_irq_handler, NULL,
+					IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+					pdev->driver_override, ctl);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request an irq, %d", ret);
+		return ret;
+	}
+
+	ret = eswin_fan_control_init(ctl, pdev->dev.of_node);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to initialize device\n");
+		return ret;
+	}
+	ctl->pwm = pwm_get(&pdev->dev, NULL);
+	if (IS_ERR(ctl->pwm)) {
+		ret = PTR_ERR(ctl->pwm);
+		dev_err(&pdev->dev, "Failed to request pwm device: %d\n", ret);
+		return ret;
+	}
+
+	pwm_get_state(ctl->pwm, &state);
+
+	/* Then fill it with the reference config */
+	pwm_get_args(ctl->pwm, &pwm_args);
+
+	if (0 == ctl->pwm_inverted)
+	{
+		state.period = pwm_args.period;
+		state.duty_cycle = state.period * 50 / 100; /* default set medium speed */
+	}
+	else
+	{
+		state.period = pwm_args.period;
+		state.duty_cycle = state.period / 100; /* default set medium speed */
+		if(0 == state.duty_cycle)
+		{
+			state.duty_cycle = 50;
+		}
+	}
+	dev_err(&pdev->dev, "state.period: %d state.duty_cycle: %d\n",
+			state.period,state.duty_cycle);
+	ret = pwm_apply_might_sleep(ctl->pwm, &state);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to apply initial PWM state: %d\n",
+			ret);
+	}
+	pwm_enable(ctl->pwm);
+
+	ret = devm_add_action_or_reset(&pdev->dev, eswin_fan_control_remove, ctl);
+	if (ret)
+		return ret;
+
+	ctl->hdev = devm_hwmon_device_register_with_info(&pdev->dev,
+							 name,
+							 ctl,
+							 &eswin_chip_info,
+							 eswin_fan_control_groups);
+	mutex_init(&ctl->fan_lock);
+	dev_err(&pdev->dev, "eswin fan control init exit\n");
+
+	return PTR_ERR_OR_ZERO(ctl->hdev);
+}
+
+static struct platform_driver eswin_fan_control_driver = {
+	.driver = {
+		.name = "eswin_fan_control_driver",
+		.of_match_table = eswin_fan_control_of_match,
+	},
+	.probe = eswin_fan_control_probe,
+};
+module_platform_driver(eswin_fan_control_driver);
+
+MODULE_AUTHOR("Han Min <hanmin@eswincomputing.com>");
+MODULE_DESCRIPTION("ESWIN Fan Control CORE driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/eswin_dummy_pvt.c b/drivers/hwmon/eswin_dummy_pvt.c
new file mode 100644
index 000000000000..711f28481427
--- /dev/null
+++ b/drivers/hwmon/eswin_dummy_pvt.c
@@ -0,0 +1,732 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN PVT device driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ *          Huan He <hehuan1@eswincomputing.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/hwmon.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/ktime.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/seqlock.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include "eswin_pvt.h"
+#include <linux/regmap.h>
+#include  <linux/mfd/syscon.h>
+#include <linux/of.h>
+
+
+/*
+ * For the sake of the code simplification we created the sensors info table
+ * with the sensor names, activation modes, threshold registers base address
+ * and the thresholds bit fields.
+ */
+static const struct pvt_sensor_info pvt_info_cpu[] = {
+	PVT_SENSOR_INFO(0, "SoC Temperature", hwmon_temp, TEMP, TTHRES),
+	PVT_SENSOR_INFO(0, "SoC Voltage", hwmon_in, VOLT, VTHRES),
+	PVT_SENSOR_INFO(1, "SoC Low-Vt", hwmon_in, LVT, LTHRES),
+	PVT_SENSOR_INFO(2, "SoC UltraLow-Vt", hwmon_in, ULVT, ULTHRES),
+	PVT_SENSOR_INFO(3, "SoC Standard-Vt", hwmon_in, SVT, STHRES),
+};
+
+static const struct pvt_sensor_info pvt_info_ddr[] = {
+	PVT_SENSOR_INFO(0, "DDR Core Temperature", hwmon_temp, TEMP, TTHRES),
+	PVT_SENSOR_INFO(0, "DDR Core Voltage", hwmon_in, VOLT, VTHRES),
+	PVT_SENSOR_INFO(1, "DDR Core Low-Vt", hwmon_in, LVT, LTHRES),
+	PVT_SENSOR_INFO(2, "DDR Core UltraLow-Vt", hwmon_in, ULVT, ULTHRES),
+	PVT_SENSOR_INFO(3, "DDR Core Standard-Vt", hwmon_in, SVT, STHRES),
+};
+
+/*
+ * The original translation formulae of the temperature (in degrees of Celsius)
+ * to PVT data and vice-versa are following:
+ * N = 6.0818e-8*(T^4) +1.2873e-5*(T^3) + 7.2244e-3*(T^2) + 3.6484*(T^1) +
+ *     1.6198e2,
+ * T = -1.8439e-11*(N^4) + 8.0705e-8*(N^3) + -1.8501e-4*(N^2) +
+ *     3.2843e-1*(N^1) - 4.8690e1,
+ * where T = [-40, 125]C and N = [27, 771].
+ * They must be accordingly altered to be suitable for the integer arithmetics.
+ * The technique is called 'factor redistribution', which just makes sure the
+ * multiplications and divisions are made so to have a result of the operations
+ * within the integer numbers limit. In addition we need to translate the
+ * formulae to accept millidegrees of Celsius. Here what they look like after
+ * the alterations:
+ * N = (60818e-20*(T^4) + 12873e-14*(T^3) + 72244e-9*(T^2) + 36484e-3*T +
+ *     16198e2) / 1e4,
+ * T = -18439e-12*(N^4) + 80705e-9*(N^3) - 185010e-6*(N^2) + 328430e-3*N -
+ *     48690,
+ * where T = [-40000, 125000] mC and N = [27, 771].
+ */
+static const struct pvt_poly __maybe_unused poly_temp_to_N = {
+	.total_divider = 10000,
+	.terms = {
+		{4, 60818, 10000, 10000},
+		{3, 12873, 10000, 100},
+		{2, 72244, 10000, 10},
+		{1, 36484, 1000, 1},
+		{0, 1619800, 1, 1}
+	}
+};
+
+static const struct pvt_poly poly_N_to_temp = {
+	.total_divider = 1,
+	.terms = {
+		{4, -18439, 1000, 1},
+		{3, 80705, 1000, 1},
+		{2, -185010, 1000, 1},
+		{1, 328430, 1000, 1},
+		{0, -48690, 1, 1}
+	}
+};
+
+/*
+ * Similar alterations are performed for the voltage conversion equations.
+ * The original formulae are:
+ * N = 1.3905e3*V - 5.7685e2,
+ * V = (N + 5.7685e2) / 1.3905e3,
+ * where V = [0.72, 0.88] V and N = [424, 646].
+ * After the optimization they looks as follows:
+ * N = (13905e-3*V - 5768.5) / 10,
+ * V = (N * 10^5 / 13905 + 57685 * 10^3 / 13905) / 10.
+ * where V = [720, 880] mV and N = [424, 646].
+ */
+static const struct pvt_poly __maybe_unused poly_volt_to_N = {
+	.total_divider = 10,
+	.terms = {
+		{1, 13905, 1000, 1},
+		{0, -57685, 1, 10}
+	}
+};
+
+static const struct pvt_poly poly_N_to_volt = {
+	.total_divider = 10,
+	.terms = {
+		{1, 100000, 13905, 1},
+		{0, 57685000, 1, 13905}
+	}
+};
+
+/*
+ * Here is the polynomial calculation function, which performs the
+ * redistributed terms calculations. It's pretty straightforward. We walk
+ * over each degree term up to the free one, and perform the redistributed
+ * multiplication of the term coefficient, its divider (as for the rationale
+ * fraction representation), data power and the rational fraction divider
+ * leftover. Then all of this is collected in a total sum variable, which
+ * value is normalized by the total divider before being returned.
+ */
+static long eswin_pvt_calc_poly(const struct pvt_poly *poly, long data)
+{
+	const struct pvt_poly_term *term = poly->terms;
+	long tmp, ret = 0;
+	int deg;
+	do {
+		tmp = term->coef;
+		for (deg = 0; deg < term->deg; ++deg)
+			tmp = mult_frac(tmp, data, term->divider);
+		ret += tmp / term->divider_leftover;
+	} while ((term++)->deg);
+
+	return ret / poly->total_divider;
+}
+
+static inline u32 eswin_pvt_update(void __iomem *reg, u32 mask, u32 data)
+{
+	u32 old;
+
+	old = readl_relaxed(reg);
+	writel((old & ~mask) | (data & mask), reg);
+
+	return old & mask;
+}
+
+static inline void eswin_pvt_set_mode(struct pvt_hwmon *pvt, u32 mode)
+{
+	u32 old;
+
+	mode = FIELD_PREP(PVT_MODE_MASK, mode);
+
+	old = eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_MODE, PVT_MODE_MASK, mode);
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static inline u32 eswin_pvt_calc_trim(long temp)
+{
+	temp = clamp_val(temp, 0, PVT_TRIM_TEMP);
+
+	return DIV_ROUND_UP(temp, PVT_TRIM_STEP);
+}
+
+static inline void eswin_pvt_set_trim(struct pvt_hwmon *pvt, u32 val)
+{
+	u32 old;
+
+	old = eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	writel(val, pvt->regs + PVT_TRIM);
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+#define pvt_soft_isr NULL
+
+static inline umode_t eswin_pvt_limit_is_visible(enum pvt_sensor_type type)
+{
+	return 0;
+}
+
+static inline umode_t eswin_pvt_pvt_alarm_is_visible(enum pvt_sensor_type type)
+{
+	return 0;
+}
+
+static int eswin_pvt_read_data(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			 long *val)
+{
+	u32 data;
+	u32 offset;
+	const struct pvt_sensor_info *pvt_info;
+
+	pvt_info = of_device_get_match_data(pvt->dev);
+	if (!pvt_info) {
+		dev_err(pvt->dev, "No matching device data found\n");
+		return -EINVAL;
+	}
+
+	pvt->sensor = type;
+
+	offset = pvt->regmap_offset;
+
+	switch (type) {
+		case PVT_TEMP:
+			offset += LPCPU_PVT_TEMP_REG_OFFSET;
+			break;
+		case PVT_VOLT:
+			offset += LPCPU_PVT_VOLT_REG_OFFSET;
+			break;
+		default:
+			return -EINVAL;
+	}
+
+	regmap_read(pvt->regmap, offset, &data);
+
+	if (type == PVT_TEMP)
+		*val = eswin_pvt_calc_poly(&poly_N_to_temp, data);
+
+	else if (type == PVT_VOLT)
+		*val = eswin_pvt_calc_poly(&poly_N_to_volt, data);
+	else
+		*val = data;
+
+	return 0;
+}
+
+static int eswin_pvt_read_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			  bool is_low, long *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_write_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			   bool is_low, long val)
+{
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			  bool is_low, long *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_channel_info *pvt_channel_info[] = {
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_TYPE | HWMON_T_LABEL |
+			   HWMON_T_OFFSET),
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL),
+	NULL
+};
+
+static inline bool eswin_pvt_hwmon_channel_is_valid(enum hwmon_sensor_types type,
+					      int ch)
+{
+	switch (type) {
+	case hwmon_temp:
+		if (ch < 0 || ch >= PVT_TEMP_CHS)
+			return false;
+		break;
+	case hwmon_in:
+		if (ch < 0 || ch >= PVT_VOLT_CHS)
+			return false;
+		break;
+	default:
+		break;
+	}
+
+	/* The rest of the types are independent from the channel number. */
+	return true;
+}
+
+static umode_t eswin_pvt_hwmon_is_visible(const void *data,
+				    enum hwmon_sensor_types type,
+				    u32 attr, int ch)
+{
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return 0;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return 0644;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+		case hwmon_temp_type:
+		case hwmon_temp_label:
+			return 0444;
+		case hwmon_temp_min:
+		case hwmon_temp_max:
+			return eswin_pvt_limit_is_visible(ch);
+		case hwmon_temp_min_alarm:
+		case hwmon_temp_max_alarm:
+			return eswin_pvt_pvt_alarm_is_visible(ch);
+		case hwmon_temp_offset:
+			return 0644;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+		case hwmon_in_label:
+			return 0444;
+		case hwmon_in_min:
+		case hwmon_in_max:
+			return eswin_pvt_limit_is_visible(PVT_VOLT + ch);
+		case hwmon_in_min_alarm:
+		case hwmon_in_max_alarm:
+			return eswin_pvt_pvt_alarm_is_visible(PVT_VOLT + ch);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_read_trim(struct pvt_hwmon *pvt, long *val)
+{
+	u32 data;
+
+	data = readl(pvt->regs + PVT_TRIM);
+	/* *val = FIELD_GET(PVT_CTRL_TRIM_MASK, data) * PVT_TRIM_STEP; */
+	*val = data;
+
+	return 0;
+}
+
+static int eswin_pvt_write_trim(struct pvt_hwmon *pvt, long val)
+{
+	int ret;
+	/*
+	 * Serialize trim update, since a part of the register is changed and
+	 * the controller is supposed to be disabled during this operation.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	/* trim = eswin_pvt_calc_trim(val); */
+	eswin_pvt_set_trim(pvt, val);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_read_timeout(struct pvt_hwmon *pvt, long *val)
+{
+	int ret;
+
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	/* Return the result in msec as hwmon sysfs interface requires. */
+	*val = ktime_to_ms(pvt->timeout);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_write_timeout(struct pvt_hwmon *pvt, long val)
+{
+	unsigned long rate;
+	ktime_t kt, cache;
+	u32 data;
+	int ret;
+
+	rate = clk_get_rate(pvt->clk);
+	if (!rate)
+		return -ENODEV;
+
+	/*
+	 * If alarms are enabled, the requested timeout must be divided
+	 * between all available sensors to have the requested delay
+	 * applicable to each individual sensor.
+	 */
+	cache = kt = ms_to_ktime(val);
+
+	/*
+	 * Subtract a constant lag, which always persists due to the limited
+	 * PVT sampling rate. Make sure the timeout is not negative.
+	 */
+	kt = ktime_sub_ns(kt, PVT_TOUT_MIN);
+	if (ktime_to_ns(kt) < 0)
+		kt = ktime_set(0, 0);
+
+	/*
+	 * Finally recalculate the timeout in terms of the reference clock
+	 * period.
+	 */
+	data = ktime_divns(kt * rate, NSEC_PER_SEC);
+
+	/*
+	 * Update the measurements delay, but lock the interface first, since
+	 * we have to disable PVT in order to have the new delay actually
+	 * updated.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	pvt->timeout = cache;
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int ch, long *val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return eswin_pvt_read_timeout(pvt, val);
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			return eswin_pvt_read_data(pvt, ch, val);
+		case hwmon_temp_type:
+			*val = 1;
+			return 0;
+		case hwmon_temp_min:
+			return eswin_pvt_read_limit(pvt, ch, true, val);
+		case hwmon_temp_max:
+			return eswin_pvt_read_limit(pvt, ch, false, val);
+		case hwmon_temp_min_alarm:
+			return eswin_pvt_read_alarm(pvt, ch, true, val);
+		case hwmon_temp_max_alarm:
+			return eswin_pvt_read_alarm(pvt, ch, false, val);
+		case hwmon_temp_offset:
+			return eswin_pvt_read_trim(pvt, val);
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+			return eswin_pvt_read_data(pvt, PVT_VOLT + ch, val);
+		case hwmon_in_min:
+			return eswin_pvt_read_limit(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max:
+			return eswin_pvt_read_limit(pvt, PVT_VOLT + ch, false, val);
+		case hwmon_in_min_alarm:
+			return eswin_pvt_read_alarm(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max_alarm:
+			return eswin_pvt_read_alarm(pvt, PVT_VOLT + ch, false, val);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_hwmon_read_string(struct device *dev,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int ch, const char **str)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	const struct pvt_sensor_info *pvt_info;
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	pvt_info = of_device_get_match_data(pvt->dev);
+	if (!pvt_info) {
+		dev_err(pvt->dev, "No matching device data found\n");
+		return -EINVAL;
+	}
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_label:
+			*str = pvt_info[ch].label;
+			return 0;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_label:
+			*str = pvt_info[PVT_VOLT + ch].label;
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int ch, long val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return eswin_pvt_write_timeout(pvt, val);
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_min:
+			return eswin_pvt_write_limit(pvt, ch, true, val);
+		case hwmon_temp_max:
+			return eswin_pvt_write_limit(pvt, ch, false, val);
+		case hwmon_temp_offset:
+			return eswin_pvt_write_trim(pvt, val);
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_min:
+			return eswin_pvt_write_limit(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max:
+			return eswin_pvt_write_limit(pvt, PVT_VOLT + ch, false, val);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_ops pvt_hwmon_ops = {
+	.is_visible = eswin_pvt_hwmon_is_visible,
+	.read = eswin_pvt_hwmon_read,
+	.read_string = eswin_pvt_hwmon_read_string,
+	.write = eswin_pvt_hwmon_write
+};
+
+static const struct hwmon_chip_info pvt_hwmon_info = {
+	.ops = &pvt_hwmon_ops,
+	.info = pvt_channel_info
+};
+
+static void pvt_clear_data(void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	int idx;
+
+	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
+		complete_all(&pvt->cache[idx].conversion);
+
+	mutex_destroy(&pvt->iface_mtx);
+}
+
+static struct pvt_hwmon *eswin_pvt_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pvt_hwmon *pvt;
+	int ret, idx;
+
+	pvt = devm_kzalloc(dev, sizeof(*pvt), GFP_KERNEL);
+	if (!pvt)
+		return ERR_PTR(-ENOMEM);
+
+	ret = devm_add_action(dev, pvt_clear_data, pvt);
+	if (ret) {
+		dev_err(dev, "Can't add PVT data clear action\n");
+		return ERR_PTR(ret);
+	}
+
+	pvt->dev = dev;
+	pvt->sensor = PVT_SENSOR_FIRST;
+	mutex_init(&pvt->iface_mtx);
+
+	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
+		init_completion(&pvt->cache[idx].conversion);
+
+	return pvt;
+}
+
+static int eswin_pvt_request_regs(struct pvt_hwmon *pvt)
+{
+	struct platform_device *pdev = to_platform_device(pvt->dev);
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(pvt->dev, "Couldn't find PVT memresource\n");
+		return -EINVAL;
+	}
+
+	pvt->regs = devm_ioremap_resource(pvt->dev, res);
+	if (IS_ERR(pvt->regs))
+		return PTR_ERR(pvt->regs);
+
+	return 0;
+}
+
+static void eswin_pvt_remove(void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	if (!IS_ERR_OR_NULL(pvt->regmap)) {
+		regmap_exit(pvt->regmap);
+		pvt->regmap = NULL;
+	}
+}
+
+
+static int eswin_pvt_create_hwmon(struct pvt_hwmon *pvt)
+{
+	pvt->hwmon = devm_hwmon_device_register_with_info(pvt->dev, "dummy_pvt", pvt,
+		&pvt_hwmon_info, NULL);
+	if (IS_ERR(pvt->hwmon)) {
+		dev_err(pvt->dev, "Couldn't create hwmon device\n");
+		return PTR_ERR(pvt->hwmon);
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_probe(struct platform_device *pdev)
+{
+	struct pvt_hwmon *pvt;
+	int ret;
+
+	pvt = eswin_pvt_create_data(pdev);
+	if (IS_ERR(pvt))
+		return PTR_ERR(pvt);
+
+	pvt->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,syscon");
+	if (IS_ERR(pvt->regmap)) {
+		dev_err(&pdev->dev, "Failed to get syscon regmap\n");
+		return PTR_ERR(pvt->regmap);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscon", 1, &pvt->regmap_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get pvt reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	ret = eswin_pvt_request_regs(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_create_hwmon(pvt);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(pvt->dev, eswin_pvt_remove, pvt);
+	if (ret) {
+		dev_err(pvt->dev, "Can't add PVT clocks disable action\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id pvt_of_match[] = {
+	{ .compatible = "eswin,eswin-dummy-pvt-cpu",
+	 .data = &pvt_info_cpu},
+	{ .compatible = "eswin,eswin-dummy-pvt-ddr",
+	 .data = &pvt_info_ddr},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, pvt_of_match);
+
+static struct platform_driver pvt_driver = {
+	.probe = eswin_pvt_probe,
+	.driver = {
+		.name = "eswin-dummy-pvt",
+		.of_match_table = pvt_of_match
+	},
+};
+module_platform_driver(pvt_driver);
+
+MODULE_DESCRIPTION("Eswin PVT driver");
+MODULE_AUTHOR("Yulin Lu <luyulin@eswincomputing.com>");
+MODULE_AUTHOR("Huan He <hehuan1@eswincomputing.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/hwmon/eswin_pvt.c b/drivers/hwmon/eswin_pvt.c
new file mode 100644
index 000000000000..f4842c96bbb2
--- /dev/null
+++ b/drivers/hwmon/eswin_pvt.c
@@ -0,0 +1,930 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN PVT device driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/hwmon.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/ktime.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/seqlock.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include "eswin_pvt.h"
+
+
+/*
+ * For the sake of the code simplification we created the sensors info table
+ * with the sensor names, activation modes, threshold registers base address
+ * and the thresholds bit fields.
+ */
+static const struct pvt_sensor_info pvt_info_cpu[] = {
+	PVT_SENSOR_INFO(0, "SoC Temperature", hwmon_temp, TEMP, TTHRES),
+	PVT_SENSOR_INFO(0, "SoC Voltage", hwmon_in, VOLT, VTHRES),
+	PVT_SENSOR_INFO(1, "SoC Low-Vt", hwmon_in, LVT, LTHRES),
+	PVT_SENSOR_INFO(2, "SoC UltraLow-Vt", hwmon_in, ULVT, ULTHRES),
+	PVT_SENSOR_INFO(3, "SoC Standard-Vt", hwmon_in, SVT, STHRES),
+};
+
+static const struct pvt_sensor_info pvt_info_ddr[] = {
+	PVT_SENSOR_INFO(0, "DDR Core Temperature", hwmon_temp, TEMP, TTHRES),
+	PVT_SENSOR_INFO(0, "DDR Core Voltage", hwmon_in, VOLT, VTHRES),
+	PVT_SENSOR_INFO(1, "DDR Core Low-Vt", hwmon_in, LVT, LTHRES),
+	PVT_SENSOR_INFO(2, "DDR Core UltraLow-Vt", hwmon_in, ULVT, ULTHRES),
+	PVT_SENSOR_INFO(3, "DDR Core Standard-Vt", hwmon_in, SVT, STHRES),
+};
+
+/*
+ * The original translation formulae of the temperature (in degrees of Celsius)
+ * to PVT data and vice-versa are following:
+ * N = 6.0818e-8*(T^4) +1.2873e-5*(T^3) + 7.2244e-3*(T^2) + 3.6484*(T^1) +
+ *     1.6198e2,
+ * T = -1.8439e-11*(N^4) + 8.0705e-8*(N^3) + -1.8501e-4*(N^2) +
+ *     3.2843e-1*(N^1) - 4.8690e1,
+ * where T = [-40, 125]C and N = [27, 771].
+ * They must be accordingly altered to be suitable for the integer arithmetics.
+ * The technique is called 'factor redistribution', which just makes sure the
+ * multiplications and divisions are made so to have a result of the operations
+ * within the integer numbers limit. In addition we need to translate the
+ * formulae to accept millidegrees of Celsius. Here what they look like after
+ * the alterations:
+ * N = (60818e-20*(T^4) + 12873e-14*(T^3) + 72244e-9*(T^2) + 36484e-3*T +
+ *     16198e2) / 1e4,
+ * T = -18439e-12*(N^4) + 80705e-9*(N^3) - 185010e-6*(N^2) + 328430e-3*N -
+ *     48690,
+ * where T = [-40000, 125000] mC and N = [27, 771].
+ */
+static const struct pvt_poly __maybe_unused poly_temp_to_N = {
+	.total_divider = 10000,
+	.terms = {
+		{4, 60818, 10000, 10000},
+		{3, 12873, 10000, 100},
+		{2, 72244, 10000, 10},
+		{1, 36484, 1000, 1},
+		{0, 1619800, 1, 1}
+	}
+};
+
+static const struct pvt_poly poly_N_to_temp = {
+	.total_divider = 1,
+	.terms = {
+		{4, -18439, 1000, 1},
+		{3, 80705, 1000, 1},
+		{2, -185010, 1000, 1},
+		{1, 328430, 1000, 1},
+		{0, -48690, 1, 1}
+	}
+};
+
+/*
+ * Similar alterations are performed for the voltage conversion equations.
+ * The original formulae are:
+ * N = 1.3905e3*V - 5.7685e2,
+ * V = (N + 5.7685e2) / 1.3905e3,
+ * where V = [0.72, 0.88] V and N = [424, 646].
+ * After the optimization they looks as follows:
+ * N = (13905e-3*V - 5768.5) / 10,
+ * V = (N * 10^5 / 13905 + 57685 * 10^3 / 13905) / 10.
+ * where V = [720, 880] mV and N = [424, 646].
+ */
+static const struct pvt_poly __maybe_unused poly_volt_to_N = {
+	.total_divider = 10,
+	.terms = {
+		{1, 13905, 1000, 1},
+		{0, -57685, 1, 10}
+	}
+};
+
+static const struct pvt_poly poly_N_to_volt = {
+	.total_divider = 10,
+	.terms = {
+		{1, 100000, 13905, 1},
+		{0, 57685000, 1, 13905}
+	}
+};
+
+/*
+ * Here is the polynomial calculation function, which performs the
+ * redistributed terms calculations. It's pretty straightforward. We walk
+ * over each degree term up to the free one, and perform the redistributed
+ * multiplication of the term coefficient, its divider (as for the rationale
+ * fraction representation), data power and the rational fraction divider
+ * leftover. Then all of this is collected in a total sum variable, which
+ * value is normalized by the total divider before being returned.
+ */
+static long eswin_pvt_calc_poly(const struct pvt_poly *poly, long data)
+{
+	const struct pvt_poly_term *term = poly->terms;
+	long tmp, ret = 0;
+	int deg;
+	do {
+		tmp = term->coef;
+		for (deg = 0; deg < term->deg; ++deg)
+			tmp = mult_frac(tmp, data, term->divider);
+		ret += tmp / term->divider_leftover;
+	} while ((term++)->deg);
+
+	return ret / poly->total_divider;
+}
+
+static inline u32 eswin_pvt_update(void __iomem *reg, u32 mask, u32 data)
+{
+	u32 old;
+
+	old = readl_relaxed(reg);
+	writel((old & ~mask) | (data & mask), reg);
+
+	return old & mask;
+}
+
+static inline void eswin_pvt_set_mode(struct pvt_hwmon *pvt, u32 mode)
+{
+	u32 old;
+
+	mode = FIELD_PREP(PVT_MODE_MASK, mode);
+
+	old = eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_MODE, PVT_MODE_MASK, mode);
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static inline u32 eswin_pvt_calc_trim(long temp)
+{
+	temp = clamp_val(temp, 0, PVT_TRIM_TEMP);
+
+	return DIV_ROUND_UP(temp, PVT_TRIM_STEP);
+}
+
+static inline void eswin_pvt_set_trim(struct pvt_hwmon *pvt, u32 val)
+{
+	u32 old;
+
+	old = eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	writel(val, pvt->regs + PVT_TRIM);
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static irqreturn_t eswin_pvt_hard_isr(int irq, void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	struct pvt_cache *cache;
+	u32 val;
+
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	/*
+	 * Nothing special for alarm-less driver. Just read the data, update
+	 * the cache and notify a waiter of this event.
+	 */
+
+	val = readl(pvt->regs + PVT_DATA);
+
+	cache = &pvt->cache[pvt->sensor];
+
+	WRITE_ONCE(cache->data, FIELD_GET(PVT_DATA_OUT, val));
+
+	complete(&cache->conversion);
+
+	return IRQ_HANDLED;
+}
+
+#define pvt_soft_isr NULL
+
+static inline umode_t eswin_pvt_limit_is_visible(enum pvt_sensor_type type)
+{
+	return 0;
+}
+
+static inline umode_t eswin_pvt_pvt_alarm_is_visible(enum pvt_sensor_type type)
+{
+	return 0;
+}
+
+static int eswin_pvt_read_data(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			 long *val)
+{
+	struct pvt_cache *cache = &pvt->cache[type];
+	unsigned long timeout;
+	u32 data;
+	int ret;
+	const struct pvt_sensor_info *pvt_info;
+
+	pvt_info = of_device_get_match_data(pvt->dev);
+	if (!pvt_info) {
+		dev_err(pvt->dev, "No matching device data found\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Lock PVT conversion interface until data cache is updated. The
+	 * data read procedure is following: set the requested PVT sensor
+	 * mode, enable IRQ and conversion, wait until conversion is finished,
+	 * then disable conversion and IRQ, and read the cached data.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	pvt->sensor = type;
+	eswin_pvt_set_mode(pvt, pvt_info[type].mode);
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, PVT_ENA_EN);
+
+	/*
+	 * Wait with timeout since in case if the sensor is suddenly powered
+	 * down the request won't be completed and the caller will hang up on
+	 * this procedure until the power is back up again. Multiply the
+	 * timeout by the factor of two to prevent a false timeout.
+	 */
+	timeout = 2 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
+	if(type==PVT_TEMP){
+		timeout = 20 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
+	}
+	ret = wait_for_completion_timeout(&cache->conversion, timeout);
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	data = READ_ONCE(cache->data);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	if (!ret)
+		return -ETIMEDOUT;
+
+	if (type == PVT_TEMP)
+		*val = eswin_pvt_calc_poly(&poly_N_to_temp, data);
+
+	else if (type == PVT_VOLT)
+		*val = eswin_pvt_calc_poly(&poly_N_to_volt, data);
+	else
+		*val = data;
+
+	return 0;
+}
+
+static int eswin_pvt_read_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			  bool is_low, long *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_write_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			   bool is_low, long val)
+{
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			  bool is_low, long *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_channel_info *pvt_channel_info[] = {
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_TYPE | HWMON_T_LABEL |
+			   HWMON_T_OFFSET),
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL),
+	NULL
+};
+
+static inline bool eswin_pvt_hwmon_channel_is_valid(enum hwmon_sensor_types type,
+					      int ch)
+{
+	switch (type) {
+	case hwmon_temp:
+		if (ch < 0 || ch >= PVT_TEMP_CHS)
+			return false;
+		break;
+	case hwmon_in:
+		if (ch < 0 || ch >= PVT_VOLT_CHS)
+			return false;
+		break;
+	default:
+		break;
+	}
+
+	/* The rest of the types are independent from the channel number. */
+	return true;
+}
+
+static umode_t eswin_pvt_hwmon_is_visible(const void *data,
+				    enum hwmon_sensor_types type,
+				    u32 attr, int ch)
+{
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return 0;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return 0644;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+		case hwmon_temp_type:
+		case hwmon_temp_label:
+			return 0444;
+		case hwmon_temp_min:
+		case hwmon_temp_max:
+			return eswin_pvt_limit_is_visible(ch);
+		case hwmon_temp_min_alarm:
+		case hwmon_temp_max_alarm:
+			return eswin_pvt_pvt_alarm_is_visible(ch);
+		case hwmon_temp_offset:
+			return 0644;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+		case hwmon_in_label:
+			return 0444;
+		case hwmon_in_min:
+		case hwmon_in_max:
+			return eswin_pvt_limit_is_visible(PVT_VOLT + ch);
+		case hwmon_in_min_alarm:
+		case hwmon_in_max_alarm:
+			return eswin_pvt_pvt_alarm_is_visible(PVT_VOLT + ch);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_read_trim(struct pvt_hwmon *pvt, long *val)
+{
+	u32 data;
+
+	data = readl(pvt->regs + PVT_TRIM);
+	/* *val = FIELD_GET(PVT_CTRL_TRIM_MASK, data) * PVT_TRIM_STEP; */
+	*val = data;
+
+	return 0;
+}
+
+static int eswin_pvt_write_trim(struct pvt_hwmon *pvt, long val)
+{
+	int ret;
+	/*
+	 * Serialize trim update, since a part of the register is changed and
+	 * the controller is supposed to be disabled during this operation.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	/* trim = eswin_pvt_calc_trim(val); */
+	eswin_pvt_set_trim(pvt, val);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_read_timeout(struct pvt_hwmon *pvt, long *val)
+{
+	int ret;
+
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	/* Return the result in msec as hwmon sysfs interface requires. */
+	*val = ktime_to_ms(pvt->timeout);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_write_timeout(struct pvt_hwmon *pvt, long val)
+{
+	unsigned long rate;
+	ktime_t kt, cache;
+	u32 data;
+	int ret;
+
+	rate = clk_get_rate(pvt->clk);
+	if (!rate)
+		return -ENODEV;
+
+	/*
+	 * If alarms are enabled, the requested timeout must be divided
+	 * between all available sensors to have the requested delay
+	 * applicable to each individual sensor.
+	 */
+	cache = kt = ms_to_ktime(val);
+
+	/*
+	 * Subtract a constant lag, which always persists due to the limited
+	 * PVT sampling rate. Make sure the timeout is not negative.
+	 */
+	kt = ktime_sub_ns(kt, PVT_TOUT_MIN);
+	if (ktime_to_ns(kt) < 0)
+		kt = ktime_set(0, 0);
+
+	/*
+	 * Finally recalculate the timeout in terms of the reference clock
+	 * period.
+	 */
+	data = ktime_divns(kt * rate, NSEC_PER_SEC);
+
+	/*
+	 * Update the measurements delay, but lock the interface first, since
+	 * we have to disable PVT in order to have the new delay actually
+	 * updated.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	pvt->timeout = cache;
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int ch, long *val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return eswin_pvt_read_timeout(pvt, val);
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			return eswin_pvt_read_data(pvt, ch, val);
+		case hwmon_temp_type:
+			*val = 1;
+			return 0;
+		case hwmon_temp_min:
+			return eswin_pvt_read_limit(pvt, ch, true, val);
+		case hwmon_temp_max:
+			return eswin_pvt_read_limit(pvt, ch, false, val);
+		case hwmon_temp_min_alarm:
+			return eswin_pvt_read_alarm(pvt, ch, true, val);
+		case hwmon_temp_max_alarm:
+			return eswin_pvt_read_alarm(pvt, ch, false, val);
+		case hwmon_temp_offset:
+			return eswin_pvt_read_trim(pvt, val);
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+			return eswin_pvt_read_data(pvt, PVT_VOLT + ch, val);
+		case hwmon_in_min:
+			return eswin_pvt_read_limit(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max:
+			return eswin_pvt_read_limit(pvt, PVT_VOLT + ch, false, val);
+		case hwmon_in_min_alarm:
+			return eswin_pvt_read_alarm(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max_alarm:
+			return eswin_pvt_read_alarm(pvt, PVT_VOLT + ch, false, val);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_hwmon_read_string(struct device *dev,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int ch, const char **str)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	const struct pvt_sensor_info *pvt_info;
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	pvt_info = of_device_get_match_data(pvt->dev);
+	if (!pvt_info) {
+		dev_err(pvt->dev, "No matching device data found\n");
+		return -EINVAL;
+	}
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_label:
+			*str = pvt_info[ch].label;
+			return 0;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_label:
+			*str = pvt_info[PVT_VOLT + ch].label;
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int ch, long val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return eswin_pvt_write_timeout(pvt, val);
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_min:
+			return eswin_pvt_write_limit(pvt, ch, true, val);
+		case hwmon_temp_max:
+			return eswin_pvt_write_limit(pvt, ch, false, val);
+		case hwmon_temp_offset:
+			return eswin_pvt_write_trim(pvt, val);
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_min:
+			return eswin_pvt_write_limit(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max:
+			return eswin_pvt_write_limit(pvt, PVT_VOLT + ch, false, val);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_ops pvt_hwmon_ops = {
+	.is_visible = eswin_pvt_hwmon_is_visible,
+	.read = eswin_pvt_hwmon_read,
+	.read_string = eswin_pvt_hwmon_read_string,
+	.write = eswin_pvt_hwmon_write
+};
+
+static const struct hwmon_chip_info pvt_hwmon_info = {
+	.ops = &pvt_hwmon_ops,
+	.info = pvt_channel_info
+};
+
+static void pvt_clear_data(void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	int idx;
+
+	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
+		complete_all(&pvt->cache[idx].conversion);
+
+	mutex_destroy(&pvt->iface_mtx);
+}
+
+static struct pvt_hwmon *eswin_pvt_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pvt_hwmon *pvt;
+	int ret, idx;
+
+	pvt = devm_kzalloc(dev, sizeof(*pvt), GFP_KERNEL);
+	if (!pvt)
+		return ERR_PTR(-ENOMEM);
+
+	ret = devm_add_action(dev, pvt_clear_data, pvt);
+	if (ret) {
+		dev_err(dev, "Can't add PVT data clear action\n");
+		return ERR_PTR(ret);
+	}
+
+	pvt->dev = dev;
+	pvt->sensor = PVT_SENSOR_FIRST;
+	mutex_init(&pvt->iface_mtx);
+
+	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
+		init_completion(&pvt->cache[idx].conversion);
+
+	return pvt;
+}
+
+static int eswin_pvt_request_regs(struct pvt_hwmon *pvt)
+{
+	struct platform_device *pdev = to_platform_device(pvt->dev);
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(pvt->dev, "Couldn't find PVT memresource\n");
+		return -EINVAL;
+	}
+
+	pvt->regs = devm_ioremap_resource(pvt->dev, res);
+	if (IS_ERR(pvt->regs))
+		return PTR_ERR(pvt->regs);
+
+	return 0;
+}
+
+static void eswin_pvt_remove(void *data)
+{
+	int ret;
+	struct pvt_hwmon *pvt = data;
+	ret = reset_control_assert(pvt->pvt_rst);
+	WARN_ON(0 != ret);
+	clk_disable_unprepare(pvt->clk);
+}
+
+static int eswin_pvt_request_clks(struct pvt_hwmon *pvt)
+{
+	int ret;
+
+	pvt->clk = devm_clk_get(pvt->dev, "pvt_clk");
+	if (IS_ERR(pvt->clk)) {
+		dev_err(pvt->dev, "Couldn't get PVT clock\n");
+		return -ENODEV;
+	}
+
+	ret = clk_prepare_enable(pvt->clk);
+	if (ret) {
+		dev_err(pvt->dev, "Couldn't enable the PVT clocks\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_request_rst(struct pvt_hwmon *pvt)
+{
+	int ret;
+	pvt->pvt_rst = devm_reset_control_get_optional(pvt->dev, "pvt_rst");
+	if(IS_ERR_OR_NULL(pvt->pvt_rst)){
+		dev_err(pvt->dev, "Couldn't get PVT reset\n");
+	}
+	ret = reset_control_reset(pvt->pvt_rst);
+	WARN_ON(0 != ret);
+	return 0;
+}
+
+static int eswin_pvt_check_pwr(struct pvt_hwmon *pvt)
+{
+	unsigned long tout;
+	int ret = 0;
+	u32 data;
+
+	/*
+	 * Test out the sensor conversion functionality. If it is not done on
+	 * time then the domain must have been unpowered and we won't be able
+	 * to use the device later in this driver.
+	 * Note If the power source is lost during the normal driver work the
+	 * data read procedure will either return -ETIMEDOUT (for the
+	 * alarm-less driver configuration) or just stop the repeated
+	 * conversion. In the later case alas we won't be able to detect the
+	 * problem.
+	 */
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, PVT_ENA_EN);
+	readl(pvt->regs + PVT_DATA);
+
+	tout = PVT_TOUT_MIN / NSEC_PER_USEC;
+	usleep_range(tout, 2 * tout);
+
+	data = readl(pvt->regs + PVT_DATA);
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	return ret;
+}
+
+static int eswin_pvt_init_iface(struct pvt_hwmon *pvt)
+{
+	unsigned long rate;
+	const struct pvt_sensor_info *pvt_info;
+
+	rate = clk_get_rate(pvt->clk);
+	if (!rate) {
+		dev_err(pvt->dev, "Invalid reference clock rate\n");
+		return -ENODEV;
+	}
+	pvt_info = of_device_get_match_data(pvt->dev);
+	if (!pvt_info) {
+		dev_err(pvt->dev, "No matching device data found\n");
+		return -EINVAL;
+	}
+	/*
+	 * Make sure all interrupts and controller are disabled so not to
+	 * accidentally have ISR executed before the driver data is fully
+	 * initialized. Clear the IRQ status as well.
+	 */
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	readl(pvt->regs + PVT_DATA);
+
+	/* Setup default sensor mode, timeout and temperature trim. */
+	eswin_pvt_set_mode(pvt, pvt_info[pvt->sensor].mode);
+
+	/*
+	 * Preserve the current ref-clock based delay (Ttotal) between the
+	 * sensors data samples in the driver data so not to recalculate it
+	 * each time on the data requests and timeout reads. It consists of the
+	 * delay introduced by the internal ref-clock timer (N / Fclk) and the
+	 * constant timeout caused by each conversion latency (Tmin):
+	 *   Ttotal = N / Fclk + Tmin
+	 * If alarms are enabled the sensors are polled one after another and
+	 * in order to get the next measurement of a particular sensor the
+	 * caller will have to wait for at most until all the others are
+	 * polled. In that case the formulae will look a bit different:
+	 *   Ttotal = 5 * (N / Fclk + Tmin)
+	 */
+
+	pvt->timeout = ktime_set(PVT_TOUT_DEF, 0);
+	pvt->timeout = ktime_divns(pvt->timeout, rate);
+	pvt->timeout = ktime_add_ns(pvt->timeout, PVT_TOUT_MIN);
+
+        /*
+	if (!of_property_read_u32(pvt->dev->of_node,
+	     "pvt-temp-offset-millicelsius", &temp))
+		trim = eswin_pvt_calc_trim(temp);
+	eswin_pvt_set_trim(pvt, trim);
+        */
+
+	return 0;
+}
+
+static int eswin_pvt_request_irq(struct pvt_hwmon *pvt)
+{
+	struct platform_device *pdev = to_platform_device(pvt->dev);
+	int ret;
+
+	pvt->irq = platform_get_irq(pdev, 0);
+	if (pvt->irq < 0)
+		return pvt->irq;
+
+	ret = devm_request_threaded_irq(pvt->dev, pvt->irq,
+					eswin_pvt_hard_isr, pvt_soft_isr,
+					IRQF_SHARED | IRQF_TRIGGER_HIGH,
+					"pvt", pvt);
+	if (ret) {
+		dev_err(pvt->dev, "Couldn't request PVT IRQ\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_create_hwmon(struct pvt_hwmon *pvt)
+{
+	pvt->hwmon = devm_hwmon_device_register_with_info(pvt->dev, "pvt", pvt,
+		&pvt_hwmon_info, NULL);
+	if (IS_ERR(pvt->hwmon)) {
+		dev_err(pvt->dev, "Couldn't create hwmon device\n");
+		return PTR_ERR(pvt->hwmon);
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_enable_iface(struct pvt_hwmon *pvt)
+{
+	return 0;
+}
+
+static int eswin_pvt_probe(struct platform_device *pdev)
+{
+	struct pvt_hwmon *pvt;
+	int ret;
+
+	pvt = eswin_pvt_create_data(pdev);
+	if (IS_ERR(pvt))
+		return PTR_ERR(pvt);
+
+	ret = eswin_pvt_request_regs(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_request_clks(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_request_rst(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_check_pwr(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_init_iface(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_request_irq(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_create_hwmon(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_enable_iface(pvt);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(pvt->dev, eswin_pvt_remove, pvt);
+	if (ret) {
+		dev_err(pvt->dev, "Can't add PVT clocks disable action\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id pvt_of_match[] = {
+	{ .compatible = "eswin,eswin-pvt-cpu",
+	 .data = &pvt_info_cpu},
+	{ .compatible = "eswin,eswin-pvt-ddr",
+	 .data = &pvt_info_ddr},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, pvt_of_match);
+
+static struct platform_driver pvt_driver = {
+	.probe = eswin_pvt_probe,
+	.driver = {
+		.name = "eswin-pvt",
+		.of_match_table = pvt_of_match
+	},
+};
+module_platform_driver(pvt_driver);
+
+MODULE_DESCRIPTION("Eswin PVT driver");
+MODULE_AUTHOR("Yulin Lu <luyulin@eswincomputing.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/hwmon/eswin_pvt.h b/drivers/hwmon/eswin_pvt.h
new file mode 100644
index 000000000000..fe30569f4897
--- /dev/null
+++ b/drivers/hwmon/eswin_pvt.h
@@ -0,0 +1,246 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN PVT device driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ */
+#ifndef __HWMON_ESWIN_PVT_H__
+#define __HWMON_ESWIN_PVT_H__
+
+#include <linux/completion.h>
+#include <linux/hwmon.h>
+#include <linux/kernel.h>
+#include <linux/ktime.h>
+#include <linux/mutex.h>
+#include <linux/seqlock.h>
+
+/* Eswin PVT registers and their bitfields */
+#define PVT_DIV               0x00
+#define PVT_TRIM              0x04
+#define PVT_TRIM_G            GENMASK(4,0)
+#define PVT_TRIM_O            GENMASK(13,8)
+#define PVT_MODE              0x08
+#define PVT_MODE_MASK         GENMASK(2, 0)
+#define PVT_CTRL_MODE_TEMP    0x0
+#define PVT_CTRL_MODE_VOLT    0x4
+#define PVT_CTRL_MODE_LVT     0x1
+#define PVT_CTRL_MODE_ULVT    0x2
+#define PVT_CTRL_MODE_SVT     0x3
+#define PVT_MODE_PSAMPLE_0    BIT(0)
+#define PVT_MODE_PSAMPLE_1    BIT(1)
+#define PVT_MODE_VSAMPLE      BIT(2)
+#define PVT_ENA               0x0c
+#define PVT_ENA_EN            BIT(0)
+#define PVT_INT               0x10
+#define PVT_INT_CLR           BIT(1)
+#define PVT_DATA              0x14
+#define PVT_DATA_OUT          GENMASK(9,0)
+
+/* alarm related */
+#define PVT_TTHRES			0x08
+#define PVT_VTHRES			0x0C
+#define PVT_LTHRES			0x10
+#define PVT_ULTHRES			0x14
+#define PVT_STHRES			0x18
+#define PVT_INTR_DVALID			BIT(0)
+#define PVT_INTR_TTHRES_LO		BIT(1)
+#define PVT_INTR_TTHRES_HI		BIT(2)
+#define PVT_INTR_VTHRES_LO		BIT(3)
+#define PVT_INTR_VTHRES_HI		BIT(4)
+#define PVT_INTR_LTHRES_LO		BIT(5)
+#define PVT_INTR_LTHRES_HI		BIT(6)
+#define PVT_INTR_ULTHRES_LO		BIT(7)
+#define PVT_INTR_ULTHRES_HI		BIT(8)
+#define PVT_INTR_STHRES_LO		BIT(9)
+#define PVT_INTR_STHRES_HI		BIT(10)
+
+/*
+ * PVT sensors-related limits and default values
+ * @PVT_TEMP_MIN: Minimal temperature in millidegrees of Celsius.
+ * @PVT_TEMP_MAX: Maximal temperature in millidegrees of Celsius.
+ * @PVT_TEMP_CHS: Number of temperature hwmon channels.
+ * @PVT_VOLT_MIN: Minimal voltage in mV.
+ * @PVT_VOLT_MAX: Maximal voltage in mV.
+ * @PVT_VOLT_CHS: Number of voltage hwmon channels.
+ * @PVT_DATA_MIN: Minimal PVT raw data value.
+ * @PVT_DATA_MAX: Maximal PVT raw data value.
+ * @PVT_TRIM_MIN: Minimal temperature sensor trim value.
+ * @PVT_TRIM_MAX: Maximal temperature sensor trim value.
+ * @PVT_TRIM_DEF: Default temperature sensor trim value (set a proper value
+ *		  when one is determined for ESWIN SoC).
+ * @PVT_TRIM_TEMP: Maximum temperature encoded by the trim factor.
+ * @PVT_TRIM_STEP: Temperature stride corresponding to the trim value.
+ * @PVT_TOUT_MIN: Minimal timeout between samples in nanoseconds.
+ * @PVT_TOUT_DEF: Default data measurements timeout. In case if alarms are
+ *		  activated the PVT IRQ is enabled to be raised after each
+ *		  conversion in order to have the thresholds checked and the
+ *		  converted value cached. Too frequent conversions may cause
+ *		  the system CPU overload. Lets set the 50ms delay between
+ *		  them by default to prevent this.
+ */
+#define PVT_TEMP_MIN		-40000L
+#define PVT_TEMP_MAX		125000L
+#define PVT_TEMP_CHS		1
+#define PVT_VOLT_MIN		720L
+#define PVT_VOLT_MAX		880L
+#define PVT_VOLT_CHS		4
+#define PVT_DATA_MIN		0
+#define PVT_DATA_DATA_FLD       0
+#define PVT_CTRL_TRIM_FLD       4
+#define PVT_CTRL_TRIM_MASK      GENMASK(8,4)
+#define PVT_DATA_MAX		(PVT_DATA_DATA_MASK >> PVT_DATA_DATA_FLD)
+#define PVT_TRIM_MIN		0
+#define PVT_TRIM_MAX		(PVT_CTRL_TRIM_MASK >> PVT_CTRL_TRIM_FLD)
+#define PVT_TRIM_TEMP		7130
+#define PVT_TRIM_STEP		(PVT_TRIM_TEMP / PVT_TRIM_MAX)
+#define PVT_TRIM_DEF		0
+#define PVT_TOUT_MIN		(NSEC_PER_SEC / 3000)
+# define PVT_TOUT_DEF		0
+#define LPCPU_PVT_TEMP_REG_OFFSET	0x0
+#define LPCPU_PVT_VOLT_REG_OFFSET	0x4
+
+/*
+ * enum pvt_sensor_type - ESWIN PVT sensor types (correspond to each PVT
+ *			  sampling mode)
+ * @PVT_SENSOR*: helpers to traverse the sensors in loops.
+ * @PVT_TEMP: PVT Temperature sensor.
+ * @PVT_VOLT: PVT Voltage sensor.
+ * @PVT_LVT: PVT Low-Voltage threshold sensor.
+ * @PVT_HVT: PVT High-Voltage threshold sensor.
+ * @PVT_SVT: PVT Standard-Voltage threshold sensor.
+ */
+enum pvt_sensor_type {
+	PVT_SENSOR_FIRST,
+	PVT_TEMP = PVT_SENSOR_FIRST,
+	PVT_VOLT,
+	PVT_LVT,
+	PVT_ULVT,
+	PVT_SVT,
+	PVT_SENSOR_LAST = PVT_SVT,
+	PVT_SENSORS_NUM
+};
+
+/*
+ * struct pvt_sensor_info - ESWIN PVT sensor informational structure
+ * @channel: Sensor channel ID.
+ * @label: hwmon sensor label.
+ * @mode: PVT mode corresponding to the channel.
+ * @thres_base: upper and lower threshold values of the sensor.
+ * @thres_sts_lo: low threshold status bitfield.
+ * @thres_sts_hi: high threshold status bitfield.
+ * @type: Sensor type.
+ * @attr_min_alarm: Min alarm attribute ID.
+ * @attr_min_alarm: Max alarm attribute ID.
+ */
+struct pvt_sensor_info {
+	int channel;
+	const char *label;
+	u32 mode;
+	unsigned long thres_base;
+	u32 thres_sts_lo;
+	u32 thres_sts_hi;
+	enum hwmon_sensor_types type;
+	u32 attr_min_alarm;
+	u32 attr_max_alarm;
+};
+
+#define PVT_SENSOR_INFO(_ch, _label, _type, _mode, _thres)	\
+	{							\
+		.channel = _ch,					\
+		.label = _label,				\
+		.mode = PVT_CTRL_MODE_ ##_mode,			\
+		.thres_base = PVT_ ##_thres,			\
+		.thres_sts_lo = PVT_INTR_ ##_thres## _LO,	\
+		.thres_sts_hi = PVT_INTR_ ##_thres## _HI,	\
+		.type = _type,					\
+		.attr_min_alarm = _type## _min,			\
+		.attr_max_alarm = _type## _max,			\
+	}
+
+/*
+ * struct pvt_cache - PVT sensors data cache
+ * @data: data cache in raw format.
+ * @thres_sts_lo: low threshold status saved on the previous data conversion.
+ * @thres_sts_hi: high threshold status saved on the previous data conversion.
+ * @data_seqlock: cached data seq-lock.
+ * @conversion: data conversion completion.
+ */
+struct pvt_cache {
+	u32 data;
+	struct completion conversion;
+};
+
+/*
+ * struct pvt_hwmon - Eswin PVT private data
+ * @dev: device structure of the PVT platform device.
+ * @hwmon: hwmon device structure.
+ * @regs: pointer to the Eswin PVT registers region.
+ * @irq: PVT events IRQ number.
+ * @clk: PVT core clock (1.2MHz).
+ * @pvt_rst: pointer to the reset descriptor.
+ * @iface_mtx: Generic interface mutex (used to lock the alarm registers
+ *	       when the alarms enabled, or the data conversion interface
+ *	       if alarms are disabled).
+ * @sensor: current PVT sensor the data conversion is being performed for.
+ * @cache: data cache descriptor.
+ * @timeout: conversion timeout cache.
+ */
+struct pvt_hwmon {
+	struct device *dev;
+	struct device *hwmon;
+
+	void __iomem *regs;
+	int irq;
+
+	struct clk *clk;
+	struct reset_control *pvt_rst;
+	struct mutex iface_mtx;
+	enum pvt_sensor_type sensor;
+	struct pvt_cache cache[PVT_SENSORS_NUM];
+	ktime_t timeout;
+	struct regmap *regmap;
+	u32 regmap_offset;
+};
+
+/*
+ * struct pvt_poly_term - a term descriptor of the PVT data translation
+ *			  polynomial
+ * @deg: degree of the term.
+ * @coef: multiplication factor of the term.
+ * @divider: distributed divider per each degree.
+ * @divider_leftover: divider leftover, which couldn't be redistributed.
+ */
+struct pvt_poly_term {
+	unsigned int deg;
+	long coef;
+	long divider;
+	long divider_leftover;
+};
+
+/*
+ * struct pvt_poly - PVT data translation polynomial descriptor
+ * @total_divider: total data divider.
+ * @terms: polynomial terms up to a free one.
+ */
+struct pvt_poly {
+	long total_divider;
+	struct pvt_poly_term terms[];
+};
+
+#endif /* __HWMON_ESWIN_PVT_H__ */
+
diff --git a/drivers/hwmon/pac193x.c b/drivers/hwmon/pac193x.c
new file mode 100644
index 000000000000..c0d165a206c3
--- /dev/null
+++ b/drivers/hwmon/pac193x.c
@@ -0,0 +1,981 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Driver for microchip  pac1931,pac1932,pac1933,pac1934 power monitor chips
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yang Wei <yangwei1@eswincomputing.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/jiffies.h>
+#include <linux/of_device.h>
+#include <linux/of.h>
+#include <linux/delay.h>
+#include <linux/util_macros.h>
+
+#define PAC193X_MAX_CHAN_CNT 4
+
+#define PAC193X_REGISTERS 0x27
+#define PAC193X_VPOWERN_ACC_LEN 6
+#define PAC193X_VOLT_VALUE_LEN 2
+#define PAC193X_VPOWERN_VALUE_LEN 4
+#define PAC193X_ACC_COUNT_LEN 3
+
+#define PAC193X_CMD_REFRESH 0x0
+#define PAC193X_CMD_CTRL 0x1
+#define PAC193X_CMD_ACC_COUNT 0x2
+#define PAC193X_CMD_VPOWER1_ACC 0x3
+#define PAC193X_CMD_VPOWER2_ACC 0x4
+#define PAC193X_CMD_VPOWER3_ACC 0x5
+#define PAC193X_CMD_VPOWER4_ACC 0x6
+#define PAC193X_CMD_VBUS1 0x7
+#define PAC193X_CMD_VBUS2 0x8
+#define PAC193X_CMD_VBUS3 0x9
+#define PAC193X_CMD_VBUS4 0xa
+#define PAC193X_CMD_VSENSE1 0xb
+#define PAC193X_CMD_VSENSE2 0xc
+#define PAC193X_CMD_VSENSE3 0xd
+#define PAC193X_CMD_VSENSE4 0xe
+#define PAC193X_CMD_VBUS1_AVG 0xf
+#define PAC193X_CMD_VBUS2_AVG 0x10
+#define PAC193X_CMD_VBUS3_AVG 0x11
+#define PAC193X_CMD_VBUS4_AVG 0x12
+#define PAC193X_CMD_VSENSE1_AVG 0x13
+#define PAC193X_CMD_VSENSE2_AVG 0x14
+#define PAC193X_CMD_VSENSE3_AVG 0x15
+#define PAC193X_CMD_VSENSE4_AVG 0x16
+#define PAC193X_CMD_VPOWER1 0x17
+#define PAC193X_CMD_VPOWER2 0x18
+#define PAC193X_CMD_VPOWER3 0x19
+#define PAC193X_CMD_VPOWER4 0x1a
+#define PAC193X_CMD_CHANNEL_SMBUS 0x1c
+#define PAC193X_CMD_NEG_PWR 0x1D
+#define PAC193X_CMD_REFRESH_G 0x1E
+#define PAC193X_CMD_REFRESH_V 0x1F
+#define PAC193X_CMD_SLOW 0x20
+#define PAC193X_CMD_CTRL_ACT 0x21
+#define PAC193X_CMD_DIS_ACT 0x22
+#define PAC193X_CMD_NEG_PWR_ACT 0x23
+#define PAC193X_CMD_CTRL_LAT 0x24
+#define PAC193X_CMD_DIS_LAT 0x25
+#define PAC193X_CMD_NEG_PWR_LAT 0x26
+#define PAC193X_CMD_PID 0xFD
+#define PAC193X_CMD_MID 0xFE
+#define PAC193X_CMD_REVERSION_ID 0xFF
+
+#define PAC193X_COSTANT_PWR_M 3200000000ull /* 3.2V^2*1000mO*/
+#define PAC193X_COSTANT_CURRENT_M 100000	/* 100mv*1000mO*/
+
+struct pac193x_data
+{
+	struct mutex config_lock;
+	struct i2c_client *client;
+	u32 update_time_ms;
+	u32 energy_acc_count;
+	struct workqueue_struct *update_workqueue;
+	struct delayed_work update_work;
+	u32 vbus_denominator[PAC193X_MAX_CHAN_CNT];
+	u32 vsense_denominator[PAC193X_MAX_CHAN_CNT];
+	u32 vpower_denominator[PAC193X_MAX_CHAN_CNT];
+	u32 shunt_resistors[PAC193X_MAX_CHAN_CNT];
+	char pac193x_label[PAC193X_MAX_CHAN_CNT][16];
+	u32 sample_rate;
+};
+
+enum PAC193X_CHAN_INDEX
+{
+	pac1931_index = 0,
+	pac1932_index,
+	pac1933_index,
+	pac1934_index
+};
+
+static int pac193x_get_energy(struct device *dev, u8 commmd, u8 chan, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u64 cache = 0;
+	u8 *pcache = (u8 *)&cache;
+	u64 energy_value = 0;
+
+	commmd = commmd + chan;
+	dev_dbg(dev, "%s.%d,chan:%d,commad:%d,LEN:%ld\n", __FUNCTION__,
+			__LINE__, chan, commmd, sizeof(cache));
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_i2c_block_data(data->client, commmd,
+										PAC193X_VPOWERN_ACC_LEN, pcache);
+	mutex_unlock(&data->config_lock);
+
+	energy_value = ((cache & 0xff) << 40) | (((cache >> 8) & 0xff) << 32) |
+				   (((cache >> 16) & 0xff) << 24) |
+				   (((cache >> 24) & 0xff) << 16) |
+				   (((cache >> 32) & 0xff) << 8) | (((cache >> 40) & 0xff));
+	energy_value = energy_value & 0xffffffffffffULL;
+	dev_dbg(dev,
+			"%s.%d,commd:0x%x,value:%lld,ret:%d,resistances:%u,denominator:%u,sample_rate:%u\n",
+			__FUNCTION__, __LINE__, commmd, energy_value, ret,
+			data->shunt_resistors[chan], data->vpower_denominator[chan],
+			data->sample_rate);
+
+	/* energy=3200000*Vpower/(Rsense*denominator*sample_rate) */
+	energy_value = ((energy_value / (u64)data->vpower_denominator[chan]) *
+					(PAC193X_COSTANT_PWR_M /
+					 (data->shunt_resistors[chan] * data->sample_rate)));
+
+	*val = energy_value;
+
+	return 0;
+}
+
+static u16 pac193x_get_word(struct pac193x_data *data, u8 commmd)
+{
+	int ret;
+	u16 cache = 0;
+
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_word_data(data->client, commmd);
+	mutex_unlock(&data->config_lock);
+	cache = ((ret & 0xff) << 8) | ((ret >> 8) & 0xff);
+	dev_dbg(&data->client->dev, "%s.%d,commd:0x%x,value:0x%x,ret:0x%x\n",
+			__FUNCTION__, __LINE__, commmd, cache, ret);
+
+	return cache;
+}
+
+static int pac193x_get_volt(struct device *dev, u8 commmd, u8 chan, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u32 cache = 0;
+
+	commmd = commmd + chan;
+	dev_dbg(dev, "%s.%d,commd:0x%x,chan:%d,vbus_denominator:%d\n",
+			__FUNCTION__, __LINE__, commmd, chan,
+			data->vbus_denominator[chan]);
+	cache = pac193x_get_word(data, commmd);
+	/*to mV*/
+	cache = cache * 1000 / data->vbus_denominator[chan];
+	*val = cache;
+	return 0;
+}
+
+static int pac193x_get_current(struct device *dev, u8 commmd, u8 chan,
+							   long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u32 cache = 0;
+
+	commmd = commmd + chan;
+	dev_dbg(dev,
+			"%s.%d,commd:0x%x,chan:%d,vbus_denominator:%d,resistances:%d\n",
+			__FUNCTION__, __LINE__, commmd, chan,
+			data->vsense_denominator[chan], data->shunt_resistors[chan]);
+	cache = pac193x_get_word(data, commmd);
+	/* I=Vsense*100/(Rsense*denominator)	*/
+	cache = cache * PAC193X_COSTANT_CURRENT_M /
+			(data->vsense_denominator[chan] *
+			 data->shunt_resistors[chan]);
+	*val = cache;
+	return 0;
+}
+
+static int pac193x_get_power(struct device *dev, u8 commmd, u8 chan, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u32 cache = 0;
+	u8 *pcache = (u8 *)&cache;
+	u64 pwr_value = 0;
+
+	commmd = commmd + chan;
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_i2c_block_data(data->client, commmd,
+										PAC193X_VPOWERN_VALUE_LEN, pcache);
+	mutex_unlock(&data->config_lock);
+	pwr_value = ((cache & 0xff) << 24) | (((cache >> 8) & 0xff) << 16) |
+				(((cache >> 16) & 0xff) << 8) | ((cache >> 24) & 0xff);
+	pwr_value = pwr_value >> 4;
+	dev_dbg(dev,
+			"%s.%d,commd:0x%x,chan:%d,value:0x%x,pwr_value:0x%llx,%llu,ret:%d,resistances:%u,denominator:%u\n",
+			__FUNCTION__, __LINE__, commmd, chan, cache, pwr_value,
+			pwr_value, ret, data->shunt_resistors[chan],
+			data->vpower_denominator[chan]);
+	/* pwr=3200000*Vpower/Rsense*denominator */
+	pwr_value = pwr_value * PAC193X_COSTANT_PWR_M /
+				((u64)data->shunt_resistors[chan] *
+				 (u64)data->vpower_denominator[chan]);
+	*val = pwr_value;
+	return 0;
+}
+
+static int pac193x_send_refresh_cmd(struct pac193x_data *data, u8 command)
+{
+	int ret = 0;
+
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_write_byte(data->client, command);
+	mutex_unlock(&data->config_lock);
+	dev_dbg(&data->client->dev, "%s.%d,commd:0x%x,ret:%d\n", __FUNCTION__,
+			__LINE__, command, ret);
+	return ret;
+}
+
+static ssize_t pac193x_refresh_store(struct device *dev,
+									 struct device_attribute *da,
+									 const char *buf, size_t count)
+{
+	struct sensor_device_attribute *sensor_attr = to_sensor_dev_attr(da);
+	struct pac193x_data *data = dev_get_drvdata(dev);
+
+	pac193x_send_refresh_cmd(data, (u8)sensor_attr->index);
+	return count;
+}
+
+static struct sensor_device_attribute pac1934_refreshs[] = {
+	SENSOR_ATTR_WO(reset_energy_history, pac193x_refresh, PAC193X_CMD_REFRESH),
+	/* SENSOR_ATTR_WO(refresh_all_193x, pac193x_refresh,
+				   PAC193X_CMD_REFRESH_G),
+	SENSOR_ATTR_WO(refresh_updata_value, pac193x_refresh,
+				   PAC193X_CMD_REFRESH_V), */
+};
+
+static u8 pac193x_read_byte_data(struct pac193x_data *data, u8 command)
+{
+	int cache = 0;
+	int cnt = 0;
+	while (1)
+	{
+		mutex_lock(&data->config_lock);
+		cache = i2c_smbus_read_byte_data(data->client, command);
+		mutex_unlock(&data->config_lock);
+		if (0xff != cache)
+		{
+			break;
+		}
+		cnt++;
+		if (cnt > 100)
+		{
+			dev_err(&data->client->dev,
+					"get command:%d value error\n", command);
+			return 0xff;
+		}
+	}
+	return (u8)cache;
+}
+
+static ssize_t pac193x_ctrl_show(struct device *dev,
+								 struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0, act_val = 0, lat_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL);
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL_ACT);
+	lat_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL_LAT);
+
+	return sprintf(buf,
+				   "%16s:%6s,%6s,%6s\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n",
+				   "", "set", "act", "latch",
+				   "sample_rate", (set_val >> 6) & 0x3, (act_val >> 6) & 0x3, (lat_val >> 6) & 0x3,
+				   "SLEEP", (set_val >> 5) & 0x1, (act_val >> 5) & 0x1, (lat_val >> 5) & 0x1,
+				   "SING", (set_val >> 4) & 0x1, (act_val >> 4) & 0x1, (lat_val >> 4) & 0x1,
+				   "ALERT_PIN", (set_val >> 3) & 0x1, (act_val >> 3) & 0x1, (lat_val >> 3) & 0x1,
+				   "ALERT_CC", (set_val >> 2) & 0x1, (act_val >> 2) & 0x1, (lat_val >> 2) & 0x1,
+				   "OVF ALERT", (set_val >> 1) & 0x1, (act_val >> 1) & 0x1, (lat_val >> 1) & 0x1,
+				   "OVF", (set_val >> 0) & 0x1, (act_val >> 0) & 0x1, (lat_val >> 0) & 0x1);
+}
+
+int pac193x_common_reg_set(struct device *dev, const char *buf, u8 commad)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	int status;
+	int ret = 0;
+
+	status = kstrtoul(buf, 10, &val);
+	if (status < 0)
+		return status;
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_write_byte_data(data->client, commad, (u8)val);
+	mutex_unlock(&data->config_lock);
+	return ret;
+}
+
+static ssize_t pac193x_ctrl_store(struct device *dev,
+								  struct device_attribute *attr,
+								  const char *buf, size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_CTRL);
+}
+
+DEVICE_ATTR(control, S_IWUSR | S_IRUGO, pac193x_ctrl_show, pac193x_ctrl_store);
+
+static ssize_t pac193x_acc_count_show(struct device *dev,
+									  struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u32 cache = 0;
+	u8 *pcache = (u8 *)&cache;
+	u32 acc_cnt = 0;
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_i2c_block_data(data->client, PAC193X_CMD_ACC_COUNT,
+										PAC193X_ACC_COUNT_LEN, pcache);
+	mutex_unlock(&data->config_lock);
+
+	acc_cnt = ((cache & 0xff) << 16) | (((cache >> 8) & 0xff) << 8) |
+			  ((cache >> 16) & 0xff);
+	acc_cnt = acc_cnt & 0xffffff;
+
+	ret = sysfs_emit(buf, "%u\n", acc_cnt);
+	return ret;
+}
+
+DEVICE_ATTR(acc_count, 0400, pac193x_acc_count_show, NULL);
+
+static ssize_t pac193x_dis_show(struct device *dev,
+								struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0, act_val = 0, lat_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_CHANNEL_SMBUS);
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_DIS_ACT);
+	lat_val = pac193x_read_byte_data(data, PAC193X_CMD_DIS_LAT);
+
+	return sprintf(buf,
+				   "%16s:%6s,%6s,%6s\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n",
+				   "", "set", "act", "latch",
+				   "CH1_OFF", (set_val >> 7) & 0x1, (act_val >> 7) & 0x1, (lat_val >> 7) & 0x1,
+				   "CH2_OFF", (set_val >> 6) & 0x1, (act_val >> 6) & 0x1, (lat_val >> 6) & 0x1,
+				   "CH3_OFF", (set_val >> 5) & 0x1, (act_val >> 5) & 0x1, (lat_val >> 5) & 0x1,
+				   "CH4_OFF", (set_val >> 4) & 0x1, (act_val >> 4) & 0x1, (lat_val >> 4) & 0x1,
+				   "TIMEOUT", (set_val >> 3) & 0x1, (act_val >> 3) & 0x1, (lat_val >> 3) & 0x1,
+				   "BYTE COUNT", (set_val >> 2) & 0x1, (act_val >> 2) & 0x1, (lat_val >> 2) & 0x1,
+				   "NO SKIP", (set_val >> 1) & 0x1, (act_val >> 1) & 0x1, (lat_val >> 1) & 0x1);
+}
+
+static ssize_t pac193x_dis_store(struct device *dev,
+								 struct device_attribute *attr, const char *buf,
+								 size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_CHANNEL_SMBUS);
+}
+
+DEVICE_ATTR(disable_chan_pmbus, S_IWUSR | S_IRUGO, pac193x_dis_show,
+			pac193x_dis_store);
+
+static ssize_t pac193x_neg_pwr_show(struct device *dev,
+									struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0, act_val = 0, lat_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR);
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR_ACT);
+	lat_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR_LAT);
+
+	return sprintf(buf,
+				   "%16s:%6s,%6s,%6s\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n"
+				   "%16s:%6d,%6d,%6d\n",
+				   "", "set", "act", "latch",
+				   "CH1_BIDI", (set_val >> 7) & 0x1, (act_val >> 7) & 0x1, (lat_val >> 7) & 0x1,
+				   "CH2_BIDI", (set_val >> 6) & 0x1, (act_val >> 6) & 0x1, (lat_val >> 6) & 0x1,
+				   "CH3_BIDI", (set_val >> 5) & 0x1, (act_val >> 5) & 0x1, (lat_val >> 5) & 0x1,
+				   "CH4_BIDI", (set_val >> 4) & 0x1, (act_val >> 4) & 0x1, (lat_val >> 5) & 0x1,
+				   "CH1_BIDV", (set_val >> 3) & 0x1, (act_val >> 3) & 0x1, (lat_val >> 3) & 0x1,
+				   "CH2_BIDV", (set_val >> 2) & 0x1, (act_val >> 2) & 0x1, (lat_val >> 2) & 0x1,
+				   "CH3_BIDV", (set_val >> 1) & 0x1, (act_val >> 1) & 0x1, (lat_val >> 1) & 0x1,
+				   "CH4_BIDV", (set_val >> 0) & 0x1, (act_val >> 0) & 0x1, (lat_val >> 0) & 0x1);
+}
+
+static ssize_t pac193x_neg_pwr_store(struct device *dev,
+									 struct device_attribute *attr,
+									 const char *buf, size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_NEG_PWR);
+}
+
+DEVICE_ATTR(neg_pwr, S_IWUSR | S_IRUGO, pac193x_neg_pwr_show,
+			pac193x_neg_pwr_store);
+
+static ssize_t pac193x_slow_show(struct device *dev,
+								 struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_SLOW);
+
+	return sprintf(buf,
+				   "SLOW:%d,SLOW-LH:%d,SLOW-HL:%d, R_RISE:%d,"
+				   "R_V_RISE:%d,R_FALL:%d,R_V_FALL:%d,POR:%d\n",
+				   (set_val >> 7) & 0x1, (set_val >> 6) & 0x1,
+				   (set_val >> 5) & 0x1, (set_val >> 4) & 0x1,
+				   (set_val >> 3) & 0x1, (set_val >> 2) & 0x1,
+				   (set_val >> 1) & 0x1, (set_val >> 0) & 0x1);
+}
+
+static ssize_t pac193x_slow_store(struct device *dev,
+								  struct device_attribute *attr,
+								  const char *buf, size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_SLOW);
+}
+
+DEVICE_ATTR(slow_ctrl, S_IWUSR | S_IRUGO, pac193x_slow_show,
+			pac193x_slow_store);
+
+static ssize_t pac193x_version_show(struct device *dev,
+									struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u8 pid = 0, mid = 0, rid = 0;
+
+	pid = pac193x_read_byte_data(data, PAC193X_CMD_PID);
+	mid = pac193x_read_byte_data(data, PAC193X_CMD_MID);
+	rid = pac193x_read_byte_data(data, PAC193X_CMD_REVERSION_ID);
+
+	ret = sysfs_emit(buf, "PID:0x%x,MID:0x%x,RID:0x%x\n", pid, mid, rid);
+	return ret;
+}
+DEVICE_ATTR(pac193x_version, 0400, pac193x_version_show, NULL);
+
+static struct attribute *pac193x_attrs[] = {
+	&dev_attr_control.attr,
+	&dev_attr_acc_count.attr,
+	&dev_attr_neg_pwr.attr,
+	&dev_attr_disable_chan_pmbus.attr,
+	&dev_attr_slow_ctrl.attr,
+	&dev_attr_pac193x_version.attr,
+	&pac1934_refreshs[0].dev_attr.attr,
+	/* &pac1934_refreshs[1].dev_attr.attr,
+	&pac1934_refreshs[2].dev_attr.attr, */
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(pac193x);
+
+static const struct hwmon_channel_info *pac1931_info[] = {
+	HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT),
+	NULL};
+static const struct hwmon_channel_info *pac1932_info[] = {
+	HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_AVERAGE,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT, HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT, HWMON_E_INPUT),
+	NULL};
+static const struct hwmon_channel_info *pac1933_info[] = {
+	HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_AVERAGE,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT, HWMON_P_INPUT, HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT, HWMON_E_INPUT, HWMON_E_INPUT),
+	NULL};
+static const struct hwmon_channel_info *pac1934_info[] = {
+	HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(in,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE | HWMON_I_LABEL,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE | HWMON_I_LABEL,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE | HWMON_I_LABEL,
+					   HWMON_I_INPUT | HWMON_I_AVERAGE | HWMON_I_LABEL),
+	HWMON_CHANNEL_INFO(curr,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE | HWMON_C_LABEL,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE | HWMON_C_LABEL,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE | HWMON_C_LABEL,
+					   HWMON_C_INPUT | HWMON_C_AVERAGE | HWMON_C_LABEL),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT | HWMON_P_LABEL,
+					   HWMON_P_INPUT | HWMON_P_LABEL,
+					   HWMON_P_INPUT | HWMON_P_LABEL,
+					   HWMON_P_INPUT | HWMON_P_LABEL),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT | HWMON_E_LABEL,
+					   HWMON_E_INPUT | HWMON_E_LABEL,
+					   HWMON_E_INPUT | HWMON_E_LABEL,
+					   HWMON_E_INPUT | HWMON_E_LABEL),
+	NULL};
+
+static umode_t pac1934x_is_visible(const void *_data,
+								   enum hwmon_sensor_types type, u32 attr,
+								   int channel)
+{
+	switch (type)
+	{
+	case hwmon_chip:
+		switch (attr)
+		{
+		case hwmon_chip_update_interval:
+			return 0644;
+		}
+		break;
+	case hwmon_in:
+		switch (attr)
+		{
+		case hwmon_in_label:
+			if (channel == 0)
+			{
+				return 0;
+			}
+			else
+			{
+				return 0444;
+			}
+		case hwmon_in_input:
+		case hwmon_in_average:
+			if (channel == 0)
+			{
+				return 0;
+			}
+			else
+			{
+				return S_IRUGO;
+			}
+		}
+		break;
+	case hwmon_curr:
+		switch (attr)
+		{
+		case hwmon_curr_label:
+			return 0444;
+		case hwmon_curr_input:
+		case hwmon_curr_average:
+			return S_IRUGO;
+		}
+		break;
+	case hwmon_power:
+		switch (attr)
+		{
+		case hwmon_power_label:
+			return 0444;
+		case hwmon_power_input:
+			return S_IRUGO;
+		}
+		break;
+	case hwmon_energy:
+		switch (attr)
+		{
+		case hwmon_energy_label:
+			return 0444;
+		case hwmon_energy_input:
+			return S_IRUGO;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int pac1934x_read_string(struct device *dev,
+								enum hwmon_sensor_types type,
+								u32 attr, int channel, const char **str)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	switch (type)
+	{
+	case hwmon_in:
+		switch (attr)
+		{
+		case hwmon_in_label:
+			if (channel != 0)
+			{
+				*str = data->pac193x_label[channel - 1];
+			}
+			break;
+		}
+		break;
+	case hwmon_curr:
+		switch (attr)
+		{
+		case hwmon_curr_label:
+			*str = data->pac193x_label[channel];
+			break;
+		}
+		break;
+	case hwmon_power:
+		switch (attr)
+		{
+		case hwmon_power_label:
+			*str = data->pac193x_label[channel];
+			break;
+		}
+		break;
+	case hwmon_energy:
+		switch (attr)
+		{
+		case hwmon_energy_label:
+			*str = data->pac193x_label[channel];
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int pac193x_read(struct device *dev, enum hwmon_sensor_types type,
+						u32 attr, int channel, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	switch (type)
+	{
+	case hwmon_chip:
+		switch (attr)
+		{
+		case hwmon_chip_update_interval:
+			*val = data->update_time_ms;
+			break;
+		}
+		break;
+	case hwmon_in:
+		switch (attr)
+		{
+		case hwmon_in_input:
+			return pac193x_get_volt(dev, PAC193X_CMD_VBUS1, channel - 1,
+									val);
+		case hwmon_in_average:
+			return pac193x_get_volt(dev, PAC193X_CMD_VBUS1_AVG,
+									channel - 1, val);
+		}
+		break;
+	case hwmon_curr:
+		switch (attr)
+		{
+		case hwmon_curr_input:
+			return pac193x_get_current(dev, PAC193X_CMD_VSENSE1,
+									   channel, val);
+		case hwmon_curr_average:
+			return pac193x_get_current(dev, PAC193X_CMD_VSENSE1_AVG,
+									   channel, val);
+		}
+		break;
+	case hwmon_power:
+		switch (attr)
+		{
+		case hwmon_power_input:
+			return pac193x_get_power(dev, PAC193X_CMD_VPOWER1,
+									 channel, val);
+		}
+		break;
+	case hwmon_energy:
+		switch (attr)
+		{
+		case hwmon_energy_input:
+			return pac193x_get_energy(dev, PAC193X_CMD_VPOWER1_ACC,
+									  channel, val);
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int pac193x_write(struct device *dev, enum hwmon_sensor_types type,
+						 u32 attr, int channel, long val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	switch (type)
+	{
+	case hwmon_chip:
+		switch (attr)
+		{
+		case hwmon_chip_update_interval:
+			data->update_time_ms = val;
+			mod_delayed_work(data->update_workqueue, &data->update_work,
+					   msecs_to_jiffies(data->update_time_ms));
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static const struct hwmon_ops pac193x_hwmon_ops = {
+	.is_visible = pac1934x_is_visible,
+	.read = pac193x_read,
+	.read_string = pac1934x_read_string,
+	.write = pac193x_write,
+};
+
+static struct hwmon_chip_info pac193x_chip_info = {
+	.ops = &pac193x_hwmon_ops,
+	.info = NULL,
+};
+
+static void update_reg_data(struct work_struct *work)
+{
+	static u32 updata_cnt = 0;
+	struct pac193x_data *data;
+	int act_val = 0, ctrl_val = 0, slow_val = 0;
+	int num = 0;
+	bool is_neg = false;
+
+	data = container_of(work, struct pac193x_data, update_work.work);
+	if ((data->energy_acc_count == 0) || (updata_cnt < data->energy_acc_count))
+	{
+		pac193x_send_refresh_cmd(data, PAC193X_CMD_REFRESH_V);
+		updata_cnt++;
+	}
+	else
+	{
+		pac193x_send_refresh_cmd(data, PAC193X_CMD_REFRESH);
+		updata_cnt = 0;
+	}
+
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR_ACT);
+
+	for (num = 0; num < PAC193X_MAX_CHAN_CNT; num++)
+	{
+		is_neg = false;
+		if (0x1 == ((act_val >> num) & 0x1))
+		{
+			/* Vsource=32*Vbus/2^15 = Vbus/2^10=Vbus/1024  */
+			data->vbus_denominator[3 - num] = 1024;
+			is_neg = true;
+		}
+		else
+		{
+			/* Vsource=32*Vbus/2^16 = Vbus/2^10=Vbus/1024  */
+			data->vbus_denominator[3 - num] = 2048;
+		}
+
+		if (0x1 == ((act_val >> (num + 4)) & 0x1))
+		{
+			/* 2^15  */
+			data->vsense_denominator[3 - num] = 32768;
+			is_neg = true;
+		}
+		else
+		{
+			/*2^16 */
+			data->vsense_denominator[3 - num] = 65536;
+		}
+		if (true == is_neg)
+		{
+			/* 2^28  */
+			data->vpower_denominator[3 - num] = 134217728;
+		}
+		else
+		{
+			/* 2^29  */
+			data->vpower_denominator[3 - num] = 268435456;
+		}
+	}
+
+	slow_val = pac193x_read_byte_data(data, PAC193X_CMD_SLOW);
+	ctrl_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL_ACT);
+	if ((0x1 == ((slow_val >> 7) & 0x1)) &&
+		(0x0 == ((ctrl_val >> 3) & 0x1)))
+	{
+		data->sample_rate = 8;
+	}
+	else
+	{
+		switch ((ctrl_val >> 6) & 0x3)
+		{
+		case 0:
+			data->sample_rate = 1024;
+			break;
+		case 1:
+			data->sample_rate = 256;
+			break;
+		case 2:
+			data->sample_rate = 64;
+			break;
+		case 3:
+			data->sample_rate = 8;
+			break;
+		default:
+			break;
+		}
+	}
+	queue_delayed_work(data->update_workqueue, &data->update_work,
+					   msecs_to_jiffies(data->update_time_ms));
+}
+
+static int pac193x_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct pac193x_data *data;
+	struct device *hwmon_dev;
+	int ret = 0;
+	int num = 0;
+	const char *output_names[4];
+	struct device_node *np;
+	enum PAC193X_CHAN_INDEX chan_index =
+		(enum PAC193X_CHAN_INDEX)of_device_get_match_data(&client->dev);
+
+	switch (chan_index)
+	{
+	case pac1931_index:
+		pac193x_chip_info.info = pac1931_info;
+		break;
+	case pac1932_index:
+		pac193x_chip_info.info = pac1932_info;
+		break;
+	case pac1933_index:
+		pac193x_chip_info.info = pac1933_info;
+		break;
+	case pac1934_index:
+		pac193x_chip_info.info = pac1934_info;
+		break;
+	default:
+		break;
+	}
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	for (num = 0; num < PAC193X_MAX_CHAN_CNT; num++)
+	{
+		data->vbus_denominator[num] = 1;
+		data->vsense_denominator[num] = 1;
+		data->shunt_resistors[num] = 1;
+	}
+
+	np = of_node_get(dev->of_node);
+	if (!np)
+		return -EINVAL;
+	ret = of_property_read_u32(np, "update_time_ms",
+							   &data->update_time_ms);
+	if (0 != ret)
+	{
+		dev_err(dev, "can not get update_time_ms:%d\n", ret);
+		data->update_time_ms = 100;
+	}
+
+	ret = of_property_read_u32(np, "energy_acc_count",
+							   &data->energy_acc_count);
+	if (0 != ret)
+	{
+		dev_err(dev, "can not get energy_acc_count:%d\n", ret);
+		data->energy_acc_count = 0;
+	}
+	ret = of_property_read_u32_array(np, "shunt_resistors",
+									 data->shunt_resistors,
+									 PAC193X_MAX_CHAN_CNT);
+	if (0 != ret)
+	{
+		dev_err(dev, "can not get shunt_resistors:%d\n", ret);
+	}
+
+	dev_info(dev,
+			 "update_time:%d,energy_acc_count:%d,resistances:%d,%d,%d,%dmOhm\n",
+			 data->update_time_ms, data->energy_acc_count,
+			 data->shunt_resistors[0], data->shunt_resistors[1],
+			 data->shunt_resistors[2], data->shunt_resistors[3]);
+
+	np = of_node_get(np);
+	if (!np)
+		return -EINVAL;
+	of_property_read_string_array(np, "eswin,chan_label", output_names, 4);
+
+	strcpy(data->pac193x_label[0], output_names[0]);
+	strcpy(data->pac193x_label[1], output_names[1]);
+	strcpy(data->pac193x_label[2], output_names[2]);
+	strcpy(data->pac193x_label[3], output_names[3]);
+
+	mutex_init(&data->config_lock);
+	data->client = client;
+	mutex_lock(&data->config_lock);
+	i2c_smbus_write_byte_data(data->client, PAC193X_CMD_CTRL, 0x8);
+	mutex_unlock(&data->config_lock);
+	pac193x_send_refresh_cmd(data, PAC193X_CMD_REFRESH);
+
+	hwmon_dev = devm_hwmon_device_register_with_info(
+		dev, client->name, data, &pac193x_chip_info, pac193x_groups);
+	if (IS_ERR(hwmon_dev))
+		return PTR_ERR(hwmon_dev);
+
+	data->update_workqueue = create_workqueue("update_workqueue");
+
+	INIT_DELAYED_WORK(&data->update_work, update_reg_data);
+	queue_delayed_work(data->update_workqueue, &data->update_work,
+					   msecs_to_jiffies(0));
+
+	return 0;
+}
+
+static const struct of_device_id __maybe_unused pac193x_of_match[] = {
+	{.compatible = "microchip,pac1931", .data = (void *)pac1931_index},
+	{.compatible = "microchip,pac1932", .data = (void *)pac1932_index},
+	{.compatible = "microchip,pac1933", .data = (void *)pac1933_index},
+	{.compatible = "microchip,pac1934", .data = (void *)pac1934_index},
+	{},
+};
+MODULE_DEVICE_TABLE(of, pac193x_of_match);
+
+static struct i2c_driver pac193x_driver = {
+	.driver = {
+		.name = "pac193x",
+		.of_match_table = of_match_ptr(pac193x_of_match),
+	},
+	.probe = pac193x_probe,
+};
+
+module_i2c_driver(pac193x_driver);
+
+MODULE_AUTHOR("Yang Wei <yangwei1@eswincomputing.com");
+MODULE_DESCRIPTION("pac193x driver");
+MODULE_LICENSE("GPL");
\ No newline at end of file
diff --git a/drivers/interconnect/Kconfig b/drivers/interconnect/Kconfig
index 5faa8d2aecff..4fe3d2484919 100644
--- a/drivers/interconnect/Kconfig
+++ b/drivers/interconnect/Kconfig
@@ -14,7 +14,7 @@ if INTERCONNECT
 source "drivers/interconnect/imx/Kconfig"
 source "drivers/interconnect/qcom/Kconfig"
 source "drivers/interconnect/samsung/Kconfig"
-
+source "drivers/interconnect/eswin/Kconfig"
 config INTERCONNECT_CLK
 	tristate
 	depends on COMMON_CLK
diff --git a/drivers/interconnect/Makefile b/drivers/interconnect/Makefile
index d0888babb9a1..402059461226 100644
--- a/drivers/interconnect/Makefile
+++ b/drivers/interconnect/Makefile
@@ -7,5 +7,5 @@ obj-$(CONFIG_INTERCONNECT)		+= icc-core.o
 obj-$(CONFIG_INTERCONNECT_IMX)		+= imx/
 obj-$(CONFIG_INTERCONNECT_QCOM)		+= qcom/
 obj-$(CONFIG_INTERCONNECT_SAMSUNG)	+= samsung/
-
+obj-$(CONFIG_INTERCONNECT_ESWIN)	+= eswin/
 obj-$(CONFIG_INTERCONNECT_CLK)		+= icc-clk.o
diff --git a/drivers/interconnect/eswin/Kconfig b/drivers/interconnect/eswin/Kconfig
new file mode 100644
index 000000000000..5c77446dfaa6
--- /dev/null
+++ b/drivers/interconnect/eswin/Kconfig
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config INTERCONNECT_ESWIN
+	tristate "Enable Eswin SoC NOC Management driver"
+	default y
+	help
+	  Say yes to enable NOC management support for Eswin SoC.
+	  say M to compile it as a module.
+	  If in doubt, say N.
+
+config INTERCONNECT_ESWIN_DEBUG
+	bool "Debug Eswin NOC"
+	depends on INTERCONNECT_ESWIN
+	help
+	  This enables verbose debug output of the eswin NOC driver.
+	  If unsure, say N.
+
+config INTERCONNECT_WIN2030_WDT
+	bool "win2030 interconnect wdt driver"
+	default y
+	help
+	  This driver will log the NOC wdt interrupt, usually indicate a timeout on NOC
+	  system.
+	depends on INTERCONNECT_ESWIN
diff --git a/drivers/interconnect/eswin/Makefile b/drivers/interconnect/eswin/Makefile
new file mode 100644
index 000000000000..01e458b6e3c3
--- /dev/null
+++ b/drivers/interconnect/eswin/Makefile
@@ -0,0 +1,10 @@
+#
+# Makefile for Arteris FlexNOC on-chip interconnect
+#
+
+obj-$(CONFIG_INTERCONNECT_WIN2030_WDT)	+= win2030_noc_wdt.o
+ccflags-$(CONFIG_INTERCONNECT_ESWIN_DEBUG) += -DDEBUG
+
+obj-$(CONFIG_INTERCONNECT_ESWIN) += eswin_noc_drv.o
+eswin_noc_drv-objs:= noc.o noc_sysfs.o noc_stat_sysfs.o noc_stat.o noc_profiler.o noc_profiler_sysfs.o of_noc.o win2030_sideband_manager.o
+
diff --git a/drivers/interconnect/eswin/noc.c b/drivers/interconnect/eswin/noc.c
new file mode 100644
index 000000000000..c6d44719db36
--- /dev/null
+++ b/drivers/interconnect/eswin/noc.c
@@ -0,0 +1,2380 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/debugfs.h>
+
+#include "noc.h"
+#include "noc_regs.h"
+
+//#pragma GCC optimize("O0")
+
+#define XGOLD632_NOC_ERROR_ID 0x011B0B00
+#define XGOLD631_NOC_ERROR_ID 0x00FA1200
+#define XGOLD726_NOC_ERROR_ID 0x010C0C00
+
+struct win2030_noc_control win2030_noc_ctrl;
+
+/**
+ * Tasklet called by interrupt when a measurement is finished.
+ * @param data a pointer to the struct win2030_noc_probe which generated the
+ * interrupt
+ */
+void win2030_noc_stat_measure_tasklet(unsigned long data)
+{
+	struct win2030_noc_probe *probe = (struct win2030_noc_probe *)data;
+
+	if (probe_t_trans == probe->type) {
+		win2030_noc_prof_do_measure(probe);
+	} else {
+		win2030_noc_stat_do_measure(probe);
+	}
+}
+
+/**
+ *
+ * @param _dev
+ * @param np
+ * @param property_name
+ * @return
+ */
+const char **win2030_get_strings_from_dts(struct device *_dev,
+					       struct device_node *np,
+					       const char *property_name)
+{
+	struct property *prop;
+	const char *s;
+	const char **lut = NULL;
+	int i = 0, lut_len;
+
+	if (of_property_read_bool(np, property_name)) {
+		lut_len = of_property_count_strings(np, property_name);
+
+		if (lut_len < 0) {
+			dev_err(_dev, "Invalid description of %s for %s\n",
+				property_name, np->name);
+			return ERR_PTR(lut_len);
+		}
+
+		lut = kcalloc(lut_len + 1, sizeof(char *), GFP_KERNEL);
+		if (!lut)
+			return ERR_PTR(-ENOMEM);
+
+		of_property_for_each_string(np, property_name, prop, s)
+			lut[i++] = s;
+
+		/* To find out the end of the list */
+		lut[lut_len] = NULL;
+	}
+
+	return lut;
+}
+
+static struct win2030_bitfield *win2030_alloc_bitfield(void)
+{
+	struct win2030_bitfield *bitfield;
+
+	bitfield = kzalloc(sizeof(*bitfield), GFP_KERNEL);
+	if (!bitfield)
+		return ERR_PTR(-ENOMEM);
+
+	spin_lock_init(&bitfield->lock);
+	INIT_LIST_HEAD(&bitfield->link);
+	return bitfield;
+}
+
+struct win2030_bitfield *win2030_new_bitfield(struct win2030_register *reg,
+						 const char *name,
+						 unsigned offset,
+						 unsigned width,
+						 const char **lut)
+{
+	struct win2030_bitfield *bf;
+	unsigned long flags;
+
+	if (!reg)
+		return ERR_PTR(-EINVAL);
+
+	bf = win2030_alloc_bitfield();
+	if (!bf)
+		return ERR_PTR(-ENOMEM);
+
+	bf->parent = reg;
+	bf->name = kstrdup(name, GFP_KERNEL);
+	bf->offset = offset;
+	bf->length = width;
+	bf->mask = ((BIT(width) - 1) << offset);
+	bf->lut = lut;
+
+	spin_lock_irqsave(&reg->lock, flags);
+	list_add_tail(&bf->link, &reg->bitfields);
+	spin_unlock_irqrestore(&reg->lock, flags);
+
+	return bf;
+}
+
+static void win2030_free_bitfield(struct win2030_bitfield *bf)
+{
+	kfree(bf->name);
+	kfree(bf);
+}
+
+static struct win2030_bitfield *win2030_get_bitfield(struct device *_dev,
+						 struct device_node *np)
+{
+	int ret, i;
+	unsigned values[2];
+	struct win2030_bitfield *bitfield;
+	u64 *values_tab;
+
+	if (!np->name)
+		return ERR_PTR(-EINVAL);
+
+	bitfield = win2030_alloc_bitfield();
+	if (!bitfield)
+		return ERR_PTR(-ENOMEM);
+
+	bitfield->name = np->name;
+
+	ret = of_property_read_u32_array(np, "offset,length", values, 2);
+	if (ret) {
+		dev_err(_dev,
+			"\"Offset,length\" properties of register %s\n missing",
+			bitfield->name);
+		ret = -EINVAL;
+		goto free_bf;
+	}
+
+	bitfield->offset = values[0];
+	bitfield->length = values[1];
+	bitfield->mask = (((1UL << bitfield->length) - 1) << bitfield->offset);
+
+	if ((bitfield->offset > 31) || (bitfield->length > 32)) {
+		ret = -EINVAL;
+		goto free_bf;
+	}
+
+	of_property_read_string(np, "description", &bitfield->description);
+
+	bitfield->lut = win2030_get_strings_from_dts(_dev, np, "lut");
+	if (IS_ERR(bitfield->lut)) {
+		ret = PTR_ERR(bitfield->lut);
+		dev_err(_dev, "Error %d while parsing lut of %s\n", ret,
+			bitfield->name);
+		bitfield->lut = NULL;
+	}
+	bitfield->aperture_size = of_property_count_elems_of_size(np,
+			"aperture-idx,aperture-base", sizeof(u64) * 4);
+	if (bitfield->aperture_size > 0) {
+		bitfield->init_flow = devm_kcalloc(_dev, bitfield->aperture_size * 4,
+						sizeof(u64), GFP_KERNEL);
+		if (!bitfield->init_flow) {
+			ret = -ENOMEM;
+			goto free_all;
+		}
+		bitfield->target_flow = bitfield->init_flow + bitfield->aperture_size;
+		bitfield->target_sub_range = bitfield->target_flow + bitfield->aperture_size;
+		bitfield->aperture_base = bitfield->target_sub_range + bitfield->aperture_size;
+
+		values_tab = devm_kcalloc(_dev, bitfield->aperture_size * 4,
+				sizeof(*values_tab), GFP_KERNEL);
+		if (!values_tab) {
+			ret = -ENOMEM;
+			goto free_all;
+		}
+		ret = of_property_read_u64_array(np, "aperture-idx,aperture-base",
+				values_tab, bitfield->aperture_size * 4);
+		if (ret) {
+			dev_err(_dev,
+				"Error while parsing aperture of %s bitfield, ret %d!\n",
+				bitfield->name, ret);
+			goto free_all;
+		}
+		for (i = 0; i < bitfield->aperture_size; i++) {
+			bitfield->init_flow[i] = values_tab[4 * i];
+			bitfield->target_flow[i] = values_tab[4 * i + 1];
+			bitfield->target_sub_range[i] = values_tab[4 * i + 2];
+			bitfield->aperture_base[i] = values_tab[4 * i + 3];
+		}
+		devm_kfree(_dev, values_tab);
+	}
+
+	dev_dbg(_dev, "Bitfield %s, offset %x, length %d created\n",
+		bitfield->name, bitfield->offset, bitfield->length);
+
+	return bitfield;
+
+free_all:
+	kfree(bitfield->lut);
+free_bf:
+	kfree(bitfield);
+	return ERR_PTR(ret);
+}
+
+static struct win2030_register *win2030_alloc_register(void)
+{
+	struct win2030_register *reg;
+
+	reg = kzalloc(sizeof(*reg), GFP_KERNEL);
+	if (!reg)
+		return ERR_PTR(-ENOMEM);
+
+	spin_lock_init(&reg->lock);
+	spin_lock_init(&reg->hw_lock);
+	INIT_LIST_HEAD(&reg->link);
+	INIT_LIST_HEAD(&reg->bitfields);
+
+	return reg;
+}
+
+struct win2030_register *win2030_new_register(struct device *parent,
+				 const char *name,
+				 void __iomem *base,
+				 unsigned offset,
+				 unsigned width)
+{
+	struct win2030_register *reg;
+
+	reg = win2030_alloc_register();
+	if (!reg)
+		return ERR_PTR(-ENOMEM);
+
+	reg->parent = parent;
+	reg->name = kstrdup(name, GFP_KERNEL);
+	reg->base = base;
+	reg->offset = offset;
+	reg->length = width;
+
+	return reg;
+}
+
+static void win2030_free_register(struct win2030_register *reg)
+{
+	struct win2030_bitfield *bf, *tmp;
+
+	list_for_each_entry_safe(bf, tmp, &reg->bitfields, link)
+		win2030_free_bitfield(bf);
+
+	kfree(reg->name);
+	kfree(reg);
+}
+
+/**
+ * Create ErrorLog register from dts file
+ * @param _dev
+ * @param np
+ * @return
+ */
+static struct win2030_register *win2030_get_err_register(struct device *_dev,
+						 struct device_node *np)
+{
+	int ret;
+	struct device_node *child = NULL;
+	unsigned values[2];
+	struct win2030_register *reg;
+
+	if (!np->name)
+		return ERR_PTR(-EINVAL);
+
+	reg = win2030_alloc_register();
+	if (!reg)
+		return ERR_PTR(-ENOMEM);
+
+	reg->name = np->name;
+	reg->parent = _dev;
+	reg->base = of_iomap(_dev->of_node, 0);
+
+	ret = of_property_read_u32_array(np, "offset,length", values, 2);
+	if (ret) {
+		dev_err(_dev,
+			"\"Offset,length\" properties of register %s\n missing",
+			reg->name);
+		ret = -EINVAL;
+		goto free_reg;
+	}
+
+	reg->offset = values[0];
+	reg->length = values[1];
+	if (reg->length > 32) {
+		ret = -EINVAL;
+		goto free_reg;
+	}
+
+	/* Not mandatory properties  */
+	of_property_read_string(np, "description", &reg->description);
+
+	ret = of_property_read_u32(np, "aperture-link", values);
+	if (!ret)
+		reg->aperture_link = values[0];
+	else
+		reg->aperture_link = -1;
+
+	ret = of_property_read_u32(np, "msb-link", values);
+	if (!ret)
+		reg->msb_link = values[0];
+	else
+		reg->msb_link = -1;
+
+	/* Create bitfields */
+	for_each_child_of_node(np, child) {
+		if (of_device_is_compatible(child, "eswin,win2030,bitfield")) {
+			struct win2030_bitfield *bitfield;
+			bitfield = win2030_get_bitfield(_dev, child);
+			if (!(IS_ERR(bitfield))) {
+				list_add_tail(&bitfield->link, &reg->bitfields);
+				bitfield->parent = reg;
+			}
+		}
+	}
+	dev_dbg(_dev, "Register %s, offset %x, length %d created\n", reg->name,
+		reg->offset, reg->length);
+	return reg;
+
+free_reg:
+	kfree(reg);
+	return ERR_PTR(ret);
+}
+
+int win2030_find_register_by_name(
+	const struct win2030_noc_device *noc_device,
+	const char *name, struct win2030_register **reg)
+{
+	struct win2030_register *tmp_reg = NULL;
+
+	list_for_each_entry(tmp_reg, &noc_device->registers, link) {
+		if (!strcmp(name, tmp_reg->name)) {
+			*reg = tmp_reg;
+			return 0;
+		}
+	}
+	return -EINVAL;
+}
+#if 0
+int win2030_find_routeid_idx_by_name(struct device *_dev,
+	const char *sub_route_id, const char *name, int *idx)
+{
+	int ret;
+	struct win2030_register *reg;
+	struct win2030_bitfield *bf;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+
+	ret = win2030_find_register_by_name(noc_device, "ErrorLogger1", &reg);
+	if (0 != ret) {
+		dev_err(_dev, "%s register not found!\n", "ErrorLogger1");
+		return -EINVAL;
+	}
+	list_for_each_entry(bf, &reg->bitfields, link) {
+		if (strcmp(bf->name, sub_route_id) == 0) {
+			ret = xgold_bitfield_get_value_from_enum(bf, name, idx);
+			if (0 == ret) {
+				break;
+			}
+		}
+	}
+	return ret;
+}
+#endif
+
+/**
+ * Create ErrorLogx register set from DTS file
+ * @param _dev
+ * @param regs
+ * @return
+ */
+static int win2030_get_err_registers(struct device *_dev, struct list_head *regs)
+{
+	struct device_node *parent = _dev->of_node;
+	struct device_node *np = NULL;
+	struct win2030_register *reg;
+
+	for_each_child_of_node(parent, np) {
+		if (of_device_is_compatible(np, "eswin,win2030,register")) {
+			reg = win2030_get_err_register(_dev, np);
+			if (!(IS_ERR(reg)))
+				list_add_tail(&reg->link, regs);
+		}
+	}
+
+	return 0;
+}
+
+static const char *noc_cnt_src_evt_lut[] = {
+	"off",
+	"cycle",
+	"idle",
+	"xfer",
+	"busy",
+	"wait",
+	"pkt",
+	"lut",
+	"byte",
+	"press0",
+	"disabled",
+	"disabled",
+	"filt0",
+	"filt1",
+	"filt2",
+	"filt3",
+	"chain",
+	"lut_byte_en",
+	"lut_byte",
+	"filt_byte_en",
+	"filt_byte",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	"disabled",
+	NULL,
+};
+
+static const char *noc_cnt_alarm_mode_lut[] = {
+	"off",
+	"min",
+	"max",
+	"min_max",
+	NULL
+};
+
+static const char *win2030_noc_enable_enum[] = {
+	"disable",
+	"enable",
+	NULL,
+};
+
+static const char *win2030_noc_on_off_enum[] = {
+	"off",
+	"on",
+	NULL
+};
+
+static const char *win2030_noc_qos_mode_enum[] = {
+	"fixed",
+	"limiter",
+	"bypass",
+	"regulator",
+	NULL
+};
+
+static struct of_device_id win2030_noc_of_match[] = {
+	{
+	 .compatible = "eswin,win2030-noc",
+	},
+	{},
+};
+
+#include <linux/sched.h>
+#include <linux/sched/debug.h>
+void stack_dump(void)
+{
+	struct task_struct *task_list = NULL;
+
+	for_each_process(task_list) {
+		show_stack(task_list, NULL, KERN_EMERG);
+	}
+}
+/**
+ * win2030_noc_get_error - Handle NoC (Network-on-Chip) error interrupts
+ * @noc_device: Pointer to the NoC device structure
+ *
+ * This function is triggered by a NoC error interrupt. It reads all the
+ * error log registers, stores them in an error record, and appends the
+ * record to the device's internal error queue.
+ *
+ * Due to a hardware design limitation with the UART module, when both
+ * LPCPU and MCPU access the UART concurrently, the NoC may report a
+ * spurious error interrupt. These errors are benign and do not lead to
+ * hardware malfunction. To reduce unnecessary alarm during normal
+ * operation, such specific error cases are logged using dev_dbg().
+ *
+ * Other errors are logged with dev_err().
+ *
+ * Return: 0 on success, negative errno on failure (e.g., -ENOMEM)
+ */
+static int win2030_noc_get_error(struct win2030_noc_device *noc_device)
+{
+	void __iomem *base = noc_device->hw_base;
+	struct win2030_noc_error *noc_err;
+	unsigned long flags;
+	int i;
+	char buf[2048] = {'\0'};
+	struct device *mydev = noc_device->dev;
+	bool use_dbg = false;
+
+	noc_err = kzalloc(sizeof(struct win2030_noc_error), GFP_ATOMIC);
+	if (!noc_err)
+		return -ENOMEM;
+
+	noc_err->timestamp = get_jiffies_64();
+
+	// Read all error log registers first but do not print
+	for (i = 0; i < noc_device->error_logger_cnt; i++) {
+		noc_err->err[noc_device->err_log_lut[i]] = ioread32(ERRLOG_0_ERRLOG0(base) +
+					noc_device->err_log_lut[i] * sizeof(u32));
+	}
+
+	// Check if the high 24 bits of ErrLog1 equal 0x000028**, it means uart noc err report
+	if (noc_err->err[1] >> 8 == 0x28)  // Check if upper 24 bits equal 0x000028
+		use_dbg = true;
+
+	// Select printing function based on the flag
+	if (use_dbg) {
+		dev_dbg(mydev, "Error interrupt happen!\n");
+		for (i = 0; i < noc_device->error_logger_cnt; i++) {
+			dev_dbg(mydev, "ErrLog%d 0x%08x", noc_device->err_log_lut[i],
+				noc_err->err[noc_device->err_log_lut[i]]);
+		}
+	} else {
+		dev_err(mydev, "Error interrupt happen!\n");
+		for (i = 0; i < noc_device->error_logger_cnt; i++) {
+			dev_err(mydev, "ErrLog%d 0x%08x", noc_device->err_log_lut[i],
+				noc_err->err[noc_device->err_log_lut[i]]);
+		}
+	}
+
+	spin_lock_irqsave(&noc_device->lock, flags);
+	list_add_tail(&noc_err->link, &noc_device->err_queue);
+	spin_unlock_irqrestore(&noc_device->lock, flags);
+
+	noc_error_dump(buf, noc_device, noc_err);
+
+	// Select print level for final error dump based on the flag
+	if (use_dbg)
+		dev_dbg(mydev, "%s\n", buf);
+	else
+		dev_err(mydev, "%s\n", buf);
+
+	// stack_dump();
+	iowrite32(1, ERRLOG_0_ERRCLR(base));
+	return 0;
+}
+
+
+static irqreturn_t win2030_noc_error_irq(int irq, void *dev)
+{
+	struct win2030_noc_device *noc_device = dev;
+	void __iomem *base = noc_device->hw_base;
+
+	while (ioread32(ERRLOG_0_ERRVLD(base)))
+		win2030_noc_get_error(noc_device);
+
+	BUG_ON(noc_device->trap_on_error == true);
+
+	return IRQ_HANDLED;
+}
+
+void zebu_stop(void)
+{
+	void __iomem *base = ioremap(0x51810000, 0x1000);
+	printk("%s %d\n",__func__,__LINE__);
+	writel_relaxed(0x8000, base + 0x668);
+}
+
+/**
+ * Copy ErrorLog registers into error_registers field + Enable error interrupt
+ * @param _dev
+ * @return
+ */
+static int win2030_noc_error_init(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	char reg_name[16];
+	struct win2030_register *reg;
+	int i;
+	int ret;
+	int cnt;
+	u32 *err_log_lut = NULL;
+	struct platform_device *pdev = to_platform_device(_dev);
+	struct device_node *np = _dev->of_node;
+
+	cnt = of_property_count_u32_elems(np, "errlogger,idx");
+	if (cnt <= 0) {
+		return cnt;
+	}
+	noc_device->error_logger_cnt = cnt;
+	err_log_lut = devm_kcalloc(_dev, cnt, sizeof(u32), GFP_KERNEL);
+	if (!err_log_lut) {
+		return -ENOMEM;
+	}
+	ret = of_property_read_u32_array(np, "errlogger,idx", err_log_lut, cnt);
+	if (ret) {
+		dev_err(_dev, "Error while parsing errlogger,idx, ret %d!\n", ret);
+		return ret;
+	}
+	noc_device->err_log_lut = err_log_lut;
+
+	for (i = 0; i < noc_device->error_logger_cnt; i++) {
+		scnprintf(reg_name, ARRAY_SIZE(reg_name), "ErrorLogger%d",
+			err_log_lut[i]);
+		ret = win2030_find_register_by_name(noc_device, reg_name, &reg);
+		if (0 != ret) {
+			dev_err(_dev, "%s register not found!\n", reg_name);
+			return -EINVAL;
+		}
+		noc_device->error_registers[noc_device->err_log_lut[i]] = reg;
+	}
+	noc_device->err_irq = platform_get_irq_byname(pdev, "error");
+	if (noc_device->err_irq <= 0) {
+		dev_warn(_dev, "error irq not defined!\n");
+		ret = noc_device->err_irq;
+		return ret;
+	} else {
+		ret = devm_request_irq(_dev, noc_device->err_irq, win2030_noc_error_irq,
+				IRQF_SHARED, dev_name(_dev), noc_device);
+		if (ret) {
+			dev_err(_dev, "Error %d while installing error irq\n",
+					(int)noc_device->err_irq);
+			return ret;
+		}
+	}
+	dev_info(_dev, "enable error logging\n");
+	iowrite32(1, ERRLOG_0_FAULTEN(noc_device->hw_base));
+	return 0;
+}
+
+static struct win2030_bitfield *win2030_noc_create_register_and_bitfield(
+		struct win2030_noc_probe *probe,
+		const char *name,
+		void *hw_base,
+		unsigned reg_offset,
+		unsigned bf_offset,
+		unsigned bf_width,
+		const char **bf_lut)
+{
+	struct win2030_register *reg;
+
+	if (probe == NULL)
+		return ERR_PTR(-EINVAL);
+
+	reg = win2030_new_register(probe->dev, name, hw_base, reg_offset, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return ERR_PTR(-EINVAL);
+
+	return win2030_new_bitfield(reg, name, bf_offset, bf_width, bf_lut);
+}
+
+static irqreturn_t win2030_noc_stat_irq(int irq, void *dev)
+{
+	unsigned status;
+	struct win2030_noc_probe *probe = dev;
+	int ret = 0;
+
+	status = ioread32(probe->hw_base + PROBE_STATALARMSTATUS);
+	dev_dbg_once(probe->dev, "counter irq occur, status %d\n", status);
+	if (status) {
+		/* Disable alarm and statics*/
+		ret |= win2030_noc_reg_write(probe->main_ctl, PROBE_MAINCTL_ALARMEN_MASK,
+					   PROBE_MAINCTL_ALARMEN_OFFSET, 0);
+
+		ret |= win2030_noc_reg_write(probe->main_ctl, PROBE_MAINCTL_STATEN_MASK,
+				   PROBE_MAINCTL_STATEN_OFFSET, 0);
+		if (ret < 0) {
+			dev_err(probe->dev, "%s failed to disable alarm and "
+				"statics, ret %d", probe->id, ret);
+		}
+		//iowrite32(0, probe->hw_base + PROBE_MAINCTL);
+
+		/* Clear alarm */
+		iowrite32(1, probe->hw_base + PROBE_STATALARMCLR);
+		tasklet_schedule(&probe->tasklet);
+	}
+	return IRQ_HANDLED;
+}
+
+/**
+ * Create a counter register set (includes PortSel, AlarmMode, Src, Val)
+ * @param cnt
+ * @param probe
+ * @param cnt_id
+ * @return
+ */
+static int win2030_noc_init_counter(struct win2030_noc_counter *cnt,
+				  struct win2030_noc_probe *probe,
+				  unsigned cnt_id)
+{
+	struct win2030_bitfield *bf;
+
+	if ((cnt == NULL) || (probe == NULL))
+		return -EINVAL;
+
+	cnt->id = cnt_id;
+	cnt->parent = probe;
+	INIT_LIST_HEAD(&cnt->register_link);
+#if 0
+	/* Port Selection */
+	bf = win2030_noc_create_register_and_bitfield(probe, "PortSel",
+			    PROBE_COUNTERS_PORTSEL(cnt->id),
+			    PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_OFFSET,
+			    PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_WIDTH,
+			    probe->available_portsel);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	cnt->portsel = bf;
+#endif
+	/* AlarmMode */
+	bf = win2030_noc_create_register_and_bitfield(probe, "AlarmMode",
+			    probe->hw_base,
+			    PROBE_COUNTERS_ALARMMODE(cnt->id),
+			    PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_OFFSET,
+			    PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_WIDTH,
+			    noc_cnt_alarm_mode_lut);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	cnt->alarm_mode = bf;
+	list_add_tail(&bf->parent->parent_link, &cnt->register_link);
+
+	/* Src */
+	bf = win2030_noc_create_register_and_bitfield(probe, "Src",
+				    probe->hw_base,
+				    PROBE_COUNTERS_SRC(cnt->id),
+				    PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+				    PROBE_COUNTERS_SRC_INTEVENT_WIDTH,
+				    noc_cnt_src_evt_lut);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	cnt->source_event = bf;
+	list_add_tail(&bf->parent->parent_link, &cnt->register_link);
+
+	/* Val */
+	bf = win2030_noc_create_register_and_bitfield(probe, "Val",
+				    probe->hw_base,
+				    PROBE_COUNTERS_VAL(cnt->id),
+				    PROBE_COUNTERS_VAL_COUNTERS_VAL_OFFSET,
+				    PROBE_COUNTERS_VAL_COUNTERS_VAL_WIDTH,
+				    NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	cnt->value = bf;
+	list_add_tail(&bf->parent->parent_link, &cnt->register_link);
+
+	return 0;
+}
+
+static struct win2030_register *win2030_noc_get_compatible_register(struct
+								win2030_noc_device
+								*noc_device,
+								const char
+								*compatible)
+{
+	struct device *_dev = noc_device->dev;
+	struct device_node *parent = _dev->of_node;
+	struct device_node *np = NULL;
+	struct win2030_register *reg = NULL;
+	int ret;
+
+	for_each_child_of_node(parent, np) {
+		if (of_device_is_compatible(np, compatible)) {
+			list_for_each_entry(reg, &noc_device->registers, link) {
+				ret = strcmp(np->name, reg->name);
+				if (ret == 0)
+					return reg;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+/**
+ * Create a filter register set (includes RouteIdBase, RouteIdMask, AddrBase_Low,
+ *    WindowSize, SecurityBase, SecurityMask, Opcode, Status, Length, Urgency)
+ * @param noc_device
+ * @param filter
+ * @param probe
+ * @param id
+ * @return
+ */
+static int win2030_noc_init_filter(struct win2030_noc_device *noc_device,
+				 struct win2030_noc_filter *filter,
+				 struct win2030_noc_probe *probe, unsigned id)
+{
+	struct win2030_bitfield *bf, *bf_template = NULL;
+	struct win2030_register *reg, *reg_template;
+	unsigned route_idmask_width;
+
+	if ((noc_device == NULL) || (filter == NULL) || (probe == NULL))
+		return -EINVAL;
+
+	filter->id = id;
+	filter->parent = probe;
+	INIT_LIST_HEAD(&filter->register_link);
+
+	/*"RouteIdBase"*/
+	reg_template = win2030_noc_get_compatible_register(noc_device,
+				"eswin,win2030,noc,filter,routeid");
+	if (!reg_template)
+		return -ENODEV;
+
+	reg = win2030_new_register(probe->dev, "RouteIdBase", probe->hw_base,
+				 PROBE_FILTERS_ROUTEIDBASE(id), 32);
+
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	route_idmask_width = 0;
+	list_for_each_entry(bf_template, &reg_template->bitfields, link) {
+		bf = win2030_new_bitfield(reg, bf_template->name,
+					bf_template->offset,
+					bf_template->length, bf_template->lut);
+		if (IS_ERR_OR_NULL(bf))
+			return -EINVAL;
+		if ((bf_template->offset + bf_template->length)
+			> route_idmask_width)
+				route_idmask_width = (bf_template->offset
+					+ bf_template->length);
+	}
+	filter->route_id_base = reg;
+	list_add_tail(&reg->parent_link, &filter->register_link);
+
+	/* RouteIdMask*/
+	bf = win2030_noc_create_register_and_bitfield(probe, "RouteIdMask",
+				probe->hw_base,
+				PROBE_FILTERS_ROUTEIDMASK(id), 0,
+				route_idmask_width, NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->route_id_mask = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* AddressBase_Low*/
+	bf = win2030_noc_create_register_and_bitfield(probe, "AddressBase_Low",
+			probe->hw_base,
+			PROBE_FILTERS_ADDRBASE_LOW(id),
+			PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET,
+			PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->addr_base_low = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* AddressBase_High*/
+	bf = win2030_noc_create_register_and_bitfield(probe, "AddressBase_High",
+			probe->hw_base,
+			PROBE_FILTERS_ADDRBASE_HIGH(id),
+			PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_OFFSET,
+			PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->addr_base_high = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* Window Size */
+	bf = win2030_noc_create_register_and_bitfield(probe, "WindowSize",
+			probe->hw_base,
+			PROBE_FILTERS_WINDOWSIZE(id),
+			PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_OFFSET,
+			PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->window_size = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+#if 0
+	/*no security deployment*/
+
+	unsigned security_mask_width;
+
+	/* Security Base */
+	reg_template = win2030_noc_get_compatible_register(noc_device,
+					"eswin,win2030,noc,filter,security");
+	if (!reg_template)
+		return -ENODEV;
+
+	scnprintf(_name, ARRAY_SIZE(_name), "probe%d_filter%d_security_base",
+		  probe->id, id);
+
+	reg = win2030_new_register(noc_device->dev, _name, noc_device->hw_base,
+				 PROBE_FILTERS_SECURITYBASE(0, probe->id, id), 32);
+
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	filter->security_base = reg;
+	security_mask_width = 0;
+	list_for_each_entry(bf_template, &reg_template->bitfields, link) {
+		bf = win2030_new_bitfield(reg, bf_template->name,
+					bf_template->offset,
+					bf_template->length, bf_template->lut);
+		if (IS_ERR_OR_NULL(bf))
+			return -EINVAL;
+		if ((bf_template->offset + bf_template->length)
+			> security_mask_width)
+				security_mask_width = (bf_template->offset
+					+ bf_template->length);
+	}
+
+	/* Security Mask */
+	scnprintf(_name, ARRAY_SIZE(_name), "probe%d_filter%d_security_mask",
+		  probe->id, id);
+
+	bf = win2030_noc_create_register_and_bitfield(probe, _name,
+				PROBE_FILTERS_SECURITYMASK
+				(0,
+				probe->id, id), 0,
+				security_mask_width, NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->security_mask = bf;
+#endif
+	/* Opcode*/
+	reg = win2030_new_register(probe->dev, "Opcode",
+			probe->hw_base,
+			PROBE_FILTERS_OPCODE(id), 32);
+
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "RdEn",
+				PROBE_FILTERS_OPCODE_RDEN_OFFSET,
+				PROBE_FILTERS_OPCODE_RDEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "WrEn",
+				PROBE_FILTERS_OPCODE_WREN_OFFSET,
+				PROBE_FILTERS_OPCODE_WREN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "LockEn",
+				PROBE_FILTERS_OPCODE_LOCKEN_OFFSET,
+				PROBE_FILTERS_OPCODE_LOCKEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "UrgEn",
+				PROBE_FILTERS_OPCODE_URGEN_OFFSET,
+				PROBE_FILTERS_OPCODE_URGEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	filter->op_code = reg;
+	list_add_tail(&reg->parent_link, &filter->register_link);
+
+	/* Status */
+	reg = win2030_new_register(probe->dev, "Status", probe->hw_base,
+				 PROBE_FILTERS_STATUS(id), 32);
+
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "status_request_enable",
+				PROBE_FILTERS_STATUS_REQEN_OFFSET,
+				PROBE_FILTERS_STATUS_REQEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "status_response_enable",
+				PROBE_FILTERS_STATUS_RSPEN_OFFSET,
+				PROBE_FILTERS_STATUS_RSPEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->status = reg;
+	list_add_tail(&reg->parent_link, &filter->register_link);
+
+	/* Length */
+	bf = win2030_noc_create_register_and_bitfield(probe, "Length",
+				probe->hw_base,
+				PROBE_FILTERS_LENGTH(id),
+				PROBE_FILTERS_LENGTH_FILTERS_LENGTH_OFFSET,
+				PROBE_FILTERS_LENGTH_FILTERS_LENGTH_WIDTH,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->length = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* Urgency */
+	bf = win2030_noc_create_register_and_bitfield(probe, "Urgency",
+				probe->hw_base,
+				PROBE_FILTERS_URGENCY(id),
+				PROBE_FILTERS_URGENCY_FILTERS_URGENCY_OFFSET,
+				PROBE_FILTERS_URGENCY_FILTERS_URGENCY_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->urgency = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	return 0;
+}
+
+/**
+ * Create a filter register set (includes RouteIdBase, RouteIdMask, AddrBase_Low,
+ *    WindowSize, SecurityBase, SecurityMask, Opcode, Status, Length, Urgency)
+ * @param noc_device
+ * @param filter
+ * @param probe
+ * @param id
+ * @return
+ */
+static int win2030_noc_init_trans_filter(struct win2030_noc_device *noc_device,
+				 struct win2030_noc_trans_filter *filter, void *hw_base,
+				 struct win2030_noc_probe *probe, unsigned id)
+{
+	struct win2030_bitfield *bf;
+	struct win2030_register *reg;
+
+	if ((noc_device == NULL) || (filter == NULL) || (probe == NULL))
+		return -EINVAL;
+
+	filter->id = id;
+	filter->parent = probe;
+	INIT_LIST_HEAD(&filter->register_link);
+
+	/* Mode */
+	bf = win2030_noc_create_register_and_bitfield(probe, "Mode",
+				hw_base,
+				PROBE_TRANS_FILTERS_MODE,
+				PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_OFFSET,
+				PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_WIDTH,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->mode = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* AddressBase_Low */
+	bf = win2030_noc_create_register_and_bitfield(probe, "AddressBase_Low",
+			hw_base,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->addr_base_low = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* AddressBase_High */
+	bf = win2030_noc_create_register_and_bitfield(probe, "AddressBase_High",
+			hw_base,
+			PROBE_TRANS_FILTERS_ADDRBASE_HIGH,
+			PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_OFFSET,
+			PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->addr_base_high = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* WindowSize */
+	bf = win2030_noc_create_register_and_bitfield(probe, "WindowSize",
+			hw_base,
+			PROBE_TRANS_FILTERS_WINDOWSIZE,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->window_size = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* OpCode */
+	reg = win2030_new_register(probe->dev, "OpCode", hw_base,
+				 PROBE_TRANS_FILTERS_OPCODE, 32);
+
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "opcode_rden",
+				PROBE_TRANS_FILTERS_OPCODE_RDEN_OFFSET,
+				PROBE_TRANS_FILTERS_OPCODE_RDEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "opcode_wren",
+				PROBE_TRANS_FILTERS_OPCODE_WREN_OFFSET,
+				PROBE_TRANS_FILTERS_OPCODE_WREN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->op_code = reg;
+	list_add_tail(&reg->parent_link, &filter->register_link);
+
+	/* UserBase */
+	bf = win2030_noc_create_register_and_bitfield(probe, "UserBase",
+			hw_base,
+			PROBE_TRANS_FILTERS_USER_BASE,
+			PROBE_TRANS_FILTERS_USER_BASE_FILTERS_USER_BASE,
+			PROBE_TRANS_FILTERS_USER_BASE_FILTERS_USER_BASE_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->user_base = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	/* UserMask */
+	bf = win2030_noc_create_register_and_bitfield(probe, "UserMask",
+				hw_base,
+				PROBE_TRANS_FILTERS_USER_MASK,
+				PROBE_TRANS_FILTERS_USER_MASK_FILTERS_USER_MASK,
+				PROBE_TRANS_FILTERS_USER_MASK_FILTERS_USER_MASK_WIDTH,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	filter->user_mask = bf;
+	list_add_tail(&bf->parent->parent_link, &filter->register_link);
+
+	return 0;
+}
+
+/**
+ * Create a profiler register set (includes En, Mode, ObservedSel,
+ *    NTenureLines, Thresholds, OverFlowStatus, OverFlowReset, PendingEventMode, PreScaler)
+ * @param noc_device
+ * @param profiler
+ * @param probe
+ * @param id
+ * @return
+ */
+static int win2030_noc_init_trans_profiler(struct win2030_noc_device *noc_device,
+				 struct win2030_noc_trans_profiler *profiler, void *hw_base,
+				 struct win2030_noc_probe *probe, unsigned id)
+{
+	struct win2030_bitfield *bf;
+	int i, j;
+	char reg_name[32];
+
+	if ((noc_device == NULL) || (profiler == NULL) || (probe == NULL))
+		return -EINVAL;
+
+	profiler->id = id;
+	profiler->parent = probe;
+	INIT_LIST_HEAD(&profiler->register_link);
+
+	/* En */
+	bf = win2030_noc_create_register_and_bitfield(probe, "En",
+				hw_base,
+				PROBE_TRANS_PROFILER_EN,
+				PROBE_TRANS_PROFILER_EN_PROFILER_EN_OFFSET,
+				PROBE_TRANS_PROFILER_EN_PROFILER_EN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	profiler->en = bf;
+	list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+
+	/* Mode */
+	bf = win2030_noc_create_register_and_bitfield(probe, "Mode",
+			hw_base,
+			PROBE_TRANS_PROFILER_MODE,
+			PROBE_TRANS_PROFILER_MODE_PROFILER_MODE_OFFSET,
+			PROBE_TRANS_PROFILER_MODE_PROFILER_MODE_WIDTH,
+			NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	profiler->mode = bf;
+	list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+
+	/* ObservedSel */
+	for (i = 0; i < probe->nr_portsel; i++) {
+		scnprintf(reg_name, ARRAY_SIZE(reg_name), "ObservedSel_%d", i);
+		bf = win2030_noc_create_register_and_bitfield(probe, reg_name,
+				hw_base,
+				PROBE_TRANS_PROFILER_OBSERVED_SEL + i * 0x4,
+				PROBE_TRANS_PROFILER_OBSERVED_SEL_PROFILER_OBSERVED_SEL_OFFSET,
+				PROBE_TRANS_PROFILER_OBSERVED_SEL_PROFILER_OBSERVED_SEL_WIDTH,
+				NULL);
+
+		if (IS_ERR_OR_NULL(bf))
+			return -EINVAL;
+
+		profiler->observed_sel[i] = bf;
+		list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+	}
+
+	/* NTenureLines */
+	for (i = 0; i < probe->nr_portsel - 1; i++) {
+		scnprintf(reg_name, ARRAY_SIZE(reg_name), "NTenureLines_%d", i);
+		bf = win2030_noc_create_register_and_bitfield(probe, reg_name,
+				hw_base,
+				PROBE_TRANS_PROFILER_N_TENURE_LINES + i * 0x4,
+				PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_OFFSET,
+				PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_WIDTH,
+				NULL);
+
+		if (IS_ERR_OR_NULL(bf))
+			return -EINVAL;
+
+		profiler->n_tenure_lines[i] = bf;
+		list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+	}
+
+	/* Thresholds */
+	for (i = 0; i < probe->nr_portsel; i++) {
+		for (j = 0; j < WIN2030_NOC_BIN_CNT - 1; j++) {
+			scnprintf(reg_name, ARRAY_SIZE(reg_name), "Thresholds_%d_%d", i, j);
+			bf = win2030_noc_create_register_and_bitfield(probe, reg_name,
+					hw_base,
+					PROBE_TRANS_PROFILER_THRESHOLDS + i * 0x10 + j * 0x4,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_WIDTH,
+					NULL);
+
+			if (IS_ERR_OR_NULL(bf))
+				return -EINVAL;
+
+			profiler->thresholds[i][j] = bf;
+			list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+		}
+	}
+
+	/* OverFlowStatus */
+	bf = win2030_noc_create_register_and_bitfield(probe, "OverFlowStatus",
+				hw_base,
+				PROBE_TRANS_PROFILER_OVER_FLOW_STATUS,
+				PROBE_TRANS_PROFILER_OVER_FLOW_STATUS_PROFILER_OVER_FLOW_STATUS_OFFSET,
+				PROBE_TRANS_PROFILER_OVER_FLOW_STATUS_PROFILER_OVER_FLOW_STATUS_WIDTH,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	profiler->over_flow_status = bf;
+	list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+
+	/* OverFlowReset */
+	bf = win2030_noc_create_register_and_bitfield(probe, "OverFlowReset",
+				hw_base,
+				PROBE_TRANS_PROFILER_OVER_FLOW_RESET,
+				PROBE_TRANS_PROFILER_OVER_FLOW_RESET_PROFILER_OVER_FLOW_RESET_OFFSET,
+				PROBE_TRANS_PROFILER_OVER_FLOW_RESET_PROFILER_OVER_FLOW_RESET_WIDTH,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	profiler->over_flow_reset = bf;
+	list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+
+	/* PendingEventMode */
+	bf = win2030_noc_create_register_and_bitfield(probe, "PendingEventMode",
+				hw_base,
+				PROBE_TRANS_PROFILER_PENDING_EVENT_MODE,
+				PROBE_TRANS_PROFILER_PENDING_EVENT_MODE_PROFILER_PENDING_EVENT_MODE_OFFSET,
+				PROBE_TRANS_PROFILER_PENDING_EVENT_MODE_PROFILER_PENDING_EVENT_MODE_OFFSET,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	profiler->pending_event_mode = bf;
+	list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+
+	/* PreScaler */
+	bf = win2030_noc_create_register_and_bitfield(probe, "PreScaler",
+				hw_base,
+				PROBE_TRANS_PROFILER_PRE_SCALER,
+				PROBE_TRANS_PROFILER_PRE_SCALER_PROFILER_PRE_SCALER_OFFSET,
+				PROBE_TRANS_PROFILER_PRE_SCALER_PROFILER_PRE_SCALER_WIDTH,
+				NULL);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	profiler->pre_scaler = bf;
+	list_add_tail(&bf->parent->parent_link, &profiler->register_link);
+
+	return 0;
+}
+
+static void win2030_noc_remove_probe(struct win2030_noc_probe *probe)
+{
+	unsigned i;
+	struct win2030_noc_counter *counter;
+	struct win2030_register *reg = NULL;
+
+	/*TODO free registers */
+	if (probe_t_pkt == probe->type) {
+		struct win2030_noc_filter *filter = probe->filters;
+		for (i = 0; i < probe->nr_filters; i++) {
+			list_for_each_entry(reg, &filter->register_link, parent_link) {
+				win2030_free_register(reg);
+			}
+			filter++;
+		}
+	} else {
+		struct win2030_noc_trans_filter *filter = probe->trans_filters;
+		for (i = 0; i < probe->nr_filters; i++) {
+			list_for_each_entry(reg, &filter->register_link, parent_link) {
+				win2030_free_register(reg);
+			}
+			filter++;
+		}
+	}
+
+	for (i = 0, counter = probe->counters; i < probe->nr_counters; i++) {
+		list_for_each_entry(reg, &counter->register_link, parent_link) {
+			win2030_free_register(reg);
+		}
+		counter++;
+	}
+	kfree(probe->filters);
+	kfree(probe->counters);
+}
+
+static struct win2030_register *win2030_noc_new_probe_register(char *reg_name,
+			unsigned offset, struct win2030_noc_probe *probe)
+{
+	struct win2030_register *reg;
+
+	reg = win2030_new_register(probe->dev, reg_name, probe->hw_base,
+				 offset, 32);
+	if (!IS_ERR_OR_NULL(reg))
+		list_add_tail(&reg->parent_link, &probe->register_link);
+
+	return reg;
+}
+
+/**
+ * Create a probe register set (includes MainCtl, ConfigCtl, StatPeriod, StatAlarmMin, StatAlarmMax
+ * @param noc_device
+ * @param probe
+ * @param cnt_id
+ * @return
+ */
+static int win2030_noc_init_probe(struct win2030_noc_device *noc_device,
+				struct win2030_noc_probe *probe,
+				struct device_node *np)
+{
+	struct device *_dev = probe->dev;
+	struct win2030_register *reg;
+	struct win2030_bitfield *bf;
+
+	probe->parent = noc_device;
+	tasklet_init(&probe->tasklet, win2030_noc_stat_measure_tasklet,
+		(unsigned long)probe);
+
+	probe->nr_portsel = of_property_count_strings(np, "portsel");
+	if (probe->nr_portsel <= 0) {
+		return -EINVAL;
+	}
+	probe->available_portsel = win2030_get_strings_from_dts(_dev, np, "portsel");
+
+	/**
+	 * Register: MainCtl
+	 */
+	reg = probe->main_ctl = win2030_noc_new_probe_register("MainCtl",
+				PROBE_MAINCTL, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatEn",
+				PROBE_MAINCTL_STATEN_OFFSET,
+				PROBE_MAINCTL_STATEN_WIDTH,
+				win2030_noc_enable_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "AlarmEn",
+				PROBE_MAINCTL_ALARMEN_OFFSET,
+				PROBE_MAINCTL_ALARMEN_WIDTH,
+				win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatCondDump",
+				PROBE_MAINCTL_STATCONDDUMP_OFFSET,
+				PROBE_MAINCTL_STATCONDDUMP_WIDTH,
+				win2030_noc_on_off_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "FiltByteAlwaysChainableEn",
+				PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_OFFSET,
+				PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_WIDTH,
+				win2030_noc_on_off_enum);
+
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: CfgCtl
+	 */
+	reg = probe->cfg_ctl = win2030_noc_new_probe_register("CfgCtl",
+				PROBE_CFGCTL, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "Global_Enable",
+				PROBE_CFGCTL_GLOBALEN_OFFSET,
+				PROBE_CFGCTL_GLOBALEN_WIDTH,
+				win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "Active",
+				PROBE_CFGCTL_GLOBALEN_OFFSET,
+				PROBE_CFGCTL_GLOBALEN_WIDTH,
+				win2030_noc_on_off_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+#if 0
+	/**
+	 * Register: TracePortSel
+	 */
+	scnprintf(mystr, ARRAY_SIZE(mystr), "probe(%s)_TracePortSel", probe->id);
+	reg = win2030_new_register(noc_device->dev, mystr, probe->hw_base,
+				 PROBE_TRACEPORTSEL,
+				 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	probe->trace_port_sel = reg;
+
+	bf = win2030_new_bitfield(reg, "TracePortSel",
+				PROBE_TRACEPORTSEL_TRACEPORTSEL_OFFSET,
+				PROBE_TRACEPORTSEL_TRACEPORTSEL_WIDTH,
+				probe->available_portsel);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+#endif
+	/**
+	 * Register: FilterLut
+	 */
+	reg = probe->filter_lut = win2030_noc_new_probe_register("FilterLut",
+				PROBE_FILTERLUT, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "FilterLut",
+				PROBE_FILTERLUT_FILTERLUT_OFFSET,
+				PROBE_FILTERLUT_FILTERLUT_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: StatPeriod
+	 */
+	reg = probe->stat_period = win2030_noc_new_probe_register("StatPeriod",
+				PROBE_STATPERIOD, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatPeriod",
+				PROBE_STATPERIOD_STATPERIOD_OFFSET,
+				PROBE_STATPERIOD_STATPERIOD_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: StatGo
+	 */
+	reg = probe->stat_go = win2030_noc_new_probe_register("StatGo",
+				PROBE_STATGO, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatGo",
+				PROBE_STATGO_STATGO_OFFSET,
+				PROBE_STATGO_STATGO_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: StatAlarmMin
+	 */
+	reg = probe->stat_alarm_min = win2030_noc_new_probe_register("StatAlarmMin",
+			PROBE_STATALARMMIN, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatAlarmMin",
+				PROBE_STATALARMMIN_STATALARMMIN_OFFSET,
+				PROBE_STATALARMMIN_STATALARMMIN_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: StatAlarmMinHigh
+	 */
+	reg = probe->stat_alarm_min_high = win2030_noc_new_probe_register("StatAlarmMinHigh",
+			PROBE_STATALARMMIN_HIGH, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatAlarmMinHigh",
+				PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_OFFSET,
+				PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: StatAlarmMax
+	 */
+	reg = probe->stat_alarm_max = win2030_noc_new_probe_register("StatAlarmMax",
+				PROBE_STATALARMMAX, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatAlarmMax",
+				PROBE_STATALARMMAX_STATALARMMAX_OFFSET,
+				PROBE_STATALARMMAX_STATALARMMAX_WIDTH, NULL);
+
+	/**
+	 * Register: StatAlarmStatus
+	 */
+	reg = probe->stat_alarm_status = win2030_noc_new_probe_register("StatAlarmStatus",
+			PROBE_STATALARMSTATUS, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatAlarmStatus",
+				PROBE_STATALARMSTATUS_STATALARMSTATUS_OFFSET,
+				PROBE_STATALARMSTATUS_STATALARMSTATUS_WIDTH,
+				NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	/**
+	 * Register: StatAlarmEn
+	 */
+	reg = probe->stat_alarm_en = win2030_noc_new_probe_register("StatAlarmEn",
+			PROBE_STATALARMEN, probe);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "StatAlarmEn",
+				PROBE_STATALARMEN_STATALARMEN_OFFSET,
+				PROBE_STATALARMEN_STATALARMEN_WIDTH, win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int win2030_noc_parse_dts_qoscfg(struct device *_dev,
+				char *propname,
+				struct regcfg **config)
+{
+	int def_len;
+	struct property *prop;
+	struct device_node *np = _dev->of_node;
+
+	prop = of_find_property(np, propname, &def_len);
+	if (prop != NULL) {
+		unsigned i;
+		struct regcfg *reg;
+
+		*config = (struct regcfg *)
+			devm_kzalloc(_dev,
+					sizeof(struct regcfg), GFP_KERNEL);
+		if (*config == NULL) {
+			dev_err(_dev, "%s: Memory allocation failed!",
+					__func__);
+			BUG();
+		}
+		INIT_LIST_HEAD(&(*config)->list);
+
+		for (i = 0; i < (def_len / sizeof(u32)); i += 2) {
+			reg = (struct regcfg *)
+				devm_kzalloc(_dev,
+				sizeof(struct regcfg), GFP_KERNEL);
+			if (reg == NULL) {
+				dev_err(_dev, "%s: Memory allocation failed!",
+						__func__);
+				BUG();
+			}
+			of_property_read_u32_index(np, propname, i,
+					&reg->offset);
+			of_property_read_u32_index(np, propname, i + 1,
+					&reg->value);
+
+			list_add_tail(&reg->list,
+					&(*config)->list);
+		}
+	} else
+		*config = NULL;
+
+	return *config == NULL ? -EINVAL : 0;
+}
+
+static int win2030_noc_qos_create_register(struct device *_dev,
+		void __iomem *base, struct dev_qos_cfg *qos)
+{
+	struct win2030_register *reg;
+	struct win2030_bitfield *bf;
+
+	/**
+	 * Register: Priority
+	 */
+	reg = win2030_new_register(_dev, "Priority", base, GPU_QOS_GEN_PRIORITY, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "P0", GPU_QOS_GEN_PRIORITY_P0_OFFSET,
+		GPU_QOS_GEN_PRIORITY_P0_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "P1", GPU_QOS_GEN_PRIORITY_P1_OFFSET,
+		GPU_QOS_GEN_PRIORITY_P1_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	list_add_tail(&reg->parent_link, &qos->register_link);
+
+	/**
+	 * Register: Mode
+	 */
+	reg = win2030_new_register(_dev, "Mode", base, GPU_QOS_GEN_MODE, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "Mode", GPU_QOS_GEN_MODE_MODE_OFFSET,
+		GPU_QOS_GEN_MODE_MODE_WIDTH, win2030_noc_qos_mode_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+	list_add_tail(&reg->parent_link, &qos->register_link);
+
+	/**
+	 * Register: Bandwidth
+	 */
+	reg = win2030_new_register(_dev, "Bandwidth", base, GPU_QOS_GEN_BANDWIDTH, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "Bandwidth", GPU_QOS_GEN_BANDWIDTH_BANDWIDTH_OFFSET,
+		GPU_QOS_GEN_BANDWIDTH_BANDWIDTH_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+	list_add_tail(&reg->parent_link, &qos->register_link);
+
+	/**
+	 * Register: Saturation
+	 */
+	reg = win2030_new_register(_dev, "Saturation", base, GPU_QOS_GEN_SATURATION, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "Saturation", GPU_QOS_GEN_SATURATION_SATURATION_OFFSET,
+		GPU_QOS_GEN_SATURATION_SATURATION_WIDTH, NULL);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+	list_add_tail(&reg->parent_link, &qos->register_link);
+
+	/**
+	 * Register: ExtControl
+	 */
+	reg = win2030_new_register(_dev, "ExtControl", base, GPU_QOS_GEN_EXTCONTROL, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "SocketEn", GPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_OFFSET,
+		GPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_WIDTH, win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "ExtThrEn", GPU_QOS_GEN_EXTCONTROL_EXTTHREN_OFFSET,
+		GPU_QOS_GEN_EXTCONTROL_EXTTHREN_WIDTH, win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "IntClkEn", GPU_QOS_GEN_EXTCONTROL_INTCLKEN_OFFSET,
+		GPU_QOS_GEN_EXTCONTROL_INTCLKEN_WIDTH, win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+
+	bf = win2030_new_bitfield(reg, "ExtLimitEn", GPU_QOS_GEN_EXTCONTROL_EXTLIMITEN_OFFSET,
+		GPU_QOS_GEN_EXTCONTROL_EXTLIMITEN_WIDTH, win2030_noc_enable_enum);
+	if (IS_ERR_OR_NULL(bf))
+		return -EINVAL;
+	list_add_tail(&reg->parent_link, &qos->register_link);
+
+	return 0;
+}
+
+static int win2030_noc_parse_dts_qoslist(struct device *_dev)
+{
+	int def_len;
+	struct property *prop;
+	struct device_node *np = _dev->of_node;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	u32 qos_base;
+
+	INIT_LIST_HEAD(&noc_device->qos_list);
+	prop = of_find_property(np, "eswin,qos-configs", &def_len);
+	if (prop != NULL) {
+		char str[64] = "\0";
+		struct dev_qos_cfg *devqos;
+		int i, ncfg = of_property_count_strings(np, "eswin,qos-configs");
+
+		for (i = 0; i < ncfg; i++) {
+			int ret = 0;
+			devqos = (struct dev_qos_cfg *)devm_kzalloc(_dev, sizeof(struct dev_qos_cfg), GFP_KERNEL);
+			if (devqos == NULL) {
+				dev_err(_dev, "%s: Memory allocation failed!", __func__);
+				BUG();
+			}
+			INIT_LIST_HEAD(&devqos->register_link);
+			of_property_read_string_index(np, "eswin,qos-configs", i, &devqos->name);
+
+			/* test if controlled by noc driver */
+			sprintf(str, "eswin,%s-qos-owner", devqos->name);
+			if (of_property_read_bool(np, str))
+				devqos->noc_owner = 1;
+			else
+				devqos->noc_owner = 0;
+
+			sprintf(str, "eswin,%s-qos-base", devqos->name);
+			ret = of_property_read_u32(np, str, &qos_base);
+			if (ret) {
+				dev_info(_dev, "%s could not get reg base\n", str);
+				devm_kfree(_dev, devqos);
+				return ret;
+			} else {
+				devqos->hw_base = ioremap(qos_base, 0x100);
+				ret = win2030_noc_qos_create_register(_dev, devqos->hw_base, devqos);
+				if (ret) {
+					dev_info(_dev, "%s could not create qos register\n", str);
+					devm_kfree(_dev, devqos);
+					return ret;
+				}
+			}
+
+			sprintf(str, "eswin,%s-qos-settings", devqos->name);
+			ret = win2030_noc_parse_dts_qoscfg(_dev, str,
+					&devqos->config);
+			if (ret) {
+				dev_info(_dev, "%s not add to list\n", str);
+				devm_kfree(_dev, devqos);
+				return -EINVAL;
+			} else {
+				dev_info(_dev, "%s added to list(%d)\n",
+						str, devqos->noc_owner);
+				list_add_tail(&devqos->list, &noc_device->qos_list);
+			}
+		}
+	} else {
+		dev_info(_dev, "no QoS config list\n");
+	}
+	return 0;
+}
+#define 	NOC_DEV_NUM		10
+
+static struct win2030_noc_device *noc_dev_glob[NOC_DEV_NUM] = {NULL, NULL};
+static int noc_idev_glob;
+/*
+	notice: the "is_enalbe" flag only affect the NOC internal priority.
+	when is_enalbe == true, the NOC qos will be set according to the dts config.
+	when is_enalbe == false, the NOC qos will be fair, but the IP qos config will still pass to ddr controller.
+	If you want completely fair mode, set bypass mode in noc dts node and set the same qos priority in the IP config.
+*/
+void win2030_noc_qos_set(const char *name, bool is_enalbe)
+{
+	struct dev_qos_cfg *qos = NULL;
+	struct regcfg *reg = NULL;
+	int idev = 0;
+
+	for (idev = 0; idev < NOC_DEV_NUM; idev++) {
+		struct win2030_noc_device *noc_dev = noc_dev_glob[idev];
+
+		if (!noc_dev)
+			continue;
+		list_for_each_entry(qos, &noc_dev->qos_list, list) {
+			if (strcmp(qos->name, name) == 0) {
+				if (qos->config) {
+					dev_info(noc_dev->dev, "%s QoS config %s\n",
+						is_enalbe ? "Enable" : "Disable", qos->name);
+					list_for_each_entry(reg, &qos->config->list, list) {
+						iowrite32(reg->value, qos->hw_base + reg->offset);
+						/*when disable qos, set the mode register to bypass and disable socket*/
+						if (false == is_enalbe) {
+							if (0xC == reg->offset) {
+								iowrite32(0x2, qos->hw_base + reg->offset);
+							}
+							if (0x18 == reg->offset) {
+								iowrite32(0x0, qos->hw_base + reg->offset);
+							}
+						}
+					}
+				}
+				return;
+			}
+		}
+	}
+	pr_err("QoS config %s not found\n", name);
+}
+EXPORT_SYMBOL(win2030_noc_qos_set);
+
+int noc_device_qos_set(struct win2030_noc_device *noc_device, bool is_enalbe)
+{
+	struct dev_qos_cfg *qos = NULL;
+
+	if (!noc_device) {
+		dev_err(noc_device->dev, "%s NULL noc device\n", __func__);
+		return -ENXIO;
+	}
+	list_for_each_entry(qos, &noc_device->qos_list, list) {
+		if (qos->noc_owner) {
+			win2030_noc_qos_set(qos->name, is_enalbe);
+		}
+	}
+	return 0;
+}
+
+/**
+ * Parse dts to get number of probes and number of filters and counters per probe
+ * @param _dev
+ * @return
+ */
+static int win2030_noc_parse_dts(struct device *_dev)
+{
+	struct device_node *np = _dev->of_node;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+
+	noc_device->trap_on_error = of_property_read_bool(np, "errors,trap");
+
+	dev_info(_dev, "error policy: %s\n", noc_device->trap_on_error ? "trap" : "print");
+
+	win2030_noc_parse_dts_qoslist(_dev);
+	noc_dev_glob[noc_idev_glob] = noc_device;
+	noc_idev_glob++;
+	return 0;
+}
+
+static int win2030_noc_register_probe_common(struct device_node *np,
+	struct device *_dev, struct win2030_noc_probe **_probe)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct win2030_noc_counter *counters;
+	int ret, j;
+	struct win2030_noc_probe *probe;
+
+	/* Initialise probes */
+	probe = devm_kcalloc(_dev, 1, sizeof(struct win2030_noc_probe), GFP_KERNEL);
+	probe->hw_base = of_iomap(np, 0);
+	if (IS_ERR_OR_NULL(probe->hw_base)) {
+		return -EINVAL;
+	}
+	INIT_LIST_HEAD(&probe->register_link);
+
+	probe->dev = _dev;
+	probe->id = np->name;
+
+	probe->clock = of_clk_get_by_name(np, "clk");
+	if (IS_ERR_OR_NULL(probe->clock)) {
+		dev_dbg(probe->dev, "Error %px while getting clock of probe %s\n",
+			probe->clock, probe->id);
+		return -ENXIO;
+	}
+
+	if (!IS_ERR_OR_NULL(probe->clock)) {
+		ret = clk_prepare_enable(probe->clock);
+		if (ret) {
+			dev_err(probe->dev, "Error %d while enabling clock of probe %s\n",
+				ret, probe->id);
+			return ret;
+		}
+	}
+	if (!IS_ERR_OR_NULL(probe->clock))
+		dev_dbg(probe->dev, "Clock of probe %s rate %lu\n",
+				probe->id, clk_get_rate(probe->clock));
+
+	probe->stat_irq = of_irq_get_byname(np, "stat");
+	if (probe->stat_irq < 0) {
+		dev_err(_dev, "Error %d while extracting stat alarm irq\n",
+			(int)probe->stat_irq);
+		ret = probe->stat_irq;
+		return ret;
+
+	}
+	ret = devm_request_irq(_dev, probe->stat_irq, win2030_noc_stat_irq,
+			IRQF_SHARED, "noc stat", probe);
+	if (ret) {
+		dev_err(_dev, "Error while installing stat irq %d\n",
+			(int)probe->stat_irq);
+		return ret;
+	}
+	list_add_tail(&probe->link, &noc_device->probes);
+
+	/*initialize register*/
+	ret = win2030_noc_init_probe(noc_device, probe, np);
+	if (ret)
+		return -EINVAL;
+
+	/* initialize counters */
+	ret = of_property_read_u32(np, "counter,nr", &probe->nr_counters);
+	if (ret) {
+		dev_err(_dev, "\"counter,nr\" property missing");
+		return -EINVAL;
+	}
+	counters = devm_kcalloc(_dev, probe->nr_counters,
+			sizeof(struct win2030_noc_counter), GFP_KERNEL);
+	if (!counters)
+		return -ENOMEM;
+
+	probe->counters = counters;
+	for (j = 0; j < probe->nr_counters; j++) {
+		ret = win2030_noc_init_counter(&counters[j], probe, j);
+		if (ret)
+			return ret;
+	}
+	*_probe = probe;
+	return 0;
+}
+
+static int win2030_noc_register_packet_probe(struct device_node *np,
+		struct device *_dev)
+{
+	int ret, j;
+	struct win2030_noc_filter *filters;
+	struct win2030_noc_probe *probe;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+
+	ret = win2030_noc_register_probe_common(np, _dev, &probe);
+	if (ret) {
+		return ret;
+	}
+	probe->type = probe_t_pkt;
+
+	/* Initialise filters */
+	ret = of_property_read_u32(np, "filter,nr", &probe->nr_filters);
+	if (ret) {
+		dev_err(_dev, "\"filter,nr\" property missing");
+		return -EINVAL;
+	}
+	filters = devm_kcalloc(_dev, probe->nr_filters,
+			sizeof(struct win2030_noc_filter), GFP_KERNEL);
+	if (!filters)
+		return -ENOMEM;
+
+	for (j = 0; j < probe->nr_filters; j++) {
+		ret = win2030_noc_init_filter(noc_device, &filters[j],
+					probe, j);
+		if (ret)
+			return ret;
+	}
+	probe->filters = filters;
+	return 0;
+}
+
+static int win2030_noc_register_trans_probe(struct device_node *np,
+			struct device *_dev)
+{
+	int ret, i;
+	struct win2030_noc_trans_filter *filters;
+	struct win2030_noc_trans_profiler *profilers;
+	struct win2030_noc_probe *probe;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct device_node *child = NULL;
+	void __iomem *hw_base;
+
+	ret = win2030_noc_register_probe_common(np, _dev, &probe);
+	if (0 != ret) {
+		return ret;
+	}
+	probe->type = probe_t_trans;
+
+	/* Initialise filters */
+	i = 0;
+	ret = of_property_read_u32(np, "filter,nr", &probe->nr_filters);
+	if (ret) {
+		dev_err(_dev, "\"filter,nr\" property missing");
+		return -EINVAL;
+	}
+	filters = devm_kcalloc(_dev, probe->nr_filters,
+			sizeof(struct win2030_noc_trans_filter), GFP_KERNEL);
+	if (!filters)
+		return -ENOMEM;
+
+	for_each_child_of_node(np, child) {
+		if (of_device_is_compatible(child, "eswin,win2xxx-noc-trans-filter")) {
+			if (!of_device_is_available(child)) {
+				probe->nr_filters--;
+				continue;
+			}
+			hw_base = of_iomap(child, 0);
+			if (IS_ERR_OR_NULL(hw_base)) {
+				return -EINVAL;
+			}
+			ret = win2030_noc_init_trans_filter(noc_device,
+					&filters[i],
+					hw_base,
+					probe, i);
+			if (ret)
+				return ret;
+			i++;
+		}
+	}
+	probe->trans_filters = filters;
+
+	/* Initialise profilers */
+	i = 0;
+	ret = of_property_read_u32(np, "profiler,nr", &probe->nr_profilers);
+	if (ret) {
+		dev_err(_dev, "\"profiler,nr\" property missing");
+		return -EINVAL;
+	}
+	profilers = devm_kcalloc(_dev, probe->nr_profilers,
+			sizeof(struct win2030_noc_trans_profiler), GFP_KERNEL);
+	if (!profilers)
+		return -ENOMEM;
+
+	for_each_child_of_node(np, child) {
+		if (of_device_is_compatible(child, "eswin,win2xxx-noc-trans-profiler")) {
+			hw_base = of_iomap(child, 0);
+			if (IS_ERR_OR_NULL(hw_base)) {
+				return -EINVAL;
+			}
+			ret = win2030_noc_init_trans_profiler(noc_device,
+					&profilers[i],
+					hw_base,
+					probe, i);
+			if (ret)
+				return ret;
+			i++;
+		}
+	}
+	probe->trans_profilers = profilers;
+
+	return 0;
+}
+
+/**
+ * Create probes, filters and counters registers
+ * @param _dev
+ * @return
+ */
+static int win2030_noc_init_device(struct device *_dev)
+{
+	int ret;
+	struct device_node *child = NULL;
+
+	for_each_child_of_node(_dev->of_node, child) {
+		if (of_device_is_compatible(child, "eswin,win2xxx-noc-packet-probe")) {
+			ret = win2030_noc_register_packet_probe(child, _dev);
+			if (0 != ret) {
+				dev_err(_dev, "Error while register packet probe\n");
+				return ret;
+			}
+		}
+	}
+
+	for_each_child_of_node(_dev->of_node, child) {
+		if (of_device_is_compatible(child, "eswin,win2xxx-noc-trans-probe")) {
+			ret = win2030_noc_register_trans_probe(child, _dev);
+			if (0 != ret) {
+				dev_err(_dev, "Error while register trans probe\n");
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
+static int win2030_noc_probe(struct platform_device *pdev)
+{
+	struct device *mydev = &pdev->dev;
+	struct device_node *np = pdev->dev.of_node;
+	struct win2030_noc_device *noc_device;
+	int ret;
+	struct resource *res;
+
+	dev_info(mydev, "Begin initialization\n");
+	noc_device = devm_kcalloc(mydev, 1,
+		sizeof(struct win2030_noc_device), GFP_KERNEL);
+	if (!noc_device)
+		return -ENOMEM;
+
+	noc_device->dev = mydev;
+	dev_set_drvdata(mydev, noc_device);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(mydev, "Error while get mem resource\n");
+		return -ENODEV;
+	}
+	noc_device->hw_base = devm_ioremap_resource(&pdev->dev, res);;
+	if (IS_ERR_OR_NULL(noc_device->hw_base)) {
+		dev_err(mydev, "Error while mapping %s reg base 0x%llx!\n",
+			np->name, res->start);
+		ret = -EINVAL;
+		goto free_noc_device;
+	}
+	INIT_LIST_HEAD(&noc_device->err_queue);
+	INIT_LIST_HEAD(&noc_device->registers);
+	INIT_LIST_HEAD(&noc_device->probes);
+	spin_lock_init(&noc_device->lock);
+
+	ret = win2030_get_err_registers(mydev, &noc_device->registers);
+	if (ret) {
+		dev_err(mydev, "Error while gettings registers\n");
+		goto put_clock;
+	}
+
+	/* Get these parameters from the dts */
+	ret = win2030_noc_parse_dts(mydev);
+	if (ret) {
+		dev_err(mydev, "Parsing Dts failed\n");
+		goto put_clock;
+	}
+
+	ret = win2030_noc_init_device(mydev);
+	if (ret) {
+		dev_err(mydev, "Failed to init internal NoC modelling\n");
+		goto put_clock;
+	}
+
+	ret = win2030_noc_error_init(mydev);
+	if (ret) {
+		dev_err(mydev, "Initialisation of error handling failed\n");
+		goto put_clock;
+	}
+
+	if (IS_ERR_OR_NULL(win2030_noc_ctrl.win2030_noc_root_debug_dir)) {
+		win2030_noc_ctrl.win2030_noc_root_debug_dir = debugfs_create_dir("noc", NULL);
+		if (IS_ERR_OR_NULL(win2030_noc_ctrl.win2030_noc_root_debug_dir)) {
+			ret = -ENOENT;
+			goto free_noc_device;
+		}
+		INIT_LIST_HEAD(&win2030_noc_ctrl.sbm_link);
+	}
+	ret = win2030_noc_init_sideband_mgr(mydev);
+	if (ret) {
+		dev_err(mydev, "Error while gettings registers\n");
+		goto put_clock;
+	}
+
+	ret = win2030_noc_stat_init(mydev);
+	if (ret)
+		dev_err(mydev, "No NOC sniffer or NOC sniffer init failure\n");
+	else {
+		ret = win2030_noc_stat_debugfs_init(mydev);
+		if (ret)
+			dev_err(mydev,
+				"Initialisation of NOC sniffer sysfs failed\n");
+	}
+	ret = win2030_noc_prof_init(mydev);
+	if (ret)
+		dev_err(mydev, "No NOC profiler or NOC profiler init failure\n");
+	else {
+		ret = win2030_noc_prof_debugfs_init(mydev);
+		if (ret)
+			dev_err(mydev,
+				"Initialisation of NOC profiler sysfs failed\n");
+	}
+	ret = win2030_noc_debug_init(mydev);
+	if (0 != ret) {
+		dev_err(mydev, "Error %d while init debug\n", ret);
+		return ret;
+	}
+	dev_info(mydev, "initialization sucess\n");
+
+	return 0;
+put_clock:
+free_noc_device:
+	return ret;
+}
+
+static int win2030_noc_remove(struct platform_device *pdev)
+{
+	struct device *mydev = &pdev->dev;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(mydev);
+	struct win2030_noc_error *noc_err = NULL, *tmp_err = NULL;
+	struct win2030_register *reg = NULL, *tmp_reg = NULL;
+	struct win2030_noc_probe *probe = NULL;
+	struct win2030_noc_stat_measure *stat_measure, *tmp_stat_measure = NULL;
+	struct win2030_noc_stat_probe *stat_probe = NULL, *tmp_stat_probe = NULL;
+	struct win2030_noc_prof_probe *prof_probe = NULL, *tmp_prof_probe = NULL;
+	struct win2030_noc_prof_measure *prof_measure, *tmp_prof_measure = NULL;
+
+	dev_info(mydev, "Removing %s\n", dev_name(mydev));
+
+	list_for_each_entry_safe(stat_probe, tmp_stat_probe,
+				 &noc_device->stat->probe, link) {
+		list_for_each_entry_safe(stat_measure, tmp_stat_measure,
+					 &stat_probe->measure, link) {
+			list_del(&stat_measure->link);
+			kfree(stat_measure);
+		}
+		list_del(&stat_probe->link);
+		kfree(stat_probe);
+	}
+
+	list_for_each_entry_safe(prof_probe, tmp_prof_probe,
+				 &noc_device->prof->prof_probe, link) {
+		list_for_each_entry_safe(prof_measure, tmp_prof_measure,
+					 &prof_probe->measure, link) {
+			list_del(&prof_measure->link);
+			kfree(prof_measure);
+		}
+		list_del(&prof_probe->link);
+		kfree(prof_probe);
+	}
+
+	list_for_each_entry_safe(noc_err, tmp_err,
+				&noc_device->err_queue, link) {
+		list_del(&noc_err->link);
+		kfree(noc_err);
+	}
+
+	list_for_each_entry_safe(reg, tmp_reg, &noc_device->registers, link) {
+		list_del(&reg->link);
+		win2030_free_register(reg);
+	}
+
+	list_for_each_entry(probe, &noc_device->probes, link) {
+		win2030_noc_remove_probe(probe);
+	}
+
+	kfree(noc_device->stat);
+	kfree(noc_device->prof);
+	return 0;
+}
+#ifdef CONFIG_PM
+static int noc_resume(struct device *dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(dev);
+	void __iomem *base = noc_device->hw_base;
+
+	noc_device_qos_set(noc_device, true);
+
+	while (ioread32(ERRLOG_0_ERRVLD(base))) {
+		dev_err(dev, "NoC Error pending during resume\n");
+		win2030_noc_get_error(noc_device);
+	}
+
+	iowrite32(1, ERRLOG_0_FAULTEN(base));
+
+	return 0;
+}
+static int noc_suspend(struct device *dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(dev);
+	void __iomem *base = noc_device->hw_base;
+
+	iowrite32(0, ERRLOG_0_FAULTEN(base));
+
+	while (ioread32(ERRLOG_0_ERRVLD(base))) {
+		dev_err(dev, "NoC Error pending during suspend\n");
+		win2030_noc_get_error(noc_device);
+	}
+
+	return 0;
+}
+static const struct dev_pm_ops win2030_noc_driver_pm_ops = {
+
+	.suspend = noc_suspend,
+	.resume = noc_resume,
+
+};
+#endif
+
+static struct platform_driver win2030_noc_driver = {
+	.probe = win2030_noc_probe,
+	.remove = win2030_noc_remove,
+	.driver = {
+		.name = "win2030-noc",
+		.owner = THIS_MODULE,
+#ifdef CONFIG_PM
+		.pm = &win2030_noc_driver_pm_ops,
+#endif
+		.of_match_table = of_match_ptr(win2030_noc_of_match),},
+};
+
+static void __exit win2030_noc_exit(void)
+{
+	platform_driver_unregister(&win2030_noc_driver);
+}
+
+module_platform_driver(win2030_noc_driver);
+module_exit(win2030_noc_exit);
+
+MODULE_AUTHOR("huangyifeng<huangyifeng@eswincomputing.com>");
+MODULE_VERSION("1.0.0");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/eswin/noc.h b/drivers/interconnect/eswin/noc.h
new file mode 100644
index 000000000000..92a86788b0d0
--- /dev/null
+++ b/drivers/interconnect/eswin/noc.h
@@ -0,0 +1,363 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef NOC_H_
+#define NOC_H_
+
+#define WIN2030_NOC_ERROR_REGISTERS_MAX 8
+#define WIN2030_NOC_REG_ADDR(_reg)	((void __iomem *) (reg->base + reg->offset))
+#define MAX_ITERATION		(1 << 28)	/* Arbitrary value */
+#define DURATION		30
+#define WIN2030_NOC_TRACE_PORT_MAX	3
+#define WIN2030_NOC_BIN_CNT	4
+#define SIZE_BIG_BUF	4096
+#define SIZE_SMALL_BUF	1500
+
+extern struct win2030_noc_control win2030_noc_ctrl;
+extern int latency_bin[3];
+
+struct win2030_bitfield {
+	struct win2030_register *parent;
+	const char *name;
+	const char *description;
+	unsigned offset;
+	unsigned length;
+	unsigned mask;
+	const char **lut;
+	int aperture_size;
+	u64 *init_flow;
+	u64 *target_flow;
+	u64 *target_sub_range;
+	u16 *aperture_idx;
+	u64 *aperture_base;
+	u32 sbm_id;
+	struct list_head link;
+	spinlock_t lock;
+};
+
+struct win2030_register {
+	struct device *parent;
+	void __iomem *base;
+	const char *name;
+	const char *description;
+	unsigned offset;
+	unsigned length;
+	int aperture_link;
+	int msb_link;
+	struct list_head bitfields;
+	struct list_head link;
+	struct list_head parent_link;	//link register's parent(probe, filter or counter)
+	spinlock_t lock;
+	spinlock_t hw_lock;
+};
+
+struct win2030_noc_probe;
+
+struct win2030_noc_stat_traceport_data {
+	const char *name;
+	u16 idx_trace_port_sel;
+	char *init_flow_name;
+	u8 init_flow_idx;
+	unsigned init_flow_offset;
+	unsigned init_flow_mask;
+	char *target_flow_name;
+	u8 target_flow_idx;
+	unsigned target_flow_offset;
+	unsigned target_flow_mask;
+	int addrBase_low;
+	int addrBase_high;
+	int addrWindowSize;
+	int Opcode;
+	int Status;
+	int Length;
+	int Urgency;
+};
+
+/* One measure for the NOC sniffer */
+struct win2030_noc_stat_measure {
+	u64 min[WIN2030_NOC_TRACE_PORT_MAX];
+	u64 max[WIN2030_NOC_TRACE_PORT_MAX];
+	u64 mean[WIN2030_NOC_TRACE_PORT_MAX];
+	u64 now[WIN2030_NOC_TRACE_PORT_MAX];
+	unsigned iteration[WIN2030_NOC_TRACE_PORT_MAX];
+	struct list_head link;
+	spinlock_t lock;
+	struct win2030_noc_stat_traceport_data data;
+};
+
+/* A set of statistics assigned to a probe
+ * (as a probe can measure severals paths)
+ */
+struct win2030_noc_stat_probe {
+	struct win2030_noc_probe *probe;
+	struct list_head measure;
+	struct list_head link;
+	spinlock_t lock;
+};
+
+/* The main structure for the NOC sniffer */
+struct win2030_noc_stat {
+	struct list_head probe;
+	struct dentry *dir;
+	spinlock_t lock;
+	bool run;
+	unsigned clock_rate;
+};
+
+struct win2030_noc_filter {
+	unsigned id;
+	struct win2030_noc_probe *parent;
+	struct win2030_register *route_id_base;
+	struct win2030_bitfield *route_id_mask;
+	struct win2030_bitfield *addr_base_low;
+	struct win2030_bitfield *addr_base_high;
+	struct win2030_bitfield *window_size;
+	//struct win2030_register *security_base;
+	//struct win2030_bitfield *security_mask;
+	struct win2030_register *op_code;
+	struct win2030_register *status;
+	struct win2030_bitfield *length;
+	struct win2030_bitfield *urgency;
+	struct list_head register_link;
+};
+
+struct win2030_noc_trans_filter {
+	unsigned id;
+	struct win2030_noc_probe *parent;
+	struct win2030_bitfield *mode;
+	struct win2030_bitfield *addr_base_low;
+	struct win2030_bitfield *addr_base_high;
+	struct win2030_bitfield *window_size;
+	struct win2030_register *op_code;
+	struct win2030_bitfield *user_base;
+	struct win2030_bitfield *user_mask;
+	struct list_head register_link;
+};
+
+typedef enum {
+	latency_t,
+	pending_t,
+}measure_type;
+
+struct win2030_noc_prof_traceport_data {
+	const char *name;
+	int mode;
+	int addrBase_low;
+	int addrBase_high;
+	int addrWindowSize;
+	int Opcode;
+};
+
+/* One measure for the NOC prof */
+struct win2030_noc_prof_measure {
+	measure_type type;
+	u64 min[WIN2030_NOC_TRACE_PORT_MAX][WIN2030_NOC_BIN_CNT];
+	u64 max[WIN2030_NOC_TRACE_PORT_MAX][WIN2030_NOC_BIN_CNT];
+	u64 mean[WIN2030_NOC_TRACE_PORT_MAX][WIN2030_NOC_BIN_CNT];
+	u64 now[WIN2030_NOC_TRACE_PORT_MAX][WIN2030_NOC_BIN_CNT];
+	unsigned iteration[WIN2030_NOC_TRACE_PORT_MAX];
+	struct list_head link;
+	spinlock_t lock;
+	struct win2030_noc_prof_traceport_data data;
+};
+
+/* A set of statistics assigned to a probe
+ * (as a probe can measure severals paths)
+ */
+struct win2030_noc_prof_probe {
+	struct win2030_noc_probe *probe;
+	struct list_head measure;
+	struct list_head link;
+	spinlock_t lock;
+};
+
+/* The main structure for the NOC profiler */
+struct win2030_noc_prof {
+	struct list_head prof_probe;
+	struct dentry *dir;
+	spinlock_t lock;
+	bool run;
+};
+
+struct win2030_noc_trans_profiler {
+	unsigned id;
+	struct win2030_noc_probe *parent;
+	struct win2030_bitfield *en;
+	struct win2030_bitfield *mode;
+	struct win2030_bitfield *observed_sel[WIN2030_NOC_TRACE_PORT_MAX];
+	struct win2030_bitfield *n_tenure_lines[WIN2030_NOC_TRACE_PORT_MAX];
+	struct win2030_bitfield *thresholds[WIN2030_NOC_TRACE_PORT_MAX][WIN2030_NOC_BIN_CNT];
+	struct win2030_bitfield *over_flow_status;
+	struct win2030_bitfield *over_flow_reset;
+	struct win2030_bitfield *pending_event_mode;
+	struct win2030_bitfield *pre_scaler;
+	struct list_head register_link;
+};
+
+struct win2030_noc_counter {
+	unsigned id;
+	struct win2030_noc_probe *parent;
+	struct win2030_bitfield *portsel;
+	struct win2030_bitfield *alarm_mode;
+	struct win2030_bitfield *source_event;
+	struct win2030_bitfield *value;
+	struct list_head register_link;
+};
+
+enum probe_t {
+	probe_t_pkt = 0x8,
+	probe_t_trans,
+};
+
+struct win2030_noc_probe {
+	struct device *dev;
+	const char *id;
+	struct clk *clock;
+	enum probe_t type;
+	int nr_portsel;				/* How many trace port per probe */
+	const char **available_portsel;
+	struct win2030_register *main_ctl;
+	struct win2030_register *cfg_ctl;
+	//struct win2030_register *trace_port_sel;
+	struct win2030_register *filter_lut;
+	struct win2030_register *stat_period;
+	struct win2030_register *stat_go;
+	struct win2030_register *stat_alarm_max;
+	struct win2030_register *stat_alarm_min;
+	struct win2030_register *stat_alarm_min_high;
+	struct win2030_register *stat_alarm_status;
+	struct win2030_register *stat_alarm_en;
+	struct list_head register_link;
+	struct win2030_noc_device *parent;
+	unsigned nr_counters;				/* How many counters per probe */
+	struct win2030_noc_counter *counters;
+	unsigned nr_filters;				/* How many filters per probe */
+	struct win2030_noc_filter *filters;
+	struct win2030_noc_trans_filter *trans_filters;
+	unsigned nr_profilers;				/* How many profiler per probe */
+	struct win2030_noc_trans_profiler *trans_profilers;
+	struct tasklet_struct tasklet;
+	struct list_head link;
+	int 	stat_irq;
+	void __iomem *hw_base;
+};
+
+struct dev_qos_cfg {
+	struct list_head list;
+	const char *name;
+	void __iomem *hw_base;
+	struct regcfg *config;
+	int noc_owner;
+	bool run;
+	struct list_head register_link;
+};
+
+struct regcfg {
+	struct list_head list;
+	unsigned offset;
+	unsigned value;
+};
+
+struct win2030_noc_device {
+	struct device *dev;
+	void __iomem *hw_base;
+	int err_irq;
+	struct list_head err_queue;
+	spinlock_t lock;
+	struct list_head registers;
+	int error_logger_cnt;
+	u32 *err_log_lut;
+	struct win2030_register *error_registers[WIN2030_NOC_ERROR_REGISTERS_MAX];
+	struct dentry *dir;
+	struct win2030_noc_stat *stat;
+	struct win2030_noc_prof *prof;
+	struct list_head qos_list;
+	int qos_run;
+	bool trap_on_error;
+	struct list_head probes;
+	struct win2030_noc_sideband_magr *sbm;
+};
+
+struct win2030_noc_error {
+	struct list_head link;
+	u64 timestamp;
+	unsigned err[WIN2030_NOC_ERROR_REGISTERS_MAX];
+};
+
+struct win2030_noc_sideband_magr {
+	struct device 		*dev;
+	void __iomem 		*hw_base;
+	struct win2030_register *SenseIn0;
+	struct list_head 	link;
+};
+
+struct win2030_noc_control {
+	struct dentry *win2030_noc_root_debug_dir;
+	struct list_head sbm_link;
+};
+
+int win2030_noc_debug_init(struct device *);
+int win2030_noc_stat_init(struct device *);
+int win2030_noc_prof_init(struct device *_dev);
+
+void win2030_noc_stat_do_measure(struct win2030_noc_probe *);
+void win2030_noc_prof_do_measure(struct win2030_noc_probe *probe);
+int win2030_noc_stat_debugfs_init(struct device *);
+int win2030_noc_prof_debugfs_init(struct device *_dev);
+int win2030_noc_stat_trigger(struct device *_dev);
+void win2030_noc_stat_reset_statistics(struct device *_dev);
+int win2030_noc_prof_trigger(struct device *_dev);
+void win2030_noc_prof_reset_statistics(struct device *_dev);
+
+
+unsigned win2030_noc_reg_read(struct win2030_register *reg, unsigned mask,
+			unsigned offset);
+int win2030_noc_reg_write(struct win2030_register *reg, unsigned mask,
+			unsigned offset, unsigned value);
+int win2030_bitfield_read(struct win2030_bitfield *bf, unsigned *value);
+int win2030_noc_init_sideband_mgr(struct device *_dev);
+int win2030_noc_sideband_mgr_debug_init(struct device *_dev);
+int win2030_noc_debug_reg_init(struct win2030_register *reg, struct dentry *dir);
+
+struct win2030_register *win2030_new_register(struct device *parent,
+			const char *name,
+			void __iomem *base,
+			unsigned offset,
+			unsigned width);
+
+const char **win2030_get_strings_from_dts(struct device *_dev,
+			struct device_node *np,
+			const char *property_name);
+
+struct win2030_bitfield *win2030_new_bitfield(struct win2030_register *reg,
+			 const char *name,
+			 unsigned offset,
+			 unsigned width,
+			 const char **lut);
+
+int noc_error_dump(char *buf,
+		struct win2030_noc_device *noc_device,
+		struct win2030_noc_error *noc_err);
+
+int noc_device_qos_set(struct win2030_noc_device *noc_device, bool is_enalbe);
+
+#endif /* NOC_H_ */
diff --git a/drivers/interconnect/eswin/noc_profiler.c b/drivers/interconnect/eswin/noc_profiler.c
new file mode 100644
index 000000000000..c6621ba4155c
--- /dev/null
+++ b/drivers/interconnect/eswin/noc_profiler.c
@@ -0,0 +1,695 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/of.h>
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/bitfield.h>
+
+#include "noc.h"
+#include "noc_regs.h"
+
+void zebu_stop(void);
+
+int latency_bin[3] = {256, 320, 384};
+
+/**
+ * Configure NOC registers from a prof_measure
+ * @param noc_device A reference to the device
+ * @param prof_probe The prof_probe to find the measure
+ * @return 0 if ok, -EINVAL otherwise
+ */
+static int win2030_noc_prof_launch_measure(struct win2030_noc_device *noc_device,
+				     struct win2030_noc_prof_probe *prof_probe)
+{
+	struct win2030_noc_prof_measure *prof_measure;
+	struct win2030_noc_probe *probe;
+	int i, ret = 0;
+	struct win2030_noc_trans_profiler *profiler;
+	struct win2030_noc_prof_traceport_data *data;
+
+	prof_measure = list_first_entry(&prof_probe->measure,
+				struct win2030_noc_prof_measure, link);
+
+	data = &prof_measure->data;
+	dev_dbg_once(noc_device->dev, "measurment %s begin, type %s\n", data->name,
+		pending_t == prof_measure->type ? "pending" : "latency");
+
+	//zebu_stop();
+	probe = prof_probe->probe;
+
+	profiler = &probe->trans_profilers[0];
+
+	/*
+	  Disable probe
+	  This will reset all fileds of register MailCtl
+	*/
+	ret = win2030_noc_reg_write(probe->cfg_ctl, PROBE_CFGCTL_GLOBALEN_MASK,
+				  PROBE_CFGCTL_GLOBALEN_OFFSET, 0);
+	while (win2030_noc_reg_read(probe->cfg_ctl, PROBE_CFGCTL_ACTIVE_WIDTH,
+				  PROBE_CFGCTL_ACTIVE_OFFSET) != 0) {
+		;
+	}
+	/* --> to program the transaction filter,
+	*/
+	for (i = 0; i < probe->nr_filters; i++) {
+		ret |= win2030_noc_reg_write(probe->trans_filters[i].mode->parent,
+			PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_MASK,
+			PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_OFFSET,
+			data->mode);
+
+		ret |= win2030_noc_reg_write(probe->trans_filters[i].addr_base_low->parent,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_MASK,
+			PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET,
+			data->addrBase_low);
+
+		ret |= win2030_noc_reg_write(probe->trans_filters[i].addr_base_high->parent,
+			PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_MASK,
+			PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_OFFSET,
+			data->addrBase_high);
+
+		ret |=
+			win2030_noc_reg_write(probe->trans_filters[i].window_size->parent,
+			PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_MASK,
+			PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_OFFSET,
+			data->addrWindowSize);
+
+		ret |= win2030_noc_reg_write(probe->trans_filters[i].op_code,
+			PROBE_TRANS_FILTERS_OPCODE_RDEN_MASK,
+			PROBE_TRANS_FILTERS_OPCODE_RDEN_OFFSET,
+			FIELD_GET(GENMASK(0, 0), data->Opcode));
+
+		ret |= win2030_noc_reg_write(probe->trans_filters[i].op_code,
+			PROBE_TRANS_FILTERS_OPCODE_WREN_MASK,
+			PROBE_TRANS_FILTERS_OPCODE_WREN_OFFSET,
+			FIELD_GET(GENMASK(1, 1), data->Opcode));
+
+	}
+
+	/*--> to program the transaction profiler unit*/
+	/* Disable profiler */
+	ret |= win2030_noc_reg_write(profiler->en->parent,
+			PROBE_TRANS_PROFILER_EN_PROFILER_EN_MASK,
+			PROBE_TRANS_PROFILER_EN_PROFILER_EN_OFFSET,
+			0);
+
+	/* Select TracePort*/
+	for (i = 0; i < probe->nr_portsel; i++) {
+		ret |= win2030_noc_reg_write(profiler->observed_sel[i]->parent,
+			PROBE_TRACEPORTSEL_TRACEPORTSEL_MASK,
+			PROBE_TRACEPORTSEL_TRACEPORTSEL_OFFSET,
+			i);
+	}
+	if (pending_t == prof_measure->type) {
+		/*Set Profiler Mode to Pending*/
+		ret |= win2030_noc_reg_write(profiler->mode->parent,
+			PROBE_TRACEPORTSEL_TRACEPORTSEL_MASK,
+			PROBE_TRACEPORTSEL_TRACEPORTSEL_OFFSET,
+			1);
+	} else {
+		/*Set Profiler Mode to Delay*/
+		ret |= win2030_noc_reg_write(profiler->mode->parent,
+				PROBE_TRACEPORTSEL_TRACEPORTSEL_MASK,
+				PROBE_TRACEPORTSEL_TRACEPORTSEL_OFFSET,
+				0);
+	}
+
+	/* Each line has 16 counters*/
+	for (i = 0; i < probe->nr_portsel - 1; i++) {
+		ret |= win2030_noc_reg_write(profiler->n_tenure_lines[i]->parent,
+			PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_MASK,
+			PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_OFFSET,
+			2);
+	}
+	if (pending_t == prof_measure->type) {
+		/* Thresholds*/
+		for (i = 0; i < probe->nr_portsel; i++) {
+			/*
+			   for read/wrtie transration, the pending binis 2/5/10/10 up cycles
+			*/
+			ret |= win2030_noc_reg_write(profiler->thresholds[i][0]->parent,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					2);
+			ret |= win2030_noc_reg_write(profiler->thresholds[i][1]->parent,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					5);
+			ret |= win2030_noc_reg_write(profiler->thresholds[i][2]->parent,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					10);
+		}
+	} else {
+		/* Thresholds*/
+		for (i = 0; i < probe->nr_portsel; i++) {
+			/*
+			   for read/wrtie transration, the delay bin is 30/100/200/200 up cycles
+			*/
+			ret |= win2030_noc_reg_write(profiler->thresholds[i][0]->parent,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					latency_bin[0]);
+			ret |= win2030_noc_reg_write(profiler->thresholds[i][1]->parent,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					latency_bin[1]);
+			ret |= win2030_noc_reg_write(profiler->thresholds[i][2]->parent,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK,
+					PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET,
+					latency_bin[2]);
+		}
+	}
+
+	/*--> to program the packet probe statistics subsystemt*/
+	for (i = 0; i < probe->nr_counters; i++) {
+		/*maping counters to count profiler outputs*/
+		ret |= win2030_noc_reg_write(probe->counters[i].source_event->parent,
+			PROBE_COUNTERS_SRC_INTEVENT_MASK,
+			PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+			0x20 + i);
+
+		/*set counters alarm mode to Min,
+		  If the value of the counter is less than the StatAlarmMin register at the
+		  dump period, the StatAlarmStatus bit is set
+		*/
+		ret |= win2030_noc_reg_write(probe->counters[i].alarm_mode->parent,
+			PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_MASK,
+			PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_OFFSET,
+			1);
+	}
+
+	/* Set alarm min value */
+	ret |= win2030_noc_reg_write(probe->stat_alarm_min,
+			PROBE_STATALARMMIN_STATALARMMIN_MASK,
+			PROBE_STATALARMMIN_STATALARMMIN_OFFSET,
+			0xfffffff);
+
+	ret |= win2030_noc_reg_write(probe->stat_alarm_min_high,
+			PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_MASK,
+			PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_OFFSET,
+			0xfffffff);
+
+	/* Set period to max value */
+	ret |= win2030_noc_reg_write(probe->stat_period,
+			PROBE_STATPERIOD_STATPERIOD_MASK,
+			PROBE_STATPERIOD_STATPERIOD_OFFSET,
+			DURATION);
+
+	/* Enable Alarm */
+	ret |= win2030_noc_reg_write(probe->main_ctl,
+			PROBE_MAINCTL_ALARMEN_MASK,
+			PROBE_MAINCTL_ALARMEN_OFFSET,
+			1);
+
+	/* Set field StatEn of register MainCtl to 1 to enable the statistics counters to count delay events.*/
+	ret |= win2030_noc_reg_write(probe->main_ctl,
+			PROBE_MAINCTL_STATEN_MASK,
+			PROBE_MAINCTL_STATEN_OFFSET,
+			1);
+
+	/*--> to enable the transaction probe*/
+	/* Set field GlobalEn of register CfgCtl to 1 to enable the statistics counters. */
+	ret |= win2030_noc_reg_write(probe->cfg_ctl,
+			   PROBE_CFGCTL_GLOBALEN_MASK,
+			   PROBE_CFGCTL_GLOBALEN_OFFSET,
+			   1);
+
+	/* Set register En to 1 to enable the transaction profiling counters.. */
+	ret |= win2030_noc_reg_write(profiler->en->parent,
+			PROBE_TRANS_PROFILER_EN_PROFILER_EN_MASK,
+			PROBE_TRANS_PROFILER_EN_PROFILER_EN_OFFSET,
+			1);
+	return ret;
+}
+
+void win2030_noc_prof_reset_statistics(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct win2030_noc_prof_probe *prof_probe = NULL;
+	struct win2030_noc_probe *noc_probe = NULL;
+	struct win2030_noc_prof_measure *prof_measure = NULL;
+	struct win2030_noc_prof *noc_prof = noc_device->prof;
+	int i, j;
+	unsigned long flags;
+
+	list_for_each_entry(prof_probe, &noc_prof->prof_probe, link) {
+		noc_probe = prof_probe->probe;
+		list_for_each_entry(prof_measure, &prof_probe->measure, link) {
+			for (j = 0; j < noc_probe->nr_portsel; j++) {
+				spin_lock_irqsave(&prof_measure->lock, flags);
+				prof_measure->iteration[j] = 0;
+				for (i = 0; i < WIN2030_NOC_BIN_CNT; i++) {
+					prof_measure->min[j][i] = -1;
+					prof_measure->max[j][i] = 0;
+					prof_measure->now[j][i] = 0;
+					prof_measure->mean[j][i] = 0;
+				}
+				spin_unlock_irqrestore(&prof_measure->lock, flags);
+			}
+		}
+	}
+}
+
+/**
+ * trigger all noc prof measurment for all probes
+ * @param _dev A reference to the device
+ * @return 0
+ */
+int win2030_noc_prof_trigger(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct win2030_noc_prof_probe *prof_probe = NULL;
+	struct win2030_noc_prof *noc_prof = noc_device->prof;
+	unsigned ret;
+
+	//zebu_stop();
+	/* Loop over all probes */
+	list_for_each_entry(prof_probe, &noc_prof->prof_probe, link) {
+		/* lunch the first measurement */
+		ret = win2030_noc_prof_launch_measure(noc_device, prof_probe);
+		if (ret) {
+			dev_err(noc_device->dev,
+				"Error when trigger prof measures, probe %s!\n",
+				prof_probe->probe->id);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+/**
+ * Create a new prof_measure from parsed dts entry
+ * @param idx_trace_port_sel First entry in the dts line latency,x as index
+ * @return The created win2030_noc_prof_measure object
+ */
+static struct win2030_noc_prof_measure *win2030_noc_prof_crate_measure(
+				struct win2030_noc_prof_probe *prof_probe)
+{
+	struct win2030_noc_prof_measure *prof_measure;
+	unsigned long flags;
+
+	prof_measure = kzalloc(sizeof(struct win2030_noc_prof_measure),
+			       GFP_KERNEL);
+	if (!prof_measure)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&prof_measure->link);
+	spin_lock_init(&prof_measure->lock);
+
+	spin_lock_irqsave(&prof_probe->lock, flags);
+	list_add_tail(&prof_measure->link, &prof_probe->measure);
+	spin_unlock_irqrestore(&prof_probe->lock, flags);
+
+	return prof_measure;
+}
+
+/**
+ * Create a new prof_probe. A prof_probe contains all measurements which
+ * are linked to the same probe
+ * @param noc_prof the prof_probe will be attached to noc_prof
+ * @param probe the prof_probe will refer to probe
+ * @return The created xgold_noc_stat_probe object
+ */
+static struct win2030_noc_prof_probe *win2030_noc_prof_new_probe(struct win2030_noc_prof
+						   *noc_prof,
+						   struct win2030_noc_probe
+						   *probe)
+{
+	struct win2030_noc_prof_probe *prof_probe;
+	unsigned long flags;
+
+	prof_probe = kzalloc(sizeof(struct win2030_noc_prof_probe), GFP_KERNEL);
+	if (!prof_probe)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&prof_probe->measure);
+	INIT_LIST_HEAD(&prof_probe->link);
+	spin_lock_init(&prof_probe->lock);
+
+	spin_lock_irqsave(&noc_prof->lock, flags);
+	list_add_tail(&prof_probe->link, &noc_prof->prof_probe);
+	spin_unlock_irqrestore(&noc_prof->lock, flags);
+	prof_probe->probe = probe;
+
+	return prof_probe;
+}
+
+enum {
+	FIELD_ID_PORT = 0,
+	FIELD_ID_MODE,
+	FIELD_ID_ADDR,
+	FIELD_ID_SIZE,
+	FIELD_ID_OPCODE,
+	FIELD_ID_MAX,
+};
+
+static int win2030_noc_prof_parse_traceport_field(struct device *dev,
+	const char *str, int field_id,
+	struct win2030_noc_prof_traceport_data *data)
+{
+	u64 addr;
+	u64 size;
+	int ret;
+
+	switch (field_id) {
+	case FIELD_ID_MODE:
+		if (0 == strcmp(str, "Mode:latency")) {
+			data->mode = 1;
+		} else if (0 == strcmp(str, "Mode:handshake")) {
+			data->mode = 0;
+		} else {
+			dev_err(dev, "get unknown mode data %s when parsing traceport dts!\n", str);
+			return -EINVAL;
+		}
+		break;
+	case FIELD_ID_ADDR:
+		ret = kstrtou64(&str[strlen("AddrBase:")], 0, &addr);
+		if (ret) {
+			dev_err(dev, "get unknown addr data %s when parsing traceport dts!\n", str);
+			return ret;
+		}
+		data->addrBase_low = FIELD_GET(GENMASK(31, 0), addr);
+		data->addrBase_high = FIELD_GET(GENMASK(40, 32), addr);
+		break;
+	case FIELD_ID_SIZE:
+		ret = kstrtou64(&str[strlen("AddrSize:")], 0, &size);
+		if (ret) {
+			dev_err(dev, "get unknown mask data %s when parsing traceport dts!\n", str);
+			return ret;
+		}
+		data->addrWindowSize = size;
+		data->addrWindowSize &= PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_MASK;
+		break;
+	case FIELD_ID_OPCODE:
+		if (0 == strcmp(str, "Opcode:RdWr")) {
+			data->Opcode = 3;
+		} else if (0 == strcmp(str, "Opcode:Rd")) {
+			data->Opcode = 1;
+		} else if (0 == strcmp(str, "Opcode:Wr")) {
+			data->Opcode = 2;
+		} else {
+			dev_err(dev, "get unknown Opcode data %s when parsing traceport dts!\n", str);
+			return -EINVAL;
+		}
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+	return 0;
+}
+
+static int win2030_noc_prof_parse_traceport_dts(struct win2030_noc_device *noc_device,
+	struct device_node *np, const char *name, struct win2030_noc_prof_traceport_data *data)
+{
+	struct property *prop;
+	struct device *dev = noc_device->dev;
+	int lg;
+	int index = 0;
+	const char *str[FIELD_ID_MAX] = {NULL}, *s;
+	int ret;
+	const char *field[] = {"TracePort:", "Mode:", "AddrBase:", "AddrSize:", "Opcode:"};
+	int i, j;
+
+	/* Count number of strings per measurement
+	  (normally 1: TracePortSel)
+	 */
+	lg = of_property_count_strings(np, name);
+	if (lg > FIELD_ID_MAX || lg < 1)
+		return -EINVAL; /* Invalid data, let's skip this measurement */
+
+	of_property_for_each_string(np, name, prop, s)
+		str[index++] = s;
+
+	data->name = &str[0][strlen("TracePort:")];
+	/*if user not set, default set as below
+		Mode: 1, which means latency
+		addrBase: 0, which means all address
+		addrWindowSize : 0, which means all range
+		Opcode: 3, which means enable both read and write
+	*/
+	data->mode = 1;
+	data->addrBase_low = 0;
+	data->addrBase_high = 0;
+	data->addrWindowSize = 0x3f;
+	data->Opcode = 3;
+
+	for (i = 1; i < lg; i++) {
+		for (j = 0; j < ARRAY_SIZE(field); j++) {
+			if (0 == strncmp(str[i], field[j], strlen(field[j]))) {
+				ret = win2030_noc_prof_parse_traceport_field(dev, str[i], j, data);
+				if (ret) {
+					dev_err(dev, "error when parsing traceport dts, ret %d!\n", ret);
+					return ret;
+				}
+			}
+		}
+	}
+	dev_dbg(dev, "get traceport data, name: %s\n"
+		"\t\t\t\t Mode: %d, addrBase_low: 0x%x, addrBase_high: 0x%x\n"
+		"\t\t\t\t addrWindowSize: 0x%x, Opcode: 0x%x\n",data->name, data->mode, data->addrBase_low,
+		data->addrBase_high, data->addrWindowSize, data->Opcode);
+	return 0;
+}
+
+/**
+ * From one line of dts, add a new prof_measure and a prof_probe (if needed)
+ * @param noc_device A ref to the device
+ * @param np A ref to the dts node
+ * @param name the line of the dts
+ * @param type profiler measurment type
+ * @return
+ */
+static int win2030_noc_prof_new_measure(struct win2030_noc_device *noc_device,
+				 struct device_node *np, const char *name, measure_type type)
+{
+	int ret;
+	struct win2030_noc_prof_probe *prof_probe = NULL;
+	struct win2030_noc_prof *noc_prof = noc_device->prof;
+	struct win2030_noc_probe *noc_probe = NULL;
+	struct win2030_noc_prof_measure *prof_measure;
+	struct win2030_noc_prof_traceport_data data;
+	bool found;
+
+	ret = win2030_noc_prof_parse_traceport_dts(noc_device, np, name, &data);
+	if (ret) {
+		dev_err(noc_device->dev, "error when parsing traceport dts, ret %d!\n", ret);
+		return ret;
+	}
+
+	/* Search which probe can be used first value in dts */
+	found = false;
+	list_for_each_entry(noc_probe, &noc_device->probes, link) {
+		if (strcmp(data.name, noc_probe->id) == 0) {
+			found = true;
+			break;
+		}
+	}
+
+	if (false == found)
+		return -EINVAL;	/* Invalid data, let's skip this measurement. */
+
+	/* Look if the prof probe was already existing */
+	found = false;
+	list_for_each_entry(prof_probe, &noc_prof->prof_probe, link) {
+		if (prof_probe->probe == noc_probe) {
+			found = true;
+			break;
+		}
+	}
+	if (false == found) {
+		/* Not found, we must create a prof_probe */
+		prof_probe = win2030_noc_prof_new_probe(noc_prof, noc_probe);
+		if (IS_ERR(prof_probe))
+			goto error;
+
+	}
+	/* create a new measure */
+	prof_measure = win2030_noc_prof_crate_measure(prof_probe);
+	if (IS_ERR(prof_measure))
+		goto error;
+
+	prof_measure->type = type;
+	memcpy(&prof_measure->data, &data, sizeof(struct win2030_noc_prof_traceport_data));
+	return 0;
+error:
+	return -ENOMEM;
+}
+
+/**
+ * Entry point for the NOC profiler
+ * @param _dev A reference to the device
+ * @return 0 if ok, if error returns 0 but print an error message
+ */
+int win2030_noc_prof_init(struct device *_dev)
+{
+	struct win2030_noc_prof *noc_prof;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct device_node *np = _dev->of_node;
+	int latency_idx = 0;
+	int pending_idx = 0;
+	bool ret;
+	char mystr[32];
+	int err;
+
+	/* Create a prof device. */
+	noc_prof = kzalloc(sizeof(struct win2030_noc_prof), GFP_KERNEL);
+	if (!noc_prof)
+		return -ENOMEM;
+	noc_device->prof = noc_prof;
+	noc_prof->run = false;
+	spin_lock_init(&noc_prof->lock);
+	INIT_LIST_HEAD(&noc_prof->prof_probe);
+
+	/* Parse dts and build matrix node */
+	do {
+		scnprintf(mystr, ARRAY_SIZE(mystr), "latency,%d", latency_idx);
+		ret = of_property_read_bool(np, mystr);
+		if (ret == true) {
+			latency_idx++;
+			err = win2030_noc_prof_new_measure(noc_device, np, mystr, latency_t);
+			if (err)
+				dev_err(_dev, "Error when adding latency measurment: %s!\n", mystr);
+		} else {
+			scnprintf(mystr, ARRAY_SIZE(mystr), "pending,%d", pending_idx);
+			ret = of_property_read_bool(np, mystr);
+			if (ret == true) {
+				pending_idx++;
+				err = win2030_noc_prof_new_measure(noc_device, np, mystr, pending_t);
+				if (err)
+					dev_err(_dev, "Error when adding pending measurment: %s!\n", mystr);
+			}
+		}
+	} while (ret == true);
+
+	/* If no prof line in dts then we can release the noc_prof object */
+	if (latency_idx == 0 && pending_idx == 0)
+		goto no_profiler;
+
+	/* Trigger measurement */
+	if (noc_prof->run) {
+		ret = win2030_noc_prof_trigger(_dev);
+		if (ret)
+			dev_err(_dev,
+				"Error when configuring prof measures!\n");
+	}
+	return 0;
+
+no_profiler:
+	kfree(noc_prof);
+	return -EINVAL;
+}
+
+/**
+ * Interrupt processing when a measurement is done
+ * @param noc_device A reference to the device
+ * @param probe The probe for which a measurement is done
+ */
+void win2030_noc_prof_do_measure(struct win2030_noc_probe *probe)
+{
+	struct win2030_noc_device *noc_device = probe->parent;
+	struct win2030_noc_prof_probe *prof_probe = NULL;
+	struct win2030_noc_prof_measure *prof_measure;
+	unsigned counter[WIN2030_NOC_BIN_CNT] = {0};
+	unsigned ret;
+	unsigned long flags;
+	int i, j;
+	u64 total = 0;
+	int percentage = 0;
+
+	/* Find which prof_probe is linked to the probe
+	 * which generated the interrupt
+	 */
+	list_for_each_entry(prof_probe, &noc_device->prof->prof_probe, link) {
+		if (prof_probe->probe == probe)
+			break;
+	}
+
+	/* Take measure object */
+	prof_measure = list_first_entry(&prof_probe->measure,
+				struct win2030_noc_prof_measure, link);
+	for (j = 0; j < probe->nr_portsel; j++) {
+		total = 0;
+		for (i = 0; i < WIN2030_NOC_BIN_CNT; i++) {
+			/**
+			  Read HW counters
+			*/
+			ret = win2030_bitfield_read(probe->counters[j * 4 + i].value, &counter[i]);
+			WARN_ON(0 != ret);
+			total += counter[i];
+		}
+		spin_lock_irqsave(&prof_measure->lock, flags);
+		if (0 == total) {
+			/*if total == 0, each bin's percentage will be 0*/
+			total = (-1U);
+		} else {
+			/*
+			   only update iteration when there are traffic,
+			   otherwize the mean value will be calculate by a wrong iteration number.
+			*/
+			if (prof_measure->iteration[j] != MAX_ITERATION) {
+				prof_measure->iteration[j] += 1;
+			} else {
+				/*OverFlow*/
+				dev_info(noc_device->dev, "TracePort %s iteration overflow, reset measure statistics!\n",
+					prof_probe->probe->available_portsel[j]);
+				prof_measure->iteration[j] = 1;
+				for (i = 0; i < WIN2030_NOC_BIN_CNT; i++) {
+					prof_measure->mean[j][i] = 0;
+				}
+			}
+		}
+		for (i = 0; i < WIN2030_NOC_BIN_CNT; i++) {
+			/* calculate each bin's percentage in the total traffic*/
+			percentage = counter[i] * 100 / total;
+			/* Update prof_measure statistics */
+			if (percentage > prof_measure->max[j][i]) {
+				prof_measure->max[j][i] = percentage;
+			}
+			if (percentage < prof_measure->min[j][i]) {
+				prof_measure->min[j][i] = percentage;
+			}
+			prof_measure->now[j][i] = percentage;
+			prof_measure->mean[j][i] += percentage;
+		}
+		spin_unlock_irqrestore(&prof_measure->lock, flags);
+	}
+
+	/* Program next measure */
+	if (noc_device->prof->run == true) {
+		/* Move to the next measure */
+		spin_lock_irqsave(&prof_probe->lock, flags);
+		list_move(&prof_measure->link, &prof_probe->measure);
+		spin_unlock_irqrestore(&prof_probe->lock, flags);
+
+		/* Program it */
+		ret = win2030_noc_prof_launch_measure(noc_device, prof_probe);
+		if (ret)
+			dev_err(noc_device->dev, "Error when configuring prof measures!\n");
+	}
+}
diff --git a/drivers/interconnect/eswin/noc_profiler_sysfs.c b/drivers/interconnect/eswin/noc_profiler_sysfs.c
new file mode 100644
index 000000000000..e36c6ff61706
--- /dev/null
+++ b/drivers/interconnect/eswin/noc_profiler_sysfs.c
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/device.h>
+#include <linux/of.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <asm/div64.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+
+#include "noc.h"
+
+/**
+ * Called when writing run file
+ */
+static ssize_t noc_prof_run_write(struct file *filp, const char __user *ubuf,
+				  size_t cnt, loff_t *ppos)
+{
+	struct win2030_noc_device *noc_device = filp->private_data;
+	char buf[SIZE_SMALL_BUF];
+	unsigned ret;
+
+	if (cnt > SIZE_SMALL_BUF)
+		cnt = SIZE_SMALL_BUF - 1;
+
+	if (copy_from_user(&buf, ubuf, cnt))
+		return -EFAULT;
+
+	if (buf[0] == '0') {
+		noc_device->prof->run = false;	/* FIXME spinlock? */
+		/*wait for the latest measure to be finished*/
+		msleep(5);
+		win2030_noc_prof_reset_statistics(noc_device->dev);
+	} else if (buf[0] == '1') {
+		if (true != noc_device->prof->run) {
+			noc_device->prof->run = true;
+			/* trigger measurements */
+			ret = win2030_noc_prof_trigger(noc_device->dev);
+			WARN_ON(ret != 0);
+		}
+	} else
+		return -EFAULT;
+
+	*ppos += cnt;
+
+	return cnt;
+}
+
+/**
+ * Called when reading run file
+ */
+static ssize_t noc_prof_run_read(struct file *filp, char __user *ubuf,
+				 size_t cnt, loff_t *ppos)
+{
+#define RUN_STR_SIZE 11
+	struct win2030_noc_device *noc_device = filp->private_data;
+	char buf[RUN_STR_SIZE];
+	int r;
+
+	r = snprintf(buf, RUN_STR_SIZE, "%i\n", noc_device->prof->run);
+
+	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+/**
+ * Called when writing config file
+ */
+static ssize_t noc_prof_config_write(struct file *filp,
+				     const char __user *ubuf, size_t cnt,
+				     loff_t *ppos)
+{
+	char buf[SIZE_SMALL_BUF] = {0};
+	char *bin;
+	int value;
+	int ret;
+	char bin_name[20] = {0};
+	int i;
+
+	if (cnt > SIZE_SMALL_BUF)
+		cnt = SIZE_SMALL_BUF - 1;
+
+	if (copy_from_user(&buf, ubuf, cnt))
+		return -EFAULT;
+
+	*ppos += cnt;
+	for (i = 0; i < 3; i++) {
+		sprintf(bin_name, "latency_bin%d:", i);
+		bin = strstr(buf, bin_name);
+		if (bin) {
+			ret = kstrtoint(bin + strlen(bin_name), 0, &value);
+			if (!ret) {
+				latency_bin[i] = value;
+				return cnt;
+			}
+		}
+	}
+	pr_err("invalid config %s", buf);
+	return cnt;
+}
+
+/**
+ * Called when reading config file
+ */
+static ssize_t noc_prof_config_read(struct file *filp, char __user *ubuf,
+				    size_t cnt, loff_t *ppos)
+{
+	return 0;
+}
+
+/**
+ * Called when reading readme file
+ * @param m
+ * @param p
+ * @return
+ */
+static int noc_prof_readme_show(struct seq_file *m, void *p)
+{
+	seq_puts(m,
+		 "How to use the NOC profiler:\nFile run: write '1' to enable "
+		 "profiler (enabling profiler resets statistics),"
+		 " write '0' to disable profiler.\n"
+		 "File config: Not yet implemented.\n"
+		 "File results: contain the results of measurement(each bin's percentage in the total traffic)"
+		 "for all nodes defined in dts.\n");
+	return 0;
+}
+
+/**
+ * Called when reading results file.
+ */
+static ssize_t noc_prof_results_read(struct file *filp, char __user *ubuf,
+				     size_t cnt, loff_t *ppos)
+{
+	struct win2030_noc_device *noc_device = filp->private_data;
+	struct win2030_noc_probe *probe;
+	struct win2030_noc_prof *prof;
+	struct win2030_noc_prof_probe *prof_probe = NULL;
+	struct win2030_noc_prof_measure *prof_measure = NULL;
+	u64 meas_mean;
+	int i, j;
+	char *buf1;
+	char buf2[SIZE_SMALL_BUF + 2];
+	int r;
+	ssize_t ret;
+	int count;	/* Used to avoid write over buffer size */
+	char latency_bin_name[4][20] = {{0}};
+	char *pending_bin_name[] = {"0~2", "2~5","5~10", "10~"};
+	unsigned long flags;
+
+	if (noc_device == NULL)
+		return -ENODEV;
+
+	prof = noc_device->prof;
+	if (!prof)
+		return -ENODEV;
+
+	buf1 = kzalloc(sizeof(*buf1) * SIZE_BIG_BUF, GFP_KERNEL);
+	if (!buf1)
+		return -ENODEV;
+
+	r = snprintf(buf1, SIZE_BIG_BUF,"\t##### RESULTS #####\n\n");
+
+	count = SIZE_BIG_BUF - strlen(buf1);
+	list_for_each_entry(prof_probe, &prof->prof_probe, link) {
+		probe = prof_probe->probe;
+		list_for_each_entry(prof_measure, &prof_probe->measure, link) {
+			if (count > 0) {
+				r = snprintf(buf2, SIZE_SMALL_BUF,
+					"Probe %s\n", prof_probe->probe->id);
+				strncat(buf1, buf2, count);
+				count -= strlen(buf2);
+			}
+			if (pending_t == prof_measure->type) {
+
+			} else {
+				sprintf(latency_bin_name[0], "\"0~%d\"", latency_bin[0]);
+				sprintf(latency_bin_name[1], "\"%d~%d\"",latency_bin[0], latency_bin[1]);
+				sprintf(latency_bin_name[2], "\"%d~%d\"",latency_bin[1], latency_bin[2]);
+				sprintf(latency_bin_name[3], "\"%d~\"", latency_bin[2]);
+			}
+			spin_lock_irqsave(&prof_measure->lock, flags);
+			for (j = 0; j < probe->nr_portsel; j++) {
+				r = snprintf(buf2, SIZE_SMALL_BUF,
+					"\tTracePort %s, NbOfMeasure=%i, type %s\n",
+					prof_probe->probe->available_portsel[j],
+					prof_measure->iteration[j],
+					prof_measure->type == pending_t ? "pedning" : "latency");
+				strncat(buf1, buf2, count);
+				count -= strlen(buf2);
+				if (count > 0) {
+					if (prof_measure->iteration[j] != 0) {
+						for (i = 0; i < WIN2030_NOC_BIN_CNT; i++) {
+							meas_mean = prof_measure->mean[j][i];
+							do_div(meas_mean, prof_measure->iteration[j]);
+							r = snprintf(buf2, SIZE_SMALL_BUF,
+								     "\t\t%s(cycles) percentage: min=%llu%%  max=%llu%%  mean=%llu%%  "
+								     "current=%llu%% \n\n",
+								     pending_t == prof_measure->type ? pending_bin_name[i] : latency_bin_name[i],
+								     prof_measure->min[j][i],
+								     prof_measure->max[j][i],
+								     meas_mean,
+								     prof_measure->now[j][i]);
+							strncat(buf1, buf2, count);
+							count -= strlen(buf2);
+						}
+					} else {
+						r = snprintf(buf2, SIZE_SMALL_BUF,
+						     "\t\t No results\n\n");
+						strncat(buf1, buf2, count);
+						count -= strlen(buf2);
+					}
+				}
+			}
+			spin_unlock_irqrestore(&prof_measure->lock, flags);
+		}
+	}
+
+	r = strlen(buf1) + 1;
+
+	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf1, r);
+
+	kfree(buf1);
+	return ret;
+
+}
+
+/**
+ * Called when open results file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_prof_results_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * Called when opening config file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_prof_config_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * Called when opening run file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_prof_run_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * Called when reading readme file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_prof_readme_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, noc_prof_readme_show, inode->i_private);
+}
+
+static const struct file_operations noc_prof_results_fops = {
+	.open = noc_prof_results_open,
+	.read = noc_prof_results_read,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_prof_config_fops = {
+	.open = noc_prof_config_open,
+	.read = noc_prof_config_read,
+	.write = noc_prof_config_write,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_prof_run_fops = {
+	.open = noc_prof_run_open,
+	.read = noc_prof_run_read,
+	.write = noc_prof_run_write,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_prof_readme_fops = {
+	.open = noc_prof_readme_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+/**
+ * Entry point for NOC profiler sysfs
+ * @param _dev A reference to the device
+ * @return 0
+ */
+int win2030_noc_prof_debugfs_init(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct device_node *np = _dev->of_node;
+	struct dentry *dir, *d;
+	char name[32];
+
+	scnprintf(name, ARRAY_SIZE(name), "%s_prof", np->name);
+
+	dir = debugfs_create_dir(name, win2030_noc_ctrl.win2030_noc_root_debug_dir);
+	if (IS_ERR(dir))
+		return PTR_ERR(dir);
+
+	noc_device->prof->dir = dir;
+
+	d = debugfs_create_file("results", S_IRUGO, dir, noc_device,
+				&noc_prof_results_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+	d = debugfs_create_file("config", S_IRUGO | S_IWUSR, dir, noc_device,
+				&noc_prof_config_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+	d = debugfs_create_file("run", S_IRUGO | S_IWUSR, dir, noc_device,
+				&noc_prof_run_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+	d = debugfs_create_file("readme.txt", S_IRUGO, dir, noc_device,
+				&noc_prof_readme_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+
+	return 0;
+}
diff --git a/drivers/interconnect/eswin/noc_regs.h b/drivers/interconnect/eswin/noc_regs.h
new file mode 100644
index 000000000000..2dc67a042540
--- /dev/null
+++ b/drivers/interconnect/eswin/noc_regs.h
@@ -0,0 +1,633 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef _NOC_REGS_H_
+#define _NOC_REGS_H_
+
+#define ERRLOG_0_ID_COREID(_base) ((_base) + 0x0)
+#define ERRLOG_0_ID_COREID_CORETYPEID_OFFSET 0x0
+#define ERRLOG_0_ID_COREID_CORETYPEID_WIDTH 0x8
+#define ERRLOG_0_ID_COREID_CORETYPEID_MASK 0xff
+#define ERRLOG_0_ID_COREID_CORETYPEID(_reg) (((_reg) & 0xff) >> 0x0)
+#define ERRLOG_0_ID_COREID_CORECHECKSUM_OFFSET 0x8
+#define ERRLOG_0_ID_COREID_CORECHECKSUM_WIDTH 0x18
+#define ERRLOG_0_ID_COREID_CORECHECKSUM_MASK 0xffffff00
+#define ERRLOG_0_ID_COREID_CORECHECKSUM(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define ERRLOG_0_ID_REVISIONID(_base) ((_base) + 0x4)
+#define ERRLOG_0_ID_REVISIONID_USERID_OFFSET 0x0
+#define ERRLOG_0_ID_REVISIONID_USERID_WIDTH 0x8
+#define ERRLOG_0_ID_REVISIONID_USERID_MASK 0xff
+#define ERRLOG_0_ID_REVISIONID_USERID(_reg) (((_reg) & 0xff) >> 0x0)
+#define ERRLOG_0_ID_REVISIONID_FLEXNOCID_OFFSET 0x8
+#define ERRLOG_0_ID_REVISIONID_FLEXNOCID_WIDTH 0x18
+#define ERRLOG_0_ID_REVISIONID_FLEXNOCID_MASK 0xffffff00
+#define ERRLOG_0_ID_REVISIONID_FLEXNOCID(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define ERRLOG_0_FAULTEN(_base) ((_base) + 0x8)
+#define ERRLOG_0_FAULTEN_FAULTEN_OFFSET 0x0
+#define ERRLOG_0_FAULTEN_FAULTEN_WIDTH 0x1
+#define ERRLOG_0_FAULTEN_FAULTEN_MASK 0x1
+#define ERRLOG_0_FAULTEN_FAULTEN(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define ERRLOG_0_ERRVLD(_base) ((_base) + 0xc)
+#define ERRLOG_0_ERRVLD_ERRVLD_OFFSET 0x0
+#define ERRLOG_0_ERRVLD_ERRVLD_WIDTH 0x1
+#define ERRLOG_0_ERRVLD_ERRVLD_MASK 0x1
+#define ERRLOG_0_ERRVLD_ERRVLD(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define ERRLOG_0_ERRCLR(_base) ((_base) + 0x10)
+#define ERRLOG_0_ERRCLR_ERRCLR_OFFSET 0x0
+#define ERRLOG_0_ERRCLR_ERRCLR_WIDTH 0x1
+#define ERRLOG_0_ERRCLR_ERRCLR_MASK 0x1
+#define ERRLOG_0_ERRCLR_ERRCLR(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define ERRLOG_0_ERRLOG0(_base) ((_base) + 0x14)
+#define ERRLOG_0_ERRLOG0_LOCK_OFFSET 0x0
+#define ERRLOG_0_ERRLOG0_LOCK_WIDTH 0x1
+#define ERRLOG_0_ERRLOG0_LOCK_MASK 0x1
+#define ERRLOG_0_ERRLOG0_LOCK(_reg) (((_reg) & 0x1) >> 0x0)
+#define ERRLOG_0_ERRLOG0_OPC_OFFSET 0x1
+#define ERRLOG_0_ERRLOG0_OPC_WIDTH 0x4
+#define ERRLOG_0_ERRLOG0_OPC_MASK 0x1e
+#define ERRLOG_0_ERRLOG0_OPC(_reg) (((_reg) & 0x1e) >> 0x1)
+#define ERRLOG_0_ERRLOG0_ERRCODE_OFFSET 0x8
+#define ERRLOG_0_ERRLOG0_ERRCODE_WIDTH 0x3
+#define ERRLOG_0_ERRLOG0_ERRCODE_MASK 0x700
+#define ERRLOG_0_ERRLOG0_ERRCODE(_reg) (((_reg) & 0x700) >> 0x8)
+#define ERRLOG_0_ERRLOG0_LEN1_OFFSET 0x10
+#define ERRLOG_0_ERRLOG0_LEN1_WIDTH 0x7
+#define ERRLOG_0_ERRLOG0_LEN1_MASK 0x7f0000
+#define ERRLOG_0_ERRLOG0_LEN1(_reg) (((_reg) & 0x7f0000) >> 0x10)
+#define ERRLOG_0_ERRLOG0_FORMAT_OFFSET 0x1f
+#define ERRLOG_0_ERRLOG0_FORMAT_WIDTH 0x1
+#define ERRLOG_0_ERRLOG0_FORMAT_MASK 0x80000000
+#define ERRLOG_0_ERRLOG0_FORMAT(_reg) (((_reg) & 0x80000000) >> 0x1f)
+
+#define ERRLOG_0_ERRLOG1(_base) ((_base) + 0x18)
+#define ERRLOG_0_ERRLOG1_ERRLOG1_OFFSET 0x0
+#define ERRLOG_0_ERRLOG1_ERRLOG1_WIDTH 0x11
+#define ERRLOG_0_ERRLOG1_ERRLOG1_MASK 0x1ffff
+#define ERRLOG_0_ERRLOG1_ERRLOG1(_reg) (((_reg) & 0x1ffff) >> 0x0)
+
+#define ERRLOG_0_ERRLOG3(_base) ((_base) + 0x20)
+#define ERRLOG_0_ERRLOG3_ERRLOG3_OFFSET 0x0
+#define ERRLOG_0_ERRLOG3_ERRLOG3_WIDTH 0x20
+#define ERRLOG_0_ERRLOG3_ERRLOG3_MASK 0xffffffff
+#define ERRLOG_0_ERRLOG3_ERRLOG3(_reg) (((_reg) & 0xffffffff) >> 0x0)
+
+#define ERRLOG_0_ERRLOG5(_base) ((_base) + 0x28)
+#define ERRLOG_0_ERRLOG5_ERRLOG5_OFFSET 0x0
+#define ERRLOG_0_ERRLOG5_ERRLOG5_WIDTH 0x9
+#define ERRLOG_0_ERRLOG5_ERRLOG5_MASK 0x1ff
+#define ERRLOG_0_ERRLOG5_ERRLOG5(_reg) (((_reg) & 0x1ff) >> 0x0)
+
+#define ERRLOG_0_ERRLOG7(_base) ((_base) + 0x30)
+#define ERRLOG_0_ERRLOG7_ERRLOG7_OFFSET 0x0
+#define ERRLOG_0_ERRLOG7_ERRLOG7_WIDTH 0x4
+#define ERRLOG_0_ERRLOG7_ERRLOG7_MASK 0xf
+#define ERRLOG_0_ERRLOG7_ERRLOG7(_reg) (((_reg) & 0xf) >> 0x0)
+
+#define PROBE_ID_COREID(_base) ((_base) + 0x1000)
+#define PROBE_ID_COREID_CORETYPEID_OFFSET 0x0
+#define PROBE_ID_COREID_CORETYPEID_WIDTH 0x8
+#define PROBE_ID_COREID_CORETYPEID_MASK 0xff
+#define PROBE_ID_COREID_CORETYPEID(_reg) (((_reg) & 0xff) >> 0x0)
+#define PROBE_ID_COREID_CORECHECKSUM_OFFSET 0x8
+#define PROBE_ID_COREID_CORECHECKSUM_WIDTH 0x18
+#define PROBE_ID_COREID_CORECHECKSUM_MASK 0xffffff00
+#define PROBE_ID_COREID_CORECHECKSUM(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define PROBE_ID_REVISIONID(_base) ((_base) + 0x1004)
+#define PROBE_ID_REVISIONID_USERID_OFFSET 0x0
+#define PROBE_ID_REVISIONID_USERID_WIDTH 0x8
+#define PROBE_ID_REVISIONID_USERID_MASK 0xff
+#define PROBE_ID_REVISIONID_USERID(_reg) (((_reg) & 0xff) >> 0x0)
+#define PROBE_ID_REVISIONID_FLEXNOCID_OFFSET 0x8
+#define PROBE_ID_REVISIONID_FLEXNOCID_WIDTH 0x18
+#define PROBE_ID_REVISIONID_FLEXNOCID_MASK 0xffffff00
+#define PROBE_ID_REVISIONID_FLEXNOCID(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define PROBE_MAINCTL (0x8)
+#define PROBE_MAINCTL_ERREN_OFFSET 0x0
+#define PROBE_MAINCTL_ERREN_WIDTH 0x1
+#define PROBE_MAINCTL_ERREN_MASK 0x1
+#define PROBE_MAINCTL_ERREN(_reg) (((_reg) & 0x1) >> 0x0)
+#define PROBE_MAINCTL_TRACEEN_OFFSET 0x1
+#define PROBE_MAINCTL_TRACEEN_WIDTH 0x1
+#define PROBE_MAINCTL_TRACEEN_MASK 0x2
+#define PROBE_MAINCTL_TRACEEN(_reg) (((_reg) & 0x2) >> 0x1)
+#define PROBE_MAINCTL_PAYLOADEN_OFFSET 0x2
+#define PROBE_MAINCTL_PAYLOADEN_WIDTH 0x1
+#define PROBE_MAINCTL_PAYLOADEN_MASK 0x4
+#define PROBE_MAINCTL_PAYLOADEN(_reg) (((_reg) & 0x4) >> 0x2)
+#define PROBE_MAINCTL_STATEN_OFFSET 0x3
+#define PROBE_MAINCTL_STATEN_WIDTH 0x1
+#define PROBE_MAINCTL_STATEN_MASK 0x8
+#define PROBE_MAINCTL_STATEN(_reg) (((_reg) & 0x8) >> 0x3)
+#define PROBE_MAINCTL_ALARMEN_OFFSET 0x4
+#define PROBE_MAINCTL_ALARMEN_WIDTH 0x1
+#define PROBE_MAINCTL_ALARMEN_MASK 0x10
+#define PROBE_MAINCTL_ALARMEN(_reg) (((_reg) & 0x10) >> 0x4)
+#define PROBE_MAINCTL_STATCONDDUMP_OFFSET 0x5
+#define PROBE_MAINCTL_STATCONDDUMP_WIDTH 0x1
+#define PROBE_MAINCTL_STATCONDDUMP_MASK 0x20
+#define PROBE_MAINCTL_STATCONDDUMP(_reg) (((_reg) & 0x20) >> 0x5)
+#define PROBE_MAINCTL_INTRUSIVEMODE_OFFSET 0x6
+#define PROBE_MAINCTL_INTRUSIVEMODE_WIDTH 0x1
+#define PROBE_MAINCTL_INTRUSIVEMODE_MASK 0x40
+#define PROBE_MAINCTL_INTRUSIVEMODE(_reg) (((_reg) & 0x40) >> 0x6)
+#define PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_OFFSET 0x7
+#define PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_WIDTH 0x1
+#define PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_MASK 0x80
+#define PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN(_reg) (((_reg) & 0x80) >> 0x7)
+
+#define PROBE_CFGCTL (0xc)
+#define PROBE_CFGCTL_GLOBALEN_OFFSET 0x0
+#define PROBE_CFGCTL_GLOBALEN_WIDTH 0x1
+#define PROBE_CFGCTL_GLOBALEN_MASK 0x1
+#define PROBE_CFGCTL_GLOBALEN(_reg) (((_reg) & 0x1) >> 0x0)
+#define PROBE_CFGCTL_ACTIVE_OFFSET 0x1
+#define PROBE_CFGCTL_ACTIVE_WIDTH 0x1
+#define PROBE_CFGCTL_ACTIVE_MASK 0x2
+#define PROBE_CFGCTL_ACTIVE(_reg) (((_reg) & 0x2) >> 0x1)
+
+#define PROBE_TRACEPORTSEL (0x10)
+#define PROBE_TRACEPORTSEL_TRACEPORTSEL_OFFSET 0x0
+#define PROBE_TRACEPORTSEL_TRACEPORTSEL_WIDTH 0x3
+#define PROBE_TRACEPORTSEL_TRACEPORTSEL_MASK 0x7
+#define PROBE_TRACEPORTSEL_TRACEPORTSEL(_reg) (((_reg) & 0x7) >> 0x0)
+
+#define PROBE_FILTERLUT (0x14)
+#define PROBE_FILTERLUT_FILTERLUT_OFFSET 0x0
+#define PROBE_FILTERLUT_FILTERLUT_WIDTH 0x2
+#define PROBE_FILTERLUT_FILTERLUT_MASK 0x3
+#define PROBE_FILTERLUT_FILTERLUT(_reg) (((_reg) & PROBE_FILTERLUT_FILTERLUT_MASK) >> 0x0)
+
+#define PROBE_TRACEALARMEN (0x18)
+#define PROBE_TRACEALARMEN_TRACEALARMEN_OFFSET 0x0
+#define PROBE_TRACEALARMEN_TRACEALARMEN_WIDTH 0x5
+#define PROBE_TRACEALARMEN_TRACEALARMEN_MASK 0x1f
+#define PROBE_TRACEALARMEN_TRACEALARMEN(_reg) (((_reg) & 0x1f) >> 0x0)
+
+#define PROBE_TRACEALARMSTATUS (0x1c)
+#define PROBE_TRACEALARMSTATUS_TRACEALARMSTATUS_OFFSET 0x0
+#define PROBE_TRACEALARMSTATUS_TRACEALARMSTATUS_WIDTH 0x5
+#define PROBE_TRACEALARMSTATUS_TRACEALARMSTATUS_MASK 0x1f
+#define PROBE_TRACEALARMSTATUS_TRACEALARMSTATUS(_reg) (((_reg) & 0x1f) >> 0x0)
+
+#define PROBE_TRACEALARMCLR (0x20)
+#define PROBE_TRACEALARMCLR_TRACEALARMCLR_OFFSET 0x0
+#define PROBE_TRACEALARMCLR_TRACEALARMCLR_WIDTH 0x5
+#define PROBE_TRACEALARMCLR_TRACEALARMCLR_MASK 0x1f
+#define PROBE_TRACEALARMCLR_TRACEALARMCLR(_reg) (((_reg) & 0x1f) >> 0x0)
+
+#define PROBE_STATPERIOD (0x24)
+#define PROBE_STATPERIOD_STATPERIOD_OFFSET 0x0
+#define PROBE_STATPERIOD_STATPERIOD_WIDTH 0x5
+#define PROBE_STATPERIOD_STATPERIOD_MASK 0x1f
+#define PROBE_STATPERIOD_STATPERIOD(_reg) (((_reg) & 0x1f) >> 0x0)
+
+#define PROBE_STATGO (0x28)
+#define PROBE_STATGO_STATGO_OFFSET 0x0
+#define PROBE_STATGO_STATGO_WIDTH 0x1
+#define PROBE_STATGO_STATGO_MASK 0x1
+#define PROBE_STATGO_STATGO(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define PROBE_STATALARMMIN (0x2c)
+#define PROBE_STATALARMMIN_STATALARMMIN_OFFSET 0x0
+#define PROBE_STATALARMMIN_STATALARMMIN_WIDTH 0x20
+#define PROBE_STATALARMMIN_STATALARMMIN_MASK 0xffffffff
+#define PROBE_STATALARMMIN_STATALARMMIN(_reg) (((_reg) & PROBE_STATALARMMIN_STATALARMMIN_MASK) >> 0x0)
+
+#define PROBE_STATALARMMIN_HIGH (0x30)
+#define PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_OFFSET 0x0
+#define PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_WIDTH 0x20
+#define PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_MASK 0xffffffff
+#define PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH(_reg) \
+					(((_reg) & PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_MASK) >> 0x0)
+#define PROBE_STATALARMMAX (0x34)
+#define PROBE_STATALARMMAX_STATALARMMAX_OFFSET 0x0
+#define PROBE_STATALARMMAX_STATALARMMAX_WIDTH 0x20
+#define PROBE_STATALARMMAX_STATALARMMAX_MASK 0xffffffff
+#define PROBE_STATALARMMAX_STATALARMMAX(_reg) (((_reg) & PROBE_STATALARMMAX_STATALARMMAX_MASK) >> 0x0)
+
+#define PROBE_STATALARMSTATUS (0x3C)
+#define PROBE_STATALARMSTATUS_STATALARMSTATUS_OFFSET 0x0
+#define PROBE_STATALARMSTATUS_STATALARMSTATUS_WIDTH 0x1
+#define PROBE_STATALARMSTATUS_STATALARMSTATUS_MASK 0x1
+#define PROBE_STATALARMSTATUS_STATALARMSTATUS(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define PROBE_STATALARMCLR (0x40)
+#define PROBE_STATALARMCLR_STATALARMCLR_OFFSET 0x0
+#define PROBE_STATALARMCLR_STATALARMCLR_WIDTH 0x1
+#define PROBE_STATALARMCLR_STATALARMCLR_MASK 0x1
+#define PROBE_STATALARMCLR_STATALARMCLR(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define PROBE_STATALARMEN (0x44)
+#define PROBE_STATALARMEN_STATALARMEN_OFFSET 0x0
+#define PROBE_STATALARMEN_STATALARMEN_WIDTH 0x1
+#define PROBE_STATALARMEN_STATALARMEN_MASK 0x1
+#define PROBE_STATALARMEN_STATALARMEN(_reg) (((_reg) & 0x1) >> 0x0)
+
+/*Trans probe filter begin*/
+#define PROBE_TRANS_FILTERS_MODE 	(0x08)
+#define PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_OFFSET 0x0
+#define PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_WIDTH 0x01
+#define PROBE_TRANS_FILTERS_MODE_FILTERS_MODE_MASK 0x01
+
+#define PROBE_TRANS_FILTERS_ADDRBASE_LOW 	(0x0C)
+#define PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET 0x0
+#define PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_WIDTH 0x20
+#define PROBE_TRANS_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_MASK 0xffffffff
+
+#define PROBE_TRANS_FILTERS_ADDRBASE_HIGH 	(0x10)
+#define PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_OFFSET 0x0
+#define PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_WIDTH 0x09
+#define PROBE_TRANS_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_MASK 0x1FF
+
+#define PROBE_TRANS_FILTERS_WINDOWSIZE 		(0x14)
+#define PROBE_TRANS_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_OFFSET 0x0
+#define PROBE_TRANS_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_WIDTH 0x06
+
+#define PROBE_TRANS_FILTERS_OPCODE 		(0x20)
+#define PROBE_TRANS_FILTERS_OPCODE_RDEN_OFFSET	0x0
+#define PROBE_TRANS_FILTERS_OPCODE_RDEN_WIDTH	0x1
+#define PROBE_TRANS_FILTERS_OPCODE_RDEN_MASK	0x1
+
+#define PROBE_TRANS_FILTERS_OPCODE_WREN_OFFSET	0x1
+#define PROBE_TRANS_FILTERS_OPCODE_WREN_WIDTH	0x1
+#define PROBE_TRANS_FILTERS_OPCODE_WREN_MASK	0x2
+
+#define PROBE_TRANS_FILTERS_USER_BASE 		(0x24)
+#define PROBE_TRANS_FILTERS_USER_BASE_FILTERS_USER_BASE 0x0
+#define PROBE_TRANS_FILTERS_USER_BASE_FILTERS_USER_BASE_WIDTH 0x12
+
+#define PROBE_TRANS_FILTERS_USER_MASK 		(0x28)
+#define PROBE_TRANS_FILTERS_USER_MASK_FILTERS_USER_MASK 0x0
+#define PROBE_TRANS_FILTERS_USER_MASK_FILTERS_USER_MASK_WIDTH 0x12
+
+/**trans probe filter end*/
+
+/*Trans probe profiler begin*/
+#define PROBE_TRANS_PROFILER_EN 	(0x08)
+#define PROBE_TRANS_PROFILER_EN_PROFILER_EN_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_EN_PROFILER_EN_WIDTH 0x01
+#define PROBE_TRANS_PROFILER_EN_PROFILER_EN_MASK 0x01
+
+#define PROBE_TRANS_PROFILER_MODE 	(0x0C)
+#define PROBE_TRANS_PROFILER_MODE_PROFILER_MODE_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_MODE_PROFILER_MODE_WIDTH 0x2
+
+#define PROBE_TRANS_PROFILER_OBSERVED_SEL 	(0x10)
+#define PROBE_TRANS_PROFILER_OBSERVED_SEL_PROFILER_OBSERVED_SEL_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_OBSERVED_SEL_PROFILER_OBSERVED_SEL_WIDTH 0x01
+#define PROBE_TRANS_PROFILER_OBSERVED_SEL_PROFILER_OBSERVED_SEL_MASK 0x01
+
+#define PROBE_TRANS_PROFILER_N_TENURE_LINES 	(0x20)
+#define PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_WIDTH 0x2
+#define PROBE_TRANS_PROFILER_N_TENURE_LINES_PROFILER_N_TENURE_LINES_MASK 0x3
+
+#define PROBE_TRANS_PROFILER_THRESHOLDS 	(0x2C)
+#define PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_WIDTH 0xa
+#define PROBE_TRANS_PROFILER_THRESHOLDS_PROFILER_THRESHOLDS_MASK 0x3FF
+
+#define PROBE_TRANS_PROFILER_OVER_FLOW_STATUS 	(0x6C)
+#define PROBE_TRANS_PROFILER_OVER_FLOW_STATUS_PROFILER_OVER_FLOW_STATUS_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_OVER_FLOW_STATUS_PROFILER_OVER_FLOW_STATUS_WIDTH 0x2
+
+#define PROBE_TRANS_PROFILER_OVER_FLOW_RESET 	(0x70)
+#define PROBE_TRANS_PROFILER_OVER_FLOW_RESET_PROFILER_OVER_FLOW_RESET_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_OVER_FLOW_RESET_PROFILER_OVER_FLOW_RESET_WIDTH 0x2
+
+#define PROBE_TRANS_PROFILER_PENDING_EVENT_MODE 	(0x74)
+#define PROBE_TRANS_PROFILER_PENDING_EVENT_MODE_PROFILER_PENDING_EVENT_MODE_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_PENDING_EVENT_MODE_PROFILER_PENDING_EVENT_MODE_WIDTH 0x1
+
+#define PROBE_TRANS_PROFILER_PRE_SCALER 	(0x78)
+#define PROBE_TRANS_PROFILER_PRE_SCALER_PROFILER_PRE_SCALER_OFFSET 0x0
+#define PROBE_TRANS_PROFILER_PRE_SCALER_PROFILER_PRE_SCALER_WIDTH 0x8
+/*trans probe profiler end*/
+
+#define PROBE_FILTERS_ROUTEIDBASE(_filter) \
+	((_filter * 0x3C) + 0x80)
+#define PROBE_FILTERS_ROUTEIDBASE_FILTERS_ROUTEIDBASE_OFFSET 0x0
+#define PROBE_FILTERS_ROUTEIDBASE_FILTERS_ROUTEIDBASE_WIDTH 0x11
+#define PROBE_FILTERS_ROUTEIDBASE_FILTERS_ROUTEIDBASE_MASK 0x1ffff
+#define PROBE_FILTERS_ROUTEIDBASE_FILTERS_ROUTEIDBASE(_reg) \
+			(((_reg) & 0x1ffff) >> 0x0)
+
+#define PROBE_FILTERS_ROUTEIDMASK(_filter) \
+	((_filter * 0x3C) + 0x84)
+#define PROBE_FILTERS_ROUTEIDMASK_FILTERS_ROUTEIDMASK_OFFSET 0x0
+#define PROBE_FILTERS_ROUTEIDMASK_FILTERS_ROUTEIDMASK_WIDTH 0x11
+#define PROBE_FILTERS_ROUTEIDMASK_FILTERS_ROUTEIDMASK_MASK 0x1ffff
+#define PROBE_FILTERS_ROUTEIDMASK_FILTERS_ROUTEIDMASK(_reg) \
+			(((_reg) & 0x1ffff) >> 0x0)
+
+#define PROBE_FILTERS_ADDRBASE_LOW(_filter) \
+	((_filter * 0x3C) + 0x88)
+#define PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET 0x0
+#define PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_WIDTH 0x20
+#define PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_MASK 0xffffffff
+#define PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW(_reg) \
+			(((_reg) & PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_MASK) >> 0x0)
+
+#define PROBE_FILTERS_ADDRBASE_HIGH(_filter) \
+	((_filter * 0x3C) + 0x8C)
+#define PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_OFFSET 0x0
+#define PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_WIDTH 0x09
+#define PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_MASK 0x1ff
+#define PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH(_reg) \
+				(((_reg) & PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_MASK) >> 0x0)
+
+#define PROBE_FILTERS_WINDOWSIZE(_filter) \
+	((_filter * 0x3C) + 0x90)
+#define PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_OFFSET 0x0
+#define PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_WIDTH 0x6
+#define PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_MASK 0x3f
+#define PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE(_reg) \
+			(((_reg) & 0x3f) >> 0x0)
+
+#define PROBE_FILTERS_SECURITYBASE(_filter) \
+	((_filter * 0x3C) + 0x94)
+#define PROBE_FILTERS_SECURITYBASE_FILTERS_SECURITYBASE_OFFSET 0x0
+#define PROBE_FILTERS_SECURITYBASE_FILTERS_SECURITYBASE_WIDTH 0x4
+#define PROBE_FILTERS_SECURITYBASE_FILTERS_SECURITYBASE_MASK 0xf
+#define PROBE_FILTERS_SECURITYBASE_FILTERS_SECURITYBASE(_reg) \
+			(((_reg) & 0xf) >> 0x0)
+
+#define PROBE_FILTERS_SECURITYMASK(_filter) \
+	((_filter * 0x3C) + 0x98)
+#define PROBE_FILTERS_SECURITYMASK_FILTERS_SECURITYMASK_OFFSET 0x0
+#define PROBE_FILTERS_SECURITYMASK_FILTERS_SECURITYMASK_WIDTH 0x4
+#define PROBE_FILTERS_SECURITYMASK_FILTERS_SECURITYMASK_MASK 0xf
+#define PROBE_FILTERS_SECURITYMASK_FILTERS_SECURITYMASK(_reg) \
+			(((_reg) & 0xf) >> 0x0)
+
+#define PROBE_FILTERS_OPCODE(_filter) \
+	((_filter * 0x3C) + 0x9c)
+#define PROBE_FILTERS_OPCODE_RDEN_OFFSET 0x0
+#define PROBE_FILTERS_OPCODE_RDEN_WIDTH 0x1
+#define PROBE_FILTERS_OPCODE_RDEN_MASK 0x1
+#define PROBE_FILTERS_OPCODE_RDEN(_reg) (((_reg) & 0x1) >> 0x0)
+#define PROBE_FILTERS_OPCODE_WREN_OFFSET 0x1
+#define PROBE_FILTERS_OPCODE_WREN_WIDTH 0x1
+#define PROBE_FILTERS_OPCODE_WREN_MASK 0x2
+#define PROBE_FILTERS_OPCODE_WREN(_reg) (((_reg) & 0x2) >> 0x1)
+#define PROBE_FILTERS_OPCODE_LOCKEN_OFFSET 0x2
+#define PROBE_FILTERS_OPCODE_LOCKEN_WIDTH 0x1
+#define PROBE_FILTERS_OPCODE_LOCKEN_MASK 0x4
+#define PROBE_FILTERS_OPCODE_LOCKEN(_reg) (((_reg) & 0x4) >> 0x2)
+#define PROBE_FILTERS_OPCODE_URGEN_OFFSET 0x3
+#define PROBE_FILTERS_OPCODE_URGEN_WIDTH 0x1
+#define PROBE_FILTERS_OPCODE_URGEN_MASK 0x8
+#define PROBE_FILTERS_OPCODE_URGEN(_reg) (((_reg) & 0x8) >> 0x3)
+
+#define PROBE_FILTERS_STATUS(_filter) \
+	((_filter * 0x3C) + 0xA0)
+#define PROBE_FILTERS_STATUS_REQEN_OFFSET 0x0
+#define PROBE_FILTERS_STATUS_REQEN_WIDTH 0x1
+#define PROBE_FILTERS_STATUS_REQEN_MASK 0x1
+#define PROBE_FILTERS_STATUS_REQEN(_reg) (((_reg) & 0x1) >> 0x0)
+#define PROBE_FILTERS_STATUS_RSPEN_OFFSET 0x1
+#define PROBE_FILTERS_STATUS_RSPEN_WIDTH 0x1
+#define PROBE_FILTERS_STATUS_RSPEN_MASK 0x2
+#define PROBE_FILTERS_STATUS_RSPEN(_reg) (((_reg) & 0x2) >> 0x1)
+
+#define PROBE_FILTERS_LENGTH(_filter) \
+	((_filter * 0x3C) + 0xA4)
+#define PROBE_FILTERS_LENGTH_FILTERS_LENGTH_OFFSET 0x0
+#define PROBE_FILTERS_LENGTH_FILTERS_LENGTH_WIDTH 0x4
+#define PROBE_FILTERS_LENGTH_FILTERS_LENGTH_MASK 0xf
+#define PROBE_FILTERS_LENGTH_FILTERS_LENGTH(_reg) (((_reg) & 0xf) >> 0x0)
+
+#define PROBE_FILTERS_URGENCY(_filter) \
+	((_filter * 0x3C) + 0xA8)
+#define PROBE_FILTERS_URGENCY_FILTERS_URGENCY_OFFSET 0x0
+#define PROBE_FILTERS_URGENCY_FILTERS_URGENCY_WIDTH 0x3
+#define PROBE_FILTERS_URGENCY_FILTERS_URGENCY_MASK 0x7
+#define PROBE_FILTERS_URGENCY_FILTERS_URGENCY(_reg) (((_reg) & PROBE_FILTERS_URGENCY_FILTERS_URGENCY_MASK) >> 0x0)
+
+#define PROBE_COUNTERS_PORTSEL(_offset, _probe, _cnt) \
+	((_cnt * 0x10) + 0x200)
+#define PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_OFFSET 0x0
+#define PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_WIDTH 0x3
+#define PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_MASK 0x7
+#define PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL(_reg) (((_reg) & 0x7) >> 0x0)
+
+#define PROBE_COUNTERS_SRC(_cnt) \
+	((_cnt * 0x10) + 0x204)
+#define PROBE_COUNTERS_SRC_INTEVENT_OFFSET 0x0
+#define PROBE_COUNTERS_SRC_INTEVENT_WIDTH 0x6
+#define PROBE_COUNTERS_SRC_INTEVENT_MASK 0x3f
+#define PROBE_COUNTERS_SRC_INTEVENT(_reg) (((_reg) & PROBE_COUNTERS_SRC_INTEVENT_MASK) >> 0x0)
+
+#define PROBE_COUNTERS_ALARMMODE(_cnt) \
+	((_cnt * 0x10) + 0x208)
+#define PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_OFFSET 0x0
+#define PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_WIDTH 0x2
+#define PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_MASK 0x3
+#define PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE(_reg) \
+			(((_reg) & 0x3) >> 0x0)
+
+#define PROBE_COUNTERS_VAL(_cnt) \
+	((_cnt * 0x10) + 0x20C)
+#define PROBE_COUNTERS_VAL_COUNTERS_VAL_OFFSET 0x0
+#define PROBE_COUNTERS_VAL_COUNTERS_VAL_WIDTH 0x20
+#define PROBE_COUNTERS_VAL_COUNTERS_VAL_MASK 0xffff
+#define PROBE_COUNTERS_VAL_COUNTERS_VAL(_reg) (((_reg) & PROBE_COUNTERS_VAL_COUNTERS_VAL_MASK) >> 0x0)
+
+#define SBM_SENSE_IN0 (0xB0)
+#define SBM_SENSE_IN0_SENSE_IN0_OFFSET 0x0
+#define SBM_SENSE_IN0_SENSE_IN0_WIDTH 0x1
+#define SBM_SENSE_IN0_SENSE_IN0_MASK 0xffff
+#define SBM_SENSE_IN0_SENSE_IN0(_reg) (((_reg) & SBM_SENSE_IN0_SENSE_IN0_MASK) >> 0x0)
+
+#define OBS_L1_ATB_ID_COREID(_base) ((_base) + 0x1c00)
+#define OBS_L1_ATB_ID_COREID_CORETYPEID_OFFSET 0x0
+#define OBS_L1_ATB_ID_COREID_CORETYPEID_WIDTH 0x8
+#define OBS_L1_ATB_ID_COREID_CORETYPEID_MASK 0xff
+#define OBS_L1_ATB_ID_COREID_CORETYPEID(_reg) (((_reg) & 0xff) >> 0x0)
+#define OBS_L1_ATB_ID_COREID_CORECHECKSUM_OFFSET 0x8
+#define OBS_L1_ATB_ID_COREID_CORECHECKSUM_WIDTH 0x18
+#define OBS_L1_ATB_ID_COREID_CORECHECKSUM_MASK 0xffffff00
+#define OBS_L1_ATB_ID_COREID_CORECHECKSUM(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define OBS_L1_ATB_ID_REVISIONID(_base) ((_base) + 0x1c04)
+#define OBS_L1_ATB_ID_REVISIONID_USERID_OFFSET 0x0
+#define OBS_L1_ATB_ID_REVISIONID_USERID_WIDTH 0x8
+#define OBS_L1_ATB_ID_REVISIONID_USERID_MASK 0xff
+#define OBS_L1_ATB_ID_REVISIONID_USERID(_reg) (((_reg) & 0xff) >> 0x0)
+#define OBS_L1_ATB_ID_REVISIONID_FLEXNOCID_OFFSET 0x8
+#define OBS_L1_ATB_ID_REVISIONID_FLEXNOCID_WIDTH 0x18
+#define OBS_L1_ATB_ID_REVISIONID_FLEXNOCID_MASK 0xffffff00
+#define OBS_L1_ATB_ID_REVISIONID_FLEXNOCID(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define OBS_L1_ATB_ATBID(_base) ((_base) + 0x1c08)
+#define OBS_L1_ATB_ATBID_ATBID_OFFSET 0x0
+#define OBS_L1_ATB_ATBID_ATBID_WIDTH 0x7
+#define OBS_L1_ATB_ATBID_ATBID_MASK 0x7f
+#define OBS_L1_ATB_ATBID_ATBID(_reg) (((_reg) & 0x7f) >> 0x0)
+
+#define OBS_L1_ATB_ATBEN(_base) ((_base) + 0x1c0c)
+#define OBS_L1_ATB_ATBEN_ATBEN_OFFSET 0x0
+#define OBS_L1_ATB_ATBEN_ATBEN_WIDTH 0x1
+#define OBS_L1_ATB_ATBEN_ATBEN_MASK 0x1
+#define OBS_L1_ATB_ATBEN_ATBEN(_reg) (((_reg) & 0x1) >> 0x0)
+
+#define OBS_L1_ATB_SYNCPERIOD(_base) ((_base) + 0x1c10)
+#define OBS_L1_ATB_SYNCPERIOD_SYNCPERIOD_OFFSET 0x0
+#define OBS_L1_ATB_SYNCPERIOD_SYNCPERIOD_WIDTH 0x5
+#define OBS_L1_ATB_SYNCPERIOD_SYNCPERIOD_MASK 0x1f
+#define OBS_L1_ATB_SYNCPERIOD_SYNCPERIOD(_reg) (((_reg) & 0x1f) >> 0x0)
+
+#define GPU_QOS_GEN_ID_COREID(_base) ((_base) + 0x1d00)
+#define GPU_QOS_GEN_ID_COREID_CORETYPEID_OFFSET 0x0
+#define GPU_QOS_GEN_ID_COREID_CORETYPEID_WIDTH 0x8
+#define GPU_QOS_GEN_ID_COREID_CORETYPEID_MASK 0xff
+#define GPU_QOS_GEN_ID_COREID_CORETYPEID(_reg) (((_reg) & 0xff) >> 0x0)
+#define GPU_QOS_GEN_ID_COREID_CORECHECKSUM_OFFSET 0x8
+#define GPU_QOS_GEN_ID_COREID_CORECHECKSUM_WIDTH 0x18
+#define GPU_QOS_GEN_ID_COREID_CORECHECKSUM_MASK 0xffffff00
+#define GPU_QOS_GEN_ID_COREID_CORECHECKSUM(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define GPU_QOS_GEN_ID_REVISIONID(_base) ((_base) + 0x1d04)
+#define GPU_QOS_GEN_ID_REVISIONID_USERID_OFFSET 0x0
+#define GPU_QOS_GEN_ID_REVISIONID_USERID_WIDTH 0x8
+#define GPU_QOS_GEN_ID_REVISIONID_USERID_MASK 0xff
+#define GPU_QOS_GEN_ID_REVISIONID_USERID(_reg) (((_reg) & 0xff) >> 0x0)
+#define GPU_QOS_GEN_ID_REVISIONID_FLEXNOCID_OFFSET 0x8
+#define GPU_QOS_GEN_ID_REVISIONID_FLEXNOCID_WIDTH 0x18
+#define GPU_QOS_GEN_ID_REVISIONID_FLEXNOCID_MASK 0xffffff00
+#define GPU_QOS_GEN_ID_REVISIONID_FLEXNOCID(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define GPU_QOS_GEN_PRIORITY 0x8
+#define GPU_QOS_GEN_PRIORITY_P0_OFFSET 0x0
+#define GPU_QOS_GEN_PRIORITY_P0_WIDTH 0x3
+#define GPU_QOS_GEN_PRIORITY_P0_MASK 0x7
+#define GPU_QOS_GEN_PRIORITY_P0(_reg) (((_reg) & 0x7) >> 0x0)
+#define GPU_QOS_GEN_PRIORITY_P1_OFFSET 0x8
+#define GPU_QOS_GEN_PRIORITY_P1_WIDTH 0x3
+#define GPU_QOS_GEN_PRIORITY_P1_MASK 0x700
+#define GPU_QOS_GEN_PRIORITY_P1(_reg) (((_reg) & 0x700) >> 0x8)
+
+#define GPU_QOS_GEN_MODE 0xc
+#define GPU_QOS_GEN_MODE_MODE_OFFSET 0x0
+#define GPU_QOS_GEN_MODE_MODE_WIDTH 0x2
+#define GPU_QOS_GEN_MODE_MODE_MASK 0x3
+#define GPU_QOS_GEN_MODE_MODE(_reg) (((_reg) & 0x3) >> 0x0)
+
+#define GPU_QOS_GEN_BANDWIDTH 0x10
+#define GPU_QOS_GEN_BANDWIDTH_BANDWIDTH_OFFSET 0x0
+#define GPU_QOS_GEN_BANDWIDTH_BANDWIDTH_WIDTH 0xf
+#define GPU_QOS_GEN_BANDWIDTH_BANDWIDTH_MASK 0x7ff
+#define GPU_QOS_GEN_BANDWIDTH_BANDWIDTH(_reg) (((_reg) & 0x7ff) >> 0x0)
+
+#define GPU_QOS_GEN_SATURATION 0x14
+#define GPU_QOS_GEN_SATURATION_SATURATION_OFFSET 0x0
+#define GPU_QOS_GEN_SATURATION_SATURATION_WIDTH 0xa
+#define GPU_QOS_GEN_SATURATION_SATURATION_MASK 0x3ff
+#define GPU_QOS_GEN_SATURATION_SATURATION(_reg) (((_reg) & 0x3ff) >> 0x0)
+
+#define GPU_QOS_GEN_EXTCONTROL 0x18
+#define GPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_OFFSET 0x0
+#define GPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_WIDTH 0x1
+#define GPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_MASK 0x1
+#define GPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN(_reg) (((_reg) & 0x1) >> 0x0)
+#define GPU_QOS_GEN_EXTCONTROL_EXTTHREN_OFFSET 0x1
+#define GPU_QOS_GEN_EXTCONTROL_EXTTHREN_WIDTH 0x1
+#define GPU_QOS_GEN_EXTCONTROL_EXTTHREN_MASK 0x2
+#define GPU_QOS_GEN_EXTCONTROL_EXTTHREN(_reg) (((_reg) & 0x2) >> 0x1)
+#define GPU_QOS_GEN_EXTCONTROL_INTCLKEN_OFFSET 0x2
+#define GPU_QOS_GEN_EXTCONTROL_INTCLKEN_WIDTH 0x1
+#define GPU_QOS_GEN_EXTCONTROL_INTCLKEN_MASK 0x4
+#define GPU_QOS_GEN_EXTCONTROL_INTCLKEN(_reg) (((_reg) & 0x4) >> 0x2)
+#define GPU_QOS_GEN_EXTCONTROL_EXTLIMITEN_OFFSET 0x3
+#define GPU_QOS_GEN_EXTCONTROL_EXTLIMITEN_WIDTH 0x1
+#define GPU_QOS_GEN_EXTCONTROL_EXTLIMITEN_MASK 0x8
+#define GPU_QOS_GEN_EXTCONTROL_EXTLIMITEN(_reg) (((_reg) & 0x8) >> 0x2)
+
+#define VPU_QOS_GEN_ID_COREID(_base) ((_base) + 0x1e00)
+#define VPU_QOS_GEN_ID_COREID_CORETYPEID_OFFSET 0x0
+#define VPU_QOS_GEN_ID_COREID_CORETYPEID_WIDTH 0x8
+#define VPU_QOS_GEN_ID_COREID_CORETYPEID_MASK 0xff
+#define VPU_QOS_GEN_ID_COREID_CORETYPEID(_reg) (((_reg) & 0xff) >> 0x0)
+#define VPU_QOS_GEN_ID_COREID_CORECHECKSUM_OFFSET 0x8
+#define VPU_QOS_GEN_ID_COREID_CORECHECKSUM_WIDTH 0x18
+#define VPU_QOS_GEN_ID_COREID_CORECHECKSUM_MASK 0xffffff00
+#define VPU_QOS_GEN_ID_COREID_CORECHECKSUM(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define VPU_QOS_GEN_ID_REVISIONID(_base) ((_base) + 0x1e04)
+#define VPU_QOS_GEN_ID_REVISIONID_USERID_OFFSET 0x0
+#define VPU_QOS_GEN_ID_REVISIONID_USERID_WIDTH 0x8
+#define VPU_QOS_GEN_ID_REVISIONID_USERID_MASK 0xff
+#define VPU_QOS_GEN_ID_REVISIONID_USERID(_reg) (((_reg) & 0xff) >> 0x0)
+#define VPU_QOS_GEN_ID_REVISIONID_FLEXNOCID_OFFSET 0x8
+#define VPU_QOS_GEN_ID_REVISIONID_FLEXNOCID_WIDTH 0x18
+#define VPU_QOS_GEN_ID_REVISIONID_FLEXNOCID_MASK 0xffffff00
+#define VPU_QOS_GEN_ID_REVISIONID_FLEXNOCID(_reg) (((_reg) & 0xffffff00) >> 0x8)
+
+#define VPU_QOS_GEN_PRIORITY(_base) ((_base) + 0x1e08)
+#define VPU_QOS_GEN_PRIORITY_P0_OFFSET 0x0
+#define VPU_QOS_GEN_PRIORITY_P0_WIDTH 0x2
+#define VPU_QOS_GEN_PRIORITY_P0_MASK 0x3
+#define VPU_QOS_GEN_PRIORITY_P0(_reg) (((_reg) & 0x3) >> 0x0)
+#define VPU_QOS_GEN_PRIORITY_P1_OFFSET 0x2
+#define VPU_QOS_GEN_PRIORITY_P1_WIDTH 0x2
+#define VPU_QOS_GEN_PRIORITY_P1_MASK 0xc
+#define VPU_QOS_GEN_PRIORITY_P1(_reg) (((_reg) & 0xc) >> 0x2)
+
+#define VPU_QOS_GEN_MODE(_base) ((_base) + 0x1e0c)
+#define VPU_QOS_GEN_MODE_MODE_OFFSET 0x0
+#define VPU_QOS_GEN_MODE_MODE_WIDTH 0x2
+#define VPU_QOS_GEN_MODE_MODE_MASK 0x3
+#define VPU_QOS_GEN_MODE_MODE(_reg) (((_reg) & 0x3) >> 0x0)
+
+#define VPU_QOS_GEN_BANDWIDTH(_base) ((_base) + 0x1e10)
+#define VPU_QOS_GEN_BANDWIDTH_BANDWIDTH_OFFSET 0x0
+#define VPU_QOS_GEN_BANDWIDTH_BANDWIDTH_WIDTH 0xb
+#define VPU_QOS_GEN_BANDWIDTH_BANDWIDTH_MASK 0x7ff
+#define VPU_QOS_GEN_BANDWIDTH_BANDWIDTH(_reg) (((_reg) & 0x7ff) >> 0x0)
+
+#define VPU_QOS_GEN_SATURATION(_base) ((_base) + 0x1e14)
+#define VPU_QOS_GEN_SATURATION_SATURATION_OFFSET 0x0
+#define VPU_QOS_GEN_SATURATION_SATURATION_WIDTH 0xa
+#define VPU_QOS_GEN_SATURATION_SATURATION_MASK 0x3ff
+#define VPU_QOS_GEN_SATURATION_SATURATION(_reg) (((_reg) & 0x3ff) >> 0x0)
+
+#define VPU_QOS_GEN_EXTCONTROL(_base) ((_base) + 0x1e18)
+#define VPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_OFFSET 0x0
+#define VPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_WIDTH 0x1
+#define VPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN_MASK 0x1
+#define VPU_QOS_GEN_EXTCONTROL_SOCKETQOSEN(_reg) (((_reg) & 0x1) >> 0x0)
+#define VPU_QOS_GEN_EXTCONTROL_EXTTHREN_OFFSET 0x1
+#define VPU_QOS_GEN_EXTCONTROL_EXTTHREN_WIDTH 0x1
+#define VPU_QOS_GEN_EXTCONTROL_EXTTHREN_MASK 0x2
+#define VPU_QOS_GEN_EXTCONTROL_EXTTHREN(_reg) (((_reg) & 0x2) >> 0x1)
+#define VPU_QOS_GEN_EXTCONTROL_INTCLKEN_OFFSET 0x2
+#define VPU_QOS_GEN_EXTCONTROL_INTCLKEN_WIDTH 0x1
+#define VPU_QOS_GEN_EXTCONTROL_INTCLKEN_MASK 0x4
+#define VPU_QOS_GEN_EXTCONTROL_INTCLKEN(_reg) (((_reg) & 0x4) >> 0x2)
+#endif
diff --git a/drivers/interconnect/eswin/noc_stat.c b/drivers/interconnect/eswin/noc_stat.c
new file mode 100644
index 000000000000..a4f6a90f1426
--- /dev/null
+++ b/drivers/interconnect/eswin/noc_stat.c
@@ -0,0 +1,966 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/of.h>
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/bitfield.h>
+
+#include "noc.h"
+#include "noc_regs.h"
+
+//#pragma GCC optimize("O0")
+
+/**
+ * Create a new stat_measure from parsed dts entry
+ * @param stat_probe
+ * @return The created win2030_noc_stat_measure object
+ */
+static struct win2030_noc_stat_measure *new_stat_measure(
+				struct win2030_noc_stat_probe *stat_probe)
+{
+	struct win2030_noc_stat_measure *stat_measure;
+	unsigned long flags;
+
+	stat_measure = kzalloc(sizeof(struct win2030_noc_stat_measure),
+			       GFP_KERNEL);
+	if (!stat_measure)
+		return ERR_PTR(-ENOMEM);
+	INIT_LIST_HEAD(&stat_measure->link);
+	spin_lock_init(&stat_measure->lock);
+
+	spin_lock_irqsave(&stat_probe->lock, flags);
+	list_add_tail(&stat_measure->link, &stat_probe->measure);
+	spin_unlock_irqrestore(&stat_probe->lock, flags);
+
+	return stat_measure;
+}
+
+/**
+ * Create a new stat_probe. A stat_probe contains all measurements which
+ * are linked to the same probe
+ * @param noc_stat the stat_probe will be attached to noc_stat
+ * @param probe the stat_probe will refer to probe
+ * @return The created xgold_noc_stat_probe object
+ */
+static struct win2030_noc_stat_probe *new_stat_probe(struct win2030_noc_stat
+						   *noc_stat,
+						   struct win2030_noc_probe
+						   *probe)
+{
+	struct win2030_noc_stat_probe *stat_probe;
+	unsigned long flags;
+
+	stat_probe = kzalloc(sizeof(struct win2030_noc_stat_probe), GFP_KERNEL);
+	if (!stat_probe)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&stat_probe->measure);
+	INIT_LIST_HEAD(&stat_probe->link);
+	spin_lock_init(&stat_probe->lock);
+
+	spin_lock_irqsave(&noc_stat->lock, flags);
+	list_add_tail(&stat_probe->link, &noc_stat->probe);
+	spin_unlock_irqrestore(&noc_stat->lock, flags);
+	stat_probe->probe = probe;
+
+	return stat_probe;
+}
+
+enum {
+	FIELD_ID_TRACE_PORT = 0,
+	FIELD_ID_INIT_FLOW,
+	FIELD_ID_TARGET_FLOW,
+	FIELD_ID_ADDR_BASE,
+	FIELD_ID_ADDR_SIZE,
+	FIELD_ID_OPCODE,
+	FIELD_ID_STATUS,
+	FIELD_ID_LENGTH,
+	FIELD_ID_URGENCY,
+	FIELD_ID_MAX,
+};
+
+static int win2030_noc_stat_parse_traceport_field(struct win2030_noc_probe *probe,
+	struct device *dev, const char *str, int field_id,
+	struct win2030_noc_stat_traceport_data *data)
+{
+	int i;
+	u64 vaule;
+	int ret;
+	struct win2030_register *reg;
+	struct win2030_bitfield *bf = NULL;
+	const char *str_tmp;
+	bool found = false;
+
+	switch (field_id) {
+	case FIELD_ID_INIT_FLOW:
+		str_tmp = &str[strlen("InitFlow:")];
+		reg = probe->filters[0].route_id_base;
+		found = false;
+		list_for_each_entry(bf, &reg->bitfields, link) {
+			if (strcmp(bf->name, "InitFlow") == 0) {
+				i = 0;
+				while (bf->lut[i] != NULL) {
+					if (str_tmp && strcmp(str_tmp, bf->lut[i]) == 0) {
+						data->init_flow_name = kstrdup(str_tmp, GFP_KERNEL);
+						data->init_flow_idx = i;
+						data->init_flow_mask = bf->mask;
+						data->init_flow_offset = bf->offset;
+						found = true;
+						break;
+					}
+					i += 1;
+				}
+			}
+		}
+		if (false == found) {
+			dev_err(dev, "get unknown InitFlow data %s when parsing traceport dts!\n", str);
+			return -EINVAL;
+		}
+		break;
+	case FIELD_ID_TARGET_FLOW:
+		str_tmp = &str[strlen("TargetFlow:")];
+		reg = probe->filters[0].route_id_base;
+		found = false;
+		list_for_each_entry(bf, &reg->bitfields, link) {
+			if (strcmp(bf->name, "TargetFlow") == 0) {
+				i = 0;
+				while (bf->lut[i] != NULL) {
+					if (str_tmp && strcmp(str_tmp, bf->lut[i]) == 0) {
+						data->target_flow_name = kstrdup(str_tmp, GFP_KERNEL);
+						data->target_flow_idx = i;
+						data->target_flow_mask = bf->mask;
+						data->target_flow_offset = bf->offset;
+						found = true;
+						break;
+					}
+					i += 1;
+				}
+			}
+		}
+		if (false == found) {
+			dev_err(dev, "get unknown TargetFlow data %s when parsing traceport dts!\n", str);
+			return -EINVAL;
+		}
+		break;
+	case FIELD_ID_ADDR_BASE:
+		ret = kstrtou64(&str[strlen("AddrBase:")], 0, &vaule);
+		if (ret) {
+			dev_err(dev, "get unknown AddrBase data %s when parsing traceport dts!\n", str);
+			return ret;
+		}
+		data->addrBase_low = FIELD_GET(GENMASK(31, 0), vaule);
+		data->addrBase_high = FIELD_GET(GENMASK(40, 32), vaule);
+		break;
+	case FIELD_ID_ADDR_SIZE:
+		ret = kstrtou64(&str[strlen("AddrSize:")], 0, &vaule);
+		if (ret) {
+			dev_err(dev, "get unknown AddrSize data %s when parsing traceport dts!\n", str);
+			return ret;
+		}
+		data->addrWindowSize = vaule;
+		/*should not exceed the max vaule*/
+		data->addrWindowSize &= PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_MASK;
+		break;
+	case FIELD_ID_OPCODE:
+		if (0 == strcmp(str, "Opcode:RdWrLockUrg")) {
+			data->Opcode = 0xf;
+		} else if (0 == strcmp(str, "Opcode:WrLockUrg")) {
+			data->Opcode = 0xe;
+		} else if (0 == strcmp(str, "Opcode:RdLockUrg")) {
+			data->Opcode = 0xd;
+		} else if (0 == strcmp(str, "Opcode:LockUrg")) {
+			data->Opcode = 0xc;
+		} else if (0 == strcmp(str, "Opcode:RdWrUrg")) {
+			data->Opcode = 0xb;
+		} else if (0 == strcmp(str, "Opcode:WrUrg")) {
+			data->Opcode = 0xa;
+		} else if (0 == strcmp(str, "Opcode:RdUrg")) {
+			data->Opcode = 0x9;
+		} else if (0 == strcmp(str, "Opcode:Urg")) {
+			data->Opcode = 0x8;
+		} else if (0 == strcmp(str, "Opcode:RdWrLock")) {
+			data->Opcode = 0x7;
+		} else if (0 == strcmp(str, "Opcode:WrLock")) {
+			data->Opcode = 0x6;
+		} else if (0 == strcmp(str, "Opcode:RdLock")) {
+			data->Opcode = 0x5;
+		} else if (0 == strcmp(str, "Opcode:Lock")) {
+			data->Opcode = 0x4;
+		} else if (0 == strcmp(str, "Opcode:RdWr")) {
+			data->Opcode = 0x3;
+		} else if (0 == strcmp(str, "Opcode:Wr")) {
+			data->Opcode = 0x2;
+		} else if (0 == strcmp(str, "Opcode:Rd")) {
+			data->Opcode = 0x1;
+		} else {
+			dev_err(dev, "get unknown Opcode data %s when parsing traceport dts!\n", str);
+			return -EINVAL;
+		}
+		break;
+
+	case FIELD_ID_STATUS:
+		if (0 == strcmp(str, "Status:ReqRsp")) {
+			data->Status = 3;
+		} else if (0 == strcmp(str, "Status:Req")) {
+			data->Status = 1;
+		} else if (0 == strcmp(str, "Status:Rsp")) {
+			data->Status = 2;
+		} else {
+			dev_err(dev, "get unknown Status data %s when parsing traceport dts!\n", str);
+			return -EINVAL;
+		}
+		break;
+
+	case FIELD_ID_LENGTH:
+		ret = kstrtou64(&str[strlen("Length:")], 0, &vaule);
+		if (ret) {
+			dev_err(dev, "get unknown length data %s when parsing traceport dts!\n", str);
+			return ret;
+		}
+		data->Length = ilog2(vaule);
+		data->Length &= PROBE_FILTERS_LENGTH_FILTERS_LENGTH_MASK;
+		break;
+
+	case FIELD_ID_URGENCY:
+		ret = kstrtou64(&str[strlen("Urgency:")], 0, &vaule);
+		if (ret) {
+			dev_err(dev, "unknown length data %s when parsing traceport dts!\n", str);
+			return ret;
+		}
+		data->Urgency = vaule;
+		data->Urgency &= PROBE_FILTERS_URGENCY_FILTERS_URGENCY_MASK;
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+	return 0;
+}
+
+static int win2030_noc_stat_parse_traceport_dts(struct win2030_noc_probe **probe_out,
+	struct win2030_noc_device *noc_device, struct device_node *np,
+	const char *name, struct win2030_noc_stat_traceport_data *data)
+{
+	struct property *prop;
+	struct device *dev = noc_device->dev;
+	int lg;
+	int index = 0;
+	const char *str[FIELD_ID_MAX] = {NULL}, *s;
+	int ret;
+	const char *field[] = {"TracePort:", "InitFlow:", "TargetFlow:",
+				"AddrBase:", "AddrSize:", "Opcode:", "Status:", "Length:", "Urgency:"};
+	int i, j;
+	bool found = false;
+	struct win2030_noc_probe *probe = NULL;
+	const char *str_tmp;
+
+	/* Count number of strings per measurement
+	  (normally 1: TracePortSel)
+	 */
+	lg = of_property_count_strings(np, name);
+	if (lg > FIELD_ID_MAX || lg < 1)
+		return -EINVAL; /* Invalid data, let's skip this measurement */
+
+	of_property_for_each_string(np, name, prop, s)
+		str[index++] = s;
+
+	str_tmp = &str[0][strlen("TracePort:")];
+	list_for_each_entry(probe, &noc_device->probes, link) {
+		for (j = 0; probe->available_portsel[j] != NULL; j++) {
+			if (str_tmp && strcmp(str_tmp, probe->available_portsel[j]) == 0)
+				break;
+		}
+		if (probe->available_portsel[j] != NULL) {
+			found = true;
+			break;
+		}
+	}
+	if (false == found)
+		return -EINVAL;	/* Invalid data, let's skip this measurement. */
+
+	*probe_out = probe;
+	data->name = str_tmp;
+	data->idx_trace_port_sel = j;
+
+	/*if user not set, default set as below
+		route_id_mask: -1, which means all flow
+		target_flow_idx: -1, which means all flow
+		addrBase: 0, which means all address
+		addrWindowSize : 0, which means all range
+		Opcode: 0xf, which means enable Rd,Wr,Lock,Urg
+		Status:0x3,which means Req,Rsp
+		Length:0xf, which means the max packets length could be 0x10000
+		Urgency:0x0, which means all urgency level be selected
+	*/
+	data->init_flow_name = NULL;
+	data->init_flow_idx = -1;
+	data->init_flow_offset = -1;
+	data->init_flow_mask = -1;
+	data->target_flow_name = NULL;
+	data->target_flow_idx = -1;
+	data->target_flow_offset = -1;
+	data->target_flow_mask = -1;
+	data->addrBase_low = 0;
+	data->addrBase_high = 0;
+	data->addrWindowSize = 0x3f;
+	data->Opcode = 0xf;
+	data->Status = 0x3;
+	data->Length = 0xF;
+	data->Urgency = 0x0;
+
+	for (i = 1; i < lg; i++) {
+		for (j = 0; j < ARRAY_SIZE(field); j++) {
+			if (0 == strncmp(str[i], field[j], strlen(field[j]))) {
+				ret = win2030_noc_stat_parse_traceport_field(probe, dev, str[i], j, data);
+				if (ret) {
+					dev_err(dev, "error when parsing traceport dts, ret %d!\n", ret);
+					return ret;
+				}
+			}
+		}
+	}
+
+	dev_dbg(dev, "get traceport data, name: %s\n"
+		"\t\t\t\t init_flow: %s init_flow_idx: 0x%x, init_flow_offset: 0x%x, init_flow_mask: 0x%x\n"
+		"\t\t\t\t target_flow: %s, target_flow_idx: 0x%x, target_flow_offset: 0x%x, target_flow_mask: 0x%x\n"
+		"\t\t\t\t addrBase_low: 0x%x, addrBase_high: 0x%x, addrWindowSize: 0x%x\n"
+		"\t\t\t\t Opcode: 0x%x, Status: 0x%x, Length: 0x%llx, Urgency: 0x%x!\n",
+		data->name,
+		data->init_flow_name == NULL ? "not set" : data->init_flow_name, data->init_flow_idx, data->init_flow_offset,data->init_flow_mask,
+		data->target_flow_name == NULL ? "not set" : data->target_flow_name, data->target_flow_idx, data->target_flow_offset, data->target_flow_mask,
+		data->addrBase_low, data->addrBase_high, data->addrWindowSize, data->Opcode, data->Status,
+		int_pow(2, data->Length), data->Urgency);
+
+	return 0;
+}
+
+/**
+ * From one line of dts, add a new stat_measure and a stat_probe (if needed)
+ * @param noc_device A ref to the device
+ * @param np A ref to the dts node
+ * @param name the line of the dts
+ * @return
+ */
+static int new_probe_and_measure(struct win2030_noc_device *noc_device,
+				 struct device_node *np, const char *name)
+{
+	struct win2030_noc_stat_probe *stat_probe, *entry = NULL;
+	struct win2030_noc_stat *noc_stat = noc_device->stat;
+	struct win2030_noc_probe *probe = NULL;
+	struct win2030_noc_stat_measure *stat_measure;
+	struct win2030_noc_stat_traceport_data data;
+	bool found = false;
+	int ret;
+
+	ret = win2030_noc_stat_parse_traceport_dts(&probe, noc_device, np, name, &data);
+	if (ret) {
+		return ret;
+	}
+
+	/* Look if the probe was already existing in noc_stat */
+	if (!list_empty(&noc_stat->probe)) {
+		found = false;
+		list_for_each_entry(entry, &noc_stat->probe, link) {
+			if (entry->probe == probe) {
+				/* It is found, create a new measure */
+				stat_measure = new_stat_measure(entry);
+				if (IS_ERR(stat_measure))
+					goto error;
+				found = true;
+				break;
+			}
+		}
+		if (found == false) {
+			/* Not found, we must create a stat_probe */
+			stat_probe = new_stat_probe(noc_stat, probe);
+			if (IS_ERR(stat_probe))
+				goto error;
+			/* Then add the measurement */
+			stat_measure = new_stat_measure(stat_probe);
+			if (IS_ERR(stat_measure))
+				goto free_stat_probe;
+		}
+	} else {
+		/* No stat_probe created up to now, let's create one. */
+		stat_probe = new_stat_probe(noc_stat, probe);
+		if (IS_ERR(stat_probe))
+			goto error;
+		/* Then add the measurement */
+		stat_measure = new_stat_measure(stat_probe);
+		if (IS_ERR(stat_measure))
+			goto free_stat_probe;
+	}
+	memcpy(&stat_measure->data, &data, sizeof(struct win2030_noc_stat_traceport_data));
+	return 0;
+
+free_stat_probe:
+	kfree(stat_probe);
+error:
+	return -ENOMEM;
+}
+
+/**
+ * Return the value of a bitfield
+ * @param bf the bitfield to read
+ * @return the value of the bitfield
+ */
+static unsigned win2030_noc_bf_read(struct win2030_bitfield *bf)
+{
+	struct win2030_register *reg;
+	unsigned bf_value;
+
+	reg = bf->parent;
+	bf_value = ioread32(WIN2030_NOC_REG_ADDR(reg));
+	bf_value &= bf->mask;
+	bf_value >>= bf->offset;
+
+	return bf_value;
+}
+
+/**
+ * Write some bits in a register
+ * @param reg The register to write
+ * @param mask The mask for the bits (shifted)
+ * @param offset The offset of the bits
+ * @param value The value to write
+ * @return 0 if ok, -EINVAL if value is not valid
+ */
+int win2030_noc_reg_write(struct win2030_register *reg, unsigned mask,
+			       unsigned offset, unsigned value)
+{
+	unsigned reg_value;
+	unsigned long flags;
+
+	if (value > (mask >> offset)) {
+		dev_err(reg->parent, "reg_write, error para, name %s, value 0x%x, mask 0x%x, offset 0x%x!\n",
+			reg->name, value, mask, offset);
+		return -EINVAL;
+	}
+	spin_lock_irqsave(&reg->hw_lock, flags);
+	reg_value = ioread32(WIN2030_NOC_REG_ADDR(reg));
+	reg_value &= ~mask;
+	reg_value |= (value << offset);
+	iowrite32(reg_value, WIN2030_NOC_REG_ADDR(reg));
+	spin_unlock_irqrestore(&reg->hw_lock, flags);
+
+	return 0;
+}
+
+/**
+ * Read some bits in a register
+ * @param reg The register to read
+ * @param mask The mask for the bits (shifted)
+ * @param offset The offset of the bits
+ * @return the read value
+ */
+unsigned win2030_noc_reg_read(struct win2030_register *reg, unsigned mask,
+				   unsigned offset)
+{
+	unsigned reg_value;
+
+	reg_value = ioread32(WIN2030_NOC_REG_ADDR(reg));
+	reg_value &= ~mask;
+	reg_value >>= offset;
+
+	return reg_value;
+}
+
+int win2030_noc_stat_packet_probe_launch_measure(
+			struct win2030_noc_device *noc_device,
+			struct win2030_noc_stat_probe *stat_probe)
+{
+	struct win2030_noc_stat_measure *stat_measure;
+	struct win2030_noc_probe *probe;
+	int i, ret = 0;
+	struct win2030_noc_stat_traceport_data *data;
+
+	stat_measure = list_first_entry(&stat_probe->measure,
+					struct win2030_noc_stat_measure, link);
+	data = &stat_measure->data;
+
+	probe = stat_probe->probe;
+
+	/* Set Filters */
+	for (i = 0; i < probe->nr_filters; i++) {
+		/*Address*/
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].addr_base_low->parent,
+			PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_MASK,
+			PROBE_FILTERS_ADDRBASE_LOW_FILTERS_ADDRBASE_LOW_OFFSET,
+			data->addrBase_low);
+
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].addr_base_high->parent,
+			PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_MASK,
+			PROBE_FILTERS_ADDRBASE_HIGH_FILTERS_ADDRBASE_HIGH_OFFSET,
+			data->addrBase_high);
+
+		/*Windowsize*/
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].window_size->parent,
+			PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_MASK,
+			PROBE_FILTERS_WINDOWSIZE_FILTERS_WINDOWSIZE_OFFSET,
+			data->addrWindowSize);
+
+		/*opcode*/
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].op_code, 0xf, 0, data->Opcode);
+
+		/*status*/
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].status, 0x3, 0, data->Status);
+
+		/*length*/
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].length->parent,
+				PROBE_FILTERS_LENGTH_FILTERS_LENGTH_MASK,
+				PROBE_FILTERS_LENGTH_FILTERS_LENGTH_OFFSET,
+				data->Length);
+
+		/*urgency*/
+		ret |=
+		    win2030_noc_reg_write(probe->filters[i].urgency->parent,
+				PROBE_FILTERS_URGENCY_FILTERS_URGENCY_MASK,
+				PROBE_FILTERS_URGENCY_FILTERS_URGENCY_OFFSET,
+				data->Urgency);
+	}
+
+	if (data->init_flow_idx != 0xff) {
+		ret |= win2030_noc_reg_write(probe->filters[0].route_id_mask->parent,
+					   data->init_flow_mask, data->init_flow_offset,
+					   (data->init_flow_mask >> data->init_flow_offset));
+
+		ret |= win2030_noc_reg_write(probe->filters[0].route_id_base,
+					   data->init_flow_mask, data->init_flow_offset,
+					   data->init_flow_idx);
+	}
+	if (data->target_flow_idx != 0xff) {
+		ret |= win2030_noc_reg_write(probe->filters[0].route_id_mask->parent,
+					   data->target_flow_mask, data->target_flow_offset,
+					   (data->target_flow_mask >> data->target_flow_offset));
+
+		ret |= win2030_noc_reg_write(probe->filters[0].route_id_base,
+					   data->target_flow_mask, data->target_flow_offset,
+					   data->target_flow_idx);
+	}
+	//if (data->init_flow_idx != 0xff || data->target_flow_idx != 0xff) {
+		/* Enable Filter LUT */
+		/* To select only filter 0 */
+		ret |= win2030_noc_reg_write(probe->filter_lut,
+				PROBE_FILTERLUT_FILTERLUT_MASK,
+				PROBE_FILTERLUT_FILTERLUT_OFFSET,
+				0x1);
+	//}
+	/* Enable Alarm and statistics */
+	ret |= win2030_noc_reg_write(probe->main_ctl, PROBE_MAINCTL_STATEN_MASK,
+				PROBE_MAINCTL_STATEN_OFFSET, 1);
+
+	ret |= win2030_noc_reg_write(probe->main_ctl, PROBE_MAINCTL_ALARMEN_MASK,
+				PROBE_MAINCTL_ALARMEN_OFFSET, 1);
+
+	/* Set period to max value */
+	ret |= win2030_noc_reg_write(probe->stat_period,
+				PROBE_STATPERIOD_STATPERIOD_MASK,
+				PROBE_STATPERIOD_STATPERIOD_OFFSET,
+				DURATION);
+
+	/* Select sources for counter0 and 1 */
+	//if (data->init_flow_idx != 0xff || data->target_flow_idx != 0xff) {
+		ret |= win2030_noc_reg_write(probe->main_ctl,
+				PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_MASK,
+				PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_OFFSET,
+				1);
+
+		ret |=
+		    win2030_noc_reg_write(probe->counters[0].source_event->parent,
+				PROBE_COUNTERS_SRC_INTEVENT_MASK,
+				/*FILT_BYTE = 0x14 */
+				PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+				0x14);
+		ret |=
+		    win2030_noc_reg_write(probe->counters[1].source_event->parent,
+				PROBE_COUNTERS_SRC_INTEVENT_MASK,
+				/* chain */
+				PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+				0x10);
+	#if 0
+	} else {
+		ret |= win2030_noc_reg_write(probe->main_ctl,
+				PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_MASK,
+				PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_OFFSET,
+				0);
+		ret |=
+		    win2030_noc_reg_write(probe->counters[0].source_event->parent,
+				PROBE_COUNTERS_SRC_INTEVENT_MASK,
+				/* BYTE */
+				PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+				0x8);
+		ret |=
+		    win2030_noc_reg_write(probe->counters[1].source_event->parent,
+				PROBE_COUNTERS_SRC_INTEVENT_MASK,
+				/* chain */
+				PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+				0x10);
+	}
+	#endif
+	/* Set alarm mode */
+	/* min */
+	ret |= win2030_noc_reg_write(probe->counters[0].alarm_mode->parent,
+			   PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_MASK,
+			   PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_OFFSET,
+			   1);
+	return ret;
+}
+
+int win2030_noc_stat_tranc_probe_launch_measure(
+			struct win2030_noc_device *noc_device,
+			struct win2030_noc_stat_probe *stat_probe)
+{
+	struct win2030_noc_stat_measure *stat_measure;
+	struct win2030_noc_probe *probe;
+	int i, ret = 0;
+
+	stat_measure = list_first_entry(&stat_probe->measure,
+					struct win2030_noc_stat_measure, link);
+
+	probe = stat_probe->probe;
+
+	/* Reset Filters */
+	for (i = 0; i < probe->nr_filters; i++) {
+		/*select all kinds opcode*/
+		ret |=
+		    win2030_noc_reg_write(probe->trans_filters[i].op_code, 0xf, 0, 0xf);
+	}
+
+	/* Enable Alarm and statistics */
+	ret |= win2030_noc_reg_write(probe->main_ctl, PROBE_MAINCTL_STATEN_MASK,
+			PROBE_MAINCTL_STATEN_OFFSET, 1);
+
+	ret |= win2030_noc_reg_write(probe->main_ctl, PROBE_MAINCTL_ALARMEN_MASK,
+			PROBE_MAINCTL_ALARMEN_OFFSET, 1);
+
+	/* Set period to max value */
+	ret |= win2030_noc_reg_write(probe->stat_period,
+			PROBE_STATPERIOD_STATPERIOD_MASK,
+			PROBE_STATPERIOD_STATPERIOD_OFFSET,
+			DURATION);
+
+	/* map counters to trace port*/
+	ret |= win2030_noc_reg_write(probe->main_ctl,
+			PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_MASK,
+			PROBE_MAINCTL_FILT_BYTE_ALWAYS_CAHINABLE_EN_OFFSET,
+			1);
+
+	for (i = 0; i < probe->nr_portsel; i++) {
+		ret |=
+		    win2030_noc_reg_write(probe->counters[i * 2].source_event->parent,
+					PROBE_COUNTERS_SRC_INTEVENT_MASK,
+					/* BYTE */
+					PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+					0x8);
+		/* Set alarm mode min */
+		ret |=
+		    win2030_noc_reg_write(probe->counters[i * 2].alarm_mode->parent,
+				   PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_MASK,
+				   PROBE_COUNTERS_ALARMMODE_COUNTERS_ALARMMODE_OFFSET,
+				   1);
+		ret |=
+		    win2030_noc_reg_write(probe->counters[i * 2 + 1].source_event->parent,
+					PROBE_COUNTERS_SRC_INTEVENT_MASK,
+					/* chain */
+					PROBE_COUNTERS_SRC_INTEVENT_OFFSET,
+					0x10);
+	}
+	return ret;
+}
+
+/**
+ * Configure NOC registers from a stat_measure
+ * @param noc_device A reference to the device
+ * @param stat_probe The stat_probe to find the measure
+ * @return 0 if ok, -EINVAL otherwise
+ */
+
+static int win2030_noc_stat_launch_measure(struct win2030_noc_device *noc_device,
+				     struct win2030_noc_stat_probe *stat_probe)
+{
+	struct win2030_noc_stat_measure *stat_measure;
+	struct win2030_noc_probe *probe;
+	int ret = 0;
+
+	stat_measure = list_first_entry(&stat_probe->measure,
+					struct win2030_noc_stat_measure, link);
+
+	dev_dbg_once(noc_device->dev, "stat measurment %s begin\n", stat_measure->data.name);
+	probe = stat_probe->probe;
+	/* Disable NOC */
+	ret = win2030_noc_reg_write(probe->cfg_ctl, PROBE_CFGCTL_GLOBALEN_MASK,
+				  PROBE_CFGCTL_GLOBALEN_OFFSET, 0);
+	while (win2030_noc_reg_read(probe->cfg_ctl, PROBE_CFGCTL_ACTIVE_WIDTH,
+				  PROBE_CFGCTL_ACTIVE_OFFSET) != 0)
+		;
+#if 0
+	/*there is only one probe point, so there is no need to choose probe point*/
+	if (stat_measure->idx_init_flow != 0xff) {
+		/* Select probe TracePortSel if need filter*/
+		ret |= win2030_noc_reg_write(probe->trace_port_sel,
+				   PROBE_TRACEPORTSEL_TRACEPORTSEL_MASK,
+				   PROBE_TRACEPORTSEL_TRACEPORTSEL_OFFSET,
+				   stat_measure->idx_trace_port_sel);
+	} else {
+		/* Select counter PortSel if no need filter*/
+		ret |= win2030_noc_reg_write(probe->counters[0].portsel->parent,
+			   PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_MASK,
+			   PROBE_COUNTERS_PORTSEL_COUNTERS_PORTSEL_OFFSET,
+			   stat_measure->idx_trace_port_sel);
+	}
+#endif
+	if (probe_t_pkt == probe->type) {
+		ret |= win2030_noc_stat_packet_probe_launch_measure(noc_device, stat_probe);
+	} else if (probe_t_trans == probe->type) {
+		ret |= win2030_noc_stat_tranc_probe_launch_measure(noc_device, stat_probe);
+	}
+
+	/* Set alarm min value */
+	ret |= win2030_noc_reg_write(probe->stat_alarm_min,
+			   PROBE_STATALARMMIN_STATALARMMIN_MASK,
+			   PROBE_STATALARMMIN_STATALARMMIN_OFFSET,
+			   0xfffffff);
+
+	ret |= win2030_noc_reg_write(probe->stat_alarm_min_high,
+			   PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_MASK,
+			   PROBE_STATALARMMIN_HIGH_STATALARMMIN_HIGH_OFFSET,
+			   0xfffffff);
+
+	/* Enable */
+	ret |= win2030_noc_reg_write(probe->stat_alarm_en,
+			   PROBE_STATALARMEN_STATALARMEN_MASK,
+			   PROBE_STATALARMEN_STATALARMEN_OFFSET,
+			   1);
+
+	ret |= win2030_noc_reg_write(probe->cfg_ctl,
+			   PROBE_CFGCTL_GLOBALEN_MASK,
+			   PROBE_CFGCTL_GLOBALEN_OFFSET,
+			   1);
+	return ret;
+}
+
+void win2030_noc_stat_reset_statistics(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct win2030_noc_stat_probe *stat_probe = NULL;
+	struct win2030_noc_stat_measure *stat_measure = NULL;
+	unsigned long flags;
+	int j;
+	struct win2030_noc_probe *probe;
+
+	list_for_each_entry(stat_probe, &noc_device->stat->probe, link) {
+		probe = stat_probe->probe;
+		list_for_each_entry(stat_measure, &stat_probe->measure, link) {
+			for (j = 0; j < probe->nr_portsel; j++) {
+				spin_lock_irqsave(&stat_measure->lock, flags);
+				stat_measure->min[j] = -1;
+				stat_measure->max[j] = 0;
+				stat_measure->now[j] = 0;
+				stat_measure->iteration[j] = 0;
+				stat_measure->mean[j] = 0;
+				spin_unlock_irqrestore(&stat_measure->lock,
+						       flags);
+			}
+
+		}
+	}
+}
+
+/**
+ * Program all noc registers for all probes
+ * @param _dev A reference to the device
+ * @return 0
+ */
+int win2030_noc_stat_trigger(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct win2030_noc_stat_probe *stat_probe = NULL;
+	struct win2030_noc_stat *noc_stat = noc_device->stat;
+	struct list_head *ptr = NULL;
+	unsigned ret;
+
+	/* Loop over all probes */
+	list_for_each(ptr, &noc_stat->probe) {
+		stat_probe = list_entry(ptr, struct win2030_noc_stat_probe, link);
+		/* Program first measurement */
+		ret = win2030_noc_stat_launch_measure(noc_device, stat_probe);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+/**
+ * Entry point for the NOC sniffer
+ * @param _dev A reference to the device
+ * @return 0 if ok, if error returns 0 but print an error message
+ */
+int win2030_noc_stat_init(struct device *_dev)
+{
+	struct win2030_noc_stat *noc_stat;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct device_node *np = _dev->of_node;
+	int stat_idx = 0;
+	bool ret;
+	char mystr[32];
+	int err;
+
+	/* Create a stat device. */
+	noc_stat = kzalloc(sizeof(struct win2030_noc_stat), GFP_KERNEL);
+	if (!noc_stat)
+		return -ENOMEM;
+	noc_device->stat = noc_stat;
+	noc_stat->run = false;
+	spin_lock_init(&noc_stat->lock);
+	INIT_LIST_HEAD(&noc_stat->probe);
+
+	/* Parse dts and build matrix node */
+	do {
+		scnprintf(mystr, ARRAY_SIZE(mystr), "stat,%d", stat_idx);
+		ret = of_property_read_bool(np, mystr);
+		if (ret == true) {
+			stat_idx++;
+			err = new_probe_and_measure(noc_device, np, mystr);
+			if (err)
+				dev_err(_dev,
+					"Error when adding measure: %s!\n",
+					mystr);
+		}
+	} while (ret == true);
+
+	/* If no stat line in dts then we can release the noc_stat object */
+	if (stat_idx == 0)
+		goto no_sniffer;
+
+	ret = of_property_read_u32(np, "clock,rate", &noc_stat->clock_rate);
+	if (ret) {
+		dev_err(_dev,
+			"\"clock,rate\" property missing. Will use clk_get_rate() "
+			"to discover clock rate.");
+	}
+
+	/* Trigger measurement */
+	if (noc_stat->run) {
+		ret = win2030_noc_stat_trigger(_dev);
+		if (ret)
+			dev_err(_dev,
+				"Error when trigger stat measures!\n");
+	}
+	return 0;
+
+no_sniffer:
+	kfree(noc_stat);
+	return -EINVAL;
+}
+
+/**
+ * Interrupt processing when a measurement is done
+ * @param noc_device A reference to the device
+ * @param probe The probe for which a measurement is done
+ */
+void win2030_noc_stat_do_measure(struct win2030_noc_probe *probe)
+{
+	struct win2030_noc_device *noc_device = probe->parent;
+	struct win2030_noc_stat_probe *stat_probe = NULL;
+	struct win2030_noc_stat_measure *stat_measure;
+	u64 counter = 0;
+	unsigned ret;
+	unsigned long flags;
+	int j;
+
+	/* Find which stat_probe is linked to the probe
+	 * which generated the interrupt
+	 */
+	list_for_each_entry(stat_probe, &noc_device->stat->probe, link) {
+		if (stat_probe->probe == probe)
+			break;
+	}
+
+	/* Take measure object */
+	stat_measure = list_first_entry(&stat_probe->measure,
+				struct win2030_noc_stat_measure, link);
+
+	for (j = 0; j < probe->nr_portsel; j++) {
+		/* Read HW counters */
+		counter = win2030_noc_bf_read(probe->counters[j * 2  + 1].value);
+		counter = ((counter << 32)
+			   | (win2030_noc_bf_read(probe->counters[j * 2].value)));
+
+		/* Update stat_measure statistics */
+		if (counter > stat_measure->max[j * 2]) {
+			spin_lock_irqsave(&stat_measure->lock, flags);
+			stat_measure->max[j * 2] = counter;
+			spin_unlock_irqrestore(&stat_measure->lock, flags);
+		}
+		if (counter < stat_measure->min[j * 2]) {
+			spin_lock_irqsave(&stat_measure->lock, flags);
+			stat_measure->min[j * 2] = counter;
+			spin_unlock_irqrestore(&stat_measure->lock, flags);
+		}
+		stat_measure->now[j * 2] = counter;
+		/*if no traffic, don't update iteration and mean value*/
+		if (0 == counter) {
+			continue;
+		}
+		if (stat_measure->iteration[j * 2] != MAX_ITERATION) {
+			spin_lock_irqsave(&stat_measure->lock, flags);
+			stat_measure->iteration[j * 2] += 1;
+			spin_unlock_irqrestore(&stat_measure->lock, flags);
+
+			spin_lock_irqsave(&stat_measure->lock, flags);
+			stat_measure->mean[j * 2] += counter;
+			spin_unlock_irqrestore(&stat_measure->lock, flags);
+		} else {
+			dev_info(noc_device->dev, "TracePort %s iteration overflow, reset measure statistics!\n",
+				stat_probe->probe->available_portsel[j]);
+			spin_lock_irqsave(&stat_measure->lock, flags);
+			stat_measure->iteration[j * 2] = 1;
+			stat_measure->mean[j * 2] = 0;
+			spin_unlock_irqrestore(&stat_measure->lock, flags);
+		}
+	}
+	/* Program next measure */
+	if (noc_device->stat->run == true) {
+		/* Move to the next measure */
+		spin_lock_irqsave(&stat_probe->lock, flags);
+		list_move(&stat_measure->link, &stat_probe->measure);
+		spin_unlock_irqrestore(&stat_probe->lock, flags);
+
+		/* Program it */
+		ret = win2030_noc_stat_launch_measure(noc_device, stat_probe);
+		if (ret)
+			dev_err(noc_device->dev,
+				"Error when launch stat measures!\n");
+	}
+}
diff --git a/drivers/interconnect/eswin/noc_stat_sysfs.c b/drivers/interconnect/eswin/noc_stat_sysfs.c
new file mode 100644
index 000000000000..891e627dc2bf
--- /dev/null
+++ b/drivers/interconnect/eswin/noc_stat_sysfs.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/device.h>
+#include <linux/of.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <asm/div64.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+
+#include "noc.h"
+
+/**
+ * Called when writing run file
+ */
+static ssize_t noc_stat_run_write(struct file *filp, const char __user *ubuf,
+				  size_t cnt, loff_t *ppos)
+{
+	struct win2030_noc_device *noc_device = filp->private_data;
+	char buf[SIZE_SMALL_BUF];
+	unsigned ret;
+
+	if (cnt > SIZE_SMALL_BUF)
+		cnt = SIZE_SMALL_BUF - 1;
+
+	if (copy_from_user(&buf, ubuf, cnt))
+		return -EFAULT;
+
+	if (buf[0] == '0') {
+		noc_device->stat->run = false;	/* FIXME spinlock? */
+		/*wait for the latest measure to be finished*/
+		msleep(5);
+		win2030_noc_stat_reset_statistics(noc_device->dev);
+	} else if (buf[0] == '1') {
+		if (true != noc_device->stat->run) {
+			noc_device->stat->run = true;
+			win2030_noc_stat_reset_statistics(noc_device->dev);
+			ret = win2030_noc_stat_trigger(noc_device->dev);
+			if (ret)
+				dev_err(noc_device->dev, "Error when trigger stat measures!\n");
+		}
+	} else
+		return -EFAULT;
+
+	*ppos += cnt;
+
+	return cnt;
+}
+
+/**
+ * Called when reading run file
+ */
+static ssize_t noc_stat_run_read(struct file *filp, char __user *ubuf,
+				 size_t cnt, loff_t *ppos)
+{
+#define RUN_STR_SIZE 11
+	struct win2030_noc_device *noc_device = filp->private_data;
+	char buf[RUN_STR_SIZE];
+	int r;
+
+	r = snprintf(buf, RUN_STR_SIZE, "%i\n", noc_device->stat->run);
+
+	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+/**
+ * Called when writing config file
+ */
+static ssize_t noc_stat_config_write(struct file *filp,
+				     const char __user *ubuf, size_t cnt,
+				     loff_t *ppos)
+{
+	return 0;
+}
+
+/**
+ * Called when reading config file
+ */
+static ssize_t noc_stat_config_read(struct file *filp, char __user *ubuf,
+				    size_t cnt, loff_t *ppos)
+{
+	return 0;
+}
+
+/**
+ * Called when reading readme file
+ * @param m
+ * @param p
+ * @return
+ */
+static int noc_stat_readme_show(struct seq_file *m, void *p)
+{
+	seq_puts(m,
+		 "How to use the NOC sniffer:\nFile run: write '1' to enable "
+		 "sniffer (enabling sniffer resets statistics),"
+		 " write '0' to disable sniffer.\n"
+		 "File config: Not yet implemented.\n"
+		 "File results: contain the results of measurement in KB/s "
+		 "for all nodes defined in dts.\n");
+	return 0;
+}
+
+/**
+ * Called when reading results file.
+ */
+static ssize_t noc_stat_results_read(struct file *filp, char __user *ubuf,
+				     size_t cnt, loff_t *ppos)
+{
+	struct win2030_noc_device *noc_device = filp->private_data;
+	struct win2030_noc_probe *probe;
+	struct win2030_noc_stat *stat;
+	struct win2030_noc_stat_probe *stat_probe = NULL;
+	struct win2030_noc_stat_measure *stat_measure = NULL;
+	u64 meas_min, meas_max, meas_mean, meas_now;
+	u64 clock_noc = 0;
+	u64 period_cycle = int_pow(2, DURATION);
+	int j;
+	char *buf1;
+	char buf2[SIZE_SMALL_BUF + 2];
+	int r;
+	ssize_t ret;
+	int count;		/* Used to avoid write over buffer size */
+	struct win2030_noc_stat_traceport_data *data;
+
+	//extern int win2030_noc_get_error(struct win2030_noc_device *noc_device);
+	//win2030_noc_get_error(noc_device);
+
+	if (noc_device == NULL)
+		return -ENODEV;
+
+	stat = noc_device->stat;
+	if (!stat)
+		return -ENODEV;
+	clock_noc = (u64) stat->clock_rate;
+
+	buf1 = kzalloc(sizeof(*buf1) * SIZE_BIG_BUF, GFP_KERNEL);
+	if (!buf1)
+		return -ENODEV;
+
+	r = snprintf(buf1, SIZE_BIG_BUF,"\t##### RESULTS #####\n\n");
+
+	count = SIZE_BIG_BUF - strlen(buf1);
+	list_for_each_entry(stat_probe, &stat->probe, link) {
+		probe = stat_probe->probe;
+		if (clock_noc == 0) {
+			clock_noc = (u64) clk_get_rate(probe->clock);
+		}
+		list_for_each_entry(stat_measure, &stat_probe->measure, link) {
+			data = &stat_measure->data;
+			if (count > 0) {
+				if (data->init_flow_name != NULL)
+					r = snprintf(buf2, SIZE_SMALL_BUF,
+						     "Probe %s, clk %llu, (%s): %s-->%s\n",
+						     stat_probe->probe->id,
+						     clock_noc,
+						     stat_probe->probe->available_portsel[data->idx_trace_port_sel],
+						     data->init_flow_name,
+						     data->target_flow_name);
+				else
+					r = snprintf(buf2, SIZE_SMALL_BUF,
+						     "Probe %s, clk %llu, (%s):\n",
+						     stat_probe->probe->id,
+						     clock_noc,
+						     stat_probe->probe->available_portsel[data->idx_trace_port_sel]);
+
+				strncat(buf1, buf2, count);
+				count -= strlen(buf2);
+			}
+			for (j = 0; j < probe->nr_portsel; j++) {
+				r = snprintf(buf2, SIZE_SMALL_BUF,
+					"\tTracePort %s, NbOfMeasure=%i\n",
+					stat_probe->probe->available_portsel[j],
+					stat_measure->iteration[j]);
+				strncat(buf1, buf2, count);
+				count -= strlen(buf2);
+
+				if (count > 0) {
+					if (stat_measure->iteration[j] != 0) {
+						meas_min = stat_measure->min[j];
+						meas_max = stat_measure->max[j];
+						meas_mean = stat_measure->mean[j];
+						do_div(meas_mean,
+						       stat_measure->iteration[j]);
+						meas_now = stat_measure->now[j];
+						r = snprintf(buf2, SIZE_SMALL_BUF,
+							     "\t\t total=%llu (KB), period time %lld (us), total time %lld (s)\n"
+							     "\t\t min=%llu (KB/s), max=%llu (KB/s) mean=%llu (KB/s) current=%llu (KB/s)\n\n",
+						     stat_measure->mean[j] / 1000,
+						     period_cycle * 1000 * 1000 / clock_noc,
+						     (stat_measure->iteration[j] * period_cycle) / clock_noc,
+						     ((meas_min * clock_noc) / (period_cycle * 1000)),
+						     ((meas_max * clock_noc) / (period_cycle * 1000)),
+						     ((meas_mean * clock_noc) / (period_cycle * 1000)),
+						     ((meas_now * clock_noc) / (period_cycle * 1000)));
+					} else {
+						r = snprintf(buf2, SIZE_SMALL_BUF,
+						     "\t\t No results\n\n");
+					}
+					strncat(buf1, buf2, count);
+					count -= strlen(buf2);
+				}
+			}
+		}
+	}
+
+	r = strlen(buf1) + 1;
+
+	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf1, r);
+
+	kfree(buf1);
+	return ret;
+
+}
+
+/**
+ * Called when open results file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_stat_results_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * Called when opening config file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_stat_config_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * Called when opening run file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_stat_run_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * Called when reading readme file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_stat_readme_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, noc_stat_readme_show, inode->i_private);
+}
+
+static const struct file_operations noc_stat_results_fops = {
+	.open = noc_stat_results_open,
+	.read = noc_stat_results_read,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_stat_config_fops = {
+	.open = noc_stat_config_open,
+	.read = noc_stat_config_read,
+	.write = noc_stat_config_write,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_stat_run_fops = {
+	.open = noc_stat_run_open,
+	.read = noc_stat_run_read,
+	.write = noc_stat_run_write,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_stat_readme_fops = {
+	.open = noc_stat_readme_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+/**
+ * Entry point for NOC sniffer sysfs
+ * @param _dev A reference to the device
+ * @return 0
+ */
+int win2030_noc_stat_debugfs_init(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct device_node *np = _dev->of_node;
+	struct dentry *dir, *d;
+	char name[32];
+
+	scnprintf(name, ARRAY_SIZE(name), "%s_stat", np->name);
+
+	dir = debugfs_create_dir(name, win2030_noc_ctrl.win2030_noc_root_debug_dir);
+	if (IS_ERR(dir))
+		return PTR_ERR(dir);
+
+	noc_device->stat->dir = dir;
+
+	d = debugfs_create_file("results", S_IRUGO, dir, noc_device,
+				&noc_stat_results_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+	d = debugfs_create_file("config", S_IRUGO | S_IWUSR, dir, noc_device,
+				&noc_stat_config_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+	d = debugfs_create_file("run", S_IRUGO | S_IWUSR, dir, noc_device,
+				&noc_stat_run_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+	d = debugfs_create_file("readme.txt", S_IRUGO, dir, noc_device,
+				&noc_stat_readme_fops);
+	if (IS_ERR(d))
+		return PTR_ERR(d);
+
+	return 0;
+}
diff --git a/drivers/interconnect/eswin/noc_sysfs.c b/drivers/interconnect/eswin/noc_sysfs.c
new file mode 100644
index 000000000000..fac895eedcb6
--- /dev/null
+++ b/drivers/interconnect/eswin/noc_sysfs.c
@@ -0,0 +1,783 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#include <linux/ctype.h>
+#include <linux/io.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/interrupt.h>
+
+#include "noc.h"
+
+#define MAX_ENUM_SIZE 254
+
+static int win2030_noc_user_flag_dump(char *buf,
+	struct win2030_bitfield *bitfield, unsigned long bf_value)
+{
+	int i;
+	int count = 1;
+
+	for (i = 0; i < bitfield->length; i++) {
+		if (test_bit(i, (unsigned long *)&bf_value)) {
+			strncat(buf, bitfield->lut[i], strlen(bitfield->lut[i]));
+			strncat(buf, "|", 1);
+			count += strlen(bitfield->lut[i]);
+		}
+	}
+	count--;
+	return count;
+}
+
+static void win2030_bitfield_debug_print(char *buf,
+				struct win2030_bitfield *bitfield,
+				unsigned value,
+				const char *prefix)
+{
+	unsigned bf_value = (value & bitfield->mask)
+		>> bitfield->offset;
+	int count = 0;
+	char buf2[SIZE_SMALL_BUF + 2];
+
+	if (bitfield->lut) {
+		if (!strcmp(bitfield->name, "User_flag")) {
+			count = sprintf(buf2, "%s%s: ", prefix, bitfield->name);
+			count += win2030_noc_user_flag_dump(buf2, bitfield, bf_value);
+		} else {
+			count = sprintf(buf2, "%s%s: %s", prefix,
+				  bitfield->name,
+				  bitfield->lut[bf_value]);
+		}
+	} else {
+		count = sprintf(buf2, "%s%s: %x", prefix,
+				  bitfield->name, bf_value);
+	}
+	strncat(buf, buf2, count);
+	strncat(buf, "\n", 1);
+	return;
+}
+
+static int win2030_register_debug_print(char *buf,
+				struct win2030_register *reg,
+				unsigned value,
+				const char *prefix)
+{
+	struct win2030_bitfield *bf = NULL;
+	char myprefix[64];
+	int count = 0;
+	char buf2[SIZE_SMALL_BUF + 2];
+
+	scnprintf(myprefix, ARRAY_SIZE(myprefix), "\t%s", prefix);
+	count = sprintf(buf2, "%s%s: %#x\n", prefix, reg->name, value);
+	strncat(buf, buf2, count);
+
+	list_for_each_entry(bf, &reg->bitfields, link) {
+		win2030_bitfield_debug_print(buf, bf, value,
+			myprefix);
+	}
+	return 0;
+}
+
+int noc_debug_open(struct inode *inode, struct file *filp)
+{
+	filp->private_data = inode->i_private;
+	return 0;
+}
+static int noc_err_get_bitfield_vaule(struct win2030_register *reg,
+		int reg_vaule, char *bitfield_name)
+{
+	struct win2030_bitfield *bf = NULL;
+	int bitfield_vaule = -1;
+
+	list_for_each_entry(bf, &reg->bitfields, link) {
+		if (!strcmp(bf->name, bitfield_name)) {
+			bitfield_vaule = (reg_vaule & bf->mask) >> bf->offset;
+			break;
+		}
+	}
+	return bitfield_vaule;
+}
+
+static void win2030_noc_err_addr_dump(char *buf, struct win2030_noc_device *noc_device,
+					struct win2030_noc_error *noc_err,
+					int reg_index,
+					const char *prefix)
+{
+	struct win2030_register *reg1, *reg3, *reg4;
+	u32 vaule_init_flow, vaule_target_flow, vaule_target_sub_range;
+	struct win2030_bitfield *bf = NULL;
+	struct win2030_bitfield *bf_AbsoluteAddress;
+	unsigned vaule_addr_msb;
+	u64 vaule_addr;
+	int i;
+	unsigned vaule_err_RouteId;
+	bool found = false;
+	char myprefix[64];
+	char buf2[SIZE_SMALL_BUF + 2];
+	int count = 0;
+
+	scnprintf(myprefix, ARRAY_SIZE(myprefix), "\t%s", prefix);
+
+	reg3 = noc_device->error_registers[reg_index];
+
+	/*Get the InitFlow, TargetFlow, TargetSubRange of the err*/
+	vaule_err_RouteId = noc_err->err[reg3->aperture_link];
+
+	reg1 = noc_device->error_registers[reg3->aperture_link];
+	vaule_init_flow = noc_err_get_bitfield_vaule(reg1, vaule_err_RouteId, "InitFlow");
+	vaule_target_flow = noc_err_get_bitfield_vaule(reg1, vaule_err_RouteId, "TargetFlow");
+	vaule_target_sub_range = noc_err_get_bitfield_vaule(reg1, vaule_err_RouteId, "TargetSubRange");
+
+	vaule_addr = noc_err->err[reg_index];
+
+	if (-1 != reg3->msb_link) {
+		/*if addr msb exist, get it*/
+		vaule_addr_msb = noc_err->err[reg3->msb_link];
+		reg4 = noc_device->error_registers[reg3->msb_link];
+		list_for_each_entry(bf, &reg4->bitfields, link) {
+			if (!strcmp(bf->name, "addr_msb")) {
+				vaule_addr |= ((u64)(vaule_addr_msb & bf->mask)) << 32;
+				break;
+			}
+		}
+	}
+	/*check if any recorded RouteId match the err pkt's RouteId*/
+	list_for_each_entry(bf, &reg3->bitfields, link) {
+		if (!strcmp(bf->name, "AbsoluteAddress")) {
+			bf_AbsoluteAddress = bf;
+			for (i = 0; i < bf_AbsoluteAddress->aperture_size; i++) {
+				if (bf_AbsoluteAddress->target_sub_range[i] == vaule_target_sub_range
+					&& bf_AbsoluteAddress->target_flow[i] == vaule_target_flow
+					&& bf_AbsoluteAddress->init_flow[i] == vaule_init_flow) {
+						found = true;
+						break;
+					}
+			}
+		}
+	}
+
+	if (true == found) {
+		vaule_addr |= bf_AbsoluteAddress->aperture_base[i];
+		count = sprintf(buf2, "%s%s: 0x%llx\n", prefix, bf_AbsoluteAddress->name, vaule_addr);
+	} else {
+		count = sprintf(buf2, "%s%s: 0x%llx\n", prefix, "OffsetAddr", vaule_addr);
+	}
+	strncat(buf, buf2, count);
+	return;
+}
+
+int noc_error_dump(char *buf,
+		struct win2030_noc_device *noc_device,
+		struct win2030_noc_error *noc_err)
+{
+	unsigned i;
+	unsigned err;
+	struct win2030_register *reg;
+	int count = 0;
+	char buf2[SIZE_SMALL_BUF + 2];
+
+	count = sprintf(buf2, "timestamp: %lld:\n", noc_err->timestamp);
+	strncat(buf, buf2, count);
+	for (i = 0; i < noc_device->error_logger_cnt; i++) {
+		reg = noc_device->error_registers[noc_device->err_log_lut[i]];
+		if (-1 != reg->aperture_link) {
+			/*should ErrLog3 come here
+			  get err_init_target_subrange form ErrLog1
+			  if ErrLog4 exist, add it to the addr vaule
+			*/
+			win2030_noc_err_addr_dump(buf, noc_device, noc_err,
+				noc_device->err_log_lut[i], "\t");
+		}
+		err = noc_err->err[noc_device->err_log_lut[i]];
+		win2030_register_debug_print(buf, reg, err, "\t");
+	}
+	return 0;
+}
+
+static const char *win2030_bitfield_get_enum_from_value(struct win2030_bitfield *bf,
+						      unsigned value)
+{
+	const char **lut = bf->lut;
+	const char *mystr;
+	unsigned i;
+
+	if (lut == NULL)
+		return ERR_PTR(-EINVAL);
+
+	for (mystr = lut[0], i = 0; mystr; mystr = lut[++i])
+		;
+
+	if (value > i)
+		return ERR_PTR(-EINVAL);
+
+	return lut[value];
+}
+
+int win2030_bitfield_read(struct win2030_bitfield *bf, unsigned *value)
+{
+	struct win2030_register *reg = bf->parent;
+
+	if (reg == NULL)
+		return -EINVAL;
+
+	if (reg->base == NULL)
+		return -EINVAL;
+
+	if (value == NULL)
+		return -EINVAL;
+
+	*value = ioread32(WIN2030_NOC_REG_ADDR(reg));
+	*value &= bf->mask;
+	*value >>= bf->offset;
+
+	dev_dbg_once(reg->parent,
+		"Read Register 0x%08x @0x%px (%s), bitfield (%s) "
+		"val=0x%x, %d bit at offset %d, mask %x\n",
+		ioread32(WIN2030_NOC_REG_ADDR(reg)), WIN2030_NOC_REG_ADDR(reg),
+		reg->name, bf->name, *value, bf->length, bf->offset, bf->mask);
+
+	return 0;
+}
+
+static const char *win2030_bitfield_read_enum(struct win2030_bitfield *bf)
+{
+	int ret;
+	unsigned value;
+
+	ret = win2030_bitfield_read(bf, &value);
+	if (ret)
+		return NULL;
+
+	return win2030_bitfield_get_enum_from_value(bf, value);
+}
+
+static int win2030_bitfield_get_value_from_enum(struct win2030_bitfield *bf,
+					      const char *str_value,
+					      unsigned *value)
+{
+	const char **lut = bf->lut;
+	const char *mystr;
+	unsigned i;
+
+	if (lut == NULL) {
+		if (sscanf(str_value, "%x", value) == 1)
+			return 0;
+
+		return -EINVAL;
+	}
+
+	for (mystr = lut[0], i = 0; mystr; mystr = lut[++i]) {
+		if (!(strcmp(str_value, mystr))) {
+			*value = i;
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int win2030_bitfield_write(struct win2030_bitfield *bf, unsigned value)
+{
+	unsigned long flags;
+	unsigned reg_value;
+	struct win2030_register *reg = bf->parent;
+
+	if (reg == NULL)
+		return -EINVAL;
+
+	if (reg->base == NULL)
+		return -EINVAL;
+
+	/*if (value >= (BIT(bf->length)-1)) */
+	/*      return -EINVAL; */
+
+	spin_lock_irqsave(&reg->hw_lock, flags);
+	reg_value = ioread32(WIN2030_NOC_REG_ADDR(reg));
+	reg_value &= ~bf->mask;
+	reg_value |= (value << bf->offset);
+	iowrite32(reg_value, WIN2030_NOC_REG_ADDR(reg));
+	spin_unlock_irqrestore(&reg->hw_lock, flags);
+
+	dev_dbg(reg->parent,
+		"Write Register 0x%08x @%p (%s), bitfield (%s) val=0x%x"
+		", %d bit at offset %d, mask %x\n",
+		reg_value, WIN2030_NOC_REG_ADDR(reg), reg->name, bf->name,
+		value, bf->length, bf->offset, bf->mask);
+	return 0;
+}
+
+static int win2030_bitfield_write_enum(struct win2030_bitfield *bf, const char *str)
+{
+	int value, ret;
+
+	ret = win2030_bitfield_get_value_from_enum(bf, str, &value);
+	if (ret)
+		return ret;
+
+	ret = win2030_bitfield_write(bf, value);
+
+	return ret;
+}
+
+static int noc_debug_available_show(struct seq_file *m, void *p)
+{
+	const char **to_show = m->private;
+	const char *str;
+	int i = 0;
+	for (str = to_show[0]; str; str = to_show[++i])
+		seq_printf(m, "%s ", str);
+
+	seq_puts(m, "\r\n");
+	return 0;
+}
+
+static int noc_debug_available_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, noc_debug_available_show, inode->i_private);
+}
+
+static ssize_t noc_debug_bf_write(struct file *filp, const char __user *ubuf,
+				  size_t cnt, loff_t *ppos)
+{
+	struct win2030_bitfield *bf = filp->private_data;
+	char buf[MAX_ENUM_SIZE + 1];
+	int i;
+	size_t ret;
+	int err;
+
+	ret = cnt;
+
+	if (cnt > MAX_ENUM_SIZE)
+		cnt = MAX_ENUM_SIZE;
+
+	if (copy_from_user(&buf, ubuf, cnt))
+		return -EFAULT;
+
+	buf[cnt] = 0;
+
+	/* strip ending whitespace. */
+	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
+		buf[i] = 0;
+
+	if (bf->lut) {
+		err = win2030_bitfield_write_enum(bf, buf);
+		if (err)
+			return err;
+	} else {
+		unsigned value = 0;
+		err = sscanf(buf, "%x\n", &value);
+		/*if (err) */
+		/*      return -ENODEV; */
+		err = win2030_bitfield_write(bf, value);
+		if (err)
+			return -ENODEV;
+	}
+	*ppos += ret;
+
+	return ret;
+}
+
+static ssize_t noc_debug_bf_read(struct file *filp, char __user *ubuf,
+				 size_t cnt, loff_t *ppos)
+{
+	struct win2030_bitfield *bf = filp->private_data;
+	char buf[MAX_ENUM_SIZE + 2];
+	int r;
+	const char *str;
+	unsigned value;
+
+	if (bf == NULL)
+		return -ENODEV;
+
+	if (bf->lut) {
+		str = win2030_bitfield_read_enum(bf);
+		if (IS_ERR_OR_NULL(str))
+			str = "UNKNOWN BITFIELD VALUE";
+		r = sprintf(buf, "%s\n", str);
+	} else {
+		r = win2030_bitfield_read(bf, &value);
+		if (r)
+			return -ENODEV;
+
+		r = sprintf(buf, "%#x\n", value);
+	}
+
+	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static int noc_debug_error_show(struct seq_file *m, void *p)
+{
+	struct win2030_noc_device *noc_device = m->private;
+	struct win2030_noc_error *noc_err = NULL;
+	unsigned long flags;
+	char *buf;
+
+	buf = kzalloc(sizeof(*buf) * SIZE_BIG_BUF, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&noc_device->lock, flags);
+	list_for_each_entry(noc_err, &noc_device->err_queue, link) {
+		spin_unlock_irqrestore(&noc_device->lock, flags);
+		noc_error_dump(buf, noc_device, noc_err);
+		spin_lock_irqsave(&noc_device->lock, flags);
+	}
+	spin_unlock_irqrestore(&noc_device->lock, flags);
+	seq_printf(m, "%s\n", buf);
+	kfree(buf);
+	return 0;
+}
+
+static int noc_debug_error_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, noc_debug_error_show, inode->i_private);
+}
+
+/**
+ * Called when writing run file
+ */
+static ssize_t noc_qos_run_write(struct file *filp, const char __user *ubuf,
+				  size_t cnt, loff_t *ppos)
+{
+	struct win2030_noc_device *noc_device = filp->private_data;
+	char buf[SIZE_SMALL_BUF];
+	unsigned ret;
+
+	if (cnt > SIZE_SMALL_BUF)
+		cnt = SIZE_SMALL_BUF - 1;
+
+	if (copy_from_user(&buf, ubuf, cnt))
+		return -EFAULT;
+
+	if (buf[0] == '0') {
+		/* FIXME spinlock? */
+		if (false != noc_device->qos_run) {
+			noc_device->qos_run = false;
+			ret = noc_device_qos_set(noc_device, false);
+			if (ret)
+				dev_err(noc_device->dev, "Error when disable qos config!\n");
+		}
+	} else if (buf[0] == '1') {
+		if (true != noc_device->qos_run) {
+			noc_device->qos_run = true;
+			ret = noc_device_qos_set(noc_device, true);
+			if (ret)
+				dev_err(noc_device->dev, "Error when enable qos config!\n");
+		}
+	} else
+		return -EFAULT;
+
+	*ppos += cnt;
+
+	return cnt;
+}
+
+/**
+ * Called when reading run file
+ */
+static ssize_t noc_qos_run_read(struct file *filp, char __user *ubuf,
+				 size_t cnt, loff_t *ppos)
+{
+#define RUN_STR_SIZE 11
+	struct win2030_noc_device *noc_device = filp->private_data;
+	char buf[RUN_STR_SIZE];
+	int r;
+
+	r = snprintf(buf, RUN_STR_SIZE, "%i\n", noc_device->qos_run);
+
+	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+/**
+ * Called when opening run file
+ * @param inode
+ * @param file
+ * @return
+ */
+static int noc_qos_run_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+static const struct file_operations noc_debug_error_fops = {
+	.open = noc_debug_error_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static const struct file_operations noc_debug_qos_fops = {
+	.open = noc_qos_run_open,
+	.read = noc_qos_run_read,
+	.write = noc_qos_run_write,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_debug_fops = {
+	.open = noc_debug_open,
+	.read = noc_debug_bf_read,
+	.write = noc_debug_bf_write,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations noc_debug_available_fops = {
+	.open = noc_debug_available_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+int win2030_noc_debug_reg_init(struct win2030_register *reg, struct dentry *dir)
+{
+	struct dentry *reg_dir;
+	struct win2030_bitfield *bf = NULL;
+
+	reg_dir = debugfs_create_dir(reg->name, dir);
+	if (IS_ERR(reg_dir))
+		return PTR_ERR(reg_dir);
+
+	list_for_each_entry(bf, &reg->bitfields, link) {
+		debugfs_create_file(bf->name,
+				    S_IRUGO | S_IWUSR, reg_dir, bf,
+				    &noc_debug_fops);
+	}
+	return 0;
+}
+
+int win2030_noc_packet_probe_debug_init(struct win2030_noc_probe *probe,
+	struct dentry *probe_dir)
+
+{
+	int j;
+	char name[64];
+	struct win2030_register *reg;
+	struct win2030_bitfield *bf = NULL;
+	int ret = 0;
+
+	for (j = 0; j < probe->nr_filters; j++) {
+		struct win2030_noc_filter *filter;
+		struct dentry *filter_dir;
+
+		filter = &probe->filters[j];
+
+		scnprintf(name, ARRAY_SIZE(name), "filter%d", j);
+		filter_dir = debugfs_create_dir(name, probe_dir);
+		if (IS_ERR(filter_dir))
+			return PTR_ERR(filter_dir);
+
+		reg = filter->route_id_base;
+		list_for_each_entry(bf, &reg->bitfields, link) {
+			if (j != 0)
+				continue;
+			if ((!(strcmp(bf->name, "InitFlow")))
+			    || (!(strcmp(bf->name, "TargetFlow")))) {
+				scnprintf(name, ARRAY_SIZE(name),
+					  "available_%s_source_event", bf->name);
+				debugfs_create_file(name, S_IRUGO,
+					    filter_dir, bf->lut,
+					    &noc_debug_available_fops);
+			}
+		}
+
+		list_for_each_entry(reg, &filter->register_link, parent_link) {
+			ret = win2030_noc_debug_reg_init(reg, filter_dir);
+			if (ret < 0) {
+				return ret;
+			}
+		}
+	}
+	return ret;
+}
+
+int win2030_noc_trans_probe_debug_init(struct win2030_noc_probe *probe,
+	struct dentry *probe_dir)
+{
+	int j;
+	char name[64];
+	struct win2030_register *reg = NULL;
+	int ret = 0;
+
+	for (j = 0; j < probe->nr_filters; j++) {
+		struct win2030_noc_trans_filter *filter;
+		struct dentry *filter_dir;
+
+		filter = &probe->trans_filters[j];
+
+		scnprintf(name, ARRAY_SIZE(name), "filter%d", j);
+		filter_dir = debugfs_create_dir(name, probe_dir);
+		if (IS_ERR(filter_dir))
+			return PTR_ERR(filter_dir);
+
+		list_for_each_entry(reg, &filter->register_link, parent_link) {
+			ret = win2030_noc_debug_reg_init(reg, filter_dir);
+			if (ret < 0) {
+				return ret;
+			}
+		}
+	}
+
+	for (j = 0; j < probe->nr_profilers; j++) {
+		struct win2030_noc_trans_profiler *profiler;
+		struct dentry *profiler_dir;
+
+		profiler = &probe->trans_profilers[j];
+
+		scnprintf(name, ARRAY_SIZE(name), "profiler%d", j);
+		profiler_dir = debugfs_create_dir(name, probe_dir);
+		if (IS_ERR(profiler_dir))
+			return PTR_ERR(profiler_dir);
+
+		list_for_each_entry(reg, &profiler->register_link, parent_link) {
+			ret = win2030_noc_debug_reg_init(reg, profiler_dir);
+			if (ret < 0) {
+				return ret;
+			}
+		}
+	}
+
+	return ret;
+}
+
+
+/**
+ * Create sysfs
+ * @param _dev
+ * @return
+ */
+int win2030_noc_debug_init(struct device *_dev)
+{
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct device_node *np = _dev->of_node;
+	struct dentry *dir;
+	char name[64];
+	int j;
+	struct win2030_noc_probe *probe = NULL;
+	int ret;
+	struct dev_qos_cfg *qos = NULL;
+	struct dentry *dir_qos;
+	struct dentry *dir_qos_module;
+	struct win2030_register *reg = NULL;
+
+	scnprintf(name, ARRAY_SIZE(name), "%s_debug", np->name);
+
+	dir = debugfs_create_dir(name, win2030_noc_ctrl.win2030_noc_root_debug_dir);
+	if (IS_ERR(dir))
+		return PTR_ERR(dir);
+
+	noc_device->dir = dir;
+
+	debugfs_create_file("errors", S_IRUGO, dir, noc_device, &noc_debug_error_fops);
+
+	list_for_each_entry(probe, &noc_device->probes, link) {
+		struct dentry *probe_dir;
+
+		scnprintf(name, ARRAY_SIZE(name), "probe_%s", probe->id);
+		probe_dir = debugfs_create_dir(name, noc_device->dir);
+		if (IS_ERR(probe_dir))
+			return PTR_ERR(probe_dir);
+
+		list_for_each_entry(reg, &probe->register_link, parent_link) {
+			ret = win2030_noc_debug_reg_init(reg, probe_dir);
+			if (ret < 0) {
+				return ret;
+			}
+		}
+
+		if (probe_t_pkt == probe->type) {
+			ret = win2030_noc_packet_probe_debug_init(probe, probe_dir);
+		} else {
+			ret = win2030_noc_trans_probe_debug_init(probe, probe_dir);
+		}
+		if (0 != ret) {
+			return ret;
+		}
+		for (j = 0; j < probe->nr_counters; j++) {
+			struct win2030_noc_counter *cnt;
+			struct dentry *dir_cnt;
+			cnt = &probe->counters[j];
+
+			scnprintf(name, ARRAY_SIZE(name), "counter%d", j);
+			dir_cnt = debugfs_create_dir(name, probe_dir);
+			if (IS_ERR(dir_cnt))
+				return PTR_ERR(dir_cnt);
+
+			/* We assume the counters are the same */
+			if (j == 0) {
+				/*
+				scnprintf(name, ARRAY_SIZE(name),
+					  "available_counter_port_selection");
+				debugfs_create_file(name, S_IRUGO, probe_dir,
+						    cnt->portsel->lut,
+						    &noc_debug_available_fops);
+				*/
+				scnprintf(name, ARRAY_SIZE(name),
+					  "available_counter_alarm_mode");
+				debugfs_create_file(name, S_IRUGO, probe_dir,
+						    cnt->alarm_mode->lut,
+						    &noc_debug_available_fops);
+
+				scnprintf(name, ARRAY_SIZE(name),
+					  "available_counter_source_event");
+				debugfs_create_file(name, S_IRUGO, probe_dir,
+						    cnt->source_event->lut,
+						    &noc_debug_available_fops);
+			}
+			list_for_each_entry(reg, &cnt->register_link, parent_link) {
+				ret = win2030_noc_debug_reg_init(reg, dir_cnt);
+				if (ret < 0) {
+					return ret;
+				}
+			}
+		}
+	}
+	if (!list_empty (&noc_device->qos_list)) {
+		dir_qos = debugfs_create_dir("qos", noc_device->dir);
+		if (IS_ERR(dir_qos))
+			return PTR_ERR(dir_qos);
+
+		debugfs_create_file("enable", S_IRUGO, dir_qos, noc_device, &noc_debug_qos_fops);
+		list_for_each_entry(qos, &noc_device->qos_list, list) {
+			dir_qos_module = debugfs_create_dir(qos->name, dir_qos);
+			list_for_each_entry(reg, &qos->register_link, parent_link) {
+				ret = win2030_noc_debug_reg_init(reg, dir_qos_module);
+				if (ret < 0) {
+					dev_err(_dev, "Error %d while init qos debug register\n", ret);
+					return ret;
+				}
+			}
+		}
+	}
+
+	ret = win2030_noc_sideband_mgr_debug_init(_dev);
+	if (0 != ret) {
+		dev_err(_dev, "Error %d while init sideband mgr debug\n", ret);
+		return ret;
+	}
+	return 0;
+}
diff --git a/drivers/interconnect/eswin/of_noc.c b/drivers/interconnect/eswin/of_noc.c
new file mode 100644
index 000000000000..bc5d874593da
--- /dev/null
+++ b/drivers/interconnect/eswin/of_noc.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/spinlock.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#include <linux/ctype.h>
+#include <linux/interrupt.h>
+#include <linux/win2030_noc.h>
+
+#include "noc.h"
+
+
+void of_noc_qos_free(struct noc_qos_list *qos_root)
+{
+	struct noc_qos_list *qos = NULL;
+	list_for_each_entry(qos, &qos_root->list, list) {
+		kfree(qos);
+	}
+}
+EXPORT_SYMBOL(of_noc_qos_free);
+
+int of_noc_qos_populate(struct device *dev,
+			struct device_node *np,
+			struct noc_qos_list **qos_root)
+{
+	int def_len;
+	struct property *prop;
+
+	prop = of_find_property(np, "intel,qos-ports", &def_len);
+	if (prop != NULL) {
+		struct noc_qos_list *qos;
+		int i, ncfg = of_property_count_strings(np, "intel,qos-ports");
+		*qos_root = (struct noc_qos_list *)
+			devm_kzalloc(dev,
+				sizeof(struct noc_qos_list), GFP_KERNEL);
+		if (*qos_root == NULL) {
+			dev_err(dev, "%s: Memory allocation failed!",
+					__func__);
+			return -ENOMEM;
+		}
+		INIT_LIST_HEAD(&(*qos_root)->list);
+		for (i = 0; i < ncfg; i++) {
+			qos = (struct noc_qos_list *)
+				devm_kzalloc(dev,
+					sizeof(struct noc_qos_list),
+					GFP_KERNEL);
+			if (qos == NULL) {
+				dev_err(dev, "%s: Memory allocation failed!",
+						__func__);
+				of_noc_qos_free(*qos_root);
+				return -ENOMEM;
+			}
+			of_property_read_string_index(np,
+					"intel,qos-ports", i,
+					&qos->name);
+
+			dev_info(dev, "QoS port %s added to list\n", qos->name);
+			list_add_tail(&qos->list,
+					&(*qos_root)->list);
+		}
+	}
+	return 0;
+}
+EXPORT_SYMBOL(of_noc_qos_populate);
diff --git a/drivers/interconnect/eswin/win2030_noc_wdt.c b/drivers/interconnect/eswin/win2030_noc_wdt.c
new file mode 100644
index 000000000000..ed9ed6827ad2
--- /dev/null
+++ b/drivers/interconnect/eswin/win2030_noc_wdt.c
@@ -0,0 +1,189 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/module.h>
+#include <linux/irq.h>
+#include <linux/printk.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+
+#define 	SYSCON_NOC_CFG0_OFFSET			0x324
+#define		NOC_CFG0_REG_BIT_WDT_MASK		GENMASK(16, 0)
+#define 	SYSCON_NOC_CFG1_OFFSET			0x328
+#define		NOC_CFG1_REG_BIT_WDT_MASK		GENMASK(20, 0)
+
+char *noc_wdt_irq_src[] = {
+	"cnoc_vo_timeout",
+	"cnoc_vi_timeout",
+	"cnoc_vc_timeout",
+	"cnoc_tcu_timeout",
+	"cnoc_pciet_x_timeout",
+	"cnoc_pciet_p_timeout",
+	"cnoc_npu_timeout",
+	"cnoc_mcput_d2d_timeout",
+	"cnoc_lsp_apb6_timeout",
+	"cnoc_lsp_apb4_timeout",
+	"cnoc_lsp_apb3_timeout",
+	"cnoc_lsp_apb2_timeout",
+	"cnoc_hsp_timeout",
+	"cnoc_gpu_timeout",
+	"cnoc_dspt_timeout",
+	"cnoc_ddrt1_phy_timeout",
+	"cnoc_ddrt1_ctrl_timeout",
+	"cnoc_ddr0_phy_timeout",
+	"cnoc_ddr0_ctrl_timeout",
+	"cnoc_aon_timeout",
+	"clmm_timeout",
+	"rnoc_ddrt1_p4_timeout",
+	"rnoc_ddrt0_p4_timeout",
+	"mnoc_ddr1_p3_timeout",
+	"mnoc_ddr0_p3_timeout",
+	"snoc_pciet_timeout",
+	"snoc_npu_timeout",
+	"snoc_dspt_timeout",
+	"snoc_ddrt1_p2_timeout",
+	"snoc_ddrt1_p1_timeout",
+	"snoc_ddrt0_p2_timeout",
+	"snoc_ddrt0_p1_timeout",
+	"snoc_aon_timeout",
+	"lnoc_ddrt1_p0_timeout",
+	"lnoc_ddrt0_p0_timeout",
+};
+
+#define 	NOC_WDT_IRQ_NUMBER			(ARRAY_SIZE(noc_wdt_irq_src))
+
+static irqreturn_t noc_wdt_interrupt(int irq, void *dev_id)
+{
+	struct irq_data *data = NULL;
+	char *irq_src = NULL;
+
+	data = irq_get_irq_data(irq);
+	if (NULL == data) {
+		pr_err("noc-wdt: invalid irq data\n");
+	}
+	irq_src = (char *)dev_id;
+	pr_warn_once("noc-wdt: interrupt %s occurred on irq %d, hw irq %ld!\n", irq_src,
+		irq, data->hwirq);
+	return IRQ_HANDLED;
+}
+
+static int eswin_noc_wdt_probe(struct platform_device *pdev)
+{
+	int i;
+	int req_irq;
+	struct device	*dev;
+	int ret;
+	struct device *parent;
+	struct regmap *regmap;
+	int noc_ctl_reg;
+	int wd_ref_divsor;
+
+	for (i = 0; i < NOC_WDT_IRQ_NUMBER; i++) {
+		req_irq = platform_get_irq(pdev, i);
+		if (req_irq < 0)
+			return req_irq;
+
+		ret = devm_request_irq(&pdev->dev, req_irq, &noc_wdt_interrupt,
+				IRQF_SHARED |IRQF_ONESHOT ,
+				noc_wdt_irq_src[i], (void *)noc_wdt_irq_src[i]);
+		if (ret) {
+			dev_err(&pdev->dev, "cannot register irq %d, ret %d\n", req_irq, ret);
+			return ret;
+		}
+		dev_dbg(&pdev->dev,"registered irq %s, base %d, num %ld\n", noc_wdt_irq_src[i],
+			platform_get_irq(pdev, 0), NOC_WDT_IRQ_NUMBER);
+	}
+	dev = &pdev->dev;
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscrg_csr");
+	if (!IS_ERR(regmap)) {
+		ret = of_property_read_u32_index(dev->of_node, "eswin,syscrg_csr", 1,
+						&noc_ctl_reg);
+		if (ret) {
+			dev_err(dev, "can't get noc_ctl_reg offset from dts node(errno:%d)\n", ret);
+			return ret;
+		}
+		ret = of_property_read_u32_index(dev->of_node, "eswin,syscrg_csr", 2,
+						&wd_ref_divsor);
+		if (ret) {
+			dev_err(dev, "can't get wd_ref_divsor val from dts node(errno:%d)\n", ret);
+			return ret;
+		}
+		ret = regmap_update_bits(regmap, noc_ctl_reg, GENMASK(19, 4),
+			FIELD_PREP(GENMASK(19, 4), wd_ref_divsor));
+		if (ret) {
+			dev_err(dev, "can't set wd_ref_divsor val in sys_crg(errno:%d)\n", ret);
+			return ret;
+		}
+	}
+	parent = pdev->dev.parent;
+	if (!parent) {
+		dev_err(&pdev->dev, "no parent\n");
+		return -ENODEV;
+	}
+
+	regmap = syscon_node_to_regmap(parent->of_node);
+	if (IS_ERR(regmap)) {
+		dev_err(&pdev->dev, "failed to get regmap\n");
+		return PTR_ERR(regmap);
+	}
+	ret = regmap_write_bits(regmap, SYSCON_NOC_CFG0_OFFSET,
+		NOC_CFG0_REG_BIT_WDT_MASK, 0x1FFFF);
+	if (0 != ret) {
+		dev_err(&pdev->dev,"failed to enable timeout wdt\n");
+	}
+	ret = regmap_write_bits(regmap, SYSCON_NOC_CFG1_OFFSET,
+		NOC_CFG1_REG_BIT_WDT_MASK, 0x1FFFFF);
+	if (0 != ret) {
+		dev_err(&pdev->dev,"failed to enable timeout wdt\n");
+	}
+	return ret;
+}
+
+static int eswin_noc_wdt_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static const struct of_device_id eswin_noc_wdt_dt_ids[] = {
+	 { .compatible = "eswin,win2030-noc-wdt", },
+	 { /* sentinel */ },
+};
+
+static struct platform_driver eswin_noc_wdt_driver = {
+	.probe  = eswin_noc_wdt_probe,
+	.remove = eswin_noc_wdt_remove,
+	.driver = {
+		.name		= "eswin-noc-wdt",
+		.of_match_table	= eswin_noc_wdt_dt_ids,
+	},
+};
+
+static int __init eswin_noc_wdt_init(void)
+{
+	return platform_driver_register(&eswin_noc_wdt_driver);
+}
+
+subsys_initcall(eswin_noc_wdt_init);
diff --git a/drivers/interconnect/eswin/win2030_sideband_manager.c b/drivers/interconnect/eswin/win2030_sideband_manager.c
new file mode 100644
index 000000000000..6c58a90c54fe
--- /dev/null
+++ b/drivers/interconnect/eswin/win2030_sideband_manager.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Noc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+
+#include "noc.h"
+#include "noc_regs.h"
+
+static const char *win2030_noc_busy_idle_enum[] = {
+	"busy",
+	"idle",
+	NULL
+};
+
+int win2030_noc_sideband_mgr_query(u32 sbm_id)
+{
+	struct win2030_bitfield *bf = NULL;
+	struct win2030_register *reg;
+	struct win2030_noc_sideband_magr *sbm = NULL;
+	int ret;
+	unsigned value;
+
+	if (IS_ERR_OR_NULL(win2030_noc_ctrl.win2030_noc_root_debug_dir) ||
+		list_empty(&win2030_noc_ctrl.sbm_link)) {
+			pr_info("Win2030 NOC SideBand driver not initialisation, will return idle directly!");
+			return 1;
+	}
+	list_for_each_entry(sbm, &win2030_noc_ctrl.sbm_link, link) {
+		reg = sbm->SenseIn0;
+		list_for_each_entry(bf, &reg->bitfields, link) {
+			if (sbm_id == bf->sbm_id) {
+				ret = win2030_bitfield_read(bf, &value);
+				if (ret) {
+					return -EIO;
+				} else {
+					return value;
+				}
+			}
+		}
+	}
+	return -EINVAL;
+}
+EXPORT_SYMBOL(win2030_noc_sideband_mgr_query);
+
+int win2030_noc_sideband_mgr_debug_init(struct device *_dev)
+{
+	int ret;
+	struct dentry *sbm_dir;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	struct win2030_noc_sideband_magr *sbm = noc_device->sbm;
+
+	if (NULL == sbm) {
+		return 0;
+	}
+
+	sbm_dir = debugfs_create_dir("sideband_manager", noc_device->dir);
+	if (IS_ERR(sbm_dir))
+		return PTR_ERR(sbm_dir);
+
+	ret = win2030_noc_debug_reg_init(sbm->SenseIn0, sbm_dir);
+	if (ret < 0) {
+		return ret;
+	}
+	return 0;
+}
+
+static int win2030_noc_register_sideband_mgr(struct device_node *np,
+		struct device *_dev)
+{
+	int ret, i;
+	struct win2030_noc_sideband_magr *sbm;
+	struct win2030_register *reg;
+	struct win2030_bitfield *bf;
+	int size;
+	u32 *values_tab;
+	int offset;
+	struct win2030_noc_device *noc_device = dev_get_drvdata(_dev);
+	const char **bf_name;
+
+	/* Initialise mgr */
+	sbm = devm_kcalloc(_dev, 1, sizeof(struct win2030_noc_sideband_magr), GFP_KERNEL);
+	sbm->hw_base = of_iomap(np, 0);
+	if (IS_ERR_OR_NULL(sbm->hw_base)) {
+		return -EINVAL;
+	}
+	sbm->dev = _dev;
+
+	/**
+	 * Register: SenseIn0
+	 */
+	reg = sbm->SenseIn0 = win2030_new_register(_dev, "SenseIn0",
+			sbm->hw_base, SBM_SENSE_IN0, 32);
+	if (IS_ERR_OR_NULL(reg))
+		return -EINVAL;
+
+	size = of_property_count_elems_of_size(np, "SenseIn0", sizeof(u32) * 2);
+	if (size < 0) {
+		return -ENOENT;
+	}
+	values_tab = devm_kcalloc(_dev, size * 2, sizeof(u32), GFP_KERNEL);
+	if (!values_tab) {
+		return -ENOMEM;
+	}
+	ret = of_property_read_u32_array(np, "SenseIn0", values_tab, size * 2);
+	if (ret) {
+		dev_err(_dev, "Error while parsing SenseIn0 bitfield, ret %d!\n", ret);
+		return -ENOENT;
+	}
+	bf_name = win2030_get_strings_from_dts(_dev, np, "bf-name");
+	if (IS_ERR(bf_name)) {
+		ret = PTR_ERR(bf_name);
+		dev_err(_dev, "Error %d while get sbm bf name!\n", ret);
+		return ret;
+	}
+	for (i = 0; i < size; i++) {
+		offset = values_tab[2 * i + 1];
+		bf = win2030_new_bitfield(reg, bf_name[i], offset, SBM_SENSE_IN0_SENSE_IN0_WIDTH,
+				win2030_noc_busy_idle_enum);
+		if (IS_ERR_OR_NULL(bf))
+			return -EIO;
+
+		bf->sbm_id = values_tab[2 * i];
+	}
+	kfree(bf_name);
+	noc_device->sbm = sbm;
+	list_add_tail(&sbm->link, &win2030_noc_ctrl.sbm_link);
+	return 0;
+}
+
+
+/**
+ * Create sideband manager
+ * @param _dev
+ * @return
+ */
+int win2030_noc_init_sideband_mgr(struct device *_dev)
+{
+	int ret;
+	struct device_node *child = NULL;
+
+	for_each_child_of_node(_dev->of_node, child) {
+		if (of_device_is_compatible(child, "eswin,win2xxx-noc-sideband-manager")) {
+			ret = win2030_noc_register_sideband_mgr(child, _dev);
+			if (0 != ret) {
+				dev_err(_dev, "Error while register sideband manager\n");
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 20aa5ed80aa3..956c9d54f7a0 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -28,7 +28,7 @@ config IOMMU_IO_PGTABLE
 config IOMMU_IO_PGTABLE_LPAE
 	bool "ARMv7/v8 Long Descriptor Format"
 	select IOMMU_IO_PGTABLE
-	depends on ARM || ARM64 || COMPILE_TEST
+	depends on ARM || ARM64 || ARCH_ESWIN_EIC770X_SOC_FAMILY || COMPILE_TEST
 	depends on !GENERIC_ATOMIC64	# for cmpxchg64()
 	help
 	  Enable support for the ARM long descriptor pagetable format.
@@ -388,7 +388,7 @@ config ARM_SMMU_QCOM_DEBUG
 
 config ARM_SMMU_V3
 	tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
-	depends on ARM64
+	depends on ARM64 || ARCH_ESWIN_EIC770X_SOC_FAMILY
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE
 	select GENERIC_MSI_IRQ
@@ -411,6 +411,12 @@ config ARM_SMMU_V3_SVA
 	  Say Y here if your system supports SVA extensions such as PCIe PASID
 	  and PRI.
 
+config ESWIN_PCIE_VPU
+	bool "Eswin PCIe VPU Card StreamID Macro"
+	depends on ARM_SMMU_V3 || ARCH_ESWIN_EIC770X_SOC_FAMILY
+	help
+	  support for eswin pcie card smmu problem
+
 config S390_IOMMU
 	def_bool y if S390 && PCI
 	depends on S390 && PCI
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 769e43d780ce..42b2d74869aa 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-y += amd/ intel/ arm/ iommufd/
+obj-y += amd/ intel/ arm/ iommufd/ eswin/
 obj-$(CONFIG_IOMMU_API) += iommu.o
 obj-$(CONFIG_IOMMU_API) += iommu-traces.o
 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index f2260f45728e..b06aee3686b3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -30,8 +30,26 @@
 #include "arm-smmu-v3.h"
 #include "../../dma-iommu.h"
 #include "../../iommu-sva.h"
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#include <dt-bindings/memory/eswin-win2030-sid.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#define ESWIN_SMMU_IRQ_CLEAR_REG	1
+
+/* smmu interrupt clear bits */
+#define TCU_U84_EVENT_Q_IRPT_NS_CLR_BIT     9
+#define TCU_U84_PRI_Q_IRPT_NS_CLR_BIT       10
+#define TCU_U84_CMD_SYNC_IRPT_NS_CLR_BIT    11
+#define TCU_U84_GLOBAL_IRPT_NS_CLR_BIT      13
+#endif
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static bool disable_bypass = false;
+#else
 static bool disable_bypass = true;
+#endif
 module_param(disable_bypass, bool, 0444);
 MODULE_PARM_DESC(disable_bypass,
 	"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
@@ -74,6 +92,21 @@ struct arm_smmu_option_prop {
 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
 DEFINE_MUTEX(arm_smmu_asid_lock);
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static unsigned long get_tcu_node_status(struct arm_smmu_device *smmu)
+{
+	unsigned long reg_val;
+	unsigned long tcu_node_status = 0;
+	int i;
+
+	for (i = 0; i < 62; i++) {
+		reg_val = readl_relaxed(smmu->s_base + ARM_SMMU_TCU_NODE_STATUSn_OFFSET + (4*i));
+		tcu_node_status |= (reg_val & 0x1) << i;
+	}
+	return tcu_node_status;
+}
+#endif
+
 /*
  * Special value used by SVA when a process dies, to quiesce a CD without
  * disabling it.
@@ -840,11 +873,20 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
 		llq.prod = queue_inc_prod_n(&llq, n);
 		ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
 		if (ret) {
+			#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+			dev_err_ratelimited(smmu->dev,
+					    "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x], TCU_NODE_STATUS=0x%016lx\n",
+					    llq.prod,
+					    readl_relaxed(cmdq->q.prod_reg),
+					    readl_relaxed(cmdq->q.cons_reg),
+					    get_tcu_node_status(smmu));
+			#else
 			dev_err_ratelimited(smmu->dev,
 					    "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
 					    llq.prod,
 					    readl_relaxed(cmdq->q.prod_reg),
 					    readl_relaxed(cmdq->q.cons_reg));
+			#endif
 		}
 
 		/*
@@ -1348,7 +1390,14 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 		u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
 			STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
 
+		#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+		if (ste_live) {
+			dev_dbg(master->dev, "%s:%d, smmu_dbg, duplicated stream, sharing same ste, return!\n", __func__, __LINE__);
+			return;
+		}
+		#else
 		BUG_ON(ste_live);
+		#endif
 		dst[1] = cpu_to_le64(
 			 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
 			 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
@@ -1367,7 +1416,14 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 	}
 
 	if (s2_cfg) {
+		#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+		if (ste_live) {
+			dev_dbg(master->dev, "%s:%d, smmu_dbg, duplicated stream, sharing same ste, return!\n", __func__, __LINE__);
+			return;
+		}
+		#else
 		BUG_ON(ste_live);
+		#endif
 		dst[2] = cpu_to_le64(
 			 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
 			 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
@@ -1415,6 +1471,28 @@ static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool fo
 	}
 }
 
+#ifdef CONFIG_ESWIN_PCIE_VPU
+static void eswin_pcie_vpu_smmu_workaround(struct arm_smmu_device *smmu, u32 sid)
+{
+	void *strtab;
+	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+	struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
+	struct arm_smmu_strtab_l1_desc *desc_busx;
+	int i;
+	u32 streamid;
+
+	for (i = 1; i < 256; i++) {
+		streamid = 0xff0000 + (i << 8);
+		strtab = &cfg->strtab[(streamid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
+ 		desc_busx = &cfg->l1_desc[streamid >> STRTAB_SPLIT];
+		desc_busx->span = STRTAB_SPLIT + 1;
+		desc_busx->l2ptr = desc->l2ptr;
+		desc_busx->l2ptr_dma = desc->l2ptr_dma;
+		arm_smmu_write_strtab_l1_desc(strtab, desc_busx);
+	}
+}
+#endif
+
 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
 {
 	size_t size;
@@ -1440,6 +1518,12 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
 
 	arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false);
 	arm_smmu_write_strtab_l1_desc(strtab, desc);
+
+#ifdef CONFIG_ESWIN_PCIE_VPU
+	if (sid & 0xff0000) {
+		eswin_pcie_vpu_smmu_workaround(smmu, sid);
+	}
+#endif
 	return 0;
 }
 
@@ -1571,6 +1655,33 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
 	return ret;
 }
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static void eswin_smmu_irq_clear(struct arm_smmu_device *smmu, int clearbit)
+{
+	int bitmask;
+	bitmask = BIT(clearbit);
+
+	regmap_write(smmu->regmap, smmu->smmu_irq_clear_reg, bitmask);
+}
+
+static irqreturn_t eswin_smmu_irq_clear_handler(int irq, void *dev)
+{
+	struct arm_smmu_device *smmu = dev;
+
+	if (irq == smmu->evtq.q.irq) {
+			eswin_smmu_irq_clear(smmu, TCU_U84_EVENT_Q_IRPT_NS_CLR_BIT);
+	}
+	else if (irq == smmu->priq.q.irq) {
+			eswin_smmu_irq_clear(smmu, TCU_U84_PRI_Q_IRPT_NS_CLR_BIT);
+	}
+	else {
+		return IRQ_NONE;
+	}
+
+    return IRQ_WAKE_THREAD;
+}
+#endif
+
 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
 {
 	int i, ret;
@@ -1589,9 +1700,9 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
 			if (!ret || !__ratelimit(&rs))
 				continue;
 
-			dev_info(smmu->dev, "event 0x%02x received:\n", id);
+			dev_dbg(smmu->dev, "event 0x%02x received:\n", id);
 			for (i = 0; i < ARRAY_SIZE(evt); ++i)
-				dev_info(smmu->dev, "\t0x%016llx\n",
+				dev_dbg(smmu->dev, "\t0x%016llx\n",
 					 (unsigned long long)evt[i]);
 
 			cond_resched();
@@ -1675,6 +1786,10 @@ static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
 	u32 gerror, gerrorn, active;
 	struct arm_smmu_device *smmu = dev;
 
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	eswin_smmu_irq_clear(smmu, TCU_U84_GLOBAL_IRPT_NS_CLR_BIT);
+	#endif
+
 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
 
@@ -2397,6 +2512,15 @@ static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
 	pci_disable_pasid(pdev);
 }
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static void arm_smmu_attach_ref_release(struct kref *kref)
+{
+	/* no related data needs to be released when kref is 0 */
+	pr_debug("smmu_dbg, %s\n", __func__);
+	return;
+}
+#endif
+
 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
 {
 	unsigned long flags;
@@ -2413,6 +2537,19 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
 
 	master->domain = NULL;
 	master->ats_enabled = false;
+
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	/* Eswin, for generic dev only.If other masters are still attached to this domain,
+	*  do NOT allow to change the ste right now
+	*/
+	if (!dev_is_pci(master->dev)) {
+		if (!kref_put(&smmu_domain->attach_refcount, arm_smmu_attach_ref_release)) {
+			dev_dbg(master->dev, "smmu_dbg, %s:%d, other masters are still on smmu_domain, return!\n",
+				__func__, __LINE__);
+			return;
+		}
+	}
+	#endif
 	arm_smmu_install_ste_for_dev(master);
 }
 
@@ -2441,7 +2578,14 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 		return -EBUSY;
 	}
 
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	/* Eswin, to support duplicated streamID within one group/smmu_domain, skip detach for generic devices*/
+	if (dev_is_pci(dev)) {
+		arm_smmu_detach_dev(master);
+	}
+	#else
 	arm_smmu_detach_dev(master);
+	#endif
 
 	mutex_lock(&smmu_domain->init_mutex);
 
@@ -2452,6 +2596,11 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 			smmu_domain->smmu = NULL;
 			goto out_unlock;
 		}
+		#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+		if (!dev_is_pci(dev)) { //Eswin, to support generic devices share the same streamID, i.e in the same domain
+			kref_init(&smmu_domain->attach_refcount);
+		}
+		#endif
 	} else if (smmu_domain->smmu != smmu) {
 		ret = -EINVAL;
 		goto out_unlock;
@@ -2464,6 +2613,17 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 		ret = -EINVAL;
 		goto out_unlock;
 	}
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	else if (!dev_is_pci(dev)){
+		/* Eswin,for supportting generic devices sharing the same streamID dev.
+		*  This domain has already been initialized by the first dev that was
+		*  attached to this domain previously.So, only needs to increase attach_refcount.
+		*/
+		kref_get(&smmu_domain->attach_refcount);
+		dev_dbg(dev, "smmu_dbg, generic dev sharing the same domain, attach_refcount = %d after add 1\n",
+			kref_read(&smmu_domain->attach_refcount));
+	}
+	#endif
 
 	master->domain = smmu_domain;
 
@@ -2728,6 +2888,147 @@ static void arm_smmu_release_device(struct device *dev)
 	kfree(master);
 }
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static void arm_smmu_group_lookup_delete(void *iommu_data)
+{
+	struct arm_smmu_group *smmu_group = iommu_data;
+	struct arm_smmu_device *smmu;
+
+	if (WARN_ON_ONCE(smmu_group == NULL))
+		return;
+
+	if (IS_ERR(smmu_group))
+		return;
+
+	smmu = smmu_group->smmu;
+	mutex_lock(&smmu->smmu_groups_mutex);
+	rb_erase(&smmu_group->node, &smmu->smmu_groups);
+	mutex_unlock(&smmu->smmu_groups_mutex);
+
+	kfree(smmu_group);
+}
+
+static struct iommu_group *arm_smmu_group_lookup(struct device *dev)
+{
+	struct arm_smmu_device *smmu;
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct rb_node *node;
+	struct arm_smmu_group *smmu_group;
+	u32 sid;
+
+	if (!master)
+		return ERR_PTR(-EFAULT);
+
+	smmu = master->smmu;
+	lockdep_assert_held(&smmu->smmu_groups_mutex);
+
+	/* pick the first sid, since only one sid for each device is allowed */
+	sid = fwspec->ids[0];
+
+	node = smmu->smmu_groups.rb_node;
+	while (node) {
+		smmu_group = rb_entry(node, struct arm_smmu_group, node);
+		if (smmu_group->streamid < sid)
+			node = node->rb_right;
+		else if (smmu_group->streamid > sid)
+			node = node->rb_left;
+		else {
+			return iommu_group_ref_get(smmu_group->group); // Found, iommu_group refcnt add, then return iommu_group
+		}
+	}
+
+	return NULL;
+}
+
+static struct arm_smmu_group *arm_smmu_insert_to_group_lookup(struct device *dev, struct iommu_group *group)
+{
+	struct arm_smmu_device *smmu;
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct arm_smmu_group *new_smmu_group, *cur_smmu_group;
+	struct rb_node **new_node, *parent_node = NULL;
+
+	if (!master)
+		return ERR_PTR(-EFAULT);
+
+	smmu = master->smmu;
+	lockdep_assert_held(&smmu->smmu_groups_mutex);
+
+	new_smmu_group = kzalloc(sizeof(*new_smmu_group), GFP_KERNEL);
+	if (!new_smmu_group)
+		return ERR_PTR(-ENOMEM);
+	/* pick the first sid, since only one sid for each device is allowed */
+	new_smmu_group->smmu = smmu;
+	new_smmu_group->streamid = fwspec->ids[0];
+	new_smmu_group->group = group;
+
+	new_node = &(smmu->smmu_groups.rb_node);
+	while (*new_node) {
+		cur_smmu_group = rb_entry(*new_node, struct arm_smmu_group,
+						node);
+		parent_node = *new_node;
+		if (cur_smmu_group->streamid > new_smmu_group->streamid) {
+			new_node = &((*new_node)->rb_left);
+		} else if (cur_smmu_group->streamid < new_smmu_group->streamid) {
+			new_node = &((*new_node)->rb_right);
+		} else {
+			dev_warn(dev,
+					"group %u already in tree\n",
+					cur_smmu_group->streamid);
+			kfree(new_smmu_group);
+			return ERR_PTR(-EINVAL);
+		}
+	}
+
+	rb_link_node(&new_smmu_group->node, parent_node, new_node);
+	rb_insert_color(&new_smmu_group->node, &smmu->smmu_groups);
+
+	return new_smmu_group;
+}
+
+static struct iommu_group *arm_smmu_device_group(struct device *dev)
+{
+	struct iommu_group *group;
+	struct arm_smmu_device *smmu;
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	struct arm_smmu_group *smmu_group = NULL;
+
+	if (!master)
+		return ERR_PTR(-EFAULT);
+
+	smmu = master->smmu;
+	/*
+	 * We don't support devices sharing stream IDs other than PCI RID
+	 * aliases, since the necessary ID-to-device lookup becomes rather
+	 * impractical given a potential sparse 32-bit stream ID space.
+	 */
+	if (dev_is_pci(dev))
+		group = pci_device_group(dev);
+	else {
+		mutex_lock(&smmu->smmu_groups_mutex);
+		group = arm_smmu_group_lookup(dev);
+		if (!group) {
+			dev_dbg(dev, "smmu_dbg, generic dev,group was NOT found in lut, alloc new group!\n");
+			group = generic_device_group(dev);
+
+			if (group) {
+				smmu_group = arm_smmu_insert_to_group_lookup(dev, group);
+				if (!IS_ERR(smmu_group)) {
+					iommu_group_set_iommudata(group, smmu_group, arm_smmu_group_lookup_delete);
+					dev_dbg(dev, "smmu_dbg, generic dev,new smmu_group added in lut\n");
+				}
+			}
+		}
+		else {
+			dev_dbg(dev, "smmu_dbg, generic dev,group was found in lut!\n");
+		}
+		mutex_unlock(&smmu->smmu_groups_mutex);
+	}
+
+	return group;
+}
+#else
 static struct iommu_group *arm_smmu_device_group(struct device *dev)
 {
 	struct iommu_group *group;
@@ -2744,6 +3045,7 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
 
 	return group;
 }
+#endif
 
 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
 {
@@ -3104,6 +3406,11 @@ static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
 	mutex_init(&smmu->streams_mutex);
 	smmu->streams = RB_ROOT;
 
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	mutex_init(&smmu->smmu_groups_mutex);
+	smmu->smmu_groups = RB_ROOT;
+	#endif
+
 	ret = arm_smmu_init_queues(smmu);
 	if (ret)
 		return ret;
@@ -3210,10 +3517,17 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
 	/* Request interrupt lines */
 	irq = smmu->evtq.q.irq;
 	if (irq) {
+		#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+		ret = devm_request_threaded_irq(smmu->dev, irq, eswin_smmu_irq_clear_handler,
+						arm_smmu_evtq_thread,
+						IRQF_ONESHOT,
+						"arm-smmu-v3-evtq", smmu);
+		#else
 		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 						arm_smmu_evtq_thread,
 						IRQF_ONESHOT,
 						"arm-smmu-v3-evtq", smmu);
+		#endif
 		if (ret < 0)
 			dev_warn(smmu->dev, "failed to enable evtq irq\n");
 	} else {
@@ -3233,11 +3547,19 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
 		irq = smmu->priq.q.irq;
 		if (irq) {
+			#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+			ret = devm_request_threaded_irq(smmu->dev, irq, eswin_smmu_irq_clear_handler,
+							arm_smmu_priq_thread,
+							IRQF_ONESHOT,
+							"arm-smmu-v3-priq",
+							smmu);
+			#else
 			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 							arm_smmu_priq_thread,
 							IRQF_ONESHOT,
 							"arm-smmu-v3-priq",
 							smmu);
+			#endif
 			if (ret < 0)
 				dev_warn(smmu->dev,
 					 "failed to enable priq irq\n");
@@ -3528,8 +3850,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
 
 	if (reg & IDR0_HYP) {
 		smmu->features |= ARM_SMMU_FEAT_HYP;
+		#ifdef CONFIG_ARM64
 		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
 			smmu->features |= ARM_SMMU_FEAT_E2H;
+		#endif
 	}
 
 	/*
@@ -3802,6 +4126,87 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
 	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
 }
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static int eswin_smmu_reset_release(struct arm_smmu_device *smmu)
+{
+	int ret = 0;
+	int i;
+	char tbu_rst_name[16] = {0};
+	struct device *dev = smmu->dev;
+	struct eswin_smmu_reset_control *eswin_smmu_rst_ctl_p = &smmu->eswin_smmu_rst_ctl;
+
+	dev_dbg(dev, "Try %s !\n", __func__);
+
+	eswin_smmu_rst_ctl_p->smmu_axi_rst = devm_reset_control_get_optional(dev, "axi_rst");
+	if (IS_ERR_OR_NULL(eswin_smmu_rst_ctl_p->smmu_axi_rst)) {
+		dev_err(dev, "Failed to get eswin smmu_axi_rst handle\n");
+		return -EFAULT;
+	}
+
+	eswin_smmu_rst_ctl_p->smmu_cfg_rst = devm_reset_control_get_optional(dev, "cfg_rst");
+	if (IS_ERR_OR_NULL(eswin_smmu_rst_ctl_p->smmu_cfg_rst)) {
+		dev_err(dev, "Failed to get eswin smmu_cfg_rst handle\n");
+		return -EFAULT;
+	}
+
+	for(i = 0; i < ESWIN_MAX_TBU_COUNT; i++) {
+		snprintf(tbu_rst_name, sizeof(tbu_rst_name), "tbu%d_rst", i);
+		eswin_smmu_rst_ctl_p->tbu_rst[i] = devm_reset_control_get_optional(dev, tbu_rst_name);
+		if (IS_ERR_OR_NULL(eswin_smmu_rst_ctl_p->tbu_rst[i])) {
+			dev_err(dev, "Failed to get eswin %s handle\n", tbu_rst_name);
+			return -EFAULT;
+		}
+	}
+
+	// The order of the reset must be TCU_cfg_rst ---> TCU_axi_rst ---> TBU_rst
+	ret = reset_control_reset(eswin_smmu_rst_ctl_p->smmu_cfg_rst);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(eswin_smmu_rst_ctl_p->smmu_axi_rst);
+	WARN_ON(0 != ret);
+
+	for(i = 0; i < ESWIN_MAX_TBU_COUNT; i++) {
+		ret = reset_control_reset(eswin_smmu_rst_ctl_p->tbu_rst[i]);
+		WARN_ON(0 != ret);
+	}
+
+	dev_dbg(dev, "%s successfully!\n", __func__);
+
+	return ret;
+}
+
+static int eswin_smmu_reset_assert(struct arm_smmu_device *smmu)
+{
+	int ret = 0;
+	int i;
+	struct device *dev = smmu->dev;
+	struct eswin_smmu_reset_control *eswin_smmu_rst_ctl_p = &smmu->eswin_smmu_rst_ctl;
+
+	dev_dbg(dev, "Try %s !\n", __func__);
+
+	for(i = 0; i < ESWIN_MAX_TBU_COUNT; i++) {
+		if (eswin_smmu_rst_ctl_p->tbu_rst[i]) {
+			ret = reset_control_assert(eswin_smmu_rst_ctl_p->tbu_rst[i]);
+			WARN_ON(0 != ret);
+		}
+	}
+
+	if (eswin_smmu_rst_ctl_p->smmu_axi_rst) {
+		ret = reset_control_assert(eswin_smmu_rst_ctl_p->smmu_axi_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_smmu_rst_ctl_p->smmu_cfg_rst) {
+		ret = reset_control_assert(eswin_smmu_rst_ctl_p->smmu_cfg_rst);
+		WARN_ON(0 != ret);
+	}
+
+	dev_dbg(dev, "%s successfully!\n", __func__);
+
+	return ret;
+}
+#endif
+
 static int arm_smmu_device_probe(struct platform_device *pdev)
 {
 	int irq, ret;
@@ -3854,6 +4259,34 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 		smmu->page1 = smmu->base;
 	}
 
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	/* eswin, map the tcu microarchitectural register region */
+	smmu->s_base = arm_smmu_ioremap(dev, ioaddr + ARM_SMMU_S_BASE, ARM_SMMU_S_AND_TCU_MICRO_REG_SZ);
+	if (IS_ERR(smmu->s_base))
+		return PTR_ERR(smmu->s_base);
+
+	/* eswin, release the reset of smmu */
+	ret = eswin_smmu_reset_release(smmu);
+	if (ret) {
+		dev_err(dev, "failed to release the reset of SMMU\n");
+		return ret;
+	}
+
+	/* eswin, syscon devie is used for clearing the smmu interrupt */
+	smmu->regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(smmu->regmap)) {
+		dev_err(smmu->dev, "No syscfg phandle specified\n");
+		return PTR_ERR(smmu->regmap);
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", ESWIN_SMMU_IRQ_CLEAR_REG,
+					&smmu->smmu_irq_clear_reg);
+	if (ret) {
+		dev_err(dev, "can't get SMMU irq clear reg offset (%d)\n", ret);
+		return ret;
+	}
+	#endif
+
 	/* Interrupt lines */
 
 	irq = platform_get_irq_byname_optional(pdev, "combined");
@@ -3925,6 +4358,11 @@ static void arm_smmu_device_remove(struct platform_device *pdev)
 	arm_smmu_device_disable(smmu);
 	iopf_queue_free(smmu->evtq.iopf);
 	ida_destroy(&smmu->vmid_map);
+
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	/* eswin, hold the reset of the smmu */
+	eswin_smmu_reset_assert(smmu);
+	#endif
 }
 
 static void arm_smmu_device_shutdown(struct platform_device *pdev)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 9915850dd4db..25a73a177b59 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -160,6 +160,19 @@
 
 #define ARM_SMMU_REG_SZ			0xe00
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+/* Secure programming register */
+#define ARM_SMMU_S_BASE			0x8000
+#define ARM_SMMU_S_INIT_OFFSET		(0x803c - ARM_SMMU_S_BASE)
+/* TCU Microarchitectural registers */
+#define ARM_SMMU_TCU_QOS_OFFSET			(0x8e04 - ARM_SMMU_S_BASE)
+#define ARM_SMMU_TCU_NODE_STATUSn_OFFSET	(0x9400 - ARM_SMMU_S_BASE)
+
+#define ARM_SMMU_S_AND_TCU_MICRO_REG_SZ	(0x9800 - ARM_SMMU_S_BASE)
+
+#define ARM_SMMU_TCU_QOS_VALUE		0xffffffff
+#endif
+
 /* Common MSI config fields */
 #define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
 #define MSI_CFG2_SH			GENMASK(5, 4)
@@ -620,11 +633,23 @@ struct arm_smmu_strtab_cfg {
 	u32				strtab_base_cfg;
 };
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+#define ESWIN_MAX_TBU_COUNT	8
+struct eswin_smmu_reset_control {
+	struct reset_control* smmu_axi_rst;
+	struct reset_control* smmu_cfg_rst;
+	struct reset_control* tbu_rst[ESWIN_MAX_TBU_COUNT];
+};
+#endif
+
 /* An SMMUv3 instance */
 struct arm_smmu_device {
 	struct device			*dev;
 	void __iomem			*base;
 	void __iomem			*page1;
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	void __iomem			*s_base;
+	#endif
 
 #define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
 #define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
@@ -682,6 +707,17 @@ struct arm_smmu_device {
 
 	struct rb_root			streams;
 	struct mutex			streams_mutex;
+
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	/* eswin, to support duplicated streamID */
+	struct rb_root			smmu_groups;
+	struct mutex			smmu_groups_mutex;
+
+	/* eswin, syscon device is used for clearing the smmu interrupt*/
+	struct regmap *regmap;
+	int	smmu_irq_clear_reg;
+	struct eswin_smmu_reset_control eswin_smmu_rst_ctl;
+	#endif
 };
 
 struct arm_smmu_stream {
@@ -690,6 +726,15 @@ struct arm_smmu_stream {
 	struct rb_node			node;
 };
 
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+struct arm_smmu_group {
+	u32			streamid;
+	struct arm_smmu_device	*smmu;
+	struct iommu_group	*group;
+	struct rb_node		node;
+};
+#endif
+
 /* SMMU private data for each master */
 struct arm_smmu_master {
 	struct arm_smmu_device		*smmu;
@@ -734,6 +779,9 @@ struct arm_smmu_domain {
 	spinlock_t			devices_lock;
 
 	struct list_head		mmu_notifiers;
+	#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+	struct kref				attach_refcount;
+	#endif
 };
 
 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
diff --git a/drivers/iommu/eswin/Makefile b/drivers/iommu/eswin/Makefile
new file mode 100644
index 000000000000..563a2ca1a91d
--- /dev/null
+++ b/drivers/iommu/eswin/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY) 		+= eswin-win2030-sid.o
diff --git a/drivers/iommu/eswin/eswin-win2030-sid.c b/drivers/iommu/eswin/eswin-win2030-sid.c
new file mode 100644
index 000000000000..7308e0dcabda
--- /dev/null
+++ b/drivers/iommu/eswin/eswin-win2030-sid.c
@@ -0,0 +1,1103 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * APIs for ESWIN SMMU stream ID and TBU configuration
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Min Lin <linmin@eswincomputing.com>
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/bitfield.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/regmap.h>
+#include <linux/iommu.h>
+#include <linux/mfd/syscon.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <dt-bindings/memory/eswin-win2030-sid.h>
+
+static void trigger_waveform_ioremap_resource(void);
+
+#define DYMN_CSR_EN_REG_OFFSET            0x0
+#define DYMN_CSR_GNT_REG_OFFSET           0x4
+
+#define MCPU_SP0_DYMN_CSR_EN_BIT    3
+#define MCPU_SP0_DYMN_CSR_GNT_BIT   3
+
+#define AWSMMUSID	GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID	GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID	GENMASK(15, 8)	// The sid of read operation
+#define ARSMMUSSID	GENMASK(7, 0)	// The ssid of read operation
+
+struct win2030_sid_client {
+	const char *name;
+	unsigned int sid;
+	unsigned int reg_offset;
+};
+
+struct win2030_sid_soc {
+	const struct win2030_sid_client *clients;
+	unsigned int num_clients;
+};
+
+/* The syscon registers for tbu power up(down) must be configured so that
+    tcu can be aware of tbu up and down.
+
+ */
+struct tbu_reg_cfg_info {
+	unsigned int reg_offset;
+	unsigned int qreqn_pd_bit;
+	unsigned int qacceptn_pd_bit;
+};
+
+struct tbu_attachment {
+	struct device *dev;
+	struct list_head list;
+	atomic_t f_count;
+};
+
+struct tbu_priv {
+	atomic_t refcount;
+	int nid;
+	const struct win2030_tbu_client *tbu_client_p;
+	struct list_head attachments;
+	struct mutex tbu_priv_lock;
+};
+
+struct win2030_tbu_client {
+	/* tbu_id: bit[3:0] is for major ID, bit[7:4] is for minor ID;
+	   For example, tbu of dsp3 is tbu7_3, the tbu_ID is 0x73. It measn tbu7_3
+	*/
+	u32 tbu_id;
+	struct tbu_reg_cfg_info tbu_reg_info;
+	int (*tbu_power_ctl_register) (struct tbu_priv *tbu_priv_p, bool is_powerUp, struct device *dev);
+};
+
+struct win2030_tbu_soc {
+	const struct win2030_tbu_client *tbu_clients;
+	unsigned int num_tbuClients;
+};
+
+
+
+struct tbu_power_soc {
+	struct tbu_priv *tbu_priv_array;
+	unsigned int num_tbuClients;
+};
+
+struct win2030_sid {
+	void __iomem *regs;
+	resource_size_t start;
+	const struct win2030_sid_soc *soc;
+	struct mutex eswin_dynm_sid_cfg_en_lock;
+	struct tbu_power_soc *tbu_power_soc;
+	struct mutex tbu_reg_lock;
+};
+struct win2030_sid *syscon_sid_cfg[MAX_NUMNODES] = {NULL};
+
+static int win2030_tbu_power_ctl_register(struct tbu_priv *tbu_priv_p, bool is_powerUp, struct device *dev);
+
+static int win2030_tbu_powr_priv_init(struct tbu_power_soc **tbu_power_soc_pp, int nid);
+static int ioremap_tcu_resource(int nid);
+void print_tcu_node_status(const char *call_name, int call_line, int nid);
+static int __init tcu_proc_init(void);
+
+static int g_nodes_cnt = 0;
+static int win2030_tbu_power_all(int nid, bool is_powerUp);
+
+int win2030_dynm_sid_enable(int nid)
+{
+	unsigned long reg_val;
+	struct win2030_sid *mc = NULL;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		pr_err("%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		pr_debug("%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	mc = syscon_sid_cfg[nid];
+	if (mc == NULL)
+		return -EFAULT;
+
+	mutex_lock(&mc->eswin_dynm_sid_cfg_en_lock);
+	reg_val = readl(mc->regs + DYMN_CSR_EN_REG_OFFSET);
+	set_bit(MCPU_SP0_DYMN_CSR_EN_BIT, &reg_val);
+	writel(reg_val, mc->regs + DYMN_CSR_EN_REG_OFFSET);
+
+	while(1) {
+		reg_val = readl(mc->regs + DYMN_CSR_GNT_REG_OFFSET) & (1 << MCPU_SP0_DYMN_CSR_GNT_BIT);
+		if (reg_val)
+			break;
+
+		msleep(10);
+	}
+	reg_val = readl(mc->regs + DYMN_CSR_EN_REG_OFFSET);
+	clear_bit(MCPU_SP0_DYMN_CSR_EN_BIT, &reg_val);
+	writel(reg_val, mc->regs + DYMN_CSR_EN_REG_OFFSET);
+	mutex_unlock(&mc->eswin_dynm_sid_cfg_en_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(win2030_dynm_sid_enable);
+
+int win2030_aon_sid_cfg(struct device *dev)
+{
+	int ret = 0;
+	struct regmap *regmap;
+	int aon_sid_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	int i,sid_count;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct device_node *np_syscon;
+	int syscon_cell_size = 0;
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_info(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "No eswin,syscfg phandle specified\n");
+		return -1;
+	}
+
+	np_syscon = of_parse_phandle(dev->of_node, "eswin,syscfg", 0);
+	if (np_syscon) {
+		if (of_property_read_u32(np_syscon, "#syscon-cells", &syscon_cell_size)) {
+			of_node_put(np_syscon);
+			dev_err(dev, "failed to get #syscon-cells of sys_con\n");
+			return -1;
+		}
+		of_node_put(np_syscon);
+	}
+
+	sid_count = of_count_phandle_with_args(dev->of_node,
+					"eswin,syscfg", "#syscon-cells");
+
+	dev_dbg(dev, "sid_count=%d, fwspec->num_ids=%d, syscon_cell_size=%d\n",
+			sid_count, fwspec->num_ids, syscon_cell_size);
+
+	if (sid_count < 0) {
+		dev_err(dev, "failed to parse eswin,syscfg property!\n");
+		return -1;
+	}
+
+	if (fwspec->num_ids != sid_count) {
+		dev_err(dev, "num_ids(%d) is NOT equal to num of sid regs(%d)!\n",
+			fwspec->num_ids, sid_count);
+		return -1;
+	}
+
+	for (i = 0; i < sid_count; i++) {
+		sid = fwspec->ids[i];
+		ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", (syscon_cell_size + 1)*i+1,
+					&aon_sid_reg);
+		if (ret) {
+			dev_err(dev, "can't get sid cfg reg offset in sys_con(errno:%d)\n", ret);
+			return ret;
+		}
+
+		/* make the reading sid the same as writing sid, ssid is fixed to zero */
+		rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+		rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+		rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+		rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+		regmap_write(regmap, aon_sid_reg, rdwr_sid_ssid);
+
+		ret = win2030_dynm_sid_enable(dev_to_node(dev));
+		if (ret < 0)
+			dev_err(dev, "failed to config streamID(%d) for %s!\n", sid, of_node_full_name(dev->of_node));
+		else
+			dev_info(dev, "success to config dma streamID(%d) for %s!\n", sid, of_node_full_name(dev->of_node));
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(win2030_aon_sid_cfg);
+
+
+int win2030_dma_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_dma_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+					&hsp_mmu_dma_reg);
+	if (ret) {
+		dev_err(dev, "can't get dma sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_dma_reg, rdwr_sid_ssid);
+
+	ret = win2030_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config dma streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config dma streamID(%d)!\n", sid);
+
+	return ret;
+}
+EXPORT_SYMBOL(win2030_dma_sid_cfg);
+
+static int of_parse_syscon_nodes(struct device_node *np, int *nid_p)
+{
+	#ifdef CONFIG_NUMA
+	int nid;
+	int r;
+
+	r = of_property_read_u32(np, "numa-node-id", &nid);
+	if (r)
+		return -EINVAL;
+
+	pr_debug("Syscon on %u\n", nid);
+	if (nid >= MAX_NUMNODES) {
+		pr_warn("Node id %u exceeds maximum value\n", nid);
+		return -EINVAL;
+	}
+	else
+		*nid_p = nid;
+	#else
+		*nid_p = 0;
+	#endif
+
+	pr_debug("%s, nid = %d\n", __func__, *nid_p);
+
+	return 0;
+}
+
+#if 0
+static int win2030_program_sid(int nid)
+{
+	unsigned int i;
+	u32 rdwr_sid_ssid;
+	struct win2030_sid *mc = NULL;
+	int ret = 0;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		pr_err("%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		pr_debug("%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	mc = syscon_sid_cfg[nid];
+	if (mc == NULL)
+		return -EFAULT;
+
+	for (i = 0; i < mc->soc->num_clients; i++) {
+		const struct win2030_sid_client *client = &mc->soc->clients[i];
+
+		/* make the reading sid the same as writing sid, ssid is fixed to zero */
+		rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, client->sid);
+		rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, client->sid);
+		rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+		rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+		pr_debug("smmu_dbg, setting SID %u for %s\n", client->sid,
+			client->name);
+		writel(rdwr_sid_ssid, mc->regs + client->reg_offset);
+
+		rdwr_sid_ssid = readl(mc->regs + client->reg_offset);
+
+		pr_debug( "smmu_dbg, client %s: rdwr_sid_ssid: 0x%x\n",
+			client->name, rdwr_sid_ssid);
+	}
+
+	ret = win2030_dynm_sid_enable(nid);
+	if (ret < 0)
+		pr_err( "smmu_dbg, %s 0x%0llx fail!\n", __func__, mc->start);
+	else
+		pr_info( "smmu_dbg, %s 0x%0llx done!\n", __func__, mc->start);
+
+	return ret;
+}
+#endif
+
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+static const struct win2030_sid_client win2030_sid_clients[] = {
+	{
+		.name = "scpu",
+		.sid = WIN2030_SID_SCPU,
+		.reg_offset = SCPU_SID_REG_OFFSET,
+	},
+	/* remove the configuration for lcpu, it should be configured by lcpu driver
+	*  since there are clk&reset needs to be configured before setting the
+	   streamID register in sys_con.
+	 */
+	//{
+		// .name = "lcpu",
+		// .sid = WIN2030_SID_LCPU,
+		// .reg_offset = LCPU_SID_REG_OFFSET,
+	// },
+	{
+		.name = "dma1",
+		.sid = WIN2030_SID_DMA1,
+		.reg_offset = DMA1_SID_REG_OFFSET,
+	}, {
+		.name = "crypt",
+		.sid = WIN2030_SID_CRYPT,
+		.reg_offset = CRYPT_SID_REG_OFFSET,
+	}
+};
+
+static const struct win2030_sid_soc win2030_sid_soc = {
+	.num_clients = ARRAY_SIZE(win2030_sid_clients),
+	.clients = win2030_sid_clients,
+};
+#endif
+
+static const struct of_device_id win2030_sid_of_match[] = {
+	{ .compatible = "eswin,win2030-scu-sys-con", .data = &win2030_sid_soc },
+	{ /* sentinel */ }
+};
+
+static int __init win2030_init_streamID(void)
+{
+	const struct of_device_id *match;
+	struct device_node *root, *child = NULL;
+	struct resource regs;
+	struct win2030_sid *mc = NULL;
+	int nid;
+	int ret = 0;
+
+	/* Mapping trigger reg base for capturing wave in zebu */
+	trigger_waveform_ioremap_resource();
+
+	root = of_find_node_by_name(NULL, "soc");
+	for_each_child_of_node(root, child) {
+		match = of_match_node(win2030_sid_of_match, child);
+		if (match && of_node_get(child)) {
+			if (of_address_to_resource(child, 0, &regs) < 0) {
+				pr_err("failed to get scu register\n");
+				of_node_put(child);
+				ret = -ENXIO;
+				break;
+			}
+			if (of_parse_syscon_nodes(child, &nid) < 0) {
+				pr_err("failed to get numa-node-id\n");
+				of_node_put(child);
+				ret = -ENXIO;
+				break;
+			}
+
+			/* program scu sreamID related registers */
+			mc = kzalloc(sizeof(*mc), GFP_KERNEL);
+			if (!mc) {
+				of_node_put(child);
+				pr_err("failed to kzalloc\n");
+				ret = -ENOMEM;
+				break;
+			}
+
+			mc->soc = match->data;
+			mc->regs = ioremap(regs.start, resource_size(&regs));
+			if (IS_ERR(mc->regs)) {
+				pr_err("failed to ioremap scu reges\n");
+				of_node_put(child);
+				ret = PTR_ERR(mc->regs);
+				kfree(mc);
+				break;
+			}
+			mc->start = regs.start;
+			mutex_init(&mc->eswin_dynm_sid_cfg_en_lock);
+
+			if (win2030_tbu_powr_priv_init(&mc->tbu_power_soc, nid)) {
+				pr_err("failed to kzalloc for tbu_power_priv_arry\n");
+				of_node_put(child);
+				iounmap(mc->regs);
+				kfree(mc);
+				ret = -ENOMEM;
+				WARN_ON(1);
+				break;
+			}
+			mutex_init(&mc->tbu_reg_lock);
+
+			syscon_sid_cfg[nid] = mc;
+			g_nodes_cnt++;
+			pr_debug("%s, syscon_sid_cfg[%d] addr is 0x%px\n", __func__, nid, syscon_sid_cfg[nid]);
+			pr_info("power down all tbus of node %d at start up\n", nid);
+			win2030_tbu_power_all(nid, false);
+			print_tcu_node_status(__func__, __LINE__, nid);
+			/* sid configuration was moved into each driver, so skip win2030_program_sid*/
+			// win2030_program_sid(nid);
+
+			of_node_put(child);
+		}
+	}
+	of_node_put(root);
+
+	return ret;
+}
+
+early_initcall(win2030_init_streamID);
+
+
+
+static const struct win2030_tbu_client win2030_tbu_clients[] = {
+	{
+		.tbu_id = WIN2030_TBUID_0x0, // ISP, DW200 share the tbu0
+		.tbu_reg_info = {0x3d8, 7, 6},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x10, // tbu1_0 is only for video decoder
+		.tbu_reg_info = {0x3d4, 31, 30},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x11, // tbu1_1 is only video encoder
+		.tbu_reg_info = {0x3d4, 23, 22},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x12, // tbu1_2 is only Jpeg encoder
+		.tbu_reg_info = {0x3d4, 7, 6},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x13, // tbu1_3 is only Jpeg decoder
+		.tbu_reg_info = {0x3d4, 15, 14},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x2, // Ethernet, sata, usb, dma0, emmc, sd, sdio share the tbu2
+		.tbu_reg_info = {0x3d8, 15, 14},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x3, // tbu3 is only for pcie
+		.tbu_reg_info = {0x3d8, 23, 22},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x4, // scpu, crypto, lpcpu, dma1 share the tbu4
+		.tbu_reg_info = {0x3d8, 31, 30},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x5, // tbu5 is only NPU
+		.tbu_reg_info = {0x3d0, 15, 14},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x70, // tbu7_0 is only dsp0
+		.tbu_reg_info = {0x3f8, 7, 6},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x71, // tbu7_1 is only dsp1
+		.tbu_reg_info = {0x3f8, 15, 14},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x72, // tbu7_2 is only dsp2
+		.tbu_reg_info = {0x3f8, 23, 22},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = WIN2030_TBUID_0x73, // tbu7_3 is only dsp3
+		.tbu_reg_info = {0x3f8, 31, 30},
+		.tbu_power_ctl_register = win2030_tbu_power_ctl_register,
+	},
+};
+
+static const struct win2030_tbu_soc win2030_tbu_soc = {
+	.num_tbuClients = ARRAY_SIZE(win2030_tbu_clients),
+	.tbu_clients = win2030_tbu_clients,
+};
+
+static int __do_win2030_tbu_power_ctl(int nid, bool is_powerUp, const struct tbu_reg_cfg_info *tbu_reg_info_p)
+{
+	int ret = 0;
+	unsigned long reg_val;
+	struct win2030_sid *mc = NULL;
+	int loop_cnt = 0;
+	// int bitmask; = BIT(clearbit);
+	mc = syscon_sid_cfg[nid];
+	if (mc == NULL)
+		return -EFAULT;
+
+	mutex_lock(&mc->tbu_reg_lock);
+	if (is_powerUp) {
+		reg_val = readl(mc->regs + tbu_reg_info_p->reg_offset);
+		set_bit(tbu_reg_info_p->qreqn_pd_bit, &reg_val);
+		writel(reg_val, mc->regs + tbu_reg_info_p->reg_offset);
+		pr_debug("reg_offset=0x%03x, tbu_val=0x%x\n",
+			tbu_reg_info_p->reg_offset, readl(mc->regs + tbu_reg_info_p->reg_offset));
+	}
+	else {
+		reg_val = readl(mc->regs + tbu_reg_info_p->reg_offset);
+		clear_bit(tbu_reg_info_p->qreqn_pd_bit, &reg_val);
+		writel(reg_val, mc->regs + tbu_reg_info_p->reg_offset);
+		do {
+			reg_val = readl(mc->regs + tbu_reg_info_p->reg_offset);
+			pr_debug("reg_offset=0x%03x, tbu_val=0x%lx, BIT(qacceptn_pd_bit)=0x%lx\n",
+				tbu_reg_info_p->reg_offset, reg_val, BIT(tbu_reg_info_p->qacceptn_pd_bit));
+			if ((reg_val & BIT(tbu_reg_info_p->qacceptn_pd_bit)) == 0) {
+				break;
+			}
+			mdelay(10);
+			loop_cnt++;
+			if (loop_cnt > 10) {
+				WARN_ON(1); // it should never happen.
+				break;
+			}
+		}while (1);
+
+		if(loop_cnt > 10) {
+			ret = -1;
+		}
+	}
+	mutex_unlock(&mc->tbu_reg_lock);
+
+	return ret;
+}
+
+#define do_win2030_tbu_power_up(nid, tbu_reg_info_p)	__do_win2030_tbu_power_ctl(nid, true, tbu_reg_info_p)
+#define do_win2030_tbu_power_down(nid, tbu_reg_info_p)	__do_win2030_tbu_power_ctl(nid, false, tbu_reg_info_p)
+
+
+
+static int tbu_power_down_ref_release(atomic_t *ref)
+{
+	int ret = 0;
+	struct tbu_priv *tbu_priv_p = container_of(ref, struct tbu_priv, refcount);
+	int nid = tbu_priv_p->nid;
+	const struct tbu_reg_cfg_info *tbu_reg_info_p = &tbu_priv_p->tbu_client_p->tbu_reg_info;
+
+	WARN_ON(!tbu_priv_p);
+	if (!tbu_priv_p)
+		return -1;
+
+	ret = do_win2030_tbu_power_down(nid, tbu_reg_info_p);
+
+	return ret;
+}
+
+static int eic770x_tbu_attach(struct device *dev, struct tbu_priv *tbu_priv_p)
+{
+	struct tbu_attachment *a;
+	bool attached = false;
+
+	WARN_ON(!mutex_is_locked(&tbu_priv_p->tbu_priv_lock));
+
+	list_for_each_entry(a, &tbu_priv_p->attachments, list) {
+		if (a->dev == dev) {
+			attached = true;
+			break;
+		}
+	}
+
+	/* device already attached, then just add refcount */
+	if (attached) {
+		atomic_add(1, &a->f_count);
+		return 0;
+	}
+
+	/* allocate tbu_attachement and attach it to the tbu */
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a) {
+		dev_WARN_ONCE(dev, true, "Failed to attach device to tbu %d, errno %d\n", tbu_priv_p->tbu_client_p->tbu_id, -ENOMEM);
+		return -ENOMEM;
+	}
+	a->dev = dev;
+	INIT_LIST_HEAD(&a->list);
+	list_add(&a->list, &tbu_priv_p->attachments);
+	atomic_add(1, &a->f_count);
+
+	return 0;
+}
+
+static int eic770x_tbu_detach(struct device *dev, struct tbu_priv *tbu_priv_p)
+{
+	struct tbu_attachment *a;
+	bool attached = false;
+
+	WARN_ON(!mutex_is_locked(&tbu_priv_p->tbu_priv_lock));
+
+	list_for_each_entry(a, &tbu_priv_p->attachments, list) {
+		if (a->dev == dev) {
+			attached = true;
+			break;
+		}
+	}
+
+	if (!attached) {
+		dev_WARN_ONCE(dev, true, "Failed to detach device from tbu %d, dev not in the tbu list!!!\n", tbu_priv_p->tbu_client_p->tbu_id);
+		return -ENOENT;
+	}
+
+	if (atomic_sub_return(1, &a->f_count) == 0) {
+		list_del(&a->list);
+		kfree(a);
+	}
+
+	return 0;
+}
+
+static void eic770x_tbu_dump_attachment(struct tbu_priv *tbu_priv_p)
+{
+	struct tbu_attachment *a;
+
+	WARN_ON(!mutex_is_locked(&tbu_priv_p->tbu_priv_lock));
+
+	pr_debug("------Dump:(node %d) tbu 0x%02x total refcnt %d, attached device(s):------\n",
+		tbu_priv_p->nid, tbu_priv_p->tbu_client_p->tbu_id, atomic_read(&tbu_priv_p->refcount));
+	list_for_each_entry(a, &tbu_priv_p->attachments, list) {
+		pr_debug("(node %d) %-32s refcnt %d\n", tbu_priv_p->nid, dev_name(a->dev), atomic_read(&a->f_count));
+	}
+	pr_debug("------End of dump:(node %d) tbu 0x%02x------------------------------------\n",
+		tbu_priv_p->nid, tbu_priv_p->tbu_client_p->tbu_id);
+
+}
+
+static int win2030_tbu_power_ctl_register(struct tbu_priv *tbu_priv_p, bool is_powerUp, struct device *dev)
+{
+	int ret = 0;
+	int nid = tbu_priv_p->nid;
+	const struct win2030_tbu_client *tbu_client_p = tbu_priv_p->tbu_client_p;
+	const struct tbu_reg_cfg_info *tbu_reg_info_p = &tbu_priv_p->tbu_client_p->tbu_reg_info;
+	unsigned int old_refcount;
+
+	mutex_lock(&tbu_priv_p->tbu_priv_lock);
+	old_refcount = atomic_read(&tbu_priv_p->refcount);
+
+	if (is_powerUp == false) { //power down
+		if (unlikely(0 == old_refcount)) {
+			pr_debug("tbu 0x%02x(node %d) is down already!\n",
+				tbu_client_p->tbu_id, tbu_priv_p->nid);
+			goto tbu_finish;
+		}
+
+		eic770x_tbu_detach(dev, tbu_priv_p);
+		if (atomic_sub_return(1, &tbu_priv_p->refcount) == 0) {
+			ret = tbu_power_down_ref_release(&tbu_priv_p->refcount);
+		}
+		else {
+			pr_debug("tbu 0x%02x(node %d) is used by other module(s) right now!\n",
+				tbu_client_p->tbu_id, tbu_priv_p->nid);
+		}
+	}
+	else { //power up
+		if (0 == old_refcount) {
+			ret = do_win2030_tbu_power_up(nid, tbu_reg_info_p);
+		}
+		else {
+			pr_debug("tbu 0x%02x(node %d) is already power up!",
+				tbu_client_p->tbu_id, tbu_priv_p->nid);
+		}
+		atomic_add(1, &tbu_priv_p->refcount);
+		eic770x_tbu_attach(dev, tbu_priv_p);
+	}
+	eic770x_tbu_dump_attachment(tbu_priv_p);
+
+tbu_finish:
+	mutex_unlock(&tbu_priv_p->tbu_priv_lock);
+
+	return ret;
+
+}
+
+static int win2030_tbu_powr_priv_init(struct tbu_power_soc **tbu_power_soc_pp, int nid)
+{
+	int ret = 0;
+	int i;
+	unsigned int num_tbuClients = win2030_tbu_soc.num_tbuClients;
+	struct tbu_power_soc *tbu_power_soc_p;
+	struct tbu_priv *tbu_priv_p;
+	unsigned int alloc_size;
+
+	tbu_power_soc_p = kzalloc(sizeof(struct tbu_power_soc), GFP_KERNEL);
+	if (!tbu_power_soc_p)
+		return -ENOMEM;
+
+	alloc_size = num_tbuClients * sizeof(struct tbu_priv);
+	tbu_priv_p = kzalloc(alloc_size, GFP_KERNEL);
+	if (!tbu_priv_p) {
+		ret = -ENOMEM;
+		goto err_tbu_priv_p;
+	}
+	tbu_power_soc_p->tbu_priv_array = tbu_priv_p;
+	pr_debug("%s:%d, num_tbu=%d,sizeof(struct tbu_priv)=0x%lx, alloc_size=0x%x, tbu_priv_p=0x%px\n",
+		__func__, __LINE__, num_tbuClients, sizeof(struct tbu_priv), alloc_size, tbu_priv_p);
+
+	for (i = 0; i < win2030_tbu_soc.num_tbuClients; i++) {
+		tbu_priv_p->nid = nid;
+		atomic_set(&tbu_priv_p->refcount, 0);
+		tbu_priv_p->tbu_client_p = &win2030_tbu_soc.tbu_clients[i];
+		INIT_LIST_HEAD(&tbu_priv_p->attachments);
+		mutex_init(&tbu_priv_p->tbu_priv_lock);
+		pr_debug("%s, nid %d, tbu 0x%02x, tbu_priv_p(0x%px), sizeof(struct tbu_priv)=0x%lx\n", __func__, nid, tbu_priv_p->tbu_client_p->tbu_id, tbu_priv_p, sizeof(struct tbu_priv));
+		tbu_priv_p++;
+	}
+	tbu_power_soc_p->num_tbuClients = num_tbuClients;
+
+	*tbu_power_soc_pp = tbu_power_soc_p;
+
+	ret = ioremap_tcu_resource(nid);
+	if (ret) {
+		WARN_ON(1);
+	}
+	ret = tcu_proc_init();
+	if (ret) {
+		pr_err("failed to create proc for tcu node %d!!!\n", nid);
+	}
+	print_tcu_node_status(__func__, __LINE__, nid);
+	pr_info("%s finished!\n", __func__);
+
+	return 0;
+
+err_tbu_priv_p:
+	kfree(tbu_power_soc_p);
+
+	return ret;
+
+}
+
+static int win2030_get_tbu_priv(int nid, u32 tbu_id, struct tbu_priv **tbu_priv_pp)
+{
+	int i;
+	struct win2030_sid *mc = syscon_sid_cfg[nid];
+	struct tbu_power_soc *tbu_power_soc_p = mc->tbu_power_soc;
+	struct tbu_priv *tbu_priv_p = tbu_power_soc_p->tbu_priv_array;
+
+	pr_debug("%s,  syscon_sid_cfg[%d] addr is 0x%px, tbu_id=0x%02x, tbu_power_soc_p is 0x%px\n",
+		__func__, nid, syscon_sid_cfg[nid], tbu_id, tbu_power_soc_p);
+
+	for (i = 0; i < tbu_power_soc_p->num_tbuClients; i++) {
+		if (tbu_id == tbu_priv_p->tbu_client_p->tbu_id) {
+			*tbu_priv_pp = tbu_priv_p;
+			pr_debug("%s, found tbu_id 0x%02x, tbu_priv_array[%d] tbu_priv_p is 0x%px\n",
+				__func__, tbu_id, i, tbu_priv_p);
+			return 0;
+		}
+		tbu_priv_p++;
+	}
+
+	return -1;
+}
+
+static int win2030_tbu_power_all(int nid, bool is_powerUp)
+{
+
+	int i;
+	struct win2030_sid *mc = syscon_sid_cfg[nid];
+	struct tbu_power_soc *tbu_power_soc_p = mc->tbu_power_soc;
+	struct tbu_priv *tbu_priv_p = tbu_power_soc_p->tbu_priv_array;
+	const struct win2030_tbu_client *tbu_client_p = NULL;
+
+	for (i = 0; i < tbu_power_soc_p->num_tbuClients; i++) {
+		tbu_client_p = tbu_priv_p->tbu_client_p;
+		pr_debug("%s, tbu 0x%x %s\n", __func__, tbu_client_p->tbu_id, is_powerUp?"power up":"power down");
+		__do_win2030_tbu_power_ctl(nid, is_powerUp, &tbu_client_p->tbu_reg_info);
+		tbu_priv_p++;
+	}
+
+	return 0;
+}
+/***********************************************************************************************
+   win2030_tbu_power(struct device *dev, bool is_powerUp) is for powering up or down
+   the tbus of the device module which is under smmu.
+   Drivers should call win2030_tbu_power(dev, true) when probing afer clk of the tbu is on,
+   and call call win2030_tbu_power(dev, false) when removing driver before clk of the tbu is off.
+
+   Input:
+	struct device *dev	The struct device of the driver that calls this API.
+	bool is_powerUp		true: power up the tbus;  false: power down the tbus.
+   Return:
+	zero:		successfully power up/down
+	none zero:	faild to power up/down
+***********************************************************************************************/
+int win2030_tbu_power(struct device *dev, bool is_powerUp)
+{
+	int ret = 0;
+	struct device_node *node = dev->of_node;
+	int nid = dev_to_node(dev);
+	u32 tbu_id;
+	const struct win2030_tbu_client *tbu_client_p = NULL;
+	struct tbu_priv *tbu_priv_p;
+	int tbu_num = 0;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		pr_err("%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		pr_debug("%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	dev_dbg(dev, "%s %s!\n", __func__, (is_powerUp == true)? "up":"down");
+	of_property_for_each_u32(node, "tbus", tbu_id) {
+		pr_debug("tbus = <0x%02x>\n", tbu_id);
+		if (0 == win2030_get_tbu_priv(nid, tbu_id, &tbu_priv_p)) {
+			tbu_client_p = tbu_priv_p->tbu_client_p;
+			if (tbu_client_p->tbu_power_ctl_register) {
+				ret = tbu_client_p->tbu_power_ctl_register(tbu_priv_p, is_powerUp, dev);
+				if (ret)
+					return ret;
+			}
+			else {
+				ret = __do_win2030_tbu_power_ctl(nid, is_powerUp, &tbu_client_p->tbu_reg_info);
+				if (ret)
+					return ret;
+			}
+			tbu_num++;
+		}
+		else if (tbu_id == WIN2030_TBUID_0xF00) {
+			tbu_num++;
+		}
+		else {
+			pr_err("tbu power ctl failed!, Couldn't find tbu 0x%x\n", tbu_id);
+			return -1;
+		}
+	}
+
+	if (tbu_num == 0) {
+		pr_err("Err,tbu NOT defined in dts!!!!\n");
+		WARN_ON(1);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(win2030_tbu_power);
+
+int win2030_tbu_power_by_dev_and_node(struct device *dev, struct device_node *node, bool is_powerUp)
+{
+	int ret = 0;
+	int nid = dev_to_node(dev);
+	u32 tbu_id;
+	const struct win2030_tbu_client *tbu_client_p = NULL;
+	struct tbu_priv *tbu_priv_p;
+	int tbu_num = 0;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		pr_err("%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		pr_debug("%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	of_property_for_each_u32(node, "tbus", tbu_id) {
+		pr_debug("tbus = <0x%02x>\n", tbu_id);
+		if (0 == win2030_get_tbu_priv(nid, tbu_id, &tbu_priv_p)) {
+			tbu_client_p = tbu_priv_p->tbu_client_p;
+			if (tbu_client_p->tbu_power_ctl_register) {
+				ret = tbu_client_p->tbu_power_ctl_register(tbu_priv_p, is_powerUp, dev);
+				if (ret)
+					return ret;
+			}
+			else {
+				ret = __do_win2030_tbu_power_ctl(nid, is_powerUp, &tbu_client_p->tbu_reg_info);
+				if (ret)
+					return ret;
+			}
+			tbu_num++;
+		}
+		else if (tbu_id == WIN2030_TBUID_0xF00) {
+			tbu_num++;
+		}
+		else {
+			pr_err("tbu power ctl failed!, Couldn't find tbu 0x%x\n", tbu_id);
+			return -1;
+		}
+	}
+
+	if (tbu_num == 0) {
+		pr_err("Err,tbu NOT defined in dts!!!!\n");
+		WARN_ON(1);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(win2030_tbu_power_by_dev_and_node);
+
+#define WAVE_TRIGGER_REG_OFFSET    0x668
+#define WAVE_TRIGGER_REG_BASE     0x51810000
+static void __iomem *trigger_reg_base;
+static void trigger_waveform_ioremap_resource(void)
+{
+	trigger_reg_base = ioremap(WAVE_TRIGGER_REG_BASE, PAGE_SIZE);
+}
+
+void trigger_waveform_start(void)
+{
+	printk("trigger waveform capture!\n");
+
+	writel(0x8000, trigger_reg_base + WAVE_TRIGGER_REG_OFFSET);
+}
+EXPORT_SYMBOL(trigger_waveform_start);
+
+void trigger_waveform_stop(void)
+{
+	writel(0x0, trigger_reg_base + WAVE_TRIGGER_REG_OFFSET);
+}
+EXPORT_SYMBOL(trigger_waveform_stop);
+
+void *tcu_base[MAX_NUMNODES] = {NULL};
+#define TCU_NODE_STATUSn	0x9400
+static int ioremap_tcu_resource(int nid)
+{
+	tcu_base[nid] = ioremap(0x50c00000 + nid*0x20000000, 0x40000);
+	if (IS_ERR(tcu_base[nid])) {
+		pr_err("failed to ioremap tcu reges\n");
+		return PTR_ERR(tcu_base[nid]);
+	}
+	return 0;
+}
+
+static int get_tcu_node_status(unsigned long *tcu_node_status_p, int nid)
+{
+	unsigned long reg_val;
+	unsigned long tcu_node_status = 0;
+	int i;
+
+	for (i = 0; i < 62; i++) {
+		reg_val = readl(tcu_base[nid] + TCU_NODE_STATUSn + (4*i));
+		tcu_node_status |= (reg_val & 0x1) << i;
+	}
+	*tcu_node_status_p = tcu_node_status;
+
+	return 0;
+}
+
+void print_tcu_node_status(const char *call_name, int call_line, int nid)
+{
+	unsigned long tcu_node_status = 0;
+
+	get_tcu_node_status(&tcu_node_status, nid);
+	pr_debug("%s:%d, (node %d) TCU_NODE_STATUS=0x%016lx\n",
+		call_name, call_line, nid, tcu_node_status);
+}
+EXPORT_SYMBOL(print_tcu_node_status);
+
+static int tcu_proc_show(struct seq_file *m, void *v)
+{
+	unsigned long tcu_node_status = 0;
+	int nid,i;
+	struct win2030_sid *mc;
+	struct tbu_power_soc *tbu_power_soc_p;
+	struct tbu_priv *tbu_priv_p;
+	const struct win2030_tbu_client *tbu_client_p = NULL;
+	struct tbu_attachment *a;
+
+	seq_printf(m, "--------------------------------------------------------------------------------------------------------------------------------------------\n");
+	seq_printf(m, "TCU bits(ID):|13(0x%02x)|12(0x%02x)|11(0x%2x)|10(0x%02x)|9(null) |8(0x%02x) |7(0x%02x) |6(0x%02x) |5(0x%02x) |4(0x%02x) |3(0x%02x) |2(0x%02x) |1(0x%02x) |0(0x%02x) |\n",
+			WIN2030_TBUID_DSP3, WIN2030_TBUID_DSP2, WIN2030_TBUID_DSP1, WIN2030_TBUID_DSP0, WIN2030_TBUID_NPU, WIN2030_TBUID_SCPU,
+			WIN2030_TBUID_PCIE, WIN2030_TBUID_DMA0, WIN2030_TBUID_JDEC, WIN2030_TBUID_JENC, WIN2030_TBUID_VENC, WIN2030_TBUID_VDEC, WIN2030_TBUID_ISP);
+	seq_printf(m, "  Device(s)  |  DSP3  |  DSP2  |  DSP1  |  DSP0  |  null  |  NPU   |  SCPU  |  PCIE  |hsp DMAC|  JDEC  |  JENC  |  VENC  |  VDEC  |  ISP   |\n");
+	seq_printf(m, "             |        |        |        |        |        |        |  CRYPT |        |  USB   |        |        |        |        |  DW    |\n");
+	seq_printf(m, "             |        |        |        |        |        |        |aon DMAC|        |  ETH   |        |        |        |        |        |\n");
+	seq_printf(m, "             |        |        |        |        |        |        |  LPCPU |        |  SATA  |        |        |        |        |        |\n");
+	seq_printf(m, "             |        |        |        |        |        |        |        |        |  EMMC  |        |        |        |        |        |\n");
+	seq_printf(m, "             |        |        |        |        |        |        |        |        |   SD   |        |        |        |        |        |\n");
+	seq_printf(m, "--------------------------------------------------------------------------------------------------------------------------------------------\n");
+	for (nid = 0; nid < g_nodes_cnt; nid++) {
+		get_tcu_node_status(&tcu_node_status, nid);
+		seq_printf(m, "(node %d) TCU:|", nid);
+		for (i = 13; i >= 0; i--) {
+			seq_printf(m, "   %d    |", (tcu_node_status&(1<<i))?1:0);
+		}
+		seq_printf(m, "\n");
+	}
+	seq_printf(m, "--------------------------------------------------------------------------------------------------------------------------------------------\n");
+
+	for (nid = 0; nid < g_nodes_cnt; nid++) {
+		seq_printf(m, "------(node %d) TBU attached device(s) list info:------\n", nid);
+		mc = syscon_sid_cfg[nid];
+		tbu_power_soc_p = mc->tbu_power_soc;
+		tbu_priv_p = tbu_power_soc_p->tbu_priv_array;
+		for (i = 0; i < tbu_power_soc_p->num_tbuClients; i++) {
+			tbu_client_p = tbu_priv_p->tbu_client_p;
+			if (tbu_client_p->tbu_power_ctl_register) {
+				mutex_lock(&tbu_priv_p->tbu_priv_lock);
+				seq_printf(m, "tbu(0x%02x) total refcnt %d, attached device(s):\n",
+						tbu_priv_p->tbu_client_p->tbu_id, atomic_read(&tbu_priv_p->refcount));
+				list_for_each_entry(a, &tbu_priv_p->attachments, list) {
+					seq_printf(m, "  %-32s refcnt %d\n",dev_name(a->dev), atomic_read(&a->f_count));
+				}
+				seq_printf(m, "\n");
+				// seq_printf(m, "<(node %d) end of tbu 0x%02x\n",
+						// tbu_priv_p->nid, tbu_priv_p->tbu_client_p->tbu_id);
+				mutex_unlock(&tbu_priv_p->tbu_priv_lock);
+			}
+			tbu_priv_p++;
+		}
+		seq_printf(m, "----------(node %d) end of list info-------------------\n", nid);
+	}
+
+	return 0;
+}
+
+static bool tcp_proc_exist = false;
+static int __init tcu_proc_init(void)
+{
+	char proc_name[64];
+
+	if (tcp_proc_exist)
+		return 0;
+
+	tcp_proc_exist = true;
+	sprintf(proc_name, "%s_info", "tcu");
+	pr_debug("%s, proc_name:%s\n", __func__, proc_name);
+	if (NULL == proc_create_single_data(proc_name, 0, NULL, tcu_proc_show, NULL)) {
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index bc2e265cb02d..a44784068faf 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -294,5 +294,21 @@ config QCOM_IPCC
 	  sending interrupts to the clients. On the other hand, the driver also
 	  acts as an interrupt controller for receiving interrupts from clients.
 	  Say Y here if you want to build this driver.
+config ESWIN_MBOX
+	tristate "Eswin Mailbox"
+	depends on OF
+	depends on HAS_IOMEM
+	depends on ARCH_ESWIN_EIC770X_SOC_FAMILY
+	help
+	  Mailbox driver implementation for the eswin platform. It is used
+	  to send message between application processors and MCU. Say Y here if
+	  you want to build the eswin mailbox controller driver.
+
+config ESWIN_LPCPU
+	tristate "Eswin LPCPU"
+	depends on ESWIN_MBOX
+	help
+	  Lpcpu driver implementation for the eswin platform. Say Y here if
+	  you want to build the eswin lpcpu driver.
 
 endif
diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile
index fc9376117111..917a154a1b64 100644
--- a/drivers/mailbox/Makefile
+++ b/drivers/mailbox/Makefile
@@ -62,3 +62,7 @@ obj-$(CONFIG_SPRD_MBOX)		+= sprd-mailbox.o
 obj-$(CONFIG_QCOM_IPCC)		+= qcom-ipcc.o
 
 obj-$(CONFIG_APPLE_MAILBOX)	+= apple-mailbox.o
+
+obj-$(CONFIG_ESWIN_MBOX)	+= eswin-mailbox.o
+
+obj-$(CONFIG_ESWIN_LPCPU)	+= eswin-lpcpu.o
diff --git a/drivers/mailbox/eswin-lpcpu.c b/drivers/mailbox/eswin-lpcpu.c
new file mode 100644
index 000000000000..0568e1ef57f5
--- /dev/null
+++ b/drivers/mailbox/eswin-lpcpu.c
@@ -0,0 +1,433 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2021 ESWIN
+ *
+ * Implementation of the WIN2030 lpcpu (client side).
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/random.h>
+#include <uapi/linux/dma-heap.h>
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/delay.h>
+#include <linux/miscdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/debugfs.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
+#include <linux/wait.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/firmware.h>
+#include <linux/of_address.h>
+#include <linux/dma-map-ops.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/iommu.h>
+// #include <linux/mailbox/eswin-ipc-scpu.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+
+#define LPCPU_FW_RESERVED
+#define FW_BOOT_ADDR 0x80000000
+
+struct lowpower_info {
+	int level;
+};
+
+#define LPCPU_IOC_MAGIC 'L'
+#define LPCPU_IOC_SAVEPOWER                          \
+	_IOWR(LPCPU_IOC_MAGIC, 0x1, struct lowpower_info)
+
+#define FW_LOAD_UNKNOW 0
+#define FW_LOAD_SUCC   0xacce55
+
+struct lpcpu_dev {
+	struct miscdevice mdev;
+	struct mutex lock;
+	struct mbox_chan *mbox_channel;
+	// struct dma_allocation_data send_buff;
+	wait_queue_head_t waitq;
+	u32 num;
+	u8 *req_msg;
+	u32 res_size;
+	int numa_id;
+	struct clk *core_clk;
+	struct clk *bus_clk;
+	struct reset_control *core_rst;
+	struct reset_control *bus_rst;
+	struct reset_control *dbg_rst;
+	void __iomem *mmio;
+	size_t fw_size;
+};
+
+static struct lpcpu_dev *lpcpu;
+static u32 load_event = FW_LOAD_UNKNOW;
+
+struct mbox_msg{
+	u32 data_l;
+	u32 data_h;
+};
+
+static void eswin_lpcpu_rx_callback(struct mbox_client *client, void *msg)
+{
+	struct mbox_msg *umsg = msg;
+	struct device *dev = lpcpu->mdev.parent;
+	dev_dbg(dev, "lpcpu rx callback : %llx\n",*(u64 *)msg);
+	printk("data_l= %x, data_h = %x\n",umsg->data_l,umsg->data_h);
+	if (load_event != FW_LOAD_SUCC) {
+		load_event = *(u32 *)msg;
+		wake_up(&lpcpu->waitq);
+		printk("eswin_lpcpu_rx_callback returned \n");
+	}
+
+	return;
+}
+
+static void eswin_lpcpu_tx_done(struct mbox_client *client, void *msg, int r)
+{
+	if (r)
+		dev_warn(client->dev, "Client: Message could not be sent:%d\n", r);
+	else
+		dev_dbg(client->dev, "Client: Message sent\n");
+}
+
+static struct mbox_chan *eswin_lpcpu_request_channel(struct platform_device *pdev,
+						   const char *name)
+{
+	struct mbox_client *client;
+	struct mbox_chan *channel;
+
+	client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
+	if (!client)
+		return ERR_PTR(-ENOMEM);
+
+	client->dev = &pdev->dev;
+	client->rx_callback = eswin_lpcpu_rx_callback;
+	client->tx_prepare = NULL;
+	client->tx_done = eswin_lpcpu_tx_done;
+	client->tx_block = false;
+	client->knows_txdone = false;
+
+	channel = mbox_request_channel_byname(client, name);
+	if (IS_ERR(channel)) {
+		dev_warn(&pdev->dev, "Failed to request %s channel\n", name);
+		return NULL;
+	}
+	dev_dbg(&pdev->dev, "request mbox chan %s\n", name);
+
+	return channel;
+}
+
+static int eswin_lpcpu_open(struct inode *inode, struct file *filp)
+{
+	int ret = 0;
+	struct device *dev = NULL;
+
+	dev = lpcpu->mdev.parent;
+
+	dev_info(dev, "%s\n", __func__);
+
+	return ret;
+}
+
+static ssize_t eswin_lpcpu_read(struct file *filp, char __user *userbuf,
+				      size_t count, loff_t *ppos)
+{
+	printk("lpcpu read\n");
+	return 0;
+}
+
+static ssize_t eswin_lpcpu_write(struct file *filp,
+				       const char __user *userbuf, size_t count,
+				       loff_t *ppos)
+{
+	printk("lpcpu write\n");
+	u8 msg[8];
+	msg[0] = 0xca;
+	msg[1] = 0xec;
+	msg[2] = 0x55;
+	int ret = 0;
+	ret = mbox_send_message(lpcpu->mbox_channel, msg);
+	if (ret < 0){
+		ret = -EAGAIN;
+		// dev_dbg(dev, "Failed to send message via mailbox\r\n");
+	}
+	return count;
+}
+
+static __poll_t eswin_lpcpu_poll(struct file *filp,
+					  struct poll_table_struct *wait)
+{
+	poll_wait(filp, &lpcpu->waitq, wait);
+
+	// if (eswin_ipc_service_ready(session))
+	// 	return EPOLLIN | EPOLLRDNORM;
+	return 0;
+}
+
+static int eswin_lpcpu_release(struct inode *inode, struct file *filp)
+{
+
+	pr_info("%s called!\n", __func__);
+
+	return 0;
+}
+
+static long eswin_lpcpu_ioctl(struct file *filp, unsigned int cmd,
+			    unsigned long arg)
+{
+	struct lowpower_info lowpower;
+	struct device *dev = lpcpu->mdev.parent;
+	u8 msg[8];
+	// void *cpu_vaddr = NULL;
+	// struct dmabuf_bank_info *info, *info_free;
+	// unsigned int cmd_size = 0;
+	// size_t buf_size = 0;
+	int ret = 0;
+	printk("ioctl: %x",cmd);
+
+	if (cmd & IOC_IN) {
+		if (copy_from_user(&lowpower, (void __user *)arg, sizeof(struct lowpower_info)) != 0) {
+			pr_err("ioctl copy_from_user failed.\n");
+			ret = -EFAULT;
+			return ret;
+		}
+	}
+	// else if (cmd & IOC_OUT) {
+	// 	memset(kdata, 0, usize);
+	// }
+
+	switch (cmd) {
+		/* alloc memory by driver using dmabuf heap helper API  */
+		case LPCPU_IOC_SAVEPOWER: {
+			msg[0] = 0x55;
+			msg[1] = 0xaa;
+
+			ret = mbox_send_message(lpcpu->mbox_channel, msg);
+			if (ret < 0){
+				ret = -EAGAIN;
+				dev_dbg(dev, "Failed to send message via mailbox\r\n");
+			}
+
+			break;
+		}
+
+		default: {
+			dev_err(dev, "Invalid IOCTL command %u\n", cmd);
+			return -ENOTTY;
+		}
+	}
+
+	return ret;
+}
+
+static const struct file_operations eswin_lpcpu_ops = {
+	.owner = THIS_MODULE,
+	.write = eswin_lpcpu_write,
+	.read = eswin_lpcpu_read,
+	.open = eswin_lpcpu_open,
+	.poll = eswin_lpcpu_poll,
+	.release = eswin_lpcpu_release,
+	.unlocked_ioctl = eswin_lpcpu_ioctl,
+};
+
+static int lpcpu_boot_status(struct mbox_chan *mbox_channel)
+{
+	int ret = 0;
+	u8 msg[8];
+	msg[0] = 0xca;
+	msg[1] = 0xec;
+	msg[2] = 0x55;
+
+	ret = mbox_send_message(mbox_channel, msg);
+	if (ret < 0){
+		ret = -EAGAIN;
+		printk("Failed to send message via mailbox\r\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+// TODO: add clk, reset tbu config
+static int eswin_lpcpu_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const char *mbox_channel_name;
+	const struct firmware *lpcpu_fw;
+	int numa_id = 0;
+	int ret;
+	struct device_node *node;
+	struct resource res_fw;
+	void __iomem *mem_fw;
+	long timeout;
+
+	lpcpu = devm_kzalloc(dev, sizeof(*lpcpu), GFP_KERNEL);
+	if (!lpcpu)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, lpcpu);
+
+	lpcpu->mdev.minor = MISC_DYNAMIC_MINOR;
+	lpcpu->mdev.name = "lpcpu";
+	lpcpu->mdev.fops = &eswin_lpcpu_ops;
+	lpcpu->mdev.parent = dev;
+
+	if(of_property_read_u32(pdev->dev.of_node, "numa-node-id", &numa_id)) {
+		numa_id = 0;
+	}
+	dev_dbg(&pdev->dev, "numa_id=%d\n", numa_id);
+	lpcpu->numa_id = numa_id;
+
+	ret = misc_register(&lpcpu->mdev);
+	if (ret) {
+		dev_err(dev, "failed to register misc device: %d\n", ret);
+		goto err_misc;
+	}
+
+	/*use eswin mailbox0 to send msg and mailbox1 receive msg*/
+	ret = device_property_read_string(&pdev->dev, "mbox-names",
+					  &mbox_channel_name);
+	if (ret == 0) {
+		lpcpu->mbox_channel = eswin_lpcpu_request_channel(pdev, mbox_channel_name);
+	}else{
+		dev_err(dev, "given arguments are not valid: %d\n", ret);
+		goto err_mailbox;
+	}
+
+	mutex_init(&lpcpu->lock);
+	init_waitqueue_head(&lpcpu->waitq);
+
+	lpcpu->core_clk = devm_clk_get(dev, "core_clk");
+	if (IS_ERR(lpcpu->core_clk)) {
+		dev_err(dev, "core clock source missing or invalid\n");
+		goto err_clkrst;
+	}
+
+	lpcpu->bus_clk = devm_clk_get(dev, "bus_clk");
+	if (IS_ERR(lpcpu->bus_clk)) {
+		dev_err(dev, "bus clock source missing or invalid\n");
+		goto err_clkrst;
+	}
+
+	lpcpu->core_rst = devm_reset_control_get_optional(&pdev->dev, "core_rst");
+	if (IS_ERR_OR_NULL(lpcpu->core_rst)) {
+		dev_err_probe(dev, PTR_ERR(lpcpu->core_rst), "unable to get core reset\n");
+		goto err_clkrst;
+	}
+
+	lpcpu->bus_rst = devm_reset_control_get_optional(&pdev->dev, "bus_rst");
+	if (IS_ERR_OR_NULL(lpcpu->bus_rst)) {
+		dev_err_probe(dev, PTR_ERR(lpcpu->bus_rst), "unable to get bus reset\n");
+		goto err_clkrst;
+	}
+
+	lpcpu->dbg_rst = devm_reset_control_get_optional(&pdev->dev, "dbg_rst");
+	if (IS_ERR_OR_NULL(lpcpu->dbg_rst)) {
+		dev_err_probe(dev, PTR_ERR(lpcpu->dbg_rst), "unable to get dbg reset\n");
+		goto err_clkrst;
+	}
+	#define LPCPU_BOOT_ADDR 0x51828314
+	if (!lpcpu->numa_id)
+		lpcpu->mmio = ioremap(LPCPU_BOOT_ADDR, 4);
+	else
+		lpcpu->mmio = ioremap(LPCPU_BOOT_ADDR+0x20000000, 4);
+
+	if (!lpcpu->mmio) {
+		pr_err("lpcpu ioremap fail.\n");
+		ret = -ENOMEM;
+		goto err_mmio;
+	}
+
+	ret = lpcpu_boot_status(lpcpu->mbox_channel);
+	if (ret < 0) {
+		dev_err(dev, "Send message to lpcpu via mailbox failed!\n");
+		goto err_mmio;
+	}
+	timeout = wait_event_timeout(lpcpu->waitq,
+			load_event == FW_LOAD_SUCC,usecs_to_jiffies(100000));
+
+	if (!timeout) {
+		dev_err(dev, "Lpcpu is not boot!\n");
+		ret = -EBUSY;
+		goto err_mmio;
+	}
+
+	dev_info(dev, "eswin lpcpu initialized\n");
+
+	return 0;
+
+err_mmio:
+err_clkrst:
+	mbox_free_channel(lpcpu->mbox_channel);
+err_mailbox:
+	misc_deregister(&lpcpu->mdev);
+err_misc:
+	devm_kfree(&pdev->dev,lpcpu);
+	return ret;
+}
+
+static int eswin_lpcpu_remove(struct platform_device *pdev)
+{
+	struct lpcpu_dev *_dev = platform_get_drvdata(pdev);
+
+	if (_dev->mbox_channel)
+		mbox_free_channel(_dev->mbox_channel);
+	misc_deregister(&_dev->mdev);
+	iounmap(_dev->mmio);
+	devm_kfree(&pdev->dev,_dev);
+	dev_dbg(&pdev->dev, "%s remove!\n", pdev->name);
+
+	return 0;
+}
+
+static const struct of_device_id eswin_lpcpu_match[] = {
+	{
+		.compatible = "eswin,win2030-lpcpu",
+	},
+	{ /* Sentinel */ }
+};
+
+static struct platform_driver eswin_lpcpu_driver = {
+    .driver = {
+	.name = "win2030-lpcpu",
+	.of_match_table = eswin_lpcpu_match,
+    },
+    .probe = eswin_lpcpu_probe,
+    .remove = eswin_lpcpu_remove,
+};
+
+static int __init lpcpu_modules_init(void)
+{
+	int err;
+
+	err = platform_driver_register(&eswin_lpcpu_driver);
+	if (err < 0){
+		pr_err("lpcpu:platform_register_drivers failed!err=%d\n",err);
+	}
+
+	return err;
+}
+module_init(lpcpu_modules_init);
+
+static void __exit lpcpu_modules_exit(void)
+{
+	platform_driver_unregister(&eswin_lpcpu_driver);
+}
+module_exit(lpcpu_modules_exit);
+
+MODULE_DESCRIPTION("ESWIN WIN2030 lpcpu driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mailbox/eswin-mailbox.c b/drivers/mailbox/eswin-mailbox.c
new file mode 100755
index 000000000000..6727882ee682
--- /dev/null
+++ b/drivers/mailbox/eswin-mailbox.c
@@ -0,0 +1,577 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Mailbox Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_controller.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/mailbox/eswin-mailbox.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/pm_runtime.h>
+
+
+#define ESWIN_MBOX_FIFO_DEPTH 8
+
+struct eswin_mbox_data {
+	int num_chans;
+};
+
+struct eswin_mbox_chan {
+	int idx;
+	int irq;
+	struct eswin_mbox_msg msg[ESWIN_MBOX_FIFO_DEPTH];
+	int msg_cnt;
+	struct eswin_mbox *mb;
+};
+
+/*
+ * Registers offset
+ */
+#define ESWIN_MBOX_WR_DATA0 0x00
+#define ESWIN_MBOX_WR_DATA1 0x04
+#define ESWIN_MBOX_RD_DATA0 0x08
+#define ESWIN_MBOX_RD_DATA1 0x0C
+#define ESWIN_MBOX_FIFO_STATUS 0x10
+#define ESWIN_MBOX_MB_ERR 0x14
+#define ESWIN_MBOX_INT_CTRL 0x18
+#define ESWIN_MBOX_WR_LOCK 0x1C
+
+struct eswin_mbox {
+	struct mbox_controller mbox;
+	struct clk *pclk;
+	struct clk *pclk_device;
+	struct reset_control *rst;
+	struct reset_control *rst_device;
+	void __iomem *mbox_base;
+	void __iomem *mbox_rx_base;
+	struct regmap *map;
+	struct regmap *rx_map;
+	struct device *dev;
+	u32 lock_bit;
+	u32 irq_bit;
+	struct eswin_mbox_chan *chans;
+	spinlock_t rx_lock;
+};
+
+static int eswin_mbox_send_data(struct mbox_chan *chan, void *data)
+{
+	u32 tmp_data;
+
+	struct eswin_mbox *mb = dev_get_drvdata(chan->mbox->dev);
+	struct eswin_mbox_msg *msg = (struct eswin_mbox_msg *)data;
+
+	dev_dbg(mb->mbox.dev, "send_data\n");
+	if (!msg)
+		return -EINVAL;
+
+	// TX FIFO FULL?
+	if (regmap_test_bits(mb->map, ESWIN_MBOX_FIFO_STATUS, BIT_ULL(0))) {
+		return -EBUSY;
+	}
+
+	tmp_data = (u32)msg->data;
+	regmap_write(mb->map, ESWIN_MBOX_WR_DATA0, tmp_data);
+
+	tmp_data = (u32)(msg->data >> 32) | BIT(31);
+	regmap_write(mb->map, ESWIN_MBOX_WR_DATA1, tmp_data);
+	// Write interrupt enable bit.
+	regmap_set_bits(mb->map, ESWIN_MBOX_INT_CTRL, mb->irq_bit);
+	return 0;
+}
+
+static int eswin_mbox_startup(struct mbox_chan *chan)
+{
+	struct eswin_mbox *mb = dev_get_drvdata(chan->mbox->dev);
+	int ret;
+
+	pm_runtime_get_sync(mb->dev);
+
+	if (regmap_test_bits(mb->map, ESWIN_MBOX_WR_LOCK, mb->lock_bit)) {
+		pm_runtime_put(mb->dev);
+		return -1;
+	}
+	ret = regmap_set_bits(mb->map, ESWIN_MBOX_WR_LOCK, mb->lock_bit);
+
+	/* Successfully write the occupancy flag bit, indicating successful occupancy */
+	dev_dbg(mb->mbox.dev, "start, ret %d, lock_bit 0x%x\n", ret,
+		mb->lock_bit);
+	return ret;
+}
+
+static void eswin_mbox_shutdown(struct mbox_chan *chan)
+{
+	struct eswin_mbox *mb = dev_get_drvdata(chan->mbox->dev);
+	int ret;
+
+	ret = regmap_clear_bits(mb->map, ESWIN_MBOX_WR_LOCK, mb->lock_bit);
+	if (0 != ret)
+		dev_err(mb->mbox.dev, "failed to shutdown mailbox\n");
+
+	ret = regmap_clear_bits(mb->map, ESWIN_MBOX_INT_CTRL, mb->irq_bit);
+	if (0 != ret)
+		dev_err(mb->mbox.dev, "failed to disable  mailbox int\n");
+
+	pm_runtime_put(mb->dev);
+	return;
+}
+
+static int eswin_mbox_receive_data(struct eswin_mbox *mb,
+				   struct eswin_mbox_msg *msg)
+{
+	u32 tmp_data;
+	u32 tmp_data0;
+	u64 tmp;
+
+	regmap_read(mb->rx_map, ESWIN_MBOX_RD_DATA0, &tmp_data0);
+	regmap_read(mb->rx_map, ESWIN_MBOX_RD_DATA1, &tmp_data);
+
+	// RX FIFO empty ?
+	if (tmp_data == 0) {
+		return -1;
+	}
+	tmp = (u64)tmp_data << 32 | tmp_data0;
+
+	msg->data = tmp;
+
+	/*trigger FIFO dequeue*/
+	regmap_write(mb->rx_map, ESWIN_MBOX_RD_DATA1, 0x0);
+	return 0;
+}
+
+static bool eswin_mbox_peek_data(struct mbox_chan *chan)
+{
+	int idx;
+	struct mbox_controller *mbox = chan->mbox;
+	struct eswin_mbox_msg *msg = NULL;
+	struct eswin_mbox *mb = container_of(mbox, struct eswin_mbox, mbox);
+	bool IsData = false;
+	unsigned long flags;
+
+	for (idx = 0; idx < mb->mbox.num_chans; idx++) {
+		spin_lock_irqsave(&mb->rx_lock, flags);
+		msg = &mb->chans[idx].msg[0];
+		if (0 != eswin_mbox_receive_data(mb, msg)) {
+			spin_unlock_irqrestore(&mb->rx_lock, flags);
+			continue;
+		}
+		IsData = true;
+		mb->chans[idx].msg_cnt--;
+		if (NULL != mb->mbox.chans[idx].cl) {
+			mbox_chan_received_data(&mb->mbox.chans[idx], msg);
+		}
+		spin_unlock_irqrestore(&mb->rx_lock, flags);
+	}
+	return IsData;
+}
+
+/*
+    once the data has been enqueued to mailbox hw FIFO in send_data function,
+    we believe that tx is done
+*/
+static bool eswin_mbox_last_tx_done(struct mbox_chan *chan)
+{
+	return true;
+}
+
+static const struct mbox_chan_ops eswin_mbox_chan_ops = {
+	.send_data = eswin_mbox_send_data,
+	.peek_data = eswin_mbox_peek_data,
+	.startup = eswin_mbox_startup,
+	.shutdown = eswin_mbox_shutdown,
+	.last_tx_done = eswin_mbox_last_tx_done,
+};
+
+static irqreturn_t eswin_mbox_irq(int irq, void *dev_id)
+{
+	int idx;
+	struct eswin_mbox *mb = (struct eswin_mbox *)dev_id;
+	unsigned long flags;
+
+	for (idx = 0; idx < mb->mbox.num_chans; idx++) {
+		if (irq != mb->chans[idx].irq)
+			continue;
+
+		spin_lock_irqsave(&mb->rx_lock, flags);
+		WARN_ON(0 != mb->chans[idx].msg_cnt);
+		while (0 ==
+		       eswin_mbox_receive_data(
+			       mb,
+			       &mb->chans[idx].msg[mb->chans[idx].msg_cnt])) {
+			mb->chans[idx].msg_cnt++;
+			/*
+				 MCU may continue enqeuing fifo when we are dequeuing fifo,
+				 So we receive up to ESWIN_MBOX_FIFO_DEPTH cnt msgs one time.
+				 The left msgs will be handled after eswin_mbox_isr finished.
+				*/
+			if (ESWIN_MBOX_FIFO_DEPTH == mb->chans[idx].msg_cnt) {
+				break;
+			}
+		};
+		spin_unlock_irqrestore(&mb->rx_lock, flags);
+	}
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t eswin_mbox_isr(int irq, void *dev_id)
+{
+	int idx;
+	struct eswin_mbox_msg *msg = NULL;
+	struct eswin_mbox *mb = (struct eswin_mbox *)dev_id;
+	unsigned long flags;
+	int i;
+
+	for (idx = 0; idx < mb->mbox.num_chans; idx++) {
+		if (irq != mb->chans[idx].irq)
+			continue;
+
+		i = 0;
+		spin_lock_irqsave(&mb->rx_lock, flags);
+		while (mb->chans[idx].msg_cnt) {
+			msg = &mb->chans[idx].msg[i++];
+			if (NULL != mb->mbox.chans[idx].cl) {
+				dev_dbg(mb->mbox.dev,
+					"Chan[%d]: receive MCU message, msg %p\n",
+					idx, msg);
+				mbox_chan_received_data(&mb->mbox.chans[idx],
+							msg);
+			}
+			mb->chans[idx].msg_cnt--;
+		}
+		spin_unlock_irqrestore(&mb->rx_lock, flags);
+	}
+	return IRQ_HANDLED;
+}
+
+static const struct eswin_mbox_data win2030_drv_data = {
+	.num_chans = 1,
+};
+
+static const struct of_device_id eswin_mbox_of_match[] = {
+	{ .compatible = "eswin,win2030-mailbox", .data = &win2030_drv_data },
+	{},
+};
+MODULE_DEVICE_TABLE(of, eswin_mbox_of_match);
+
+static int eswin_mbox_reg_read(void *context, unsigned int reg,
+		unsigned int *val)
+{
+	struct eswin_mbox *mb = context;
+
+	*val = readl_relaxed(mb->mbox_base + reg);
+	return 0;
+}
+
+static int eswin_mbox_reg_write(void *context, unsigned int reg,
+				unsigned int val)
+{
+	struct eswin_mbox *mb = context;
+
+	writel_relaxed(val, mb->mbox_base + reg);
+	return 0;
+}
+
+static int eswin_mbox_rx_reg_read(void *context, unsigned int reg,
+				  unsigned int *val)
+{
+	struct eswin_mbox *mb = context;
+
+	*val = readl_relaxed(mb->mbox_rx_base + reg);
+	return 0;
+}
+
+static int eswin_mbox_rx_reg_write(void *context, unsigned int reg,
+				    unsigned int val)
+{
+	struct eswin_mbox *mb = context;
+
+	writel_relaxed(val, mb->mbox_rx_base + reg);
+	return 0;
+}
+
+/**
+ * eswin_mbox_init_regmap() - Initialize registers map
+ * @dev: device private data
+ *
+ * Autodetects needed register access mode and creates the regmap with
+ * corresponding read/write callbacks. This must be called before doing any
+ * other register access.
+ */
+int eswin_mbox_init_regmap(struct eswin_mbox *mb)
+{
+	struct regmap_config map_cfg = {
+		.reg_bits = 32,
+		.val_bits = 32,
+		.reg_stride = 4,
+		.use_hwlock = true,
+		.cache_type = REGCACHE_NONE,
+		.can_sleep = false,
+		.reg_read = eswin_mbox_reg_read,
+		.reg_write = eswin_mbox_reg_write,
+	};
+	struct regmap_config rx_map_cfg = {
+		.reg_bits = 32,
+		.val_bits = 32,
+		.reg_stride = 4,
+		.use_hwlock = true,
+		.cache_type = REGCACHE_NONE,
+		.can_sleep = false,
+		.reg_read = eswin_mbox_rx_reg_read,
+		.reg_write = eswin_mbox_rx_reg_write,
+	};
+
+	/*
+	 * Note we'll check the return value of the regmap IO accessors only
+	 * at the probe stage. The rest of the code won't do this because
+	 * basically we have MMIO-based regmap so non of the read/write methods
+	 * can fail.
+	 */
+	mb->map = devm_regmap_init(mb->dev, NULL, mb, &map_cfg);
+	if (IS_ERR(mb->map)) {
+		dev_err(mb->dev, "Failed to init the registers map\n");
+		return PTR_ERR(mb->map);
+	}
+	mb->rx_map = devm_regmap_init(mb->dev, NULL, mb, &rx_map_cfg);
+	if (IS_ERR(mb->rx_map)) {
+		dev_err(mb->dev, "Failed to init the registers rx map\n");
+		regmap_exit(mb->map);
+		return PTR_ERR(mb->rx_map);
+	}
+
+	return 0;
+}
+
+static int eswin_mbox_prepare_clk(struct device *dev, bool enable)
+{
+	int ret = 0;
+	struct eswin_mbox *mb = dev_get_drvdata(dev);
+
+	if (enable) {
+		ret = clk_prepare_enable(mb->pclk);
+		if (ret) {
+			dev_err(dev, "failed to enable host mailbox pclk: %d\n", ret);
+			return ret;
+		}
+		ret = clk_prepare_enable(mb->pclk_device);
+		if (ret) {
+			dev_err(dev, "failed to enable device mailbox pclk: %d\n", ret);
+			return ret;
+		}
+	} else {
+		clk_disable_unprepare(mb->pclk);
+		clk_disable_unprepare(mb->pclk_device);
+	}
+	return ret;
+}
+
+static int eswin_mbox_probe(struct platform_device *pdev)
+{
+	struct eswin_mbox *mb;
+	const struct of_device_id *match;
+	const struct eswin_mbox_data *drv_data;
+	struct resource *res;
+	int ret, irq, i;
+
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
+	match = of_match_node(eswin_mbox_of_match, pdev->dev.of_node);
+	drv_data = (const struct eswin_mbox_data *)match->data;
+
+	mb = devm_kzalloc(&pdev->dev, sizeof(*mb), GFP_KERNEL);
+	if (!mb)
+		return -ENOMEM;
+
+	if (of_property_read_u32(pdev->dev.of_node, "lock-bit",
+				 &mb->lock_bit)) {
+		dev_err(&pdev->dev, "failed to get lock_bit: %d\n", ret);
+	}
+
+	if (of_property_read_u32(pdev->dev.of_node, "irq-bit", &mb->irq_bit)) {
+		dev_err(&pdev->dev, "failed to get irq_bit: %d\n", ret);
+	}
+	mb->chans = devm_kcalloc(&pdev->dev, drv_data->num_chans,
+				 sizeof(*mb->chans), GFP_KERNEL);
+	if (!mb->chans)
+		return -ENOMEM;
+
+	mb->mbox.chans = devm_kcalloc(&pdev->dev, drv_data->num_chans,
+				      sizeof(*mb->mbox.chans), GFP_KERNEL);
+	if (!mb->mbox.chans)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mb);
+
+	mb->mbox.dev = &pdev->dev;
+	mb->mbox.num_chans = drv_data->num_chans;
+	mb->mbox.ops = &eswin_mbox_chan_ops;
+	mb->mbox.txdone_irq = false;
+	mb->mbox.txdone_poll = true;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+
+	mb->mbox_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mb->mbox_base))
+		return PTR_ERR(mb->mbox_base);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (!res)
+		return -ENODEV;
+
+	mb->mbox_rx_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mb->mbox_rx_base))
+		return PTR_ERR(mb->mbox_rx_base);
+
+	mb->pclk = devm_clk_get(&pdev->dev, "pclk_mailbox_host");
+	if (IS_ERR(mb->pclk)) {
+		ret = PTR_ERR(mb->pclk);
+		dev_err(&pdev->dev, "failed to get host mailbox clock: %d\n",
+			ret);
+		return ret;
+	}
+
+	mb->pclk_device = devm_clk_get(&pdev->dev, "pclk_mailbox_device");
+	if (IS_ERR(mb->pclk_device)) {
+		ret = PTR_ERR(mb->pclk_device);
+		dev_err(&pdev->dev, "failed to get device mailbox clock: %d\n",
+			ret);
+		return ret;
+	}
+	eswin_mbox_prepare_clk(&pdev->dev, true);
+
+	mb->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "rst");
+	if (IS_ERR(mb->rst))
+		return PTR_ERR(mb->rst);
+	reset_control_reset(mb->rst);
+
+	mb->rst_device = devm_reset_control_get_optional_exclusive(
+		&pdev->dev, "rst_device");
+	if (IS_ERR(mb->rst_device))
+		return PTR_ERR(mb->rst_device);
+	reset_control_reset(mb->rst_device);
+
+	for (i = 0; i < mb->mbox.num_chans; i++) {
+		irq = platform_get_irq(pdev, i);
+		if (irq < 0)
+			return irq;
+
+		ret = devm_request_threaded_irq(&pdev->dev, irq, eswin_mbox_irq,
+				eswin_mbox_isr, IRQF_ONESHOT, dev_name(&pdev->dev), mb);
+		if (ret < 0)
+			return ret;
+
+		mb->chans[i].idx = i;
+		mb->chans[i].irq = irq;
+		mb->chans[i].mb = mb;
+	}
+	mb->dev = &pdev->dev;
+	ret = eswin_mbox_init_regmap(mb);
+	if (ret)
+		return ret;
+
+	spin_lock_init(&mb->rx_lock);
+
+	/* The code below assumes runtime PM to be disabled. */
+	WARN_ON(pm_runtime_enabled(&pdev->dev));
+
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 1000);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+
+	ret = devm_mbox_controller_register(&pdev->dev, &mb->mbox);
+	if (ret < 0) {
+		pm_runtime_disable(&pdev->dev);
+		dev_err(&pdev->dev, "failed to register mailbox: %d\n", ret);
+	}
+	dev_info(&pdev->dev, "register sucessfully\n");
+	return ret;
+}
+
+static int eswin_mbox_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct eswin_mbox *mb = platform_get_drvdata(pdev);
+
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	ret = reset_control_assert(mb->rst);
+	WARN_ON(ret != 0);
+	ret = reset_control_assert(mb->rst_device);
+	WARN_ON(ret != 0);
+	return 0;
+}
+
+__maybe_unused static int eswin_mbox_suspend(struct device *dev)
+{
+	if (!pm_runtime_status_suspended(dev)) {
+		return eswin_mbox_prepare_clk(dev, false);
+	}
+	return 0;
+}
+
+__maybe_unused static int eswin_mbox_resume(struct device *dev)
+{
+	if (!pm_runtime_status_suspended(dev)) {
+		eswin_mbox_prepare_clk(dev, true);
+		pm_runtime_mark_last_busy(dev);
+		pm_request_autosuspend(dev);
+	}
+	return 0;
+}
+
+__maybe_unused static int eswin_mbox_runtime_suspend(struct device *dev)
+{
+	return eswin_mbox_prepare_clk(dev, false);
+}
+
+__maybe_unused static int eswin_mbox_runtime_resume(struct device *dev)
+{
+	return eswin_mbox_prepare_clk(dev, true);
+}
+
+static const struct dev_pm_ops eswin_mbox_dev_pm_ops = {
+	LATE_SYSTEM_SLEEP_PM_OPS(eswin_mbox_suspend, eswin_mbox_resume)
+	RUNTIME_PM_OPS(eswin_mbox_runtime_suspend, eswin_mbox_runtime_resume, NULL)
+};
+
+static struct platform_driver eswin_mbox_driver = {
+	.probe	= eswin_mbox_probe,
+	.remove = eswin_mbox_remove,
+	.driver = {
+		.name = "eswin-mailbox",
+		.of_match_table = of_match_ptr(eswin_mbox_of_match),
+		.pm	= pm_ptr(&eswin_mbox_dev_pm_ops),
+	},
+};
+
+module_platform_driver(eswin_mbox_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Eswin mailbox: communicate between CPU cores and MCUs");
+MODULE_AUTHOR("Huang Yifeng <huangyifeng@eswincomputing.com>");
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 53b443be5a59..a0205c8368a1 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -99,7 +99,7 @@ config VIDEO_IMX214
 
 config VIDEO_IMX219
 	tristate "Sony IMX219 sensor support"
-	select V4L2_CCI_I2C
+#	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
 	  IMX219 camera.
@@ -107,6 +107,17 @@ config VIDEO_IMX219
 	  To compile this driver as a module, choose M here: the
 	  module will be called imx219.
 
+config VIDEO_IMX327
+	tristate "Sony IMX327 sensor support"
+	# depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	# depends on MEDIA_CAMERA_SUPPORT
+	help
+	  This is a Video4Linux2 sensor driver for the Sony
+	  IMX327 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx327.
+
 config VIDEO_IMX258
 	tristate "Sony IMX258 sensor support"
 	help
@@ -152,6 +163,17 @@ config VIDEO_IMX319
 	  To compile this driver as a module, choose M here: the
 	  module will be called imx319.
 
+config VIDEO_IMX327
+	tristate "Sony IMX327 sensor support"
+	#depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	#depends on MEDIA_CAMERA_SUPPORT
+	help
+	  This is a Video4Linux2 sensor driver for the Sony
+	  IMX327 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx327.
+
 config VIDEO_IMX334
 	tristate "Sony IMX334 sensor support"
 	depends on OF_GPIO
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index 80b00d39b48f..f782c9be9d26 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -48,6 +48,7 @@ obj-$(CONFIG_VIDEO_IMX274) += imx274.o
 obj-$(CONFIG_VIDEO_IMX290) += imx290.o
 obj-$(CONFIG_VIDEO_IMX296) += imx296.o
 obj-$(CONFIG_VIDEO_IMX319) += imx319.o
+obj-$(CONFIG_VIDEO_IMX327) += imx327.o
 obj-$(CONFIG_VIDEO_IMX334) += imx334.o
 obj-$(CONFIG_VIDEO_IMX335) += imx335.o
 obj-$(CONFIG_VIDEO_IMX355) += imx355.o
@@ -145,3 +146,4 @@ obj-$(CONFIG_VIDEO_VP27SMPX) += vp27smpx.o
 obj-$(CONFIG_VIDEO_VPX3220) += vpx3220.o
 obj-$(CONFIG_VIDEO_WM8739) += wm8739.o
 obj-$(CONFIG_VIDEO_WM8775) += wm8775.o
+obj-$(CONFIG_VIDEO_IMX327) += imx327.o
diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c
index a3d5a8a7c660..5184683432c8 100644
--- a/drivers/media/i2c/imx219.c
+++ b/drivers/media/i2c/imx219.c
@@ -85,8 +85,10 @@
 #define IMX219_FLL_STEP			1
 #define IMX219_FLL_DEFAULT		0x0c98
 
-/* HBLANK control - read only */
-#define IMX219_PPL_DEFAULT		3448
+/* HBLANK control range */
+#define IMX219_PPL_MIN			3448
+#define IMX219_PPL_MAX			0x7ff0
+#define IMX219_REG_HTS			CCI_REG16(0x0162)
 
 #define IMX219_REG_LINE_LENGTH_A	CCI_REG16(0x0162)
 #define IMX219_REG_X_ADD_STA_A		CCI_REG16(0x0164)
@@ -102,8 +104,8 @@
 /* Binning  Mode */
 #define IMX219_REG_BINNING_MODE		CCI_REG16(0x0174)
 #define IMX219_BINNING_NONE		0x0000
-#define IMX219_BINNING_2X2		0x0101
-#define IMX219_BINNING_2X2_ANALOG	0x0303
+#define IMX219_BINNING_2X2_NORMAL	0x0101
+#define IMX219_BINNING_2X2_SPECIAL	0x0303
 
 #define IMX219_REG_CSI_DATA_FORMAT_A	CCI_REG16(0x018c)
 
@@ -146,19 +148,86 @@
 
 /* Pixel rate is fixed for all the modes */
 #define IMX219_PIXEL_RATE		182400000
-#define IMX219_PIXEL_RATE_4LANE		280800000
+#define IMX219_PIXEL_RATE_4LANE		281600000
 
 #define IMX219_DEFAULT_LINK_FREQ	456000000
-#define IMX219_DEFAULT_LINK_FREQ_4LANE	363000000
+#define IMX219_DEFAULT_LINK_FREQ_4LANE_UNSUPPORTED	363000000
+#define IMX219_DEFAULT_LINK_FREQ_4LANE	364000000
 
 /* IMX219 native and active pixel array size. */
 #define IMX219_NATIVE_WIDTH		3296U
 #define IMX219_NATIVE_HEIGHT		2480U
-#define IMX219_PIXEL_ARRAY_LEFT		8U
-#define IMX219_PIXEL_ARRAY_TOP		8U
+#define IMX219_PIXEL_ARRAY_LEFT		0U
+#define IMX219_PIXEL_ARRAY_TOP		0U
 #define IMX219_PIXEL_ARRAY_WIDTH	3280U
 #define IMX219_PIXEL_ARRAY_HEIGHT	2464U
 
+/* Embedded metadata stream structure */
+#define IMX219_EMBEDDED_LINE_WIDTH 16384
+#define IMX219_NUM_EMBEDDED_LINES 1
+
+
+#define IMX219_AGAIN_TABLE 233
+
+
+static int imx219_again_table[IMX219_AGAIN_TABLE] =
+
+{
+	0,	3,	7,	10,	14,	17,	21,	24,
+	28, 31, 35, 38, 42, 45, 49, 52,
+	56, 60, 63,	67, 71, 74, 78, 82,
+	86, 89, 93, 97, 101, 104, 108, 112,
+	116, 120, 124, 128, 132, 136, 140, 144,
+	148, 152, 156, 160, 164, 168, 172, 176,
+	180, 185, 189, 193, 197, 201, 206, 210,
+	214, 219, 223, 228, 232, 236, 241, 245,
+	250, 254, 259, 264, 268, 273, 277, 282,
+	287, 292, 296, 301, 306, 311, 316, 321, 
+	325, 330, 335, 340, 345, 350, 356, 361, 
+	366, 371, 376, 382, 387, 392, 397, 403,
+	408, 414, 419, 425, 430, 436, 441, 447,
+	453, 459, 464, 470, 476, 482, 488, 494,
+	500, 506, 512, 518, 524, 530, 537, 543,
+	549, 556, 562, 569, 575, 582, 589, 595,
+	602, 609, 616, 623, 630, 637, 644, 651,
+	658, 665, 673, 680, 688, 695, 703, 710,
+	718, 726, 734, 742, 750, 758, 766, 774,
+	782, 791, 799, 808, 816, 825, 834, 843,
+	852, 861, 870, 880, 889, 898, 908, 918, 
+	928, 937, 947, 958, 968, 978, 989, 1000,
+	1010, 1021, 1032, 1043, 1055, 1066, 1078, 1090,
+	1102, 1114, 1126, 1139, 1151, 1164, 1177, 1191,
+	1204, 1218, 1232, 1246, 1260, 1275, 1290, 1305,
+	1320, 1336, 1352, 1368, 1384, 1401, 1419, 1436, 
+	1454, 1472, 1491, 1510, 1530, 1550, 1570, 1591, 
+	1612, 1634, 1657, 1680, 1704, 1728, 1754, 1779, 
+	1806, 1834, 1862, 1892, 1922, 1954, 1987, 2021, 
+	2056
+};
+
+enum pad_types {
+	IMAGE_PAD,
+	METADATA_PAD,
+	NUM_PADS
+};
+
+enum binning_mode {
+	BINNING_NONE,
+	BINNING_NORMAL_2x2,
+	BINNING_SPECIAL_2x2,
+};
+
+enum binning_bit_depths {
+	BINNING_IDX_8_BIT,
+	BINNING_IDX_10_BIT,
+	BINNING_IDX_MAX
+};
+
+struct imx219_reg {
+	u16 address;
+	u8 val;
+};
+
 struct imx219_reg_list {
 	unsigned int num_of_regs;
 	const struct cci_reg_sequence *regs;
@@ -180,8 +249,8 @@ struct imx219_mode {
 	/* Default register values */
 	struct imx219_reg_list reg_list;
 
-	/* 2x2 binning is used */
-	bool binning;
+	/* binning mode based on format code */
+	enum binning_mode binning[BINNING_IDX_MAX];
 };
 
 static const struct cci_reg_sequence imx219_common_regs[] = {
@@ -195,15 +264,6 @@ static const struct cci_reg_sequence imx219_common_regs[] = {
 	{ CCI_REG8(0x30eb), 0x05 },
 	{ CCI_REG8(0x30eb), 0x09 },
 
-	/* PLL Clock Table */
-	{ IMX219_REG_VTPXCK_DIV, 5 },
-	{ IMX219_REG_VTSYCK_DIV, 1 },
-	{ IMX219_REG_PREPLLCK_VT_DIV, 3 },	/* 0x03 = AUTO set */
-	{ IMX219_REG_PREPLLCK_OP_DIV, 3 },	/* 0x03 = AUTO set */
-	{ IMX219_REG_PLL_VT_MPY, 57 },
-	{ IMX219_REG_OPSYCK_DIV, 1 },
-	{ IMX219_REG_PLL_OP_MPY, 114 },
-
 	/* Undocumented registers */
 	{ CCI_REG8(0x455e), 0x00 },
 	{ CCI_REG8(0x471e), 0x4b },
@@ -287,6 +347,34 @@ static const struct cci_reg_sequence raw10_framefmt_regs[] = {
 	{ IMX219_REG_OPPXCK_DIV, 10 },
 };
 
+static const struct cci_reg_sequence imx219_2lane_regs[] = {
+	/* PLL Clock Table */
+	{ IMX219_REG_VTPXCK_DIV, 5 },
+	{ IMX219_REG_VTSYCK_DIV, 1 },
+	{ IMX219_REG_PREPLLCK_VT_DIV, 3 },	/* 0x03 = AUTO set */
+	{ IMX219_REG_PREPLLCK_OP_DIV, 3 },	/* 0x03 = AUTO set */
+	{ IMX219_REG_PLL_VT_MPY, 57 },
+	{ IMX219_REG_OPSYCK_DIV, 1 },
+	{ IMX219_REG_PLL_OP_MPY, 114 },
+
+	/* 2-Lane CSI Mode */
+	{ IMX219_REG_CSI_LANE_MODE, IMX219_CSI_2_LANE_MODE },
+};
+
+static const struct cci_reg_sequence imx219_4lane_regs[] = {
+	/* PLL Clock Table */
+	{ IMX219_REG_VTPXCK_DIV, 5 },
+	{ IMX219_REG_VTSYCK_DIV, 1 },
+	{ IMX219_REG_PREPLLCK_VT_DIV, 3 },	/* 0x03 = AUTO set */
+	{ IMX219_REG_PREPLLCK_OP_DIV, 3 },	/* 0x03 = AUTO set */
+	{ IMX219_REG_PLL_VT_MPY, 88 },
+	{ IMX219_REG_OPSYCK_DIV, 1 },
+	{ IMX219_REG_PLL_OP_MPY, 91 },
+
+	/* 4-Lane CSI Mode */
+	{ IMX219_REG_CSI_LANE_MODE, IMX219_CSI_4_LANE_MODE },
+};
+
 static const s64 imx219_link_freq_menu[] = {
 	IMX219_DEFAULT_LINK_FREQ,
 };
@@ -386,7 +474,10 @@ static const struct imx219_mode supported_modes[] = {
 			.num_of_regs = ARRAY_SIZE(mode_3280x2464_regs),
 			.regs = mode_3280x2464_regs,
 		},
-		.binning = false,
+		.binning = {
+			[BINNING_IDX_8_BIT] = BINNING_NONE,
+			[BINNING_IDX_10_BIT] = BINNING_NONE,
+		},
 	},
 	{
 		/* 1080P 30fps cropped */
@@ -403,7 +494,10 @@ static const struct imx219_mode supported_modes[] = {
 			.num_of_regs = ARRAY_SIZE(mode_1920_1080_regs),
 			.regs = mode_1920_1080_regs,
 		},
-		.binning = false,
+		.binning = {
+			[BINNING_IDX_8_BIT] = BINNING_NONE,
+			[BINNING_IDX_10_BIT] = BINNING_NONE,
+		},
 	},
 	{
 		/* 2x2 binned 30fps mode */
@@ -420,7 +514,10 @@ static const struct imx219_mode supported_modes[] = {
 			.num_of_regs = ARRAY_SIZE(mode_1640_1232_regs),
 			.regs = mode_1640_1232_regs,
 		},
-		.binning = true,
+		.binning = {
+			[BINNING_IDX_8_BIT] = BINNING_SPECIAL_2x2,
+			[BINNING_IDX_10_BIT] = BINNING_NORMAL_2x2,
+		},
 	},
 	{
 		/* 640x480 30fps mode */
@@ -437,19 +534,26 @@ static const struct imx219_mode supported_modes[] = {
 			.num_of_regs = ARRAY_SIZE(mode_640_480_regs),
 			.regs = mode_640_480_regs,
 		},
-		.binning = true,
+		.binning = {
+			[BINNING_IDX_8_BIT] = BINNING_SPECIAL_2x2,
+			[BINNING_IDX_10_BIT] = BINNING_SPECIAL_2x2,
+		},
 	},
 };
 
 struct imx219 {
 	struct v4l2_subdev sd;
-	struct media_pad pad;
+	struct media_pad pad[NUM_PADS];
 
 	struct regmap *regmap;
 	struct clk *xclk; /* system clock to IMX219 */
 	u32 xclk_freq;
 
 	struct gpio_desc *reset_gpio;
+	struct gpio_desc *mclk_gpio;
+	struct gpio_desc *power_en_gpio;
+	struct gpio_desc *power_level_gpio;
+
 	struct regulator_bulk_data supplies[IMX219_NUM_SUPPLIES];
 
 	struct v4l2_ctrl_handler ctrl_handler;
@@ -468,7 +572,6 @@ struct imx219 {
 	/* Streaming on/off */
 	bool streaming;
 
-	/* Two or Four lanes */
 	u8 lanes;
 };
 
@@ -495,12 +598,79 @@ static u32 imx219_get_format_code(struct imx219 *imx219, u32 code)
 	return imx219_mbus_formats[i];
 }
 
+static int imx219_resolve_binning(struct imx219 *imx219,
+				  const struct v4l2_mbus_framefmt *format,
+				  enum binning_mode *binning)
+{
+	u32 fmt;
+
+	if (format)
+		fmt = format->code;
+	else
+		fmt = MEDIA_BUS_FMT_SRGGB10_1X10;
+
+	switch (fmt) {
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+		*binning = imx219->mode->binning[BINNING_IDX_8_BIT];
+		return 0;
+
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+		*binning = imx219->mode->binning[BINNING_IDX_10_BIT];
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static int imx219_get_rate_factor(struct imx219 *imx219,
+				  const struct v4l2_mbus_framefmt *format)
+{
+	enum binning_mode binning = BINNING_NONE;
+	int ret;
+
+	ret = imx219_resolve_binning(imx219, format, &binning);
+	if (ret < 0)
+		return ret;
+
+	switch (binning) {
+	case BINNING_NONE:
+	case BINNING_NORMAL_2x2:
+		return 1;
+	case BINNING_SPECIAL_2x2:
+		return 2;
+	}
+	return -EINVAL;
+}
+
+static void get_again_in_table(int value, int *again)
+{
+	int i;
+	for (i = 0; i < IMX219_AGAIN_TABLE - 1; i++) {
+		if (value > imx219_again_table[i] && value <= imx219_again_table[i + 1]) {
+			if ((imx219_again_table[i + 1] - value) < (value - imx219_again_table[i])) {
+				*again = i + 1;
+			} else {
+				*again = i;
+			}
+		}
+	}
+}
+
 static int imx219_set_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct imx219 *imx219 =
 		container_of(ctrl->handler, struct imx219, ctrl_handler);
 	struct i2c_client *client = v4l2_get_subdevdata(&imx219->sd);
 	int ret = 0;
+	const struct v4l2_mbus_framefmt *format;
+	struct v4l2_subdev_state *state;
+	int rate_factor;
+	int again = 0;
 
 	if (ctrl->id == V4L2_CID_VBLANK) {
 		int exposure_max, exposure_def;
@@ -522,14 +692,23 @@ static int imx219_set_ctrl(struct v4l2_ctrl *ctrl)
 	if (pm_runtime_get_if_in_use(&client->dev) == 0)
 		return 0;
 
+	state = v4l2_subdev_get_locked_active_state(&imx219->sd);
+	format = v4l2_subdev_get_pad_format(&imx219->sd, state, 0);
+	rate_factor = imx219_get_rate_factor(imx219, format);
+
 	switch (ctrl->id) {
 	case V4L2_CID_ANALOGUE_GAIN:
 		cci_write(imx219->regmap, IMX219_REG_ANALOG_GAIN,
 			  ctrl->val, &ret);
 		break;
+	case V4L2_CID_NOTIFY_GAINS:
+		get_again_in_table(ctrl->val, &again);
+		cci_write(imx219->regmap, IMX219_REG_ANALOG_GAIN,
+			  again, &ret);
+		break;
 	case V4L2_CID_EXPOSURE:
 		cci_write(imx219->regmap, IMX219_REG_EXPOSURE,
-			  ctrl->val, &ret);
+			  ctrl->val / rate_factor, &ret);
 		break;
 	case V4L2_CID_DIGITAL_GAIN:
 		cci_write(imx219->regmap, IMX219_REG_DIGITAL_GAIN,
@@ -546,7 +725,11 @@ static int imx219_set_ctrl(struct v4l2_ctrl *ctrl)
 		break;
 	case V4L2_CID_VBLANK:
 		cci_write(imx219->regmap, IMX219_REG_VTS,
-			  imx219->mode->height + ctrl->val, &ret);
+			  (imx219->mode->height + ctrl->val) / rate_factor, &ret);
+		break;
+	case V4L2_CID_HBLANK:
+		cci_write(imx219->regmap, IMX219_REG_HTS,
+			  imx219->mode->width + ctrl->val, &ret);
 		break;
 	case V4L2_CID_TEST_PATTERN_RED:
 		cci_write(imx219->regmap, IMX219_REG_TESTP_RED,
@@ -614,6 +797,13 @@ static int imx219_init_cfg(struct v4l2_subdev *sd,
 	crop->width = IMX219_PIXEL_ARRAY_WIDTH;
 	crop->height = IMX219_PIXEL_ARRAY_HEIGHT;
 
+	/* Initialize try_fmt for the embedded metadata pad */
+	// format = v4l2_subdev_get_pad_format(sd, state, 1);
+	// format->code = MEDIA_BUS_FMT_SENSOR_DATA;
+	// format->width = IMX219_EMBEDDED_LINE_WIDTH;
+	// format->height = IMX219_NUM_EMBEDDED_LINES;
+	// format->field = V4L2_FIELD_NONE;
+
 	return 0;
 }
 
@@ -623,10 +813,20 @@ static int imx219_enum_mbus_code(struct v4l2_subdev *sd,
 {
 	struct imx219 *imx219 = to_imx219(sd);
 
-	if (code->index >= (ARRAY_SIZE(imx219_mbus_formats) / 4))
+	if (code->pad >= NUM_PADS)
 		return -EINVAL;
 
-	code->code = imx219_get_format_code(imx219, imx219_mbus_formats[code->index * 4]);
+	if (code->pad == IMAGE_PAD) {
+		if (code->index >= (ARRAY_SIZE(imx219_mbus_formats) / 4))
+			return -EINVAL;
+
+		code->code = imx219_get_format_code(imx219, imx219_mbus_formats[code->index * 4]);
+	} else {
+		if (code->index > 0)
+			return -EINVAL;
+
+		// code->code = MEDIA_BUS_FMT_SENSOR_DATA;
+	}
 
 	return 0;
 }
@@ -638,68 +838,110 @@ static int imx219_enum_frame_size(struct v4l2_subdev *sd,
 	struct imx219 *imx219 = to_imx219(sd);
 	u32 code;
 
-	if (fse->index >= ARRAY_SIZE(supported_modes))
+	if (fse->pad >= NUM_PADS)
 		return -EINVAL;
 
-	code = imx219_get_format_code(imx219, fse->code);
-	if (fse->code != code)
-		return -EINVAL;
+	if (fse->pad == IMAGE_PAD) {
+		if (fse->index >= ARRAY_SIZE(supported_modes))
+			return -EINVAL;
+
+		code = imx219_get_format_code(imx219, fse->code);
+		if (fse->code != code)
+			return -EINVAL;
 
-	fse->min_width = supported_modes[fse->index].width;
-	fse->max_width = fse->min_width;
-	fse->min_height = supported_modes[fse->index].height;
-	fse->max_height = fse->min_height;
+		fse->min_width = supported_modes[fse->index].width;
+		fse->max_width = fse->min_width;
+		fse->min_height = supported_modes[fse->index].height;
+		fse->max_height = fse->min_height;
+	}
+	// } else {
+	// 	if (fse->code != MEDIA_BUS_FMT_SENSOR_DATA || fse->index > 0)
+	// 		return -EINVAL;
+
+	// 	fse->min_width = IMX219_EMBEDDED_LINE_WIDTH;
+	// 	fse->max_width = fse->min_width;
+	// 	fse->min_height = IMX219_NUM_EMBEDDED_LINES;
+	// 	fse->max_height = fse->min_height;
+	// }
 
 	return 0;
 }
 
+static unsigned long imx219_get_pixel_rate(struct imx219 *imx219,
+					   const struct v4l2_mbus_framefmt *format)
+{
+	return ((imx219->lanes == 2) ? IMX219_PIXEL_RATE :
+		IMX219_PIXEL_RATE_4LANE) * imx219_get_rate_factor(imx219, format);
+}
+
 static int imx219_set_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
 	struct imx219 *imx219 = to_imx219(sd);
 	const struct imx219_mode *mode;
-	int exposure_max, exposure_def, hblank;
+	int exposure_max, exposure_def, hblank, pixel_rate;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *crop;
 
-	mode = v4l2_find_nearest_size(supported_modes,
-				      ARRAY_SIZE(supported_modes),
-				      width, height,
-				      fmt->format.width, fmt->format.height);
-
-	imx219_update_pad_format(imx219, mode, &fmt->format, fmt->format.code);
-
-	format = v4l2_subdev_get_pad_format(sd, sd_state, 0);
-	crop = v4l2_subdev_get_pad_crop(sd, sd_state, 0);
-
-	*format = fmt->format;
-	*crop = mode->crop;
+	if (fmt->pad >= NUM_PADS)
+		return -EINVAL;
 
-	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-		imx219->mode = mode;
-		/* Update limits and set FPS to default */
-		__v4l2_ctrl_modify_range(imx219->vblank, IMX219_VBLANK_MIN,
-					 IMX219_VTS_MAX - mode->height, 1,
-					 mode->vts_def - mode->height);
-		__v4l2_ctrl_s_ctrl(imx219->vblank,
-				   mode->vts_def - mode->height);
-		/* Update max exposure while meeting expected vblanking */
-		exposure_max = mode->vts_def - 4;
-		exposure_def = (exposure_max < IMX219_EXPOSURE_DEFAULT) ?
-			exposure_max : IMX219_EXPOSURE_DEFAULT;
-		__v4l2_ctrl_modify_range(imx219->exposure,
-					 imx219->exposure->minimum,
-					 exposure_max, imx219->exposure->step,
-					 exposure_def);
-		/*
-		 * Currently PPL is fixed to IMX219_PPL_DEFAULT, so hblank
-		 * depends on mode->width only, and is not changeble in any
-		 * way other than changing the mode.
-		 */
-		hblank = IMX219_PPL_DEFAULT - mode->width;
-		__v4l2_ctrl_modify_range(imx219->hblank, hblank, hblank, 1,
-					 hblank);
+	if (fmt->pad == IMAGE_PAD) {
+		mode = v4l2_find_nearest_size(supported_modes,
+					      ARRAY_SIZE(supported_modes),
+					      width, height,
+					      fmt->format.width, fmt->format.height);
+
+		imx219_update_pad_format(imx219, mode, &fmt->format, fmt->format.code);
+
+		format = v4l2_subdev_get_pad_format(sd, sd_state, 0);
+		crop = v4l2_subdev_get_pad_crop(sd, sd_state, 0);
+		
+		*format = fmt->format;
+		*crop = mode->crop;
+
+		if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+			u32 prev_hts = imx219->mode->width + imx219->hblank->val;
+
+			imx219->mode = mode;
+			/* Update limits and set FPS to default */
+			__v4l2_ctrl_modify_range(imx219->vblank, IMX219_VBLANK_MIN,
+						 IMX219_VTS_MAX - mode->height, 1,
+						 mode->vts_def - mode->height);
+			__v4l2_ctrl_s_ctrl(imx219->vblank,
+					   mode->vts_def - mode->height);
+			/* Update max exposure while meeting expected vblanking */
+			exposure_max = mode->vts_def - 4;
+			exposure_def = (exposure_max < IMX219_EXPOSURE_DEFAULT) ?
+				exposure_max : IMX219_EXPOSURE_DEFAULT;
+			__v4l2_ctrl_modify_range(imx219->exposure,
+						 imx219->exposure->minimum,
+						 exposure_max, imx219->exposure->step,
+						 exposure_def);
+			/*
+			 * Retain PPL setting from previous mode so that the
+			 * line time does not change on a mode change.
+			 * Limits have to be recomputed as the controls define
+			 * the blanking only, so PPL values need to have the
+			 * mode width subtracted.
+			 */
+			hblank = prev_hts - mode->width;
+			__v4l2_ctrl_modify_range(imx219->hblank,
+						 IMX219_PPL_MIN - mode->width,
+						 IMX219_PPL_MAX - mode->width,
+						 1, IMX219_PPL_MIN - mode->width);
+			__v4l2_ctrl_s_ctrl(imx219->hblank, hblank);
+
+			/* Scale the pixel rate based on the mode specific factor */
+			pixel_rate = imx219_get_pixel_rate(imx219, &fmt->format);
+			__v4l2_ctrl_modify_range(imx219->pixel_rate, pixel_rate,
+						 pixel_rate, 1, pixel_rate);
+		}
+	} else {
+		format = v4l2_subdev_get_pad_format(sd, sd_state, 1);
+		/* Don't allow the embedded data format to be changed */
+		fmt->format = *format;
 	}
 
 	return 0;
@@ -730,24 +972,20 @@ static int imx219_set_framefmt(struct imx219 *imx219,
 static int imx219_set_binning(struct imx219 *imx219,
 			      const struct v4l2_mbus_framefmt *format)
 {
-	if (!imx219->mode->binning)
+	enum binning_mode binning = BINNING_NONE;
+
+	imx219_resolve_binning(imx219, format, &binning);
+
+	switch (binning) {
+	case BINNING_NONE:
 		return cci_write(imx219->regmap, IMX219_REG_BINNING_MODE,
 				 IMX219_BINNING_NONE, NULL);
-
-	switch (format->code) {
-	case MEDIA_BUS_FMT_SRGGB8_1X8:
-	case MEDIA_BUS_FMT_SGRBG8_1X8:
-	case MEDIA_BUS_FMT_SGBRG8_1X8:
-	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case BINNING_NORMAL_2x2:
 		return cci_write(imx219->regmap, IMX219_REG_BINNING_MODE,
-				 IMX219_BINNING_2X2_ANALOG, NULL);
-
-	case MEDIA_BUS_FMT_SRGGB10_1X10:
-	case MEDIA_BUS_FMT_SGRBG10_1X10:
-	case MEDIA_BUS_FMT_SGBRG10_1X10:
-	case MEDIA_BUS_FMT_SBGGR10_1X10:
+				 IMX219_BINNING_2X2_NORMAL, NULL);
+	case BINNING_SPECIAL_2x2:
 		return cci_write(imx219->regmap, IMX219_REG_BINNING_MODE,
-				 IMX219_BINNING_2X2, NULL);
+				 IMX219_BINNING_2X2_SPECIAL, NULL);
 	}
 
 	return -EINVAL;
@@ -784,11 +1022,22 @@ static int imx219_get_selection(struct v4l2_subdev *sd,
 	return -EINVAL;
 }
 
+static int imx219_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
+				struct v4l2_mbus_config *cfg)
+{
+	struct imx219 *imx219 = to_imx219(sd);
+	cfg->type = V4L2_MBUS_CSI2_DPHY;
+	cfg->bus.mipi_csi2.num_data_lanes = imx219->lanes;
+	return 0;
+}
+
 static int imx219_configure_lanes(struct imx219 *imx219)
 {
-	return cci_write(imx219->regmap, IMX219_REG_CSI_LANE_MODE,
-			 imx219->lanes == 2 ? IMX219_CSI_2_LANE_MODE :
-			 IMX219_CSI_4_LANE_MODE, NULL);
+	/* Write the appropriate PLL settings for the number of MIPI lanes */
+	return cci_multi_reg_write(imx219->regmap,
+				  imx219->lanes == 2 ? imx219_2lane_regs : imx219_4lane_regs,
+				  imx219->lanes == 2 ? ARRAY_SIZE(imx219_2lane_regs) :
+				  ARRAY_SIZE(imx219_4lane_regs), NULL);
 };
 
 static int imx219_start_streaming(struct imx219 *imx219,
@@ -915,44 +1164,44 @@ static int imx219_set_stream(struct v4l2_subdev *sd, int enable)
 static int imx219_power_on(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct imx219 *imx219 = to_imx219(sd);
+	// struct imx219 *imx219 = to_imx219(sd);
 	int ret;
 
-	ret = regulator_bulk_enable(IMX219_NUM_SUPPLIES,
-				    imx219->supplies);
-	if (ret) {
-		dev_err(dev, "%s: failed to enable regulators\n",
-			__func__);
-		return ret;
-	}
-
-	ret = clk_prepare_enable(imx219->xclk);
-	if (ret) {
-		dev_err(dev, "%s: failed to enable clock\n",
-			__func__);
-		goto reg_off;
-	}
-
-	gpiod_set_value_cansleep(imx219->reset_gpio, 1);
+	// ret = regulator_bulk_enable(IMX219_NUM_SUPPLIES,
+	// 			    imx219->supplies);
+	// if (ret) {
+	// 	dev_err(dev, "%s: failed to enable regulators\n",
+	// 		__func__);
+	// 	return ret;
+	// }
+
+	// ret = clk_prepare_enable(imx219->xclk);
+	// if (ret) {
+	// 	dev_err(dev, "%s: failed to enable clock\n",
+	// 		__func__);
+	// 	goto reg_off;
+	// }
+
+	// gpiod_set_value_cansleep(imx219->reset_gpio, 1);
 	usleep_range(IMX219_XCLR_MIN_DELAY_US,
 		     IMX219_XCLR_MIN_DELAY_US + IMX219_XCLR_DELAY_RANGE_US);
 
 	return 0;
 
-reg_off:
-	regulator_bulk_disable(IMX219_NUM_SUPPLIES, imx219->supplies);
+// reg_off:
+// 	regulator_bulk_disable(IMX219_NUM_SUPPLIES, imx219->supplies);
 
 	return ret;
 }
 
 static int imx219_power_off(struct device *dev)
 {
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct imx219 *imx219 = to_imx219(sd);
+	// struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	// struct imx219 *imx219 = to_imx219(sd);
 
-	gpiod_set_value_cansleep(imx219->reset_gpio, 0);
-	regulator_bulk_disable(IMX219_NUM_SUPPLIES, imx219->supplies);
-	clk_disable_unprepare(imx219->xclk);
+	// gpiod_set_value_cansleep(imx219->reset_gpio, 0);
+	// regulator_bulk_disable(IMX219_NUM_SUPPLIES, imx219->supplies);
+	// clk_disable_unprepare(imx219->xclk);
 
 	return 0;
 }
@@ -992,7 +1241,7 @@ static int __maybe_unused imx219_resume(struct device *dev)
 	return ret;
 }
 
-static int imx219_get_regulators(struct imx219 *imx219)
+static int __maybe_unused imx219_get_regulators(struct imx219 *imx219)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&imx219->sd);
 	unsigned int i;
@@ -1044,6 +1293,7 @@ static const struct v4l2_subdev_pad_ops imx219_pad_ops = {
 	.set_fmt = imx219_set_pad_format,
 	.get_selection = imx219_get_selection,
 	.enum_frame_size = imx219_enum_frame_size,
+	.get_mbus_config = imx219_get_mbus_config,
 };
 
 static const struct v4l2_subdev_ops imx219_subdev_ops = {
@@ -1053,11 +1303,6 @@ static const struct v4l2_subdev_ops imx219_subdev_ops = {
 };
 
 
-static unsigned long imx219_get_pixel_rate(struct imx219 *imx219)
-{
-	return (imx219->lanes == 2) ? IMX219_PIXEL_RATE : IMX219_PIXEL_RATE_4LANE;
-}
-
 /* Initialize control handlers */
 static int imx219_init_controls(struct imx219 *imx219)
 {
@@ -1065,7 +1310,7 @@ static int imx219_init_controls(struct imx219 *imx219)
 	struct v4l2_ctrl_handler *ctrl_hdlr;
 	unsigned int height = imx219->mode->height;
 	struct v4l2_fwnode_device_properties props;
-	int exposure_max, exposure_def, hblank;
+	int exposure_max, exposure_def, hblank, pixel_rate;
 	int i, ret;
 
 	ctrl_hdlr = &imx219->ctrl_handler;
@@ -1073,12 +1318,16 @@ static int imx219_init_controls(struct imx219 *imx219)
 	if (ret)
 		return ret;
 
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx219_ctrl_ops, V4L2_CID_NOTIFY_GAINS,
+			imx219_again_table[0], imx219_again_table[IMX219_AGAIN_TABLE - 1],
+			IMX219_ANA_GAIN_STEP, imx219_again_table[0]);
+
 	/* By default, PIXEL_RATE is read only */
+	pixel_rate = imx219_get_pixel_rate(imx219, NULL);
 	imx219->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &imx219_ctrl_ops,
 					       V4L2_CID_PIXEL_RATE,
-					       imx219_get_pixel_rate(imx219),
-					       imx219_get_pixel_rate(imx219), 1,
-					       imx219_get_pixel_rate(imx219));
+					       pixel_rate, pixel_rate, 1,
+					       pixel_rate);
 
 	imx219->link_freq =
 		v4l2_ctrl_new_int_menu(ctrl_hdlr, &imx219_ctrl_ops,
@@ -1094,12 +1343,11 @@ static int imx219_init_controls(struct imx219 *imx219)
 					   V4L2_CID_VBLANK, IMX219_VBLANK_MIN,
 					   IMX219_VTS_MAX - height, 1,
 					   imx219->mode->vts_def - height);
-	hblank = IMX219_PPL_DEFAULT - imx219->mode->width;
+	hblank = IMX219_PPL_MIN - imx219->mode->width;
 	imx219->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx219_ctrl_ops,
-					   V4L2_CID_HBLANK, hblank, hblank,
+					   V4L2_CID_HBLANK, hblank,
+					   IMX219_PPL_MIN - imx219->mode->width,
 					   1, hblank);
-	if (imx219->hblank)
-		imx219->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 	exposure_max = imx219->mode->vts_def - 4;
 	exposure_def = (exposure_max < IMX219_EXPOSURE_DEFAULT) ?
 		exposure_max : IMX219_EXPOSURE_DEFAULT;
@@ -1185,6 +1433,7 @@ static int imx219_check_hwcfg(struct device *dev, struct imx219 *imx219)
 		.bus_type = V4L2_MBUS_CSI2_DPHY
 	};
 	int ret = -EINVAL;
+	bool link_frequency_valid = false;
 
 	endpoint = fwnode_graph_get_next_endpoint(dev_fwnode(dev), NULL);
 	if (!endpoint) {
@@ -1204,18 +1453,40 @@ static int imx219_check_hwcfg(struct device *dev, struct imx219 *imx219)
 		goto error_out;
 	}
 	imx219->lanes = ep_cfg.bus.mipi_csi2.num_data_lanes;
-
+	dev_info(dev, "MIPI CSI2 data lanes: %d\n", imx219->lanes);
 	/* Check the link frequency set in device tree */
 	if (!ep_cfg.nr_of_link_frequencies) {
 		dev_err(dev, "link-frequency property not found in DT\n");
 		goto error_out;
 	}
 
-	if (ep_cfg.nr_of_link_frequencies != 1 ||
-	   (ep_cfg.link_frequencies[0] != ((imx219->lanes == 2) ?
-	    IMX219_DEFAULT_LINK_FREQ : IMX219_DEFAULT_LINK_FREQ_4LANE))) {
-		dev_err(dev, "Link frequency not supported: %lld\n",
-			ep_cfg.link_frequencies[0]);
+	if (ep_cfg.nr_of_link_frequencies == 1) {
+		switch (imx219->lanes) {
+		case 2:
+			if (ep_cfg.link_frequencies[0] ==
+						IMX219_DEFAULT_LINK_FREQ)
+				link_frequency_valid = true;
+			break;
+		case 4:
+			if (ep_cfg.link_frequencies[0] ==
+						IMX219_DEFAULT_LINK_FREQ_4LANE)
+				link_frequency_valid = true;
+			else if (ep_cfg.link_frequencies[0] ==
+				   IMX219_DEFAULT_LINK_FREQ_4LANE_UNSUPPORTED) {
+				dev_warn(dev, "Link frequency of %d not supported, but has been incorrectly advertised previously\n",
+					 IMX219_DEFAULT_LINK_FREQ_4LANE_UNSUPPORTED);
+				dev_warn(dev, "Using link frequency of %d\n",
+					 IMX219_DEFAULT_LINK_FREQ_4LANE);
+				link_frequency_valid = true;
+			}
+			break;
+		}
+	}
+
+	if (!link_frequency_valid) {
+		dev_err_probe(dev, -EINVAL,
+			      "Link frequency not supported: %lld\n",
+			      ep_cfg.link_frequencies[0]);
 		goto error_out;
 	}
 
@@ -1232,6 +1503,7 @@ static int imx219_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct imx219 *imx219;
+	const char *board_type;
 	int ret;
 
 	imx219 = devm_kzalloc(&client->dev, sizeof(*imx219), GFP_KERNEL);
@@ -1251,34 +1523,58 @@ static int imx219_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	/* Get system clock (xclk) */
-	imx219->xclk = devm_clk_get(dev, NULL);
-	if (IS_ERR(imx219->xclk)) {
-		dev_err(dev, "failed to get xclk\n");
-		return PTR_ERR(imx219->xclk);
+	// /* Get system clock (xclk) */
+	// imx219->xclk = devm_clk_get(dev, NULL);
+	// if (IS_ERR(imx219->xclk)) {
+	// 	dev_err(dev, "failed to get xclk\n");
+	// 	return PTR_ERR(imx219->xclk);
+	// }
+
+	imx219->mclk_gpio = devm_gpiod_get_optional(dev, "mclk-gpios",
+	 					     GPIOD_OUT_HIGH);
+	if (IS_ERR(imx219->mclk_gpio)) {
+		dev_dbg(dev, "failed to get mclk-gpios\n");
+	} else {
+		gpiod_set_value(imx219->mclk_gpio, 1);
 	}
 
-	imx219->xclk_freq = clk_get_rate(imx219->xclk);
-	if (imx219->xclk_freq != IMX219_XCLK_FREQ) {
-		dev_err(dev, "xclk frequency not supported: %d Hz\n",
-			imx219->xclk_freq);
-		return -EINVAL;
+	/* For the D560 board, raise the GPIO21 operation */
+	ret = of_property_read_string(dev->of_node, "eswin,board-type", &board_type);
+	if (ret || strcmp(board_type, "d560") != 0) {
+		dev_info(dev, "Non-d560 board, skip GPIO operations\n");
+		board_type = NULL;
 	}
 
-	ret = imx219_get_regulators(imx219);
-	if (ret) {
-		dev_err(dev, "failed to get regulators\n");
-		return ret;
+	if (board_type) {
+		imx219->power_en_gpio = devm_gpiod_get_optional(dev, "power-en", GPIOD_OUT_LOW);
+		imx219->power_level_gpio = devm_gpiod_get_optional(dev, "power-level", GPIOD_OUT_LOW);
+		imx219->mclk_gpio = devm_gpiod_get_optional(dev, "mclk-en", GPIOD_OUT_LOW);
+
+		if (IS_ERR(imx219->power_en_gpio)) {
+			ret = PTR_ERR(imx219->power_en_gpio);
+			dev_err(dev, "get power-en GPIO failed\n");
+			return ret;
+		}
+		if (IS_ERR(imx219->power_level_gpio)) {
+			ret = PTR_ERR(imx219->power_level_gpio);
+			dev_err(dev, "get power-level GPIO failed\n");
+			return ret;
+		}
+		if (IS_ERR(imx219->mclk_gpio)) {
+			ret = PTR_ERR(imx219->mclk_gpio);
+			dev_err(dev, "get mclk-en GPIO failed\n");
+			return ret;
+		}
+
+		gpiod_set_value_cansleep(imx219->power_en_gpio, 1);
+		gpiod_set_value_cansleep(imx219->power_level_gpio, 1);
+		gpiod_set_value_cansleep(imx219->mclk_gpio, 1);
 	}
 
-	/* Request optional enable pin */
-	imx219->reset_gpio = devm_gpiod_get_optional(dev, "reset",
-						     GPIOD_OUT_HIGH);
-
 	/*
-	 * The sensor must be powered for imx219_identify_module()
-	 * to be able to read the CHIP_ID register
-	 */
+	* The sensor must be powered for imx219_identify_module()
+	* to be able to read the CHIP_ID register
+	*/
 	ret = imx219_power_on(dev);
 	if (ret)
 		return ret;
@@ -1319,9 +1615,10 @@ static int imx219_probe(struct i2c_client *client)
 	imx219->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
 
 	/* Initialize source pad */
-	imx219->pad.flags = MEDIA_PAD_FL_SOURCE;
+	imx219->pad[IMAGE_PAD].flags = MEDIA_PAD_FL_SOURCE;
+	imx219->pad[METADATA_PAD].flags = MEDIA_PAD_FL_SOURCE;
 
-	ret = media_entity_pads_init(&imx219->sd.entity, 1, &imx219->pad);
+	ret = media_entity_pads_init(&imx219->sd.entity, NUM_PADS, imx219->pad);
 	if (ret) {
 		dev_err(dev, "failed to init entity pads: %d\n", ret);
 		goto error_handler_free;
@@ -1334,25 +1631,24 @@ static int imx219_probe(struct i2c_client *client)
 		goto error_media_entity;
 	}
 
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
-
 	ret = v4l2_async_register_subdev_sensor(&imx219->sd);
 	if (ret < 0) {
 		dev_err(dev, "failed to register sensor sub-device: %d\n", ret);
 		goto error_subdev_cleanup;
 	}
 
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
 	pm_runtime_idle(dev);
 
 	return 0;
 
 error_subdev_cleanup:
 	v4l2_subdev_cleanup(&imx219->sd);
-	pm_runtime_disable(dev);
-	pm_runtime_set_suspended(dev);
 
 error_media_entity:
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
 	media_entity_cleanup(&imx219->sd.entity);
 
 error_handler_free:
diff --git a/drivers/media/i2c/imx327.c b/drivers/media/i2c/imx327.c
new file mode 100644
index 000000000000..ed1950868568
--- /dev/null
+++ b/drivers/media/i2c/imx327.c
@@ -0,0 +1,2255 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/sysfs.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+
+#include <linux/of_graph.h>
+#include <media/media-entity.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-subdev.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-event.h>
+#include <linux/pinctrl/consumer.h>
+
+#define CONFIG_VIDEO_V4L2_SUBDEV_API
+#define CONFIG_MEDIA_CONTROLLER
+
+#define DRIVER_VERSION			KERNEL_VERSION(0, 0x01, 0x06)
+#ifndef V4L2_CID_DIGITAL_GAIN
+#define V4L2_CID_DIGITAL_GAIN		V4L2_CID_GAIN
+#endif
+
+#define IMX327_LINK_FREQ_111M		111370000
+#define IMX327_LINK_FREQ_222M		222750000
+#define IMX327_2LANES			2
+#define IMX327_4LANES			4
+#define IMX327_BITS_PER_SAMPLE		10
+
+/* pixel rate = link frequency * 2 * lanes / BITS_PER_SAMPLE */
+#define IMX327_PIXEL_RATE_NORMAL (IMX327_LINK_FREQ_111M * 2 / 10 * IMX327_4LANES)
+#define IMX327_PIXEL_RATE_HDR (IMX327_LINK_FREQ_222M * 2 / 10 * IMX327_4LANES)
+
+#define IMX327_XVCLK_FREQ		37125000
+
+#define CHIP_ID				0xb2
+#define IMX327_REG_CHIP_ID		0x301e
+
+#define IMX327_REG_CTRL_MODE		0x3000
+#define IMX327_MODE_SW_STANDBY		0x1
+#define IMX327_MODE_STREAMING		0x0
+
+#define IMX327_REG_SHS1_H		0x3022
+#define IMX327_REG_SHS1_M		0x3021
+#define IMX327_REG_SHS1_L		0x3020
+
+#define IMX327_REG_SHS2_H		0x3026
+#define IMX327_REG_SHS2_M		0x3025
+#define IMX327_REG_SHS2_L		0x3024
+
+#define IMX327_REG_RHS1_H		0x3032
+#define IMX327_REG_RHS1_M		0x3031
+#define IMX327_REG_RHS1_L		0x3030
+
+#define IMX327_FETCH_HIGH_BYTE_EXP(VAL)	(((VAL) >> 16) & 0x0F)
+#define IMX327_FETCH_MID_BYTE_EXP(VAL)	(((VAL) >> 8) & 0xFF)
+#define IMX327_FETCH_LOW_BYTE_EXP(VAL)	((VAL) & 0xFF)
+
+#define	IMX327_EXPOSURE_MIN		2
+#define	IMX327_EXPOSURE_STEP		1
+#define IMX327_VTS_MAX			0x7fff
+
+#define IMX327_GAIN_SWITCH_REG		0x3009
+#define IMX327_REG_LF_GAIN		0x3014
+#define IMX327_REG_SF_GAIN		0x30f2
+#define IMX327_GAIN_MIN			0x00
+#define IMX327_GAIN_MAX			0xee
+#define IMX327_GAIN_STEP		1
+#define IMX327_GAIN_DEFAULT		0x00
+
+#define IMX327_GROUP_HOLD_REG		0x3001
+#define IMX327_GROUP_HOLD_START		0x01
+#define IMX327_GROUP_HOLD_END		0x00
+
+#define USED_TEST_PATTERN
+#ifdef USED_TEST_PATTERN
+#define IMX327_REG_TEST_PATTERN		0x308c
+#define	IMX327_TEST_PATTERN_ENABLE	BIT(0)
+#endif
+
+#define IMX327_REG_VTS_H		0x301a
+#define IMX327_REG_VTS_M		0x3019
+#define IMX327_REG_VTS_L		0x3018
+#define IMX327_FETCH_HIGH_BYTE_VTS(VAL)	(((VAL) >> 16) & 0x03)
+#define IMX327_FETCH_MID_BYTE_VTS(VAL)	(((VAL) >> 8) & 0xFF)
+#define IMX327_FETCH_LOW_BYTE_VTS(VAL)	((VAL) & 0xFF)
+
+#define REG_NULL			0xFFFF
+#define REG_DELAY			0xFFFE
+
+#define IMX327_REG_VALUE_08BIT		1
+#define IMX327_REG_VALUE_16BIT		2
+#define IMX327_REG_VALUE_24BIT		3
+
+static bool g_isHCG;
+
+#define IMX327_NAME			"imx327"
+
+#define OF_CAMERA_PINCTRL_STATE_DEFAULT	"rockchip,camera_default"
+#define OF_CAMERA_PINCTRL_STATE_SLEEP	"rockchip,camera_sleep"
+
+#define IMX327_FLIP_REG			0x3007
+#define MIRROR_BIT_MASK			BIT(1)
+#define FLIP_BIT_MASK			BIT(0)
+
+static const char * const imx327_supply_names[] = {
+	"avdd",		/* Analog power */
+	"dovdd",	/* Digital I/O power */
+	"dvdd",		/* Digital core power */
+};
+
+#define IMX327_NUM_SUPPLIES ARRAY_SIZE(imx327_supply_names)
+
+enum hdr_mode {
+	NO_HDR = 0,
+	HDR_X2 = 5,
+	HDR_X3 = 6,
+	HDR_COMPR,
+};
+
+/*
+ * enum cg_mode_e conversion gain
+ *
+ */
+enum cg_mode_e {
+	GAIN_MODE_LCG,
+	GAIN_MODE_HCG,
+};
+
+struct regval {
+	u16 addr;
+	u8 val;
+};
+
+struct imx327_mode {
+	u32 bus_fmt;
+	u32 width;
+	u32 height;
+	struct v4l2_fract max_fps;
+	struct v4l2_rect crop;
+	u32 hts_def;
+	u32 vts_def;
+	u32 exp_def;
+	const struct regval *reg_list;
+	u32 hdr_mode;
+};
+
+struct imx327 {
+	struct i2c_client	*client;
+	struct clk		*xvclk;
+	struct gpio_desc	*reset_gpio;
+	struct gpio_desc	*pwdn_gpio;
+	struct regulator_bulk_data supplies[IMX327_NUM_SUPPLIES];
+
+	struct pinctrl		*pinctrl;
+	struct pinctrl_state	*pins_default;
+	struct pinctrl_state	*pins_sleep;
+
+	struct v4l2_subdev	subdev;
+	struct media_pad	pad;
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct v4l2_ctrl	*exposure;
+	struct v4l2_ctrl	*anal_gain;
+	struct v4l2_ctrl	*digi_gain;
+	struct v4l2_ctrl	*hblank;
+	struct v4l2_ctrl	*vblank;
+	struct v4l2_ctrl	*pixel_rate;
+	struct v4l2_ctrl	*link_freq;
+	struct v4l2_ctrl	*h_flip;
+	struct v4l2_ctrl	*v_flip;
+#ifdef USED_TEST_PATTERN
+	struct v4l2_ctrl	*test_pattern;
+#endif
+	struct mutex		mutex;
+	bool			streaming;
+	bool			power_on;
+	const struct imx327_mode *support_modes;
+	u32			support_modes_num;
+	const struct imx327_mode *cur_mode;
+	u32			cur_vts;
+	bool			has_init_exp;
+	struct v4l2_fwnode_endpoint bus_cfg;
+	u8			flip;
+};
+
+#define to_imx327(sd) container_of(sd, struct imx327, subdev)
+
+/*
+ * Xclk 37.125Mhz
+ */
+static const struct regval imx327_global_regs[] = {
+	{REG_NULL, 0x00},
+};
+
+/*
+ * Xclk 37.125Mhz
+ * max_framerate 30fps
+ * lvds_datarate per lane 222.75Mbps 4 lane
+ */
+static const struct regval imx327_linear_1920x1080_lvds_regs[] = {
+	{0x3003, 0x01},
+	{REG_DELAY, 0x10},
+	{0x3000, 0x01},
+	{0x3001, 0x00},
+	{0x3002, 0x01},
+	{0x3005, 0x00},
+	{0x3007, 0x00},
+	{0x3009, 0x02},
+	{0x300a, 0x3c},
+	{0x3010, 0x21},
+	{0x3011, 0x0a},
+	{0x3018, 0x46},
+	{0x3019, 0x05},
+	{0x301c, 0x30},
+	{0x301d, 0x11},
+	{0x3046, 0xe0},
+	{0x304b, 0x0a},
+	{0x305c, 0x18},
+	{0x305d, 0x00},
+	{0x305e, 0x20},
+	{0x305f, 0x01},
+	{0x309e, 0x4a},
+	{0x309f, 0x4a},
+	{0x311c, 0x0e},
+	{0x3128, 0x04},
+	{0x3129, 0x1d},
+	{0x313b, 0x41},
+	{0x315e, 0x1a},
+	{0x3164, 0x1a},
+	{0x317c, 0x12},
+	{0x31ec, 0x37},
+	{0x3480, 0x49},
+	{0x3002, 0x00},
+	{REG_NULL, 0x00},
+};
+
+/*
+ * Xclk 37.125Mhz
+ * max_framerate 30fps
+ * lvds_datarate per lane 445.5Mbps 4 lane
+ */
+static const struct regval imx327_hdr2_1920x1080_lvds_regs[] = {
+	{0x3003, 0x01},
+	{REG_DELAY, 0x10},
+	{0x3000, 0x01},
+	{0x3001, 0x00},
+	{0x3002, 0x01},
+	{0x3005, 0x00},
+	{0x3007, 0x40},
+	{0x3009, 0x01},
+	{0x300a, 0x3c},
+	{0x300c, 0x11},
+	{0x3011, 0x02},
+	{0x3018, 0xb8},/* VMAX L */
+	{0x3019, 0x05},/* VMAX M */
+	{0x301c, 0xec},/* HMAX L */
+	{0x301d, 0x07},/* HMAX H */
+	{0x3020, 0x02},//hdr+ shs1 l  short
+	{0x3021, 0x00},//hdr+ shs1 m
+	{0x3024, 0xc9},//hdr+ shs2 l
+	{0x3025, 0x07},//hdr+ shs2 m
+	{0x3030, 0xe1},//hdr+ IMX327_RHS1
+	{0x3031, 0x00},//hdr+IMX327_RHS1
+	{0x3045, 0x03},//hdr+
+	{0x3046, 0xe0},
+	{0x304b, 0x0a},
+	{0x305c, 0x18},
+	{0x305d, 0x03},
+	{0x305e, 0x20},
+	{0x305f, 0x01},
+	{0x309e, 0x4a},
+	{0x309f, 0x4a},
+	{0x30d2, 0x19},
+	{0x30d7, 0x03},
+	{0x3106, 0x11},
+	{0x3129, 0x1d},
+	{0x313b, 0x61},
+	{0x315e, 0x1a},
+	{0x3164, 0x1a},
+	{0x317c, 0x12},
+	{0x31ec, 0x37},
+	{0x3414, 0x00},
+	{0x3415, 0x00},
+	{0x3480, 0x49},
+	{0x31a0, 0xb4},
+	{0x31a1, 0x02},
+	{0x303c, 0x04},//Y offset
+	{0x303d, 0x00},
+	{0x303e, 0x41},
+	{0x303f, 0x04},//height
+	{0x303A, 0x08},//hdr+
+	{0x3010, 0x61},//hdr+ gain 1frame FPGC
+	{0x3014, 0x00},//hdr+ gain 1frame long
+	{0x30F0, 0x64},//hdr+ gain 2frame FPGC
+	{0x30f2, 0x00},//hdr+ gain 2frame short
+	{0x3002, 0x00},
+	{REG_NULL, 0x00},
+};
+
+/*
+ * Xclk 37.125Mhz
+ * max_framerate 30fps
+ * mipi_datarate per lane 222.75Mbps 4 lane
+ */
+static const struct regval imx327_linear_1920x1080_mipi_regs[] = {
+	{0x3003, 0x01},
+	{REG_DELAY, 0x10},
+	{0x3000, 0x01},
+	{0x3001, 0x00},
+	{0x3002, 0x01},
+	{0x3005, 0x00},
+	{0x3007, 0x00},
+	{0x3009, 0x02},
+	{0x300A, 0x3c},
+	{0x3010, 0x21},
+	{0x3011, 0x0a},
+	{0x3018, 0x46},
+	{0x3019, 0x05},
+	{0x301C, 0x30},
+	{0x301D, 0x11},
+	{0x3046, 0x00},
+	{0x304B, 0x0A},
+	{0x305C, 0x18},
+	{0x305D, 0x03},
+	{0x305E, 0x20},
+	{0x305F, 0x01},
+	{0x309E, 0x4A},
+	{0x309F, 0x4A},
+	{0x311c, 0x0e},
+	{0x3128, 0x04},
+	{0x3129, 0x1d},
+	{0x313B, 0x41},
+	{0x315E, 0x1A},
+	{0x3164, 0x1A},
+	{0x317C, 0x12},
+	{0x31EC, 0x37},
+	{0x3405, 0x20},
+	{0x3407, 0x03},
+	{0x3414, 0x0A},
+	{0x3418, 0x49},
+	{0x3419, 0x04},
+	{0x3441, 0x0a},
+	{0x3442, 0x0a},
+	{0x3443, 0x03},
+	{0x3444, 0x20},
+	{0x3445, 0x25},
+	{0x3446, 0x47},
+	{0x3447, 0x00},
+	{0x3448, 0x1f},
+	{0x3449, 0x00},
+	{0x344A, 0x17},
+	{0x344B, 0x00},
+	{0x344C, 0x0F},
+	{0x344D, 0x00},
+	{0x344E, 0x17},
+	{0x344F, 0x00},
+	{0x3450, 0x47},
+	{0x3451, 0x00},
+	{0x3452, 0x0F},
+	{0x3453, 0x00},
+	{0x3454, 0x0f},
+	{0x3455, 0x00},
+	{0x3472, 0x9c},
+	{0x3473, 0x07},
+	{0x3480, 0x49},
+	{0x3002, 0x00},
+	{REG_NULL, 0x00},
+};
+
+//2 lane regs config
+static const struct regval imx327_1280x720_regs[] = {
+	
+	/* standby */
+	{0x3000, 0x01},
+
+    {0x3407, 0x01}, /* IMX327_PHY_LANE_NUM */
+    {0x3443, 0x01}, /* IMX327_CSI_LANE_MODE */
+    #ifdef IMX327_BASIC_RATE_FPS
+    {0x3009, 0x02}, /* IMX327_FR_FDG_SEL 30fps*/
+    #else
+    {0x3009, 0x01}, /* 60fps is not working */
+    #endif
+    // {REG_NULL, 0x00},
+	
+    // /* read back for checking */
+    // {0x3009, 0x01},
+    // {0x3443, 0x01},
+
+    {0x3020, 0x00}, /* SHS1 setting */
+    {0x3021, 0x05},
+    {0x3022, 0x00},  
+
+    /* imx327_global_init_settings */
+    {0x3007, 0x10},  /* 720p */
+    {0x300c, 0x00},
+    {0x3045, 0x00},
+    /* imx327_720p_settings */
+	{0x303a, 0x06}, /* WINWV_OB */
+    #ifdef IMX327_USER_PACKET_ENABLE
+    {0x3414, 0x04},
+    #else
+	{0x3414, 0x00}, /* OPB_SIZE_V */
+    #endif
+
+    {0x3444, 0x20}, /* EXTCK_FREQ 37.125MHz */
+    {0x3445, 0x25},
+
+    // corner
+    {0x3040, 0x04}, /* WINPH[10:0] 0 */
+    {0x3041, 0x00},
+    {0x303c, 0x0c}, /* WINPV[10:0] 0 */
+    {0x303d, 0x00},
+
+    {0x3042, 0x10}, /* WINWH[10:0] for 1280+16 */
+    {0x3043, 0x05},
+    {0x303e, 0xd5}, /* WINWV[10:0] for 720+5 */
+    {0x303f, 0x02},
+
+	{0x3472, 0x00}, /* X OUT SIZE (1280)*/
+	{0x3473, 0x05},
+	{0x3418, 0xd0}, /* Y OUT SIZE (720)*/
+	{0x3419, 0x02},
+
+#ifdef SENSOR_OUT_12BIT
+    /* imx327_12bit_settings */
+    {0x3005, 0x01},
+    {0x3046, 0x01},
+    {0x3129, 0x00},
+    {0x317c, 0x00},
+    {0x31ec, 0x00},
+    {0x3441, 0x0c},
+    {0x3442, 0x0c},
+#else
+    /* imx327_10bit_settings */
+    {0x3005, 0x00},
+    {0x3046, 0x00},
+    {0x3129, 0x1d},
+    {0x317c, 0x12},
+    {0x31ec, 0x37},
+    {0x3441, 0x0a},
+    {0x3442, 0x0a},
+#endif
+    {0x300a, 0x3c},
+    {0x300b, 0x00},
+
+	{0x3012, 0x64},
+	{0x3013, 0x00},
+	{0x305c, 0x18},
+	{0x305d, 0x03},
+	{0x305e, 0x20},
+	{0x305f, 0x01},
+	{0x315e, 0x1a},
+	{0x3164, 0x1a},
+	{0x3480, 0x49},
+
+    #ifdef IMX327_BASIC_RATE_FPS
+	/* data rate settings */
+	{ 0x3405, 0x10 },
+
+	{ 0x3446, 0x4f },
+	{ 0x3447, 0x00 },
+
+	{ 0x3448, 0x2f },
+	{ 0x3449, 0x00 },
+
+	{ 0x344a, 0x17 },
+	{ 0x344b, 0x00 },
+
+	{ 0x344c, 0x17 },
+	{ 0x344d, 0x00 },
+
+	{ 0x344e, 0x17 },
+	{ 0x344f, 0x00 },
+
+	{ 0x3450, 0x57 },
+	{ 0x3451, 0x00 },
+
+	{ 0x3452, 0x17 },
+	{ 0x3453, 0x00 },
+
+	{ 0x3454, 0x17 },
+	{ 0x3455, 0x00 },
+
+    // {0x3018, 0xee}, /* VMAX[17:0] */
+    // {0x3019, 0x02},
+    // {0x301a, 0x00},
+    // {0x301c, 0xc8}, /* HMAX[15:0] */
+    // {0x301d, 0x19},
+    /* VMAX&HMAX*2 for 7.5 fps */
+    {0x3018, 0xdc}, /* VMAX[17:0] */
+    {0x3019, 0x05},
+    {0x301a, 0x00},
+    {0x301c, 0x90}, /* HMAX[15:0] */
+    {0x301d, 0x33},
+    #else
+	/* data rate settings */
+	{ 0x3405, 0x00 },
+
+	{ 0x3446, 0x67 },
+	{ 0x3447, 0x00 },
+
+	{ 0x3448, 0x57 },
+	{ 0x3449, 0x00 },
+
+	{ 0x344a, 0x2f },
+	{ 0x344b, 0x00 },
+
+	{ 0x344c, 0x27 },
+	{ 0x344d, 0x00 },
+
+	{ 0x344e, 0x2f },
+	{ 0x344f, 0x00 },
+
+	{ 0x3450, 0xbf },
+	{ 0x3451, 0x00 },
+
+	{ 0x3452, 0x2f },
+	{ 0x3453, 0x00 },
+
+	{ 0x3454, 0x27 },
+	{ 0x3455, 0x00 },
+
+    {0x3018, 0xee}, /* VMAX[17:0] */
+    {0x3019, 0x02},
+    {0x301a, 0x00},
+    {0x301c, 0xe4}, /* HMAX[15:0] */
+    {0x301d, 0x0c},
+    #endif
+
+    #ifdef IMX327_TEST_PATERN_ENABLE
+    // /* pattern */
+    {0x308c, 0x33, 0, 10000},
+    #else
+    {0x308c, 0x00, 0, 10000},
+    #endif
+    {0x300a, 0x3c},
+    {0x300b, 0x00},
+	// {REG_NULL, 0x00},
+    /* standby */
+    {0x3000, 0},
+
+    /* start */
+    {0x3002, 0},
+	{REG_NULL, 0x00}
+};
+
+static struct regval imx327_1920x1080_2lanes_regs[] = {
+
+    /* standby */
+    {0x3000, 0x01},
+
+    {0x3407, 0x01}, /* IMX327_PHY_LANE_NUM */
+    {0x3443, 0x01}, /* IMX327_CSI_LANE_MODE */
+    {0x3009, 0x02}, /* IMX327_FR_FDG_SEL 30fps*/
+    
+    /* read back for checking */
+    {0x3009, 0x01, 1, 0},
+    {0x3443, 0x01, 1, 0},
+
+    {0x3018, 0x65}, /* VMAX[17:0] */
+    {0x3019, 0x04},
+    {0x301a, 0x00},
+    {0x301c, 0x60}, /* HMAX[15:0] */
+    {0x301d, 0x22},
+
+    {0x3020, 0x40}, /* SHS1 setting */
+    {0x3021, 0x04},
+    {0x3022, 0x00},  
+
+    /* imx327_global_init_settings */
+    {0x3007, 0x00},  /* 1080p */
+    {0x300c, 0x00},
+    {0x3045, 0x00},
+
+	{0x303a, 0x0c}, /* WINWV_OB */
+    #ifdef IMX327_USER_PACKET_ENABLE
+    {0x3414, 0x0a},
+    #else
+	{0x3414, 0x00}, /* OPB_SIZE_V */
+    #endif
+
+    {0x3444, 0x20}, /* EXTCK_FREQ 37.125MHz */
+    {0x3445, 0x25},
+
+    // corner
+    {0x3040, 0x0c}, /* WINPH[10:0] 0 */
+    {0x3041, 0x00},
+    {0x303c, 0x08}, /* WINPV[10:0] 0 */
+    {0x303d, 0x00},
+
+    {0x3042, 0x90}, /* WINWH[10:0] 1920+16 */
+    {0x3043, 0x07},
+    {0x303e, 0x41}, /* WINWV[10:0] 1080+9 */
+    {0x303f, 0x04},
+
+	{0x3472, 0x80}, /* X OUT SIZE (1920)*/
+	{0x3473, 0x07},
+	{0x3418, 0x38}, /* Y OUT SIZE (1080)*/
+	{0x3419, 0x04},
+
+#ifdef SENSOR_OUT_12BIT
+    /* imx327_12bit_settings */
+    {0x3005, 0x01},
+    {0x3046, 0x01},
+    {0x3129, 0x00},
+    {0x317c, 0x00},
+    {0x31ec, 0x00},
+    {0x3441, 0x0c},
+    {0x3442, 0x0c},
+#else
+    /* imx327_10bit_settings */
+    {0x3005, 0x00},
+    {0x3046, 0x00},
+    {0x3129, 0x1d},
+    {0x317c, 0x12},
+    {0x31ec, 0x37},
+    {0x3441, 0x0a},
+    {0x3442, 0x0a},
+#endif
+
+	{0x3012, 0x64},
+	{0x3013, 0x00},
+	{0x305c, 0x18},
+	{0x305d, 0x03},
+	{0x305e, 0x20},
+	{0x305f, 0x01},
+	{0x315e, 0x1a},
+	{0x3164, 0x1a},
+	{0x3480, 0x49},
+
+	/* data rate settings */
+	{ 0x3405, 0x10 },
+
+	{ 0x3446, 0x57 },
+	{ 0x3447, 0x00 },
+
+	{ 0x3448, 0x37 },
+	{ 0x3449, 0x00 },
+
+	{ 0x344a, 0x1f },
+	{ 0x344b, 0x00 },
+
+	{ 0x344c, 0x1f },
+	{ 0x344d, 0x00 },
+
+	{ 0x344e, 0x1f },
+	{ 0x344f, 0x00 },
+
+	{ 0x3450, 0x77 },
+	{ 0x3451, 0x00 },
+
+	{ 0x3452, 0x1f },
+	{ 0x3453, 0x00 },
+
+	{ 0x3454, 0x17 },
+	{ 0x3455, 0x00 },
+
+
+    #ifdef IMX327_TEST_PATERN_ENABLE
+    /* pattern */
+    {0x308c, 0x33},
+    #else
+    {0x308c, 0x00},
+    #endif
+    {0x300a, 0x3c},
+    {0x300b, 0x00},
+
+    /* standby */
+    {0x3000, 0},
+
+    /* start */
+    {0x3002, 0}
+};
+
+
+/*
+ * Xclk 37.125Mhz
+ * max_framerate 30fps
+ * mipi_datarate per lane 445.5Mbps 4 lane
+ */
+static const struct regval imx327_hdr2_1920x1080_mipi_regs[] = {
+	{0x3003, 0x01},
+	{REG_DELAY, 0x10},
+	{0x3000, 0x01},
+	{0x3001, 0x00},
+	{0x3002, 0x01},
+	{0x3005, 0x00},
+	{0x3007, 0x40},
+	{0x3009, 0x01},
+	{0x300a, 0x3c},
+	{0x300c, 0x11}, //hdr+
+	{0x3011, 0x02},
+	{0x3018, 0xb8},/* VMAX L */
+	{0x3019, 0x05},/* VMAX M */
+	{0x301a, 0x00},
+	{0x301c, 0xEc},/* HMAX L */
+	{0x301d, 0x07},/* HMAX H */
+	{0x3045, 0x05},//hdr+
+	{0x3046, 0x00},
+	{0x304b, 0x0a},
+	{0x305c, 0x18},
+	{0x305d, 0x03},
+	{0x305e, 0x20},
+	{0x305f, 0x01},
+	{0x309e, 0x4a},
+	{0x309f, 0x4a},
+	{0x30d2, 0x19},
+	{0x30d7, 0x03},
+	{0x3106, 0x11},//hdr+
+	{0x3129, 0x1d},
+	{0x313b, 0x61},
+	{0x315e, 0x1a},
+	{0x3164, 0x1a},
+	{0x317c, 0x12},
+	{0x31ec, 0x37},
+	{0x3405, 0x10},
+	{0x3407, 0x03},
+	{0x3414, 0x00},
+	{0x3415, 0x00},//hdr+
+	{0x3418, 0x72},
+	{0x3419, 0x09},
+	{0x3441, 0x0a},
+	{0x3442, 0x0a},
+	{0x3443, 0x03},
+	{0x3444, 0x20},
+	{0x3445, 0x25},
+	{0x3446, 0x57},
+	{0x3447, 0x00},
+	{0x3448, 0x37},//37?
+	{0x3449, 0x00},
+	{0x344a, 0x1f},
+	{0x344b, 0x00},
+	{0x344c, 0x1f},
+	{0x344d, 0x00},
+	{0x344e, 0x1f},
+	{0x344f, 0x00},
+	{0x3450, 0x77},
+	{0x3451, 0x00},
+	{0x3452, 0x1f},
+	{0x3453, 0x00},
+	{0x3454, 0x17},
+	{0x3455, 0x00},
+	{0x3472, 0xa0},
+	{0x3473, 0x07},
+	{0x347b, 0x23},
+	{0x3480, 0x49},
+	{0x31a0, 0xb4},//hdr+
+	{0x31a1, 0x02},//hdr+
+	{0x3020, 0x02},//hdr+ shs1 l  short
+	{0x3021, 0x00},//hdr+ shs1 m
+	{0x3022, 0x00},//hdr+ shs1 h
+	{0x3030, 0xe1},//hdr+ IMX327_RHS1
+	{0x3031, 0x00},//hdr+IMX327_RHS1
+	{0x3032, 0x00},//hdr+
+	{0x31A0, 0xe8},//hdr+ HBLANK1
+	{0x31A1, 0x01},//hdr+
+	{0x303c, 0x04},
+	{0x303d, 0x00},
+	{0x303e, 0x41},
+	{0x303f, 0x04},
+	{0x303A, 0x08},//hdr+
+	{0x3024, 0xc9},//hdr+ shs2 l
+	{0x3025, 0x06},//hdr+ shs2 m
+	{0x3026, 0x00},//hdr+ shs2 h
+	{0x3010, 0x61},//hdr+ gain 1frame FPGC
+	{0x3014, 0x00},//hdr+ gain 1frame long
+	{0x30F0, 0x64},//hdr+ gain 2frame FPGC
+	{0x30f2, 0x00},//hdr+ gain 2frame short
+	{0x3002, 0x00},
+	{REG_NULL, 0x00},
+};
+
+/*
+ * The width and height must be configured to be
+ * the same as the current output resolution of the sensor.
+ * The input width of the isp needs to be 16 aligned.
+ * The input height of the isp needs to be 8 aligned.
+ * If the width or height does not meet the alignment rules,
+ * you can configure the cropping parameters with the following function to
+ * crop out the appropriate resolution.
+ * struct v4l2_subdev_pad_ops {
+ *	.get_selection
+ * }
+ */
+static const struct imx327_mode lvds_supported_modes[] = {
+	{
+		.bus_fmt = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.width = 1948,
+		.height = 1110,
+		.max_fps = {
+			.numerator = 10000,
+			.denominator = 250000,
+		},
+		.exp_def = 0x03fe,
+		.hts_def = 0x1130,
+		.vts_def = 0x0546,
+		.reg_list = imx327_linear_1920x1080_lvds_regs,
+		.hdr_mode = NO_HDR,
+	}, {
+		.bus_fmt = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.width = 1948,
+		.height = 1098,
+		.max_fps = {
+			.numerator = 10000,
+			.denominator = 250000,
+		},
+		.exp_def = 0x0473,
+		.hts_def = 0x07ec,
+		.vts_def = 0x05b8 * 2,
+		.reg_list = imx327_hdr2_1920x1080_lvds_regs,
+		.hdr_mode = HDR_X2,
+	},
+};
+
+static const struct imx327_mode mipi_supported_modes[] = {
+	
+	{
+		.bus_fmt = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.width = 1280,
+		.height = 720,
+		.crop = {
+			.left = 0,
+			.top = 0,
+			.width = 1280,
+			.height = 720
+		},
+		.max_fps = {
+			.numerator = 10000,
+			.denominator = 500000,
+		},
+		.exp_def = 0x0002,
+		.hts_def = 0x07ec,
+		.vts_def = 0x05b8,
+		.reg_list = imx327_1280x720_regs,
+	},
+	{
+		.bus_fmt = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.width = 1920,
+		.height = 1080,
+		.crop = {
+			.left = 0,
+			.top = 0,
+			.width = 1920,
+			.height = 1080
+		},
+		.max_fps = {
+			.numerator = 10000,
+			.denominator = 250000,
+		},
+		.exp_def = 0x03fe,
+		.hts_def = 0x1130,
+		.vts_def = 0x0546,
+		.reg_list = imx327_1920x1080_2lanes_regs,
+		.hdr_mode = NO_HDR,
+	},
+	{
+		.bus_fmt = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.width = 1952,
+		.height = 1089,
+		.max_fps = {
+			.numerator = 10000,
+			.denominator = 250000,
+		},
+		.exp_def = 0x0473,
+		.hts_def = 0x07ec,
+		.vts_def = 0x05b8 * 2,
+		.reg_list = imx327_hdr2_1920x1080_mipi_regs,
+		.hdr_mode = HDR_X2,
+	},
+};
+
+static const s64 link_freq_menu_items[] = {
+	IMX327_LINK_FREQ_111M,
+	IMX327_LINK_FREQ_222M
+};
+
+#ifdef USED_TEST_PATTERN
+static const char * const imx327_test_pattern_menu[] = {
+	"Disabled",
+	"Bar Type 1",
+	"Bar Type 2",
+	"Bar Type 3",
+	"Bar Type 4",
+	"Bar Type 5",
+	"Bar Type 6",
+	"Bar Type 7",
+	"Bar Type 8",
+	"Bar Type 9",
+	"Bar Type 10",
+	"Bar Type 11",
+	"Bar Type 12",
+	"Bar Type 13",
+	"Bar Type 14",
+	"Bar Type 15"
+};
+#endif
+
+/* Write registers up to 4 at a time */
+static int imx327_write_reg(struct i2c_client *client, u16 reg,
+			    u32 len, u32 val)
+{
+	// printk("********** imx327_write_reg ***********\n");
+	// printk("W: reg=0x%x,    val=0x%x\n", reg, val);
+	u32 buf_i, val_i;
+	u8 buf[6];
+	u8 *val_p;
+	__be32 val_be;
+	if (len > 4)
+		return -EINVAL;
+
+	buf[0] = reg >> 8;
+	buf[1] = reg & 0xff;
+
+	val_be = cpu_to_be32(val);
+	val_p = (u8 *)&val_be;
+	buf_i = 2;
+	val_i = 4 - len;
+
+	while (val_i < 4)
+		buf[buf_i++] = val_p[val_i++];
+
+	if (i2c_master_send(client, buf, len + 2) != len + 2)
+		return -EIO;
+	udelay(1000);////add
+	return 0;
+}
+
+////delete
+// static int imx327_write_array(struct i2c_client *client,
+// 			      const struct regval *regs)
+// {
+// 	u32 i;
+// 	int ret = 0;
+
+// 	u32 val_test;
+
+// 	printk("********** imx327_write_array ***********\n");
+// 	for (i = 0; ret == 0 && regs[i].addr != REG_NULL; i++)
+// 		if (unlikely(regs[i].addr == REG_DELAY))
+// 			usleep_range(regs[i].val * 1000, regs[i].val * 2000);
+// 		else
+// 		{
+// 			ret = imx327_write_reg(client, regs[i].addr,
+// 				IMX327_REG_VALUE_08BIT,
+// 				regs[i].val);
+
+// 			imx327_read_reg(client, regs[i].addr, IMX327_REG_VALUE_08BIT, &val_test);////add
+// 			printk("R: reg=0x%x,    val=0x%x\n", regs[i].addr, val_test);////add
+// 		}
+// 	return ret;
+// }
+
+/* Read registers up to 4 at a time */
+static int imx327_read_reg(struct i2c_client *client, u16 reg,
+			   unsigned int len, u32 *val)
+{
+	struct i2c_msg msgs[2];
+	u8 *data_be_p;
+	__be32 data_be = 0;
+	__be16 reg_addr_be = cpu_to_be16(reg);
+	int ret;
+
+	if (len > 4 || !len)
+		return -EINVAL;
+
+	data_be_p = (u8 *)&data_be;
+	/* Write register address */
+	msgs[0].addr = client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 2;
+	msgs[0].buf = (u8 *)&reg_addr_be;
+
+	/* Read data from register */
+	msgs[1].addr = client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = len;
+	msgs[1].buf = &data_be_p[4 - len];
+
+	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
+	if (ret != ARRAY_SIZE(msgs))
+		return -EIO;
+
+	*val = be32_to_cpu(data_be);
+	return 0;
+}
+
+static int imx327_write_array(struct i2c_client *client,
+			      const struct regval *regs)
+{
+	u32 i;
+	int ret = 0;
+
+	for (i = 0; ret == 0 && regs[i].addr != REG_NULL; i++)
+		if (unlikely(regs[i].addr == REG_DELAY))
+			usleep_range(regs[i].val * 1000, regs[i].val * 2000);
+		else
+		{
+			ret = imx327_write_reg(client, regs[i].addr,
+				IMX327_REG_VALUE_08BIT,
+				regs[i].val);
+		}
+	return ret;
+}
+
+static int imx327_set_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_state *sd_state,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	const struct imx327_mode *mode;
+	s64 h_blank, vblank_def;
+	s32 dst_link_freq = 0;
+	s64 dst_pixel_rate = 0;
+
+	mutex_lock(&imx327->mutex);
+
+	mode = v4l2_find_nearest_size(imx327->support_modes,
+				      imx327->support_modes_num,
+				      width, height,
+				      fmt->format.width, fmt->format.height);
+	fmt->format.code = mode->bus_fmt;
+	fmt->format.width = mode->width;
+	fmt->format.height = mode->height;
+	fmt->format.field = V4L2_FIELD_NONE;
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+		*v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format;
+#else
+		mutex_unlock(&imx327->mutex);
+		return -ENOTTY;
+#endif
+	} else {
+		imx327->cur_mode = mode;
+		h_blank = mode->hts_def - mode->width;
+		__v4l2_ctrl_modify_range(imx327->hblank, h_blank,
+					 h_blank, 1, h_blank);
+		vblank_def = mode->vts_def - mode->height;
+		__v4l2_ctrl_modify_range(imx327->vblank, vblank_def,
+					 IMX327_VTS_MAX - mode->height,
+					 1, vblank_def);
+		if (imx327->cur_mode->hdr_mode == NO_HDR) {
+			dst_link_freq = 0;
+			dst_pixel_rate = IMX327_LINK_FREQ_111M;
+		} else {
+			dst_link_freq = 1;
+			dst_pixel_rate = IMX327_LINK_FREQ_222M;
+		}
+		__v4l2_ctrl_s_ctrl_int64(imx327->pixel_rate,
+					 dst_pixel_rate);
+		__v4l2_ctrl_s_ctrl(imx327->link_freq,
+				   dst_link_freq);
+		imx327->cur_vts = mode->vts_def;
+	}
+
+	mutex_unlock(&imx327->mutex);
+
+	return 0;
+}
+
+static void imx327_reset_colorspace(struct v4l2_mbus_framefmt *fmt)
+{
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+	fmt->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(fmt->colorspace);
+	fmt->quantization = V4L2_MAP_QUANTIZATION_DEFAULT(true, fmt->colorspace,
+							  fmt->ycbcr_enc);
+	fmt->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(fmt->colorspace);
+}
+
+static void imx327_update_pad_format(struct imx327 *imx327,
+				     const struct imx327_mode *mode,
+				     struct v4l2_subdev_format *fmt)
+{
+	fmt->format.width = mode->width;
+	fmt->format.height = mode->height;
+	fmt->format.field = V4L2_FIELD_NONE;
+	imx327_reset_colorspace(&fmt->format);
+}
+static int imx327_get_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_state *sd_state,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	const struct imx327_mode *mode = imx327->cur_mode;
+
+	mutex_lock(&imx327->mutex);
+	/*
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+		fmt->format = *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad);
+#else
+		mutex_unlock(&imx327->mutex);
+		return -ENOTTY;
+#endif
+	} else {
+		fmt->format.width = mode->width;
+		fmt->format.height = mode->height;
+		fmt->format.code = mode->bus_fmt;
+		fmt->format.field = V4L2_FIELD_NONE;
+	}
+	*/
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		struct v4l2_mbus_framefmt *try_fmt = v4l2_subdev_get_try_format(
+			&imx327->subdev, sd_state, fmt->pad);
+		/* update the code which could change due to vflip or hflip: */
+		try_fmt->code = MEDIA_BUS_FMT_SRGGB10_1X10;
+		fmt->format = *try_fmt;
+	} else {
+		imx327_update_pad_format(imx327, imx327->cur_mode, fmt);
+		fmt->format.code = MEDIA_BUS_FMT_SRGGB10_1X10;
+	}
+	mutex_unlock(&imx327->mutex);
+	return 0;
+}
+
+static int imx327_enum_mbus_code(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	const struct imx327_mode *mode = imx327->cur_mode;
+
+	if (code->index != 0)
+		return -EINVAL;
+	code->code = mode->bus_fmt;
+
+	return 0;
+}
+
+static int imx327_enum_frame_sizes(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_state *sd_state,
+				   struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+
+	if (fse->index >= imx327->support_modes_num)
+		return -EINVAL;
+
+	if (fse->code != imx327->support_modes[fse->index].bus_fmt)
+		return -EINVAL;
+
+	fse->min_width  = imx327->support_modes[fse->index].width;
+	fse->max_width  = imx327->support_modes[fse->index].width;
+	fse->max_height = imx327->support_modes[fse->index].height;
+	fse->min_height = imx327->support_modes[fse->index].height;
+
+	return 0;
+}
+
+#ifdef USED_TEST_PATTERN
+static int imx327_enable_test_pattern(struct imx327 *imx327, u32 pattern)
+{
+	u32 val = 0;
+
+	imx327_read_reg(imx327->client,
+			IMX327_REG_TEST_PATTERN,
+			IMX327_REG_VALUE_08BIT,
+			&val);
+	if (pattern) {
+		val = ((pattern - 1) << 4) | IMX327_TEST_PATTERN_ENABLE;
+		imx327_write_reg(imx327->client,
+				 0x300a,
+				 IMX327_REG_VALUE_08BIT,
+				 0x00);
+		imx327_write_reg(imx327->client,
+				 0x300e,
+				 IMX327_REG_VALUE_08BIT,
+				 0x00);
+	} else {
+		val &= ~IMX327_TEST_PATTERN_ENABLE;
+		imx327_write_reg(imx327->client,
+				 0x300a,
+				 IMX327_REG_VALUE_08BIT,
+				 0x3c);
+		imx327_write_reg(imx327->client,
+				 0x300e,
+				 IMX327_REG_VALUE_08BIT,
+				 0x01);
+	}
+	return imx327_write_reg(imx327->client,
+				IMX327_REG_TEST_PATTERN,
+				IMX327_REG_VALUE_08BIT,
+				val);
+}
+#endif
+
+static int imx327_g_frame_interval(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_frame_interval *fi)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	const struct imx327_mode *mode = imx327->cur_mode;
+
+	fi->interval = mode->max_fps;
+
+	return 0;
+}
+
+// static int imx327_g_mbus_config(struct v4l2_subdev *sd, unsigned int pad_id,
+// 				struct v4l2_mbus_config *config)
+// {
+// 	struct imx327 *imx327 = to_imx327(sd);
+// 	u32 val = 0;
+
+// 	val = 1 << (IMX327_4LANES - 1) |
+// 			V4L2_MBUS_CSI2_CHANNEL_0 |
+// 			V4L2_MBUS_CSI2_CONTINUOUS_CLOCK;
+// 	config->type = imx327->bus_cfg.bus_type;
+// 	config->flags = val;
+
+// 	return 0;
+// }
+static int imx327_g_mbus_config(struct v4l2_subdev *sd, unsigned int pad_id,
+				struct v4l2_mbus_config *config)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+
+	///TODO
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	config->type = V4L2_MBUS_CSI2_DPHY;
+	config->bus.mipi_csi2.num_data_lanes = 2;///Revise
+	return 0;
+}
+
+// static int imx327_set_hdrae(struct imx327 *imx327,
+// 			    struct preisp_hdrae_exp_s *ae)
+// {
+// 	u32 l_exp_time, m_exp_time, s_exp_time;
+// 	u32 l_gain, m_gain, s_gain;
+// 	u32 shs1, shs2, rhs1;
+// 	u32 gain_switch = 0;
+// 	int ret = 0;
+// 	u8 cg_mode = 0;
+// 	u32 fsc = imx327->cur_vts;//The HDR mode vts is double by default to workaround T-line
+
+// 	if (!imx327->has_init_exp && !imx327->streaming) {
+// 		imx327->init_hdrae_exp = *ae;
+// 		imx327->has_init_exp = true;
+// 		dev_dbg(&imx327->client->dev, "imx327 don't stream, record exp for hdr!\n");
+// 		return ret;
+// 	}
+
+// 	l_exp_time = ae->long_exp_reg;
+// 	m_exp_time = ae->middle_exp_reg;
+// 	s_exp_time = ae->short_exp_reg;
+// 	l_gain = ae->long_gain_reg;
+// 	m_gain = ae->middle_gain_reg;
+// 	s_gain = ae->short_gain_reg;
+
+// 	if (imx327->cur_mode->hdr_mode == HDR_X2) {
+// 		//2 stagger
+// 		l_gain = m_gain;
+// 		l_exp_time = m_exp_time;
+// 		cg_mode = ae->middle_cg_mode;
+// 	}
+// 	dev_dbg(&imx327->client->dev,
+// 		"rev exp req: L_time=%d, gain=%d, S_time=%d, gain=%d\n",
+// 		l_exp_time, l_gain,
+// 		s_exp_time, s_gain);
+// 	ret = imx327_read_reg(imx327->client, IMX327_GAIN_SWITCH_REG,
+// 				IMX327_REG_VALUE_08BIT, &gain_switch);
+// 	if (!g_isHCG && cg_mode == GAIN_MODE_HCG) {
+// 		gain_switch |= 0x0110;
+// 		g_isHCG = true;
+// 	} else if (g_isHCG && cg_mode == GAIN_MODE_LCG) {
+// 		gain_switch &= 0xef;
+// 		gain_switch |= 0x100;
+// 		g_isHCG = false;
+// 	}
+
+// 	//long exposure and short exposure
+// 	rhs1 = 0xe1;
+// 	shs1 = rhs1 - s_exp_time - 1;
+// 	shs2 = fsc - l_exp_time - 1;
+// 	if (shs1 < 2)
+// 		shs1 = 2;
+// 	if (shs2 < (rhs1 + 2))
+// 		shs2 = rhs1 + 2;
+// 	else if (shs2 > (fsc - 2))
+// 		shs2 = fsc - 2;
+
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SHS1_L,
+// 		IMX327_REG_VALUE_08BIT,
+// 		IMX327_FETCH_LOW_BYTE_EXP(shs1));
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SHS1_M,
+// 		IMX327_REG_VALUE_08BIT,
+// 		IMX327_FETCH_MID_BYTE_EXP(shs1));
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SHS1_H,
+// 		IMX327_REG_VALUE_08BIT,
+// 		IMX327_FETCH_HIGH_BYTE_EXP(shs1));
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SHS2_L,
+// 		IMX327_REG_VALUE_08BIT,
+// 		IMX327_FETCH_LOW_BYTE_EXP(shs2));
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SHS2_M,
+// 		IMX327_REG_VALUE_08BIT,
+// 		IMX327_FETCH_MID_BYTE_EXP(shs2));
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SHS2_H,
+// 		IMX327_REG_VALUE_08BIT,
+// 		IMX327_FETCH_HIGH_BYTE_EXP(shs2));
+
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_LF_GAIN,
+// 		IMX327_REG_VALUE_08BIT,
+// 		l_gain);
+// 	ret |= imx327_write_reg(imx327->client, IMX327_REG_SF_GAIN,
+// 		IMX327_REG_VALUE_08BIT,
+// 		s_gain);
+// 	if (gain_switch & 0x100) {
+// 		ret |= imx327_write_reg(imx327->client,
+// 				IMX327_GROUP_HOLD_REG,
+// 				IMX327_REG_VALUE_08BIT,
+// 				IMX327_GROUP_HOLD_START);
+// 		ret |= imx327_write_reg(imx327->client, IMX327_GAIN_SWITCH_REG,
+// 				IMX327_REG_VALUE_08BIT, gain_switch);
+// 		ret |= imx327_write_reg(imx327->client,
+// 				IMX327_GROUP_HOLD_REG,
+// 				IMX327_REG_VALUE_08BIT,
+// 				IMX327_GROUP_HOLD_END);
+// 	}
+// 	dev_dbg(&imx327->client->dev,
+// 		"set l_gain:0x%x s_gain:0x%x shs2:0x%x shs1:0x%x\n",
+// 		l_gain, s_gain, shs2, shs1);
+// 	return ret;
+// }
+
+#define USED_SYS_DEBUG
+#ifdef USED_SYS_DEBUG
+static int imx327_set_conversion_gain(struct imx327 *imx327, u32 *cg)
+{
+	int ret = 0;
+	struct i2c_client *client = imx327->client;
+	int cur_cg = *cg;
+	u32 gain_switch = 0;
+
+	ret = imx327_read_reg(client,
+		IMX327_GAIN_SWITCH_REG,
+		IMX327_REG_VALUE_08BIT,
+		&gain_switch);
+	if (g_isHCG && cur_cg == GAIN_MODE_LCG) {
+		gain_switch &= 0xef;
+		gain_switch |= 0x0100;
+		g_isHCG = false;
+	} else if (!g_isHCG && cur_cg == GAIN_MODE_HCG) {
+		gain_switch |= 0x0110;
+		g_isHCG = true;
+	}
+
+	if (gain_switch & 0x100) {
+		ret |= imx327_write_reg(client,
+			IMX327_GROUP_HOLD_REG,
+			IMX327_REG_VALUE_08BIT,
+			IMX327_GROUP_HOLD_START);
+		ret |= imx327_write_reg(client,
+			IMX327_GAIN_SWITCH_REG,
+			IMX327_REG_VALUE_08BIT,
+			gain_switch & 0xff);
+		ret |= imx327_write_reg(client,
+			IMX327_GROUP_HOLD_REG,
+			IMX327_REG_VALUE_08BIT,
+			IMX327_GROUP_HOLD_END);
+	}
+
+	return ret;
+}
+//ag: echo 0 >  /sys/devices/platform/ff510000.i2c/i2c-1/1-0037/cam_s_cg
+static ssize_t set_conversion_gain_status(struct device *dev,
+	struct device_attribute *attr,
+	const char *buf,
+	size_t count)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx327 *imx327 = to_imx327(sd);
+	int status = 0;
+	int ret = 0;
+
+	ret = kstrtoint(buf, 0, &status);
+	if (!ret && status >= 0 && status < 2)
+		imx327_set_conversion_gain(imx327, &status);
+	else
+		dev_err(dev, "input 0 for LCG, 1 for HCG, cur %d\n", status);
+	return count;
+}
+
+static struct device_attribute attributes[] = {
+	__ATTR(cam_s_cg, S_IWUSR, NULL, set_conversion_gain_status),
+};
+
+static int add_sysfs_interfaces(struct device *dev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(attributes); i++)
+		if (device_create_file(dev, attributes + i))
+			goto undo;
+	return 0;
+undo:
+	for (i--; i >= 0 ; i--)
+		device_remove_file(dev, attributes + i);
+	dev_err(dev, "%s: failed to create sysfs interface\n", __func__);
+	return -ENODEV;
+}
+
+static int remove_sysfs_interfaces(struct device *dev)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(attributes); i++)
+		device_remove_file(dev, attributes + i);
+	return 0;
+}
+
+#endif
+
+static long imx327_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
+{
+	// struct imx327 *imx327 = to_imx327(sd);///add
+	/// return 0;
+	return 1;
+}
+
+#ifdef CONFIG_COMPAT
+static long imx327_compat_ioctl32(struct v4l2_subdev *sd,
+				  unsigned int cmd, unsigned long arg)
+{
+	// void __user *up = compat_ptr(arg);///add
+	/// return 0;
+	return 1;
+}
+#endif
+
+static int imx327_init_conversion_gain(struct imx327 *imx327)
+{
+	int ret = 0;
+	struct i2c_client *client = imx327->client;
+	u32 val = 0;
+
+	ret = imx327_read_reg(client,
+		IMX327_GAIN_SWITCH_REG,
+		IMX327_REG_VALUE_08BIT,
+		&val);
+	val &= 0xef;
+	ret = imx327_write_reg(client,
+		IMX327_GAIN_SWITCH_REG,
+		IMX327_REG_VALUE_08BIT,
+		val);
+	if (!ret)
+		g_isHCG = false;
+	return ret;
+}
+
+static int __imx327_start_stream(struct imx327 *imx327)
+{
+	int ret;
+
+
+	ret = imx327_write_array(imx327->client, imx327->cur_mode->reg_list);
+	if (ret)
+		return ret;
+	
+	// ret = imx327_init_conversion_gain(imx327);
+	// if (ret)
+	// 	return ret;
+	// /* In case these controls are set before streaming */
+	// ret = __v4l2_ctrl_handler_setup(&imx327->ctrl_handler);
+	// ret = imx327_write_reg(imx327->client,
+	// 	IMX327_REG_CTRL_MODE,
+	// 	IMX327_REG_VALUE_08BIT,
+	// 	0);
+	
+	return ret;
+}
+
+static int __imx327_stop_stream(struct imx327 *imx327)
+{
+	pr_info("imx327_stop_stream\n");
+	return imx327_write_reg(imx327->client,
+		IMX327_REG_CTRL_MODE,
+		IMX327_REG_VALUE_08BIT,
+		1);
+}
+
+static int imx327_s_stream(struct v4l2_subdev *sd, int on)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	struct i2c_client *client = imx327->client;
+	int ret = 0;
+
+	mutex_lock(&imx327->mutex);
+	on = !!on;
+	if (on == imx327->streaming)
+		goto unlock_and_return;
+
+	if (on) {
+		ret = pm_runtime_get_sync(&client->dev);
+		if (ret < 0) {
+			pm_runtime_put_noidle(&client->dev);
+			goto unlock_and_return;
+		}
+
+		ret = __imx327_start_stream(imx327);
+		if (ret) {
+			v4l2_err(sd, "start stream failed while write regs\n");
+			pm_runtime_put(&client->dev);
+			goto unlock_and_return;
+		}
+	} else {
+		__imx327_stop_stream(imx327);
+		pm_runtime_put(&client->dev);
+	}
+
+	imx327->streaming = on;
+
+unlock_and_return:
+	mutex_unlock(&imx327->mutex);
+
+	return ret;
+}
+
+static int imx327_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	struct i2c_client *client = imx327->client;
+	int ret = 0;
+
+	mutex_lock(&imx327->mutex);
+
+	/* If the power state is not modified - no work to do. */
+	if (imx327->power_on == !!on)
+		goto unlock_and_return;
+
+	if (on) {
+		ret = pm_runtime_get_sync(&client->dev);
+		if (ret < 0) {
+			pm_runtime_put_noidle(&client->dev);
+			goto unlock_and_return;
+		}
+
+		ret = imx327_write_array(imx327->client, imx327_global_regs);
+		if (ret) {
+			v4l2_err(sd, "could not set init registers\n");
+			pm_runtime_put_noidle(&client->dev);
+			goto unlock_and_return;
+		}
+
+		imx327->power_on = true;
+	} else {
+		pm_runtime_put(&client->dev);
+		imx327->power_on = false;
+	}
+
+unlock_and_return:
+	mutex_unlock(&imx327->mutex);
+
+	return ret;
+}
+
+/* Calculate the delay in us by clock rate and clock cycles */
+static inline u32 imx327_cal_delay(u32 cycles)
+{
+	return DIV_ROUND_UP(cycles, IMX327_XVCLK_FREQ / 1000 / 1000);
+}
+
+static int __imx327_power_on(struct imx327 *imx327)
+{
+	int ret;
+	u32 delay_us;
+	struct device *dev = &imx327->client->dev;
+
+	// if (!IS_ERR_OR_NULL(imx327->pins_default)) {
+	// 	ret = pinctrl_select_state(imx327->pinctrl,
+	// 				   imx327->pins_default);
+	// 	if (ret < 0)
+	// 		dev_err(dev, "could not set pins\n");
+	// }
+
+
+	ret = clk_set_rate(imx327->xvclk, IMX327_XVCLK_FREQ);
+	if (ret < 0)
+		dev_warn(dev, "Failed to set xvclk rate (37.125M Hz)\n");
+
+	if (clk_get_rate(imx327->xvclk) != IMX327_XVCLK_FREQ)
+		dev_warn(dev, "xvclk mismatched,based on 24M Hz\n");
+
+	ret = clk_prepare_enable(imx327->xvclk);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable xvclk\n");
+		return ret;
+	}
+
+
+	ret = regulator_bulk_enable(IMX327_NUM_SUPPLIES, imx327->supplies);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators\n");
+		goto disable_clk;
+	}
+
+	if (!IS_ERR(imx327->reset_gpio))
+		gpiod_set_value_cansleep(imx327->reset_gpio, 0);
+	usleep_range(500, 1000);
+	if (!IS_ERR(imx327->reset_gpio))
+		gpiod_set_value_cansleep(imx327->reset_gpio, 1);
+
+	if (!IS_ERR(imx327->pwdn_gpio))
+		gpiod_set_value_cansleep(imx327->pwdn_gpio, 0);
+
+	/* 8192 cycles prior to first SCCB transaction */
+	delay_us = imx327_cal_delay(8192);
+	usleep_range(delay_us, delay_us * 2);
+	usleep_range(5000, 10000);
+	return 0;
+
+disable_clk:
+	clk_disable_unprepare(imx327->xvclk);
+
+	return ret;
+}
+
+static void __imx327_power_off(struct imx327 *imx327)
+{
+//	int ret;
+//	struct device *dev = &imx327->client->dev;
+
+	if (!IS_ERR(imx327->pwdn_gpio))
+		gpiod_set_value_cansleep(imx327->pwdn_gpio, 0);
+	clk_disable_unprepare(imx327->xvclk);
+	if (!IS_ERR(imx327->reset_gpio))
+		gpiod_set_value_cansleep(imx327->reset_gpio, 0);
+	// if (!IS_ERR_OR_NULL(imx327->pins_sleep)) {
+	// 	ret = pinctrl_select_state(imx327->pinctrl,
+	// 				   imx327->pins_sleep);
+	// 	if (ret < 0)
+	// 		dev_dbg(dev, "could not set pins\n");
+	// }
+	regulator_bulk_disable(IMX327_NUM_SUPPLIES, imx327->supplies);
+}
+
+ int imx327_runtime_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx327 *imx327 = to_imx327(sd);
+
+	return __imx327_power_on(imx327);
+}
+
+ int imx327_runtime_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx327 *imx327 = to_imx327(sd);
+
+	__imx327_power_off(imx327);
+
+	return 0;
+}
+
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+static int imx327_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+	struct v4l2_mbus_framefmt *try_fmt =
+				v4l2_subdev_get_try_format(sd, fh->state, 0);
+	const struct imx327_mode *def_mode = &imx327->support_modes[0];
+
+	mutex_lock(&imx327->mutex);
+	/* Initialize try_fmt */
+	try_fmt->width = def_mode->width;
+	try_fmt->height = def_mode->height;
+	// try_fmt->code = def_mode->bus_fmt;
+	try_fmt->code = MEDIA_BUS_FMT_SRGGB10_1X10;
+	try_fmt->field = V4L2_FIELD_NONE;
+
+	mutex_unlock(&imx327->mutex);
+	/* No crop or compose */
+
+	return 0;
+}
+#endif
+
+static int imx327_enum_frame_interval(struct v4l2_subdev *sd,
+				       struct v4l2_subdev_state *sd_state,
+				       struct v4l2_subdev_frame_interval_enum *fie)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+
+	if (fie->index >= imx327->support_modes_num)
+		return -EINVAL;
+
+	fie->code = imx327->support_modes[fie->index].bus_fmt;
+	fie->width = imx327->support_modes[fie->index].width;
+	fie->height = imx327->support_modes[fie->index].height;
+	fie->interval = imx327->support_modes[fie->index].max_fps;
+	fie->reserved[0] = imx327->support_modes[fie->index].hdr_mode;
+	return 0;
+}
+
+#define CROP_START(SRC, DST) (((SRC) - (DST)) / 2 / 4 * 4)
+// #define DST_WIDTH 1920
+// #define DST_HEIGHT 1080
+#define DST_WIDTH 1280
+#define DST_HEIGHT 720
+
+/*
+ * The resolution of the driver configuration needs to be exactly
+ * the same as the current output resolution of the sensor,
+ * the input width of the isp needs to be 16 aligned,
+ * the input height of the isp needs to be 8 aligned.
+ * Can be cropped to standard resolution by this function,
+ * otherwise it will crop out strange resolution according
+ * to the alignment rules.
+ */
+
+static int imx327_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *sd_state,
+				struct v4l2_subdev_selection *sel)
+{
+	struct imx327 *imx327 = to_imx327(sd);
+
+	if (sel->target == V4L2_SEL_TGT_CROP_BOUNDS) {
+		sel->r.left = CROP_START(imx327->cur_mode->width, DST_WIDTH);
+		sel->r.width = imx327->cur_mode->width;
+		if (imx327->bus_cfg.bus_type == V4L2_MBUS_CCP2) {
+			if (imx327->cur_mode->hdr_mode == NO_HDR)
+				sel->r.top = 21;
+			else
+				sel->r.top = 13;
+		} else {
+			sel->r.top = CROP_START(imx327->cur_mode->height, DST_HEIGHT);
+		}
+		sel->r.height = imx327->cur_mode->crop.height;
+
+		return 0;
+	} else if(sel->target == V4L2_SEL_TGT_CROP) {
+
+		mutex_lock(&imx327->mutex);
+		// sel->r = imx327->cur_mode->crop;
+		sel->r.left = CROP_START(imx327->cur_mode->width, DST_WIDTH);
+		sel->r.width = imx327->cur_mode->width;
+		sel->r.top = CROP_START(imx327->cur_mode->height, DST_HEIGHT);
+		sel->r.height = imx327->cur_mode->crop.height;
+		mutex_unlock(&imx327->mutex);
+
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static const struct dev_pm_ops imx327_pm_ops = {
+	SET_RUNTIME_PM_OPS(imx327_runtime_suspend,
+			   imx327_runtime_resume, NULL)
+};
+
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+static const struct v4l2_subdev_internal_ops imx327_internal_ops = {
+	.open = imx327_open,
+};
+#endif
+
+static const struct v4l2_subdev_core_ops imx327_core_ops = {
+	.s_power = imx327_s_power,
+	.ioctl = imx327_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl32 = imx327_compat_ioctl32,
+#endif
+	.subscribe_event = v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static const struct v4l2_subdev_video_ops imx327_video_ops = {
+	.s_stream = imx327_s_stream,
+	.g_frame_interval = imx327_g_frame_interval,
+};
+
+static const struct v4l2_subdev_pad_ops imx327_pad_ops = {
+	.enum_mbus_code = imx327_enum_mbus_code,
+	.enum_frame_size = imx327_enum_frame_sizes,
+	.enum_frame_interval = imx327_enum_frame_interval,
+	.get_fmt = imx327_get_fmt,
+	.set_fmt = imx327_set_fmt,
+	.get_selection = imx327_get_selection,
+	.get_mbus_config = imx327_g_mbus_config,
+};
+
+static const struct v4l2_subdev_ops imx327_subdev_ops = {
+	.core	= &imx327_core_ops,
+	.video	= &imx327_video_ops,
+	.pad	= &imx327_pad_ops,
+};
+
+static int imx327_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct imx327 *imx327 = container_of(ctrl->handler,
+					     struct imx327, ctrl_handler);
+	struct i2c_client *client = imx327->client;
+	s64 max;
+	int ret = 0;
+	u32 shs1 = 0;
+	u32 vts = 0;
+	u32 val = 0;
+
+	/* Propagate change of current control to all related controls */
+	switch (ctrl->id) {
+	case V4L2_CID_VBLANK:
+		/* Update max exposure while meeting expected vblanking */
+		max = imx327->cur_mode->height + ctrl->val - 2;
+		__v4l2_ctrl_modify_range(imx327->exposure,
+					 imx327->exposure->minimum, max,
+					 imx327->exposure->step,
+					 imx327->exposure->default_value);
+		break;
+	}
+
+	if (!pm_runtime_get_if_in_use(&client->dev))
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_EXPOSURE:
+		if (imx327->cur_mode->hdr_mode == NO_HDR) {
+			shs1 = imx327->cur_vts - ctrl->val - 1;
+			ret = imx327_write_reg(imx327->client,
+				IMX327_REG_SHS1_H,
+				IMX327_REG_VALUE_08BIT,
+				IMX327_FETCH_HIGH_BYTE_EXP(shs1));
+			ret |= imx327_write_reg(imx327->client,
+				IMX327_REG_SHS1_M,
+				IMX327_REG_VALUE_08BIT,
+				IMX327_FETCH_MID_BYTE_EXP(shs1));
+			ret |= imx327_write_reg(imx327->client,
+				IMX327_REG_SHS1_L,
+				IMX327_REG_VALUE_08BIT,
+				IMX327_FETCH_LOW_BYTE_EXP(shs1));
+			dev_dbg(&client->dev, "set exposure 0x%x, cur_vts 0x%x,shs1 0x%x\n",
+				ctrl->val, imx327->cur_vts, shs1);
+		}
+		break;
+	case V4L2_CID_DIGITAL_GAIN:
+	case V4L2_CID_NOTIFY_GAINS:
+		break;
+	case V4L2_CID_ANALOGUE_GAIN:
+		if (imx327->cur_mode->hdr_mode == NO_HDR) {
+			ret = imx327_write_reg(imx327->client,
+				IMX327_REG_LF_GAIN,
+				IMX327_REG_VALUE_08BIT,
+				ctrl->val);
+			dev_dbg(&client->dev, "set analog gain 0x%x\n",
+				ctrl->val);
+		}
+		break;
+	case V4L2_CID_VBLANK:
+		vts = ctrl->val + imx327->cur_mode->height;
+		imx327->cur_vts = vts;
+		if (imx327->cur_mode->hdr_mode == HDR_X2)
+			vts /= 2;
+		ret = imx327_write_reg(imx327->client,
+			IMX327_REG_VTS_H,
+			IMX327_REG_VALUE_08BIT,
+			IMX327_FETCH_HIGH_BYTE_VTS(vts));
+		ret |= imx327_write_reg(imx327->client,
+			IMX327_REG_VTS_M,
+			IMX327_REG_VALUE_08BIT,
+			IMX327_FETCH_MID_BYTE_VTS(vts));
+		ret |= imx327_write_reg(imx327->client,
+			IMX327_REG_VTS_L,
+			IMX327_REG_VALUE_08BIT,
+			IMX327_FETCH_LOW_BYTE_VTS(vts));
+		dev_dbg(&client->dev, "set vts 0x%x\n",
+			vts);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+#ifdef USED_TEST_PATTERN
+		ret = imx327_enable_test_pattern(imx327, ctrl->val);
+#endif
+		break;
+	case V4L2_CID_HFLIP:
+		ret = imx327_read_reg(client,
+				      IMX327_FLIP_REG,
+				      IMX327_REG_VALUE_08BIT,
+				      &val);
+		if (ctrl->val)
+			val |= MIRROR_BIT_MASK;
+		else
+			val &= ~MIRROR_BIT_MASK;
+		ret |= imx327_write_reg(client,
+					IMX327_FLIP_REG,
+					IMX327_REG_VALUE_08BIT,
+					val);
+		if (ret == 0)
+			imx327->flip = val;
+		break;
+	case V4L2_CID_VFLIP:
+		ret = imx327_read_reg(client,
+				      IMX327_FLIP_REG,
+				      IMX327_REG_VALUE_08BIT,
+				      &val);
+		if (ctrl->val)
+			val |= FLIP_BIT_MASK;
+		else
+			val &= ~FLIP_BIT_MASK;
+		ret |= imx327_write_reg(client,
+					IMX327_FLIP_REG,
+					IMX327_REG_VALUE_08BIT,
+					val);
+		if (ret == 0)
+			imx327->flip = val;
+		break;
+	default:
+		dev_warn(&client->dev, "%s Unhandled id:0x%x, val:0x%x\n",
+			 __func__, ctrl->id, ctrl->val);
+		break;
+	}
+
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops imx327_ctrl_ops = {
+	.s_ctrl = imx327_set_ctrl,
+};
+
+static int imx327_initialize_controls(struct imx327 *imx327)
+{
+	const struct imx327_mode *mode;
+	struct v4l2_ctrl_handler *handler;
+	s64 exposure_max, vblank_def;
+	u32 h_blank;
+	int ret;
+	s32 dst_link_freq = 0;
+	s64 dst_pixel_rate = 0;
+
+	handler = &imx327->ctrl_handler;
+	mode = imx327->cur_mode;
+	ret = v4l2_ctrl_handler_init(handler, 9);
+	if (ret)
+		return ret;
+	handler->lock = &imx327->mutex;
+
+	imx327->link_freq = v4l2_ctrl_new_int_menu(handler, NULL, V4L2_CID_LINK_FREQ,
+				      1, 0, link_freq_menu_items);
+
+	if (imx327->cur_mode->hdr_mode == NO_HDR) {
+		dst_link_freq = 0;
+		dst_pixel_rate = IMX327_PIXEL_RATE_NORMAL;
+	} else {
+		dst_link_freq = 1;
+		dst_pixel_rate = IMX327_PIXEL_RATE_HDR;
+	}
+	__v4l2_ctrl_s_ctrl(imx327->link_freq,
+			   dst_link_freq);
+	imx327->pixel_rate = v4l2_ctrl_new_std(handler, NULL, V4L2_CID_PIXEL_RATE,
+			  0, IMX327_PIXEL_RATE_HDR, 1, dst_pixel_rate);
+
+	h_blank = mode->hts_def - mode->width;
+
+	imx327->hblank = v4l2_ctrl_new_std(handler, NULL, V4L2_CID_HBLANK,
+				h_blank, h_blank, 1, h_blank);
+	if (imx327->hblank)
+		imx327->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	vblank_def = mode->vts_def - mode->height;
+	imx327->cur_vts = mode->vts_def;
+	imx327->vblank = v4l2_ctrl_new_std(handler, &imx327_ctrl_ops,
+				V4L2_CID_VBLANK, vblank_def,
+				IMX327_VTS_MAX - mode->height,
+				1, vblank_def);
+
+	exposure_max = mode->vts_def - 2;
+
+	imx327->exposure = v4l2_ctrl_new_std(handler, &imx327_ctrl_ops,
+				V4L2_CID_EXPOSURE, IMX327_EXPOSURE_MIN,
+				exposure_max, IMX327_EXPOSURE_STEP,
+				mode->exp_def);
+
+	v4l2_ctrl_new_std(handler, &imx327_ctrl_ops, V4L2_CID_NOTIFY_GAINS,
+			IMX327_GAIN_MIN, IMX327_GAIN_MAX,
+			IMX327_GAIN_STEP, IMX327_GAIN_DEFAULT);
+
+	imx327->anal_gain = v4l2_ctrl_new_std(handler, &imx327_ctrl_ops,
+				V4L2_CID_ANALOGUE_GAIN, IMX327_GAIN_MIN,
+				IMX327_GAIN_MAX, IMX327_GAIN_STEP,
+				IMX327_GAIN_DEFAULT);
+
+	imx327->digi_gain = v4l2_ctrl_new_std(handler, &imx327_ctrl_ops,
+				V4L2_CID_DIGITAL_GAIN, IMX327_GAIN_MIN,
+				IMX327_GAIN_MAX, IMX327_GAIN_STEP,
+				IMX327_GAIN_DEFAULT);
+
+#ifdef USED_TEST_PATTERN
+	imx327->test_pattern = v4l2_ctrl_new_std_menu_items(handler,
+				&imx327_ctrl_ops, V4L2_CID_TEST_PATTERN,
+				ARRAY_SIZE(imx327_test_pattern_menu) - 1,
+				0, 0, imx327_test_pattern_menu);
+#endif
+	imx327->h_flip = v4l2_ctrl_new_std(handler, &imx327_ctrl_ops,
+				V4L2_CID_HFLIP, 0, 1, 1, 0);
+
+	imx327->v_flip = v4l2_ctrl_new_std(handler, &imx327_ctrl_ops,
+				V4L2_CID_VFLIP, 0, 1, 1, 0);
+	imx327->flip = 0;
+	if (handler->error) {
+		ret = handler->error;
+		dev_err(&imx327->client->dev,
+			"Failed to init controls(%d)\n", ret);
+		goto err_free_handler;
+	}
+
+	imx327->subdev.ctrl_handler = handler;
+	imx327->has_init_exp = false;
+
+	return 0;
+
+err_free_handler:
+	v4l2_ctrl_handler_free(handler);
+
+	return ret;
+}
+
+static int imx327_check_sensor_id(struct imx327 *imx327,
+				  struct i2c_client *client)
+{
+	struct device *dev = &imx327->client->dev;
+	u32 id = 0;
+	int ret;
+	u32 val = 0;
+
+	ret = imx327_read_reg(client, IMX327_REG_CHIP_ID,
+			      IMX327_REG_VALUE_08BIT, &id);
+
+	ret = imx327_read_reg(client, 0x3008,
+			      IMX327_REG_VALUE_08BIT, &val);
+	printk("imx327 sensor id : %06x, 0x3008:0x%x", id, val);
+
+	if (id != CHIP_ID) {
+		dev_err(dev, "Unexpected sensor id(%06x), ret(%d)\n", id, ret);
+		return -EINVAL;
+	}
+	return ret;
+}
+
+static int imx327_configure_regulators(struct imx327 *imx327)
+{
+	unsigned int i;
+
+	for (i = 0; i < IMX327_NUM_SUPPLIES; i++)
+		imx327->supplies[i].supply = imx327_supply_names[i];
+
+	return devm_regulator_bulk_get(&imx327->client->dev,
+				       IMX327_NUM_SUPPLIES,
+				       imx327->supplies);
+}
+
+static int imx327_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	//struct device_node *node = dev->of_node;
+	struct imx327 *imx327;
+	struct v4l2_subdev *sd;
+	int ret;
+	struct device_node *endpoint;
+
+	dev_info(dev, "driver version: %02x.%02x.%02x",
+		DRIVER_VERSION >> 16,
+		(DRIVER_VERSION & 0xff00) >> 8,
+		DRIVER_VERSION & 0x00ff);
+
+		// printk("%s in \n", __func__);
+
+	imx327 = devm_kzalloc(dev, sizeof(*imx327), GFP_KERNEL);
+	if (!imx327)
+		return -ENOMEM;
+
+	// endpoint = of_graph_get_next_endpoint(dev->of_node, NULL);
+	// if (!endpoint) {
+	// 	dev_err(dev, "Failed to get endpoint\n");
+	// 	return -EINVAL;
+	// }
+
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(endpoint),
+		&imx327->bus_cfg);
+	if (imx327->bus_cfg.bus_type == V4L2_MBUS_CCP2) {
+		imx327->support_modes = lvds_supported_modes;
+		imx327->support_modes_num = ARRAY_SIZE(lvds_supported_modes);
+	} else {
+		imx327->support_modes = mipi_supported_modes;
+		imx327->support_modes_num = ARRAY_SIZE(mipi_supported_modes);
+	}
+	imx327->client = client;
+	imx327->cur_mode = &imx327->support_modes[0];
+
+	imx327->xvclk = devm_clk_get(dev, "xvclk");
+	if (IS_ERR(imx327->xvclk)) {
+		dev_err(dev, "Failed to get xvclk\n");
+		return -EINVAL;
+	}
+
+	imx327->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(imx327->reset_gpio))
+		dev_warn(dev, "Failed to get reset-gpios\n");
+
+	imx327->pwdn_gpio = devm_gpiod_get(dev, "pwdn", GPIOD_OUT_LOW);
+	if (IS_ERR(imx327->pwdn_gpio))
+		dev_warn(dev, "Failed to get pwdn-gpios\n");
+
+	ret = imx327_configure_regulators(imx327);
+	if (ret) {
+		dev_err(dev, "Failed to get power regulators\n");
+		return ret;
+	}
+
+	// imx327->pinctrl = devm_pinctrl_get(dev);
+	// if (!IS_ERR(imx327->pinctrl)) {
+	// 	imx327->pins_default =
+	// 		pinctrl_lookup_state(imx327->pinctrl,
+	// 				     OF_CAMERA_PINCTRL_STATE_DEFAULT);
+	// 	if (IS_ERR(imx327->pins_default))
+	// 		dev_err(dev, "could not get default pinstate\n");
+
+	// 	imx327->pins_sleep =
+	// 		pinctrl_lookup_state(imx327->pinctrl,
+	// 				     OF_CAMERA_PINCTRL_STATE_SLEEP);
+	// 	if (IS_ERR(imx327->pins_sleep))
+	// 		dev_err(dev, "could not get sleep pinstate\n");
+	// }
+
+	mutex_init(&imx327->mutex);
+
+	sd = &imx327->subdev;
+	v4l2_i2c_subdev_init(sd, client, &imx327_subdev_ops);
+	ret = imx327_initialize_controls(imx327);
+	if (ret)
+		goto err_destroy_mutex;
+
+	ret = __imx327_power_on(imx327);
+	if (ret)
+		goto err_free_handler;
+
+	ret = imx327_check_sensor_id(imx327, client);
+	if (ret)
+		goto err_power_off;
+
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+	dev_info(dev, "set the video v4l2 subdev api\n");
+	sd->internal_ops = &imx327_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+		     V4L2_SUBDEV_FL_HAS_EVENTS;
+#endif
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	dev_info(dev, "set the media controller\n");
+	imx327->pad.flags = MEDIA_PAD_FL_SOURCE;
+	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
+	ret = media_entity_pads_init(&sd->entity, 1, &imx327->pad);
+	if (ret < 0)
+		goto err_power_off;
+#endif
+
+
+	snprintf(sd->name, sizeof(sd->name), "m%s %s", IMX327_NAME, dev_name(sd->dev));
+	ret = v4l2_async_register_subdev_sensor(sd);
+	if (ret) {
+		dev_err(dev, "v4l2 async register subdev failed\n");
+		goto err_clean_entity;
+	}
+
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	pm_runtime_idle(dev);
+	g_isHCG = false;
+#ifdef USED_SYS_DEBUG
+	add_sysfs_interfaces(dev);
+#endif
+
+	imx327_read_reg(client, 0x3008,
+			      IMX327_REG_VALUE_08BIT, &ret);
+	dev_info(dev, "v4l2 async register subdev success\n");
+	return 0;
+
+err_clean_entity:
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	media_entity_cleanup(&sd->entity);
+#endif
+err_power_off:
+	__imx327_power_off(imx327);
+err_free_handler:
+	v4l2_ctrl_handler_free(&imx327->ctrl_handler);
+err_destroy_mutex:
+	mutex_destroy(&imx327->mutex);
+
+	return ret;
+}
+
+static void imx327_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx327 *imx327 = to_imx327(sd);
+
+#ifdef USED_SYS_DEBUG
+	remove_sysfs_interfaces(&client->dev);
+#endif
+	v4l2_async_unregister_subdev(sd);
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	media_entity_cleanup(&sd->entity);
+#endif
+	v4l2_ctrl_handler_free(&imx327->ctrl_handler);
+	mutex_destroy(&imx327->mutex);
+
+	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev))
+		__imx327_power_off(imx327);
+	pm_runtime_set_suspended(&client->dev);
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id imx327_of_match[] = {
+	{ .compatible = "sony,imx327" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, imx327_of_match);
+#endif
+
+static const struct i2c_device_id imx327_match_id[] = {
+	{ "sony,imx327", 0 },
+	{ },
+};
+
+static struct i2c_driver imx327_i2c_driver = {
+	.driver = {
+		.name = IMX327_NAME,
+		.pm = &imx327_pm_ops,
+		.of_match_table = of_match_ptr(imx327_of_match),
+	},
+	.probe		= imx327_probe,
+	.remove		= imx327_remove,
+	.id_table	= imx327_match_id,
+};
+
+static int __init sensor_mod_init(void)
+{
+	return i2c_add_driver(&imx327_i2c_driver);
+}
+
+static void __exit sensor_mod_exit(void)
+{
+	i2c_del_driver(&imx327_i2c_driver);
+}
+
+device_initcall_sync(sensor_mod_init);
+module_exit(sensor_mod_exit);
+
+MODULE_DESCRIPTION("Sony imx327 sensor driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index ee579916f874..1e0124af0b73 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -85,5 +85,6 @@ source "drivers/media/platform/ti/Kconfig"
 source "drivers/media/platform/verisilicon/Kconfig"
 source "drivers/media/platform/via/Kconfig"
 source "drivers/media/platform/xilinx/Kconfig"
+source "drivers/media/platform/eswin/Kconfig"
 
 endif # MEDIA_PLATFORM_DRIVERS
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 5453bb868e67..02d06ce4d6cd 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -34,3 +34,7 @@ obj-y += xilinx/
 # (e. g. LC_ALL=C sort Makefile)
 obj-$(CONFIG_VIDEO_MEM2MEM_DEINTERLACE)	+= m2m-deinterlace.o
 obj-$(CONFIG_VIDEO_MUX)			+= video-mux.o
+obj-$(CONFIG_VIDEO_ESWIN_DVP2AXI) += eswin/dvp2axi/
+obj-$(CONFIG_VIDEO_ESWIN_DVP2AXI) += eswin/mipi-csi2/
+obj-$(CONFIG_VIDEO_ESWIN_DVP2AXI) += eswin/vi_top/
+obj-$(CONFIG_VIDEO_ESWIN_ISP) += eswin/isp/
diff --git a/drivers/media/platform/eswin/Kconfig b/drivers/media/platform/eswin/Kconfig
new file mode 100644
index 000000000000..59728263eb1b
--- /dev/null
+++ b/drivers/media/platform/eswin/Kconfig
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+source "drivers/media/platform/eswin/dvp2axi/Kconfig"
+source "drivers/media/platform/eswin/mipi-csi2/Kconfig"
+source "drivers/media/platform/eswin/vi_top/Kconfig"
+source "drivers/media/platform/eswin/isp/Kconfig"
+
+config EIC7700_EVB_VI
+    bool "ESWIN EVB Board Support"
+    default n
+
diff --git a/drivers/media/platform/eswin/dvp2axi/Kconfig b/drivers/media/platform/eswin/dvp2axi/Kconfig
new file mode 100644
index 000000000000..edd144481e3d
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/Kconfig
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: GPL-2.0
+config VIDEO_ESWIN_DVP2AXI_DEPENDENCES
+	bool "Eswin Camera AXI2DVP dependences"
+	default "y"
+	select MEDIA_CAMERA_SUPPORT
+	select VIDEO_V4L2
+	select MEDIA_CONTROLLER
+	select VIDEO_V4L2_SUBDEV_API
+	select VIDEOBUF2_DMA_CONTIG
+	select V4L2_FWNODE
+	select VIDEOBUF2_CMA_SG
+	select VIDEOBUF2_VMALLOC
+	select V4L2_CCI_I2C
+	help
+	  Support for DVP2AXi dependences
+
+config VIDEO_ESWIN_DVP2AXI
+	tristate "Eswin Camera AXI2DVP driver"
+	depends on VIDEO_ESWIN_DVP2AXI_DEPENDENCES
+#	select MEDIA_CAMERA_SUPPORT
+#	select VIDEO_V4L2
+#	select MEDIA_CONTROLLER
+#	select VIDEO_V4L2_SUBDEV_API
+#	select VIDEOBUF2_DMA_CONTIG
+#	select V4L2_FWNODE
+#	select VIDEOBUF2_CMA_SG
+#	select VIDEOBUF2_VMALLOC
+	help
+	  Support for DVP2AXi on the eswin eic770x
+
+config ES_DVP2AXI_WORKMODE_ONEFRAME
+	bool "Eswin Camera AXI2DVP Work Mode Test"
+	default "y"
+	help
+	  Support for DVP2AXi ONEFRAME mode on the eswin eic770x
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/Makefile b/drivers/media/platform/eswin/dvp2axi/Makefile
new file mode 100644
index 000000000000..038d62302199
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VIDEO_ESWIN_DVP2AXI) += video_dvp2axi.o
+video_dvp2axi-objs += dev.o \
+			capture.o \
+			dvp2axi-luma.o \
+			hw.o \
+			subdev-itf.o \
+			procfs.o \
+			dvp2axi-scale.o \
+			common.o \
diff --git a/drivers/media/platform/eswin/dvp2axi/capture.c b/drivers/media/platform/eswin/dvp2axi/capture.c
new file mode 100644
index 000000000000..c697cbc975cc
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/capture.c
@@ -0,0 +1,12622 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI capture driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#include <linux/delay.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/iommu.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include "es-isp32-config.h"
+
+#include "dev.h"
+// #include "mipi-csi2.h"
+#include "common.h"
+#include "es-dvp2axi-externel.h"
+
+/* eic770x */
+#include "dvp2axi.h"
+#include <media/eswin/common-def.h>
+
+#define CSI2_ERR_FSFE_MASK	(0xff << 8)
+#define CSI2_ERR_COUNT_ALL_MASK	(0xff)
+
+#define ES_DVP2AXI_V4L2_EVENT_ELEMS 4
+
+#define DVP2AXI_REQ_BUFS_MIN 1
+#define DVP2AXI_MIN_WIDTH 64
+#define DVP2AXI_MIN_HEIGHT 64
+#define DVP2AXI_MAX_WIDTH 8192
+#define DVP2AXI_MAX_HEIGHT 8192
+
+#define OUTPUT_STEP_WISE 8
+
+#define ES_DVP2AXI_PLANE_Y 0
+#define ES_DVP2AXI_PLANE_CBCR 1
+#define ES_DVP2AXI_MAX_PLANE 3
+
+#define STREAM_PAD_SINK 0
+#define STREAM_PAD_SOURCE 1
+
+#define DVP2AXI_TIMEOUT_FRAME_NUM (2)
+
+#define DVP2AXI_DVP_PCLK_DUAL_EDGE \
+	(V4L2_MBUS_PCLK_SAMPLE_RISING | V4L2_MBUS_PCLK_SAMPLE_FALLING)
+
+/*
+ * Round up height when allocate memory so that EIC770X encoder can
+ * use DMA buffer directly, though this may waste a bit of memory.
+ */
+#define MEMORY_ALIGN_ROUND_UP_HEIGHT 16
+
+/* Get xsubs and ysubs for fourcc formats
+ *
+ * @xsubs: horizontal color samples in a 4*4 matrix, for yuv
+ * @ysubs: vertical color samples in a 4*4 matrix, for yuv
+ */
+
+static inline void DVP2AXI_HalWriteReg(struct es_dvp2axi_hw *dvp2axi_hw, u32 address, u32 data) {
+    writel(data, dvp2axi_hw->base_addr + address);
+	// pr_info("%s: address:0x%x, data:0x%x\n", __func__, address, data);
+}
+
+static inline u32 DVP2AXI_HalReadReg(struct es_dvp2axi_hw *dvp2axi_hw, u32 address) {
+    u32 val;
+    val = readl(dvp2axi_hw->base_addr + address);
+    return val;
+}
+
+static int fcc_xysubs(u32 fcc, u32 *xsubs, u32 *ysubs)
+{
+	switch (fcc) {
+	case V4L2_PIX_FMT_NV16:
+	case V4L2_PIX_FMT_NV61:
+	case V4L2_PIX_FMT_UYVY:
+	case V4L2_PIX_FMT_VYUY:
+	case V4L2_PIX_FMT_YUYV:
+	case V4L2_PIX_FMT_YVYU:
+		*xsubs = 2;
+		*ysubs = 1;
+		break;
+	case V4L2_PIX_FMT_NV21:
+	case V4L2_PIX_FMT_NV12:
+		*xsubs = 2;
+		*ysubs = 2;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct dvp2axi_output_fmt out_fmts[] = {
+	{
+		.fourcc = V4L2_PIX_FMT_NV16,
+		.cplanes = 2,
+		.mplanes = 1,
+		.fmt_val = YUV_OUTPUT_422 | UV_STORAGE_ORDER_UVUV,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_NV61,
+		.fmt_val = YUV_OUTPUT_422 | UV_STORAGE_ORDER_VUVU,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_NV12,
+		.fmt_val = YUV_OUTPUT_420 | UV_STORAGE_ORDER_UVUV,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV420SP,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_NV21,
+		.fmt_val = YUV_OUTPUT_420 | UV_STORAGE_ORDER_VUVU,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV420SP,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_YUYV,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_YVYU,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_Y210,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 10, 20 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_UYVY,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_VYUY,
+		.cplanes = 2,
+		.mplanes = 1,
+		.bpp = { 8, 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_RGB24,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 24 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB888,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_BGR24,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 24 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB888,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_RGB565,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB565,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_BGR666,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 18 },
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SRGGB8,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 8 },
+		.raw_bpp = 8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGRBG8,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 8 },
+		.raw_bpp = 8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGBRG8,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 8 },
+		.raw_bpp = 8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SBGGR8,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 8 },
+		.raw_bpp = 8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SRGGB10,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGRBG10,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGBRG10,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SBGGR10,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SRGGB12,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGRBG12,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGBRG12,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SBGGR12,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SBGGR16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGBRG16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGRBG16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SRGGB16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_Y16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_GREY,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 8 },
+		.raw_bpp = 8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4l2_PIX_FMT_EBD8,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 8 },
+		.raw_bpp = 8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4l2_PIX_FMT_SPD16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_Y12,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_Y10,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SRGGB16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGRBG16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SGBRG16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	},
+	{
+		.fourcc = V4L2_PIX_FMT_SBGGR16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	}
+	/* TODO: We can support NV12M/NV21M/NV16M/NV61M too */
+};
+
+static const struct dvp2axi_input_fmt in_fmts[] = {
+	{
+		.mbus_code = MEDIA_BUS_FMT_YUYV8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_YUYV,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_YUYV,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_YUYV8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_YUYV,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_YUYV,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_INTERLACED,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_YVYU8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_YVYU,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_YVYU,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_YVYU8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_YVYU,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_YVYU,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_INTERLACED,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_UYVY8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_UYVY,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_UYVY,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_UYVY8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_UYVY,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_UYVY,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_INTERLACED,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_VYUY8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_VYUY,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_VYUY,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_VYUY8_2X8,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_VYUY,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_VYUY,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_INTERLACED,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_YVYU10_2X10,
+		.dvp_fmt_val = YUV_INPUT_422 | YUV_INPUT_ORDER_YVYU,
+		.csi_fmt_val = CSI_WRDDR_TYPE_YUV422,
+		.csi_yuv_order = CSI_YUV_INPUT_ORDER_YVYU,
+		.fmt_type = DVP2AXI_FMT_TYPE_YUV,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_RGB888_1X24,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB888,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_BGR888_1X24,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB888,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_GBR888_1X24,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB888,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_RGB565_1X16,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RGB565,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_Y8_1X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_Y10_1X10,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_10,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW10,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_Y12_1X12,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_EBD_1X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_8,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW8,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SPD_2X8,
+		.dvp_fmt_val = INPUT_MODE_RAW | RAW_DATA_WIDTH_12,
+		.csi_fmt_val = CSI_WRDDR_TYPE_RAW12,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+		.field = V4L2_FIELD_NONE,
+	}
+};
+
+// static inline void DVP2AXI_HalWriteReg(struct es_dvp2axi_hw *dvp2axi_hw, u32 address, u32 data) {
+//     writel(data, dvp2axi_hw->base_addr + address);
+// }
+
+// static inline u32 DVP2AXI_HalReadReg(struct es_dvp2axi_hw *dvp2axi_hw, u32 address) {
+//     u32 val;
+//     val = readl(dvp2axi_hw->base_addr + address);
+//     return val;
+// }
+
+
+static int es_dvp2axi_output_fmt_check(struct es_dvp2axi_stream *stream,
+				  const struct dvp2axi_output_fmt *output_fmt)
+{
+	const struct dvp2axi_input_fmt *input_fmt = stream->dvp2axi_fmt_in;
+	struct csi_channel_info *channel =
+		&stream->dvp2axidev->channels[stream->id];
+	int ret = -EINVAL;
+
+	switch (input_fmt->mbus_code) {
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+	case MEDIA_BUS_FMT_VYUY8_2X8:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_NV16 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_NV61 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_NV12 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_NV21 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_YUYV ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_YVYU ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_UYVY ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_VYUY ||
+			output_fmt->fourcc == V4L2_PIX_FMT_Y210)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_YUYV10_2X10:
+	case MEDIA_BUS_FMT_YVYU10_2X10:
+	case MEDIA_BUS_FMT_UYVY10_2X10:
+	case MEDIA_BUS_FMT_VYUY10_2X10:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_Y210)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	case MEDIA_BUS_FMT_Y8_1X8:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB8 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SGRBG8 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SGBRG8 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SBGGR8 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_GREY)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_Y10_1X10:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB10 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SGRBG10 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SGBRG10 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SBGGR10 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_Y10)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+	case MEDIA_BUS_FMT_SGBRG12_1X12:
+	case MEDIA_BUS_FMT_SGRBG12_1X12:
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+	case MEDIA_BUS_FMT_Y12_1X12:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB12 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SGRBG12 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SGBRG12 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_SBGGR12 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_Y12)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_BGR888_1X24:
+	case MEDIA_BUS_FMT_GBR888_1X24:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_RGB24 ||
+		    output_fmt->fourcc == V4L2_PIX_FMT_BGR24)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_RGB565_1X16:
+		if (output_fmt->fourcc == V4L2_PIX_FMT_RGB565)
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_EBD_1X8:
+		if (output_fmt->fourcc == V4l2_PIX_FMT_EBD8 ||
+		    (channel->data_bit == 8 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB8 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG8 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG8 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR8)) ||
+		    (channel->data_bit == 10 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB10 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG10 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG10 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR10)) ||
+		    (channel->data_bit == 12 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB12 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG12 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG12 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR12)) ||
+		    (channel->data_bit == 16 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB16 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG16 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG16 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR16)))
+			ret = 0;
+		break;
+	case MEDIA_BUS_FMT_SPD_2X8:
+		if (output_fmt->fourcc == V4l2_PIX_FMT_SPD16 ||
+		    (channel->data_bit == 8 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB8 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG8 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG8 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR8)) ||
+		    (channel->data_bit == 10 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB10 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG10 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG10 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR10)) ||
+		    (channel->data_bit == 12 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB12 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG12 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG12 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR12)) ||
+		    (channel->data_bit == 16 &&
+		     (output_fmt->fourcc == V4L2_PIX_FMT_SRGGB16 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGRBG16 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SGBRG16 ||
+		      output_fmt->fourcc == V4L2_PIX_FMT_SBGGR16)))
+			ret = 0;
+		break;
+	default:
+		break;
+	}
+	if (ret)
+		v4l2_dbg(4, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			 "input mbus_code 0x%x, can't transform to %c%c%c%c\n",
+			 input_fmt->mbus_code, output_fmt->fourcc & 0xff,
+			 (output_fmt->fourcc >> 8) & 0xff,
+			 (output_fmt->fourcc >> 16) & 0xff,
+			 (output_fmt->fourcc >> 24) & 0xff);
+	return ret;
+}
+
+static int es_dvp2axi_stop_dma_capture(struct es_dvp2axi_stream *stream);
+
+struct es_dvp2axi_rx_buffer *to_dvp2axi_rx_buf(struct esisp_rx_buf *dbufs)
+{
+	return container_of(dbufs, struct es_dvp2axi_rx_buffer, dbufs);
+}
+
+static struct v4l2_subdev *get_remote_sensor(struct es_dvp2axi_stream *stream,
+					     u16 *index)
+{
+	struct media_pad *local, *remote;
+	struct media_entity *sensor_me;
+	struct v4l2_subdev *sub = NULL;
+
+	local = &stream->vnode.vdev.entity.pads[0];
+	if (!local) {
+		v4l2_err(&stream->dvp2axidev->v4l2_dev,
+			 "%s: video pad[0] is null\n", __func__);
+		return NULL;
+	}
+
+	remote = media_pad_remote_pad_first(local);
+	if (!remote) {
+		v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev, "%s: remote pad is null\n",
+			 __func__);
+		return NULL;
+	}
+
+	if (index)
+		*index = remote->index;
+
+	sensor_me = remote->entity;
+
+	sub = media_entity_to_v4l2_subdev(sensor_me);
+	pr_debug("get_remote_sensors sd:%s\n", sub->name);
+
+	return sub;
+}
+
+static void get_remote_terminal_sensor(struct es_dvp2axi_stream *stream,
+				       struct v4l2_subdev **sensor_sd)
+{
+	struct media_graph graph;
+	struct media_entity *entity = &stream->vnode.vdev.entity;
+	struct media_device *mdev = entity->graph_obj.mdev;
+	int ret;
+
+	/* Walk the graph to locate sensor nodes. */
+	mutex_lock(&mdev->graph_mutex);
+	ret = media_graph_walk_init(&graph, mdev);
+	if (ret) {
+		mutex_unlock(&mdev->graph_mutex);
+		*sensor_sd = NULL;
+		return;
+	}
+
+	media_graph_walk_start(&graph, entity);
+	while ((entity = media_graph_walk_next(&graph))) {
+		if (entity->function == MEDIA_ENT_F_CAM_SENSOR)
+			break;
+	}
+	mutex_unlock(&mdev->graph_mutex);
+	media_graph_walk_cleanup(&graph);
+	if (entity){
+			*sensor_sd = media_entity_to_v4l2_subdev(entity);
+			pr_debug("%s %d sd->name :%s\n", __func__, __LINE__, (*sensor_sd)->name);
+	}
+	else
+		*sensor_sd = NULL;
+}
+
+static struct es_dvp2axi_sensor_info *sd_to_sensor(struct es_dvp2axi_device *dev,
+					      struct v4l2_subdev *sd)
+{
+	u32 i;
+
+	for (i = 0; i < dev->num_sensors; ++i)
+		if (dev->sensors[i].sd == sd)
+			return &dev->sensors[i];
+
+	if (i == dev->num_sensors) {
+		for (i = 0; i < dev->num_sensors; ++i) {
+			if (dev->sensors[i].mbus.type == V4L2_MBUS_CCP2)
+				return &dev->lvds_subdev.sensor_self;
+		}
+	}
+
+	return NULL;
+}
+
+static unsigned char get_data_type(u32 pixelformat, u8 cmd_mode_en,
+				   u8 dsi_input)
+{
+	switch (pixelformat) {
+	/* csi raw8 */
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	case MEDIA_BUS_FMT_Y8_1X8:
+		return 0x2a;
+	/* csi raw10 */
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_Y10_1X10:
+		return 0x2b;
+	/* csi raw12 */
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+	case MEDIA_BUS_FMT_SGBRG12_1X12:
+	case MEDIA_BUS_FMT_SGRBG12_1X12:
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+	case MEDIA_BUS_FMT_Y12_1X12:
+		return 0x2c;
+	/* csi uyvy 422 */
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+	case MEDIA_BUS_FMT_VYUY8_2X8:
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+		return 0x1e;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_BGR888_1X24:
+	case MEDIA_BUS_FMT_GBR888_1X24:
+		if (dsi_input) {
+			if (cmd_mode_en) /* dsi command mode*/
+				return 0x39;
+			else /* dsi video mode */
+				return 0x3e;
+		} else {
+			return 0x24;
+		}
+	case MEDIA_BUS_FMT_RGB565_1X16:
+		if (dsi_input) {
+			if (cmd_mode_en) /* dsi command mode*/
+				return 0x39;
+			else /* dsi video mode */
+				return 0x0e;
+		} else {
+			return 0x22;
+		}
+	case MEDIA_BUS_FMT_EBD_1X8:
+		return 0x12;
+	case MEDIA_BUS_FMT_SPD_2X8:
+		return 0x2f;
+
+	default:
+		return 0x2b;
+	}
+}
+
+static int get_csi_crop_align(const struct dvp2axi_input_fmt *fmt_in)
+{
+	switch (fmt_in->csi_fmt_val) {
+	case CSI_WRDDR_TYPE_RGB888:
+		return 24;
+	case CSI_WRDDR_TYPE_RGB565:
+		return 16;
+	case CSI_WRDDR_TYPE_RAW10:
+	case CSI_WRDDR_TYPE_RAW12:
+		return 4;
+	case CSI_WRDDR_TYPE_RAW8:
+	case CSI_WRDDR_TYPE_YUV422:
+		return 8;
+	default:
+		return -1;
+	}
+}
+
+const struct dvp2axi_input_fmt *
+es_dvp2axi_get_input_fmt(struct es_dvp2axi_device *dev, struct v4l2_rect *rect,
+		    u32 pad_id, struct csi_channel_info *csi_info)
+{
+	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev *sd = dev->terminal_sensor.sd;
+	struct esmodule_channel_info ch_info = { 0 };
+	struct esmodule_capture_info capture_info;
+	int ret;
+	u32 i;
+
+	fmt.pad = 0;
+	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	fmt.reserved[0] = 0;
+	fmt.format.field = V4L2_FIELD_NONE;
+	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
+	if (ret < 0) {
+		v4l2_warn(sd->v4l2_dev,
+			  "sensor fmt invalid, set to default size\n");
+		goto set_default;
+	}
+	ch_info.index = pad_id;
+	ret = v4l2_subdev_call(sd, core, ioctl, ESMODULE_GET_CHANNEL_INFO,
+			       &ch_info);
+	if (!ret) {
+		fmt.format.width = ch_info.width;
+		fmt.format.height = ch_info.height;
+		fmt.format.code = ch_info.bus_fmt;
+		if (ch_info.vc >= 0 && ch_info.vc <= 3)
+			csi_info->vc = ch_info.vc;
+		else
+			csi_info->vc = 0xff;
+		if (ch_info.bus_fmt == MEDIA_BUS_FMT_SPD_2X8 ||
+		    ch_info.bus_fmt == MEDIA_BUS_FMT_EBD_1X8) {
+			if (ch_info.data_type > 0)
+				csi_info->data_type = ch_info.data_type;
+			if (ch_info.data_bit > 0)
+				csi_info->data_bit = ch_info.data_bit;
+		}
+	} else {
+		csi_info->vc = 0xff;
+	}
+
+	v4l2_dbg(1, es_dvp2axi_debug, sd->v4l2_dev,
+		 "remote fmt: mbus code:0x%x, size:%dx%d, field: %d\n",
+		 fmt.format.code, fmt.format.width, fmt.format.height,
+		 fmt.format.field);
+	rect->left = 0;
+	rect->top = 0;
+	rect->width = fmt.format.width;
+	rect->height = fmt.format.height;
+	ret = v4l2_subdev_call(sd, core, ioctl, ESMODULE_GET_CAPTURE_MODE,
+			       &capture_info);
+	if (!ret) {
+		if (capture_info.mode == ESMODULE_MULTI_DEV_COMBINE_ONE &&
+		    dev->hw_dev->is_eic770xs2) {
+			for (i = 0; i < capture_info.multi_dev.dev_num; i++) {
+				if (capture_info.multi_dev.dev_idx[i] == 0)
+					capture_info.multi_dev.dev_idx[i] = 2;
+				else if (capture_info.multi_dev.dev_idx[i] == 2)
+					capture_info.multi_dev.dev_idx[i] = 4;
+				else if (capture_info.multi_dev.dev_idx[i] == 3)
+					capture_info.multi_dev.dev_idx[i] = 5;
+			}
+		}
+		csi_info->capture_info = capture_info;
+	} else {
+		csi_info->capture_info.mode = ESMODULE_CAPTURE_MODE_NONE;
+	}
+	for (i = 0; i < ARRAY_SIZE(in_fmts); i++)
+		if (fmt.format.code == in_fmts[i].mbus_code &&
+		    fmt.format.field == in_fmts[i].field)
+			return &in_fmts[i];
+
+	v4l2_err(sd->v4l2_dev, "remote sensor mbus code not supported\n");
+
+set_default:
+	rect->left = 0;
+	rect->top = 0;
+	rect->width = ES_DVP2AXI_DEFAULT_WIDTH;
+	rect->height = ES_DVP2AXI_DEFAULT_HEIGHT;
+
+	return NULL;
+}
+
+const struct dvp2axi_output_fmt *es_dvp2axi_find_output_fmt(struct es_dvp2axi_stream *stream,
+						   u32 pixelfmt)
+{
+	const struct dvp2axi_output_fmt *fmt;
+	u32 i;
+
+	pr_debug("%s pixelfmt =0x%x\n", __func__, pixelfmt);
+	for (i = 0; i < ARRAY_SIZE(out_fmts); i++) {
+		fmt = &out_fmts[i];
+		if (fmt->fourcc == pixelfmt)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_id_ctrl0(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_ID1_CTRL0;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_ID2_CTRL0;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_ID3_CTRL0;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0;
+		break;
+	}
+
+	return index;
+}
+
+#if 0
+static enum dvp2axi_reg_index get_reg_index_of_lvds_id_ctrl0(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_LVDS_ID0_CTRL0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_LVDS_ID1_CTRL0;
+		break;
+	case 2:
+		index = DVP2AXI_REG_LVDS_ID2_CTRL0;
+		break;
+	case 3:
+		index = DVP2AXI_REG_LVDS_ID3_CTRL0;
+		break;
+	default:
+		index = DVP2AXI_REG_LVDS_ID0_CTRL0;
+		break;
+	}
+
+	return index;
+}
+#endif
+
+static enum dvp2axi_reg_index get_reg_index_of_id_ctrl1(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_ID0_CTRL1;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_ID1_CTRL1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_ID2_CTRL1;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_ID3_CTRL1;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_ID0_CTRL1;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm0_y_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm1_y_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm0_uv_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm1_uv_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm0_y_vlw(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm1_y_vlw(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm0_uv_vlw(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_frm1_uv_vlw(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_id_crop_start(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_MIPI_LVDS_ID0_CROP_START;
+		break;
+	case 1:
+		index = DVP2AXI_REG_MIPI_LVDS_ID1_CROP_START;
+		break;
+	case 2:
+		index = DVP2AXI_REG_MIPI_LVDS_ID2_CROP_START;
+		break;
+	case 3:
+		index = DVP2AXI_REG_MIPI_LVDS_ID3_CROP_START;
+		break;
+	default:
+		index = DVP2AXI_REG_MIPI_LVDS_ID0_CROP_START;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_lvds_sav_eav_act0(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_lvds_sav_eav_act1(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_lvds_sav_eav_blk0(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_lvds_sav_eav_blk1(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_dvp_reg_index_of_frm0_y_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_Y;
+		break;
+	case 1:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_Y;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_dvp_reg_index_of_frm1_y_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_Y;
+		break;
+	case 1:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_Y;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_dvp_reg_index_of_frm0_uv_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_UV;
+		break;
+	case 1:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_DVP_FRM0_ADDR_UV;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_dvp_reg_index_of_frm1_uv_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_UV;
+		break;
+	case 1:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID3;
+		break;
+	default:
+		index = DVP2AXI_REG_DVP_FRM1_ADDR_UV;
+		break;
+	}
+
+	return index;
+}
+
+int es_dvp2axi_get_linetime(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor = &dvp2axi_dev->terminal_sensor;
+	u32 numerator, denominator;
+	u32 def_fps = 0;
+	int line_time = 0;
+	int vblank_def = 0;
+	int vblank_curr = 0;
+
+	numerator = sensor->fi.interval.numerator;
+	denominator = sensor->fi.interval.denominator;
+	if (!numerator || !denominator) {
+		v4l2_err(
+			&dvp2axi_dev->v4l2_dev,
+			"get frame interval fail, numerator %d, denominator %d\n",
+			numerator, denominator);
+		return -EINVAL;
+	}
+	def_fps = denominator / numerator;
+	if (!def_fps) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev,
+			 "get fps fail, numerator %d, denominator %d\n",
+			 numerator, denominator);
+		return -EINVAL;
+	}
+	vblank_def = es_dvp2axi_get_sensor_vblank_def(dvp2axi_dev);
+	vblank_curr = es_dvp2axi_get_sensor_vblank(dvp2axi_dev);
+	if (!vblank_def || !vblank_curr) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev,
+			 "get vblank fail, vblank_def %d, vblank_curr %d\n",
+			 vblank_def, vblank_curr);
+		return -EINVAL;
+	}
+	line_time = div_u64(1000000000, def_fps);
+	line_time = div_u64(line_time, vblank_def + sensor->raw_rect.height);
+	return line_time;
+}
+
+/***************************** stream operations ******************************/
+static int es_dvp2axi_assign_new_buffer_oneframe(struct es_dvp2axi_stream *stream,
+					    enum es_dvp2axi_yuvaddr_state stat)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	struct es_dvp2axi_buffer *buffer = NULL;
+	// u32 frm_addr_y = DVP2AXI_REG_DVP_FRM0_ADDR_Y;
+	// u32 frm_addr_uv = DVP2AXI_REG_DVP_FRM0_ADDR_UV;
+	unsigned long flags;
+	int ret = 0;
+	int hdr_id;
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (stat == ES_DVP2AXI_YUV_ADDR_STATE_INIT) {
+		
+		if (!stream->curr_buf) {
+			if (!list_empty(&stream->buf_head)) {
+				stream->curr_buf = list_first_entry(
+					&stream->buf_head, struct es_dvp2axi_buffer,
+					queue);
+				list_del(&stream->curr_buf->queue);
+			}
+		}
+
+		if (stream->curr_buf) {
+			DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL9_CSR + stream->id * 0xc, stream->curr_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+		} else {
+			if (dummy_buf->vaddr) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL9_CSR + stream->id * 0xc, dummy_buf->dma_addr);
+			}
+		}
+
+		if(dev->hdr.hdr_mode == HDR_X2) {
+			hdr_id = DVP2AXI_HalReadReg(dev->hw_dev, VI_DVP2AXI_CTRL36_CSR);
+			hdr_id |= 0x1 << (VI_DVP2AXI_DVP0_LAST_ID_OFFSET + stream->id * 2);
+			DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL36_CSR, 0);
+			DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL36_CSR, 0);
+			if (!stream->next_buf) {
+				if (!list_empty(&stream->buf_head)) {
+					stream->next_buf = list_first_entry(
+						&stream->buf_head, struct es_dvp2axi_buffer,
+						queue);
+					list_del(&stream->next_buf->queue);
+				}
+			}
+			if (stream->next_buf) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL10_CSR + stream->id * 0xc, stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			} else {
+				if (dummy_buf->vaddr) {
+					DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL10_CSR+stream->id * 0xc, dummy_buf->dma_addr);
+				}
+			}
+		} else if(dev->hdr.hdr_mode == HDR_X3) {
+			hdr_id = DVP2AXI_HalReadReg(dev->hw_dev, VI_DVP2AXI_CTRL36_CSR);
+			hdr_id |= 0x2 << (VI_DVP2AXI_DVP0_LAST_ID_OFFSET + stream->id * 2);
+			if (!stream->next_buf) {
+				if (!list_empty(&stream->buf_head)) {
+					stream->next_buf = list_first_entry(
+						&stream->buf_head, struct es_dvp2axi_buffer,
+						queue);
+					list_del(&stream->next_buf->queue);
+				}
+			}
+			if (stream->next_buf) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL10_CSR+stream->id * 0xc, stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			} else {
+				if (dummy_buf->vaddr) {
+					DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL10_CSR+stream->id * 0xc, dummy_buf->dma_addr);
+				}
+			}
+
+			if (!stream->last_buf) {
+				if (!list_empty(&stream->buf_head)) {
+					stream->last_buf = list_first_entry(
+						&stream->buf_head, struct es_dvp2axi_buffer,
+						queue);
+					list_del(&stream->last_buf->queue);
+				}
+			}
+			if (stream->last_buf) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL11_CSR+stream->id * 0xc, stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			} else {
+				if (dummy_buf->vaddr) {
+					DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL11_CSR+stream->id * 0xc, dummy_buf->dma_addr);
+				}
+			}
+
+		}
+	} else if (stat == ES_DVP2AXI_YUV_ADDR_STATE_UPDATE) {
+		if (!list_empty(&stream->buf_head)) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				stream->curr_buf = list_first_entry(
+					&stream->buf_head, struct es_dvp2axi_buffer,
+					queue);
+				list_del(&stream->curr_buf->queue);
+				buffer = stream->curr_buf;
+			} else if (stream->frame_phase ==
+				   DVP2AXI_CSI_FRAME1_READY) {
+				stream->next_buf = list_first_entry(
+					&stream->buf_head, struct es_dvp2axi_buffer,
+					queue);
+				list_del(&stream->next_buf->queue);
+				buffer = stream->next_buf;
+			} else if(stream->frame_phase == DVP2AXI_CSI_FRAME2_READY) {
+				stream->last_buf = list_first_entry(
+					&stream->buf_head, struct es_dvp2axi_buffer,
+					queue);
+				list_del(&stream->last_buf->queue);
+				buffer = stream->last_buf;
+			}
+		} else {
+			buffer = NULL;
+		}
+		
+
+		if (buffer) {
+			if(stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL9_CSR+stream->id * 0xc, buffer->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			} else if(stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL10_CSR+stream->id * 0xc, buffer->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			} else if(stream->frame_phase == DVP2AXI_CSI_FRAME2_READY) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL11_CSR+stream->id * 0xc, buffer->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			}
+		} else {
+			// ret = -EINVAL;
+			if(stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				stream->curr_buf = NULL;
+			}
+			if(stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+				stream->next_buf = NULL;
+			}
+			if(stream->frame_phase == DVP2AXI_CSI_FRAME2_READY) {
+				stream->last_buf = NULL;
+			}
+			if(dummy_buf->vaddr) {
+				DVP2AXI_HalWriteReg(dev->hw_dev, VI_DVP2AXI_CTRL11_CSR+stream->id * 0xc, dummy_buf->dma_addr);
+			}
+			dev->err_state |= (ES_DVP2AXI_ERR_ID0_NOT_BUF << stream->id);
+			dev->irq_stats.not_active_buf_cnt[stream->id]++;
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	return ret;
+}
+
+static struct v4l2_subdev *get_esisp_sd(struct sditf_priv *priv)
+{
+	struct media_pad *pad = NULL;
+
+	if (priv && priv->pads[0].entity->num_links) {
+		if (priv->is_combine_mode)
+			pad = media_pad_remote_pad_first(&priv->pads[1]);
+		else
+			pad = media_pad_remote_pad_first(&priv->pads[0]);
+		if (pad)
+			return media_entity_to_v4l2_subdev(pad->entity);
+	}
+	return NULL;
+}
+
+static void es_dvp2axi_rx_buffer_free(struct es_dvp2axi_stream *stream)
+{
+	struct v4l2_subdev *sd;
+	struct esisp_rx_buf *dbufs;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+
+	sd = get_esisp_sd(dev->sditf[0]);
+	if (!sd)
+		return;
+
+	while (!list_empty(&stream->rx_buf_head_vicap)) {
+		dbufs = list_first_entry(&stream->rx_buf_head_vicap,
+					 struct esisp_rx_buf, list);
+		if (dbufs->is_init)
+			v4l2_subdev_call(sd, core, ioctl,
+					 ESISP_VICAP_CMD_RX_BUFFER_FREE, dbufs);
+		dma_buf_put(dbufs->dbuf);
+		list_del(&dbufs->list);
+		kfree(dbufs);
+	}
+}
+
+static void es_dvp2axi_s_rx_buffer(struct es_dvp2axi_stream *stream,
+			      struct esisp_rx_buf *dbufs)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_subdev *sd;
+	struct es_dvp2axi_rx_buffer *rx_buf = NULL;
+
+	sd = get_esisp_sd(dev->sditf[0]);
+	if (!sd)
+		return;
+	if ((dev->rdbk_debug && dbufs->sequence < 15) || es_dvp2axi_debug == 3) {
+		rx_buf = to_dvp2axi_rx_buf(dbufs);
+		v4l2_info(&dev->v4l2_dev,
+			  "s_buf seq %d type %d, dma addr %x, %lld\n",
+			  dbufs->sequence, dbufs->type,
+			  (u32)rx_buf->dummy.dma_addr, es_dvp2axi_time_get_ns(dev));
+	}
+	v4l2_subdev_call(sd, video, s_rx_buffer, dbufs, NULL);
+}
+
+#if 0
+static void es_dvp2axi_enable_skip_frame(struct es_dvp2axi_stream *stream, int cap_m,
+				    int skip_n)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	u32 val = 0;
+
+	val = es_dvp2axi_read_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL);
+	val &= 0xc00fffff;
+	val |= cap_m << ES_DVP2AXI_CAP_SHIFT | skip_n << ES_DVP2AXI_SKIP_SHIFT |
+	       ES_DVP2AXI_SKIP_EN(stream->id);
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL, val);
+	stream->skip_info.skip_en = true;
+}
+#endif
+
+#if 0
+static void es_dvp2axi_disable_skip_frame(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	u32 val = 0;
+
+	val = es_dvp2axi_read_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL);
+	val &= ~(ES_DVP2AXI_SKIP_EN(stream->id));
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL, val);
+	stream->skip_info.skip_en = false;
+}
+#endif
+
+static void es_dvp2axi_rdbk_with_tools(struct es_dvp2axi_stream *stream,
+				  struct es_dvp2axi_rx_buffer *active_buf)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&stream->tools_vdev->vbq_lock, flags);
+	if (stream->tools_vdev->state == ES_DVP2AXI_STATE_STREAMING) {
+		list_add_tail(&active_buf->list,
+			      &stream->tools_vdev->buf_done_head);
+		if (!work_busy(&stream->tools_vdev->work))
+			schedule_work(&stream->tools_vdev->work);
+	}
+	spin_unlock_irqrestore(&stream->tools_vdev->vbq_lock, flags);
+}
+
+static void es_dvp2axi_rdbk_frame_end_toisp(struct es_dvp2axi_stream *stream,
+				       struct es_dvp2axi_rx_buffer *buffer)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor = &stream->dvp2axidev->terminal_sensor;
+	u32 denominator, numerator;
+	u64 l_ts, m_ts, s_ts, time = 30000000LL;
+	int ret, fps = -1;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->hdr_lock, flags);
+	if (dev->rdbk_rx_buf[stream->id]) {
+		list_add_tail(&dev->rdbk_rx_buf[stream->id]->list,
+			      &stream->rx_buf_head);
+		dev->rdbk_rx_buf[stream->id] = buffer;
+	} else {
+		dev->rdbk_rx_buf[stream->id] = buffer;
+	}
+
+	numerator = sensor->fi.interval.numerator;
+	denominator = sensor->fi.interval.denominator;
+	if (denominator && numerator)
+		time = numerator * 1000 / denominator * 1000 * 1000;
+
+	if (dev->hdr.hdr_mode == HDR_X3 && dev->rdbk_rx_buf[RDBK_L] &&
+	    dev->rdbk_rx_buf[RDBK_M] && dev->rdbk_rx_buf[RDBK_S]) {
+		l_ts = dev->rdbk_rx_buf[RDBK_L]->fe_timestamp;
+		m_ts = dev->rdbk_rx_buf[RDBK_M]->fe_timestamp;
+		s_ts = dev->rdbk_rx_buf[RDBK_S]->fe_timestamp;
+
+		if (m_ts < l_ts || s_ts < m_ts) {
+			v4l2_err(
+				&dev->v4l2_dev,
+				"s/m/l frame err, timestamp s:%lld m:%lld l:%lld\n",
+				s_ts, m_ts, l_ts);
+			goto RDBK_TOISP_UNMATCH;
+		}
+
+		if ((m_ts - l_ts) > time || (s_ts - m_ts) > time) {
+			ret = v4l2_subdev_call(sensor->sd, video,
+					       g_frame_interval, &sensor->fi);
+			if (!ret) {
+				denominator = sensor->fi.interval.denominator;
+				numerator = sensor->fi.interval.numerator;
+				if (denominator && numerator) {
+					time = numerator * 1000 / denominator *
+					       1000 * 1000;
+					fps = denominator / numerator;
+				}
+			}
+
+			if ((m_ts - l_ts) > time || (s_ts - m_ts) > time) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"timestamp no match, s:%lld m:%lld l:%lld, fps:%d\n",
+					s_ts, m_ts, l_ts, fps);
+				goto RDBK_TOISP_UNMATCH;
+			}
+		}
+		dev->rdbk_rx_buf[RDBK_M]->dbufs.sequence =
+			dev->rdbk_rx_buf[RDBK_L]->dbufs.sequence;
+		dev->rdbk_rx_buf[RDBK_S]->dbufs.sequence =
+			dev->rdbk_rx_buf[RDBK_L]->dbufs.sequence;
+		es_dvp2axi_s_rx_buffer(&dev->stream[RDBK_L],
+				  &dev->rdbk_rx_buf[RDBK_L]->dbufs);
+		es_dvp2axi_s_rx_buffer(&dev->stream[RDBK_M],
+				  &dev->rdbk_rx_buf[RDBK_M]->dbufs);
+		es_dvp2axi_s_rx_buffer(&dev->stream[RDBK_S],
+				  &dev->rdbk_rx_buf[RDBK_S]->dbufs);
+		es_dvp2axi_rdbk_with_tools(&dev->stream[RDBK_L],
+				      dev->rdbk_rx_buf[RDBK_L]);
+		es_dvp2axi_rdbk_with_tools(&dev->stream[RDBK_M],
+				      dev->rdbk_rx_buf[RDBK_M]);
+		es_dvp2axi_rdbk_with_tools(&dev->stream[RDBK_S],
+				      dev->rdbk_rx_buf[RDBK_S]);
+		atomic_dec(&dev->stream[RDBK_L].buf_cnt);
+		atomic_dec(&dev->stream[RDBK_M].buf_cnt);
+		atomic_dec(&dev->stream[RDBK_S].buf_cnt);
+		dev->rdbk_rx_buf[RDBK_L] = NULL;
+		dev->rdbk_rx_buf[RDBK_M] = NULL;
+		dev->rdbk_rx_buf[RDBK_S] = NULL;
+	} else if (dev->hdr.hdr_mode == HDR_X2 && dev->rdbk_rx_buf[RDBK_L] &&
+		   dev->rdbk_rx_buf[RDBK_M]) {
+		l_ts = dev->rdbk_rx_buf[RDBK_L]->fe_timestamp;
+		s_ts = dev->rdbk_rx_buf[RDBK_M]->fe_timestamp;
+
+		if (s_ts < l_ts) {
+			v4l2_err(&dev->v4l2_dev,
+				 "s/l frame err, timestamp s:%lld l:%lld\n",
+				 s_ts, l_ts);
+			goto RDBK_TOISP_UNMATCH;
+		}
+
+		if ((s_ts - l_ts) > time) {
+			ret = v4l2_subdev_call(sensor->sd, video,
+					       g_frame_interval, &sensor->fi);
+			if (!ret) {
+				denominator = sensor->fi.interval.denominator;
+				numerator = sensor->fi.interval.numerator;
+				if (denominator && numerator) {
+					time = numerator * 1000 / denominator *
+					       1000 * 1000;
+					fps = denominator / numerator;
+				}
+			}
+			if ((s_ts - l_ts) > time) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"timestamp no match, s:%lld l:%lld, fps:%d\n",
+					s_ts, l_ts, fps);
+				goto RDBK_TOISP_UNMATCH;
+			}
+		}
+		dev->rdbk_rx_buf[RDBK_M]->dbufs.sequence =
+			dev->rdbk_rx_buf[RDBK_L]->dbufs.sequence;
+		es_dvp2axi_s_rx_buffer(&dev->stream[RDBK_L],
+				  &dev->rdbk_rx_buf[RDBK_L]->dbufs);
+		es_dvp2axi_s_rx_buffer(&dev->stream[RDBK_M],
+				  &dev->rdbk_rx_buf[RDBK_M]->dbufs);
+		es_dvp2axi_rdbk_with_tools(&dev->stream[RDBK_L],
+				      dev->rdbk_rx_buf[RDBK_L]);
+		es_dvp2axi_rdbk_with_tools(&dev->stream[RDBK_M],
+				      dev->rdbk_rx_buf[RDBK_M]);
+		atomic_dec(&dev->stream[RDBK_L].buf_cnt);
+		atomic_dec(&dev->stream[RDBK_M].buf_cnt);
+		dev->rdbk_rx_buf[RDBK_L] = NULL;
+		dev->rdbk_rx_buf[RDBK_M] = NULL;
+	}
+
+	spin_unlock_irqrestore(&dev->hdr_lock, flags);
+	return;
+
+RDBK_TOISP_UNMATCH:
+	spin_unlock_irqrestore(&dev->hdr_lock, flags);
+}
+
+static void es_dvp2axi_write_buff_addr_multi_dev_combine(
+	struct es_dvp2axi_stream *stream, u32 frm_addr_y, u32 frm_addr_uv,
+	u32 buff_addr_y, u32 buff_addr_cbcr, bool is_dummy_buf)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct esmodule_capture_info *capture_info =
+		&dev->channels[stream->id].capture_info;
+	u32 addr_y, addr_cbcr;
+	int addr_offset = 0;
+	int i = 0;
+	int tmp_host_index = dev->csi_host_idx;
+
+	for (i = 0; i < capture_info->multi_dev.dev_num; i++) {
+		if (is_dummy_buf) {
+			addr_y = buff_addr_y;
+		} else {
+			addr_offset =
+				dev->channels[stream->id].left_virtual_width;
+			addr_y = buff_addr_y + addr_offset * i;
+		}
+		dev->csi_host_idx = capture_info->multi_dev.dev_idx[i];
+		es_dvp2axi_write_register(dev, frm_addr_y, addr_y);
+		if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW &&
+		    frm_addr_uv && buff_addr_cbcr) {
+			if (is_dummy_buf) {
+				addr_cbcr = buff_addr_cbcr;
+			} else {
+				addr_offset = dev->channels[stream->id]
+						      .left_virtual_width;
+				addr_cbcr = buff_addr_cbcr + addr_offset * i;
+			}
+			es_dvp2axi_write_register(dev, frm_addr_uv, addr_cbcr);
+		}
+	}
+	dev->csi_host_idx = tmp_host_index;
+}
+
+static void es_dvp2axi_assign_new_buffer_init_toisp(struct es_dvp2axi_stream *stream,
+					       int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_rx_buffer *rx_buf;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct esmodule_capture_info *capture_info =
+		&dev->channels[channel_id].capture_info;
+	u32 frm0_addr_y;
+	u32 frm1_addr_y;
+	u32 buff_addr_y;
+	unsigned long flags;
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm0_addr_y = get_reg_index_of_frm0_y_addr(channel_id);
+		frm1_addr_y = get_reg_index_of_frm1_y_addr(channel_id);
+	} else {
+		frm0_addr_y = get_dvp_reg_index_of_frm0_y_addr(channel_id);
+		frm1_addr_y = get_dvp_reg_index_of_frm1_y_addr(channel_id);
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+
+	if (!stream->curr_buf_toisp) {
+		if (!list_empty(&stream->rx_buf_head)) {
+			rx_buf = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (rx_buf) {
+				list_del(&rx_buf->list);
+				stream->curr_buf_toisp = rx_buf;
+			}
+		}
+	}
+
+	if (stream->curr_buf_toisp) {
+		buff_addr_y = stream->curr_buf_toisp->dummy.dma_addr;
+		if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm0_addr_y, 0, buff_addr_y, 0, false);
+		} else {
+			es_dvp2axi_write_register(dev, frm0_addr_y, buff_addr_y);
+		}
+	} else {
+		if (stream->lack_buf_cnt < 2)
+			stream->lack_buf_cnt++;
+	}
+
+	if (!stream->next_buf_toisp) {
+		if (!list_empty(&stream->rx_buf_head)) {
+			rx_buf = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (rx_buf) {
+				list_del(&rx_buf->list);
+				stream->next_buf_toisp = rx_buf;
+			} else {
+				stream->next_buf_toisp = stream->curr_buf_toisp;
+			}
+		} else {
+			stream->next_buf_toisp = stream->curr_buf_toisp;
+		}
+	}
+
+	if (stream->next_buf_toisp) {
+		buff_addr_y = stream->next_buf_toisp->dummy.dma_addr;
+		if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm1_addr_y, 0, buff_addr_y, 0, false);
+		} else {
+			es_dvp2axi_write_register(dev, frm1_addr_y, buff_addr_y);
+		}
+	} else {
+		if (stream->lack_buf_cnt < 2)
+			stream->lack_buf_cnt++;
+	}
+
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	stream->buf_owner = ES_DVP2AXI_DMAEN_BY_ISP;
+}
+
+static void es_dvp2axi_dphy_quick_stream(struct es_dvp2axi_device *dev, int on)
+{
+	struct es_dvp2axi_pipeline *p = NULL;
+	int j = 0;
+	if (dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+	    dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+	    dev->active_sensor->mbus.type == V4L2_MBUS_CCP2) {
+		p = &dev->pipe;
+		for (j = 0; j < p->num_subdevs; j++) {
+			if (p->subdevs[j] != dev->terminal_sensor.sd &&
+			    p->subdevs[j] != dev->active_sensor->sd) {
+				v4l2_subdev_call(p->subdevs[j], core, ioctl,
+						 ESMODULE_SET_QUICK_STREAM,
+						 &on);
+				break;
+			}
+		}
+	}
+}
+
+static int es_dvp2axi_assign_new_buffer_update_toisp(struct es_dvp2axi_stream *stream,
+						int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct esmodule_capture_info *capture_info =
+		&dev->channels[channel_id].capture_info;
+	struct es_dvp2axi_rx_buffer *buffer = NULL;
+	struct es_dvp2axi_rx_buffer *active_buf = NULL;
+	struct sditf_priv *priv = dev->sditf[0];
+	u32 frm_addr_y, buff_addr_y;
+	unsigned long flags;
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm_addr_y = stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				     get_reg_index_of_frm0_y_addr(channel_id) :
+				     get_reg_index_of_frm1_y_addr(channel_id);
+	} else {
+		frm_addr_y =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_y_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_y_addr(channel_id);
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (stream->cur_skip_frame)
+		goto out_get_buf;
+	memset(&stream->toisp_buf_state, 0, sizeof(stream->toisp_buf_state));
+	if (!list_empty(&stream->rx_buf_head)) {
+		if (stream->curr_buf_toisp && stream->next_buf_toisp &&
+		    stream->curr_buf_toisp != stream->next_buf_toisp)
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_ROTATE;
+		else
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_LOSS;
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+			active_buf = stream->curr_buf_toisp;
+
+			buffer = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (buffer) {
+				list_del(&buffer->list);
+				stream->curr_buf_toisp = buffer;
+			}
+			if (priv &&
+			    priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO) {
+				if (!active_buf)
+					goto out_get_buf;
+				if (stream->frame_idx == 1)
+					active_buf->dbufs.is_first = true;
+				active_buf->dbufs.sequence =
+					stream->frame_idx - 1;
+				active_buf->dbufs.timestamp =
+					stream->readout.fs_timestamp;
+				active_buf->fe_timestamp =
+					es_dvp2axi_time_get_ns(dev);
+				stream->last_frame_idx = stream->frame_idx;
+				if (dev->hdr.hdr_mode == NO_HDR) {
+					es_dvp2axi_s_rx_buffer(stream,
+							  &active_buf->dbufs);
+					if (dev->is_support_tools &&
+					    stream->tools_vdev)
+						es_dvp2axi_rdbk_with_tools(
+							stream, active_buf);
+					atomic_dec(&stream->buf_cnt);
+				} else {
+					es_dvp2axi_rdbk_frame_end_toisp(stream,
+								   active_buf);
+				}
+			} else {
+				if (active_buf)
+					es_dvp2axi_s_rx_buffer(stream,
+							  &active_buf->dbufs);
+				if (dev->is_support_tools && stream->tools_vdev)
+					es_dvp2axi_rdbk_with_tools(stream,
+							      active_buf);
+			}
+		} else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			active_buf = stream->next_buf_toisp;
+			buffer = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (buffer) {
+				list_del(&buffer->list);
+				stream->next_buf_toisp = buffer;
+			}
+			if (priv &&
+			    priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO) {
+				if (!active_buf)
+					goto out_get_buf;
+				if (stream->frame_idx == 1)
+					active_buf->dbufs.is_first = true;
+				active_buf->dbufs.sequence =
+					stream->frame_idx - 1;
+				active_buf->dbufs.timestamp =
+					stream->readout.fs_timestamp;
+				active_buf->fe_timestamp =
+					es_dvp2axi_time_get_ns(dev);
+				stream->last_frame_idx = stream->frame_idx;
+				if (dev->hdr.hdr_mode == NO_HDR) {
+					es_dvp2axi_s_rx_buffer(stream,
+							  &active_buf->dbufs);
+					if (dev->is_support_tools &&
+					    stream->tools_vdev)
+						es_dvp2axi_rdbk_with_tools(
+							stream, active_buf);
+					atomic_dec(&stream->buf_cnt);
+				} else {
+					es_dvp2axi_rdbk_frame_end_toisp(stream,
+								   active_buf);
+				}
+			} else {
+				if (active_buf)
+					es_dvp2axi_s_rx_buffer(stream,
+							  &active_buf->dbufs);
+				if (dev->is_support_tools && stream->tools_vdev)
+					es_dvp2axi_rdbk_with_tools(stream,
+							      active_buf);
+			}
+		}
+		if (stream->lack_buf_cnt)
+			stream->lack_buf_cnt--;
+	} else {
+		if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE)
+			goto out_get_buf;
+		if (stream->lack_buf_cnt < 2)
+			stream->lack_buf_cnt++;
+		if (dev->hw_dev->dummy_buf.vaddr) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				active_buf = stream->curr_buf_toisp;
+			} else {
+				active_buf = stream->next_buf_toisp;
+			}
+		} else if (stream->curr_buf_toisp && stream->next_buf_toisp &&
+			   stream->curr_buf_toisp != stream->next_buf_toisp) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				active_buf = stream->curr_buf_toisp;
+				stream->curr_buf_toisp = stream->next_buf_toisp;
+				buffer = stream->next_buf_toisp;
+			} else if (stream->frame_phase ==
+				   DVP2AXI_CSI_FRAME1_READY) {
+				active_buf = stream->next_buf_toisp;
+				stream->next_buf_toisp = stream->curr_buf_toisp;
+				buffer = stream->curr_buf_toisp;
+			}
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_THESAME;
+			if (stream->dvp2axidev->rdbk_debug)
+				v4l2_info(&stream->dvp2axidev->v4l2_dev,
+					  "stream[%d] hold buf %x\n",
+					  stream->id,
+					  (u32)stream->next_buf_toisp->dummy
+						  .dma_addr);
+		} else {
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_LOSS;
+			if (stream->is_single_cap) {
+				active_buf = stream->curr_buf_toisp;
+				stream->curr_buf_toisp = NULL;
+				stream->next_buf_toisp = NULL;
+			}
+		}
+
+		if (active_buf) {
+			if (stream->frame_idx == 1)
+				active_buf->dbufs.is_first = true;
+			active_buf->dbufs.sequence = stream->frame_idx - 1;
+			active_buf->dbufs.timestamp =
+				stream->readout.fs_timestamp;
+			active_buf->fe_timestamp = es_dvp2axi_time_get_ns(dev);
+			stream->last_frame_idx = stream->frame_idx;
+			if (dev->hdr.hdr_mode == NO_HDR) {
+				es_dvp2axi_s_rx_buffer(stream, &active_buf->dbufs);
+				atomic_dec(&stream->buf_cnt);
+			} else {
+				es_dvp2axi_rdbk_frame_end_toisp(stream, active_buf);
+			}
+		} else {
+			if (stream->dvp2axidev->rdbk_debug &&
+			    dev->hw_dev->dummy_buf.vaddr)
+				v4l2_info(&stream->dvp2axidev->v4l2_dev,
+					  "stream[%d] loss frame %d\n",
+					  stream->id, stream->frame_idx - 1);
+		}
+		if (dev->is_support_tools && stream->tools_vdev && active_buf)
+			es_dvp2axi_rdbk_with_tools(stream, active_buf);
+	}
+out_get_buf:
+	stream->frame_phase_cache = stream->frame_phase;
+	if (buffer) {
+		buff_addr_y = buffer->dummy.dma_addr;
+		if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm_addr_y, 0, buff_addr_y, 0, false);
+		} else {
+			es_dvp2axi_write_register(dev, frm_addr_y, buff_addr_y);
+		}
+		if (dev->rdbk_debug > 1 && stream->frame_idx < 15)
+			v4l2_info(&dev->v4l2_dev,
+				  "stream[%d] update, seq %d, reg %x, buf %x\n",
+				  stream->id, stream->frame_idx - 1, frm_addr_y,
+				  (u32)buffer->dummy.dma_addr);
+	} else if (dev->hw_dev->dummy_buf.vaddr && priv &&
+		   priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO) {
+		buff_addr_y = dev->hw_dev->dummy_buf.dma_addr;
+		if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm_addr_y, 0, buff_addr_y, 0, true);
+		} else {
+			es_dvp2axi_write_register(dev, frm_addr_y, buff_addr_y);
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	return 0;
+}
+
+static int es_dvp2axi_assign_new_buffer_pingpong_toisp(struct es_dvp2axi_stream *stream,
+						  int init, int channel_id)
+{
+	int ret = 0;
+
+	if (init)
+		es_dvp2axi_assign_new_buffer_init_toisp(stream, channel_id);
+	else
+		ret = es_dvp2axi_assign_new_buffer_update_toisp(stream, channel_id);
+	return ret;
+}
+
+void es_dvp2axi_assign_check_buffer_update_toisp(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct es_dvp2axi_rx_buffer *buffer = NULL;
+	struct esmodule_capture_info *capture_info =
+		&dev->channels[stream->id].capture_info;
+	struct es_dvp2axi_rx_buffer *active_buf = NULL;
+	u32 frm_addr_y, buff_addr_y;
+	u32 vblank = 0;
+	u32 vblank_ns = 0;
+	u64 cur_time = 0;
+	int frame_phase = 0;
+	int frame_phase_next = 0;
+
+	if (stream->toisp_buf_state.state == ES_DVP2AXI_TOISP_BUF_ROTATE ||
+	    (stream->toisp_buf_state.state == ES_DVP2AXI_TOISP_BUF_THESAME &&
+	     stream->toisp_buf_state.check_cnt >= 1) ||
+	    (stream->toisp_buf_state.state == ES_DVP2AXI_TOISP_BUF_LOSS &&
+	     stream->toisp_buf_state.check_cnt >= 2)) {
+		if (dev->rdbk_debug > 2 && stream->frame_idx < 15)
+			v4l2_info(&dev->v4l2_dev,
+				  "stream[%d] addr check not equal 0x%x 0x%x\n",
+				  stream->id,
+				  (u32)stream->curr_buf_toisp->dummy.dma_addr,
+				  (u32)stream->next_buf_toisp->dummy.dma_addr);
+		return;
+	}
+	if (!dev->sensor_linetime)
+		dev->sensor_linetime = es_dvp2axi_get_linetime(stream);
+	vblank = es_dvp2axi_get_sensor_vblank(dev);
+	vblank_ns = vblank * dev->sensor_linetime;
+	cur_time = es_dvp2axi_time_get_ns(dev);
+
+	if (stream->toisp_buf_state.state == ES_DVP2AXI_TOISP_BUF_THESAME) {
+		frame_phase = stream->frame_phase;
+	} else {
+		if (stream->toisp_buf_state.state == ES_DVP2AXI_TOISP_BUF_LOSS &&
+		    stream->toisp_buf_state.check_cnt == 0 &&
+		    cur_time - stream->readout.fe_timestamp <
+			    (vblank_ns - 500000)) {
+			stream->toisp_buf_state.is_early_update = true;
+			frame_phase = stream->frame_phase &
+						      DVP2AXI_CSI_FRAME0_READY ?
+					      DVP2AXI_CSI_FRAME1_READY :
+					      DVP2AXI_CSI_FRAME0_READY;
+			frame_phase_next =
+				stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+					DVP2AXI_CSI_FRAME0_READY :
+					DVP2AXI_CSI_FRAME1_READY;
+		} else {
+			if (stream->toisp_buf_state.check_cnt == 1 &&
+			    (!stream->toisp_buf_state.is_early_update))
+				return;
+			frame_phase = stream->frame_phase;
+			stream->toisp_buf_state.is_early_update = false;
+		}
+	}
+	if (dev->rdbk_debug > 2 && stream->frame_idx < 15)
+		v4l2_info(
+			&dev->v4l2_dev,
+			"stream[%d] check update, cur %lld, fe %lld, vb %u lack_buf %d\n",
+			stream->id, cur_time, stream->readout.fe_timestamp,
+			vblank_ns, stream->lack_buf_cnt);
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm_addr_y = frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				     get_reg_index_of_frm0_y_addr(stream->id) :
+				     get_reg_index_of_frm1_y_addr(stream->id);
+	} else {
+		frm_addr_y =
+			frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_y_addr(stream->id) :
+				get_dvp_reg_index_of_frm1_y_addr(stream->id);
+	}
+	if (!list_empty(&stream->rx_buf_head)) {
+		if (frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+			active_buf = stream->curr_buf_toisp;
+			buffer = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (buffer) {
+				list_del(&buffer->list);
+				stream->curr_buf_toisp = buffer;
+				buff_addr_y =
+					stream->curr_buf_toisp->dummy.dma_addr;
+				if (capture_info->mode ==
+				    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+					es_dvp2axi_write_buff_addr_multi_dev_combine(
+						stream, frm_addr_y, 0,
+						buff_addr_y, 0, false);
+				} else {
+					es_dvp2axi_write_register(dev, frm_addr_y,
+							     buff_addr_y);
+				}
+				if (dev->rdbk_debug > 1 &&
+				    stream->frame_idx < 15)
+					v4l2_info(
+						&dev->v4l2_dev,
+						"stream[%d] check update, seq %d, addr 0x%x, buf 0x%x\n",
+						stream->id,
+						stream->frame_idx - 1,
+						frm_addr_y,
+						(u32)stream->curr_buf_toisp
+							->dummy.dma_addr);
+			}
+		} else if (frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			active_buf = stream->next_buf_toisp;
+			buffer = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (buffer) {
+				list_del(&buffer->list);
+				stream->next_buf_toisp = buffer;
+				buff_addr_y =
+					stream->next_buf_toisp->dummy.dma_addr;
+				if (capture_info->mode ==
+				    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+					es_dvp2axi_write_buff_addr_multi_dev_combine(
+						stream, frm_addr_y, 0,
+						buff_addr_y, 0, false);
+				} else {
+					es_dvp2axi_write_register(dev, frm_addr_y,
+							     buff_addr_y);
+				}
+				if (dev->rdbk_debug > 1 &&
+				    stream->frame_idx < 15)
+					v4l2_info(
+						&dev->v4l2_dev,
+						"stream[%d] check update, seq %d, addr 0x%x, buf 0x%x\n",
+						stream->id,
+						stream->frame_idx - 1,
+						frm_addr_y,
+						(u32)stream->next_buf_toisp
+							->dummy.dma_addr);
+			}
+		}
+		if (stream->lack_buf_cnt)
+			stream->lack_buf_cnt--;
+	}
+	if (stream->toisp_buf_state.is_early_update) {
+		if (dev->rdbk_debug > 1 && stream->frame_idx < 15)
+			v4l2_info(&dev->v4l2_dev,
+				  "stream[%d] early update, seq %d\n",
+				  stream->id, stream->frame_idx - 1);
+		if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+		    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY)
+			es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_CTRL,
+						0x00010000);
+		else
+			es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_CTRL,
+						0x00010000);
+		if (active_buf) {
+			active_buf->dbufs.sequence = stream->frame_idx - 1;
+			active_buf->dbufs.timestamp =
+				stream->readout.fs_timestamp;
+			stream->last_frame_idx = stream->frame_idx;
+			es_dvp2axi_s_rx_buffer(stream, &active_buf->dbufs);
+		}
+		if (dev->hw_dev->dummy_buf.vaddr)
+			return;
+		if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+		    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+		    mbus_cfg->type == V4L2_MBUS_CCP2) {
+			frm_addr_y = frame_phase_next & DVP2AXI_CSI_FRAME0_READY ?
+					     get_reg_index_of_frm0_y_addr(
+						     stream->id) :
+					     get_reg_index_of_frm1_y_addr(
+						     stream->id);
+		} else {
+			frm_addr_y = frame_phase_next & DVP2AXI_CSI_FRAME0_READY ?
+					     get_dvp_reg_index_of_frm0_y_addr(
+						     stream->id) :
+					     get_dvp_reg_index_of_frm1_y_addr(
+						     stream->id);
+		}
+		if (frame_phase == DVP2AXI_CSI_FRAME0_READY)
+			stream->next_buf_toisp = stream->curr_buf_toisp;
+		else
+			stream->curr_buf_toisp = stream->next_buf_toisp;
+		buff_addr_y = stream->curr_buf_toisp->dummy.dma_addr;
+		if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm_addr_y, 0, buff_addr_y, 0, false);
+		} else {
+			es_dvp2axi_write_register(dev, frm_addr_y, buff_addr_y);
+		}
+	}
+	stream->toisp_buf_state.check_cnt++;
+}
+
+static void es_dvp2axi_assign_new_buffer_init(struct es_dvp2axi_stream *stream,
+					 int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	u32 frm0_addr_y, frm0_addr_uv;
+	u32 frm1_addr_y, frm1_addr_uv;
+	u32 buff_addr_y, buff_addr_cbcr;
+	unsigned long flags;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	struct csi_channel_info *channel = &dev->channels[channel_id];
+	stream->lack_buf_cnt = 0;
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm0_addr_y = get_reg_index_of_frm0_y_addr(channel_id);
+		frm0_addr_uv = get_reg_index_of_frm0_uv_addr(channel_id);
+		frm1_addr_y = get_reg_index_of_frm1_y_addr(channel_id);
+		frm1_addr_uv = get_reg_index_of_frm1_uv_addr(channel_id);
+	} else {
+		frm0_addr_y = get_dvp_reg_index_of_frm0_y_addr(channel_id);
+		frm0_addr_uv = get_dvp_reg_index_of_frm0_uv_addr(channel_id);
+		frm1_addr_y = get_dvp_reg_index_of_frm1_y_addr(channel_id);
+		frm1_addr_uv = get_dvp_reg_index_of_frm1_uv_addr(channel_id);
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+
+	if (!stream->curr_buf) {
+		if (!list_empty(&stream->buf_head)) {
+			stream->curr_buf = list_first_entry(
+				&stream->buf_head, struct es_dvp2axi_buffer, queue);
+			v4l2_dbg(4, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "%s %d, stream[%d] buf idx %d\n", __func__,
+				 __LINE__, stream->id,
+				 stream->curr_buf->vb.vb2_buf.index);
+			list_del(&stream->curr_buf->queue);
+		}
+	}
+
+	if (stream->curr_buf) {
+		buff_addr_y = stream->curr_buf->buff_addr[ES_DVP2AXI_PLANE_Y];
+		buff_addr_cbcr = stream->curr_buf->buff_addr[ES_DVP2AXI_PLANE_CBCR];
+		if (channel->capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm0_addr_y, frm0_addr_uv, buff_addr_y,
+				buff_addr_cbcr, false);
+		} else {
+			es_dvp2axi_write_register(
+				dev, frm0_addr_y,
+				stream->curr_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(
+					dev, frm0_addr_uv,
+					stream->curr_buf
+						->buff_addr[ES_DVP2AXI_PLANE_CBCR]);
+		}
+	} else {
+		if (dummy_buf->vaddr) {
+			buff_addr_y = dummy_buf->dma_addr;
+			buff_addr_cbcr = dummy_buf->dma_addr;
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm0_addr_y, frm0_addr_uv,
+					buff_addr_y, buff_addr_cbcr, true);
+			} else {
+				es_dvp2axi_write_register(dev, frm0_addr_y,
+						     buff_addr_y);
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(dev, frm0_addr_uv,
+							     buff_addr_cbcr);
+			}
+		} else {
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		}
+	}
+
+	if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED) {
+		stream->next_buf = stream->curr_buf;
+		if (stream->next_buf) {
+			buff_addr_y =
+				stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_Y];
+			buff_addr_cbcr =
+				stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_CBCR];
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm1_addr_y, frm1_addr_uv,
+					buff_addr_y, buff_addr_cbcr, false);
+			} else {
+				es_dvp2axi_write_register(
+					dev, frm1_addr_y,
+					buff_addr_y +
+						(channel->virtual_width / 2));
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(
+						dev, frm1_addr_uv,
+						buff_addr_cbcr +
+							(channel->virtual_width /
+							 2));
+			}
+		}
+	} else {
+		if (!stream->next_buf) {
+			if (!list_empty(&stream->buf_head)) {
+				stream->next_buf = list_first_entry(
+					&stream->buf_head, struct es_dvp2axi_buffer,
+					queue);
+				v4l2_dbg(4, es_dvp2axi_debug, &dev->v4l2_dev,
+					 "%s %d, stream[%d] buf idx %d\n",
+					 __func__, __LINE__, stream->id,
+					 stream->next_buf->vb.vb2_buf.index);
+				list_del(&stream->next_buf->queue);
+			} else if (stream->curr_buf) {
+				stream->next_buf = stream->curr_buf;
+			}
+		}
+
+		if (!stream->next_buf && dummy_buf->vaddr) {
+			buff_addr_y = dummy_buf->dma_addr;
+			buff_addr_cbcr = dummy_buf->dma_addr;
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm1_addr_y, frm1_addr_uv,
+					buff_addr_y, buff_addr_cbcr, true);
+			} else {
+				es_dvp2axi_write_register(dev, frm1_addr_y,
+						     dummy_buf->dma_addr);
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(
+						dev, frm1_addr_uv,
+						dummy_buf->dma_addr);
+			}
+
+		} else if (!stream->next_buf && stream->curr_buf) {
+			stream->next_buf = stream->curr_buf;
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		}
+		if (stream->next_buf) {
+			buff_addr_y =
+				stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_Y];
+			buff_addr_cbcr =
+				stream->next_buf->buff_addr[ES_DVP2AXI_PLANE_CBCR];
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm1_addr_y, frm1_addr_uv,
+					buff_addr_y, buff_addr_cbcr, false);
+			} else {
+				es_dvp2axi_write_register(dev, frm1_addr_y,
+						     buff_addr_y);
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(dev, frm1_addr_uv,
+							     buff_addr_cbcr);
+			}
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	stream->is_dvp_yuv_addr_init = true;
+
+	/* for BT.656/BT.1120 multi channels function,
+	 * yuv addr of unused channel must be set
+	 */
+	if (mbus_cfg->type == V4L2_MBUS_BT656) {
+		int ch_id;
+
+		for (ch_id = 0; ch_id < ES_DVP2AXI_MAX_STREAM_DVP; ch_id++) {
+			if (dev->stream[ch_id].is_dvp_yuv_addr_init)
+				continue;
+			if (dummy_buf->dma_addr) {
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm0_y_addr(ch_id),
+					dummy_buf->dma_addr);
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm0_uv_addr(
+						ch_id),
+					dummy_buf->dma_addr);
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm1_y_addr(ch_id),
+					dummy_buf->dma_addr);
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm1_uv_addr(
+						ch_id),
+					dummy_buf->dma_addr);
+			}
+		}
+	}
+	stream->buf_owner = ES_DVP2AXI_DMAEN_BY_VICAP;
+}
+
+static int es_dvp2axi_assign_new_buffer_update(struct es_dvp2axi_stream *stream,
+					  int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct es_dvp2axi_buffer *buffer = NULL;
+	u32 frm_addr_y, frm_addr_uv;
+	struct csi_channel_info *channel = &dev->channels[channel_id];
+	struct esisp_rx_buf *dbufs = NULL;
+	struct dma_buf *dbuf = NULL;
+	int ret = 0;
+	u32 buff_addr_y, buff_addr_cbcr;
+	unsigned long flags;
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm_addr_y = stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				     get_reg_index_of_frm0_y_addr(channel_id) :
+				     get_reg_index_of_frm1_y_addr(channel_id);
+		frm_addr_uv =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_reg_index_of_frm0_uv_addr(channel_id) :
+				get_reg_index_of_frm1_uv_addr(channel_id);
+	} else {
+		frm_addr_y =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_y_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_y_addr(channel_id);
+		frm_addr_uv =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_uv_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_uv_addr(channel_id);
+	}
+
+	if (stream->to_stop_dma && (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP)) {
+		v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev, "%s %d\n", __func__,
+			 __LINE__);
+		goto stop_dma;
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (!list_empty(&stream->buf_head)) {
+		if (!dummy_buf->vaddr && stream->curr_buf == stream->next_buf &&
+		    stream->dvp2axi_fmt_in->field != V4L2_FIELD_INTERLACED)
+			ret = -EINVAL;
+
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+			if (!stream->curr_buf)
+				ret = -EINVAL;
+			stream->curr_buf = list_first_entry(
+				&stream->buf_head, struct es_dvp2axi_buffer, queue);
+			if (stream->curr_buf) {
+				list_del(&stream->curr_buf->queue);
+				buffer = stream->curr_buf;
+				v4l2_dbg(
+					3, es_dvp2axi_debug, &dev->v4l2_dev,
+					"stream[%d] update curr_buf 0x%x, buf idx %d\n",
+					stream->id, buffer->buff_addr[0],
+					stream->curr_buf->vb.vb2_buf.index);
+			}
+		} else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			if (!stream->next_buf)
+				ret = -EINVAL;
+			if (stream->dvp2axi_fmt_in->field ==
+			    V4L2_FIELD_INTERLACED) {
+				if (stream->next_buf != stream->curr_buf) {
+					stream->next_buf = stream->curr_buf;
+					buffer = stream->next_buf;
+				} else {
+					buffer = NULL;
+				}
+
+			} else {
+				stream->next_buf = list_first_entry(
+					&stream->buf_head, struct es_dvp2axi_buffer,
+					queue);
+				if (stream->next_buf) {
+					list_del(&stream->next_buf->queue);
+					buffer = stream->next_buf;
+					v4l2_dbg(
+						3, es_dvp2axi_debug, &dev->v4l2_dev,
+						"stream[%d] update next_buf 0x%x, buf idx %d\n",
+						stream->id,
+						buffer->buff_addr[0],
+						stream->next_buf->vb.vb2_buf
+							.index);
+				}
+			}
+		}
+	} else {
+		buffer = NULL;
+		if (!(stream->cur_stream_mode & ES_DVP2AXI_STREAM_MODE_TOISP) &&
+		    dummy_buf->vaddr) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				stream->curr_buf = NULL;
+			} else if (stream->frame_phase ==
+				   DVP2AXI_CSI_FRAME1_READY) {
+				if (stream->dvp2axi_fmt_in->field ==
+				    V4L2_FIELD_INTERLACED) {
+					stream->next_buf = stream->curr_buf;
+					buffer = stream->next_buf;
+				} else {
+					stream->next_buf = NULL;
+				}
+			}
+		} else if (stream->curr_buf != stream->next_buf) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY &&
+			    stream->next_buf) {
+				stream->curr_buf = stream->next_buf;
+				buffer = stream->next_buf;
+			} else if (stream->frame_phase ==
+					   DVP2AXI_CSI_FRAME1_READY &&
+				   stream->curr_buf) {
+				stream->next_buf = stream->curr_buf;
+				buffer = stream->curr_buf;
+			}
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		} else {
+			stream->curr_buf = NULL;
+			stream->next_buf = NULL;
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		}
+	}
+	stream->frame_phase_cache = stream->frame_phase;
+
+	if (buffer) {
+		buff_addr_y = buffer->buff_addr[ES_DVP2AXI_PLANE_Y];
+		buff_addr_cbcr = buffer->buff_addr[ES_DVP2AXI_PLANE_CBCR];
+		if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED &&
+		    stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm_addr_y, frm_addr_uv,
+					buff_addr_y, buff_addr_cbcr, false);
+			} else {
+				es_dvp2axi_write_register(
+					dev, frm_addr_y,
+					buff_addr_y +
+						(channel->virtual_width / 2));
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(
+						dev, frm_addr_uv,
+						buff_addr_cbcr +
+							(channel->virtual_width /
+							 2));
+			}
+		} else {
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm_addr_y, frm_addr_uv,
+					buff_addr_y, buff_addr_cbcr, false);
+			} else {
+				es_dvp2axi_write_register(dev, frm_addr_y,
+						     buff_addr_y);
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(dev, frm_addr_uv,
+							     buff_addr_cbcr);
+			}
+		}
+		if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP) {
+			if (stream->buf_replace_cnt < 2)
+				stream->buf_replace_cnt++;
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY &&
+			    stream->next_buf)
+				dbuf = stream->next_buf->dbuf;
+			else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY &&
+				 stream->curr_buf)
+				dbuf = stream->curr_buf->dbuf;
+
+			if (dbuf) {
+				list_for_each_entry(dbufs,
+						    &stream->rx_buf_head_vicap,
+						    list) {
+					if (dbufs->dbuf == dbuf)
+						break;
+				}
+			}
+			if (dbufs)
+				es_dvp2axi_s_rx_buffer(stream, dbufs);
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	return ret;
+stop_dma:
+	if (stream->buf_replace_cnt) {
+		spin_lock_irqsave(&stream->vbq_lock, flags);
+		buff_addr_y = stream->curr_buf_toisp->dummy.dma_addr;
+		if (channel->capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE)
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm_addr_y, 0, buff_addr_y, 0, false);
+		else
+			es_dvp2axi_write_register(dev, frm_addr_y, buff_addr_y);
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY &&
+		    stream->next_buf)
+			dbuf = stream->next_buf->dbuf;
+		else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY &&
+			 stream->curr_buf)
+			dbuf = stream->curr_buf->dbuf;
+
+		if (dbuf) {
+			list_for_each_entry(dbufs, &stream->rx_buf_head_vicap,
+					    list)
+				if (dbufs->dbuf == dbuf)
+					break;
+		} else {
+			dbufs = &stream->curr_buf_toisp->dbufs;
+		}
+		if (dbufs)
+			es_dvp2axi_s_rx_buffer(stream, dbufs);
+
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY &&
+		    stream->curr_buf) {
+			stream->curr_buf = NULL;
+		} else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY &&
+			   stream->next_buf) {
+			stream->next_buf = NULL;
+		}
+		stream->buf_replace_cnt--;
+		spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	}
+	return -EINVAL;
+}
+
+static int es_dvp2axi_get_new_buffer_wake_up_mode(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	int ret = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (!list_empty(&stream->buf_head)) {
+		if (!dummy_buf->vaddr && stream->curr_buf == stream->next_buf)
+			ret = -EINVAL;
+		if (stream->line_int_cnt % 2) {
+			stream->curr_buf = list_first_entry(
+				&stream->buf_head, struct es_dvp2axi_buffer, queue);
+			if (stream->curr_buf)
+				list_del(&stream->curr_buf->queue);
+		} else {
+			stream->next_buf = list_first_entry(
+				&stream->buf_head, struct es_dvp2axi_buffer, queue);
+			if (stream->next_buf)
+				list_del(&stream->next_buf->queue);
+		}
+		stream->is_buf_active = true;
+		if (stream->lack_buf_cnt)
+			stream->lack_buf_cnt--;
+	} else {
+		stream->is_buf_active = false;
+		if (dummy_buf->vaddr) {
+			if (stream->line_int_cnt % 2)
+				stream->curr_buf = NULL;
+			else
+				stream->next_buf = NULL;
+		} else if (stream->curr_buf != stream->next_buf) {
+			if (stream->line_int_cnt % 2) {
+				stream->curr_buf = stream->next_buf;
+				stream->frame_phase_cache =
+					DVP2AXI_CSI_FRAME0_READY;
+			} else {
+				stream->next_buf = stream->curr_buf;
+				stream->frame_phase_cache =
+					DVP2AXI_CSI_FRAME1_READY;
+			}
+			stream->is_buf_active = true;
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		} else {
+			ret = 0;
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	return ret;
+}
+
+static int es_dvp2axi_update_new_buffer_wake_up_mode(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct esmodule_capture_info *capture_info =
+		&dev->channels[stream->id].capture_info;
+	struct es_dvp2axi_buffer *buffer = NULL;
+	u32 frm_addr_y, frm_addr_uv;
+	u32 buff_addr_y, buff_addr_cbcr;
+	int channel_id = stream->id;
+	int ret = 0;
+	unsigned long flags;
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm_addr_y = stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				     get_reg_index_of_frm0_y_addr(channel_id) :
+				     get_reg_index_of_frm1_y_addr(channel_id);
+		frm_addr_uv =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_reg_index_of_frm0_uv_addr(channel_id) :
+				get_reg_index_of_frm1_uv_addr(channel_id);
+	} else {
+		frm_addr_y =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_y_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_y_addr(channel_id);
+		frm_addr_uv =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_uv_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_uv_addr(channel_id);
+	}
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (stream->is_buf_active) {
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY)
+			buffer = stream->curr_buf;
+		else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY)
+			buffer = stream->next_buf;
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	if (buffer) {
+		buff_addr_y = buffer->buff_addr[ES_DVP2AXI_PLANE_Y];
+		buff_addr_cbcr = buffer->buff_addr[ES_DVP2AXI_PLANE_CBCR];
+		if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			es_dvp2axi_write_buff_addr_multi_dev_combine(
+				stream, frm_addr_y, frm_addr_uv, buff_addr_y,
+				buff_addr_cbcr, false);
+		} else {
+			es_dvp2axi_write_register(dev, frm_addr_y, buff_addr_y);
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(dev, frm_addr_uv,
+						     buff_addr_cbcr);
+		}
+	} else {
+		if (dummy_buf->vaddr) {
+			buff_addr_y = dummy_buf->dma_addr;
+			buff_addr_cbcr = dummy_buf->dma_addr;
+			if (capture_info->mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				es_dvp2axi_write_buff_addr_multi_dev_combine(
+					stream, frm_addr_y, frm_addr_uv,
+					buff_addr_y, buff_addr_cbcr, true);
+			} else {
+				es_dvp2axi_write_register(dev, frm_addr_y,
+						     buff_addr_y);
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(dev, frm_addr_uv,
+							     buff_addr_cbcr);
+			}
+		} else {
+			ret = 0;
+		}
+		dev->err_state |= (ES_DVP2AXI_ERR_ID0_NOT_BUF << stream->id);
+		dev->irq_stats.not_active_buf_cnt[stream->id]++;
+	}
+
+	return ret;
+}
+
+static int es_dvp2axi_get_new_buffer_wake_up_mode_rdbk(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_rx_buffer *buffer = NULL;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	int ret = 0;
+	unsigned long flags;
+	u32 frm_addr_y;
+	int frame_phase = 0;
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	memset(&stream->toisp_buf_state, 0, sizeof(stream->toisp_buf_state));
+	if (!list_empty(&stream->rx_buf_head)) {
+		if (stream->curr_buf_toisp && stream->next_buf_toisp &&
+		    stream->curr_buf_toisp != stream->next_buf_toisp)
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_ROTATE;
+		else
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_LOSS;
+		if (stream->line_int_cnt % 2) {
+			buffer = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (buffer) {
+				list_del(&buffer->list);
+				stream->curr_buf_toisp = buffer;
+			}
+			frame_phase = DVP2AXI_CSI_FRAME0_READY;
+		} else {
+			buffer = list_first_entry(&stream->rx_buf_head,
+						  struct es_dvp2axi_rx_buffer, list);
+			if (buffer) {
+				list_del(&buffer->list);
+				stream->next_buf_toisp = buffer;
+			}
+			frame_phase = DVP2AXI_CSI_FRAME1_READY;
+		}
+		if (stream->lack_buf_cnt)
+			stream->lack_buf_cnt--;
+	} else {
+		if (stream->lack_buf_cnt < 2)
+			stream->lack_buf_cnt++;
+		if (stream->curr_buf_toisp && stream->next_buf_toisp &&
+		    stream->curr_buf_toisp != stream->next_buf_toisp) {
+			if (stream->line_int_cnt % 2)
+				stream->curr_buf_toisp = stream->next_buf_toisp;
+			else
+				stream->next_buf_toisp = stream->curr_buf_toisp;
+			buffer = stream->curr_buf_toisp;
+			ret = 0;
+			if (stream->dvp2axidev->rdbk_debug)
+				v4l2_info(&stream->dvp2axidev->v4l2_dev,
+					  "stream[%d] hold buf %x\n",
+					  stream->id,
+					  (u32)stream->next_buf_toisp->dummy
+						  .dma_addr);
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_THESAME;
+		} else {
+			ret = -EINVAL;
+			stream->toisp_buf_state.state = ES_DVP2AXI_TOISP_BUF_LOSS;
+		}
+	}
+	if (buffer) {
+		if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+		    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+		    mbus_cfg->type == V4L2_MBUS_CCP2) {
+			frm_addr_y = frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+					     get_reg_index_of_frm0_y_addr(
+						     stream->id) :
+					     get_reg_index_of_frm1_y_addr(
+						     stream->id);
+		} else {
+			frm_addr_y = frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+					     get_dvp_reg_index_of_frm0_y_addr(
+						     stream->id) :
+					     get_dvp_reg_index_of_frm1_y_addr(
+						     stream->id);
+		}
+		es_dvp2axi_write_register(dev, frm_addr_y, buffer->dummy.dma_addr);
+		if (dev->rdbk_debug > 1 && stream->frame_idx < 15)
+			v4l2_info(
+				&dev->v4l2_dev,
+				"stream[%d] rdbk update, seq %d, reg %x, buf %x\n",
+				stream->id, stream->frame_idx - 1, frm_addr_y,
+				(u32)buffer->dummy.dma_addr);
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	return ret;
+}
+
+#if 0
+static void es_dvp2axi_assign_dummy_buffer(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	unsigned long flags;
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+
+	/* for BT.656/BT.1120 multi channels function,
+	 * yuv addr of unused channel must be set
+	 */
+	if (mbus_cfg->type == V4L2_MBUS_BT656 && dummy_buf->vaddr) {
+		es_dvp2axi_write_register(
+			dev, get_dvp_reg_index_of_frm0_y_addr(stream->id),
+			dummy_buf->dma_addr);
+		es_dvp2axi_write_register(
+			dev, get_dvp_reg_index_of_frm0_uv_addr(stream->id),
+			dummy_buf->dma_addr);
+		es_dvp2axi_write_register(
+			dev, get_dvp_reg_index_of_frm1_y_addr(stream->id),
+			dummy_buf->dma_addr);
+		es_dvp2axi_write_register(
+			dev, get_dvp_reg_index_of_frm1_uv_addr(stream->id),
+			dummy_buf->dma_addr);
+	}
+
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+}
+#endif
+
+static int es_dvp2axi_assign_new_buffer_pingpong(struct es_dvp2axi_stream *stream,
+					    int init, int channel_id)
+{
+	int ret = 0;
+
+	if (init)
+		es_dvp2axi_assign_new_buffer_init(stream, channel_id);
+	else
+		ret = es_dvp2axi_assign_new_buffer_update(stream, channel_id);
+	return ret;
+}
+
+static void es_dvp2axi_assign_new_buffer_init_eskit(struct es_dvp2axi_stream *stream,
+						int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	u32 frm0_addr_y, frm0_addr_uv;
+	u32 frm1_addr_y, frm1_addr_uv;
+	unsigned long flags;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	struct csi_channel_info *channel = &dev->channels[channel_id];
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm0_addr_y = get_reg_index_of_frm0_y_addr(channel_id);
+		frm0_addr_uv = get_reg_index_of_frm0_uv_addr(channel_id);
+		frm1_addr_y = get_reg_index_of_frm1_y_addr(channel_id);
+		frm1_addr_uv = get_reg_index_of_frm1_uv_addr(channel_id);
+	} else {
+		frm0_addr_y = get_dvp_reg_index_of_frm0_y_addr(channel_id);
+		frm0_addr_uv = get_dvp_reg_index_of_frm0_uv_addr(channel_id);
+		frm1_addr_y = get_dvp_reg_index_of_frm1_y_addr(channel_id);
+		frm1_addr_uv = get_dvp_reg_index_of_frm1_uv_addr(channel_id);
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+
+	if (!stream->curr_buf_eskit) {
+		if (!list_empty(&stream->eskit_buf_head)) {
+			stream->curr_buf_eskit =
+				list_first_entry(&stream->eskit_buf_head,
+						 struct es_dvp2axi_buffer, queue);
+			list_del(&stream->curr_buf_eskit->queue);
+		}
+	}
+
+	if (stream->curr_buf_eskit) {
+		es_dvp2axi_write_register(
+			dev, frm0_addr_y,
+			stream->curr_buf_eskit->buff_addr[ES_DVP2AXI_PLANE_Y]);
+		if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+			es_dvp2axi_write_register(
+				dev, frm0_addr_uv,
+				stream->curr_buf_eskit
+					->buff_addr[ES_DVP2AXI_PLANE_CBCR]);
+	} else {
+		if (dummy_buf->vaddr) {
+			es_dvp2axi_write_register(dev, frm0_addr_y,
+					     dummy_buf->dma_addr);
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(dev, frm0_addr_uv,
+						     dummy_buf->dma_addr);
+		} else {
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		}
+	}
+
+	if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED) {
+		stream->next_buf_eskit = stream->curr_buf_eskit;
+		if (stream->next_buf_eskit) {
+			es_dvp2axi_write_register(
+				dev, frm1_addr_y,
+				stream->next_buf_eskit
+						->buff_addr[ES_DVP2AXI_PLANE_Y] +
+					(channel->virtual_width / 2));
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(
+					dev, frm1_addr_uv,
+					stream->next_buf_eskit->buff_addr
+							[ES_DVP2AXI_PLANE_CBCR] +
+						(channel->virtual_width / 2));
+		}
+	} else {
+		if (!stream->next_buf_eskit) {
+			if (!list_empty(&stream->eskit_buf_head)) {
+				stream->next_buf_eskit = list_first_entry(
+					&stream->eskit_buf_head,
+					struct es_dvp2axi_buffer, queue);
+				list_del(&stream->next_buf_eskit->queue);
+			}
+		}
+
+		if (stream->next_buf_eskit) {
+			es_dvp2axi_write_register(
+				dev, frm1_addr_y,
+				stream->next_buf_eskit
+					->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(
+					dev, frm1_addr_uv,
+					stream->next_buf_eskit
+						->buff_addr[ES_DVP2AXI_PLANE_CBCR]);
+		} else {
+			if (dummy_buf->vaddr) {
+				es_dvp2axi_write_register(dev, frm1_addr_y,
+						     dummy_buf->dma_addr);
+				if (stream->dvp2axi_fmt_out->fmt_type !=
+				    DVP2AXI_FMT_TYPE_RAW)
+					es_dvp2axi_write_register(
+						dev, frm1_addr_uv,
+						dummy_buf->dma_addr);
+			} else {
+				if (stream->curr_buf_eskit) {
+					stream->next_buf_eskit =
+						stream->curr_buf_eskit;
+					es_dvp2axi_write_register(
+						dev, frm1_addr_y,
+						stream->next_buf_eskit->buff_addr
+							[ES_DVP2AXI_PLANE_Y]);
+					if (stream->dvp2axi_fmt_out->fmt_type !=
+					    DVP2AXI_FMT_TYPE_RAW)
+						es_dvp2axi_write_register(
+							dev, frm1_addr_uv,
+							stream->next_buf_eskit->buff_addr
+								[ES_DVP2AXI_PLANE_CBCR]);
+				}
+				if (stream->lack_buf_cnt < 2)
+					stream->lack_buf_cnt++;
+			}
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	stream->is_dvp_yuv_addr_init = true;
+
+	/* for BT.656/BT.1120 multi channels function,
+	 * yuv addr of unused channel must be set
+	 */
+	if (mbus_cfg->type == V4L2_MBUS_BT656) {
+		int ch_id;
+
+		for (ch_id = 0; ch_id < ES_DVP2AXI_MAX_STREAM_DVP; ch_id++) {
+			if (dev->stream[ch_id].is_dvp_yuv_addr_init)
+				continue;
+			if (dummy_buf->dma_addr) {
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm0_y_addr(ch_id),
+					dummy_buf->dma_addr);
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm0_uv_addr(
+						ch_id),
+					dummy_buf->dma_addr);
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm1_y_addr(ch_id),
+					dummy_buf->dma_addr);
+				es_dvp2axi_write_register(
+					dev,
+					get_dvp_reg_index_of_frm1_uv_addr(
+						ch_id),
+					dummy_buf->dma_addr);
+			}
+		}
+	}
+	stream->buf_owner = ES_DVP2AXI_DMAEN_BY_ESKIT;
+}
+
+static int es_dvp2axi_assign_new_buffer_update_eskit(struct es_dvp2axi_stream *stream,
+						 int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct es_dvp2axi_buffer *buffer = NULL;
+	u32 frm_addr_y, frm_addr_uv;
+	struct csi_channel_info *channel = &dev->channels[channel_id];
+	int ret = 0;
+	unsigned long flags;
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm_addr_y = stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				     get_reg_index_of_frm0_y_addr(channel_id) :
+				     get_reg_index_of_frm1_y_addr(channel_id);
+		frm_addr_uv =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_reg_index_of_frm0_uv_addr(channel_id) :
+				get_reg_index_of_frm1_uv_addr(channel_id);
+	} else {
+		frm_addr_y =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_y_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_y_addr(channel_id);
+		frm_addr_uv =
+			stream->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+				get_dvp_reg_index_of_frm0_uv_addr(channel_id) :
+				get_dvp_reg_index_of_frm1_uv_addr(channel_id);
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (!list_empty(&stream->eskit_buf_head)) {
+		if (!dummy_buf->vaddr &&
+		    stream->curr_buf_eskit == stream->next_buf_eskit &&
+		    stream->dvp2axi_fmt_in->field != V4L2_FIELD_INTERLACED)
+			ret = -EINVAL;
+
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+			if (!stream->curr_buf_eskit)
+				ret = -EINVAL;
+			stream->curr_buf_eskit =
+				list_first_entry(&stream->eskit_buf_head,
+						 struct es_dvp2axi_buffer, queue);
+			if (stream->curr_buf_eskit) {
+				list_del(&stream->curr_buf_eskit->queue);
+				buffer = stream->curr_buf_eskit;
+			}
+		} else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			if (!stream->next_buf_eskit)
+				ret = -EINVAL;
+			if (stream->dvp2axi_fmt_in->field ==
+			    V4L2_FIELD_INTERLACED) {
+				if (stream->next_buf_eskit !=
+				    stream->curr_buf_eskit) {
+					stream->next_buf_eskit =
+						stream->curr_buf_eskit;
+					buffer = stream->next_buf_eskit;
+				} else {
+					buffer = NULL;
+				}
+
+			} else {
+				stream->next_buf_eskit = list_first_entry(
+					&stream->eskit_buf_head,
+					struct es_dvp2axi_buffer, queue);
+				if (stream->next_buf_eskit) {
+					list_del(
+						&stream->next_buf_eskit->queue);
+					buffer = stream->next_buf_eskit;
+				}
+			}
+		}
+	} else {	
+		buffer = NULL;
+		if (dummy_buf->vaddr) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				stream->curr_buf_eskit = NULL;
+			} else if (stream->frame_phase ==
+				   DVP2AXI_CSI_FRAME1_READY) {
+				if (stream->dvp2axi_fmt_in->field ==
+				    V4L2_FIELD_INTERLACED) {
+					stream->next_buf_eskit =
+						stream->curr_buf_eskit;
+					buffer = stream->next_buf_eskit;
+				} else {
+					stream->next_buf_eskit = NULL;
+				}
+			}
+		} else if (stream->curr_buf_eskit && stream->next_buf_eskit &&
+			   stream->curr_buf_eskit != stream->next_buf_eskit) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+				stream->curr_buf_eskit =
+					stream->next_buf_eskit;
+				buffer = stream->next_buf_eskit;
+			} else if (stream->frame_phase ==
+				   DVP2AXI_CSI_FRAME1_READY) {
+				stream->next_buf_eskit =
+					stream->curr_buf_eskit;
+				buffer = stream->curr_buf_eskit;
+			}
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		} else {
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
+		}
+	}
+	stream->frame_phase_cache = stream->frame_phase;
+
+	if (buffer) {
+		if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED &&
+		    stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			es_dvp2axi_write_register(
+				dev, frm_addr_y,
+				buffer->buff_addr[ES_DVP2AXI_PLANE_Y] +
+					(channel->virtual_width / 2));
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(
+					dev, frm_addr_uv,
+					buffer->buff_addr[ES_DVP2AXI_PLANE_CBCR] +
+						(channel->virtual_width / 2));
+		} else {
+			es_dvp2axi_write_register(dev, frm_addr_y,
+					     buffer->buff_addr[ES_DVP2AXI_PLANE_Y]);
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(
+					dev, frm_addr_uv,
+					buffer->buff_addr[ES_DVP2AXI_PLANE_CBCR]);
+		}
+	} else {
+		if (dummy_buf->vaddr) {
+			es_dvp2axi_write_register(dev, frm_addr_y,
+					     dummy_buf->dma_addr);
+			if (stream->dvp2axi_fmt_out->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+				es_dvp2axi_write_register(dev, frm_addr_uv,
+						     dummy_buf->dma_addr);
+			dev->err_state |= (ES_DVP2AXI_ERR_ID0_NOT_BUF << stream->id);
+			dev->irq_stats.not_active_buf_cnt[stream->id]++;
+		} else {
+			ret = -EINVAL;
+			dev->err_state |= (ES_DVP2AXI_ERR_ID0_NOT_BUF << stream->id);
+			dev->irq_stats.not_active_buf_cnt[stream->id]++;
+		}
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	return ret;
+}
+
+static int es_dvp2axi_assign_new_buffer_pingpong_eskit(struct es_dvp2axi_stream *stream,
+						   int init, int channel_id)
+{
+	int ret = 0;
+
+	if (init)
+		es_dvp2axi_assign_new_buffer_init_eskit(stream, channel_id);
+	else
+		ret = es_dvp2axi_assign_new_buffer_update_eskit(stream, channel_id);
+	return ret;
+}
+
+static void es_dvp2axi_csi_set_lvds_sav_eav(struct es_dvp2axi_stream *stream,
+				       struct csi_channel_info *channel)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct esmodule_lvds_cfg *lvds_cfg = &channel->lvds_cfg;
+	struct esmodule_lvds_frame_sync_code *frm_sync_code = NULL;
+	struct esmodule_lvds_frm_sync_code *odd_sync_code = NULL;
+	struct esmodule_lvds_frm_sync_code *even_sync_code = NULL;
+
+	if (dev->hdr.hdr_mode == NO_HDR || dev->hdr.hdr_mode == HDR_COMPR) {
+		frm_sync_code = &lvds_cfg->frm_sync_code[LVDS_CODE_GRP_LINEAR];
+		odd_sync_code = &frm_sync_code->odd_sync_code;
+		even_sync_code = odd_sync_code;
+	} else {
+		if (channel->id == ES_DVP2AXI_STREAM_MIPI_ID0)
+			frm_sync_code =
+				&lvds_cfg->frm_sync_code[LVDS_CODE_GRP_LONG];
+
+		if (dev->hdr.hdr_mode == HDR_X2) {
+			if (channel->id == ES_DVP2AXI_STREAM_MIPI_ID1)
+				frm_sync_code = &lvds_cfg->frm_sync_code
+							 [LVDS_CODE_GRP_SHORT];
+			else
+				frm_sync_code = &lvds_cfg->frm_sync_code
+							 [LVDS_CODE_GRP_LONG];
+		} else if (dev->hdr.hdr_mode == HDR_X3) {
+			if (channel->id == ES_DVP2AXI_STREAM_MIPI_ID1)
+				frm_sync_code = &lvds_cfg->frm_sync_code
+							 [LVDS_CODE_GRP_MEDIUM];
+			else if (channel->id == ES_DVP2AXI_STREAM_MIPI_ID2)
+				frm_sync_code = &lvds_cfg->frm_sync_code
+							 [LVDS_CODE_GRP_SHORT];
+			else
+				frm_sync_code = &lvds_cfg->frm_sync_code
+							 [LVDS_CODE_GRP_LONG];
+		}
+
+		odd_sync_code = &frm_sync_code->odd_sync_code;
+		even_sync_code = &frm_sync_code->even_sync_code;
+	}
+
+	if (odd_sync_code && even_sync_code) {
+		es_dvp2axi_write_register(
+			stream->dvp2axidev,
+			get_reg_index_of_lvds_sav_eav_act0(channel->id),
+			SW_LVDS_EAV_ACT(odd_sync_code->act.eav) |
+				SW_LVDS_SAV_ACT(odd_sync_code->act.sav));
+
+		es_dvp2axi_write_register(
+			stream->dvp2axidev,
+			get_reg_index_of_lvds_sav_eav_blk0(channel->id),
+			SW_LVDS_EAV_BLK(odd_sync_code->blk.eav) |
+				SW_LVDS_SAV_BLK(odd_sync_code->blk.sav));
+
+		es_dvp2axi_write_register(
+			stream->dvp2axidev,
+			get_reg_index_of_lvds_sav_eav_act1(channel->id),
+			SW_LVDS_EAV_ACT(even_sync_code->act.eav) |
+				SW_LVDS_SAV_ACT(even_sync_code->act.sav));
+
+		es_dvp2axi_write_register(
+			stream->dvp2axidev,
+			get_reg_index_of_lvds_sav_eav_blk1(channel->id),
+			SW_LVDS_EAV_BLK(even_sync_code->blk.eav) |
+				SW_LVDS_SAV_BLK(even_sync_code->blk.sav));
+	}
+}
+
+static unsigned char get_csi_fmt_val(const struct dvp2axi_input_fmt *dvp2axi_fmt_in,
+				     struct csi_channel_info *csi_info)
+{
+	unsigned char csi_fmt_val = 0;
+
+	if (dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_SPD_2X8 ||
+	    dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_EBD_1X8) {
+		switch (csi_info->data_bit) {
+		case 8:
+			csi_fmt_val = CSI_WRDDR_TYPE_RAW8;
+			break;
+		case 10:
+			csi_fmt_val = CSI_WRDDR_TYPE_RAW10;
+			break;
+		case 12:
+			csi_fmt_val = CSI_WRDDR_TYPE_RAW12;
+			break;
+		default:
+			csi_fmt_val = CSI_WRDDR_TYPE_RAW12;
+			break;
+		}
+	} else if (dvp2axi_fmt_in->csi_fmt_val == CSI_WRDDR_TYPE_RGB888 ||
+		   dvp2axi_fmt_in->csi_fmt_val == CSI_WRDDR_TYPE_RGB565) {
+		csi_fmt_val = CSI_WRDDR_TYPE_RAW8;
+	} else {
+		csi_fmt_val = dvp2axi_fmt_in->csi_fmt_val;
+	}
+	return csi_fmt_val;
+}
+
+static int es_dvp2axi_csi_channel_init(struct es_dvp2axi_stream *stream,
+				  struct csi_channel_info *channel)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct sditf_priv *priv = dev->sditf[0];
+	const struct dvp2axi_output_fmt *fmt;
+	u32 fourcc;
+	int vc = dev->channels[stream->id].vc;
+
+	channel->enable = 1;
+	channel->width = stream->pixm.width;
+	channel->height = stream->pixm.height;
+
+	channel->fmt_val = stream->dvp2axi_fmt_out->csi_fmt_val;
+
+	channel->cmd_mode_en = 0; /* default use DSI Video Mode */
+	channel->dsi_input = dev->terminal_sensor.dsi_input_en;
+
+	if (stream->crop_enable) {
+		channel->crop_en = 1;
+
+		if (channel->fmt_val == CSI_WRDDR_TYPE_RGB888)
+			channel->crop_st_x =
+				3 * stream->crop[CROP_SRC_ACT].left;
+		else if (channel->fmt_val == CSI_WRDDR_TYPE_RGB565)
+			channel->crop_st_x =
+				2 * stream->crop[CROP_SRC_ACT].left;
+		else
+			channel->crop_st_x = stream->crop[CROP_SRC_ACT].left;
+
+		channel->crop_st_y = stream->crop[CROP_SRC_ACT].top;
+		if (priv && priv->is_combine_mode &&
+		    dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF)
+			channel->crop_st_y *= dev->sditf_cnt;
+		channel->width = stream->crop[CROP_SRC_ACT].width;
+		channel->height = stream->crop[CROP_SRC_ACT].height;
+	} else {
+		channel->width = stream->pixm.width;
+		channel->height = stream->pixm.height;
+		channel->crop_en = 0;
+	}
+
+	if (priv && priv->is_combine_mode && dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF)
+		channel->height *= dev->sditf_cnt;
+
+	fmt = es_dvp2axi_find_output_fmt(stream, stream->pixm.pixelformat);
+	if (!fmt) {
+		v4l2_err(&dev->v4l2_dev, "can not find output format: 0x%x",
+			 stream->pixm.pixelformat);
+		return -EINVAL;
+	}
+
+	if (channel->capture_info.mode == ESMODULE_MULTI_DEV_COMBINE_ONE)
+		channel->width /= channel->capture_info.multi_dev.dev_num;
+	/*
+	 * for mipi or lvds, when enable compact, the virtual width of raw10/raw12
+	 * needs aligned with :ALIGN(bits_per_pixel * width / 8, 8), if enable 16bit mode
+	 * needs aligned with :ALIGN(bits_per_pixel * width * 2, 8), to optimize reading and
+	 * writing of ddr, aliged with 256
+	 */
+	if (fmt->fmt_type == DVP2AXI_FMT_TYPE_RAW && stream->is_compact &&
+	    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB888 &&
+	    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB565) {
+		if (channel->capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			channel->virtual_width = ALIGN(
+				channel->width * 2 * fmt->raw_bpp / 8, 256);
+			channel->left_virtual_width =
+				channel->width * fmt->raw_bpp / 8;
+		} else {
+			channel->virtual_width =
+				ALIGN(channel->width * fmt->raw_bpp / 8, 256);
+		}
+	} else {
+		if (channel->capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			channel->virtual_width =
+				ALIGN(channel->width * 2 * fmt->bpp[0] / 8, 8);
+			channel->left_virtual_width =
+				ALIGN(channel->width * fmt->bpp[0] / 8, 8);
+		} else {
+			channel->virtual_width =
+				ALIGN(channel->width * fmt->bpp[0] / 8, 8);
+		}
+	}
+
+	if (channel->fmt_val == CSI_WRDDR_TYPE_RGB888 ||
+	    channel->fmt_val == CSI_WRDDR_TYPE_RGB565)
+		channel->width = channel->width * fmt->bpp[0] / 8;
+	/*
+	 * dvp2axi don't support output yuyv fmt data
+	 * if user request yuyv fmt, the input mode must be RAW8
+	 * and the width is double Because the real input fmt is
+	 * yuyv
+	 */
+	fourcc = stream->dvp2axi_fmt_out->fourcc;
+	if (fourcc == V4L2_PIX_FMT_YUYV || fourcc == V4L2_PIX_FMT_YVYU ||
+	    fourcc == V4L2_PIX_FMT_UYVY || fourcc == V4L2_PIX_FMT_VYUY) {
+		channel->virtual_width *= 2;
+		if (channel->capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE)
+			channel->left_virtual_width *= 2;
+	}
+	if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED) {
+		channel->virtual_width *= 2;
+		channel->height /= 2;
+	}
+	if (stream->dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_EBD_1X8 ||
+	    stream->dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_SPD_2X8) {
+		if (dev->channels[stream->id].data_type)
+			channel->data_type =
+				dev->channels[stream->id].data_type;
+		else
+			channel->data_type = get_data_type(
+				stream->dvp2axi_fmt_in->mbus_code,
+				channel->cmd_mode_en, channel->dsi_input);
+	} else {
+		channel->data_type =
+			get_data_type(stream->dvp2axi_fmt_in->mbus_code,
+				      channel->cmd_mode_en, channel->dsi_input);
+	}
+	channel->csi_fmt_val =
+		get_csi_fmt_val(stream->dvp2axi_fmt_in, &dev->channels[stream->id]);
+
+	if (dev->hdr.hdr_mode == NO_HDR || dev->hdr.hdr_mode == HDR_COMPR ||
+	    (dev->hdr.hdr_mode == HDR_X2 && stream->id > 1) ||
+	    (dev->hdr.hdr_mode == HDR_X3 && stream->id > 2))
+		channel->vc = vc < 4 ? vc : channel->id;
+	else
+		channel->vc = channel->id;
+	v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+		 "%s: channel width %d, height %d, virtual_width %d, vc %d\n",
+		 __func__, channel->width, channel->height,
+		 channel->virtual_width, channel->vc);
+	return 0;
+}
+
+static int es_dvp2axi_csi_channel_set(struct es_dvp2axi_stream *stream,
+				 struct csi_channel_info *channel,
+				 enum v4l2_mbus_type mbus_type)
+{
+	unsigned int val = 0x0;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_stream *detect_stream = &dev->stream[0];
+	unsigned int wait_line = 0x3fff;
+
+	if (channel->id >= 4)
+		return -EINVAL;
+
+	if (!channel->enable) {
+		es_dvp2axi_write_register(dev,
+				     get_reg_index_of_id_ctrl0(channel->id),
+				     CSI_DISABLE_CAPTURE);
+		return 0;
+	}
+
+	es_dvp2axi_write_register_and(dev, DVP2AXI_REG_MIPI_LVDS_INTSTAT,
+				 ~(CSI_START_INTSTAT(channel->id) |
+				   CSI_DMA_END_INTSTAT(channel->id) |
+				   CSI_LINE_INTSTAT(channel->id)));
+
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+				CSI_START_INTEN(channel->id));
+
+	if (detect_stream->is_line_wake_up) {
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					CSI_LINE_INTEN(channel->id));
+		wait_line = dev->wait_line;
+	}
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID0_1,
+			     wait_line << 16 | wait_line);
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID2_3,
+			     wait_line << 16 | wait_line);
+
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+				CSI_DMA_END_INTEN(channel->id));
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_WATER_LINE,
+			     DVP2AXI_MIPI_LVDS_SW_WATER_LINE_25_ES1808 |
+				     DVP2AXI_MIPI_LVDS_SW_WATER_LINE_ENABLE_ES1808 |
+				     DVP2AXI_MIPI_LVDS_SW_HURRY_VALUE_ES1808(0x3) |
+				     DVP2AXI_MIPI_LVDS_SW_HURRY_ENABLE_ES1808);
+
+	val = DVP2AXI_MIPI_LVDS_SW_PRESS_VALUE(0x3) |
+	      DVP2AXI_MIPI_LVDS_SW_PRESS_ENABLE |
+	      DVP2AXI_MIPI_LVDS_SW_HURRY_VALUE(0x3) |
+	      DVP2AXI_MIPI_LVDS_SW_HURRY_ENABLE | DVP2AXI_MIPI_LVDS_SW_WATER_LINE_25 |
+	      DVP2AXI_MIPI_LVDS_SW_WATER_LINE_ENABLE;
+	if (mbus_type == V4L2_MBUS_CSI2_DPHY) {
+		val &= ~DVP2AXI_MIPI_LVDS_SW_SEL_LVDS;
+	} else if (mbus_type == V4L2_MBUS_CCP2) {
+		if (channel->fmt_val == CSI_WRDDR_TYPE_RAW12)
+			val |= DVP2AXI_MIPI_LVDS_SW_LVDS_WIDTH_12BITS;
+		else if (channel->fmt_val == CSI_WRDDR_TYPE_RAW10)
+			val |= DVP2AXI_MIPI_LVDS_SW_LVDS_WIDTH_10BITS;
+		else
+			val |= DVP2AXI_MIPI_LVDS_SW_LVDS_WIDTH_8BITS;
+		val |= DVP2AXI_MIPI_LVDS_SW_SEL_LVDS;
+	}
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL, val);
+
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+				CSI_ALL_ERROR_INTEN);
+
+	es_dvp2axi_write_register(dev, get_reg_index_of_id_ctrl1(channel->id),
+			     channel->width | (channel->height << 16));
+
+	es_dvp2axi_write_register(dev, get_reg_index_of_frm0_y_vlw(channel->id),
+			     channel->virtual_width);
+	es_dvp2axi_write_register(dev, get_reg_index_of_frm1_y_vlw(channel->id),
+			     channel->virtual_width);
+	es_dvp2axi_write_register(dev, get_reg_index_of_frm0_uv_vlw(channel->id),
+			     channel->virtual_width);
+	es_dvp2axi_write_register(dev, get_reg_index_of_frm1_uv_vlw(channel->id),
+			     channel->virtual_width);
+
+	if (channel->crop_en)
+		es_dvp2axi_write_register(
+			dev, get_reg_index_of_id_crop_start(channel->id),
+			channel->crop_st_y << 16 | channel->crop_st_x);
+
+	/* Set up an buffer for the next frame */
+	es_dvp2axi_assign_new_buffer_pingpong(stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT,
+					 channel->id);
+
+	if (mbus_type == V4L2_MBUS_CSI2_DPHY) {
+		//need always enable crop
+		val = CSI_ENABLE_CAPTURE | channel->fmt_val |
+		      channel->cmd_mode_en << 4 | CSI_ENABLE_CROP |
+		      channel->vc << 8 | channel->data_type << 10;
+		if (stream->is_compact)
+			val |= CSI_ENABLE_MIPI_COMPACT;
+		else
+			val &= ~CSI_ENABLE_MIPI_COMPACT;
+	} else if (mbus_type == V4L2_MBUS_CCP2) {
+		es_dvp2axi_csi_set_lvds_sav_eav(stream, channel);
+		val = LVDS_ENABLE_CAPTURE | LVDS_MODE(channel->lvds_cfg.mode) |
+		      LVDS_MAIN_LANE(0) | LVDS_FID(0) |
+		      LVDS_LANES_ENABLED(dev->active_sensor->lanes);
+
+		if (stream->is_compact)
+			val |= LVDS_COMPACT;
+		else
+			val &= ~LVDS_COMPACT;
+	}
+	if (stream->is_high_align)
+		val |= CSI_HIGH_ALIGN;
+	else
+		val &= ~CSI_HIGH_ALIGN;
+	es_dvp2axi_write_register(dev, get_reg_index_of_id_ctrl0(channel->id), val);
+
+	dev->intr_mask = es_dvp2axi_read_register(dev, DVP2AXI_REG_MIPI_LVDS_INTEN);
+	return 0;
+}
+
+static int es_dvp2axi_dvp_get_input_yuv_order(struct es_dvp2axi_stream *stream)
+{
+	unsigned int mask;
+	const struct dvp2axi_input_fmt *fmt = stream->dvp2axi_fmt_in;
+
+	switch (fmt->mbus_code) {
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+		mask = CSI_YUV_INPUT_ORDER_UYVY >> 11;
+		break;
+	case MEDIA_BUS_FMT_VYUY8_2X8:
+		mask = CSI_YUV_INPUT_ORDER_VYUY >> 11;
+		break;
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+		mask = CSI_YUV_INPUT_ORDER_YUYV >> 11;
+		break;
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+		mask = CSI_YUV_INPUT_ORDER_YVYU >> 11;
+		break;
+	default:
+		mask = CSI_YUV_INPUT_ORDER_UYVY >> 11;
+		break;
+	}
+	return mask;
+}
+
+static int es_dvp2axi_csi_get_output_type_mask(struct es_dvp2axi_stream *stream)
+{
+	unsigned int mask;
+	const struct dvp2axi_output_fmt *fmt = stream->dvp2axi_fmt_out;
+
+	switch (fmt->fourcc) {
+	case V4L2_PIX_FMT_NV16:
+		mask = CSI_WRDDR_TYPE_YUV422SP_EIC770X |
+		       CSI_YUV_OUTPUT_ORDER_UYVY;
+		break;
+	case V4L2_PIX_FMT_NV61:
+		mask = CSI_WRDDR_TYPE_YUV422SP_EIC770X |
+		       CSI_YUV_OUTPUT_ORDER_VYUY;
+		break;
+	case V4L2_PIX_FMT_NV12:
+		mask = CSI_WRDDR_TYPE_YUV420SP_EIC770X |
+		       CSI_YUV_OUTPUT_ORDER_UYVY;
+		break;
+	case V4L2_PIX_FMT_NV21:
+		mask = CSI_WRDDR_TYPE_YUV420SP_EIC770X |
+		       CSI_YUV_OUTPUT_ORDER_VYUY;
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		mask = CSI_WRDDR_TYPE_YUV_PACKET | CSI_YUV_OUTPUT_ORDER_YUYV;
+		break;
+	case V4L2_PIX_FMT_YVYU:
+		mask = CSI_WRDDR_TYPE_YUV_PACKET | CSI_YUV_OUTPUT_ORDER_YVYU;
+		break;
+	case V4L2_PIX_FMT_UYVY:
+		mask = CSI_WRDDR_TYPE_YUV_PACKET | CSI_YUV_OUTPUT_ORDER_UYVY;
+		break;
+	case V4L2_PIX_FMT_VYUY:
+		mask = CSI_WRDDR_TYPE_YUV_PACKET | CSI_YUV_OUTPUT_ORDER_VYUY;
+		break;
+	case V4L2_PIX_FMT_RGB24:
+	case V4L2_PIX_FMT_BGR24:
+	case V4L2_PIX_FMT_RGB565:
+	case V4L2_PIX_FMT_BGR666:
+		mask = CSI_WRDDR_TYPE_RAW_COMPACT;
+		break;
+	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SBGGR8:
+	case V4L2_PIX_FMT_SRGGB10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SRGGB12:
+	case V4L2_PIX_FMT_SGRBG12:
+	case V4L2_PIX_FMT_SGBRG12:
+	case V4L2_PIX_FMT_SBGGR12:
+	case V4L2_PIX_FMT_GREY:
+	case V4L2_PIX_FMT_Y10:
+	case V4L2_PIX_FMT_Y12:
+		if (stream->is_compact)
+			mask = CSI_WRDDR_TYPE_RAW_COMPACT;
+		else
+			mask = CSI_WRDDR_TYPE_RAW_UNCOMPACT;
+		break;
+	case V4L2_PIX_FMT_SBGGR16:
+	case V4L2_PIX_FMT_SGBRG16:
+	case V4L2_PIX_FMT_SGRBG16:
+	case V4L2_PIX_FMT_SRGGB16:
+	case V4L2_PIX_FMT_Y16:
+		mask = CSI_WRDDR_TYPE_RAW_UNCOMPACT;
+		break;
+	default:
+		mask = CSI_WRDDR_TYPE_RAW_COMPACT;
+		break;
+	}
+	return mask;
+}
+
+#if 0
+static int es_dvp2axi_lvds_get_output_type_mask(struct es_dvp2axi_stream *stream)
+{
+	unsigned int mask;
+	const struct dvp2axi_output_fmt *fmt = stream->dvp2axi_fmt_out;
+	int wr_type_offset = 0;
+	int yuvout_offset = 0;
+
+	switch (fmt->fourcc) {
+	case V4L2_PIX_FMT_NV16:
+		mask = (CSI_WRDDR_TYPE_YUV422SP_EIC770X << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_UYVY << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_NV61:
+		mask = (CSI_WRDDR_TYPE_YUV422SP_EIC770X << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_VYUY << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_NV12:
+		mask = (CSI_WRDDR_TYPE_YUV420SP_EIC770X << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_UYVY << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_NV21:
+		mask = (CSI_WRDDR_TYPE_YUV420SP_EIC770X << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_VYUY << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_YUYV << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_YVYU:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_YVYU << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_UYVY:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_UYVY << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_VYUY:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << wr_type_offset) |
+		       (CSI_YUV_OUTPUT_ORDER_VYUY << yuvout_offset);
+		break;
+	case V4L2_PIX_FMT_RGB24:
+	case V4L2_PIX_FMT_BGR24:
+	case V4L2_PIX_FMT_RGB565:
+	case V4L2_PIX_FMT_BGR666:
+		mask = CSI_WRDDR_TYPE_RAW_COMPACT << wr_type_offset;
+		break;
+	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SBGGR8:
+	case V4L2_PIX_FMT_SRGGB10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SRGGB12:
+	case V4L2_PIX_FMT_SGRBG12:
+	case V4L2_PIX_FMT_SGBRG12:
+	case V4L2_PIX_FMT_SBGGR12:
+	case V4L2_PIX_FMT_GREY:
+	case V4L2_PIX_FMT_Y10:
+	case V4L2_PIX_FMT_Y12:
+		if (stream->is_compact)
+			mask = CSI_WRDDR_TYPE_RAW_COMPACT << wr_type_offset;
+		else
+			mask = CSI_WRDDR_TYPE_RAW_UNCOMPACT << wr_type_offset;
+		break;
+	case V4L2_PIX_FMT_SBGGR16:
+	case V4L2_PIX_FMT_SGBRG16:
+	case V4L2_PIX_FMT_SGRBG16:
+	case V4L2_PIX_FMT_SRGGB16:
+	case V4L2_PIX_FMT_Y16:
+		mask = CSI_WRDDR_TYPE_RAW_UNCOMPACT << wr_type_offset;
+		break;
+	default:
+		mask = CSI_WRDDR_TYPE_RAW_COMPACT << wr_type_offset;
+		break;
+	}
+	return mask;
+}
+#endif
+
+#if 0
+static void es_dvp2axi_modify_frame_skip_config(struct es_dvp2axi_stream *stream)
+{
+	if (stream->skip_info.skip_to_en) {
+		es_dvp2axi_disable_skip_frame(stream);
+		es_dvp2axi_enable_skip_frame(stream, stream->skip_info.cap_m,
+					stream->skip_info.skip_n);
+		stream->skip_info.skip_to_en = false;
+	} else if (stream->skip_info.skip_to_dis) {
+		es_dvp2axi_disable_skip_frame(stream);
+	}
+}
+#endif
+
+/*config reg for eic770x*/
+static int es_dvp2axi_csi_channel_set_v1(struct es_dvp2axi_stream *stream,
+				    struct csi_channel_info *channel,
+				    enum v4l2_mbus_type mbus_type,
+				    unsigned int mode, int index)
+{
+	unsigned int val = 0x0;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_stream *detect_stream = &dev->stream[0];
+	struct sditf_priv *priv = dev->sditf[0];
+	struct esmodule_capture_info *capture_info = &channel->capture_info;
+	unsigned int wait_line = 0x3fff;
+	unsigned int dma_en = 0;
+	int offset = 0;
+
+	// pr_info("%s:%d yfx !!!!! \n ", __func__, __LINE__);
+	if (channel->id >= 4)
+		return -EINVAL;
+
+	if (!channel->enable) {
+		es_dvp2axi_write_register(dev,
+				     get_reg_index_of_id_ctrl0(channel->id),
+				     CSI_DISABLE_CAPTURE);
+		return 0;
+	}
+
+	es_dvp2axi_write_register_and(dev, DVP2AXI_REG_MIPI_LVDS_INTSTAT,
+				 ~(CSI_START_INTSTAT(channel->id) |
+				   CSI_DMA_END_INTSTAT(channel->id) |
+				   CSI_LINE_INTSTAT_V1(channel->id)));
+
+	if (!(capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE &&
+	      index < capture_info->multi_dev.dev_num - 1)) {
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					CSI_START_INTEN(channel->id));
+
+		if (priv && priv->mode.rdbk_mode &&
+		    detect_stream->is_line_wake_up) {
+			es_dvp2axi_write_register_or(
+				dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+				CSI_LINE_INTEN_EIC770X(channel->id));
+			wait_line = dev->wait_line;
+		}
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID0_1,
+				     wait_line << 16 | wait_line);
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID2_3,
+				     wait_line << 16 | wait_line);
+
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					CSI_DMA_END_INTEN(channel->id));
+
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					CSI_ALL_ERROR_INTEN_V1);
+	}
+	if (stream->dvp2axidev->id_use_cnt == 0) {
+		val = DVP2AXI_MIPI_LVDS_SW_PRESS_VALUE_EIC770X(0x3) |
+		      DVP2AXI_MIPI_LVDS_SW_PRESS_ENABLE |
+		      DVP2AXI_MIPI_LVDS_SW_HURRY_VALUE_EIC770X(0x3) |
+		      DVP2AXI_MIPI_LVDS_SW_HURRY_ENABLE |
+		      DVP2AXI_MIPI_LVDS_SW_WATER_LINE_25 |
+		      DVP2AXI_MIPI_LVDS_SW_WATER_LINE_ENABLE;
+		if (mbus_type == V4L2_MBUS_CSI2_DPHY ||
+		    mbus_type == V4L2_MBUS_CSI2_CPHY)
+			val &= ~DVP2AXI_MIPI_LVDS_SW_SEL_LVDS_RV1106;
+		else
+			val |= DVP2AXI_MIPI_LVDS_SW_SEL_LVDS_RV1106;
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL, val);
+	}
+	if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE && priv &&
+	    priv->mode.rdbk_mode == ESISP_VICAP_ONLINE &&
+	    (dev->hdr.hdr_mode == NO_HDR ||
+	     (dev->hdr.hdr_mode == HDR_X2 && stream->id == 1) ||
+	     (dev->hdr.hdr_mode == HDR_X3 && stream->id == 2)))
+		offset = channel->capture_info.multi_dev.pixel_offset;
+
+	es_dvp2axi_write_register(dev, get_reg_index_of_id_ctrl1(channel->id),
+			     (channel->width + offset) |
+				     (channel->height << 16));
+
+	if (channel->crop_en)
+		es_dvp2axi_write_register(
+			dev, get_reg_index_of_id_crop_start(channel->id),
+			channel->crop_st_y << 16 | channel->crop_st_x);
+
+	if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+		// pr_info("%s: %d  yfx !!!!! \n", __func__, __LINE__);
+		// pr_info("%s: %d  yfx assing new buffer one frame !!!!! \n", __func__, __LINE__);
+
+		es_dvp2axi_assign_new_buffer_oneframe(stream , ES_DVP2AXI_YUV_ADDR_STATE_INIT);
+		// es_dvp2axi_assign_new_buffer_pingpong(
+			// stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, channel->id);
+	}
+	else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP ||
+		 mode == ES_DVP2AXI_STREAM_MODE_TOISP_RDBK)
+		es_dvp2axi_assign_new_buffer_pingpong_toisp(
+			stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, channel->id);
+	else if (mode == ES_DVP2AXI_STREAM_MODE_ESKIT)
+		es_dvp2axi_assign_new_buffer_pingpong_eskit(
+			stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, channel->id);
+
+	if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE &&
+	    index == (capture_info->multi_dev.dev_num - 1) && priv &&
+	    priv->mode.rdbk_mode != ESISP_VICAP_ONLINE)
+		es_dvp2axi_write_register(
+			dev, get_reg_index_of_id_crop_start(channel->id),
+			channel->crop_st_y << 16 |
+				(channel->crop_st_x +
+				 capture_info->multi_dev.pixel_offset));
+
+	es_dvp2axi_write_register(dev, get_reg_index_of_frm0_y_vlw(channel->id),
+			     channel->virtual_width);
+
+	if (stream->lack_buf_cnt == 2)
+		stream->dma_en = 0;
+
+	if (stream->dma_en) {
+		// if (mbus_type == V4L2_MBUS_CSI2_DPHY ||
+		//     mbus_type == V4L2_MBUS_CSI2_CPHY)
+		// 	dma_en = CSI_DMA_ENABLE;
+
+		dma_en = CSI_DMA_ENABLE;
+	}
+	if (mbus_type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_type == V4L2_MBUS_CSI2_CPHY) {
+		if (stream->dvp2axidev->hdr.esp.mode == HDR_LINE_CNT ||
+		    stream->dvp2axidev->hdr.esp.mode == HDR_ID_CODE)
+			channel->vc = 0;
+
+		val = CSI_ENABLE_CAPTURE | dma_en | channel->cmd_mode_en << 26 |
+		      CSI_ENABLE_CROP_V1 | channel->vc << 8 |
+		      channel->data_type << 10 | channel->csi_fmt_val;
+
+		val |= stream->dvp2axi_fmt_in->csi_yuv_order;
+		val |= es_dvp2axi_csi_get_output_type_mask(stream);
+		if (stream->dvp2axidev->hdr.hdr_mode == NO_HDR ||
+		    stream->dvp2axidev->hdr.hdr_mode == HDR_COMPR)
+			val |= CSI_NO_HDR;
+		else if (stream->dvp2axidev->hdr.hdr_mode == HDR_X2)
+			val |= CSI_HDR2;
+		else if (stream->dvp2axidev->hdr.hdr_mode == HDR_X3)
+			val |= CSI_HDR3;
+		if (stream->dvp2axidev->hdr.esp.mode == HDR_NORMAL_VC)
+			val |= CSI_HDR_MODE_VC;
+		else if (stream->dvp2axidev->hdr.esp.mode == HDR_LINE_CNT)
+			val |= CSI_HDR_MODE_LINE_CNT;
+		else if (stream->dvp2axidev->hdr.esp.mode == HDR_ID_CODE)
+			val |= CSI_HDR_MODE_LINE_INFO;
+		if (stream->dvp2axidev->hdr.hdr_mode != NO_HDR &&
+		    stream->dvp2axidev->hdr.esp.mode == HDR_NORMAL_VC)
+			val |= CSI_HDR_VC_MODE_PROTECT;
+		if (stream->is_high_align)
+			val |= CSI_HIGH_ALIGN_EIC770X;
+		else
+			val &= ~CSI_HIGH_ALIGN_EIC770X;
+		es_dvp2axi_write_register(
+			dev, get_reg_index_of_id_ctrl0(channel->id), val);
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_EFFECT_CODE_ID0,
+				     0x02410251);
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_EFFECT_CODE_ID1,
+				     0x02420252);
+	} 
+	if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE) {
+		if (index == (capture_info->multi_dev.dev_num - 1))
+			stream->dvp2axidev->id_use_cnt++;
+	} else {
+		stream->dvp2axidev->id_use_cnt++;
+	}
+	if (!(capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE &&
+	      index < capture_info->multi_dev.dev_num - 1)) {
+		if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+			// pr_info("%s:%d yfx!!!! \n", __func__, __LINE__);
+			// es_dvp2axi_assign_new_buffer_pingpong(
+			// 	stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, channel->id);
+		}
+		else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP ||
+			 mode == ES_DVP2AXI_STREAM_MODE_TOISP_RDBK)
+			es_dvp2axi_assign_new_buffer_pingpong_toisp(
+				stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, channel->id);
+	}
+	dev->intr_mask = es_dvp2axi_read_register(dev, DVP2AXI_REG_MIPI_LVDS_INTEN);
+	return 0;
+}
+
+static int es_dvp2axi_csi_stream_start(struct es_dvp2axi_stream *stream,
+				  unsigned int mode)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *active_sensor = dev->active_sensor;
+	enum v4l2_mbus_type mbus_type = active_sensor->mbus.type;
+	struct csi_channel_info *channel;
+	u32 ret = 0;
+	int i;
+
+	if (stream->state < ES_DVP2AXI_STATE_STREAMING) {
+		stream->frame_idx = 0;
+		stream->buf_wake_up_cnt = 0;
+		stream->frame_phase = 0;
+		stream->lack_buf_cnt = 0;
+		stream->is_in_vblank = false;
+		stream->is_change_toisp = false;
+	}
+
+	channel = &dev->channels[stream->id];
+	channel->id = stream->id;
+	if (mbus_type == V4L2_MBUS_CCP2) {
+		ret = v4l2_subdev_call(dev->terminal_sensor.sd, core, ioctl,
+				       ESMODULE_GET_LVDS_CFG,
+				       &channel->lvds_cfg);
+		if (ret) {
+			v4l2_err(&dev->v4l2_dev,
+				 "Err: get lvds config failed!!\n");
+			return ret;
+		}
+	}
+	es_dvp2axi_csi_channel_init(stream, channel);
+	if (stream->state != ES_DVP2AXI_STATE_STREAMING) {
+		if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+			stream->dma_en |= ES_DVP2AXI_DMAEN_BY_VICAP;
+		} else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP_RDBK) {
+			stream->dma_en |= ES_DVP2AXI_DMAEN_BY_ISP;
+		} else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP) {
+			if (dev->hdr.hdr_mode == HDR_X2 && stream->id == 0)
+				stream->dma_en |= ES_DVP2AXI_DMAEN_BY_ISP;
+			else if (dev->hdr.hdr_mode == HDR_X3 &&
+				 (stream->id == 0 || stream->id == 1))
+				stream->dma_en |= ES_DVP2AXI_DMAEN_BY_ISP;
+		} else if (mode == ES_DVP2AXI_STREAM_MODE_ESKIT) {
+			stream->dma_en |= ES_DVP2AXI_DMAEN_BY_ESKIT;
+		}
+		if (stream->dvp2axidev->chip_id < CHIP_EIC770X_DVP2AXI) {
+			es_dvp2axi_csi_channel_set(stream, channel, mbus_type);
+		} else {
+			if (channel->capture_info.mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+				// pr_info("%s:%d yfx !!!!! combine one \n", __func__,__LINE__);
+				for (i = 0;
+				     i <
+				     channel->capture_info.multi_dev.dev_num;
+				     i++) {
+					dev->csi_host_idx =
+						channel->capture_info.multi_dev
+							.dev_idx[i];
+					es_dvp2axi_csi_channel_set_v1(stream,
+								 channel,
+								 mbus_type,
+								 mode, i);
+				}
+			} else {
+				// pr_info("%s:%d yfx !!!!! combine one \n", __func__,__LINE__);
+				es_dvp2axi_csi_channel_set_v1(stream, channel,
+							 mbus_type, mode, 0);
+			}
+		}
+	} else {
+		if (stream->dvp2axidev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+			if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+				stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_VICAP;
+			} else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP_RDBK) {
+				stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_ISP;
+			} else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP) {
+				if (dev->hdr.hdr_mode == HDR_X2 &&
+				    stream->id == 0 && (!stream->dma_en))
+					stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_ISP;
+				else if (dev->hdr.hdr_mode == HDR_X3 &&
+					 (stream->id == 0 || stream->id == 1) &&
+					 (!stream->dma_en))
+					stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_ISP;
+			} else if (mode == ES_DVP2AXI_STREAM_MODE_ESKIT) {
+				stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_ESKIT;
+			}
+		}
+	}
+	if (stream->state != ES_DVP2AXI_STATE_STREAMING) {
+		stream->line_int_cnt = 0;
+		if (stream->is_line_wake_up)
+			stream->is_can_stop = false;
+		else
+			stream->is_can_stop = true;
+		stream->state = ES_DVP2AXI_STATE_STREAMING;
+		dev->workmode = ES_DVP2AXI_WORKMODE_PINGPONG;
+	}
+
+	return 0;
+}
+
+static void es_dvp2axi_stream_stop(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dvp2axi_dev->active_sensor->mbus;
+	u32 val;
+	int id;
+	int i = 0;
+	stream->dvp2axidev->id_use_cnt--;
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		id = stream->id;
+		val = es_dvp2axi_read_register(dvp2axi_dev,
+					  get_reg_index_of_id_ctrl0(id));
+		if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+		    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY)
+			val &= ~(CSI_ENABLE_CAPTURE | CSI_DMA_ENABLE);
+		else
+			val &= ~LVDS_ENABLE_CAPTURE;
+
+		if (dvp2axi_dev->channels[id].capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			for (i = 0; i < dvp2axi_dev->channels[id]
+						.capture_info.multi_dev.dev_num;
+			     i++) {
+				dvp2axi_dev->csi_host_idx =
+					dvp2axi_dev->channels[id]
+						.capture_info.multi_dev
+						.dev_idx[i];
+				es_dvp2axi_write_register(
+					dvp2axi_dev, get_reg_index_of_id_ctrl0(id),
+					val);
+			}
+		} else {
+			es_dvp2axi_write_register(
+				dvp2axi_dev, get_reg_index_of_id_ctrl0(id), val);
+		}
+
+		es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTSTAT,
+					CSI_START_INTSTAT(id) |
+						CSI_DMA_END_INTSTAT(id) |
+						CSI_LINE_INTSTAT(id));
+
+		es_dvp2axi_write_register_and(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					 ~(CSI_START_INTEN(id) |
+					   CSI_DMA_END_INTEN(id) |
+					   CSI_LINE_INTEN(id)));
+
+		if (stream->dvp2axidev->chip_id < CHIP_EIC770X_DVP2AXI) {
+			es_dvp2axi_write_register_and(dvp2axi_dev,
+						 DVP2AXI_REG_MIPI_LVDS_INTEN,
+						 ~CSI_ALL_ERROR_INTEN);
+		} else {
+			if (stream->dvp2axidev->id_use_cnt == 0) {
+				es_dvp2axi_write_register_and(
+					dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					~CSI_ALL_ERROR_INTEN_V1);
+				if (dvp2axi_dev->channels[id].capture_info.mode ==
+				    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+					for (i = 0;
+					     i < dvp2axi_dev->channels[id]
+							 .capture_info.multi_dev
+							 .dev_num;
+					     i++) {
+						dvp2axi_dev->csi_host_idx =
+							dvp2axi_dev->channels[id]
+								.capture_info
+								.multi_dev
+								.dev_idx[i];
+						es_dvp2axi_write_register_and(
+							dvp2axi_dev,
+							DVP2AXI_REG_MIPI_LVDS_CTRL,
+							~CSI_ENABLE_CAPTURE);
+					}
+				} else {
+					es_dvp2axi_write_register_and(
+						dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_CTRL,
+						~CSI_ENABLE_CAPTURE);
+				}
+			}
+		}
+
+	} else {
+		if (atomic_read(&dvp2axi_dev->pipe.stream_cnt) == 1) {
+			val = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL);
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL,
+					     val & (~ENABLE_CAPTURE));
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTEN, 0x0);
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT,
+					     0x3ff);
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_FRAME_STATUS,
+					     0x0);
+		}
+	}
+	stream->state = ES_DVP2AXI_STATE_READY;
+	stream->dma_en = 0;
+}
+
+static bool es_dvp2axi_is_extending_line_for_height(struct es_dvp2axi_device *dev,
+					       struct es_dvp2axi_stream *stream,
+					       const struct dvp2axi_input_fmt *fmt)
+{
+	bool is_extended = false;
+
+	return is_extended;
+}
+
+static int es_dvp2axi_queue_setup(struct vb2_queue *queue, unsigned int *num_buffers,
+			     unsigned int *num_planes, unsigned int sizes[],
+			     struct device *alloc_ctxs[])
+{
+	struct es_dvp2axi_stream *stream = queue->drv_priv;
+	struct es_dvp2axi_extend_info *extend_line = &stream->extend_line;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	const struct v4l2_pix_format_mplane *pixm = NULL;
+	const struct dvp2axi_output_fmt *dvp2axi_fmt;
+	const struct dvp2axi_input_fmt *in_fmt;
+	bool is_extended = false;
+	u32 i, height;
+
+	pixm = &stream->pixm;
+	dvp2axi_fmt = stream->dvp2axi_fmt_out;
+	in_fmt = stream->dvp2axi_fmt_in;
+
+	*num_planes = dvp2axi_fmt->mplanes;
+
+	if (stream->crop_enable)
+		height = stream->crop[CROP_SRC_ACT].height;
+	else
+		height = pixm->height;
+
+	is_extended = es_dvp2axi_is_extending_line_for_height(dev, stream, in_fmt);
+	if (is_extended && extend_line->is_extended) {
+		height = extend_line->pixm.height;
+		pixm = &extend_line->pixm;
+	}
+
+	for (i = 0; i < dvp2axi_fmt->mplanes; i++) {
+		const struct v4l2_plane_pix_format *plane_fmt;
+		int h = round_up(height, MEMORY_ALIGN_ROUND_UP_HEIGHT);
+
+		plane_fmt = &pixm->plane_fmt[i];
+		sizes[i] = plane_fmt->sizeimage / height * h;
+	}
+	stream->total_buf_num = *num_buffers;
+	v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+		 "%s count %d, size %d, extended(%d, %d)\n",
+		 v4l2_type_names[queue->type], *num_buffers, sizes[0],
+		 is_extended, extend_line->is_extended);
+
+	return 0;
+}
+
+static void es_dvp2axi_check_buffer_update_pingpong(struct es_dvp2axi_stream *stream,
+					       int channel_id)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	struct es_dvp2axi_buffer *buffer = NULL;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+	u32 frm_addr_y = 0, frm_addr_uv = 0;
+	u32 frm0_addr_y = 0, frm0_addr_uv = 0;
+	u32 frm1_addr_y = 0, frm1_addr_uv = 0;
+	u32 buff_addr_y = 0, buff_addr_cbcr = 0;
+	struct esmodule_capture_info *capture_info =
+		&dev->channels[channel_id].capture_info;
+	unsigned long flags;
+	int frame_phase = 0;
+	bool is_dual_update_buf = false;
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	if (stream->state == ES_DVP2AXI_STATE_STREAMING &&
+	    ((stream->curr_buf == stream->next_buf &&
+	      stream->dvp2axi_fmt_in->field != V4L2_FIELD_INTERLACED &&
+	      (!dummy_buf->vaddr)) ||
+	     stream->curr_buf == NULL || stream->next_buf == NULL)) {
+		frame_phase = stream->frame_phase_cache;
+		if (!stream->is_line_wake_up ||
+		    (stream->is_line_wake_up && stream->frame_idx < 2)) {
+			if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+			    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+			    mbus_cfg->type == V4L2_MBUS_CCP2) {
+				frm0_addr_y = get_reg_index_of_frm0_y_addr(
+					channel_id);
+				frm1_addr_y = get_reg_index_of_frm1_y_addr(
+					channel_id);
+				frm0_addr_uv = get_reg_index_of_frm0_uv_addr(
+					channel_id);
+				frm1_addr_uv = get_reg_index_of_frm1_uv_addr(
+					channel_id);
+			} else {
+				frm0_addr_y = get_dvp_reg_index_of_frm0_y_addr(
+					channel_id);
+				frm1_addr_y = get_dvp_reg_index_of_frm1_y_addr(
+					channel_id);
+				frm0_addr_uv =
+					get_dvp_reg_index_of_frm0_uv_addr(
+						channel_id);
+				frm1_addr_uv =
+					get_dvp_reg_index_of_frm1_uv_addr(
+						channel_id);
+			}
+			if (frame_phase & DVP2AXI_CSI_FRAME0_READY) {
+				frm_addr_y = frm0_addr_y;
+				frm_addr_uv = frm0_addr_uv;
+			} else {
+				frm_addr_y = frm1_addr_y;
+				frm_addr_uv = frm1_addr_uv;
+			}
+			if (!stream->dma_en && stream->curr_buf == NULL &&
+			    stream->next_buf == NULL)
+				is_dual_update_buf = true;
+			if (!list_empty(&stream->buf_head)) {
+				if (frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+					stream->curr_buf = list_first_entry(
+						&stream->buf_head,
+						struct es_dvp2axi_buffer, queue);
+					if (stream->curr_buf) {
+						list_del(&stream->curr_buf
+								  ->queue);
+						buffer = stream->curr_buf;
+					}
+					if (buffer && is_dual_update_buf)
+						stream->next_buf = buffer;
+				} else if (frame_phase ==
+					   DVP2AXI_CSI_FRAME1_READY) {
+					if (stream->next_buf == NULL &&
+					    stream->dvp2axi_fmt_in->field ==
+						    V4L2_FIELD_INTERLACED) {
+						stream->next_buf =
+							stream->curr_buf;
+					} else {
+						stream->next_buf = list_first_entry(
+							&stream->buf_head,
+							struct es_dvp2axi_buffer,
+							queue);
+						if (stream->next_buf) {
+							list_del(
+								&stream->next_buf
+									 ->queue);
+							buffer =
+								stream->next_buf;
+						}
+						if (buffer &&
+						    is_dual_update_buf)
+							stream->curr_buf =
+								buffer;
+					}
+				}
+			} else {
+				v4l2_info(&dev->v4l2_dev, "%s %d\n", __func__,
+					  __LINE__);
+			}
+			if (buffer) {
+				if (is_dual_update_buf) {
+					buff_addr_y =
+						buffer->buff_addr[ES_DVP2AXI_PLANE_Y];
+					buff_addr_cbcr =
+						buffer->buff_addr
+							[ES_DVP2AXI_PLANE_CBCR];
+					if (capture_info->mode ==
+					    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+						es_dvp2axi_write_buff_addr_multi_dev_combine(
+							stream, frm0_addr_y,
+							frm0_addr_uv,
+							buff_addr_y,
+							buff_addr_cbcr, false);
+						es_dvp2axi_write_buff_addr_multi_dev_combine(
+							stream, frm1_addr_y,
+							frm1_addr_uv,
+							buff_addr_y,
+							buff_addr_cbcr, false);
+					} else {
+						es_dvp2axi_write_register(
+							dev, frm0_addr_y,
+							buff_addr_y);
+						if (stream->dvp2axi_fmt_out
+							    ->fmt_type !=
+						    DVP2AXI_FMT_TYPE_RAW)
+							es_dvp2axi_write_register(
+								dev,
+								frm0_addr_uv,
+								buff_addr_cbcr);
+						es_dvp2axi_write_register(
+							dev, frm1_addr_y,
+							buff_addr_y);
+						if (stream->dvp2axi_fmt_out
+							    ->fmt_type !=
+						    DVP2AXI_FMT_TYPE_RAW)
+							es_dvp2axi_write_register(
+								dev,
+								frm1_addr_uv,
+								buff_addr_cbcr);
+					}
+				} else {
+					buff_addr_y =
+						buffer->buff_addr[ES_DVP2AXI_PLANE_Y];
+					buff_addr_cbcr =
+						buffer->buff_addr
+							[ES_DVP2AXI_PLANE_CBCR];
+					if (capture_info->mode ==
+					    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+						es_dvp2axi_write_buff_addr_multi_dev_combine(
+							stream, frm_addr_y,
+							frm_addr_uv,
+							buff_addr_y,
+							buff_addr_cbcr, false);
+					} else {
+						es_dvp2axi_write_register(
+							dev, frm_addr_y,
+							buff_addr_y);
+						if (stream->dvp2axi_fmt_out
+							    ->fmt_type !=
+						    DVP2AXI_FMT_TYPE_RAW)
+							es_dvp2axi_write_register(
+								dev,
+								frm_addr_uv,
+								buff_addr_cbcr);
+					}
+				}
+			}
+		} else {
+			v4l2_dbg(3, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+				 "%s %d, is_wake_up %d, frame_idx %d\n",
+				 __func__, __LINE__, stream->is_line_wake_up,
+				 stream->frame_idx);
+			if (stream->curr_buf == stream->next_buf) {
+				if (stream->frame_phase_cache ==
+				    DVP2AXI_CSI_FRAME0_READY) {
+					stream->curr_buf = list_first_entry(
+						&stream->buf_head,
+						struct es_dvp2axi_buffer, queue);
+					v4l2_dbg(
+						3, es_dvp2axi_debug, &dev->v4l2_dev,
+						"%s %d, stream[%d] buf idx %d\n",
+						__func__, __LINE__, stream->id,
+						stream->curr_buf->vb.vb2_buf
+							.index);
+					if (stream->curr_buf)
+						list_del(&stream->curr_buf
+								  ->queue);
+				} else if (stream->frame_phase_cache ==
+					   DVP2AXI_CSI_FRAME1_READY) {
+					stream->next_buf = list_first_entry(
+						&stream->buf_head,
+						struct es_dvp2axi_buffer, queue);
+					v4l2_dbg(
+						4, es_dvp2axi_debug, &dev->v4l2_dev,
+						"%s %d, stream[%d] buf idx %d\n",
+						__func__, __LINE__, stream->id,
+						stream->next_buf->vb.vb2_buf
+							.index);
+					if (stream->next_buf)
+						list_del(&stream->next_buf
+								  ->queue);
+				}
+				stream->is_buf_active = true;
+			}
+		}
+		v4l2_dbg(
+			3, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			"%s, stream[%d] update buffer, frame_phase %d, is_stop %s, lack_buf_cnt %d\n",
+			__func__, stream->id, frame_phase,
+			(stream->dma_en ? "false" : "true"),
+			stream->lack_buf_cnt);
+		if (!stream->dma_en) {
+			if (stream->to_stop_dma) {
+				stream->to_stop_dma = 0;
+				wake_up(&stream->wq_stopped);
+			} else {
+				stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_VICAP;
+				v4l2_dbg(
+					3, es_dvp2axi_debug,
+					&stream->dvp2axidev->v4l2_dev,
+					"%s stream[%d] start dma capture, frame cnt %d\n",
+					__func__, stream->id,
+					stream->frame_idx);
+			}
+		} else {
+			v4l2_dbg(3, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+				 "%s %d, dma_en 0x%x, frame cnt %d\n", __func__,
+				 __LINE__, stream->dma_en, stream->frame_idx);
+		}
+		if (stream->lack_buf_cnt)
+			stream->lack_buf_cnt--;
+
+	} else {
+		v4l2_info(&dev->v4l2_dev,
+			  "%s %d, state %d, curr_buf %p, next_buf %p\n",
+			  __func__, __LINE__, stream->state, stream->curr_buf,
+			  stream->next_buf);
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	if (stream->to_en_dma)
+		es_dvp2axi_enable_dma_capture(stream, true);
+}
+
+/*
+ * The vb2_buffer are stored in es_dvp2axi_buffer, in order to unify
+ * mplane buffer and none-mplane buffer.
+ */
+void es_dvp2axi_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct es_dvp2axi_buffer *dvp2axibuf = to_es_dvp2axi_buffer(vbuf);
+	struct vb2_queue *queue = vb->vb2_queue;
+	struct es_dvp2axi_stream *stream = queue->drv_priv;
+	struct v4l2_pix_format_mplane *pixm = &stream->pixm;
+	const struct dvp2axi_output_fmt *fmt = stream->dvp2axi_fmt_out;
+	struct es_dvp2axi_hw *hw_dev = stream->dvp2axidev->hw_dev;
+	struct es_dvp2axi_tools_buffer *tools_buf = NULL;
+	struct es_dvp2axi_tools_vdev *tools_vdev = stream->tools_vdev;
+	unsigned long flags;
+	int i;
+	// static int first = 1;
+	bool is_find_tools_buf = false;
+
+	if (tools_vdev) {
+		spin_lock_irqsave(&stream->tools_vdev->vbq_lock, flags);
+		if (!list_empty(&tools_vdev->src_buf_head)) {
+			list_for_each_entry(tools_buf,
+					    &tools_vdev->src_buf_head, list) {
+				if (tools_buf->vb == vbuf) {
+					is_find_tools_buf = true;
+					break;
+				}
+			}
+			if (is_find_tools_buf) {
+				if (tools_buf->use_cnt)
+					tools_buf->use_cnt--;
+				if (tools_buf->use_cnt) {
+					spin_unlock_irqrestore(
+						&stream->tools_vdev->vbq_lock,
+						flags);
+					return;
+				}
+			}
+		}
+		spin_unlock_irqrestore(&stream->tools_vdev->vbq_lock, flags);
+	}
+
+	// uint32_t dvp2axi_out_addr_csr[] = { 0x24, 0x28, 0x2c };
+	memset(dvp2axibuf->buff_addr, 0, sizeof(dvp2axibuf->buff_addr));
+	/* If mplanes > 1, every c-plane has its own m-plane,
+	 * otherwise, multiple c-planes are in the same m-plane
+	 */
+	for (i = 0; i < fmt->mplanes; i++) {
+		void *addr = vb2_plane_vaddr(vb, i);
+
+		if (hw_dev->is_dma_sg_ops) {
+			struct sg_table *sgt = vb2_dma_sg_plane_desc(vb, i);
+
+			dvp2axibuf->buff_addr[i] = sg_dma_address(sgt->sgl);
+		} else {
+			dvp2axibuf->buff_addr[i] =
+				vb2_dma_contig_plane_dma_addr(vb, i);
+				// writel(dvp2axibuf->buff_addr[i], hw_dev->base_addr + dvp2axi_out_addr_csr[i%3]);
+				// DVP2AXI_HalWriteReg(hw_dev, dvp2axi_out_addr_csr[i%3], dvp2axibuf->buff_addr[i]);
+				//debug
+				// memset(addr, 0xff, pixm->plane_fmt[i].sizeimage);
+		}
+		if (es_dvp2axi_debug && addr && !hw_dev->iommu_en) {
+			//memset(addr, 255, pixm->plane_fmt[i].sizeimage);
+			v4l2_dbg(3, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+				 "Clear buffer, size: 0x%08x\n",
+				 pixm->plane_fmt[i].sizeimage);
+		}
+	}
+
+	if (fmt->mplanes == 1) {
+		for (i = 0; i < fmt->cplanes - 1; i++)
+			dvp2axibuf->buff_addr[i + 1] =
+				dvp2axibuf->buff_addr[i] +
+				pixm->plane_fmt[i].bytesperline * pixm->height;
+	}
+	// pr_info("%s:%d yfx !!!!! dvp2axibuf->buff_addr[0] = 0x%x \n", __func__, __LINE__, dvp2axibuf->buff_addr[0]);
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	list_add_tail(&dvp2axibuf->queue, &stream->buf_head);
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+	if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP && (!dvp2axibuf->dbuf)) {
+		struct esisp_rx_buf *dbufs = NULL;
+
+		dbufs = kzalloc(sizeof(struct esisp_rx_buf), GFP_KERNEL);
+
+		memset(dbufs, 0, sizeof(struct esisp_rx_buf));
+		if (stream->dvp2axidev->hdr.hdr_mode == HDR_X2 && stream->id == 0)
+			dbufs->type = BUF_MIDDLE;
+		else if (stream->dvp2axidev->hdr.hdr_mode == HDR_X3 &&
+			 stream->id == 0)
+			dbufs->type = BUF_LONG;
+		else if (stream->dvp2axidev->hdr.hdr_mode == HDR_X3 &&
+			 stream->id == 1)
+			dbufs->type = BUF_MIDDLE;
+		dvp2axibuf->dbuf = hw_dev->mem_ops->get_dmabuf(
+			vb, vb->planes[0].mem_priv, O_RDWR);
+		if (dvp2axibuf->dbuf)
+			dbufs->dbuf = dvp2axibuf->dbuf;
+		list_add_tail(&dbufs->list, &stream->rx_buf_head_vicap);
+	}
+	if (stream->dvp2axidev->workmode == ES_DVP2AXI_WORKMODE_PINGPONG &&
+	    stream->lack_buf_cnt &&
+	    stream->cur_stream_mode & ES_DVP2AXI_STREAM_MODE_CAPTURE)
+		es_dvp2axi_check_buffer_update_pingpong(stream, stream->id);
+	v4l2_dbg(3, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		 "stream[%d] buf queue, index: %d, dma_addr 0x%x\n", stream->id,
+		 vb->index, dvp2axibuf->buff_addr[0]);
+	atomic_inc(&stream->buf_cnt);
+}
+
+void es_dvp2axi_free_rx_buf(struct es_dvp2axi_stream *stream, int buf_num)
+{
+	struct es_dvp2axi_rx_buffer *buf;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct sditf_priv *priv = dev->sditf[0];
+	struct v4l2_subdev *sd;
+	int i = 0;
+	unsigned long flags;
+
+	if (!priv)
+		return;
+
+	sd = get_esisp_sd(dev->sditf[0]);
+	if (!sd)
+		return;
+
+	if ((dev->is_rtt_suspend || dev->is_aov_reserved) &&
+	    dev->is_thunderboot) {
+		stream->curr_buf_toisp = NULL;
+		stream->next_buf_toisp = NULL;
+		INIT_LIST_HEAD(&stream->rx_buf_head);
+
+		for (i = 0; i < buf_num; i++) {
+			buf = &stream->rx_buf[i];
+			if (buf->dbufs.is_init)
+				v4l2_subdev_call(sd, core, ioctl,
+						 ESISP_VICAP_CMD_RX_BUFFER_FREE,
+						 &buf->dbufs);
+			buf->dummy.is_free = true;
+		}
+		atomic_set(&stream->buf_cnt, 0);
+		stream->total_buf_num = 0;
+		stream->rx_buf_num = 0;
+
+		return;
+	}
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	stream->curr_buf_toisp = NULL;
+	stream->next_buf_toisp = NULL;
+	INIT_LIST_HEAD(&stream->rx_buf_head);
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	if (dev->is_thunderboot)
+		spin_lock_irqsave(&dev->buffree_lock, flags);
+	for (i = 0; i < buf_num; i++) {
+		buf = &stream->rx_buf[i];
+		if (buf->dummy.is_free)
+			continue;
+		if (buf->dbufs.is_init)
+			v4l2_subdev_call(sd, core, ioctl,
+					 ESISP_VICAP_CMD_RX_BUFFER_FREE,
+					 &buf->dbufs);
+		if (!dev->is_thunderboot)
+			es_dvp2axi_free_buffer(dev, &buf->dummy);
+		else
+			list_add_tail(&buf->list_free, &priv->buf_free_list);
+		atomic_dec(&stream->buf_cnt);
+		stream->total_buf_num--;
+	}
+	stream->rx_buf_num = 0;
+
+	if (dev->is_thunderboot) {
+		spin_unlock_irqrestore(&dev->buffree_lock, flags);
+		schedule_work(&priv->buffree_work.work);
+	}
+	stream->dma_en &= ~ES_DVP2AXI_DMAEN_BY_ISP;
+	v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		 "free rx_buf, buf_num %d\n", buf_num);
+}
+
+static void es_dvp2axi_get_resmem_head(struct es_dvp2axi_device *dvp2axi_dev);
+int es_dvp2axi_init_rx_buf(struct es_dvp2axi_stream *stream, int buf_num)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_pix_format_mplane *pixm = &stream->pixm;
+	struct es_dvp2axi_dummy_buffer *dummy;
+	struct es_dvp2axi_rx_buffer *buf;
+	struct sditf_priv *priv = dev->sditf[0];
+	int frm_type = 0;
+	int i = 0;
+	int ret = 0;
+
+	if (!priv)
+		return -EINVAL;
+
+	if (buf_num > ESISP_VICAP_BUF_CNT_MAX)
+		return -EINVAL;
+
+	if (dev->hdr.hdr_mode == NO_HDR) {
+		if (stream->id == 0)
+			frm_type = BUF_SHORT;
+		else
+			return -EINVAL;
+	} else if (dev->hdr.hdr_mode == HDR_X2) {
+		if (stream->id == 0)
+			frm_type = BUF_MIDDLE;
+		else if (stream->id == 1)
+			frm_type = BUF_SHORT;
+		else
+			return -EINVAL;
+	} else if (dev->hdr.hdr_mode == HDR_X3) {
+		if (stream->id == 0)
+			frm_type = BUF_LONG;
+		else if (stream->id == 1)
+			frm_type = BUF_MIDDLE;
+		else if (stream->id == 2)
+			frm_type = BUF_SHORT;
+		else
+			return -EINVAL;
+	}
+	while (true) {
+		buf = &stream->rx_buf[i];
+		memset(buf, 0, sizeof(*buf));
+		dummy = &buf->dummy;
+		dummy->size = pixm->plane_fmt[0].sizeimage;
+		dummy->is_need_vaddr = true;
+		dummy->is_need_dbuf = true;
+		if (dev->is_thunderboot || dev->is_rtt_suspend ||
+		    dev->is_aov_reserved) {
+			if (i == 0)
+				es_dvp2axi_get_resmem_head(dev);
+			buf->buf_idx = i;
+			ret = es_dvp2axi_alloc_reserved_mem_buf(dev, buf);
+			if (ret) {
+				stream->rx_buf_num = i;
+				v4l2_info(
+					&dev->v4l2_dev,
+					"reserved mem support alloc buf num %d, require buf num %d\n",
+					i, buf_num);
+				break;
+			}
+			if (dev->rdbk_debug)
+				v4l2_info(&dev->v4l2_dev,
+					  "stream[%d] buf addr 0x%llx\n",
+					  stream->id, (u64)dummy->dma_addr);
+		} else {
+			ret = es_dvp2axi_alloc_buffer(dev, dummy);
+			if (ret) {
+				stream->rx_buf_num = i;
+				v4l2_info(
+					&dev->v4l2_dev,
+					"alloc buf num %d, require buf num %d\n",
+					i, buf_num);
+				break;
+			}
+			buf->dbufs.dbuf = dummy->dbuf;
+		}
+		buf->dbufs.is_init = false;
+		buf->dbufs.type = frm_type;
+		list_add_tail(&buf->list, &stream->rx_buf_head);
+		dummy->is_free = false;
+		if (stream->is_compact)
+			buf->dbufs.is_uncompact = false;
+		else
+			buf->dbufs.is_uncompact = true;
+		if (priv && i == 0) {
+			buf->dbufs.is_first = true;
+			if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE)
+				es_dvp2axi_s_rx_buffer(stream, &buf->dbufs);
+		}
+		i++;
+		if (!dev->is_thunderboot && i >= buf_num) {
+			stream->rx_buf_num = buf_num;
+			break;
+		} else if (i >= ESISP_VICAP_BUF_CNT_MAX) {
+			stream->rx_buf_num = i;
+			v4l2_info(&dev->v4l2_dev,
+				  "reserved mem alloc buf num %d\n", i);
+			break;
+		}
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+			 "init rx_buf,dma_addr 0x%llx size: 0x%x\n",
+			 (u64)dummy->dma_addr, pixm->plane_fmt[0].sizeimage);
+	}
+	if (stream->rx_buf_num) {
+		stream->total_buf_num = stream->rx_buf_num;
+		atomic_set(&stream->buf_cnt, stream->rx_buf_num);
+		return 0;
+	} else {
+		return -EINVAL;
+	}
+}
+
+static int es_dvp2axi_create_dummy_buf(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_hw *hw = dev->hw_dev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &hw->dummy_buf;
+	struct es_dvp2axi_device *tmp_dev = NULL;
+	struct v4l2_subdev_frame_interval_enum fie;
+	struct v4l2_subdev_format fmt;
+	u32 max_size = 0;
+	u32 size = 0;
+	int ret = 0;
+	int i, j;
+	
+	for (i = 0; i < hw->dev_num; i++) {
+		tmp_dev = hw->dvp2axi_dev[i];
+		if (tmp_dev->terminal_sensor.sd) {
+			for (j = 0; j < 32; j++) {
+				memset(&fie, 0, sizeof(fie));
+				fie.index = j;
+				fie.pad = 0;
+				fie.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+				ret = v4l2_subdev_call(
+					tmp_dev->terminal_sensor.sd, pad,
+					enum_frame_interval, NULL, &fie);
+				if (!ret) {
+					if (fie.code ==
+						    MEDIA_BUS_FMT_RGB888_1X24 ||
+					    fie.code ==
+						    MEDIA_BUS_FMT_BGR888_1X24 ||
+					    fie.code ==
+						    MEDIA_BUS_FMT_GBR888_1X24)
+						size = fie.width * fie.height *
+						       3;
+					else
+						size = fie.width * fie.height *
+						       2;
+					v4l2_dbg(
+						1, es_dvp2axi_debug, &dev->v4l2_dev,
+						"%s enum fmt, width %d, height %d\n",
+						__func__, fie.width,
+						fie.height);
+				} else {
+					break;
+				}
+				if (size > max_size)
+					max_size = size;
+			}
+		} else {
+			continue;
+		}
+	}
+
+	if (max_size == 0 && dev->terminal_sensor.sd) {
+		fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		ret = v4l2_subdev_call(dev->terminal_sensor.sd, pad, get_fmt,
+				       NULL, &fmt);
+		if (!ret) {
+			if (fmt.format.code == MEDIA_BUS_FMT_RGB888_1X24 ||
+			    fmt.format.code == MEDIA_BUS_FMT_BGR888_1X24 ||
+			    fmt.format.code == MEDIA_BUS_FMT_GBR888_1X24)
+				size = fmt.format.width * fmt.format.height * 3;
+			else
+				size = fmt.format.width * fmt.format.height * 2;
+			if (size > max_size)
+				max_size = size;
+		}
+	}
+
+	dummy_buf->size = max_size;
+
+	dummy_buf->is_need_vaddr = true;
+	dummy_buf->is_need_dbuf = true;
+	ret = es_dvp2axi_alloc_buffer(dev, dummy_buf);
+	if (ret) {
+		v4l2_err(&dev->v4l2_dev,
+			 "Failed to allocate the memory for dummy buffer\n");
+		return -ENOMEM;
+	}
+
+	v4l2_info(&dev->v4l2_dev, "Allocate dummy buffer, size: 0x%08x\n",
+		  dummy_buf->size);
+
+	return ret;
+}
+
+static void es_dvp2axi_destroy_dummy_buf(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
+
+	if (dummy_buf->vaddr)
+		es_dvp2axi_free_buffer(dev, dummy_buf);
+	dummy_buf->dma_addr = 0;
+	dummy_buf->vaddr = NULL;
+}
+
+static void es_dvp2axi_do_cru_reset(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev->hw_dev;
+
+	unsigned int val, i;
+	if (dev->luma_vdev.enable)
+		es_dvp2axi_stop_luma(&dev->luma_vdev);
+
+	if (dev->hdr.hdr_mode != NO_HDR) {
+		val = es_dvp2axi_read_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL);
+		val |= DVP2AXI_MIPI_LVDS_SW_DMA_IDLE;
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_MIPI_LVDS_CTRL, val);
+
+		udelay(5);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dvp2axi_hw->dvp2axi_rst); i++)
+		if (dvp2axi_hw->dvp2axi_rst[i])
+			reset_control_assert(dvp2axi_hw->dvp2axi_rst[i]);
+
+	udelay(10);
+
+	for (i = 0; i < ARRAY_SIZE(dvp2axi_hw->dvp2axi_rst); i++)
+		if (dvp2axi_hw->dvp2axi_rst[i])
+			reset_control_deassert(dvp2axi_hw->dvp2axi_rst[i]);
+}
+
+void es_dvp2axi_do_soft_reset(struct es_dvp2axi_device *dev)
+{
+	if (dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+	    dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+	    dev->active_sensor->mbus.type == V4L2_MBUS_CCP2)
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_MIPI_LVDS_CTRL,
+					0x000A0000);
+	else
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_CTRL, 0x000A0000);
+	usleep_range(10, 20);
+	v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev, "vicap do soft reset 0x%x\n",
+		 0x000A0000);
+}
+
+static void es_dvp2axi_release_rdbk_buf(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_buffer *rdbk_buf = NULL;
+	struct es_dvp2axi_buffer *tmp_buf = NULL;
+	unsigned long flags;
+	bool has_added;
+	int index = 0;
+
+	if (stream->id == ES_DVP2AXI_STREAM_MIPI_ID0)
+		index = RDBK_L;
+	else if (stream->id == ES_DVP2AXI_STREAM_MIPI_ID1)
+		index = RDBK_M;
+	else if (stream->id == ES_DVP2AXI_STREAM_MIPI_ID2)
+		index = RDBK_S;
+	else
+		return;
+
+	spin_lock_irqsave(&dev->hdr_lock, flags);
+	rdbk_buf = dev->rdbk_buf[index];
+	if (rdbk_buf) {
+		if (rdbk_buf != stream->curr_buf &&
+		    rdbk_buf != stream->next_buf) {
+			has_added = false;
+
+			list_for_each_entry(tmp_buf, &stream->buf_head, queue) {
+				if (tmp_buf == rdbk_buf) {
+					has_added = true;
+					break;
+				}
+			}
+
+			if (!has_added)
+				list_add_tail(&rdbk_buf->queue,
+					      &stream->buf_head);
+		}
+		dev->rdbk_buf[index] = NULL;
+	}
+	spin_unlock_irqrestore(&dev->hdr_lock, flags);
+}
+
+static void es_dvp2axi_detach_sync_mode(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	int i = 0;
+	struct es_dvp2axi_hw *hw = dvp2axi_dev->hw_dev;
+	struct es_dvp2axi_device *tmp_dev;
+	struct es_dvp2axi_multi_sync_config *sync_config;
+
+	if ((!dvp2axi_dev->sync_cfg.type) ||
+	    (atomic_read(&dvp2axi_dev->pipe.stream_cnt) != 0))
+		return;
+	mutex_lock(&hw->dev_lock);
+	memset(&dvp2axi_dev->sync_cfg, 0, sizeof(dvp2axi_dev->sync_cfg));
+	sync_config = &hw->sync_config[dvp2axi_dev->sync_cfg.group];
+	sync_config->streaming_cnt--;
+	if (dvp2axi_dev->sync_cfg.type == EXTERNAL_MASTER_MODE) {
+		for (i = 0; i < sync_config->ext_master.count; i++) {
+			tmp_dev = sync_config->ext_master.dvp2axi_dev[i];
+			if (tmp_dev == dvp2axi_dev) {
+				sync_config->ext_master.is_streaming[i] = false;
+				break;
+			}
+		}
+	}
+	if (dvp2axi_dev->sync_cfg.type == INTERNAL_MASTER_MODE)
+		sync_config->int_master.is_streaming[0] = false;
+	if (dvp2axi_dev->sync_cfg.type == SLAVE_MODE) {
+		for (i = 0; i < sync_config->slave.count; i++) {
+			tmp_dev = sync_config->slave.dvp2axi_dev[i];
+			if (tmp_dev == dvp2axi_dev) {
+				sync_config->slave.is_streaming[i] = false;
+				break;
+			}
+		}
+	}
+
+	if (!sync_config->streaming_cnt && sync_config->is_attach) {
+		sync_config->is_attach = false;
+		sync_config->mode = ES_DVP2AXI_NOSYNC_MODE;
+		sync_config->dev_cnt = 0;
+	}
+	mutex_unlock(&hw->dev_lock);
+}
+
+void es_dvp2axi_do_stop_stream(struct es_dvp2axi_stream *stream,
+			  enum es_dvp2axi_stream_mode mode)
+{
+	struct es_dvp2axi_vdev_node *node = &stream->vnode;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+	struct es_dvp2axi_buffer *buf = NULL;
+	int ret;
+	struct es_dvp2axi_hw *hw_dev = dev->hw_dev;
+	bool can_reset = true;
+	int i;
+	unsigned long flags;
+	u32 vblank = 0;
+	u32 frame_time_ns = 0;
+	u64 cur_time = 0;
+	u64 fs_time = 0;
+
+	mutex_lock(&dev->stream_lock);
+
+	v4l2_info(&dev->v4l2_dev,
+		  "stream[%d] start stopping, total mode 0x%x, cur 0x%x\n",
+		  stream->id, stream->cur_stream_mode, mode);
+
+	if (mode == stream->cur_stream_mode) {
+		if (stream->dma_en) {
+			if (!dev->sensor_linetime)
+				dev->sensor_linetime =
+					es_dvp2axi_get_linetime(stream);
+			vblank = es_dvp2axi_get_sensor_vblank(dev);
+			if (vblank) {
+				frame_time_ns =
+					(vblank +
+					 dev->terminal_sensor.raw_rect.height) *
+					dev->sensor_linetime;
+				spin_lock_irqsave(&stream->fps_lock, flags);
+				fs_time = stream->readout.fs_timestamp;
+				spin_unlock_irqrestore(&stream->fps_lock,
+						       flags);
+				cur_time = es_dvp2axi_time_get_ns(dev);
+				if (cur_time > fs_time &&
+				    cur_time - fs_time <
+					    (frame_time_ns - 10000000)) {
+					spin_lock_irqsave(&stream->vbq_lock,
+							  flags);
+					if (stream->dma_en &
+					    ES_DVP2AXI_DMAEN_BY_VICAP)
+						stream->to_stop_dma =
+							ES_DVP2AXI_DMAEN_BY_VICAP;
+					else if (stream->dma_en &
+						 ES_DVP2AXI_DMAEN_BY_ISP)
+						stream->to_stop_dma =
+							ES_DVP2AXI_DMAEN_BY_ISP;
+					stream->is_stop_capture = true;
+					es_dvp2axi_stop_dma_capture(stream);
+					spin_unlock_irqrestore(
+						&stream->vbq_lock, flags);
+				}
+			}
+		}
+		stream->stopping = true;
+		ret = wait_event_timeout(stream->wq_stopped,
+					 stream->state != ES_DVP2AXI_STATE_STREAMING,
+					 msecs_to_jiffies(500));
+		if (!ret) {
+			es_dvp2axi_stream_stop(stream);
+			stream->stopping = false;
+		}
+
+		video_device_pipeline_stop(&node->vdev);
+		ret = dev->pipe.set_stream(&dev->pipe, false);
+		if (ret < 0)
+			v4l2_err(v4l2_dev,
+				 "pipeline stream-off failed error:%d\n", ret);
+
+		dev->is_start_hdr = false;
+		stream->is_dvp_yuv_addr_init = false;
+		if (stream->skip_info.skip_en) {
+			stream->skip_info.skip_en = false;
+			stream->skip_info.skip_to_en = true;
+		}
+	} else if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE &&
+		   stream->dma_en & ES_DVP2AXI_DMAEN_BY_VICAP) {
+		//only stop dma
+		stream->to_stop_dma = ES_DVP2AXI_DMAEN_BY_VICAP;
+		stream->is_wait_dma_stop = true;
+		wait_event_timeout(stream->wq_stopped,
+				   !stream->is_wait_dma_stop,
+				   msecs_to_jiffies(1000));
+	} else if (mode == ES_DVP2AXI_STREAM_MODE_TOISP &&
+		   stream->dma_en & ES_DVP2AXI_DMAEN_BY_VICAP) {
+		//only stop dma
+		stream->to_stop_dma = ES_DVP2AXI_DMAEN_BY_ISP;
+		stream->is_wait_dma_stop = true;
+		wait_event_timeout(stream->wq_stopped,
+				   !stream->is_wait_dma_stop,
+				   msecs_to_jiffies(1000));
+	}
+	if ((mode & ES_DVP2AXI_STREAM_MODE_CAPTURE) == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+		/* release buffers */
+		spin_lock_irqsave(&stream->vbq_lock, flags);
+		if (stream->curr_buf)
+			list_add_tail(&stream->curr_buf->queue,
+				      &stream->buf_head);
+		if (stream->next_buf && stream->next_buf != stream->curr_buf)
+			list_add_tail(&stream->next_buf->queue,
+				      &stream->buf_head);
+		spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+		if (dev->hdr.hdr_mode == HDR_X2 || dev->hdr.hdr_mode == HDR_X3)
+			es_dvp2axi_release_rdbk_buf(stream);
+
+		stream->curr_buf = NULL;
+		stream->next_buf = NULL;
+
+		es_dvp2axi_rx_buffer_free(stream);
+		list_for_each_entry(buf, &stream->buf_head, queue) {
+			v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "stream[%d] buf return addr 0x%x\n",
+				 stream->id, buf->buff_addr[0]);
+			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		}
+		INIT_LIST_HEAD(&stream->buf_head);
+		while (!list_empty(&stream->vb_done_list)) {
+			buf = list_first_entry(&stream->vb_done_list,
+					       struct es_dvp2axi_buffer, queue);
+			if (buf) {
+				list_del(&buf->queue);
+				vb2_buffer_done(&buf->vb.vb2_buf,
+						VB2_BUF_STATE_ERROR);
+			}
+		}
+		stream->total_buf_num = 0;
+		atomic_set(&stream->buf_cnt, 0);
+		stream->lack_buf_cnt = 0;
+		stream->dma_en &= ~ES_DVP2AXI_DMAEN_BY_VICAP;
+	}
+
+	if (mode == stream->cur_stream_mode) {
+		ret = dev->pipe.close(&dev->pipe);
+		if (ret < 0)
+			v4l2_err(v4l2_dev, "pipeline close failed error:%d\n",
+				 ret);
+		if (dev->hdr.hdr_mode == HDR_X2) {
+			if (dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0].state ==
+				    ES_DVP2AXI_STATE_READY &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1].state ==
+				    ES_DVP2AXI_STATE_READY) {
+				dev->can_be_reset = true;
+			}
+		} else if (dev->hdr.hdr_mode == HDR_X3) {
+			if (dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0].state ==
+				    ES_DVP2AXI_STATE_READY &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1].state ==
+				    ES_DVP2AXI_STATE_READY &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID2].state ==
+				    ES_DVP2AXI_STATE_READY) {
+				dev->can_be_reset = true;
+			}
+		} else {
+			dev->can_be_reset = true;
+		}
+		mutex_lock(&hw_dev->dev_lock);
+		for (i = 0; i < hw_dev->dev_num; i++) {
+			if (atomic_read(&hw_dev->dvp2axi_dev[i]->pipe.stream_cnt) !=
+			    0) {
+				can_reset = false;
+				break;
+			}
+		}
+		mutex_unlock(&hw_dev->dev_lock);
+		if (dev->can_be_reset && dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+			es_dvp2axi_do_soft_reset(dev);
+			atomic_set(&dev->streamoff_cnt, 0);
+		}
+		if (dev->can_be_reset && can_reset) {
+			dev->can_be_reset = false;
+			dev->reset_work_cancel = true;
+			dev->early_line = 0;
+			dev->sensor_linetime = 0;
+			dev->wait_line = 0;
+			stream->is_line_wake_up = false;
+		}
+		if (can_reset && hw_dev->dummy_buf.vaddr)
+			es_dvp2axi_destroy_dummy_buf(stream);
+	}
+	if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE)
+		tasklet_disable(&stream->vb_done_tasklet);
+
+	stream->cur_stream_mode &= ~mode;
+	INIT_LIST_HEAD(&stream->vb_done_list);
+	v4l2_info(&dev->v4l2_dev, "stream[%d] stopping finished, dma_en 0x%x\n",
+		  stream->id, stream->dma_en);
+	mutex_unlock(&dev->stream_lock);
+	es_dvp2axi_detach_sync_mode(dev);
+}
+
+static void es_dvp2axi_stop_streaming(struct vb2_queue *queue)
+{
+	struct es_dvp2axi_stream *stream = queue->drv_priv;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	int count = 0;
+	uint32_t soft_rstn;
+	uint32_t csr0 = DVP2AXI_HalReadReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR);
+
+	pr_info("%s IN\n",  __func__);
+
+	csr0 &= (~0x1f);  // disable dvp chn
+    DVP2AXI_HalWriteReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR, csr0);
+
+	
+	dvp2axi_hw_int_mask(stream->dvp2axidev->hw_dev, 1);
+
+	// do software reset
+	soft_rstn = DVP2AXI_HalReadReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR) & (~VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+	DVP2AXI_HalWriteReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR, soft_rstn);
+	do {
+		soft_rstn = DVP2AXI_HalReadReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR);
+		udelay(1000);
+    } while ((soft_rstn & VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK) != VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK && count++ < 100);
+    	// release reset
+    DVP2AXI_HalWriteReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR, soft_rstn | VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+	
+	if(count >= 100)
+		pr_err("dvp2axi soft reset timeout\n");
+
+    DVP2AXI_HalWriteReg(stream->dvp2axidev->hw_dev, VI_DVP2AXI_CTRL0_CSR, soft_rstn | VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+	
+	es_dvp2axi_do_stop_stream(stream, ES_DVP2AXI_STREAM_MODE_CAPTURE);
+	
+	if (dev->irq_stats.not_active_buf_cnt[stream->id] > 0)
+		pr_warn("%s: stream %d discard frame num %llu \n", __func__, stream->id, dev->irq_stats.not_active_buf_cnt[stream->id]);
+
+	pr_info("%s OUT\n",  __func__);
+}
+
+/*
+ * DVP2AXI supports the following input modes,
+ *    YUV, the default mode
+ *    PAL,
+ *    NTSC,
+ *    RAW, if the input format is raw bayer
+ *    JPEG, TODO
+ *    MIPI, TODO
+ */
+static u32 es_dvp2axi_determine_input_mode(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor_info = dev->active_sensor;
+	u32 mode = INPUT_MODE_YUV;
+	v4l2_std_id std;
+	int ret;
+
+	ret = v4l2_subdev_call(sensor_info->sd, video, querystd, &std);
+	if (ret == 0) {
+		/* retrieve std from sensor if exist */
+		switch (std) {
+		case V4L2_STD_NTSC:
+			mode = INPUT_MODE_NTSC;
+			break;
+		case V4L2_STD_PAL:
+			mode = INPUT_MODE_PAL;
+			break;
+		case V4L2_STD_ATSC:
+			mode = INPUT_MODE_BT1120;
+			break;
+		default:
+			v4l2_err(&dev->v4l2_dev, "std: %lld is not supported",
+				 std);
+		}
+	} else {
+		/* determine input mode by mbus_code (fmt_type) */
+		switch (stream->dvp2axi_fmt_in->fmt_type) {
+		case DVP2AXI_FMT_TYPE_YUV:
+			mode = INPUT_MODE_YUV;
+			break;
+		case DVP2AXI_FMT_TYPE_RAW:
+			mode = INPUT_MODE_RAW;
+			break;
+		}
+	}
+
+	return mode;
+}
+
+static u32 es_dvp2axi_determine_input_mode_eic770x(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor_info = dev->active_sensor;
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dev->terminal_sensor;
+	__u32 intf = BT656_STD_RAW;
+	u32 mode = INPUT_MODE_YUV;
+	v4l2_std_id std;
+	int ret;
+
+	ret = v4l2_subdev_call(sensor_info->sd, video, querystd, &std);
+	if (ret == 0) {
+		/* retrieve std from sensor if exist */
+		switch (std) {
+		case V4L2_STD_NTSC:
+		case V4L2_STD_PAL:
+			mode = INPUT_BT656_YUV422;
+			break;
+		case V4L2_STD_ATSC:
+			mode = INPUT_BT1120_YUV422;
+			break;
+		default:
+			v4l2_err(&dev->v4l2_dev, "std: %lld is not supported",
+				 std);
+		}
+		mode |= CSI_WRDDR_TYPE_RAW8 << 6;
+	} else {
+		/* determine input mode by mbus_code (fmt_type) */
+		switch (stream->dvp2axi_fmt_in->fmt_type) {
+		case DVP2AXI_FMT_TYPE_YUV:
+			if (sensor_info->mbus.type == V4L2_MBUS_BT656) {
+				if ((sensor_info->mbus.bus.parallel.flags &
+				     DVP2AXI_DVP_PCLK_DUAL_EDGE) ==
+				    DVP2AXI_DVP_PCLK_DUAL_EDGE)
+					mode = INPUT_BT1120_YUV422;
+				else
+					mode = INPUT_BT656_YUV422;
+			} else {
+				mode = INPUT_BT601_YUV422;
+			}
+			mode |= CSI_WRDDR_TYPE_RAW8 << 6;
+			break;
+		case DVP2AXI_FMT_TYPE_RAW:
+			ret = v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+					       ESMODULE_GET_BT656_INTF_TYPE,
+					       &intf);
+			if (!ret) {
+				if (intf == BT656_SONY_RAW)
+					mode = INPUT_SONY_RAW;
+				else
+					mode = INPUT_BT601_RAW;
+			} else {
+				mode = INPUT_BT601_RAW;
+			}
+			mode |= stream->dvp2axi_fmt_in->csi_fmt_val << 6;
+			break;
+		}
+	}
+	if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_NONE)
+		mode |= TRANSMIT_PROGRESS_EIC770X;
+	else
+		mode |= TRANSMIT_INTERFACE_EIC770X;
+	return mode;
+}
+
+static inline u32 es_dvp2axi_scl_ctl(struct es_dvp2axi_stream *stream)
+{
+	u32 fmt_type = stream->dvp2axi_fmt_in->fmt_type;
+	return (fmt_type == DVP2AXI_FMT_TYPE_YUV) ? ENABLE_YUV_16BIT_BYPASS :
+						ENABLE_RAW_16BIT_BYPASS;
+}
+
+/**
+ * es_dvp2axi_align_bits_per_pixel() - return the bit width of per pixel for stored
+ * In raw or jpeg mode, data is stored by 16-bits,so need to align it.
+ */
+static u32 es_dvp2axi_align_bits_per_pixel(struct es_dvp2axi_stream *stream,
+				      const struct dvp2axi_output_fmt *fmt,
+				      int plane_index)
+{
+	u32 bpp = 0, i, cal = 0;
+
+	if (fmt) {
+		switch (fmt->fourcc) {
+		case V4L2_PIX_FMT_NV16:
+		case V4L2_PIX_FMT_NV61:
+		case V4L2_PIX_FMT_NV12:
+		case V4L2_PIX_FMT_NV21:
+		case V4L2_PIX_FMT_GREY:
+		case V4L2_PIX_FMT_Y16:
+			bpp = fmt->bpp[plane_index];
+			break;
+		case V4L2_PIX_FMT_YUYV:
+		case V4L2_PIX_FMT_YVYU:
+		case V4L2_PIX_FMT_UYVY:
+		case V4L2_PIX_FMT_VYUY:
+		case V4L2_PIX_FMT_Y210:
+			if (stream->dvp2axidev->chip_id < CHIP_EIC770X_DVP2AXI)
+				bpp = fmt->bpp[plane_index];
+			else
+				bpp = fmt->bpp[plane_index + 1];
+			break;
+		case V4L2_PIX_FMT_RGB24:
+		case V4L2_PIX_FMT_BGR24:
+		case V4L2_PIX_FMT_RGB565:
+		case V4L2_PIX_FMT_BGR666:
+		case V4L2_PIX_FMT_SRGGB8:
+		case V4L2_PIX_FMT_SGRBG8:
+		case V4L2_PIX_FMT_SGBRG8:
+		case V4L2_PIX_FMT_SBGGR8:
+		case V4L2_PIX_FMT_SRGGB10:
+		case V4L2_PIX_FMT_SGRBG10:
+		case V4L2_PIX_FMT_SGBRG10:
+		case V4L2_PIX_FMT_SBGGR10:
+		case V4L2_PIX_FMT_SRGGB12:
+		case V4L2_PIX_FMT_SGRBG12:
+		case V4L2_PIX_FMT_SGBRG12:
+		case V4L2_PIX_FMT_SBGGR12:
+		case V4L2_PIX_FMT_SBGGR16:
+		case V4L2_PIX_FMT_SGBRG16:
+		case V4L2_PIX_FMT_SGRBG16:
+		case V4L2_PIX_FMT_SRGGB16:
+		case V4l2_PIX_FMT_SPD16:
+		case V4l2_PIX_FMT_EBD8:
+		case V4L2_PIX_FMT_Y10:
+		case V4L2_PIX_FMT_Y12:
+			bpp = max(fmt->bpp[plane_index],
+			(u8)DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126);
+	  		cal = DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126;
+			for (i = 1; i < 5; i++) {
+				if (i * cal >= bpp) {
+					bpp = i * cal;
+					break;
+				}
+			}
+			break;
+		default:
+			v4l2_err(&stream->dvp2axidev->v4l2_dev,
+				 "fourcc: %d is not supported!\n", fmt->fourcc);
+			break;
+		}
+	}
+
+	return bpp;
+}
+
+/**
+ * es_dvp2axi_cal_raw_vir_line_ratio() - return ratio for virtual line width setting
+ * In raw or jpeg mode, data is stored by 16-bits,
+ * so need to align virtual line width.
+ */
+static u32 es_dvp2axi_cal_raw_vir_line_ratio(struct es_dvp2axi_stream *stream,
+					const struct dvp2axi_output_fmt *fmt)
+{
+	u32 ratio = 0, bpp = 0;
+	if (fmt) {
+		bpp = es_dvp2axi_align_bits_per_pixel(stream, fmt, 0);
+		ratio = bpp / DVP2AXI_YUV_STORED_BIT_WIDTH;
+	}
+
+	return ratio;
+}
+
+static void es_dvp2axi_sync_crop_info(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_subdev_selection input_sel;
+	int ret;
+	if (dev->terminal_sensor.sd) {
+		input_sel.target = V4L2_SEL_TGT_CROP_BOUNDS;
+		input_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		input_sel.pad = 0;
+		ret = v4l2_subdev_call(dev->terminal_sensor.sd, pad,
+				       get_selection, NULL, &input_sel);
+		if (!ret) {
+			stream->crop[CROP_SRC_SENSOR] = input_sel.r;
+			stream->crop_enable = true;
+			stream->crop_mask |= CROP_SRC_SENSOR_MASK;
+			dev->terminal_sensor.selection = input_sel;
+		} else {
+			stream->crop_mask &= ~CROP_SRC_SENSOR_MASK;
+			dev->terminal_sensor.selection.r =
+				dev->terminal_sensor.raw_rect;
+		}
+	}
+
+	if ((stream->crop_mask & 0x3) ==
+	    (CROP_SRC_USR_MASK | CROP_SRC_SENSOR_MASK)) {
+		if (stream->crop[CROP_SRC_USR].left +
+				    stream->crop[CROP_SRC_USR].width >
+			    stream->crop[CROP_SRC_SENSOR].width ||
+		    stream->crop[CROP_SRC_USR].top +
+				    stream->crop[CROP_SRC_USR].height >
+			    stream->crop[CROP_SRC_SENSOR].height)
+			stream->crop[CROP_SRC_USR] =
+				stream->crop[CROP_SRC_SENSOR];
+	}
+
+	if (stream->crop_mask & CROP_SRC_USR_MASK) {
+		stream->crop[CROP_SRC_ACT] = stream->crop[CROP_SRC_USR];
+		if (stream->crop_mask & CROP_SRC_SENSOR_MASK) {
+			stream->crop[CROP_SRC_ACT].left =
+				stream->crop[CROP_SRC_USR].left +
+				stream->crop[CROP_SRC_SENSOR].left;
+			stream->crop[CROP_SRC_ACT].top =
+				stream->crop[CROP_SRC_USR].top +
+				stream->crop[CROP_SRC_SENSOR].top;
+		}
+	} else if (stream->crop_mask & CROP_SRC_SENSOR_MASK) {
+		stream->crop[CROP_SRC_ACT] = stream->crop[CROP_SRC_SENSOR];
+	} else {
+		stream->crop[CROP_SRC_ACT] = dev->terminal_sensor.raw_rect;
+	}
+}
+
+/**es_dvp2axi_sanity_check_fmt - check fmt for setting
+ * @stream - the stream for setting
+ * @s_crop - the crop information
+ */
+static int es_dvp2axi_sanity_check_fmt(struct es_dvp2axi_stream *stream,
+				  const struct v4l2_rect *s_crop)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+	struct v4l2_rect input, *crop;
+
+	if (dev->terminal_sensor.sd) {
+		stream->dvp2axi_fmt_in = es_dvp2axi_get_input_fmt(
+			dev, &input, stream->id, &dev->channels[stream->id]);
+		if (!stream->dvp2axi_fmt_in) {
+			v4l2_err(v4l2_dev, "Input fmt is invalid\n");
+			return -EINVAL;
+		}
+		pr_info("%s:%d input.width %d, input.height %d\n", __func__, __LINE__, input.width, input.height);
+	} else {
+		v4l2_err(v4l2_dev, "terminal_sensor is invalid\n");
+		return -EINVAL;
+	}
+
+	if (stream->dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_EBD_1X8 ||
+	    stream->dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_SPD_2X8) {
+		stream->crop_enable = false;
+		return 0;
+	}
+
+	if (s_crop)
+		crop = (struct v4l2_rect *)s_crop;
+	else
+		crop = &stream->crop[CROP_SRC_ACT];
+
+	if (crop->width + crop->left > input.width ||
+	    crop->height + crop->top > input.height) {
+		pr_debug("%s:%d crop->width %d, crop->left %d, crop->height %d,crop->top %d, input.width %d, input.height %d\n", __func__, __LINE__, crop->width, crop->left, crop->height, crop->top, input.width, input.height);
+		v4l2_err(v4l2_dev, "crop size is bigger than input\n");
+		return -EINVAL;
+	}
+
+	if (dev->active_sensor &&
+	    (dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+	     dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY)) {
+		if (crop->left > 0) {
+			int align_x = get_csi_crop_align(stream->dvp2axi_fmt_in);
+
+			if (align_x > 0 && crop->left % align_x != 0) {
+				v4l2_err(v4l2_dev,
+					 "ERROR: crop left must align %d\n",
+					 align_x);
+				return -EINVAL;
+			}
+		}
+	} else if (dev->active_sensor &&
+		   dev->active_sensor->mbus.type == V4L2_MBUS_CCP2) {
+		if (crop->left % 4 != 0 && crop->width % 4 != 0) {
+			v4l2_err(
+				v4l2_dev,
+				"ERROR: lvds crop left and width must align %d\n",
+				4);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int es_dvp2axi_update_sensor_info(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_sensor_info *sensor, *terminal_sensor;
+	struct v4l2_subdev *sensor_sd;
+	int ret = 0;
+
+	sensor_sd = get_remote_sensor(stream, NULL);
+	if (!sensor_sd) {
+		v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			 "%s: stream[%d] get remote sensor_sd failed!\n",
+			 __func__, stream->id);
+		return -ENODEV;
+	}
+
+	sensor = sd_to_sensor(stream->dvp2axidev, sensor_sd);
+	if (!sensor) {
+		v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			 "%s: stream[%d] get remote sensor failed!\n", __func__,
+			 stream->id);
+		return -ENODEV;
+	}
+	ret = v4l2_subdev_call(sensor->sd, pad, get_mbus_config, 0,
+			       &sensor->mbus);
+	if (ret && ret != -ENOIOCTLCMD) {
+		v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			 "%s: get remote %s mbus failed!\n", __func__,
+			 sensor->sd->name);
+		return ret;
+	}
+
+	stream->dvp2axidev->active_sensor = sensor;
+
+	terminal_sensor = &stream->dvp2axidev->terminal_sensor;
+	get_remote_terminal_sensor(stream, &terminal_sensor->sd);
+	if (terminal_sensor->sd) {
+		ret = v4l2_subdev_call(terminal_sensor->sd, pad,
+				       get_mbus_config, 0,
+				       &terminal_sensor->mbus);
+		if (ret && ret != -ENOIOCTLCMD) {
+			v4l2_err(&stream->dvp2axidev->v4l2_dev,
+				 "%s: get terminal %s mbus failed!\n", __func__,
+				 terminal_sensor->sd->name);
+			return ret;
+		}
+		ret = v4l2_subdev_call(terminal_sensor->sd, video,
+				       g_frame_interval, &terminal_sensor->fi);
+		if (ret) {
+			v4l2_err(
+				&stream->dvp2axidev->v4l2_dev,
+				"%s: get terminal %s g_frame_interval failed!\n",
+				__func__, terminal_sensor->sd->name);
+			return ret;
+		}
+		// if (v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+		// 		     ESMODULE_GET_CSI_DSI_INFO,
+		// 		     &terminal_sensor->dsi_input_en)) {
+		// 	v4l2_dbg(
+		// 		1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		// 		"%s: get terminal %s CSI/DSI sel failed, default csi input!\n",
+		// 		__func__, terminal_sensor->sd->name);
+		// 	terminal_sensor->dsi_input_en = 0;
+		// }
+		terminal_sensor->dsi_input_en= 1;
+	} else {
+		v4l2_err(&stream->dvp2axidev->v4l2_dev,
+			 "%s: stream[%d] get remote terminal sensor failed!\n",
+			 __func__, stream->id);
+		return -ENODEV;
+	}
+
+	if (terminal_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+	    terminal_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY)
+		terminal_sensor->lanes =
+			terminal_sensor->mbus.bus.mipi_csi2.num_data_lanes;
+	else if (terminal_sensor->mbus.type == V4L2_MBUS_CCP2)
+		terminal_sensor->lanes =
+			terminal_sensor->mbus.bus.mipi_csi1.data_lane;
+	return ret;
+}
+
+static int es_dvp2axi_dvp_get_output_type_mask(struct es_dvp2axi_stream *stream)
+{
+	unsigned int mask;
+	const struct dvp2axi_output_fmt *fmt = stream->dvp2axi_fmt_out;
+	switch (fmt->fourcc) {
+	case V4L2_PIX_FMT_NV16:
+		mask = (CSI_WRDDR_TYPE_YUV422SP_EIC770X << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_UYVY << 1);
+		break;
+	case V4L2_PIX_FMT_NV61:
+		mask = (CSI_WRDDR_TYPE_YUV422SP_EIC770X << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_VYUY << 1);
+		break;
+	case V4L2_PIX_FMT_NV12:
+		mask = (CSI_WRDDR_TYPE_YUV420SP_EIC770X << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_UYVY << 1);
+		break;
+	case V4L2_PIX_FMT_NV21:
+		mask = (CSI_WRDDR_TYPE_YUV420SP_EIC770X << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_VYUY << 1);
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_YUYV << 1);
+		break;
+	case V4L2_PIX_FMT_YVYU:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_YVYU << 1);
+		break;
+	case V4L2_PIX_FMT_UYVY:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_UYVY << 1);
+		break;
+	case V4L2_PIX_FMT_VYUY:
+		mask = (CSI_WRDDR_TYPE_YUV_PACKET << 11) |
+		       (CSI_YUV_OUTPUT_ORDER_VYUY << 1);
+		break;
+	case V4L2_PIX_FMT_RGB24:
+	case V4L2_PIX_FMT_BGR24:
+	case V4L2_PIX_FMT_RGB565:
+	case V4L2_PIX_FMT_BGR666:
+		mask = CSI_WRDDR_TYPE_RAW_COMPACT << 11;
+		break;
+	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SBGGR8:
+	case V4L2_PIX_FMT_SRGGB10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SRGGB12:
+	case V4L2_PIX_FMT_SGRBG12:
+	case V4L2_PIX_FMT_SGBRG12:
+	case V4L2_PIX_FMT_SBGGR12:
+	case V4L2_PIX_FMT_GREY:
+	case V4L2_PIX_FMT_Y10:
+	case V4L2_PIX_FMT_Y12:
+		if (stream->is_compact)
+			mask = CSI_WRDDR_TYPE_RAW_COMPACT << 11;
+		else
+			mask = CSI_WRDDR_TYPE_RAW_UNCOMPACT << 11;
+		break;
+	case V4L2_PIX_FMT_SBGGR16:
+	case V4L2_PIX_FMT_SGBRG16:
+	case V4L2_PIX_FMT_SGRBG16:
+	case V4L2_PIX_FMT_SRGGB16:
+	case V4L2_PIX_FMT_Y16:
+		mask = CSI_WRDDR_TYPE_RAW_UNCOMPACT << 11;
+		break;
+	default:
+		mask = CSI_WRDDR_TYPE_RAW_COMPACT << 11;
+		break;
+	}
+	return mask;
+}
+
+static int es_dvp2axi_stream_start(struct es_dvp2axi_stream *stream, unsigned int mode)
+{
+	u32 val, mbus_flags, href_pol, vsync_pol,
+		xfer_mode = 0, yc_swap = 0, inputmode = 0, mipimode = 0,
+		workmode = 0, multi_id = 0,
+		multi_id_en = BT656_1120_MULTI_ID_DISABLE,
+		multi_id_mode = BT656_1120_MULTI_ID_MODE_1,
+		multi_id_sel = BT656_1120_MULTI_ID_SEL_LSB,
+		bt1120_edge_mode = BT1120_CLOCK_SINGLE_EDGES, bt1120_flags = 0,
+		out_fmt_mask = 0, in_fmt_yuv_order = 0;
+	struct esmodule_bt656_mbus_info bt1120_info;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor_info;
+	struct v4l2_mbus_config *mbus;
+	struct es_dvp2axi_dvp_sof_subdev *sof_sd = &dev->dvp_sof_subdev;
+	const struct dvp2axi_output_fmt *fmt;
+	unsigned int dma_en = 0;
+	unsigned int dma_state = 0;
+	int i = 0;
+	u32 sav_detect = BT656_DETECT_SAV;
+	u32 reserved = 0;
+
+	if (stream->state < ES_DVP2AXI_STATE_STREAMING) {
+		stream->frame_idx = 0;
+		stream->buf_wake_up_cnt = 0;
+		stream->lack_buf_cnt = 0;
+		stream->frame_phase = 0;
+		stream->is_in_vblank = false;
+		stream->is_change_toisp = false;
+	}
+
+	sensor_info = dev->active_sensor;
+	mbus = &sensor_info->mbus;
+
+	dma_state = stream->dma_en;
+	if ((mode & ES_DVP2AXI_STREAM_MODE_CAPTURE) == ES_DVP2AXI_STREAM_MODE_CAPTURE)
+		stream->dma_en |= ES_DVP2AXI_DMAEN_BY_VICAP;
+	else if ((mode & ES_DVP2AXI_STREAM_MODE_TOISP_RDBK) ==
+		 ES_DVP2AXI_STREAM_MODE_TOISP_RDBK)
+		stream->dma_en |= ES_DVP2AXI_DMAEN_BY_ISP;
+	else if ((mode & ES_DVP2AXI_STREAM_MODE_ESKIT) == ES_DVP2AXI_STREAM_MODE_ESKIT)
+		stream->dma_en |= ES_DVP2AXI_DMAEN_BY_ESKIT;
+
+	if (dma_state)
+		return 0;
+
+	mbus_flags = mbus->bus.parallel.flags;
+	if ((mbus_flags & DVP2AXI_DVP_PCLK_DUAL_EDGE) == DVP2AXI_DVP_PCLK_DUAL_EDGE) {
+		bt1120_edge_mode = (dev->chip_id < CHIP_EIC770X_DVP2AXI ?
+					    BT1120_CLOCK_DOUBLE_EDGES :
+					    BT1120_CLOCK_DOUBLE_EDGES_EIC770X);
+		es_dvp2axi_enable_dvp_clk_dual_edge(dev, true);
+	} else {
+		bt1120_edge_mode = dev->chip_id < CHIP_EIC770X_DVP2AXI ?
+					   BT1120_CLOCK_SINGLE_EDGES :
+					   BT1120_CLOCK_SINGLE_EDGES_EIC770X;
+		es_dvp2axi_enable_dvp_clk_dual_edge(dev, false);
+	}
+
+	if (mbus_flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
+		es_dvp2axi_config_dvp_clk_sampling_edge(dev, ES_DVP2AXI_CLK_RISING);
+	else
+		es_dvp2axi_config_dvp_clk_sampling_edge(dev, ES_DVP2AXI_CLK_FALLING);
+
+	if (sensor_info->sd && mbus->type == V4L2_MBUS_BT656) {
+		int ret;
+
+		multi_id_en = BT656_1120_MULTI_ID_ENABLE;
+
+		ret = v4l2_subdev_call(sensor_info->sd, core, ioctl,
+				       ESMODULE_GET_BT656_MBUS_INFO,
+				       &bt1120_info);
+		if (ret) {
+			v4l2_warn(&dev->v4l2_dev,
+				  "waring: no muti channel info for BT.656\n");
+		} else {
+			bt1120_flags = bt1120_info.flags;
+			if (bt1120_flags & ESMODULE_CAMERA_BT656_PARSE_ID_LSB)
+				multi_id_sel = BT656_1120_MULTI_ID_SEL_LSB;
+			else
+				multi_id_sel = BT656_1120_MULTI_ID_SEL_MSB;
+
+			if (((bt1120_flags & ESMODULE_CAMERA_BT656_CHANNELS) >>
+			     2) > 3)
+				multi_id_mode = BT656_1120_MULTI_ID_MODE_4;
+			else if (((bt1120_flags &
+				   ESMODULE_CAMERA_BT656_CHANNELS) >>
+				  2) > 1)
+				multi_id_mode = BT656_1120_MULTI_ID_MODE_2;
+			for (i = 0; i < 4; i++)
+				multi_id |= DVP_SW_MULTI_ID(
+					i, i, bt1120_info.id_en_bits);
+			es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_MULTI_ID,
+						multi_id);
+		}
+	}
+
+	href_pol = (mbus_flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH) ?
+			   HSY_HIGH_ACTIVE :
+			   HSY_LOW_ACTIVE;
+	vsync_pol = (mbus_flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH) ?
+			    VSY_HIGH_ACTIVE :
+			    VSY_LOW_ACTIVE;
+
+	if (dev->chip_id < CHIP_EIC770X_DVP2AXI)
+		inputmode = es_dvp2axi_determine_input_mode(stream);
+	else
+		inputmode = es_dvp2axi_determine_input_mode_eic770x(stream);
+	if ((inputmode & INPUT_BT1120_YUV422) == INPUT_BT1120_YUV422)
+	if (DVP2AXI_FETCH_IS_Y_FIRST(
+			stream->dvp2axi_fmt_in->dvp_fmt_val))
+		yc_swap = BT1120_YC_SWAP_EIC770X;
+
+	if (dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+	    dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY) {
+		inputmode = INPUT_MODE_MIPI;
+
+		/* if dvp2axi is linked with mipi,
+		 * href pol must be set as high active,
+		 * vsyn pol must be set as low active.
+		 */
+		href_pol = HSY_HIGH_ACTIVE;
+		vsync_pol = VSY_LOW_ACTIVE;
+
+		if (stream->dvp2axi_fmt_in->fmt_type == DVP2AXI_FMT_TYPE_YUV)
+			mipimode = MIPI_MODE_YUV;
+		else if (stream->dvp2axi_fmt_in->fmt_type == DVP2AXI_FMT_TYPE_RAW)
+			mipimode = MIPI_MODE_RGB;
+		else
+			mipimode = MIPI_MODE_32BITS_BYPASS;
+	}
+
+	if (dev->chip_id < CHIP_EIC770X_DVP2AXI) {
+		val = vsync_pol | href_pol | inputmode | mipimode |
+		      stream->dvp2axi_fmt_out->fmt_val |
+		      stream->dvp2axi_fmt_in->dvp_fmt_val | xfer_mode | yc_swap |
+		      multi_id_en | multi_id_sel | multi_id_mode |
+		      bt1120_edge_mode;
+		if (stream->is_high_align)
+			val |= DVP2AXI_HIGH_ALIGN;
+		else
+			val &= ~DVP2AXI_HIGH_ALIGN;
+	} else {
+		out_fmt_mask = es_dvp2axi_dvp_get_output_type_mask(stream);
+		in_fmt_yuv_order = es_dvp2axi_dvp_get_input_yuv_order(stream);
+		val = vsync_pol | href_pol | inputmode | yc_swap |
+		      out_fmt_mask | in_fmt_yuv_order | multi_id_en |
+		      sav_detect | multi_id_sel | multi_id_mode |
+		      bt1120_edge_mode;
+		if (stream->is_high_align)
+			val |= DVP2AXI_HIGH_ALIGN_EIC770X;
+		else
+			val &= ~DVP2AXI_HIGH_ALIGN_EIC770X;
+	}
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_FOR, val);
+
+	if (dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+		val = stream->pixm.plane_fmt[0].bytesperline;
+	} else {
+		fmt = es_dvp2axi_find_output_fmt(stream, stream->pixm.pixelformat);
+		if (fmt->fmt_type == DVP2AXI_FMT_TYPE_RAW &&
+		    fmt->csi_fmt_val == CSI_WRDDR_TYPE_RAW8)
+			val = ALIGN(stream->pixm.width * fmt->raw_bpp / 8, 256);
+		else
+			val = stream->pixm.width *
+			      es_dvp2axi_cal_raw_vir_line_ratio(stream, fmt);
+	}
+
+	if (stream->crop_enable) {
+		dev->channels[stream->id].crop_en = 1;
+		dev->channels[stream->id].crop_st_x =
+			stream->crop[CROP_SRC_ACT].left;
+		dev->channels[stream->id].crop_st_y =
+			stream->crop[CROP_SRC_ACT].top;
+		dev->channels[stream->id].width =
+			stream->crop[CROP_SRC_ACT].width;
+		dev->channels[stream->id].height =
+			stream->crop[CROP_SRC_ACT].height;
+	} else {
+		dev->channels[stream->id].crop_st_y = 0;
+		dev->channels[stream->id].crop_st_x = 0;
+		dev->channels[stream->id].width = stream->pixm.width;
+		dev->channels[stream->id].height = stream->pixm.height;
+		dev->channels[stream->id].crop_en = 0;
+	}
+
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_VIR_LINE_WIDTH, val);
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_SET_SIZE,
+			     dev->channels[stream->id].width |
+				     (dev->channels[stream->id].height << 16));
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_CROP,
+			     dev->channels[stream->id].crop_st_y
+					     << DVP2AXI_CROP_Y_SHIFT |
+				     dev->channels[stream->id].crop_st_x);
+
+	if (atomic_read(&dev->pipe.stream_cnt) <= 1)
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_FRAME_STATUS,
+				     FRAME_STAT_CLS);
+
+	if (dev->chip_id < CHIP_EIC770X_DVP2AXI) {
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_INTSTAT, INTSTAT_CLS);
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_SCL_CTRL,
+				     es_dvp2axi_scl_ctl(stream));
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_INTEN,
+					DVP_DMA_END_INTSTAT(stream->id) |
+						INTSTAT_ERR |
+						PST_INF_FRAME_END);
+		/* enable line int for sof */
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_LINE_INT_NUM, 0x1);
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_INTEN, LINE_INT_EN);
+	} else {
+		reserved = 0;
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_INTSTAT,
+				     0x3c3ffff | reserved);
+		es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_INTEN,
+					0x033ffff); //0x3c3ffff
+	}
+
+	// if (stream->dma_en) {
+	// 	pr_info("%s:%d yfx !!!! dma enable \n", __func__, __LINE__);
+	// 	pr_info("%s:%d \n", __func__, __LINE__);
+
+	// }
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_INTEN,
+				DVP_DMA_END_INTSTAT(stream->id) | INTSTAT_ERR |
+					PST_INF_FRAME_END);
+
+	/* enable line int for sof */
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_LINE_INT_NUM, 0x1);
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_DVP_INTEN, LINE_INT_EN);
+
+	if (dev->workmode == ES_DVP2AXI_WORKMODE_ONEFRAME)
+		workmode = MODE_ONEFRAME;
+	else if (dev->workmode == ES_DVP2AXI_WORKMODE_PINGPONG)
+		workmode = MODE_PINGPONG;
+	else
+		workmode = MODE_LINELOOP;
+
+	if ((inputmode & INPUT_MODE_BT1120) == INPUT_MODE_BT1120) {
+		workmode = MODE_PINGPONG;
+		dev->workmode = ES_DVP2AXI_WORKMODE_PINGPONG;
+	}
+
+	if (dev->chip_id == CHIP_EIC770X_DVP2AXI) {
+		if (stream->dma_en)
+			dma_en = DVP_DMA_EN;
+		if (stream->lack_buf_cnt == 2)
+			dma_en = 0;
+		es_dvp2axi_write_register(
+			dev, DVP2AXI_REG_DVP_CTRL,
+			DVP_SW_WATER_LINE_25 | dma_en | DVP_PRESS_EN |
+				DVP_HURRY_EN | DVP_SW_WATER_LINE_25 |
+				DVP_SW_PRESS_VALUE(3) | DVP_SW_HURRY_VALUE(3) |
+				ENABLE_CAPTURE);
+	} else {
+		es_dvp2axi_write_register(dev, DVP2AXI_REG_DVP_CTRL,
+				     AXI_BURST_16 | workmode | ENABLE_CAPTURE);
+	}
+	dev->intr_mask = es_dvp2axi_read_register(dev, DVP2AXI_REG_DVP_INTSTAT);
+
+	atomic_set(&sof_sd->frm_sync_seq, 0);
+	stream->state = ES_DVP2AXI_STATE_STREAMING;
+	stream->dvp2axidev->dvp_sof_in_oneframe = 1;
+
+	return 0;
+}
+
+static void es_dvp2axi_attach_sync_mode(struct es_dvp2axi_device *dvp2axidev)
+{
+	struct es_dvp2axi_hw *hw = dvp2axidev->hw_dev;
+	struct es_dvp2axi_device *dev;
+	struct sditf_priv *priv;
+	int i = 0, j = 0;
+	int ret = 0;
+	int count = 0;
+	int sync_type = NO_SYNC_MODE;
+	int sync_group = 0;
+	struct es_dvp2axi_sync_cfg sync_cfg;
+	struct es_dvp2axi_multi_sync_config *sync_config;
+
+	mutex_lock(&hw->dev_lock);
+	if (dvp2axidev->sditf_cnt <= 1) {
+		ret = v4l2_subdev_call(dvp2axidev->terminal_sensor.sd, core, ioctl,
+				       ESMODULE_GET_SYNC_MODE, &sync_type);
+		if (!ret)
+			sync_cfg.type = sync_type;
+		else
+			sync_cfg.type = NO_SYNC_MODE;
+		ret = v4l2_subdev_call(dvp2axidev->terminal_sensor.sd, core, ioctl,
+				       ESMODULE_GET_GROUP_ID, &sync_group);
+		if (!ret && sync_group < ES_DVP2AXI_MAX_GROUP)
+			sync_cfg.group = sync_group;
+		else
+			sync_cfg.group = 0;
+	} else {
+		for (j = 0; j < dvp2axidev->sditf_cnt; j++) {
+			ret |= v4l2_subdev_call(dvp2axidev->sditf[j]->sensor_sd,
+						core, ioctl,
+						ESMODULE_GET_SYNC_MODE,
+						&sync_type);
+			if (!ret && sync_type)
+				break;
+		}
+		if (!ret)
+			sync_cfg.type = sync_type;
+		else
+			sync_cfg.type = NO_SYNC_MODE;
+		ret = v4l2_subdev_call(dvp2axidev->sditf[j]->sensor_sd, core, ioctl,
+				       ESMODULE_GET_GROUP_ID, &sync_group);
+		if (!ret && sync_group < ES_DVP2AXI_MAX_GROUP)
+			sync_cfg.group = sync_group;
+		else
+			sync_cfg.group = 0;
+	}
+	dvp2axidev->sync_cfg = sync_cfg;
+	if (sync_cfg.type == NO_SYNC_MODE ||
+	    hw->sync_config[sync_cfg.group].is_attach) {
+		mutex_unlock(&hw->dev_lock);
+		return;
+	}
+
+	sync_config = &hw->sync_config[sync_cfg.group];
+	memset(sync_config, 0, sizeof(struct es_dvp2axi_multi_sync_config));
+	for (i = 0; i < hw->dev_num; i++) {
+		dev = hw->dvp2axi_dev[i];
+		if (dev->sditf_cnt <= 1) {
+			ret = v4l2_subdev_call(dev->terminal_sensor.sd, core,
+					       ioctl, ESMODULE_GET_SYNC_MODE,
+					       &sync_type);
+			if (!ret)
+				sync_cfg.type = sync_type;
+			else
+				sync_cfg.type = NO_SYNC_MODE;
+			ret = v4l2_subdev_call(dev->terminal_sensor.sd, core,
+					       ioctl, ESMODULE_GET_GROUP_ID,
+					       &sync_group);
+			if (!ret && sync_group < ES_DVP2AXI_MAX_GROUP)
+				sync_cfg.group = sync_group;
+			else
+				sync_cfg.group = 0;
+		} else {
+			priv = dev->sditf[0];
+			if (priv && priv->is_combine_mode &&
+			    dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF) {
+				for (j = 0; j < dev->sditf_cnt; j++) {
+					ret |= v4l2_subdev_call(
+						dev->sditf[j]->sensor_sd, core,
+						ioctl, ESMODULE_GET_SYNC_MODE,
+						&sync_type);
+					if (!ret && sync_type) {
+						priv = dev->sditf[j];
+						break;
+					}
+				}
+				if (!ret)
+					sync_cfg.type = sync_type;
+				else
+					sync_cfg.type = NO_SYNC_MODE;
+				ret = v4l2_subdev_call(priv->sensor_sd, core,
+						       ioctl,
+						       ESMODULE_GET_GROUP_ID,
+						       &sync_group);
+				if (!ret && sync_group < ES_DVP2AXI_MAX_GROUP)
+					sync_cfg.group = sync_group;
+				else
+					sync_cfg.group = 0;
+			}
+		}
+		if (sync_cfg.group == dvp2axidev->sync_cfg.group) {
+			if (sync_cfg.type == EXTERNAL_MASTER_MODE) {
+				count = sync_config->ext_master.count;
+				sync_config->ext_master.dvp2axi_dev[count] = dev;
+				sync_config->ext_master.count++;
+				sync_config->dev_cnt++;
+				sync_config->sync_mask |=
+					BIT(dev->csi_host_idx);
+			} else if (sync_cfg.type == INTERNAL_MASTER_MODE) {
+				count = sync_config->int_master.count;
+				sync_config->int_master.dvp2axi_dev[count] = dev;
+				sync_config->int_master.count++;
+				sync_config->dev_cnt++;
+				sync_config->sync_mask |=
+					BIT(dev->csi_host_idx);
+			} else if (sync_cfg.type == SLAVE_MODE) {
+				count = sync_config->slave.count;
+				sync_config->slave.dvp2axi_dev[count] = dev;
+				sync_config->slave.count++;
+				sync_config->dev_cnt++;
+				sync_config->sync_mask |=
+					BIT(dev->csi_host_idx);
+			}
+			dev->sync_cfg = sync_cfg;
+		} else {
+			ret = v4l2_subdev_call(dev->terminal_sensor.sd, core,
+					       ioctl, ESMODULE_GET_SYNC_MODE,
+					       &sync_type);
+		}
+	}
+	if (sync_config->int_master.count == 1) {
+		if (sync_config->ext_master.count) {
+			sync_config->mode = ES_DVP2AXI_MASTER_MASTER;
+			sync_config->is_attach = true;
+		} else if (sync_config->slave.count) {
+			sync_config->mode = ES_DVP2AXI_MASTER_SLAVE;
+			sync_config->is_attach = true;
+		} else {
+			dev_info(
+				hw->dev,
+				"Missing slave device, do not use sync mode\n");
+		}
+		if (sync_config->is_attach)
+			dev_info(
+				hw->dev,
+				"group %d, int_master %d, ext_master %d, slave %d\n",
+				i, sync_config->int_master.count,
+				sync_config->ext_master.count,
+				sync_config->slave.count);
+	}
+	mutex_unlock(&hw->dev_lock);
+}
+
+static void es_dvp2axi_monitor_reset_event(struct es_dvp2axi_device *dev);
+
+int es_dvp2axi_do_start_stream(struct es_dvp2axi_stream *stream,
+			  enum es_dvp2axi_stream_mode mode)
+{
+	struct es_dvp2axi_vdev_node *node = &stream->vnode;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_hw *hw_dev = dev->hw_dev;
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+	struct es_dvp2axi_sensor_info *sensor_info = dev->active_sensor;
+	struct es_dvp2axi_sensor_info *terminal_sensor = NULL;
+	struct esmodule_hdr_cfg hdr_cfg;
+	struct es_dvp2axi_csi_info csi_info = { 0 };
+	int esmodule_stream_seq = ESMODULE_START_STREAM_DEFAULT;
+	int ret;
+	int i = 0;
+	u32 skip_frame = 0;
+
+	v4l2_info(&dev->v4l2_dev, "stream[%d] start streaming\n", stream->id);
+
+	es_dvp2axi_attach_sync_mode(dev);
+	mutex_lock(&dev->stream_lock);
+
+	if ((stream->cur_stream_mode & ES_DVP2AXI_STREAM_MODE_CAPTURE) == mode) {
+		ret = -EBUSY;
+		v4l2_err(v4l2_dev, "stream in busy state\n");
+		goto destroy_buf;
+	}
+	if (stream->dma_en == 0)
+		stream->fs_cnt_in_single_frame = 0;
+	if (stream->is_line_wake_up)
+		stream->is_line_inten = true;
+	else
+		stream->is_line_inten = false;
+
+	if (!dev->active_sensor) {
+		ret = es_dvp2axi_update_sensor_info(stream);
+		if (ret < 0) {
+			v4l2_dbg(1, es_dvp2axi_debug,v4l2_dev, "update sensor info failed %d\n",
+				 ret);
+			goto out;
+		}
+	}
+	terminal_sensor = &dev->terminal_sensor;
+	if (terminal_sensor->sd) {
+		ret = v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+				       ESMODULE_GET_HDR_CFG, &hdr_cfg);
+		if (!ret)
+			dev->hdr = hdr_cfg;
+		else
+			dev->hdr.hdr_mode = NO_HDR;
+
+		ret = v4l2_subdev_call(terminal_sensor->sd, video,
+				       g_frame_interval, &terminal_sensor->fi);
+		if (ret)
+			terminal_sensor->fi.interval =
+				(struct v4l2_fract){ 1, 30 };
+
+		ret = v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+				       ESMODULE_GET_START_STREAM_SEQ,
+				       &esmodule_stream_seq);
+		if (ret)
+			esmodule_stream_seq = ESMODULE_START_STREAM_DEFAULT;
+
+		es_dvp2axi_sync_crop_info(stream);
+	}
+
+	ret = es_dvp2axi_sanity_check_fmt(stream, NULL);
+	if (ret < 0)
+		goto destroy_buf;
+
+	mutex_lock(&hw_dev->dev_lock);
+	if (atomic_read(&dev->pipe.stream_cnt) == 0 && dev->active_sensor &&
+	    (dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+	     dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+	     dev->active_sensor->mbus.type == V4L2_MBUS_CCP2)) {
+		if (dev->channels[0].capture_info.mode ==
+		    ESMODULE_MULTI_DEV_COMBINE_ONE) {
+			csi_info.csi_num =
+				dev->channels[0].capture_info.multi_dev.dev_num;
+			if (csi_info.csi_num > ES_DVP2AXI_MAX_CSI_NUM) {
+				v4l2_err(v4l2_dev, "csi num %d, max %d\n",
+					 csi_info.csi_num, ES_DVP2AXI_MAX_CSI_NUM);
+				goto out;
+			}
+			for (i = 0; i < csi_info.csi_num; i++) {
+				csi_info.csi_idx[i] =
+					dev->channels[0]
+						.capture_info.multi_dev
+						.dev_idx[i];
+				if (dev->hw_dev->is_eic770xs2)
+					v4l2_info(
+						v4l2_dev,
+						"eic770xs2 combine mode attach to mipi%d\n",
+						csi_info.csi_idx[i]);
+			}
+		} else {
+			csi_info.csi_num = 1;
+			dev->csi_host_idx = dev->csi_host_idx_def;
+			csi_info.csi_idx[0] = dev->csi_host_idx;
+		}
+		ret = v4l2_subdev_call(dev->active_sensor->sd, core, ioctl,
+				       ES_DVP2AXI_CMD_SET_CSI_IDX, &csi_info);
+		if (ret)
+			v4l2_err(&dev->v4l2_dev, "set csi idx %d fail\n",
+				 dev->csi_host_idx);
+	}
+
+	if (((dev->active_sensor &&
+	      dev->active_sensor->mbus.type == V4L2_MBUS_BT656) ||
+	     dev->is_use_dummybuf) &&
+	    (!dev->hw_dev->dummy_buf.vaddr) &&
+	    mode == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+		ret = es_dvp2axi_create_dummy_buf(stream);
+		if (ret < 0) {
+			mutex_unlock(&hw_dev->dev_lock);
+			v4l2_err(v4l2_dev, "Failed to create dummy_buf, %d\n",
+				 ret);
+			goto destroy_buf;
+		}
+	}
+	mutex_unlock(&hw_dev->dev_lock);
+
+	if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE) {
+		// pr_info("%s:%d yfx!!!! enable tasklet \n", __func__, __LINE__);
+		tasklet_enable(&stream->vb_done_tasklet);
+	}
+
+	if (stream->cur_stream_mode == ES_DVP2AXI_STREAM_MODE_NONE) {
+		ret = dev->pipe.open(&dev->pipe, &node->vdev.entity, true);
+		if (ret < 0) {
+			v4l2_err(v4l2_dev, "open dvp2axi pipeline failed %d\n",
+				 ret);
+			goto destroy_buf;
+		}
+
+		/*
+		 * start sub-devices
+		 * When use bt601, the sampling edge of dvp2axi is random,
+		 * can be rising or fallling after powering on dvp2axi.
+		 * To keep the coherence of edge, open sensor in advance.
+		 */
+		if (sensor_info->mbus.type == V4L2_MBUS_PARALLEL ||
+		    esmodule_stream_seq == ESMODULE_START_STREAM_FRONT) {
+			ret = dev->pipe.set_stream(&dev->pipe, true);
+			if (ret < 0)
+				goto destroy_buf;
+		}
+		ret = v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+				       ESMODULE_GET_SKIP_FRAME, &skip_frame);
+		if (!ret && skip_frame < ES_DVP2AXI_SKIP_FRAME_MAX)
+			stream->skip_frame = skip_frame;
+		else
+			stream->skip_frame = 0;
+		stream->cur_skip_frame = stream->skip_frame;
+	}
+	if (dev->active_sensor &&
+		(dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+		 dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+		 dev->active_sensor->mbus.type == V4L2_MBUS_CCP2))
+		ret = es_dvp2axi_csi_stream_start(stream, mode);
+	else
+		ret = es_dvp2axi_stream_start(stream, mode);
+
+	if (ret < 0)
+		goto destroy_buf;
+
+	if (stream->cur_stream_mode == ES_DVP2AXI_STREAM_MODE_NONE) {
+		ret = video_device_pipeline_start(&node->vdev, &dev->pipe.pipe);
+		if (ret < 0) {
+			v4l2_err(&dev->v4l2_dev, "start pipeline failed %d\n",
+				 ret);
+			goto pipe_stream_off;
+		}
+
+		if (sensor_info->mbus.type != V4L2_MBUS_PARALLEL &&
+		    esmodule_stream_seq != ESMODULE_START_STREAM_FRONT) {
+			ret = dev->pipe.set_stream(&dev->pipe, true);
+			if (ret < 0)
+				goto stop_stream;
+		}
+	}
+	dev->reset_work_cancel = false;
+	stream->cur_stream_mode |= mode;
+	es_dvp2axi_monitor_reset_event(dev);
+	goto out;
+
+stop_stream:
+	es_dvp2axi_stream_stop(stream);
+pipe_stream_off:
+	dev->pipe.set_stream(&dev->pipe, false);
+
+destroy_buf:
+	if (mode == ES_DVP2AXI_STREAM_MODE_CAPTURE)
+		tasklet_disable(&stream->vb_done_tasklet);
+	if (stream->curr_buf)
+		list_add_tail(&stream->curr_buf->queue, &stream->buf_head);
+	if (stream->next_buf && stream->next_buf != stream->curr_buf)
+		list_add_tail(&stream->next_buf->queue, &stream->buf_head);
+
+	stream->curr_buf = NULL;
+	stream->next_buf = NULL;
+	atomic_set(&stream->buf_cnt, 0);
+	while (!list_empty(&stream->buf_head)) {
+		struct es_dvp2axi_buffer *buf;
+
+		buf = list_first_entry(&stream->buf_head, struct es_dvp2axi_buffer,
+				       queue);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+		list_del(&buf->queue);
+	}
+
+out:
+	mutex_unlock(&dev->stream_lock);
+	return ret;
+}
+
+/* CTRL1 */
+#define DVP2AXI_DVP0_PWIDTH 16
+#define DVP2AXI_DVP1_PWIDTH 16
+
+#define DVP2AXI_IO_DVP_DIS  0
+#define DVP2AXI_IO_DVP_ENA  1
+#define DVP2AXI_PIXEL_MODE_1WORD 0
+#define DVP2AXI_PIXEL_MODE_2WORD 1
+#define DVP2AXI_PIXEL_MODE_3WORD 2
+
+/* CTRL2 */
+#define DVP2AXI_DVP2_PWIDTH 16
+#define DVP2AXI_DVP3_PWIDTH 16
+#define DVP2AXI_DVP4_PWIDTH 16
+#define DVP2AXI_DVP5_PWIDTH 16
+#define DVP2AXI_OUTSTANDING_SIZE 16
+#define DVP2AXI_WQOS_CFG 0
+
+static int es_dvp2axi_start_streaming(struct vb2_queue *queue, unsigned int count)
+{
+	struct es_dvp2axi_stream *stream = queue->drv_priv;
+	struct es_dvp2axi_hw *dvp2axi_hw = stream->dvp2axidev->hw_dev;
+
+	uint32_t outstand_reg;
+	// uint32_t int0, int1, int_err;
+	// uint32_t stride, shnum;
+	u32 dvp2axi_bpp;
+	dvp2axi_bpp = (DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL1_CSR) & VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK ) >> 20;
+
+	/* outstanding 16 */
+	outstand_reg = (DVP2AXI_OUTSTANDING_SIZE-1) << 24;
+	DVP2AXI_HalWriteReg( dvp2axi_hw, VI_DVP2AXI_CTRL2_CSR, outstand_reg);
+
+	#ifdef DVP2AXI_DVP0_ENABLE
+	DVP2AXI_HalWriteReg( dvp2axi_hw, VI_DVP2AXI_CTRL3_CSR,   (stream->pixm.width )| (stream->pixm.height << 16));//DVP0
+	#endif
+	
+	uint32_t bpl = 0;
+	uint32_t dvpx_bpl = 0;
+	#ifdef DVP2AXI_DVP0_ENABLE
+	if(stream->crop_enable) {
+		bpl = ALIGN(stream->crop[CROP_SRC_ACT].width * dvp2axi_bpp / 8, 256);
+		dvpx_bpl |= bpl;
+	} else {
+		bpl = ALIGN(stream->pixm.width * dvp2axi_bpp / 8, 256);
+		dvpx_bpl |= bpl;
+	}
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL33_CSR, dvpx_bpl);
+	dvpx_bpl = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL33_CSR);
+	pr_info("%s:%d dvpx_bpl:%x\n", __func__, __LINE__, dvpx_bpl);
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL33_CSR, dvpx_bpl);
+	#endif
+
+	int ret = 0;
+	ret = es_dvp2axi_do_start_stream(stream, ES_DVP2AXI_STREAM_MODE_CAPTURE);
+	stream->frame_phase = DVP2AXI_CSI_FRAME_UNREADY;
+	dvp2axi_hw_int_mask(stream->dvp2axidev->hw_dev, 0);
+	dvp2axi_hw_soft_reset(stream->dvp2axidev->hw_dev);
+	uint32_t csr0 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR);
+	#ifdef DVP2AXI_DVP0_ENABLE
+    csr0 = csr0 | 1;
+    #endif
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, csr0);
+	pr_info("dvp2axi enabled=0x%x\n", csr0);
+
+	return ret;
+}
+
+static struct vb2_ops es_dvp2axi_vb2_ops = {
+	.queue_setup = es_dvp2axi_queue_setup,
+	.buf_queue = es_dvp2axi_buf_queue,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+	.stop_streaming = es_dvp2axi_stop_streaming,
+	.start_streaming = es_dvp2axi_start_streaming,
+};
+
+static int es_dvp2axi_init_vb2_queue(struct vb2_queue *q,
+				struct es_dvp2axi_stream *stream,
+				enum v4l2_buf_type buf_type)
+{
+	struct es_dvp2axi_hw *hw_dev = stream->dvp2axidev->hw_dev;
+
+	q->type = buf_type;
+	q->io_modes = VB2_MMAP | VB2_DMABUF;
+	q->drv_priv = stream;
+	q->ops = &es_dvp2axi_vb2_ops;
+	// q->mem_ops = hw_dev->mem_ops;
+	q->mem_ops = &vb2_dma_contig_memops;
+	q->buf_struct_size = sizeof(struct es_dvp2axi_buffer);
+	if (stream->dvp2axidev->is_use_dummybuf)
+		q->min_buffers_needed = 1;
+	else
+		q->min_buffers_needed = DVP2AXI_REQ_BUFS_MIN;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->lock = &stream->vnode.vlock;
+	q->dev = hw_dev->dev;
+	q->allow_cache_hints = 1;
+	q->bidirectional = 1;
+	// q->gfp_flags = GFP_DMA32;
+	dma_set_mask_and_coherent(q->dev, DMA_BIT_MASK(41));
+	if (hw_dev->is_dma_contig)
+		q->dma_attrs = 0;//DMA_ATTR_FORCE_CONTIGUOUS;
+
+	return vb2_queue_init(q);
+}
+
+int es_dvp2axi_set_fmt(struct es_dvp2axi_stream *stream,
+		  struct v4l2_pix_format_mplane *pixm, bool try)
+{
+	// struct v4l2_subdev_format subdev_fmt;
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct sditf_priv *priv = dev->sditf[0];
+	const struct dvp2axi_output_fmt *fmt;
+	const struct dvp2axi_input_fmt *dvp2axi_fmt_in = NULL;
+	struct v4l2_rect input_rect;
+	unsigned int imagesize = 0, ex_imagesize = 0, planes;
+	u32 xsubs = 1, ysubs = 1, i;
+	struct esmodule_hdr_cfg hdr_cfg;
+	struct es_dvp2axi_extend_info *extend_line = &stream->extend_line;
+	struct csi_channel_info *channel_info = &dev->channels[stream->id];
+	int ret;
+	struct es_dvp2axi_hw *hw_dev = stream->dvp2axidev->hw_dev;
+	int fmt_w_h_val;
+	// int dvp2axi_bpp;
+	int dvp2axi_bpp_clear;
+	int dvp2axi_mode_clear;
+
+	for (i = 0; i < ES_DVP2AXI_MAX_PLANE; i++)
+		memset(&pixm->plane_fmt[i], 0,
+		       sizeof(struct v4l2_plane_pix_format));
+	fmt = es_dvp2axi_find_output_fmt(stream, pixm->pixelformat);
+	if (!fmt)
+		fmt = &out_fmts[0];
+
+	// input_rect.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	// input_rect.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+	input_rect.width = pixm->width;
+	input_rect.height = pixm->height;
+	if (dev->terminal_sensor.sd) {
+		dvp2axi_fmt_in = es_dvp2axi_get_input_fmt(dev, &input_rect, stream->id,
+						 channel_info);
+
+		stream->dvp2axi_fmt_in = dvp2axi_fmt_in;
+	} else {
+		pr_debug("terminal subdev does not exist\n");
+		return -EINVAL;
+	}
+	ret = es_dvp2axi_output_fmt_check(stream, fmt);
+	if (ret)
+		return -EINVAL;
+
+	pr_debug("%s:%d input_rect.width:%d height:%d \n", __func__, __LINE__, input_rect.width, input_rect.height);
+	fmt_w_h_val = (input_rect.height << 16) | (input_rect.width);
+	DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL3_CSR, fmt_w_h_val);
+
+	if (dev->terminal_sensor.sd) {
+		ret = v4l2_subdev_call(dev->terminal_sensor.sd, core, ioctl,
+				       ESMODULE_GET_HDR_CFG, &hdr_cfg);
+		if (!ret)
+			dev->hdr = hdr_cfg;
+		else
+			dev->hdr.hdr_mode = NO_HDR;
+
+		dev->terminal_sensor.raw_rect = input_rect;
+	}
+
+	/* DVP2AXI has not scale function,
+	 * the size should not be larger than input
+	 */
+	pr_debug("input_rect.width = 0x%x \n", input_rect.width);
+	pr_debug("input_rect.height = 0x%x \n", input_rect.height);
+	pr_debug("pixm->width = 0x%x \n", pixm->width);
+	pr_debug("pixm->height = 0x%x \n", pixm->height);
+	pixm->width =
+		clamp_t(u32, pixm->width, DVP2AXI_MIN_WIDTH, input_rect.width);
+	pixm->height =
+		clamp_t(u32, pixm->height, DVP2AXI_MIN_HEIGHT, input_rect.height);
+	pixm->num_planes = fmt->mplanes;
+	pixm->field = V4L2_FIELD_NONE;
+	pixm->quantization = V4L2_QUANTIZATION_DEFAULT;
+	es_dvp2axi_sync_crop_info(stream);
+	/* calculate plane size and image size */
+	fcc_xysubs(fmt->fourcc, &xsubs, &ysubs);
+	planes = fmt->cplanes ? fmt->cplanes : fmt->mplanes;
+
+	if (dvp2axi_fmt_in && (dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_SPD_2X8 ||
+			   dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_EBD_1X8))
+		stream->crop_enable = false;
+
+	for (i = 0; i < planes; i++) {
+		struct v4l2_plane_pix_format *plane_fmt;
+		int width, height, bpl, size, bpp, ex_size;
+
+		if (i == 0) {
+			if (stream->crop_enable) {
+				width = stream->crop[CROP_SRC_ACT].width;
+				height = stream->crop[CROP_SRC_ACT].height;
+			} else {
+				width = pixm->width;
+				height = pixm->height;
+			}
+		} else {
+			if (stream->crop_enable) {
+				width = stream->crop[CROP_SRC_ACT].width /
+					xsubs;
+				height = stream->crop[CROP_SRC_ACT].height /
+					 ysubs;
+			} else {
+				width = pixm->width / xsubs;
+				height = pixm->height / ysubs;
+			}
+		}
+
+		if (priv && priv->is_combine_mode &&
+		    dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF)
+			height *= dev->sditf_cnt;
+
+		extend_line->pixm.height = height + ESMODULE_EXTEND_LINE;
+
+		/* compact mode need bytesperline 4bytes align,
+		 * align 8 to bring into correspondence with virtual width.
+		 * to optimize reading and writing of ddr, aliged with 256.
+		 */
+		if (fmt->fmt_type == DVP2AXI_FMT_TYPE_RAW && dvp2axi_fmt_in &&
+		    (dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_EBD_1X8 ||
+		     dvp2axi_fmt_in->mbus_code == MEDIA_BUS_FMT_SPD_2X8)) {
+			stream->is_compact = false;
+		}
+		pr_debug("*** modet fmt->fmt_type = 0x%x ***\n",fmt->fmt_type);
+		if (fmt->fmt_type == DVP2AXI_FMT_TYPE_RAW && stream->is_compact &&
+		    (dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+		     dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+		     dev->active_sensor->mbus.type == V4L2_MBUS_CCP2) &&
+		    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB888 &&
+		    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB565) {
+			// bpl = ALIGN(width * fmt->raw_bpp / 8, 256);
+			// dvp2axi_bpp = (readl(dev->hw_dev->base_addr+VI_DVP2AXI_CTRL1_CSR) & VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK ) >> 20;
+			// if(dvp2axi_bpp == 16) {
+			// 	bpl = ALIGN(width * ALIGN(fmt->raw_bpp, 16) / 8, 256);
+			// } else {
+			// 	bpl = ALIGN(width * fmt->raw_bpp / 8, 256);
+			// }
+			dvp2axi_bpp_clear = DVP2AXI_HalReadReg(hw_dev, VI_DVP2AXI_CTRL1_CSR) & ~VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK;
+			if(fmt->raw_bpp == 10) {
+				bpl = ALIGN(width * ALIGN(fmt->raw_bpp, 16) / 8, 256);
+				DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_bpp_clear | ((16 & 0x1F) << 20));
+			} else {
+				bpl = ALIGN(width * fmt->raw_bpp / 8, 256);
+				DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_bpp_clear | ((fmt->raw_bpp & 0x1F) << 20));
+			}
+			dvp2axi_mode_clear = DVP2AXI_HalReadReg(hw_dev, VI_DVP2AXI_CTRL1_CSR) & ~0x300;
+			DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_mode_clear | ((DVP_PIXEL_MODE_RAW & 0x3) << 8));
+		} else {
+			if (fmt->fmt_type == DVP2AXI_FMT_TYPE_RAW &&
+			    stream->is_compact &&
+			    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB888 &&
+			    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB565 &&
+			    dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+				bpl = ALIGN(width * fmt->raw_bpp / 8, 256);
+			} else {
+				if(fmt->fmt_type == DVP2AXI_FMT_TYPE_YUV){
+					//YUV
+					pr_debug("*** mode yuv ***\n");
+					dvp2axi_mode_clear = DVP2AXI_HalReadReg(hw_dev, VI_DVP2AXI_CTRL1_CSR) & ~0x300;
+					DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_mode_clear | ((DVP_PIXEL_MODE_YUV & 0x3) << 8));
+					bpp = es_dvp2axi_align_bits_per_pixel(stream, fmt,
+										i);
+					dvp2axi_bpp_clear = DVP2AXI_HalReadReg(hw_dev, VI_DVP2AXI_CTRL1_CSR) & ~VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK;
+					DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_bpp_clear | ((bpp & 0x1F) << 20));
+					bpl = width * bpp / DVP2AXI_YUV_STORED_BIT_WIDTH;
+					pr_debug("*** mode yuv width = 0x%x ***\n", width);
+					pr_debug("*** mode yuv bpp = 0x%x ***\n", bpp);
+					pr_debug("*** mode yuv DVP2AXI_YUV_STORED_BIT_WIDTH = 0x%x ***\n", DVP2AXI_YUV_STORED_BIT_WIDTH);
+					pr_debug("*** mode yuv bpl = 0x%x ***\n", bpl);
+				}
+				if(fmt->csi_fmt_val == CSI_WRDDR_TYPE_RGB888){
+					//RGB888
+					dvp2axi_mode_clear = DVP2AXI_HalReadReg(hw_dev, VI_DVP2AXI_CTRL1_CSR) & ~0x300;
+					DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_mode_clear | ((DVP_PIXEL_MODE_RGB & 0x3) << 8));
+					bpp = es_dvp2axi_align_bits_per_pixel(stream, fmt,
+										i);
+					dvp2axi_bpp_clear = DVP2AXI_HalReadReg(hw_dev, VI_DVP2AXI_CTRL1_CSR) & ~VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK;
+					DVP2AXI_HalWriteReg(hw_dev, VI_DVP2AXI_CTRL1_CSR, dvp2axi_bpp_clear | ((bpp & 0x1F) << 20));
+					bpl = width * bpp / DVP2AXI_YUV_STORED_BIT_WIDTH;
+					pr_debug("*** mode rgb width = 0x%x ***\n", width);
+					pr_debug("*** mode rgb bpp = 0x%x ***\n", bpp);
+					pr_debug("*** mode rgb DVP2AXI_YUV_STORED_BIT_WIDTH = 0x%x ***\n", DVP2AXI_YUV_STORED_BIT_WIDTH);
+					pr_debug("*** mode rgb bpl = 0x%x ***\n", bpl);
+				}
+			}
+		}
+		
+		pr_debug("***** bpl = 0x%x *****\n", bpl);
+		
+		size = bpl * height;
+		imagesize += size;
+		ex_size = bpl * extend_line->pixm.height;
+		ex_imagesize += ex_size;
+
+		pr_debug("*** mode size = 0x%x ***\n", size);
+		pr_debug("*** mode imagesize = 0x%x ***\n", imagesize);
+		pr_debug("*** mode ex_size = 0x%x ***\n", ex_size);
+		pr_debug("*** mode ex_imagesize = 0x%x ***\n", ex_imagesize);
+
+		if (fmt->mplanes > i) {
+			/* Set bpl and size for each mplane */
+			plane_fmt = pixm->plane_fmt + i;
+			plane_fmt->bytesperline = bpl;
+			plane_fmt->sizeimage = size;
+
+			plane_fmt = extend_line->pixm.plane_fmt + i;
+			plane_fmt->bytesperline = bpl;
+			plane_fmt->sizeimage = ex_size;
+		}
+		v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			 "C-Plane %i size: %d, Total imagesize: %d\n", i, size,
+			 imagesize);
+	}
+
+	/* convert to non-MPLANE format.
+	 * It's important since we want to unify non-MPLANE
+	 * and MPLANE.
+	 */
+	if (fmt->mplanes == 1) {
+		pixm->plane_fmt[0].sizeimage = imagesize;
+		extend_line->pixm.plane_fmt[0].sizeimage = ex_imagesize;
+	}
+
+	if (!try) {
+		stream->dvp2axi_fmt_out = fmt;
+		stream->pixm = *pixm;
+
+		v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+			 "%s: req(%d, %d) out(%d, %d)\n", __func__, pixm->width,
+			 pixm->height, stream->pixm.width, stream->pixm.height);
+	}
+	return 0;
+}
+
+void es_dvp2axi_stream_init(struct es_dvp2axi_device *dev, u32 id)
+{
+	struct es_dvp2axi_stream *stream = &dev->stream[id];
+	struct v4l2_pix_format_mplane pixm;
+	int i;
+
+	memset(stream, 0, sizeof(*stream));
+	memset(&pixm, 0, sizeof(pixm));
+	stream->id = id;
+	stream->dvp2axidev = dev;
+
+	INIT_LIST_HEAD(&stream->buf_head);
+	INIT_LIST_HEAD(&stream->rx_buf_head);
+	INIT_LIST_HEAD(&stream->rx_buf_head_vicap);
+	INIT_LIST_HEAD(&stream->eskit_buf_head);
+	spin_lock_init(&stream->vbq_lock);
+	spin_lock_init(&stream->fps_lock);
+	stream->state = ES_DVP2AXI_STATE_READY;
+	init_waitqueue_head(&stream->wq_stopped);
+
+	/* Set default format */
+	pixm.pixelformat = V4L2_PIX_FMT_NV12;
+	pixm.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	pixm.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+	es_dvp2axi_set_fmt(stream, &pixm, false);
+
+	for (i = 0; i < CROP_SRC_MAX; i++) {
+		stream->crop[i].left = 0;
+		stream->crop[i].top = 0;
+		stream->crop[i].width = ES_DVP2AXI_DEFAULT_WIDTH;
+		stream->crop[i].height = ES_DVP2AXI_DEFAULT_HEIGHT;
+	}
+
+	stream->crop_enable = false;
+	stream->crop_dyn_en = false;
+	stream->crop_mask = 0x0;
+
+	if (dev->inf_id == ES_DVP2AXI_DVP) {
+		stream->is_compact = true;
+	} else {
+		stream->is_compact = true;
+	}
+
+	stream->is_high_align = false;
+	stream->is_finish_stop_dma = false;
+	stream->is_wait_dma_stop = false;
+
+	stream->extend_line.is_extended = false;
+
+	stream->is_dvp_yuv_addr_init = false;
+	stream->is_fs_fe_not_paired = false;
+	stream->fs_cnt_in_single_frame = 0;
+	if (dev->wait_line) {
+		dev->wait_line_cache = dev->wait_line;
+		dev->wait_line_bak = dev->wait_line;
+		stream->is_line_wake_up = true;
+	} else {
+		stream->is_line_wake_up = false;
+		dev->wait_line_cache = 0;
+		dev->wait_line_bak = 0;
+	}
+	stream->cur_stream_mode = 0;
+	stream->dma_en = 0;
+	stream->to_en_dma = 0;
+	stream->to_stop_dma = 0;
+	stream->to_en_scale = false;
+	stream->buf_owner = 0;
+	stream->buf_replace_cnt = 0;
+	stream->is_stop_capture = false;
+	stream->is_single_cap = false;
+	atomic_set(&stream->buf_cnt, 0);
+	stream->rx_buf_num = 0;
+	init_completion(&stream->stop_complete);
+	stream->is_wait_stop_complete = false;
+}
+
+static int es_dvp2axi_fh_open(struct file *filp)
+{
+	struct video_device *vdev = video_devdata(filp);
+	struct es_dvp2axi_vdev_node *vnode = vdev_to_node(vdev);
+	struct es_dvp2axi_stream *stream = to_es_dvp2axi_stream(vnode);
+	struct es_dvp2axi_device *dvp2axidev = stream->dvp2axidev;
+	int ret;
+	int i = 0;
+
+	stream->is_first_flush = true;
+	ret = es_dvp2axi_attach_hw(dvp2axidev);
+	if (ret)
+		return ret;
+
+	/* Make sure active sensor is valid before .set_fmt() */
+	ret = es_dvp2axi_update_sensor_info(stream);
+	if (ret < 0) {
+		v4l2_dbg(1, es_dvp2axi_debug, vdev, "update sensor info failed %d\n", ret);
+
+		return ret;
+	}
+
+	ret = pm_runtime_resume_and_get(dvp2axidev->dev);
+	if (ret < 0) {
+		v4l2_err(vdev, "Failed to get runtime pm, %d\n", ret);
+		return ret;
+	}
+
+	ret = v4l2_fh_open(filp);
+	
+	if (ret < 0)
+		vb2_fop_release(filp);
+	
+	if (dvp2axidev->sditf_cnt > 1) {
+		for (i = 0; i < dvp2axidev->sditf_cnt; i++) {
+			if (dvp2axidev->sditf[i]->sensor_sd)
+				ret |= v4l2_subdev_call(
+					dvp2axidev->sditf[i]->sensor_sd, core,
+					s_power, 1);
+		}
+		if (ret < 0)
+			v4l2_err(vdev, "set sensor power on fail, ret %d\n",
+				 ret);
+	}
+	return ret;
+}
+
+static int es_dvp2axi_fh_release(struct file *filp)
+{
+	struct video_device *vdev = video_devdata(filp);
+	struct es_dvp2axi_vdev_node *vnode = vdev_to_node(vdev);
+	struct es_dvp2axi_stream *stream = to_es_dvp2axi_stream(vnode);
+	struct es_dvp2axi_device *dvp2axidev = stream->dvp2axidev;
+	int ret = 0;
+	int i = 0;
+
+	stream->is_first_flush = true;
+
+	ret = vb2_fop_release(filp);
+	
+	pm_runtime_put_sync(dvp2axidev->dev);
+	if (dvp2axidev->sditf_cnt > 1) {
+		for (i = 0; i < dvp2axidev->sditf_cnt; i++) {
+			if (dvp2axidev->sditf[i]->sensor_sd)
+				ret |= v4l2_subdev_call(
+					dvp2axidev->sditf[i]->sensor_sd, core,
+					s_power, 0);
+		}
+		if (ret < 0)
+			v4l2_err(vdev, "set sensor power on fail, ret %d\n",
+				 ret);
+	}
+	return ret;
+}
+
+unsigned int eic770x_vb2_fop_poll(struct file *file, poll_table *wait)
+{
+	int int0, int1, int_err;
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct video_device *vdev = video_devdata(file);
+	struct es_dvp2axi_hw	*dvp2axi_hw =  stream->dvp2axidev->hw_dev;
+	//unsigned long req_events = poll_requested_events(wait);
+	unsigned int mask = 0;
+	int buf_idx = 0;
+	for(int i=0; i<100; i++) {
+		int0 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR);
+		int1 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR);
+		int_err = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT2_CSR);
+
+		if ((int0 >> 9) & 0x7) {  // done handle
+			// clear interrupt
+			DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR, int0);
+			//send q to donelist
+			buf_idx = (((int0 >> 9) & 0x7) & 0x1) ? 0 : (((int0 >> 9) & 0x7) & 0x2)? 1: 2;
+			vdev->queue->bufs[buf_idx]->state = VB2_BUF_STATE_DONE;
+			vdev->queue->bufs[buf_idx]->planes[0].bytesused = stream->pixm.plane_fmt[0].sizeimage;
+			list_add_tail(&vdev->queue->bufs[buf_idx]->done_entry, &vdev->queue->done_list); 
+			mask = POLLIN | POLLRDNORM;
+			break;
+		}
+
+		if (int0 & 0x1ff) {  // flush handle
+			// clear interrupt
+			DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR, int0);
+		}
+		if (int1) {  // done handle
+			// clear interrupt
+			DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, int1);
+			mask = POLLIN | POLLRDNORM;
+			break;
+
+		}
+
+		if (int1 & 0x1ff) {  // flush handle
+			// clear interrupt
+			DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, int1);
+		}
+
+		if (int_err) {  // axi error: reset required
+			DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT2_CSR, int_err);
+		}
+	}
+	return mask;
+}
+
+void dvp2axi_hw_int_mask(struct es_dvp2axi_hw *dvp2axi_hw, int mask)
+{
+
+	
+    if(mask) {
+		pr_info("%s: mask dvp2axi interrupt\n", __func__);
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT_MASK0_CSR, 0xffffffff);
+    	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT_MASK1_CSR, 0xffffffff);
+    	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT_MASK2_CSR, 0xffffffff);
+	} else {
+		pr_info("%s: unmask dvp2axi interrupt\n", __func__);
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT_MASK0_CSR, 0x0);
+    	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT_MASK1_CSR, 0x0);
+    	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT_MASK2_CSR, 0x0);
+	}
+
+}
+
+
+static const struct v4l2_file_operations es_dvp2axi_fops = {
+	.open = es_dvp2axi_fh_open,
+	.release = es_dvp2axi_fh_release,
+	.unlocked_ioctl = video_ioctl2,
+	.poll = vb2_fop_poll,
+	// .poll = eic770x_vb2_fop_poll,
+	.mmap = vb2_fop_mmap,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl32 = video_ioctl2,
+#endif
+};
+
+static int es_dvp2axi_enum_input(struct file *file, void *priv,
+			    struct v4l2_input *input)
+{
+	if (input->index > 0)
+		return -EINVAL;
+
+	input->type = V4L2_INPUT_TYPE_CAMERA;
+	strlcpy(input->name, "Camera", sizeof(input->name));
+
+	return 0;
+}
+
+static int es_dvp2axi_try_fmt_vid_cap_mplane(struct file *file, void *fh,
+					struct v4l2_format *f)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	int ret = 0;
+	ret = es_dvp2axi_set_fmt(stream, &f->fmt.pix_mp, true);
+	return ret;
+}
+
+static int es_dvp2axi_enum_framesizes(struct file *file, void *prov,
+				 struct v4l2_frmsizeenum *fsize)
+{
+	struct v4l2_frmsize_discrete *d = &fsize->discrete;
+	struct v4l2_frmsize_stepwise *s = &fsize->stepwise;
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_rect input_rect;
+	struct csi_channel_info csi_info;
+	
+	
+	if (fsize->index != 0)
+		return -EINVAL;
+
+	pr_debug("***** fsize->pixel_format =0x%x *****\n", fsize->pixel_format);
+	if (!es_dvp2axi_find_output_fmt(stream, fsize->pixel_format))
+		return -EINVAL;
+
+	input_rect.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	input_rect.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+
+	if (dev->terminal_sensor.sd)
+		es_dvp2axi_get_input_fmt(dev, &input_rect, stream->id, &csi_info);
+
+	if (dev->hw_dev->adapt_to_usbcamerahal) {
+		fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+		d->width = input_rect.width;
+		d->height = input_rect.height;
+	} else {
+		fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
+		s->min_width = DVP2AXI_MIN_WIDTH;
+		s->min_height = DVP2AXI_MIN_HEIGHT;
+		s->max_width = input_rect.width;
+		s->max_height = input_rect.height;
+		s->step_width = OUTPUT_STEP_WISE;
+		s->step_height = OUTPUT_STEP_WISE;
+	}
+
+	return 0;
+}
+
+static int es_dvp2axi_enum_frameintervals(struct file *file, void *fh,
+				     struct v4l2_frmivalenum *fival)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor = dev->active_sensor;
+	struct v4l2_subdev_frame_interval fi;
+	int ret;
+
+	if (fival->index != 0)
+		return -EINVAL;
+
+	if (!sensor || !sensor->sd) {
+		/* TODO: active_sensor is NULL if using DMARX path */
+		v4l2_err(&dev->v4l2_dev, "%s Not active sensor\n", __func__);
+		return -ENODEV;
+	}
+
+	ret = v4l2_subdev_call(sensor->sd, video, g_frame_interval, &fi);
+	if (ret && ret != -ENOIOCTLCMD) {
+		return ret;
+	} else if (ret == -ENOIOCTLCMD) {
+		/* Set a default value for sensors not implements ioctl */
+		fi.interval.numerator = 1;
+		fi.interval.denominator = 30;
+	}
+
+	if (dev->hw_dev->adapt_to_usbcamerahal) {
+		fival->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+		fival->discrete.numerator = fi.interval.numerator;
+		fival->discrete.denominator = fi.interval.denominator;
+	} else {
+		fival->type = V4L2_FRMIVAL_TYPE_CONTINUOUS;
+		fival->stepwise.step.numerator = 1;
+		fival->stepwise.step.denominator = 1;
+		fival->stepwise.max.numerator = 1;
+		fival->stepwise.max.denominator = 1;
+		fival->stepwise.min.numerator = fi.interval.numerator;
+		fival->stepwise.min.denominator = fi.interval.denominator;
+	}
+
+	return 0;
+}
+
+static int es_dvp2axi_enum_fmt_vid_cap_mplane(struct file *file, void *priv,
+					 struct v4l2_fmtdesc *f)
+{
+	const struct dvp2axi_output_fmt *fmt = NULL;
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	const struct dvp2axi_input_fmt *dvp2axi_fmt_in = NULL;
+	struct v4l2_rect input_rect;
+	int i = 0;
+	int ret = 0;
+	int fource_idx = 0;
+
+	if (f->index >= ARRAY_SIZE(out_fmts))
+		return -EINVAL;
+
+	if (dev->terminal_sensor.sd) {
+		dvp2axi_fmt_in = es_dvp2axi_get_input_fmt(dev, &input_rect, stream->id,
+						 &dev->channels[stream->id]);
+		stream->dvp2axi_fmt_in = dvp2axi_fmt_in;
+		pr_debug("*** stream->dvp2axi_fmt_in->mbus_code = %d ***\n", stream->dvp2axi_fmt_in->mbus_code);
+	} else {
+		pr_debug("terminal subdev does not exist\n");
+		return -EINVAL;
+	}
+
+	if (f->index != 0)
+		fource_idx = stream->new_fource_idx;
+
+	for (i = fource_idx; i < ARRAY_SIZE(out_fmts); i++) {
+		fmt = &out_fmts[i];
+		ret = es_dvp2axi_output_fmt_check(stream, fmt);
+		if (!ret) {
+			f->pixelformat = fmt->fourcc;
+			stream->new_fource_idx = i + 1;
+			break;
+		}
+	}
+	if (i == ARRAY_SIZE(out_fmts))
+		return -EINVAL;
+
+	switch (f->pixelformat) {
+	case V4l2_PIX_FMT_EBD8:
+		strscpy(f->description, "Embedded data 8-bit",
+			sizeof(f->description));
+		break;
+	case V4l2_PIX_FMT_SPD16:
+		strscpy(f->description, "Shield pix data 16-bit",
+			sizeof(f->description));
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int es_dvp2axi_s_fmt_vid_cap_mplane(struct file *file, void *priv,
+				      struct v4l2_format *f)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	int ret = 0;
+
+	if (vb2_is_busy(&stream->vnode.buf_queue)) {
+		v4l2_err(&dev->v4l2_dev, "%s queue busy\n", __func__);
+		return -EBUSY;
+	}
+	ret = es_dvp2axi_set_fmt(stream, &f->fmt.pix_mp, false);
+	return ret;
+}
+
+static int es_dvp2axi_g_fmt_vid_cap_mplane(struct file *file, void *fh,
+				      struct v4l2_format *f)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	f->fmt.pix_mp = stream->pixm;
+
+	return 0;
+}
+
+static int es_dvp2axi_querycap(struct file *file, void *priv,
+			  struct v4l2_capability *cap)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct device *dev = stream->dvp2axidev->dev;
+
+	strlcpy(cap->driver, dev->driver->name, sizeof(cap->driver));
+	strlcpy(cap->card, dev->driver->name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 dev_name(dev));
+
+	return 0;
+}
+
+static __maybe_unused int es_dvp2axi_cropcap(struct file *file, void *fh,
+					struct v4l2_cropcap *cap)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_rect *raw_rect = &dev->terminal_sensor.raw_rect;
+	int ret = 0;
+
+	if (stream->crop_mask & CROP_SRC_SENSOR) {
+		cap->bounds.left = stream->crop[CROP_SRC_SENSOR].left;
+		cap->bounds.top = stream->crop[CROP_SRC_SENSOR].top;
+		cap->bounds.width = stream->crop[CROP_SRC_SENSOR].width;
+		cap->bounds.height = stream->crop[CROP_SRC_SENSOR].height;
+	} else {
+		cap->bounds.left = raw_rect->left;
+		cap->bounds.top = raw_rect->top;
+		cap->bounds.width = raw_rect->width;
+		cap->bounds.height = raw_rect->height;
+	}
+
+	cap->defrect = cap->bounds;
+	cap->pixelaspect.numerator = 1;
+	cap->pixelaspect.denominator = 1;
+
+	return ret;
+}
+
+static int es_dvp2axi_s_selection(struct file *file, void *fh,
+			     struct v4l2_selection *s)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_subdev *sensor_sd;
+	struct v4l2_subdev_selection sd_sel;
+	const struct v4l2_rect *rect = &s->r;
+	struct v4l2_rect sensor_crop;
+	struct v4l2_rect *raw_rect = &dev->terminal_sensor.raw_rect;
+	u16 pad = 0;
+	int ret = 0;
+
+	if (!s) {
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev, "sel is null\n");
+		goto err;
+	}
+
+	if (s->target == V4L2_SEL_TGT_CROP_BOUNDS) {
+		sensor_sd = get_remote_sensor(stream, &pad);
+
+		sd_sel.r = s->r;
+		sd_sel.pad = pad;
+		sd_sel.target = s->target;
+		sd_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		pr_debug("%s:%d s->r.left %d,  s->r.top %d,  s->r.width %d, s->r.height %d \n", __func__, __LINE__, s->r.left, s->r.top, s->r.width, s->r.height);
+		ret = v4l2_subdev_call(sensor_sd, pad, set_selection, NULL,
+				       &sd_sel);
+		if (!ret) {
+			s->r = sd_sel.r;
+			v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "%s: pad:%d, which:%d, target:%d\n", __func__,
+				 pad, sd_sel.which, sd_sel.target);
+		}
+	} else if (s->target == V4L2_SEL_TGT_CROP) {
+		pr_debug("%s:%d start check fmt\n", __func__, __LINE__);
+		ret = es_dvp2axi_sanity_check_fmt(stream, rect);
+		if (ret) {
+			v4l2_err(&dev->v4l2_dev, "set crop failed\n");
+			return ret;
+		}
+
+		if (stream->crop_mask & CROP_SRC_SENSOR) {
+			sensor_crop = stream->crop[CROP_SRC_SENSOR];
+			if (rect->left + rect->width > sensor_crop.width ||
+			    rect->top + rect->height > sensor_crop.height) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"crop size is bigger than sensor input:left:%d, top:%d, width:%d, height:%d\n",
+					sensor_crop.left, sensor_crop.top,
+					sensor_crop.width, sensor_crop.height);
+				return -EINVAL;
+			}
+		} else {
+			if (rect->left + rect->width > raw_rect->width ||
+			    rect->top + rect->height > raw_rect->height) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"crop size is bigger than sensor raw input:left:%d, top:%d, width:%d, height:%d\n",
+					raw_rect->left, raw_rect->top,
+					raw_rect->width, raw_rect->height);
+				return -EINVAL;
+			}
+		}
+
+		stream->crop[CROP_SRC_USR] = *rect;
+		stream->crop_enable = true;
+		stream->crop_mask |= CROP_SRC_USR_MASK;
+		stream->crop[CROP_SRC_ACT] = stream->crop[CROP_SRC_USR];
+		if (stream->crop_mask & CROP_SRC_SENSOR) {
+			sensor_crop = stream->crop[CROP_SRC_SENSOR];
+			stream->crop[CROP_SRC_ACT].left =
+				sensor_crop.left +
+				stream->crop[CROP_SRC_USR].left;
+			stream->crop[CROP_SRC_ACT].top =
+				sensor_crop.top +
+				stream->crop[CROP_SRC_USR].top;
+		}
+
+		if (stream->state == ES_DVP2AXI_STATE_STREAMING) {
+			stream->crop_dyn_en = true;
+
+			v4l2_info(
+				&dev->v4l2_dev,
+				"enable dynamic crop, S_SELECTION(%ux%u@%u:%u) target: %d\n",
+				rect->width, rect->height, rect->left,
+				rect->top, s->target);
+		} else {
+			v4l2_info(
+				&dev->v4l2_dev,
+				"static crop, S_SELECTION(%ux%u@%u:%u) target: %d\n",
+				rect->width, rect->height, rect->left,
+				rect->top, s->target);
+		}
+	} else {
+		goto err;
+	}
+
+	return ret;
+
+err:
+	return -EINVAL;
+}
+
+static int es_dvp2axi_g_selection(struct file *file, void *fh,
+			     struct v4l2_selection *s)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_subdev *sensor_sd;
+	struct v4l2_subdev_selection sd_sel;
+	u16 pad = 0;
+	int ret = 0;
+
+	if (!s) {
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev, "sel is null\n");
+		goto err;
+	}
+
+	if (s->target == V4L2_SEL_TGT_CROP_BOUNDS) {
+		sensor_sd = get_remote_sensor(stream, &pad);
+
+		sd_sel.pad = pad;
+		sd_sel.target = s->target;
+		sd_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+			 "%s(line:%d): sd:%s pad:%d, which:%d, target:%d\n",
+			 __func__, __LINE__, sensor_sd->name, pad, sd_sel.which,
+			 sd_sel.target);
+
+		ret = v4l2_subdev_call(sensor_sd, pad, get_selection, NULL,
+				       &sd_sel);
+		if (!ret) {
+			s->r = sd_sel.r;
+		} else {
+			s->r.left = 0;
+			s->r.top = 0;
+			s->r.width = stream->pixm.width;
+			s->r.height = stream->pixm.height;
+		}
+	} else if (s->target == V4L2_SEL_TGT_CROP) {
+		if (stream->crop_mask &
+		    (CROP_SRC_USR_MASK | CROP_SRC_SENSOR_MASK)) {
+			s->r = stream->crop[CROP_SRC_ACT];
+		} else {
+			s->r.left = 0;
+			s->r.top = 0;
+			s->r.width = stream->pixm.width;
+			s->r.height = stream->pixm.height;
+		}
+	} else {
+		goto err;
+	}
+
+	return ret;
+err:
+	return -EINVAL;
+}
+
+static int es_dvp2axi_get_max_common_div(int a, int b)
+{
+	int remainder = a % b;
+
+	while (remainder != 0) {
+		a = b;
+		b = remainder;
+		remainder = a % b;
+	}
+	return b;
+}
+
+void es_dvp2axi_set_fps(struct es_dvp2axi_stream *stream, struct es_dvp2axi_fps *fps)
+{
+	struct es_dvp2axi_sensor_info *sensor = &stream->dvp2axidev->terminal_sensor;
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct es_dvp2axi_stream *tmp_stream = NULL;
+	u32 numerator, denominator;
+	u32 def_fps = 0;
+	u32 cur_fps = 0;
+	int cap_m, skip_n;
+	int i = 0;
+	int max_common_div;
+	bool skip_en = false;
+	s32 vblank_def = 0;
+	s32 vblank_curr = 0;
+	int ret = 0;
+
+	if (!stream->dvp2axidev->terminal_sensor.sd) {
+		ret = es_dvp2axi_update_sensor_info(stream);
+		if (ret) {
+			v4l2_dbg(1, es_dvp2axi_debug ,&stream->dvp2axidev->v4l2_dev,
+				 "%s update sensor info fail\n", __func__);
+			return;
+		}
+	}
+	if (!stream->dvp2axidev->terminal_sensor.sd)
+		return;
+	numerator = sensor->fi.interval.numerator;
+	denominator = sensor->fi.interval.denominator;
+	def_fps = denominator / numerator;
+
+	vblank_def = es_dvp2axi_get_sensor_vblank_def(dvp2axi_dev);
+	vblank_curr = es_dvp2axi_get_sensor_vblank(dvp2axi_dev);
+	if (vblank_def)
+		cur_fps = def_fps *
+			  (u32)(vblank_def + sensor->raw_rect.height) /
+			  (u32)(vblank_curr + sensor->raw_rect.height);
+	else
+		cur_fps = def_fps;
+
+	if (fps->fps == 0 || fps->fps > cur_fps) {
+		v4l2_err(&stream->dvp2axidev->v4l2_dev,
+			 "set fps %d fps failed, current fps %d fps\n",
+			 fps->fps, cur_fps);
+		return;
+	}
+	cap_m = fps->fps;
+	skip_n = cur_fps - fps->fps;
+	max_common_div = es_dvp2axi_get_max_common_div(cap_m, skip_n);
+	cap_m /= max_common_div;
+	skip_n /= max_common_div;
+	if (cap_m > 64) {
+		skip_n = skip_n / (cap_m / 64);
+		if (skip_n == 0)
+			skip_n = 1;
+		cap_m = 64;
+	}
+	if (skip_n > 7) {
+		cap_m = cap_m / (skip_n / 7);
+		if (cap_m == 0)
+			cap_m = 1;
+		skip_n = 7;
+	}
+
+	if (fps->fps == cur_fps)
+		skip_en = false;
+	else
+		skip_en = true;
+
+	if (fps->ch_num > 1 && fps->ch_num < 4) {
+		for (i = 0; i < fps->ch_num; i++) {
+			tmp_stream = &dvp2axi_dev->stream[i];
+			if (skip_en) {
+				tmp_stream->skip_info.skip_to_en = true;
+				tmp_stream->skip_info.cap_m = cap_m;
+				tmp_stream->skip_info.skip_n = skip_n;
+			} else {
+				tmp_stream->skip_info.skip_to_dis = true;
+			}
+		}
+	} else {
+		if (skip_en) {
+			stream->skip_info.skip_to_en = true;
+			stream->skip_info.cap_m = cap_m;
+			stream->skip_info.skip_n = skip_n;
+		} else {
+			stream->skip_info.skip_to_dis = true;
+		}
+	}
+	v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		 "skip_to_en %d, cap_m %d, skip_n %d\n",
+		 stream->skip_info.skip_to_en, cap_m, skip_n);
+}
+
+static bool es_dvp2axi_check_can_be_online(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	return true;
+}
+
+static int es_dvp2axi_do_reset_work(struct es_dvp2axi_device *dvp2axi_dev,
+			       enum esmodule_reset_src reset_src);
+static long es_dvp2axi_ioctl_default(struct file *file, void *fh, bool valid_prio,
+				unsigned int cmd, void *arg)
+{
+	struct es_dvp2axi_stream *stream = video_drvdata(file);
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	const struct dvp2axi_input_fmt *in_fmt;
+	struct v4l2_rect rect;
+	struct csi_channel_info csi_info;
+	struct es_dvp2axi_fps fps;
+	int reset_src;
+	struct es_dvp2axi_quick_stream_param *stream_param;
+	bool is_single_dev = false;
+	struct v4l2_subdev *sd;
+	int ret = -EINVAL;
+	int i = 0;
+	int stream_num = 0;
+	bool is_can_be_online = false;
+
+	switch (cmd) {
+	case ES_DVP2AXI_CMD_GET_CSI_MEMORY_MODE:
+		if (stream->is_compact) {
+			*(int *)arg = CSI_LVDS_MEM_COMPACT;
+		} else {
+			if (stream->is_high_align)
+				*(int *)arg = CSI_LVDS_MEM_WORD_HIGH_ALIGN;
+			else
+				*(int *)arg = CSI_LVDS_MEM_WORD_LOW_ALIGN;
+		}
+		break;
+	case ES_DVP2AXI_CMD_SET_CSI_MEMORY_MODE:
+		if (dev->terminal_sensor.sd) {
+			in_fmt = es_dvp2axi_get_input_fmt(dev, &rect, 0, &csi_info);
+			if (in_fmt == NULL) {
+				v4l2_err(&dev->v4l2_dev,
+					 "can't get sensor input format\n");
+				return -EINVAL;
+			}
+		} else {
+			v4l2_err(&dev->v4l2_dev, "can't get sensor device\n");
+			return -EINVAL;
+		}
+		if (*(int *)arg == CSI_LVDS_MEM_COMPACT) {
+			stream->is_compact = true;
+			stream->is_high_align = false;
+		} else if (*(int *)arg == CSI_LVDS_MEM_WORD_HIGH_ALIGN) {
+			stream->is_compact = false;
+			stream->is_high_align = true;
+		} else {
+			stream->is_compact = false;
+			stream->is_high_align = false;
+		}
+		break;
+	case ES_DVP2AXI_CMD_SET_FPS:
+		fps = *(struct es_dvp2axi_fps *)arg;
+		es_dvp2axi_set_fps(stream, &fps);
+		break;
+	case ES_DVP2AXI_CMD_SET_RESET:
+		reset_src = *(int *)arg;
+		return es_dvp2axi_do_reset_work(dev, reset_src);
+	case ES_DVP2AXI_CMD_SET_QUICK_STREAM:
+		stream_param = (struct es_dvp2axi_quick_stream_param *)arg;
+		if (!dev->sditf[0])
+			return -EINVAL;
+		if (dev->hdr.hdr_mode == HDR_X2)
+			stream_num = 2;
+		else if (dev->hdr.hdr_mode == HDR_X3)
+			stream_num = 3;
+		else
+			stream_num = 1;
+		if (stream_param->on) {
+			for (i = 0; i < stream_num; i++)
+				dev->stream[i].cur_skip_frame =
+					dev->stream[i].skip_frame;
+			is_single_dev =
+				es_dvp2axi_check_single_dev_stream_on(dev->hw_dev);
+			is_can_be_online = es_dvp2axi_check_can_be_online(dev);
+			if (is_single_dev && is_can_be_online) {
+				for (i = 0; i < stream_num - 1; i++) {
+					dev->stream[i].to_en_dma =
+						ES_DVP2AXI_DMAEN_BY_ISP;
+					es_dvp2axi_enable_dma_capture(
+						&dev->stream[i], true);
+				}
+				dev->sditf[0]->mode.rdbk_mode =
+					ESISP_VICAP_ONLINE;
+				sditf_change_to_online(dev->sditf[0]);
+				sd = get_esisp_sd(dev->sditf[0]);
+				if (sd)
+					ret = v4l2_subdev_call(
+						sd, core, ioctl,
+						ESISP_VICAP_CMD_MODE,
+						&dev->sditf[0]->mode);
+				if (ret) {
+					v4l2_err(
+						&dev->v4l2_dev,
+						"set isp work mode online fail\n");
+					return -EINVAL;
+				}
+			} else {
+				sditf_disable_immediately(dev->sditf[0]);
+				dev->sditf[0]->mode.rdbk_mode =
+					ESISP_VICAP_RDBK_AUTO;
+				sd = get_esisp_sd(dev->sditf[0]);
+				if (sd)
+					ret = v4l2_subdev_call(
+						sd, core, ioctl,
+						ESISP_VICAP_CMD_MODE,
+						&dev->sditf[0]->mode);
+				for (i = 0; i < stream_num; i++) {
+					if (dev->sditf[0]->mode.rdbk_mode ==
+					    ESISP_VICAP_RDBK_AUTO)
+						dev->stream[i].to_en_dma =
+							ES_DVP2AXI_DMAEN_BY_ISP;
+					else
+						dev->stream[i].to_en_dma =
+							ES_DVP2AXI_DMAEN_BY_VICAP;
+					es_dvp2axi_enable_dma_capture(
+						&dev->stream[i], true);
+				}
+			}
+			es_dvp2axi_dphy_quick_stream(dev, stream_param->on);
+			v4l2_subdev_call(dev->terminal_sensor.sd, core, ioctl,
+					 ESMODULE_SET_QUICK_STREAM,
+					 &stream_param->on);
+		} else {
+			if (dev->sditf[0]->mode.rdbk_mode ==
+			    ESISP_VICAP_ONLINE) {
+				for (i = 0; i < stream_num - 1; i++) {
+					reinit_completion(
+						&dev->stream[i].stop_complete);
+					dev->stream[i].is_wait_stop_complete =
+						true;
+					dev->stream[i].to_stop_dma =
+						ES_DVP2AXI_DMAEN_BY_ISP;
+					wait_for_completion_timeout(
+						&dev->stream[i].stop_complete,
+						msecs_to_jiffies(
+							ES_DVP2AXI_STOP_MAX_WAIT_TIME_MS));
+				}
+				stream->dvp2axidev->sensor_state = stream_param->on;
+				stream->dvp2axidev->sensor_state_change = true;
+				dev->stream[i].is_wait_stop_complete = true;
+				wait_for_completion_timeout(
+					&dev->stream[i].stop_complete,
+					msecs_to_jiffies(
+						ES_DVP2AXI_STOP_MAX_WAIT_TIME_MS));
+			} else {
+				for (i = 0; i < stream_num; i++) {
+					dev->stream[i].is_wait_stop_complete =
+						true;
+					reinit_completion(
+						&dev->stream[i].stop_complete);
+					if (dev->sditf[0]->mode.rdbk_mode ==
+					    ESISP_VICAP_RDBK_AUTO)
+						dev->stream[i].to_stop_dma =
+							ES_DVP2AXI_DMAEN_BY_ISP;
+					else
+						dev->stream[i].to_stop_dma =
+							ES_DVP2AXI_DMAEN_BY_VICAP;
+					wait_for_completion_timeout(
+						&dev->stream[i].stop_complete,
+						msecs_to_jiffies(
+							ES_DVP2AXI_STOP_MAX_WAIT_TIME_MS));
+				}
+				es_dvp2axi_dphy_quick_stream(dev, stream_param->on);
+				v4l2_subdev_call(dev->terminal_sensor.sd, core,
+						 ioctl,
+						 ESMODULE_SET_QUICK_STREAM,
+						 &stream_param->on);
+			}
+			stream_param->frame_num = dev->stream[0].frame_idx - 1;
+			if (!dev->is_rtt_suspend) {
+				dev->resume_mode = stream_param->resume_mode;
+				v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev,
+					 "set resume mode %d\n",
+					 dev->resume_mode);
+			}
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops es_dvp2axi_v4l2_ioctl_ops = {
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+	.vidioc_enum_input = es_dvp2axi_enum_input,
+	.vidioc_try_fmt_vid_cap_mplane = es_dvp2axi_try_fmt_vid_cap_mplane,
+	.vidioc_enum_fmt_vid_cap = es_dvp2axi_enum_fmt_vid_cap_mplane,
+	.vidioc_s_fmt_vid_cap_mplane = es_dvp2axi_s_fmt_vid_cap_mplane,
+	.vidioc_g_fmt_vid_cap_mplane = es_dvp2axi_g_fmt_vid_cap_mplane,
+	.vidioc_querycap = es_dvp2axi_querycap,
+	.vidioc_s_selection = es_dvp2axi_s_selection,
+	.vidioc_g_selection = es_dvp2axi_g_selection,
+	.vidioc_enum_frameintervals = es_dvp2axi_enum_frameintervals,
+	.vidioc_enum_framesizes = es_dvp2axi_enum_framesizes,
+	.vidioc_default = es_dvp2axi_ioctl_default,
+};
+
+void es_dvp2axi_vb_done_oneframe(struct es_dvp2axi_stream *stream,
+			    struct vb2_v4l2_buffer *vb_done)
+{
+	const struct dvp2axi_output_fmt *fmt = stream->dvp2axi_fmt_out;
+	u32 i;
+
+	/* Dequeue a filled buffer */
+	for (i = 0; i < fmt->mplanes; i++) {
+		vb2_set_plane_payload(&vb_done->vb2_buf, i,
+				      stream->pixm.plane_fmt[i].sizeimage);
+	}
+
+	// pr_info("%s:%d yfx!!!!! \n", __func__, __LINE__);
+	vb2_buffer_done(&vb_done->vb2_buf, VB2_BUF_STATE_DONE);
+	v4l2_dbg(2, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		 "stream[%d] vb done, index: %d, sequence %d\n", stream->id,
+		 vb_done->vb2_buf.index, vb_done->sequence);
+	atomic_dec(&stream->buf_cnt);
+}
+
+static void es_dvp2axi_tasklet_handle(unsigned long data)
+{
+	struct es_dvp2axi_stream *stream = (struct es_dvp2axi_stream *)data;
+	struct es_dvp2axi_buffer *buf = NULL;
+	unsigned long flags = 0;
+	LIST_HEAD(local_list);
+
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	list_replace_init(&stream->vb_done_list, &local_list);
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	while (!list_empty(&local_list)) {
+		buf = list_first_entry(&local_list, struct es_dvp2axi_buffer, queue);
+		list_del(&buf->queue);
+		es_dvp2axi_vb_done_oneframe(stream, &buf->vb);
+	}
+}
+
+void es_dvp2axi_vb_done_tasklet(struct es_dvp2axi_stream *stream,
+			   struct es_dvp2axi_buffer *buf)
+{
+	unsigned long flags = 0;
+
+	if (!stream || !buf)
+		return;
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	list_add_tail(&buf->queue, &stream->vb_done_list);
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	tasklet_schedule(&stream->vb_done_tasklet);
+}
+
+static void es_dvp2axi_unregister_stream_vdev(struct es_dvp2axi_stream *stream)
+{
+	tasklet_kill(&stream->vb_done_tasklet);
+	media_entity_cleanup(&stream->vnode.vdev.entity);
+	video_unregister_device(&stream->vnode.vdev);
+}
+
+static int es_dvp2axi_register_stream_vdev(struct es_dvp2axi_stream *stream,
+				      bool is_multi_input)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+	struct video_device *vdev = &stream->vnode.vdev;
+	struct es_dvp2axi_vdev_node *node;
+	int ret = 0;
+	char *vdev_name;
+
+	if (dev->inf_id == ES_DVP2AXI_MIPI_LVDS) {
+		switch (stream->id) {
+		case ESDVP2AXI_STREAM_MIPI_ID0:
+			vdev_name = DVP2AXI_MIPI_ID0_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID1:
+			vdev_name = DVP2AXI_MIPI_ID1_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID2:
+			vdev_name = DVP2AXI_MIPI_ID2_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID3:
+			vdev_name = DVP2AXI_MIPI_ID3_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID4:
+			vdev_name = DVP2AXI_MIPI_ID4_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID5:
+			vdev_name = DVP2AXI_MIPI_ID5_VDEV_NAME;
+			break;
+		default:
+			ret = -EINVAL;
+			v4l2_err(v4l2_dev, "Invalid stream\n");
+			goto unreg;
+		}
+	} else {
+		switch (stream->id) {
+		case ESDVP2AXI_STREAM_MIPI_ID0:
+			vdev_name = DVP2AXI_MIPI_ID0_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID1:
+			vdev_name = DVP2AXI_MIPI_ID1_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID2:
+			vdev_name = DVP2AXI_MIPI_ID2_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID3:
+			vdev_name = DVP2AXI_MIPI_ID3_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID4:
+			vdev_name = DVP2AXI_MIPI_ID4_VDEV_NAME;
+			break;
+		case ESDVP2AXI_STREAM_MIPI_ID5:
+			vdev_name = DVP2AXI_MIPI_ID5_VDEV_NAME;
+			break;
+		default:
+			ret = -EINVAL;
+			v4l2_err(v4l2_dev, "Invalid stream\n");
+			goto unreg;
+		}
+	}
+
+	strlcpy(vdev->name, vdev_name, sizeof(vdev->name));
+	v4l2_err(v4l2_dev, "vdev->name %s \n", vdev->name);
+	node = vdev_to_node(vdev);
+	mutex_init(&node->vlock);
+	vdev->ioctl_ops = &es_dvp2axi_v4l2_ioctl_ops;
+	vdev->release = video_device_release_empty;
+	vdev->fops = &es_dvp2axi_fops;
+	vdev->minor = -1;
+	vdev->v4l2_dev = v4l2_dev;
+	vdev->lock = &node->vlock;
+	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_STREAMING;
+	video_set_drvdata(vdev, stream);
+	vdev->vfl_dir = VFL_DIR_RX;
+	node->pad.flags = MEDIA_PAD_FL_SINK;
+	es_dvp2axi_init_vb2_queue(&node->buf_queue, stream,
+			     V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
+	vdev->queue = &node->buf_queue;
+	
+	ret = media_entity_pads_init(&vdev->entity, 1, &node->pad);
+	if (ret < 0)
+		goto unreg;
+
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret < 0) {
+		v4l2_err(v4l2_dev,
+			 "video_register_device failed with error %d\n", ret);
+		return ret;
+	}
+
+
+	INIT_LIST_HEAD(&stream->vb_done_list);
+	tasklet_init(&stream->vb_done_tasklet, es_dvp2axi_tasklet_handle,
+		     (unsigned long)stream);
+	tasklet_disable(&stream->vb_done_tasklet);
+	return 0;
+unreg:
+	video_unregister_device(vdev);
+	return ret;
+}
+
+void es_dvp2axi_unregister_stream_vdevs(struct es_dvp2axi_device *dev, int stream_num)
+{
+	struct es_dvp2axi_stream *stream;
+	int i;
+
+	for (i = 0; i < stream_num; i++) {
+		stream = &dev->stream[i];
+		es_dvp2axi_unregister_stream_vdev(stream);
+	}
+}
+
+int es_dvp2axi_register_stream_vdevs(struct es_dvp2axi_device *dev, int stream_num,
+				bool is_multi_input)
+{
+	struct es_dvp2axi_stream *stream;
+	int i, j, ret;
+
+	for (i = 0; i < stream_num; i++) {
+		stream = &dev->stream[i];
+		stream->dvp2axidev = dev;
+		ret = es_dvp2axi_register_stream_vdev(stream, is_multi_input);
+		if (ret < 0)
+			goto err;
+	}
+	dev->num_channels = stream_num;
+	return 0;
+err:
+	for (j = 0; j < i; j++) {
+		stream = &dev->stream[j];
+		es_dvp2axi_unregister_stream_vdev(stream);
+	}
+
+	return ret;
+}
+
+static struct v4l2_subdev *get_lvds_remote_sensor(struct v4l2_subdev *sd)
+{
+	struct media_pad *local, *remote;
+	struct media_entity *sensor_me;
+
+	local = &sd->entity.pads[ES_DVP2AXI_LVDS_PAD_SINK];
+	remote = media_pad_remote_pad_first(local);
+	if (!remote) {
+		v4l2_warn(sd, "No link between dphy and sensor with lvds\n");
+		return NULL;
+	}
+
+	sensor_me = media_pad_remote_pad_first(local)->entity;
+	return media_entity_to_v4l2_subdev(sensor_me);
+}
+
+static int es_dvp2axi_lvds_subdev_link_setup(struct media_entity *entity,
+					const struct media_pad *local,
+					const struct media_pad *remote,
+					u32 flags)
+{
+	return 0;
+}
+
+static int es_dvp2axi_lvds_sd_set_fmt(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct v4l2_subdev *sensor = get_lvds_remote_sensor(sd);
+
+	/*
+	 * Do not allow format changes and just relay whatever
+	 * set currently in the sensor.
+	 */
+	return v4l2_subdev_call(sensor, pad, get_fmt, NULL, fmt);
+}
+
+static int es_dvp2axi_lvds_sd_get_fmt(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct es_dvp2axi_lvds_subdev *subdev =
+		container_of(sd, struct es_dvp2axi_lvds_subdev, sd);
+	struct v4l2_subdev *sensor = get_lvds_remote_sensor(sd);
+	int ret;
+
+	/*
+	 * Do not allow format changes and just relay whatever
+	 * set currently in the sensor.
+	 */
+	ret = v4l2_subdev_call(sensor, pad, get_fmt, NULL, fmt);
+	if (!ret)
+		subdev->in_fmt = fmt->format;
+
+	return ret;
+}
+
+static struct v4l2_rect *
+es_dvp2axi_lvds_sd_get_crop(struct es_dvp2axi_lvds_subdev *subdev,
+		       struct v4l2_subdev_state *sd_state,
+		       enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_crop(&subdev->sd, sd_state,
+						ES_DVP2AXI_LVDS_PAD_SINK);
+	else
+		return &subdev->crop;
+}
+
+static int es_dvp2axi_lvds_sd_set_selection(struct v4l2_subdev *sd,
+				       struct v4l2_subdev_state *sd_state,
+				       struct v4l2_subdev_selection *sel)
+{
+	struct es_dvp2axi_lvds_subdev *subdev =
+		container_of(sd, struct es_dvp2axi_lvds_subdev, sd);
+	struct v4l2_subdev *sensor = get_lvds_remote_sensor(sd);
+	int ret = 0;
+
+	ret = v4l2_subdev_call(sensor, pad, set_selection, sd_state, sel);
+	if (!ret)
+		subdev->crop = sel->r;
+
+	return ret;
+}
+
+static int es_dvp2axi_lvds_sd_get_selection(struct v4l2_subdev *sd,
+				       struct v4l2_subdev_state *sd_state,
+				       struct v4l2_subdev_selection *sel)
+{
+	struct es_dvp2axi_lvds_subdev *subdev =
+		container_of(sd, struct es_dvp2axi_lvds_subdev, sd);
+	struct v4l2_subdev *sensor = get_lvds_remote_sensor(sd);
+	struct v4l2_subdev_format fmt;
+	int ret = 0;
+
+	if (!sel) {
+		v4l2_dbg(1, es_dvp2axi_debug, sd, "sel is null\n");
+		goto err;
+	}
+
+	if (sel->pad > ES_DVP2AXI_LVDS_PAD_SRC_ID3) {
+		v4l2_dbg(1, es_dvp2axi_debug, sd, "pad[%d] isn't matched\n",
+			 sel->pad);
+		goto err;
+	}
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+			ret = v4l2_subdev_call(sensor, pad, get_selection,
+					       sd_state, sel);
+			if (ret) {
+				fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+				ret = v4l2_subdev_call(sensor, pad, get_fmt,
+						       NULL, &fmt);
+				if (!ret) {
+					subdev->in_fmt = fmt.format;
+					sel->r.top = 0;
+					sel->r.left = 0;
+					sel->r.width = subdev->in_fmt.width;
+					sel->r.height = subdev->in_fmt.height;
+					subdev->crop = sel->r;
+				} else {
+					sel->r = subdev->crop;
+				}
+			} else {
+				subdev->crop = sel->r;
+			}
+		} else {
+			sel->r = *v4l2_subdev_get_try_crop(sd, sd_state,
+							   sel->pad);
+		}
+		break;
+
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *es_dvp2axi_lvds_sd_get_crop(subdev, sd_state, sel->which);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+err:
+	return -EINVAL;
+}
+
+static int es_dvp2axi_lvds_g_mbus_config(struct v4l2_subdev *sd, unsigned int pad_id,
+				    struct v4l2_mbus_config *mbus)
+{
+	struct v4l2_subdev *sensor_sd = get_lvds_remote_sensor(sd);
+	int ret;
+
+	ret = v4l2_subdev_call(sensor_sd, pad, get_mbus_config, 0, mbus);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int es_dvp2axi_lvds_sd_s_stream(struct v4l2_subdev *sd, int on)
+{
+	struct es_dvp2axi_lvds_subdev *subdev =
+		container_of(sd, struct es_dvp2axi_lvds_subdev, sd);
+
+	if (on)
+		atomic_set(&subdev->frm_sync_seq, 0);
+
+	return 0;
+}
+
+static int es_dvp2axi_lvds_sd_s_power(struct v4l2_subdev *sd, int on)
+{
+	return 0;
+}
+
+#define ESCIF_V4L2_EVENT_ELEMS 4
+
+static int es_dvp2axi_sof_subscribe_event(struct v4l2_subdev *sd, struct v4l2_fh *fh,
+				     struct v4l2_event_subscription *sub)
+{
+	if (sub->type == V4L2_EVENT_FRAME_SYNC ||
+	    sub->type == V4L2_EVENT_RESET_DEV)
+		return v4l2_event_subscribe(fh, sub, ES_DVP2AXI_V4L2_EVENT_ELEMS,
+					    NULL);
+	else
+		return -EINVAL;
+}
+
+static const struct media_entity_operations es_dvp2axi_lvds_sd_media_ops = {
+	.link_setup = es_dvp2axi_lvds_subdev_link_setup,
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+static const struct v4l2_subdev_pad_ops es_dvp2axi_lvds_sd_pad_ops = {
+	.set_fmt = es_dvp2axi_lvds_sd_set_fmt,
+	.get_fmt = es_dvp2axi_lvds_sd_get_fmt,
+	.set_selection = es_dvp2axi_lvds_sd_set_selection,
+	.get_selection = es_dvp2axi_lvds_sd_get_selection,
+	.get_mbus_config = es_dvp2axi_lvds_g_mbus_config,
+};
+
+static const struct v4l2_subdev_video_ops es_dvp2axi_lvds_sd_video_ops = {
+	.s_stream = es_dvp2axi_lvds_sd_s_stream,
+};
+
+static const struct v4l2_subdev_core_ops es_dvp2axi_lvds_sd_core_ops = {
+	.s_power = es_dvp2axi_lvds_sd_s_power,
+	.subscribe_event = es_dvp2axi_sof_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static struct v4l2_subdev_ops es_dvp2axi_lvds_sd_ops = {
+	.core = &es_dvp2axi_lvds_sd_core_ops,
+	.video = &es_dvp2axi_lvds_sd_video_ops,
+	.pad = &es_dvp2axi_lvds_sd_pad_ops,
+};
+
+static void es_dvp2axi_lvds_event_inc_sof(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_lvds_subdev *subdev = &dev->lvds_subdev;
+
+	if (subdev) {
+		struct v4l2_event event = {
+			.type = V4L2_EVENT_FRAME_SYNC,
+			.u.frame_sync.frame_sequence =
+				atomic_inc_return(&subdev->frm_sync_seq) - 1,
+		};
+		v4l2_event_queue(subdev->sd.devnode, &event);
+	}
+}
+
+static u32 es_dvp2axi_lvds_get_sof(struct es_dvp2axi_device *dev)
+{
+	if (dev)
+		return atomic_read(&dev->lvds_subdev.frm_sync_seq) - 1;
+
+	return 0;
+}
+
+static u32 es_dvp2axi_lvds_set_sof(struct es_dvp2axi_device *dev, u32 seq)
+{
+	if (dev)
+		atomic_set(&dev->lvds_subdev.frm_sync_seq, seq);
+
+	return 0;
+}
+
+int es_dvp2axi_register_lvds_subdev(struct es_dvp2axi_device *dev)
+{
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+	struct es_dvp2axi_lvds_subdev *lvds_subdev = &dev->lvds_subdev;
+	struct v4l2_subdev *sd;
+	int ret;
+	int pad_num = 4;
+
+	memset(lvds_subdev, 0, sizeof(*lvds_subdev));
+	lvds_subdev->dvp2axidev = dev;
+	sd = &lvds_subdev->sd;
+	lvds_subdev->state = ES_DVP2AXI_LVDS_STOP;
+	v4l2_subdev_init(sd, &es_dvp2axi_lvds_sd_ops);
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
+	sd->entity.ops = &es_dvp2axi_lvds_sd_media_ops;
+
+	snprintf(sd->name, sizeof(sd->name), "es_dvp2axi-lvds-subdev");
+
+	lvds_subdev->pads[ES_DVP2AXI_LVDS_PAD_SINK].flags =
+		MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT;
+	lvds_subdev->pads[ES_DVP2AXI_LVDS_PAD_SRC_ID0].flags = MEDIA_PAD_FL_SOURCE;
+	lvds_subdev->pads[ES_DVP2AXI_LVDS_PAD_SRC_ID1].flags = MEDIA_PAD_FL_SOURCE;
+	lvds_subdev->pads[ES_DVP2AXI_LVDS_PAD_SRC_ID2].flags = MEDIA_PAD_FL_SOURCE;
+	lvds_subdev->pads[ES_DVP2AXI_LVDS_PAD_SRC_ID3].flags = MEDIA_PAD_FL_SOURCE;
+	lvds_subdev->in_fmt.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	lvds_subdev->in_fmt.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+	lvds_subdev->crop.left = 0;
+	lvds_subdev->crop.top = 0;
+	lvds_subdev->crop.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	lvds_subdev->crop.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+
+	ret = media_entity_pads_init(&sd->entity, pad_num, lvds_subdev->pads);
+	if (ret < 0)
+		return ret;
+	sd->owner = THIS_MODULE;
+	v4l2_set_subdevdata(sd, lvds_subdev);
+	ret = v4l2_device_register_subdev(v4l2_dev, sd);
+	if (ret < 0)
+		goto free_media;
+
+	ret = v4l2_device_register_subdev_nodes(v4l2_dev);
+	if (ret < 0)
+		goto free_subdev;
+	return ret;
+free_subdev:
+	v4l2_device_unregister_subdev(sd);
+free_media:
+	media_entity_cleanup(&sd->entity);
+	v4l2_err(sd, "Failed to register subdev, ret:%d\n", ret);
+	return ret;
+}
+
+void es_dvp2axi_unregister_lvds_subdev(struct es_dvp2axi_device *dev)
+{
+	struct v4l2_subdev *sd = &dev->lvds_subdev.sd;
+
+	v4l2_device_unregister_subdev(sd);
+	media_entity_cleanup(&sd->entity);
+}
+
+static void es_dvp2axi_dvp_event_inc_sof(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_dvp_sof_subdev *subdev = &dev->dvp_sof_subdev;
+
+	if (subdev) {
+		struct v4l2_event event = {
+			.type = V4L2_EVENT_FRAME_SYNC,
+			.u.frame_sync.frame_sequence =
+				atomic_inc_return(&subdev->frm_sync_seq) - 1,
+		};
+		v4l2_event_queue(subdev->sd.devnode, &event);
+	}
+}
+
+static u32 es_dvp2axi_dvp_get_sof(struct es_dvp2axi_device *dev)
+{
+	if (dev)
+		return atomic_read(&dev->dvp_sof_subdev.frm_sync_seq) - 1;
+
+	return 0;
+}
+
+static u32 es_dvp2axi_dvp_set_sof(struct es_dvp2axi_device *dev, u32 seq)
+{
+	if (dev)
+		atomic_set(&dev->dvp_sof_subdev.frm_sync_seq, seq);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops es_dvp2axi_dvp_sof_sd_core_ops = {
+	.subscribe_event = es_dvp2axi_sof_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static struct v4l2_subdev_ops es_dvp2axi_dvp_sof_sd_ops = {
+	.core = &es_dvp2axi_dvp_sof_sd_core_ops,
+};
+
+int es_dvp2axi_register_dvp_sof_subdev(struct es_dvp2axi_device *dev)
+{
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+	struct es_dvp2axi_dvp_sof_subdev *subdev = &dev->dvp_sof_subdev;
+	struct v4l2_subdev *sd;
+	int ret;
+
+	memset(subdev, 0, sizeof(*subdev));
+	subdev->dvp2axidev = dev;
+	sd = &subdev->sd;
+	v4l2_subdev_init(sd, &es_dvp2axi_dvp_sof_sd_ops);
+	sd->owner = THIS_MODULE;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
+	snprintf(sd->name, sizeof(sd->name), "es_dvp2axi-dvp-sof");
+
+	v4l2_set_subdevdata(sd, subdev);
+	ret = v4l2_device_register_subdev(v4l2_dev, sd);
+	if (ret < 0)
+		goto end;
+
+	ret = v4l2_device_register_subdev_nodes(v4l2_dev);
+	if (ret < 0)
+		goto free_subdev;
+
+	return ret;
+
+free_subdev:
+	v4l2_device_unregister_subdev(sd);
+
+end:
+	v4l2_err(sd, "Failed to register subdev, ret:%d\n", ret);
+	return ret;
+}
+
+void es_dvp2axi_unregister_dvp_sof_subdev(struct es_dvp2axi_device *dev)
+{
+	struct v4l2_subdev *sd = &dev->dvp_sof_subdev.sd;
+
+	v4l2_device_unregister_subdev(sd);
+}
+
+void dvp2axi_hw_soft_reset(struct es_dvp2axi_hw *dvp2axi_hw)
+{
+	uint32_t soft_rstn;
+	u32 count = 0;
+		// do software reset
+	soft_rstn = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR) & (~VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, soft_rstn);
+	do {
+		volatile uint32_t cycle = 0;
+		while (cycle++ < 100);
+		soft_rstn = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR);
+		udelay(1);
+	} while ((soft_rstn & VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK) != VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK && count++ < 100);
+    	// release reset
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, soft_rstn | VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+}
+
+void es_irq_oneframe(struct device *dev, struct es_dvp2axi_device *dvp2axi_dev)
+{
+	u32 vi_dvp2axi_int0, vi_dvp2axi_int1;
+	u32 vi_dvp2axi_int_err;
+	int ret = 0;
+	struct es_dvp2axi_hw	*dvp2axi_hw =  dev_get_drvdata(dev);
+	struct es_dvp2axi_stream *stream = &dvp2axi_dev->stream[ES_DVP2AXI_STREAM_DVP2AXI];
+	//struct es_dvp2axi_stream *stream;
+	struct es_dvp2axi_buffer *active_buf = NULL;
+	//int dvp_chn;
+	int es_dvp2axi_addr_state;
+	
+	vi_dvp2axi_int0 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR);
+	vi_dvp2axi_int1 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR);
+	vi_dvp2axi_int_err = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT2_CSR);
+	
+	//TODO add error process into err tasklet
+	if (vi_dvp2axi_int_err & VI_DVP2AXI_INT2_AXI_IDBUFFER_AFULL || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP5_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP4_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP3_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP2_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP1_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP0_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_AXI_RESP_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_AXI_IDBUFFER_FULL) {  // error handle
+		
+		uint32_t soft_rstn;
+		u32 count = 0;
+		// do software reset
+		soft_rstn = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR) & (~VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, soft_rstn);
+		do {
+			volatile uint32_t cycle = 0;
+			while (cycle++ < 100);
+			soft_rstn = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR);
+			udelay(1);
+		} while ((soft_rstn & VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK) != VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK && count++ < 100);
+
+    	// release reset
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, soft_rstn | VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+	}
+
+	if (vi_dvp2axi_int0 & VI_DVP2AXI_DVP0_ID0_FRAME_DONE_MASK) {  // done handle
+		active_buf = stream->curr_buf;
+		stream->frame_phase = DVP2AXI_CSI_FRAME0_READY;
+	}
+	if (vi_dvp2axi_int0 & VI_DVP2AXI_DVP0_ID1_FRAME_DONE_MASK) {
+		active_buf = stream->next_buf;
+		stream->frame_phase = DVP2AXI_CSI_FRAME1_READY;
+	} 
+	if (vi_dvp2axi_int0 & VI_DVP2AXI_DVP0_ID2_FRAME_DONE_MASK) {
+		//TODO not use now
+		active_buf = stream->last_buf;
+		stream->frame_phase = DVP2AXI_CSI_FRAME2_READY;
+	}
+
+
+
+	if(stream->is_first_flush) {
+		stream->is_first_flush = false;
+	} 
+	es_dvp2axi_addr_state = ES_DVP2AXI_YUV_ADDR_STATE_UPDATE;
+	//TODO need to check whether the frame is done but not get flush signal
+	// if ((vi_dvp2axi_int0 & VI_DVP2AXI_DVP0_ID0_FRAME_FLUSH_MASK) && (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY)) {  // done handle
+	// 	ret = es_dvp2axi_assign_new_buffer_oneframe(stream, es_dvp2axi_addr_state);
+	// }
+	// if((vi_dvp2axi_int0 & VI_DVP2AXI_DVP0_ID1_FRAME_FLUSH_MASK) && (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY)) {
+	// 	ret = es_dvp2axi_assign_new_buffer_oneframe(stream, es_dvp2axi_addr_state);
+	// }
+	// if((vi_dvp2axi_int0 & VI_DVP2AXI_DVP0_ID2_FRAME_FLUSH_MASK) && (stream->frame_phase == DVP2AXI_CSI_FRAME2_READY)) {
+	// 		//TODO not use now
+	// 	ret = es_dvp2axi_assign_new_buffer_oneframe(stream, es_dvp2axi_addr_state);
+	// }
+
+	if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY) {  // done handle
+		ret = es_dvp2axi_assign_new_buffer_oneframe(stream, es_dvp2axi_addr_state);
+	}
+	if(stream->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+		ret = es_dvp2axi_assign_new_buffer_oneframe(stream, es_dvp2axi_addr_state);
+	}
+	if(stream->frame_phase == DVP2AXI_CSI_FRAME2_READY) {
+		ret = es_dvp2axi_assign_new_buffer_oneframe(stream, es_dvp2axi_addr_state);
+	}
+
+	if (active_buf && (!ret)) {
+		active_buf->vb.sequence = stream->frame_idx - 1;
+		es_dvp2axi_vb_done_tasklet(stream, active_buf);
+		dvp2axi_dev->irq_stats.frm_end_cnt[stream->id]++;
+	}
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR, vi_dvp2axi_int0);
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, vi_dvp2axi_int1);
+	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT2_CSR, vi_dvp2axi_int_err);
+	stream->frame_phase = DVP2AXI_CSI_FRAME_UNREADY;
+}
+
+void es_irq_oneframe_new(struct device *dev, struct es_dvp2axi_device *dvp2axi_dev)
+{
+	u32 vi_dvp2axi_int0, vi_dvp2axi_int1;
+	u32 vi_dvp2axi_int_err;
+	int ret = 0;
+	struct es_dvp2axi_hw	*dvp2axi_hw =  dev_get_drvdata(dev);
+	// struct es_dvp2axi_stream *stream = &dvp2axi_dev->stream[ES_DVP2AXI_STREAM_DVP2AXI];
+	struct es_dvp2axi_stream *stream;
+	struct es_dvp2axi_buffer *active_buf[ESDVP2AXI_MAX_STREAM_MIPI][3] = {NULL};
+	int dvp_chn;
+	int esdvp2axi_addr_state;
+	dvp2axi_irq_status multi_dvp_irq_status[ESDVP2AXI_MAX_STREAM_MIPI];
+
+	memset(multi_dvp_irq_status, 0, sizeof(dvp2axi_irq_status)* ESDVP2AXI_MAX_STREAM_MIPI);
+	vi_dvp2axi_int0 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR);
+	vi_dvp2axi_int1 = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR);
+	vi_dvp2axi_int_err = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_INT2_CSR);
+
+	pr_info("%s: vi_dvp2axi_int0 0x%x, vi_dvp2axi_int1 0x%x, vi_dvp2axi_int_err 0x%x\n", __func__, vi_dvp2axi_int0, vi_dvp2axi_int1, vi_dvp2axi_int_err);
+	//TODO add error process into err tasklet
+	if (vi_dvp2axi_int_err & VI_DVP2AXI_INT2_AXI_IDBUFFER_AFULL || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP5_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP4_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP3_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP2_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP1_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_DVP0_FRAME_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_AXI_RESP_ERROR || \
+		vi_dvp2axi_int_err & VI_DVP2AXI_INT2_AXI_IDBUFFER_FULL) {  // error handle
+
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR, vi_dvp2axi_int0);
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, vi_dvp2axi_int1);
+
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT2_CSR, vi_dvp2axi_int_err);
+		
+		uint32_t soft_rstn;
+
+		// do software reset
+		soft_rstn = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR) & (~VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+		DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, soft_rstn);
+		do {
+			volatile uint32_t cycle = 0;
+			while (cycle++ < 100);
+			soft_rstn = DVP2AXI_HalReadReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR);
+    	} while ((soft_rstn & VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK) != VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK);
+    	// release reset
+    	DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_CTRL0_CSR, soft_rstn | VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK);
+
+		//return ;
+	}
+
+	if(vi_dvp2axi_int0 & VI_DVP2AXI_INT_FRAMEDONE_DVP0) {
+		dvp_chn = 0;
+		multi_dvp_irq_status[0].dvp_frame_done_irq = 1;
+		multi_dvp_irq_status[0].chn_frame_done[0] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP0_ID0_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP0_ID0_FRAME_DONE:0;
+		multi_dvp_irq_status[0].chn_frame_done[1] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP0_ID1_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP0_ID1_FRAME_DONE:0;
+		multi_dvp_irq_status[0].chn_frame_done[2] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP0_ID2_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP0_ID2_FRAME_DONE:0;
+	} 
+	if(vi_dvp2axi_int0 & VI_DVP2AXI_INT_FRAMEDONE_DVP1) {
+		dvp_chn = 1;
+		multi_dvp_irq_status[1].dvp_frame_done_irq = 1;
+		multi_dvp_irq_status[1].chn_frame_done[0] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP1_ID0_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP1_ID0_FRAME_DONE: 0 ;
+		multi_dvp_irq_status[1].chn_frame_done[1] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP1_ID1_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP1_ID1_FRAME_DONE: 0;
+		multi_dvp_irq_status[1].chn_frame_done[2] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP1_ID2_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP1_ID2_FRAME_DONE: 0;
+	} 
+	
+	if(vi_dvp2axi_int0 & VI_DVP2AXI_INT_FRAMEDONE_DVP2) {
+		dvp_chn = 2;
+		multi_dvp_irq_status[2].dvp_frame_done_irq = 1;
+		multi_dvp_irq_status[2].chn_frame_done[0] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP2_ID0_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP2_ID0_FRAME_DONE: 0;
+		multi_dvp_irq_status[2].chn_frame_done[1] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP2_ID1_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP2_ID1_FRAME_DONE: 0;
+		multi_dvp_irq_status[2].chn_frame_done[2] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP2_ID2_FRAME_DONE) ? VI_DVP2AXI_INT0_DVP2_ID2_FRAME_DONE: 0;
+	}
+
+	if(vi_dvp2axi_int1 & VI_DVP2AXI_INT_FRAMEDONE_DVP3) {
+		dvp_chn = 3;
+		multi_dvp_irq_status[3].dvp_frame_done_irq = 1;
+		multi_dvp_irq_status[3].chn_frame_done[0] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID0_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP3_ID0_FRAME_DONE:0;
+		multi_dvp_irq_status[3].chn_frame_done[1] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID1_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP3_ID1_FRAME_DONE:0;
+		multi_dvp_irq_status[3].chn_frame_done[2] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID2_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP3_ID2_FRAME_DONE:0;
+	} 
+	if(vi_dvp2axi_int1 & VI_DVP2AXI_INT_FRAMEDONE_DVP4) {
+		dvp_chn = 4;
+		multi_dvp_irq_status[4].dvp_frame_done_irq = 1;
+		multi_dvp_irq_status[4].chn_frame_done[0] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP4_ID0_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP4_ID0_FRAME_DONE:0;
+		multi_dvp_irq_status[4].chn_frame_done[1] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP4_ID1_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP4_ID1_FRAME_DONE:0;
+		multi_dvp_irq_status[4].chn_frame_done[2] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP4_ID2_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP4_ID2_FRAME_DONE:0;
+	} 
+	if(vi_dvp2axi_int1 & VI_DVP2AXI_INT_FRAMEDONE_DVP5) {
+		dvp_chn = 5;
+		multi_dvp_irq_status[5].dvp_frame_done_irq = 1;
+		multi_dvp_irq_status[5].chn_frame_done[0] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP5_ID0_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP5_ID0_FRAME_DONE:0;
+		multi_dvp_irq_status[5].chn_frame_done[1] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP5_ID1_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP5_ID1_FRAME_DONE:0;
+		multi_dvp_irq_status[5].chn_frame_done[2] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP5_ID2_FRAME_DONE) ? VI_DVP2AXI_INT1_DVP5_ID2_FRAME_DONE:0;
+	}
+
+	// flush 
+
+	if(vi_dvp2axi_int0 & VI_DVP2AXI_INT_FRAMEFLUSH_DVP0) {
+		dvp_chn = 0;
+		multi_dvp_irq_status[0].dvp_frame_flush_irq = 1;
+		multi_dvp_irq_status[0].chn_frame_flush[0] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP0_ID0_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP0_ID0_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[0].chn_frame_flush[1] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP0_ID1_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP0_ID1_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[0].chn_frame_flush[2] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP0_ID2_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP0_ID2_FRAME_FLUSH: 0;
+	} 
+	if(vi_dvp2axi_int0 & VI_DVP2AXI_INT_FRAMEFLUSH_DVP1) {
+		dvp_chn = 1;
+		multi_dvp_irq_status[1].dvp_frame_flush_irq = 1;
+		multi_dvp_irq_status[1].chn_frame_flush[0] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP1_ID0_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP1_ID0_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[1].chn_frame_flush[1] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP1_ID1_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP1_ID1_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[1].chn_frame_flush[2] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP1_ID2_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP1_ID2_FRAME_FLUSH: 0;
+	} 
+	
+	if(vi_dvp2axi_int0 & VI_DVP2AXI_INT_FRAMEFLUSH_DVP2) {
+		dvp_chn = 2;
+		multi_dvp_irq_status[2].dvp_frame_flush_irq = 1;
+		multi_dvp_irq_status[2].chn_frame_flush[0] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP2_ID0_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP2_ID0_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[2].chn_frame_flush[1] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP2_ID1_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP2_ID1_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[2].chn_frame_flush[2] = (vi_dvp2axi_int0 & VI_DVP2AXI_INT0_DVP2_ID2_FRAME_FLUSH) ? VI_DVP2AXI_INT0_DVP2_ID2_FRAME_FLUSH: 0;
+	}
+
+	if(vi_dvp2axi_int1 & VI_DVP2AXI_INT_FRAMEFLUSH_DVP3) {
+		dvp_chn = 3;
+		multi_dvp_irq_status[3].dvp_frame_flush_irq = 1;
+		multi_dvp_irq_status[3].chn_frame_flush[0] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[3].chn_frame_flush[1] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[3].chn_frame_flush[2] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH: 0;
+	} 
+	if(vi_dvp2axi_int1 & VI_DVP2AXI_INT_FRAMEFLUSH_DVP4) {
+		dvp_chn = 4;
+		multi_dvp_irq_status[4].dvp_frame_flush_irq = 1;
+		multi_dvp_irq_status[4].chn_frame_flush[0] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[4].chn_frame_flush[1] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[4].chn_frame_flush[2] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH: 0;
+	} 
+	if(vi_dvp2axi_int1 & VI_DVP2AXI_INT_FRAMEFLUSH_DVP5) {
+		dvp_chn = 5;
+		multi_dvp_irq_status[5].dvp_frame_flush_irq = 1;
+		multi_dvp_irq_status[5].chn_frame_flush[0] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[5].chn_frame_flush[1] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH: 0;
+		multi_dvp_irq_status[5].chn_frame_flush[2] = (vi_dvp2axi_int1 & VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH) ? VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH: 0;
+	}
+
+	
+	for(int i = 0; i < ESDVP2AXI_MAX_STREAM_MIPI; i++) {
+		stream = &dvp2axi_dev->stream[i];
+		
+		if(multi_dvp_irq_status[i].dvp_frame_done_irq) {
+			for(int j=0; j<3; j++) {
+				if(multi_dvp_irq_status[i].chn_frame_done[j] != 0) {
+					active_buf[i][j] = (j == 0)? stream->curr_buf: (j == 1)? stream->next_buf: (j == 2)? stream->last_buf: NULL;
+					stream->frame_phase = j;
+					if(i <3) {
+						pr_info("%s:%d frame done %d \n", __func__, __LINE__, multi_dvp_irq_status[i].chn_frame_done[j]);
+						DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR, multi_dvp_irq_status[i].chn_frame_done[j]);
+					}
+					else {
+						DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, multi_dvp_irq_status[i].chn_frame_done[j]);
+					}
+				}
+			}
+		}
+		if(multi_dvp_irq_status[i].dvp_frame_flush_irq) {
+			if(stream->is_first_flush) {
+				stream->is_first_flush = false;
+				if(i <3)
+					DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR, multi_dvp_irq_status[i].chn_frame_flush[0] | multi_dvp_irq_status[i].chn_frame_flush[1] | multi_dvp_irq_status[i].chn_frame_flush[2]);
+				else {
+					DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, multi_dvp_irq_status[i].chn_frame_flush[0] | multi_dvp_irq_status[i].chn_frame_flush[1] | multi_dvp_irq_status[i].chn_frame_flush[2]);
+				}
+			} else {
+				for(int j=0; j<3; j++) {
+					if(multi_dvp_irq_status[i].chn_frame_flush[j] != 0) {
+						esdvp2axi_addr_state = ES_DVP2AXI_YUV_ADDR_STATE_UPDATE;
+						ret = es_dvp2axi_assign_new_buffer_oneframe(stream, esdvp2axi_addr_state);
+						if( i < 3) {
+							DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT0_CSR,multi_dvp_irq_status[i].chn_frame_flush[j]);
+						} else {
+							DVP2AXI_HalWriteReg(dvp2axi_hw, VI_DVP2AXI_INT1_CSR, multi_dvp_irq_status[i].chn_frame_flush[j]);
+						}
+					}
+				}
+			}
+		}
+
+		
+
+		for(int j=0; j<3; j++) {
+			if (active_buf[i][j] && (!ret)) {
+				active_buf[i][j]->vb.sequence = stream->frame_idx - 1;
+				es_dvp2axi_vb_done_tasklet(stream, active_buf[i][j]);
+				dvp2axi_dev->irq_stats.frm_end_cnt[stream->id]++;
+			}
+		}
+	}
+	
+
+}
+
+
+
+void es_dvp2axi_irq_oneframe(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	/* TODO: xuhf-debug: add stream type */
+	struct es_dvp2axi_stream *stream;
+	u32 lastline, lastpix, ctl, dvp2axi_frmst, intstat, frmid;
+	int ret = 0;
+	
+
+
+
+	intstat = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT);
+	dvp2axi_frmst = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_FRAME_STATUS);
+	lastline = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_LAST_LINE);
+	lastpix = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_LAST_PIX);
+	ctl = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL);
+	frmid = DVP2AXI_GET_FRAME_ID(dvp2axi_frmst);
+
+	/* There are two irqs enabled:
+	 *  - PST_INF_FRAME_END: dvp2axi FIFO is ready, this is prior to FRAME_END
+	 *  -         FRAME_END: dvp2axi has saved frame to memory, a frame ready
+	 */
+	stream = &dvp2axi_dev->stream[ES_DVP2AXI_STREAM_DVP2AXI];
+
+	if ((intstat & PST_INF_FRAME_END)) {
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT,
+				     PST_INF_FRAME_END_CLR);
+
+		if (stream->stopping)
+			/* To stop DVP2AXI ASAP, before FRAME_END irq */
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL,
+					     ctl & (~ENABLE_CAPTURE));
+	}
+
+	if ((intstat & FRAME_END)) {
+		struct es_dvp2axi_buffer *active_buf = NULL;
+
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT,
+				     FRAME_END_CLR);
+
+		if (stream->stopping) {
+			es_dvp2axi_stream_stop(stream);
+			stream->stopping = false;
+			wake_up(&stream->wq_stopped);
+			return;
+		}
+
+		if (lastline != stream->pixm.height ||
+		    !(dvp2axi_frmst & DVP2AXI_F0_READY)) {
+			/* Clearing status must be complete before fe packet
+			 * arrives while dvp2axi is connected with mipi,
+			 * so it should be placed before printing log here,
+			 * otherwise it would be delayed.
+			 * At the same time, don't clear the frame id
+			 * for switching address.
+			 */
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_FRAME_STATUS,
+					     FRM0_STAT_CLS);
+			v4l2_err(&dvp2axi_dev->v4l2_dev,
+				 "Bad frame, irq:0x%x frmst:0x%x size:%dx%d\n",
+				 intstat, dvp2axi_frmst, lastline, lastpix);
+
+			return;
+		}
+
+		if (frmid % 2 != 0) {
+			stream->frame_phase = DVP2AXI_CSI_FRAME0_READY;
+			if (stream->curr_buf)
+				active_buf = stream->curr_buf;
+		} else {
+			stream->frame_phase = DVP2AXI_CSI_FRAME1_READY;
+			if (stream->next_buf)
+				active_buf = stream->next_buf;
+		}
+
+		/* In one-frame mode:
+		 * 1,must clear status manually by writing 0 to enable
+		 * the next frame end irq;
+		 * 2,do not clear the frame id for switching address.
+		 */
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_FRAME_STATUS,
+				     dvp2axi_frmst & FRM0_STAT_CLS);
+		ret = es_dvp2axi_assign_new_buffer_oneframe(
+			stream, ES_DVP2AXI_YUV_ADDR_STATE_UPDATE);
+
+		if (active_buf && (!ret)) {
+			active_buf->vb.sequence = stream->frame_idx - 1;
+			es_dvp2axi_vb_done_tasklet(stream, active_buf);
+		}
+
+		dvp2axi_dev->irq_stats.frm_end_cnt[stream->id]++;
+	}
+}
+
+static int es_dvp2axi_csi_g_mipi_id(struct v4l2_device *v4l2_dev,
+			       unsigned int intstat)
+{
+	if (intstat & CSI_FRAME_END_ID0)
+		return ES_DVP2AXI_STREAM_MIPI_ID0;
+
+	if (intstat & CSI_FRAME_END_ID1)
+		return ES_DVP2AXI_STREAM_MIPI_ID1;
+
+	if (intstat & CSI_FRAME_END_ID2)
+		return ES_DVP2AXI_STREAM_MIPI_ID2;
+	if (intstat & CSI_FRAME_END_ID3)
+		return ES_DVP2AXI_STREAM_MIPI_ID3;
+
+	return -EINVAL;
+}
+
+static int es_dvp2axi_dvp_g_ch_id(struct v4l2_device *v4l2_dev, u32 *intstat,
+			     u32 frm_stat)
+{
+	if (*intstat & DVP_FRAME_END_ID0) {
+		*intstat &= ~DVP_FRAME_END_ID0;
+		return ES_DVP2AXI_STREAM_MIPI_ID0;
+	}
+
+	if (*intstat & DVP_FRAME_END_ID1) {
+		*intstat &= ~DVP_FRAME_END_ID1;
+		return ES_DVP2AXI_STREAM_MIPI_ID1;
+	}
+
+	if (*intstat & DVP_FRAME_END_ID2) {
+		*intstat &= ~DVP_FRAME_END_ID2;
+		return ES_DVP2AXI_STREAM_MIPI_ID2;
+	}
+
+	if (*intstat & DVP_FRAME_END_ID3) {
+		*intstat &= ~DVP_FRAME_END_ID3;
+		return ES_DVP2AXI_STREAM_MIPI_ID3;
+	}
+
+	return -EINVAL;
+}
+
+static int es_dvp2axi_dvp_g_ch_id_by_fe(struct v4l2_device *v4l2_dev, u32 intstat)
+{
+	if (intstat & DVP_ALL_END_ID0)
+		return ES_DVP2AXI_STREAM_MIPI_ID0;
+
+	if (intstat & DVP_ALL_END_ID1)
+		return ES_DVP2AXI_STREAM_MIPI_ID1;
+
+	if (intstat & DVP_ALL_END_ID2)
+		return ES_DVP2AXI_STREAM_MIPI_ID2;
+
+	if (intstat & DVP_ALL_END_ID3)
+		return ES_DVP2AXI_STREAM_MIPI_ID3;
+
+	return -EINVAL;
+}
+
+static bool es_dvp2axi_is_csi2_err_trigger_reset(struct es_dvp2axi_timer *timer)
+{
+	struct es_dvp2axi_device *dev =
+		container_of(timer, struct es_dvp2axi_device, reset_watchdog_timer);
+	struct es_dvp2axi_stream *stream = &dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0];
+	bool is_triggered = false, is_assign_triggered = false,
+	     is_first_err = false;
+	unsigned long flags;
+	u64 cur_time, diff_time;
+
+	spin_lock_irqsave(&timer->csi2_err_lock, flags);
+
+	if (timer->csi2_err_cnt_even != 0 && timer->csi2_err_cnt_odd != 0) {
+		timer->csi2_err_cnt_odd = 0;
+		timer->csi2_err_cnt_even = 0;
+		timer->reset_src = ES_RESET_SRC_ERR_CSI2;
+		timer->csi2_err_triggered_cnt++;
+		if (timer->csi2_err_triggered_cnt == 1) {
+			is_first_err = true;
+			timer->csi2_first_err_timestamp =
+				es_dvp2axi_time_get_ns(dev);
+		}
+
+		is_assign_triggered = true;
+
+		v4l2_info(&dev->v4l2_dev, "find csi2 err cnt is:%d\n",
+			  timer->csi2_err_triggered_cnt);
+	}
+
+	if (!is_first_err) {
+		if (timer->csi2_err_triggered_cnt >= 1) {
+			cur_time = es_dvp2axi_time_get_ns(dev);
+			diff_time = cur_time - timer->csi2_first_err_timestamp;
+			diff_time = div_u64(diff_time, 1000000);
+			if (diff_time >= timer->err_time_interval) {
+				is_triggered = true;
+				v4l2_info(
+					&dev->v4l2_dev,
+					"trigger reset for time out of csi err\n");
+				goto end_judge;
+			}
+
+			if (!is_assign_triggered &&
+			    (timer->csi2_err_cnt_odd == 0 ||
+			     timer->csi2_err_cnt_even == 0)) {
+				is_triggered = true;
+				v4l2_info(&dev->v4l2_dev,
+					  "trigger reset for csi err\n");
+				goto end_judge;
+			}
+		}
+	}
+
+	/*
+	 * when fs cnt is beyond 2, it indicates that frame end is not coming,
+	 * or fs and fe had been not paired.
+	 */
+	if (stream->is_fs_fe_not_paired ||
+	    stream->fs_cnt_in_single_frame > ES_DVP2AXI_FS_DETECTED_NUM) {
+		is_triggered = true;
+		v4l2_info(&dev->v4l2_dev, "reset for fs & fe not paired\n");
+	}
+end_judge:
+	spin_unlock_irqrestore(&timer->csi2_err_lock, flags);
+
+	return is_triggered;
+}
+
+static bool es_dvp2axi_is_triggered_monitoring(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_timer *timer = &dev->reset_watchdog_timer;
+	struct es_dvp2axi_stream *stream = &dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0];
+	bool ret = false;
+
+	if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_IDLE)
+		ret = false;
+
+	if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_CONTINUE ||
+	    timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_HOTPLUG) {
+		if (stream->frame_idx >= timer->triggered_frame_num)
+			ret = true;
+	}
+
+	if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_TRIGGER) {
+		timer->is_csi2_err_occurred =
+			es_dvp2axi_is_csi2_err_trigger_reset(timer);
+		ret = timer->is_csi2_err_occurred;
+	}
+
+	return ret;
+}
+
+s32 es_dvp2axi_get_sensor_vblank(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dev->terminal_sensor;
+	struct v4l2_subdev *sd = terminal_sensor->sd;
+	struct v4l2_ctrl_handler *hdl = sd->ctrl_handler;
+	struct v4l2_ctrl *ctrl = NULL;
+
+	if (!list_empty(&hdl->ctrls)) {
+		list_for_each_entry(ctrl, &hdl->ctrls, node) {
+			if (ctrl->id == V4L2_CID_VBLANK)
+				return ctrl->val;
+		}
+	}
+
+	return 0;
+}
+
+s32 es_dvp2axi_get_sensor_vblank_def(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dev->terminal_sensor;
+	struct v4l2_subdev *sd = terminal_sensor->sd;
+	struct v4l2_ctrl_handler *hdl = sd->ctrl_handler;
+	struct v4l2_ctrl *ctrl = NULL;
+
+	if (!list_empty(&hdl->ctrls)) {
+		list_for_each_entry(ctrl, &hdl->ctrls, node) {
+			if (ctrl->id == V4L2_CID_VBLANK)
+				return ctrl->default_value;
+		}
+	}
+
+	return 0;
+}
+
+static void es_dvp2axi_cal_csi_crop_width_vwidth(struct es_dvp2axi_stream *stream,
+					    u32 raw_width, u32 *crop_width,
+					    u32 *crop_vwidth)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct csi_channel_info *channel = &dev->channels[stream->id];
+	const struct dvp2axi_output_fmt *fmt;
+	u32 fourcc;
+
+	fmt = es_dvp2axi_find_output_fmt(stream, stream->pixm.pixelformat);
+	if (!fmt) {
+		v4l2_err(&dev->v4l2_dev, "can not find output format: 0x%x",
+			 stream->pixm.pixelformat);
+		return;
+	}
+
+	*crop_width = raw_width;
+
+	/*
+	 * for mipi or lvds, when enable compact, the virtual width of raw10/raw12
+	 * needs aligned with :ALIGN(bits_per_pixel * width / 8, 8), if enable 16bit mode
+	 * needs aligned with :ALIGN(bits_per_pixel * width * 2, 8), to optimize reading and
+	 * writing of ddr, aliged with 256
+	 */
+	if (fmt->fmt_type == DVP2AXI_FMT_TYPE_RAW && stream->is_compact &&
+	    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB888 &&
+	    fmt->csi_fmt_val != CSI_WRDDR_TYPE_RGB565) {
+		*crop_vwidth = ALIGN(raw_width * fmt->raw_bpp / 8, 256);
+	} else {
+		*crop_vwidth = ALIGN(raw_width * fmt->bpp[0] / 8, 8);
+	}
+
+	if (channel->fmt_val == CSI_WRDDR_TYPE_RGB888 ||
+	    channel->fmt_val == CSI_WRDDR_TYPE_RGB565)
+		*crop_width = raw_width * fmt->bpp[0] / 8;
+	/*
+	 * es dvp2axi don't support output yuyv fmt data
+	 * if user request yuyv fmt, the input mode must be RAW8
+	 * and the width is double Because the real input fmt is
+	 * yuyv
+	 */
+	fourcc = stream->dvp2axi_fmt_out->fourcc;
+	if (fourcc == V4L2_PIX_FMT_YUYV || fourcc == V4L2_PIX_FMT_YVYU ||
+	    fourcc == V4L2_PIX_FMT_UYVY || fourcc == V4L2_PIX_FMT_VYUY) {
+		*crop_width = 2 * raw_width;
+		*crop_vwidth *= 2;
+	}
+}
+
+static void es_dvp2axi_dynamic_crop(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus;
+	const struct dvp2axi_output_fmt *fmt;
+	u32 raw_width, crop_width = 64, crop_vwidth = 64, crop_height = 64,
+		       crop_x = 0, crop_y = 0;
+
+	if (!dvp2axi_dev->active_sensor)
+		return;
+
+	mbus = &dvp2axi_dev->active_sensor->mbus;
+	if (mbus->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus->type == V4L2_MBUS_CSI2_CPHY || mbus->type == V4L2_MBUS_CCP2) {
+		struct csi_channel_info *channel =
+			&dvp2axi_dev->channels[stream->id];
+
+		if (channel->fmt_val == CSI_WRDDR_TYPE_RGB888)
+			crop_x = 3 * stream->crop[CROP_SRC_ACT].left;
+		else if (channel->fmt_val == CSI_WRDDR_TYPE_RGB565)
+			crop_x = 2 * stream->crop[CROP_SRC_ACT].left;
+		else
+			crop_x = stream->crop[CROP_SRC_ACT].left;
+
+		crop_y = stream->crop[CROP_SRC_ACT].top;
+		raw_width = stream->crop[CROP_SRC_ACT].width;
+		crop_height = stream->crop[CROP_SRC_ACT].height;
+
+		es_dvp2axi_cal_csi_crop_width_vwidth(stream, raw_width, &crop_width,
+						&crop_vwidth);
+		es_dvp2axi_write_register(
+			dvp2axi_dev, get_reg_index_of_id_crop_start(channel->id),
+			crop_y << 16 | crop_x);
+		es_dvp2axi_write_register(dvp2axi_dev,
+				     get_reg_index_of_id_ctrl1(channel->id),
+				     crop_height << 16 | crop_width);
+
+		es_dvp2axi_write_register(dvp2axi_dev,
+				     get_reg_index_of_frm0_y_vlw(channel->id),
+				     crop_vwidth);
+		es_dvp2axi_write_register(dvp2axi_dev,
+				     get_reg_index_of_frm1_y_vlw(channel->id),
+				     crop_vwidth);
+		es_dvp2axi_write_register(dvp2axi_dev,
+				     get_reg_index_of_frm0_uv_vlw(channel->id),
+				     crop_vwidth);
+		es_dvp2axi_write_register(dvp2axi_dev,
+				     get_reg_index_of_frm1_uv_vlw(channel->id),
+				     crop_vwidth);
+	} else {
+		raw_width = stream->crop[CROP_SRC_ACT].width;
+		crop_width = raw_width;
+		crop_vwidth = raw_width;
+		crop_height = stream->crop[CROP_SRC_ACT].height;
+		crop_x = stream->crop[CROP_SRC_ACT].left;
+		crop_y = stream->crop[CROP_SRC_ACT].top;
+
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_CROP,
+				     crop_y << DVP2AXI_CROP_Y_SHIFT | crop_x);
+
+		if (stream->dvp2axi_fmt_in->fmt_type == DVP2AXI_FMT_TYPE_RAW) {
+			fmt = es_dvp2axi_find_output_fmt(stream,
+						    stream->pixm.pixelformat);
+			crop_vwidth = raw_width *
+				      es_dvp2axi_cal_raw_vir_line_ratio(stream, fmt);
+		}
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_VIR_LINE_WIDTH,
+				     crop_vwidth);
+
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_SET_SIZE,
+				     crop_height << 16 | crop_width);
+	}
+
+	stream->crop_dyn_en = false;
+}
+
+static void es_dvp2axi_monitor_reset_event(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_stream *stream = NULL;
+	struct es_dvp2axi_timer *timer = &dev->reset_watchdog_timer;
+	unsigned int cycle = 0;
+	u64 fps, timestamp0, timestamp1;
+	unsigned long flags, fps_flags;
+	int i = 0;
+
+	if (timer->is_running)
+		return;
+
+	if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_IDLE)
+		return;
+
+	for (i = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+		stream = &dev->stream[i];
+		if (stream->state == ES_DVP2AXI_STATE_STREAMING)
+			break;
+	}
+
+	if (i >= ES_DVP2AXI_MAX_STREAM_MIPI)
+		return;
+
+	timer->is_triggered = es_dvp2axi_is_triggered_monitoring(dev);
+
+	if (timer->is_triggered) {
+		struct v4l2_rect *raw_rect = &dev->terminal_sensor.raw_rect;
+		enum es_dvp2axi_monitor_mode mode;
+		s32 vblank = 0;
+		u32 vts = 0;
+		u64 numerator = 0;
+		u64 denominator = 0;
+
+		if (stream->frame_idx > 2) {
+			spin_lock_irqsave(&stream->fps_lock, fps_flags);
+			timestamp0 = stream->fps_stats.frm0_timestamp;
+			timestamp1 = stream->fps_stats.frm1_timestamp;
+			spin_unlock_irqrestore(&stream->fps_lock, fps_flags);
+
+			fps = timestamp0 > timestamp1 ?
+				      timestamp0 - timestamp1 :
+				      timestamp1 - timestamp0;
+			fps = div_u64(fps, 1000);
+		} else {
+			numerator = dev->terminal_sensor.fi.interval.numerator;
+			denominator =
+				dev->terminal_sensor.fi.interval.denominator;
+			fps = div_u64(1000000 * numerator, denominator);
+		}
+		spin_lock_irqsave(&timer->timer_lock, flags);
+
+		timer->frame_end_cycle_us = fps;
+
+		vblank = es_dvp2axi_get_sensor_vblank(dev);
+		timer->raw_height = raw_rect->height;
+		vts = timer->raw_height + vblank;
+		timer->vts = vts;
+
+		timer->line_end_cycle =
+			div_u64(timer->frame_end_cycle_us, timer->vts);
+		fps = div_u64(timer->frame_end_cycle_us, 1000);
+		cycle = fps * timer->frm_num_of_monitor_cycle;
+		timer->cycle = msecs_to_jiffies(cycle);
+
+		timer->run_cnt = 0;
+		timer->is_running = true;
+		timer->is_buf_stop_update = false;
+		for (i = 0; i < dev->num_channels; i++) {
+			stream = &dev->stream[i];
+			if (stream->state == ES_DVP2AXI_STATE_STREAMING)
+				timer->last_buf_wakeup_cnt[i] =
+					stream->buf_wake_up_cnt;
+		}
+		/* in trigger mode, monitoring count is fps */
+		mode = timer->monitor_mode;
+		if (mode == ES_DVP2AXI_MONITOR_MODE_CONTINUE ||
+		    mode == ES_DVP2AXI_MONITOR_MODE_HOTPLUG)
+			timer->max_run_cnt = 0xffffffff - DVP2AXI_TIMEOUT_FRAME_NUM;
+		else
+			timer->max_run_cnt = div_u64(1000, fps) * 1;
+
+		timer->timer.expires = jiffies + timer->cycle;
+		mod_timer(&timer->timer, timer->timer.expires);
+
+		spin_unlock_irqrestore(&timer->timer_lock, flags);
+
+		v4l2_dbg(
+			3, es_dvp2axi_debug, &dev->v4l2_dev,
+			"%s:mode:%d, raw height:%d,vblank:%d, cycle:%ld, fps:%llu\n",
+			__func__, timer->monitor_mode, raw_rect->height, vblank,
+			timer->cycle, div_u64(1000, fps));
+	}
+}
+
+static void es_dvp2axi_rdbk_frame_end(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor = &stream->dvp2axidev->terminal_sensor;
+	u32 denominator, numerator;
+	u64 l_ts, m_ts, s_ts, time = 30000000LL;
+	int ret, fps = -1;
+	int i = 0;
+	unsigned long flags;
+
+	if (dev->hdr.hdr_mode == HDR_X2) {
+		if (stream->id != ES_DVP2AXI_STREAM_MIPI_ID1 ||
+		    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0].state !=
+			    ES_DVP2AXI_STATE_STREAMING ||
+		    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1].state !=
+			    ES_DVP2AXI_STATE_STREAMING)
+			return;
+	} else if (dev->hdr.hdr_mode == HDR_X3) {
+		if (stream->id != ES_DVP2AXI_STREAM_MIPI_ID2 ||
+		    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0].state !=
+			    ES_DVP2AXI_STATE_STREAMING ||
+		    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1].state !=
+			    ES_DVP2AXI_STATE_STREAMING ||
+		    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID2].state !=
+			    ES_DVP2AXI_STATE_STREAMING)
+			return;
+	}
+
+	numerator = sensor->fi.interval.numerator;
+	denominator = sensor->fi.interval.denominator;
+	if (denominator && numerator)
+		time = numerator * 1000 / denominator * 1000 * 1000;
+
+	if (dev->hdr.hdr_mode == HDR_X3) {
+		if (dev->rdbk_buf[RDBK_L] && dev->rdbk_buf[RDBK_M] &&
+		    dev->rdbk_buf[RDBK_S]) {
+			l_ts = dev->rdbk_buf[RDBK_L]->fe_timestamp;
+			m_ts = dev->rdbk_buf[RDBK_M]->fe_timestamp;
+			s_ts = dev->rdbk_buf[RDBK_S]->fe_timestamp;
+
+			if (m_ts < l_ts || s_ts < m_ts) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"s/m/l frame err, timestamp s:%lld m:%lld l:%lld\n",
+					s_ts, m_ts, l_ts);
+				return;
+			}
+
+			if ((m_ts - l_ts) > time || (s_ts - m_ts) > time) {
+				ret = v4l2_subdev_call(sensor->sd, video,
+						       g_frame_interval,
+						       &sensor->fi);
+				if (!ret) {
+					denominator =
+						sensor->fi.interval.denominator;
+					numerator =
+						sensor->fi.interval.numerator;
+					if (denominator && numerator) {
+						time = numerator * 1000 /
+						       denominator * 1000 *
+						       1000;
+						fps = denominator / numerator;
+					}
+				}
+
+				if ((m_ts - l_ts) > time ||
+				    (s_ts - m_ts) > time) {
+					v4l2_err(
+						&dev->v4l2_dev,
+						"timestamp no match, s:%lld m:%lld l:%lld, fps:%d\n",
+						s_ts, m_ts, l_ts, fps);
+					return;
+				}
+			}
+			dev->rdbk_buf[RDBK_M]->vb.sequence =
+				dev->rdbk_buf[RDBK_L]->vb.sequence;
+			dev->rdbk_buf[RDBK_S]->vb.sequence =
+				dev->rdbk_buf[RDBK_L]->vb.sequence;
+			if (dev->is_support_tools &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0]
+					    .tools_vdev->state ==
+				    ES_DVP2AXI_STATE_STREAMING &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1]
+					    .tools_vdev->state ==
+				    ES_DVP2AXI_STATE_STREAMING &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID2]
+					    .tools_vdev->state ==
+				    ES_DVP2AXI_STATE_STREAMING) {
+				for (i = 0; i < 3; i++) {
+					spin_lock_irqsave(
+						&dev->stream[i]
+							 .tools_vdev->vbq_lock,
+						flags);
+					list_add_tail(&dev->rdbk_buf[i]->queue,
+						      &dev->stream[i]
+							       .tools_vdev
+							       ->buf_done_head);
+					if (!work_busy(
+						    &dev->stream[i]
+							     .tools_vdev->work))
+						schedule_work(
+							&dev->stream[i]
+								 .tools_vdev
+								 ->work);
+					spin_unlock_irqrestore(
+						&dev->stream[i]
+							 .tools_vdev->vbq_lock,
+						flags);
+				}
+			} else {
+				es_dvp2axi_vb_done_tasklet(
+					&dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0],
+					dev->rdbk_buf[RDBK_L]);
+				es_dvp2axi_vb_done_tasklet(
+					&dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1],
+					dev->rdbk_buf[RDBK_M]);
+				es_dvp2axi_vb_done_tasklet(
+					&dev->stream[ES_DVP2AXI_STREAM_MIPI_ID2],
+					dev->rdbk_buf[RDBK_S]);
+			}
+		} else {
+			if (!dev->rdbk_buf[RDBK_L])
+				v4l2_err(&dev->v4l2_dev, "lost long frames\n");
+			if (!dev->rdbk_buf[RDBK_M])
+				v4l2_err(&dev->v4l2_dev,
+					 "lost medium frames\n");
+			if (!dev->rdbk_buf[RDBK_S])
+				v4l2_err(&dev->v4l2_dev, "lost short frames\n");
+			return;
+		}
+	} else if (dev->hdr.hdr_mode == HDR_X2) {
+		if (dev->rdbk_buf[RDBK_L] && dev->rdbk_buf[RDBK_M]) {
+			l_ts = dev->rdbk_buf[RDBK_L]->fe_timestamp;
+			s_ts = dev->rdbk_buf[RDBK_M]->fe_timestamp;
+
+			if (s_ts < l_ts) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"s/l frame err, timestamp s:%lld l:%lld\n",
+					s_ts, l_ts);
+				return;
+			}
+
+			if ((s_ts - l_ts) > time) {
+				ret = v4l2_subdev_call(sensor->sd, video,
+						       g_frame_interval,
+						       &sensor->fi);
+				if (!ret) {
+					denominator =
+						sensor->fi.interval.denominator;
+					numerator =
+						sensor->fi.interval.numerator;
+					if (denominator && numerator) {
+						time = numerator * 1000 /
+						       denominator * 1000 *
+						       1000;
+						fps = denominator / numerator;
+					}
+				}
+				if ((s_ts - l_ts) > time) {
+					v4l2_err(
+						&dev->v4l2_dev,
+						"timestamp no match, s:%lld l:%lld, fps:%d\n",
+						s_ts, l_ts, fps);
+					return;
+				}
+			}
+			dev->rdbk_buf[RDBK_M]->vb.sequence =
+				dev->rdbk_buf[RDBK_L]->vb.sequence;
+			if (dev->is_support_tools &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0]
+					    .tools_vdev->state ==
+				    ES_DVP2AXI_STATE_STREAMING &&
+			    dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1]
+					    .tools_vdev->state ==
+				    ES_DVP2AXI_STATE_STREAMING) {
+				for (i = 0; i < 2; i++) {
+					spin_lock_irqsave(
+						&dev->stream[i]
+							 .tools_vdev->vbq_lock,
+						flags);
+					list_add_tail(&dev->rdbk_buf[i]->queue,
+						      &dev->stream[i]
+							       .tools_vdev
+							       ->buf_done_head);
+					if (!work_busy(
+						    &dev->stream[i]
+							     .tools_vdev->work))
+						schedule_work(
+							&dev->stream[i]
+								 .tools_vdev
+								 ->work);
+					spin_unlock_irqrestore(
+						&dev->stream[i]
+							 .tools_vdev->vbq_lock,
+						flags);
+				}
+			} else {
+				es_dvp2axi_vb_done_tasklet(
+					&dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0],
+					dev->rdbk_buf[RDBK_L]);
+				es_dvp2axi_vb_done_tasklet(
+					&dev->stream[ES_DVP2AXI_STREAM_MIPI_ID1],
+					dev->rdbk_buf[RDBK_M]);
+			}
+		} else {
+			if (!dev->rdbk_buf[RDBK_L])
+				v4l2_err(&dev->v4l2_dev, "lost long frames\n");
+			if (!dev->rdbk_buf[RDBK_M])
+				v4l2_err(&dev->v4l2_dev, "lost short frames\n");
+			return;
+		}
+	}
+
+	dev->rdbk_buf[RDBK_L] = NULL;
+	dev->rdbk_buf[RDBK_M] = NULL;
+	dev->rdbk_buf[RDBK_S] = NULL;
+}
+
+static void es_dvp2axi_buf_done_with_tools(struct es_dvp2axi_stream *stream,
+				      struct es_dvp2axi_buffer *active_buf)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&stream->tools_vdev->vbq_lock, flags);
+	if (stream->tools_vdev->state == ES_DVP2AXI_STATE_STREAMING) {
+		list_add_tail(&active_buf->queue,
+			      &stream->tools_vdev->buf_done_head);
+		if (!work_busy(&stream->tools_vdev->work))
+			schedule_work(&stream->tools_vdev->work);
+	} else {
+		es_dvp2axi_vb_done_tasklet(stream, active_buf);
+	}
+	spin_unlock_irqrestore(&stream->tools_vdev->vbq_lock, flags);
+}
+
+static void es_dvp2axi_buf_done_prepare(struct es_dvp2axi_stream *stream,
+				   struct es_dvp2axi_buffer *active_buf, int mipi_id,
+				   u32 mode)
+{
+	unsigned long flags;
+	struct vb2_v4l2_buffer *vb_done = NULL;
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+
+	if (active_buf) {
+		vb_done = &active_buf->vb;
+		if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI &&
+		    dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_BT656)
+			vb_done->vb2_buf.timestamp =
+				stream->readout.fe_timestamp;
+		else
+			vb_done->vb2_buf.timestamp =
+				stream->readout.fs_timestamp;
+		vb_done->sequence = stream->frame_idx - 1;
+		active_buf->fe_timestamp = es_dvp2axi_time_get_ns(dvp2axi_dev);
+		if (stream->is_line_wake_up) {
+			spin_lock_irqsave(&stream->fps_lock, flags);
+			if (mode)
+				stream->fps_stats.frm0_timestamp =
+					vb_done->vb2_buf.timestamp;
+			else
+				stream->fps_stats.frm1_timestamp =
+					vb_done->vb2_buf.timestamp;
+			stream->readout.wk_timestamp =
+				vb_done->vb2_buf.timestamp;
+			spin_unlock_irqrestore(&stream->fps_lock, flags);
+		}
+		if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED)
+			vb_done->sequence /= 2;
+		if (stream->cur_skip_frame) {
+			es_dvp2axi_buf_queue(&active_buf->vb.vb2_buf);
+			return;
+		}
+	} else if (dvp2axi_dev->rdbk_buf[stream->id]) {
+		vb_done = &dvp2axi_dev->rdbk_buf[stream->id]->vb;
+		if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI &&
+		    dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_BT656)
+			vb_done->vb2_buf.timestamp =
+				stream->readout.fe_timestamp;
+		else
+			vb_done->vb2_buf.timestamp =
+				stream->readout.fs_timestamp;
+		vb_done->sequence = stream->frame_idx - 1;
+		dvp2axi_dev->rdbk_buf[stream->id]->fe_timestamp =
+			es_dvp2axi_time_get_ns(dvp2axi_dev);
+	}
+
+	if (dvp2axi_dev->hdr.hdr_mode == NO_HDR ||
+	    dvp2axi_dev->hdr.hdr_mode == HDR_COMPR) {
+		if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED) {
+			if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY &&
+			    active_buf) {
+				if (dvp2axi_dev->is_support_tools &&
+				    stream->tools_vdev)
+					es_dvp2axi_buf_done_with_tools(stream,
+								  active_buf);
+				else
+					es_dvp2axi_vb_done_tasklet(stream,
+							      active_buf);
+			}
+		} else {
+			if (active_buf) {
+				if (dvp2axi_dev->is_support_tools &&
+				    stream->tools_vdev)
+					es_dvp2axi_buf_done_with_tools(stream,
+								  active_buf);
+				else
+					es_dvp2axi_vb_done_tasklet(stream,
+							      active_buf);
+			}
+		}
+	} else {
+		if (dvp2axi_dev->is_start_hdr) {
+			spin_lock_irqsave(&dvp2axi_dev->hdr_lock, flags);
+			if (mipi_id == ES_DVP2AXI_STREAM_MIPI_ID0) {
+				if (dvp2axi_dev->rdbk_buf[RDBK_L] && active_buf) {
+					v4l2_err(
+						&dvp2axi_dev->v4l2_dev,
+						"multiple long data in %s frame,frm_idx:%d,state:0x%x\n",
+						dvp2axi_dev->hdr.hdr_mode ==
+								HDR_X2 ?
+							"hdr_x2" :
+							"hdr_x3",
+						stream->frame_idx,
+						dvp2axi_dev->rdbk_buf[RDBK_L]
+							->vb.vb2_buf.state);
+					dvp2axi_dev->rdbk_buf[RDBK_L]
+						->vb.vb2_buf.state =
+						VB2_BUF_STATE_ACTIVE;
+					es_dvp2axi_buf_queue(
+						&dvp2axi_dev->rdbk_buf[RDBK_L]
+							 ->vb.vb2_buf);
+					dvp2axi_dev->rdbk_buf[RDBK_L] = NULL;
+				}
+				if (active_buf)
+					dvp2axi_dev->rdbk_buf[RDBK_L] = active_buf;
+			} else if (mipi_id == ES_DVP2AXI_STREAM_MIPI_ID1) {
+				if (dvp2axi_dev->rdbk_buf[RDBK_M] && active_buf) {
+					v4l2_err(
+						&dvp2axi_dev->v4l2_dev,
+						"multiple %s frame,frm_idx:%d,state:0x%x\n",
+						dvp2axi_dev->hdr.hdr_mode ==
+								HDR_X2 ?
+							"short data in hdr_x2" :
+							"medium data in hdr_x3",
+						stream->frame_idx,
+						dvp2axi_dev->rdbk_buf[RDBK_M]
+							->vb.vb2_buf.state);
+					dvp2axi_dev->rdbk_buf[RDBK_M]
+						->vb.vb2_buf.state =
+						VB2_BUF_STATE_ACTIVE;
+					es_dvp2axi_buf_queue(
+						&dvp2axi_dev->rdbk_buf[RDBK_M]
+							 ->vb.vb2_buf);
+					dvp2axi_dev->rdbk_buf[RDBK_M] = NULL;
+				}
+				if (active_buf)
+					dvp2axi_dev->rdbk_buf[RDBK_M] = active_buf;
+				if (dvp2axi_dev->hdr.hdr_mode == HDR_X2)
+					es_dvp2axi_rdbk_frame_end(stream);
+			} else if (mipi_id == ES_DVP2AXI_STREAM_MIPI_ID2) {
+				if (dvp2axi_dev->rdbk_buf[RDBK_S] && active_buf) {
+					v4l2_err(
+						&dvp2axi_dev->v4l2_dev,
+						"multiple %s frame, frm_idx:%d,state:0x%x\n",
+						dvp2axi_dev->hdr.hdr_mode ==
+								HDR_X2 ?
+							"err short data in hdr_x3" :
+							"short data in hdr_x3",
+						stream->frame_idx,
+						dvp2axi_dev->rdbk_buf[RDBK_S]
+							->vb.vb2_buf.state);
+					dvp2axi_dev->rdbk_buf[RDBK_S]
+						->vb.vb2_buf.state =
+						VB2_BUF_STATE_ACTIVE;
+					es_dvp2axi_buf_queue(
+						&dvp2axi_dev->rdbk_buf[RDBK_S]
+							 ->vb.vb2_buf);
+					dvp2axi_dev->rdbk_buf[RDBK_S] = NULL;
+				}
+				if (active_buf)
+					dvp2axi_dev->rdbk_buf[RDBK_S] = active_buf;
+				if (dvp2axi_dev->hdr.hdr_mode == HDR_X3)
+					es_dvp2axi_rdbk_frame_end(stream);
+			}
+			spin_unlock_irqrestore(&dvp2axi_dev->hdr_lock, flags);
+		} else {
+			if (active_buf) {
+				vb_done->vb2_buf.state = VB2_BUF_STATE_ACTIVE;
+				es_dvp2axi_buf_queue(&vb_done->vb2_buf);
+			}
+
+			v4l2_info(
+				&dvp2axi_dev->v4l2_dev,
+				"warning:hdr runs stream[%d], stream[0]:%s stream[1]:%s stream[2]:%s stream[3]:%s\n",
+				stream->id,
+				dvp2axi_dev->stream[0].state !=
+						ES_DVP2AXI_STATE_STREAMING ?
+					"stopped" :
+					"running",
+				dvp2axi_dev->stream[1].state !=
+						ES_DVP2AXI_STATE_STREAMING ?
+					"stopped" :
+					"running",
+				dvp2axi_dev->stream[2].state !=
+						ES_DVP2AXI_STATE_STREAMING ?
+					"stopped" :
+					"running",
+				dvp2axi_dev->stream[3].state !=
+						ES_DVP2AXI_STATE_STREAMING ?
+					"stopped" :
+					"running");
+		}
+	}
+}
+
+static void es_dvp2axi_line_wake_up(struct es_dvp2axi_stream *stream, int mipi_id)
+{
+	u32 mode;
+	struct es_dvp2axi_buffer *active_buf = NULL;
+	int ret = 0;
+
+	mode = stream->line_int_cnt % 2;
+	if (mode) {
+		if (stream->curr_buf)
+			active_buf = stream->curr_buf;
+	} else {
+		if (stream->next_buf)
+			active_buf = stream->next_buf;
+	}
+
+	if (stream->stopping) {
+		stream->is_can_stop = true;
+		return;
+	}
+	ret = es_dvp2axi_get_new_buffer_wake_up_mode(stream);
+	if (ret)
+		return;
+	es_dvp2axi_buf_done_prepare(stream, active_buf, mipi_id, mode);
+}
+
+static void es_dvp2axi_store_last_buf_for_online(struct es_dvp2axi_stream *stream,
+					    struct es_dvp2axi_rx_buffer *buf)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dev->active_sensor->mbus;
+	u32 frm0_addr_y, frm1_addr_y;
+
+	INIT_LIST_HEAD(&stream->rx_buf_head);
+	stream->curr_buf_toisp = buf;
+	stream->next_buf_toisp = buf;
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CCP2) {
+		frm0_addr_y = get_reg_index_of_frm0_y_addr(stream->id);
+		frm1_addr_y = get_reg_index_of_frm1_y_addr(stream->id);
+	} else {
+		frm0_addr_y = get_dvp_reg_index_of_frm0_y_addr(stream->id);
+		frm1_addr_y = get_dvp_reg_index_of_frm1_y_addr(stream->id);
+	}
+	es_dvp2axi_write_register(dev, frm0_addr_y, buf->dummy.dma_addr);
+	es_dvp2axi_write_register(dev, frm1_addr_y, buf->dummy.dma_addr);
+}
+
+static void
+es_dvp2axi_release_unnecessary_buf_for_online(struct es_dvp2axi_stream *stream,
+					 struct es_dvp2axi_rx_buffer *buf)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct sditf_priv *priv = dev->sditf[0];
+	struct es_dvp2axi_rx_buffer *rx_buf = NULL;
+	unsigned long flags;
+	int i = 0;
+
+	spin_lock_irqsave(&priv->dvp2axi_dev->buffree_lock, flags);
+	for (i = 0; i < priv->buf_num; i++) {
+		rx_buf = &stream->rx_buf[i];
+		if (rx_buf && (!rx_buf->dummy.is_free) && rx_buf != buf) {
+			list_add_tail(&rx_buf->list_free, &priv->buf_free_list);
+			stream->total_buf_num--;
+		}
+	}
+	spin_unlock_irqrestore(&priv->dvp2axi_dev->buffree_lock, flags);
+	schedule_work(&priv->buffree_work.work);
+}
+
+static void es_dvp2axi_line_wake_up_rdbk(struct es_dvp2axi_stream *stream, int mipi_id)
+{
+	u32 mode;
+	struct es_dvp2axi_rx_buffer *active_buf = NULL;
+	struct sditf_priv *priv = NULL;
+	unsigned long flags;
+	int ret = 0;
+
+	mode = stream->line_int_cnt % 2;
+	if (mode) {
+		if (stream->curr_buf_toisp)
+			active_buf = stream->curr_buf_toisp;
+		stream->frame_phase = DVP2AXI_CSI_FRAME0_READY;
+	} else {
+		if (stream->next_buf_toisp)
+			active_buf = stream->next_buf_toisp;
+		stream->frame_phase = DVP2AXI_CSI_FRAME1_READY;
+	}
+
+	if (!active_buf) {
+		v4l2_err(&stream->dvp2axidev->v4l2_dev, "err buffer state in %s\n",
+			 __func__);
+		return;
+	}
+
+	if (stream->stopping) {
+		stream->is_can_stop = true;
+		return;
+	}
+	ret = es_dvp2axi_get_new_buffer_wake_up_mode_rdbk(stream);
+	v4l2_dbg(3, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		 "%d frame_idx %d, last_rx_buf_idx %d cur dma buf %x\n",
+		 __LINE__, stream->frame_idx, stream->last_rx_buf_idx,
+		 (u32)active_buf->dummy.dma_addr);
+	if (!ret) {
+		priv = stream->dvp2axidev->sditf[0];
+		if (stream->cur_stream_mode & ES_DVP2AXI_STREAM_MODE_TOISP_RDBK) {
+			spin_lock_irqsave(&stream->vbq_lock, flags);
+			if (stream->dvp2axidev->is_thunderboot &&
+			    (stream->frame_idx - 1) ==
+				    stream->last_rx_buf_idx &&
+			    stream->dvp2axidev->is_rdbk_to_online) {
+				stream->cur_stream_mode &=
+					~ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+				stream->cur_stream_mode |=
+					ES_DVP2AXI_STREAM_MODE_TOISP;
+				stream->dvp2axidev->wait_line = 0;
+				stream->is_line_wake_up = false;
+				if (stream->dvp2axidev->hdr.hdr_mode == NO_HDR ||
+				    (priv->hdr_cfg.hdr_mode == HDR_X2 &&
+				     stream->id == 1) ||
+				    (priv->hdr_cfg.hdr_mode == HDR_X3 &&
+				     stream->id == 2)) {
+					stream->to_stop_dma =
+						ES_DVP2AXI_DMAEN_BY_ISP;
+					es_dvp2axi_stop_dma_capture(stream);
+				}
+				active_buf->dbufs.is_switch = true;
+				if ((priv->hdr_cfg.hdr_mode == HDR_X2 &&
+				     stream->id != 1) ||
+				    (priv->hdr_cfg.hdr_mode == HDR_X3 &&
+				     stream->id != 2)) {
+					es_dvp2axi_store_last_buf_for_online(
+						stream, active_buf);
+					stream->is_change_toisp = true;
+				}
+			}
+			spin_unlock_irqrestore(&stream->vbq_lock, flags);
+			active_buf->dbufs.sequence = stream->frame_idx - 1;
+			active_buf->dbufs.timestamp =
+				stream->readout.fs_timestamp;
+			active_buf->fe_timestamp =
+				es_dvp2axi_time_get_ns(stream->dvp2axidev);
+			stream->last_frame_idx = stream->frame_idx;
+			if (stream->dvp2axidev->hdr.hdr_mode == NO_HDR) {
+				es_dvp2axi_s_rx_buffer(stream, &active_buf->dbufs);
+				if (stream->dvp2axidev->is_support_tools &&
+				    stream->tools_vdev)
+					es_dvp2axi_rdbk_with_tools(stream,
+							      active_buf);
+			} else {
+				es_dvp2axi_rdbk_frame_end_toisp(stream, active_buf);
+			}
+		}
+	}
+}
+
+static void es_dvp2axi_deal_readout_time(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct es_dvp2axi_stream *detect_stream = &dvp2axi_dev->stream[0];
+	unsigned long flags;
+
+	spin_lock_irqsave(&stream->fps_lock, flags);
+	stream->readout.fe_timestamp = es_dvp2axi_time_get_ns(dvp2axi_dev);
+
+	if (dvp2axi_dev->inf_id == ES_DVP2AXI_DVP) {
+		spin_unlock_irqrestore(&stream->fps_lock, flags);
+		return;
+	}
+
+	if (stream->id == ES_DVP2AXI_STREAM_MIPI_ID0)
+		detect_stream->readout.readout_time =
+			stream->readout.fe_timestamp -
+			stream->readout.fs_timestamp;
+
+	if ((dvp2axi_dev->hdr.hdr_mode == NO_HDR ||
+	     dvp2axi_dev->hdr.hdr_mode == HDR_COMPR) &&
+	    (stream->id == ES_DVP2AXI_STREAM_MIPI_ID0)) {
+		detect_stream->readout.early_time =
+			stream->readout.fe_timestamp -
+			stream->readout.wk_timestamp;
+
+	} else if ((dvp2axi_dev->hdr.hdr_mode == HDR_X2) &&
+		   (stream->id == ES_DVP2AXI_STREAM_MIPI_ID1)) {
+		detect_stream->readout.early_time =
+			stream->readout.fe_timestamp -
+			stream->readout.wk_timestamp;
+		detect_stream->readout.total_time =
+			stream->readout.fe_timestamp -
+			detect_stream->readout.fe_timestamp;
+		detect_stream->readout.total_time +=
+			detect_stream->readout.readout_time;
+	} else if ((dvp2axi_dev->hdr.hdr_mode == HDR_X3) &&
+		   (stream->id == ES_DVP2AXI_STREAM_MIPI_ID2)) {
+		detect_stream->readout.early_time =
+			stream->readout.fe_timestamp -
+			stream->readout.wk_timestamp;
+		detect_stream->readout.total_time =
+			stream->readout.fe_timestamp -
+			detect_stream->readout.fe_timestamp;
+		detect_stream->readout.total_time +=
+			detect_stream->readout.readout_time;
+	}
+	if (!stream->is_line_wake_up)
+		detect_stream->readout.early_time = 0;
+	spin_unlock_irqrestore(&stream->fps_lock, flags);
+}
+
+static void es_dvp2axi_update_stream(struct es_dvp2axi_device *dvp2axi_dev,
+				struct es_dvp2axi_stream *stream, int mipi_id)
+{
+	struct es_dvp2axi_buffer *active_buf = NULL;
+	unsigned long flags;
+	int ret = 0;
+
+	if (stream->frame_phase ==
+	    (DVP2AXI_CSI_FRAME0_READY | DVP2AXI_CSI_FRAME1_READY)) {
+		dvp2axi_dev->err_state |= (ES_DVP2AXI_ERR_ID0_TRIG_SIMULT << stream->id);
+		dvp2axi_dev->irq_stats.trig_simult_cnt[stream->id]++;
+		return;
+	}
+	if (!stream->is_line_wake_up) {
+		spin_lock_irqsave(&stream->fps_lock, flags);
+		if (stream->frame_phase & DVP2AXI_CSI_FRAME0_READY) {
+			if (stream->curr_buf)
+				active_buf = stream->curr_buf;
+			stream->fps_stats.frm0_timestamp =
+				es_dvp2axi_time_get_ns(dvp2axi_dev);
+		} else if (stream->frame_phase & DVP2AXI_CSI_FRAME1_READY) {
+			if (stream->next_buf)
+				active_buf = stream->next_buf;
+			stream->fps_stats.frm1_timestamp =
+				es_dvp2axi_time_get_ns(dvp2axi_dev);
+		}
+		spin_unlock_irqrestore(&stream->fps_lock, flags);
+	}
+
+	es_dvp2axi_deal_readout_time(stream);
+
+	if (!stream->is_line_wake_up) {
+		ret = es_dvp2axi_assign_new_buffer_pingpong(
+			stream, ES_DVP2AXI_YUV_ADDR_STATE_UPDATE, mipi_id);
+		if (ret && dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI)
+			return;
+	} else {
+		ret = es_dvp2axi_update_new_buffer_wake_up_mode(stream);
+		if (ret && dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI)
+			return;
+	}
+	if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI &&
+	    dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_BT656 &&
+	    stream->id != 0)
+		stream->frame_idx++;
+	if (!stream->is_line_wake_up)
+		es_dvp2axi_buf_done_prepare(stream, active_buf, mipi_id, 0);
+}
+
+static void es_dvp2axi_update_stream_toisp(struct es_dvp2axi_device *dvp2axi_dev,
+				      struct es_dvp2axi_stream *stream, int mipi_id)
+{
+	if (stream->frame_phase ==
+	    (DVP2AXI_CSI_FRAME0_READY | DVP2AXI_CSI_FRAME1_READY)) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev,
+			 "stream[%d], frm0/frm1 end simultaneously,frm id:%d\n",
+			 stream->id, stream->frame_idx);
+		return;
+	}
+
+	spin_lock(&stream->fps_lock);
+	if (stream->frame_phase & DVP2AXI_CSI_FRAME0_READY)
+		stream->fps_stats.frm0_timestamp = es_dvp2axi_time_get_ns(dvp2axi_dev);
+	else if (stream->frame_phase & DVP2AXI_CSI_FRAME1_READY)
+		stream->fps_stats.frm1_timestamp = es_dvp2axi_time_get_ns(dvp2axi_dev);
+	spin_unlock(&stream->fps_lock);
+
+	if (dvp2axi_dev->inf_id == ES_DVP2AXI_MIPI_LVDS)
+		es_dvp2axi_deal_readout_time(stream);
+
+	if (!stream->is_line_wake_up)
+		es_dvp2axi_assign_new_buffer_pingpong_toisp(
+			stream, ES_DVP2AXI_YUV_ADDR_STATE_UPDATE, mipi_id);
+}
+
+// static void es_dvp2axi_update_stream_eskit(struct es_dvp2axi_device *dvp2axi_dev,
+// 				       struct es_dvp2axi_stream *stream, int mipi_id)
+// {
+// 	struct es_dvp2axi_buffer *active_buf = NULL;
+// 	unsigned long flags;
+// 	int ret = 0;
+
+// 	if (stream->frame_phase ==
+// 	    (DVP2AXI_CSI_FRAME0_READY | DVP2AXI_CSI_FRAME1_READY)) {
+// 		v4l2_err(&dvp2axi_dev->v4l2_dev,
+// 			 "stream[%d], frm0/frm1 end simultaneously,frm id:%d\n",
+// 			 stream->id, stream->frame_idx);
+// 		return;
+// 	}
+// 	if (!stream->is_line_wake_up) {
+// 		spin_lock_irqsave(&stream->fps_lock, flags);
+// 		if (stream->frame_phase & DVP2AXI_CSI_FRAME0_READY) {
+// 			if (stream->curr_buf_eskit)
+// 				active_buf = stream->curr_buf_eskit;
+// 			stream->fps_stats.frm0_timestamp =
+// 				es_dvp2axi_time_get_ns(dvp2axi_dev);
+// 		} else if (stream->frame_phase & DVP2AXI_CSI_FRAME1_READY) {
+// 			if (stream->next_buf_eskit)
+// 				active_buf = stream->next_buf_eskit;
+// 			stream->fps_stats.frm1_timestamp =
+// 				es_dvp2axi_time_get_ns(dvp2axi_dev);
+// 		}
+// 		spin_unlock_irqrestore(&stream->fps_lock, flags);
+// 	}
+
+// 	if (dvp2axi_dev->inf_id == ES_DVP2AXI_MIPI_LVDS)
+// 		es_dvp2axi_deal_readout_time(stream);
+
+// 	ret = es_dvp2axi_assign_new_buffer_pingpong_eskit(
+// 		stream, ES_DVP2AXI_YUV_ADDR_STATE_UPDATE, mipi_id);
+// 	if (ret)
+// 		return;
+
+// 	if (active_buf) {
+// 		active_buf->vb.vb2_buf.timestamp = stream->readout.fs_timestamp;
+// 		active_buf->vb.sequence = stream->frame_idx - 1;
+// 		es_dvp2axi_eskit_buf_done(stream, active_buf);
+// 	}
+// }
+
+static u32 es_dvp2axi_get_sof(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	u32 val = 0x0;
+	struct es_dvp2axi_sensor_info *sensor = dvp2axi_dev->active_sensor;
+	// struct csi2_dev *csi;
+
+	if (sensor->mbus.type == V4L2_MBUS_CCP2) {
+		val = es_dvp2axi_lvds_get_sof(dvp2axi_dev);
+	} else if (sensor->mbus.type == V4L2_MBUS_PARALLEL ||
+		   sensor->mbus.type == V4L2_MBUS_BT656) {
+		val = es_dvp2axi_dvp_get_sof(dvp2axi_dev);
+	}
+	return val;
+}
+
+static void es_dvp2axi_set_sof(struct es_dvp2axi_device *dvp2axi_dev, u32 seq)
+{
+	struct es_dvp2axi_sensor_info *sensor = dvp2axi_dev->active_sensor;
+	// struct csi2_dev *csi;
+
+	if (sensor->mbus.type == V4L2_MBUS_CCP2) {
+		es_dvp2axi_lvds_set_sof(dvp2axi_dev, seq);
+	} else if (sensor->mbus.type == V4L2_MBUS_PARALLEL ||
+		   sensor->mbus.type == V4L2_MBUS_BT656) {
+		es_dvp2axi_dvp_set_sof(dvp2axi_dev, seq);
+	}
+}
+
+static void es_dvp2axi_toisp_set_stream(struct es_dvp2axi_device *dev, int on)
+{
+	struct v4l2_subdev *sd = get_esisp_sd(dev->sditf[0]);
+
+	if (sd)
+		v4l2_subdev_call(sd, core, ioctl, ESISP_VICAP_CMD_SET_STREAM,
+				 &on);
+}
+
+static int es_dvp2axi_do_reset_work(struct es_dvp2axi_device *dvp2axi_dev,
+			       enum esmodule_reset_src reset_src)
+{
+	struct es_dvp2axi_pipeline *p = &dvp2axi_dev->pipe;
+	struct es_dvp2axi_stream *stream = NULL;
+	struct es_dvp2axi_stream *resume_stream[ES_DVP2AXI_MAX_STREAM_MIPI] = { NULL };
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dvp2axi_dev->terminal_sensor;
+	struct es_dvp2axi_resume_info *resume_info =
+		&dvp2axi_dev->reset_work.resume_info;
+	struct es_dvp2axi_timer *timer = &dvp2axi_dev->reset_watchdog_timer;
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	int i, j, ret = 0;
+	u32 on, sof_cnt;
+	int capture_mode = 0;
+
+	mutex_lock(&dvp2axi_dev->stream_lock);
+	if (dvp2axi_dev->reset_work_cancel) {
+		ret = 0;
+		goto unlock_stream;
+	}
+	v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev, "do es_dvp2axi reset\n");
+
+	for (i = 0, j = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+		stream = &dvp2axi_dev->stream[i];
+
+		if (stream->state == ES_DVP2AXI_STATE_STREAMING) {
+			v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				 "stream[%d] stopping\n", stream->id);
+
+			es_dvp2axi_stream_stop(stream);
+
+			if (stream->id == ES_DVP2AXI_STREAM_MIPI_ID0) {
+				sof_cnt = es_dvp2axi_get_sof(dvp2axi_dev);
+				v4l2_dbg(
+					1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+					"%s: stream[%d] sync frmid & csi_sof, frm_id:%d, csi_sof:%d\n",
+					__func__, stream->id, stream->frame_idx,
+					sof_cnt);
+
+				resume_info->frm_sync_seq = stream->frame_idx;
+			}
+
+			stream->state = ES_DVP2AXI_STATE_RESET_IN_STREAMING;
+			stream->is_fs_fe_not_paired = false;
+			stream->fs_cnt_in_single_frame = 0;
+			resume_stream[j] = stream;
+			j += 1;
+
+			v4l2_dbg(
+				1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				"%s stop stream[%d] in streaming, frm_id:%d, csi_sof:%d\n",
+				__func__, stream->id, stream->frame_idx,
+				es_dvp2axi_get_sof(dvp2axi_dev));
+		}
+	}
+
+	on = 0;
+	for (i = 0; i < p->num_subdevs; i++) {
+		if (p->subdevs[i] == terminal_sensor->sd) {
+			if (reset_src != ES_RESET_SRC_ERR_APP) {
+				ret = v4l2_subdev_call(
+					p->subdevs[i], core, ioctl,
+					ESMODULE_SET_QUICK_STREAM, &on);
+				if (ret)
+					v4l2_dbg(
+						1, es_dvp2axi_debug,
+						&dvp2axi_dev->v4l2_dev,
+						"quick stream off subdev:%s failed\n",
+						p->subdevs[i]->name);
+			}
+		} else {
+			ret = v4l2_subdev_call(p->subdevs[i], video, s_stream,
+					       on);
+		}
+		if (ret)
+			v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				 "%s:stream %s subdev:%s failed\n", __func__,
+				 on ? "on" : "off", p->subdevs[i]->name);
+	}
+
+	if (priv && priv->is_combine_mode &&
+	    dvp2axi_dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF) {
+		for (i = 0; i < dvp2axi_dev->sditf_cnt; i++) {
+			if (dvp2axi_dev->sditf[i] && dvp2axi_dev->sditf[i]->sensor_sd)
+				ret = v4l2_subdev_call(
+					dvp2axi_dev->sditf[i]->sensor_sd, core,
+					ioctl, ESMODULE_SET_QUICK_STREAM, &on);
+		}
+	}
+
+	if (dvp2axi_dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+		es_dvp2axi_do_soft_reset(dvp2axi_dev);
+	} else {
+		es_dvp2axi_do_cru_reset(dvp2axi_dev);
+
+		es_dvp2axi_disable_sys_clk(dvp2axi_dev->hw_dev);
+
+		udelay(5);
+
+		ret = es_dvp2axi_enable_sys_clk(dvp2axi_dev->hw_dev);
+
+		if (ret < 0) {
+			v4l2_err(&dvp2axi_dev->v4l2_dev,
+				 "%s:resume dvp2axi clk failed\n", __func__);
+			goto unlock_stream;
+		}
+	}
+
+	if (priv && priv->mode.rdbk_mode == ESISP_VICAP_ONLINE)
+		es_dvp2axi_toisp_set_stream(dvp2axi_dev, 1);
+
+	for (i = 0; i < j; i++) {
+		stream = resume_stream[i];
+		stream->fs_cnt_in_single_frame = 0;
+		if (stream->dvp2axi_fmt_in->field == V4L2_FIELD_INTERLACED) {
+			if (stream->curr_buf == stream->next_buf) {
+				if (stream->curr_buf)
+					list_add_tail(&stream->curr_buf->queue,
+						      &stream->buf_head);
+			} else {
+				if (stream->curr_buf)
+					list_add_tail(&stream->curr_buf->queue,
+						      &stream->buf_head);
+				if (stream->next_buf)
+					list_add_tail(&stream->next_buf->queue,
+						      &stream->buf_head);
+			}
+			stream->curr_buf = NULL;
+			stream->next_buf = NULL;
+		}
+		if (!dvp2axi_dev->sditf[0] ||
+		    dvp2axi_dev->sditf[0]->mode.rdbk_mode == ESISP_VICAP_RDBK_AIQ)
+			capture_mode = ES_DVP2AXI_STREAM_MODE_CAPTURE;
+		else {
+			if (dvp2axi_dev->sditf[0]->mode.rdbk_mode ==
+			    ESISP_VICAP_ONLINE)
+				capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+			else
+				capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+		}
+		if (dvp2axi_dev->active_sensor &&
+		    (dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+		     dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+		     dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CCP2)) {
+			//  pr_info("%s:%d yfx !!!!!! call es_dvp2axi csi stream start \n", __func__, __LINE__);
+			ret = es_dvp2axi_csi_stream_start(stream, capture_mode);
+		}
+		else
+			ret = es_dvp2axi_stream_start(stream, capture_mode);
+		if (ret) {
+			v4l2_err(&dvp2axi_dev->v4l2_dev,
+				 "%s:resume stream[%d] failed\n", __func__,
+				 stream->id);
+			goto unlock_stream;
+		}
+
+		v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "resume stream[%d], frm_idx:%d, csi_sof:%d\n",
+			 stream->id, stream->frame_idx, es_dvp2axi_get_sof(dvp2axi_dev));
+	}
+
+	on = 1;
+	for (i = 0; i < p->num_subdevs; i++) {
+		if (p->subdevs[i] == terminal_sensor->sd) {
+			es_dvp2axi_set_sof(dvp2axi_dev, resume_info->frm_sync_seq);
+
+			if (reset_src != ES_RESET_SRC_ERR_APP) {
+				ret = v4l2_subdev_call(
+					p->subdevs[i], core, ioctl,
+					ESMODULE_SET_QUICK_STREAM, &on);
+				if (ret)
+					v4l2_err(
+						&dvp2axi_dev->v4l2_dev,
+						"quick stream on subdev:%s failed\n",
+						p->subdevs[i]->name);
+			}
+		} else {
+			if (p->subdevs[i] == terminal_sensor->sd)
+				es_dvp2axi_set_sof(dvp2axi_dev,
+					      resume_info->frm_sync_seq);
+
+			ret = v4l2_subdev_call(p->subdevs[i], video, s_stream,
+					       on);
+		}
+
+		if (ret)
+			v4l2_err(&dvp2axi_dev->v4l2_dev, "reset subdev:%s failed\n",
+				 p->subdevs[i]->name);
+	}
+
+	if (priv && priv->is_combine_mode &&
+	    dvp2axi_dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF) {
+		for (i = 0; i < dvp2axi_dev->sditf_cnt; i++) {
+			if (dvp2axi_dev->sditf[i] && dvp2axi_dev->sditf[i]->sensor_sd)
+				v4l2_subdev_call(dvp2axi_dev->sditf[i]->sensor_sd,
+						 core, ioctl,
+						 ESMODULE_SET_QUICK_STREAM,
+						 &on);
+		}
+	}
+
+	if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI)
+		es_dvp2axi_start_luma(
+			&dvp2axi_dev->luma_vdev,
+			dvp2axi_dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0].dvp2axi_fmt_in);
+
+	timer->csi2_err_triggered_cnt = 0;
+	es_dvp2axi_monitor_reset_event(dvp2axi_dev);
+
+	v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+		 "do es_dvp2axi reset successfully!\n");
+	mutex_unlock(&dvp2axi_dev->stream_lock);
+	return 0;
+
+unlock_stream:
+	mutex_unlock(&dvp2axi_dev->stream_lock);
+	return ret;
+}
+
+void es_dvp2axi_reset_work(struct work_struct *work)
+{
+	struct es_dvp2axi_work_struct *reset_work =
+		container_of(work, struct es_dvp2axi_work_struct, work);
+	struct es_dvp2axi_device *dev =
+		container_of(reset_work, struct es_dvp2axi_device, reset_work);
+	int ret;
+
+	ret = es_dvp2axi_do_reset_work(dev, reset_work->reset_src);
+	if (ret)
+		v4l2_info(&dev->v4l2_dev, "do reset work failed!\n");
+}
+
+static bool es_dvp2axi_is_reduced_frame_rate(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_timer *timer = &dev->reset_watchdog_timer;
+	struct es_dvp2axi_stream *stream = &dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0];
+	struct v4l2_rect *raw_rect = &dev->terminal_sensor.raw_rect;
+	u64 fps, timestamp0, timestamp1, diff_time;
+	unsigned long fps_flags;
+	unsigned int deviation = 1;
+	bool is_reduced = false;
+	u64 cur_time = 0;
+	u64 time_distance = 0;
+
+	spin_lock_irqsave(&stream->fps_lock, fps_flags);
+	timestamp0 = stream->fps_stats.frm0_timestamp;
+	timestamp1 = stream->fps_stats.frm1_timestamp;
+	spin_unlock_irqrestore(&stream->fps_lock, fps_flags);
+
+	fps = timestamp0 > timestamp1 ? timestamp0 - timestamp1 :
+					timestamp1 - timestamp0;
+	fps = div_u64(fps, 1000);
+	diff_time = fps > timer->frame_end_cycle_us ?
+			    fps - timer->frame_end_cycle_us :
+			    0;
+	deviation = DIV_ROUND_UP(timer->vts, 100);
+
+	v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev,
+		 "diff_time:%lld,devi_t:%ld,devi_h:%d\n", diff_time,
+		 timer->line_end_cycle * deviation, deviation);
+
+	cur_time = es_dvp2axi_time_get_ns(dev);
+	time_distance = timestamp0 > timestamp1 ? cur_time - timestamp0 :
+						  cur_time - timestamp1;
+	time_distance = div_u64(time_distance, 1000);
+	if (time_distance > fps * 2)
+		return false;
+
+	if (diff_time > timer->line_end_cycle * deviation) {
+		s32 vblank = 0;
+		unsigned int vts;
+
+		is_reduced = true;
+		vblank = es_dvp2axi_get_sensor_vblank(dev);
+		vts = vblank + timer->raw_height;
+
+		v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev,
+			 "old vts:%d,new vts:%d\n", timer->vts, vts);
+
+		v4l2_dbg(
+			3, es_dvp2axi_debug, &dev->v4l2_dev,
+			"reduce frame rate,vblank:%d, height(raw output):%d, fps:%lld, frm_end_t:%ld, line_t:%ld, diff:%lld\n",
+			es_dvp2axi_get_sensor_vblank(dev), raw_rect->height, fps,
+			timer->frame_end_cycle_us, timer->line_end_cycle,
+			diff_time);
+
+		timer->vts = vts;
+		timer->frame_end_cycle_us = fps;
+		timer->line_end_cycle =
+			div_u64(timer->frame_end_cycle_us, timer->vts);
+	} else {
+		is_reduced = false;
+	}
+
+	timer->frame_end_cycle_us = fps;
+
+	fps = div_u64(fps, 1000);
+	fps = fps * timer->frm_num_of_monitor_cycle;
+	timer->cycle = msecs_to_jiffies(fps);
+	timer->timer.expires = jiffies + timer->cycle;
+
+	return is_reduced;
+}
+
+static void es_dvp2axi_dvp_event_reset_pipe(struct es_dvp2axi_device *dev, int reset_src)
+{
+	struct es_dvp2axi_dvp_sof_subdev *subdev = &dev->dvp_sof_subdev;
+
+	if (subdev) {
+		struct v4l2_event event = {
+			.type = V4L2_EVENT_RESET_DEV,
+			.reserved[0] = reset_src,
+		};
+		v4l2_event_queue(subdev->sd.devnode, &event);
+	}
+}
+
+static void es_dvp2axi_lvds_event_reset_pipe(struct es_dvp2axi_device *dev, int reset_src)
+{
+	struct es_dvp2axi_lvds_subdev *subdev = &dev->lvds_subdev;
+
+	if (subdev) {
+		struct v4l2_event event = {
+			.type = V4L2_EVENT_RESET_DEV,
+			.reserved[0] = reset_src,
+		};
+		v4l2_event_queue(subdev->sd.devnode, &event);
+	}
+}
+
+static void es_dvp2axi_send_reset_event(struct es_dvp2axi_device *dvp2axi_dev, int reset_src)
+{
+	struct v4l2_mbus_config *mbus = &dvp2axi_dev->active_sensor->mbus;
+	// struct csi2_dev *csi;
+
+	if (mbus->type == V4L2_MBUS_CCP2) {
+		es_dvp2axi_lvds_event_reset_pipe(dvp2axi_dev, reset_src);
+	} else {
+		es_dvp2axi_dvp_event_reset_pipe(dvp2axi_dev, reset_src);
+	}
+	v4l2_dbg(3, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+		 "send reset event,bus type 0x%x\n", mbus->type);
+}
+
+static void es_dvp2axi_init_reset_work(struct es_dvp2axi_timer *timer)
+{
+	struct es_dvp2axi_device *dev =
+		container_of(timer, struct es_dvp2axi_device, reset_watchdog_timer);
+	struct es_dvp2axi_stream *stream = NULL;
+	unsigned long flags;
+	int i = 0;
+
+	v4l2_info(&dev->v4l2_dev,
+		  "do reset work schedule, run_cnt:%d, reset source:%d\n",
+		  timer->run_cnt, timer->reset_src);
+
+	spin_lock_irqsave(&timer->timer_lock, flags);
+	timer->is_running = false;
+	timer->is_triggered = false;
+	timer->csi2_err_cnt_odd = 0;
+	timer->csi2_err_cnt_even = 0;
+	timer->csi2_err_fs_fe_cnt = 0;
+	timer->notifer_called_cnt = 0;
+	dev->is_toisp_reset = false;
+	for (i = 0; i < dev->num_channels; i++) {
+		stream = &dev->stream[i];
+		if (stream->state == ES_DVP2AXI_STATE_STREAMING)
+			timer->last_buf_wakeup_cnt[stream->id] =
+				stream->buf_wake_up_cnt;
+	}
+	spin_unlock_irqrestore(&timer->timer_lock, flags);
+	if (timer->is_ctrl_by_user) {
+		es_dvp2axi_send_reset_event(dev, timer->reset_src);
+	} else {
+		dev->reset_work.reset_src = timer->reset_src;
+		if (!schedule_work(&dev->reset_work.work))
+			v4l2_info(&dev->v4l2_dev,
+				  "schedule reset work failed\n");
+	}
+}
+
+static int es_dvp2axi_detect_reset_event(struct es_dvp2axi_stream *stream,
+				    struct es_dvp2axi_timer *timer, int check_cnt,
+				    bool *is_mod_timer)
+{
+	struct es_dvp2axi_device *dev = stream->dvp2axidev;
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dev->terminal_sensor;
+	unsigned long flags;
+	int ret, is_reset = 0;
+	struct esmodule_vicap_reset_info rst_info;
+
+	if (dev->is_toisp_reset) {
+		is_reset = 1;
+		timer->reset_src = ES_RESET_SRC_ERR_ISP;
+	}
+	if (is_reset) {
+		es_dvp2axi_init_reset_work(timer);
+		return is_reset;
+	}
+
+	if (timer->last_buf_wakeup_cnt[stream->id] < stream->buf_wake_up_cnt &&
+	    check_cnt == 0) {
+		v4l2_dbg(
+			1, es_dvp2axi_debug, &dev->v4l2_dev,
+			"info: frame end still update(%d, %d) in detecting cnt:%d, mode:%d\n",
+			timer->last_buf_wakeup_cnt[stream->id],
+			stream->frame_idx, timer->run_cnt, timer->monitor_mode);
+
+		timer->last_buf_wakeup_cnt[stream->id] =
+			stream->buf_wake_up_cnt;
+
+		if (stream->frame_idx > 2)
+			es_dvp2axi_is_reduced_frame_rate(dev);
+
+		if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_HOTPLUG) {
+			ret = v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+					       ESMODULE_GET_VICAP_RST_INFO,
+					       &rst_info);
+			if (ret)
+				is_reset = 0;
+			else
+				is_reset = rst_info.is_reset;
+			rst_info.is_reset = 0;
+			if (is_reset)
+				timer->reset_src = ES_RESET_SRC_ERR_HOTPLUG;
+			v4l2_subdev_call(terminal_sensor->sd, core, ioctl,
+					 ESMODULE_SET_VICAP_RST_INFO,
+					 &rst_info);
+			if (!is_reset) {
+				is_reset =
+					es_dvp2axi_is_csi2_err_trigger_reset(timer);
+				if (is_reset)
+					timer->reset_src =
+						ES_RESET_SRC_ERR_CSI2;
+			}
+		} else if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_CONTINUE) {
+			is_reset = es_dvp2axi_is_csi2_err_trigger_reset(timer);
+		} else if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_TRIGGER) {
+			is_reset = timer->is_csi2_err_occurred;
+			if (is_reset)
+				timer->reset_src = ES_RESET_SRC_ERR_CSI2;
+			timer->is_csi2_err_occurred = false;
+		}
+
+		if (is_reset) {
+			es_dvp2axi_init_reset_work(timer);
+			return is_reset;
+		}
+
+		if (timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_CONTINUE ||
+		    timer->monitor_mode == ES_DVP2AXI_MONITOR_MODE_HOTPLUG) {
+			if (timer->run_cnt == timer->max_run_cnt)
+				timer->run_cnt = 0x0;
+			*is_mod_timer = true;
+		} else {
+			if (timer->run_cnt <= timer->max_run_cnt) {
+				*is_mod_timer = true;
+			} else {
+				spin_lock_irqsave(&timer->timer_lock, flags);
+				timer->is_triggered = false;
+				timer->is_running = false;
+				spin_unlock_irqrestore(&timer->timer_lock,
+						       flags);
+				v4l2_info(&dev->v4l2_dev,
+					  "stop reset detecting!\n");
+			}
+		}
+	} else if (timer->last_buf_wakeup_cnt[stream->id] ==
+		   stream->buf_wake_up_cnt) {
+		bool is_reduced = false;
+
+		if (stream->frame_idx > 2)
+			is_reduced = es_dvp2axi_is_reduced_frame_rate(dev);
+		else if (timer->run_cnt < 20)
+			is_reduced = true;
+
+		if (is_reduced) {
+			*is_mod_timer = true;
+			v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "%s fps is reduced\n", __func__);
+		} else {
+			v4l2_info(
+				&dev->v4l2_dev,
+				"do reset work due to frame end is stopped, run_cnt:%d\n",
+				timer->run_cnt);
+
+			timer->reset_src = ES_RESET_SRC_ERR_CUTOFF;
+			es_dvp2axi_init_reset_work(timer);
+			is_reset = true;
+		}
+	}
+
+	return is_reset;
+}
+
+void es_dvp2axi_reset_watchdog_timer_handler(struct timer_list *t)
+{
+	struct es_dvp2axi_timer *timer = container_of(t, struct es_dvp2axi_timer, timer);
+	struct es_dvp2axi_device *dev =
+		container_of(timer, struct es_dvp2axi_device, reset_watchdog_timer);
+	struct es_dvp2axi_stream *stream = NULL;
+	unsigned long flags;
+	unsigned int i;
+	int is_reset = 0;
+	int check_cnt = 0;
+	bool is_mod_timer = false;
+
+	for (i = 0; i < dev->num_channels; i++) {
+		stream = &dev->stream[i];
+		if (stream->state == ES_DVP2AXI_STATE_STREAMING) {
+			is_reset = es_dvp2axi_detect_reset_event(
+				stream, timer, check_cnt, &is_mod_timer);
+			check_cnt++;
+			if (is_reset)
+				break;
+		}
+	}
+	if (!is_reset && is_mod_timer)
+		mod_timer(&timer->timer, jiffies + timer->cycle);
+
+	timer->run_cnt += 1;
+
+	if (!check_cnt) {
+		spin_lock_irqsave(&timer->timer_lock, flags);
+		timer->is_triggered = false;
+		timer->is_running = false;
+		spin_unlock_irqrestore(&timer->timer_lock, flags);
+
+		v4l2_info(&dev->v4l2_dev,
+			  "all stream is stopped, stop reset detect!\n");
+	}
+}
+
+int es_dvp2axi_reset_notifier(struct notifier_block *nb, unsigned long action,
+			 void *data)
+{
+	struct es_dvp2axi_hw *hw = container_of(nb, struct es_dvp2axi_hw, reset_notifier);
+	struct es_dvp2axi_device *dev = NULL;
+	struct es_dvp2axi_timer *timer = NULL;
+	unsigned long flags, val;
+	u32 *csi_idx = data;
+	int i = 0;
+	bool is_match_dev = false;
+
+	for (i = 0; i < hw->dev_num; i++) {
+		dev = hw->dvp2axi_dev[i];
+		if (*csi_idx == dev->csi_host_idx) {
+			is_match_dev = true;
+			break;
+		}
+	}
+	if (!is_match_dev)
+		return -EINVAL;
+	timer = &dev->reset_watchdog_timer;
+	if (timer->is_running) {
+		val = action & CSI2_ERR_COUNT_ALL_MASK;
+		spin_lock_irqsave(&timer->csi2_err_lock, flags);
+		if ((val % timer->csi2_err_ref_cnt) == 0) {
+			timer->notifer_called_cnt++;
+			if ((timer->notifer_called_cnt % 2) == 0)
+				timer->csi2_err_cnt_even = val;
+			else
+				timer->csi2_err_cnt_odd = val;
+		}
+
+		timer->csi2_err_fs_fe_cnt = (action & CSI2_ERR_FSFE_MASK) >> 8;
+		spin_unlock_irqrestore(&timer->csi2_err_lock, flags);
+	}
+
+	return 0;
+}
+
+static void es_dvp2axi_modify_line_int(struct es_dvp2axi_stream *stream, bool en)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	u32 line_intr_en = 0;
+
+	if (dvp2axi_dev->chip_id >= CHIP_EIC770X_DVP2AXI)
+		line_intr_en = CSI_LINE_INTEN_EIC770X(stream->id);
+	else
+		line_intr_en = CSI_LINE_INTEN(stream->id);
+	if (en) {
+		if (dvp2axi_dev->wait_line_bak != dvp2axi_dev->wait_line) {
+			dvp2axi_dev->wait_line_bak = dvp2axi_dev->wait_line;
+			es_dvp2axi_write_register(
+				dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID0_1,
+				dvp2axi_dev->wait_line << 16 | dvp2axi_dev->wait_line);
+			es_dvp2axi_write_register(
+				dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID2_3,
+				dvp2axi_dev->wait_line << 16 | dvp2axi_dev->wait_line);
+		}
+		es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					line_intr_en);
+		stream->is_line_inten = true;
+	} else {
+		es_dvp2axi_write_register_and(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					 ~line_intr_en);
+	}
+}
+
+static void es_dvp2axi_detect_wake_up_mode_change(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	bool is_change = false;
+	int ch = 0;
+	int i = 0;
+
+	if (!priv || priv->mode.rdbk_mode == ESISP_VICAP_ONLINE)
+		return;
+
+	if ((dvp2axi_dev->hdr.hdr_mode == NO_HDR ||
+	     dvp2axi_dev->hdr.hdr_mode == HDR_COMPR) &&
+	    stream->id == ES_DVP2AXI_STREAM_MIPI_ID0) {
+		if (dvp2axi_dev->wait_line != dvp2axi_dev->wait_line_cache)
+			dvp2axi_dev->wait_line = dvp2axi_dev->wait_line_cache;
+	} else if (dvp2axi_dev->hdr.hdr_mode == HDR_X2 &&
+		   stream->id == ES_DVP2AXI_STREAM_MIPI_ID1) {
+		if (dvp2axi_dev->wait_line != dvp2axi_dev->wait_line_cache)
+			dvp2axi_dev->wait_line = dvp2axi_dev->wait_line_cache;
+	} else if (dvp2axi_dev->hdr.hdr_mode == HDR_X3 &&
+		   stream->id == ES_DVP2AXI_STREAM_MIPI_ID2) {
+		if (dvp2axi_dev->wait_line != dvp2axi_dev->wait_line_cache)
+			dvp2axi_dev->wait_line = dvp2axi_dev->wait_line_cache;
+	}
+
+	if (dvp2axi_dev->wait_line && (!stream->is_line_wake_up)) {
+		is_change = true;
+		stream->is_line_wake_up = true;
+		if (stream->frame_phase == DVP2AXI_CSI_FRAME0_READY)
+			stream->line_int_cnt = 1;
+		else if (stream->frame_phase == DVP2AXI_CSI_FRAME1_READY)
+			stream->line_int_cnt = 0;
+		if (dvp2axi_dev->hdr.hdr_mode == HDR_X2) {
+			dvp2axi_dev->stream[0].is_line_wake_up = true;
+			dvp2axi_dev->stream[0].line_int_cnt = stream->line_int_cnt;
+		} else if (dvp2axi_dev->hdr.hdr_mode == HDR_X3) {
+			dvp2axi_dev->stream[0].is_line_wake_up = true;
+			dvp2axi_dev->stream[1].is_line_wake_up = true;
+			dvp2axi_dev->stream[0].line_int_cnt = stream->line_int_cnt;
+			dvp2axi_dev->stream[1].line_int_cnt = stream->line_int_cnt;
+		}
+	} else if ((dvp2axi_dev->wait_line == 0) && stream->is_line_wake_up) {
+		stream->is_line_wake_up = false;
+	}
+	if (stream->is_line_wake_up) {
+		if (is_change) {
+			if (dvp2axi_dev->hdr.hdr_mode == HDR_X2)
+				ch = 2;
+			else if (dvp2axi_dev->hdr.hdr_mode == HDR_X3)
+				ch = 3;
+			else
+				ch = 1;
+			for (i = 0; i < ch; i++)
+				es_dvp2axi_modify_line_int(&dvp2axi_dev->stream[i],
+						      true);
+		} else {
+			es_dvp2axi_modify_line_int(stream, true);
+		}
+	}
+}
+
+u32 es_dvp2axi_mbus_pixelcode_to_v4l2(u32 pixelcode)
+{
+	u32 pixelformat;
+	switch (pixelcode) {
+	case MEDIA_BUS_FMT_Y8_1X8:
+		pixelformat = V4L2_PIX_FMT_GREY;
+		break;
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+		pixelformat = V4L2_PIX_FMT_SBGGR8;
+		break;
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+		pixelformat = V4L2_PIX_FMT_SGBRG8;
+		break;
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+		pixelformat = V4L2_PIX_FMT_SGRBG8;
+		break;
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+		pixelformat = V4L2_PIX_FMT_SRGGB8;
+		break;
+	case MEDIA_BUS_FMT_Y10_1X10:
+		pixelformat = V4L2_PIX_FMT_Y10;
+		break;
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+		pixelformat = V4L2_PIX_FMT_SBGGR10;
+		break;
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+		pixelformat = V4L2_PIX_FMT_SGBRG10;
+		break;
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+		pixelformat = V4L2_PIX_FMT_SGRBG10;
+		break;
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+		pixelformat = V4L2_PIX_FMT_SRGGB10;
+		break;
+	case MEDIA_BUS_FMT_Y12_1X12:
+		pixelformat = V4L2_PIX_FMT_Y12;
+		break;
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+		pixelformat = V4L2_PIX_FMT_SBGGR12;
+		break;
+	case MEDIA_BUS_FMT_SGBRG12_1X12:
+		pixelformat = V4L2_PIX_FMT_SGBRG12;
+		break;
+	case MEDIA_BUS_FMT_SGRBG12_1X12:
+		pixelformat = V4L2_PIX_FMT_SGRBG12;
+		break;
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+		pixelformat = V4L2_PIX_FMT_SRGGB12;
+		break;
+	case MEDIA_BUS_FMT_SPD_2X8:
+		pixelformat = V4l2_PIX_FMT_SPD16;
+		break;
+	case MEDIA_BUS_FMT_EBD_1X8:
+		pixelformat = V4l2_PIX_FMT_EBD8;
+		break;
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+		pixelformat = V4L2_PIX_FMT_YVYU;
+		break;
+	case MEDIA_BUS_FMT_YVYU10_2X10:
+		pixelformat = V4L2_PIX_FMT_Y210;
+		break;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		pixelformat = V4L2_PIX_FMT_RGB24;
+		break;
+	default:
+		pixelformat = V4L2_PIX_FMT_SRGGB10;
+	}
+
+	return pixelformat;
+}
+
+void es_dvp2axi_set_default_fmt(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct v4l2_subdev_selection input_sel;
+	struct v4l2_pix_format_mplane pixm;
+	struct v4l2_subdev_format fmt;
+	int stream_num = 0;
+	int ret, i;
+
+	stream_num = ES_DVP2AXI_MAX_STREAM_MIPI;
+
+	if (!dvp2axi_dev->terminal_sensor.sd)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->terminal_sensor.sd) {
+		for (i = 0; i < stream_num; i++) {
+			if (i == ES_DVP2AXI_STREAM_MIPI_ID3)
+				dvp2axi_dev->stream[i].is_compact = false;
+			memset(&fmt, 0, sizeof(fmt));
+			fmt.pad = i;
+			fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+			v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, pad,
+					 get_fmt, NULL, &fmt);
+			memset(&pixm, 0, sizeof(pixm));
+			pixm.pixelformat =
+				es_dvp2axi_mbus_pixelcode_to_v4l2(fmt.format.code);
+			pixm.width = fmt.format.width;
+			pixm.height = fmt.format.height;
+			memset(&input_sel, 0, sizeof(input_sel));
+			input_sel.pad = i;
+			input_sel.target = V4L2_SEL_TGT_CROP_BOUNDS;
+			input_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+			ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, pad,
+					       get_selection, NULL, &input_sel);
+			if (!ret) {
+				pixm.width = input_sel.r.width;
+				pixm.height = input_sel.r.height;
+			}
+			pr_debug("pixm.pixelformat:%d, pixm.width:%d, pixm.height:%d\n",pixm.pixelformat, pixm.width, pixm.height);
+			pr_debug("dvp2axi_dev->terminal_sensor.sd->name :%s\n",dvp2axi_dev->terminal_sensor.sd->name);
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[i], &pixm, false);
+		}
+	}
+}
+
+void es_dvp2axi_enable_dma_capture(struct es_dvp2axi_stream *stream, bool is_only_enable)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dvp2axi_dev->active_sensor->mbus;
+	struct csi_channel_info *channel = &dvp2axi_dev->channels[stream->id];
+	u32 val = 0;
+
+	if (stream->buf_owner == ES_DVP2AXI_DMAEN_BY_ISP)
+		stream->buf_owner = ES_DVP2AXI_DMAEN_BY_ISP_TO_VICAP;
+
+	atomic_dec(&dvp2axi_dev->streamoff_cnt);
+	if (stream->dma_en) {
+		stream->dma_en |= stream->to_en_dma;
+		stream->to_en_dma = 0;
+		return;
+	}
+
+	stream->dma_en |= stream->to_en_dma;
+	if (!is_only_enable) {
+		if (stream->to_en_dma == ES_DVP2AXI_DMAEN_BY_VICAP) {
+			es_dvp2axi_assign_new_buffer_pingpong(
+				stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, stream->id);
+			es_dvp2axi_write_register(
+				dvp2axi_dev,
+				get_reg_index_of_frm0_y_vlw(stream->id),
+				channel->virtual_width);
+		} else if (stream->to_en_dma == ES_DVP2AXI_DMAEN_BY_ISP) {
+			es_dvp2axi_assign_new_buffer_pingpong_toisp(
+				stream, ES_DVP2AXI_YUV_ADDR_STATE_INIT, stream->id);
+		}
+	}
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY)
+		es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_CTRL,
+					0x00010000);
+	else
+		es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL, 0x00010000);
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY) {
+		val = es_dvp2axi_read_register(
+			dvp2axi_dev, get_reg_index_of_id_ctrl0(stream->id));
+		if (!stream->is_compact)
+			val |= CSI_WRDDR_TYPE_RAW_UNCOMPACT;
+		else
+			val &= ~CSI_WRDDR_TYPE_RAW_UNCOMPACT;
+		val |= CSI_DMA_ENABLE;
+		es_dvp2axi_write_register(
+			dvp2axi_dev, get_reg_index_of_id_ctrl0(stream->id), val);
+	} else {
+		val = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_FOR);
+		if (!stream->is_compact)
+			val |= CSI_WRDDR_TYPE_RAW_UNCOMPACT << 11;
+		else
+			val &= ~(CSI_WRDDR_TYPE_RAW_UNCOMPACT << 11);
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_FOR, val);
+		val = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL);
+		if (dvp2axi_dev->chip_id == CHIP_EIC770X_DVP2AXI)
+			val |= DVP_DMA_EN;
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL, val);
+	}
+
+	stream->to_en_dma = 0;
+}
+
+static int es_dvp2axi_stop_dma_capture(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	struct v4l2_mbus_config *mbus_cfg = &dvp2axi_dev->active_sensor->mbus;
+	u32 val = 0;
+
+	if (stream->buf_replace_cnt)
+		return -EINVAL;
+
+	stream->dma_en &= ~stream->to_stop_dma;
+	atomic_inc(&dvp2axi_dev->streamoff_cnt);
+	if (stream->dma_en != 0) {
+		if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP)
+			stream->buf_owner = ES_DVP2AXI_DMAEN_BY_ISP;
+		stream->to_stop_dma = 0;
+		return 0;
+	}
+
+	if (mbus_cfg->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus_cfg->type == V4L2_MBUS_CSI2_CPHY) {
+		val = es_dvp2axi_read_register(
+			dvp2axi_dev, get_reg_index_of_id_ctrl0(stream->id));
+		val &= ~CSI_DMA_ENABLE;
+		if (stream->is_stop_capture) {
+			val &= ~CSI_ENABLE_CAPTURE;
+			stream->is_stop_capture = false;
+		}
+		es_dvp2axi_write_register(
+			dvp2axi_dev, get_reg_index_of_id_ctrl0(stream->id), val);
+	} else {
+		val = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL);
+		if (dvp2axi_dev->chip_id == CHIP_EIC770X_DVP2AXI)
+			val &= ~DVP_DMA_EN;
+		if (stream->is_stop_capture) {
+			val &= ~ENABLE_CAPTURE;
+			stream->is_stop_capture = false;
+		}
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL, val);
+	}
+	stream->to_stop_dma = 0;
+	v4l2_dbg(4, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+		 "stream[%d] replace_cnt %d, y_addr 0x%x, 0x%x\n", stream->id,
+		 stream->buf_replace_cnt,
+		 es_dvp2axi_read_register(dvp2axi_dev,
+				     get_reg_index_of_frm0_y_addr(stream->id)),
+		 es_dvp2axi_read_register(dvp2axi_dev,
+				     get_reg_index_of_frm1_y_addr(stream->id)));
+	return 0;
+}
+
+static void es_dvp2axi_send_sof(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct v4l2_mbus_config *mbus = &dvp2axi_dev->active_sensor->mbus;
+	// struct csi2_dev *csi;
+
+	if (mbus->type == V4L2_MBUS_CCP2) {
+		es_dvp2axi_lvds_event_inc_sof(dvp2axi_dev);
+	} else {
+		es_dvp2axi_dvp_event_inc_sof(dvp2axi_dev);
+	}
+}
+
+static int es_dvp2axi_g_toisp_ch(unsigned int intstat_glb, int index)
+{
+	if (intstat_glb & TOISP_END_CH0(index))
+		return ES_DVP2AXI_TOISP_CH0;
+	if (intstat_glb & TOISP_END_CH1(index))
+		return ES_DVP2AXI_TOISP_CH1;
+	if (intstat_glb & TOISP_END_CH2(index))
+		return ES_DVP2AXI_TOISP_CH2;
+
+	return -EINVAL;
+}
+
+static int es_dvp2axi_g_toisp_fs(unsigned int intstat_glb, int index)
+{
+	if (intstat_glb & TOISP_FS_CH0(index))
+		return ES_DVP2AXI_TOISP_CH0;
+	if (intstat_glb & TOISP_FS_CH1(index))
+		return ES_DVP2AXI_TOISP_CH1;
+	if (intstat_glb & TOISP_FS_CH2(index))
+		return ES_DVP2AXI_TOISP_CH2;
+
+	return -EINVAL;
+}
+
+static void es_dvp2axi_toisp_check_stop_status(struct sditf_priv *priv,
+					  unsigned int intstat_glb, int index)
+{
+	int ch = 0;
+	struct es_dvp2axi_stream *stream;
+	int src_id = 0;
+	int i = 0;
+	u32 val = 0;
+	u64 cur_time = 0;
+	int on = 0;
+
+	for (i = 0; i < TOISP_CH_MAX; i++) {
+		ch = es_dvp2axi_g_toisp_ch(intstat_glb, index);
+		if (ch >= 0) {
+			src_id = priv->toisp_inf.ch_info[ch].id;
+			if (src_id == 24)
+				stream = &priv->dvp2axi_dev->stream[0];
+			else
+				stream = &priv->dvp2axi_dev->stream[src_id % 4];
+			if (stream->stopping) {
+				v4l2_dbg(3, es_dvp2axi_debug,
+					 &priv->dvp2axi_dev->v4l2_dev,
+					 "stream[%d] stop\n", stream->id);
+				es_dvp2axi_stream_stop(stream);
+				stream->stopping = false;
+				wake_up(&stream->wq_stopped);
+			}
+			if (stream->dvp2axidev->sensor_state_change) {
+				es_dvp2axi_dphy_quick_stream(stream->dvp2axidev, on);
+				stream->dvp2axidev->sensor_work.on =
+					stream->dvp2axidev->sensor_state;
+				schedule_work(
+					&stream->dvp2axidev->sensor_work.work);
+				stream->dvp2axidev->sensor_state_change = false;
+				if (stream->is_wait_stop_complete) {
+					stream->is_wait_stop_complete = false;
+					complete(&stream->stop_complete);
+				}
+			}
+			if (stream->is_single_cap &&
+			    (!stream->cur_skip_frame) &&
+			    (stream->dvp2axidev->hdr.hdr_mode == NO_HDR ||
+			     (stream->dvp2axidev->hdr.hdr_mode == HDR_X2 &&
+			      stream->id == 1) ||
+			     (stream->dvp2axidev->hdr.hdr_mode == HDR_X3 &&
+			      stream->id == 2))) {
+				es_dvp2axi_dphy_quick_stream(stream->dvp2axidev, on);
+				stream->dvp2axidev->sensor_work.on = 0;
+				schedule_work(
+					&stream->dvp2axidev->sensor_work.work);
+				stream->is_single_cap = false;
+			}
+			if (stream->cur_skip_frame &&
+			    (stream->dvp2axidev->hdr.hdr_mode == NO_HDR ||
+			     (stream->dvp2axidev->hdr.hdr_mode == HDR_X2 &&
+			      stream->id == 1) ||
+			     (stream->dvp2axidev->hdr.hdr_mode == HDR_X3 &&
+			      stream->id == 2)))
+				stream->cur_skip_frame--;
+			if (stream->dvp2axidev->rdbk_debug &&
+			    stream->frame_idx < 15)
+				v4l2_info(&priv->dvp2axi_dev->v4l2_dev,
+					  "stream[%d] toisp fe %d\n",
+					  stream->id, stream->frame_idx - 1);
+
+			switch (ch) {
+			case ES_DVP2AXI_TOISP_CH0:
+				val = TOISP_END_CH0(index);
+				intstat_glb = intstat_glb & (~val);
+				break;
+			case ES_DVP2AXI_TOISP_CH1:
+				val = TOISP_END_CH1(index);
+				intstat_glb = intstat_glb & (~val);
+				break;
+			case ES_DVP2AXI_TOISP_CH2:
+				val = TOISP_END_CH2(index);
+				intstat_glb = intstat_glb & (~val);
+				break;
+			default:
+				break;
+			}
+		}
+		ch = es_dvp2axi_g_toisp_fs(intstat_glb, index);
+		if (ch >= 0) {
+			src_id = priv->toisp_inf.ch_info[ch].id;
+			if (src_id == 24)
+				stream = &priv->dvp2axi_dev->stream[0];
+			else
+				stream = &priv->dvp2axi_dev->stream[src_id % 4];
+			if (stream->id == 0)
+				es_dvp2axi_send_sof(stream->dvp2axidev);
+			stream->frame_idx++;
+			cur_time = es_dvp2axi_time_get_ns(stream->dvp2axidev);
+			stream->readout.readout_time =
+				cur_time - stream->readout.fs_timestamp;
+			stream->readout.fs_timestamp = cur_time;
+			stream->buf_wake_up_cnt++;
+			if (stream->frame_idx % 2)
+				stream->fps_stats.frm0_timestamp =
+					es_dvp2axi_time_get_ns(stream->dvp2axidev);
+			else
+				stream->fps_stats.frm1_timestamp =
+					es_dvp2axi_time_get_ns(stream->dvp2axidev);
+			if (stream->dvp2axidev->rdbk_debug &&
+			    stream->frame_idx < 15)
+				v4l2_info(&priv->dvp2axi_dev->v4l2_dev,
+					  "stream[%d] toisp sof seq %d\n",
+					  stream->id, stream->frame_idx - 1);
+			if (stream->to_en_dma)
+				es_dvp2axi_enable_dma_capture(stream, false);
+			if (stream->to_en_scale) {
+				stream->to_en_scale = false;
+				es_dvp2axi_scale_start(stream->scale_vdev);
+			}
+			switch (ch) {
+			case ES_DVP2AXI_TOISP_CH0:
+				val = TOISP_FS_CH0(index);
+				intstat_glb = intstat_glb & (~val);
+				break;
+			case ES_DVP2AXI_TOISP_CH1:
+				val = TOISP_FS_CH1(index);
+				intstat_glb = intstat_glb & (~val);
+				break;
+			case ES_DVP2AXI_TOISP_CH2:
+				val = TOISP_FS_CH2(index);
+				intstat_glb = intstat_glb & (~val);
+				break;
+			default:
+				break;
+			}
+		}
+	}
+}
+
+void es_dvp2axi_irq_handle_toisp(struct es_dvp2axi_device *dvp2axi_dev,
+			    unsigned int intstat_glb)
+{
+	int i = 0;
+	bool to_check = false;
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+
+	if (!priv || priv->mode.rdbk_mode != ESISP_VICAP_ONLINE)
+		return;
+
+	for (i = 0; i < 2; i++) {
+		if (priv->toisp_inf.link_mode == TOISP0 && i == 0) {
+			to_check = true;
+		} else if (priv->toisp_inf.link_mode == TOISP1 && i == 1) {
+			to_check = true;
+		} else if (priv->toisp_inf.link_mode == TOISP_UNITE && i == 1) {
+			to_check = true;
+		}
+		if (to_check)
+			es_dvp2axi_toisp_check_stop_status(priv, intstat_glb, i);
+	}
+}
+
+static int es_dvp2axi_check_group_sync_state(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_stream *detect_stream = &dvp2axi_dev->stream[0];
+	struct es_dvp2axi_stream *next_stream = NULL;
+	struct es_dvp2axi_hw *hw = dvp2axi_dev->hw_dev;
+	u64 fs_interval = 0;
+	int i = 0;
+	int ret = 0;
+	struct es_dvp2axi_multi_sync_config *sync_config;
+
+	sync_config = &hw->sync_config[dvp2axi_dev->sync_cfg.group];
+	sync_config->sync_code |= BIT(dvp2axi_dev->csi_host_idx);
+	v4l2_dbg(
+		3, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+		"sync code 0x%x, mask 0x%x, update 0x%x, cache 0x%x, timestamp %llu\n",
+		sync_config->sync_code, sync_config->sync_mask,
+		sync_config->update_code, sync_config->update_cache,
+		detect_stream->readout.fs_timestamp);
+
+	if (sync_config->sync_code != sync_config->sync_mask)
+		return -EINVAL;
+
+	for (i = 0; i < sync_config->dev_cnt; i++) {
+		if (sync_config->mode == ES_DVP2AXI_MASTER_MASTER) {
+			if (i < sync_config->ext_master.count)
+				next_stream =
+					&sync_config->ext_master.dvp2axi_dev[i]
+						 ->stream[0];
+			else
+				next_stream =
+					&sync_config->int_master.dvp2axi_dev[0]
+						 ->stream[0];
+		} else if (sync_config->mode == ES_DVP2AXI_MASTER_SLAVE) {
+			if (i < sync_config->slave.count)
+				next_stream = &sync_config->slave.dvp2axi_dev[i]
+						       ->stream[0];
+			else
+				next_stream =
+					&sync_config->int_master.dvp2axi_dev[0]
+						 ->stream[0];
+		} else {
+			v4l2_err(&dvp2axi_dev->v4l2_dev,
+				 "ERROR: invalid group sync mode\n");
+			ret = -EINVAL;
+			break;
+		}
+		if (detect_stream == next_stream)
+			continue;
+		fs_interval = abs(detect_stream->readout.fs_timestamp -
+				  next_stream->readout.fs_timestamp);
+		if (fs_interval > ES_DVP2AXI_MAX_INTERVAL_NS) {
+			ret = -EINVAL;
+			break;
+		}
+	}
+	return ret;
+}
+
+static void es_dvp2axi_deal_sof(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_stream *detect_stream = &dvp2axi_dev->stream[0];
+	struct es_dvp2axi_hw *hw = dvp2axi_dev->hw_dev;
+	struct es_dvp2axi_device *tmp_dev = NULL;
+	unsigned long flags;
+	int i = 0;
+	int ret = 0;
+
+	if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI)
+		detect_stream->fs_cnt_in_single_frame++;
+	spin_lock_irqsave(&detect_stream->fps_lock, flags);
+	detect_stream->readout.fs_timestamp = es_dvp2axi_time_get_ns(dvp2axi_dev);
+	spin_unlock_irqrestore(&detect_stream->fps_lock, flags);
+
+	if (dvp2axi_dev->sync_cfg.type != ES_DVP2AXI_NOSYNC_MODE) {
+		struct es_dvp2axi_multi_sync_config *sync_config;
+
+		sync_config = &hw->sync_config[dvp2axi_dev->sync_cfg.group];
+		ret = es_dvp2axi_check_group_sync_state(dvp2axi_dev);
+		if (!ret) {
+			sync_config->sync_code = 0;
+			sync_config->frame_idx++;
+			spin_lock_irqsave(&hw->group_lock, flags);
+			sync_config->update_cache = sync_config->sync_mask;
+			if (!sync_config->update_code) {
+				sync_config->update_code =
+					sync_config->update_cache;
+				sync_config->update_cache = 0;
+			}
+			spin_unlock_irqrestore(&hw->group_lock, flags);
+			for (i = 0; i < sync_config->dev_cnt; i++) {
+				if (sync_config->mode == ES_DVP2AXI_MASTER_MASTER) {
+					if (i < sync_config->ext_master.count)
+						tmp_dev =
+							sync_config->ext_master
+								.dvp2axi_dev[i];
+					else
+						tmp_dev =
+							sync_config->int_master
+								.dvp2axi_dev[0];
+				} else if (sync_config->mode ==
+					   ES_DVP2AXI_MASTER_SLAVE) {
+					if (i < sync_config->slave.count)
+						tmp_dev = sync_config->slave
+								  .dvp2axi_dev[i];
+					else
+						tmp_dev =
+							sync_config->int_master
+								.dvp2axi_dev[0];
+				} else {
+					v4l2_err(
+						&dvp2axi_dev->v4l2_dev,
+						"ERROR: invalid group sync mode\n");
+				}
+				if (tmp_dev) {
+					es_dvp2axi_send_sof(tmp_dev);
+					tmp_dev->stream[0].frame_idx =
+						sync_config->frame_idx;
+				}
+			}
+		}
+	} else {
+		if (!dvp2axi_dev->sditf[0] || dvp2axi_dev->sditf[0]->mode.rdbk_mode) {
+			es_dvp2axi_send_sof(dvp2axi_dev);
+			detect_stream->frame_idx++;
+		}
+		if (detect_stream->dvp2axidev->rdbk_debug &&
+		    detect_stream->frame_idx < 15 &&
+		    (!dvp2axi_dev->sditf[0] || dvp2axi_dev->sditf[0]->mode.rdbk_mode))
+			v4l2_info(&dvp2axi_dev->v4l2_dev,
+				  "stream[%d] sof %d %lld\n", detect_stream->id,
+				  detect_stream->frame_idx - 1,
+				  es_dvp2axi_time_get_ns(dvp2axi_dev));
+	}
+}
+
+unsigned int es_dvp2axi_irq_global(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	unsigned int intstat_glb = 0;
+
+	intstat_glb = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_GLB_INTST);
+	if (intstat_glb)
+		v4l2_dbg(2, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "intstat_glb 0x%x\n", intstat_glb);
+	else
+		return intstat_glb;
+
+	if (intstat_glb & SCALE_TOISP_AXI0_ERR) {
+		v4l2_err(
+			&dvp2axi_dev->v4l2_dev,
+			"ERROR: scale channel, AXI0 bus err intstat_glb:0x%x !!\n",
+			intstat_glb);
+		return 0;
+	}
+	if (intstat_glb & SCALE_TOISP_AXI1_ERR) {
+		v4l2_err(
+			&dvp2axi_dev->v4l2_dev,
+			"ERROR: scale channel, AXI1 bus err intstat_glb:0x%x !!\n",
+			intstat_glb);
+		return 0;
+	}
+	es_dvp2axi_irq_handle_scale(dvp2axi_dev, intstat_glb);
+	return intstat_glb;
+}
+
+static bool es_dvp2axi_check_buffer_prepare(struct es_dvp2axi_stream *stream)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = stream->dvp2axidev;
+	unsigned long flags;
+	bool is_update = false;
+	struct es_dvp2axi_multi_sync_config *sync_config;
+
+	spin_lock_irqsave(&dvp2axi_dev->hw_dev->group_lock, flags);
+	sync_config = &dvp2axi_dev->hw_dev->sync_config[dvp2axi_dev->sync_cfg.group];
+	if (stream->id == 0 &&
+	    sync_config->update_code & BIT(dvp2axi_dev->csi_host_idx)) {
+		is_update = true;
+		sync_config->update_code &= ~(BIT(dvp2axi_dev->csi_host_idx));
+		if (!sync_config->update_code && sync_config->update_cache) {
+			sync_config->update_code = sync_config->update_cache;
+			sync_config->update_cache = 0;
+		}
+	} else {
+		if (dvp2axi_dev->rdbk_buf[RDBK_L])
+			is_update = true;
+	}
+
+	spin_unlock_irqrestore(&dvp2axi_dev->hw_dev->group_lock, flags);
+	return is_update;
+}
+
+bool es_dvp2axi_check_single_dev_stream_on(struct es_dvp2axi_hw *hw)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = NULL;
+	struct es_dvp2axi_stream *stream = NULL;
+	int i = 0, j = 0;
+	int stream_cnt = 0;
+
+	if (hw->dev_num == 1)
+		return true;
+	for (i = 0; i < hw->dev_num; i++) {
+		dvp2axi_dev = hw->dvp2axi_dev[i];
+		for (j = 0; j < ES_DVP2AXI_MAX_STREAM_MIPI; j++) {
+			stream = &dvp2axi_dev->stream[j];
+			if (stream->state == ES_DVP2AXI_STATE_STREAMING ||
+			    stream->state == ES_DVP2AXI_STATE_RESET_IN_STREAMING) {
+				stream_cnt++;
+				break;
+			}
+		}
+	}
+	if (stream_cnt > 1)
+		return false;
+	return true;
+}
+
+static void es_dvp2axi_get_resmem_head(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	// void *resmem_va = phys_to_virt(dvp2axi_dev->resmem_pa);
+	// struct esisp_thunderboot_resmem_head *head = NULL;
+	int size = 0;
+	int offset = 0;
+	int ret = 0;
+	int cam_idx = 0;
+	char cam_idx_str[3] = { 0 };
+
+	if (!dvp2axi_dev->is_rtt_suspend)
+		return;
+	strscpy(cam_idx_str, dvp2axi_dev->terminal_sensor.sd->name + 1, 2);
+	cam_idx_str[2] = '\0';
+	ret = kstrtoint(cam_idx_str, 0, &cam_idx);
+	if (ret) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev, "get camera index fail\n");
+		return;
+	}
+
+	/* currently, thunderboot with mcu only run one camera */
+	offset = 0;
+
+	if (size && size < dvp2axi_dev->resmem_size) {
+		dma_sync_single_for_cpu(dvp2axi_dev->dev,
+					dvp2axi_dev->resmem_addr + offset, size,
+					DMA_FROM_DEVICE);
+	}
+	v4l2_err(&dvp2axi_dev->v4l2_dev,
+		 "get camera index %02x, resume_mode 0x%x, nr_buf_size %d\n",
+		 cam_idx, dvp2axi_dev->resume_mode, dvp2axi_dev->nr_buf_size);
+}
+
+static int es_dvp2axi_subdevs_set_power(struct es_dvp2axi_device *dvp2axi_dev, int on)
+{
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	int ret = 0;
+	int i = 0;
+
+	if (dvp2axi_dev->terminal_sensor.sd)
+		ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, core,
+				       s_power, on);
+	if (priv && priv->is_combine_mode &&
+	    dvp2axi_dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF) {
+		for (i = 0; i < dvp2axi_dev->sditf_cnt; i++) {
+			if (dvp2axi_dev->sditf[i] && dvp2axi_dev->sditf[i]->sensor_sd)
+				v4l2_subdev_call(dvp2axi_dev->sditf[i]->sensor_sd,
+						 core, s_power, on);
+		}
+	}
+	return ret;
+}
+
+#if 0
+static void es_dvp2axi_sensor_quick_streaming_cb(void *data)
+{
+	struct v4l2_subdev *subdevs = (struct v4l2_subdev *)data;
+	int on = 1;
+
+	v4l2_subdev_call(subdevs, core, ioctl, ESMODULE_SET_QUICK_STREAM, &on);
+}
+#endif
+
+static int es_dvp2axi_subdevs_set_stream(struct es_dvp2axi_device *dvp2axi_dev, int on)
+{
+	struct es_dvp2axi_pipeline *p = &dvp2axi_dev->pipe;
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dvp2axi_dev->terminal_sensor;
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	int i = 0;
+	int ret = 0;
+
+	for (i = 0; i < p->num_subdevs; i++) {
+		if (p->subdevs[i] == terminal_sensor->sd && on)
+			es_dvp2axi_set_sof(dvp2axi_dev, dvp2axi_dev->stream[0].frame_idx);
+		ret = v4l2_subdev_call(p->subdevs[i], video, s_stream,
+				on);
+		if (ret)
+			v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				"%s:stream %s subdev:%s failed\n",
+				__func__, on ? "on" : "off",
+				p->subdevs[i]->name);
+	}
+
+	if (priv && priv->is_combine_mode &&
+	    dvp2axi_dev->sditf_cnt <= ES_DVP2AXI_MAX_SDITF) {
+		for (i = 0; i < dvp2axi_dev->sditf_cnt; i++) {
+			if (dvp2axi_dev->sditf[i] && dvp2axi_dev->sditf[i]->sensor_sd) {
+				ret = v4l2_subdev_call(
+					dvp2axi_dev->sditf[i]->sensor_sd, video,
+					s_stream, on);
+				if (ret)
+					v4l2_dbg(
+						1, es_dvp2axi_debug,
+						&dvp2axi_dev->v4l2_dev,
+						"%s:stream %s subdev:%s failed\n",
+						__func__, on ? "on" : "off",
+						dvp2axi_dev->sditf[i]
+							->sensor_sd->name);
+			}
+		}
+	}
+	return ret;
+}
+
+int es_dvp2axi_stream_suspend(struct es_dvp2axi_device *dvp2axi_dev, int mode)
+{
+	struct es_dvp2axi_stream *stream = NULL;
+	struct es_dvp2axi_resume_info *resume_info =
+		&dvp2axi_dev->reset_work.resume_info;
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	int ret = 0;
+	int i = 0;
+	int sof_cnt = 0;
+	int on = 0;
+	int suspend_cnt = 0;
+
+	mutex_lock(&dvp2axi_dev->stream_lock);
+
+	if (priv && priv->mode.rdbk_mode == ESISP_VICAP_ONLINE &&
+	    mode == ES_DVP2AXI_RESUME_DVP2AXI)
+		goto out_suspend;
+
+	for (i = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+		stream = &dvp2axi_dev->stream[i];
+
+		if (stream->state == ES_DVP2AXI_STATE_STREAMING) {
+			suspend_cnt++;
+			v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				 "stream[%d] stopping\n", stream->id);
+			if (stream->dma_en) {
+				stream->stopping = true;
+				ret = wait_event_timeout(
+					stream->wq_stopped,
+					stream->state != ES_DVP2AXI_STATE_STREAMING,
+					msecs_to_jiffies(500));
+				if (!ret) {
+					es_dvp2axi_stream_stop(stream);
+					stream->stopping = false;
+				}
+			} else {
+				es_dvp2axi_stream_stop(stream);
+			}
+
+			if (stream->id == ES_DVP2AXI_STREAM_MIPI_ID0) {
+				sof_cnt = es_dvp2axi_get_sof(dvp2axi_dev);
+				v4l2_dbg(
+					1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+					"%s: stream[%d] sync frmid & csi_sof, frm_id:%d, csi_sof:%d\n",
+					__func__, stream->id, stream->frame_idx,
+					sof_cnt);
+
+				resume_info->frm_sync_seq = stream->frame_idx;
+			}
+
+			stream->state = ES_DVP2AXI_STATE_RESET_IN_STREAMING;
+			stream->is_fs_fe_not_paired = false;
+			stream->fs_cnt_in_single_frame = 0;
+
+			v4l2_dbg(
+				1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				"%s stop stream[%d] in streaming, frm_id:%d, csi_sof:%d\n",
+				__func__, stream->id, stream->frame_idx,
+				es_dvp2axi_get_sof(dvp2axi_dev));
+		}
+	}
+
+	if (suspend_cnt == 0)
+		goto out_suspend;
+
+	if (!dvp2axi_dev->resume_mode)
+		es_dvp2axi_subdevs_set_power(dvp2axi_dev, on);
+
+	es_dvp2axi_subdevs_set_stream(dvp2axi_dev, on);
+
+out_suspend:
+	mutex_unlock(&dvp2axi_dev->stream_lock);
+	return 0;
+}
+
+int es_dvp2axi_stream_resume(struct es_dvp2axi_device *dvp2axi_dev, int mode)
+{
+	struct es_dvp2axi_stream *stream = NULL;
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	struct v4l2_subdev *sd = NULL;
+	int ret = 0;
+	int i = 0;
+	u32 capture_mode = 0;
+	int on = 1;
+	int resume_cnt = 0;
+	unsigned long flags;
+	bool is_single_dev = false;
+	bool is_can_be_online = false;
+	struct esisp_vicap_mode vicap_mode;
+
+	mutex_lock(&dvp2axi_dev->stream_lock);
+
+	es_dvp2axi_get_resmem_head(dvp2axi_dev);
+	is_single_dev = es_dvp2axi_check_single_dev_stream_on(dvp2axi_dev->hw_dev);
+	is_can_be_online = es_dvp2axi_check_can_be_online(dvp2axi_dev);
+	if (dvp2axi_dev->resume_mode == ESISP_RTT_MODE_ONE_FRAME) {
+		if (dvp2axi_dev->is_rtt_suspend) {
+			capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+			if (priv)
+				priv->mode.rdbk_mode = ESISP_VICAP_RDBK_AUTO;
+		} else {
+			if (priv &&
+			    priv->mode.rdbk_mode == ESISP_VICAP_ONLINE) {
+				capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+			} else if (priv &&
+				   (priv->mode.rdbk_mode ==
+					    ESISP_VICAP_RDBK_AUTO ||
+				    priv->mode.rdbk_mode ==
+					    ESISP_VICAP_RDBK_AUTO_ONE_FRAME)) {
+				capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+			} else {
+				capture_mode = ES_DVP2AXI_STREAM_MODE_CAPTURE;
+			}
+		}
+	} else if (dvp2axi_dev->resume_mode == ESISP_RTT_MODE_MULTI_FRAME) {
+		if (is_single_dev && is_can_be_online) {
+			capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+			if (priv)
+				priv->mode.rdbk_mode = ESISP_VICAP_ONLINE;
+		} else {
+			if (dvp2axi_dev->is_thunderboot) {
+				capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+				if (priv)
+					priv->mode.rdbk_mode =
+						ESISP_VICAP_RDBK_AUTO;
+			} else {
+				capture_mode = ES_DVP2AXI_STREAM_MODE_CAPTURE;
+				if (priv)
+					priv->mode.rdbk_mode =
+						ESISP_VICAP_RDBK_AIQ;
+			}
+		}
+	} else {
+		if (priv && priv->mode.rdbk_mode == ESISP_VICAP_ONLINE)
+			capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+		else if (priv &&
+			 (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO ||
+			  priv->mode.rdbk_mode ==
+				  ESISP_VICAP_RDBK_AUTO_ONE_FRAME))
+			capture_mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+		else
+			capture_mode = ES_DVP2AXI_STREAM_MODE_CAPTURE;
+	}
+	if (priv && priv->mode.rdbk_mode == ESISP_VICAP_ONLINE &&
+	    mode == ES_DVP2AXI_RESUME_DVP2AXI)
+		goto out_resume;
+
+	for (i = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+		stream = &dvp2axi_dev->stream[i];
+		if (stream->state != ES_DVP2AXI_STATE_RESET_IN_STREAMING)
+			continue;
+
+		stream->fs_cnt_in_single_frame = 0;
+		if (dvp2axi_dev->resume_mode == ESISP_RTT_MODE_ONE_FRAME)
+			stream->is_single_cap = true;
+		spin_lock_irqsave(&stream->vbq_lock, flags);
+		if (!priv || priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AIQ) {
+			if (stream->dvp2axi_fmt_in->field ==
+			    V4L2_FIELD_INTERLACED) {
+				if (stream->curr_buf == stream->next_buf) {
+					if (stream->curr_buf)
+						list_add_tail(
+							&stream->curr_buf->queue,
+							&stream->buf_head);
+				} else {
+					if (stream->curr_buf)
+						list_add_tail(
+							&stream->curr_buf->queue,
+							&stream->buf_head);
+					if (stream->next_buf)
+						list_add_tail(
+							&stream->next_buf->queue,
+							&stream->buf_head);
+				}
+				stream->curr_buf = NULL;
+				stream->next_buf = NULL;
+			}
+		} else {
+			if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE) {
+				if (stream->curr_buf_toisp ==
+				    stream->next_buf_toisp) {
+					if (stream->curr_buf_toisp)
+						list_add_tail(
+							&stream->curr_buf_toisp
+								 ->list,
+							&stream->rx_buf_head);
+				} else {
+					if (stream->curr_buf_toisp)
+						list_add_tail(
+							&stream->curr_buf_toisp
+								 ->list,
+							&stream->rx_buf_head);
+					if (stream->next_buf_toisp)
+						list_add_tail(
+							&stream->next_buf_toisp
+								 ->list,
+							&stream->rx_buf_head);
+				}
+				stream->curr_buf_toisp = NULL;
+				stream->next_buf_toisp = NULL;
+			} else {
+				if (stream->curr_buf_toisp ==
+				    stream->next_buf_toisp) {
+					if (stream->curr_buf_toisp)
+						list_add_tail(
+							&stream->curr_buf_toisp
+								 ->list,
+							&stream->rx_buf_head);
+				} else {
+					if (stream->curr_buf_toisp)
+						list_add_tail(
+							&stream->curr_buf_toisp
+								 ->list,
+							&stream->rx_buf_head);
+					if (stream->next_buf_toisp)
+						list_add_tail(
+							&stream->next_buf_toisp
+								 ->list,
+							&stream->rx_buf_head);
+				}
+				stream->curr_buf_toisp = NULL;
+				stream->next_buf_toisp = NULL;
+			}
+		}
+
+		spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+		if (priv) {
+			if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE) {
+				sditf_change_to_online(priv);
+				if (dvp2axi_dev->resume_mode ==
+					    ESISP_RTT_MODE_MULTI_FRAME &&
+				    stream->rx_buf_num &&
+				    (priv->hdr_cfg.hdr_mode == NO_HDR ||
+				     (priv->hdr_cfg.hdr_mode == HDR_X2 &&
+				      stream->id == 1) ||
+				     (priv->hdr_cfg.hdr_mode == HDR_X3 &&
+				      stream->id == 2)))
+					es_dvp2axi_free_rx_buf(stream,
+							  priv->buf_num);
+				else if (!stream->rx_buf_num &&
+					 ((priv->hdr_cfg.hdr_mode == HDR_X2 &&
+					   stream->id == 0) ||
+					  (priv->hdr_cfg.hdr_mode == HDR_X3 &&
+					   (stream->id == 0 ||
+					    stream->id == 1))))
+					es_dvp2axi_init_rx_buf(stream, 1);
+			} else {
+				if (stream->is_single_cap && stream->id == 0) {
+					vicap_mode = priv->mode;
+					vicap_mode.rdbk_mode =
+						ESISP_VICAP_RDBK_AUTO_ONE_FRAME;
+					sd = get_esisp_sd(priv);
+					if (sd) {
+						ret = v4l2_subdev_call(
+							sd, core, ioctl,
+							ESISP_VICAP_CMD_MODE,
+							&vicap_mode);
+						if (ret)
+							v4l2_err(
+								&dvp2axi_dev->v4l2_dev,
+								"set isp work mode rdbk aotu oneframe fail\n");
+					}
+				}
+				sditf_disable_immediately(priv);
+				if (!stream->rx_buf_num &&
+				    capture_mode ==
+					    ES_DVP2AXI_STREAM_MODE_TOISP_RDBK) {
+					if (dvp2axi_dev->resume_mode ==
+					    ESISP_RTT_MODE_ONE_FRAME)
+						es_dvp2axi_init_rx_buf(stream, 1);
+					else
+						es_dvp2axi_init_rx_buf(
+							stream, priv->buf_num);
+				}
+			}
+		}
+
+		stream->lack_buf_cnt = 0;
+		if (dvp2axi_dev->active_sensor &&
+		    (dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+		     dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ||
+		     dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CCP2))
+			ret = es_dvp2axi_csi_stream_start(stream, capture_mode);
+		else
+			ret = es_dvp2axi_stream_start(stream, capture_mode);
+		if (ret)
+			v4l2_err(&dvp2axi_dev->v4l2_dev,
+				 "%s:resume stream[%d] failed\n", __func__,
+				 stream->id);
+
+		resume_cnt++;
+		stream->cur_skip_frame = stream->skip_frame;
+		v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "resume stream[%d], frm_idx:%d, csi_sof:%d\n",
+			 stream->id, stream->frame_idx, es_dvp2axi_get_sof(dvp2axi_dev));
+	}
+
+	if (resume_cnt == 0)
+		goto out_resume;
+
+	if (!dvp2axi_dev->resume_mode)
+		es_dvp2axi_subdevs_set_power(dvp2axi_dev, on);
+
+	atomic_set(&dvp2axi_dev->streamoff_cnt, 0);
+	es_dvp2axi_subdevs_set_stream(dvp2axi_dev, on);
+
+	if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI)
+		es_dvp2axi_start_luma(
+			&dvp2axi_dev->luma_vdev,
+			dvp2axi_dev->stream[ES_DVP2AXI_STREAM_MIPI_ID0].dvp2axi_fmt_in);
+out_resume:
+	mutex_unlock(&dvp2axi_dev->stream_lock);
+	return 0;
+}
+
+void es_dvp2axi_err_print_work(struct work_struct *work)
+{
+	struct es_dvp2axi_err_state_work *err_state_work =
+		container_of(work, struct es_dvp2axi_err_state_work, work);
+	struct es_dvp2axi_device *dev = container_of(
+		err_state_work, struct es_dvp2axi_device, err_state_work);
+	u32 err_state = 0;
+	int intstat = 0;
+	int lastline = 0;
+	int lastpixel = 0;
+	u64 cur_time = 0;
+	bool is_print = false;
+
+	cur_time = es_dvp2axi_time_get_ns(dev);
+	if (err_state_work->last_timestamp == 0) {
+		is_print = true;
+	} else {
+		if (cur_time - err_state_work->last_timestamp > 500000000)
+			is_print = true;
+	}
+	err_state_work->last_timestamp = cur_time;
+	err_state = err_state_work->err_state;
+	intstat = err_state_work->intstat;
+	lastline = err_state_work->lastline;
+	lastpixel = err_state_work->lastpixel;
+	if (err_state & ES_DVP2AXI_ERR_ID0_NOT_BUF && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[0] not active buffer, frame num %d, cnt %llu\n",
+			dev->stream[0].frame_idx,
+			dev->irq_stats.not_active_buf_cnt[0]);
+	if (err_state & ES_DVP2AXI_ERR_ID1_NOT_BUF && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[1] not active buffer, frame num %d, cnt %llu\n",
+			dev->stream[1].frame_idx,
+			dev->irq_stats.not_active_buf_cnt[1]);
+	if (err_state & ES_DVP2AXI_ERR_ID2_NOT_BUF && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[2] not active buffer, frame num %d, cnt %llu\n",
+			dev->stream[2].frame_idx,
+			dev->irq_stats.not_active_buf_cnt[2]);
+	if (err_state & ES_DVP2AXI_ERR_ID3_NOT_BUF && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[3] not active buffer, frame num %d, cnt %llu\n",
+			dev->stream[3].frame_idx,
+			dev->irq_stats.not_active_buf_cnt[3]);
+	if (err_state & ES_DVP2AXI_ERR_ID0_TRIG_SIMULT && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[0], frm0/frm1 end simultaneously,frm id:%d, cnt %llu\n",
+			dev->stream[0].frame_idx,
+			dev->irq_stats.trig_simult_cnt[0]);
+	if (err_state & ES_DVP2AXI_ERR_ID1_TRIG_SIMULT && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[1], frm0/frm1 end simultaneously,frm id:%d, cnt %llu\n",
+			dev->stream[1].frame_idx,
+			dev->irq_stats.trig_simult_cnt[1]);
+	if (err_state & ES_DVP2AXI_ERR_ID2_TRIG_SIMULT && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[2], frm0/frm1 end simultaneously,frm id:%d, cnt %llu\n",
+			dev->stream[2].frame_idx,
+			dev->irq_stats.trig_simult_cnt[2]);
+	if (err_state & ES_DVP2AXI_ERR_ID3_TRIG_SIMULT && is_print)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"stream[3], frm0/frm1 end simultaneously,frm id:%d, cnt %llu\n",
+			dev->stream[3].frame_idx,
+			dev->irq_stats.trig_simult_cnt[3]);
+	if (err_state & ES_DVP2AXI_ERR_SIZE) {
+		if (dev->chip_id >= CHIP_EIC770X_DVP2AXI)
+			v4l2_err(
+				&dev->v4l2_dev,
+				"ERROR: csi size err, intstat:0x%x, size:0x%x,0x%x,0x%x,0x%x, cnt %llu\n",
+				intstat, err_state_work->size_id0,
+				err_state_work->size_id1,
+				err_state_work->size_id2,
+				err_state_work->size_id3,
+				dev->irq_stats.csi_size_err_cnt);
+		else
+			v4l2_err(
+				&dev->v4l2_dev,
+				"ERROR: csi size err, intstat:0x%x, lastline:0x%x, cnt %llu\n",
+				intstat, lastline,
+				dev->irq_stats.csi_size_err_cnt);
+	}
+	if (err_state & ES_DVP2AXI_ERR_OVERFLOW)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"ERROR: csi fifo overflow, intstat:0x%x, lastline:0x%x, cnt %llu\n",
+			intstat, lastline, dev->irq_stats.csi_overflow_cnt);
+	if (err_state & ES_DVP2AXI_ERR_BANDWIDTH_LACK)
+		v4l2_err(
+			&dev->v4l2_dev,
+			"ERROR: csi bandwidth lack, intstat:0x%x, lastline:0x%x, cnt %llu\n",
+			intstat, lastline, dev->irq_stats.csi_bwidth_lack_cnt);
+	if (err_state & ES_DVP2AXI_ERR_ID0_MULTI_FS)
+		v4l2_err(&dev->v4l2_dev,
+			 "ERR: multi fs in oneframe in id0, fs_num:%u\n",
+			 dev->stream[0].fs_cnt_in_single_frame);
+	if (err_state & ES_DVP2AXI_ERR_BUS)
+		v4l2_err(&dev->v4l2_dev,
+			 "dvp bus err, intstat 0x%x, last line 0x%x\n", intstat,
+			 lastline);
+	if (err_state & ES_DVP2AXI_ERR_PIXEL)
+		v4l2_err(&dev->v4l2_dev,
+			 "dvp pix err, intstat 0x%x, last pixel 0x%x\n",
+			 intstat, lastpixel);
+	if (err_state & ES_DVP2AXI_ERR_LINE)
+		v4l2_err(&dev->v4l2_dev,
+			 "dvp line err, intstat 0x%x, last line 0x%x\n",
+			 intstat, lastline);
+}
+
+/* pingpong irq for eic770x and next */
+void es_dvp2axi_irq_pingpong_v1(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_stream *stream;
+	struct es_dvp2axi_stream *detect_stream = &dvp2axi_dev->stream[0];
+	struct v4l2_mbus_config *mbus;
+	unsigned int intstat, i = 0xff;
+	unsigned long flags;
+	bool is_update = false;
+	int ret = 0;
+	int on = 0;
+
+	if (!dvp2axi_dev->active_sensor)
+		return;
+
+	mbus = &dvp2axi_dev->active_sensor->mbus;
+	if (mbus->type == V4L2_MBUS_CSI2_DPHY ||
+	    mbus->type == V4L2_MBUS_CSI2_CPHY || mbus->type == V4L2_MBUS_CCP2) {
+		int mipi_id;
+		u32 lastline = 0;
+
+		intstat =
+			es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTSTAT);
+		lastline = es_dvp2axi_read_register(
+			dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_LINE_LINE_CNT_ID0_1);
+		dvp2axi_dev->err_state_work.lastline = lastline;
+		dvp2axi_dev->err_state_work.intstat = intstat;
+
+		/* clear all interrupts that has been triggered */
+		if (intstat) {
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTSTAT,
+					     intstat);
+			v4l2_dbg(2, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+				 "intstat 0x%x\n", intstat);
+		} else {
+			return;
+		}
+
+		if (intstat & CSI_SIZE_ERR) {
+			if (dvp2axi_dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+				dvp2axi_dev->err_state_work.size_id0 =
+					es_dvp2axi_read_register(
+						dvp2axi_dev,
+						DVP2AXI_REG_MIPI_FRAME_SIZE_ID0);
+				dvp2axi_dev->err_state_work.size_id1 =
+					es_dvp2axi_read_register(
+						dvp2axi_dev,
+						DVP2AXI_REG_MIPI_FRAME_SIZE_ID1);
+				dvp2axi_dev->err_state_work.size_id2 =
+					es_dvp2axi_read_register(
+						dvp2axi_dev,
+						DVP2AXI_REG_MIPI_FRAME_SIZE_ID2);
+				dvp2axi_dev->err_state_work.size_id3 =
+					es_dvp2axi_read_register(
+						dvp2axi_dev,
+						DVP2AXI_REG_MIPI_FRAME_SIZE_ID3);
+			}
+			dvp2axi_dev->irq_stats.csi_size_err_cnt++;
+			dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_SIZE;
+			es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_CTRL,
+						0x000A0000);
+			return;
+		}
+
+		if (intstat & CSI_FIFO_OVERFLOW_V1) {
+			dvp2axi_dev->irq_stats.csi_overflow_cnt++;
+			dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_OVERFLOW;
+			return;
+		}
+
+		if (intstat & CSI_BANDWIDTH_LACK_V1 &&
+		    dvp2axi_dev->intr_mask & CSI_BANDWIDTH_LACK_V1) {
+			dvp2axi_dev->irq_stats.csi_bwidth_lack_cnt++;
+			dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_BANDWIDTH_LACK;
+			if (dvp2axi_dev->irq_stats.csi_bwidth_lack_cnt > 10) {
+				es_dvp2axi_write_register_and(
+					dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+					~(CSI_BANDWIDTH_LACK_V1));
+				dvp2axi_dev->intr_mask &= ~(CSI_BANDWIDTH_LACK_V1);
+				schedule_delayed_work(&dvp2axi_dev->work_deal_err,
+						      msecs_to_jiffies(1000));
+			}
+		}
+
+		if (intstat & CSI_ALL_ERROR_INTEN_V1) {
+			dvp2axi_dev->irq_stats.all_err_cnt++;
+			return;
+		}
+
+		for (i = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+			mipi_id = es_dvp2axi_csi_g_mipi_id(&dvp2axi_dev->v4l2_dev,
+						      intstat);
+			if (mipi_id < 0)
+				continue;
+
+			stream = &dvp2axi_dev->stream[mipi_id];
+			if (!dvp2axi_dev->sditf[0] ||
+			    dvp2axi_dev->sditf[0]->mode.rdbk_mode)
+				stream->buf_wake_up_cnt++;
+
+			if (stream->stopping && (!stream->dma_en)) {
+				es_dvp2axi_stream_stop(stream);
+				stream->stopping = false;
+				wake_up(&stream->wq_stopped);
+				continue;
+			}
+
+			if (stream->state != ES_DVP2AXI_STATE_STREAMING)
+				continue;
+			stream->is_in_vblank = true;
+			switch (mipi_id) {
+			case ES_DVP2AXI_STREAM_MIPI_ID0:
+				stream->frame_phase = SW_FRM_END_ID0(intstat);
+				intstat &= ~CSI_FRAME_END_ID0;
+				break;
+			case ES_DVP2AXI_STREAM_MIPI_ID1:
+				stream->frame_phase = SW_FRM_END_ID1(intstat);
+				intstat &= ~CSI_FRAME_END_ID1;
+				break;
+			case ES_DVP2AXI_STREAM_MIPI_ID2:
+				stream->frame_phase = SW_FRM_END_ID2(intstat);
+				intstat &= ~CSI_FRAME_END_ID2;
+				break;
+			case ES_DVP2AXI_STREAM_MIPI_ID3:
+				stream->frame_phase = SW_FRM_END_ID3(intstat);
+				intstat &= ~CSI_FRAME_END_ID3;
+				break;
+			}
+			if (stream->dvp2axidev->rdbk_debug &&
+			    stream->frame_idx < 15 &&
+			    (!dvp2axi_dev->sditf[0] ||
+			     dvp2axi_dev->sditf[0]->mode.rdbk_mode))
+				v4l2_info(&dvp2axi_dev->v4l2_dev,
+					  "stream[%d] fe %d, phase %d, %lld\n",
+					  stream->id, stream->frame_idx - 1,
+					  stream->frame_phase,
+					  es_dvp2axi_time_get_ns(dvp2axi_dev));
+			if (stream->is_finish_stop_dma &&
+			    stream->is_wait_dma_stop) {
+				stream->is_wait_dma_stop = false;
+				wake_up(&stream->wq_stopped);
+				stream->is_finish_stop_dma = false;
+				continue;
+			}
+
+			if (stream->is_finish_stop_dma &&
+			    stream->is_wait_stop_complete) {
+				stream->is_finish_stop_dma = false;
+				stream->is_wait_stop_complete = false;
+				complete(&stream->stop_complete);
+			}
+
+			if (stream->crop_dyn_en)
+				es_dvp2axi_dynamic_crop(stream);
+
+			if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_VICAP) {
+				if (dvp2axi_dev->sync_cfg.type == ES_DVP2AXI_NOSYNC_MODE)
+					is_update = true;
+				else
+					is_update = es_dvp2axi_check_buffer_prepare(
+						stream);
+				v4l2_dbg(
+					4, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+					"dma capture by vicap, is_updata %d, group mode %d, dma_en %d\n",
+					is_update, dvp2axi_dev->sync_cfg.type,
+					stream->dma_en);
+				if (is_update)
+					es_dvp2axi_update_stream(dvp2axi_dev, stream,
+							    mipi_id);
+			} else if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP) {
+				v4l2_dbg(4, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+					 "dma capture by isp, dma_en 0x%x\n",
+					 stream->dma_en);
+				es_dvp2axi_update_stream_toisp(dvp2axi_dev, stream,
+							  mipi_id);
+			} 
+			// else if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ESKIT) {
+			// 	v4l2_dbg(4, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			// 		 "dma capture by eskit, dma_en 0x%x\n",
+			// 		 stream->dma_en);
+			// 	es_dvp2axi_update_stream_eskit(dvp2axi_dev, stream,
+			// 				   mipi_id);
+			// }
+			if (stream->is_single_cap && !stream->cur_skip_frame) {
+				if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP)
+					stream->to_stop_dma =
+						ES_DVP2AXI_DMAEN_BY_ISP;
+				else if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_VICAP)
+					stream->to_stop_dma =
+						ES_DVP2AXI_DMAEN_BY_VICAP;
+				else if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ESKIT)
+					stream->to_stop_dma =
+						ES_DVP2AXI_DMAEN_BY_ESKIT;
+				es_dvp2axi_stop_dma_capture(stream);
+				stream->is_single_cap = false;
+				if ((dvp2axi_dev->hdr.hdr_mode == NO_HDR &&
+				     atomic_read(&dvp2axi_dev->streamoff_cnt) ==
+					     1) ||
+				    (dvp2axi_dev->hdr.hdr_mode == HDR_X2 &&
+				     atomic_read(&dvp2axi_dev->streamoff_cnt) ==
+					     2) ||
+				    (dvp2axi_dev->hdr.hdr_mode == HDR_X3 &&
+				     atomic_read(&dvp2axi_dev->streamoff_cnt) ==
+					     3)) {
+					es_dvp2axi_dphy_quick_stream(stream->dvp2axidev,
+								on);
+					dvp2axi_dev->sensor_work.on = 0;
+					schedule_work(
+						&dvp2axi_dev->sensor_work.work);
+				}
+			}
+
+			if (stream->cur_skip_frame)
+				stream->cur_skip_frame--;
+
+			if (stream->is_change_toisp) {
+				stream->is_change_toisp = false;
+				if ((dvp2axi_dev->hdr.hdr_mode == HDR_X2 &&
+				     stream->id != 1) ||
+				    (dvp2axi_dev->hdr.hdr_mode == HDR_X3 &&
+				     stream->id != 2))
+					es_dvp2axi_release_unnecessary_buf_for_online(
+						stream, stream->curr_buf_toisp);
+				else
+					sditf_change_to_online(
+						dvp2axi_dev->sditf[0]);
+			}
+
+			spin_lock_irqsave(&stream->vbq_lock, flags);
+			if (!(stream->cur_stream_mode &
+			      ES_DVP2AXI_STREAM_MODE_TOISP) &&
+			    stream->lack_buf_cnt == 2) {
+				stream->to_stop_dma = ES_DVP2AXI_DMAEN_BY_VICAP;
+				es_dvp2axi_stop_dma_capture(stream);
+				v4l2_dbg(
+					4, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+					"stream[%d] to stop dma, lack_buf_cnt %d\n",
+					stream->id, stream->lack_buf_cnt);
+			}
+			spin_unlock_irqrestore(&stream->vbq_lock, flags);
+			if (stream->to_en_scale) {
+				stream->to_en_scale = false;
+				es_dvp2axi_scale_start(stream->scale_vdev);
+			}
+			es_dvp2axi_detect_wake_up_mode_change(stream);
+			if (dvp2axi_dev->chip_id < CHIP_EIC770X_DVP2AXI &&
+			    mipi_id == ES_DVP2AXI_STREAM_MIPI_ID0) {
+				if ((intstat & (CSI_FRAME1_START_ID0 |
+						CSI_FRAME0_START_ID0)) == 0 &&
+				    detect_stream->fs_cnt_in_single_frame > 1) {
+					dvp2axi_dev->err_state |=
+						ES_DVP2AXI_ERR_ID0_MULTI_FS;
+					detect_stream->is_fs_fe_not_paired =
+						true;
+					detect_stream->fs_cnt_in_single_frame =
+						0;
+				} else {
+					detect_stream->fs_cnt_in_single_frame--;
+				}
+			}
+			es_dvp2axi_monitor_reset_event(dvp2axi_dev);
+			dvp2axi_dev->irq_stats.frm_end_cnt[stream->id]++;
+		}
+
+		for (i = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+			if (intstat & CSI_START_INTSTAT(i)) {
+				stream = &dvp2axi_dev->stream[i];
+				if (i == 0) {
+					if (!stream->cur_skip_frame)
+						es_dvp2axi_deal_sof(dvp2axi_dev);
+				} else {
+					spin_lock_irqsave(&stream->fps_lock,
+							  flags);
+					stream->readout.fs_timestamp =
+						es_dvp2axi_time_get_ns(dvp2axi_dev);
+					stream->frame_idx++;
+					spin_unlock_irqrestore(
+						&stream->fps_lock, flags);
+				}
+				stream->is_in_vblank = false;
+				spin_lock_irqsave(&stream->vbq_lock, flags);
+				if (stream->stopping && stream->dma_en) {
+					if (stream->dma_en &
+					    ES_DVP2AXI_DMAEN_BY_VICAP)
+						stream->to_stop_dma =
+							ES_DVP2AXI_DMAEN_BY_VICAP;
+					else if (stream->dma_en &
+						 ES_DVP2AXI_DMAEN_BY_ISP)
+						stream->to_stop_dma =
+							ES_DVP2AXI_DMAEN_BY_ISP;
+					stream->is_stop_capture = true;
+				}
+				if (stream->to_stop_dma) {
+					ret = es_dvp2axi_stop_dma_capture(stream);
+					if (!ret)
+						stream->is_finish_stop_dma =
+							true;
+				}
+				if (stream->to_en_dma)
+					es_dvp2axi_enable_dma_capture(stream, false);
+				spin_unlock_irqrestore(&stream->vbq_lock,
+						       flags);
+			}
+			if (intstat & CSI_LINE_INTSTAT_V1(i)) {
+				stream = &dvp2axi_dev->stream[i];
+				if (stream->is_line_inten) {
+					stream->line_int_cnt++;
+					if (dvp2axi_dev->rdbk_debug > 1 &&
+					    stream->frame_idx < 15)
+						v4l2_info(&dvp2axi_dev->v4l2_dev,
+							  "line int %lld\n",
+							  stream->line_int_cnt);
+					if (dvp2axi_dev->sditf[0] &&
+					    (dvp2axi_dev->sditf[0]->mode.rdbk_mode ==
+						     ESISP_VICAP_RDBK_AUTO ||
+					     dvp2axi_dev->sditf[0]->mode.rdbk_mode ==
+						     ESISP_VICAP_RDBK_AUTO_ONE_FRAME))
+						es_dvp2axi_line_wake_up_rdbk(
+							stream, stream->id);
+					else
+						es_dvp2axi_line_wake_up(stream,
+								   stream->id);
+					es_dvp2axi_modify_line_int(stream, false);
+					stream->is_line_inten = false;
+				}
+				v4l2_dbg(3, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+					 "%s: id0 cur line:%d\n", __func__,
+					 lastline & 0x3fff);
+			}
+		}
+	} else {
+		struct es_dvp2axi_stream *stream;
+		int ch_id;
+		int lastline;
+
+		intstat = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT);
+		if (intstat)
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT,
+					     intstat);
+		else
+			return;
+		lastline = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_LINE_CNT);
+		dvp2axi_dev->err_state_work.lastline = lastline;
+		dvp2axi_dev->err_state_work.intstat = intstat;
+		stream = &dvp2axi_dev->stream[ES_DVP2AXI_STREAM_DVP2AXI];
+
+		if (intstat & DVP_SIZE_ERR) {
+			dvp2axi_dev->irq_stats.dvp_size_err_cnt++;
+			es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL,
+						0x000A0000);
+			dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_SIZE;
+		}
+
+		if (intstat & DVP_FIFO_OVERFLOW) {
+			dvp2axi_dev->irq_stats.dvp_overflow_cnt++;
+			dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_OVERFLOW;
+		}
+
+		if (intstat & DVP_BANDWIDTH_LACK) {
+			dvp2axi_dev->irq_stats.dvp_bwidth_lack_cnt++;
+			dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_BANDWIDTH_LACK;
+		}
+
+		if (intstat & INTSTAT_ERR_EIC770X) {
+			dvp2axi_dev->irq_stats.all_err_cnt++;
+		}
+
+		for (i = 0; i < ES_DVP2AXI_MAX_STREAM_DVP; i++) {
+			ch_id = es_dvp2axi_dvp_g_ch_id_by_fe(&dvp2axi_dev->v4l2_dev,
+							intstat);
+
+			if (ch_id < 0)
+				continue;
+
+			stream = &dvp2axi_dev->stream[ch_id];
+			if (!dvp2axi_dev->sditf[0] ||
+			    dvp2axi_dev->sditf[0]->mode.rdbk_mode)
+				stream->buf_wake_up_cnt++;
+
+			if (stream->stopping) {
+				es_dvp2axi_stream_stop(stream);
+				stream->stopping = false;
+				wake_up(&stream->wq_stopped);
+				continue;
+			}
+
+			if (stream->state != ES_DVP2AXI_STATE_STREAMING)
+				continue;
+			stream->is_in_vblank = true;
+			switch (ch_id) {
+			case ES_DVP2AXI_STREAM_MIPI_ID0:
+				stream->frame_phase = SW_FRM_END_ID0(intstat);
+				intstat &= ~DVP_ALL_END_ID0;
+				break;
+			case ES_DVP2AXI_STREAM_MIPI_ID1:
+				stream->frame_phase = SW_FRM_END_ID1(intstat);
+				intstat &= ~DVP_ALL_END_ID1;
+				break;
+			case ES_DVP2AXI_STREAM_MIPI_ID2:
+				stream->frame_phase = SW_FRM_END_ID2(intstat);
+				intstat &= ~DVP_ALL_END_ID2;
+				break;
+			case ES_DVP2AXI_STREAM_MIPI_ID3:
+				stream->frame_phase = SW_FRM_END_ID3(intstat);
+				intstat &= ~DVP_ALL_END_ID3;
+				break;
+			}
+			if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_VICAP) {
+				if (dvp2axi_dev->sync_cfg.type == ES_DVP2AXI_NOSYNC_MODE)
+					is_update = true;
+				else
+					is_update = es_dvp2axi_check_buffer_prepare(
+						stream);
+				if (is_update)
+					es_dvp2axi_update_stream(dvp2axi_dev, stream,
+							    ch_id);
+			} else if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ISP) {
+				es_dvp2axi_update_stream_toisp(dvp2axi_dev, stream,
+							  ch_id);
+			} 
+			// else if (stream->dma_en & ES_DVP2AXI_DMAEN_BY_ESKIT) {
+			// 	es_dvp2axi_update_stream_eskit(dvp2axi_dev, stream,
+			// 				   ch_id);
+			// }
+
+			if (stream->to_en_dma)
+				es_dvp2axi_enable_dma_capture(stream, false);
+			if (stream->to_stop_dma) {
+				es_dvp2axi_stop_dma_capture(stream);
+				wake_up(&stream->wq_stopped);
+			}
+			dvp2axi_dev->irq_stats.frm_end_cnt[stream->id]++;
+		}
+
+		if (intstat & DVP_FRAME0_START_ID0 ||
+		    intstat & DVP_FRAME1_START_ID0) {
+			stream->is_in_vblank = false;
+			if (!stream->cur_skip_frame)
+				es_dvp2axi_deal_sof(dvp2axi_dev);
+		}
+
+		if (stream->crop_dyn_en)
+			es_dvp2axi_dynamic_crop(stream);
+	}
+}
+
+void es_dvp2axi_irq_pingpong(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_stream *stream;
+	// struct es_dvp2axi_stream *detect_stream = &dvp2axi_dev->stream[0];
+	struct v4l2_mbus_config *mbus;
+	unsigned int intstat = 0x0, i = 0xff;
+	// unsigned long flags;
+	// int ret = 0;
+
+	if (!dvp2axi_dev->active_sensor)
+		return;
+
+	mbus = &dvp2axi_dev->active_sensor->mbus;
+
+	u32 lastline, lastpix, ctl;
+	u32 dvp2axi_frmst, frmid, int_en;
+	int ch_id;
+
+	intstat = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT);
+	dvp2axi_frmst =
+		es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_FRAME_STATUS);
+	lastline = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_LAST_LINE);
+	lastline = DVP2AXI_FETCH_Y_LAST_LINE(lastline);
+	lastpix = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_LAST_PIX);
+	lastpix = DVP2AXI_FETCH_Y_LAST_LINE(lastpix);
+	ctl = es_dvp2axi_read_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL);
+	dvp2axi_dev->err_state_work.lastline = lastline;
+	dvp2axi_dev->err_state_work.lastpixel = lastpix;
+	dvp2axi_dev->err_state_work.intstat = intstat;
+
+	es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_INTSTAT, intstat);
+
+	stream = &dvp2axi_dev->stream[ES_DVP2AXI_STREAM_DVP2AXI];
+
+	if (intstat & BUS_ERR) {
+		dvp2axi_dev->irq_stats.dvp_bus_err_cnt++;
+		dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_BUS;
+	}
+
+	if (intstat & DVP_ALL_OVERFLOW) {
+		dvp2axi_dev->irq_stats.dvp_overflow_cnt++;
+		dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_OVERFLOW;
+	}
+
+	if (intstat & LINE_ERR) {
+		dvp2axi_dev->irq_stats.dvp_line_err_cnt++;
+		dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_LINE;
+	}
+
+	if (intstat & PIX_ERR) {
+		dvp2axi_dev->irq_stats.dvp_pix_err_cnt++;
+		dvp2axi_dev->err_state |= ES_DVP2AXI_ERR_PIXEL;
+	}
+
+	if (intstat & INTSTAT_ERR)
+		dvp2axi_dev->irq_stats.all_err_cnt++;
+
+	/* There are two irqs enabled:
+		*  - PST_INF_FRAME_END: dvp2axi FIFO is ready,
+		*    this is prior to FRAME_END
+		*  - FRAME_END: dvp2axi has saved frame to memory,
+		*    a frame ready
+		*/
+	if ((intstat & PST_INF_FRAME_END)) {
+		stream = &dvp2axi_dev->stream[ES_DVP2AXI_STREAM_DVP2AXI];
+		if (stream->stopping)
+			/* To stop DVP2AXI ASAP, before FRAME_END irq */
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_DVP_CTRL,
+							ctl & (~ENABLE_CAPTURE));
+	}
+
+	for (i = 0; i < ES_DVP2AXI_MAX_STREAM_DVP; i++) {
+		ch_id = es_dvp2axi_dvp_g_ch_id(&dvp2axi_dev->v4l2_dev,
+						&intstat, dvp2axi_frmst);
+
+		if (ch_id < 0)
+			continue;
+
+		if (ch_id == ES_DVP2AXI_STREAM_MIPI_ID0) {
+			int_en = es_dvp2axi_read_register(
+				dvp2axi_dev, DVP2AXI_REG_DVP_INTEN);
+			int_en |= LINE_INT_EN;
+			es_dvp2axi_write_register(dvp2axi_dev,
+							DVP2AXI_REG_DVP_INTEN,
+							int_en);
+			dvp2axi_dev->dvp_sof_in_oneframe = 0;
+		}
+
+		stream = &dvp2axi_dev->stream[ch_id];
+
+		if (stream->stopping) {
+			es_dvp2axi_stream_stop(stream);
+			stream->stopping = false;
+			wake_up(&stream->wq_stopped);
+			continue;
+		}
+
+		if (stream->state != ES_DVP2AXI_STATE_STREAMING)
+			continue;
+
+		switch (ch_id) {
+		case ES_DVP2AXI_STREAM_MIPI_ID0:
+			stream->frame_phase =
+				DVP_FRM_STS_ID0(dvp2axi_frmst);
+			break;
+		case ES_DVP2AXI_STREAM_MIPI_ID1:
+			stream->frame_phase =
+				DVP_FRM_STS_ID1(dvp2axi_frmst);
+			break;
+		case ES_DVP2AXI_STREAM_MIPI_ID2:
+			stream->frame_phase =
+				DVP_FRM_STS_ID2(dvp2axi_frmst);
+			break;
+		case ES_DVP2AXI_STREAM_MIPI_ID3:
+			stream->frame_phase =
+				DVP_FRM_STS_ID3(dvp2axi_frmst);
+			break;
+		}
+
+		frmid = DVP2AXI_GET_FRAME_ID(dvp2axi_frmst);
+		if ((frmid == 0xfffd) || (frmid == 0xfffe)) {
+			v4l2_info(&dvp2axi_dev->v4l2_dev,
+					"frmid:%d, frmstat:0x%x\n",
+					frmid, dvp2axi_frmst);
+			es_dvp2axi_write_register(
+				dvp2axi_dev,
+				DVP2AXI_REG_DVP_FRAME_STATUS,
+				FRAME_STAT_CLS);
+		}
+		es_dvp2axi_update_stream(dvp2axi_dev, stream, ch_id);
+		dvp2axi_dev->irq_stats.frm_end_cnt[stream->id]++;
+	}
+
+	if ((intstat & LINE_INT_END) && !(intstat & FRAME_END) &&
+		(dvp2axi_dev->dvp_sof_in_oneframe == 0)) {
+		if ((intstat &
+				(PRE_INF_FRAME_END | PST_INF_FRAME_END)) == 0x0) {
+			if ((intstat & INTSTAT_ERR) == 0x0) {
+				if (!stream->cur_skip_frame)
+					es_dvp2axi_deal_sof(dvp2axi_dev);
+				int_en = es_dvp2axi_read_register(
+					dvp2axi_dev, DVP2AXI_REG_DVP_INTEN);
+				int_en &= ~LINE_INT_EN;
+				es_dvp2axi_write_register(dvp2axi_dev,
+								DVP2AXI_REG_DVP_INTEN,
+								int_en);
+				dvp2axi_dev->dvp_sof_in_oneframe = 1;
+			}
+		}
+	}
+
+	if (stream->crop_dyn_en)
+		es_dvp2axi_dynamic_crop(stream);
+}
+
+int es_dvp2axi_sditf_disconnect(struct video_device *vdev)
+{
+	struct es_dvp2axi_vdev_node *vnode = vdev_to_node(vdev);
+	struct es_dvp2axi_stream *stream = to_es_dvp2axi_stream(vnode);
+	struct es_dvp2axi_device *dvp2axidev = stream->dvp2axidev;
+	struct media_link *link;
+	int ret;
+
+	link = list_first_entry(&dvp2axidev->sditf[0]->sd.entity.links,
+				struct media_link, list);
+	ret = media_entity_setup_link(link, 0);
+	if (ret)
+		dev_err(dvp2axidev->dev,
+			"failed to disable link of sditf with isp");
+
+	return ret;
+}
+EXPORT_SYMBOL(es_dvp2axi_sditf_disconnect);
diff --git a/drivers/media/platform/eswin/dvp2axi/common.c b/drivers/media/platform/eswin/dvp2axi/common.c
new file mode 100644
index 000000000000..c491abab495e
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/common.c
@@ -0,0 +1,419 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI common driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include <linux/of_platform.h>
+#include "dev.h"
+#include "common.h"
+
+static void es_dvp2axi_init_dummy_vb2(struct es_dvp2axi_device *dev,
+				struct es_dvp2axi_dummy_buffer *buf)
+{
+	unsigned long attrs = buf->is_need_vaddr ? 0 : DMA_ATTR_NO_KERNEL_MAPPING;
+
+	memset(&buf->vb2_queue, 0, sizeof(struct vb2_queue));
+	memset(&buf->vb, 0, sizeof(struct vb2_buffer));
+	buf->vb2_queue.gfp_flags = GFP_KERNEL | GFP_DMA32;
+	buf->vb2_queue.dma_dir = DMA_BIDIRECTIONAL;
+	if (dev->hw_dev->is_dma_contig)
+		attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
+	buf->vb2_queue.dma_attrs = attrs;
+	buf->vb.vb2_queue = &buf->vb2_queue;
+}
+
+int es_dvp2axi_alloc_buffer(struct es_dvp2axi_device *dev,
+		       struct es_dvp2axi_dummy_buffer *buf)
+{
+	const struct vb2_mem_ops *g_ops = dev->hw_dev->mem_ops;
+	struct sg_table	 *sg_tbl;
+	void *mem_priv;
+	int ret = 0;
+
+	if (!buf->size) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	es_dvp2axi_init_dummy_vb2(dev, buf);
+
+	buf->size = PAGE_ALIGN(buf->size);
+	mem_priv = g_ops->alloc(&buf->vb, dev->hw_dev->dev, buf->size);
+	if (IS_ERR_OR_NULL(mem_priv)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	buf->mem_priv = mem_priv;
+	if (dev->hw_dev->is_dma_sg_ops) {
+		sg_tbl = (struct sg_table *)g_ops->cookie(&buf->vb, mem_priv);
+		buf->dma_addr = sg_dma_address(sg_tbl->sgl);
+		g_ops->prepare(mem_priv);
+	} else {
+		buf->dma_addr = *((dma_addr_t *)g_ops->cookie(&buf->vb, mem_priv));
+	}
+	if (buf->is_need_vaddr)
+		buf->vaddr = g_ops->vaddr(&buf->vb, mem_priv);
+	if (buf->is_need_dbuf) {
+		buf->dbuf = g_ops->get_dmabuf(&buf->vb, mem_priv, O_RDWR);
+		if (buf->is_need_dmafd) {
+			buf->dma_fd = dma_buf_fd(buf->dbuf, O_CLOEXEC);
+			if (buf->dma_fd < 0) {
+				dma_buf_put(buf->dbuf);
+				ret = buf->dma_fd;
+				goto err;
+			}
+			get_dma_buf(buf->dbuf);
+		}
+	}
+	v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+		 "%s buf:0x%x~0x%x size:%d\n", __func__,
+		 (u32)buf->dma_addr, (u32)buf->dma_addr + buf->size, buf->size);
+	return ret;
+err:
+	dev_err(dev->dev, "%s failed ret:%d\n", __func__, ret);
+	return ret;
+}
+
+void es_dvp2axi_free_buffer(struct es_dvp2axi_device *dev,
+			struct es_dvp2axi_dummy_buffer *buf)
+{
+	const struct vb2_mem_ops *g_ops = dev->hw_dev->mem_ops;
+
+	if (buf && buf->mem_priv) {
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+			 "%s buf:0x%x~0x%x\n", __func__,
+			 (u32)buf->dma_addr, (u32)buf->dma_addr + buf->size);
+		if (buf->dbuf)
+			dma_buf_put(buf->dbuf);
+		g_ops->put(buf->mem_priv);
+		buf->size = 0;
+		buf->dbuf = NULL;
+		buf->vaddr = NULL;
+		buf->mem_priv = NULL;
+		buf->is_need_dbuf = false;
+		buf->is_need_vaddr = false;
+		buf->is_need_dmafd = false;
+		buf->is_free = true;
+	}
+}
+
+static int es_dvp2axi_alloc_page_dummy_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_dummy_buffer *buf)
+{
+	struct es_dvp2axi_hw *hw = dev->hw_dev;
+	u32 i, n_pages = PAGE_ALIGN(buf->size) >> PAGE_SHIFT;
+	struct page *page = NULL, **pages = NULL;
+	struct sg_table *sg = NULL;
+	int ret = -ENOMEM;
+
+	page = alloc_pages(GFP_KERNEL | GFP_DMA32, 0);
+	if (!page)
+		goto err;
+
+	pages = kvmalloc_array(n_pages, sizeof(struct page *), GFP_KERNEL);
+	if (!pages)
+		goto free_page;
+	for (i = 0; i < n_pages; i++)
+		pages[i] = page;
+
+	sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+	if (!sg)
+		goto free_pages;
+	ret = sg_alloc_table_from_pages(sg, pages, n_pages, 0,
+					n_pages << PAGE_SHIFT, GFP_KERNEL);
+	if (ret)
+		goto free_sg;
+
+	ret = dma_map_sg(hw->dev, sg->sgl, sg->nents, DMA_BIDIRECTIONAL);
+	buf->dma_addr = sg_dma_address(sg->sgl);
+	buf->mem_priv = sg;
+	buf->pages = pages;
+	v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+		 "%s buf:0x%x map cnt:%d size:%d\n", __func__,
+		 (u32)buf->dma_addr, ret, buf->size);
+	return 0;
+free_sg:
+	kfree(sg);
+free_pages:
+	kvfree(pages);
+free_page:
+	__free_pages(page, 0);
+err:
+	return ret;
+}
+
+static void es_dvp2axi_free_page_dummy_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_dummy_buffer *buf)
+{
+	struct sg_table *sg = buf->mem_priv;
+
+	if (!sg)
+		return;
+	dma_unmap_sg(dev->hw_dev->dev, sg->sgl, sg->nents, DMA_BIDIRECTIONAL);
+	sg_free_table(sg);
+	kfree(sg);
+	__free_pages(buf->pages[0], 0);
+	kvfree(buf->pages);
+	buf->mem_priv = NULL;
+	buf->pages = NULL;
+}
+
+int es_dvp2axi_alloc_common_dummy_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_dummy_buffer *buf)
+{
+	struct es_dvp2axi_hw *hw = dev->hw_dev;
+	int ret = 0;
+
+	mutex_lock(&hw->dev_lock);
+	if (buf->mem_priv)
+		goto end;
+
+	if (buf->size == 0)
+		goto end;
+
+	if (hw->iommu_en) {
+		ret = es_dvp2axi_alloc_page_dummy_buf(dev, buf);
+		goto end;
+	}
+
+	ret = es_dvp2axi_alloc_buffer(dev, buf);
+	if (!ret)
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+			 "%s buf:0x%x size:%d\n", __func__,
+			 (u32)buf->dma_addr, buf->size);
+end:
+	if (ret < 0)
+		v4l2_err(&dev->v4l2_dev, "%s failed:%d\n", __func__, ret);
+	mutex_unlock(&hw->dev_lock);
+	return ret;
+}
+
+void es_dvp2axi_free_common_dummy_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_dummy_buffer *buf)
+{
+	struct es_dvp2axi_hw *hw = dev->hw_dev;
+
+	mutex_lock(&hw->dev_lock);
+
+	if (hw->iommu_en)
+		es_dvp2axi_free_page_dummy_buf(dev, buf);
+	else
+		es_dvp2axi_free_buffer(dev, buf);
+	mutex_unlock(&hw->dev_lock);
+}
+
+struct es_dvp2axi_shm_data {
+	void *vaddr;
+	int vmap_cnt;
+	int npages;
+	struct page *pages[];
+};
+
+static struct sg_table *es_dvp2axi_shm_map_dma_buf(struct dma_buf_attachment *attachment,
+					enum dma_data_direction dir)
+{
+	struct es_dvp2axi_shm_data *data = attachment->dmabuf->priv;
+	struct sg_table *table;
+
+	table = kmalloc(sizeof(*table), GFP_KERNEL);
+	if (!table)
+		return ERR_PTR(-ENOMEM);
+
+	sg_alloc_table_from_pages(table, data->pages, data->npages, 0,
+				  data->npages << PAGE_SHIFT, GFP_KERNEL);
+
+	dma_map_sgtable(attachment->dev, table, dir, DMA_ATTR_SKIP_CPU_SYNC);
+
+	return table;
+}
+
+static void es_dvp2axi_shm_unmap_dma_buf(struct dma_buf_attachment *attachment,
+			      struct sg_table *table,
+			      enum dma_data_direction dir)
+{
+	dma_unmap_sgtable(attachment->dev, table, dir, DMA_ATTR_SKIP_CPU_SYNC);
+	sg_free_table(table);
+	kfree(table);
+}
+
+static int es_dvp2axi_shm_vmap(struct dma_buf *dma_buf, struct iosys_map *map)
+{
+	struct es_dvp2axi_shm_data *data = dma_buf->priv;
+
+	data->vaddr = vmap(data->pages, data->npages, VM_MAP, PAGE_KERNEL);
+	data->vmap_cnt++;
+	iosys_map_set_vaddr(map, data->vaddr);
+	return 0;
+}
+
+static void es_dvp2axi_shm_vunmap(struct dma_buf *dma_buf, struct iosys_map *map)
+{
+	struct es_dvp2axi_shm_data *data = dma_buf->priv;
+
+	vunmap(data->vaddr);
+	data->vaddr = NULL;
+	data->vmap_cnt--;
+}
+
+static int es_dvp2axi_shm_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
+{
+	struct es_dvp2axi_shm_data *data = dma_buf->priv;
+	unsigned long vm_start = vma->vm_start;
+	int i;
+
+	for (i = 0; i < data->npages; i++) {
+		remap_pfn_range(vma, vm_start, page_to_pfn(data->pages[i]),
+				PAGE_SIZE, vma->vm_page_prot);
+		vm_start += PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+static int es_dvp2axi_shm_begin_cpu_access(struct dma_buf *dmabuf, enum dma_data_direction dir)
+{
+	struct dma_buf_attachment *attachment;
+	struct sg_table *table;
+
+	attachment = list_first_entry(&dmabuf->attachments, struct dma_buf_attachment, node);
+	table = attachment->priv;
+	dma_sync_sg_for_cpu(NULL, table->sgl, table->nents, dir);
+
+	return 0;
+}
+
+static int es_dvp2axi_shm_end_cpu_access(struct dma_buf *dmabuf, enum dma_data_direction dir)
+{
+	struct dma_buf_attachment *attachment;
+	struct sg_table *table;
+
+	attachment = list_first_entry(&dmabuf->attachments, struct dma_buf_attachment, node);
+	table = attachment->priv;
+	dma_sync_sg_for_device(NULL, table->sgl, table->nents, dir);
+
+	return 0;
+}
+
+static void es_dvp2axi_shm_release(struct dma_buf *dma_buf)
+{
+	struct es_dvp2axi_shm_data *data = dma_buf->priv;
+
+	if (data->vmap_cnt) {
+		WARN(1, "%s: buffer still mapped in the kernel\n", __func__);
+		es_dvp2axi_shm_vunmap(dma_buf, NULL);
+	}
+	kfree(data);
+}
+
+static const struct dma_buf_ops es_dvp2axi_shm_dmabuf_ops = {
+	.map_dma_buf = es_dvp2axi_shm_map_dma_buf,
+	.unmap_dma_buf = es_dvp2axi_shm_unmap_dma_buf,
+	.release = es_dvp2axi_shm_release,
+	.mmap = es_dvp2axi_shm_mmap,
+	.vmap = es_dvp2axi_shm_vmap,
+	.vunmap = es_dvp2axi_shm_vunmap,
+	.begin_cpu_access = es_dvp2axi_shm_begin_cpu_access,
+	.end_cpu_access = es_dvp2axi_shm_end_cpu_access,
+};
+
+static struct dma_buf *es_dvp2axi_shm_alloc(struct esisp_thunderboot_shmem *shmem)
+{
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct dma_buf *dmabuf;
+	struct es_dvp2axi_shm_data *data;
+	int i, npages;
+
+	npages = PAGE_ALIGN(shmem->shm_size) / PAGE_SIZE;
+	data = kmalloc(sizeof(*data) + npages * sizeof(struct page *), GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
+	data->vmap_cnt = 0;
+	data->npages = npages;
+	for (i = 0; i < npages; i++)
+		data->pages[i] = phys_to_page(shmem->shm_start + i * PAGE_SIZE);
+
+	exp_info.ops = &es_dvp2axi_shm_dmabuf_ops;
+	exp_info.size = npages * PAGE_SIZE;
+	exp_info.flags = O_RDWR;
+	exp_info.priv = data;
+
+	dmabuf = dma_buf_export(&exp_info);
+
+	return dmabuf;
+}
+
+int es_dvp2axi_alloc_reserved_mem_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_rx_buffer *buf)
+{
+	struct es_dvp2axi_dummy_buffer *dummy = &buf->dummy;
+
+	dummy->dma_addr = dev->resmem_pa + dummy->size * buf->buf_idx;
+	if (dummy->dma_addr + dummy->size > dev->resmem_pa + dev->resmem_size)
+		return -EINVAL;
+	buf->dbufs.dma = dummy->dma_addr;
+	buf->dbufs.is_resmem = true;
+	buf->shmem.shm_start = dummy->dma_addr;
+	buf->shmem.shm_size = dummy->size;
+	dummy->dbuf = es_dvp2axi_shm_alloc(&buf->shmem);
+	if (dummy->is_need_vaddr) {
+		struct iosys_map map;
+
+		dummy->dbuf->ops->vmap(dummy->dbuf, &map);
+		dummy->vaddr = map.vaddr;
+	}
+	return 0;
+}
+
+void es_dvp2axi_free_reserved_mem_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_rx_buffer *buf)
+{
+	struct es_dvp2axi_dummy_buffer *dummy = &buf->dummy;
+	struct media_pad *pad = NULL;
+	struct v4l2_subdev *sd;
+
+	if (buf->dummy.is_free)
+		return;
+
+	if (dev->rdbk_debug)
+		v4l2_info(&dev->v4l2_dev,
+			  "free reserved mem addr 0x%x\n",
+			  (u32)dummy->dma_addr);
+	if (dev->sditf[0]) {
+		if (dev->sditf[0]->is_combine_mode)
+			pad = media_pad_remote_pad_first(&dev->sditf[0]->pads[1]);
+		else
+			pad = media_pad_remote_pad_first(&dev->sditf[0]->pads[0]);
+	} else {
+		v4l2_info(&dev->v4l2_dev,
+			  "not find sditf\n");
+		return;
+	}
+	if (pad) {
+		sd = media_entity_to_v4l2_subdev(pad->entity);
+	} else {
+		v4l2_info(&dev->v4l2_dev,
+			  "not find remote pad\n");
+		return;
+	}
+	if (buf->dbufs.is_init)
+		v4l2_subdev_call(sd, core, ioctl,
+				 ESISP_VICAP_CMD_RX_BUFFER_FREE, &buf->dbufs);
+	if (dummy->is_need_vaddr)
+		dummy->dbuf->ops->vunmap(dummy->dbuf, NULL);
+	buf->dummy.is_free = true;
+}
+
diff --git a/drivers/media/platform/eswin/dvp2axi/common.h b/drivers/media/platform/eswin/dvp2axi/common.h
new file mode 100644
index 000000000000..1f0c1b41a839
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/common.h
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI common driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_COMMON_H
+#define _ES_DVP2AXI_COMMON_H
+#include <linux/clk.h>
+#include <linux/mutex.h>
+#include <linux/media.h>
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-mc.h>
+#include "dev.h"
+
+int es_dvp2axi_alloc_buffer(struct es_dvp2axi_device *dev,
+		       struct es_dvp2axi_dummy_buffer *buf);
+void es_dvp2axi_free_buffer(struct es_dvp2axi_device *dev,
+			struct es_dvp2axi_dummy_buffer *buf);
+
+int es_dvp2axi_alloc_common_dummy_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_dummy_buffer *buf);
+void es_dvp2axi_free_common_dummy_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_dummy_buffer *buf);
+
+int es_dvp2axi_alloc_reserved_mem_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_rx_buffer *buf);
+void es_dvp2axi_free_reserved_mem_buf(struct es_dvp2axi_device *dev, struct es_dvp2axi_rx_buffer *buf);
+
+#endif /* _ES_DVP2AXI_COMMON_H */
+
diff --git a/drivers/media/platform/eswin/dvp2axi/dev.c b/drivers/media/platform/eswin/dvp2axi/dev.c
new file mode 100644
index 000000000000..8024b9d2241b
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dev.c
@@ -0,0 +1,2108 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI dev driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/reset.h>
+#include <linux/pm_runtime.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/regmap.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include <linux/iommu.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include "dev.h"
+#include "procfs.h"
+#include <linux/kthread.h>
+#include "phy-es-csi2-dphy-common.h"
+#include <linux/of_reserved_mem.h>
+#include <linux/of_address.h>
+
+#include <media/eswin/common-def.h>
+
+
+#define ES_DVP2AXI_VERNO_LEN 10
+
+int es_dvp2axi_debug = 0;
+module_param_named(debug, es_dvp2axi_debug, int, 0644);
+MODULE_PARM_DESC(debug, "Debug level (0-1)");
+
+static char es_dvp2axi_version[ES_DVP2AXI_VERNO_LEN];
+module_param_string(version, es_dvp2axi_version, ES_DVP2AXI_VERNO_LEN, 0444);
+MODULE_PARM_DESC(version, "version number");
+
+static DEFINE_MUTEX(es_dvp2axi_dev_mutex);
+static LIST_HEAD(es_dvp2axi_device_list);
+
+/* show the compact mode of each stream in stream index order,
+ * 1 for compact, 0 for 16bit
+ */
+static ssize_t es_dvp2axi_show_compact_mode(struct device *dev,
+				       struct device_attribute *attr, char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(buf, PAGE_SIZE, "%d %d %d %d\n",
+		       dvp2axi_dev->stream[0].is_compact ? 1 : 0,
+		       dvp2axi_dev->stream[1].is_compact ? 1 : 0,
+		       dvp2axi_dev->stream[2].is_compact ? 1 : 0,
+		       dvp2axi_dev->stream[3].is_compact ? 1 : 0);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_compact_mode(struct device *dev,
+					struct device_attribute *attr,
+					const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i, index;
+	char val[4];
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (buf[i] == ' ') {
+				continue;
+			} else if (buf[i] == '\0') {
+				break;
+			} else {
+				val[index] = buf[i];
+				index++;
+				if (index == 4)
+					break;
+			}
+		}
+
+		for (i = 0; i < index; i++) {
+			if (val[i] - '0' == 0)
+				dvp2axi_dev->stream[i].is_compact = false;
+			else
+				dvp2axi_dev->stream[i].is_compact = true;
+		}
+	}
+
+	return len;
+}
+
+static DEVICE_ATTR(compact_test, S_IWUSR | S_IRUSR, es_dvp2axi_show_compact_mode,
+		   es_dvp2axi_store_compact_mode);
+
+static ssize_t es_dvp2axi_show_line_int_num(struct device *dev,
+				       struct device_attribute *attr, char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(buf, PAGE_SIZE, "%d\n", dvp2axi_dev->wait_line_cache);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_line_int_num(struct device *dev,
+					struct device_attribute *attr,
+					const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	struct sditf_priv *priv = dvp2axi_dev->sditf[0];
+	int val = 0;
+	int ret = 0;
+
+	if (priv && priv->mode.rdbk_mode == ESISP_VICAP_ONLINE) {
+		dev_info(
+			dvp2axi_dev->dev,
+			"current mode is on the fly, wake up mode wouldn't used\n");
+		return len;
+	}
+	ret = kstrtoint(buf, 0, &val);
+	if (!ret && val >= 0 && val <= 0x3fff)
+		dvp2axi_dev->wait_line_cache = val;
+	else
+		dev_info(dvp2axi_dev->dev, "set line int num failed\n");
+	return len;
+}
+
+static DEVICE_ATTR(wait_line, S_IWUSR | S_IRUSR, es_dvp2axi_show_line_int_num,
+		   es_dvp2axi_store_line_int_num);
+
+static ssize_t es_dvp2axi_show_dummybuf_mode(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(buf, PAGE_SIZE, "%d\n", dvp2axi_dev->is_use_dummybuf);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_dummybuf_mode(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int val = 0;
+	int ret = 0;
+
+	ret = kstrtoint(buf, 0, &val);
+	if (!ret) {
+		if (val)
+			dvp2axi_dev->is_use_dummybuf = true;
+		else
+			dvp2axi_dev->is_use_dummybuf = false;
+	} else {
+		dev_info(dvp2axi_dev->dev, "set dummy buf mode failed\n");
+	}
+	return len;
+}
+
+static DEVICE_ATTR(is_use_dummybuf, S_IWUSR | S_IRUSR, es_dvp2axi_show_dummybuf_mode,
+		   es_dvp2axi_store_dummybuf_mode);
+
+/* show the memory mode of each stream in stream index order,
+ * 1 for high align, 0 for low align
+ */
+static ssize_t es_dvp2axi_show_memory_mode(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(
+		buf, PAGE_SIZE,
+		"stream[0~3] %d %d %d %d, 0(low align) 1(high align) 2(compact)\n",
+		dvp2axi_dev->stream[0].is_compact ?
+			2 :
+			(dvp2axi_dev->stream[0].is_high_align ? 1 : 0),
+		dvp2axi_dev->stream[1].is_compact ?
+			2 :
+			(dvp2axi_dev->stream[1].is_high_align ? 1 : 0),
+		dvp2axi_dev->stream[2].is_compact ?
+			2 :
+			(dvp2axi_dev->stream[2].is_high_align ? 1 : 0),
+		dvp2axi_dev->stream[3].is_compact ?
+			2 :
+			(dvp2axi_dev->stream[3].is_high_align ? 1 : 0));
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_memory_mode(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i, index;
+	char val[4];
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (buf[i] == ' ') {
+				continue;
+			} else if (buf[i] == '\0') {
+				break;
+			} else {
+				val[index] = buf[i];
+				index++;
+				if (index == 4)
+					break;
+			}
+		}
+
+		for (i = 0; i < index; i++) {
+			if (dvp2axi_dev->stream[i].is_compact) {
+				dev_info(
+					dvp2axi_dev->dev,
+					"stream[%d] set memory align fail, is compact mode\n",
+					i);
+				continue;
+			}
+			if (val[i] - '0' == 0)
+				dvp2axi_dev->stream[i].is_high_align = false;
+			else
+				dvp2axi_dev->stream[i].is_high_align = true;
+		}
+	}
+
+	return len;
+}
+
+static DEVICE_ATTR(is_high_align, S_IWUSR | S_IRUSR, es_dvp2axi_show_memory_mode,
+		   es_dvp2axi_store_memory_mode);
+
+static ssize_t es_dvp2axi_show_scale_ch0_blc(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(
+		buf, PAGE_SIZE,
+		"ch0 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+		dvp2axi_dev->scale_vdev[0].blc.pattern00,
+		dvp2axi_dev->scale_vdev[0].blc.pattern01,
+		dvp2axi_dev->scale_vdev[0].blc.pattern02,
+		dvp2axi_dev->scale_vdev[0].blc.pattern03);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_scale_ch0_blc(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i = 0, index = 0;
+	unsigned int val[4] = { 0 };
+	unsigned int temp = 0;
+	int ret = 0;
+	int j = 0;
+	char cha[2] = { 0 };
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (((buf[i] == ' ') || (buf[i] == '\n')) && j) {
+				index++;
+				j = 0;
+				if (index == 4)
+					break;
+				continue;
+			} else {
+				if (buf[i] < '0' || buf[i] > '9')
+					continue;
+				cha[0] = buf[i];
+				cha[1] = '\0';
+				ret = kstrtoint(cha, 0, &temp);
+				if (!ret) {
+					if (j)
+						val[index] *= 10;
+					val[index] += temp;
+					j++;
+				}
+			}
+		}
+		if (val[0] > 255 || val[1] > 255 || val[2] > 255 ||
+		    val[3] > 255)
+			return -EINVAL;
+		dvp2axi_dev->scale_vdev[0].blc.pattern00 = val[0];
+		dvp2axi_dev->scale_vdev[0].blc.pattern01 = val[1];
+		dvp2axi_dev->scale_vdev[0].blc.pattern02 = val[2];
+		dvp2axi_dev->scale_vdev[0].blc.pattern03 = val[3];
+		dev_info(
+			dvp2axi_dev->dev,
+			"set ch0 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+			dvp2axi_dev->scale_vdev[0].blc.pattern00,
+			dvp2axi_dev->scale_vdev[0].blc.pattern01,
+			dvp2axi_dev->scale_vdev[0].blc.pattern02,
+			dvp2axi_dev->scale_vdev[0].blc.pattern03);
+	}
+
+	return len;
+}
+
+static DEVICE_ATTR(scale_ch0_blc, S_IWUSR | S_IRUSR, es_dvp2axi_show_scale_ch0_blc,
+		   es_dvp2axi_store_scale_ch0_blc);
+
+static ssize_t es_dvp2axi_show_scale_ch1_blc(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(
+		buf, PAGE_SIZE,
+		"ch1 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+		dvp2axi_dev->scale_vdev[1].blc.pattern00,
+		dvp2axi_dev->scale_vdev[1].blc.pattern01,
+		dvp2axi_dev->scale_vdev[1].blc.pattern02,
+		dvp2axi_dev->scale_vdev[1].blc.pattern03);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_scale_ch1_blc(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i = 0, index = 0;
+	unsigned int val[4] = { 0 };
+	unsigned int temp = 0;
+	int ret = 0;
+	int j = 0;
+	char cha[2] = { 0 };
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (((buf[i] == ' ') || (buf[i] == '\n')) && j) {
+				index++;
+				j = 0;
+				if (index == 4)
+					break;
+				continue;
+			} else {
+				if (buf[i] < '0' || buf[i] > '9')
+					continue;
+				cha[0] = buf[i];
+				cha[1] = '\0';
+				ret = kstrtoint(cha, 0, &temp);
+				if (!ret) {
+					if (j)
+						val[index] *= 10;
+					val[index] += temp;
+					j++;
+				}
+			}
+		}
+		if (val[0] > 255 || val[1] > 255 || val[2] > 255 ||
+		    val[3] > 255)
+			return -EINVAL;
+
+		dvp2axi_dev->scale_vdev[1].blc.pattern00 = val[0];
+		dvp2axi_dev->scale_vdev[1].blc.pattern01 = val[1];
+		dvp2axi_dev->scale_vdev[1].blc.pattern02 = val[2];
+		dvp2axi_dev->scale_vdev[1].blc.pattern03 = val[3];
+
+		dev_info(
+			dvp2axi_dev->dev,
+			"set ch1 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+			dvp2axi_dev->scale_vdev[1].blc.pattern00,
+			dvp2axi_dev->scale_vdev[1].blc.pattern01,
+			dvp2axi_dev->scale_vdev[1].blc.pattern02,
+			dvp2axi_dev->scale_vdev[1].blc.pattern03);
+	}
+
+	return len;
+}
+
+static DEVICE_ATTR(scale_ch1_blc, S_IWUSR | S_IRUSR, es_dvp2axi_show_scale_ch1_blc,
+		   es_dvp2axi_store_scale_ch1_blc);
+
+static ssize_t es_dvp2axi_show_scale_ch2_blc(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(
+		buf, PAGE_SIZE,
+		"ch2 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+		dvp2axi_dev->scale_vdev[2].blc.pattern00,
+		dvp2axi_dev->scale_vdev[2].blc.pattern01,
+		dvp2axi_dev->scale_vdev[2].blc.pattern02,
+		dvp2axi_dev->scale_vdev[2].blc.pattern03);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_scale_ch2_blc(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i = 0, index = 0;
+	unsigned int val[4] = { 0 };
+	unsigned int temp = 0;
+	int ret = 0;
+	int j = 0;
+	char cha[2] = { 0 };
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (((buf[i] == ' ') || (buf[i] == '\n')) && j) {
+				index++;
+				j = 0;
+				if (index == 4)
+					break;
+				continue;
+			} else {
+				if (buf[i] < '0' || buf[i] > '9')
+					continue;
+				cha[0] = buf[i];
+				cha[1] = '\0';
+				ret = kstrtoint(cha, 0, &temp);
+				if (!ret) {
+					if (j)
+						val[index] *= 10;
+					val[index] += temp;
+					j++;
+				}
+			}
+		}
+		if (val[0] > 255 || val[1] > 255 || val[2] > 255 ||
+		    val[3] > 255)
+			return -EINVAL;
+
+		dvp2axi_dev->scale_vdev[2].blc.pattern00 = val[0];
+		dvp2axi_dev->scale_vdev[2].blc.pattern01 = val[1];
+		dvp2axi_dev->scale_vdev[2].blc.pattern02 = val[2];
+		dvp2axi_dev->scale_vdev[2].blc.pattern03 = val[3];
+
+		dev_info(
+			dvp2axi_dev->dev,
+			"set ch2 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+			dvp2axi_dev->scale_vdev[2].blc.pattern00,
+			dvp2axi_dev->scale_vdev[2].blc.pattern01,
+			dvp2axi_dev->scale_vdev[2].blc.pattern02,
+			dvp2axi_dev->scale_vdev[2].blc.pattern03);
+	}
+
+	return len;
+}
+static DEVICE_ATTR(scale_ch2_blc, S_IWUSR | S_IRUSR, es_dvp2axi_show_scale_ch2_blc,
+		   es_dvp2axi_store_scale_ch2_blc);
+
+static ssize_t es_dvp2axi_show_scale_ch3_blc(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(
+		buf, PAGE_SIZE,
+		"ch3 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+		dvp2axi_dev->scale_vdev[3].blc.pattern00,
+		dvp2axi_dev->scale_vdev[3].blc.pattern01,
+		dvp2axi_dev->scale_vdev[3].blc.pattern02,
+		dvp2axi_dev->scale_vdev[3].blc.pattern03);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_scale_ch3_blc(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i = 0, index = 0;
+	unsigned int val[4] = { 0 };
+	unsigned int temp = 0;
+	int ret = 0;
+	int j = 0;
+	char cha[2] = { 0 };
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (((buf[i] == ' ') || (buf[i] == '\n')) && j) {
+				index++;
+				j = 0;
+				if (index == 4)
+					break;
+				continue;
+			} else {
+				if (buf[i] < '0' || buf[i] > '9')
+					continue;
+				cha[0] = buf[i];
+				cha[1] = '\0';
+				ret = kstrtoint(cha, 0, &temp);
+				if (!ret) {
+					if (j)
+						val[index] *= 10;
+					val[index] += temp;
+					j++;
+				}
+			}
+		}
+		if (val[0] > 255 || val[1] > 255 || val[2] > 255 ||
+		    val[3] > 255)
+			return -EINVAL;
+
+		dvp2axi_dev->scale_vdev[3].blc.pattern00 = val[0];
+		dvp2axi_dev->scale_vdev[3].blc.pattern01 = val[1];
+		dvp2axi_dev->scale_vdev[3].blc.pattern02 = val[2];
+		dvp2axi_dev->scale_vdev[3].blc.pattern03 = val[3];
+
+		dev_info(
+			dvp2axi_dev->dev,
+			"set ch3 pattern00: %d, pattern01: %d, pattern02: %d, pattern03: %d\n",
+			dvp2axi_dev->scale_vdev[3].blc.pattern00,
+			dvp2axi_dev->scale_vdev[3].blc.pattern01,
+			dvp2axi_dev->scale_vdev[3].blc.pattern02,
+			dvp2axi_dev->scale_vdev[3].blc.pattern03);
+	}
+
+	return len;
+}
+
+static DEVICE_ATTR(scale_ch3_blc, S_IWUSR | S_IRUSR, es_dvp2axi_show_scale_ch3_blc,
+		   es_dvp2axi_store_scale_ch3_blc);
+
+static ssize_t es_dvp2axi_store_capture_fps(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int i = 0, index = 0;
+	unsigned int val[4] = { 0 };
+	unsigned int temp = 0;
+	int ret = 0;
+	int j = 0;
+	char cha[2] = { 0 };
+
+	if (buf) {
+		index = 0;
+		for (i = 0; i < len; i++) {
+			if (((buf[i] == ' ') || (buf[i] == '\n')) && j) {
+				index++;
+				j = 0;
+				if (index == 4)
+					break;
+				continue;
+			} else {
+				if (buf[i] < '0' || buf[i] > '9')
+					continue;
+				cha[0] = buf[i];
+				cha[1] = '\0';
+				ret = kstrtoint(cha, 0, &temp);
+				if (!ret) {
+					if (j)
+						val[index] *= 10;
+					val[index] += temp;
+					j++;
+				}
+			}
+		}
+		dev_info(dvp2axi_dev->dev,
+			 "set fps id0: %d, id1: %d, id2: %d, id3: %d\n", val[0],
+			 val[1], val[2], val[3]);
+	}
+
+	return len;
+}
+static DEVICE_ATTR(fps, 0200, NULL, es_dvp2axi_store_capture_fps);
+
+static ssize_t es_dvp2axi_show_rdbk_debug(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int ret;
+
+	ret = snprintf(buf, PAGE_SIZE, "%d\n", dvp2axi_dev->rdbk_debug);
+	return ret;
+}
+
+static ssize_t es_dvp2axi_store_rdbk_debug(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t len)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		(struct es_dvp2axi_device *)dev_get_drvdata(dev);
+	int val = 0;
+	int ret = 0;
+
+	ret = kstrtoint(buf, 0, &val);
+	if (!ret)
+		dvp2axi_dev->rdbk_debug = val;
+	else
+		dev_info(dvp2axi_dev->dev, "set rdbk debug failed\n");
+	return len;
+}
+static DEVICE_ATTR(rdbk_debug, 0200, es_dvp2axi_show_rdbk_debug,
+		   es_dvp2axi_store_rdbk_debug);
+
+static struct attribute *dev_attrs[] = {
+	&dev_attr_compact_test.attr,
+	&dev_attr_wait_line.attr,
+	&dev_attr_is_use_dummybuf.attr,
+	&dev_attr_is_high_align.attr,
+	&dev_attr_scale_ch0_blc.attr,
+	&dev_attr_scale_ch1_blc.attr,
+	&dev_attr_scale_ch2_blc.attr,
+	&dev_attr_scale_ch3_blc.attr,
+	&dev_attr_fps.attr,
+	&dev_attr_rdbk_debug.attr,
+	NULL,
+};
+
+static struct attribute_group dev_attr_grp = {
+	.attrs = dev_attrs,
+};
+
+struct es_dvp2axi_match_data {
+	int inf_id;
+};
+
+void es_dvp2axi_write_register(struct es_dvp2axi_device *dev, enum dvp2axi_reg_index index,
+			  u32 val)
+{
+	return ;
+	void __iomem *base = dev->hw_dev->base_addr;
+	const struct dvp2axi_reg *reg = &dev->hw_dev->dvp2axi_regs[index];
+	int csi_offset = 0;
+
+	if (dev->inf_id == ES_DVP2AXI_MIPI_LVDS &&
+	    index >= DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0 &&
+	    index <= DVP2AXI_REG_MIPI_ON_PAD) {
+		csi_offset = dev->csi_host_idx * 0x100;
+	}
+	if (index < DVP2AXI_REG_INDEX_MAX) {
+		if (index == DVP2AXI_REG_DVP_CTRL || reg->offset != 0x0) {
+			write_dvp2axi_reg(base, reg->offset + csi_offset, val);
+			v4l2_dbg(4, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "write reg[0x%x]:0x%x!!!\n",
+				 reg->offset + csi_offset, val);
+		} else {
+			v4l2_dbg(
+				1, es_dvp2axi_debug, &dev->v4l2_dev,
+				"write reg[%d]:0x%x failed, maybe useless!!!\n",
+				index, val);
+		}
+	}
+}
+
+void es_dvp2axi_write_register_or(struct es_dvp2axi_device *dev, enum dvp2axi_reg_index index,
+			     u32 val)
+{
+	return ;
+	unsigned int reg_val = 0x0;
+	void __iomem *base = dev->hw_dev->base_addr;
+	const struct dvp2axi_reg *reg = &dev->hw_dev->dvp2axi_regs[index];
+	int csi_offset = 0;
+
+	if (dev->inf_id == ES_DVP2AXI_MIPI_LVDS &&
+	    index >= DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0 &&
+	    index <= DVP2AXI_REG_MIPI_ON_PAD) {
+			csi_offset = dev->csi_host_idx * 0x100;
+	}
+
+	if (index < DVP2AXI_REG_INDEX_MAX) {
+		if (index == DVP2AXI_REG_DVP_CTRL || reg->offset != 0x0) {
+			reg_val = read_dvp2axi_reg(base, reg->offset + csi_offset);
+			reg_val |= val;
+			write_dvp2axi_reg(base, reg->offset + csi_offset, reg_val);
+			v4l2_dbg(4, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "write or reg[0x%x]:0x%x!!!\n",
+				 reg->offset + csi_offset, val);
+		} else {
+			v4l2_dbg(
+				1, es_dvp2axi_debug, &dev->v4l2_dev,
+				"write reg[%d]:0x%x with OR failed, maybe useless!!!\n",
+				index, val);
+		}
+	}
+}
+
+void es_dvp2axi_write_register_and(struct es_dvp2axi_device *dev,
+			      enum dvp2axi_reg_index index, u32 val)
+{
+	return ;
+	unsigned int reg_val = 0x0;
+	void __iomem *base = dev->hw_dev->base_addr;
+	const struct dvp2axi_reg *reg = &dev->hw_dev->dvp2axi_regs[index];
+	int csi_offset = 0;
+
+	if (dev->inf_id == ES_DVP2AXI_MIPI_LVDS &&
+	    index >= DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0 &&
+	    index <= DVP2AXI_REG_MIPI_ON_PAD) {
+		csi_offset = dev->csi_host_idx * 0x100;
+	}
+
+	if (index < DVP2AXI_REG_INDEX_MAX) {
+		if (index == DVP2AXI_REG_DVP_CTRL || reg->offset != 0x0) {
+			reg_val = read_dvp2axi_reg(base, reg->offset + csi_offset);
+			reg_val &= val;
+			write_dvp2axi_reg(base, reg->offset + csi_offset, reg_val);
+			v4l2_dbg(4, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "write and reg[0x%x]:0x%x!!!\n",
+				 reg->offset + csi_offset, val);
+		} else {
+			v4l2_dbg(
+				1, es_dvp2axi_debug, &dev->v4l2_dev,
+				"write reg[%d]:0x%x with OR failed, maybe useless!!!\n",
+				index, val);
+		}
+	}
+}
+
+unsigned int es_dvp2axi_read_register(struct es_dvp2axi_device *dev,
+				 enum dvp2axi_reg_index index)
+{
+	unsigned int val = 0x0;
+	void __iomem *base = dev->hw_dev->base_addr;
+	const struct dvp2axi_reg *reg = &dev->hw_dev->dvp2axi_regs[index];
+	int csi_offset = 0;
+
+	if (dev->inf_id == ES_DVP2AXI_MIPI_LVDS &&
+	    index >= DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0 &&
+	    index <= DVP2AXI_REG_MIPI_ON_PAD) {
+		csi_offset = dev->csi_host_idx * 0x100;
+	}
+
+	if (index < DVP2AXI_REG_INDEX_MAX) {
+		if (index == DVP2AXI_REG_DVP_CTRL || reg->offset != 0x0)
+			val = read_dvp2axi_reg(base, reg->offset + csi_offset);
+		else
+			v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "read reg[%d] failed, maybe useless!!!\n",
+				 index);
+	}
+
+	return val;
+}
+
+void es_dvp2axi_write_grf_reg(struct es_dvp2axi_device *dev, enum dvp2axi_reg_index index,
+			 u32 val)
+{
+	return ;
+	struct es_dvp2axi_hw *dvp2axi_hw = dev->hw_dev;
+	const struct dvp2axi_reg *reg = &dvp2axi_hw->dvp2axi_regs[index];
+
+	if (index < DVP2AXI_REG_INDEX_MAX) {
+		if (index > DVP2AXI_REG_DVP_CTRL) {
+			if (!IS_ERR(dvp2axi_hw->grf))
+				regmap_write(dvp2axi_hw->grf, reg->offset, val);
+		} else {
+			v4l2_dbg(
+				1, es_dvp2axi_debug, &dev->v4l2_dev,
+				"write reg[%d]:0x%x failed, maybe useless!!!\n",
+				index, val);
+		}
+	}
+}
+
+u32 es_dvp2axi_read_grf_reg(struct es_dvp2axi_device *dev, enum dvp2axi_reg_index index)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev->hw_dev;
+	const struct dvp2axi_reg *reg = &dvp2axi_hw->dvp2axi_regs[index];
+	u32 val = 0xffff;
+
+	if (index < DVP2AXI_REG_INDEX_MAX) {
+		if (index > DVP2AXI_REG_DVP_CTRL) {
+			if (!IS_ERR(dvp2axi_hw->grf))
+				regmap_read(dvp2axi_hw->grf, reg->offset, &val);
+		} else {
+			v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "read reg[%d] failed, maybe useless!!!\n",
+				 index);
+		}
+	}
+
+	return val;
+}
+
+void es_dvp2axi_enable_dvp_clk_dual_edge(struct es_dvp2axi_device *dev, bool on)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev->hw_dev;
+	u32 val = 0x0;
+
+	if (!IS_ERR(dvp2axi_hw->grf)) {
+		if (dev->chip_id == CHIP_EIC770X_DVP2AXI) {
+			if (on)
+				val = EIC770X_DVP2AXI_PCLK_DUAL_EDGE;
+			else
+				val = EIC770X_DVP2AXI_PCLK_SINGLE_EDGE;
+			es_dvp2axi_write_grf_reg(dev, DVP2AXI_REG_GRF_DVP2AXIIO_CON, val);
+		}
+	}
+
+	v4l2_info(&dev->v4l2_dev, "set dual edge mode(%s,0x%x)!!!\n",
+		  on ? "on" : "off", val);
+}
+
+void es_dvp2axi_config_dvp_clk_sampling_edge(struct es_dvp2axi_device *dev,
+					enum es_dvp2axi_clk_edge edge)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev->hw_dev;
+	u32 val = 0x0;
+
+	if (!IS_ERR(dvp2axi_hw->grf)) {
+		if (edge == ES_DVP2AXI_CLK_RISING)
+			val = EIC770X_DVP2AXI_PCLK_SAMPLING_EDGE_RISING;
+		else
+			val = EIC770X_DVP2AXI_PCLK_SAMPLING_EDGE_FALLING;
+		es_dvp2axi_write_grf_reg(dev, DVP2AXI_REG_GRF_DVP2AXIIO_CON, val);
+	}
+}
+
+/**************************** pipeline operations *****************************/
+static int __dvp2axi_pipeline_prepare(struct es_dvp2axi_pipeline *p,
+				  struct media_entity *me)
+{
+	struct v4l2_subdev *sd;
+	int i;
+
+	p->num_subdevs = 0;
+	memset(p->subdevs, 0, sizeof(p->subdevs));
+
+	while (1) {
+		struct media_pad *pad = NULL;
+
+		/* Find remote source pad */
+		for (i = 0; i < me->num_pads; i++) {
+			struct media_pad *spad = &me->pads[i];
+
+			if (!(spad->flags & MEDIA_PAD_FL_SINK))
+				continue;
+			pad = media_pad_remote_pad_first(spad);
+			if (pad)
+				break;
+		}
+
+		if (!pad)
+			break;
+
+		sd = media_entity_to_v4l2_subdev(pad->entity);
+		p->subdevs[p->num_subdevs++] = sd;
+		me = &sd->entity;
+		if (me->num_pads == 1)
+			break;
+	}
+
+	return 0;
+}
+
+static int __dvp2axi_pipeline_s_dvp2axi_clk(struct es_dvp2axi_pipeline *p)
+{
+	return 0;
+}
+
+static int es_dvp2axi_pipeline_open(struct es_dvp2axi_pipeline *p,
+			       struct media_entity *me, bool prepare)
+{
+	int ret;
+
+	if (WARN_ON(!p || !me))
+		return -EINVAL;
+	if (atomic_inc_return(&p->power_cnt) > 1)
+		return 0;
+
+	/* go through media graphic and get subdevs */
+	if (prepare)
+		__dvp2axi_pipeline_prepare(p, me);
+
+	if (!p->num_subdevs)
+		return -EINVAL;
+
+	ret = __dvp2axi_pipeline_s_dvp2axi_clk(p);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int es_dvp2axi_pipeline_close(struct es_dvp2axi_pipeline *p)
+{
+	atomic_dec_return(&p->power_cnt);
+
+	return 0;
+}
+
+static void es_dvp2axi_set_sensor_streamon_in_sync_mode(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_hw *hw = dvp2axi_dev->hw_dev;
+	struct es_dvp2axi_device *dev = NULL;
+	int i = 0, j = 0;
+	int on = 1;
+	int ret = 0;
+	bool is_streaming = false;
+	struct es_dvp2axi_multi_sync_config *sync_config;
+
+	if (!dvp2axi_dev->sync_cfg.type)
+		return;
+
+	mutex_lock(&hw->dev_lock);
+	sync_config = &hw->sync_config[dvp2axi_dev->sync_cfg.group];
+	sync_config->streaming_cnt++;
+	if (sync_config->streaming_cnt < sync_config->dev_cnt) {
+		mutex_unlock(&hw->dev_lock);
+		return;
+	}
+
+	if (sync_config->mode == ES_DVP2AXI_MASTER_MASTER ||
+	    sync_config->mode == ES_DVP2AXI_MASTER_SLAVE) {
+		for (i = 0; i < sync_config->slave.count; i++) {
+			dev = sync_config->slave.dvp2axi_dev[i];
+			is_streaming = sync_config->slave.is_streaming[i];
+			if (!is_streaming) {
+				if (dev->sditf_cnt == 1) {
+					ret = v4l2_subdev_call(
+						dev->terminal_sensor.sd, core,
+						ioctl,
+						ESMODULE_SET_QUICK_STREAM, &on);
+					if (ret)
+						dev_info(
+							dev->dev,
+							"set ESMODULE_SET_QUICK_STREAM failed\n");
+				} else {
+					for (j = 0; j < dev->sditf_cnt; j++)
+						ret |= v4l2_subdev_call(
+							dev->sditf[j]->sensor_sd,
+							core, ioctl,
+							ESMODULE_SET_QUICK_STREAM,
+							&on);
+					if (ret)
+						dev_info(
+							dev->dev,
+							"set ESMODULE_SET_QUICK_STREAM failed\n");
+				}
+				sync_config->slave.is_streaming[i] = true;
+			}
+			v4l2_dbg(3, es_dvp2axi_debug, &dev->v4l2_dev,
+				 "quick stream in sync mode, slave_dev[%d]\n",
+				 i);
+		}
+		for (i = 0; i < sync_config->ext_master.count; i++) {
+			dev = sync_config->ext_master.dvp2axi_dev[i];
+			is_streaming = sync_config->ext_master.is_streaming[i];
+			if (!is_streaming) {
+				if (dev->sditf_cnt == 1) {
+					ret = v4l2_subdev_call(
+						dev->terminal_sensor.sd, core,
+						ioctl,
+						ESMODULE_SET_QUICK_STREAM, &on);
+					if (ret)
+						dev_info(
+							dev->dev,
+							"set ESMODULE_SET_QUICK_STREAM failed\n");
+				} else {
+					for (j = 0; j < dev->sditf_cnt; j++)
+						ret |= v4l2_subdev_call(
+							dev->sditf[j]->sensor_sd,
+							core, ioctl,
+							ESMODULE_SET_QUICK_STREAM,
+							&on);
+					if (ret)
+						dev_info(
+							dev->dev,
+							"set ESMODULE_SET_QUICK_STREAM failed\n");
+				}
+				sync_config->ext_master.is_streaming[i] = true;
+			}
+			v4l2_dbg(
+				3, es_dvp2axi_debug, &dev->v4l2_dev,
+				"quick stream in sync mode, ext_master_dev[%d]\n",
+				i);
+		}
+		for (i = 0; i < sync_config->int_master.count; i++) {
+			dev = sync_config->int_master.dvp2axi_dev[i];
+			is_streaming = sync_config->int_master.is_streaming[i];
+			if (!is_streaming) {
+				if (dev->sditf_cnt == 1) {
+					ret = v4l2_subdev_call(
+						dev->terminal_sensor.sd, core,
+						ioctl,
+						ESMODULE_SET_QUICK_STREAM, &on);
+					if (ret)
+						dev_info(
+							hw->dev,
+							"set ESMODULE_SET_QUICK_STREAM failed\n");
+				} else {
+					for (j = 0; j < dev->sditf_cnt; j++)
+						ret |= v4l2_subdev_call(
+							dev->sditf[j]->sensor_sd,
+							core, ioctl,
+							ESMODULE_SET_QUICK_STREAM,
+							&on);
+					if (ret)
+						dev_info(
+							dev->dev,
+							"set ESMODULE_SET_QUICK_STREAM failed\n");
+				}
+				sync_config->int_master.is_streaming[i] = true;
+			}
+			v4l2_dbg(
+				3, es_dvp2axi_debug, &dev->v4l2_dev,
+				"quick stream in sync mode, int_master_dev[%d]\n",
+				i);
+		}
+	}
+	mutex_unlock(&hw->dev_lock);
+}
+
+#if 0
+static void es_dvp2axi_sensor_streaming_cb(void *data)
+{
+	struct v4l2_subdev *subdevs = (struct v4l2_subdev *)data;
+
+	v4l2_subdev_call(subdevs, video, s_stream, 1);
+}
+#endif
+
+/*
+ * stream-on order: isp_subdev, mipi dphy, sensor
+ * stream-off order: mipi dphy, sensor, isp_subdev
+ */
+static int es_dvp2axi_pipeline_set_stream(struct es_dvp2axi_pipeline *p, bool on)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		container_of(p, struct es_dvp2axi_device, pipe);
+	bool can_be_set = false;
+	int i, ret = 0;
+
+	if (dvp2axi_dev->hdr.hdr_mode == NO_HDR ||
+	    dvp2axi_dev->hdr.hdr_mode == HDR_COMPR) {
+		if ((on && atomic_inc_return(&p->stream_cnt) > 1) ||
+		    (!on && atomic_dec_return(&p->stream_cnt) > 0))
+			return 0;
+
+		if (on) {
+			dvp2axi_dev->irq_stats.csi_overflow_cnt = 0;
+			dvp2axi_dev->irq_stats.csi_bwidth_lack_cnt = 0;
+			dvp2axi_dev->irq_stats.dvp_bus_err_cnt = 0;
+			dvp2axi_dev->irq_stats.dvp_line_err_cnt = 0;
+			dvp2axi_dev->irq_stats.dvp_overflow_cnt = 0;
+			dvp2axi_dev->irq_stats.dvp_pix_err_cnt = 0;
+			dvp2axi_dev->irq_stats.all_err_cnt = 0;
+			dvp2axi_dev->irq_stats.csi_size_err_cnt = 0;
+			dvp2axi_dev->irq_stats.dvp_size_err_cnt = 0;
+			dvp2axi_dev->irq_stats.dvp_bwidth_lack_cnt = 0;
+			dvp2axi_dev->irq_stats.frm_end_cnt[0] = 0;
+			dvp2axi_dev->irq_stats.frm_end_cnt[1] = 0;
+			dvp2axi_dev->irq_stats.frm_end_cnt[2] = 0;
+			dvp2axi_dev->irq_stats.frm_end_cnt[3] = 0;
+			dvp2axi_dev->irq_stats.not_active_buf_cnt[0] = 0;
+			dvp2axi_dev->irq_stats.not_active_buf_cnt[1] = 0;
+			dvp2axi_dev->irq_stats.not_active_buf_cnt[2] = 0;
+			dvp2axi_dev->irq_stats.not_active_buf_cnt[3] = 0;
+			dvp2axi_dev->irq_stats.trig_simult_cnt[0] = 0;
+			dvp2axi_dev->irq_stats.trig_simult_cnt[1] = 0;
+			dvp2axi_dev->irq_stats.trig_simult_cnt[2] = 0;
+			dvp2axi_dev->irq_stats.trig_simult_cnt[3] = 0;
+			dvp2axi_dev->reset_watchdog_timer.is_triggered = false;
+			dvp2axi_dev->reset_watchdog_timer.is_running = false;
+			dvp2axi_dev->err_state_work.last_timestamp = 0;
+			dvp2axi_dev->is_toisp_reset = false;
+			for (i = 0; i < dvp2axi_dev->num_channels; i++)
+				dvp2axi_dev->reset_watchdog_timer
+					.last_buf_wakeup_cnt[i] = 0;
+			dvp2axi_dev->reset_watchdog_timer.run_cnt = 0;
+		}
+
+		/* phy -> sensor */
+		for (i = 0; i < p->num_subdevs; i++) {
+			if (p->subdevs[i] == dvp2axi_dev->terminal_sensor.sd &&
+			    on && dvp2axi_dev->is_thunderboot) {
+				// dvp2axi_dev->tb_client.data = p->subdevs[i];
+				// dvp2axi_dev->tb_client.cb = es_dvp2axi_sensor_streaming_cb;
+				// es_tb_client_register_cb(&dvp2axi_dev->tb_client);
+			} else {
+				ret = v4l2_subdev_call(p->subdevs[i], video,
+						       s_stream, on);
+			}
+			if (on && ret < 0 && ret != -ENOIOCTLCMD &&
+			    ret != -ENODEV)
+				goto err_stream_off;
+		}
+
+		if (dvp2axi_dev->sditf_cnt > 1) {
+			for (i = 0; i < dvp2axi_dev->sditf_cnt; i++) {
+				ret = v4l2_subdev_call(
+					dvp2axi_dev->sditf[i]->sensor_sd, video,
+					s_stream, on);
+				if (on && ret < 0 && ret != -ENOIOCTLCMD &&
+				    ret != -ENODEV)
+					goto err_stream_off;
+			}
+		}
+
+		if (on)
+			es_dvp2axi_set_sensor_streamon_in_sync_mode(dvp2axi_dev);
+	} else {
+		if (!on && atomic_dec_return(&p->stream_cnt) > 0)
+			return 0;
+
+		if (on) {
+			atomic_inc(&p->stream_cnt);
+			if (dvp2axi_dev->hdr.hdr_mode == HDR_X2) {
+				if (atomic_read(&p->stream_cnt) == 1) {
+					can_be_set = false;
+				} else if (atomic_read(&p->stream_cnt) == 2) {
+					can_be_set = true;
+				}
+			} else if (dvp2axi_dev->hdr.hdr_mode == HDR_X3) {
+				if (atomic_read(&p->stream_cnt) == 1) {
+					can_be_set = false;
+				} else if (atomic_read(&p->stream_cnt) == 3) {
+					can_be_set = true;
+				}
+			}
+		}
+
+		if ((on && can_be_set) || !on) {
+			if (on) {
+				dvp2axi_dev->irq_stats.csi_overflow_cnt = 0;
+				dvp2axi_dev->irq_stats.csi_bwidth_lack_cnt = 0;
+				dvp2axi_dev->irq_stats.dvp_bus_err_cnt = 0;
+				dvp2axi_dev->irq_stats.dvp_line_err_cnt = 0;
+				dvp2axi_dev->irq_stats.dvp_overflow_cnt = 0;
+				dvp2axi_dev->irq_stats.dvp_pix_err_cnt = 0;
+				dvp2axi_dev->irq_stats.dvp_bwidth_lack_cnt = 0;
+				dvp2axi_dev->irq_stats.all_err_cnt = 0;
+				dvp2axi_dev->irq_stats.csi_size_err_cnt = 0;
+				dvp2axi_dev->irq_stats.dvp_size_err_cnt = 0;
+				dvp2axi_dev->irq_stats.frm_end_cnt[0] = 0;
+				dvp2axi_dev->irq_stats.frm_end_cnt[1] = 0;
+				dvp2axi_dev->irq_stats.frm_end_cnt[2] = 0;
+				dvp2axi_dev->irq_stats.frm_end_cnt[3] = 0;
+				dvp2axi_dev->irq_stats.not_active_buf_cnt[0] = 0;
+				dvp2axi_dev->irq_stats.not_active_buf_cnt[1] = 0;
+				dvp2axi_dev->irq_stats.not_active_buf_cnt[2] = 0;
+				dvp2axi_dev->irq_stats.not_active_buf_cnt[3] = 0;
+				dvp2axi_dev->irq_stats.trig_simult_cnt[0] = 0;
+				dvp2axi_dev->irq_stats.trig_simult_cnt[1] = 0;
+				dvp2axi_dev->irq_stats.trig_simult_cnt[2] = 0;
+				dvp2axi_dev->irq_stats.trig_simult_cnt[3] = 0;
+				dvp2axi_dev->is_start_hdr = true;
+				dvp2axi_dev->reset_watchdog_timer.is_triggered =
+					false;
+				dvp2axi_dev->reset_watchdog_timer.is_running =
+					false;
+				dvp2axi_dev->is_toisp_reset = false;
+				for (i = 0; i < dvp2axi_dev->num_channels; i++)
+					dvp2axi_dev->reset_watchdog_timer
+						.last_buf_wakeup_cnt[i] = 0;
+				dvp2axi_dev->reset_watchdog_timer.run_cnt = 0;
+			}
+
+			/* phy -> sensor */
+			for (i = 0; i < p->num_subdevs; i++) {
+				if (p->subdevs[i] ==
+					    dvp2axi_dev->terminal_sensor.sd &&
+				    on && dvp2axi_dev->is_thunderboot) {
+					// dvp2axi_dev->tb_client.data = p->subdevs[i];
+					// dvp2axi_dev->tb_client.cb = es_dvp2axi_sensor_streaming_cb;
+					// es_tb_client_register_cb(&dvp2axi_dev->tb_client);
+				} else {
+					ret = v4l2_subdev_call(p->subdevs[i],
+							       video, s_stream,
+							       on);
+				}
+				if (on && ret < 0 && ret != -ENOIOCTLCMD &&
+				    ret != -ENODEV)
+					goto err_stream_off;
+			}
+			if (dvp2axi_dev->sditf_cnt > 1) {
+				for (i = 0; i < dvp2axi_dev->sditf_cnt; i++) {
+					ret = v4l2_subdev_call(
+						dvp2axi_dev->sditf[i]->sensor_sd,
+						video, s_stream, on);
+					if (on && ret < 0 &&
+					    ret != -ENOIOCTLCMD &&
+					    ret != -ENODEV)
+						goto err_stream_off;
+				}
+			}
+
+			if (on)
+				es_dvp2axi_set_sensor_streamon_in_sync_mode(dvp2axi_dev);
+		}
+	}
+
+
+	return 0;
+
+err_stream_off:
+	for (--i; i >= 0; --i)
+		v4l2_subdev_call(p->subdevs[i], video, s_stream, false);
+	return ret;
+}
+
+static int es_dvp2axi_create_link(struct es_dvp2axi_device *dev,
+			     struct es_dvp2axi_sensor_info *sensor, u32 stream_num, u32 sensor_id,
+			     bool *mipi_lvds_linked)
+{
+	struct es_dvp2axi_sensor_info linked_sensor;
+	struct media_entity *source_entity, *sink_entity;
+	int ret = 0;
+	u32 flags, pad, id;
+	int pad_offset = 0;
+
+	if (dev->chip_id >= CHIP_EIC770X_DVP2AXI)
+		pad_offset = 4;
+
+	linked_sensor.lanes = sensor->lanes;
+
+	if (sensor->mbus.type == V4L2_MBUS_CCP2) {
+		linked_sensor.sd = &dev->lvds_subdev.sd;
+		dev->lvds_subdev.sensor_self.sd = &dev->lvds_subdev.sd;
+		dev->lvds_subdev.sensor_self.lanes = sensor->lanes;
+		memcpy(&dev->lvds_subdev.sensor_self.mbus, &sensor->mbus,
+		       sizeof(struct v4l2_mbus_config));
+	} else {
+		linked_sensor.sd = sensor->sd;
+	}
+
+	memcpy(&linked_sensor.mbus, &sensor->mbus,
+	       sizeof(struct v4l2_mbus_config));
+
+	for (pad = 0; pad < linked_sensor.sd->entity.num_pads; pad++) {
+		if (linked_sensor.sd->entity.pads[pad].flags &
+		    MEDIA_PAD_FL_SOURCE) {
+			if (pad == linked_sensor.sd->entity.num_pads) {
+				dev_err(dev->dev,
+					"failed to find src pad for %s\n",
+					linked_sensor.sd->name);
+
+				break;
+			}
+			for (id = 0; id < stream_num; id++) {
+				source_entity = &linked_sensor.sd->entity;
+				sink_entity =
+					&dev->stream[id].vnode.vdev.entity;
+				if ((id == pad - 1 && !(*mipi_lvds_linked)))
+					flags = MEDIA_LNK_FL_ENABLED;
+				else
+					flags = 0;
+				ret = media_create_pad_link(source_entity, pad,
+							    sink_entity, 0,
+							    flags);
+				if (ret) {
+					dev_err(dev->dev,
+						"failed to create link for %s\n",
+						linked_sensor.sd->name);
+					break;
+				}
+			}
+		}
+	}
+
+	if (sensor->mbus.type == V4L2_MBUS_CCP2) {
+		source_entity = &sensor->sd->entity;
+		sink_entity = &linked_sensor.sd->entity;
+		ret = media_create_pad_link(source_entity, 1, sink_entity, 0,
+					    MEDIA_LNK_FL_ENABLED);
+		if (ret)
+			dev_err(dev->dev,
+				"failed to create link between %s and %s\n",
+				linked_sensor.sd->name, sensor->sd->name);
+	}
+
+	if (linked_sensor.mbus.type != V4L2_MBUS_BT656 &&
+	    linked_sensor.mbus.type != V4L2_MBUS_PARALLEL) {
+			// pr_info("%s:%d yfx is linked \n", __func__, __LINE__);
+			*mipi_lvds_linked = true;
+		}
+	return ret;
+}
+
+/***************************** media controller *******************************/
+static int es_dvp2axi_create_links(struct es_dvp2axi_device *dev)
+{
+	u32 s = 0;
+	u32 stream_num = 0;
+	bool mipi_lvds_linked= false;
+
+	stream_num = ESDVP2AXI_MULTI_STREAM;
+
+	/* sensor links(or mipi-phy) */
+	for (s = 0; s < dev->num_sensors; ++s) {
+		struct es_dvp2axi_sensor_info *sensor = &dev->sensors[s];
+		es_dvp2axi_create_link(dev, sensor, stream_num, s, &mipi_lvds_linked);
+	}
+	
+	return 0;
+}
+
+static int _set_pipeline_default_fmt(struct es_dvp2axi_device *dev)
+{
+	es_dvp2axi_set_default_fmt(dev);
+	return 0;
+}
+
+static int subdev_asyn_register_itf(struct es_dvp2axi_device *dev)
+{
+	struct sditf_priv *sditf = NULL;
+	int ret = 0;
+
+	if (IS_ENABLED(CONFIG_NO_GKI)) {
+		ret = es_dvp2axi_update_sensor_info(&dev->stream[0]);
+		if (ret) {
+			v4l2_err(
+				&dev->v4l2_dev,
+				"There is not terminal subdev, not synchronized with ISP\n");
+			return 0;
+		}
+	}
+	sditf = dev->sditf[0];
+	if (sditf && (!sditf->is_combine_mode) && (!dev->is_notifier_isp)) {
+		ret = v4l2_async_register_subdev_sensor(&sditf->sd);
+		dev->is_notifier_isp = true;
+	}
+
+	return ret;
+}
+
+static int subdev_notifier_complete(struct v4l2_async_notifier *notifier)
+{
+	struct es_dvp2axi_device *dev;
+	struct es_dvp2axi_sensor_info *sensor;
+	struct v4l2_subdev *sd;
+	struct v4l2_device *v4l2_dev = NULL;
+	int ret, index;
+
+	dev = container_of(notifier, struct es_dvp2axi_device, notifier);
+
+	v4l2_dev = &dev->v4l2_dev;
+	
+	for (index = 0; index < dev->num_sensors; index++) {
+		sensor = &dev->sensors[index];
+
+		list_for_each_entry(sd, &v4l2_dev->subdevs, list) {
+			if (sd->ops) {
+				if (sd == sensor->sd) {
+					ret = v4l2_subdev_call(sd, pad,
+							       get_mbus_config,
+							       0,
+							       &sensor->mbus);
+					if (ret)
+						v4l2_err(
+							v4l2_dev,
+							"get mbus config failed for linking\n");
+				}
+			}
+		}
+		if (sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ||
+		    sensor->mbus.type == V4L2_MBUS_CSI2_CPHY) {
+			sensor->lanes =
+				sensor->mbus.bus.mipi_csi2.num_data_lanes;
+		} else if (sensor->mbus.type == V4L2_MBUS_CCP2) {
+			sensor->lanes = sensor->mbus.bus.mipi_csi1.data_lane;
+		}
+		
+		if (sensor->mbus.type == V4L2_MBUS_CCP2) {
+			ret = es_dvp2axi_register_lvds_subdev(dev);
+			if (ret < 0) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"Err: register lvds subdev failed!!!\n");
+				goto notifier_end;
+			}
+			break;
+		}
+
+		if (sensor->mbus.type == V4L2_MBUS_PARALLEL ||
+		    sensor->mbus.type == V4L2_MBUS_BT656) {
+			ret = es_dvp2axi_register_dvp_sof_subdev(dev);
+			if (ret < 0) {
+				v4l2_err(
+					&dev->v4l2_dev,
+					"Err: register dvp sof subdev failed!!!\n");
+				goto notifier_end;
+			}
+			break;
+		}
+	}
+
+	ret = es_dvp2axi_create_links(dev);
+	if (ret < 0)
+		goto unregister_lvds;
+	ret = v4l2_device_register_subdev_nodes(&dev->v4l2_dev);
+	if (ret < 0)
+		goto unregister_lvds;
+	ret = _set_pipeline_default_fmt(dev);
+	if (ret < 0)
+		goto unregister_lvds;
+	if (!completion_done(&dev->cmpl_ntf))
+		complete(&dev->cmpl_ntf);
+	v4l2_info(&dev->v4l2_dev, "Async subdev notifier completed\n");
+	return ret;
+
+unregister_lvds:
+	es_dvp2axi_unregister_lvds_subdev(dev);
+	es_dvp2axi_unregister_dvp_sof_subdev(dev);
+notifier_end:
+	return ret;
+}
+
+struct es_dvp2axi_async_subdev {
+	struct v4l2_async_connection asd;
+	struct v4l2_mbus_config mbus;
+	u32 port;
+	int lanes;
+};
+
+static int subdev_notifier_bound(struct v4l2_async_notifier *notifier,
+				 struct v4l2_subdev *subdev,
+				 struct v4l2_async_connection *asd)
+{
+	struct es_dvp2axi_device *dvp2axi_dev =
+		container_of(notifier, struct es_dvp2axi_device, notifier);
+	struct es_dvp2axi_async_subdev *s_asd =
+		container_of(asd, struct es_dvp2axi_async_subdev, asd);
+
+	// pr_info("%s:%d yfx! subdev %p, asd %p \n", __func__, __LINE__,subdev, asd);
+
+	if (dvp2axi_dev->num_sensors == ARRAY_SIZE(dvp2axi_dev->sensors)) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev,
+			 "%s: the num of subdev is beyond %d\n", __func__,
+			 dvp2axi_dev->num_sensors);
+		return -EBUSY;
+	}
+
+	dvp2axi_dev->sensors[dvp2axi_dev->num_sensors].lanes = s_asd->lanes;
+	dvp2axi_dev->sensors[dvp2axi_dev->num_sensors].mbus = s_asd->mbus;
+	dvp2axi_dev->sensors[dvp2axi_dev->num_sensors].sd = subdev;
+	++dvp2axi_dev->num_sensors;
+
+
+	v4l2_err(subdev, "Async registered subdev\n");
+
+	return 0;
+}
+
+// static int es_dvp2axi_fwnode_parse(struct device *dev,
+// 			      struct v4l2_fwnode_endpoint *vep,
+// 			      struct v4l2_async_connection *asd)
+// {
+// 	struct es_dvp2axi_async_subdev *es_asd =
+// 			container_of(asd, struct es_dvp2axi_async_subdev, asd);
+
+// 	if (vep->bus_type != V4L2_MBUS_BT656 &&
+// 	    vep->bus_type != V4L2_MBUS_PARALLEL &&
+// 	    vep->bus_type != V4L2_MBUS_CSI2_DPHY &&
+// 	    vep->bus_type != V4L2_MBUS_CSI2_CPHY &&
+// 	    vep->bus_type != V4L2_MBUS_CCP2)
+// 		return 0;
+
+// 	es_asd->mbus.type = vep->bus_type;
+
+// 	return 0;
+// }
+
+static const struct v4l2_async_notifier_operations subdev_notifier_ops = {
+	.bound = subdev_notifier_bound,
+	.complete = subdev_notifier_complete,
+};
+
+static int es_dvp2axi_fwnode_parse(struct es_dvp2axi_device *sditf)
+{
+	struct device *dev = sditf->dev;
+
+	int ret = 0, i = 0;
+
+	for (i = 0; i < 2; i++) {
+		struct v4l2_fwnode_endpoint vep = {
+			// .bus_type = V4L2_MBUS_CSI2_DPHY
+			.bus_type = V4L2_MBUS_UNKNOWN
+		};
+		// pr_info("%s:%d yfx! i %d \n", __func__, __LINE__, i);
+		struct es_dvp2axi_async_subdev *s_asd;
+		struct fwnode_handle *ep;
+		struct fwnode_handle *remote_ep = NULL;
+
+		ep = fwnode_graph_get_endpoint_by_id(
+			dev_fwnode(dev), i, 0, FWNODE_GRAPH_ENDPOINT_NEXT);
+		if (!ep)
+			continue;
+
+		remote_ep = fwnode_graph_get_remote_port_parent(ep);
+
+		if (!fwnode_device_is_available(remote_ep)) {
+			fwnode_handle_put(remote_ep);
+			continue;
+		}
+
+		ret = v4l2_fwnode_endpoint_parse(ep, &vep);
+		// pr_info("%s:%d yfx!!!\n", __func__, __LINE__);
+		if (ret)
+			goto err_parse;
+		
+		s_asd = v4l2_async_nf_add_fwnode_remote(
+			&sditf->notifier, ep, struct es_dvp2axi_async_subdev);
+		if (IS_ERR(s_asd)) {
+			ret = PTR_ERR(s_asd);
+			goto err_parse;
+		}
+
+		s_asd->port = vep.base.port;
+		s_asd->lanes = vep.bus.mipi_csi2.num_data_lanes;
+		s_asd->mbus.type = vep.bus_type;
+
+		fwnode_handle_put(ep);
+		
+		continue;
+
+err_parse:
+		fwnode_handle_put(ep);
+		return ret;
+	}
+
+	/*
+	 * Proceed even without sensors connected to allow the device to
+	 * suspend.
+	 */
+	sditf->notifier.ops = &subdev_notifier_ops;
+	ret = v4l2_async_nf_register(&sditf->notifier);
+	if (ret)
+		dev_err(dev, "failed to register async notifier : %d\n", ret);
+	return ret;
+}
+
+static int dvp2axi_subdev_notifier(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct v4l2_async_notifier *ntf = &dvp2axi_dev->notifier;
+	int ret;
+
+	v4l2_async_nf_init(ntf, &dvp2axi_dev->v4l2_dev);
+	ret = es_dvp2axi_fwnode_parse(dvp2axi_dev);
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+static int notifier_isp_thread(void *data)
+{
+	struct es_dvp2axi_device *dev = data;
+	int ret = 0;
+
+	ret = wait_for_completion_timeout(&dev->cmpl_ntf,
+					  msecs_to_jiffies(5000));
+	if (ret) {
+		mutex_lock(&es_dvp2axi_dev_mutex);
+		subdev_asyn_register_itf(dev);
+		mutex_unlock(&es_dvp2axi_dev_mutex);
+	}
+	return 0;
+}
+
+/***************************** platform deive *******************************/
+
+static int es_dvp2axi_register_platform_subdevs(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	int stream_num = 0, ret;
+
+	//stream_num = ES_DVP2AXI_SINGLE_STREAM;
+	stream_num = ESDVP2AXI_MAX_STREAM_MIPI;
+	ret = es_dvp2axi_register_stream_vdevs(dvp2axi_dev, stream_num, true);
+	//ret = es_dvp2axi_register_stream_vdevs(dvp2axi_dev, stream_num, false);
+	if (ret < 0) {
+		dev_err(dvp2axi_dev->dev, "dvp2axi register stream[%d] failed!\n",
+			stream_num);
+		return -EINVAL;
+	}
+	dvp2axi_dev->is_support_tools = false;
+
+	init_completion(&dvp2axi_dev->cmpl_ntf);
+	kthread_run(notifier_isp_thread, dvp2axi_dev, "notifier isp");
+	ret = dvp2axi_subdev_notifier(dvp2axi_dev);
+	if (ret < 0) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev,
+			 "Failed to register subdev notifier(%d)\n", ret);
+		goto err_unreg_stream_vdev;
+	}
+	return 0;
+err_unreg_stream_vdev:
+	es_dvp2axi_unregister_stream_vdevs(dvp2axi_dev, stream_num);
+	es_dvp2axi_unregister_scale_vdevs(dvp2axi_dev, ES_DVP2AXI_MAX_SCALE_CH);
+
+	return ret;
+}
+
+static irqreturn_t es_dvp2axi_irq_handler(int irq, struct es_dvp2axi_device *dvp2axi_dev)
+{
+	if (dvp2axi_dev->workmode == ES_DVP2AXI_WORKMODE_PINGPONG) {
+		es_dvp2axi_irq_pingpong_v1(dvp2axi_dev);
+	} else {
+		es_dvp2axi_irq_oneframe(dvp2axi_dev);
+	}
+	return IRQ_HANDLED;
+}
+
+#if 0
+static irqreturn_t es_dvp2axi_irq_lite_handler(int irq, struct es_dvp2axi_device *dvp2axi_dev)
+{
+	es_dvp2axi_irq_lite_lvds(dvp2axi_dev);
+
+	return IRQ_HANDLED;
+}
+#endif
+
+#if 0
+static void es_dvp2axi_attach_dphy_hw(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct platform_device *plat_dev;
+	struct device *dev = dvp2axi_dev->dev;
+	struct device_node *np;
+	struct csi2_dphy_hw *dphy_hw;
+
+	np = of_parse_phandle(dev->of_node, "eic770x,dphy_hw", 0);
+	if (!np || !of_device_is_available(np)) {
+		dev_err(dev, "failed to get dphy hw node\n");
+		return;
+	}
+
+	plat_dev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!plat_dev) {
+		dev_err(dev, "failed to get dphy hw from node\n");
+		return;
+	}
+
+	dphy_hw = platform_get_drvdata(plat_dev);
+	if (!dphy_hw) {
+		dev_err(dev, "failed attach dphy hw\n");
+		return;
+	}
+	dvp2axi_dev->dphy_hw = dphy_hw;
+}
+#endif
+
+int es_dvp2axi_attach_hw(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct device_node *np;
+	struct platform_device *pdev;
+	struct es_dvp2axi_hw *hw;
+
+	if (dvp2axi_dev->hw_dev)
+		return 0;
+
+	dvp2axi_dev->chip_id = CHIP_EIC770X_DVP2AXI;
+	np = of_parse_phandle(dvp2axi_dev->dev->of_node, "eswin,hw", 0);
+	if (!np || !of_device_is_available(np)) {
+		dev_err(dvp2axi_dev->dev, "failed to get dvp2axi hw node\n");
+		return -ENODEV;
+	}
+
+	pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!pdev) {
+		dev_err(dvp2axi_dev->dev, "failed to get dvp2axi hw from node\n");
+		return -ENODEV;
+	}
+
+	hw = platform_get_drvdata(pdev);
+	if (!hw) {
+		dev_err(dvp2axi_dev->dev, "failed attach dvp2axi hw\n");
+		return -EINVAL;
+	}
+
+	hw->dvp2axi_dev[hw->dev_num] = dvp2axi_dev;
+	hw->dev_num++;
+	dvp2axi_dev->hw_dev = hw;
+	dvp2axi_dev->chip_id = hw->chip_id;
+	dev_info(dvp2axi_dev->dev, "attach to dvp2axi hw node\n");
+
+	return 0;
+}
+
+static int es_dvp2axi_detach_hw(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_hw *hw = dvp2axi_dev->hw_dev;
+	int i;
+
+	for (i = 0; i < hw->dev_num; i++) {
+		if (hw->dvp2axi_dev[i] == dvp2axi_dev) {
+			if ((i + 1) < hw->dev_num) {
+				hw->dvp2axi_dev[i] = hw->dvp2axi_dev[i + 1];
+				hw->dvp2axi_dev[i + 1] = NULL;
+			} else {
+				hw->dvp2axi_dev[i] = NULL;
+			}
+
+			hw->dev_num--;
+			dev_info(dvp2axi_dev->dev, "detach to dvp2axi hw node\n");
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static void es_dvp2axi_init_reset_monitor(struct es_dvp2axi_device *dev)
+{
+	struct es_dvp2axi_timer *timer = &dev->reset_watchdog_timer;
+
+	timer->monitor_mode = ES_DVP2AXI_MONITOR_MODE_IDLE;
+	timer->err_time_interval = 0xffffffff;
+	timer->frm_num_of_monitor_cycle = 0xffffffff;
+	timer->triggered_frame_num = 0xffffffff;
+	timer->csi2_err_ref_cnt = 0xffffffff;
+	timer->is_running = false;
+	timer->is_triggered = false;
+	timer->is_buf_stop_update = false;
+	timer->csi2_err_cnt_even = 0;
+	timer->csi2_err_cnt_odd = 0;
+	timer->csi2_err_fs_fe_cnt = 0;
+	timer->csi2_err_fs_fe_detect_cnt = 0;
+	timer->csi2_err_triggered_cnt = 0;
+	timer->csi2_first_err_timestamp = 0;
+
+	timer_setup(&timer->timer, es_dvp2axi_reset_watchdog_timer_handler, 0);
+
+	INIT_WORK(&dev->reset_work.work, es_dvp2axi_reset_work);
+}
+
+void es_dvp2axi_set_sensor_stream(struct work_struct *work)
+{
+	struct es_dvp2axi_sensor_work *sensor_work =
+		container_of(work, struct es_dvp2axi_sensor_work, work);
+	struct es_dvp2axi_device *dvp2axi_dev =
+		container_of(sensor_work, struct es_dvp2axi_device, sensor_work);
+
+	v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, core, ioctl,
+			 ESMODULE_SET_QUICK_STREAM, &sensor_work->on);
+}
+
+static void es_dvp2axi_deal_err_intr(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct es_dvp2axi_device *dvp2axi_dev =
+		container_of(dwork, struct es_dvp2axi_device, work_deal_err);
+
+	dvp2axi_dev->intr_mask |= CSI_BANDWIDTH_LACK_V1;
+	es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_MIPI_LVDS_INTEN,
+				CSI_BANDWIDTH_LACK_V1);
+}
+
+//2025-3-14 use es_vi_dev->media_dev to replace es_dvp2axi_media_dev
+int es_dvp2axi_plat_init(struct es_dvp2axi_device *dvp2axi_dev, struct device_node *node,
+		    int inf_id)
+{
+	struct device *dev = dvp2axi_dev->dev;
+	struct v4l2_device *v4l2_dev;
+	struct eswin_vi_device* es_vi_dev;
+	int ret;
+
+	dvp2axi_dev->hdr.hdr_mode = NO_HDR;
+	dvp2axi_dev->inf_id = inf_id;
+
+	es_vi_dev =  dev_get_drvdata(dvp2axi_dev->dev->parent);
+
+	mutex_init(&dvp2axi_dev->stream_lock);
+	mutex_init(&dvp2axi_dev->scale_lock);
+	mutex_init(&dvp2axi_dev->tools_lock);
+	spin_lock_init(&dvp2axi_dev->hdr_lock);
+	spin_lock_init(&dvp2axi_dev->buffree_lock);
+	spin_lock_init(&dvp2axi_dev->reset_watchdog_timer.timer_lock);
+	spin_lock_init(&dvp2axi_dev->reset_watchdog_timer.csi2_err_lock);
+	atomic_set(&dvp2axi_dev->pipe.power_cnt, 0);
+	atomic_set(&dvp2axi_dev->pipe.stream_cnt, 0);
+	atomic_set(&dvp2axi_dev->power_cnt, 0);
+	atomic_set(&dvp2axi_dev->streamoff_cnt, 0);
+	dvp2axi_dev->is_start_hdr = false;
+	dvp2axi_dev->pipe.open = es_dvp2axi_pipeline_open;
+	dvp2axi_dev->pipe.close = es_dvp2axi_pipeline_close;
+	dvp2axi_dev->pipe.set_stream = es_dvp2axi_pipeline_set_stream;
+	dvp2axi_dev->isr_hdl = es_dvp2axi_irq_handler;
+	dvp2axi_dev->id_use_cnt = 0;
+	memset(&dvp2axi_dev->sync_cfg, 0, sizeof(dvp2axi_dev->sync_cfg));
+	dvp2axi_dev->sditf_cnt = 0;
+	dvp2axi_dev->is_notifier_isp = false;
+	dvp2axi_dev->sensor_linetime = 0;
+	dvp2axi_dev->early_line = 0;
+	dvp2axi_dev->is_thunderboot = false;
+	dvp2axi_dev->rdbk_debug = 0;
+
+	dvp2axi_dev->resume_mode = 0;
+	memset(&dvp2axi_dev->channels[0].capture_info, 0,
+	       sizeof(dvp2axi_dev->channels[0].capture_info));
+
+	INIT_WORK(&dvp2axi_dev->err_state_work.work, es_dvp2axi_err_print_work);
+	INIT_WORK(&dvp2axi_dev->sensor_work.work, es_dvp2axi_set_sensor_stream);
+	INIT_DELAYED_WORK(&dvp2axi_dev->work_deal_err, es_dvp2axi_deal_err_intr);
+	es_dvp2axi_stream_init(dvp2axi_dev, ESDVP2AXI_STREAM_MIPI_ID0);
+	es_dvp2axi_stream_init(dvp2axi_dev, ESDVP2AXI_STREAM_MIPI_ID1);
+	es_dvp2axi_stream_init(dvp2axi_dev, ESDVP2AXI_STREAM_MIPI_ID2);
+	es_dvp2axi_stream_init(dvp2axi_dev, ESDVP2AXI_STREAM_MIPI_ID3);
+	es_dvp2axi_stream_init(dvp2axi_dev, ESDVP2AXI_STREAM_MIPI_ID4);
+	es_dvp2axi_stream_init(dvp2axi_dev, ESDVP2AXI_STREAM_MIPI_ID5);
+
+	
+	dvp2axi_dev->workmode = ES_DVP2AXI_WORKMODE_PINGPONG;
+
+	dvp2axi_dev->is_use_dummybuf = false;
+	// strlcpy(dvp2axi_dev->media_dev.model, dev_name(dev),
+		// sizeof(dvp2axi_dev->media_dev.model));
+	dvp2axi_dev->csi_host_idx = of_alias_get_id(node, "es_dvp2axi_mipi_lvds");
+	if (dvp2axi_dev->csi_host_idx < 0 || dvp2axi_dev->csi_host_idx > 5)
+		dvp2axi_dev->csi_host_idx = 0;
+	if (dvp2axi_dev->hw_dev->is_eic770xs2) {
+		if (dvp2axi_dev->csi_host_idx == 0)
+			dvp2axi_dev->csi_host_idx = 2;
+		else if (dvp2axi_dev->csi_host_idx == 2)
+			dvp2axi_dev->csi_host_idx = 4;
+		else if (dvp2axi_dev->csi_host_idx == 3)
+			dvp2axi_dev->csi_host_idx = 5;
+		v4l2_info(&dvp2axi_dev->v4l2_dev, "eic770xs2 attach to mipi%d\n",
+			  dvp2axi_dev->csi_host_idx);
+	}
+	dvp2axi_dev->csi_host_idx_def = dvp2axi_dev->csi_host_idx;
+	
+	dvp2axi_dev->media_dev = es_vi_dev->media_dev;
+	v4l2_dev = &dvp2axi_dev->v4l2_dev;
+	v4l2_dev->mdev = dvp2axi_dev->media_dev;
+	strlcpy(v4l2_dev->name, dev_name(dev), sizeof(v4l2_dev->name));
+	ret = v4l2_device_register(dvp2axi_dev->dev, &dvp2axi_dev->v4l2_dev);
+	if (ret < 0)
+		return ret;
+
+
+	// media_device_init(&dvp2axi_dev->media_dev);
+	// ret = media_device_register(&dvp2axi_dev->media_dev);
+	// if (ret < 0) {
+	// 	v4l2_err(v4l2_dev, "Failed to register media device: %d\n",
+	// 		 ret);
+	// 	goto err_unreg_v4l2_dev;
+	// }
+	/* create & register platefom subdev (from of_node) */
+	ret = es_dvp2axi_register_platform_subdevs(dvp2axi_dev);
+	// if (ret < 0)
+	// 	goto err_unreg_media_dev;
+
+	mutex_lock(&es_dvp2axi_dev_mutex);
+	list_add_tail(&dvp2axi_dev->list, &es_dvp2axi_device_list);
+	mutex_unlock(&es_dvp2axi_dev_mutex);
+	return 0;
+}
+
+int es_dvp2axi_plat_uninit(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	int stream_num = 0;
+
+	if (dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_CCP2)
+		es_dvp2axi_unregister_lvds_subdev(dvp2axi_dev);
+
+	if (dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_BT656 ||
+	    dvp2axi_dev->active_sensor->mbus.type == V4L2_MBUS_PARALLEL)
+		es_dvp2axi_unregister_dvp_sof_subdev(dvp2axi_dev);
+
+	// media_device_unregister(&dvp2axi_dev->media_dev);
+	v4l2_device_unregister(&dvp2axi_dev->v4l2_dev);
+
+	stream_num = ES_DVP2AXI_MAX_STREAM_MIPI;
+	es_dvp2axi_unregister_stream_vdevs(dvp2axi_dev, stream_num);
+	return 0;
+}
+
+static const struct es_dvp2axi_match_data es_dvp2axi_dvp_match_data = {
+	.inf_id = ES_DVP2AXI_DVP,
+};
+
+static const struct es_dvp2axi_match_data es_dvp2axi_mipi_lvds_match_data = {
+	.inf_id = ES_DVP2AXI_MIPI_LVDS,
+};
+
+static const struct of_device_id es_dvp2axi_plat_of_match[] = {
+	{
+		.compatible = "eswin,dvp2axi-mipi-lvds",
+		.data = &es_dvp2axi_mipi_lvds_match_data,
+	},
+	{},
+};
+
+static void es_dvp2axi_parse_dts(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	int ret = 0;
+	struct device_node *node = dvp2axi_dev->dev->of_node;
+
+	ret = of_property_read_u32(node, OF_DVP2AXI_WAIT_LINE, &dvp2axi_dev->wait_line);
+	if (ret != 0)
+		dvp2axi_dev->wait_line = 0;
+	dev_info(dvp2axi_dev->dev, "es_dvp2axi wait line %d\n", dvp2axi_dev->wait_line);
+}
+
+static int es_dvp2axi_plat_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	struct device_node *node = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct es_dvp2axi_device *dvp2axi_dev;
+	const struct es_dvp2axi_match_data *data;
+	int ret;
+
+	sprintf(es_dvp2axi_version, "v%02x.%02x.%02x", ES_DVP2AXI_DRIVER_VERSION >> 16,
+		(ES_DVP2AXI_DRIVER_VERSION & 0xff00) >> 8,
+		ES_DVP2AXI_DRIVER_VERSION & 0x00ff);
+
+	dev_info(dev, "es_dvp2axi driver version: %s\n", es_dvp2axi_version);
+
+	match = of_match_node(es_dvp2axi_plat_of_match, node);
+	if (IS_ERR(match))
+		return PTR_ERR(match);
+	data = match->data;
+
+	dvp2axi_dev = devm_kzalloc(dev, sizeof(*dvp2axi_dev), GFP_KERNEL);
+	if (!dvp2axi_dev)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, dvp2axi_dev);
+	dvp2axi_dev->dev = dev;
+
+	if (sysfs_create_group(&pdev->dev.kobj, &dev_attr_grp))
+		return -ENODEV;
+	ret = es_dvp2axi_attach_hw(dvp2axi_dev);
+	if (ret)
+		return ret;
+	es_dvp2axi_parse_dts(dvp2axi_dev);
+	ret = es_dvp2axi_plat_init(dvp2axi_dev, node, data->inf_id);
+	if (ret) {
+		es_dvp2axi_detach_hw(dvp2axi_dev);
+		return ret;
+	}
+
+	if (es_dvp2axi_proc_init(dvp2axi_dev))
+		dev_warn(dev, "dev:%s create proc failed\n", dev_name(dev));
+	es_dvp2axi_init_reset_monitor(dvp2axi_dev);
+	pm_runtime_enable(&pdev->dev);
+	pr_info("%s succsess\n", __func__);
+	return 0;
+}
+
+static int es_dvp2axi_plat_remove(struct platform_device *pdev)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = platform_get_drvdata(pdev);
+
+	es_dvp2axi_plat_uninit(dvp2axi_dev);
+	es_dvp2axi_detach_hw(dvp2axi_dev);
+	es_dvp2axi_proc_cleanup(dvp2axi_dev);
+	sysfs_remove_group(&pdev->dev.kobj, &dev_attr_grp);
+	del_timer_sync(&dvp2axi_dev->reset_watchdog_timer.timer);
+
+	return 0;
+}
+
+static int __maybe_unused es_dvp2axi_sleep_suspend(struct device *dev)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = dev_get_drvdata(dev);
+
+	es_dvp2axi_stream_suspend(dvp2axi_dev, ES_DVP2AXI_RESUME_DVP2AXI);
+	return 0;
+}
+
+static int __maybe_unused es_dvp2axi_sleep_resume(struct device *dev)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = dev_get_drvdata(dev);
+
+	es_dvp2axi_stream_resume(dvp2axi_dev, ES_DVP2AXI_RESUME_DVP2AXI);
+	return 0;
+}
+
+static int __maybe_unused es_dvp2axi_runtime_suspend(struct device *dev)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (atomic_dec_return(&dvp2axi_dev->power_cnt))
+		return 0;
+
+	mutex_lock(&dvp2axi_dev->hw_dev->dev_lock);
+	ret = pm_runtime_put_sync(dvp2axi_dev->hw_dev->dev);
+	mutex_unlock(&dvp2axi_dev->hw_dev->dev_lock);
+	return (ret > 0) ? 0 : ret;
+}
+
+static int __maybe_unused es_dvp2axi_runtime_resume(struct device *dev)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (atomic_inc_return(&dvp2axi_dev->power_cnt) > 1)
+		return 0;
+
+	mutex_lock(&dvp2axi_dev->hw_dev->dev_lock);
+	ret = pm_runtime_resume_and_get(dvp2axi_dev->hw_dev->dev);
+	mutex_unlock(&dvp2axi_dev->hw_dev->dev_lock);
+	es_dvp2axi_do_soft_reset(dvp2axi_dev);
+	return (ret > 0) ? 0 : ret;
+}
+
+static int __maybe_unused __es_dvp2axi_clr_unready_dev(void)
+{
+	struct es_dvp2axi_device *dvp2axi_dev;
+
+	mutex_lock(&es_dvp2axi_dev_mutex);
+
+	list_for_each_entry(dvp2axi_dev, &es_dvp2axi_device_list, list) {
+		// v4l2_async_notifier_clr_unready_dev(&dvp2axi_dev->notifier);
+		v4l2_async_nf_cleanup(&dvp2axi_dev->notifier);
+		subdev_asyn_register_itf(dvp2axi_dev);
+	}
+
+	mutex_unlock(&es_dvp2axi_dev_mutex);
+
+	return 0;
+}
+
+static int es_dvp2axi_clr_unready_dev_param_set(const char *val,
+					   const struct kernel_param *kp)
+{
+#ifdef MODULE
+	__es_dvp2axi_clr_unready_dev();
+#endif
+
+	return 0;
+}
+
+module_param_call(clr_unready_dev, es_dvp2axi_clr_unready_dev_param_set, NULL, NULL,
+		  0200);
+MODULE_PARM_DESC(clr_unready_dev, "clear unready devices");
+
+#ifndef MODULE
+int es_dvp2axi_clr_unready_dev(void)
+{
+	__es_dvp2axi_clr_unready_dev();
+
+	return 0;
+}
+#ifndef CONFIG_VIDEO_REVERSE_IMAGE
+late_initcall(es_dvp2axi_clr_unready_dev);
+#endif
+#endif
+
+static const struct dev_pm_ops es_dvp2axi_plat_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(es_dvp2axi_sleep_suspend, es_dvp2axi_sleep_resume)
+		SET_RUNTIME_PM_OPS(es_dvp2axi_runtime_suspend, es_dvp2axi_runtime_resume,
+				   NULL)
+};
+
+struct platform_driver es_dvp2axi_plat_drv = {
+	.driver = {
+		.name = DVP2AXI_DRIVER_NAME,
+		.of_match_table = of_match_ptr(es_dvp2axi_plat_of_match),
+		.pm = &es_dvp2axi_plat_pm_ops,
+	},
+	.probe = es_dvp2axi_plat_probe,
+	.remove = es_dvp2axi_plat_remove,
+};
+EXPORT_SYMBOL(es_dvp2axi_plat_drv);
+
+MODULE_AUTHOR("Eswin VI team");
+MODULE_DESCRIPTION("Eswin DVP2AXI platform driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/eswin/dvp2axi/dev.h b/drivers/media/platform/eswin/dvp2axi/dev.h
new file mode 100644
index 000000000000..c1590db40769
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dev.h
@@ -0,0 +1,1062 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI dev driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.yy
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_DEV_H
+#define _ES_DVP2AXI_DEV_H
+
+#include <linux/mutex.h>
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-mc.h>
+#include <linux/workqueue.h>
+#include <media/eswin/eswin_vi.h>
+#include "../../../../phy/eswin/es-camera-module.h"
+#include "../../../../phy/eswin/es-dvp2axi-config.h"
+
+#include "regs.h"
+#include "hw.h"
+
+#define DVP2AXI_DRIVER_NAME		"es_dvp2axi"
+#define DVP2AXI_DRIVER_NAME_D1		"es_dvp2axi_d1"
+#define DVP2AXI_VIDEODEVICE_NAME	"stream_dvp2axi"
+
+#define OF_DVP2AXI_MONITOR_PARA	"eic770x,dvp2axi-monitor"
+#define OF_DVP2AXI_WAIT_LINE	"wait-line"
+
+#define DVP2AXI_MONITOR_PARA_NUM	(5)
+
+#define ES_DVP2AXI_SINGLE_STREAM	1
+#define ESDVP2AXI_MULTI_STREAM 6
+#define ESDVP2AXI_MAX_STREAM_MIPI 6
+#define ESDVP2AXI_STREAM_MIPI_ID0 0
+#define ESDVP2AXI_STREAM_MIPI_ID1 1
+#define ESDVP2AXI_STREAM_MIPI_ID2 2
+#define ESDVP2AXI_STREAM_MIPI_ID3 3
+#define ESDVP2AXI_STREAM_MIPI_ID4 4
+#define ESDVP2AXI_STREAM_MIPI_ID5 5
+
+#define ESDVP2AXI_STREAM_DVP	6
+
+#define ES_DVP2AXI_STREAM_DVP2AXI	0
+#define DVP2AXI_DVP_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME		"_dvp"
+#define DVP2AXI_MIPI_ID0_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_mipi_id0"
+#define DVP2AXI_MIPI_ID1_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_mipi_id1"
+#define DVP2AXI_MIPI_ID2_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_mipi_id2"
+#define DVP2AXI_MIPI_ID3_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_mipi_id3"
+#define DVP2AXI_MIPI_ID4_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_mipi_id4"
+#define DVP2AXI_MIPI_ID5_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_mipi_id5"
+
+#define DVP2AXI_DVP_ID0_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_dvp_id0"
+#define DVP2AXI_DVP_ID1_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_dvp_id1"
+#define DVP2AXI_DVP_ID2_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_dvp_id2"
+#define DVP2AXI_DVP_ID3_VDEV_NAME DVP2AXI_VIDEODEVICE_NAME	"_dvp_id3"
+
+#define ES_DVP2AXI_PLANE_Y		0
+#define ES_DVP2AXI_PLANE_CBCR	1
+
+/*
+ * ES1808 support 5 channel inputs simultaneously:
+ * dvp + 4 mipi virtual channels;
+ * RV1126/ES356X support 4 channels of BT.656/BT.1120/MIPI
+ */
+#define ES_DVP2AXI_MULTI_STREAMS_NUM	5
+#define ES_DVP2AXI_STREAM_MIPI_ID0	0
+#define ES_DVP2AXI_STREAM_MIPI_ID1	1
+#define ES_DVP2AXI_STREAM_MIPI_ID2	2
+#define ES_DVP2AXI_STREAM_MIPI_ID3	3
+// #define ES_DVP2AXI_MAX_STREAM_MIPI	4
+#define ES_DVP2AXI_MAX_STREAM_MIPI	1
+#define ES_DVP2AXI_MAX_STREAM_LVDS	4
+#define ES_DVP2AXI_MAX_STREAM_DVP	4
+#define ES_DVP2AXI_STREAM_DVP	4
+
+#define ES_DVP2AXI_MAX_SENSOR	6
+#define ES_DVP2AXI_MAX_CSI_CHANNEL	4
+#define ES_DVP2AXI_MAX_PIPELINE	4
+
+#define ES_DVP2AXI_DEFAULT_WIDTH	3280
+#define ES_DVP2AXI_DEFAULT_HEIGHT	2464
+#define ES_DVP2AXI_FS_DETECTED_NUM	2
+
+#define ES_DVP2AXI_MAX_INTERVAL_NS	5000000
+/*
+ * for HDR mode sync buf
+ */
+#define RDBK_MAX		3
+#define RDBK_TOISP_MAX		2
+#define RDBK_L			0
+#define RDBK_M			1
+#define RDBK_S			2
+
+/*
+ * for distinguishing cropping from senosr or usr
+ */
+#define CROP_SRC_SENSOR_MASK		(0x1 << 0)
+#define CROP_SRC_USR_MASK		(0x1 << 1)
+
+/*
+ * max wait time for stream stop
+ */
+#define ES_DVP2AXI_STOP_MAX_WAIT_TIME_MS	(500)
+
+#define ES_DVP2AXI_SKIP_FRAME_MAX		(16)
+
+enum es_dvp2axi_workmode {
+	ES_DVP2AXI_WORKMODE_ONEFRAME = 0x00,
+	ES_DVP2AXI_WORKMODE_PINGPONG = 0x01,
+	ES_DVP2AXI_WORKMODE_LINELOOP = 0x02
+};
+
+enum es_dvp2axi_stream_mode {
+	ES_DVP2AXI_STREAM_MODE_NONE       = 0x0,
+	ES_DVP2AXI_STREAM_MODE_CAPTURE    = 0x01,
+	ES_DVP2AXI_STREAM_MODE_TOISP      = 0x02,
+	ES_DVP2AXI_STREAM_MODE_TOSCALE    = 0x04,
+	ES_DVP2AXI_STREAM_MODE_TOISP_RDBK = 0x08,
+	ES_DVP2AXI_STREAM_MODE_ESKIT     = 0x10
+};
+
+enum es_dvp2axi_yuvaddr_state {
+	ES_DVP2AXI_YUV_ADDR_STATE_UPDATE = 0x0,
+	ES_DVP2AXI_YUV_ADDR_STATE_INIT = 0x1
+};
+
+enum es_dvp2axi_state {
+	ES_DVP2AXI_STATE_DISABLED,
+	ES_DVP2AXI_STATE_READY,
+	ES_DVP2AXI_STATE_STREAMING,
+	ES_DVP2AXI_STATE_RESET_IN_STREAMING,
+};
+
+enum es_dvp2axi_lvds_pad {
+	ES_DVP2AXI_LVDS_PAD_SINK = 0x0,
+	ES_DVP2AXI_LVDS_PAD_SRC_ID0,
+	ES_DVP2AXI_LVDS_PAD_SRC_ID1,
+	ES_DVP2AXI_LVDS_PAD_SRC_ID2,
+	ES_DVP2AXI_LVDS_PAD_SRC_ID3,
+	ES_DVP2AXI_LVDS_PAD_SCL_ID0,
+	ES_DVP2AXI_LVDS_PAD_SCL_ID1,
+	ES_DVP2AXI_LVDS_PAD_SCL_ID2,
+	ES_DVP2AXI_LVDS_PAD_SCL_ID3,
+	ES_DVP2AXI_LVDS_PAD_MAX,
+};
+
+enum es_dvp2axi_lvds_state {
+	ES_DVP2AXI_LVDS_STOP = 0,
+	ES_DVP2AXI_LVDS_START,
+};
+
+enum es_dvp2axi_inf_id {
+	ES_DVP2AXI_DVP,
+	ES_DVP2AXI_MIPI_LVDS,
+};
+
+enum es_dvp2axi_clk_edge {
+	ES_DVP2AXI_CLK_RISING = 0x0,
+	ES_DVP2AXI_CLK_FALLING,
+};
+
+/*
+ * for distinguishing cropping from senosr or usr
+ */
+enum es_dvp2axi_crop_src {
+	CROP_SRC_ACT	= 0x0,
+	CROP_SRC_SENSOR,
+	CROP_SRC_USR,
+	CROP_SRC_MAX
+};
+
+/*
+ * struct es_dvp2axi_pipeline - An DVP2AXI hardware pipeline
+ *
+ * Capture device call other devices via pipeline
+ *
+ * @num_subdevs: number of linked subdevs
+ * @power_cnt: pipeline power count
+ * @stream_cnt: stream power count
+ */
+struct es_dvp2axi_pipeline {
+	struct media_pipeline pipe;
+	int num_subdevs;
+	atomic_t power_cnt;
+	atomic_t stream_cnt;
+	struct v4l2_subdev *subdevs[ES_DVP2AXI_MAX_PIPELINE];
+	int (*open)(struct es_dvp2axi_pipeline *p,
+		    struct media_entity *me, bool prepare);
+	int (*close)(struct es_dvp2axi_pipeline *p);
+	int (*set_stream)(struct es_dvp2axi_pipeline *p, bool on);
+};
+
+struct es_dvp2axi_buffer {
+	struct vb2_v4l2_buffer vb;
+	struct list_head queue;
+	union {
+		u32 buff_addr[VIDEO_MAX_PLANES];
+		void *vaddr[VIDEO_MAX_PLANES];
+	};
+	struct dma_buf *dbuf;
+	u64 fe_timestamp;
+};
+
+struct es_dvp2axi_tools_buffer {
+	struct vb2_v4l2_buffer *vb;
+	struct list_head list;
+	u32 frame_idx;
+	u64 timestamp;
+	int use_cnt;
+};
+
+extern int es_dvp2axi_debug;
+
+/*
+ * struct es_dvp2axi_sensor_info - Sensor infomations
+ * @sd: v4l2 subdev of sensor
+ * @mbus: media bus configuration
+ * @fi: v4l2 subdev frame interval
+ * @lanes: lane num of sensor
+ * @raw_rect: raw output rectangle of sensor, not crop or selection
+ * @selection: selection info of sensor
+ */
+struct es_dvp2axi_sensor_info {
+	struct v4l2_subdev *sd;
+	struct v4l2_mbus_config mbus;
+	struct v4l2_subdev_frame_interval fi;
+	int lanes;
+	struct v4l2_rect raw_rect;
+	struct v4l2_subdev_selection selection;
+	int dsi_input_en;
+};
+
+enum dvp2axi_fmt_type {
+	DVP2AXI_FMT_TYPE_YUV = 0,
+	DVP2AXI_FMT_TYPE_RAW,
+};
+
+/*
+ * struct dvp2axi_output_fmt - The output format
+ *
+ * @bpp: bits per pixel for each cplanes
+ * @fourcc: pixel format in fourcc
+ * @fmt_val: the fmt val corresponding to DVP2AXI_FOR register
+ * @csi_fmt_val: the fmt val corresponding to DVP2AXI_CSI_ID_CTRL
+ * @cplanes: number of colour planes
+ * @mplanes: number of planes for format
+ * @raw_bpp: bits per pixel for raw format
+ * @fmt_type: image format, raw or yuv
+ */
+struct dvp2axi_output_fmt {
+	u8 bpp[VIDEO_MAX_PLANES];
+	u32 fourcc;
+	u32 fmt_val;
+	u32 csi_fmt_val;
+	u8 cplanes;
+	u8 mplanes;
+	u8 raw_bpp;
+	enum dvp2axi_fmt_type fmt_type;
+};
+
+/*
+ * struct dvp2axi_input_fmt - The input mbus format from sensor
+ *
+ * @mbus_code: mbus format
+ * @dvp_fmt_val: the fmt val corresponding to DVP2AXI_FOR register
+ * @csi_fmt_val: the fmt val corresponding to DVP2AXI_CSI_ID_CTRL
+ * @fmt_type: image format, raw or yuv
+ * @field: the field type of the input from sensor
+ */
+struct dvp2axi_input_fmt {
+	u32 mbus_code;
+	u32 dvp_fmt_val;
+	u32 csi_fmt_val;
+	u32 csi_yuv_order;
+	enum dvp2axi_fmt_type fmt_type;
+	enum v4l2_field field;
+};
+
+struct csi_channel_info {
+	unsigned char id;
+	unsigned char enable;	/* capture enable */
+	unsigned char vc;
+	unsigned char data_type;
+	unsigned char data_bit;
+	unsigned char crop_en;
+	unsigned char cmd_mode_en;
+	unsigned char fmt_val;
+	unsigned char csi_fmt_val;
+	unsigned int width;
+	unsigned int height;
+	unsigned int virtual_width;
+	unsigned int left_virtual_width;
+	unsigned int crop_st_x;
+	unsigned int crop_st_y;
+	unsigned int dsi_input;
+	struct esmodule_lvds_cfg lvds_cfg;
+	struct esmodule_capture_info capture_info;
+};
+
+struct es_dvp2axi_vdev_node {
+	struct vb2_queue buf_queue;
+	/* vfd lock */
+	struct mutex vlock;
+	struct video_device vdev;
+	struct media_pad pad;
+};
+
+/*
+ * the mark that csi frame0/1 interrupts enabled
+ * in DVP2AXI_MIPI_INTEN
+ */
+enum dvp2axi_frame_ready {
+	DVP2AXI_CSI_FRAME0_READY = 0,
+	DVP2AXI_CSI_FRAME1_READY,
+	DVP2AXI_CSI_FRAME2_READY,
+	DVP2AXI_CSI_FRAME_UNREADY
+};
+
+/* struct es_dvp2axi_hdr - hdr configured
+ * @op_mode: hdr optional mode
+ */
+struct es_dvp2axi_hdr {
+	u8 mode;
+};
+
+/* struct es_dvp2axi_fps_stats - take notes on timestamp of buf
+ * @frm0_timestamp: timesstamp of buf in frm0
+ * @frm1_timestamp: timesstamp of buf in frm1
+ */
+struct es_dvp2axi_fps_stats {
+	u64 frm0_timestamp;
+	u64 frm1_timestamp;
+};
+
+/* struct es_dvp2axi_fps_stats - take notes on timestamp of buf
+ * @fs_timestamp: timesstamp of frame start
+ * @fe_timestamp: timesstamp of frame end
+ * @wk_timestamp: timesstamp of buf send to user in wake up mode
+ * @readout_time: one frame of readout time
+ * @early_time: early time of buf send to user
+ * @total_time: totaltime of readout time in hdr
+ */
+struct es_dvp2axi_readout_stats {
+	u64 fs_timestamp;
+	u64 fe_timestamp;
+	u64 wk_timestamp;
+	u64 readout_time;
+	u64 early_time;
+	u64 total_time;
+};
+
+/* struct es_dvp2axi_irq_stats - take notes on irq number
+ * @csi_overflow_cnt: count of csi overflow irq
+ * @csi_bwidth_lack_cnt: count of csi bandwidth lack irq
+ * @dvp_bus_err_cnt: count of dvp bus err irq
+ * @dvp_overflow_cnt: count dvp overflow irq
+ * @dvp_line_err_cnt: count dvp line err irq
+ * @dvp_pix_err_cnt: count dvp pix err irq
+ * @all_frm_end_cnt: raw frame end count
+ * @all_err_cnt: all err count
+ * @
+ */
+struct es_dvp2axi_irq_stats {
+	u64 csi_overflow_cnt;
+	u64 csi_bwidth_lack_cnt;
+	u64 csi_size_err_cnt;
+	u64 dvp_bus_err_cnt;
+	u64 dvp_overflow_cnt;
+	u64 dvp_line_err_cnt;
+	u64 dvp_pix_err_cnt;
+	u64 dvp_size_err_cnt;
+	u64 dvp_bwidth_lack_cnt;
+	u64 frm_end_cnt[ES_DVP2AXI_MAX_STREAM_MIPI];
+	u64 not_active_buf_cnt[ES_DVP2AXI_MAX_STREAM_MIPI];
+	u64 trig_simult_cnt[ES_DVP2AXI_MAX_STREAM_MIPI];
+	u64 all_err_cnt;
+};
+
+/*
+ * the detecting mode of dvp2axi reset timer
+ * related with dts property:eic770x,dvp2axi-monitor
+ */
+enum es_dvp2axi_monitor_mode {
+	ES_DVP2AXI_MONITOR_MODE_IDLE = 0x0,
+	ES_DVP2AXI_MONITOR_MODE_CONTINUE,
+	ES_DVP2AXI_MONITOR_MODE_TRIGGER,
+	ES_DVP2AXI_MONITOR_MODE_HOTPLUG,
+};
+
+/*
+ * the parameters to resume when reset dvp2axi in running
+ */
+struct es_dvp2axi_resume_info {
+	u32 frm_sync_seq;
+};
+
+struct es_dvp2axi_work_struct {
+	struct work_struct	work;
+	enum esmodule_reset_src	reset_src;
+	struct es_dvp2axi_resume_info	resume_info;
+};
+
+struct es_dvp2axi_timer {
+	struct timer_list	timer;
+	spinlock_t		timer_lock;
+	spinlock_t		csi2_err_lock;
+	unsigned long		cycle;
+	/* unit: us */
+	unsigned long		line_end_cycle;
+	unsigned int		run_cnt;
+	unsigned int		max_run_cnt;
+	unsigned int		stop_index_of_run_cnt;
+	unsigned int		last_buf_wakeup_cnt[ES_DVP2AXI_MAX_STREAM_MIPI];
+	unsigned long		csi2_err_cnt_even;
+	unsigned long		csi2_err_cnt_odd;
+	unsigned int		csi2_err_ref_cnt;
+	unsigned int		csi2_err_fs_fe_cnt;
+	unsigned int		csi2_err_fs_fe_detect_cnt;
+	unsigned int		frm_num_of_monitor_cycle;
+	unsigned int		triggered_frame_num;
+	unsigned int		vts;
+	unsigned int		raw_height;
+	/* unit: ms */
+	unsigned int		err_time_interval;
+	unsigned int		csi2_err_triggered_cnt;
+	unsigned int		notifer_called_cnt;
+	unsigned long		frame_end_cycle_us;
+	u64			csi2_first_err_timestamp;
+	bool			is_triggered;
+	bool			is_buf_stop_update;
+	bool			is_running;
+	bool			is_csi2_err_occurred;
+	bool			has_been_init;
+	bool			is_ctrl_by_user;
+	enum es_dvp2axi_monitor_mode	monitor_mode;
+	enum esmodule_reset_src	reset_src;
+};
+
+struct es_dvp2axi_extend_info {
+	struct v4l2_pix_format_mplane	pixm;
+	bool is_extended;
+};
+
+enum es_dvp2axi_capture_mode {
+	ES_DVP2AXI_TO_DDR = 0,
+	ES_DVP2AXI_TO_ISP_DDR,
+	ES_DVP2AXI_TO_ISP_DMA,
+};
+
+/*
+ * list: used for buf rotation
+ * list_free: only used to release buf asynchronously
+ */
+struct es_dvp2axi_rx_buffer {
+	int buf_idx;
+	struct list_head list;
+	struct list_head list_free;
+	struct es_dvp2axi_dummy_buffer dummy;
+	u64 fe_timestamp;
+};
+
+enum es_dvp2axi_dma_en_mode {
+	ES_DVP2AXI_DMAEN_BY_VICAP = 0x1,
+	ES_DVP2AXI_DMAEN_BY_ISP = 0x2,
+	ES_DVP2AXI_DMAEN_BY_VICAP_TO_ISP = 0x4,
+	ES_DVP2AXI_DMAEN_BY_ISP_TO_VICAP = 0x8,
+	ES_DVP2AXI_DMAEN_BY_ESKIT = 0x10,
+};
+
+struct es_dvp2axi_skip_info {
+	u8 cap_m;
+	u8 skip_n;
+	bool skip_en;
+	bool skip_to_en;
+	bool skip_to_dis;
+};
+
+struct es_dvp2axi_sync_cfg {
+	u32 type;
+	u32 group;
+};
+
+enum es_dvp2axi_toisp_buf_update_state {
+	ES_DVP2AXI_TOISP_BUF_ROTATE,
+	ES_DVP2AXI_TOISP_BUF_THESAME,
+	ES_DVP2AXI_TOISP_BUF_LOSS,
+};
+
+struct es_dvp2axi_toisp_buf_state {
+	enum es_dvp2axi_toisp_buf_update_state state;
+	int check_cnt;
+	bool is_early_update;
+};
+
+/*
+ * struct es_dvp2axi_stream - Stream states TODO
+ *
+ * @vbq_lock: lock to protect buf_queue
+ * @buf_queue: queued buffer list
+ * @dummy_buf: dummy space to store dropped data
+ * @crop_enable: crop status when stream off
+ * @crop_dyn_en: crop status when streaming
+ * es_dvp2axi use shadowsock registers, so it need two buffer at a time
+ * @curr_buf: the buffer used for current frame
+ * @next_buf: the buffer used for next frame
+ * @fps_lock: to protect parameters about calculating fps
+ */
+struct es_dvp2axi_stream {
+	unsigned id:3;
+	struct es_dvp2axi_device		*dvp2axidev;
+	struct es_dvp2axi_vdev_node		vnode;
+	enum es_dvp2axi_state		state;
+	wait_queue_head_t		wq_stopped;
+	unsigned int			frame_idx;
+	enum dvp2axi_frame_ready	frame_phase;
+	int				frame_phase_cache;
+	unsigned int			crop_mask;
+	/* lock between irq and buf_queue */
+	struct list_head		buf_head;
+	struct es_dvp2axi_buffer		*curr_buf;
+	struct es_dvp2axi_buffer		*next_buf;
+	struct es_dvp2axi_buffer		*last_buf;
+	struct es_dvp2axi_rx_buffer		*curr_buf_toisp;
+	struct es_dvp2axi_rx_buffer		*next_buf_toisp;
+	struct es_dvp2axi_rx_buffer		*last_buf_toisp;
+	struct list_head		eskit_buf_head;
+	struct es_dvp2axi_buffer		*curr_buf_eskit;
+	struct es_dvp2axi_buffer		*next_buf_eskit;
+
+	spinlock_t vbq_lock; /* vfd lock */
+	spinlock_t fps_lock;
+	/* TODO: pad for dvp and mipi separately? */
+	struct media_pad		pad;
+
+	const struct dvp2axi_output_fmt	*dvp2axi_fmt_out;
+	const struct dvp2axi_input_fmt	*dvp2axi_fmt_in;
+	struct v4l2_pix_format_mplane	pixm;
+	struct v4l2_rect		crop[CROP_SRC_MAX];
+	struct es_dvp2axi_fps_stats		fps_stats;
+	struct es_dvp2axi_extend_info	extend_line;
+	struct es_dvp2axi_readout_stats	readout;
+	unsigned int			fs_cnt_in_single_frame;
+	unsigned int			capture_mode;
+	struct es_dvp2axi_scale_vdev		*scale_vdev;
+	struct es_dvp2axi_tools_vdev		*tools_vdev;
+	int				dma_en;
+	int				to_en_dma;
+	int				to_stop_dma;
+	int				buf_owner;
+	int				buf_replace_cnt;
+	struct list_head		rx_buf_head_vicap;
+	unsigned int			cur_stream_mode;
+	struct list_head		rx_buf_head;
+	int				total_buf_num;
+	int				rx_buf_num;
+	u64				line_int_cnt;
+	int				lack_buf_cnt;
+	unsigned int			buf_wake_up_cnt;
+	struct es_dvp2axi_skip_info		skip_info;
+	struct tasklet_struct		vb_done_tasklet;
+	struct tasklet_struct		dvp2axi_err_tasklet;
+	struct list_head		vb_done_list;
+	int				last_rx_buf_idx;
+	int				last_frame_idx;
+	int				new_fource_idx;
+	atomic_t			buf_cnt;
+	struct completion		stop_complete;
+	struct es_dvp2axi_toisp_buf_state	toisp_buf_state;
+	u32				skip_frame;
+	u32				cur_skip_frame;
+	bool				stopping;
+	bool				crop_enable;
+	bool				crop_dyn_en;
+	bool				is_compact;
+	bool				is_dvp_yuv_addr_init;
+	bool				is_fs_fe_not_paired;
+	bool				is_line_wake_up;
+	bool				is_line_inten;
+	bool				is_can_stop;
+	bool				is_buf_active;
+	bool				is_high_align;
+	bool				to_en_scale;
+	bool				is_finish_stop_dma;
+	bool				is_in_vblank;
+	bool				is_change_toisp;
+	bool				is_stop_capture;
+	bool				is_wait_dma_stop;
+	bool				is_single_cap;
+	bool				is_wait_stop_complete;
+
+	bool 				is_first_flush;
+	bool				is_dummy_buffer;
+};
+
+struct es_dvp2axi_lvds_subdev {
+	struct es_dvp2axi_device	*dvp2axidev;
+	struct v4l2_subdev sd;
+	struct v4l2_subdev *remote_sd;
+	struct media_pad pads[ES_DVP2AXI_LVDS_PAD_MAX];
+	struct v4l2_mbus_framefmt in_fmt;
+	struct v4l2_rect crop;
+	const struct dvp2axi_output_fmt	*dvp2axi_fmt_out;
+	const struct dvp2axi_input_fmt	*dvp2axi_fmt_in;
+	enum es_dvp2axi_lvds_state		state;
+	struct es_dvp2axi_sensor_info	sensor_self;
+	atomic_t			frm_sync_seq;
+};
+
+struct es_dvp2axi_dvp_sof_subdev {
+	struct es_dvp2axi_device *dvp2axidev;
+	struct v4l2_subdev sd;
+	atomic_t			frm_sync_seq;
+};
+
+static inline struct es_dvp2axi_buffer *to_es_dvp2axi_buffer(struct vb2_v4l2_buffer *vb)
+{
+	return container_of(vb, struct es_dvp2axi_buffer, vb);
+}
+
+static inline
+struct es_dvp2axi_vdev_node *vdev_to_node(struct video_device *vdev)
+{
+	return container_of(vdev, struct es_dvp2axi_vdev_node, vdev);
+}
+
+static inline
+struct es_dvp2axi_stream *to_es_dvp2axi_stream(struct es_dvp2axi_vdev_node *vnode)
+{
+	return container_of(vnode, struct es_dvp2axi_stream, vnode);
+}
+
+static inline struct es_dvp2axi_vdev_node *queue_to_node(struct vb2_queue *q)
+{
+	return container_of(q, struct es_dvp2axi_vdev_node, buf_queue);
+}
+
+static inline struct vb2_queue *to_vb2_queue(struct file *file)
+{
+	struct es_dvp2axi_vdev_node *vnode = video_drvdata(file);
+
+	return &vnode->buf_queue;
+}
+
+#define SCALE_DRIVER_NAME		"es_dvp2axi_scale"
+
+#define ES_DVP2AXI_SCALE_CH0		0
+#define ES_DVP2AXI_SCALE_CH1		1
+#define ES_DVP2AXI_SCALE_CH2		2
+#define ES_DVP2AXI_SCALE_CH3		3
+#define ES_DVP2AXI_MAX_SCALE_CH	4
+
+#define DVP2AXI_SCALE_CH0_VDEV_NAME DVP2AXI_DRIVER_NAME	"_scale_ch0"
+#define DVP2AXI_SCALE_CH1_VDEV_NAME DVP2AXI_DRIVER_NAME	"_scale_ch1"
+#define DVP2AXI_SCALE_CH2_VDEV_NAME DVP2AXI_DRIVER_NAME	"_scale_ch2"
+#define DVP2AXI_SCALE_CH3_VDEV_NAME DVP2AXI_DRIVER_NAME	"_scale_ch3"
+
+#define ES_DVP2AXI_SCALE_ENUM_SIZE_MAX	3
+#define ES_DVP2AXI_MAX_SDITF			4
+
+enum scale_ch_sw {
+	SCALE_MIPI0_ID0,
+	SCALE_MIPI0_ID1,
+	SCALE_MIPI0_ID2,
+	SCALE_MIPI0_ID3,
+	SCALE_MIPI1_ID0,
+	SCALE_MIPI1_ID1,
+	SCALE_MIPI1_ID2,
+	SCALE_MIPI1_ID3,
+	SCALE_MIPI2_ID0,
+	SCALE_MIPI2_ID1,
+	SCALE_MIPI2_ID2,
+	SCALE_MIPI2_ID3,
+	SCALE_MIPI3_ID0,
+	SCALE_MIPI3_ID1,
+	SCALE_MIPI3_ID2,
+	SCALE_MIPI3_ID3,
+	SCALE_MIPI4_ID0,
+	SCALE_MIPI4_ID1,
+	SCALE_MIPI4_ID2,
+	SCALE_MIPI4_ID3,
+	SCALE_MIPI5_ID0,
+	SCALE_MIPI5_ID1,
+	SCALE_MIPI5_ID2,
+	SCALE_MIPI5_ID3,
+	SCALE_DVP,
+	SCALE_CH_MAX,
+};
+
+enum scale_mode {
+	SCALE_8TIMES,
+	SCALE_16TIMES,
+	SCALE_32TIMES,
+};
+
+struct es_dvp2axi_scale_ch_info {
+	u32 width;
+	u32 height;
+	u32 vir_width;
+};
+
+struct es_dvp2axi_scale_src_res {
+	u32 width;
+	u32 height;
+};
+
+/*
+ * struct es_dvp2axi_scale_vdev - DVP2AXI Capture device
+ *
+ * @irq_lock: buffer queue lock
+ * @stat: stats buffer list
+ * @readout_wq: workqueue for statistics information read
+ */
+struct es_dvp2axi_scale_vdev {
+	unsigned int ch:3;
+	struct es_dvp2axi_device *dvp2axidev;
+	struct es_dvp2axi_vdev_node vnode;
+	struct es_dvp2axi_stream *stream;
+	struct list_head buf_head;
+	spinlock_t vbq_lock; /* vfd lock */
+	wait_queue_head_t wq_stopped;
+	struct v4l2_pix_format_mplane	pixm;
+	const struct dvp2axi_output_fmt *scale_out_fmt;
+	struct es_dvp2axi_scale_ch_info ch_info;
+	struct es_dvp2axi_scale_src_res src_res;
+	struct es_dvp2axi_buffer *curr_buf;
+	struct es_dvp2axi_buffer *next_buf;
+	struct bayer_blc blc;
+	enum es_dvp2axi_state state;
+	unsigned int ch_src;
+	unsigned int scale_mode;
+	int frame_phase;
+	unsigned int frame_idx;
+	bool stopping;
+};
+
+static inline
+struct es_dvp2axi_scale_vdev *to_es_dvp2axi_scale_vdev(struct es_dvp2axi_vdev_node *vnode)
+{
+	return container_of(vnode, struct es_dvp2axi_scale_vdev, vnode);
+}
+
+
+#define TOOLS_DRIVER_NAME		"es_dvp2axi_tools"
+
+#define ES_DVP2AXI_TOOLS_CH0		0
+#define ES_DVP2AXI_TOOLS_CH1		1
+#define ES_DVP2AXI_TOOLS_CH2		2
+#define ES_DVP2AXI_MAX_TOOLS_CH	3
+
+#define DVP2AXI_TOOLS_CH0_VDEV_NAME DVP2AXI_DRIVER_NAME	"_tools_id0"
+#define DVP2AXI_TOOLS_CH1_VDEV_NAME DVP2AXI_DRIVER_NAME	"_tools_id1"
+#define DVP2AXI_TOOLS_CH2_VDEV_NAME DVP2AXI_DRIVER_NAME	"_tools_id2"
+
+/*
+ * struct es_dvp2axi_tools_vdev - DVP2AXI Capture device
+ *
+ * @irq_lock: buffer queue lock
+ * @stat: stats buffer list
+ * @readout_wq: workqueue for statistics information read
+ */
+struct es_dvp2axi_tools_vdev {
+	unsigned int ch:3;
+	struct es_dvp2axi_device *dvp2axidev;
+	struct es_dvp2axi_vdev_node vnode;
+	struct es_dvp2axi_stream *stream;
+	struct list_head buf_head;
+	struct list_head buf_done_head;
+	struct list_head src_buf_head;
+	spinlock_t vbq_lock; /* vfd lock */
+	wait_queue_head_t wq_stopped;
+	struct v4l2_pix_format_mplane	pixm;
+	const struct dvp2axi_output_fmt *tools_out_fmt;
+	struct es_dvp2axi_buffer *curr_buf;
+	struct work_struct work;
+	enum es_dvp2axi_state state;
+	int frame_phase;
+	unsigned int frame_idx;
+	bool stopping;
+};
+
+static inline
+struct es_dvp2axi_tools_vdev *to_es_dvp2axi_tools_vdev(struct es_dvp2axi_vdev_node *vnode)
+{
+	return container_of(vnode, struct es_dvp2axi_tools_vdev, vnode);
+}
+
+void es_dvp2axi_init_tools_vdev(struct es_dvp2axi_device *dvp2axi_dev, u32 ch);
+int es_dvp2axi_register_tools_vdevs(struct es_dvp2axi_device *dvp2axi_dev,
+				int stream_num,
+				bool is_multi_input);
+// void es_dvp2axi_unregister_tools_vdevs(struct es_dvp2axi_device *dvp2axi_dev,
+// 				   int stream_num);
+
+enum es_dvp2axi_err_state {
+	ES_DVP2AXI_ERR_ID0_NOT_BUF = 0x1,
+	ES_DVP2AXI_ERR_ID1_NOT_BUF = 0x2,
+	ES_DVP2AXI_ERR_ID2_NOT_BUF = 0x4,
+	ES_DVP2AXI_ERR_ID3_NOT_BUF = 0x8,
+	ES_DVP2AXI_ERR_ID0_TRIG_SIMULT = 0x10,
+	ES_DVP2AXI_ERR_ID1_TRIG_SIMULT = 0x20,
+	ES_DVP2AXI_ERR_ID2_TRIG_SIMULT = 0x40,
+	ES_DVP2AXI_ERR_ID3_TRIG_SIMULT = 0x80,
+	ES_DVP2AXI_ERR_SIZE = 0x100,
+	ES_DVP2AXI_ERR_OVERFLOW = 0x200,
+	ES_DVP2AXI_ERR_BANDWIDTH_LACK = 0x400,
+	ES_DVP2AXI_ERR_BUS = 0X800,
+	ES_DVP2AXI_ERR_ID0_MULTI_FS = 0x1000,
+	ES_DVP2AXI_ERR_ID1_MULTI_FS = 0x2000,
+	ES_DVP2AXI_ERR_ID2_MULTI_FS = 0x4000,
+	ES_DVP2AXI_ERR_ID3_MULTI_FS = 0x8000,
+	ES_DVP2AXI_ERR_PIXEL = 0x10000,
+	ES_DVP2AXI_ERR_LINE = 0x20000,
+};
+
+struct es_dvp2axi_err_state_work {
+	struct work_struct	work;
+	u64 last_timestamp;
+	u32 err_state;
+	u32 intstat;
+	u32 lastline;
+	u32 lastpixel;
+	u32 size_id0;
+	u32 size_id1;
+	u32 size_id2;
+	u32 size_id3;
+};
+
+enum es_dvp2axi_resume_user {
+	ES_DVP2AXI_RESUME_DVP2AXI,
+	ES_DVP2AXI_RESUME_ISP,
+};
+
+struct es_dvp2axi_sensor_work {
+	struct work_struct work;
+	int on;
+};
+
+/*
+ * struct es_dvp2axi_device - ISP platform device
+ * @base_addr: base register address
+ * @active_sensor: sensor in-use, set when streaming on
+ * @stream: capture video device
+ */
+struct es_dvp2axi_device {
+	struct list_head		list;
+	struct device			*dev;
+	struct v4l2_device		v4l2_dev;
+	struct media_device		*media_dev;
+	struct v4l2_async_notifier	notifier;
+
+	struct es_dvp2axi_sensor_info	sensors[ES_DVP2AXI_MAX_SENSOR];
+	u32				num_sensors;
+	struct es_dvp2axi_sensor_info	*active_sensor;
+	struct es_dvp2axi_sensor_info	terminal_sensor;
+
+	struct es_dvp2axi_stream		stream[ESDVP2AXI_MULTI_STREAM];
+	struct es_dvp2axi_scale_vdev		scale_vdev[ESDVP2AXI_MULTI_STREAM];
+	struct es_dvp2axi_tools_vdev		tools_vdev[ES_DVP2AXI_MAX_TOOLS_CH];
+	struct es_dvp2axi_pipeline		pipe;
+
+	struct csi_channel_info		channels[ES_DVP2AXI_MAX_CSI_CHANNEL];
+	int				num_channels;
+	int				chip_id;
+	atomic_t			stream_cnt;
+	atomic_t			power_cnt;
+	atomic_t			streamoff_cnt;
+	struct mutex			stream_lock; /* lock between streams */
+	struct mutex			scale_lock; /* lock between scale dev */
+	struct mutex			tools_lock; /* lock between tools dev */
+	enum es_dvp2axi_workmode		workmode;
+	bool				can_be_reset;
+	struct esmodule_hdr_cfg		hdr;
+	struct es_dvp2axi_buffer		*rdbk_buf[RDBK_MAX];
+	struct es_dvp2axi_rx_buffer		*rdbk_rx_buf[RDBK_MAX];
+	struct es_dvp2axi_lvds_subdev	lvds_subdev;
+	struct es_dvp2axi_dvp_sof_subdev	dvp_sof_subdev;
+	struct es_dvp2axi_hw *hw_dev;
+	irqreturn_t (*isr_hdl)(int irq, struct es_dvp2axi_device *dvp2axi_dev);
+	int inf_id;
+
+	struct proc_dir_entry		*proc_dir;
+	struct es_dvp2axi_irq_stats		irq_stats;
+	spinlock_t			hdr_lock; /* lock for hdr buf sync */
+	spinlock_t			buffree_lock;
+	struct es_dvp2axi_timer		reset_watchdog_timer;
+	struct es_dvp2axi_work_struct	reset_work;
+	int				id_use_cnt;
+	unsigned int			csi_host_idx;
+	unsigned int			csi_host_idx_def;
+	unsigned int			dvp_sof_in_oneframe;
+	unsigned int			wait_line;
+	unsigned int			wait_line_bak;
+	unsigned int			wait_line_cache;
+	struct completion		cmpl_ntf;
+	phys_addr_t			resmem_pa;
+	dma_addr_t			resmem_addr;
+	size_t				resmem_size;
+	// struct es_tb_client		tb_client;
+	bool				is_start_hdr;
+	bool				reset_work_cancel;
+	bool				iommu_en;
+	bool				is_use_dummybuf;
+	bool				is_notifier_isp;
+	bool				is_thunderboot;
+	bool				is_rdbk_to_online;
+	bool				is_support_tools;
+	bool				is_rtt_suspend;
+	bool				is_aov_reserved;
+	bool				sensor_state_change;
+	bool				is_toisp_reset;
+	int				rdbk_debug;
+	struct es_dvp2axi_sync_cfg		sync_cfg;
+	int				sditf_cnt;
+	u32				early_line;
+	int				isp_runtime_max;
+	int				sensor_linetime;
+	u32				err_state;
+	struct es_dvp2axi_err_state_work	err_state_work;
+	struct es_dvp2axi_sensor_work	sensor_work;
+	int				resume_mode;
+	u32				nr_buf_size;
+	u32				share_mem_size;
+	u32				thunderboot_sensor_num;
+	int				sensor_state;
+	u32				intr_mask;
+	struct delayed_work		work_deal_err;
+	u32						dvp2axi_id;
+};
+
+
+void dvp2axi_hw_soft_reset(struct es_dvp2axi_hw *es_dvp2axi_hw);
+
+void dvp2axi_hw_irq_mask(struct es_dvp2axi_hw *dvp2axi_hw, struct es_dvp2axi_stream *stream, int mask);
+
+extern struct platform_driver es_dvp2axi_plat_drv;
+#ifdef CONFIG_NUMA
+extern struct platform_driver es_dvp2axi_plat_drv_d1;
+#endif
+void es_dvp2axi_set_fps(struct es_dvp2axi_stream *stream, struct es_dvp2axi_fps *fps);
+int es_dvp2axi_do_start_stream(struct es_dvp2axi_stream *stream,
+				enum es_dvp2axi_stream_mode mode);
+void es_dvp2axi_do_stop_stream(struct es_dvp2axi_stream *stream,
+				enum es_dvp2axi_stream_mode mode);
+void es_dvp2axi_irq_handle_scale(struct es_dvp2axi_device *dvp2axi_dev,
+				  unsigned int intstat_glb);
+void es_dvp2axi_buf_queue(struct vb2_buffer *vb);
+
+void es_dvp2axi_vb_done_tasklet(struct es_dvp2axi_stream *stream, struct es_dvp2axi_buffer *buf);
+
+void dvp2axi_interrupt_handler(struct device *dev);
+
+void es_irq_oneframe(struct device *dev, struct es_dvp2axi_device *dvp2axi_dev);
+void es_irq_err_handle(struct device *dev, struct es_dvp2axi_device *dvp2axi_dev);
+
+void es_irq_oneframe_new(struct device *dev, struct es_dvp2axi_device *dvp2axi_dev);
+
+int es_dvp2axi_scale_start(struct es_dvp2axi_scale_vdev *scale_vdev);
+
+const struct
+dvp2axi_input_fmt *es_dvp2axi_get_input_fmt(struct es_dvp2axi_device *dev,
+				 struct v4l2_rect *rect,
+				 u32 pad_id, struct csi_channel_info *csi_info);
+
+void es_dvp2axi_write_register(struct es_dvp2axi_device *dev,
+			  enum dvp2axi_reg_index index, u32 val);
+void es_dvp2axi_write_register_or(struct es_dvp2axi_device *dev,
+			     enum dvp2axi_reg_index index, u32 val);
+void es_dvp2axi_write_register_and(struct es_dvp2axi_device *dev,
+			      enum dvp2axi_reg_index index, u32 val);
+unsigned int es_dvp2axi_read_register(struct es_dvp2axi_device *dev,
+				 enum dvp2axi_reg_index index);
+void es_dvp2axi_write_grf_reg(struct es_dvp2axi_device *dev,
+			 enum dvp2axi_reg_index index, u32 val);
+u32 es_dvp2axi_read_grf_reg(struct es_dvp2axi_device *dev,
+		       enum dvp2axi_reg_index index);
+void es_dvp2axi_unregister_stream_vdevs(struct es_dvp2axi_device *dev,
+				   int stream_num);
+int es_dvp2axi_register_stream_vdevs(struct es_dvp2axi_device *dev,
+				int stream_num,
+				bool is_multi_input);
+void es_dvp2axi_stream_init(struct es_dvp2axi_device *dev, u32 id);
+void es_dvp2axi_set_default_fmt(struct es_dvp2axi_device *dvp2axi_dev);
+void es_dvp2axi_irq_oneframe(struct es_dvp2axi_device *dvp2axi_dev);
+void es_dvp2axi_irq_pingpong(struct es_dvp2axi_device *dvp2axi_dev);
+void es_dvp2axi_irq_pingpong_v1(struct es_dvp2axi_device *dvp2axi_dev);
+unsigned int es_dvp2axi_irq_global(struct es_dvp2axi_device *dvp2axi_dev);
+void es_dvp2axi_irq_handle_toisp(struct es_dvp2axi_device *dvp2axi_dev, unsigned int intstat_glb);
+int es_dvp2axi_register_lvds_subdev(struct es_dvp2axi_device *dev);
+void es_dvp2axi_unregister_lvds_subdev(struct es_dvp2axi_device *dev);
+int es_dvp2axi_register_dvp_sof_subdev(struct es_dvp2axi_device *dev);
+void es_dvp2axi_unregister_dvp_sof_subdev(struct es_dvp2axi_device *dev);
+void es_dvp2axi_irq_lite_lvds(struct es_dvp2axi_device *dvp2axi_dev);
+int es_dvp2axi_plat_init(struct es_dvp2axi_device *dvp2axi_dev, struct device_node *node, int inf_id);
+int es_dvp2axi_plat_uninit(struct es_dvp2axi_device *dvp2axi_dev);
+int es_dvp2axi_attach_hw(struct es_dvp2axi_device *dvp2axi_dev);
+int es_dvp2axi_update_sensor_info(struct es_dvp2axi_stream *stream);
+int es_dvp2axi_reset_notifier(struct notifier_block *nb, unsigned long action, void *data);
+void es_dvp2axi_reset_watchdog_timer_handler(struct timer_list *t);
+void es_dvp2axi_config_dvp_clk_sampling_edge(struct es_dvp2axi_device *dev,
+					enum es_dvp2axi_clk_edge edge);
+void es_dvp2axi_enable_dvp_clk_dual_edge(struct es_dvp2axi_device *dev, bool on);
+void es_dvp2axi_reset_work(struct work_struct *work);
+
+void es_dvp2axi_vb_done_oneframe(struct es_dvp2axi_stream *stream,
+			    struct vb2_v4l2_buffer *vb_done);
+
+int es_dvp2axi_init_rx_buf(struct es_dvp2axi_stream *stream, int buf_num);
+
+int es_dvp2axi_set_fmt(struct es_dvp2axi_stream *stream,
+		       struct v4l2_pix_format_mplane *pixm,
+		       bool try);
+void es_dvp2axi_enable_dma_capture(struct es_dvp2axi_stream *stream, bool is_only_enable);
+
+void es_dvp2axi_do_soft_reset(struct es_dvp2axi_device *dev);
+
+u32 es_dvp2axi_mbus_pixelcode_to_v4l2(u32 pixelcode);
+
+void es_dvp2axi_config_dvp_pin(struct es_dvp2axi_device *dev, bool on);
+
+s32 es_dvp2axi_get_sensor_vblank_def(struct es_dvp2axi_device *dev);
+s32 es_dvp2axi_get_sensor_vblank(struct es_dvp2axi_device *dev);
+int es_dvp2axi_get_linetime(struct es_dvp2axi_stream *stream);
+
+void es_dvp2axi_assign_check_buffer_update_toisp(struct es_dvp2axi_stream *stream);
+
+
+int es_dvp2axi_clr_unready_dev(void);
+
+const struct
+dvp2axi_output_fmt *es_dvp2axi_find_output_fmt(struct es_dvp2axi_stream *stream, u32 pixelfmt);
+
+void es_dvp2axi_err_print_work(struct work_struct *work);
+int es_dvp2axi_stream_suspend(struct es_dvp2axi_device *dvp2axi_dev, int mode);
+int es_dvp2axi_stream_resume(struct es_dvp2axi_device *dvp2axi_dev, int mode);
+
+static inline u64 es_dvp2axi_time_get_ns(struct es_dvp2axi_device *dev)
+{
+	return ktime_get_ns();
+}
+
+bool es_dvp2axi_check_single_dev_stream_on(struct es_dvp2axi_hw *hw);
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/dvp2axi-luma.c b/drivers/media/platform/eswin/dvp2axi/dvp2axi-luma.c
new file mode 100644
index 000000000000..400a70f63060
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dvp2axi-luma.c
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI luma driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#include <linux/kfifo.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-vmalloc.h>
+#include "dev.h"
+#include "regs.h"
+#include "dvp2axi-luma.h"
+#include "mipi-csi2.h"
+
+#define ES_DVP2AXI_LUMA_REQ_BUFS_MIN		2
+#define ES_DVP2AXI_LUMA_REQ_BUFS_MAX		8
+#define SW_Y_STAT_RD_ID_MASK		GENMASK(5, 4)
+#define SW_Y_STAT_RD_BLOCK_MASK		GENMASK(7, 6)
+#define SW_Y_STAT_EN			BIT(0)
+#define SW_Y_STAT_RD_EN			BIT(3)
+#define SW_Y_STAT_BAYER_TYPE(a)		(((a) & 0x3) << 1)
+#define SW_Y_STAT_RD_ID(a)		(((a) & 0x3) << 4)
+#define SW_Y_STAT_RD_BLOCK(a)		(((a) & 0x3) << 6)
+
+static int es_dvp2axi_luma_enum_fmt_meta_cap(struct file *file, void *priv,
+					struct v4l2_fmtdesc *f)
+{
+	struct video_device *video = video_devdata(file);
+	struct es_dvp2axi_luma_vdev *luma_vdev = video_get_drvdata(video);
+
+	if (f->index > 0 || f->type != video->queue->type)
+		return -EINVAL;
+
+	f->pixelformat = luma_vdev->vdev_fmt.fmt.meta.dataformat;
+	return 0;
+}
+
+static int es_dvp2axi_luma_g_fmt_meta_cap(struct file *file, void *priv,
+				     struct v4l2_format *f)
+{
+	struct video_device *video = video_devdata(file);
+	struct es_dvp2axi_luma_vdev *luma_vdev = video_get_drvdata(video);
+	struct v4l2_meta_format *meta = &f->fmt.meta;
+
+	if (f->type != video->queue->type)
+		return -EINVAL;
+
+	memset(meta, 0, sizeof(*meta));
+	meta->dataformat = luma_vdev->vdev_fmt.fmt.meta.dataformat;
+	meta->buffersize = luma_vdev->vdev_fmt.fmt.meta.buffersize;
+
+	return 0;
+}
+
+static int es_dvp2axi_luma_querycap(struct file *file,
+			       void *priv, struct v4l2_capability *cap)
+{
+	struct video_device *vdev = video_devdata(file);
+	struct es_dvp2axi_luma_vdev *luma_vdev = video_get_drvdata(vdev);
+	struct device *dev = luma_vdev->dvp2axidev->dev;
+
+	strlcpy(cap->driver, dev->driver->name, sizeof(cap->driver));
+	strlcpy(cap->card, dev->driver->name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		 "platform:%s", dev_name(dev));
+
+	return 0;
+}
+
+/* ISP video device IOCTLs */
+static const struct v4l2_ioctl_ops es_dvp2axi_luma_ioctl = {
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+	.vidioc_enum_fmt_meta_cap = es_dvp2axi_luma_enum_fmt_meta_cap,
+	.vidioc_g_fmt_meta_cap = es_dvp2axi_luma_g_fmt_meta_cap,
+	.vidioc_s_fmt_meta_cap = es_dvp2axi_luma_g_fmt_meta_cap,
+	.vidioc_try_fmt_meta_cap = es_dvp2axi_luma_g_fmt_meta_cap,
+	.vidioc_querycap = es_dvp2axi_luma_querycap
+};
+
+static int es_dvp2axi_luma_fh_open(struct file *filp)
+{
+	struct es_dvp2axi_luma_vdev *params = video_drvdata(filp);
+	int ret;
+
+	ret = v4l2_fh_open(filp);
+	if (!ret) {
+		ret = v4l2_pipeline_pm_get(&params->vnode.vdev.entity);
+		if (ret < 0)
+			vb2_fop_release(filp);
+	}
+
+	return ret;
+}
+
+static int es_dvp2axi_luma_fop_release(struct file *file)
+{
+	struct es_dvp2axi_luma_vdev *luma = video_drvdata(file);
+	int ret;
+
+	ret = vb2_fop_release(file);
+	if (!ret)
+		v4l2_pipeline_pm_put(&luma->vnode.vdev.entity);
+	return ret;
+}
+
+struct v4l2_file_operations es_dvp2axi_luma_fops = {
+	.mmap = vb2_fop_mmap,
+	.unlocked_ioctl = video_ioctl2,
+	.poll = vb2_fop_poll,
+	.open = es_dvp2axi_luma_fh_open,
+	.release = es_dvp2axi_luma_fop_release
+};
+
+static int es_dvp2axi_luma_vb2_queue_setup(struct vb2_queue *vq,
+				      unsigned int *num_buffers,
+				      unsigned int *num_planes,
+				      unsigned int sizes[],
+				      struct device *alloc_ctxs[])
+{
+	struct es_dvp2axi_luma_vdev *luma_vdev = vq->drv_priv;
+
+	*num_planes = 1;
+
+	*num_buffers = clamp_t(u32, *num_buffers, ES_DVP2AXI_LUMA_REQ_BUFS_MIN,
+			       ES_DVP2AXI_LUMA_REQ_BUFS_MAX);
+
+	sizes[0] = sizeof(struct esisp_isp2x_luma_buffer);
+
+	INIT_LIST_HEAD(&luma_vdev->stat);
+
+	return 0;
+}
+
+static void es_dvp2axi_luma_vb2_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct es_dvp2axi_buffer *luma_buf = to_es_dvp2axi_buffer(vbuf);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct es_dvp2axi_luma_vdev *luma_dev = vq->drv_priv;
+
+	luma_buf->vaddr[0] = vb2_plane_vaddr(vb, 0);
+
+	spin_lock_bh(&luma_dev->rd_lock);
+	list_add_tail(&luma_buf->queue, &luma_dev->stat);
+	spin_unlock_bh(&luma_dev->rd_lock);
+}
+
+static void es_dvp2axi_luma_vb2_stop_streaming(struct vb2_queue *vq)
+{
+	struct es_dvp2axi_luma_vdev *luma_vdev = vq->drv_priv;
+	struct es_dvp2axi_buffer *buf;
+	unsigned long flags;
+	int i;
+
+	/* Make sure no new work queued in isr before draining wq */
+	spin_lock_irqsave(&luma_vdev->irq_lock, flags);
+	luma_vdev->streamon = false;
+	spin_unlock_irqrestore(&luma_vdev->irq_lock, flags);
+
+	tasklet_disable(&luma_vdev->rd_tasklet);
+
+	spin_lock_bh(&luma_vdev->rd_lock);
+	for (i = 0; i < ES_DVP2AXI_LUMA_REQ_BUFS_MAX; i++) {
+		if (list_empty(&luma_vdev->stat))
+			break;
+		buf = list_first_entry(&luma_vdev->stat,
+				       struct es_dvp2axi_buffer, queue);
+		list_del(&buf->queue);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+	spin_unlock_bh(&luma_vdev->rd_lock);
+}
+
+static int
+es_dvp2axi_luma_vb2_start_streaming(struct vb2_queue *queue,
+			       unsigned int count)
+{
+	struct es_dvp2axi_luma_vdev *luma_vdev = queue->drv_priv;
+	u32 i;
+
+	for (i = 0; i < ES_DVP2AXI_RAW_MAX; i++)
+		luma_vdev->ystat_rdflg[i] = false;
+
+	luma_vdev->streamon = true;
+	kfifo_reset(&luma_vdev->rd_kfifo);
+	tasklet_enable(&luma_vdev->rd_tasklet);
+
+	return 0;
+}
+
+static struct vb2_ops es_dvp2axi_luma_vb2_ops = {
+	.queue_setup = es_dvp2axi_luma_vb2_queue_setup,
+	.buf_queue = es_dvp2axi_luma_vb2_buf_queue,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+	.stop_streaming = es_dvp2axi_luma_vb2_stop_streaming,
+	.start_streaming = es_dvp2axi_luma_vb2_start_streaming,
+};
+
+static int es_dvp2axi_luma_init_vb2_queue(struct vb2_queue *q,
+				     struct es_dvp2axi_luma_vdev *luma_vdev)
+{
+	q->type = V4L2_BUF_TYPE_META_CAPTURE;
+	q->io_modes = VB2_MMAP | VB2_USERPTR;
+	q->drv_priv = luma_vdev;
+	q->ops = &es_dvp2axi_luma_vb2_ops;
+	q->mem_ops = &vb2_vmalloc_memops;
+	q->buf_struct_size = sizeof(struct es_dvp2axi_buffer);
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->lock = &luma_vdev->vnode.vlock;
+	q->dev = luma_vdev->dvp2axidev->dev;
+
+	return vb2_queue_init(q);
+}
+
+static void
+es_dvp2axi_stats_send_luma(struct es_dvp2axi_luma_vdev *vdev,
+		      struct es_dvp2axi_luma_readout_work *work)
+{
+	unsigned int cur_frame_id;
+	struct esisp_isp2x_luma_buffer *cur_stat_buf;
+	struct es_dvp2axi_buffer *cur_buf = NULL;
+	u32 i, j;
+
+	spin_lock(&vdev->rd_lock);
+	/* get one empty buffer */
+	if (!list_empty(&vdev->stat)) {
+		cur_buf = list_first_entry(&vdev->stat,
+					   struct es_dvp2axi_buffer, queue);
+		list_del(&cur_buf->queue);
+	}
+	spin_unlock(&vdev->rd_lock);
+
+	if (!cur_buf) {
+		v4l2_warn(vdev->vnode.vdev.v4l2_dev,
+			  "no luma buffer available\n");
+		return;
+	}
+
+	cur_stat_buf =
+		(struct esisp_isp2x_luma_buffer *)(cur_buf->vaddr[0]);
+	if (!cur_stat_buf) {
+		v4l2_err(vdev->vnode.vdev.v4l2_dev,
+			 "cur_stat_buf is NULL\n");
+		return;
+	}
+
+	cur_stat_buf->frame_id = work->frame_id;
+	cur_stat_buf->meas_type = work->meas_type;
+	for (i = 0; i < ES_DVP2AXI_RAW_MAX; i++) {
+		for (j = 0; j < ISP2X_MIPI_LUMA_MEAN_MAX; j++) {
+			cur_stat_buf->luma[i].exp_mean[j] =
+				work->luma[i].exp_mean[j];
+		}
+	}
+
+	cur_frame_id = cur_stat_buf->frame_id;
+	vb2_set_plane_payload(&cur_buf->vb.vb2_buf, 0,
+			      sizeof(struct esisp_isp2x_luma_buffer));
+	cur_buf->vb.sequence = cur_frame_id;
+	cur_buf->vb.vb2_buf.timestamp = work->timestamp;
+	vb2_buffer_done(&cur_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+}
+
+static void es_dvp2axi_luma_readout_task(unsigned long data)
+{
+	unsigned int out = 0;
+	struct es_dvp2axi_luma_readout_work work;
+	struct es_dvp2axi_luma_vdev *vdev =
+		(struct es_dvp2axi_luma_vdev *)data;
+
+	while (!kfifo_is_empty(&vdev->rd_kfifo)) {
+		out = kfifo_out(&vdev->rd_kfifo,
+				&work, sizeof(work));
+		if (!out)
+			break;
+
+		if (work.readout == ES_DVP2AXI_READOUT_LUMA)
+			es_dvp2axi_stats_send_luma(vdev, &work);
+	}
+}
+
+void es_dvp2axi_luma_isr(struct es_dvp2axi_luma_vdev *luma_vdev, int mipi_id, u32 frame_id)
+{
+	u8 hdr_mode = luma_vdev->dvp2axidev->hdr.hdr_mode;
+	enum es_dvp2axi_luma_frm_mode frm_mode;
+	bool send_task;
+	u32 i, value;
+
+	spin_lock(&luma_vdev->irq_lock);
+	if (!luma_vdev->streamon)
+		goto unlock;
+
+	switch (hdr_mode) {
+	case NO_HDR:
+		frm_mode = ES_DVP2AXI_LUMA_ONEFRM;
+		break;
+	case HDR_X2:
+		frm_mode = ES_DVP2AXI_LUMA_TWOFRM;
+		break;
+	case HDR_X3:
+		frm_mode = ES_DVP2AXI_LUMA_THREEFRM;
+		break;
+	default:
+		goto unlock;
+	}
+
+	if (mipi_id == ES_DVP2AXI_STREAM_MIPI_ID0 && !luma_vdev->ystat_rdflg[0]) {
+		value = es_dvp2axi_read_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL);
+		value &= ~(SW_Y_STAT_RD_ID_MASK | SW_Y_STAT_RD_BLOCK_MASK);
+		value |= SW_Y_STAT_RD_ID(0x0) | SW_Y_STAT_RD_EN;
+		es_dvp2axi_write_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL, value);
+		for (i = 0; i < ISP2X_MIPI_LUMA_MEAN_MAX; i++)
+			luma_vdev->work.luma[0].exp_mean[i] =
+				es_dvp2axi_read_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_VALUE);
+
+		luma_vdev->ystat_rdflg[0] = true;
+	}
+	if (mipi_id == ES_DVP2AXI_STREAM_MIPI_ID1 && !luma_vdev->ystat_rdflg[1]) {
+		value = es_dvp2axi_read_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL);
+		value &= ~(SW_Y_STAT_RD_ID_MASK | SW_Y_STAT_RD_BLOCK_MASK);
+		value |= SW_Y_STAT_RD_ID(0x1) | SW_Y_STAT_RD_EN;
+		es_dvp2axi_write_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL, value);
+		for (i = 0; i < ISP2X_MIPI_LUMA_MEAN_MAX; i++)
+			luma_vdev->work.luma[1].exp_mean[i] =
+				es_dvp2axi_read_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_VALUE);
+
+		luma_vdev->ystat_rdflg[1] = true;
+	}
+	if (mipi_id == ES_DVP2AXI_STREAM_MIPI_ID2 && !luma_vdev->ystat_rdflg[2]) {
+		value = es_dvp2axi_read_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL);
+		value &= ~(SW_Y_STAT_RD_ID_MASK | SW_Y_STAT_RD_BLOCK_MASK);
+		value |= SW_Y_STAT_RD_ID(0x2) | SW_Y_STAT_RD_EN;
+		es_dvp2axi_write_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL, value);
+		for (i = 0; i < ISP2X_MIPI_LUMA_MEAN_MAX; i++)
+			luma_vdev->work.luma[2].exp_mean[i] =
+				es_dvp2axi_read_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_VALUE);
+
+		luma_vdev->ystat_rdflg[2] = true;
+	}
+
+	send_task = false;
+	if (frm_mode == ES_DVP2AXI_LUMA_THREEFRM) {
+		if (luma_vdev->ystat_rdflg[0] && luma_vdev->ystat_rdflg[1] &&
+		    luma_vdev->ystat_rdflg[2])
+			send_task = true;
+	} else if (frm_mode == ES_DVP2AXI_LUMA_TWOFRM) {
+		if (luma_vdev->ystat_rdflg[0] && luma_vdev->ystat_rdflg[1])
+			send_task = true;
+	} else {
+		if (luma_vdev->ystat_rdflg[0])
+			send_task = true;
+	}
+
+	if (send_task) {
+		luma_vdev->work.readout = ES_DVP2AXI_READOUT_LUMA;
+		luma_vdev->work.timestamp = es_dvp2axi_time_get_ns(luma_vdev->dvp2axidev);
+		luma_vdev->work.frame_id = frame_id;
+
+		if (frm_mode == ES_DVP2AXI_LUMA_THREEFRM)
+			luma_vdev->work.meas_type = ISP2X_RAW0_Y_STATE | ISP2X_RAW1_Y_STATE |
+						    ISP2X_RAW2_Y_STATE;
+		else if (frm_mode == ES_DVP2AXI_LUMA_TWOFRM)
+			luma_vdev->work.meas_type = ISP2X_RAW0_Y_STATE | ISP2X_RAW1_Y_STATE;
+		else
+			luma_vdev->work.meas_type = ISP2X_RAW0_Y_STATE;
+
+		if (!kfifo_is_full(&luma_vdev->rd_kfifo))
+			kfifo_in(&luma_vdev->rd_kfifo,
+				 &luma_vdev->work, sizeof(luma_vdev->work));
+		else
+			v4l2_err(luma_vdev->vnode.vdev.v4l2_dev,
+				 "stats kfifo is full\n");
+
+		tasklet_schedule(&luma_vdev->rd_tasklet);
+
+		for (i = 0; i < ES_DVP2AXI_RAW_MAX; i++)
+			luma_vdev->ystat_rdflg[i] = false;
+
+		memset(&luma_vdev->work, 0, sizeof(luma_vdev->work));
+	}
+
+unlock:
+	spin_unlock(&luma_vdev->irq_lock);
+}
+
+void es_dvp2axi_start_luma(struct es_dvp2axi_luma_vdev *luma_vdev, const struct dvp2axi_input_fmt *dvp2axi_fmt_in)
+{
+	u32 bayer = 0;
+
+	if (dvp2axi_fmt_in->fmt_type != DVP2AXI_FMT_TYPE_RAW)
+		return;
+
+	switch (dvp2axi_fmt_in->mbus_code) {
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+		bayer = 3;
+		break;
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SGBRG12_1X12:
+		bayer = 2;
+		break;
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SGRBG12_1X12:
+		bayer = 1;
+		break;
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+		bayer = 0;
+		break;
+	}
+
+	es_dvp2axi_write_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL,
+			     SW_Y_STAT_BAYER_TYPE(bayer) | SW_Y_STAT_EN);
+	luma_vdev->enable = true;
+}
+
+void es_dvp2axi_stop_luma(struct es_dvp2axi_luma_vdev *luma_vdev)
+{
+	es_dvp2axi_write_register(luma_vdev->dvp2axidev, DVP2AXI_REG_Y_STAT_CONTROL, 0x0);
+	luma_vdev->enable = false;
+}
+
+static void es_dvp2axi_init_luma_vdev(struct es_dvp2axi_luma_vdev *luma_vdev)
+{
+	luma_vdev->vdev_fmt.fmt.meta.dataformat =
+		V4L2_META_FMT_ES_ISP1_STAT_LUMA;
+	luma_vdev->vdev_fmt.fmt.meta.buffersize =
+		sizeof(struct esisp_isp2x_luma_buffer);
+}
+
+int es_dvp2axi_register_luma_vdev(struct es_dvp2axi_luma_vdev *luma_vdev,
+			     struct v4l2_device *v4l2_dev,
+			     struct es_dvp2axi_device *dev)
+{
+	int ret;
+	struct es_dvp2axi_luma_node *node = &luma_vdev->vnode;
+	struct video_device *vdev = &node->vdev;
+
+	luma_vdev->dvp2axidev = dev;
+
+	INIT_LIST_HEAD(&luma_vdev->stat);
+	spin_lock_init(&luma_vdev->irq_lock);
+	spin_lock_init(&luma_vdev->rd_lock);
+
+	strlcpy(vdev->name, "es_dvp2axi-mipi-luma", sizeof(vdev->name));
+	mutex_init(&node->vlock);
+
+	vdev->ioctl_ops = &es_dvp2axi_luma_ioctl;
+	vdev->fops = &es_dvp2axi_luma_fops;
+	vdev->release = video_device_release_empty;
+	vdev->lock = &node->vlock;
+	vdev->v4l2_dev = v4l2_dev;
+	vdev->queue = &node->buf_queue;
+	vdev->device_caps = V4L2_CAP_META_CAPTURE | V4L2_CAP_STREAMING;
+	vdev->vfl_dir =  VFL_DIR_RX;
+	es_dvp2axi_luma_init_vb2_queue(vdev->queue, luma_vdev);
+	es_dvp2axi_init_luma_vdev(luma_vdev);
+	video_set_drvdata(vdev, luma_vdev);
+
+	node->pad.flags = MEDIA_PAD_FL_SINK;
+	ret = media_entity_pads_init(&vdev->entity, 0, &node->pad);
+	if (ret < 0)
+		goto err_release_queue;
+
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret < 0) {
+		dev_err(&vdev->dev,
+			"could not register Video for Linux device\n");
+		goto err_cleanup_media_entity;
+	}
+
+	ret = kfifo_alloc(&luma_vdev->rd_kfifo,
+			  ES_DVP2AXI_LUMA_READOUT_WORK_SIZE,
+			  GFP_KERNEL);
+	if (ret) {
+		dev_err(&vdev->dev,
+			"kfifo_alloc failed with error %d\n",
+			ret);
+		goto err_unregister_video;
+	}
+
+	tasklet_init(&luma_vdev->rd_tasklet,
+		     es_dvp2axi_luma_readout_task,
+		     (unsigned long)luma_vdev);
+	tasklet_disable(&luma_vdev->rd_tasklet);
+
+	return 0;
+
+err_unregister_video:
+	video_unregister_device(vdev);
+err_cleanup_media_entity:
+	media_entity_cleanup(&vdev->entity);
+err_release_queue:
+	vb2_queue_release(vdev->queue);
+	return ret;
+}
+
+void es_dvp2axi_unregister_luma_vdev(struct es_dvp2axi_luma_vdev *luma_vdev)
+{
+	struct es_dvp2axi_luma_node *node = &luma_vdev->vnode;
+	struct video_device *vdev = &node->vdev;
+
+	kfifo_free(&luma_vdev->rd_kfifo);
+	tasklet_kill(&luma_vdev->rd_tasklet);
+	video_unregister_device(vdev);
+	media_entity_cleanup(&vdev->entity);
+	vb2_queue_release(vdev->queue);
+}
diff --git a/drivers/media/platform/eswin/dvp2axi/dvp2axi-luma.h b/drivers/media/platform/eswin/dvp2axi/dvp2axi-luma.h
new file mode 100644
index 000000000000..0aace2e971e0
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dvp2axi-luma.h
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI luma driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+#ifndef _ES_DVP2AXI_LUMA_H
+#define _ES_DVP2AXI_LUMA_H
+
+#include "es-isp1-config.h"
+#include <linux/interrupt.h>
+#include <linux/kfifo.h>
+#include "dev.h"
+
+#define ES_DVP2AXI_LUMA_READOUT_WORK_SIZE	\
+	(9 * sizeof(struct es_dvp2axi_luma_readout_work))
+#define ES_DVP2AXI_LUMA_YSTAT_ISR_NUM	1
+#define ES_DVP2AXI_RAW_MAX			3
+
+struct es_dvp2axi_luma_vdev;
+struct dvp2axi_input_fmt;
+
+enum es_dvp2axi_luma_readout_cmd {
+	ES_DVP2AXI_READOUT_LUMA,
+};
+
+enum es_dvp2axi_luma_frm_mode {
+	ES_DVP2AXI_LUMA_ONEFRM,
+	ES_DVP2AXI_LUMA_TWOFRM,
+	ES_DVP2AXI_LUMA_THREEFRM,
+};
+
+struct es_dvp2axi_luma_readout_work {
+	enum es_dvp2axi_luma_readout_cmd readout;
+	unsigned long long timestamp;
+	unsigned int meas_type;
+	unsigned int frame_id;
+	struct esisp_mipi_luma luma[ES_DVP2AXI_RAW_MAX];
+};
+
+struct es_dvp2axi_luma_node {
+	struct vb2_queue buf_queue;
+	/* vfd lock */
+	struct mutex vlock;
+	struct video_device vdev;
+	struct media_pad pad;
+};
+
+/*
+ * struct es_dvp2axi_luma_vdev - DVP2AXI Statistics device
+ *
+ * @irq_lock: buffer queue lock
+ * @stat: stats buffer list
+ * @readout_wq: workqueue for statistics information read
+ */
+struct es_dvp2axi_luma_vdev {
+	struct es_dvp2axi_luma_node vnode;
+	struct es_dvp2axi_device *dvp2axidev;
+	bool enable;
+
+	spinlock_t irq_lock;	/* tasklet queue lock */
+	struct list_head stat;
+	struct v4l2_format vdev_fmt;
+	bool streamon;
+
+	spinlock_t rd_lock;	/* buffer queue lock */
+	struct kfifo rd_kfifo;
+	struct tasklet_struct rd_tasklet;
+
+	bool ystat_rdflg[ISP2X_MIPI_RAW_MAX];
+	struct es_dvp2axi_luma_readout_work work;
+};
+
+void es_dvp2axi_start_luma(struct es_dvp2axi_luma_vdev *luma_vdev, const struct dvp2axi_input_fmt *dvp2axi_fmt_in);
+
+void es_dvp2axi_stop_luma(struct es_dvp2axi_luma_vdev *luma_vdev);
+
+void es_dvp2axi_luma_isr(struct es_dvp2axi_luma_vdev *luma_vdev, int isp_stat, u32 frame_id);
+
+int es_dvp2axi_register_luma_vdev(struct es_dvp2axi_luma_vdev *luma_vdev,
+			     struct v4l2_device *v4l2_dev,
+			     struct es_dvp2axi_device *dev);
+
+void es_dvp2axi_unregister_luma_vdev(struct es_dvp2axi_luma_vdev *luma_vdev);
+
+#endif /* _ES_DVP2AXI_LUMA_H */
diff --git a/drivers/media/platform/eswin/dvp2axi/dvp2axi-scale.c b/drivers/media/platform/eswin/dvp2axi/dvp2axi-scale.c
new file mode 100644
index 000000000000..6006c597c31e
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dvp2axi-scale.c
@@ -0,0 +1,1213 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI scale driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#include <linux/kfifo.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-vmalloc.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <media/v4l2-event.h>
+#include "dev.h"
+#include "regs.h"
+#include "mipi-csi2.h"
+#include <media/v4l2-fwnode.h>
+#include <linux/pm_runtime.h>
+
+#define MEMORY_ALIGN_ROUND_UP_HEIGHT		16
+
+#define SCALE_MIN_WIDTH		4
+#define SCALE_MIN_HEIGHT	4
+#define SCALE_OUTPUT_STEP_WISE	1
+#define DVP2AXI_SCALE_REQ_BUFS_MIN	3
+
+static const struct dvp2axi_output_fmt scale_out_fmts[] = {
+	{
+		.fourcc = V4L2_PIX_FMT_SRGGB16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	}, {
+		.fourcc = V4L2_PIX_FMT_SGRBG16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	}, {
+		.fourcc = V4L2_PIX_FMT_SGBRG16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	}, {
+		.fourcc = V4L2_PIX_FMT_SBGGR16,
+		.cplanes = 1,
+		.mplanes = 1,
+		.bpp = { 16 },
+		.raw_bpp = 16,
+		.fmt_type = DVP2AXI_FMT_TYPE_RAW,
+	}
+};
+
+static int es_dvp2axi_scale_enum_fmt_vid_cap(struct file *file, void *priv,
+					struct v4l2_fmtdesc *f)
+{
+	const struct dvp2axi_output_fmt *fmt = NULL;
+
+	if (f->index >= ARRAY_SIZE(scale_out_fmts))
+		return -EINVAL;
+	fmt = &scale_out_fmts[f->index];
+	f->pixelformat = fmt->fourcc;
+	return 0;
+}
+
+static int es_dvp2axi_scale_g_fmt_vid_cap_mplane(struct file *file, void *priv,
+					    struct v4l2_format *f)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+
+	f->fmt.pix_mp = scale_vdev->pixm;
+	return 0;
+}
+
+static u32 es_dvp2axi_scale_align_bits_per_pixel(struct es_dvp2axi_device *dvp2axi_dev,
+					    const struct dvp2axi_output_fmt *fmt,
+					    int plane_index)
+{
+	u32 bpp = 0, i;
+
+	if (fmt) {
+		switch (fmt->fourcc) {
+		case V4L2_PIX_FMT_SBGGR16:
+		case V4L2_PIX_FMT_SGBRG16:
+		case V4L2_PIX_FMT_SGRBG16:
+		case V4L2_PIX_FMT_SRGGB16:
+			bpp = max(fmt->bpp[plane_index], (u8)DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126);
+			for (i = 1; i < 5; i++) {
+				if (i * DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126 >= bpp) {
+					bpp = i * DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126;
+					break;
+				}
+			}
+			break;
+		default:
+			v4l2_err(&dvp2axi_dev->v4l2_dev, "fourcc: %d is not supported!\n",
+				 fmt->fourcc);
+			break;
+		}
+	}
+
+	return bpp;
+}
+
+
+static const struct
+dvp2axi_output_fmt *es_dvp2axi_scale_find_output_fmt(u32 pixelfmt)
+{
+	const struct dvp2axi_output_fmt *fmt;
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(scale_out_fmts); i++) {
+		fmt = &scale_out_fmts[i];
+		if (fmt->fourcc == pixelfmt)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+static int es_dvp2axi_scale_set_fmt(struct es_dvp2axi_scale_vdev *scale_vdev,
+			       struct v4l2_pix_format_mplane *pixm,
+			       bool try)
+{
+	struct es_dvp2axi_stream *stream = scale_vdev->stream;
+	struct es_dvp2axi_device *dvp2axi_dev = scale_vdev->dvp2axidev;
+	struct v4l2_subdev_selection input_sel;
+	struct v4l2_subdev_format fmt_src;
+	const struct dvp2axi_output_fmt *fmt;
+	unsigned int imagesize = 0;
+	int bpl, size, bpp;
+	int scale_times = 0;
+	u32 scale_ratio = 0;
+	u32 width = 3280;
+	u32 height = 2464;
+	int ret = 0;
+
+	if (!dvp2axi_dev->terminal_sensor.sd)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->terminal_sensor.sd) {
+		fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		fmt_src.pad = 0;
+		ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, pad, get_fmt, NULL, &fmt_src);
+		if (ret) {
+			v4l2_err(&scale_vdev->dvp2axidev->v4l2_dev,
+				 "%s: get sensor format failed\n", __func__);
+			return ret;
+		}
+
+		input_sel.target = V4L2_SEL_TGT_CROP_BOUNDS;
+		input_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		input_sel.pad = 0;
+		ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd,
+				       pad, get_selection, NULL,
+				       &input_sel);
+		if (!ret) {
+			fmt_src.format.width = input_sel.r.width;
+			fmt_src.format.height = input_sel.r.height;
+		}
+		scale_vdev->src_res.width = fmt_src.format.width;
+		scale_vdev->src_res.height = fmt_src.format.height;
+	}
+	fmt = es_dvp2axi_scale_find_output_fmt(pixm->pixelformat);
+	if (fmt == NULL) {
+		v4l2_err(&scale_vdev->dvp2axidev->v4l2_dev,
+			"format of source channel are not bayer raw, not support scale\n");
+		return -1;
+	}
+	if (scale_vdev->src_res.width && scale_vdev->src_res.height) {
+		width = scale_vdev->src_res.width;
+		height = scale_vdev->src_res.height;
+	}
+	scale_ratio = width / pixm->width;
+	if (scale_ratio <= 8) {
+		scale_vdev->scale_mode = SCALE_8TIMES;
+		scale_times = 8;
+	} else if (scale_ratio <= 16) {
+		scale_vdev->scale_mode = SCALE_16TIMES;
+		scale_times = 16;
+	} else {
+		scale_vdev->scale_mode = SCALE_32TIMES;
+		scale_times = 32;
+	}
+	//source resolution align (scale_times * 2)
+	width = ALIGN(width, scale_times * 2);
+	pixm->width = width  / (scale_times * 2) * 2;
+	pixm->height = height / (scale_times * 2) * 2;
+	pixm->num_planes = fmt->mplanes;
+	pixm->field = V4L2_FIELD_NONE;
+	pixm->quantization = V4L2_QUANTIZATION_DEFAULT;
+
+	bpp = es_dvp2axi_scale_align_bits_per_pixel(dvp2axi_dev, fmt, 0);
+	bpl = pixm->width * bpp / DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126;
+	bpl = ALIGN(bpl, 8);
+	size = bpl * pixm->height;
+	imagesize += size;
+
+	v4l2_dbg(1, es_dvp2axi_debug, &stream->dvp2axidev->v4l2_dev,
+		 "%s C-Plane %i size: %d, Total imagesize: %d\n",
+		 __func__, 0, size, imagesize);
+
+	if (fmt->mplanes == 1) {
+		pixm->plane_fmt[0].bytesperline = bpl;
+		pixm->plane_fmt[0].sizeimage = imagesize;
+	}
+
+	if (!try) {
+		scale_vdev->scale_out_fmt = fmt;
+		scale_vdev->pixm = *pixm;
+
+		v4l2_info(&stream->dvp2axidev->v4l2_dev,
+			  "%s: req(%d, %d) src out(%d, %d)\n", __func__,
+			  pixm->width, pixm->height,
+			  scale_vdev->src_res.width, scale_vdev->src_res.height);
+	}
+	return 0;
+}
+
+static int es_dvp2axi_scale_s_fmt_vid_cap_mplane(struct file *file,
+					    void *priv, struct v4l2_format *f)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+	int ret = 0;
+
+	if (vb2_is_busy(&scale_vdev->vnode.buf_queue)) {
+		v4l2_err(&scale_vdev->dvp2axidev->v4l2_dev, "%s queue busy\n", __func__);
+		return -EBUSY;
+	}
+
+	ret = es_dvp2axi_scale_set_fmt(scale_vdev, &f->fmt.pix_mp, false);
+
+	return ret;
+}
+
+static int es_dvp2axi_scale_querycap(struct file *file,
+				void *priv, struct v4l2_capability *cap)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+	struct device *dev = scale_vdev->dvp2axidev->dev;
+
+	strscpy(cap->driver, dev->driver->name, sizeof(cap->driver));
+	strscpy(cap->card, dev->driver->name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		 "platform:%s", dev_name(dev));
+	return 0;
+}
+
+static long es_dvp2axi_scale_ioctl_default(struct file *file, void *fh,
+				    bool valid_prio, unsigned int cmd, void *arg)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	struct bayer_blc *pblc;
+
+	switch (cmd) {
+	case ES_DVP2AXI_CMD_GET_SCALE_BLC:
+		pblc = (struct bayer_blc *)arg;
+		*pblc = scale_vdev->blc;
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev, "get scale blc %d %d %d %d\n",
+			 pblc->pattern00, pblc->pattern01, pblc->pattern02, pblc->pattern03);
+		break;
+	case ES_DVP2AXI_CMD_SET_SCALE_BLC:
+		pblc = (struct bayer_blc *)arg;
+		scale_vdev->blc = *pblc;
+		v4l2_dbg(1, es_dvp2axi_debug, &dev->v4l2_dev, "set scale blc %d %d %d %d\n",
+			 pblc->pattern00, pblc->pattern01, pblc->pattern02, pblc->pattern03);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int es_dvp2axi_scale_enum_input(struct file *file, void *priv,
+				  struct v4l2_input *input)
+{
+
+	if (input->index > 0)
+		return -EINVAL;
+
+	input->type = V4L2_INPUT_TYPE_CAMERA;
+	strscpy(input->name, "Camera", sizeof(input->name));
+
+	return 0;
+}
+
+static int es_dvp2axi_scale_try_fmt_vid_cap_mplane(struct file *file, void *fh,
+					      struct v4l2_format *f)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+	int ret = 0;
+
+	ret = es_dvp2axi_scale_set_fmt(scale_vdev, &f->fmt.pix_mp, true);
+
+	return ret;
+}
+
+static int es_dvp2axi_scale_enum_frameintervals(struct file *file, void *fh,
+					   struct v4l2_frmivalenum *fival)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	struct es_dvp2axi_sensor_info *sensor = &dev->terminal_sensor;
+	struct v4l2_subdev_frame_interval fi;
+	int ret;
+
+	if (fival->index != 0)
+		return -EINVAL;
+
+	if (!sensor || !sensor->sd) {
+		/* TODO: active_sensor is NULL if using DMARX path */
+		v4l2_err(&dev->v4l2_dev, "%s Not active sensor\n", __func__);
+		return -ENODEV;
+	}
+
+	ret = v4l2_subdev_call(sensor->sd, video, g_frame_interval, &fi);
+	if (ret && ret != -ENOIOCTLCMD) {
+		return ret;
+	} else if (ret == -ENOIOCTLCMD) {
+		/* Set a default value for sensors not implements ioctl */
+		fi.interval.numerator = 1;
+		fi.interval.denominator = 30;
+	}
+
+	fival->type = V4L2_FRMIVAL_TYPE_CONTINUOUS;
+	fival->stepwise.step.numerator = 1;
+	fival->stepwise.step.denominator = 1;
+	fival->stepwise.max.numerator = 1;
+	fival->stepwise.max.denominator = 1;
+	fival->stepwise.min.numerator = fi.interval.numerator;
+	fival->stepwise.min.denominator = fi.interval.denominator;
+
+	return 0;
+}
+
+static int es_dvp2axi_scale_enum_framesizes(struct file *file, void *prov,
+				       struct v4l2_frmsizeenum *fsize)
+{
+	struct v4l2_frmsize_discrete *s = &fsize->discrete;
+	struct es_dvp2axi_scale_vdev *scale_vdev = video_drvdata(file);
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	struct v4l2_rect input_rect;
+	struct es_dvp2axi_sensor_info *terminal_sensor = &dev->terminal_sensor;
+	struct csi_channel_info csi_info;
+	int scale_times = 0;
+
+	if (fsize->index >= ES_DVP2AXI_SCALE_ENUM_SIZE_MAX)
+		return -EINVAL;
+
+	if (!es_dvp2axi_scale_find_output_fmt(fsize->pixel_format))
+		return -EINVAL;
+
+	input_rect.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	input_rect.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+
+	if (terminal_sensor && terminal_sensor->sd)
+		es_dvp2axi_get_input_fmt(dev,
+				    &input_rect, 0, &csi_info);
+
+	switch (fsize->index) {
+	case SCALE_8TIMES:
+		scale_times = 8;
+		break;
+	case SCALE_16TIMES:
+		scale_times = 16;
+		break;
+	case SCALE_32TIMES:
+		scale_times = 32;
+		break;
+	default:
+		scale_times = 32;
+		break;
+	}
+	fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+	s->width = input_rect.width  / (scale_times * 2) * 2;
+	s->height = input_rect.height / (scale_times * 2) * 2;
+
+	return 0;
+}
+
+/* ISP video device IOCTLs */
+static const struct v4l2_ioctl_ops es_dvp2axi_scale_ioctl = {
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+	.vidioc_enum_input = es_dvp2axi_scale_enum_input,
+	.vidioc_enum_fmt_vid_cap = es_dvp2axi_scale_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap_mplane = es_dvp2axi_scale_g_fmt_vid_cap_mplane,
+	.vidioc_s_fmt_vid_cap_mplane = es_dvp2axi_scale_s_fmt_vid_cap_mplane,
+	.vidioc_try_fmt_vid_cap_mplane = es_dvp2axi_scale_try_fmt_vid_cap_mplane,
+	.vidioc_querycap = es_dvp2axi_scale_querycap,
+	.vidioc_enum_frameintervals = es_dvp2axi_scale_enum_frameintervals,
+	.vidioc_enum_framesizes = es_dvp2axi_scale_enum_framesizes,
+	.vidioc_default = es_dvp2axi_scale_ioctl_default,
+};
+
+static int es_dvp2axi_scale_fh_open(struct file *file)
+{
+	struct video_device *vdev = video_devdata(file);
+	struct es_dvp2axi_vdev_node *vnode = vdev_to_node(vdev);
+	struct es_dvp2axi_scale_vdev *scale_vdev = to_es_dvp2axi_scale_vdev(vnode);
+	struct es_dvp2axi_device *dvp2axidev = scale_vdev->dvp2axidev;
+	int ret;
+
+	ret = es_dvp2axi_update_sensor_info(scale_vdev->stream);
+	if (ret < 0) {
+		v4l2_err(vdev,
+			 "update sensor info failed %d\n",
+			 ret);
+
+		return ret;
+	}
+
+	ret = pm_runtime_resume_and_get(dvp2axidev->dev);
+	if (ret < 0)
+		v4l2_err(&dvp2axidev->v4l2_dev, "Failed to get runtime pm, %d\n",
+			 ret);
+
+	ret = v4l2_fh_open(file);
+	if (!ret) {
+		ret = v4l2_pipeline_pm_get(&vnode->vdev.entity);
+		if (ret < 0)
+			vb2_fop_release(file);
+	}
+
+	return ret;
+}
+
+static int es_dvp2axi_scale_fop_release(struct file *file)
+{
+	struct video_device *vdev = video_devdata(file);
+	struct es_dvp2axi_vdev_node *vnode = vdev_to_node(vdev);
+	struct es_dvp2axi_scale_vdev *scale_vdev = to_es_dvp2axi_scale_vdev(vnode);
+	struct es_dvp2axi_device *dvp2axidev = scale_vdev->dvp2axidev;
+	int ret;
+
+	ret = vb2_fop_release(file);
+	if (!ret)
+		v4l2_pipeline_pm_put(&vnode->vdev.entity);
+
+	pm_runtime_put_sync(dvp2axidev->dev);
+	return ret;
+}
+
+struct v4l2_file_operations es_dvp2axi_scale_fops = {
+	.mmap = vb2_fop_mmap,
+	.unlocked_ioctl = video_ioctl2,
+	.poll = vb2_fop_poll,
+	.open = es_dvp2axi_scale_fh_open,
+	.release = es_dvp2axi_scale_fop_release
+};
+
+static int es_dvp2axi_scale_vb2_queue_setup(struct vb2_queue *queue,
+				       unsigned int *num_buffers,
+				       unsigned int *num_planes,
+				       unsigned int sizes[],
+				       struct device *alloc_ctxs[])
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = queue->drv_priv;
+	struct es_dvp2axi_device *dvp2axi_dev = scale_vdev->dvp2axidev;
+	const struct v4l2_pix_format_mplane *pixm = NULL;
+	const struct dvp2axi_output_fmt *dvp2axi_fmt;
+	u32 i;
+	const struct v4l2_plane_pix_format *plane_fmt;
+
+	pixm = &scale_vdev->pixm;
+	dvp2axi_fmt = scale_vdev->scale_out_fmt;
+	*num_planes = dvp2axi_fmt->mplanes;
+
+	for (i = 0; i < dvp2axi_fmt->mplanes; i++) {
+		plane_fmt = &pixm->plane_fmt[i];
+		sizes[i] = plane_fmt->sizeimage;
+	}
+
+	v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev, "%s count %d, size %d\n",
+		 v4l2_type_names[queue->type], *num_buffers, sizes[0]);
+	return 0;
+
+}
+
+static void es_dvp2axi_scale_vb2_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct es_dvp2axi_buffer *dvp2axibuf = to_es_dvp2axi_buffer(vbuf);
+	struct vb2_queue *queue = vb->vb2_queue;
+	struct es_dvp2axi_scale_vdev *scale_vdev = queue->drv_priv;
+	struct v4l2_pix_format_mplane *pixm = &scale_vdev->pixm;
+	const struct dvp2axi_output_fmt *fmt = scale_vdev->scale_out_fmt;
+	struct es_dvp2axi_hw *hw_dev = scale_vdev->dvp2axidev->hw_dev;
+	unsigned long lock_flags = 0;
+	int i;
+
+	memset(dvp2axibuf->buff_addr, 0, sizeof(dvp2axibuf->buff_addr));
+	/* If mplanes > 1, every c-plane has its own m-plane,
+	 * otherwise, multiple c-planes are in the same m-plane
+	 */
+	for (i = 0; i < fmt->mplanes; i++) {
+		void *addr = vb2_plane_vaddr(vb, i);
+
+		if (hw_dev->is_dma_sg_ops) {
+			struct sg_table *sgt = vb2_dma_sg_plane_desc(vb, i);
+
+			dvp2axibuf->buff_addr[i] = sg_dma_address(sgt->sgl);
+		} else {
+			dvp2axibuf->buff_addr[i] = vb2_dma_contig_plane_dma_addr(vb, i);
+		}
+		if (es_dvp2axi_debug && addr && !hw_dev->iommu_en) {
+			memset(addr, 0, pixm->plane_fmt[i].sizeimage);
+			v4l2_dbg(3, es_dvp2axi_debug, &scale_vdev->dvp2axidev->v4l2_dev,
+				 "Clear buffer, size: 0x%08x\n",
+				 pixm->plane_fmt[i].sizeimage);
+		}
+	}
+
+	if (fmt->mplanes == 1) {
+		for (i = 0; i < fmt->cplanes - 1; i++)
+			dvp2axibuf->buff_addr[i + 1] = dvp2axibuf->buff_addr[i] +
+				pixm->plane_fmt[i].bytesperline * pixm->height;
+	}
+	spin_lock_irqsave(&scale_vdev->vbq_lock, lock_flags);
+	list_add_tail(&dvp2axibuf->queue, &scale_vdev->buf_head);
+	spin_unlock_irqrestore(&scale_vdev->vbq_lock, lock_flags);
+}
+
+static int es_dvp2axi_scale_stop(struct es_dvp2axi_scale_vdev *scale_vdev)
+{
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	int ch = scale_vdev->ch;
+
+	es_dvp2axi_write_register_and(dev, DVP2AXI_REG_SCL_CH_CTRL,
+				 ~(DVP2AXI_SCALE_EN(ch) |
+				 DVP2AXI_SCALE_SW_SRC_CH(0x1f, ch) |
+				 DVP2AXI_SCALE_SW_MODE(0x03, ch)));
+	scale_vdev->state = ES_DVP2AXI_STATE_READY;
+	scale_vdev->frame_idx = 0;
+	return 0;
+}
+
+static void es_dvp2axi_scale_vb2_stop_streaming(struct vb2_queue *vq)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = vq->drv_priv;
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	struct es_dvp2axi_buffer *buf = NULL;
+	int ret = 0;
+
+	mutex_lock(&dev->scale_lock);
+	/* Make sure no new work queued in isr before draining wq */
+	scale_vdev->stopping = true;
+	ret = wait_event_timeout(scale_vdev->wq_stopped,
+				 scale_vdev->state != ES_DVP2AXI_STATE_STREAMING,
+				 msecs_to_jiffies(1000));
+	if (!ret) {
+		es_dvp2axi_scale_stop(scale_vdev);
+		scale_vdev->stopping = false;
+	}
+	/* release buffers */
+	if (scale_vdev->curr_buf)
+		list_add_tail(&scale_vdev->curr_buf->queue, &scale_vdev->buf_head);
+
+	if (scale_vdev->next_buf &&
+	    scale_vdev->next_buf != scale_vdev->curr_buf)
+		list_add_tail(&scale_vdev->next_buf->queue, &scale_vdev->buf_head);
+	scale_vdev->curr_buf = NULL;
+	scale_vdev->next_buf = NULL;
+	while (!list_empty(&scale_vdev->buf_head)) {
+		buf = list_first_entry(&scale_vdev->buf_head,
+				       struct es_dvp2axi_buffer, queue);
+		list_del(&buf->queue);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+	mutex_unlock(&dev->scale_lock);
+}
+
+static int es_dvp2axi_scale_channel_init(struct es_dvp2axi_scale_vdev *scale_vdev)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = scale_vdev->dvp2axidev;
+	struct es_dvp2axi_scale_ch_info *ch_info = &scale_vdev->ch_info;
+	struct v4l2_pix_format_mplane pixm = scale_vdev->pixm;
+	const struct dvp2axi_output_fmt *fmt = scale_vdev->scale_out_fmt;
+
+	if (dvp2axi_dev->inf_id == ES_DVP2AXI_DVP)
+		scale_vdev->ch_src = SCALE_DVP;
+	else
+		scale_vdev->ch_src = 4 * dvp2axi_dev->csi_host_idx + scale_vdev->ch;
+	ch_info->width = pixm.width;
+	ch_info->height = pixm.height;
+	ch_info->vir_width = ALIGN(ch_info->width  * fmt->bpp[0] / 8, 8);
+	return 0;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_scale_vlw(int ch)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (ch) {
+	case 0:
+		index = DVP2AXI_REG_SCL_VLW_CH0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_SCL_VLW_CH1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_SCL_VLW_CH2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_SCL_VLW_CH3;
+		break;
+	default:
+		index = DVP2AXI_REG_SCL_VLW_CH0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_scale_frm0_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_SCL_FRM0_ADDR_CH0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_SCL_FRM0_ADDR_CH1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_SCL_FRM0_ADDR_CH2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_SCL_FRM0_ADDR_CH3;
+		break;
+	default:
+		index = DVP2AXI_REG_SCL_FRM0_ADDR_CH0;
+		break;
+	}
+
+	return index;
+}
+
+static enum dvp2axi_reg_index get_reg_index_of_scale_frm1_addr(int channel_id)
+{
+	enum dvp2axi_reg_index index;
+
+	switch (channel_id) {
+	case 0:
+		index = DVP2AXI_REG_SCL_FRM1_ADDR_CH0;
+		break;
+	case 1:
+		index = DVP2AXI_REG_SCL_FRM1_ADDR_CH1;
+		break;
+	case 2:
+		index = DVP2AXI_REG_SCL_FRM1_ADDR_CH2;
+		break;
+	case 3:
+		index = DVP2AXI_REG_SCL_FRM1_ADDR_CH3;
+		break;
+	default:
+		index = DVP2AXI_REG_SCL_FRM1_ADDR_CH0;
+		break;
+	}
+
+	return index;
+}
+
+static void es_dvp2axi_assign_scale_buffer_init(struct es_dvp2axi_scale_vdev *scale_vdev,
+					   int ch)
+{
+	struct es_dvp2axi_device *dev = scale_vdev->stream->dvp2axidev;
+	u32 frm0_addr;
+	u32 frm1_addr;
+	unsigned long flags;
+
+	frm0_addr = get_reg_index_of_scale_frm0_addr(ch);
+	frm1_addr = get_reg_index_of_scale_frm1_addr(ch);
+
+	spin_lock_irqsave(&scale_vdev->vbq_lock, flags);
+
+	if (!scale_vdev->curr_buf) {
+		if (!list_empty(&scale_vdev->buf_head)) {
+			scale_vdev->curr_buf = list_first_entry(&scale_vdev->buf_head,
+							    struct es_dvp2axi_buffer,
+							    queue);
+			list_del(&scale_vdev->curr_buf->queue);
+		}
+	}
+
+	if (scale_vdev->curr_buf)
+		es_dvp2axi_write_register(dev, frm0_addr,
+				     scale_vdev->curr_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+
+	if (!scale_vdev->next_buf) {
+		if (!list_empty(&scale_vdev->buf_head)) {
+			scale_vdev->next_buf = list_first_entry(&scale_vdev->buf_head,
+							    struct es_dvp2axi_buffer, queue);
+			list_del(&scale_vdev->next_buf->queue);
+		}
+	}
+
+	if (scale_vdev->next_buf)
+		es_dvp2axi_write_register(dev, frm1_addr,
+				     scale_vdev->next_buf->buff_addr[ES_DVP2AXI_PLANE_Y]);
+
+	spin_unlock_irqrestore(&scale_vdev->vbq_lock, flags);
+}
+
+static int es_dvp2axi_assign_scale_buffer_update(struct es_dvp2axi_scale_vdev *scale_vdev,
+					    int channel_id)
+{
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	struct es_dvp2axi_buffer *buffer = NULL;
+	u32 frm_addr;
+	int ret = 0;
+	unsigned long flags;
+
+	frm_addr = scale_vdev->frame_phase & DVP2AXI_CSI_FRAME0_READY ?
+		   get_reg_index_of_scale_frm0_addr(channel_id) :
+		   get_reg_index_of_scale_frm1_addr(channel_id);
+
+	spin_lock_irqsave(&scale_vdev->vbq_lock, flags);
+	if (!list_empty(&scale_vdev->buf_head)) {
+		if (scale_vdev->frame_phase == DVP2AXI_CSI_FRAME0_READY) {
+			scale_vdev->curr_buf = list_first_entry(&scale_vdev->buf_head,
+							    struct es_dvp2axi_buffer, queue);
+			if (scale_vdev->curr_buf) {
+				list_del(&scale_vdev->curr_buf->queue);
+				buffer = scale_vdev->curr_buf;
+			}
+		} else if (scale_vdev->frame_phase == DVP2AXI_CSI_FRAME1_READY) {
+			scale_vdev->next_buf = list_first_entry(&scale_vdev->buf_head,
+							    struct es_dvp2axi_buffer, queue);
+			if (scale_vdev->next_buf) {
+				list_del(&scale_vdev->next_buf->queue);
+				buffer = scale_vdev->next_buf;
+			}
+		}
+	} else {
+		buffer = NULL;
+	}
+	spin_unlock_irqrestore(&scale_vdev->vbq_lock, flags);
+
+	if (buffer) {
+		es_dvp2axi_write_register(dev, frm_addr,
+				     buffer->buff_addr[ES_DVP2AXI_PLANE_Y]);
+	} else {
+		ret = -EINVAL;
+		v4l2_info(&dev->v4l2_dev,
+			 "not active buffer,skip frame, scale ch[%d]\n",
+			  scale_vdev->ch);
+	}
+	return ret;
+}
+
+static int es_dvp2axi_assign_scale_buffer_pingpong(struct es_dvp2axi_scale_vdev *scale_vdev,
+					      int init, int channel_id)
+{
+	int ret = 0;
+
+	if (init)
+		es_dvp2axi_assign_scale_buffer_init(scale_vdev, channel_id);
+	else
+		ret = es_dvp2axi_assign_scale_buffer_update(scale_vdev, channel_id);
+	return ret;
+}
+
+static int es_dvp2axi_scale_channel_set(struct es_dvp2axi_scale_vdev *scale_vdev)
+{
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	u32 val = 0;
+	u32 ch  = scale_vdev->ch;
+
+	val = es_dvp2axi_read_register(dev, DVP2AXI_REG_SCL_CH_CTRL);
+	if (val & DVP2AXI_SCALE_EN(ch)) {
+		v4l2_err(&dev->v4l2_dev, "scale_vdev[%d] has been used by other device\n", ch);
+		return -EINVAL;
+	}
+
+	es_dvp2axi_assign_scale_buffer_pingpong(scale_vdev,
+					   ES_DVP2AXI_YUV_ADDR_STATE_INIT,
+					   ch);
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_SCL_CTRL, SCALE_SOFT_RESET(scale_vdev->ch));
+
+	es_dvp2axi_write_register_and(dev, DVP2AXI_REG_GLB_INTST,
+				 ~(SCALE_END_INTSTAT(ch) |
+				 SCALE_FIFO_OVERFLOW(ch)));
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_GLB_INTEN,
+				(SCALE_END_INTSTAT(ch) |
+				SCALE_FIFO_OVERFLOW(ch) |
+				SCALE_TOISP_AXI0_ERR |
+				SCALE_TOISP_AXI1_ERR));
+	val = DVP2AXI_SCALE_SW_PRESS_ENABLE |
+	      DVP2AXI_SCALE_SW_PRESS_VALUE(7) |
+	      DVP2AXI_SCALE_SW_HURRY_ENABLE |
+	      DVP2AXI_SCALE_SW_HURRY_VALUE(7) |
+	      DVP2AXI_SCALE_SW_WATER_LINE(1);
+
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_SCL_CTRL, val);
+	val = scale_vdev->blc.pattern00 |
+	      (scale_vdev->blc.pattern01 << 8) |
+	      (scale_vdev->blc.pattern02 << 16) |
+	      (scale_vdev->blc.pattern03 << 24);
+	es_dvp2axi_write_register(dev, DVP2AXI_REG_SCL_BLC_CH0 + ch,
+			     val);
+	es_dvp2axi_write_register(dev, get_reg_index_of_scale_vlw(ch),
+			     scale_vdev->ch_info.vir_width);
+	val = DVP2AXI_SCALE_SW_SRC_CH(scale_vdev->ch_src, ch) |
+	      DVP2AXI_SCALE_SW_MODE(scale_vdev->scale_mode, ch) |
+	      DVP2AXI_SCALE_EN(ch);
+	es_dvp2axi_write_register_or(dev, DVP2AXI_REG_SCL_CH_CTRL,
+				val);
+	return 0;
+}
+
+
+int es_dvp2axi_scale_start(struct es_dvp2axi_scale_vdev *scale_vdev)
+{
+	int ret = 0;
+	struct es_dvp2axi_device *dev = scale_vdev->dvp2axidev;
+	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
+
+	mutex_lock(&dev->scale_lock);
+	if (scale_vdev->state == ES_DVP2AXI_STATE_STREAMING) {
+		ret = -EBUSY;
+		v4l2_err(v4l2_dev, "stream in busy state\n");
+		goto destroy_buf;
+	}
+
+	es_dvp2axi_scale_channel_init(scale_vdev);
+	ret = es_dvp2axi_scale_channel_set(scale_vdev);
+	if (ret)
+		goto destroy_buf;
+	scale_vdev->frame_idx = 0;
+	scale_vdev->state = ES_DVP2AXI_STATE_STREAMING;
+	mutex_unlock(&dev->scale_lock);
+	return 0;
+
+destroy_buf:
+	if (scale_vdev->next_buf)
+		vb2_buffer_done(&scale_vdev->next_buf->vb.vb2_buf,
+				VB2_BUF_STATE_QUEUED);
+	if (scale_vdev->curr_buf)
+		vb2_buffer_done(&scale_vdev->curr_buf->vb.vb2_buf,
+				VB2_BUF_STATE_QUEUED);
+	while (!list_empty(&scale_vdev->buf_head)) {
+		struct es_dvp2axi_buffer *buf;
+
+		buf = list_first_entry(&scale_vdev->buf_head,
+				       struct es_dvp2axi_buffer, queue);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+		list_del(&buf->queue);
+	}
+	mutex_unlock(&dev->scale_lock);
+	return ret;
+}
+
+static int
+es_dvp2axi_scale_vb2_start_streaming(struct vb2_queue *queue,
+				unsigned int count)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = queue->drv_priv;
+	struct es_dvp2axi_stream *stream = scale_vdev->stream;
+	int ret = 0;
+
+	if (stream->state == ES_DVP2AXI_STATE_STREAMING) {
+		stream->to_en_scale = true;
+	} else {
+		ret = es_dvp2axi_scale_start(scale_vdev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static struct vb2_ops es_dvp2axi_scale_vb2_ops = {
+	.queue_setup = es_dvp2axi_scale_vb2_queue_setup,
+	.buf_queue = es_dvp2axi_scale_vb2_buf_queue,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+	.stop_streaming = es_dvp2axi_scale_vb2_stop_streaming,
+	.start_streaming = es_dvp2axi_scale_vb2_start_streaming,
+};
+
+static int es_dvp2axi_scale_init_vb2_queue(struct vb2_queue *q,
+				      struct es_dvp2axi_scale_vdev *scale_vdev,
+				      enum v4l2_buf_type buf_type)
+{
+	struct es_dvp2axi_hw *hw_dev = scale_vdev->dvp2axidev->hw_dev;
+
+	q->type = buf_type;
+	q->io_modes = VB2_MMAP | VB2_DMABUF;
+	q->drv_priv = scale_vdev;
+	q->ops = &es_dvp2axi_scale_vb2_ops;
+	q->mem_ops = hw_dev->mem_ops;
+	q->buf_struct_size = sizeof(struct es_dvp2axi_buffer);
+	q->min_buffers_needed = DVP2AXI_SCALE_REQ_BUFS_MIN;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->lock = &scale_vdev->vnode.vlock;
+	q->dev = hw_dev->dev;
+	q->allow_cache_hints = 1;
+	q->bidirectional = 1;
+	q->gfp_flags = GFP_DMA32;
+	if (hw_dev->is_dma_contig)
+		q->dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS;
+	return vb2_queue_init(q);
+}
+
+
+static int es_dvp2axi_scale_g_ch(struct v4l2_device *v4l2_dev,
+			    unsigned int intstat)
+{
+	if (intstat & SCALE_END_INTSTAT(0)) {
+		if ((intstat & SCALE_END_INTSTAT(0)) ==
+		    SCALE_END_INTSTAT(0))
+			v4l2_warn(v4l2_dev, "frame0/1 trigger simultaneously in CH0\n");
+		return ES_DVP2AXI_SCALE_CH0;
+	}
+
+	if (intstat & SCALE_END_INTSTAT(1)) {
+		if ((intstat & SCALE_END_INTSTAT(1)) ==
+		    SCALE_END_INTSTAT(1))
+			v4l2_warn(v4l2_dev, "frame0/1 trigger simultaneously in CH1\n");
+		return ES_DVP2AXI_SCALE_CH1;
+	}
+
+	if (intstat & SCALE_END_INTSTAT(2)) {
+		if ((intstat & SCALE_END_INTSTAT(2)) ==
+		    SCALE_END_INTSTAT(2))
+			v4l2_warn(v4l2_dev, "frame0/1 trigger simultaneously in CH2\n");
+		return ES_DVP2AXI_SCALE_CH2;
+	}
+
+	if (intstat & SCALE_END_INTSTAT(3)) {
+		if ((intstat & SCALE_END_INTSTAT(3)) ==
+		    SCALE_END_INTSTAT(3))
+			v4l2_warn(v4l2_dev, "frame0/1 trigger simultaneously in CH3\n");
+		return ES_DVP2AXI_SCALE_CH3;
+	}
+
+	return -EINVAL;
+}
+
+static void es_dvp2axi_scale_vb_done_oneframe(struct es_dvp2axi_scale_vdev *scale_vdev,
+					 struct vb2_v4l2_buffer *vb_done)
+{
+	const struct dvp2axi_output_fmt *fmt = scale_vdev->scale_out_fmt;
+	u32 i;
+
+	/* Dequeue a filled buffer */
+	for (i = 0; i < fmt->mplanes; i++) {
+		vb2_set_plane_payload(&vb_done->vb2_buf, i,
+				      scale_vdev->pixm.plane_fmt[i].sizeimage);
+	}
+
+	vb_done->vb2_buf.timestamp = es_dvp2axi_time_get_ns(scale_vdev->dvp2axidev);
+
+	vb2_buffer_done(&vb_done->vb2_buf, VB2_BUF_STATE_DONE);
+}
+
+static void es_dvp2axi_scale_update_stream(struct es_dvp2axi_scale_vdev *scale_vdev, int ch)
+{
+	struct es_dvp2axi_buffer *active_buf = NULL;
+	struct vb2_v4l2_buffer *vb_done = NULL;
+	int ret = 0;
+
+	if (scale_vdev->frame_phase & DVP2AXI_CSI_FRAME0_READY) {
+		if (scale_vdev->curr_buf)
+			active_buf = scale_vdev->curr_buf;
+	} else if (scale_vdev->frame_phase & DVP2AXI_CSI_FRAME1_READY) {
+		if (scale_vdev->next_buf)
+			active_buf = scale_vdev->next_buf;
+	}
+
+	ret = es_dvp2axi_assign_scale_buffer_pingpong(scale_vdev,
+					 ES_DVP2AXI_YUV_ADDR_STATE_UPDATE,
+					 ch);
+
+	scale_vdev->frame_idx = scale_vdev->stream->frame_idx;
+	if (active_buf && (!ret)) {
+		vb_done = &active_buf->vb;
+		vb_done->vb2_buf.timestamp = ktime_get_ns();
+		vb_done->sequence = scale_vdev->frame_idx;
+		es_dvp2axi_scale_vb_done_oneframe(scale_vdev, vb_done);
+	}
+}
+
+void es_dvp2axi_irq_handle_scale(struct es_dvp2axi_device *dvp2axi_dev, unsigned int intstat_glb)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev;
+	struct es_dvp2axi_stream *stream;
+	int ch;
+	int i = 0;
+	u32 val = 0;
+
+	val = SCALE_FIFO_OVERFLOW(0) |
+	      SCALE_FIFO_OVERFLOW(1) |
+	      SCALE_FIFO_OVERFLOW(2) |
+	      SCALE_FIFO_OVERFLOW(3);
+	if (intstat_glb & val) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev,
+			"ERROR: scale channel, overflow intstat_glb:0x%x !!\n",
+			intstat_glb);
+		return;
+	}
+
+	ch = es_dvp2axi_scale_g_ch(&dvp2axi_dev->v4l2_dev,
+				      intstat_glb);
+	if (ch < 0)
+		return;
+
+	for (i = 0; i < ES_DVP2AXI_MAX_STREAM_MIPI; i++) {
+		ch = es_dvp2axi_scale_g_ch(&dvp2axi_dev->v4l2_dev,
+				      intstat_glb);
+		if (ch < 0)
+			continue;
+
+		scale_vdev = &dvp2axi_dev->scale_vdev[ch];
+
+		if (scale_vdev->state != ES_DVP2AXI_STATE_STREAMING)
+			continue;
+
+		if (scale_vdev->stopping) {
+			es_dvp2axi_scale_stop(scale_vdev);
+			scale_vdev->stopping = false;
+			wake_up(&scale_vdev->wq_stopped);
+			continue;
+		}
+
+		scale_vdev->frame_phase = SW_SCALE_END(intstat_glb, ch);
+		intstat_glb &= ~(SCALE_END_INTSTAT(ch));
+		es_dvp2axi_scale_update_stream(scale_vdev, ch);
+		stream = scale_vdev->stream;
+		if (stream->to_en_dma)
+			es_dvp2axi_enable_dma_capture(stream, false);
+	}
+}
+
+void es_dvp2axi_init_scale_vdev(struct es_dvp2axi_device *dvp2axi_dev, u32 ch)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev = &dvp2axi_dev->scale_vdev[ch];
+	struct es_dvp2axi_stream *stream = &dvp2axi_dev->stream[ch];
+	struct v4l2_pix_format_mplane pixm;
+
+	memset(scale_vdev, 0, sizeof(*scale_vdev));
+	memset(&pixm, 0, sizeof(pixm));
+	scale_vdev->dvp2axidev = dvp2axi_dev;
+	scale_vdev->stream = stream;
+	stream->scale_vdev = scale_vdev;
+	scale_vdev->ch = ch;
+	scale_vdev->ch_src = 0;
+	scale_vdev->frame_idx = 0;
+	pixm.pixelformat = V4L2_PIX_FMT_SBGGR16;
+	pixm.width = ES_DVP2AXI_DEFAULT_WIDTH;
+	pixm.height = ES_DVP2AXI_DEFAULT_HEIGHT;
+	scale_vdev->state = ES_DVP2AXI_STATE_READY;
+	scale_vdev->stopping = false;
+	scale_vdev->blc.pattern00 = 0;
+	scale_vdev->blc.pattern01 = 0;
+	scale_vdev->blc.pattern02 = 0;
+	scale_vdev->blc.pattern03 = 0;
+	INIT_LIST_HEAD(&scale_vdev->buf_head);
+	spin_lock_init(&scale_vdev->vbq_lock);
+	init_waitqueue_head(&scale_vdev->wq_stopped);
+	es_dvp2axi_scale_set_fmt(scale_vdev, &pixm, false);
+}
+
+static int es_dvp2axi_register_scale_vdev(struct es_dvp2axi_scale_vdev *scale_vdev, bool is_multi_input)
+{
+	int ret = 0;
+	struct video_device *vdev = &scale_vdev->vnode.vdev;
+	struct es_dvp2axi_vdev_node *node;
+	char *vdev_name;
+
+	switch (scale_vdev->ch) {
+	case ES_DVP2AXI_SCALE_CH0:
+		vdev_name = DVP2AXI_SCALE_CH0_VDEV_NAME;
+		break;
+	case ES_DVP2AXI_SCALE_CH1:
+		vdev_name = DVP2AXI_SCALE_CH1_VDEV_NAME;
+		break;
+	case ES_DVP2AXI_SCALE_CH2:
+		vdev_name = DVP2AXI_SCALE_CH2_VDEV_NAME;
+		break;
+	case ES_DVP2AXI_SCALE_CH3:
+		vdev_name = DVP2AXI_SCALE_CH3_VDEV_NAME;
+		break;
+	default:
+		ret = -EINVAL;
+		v4l2_err(&scale_vdev->dvp2axidev->v4l2_dev, "Invalid stream\n");
+		goto err_cleanup_media_entity;
+	}
+
+	strscpy(vdev->name, vdev_name, sizeof(vdev->name));
+	node = container_of(vdev, struct es_dvp2axi_vdev_node, vdev);
+	mutex_init(&node->vlock);
+
+	vdev->ioctl_ops = &es_dvp2axi_scale_ioctl;
+	vdev->fops = &es_dvp2axi_scale_fops;
+	vdev->release = video_device_release_empty;
+	vdev->lock = &node->vlock;
+	vdev->v4l2_dev = &scale_vdev->dvp2axidev->v4l2_dev;
+	vdev->queue = &node->buf_queue;
+	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE |
+			    V4L2_CAP_STREAMING;
+	vdev->vfl_dir =  VFL_DIR_RX;
+	node->pad.flags = MEDIA_PAD_FL_SINK;
+	video_set_drvdata(vdev, scale_vdev);
+
+	es_dvp2axi_scale_init_vb2_queue(&node->buf_queue,
+				   scale_vdev,
+				   V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
+	vdev->queue = &node->buf_queue;
+
+	ret = media_entity_pads_init(&vdev->entity, 1, &node->pad);
+	if (ret < 0)
+		goto err_release_queue;
+
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret < 0) {
+		dev_err(&vdev->dev,
+			"could not register Video for Linux device\n");
+		goto err_cleanup_media_entity;
+	}
+	return 0;
+
+err_cleanup_media_entity:
+	media_entity_cleanup(&vdev->entity);
+err_release_queue:
+	vb2_queue_release(vdev->queue);
+	return ret;
+}
+
+static void es_dvp2axi_unregister_scale_vdev(struct es_dvp2axi_scale_vdev *scale_vdev)
+{
+	struct es_dvp2axi_vdev_node *node = &scale_vdev->vnode;
+	struct video_device *vdev = &node->vdev;
+
+	video_unregister_device(vdev);
+	media_entity_cleanup(&vdev->entity);
+	vb2_queue_release(vdev->queue);
+}
+
+int es_dvp2axi_register_scale_vdevs(struct es_dvp2axi_device *dvp2axi_dev,
+			       int stream_num,
+			       bool is_multi_input)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev;
+	int i, j, ret;
+
+	for (i = 0; i < stream_num; i++) {
+		scale_vdev = &dvp2axi_dev->scale_vdev[i];
+		ret = es_dvp2axi_register_scale_vdev(scale_vdev, is_multi_input);
+		if (ret < 0)
+			goto err;
+	}
+
+	return 0;
+err:
+	for (j = 0; j < i; j++) {
+		scale_vdev = &dvp2axi_dev->scale_vdev[j];
+		es_dvp2axi_unregister_scale_vdev(scale_vdev);
+	}
+
+	return ret;
+}
+
+void es_dvp2axi_unregister_scale_vdevs(struct es_dvp2axi_device *dvp2axi_dev,
+				  int stream_num)
+{
+	struct es_dvp2axi_scale_vdev *scale_vdev;
+	int i;
+
+	for (i = 0; i < stream_num; i++) {
+		scale_vdev = &dvp2axi_dev->scale_vdev[i];
+		es_dvp2axi_unregister_scale_vdev(scale_vdev);
+	}
+}
+
diff --git a/drivers/media/platform/eswin/dvp2axi/dvp2axi.h b/drivers/media/platform/eswin/dvp2axi/dvp2axi.h
new file mode 100644
index 000000000000..7cd41703db2c
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dvp2axi.h
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef __DVP2AXI_HEADER__
+#define __DVP2AXI_HEADER__
+
+#define VI_DVP2AXI_CTRL0_CSR  0x0
+#define VI_DVP2AXI_CTRL0_DVP3_SHIFT_NUM_MASK 0xf0000000
+#define VI_DVP2AXI_CTRL0_DVP2_SHIFT_NUM_MASK 0x0f000000
+#define VI_DVP2AXI_CTRL0_DVP1_SHIFT_NUM_MASK 0x00f00000
+#define VI_DVP2AXI_CTRL0_DVP0_SHIFT_NUM_MASK 0x000f0000
+#define VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK 0x00004000
+#define VI_DVP2AXI_CTRL0_DVP_DATA_SHIFT_MASK 0x00003f00
+#define VI_DVP2AXI_CTRL0_AXI_BURST_LEN_MASK 0x00000080
+#define VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK 0x00000040
+#define VI_DVP2AXI_CTRL0_DVP_CHN_EN_MASK 0x0000003f
+
+#define VI_DVP2AXI_CTRL1_CSR  0x4
+#define VI_DVP2AXI_CTRL1_IO_DVP_MUX_MASK 0xc0000000
+#define VI_DVP2AXI_CTRL1_DVP1_PIXEL_WIDTH_MASK 0x3e000000
+#define VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK 0x01f00000
+#define VI_DVP2AXI_CTRL1_DVP5_PIXEL_MODE_MASK 0x000c0000
+#define VI_DVP2AXI_CTRL1_DVP4_PIXEL_MODE_MASK 0x00030000
+#define VI_DVP2AXI_CTRL1_DVP3_PIXEL_MODE_MASK 0x0000c000
+#define VI_DVP2AXI_CTRL1_DVP2_PIXEL_MODE_MASK 0x00003000
+#define VI_DVP2AXI_CTRL1_DVP1_PIXEL_MODE_MASK 0x00000c00
+#define VI_DVP2AXI_CTRL1_DVP0_PIXEL_MODE_MASK 0x00000300
+#define VI_DVP2AXI_CTRL1_DVP5_SHIFT_NUM_MASK 0x000000f0
+#define VI_DVP2AXI_CTRL1_DVP4_SHIFT_NUM_MASK 0x0000000f
+
+
+#define VI_DVP2AXI_CTRL2_CSR  0x8
+#define VI_DVP2AXI_CTRL2_AXI_OUTSTANDING_MASK 0xff000000
+#define VI_DVP2AXI_CTRL2_AXI_MSTR_WQOS_CFG_MASK 0x00f00000
+#define VI_DVP2AXI_CTRL2_DVP5_PIXEL_WIDTH_MASK 0x000f8000
+#define VI_DVP2AXI_CTRL2_DVP4_PIXEL_WIDTH_MASK 0x00007c00
+#define VI_DVP2AXI_CTRL2_DVP3_PIXEL_WIDTH_MASK 0x000003e0
+#define VI_DVP2AXI_CTRL2_DVP2_PIXEL_WIDTH_MASK 0x0000001f
+
+/* ctrl3~8: dvp 0~5 size */
+#define VI_DVP2AXI_CTRL3_CSR  0xc
+#define VI_DVP2AXI_CTRL4_CSR  0x10
+#define VI_DVP2AXI_CTRL5_CSR  0x14
+#define VI_DVP2AXI_CTRL6_CSR  0x18
+#define VI_DVP2AXI_CTRL7_CSR  0x1c
+#define VI_DVP2AXI_CTRL8_CSR  0x20
+#define VI_DVP2AXI_CTRL_DVP_HS_NUM_MASK  0x0000ffff
+#define VI_DVP2AXI_CTRL_DVP_VS_NUM_MASK  0xffff0000
+
+/* crtl9~32: dvp 0~5, virtual channel 0~2 and embedded(3) base address */
+#define VI_DVP2AXI_CTRL9_CSR  0x24
+#define VI_DVP2AXI_CTRL10_CSR 0x28
+#define VI_DVP2AXI_CTRL11_CSR 0x2c
+#define VI_DVP2AXI_CTRL12_CSR 0x30
+#define VI_DVP2AXI_CTRL13_CSR 0x34
+#define VI_DVP2AXI_CTRL14_CSR 0x38
+#define VI_DVP2AXI_CTRL15_CSR 0x3c
+#define VI_DVP2AXI_CTRL16_CSR 0x40
+#define VI_DVP2AXI_CTRL17_CSR 0x44
+#define VI_DVP2AXI_CTRL18_CSR 0x48
+#define VI_DVP2AXI_CTRL19_CSR 0x4c
+#define VI_DVP2AXI_CTRL20_CSR 0x50
+#define VI_DVP2AXI_CTRL21_CSR 0x54
+#define VI_DVP2AXI_CTRL22_CSR 0x58
+#define VI_DVP2AXI_CTRL23_CSR 0x5c
+#define VI_DVP2AXI_CTRL24_CSR 0x60
+#define VI_DVP2AXI_CTRL25_CSR 0x64
+#define VI_DVP2AXI_CTRL26_CSR 0x68
+#define VI_DVP2AXI_CTRL27_CSR 0x6c
+#define VI_DVP2AXI_CTRL28_CSR 0x70
+#define VI_DVP2AXI_CTRL29_CSR 0x74
+#define VI_DVP2AXI_CTRL30_CSR 0x78
+#define VI_DVP2AXI_CTRL31_CSR 0x7c
+#define VI_DVP2AXI_CTRL32_CSR 0x80
+
+/* ctrl33~35: dvp0~5 H stride setting: 16byte aligned per image line */
+#define VI_DVP2AXI_CTRL33_CSR 0x84
+#define VI_DVP2AXI_CTRL34_CSR 0x88
+#define VI_DVP2AXI_CTRL35_CSR 0x8c
+
+/* ctrl36: dvp virtual id enabled */
+#define VI_DVP2AXI_CTRL36_CSR 0x90
+#define VI_DVP2AXI_DVP0_FIRST_ID_OFFSET 0
+#define VI_DVP2AXI_DVP1_FIRST_ID_OFFSET 2
+#define VI_DVP2AXI_DVP2_FIRST_ID_OFFSET 4
+#define VI_DVP2AXI_DVP3_FIRST_ID_OFFSET 6
+#define VI_DVP2AXI_DVP4_FIRST_ID_OFFSET 8
+#define VI_DVP2AXI_DVP5_FIRST_ID_OFFSET 10
+#define VI_DVP2AXI_DVP0_LAST_ID_OFFSET 18
+#define VI_DVP2AXI_DVP1_LAST_ID_OFFSET 20
+#define VI_DVP2AXI_DVP2_LAST_ID_OFFSET 22
+#define VI_DVP2AXI_DVP3_LAST_ID_OFFSET 24
+#define VI_DVP2AXI_DVP5_LAST_ID_OFFSET 26
+#define VI_DVP2AXI_DVP6_LAST_ID_OFFSET 28
+
+#define VI_DVP2AXI_INT0_CSR 0xa0
+#define VI_DVP2AXI_INT0_DVP2_ID2_FRAME_DONE 0x20000
+#define VI_DVP2AXI_INT0_DVP2_ID1_FRAME_DONE 0x10000
+#define VI_DVP2AXI_INT0_DVP2_ID0_FRAME_DONE 0x8000
+#define VI_DVP2AXI_INT0_DVP1_ID2_FRAME_DONE 0x4000
+#define VI_DVP2AXI_INT0_DVP1_ID1_FRAME_DONE 0x2000
+#define VI_DVP2AXI_INT0_DVP1_ID0_FRAME_DONE 0x1000
+#define VI_DVP2AXI_INT0_DVP0_ID2_FRAME_DONE 0x800
+#define VI_DVP2AXI_INT0_DVP0_ID1_FRAME_DONE 0x400
+#define VI_DVP2AXI_INT0_DVP0_ID0_FRAME_DONE 0x200
+#define VI_DVP2AXI_INT0_DVP2_ID2_FRAME_FLUSH 0x100
+#define VI_DVP2AXI_INT0_DVP2_ID1_FRAME_FLUSH 0x80
+#define VI_DVP2AXI_INT0_DVP2_ID0_FRAME_FLUSH 0x40
+#define VI_DVP2AXI_INT0_DVP1_ID2_FRAME_FLUSH 0x20
+#define VI_DVP2AXI_INT0_DVP1_ID1_FRAME_FLUSH 0x10
+#define VI_DVP2AXI_INT0_DVP1_ID0_FRAME_FLUSH 0x8
+#define VI_DVP2AXI_INT0_DVP0_ID2_FRAME_FLUSH 0x4
+#define VI_DVP2AXI_INT0_DVP0_ID1_FRAME_FLUSH 0x2
+#define VI_DVP2AXI_INT0_DVP0_ID0_FRAME_FLUSH 0x1
+
+
+#define VI_DVP2AXI_INT1_CSR 0xa4
+#define VI_DVP2AXI_INT1_DVP5_ID2_FRAME_DONE 0x20000
+#define VI_DVP2AXI_INT1_DVP5_ID1_FRAME_DONE 0x10000
+#define VI_DVP2AXI_INT1_DVP5_ID0_FRAME_DONE 0x8000
+#define VI_DVP2AXI_INT1_DVP4_ID2_FRAME_DONE 0x4000
+#define VI_DVP2AXI_INT1_DVP4_ID1_FRAME_DONE 0x2000
+#define VI_DVP2AXI_INT1_DVP4_ID0_FRAME_DONE 0x1000
+#define VI_DVP2AXI_INT1_DVP3_ID2_FRAME_DONE 0x800
+#define VI_DVP2AXI_INT1_DVP3_ID1_FRAME_DONE 0x400
+#define VI_DVP2AXI_INT1_DVP3_ID0_FRAME_DONE 0x200
+#define VI_DVP2AXI_INT1_DVP5_ID2_FRAME_FLUSH 0x100
+#define VI_DVP2AXI_INT1_DVP5_ID1_FRAME_FLUSH 0x80
+#define VI_DVP2AXI_INT1_DVP5_ID0_FRAME_FLUSH 0x40
+#define VI_DVP2AXI_INT1_DVP4_ID2_FRAME_FLUSH 0x20
+#define VI_DVP2AXI_INT1_DVP4_ID1_FRAME_FLUSH 0x10
+#define VI_DVP2AXI_INT1_DVP4_ID0_FRAME_FLUSH 0x8
+#define VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH 0x4
+#define VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH 0x2
+#define VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH 0x1
+
+#define VI_DVP2AXI_INT2_CSR 0xa8
+#define VI_DVP2AXI_INT2_AXI_IDBUFFER_AFULL  0x100
+#define VI_DVP2AXI_INT2_DVP5_FRAME_ERROR    0x80
+#define VI_DVP2AXI_INT2_DVP4_FRAME_ERROR    0x40
+#define VI_DVP2AXI_INT2_DVP3_FRAME_ERROR    0x20
+#define VI_DVP2AXI_INT2_DVP2_FRAME_ERROR    0x10
+#define VI_DVP2AXI_INT2_DVP1_FRAME_ERROR    0x8
+#define VI_DVP2AXI_INT2_DVP0_FRAME_ERROR    0x4
+#define VI_DVP2AXI_INT2_AXI_RESP_ERROR      0x2
+#define VI_DVP2AXI_INT2_AXI_IDBUFFER_FULL   0x1
+
+#define VI_DVP2AXI_INT_FRAMEDONE_DVP0 (VI_DVP2AXI_INT0_DVP0_ID2_FRAME_DONE |VI_DVP2AXI_INT0_DVP0_ID1_FRAME_DONE |VI_DVP2AXI_INT0_DVP0_ID0_FRAME_DONE)
+#define VI_DVP2AXI_INT_FRAMEFLUSH_DVP0 (VI_DVP2AXI_INT0_DVP0_ID2_FRAME_FLUSH | VI_DVP2AXI_INT0_DVP0_ID1_FRAME_FLUSH | VI_DVP2AXI_INT0_DVP0_ID0_FRAME_FLUSH)
+
+#define VI_DVP2AXI_INT_FRAMEDONE_DVP1 (VI_DVP2AXI_INT0_DVP1_ID2_FRAME_DONE |VI_DVP2AXI_INT0_DVP1_ID1_FRAME_DONE | VI_DVP2AXI_INT0_DVP1_ID0_FRAME_DONE)
+#define VI_DVP2AXI_INT_FRAMEFLUSH_DVP1 (VI_DVP2AXI_INT0_DVP1_ID2_FRAME_FLUSH | VI_DVP2AXI_INT0_DVP1_ID1_FRAME_FLUSH | VI_DVP2AXI_INT0_DVP1_ID0_FRAME_FLUSH)
+
+#define VI_DVP2AXI_INT_FRAMEDONE_DVP2 (VI_DVP2AXI_INT0_DVP2_ID2_FRAME_DONE |VI_DVP2AXI_INT0_DVP2_ID1_FRAME_DONE | VI_DVP2AXI_INT0_DVP2_ID0_FRAME_DONE)
+#define VI_DVP2AXI_INT_FRAMEFLUSH_DVP2 (VI_DVP2AXI_INT0_DVP2_ID2_FRAME_FLUSH | VI_DVP2AXI_INT0_DVP2_ID1_FRAME_FLUSH | VI_DVP2AXI_INT0_DVP2_ID0_FRAME_FLUSH)
+
+#define VI_DVP2AXI_INT_FRAMEDONE_DVP3 (VI_DVP2AXI_INT1_DVP3_ID2_FRAME_DONE | VI_DVP2AXI_INT1_DVP3_ID1_FRAME_DONE | VI_DVP2AXI_INT1_DVP3_ID0_FRAME_DONE)
+#define VI_DVP2AXI_INT_FRAMEFLUSH_DVP3 (VI_DVP2AXI_INT1_DVP3_ID2_FRAME_FLUSH | VI_DVP2AXI_INT1_DVP3_ID1_FRAME_FLUSH | VI_DVP2AXI_INT1_DVP3_ID0_FRAME_FLUSH)
+
+#define VI_DVP2AXI_INT_FRAMEDONE_DVP4 (VI_DVP2AXI_INT1_DVP4_ID2_FRAME_DONE | VI_DVP2AXI_INT1_DVP4_ID1_FRAME_DONE | VI_DVP2AXI_INT1_DVP4_ID0_FRAME_DONE)
+#define VI_DVP2AXI_INT_FRAMEFLUSH_DVP4 (VI_DVP2AXI_INT1_DVP4_ID2_FRAME_FLUSH | VI_DVP2AXI_INT1_DVP4_ID1_FRAME_FLUSH | VI_DVP2AXI_INT1_DVP4_ID0_FRAME_FLUSH)
+
+#define VI_DVP2AXI_INT_FRAMEDONE_DVP5 (VI_DVP2AXI_INT1_DVP5_ID2_FRAME_DONE | VI_DVP2AXI_INT1_DVP5_ID1_FRAME_DONE | VI_DVP2AXI_INT1_DVP5_ID0_FRAME_DONE)
+#define VI_DVP2AXI_INT_FRAMEFLUSH_DVP5 (VI_DVP2AXI_INT1_DVP5_ID2_FRAME_FLUSH | VI_DVP2AXI_INT1_DVP5_ID1_FRAME_FLUSH | VI_DVP2AXI_INT1_DVP5_ID0_FRAME_FLUSH)
+
+
+#define DVP_PIXEL_MODE_RAW 0x0
+#define DVP_PIXEL_MODE_YUV 0x1
+#define DVP_PIXEL_MODE_RGB 0x2
+
+typedef struct dvp2axi_irq_status {
+    uint32_t dvp_frame_done_irq;
+    uint32_t dvp_frame_flush_irq;
+    uint32_t chn_frame_done[3];
+    uint32_t chn_frame_flush[3];
+}dvp2axi_irq_status;
+
+//little-endian
+typedef union dvp2axi_ctrl36_csr_s {
+    struct {
+        uint32_t dvp0_first_id:2;
+        uint32_t dvp1_first_id:2;
+        uint32_t dvp2_first_id:2;
+        uint32_t dvp3_first_id:2;
+        uint32_t dvp4_first_id:2;
+        uint32_t dvp5_first_id:2;
+        uint32_t dvp_embed_con_en:6;
+        uint32_t dvp0_last_id:2;
+        uint32_t dvp1_last_id:2;
+        uint32_t dvp2_last_id:2;
+        uint32_t dvp3_last_id:2;
+        uint32_t dvp4_last_id:2;
+        uint32_t dvp5_last_id:2;
+        uint32_t reserved:2;
+    } bf;
+    uint32_t dw;
+} dvp2axi_ctrl36_csr_t;
+
+#define VI_DVP2AXI_INT0_CSR 0xa0
+#define VI_DVP2AXI_DVP0_ID0_FRAME_FLUSH_MASK (1 << 0)
+#define VI_DVP2AXI_DVP0_ID1_FRAME_FLUSH_MASK (1 << 1)
+#define VI_DVP2AXI_DVP0_ID2_FRAME_FLUSH_MASK (1 << 2)
+#define VI_DVP2AXI_DVP1_ID0_FRAME_FLUSH_MASK (1 << 3)
+#define VI_DVP2AXI_DVP1_ID1_FRAME_FLUSH_MASK (1 << 4)
+#define VI_DVP2AXI_DVP1_ID2_FRAME_FLUSH_MASK (1 << 5)
+#define VI_DVP2AXI_DVP2_ID0_FRAME_FLUSH_MASK (1 << 6)
+#define VI_DVP2AXI_DVP2_ID1_FRAME_FLUSH_MASK (1 << 7)
+#define VI_DVP2AXI_DVP2_ID2_FRAME_FLUSH_MASK (1 << 8)
+#define VI_DVP2AXI_DVP0_ID0_FRAME_DONE_MASK  (1 << 9)
+#define VI_DVP2AXI_DVP0_ID1_FRAME_DONE_MASK  (1 << 10)
+#define VI_DVP2AXI_DVP0_ID2_FRAME_DONE_MASK  (1 << 11)
+#define VI_DVP2AXI_DVP1_ID0_FRAME_DONE_MASK  (1 << 12)
+#define VI_DVP2AXI_DVP1_ID1_FRAME_DONE_MASK  (1 << 13)
+#define VI_DVP2AXI_DVP1_ID2_FRAME_DONE_MASK  (1 << 14)
+#define VI_DVP2AXI_DVP2_ID0_FRAME_DONE_MASK  (1 << 15)
+#define VI_DVP2AXI_DVP2_ID1_FRAME_DONE_MASK  (1 << 16)
+#define VI_DVP2AXI_DVP2_ID2_FRAME_DONE_MASK  (1 << 17)
+
+#define VI_DVP2AXI_INT1_CSR 0xa4
+#define VI_DVP2AXI_DVP3_ID0_FRAME_FLUSH_MASK (1 << 0)
+#define VI_DVP2AXI_DVP3_ID1_FRAME_FLUSH_MASK (1 << 1)
+#define VI_DVP2AXI_DVP3_ID2_FRAME_FLUSH_MASK (1 << 2)
+#define VI_DVP2AXI_DVP4_ID0_FRAME_FLUSH_MASK (1 << 3)
+#define VI_DVP2AXI_DVP4_ID1_FRAME_FLUSH_MASK (1 << 4)
+#define VI_DVP2AXI_DVP4_ID2_FRAME_FLUSH_MASK (1 << 5)
+#define VI_DVP2AXI_DVP5_ID0_FRAME_FLUSH_MASK (1 << 6)
+#define VI_DVP2AXI_DVP5_ID1_FRAME_FLUSH_MASK (1 << 7)
+#define VI_DVP2AXI_DVP5_ID2_FRAME_FLUSH_MASK (1 << 8)
+#define VI_DVP2AXI_DVP3_ID0_FRAME_DONE_MASK  (1 << 9)
+#define VI_DVP2AXI_DVP3_ID1_FRAME_DONE_MASK  (1 << 10)
+#define VI_DVP2AXI_DVP3_ID2_FRAME_DONE_MASK  (1 << 11)
+#define VI_DVP2AXI_DVP4_ID0_FRAME_DONE_MASK  (1 << 12)
+#define VI_DVP2AXI_DVP4_ID1_FRAME_DONE_MASK  (1 << 13)
+#define VI_DVP2AXI_DVP4_ID2_FRAME_DONE_MASK  (1 << 14)
+#define VI_DVP2AXI_DVP5_ID0_FRAME_DONE_MASK  (1 << 15)
+#define VI_DVP2AXI_DVP5_ID1_FRAME_DONE_MASK  (1 << 16)
+#define VI_DVP2AXI_DVP5_ID2_FRAME_DONE_MASK  (1 << 17)
+
+#define VI_DVP2AXI_INT2_CSR 0xa8
+#define VI_DVP2AXI_AXI_IDBUFFER_FULL_MASK  (1 << 0)
+#define VI_DVP2AXI_AXI_RESP_ERROR_MASK     (1 << 1)
+#define VI_DVP2AXI_DVP0_FRAME_ERROR_MASK   (1 << 2)
+#define VI_DVP2AXI_DVP1_FRAME_ERROR_MASK   (1 << 3)
+#define VI_DVP2AXI_DVP2_FRAME_ERROR_MASK   (1 << 4)
+#define VI_DVP2AXI_DVP3_FRAME_ERROR_MASK   (1 << 5)
+#define VI_DVP2AXI_DVP4_FRAME_ERROR_MASK   (1 << 6)
+#define VI_DVP2AXI_DVP5_FRAME_ERROR_MASK   (1 << 7)
+#define VI_DVP2AXI_AXI_DIBUFFER_AFULL_MASK (1 << 8)
+
+#define VI_DVP2AXI_INT_MASK0_CSR 0xac
+#define VI_DVP2AXI_INT_MASK1_CSR 0xb0
+#define VI_DVP2AXI_INT_MASK2_CSR 0xb4
+
+/* embedded channel setting */
+#define VI_DVP2AXI_CTRL37 0x94
+#define VI_DVP2AXI_CTRL38 0x98
+#define VI_DVP2AXI_CTRL39 0x9c
+#define VI_DVP2AXI_CTRL40 0xe0
+#define VI_DVP2AXI_CTRL41 0xe4
+#define VI_DVP2AXI_CTRL42 0xe8
+#define VI_DVP2AXI_CTRL43 0xec
+
+
+#define VI_DVP2AXI_DVP_CHANNELS     6
+#define VI_DVP2AXI_VIRTUAL_CHANNELS 4
+#define VI_DVP2AXI_CAP_FRAME_NUMBER 0
+
+/* interrupt */
+// #define VI_DVP2AXI_IRQ_ENABLE
+#define VI_DVP2AXI_IRQ_PRIORITY  0
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/dvp2axi_reg.h b/drivers/media/platform/eswin/dvp2axi/dvp2axi_reg.h
new file mode 100644
index 000000000000..bb52b40ceab7
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dvp2axi_reg.h
@@ -0,0 +1,278 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI dvp2axi_reg driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef __DVP2AXI_HEADER__
+#define __DVP2AXI_HEADER__
+
+// #include <u84_metal_ctype.h>
+
+#define VI_DVP2AXI_CTRL0_CSR  0x0
+#define VI_DVP2AXI_CTRL0_DVP3_SHIFT_NUM_MASK 0xf0000000
+#define VI_DVP2AXI_CTRL0_DVP2_SHIFT_NUM_MASK 0x0f000000
+#define VI_DVP2AXI_CTRL0_DVP1_SHIFT_NUM_MASK 0x00f00000
+#define VI_DVP2AXI_CTRL0_DVP0_SHIFT_NUM_MASK 0x000f0000
+#define VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_DONE_MASK 0x00004000
+#define VI_DVP2AXI_CTRL0_DVP_DATA_SHIFT_MASK 0x00003f00
+#define VI_DVP2AXI_CTRL0_AXI_BURST_LEN_MASK 0x00000080
+#define VI_DVP2AXI_CTRL0_AXI_SOFT_RSTN_MASK 0x00000040
+#define VI_DVP2AXI_CTRL0_DVP_CHN_EN_MASK 0x0000003f
+
+#define VI_DVP2AXI_CTRL1_CSR  0x4
+#define VI_DVP2AXI_CTRL1_IO_DVP_MUX_MASK 0xc0000000
+#define VI_DVP2AXI_CTRL1_DVP1_PIXEL_WIDTH_MASK 0x3e000000
+#define VI_DVP2AXI_CTRL1_DVP0_PIXEL_WIDTH_MASK 0x01f00000
+#define VI_DVP2AXI_CTRL1_DVP5_PIXEL_MODE_MASK 0x000c0000
+#define VI_DVP2AXI_CTRL1_DVP4_PIXEL_MODE_MASK 0x00030000
+#define VI_DVP2AXI_CTRL1_DVP3_PIXEL_MODE_MASK 0x0000c000
+#define VI_DVP2AXI_CTRL1_DVP2_PIXEL_MODE_MASK 0x00003000
+#define VI_DVP2AXI_CTRL1_DVP1_PIXEL_MODE_MASK 0x00000c00
+#define VI_DVP2AXI_CTRL1_DVP0_PIXEL_MODE_MASK 0x00000300
+#define VI_DVP2AXI_CTRL1_DVP5_SHIFT_NUM_MASK 0x000000f0
+#define VI_DVP2AXI_CTRL1_DVP4_SHIFT_NUM_MASK 0x0000000f
+
+
+#define VI_DVP2AXI_CTRL2_CSR  0x8
+#define VI_DVP2AXI_CTRL2_AXI_OUTSTANDING_MASK 0xff000000
+#define VI_DVP2AXI_CTRL2_AXI_MSTR_WQOS_CFG_MASK 0x00f00000
+#define VI_DVP2AXI_CTRL2_DVP5_PIXEL_WIDTH_MASK 0x000f8000
+#define VI_DVP2AXI_CTRL2_DVP4_PIXEL_WIDTH_MASK 0x00007c00
+#define VI_DVP2AXI_CTRL2_DVP3_PIXEL_WIDTH_MASK 0x000003e0
+#define VI_DVP2AXI_CTRL2_DVP2_PIXEL_WIDTH_MASK 0x0000001f
+
+/* ctrl3~8: dvp 0~5 size */
+#define VI_DVP2AXI_CTRL3_CSR  0xc
+#define VI_DVP2AXI_CTRL4_CSR  0x10
+#define VI_DVP2AXI_CTRL5_CSR  0x14
+#define VI_DVP2AXI_CTRL6_CSR  0x18
+#define VI_DVP2AXI_CTRL7_CSR  0x1c
+#define VI_DVP2AXI_CTRL8_CSR  0x20
+#define VI_DVP2AXI_CTRL_DVP_HS_NUM_MASK  0x0000ffff
+#define VI_DVP2AXI_CTRL_DVP_VS_NUM_MASK  0xffff0000
+
+/* crtl9~32: dvp 0~5, virtual channel 0~2 and embedded(3) base address */
+#define VI_DVP2AXI_CTRL9_CSR  0x24
+#define VI_DVP2AXI_CTRL10_CSR 0x28
+#define VI_DVP2AXI_CTRL11_CSR 0x2c
+#define VI_DVP2AXI_CTRL12_CSR 0x30
+#define VI_DVP2AXI_CTRL13_CSR 0x34
+#define VI_DVP2AXI_CTRL14_CSR 0x38
+#define VI_DVP2AXI_CTRL15_CSR 0x3c
+#define VI_DVP2AXI_CTRL16_CSR 0x40
+#define VI_DVP2AXI_CTRL17_CSR 0x44
+#define VI_DVP2AXI_CTRL18_CSR 0x48
+#define VI_DVP2AXI_CTRL19_CSR 0x4c
+#define VI_DVP2AXI_CTRL20_CSR 0x50
+#define VI_DVP2AXI_CTRL21_CSR 0x54
+#define VI_DVP2AXI_CTRL22_CSR 0x58
+#define VI_DVP2AXI_CTRL23_CSR 0x5c
+#define VI_DVP2AXI_CTRL24_CSR 0x60
+#define VI_DVP2AXI_CTRL25_CSR 0x64
+#define VI_DVP2AXI_CTRL26_CSR 0x68
+#define VI_DVP2AXI_CTRL27_CSR 0x6c
+#define VI_DVP2AXI_CTRL28_CSR 0x70
+#define VI_DVP2AXI_CTRL29_CSR 0x74
+#define VI_DVP2AXI_CTRL30_CSR 0x78
+#define VI_DVP2AXI_CTRL31_CSR 0x7c
+#define VI_DVP2AXI_CTRL32_CSR 0x80
+
+/* ctrl33~35: dvp0~5 H stride setting: 16byte aligned per image line */
+#define VI_DVP2AXI_CTRL33_CSR 0x84
+#define VI_DVP2AXI_CTRL34_CSR 0x88
+#define VI_DVP2AXI_CTRL35_CSR 0x8c
+
+/* ctrl36: dvp virtual id enabled */
+#define VI_DVP2AXI_CTRL36_CSR 0x90
+#define VI_DVP2AXI_DVP0_FIRST_ID_OFFSET 0
+#define VI_DVP2AXI_DVP1_FIRST_ID_OFFSET 2
+#define VI_DVP2AXI_DVP2_FIRST_ID_OFFSET 4
+#define VI_DVP2AXI_DVP3_FIRST_ID_OFFSET 6
+#define VI_DVP2AXI_DVP4_FIRST_ID_OFFSET 8
+#define VI_DVP2AXI_DVP5_FIRST_ID_OFFSET 10
+#define VI_DVP2AXI_DVP0_LAST_ID_OFFSET 18
+#define VI_DVP2AXI_DVP1_LAST_ID_OFFSET 20
+#define VI_DVP2AXI_DVP2_LAST_ID_OFFSET 22
+#define VI_DVP2AXI_DVP3_LAST_ID_OFFSET 24
+#define VI_DVP2AXI_DVP5_LAST_ID_OFFSET 26
+#define VI_DVP2AXI_DVP6_LAST_ID_OFFSET 28
+
+#define VI_DVP2AXI_INT0_CSR 0xa0
+#define VI_DVP2AXI_INT0_DVP2_ID2_FRAME_DONE 0x20000
+#define VI_DVP2AXI_INT0_DVP2_ID1_FRAME_DONE 0x10000
+#define VI_DVP2AXI_INT0_DVP2_ID0_FRAME_DONE 0x8000
+#define VI_DVP2AXI_INT0_DVP1_ID2_FRAME_DONE 0x4000
+#define VI_DVP2AXI_INT0_DVP1_ID1_FRAME_DONE 0x2000
+#define VI_DVP2AXI_INT0_DVP1_ID0_FRAME_DONE 0x1000
+#define VI_DVP2AXI_INT0_DVP0_ID2_FRAME_DONE 0x800
+#define VI_DVP2AXI_INT0_DVP0_ID1_FRAME_DONE 0x400
+#define VI_DVP2AXI_INT0_DVP0_ID0_FRAME_DONE 0x200
+#define VI_DVP2AXI_INT0_DVP2_ID2_FRAME_FLUSH 0x100
+#define VI_DVP2AXI_INT0_DVP2_ID1_FRAME_FLUSH 0x80
+#define VI_DVP2AXI_INT0_DVP2_ID0_FRAME_FLUSH 0x40
+#define VI_DVP2AXI_INT0_DVP1_ID2_FRAME_FLUSH 0x20
+#define VI_DVP2AXI_INT0_DVP1_ID1_FRAME_FLUSH 0x10
+#define VI_DVP2AXI_INT0_DVP1_ID0_FRAME_FLUSH 0x8
+#define VI_DVP2AXI_INT0_DVP0_ID2_FRAME_FLUSH 0x4
+#define VI_DVP2AXI_INT0_DVP0_ID1_FRAME_FLUSH 0x2
+#define VI_DVP2AXI_INT0_DVP0_ID0_FRAME_FLUSH 0x1
+
+
+#define VI_DVP2AXI_INT1_CSR 0xa4
+#define VI_DVP2AXI_INT0_DVP5_ID2_FRAME_DONE 0x20000
+#define VI_DVP2AXI_INT0_DVP5_ID1_FRAME_DONE 0x10000
+#define VI_DVP2AXI_INT0_DVP5_ID0_FRAME_DONE 0x8000
+#define VI_DVP2AXI_INT0_DVP4_ID2_FRAME_DONE 0x4000
+#define VI_DVP2AXI_INT0_DVP4_ID1_FRAME_DONE 0x2000
+#define VI_DVP2AXI_INT0_DVP4_ID0_FRAME_DONE 0x1000
+#define VI_DVP2AXI_INT0_DVP3_ID2_FRAME_DONE 0x800
+#define VI_DVP2AXI_INT0_DVP3_ID1_FRAME_DONE 0x400
+#define VI_DVP2AXI_INT0_DVP3_ID0_FRAME_DONE 0x200
+#define VI_DVP2AXI_INT0_DVP5_ID2_FRAME_FLUSH 0x100
+#define VI_DVP2AXI_INT0_DVP5_ID1_FRAME_FLUSH 0x80
+#define VI_DVP2AXI_INT0_DVP5_ID0_FRAME_FLUSH 0x40
+#define VI_DVP2AXI_INT0_DVP4_ID2_FRAME_FLUSH 0x20
+#define VI_DVP2AXI_INT0_DVP4_ID1_FRAME_FLUSH 0x10
+#define VI_DVP2AXI_INT0_DVP4_ID0_FRAME_FLUSH 0x8
+#define VI_DVP2AXI_INT0_DVP3_ID2_FRAME_FLUSH 0x4
+#define VI_DVP2AXI_INT0_DVP3_ID1_FRAME_FLUSH 0x2
+#define VI_DVP2AXI_INT0_DVP3_ID0_FRAME_FLUSH 0x1
+
+#define VI_DVP2AXI_INT2_CSR 0xa8
+#define VI_DVP2AXI_INT2_AXI_IDBUFFER_AFULL  0x100
+#define VI_DVP2AXI_INT2_DVP5_FRAME_ERROR    0x80
+#define VI_DVP2AXI_INT2_DVP4_FRAME_ERROR    0x40
+#define VI_DVP2AXI_INT2_DVP3_FRAME_ERROR    0x20
+#define VI_DVP2AXI_INT2_DVP2_FRAME_ERROR    0x10
+#define VI_DVP2AXI_INT2_DVP1_FRAME_ERROR    0x8
+#define VI_DVP2AXI_INT2_DVP0_FRAME_ERROR    0x4
+#define VI_DVP2AXI_INT2_AXI_RESP_ERROR      0x2
+#define VI_DVP2AXI_INT2_AXI_IDBUFFER_FULL   0x1
+
+#define DVP_PIXEL_MODE_RAW 0x0
+#define DVP_PIXEL_MODE_YUV 0x1
+#define DVP_PIXEL_MODE_RGB 0x2
+
+//little-endian
+typedef union dvp2axi_ctrl36_csr_s {
+    struct {
+        uint32_t dvp0_first_id:2;
+        uint32_t dvp1_first_id:2;
+        uint32_t dvp2_first_id:2;
+        uint32_t dvp3_first_id:2;
+        uint32_t dvp4_first_id:2;
+        uint32_t dvp5_first_id:2;
+        uint32_t dvp_embed_con_en:6;
+        uint32_t dvp0_last_id:2;
+        uint32_t dvp1_last_id:2;
+        uint32_t dvp2_last_id:2;
+        uint32_t dvp3_last_id:2;
+        uint32_t dvp4_last_id:2;
+        uint32_t dvp5_last_id:2;
+        uint32_t reserved:2;
+    } bf;
+    uint32_t dw;
+} dvp2axi_ctrl36_csr_t;
+
+#define VI_DVP2AXI_INT0_CSR 0xa0
+#define VI_DVP2AXI_DVP0_ID0_FRAME_FLUSH_MASK (1 << 0)
+#define VI_DVP2AXI_DVP0_ID1_FRAME_FLUSH_MASK (1 << 1)
+#define VI_DVP2AXI_DVP0_ID2_FRAME_FLUSH_MASK (1 << 2)
+#define VI_DVP2AXI_DVP1_ID0_FRAME_FLUSH_MASK (1 << 3)
+#define VI_DVP2AXI_DVP1_ID1_FRAME_FLUSH_MASK (1 << 4)
+#define VI_DVP2AXI_DVP1_ID2_FRAME_FLUSH_MASK (1 << 5)
+#define VI_DVP2AXI_DVP2_ID0_FRAME_FLUSH_MASK (1 << 6)
+#define VI_DVP2AXI_DVP2_ID1_FRAME_FLUSH_MASK (1 << 7)
+#define VI_DVP2AXI_DVP2_ID2_FRAME_FLUSH_MASK (1 << 8)
+#define VI_DVP2AXI_DVP0_ID0_FRAME_DONE_MASK  (1 << 9)
+#define VI_DVP2AXI_DVP0_ID1_FRAME_DONE_MASK  (1 << 10)
+#define VI_DVP2AXI_DVP0_ID2_FRAME_DONE_MASK  (1 << 11)
+#define VI_DVP2AXI_DVP1_ID0_FRAME_DONE_MASK  (1 << 12)
+#define VI_DVP2AXI_DVP1_ID1_FRAME_DONE_MASK  (1 << 13)
+#define VI_DVP2AXI_DVP1_ID2_FRAME_DONE_MASK  (1 << 14)
+#define VI_DVP2AXI_DVP2_ID0_FRAME_DONE_MASK  (1 << 15)
+#define VI_DVP2AXI_DVP2_ID1_FRAME_DONE_MASK  (1 << 16)
+#define VI_DVP2AXI_DVP2_ID2_FRAME_DONE_MASK  (1 << 17)
+
+#define VI_DVP2AXI_INT1_CSR 0xa4
+#define VI_DVP2AXI_DVP3_ID0_FRAME_FLUSH_MASK (1 << 0)
+#define VI_DVP2AXI_DVP3_ID1_FRAME_FLUSH_MASK (1 << 1)
+#define VI_DVP2AXI_DVP3_ID2_FRAME_FLUSH_MASK (1 << 2)
+#define VI_DVP2AXI_DVP4_ID0_FRAME_FLUSH_MASK (1 << 3)
+#define VI_DVP2AXI_DVP4_ID1_FRAME_FLUSH_MASK (1 << 4)
+#define VI_DVP2AXI_DVP4_ID2_FRAME_FLUSH_MASK (1 << 5)
+#define VI_DVP2AXI_DVP5_ID0_FRAME_FLUSH_MASK (1 << 6)
+#define VI_DVP2AXI_DVP5_ID1_FRAME_FLUSH_MASK (1 << 7)
+#define VI_DVP2AXI_DVP5_ID2_FRAME_FLUSH_MASK (1 << 8)
+#define VI_DVP2AXI_DVP3_ID0_FRAME_DONE_MASK  (1 << 9)
+#define VI_DVP2AXI_DVP3_ID1_FRAME_DONE_MASK  (1 << 10)
+#define VI_DVP2AXI_DVP3_ID2_FRAME_DONE_MASK  (1 << 11)
+#define VI_DVP2AXI_DVP4_ID0_FRAME_DONE_MASK  (1 << 12)
+#define VI_DVP2AXI_DVP4_ID1_FRAME_DONE_MASK  (1 << 13)
+#define VI_DVP2AXI_DVP4_ID2_FRAME_DONE_MASK  (1 << 14)
+#define VI_DVP2AXI_DVP5_ID0_FRAME_DONE_MASK  (1 << 15)
+#define VI_DVP2AXI_DVP5_ID1_FRAME_DONE_MASK  (1 << 16)
+#define VI_DVP2AXI_DVP5_ID2_FRAME_DONE_MASK  (1 << 17)
+
+#define VI_DVP2AXI_INT2_CSR 0xa8
+#define VI_DVP2AXI_AXI_IDBUFFER_FULL_MASK  (1 << 0)
+#define VI_DVP2AXI_AXI_RESP_ERROR_MASK     (1 << 1)
+#define VI_DVP2AXI_DVP0_FRAME_ERROR_MASK   (1 << 2)
+#define VI_DVP2AXI_DVP1_FRAME_ERROR_MASK   (1 << 3)
+#define VI_DVP2AXI_DVP2_FRAME_ERROR_MASK   (1 << 4)
+#define VI_DVP2AXI_DVP3_FRAME_ERROR_MASK   (1 << 5)
+#define VI_DVP2AXI_DVP4_FRAME_ERROR_MASK   (1 << 6)
+#define VI_DVP2AXI_DVP5_FRAME_ERROR_MASK   (1 << 7)
+#define VI_DVP2AXI_AXI_DIBUFFER_AFULL_MASK (1 << 8)
+
+#define VI_DVP2AXI_INT_MASK0_CSR 0xac
+#define VI_DVP2AXI_INT_MASK1_CSR 0xb0
+#define VI_DVP2AXI_INT_MASK2_CSR 0xb4
+
+/* embedded channel setting */
+#define VI_DVP2AXI_CTRL37 0x94
+#define VI_DVP2AXI_CTRL38 0x98
+#define VI_DVP2AXI_CTRL39 0x9c
+#define VI_DVP2AXI_CTRL40 0xe0
+#define VI_DVP2AXI_CTRL41 0xe4
+#define VI_DVP2AXI_CTRL42 0xe8
+#define VI_DVP2AXI_CTRL43 0xec
+
+
+#define VI_DVP2AXI_DVP_CHANNELS     6
+#define VI_DVP2AXI_VIRTUAL_CHANNELS 4
+#define VI_DVP2AXI_CAP_FRAME_NUMBER 0
+
+/* interrupt */
+// #define VI_DVP2AXI_IRQ_ENABLE
+#define VI_DVP2AXI_IRQ_PRIORITY  0
+
+// void bmtest_dvp2axi_soft_reset(void);
+// void bmtest_dvp2axi_hw_program(void);
+// void bmtest_dvp2axi_hw_int_setup(void);
+// void bmtest_dvp2axi_hw_enable(void);
+// void bmtest_dvp2axi_hw_wait(void);
+// void bmtest_dvp2axi_hw_dump_regs(void);
+// void bmtest_dvp2axi_hw_notify(int result);
+
+// void bmtest_dvp2axi_buffer_clear(void);
+// int bmtest_dvp2axi_buffer_check(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/dw-mipi-csi-hal.h b/drivers/media/platform/eswin/dvp2axi/dw-mipi-csi-hal.h
new file mode 100644
index 000000000000..e29a6bbef484
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/dw-mipi-csi-hal.h
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI dw-mipi-csi-hal driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _DW_MIPI_CSI_HAL_
+#define _DW_MIPI_CSI_HAL_
+// #include "sys_port.h"
+#include "../vi_top/vitop.h"
+#include <media/eswin/common-def.h>
+
+#define MIPI_CSI_REG_ADDR_0 VI_CSI2_CTRL0_REGISTER_BASE_ADDRESS
+#define MIPI_CSI_REG_SIZE_0 0x00010000UL
+#define MIPI_CSI_IRQ_NUM_0 VI_CSI2_CTRL0_IRQ_NUM
+#define MIPI_CSI_IRQ_NUM_1 VI_CSI2_CTRL1_IRQ_NUM
+#define MIPI_CSI_IRQ_NUM_2 VI_CSI2_CTRL2_IRQ_NUM
+#define MIPI_CSI_IRQ_NUM_3 VI_CSI2_CTRL3_IRQ_NUM
+#define MIPI_CSI_IRQ_NUM_4 VI_CSI2_CTRL4_IRQ_NUM
+#define MIPI_CSI_IRQ_NUM_5 VI_CSI2_CTRL5_IRQ_NUM
+
+/* MIPI CSI2 REGISTERS OFFSET */
+#define VERSION 0x00
+#define N_LANES 0x04
+#define CSI2_RESETN 0x08
+#define INT_ST_MAIN 0x0c
+// #define DATA_IDS_1 0x10
+// #define DATA_IDS_2 0x14
+
+#define PHY_CFG 0x18
+#define PHY_MODE 0x1c
+
+#define INT_ST_AP_MAIN 0x2c
+// #define DATA_IDS_VC_1 0x30
+// #define DATA_IDS_VC_2 0x34
+
+#define PHY_SHUTDOWNZ 0x40
+#define DPHY_RSTZ 0x44
+// #define PHY_RX 0x48
+// #define PHY_STOPSTATE 0x4c
+#define PHY_TEST_CTRL0 0x50
+#define PHY_TEST_CTRL1 0x54
+#define PHY2_TEST_CTRL0 0x58
+#define PHY2_TEST_CTRL1 0x5c
+
+#define PPI_PG_PATTERN_VRES 0x60
+#define PPI_PG_PATTERN_HRES 0x64
+#define PPI_PG_CONFIG 0x68
+#define PPI_PG_ENABLE 0x6c
+#define PPI_PG_STATUS 0x70
+
+#define IPI_MODE 0x80
+#define IPI_VCID 0x84
+#define IPI_DATA_TYPE 0x88
+#define IPI_MEM_FLUSH 0x8c
+#define IPI_HSA_TIME 0x90
+#define IPI_HBP_TIME 0x94
+#define IPI_HSD_TIME 0x98
+#define IPI_HLINE_TIME 0x9c
+#define IPI_SOFTRSTN 0xa0
+#define IPI_ADV_FEATURES 0xac
+#define IPI_VSA_LINES 0xb0
+#define IPI_VBP_LINES 0xb4
+#define IPI_VFP_LINES 0xb8
+#define IPI_VACTIVE_LINES 0xbc
+// #define VC_EXTENSION 0xc8
+// #define PHY_CAL 0xcc
+
+#define IPI2_MODE 0x200
+#define IPI2_VCID 0x204
+#define IPI2_DATA_TYPE 0x208
+#define IPI2_MEM_FLUSH 0x20c
+#define IPI2_HSA_TIME 0x210
+#define IPI2_HBP_TIME 0x214
+#define IPI2_HSD_TIME 0x218
+#define IPI2_ADV_FEATURES 0x21c
+
+#define IPI3_MODE 0x220
+#define IPI3_VCID 0x224
+#define IPI3_DATA_TYPE 0x228
+#define IPI3_MEM_FLUSH 0x22c
+#define IPI3_HSA_TIME 0x230
+#define IPI3_HBP_TIME 0x234
+#define IPI3_HSD_TIME 0x238
+#define IPI3_ADV_FEATURES 0x23c
+
+#define LINE_EVENT_SELECTION_BIT (1 << 16)
+#define EN_VIDEO_BIT (1 << 17)
+#define EN_LINE_START_BIT (1 << 18)
+#define EN_NULL_BIT (1 << 19)
+#define EN_BLANKING_BIT (1 << 20)
+#define EN_EMBEDDED_BIT (1 << 21)
+
+#define LINE_ALL_EVENT_SELECT (EN_EMBEDDED_BIT | EN_BLANKING_BIT | EN_NULL_BIT |    \
+                            EN_LINE_START_BIT | EN_VIDEO_BIT | LINE_EVENT_SELECTION_BIT)
+
+// #define MIPI_CSI_PPI_PATTERN
+// #define MIPI_CSI_PPI_PATTERN_CAMERA_TIMING
+
+enum {
+    D_PHY_PPI_8 = 0,
+    D_PHY_PPI_16 = 1
+};
+
+enum {
+	CAMERA_TIMING = 0,
+	CONTROLLER_TIMING = 1
+};
+
+enum {
+    IPI_DATA_48_BIT = 0,
+    IPI_DATA_16_BIT = 1
+};
+
+/* IPI cut through */
+enum cut_through {
+	IPI_CTINACTIVE = 0,
+	IPI_CTACTIVE = 1
+};
+
+enum {
+	D_PHY_MODE = 0,
+	C_PHY_MODE = 1
+};
+
+/* IPI Data Types */
+enum ipi_data_type {
+	CSI_2_YUV420_8 = 0x18,
+	CSI_2_YUV420_10 = 0x19,
+	CSI_2_YUV420_8_LEG = 0x1A,
+	CSI_2_YUV420_8_SHIFT = 0x1C,
+	CSI_2_YUV420_10_SHIFT = 0x1D,
+	CSI_2_YUV422_8 = 0x1E,
+	CSI_2_YUV422_10 = 0x1F,
+	CSI_2_RGB444 = 0x20,
+	CSI_2_RGB555 = 0x21,
+	CSI_2_RGB565 = 0x22,
+	CSI_2_RGB666 = 0x23,
+	CSI_2_RGB888 = 0x24,
+	CSI_2_RAW6 = 0x28,
+	CSI_2_RAW7 = 0x29,
+	CSI_2_RAW8 = 0x2A,
+	CSI_2_RAW10 = 0x2B,
+	CSI_2_RAW12 = 0x2C,
+	CSI_2_RAW14 = 0x2D,
+	CSI_2_RAW16 = 0x2E,
+	CSI_2_RAW20 = 0x2F,
+	USER_DEFINED_1 = 0x30,
+	USER_DEFINED_2 = 0x31,
+	USER_DEFINED_3 = 0x32,
+	USER_DEFINED_4 = 0x33,
+	USER_DEFINED_5 = 0x34,
+	USER_DEFINED_6 = 0x35,
+	USER_DEFINED_7 = 0x36,
+	USER_DEFINED_8 = 0x37,
+};
+
+// #ifdef MIPI_CSI_PPI_PATTERN
+struct ppi_pattern_info {
+    u32 ppi_patn_vres:16;
+    u32 ppi_patn_hres:16;
+
+    u32 phy_mode:1;
+    u32 ppi_patn_dphy_vc:4;
+    u32 ppi_patn_cphy_vc:5;
+
+    u32 ppi_patn_datatype:6;
+
+    u32 ppi_pg_pattern:1;
+};
+// #endif
+struct control_timing {
+    u32 hsa;
+	u32 hbp;
+	u32 hsd;
+	u32 htotal;
+
+    u32 vsa;
+	u32 vbp;
+	u32 vfp;
+	u32 vactive;
+};
+
+struct csi_data {
+    u32 num_lanes:4;
+    u32 ppi_width:2;
+	u32 phy_mode:1;
+
+    u32 vc:5;
+    u32 dt:6;	
+	u32 emb:1;
+	u32 frame_det:1;
+
+    u32 ipi_mode:1;
+    u32 ipi_color_com:1;
+    u32 ipi_auto_flush:1;
+    u32 ipi_cut_through:1;
+    u32 ipi_line_event;
+
+    u32 hsa;
+	u32 hbp;
+	u32 hsd;
+	u32 htotal;
+
+    u32 vsa;
+	u32 vbp;
+	u32 vfp;
+	u32 vactive;
+};
+
+struct dw_csi {
+	void volatile *base_address;
+
+    struct csi_data hw;
+};
+
+// void mipi_csi_init(uint32_t controller_id);
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/es-dvp2axi-externel.h b/drivers/media/platform/eswin/dvp2axi/es-dvp2axi-externel.h
new file mode 100644
index 000000000000..37e8e3be70bf
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/es-dvp2axi-externel.h
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI externel driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _RKCIF_EXTERNEL_H
+ #define _RKCIF_EXTERNEL_H
+ 
+ #include <media/v4l2-dev.h>
+ 
+ int es_dvp2axi_sditf_disconnect(struct video_device *vdev);
+ 
+ #endif
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/es-isp1-config.h b/drivers/media/platform/eswin/dvp2axi/es-isp1-config.h
new file mode 100644
index 000000000000..0857da8aca00
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/es-isp1-config.h
@@ -0,0 +1,889 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI isp config driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _UAPI_ES_ISP1_CONFIG_H
+ #define _UAPI_ES_ISP1_CONFIG_H
+ 
+ #include <linux/types.h>
+ #include <linux/v4l2-controls.h>
+ #include "es-isp21-config.h"
+ 
+ #define DVP2AXIISP_MODULE_DPCC			(1 << 0)
+ #define DVP2AXIISP_MODULE_BLS			(1 << 1)
+ #define DVP2AXIISP_MODULE_SDG			(1 << 2)
+ #define DVP2AXIISP_MODULE_HST			(1 << 3)
+ #define DVP2AXIISP_MODULE_LSC			(1 << 4)
+ #define DVP2AXIISP_MODULE_AWB_GAIN			(1 << 5)
+ #define DVP2AXIISP_MODULE_FLT			(1 << 6)
+ #define DVP2AXIISP_MODULE_BDM			(1 << 7)
+ #define DVP2AXIISP_MODULE_CTK			(1 << 8)
+ #define DVP2AXIISP_MODULE_GOC			(1 << 9)
+ #define DVP2AXIISP_MODULE_CPROC			(1 << 10)
+ #define DVP2AXIISP_MODULE_AFC			(1 << 11)
+ #define DVP2AXIISP_MODULE_AWB			(1 << 12)
+ #define DVP2AXIISP_MODULE_IE			(1 << 13)
+ #define DVP2AXIISP_MODULE_AEC			(1 << 14)
+ #define DVP2AXIISP_MODULE_WDR			(1 << 15)
+ #define DVP2AXIISP_MODULE_DPF			(1 << 16)
+ #define DVP2AXIISP_MODULE_DPF_STRENGTH		(1 << 17)
+ #define DVP2AXIISP_MODULE_DEMOSAICLP		(1 << 18)
+ #define DVP2AXIISP_MODULE_ES_IESHARP		(1 << 19)
+ 
+ #define DVP2AXIISP_CTK_COEFF_MAX			0x100
+ #define DVP2AXIISP_CTK_OFFSET_MAX			0x800
+ 
+ #define DVP2AXIISP_AE_MEAN_MAX			81
+ #define DVP2AXIISP_HIST_BIN_N_MAX			32
+ #define DVP2AXIISP_AFM_MAX_WINDOWS			3
+ #define DVP2AXIISP_DEGAMMA_CURVE_SIZE		17
+ 
+ #define DVP2AXIISP_BDM_MAX_TH			0xFF
+ 
+ /*
+  * Black level compensation
+  */
+ /* maximum value for horizontal start address */
+ #define DVP2AXIISP_BLS_START_H_MAX			0x00000FFF
+ /* maximum value for horizontal stop address */
+ #define DVP2AXIISP_BLS_STOP_H_MAX			0x00000FFF
+ /* maximum value for vertical start address */
+ #define DVP2AXIISP_BLS_START_V_MAX			0x00000FFF
+ /* maximum value for vertical stop address */
+ #define DVP2AXIISP_BLS_STOP_V_MAX			0x00000FFF
+ /* maximum is 2^18 = 262144*/
+ #define DVP2AXIISP_BLS_SAMPLES_MAX			0x00000012
+ /* maximum value for fixed black level */
+ #define DVP2AXIISP_BLS_FIX_SUB_MAX			0x00000FFF
+ /* minimum value for fixed black level */
+ #define DVP2AXIISP_BLS_FIX_SUB_MIN			0xFFFFF000
+ /* 13 bit range (signed)*/
+ #define DVP2AXIISP_BLS_FIX_MASK			0x00001FFF
+ 
+ /*
+  * Automatic white balance measurments
+  */
+ #define DVP2AXIISP_AWB_MAX_GRID			1
+ #define DVP2AXIISP_AWB_MAX_FRAMES			7
+ 
+ /*
+  * Gamma out
+  */
+ /* Maximum number of color samples supported */
+ #define DVP2AXIISP_GAMMA_OUT_MAX_SAMPLES		34
+ 
+ /*
+  * Lens shade correction
+  */
+ #define DVP2AXIISP_LSC_GRAD_TBL_SIZE		8
+ #define DVP2AXIISP_LSC_SIZE_TBL_SIZE		8
+ /*
+  * The following matches the tuning process,
+  * not the max capabilities of the chip.
+  * Last value unused.
+  */
+ #define	DVP2AXIISP_LSC_DATA_TBL_SIZE		290
+ 
+ /*
+  * Histogram calculation
+  */
+ /* Last 3 values unused. */
+ #define DVP2AXIISP_HISTOGRAM_WEIGHT_GRIDS_SIZE	81
+ 
+ /*
+  * Defect Pixel Cluster Correction
+  */
+ #define DVP2AXIISP_DPCC_METHODS_MAX			3
+ 
+ /*
+  * Denoising pre filter
+  */
+ #define DVP2AXIISP_DPF_MAX_NLF_COEFFS		17
+ #define DVP2AXIISP_DPF_MAX_SPATIAL_COEFFS		6
+ 
+ /* WDR */
+ #define DVP2AXIISP_WDR_SIZE				48
+ 
+ /*
+  * Measurement types
+  */
+ #define DVP2AXIISP_STAT_AWB				(1 << 0)
+ #define DVP2AXIISP_STAT_AUTOEXP			(1 << 1)
+ #define DVP2AXIISP_STAT_AFM_FIN			(1 << 2)
+ #define DVP2AXIISP_STAT_HIST			(1 << 3)
+ #define DVP2AXIISP_STAT_EMB_DATA			(1 << 4)
+ 
+ /* ADD DATA */
+ #define DVP2AXIISP_ADD_DATA_FIFO_SIZE		(2048 * 4)
+ 
+ /* Private v4l2 event */
+ #define DVP2AXIISP_V4L2_EVENT_STREAM_START	\
+                 (V4L2_EVENT_PRIVATE_START + 1)
+ #define DVP2AXIISP_V4L2_EVENT_STREAM_STOP	\
+                 (V4L2_EVENT_PRIVATE_START + 2)
+ 
+ /*
+  * private control id
+  */
+ enum dvp2axiisp_ctrl_id {
+     DVP2AXIISP_CID_EMB_VC = (V4L2_CTRL_CLASS_CAMERA | 0x1001),
+     DVP2AXIISP_CID_EMB_DT,
+     DVP2AXIISP_CID_LAST
+ };
+ 
+ enum dvp2axiisp_histogram_mode {
+     DVP2AXIISP_HISTOGRAM_MODE_DISABLE,
+     DVP2AXIISP_HISTOGRAM_MODE_RGB_COMBINED,
+     DVP2AXIISP_HISTOGRAM_MODE_R_HISTOGRAM,
+     DVP2AXIISP_HISTOGRAM_MODE_G_HISTOGRAM,
+     DVP2AXIISP_HISTOGRAM_MODE_B_HISTOGRAM,
+     DVP2AXIISP_HISTOGRAM_MODE_Y_HISTOGRAM
+ };
+ 
+ enum dvp2axiisp_awb_mode_type {
+     DVP2AXIISP_AWB_MODE_MANUAL,
+     DVP2AXIISP_AWB_MODE_RGB,
+     DVP2AXIISP_AWB_MODE_YCBCR
+ };
+ 
+ enum dvp2axiisp_flt_mode {
+     DVP2AXIISP_FLT_STATIC_MODE,
+     DVP2AXIISP_FLT_DYNAMIC_MODE
+ };
+ 
+ /**
+  * enum dvp2axiisp_exp_ctrl_auotostop - stop modes
+  * @DVP2AXIISP_EXP_CTRL_AUTOSTOP_0: continuous measurement
+  * @DVP2AXIISP_EXP_CTRL_AUTOSTOP_1: stop measuring after a complete frame
+  */
+ enum dvp2axiisp_exp_ctrl_auotostop {
+     DVP2AXIISP_EXP_CTRL_AUTOSTOP_0 = 0,
+     DVP2AXIISP_EXP_CTRL_AUTOSTOP_1 = 1,
+ };
+ 
+ /**
+  * enum dvp2axiisp_exp_meas_mode - Exposure measure mode
+  * @DVP2AXIISP_EXP_MEASURING_MODE_0: Y = 16 + 0.25R + 0.5G + 0.1094B
+  * @DVP2AXIISP_EXP_MEASURING_MODE_1: Y = (R + G + B) x (85/256)
+  */
+ enum dvp2axiisp_exp_meas_mode {
+     DVP2AXIISP_EXP_MEASURING_MODE_0,
+     DVP2AXIISP_EXP_MEASURING_MODE_1,
+ };
+ 
+ /*---------- PART1: Input Parameters ------------*/
+ 
+ struct dvp2axiisp_window {
+     unsigned short h_offs;
+     unsigned short v_offs;
+     unsigned short h_size;
+     unsigned short v_size;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_bls_fixed_val - BLS fixed subtraction values
+  *
+  * The values will be subtracted from the sensor
+  * values. Therefore a negative value means addition instead of subtraction!
+  *
+  * @r: Fixed (signed!) subtraction value for Bayer pattern R
+  * @gr: Fixed (signed!) subtraction value for Bayer pattern Gr
+  * @gb: Fixed (signed!) subtraction value for Bayer pattern Gb
+  * @b: Fixed (signed!) subtraction value for Bayer pattern B
+  */
+ struct dvp2axiisp_bls_fixed_val {
+     signed short r;
+     signed short gr;
+     signed short gb;
+     signed short b;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_bls_config - Configuration used by black level subtraction
+  *
+  * @enable_auto: Automatic mode activated means that the measured values
+  * are subtracted.Otherwise the fixed subtraction
+  * values will be subtracted.
+  * @en_windows: enabled window
+  * @bls_window1: Measurement window 1 size
+  * @bls_window2: Measurement window 2 size
+  * @bls_samples: Set amount of measured pixels for each Bayer position
+  * (A, B,C and D) to 2^bls_samples.
+  * @dvp2axiisp_bls_fixed_val: Fixed subtraction values
+  */
+ struct dvp2axiisp_bls_config {
+     unsigned char enable_auto;
+     unsigned char en_windows;
+     struct dvp2axiisp_window bls_window1;
+     struct dvp2axiisp_window bls_window2;
+     unsigned char bls_samples;
+     struct dvp2axiisp_bls_fixed_val fixed_val;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_dpcc_methods_config - Methods Configuration used by Defect Pixel Cluster Correction
+  *
+  * @method:
+  * @line_thresh:
+  * @line_mad_fac:
+  * @pg_fac:
+  * @rnd_thresh:
+  * @rg_fac:
+  */
+ struct dvp2axiisp_dpcc_methods_config {
+     unsigned int method;
+     unsigned int line_thresh;
+     unsigned int line_mad_fac;
+     unsigned int pg_fac;
+     unsigned int rnd_thresh;
+     unsigned int rg_fac;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_dpcc_methods_config - Configuration used by Defect Pixel Cluster Correction
+  *
+  * @mode: dpcc output mode
+  * @output_mode: whether use hard coded methods
+  * @set_use: stage1 methods set
+  * @methods: methods config
+  * @ro_limits: rank order limits
+  * @rnd_offs: differential rank offsets for rank neighbor difference
+  */
+ struct dvp2axiisp_dpcc_config {
+     unsigned int mode;
+     unsigned int output_mode;
+     unsigned int set_use;
+     struct dvp2axiisp_dpcc_methods_config methods[DVP2AXIISP_DPCC_METHODS_MAX];
+     unsigned int ro_limits;
+     unsigned int rnd_offs;
+ } __attribute__ ((packed));
+ 
+ struct dvp2axiisp_gamma_corr_curve {
+     unsigned short gamma_y[DVP2AXIISP_DEGAMMA_CURVE_SIZE];
+ } __attribute__ ((packed));
+ 
+ struct dvp2axiisp_gamma_curve_x_axis_pnts {
+     unsigned int gamma_dx0;
+     unsigned int gamma_dx1;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_gamma_corr_curve - Configuration used by sensor degamma
+  *
+  * @curve_x: gamma curve point definition axis for x
+  * @xa_pnts: x increments
+  */
+ struct dvp2axiisp_sdg_config {
+     struct dvp2axiisp_gamma_corr_curve curve_r;
+     struct dvp2axiisp_gamma_corr_curve curve_g;
+     struct dvp2axiisp_gamma_corr_curve curve_b;
+     struct dvp2axiisp_gamma_curve_x_axis_pnts xa_pnts;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_lsc_config - Configuration used by Lens shading correction
+  *
+  * refer to datasheet for details
+  */
+ struct dvp2axiisp_lsc_config {
+     unsigned int r_data_tbl[DVP2AXIISP_LSC_DATA_TBL_SIZE];
+     unsigned int gr_data_tbl[DVP2AXIISP_LSC_DATA_TBL_SIZE];
+     unsigned int gb_data_tbl[DVP2AXIISP_LSC_DATA_TBL_SIZE];
+     unsigned int b_data_tbl[DVP2AXIISP_LSC_DATA_TBL_SIZE];
+ 
+     unsigned int x_grad_tbl[DVP2AXIISP_LSC_GRAD_TBL_SIZE];
+     unsigned int y_grad_tbl[DVP2AXIISP_LSC_GRAD_TBL_SIZE];
+ 
+     unsigned int x_size_tbl[DVP2AXIISP_LSC_SIZE_TBL_SIZE];
+     unsigned int y_size_tbl[DVP2AXIISP_LSC_SIZE_TBL_SIZE];
+     unsigned short config_width;
+     unsigned short config_height;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_ie_config - Configuration used by image effects
+  *
+  * @eff_mat_1: 3x3 Matrix Coefficients for Emboss Effect 1
+  * @eff_mat_2: 3x3 Matrix Coefficients for Emboss Effect 2
+  * @eff_mat_3: 3x3 Matrix Coefficients for Emboss 3/Sketch 1
+  * @eff_mat_4: 3x3 Matrix Coefficients for Sketch Effect 2
+  * @eff_mat_5: 3x3 Matrix Coefficients for Sketch Effect 3
+  * @eff_tint: Chrominance increment values of tint (used for sepia effect)
+  */
+ struct dvp2axiisp_ie_config {
+     unsigned short effect;
+     unsigned short color_sel;
+     unsigned short eff_mat_1;
+     unsigned short eff_mat_2;
+     unsigned short eff_mat_3;
+     unsigned short eff_mat_4;
+     unsigned short eff_mat_5;
+     unsigned short eff_tint;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_cproc_config - Configuration used by Color Processing
+  *
+  * @c_out_range: Chrominance pixel clipping range at output. (0 for limit, 1 for full)
+  * @y_in_range: Luminance pixel clipping range at output.
+  * @y_out_range: Luminance pixel clipping range at output.
+  * @contrast: 00~ff, 0.0~1.992
+  * @brightness: 80~7F, -128~+127
+  * @sat: saturation, 00~FF, 0.0~1.992
+  * @hue: 80~7F, -90~+87.188
+  */
+ struct dvp2axiisp_cproc_config {
+     unsigned char c_out_range;
+     unsigned char y_in_range;
+     unsigned char y_out_range;
+     unsigned char contrast;
+     unsigned char brightness;
+     unsigned char sat;
+     unsigned char hue;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_awb_meas_config - Configuration used by auto white balance
+  *
+  * @awb_wnd: white balance measurement window (in pixels)
+  * @max_y: only pixels values < max_y contribute to awb measurement, set to 0 to disable this feature
+  * @min_y: only pixels values > min_y contribute to awb measurement
+  * @max_csum: Chrominance sum maximum value, only consider pixels with Cb+Cr, smaller than threshold for awb measurements
+  * @min_c: Chrominance minimum value, only consider pixels with Cb/Cr each greater than threshold value for awb measurements
+  * @frames: number of frames - 1 used for mean value calculation(ucFrames=0 means 1 Frame)
+  * @awb_ref_cr: reference Cr value for AWB regulation, target for AWB
+  * @awb_ref_cb: reference Cb value for AWB regulation, target for AWB
+  */
+ struct dvp2axiisp_awb_meas_config {
+     /*
+      * Note: currently the h and v offsets are mapped to grid offsets
+      */
+     struct dvp2axiisp_window awb_wnd;
+     enum dvp2axiisp_awb_mode_type awb_mode;
+     unsigned char max_y;
+     unsigned char min_y;
+     unsigned char max_csum;
+     unsigned char min_c;
+     unsigned char frames;
+     unsigned char awb_ref_cr;
+     unsigned char awb_ref_cb;
+     _Bool enable_ymax_cmp;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_awb_gain_config - Configuration used by auto white balance gain
+  *
+  * out_data_x = ( AWB_GEAIN_X * in_data + 128) >> 8
+  */
+ struct dvp2axiisp_awb_gain_config {
+     unsigned short gain_red;
+     unsigned short gain_green_r;
+     unsigned short gain_blue;
+     unsigned short gain_green_b;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_flt_config - Configuration used by ISP filtering
+  *
+  * @mode: ISP_FILT_MODE register fields
+  * @grn_stage1: ISP_FILT_MODE register fields
+  * @chr_h_mode: ISP_FILT_MODE register fields
+  * @chr_v_mode: ISP_FILT_MODE register fields
+  *
+  * refer to datasheet for details.
+  */
+ struct dvp2axiisp_flt_config {
+     enum dvp2axiisp_flt_mode mode;
+     unsigned char grn_stage1;
+     unsigned char chr_h_mode;
+     unsigned char chr_v_mode;
+     unsigned int thresh_bl0;
+     unsigned int thresh_bl1;
+     unsigned int thresh_sh0;
+     unsigned int thresh_sh1;
+     unsigned int lum_weight;
+     unsigned int fac_sh1;
+     unsigned int fac_sh0;
+     unsigned int fac_mid;
+     unsigned int fac_bl0;
+     unsigned int fac_bl1;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_bdm_config - Configuration used by Bayer DeMosaic
+  *
+  * @demosaic_th: threshod for bayer demosaicing texture detection
+  */
+ struct dvp2axiisp_bdm_config {
+     unsigned char demosaic_th;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_ctk_config - Configuration used by Cross Talk correction
+  *
+  * @coeff: color correction matrix
+  * @ct_offset_b: offset for the crosstalk correction matrix
+  */
+ struct dvp2axiisp_ctk_config {
+     unsigned short coeff0;
+     unsigned short coeff1;
+     unsigned short coeff2;
+     unsigned short coeff3;
+     unsigned short coeff4;
+     unsigned short coeff5;
+     unsigned short coeff6;
+     unsigned short coeff7;
+     unsigned short coeff8;
+     unsigned short ct_offset_r;
+     unsigned short ct_offset_g;
+     unsigned short ct_offset_b;
+ } __attribute__ ((packed));
+ 
+ enum dvp2axiisp_goc_mode {
+     DVP2AXIISP_GOC_MODE_LOGARITHMIC,
+     DVP2AXIISP_GOC_MODE_EQUIDISTANT
+ };
+ 
+ /**
+  * struct dvp2axiisp_goc_config - Configuration used by Gamma Out correction
+  *
+  * @mode: goc mode
+  * @gamma_y: gamma out curve y-axis for all color components
+  */
+ struct dvp2axiisp_goc_config {
+     enum dvp2axiisp_goc_mode mode;
+     unsigned short gamma_y[DVP2AXIISP_GAMMA_OUT_MAX_SAMPLES];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_hst_config - Configuration used by Histogram
+  *
+  * @mode: histogram mode
+  * @histogram_predivider: process every stepsize pixel, all other pixels are skipped
+  * @meas_window: coordinates of the meas window
+  * @hist_weight: weighting factor for sub-windows
+  */
+ struct dvp2axiisp_hst_config {
+     enum dvp2axiisp_histogram_mode mode;
+     unsigned char histogram_predivider;
+     struct dvp2axiisp_window meas_window;
+     unsigned char hist_weight[DVP2AXIISP_HISTOGRAM_WEIGHT_GRIDS_SIZE];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_aec_config - Configuration used by Auto Exposure Control
+  *
+  * @mode: Exposure measure mode
+  * @autostop: stop mode (from enum dvp2axiisp_exp_ctrl_auotostop)
+  * @meas_window: coordinates of the meas window
+  */
+ struct dvp2axiisp_aec_config {
+     enum dvp2axiisp_exp_meas_mode mode;
+     __u32 autostop;
+     struct dvp2axiisp_window meas_window;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_afc_config - Configuration used by Auto Focus Control
+  *
+  * @num_afm_win: max DVP2AXIISP_AFM_MAX_WINDOWS
+  * @afm_win: coordinates of the meas window
+  * @thres: threshold used for minimizing the influence of noise
+  * @var_shift: the number of bits for the shift operation at the end of the calculation chain.
+  */
+ struct dvp2axiisp_afc_config {
+     unsigned char num_afm_win;
+     struct dvp2axiisp_window afm_win[DVP2AXIISP_AFM_MAX_WINDOWS];
+     unsigned int thres;
+     unsigned int var_shift;
+ } __attribute__ ((packed));
+ 
+ /**
+  * enum dvp2axiisp_dpf_gain_usage - dpf gain usage
+  * @DVP2AXIISP_DPF_GAIN_USAGE_DISABLED: don't use any gains in preprocessing stage
+  * @DVP2AXIISP_DPF_GAIN_USAGE_NF_GAINS: use only the noise function gains from registers DPF_NF_GAIN_R, ...
+  * @DVP2AXIISP_DPF_GAIN_USAGE_LSC_GAINS:  use only the gains from LSC module
+  * @DVP2AXIISP_DPF_GAIN_USAGE_NF_LSC_GAINS: use the noise function gains and the gains from LSC module
+  * @DVP2AXIISP_DPF_GAIN_USAGE_AWB_GAINS: use only the gains from AWB module
+  * @DVP2AXIISP_DPF_GAIN_USAGE_AWB_LSC_GAINS: use the gains from AWB and LSC module
+  * @DVP2AXIISP_DPF_GAIN_USAGE_MAX: upper border (only for an internal evaluation)
+  */
+ enum dvp2axiisp_dpf_gain_usage {
+     DVP2AXIISP_DPF_GAIN_USAGE_DISABLED,
+     DVP2AXIISP_DPF_GAIN_USAGE_NF_GAINS,
+     DVP2AXIISP_DPF_GAIN_USAGE_LSC_GAINS,
+     DVP2AXIISP_DPF_GAIN_USAGE_NF_LSC_GAINS,
+     DVP2AXIISP_DPF_GAIN_USAGE_AWB_GAINS,
+     DVP2AXIISP_DPF_GAIN_USAGE_AWB_LSC_GAINS,
+     DVP2AXIISP_DPF_GAIN_USAGE_MAX
+ };
+ 
+ /**
+  * enum dvp2axiisp_dpf_gain_usage - dpf gain usage
+  * @DVP2AXIISP_DPF_RB_FILTERSIZE_13x9: red and blue filter kernel size 13x9 (means 7x5 active pixel)
+  * @DVP2AXIISP_DPF_RB_FILTERSIZE_9x9: red and blue filter kernel size 9x9 (means 5x5 active pixel)
+  */
+ enum dvp2axiisp_dpf_rb_filtersize {
+     DVP2AXIISP_DPF_RB_FILTERSIZE_13x9,
+     DVP2AXIISP_DPF_RB_FILTERSIZE_9x9,
+ };
+ 
+ /**
+  * enum dvp2axiisp_dpf_nll_scale_mode - dpf noise level scale mode
+  * @DVP2AXIISP_NLL_SCALE_LINEAR: use a linear scaling
+  * @DVP2AXIISP_NLL_SCALE_LOGARITHMIC: use a logarithmic scaling
+  */
+ enum dvp2axiisp_dpf_nll_scale_mode {
+     DVP2AXIISP_NLL_SCALE_LINEAR,
+     DVP2AXIISP_NLL_SCALE_LOGARITHMIC,
+ };
+ 
+ struct dvp2axiisp_dpf_nll {
+     unsigned short coeff[DVP2AXIISP_DPF_MAX_NLF_COEFFS];
+     enum dvp2axiisp_dpf_nll_scale_mode scale_mode;
+ } __attribute__ ((packed));
+ 
+ struct dvp2axiisp_dpf_rb_flt {
+     enum dvp2axiisp_dpf_rb_filtersize fltsize;
+     unsigned char spatial_coeff[DVP2AXIISP_DPF_MAX_SPATIAL_COEFFS];
+     _Bool r_enable;
+     _Bool b_enable;
+ } __attribute__ ((packed));
+ 
+ struct dvp2axiisp_dpf_g_flt {
+     unsigned char spatial_coeff[DVP2AXIISP_DPF_MAX_SPATIAL_COEFFS];
+     _Bool gr_enable;
+     _Bool gb_enable;
+ } __attribute__ ((packed));
+ 
+ struct dvp2axiisp_dpf_gain {
+     enum dvp2axiisp_dpf_gain_usage mode;
+     unsigned short nf_r_gain;
+     unsigned short nf_b_gain;
+     unsigned short nf_gr_gain;
+     unsigned short nf_gb_gain;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_dpf_config - Configuration used by De-noising pre-filter
+  *
+  * @gain: noise function gain
+  * @g_flt: green filter config
+  * @rb_flt: red blue filter config
+  * @nll: noise level lookup
+  */
+ struct dvp2axiisp_dpf_config {
+     struct dvp2axiisp_dpf_gain gain;
+     struct dvp2axiisp_dpf_g_flt g_flt;
+     struct dvp2axiisp_dpf_rb_flt rb_flt;
+     struct dvp2axiisp_dpf_nll nll;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_dpf_strength_config - strength of the filter
+  *
+  * @r: filter strength of the RED filter
+  * @g: filter strength of the GREEN filter
+  * @b: filter strength of the BLUE filter
+  */
+ struct dvp2axiisp_dpf_strength_config {
+     unsigned char r;
+     unsigned char g;
+     unsigned char b;
+ } __attribute__ ((packed));
+ 
+ /**
+  * enum dvp2axiisp_wdr_mode - wdr mode
+  * @DVP2AXIISP_WDR_MODE_BLOCK: use a linear scaling
+  * @DVP2AXIISP_WDR_MODE_GLOBAL: use a logarithmic scaling
+  */
+ enum dvp2axiisp_wdr_mode {
+     DVP2AXIISP_WDR_MODE_BLOCK,
+     DVP2AXIISP_WDR_MODE_GLOBAL
+ };
+ 
+ /**
+  * struct dvp2axiisp_wdr_config - Gamma Out correction
+  */
+ struct dvp2axiisp_wdr_config {
+     enum dvp2axiisp_wdr_mode mode;
+     unsigned int c_wdr[DVP2AXIISP_WDR_SIZE];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_demosaiclp_config - es demosiac low pass
+  */
+ struct dvp2axiisp_demosaiclp_config {
+     unsigned char rb_filter_en;
+     unsigned char hp_filter_en;
+     unsigned char lu_divided[4];
+     unsigned char thgrad_divided[5];
+     unsigned char thdiff_divided[5];
+     unsigned char thcsc_divided[5];
+     unsigned short thvar_divided[5];
+     unsigned char th_grad;
+     unsigned char th_diff;
+     unsigned char th_csc;
+     unsigned short th_var;
+     unsigned char th_var_en;
+     unsigned char th_csc_en;
+     unsigned char th_diff_en;
+     unsigned char th_grad_en;
+     unsigned char use_old_lp;
+     unsigned char similarity_th;
+     unsigned char flat_level_sel;
+     unsigned char pattern_level_sel;
+     unsigned char edge_level_sel;
+     unsigned char thgrad_r_fct;
+     unsigned char thdiff_r_fct;
+     unsigned char thvar_r_fct;
+     unsigned char thgrad_b_fct;
+     unsigned char thdiff_b_fct;
+     unsigned char thvar_b_fct;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_esiesharp_config - es ie sharp
+  */
+ struct dvp2axiisp_esiesharp_config {
+     unsigned char coring_thr;
+     unsigned char full_range;
+     unsigned char switch_avg;
+     unsigned char yavg_thr[4];
+     unsigned char delta1[5];
+     unsigned char delta2[5];
+     unsigned char maxnumber[5];
+     unsigned char minnumber[5];
+     unsigned char gauss_flat_coe[9];
+     unsigned char gauss_noise_coe[9];
+     unsigned char gauss_other_coe[9];
+     unsigned char line1_filter_coe[6];
+     unsigned char line2_filter_coe[9];
+     unsigned char line3_filter_coe[6];
+     unsigned short grad_seq[4];
+     unsigned char sharp_factor[5];
+     unsigned char uv_gauss_flat_coe[15];
+     unsigned char uv_gauss_noise_coe[15];
+     unsigned char uv_gauss_other_coe[15];
+     unsigned char lap_mat_coe[9];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_isp_other_cfg - Parameters for some blocks in ESWIN isp1
+  *
+  * @dpcc_config: Defect Pixel Cluster Correction config
+  * @bls_config: Black Level Subtraction config
+  * @sdg_config: sensor degamma config
+  * @lsc_config: Lens Shade config
+  * @awb_gain_config: Auto White balance gain config
+  * @flt_config: filter config
+  * @bdm_config: demosaic config
+  * @ctk_config: cross talk config
+  * @goc_config: gamma out config
+  * @bls_config: black level suntraction config
+  * @dpf_config: De-noising pre-filter config
+  * @dpf_strength_config: dpf strength config
+  * @cproc_config: color process config
+  * @ie_config: image effects config
+  */
+ struct dvp2axiisp_isp_other_cfg {
+     struct dvp2axiisp_dpcc_config dpcc_config;
+     struct dvp2axiisp_bls_config bls_config;
+     struct dvp2axiisp_sdg_config sdg_config;
+     struct dvp2axiisp_lsc_config lsc_config;
+     struct dvp2axiisp_awb_gain_config awb_gain_config;
+     struct dvp2axiisp_flt_config flt_config;
+     struct dvp2axiisp_bdm_config bdm_config;
+     struct dvp2axiisp_ctk_config ctk_config;
+     struct dvp2axiisp_goc_config goc_config;
+     struct dvp2axiisp_dpf_config dpf_config;
+     struct dvp2axiisp_dpf_strength_config dpf_strength_config;
+     struct dvp2axiisp_cproc_config cproc_config;
+     struct dvp2axiisp_ie_config ie_config;
+     struct dvp2axiisp_wdr_config wdr_config;
+     struct dvp2axiisp_demosaiclp_config demosaiclp_config;
+     struct dvp2axiisp_esiesharp_config esiesharp_config;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_isp_meas_cfg - ESWIN ISP1 Measure Parameters
+  *
+  * @awb_meas_config: auto white balance config
+  * @hst_config: histogram config
+  * @aec_config: auto exposure config
+  * @afc_config: auto focus config
+  */
+ struct dvp2axiisp_isp_meas_cfg {
+     struct dvp2axiisp_awb_meas_config awb_meas_config;
+     struct dvp2axiisp_hst_config hst_config;
+     struct dvp2axiisp_aec_config aec_config;
+     struct dvp2axiisp_afc_config afc_config;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp1_isp_params_cfg - ESWIN ISP1 Input Parameters Meta Data
+  *
+  * @module_en_update: mask the enable bits of which module  should be updated
+  * @module_ens: mask the enable value of each module, only update the module
+  * which correspond bit was set in module_en_update
+  * @module_cfg_update: mask the config bits of which module  should be updated
+  * @meas: measurement config
+  * @others: other config
+  */
+ struct esisp1_isp_params_cfg {
+     unsigned int module_en_update;
+     unsigned int module_ens;
+     unsigned int module_cfg_update;
+ 
+     struct dvp2axiisp_isp_meas_cfg meas;
+     struct dvp2axiisp_isp_other_cfg others;
+ } __attribute__ ((packed));
+ 
+ /*---------- PART2: Measurement Statistics ------------*/
+ 
+ /**
+  * struct dvp2axiisp_bls_meas_val - AWB measured values
+  *
+  * @cnt: White pixel count, number of "white pixels" found during laster measurement
+  * @mean_y_or_g: Mean value of Y within window and frames, Green if RGB is selected.
+  * @mean_cb_or_b: Mean value of Cb within window and frames, Blue if RGB is selected.
+  * @mean_cr_or_r: Mean value of Cr within window and frames, Red if RGB is selected.
+  */
+ struct dvp2axiisp_awb_meas {
+     unsigned int cnt;
+     unsigned char mean_y_or_g;
+     unsigned char mean_cb_or_b;
+     unsigned char mean_cr_or_r;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_awb_stat - statistics automatic white balance data
+  *
+  * @awb_mean: Mean measured data
+  */
+ struct dvp2axiisp_awb_stat {
+     struct dvp2axiisp_awb_meas awb_mean[DVP2AXIISP_AWB_MAX_GRID];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_bls_meas_val - BLS measured values
+  *
+  * @meas_r: Mean measured value for Bayer pattern R
+  * @meas_gr: Mean measured value for Bayer pattern Gr
+  * @meas_gb: Mean measured value for Bayer pattern Gb
+  * @meas_b: Mean measured value for Bayer pattern B
+  */
+ struct dvp2axiisp_bls_meas_val {
+     unsigned short meas_r;
+     unsigned short meas_gr;
+     unsigned short meas_gb;
+     unsigned short meas_b;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_ae_stat - statistics auto exposure data
+  *
+  * @exp_mean: Mean luminance value of block xx
+  * @bls_val: available wit exposure results
+  *
+  * Image is divided into 5x5 blocks.
+  */
+ struct dvp2axiisp_ae_stat {
+     unsigned char exp_mean[DVP2AXIISP_AE_MEAN_MAX];
+     struct dvp2axiisp_bls_meas_val bls_val;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_af_meas_val - AF measured values
+  *
+  * @sum: sharpness, refer to datasheet for definition
+  * @lum: luminance, refer to datasheet for definition
+  */
+ struct dvp2axiisp_af_meas_val {
+     unsigned int sum;
+     unsigned int lum;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_af_stat - statistics auto focus data
+  *
+  * @window: AF measured value of window x
+  *
+  * The module measures the sharpness in 3 windows of selectable size via
+  * register settings(ISP_AFM_*_A/B/C)
+  */
+ struct dvp2axiisp_af_stat {
+     struct dvp2axiisp_af_meas_val window[DVP2AXIISP_AFM_MAX_WINDOWS];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_hist_stat - statistics histogram data
+  *
+  * @hist_bins: measured bin counters
+  *
+  * Measurement window divided into 25 sub-windows, set
+  * with ISP_HIST_XXX
+  */
+ struct dvp2axiisp_hist_stat {
+     unsigned int hist_bins[DVP2AXIISP_HIST_BIN_N_MAX];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct dvp2axiisp_embedded_data - embedded data
+  *
+  * @data: embedded data
+  *
+  */
+ struct dvp2axiisp_embedded_data {
+     unsigned char data[DVP2AXIISP_ADD_DATA_FIFO_SIZE];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp1_stat_buffer - ESWIN ISP1 Statistics Data
+  *
+  * @dvp2axiisp_awb_stat: statistics data for automatic white balance
+  * @dvp2axiisp_ae_stat: statistics data for auto exposure
+  * @dvp2axiisp_af_stat: statistics data for auto focus
+  * @dvp2axiisp_hist_stat: statistics histogram data
+  */
+ struct dvp2axiisp_stat {
+     struct dvp2axiisp_awb_stat awb;
+     struct dvp2axiisp_ae_stat ae;
+     struct dvp2axiisp_af_stat af;
+     struct dvp2axiisp_hist_stat hist;
+     struct dvp2axiisp_embedded_data emd;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp1_stat_buffer - ESWIN ISP1 Statistics Meta Data
+  *
+  * @meas_type: measurement types (DVP2AXIISP_STAT_ definitions)
+  * @frame_id: frame ID for sync
+  * @params: statistics data
+  */
+ struct esisp1_stat_buffer {
+     unsigned int meas_type;
+     unsigned int frame_id;
+     struct dvp2axiisp_stat params;
+ } __attribute__ ((packed));
+ 
+ #endif /* _UAPI_ES_ISP1_CONFIG_H */
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/es-isp2-config.h b/drivers/media/platform/eswin/dvp2axi/es-isp2-config.h
new file mode 100644
index 000000000000..22d1222ad9e7
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/es-isp2-config.h
@@ -0,0 +1,2035 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI isp config driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _UAPI_ES_ISP2_CONFIG_H
+ #define _UAPI_ES_ISP2_CONFIG_H
+ 
+ #include <linux/const.h>
+ #include <linux/types.h>
+ #include <linux/v4l2-controls.h>
+ 
+ #define ESISP_API_VERSION		KERNEL_VERSION(2, 4, 0)
+ 
+ /****************ISP SUBDEV IOCTL*****************************/
+ 
+ #define ESISP_CMD_TRIGGER_READ_BACK \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 0, struct isp2x_csi_trigger)
+ 
+ #define ESISP_CMD_GET_ISP_INFO \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 1, struct esisp_isp_info)
+ 
+ #define ESISP_CMD_GET_SHARED_BUF \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 2, struct esisp_thunderboot_resmem)
+ 
+ #define ESISP_CMD_FREE_SHARED_BUF \
+     _IO('V', BASE_VIDIOC_PRIVATE + 3)
+ 
+ #define ESISP_CMD_GET_LDCHBUF_INFO \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 4, struct esisp_ldchbuf_info)
+ 
+ #define ESISP_CMD_SET_LDCHBUF_SIZE \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 5, struct esisp_ldchbuf_size)
+ 
+ #define ESISP_CMD_GET_SHM_BUFFD \
+     _IOWR('V', BASE_VIDIOC_PRIVATE + 6, struct esisp_thunderboot_shmem)
+ 
+ #define ESISP_CMD_GET_FBCBUF_FD \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 7, struct isp2x_buf_idxfd)
+ 
+ #define ESISP_CMD_GET_MESHBUF_INFO \
+     _IOWR('V', BASE_VIDIOC_PRIVATE + 8, struct esisp_meshbuf_info)
+ 
+ #define ESISP_CMD_SET_MESHBUF_SIZE \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 9, struct esisp_meshbuf_size)
+ 
+ #define ESISP_CMD_INFO2DDR \
+     _IOWR('V', BASE_VIDIOC_PRIVATE + 10, struct esisp_info2ddr)
+ 
+ #define ESISP_CMD_MESHBUF_FREE \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 11, long long)
+ 
+ /* BASE_VIDIOC_PRIVATE + 12 for ESISP_CMD_GET_TB_HEAD_V32 */
+ 
+ /* for all isp device stop and no power off but resolution change */
+ #define ESISP_CMD_MULTI_DEV_FORCE_ENUM \
+     _IO('V', BASE_VIDIOC_PRIVATE + 13)
+ 
+ /****************ISP VIDEO IOCTL******************************/
+ 
+ #define ESISP_CMD_GET_CSI_MEMORY_MODE \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 100, int)
+ 
+ #define ESISP_CMD_SET_CSI_MEMORY_MODE \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 101, int)
+ 
+ #define ESISP_CMD_GET_CMSK \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 102, struct esisp_cmsk_cfg)
+ 
+ #define ESISP_CMD_SET_CMSK \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 103, struct esisp_cmsk_cfg)
+ 
+ #define ESISP_CMD_GET_STREAM_INFO \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 104, struct esisp_stream_info)
+ 
+ #define ESISP_CMD_GET_MIRROR_FLIP \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 105, struct esisp_mirror_flip)
+ 
+ #define ESISP_CMD_SET_MIRROR_FLIP \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 106, struct esisp_mirror_flip)
+ 
+ #define ESISP_CMD_GET_WRAP_LINE \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 107, struct esisp_wrap_info)
+ /* set wrap line before VIDIOC_S_FMT */
+ #define ESISP_CMD_SET_WRAP_LINE \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 108, struct esisp_wrap_info)
+ 
+ #define ESISP_CMD_SET_FPS \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 109, int)
+ 
+ #define ESISP_CMD_GET_FPS \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 110, int)
+ 
+ #define ESISP_CMD_GET_TB_STREAM_INFO \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 111, struct esisp_tb_stream_info)
+ 
+ #define ESISP_CMD_FREE_TB_STREAM_BUF \
+     _IO('V', BASE_VIDIOC_PRIVATE + 112)
+ 
+ #define ESISP_CMD_SET_IQTOOL_CONN_ID \
+     _IOW('V', BASE_VIDIOC_PRIVATE + 113, int)
+ /*************************************************************/
+ 
+ #define ISP2X_ID_DPCC			(0)
+ #define ISP2X_ID_BLS			(1)
+ #define ISP2X_ID_SDG			(2)
+ #define ISP2X_ID_SIHST			(3)
+ #define ISP2X_ID_LSC			(4)
+ #define ISP2X_ID_AWB_GAIN		(5)
+ #define ISP2X_ID_BDM			(7)
+ #define ISP2X_ID_CCM			(8)
+ #define ISP2X_ID_GOC			(9)
+ #define ISP2X_ID_CPROC			(10)
+ #define ISP2X_ID_SIAF			(11)
+ #define ISP2X_ID_SIAWB			(12)
+ #define ISP2X_ID_IE			(13)
+ #define ISP2X_ID_YUVAE			(14)
+ #define ISP2X_ID_WDR			(15)
+ #define ISP2X_ID_ES_IESHARP		(16)
+ #define ISP2X_ID_RAWAF			(17)
+ #define ISP2X_ID_RAWAE0			(18)
+ #define ISP2X_ID_RAWAE1			(19)
+ #define ISP2X_ID_RAWAE2			(20)
+ #define ISP2X_ID_RAWAE3			(21)
+ #define ISP2X_ID_RAWAWB			(22)
+ #define ISP2X_ID_RAWHIST0		(23)
+ #define ISP2X_ID_RAWHIST1		(24)
+ #define ISP2X_ID_RAWHIST2		(25)
+ #define ISP2X_ID_RAWHIST3		(26)
+ #define ISP2X_ID_HDRMGE			(27)
+ #define ISP2X_ID_RAWNR			(28)
+ #define ISP2X_ID_HDRTMO			(29)
+ #define ISP2X_ID_GIC			(30)
+ #define ISP2X_ID_DHAZ			(31)
+ #define ISP2X_ID_3DLUT			(32)
+ #define ISP2X_ID_LDCH			(33)
+ #define ISP2X_ID_GAIN			(34)
+ #define ISP2X_ID_DEBAYER		(35)
+ #define ISP2X_ID_MAX			(63)
+ 
+ #define ISP2X_MODULE_DPCC		_BITULL(ISP2X_ID_DPCC)
+ #define ISP2X_MODULE_BLS		_BITULL(ISP2X_ID_BLS)
+ #define ISP2X_MODULE_SDG		_BITULL(ISP2X_ID_SDG)
+ #define ISP2X_MODULE_SIHST		_BITULL(ISP2X_ID_SIHST)
+ #define ISP2X_MODULE_LSC		_BITULL(ISP2X_ID_LSC)
+ #define ISP2X_MODULE_AWB_GAIN		_BITULL(ISP2X_ID_AWB_GAIN)
+ #define ISP2X_MODULE_BDM		_BITULL(ISP2X_ID_BDM)
+ #define ISP2X_MODULE_CCM		_BITULL(ISP2X_ID_CCM)
+ #define ISP2X_MODULE_GOC		_BITULL(ISP2X_ID_GOC)
+ #define ISP2X_MODULE_CPROC		_BITULL(ISP2X_ID_CPROC)
+ #define ISP2X_MODULE_SIAF		_BITULL(ISP2X_ID_SIAF)
+ #define ISP2X_MODULE_SIAWB		_BITULL(ISP2X_ID_SIAWB)
+ #define ISP2X_MODULE_IE			_BITULL(ISP2X_ID_IE)
+ #define ISP2X_MODULE_YUVAE		_BITULL(ISP2X_ID_YUVAE)
+ #define ISP2X_MODULE_WDR		_BITULL(ISP2X_ID_WDR)
+ #define ISP2X_MODULE_ES_IESHARP		_BITULL(ISP2X_ID_ES_IESHARP)
+ #define ISP2X_MODULE_RAWAF		_BITULL(ISP2X_ID_RAWAF)
+ #define ISP2X_MODULE_RAWAE0		_BITULL(ISP2X_ID_RAWAE0)
+ #define ISP2X_MODULE_RAWAE1		_BITULL(ISP2X_ID_RAWAE1)
+ #define ISP2X_MODULE_RAWAE2		_BITULL(ISP2X_ID_RAWAE2)
+ #define ISP2X_MODULE_RAWAE3		_BITULL(ISP2X_ID_RAWAE3)
+ #define ISP2X_MODULE_RAWAWB		_BITULL(ISP2X_ID_RAWAWB)
+ #define ISP2X_MODULE_RAWHIST0		_BITULL(ISP2X_ID_RAWHIST0)
+ #define ISP2X_MODULE_RAWHIST1		_BITULL(ISP2X_ID_RAWHIST1)
+ #define ISP2X_MODULE_RAWHIST2		_BITULL(ISP2X_ID_RAWHIST2)
+ #define ISP2X_MODULE_RAWHIST3		_BITULL(ISP2X_ID_RAWHIST3)
+ #define ISP2X_MODULE_HDRMGE		_BITULL(ISP2X_ID_HDRMGE)
+ #define ISP2X_MODULE_RAWNR		_BITULL(ISP2X_ID_RAWNR)
+ #define ISP2X_MODULE_HDRTMO		_BITULL(ISP2X_ID_HDRTMO)
+ #define ISP2X_MODULE_GIC		_BITULL(ISP2X_ID_GIC)
+ #define ISP2X_MODULE_DHAZ		_BITULL(ISP2X_ID_DHAZ)
+ #define ISP2X_MODULE_3DLUT		_BITULL(ISP2X_ID_3DLUT)
+ #define ISP2X_MODULE_LDCH		_BITULL(ISP2X_ID_LDCH)
+ #define ISP2X_MODULE_GAIN		_BITULL(ISP2X_ID_GAIN)
+ #define ISP2X_MODULE_DEBAYER		_BITULL(ISP2X_ID_DEBAYER)
+ 
+ #define ISP2X_MODULE_FORCE		_BITULL(ISP2X_ID_MAX)
+ 
+ /*
+  * Measurement types
+  */
+ #define ISP2X_STAT_SIAWB		_BITUL(0)
+ #define ISP2X_STAT_YUVAE		_BITUL(1)
+ #define ISP2X_STAT_SIAF			_BITUL(2)
+ #define ISP2X_STAT_SIHST		_BITUL(3)
+ #define ISP2X_STAT_EMB_DATA		_BITUL(4)
+ #define ISP2X_STAT_RAWAWB		_BITUL(5)
+ #define ISP2X_STAT_RAWAF		_BITUL(6)
+ #define ISP2X_STAT_RAWAE0		_BITUL(7)
+ #define ISP2X_STAT_RAWAE1		_BITUL(8)
+ #define ISP2X_STAT_RAWAE2		_BITUL(9)
+ #define ISP2X_STAT_RAWAE3		_BITUL(10)
+ #define ISP2X_STAT_RAWHST0		_BITUL(11)
+ #define ISP2X_STAT_RAWHST1		_BITUL(12)
+ #define ISP2X_STAT_RAWHST2		_BITUL(13)
+ #define ISP2X_STAT_RAWHST3		_BITUL(14)
+ #define ISP2X_STAT_BLS			_BITUL(15)
+ #define ISP2X_STAT_HDRTMO		_BITUL(16)
+ #define ISP2X_STAT_DHAZ			_BITUL(17)
+ 
+ #define ISP2X_LSC_GRAD_TBL_SIZE		8
+ #define ISP2X_LSC_SIZE_TBL_SIZE		8
+ #define ISP2X_LSC_DATA_TBL_SIZE		290
+ 
+ #define ISP2X_DEGAMMA_CURVE_SIZE	17
+ 
+ #define ISP2X_GAIN_HDRMGE_GAIN_NUM	3
+ #define ISP2X_GAIN_IDX_NUM		15
+ #define ISP2X_GAIN_LUT_NUM		17
+ 
+ #define ISP2X_AWB_MAX_GRID		1
+ #define ISP2X_RAWAWB_SUM_NUM		7
+ #define ISP2X_RAWAWB_MULWD_NUM		8
+ #define ISP2X_RAWAWB_RAMDATA_NUM	225
+ 
+ #define ISP2X_RAWAEBIG_SUBWIN_NUM	4
+ #define ISP2X_RAWAEBIG_MEAN_NUM		225
+ #define ISP2X_RAWAELITE_MEAN_NUM	25
+ #define ISP2X_YUVAE_SUBWIN_NUM		4
+ #define ISP2X_YUVAE_MEAN_NUM		225
+ 
+ #define ISP2X_RAWHISTBIG_SUBWIN_NUM	225
+ #define ISP2X_RAWHISTLITE_SUBWIN_NUM	25
+ #define ISP2X_SIHIST_WIN_NUM		1
+ #define ISP2X_HIST_WEIGHT_NUM		225
+ #define ISP2X_HIST_BIN_N_MAX		256
+ #define ISP2X_SIHIST_BIN_N_MAX		32
+ 
+ #define ISP2X_RAWAF_WIN_NUM		2
+ #define ISP2X_RAWAF_LINE_NUM		5
+ #define ISP2X_RAWAF_GAMMA_NUM		17
+ #define ISP2X_RAWAF_SUMDATA_ROW		15
+ #define ISP2X_RAWAF_SUMDATA_COLUMN	15
+ #define ISP2X_RAWAF_SUMDATA_NUM		225
+ #define ISP2X_AFM_MAX_WINDOWS		3
+ 
+ #define ISP2X_DPCC_PDAF_POINT_NUM	16
+ 
+ #define ISP2X_HDRMGE_L_CURVE_NUM	17
+ #define ISP2X_HDRMGE_E_CURVE_NUM	17
+ 
+ #define ISP2X_RAWNR_LUMA_RATION_NUM	8
+ 
+ #define ISP2X_HDRTMO_MINMAX_NUM		32
+ 
+ #define ISP2X_GIC_SIGMA_Y_NUM		15
+ 
+ #define ISP2X_CCM_CURVE_NUM		17
+ 
+ /* WDR */
+ #define ISP2X_WDR_SIZE			48
+ 
+ #define ISP2X_DHAZ_CONV_COEFF_NUM	6
+ #define ISP2X_DHAZ_HIST_IIR_NUM		64
+ 
+ #define ISP2X_GAMMA_OUT_MAX_SAMPLES	45
+ 
+ #define ISP2X_MIPI_LUMA_MEAN_MAX	16
+ #define ISP2X_MIPI_RAW_MAX		3
+ #define ISP2X_RAW0_Y_STATE		(1 << 0)
+ #define ISP2X_RAW1_Y_STATE		(1 << 1)
+ #define ISP2X_RAW2_Y_STATE		(1 << 2)
+ 
+ #define ISP2X_3DLUT_DATA_NUM		729
+ 
+ #define ISP2X_LDCH_MESH_XY_NUM		0x80000
+ #define ISP2X_LDCH_BUF_NUM		2
+ 
+ #define ISP2X_THUNDERBOOT_VIDEO_BUF_NUM	30
+ 
+ #define ISP2X_FBCBUF_FD_NUM		64
+ 
+ #define ISP2X_MESH_BUF_NUM		2
+ 
+ enum esisp_isp_mode {
+     /* frame input related */
+     ESISP_ISP_NORMAL = _BITUL(0),
+     ESISP_ISP_HDR2 = _BITUL(1),
+     ESISP_ISP_HDR3 = _BITUL(2),
+     ESISP_ISP_COMPR = _BITUL(3),
+ 
+     /* isp function related */
+     ESISP_ISP_BIGMODE = _BITUL(28),
+ };
+ 
+ struct esisp_isp_info {
+     enum esisp_isp_mode mode;
+     __u32 act_width;
+     __u32 act_height;
+     __u8 compr_bit;
+ } __attribute__ ((packed));
+ 
+ enum isp2x_mesh_buf_stat {
+     MESH_BUF_INIT = 0,
+     MESH_BUF_WAIT2CHIP,
+     MESH_BUF_CHIPINUSE,
+ };
+ 
+ struct esisp_meshbuf_info {
+     __u64 module_id;
+     __u32 unite_isp_id;
+     __s32 buf_fd[ISP2X_MESH_BUF_NUM];
+     __u32 buf_size[ISP2X_MESH_BUF_NUM];
+ } __attribute__ ((packed));
+ 
+ struct esisp_meshbuf_size {
+     __u64 module_id;
+     __u32 unite_isp_id;
+     __u32 meas_width;
+     __u32 meas_height;
+     int buf_cnt;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_mesh_head {
+     enum isp2x_mesh_buf_stat stat;
+     __u32 data_oft;
+ } __attribute__ ((packed));
+ 
+ #define ESISP_CMSK_WIN_MAX 12
+ #define ESISP_CMSK_WIN_MAX_V30 8
+ #define ESISP_CMSK_MOSAIC_MODE 0
+ #define ESISP_CMSK_COVER_MODE 1
+ 
+ /* struct esisp_cmsk_win
+  * Priacy Mask Window configture, support windows
+  * ESISP_CMSK_WIN_MAX_V30 for es3588 support 8 windows, and
+  * support for mainpath and selfpath output stream channel.
+  *
+  * ESISP_CMSK_WIN_MAX for rv1106 support 12 windows, and
+  * support for mainpath selfpath and bypasspath output stream channel.
+  *
+  * mode: 0:mosaic mode, 1:cover mode
+  * win_index: window index 0~11. windows overlap, priority win11 > win0.
+  * cover_color_y: cover mode effective, share for stream channel when same win_index.
+  * cover_color_u: cover mode effective, share for stream channel when same win_index.
+  * cover_color_v: cover mode effective, share for stream channel when same win_index.
+  *
+  * h_offs: window horizontal offset, share for stream channel when same win_index. 2 align.
+  * v_offs: window vertical offset, share for stream channel when same win_index. 2 align.
+  * h_size: window horizontal size, share for stream channel when same win_index. 8 align for es3588, 2 align for rv1106.
+  * v_size: window vertical size, share for stream channel when same win_index. 8 align for es3588, 2 align for rv1106.
+  */
+ struct esisp_cmsk_win {
+     unsigned short mode;
+     unsigned short win_en;
+ 
+     unsigned char cover_color_y;
+     unsigned char cover_color_u;
+     unsigned char cover_color_v;
+ 
+     unsigned short h_offs;
+     unsigned short v_offs;
+     unsigned short h_size;
+     unsigned short v_size;
+ } __attribute__ ((packed));
+ 
+ /* struct esisp_cmsk_cfg
+  * win: priacy mask window
+  * mosaic_block: Mosaic block size, 0:8x8 1:16x16 2:32x32 3:64x64, share for all windows
+  * width_ro: isp full resolution, h_offs + h_size <= width_ro.
+  * height_ro: isp full resolution, v_offs + v_size <= height_ro.
+  */
+ struct esisp_cmsk_cfg {
+     struct esisp_cmsk_win win[ESISP_CMSK_WIN_MAX];
+     unsigned int mosaic_block;
+     unsigned int width_ro;
+     unsigned int height_ro;
+ } __attribute__ ((packed));
+ 
+ /* struct esisp_stream_info
+  * cur_frame_id: stream current frame id
+  * input_frame_loss: isp input frame loss num
+  * output_frame_loss: stream output frame loss num
+  * stream_on: stream on/off
+  */
+ struct esisp_stream_info {
+     unsigned int cur_frame_id;
+     unsigned int input_frame_loss;
+     unsigned int output_frame_loss;
+     unsigned char stream_on;
+     unsigned char stream_id;
+ } __attribute__ ((packed));
+ 
+ /* struct esisp_mirror_flip
+  * mirror: global for all output stream
+  * flip: independent for all output stream
+  */
+ struct esisp_mirror_flip {
+     unsigned char mirror;
+     unsigned char flip;
+ } __attribute__ ((packed));
+ 
+ struct esisp_wrap_info {
+     int width;
+     int height;
+ };
+ 
+ #define ESISP_TB_STREAM_BUF_MAX 5
+ struct esisp_tb_stream_buf {
+     unsigned int dma_addr;
+     unsigned int sequence;
+     long long timestamp;
+ } __attribute__ ((packed));
+ 
+ /* struct esisp_tb_stream_info
+  * frame_size: nv12 frame buf size, bytesperline * height_16align * 1.5
+  * buf_max: memory size / frame_size
+  * buf_cnt: the num of frame write to buf.
+  */
+ struct esisp_tb_stream_info {
+     unsigned int width;
+     unsigned int height;
+     unsigned int bytesperline;
+     unsigned int frame_size;
+     unsigned int buf_max;
+     unsigned int buf_cnt;
+     struct esisp_tb_stream_buf buf[ESISP_TB_STREAM_BUF_MAX];
+ } __attribute__ ((packed));
+ 
+ /* trigger event mode
+  * T_TRY: trigger maybe with retry
+  * T_TRY_YES: trigger to retry
+  * T_TRY_NO: trigger no to retry
+  *
+  * T_START_X1: isp read one frame
+  * T_START_X2: isp read hdr two frame
+  * T_START_X3: isp read hdr three frame
+  * T_START_C: isp read hdr linearised and compressed data
+  */
+ enum isp2x_trigger_mode {
+     T_TRY = _BITUL(0),
+     T_TRY_YES = _BITUL(1),
+     T_TRY_NO = _BITUL(2),
+ 
+     T_START_X1 = _BITUL(4),
+     T_START_X2 = _BITUL(5),
+     T_START_X3 = _BITUL(6),
+     T_START_C = _BITUL(7),
+ };
+ 
+ struct isp2x_csi_trigger {
+     /* timestamp in ns */
+     __u64 sof_timestamp;
+     __u64 frame_timestamp;
+     __u32 frame_id;
+     int times;
+     enum isp2x_trigger_mode mode;
+ } __attribute__ ((packed));
+ 
+ /* isp csi dmatx/dmarx memory mode
+  * 0: raw12/raw10/raw8 8bit memory compact
+  * 1: raw12/raw10 16bit memory one pixel
+  *    big endian for rv1126/rv1109
+  *    |15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
+  *    | 3| 2| 1| 0| -| -| -| -|11|10| 9| 8| 7| 6| 5| 4|
+  *    little align for es356x
+  *    |15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
+  *    | -| -| -| -|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
+  * 2: raw12/raw10 16bit memory one pixel
+  *    big align for rv1126/rv1109/es356x
+  *    |15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
+  *    |11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0| -| -| -| -|
+  */
+ enum isp_csi_memory {
+     CSI_MEM_COMPACT = 0,
+     CSI_MEM_WORD_BIG_END = 1,
+     CSI_MEM_WORD_LITTLE_ALIGN = 1,
+     CSI_MEM_WORD_BIG_ALIGN = 2,
+ };
+ 
+ #define ESISP_INFO2DDR_BUF_MAX	4
+ /* 32bit flag for user set to memory after buf used */
+ #define ESISP_INFO2DDR_BUF_INIT 0x5AA5
+ 
+ enum esisp_info2ddr_owner {
+     ESISP_INFO2DRR_OWNER_NULL,
+     ESISP_INFO2DRR_OWNER_GAIN,
+     ESISP_INFO2DRR_OWNER_AWB,
+ };
+ 
+ /* struct esisp_info2ddr
+  * awb and gain debug info write to ddr
+  *
+  * owner: 0: off, 1: gain, 2: awb.
+  * u: gain or awb mode parameters.
+  * buf_cnt: buf num to request. return actual result.
+  * buf_fd: fd of memory alloc result.
+  * wsize: data width to request. if useless to 0. return actual result.
+  * vsize: data height to request. if useless to 0. return actual result.
+  */
+ struct esisp_info2ddr {
+     enum esisp_info2ddr_owner owner;
+ 
+     union {
+         struct {
+             __u8 gain2ddr_mode;
+         } gain;
+ 
+         struct {
+             __u8 awb2ddr_sel;
+         } awb;
+     } u;
+ 
+     __u8 buf_cnt;
+     __s32 buf_fd[ESISP_INFO2DDR_BUF_MAX];
+ 
+     __u32 wsize;
+     __u32 vsize;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_ispgain_buf {
+     __u32 gain_dmaidx;
+     __u32 mfbc_dmaidx;
+     __u32 gain_size;
+     __u32 mfbc_size;
+     __u32 frame_id;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_buf_idxfd {
+     __u32 buf_num;
+     __u32 index[ISP2X_FBCBUF_FD_NUM];
+     __s32 dmafd[ISP2X_FBCBUF_FD_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_window {
+     __u16 h_offs;
+     __u16 v_offs;
+     __u16 h_size;
+     __u16 v_size;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_bls_fixed_val {
+     __s16 r;
+     __s16 gr;
+     __s16 gb;
+     __s16 b;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_bls_cfg {
+     __u8 enable_auto;
+     __u8 en_windows;
+     struct isp2x_window bls_window1;
+     struct isp2x_window bls_window2;
+     __u8 bls_samples;
+     struct isp2x_bls_fixed_val fixed_val;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_bls_stat {
+     __u16 meas_r;
+     __u16 meas_gr;
+     __u16 meas_gb;
+     __u16 meas_b;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_dpcc_pdaf_point {
+     __u8 y;
+     __u8 x;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_dpcc_cfg {
+     /* mode 0x0000 */
+     __u8 stage1_enable;
+     __u8 grayscale_mode;
+ 
+     /* output_mode 0x0004 */
+     __u8 sw_es_out_sel;
+     __u8 sw_dpcc_output_sel;
+     __u8 stage1_rb_3x3;
+     __u8 stage1_g_3x3;
+     __u8 stage1_incl_rb_center;
+     __u8 stage1_incl_green_center;
+ 
+     /* set_use 0x0008 */
+     __u8 stage1_use_fix_set;
+     __u8 stage1_use_set_3;
+     __u8 stage1_use_set_2;
+     __u8 stage1_use_set_1;
+ 
+     /* methods_set_1 0x000c */
+     __u8 sw_es_red_blue1_en;
+     __u8 rg_red_blue1_enable;
+     __u8 rnd_red_blue1_enable;
+     __u8 ro_red_blue1_enable;
+     __u8 lc_red_blue1_enable;
+     __u8 pg_red_blue1_enable;
+     __u8 sw_es_green1_en;
+     __u8 rg_green1_enable;
+     __u8 rnd_green1_enable;
+     __u8 ro_green1_enable;
+     __u8 lc_green1_enable;
+     __u8 pg_green1_enable;
+ 
+     /* methods_set_2 0x0010 */
+     __u8 sw_es_red_blue2_en;
+     __u8 rg_red_blue2_enable;
+     __u8 rnd_red_blue2_enable;
+     __u8 ro_red_blue2_enable;
+     __u8 lc_red_blue2_enable;
+     __u8 pg_red_blue2_enable;
+     __u8 sw_es_green2_en;
+     __u8 rg_green2_enable;
+     __u8 rnd_green2_enable;
+     __u8 ro_green2_enable;
+     __u8 lc_green2_enable;
+     __u8 pg_green2_enable;
+ 
+     /* methods_set_3 0x0014 */
+     __u8 sw_es_red_blue3_en;
+     __u8 rg_red_blue3_enable;
+     __u8 rnd_red_blue3_enable;
+     __u8 ro_red_blue3_enable;
+     __u8 lc_red_blue3_enable;
+     __u8 pg_red_blue3_enable;
+     __u8 sw_es_green3_en;
+     __u8 rg_green3_enable;
+     __u8 rnd_green3_enable;
+     __u8 ro_green3_enable;
+     __u8 lc_green3_enable;
+     __u8 pg_green3_enable;
+ 
+     /* line_thresh_1 0x0018 */
+     __u8 sw_mindis1_rb;
+     __u8 sw_mindis1_g;
+     __u8 line_thr_1_rb;
+     __u8 line_thr_1_g;
+ 
+     /* line_mad_fac_1 0x001c */
+     __u8 sw_dis_scale_min1;
+     __u8 sw_dis_scale_max1;
+     __u8 line_mad_fac_1_rb;
+     __u8 line_mad_fac_1_g;
+ 
+     /* pg_fac_1 0x0020 */
+     __u8 pg_fac_1_rb;
+     __u8 pg_fac_1_g;
+ 
+     /* rnd_thresh_1 0x0024 */
+     __u8 rnd_thr_1_rb;
+     __u8 rnd_thr_1_g;
+ 
+     /* rg_fac_1 0x0028 */
+     __u8 rg_fac_1_rb;
+     __u8 rg_fac_1_g;
+ 
+     /* line_thresh_2 0x002c */
+     __u8 sw_mindis2_rb;
+     __u8 sw_mindis2_g;
+     __u8 line_thr_2_rb;
+     __u8 line_thr_2_g;
+ 
+     /* line_mad_fac_2 0x0030 */
+     __u8 sw_dis_scale_min2;
+     __u8 sw_dis_scale_max2;
+     __u8 line_mad_fac_2_rb;
+     __u8 line_mad_fac_2_g;
+ 
+     /* pg_fac_2 0x0034 */
+     __u8 pg_fac_2_rb;
+     __u8 pg_fac_2_g;
+ 
+     /* rnd_thresh_2 0x0038 */
+     __u8 rnd_thr_2_rb;
+     __u8 rnd_thr_2_g;
+ 
+     /* rg_fac_2 0x003c */
+     __u8 rg_fac_2_rb;
+     __u8 rg_fac_2_g;
+ 
+     /* line_thresh_3 0x0040 */
+     __u8 sw_mindis3_rb;
+     __u8 sw_mindis3_g;
+     __u8 line_thr_3_rb;
+     __u8 line_thr_3_g;
+ 
+     /* line_mad_fac_3 0x0044 */
+     __u8 sw_dis_scale_min3;
+     __u8 sw_dis_scale_max3;
+     __u8 line_mad_fac_3_rb;
+     __u8 line_mad_fac_3_g;
+ 
+     /* pg_fac_3 0x0048 */
+     __u8 pg_fac_3_rb;
+     __u8 pg_fac_3_g;
+ 
+     /* rnd_thresh_3 0x004c */
+     __u8 rnd_thr_3_rb;
+     __u8 rnd_thr_3_g;
+ 
+     /* rg_fac_3 0x0050 */
+     __u8 rg_fac_3_rb;
+     __u8 rg_fac_3_g;
+ 
+     /* ro_limits 0x0054 */
+     __u8 ro_lim_3_rb;
+     __u8 ro_lim_3_g;
+     __u8 ro_lim_2_rb;
+     __u8 ro_lim_2_g;
+     __u8 ro_lim_1_rb;
+     __u8 ro_lim_1_g;
+ 
+     /* rnd_offs 0x0058 */
+     __u8 rnd_offs_3_rb;
+     __u8 rnd_offs_3_g;
+     __u8 rnd_offs_2_rb;
+     __u8 rnd_offs_2_g;
+     __u8 rnd_offs_1_rb;
+     __u8 rnd_offs_1_g;
+ 
+     /* bpt_ctrl 0x005c */
+     __u8 bpt_rb_3x3;
+     __u8 bpt_g_3x3;
+     __u8 bpt_incl_rb_center;
+     __u8 bpt_incl_green_center;
+     __u8 bpt_use_fix_set;
+     __u8 bpt_use_set_3;
+     __u8 bpt_use_set_2;
+     __u8 bpt_use_set_1;
+     __u8 bpt_cor_en;
+     __u8 bpt_det_en;
+ 
+     /* bpt_number 0x0060 */
+     __u16 bp_number;
+ 
+     /* bpt_addr 0x0064 */
+     __u16 bp_table_addr;
+ 
+     /* bpt_data 0x0068 */
+     __u16 bpt_v_addr;
+     __u16 bpt_h_addr;
+ 
+     /* bp_cnt 0x006c */
+     __u32 bp_cnt;
+ 
+     /* pdaf_en 0x0070 */
+     __u8 sw_pdaf_en;
+ 
+     /* pdaf_point_en 0x0074 */
+     __u8 pdaf_point_en[ISP2X_DPCC_PDAF_POINT_NUM];
+ 
+     /* pdaf_offset 0x0078 */
+     __u16 pdaf_offsety;
+     __u16 pdaf_offsetx;
+ 
+     /* pdaf_wrap 0x007c */
+     __u16 pdaf_wrapy;
+     __u16 pdaf_wrapx;
+ 
+     /* pdaf_scope 0x0080 */
+     __u16 pdaf_wrapy_num;
+     __u16 pdaf_wrapx_num;
+ 
+     /* pdaf_point_0 0x0084 */
+     struct isp2x_dpcc_pdaf_point point[ISP2X_DPCC_PDAF_POINT_NUM];
+ 
+     /* pdaf_forward_med 0x00a4 */
+     __u8 pdaf_forward_med;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_hdrmge_curve {
+     __u16 curve_1[ISP2X_HDRMGE_L_CURVE_NUM];
+     __u16 curve_0[ISP2X_HDRMGE_L_CURVE_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_hdrmge_cfg {
+     __u8 mode;
+ 
+     __u16 gain0_inv;
+     __u16 gain0;
+ 
+     __u16 gain1_inv;
+     __u16 gain1;
+ 
+     __u8 gain2;
+ 
+     __u8 lm_dif_0p15;
+     __u8 lm_dif_0p9;
+     __u8 ms_diff_0p15;
+     __u8 ms_dif_0p8;
+ 
+     struct isp2x_hdrmge_curve curve;
+     __u16 e_y[ISP2X_HDRMGE_E_CURVE_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawnr_cfg {
+     __u8 gauss_en;
+     __u8 log_bypass;
+ 
+     __u16 filtpar0;
+     __u16 filtpar1;
+     __u16 filtpar2;
+ 
+     __u32 dgain0;
+     __u32 dgain1;
+     __u32 dgain2;
+ 
+     __u16 luration[ISP2X_RAWNR_LUMA_RATION_NUM];
+     __u16 lulevel[ISP2X_RAWNR_LUMA_RATION_NUM];
+ 
+     __u32 gauss;
+     __u16 sigma;
+     __u16 pix_diff;
+ 
+     __u32 thld_diff;
+ 
+     __u8 gas_weig_scl2;
+     __u8 gas_weig_scl1;
+     __u16 thld_chanelw;
+ 
+     __u16 lamda;
+ 
+     __u16 fixw0;
+     __u16 fixw1;
+     __u16 fixw2;
+     __u16 fixw3;
+ 
+     __u32 wlamda0;
+     __u32 wlamda1;
+     __u32 wlamda2;
+ 
+     __u16 rgain_filp;
+     __u16 bgain_filp;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_lsc_cfg {
+     __u16 r_data_tbl[ISP2X_LSC_DATA_TBL_SIZE];
+     __u16 gr_data_tbl[ISP2X_LSC_DATA_TBL_SIZE];
+     __u16 gb_data_tbl[ISP2X_LSC_DATA_TBL_SIZE];
+     __u16 b_data_tbl[ISP2X_LSC_DATA_TBL_SIZE];
+ 
+     __u16 x_grad_tbl[ISP2X_LSC_GRAD_TBL_SIZE];
+     __u16 y_grad_tbl[ISP2X_LSC_GRAD_TBL_SIZE];
+ 
+     __u16 x_size_tbl[ISP2X_LSC_SIZE_TBL_SIZE];
+     __u16 y_size_tbl[ISP2X_LSC_SIZE_TBL_SIZE];
+ } __attribute__ ((packed));
+ 
+ enum isp2x_goc_mode {
+     ISP2X_GOC_MODE_LOGARITHMIC,
+     ISP2X_GOC_MODE_EQUIDISTANT
+ };
+ 
+ struct isp2x_goc_cfg {
+     enum isp2x_goc_mode mode;
+     __u8 gamma_y[17];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_hdrtmo_predict {
+     __u8 global_tmo;
+     __s32 iir_max;
+     __s32 global_tmo_strength;
+ 
+     __u8 scene_stable;
+     __s32 k_rolgmean;
+     __s32 iir;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_hdrtmo_cfg {
+     __u16 cnt_vsize;
+     __u8 gain_ld_off2;
+     __u8 gain_ld_off1;
+     __u8 big_en;
+     __u8 nobig_en;
+     __u8 newhst_en;
+     __u8 cnt_mode;
+ 
+     __u16 expl_lgratio;
+     __u8 lgscl_ratio;
+     __u8 cfg_alpha;
+ 
+     __u16 set_gainoff;
+     __u16 set_palpha;
+ 
+     __u16 set_lgmax;
+     __u16 set_lgmin;
+ 
+     __u8 set_weightkey;
+     __u16 set_lgmean;
+ 
+     __u16 set_lgrange1;
+     __u16 set_lgrange0;
+ 
+     __u16 set_lgavgmax;
+ 
+     __u8 clipgap1_i;
+     __u8 clipgap0_i;
+     __u8 clipratio1;
+     __u8 clipratio0;
+     __u8 ratiol;
+ 
+     __u16 lgscl_inv;
+     __u16 lgscl;
+ 
+     __u16 lgmax;
+ 
+     __u16 hist_low;
+     __u16 hist_min;
+ 
+     __u8 hist_shift;
+     __u16 hist_0p3;
+     __u16 hist_high;
+ 
+     __u16 palpha_lwscl;
+     __u16 palpha_lw0p5;
+     __u16 palpha_0p18;
+ 
+     __u16 maxgain;
+     __u16 maxpalpha;
+ 
+     struct isp2x_hdrtmo_predict predict;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_hdrtmo_stat {
+     __u16 lglow;
+     __u16 lgmin;
+     __u16 lghigh;
+     __u16 lgmax;
+     __u16 weightkey;
+     __u16 lgmean;
+     __u16 lgrange1;
+     __u16 lgrange0;
+     __u16 palpha;
+     __u16 lgavgmax;
+     __u16 linecnt;
+     __u32 min_max[ISP2X_HDRTMO_MINMAX_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_gic_cfg {
+     __u8 edge_open;
+ 
+     __u16 regmingradthrdark2;
+     __u16 regmingradthrdark1;
+     __u16 regminbusythre;
+ 
+     __u16 regdarkthre;
+     __u16 regmaxcorvboth;
+     __u16 regdarktthrehi;
+ 
+     __u8 regkgrad2dark;
+     __u8 regkgrad1dark;
+     __u8 regstrengthglobal_fix;
+     __u8 regdarkthrestep;
+     __u8 regkgrad2;
+     __u8 regkgrad1;
+     __u8 reggbthre;
+ 
+     __u16 regmaxcorv;
+     __u16 regmingradthr2;
+     __u16 regmingradthr1;
+ 
+     __u8 gr_ratio;
+     __u16 dnloscale;
+     __u16 dnhiscale;
+     __u8 reglumapointsstep;
+ 
+     __u16 gvaluelimitlo;
+     __u16 gvaluelimithi;
+     __u8 fusionratiohilimt1;
+ 
+     __u8 regstrength_fix;
+ 
+     __u16 sigma_y[ISP2X_GIC_SIGMA_Y_NUM];
+ 
+     __u8 noise_cut_en;
+     __u16 noise_coe_a;
+ 
+     __u16 noise_coe_b;
+     __u16 diff_clip;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_debayer_cfg {
+     __u8 filter_c_en;
+     __u8 filter_g_en;
+ 
+     __u8 thed1;
+     __u8 thed0;
+     __u8 dist_scale;
+     __u8 max_ratio;
+     __u8 clip_en;
+ 
+     __s8 filter1_coe5;
+     __s8 filter1_coe4;
+     __s8 filter1_coe3;
+     __s8 filter1_coe2;
+     __s8 filter1_coe1;
+ 
+     __s8 filter2_coe5;
+     __s8 filter2_coe4;
+     __s8 filter2_coe3;
+     __s8 filter2_coe2;
+     __s8 filter2_coe1;
+ 
+     __u16 hf_offset;
+     __u8 gain_offset;
+     __u8 offset;
+ 
+     __u8 shift_num;
+     __u8 order_max;
+     __u8 order_min;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_ccm_cfg {
+     __s16 coeff0_r;
+     __s16 coeff1_r;
+     __s16 coeff2_r;
+     __s16 offset_r;
+ 
+     __s16 coeff0_g;
+     __s16 coeff1_g;
+     __s16 coeff2_g;
+     __s16 offset_g;
+ 
+     __s16 coeff0_b;
+     __s16 coeff1_b;
+     __s16 coeff2_b;
+     __s16 offset_b;
+ 
+     __u16 coeff0_y;
+     __u16 coeff1_y;
+     __u16 coeff2_y;
+ 
+     __u16 alp_y[ISP2X_CCM_CURVE_NUM];
+ 
+     __u8 bound_bit;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_gammaout_cfg {
+     __u8 equ_segm;
+     __u16 offset;
+     __u16 gamma_y[ISP2X_GAMMA_OUT_MAX_SAMPLES];
+ } __attribute__ ((packed));
+ 
+ enum isp2x_wdr_mode {
+     ISP2X_WDR_MODE_BLOCK,
+     ISP2X_WDR_MODE_GLOBAL
+ };
+ 
+ struct isp2x_wdr_cfg {
+     enum isp2x_wdr_mode mode;
+     unsigned int c_wdr[ISP2X_WDR_SIZE];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_dhaz_cfg {
+     __u8 enhance_en;
+     __u8 hist_chn;
+     __u8 hpara_en;
+     __u8 hist_en;
+     __u8 dc_en;
+     __u8 big_en;
+     __u8 nobig_en;
+ 
+     __u8 yblk_th;
+     __u8 yhist_th;
+     __u8 dc_max_th;
+     __u8 dc_min_th;
+ 
+     __u16 wt_max;
+     __u8 bright_max;
+     __u8 bright_min;
+ 
+     __u8 tmax_base;
+     __u8 dark_th;
+     __u8 air_max;
+     __u8 air_min;
+ 
+     __u16 tmax_max;
+     __u16 tmax_off;
+ 
+     __u8 hist_th_off;
+     __u8 hist_gratio;
+ 
+     __u16 hist_min;
+     __u16 hist_k;
+ 
+     __u16 enhance_value;
+     __u16 hist_scale;
+ 
+     __u16 iir_wt_sigma;
+     __u16 iir_sigma;
+     __u16 stab_fnum;
+ 
+     __u16 iir_tmax_sigma;
+     __u16 iir_air_sigma;
+ 
+     __u16 cfg_wt;
+     __u16 cfg_air;
+     __u16 cfg_alpha;
+ 
+     __u16 cfg_gratio;
+     __u16 cfg_tmax;
+ 
+     __u16 dc_weitcur;
+     __u16 dc_thed;
+ 
+     __u8 sw_dhaz_dc_bf_h3;
+     __u8 sw_dhaz_dc_bf_h2;
+     __u8 sw_dhaz_dc_bf_h1;
+     __u8 sw_dhaz_dc_bf_h0;
+ 
+     __u8 sw_dhaz_dc_bf_h5;
+     __u8 sw_dhaz_dc_bf_h4;
+ 
+     __u16 air_weitcur;
+     __u16 air_thed;
+ 
+     __u8 air_bf_h2;
+     __u8 air_bf_h1;
+     __u8 air_bf_h0;
+ 
+     __u8 gaus_h2;
+     __u8 gaus_h1;
+     __u8 gaus_h0;
+ 
+     __u8 conv_t0[ISP2X_DHAZ_CONV_COEFF_NUM];
+     __u8 conv_t1[ISP2X_DHAZ_CONV_COEFF_NUM];
+     __u8 conv_t2[ISP2X_DHAZ_CONV_COEFF_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_dhaz_stat {
+     __u16 dhaz_adp_air_base;
+     __u16 dhaz_adp_wt;
+ 
+     __u16 dhaz_adp_gratio;
+     __u16 dhaz_adp_tmax;
+ 
+     __u16 h_r_iir[ISP2X_DHAZ_HIST_IIR_NUM];
+     __u16 h_g_iir[ISP2X_DHAZ_HIST_IIR_NUM];
+     __u16 h_b_iir[ISP2X_DHAZ_HIST_IIR_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_cproc_cfg {
+     __u8 c_out_range;
+     __u8 y_in_range;
+     __u8 y_out_range;
+     __u8 contrast;
+     __u8 brightness;
+     __u8 sat;
+     __u8 hue;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_ie_cfg {
+     __u16 effect;
+     __u16 color_sel;
+     __u16 eff_mat_1;
+     __u16 eff_mat_2;
+     __u16 eff_mat_3;
+     __u16 eff_mat_4;
+     __u16 eff_mat_5;
+     __u16 eff_tint;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_esiesharp_cfg {
+     __u8 coring_thr;
+     __u8 full_range;
+     __u8 switch_avg;
+     __u8 yavg_thr[4];
+     __u8 delta1[5];
+     __u8 delta2[5];
+     __u8 maxnumber[5];
+     __u8 minnumber[5];
+     __u8 gauss_flat_coe[9];
+     __u8 gauss_noise_coe[9];
+     __u8 gauss_other_coe[9];
+     __u8 line1_filter_coe[6];
+     __u8 line2_filter_coe[9];
+     __u8 line3_filter_coe[6];
+     __u16 grad_seq[4];
+     __u8 sharp_factor[5];
+     __u8 uv_gauss_flat_coe[15];
+     __u8 uv_gauss_noise_coe[15];
+     __u8 uv_gauss_other_coe[15];
+     __u8 lap_mat_coe[9];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_superimp_cfg {
+     __u8 transparency_mode;
+     __u8 ref_image;
+ 
+     __u16 offset_x;
+     __u16 offset_y;
+ 
+     __u8 y_comp;
+     __u8 cb_comp;
+     __u8 cr_comp;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_gamma_corr_curve {
+     __u16 gamma_y[ISP2X_DEGAMMA_CURVE_SIZE];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_gamma_curve_x_axis_pnts {
+     __u32 gamma_dx0;
+     __u32 gamma_dx1;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_sdg_cfg {
+     struct isp2x_gamma_corr_curve curve_r;
+     struct isp2x_gamma_corr_curve curve_g;
+     struct isp2x_gamma_corr_curve curve_b;
+     struct isp2x_gamma_curve_x_axis_pnts xa_pnts;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_bdm_config {
+     unsigned char demosaic_th;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_gain_cfg {
+     __u8 dhaz_en;
+     __u8 wdr_en;
+     __u8 tmo_en;
+     __u8 lsc_en;
+     __u8 mge_en;
+ 
+     __u32 mge_gain[ISP2X_GAIN_HDRMGE_GAIN_NUM];
+     __u16 idx[ISP2X_GAIN_IDX_NUM];
+     __u16 lut[ISP2X_GAIN_LUT_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_3dlut_cfg {
+     __u8 bypass_en;
+     __u32 actual_size;	/* word unit */
+     __u16 lut_r[ISP2X_3DLUT_DATA_NUM];
+     __u16 lut_g[ISP2X_3DLUT_DATA_NUM];
+     __u16 lut_b[ISP2X_3DLUT_DATA_NUM];
+ } __attribute__ ((packed));
+ 
+ enum isp2x_ldch_buf_stat {
+     LDCH_BUF_INIT = 0,
+     LDCH_BUF_WAIT2CHIP,
+     LDCH_BUF_CHIPINUSE,
+ };
+ 
+ struct esisp_ldchbuf_info {
+     __s32 buf_fd[ISP2X_LDCH_BUF_NUM];
+     __u32 buf_size[ISP2X_LDCH_BUF_NUM];
+ } __attribute__ ((packed));
+ 
+ struct esisp_ldchbuf_size {
+     __u32 meas_width;
+     __u32 meas_height;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_ldch_head {
+     enum isp2x_ldch_buf_stat stat;
+     __u32 data_oft;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_ldch_cfg {
+     __u32 hsize;
+     __u32 vsize;
+     __s32 buf_fd;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_awb_gain_cfg {
+     __u16 gain_red;
+     __u16 gain_green_r;
+     __u16 gain_blue;
+     __u16 gain_green_b;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siawb_meas_cfg {
+     struct isp2x_window awb_wnd;
+     __u8 awb_mode;
+     __u8 max_y;
+     __u8 min_y;
+     __u8 max_csum;
+     __u8 min_c;
+     __u8 frames;
+     __u8 awb_ref_cr;
+     __u8 awb_ref_cb;
+     __u8 enable_ymax_cmp;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawawb_meas_cfg {
+     __u8 rawawb_sel;
+     __u8 sw_rawawb_light_num;		/* CTRL */
+     __u8 sw_rawawb_wind_size;		/* CTRL */
+     __u8 sw_rawawb_c_range;			/* CTRL */
+     __u8 sw_rawawb_y_range;			/* CTRL */
+     __u8 sw_rawawb_3dyuv_ls_idx3;		/* CTRL */
+     __u8 sw_rawawb_3dyuv_ls_idx2;		/* CTRL */
+     __u8 sw_rawawb_3dyuv_ls_idx1;		/* CTRL */
+     __u8 sw_rawawb_3dyuv_ls_idx0;		/* CTRL */
+     __u8 sw_rawawb_xy_en;			/* CTRL */
+     __u8 sw_rawawb_uv_en;			/* CTRL */
+     __u8 sw_rawlsc_bypass_en;		/* CTRL */
+     __u8 sw_rawawb_blk_measure_mode;	/* BLK_CTRL */
+     __u8 sw_rawawb_store_wp_flag_ls_idx2;	/* BLK_CTRL */
+     __u8 sw_rawawb_store_wp_flag_ls_idx1;	/* BLK_CTRL */
+     __u8 sw_rawawb_store_wp_flag_ls_idx0;	/* BLK_CTRL */
+     __u16 sw_rawawb_store_wp_th0;		/* BLK_CTRL */
+     __u16 sw_rawawb_store_wp_th1;		/* BLK_CTRL */
+     __u16 sw_rawawb_store_wp_th2;		/* RAW_CTRL */
+     __u16 sw_rawawb_v_offs;			/* WIN_OFFS */
+     __u16 sw_rawawb_h_offs;			/* WIN_OFFS */
+     __u16 sw_rawawb_v_size;			/* WIN_SIZE */
+     __u16 sw_rawawb_h_size;			/* WIN_SIZE */
+     __u16 sw_rawawb_g_max;			/* LIMIT_RG_MAX */
+     __u16 sw_rawawb_r_max;			/* LIMIT_RG_MAX */
+     __u16 sw_rawawb_y_max;			/* LIMIT_BY_MAX */
+     __u16 sw_rawawb_b_max;			/* LIMIT_BY_MAX */
+     __u16 sw_rawawb_g_min;			/* LIMIT_RG_MIN */
+     __u16 sw_rawawb_r_min;			/* LIMIT_RG_MIN */
+     __u16 sw_rawawb_y_min;			/* LIMIT_BY_MIN */
+     __u16 sw_rawawb_b_min;			/* LIMIT_BY_MIN */
+     __u16 sw_rawawb_coeff_y_g;		/* RGB2Y_0 */
+     __u16 sw_rawawb_coeff_y_r;		/* RGB2Y_0 */
+     __u16 sw_rawawb_coeff_y_b;		/* RGB2Y_1 */
+     __u16 sw_rawawb_coeff_u_g;		/* RGB2U_0 */
+     __u16 sw_rawawb_coeff_u_r;		/* RGB2U_0 */
+     __u16 sw_rawawb_coeff_u_b;		/* RGB2U_1 */
+     __u16 sw_rawawb_coeff_v_g;		/* RGB2V_0 */
+     __u16 sw_rawawb_coeff_v_r;		/* RGB2V_0 */
+     __u16 sw_rawawb_coeff_v_b;		/* RGB2V_1 */
+     __u16 sw_rawawb_vertex0_v_0;		/* UV_DETC_VERTEX0_0 */
+     __u16 sw_rawawb_vertex0_u_0;		/* UV_DETC_VERTEX0_0 */
+     __u16 sw_rawawb_vertex1_v_0;		/* UV_DETC_VERTEX1_0 */
+     __u16 sw_rawawb_vertex1_u_0;		/* UV_DETC_VERTEX1_0 */
+     __u16 sw_rawawb_vertex2_v_0;		/* UV_DETC_VERTEX2_0 */
+     __u16 sw_rawawb_vertex2_u_0;		/* UV_DETC_VERTEX2_0 */
+     __u16 sw_rawawb_vertex3_v_0;		/* UV_DETC_VERTEX3_0 */
+     __u16 sw_rawawb_vertex3_u_0;		/* UV_DETC_VERTEX3_0 */
+     __u32 sw_rawawb_islope01_0;		/* UV_DETC_ISLOPE01_0 */
+     __u32 sw_rawawb_islope12_0;		/* UV_DETC_ISLOPE12_0 */
+     __u32 sw_rawawb_islope23_0;		/* UV_DETC_ISLOPE23_0 */
+     __u32 sw_rawawb_islope30_0;		/* UV_DETC_ISLOPE30_0 */
+     __u16 sw_rawawb_vertex0_v_1;		/* UV_DETC_VERTEX0_1 */
+     __u16 sw_rawawb_vertex0_u_1;		/* UV_DETC_VERTEX0_1 */
+     __u16 sw_rawawb_vertex1_v_1;		/* UV_DETC_VERTEX1_1 */
+     __u16 sw_rawawb_vertex1_u_1;		/* UV_DETC_VERTEX1_1 */
+     __u16 sw_rawawb_vertex2_v_1;		/* UV_DETC_VERTEX2_1 */
+     __u16 sw_rawawb_vertex2_u_1;		/* UV_DETC_VERTEX2_1 */
+     __u16 sw_rawawb_vertex3_v_1;		/* UV_DETC_VERTEX3_1 */
+     __u16 sw_rawawb_vertex3_u_1;		/* UV_DETC_VERTEX3_1 */
+     __u32 sw_rawawb_islope01_1;		/* UV_DETC_ISLOPE01_1 */
+     __u32 sw_rawawb_islope12_1;		/* UV_DETC_ISLOPE12_1 */
+     __u32 sw_rawawb_islope23_1;		/* UV_DETC_ISLOPE23_1 */
+     __u32 sw_rawawb_islope30_1;		/* UV_DETC_ISLOPE30_1 */
+     __u16 sw_rawawb_vertex0_v_2;		/* UV_DETC_VERTEX0_2 */
+     __u16 sw_rawawb_vertex0_u_2;		/* UV_DETC_VERTEX0_2 */
+     __u16 sw_rawawb_vertex1_v_2;		/* UV_DETC_VERTEX1_2 */
+     __u16 sw_rawawb_vertex1_u_2;		/* UV_DETC_VERTEX1_2 */
+     __u16 sw_rawawb_vertex2_v_2;		/* UV_DETC_VERTEX2_2 */
+     __u16 sw_rawawb_vertex2_u_2;		/* UV_DETC_VERTEX2_2 */
+     __u16 sw_rawawb_vertex3_v_2;		/* UV_DETC_VERTEX3_2 */
+     __u16 sw_rawawb_vertex3_u_2;		/* UV_DETC_VERTEX3_2 */
+     __u32 sw_rawawb_islope01_2;		/* UV_DETC_ISLOPE01_2 */
+     __u32 sw_rawawb_islope12_2;		/* UV_DETC_ISLOPE12_2 */
+     __u32 sw_rawawb_islope23_2;		/* UV_DETC_ISLOPE23_2 */
+     __u32 sw_rawawb_islope30_2;		/* UV_DETC_ISLOPE30_2 */
+     __u16 sw_rawawb_vertex0_v_3;		/* UV_DETC_VERTEX0_3 */
+     __u16 sw_rawawb_vertex0_u_3;		/* UV_DETC_VERTEX0_3 */
+     __u16 sw_rawawb_vertex1_v_3;		/* UV_DETC_VERTEX1_3 */
+     __u16 sw_rawawb_vertex1_u_3;		/* UV_DETC_VERTEX1_3 */
+     __u16 sw_rawawb_vertex2_v_3;		/* UV_DETC_VERTEX2_3 */
+     __u16 sw_rawawb_vertex2_u_3;		/* UV_DETC_VERTEX2_3 */
+     __u16 sw_rawawb_vertex3_v_3;		/* UV_DETC_VERTEX3_3 */
+     __u16 sw_rawawb_vertex3_u_3;		/* UV_DETC_VERTEX3_3 */
+     __u32 sw_rawawb_islope01_3;		/* UV_DETC_ISLOPE01_3 */
+     __u32 sw_rawawb_islope12_3;		/* UV_DETC_ISLOPE12_3 */
+     __u32 sw_rawawb_islope23_3;		/* UV_DETC_ISLOPE23_3 */
+     __u32 sw_rawawb_islope30_3;		/* UV_DETC_ISLOPE30_3 */
+     __u16 sw_rawawb_vertex0_v_4;		/* UV_DETC_VERTEX0_4 */
+     __u16 sw_rawawb_vertex0_u_4;		/* UV_DETC_VERTEX0_4 */
+     __u16 sw_rawawb_vertex1_v_4;		/* UV_DETC_VERTEX1_4 */
+     __u16 sw_rawawb_vertex1_u_4;		/* UV_DETC_VERTEX1_4 */
+     __u16 sw_rawawb_vertex2_v_4;		/* UV_DETC_VERTEX2_4 */
+     __u16 sw_rawawb_vertex2_u_4;		/* UV_DETC_VERTEX2_4 */
+     __u16 sw_rawawb_vertex3_v_4;		/* UV_DETC_VERTEX3_4 */
+     __u16 sw_rawawb_vertex3_u_4;		/* UV_DETC_VERTEX3_4 */
+     __u32 sw_rawawb_islope01_4;		/* UV_DETC_ISLOPE01_4 */
+     __u32 sw_rawawb_islope12_4;		/* UV_DETC_ISLOPE12_4 */
+     __u32 sw_rawawb_islope23_4;		/* UV_DETC_ISLOPE23_4 */
+     __u32 sw_rawawb_islope30_4;		/* UV_DETC_ISLOPE30_4 */
+     __u16 sw_rawawb_vertex0_v_5;		/* UV_DETC_VERTEX0_5 */
+     __u16 sw_rawawb_vertex0_u_5;		/* UV_DETC_VERTEX0_5 */
+     __u16 sw_rawawb_vertex1_v_5;		/* UV_DETC_VERTEX1_5 */
+     __u16 sw_rawawb_vertex1_u_5;		/* UV_DETC_VERTEX1_5 */
+     __u16 sw_rawawb_vertex2_v_5;		/* UV_DETC_VERTEX2_5 */
+     __u16 sw_rawawb_vertex2_u_5;		/* UV_DETC_VERTEX2_5 */
+     __u16 sw_rawawb_vertex3_v_5;		/* UV_DETC_VERTEX3_5 */
+     __u16 sw_rawawb_vertex3_u_5;		/* UV_DETC_VERTEX3_5 */
+     __u32 sw_rawawb_islope01_5;		/* UV_DETC_ISLOPE01_5 */
+     __u32 sw_rawawb_islope12_5;		/* UV_DETC_ISLOPE10_5 */
+     __u32 sw_rawawb_islope23_5;		/* UV_DETC_ISLOPE23_5 */
+     __u32 sw_rawawb_islope30_5;		/* UV_DETC_ISLOPE30_5 */
+     __u16 sw_rawawb_vertex0_v_6;		/* UV_DETC_VERTEX0_6 */
+     __u16 sw_rawawb_vertex0_u_6;		/* UV_DETC_VERTEX0_6 */
+     __u16 sw_rawawb_vertex1_v_6;		/* UV_DETC_VERTEX1_6 */
+     __u16 sw_rawawb_vertex1_u_6;		/* UV_DETC_VERTEX1_6 */
+     __u16 sw_rawawb_vertex2_v_6;		/* UV_DETC_VERTEX2_6 */
+     __u16 sw_rawawb_vertex2_u_6;		/* UV_DETC_VERTEX2_6 */
+     __u16 sw_rawawb_vertex3_v_6;		/* UV_DETC_VERTEX3_6 */
+     __u16 sw_rawawb_vertex3_u_6;		/* UV_DETC_VERTEX3_6 */
+     __u32 sw_rawawb_islope01_6;		/* UV_DETC_ISLOPE01_6 */
+     __u32 sw_rawawb_islope12_6;		/* UV_DETC_ISLOPE10_6 */
+     __u32 sw_rawawb_islope23_6;		/* UV_DETC_ISLOPE23_6 */
+     __u32 sw_rawawb_islope30_6;		/* UV_DETC_ISLOPE30_6 */
+     __u32 sw_rawawb_b_uv_0;			/* YUV_DETC_B_UV_0 */
+     __u32 sw_rawawb_slope_vtcuv_0;		/* YUV_DETC_SLOPE_VTCUV_0 */
+     __u32 sw_rawawb_inv_dslope_0;		/* YUV_DETC_INV_DSLOPE_0 */
+     __u32 sw_rawawb_slope_ydis_0;		/* YUV_DETC_SLOPE_YDIS_0 */
+     __u32 sw_rawawb_b_ydis_0;		/* YUV_DETC_B_YDIS_0 */
+     __u32 sw_rawawb_b_uv_1;			/* YUV_DETC_B_UV_1 */
+     __u32 sw_rawawb_slope_vtcuv_1;		/* YUV_DETC_SLOPE_VTCUV_1 */
+     __u32 sw_rawawb_inv_dslope_1;		/* YUV_DETC_INV_DSLOPE_1 */
+     __u32 sw_rawawb_slope_ydis_1;		/* YUV_DETC_SLOPE_YDIS_1 */
+     __u32 sw_rawawb_b_ydis_1;		/* YUV_DETC_B_YDIS_1 */
+     __u32 sw_rawawb_b_uv_2;			/* YUV_DETC_B_UV_2 */
+     __u32 sw_rawawb_slope_vtcuv_2;		/* YUV_DETC_SLOPE_VTCUV_2 */
+     __u32 sw_rawawb_inv_dslope_2;		/* YUV_DETC_INV_DSLOPE_2 */
+     __u32 sw_rawawb_slope_ydis_2;		/* YUV_DETC_SLOPE_YDIS_2 */
+     __u32 sw_rawawb_b_ydis_2;		/* YUV_DETC_B_YDIS_2 */
+     __u32 sw_rawawb_b_uv_3;			/* YUV_DETC_B_UV_3 */
+     __u32 sw_rawawb_slope_vtcuv_3;		/* YUV_DETC_SLOPE_VTCUV_3 */
+     __u32 sw_rawawb_inv_dslope_3;		/* YUV_DETC_INV_DSLOPE_3 */
+     __u32 sw_rawawb_slope_ydis_3;		/* YUV_DETC_SLOPE_YDIS_3 */
+     __u32 sw_rawawb_b_ydis_3;		/* YUV_DETC_B_YDIS_3 */
+     __u32 sw_rawawb_ref_u;			/* YUV_DETC_REF_U */
+     __u8 sw_rawawb_ref_v_3;			/* YUV_DETC_REF_V_1 */
+     __u8 sw_rawawb_ref_v_2;			/* YUV_DETC_REF_V_1 */
+     __u8 sw_rawawb_ref_v_1;			/* YUV_DETC_REF_V_1 */
+     __u8 sw_rawawb_ref_v_0;			/* YUV_DETC_REF_V_1 */
+     __u16 sw_rawawb_dis1_0;			/* YUV_DETC_DIS01_0 */
+     __u16 sw_rawawb_dis0_0;			/* YUV_DETC_DIS01_0 */
+     __u16 sw_rawawb_dis3_0;			/* YUV_DETC_DIS23_0 */
+     __u16 sw_rawawb_dis2_0;			/* YUV_DETC_DIS23_0 */
+     __u16 sw_rawawb_dis5_0;			/* YUV_DETC_DIS45_0 */
+     __u16 sw_rawawb_dis4_0;			/* YUV_DETC_DIS45_0 */
+     __u8 sw_rawawb_th3_0;			/* YUV_DETC_TH03_0 */
+     __u8 sw_rawawb_th2_0;			/* YUV_DETC_TH03_0 */
+     __u8 sw_rawawb_th1_0;			/* YUV_DETC_TH03_0 */
+     __u8 sw_rawawb_th0_0;			/* YUV_DETC_TH03_0 */
+     __u8 sw_rawawb_th5_0;			/* YUV_DETC_TH45_0 */
+     __u8 sw_rawawb_th4_0;			/* YUV_DETC_TH45_0 */
+     __u16 sw_rawawb_dis1_1;			/* YUV_DETC_DIS01_1 */
+     __u16 sw_rawawb_dis0_1;			/* YUV_DETC_DIS01_1 */
+     __u16 sw_rawawb_dis3_1;			/* YUV_DETC_DIS23_1 */
+     __u16 sw_rawawb_dis2_1;			/* YUV_DETC_DIS23_1 */
+     __u16 sw_rawawb_dis5_1;			/* YUV_DETC_DIS45_1 */
+     __u16 sw_rawawb_dis4_1;			/* YUV_DETC_DIS45_1 */
+     __u8 sw_rawawb_th3_1;			/* YUV_DETC_TH03_1 */
+     __u8 sw_rawawb_th2_1;			/* YUV_DETC_TH03_1 */
+     __u8 sw_rawawb_th1_1;			/* YUV_DETC_TH03_1 */
+     __u8 sw_rawawb_th0_1;			/* YUV_DETC_TH03_1 */
+     __u8 sw_rawawb_th5_1;			/* YUV_DETC_TH45_1 */
+     __u8 sw_rawawb_th4_1;			/* YUV_DETC_TH45_1 */
+     __u16 sw_rawawb_dis1_2;			/* YUV_DETC_DIS01_2 */
+     __u16 sw_rawawb_dis0_2;			/* YUV_DETC_DIS01_2 */
+     __u16 sw_rawawb_dis3_2;			/* YUV_DETC_DIS23_2 */
+     __u16 sw_rawawb_dis2_2;			/* YUV_DETC_DIS23_2 */
+     __u16 sw_rawawb_dis5_2;			/* YUV_DETC_DIS45_2 */
+     __u16 sw_rawawb_dis4_2;			/* YUV_DETC_DIS45_2 */
+     __u8 sw_rawawb_th3_2;			/* YUV_DETC_TH03_2 */
+     __u8 sw_rawawb_th2_2;			/* YUV_DETC_TH03_2 */
+     __u8 sw_rawawb_th1_2;			/* YUV_DETC_TH03_2 */
+     __u8 sw_rawawb_th0_2;			/* YUV_DETC_TH03_2 */
+     __u8 sw_rawawb_th5_2;			/* YUV_DETC_TH45_2 */
+     __u8 sw_rawawb_th4_2;			/* YUV_DETC_TH45_2 */
+     __u16 sw_rawawb_dis1_3;			/* YUV_DETC_DIS01_3 */
+     __u16 sw_rawawb_dis0_3;			/* YUV_DETC_DIS01_3 */
+     __u16 sw_rawawb_dis3_3;			/* YUV_DETC_DIS23_3 */
+     __u16 sw_rawawb_dis2_3;			/* YUV_DETC_DIS23_3 */
+     __u16 sw_rawawb_dis5_3;			/* YUV_DETC_DIS45_3 */
+     __u16 sw_rawawb_dis4_3;			/* YUV_DETC_DIS45_3 */
+     __u8 sw_rawawb_th3_3;			/* YUV_DETC_TH03_3 */
+     __u8 sw_rawawb_th2_3;			/* YUV_DETC_TH03_3 */
+     __u8 sw_rawawb_th1_3;			/* YUV_DETC_TH03_3 */
+     __u8 sw_rawawb_th0_3;			/* YUV_DETC_TH03_3 */
+     __u8 sw_rawawb_th5_3;			/* YUV_DETC_TH45_3 */
+     __u8 sw_rawawb_th4_3;			/* YUV_DETC_TH45_3 */
+     __u16 sw_rawawb_wt1;			/* RGB2XY_WT01 */
+     __u16 sw_rawawb_wt0;			/* RGB2XY_WT01 */
+     __u16 sw_rawawb_wt2;			/* RGB2XY_WT2 */
+     __u16 sw_rawawb_mat0_y;			/* RGB2XY_MAT0_XY */
+     __u16 sw_rawawb_mat0_x;			/* RGB2XY_MAT0_XY */
+     __u16 sw_rawawb_mat1_y;			/* RGB2XY_MAT1_XY */
+     __u16 sw_rawawb_mat1_x;			/* RGB2XY_MAT1_XY */
+     __u16 sw_rawawb_mat2_y;			/* RGB2XY_MAT2_XY */
+     __u16 sw_rawawb_mat2_x;			/* RGB2XY_MAT2_XY */
+     __u16 sw_rawawb_nor_x1_0;		/* XY_DETC_NOR_X_0 */
+     __u16 sw_rawawb_nor_x0_0;		/* XY_DETC_NOR_X_0 */
+     __u16 sw_rawawb_nor_y1_0;		/* XY_DETC_NOR_Y_0 */
+     __u16 sw_rawawb_nor_y0_0;		/* XY_DETC_NOR_Y_0 */
+     __u16 sw_rawawb_big_x1_0;		/* XY_DETC_BIG_X_0 */
+     __u16 sw_rawawb_big_x0_0;		/* XY_DETC_BIG_X_0 */
+     __u16 sw_rawawb_big_y1_0;		/* XY_DETC_BIG_Y_0 */
+     __u16 sw_rawawb_big_y0_0;		/* XY_DETC_BIG_Y_0 */
+     __u16 sw_rawawb_sma_x1_0;		/* XY_DETC_SMA_X_0 */
+     __u16 sw_rawawb_sma_x0_0;		/* XY_DETC_SMA_X_0 */
+     __u16 sw_rawawb_sma_y1_0;		/* XY_DETC_SMA_Y_0 */
+     __u16 sw_rawawb_sma_y0_0;		/* XY_DETC_SMA_Y_0 */
+     __u16 sw_rawawb_nor_x1_1;		/* XY_DETC_NOR_X_1 */
+     __u16 sw_rawawb_nor_x0_1;		/* XY_DETC_NOR_X_1 */
+     __u16 sw_rawawb_nor_y1_1;		/* XY_DETC_NOR_Y_1 */
+     __u16 sw_rawawb_nor_y0_1;		/* XY_DETC_NOR_Y_1 */
+     __u16 sw_rawawb_big_x1_1;		/* XY_DETC_BIG_X_1 */
+     __u16 sw_rawawb_big_x0_1;		/* XY_DETC_BIG_X_1 */
+     __u16 sw_rawawb_big_y1_1;		/* XY_DETC_BIG_Y_1 */
+     __u16 sw_rawawb_big_y0_1;		/* XY_DETC_BIG_Y_1 */
+     __u16 sw_rawawb_sma_x1_1;		/* XY_DETC_SMA_X_1 */
+     __u16 sw_rawawb_sma_x0_1;		/* XY_DETC_SMA_X_1 */
+     __u16 sw_rawawb_sma_y1_1;		/* XY_DETC_SMA_Y_1 */
+     __u16 sw_rawawb_sma_y0_1;		/* XY_DETC_SMA_Y_1 */
+     __u16 sw_rawawb_nor_x1_2;		/* XY_DETC_NOR_X_2 */
+     __u16 sw_rawawb_nor_x0_2;		/* XY_DETC_NOR_X_2 */
+     __u16 sw_rawawb_nor_y1_2;		/* XY_DETC_NOR_Y_2 */
+     __u16 sw_rawawb_nor_y0_2;		/* XY_DETC_NOR_Y_2 */
+     __u16 sw_rawawb_big_x1_2;		/* XY_DETC_BIG_X_2 */
+     __u16 sw_rawawb_big_x0_2;		/* XY_DETC_BIG_X_2 */
+     __u16 sw_rawawb_big_y1_2;		/* XY_DETC_BIG_Y_2 */
+     __u16 sw_rawawb_big_y0_2;		/* XY_DETC_BIG_Y_2 */
+     __u16 sw_rawawb_sma_x1_2;		/* XY_DETC_SMA_X_2 */
+     __u16 sw_rawawb_sma_x0_2;		/* XY_DETC_SMA_X_2 */
+     __u16 sw_rawawb_sma_y1_2;		/* XY_DETC_SMA_Y_2 */
+     __u16 sw_rawawb_sma_y0_2;		/* XY_DETC_SMA_Y_2 */
+     __u16 sw_rawawb_nor_x1_3;		/* XY_DETC_NOR_X_3 */
+     __u16 sw_rawawb_nor_x0_3;		/* XY_DETC_NOR_X_3 */
+     __u16 sw_rawawb_nor_y1_3;		/* XY_DETC_NOR_Y_3 */
+     __u16 sw_rawawb_nor_y0_3;		/* XY_DETC_NOR_Y_3 */
+     __u16 sw_rawawb_big_x1_3;		/* XY_DETC_BIG_X_3 */
+     __u16 sw_rawawb_big_x0_3;		/* XY_DETC_BIG_X_3 */
+     __u16 sw_rawawb_big_y1_3;		/* XY_DETC_BIG_Y_3 */
+     __u16 sw_rawawb_big_y0_3;		/* XY_DETC_BIG_Y_3 */
+     __u16 sw_rawawb_sma_x1_3;		/* XY_DETC_SMA_X_3 */
+     __u16 sw_rawawb_sma_x0_3;		/* XY_DETC_SMA_X_3 */
+     __u16 sw_rawawb_sma_y1_3;		/* XY_DETC_SMA_Y_3 */
+     __u16 sw_rawawb_sma_y0_3;		/* XY_DETC_SMA_Y_3 */
+     __u16 sw_rawawb_nor_x1_4;		/* XY_DETC_NOR_X_4 */
+     __u16 sw_rawawb_nor_x0_4;		/* XY_DETC_NOR_X_4 */
+     __u16 sw_rawawb_nor_y1_4;		/* XY_DETC_NOR_Y_4 */
+     __u16 sw_rawawb_nor_y0_4;		/* XY_DETC_NOR_Y_4 */
+     __u16 sw_rawawb_big_x1_4;		/* XY_DETC_BIG_X_4 */
+     __u16 sw_rawawb_big_x0_4;		/* XY_DETC_BIG_X_4 */
+     __u16 sw_rawawb_big_y1_4;		/* XY_DETC_BIG_Y_4 */
+     __u16 sw_rawawb_big_y0_4;		/* XY_DETC_BIG_Y_4 */
+     __u16 sw_rawawb_sma_x1_4;		/* XY_DETC_SMA_X_4 */
+     __u16 sw_rawawb_sma_x0_4;		/* XY_DETC_SMA_X_4 */
+     __u16 sw_rawawb_sma_y1_4;		/* XY_DETC_SMA_Y_4 */
+     __u16 sw_rawawb_sma_y0_4;		/* XY_DETC_SMA_Y_4 */
+     __u16 sw_rawawb_nor_x1_5;		/* XY_DETC_NOR_X_5 */
+     __u16 sw_rawawb_nor_x0_5;		/* XY_DETC_NOR_X_5 */
+     __u16 sw_rawawb_nor_y1_5;		/* XY_DETC_NOR_Y_5 */
+     __u16 sw_rawawb_nor_y0_5;		/* XY_DETC_NOR_Y_5 */
+     __u16 sw_rawawb_big_x1_5;		/* XY_DETC_BIG_X_5 */
+     __u16 sw_rawawb_big_x0_5;		/* XY_DETC_BIG_X_5 */
+     __u16 sw_rawawb_big_y1_5;		/* XY_DETC_BIG_Y_5 */
+     __u16 sw_rawawb_big_y0_5;		/* XY_DETC_BIG_Y_5 */
+     __u16 sw_rawawb_sma_x1_5;		/* XY_DETC_SMA_X_5 */
+     __u16 sw_rawawb_sma_x0_5;		/* XY_DETC_SMA_X_5 */
+     __u16 sw_rawawb_sma_y1_5;		/* XY_DETC_SMA_Y_5 */
+     __u16 sw_rawawb_sma_y0_5;		/* XY_DETC_SMA_Y_5 */
+     __u16 sw_rawawb_nor_x1_6;		/* XY_DETC_NOR_X_6 */
+     __u16 sw_rawawb_nor_x0_6;		/* XY_DETC_NOR_X_6 */
+     __u16 sw_rawawb_nor_y1_6;		/* XY_DETC_NOR_Y_6 */
+     __u16 sw_rawawb_nor_y0_6;		/* XY_DETC_NOR_Y_6 */
+     __u16 sw_rawawb_big_x1_6;		/* XY_DETC_BIG_X_6 */
+     __u16 sw_rawawb_big_x0_6;		/* XY_DETC_BIG_X_6 */
+     __u16 sw_rawawb_big_y1_6;		/* XY_DETC_BIG_Y_6 */
+     __u16 sw_rawawb_big_y0_6;		/* XY_DETC_BIG_Y_6 */
+     __u16 sw_rawawb_sma_x1_6;		/* XY_DETC_SMA_X_6 */
+     __u16 sw_rawawb_sma_x0_6;		/* XY_DETC_SMA_X_6 */
+     __u16 sw_rawawb_sma_y1_6;		/* XY_DETC_SMA_Y_6 */
+     __u16 sw_rawawb_sma_y0_6;		/* XY_DETC_SMA_Y_6 */
+     __u8 sw_rawawb_multiwindow_en;		/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region6_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region6_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region6_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region5_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region5_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region5_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region4_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region4_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region4_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region3_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region3_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region3_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region2_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region2_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region2_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region1_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region1_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region1_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region0_domain;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region0_measen;	/* MULTIWINDOW_EXC_CTRL */
+     __u8 sw_rawawb_exc_wp_region0_excen;	/* MULTIWINDOW_EXC_CTRL */
+     __u16 sw_rawawb_multiwindow0_v_offs;	/* MULTIWINDOW0_OFFS */
+     __u16 sw_rawawb_multiwindow0_h_offs;	/* MULTIWINDOW0_OFFS */
+     __u16 sw_rawawb_multiwindow0_v_size;	/* MULTIWINDOW0_SIZE */
+     __u16 sw_rawawb_multiwindow0_h_size;	/* MULTIWINDOW0_SIZE */
+     __u16 sw_rawawb_multiwindow1_v_offs;	/* MULTIWINDOW1_OFFS */
+     __u16 sw_rawawb_multiwindow1_h_offs;	/* MULTIWINDOW1_OFFS */
+     __u16 sw_rawawb_multiwindow1_v_size;	/* MULTIWINDOW1_SIZE */
+     __u16 sw_rawawb_multiwindow1_h_size;	/* MULTIWINDOW1_SIZE */
+     __u16 sw_rawawb_multiwindow2_v_offs;	/* MULTIWINDOW2_OFFS */
+     __u16 sw_rawawb_multiwindow2_h_offs;	/* MULTIWINDOW2_OFFS */
+     __u16 sw_rawawb_multiwindow2_v_size;	/* MULTIWINDOW2_SIZE */
+     __u16 sw_rawawb_multiwindow2_h_size;	/* MULTIWINDOW2_SIZE */
+     __u16 sw_rawawb_multiwindow3_v_offs;	/* MULTIWINDOW3_OFFS */
+     __u16 sw_rawawb_multiwindow3_h_offs;	/* MULTIWINDOW3_OFFS */
+     __u16 sw_rawawb_multiwindow3_v_size;	/* MULTIWINDOW3_SIZE */
+     __u16 sw_rawawb_multiwindow3_h_size;	/* MULTIWINDOW3_SIZE */
+     __u16 sw_rawawb_multiwindow4_v_offs;	/* MULTIWINDOW4_OFFS */
+     __u16 sw_rawawb_multiwindow4_h_offs;	/* MULTIWINDOW4_OFFS */
+     __u16 sw_rawawb_multiwindow4_v_size;	/* MULTIWINDOW4_SIZE */
+     __u16 sw_rawawb_multiwindow4_h_size;	/* MULTIWINDOW4_SIZE */
+     __u16 sw_rawawb_multiwindow5_v_offs;	/* MULTIWINDOW5_OFFS */
+     __u16 sw_rawawb_multiwindow5_h_offs;	/* MULTIWINDOW5_OFFS */
+     __u16 sw_rawawb_multiwindow5_v_size;	/* MULTIWINDOW5_SIZE */
+     __u16 sw_rawawb_multiwindow5_h_size;	/* MULTIWINDOW5_SIZE */
+     __u16 sw_rawawb_multiwindow6_v_offs;	/* MULTIWINDOW6_OFFS */
+     __u16 sw_rawawb_multiwindow6_h_offs;	/* MULTIWINDOW6_OFFS */
+     __u16 sw_rawawb_multiwindow6_v_size;	/* MULTIWINDOW6_SIZE */
+     __u16 sw_rawawb_multiwindow6_h_size;	/* MULTIWINDOW6_SIZE */
+     __u16 sw_rawawb_multiwindow7_v_offs;	/* MULTIWINDOW7_OFFS */
+     __u16 sw_rawawb_multiwindow7_h_offs;	/* MULTIWINDOW7_OFFS */
+     __u16 sw_rawawb_multiwindow7_v_size;	/* MULTIWINDOW7_SIZE */
+     __u16 sw_rawawb_multiwindow7_h_size;	/* MULTIWINDOW7_SIZE */
+     __u16 sw_rawawb_exc_wp_region0_xu1;	/* EXC_WP_REGION0_XU */
+     __u16 sw_rawawb_exc_wp_region0_xu0;	/* EXC_WP_REGION0_XU */
+     __u16 sw_rawawb_exc_wp_region0_yv1;	/* EXC_WP_REGION0_YV */
+     __u16 sw_rawawb_exc_wp_region0_yv0;	/* EXC_WP_REGION0_YV */
+     __u16 sw_rawawb_exc_wp_region1_xu1;	/* EXC_WP_REGION1_XU */
+     __u16 sw_rawawb_exc_wp_region1_xu0;	/* EXC_WP_REGION1_XU */
+     __u16 sw_rawawb_exc_wp_region1_yv1;	/* EXC_WP_REGION1_YV */
+     __u16 sw_rawawb_exc_wp_region1_yv0;	/* EXC_WP_REGION1_YV */
+     __u16 sw_rawawb_exc_wp_region2_xu1;	/* EXC_WP_REGION2_XU */
+     __u16 sw_rawawb_exc_wp_region2_xu0;	/* EXC_WP_REGION2_XU */
+     __u16 sw_rawawb_exc_wp_region2_yv1;	/* EXC_WP_REGION2_YV */
+     __u16 sw_rawawb_exc_wp_region2_yv0;	/* EXC_WP_REGION2_YV */
+     __u16 sw_rawawb_exc_wp_region3_xu1;	/* EXC_WP_REGION3_XU */
+     __u16 sw_rawawb_exc_wp_region3_xu0;	/* EXC_WP_REGION3_XU */
+     __u16 sw_rawawb_exc_wp_region3_yv1;	/* EXC_WP_REGION3_YV */
+     __u16 sw_rawawb_exc_wp_region3_yv0;	/* EXC_WP_REGION3_YV */
+     __u16 sw_rawawb_exc_wp_region4_xu1;	/* EXC_WP_REGION4_XU */
+     __u16 sw_rawawb_exc_wp_region4_xu0;	/* EXC_WP_REGION4_XU */
+     __u16 sw_rawawb_exc_wp_region4_yv1;	/* EXC_WP_REGION4_YV */
+     __u16 sw_rawawb_exc_wp_region4_yv0;	/* EXC_WP_REGION4_YV */
+     __u16 sw_rawawb_exc_wp_region5_xu1;	/* EXC_WP_REGION5_XU */
+     __u16 sw_rawawb_exc_wp_region5_xu0;	/* EXC_WP_REGION5_XU */
+     __u16 sw_rawawb_exc_wp_region5_yv1;	/* EXC_WP_REGION5_YV */
+     __u16 sw_rawawb_exc_wp_region5_yv0;	/* EXC_WP_REGION5_YV */
+     __u16 sw_rawawb_exc_wp_region6_xu1;	/* EXC_WP_REGION6_XU */
+     __u16 sw_rawawb_exc_wp_region6_xu0;	/* EXC_WP_REGION6_XU */
+     __u16 sw_rawawb_exc_wp_region6_yv1;	/* EXC_WP_REGION6_YV */
+     __u16 sw_rawawb_exc_wp_region6_yv0;	/* EXC_WP_REGION6_YV */
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawaebig_meas_cfg {
+     __u8 rawae_sel;
+     __u8 wnd_num;
+     __u8 subwin_en[ISP2X_RAWAEBIG_SUBWIN_NUM];
+     struct isp2x_window win;
+     struct isp2x_window subwin[ISP2X_RAWAEBIG_SUBWIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawaelite_meas_cfg {
+     __u8 rawae_sel;
+     __u8 wnd_num;
+     struct isp2x_window win;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_yuvae_meas_cfg {
+     __u8 ysel;
+     __u8 wnd_num;
+     __u8 subwin_en[ISP2X_YUVAE_SUBWIN_NUM];
+     struct isp2x_window win;
+     struct isp2x_window subwin[ISP2X_YUVAE_SUBWIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawaf_meas_cfg {
+     __u8 rawaf_sel;
+     __u8 num_afm_win;
+     __u8 gaus_en;
+     __u8 gamma_en;
+     struct isp2x_window win[ISP2X_RAWAF_WIN_NUM];
+     __u8 line_en[ISP2X_RAWAF_LINE_NUM];
+     __u8 line_num[ISP2X_RAWAF_LINE_NUM];
+     __u8 gaus_coe_h2;
+     __u8 gaus_coe_h1;
+     __u8 gaus_coe_h0;
+     __u16 afm_thres;
+     __u8 lum_var_shift[ISP2X_RAWAF_WIN_NUM];
+     __u8 afm_var_shift[ISP2X_RAWAF_WIN_NUM];
+     __u16 gamma_y[ISP2X_RAWAF_GAMMA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siaf_win_cfg {
+     __u8 sum_shift;
+     __u8 lum_shift;
+     struct isp2x_window win;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siaf_cfg {
+     __u8 num_afm_win;
+     __u32 thres;
+     struct isp2x_siaf_win_cfg afm_win[ISP2X_AFM_MAX_WINDOWS];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawhistbig_cfg {
+     __u8 wnd_num;
+     __u8 data_sel;
+     __u8 waterline;
+     __u8 mode;
+     __u8 stepsize;
+     __u8 off;
+     __u8 bcc;
+     __u8 gcc;
+     __u8 rcc;
+     struct isp2x_window win;
+     __u8 weight[ISP2X_RAWHISTBIG_SUBWIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawhistlite_cfg {
+     __u8 data_sel;
+     __u8 waterline;
+     __u8 mode;
+     __u8 stepsize;
+     __u8 off;
+     __u8 bcc;
+     __u8 gcc;
+     __u8 rcc;
+     struct isp2x_window win;
+     __u8 weight[ISP2X_RAWHISTLITE_SUBWIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_sihst_win_cfg {
+     __u8 data_sel;
+     __u8 waterline;
+     __u8 auto_stop;
+     __u8 mode;
+     __u8 stepsize;
+     struct isp2x_window win;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_sihst_cfg {
+     __u8 wnd_num;
+     struct isp2x_sihst_win_cfg win_cfg[ISP2X_SIHIST_WIN_NUM];
+     __u8 hist_weight[ISP2X_HIST_WEIGHT_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_isp_other_cfg {
+     struct isp2x_bls_cfg bls_cfg;
+     struct isp2x_dpcc_cfg dpcc_cfg;
+     struct isp2x_hdrmge_cfg hdrmge_cfg;
+     struct isp2x_rawnr_cfg rawnr_cfg;
+     struct isp2x_lsc_cfg lsc_cfg;
+     struct isp2x_awb_gain_cfg awb_gain_cfg;
+     /* struct isp2x_goc_cfg goc_cfg; */
+     struct isp2x_gic_cfg gic_cfg;
+     struct isp2x_debayer_cfg debayer_cfg;
+     struct isp2x_ccm_cfg ccm_cfg;
+     struct isp2x_gammaout_cfg gammaout_cfg;
+     struct isp2x_wdr_cfg wdr_cfg;
+     struct isp2x_cproc_cfg cproc_cfg;
+     struct isp2x_ie_cfg ie_cfg;
+     struct isp2x_esiesharp_cfg esiesharp_cfg;
+     struct isp2x_superimp_cfg superimp_cfg;
+     struct isp2x_sdg_cfg sdg_cfg;
+     struct isp2x_bdm_config bdm_cfg;
+     struct isp2x_hdrtmo_cfg hdrtmo_cfg;
+     struct isp2x_dhaz_cfg dhaz_cfg;
+     struct isp2x_gain_cfg gain_cfg;
+     struct isp2x_3dlut_cfg isp3dlut_cfg;
+     struct isp2x_ldch_cfg ldch_cfg;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_isp_meas_cfg {
+     struct isp2x_siawb_meas_cfg siawb;
+     struct isp2x_rawawb_meas_cfg rawawb;
+     struct isp2x_rawaelite_meas_cfg rawae0;
+     struct isp2x_rawaebig_meas_cfg rawae1;
+     struct isp2x_rawaebig_meas_cfg rawae2;
+     struct isp2x_rawaebig_meas_cfg rawae3;
+     struct isp2x_yuvae_meas_cfg yuvae;
+     struct isp2x_rawaf_meas_cfg rawaf;
+     struct isp2x_siaf_cfg siaf;
+     struct isp2x_rawhistlite_cfg rawhist0;
+     struct isp2x_rawhistbig_cfg rawhist1;
+     struct isp2x_rawhistbig_cfg rawhist2;
+     struct isp2x_rawhistbig_cfg rawhist3;
+     struct isp2x_sihst_cfg sihst;
+ } __attribute__ ((packed));
+ 
+ struct sensor_exposure_s {
+     __u32 fine_integration_time;
+     __u32 coarse_integration_time;
+     __u32 analog_gain_code_global;
+     __u32 digital_gain_global;
+     __u32 isp_digital_gain;
+ } __attribute__ ((packed));
+ 
+ struct sensor_exposure_cfg {
+     struct sensor_exposure_s linear_exp;
+     struct sensor_exposure_s hdr_exp[3];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_isp_params_cfg {
+     __u64 module_en_update;
+     __u64 module_ens;
+     __u64 module_cfg_update;
+ 
+     __u32 frame_id;
+     struct isp2x_isp_meas_cfg meas;
+     struct isp2x_isp_other_cfg others;
+     struct sensor_exposure_cfg exposure;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siawb_meas {
+     __u32 cnt;
+     __u8 mean_y_or_g;
+     __u8 mean_cb_or_b;
+     __u8 mean_cr_or_r;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siawb_stat {
+     struct isp2x_siawb_meas awb_mean[ISP2X_AWB_MAX_GRID];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawawb_ramdata {
+     __u32 wp;
+     __u32 r;
+     __u32 g;
+     __u32 b;
+ };
+ 
+ struct isp2x_rawawb_meas_stat {
+     __u32 ro_rawawb_sum_r_nor[ISP2X_RAWAWB_SUM_NUM];	/* SUM_R_NOR_0 */
+     __u32 ro_rawawb_sum_g_nor[ISP2X_RAWAWB_SUM_NUM];	/* SUM_G_NOR_0 */
+     __u32 ro_rawawb_sum_b_nor[ISP2X_RAWAWB_SUM_NUM];	/* SUM_B_NOR_0 */
+     __u32 ro_rawawb_wp_num_nor[ISP2X_RAWAWB_SUM_NUM];	/* WP_NUM_NOR_0 */
+     __u32 ro_rawawb_sum_r_big[ISP2X_RAWAWB_SUM_NUM];	/* SUM_R_BIG_0 */
+     __u32 ro_rawawb_sum_g_big[ISP2X_RAWAWB_SUM_NUM];	/* SUM_G_BIG_0 */
+     __u32 ro_rawawb_sum_b_big[ISP2X_RAWAWB_SUM_NUM];	/* SUM_B_BIG_0 */
+     __u32 ro_rawawb_wp_num_big[ISP2X_RAWAWB_SUM_NUM];	/* WP_NUM_BIG_0 */
+     __u32 ro_rawawb_sum_r_sma[ISP2X_RAWAWB_SUM_NUM];	/* SUM_R_SMA_0 */
+     __u32 ro_rawawb_sum_g_sma[ISP2X_RAWAWB_SUM_NUM];	/* SUM_G_SMA_0 */
+     __u32 ro_rawawb_sum_b_sma[ISP2X_RAWAWB_SUM_NUM];	/* SUM_B_SMA_0 */
+     __u32 ro_rawawb_wp_num_sma[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_sum_r_nor_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_R_NOR_MULTIWINDOW_0 */
+     __u32 ro_sum_g_nor_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_G_NOR_MULTIWINDOW_0 */
+     __u32 ro_sum_b_nor_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_B_NOR_MULTIWINDOW_0 */
+     __u32 ro_wp_nm_nor_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* WP_NM_NOR_MULTIWINDOW_0 */
+     __u32 ro_sum_r_big_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_R_BIG_MULTIWINDOW_0 */
+     __u32 ro_sum_g_big_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_G_BIG_MULTIWINDOW_0 */
+     __u32 ro_sum_b_big_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_B_BIG_MULTIWINDOW_0 */
+     __u32 ro_wp_nm_big_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* WP_NM_BIG_MULTIWINDOW_0 */
+     __u32 ro_sum_r_sma_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_R_SMA_MULTIWINDOW_0 */
+     __u32 ro_sum_g_sma_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_G_SMA_MULTIWINDOW_0 */
+     __u32 ro_sum_b_sma_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* SUM_B_SMA_MULTIWINDOW_0 */
+     __u32 ro_wp_nm_sma_multiwindow[ISP2X_RAWAWB_MULWD_NUM];	/* WP_NM_SMA_MULTIWINDOW_0 */
+     __u32 ro_sum_r_exc[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_sum_g_exc[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_sum_b_exc[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_wp_nm_exc[ISP2X_RAWAWB_SUM_NUM];
+     struct isp2x_rawawb_ramdata ramdata[ISP2X_RAWAWB_RAMDATA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawae_meas_data {
+     __u16 channelr_xy;
+     __u16 channelb_xy;
+     __u16 channelg_xy;
+ };
+ 
+ struct isp2x_rawaebig_stat {
+     __u32 sumr[ISP2X_RAWAEBIG_SUBWIN_NUM];
+     __u32 sumg[ISP2X_RAWAEBIG_SUBWIN_NUM];
+     __u32 sumb[ISP2X_RAWAEBIG_SUBWIN_NUM];
+     struct isp2x_rawae_meas_data data[ISP2X_RAWAEBIG_MEAN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawaelite_stat {
+     struct isp2x_rawae_meas_data data[ISP2X_RAWAELITE_MEAN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_yuvae_stat {
+     __u32 ro_yuvae_sumy[ISP2X_YUVAE_SUBWIN_NUM];
+     __u8 mean[ISP2X_YUVAE_MEAN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawaf_stat {
+     __u32 int_state;
+     __u32 afm_sum[ISP2X_RAWAF_WIN_NUM];
+     __u32 afm_lum[ISP2X_RAWAF_WIN_NUM];
+     __u32 ramdata[ISP2X_RAWAF_SUMDATA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siaf_meas_val {
+     __u32 sum;
+     __u32 lum;
+ } __attribute__ ((packed));
+ 
+ struct isp2x_siaf_stat {
+     struct isp2x_siaf_meas_val win[ISP2X_AFM_MAX_WINDOWS];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawhistbig_stat {
+     __u32 hist_bin[ISP2X_HIST_BIN_N_MAX];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_rawhistlite_stat {
+     __u32 hist_bin[ISP2X_HIST_BIN_N_MAX];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_sihst_win_stat {
+     __u32 hist_bins[ISP2X_SIHIST_BIN_N_MAX];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_sihst_stat {
+     struct isp2x_sihst_win_stat win_stat[ISP2X_SIHIST_WIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp2x_stat {
+     struct isp2x_siawb_stat siawb;
+     struct isp2x_rawawb_meas_stat rawawb;
+     struct isp2x_rawaelite_stat rawae0;
+     struct isp2x_rawaebig_stat rawae1;
+     struct isp2x_rawaebig_stat rawae2;
+     struct isp2x_rawaebig_stat rawae3;
+     struct isp2x_yuvae_stat yuvae;
+     struct isp2x_rawaf_stat rawaf;
+     struct isp2x_siaf_stat siaf;
+     struct isp2x_rawhistlite_stat rawhist0;
+     struct isp2x_rawhistbig_stat rawhist1;
+     struct isp2x_rawhistbig_stat rawhist2;
+     struct isp2x_rawhistbig_stat rawhist3;
+     struct isp2x_sihst_stat sihst;
+ 
+     struct isp2x_bls_stat bls;
+     struct isp2x_hdrtmo_stat hdrtmo;
+     struct isp2x_dhaz_stat dhaz;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp_isp2x_stat_buffer - ESWIN ISP2 Statistics Meta Data
+  *
+  * @meas_type: measurement types (CIFISP_STAT_ definitions)
+  * @frame_id: frame ID for sync
+  * @params: statistics data
+  */
+ struct esisp_isp2x_stat_buffer {
+     unsigned int meas_type;
+     unsigned int frame_id;
+     struct isp2x_stat params;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp_mipi_luma - statistics mipi y statistic
+  *
+  * @exp_mean: Mean luminance value of block xx
+  *
+  * Image is divided into 5x5 blocks.
+  */
+ struct esisp_mipi_luma {
+     unsigned int exp_mean[ISP2X_MIPI_LUMA_MEAN_MAX];
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp_isp2x_luma_buffer - ESWIN ISP1 Statistics Mipi Luma
+  *
+  * @meas_type: measurement types (CIFISP_STAT_ definitions)
+  * @frame_id: frame ID for sync
+  * @params: statistics data
+  */
+ struct esisp_isp2x_luma_buffer {
+     unsigned int meas_type;
+     unsigned int frame_id;
+     struct esisp_mipi_luma luma[ISP2X_MIPI_RAW_MAX];
+ } __attribute__ ((packed));
+ 
+ enum {
+     ESISP_RTT_MODE_NORMAL = 0,
+     ESISP_RTT_MODE_MULTI_FRAME,
+     ESISP_RTT_MODE_ONE_FRAME,
+ };
+ 
+ /**
+  * struct esisp_thunderboot_resmem_head
+  */
+ struct esisp_thunderboot_resmem_head {
+     __u16 enable;
+     __u16 complete;
+     __u16 frm_total;
+     __u16 hdr_mode;
+     __u16 rtt_mode;
+     __u16 width;
+     __u16 height;
+     __u16 camera_num;
+     __u16 camera_index;
+     __u16 md_flag;
+ 
+     __u32 exp_time[3];
+     __u32 exp_gain[3];
+     __u32 exp_time_reg[3];
+     __u32 exp_gain_reg[3];
+     __u32 exp_isp_dgain[3];
+     __u32 nr_buf_size;
+     __u32 share_mem_size;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp_thunderboot_resmem - shared buffer for thunderboot with risc-v side
+  */
+ struct esisp_thunderboot_resmem {
+     __u32 resmem_padr;
+     __u32 resmem_size;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp_thunderboot_shmem
+  */
+ struct esisp_thunderboot_shmem {
+     __u32 shm_start;
+     __u32 shm_size;
+     __s32 shm_fd;
+ } __attribute__ ((packed));
+ 
+ #endif /* _UAPI_ES_ISP2_CONFIG_H */
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/es-isp21-config.h b/drivers/media/platform/eswin/dvp2axi/es-isp21-config.h
new file mode 100644
index 000000000000..214400872216
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/es-isp21-config.h
@@ -0,0 +1,830 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI isp config driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _UAPI_ES_ISP21_CONFIG_H
+ #define _UAPI_ES_ISP21_CONFIG_H
+ 
+ #include <linux/const.h>
+ #include <linux/types.h>
+ #include <linux/v4l2-controls.h>
+ #include "es-isp2-config.h"
+ 
+ #define ISP2X_MODULE_BAYNR			BIT_ULL(36)
+ #define ISP2X_MODULE_BAY3D			BIT_ULL(37)
+ #define ISP2X_MODULE_YNR			BIT_ULL(38)
+ #define ISP2X_MODULE_CNR			BIT_ULL(39)
+ #define ISP2X_MODULE_SHARP			BIT_ULL(40)
+ #define ISP2X_MODULE_DRC			BIT_ULL(41)
+ #define ISP2X_MODULE_CSM			BIT_ULL(43)
+ #define ISP2X_MODULE_CGC			BIT_ULL(44)
+ 
+ #define ISP21_DHAZ_ENH_CURVE_NUM		17
+ #define ISP21_DHAZ_HIST_IIR_NUM			64
+ #define ISP21_RAWAWB_HSTBIN_NUM			8
+ #define ISP21_RAWAWB_WEIGHT_NUM			225
+ #define ISP21_DRC_Y_NUM				17
+ #define ISP21_YNR_XY_NUM			17
+ #define ISP21_BAYNR_XY_NUM			16
+ #define ISP21_BAY3D_XY_NUM			16
+ #define ISP21_SHARP_X_NUM			7
+ #define ISP21_SHARP_Y_NUM			8
+ #define ISP21_CSM_COEFF_NUM			9
+ 
+ struct isp21_cgc_cfg {
+     __u8 yuv_limit;
+     __u8 ratio_en;
+ } __attribute__ ((packed));
+ 
+ struct isp21_csm_cfg {
+     __u8 csm_full_range;
+     __u16 csm_y_offset;
+     __u16 csm_c_offset;
+ 
+     __u32 csm_coeff[ISP21_CSM_COEFF_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_bls_cfg {
+     __u8 enable_auto;
+     __u8 en_windows;
+     __u8 bls1_en;
+     struct isp2x_window bls_window1;
+     struct isp2x_window bls_window2;
+     __u8 bls_samples;
+     struct isp2x_bls_fixed_val fixed_val;
+     struct isp2x_bls_fixed_val bls1_val;
+ } __attribute__ ((packed));
+ 
+ struct isp21_awb_gain_cfg {
+     __u16 gain0_red;
+     __u16 gain0_green_r;
+     __u16 gain0_blue;
+     __u16 gain0_green_b;
+     __u16 gain1_red;
+     __u16 gain1_green_r;
+     __u16 gain1_blue;
+     __u16 gain1_green_b;
+     __u16 gain2_red;
+     __u16 gain2_green_r;
+     __u16 gain2_blue;
+     __u16 gain2_green_b;
+ } __attribute__ ((packed));
+ 
+ struct isp21_gic_cfg {
+     __u16 regmingradthrdark2;
+     __u16 regmingradthrdark1;
+     __u16 regminbusythre;
+ 
+     __u16 regdarkthre;
+     __u16 regmaxcorvboth;
+     __u16 regdarktthrehi;
+ 
+     __u8 regkgrad2dark;
+     __u8 regkgrad1dark;
+     __u8 regstrengthglobal_fix;
+     __u8 regdarkthrestep;
+     __u8 regkgrad2;
+     __u8 regkgrad1;
+     __u8 reggbthre;
+ 
+     __u16 regmaxcorv;
+     __u16 regmingradthr2;
+     __u16 regmingradthr1;
+ 
+     __u8 gr_ratio;
+     __u8 noise_scale;
+     __u16 noise_base;
+     __u16 diff_clip;
+ 
+     __u16 sigma_y[ISP2X_GIC_SIGMA_Y_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_baynr_cfg {
+     __u8 sw_baynr_gauss_en;
+     __u8 sw_baynr_log_bypass;
+     __u16 sw_baynr_dgain1;
+     __u16 sw_baynr_dgain0;
+     __u16 sw_baynr_dgain2;
+     __u16 sw_baynr_pix_diff;
+     __u16 sw_baynr_diff_thld;
+     __u16 sw_baynr_softthld;
+     __u16 sw_bltflt_streng;
+     __u16 sw_baynr_reg_w1;
+     __u16 sw_sigma_x[ISP21_BAYNR_XY_NUM];
+     __u16 sw_sigma_y[ISP21_BAYNR_XY_NUM];
+     __u16 weit_d2;
+     __u16 weit_d1;
+     __u16 weit_d0;
+ } __attribute__ ((packed));
+ 
+ struct isp21_bay3d_cfg {
+     __u8 sw_bay3d_exp_sel;
+     __u8 sw_bay3d_bypass_en;
+     __u8 sw_bay3d_pk_en;
+     __u16 sw_bay3d_softwgt;
+     __u16 sw_bay3d_sigratio;
+     __u32 sw_bay3d_glbpk2;
+     __u16 sw_bay3d_exp_str;
+     __u16 sw_bay3d_str;
+     __u16 sw_bay3d_wgtlmt_h;
+     __u16 sw_bay3d_wgtlmt_l;
+     __u16 sw_bay3d_sig_x[ISP21_BAY3D_XY_NUM];
+     __u16 sw_bay3d_sig_y[ISP21_BAY3D_XY_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_ynr_cfg {
+     __u8 sw_ynr_thumb_mix_cur_en;
+     __u8 sw_ynr_global_gain_alpha;
+     __u8 sw_ynr_global_gain;
+     __u8 sw_ynr_flt1x1_bypass_sel;
+     __u8 sw_ynr_sft5x5_bypass;
+     __u8 sw_ynr_flt1x1_bypass;
+     __u8 sw_ynr_lgft3x3_bypass;
+     __u8 sw_ynr_lbft5x5_bypass;
+     __u8 sw_ynr_bft3x3_bypass;
+ 
+     __u16 sw_ynr_rnr_max_r;
+ 
+     __u16 sw_ynr_low_bf_inv1;
+     __u16 sw_ynr_low_bf_inv0;
+ 
+     __u16 sw_ynr_low_peak_supress;
+     __u16 sw_ynr_low_thred_adj;
+ 
+     __u16 sw_ynr_low_dist_adj;
+     __u16 sw_ynr_low_edge_adj_thresh;
+ 
+     __u16 sw_ynr_low_bi_weight;
+     __u16 sw_ynr_low_weight;
+     __u16 sw_ynr_low_center_weight;
+     __u16 sw_ynr_hi_min_adj;
+     __u16 sw_ynr_high_thred_adj;
+     __u8 sw_ynr_high_retain_weight;
+     __u8 sw_ynr_hi_edge_thed;
+     __u8 sw_ynr_base_filter_weight2;
+     __u8 sw_ynr_base_filter_weight1;
+     __u8 sw_ynr_base_filter_weight0;
+     __u16 sw_ynr_low_gauss1_coeff2;
+     __u16 sw_ynr_low_gauss1_coeff1;
+     __u16 sw_ynr_low_gauss1_coeff0;
+     __u16 sw_ynr_low_gauss2_coeff2;
+     __u16 sw_ynr_low_gauss2_coeff1;
+     __u16 sw_ynr_low_gauss2_coeff0;
+     __u8 sw_ynr_direction_weight3;
+     __u8 sw_ynr_direction_weight2;
+     __u8 sw_ynr_direction_weight1;
+     __u8 sw_ynr_direction_weight0;
+     __u8 sw_ynr_direction_weight7;
+     __u8 sw_ynr_direction_weight6;
+     __u8 sw_ynr_direction_weight5;
+     __u8 sw_ynr_direction_weight4;
+     __u16 sw_ynr_luma_points_x[ISP21_YNR_XY_NUM];
+     __u16 sw_ynr_lsgm_y[ISP21_YNR_XY_NUM];
+     __u16 sw_ynr_hsgm_y[ISP21_YNR_XY_NUM];
+     __u8 sw_ynr_rnr_strength3[ISP21_YNR_XY_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_cnr_cfg {
+     __u8 sw_cnr_thumb_mix_cur_en;
+     __u8 sw_cnr_lq_bila_bypass;
+     __u8 sw_cnr_hq_bila_bypass;
+     __u8 sw_cnr_exgain_bypass;
+     __u8 sw_cnr_exgain_mux;
+     __u8 sw_cnr_gain_iso;
+     __u8 sw_cnr_gain_offset;
+     __u8 sw_cnr_gain_1sigma;
+     __u8 sw_cnr_gain_uvgain1;
+     __u8 sw_cnr_gain_uvgain0;
+     __u8 sw_cnr_lmed3_alpha;
+     __u8 sw_cnr_lbf5_gain_y;
+     __u8 sw_cnr_lbf5_gain_c;
+     __u8 sw_cnr_lbf5_weit_d3;
+     __u8 sw_cnr_lbf5_weit_d2;
+     __u8 sw_cnr_lbf5_weit_d1;
+     __u8 sw_cnr_lbf5_weit_d0;
+     __u8 sw_cnr_lbf5_weit_d4;
+     __u8 sw_cnr_hmed3_alpha;
+     __u16 sw_cnr_hbf5_weit_src;
+     __u16 sw_cnr_hbf5_min_wgt;
+     __u16 sw_cnr_hbf5_sigma;
+     __u16 sw_cnr_lbf5_weit_src;
+     __u16 sw_cnr_lbf3_sigma;
+ } __attribute__ ((packed));
+ 
+ struct isp21_sharp_cfg {
+     __u8 sw_sharp_bypass;
+     __u8 sw_sharp_sharp_ratio;
+     __u8 sw_sharp_bf_ratio;
+     __u8 sw_sharp_gaus_ratio;
+     __u8 sw_sharp_pbf_ratio;
+     __u8 sw_sharp_luma_dx[ISP21_SHARP_X_NUM];
+     __u16 sw_sharp_pbf_sigma_inv[ISP21_SHARP_Y_NUM];
+     __u16 sw_sharp_bf_sigma_inv[ISP21_SHARP_Y_NUM];
+     __u8 sw_sharp_bf_sigma_shift;
+     __u8 sw_sharp_pbf_sigma_shift;
+     __u16 sw_sharp_ehf_th[ISP21_SHARP_Y_NUM];
+     __u16 sw_sharp_clip_hf[ISP21_SHARP_Y_NUM];
+     __u8 sw_sharp_pbf_coef_2;
+     __u8 sw_sharp_pbf_coef_1;
+     __u8 sw_sharp_pbf_coef_0;
+     __u8 sw_sharp_bf_coef_2;
+     __u8 sw_sharp_bf_coef_1;
+     __u8 sw_sharp_bf_coef_0;
+     __u8 sw_sharp_gaus_coef_2;
+     __u8 sw_sharp_gaus_coef_1;
+     __u8 sw_sharp_gaus_coef_0;
+ } __attribute__ ((packed));
+ 
+ struct isp21_ccm_cfg {
+     __u8 highy_adjust_dis;
+     __u8 bound_bit;
+ 
+     __s16 coeff0_r;
+     __s16 coeff1_r;
+     __s16 coeff2_r;
+     __s16 offset_r;
+ 
+     __s16 coeff0_g;
+     __s16 coeff1_g;
+     __s16 coeff2_g;
+     __s16 offset_g;
+ 
+     __s16 coeff0_b;
+     __s16 coeff1_b;
+     __s16 coeff2_b;
+     __s16 offset_b;
+ 
+     __u16 coeff0_y;
+     __u16 coeff1_y;
+     __u16 coeff2_y;
+ 
+     __u16 alp_y[ISP21_DHAZ_ENH_CURVE_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_dhaz_cfg {
+     __u8 enhance_en;
+     __u8 air_lc_en;
+     __u8 hpara_en;
+     __u8 hist_en;
+     __u8 dc_en;
+ 
+     __u8 yblk_th;
+     __u8 yhist_th;
+     __u8 dc_max_th;
+     __u8 dc_min_th;
+ 
+     __u16 wt_max;
+     __u8 bright_max;
+     __u8 bright_min;
+ 
+     __u8 tmax_base;
+     __u8 dark_th;
+     __u8 air_max;
+     __u8 air_min;
+ 
+     __u16 tmax_max;
+     __u16 tmax_off;
+ 
+     __u8 hist_k;
+     __u8 hist_th_off;
+     __u16 hist_min;
+ 
+     __u16 hist_gratio;
+     __u16 hist_scale;
+ 
+     __u16 enhance_value;
+     __u16 enhance_chroma;
+ 
+     __u16 iir_wt_sigma;
+     __u16 iir_sigma;
+     __u16 stab_fnum;
+ 
+     __u16 iir_tmax_sigma;
+     __u16 iir_air_sigma;
+     __u8 iir_pre_wet;
+ 
+     __u16 cfg_wt;
+     __u16 cfg_air;
+     __u16 cfg_alpha;
+ 
+     __u16 cfg_gratio;
+     __u16 cfg_tmax;
+ 
+     __u16 range_sima;
+     __u8 space_sigma_pre;
+     __u8 space_sigma_cur;
+ 
+     __u16 dc_weitcur;
+     __u16 bf_weight;
+ 
+     __u16 enh_curve[ISP21_DHAZ_ENH_CURVE_NUM];
+ 
+     __u8 gaus_h2;
+     __u8 gaus_h1;
+     __u8 gaus_h0;
+ } __attribute__ ((packed));
+ 
+ struct isp21_dhaz_stat {
+     __u16 dhaz_adp_air_base;
+     __u16 dhaz_adp_wt;
+ 
+     __u16 dhaz_adp_gratio;
+     __u16 dhaz_adp_tmax;
+ 
+     __u16 h_rgb_iir[ISP21_DHAZ_HIST_IIR_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_drc_cfg {
+     __u8 sw_drc_offset_pow2;
+     __u16 sw_drc_compres_scl;
+     __u16 sw_drc_position;
+     __u16 sw_drc_delta_scalein;
+     __u16 sw_drc_hpdetail_ratio;
+     __u16 sw_drc_lpdetail_ratio;
+     __u8 sw_drc_weicur_pix;
+     __u8 sw_drc_weipre_frame;
+     __u16 sw_drc_force_sgm_inv0;
+     __u8 sw_drc_motion_scl;
+     __u8 sw_drc_edge_scl;
+     __u16 sw_drc_space_sgm_inv1;
+     __u16 sw_drc_space_sgm_inv0;
+     __u16 sw_drc_range_sgm_inv1;
+     __u16 sw_drc_range_sgm_inv0;
+     __u8 sw_drc_weig_maxl;
+     __u8 sw_drc_weig_bilat;
+     __u16 sw_drc_gain_y[ISP21_DRC_Y_NUM];
+     __u16 sw_drc_compres_y[ISP21_DRC_Y_NUM];
+     __u16 sw_drc_scale_y[ISP21_DRC_Y_NUM];
+     __u16 sw_drc_iir_weight;
+     __u16 sw_drc_min_ogain;
+ } __attribute__ ((packed));
+ 
+ struct isp21_rawawb_meas_cfg {
+     __u8 rawawb_sel;
+     __u8 sw_rawawb_xy_en0;
+     __u8 sw_rawawb_uv_en0;
+     __u8 sw_rawawb_xy_en1;
+     __u8 sw_rawawb_uv_en1;
+     __u8 sw_rawawb_3dyuv_en0;
+     __u8 sw_rawawb_3dyuv_en1;
+     __u8 sw_rawawb_wp_blk_wei_en0;
+     __u8 sw_rawawb_wp_blk_wei_en1;
+     __u8 sw_rawawb_wp_luma_wei_en0;
+     __u8 sw_rawawb_wp_luma_wei_en1;
+     __u8 sw_rawlsc_bypass_en;
+     __u8 sw_rawawb_blk_measure_enable;
+     __u8 sw_rawawb_blk_measure_mode;
+     __u8 sw_rawawb_blk_measure_xytype;
+     __u8 sw_rawawb_blk_measure_illu_idx;
+     __u8 sw_rawawb_wp_hist_xytype;
+     __u8 sw_rawawb_light_num;
+     __u8 sw_rawawb_wind_size;
+     __u8 sw_rawawb_r_max;
+     __u8 sw_rawawb_g_max;
+     __u8 sw_rawawb_b_max;
+     __u8 sw_rawawb_y_max;
+     __u8 sw_rawawb_r_min;
+     __u8 sw_rawawb_g_min;
+     __u8 sw_rawawb_b_min;
+     __u8 sw_rawawb_y_min;
+     __u8 sw_rawawb_3dyuv_ls_idx0;
+     __u8 sw_rawawb_3dyuv_ls_idx1;
+     __u8 sw_rawawb_3dyuv_ls_idx2;
+     __u8 sw_rawawb_3dyuv_ls_idx3;
+     __u8 sw_rawawb_exc_wp_region0_excen0;
+     __u8 sw_rawawb_exc_wp_region0_excen1;
+     __u8 sw_rawawb_exc_wp_region0_domain;
+     __u8 sw_rawawb_exc_wp_region1_excen0;
+     __u8 sw_rawawb_exc_wp_region1_excen1;
+     __u8 sw_rawawb_exc_wp_region1_domain;
+     __u8 sw_rawawb_exc_wp_region2_excen0;
+     __u8 sw_rawawb_exc_wp_region2_excen1;
+     __u8 sw_rawawb_exc_wp_region2_domain;
+     __u8 sw_rawawb_exc_wp_region3_excen0;
+     __u8 sw_rawawb_exc_wp_region3_excen1;
+     __u8 sw_rawawb_exc_wp_region3_domain;
+     __u8 sw_rawawb_exc_wp_region4_excen0;
+     __u8 sw_rawawb_exc_wp_region4_excen1;
+     __u8 sw_rawawb_exc_wp_region4_domain;
+     __u8 sw_rawawb_exc_wp_region5_excen0;
+     __u8 sw_rawawb_exc_wp_region5_excen1;
+     __u8 sw_rawawb_exc_wp_region5_domain;
+     __u8 sw_rawawb_exc_wp_region6_excen0;
+     __u8 sw_rawawb_exc_wp_region6_excen1;
+     __u8 sw_rawawb_exc_wp_region6_domain;
+     __u8 sw_rawawb_wp_luma_weicurve_y0;
+     __u8 sw_rawawb_wp_luma_weicurve_y1;
+     __u8 sw_rawawb_wp_luma_weicurve_y2;
+     __u8 sw_rawawb_wp_luma_weicurve_y3;
+     __u8 sw_rawawb_wp_luma_weicurve_y4;
+     __u8 sw_rawawb_wp_luma_weicurve_y5;
+     __u8 sw_rawawb_wp_luma_weicurve_y6;
+     __u8 sw_rawawb_wp_luma_weicurve_y7;
+     __u8 sw_rawawb_wp_luma_weicurve_y8;
+     __u8 sw_rawawb_wp_luma_weicurve_w0;
+     __u8 sw_rawawb_wp_luma_weicurve_w1;
+     __u8 sw_rawawb_wp_luma_weicurve_w2;
+     __u8 sw_rawawb_wp_luma_weicurve_w3;
+     __u8 sw_rawawb_wp_luma_weicurve_w4;
+     __u8 sw_rawawb_wp_luma_weicurve_w5;
+     __u8 sw_rawawb_wp_luma_weicurve_w6;
+     __u8 sw_rawawb_wp_luma_weicurve_w7;
+     __u8 sw_rawawb_wp_luma_weicurve_w8;
+     __u8 sw_rawawb_rotu0_ls0;
+     __u8 sw_rawawb_rotu1_ls0;
+     __u8 sw_rawawb_rotu2_ls0;
+     __u8 sw_rawawb_rotu3_ls0;
+     __u8 sw_rawawb_rotu4_ls0;
+     __u8 sw_rawawb_rotu5_ls0;
+     __u8 sw_rawawb_dis_x1x2_ls0;
+     __u8 sw_rawawb_rotu0_ls1;
+     __u8 sw_rawawb_rotu1_ls1;
+     __u8 sw_rawawb_rotu2_ls1;
+     __u8 sw_rawawb_rotu3_ls1;
+     __u8 sw_rawawb_rotu4_ls1;
+     __u8 sw_rawawb_rotu5_ls1;
+     __u8 sw_rawawb_dis_x1x2_ls1;
+     __u8 sw_rawawb_rotu0_ls2;
+     __u8 sw_rawawb_rotu1_ls2;
+     __u8 sw_rawawb_rotu2_ls2;
+     __u8 sw_rawawb_rotu3_ls2;
+     __u8 sw_rawawb_rotu4_ls2;
+     __u8 sw_rawawb_rotu5_ls2;
+     __u8 sw_rawawb_dis_x1x2_ls2;
+     __u8 sw_rawawb_rotu0_ls3;
+     __u8 sw_rawawb_rotu1_ls3;
+     __u8 sw_rawawb_rotu2_ls3;
+     __u8 sw_rawawb_rotu3_ls3;
+     __u8 sw_rawawb_rotu4_ls3;
+     __u8 sw_rawawb_rotu5_ls3;
+     __u8 sw_rawawb_dis_x1x2_ls3;
+     __u8 sw_rawawb_blk_rtdw_measure_en;
+     __u8 sw_rawawb_blk_with_luma_wei_en;
+     __u8 sw_rawawb_wp_blk_wei_w[ISP21_RAWAWB_WEIGHT_NUM];
+ 
+     __u16 sw_rawawb_h_offs;
+     __u16 sw_rawawb_v_offs;
+     __u16 sw_rawawb_h_size;
+     __u16 sw_rawawb_v_size;
+     __u16 sw_rawawb_vertex0_u_0;
+     __u16 sw_rawawb_vertex0_v_0;
+     __u16 sw_rawawb_vertex1_u_0;
+     __u16 sw_rawawb_vertex1_v_0;
+     __u16 sw_rawawb_vertex2_u_0;
+     __u16 sw_rawawb_vertex2_v_0;
+     __u16 sw_rawawb_vertex3_u_0;
+     __u16 sw_rawawb_vertex3_v_0;
+     __u16 sw_rawawb_vertex0_u_1;
+     __u16 sw_rawawb_vertex0_v_1;
+     __u16 sw_rawawb_vertex1_u_1;
+     __u16 sw_rawawb_vertex1_v_1;
+     __u16 sw_rawawb_vertex2_u_1;
+     __u16 sw_rawawb_vertex2_v_1;
+     __u16 sw_rawawb_vertex3_u_1;
+     __u16 sw_rawawb_vertex3_v_1;
+     __u16 sw_rawawb_vertex0_u_2;
+     __u16 sw_rawawb_vertex0_v_2;
+     __u16 sw_rawawb_vertex1_u_2;
+     __u16 sw_rawawb_vertex1_v_2;
+     __u16 sw_rawawb_vertex2_u_2;
+     __u16 sw_rawawb_vertex2_v_2;
+     __u16 sw_rawawb_vertex3_u_2;
+     __u16 sw_rawawb_vertex3_v_2;
+     __u16 sw_rawawb_vertex0_u_3;
+     __u16 sw_rawawb_vertex0_v_3;
+     __u16 sw_rawawb_vertex1_u_3;
+     __u16 sw_rawawb_vertex1_v_3;
+     __u16 sw_rawawb_vertex2_u_3;
+     __u16 sw_rawawb_vertex2_v_3;
+     __u16 sw_rawawb_vertex3_u_3;
+     __u16 sw_rawawb_vertex3_v_3;
+     __u16 sw_rawawb_vertex0_u_4;
+     __u16 sw_rawawb_vertex0_v_4;
+     __u16 sw_rawawb_vertex1_u_4;
+     __u16 sw_rawawb_vertex1_v_4;
+     __u16 sw_rawawb_vertex2_u_4;
+     __u16 sw_rawawb_vertex2_v_4;
+     __u16 sw_rawawb_vertex3_u_4;
+     __u16 sw_rawawb_vertex3_v_4;
+     __u16 sw_rawawb_vertex0_u_5;
+     __u16 sw_rawawb_vertex0_v_5;
+     __u16 sw_rawawb_vertex1_u_5;
+     __u16 sw_rawawb_vertex1_v_5;
+     __u16 sw_rawawb_vertex2_u_5;
+     __u16 sw_rawawb_vertex2_v_5;
+     __u16 sw_rawawb_vertex3_u_5;
+     __u16 sw_rawawb_vertex3_v_5;
+     __u16 sw_rawawb_vertex0_u_6;
+     __u16 sw_rawawb_vertex0_v_6;
+     __u16 sw_rawawb_vertex1_u_6;
+     __u16 sw_rawawb_vertex1_v_6;
+     __u16 sw_rawawb_vertex2_u_6;
+     __u16 sw_rawawb_vertex2_v_6;
+     __u16 sw_rawawb_vertex3_u_6;
+     __u16 sw_rawawb_vertex3_v_6;
+ 
+     __u16 sw_rawawb_wt0;
+     __u16 sw_rawawb_wt1;
+     __u16 sw_rawawb_wt2;
+     __u16 sw_rawawb_mat0_x;
+     __u16 sw_rawawb_mat1_x;
+     __u16 sw_rawawb_mat2_x;
+     __u16 sw_rawawb_mat0_y;
+     __u16 sw_rawawb_mat1_y;
+     __u16 sw_rawawb_mat2_y;
+     __u16 sw_rawawb_nor_x0_0;
+     __u16 sw_rawawb_nor_x1_0;
+     __u16 sw_rawawb_nor_y0_0;
+     __u16 sw_rawawb_nor_y1_0;
+     __u16 sw_rawawb_big_x0_0;
+     __u16 sw_rawawb_big_x1_0;
+     __u16 sw_rawawb_big_y0_0;
+     __u16 sw_rawawb_big_y1_0;
+     __u16 sw_rawawb_nor_x0_1;
+     __u16 sw_rawawb_nor_x1_1;
+     __u16 sw_rawawb_nor_y0_1;
+     __u16 sw_rawawb_nor_y1_1;
+     __u16 sw_rawawb_big_x0_1;
+     __u16 sw_rawawb_big_x1_1;
+     __u16 sw_rawawb_big_y0_1;
+     __u16 sw_rawawb_big_y1_1;
+     __u16 sw_rawawb_nor_x0_2;
+     __u16 sw_rawawb_nor_x1_2;
+     __u16 sw_rawawb_nor_y0_2;
+     __u16 sw_rawawb_nor_y1_2;
+     __u16 sw_rawawb_big_x0_2;
+     __u16 sw_rawawb_big_x1_2;
+     __u16 sw_rawawb_big_y0_2;
+     __u16 sw_rawawb_big_y1_2;
+     __u16 sw_rawawb_nor_x0_3;
+     __u16 sw_rawawb_nor_x1_3;
+     __u16 sw_rawawb_nor_y0_3;
+     __u16 sw_rawawb_nor_y1_3;
+     __u16 sw_rawawb_big_x0_3;
+     __u16 sw_rawawb_big_x1_3;
+     __u16 sw_rawawb_big_y0_3;
+     __u16 sw_rawawb_big_y1_3;
+     __u16 sw_rawawb_nor_x0_4;
+     __u16 sw_rawawb_nor_x1_4;
+     __u16 sw_rawawb_nor_y0_4;
+     __u16 sw_rawawb_nor_y1_4;
+     __u16 sw_rawawb_big_x0_4;
+     __u16 sw_rawawb_big_x1_4;
+     __u16 sw_rawawb_big_y0_4;
+     __u16 sw_rawawb_big_y1_4;
+     __u16 sw_rawawb_nor_x0_5;
+     __u16 sw_rawawb_nor_x1_5;
+     __u16 sw_rawawb_nor_y0_5;
+     __u16 sw_rawawb_nor_y1_5;
+     __u16 sw_rawawb_big_x0_5;
+     __u16 sw_rawawb_big_x1_5;
+     __u16 sw_rawawb_big_y0_5;
+     __u16 sw_rawawb_big_y1_5;
+     __u16 sw_rawawb_nor_x0_6;
+     __u16 sw_rawawb_nor_x1_6;
+     __u16 sw_rawawb_nor_y0_6;
+     __u16 sw_rawawb_nor_y1_6;
+     __u16 sw_rawawb_big_x0_6;
+     __u16 sw_rawawb_big_x1_6;
+     __u16 sw_rawawb_big_y0_6;
+     __u16 sw_rawawb_big_y1_6;
+     __u16 sw_rawawb_pre_wbgain_inv_r;
+     __u16 sw_rawawb_pre_wbgain_inv_g;
+     __u16 sw_rawawb_pre_wbgain_inv_b;
+     __u16 sw_rawawb_exc_wp_region0_xu0;
+     __u16 sw_rawawb_exc_wp_region0_xu1;
+     __u16 sw_rawawb_exc_wp_region0_yv0;
+     __u16 sw_rawawb_exc_wp_region0_yv1;
+     __u16 sw_rawawb_exc_wp_region1_xu0;
+     __u16 sw_rawawb_exc_wp_region1_xu1;
+     __u16 sw_rawawb_exc_wp_region1_yv0;
+     __u16 sw_rawawb_exc_wp_region1_yv1;
+     __u16 sw_rawawb_exc_wp_region2_xu0;
+     __u16 sw_rawawb_exc_wp_region2_xu1;
+     __u16 sw_rawawb_exc_wp_region2_yv0;
+     __u16 sw_rawawb_exc_wp_region2_yv1;
+     __u16 sw_rawawb_exc_wp_region3_xu0;
+     __u16 sw_rawawb_exc_wp_region3_xu1;
+     __u16 sw_rawawb_exc_wp_region3_yv0;
+     __u16 sw_rawawb_exc_wp_region3_yv1;
+     __u16 sw_rawawb_exc_wp_region4_xu0;
+     __u16 sw_rawawb_exc_wp_region4_xu1;
+     __u16 sw_rawawb_exc_wp_region4_yv0;
+     __u16 sw_rawawb_exc_wp_region4_yv1;
+     __u16 sw_rawawb_exc_wp_region5_xu0;
+     __u16 sw_rawawb_exc_wp_region5_xu1;
+     __u16 sw_rawawb_exc_wp_region5_yv0;
+     __u16 sw_rawawb_exc_wp_region5_yv1;
+     __u16 sw_rawawb_exc_wp_region6_xu0;
+     __u16 sw_rawawb_exc_wp_region6_xu1;
+     __u16 sw_rawawb_exc_wp_region6_yv0;
+     __u16 sw_rawawb_exc_wp_region6_yv1;
+     __u16 sw_rawawb_rgb2ryuvmat0_u;
+     __u16 sw_rawawb_rgb2ryuvmat1_u;
+     __u16 sw_rawawb_rgb2ryuvmat2_u;
+     __u16 sw_rawawb_rgb2ryuvofs_u;
+     __u16 sw_rawawb_rgb2ryuvmat0_v;
+     __u16 sw_rawawb_rgb2ryuvmat1_v;
+     __u16 sw_rawawb_rgb2ryuvmat2_v;
+     __u16 sw_rawawb_rgb2ryuvofs_v;
+     __u16 sw_rawawb_rgb2ryuvmat0_y;
+     __u16 sw_rawawb_rgb2ryuvmat1_y;
+     __u16 sw_rawawb_rgb2ryuvmat2_y;
+     __u16 sw_rawawb_rgb2ryuvofs_y;
+     __u16 sw_rawawb_th0_ls0;
+     __u16 sw_rawawb_th1_ls0;
+     __u16 sw_rawawb_th2_ls0;
+     __u16 sw_rawawb_th3_ls0;
+     __u16 sw_rawawb_th4_ls0;
+     __u16 sw_rawawb_th5_ls0;
+     __u16 sw_rawawb_coor_x1_ls0_u;
+     __u16 sw_rawawb_coor_x1_ls0_v;
+     __u16 sw_rawawb_coor_x1_ls0_y;
+     __u16 sw_rawawb_vec_x21_ls0_u;
+     __u16 sw_rawawb_vec_x21_ls0_v;
+     __u16 sw_rawawb_vec_x21_ls0_y;
+     __u16 sw_rawawb_th0_ls1;
+     __u16 sw_rawawb_th1_ls1;
+     __u16 sw_rawawb_th2_ls1;
+     __u16 sw_rawawb_th3_ls1;
+     __u16 sw_rawawb_th4_ls1;
+     __u16 sw_rawawb_th5_ls1;
+     __u16 sw_rawawb_coor_x1_ls1_u;
+     __u16 sw_rawawb_coor_x1_ls1_v;
+     __u16 sw_rawawb_coor_x1_ls1_y;
+     __u16 sw_rawawb_vec_x21_ls1_u;
+     __u16 sw_rawawb_vec_x21_ls1_v;
+     __u16 sw_rawawb_vec_x21_ls1_y;
+     __u16 sw_rawawb_th0_ls2;
+     __u16 sw_rawawb_th1_ls2;
+     __u16 sw_rawawb_th2_ls2;
+     __u16 sw_rawawb_th3_ls2;
+     __u16 sw_rawawb_th4_ls2;
+     __u16 sw_rawawb_th5_ls2;
+     __u16 sw_rawawb_coor_x1_ls2_u;
+     __u16 sw_rawawb_coor_x1_ls2_v;
+     __u16 sw_rawawb_coor_x1_ls2_y;
+     __u16 sw_rawawb_vec_x21_ls2_u;
+     __u16 sw_rawawb_vec_x21_ls2_v;
+     __u16 sw_rawawb_vec_x21_ls2_y;
+     __u16 sw_rawawb_th0_ls3;
+     __u16 sw_rawawb_th1_ls3;
+     __u16 sw_rawawb_th2_ls3;
+     __u16 sw_rawawb_th3_ls3;
+     __u16 sw_rawawb_th4_ls3;
+     __u16 sw_rawawb_th5_ls3;
+     __u16 sw_rawawb_coor_x1_ls3_u;
+     __u16 sw_rawawb_coor_x1_ls3_v;
+     __u16 sw_rawawb_coor_x1_ls3_y;
+     __u16 sw_rawawb_vec_x21_ls3_u;
+     __u16 sw_rawawb_vec_x21_ls3_v;
+     __u16 sw_rawawb_vec_x21_ls3_y;
+ 
+     __u32 sw_rawawb_islope01_0;
+     __u32 sw_rawawb_islope12_0;
+     __u32 sw_rawawb_islope23_0;
+     __u32 sw_rawawb_islope30_0;
+     __u32 sw_rawawb_islope01_1;
+     __u32 sw_rawawb_islope12_1;
+     __u32 sw_rawawb_islope23_1;
+     __u32 sw_rawawb_islope30_1;
+     __u32 sw_rawawb_islope01_2;
+     __u32 sw_rawawb_islope12_2;
+     __u32 sw_rawawb_islope23_2;
+     __u32 sw_rawawb_islope30_2;
+     __u32 sw_rawawb_islope01_3;
+     __u32 sw_rawawb_islope12_3;
+     __u32 sw_rawawb_islope23_3;
+     __u32 sw_rawawb_islope30_3;
+     __u32 sw_rawawb_islope01_4;
+     __u32 sw_rawawb_islope12_4;
+     __u32 sw_rawawb_islope23_4;
+     __u32 sw_rawawb_islope30_4;
+     __u32 sw_rawawb_islope01_5;
+     __u32 sw_rawawb_islope12_5;
+     __u32 sw_rawawb_islope23_5;
+     __u32 sw_rawawb_islope30_5;
+     __u32 sw_rawawb_islope01_6;
+     __u32 sw_rawawb_islope12_6;
+     __u32 sw_rawawb_islope23_6;
+     __u32 sw_rawawb_islope30_6;
+ } __attribute__ ((packed));
+ 
+ struct isp21_isp_other_cfg {
+     struct isp21_bls_cfg bls_cfg;
+     struct isp2x_dpcc_cfg dpcc_cfg;
+     struct isp2x_lsc_cfg lsc_cfg;
+     struct isp21_awb_gain_cfg awb_gain_cfg;
+     struct isp21_gic_cfg gic_cfg;
+     struct isp2x_debayer_cfg debayer_cfg;
+     struct isp21_ccm_cfg ccm_cfg;
+     struct isp2x_gammaout_cfg gammaout_cfg;
+     struct isp2x_cproc_cfg cproc_cfg;
+     struct isp2x_ie_cfg ie_cfg;
+     struct isp2x_sdg_cfg sdg_cfg;
+     struct isp21_drc_cfg drc_cfg;
+     struct isp2x_hdrmge_cfg hdrmge_cfg;
+     struct isp21_dhaz_cfg dhaz_cfg;
+     struct isp2x_3dlut_cfg isp3dlut_cfg;
+     struct isp2x_ldch_cfg ldch_cfg;
+     struct isp21_baynr_cfg baynr_cfg;
+     struct isp21_bay3d_cfg bay3d_cfg;
+     struct isp21_ynr_cfg ynr_cfg;
+     struct isp21_cnr_cfg cnr_cfg;
+     struct isp21_sharp_cfg sharp_cfg;
+     struct isp21_csm_cfg csm_cfg;
+     struct isp21_cgc_cfg cgc_cfg;
+ } __attribute__ ((packed));
+ 
+ struct isp21_isp_meas_cfg {
+     struct isp2x_siawb_meas_cfg siawb;
+     struct isp21_rawawb_meas_cfg rawawb;
+     struct isp2x_rawaelite_meas_cfg rawae0;
+     struct isp2x_rawaebig_meas_cfg rawae1;
+     struct isp2x_rawaebig_meas_cfg rawae2;
+     struct isp2x_rawaebig_meas_cfg rawae3;
+     struct isp2x_yuvae_meas_cfg yuvae;
+     struct isp2x_rawaf_meas_cfg rawaf;
+     struct isp2x_siaf_cfg siaf;
+     struct isp2x_rawhistlite_cfg rawhist0;
+     struct isp2x_rawhistbig_cfg rawhist1;
+     struct isp2x_rawhistbig_cfg rawhist2;
+     struct isp2x_rawhistbig_cfg rawhist3;
+     struct isp2x_sihst_cfg sihst;
+ } __attribute__ ((packed));
+ 
+ struct isp21_isp_params_cfg {
+     __u64 module_en_update;
+     __u64 module_ens;
+     __u64 module_cfg_update;
+ 
+     __u32 frame_id;
+     struct isp21_isp_meas_cfg meas;
+     struct isp21_isp_other_cfg others;
+ } __attribute__ ((packed));
+ 
+ struct isp21_rawawb_meas_stat {
+     __u16 ro_yhist_bin[ISP21_RAWAWB_HSTBIN_NUM];
+     __u32 ro_rawawb_sum_rgain_nor[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_sum_bgain_nor[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_wp_num_nor[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_sum_rgain_big[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_sum_bgain_big[ISP2X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_wp_num_big[ISP2X_RAWAWB_SUM_NUM];
+     struct isp2x_rawawb_ramdata ramdata[ISP2X_RAWAWB_RAMDATA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp21_stat {
+     struct isp2x_siawb_stat siawb;
+     struct isp21_rawawb_meas_stat rawawb;
+     struct isp2x_rawaelite_stat rawae0;
+     struct isp2x_rawaebig_stat rawae1;
+     struct isp2x_rawaebig_stat rawae2;
+     struct isp2x_rawaebig_stat rawae3;
+     struct isp2x_yuvae_stat yuvae;
+     struct isp2x_rawaf_stat rawaf;
+     struct isp2x_siaf_stat siaf;
+     struct isp2x_rawhistlite_stat rawhist0;
+     struct isp2x_rawhistbig_stat rawhist1;
+     struct isp2x_rawhistbig_stat rawhist2;
+     struct isp2x_rawhistbig_stat rawhist3;
+     struct isp2x_sihst_stat sihst;
+ 
+     struct isp2x_bls_stat bls;
+     struct isp21_dhaz_stat dhaz;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp_isp21_stat_buffer - ESWIN ISP2 Statistics Meta Data
+  *
+  * @meas_type: measurement types (CIFISP_STAT_ definitions)
+  * @frame_id: frame ID for sync
+  * @params: statistics data
+  */
+ struct esisp_isp21_stat_buffer {
+     unsigned int meas_type;
+     unsigned int frame_id;
+     unsigned int params_id;
+     struct isp21_stat params;
+ } __attribute__ ((packed));
+ 
+ #endif /* _UAPI_ES_ISP21_CONFIG_H */
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/es-isp3-config.h b/drivers/media/platform/eswin/dvp2axi/es-isp3-config.h
new file mode 100644
index 000000000000..099ee57e0b83
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/es-isp3-config.h
@@ -0,0 +1,1094 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI isp config driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _UAPI_ES_ISP3_CONFIG_H
+ #define _UAPI_ES_ISP3_CONFIG_H
+ 
+ #include <linux/types.h>
+ #include <linux/v4l2-controls.h>
+ #include "es-isp21-config.h"
+ 
+ #define ISP3X_MODULE_DPCC		ISP2X_MODULE_DPCC
+ #define ISP3X_MODULE_BLS		ISP2X_MODULE_BLS
+ #define ISP3X_MODULE_SDG		ISP2X_MODULE_SDG
+ #define ISP3X_MODULE_LSC		ISP2X_MODULE_LSC
+ #define ISP3X_MODULE_AWB_GAIN		ISP2X_MODULE_AWB_GAIN
+ #define ISP3X_MODULE_BDM		ISP2X_MODULE_BDM
+ #define ISP3X_MODULE_CCM		ISP2X_MODULE_CCM
+ #define ISP3X_MODULE_GOC		ISP2X_MODULE_GOC
+ #define ISP3X_MODULE_CPROC		ISP2X_MODULE_CPROC
+ #define ISP3X_MODULE_IE			ISP2X_MODULE_IE
+ #define ISP3X_MODULE_RAWAF		ISP2X_MODULE_RAWAF
+ #define ISP3X_MODULE_RAWAE0		ISP2X_MODULE_RAWAE0
+ #define ISP3X_MODULE_RAWAE1		ISP2X_MODULE_RAWAE1
+ #define ISP3X_MODULE_RAWAE2		ISP2X_MODULE_RAWAE2
+ #define ISP3X_MODULE_RAWAE3		ISP2X_MODULE_RAWAE3
+ #define ISP3X_MODULE_RAWAWB		ISP2X_MODULE_RAWAWB
+ #define ISP3X_MODULE_RAWHIST0		ISP2X_MODULE_RAWHIST0
+ #define ISP3X_MODULE_RAWHIST1		ISP2X_MODULE_RAWHIST1
+ #define ISP3X_MODULE_RAWHIST2		ISP2X_MODULE_RAWHIST2
+ #define ISP3X_MODULE_RAWHIST3		ISP2X_MODULE_RAWHIST3
+ #define ISP3X_MODULE_HDRMGE		ISP2X_MODULE_HDRMGE
+ #define ISP3X_MODULE_RAWNR		ISP2X_MODULE_RAWNR
+ #define ISP3X_MODULE_GIC		ISP2X_MODULE_GIC
+ #define ISP3X_MODULE_DHAZ		ISP2X_MODULE_DHAZ
+ #define ISP3X_MODULE_3DLUT		ISP2X_MODULE_3DLUT
+ #define ISP3X_MODULE_LDCH		ISP2X_MODULE_LDCH
+ #define ISP3X_MODULE_GAIN		ISP2X_MODULE_GAIN
+ #define ISP3X_MODULE_DEBAYER		ISP2X_MODULE_DEBAYER
+ #define ISP3X_MODULE_BAYNR		ISP2X_MODULE_BAYNR
+ #define ISP3X_MODULE_BAY3D		ISP2X_MODULE_BAY3D
+ #define ISP3X_MODULE_YNR		ISP2X_MODULE_YNR
+ #define ISP3X_MODULE_CNR		ISP2X_MODULE_CNR
+ #define ISP3X_MODULE_SHARP		ISP2X_MODULE_SHARP
+ #define ISP3X_MODULE_DRC		ISP2X_MODULE_DRC
+ #define ISP3X_MODULE_CAC		BIT_ULL(42)
+ #define ISP3X_MODULE_CSM		ISP2X_MODULE_CSM
+ #define ISP3X_MODULE_CGC		ISP2X_MODULE_CGC
+ 
+ #define ISP3X_MODULE_FORCE		ISP2X_MODULE_FORCE
+ 
+ /* Measurement types */
+ #define ISP3X_STAT_RAWAWB		ISP2X_STAT_RAWAWB
+ #define ISP3X_STAT_RAWAF		ISP2X_STAT_RAWAF
+ #define ISP3X_STAT_RAWAE0		ISP2X_STAT_RAWAE0
+ #define ISP3X_STAT_RAWAE1		ISP2X_STAT_RAWAE1
+ #define ISP3X_STAT_RAWAE2		ISP2X_STAT_RAWAE2
+ #define ISP3X_STAT_RAWAE3		ISP2X_STAT_RAWAE3
+ #define ISP3X_STAT_RAWHST0		ISP2X_STAT_RAWHST0
+ #define ISP3X_STAT_RAWHST1		ISP2X_STAT_RAWHST1
+ #define ISP3X_STAT_RAWHST2		ISP2X_STAT_RAWHST2
+ #define ISP3X_STAT_RAWHST3		ISP2X_STAT_RAWHST3
+ #define ISP3X_STAT_BLS			ISP2X_STAT_BLS
+ #define ISP3X_STAT_DHAZ			ISP2X_STAT_DHAZ
+ 
+ #define ISP3X_MESH_BUF_NUM		ISP2X_MESH_BUF_NUM
+ 
+ #define ISP3X_LSC_GRAD_TBL_SIZE		16
+ #define ISP3X_LSC_SIZE_TBL_SIZE		16
+ #define ISP3X_LSC_DATA_TBL_SIZE		ISP2X_LSC_DATA_TBL_SIZE
+ 
+ #define ISP3X_DEGAMMA_CURVE_SIZE	ISP2X_DEGAMMA_CURVE_SIZE
+ 
+ #define ISP3X_GAIN_IDX_NUM		ISP2X_GAIN_IDX_NUM
+ #define ISP3X_GAIN_LUT_NUM		ISP2X_GAIN_LUT_NUM
+ 
+ #define ISP3X_RAWAWB_MULWD_NUM		4
+ #define ISP3X_RAWAWB_EXCL_STAT_NUM	4
+ #define ISP3X_RAWAWB_HSTBIN_NUM		ISP21_RAWAWB_HSTBIN_NUM
+ #define ISP3X_RAWAWB_WEIGHT_NUM		ISP21_RAWAWB_WEIGHT_NUM
+ #define ISP3X_RAWAWB_SUM_NUM		ISP2X_RAWAWB_SUM_NUM
+ #define ISP3X_RAWAWB_RAMDATA_NUM	ISP2X_RAWAWB_RAMDATA_NUM
+ 
+ #define	ISP3X_RAWAEBIG_SUBWIN_NUM	ISP2X_RAWAEBIG_SUBWIN_NUM
+ #define ISP3X_RAWAEBIG_MEAN_NUM		ISP2X_RAWAEBIG_MEAN_NUM
+ #define ISP3X_RAWAELITE_MEAN_NUM	ISP2X_RAWAELITE_MEAN_NUM
+ 
+ #define ISP3X_RAWHISTBIG_SUBWIN_NUM	ISP2X_RAWHISTBIG_SUBWIN_NUM
+ #define ISP3X_RAWHISTLITE_SUBWIN_NUM	ISP2X_RAWHISTLITE_SUBWIN_NUM
+ #define ISP3X_HIST_BIN_N_MAX		ISP2X_HIST_BIN_N_MAX
+ 
+ #define ISP3X_RAWAF_CURVE_NUM		2
+ #define ISP3X_RAWAF_HIIR_COE_NUM	6
+ #define ISP3X_RAWAF_V1IIR_COE_NUM	9
+ #define ISP3X_RAWAF_V2IIR_COE_NUM	3
+ #define ISP3X_RAWAF_VFIR_COE_NUM	3
+ #define ISP3X_RAWAF_WIN_NUM		ISP2X_RAWAF_WIN_NUM
+ #define ISP3X_RAWAF_LINE_NUM		ISP2X_RAWAF_LINE_NUM
+ #define ISP3X_RAWAF_GAMMA_NUM		ISP2X_RAWAF_GAMMA_NUM
+ #define ISP3X_RAWAF_SUMDATA_NUM		ISP2X_RAWAF_SUMDATA_NUM
+ 
+ #define ISP3X_DPCC_PDAF_POINT_NUM	ISP2X_DPCC_PDAF_POINT_NUM
+ 
+ #define ISP3X_HDRMGE_L_CURVE_NUM	ISP2X_HDRMGE_L_CURVE_NUM
+ #define ISP3X_HDRMGE_E_CURVE_NUM	ISP2X_HDRMGE_E_CURVE_NUM
+ 
+ #define ISP3X_GIC_SIGMA_Y_NUM		ISP2X_GIC_SIGMA_Y_NUM
+ 
+ #define ISP3X_CCM_CURVE_NUM		ISP2X_CCM_CURVE_NUM
+ 
+ #define ISP3X_3DLUT_DATA_NUM		ISP2X_3DLUT_DATA_NUM
+ 
+ #define ISP3X_LDCH_MESH_XY_NUM		ISP2X_LDCH_MESH_XY_NUM
+ 
+ #define ISP3X_GAMMA_OUT_MAX_SAMPLES     49
+ 
+ #define ISP3X_DHAZ_SIGMA_IDX_NUM	15
+ #define ISP3X_DHAZ_SIGMA_LUT_NUM	17
+ #define ISP3X_DHAZ_HIST_WR_NUM		64
+ #define ISP3X_DHAZ_ENH_CURVE_NUM	ISP21_DHAZ_ENH_CURVE_NUM
+ #define ISP3X_DHAZ_HIST_IIR_NUM		ISP21_DHAZ_HIST_IIR_NUM
+ 
+ #define ISP3X_DRC_Y_NUM			ISP21_DRC_Y_NUM
+ 
+ #define ISP3X_CNR_SIGMA_Y_NUM		13
+ 
+ #define ISP3X_YNR_XY_NUM		ISP21_YNR_XY_NUM
+ 
+ #define ISP3X_BAYNR_XY_NUM		ISP21_BAYNR_XY_NUM
+ 
+ #define ISP3X_BAY3D_XY_NUM		ISP21_BAY3D_XY_NUM
+ 
+ #define ISP3X_SHARP_X_NUM		ISP21_SHARP_X_NUM
+ #define ISP3X_SHARP_Y_NUM		ISP21_SHARP_Y_NUM
+ #define ISP3X_SHARP_GAUS_COEF_NUM	6
+ 
+ #define ISP3X_CAC_STRENGTH_NUM		22
+ 
+ #define ISP3X_CSM_COEFF_NUM		ISP21_CSM_COEFF_NUM
+ 
+ enum isp3x_unite_id {
+     ISP3_LEFT = 0,
+     ISP3_RIGHT,
+     ISP3_UNITE_MAX,
+ };
+ 
+ struct isp3x_gammaout_cfg {
+     __u8 equ_segm;
+     __u8 finalx4_dense_en;
+     __u16 offset;
+     __u16 gamma_y[ISP3X_GAMMA_OUT_MAX_SAMPLES];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_lsc_cfg {
+     __u8 sector_16x16;
+ 
+     __u16 r_data_tbl[ISP3X_LSC_DATA_TBL_SIZE];
+     __u16 gr_data_tbl[ISP3X_LSC_DATA_TBL_SIZE];
+     __u16 gb_data_tbl[ISP3X_LSC_DATA_TBL_SIZE];
+     __u16 b_data_tbl[ISP3X_LSC_DATA_TBL_SIZE];
+ 
+     __u16 x_grad_tbl[ISP3X_LSC_GRAD_TBL_SIZE];
+     __u16 y_grad_tbl[ISP3X_LSC_GRAD_TBL_SIZE];
+ 
+     __u16 x_size_tbl[ISP3X_LSC_SIZE_TBL_SIZE];
+     __u16 y_size_tbl[ISP3X_LSC_SIZE_TBL_SIZE];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_baynr_cfg {
+     __u8 lg2_mode;
+     __u8 gauss_en;
+     __u8 log_bypass;
+ 
+     __u16 dgain1;
+     __u16 dgain0;
+     __u16 dgain2;
+ 
+     __u16 pix_diff;
+ 
+     __u16 diff_thld;
+     __u16 softthld;
+ 
+     __u16 bltflt_streng;
+     __u16 reg_w1;
+ 
+     __u16 sigma_x[ISP3X_BAYNR_XY_NUM];
+     __u16 sigma_y[ISP3X_BAYNR_XY_NUM];
+ 
+     __u16 weit_d2;
+     __u16 weit_d1;
+     __u16 weit_d0;
+ 
+     __u16 lg2_lgoff;
+     __u16 lg2_off;
+ 
+     __u32 dat_max;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_bay3d_cfg {
+     __u8 bypass_en;
+     __u8 hibypass_en;
+     __u8 lobypass_en;
+     __u8 himed_bypass_en;
+     __u8 higaus_bypass_en;
+     __u8 hiabs_possel;
+     __u8 hichnsplit_en;
+     __u8 lomed_bypass_en;
+     __u8 logaus5_bypass_en;
+     __u8 logaus3_bypass_en;
+     __u8 glbpk_en;
+     __u8 loswitch_protect;
+ 
+     __u16 softwgt;
+     __u16 hidif_th;
+ 
+     __u32 glbpk2;
+ 
+     __u16 wgtlmt;
+     __u16 wgtratio;
+ 
+     __u16 sig0_x[ISP3X_BAY3D_XY_NUM];
+     __u16 sig0_y[ISP3X_BAY3D_XY_NUM];
+     __u16 sig1_x[ISP3X_BAY3D_XY_NUM];
+     __u16 sig1_y[ISP3X_BAY3D_XY_NUM];
+     __u16 sig2_x[ISP3X_BAY3D_XY_NUM];
+     __u16 sig2_y[ISP3X_BAY3D_XY_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_ynr_cfg {
+     __u8 rnr_en;
+     __u8 thumb_mix_cur_en;
+     __u8 global_gain_alpha;
+     __u8 flt1x1_bypass_sel;
+     __u8 sft5x5_bypass;
+     __u8 flt1x1_bypass;
+     __u8 lgft3x3_bypass;
+     __u8 lbft5x5_bypass;
+     __u8 bft3x3_bypass;
+     __u16 global_gain;
+ 
+     __u16 rnr_max_r;
+     __u16 local_gainscale;
+ 
+     __u16 rnr_center_coorh;
+     __u16 rnr_center_coorv;
+ 
+     __u16 loclagain_adj_thresh;
+     __u16 localgain_adj;
+ 
+     __u16 low_bf_inv1;
+     __u16 low_bf_inv0;
+ 
+     __u16 low_peak_supress;
+     __u16 low_thred_adj;
+ 
+     __u16 low_dist_adj;
+     __u16 low_edge_adj_thresh;
+ 
+     __u16 low_bi_weight;
+     __u16 low_weight;
+     __u16 low_center_weight;
+     __u16 hi_min_adj;
+     __u16 high_thred_adj;
+     __u8 high_retain_weight;
+     __u8 hi_edge_thed;
+     __u8 base_filter_weight2;
+     __u8 base_filter_weight1;
+     __u8 base_filter_weight0;
+     __u16 frame_full_size;
+     __u16 lbf_weight_thres;
+     __u16 low_gauss1_coeff2;
+     __u16 low_gauss1_coeff1;
+     __u16 low_gauss1_coeff0;
+     __u16 low_gauss2_coeff2;
+     __u16 low_gauss2_coeff1;
+     __u16 low_gauss2_coeff0;
+     __u8 direction_weight3;
+     __u8 direction_weight2;
+     __u8 direction_weight1;
+     __u8 direction_weight0;
+     __u8 direction_weight7;
+     __u8 direction_weight6;
+     __u8 direction_weight5;
+     __u8 direction_weight4;
+     __u16 luma_points_x[ISP3X_YNR_XY_NUM];
+     __u16 lsgm_y[ISP3X_YNR_XY_NUM];
+     __u16 hsgm_y[ISP3X_YNR_XY_NUM];
+     __u8 rnr_strength3[ISP3X_YNR_XY_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_cnr_cfg {
+     __u8 thumb_mix_cur_en;
+     __u8 lq_bila_bypass;
+     __u8 hq_bila_bypass;
+     __u8 exgain_bypass;
+ 
+     __u8 global_gain_alpha;
+     __u16 global_gain;
+ 
+     __u8 gain_iso;
+     __u8 gain_offset;
+     __u8 gain_1sigma;
+ 
+     __u8 gain_uvgain1;
+     __u8 gain_uvgain0;
+ 
+     __u8 lmed3_alpha;
+ 
+     __u8 lbf5_gain_y;
+     __u8 lbf5_gain_c;
+ 
+     __u8 lbf5_weit_d3;
+     __u8 lbf5_weit_d2;
+     __u8 lbf5_weit_d1;
+     __u8 lbf5_weit_d0;
+ 
+     __u8 lbf5_weit_d4;
+ 
+     __u8 hmed3_alpha;
+ 
+     __u16 hbf5_weit_src;
+     __u16 hbf5_min_wgt;
+     __u16 hbf5_sigma;
+ 
+     __u16 lbf5_weit_src;
+     __u16 lbf3_sigma;
+ 
+     __u8 sigma_y[ISP3X_CNR_SIGMA_Y_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_sharp_cfg {
+     __u8 bypass;
+     __u8 center_mode;
+     __u8 exgain_bypass;
+ 
+     __u8 sharp_ratio;
+     __u8 bf_ratio;
+     __u8 gaus_ratio;
+     __u8 pbf_ratio;
+ 
+     __u8 luma_dx[ISP3X_SHARP_X_NUM];
+ 
+     __u16 pbf_sigma_inv[ISP3X_SHARP_Y_NUM];
+ 
+     __u16 bf_sigma_inv[ISP3X_SHARP_Y_NUM];
+ 
+     __u8 bf_sigma_shift;
+     __u8 pbf_sigma_shift;
+ 
+     __u16 ehf_th[ISP3X_SHARP_Y_NUM];
+ 
+     __u16 clip_hf[ISP3X_SHARP_Y_NUM];
+ 
+     __u8 pbf_coef2;
+     __u8 pbf_coef1;
+     __u8 pbf_coef0;
+ 
+     __u8 bf_coef2;
+     __u8 bf_coef1;
+     __u8 bf_coef0;
+ 
+     __u8 gaus_coef[ISP3X_SHARP_GAUS_COEF_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_dhaz_cfg {
+     __u8 round_en;
+     __u8 soft_wr_en;
+     __u8 enhance_en;
+     __u8 air_lc_en;
+     __u8 hpara_en;
+     __u8 hist_en;
+     __u8 dc_en;
+ 
+     __u8 yblk_th;
+     __u8 yhist_th;
+     __u8 dc_max_th;
+     __u8 dc_min_th;
+ 
+     __u16 wt_max;
+     __u8 bright_max;
+     __u8 bright_min;
+ 
+     __u8 tmax_base;
+     __u8 dark_th;
+     __u8 air_max;
+     __u8 air_min;
+ 
+     __u16 tmax_max;
+     __u16 tmax_off;
+ 
+     __u8 hist_k;
+     __u8 hist_th_off;
+     __u16 hist_min;
+ 
+     __u16 hist_gratio;
+     __u16 hist_scale;
+ 
+     __u16 enhance_value;
+     __u16 enhance_chroma;
+ 
+     __u16 iir_wt_sigma;
+     __u16 iir_sigma;
+     __u16 stab_fnum;
+ 
+     __u16 iir_tmax_sigma;
+     __u16 iir_air_sigma;
+     __u8 iir_pre_wet;
+ 
+     __u16 cfg_wt;
+     __u16 cfg_air;
+     __u16 cfg_alpha;
+ 
+     __u16 cfg_gratio;
+     __u16 cfg_tmax;
+ 
+     __u16 range_sima;
+     __u8 space_sigma_pre;
+     __u8 space_sigma_cur;
+ 
+     __u16 dc_weitcur;
+     __u16 bf_weight;
+ 
+     __u16 enh_curve[ISP3X_DHAZ_ENH_CURVE_NUM];
+ 
+     __u8 gaus_h2;
+     __u8 gaus_h1;
+     __u8 gaus_h0;
+ 
+     __u8 sigma_idx[ISP3X_DHAZ_SIGMA_IDX_NUM];
+     __u16 sigma_lut[ISP3X_DHAZ_SIGMA_LUT_NUM];
+ 
+     __u16 adp_wt_wr;
+     __u16 adp_air_wr;
+ 
+     __u16 adp_tmax_wr;
+     __u16 adp_gratio_wr;
+ 
+     __u16 hist_wr[ISP3X_DHAZ_HIST_WR_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_dhaz_stat {
+     __u32 dhaz_pic_sumh;
+ 
+     __u16 dhaz_adp_air_base;
+     __u16 dhaz_adp_wt;
+ 
+     __u16 dhaz_adp_gratio;
+     __u16 dhaz_adp_tmax;
+ 
+     __u16 h_rgb_iir[ISP3X_DHAZ_HIST_IIR_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_drc_cfg {
+     __u8 bypass_en;
+     __u8 offset_pow2;
+     __u16 compres_scl;
+     __u16 position;
+     __u16 delta_scalein;
+     __u16 hpdetail_ratio;
+     __u16 lpdetail_ratio;
+     __u8 weicur_pix;
+     __u8 weipre_frame;
+     __u8 bilat_wt_off;
+     __u16 force_sgm_inv0;
+     __u8 motion_scl;
+     __u8 edge_scl;
+     __u16 space_sgm_inv1;
+     __u16 space_sgm_inv0;
+     __u16 range_sgm_inv1;
+     __u16 range_sgm_inv0;
+     __u8 weig_maxl;
+     __u8 weig_bilat;
+     __u8 enable_soft_thd;
+     __u16 bilat_soft_thd;
+     __u16 gain_y[ISP3X_DRC_Y_NUM];
+     __u16 compres_y[ISP3X_DRC_Y_NUM];
+     __u16 scale_y[ISP3X_DRC_Y_NUM];
+     __u16 wr_cycle;
+     __u16 iir_weight;
+     __u16 min_ogain;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_hdrmge_cfg {
+     __u8 s_base;
+     __u8 mode;
+ 
+     __u16 gain0_inv;
+     __u16 gain0;
+     __u16 gain1_inv;
+     __u16 gain1;
+     __u8 gain2;
+ 
+     __u8 lm_dif_0p15;
+     __u8 lm_dif_0p9;
+     __u8 ms_diff_0p15;
+     __u8 ms_dif_0p8;
+ 
+     __u16 ms_thd1;
+     __u16 ms_thd0;
+     __u16 ms_scl;
+     __u16 lm_thd1;
+     __u16 lm_thd0;
+     __u16 lm_scl;
+     struct isp2x_hdrmge_curve curve;
+     __u16 e_y[ISP3X_HDRMGE_E_CURVE_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_rawawb_meas_cfg {
+     __u8 rawawb_sel;
+     __u8 sw_rawawb_xy_en0;
+     __u8 sw_rawawb_uv_en0;
+     __u8 sw_rawawb_xy_en1;
+     __u8 sw_rawawb_uv_en1;
+     __u8 sw_rawawb_3dyuv_en0;
+     __u8 sw_rawawb_3dyuv_en1;
+     __u8 sw_rawawb_wp_blk_wei_en0;
+     __u8 sw_rawawb_wp_blk_wei_en1;
+     __u8 sw_rawawb_wp_luma_wei_en0;
+     __u8 sw_rawawb_wp_luma_wei_en1;
+     __u8 sw_rawlsc_bypass_en;
+     __u8 sw_rawawb_blk_measure_enable;
+     __u8 sw_rawawb_blk_measure_mode;
+     __u8 sw_rawawb_blk_measure_xytype;
+     __u8 sw_rawawb_blk_measure_illu_idx;
+     __u8 sw_rawawb_wp_hist_xytype;
+     __u8 sw_rawawb_light_num;
+     __u8 sw_rawawb_wind_size;
+     __u8 sw_rawawb_r_max;
+     __u8 sw_rawawb_g_max;
+     __u8 sw_rawawb_b_max;
+     __u8 sw_rawawb_y_max;
+     __u8 sw_rawawb_r_min;
+     __u8 sw_rawawb_g_min;
+     __u8 sw_rawawb_b_min;
+     __u8 sw_rawawb_y_min;
+     __u8 sw_rawawb_3dyuv_ls_idx0;
+     __u8 sw_rawawb_3dyuv_ls_idx1;
+     __u8 sw_rawawb_3dyuv_ls_idx2;
+     __u8 sw_rawawb_3dyuv_ls_idx3;
+     __u8 sw_rawawb_multiwindow_en;
+     __u8 sw_rawawb_exc_wp_region0_excen0;
+     __u8 sw_rawawb_exc_wp_region0_excen1;
+     __u8 sw_rawawb_exc_wp_region0_measen;
+     __u8 sw_rawawb_exc_wp_region0_domain;
+     __u8 sw_rawawb_exc_wp_region1_excen0;
+     __u8 sw_rawawb_exc_wp_region1_excen1;
+     __u8 sw_rawawb_exc_wp_region1_measen;
+     __u8 sw_rawawb_exc_wp_region1_domain;
+     __u8 sw_rawawb_exc_wp_region2_excen0;
+     __u8 sw_rawawb_exc_wp_region2_excen1;
+     __u8 sw_rawawb_exc_wp_region2_measen;
+     __u8 sw_rawawb_exc_wp_region2_domain;
+     __u8 sw_rawawb_exc_wp_region3_excen0;
+     __u8 sw_rawawb_exc_wp_region3_excen1;
+     __u8 sw_rawawb_exc_wp_region3_measen;
+     __u8 sw_rawawb_exc_wp_region3_domain;
+     __u8 sw_rawawb_exc_wp_region4_excen0;
+     __u8 sw_rawawb_exc_wp_region4_excen1;
+     __u8 sw_rawawb_exc_wp_region4_domain;
+     __u8 sw_rawawb_exc_wp_region5_excen0;
+     __u8 sw_rawawb_exc_wp_region5_excen1;
+     __u8 sw_rawawb_exc_wp_region5_domain;
+     __u8 sw_rawawb_exc_wp_region6_excen0;
+     __u8 sw_rawawb_exc_wp_region6_excen1;
+     __u8 sw_rawawb_exc_wp_region6_domain;
+     __u8 sw_rawawb_wp_luma_weicurve_y0;
+     __u8 sw_rawawb_wp_luma_weicurve_y1;
+     __u8 sw_rawawb_wp_luma_weicurve_y2;
+     __u8 sw_rawawb_wp_luma_weicurve_y3;
+     __u8 sw_rawawb_wp_luma_weicurve_y4;
+     __u8 sw_rawawb_wp_luma_weicurve_y5;
+     __u8 sw_rawawb_wp_luma_weicurve_y6;
+     __u8 sw_rawawb_wp_luma_weicurve_y7;
+     __u8 sw_rawawb_wp_luma_weicurve_y8;
+     __u8 sw_rawawb_wp_luma_weicurve_w0;
+     __u8 sw_rawawb_wp_luma_weicurve_w1;
+     __u8 sw_rawawb_wp_luma_weicurve_w2;
+     __u8 sw_rawawb_wp_luma_weicurve_w3;
+     __u8 sw_rawawb_wp_luma_weicurve_w4;
+     __u8 sw_rawawb_wp_luma_weicurve_w5;
+     __u8 sw_rawawb_wp_luma_weicurve_w6;
+     __u8 sw_rawawb_wp_luma_weicurve_w7;
+     __u8 sw_rawawb_wp_luma_weicurve_w8;
+     __u8 sw_rawawb_rotu0_ls0;
+     __u8 sw_rawawb_rotu1_ls0;
+     __u8 sw_rawawb_rotu2_ls0;
+     __u8 sw_rawawb_rotu3_ls0;
+     __u8 sw_rawawb_rotu4_ls0;
+     __u8 sw_rawawb_rotu5_ls0;
+     __u8 sw_rawawb_dis_x1x2_ls0;
+     __u8 sw_rawawb_rotu0_ls1;
+     __u8 sw_rawawb_rotu1_ls1;
+     __u8 sw_rawawb_rotu2_ls1;
+     __u8 sw_rawawb_rotu3_ls1;
+     __u8 sw_rawawb_rotu4_ls1;
+     __u8 sw_rawawb_rotu5_ls1;
+     __u8 sw_rawawb_dis_x1x2_ls1;
+     __u8 sw_rawawb_rotu0_ls2;
+     __u8 sw_rawawb_rotu1_ls2;
+     __u8 sw_rawawb_rotu2_ls2;
+     __u8 sw_rawawb_rotu3_ls2;
+     __u8 sw_rawawb_rotu4_ls2;
+     __u8 sw_rawawb_rotu5_ls2;
+     __u8 sw_rawawb_dis_x1x2_ls2;
+     __u8 sw_rawawb_rotu0_ls3;
+     __u8 sw_rawawb_rotu1_ls3;
+     __u8 sw_rawawb_rotu2_ls3;
+     __u8 sw_rawawb_rotu3_ls3;
+     __u8 sw_rawawb_rotu4_ls3;
+     __u8 sw_rawawb_rotu5_ls3;
+     __u8 sw_rawawb_dis_x1x2_ls3;
+     __u8 sw_rawawb_blk_rtdw_measure_en;
+     __u8 sw_rawawb_blk_with_luma_wei_en;
+     __u8 sw_rawawb_wp_blk_wei_w[ISP3X_RAWAWB_WEIGHT_NUM];
+ 
+     __u16 sw_rawawb_h_offs;
+     __u16 sw_rawawb_v_offs;
+     __u16 sw_rawawb_h_size;
+     __u16 sw_rawawb_v_size;
+     __u16 sw_rawawb_vertex0_u_0;
+     __u16 sw_rawawb_vertex0_v_0;
+     __u16 sw_rawawb_vertex1_u_0;
+     __u16 sw_rawawb_vertex1_v_0;
+     __u16 sw_rawawb_vertex2_u_0;
+     __u16 sw_rawawb_vertex2_v_0;
+     __u16 sw_rawawb_vertex3_u_0;
+     __u16 sw_rawawb_vertex3_v_0;
+     __u16 sw_rawawb_vertex0_u_1;
+     __u16 sw_rawawb_vertex0_v_1;
+     __u16 sw_rawawb_vertex1_u_1;
+     __u16 sw_rawawb_vertex1_v_1;
+     __u16 sw_rawawb_vertex2_u_1;
+     __u16 sw_rawawb_vertex2_v_1;
+     __u16 sw_rawawb_vertex3_u_1;
+     __u16 sw_rawawb_vertex3_v_1;
+     __u16 sw_rawawb_vertex0_u_2;
+     __u16 sw_rawawb_vertex0_v_2;
+     __u16 sw_rawawb_vertex1_u_2;
+     __u16 sw_rawawb_vertex1_v_2;
+     __u16 sw_rawawb_vertex2_u_2;
+     __u16 sw_rawawb_vertex2_v_2;
+     __u16 sw_rawawb_vertex3_u_2;
+     __u16 sw_rawawb_vertex3_v_2;
+     __u16 sw_rawawb_vertex0_u_3;
+     __u16 sw_rawawb_vertex0_v_3;
+     __u16 sw_rawawb_vertex1_u_3;
+     __u16 sw_rawawb_vertex1_v_3;
+     __u16 sw_rawawb_vertex2_u_3;
+     __u16 sw_rawawb_vertex2_v_3;
+     __u16 sw_rawawb_vertex3_u_3;
+     __u16 sw_rawawb_vertex3_v_3;
+     __u16 sw_rawawb_vertex0_u_4;
+     __u16 sw_rawawb_vertex0_v_4;
+     __u16 sw_rawawb_vertex1_u_4;
+     __u16 sw_rawawb_vertex1_v_4;
+     __u16 sw_rawawb_vertex2_u_4;
+     __u16 sw_rawawb_vertex2_v_4;
+     __u16 sw_rawawb_vertex3_u_4;
+     __u16 sw_rawawb_vertex3_v_4;
+     __u16 sw_rawawb_vertex0_u_5;
+     __u16 sw_rawawb_vertex0_v_5;
+     __u16 sw_rawawb_vertex1_u_5;
+     __u16 sw_rawawb_vertex1_v_5;
+     __u16 sw_rawawb_vertex2_u_5;
+     __u16 sw_rawawb_vertex2_v_5;
+     __u16 sw_rawawb_vertex3_u_5;
+     __u16 sw_rawawb_vertex3_v_5;
+     __u16 sw_rawawb_vertex0_u_6;
+     __u16 sw_rawawb_vertex0_v_6;
+     __u16 sw_rawawb_vertex1_u_6;
+     __u16 sw_rawawb_vertex1_v_6;
+     __u16 sw_rawawb_vertex2_u_6;
+     __u16 sw_rawawb_vertex2_v_6;
+     __u16 sw_rawawb_vertex3_u_6;
+     __u16 sw_rawawb_vertex3_v_6;
+ 
+     __u16 sw_rawawb_wt0;
+     __u16 sw_rawawb_wt1;
+     __u16 sw_rawawb_wt2;
+     __u16 sw_rawawb_mat0_x;
+     __u16 sw_rawawb_mat1_x;
+     __u16 sw_rawawb_mat2_x;
+     __u16 sw_rawawb_mat0_y;
+     __u16 sw_rawawb_mat1_y;
+     __u16 sw_rawawb_mat2_y;
+     __u16 sw_rawawb_nor_x0_0;
+     __u16 sw_rawawb_nor_x1_0;
+     __u16 sw_rawawb_nor_y0_0;
+     __u16 sw_rawawb_nor_y1_0;
+     __u16 sw_rawawb_big_x0_0;
+     __u16 sw_rawawb_big_x1_0;
+     __u16 sw_rawawb_big_y0_0;
+     __u16 sw_rawawb_big_y1_0;
+     __u16 sw_rawawb_nor_x0_1;
+     __u16 sw_rawawb_nor_x1_1;
+     __u16 sw_rawawb_nor_y0_1;
+     __u16 sw_rawawb_nor_y1_1;
+     __u16 sw_rawawb_big_x0_1;
+     __u16 sw_rawawb_big_x1_1;
+     __u16 sw_rawawb_big_y0_1;
+     __u16 sw_rawawb_big_y1_1;
+     __u16 sw_rawawb_nor_x0_2;
+     __u16 sw_rawawb_nor_x1_2;
+     __u16 sw_rawawb_nor_y0_2;
+     __u16 sw_rawawb_nor_y1_2;
+     __u16 sw_rawawb_big_x0_2;
+     __u16 sw_rawawb_big_x1_2;
+     __u16 sw_rawawb_big_y0_2;
+     __u16 sw_rawawb_big_y1_2;
+     __u16 sw_rawawb_nor_x0_3;
+     __u16 sw_rawawb_nor_x1_3;
+     __u16 sw_rawawb_nor_y0_3;
+     __u16 sw_rawawb_nor_y1_3;
+     __u16 sw_rawawb_big_x0_3;
+     __u16 sw_rawawb_big_x1_3;
+     __u16 sw_rawawb_big_y0_3;
+     __u16 sw_rawawb_big_y1_3;
+     __u16 sw_rawawb_nor_x0_4;
+     __u16 sw_rawawb_nor_x1_4;
+     __u16 sw_rawawb_nor_y0_4;
+     __u16 sw_rawawb_nor_y1_4;
+     __u16 sw_rawawb_big_x0_4;
+     __u16 sw_rawawb_big_x1_4;
+     __u16 sw_rawawb_big_y0_4;
+     __u16 sw_rawawb_big_y1_4;
+     __u16 sw_rawawb_nor_x0_5;
+     __u16 sw_rawawb_nor_x1_5;
+     __u16 sw_rawawb_nor_y0_5;
+     __u16 sw_rawawb_nor_y1_5;
+     __u16 sw_rawawb_big_x0_5;
+     __u16 sw_rawawb_big_x1_5;
+     __u16 sw_rawawb_big_y0_5;
+     __u16 sw_rawawb_big_y1_5;
+     __u16 sw_rawawb_nor_x0_6;
+     __u16 sw_rawawb_nor_x1_6;
+     __u16 sw_rawawb_nor_y0_6;
+     __u16 sw_rawawb_nor_y1_6;
+     __u16 sw_rawawb_big_x0_6;
+     __u16 sw_rawawb_big_x1_6;
+     __u16 sw_rawawb_big_y0_6;
+     __u16 sw_rawawb_big_y1_6;
+     __u16 sw_rawawb_pre_wbgain_inv_r;
+     __u16 sw_rawawb_pre_wbgain_inv_g;
+     __u16 sw_rawawb_pre_wbgain_inv_b;
+     __u16 sw_rawawb_multiwindow0_v_offs;
+     __u16 sw_rawawb_multiwindow0_h_offs;
+     __u16 sw_rawawb_multiwindow0_v_size;
+     __u16 sw_rawawb_multiwindow0_h_size;
+     __u16 sw_rawawb_multiwindow1_v_offs;
+     __u16 sw_rawawb_multiwindow1_h_offs;
+     __u16 sw_rawawb_multiwindow1_v_size;
+     __u16 sw_rawawb_multiwindow1_h_size;
+     __u16 sw_rawawb_multiwindow2_v_offs;
+     __u16 sw_rawawb_multiwindow2_h_offs;
+     __u16 sw_rawawb_multiwindow2_v_size;
+     __u16 sw_rawawb_multiwindow2_h_size;
+     __u16 sw_rawawb_multiwindow3_v_offs;
+     __u16 sw_rawawb_multiwindow3_h_offs;
+     __u16 sw_rawawb_multiwindow3_v_size;
+     __u16 sw_rawawb_multiwindow3_h_size;
+     __u16 sw_rawawb_exc_wp_region0_xu0;
+     __u16 sw_rawawb_exc_wp_region0_xu1;
+     __u16 sw_rawawb_exc_wp_region0_yv0;
+     __u16 sw_rawawb_exc_wp_region0_yv1;
+     __u16 sw_rawawb_exc_wp_region1_xu0;
+     __u16 sw_rawawb_exc_wp_region1_xu1;
+     __u16 sw_rawawb_exc_wp_region1_yv0;
+     __u16 sw_rawawb_exc_wp_region1_yv1;
+     __u16 sw_rawawb_exc_wp_region2_xu0;
+     __u16 sw_rawawb_exc_wp_region2_xu1;
+     __u16 sw_rawawb_exc_wp_region2_yv0;
+     __u16 sw_rawawb_exc_wp_region2_yv1;
+     __u16 sw_rawawb_exc_wp_region3_xu0;
+     __u16 sw_rawawb_exc_wp_region3_xu1;
+     __u16 sw_rawawb_exc_wp_region3_yv0;
+     __u16 sw_rawawb_exc_wp_region3_yv1;
+     __u16 sw_rawawb_exc_wp_region4_xu0;
+     __u16 sw_rawawb_exc_wp_region4_xu1;
+     __u16 sw_rawawb_exc_wp_region4_yv0;
+     __u16 sw_rawawb_exc_wp_region4_yv1;
+     __u16 sw_rawawb_exc_wp_region5_xu0;
+     __u16 sw_rawawb_exc_wp_region5_xu1;
+     __u16 sw_rawawb_exc_wp_region5_yv0;
+     __u16 sw_rawawb_exc_wp_region5_yv1;
+     __u16 sw_rawawb_exc_wp_region6_xu0;
+     __u16 sw_rawawb_exc_wp_region6_xu1;
+     __u16 sw_rawawb_exc_wp_region6_yv0;
+     __u16 sw_rawawb_exc_wp_region6_yv1;
+     __u16 sw_rawawb_rgb2ryuvmat0_u;
+     __u16 sw_rawawb_rgb2ryuvmat1_u;
+     __u16 sw_rawawb_rgb2ryuvmat2_u;
+     __u16 sw_rawawb_rgb2ryuvofs_u;
+     __u16 sw_rawawb_rgb2ryuvmat0_v;
+     __u16 sw_rawawb_rgb2ryuvmat1_v;
+     __u16 sw_rawawb_rgb2ryuvmat2_v;
+     __u16 sw_rawawb_rgb2ryuvofs_v;
+     __u16 sw_rawawb_rgb2ryuvmat0_y;
+     __u16 sw_rawawb_rgb2ryuvmat1_y;
+     __u16 sw_rawawb_rgb2ryuvmat2_y;
+     __u16 sw_rawawb_rgb2ryuvofs_y;
+     __u16 sw_rawawb_th0_ls0;
+     __u16 sw_rawawb_th1_ls0;
+     __u16 sw_rawawb_th2_ls0;
+     __u16 sw_rawawb_th3_ls0;
+     __u16 sw_rawawb_th4_ls0;
+     __u16 sw_rawawb_th5_ls0;
+     __u16 sw_rawawb_coor_x1_ls0_u;
+     __u16 sw_rawawb_coor_x1_ls0_v;
+     __u16 sw_rawawb_coor_x1_ls0_y;
+     __u16 sw_rawawb_vec_x21_ls0_u;
+     __u16 sw_rawawb_vec_x21_ls0_v;
+     __u16 sw_rawawb_vec_x21_ls0_y;
+     __u16 sw_rawawb_th0_ls1;
+     __u16 sw_rawawb_th1_ls1;
+     __u16 sw_rawawb_th2_ls1;
+     __u16 sw_rawawb_th3_ls1;
+     __u16 sw_rawawb_th4_ls1;
+     __u16 sw_rawawb_th5_ls1;
+     __u16 sw_rawawb_coor_x1_ls1_u;
+     __u16 sw_rawawb_coor_x1_ls1_v;
+     __u16 sw_rawawb_coor_x1_ls1_y;
+     __u16 sw_rawawb_vec_x21_ls1_u;
+     __u16 sw_rawawb_vec_x21_ls1_v;
+     __u16 sw_rawawb_vec_x21_ls1_y;
+     __u16 sw_rawawb_th0_ls2;
+     __u16 sw_rawawb_th1_ls2;
+     __u16 sw_rawawb_th2_ls2;
+     __u16 sw_rawawb_th3_ls2;
+     __u16 sw_rawawb_th4_ls2;
+     __u16 sw_rawawb_th5_ls2;
+     __u16 sw_rawawb_coor_x1_ls2_u;
+     __u16 sw_rawawb_coor_x1_ls2_v;
+     __u16 sw_rawawb_coor_x1_ls2_y;
+     __u16 sw_rawawb_vec_x21_ls2_u;
+     __u16 sw_rawawb_vec_x21_ls2_v;
+     __u16 sw_rawawb_vec_x21_ls2_y;
+     __u16 sw_rawawb_th0_ls3;
+     __u16 sw_rawawb_th1_ls3;
+     __u16 sw_rawawb_th2_ls3;
+     __u16 sw_rawawb_th3_ls3;
+     __u16 sw_rawawb_th4_ls3;
+     __u16 sw_rawawb_th5_ls3;
+     __u16 sw_rawawb_coor_x1_ls3_u;
+     __u16 sw_rawawb_coor_x1_ls3_v;
+     __u16 sw_rawawb_coor_x1_ls3_y;
+     __u16 sw_rawawb_vec_x21_ls3_u;
+     __u16 sw_rawawb_vec_x21_ls3_v;
+     __u16 sw_rawawb_vec_x21_ls3_y;
+ 
+     __u32 sw_rawawb_islope01_0;
+     __u32 sw_rawawb_islope12_0;
+     __u32 sw_rawawb_islope23_0;
+     __u32 sw_rawawb_islope30_0;
+     __u32 sw_rawawb_islope01_1;
+     __u32 sw_rawawb_islope12_1;
+     __u32 sw_rawawb_islope23_1;
+     __u32 sw_rawawb_islope30_1;
+     __u32 sw_rawawb_islope01_2;
+     __u32 sw_rawawb_islope12_2;
+     __u32 sw_rawawb_islope23_2;
+     __u32 sw_rawawb_islope30_2;
+     __u32 sw_rawawb_islope01_3;
+     __u32 sw_rawawb_islope12_3;
+     __u32 sw_rawawb_islope23_3;
+     __u32 sw_rawawb_islope30_3;
+     __u32 sw_rawawb_islope01_4;
+     __u32 sw_rawawb_islope12_4;
+     __u32 sw_rawawb_islope23_4;
+     __u32 sw_rawawb_islope30_4;
+     __u32 sw_rawawb_islope01_5;
+     __u32 sw_rawawb_islope12_5;
+     __u32 sw_rawawb_islope23_5;
+     __u32 sw_rawawb_islope30_5;
+     __u32 sw_rawawb_islope01_6;
+     __u32 sw_rawawb_islope12_6;
+     __u32 sw_rawawb_islope23_6;
+     __u32 sw_rawawb_islope30_6;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_rawawb_meas_stat {
+     __u16 ro_yhist_bin[ISP3X_RAWAWB_HSTBIN_NUM];
+     __u32 ro_rawawb_sum_rgain_nor[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_sum_bgain_nor[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_wp_num_nor[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_sum_rgain_big[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_sum_bgain_big[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_rawawb_wp_num_big[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_wp_num2[ISP3X_RAWAWB_SUM_NUM];
+     __u32 ro_sum_r_nor_multiwindow[ISP3X_RAWAWB_MULWD_NUM];
+     __u32 ro_sum_b_nor_multiwindow[ISP3X_RAWAWB_MULWD_NUM];
+     __u32 ro_wp_nm_nor_multiwindow[ISP3X_RAWAWB_MULWD_NUM];
+     __u32 ro_sum_r_big_multiwindow[ISP3X_RAWAWB_MULWD_NUM];
+     __u32 ro_sum_b_big_multiwindow[ISP3X_RAWAWB_MULWD_NUM];
+     __u32 ro_wp_nm_big_multiwindow[ISP3X_RAWAWB_MULWD_NUM];
+     __u32 ro_sum_r_exc[ISP3X_RAWAWB_EXCL_STAT_NUM];
+     __u32 ro_sum_b_exc[ISP3X_RAWAWB_EXCL_STAT_NUM];
+     __u32 ro_wp_nm_exc[ISP3X_RAWAWB_EXCL_STAT_NUM];
+     struct isp2x_rawawb_ramdata ramdata[ISP3X_RAWAWB_RAMDATA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_rawaf_curve {
+     __u8 ldg_lumth;
+     __u8 ldg_gain;
+     __u16 ldg_gslp;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_rawaf_meas_cfg {
+     __u8 rawaf_sel;
+     __u8 num_afm_win;
+     /* CTRL */
+     __u8 gamma_en;
+     __u8 gaus_en;
+     __u8 v1_fir_sel;
+     __u8 hiir_en;
+     __u8 viir_en;
+     __u8 accu_8bit_mode;
+     __u8 ldg_en;
+     __u8 h1_fv_mode;
+     __u8 h2_fv_mode;
+     __u8 v1_fv_mode;
+     __u8 v2_fv_mode;
+     __u8 ae_mode;
+     __u8 y_mode;
+     /* WINA_B */
+     struct isp2x_window win[ISP3X_RAWAF_WIN_NUM];
+     /* INT_LINE */
+     __u8 line_num[ISP3X_RAWAF_LINE_NUM];
+     __u8 line_en[ISP3X_RAWAF_LINE_NUM];
+     /* THRES */
+     __u16 afm_thres;
+     /* VAR_SHIFT */
+     __u8 afm_var_shift[ISP3X_RAWAF_WIN_NUM];
+     __u8 lum_var_shift[ISP3X_RAWAF_WIN_NUM];
+     /* HVIIR_VAR_SHIFT */
+     __u8 h1iir_var_shift;
+     __u8 h2iir_var_shift;
+     __u8 v1iir_var_shift;
+     __u8 v2iir_var_shift;
+     /* GAMMA_Y */
+     __u16 gamma_y[ISP3X_RAWAF_GAMMA_NUM];
+     /* HIIR_THRESH */
+     __u16 h_fv_thresh;
+     __u16 v_fv_thresh;
+     struct isp3x_rawaf_curve curve_h[ISP3X_RAWAF_CURVE_NUM];
+     struct isp3x_rawaf_curve curve_v[ISP3X_RAWAF_CURVE_NUM];
+     __s16 h1iir1_coe[ISP3X_RAWAF_HIIR_COE_NUM];
+     __s16 h1iir2_coe[ISP3X_RAWAF_HIIR_COE_NUM];
+     __s16 h2iir1_coe[ISP3X_RAWAF_HIIR_COE_NUM];
+     __s16 h2iir2_coe[ISP3X_RAWAF_HIIR_COE_NUM];
+     __s16 v1iir_coe[ISP3X_RAWAF_V1IIR_COE_NUM];
+     __s16 v2iir_coe[ISP3X_RAWAF_V2IIR_COE_NUM];
+     __s16 v1fir_coe[ISP3X_RAWAF_VFIR_COE_NUM];
+     __s16 v2fir_coe[ISP3X_RAWAF_VFIR_COE_NUM];
+     __u16 highlit_thresh;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_rawaf_ramdata {
+     __u32 v1;
+     __u32 v2;
+     __u32 h1;
+     __u32 h2;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_rawaf_stat {
+     __u32 int_state;
+     __u32 afm_sum_b;
+     __u32 afm_lum_b;
+     __u32 highlit_cnt_winb;
+     struct isp3x_rawaf_ramdata ramdata[ISP3X_RAWAF_SUMDATA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp3x_cac_cfg {
+     __u8 bypass_en;
+     __u8 center_en;
+ 
+     __u8 psf_sft_bit;
+     __u16 cfg_num;
+ 
+     __u16 center_width;
+     __u16 center_height;
+ 
+     __u16 strength[ISP3X_CAC_STRENGTH_NUM];
+ 
+     __u32 hsize;
+     __u32 vsize;
+     __s32 buf_fd;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_gain_cfg {
+     __u32 g0;
+     __u16 g1;
+     __u16 g2;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_isp_other_cfg {
+     struct isp21_bls_cfg bls_cfg;
+     struct isp2x_dpcc_cfg dpcc_cfg;
+     struct isp3x_lsc_cfg lsc_cfg;
+     struct isp21_awb_gain_cfg awb_gain_cfg;
+     struct isp21_gic_cfg gic_cfg;
+     struct isp2x_debayer_cfg debayer_cfg;
+     struct isp21_ccm_cfg ccm_cfg;
+     struct isp3x_gammaout_cfg gammaout_cfg;
+     struct isp2x_cproc_cfg cproc_cfg;
+     struct isp2x_ie_cfg ie_cfg;
+     struct isp2x_sdg_cfg sdg_cfg;
+     struct isp3x_drc_cfg drc_cfg;
+     struct isp3x_hdrmge_cfg hdrmge_cfg;
+     struct isp3x_dhaz_cfg dhaz_cfg;
+     struct isp2x_3dlut_cfg isp3dlut_cfg;
+     struct isp2x_ldch_cfg ldch_cfg;
+     struct isp3x_baynr_cfg baynr_cfg;
+     struct isp3x_bay3d_cfg bay3d_cfg;
+     struct isp3x_ynr_cfg ynr_cfg;
+     struct isp3x_cnr_cfg cnr_cfg;
+     struct isp3x_sharp_cfg sharp_cfg;
+     struct isp3x_cac_cfg cac_cfg;
+     struct isp3x_gain_cfg gain_cfg;
+     struct isp21_csm_cfg csm_cfg;
+     struct isp21_cgc_cfg cgc_cfg;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_isp_meas_cfg {
+     struct isp3x_rawaf_meas_cfg rawaf;
+     struct isp3x_rawawb_meas_cfg rawawb;
+     struct isp2x_rawaelite_meas_cfg rawae0;
+     struct isp2x_rawaebig_meas_cfg rawae1;
+     struct isp2x_rawaebig_meas_cfg rawae2;
+     struct isp2x_rawaebig_meas_cfg rawae3;
+     struct isp2x_rawhistlite_cfg rawhist0;
+     struct isp2x_rawhistbig_cfg rawhist1;
+     struct isp2x_rawhistbig_cfg rawhist2;
+     struct isp2x_rawhistbig_cfg rawhist3;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_isp_params_cfg {
+     __u64 module_en_update;
+     __u64 module_ens;
+     __u64 module_cfg_update;
+ 
+     __u32 frame_id;
+     struct isp3x_isp_meas_cfg meas;
+     struct isp3x_isp_other_cfg others;
+ } __attribute__ ((packed));
+ 
+ struct isp3x_stat {
+     struct isp2x_rawaebig_stat rawae3;
+     struct isp2x_rawaebig_stat rawae1;
+     struct isp2x_rawaebig_stat rawae2;
+     struct isp2x_rawaelite_stat rawae0;
+     struct isp2x_rawhistbig_stat rawhist3;
+     struct isp2x_rawhistlite_stat rawhist0;
+     struct isp2x_rawhistbig_stat rawhist1;
+     struct isp2x_rawhistbig_stat rawhist2;
+     struct isp3x_rawaf_stat rawaf;
+     struct isp3x_rawawb_meas_stat rawawb;
+     struct isp3x_dhaz_stat dhaz;
+     struct isp2x_bls_stat bls;
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp3x_isp_stat_buffer - Rockchip ISP3 Statistics Meta Data
+  *
+  * @meas_type: measurement types (ISP3X_STAT_ definitions)
+  * @frame_id: frame ID for sync
+  * @params: statistics data
+  */
+ struct esisp3x_isp_stat_buffer {
+     __u32 meas_type;
+     __u32 frame_id;
+     __u32 params_id;
+     struct isp3x_stat params;
+ } __attribute__ ((packed));
+ 
+ #endif /* _UAPI_ES_ISP3_CONFIG_H */
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/es-isp32-config.h b/drivers/media/platform/eswin/dvp2axi/es-isp32-config.h
new file mode 100644
index 000000000000..cf9670f60b9d
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/es-isp32-config.h
@@ -0,0 +1,1545 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI isp config driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _UAPI_RK_ISP32_CONFIG_H
+ #define _UAPI_RK_ISP32_CONFIG_H
+ 
+ #include <linux/types.h>
+ #include <linux/v4l2-controls.h>
+ #include "es-isp3-config.h"
+ 
+ #define RKISP_CMD_GET_TB_HEAD_V32 \
+     _IOR('V', BASE_VIDIOC_PRIVATE + 12, struct esisp32_thunderboot_resmem_head)
+ 
+ #define ISP32_MODULE_DPCC		ISP3X_MODULE_DPCC
+ #define ISP32_MODULE_BLS		ISP3X_MODULE_BLS
+ #define ISP32_MODULE_SDG		ISP3X_MODULE_SDG
+ #define ISP32_MODULE_LSC		ISP3X_MODULE_LSC
+ #define ISP32_MODULE_AWB_GAIN		ISP3X_MODULE_AWB_GAIN
+ #define ISP32_MODULE_BDM		ISP3X_MODULE_BDM
+ #define ISP32_MODULE_CCM		ISP3X_MODULE_CCM
+ #define ISP32_MODULE_GOC		ISP3X_MODULE_GOC
+ #define ISP32_MODULE_CPROC		ISP3X_MODULE_CPROC
+ #define ISP32_MODULE_IE			ISP3X_MODULE_IE
+ #define ISP32_MODULE_RAWAF		ISP3X_MODULE_RAWAF
+ #define ISP32_MODULE_RAWAE0		ISP3X_MODULE_RAWAE0
+ #define ISP32_MODULE_RAWAE1		ISP3X_MODULE_RAWAE1
+ #define ISP32_MODULE_RAWAE2		ISP3X_MODULE_RAWAE2
+ #define ISP32_MODULE_RAWAE3		ISP3X_MODULE_RAWAE3
+ #define ISP32_MODULE_RAWAWB		ISP3X_MODULE_RAWAWB
+ #define ISP32_MODULE_RAWHIST0		ISP3X_MODULE_RAWHIST0
+ #define ISP32_MODULE_RAWHIST1		ISP3X_MODULE_RAWHIST1
+ #define ISP32_MODULE_RAWHIST2		ISP3X_MODULE_RAWHIST2
+ #define ISP32_MODULE_RAWHIST3		ISP3X_MODULE_RAWHIST3
+ #define ISP32_MODULE_HDRMGE		ISP3X_MODULE_HDRMGE
+ #define ISP32_MODULE_RAWNR		ISP3X_MODULE_RAWNR
+ #define ISP32_MODULE_GIC		ISP3X_MODULE_GIC
+ #define ISP32_MODULE_DHAZ		ISP3X_MODULE_DHAZ
+ #define ISP32_MODULE_3DLUT		ISP3X_MODULE_3DLUT
+ #define ISP32_MODULE_LDCH		ISP3X_MODULE_LDCH
+ #define ISP32_MODULE_GAIN		ISP3X_MODULE_GAIN
+ #define ISP32_MODULE_DEBAYER		ISP3X_MODULE_DEBAYER
+ #define ISP32_MODULE_BAYNR		ISP3X_MODULE_BAYNR
+ #define ISP32_MODULE_BAY3D		ISP3X_MODULE_BAY3D
+ #define ISP32_MODULE_YNR		ISP3X_MODULE_YNR
+ #define ISP32_MODULE_CNR		ISP3X_MODULE_CNR
+ #define ISP32_MODULE_SHARP		ISP3X_MODULE_SHARP
+ #define ISP32_MODULE_DRC		ISP3X_MODULE_DRC
+ #define ISP32_MODULE_CAC		ISP3X_MODULE_CAC
+ #define ISP32_MODULE_CSM		ISP3X_MODULE_CSM
+ #define ISP32_MODULE_CGC		ISP3X_MODULE_CGC
+ #define ISP32_MODULE_VSM		BIT_ULL(45)
+ #define ISP32_MODULE_RTT_FST		BIT_ULL(62)
+ #define ISP32_MODULE_FORCE		ISP3X_MODULE_FORCE
+ 
+ /* Measurement types */
+ #define ISP32_STAT_RAWAWB		ISP3X_STAT_RAWAWB
+ #define ISP32_STAT_RAWAF		ISP3X_STAT_RAWAF
+ #define ISP32_STAT_RAWAE0		ISP3X_STAT_RAWAE0
+ #define ISP32_STAT_RAWAE1		ISP3X_STAT_RAWAE1
+ #define ISP32_STAT_RAWAE2		ISP3X_STAT_RAWAE2
+ #define ISP32_STAT_RAWAE3		ISP3X_STAT_RAWAE3
+ #define ISP32_STAT_RAWHST0		ISP3X_STAT_RAWHST0
+ #define ISP32_STAT_RAWHST1		ISP3X_STAT_RAWHST1
+ #define ISP32_STAT_RAWHST2		ISP3X_STAT_RAWHST2
+ #define ISP32_STAT_RAWHST3		ISP3X_STAT_RAWHST3
+ #define ISP32_STAT_BLS			ISP3X_STAT_BLS
+ #define ISP32_STAT_DHAZ			ISP3X_STAT_DHAZ
+ #define ISP32_STAT_VSM			BIT(18)
+ #define ISP32_STAT_INFO2DDR		BIT(19)
+ #define ISP32_STAT_RTT_FST		BIT(31)
+ 
+ #define ISP32_MESH_BUF_NUM		ISP3X_MESH_BUF_NUM
+ 
+ #define ISP32_LSC_GRAD_TBL_SIZE		ISP3X_LSC_GRAD_TBL_SIZE
+ #define ISP32_LSC_SIZE_TBL_SIZE		ISP3X_LSC_SIZE_TBL_SIZE
+ #define ISP32_LSC_DATA_TBL_SIZE		ISP3X_LSC_DATA_TBL_SIZE
+ 
+ #define ISP32_DEGAMMA_CURVE_SIZE	ISP3X_DEGAMMA_CURVE_SIZE
+ 
+ #define ISP32_GAIN_IDX_NUM		ISP3X_GAIN_IDX_NUM
+ #define ISP32_GAIN_LUT_NUM		ISP3X_GAIN_LUT_NUM
+ 
+ #define ISP32_RAWAWB_EXCL_STAT_NUM	ISP3X_RAWAWB_EXCL_STAT_NUM
+ #define ISP32_RAWAWB_HSTBIN_NUM		ISP3X_RAWAWB_HSTBIN_NUM
+ #define ISP32_RAWAWB_WEIGHT_NUM		ISP3X_RAWAWB_WEIGHT_NUM
+ #define ISP32_RAWAWB_SUM_NUM		4
+ #define ISP32_RAWAWB_RAMDATA_NUM	ISP3X_RAWAWB_RAMDATA_NUM
+ #define ISP32L_RAWAWB_WEIGHT_NUM	5
+ #define ISP32L_RAWAWB_RAMDATA_RGB_NUM	25
+ #define ISP32L_RAWAWB_RAMDATA_WP_NUM	13
+ 
+ #define	ISP32_RAWAEBIG_SUBWIN_NUM	ISP3X_RAWAEBIG_SUBWIN_NUM
+ #define ISP32_RAWAEBIG_MEAN_NUM		ISP3X_RAWAEBIG_MEAN_NUM
+ #define ISP32_RAWAELITE_MEAN_NUM	ISP3X_RAWAELITE_MEAN_NUM
+ 
+ #define ISP32_RAWHISTBIG_SUBWIN_NUM	ISP3X_RAWHISTBIG_SUBWIN_NUM
+ #define ISP32_RAWHISTLITE_SUBWIN_NUM	ISP3X_RAWHISTLITE_SUBWIN_NUM
+ #define ISP32_HIST_BIN_N_MAX		ISP3X_HIST_BIN_N_MAX
+ #define ISP32L_HIST_LITE_BIN_N_MAX	64
+ 
+ #define ISP32_RAWAF_CURVE_NUM		ISP3X_RAWAF_CURVE_NUM
+ #define ISP32_RAWAF_HIIR_COE_NUM	ISP3X_RAWAF_HIIR_COE_NUM
+ #define ISP32_RAWAF_VFIR_COE_NUM	ISP3X_RAWAF_VFIR_COE_NUM
+ #define ISP32_RAWAF_WIN_NUM		ISP3X_RAWAF_WIN_NUM
+ #define ISP32_RAWAF_LINE_NUM		ISP3X_RAWAF_LINE_NUM
+ #define ISP32_RAWAF_GAMMA_NUM		ISP3X_RAWAF_GAMMA_NUM
+ #define ISP32_RAWAF_SUMDATA_NUM		ISP3X_RAWAF_SUMDATA_NUM
+ #define ISP32_RAWAF_VIIR_COE_NUM	3
+ #define ISP32_RAWAF_GAUS_COE_NUM	9
+ #define ISP32L_RAWAF_WND_DATA		25
+ 
+ #define ISP32_DPCC_PDAF_POINT_NUM	ISP3X_DPCC_PDAF_POINT_NUM
+ 
+ #define ISP32_HDRMGE_L_CURVE_NUM	ISP3X_HDRMGE_L_CURVE_NUM
+ #define ISP32_HDRMGE_E_CURVE_NUM	ISP3X_HDRMGE_E_CURVE_NUM
+ 
+ #define ISP32_GIC_SIGMA_Y_NUM		ISP3X_GIC_SIGMA_Y_NUM
+ 
+ #define ISP32_CCM_CURVE_NUM		18
+ 
+ #define ISP32_3DLUT_DATA_NUM		ISP3X_3DLUT_DATA_NUM
+ 
+ #define ISP32_LDCH_MESH_XY_NUM		ISP3X_LDCH_MESH_XY_NUM
+ #define ISP32_LDCH_BIC_NUM		36
+ 
+ #define ISP32_GAMMA_OUT_MAX_SAMPLES     ISP3X_GAMMA_OUT_MAX_SAMPLES
+ 
+ #define ISP32_DHAZ_SIGMA_IDX_NUM	ISP3X_DHAZ_SIGMA_IDX_NUM
+ #define ISP32_DHAZ_SIGMA_LUT_NUM	ISP3X_DHAZ_SIGMA_LUT_NUM
+ #define ISP32_DHAZ_HIST_WR_NUM		ISP3X_DHAZ_HIST_WR_NUM
+ #define ISP32_DHAZ_ENH_CURVE_NUM	ISP3X_DHAZ_ENH_CURVE_NUM
+ #define ISP32_DHAZ_HIST_IIR_NUM		ISP3X_DHAZ_HIST_IIR_NUM
+ #define ISP32_DHAZ_ENH_LUMA_NUM		17
+ 
+ #define ISP32_DRC_Y_NUM			ISP3X_DRC_Y_NUM
+ 
+ #define ISP32_CNR_SIGMA_Y_NUM		ISP3X_CNR_SIGMA_Y_NUM
+ #define ISP32_CNR_GAUS_COE_NUM		6
+ 
+ #define ISP32_YNR_XY_NUM		ISP3X_YNR_XY_NUM
+ #define ISP32_YNR_NLM_COE_NUM		6
+ 
+ #define ISP32_BAYNR_XY_NUM		ISP3X_BAYNR_XY_NUM
+ #define ISP32_BAYNR_GAIN_NUM		16
+ 
+ #define ISP32_BAY3D_XY_NUM		ISP3X_BAY3D_XY_NUM
+ 
+ #define ISP32_SHARP_X_NUM		ISP3X_SHARP_X_NUM
+ #define ISP32_SHARP_Y_NUM		ISP3X_SHARP_Y_NUM
+ #define ISP32_SHARP_GAUS_COEF_NUM	ISP3X_SHARP_GAUS_COEF_NUM
+ #define ISP32_SHARP_GAIN_ADJ_NUM	14
+ #define ISP32_SHARP_STRENGTH_NUM	22
+ 
+ #define ISP32_CAC_STRENGTH_NUM		ISP3X_CAC_STRENGTH_NUM
+ 
+ #define ISP32_CSM_COEFF_NUM		ISP3X_CSM_COEFF_NUM
+ 
+ struct isp32_ldch_cfg {
+     __u8 frm_end_dis;
+     __u8 zero_interp_en;
+     __u8 sample_avr_en;
+     __u8 bic_mode_en;
+     __u8 force_map_en;
+     __u8 map13p3_en;
+ 
+     __u8 bicubic[ISP32_LDCH_BIC_NUM];
+ 
+     __u32 hsize;
+     __u32 vsize;
+     __s32 buf_fd;
+ } __attribute__ ((packed));
+ 
+ struct isp32_awb_gain_cfg {
+     /* AWB1_GAIN_G */
+     __u16 awb1_gain_gb;
+     __u16 awb1_gain_gr;
+     /* AWB1_GAIN_RB */
+     __u16 awb1_gain_b;
+     __u16 awb1_gain_r;
+     /* AWB0_GAIN0_G */
+     __u16 gain0_green_b;
+     __u16 gain0_green_r;
+     /* AWB0_GAIN0_RB*/
+     __u16 gain0_blue;
+     __u16 gain0_red;
+     /* AWB0_GAIN1_G */
+     __u16 gain1_green_b;
+     __u16 gain1_green_r;
+     /* AWB0_GAIN1_RB*/
+     __u16 gain1_blue;
+     __u16 gain1_red;
+     /* AWB0_GAIN2_G */
+     __u16 gain2_green_b;
+     __u16 gain2_green_r;
+     /* AWB0_GAIN2_RB*/
+     __u16 gain2_blue;
+     __u16 gain2_red;
+ } __attribute__ ((packed));
+ 
+ struct isp32_bls_cfg {
+     __u8 enable_auto;
+     __u8 en_windows;
+     __u8 bls1_en;
+ 
+     __u8 bls_samples;
+ 
+     struct isp2x_window bls_window1;
+     struct isp2x_window bls_window2;
+     struct isp2x_bls_fixed_val fixed_val;
+     struct isp2x_bls_fixed_val bls1_val;
+ 
+     __u16 isp_ob_offset;
+     __u16 isp_ob_predgain;
+     __u32 isp_ob_max;
+ } __attribute__ ((packed));
+ 
+ struct isp32_ccm_cfg {
+     /* CTRL */
+     __u8 highy_adjust_dis;
+     __u8 enh_adj_en;
+     __u8 asym_adj_en;
+     /* BOUND_BIT */
+     __u8 bound_bit;
+     __u8 right_bit;
+     /* COEFF0_R */
+     __s16 coeff0_r;
+     __s16 coeff1_r;
+     /* COEFF1_R */
+     __s16 coeff2_r;
+     __s16 offset_r;
+     /* COEFF0_G */
+     __s16 coeff0_g;
+     __s16 coeff1_g;
+     /* COEFF1_G */
+     __s16 coeff2_g;
+     __s16 offset_g;
+     /* COEFF0_B */
+     __s16 coeff0_b;
+     __s16 coeff1_b;
+     /* COEFF1_B */
+     __s16 coeff2_b;
+     __s16 offset_b;
+     /* COEFF0_Y */
+     __u16 coeff0_y;
+     __u16 coeff1_y;
+     /* COEFF1_Y */
+     __u16 coeff2_y;
+     /* ALP_Y */
+     __u16 alp_y[ISP32_CCM_CURVE_NUM];
+     /* ENHANCE0 */
+     __u16 color_coef0_r2y;
+     __u16 color_coef1_g2y;
+     /* ENHANCE1 */
+     __u16 color_coef2_b2y;
+     __u16 color_enh_rat_max;
+ } __attribute__ ((packed));
+ 
+ struct isp32_debayer_cfg {
+     /* CONTROL */
+     __u8 filter_g_en;
+     __u8 filter_c_en;
+     /* G_INTERP */
+     __u8 clip_en;
+     __u8 dist_scale;
+     __u8 thed0;
+     __u8 thed1;
+     __u8 select_thed;
+     __u8 max_ratio;
+     /* G_INTERP_FILTER1 */
+     __s8 filter1_coe1;
+     __s8 filter1_coe2;
+     __s8 filter1_coe3;
+     __s8 filter1_coe4;
+     /* G_INTERP_FILTER2 */
+     __s8 filter2_coe1;
+     __s8 filter2_coe2;
+     __s8 filter2_coe3;
+     __s8 filter2_coe4;
+     /* C_FILTER_GUIDE_GAUS */
+     __s8 guid_gaus_coe0;
+     __s8 guid_gaus_coe1;
+     __s8 guid_gaus_coe2;
+     /* C_FILTER_CE_GAUS */
+     __s8 ce_gaus_coe0;
+     __s8 ce_gaus_coe1;
+     __s8 ce_gaus_coe2;
+     /* C_FILTER_ALPHA_GAUS */
+     __s8 alpha_gaus_coe0;
+     __s8 alpha_gaus_coe1;
+     __s8 alpha_gaus_coe2;
+     /* C_FILTER_IIR_0 */
+     __u8 ce_sgm;
+     __u8 exp_shift;
+     /* C_FILTER_IIR_1 */
+     __u8 wet_clip;
+     __u8 wet_ghost;
+     /* C_FILTER_BF */
+     __u8 bf_clip;
+     __u8 bf_curwgt;
+     __u16 bf_sgm;
+     /* G_INTERP_OFFSET */
+     __u16 hf_offset;
+     __u16 gain_offset;
+     /* G_FILTER_OFFSET */
+     __u16 offset;
+     /* C_FILTER_LOG_OFFSET */
+     __u16 loghf_offset;
+     __u16 loggd_offset;
+     /* C_FILTER_IIR_0 */
+     __u16 wgtslope;
+     /* C_FILTER_ALPHA */
+     __u16 alpha_offset;
+     /* C_FILTER_EDGE */
+     __u16 edge_offset;
+     __u32 edge_scale;
+     /* C_FILTER_ALPHA */
+     __u32 alpha_scale;
+ } __attribute__ ((packed));
+ 
+ struct isp32_baynr_cfg {
+     /* BAYNR_CTRL */
+     __u8 bay3d_gain_en;
+     __u8 lg2_mode;
+     __u8 gauss_en;
+     __u8 log_bypass;
+     /* BAYNR_DGAIN */
+     __u16 dgain1;
+     __u16 dgain0;
+     __u16 dgain2;
+     /* BAYNR_PIXDIFF */
+     __u16 pix_diff;
+     /* BAYNR_THLD */
+     __u16 diff_thld;
+     __u16 softthld;
+     /* BAYNR_W1_STRENG */
+     __u16 bltflt_streng;
+     __u16 reg_w1;
+     /* BAYNR_SIGMA */
+     __u16 sigma_x[ISP32_BAYNR_XY_NUM];
+     __u16 sigma_y[ISP32_BAYNR_XY_NUM];
+     /* BAYNR_WRIT_D */
+     __u16 weit_d2;
+     __u16 weit_d1;
+     __u16 weit_d0;
+     /* BAYNR_LG_OFF */
+     __u16 lg2_lgoff;
+     __u16 lg2_off;
+     /* BAYNR_DAT_MAX */
+     __u32 dat_max;
+     /* BAYNR_SIGOFF */
+     __u16 rgain_off;
+     __u16 bgain_off;
+     /* BAYNR_GAIN */
+     __u8 gain_x[ISP32_BAYNR_GAIN_NUM];
+     __u16 gain_y[ISP32_BAYNR_GAIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_bay3d_cfg {
+     /* BAY3D_CTRL */
+     __u8 bypass_en;
+     __u8 hibypass_en;
+     __u8 lobypass_en;
+     __u8 himed_bypass_en;
+     __u8 higaus_bypass_en;
+     __u8 hiabs_possel;
+     __u8 hichnsplit_en;
+     __u8 lomed_bypass_en;
+     __u8 logaus5_bypass_en;
+     __u8 logaus3_bypass_en;
+     __u8 glbpk_en;
+     __u8 loswitch_protect;
+     __u8 bwsaving_en;
+     /* BAY3D_CTRL1 */
+     __u8 hiwgt_opt_en;
+     __u8 hichncor_en;
+     __u8 bwopt_gain_dis;
+     __u8 lo4x8_en;
+     __u8 lo4x4_en;
+     __u8 hisig_ind_sel;
+     __u8 pksig_ind_sel;
+     __u8 iirwr_rnd_en;
+     __u8 curds_high_en;
+     __u8 higaus3_mode;
+     __u8 higaus5x5_en;
+     __u8 wgtmix_opt_en;
+ 
+     /* for isp32_lite */
+     __u8 wgtmm_opt_en;
+     __u8 wgtmm_sel_en;
+ 
+     /* BAY3D_SIGGAUS */
+     __u8 siggaus0;
+     __u8 siggaus1;
+     __u8 siggaus2;
+     __u8 siggaus3;
+     /* BAY3D_KALRATIO */
+     __u16 softwgt;
+     __u16 hidif_th;
+     /* BAY3D_WGTLMT */
+     __u16 wgtlmt;
+     __u16 wgtratio;
+     /* BAY3D_SIG */
+     __u16 sig0_x[ISP32_BAY3D_XY_NUM];
+     __u16 sig0_y[ISP32_BAY3D_XY_NUM];
+     __u16 sig1_x[ISP32_BAY3D_XY_NUM];
+     __u16 sig1_y[ISP32_BAY3D_XY_NUM];
+     __u16 sig2_x[ISP32_BAY3D_XY_NUM];
+     __u16 sig2_y[ISP32_BAY3D_XY_NUM];
+ 
+     /* LODIF_STAT1 for isp32_lite */
+     __u16 wgtmin;
+ 
+     /* BAY3D_HISIGRAT */
+     __u16 hisigrat0;
+     __u16 hisigrat1;
+     /* BAY3D_HISIGOFF */
+     __u16 hisigoff0;
+     __u16 hisigoff1;
+     /* BAY3D_LOSIG */
+     __u16 losigoff;
+     __u16 losigrat;
+     /* BAY3D_SIGPK */
+     __u16 rgain_off;
+     __u16 bgain_off;
+     /* BAY3D_GLBPK2 */
+     __u32 glbpk2;
+ } __attribute__ ((packed));
+ 
+ struct isp32_ynr_cfg {
+     /* YNR_GLOBAL_CTRL */
+     __u8 rnr_en;
+     __u8 thumb_mix_cur_en;
+     __u8 global_gain_alpha;
+     __u8 flt1x1_bypass_sel;
+     __u8 nlm11x11_bypass;
+     __u8 flt1x1_bypass;
+     __u8 lgft3x3_bypass;
+     __u8 lbft5x5_bypass;
+     __u8 bft3x3_bypass;
+     /* YNR_RNR_STRENGTH */
+     __u8 rnr_strength3[ISP32_YNR_XY_NUM];
+     /* YNR_NLM_SIGMA_GAIN */
+     __u8 nlm_hi_gain_alpha;
+     /* YNR_NLM_COE */
+     __u8 nlm_coe[ISP32_YNR_NLM_COE_NUM];
+ 
+     /* LOWNR_CTRL4 for isp32_lite */
+     __u8 frame_add4line;
+ 
+     __u16 global_gain;
+ 
+     /* YNR_RNR_MAX_R */
+     __u16 rnr_max_r;
+     __u16 local_gainscale;
+     /* YNR_RNR_CENTER_COOR */
+     __u16 rnr_center_coorh;
+     __u16 rnr_center_coorv;
+     /* YNR_LOCAL_GAIN_CTRL */
+     __u16 loclagain_adj_thresh;
+     __u16 localgain_adj;
+     /* YNR_LOWNR_CTRL0 */
+     __u16 low_bf_inv1;
+     __u16 low_bf_inv0;
+     /* YNR_LOWNR_CTRL1 */
+     __u16 low_peak_supress;
+     __u16 low_thred_adj;
+     /* YNR_LOWNR_CTRL2 */
+     __u16 low_dist_adj;
+     __u16 low_edge_adj_thresh;
+     /* YNR_LOWNR_CTRL3 */
+     __u16 low_bi_weight;
+     __u16 low_weight;
+     __u16 low_center_weight;
+     /* YNR_LOWNR_CTRL4 */
+     __u16 frame_full_size;
+     __u16 lbf_weight_thres;
+     /* YNR_GAUSS1_COEFF */
+     __u16 low_gauss1_coeff2;
+     __u16 low_gauss1_coeff1;
+     __u16 low_gauss1_coeff0;
+     /* YNR_GAUSS2_COEFF */
+     __u16 low_gauss2_coeff2;
+     __u16 low_gauss2_coeff1;
+     __u16 low_gauss2_coeff0;
+     /* YNR_SGM_DX */
+     __u16 luma_points_x[ISP32_YNR_XY_NUM];
+     /* YNR_LSGM_Y */
+     __u16 lsgm_y[ISP32_YNR_XY_NUM];
+     /* YNR_NLM_SIGMA_GAIN */
+     __u16 nlm_min_sigma;
+     __u16 nlm_hi_bf_scale;
+     /* YNR_NLM_WEIGHT */
+     __u16 nlm_nr_weight;
+     __u16 nlm_weight_offset;
+     /* YNR_NLM_NR_WEIGHT */
+     __u32 nlm_center_weight;
+ } __attribute__ ((packed));
+ 
+ struct isp32_cnr_cfg {
+     /* CNR_CTRL */
+     __u8 exgain_bypass;
+     __u8 yuv422_mode;
+     __u8 thumb_mode;
+     __u8 bf3x3_wgt0_sel;
+     /* CNR_LBF_WEITD */
+     __u8 lbf1x7_weit_d0;
+     __u8 lbf1x7_weit_d1;
+     __u8 lbf1x7_weit_d2;
+     __u8 lbf1x7_weit_d3;
+     /* CNR_IIR_PARA1 */
+     __u8 iir_uvgain;
+     __u8 iir_strength;
+     __u8 exp_shift;
+     /* CNR_IIR_PARA2 */
+     __u8 chroma_ghost;
+     __u8 iir_uv_clip;
+     /* CNR_GAUS_COE */
+     __u8 gaus_coe[ISP32_CNR_GAUS_COE_NUM];
+     /* CNR_GAUS_RATIO */
+     __u8 bf_wgt_clip;
+     /* CNR_BF_PARA1 */
+     __u8 uv_gain;
+     __u8 bf_ratio;
+     /* CNR_SIGMA */
+     __u8 sigma_y[ISP32_CNR_SIGMA_Y_NUM];
+     /* CNR_IIR_GLOBAL_GAIN */
+     __u8 iir_gain_alpha;
+     __u8 iir_global_gain;
+     /* CNR_EXGAIN */
+     __u8 gain_iso;
+     __u8 global_gain_alpha;
+     __u16 global_gain;
+     /* CNR_THUMB1 */
+     __u16 thumb_sigma_c;
+     __u16 thumb_sigma_y;
+     /* CNR_THUMB_BF_RATIO */
+     __u16 thumb_bf_ratio;
+     /* CNR_IIR_PARA1 */
+     __u16 wgt_slope;
+     /* CNR_GAUS_RATIO */
+     __u16 gaus_ratio;
+     __u16 global_alpha;
+     /* CNR_BF_PARA1 */
+     __u16 sigma_r;
+     /* CNR_BF_PARA2 */
+     __u16 adj_offset;
+     __u16 adj_ratio;
+ } __attribute__ ((packed));
+ 
+ struct isp32_sharp_cfg {
+     /* SHARP_EN */
+     __u8 bypass;
+     __u8 center_mode;
+     __u8 exgain_bypass;
+     __u8 radius_ds_mode;
+     __u8 noiseclip_mode;
+ 
+     /* for isp32_lite */
+     __u8 clip_hf_mode;
+     __u8 add_mode;
+ 
+     /* SHARP_RATIO */
+     __u8 sharp_ratio;
+     __u8 bf_ratio;
+     __u8 gaus_ratio;
+     __u8 pbf_ratio;
+     /* SHARP_LUMA_DX */
+     __u8 luma_dx[ISP32_SHARP_X_NUM];
+     /* SHARP_SIGMA_SHIFT */
+     __u8 bf_sigma_shift;
+     __u8 pbf_sigma_shift;
+     /* SHARP_PBF_COEF */
+     __u8 pbf_coef2;
+     __u8 pbf_coef1;
+     __u8 pbf_coef0;
+     /* SHARP_BF_COEF */
+     __u8 bf_coef2;
+     __u8 bf_coef1;
+     __u8 bf_coef0;
+     /* SHARP_GAUS_COEF */
+     __u8 gaus_coef[ISP32_SHARP_GAUS_COEF_NUM];
+     /* SHARP_GAIN */
+     __u8 global_gain_alpha;
+     __u8 local_gainscale;
+     /* SHARP_GAIN_DIS_STRENGTH */
+     __u8 strength[ISP32_SHARP_STRENGTH_NUM];
+     /* SHARP_TEXTURE */
+     __u8 enhance_bit;
+     /* SHARP_PBF_SIGMA_INV */
+     __u16 pbf_sigma_inv[ISP32_SHARP_Y_NUM];
+     /* SHARP_BF_SIGMA_INV */
+     __u16 bf_sigma_inv[ISP32_SHARP_Y_NUM];
+     /* SHARP_CLIP_HF */
+     __u16 clip_hf[ISP32_SHARP_Y_NUM];
+     /* SHARP_GAIN */
+     __u16 global_gain;
+     /* SHARP_GAIN_ADJUST */
+     __u16 gain_adj[ISP32_SHARP_GAIN_ADJ_NUM];
+     /* SHARP_CENTER */
+     __u16 center_wid;
+     __u16 center_het;
+     /* SHARP_TEXTURE */
+     __u16 noise_sigma;
+     __u16 noise_strength;
+ 
+     /* EHF_TH for isp32_lite */
+     __u16 ehf_th[ISP32_SHARP_Y_NUM];
+     /* CLIP_NEG for isp32_lite */
+     __u16 clip_neg[ISP32_SHARP_Y_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_dhaz_cfg {
+     /* DHAZ_CTRL */
+     __u8 enh_luma_en;
+     __u8 color_deviate_en;
+     __u8 round_en;
+     __u8 soft_wr_en;
+     __u8 enhance_en;
+     __u8 air_lc_en;
+     __u8 hpara_en;
+     __u8 hist_en;
+     __u8 dc_en;
+     /* DHAZ_ADP0 */
+     __u8 yblk_th;
+     __u8 yhist_th;
+     __u8 dc_max_th;
+     __u8 dc_min_th;
+     /* DHAZ_ADP2 */
+     __u8 tmax_base;
+     __u8 dark_th;
+     __u8 air_max;
+     __u8 air_min;
+     /* DHAZ_GAUS */
+     __u8 gaus_h2;
+     __u8 gaus_h1;
+     __u8 gaus_h0;
+     /* DHAZ_GAIN_IDX */
+     __u8 sigma_idx[ISP32_DHAZ_SIGMA_IDX_NUM];
+     /* DHAZ_ADP_HIST1 */
+     __u8 hist_gratio;
+     __u16 hist_scale;
+     /* DHAZ_ADP1 */
+     __u8 bright_max;
+     __u8 bright_min;
+     __u16 wt_max;
+     /* DHAZ_ADP_TMAX */
+     __u16 tmax_max;
+     __u16 tmax_off;
+     /* DHAZ_ADP_HIST0 */
+     __u8 hist_k;
+     __u8 hist_th_off;
+     __u16 hist_min;
+     /* DHAZ_ENHANCE */
+     __u16 enhance_value;
+     __u16 enhance_chroma;
+     /* DHAZ_IIR0 */
+     __u16 iir_wt_sigma;
+     __u8 iir_sigma;
+     __u8 stab_fnum;
+     /* DHAZ_IIR1 */
+     __u16 iir_tmax_sigma;
+     __u8 iir_air_sigma;
+     __u8 iir_pre_wet;
+     /* DHAZ_SOFT_CFG0 */
+     __u16 cfg_wt;
+     __u8 cfg_air;
+     __u8 cfg_alpha;
+     /* DHAZ_SOFT_CFG1 */
+     __u16 cfg_gratio;
+     __u16 cfg_tmax;
+     /* DHAZ_BF_SIGMA */
+     __u16 range_sima;
+     __u8 space_sigma_pre;
+     __u8 space_sigma_cur;
+     /* DHAZ_BF_WET */
+     __u16 dc_weitcur;
+     __u16 bf_weight;
+     /* DHAZ_ENH_CURVE */
+     __u16 enh_curve[ISP32_DHAZ_ENH_CURVE_NUM];
+ 
+     __u16 sigma_lut[ISP32_DHAZ_SIGMA_LUT_NUM];
+ 
+     __u16 hist_wr[ISP32_DHAZ_HIST_WR_NUM];
+ 
+     __u16 enh_luma[ISP32_DHAZ_ENH_LUMA_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_drc_cfg {
+     __u8 bypass_en;
+     /* DRC_CTRL1 */
+     __u8 offset_pow2;
+     __u16 compres_scl;
+     __u16 position;
+     /* DRC_LPRATIO */
+     __u16 hpdetail_ratio;
+     __u16 lpdetail_ratio;
+     __u8 delta_scalein;
+     /* DRC_EXPLRATIO */
+     __u8 weicur_pix;
+     __u8 weipre_frame;
+     __u8 bilat_wt_off;
+     /* DRC_SIGMA */
+     __u8 edge_scl;
+     __u8 motion_scl;
+     __u16 force_sgm_inv0;
+     /* DRC_SPACESGM */
+     __u16 space_sgm_inv1;
+     __u16 space_sgm_inv0;
+     /* DRC_RANESGM */
+     __u16 range_sgm_inv1;
+     __u16 range_sgm_inv0;
+     /* DRC_BILAT */
+     __u16 bilat_soft_thd;
+     __u8 weig_maxl;
+     __u8 weig_bilat;
+     __u8 enable_soft_thd;
+     /* DRC_IIRWG_GAIN */
+     __u8 iir_weight;
+     __u16 min_ogain;
+     /* DRC_LUM3X2_CTRL */
+     __u16 gas_t;
+     /* DRC_LUM3X2_GAS */
+     __u8 gas_l0;
+     __u8 gas_l1;
+     __u8 gas_l2;
+     __u8 gas_l3;
+ 
+     __u16 gain_y[ISP32_DRC_Y_NUM];
+     __u16 compres_y[ISP32_DRC_Y_NUM];
+     __u16 scale_y[ISP32_DRC_Y_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_hdrmge_cfg {
+     __u8 s_base;
+     __u8 mode;
+     __u8 dbg_mode;
+     __u8 each_raw_en;
+ 
+     __u8 gain2;
+ 
+     __u8 lm_dif_0p15;
+     __u8 lm_dif_0p9;
+     __u8 ms_diff_0p15;
+     __u8 ms_dif_0p8;
+ 
+     __u16 gain0_inv;
+     __u16 gain0;
+     __u16 gain1_inv;
+     __u16 gain1;
+ 
+     __u16 ms_thd1;
+     __u16 ms_thd0;
+     __u16 ms_scl;
+     __u16 lm_thd1;
+     __u16 lm_thd0;
+     __u16 lm_scl;
+     struct isp2x_hdrmge_curve curve;
+     __u16 e_y[ISP32_HDRMGE_E_CURVE_NUM];
+     __u16 l_raw0[ISP32_HDRMGE_E_CURVE_NUM];
+     __u16 l_raw1[ISP32_HDRMGE_E_CURVE_NUM];
+     __u16 each_raw_gain0;
+     __u16 each_raw_gain1;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawawb_meas_cfg {
+     __u8 bls2_en;
+ 
+     __u8 rawawb_sel;
+     __u8 bnr2awb_sel;
+     __u8 drc2awb_sel;
+     /* RAWAWB_CTRL */
+     __u8 uv_en0;
+     __u8 xy_en0;
+     __u8 yuv3d_en0;
+     __u8 yuv3d_ls_idx0;
+     __u8 yuv3d_ls_idx1;
+     __u8 yuv3d_ls_idx2;
+     __u8 yuv3d_ls_idx3;
+     __u8 in_rshift_to_12bit_en;
+     __u8 in_overexposure_check_en;
+     __u8 wind_size;
+     __u8 rawlsc_bypass_en;
+     __u8 light_num;
+     __u8 uv_en1;
+     __u8 xy_en1;
+     __u8 yuv3d_en1;
+     __u8 low12bit_val;
+     /* RAWAWB_WEIGHT_CURVE_CTRL */
+     __u8 wp_luma_wei_en0;
+     __u8 wp_luma_wei_en1;
+     __u8 wp_blk_wei_en0;
+     __u8 wp_blk_wei_en1;
+     __u8 wp_hist_xytype;
+     /* RAWAWB_MULTIWINDOW_EXC_CTRL */
+     __u8 exc_wp_region0_excen;
+     __u8 exc_wp_region0_measen;
+     __u8 exc_wp_region0_domain;
+     __u8 exc_wp_region1_excen;
+     __u8 exc_wp_region1_measen;
+     __u8 exc_wp_region1_domain;
+     __u8 exc_wp_region2_excen;
+     __u8 exc_wp_region2_measen;
+     __u8 exc_wp_region2_domain;
+     __u8 exc_wp_region3_excen;
+     __u8 exc_wp_region3_measen;
+     __u8 exc_wp_region3_domain;
+     __u8 exc_wp_region4_excen;
+     __u8 exc_wp_region4_domain;
+     __u8 exc_wp_region5_excen;
+     __u8 exc_wp_region5_domain;
+     __u8 exc_wp_region6_excen;
+     __u8 exc_wp_region6_domain;
+     __u8 multiwindow_en;
+     /* RAWAWB_YWEIGHT_CURVE_XCOOR03 */
+     __u8 wp_luma_weicurve_y0;
+     __u8 wp_luma_weicurve_y1;
+     __u8 wp_luma_weicurve_y2;
+     __u8 wp_luma_weicurve_y3;
+     /* RAWAWB_YWEIGHT_CURVE_XCOOR47 */
+     __u8 wp_luma_weicurve_y4;
+     __u8 wp_luma_weicurve_y5;
+     __u8 wp_luma_weicurve_y6;
+     __u8 wp_luma_weicurve_y7;
+     /* RAWAWB_YWEIGHT_CURVE_XCOOR8 */
+     __u8 wp_luma_weicurve_y8;
+     /* RAWAWB_YWEIGHT_CURVE_YCOOR03 */
+     __u8 wp_luma_weicurve_w0;
+     __u8 wp_luma_weicurve_w1;
+     __u8 wp_luma_weicurve_w2;
+     __u8 wp_luma_weicurve_w3;
+     /* RAWAWB_YWEIGHT_CURVE_YCOOR47 */
+     __u8 wp_luma_weicurve_w4;
+     __u8 wp_luma_weicurve_w5;
+     __u8 wp_luma_weicurve_w6;
+     __u8 wp_luma_weicurve_w7;
+     /* RAWAWB_YWEIGHT_CURVE_YCOOR8 */
+     __u8 wp_luma_weicurve_w8;
+     /* RAWAWB_YUV_X1X2_DIS_0 */
+     __u8 dis_x1x2_ls0;
+     __u8 rotu0_ls0;
+     __u8 rotu1_ls0;
+     /* RAWAWB_YUV_INTERP_CURVE_UCOOR_0 */
+     __u8 rotu2_ls0;
+     __u8 rotu3_ls0;
+     __u8 rotu4_ls0;
+     __u8 rotu5_ls0;
+     /* RAWAWB_YUV_X1X2_DIS_1 */
+     __u8 dis_x1x2_ls1;
+     __u8 rotu0_ls1;
+     __u8 rotu1_ls1;
+     /* YUV_INTERP_CURVE_UCOOR_1 */
+     __u8 rotu2_ls1;
+     __u8 rotu3_ls1;
+     __u8 rotu4_ls1;
+     __u8 rotu5_ls1;
+     /* RAWAWB_YUV_X1X2_DIS_2 */
+     __u8 dis_x1x2_ls2;
+     __u8 rotu0_ls2;
+     __u8 rotu1_ls2;
+     /* YUV_INTERP_CURVE_UCOOR_2 */
+     __u8 rotu2_ls2;
+     __u8 rotu3_ls2;
+     __u8 rotu4_ls2;
+     __u8 rotu5_ls2;
+     /* RAWAWB_YUV_X1X2_DIS_3 */
+     __u8 dis_x1x2_ls3;
+     __u8 rotu0_ls3;
+     __u8 rotu1_ls3;
+     __u8 rotu2_ls3;
+     __u8 rotu3_ls3;
+     __u8 rotu4_ls3;
+     __u8 rotu5_ls3;
+     /* RAWAWB_EXC_WP_WEIGHT */
+     __u8 exc_wp_region0_weight;
+     __u8 exc_wp_region1_weight;
+     __u8 exc_wp_region2_weight;
+     __u8 exc_wp_region3_weight;
+     __u8 exc_wp_region4_weight;
+     __u8 exc_wp_region5_weight;
+     __u8 exc_wp_region6_weight;
+     /* RAWAWB_WRAM_DATA */
+     __u8 wp_blk_wei_w[ISP32_RAWAWB_WEIGHT_NUM];
+     /* RAWAWB_BLK_CTRL */
+     __u8 blk_measure_enable;
+     __u8 blk_measure_mode;
+     __u8 blk_measure_xytype;
+     __u8 blk_rtdw_measure_en;
+     __u8 blk_measure_illu_idx;
+ 
+     /* for isp32_lite */
+     __u8 ds16x8_mode_en;
+ 
+     __u8 blk_with_luma_wei_en;
+     __u16 in_overexposure_threshold;
+     /* RAWAWB_LIMIT_RG_MAX*/
+     __u16 r_max;
+     __u16 g_max;
+     /* RAWAWB_LIMIT_BY_MAX */
+     __u16 b_max;
+     __u16 y_max;
+     /* RAWAWB_LIMIT_RG_MIN */
+     __u16 r_min;
+     __u16 g_min;
+     /* RAWAWB_LIMIT_BY_MIN */
+     __u16 b_min;
+     __u16 y_min;
+     /* RAWAWB_WIN_OFFS */
+     __u16 h_offs;
+     __u16 v_offs;
+     /* RAWAWB_WIN_SIZE */
+     __u16 h_size;
+     __u16 v_size;
+     /* RAWAWB_YWEIGHT_CURVE_YCOOR8 */
+     __u16 pre_wbgain_inv_r;
+     /* RAWAWB_PRE_WBGAIN_INV */
+     __u16 pre_wbgain_inv_g;
+     __u16 pre_wbgain_inv_b;
+     /* RAWAWB_UV_DETC_VERTEX */
+     __u16 vertex0_u_0;
+     __u16 vertex0_v_0;
+ 
+     __u16 vertex1_u_0;
+     __u16 vertex1_v_0;
+ 
+     __u16 vertex2_u_0;
+     __u16 vertex2_v_0;
+ 
+     __u16 vertex3_u_0;
+     __u16 vertex3_v_0;
+ 
+     __u16 vertex0_u_1;
+     __u16 vertex0_v_1;
+ 
+     __u16 vertex1_u_1;
+     __u16 vertex1_v_1;
+ 
+     __u16 vertex2_u_1;
+     __u16 vertex2_v_1;
+ 
+     __u16 vertex3_u_1;
+     __u16 vertex3_v_1;
+ 
+     __u16 vertex0_u_2;
+     __u16 vertex0_v_2;
+ 
+     __u16 vertex1_u_2;
+     __u16 vertex1_v_2;
+ 
+     __u16 vertex2_u_2;
+     __u16 vertex2_v_2;
+ 
+     __u16 vertex3_u_2;
+     __u16 vertex3_v_2;
+ 
+     __u16 vertex0_u_3;
+     __u16 vertex0_v_3;
+ 
+     __u16 vertex1_u_3;
+     __u16 vertex1_v_3;
+ 
+     __u16 vertex2_u_3;
+     __u16 vertex2_v_3;
+ 
+     __u16 vertex3_u_3;
+     __u16 vertex3_v_3;
+     /* RAWAWB_RGB2XY_WT */
+     __u16 wt0;
+     __u16 wt1;
+     __u16 wt2;
+     /* RAWAWB_RGB2XY_MAT */
+     __u16 mat0_x;
+     __u16 mat0_y;
+ 
+     __u16 mat1_x;
+     __u16 mat1_y;
+ 
+     __u16 mat2_x;
+     __u16 mat2_y;
+     /* RAWAWB_XY_DETC_NOR */
+     __u16 nor_x0_0;
+     __u16 nor_x1_0;
+     __u16 nor_y0_0;
+     __u16 nor_y1_0;
+ 
+     __u16 nor_x0_1;
+     __u16 nor_x1_1;
+     __u16 nor_y0_1;
+     __u16 nor_y1_1;
+ 
+     __u16 nor_x0_2;
+     __u16 nor_x1_2;
+     __u16 nor_y0_2;
+     __u16 nor_y1_2;
+ 
+     __u16 nor_x0_3;
+     __u16 nor_x1_3;
+     __u16 nor_y0_3;
+     __u16 nor_y1_3;
+     /* RAWAWB_XY_DETC_BIG */
+     __u16 big_x0_0;
+     __u16 big_x1_0;
+     __u16 big_y0_0;
+     __u16 big_y1_0;
+ 
+     __u16 big_x0_1;
+     __u16 big_x1_1;
+     __u16 big_y0_1;
+     __u16 big_y1_1;
+ 
+     __u16 big_x0_2;
+     __u16 big_x1_2;
+     __u16 big_y0_2;
+     __u16 big_y1_2;
+ 
+     __u16 big_x0_3;
+     __u16 big_x1_3;
+     __u16 big_y0_3;
+     __u16 big_y1_3;
+     /* RAWAWB_MULTIWINDOW */
+     __u16 multiwindow0_v_offs;
+     __u16 multiwindow0_h_offs;
+     __u16 multiwindow0_v_size;
+     __u16 multiwindow0_h_size;
+ 
+     __u16 multiwindow1_v_offs;
+     __u16 multiwindow1_h_offs;
+     __u16 multiwindow1_v_size;
+     __u16 multiwindow1_h_size;
+ 
+     __u16 multiwindow2_v_offs;
+     __u16 multiwindow2_h_offs;
+     __u16 multiwindow2_v_size;
+     __u16 multiwindow2_h_size;
+ 
+     __u16 multiwindow3_v_offs;
+     __u16 multiwindow3_h_offs;
+     __u16 multiwindow3_v_size;
+     __u16 multiwindow3_h_size;
+     /* RAWAWB_EXC_WP_REGION */
+     __u16 exc_wp_region0_xu0;
+     __u16 exc_wp_region0_xu1;
+ 
+     __u16 exc_wp_region0_yv0;
+     __u16 exc_wp_region0_yv1;
+ 
+     __u16 exc_wp_region1_xu0;
+     __u16 exc_wp_region1_xu1;
+ 
+     __u16 exc_wp_region1_yv0;
+     __u16 exc_wp_region1_yv1;
+ 
+     __u16 exc_wp_region2_xu0;
+     __u16 exc_wp_region2_xu1;
+ 
+     __u16 exc_wp_region2_yv0;
+     __u16 exc_wp_region2_yv1;
+ 
+     __u16 exc_wp_region3_xu0;
+     __u16 exc_wp_region3_xu1;
+ 
+     __u16 exc_wp_region3_yv0;
+     __u16 exc_wp_region3_yv1;
+ 
+     __u16 exc_wp_region4_xu0;
+     __u16 exc_wp_region4_xu1;
+ 
+     __u16 exc_wp_region4_yv0;
+     __u16 exc_wp_region4_yv1;
+ 
+     __u16 exc_wp_region5_xu0;
+     __u16 exc_wp_region5_xu1;
+ 
+     __u16 exc_wp_region5_yv0;
+     __u16 exc_wp_region5_yv1;
+ 
+     __u16 exc_wp_region6_xu0;
+     __u16 exc_wp_region6_xu1;
+ 
+     __u16 exc_wp_region6_yv0;
+     __u16 exc_wp_region6_yv1;
+     /* RAWAWB_YUV_RGB2ROTY */
+     __u16 rgb2ryuvmat0_y;
+     __u16 rgb2ryuvmat1_y;
+     __u16 rgb2ryuvmat2_y;
+     __u16 rgb2ryuvofs_y;
+     /* RAWAWB_YUV_RGB2ROTU */
+     __u16 rgb2ryuvmat0_u;
+     __u16 rgb2ryuvmat1_u;
+     __u16 rgb2ryuvmat2_u;
+     __u16 rgb2ryuvofs_u;
+     /* RAWAWB_YUV_RGB2ROTV */
+     __u16 rgb2ryuvmat0_v;
+     __u16 rgb2ryuvmat1_v;
+     __u16 rgb2ryuvmat2_v;
+     __u16 rgb2ryuvofs_v;
+     /* RAWAWB_YUV_X_COOR */
+     __u16 coor_x1_ls0_y;
+     __u16 vec_x21_ls0_y;
+     __u16 coor_x1_ls0_u;
+     __u16 vec_x21_ls0_u;
+     __u16 coor_x1_ls0_v;
+     __u16 vec_x21_ls0_v;
+ 
+     __u16 coor_x1_ls1_y;
+     __u16 vec_x21_ls1_y;
+     __u16 coor_x1_ls1_u;
+     __u16 vec_x21_ls1_u;
+     __u16 coor_x1_ls1_v;
+     __u16 vec_x21_ls1_v;
+ 
+     __u16 coor_x1_ls2_y;
+     __u16 vec_x21_ls2_y;
+     __u16 coor_x1_ls2_u;
+     __u16 vec_x21_ls2_v;
+     __u16 coor_x1_ls2_v;
+     __u16 vec_x21_ls2_u;
+ 
+     __u16 coor_x1_ls3_y;
+     __u16 vec_x21_ls3_y;
+     __u16 coor_x1_ls3_u;
+     __u16 vec_x21_ls3_u;
+     __u16 coor_x1_ls3_v;
+     __u16 vec_x21_ls3_v;
+     /* RAWAWB_YUV_INTERP_CURVE_TH */
+     __u16 th0_ls0;
+     __u16 th1_ls0;
+     __u16 th2_ls0;
+     __u16 th3_ls0;
+     __u16 th4_ls0;
+     __u16 th5_ls0;
+ 
+     __u16 th0_ls1;
+     __u16 th1_ls1;
+     __u16 th2_ls1;
+     __u16 th3_ls1;
+     __u16 th4_ls1;
+     __u16 th5_ls1;
+ 
+     __u16 th0_ls2;
+     __u16 th1_ls2;
+     __u16 th2_ls2;
+     __u16 th3_ls2;
+     __u16 th4_ls2;
+     __u16 th5_ls2;
+ 
+     __u16 th0_ls3;
+     __u16 th1_ls3;
+     __u16 th2_ls3;
+     __u16 th3_ls3;
+     __u16 th4_ls3;
+     __u16 th5_ls3;
+     /* RAWAWB_UV_DETC_ISLOPE */
+     __u32 islope01_0;
+     __u32 islope12_0;
+     __u32 islope23_0;
+     __u32 islope30_0;
+     __u32 islope01_1;
+     __u32 islope12_1;
+     __u32 islope23_1;
+     __u32 islope30_1;
+     __u32 islope01_2;
+     __u32 islope12_2;
+     __u32 islope23_2;
+     __u32 islope30_2;
+     __u32 islope01_3;
+     __u32 islope12_3;
+     __u32 islope23_3;
+     __u32 islope30_3;
+ 
+     /* WIN_WEIGHT for isp32_lite */
+     __u32 win_weight[ISP32L_RAWAWB_WEIGHT_NUM];
+     struct isp2x_bls_fixed_val bls2_val;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawaf_meas_cfg {
+     __u8 rawaf_sel;
+     __u8 num_afm_win;
+     /* for isp32_lite */
+     __u8 bnr2af_sel;
+ 
+     /* CTRL */
+     __u8 gamma_en;
+     __u8 gaus_en;
+     __u8 v1_fir_sel;
+     __u8 hiir_en;
+     __u8 viir_en;
+     __u8 accu_8bit_mode;
+     __u8 ldg_en;
+     __u8 h1_fv_mode;
+     __u8 h2_fv_mode;
+     __u8 v1_fv_mode;
+     __u8 v2_fv_mode;
+     __u8 ae_mode;
+     __u8 y_mode;
+     __u8 vldg_sel;
+     __u8 sobel_sel;
+     __u8 v_dnscl_mode;
+     __u8 from_awb;
+     __u8 from_ynr;
+     __u8 ae_config_use;
+     /* for isp32_lite */
+     __u8 ae_sel;
+ 
+     /* for isp32_lite */
+     __u8 hiir_left_border_mode;
+     __u8 avg_ds_en;
+     __u8 avg_ds_mode;
+     __u8 h1_acc_mode;
+     __u8 h2_acc_mode;
+     __u8 v1_acc_mode;
+     __u8 v2_acc_mode;
+ 
+     /* CTRL1 for isp32_lite */
+     __s16 bls_offset;
+     __u8 bls_en;
+     __u8 hldg_dilate_num;
+ 
+     /* WINA_B */
+     struct isp2x_window win[ISP32_RAWAF_WIN_NUM];
+     /* INT_LINE */
+     __u8 line_num[ISP32_RAWAF_LINE_NUM];
+     __u8 line_en[ISP32_RAWAF_LINE_NUM];
+     /* THRES */
+     __u16 afm_thres;
+     /* VAR_SHIFT */
+     __u8 afm_var_shift[ISP32_RAWAF_WIN_NUM];
+     __u8 lum_var_shift[ISP32_RAWAF_WIN_NUM];
+     /* for isp32_lite */
+     __u8 tnrin_shift;
+ 
+     /* HVIIR_VAR_SHIFT */
+     __u8 h1iir_var_shift;
+     __u8 h2iir_var_shift;
+     __u8 v1iir_var_shift;
+     __u8 v2iir_var_shift;
+     /* GAUS_COE */
+     __s8 gaus_coe[ISP32_RAWAF_GAUS_COE_NUM];
+ 
+     /* GAMMA_Y */
+     __u16 gamma_y[ISP32_RAWAF_GAMMA_NUM];
+     /* HIIR_THRESH */
+     __u16 h_fv_thresh;
+     __u16 v_fv_thresh;
+     struct isp3x_rawaf_curve curve_h[ISP32_RAWAF_CURVE_NUM];
+     struct isp3x_rawaf_curve curve_v[ISP32_RAWAF_CURVE_NUM];
+     __s16 h1iir1_coe[ISP32_RAWAF_HIIR_COE_NUM];
+     __s16 h1iir2_coe[ISP32_RAWAF_HIIR_COE_NUM];
+     __s16 h2iir1_coe[ISP32_RAWAF_HIIR_COE_NUM];
+     __s16 h2iir2_coe[ISP32_RAWAF_HIIR_COE_NUM];
+     __s16 v1iir_coe[ISP32_RAWAF_VIIR_COE_NUM];
+     __s16 v2iir_coe[ISP32_RAWAF_VIIR_COE_NUM];
+     __s16 v1fir_coe[ISP32_RAWAF_VFIR_COE_NUM];
+     __s16 v2fir_coe[ISP32_RAWAF_VFIR_COE_NUM];
+     __u16 highlit_thresh;
+ 
+     /* CORING_H for isp32_lite */
+     __u16 h_fv_limit;
+     __u16 h_fv_slope;
+     /* CORING_V for isp32_lite */
+     __u16 v_fv_limit;
+     __u16 v_fv_slope;
+ } __attribute__ ((packed));
+ 
+ struct isp32_cac_cfg {
+     __u8 bypass_en;
+     __u8 center_en;
+     __u8 clip_g_mode;
+     __u8 edge_detect_en;
+     __u8 neg_clip0_en;
+ 
+     __u8 flat_thed_b;
+     __u8 flat_thed_r;
+ 
+     __u8 psf_sft_bit;
+     __u16 cfg_num;
+ 
+     __u16 center_width;
+     __u16 center_height;
+ 
+     __u16 strength[ISP32_CAC_STRENGTH_NUM];
+ 
+     __u16 offset_b;
+     __u16 offset_r;
+ 
+     __u32 expo_thed_b;
+     __u32 expo_thed_r;
+     __u32 expo_adj_b;
+     __u32 expo_adj_r;
+ 
+     __u32 hsize;
+     __u32 vsize;
+     __s32 buf_fd;
+ } __attribute__ ((packed));
+ 
+ struct isp32_vsm_cfg {
+     __u8 h_segments;
+     __u8 v_segments;
+     __u16 h_offs;
+     __u16 v_offs;
+     __u16 h_size;
+     __u16 v_size;
+ } __attribute__ ((packed));
+ 
+ struct isp32_isp_other_cfg {
+     struct isp32_bls_cfg bls_cfg;
+     struct isp2x_dpcc_cfg dpcc_cfg;
+     struct isp3x_lsc_cfg lsc_cfg;
+     struct isp32_awb_gain_cfg awb_gain_cfg;
+     struct isp21_gic_cfg gic_cfg;
+     struct isp32_debayer_cfg debayer_cfg;
+     struct isp32_ccm_cfg ccm_cfg;
+     struct isp3x_gammaout_cfg gammaout_cfg;
+     struct isp2x_cproc_cfg cproc_cfg;
+     struct isp2x_ie_cfg ie_cfg;
+     struct isp2x_sdg_cfg sdg_cfg;
+     struct isp32_drc_cfg drc_cfg;
+     struct isp32_hdrmge_cfg hdrmge_cfg;
+     struct isp32_dhaz_cfg dhaz_cfg;
+     struct isp2x_3dlut_cfg isp3dlut_cfg;
+     struct isp32_ldch_cfg ldch_cfg;
+     struct isp32_baynr_cfg baynr_cfg;
+     struct isp32_bay3d_cfg bay3d_cfg;
+     struct isp32_ynr_cfg ynr_cfg;
+     struct isp32_cnr_cfg cnr_cfg;
+     struct isp32_sharp_cfg sharp_cfg;
+     struct isp32_cac_cfg cac_cfg;
+     struct isp3x_gain_cfg gain_cfg;
+     struct isp21_csm_cfg csm_cfg;
+     struct isp21_cgc_cfg cgc_cfg;
+     struct isp32_vsm_cfg vsm_cfg;
+ } __attribute__ ((packed));
+ 
+ struct isp32_isp_meas_cfg {
+     struct isp32_rawaf_meas_cfg rawaf;
+     struct isp32_rawawb_meas_cfg rawawb;
+     struct isp2x_rawaelite_meas_cfg rawae0;
+     struct isp2x_rawaebig_meas_cfg rawae1;
+     struct isp2x_rawaebig_meas_cfg rawae2;
+     struct isp2x_rawaebig_meas_cfg rawae3;
+     struct isp2x_rawhistlite_cfg rawhist0;
+     struct isp2x_rawhistbig_cfg rawhist1;
+     struct isp2x_rawhistbig_cfg rawhist2;
+     struct isp2x_rawhistbig_cfg rawhist3;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawae_meas_data {
+     __u32 channelg_xy:12;
+     __u32 channelb_xy:10;
+     __u32 channelr_xy:10;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawaebig_stat0 {
+     struct isp32_rawae_meas_data data[ISP32_RAWAEBIG_MEAN_NUM];
+     __u32 reserved[3];
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawaebig_stat1 {
+     __u32 sumr[ISP32_RAWAEBIG_SUBWIN_NUM];
+     __u32 sumg[ISP32_RAWAEBIG_SUBWIN_NUM];
+     __u32 sumb[ISP32_RAWAEBIG_SUBWIN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawaelite_stat {
+     struct isp32_rawae_meas_data data[ISP32_RAWAELITE_MEAN_NUM];
+     __u32 reserved[21];
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawaf_stat {
+     struct isp3x_rawaf_ramdata ramdata[ISP32_RAWAF_SUMDATA_NUM];
+     __u32 int_state;
+     __u32 afm_sum_b;
+     __u32 afm_lum_b;
+     __u32 highlit_cnt_winb;
+     __u32 reserved[18];
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawawb_ramdata {
+     __u64 b:18;
+     __u64 g:18;
+     __u64 r:18;
+     __u64 wp:10;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawawb_sum {
+     __u32 rgain_nor;
+     __u32 bgain_nor;
+     __u32 wp_num_nor;
+     __u32 wp_num2;
+ 
+     __u32 rgain_big;
+     __u32 bgain_big;
+     __u32 wp_num_big;
+     __u32 reserved;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawawb_sum_exc {
+     __u32 rgain_exc;
+     __u32 bgain_exc;
+     __u32 wp_num_exc;
+     __u32 reserved;
+ } __attribute__ ((packed));
+ 
+ struct isp32_rawawb_meas_stat {
+     struct isp32_rawawb_ramdata ramdata[ISP32_RAWAWB_RAMDATA_NUM];
+     __u64 reserved;
+     struct isp32_rawawb_sum sum[ISP32_RAWAWB_SUM_NUM];
+     __u16 yhist_bin[ISP32_RAWAWB_HSTBIN_NUM];
+     struct isp32_rawawb_sum_exc sum_exc[ISP32_RAWAWB_EXCL_STAT_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_vsm_stat {
+     __u16 delta_h;
+     __u16 delta_v;
+ } __attribute__ ((packed));
+ 
+ struct isp32_info2ddr_stat {
+     __u32 owner;
+     __s32 buf_fd;
+ } __attribute__ ((packed));
+ 
+ struct isp32_isp_params_cfg {
+     __u64 module_en_update;
+     __u64 module_ens;
+     __u64 module_cfg_update;
+ 
+     __u32 frame_id;
+     struct isp32_isp_meas_cfg meas;
+     struct isp32_isp_other_cfg others;
+ } __attribute__ ((packed));
+ 
+ struct isp32_stat {
+     struct isp32_rawaebig_stat0 rawae3_0;	/* offset 0 */
+     struct isp32_rawaebig_stat0 rawae1_0;	/* offset 0x390 */
+     struct isp32_rawaebig_stat0 rawae2_0;	/* offset 0x720 */
+     struct isp32_rawaelite_stat rawae0;	/* offset 0xab0 */
+     struct isp32_rawaebig_stat1 rawae3_1;
+     struct isp32_rawaebig_stat1 rawae1_1;
+     struct isp32_rawaebig_stat1 rawae2_1;
+     struct isp2x_bls_stat bls;
+     struct isp2x_rawhistbig_stat rawhist3;	/* offset 0xc00 */
+     struct isp2x_rawhistlite_stat rawhist0;	/* offset 0x1000 */
+     struct isp2x_rawhistbig_stat rawhist1;	/* offset 0x1400 */
+     struct isp2x_rawhistbig_stat rawhist2;	/* offset 0x1800 */
+     struct isp32_rawaf_stat rawaf;		/* offset 0x1c00 */
+     struct isp3x_dhaz_stat dhaz;
+     struct isp32_vsm_stat vsm;
+     struct isp32_info2ddr_stat info2ddr;
+     struct isp32_rawawb_meas_stat rawawb;	/* offset 0x2b00 */
+ } __attribute__ ((packed));
+ 
+ /**
+  * struct esisp32_isp_stat_buffer - ESWIN ISP32 Statistics Meta Data
+  *
+  * @meas_type: measurement types (ISP3X_STAT_ definitions)
+  * @frame_id: frame ID for sync
+  * @params: statistics data
+  */
+ struct esisp32_isp_stat_buffer {
+     struct isp32_stat params;
+     __u32 meas_type;
+     __u32 frame_id;
+     __u32 params_id;
+ } __attribute__ ((packed));
+ 
+ struct esisp32_thunderboot_resmem_head {
+     struct esisp_thunderboot_resmem_head head;
+     struct isp32_isp_params_cfg cfg;
+ } __attribute__ ((packed));
+ 
+ /****************isp32 lite********************/
+ 
+ struct isp32_lite_rawaebig_stat {
+     __u32 sumr;
+     __u32 sumg;
+     __u32 sumb;
+     struct isp2x_rawae_meas_data data[ISP32_RAWAEBIG_MEAN_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_lite_rawawb_meas_stat {
+     __u32 ramdata_r[ISP32L_RAWAWB_RAMDATA_RGB_NUM];
+     __u32 ramdata_g[ISP32L_RAWAWB_RAMDATA_RGB_NUM];
+     __u32 ramdata_b[ISP32L_RAWAWB_RAMDATA_RGB_NUM];
+     __u32 ramdata_wpnum0[ISP32L_RAWAWB_RAMDATA_WP_NUM];
+     __u32 ramdata_wpnum1[ISP32L_RAWAWB_RAMDATA_WP_NUM];
+     struct isp32_rawawb_sum sum[ISP32_RAWAWB_SUM_NUM];
+     __u16 yhist_bin[ISP32_RAWAWB_HSTBIN_NUM];
+     struct isp32_rawawb_sum_exc sum_exc[ISP32_RAWAWB_EXCL_STAT_NUM];
+ } __attribute__ ((packed));
+ 
+ struct isp32_lite_rawaf_ramdata {
+     __u32 hiir_wnd_data[ISP32L_RAWAF_WND_DATA];
+     __u32 viir_wnd_data[ISP32L_RAWAF_WND_DATA];
+ } __attribute__ ((packed));
+ 
+ struct isp32_lite_rawaf_stat {
+     struct isp32_lite_rawaf_ramdata ramdata;
+     __u32 int_state;
+     __u32 afm_sum_b;
+     __u32 afm_lum_b;
+     __u32 highlit_cnt_winb;
+ } __attribute__ ((packed));
+ 
+ struct isp32_lite_rawhistlite_stat {
+     __u32 hist_bin[ISP32L_HIST_LITE_BIN_N_MAX];
+ } __attribute__ ((packed));
+ 
+ struct isp32_lite_stat {
+     struct isp2x_bls_stat bls;
+     struct isp3x_dhaz_stat dhaz;
+     struct isp32_info2ddr_stat info2ddr;
+     struct isp2x_rawaelite_stat rawae0;
+     struct isp32_lite_rawaebig_stat rawae3;
+     struct isp32_lite_rawhistlite_stat rawhist0;
+     struct isp2x_rawhistbig_stat rawhist3;
+     struct isp32_lite_rawaf_stat rawaf;
+     struct isp32_lite_rawawb_meas_stat rawawb;
+ } __attribute__ ((packed));
+ 
+ struct esisp32_lite_stat_buffer {
+     struct isp32_lite_stat params;
+     __u32 meas_type;
+     __u32 frame_id;
+     __u32 params_id;
+ } __attribute__ ((packed));
+ #endif /* _UAPI_RK_ISP32_CONFIG_H */
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/hw.c b/drivers/media/platform/eswin/dvp2axi/hw.c
new file mode 100644
index 000000000000..d999d6a4e621
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/hw.c
@@ -0,0 +1,574 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI hw driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/reset.h>
+#include <linux/pm_runtime.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/regmap.h>
+// #include <media/videobuf2-cma-sg.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include <media/v4l2-fwnode.h>
+#include <linux/iommu.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include "common.h"
+
+#include <linux/bitfield.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+
+/* eic770x */
+#include <media/eswin/common-def.h>
+#include "../vi_top/vitop.h"
+#include "dvp2axi.h"
+
+#define AWSMMUSID	GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID	GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID	GENMASK(15, 8)	// The sid of read operation
+#define ARSMMUSSID	GENMASK(7, 0)	// The ssid of read operation
+
+u32 devm_irq_num[6] = {0};
+static inline void DVP2AXI_HalWriteReg(struct es_dvp2axi_hw *dvp2axi_hw, u32 address, u32 data) {
+    writel(data, dvp2axi_hw->base_addr + address);
+}
+static inline u32 DVP2AXI_HalReadReg(struct es_dvp2axi_hw *dvp2axi_hw, u32 address) {
+    u32 val;
+    val = readl(dvp2axi_hw->base_addr + address);
+    return val;
+}
+
+static const char * const eic770x_dvp2axi_clks[] = {
+	"aclk_dvp2axi",
+	"hclk_dvp2axi",
+	"dclk_dvp2axi",
+	"iclk_host0",
+	"iclk_host1",
+};
+
+static const char * const eic770x_dvp2axi_rsts[] = {
+	"rst_dvp2axi_a",
+	"rst_dvp2axi_h",
+	"rst_dvp2axi_d",
+	"rst_dvp2axi_host0",
+	"rst_dvp2axi_host1",
+	"rst_dvp2axi_host2",
+	"rst_dvp2axi_host3",
+	"rst_dvp2axi_host4",
+	"rst_dvp2axi_host5",
+};
+
+static const struct dvp2axi_reg eic770x_dvp2axi_regs[] = {
+	[DVP2AXI_REG_DVP_CTRL] = DVP2AXI_REG(DVP_CTRL),
+	[DVP2AXI_REG_DVP_INTEN] = DVP2AXI_REG(DVP_INTEN),
+	[DVP2AXI_REG_DVP_INTSTAT] = DVP2AXI_REG(DVP_INTSTAT),
+	[DVP2AXI_REG_DVP_FOR] = DVP2AXI_REG(DVP_FOR),
+	[DVP2AXI_REG_DVP_MULTI_ID] = DVP2AXI_REG(DVP_MULTI_ID),
+	[DVP2AXI_REG_DVP_SAV_EAV] = DVP2AXI_REG(DVP_SAV_EAV),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_Y] = DVP2AXI_REG(DVP_FRM0_ADDR_Y_ID0),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_UV] = DVP2AXI_REG(DVP_FRM0_ADDR_UV_ID0),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_Y] = DVP2AXI_REG(DVP_FRM1_ADDR_Y_ID0),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_UV] = DVP2AXI_REG(DVP_FRM1_ADDR_UV_ID0),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID1] = DVP2AXI_REG(DVP_FRM0_ADDR_Y_ID1),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID1] = DVP2AXI_REG(DVP_FRM0_ADDR_UV_ID1),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID1] = DVP2AXI_REG(DVP_FRM1_ADDR_Y_ID1),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID1] = DVP2AXI_REG(DVP_FRM1_ADDR_UV_ID1),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID2] = DVP2AXI_REG(DVP_FRM0_ADDR_Y_ID2),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID2] = DVP2AXI_REG(DVP_FRM0_ADDR_UV_ID2),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID2] = DVP2AXI_REG(DVP_FRM1_ADDR_Y_ID2),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID2] = DVP2AXI_REG(DVP_FRM1_ADDR_UV_ID2),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID3] = DVP2AXI_REG(DVP_FRM0_ADDR_Y_ID3),
+	[DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID3] = DVP2AXI_REG(DVP_FRM0_ADDR_UV_ID3),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID3] = DVP2AXI_REG(DVP_FRM1_ADDR_Y_ID3),
+	[DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID3] = DVP2AXI_REG(DVP_FRM1_ADDR_UV_ID3),
+	[DVP2AXI_REG_DVP_VIR_LINE_WIDTH] = DVP2AXI_REG(DVP_VIR_LINE_WIDTH),
+	[DVP2AXI_REG_DVP_SET_SIZE] = DVP2AXI_REG(DVP_CROP_SIZE),
+	[DVP2AXI_REG_DVP_CROP] = DVP2AXI_REG(DVP_CROP),
+	[DVP2AXI_REG_DVP_LINE_INT_NUM] = DVP2AXI_REG(DVP_LINE_INT_NUM_01),
+	[DVP2AXI_REG_DVP_LINE_INT_NUM1] = DVP2AXI_REG(DVP_LINE_INT_NUM_23),
+	[DVP2AXI_REG_DVP_LINE_CNT] = DVP2AXI_REG(DVP_LINE_INT_NUM_01),
+	[DVP2AXI_REG_DVP_LINE_CNT1] = DVP2AXI_REG(DVP_LINE_INT_NUM_23),
+
+	[DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0] = DVP2AXI_REG(CSI_MIPI0_ID0_CTRL0),
+	[DVP2AXI_REG_MIPI_LVDS_ID0_CTRL1] = DVP2AXI_REG(CSI_MIPI0_ID0_CTRL1),
+	[DVP2AXI_REG_MIPI_LVDS_ID1_CTRL0] = DVP2AXI_REG(CSI_MIPI0_ID1_CTRL0),
+	[DVP2AXI_REG_MIPI_LVDS_ID1_CTRL1] = DVP2AXI_REG(CSI_MIPI0_ID1_CTRL1),
+	[DVP2AXI_REG_MIPI_LVDS_ID2_CTRL0] = DVP2AXI_REG(CSI_MIPI0_ID2_CTRL0),
+	[DVP2AXI_REG_MIPI_LVDS_ID2_CTRL1] = DVP2AXI_REG(CSI_MIPI0_ID2_CTRL1),
+	[DVP2AXI_REG_MIPI_LVDS_ID3_CTRL0] = DVP2AXI_REG(CSI_MIPI0_ID3_CTRL0),
+	[DVP2AXI_REG_MIPI_LVDS_ID3_CTRL1] = DVP2AXI_REG(CSI_MIPI0_ID3_CTRL1),
+	[DVP2AXI_REG_MIPI_LVDS_CTRL] = DVP2AXI_REG(CSI_MIPI0_CTRL),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID0] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_Y_ID0),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID0] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_Y_ID0),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID0] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_UV_ID0),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID0] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_UV_ID0),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID0] = DVP2AXI_REG(CSI_MIPI0_VLW_ID0),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID1] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_Y_ID1),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID1] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_Y_ID1),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID1] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_UV_ID1),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID1] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_UV_ID1),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID1] = DVP2AXI_REG(CSI_MIPI0_VLW_ID1),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID2] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_Y_ID2),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID2] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_Y_ID2),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID2] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_UV_ID2),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID2] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_UV_ID2),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID2] = DVP2AXI_REG(CSI_MIPI0_VLW_ID2),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID3] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_Y_ID3),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID3] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_Y_ID3),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID3] = DVP2AXI_REG(CSI_MIPI0_FRM0_ADDR_UV_ID3),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID3] = DVP2AXI_REG(CSI_MIPI0_FRM1_ADDR_UV_ID3),
+	[DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID3] = DVP2AXI_REG(CSI_MIPI0_VLW_ID3),
+	[DVP2AXI_REG_MIPI_LVDS_INTEN] = DVP2AXI_REG(CSI_MIPI0_INTEN),
+	[DVP2AXI_REG_MIPI_LVDS_INTSTAT] = DVP2AXI_REG(CSI_MIPI0_INTSTAT),
+	[DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID0_1] = DVP2AXI_REG(CSI_MIPI0_LINE_INT_NUM_ID0_1),
+	[DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID2_3] = DVP2AXI_REG(CSI_MIPI0_LINE_INT_NUM_ID2_3),
+	[DVP2AXI_REG_MIPI_LVDS_LINE_LINE_CNT_ID0_1] = DVP2AXI_REG(CSI_MIPI0_LINE_CNT_ID0_1),
+	[DVP2AXI_REG_MIPI_LVDS_LINE_LINE_CNT_ID2_3] = DVP2AXI_REG(CSI_MIPI0_LINE_CNT_ID2_3),
+	[DVP2AXI_REG_MIPI_LVDS_ID0_CROP_START] = DVP2AXI_REG(CSI_MIPI0_ID0_CROP_START),
+	[DVP2AXI_REG_MIPI_LVDS_ID1_CROP_START] = DVP2AXI_REG(CSI_MIPI0_ID1_CROP_START),
+	[DVP2AXI_REG_MIPI_LVDS_ID2_CROP_START] = DVP2AXI_REG(CSI_MIPI0_ID2_CROP_START),
+	[DVP2AXI_REG_MIPI_LVDS_ID3_CROP_START] = DVP2AXI_REG(CSI_MIPI0_ID3_CROP_START),
+	[DVP2AXI_REG_MIPI_FRAME_NUM_VC0] = DVP2AXI_REG(CSI_MIPI0_FRAME_NUM_VC0),
+	[DVP2AXI_REG_MIPI_FRAME_NUM_VC1] = DVP2AXI_REG(CSI_MIPI0_FRAME_NUM_VC1),
+	[DVP2AXI_REG_MIPI_FRAME_NUM_VC2] = DVP2AXI_REG(CSI_MIPI0_FRAME_NUM_VC2),
+	[DVP2AXI_REG_MIPI_FRAME_NUM_VC3] = DVP2AXI_REG(CSI_MIPI0_FRAME_NUM_VC3),
+	[DVP2AXI_REG_MIPI_EFFECT_CODE_ID0] = DVP2AXI_REG(CSI_MIPI0_EFFECT_CODE_ID0),
+	[DVP2AXI_REG_MIPI_EFFECT_CODE_ID1] = DVP2AXI_REG(CSI_MIPI0_EFFECT_CODE_ID1),
+	[DVP2AXI_REG_MIPI_EFFECT_CODE_ID2] = DVP2AXI_REG(CSI_MIPI0_EFFECT_CODE_ID2),
+	[DVP2AXI_REG_MIPI_EFFECT_CODE_ID3] = DVP2AXI_REG(CSI_MIPI0_EFFECT_CODE_ID3),
+	[DVP2AXI_REG_MIPI_FRAME_SIZE_ID0] = DVP2AXI_REG(CSI_MIPI0_FRAME_SIZE_ID0),
+	[DVP2AXI_REG_MIPI_FRAME_SIZE_ID1] = DVP2AXI_REG(CSI_MIPI0_FRAME_SIZE_ID1),
+	[DVP2AXI_REG_MIPI_FRAME_SIZE_ID2] = DVP2AXI_REG(CSI_MIPI0_FRAME_SIZE_ID2),
+	[DVP2AXI_REG_MIPI_FRAME_SIZE_ID3] = DVP2AXI_REG(CSI_MIPI0_FRAME_SIZE_ID3),
+	[DVP2AXI_REG_MIPI_ON_PAD] = DVP2AXI_REG(CSI_MIPI0_ON_PAD),
+
+	[DVP2AXI_REG_GLB_CTRL] = DVP2AXI_REG(GLB_CTRL),
+	[DVP2AXI_REG_GLB_INTEN] = DVP2AXI_REG(GLB_INTEN),
+	[DVP2AXI_REG_GLB_INTST] = DVP2AXI_REG(GLB_INTST),
+
+	[DVP2AXI_REG_SCL_CH_CTRL] = DVP2AXI_REG(SCL_CH_CTRL),
+	[DVP2AXI_REG_SCL_CTRL] = DVP2AXI_REG(SCL_CTRL),
+	[DVP2AXI_REG_SCL_FRM0_ADDR_CH0] = DVP2AXI_REG(SCL_FRM0_ADDR_CH0),
+	[DVP2AXI_REG_SCL_FRM1_ADDR_CH0] = DVP2AXI_REG(SCL_FRM1_ADDR_CH0),
+	[DVP2AXI_REG_SCL_VLW_CH0] = DVP2AXI_REG(SCL_VLW_CH0),
+	[DVP2AXI_REG_SCL_FRM0_ADDR_CH1] = DVP2AXI_REG(SCL_FRM0_ADDR_CH1),
+	[DVP2AXI_REG_SCL_FRM1_ADDR_CH1] = DVP2AXI_REG(SCL_FRM1_ADDR_CH1),
+	[DVP2AXI_REG_SCL_VLW_CH1] = DVP2AXI_REG(SCL_VLW_CH1),
+	[DVP2AXI_REG_SCL_FRM0_ADDR_CH2] = DVP2AXI_REG(SCL_FRM0_ADDR_CH2),
+	[DVP2AXI_REG_SCL_FRM1_ADDR_CH2] = DVP2AXI_REG(SCL_FRM1_ADDR_CH2),
+	[DVP2AXI_REG_SCL_VLW_CH2] = DVP2AXI_REG(SCL_VLW_CH2),
+	[DVP2AXI_REG_SCL_FRM0_ADDR_CH3] = DVP2AXI_REG(SCL_FRM0_ADDR_CH3),
+	[DVP2AXI_REG_SCL_FRM1_ADDR_CH3] = DVP2AXI_REG(SCL_FRM1_ADDR_CH3),
+	[DVP2AXI_REG_SCL_VLW_CH3] = DVP2AXI_REG(SCL_VLW_CH3),
+	[DVP2AXI_REG_SCL_BLC_CH0] = DVP2AXI_REG(SCL_BLC_CH0),
+	[DVP2AXI_REG_SCL_BLC_CH1] = DVP2AXI_REG(SCL_BLC_CH1),
+	[DVP2AXI_REG_SCL_BLC_CH2] = DVP2AXI_REG(SCL_BLC_CH2),
+	[DVP2AXI_REG_SCL_BLC_CH3] = DVP2AXI_REG(SCL_BLC_CH3),
+	[DVP2AXI_REG_TOISP0_CTRL] = DVP2AXI_REG(TOISP0_CH_CTRL),
+	[DVP2AXI_REG_TOISP0_SIZE] = DVP2AXI_REG(TOISP0_CROP_SIZE),
+	[DVP2AXI_REG_TOISP0_CROP] = DVP2AXI_REG(TOISP0_CROP),
+	[DVP2AXI_REG_TOISP1_CTRL] = DVP2AXI_REG(TOISP1_CH_CTRL),
+	[DVP2AXI_REG_TOISP1_SIZE] = DVP2AXI_REG(TOISP1_CROP_SIZE),
+	[DVP2AXI_REG_TOISP1_CROP] = DVP2AXI_REG(TOISP1_CROP),
+	[DVP2AXI_REG_GRF_DVP2AXIIO_CON] = DVP2AXI_REG(DVP2AXI_GRF_SOC_CON2),
+};
+
+
+static const struct of_device_id es_dvp2axi_plat_of_match[] = {
+	{
+		.compatible = "eswin,dvp2axi",
+	},
+	{},
+};
+
+static irqreturn_t es_dvp2axi_irq_handler(int irq, void *ctx)
+{
+	struct device *dev = ctx;
+	struct es_dvp2axi_hw *dvp2axi_hw = dev_get_drvdata(dev);
+	// unsigned int intstat_glb = 0;
+	u64 irq_start, irq_stop;
+	// int i;
+
+	irq_start = ktime_get_ns();
+	for(int i = 0; i < 6; i++) {
+		if(irq == dvp2axi_hw->devm_irq_num[i]) {
+			es_irq_oneframe(dev, dvp2axi_hw->dvp2axi_dev[i]);
+		}
+	}
+
+	irq_stop = ktime_get_ns();
+	dvp2axi_hw->irq_time = irq_stop - irq_start;
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t es_dvp2axi_err_irq_handler(int irq, void *ctx)
+{
+	struct device *dev = ctx;
+	struct es_dvp2axi_hw *dvp2axi_hw = dev_get_drvdata(dev);
+	u64 irq_start, irq_stop;
+
+	irq_start = ktime_get_ns();
+	for(int i = 0; i < ES_DVP2AXI_IRQ_NUM; i++) {
+		if(irq == dvp2axi_hw->devm_irq_num[i]) {
+			es_irq_err_handle(dev, dvp2axi_hw->dvp2axi_dev[i]);
+		}
+	}
+
+	irq_stop = ktime_get_ns();
+	dvp2axi_hw->irq_time = irq_stop - irq_start;
+	return IRQ_HANDLED;
+}
+
+void es_dvp2axi_disable_sys_clk(struct es_dvp2axi_hw *dvp2axi_hw)
+{
+	int i;
+
+	for (i = dvp2axi_hw->clk_size - 1; i >= 0; i--)
+		clk_disable_unprepare(dvp2axi_hw->clks[i]);
+}
+
+int es_dvp2axi_enable_sys_clk(struct es_dvp2axi_hw *dvp2axi_hw)
+{
+	int i, ret = -EINVAL;
+
+	for (i = 0; i < dvp2axi_hw->clk_size; i++) {
+		ret = clk_prepare_enable(dvp2axi_hw->clks[i]);
+
+		if (ret < 0)
+			goto err;
+	}
+
+	write_dvp2axi_reg_and(dvp2axi_hw->base_addr, DVP2AXI_CSI_INTEN, 0x0);
+	return 0;
+
+err:
+	for (--i; i >= 0; --i)
+		clk_disable_unprepare(dvp2axi_hw->clks[i]);
+
+	return ret;
+}
+
+void es_dvp2axi_hw_soft_reset(struct es_dvp2axi_hw *dvp2axi_hw, bool is_rst_iommu)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dvp2axi_hw->dvp2axi_rst); i++)
+		if (dvp2axi_hw->dvp2axi_rst[i])
+			reset_control_assert(dvp2axi_hw->dvp2axi_rst[i]);
+	udelay(5);
+	for (i = 0; i < ARRAY_SIZE(dvp2axi_hw->dvp2axi_rst); i++)
+		if (dvp2axi_hw->dvp2axi_rst[i])
+			reset_control_deassert(dvp2axi_hw->dvp2axi_rst[i]);
+}
+
+/* CTRL1 */
+#define DVP2AXI_DVP0_PWIDTH 16
+#define DVP2AXI_DVP1_PWIDTH 16
+
+#define DVP2AXI_IO_DVP_DIS  0
+#define DVP2AXI_IO_DVP_ENA  1
+#define DVP2AXI_PIXEL_MODE_1WORD 0
+#define DVP2AXI_PIXEL_MODE_2WORD 1
+#define DVP2AXI_PIXEL_MODE_3WORD 2
+
+/* CTRL2 */
+#define DVP2AXI_DVP2_PWIDTH 16
+#define DVP2AXI_DVP3_PWIDTH 16
+#define DVP2AXI_DVP4_PWIDTH 16
+#define DVP2AXI_DVP5_PWIDTH 16
+#define DVP2AXI_OUTSTANDING_SIZE 16
+#define DVP2AXI_WQOS_CFG 0
+
+
+static int dvp2axi_smmu_sid_cfg(struct device* dev)
+{
+    int ret = 0;
+    struct regmap* regmap = NULL;
+    int mmu_tbu0_vi_dvp2axi_reg = 0;
+    u32 rdwr_sid_ssid = 0;
+    u32 sid = 0;
+	u32 val;
+
+    struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+    if (fwspec == NULL) {
+        pr_info("Device is not behind SMMU, using default streamID(0)\n");
+        return 0;
+    }
+
+	if (fwspec->num_ids == 0) {
+		dev_err(dev, "No Stream IDs configured!\n");
+		return -EINVAL;
+	}
+
+    sid = fwspec->ids[0];
+
+    regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,vi_top_csr");
+    if (IS_ERR(regmap)) {
+        pr_err("No vi_top_csr phandle specified, regmap=%ld\n", PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+    }
+
+    ret = of_property_read_u32_index(dev->of_node, "eswin,vi_top_csr", 1,
+                                    &mmu_tbu0_vi_dvp2axi_reg);
+    if (ret) {
+        pr_err("Failed to get sid cfg reg offset, ret=%d\n", ret);
+        return ret;
+    }
+
+    rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+    rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+    rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+    rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+
+    regmap_write(regmap, mmu_tbu0_vi_dvp2axi_reg, rdwr_sid_ssid);
+
+
+	regmap_read(regmap, mmu_tbu0_vi_dvp2axi_reg, &val);
+
+    ret = win2030_dynm_sid_enable(dev_to_node(dev));
+    if (ret < 0)
+        pr_err("Failed to enable dynamic SID for sid=%u, ret=%d\n", sid, ret);
+
+    return ret;
+}
+
+static int es_dvp2axi_plat_hw_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct es_dvp2axi_hw *dvp2axi_hw;
+	struct resource *res;
+	int ret, irq;
+#ifdef CONFIG_NUMA
+	u32 numa_id = 0;
+#endif
+
+	dvp2axi_hw = devm_kzalloc(dev, sizeof(*dvp2axi_hw), GFP_KERNEL);
+	if (!dvp2axi_hw)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, dvp2axi_hw);
+	dvp2axi_hw->dev = dev;
+
+	ret = dvp2axi_smmu_sid_cfg(dev);
+	if (ret) {
+		dev_err(dev, "SMMU SID config failed: %d\n", ret);
+		return ret;
+	}
+
+	for(int i = 0; i < ES_DVP2AXI_IRQ_NUM; i++) {
+		irq = platform_get_irq(pdev, i);
+		dvp2axi_hw->devm_irq_num[i] = irq;
+		if (irq < 0)
+			return irq;
+		if(i == ES_DVP2AXI_ERR_IRQ || i == ES_DVP2AXI_AFULL_IRQ) 
+			ret = devm_request_irq(dev, irq, es_dvp2axi_err_irq_handler,
+			       IRQF_SHARED,
+			       dev_driver_string(dev), dev);
+		else
+			ret = devm_request_irq(dev, irq, es_dvp2axi_irq_handler,
+			       IRQF_SHARED,
+			       dev_driver_string(dev), dev);
+	
+		if (ret < 0) {
+			dev_err(dev, "request irq failed: %d\n", ret);
+			return ret;
+		}
+	}
+
+	for(int i = 0; i < ES_DVP2AXI_ERRIRQ_NUM; i++)
+		atomic_set(&dvp2axi_hw->dvp2axi_errirq_cnts[i], 0);
+
+	dvp2axi_hw->irq = irq;
+
+	res = platform_get_resource_byname(pdev,
+		IORESOURCE_MEM,
+		"dvp2axi_regs");
+	dvp2axi_hw->base_addr = devm_ioremap_resource(dev, res);
+	if (PTR_ERR(dvp2axi_hw->base_addr) == -EBUSY) {
+		resource_size_t offset = res->start;
+		resource_size_t size = resource_size(res);
+
+		dvp2axi_hw->base_addr = devm_ioremap(dev, offset, size);
+		if (IS_ERR(dvp2axi_hw->base_addr)) {
+			dev_err(dev, "ioremap failed\n");
+			return PTR_ERR(dvp2axi_hw->base_addr);
+		}
+	}
+
+	// dvp2axi_hw->is_dma_sg_ops = true;
+	dvp2axi_hw->is_dma_sg_ops = false;
+	dvp2axi_hw->is_dma_contig = true;
+	mutex_init(&dvp2axi_hw->dev_lock);
+	mutex_init(&dvp2axi_hw->dev_multi_chn_lock);
+
+	spin_lock_init(&dvp2axi_hw->group_lock);
+	spin_lock_init(&dvp2axi_hw->intr_spinlock);
+	atomic_set(&dvp2axi_hw->power_cnt, 0);
+
+	pm_runtime_enable(&pdev->dev);
+
+#ifdef CONFIG_NUMA
+	ret = of_property_read_u32(dev->of_node, "numa-node-id", &numa_id);
+	if(ret) {
+		dev_warn(dev, "Could not get numa-node-id, use default 0\n");
+		numa_id = 0;
+	}
+#endif
+
+#ifdef CONFIG_NUMA
+	if(numa_id == 0) {
+		platform_driver_register(&es_dvp2axi_plat_drv);
+	} else {
+		platform_driver_register(&es_dvp2axi_plat_drv_d1);
+	}
+#else
+	platform_driver_register(&es_dvp2axi_plat_drv);
+#endif
+	return 0;
+}
+
+static int es_dvp2axi_plat_remove(struct platform_device *pdev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = platform_get_drvdata(pdev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	mutex_destroy(&dvp2axi_hw->dev_lock);
+	mutex_destroy(&dvp2axi_hw->dev_multi_chn_lock);
+
+	return 0;
+}
+
+static void es_dvp2axi_hw_shutdown(struct platform_device *pdev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = platform_get_drvdata(pdev);
+
+	if (pm_runtime_get_if_in_use(&pdev->dev) <= 0)
+		return;
+
+	write_dvp2axi_reg(dvp2axi_hw->base_addr, 0, 0);
+	if (dvp2axi_hw->irq > 0)
+		disable_irq(dvp2axi_hw->irq);
+	pm_runtime_put(&pdev->dev);
+}
+
+static int __maybe_unused es_dvp2axi_runtime_suspend(struct device *dev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev_get_drvdata(dev);
+
+	if (atomic_dec_return(&dvp2axi_hw->power_cnt))
+		return 0;
+	es_dvp2axi_disable_sys_clk(dvp2axi_hw);
+
+	return pinctrl_pm_select_sleep_state(dev);
+}
+
+static int __maybe_unused es_dvp2axi_runtime_resume(struct device *dev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev_get_drvdata(dev);
+	int ret;
+
+	if (atomic_inc_return(&dvp2axi_hw->power_cnt) > 1)
+		return 0;
+	ret = pinctrl_pm_select_default_state(dev);
+	if (ret < 0)
+		return ret;
+	es_dvp2axi_enable_sys_clk(dvp2axi_hw);
+	es_dvp2axi_hw_soft_reset(dvp2axi_hw, true);
+
+	return 0;
+}
+
+static int __maybe_unused es_dvp2axi_sleep_suspend(struct device *dev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev_get_drvdata(dev);
+
+	if (atomic_read(&dvp2axi_hw->power_cnt) == 0)
+		return 0;
+
+	es_dvp2axi_disable_sys_clk(dvp2axi_hw);
+
+	return pinctrl_pm_select_sleep_state(dev);
+}
+
+static int __maybe_unused es_dvp2axi_sleep_resume(struct device *dev)
+{
+	struct es_dvp2axi_hw *dvp2axi_hw = dev_get_drvdata(dev);
+	int ret;
+
+	if (atomic_read(&dvp2axi_hw->power_cnt) == 0)
+		return 0;
+
+	ret = pinctrl_pm_select_default_state(dev);
+	if (ret < 0)
+		return ret;
+	es_dvp2axi_enable_sys_clk(dvp2axi_hw);
+	es_dvp2axi_hw_soft_reset(dvp2axi_hw, true);
+
+	return 0;
+}
+
+static const struct dev_pm_ops es_dvp2axi_plat_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(es_dvp2axi_sleep_suspend,
+				es_dvp2axi_sleep_resume)
+	SET_RUNTIME_PM_OPS(es_dvp2axi_runtime_suspend, es_dvp2axi_runtime_resume, NULL)
+};
+
+static struct platform_driver es_dvp2axi_hw_plat_drv = {
+	.driver = {
+		.name = ES_DVP2AXI_HW_DRIVER_NAME,
+		.of_match_table = of_match_ptr(es_dvp2axi_plat_of_match),
+		.pm = &es_dvp2axi_plat_pm_ops,
+	},
+	.probe = es_dvp2axi_plat_hw_probe,
+	.remove = es_dvp2axi_plat_remove,
+	.shutdown = es_dvp2axi_hw_shutdown,
+};
+
+int es_dvp2axi_plat_drv_init(void)
+{
+	platform_driver_register(&es_dvp2axi_hw_plat_drv);
+	return 0;
+}
+
+static void __exit es_dvp2axi_plat_drv_exit(void)
+{
+	platform_driver_unregister(&es_dvp2axi_hw_plat_drv);
+}
+
+
+late_initcall(es_dvp2axi_plat_drv_init);
+module_exit(es_dvp2axi_plat_drv_exit);
+
+MODULE_AUTHOR("ES VI team");
+MODULE_DESCRIPTION("ES DVP2AXI platform driver");
+MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(DMA_BUF);
diff --git a/drivers/media/platform/eswin/dvp2axi/hw.h b/drivers/media/platform/eswin/dvp2axi/hw.h
new file mode 100644
index 000000000000..e85cceb94fc1
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/hw.h
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI hw driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_HW_H
+#define _ES_DVP2AXI_HW_H
+
+#include <linux/mutex.h>
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-mc.h>
+#include "../../../../phy/eswin/es-camera-module.h"
+#include "regs.h"
+#include "version.h"
+#include "dev.h"
+
+#define ES_DVP2AXI_DEV_MAX		7
+#define ES_DVP2AXI_HW_DRIVER_NAME	"es_dvp2axihw"
+#define ES_DVP2AXI_MAX_BUS_CLK	15
+#define ES_DVP2AXI_MAX_RESET		15
+
+#define ES_DVP2AXI_MAX_GROUP		4
+
+#define write_dvp2axi_reg(base, addr, val) \
+	writel(val, (addr) + (base))
+#define read_dvp2axi_reg(base, addr) \
+	readl((addr) + (base))
+#define write_dvp2axi_reg_or(base, addr, val) \
+	writel(readl((addr) + (base)) | (val), (addr) + (base))
+#define write_dvp2axi_reg_and(base, addr, val) \
+	writel(readl((addr) + (base)) & (val), (addr) + (base))
+
+/*
+ * multi sensor sync mode
+ * ES_DVP2AXI_NOSYNC_MODE: not used sync mode
+ * ES_DVP2AXI_MASTER_MASTER:	internal master->external master
+ * ES_DVP2AXI_MASTER_SLAVE:	internal master->slave
+ * ES_DVP2AXI_MASTER_MASTER: pwm/gpio->external master
+ * ES_DVP2AXI_MASTER_MASTER: pwm/gpio->slave
+ */
+enum es_dvp2axi_sync_mode {
+	ES_DVP2AXI_NOSYNC_MODE,
+	ES_DVP2AXI_MASTER_MASTER,
+	ES_DVP2AXI_MASTER_SLAVE,
+	ES_DVP2AXI_EXT_MASTER,
+	ES_DVP2AXI_EXT_SLAVE,
+};
+
+struct es_dvp2axi_sync_dev {
+	struct es_dvp2axi_device *dvp2axi_dev[ES_DVP2AXI_DEV_MAX];
+	int count;
+	bool is_streaming[ES_DVP2AXI_DEV_MAX];
+};
+
+struct es_dvp2axi_multi_sync_config {
+	struct es_dvp2axi_sync_dev int_master;
+	struct es_dvp2axi_sync_dev ext_master;
+	struct es_dvp2axi_sync_dev slave;
+	enum es_dvp2axi_sync_mode mode;
+	int dev_cnt;
+	int streaming_cnt;
+	u32 sync_code;
+	u32 sync_mask;
+	u32 update_code;
+	u32 update_cache;
+	u32 frame_idx;
+	bool is_attach;
+};
+
+struct es_dvp2axi_dummy_buffer {
+	struct vb2_buffer vb;
+	struct vb2_queue vb2_queue;
+	struct list_head list;
+	struct dma_buf *dbuf;
+	dma_addr_t dma_addr;
+	struct page **pages;
+	void *mem_priv;
+	void *vaddr;
+	u32 size;
+	int dma_fd;
+	bool is_need_vaddr;
+	bool is_need_dbuf;
+	bool is_need_dmafd;
+	bool is_free;
+};
+
+/*
+ * add new chip id in tail in time order
+ * by increasing to distinguish dvp2axi version
+ */
+enum es_dvp2axi_chip_id {
+	CHIP_EIC770X_DVP2AXI,
+};
+
+struct es_dvp2axi_hw_match_data {
+	int chip_id;
+	const char * const *clks;
+	const char * const *rsts;
+	int clks_num;
+	int rsts_num;
+	const struct dvp2axi_reg *dvp2axi_regs;
+};
+
+/*
+ * struct es_dvp2axi_device - ISP platform device
+ * @base_addr: base register address
+ * @active_sensor: sensor in-use, set when streaming on
+ * @stream: capture video device
+ */
+struct es_dvp2axi_hw {
+	struct device			*dev;
+	int				irq;
+	void __iomem			*base_addr;
+	void __iomem			*csi_base;
+	struct regmap			*grf;
+	struct clk			*clks[ES_DVP2AXI_MAX_BUS_CLK];
+	int				clk_size;
+	struct iommu_domain		*domain;
+	struct reset_control		*dvp2axi_rst[ES_DVP2AXI_MAX_RESET];
+	int				chip_id;
+	const struct dvp2axi_reg		*dvp2axi_regs;
+	const struct vb2_mem_ops	*mem_ops;
+	struct es_dvp2axi_device		*dvp2axi_dev[ES_DVP2AXI_DEV_MAX];
+	int				dev_num;
+	atomic_t			power_cnt;
+	const struct es_dvp2axi_hw_match_data *match_data;
+	struct mutex			dev_lock;
+	struct es_dvp2axi_multi_sync_config	sync_config[ES_DVP2AXI_MAX_GROUP];
+	spinlock_t			group_lock;
+	struct notifier_block		reset_notifier; /* reset for mipi csi crc err */
+	struct es_dvp2axi_dummy_buffer	dummy_buf;
+	bool				iommu_en;
+	bool				can_be_reset;
+	bool				is_dma_sg_ops;
+	bool				is_dma_contig;
+	bool				adapt_to_usbcamerahal;
+	u64				irq_time;
+	bool				is_eic770xs2;
+
+	/* eic770x_mipi_csi2_init */
+	u32 num_lanes:4;
+    u32 ppi_width:2;
+	u32 phy_mode:1;
+
+    u32 vc:5;
+    u32 dt:6;	
+	u32 emb:1;
+	u32 frame_det:1;
+
+    u32 ipi_mode:1;
+    u32 ipi_color_com:1;
+    u32 ipi_auto_flush:1;
+    u32 ipi_cut_through:1;
+    u32 ipi_line_event;
+
+    u32 hsa;
+	u32 hbp;
+	u32 hsd;
+	u32 htotal;
+
+    u32 vsa;
+	u32 vbp;
+	u32 vfp;
+	u32 vactive;
+};
+
+void es_dvp2axi_disable_sys_clk(struct es_dvp2axi_hw *dvp2axi_hw);
+int es_dvp2axi_enable_sys_clk(struct es_dvp2axi_hw *dvp2axi_hw);
+int es_dvp2axi_plat_drv_init(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/isp_external.h b/drivers/media/platform/eswin/dvp2axi/isp_external.h
new file mode 100644
index 000000000000..0dd2601bc3b0
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/isp_external.h
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI isp_external driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ESISP_EXTERNAL_H
+#define _ESISP_EXTERNAL_H
+
+#define ESISP_VICAP_CMD_MODE \
+	_IOWR('V', BASE_VIDIOC_PRIVATE + 0, struct esisp_vicap_mode)
+
+#define ESISP_VICAP_CMD_INIT_BUF \
+	 _IOW('V', BASE_VIDIOC_PRIVATE + 1, int)
+
+#define ESISP_VICAP_CMD_RX_BUFFER_FREE \
+	 _IOW('V', BASE_VIDIOC_PRIVATE + 2, struct esisp_rx_buf)
+
+#define ESISP_VICAP_CMD_QUICK_STREAM \
+	_IOW('V', BASE_VIDIOC_PRIVATE + 3, int)
+
+#define ESISP_VICAP_CMD_SET_RESET \
+	 _IOW('V', BASE_VIDIOC_PRIVATE + 4, int)
+
+#define ESISP_VICAP_CMD_SET_STREAM \
+	 _IOW('V', BASE_VIDIOC_PRIVATE + 5, int)
+
+#define ESISP_VICAP_BUF_CNT 3
+#define ESISP_VICAP_BUF_CNT_MAX 8
+#define ESISP_RX_BUF_POOL_MAX (ESISP_VICAP_BUF_CNT_MAX * 3)
+
+struct esisp_vicap_input {
+	u8 merge_num;
+	u8 index;
+};
+
+enum esisp_vicap_link {
+	ESISP_VICAP_ONLINE,
+	ESISP_VICAP_RDBK_AIQ,
+	ESISP_VICAP_RDBK_AUTO,
+	ESISP_VICAP_RDBK_AUTO_ONE_FRAME,
+};
+
+struct esisp_vicap_mode {
+	char *name;
+	enum esisp_vicap_link rdbk_mode;
+
+	struct esisp_vicap_input input;
+};
+
+enum rx_buf_type {
+	BUF_SHORT,
+	BUF_MIDDLE,
+	BUF_LONG,
+};
+
+struct esisp_rx_buf {
+	struct list_head list;
+	struct dma_buf *dbuf;
+	dma_addr_t dma;
+	u64 timestamp;
+	u32 sequence;
+	u32 type;
+	u32 runtime_us;
+
+	bool is_init;
+	bool is_first;
+
+	bool is_resmem;
+	bool is_switch;
+
+	bool is_uncompact;
+};
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/mipi-csi2.h b/drivers/media/platform/eswin/dvp2axi/mipi-csi2.h
new file mode 100644
index 000000000000..5e79116dd335
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/mipi-csi2.h
@@ -0,0 +1,458 @@
+#ifndef _ES_DVP2AXI_MIPI_CSI2_H_
+#define _ES_DVP2AXI_MIPI_CSI2_H_
+
+#include <linux/notifier.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-subdev.h>
+#include <media/v4l2-event.h>
+#include "../../../../phy/eswin/es-dvp2axi-config.h"
+
+#define CSI2_ERR_FSFE_MASK	(0xff << 8)
+#define CSI2_ERR_COUNT_ALL_MASK	(0xff)
+
+#define ES_DVP2AXI_V4L2_EVENT_ELEMS 4
+
+/*
+ * there must be 5 pads: 1 input pad from sensor, and
+ * the 4 virtual channel output pads
+ */
+#define CSI2_SINK_PAD			0
+#define CSI2_NUM_SINK_PADS		4
+#define CSI2_NUM_SRC_PADS		11
+#define CSI2_NUM_PADS			3
+#define CSI2_NUM_PADS_MAX		12
+#define CSI2_NUM_PADS_SINGLE_LINK	2
+#define MAX_CSI2_SENSORS		2
+
+// #define ES_DVP2AXI_DEFAULT_WIDTH 1920	//3280
+// #define ES_DVP2AXI_DEFAULT_HEIGHT 1080 //	2464
+
+#define CSI_ERRSTR_LEN		(256)
+#define CSI_VCINFO_LEN		(12)
+
+/*
+ * The default maximum bit-rate per lane in Mbps, if the
+ * source subdev does not provide V4L2_CID_LINK_FREQ.
+ */
+#define CSI2_DEFAULT_MAX_MBPS 849
+
+#define IMX_MEDIA_GRP_ID_CSI2      BIT(8)
+#define CSIHOST_MAX_ERRINT_COUNT	10
+
+#define DEVICE_NAME "eswin-mipi-csi2"
+#define DEVICE_NAME_HW "eswin-mipi-csi2-hw"
+
+/* CSI Host Registers Define */
+#define CSIHOST_N_LANES		0x04
+#define CSIHOST_DPHY_SHUTDOWNZ	0x08
+#define CSIHOST_PHY_RSTZ	0x0c
+#define CSIHOST_RESETN		0x10
+#define CSIHOST_PHY_STATE	0x14
+#define CSIHOST_ERR1		0x20
+#define CSIHOST_ERR2		0x24
+#define CSIHOST_MSK1		0x28
+#define CSIHOST_MSK2		0x2c
+#define CSIHOST_CONTROL		0x40
+
+#define CSIHOST_ERR1_PHYERR_SPTSYNCHS	0x0000000f
+#define CSIHOST_ERR1_ERR_BNDRY_MATCH	0x000000f0
+#define CSIHOST_ERR1_ERR_SEQ		0x00000f00
+#define CSIHOST_ERR1_ERR_FRM_DATA	0x0000f000
+#define CSIHOST_ERR1_ERR_CRC		0x0f000000
+#define CSIHOST_ERR1_ERR_ECC2		0x10000000
+#define CSIHOST_ERR1_ERR_CTRL		0x000f0000
+
+#define CSIHOST_ERR2_PHYERR_ESC		0x0000000f
+#define CSIHOST_ERR2_PHYERR_SOTHS	0x000000f0
+#define CSIHOST_ERR2_ECC_CORRECTED	0x00000f00
+#define CSIHOST_ERR2_ERR_ID		0x0000f000
+#define CSIHOST_ERR2_PHYERR_CODEHS	0x01000000
+
+
+#define DWC_MIPI_CSI2_VERSION			0x0
+
+#define DWC_MIPI_CSI2_N_LANES			0x4
+#define DWC_MIPI_CSI2_N_LANES_N_LANES(x)	((x) & 0x7)
+
+#define DWC_MIPI_CSI2_HOST_RESETN		0x8
+#define DWC_MIPI_CSI2_HOST_RESETN_ENABLE	(0x1)
+
+#define DWC_MIPI_CSI2_INT_ST_MAIN		0xC
+
+#define DWC_MIPI_CSI2_DATA_IDS_1		0x10
+#define DWC_MIPI_CSI2_DATA_IDS_1_DI0_DT(x)	(((x) & 0x3f))
+#define DWC_MIPI_CSI2_DATA_IDS_1_DI1_DT(x)	(((x) & 0x3f) << 8)
+#define DWC_MIPI_CSI2_DATA_IDS_1_DI2_DT(x)	(((x) & 0x3f) << 16)
+#define DWC_MIPI_CSI2_DATA_IDS_1_DI3_DT(x)	(((x) & 0x3f) << 24)
+
+#define DWC_MIPI_CSI2_DATA_IDS_2		0x14
+#define DWC_MIPI_CSI2_DATA_IDS_2_DI0_DT(x)	(((x) & 0x3f))
+#define DWC_MIPI_CSI2_DATA_IDS_2_DI1_DT(x)	(((x) & 0x3f) << 8)
+#define DWC_MIPI_CSI2_DATA_IDS_2_DI2_DT(x)	(((x) & 0x3f) << 16)
+#define DWC_MIPI_CSI2_DATA_IDS_2_DI3_DT(x)	(((x) & 0x3f) << 24)
+
+#define DWC_MIPI_CSI2_DPHY_CFG			0x18
+#define DWC_MIPI_CSI2_DPHY_CFG_PPI8		(0x0)
+#define DWC_MIPI_CSI2_DPHY_CFG_PPI16		(0x1)
+
+#define DWC_MIPI_CSI2_DPHY_MODE			0x1C
+#define DWC_MIPI_CSI2_DPHY_MODE_DPHY		(0x0)
+#define DWC_MIPI_CSI2_DPHY_MODE_CPHY		(0x1)
+
+#define DWC_MIPI_CSI2_INT_ST_AP_MAIN		0x2C
+
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1			0x30
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI0_VC(x)		((x & 0x3))
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI0_VC_0_1(x)	((x & 0x3) << 2)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI0_VC_2		(0x1 << 4)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI1_VC(x)		((x & 0x3) << 8)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI1_VC_0_1(x)	((x & 0x3) << 10)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI1_VC_2		(0x1 << 12)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI2_VC(x)		((x & 0x3) << 16)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI2_VC_0_1(x)	((x & 0x3) << 18)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI2_VC_2		(0x1 << 20)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI3_VC(x)		((x & 0x3) << 24)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI3_VC_0_1(x)	((x & 0x3) << 26)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_1_DI3_VC_2		(0x1 << 28)
+
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2			0x34
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI4_VC(x)		((x & 0x3))
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI4_VC_0_1(x)	((x & 0x3) << 2)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI4_VC_2		(0x1 << 4)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI5_VC(x)		((x & 0x3) << 8)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI5_VC_0_1(x)	((x & 0x3) << 10)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI5_VC_2		(0x1 << 12)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI6_VC(x)		((x & 0x3) << 16)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI6_VC_0_1(x)	((x & 0x3) << 18)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI6_VC_2		(0x1 << 20)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI7_VC(x)		((x & 0x3) << 24)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI7_VC_0_1(x)	((x & 0x3) << 26)
+#define DWC_MIPI_CSI2_DATA_IDS_VC_2_DI7_VC_2		(0x1 << 28)
+
+#define DWC_MIPI_CSI2_DPHY_SHUTDOWNZ		0x40
+#define DWC_MIPI_CSI2_DPHY_SHUTDOWNZ_ENABLE	(0x1)
+
+#define DWC_MIPI_CSI2_DPHY_RSTZ			0x44
+#define DWC_MIPI_CSI2_DPHY_RSTZ_ENABLE		(0x1)
+
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS			0x48
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_CLK_LANE_HS	(0x1 << 17)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_CLK_LANE_ULP	(0x1 << 16)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE0_ULP	(0x1)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE1_ULP	(0x1 << 1)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE2_ULP	(0x1 << 2)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE3_ULP	(0x1 << 3)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE4_ULP	(0x1 << 4)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE5_ULP	(0x1 << 5)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE6_ULP	(0x1 << 6)
+#define DWC_MIPI_CSI2_DPHY_RX_STATUS_DATA_LANE7_ULP	(0x1 << 7)
+
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE			0x4C
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_CLK_LANE		(0x1 << 16)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE0		(0x1)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE1		(0x1 << 1)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE2		(0x1 << 2)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE3		(0x1 << 3)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE4		(0x1 << 4)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE5		(0x1 << 5)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE6		(0x1 << 6)
+#define DWC_MIPI_CSI2_DPHY_STOPSTATE_DATA_LANE7		(0x1 << 7)
+
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL0			0x50
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL0_TEST_CLR		(0x1)
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL0_TEST_CLKEN	(0x1 << 1)
+
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL1			0x54
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL1_TEST_DIN(x)	((x) & 0xFF)
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL1_TEST_DOUT(x)	(((x) & 0xFF) << 8)
+#define DWC_MIPI_CSI2_DPHY_TEST_CTRL1_TEST_EN		(0x1 << 16)
+
+#define DWC_MIPI_CSI2_PPI_PG_PATTERN_VRES		0x60
+#define DWC_MIPI_CSI2_PPI_PG_PATTERN_VRES_VRES(x)	((x) & 0xFFFF)
+
+#define DWC_MIPI_CSI2_PPI_PG_PATTERN_HRES		0x64
+#define DWC_MIPI_CSI2_PPI_PG_PATTERN_HRES_HRES(x)	((x) & 0xFFFF)
+
+#define DWC_MIPI_CSI2_PPI_PG_CONFIG			0x68
+#define DWC_MIPI_CSI2_PPI_PG_CONFIG_DATA_TYPE(x)	(((x) & 0x3F) << 8)
+#define DWC_MIPI_CSI2_PPI_PG_CONFIG_VIR_CHAN(x)		(((x) & 0x3) << 14)
+#define DWC_MIPI_CSI2_PPI_PG_CONFIG_VIR_CHAN_EX(x)	(((x) & 0x3) << 16)
+#define DWC_MIPI_CSI2_PPI_PG_CONFIG_VIR_CHAN_EX_2_EN	(0x1 << 18)
+#define DWC_MIPI_CSI2_PPI_PG_CONFIG_PG_MODE(x)		(x)
+
+#define DWC_MIPI_CSI2_PPI_PG_ENABLE			0x6C
+#define DWC_MIPI_CSI2_PPI_PG_ENABLE_EN			0x1
+
+#define DWC_MIPI_CSI2_PPI_PG_STATUS			0x70
+
+#define DWC_MIPI_CSI2_IPI_MODE				0x80
+#define DWC_MIPI_CSI2_IPI_MODE_CAMERA			0x0
+#define DWC_MIPI_CSI2_IPI_MODE_CONTROLLER		0x1
+#define DWC_MIPI_CSI2_IPI_MODE_COLOR_MODE16		(0x1 << 8)
+#define DWC_MIPI_CSI2_IPI_MODE_COLOR_MODE48		(0x0 << 8)
+#define DWC_MIPI_CSI2_IPI_MODE_CUT_THROUGH		(0x1 << 16)
+#define DWC_MIPI_CSI2_IPI_MODE_ENABLE			(0x1 << 24)
+
+#define DWC_MIPI_CSI2_IPI_VCID				0x84
+#define DWC_MIPI_CSI2_IPI_VCID_VC(x)			((x)  & 0x3)
+#define DWC_MIPI_CSI2_IPI_VCID_VC_0_1(x)		(((x) & 0x3) << 2)
+#define DWC_MIPI_CSI2_IPI_VCID_VC_2			(0x1 << 4)
+
+#define DWC_MIPI_CSI2_IPI_DATA_TYPE			0x88
+#define DWC_MIPI_CSI2_IPI_DATA_TYPE_DT(x)		((x) & 0x3F)
+#define DWC_MIPI_CSI2_IPI_DATA_TYPE_EMB_DATA_EN		(0x1 << 8)
+
+#define DWC_MIPI_CSI2_IPI_MEM_FLUSH			0x8C
+#define DWC_MIPI_CSI2_IPI_MEM_FLUSH_AUTO		(0x1 << 8)
+
+#define DWC_MIPI_CSI2_IPI_HSA_TIME			0x90
+#define DWC_MIPI_CSI2_IPI_HSA_TIME_VAL(x)		((x) & 0xFFF)
+
+#define DWC_MIPI_CSI2_IPI_HBP_TIME			0x94
+#define DWC_MIPI_CSI2_IPI_HBP_TIME_VAL(x)		((x) & 0xFFF)
+
+#define DWC_MIPI_CSI2_IPI_HSD_TIME			0x98
+#define DWC_MIPI_CSI2_IPI_HSD_TIME_VAL(x)		((x) & 0xFFF)
+
+#define DWC_MIPI_CSI2_IPI_HLINE_TIME			0x9C
+#define DWC_MIPI_CSI2_IPI_HLINE_TIME_VAL(x)		((x) & 0x3FFF)
+
+#define DWC_MIPI_CSI2_IPI_SOFTRSTN			0xA0
+#define DWC_MIPI_CSI2_IPI_ADV_FEATURES			0xAC
+
+#define DWC_MIPI_CSI2_IPI_VSA_LINES			0xB0
+#define DWC_MIPI_CSI2_IPI_VSA_LINES_VAL(x)		((x) & 0x3FF)
+
+#define DWC_MIPI_CSI2_IPI_VBP_LINES			0xB4
+#define DWC_MIPI_CSI2_IPI_VBP_LINES_VAL(x)		((x) & 0x3FF)
+
+#define DWC_MIPI_CSI2_IPI_VFP_LINES			0xB8
+#define DWC_MIPI_CSI2_IPI_VFP_LINES_VAL(x)		((x) & 0x3FF)
+
+#define DWC_MIPI_CSI2_IPI_VACTIVE_LINES			0xBC
+#define DWC_MIPI_CSI2_IPI_VACTIVE_LINES_VAL(x)		((x) & 0x3FFF)
+
+#define DWC_MIPI_CSI2_VC_EXTENSION			0xC8
+
+#define DWC_MIPI_CSI2_DPHY_CAL				0xCC
+#define DWC_MIPI_CSI2_INT_ST_DPHY_FATAL			0xE0
+#define DWC_MIPI_CSI2_INT_MSK_DPHY_FATAL		0xE4
+#define DWC_MIPI_CSI2_INT_FORCE_DPHY_FATAL		0xE8
+#define DWC_MIPI_CSI2_INT_ST_PKT_FATAL			0xF0
+#define DWC_MIPI_CSI2_INT_MSK_PKT_FATAL			0xF4
+#define DWC_MIPI_CSI2_INT_FORCE_PKT_FATAL		0xF8
+
+#define DWC_MIPI_CSI2_INT_ST_DPHY			0x110
+#define DWC_MIPI_CSI2_INT_MSK_DPHY			0x114
+#define DWC_MIPI_CSI2_INT_FORCE_DPHY			0x118
+#define DWC_MIPI_CSI2_INT_ST_LINE			0x130
+#define DWC_MIPI_CSI2_INT_MSK_LINE			0x134
+#define DWC_MIPI_CSI2_INT_FORCE_LINE			0x138
+#define DWC_MIPI_CSI2_INT_ST_IPI_FATAL			0x140
+#define DWC_MIPI_CSI2_INT_MSK_IPI_FATAL			0x144
+#define DWC_MIPI_CSI2_INT_FORCE_IPI_FATAL		0x148
+
+#define DWC_MIPI_CSI2_INT_ST_AP_GENERIC			0x180
+#define DWC_MIPI_CSI2_INT_MSK_AP_GENERIC		0x184
+#define DWC_MIPI_CSI2_INT_FORCE_AP_GENERIC		0x188
+#define DWC_MIPI_CSI2_INT_ST_AP_IPI_FATAL		0x190
+#define DWC_MIPI_CSI2_INT_MSK_AP_IPI_FATAL		0x194
+#define DWC_MIPI_CSI2_INT_FORCE_AP_IPI_FATAL		0x198
+
+#define DWC_MIPI_CSI2_INT_ST_BNDRY_FRAME_FATAL		0x280
+#define DWC_MIPI_CSI2_INT_MSK_BNDRY_FRAME_FATAL		0x284
+#define DWC_MIPI_CSI2_INT_FORCE_BNDRY_FRAME_FATAL	0x288
+#define DWC_MIPI_CSI2_INT_ST_SEQ_FRAME_FATAL		0x290
+#define DWC_MIPI_CSI2_INT_MSK_SEQ_FRAME_FATAL		0x294
+#define DWC_MIPI_CSI2_INT_FORCE_SEQ_FRAME_FATAL		0x298
+#define DWC_MIPI_CSI2_INT_ST_CRC_FRAME_FATAL		0x2A0
+#define DWC_MIPI_CSI2_INT_MSK_CRC_FRAME_FATAL		0x2A4
+#define DWC_MIPI_CSI2_INT_FORCE_CRC_FRAME_FATAL		0x2A8
+#define DWC_MIPI_CSI2_INT_ST_PLD_CRC_FATAL		0x2B0
+#define DWC_MIPI_CSI2_INT_MSK_PLD_CRC_FATAL		0x2B4
+#define DWC_MIPI_CSI2_INT_FORCE_PLD_CRC_FATAL		0x2B8
+#define DWC_MIPI_CSI2_INT_ST_DATA_ID			0x2C0
+#define DWC_MIPI_CSI2_INT_MSK_DATA_ID			0x2C4
+#define DWC_MIPI_CSI2_INT_FORCE_DATA_ID			0x2C8
+#define DWC_MIPI_CSI2_INT_ST_ECC_CORRECTED		0x2D0
+#define DWC_MIPI_CSI2_INT_MSK_ECC_CORRECTED		0x2D4
+#define DWC_MIPI_CSI2_INT_FORCE_ECC_CORRECTED		0x2D8
+#define DWC_MIPI_CSI2_IPI_RAM_ERR_LOG_AP		0x2E0
+#define DWC_MIPI_CSI2_IPI_RAM_ERR_ADDR_AP		0x2E4
+
+#define DWC_MIPI_CSI2_SCRAMBLING			0x300
+#define DWC_MIPI_CSI2_SCRAMBLING_SEED1			0x304
+#define DWC_MIPI_CSI2_SCRAMBLING_SEED2			0x308
+
+/* mediamix_GPR register */
+#define DISP_MIX_CAMERA_MUX				0x30
+#define DISP_MIX_CAMERA_MUX_DATA_TYPE(x)		(((x) & 0x3f) << 3)
+// #define DISP_MIX_CAMERA_MUX_GASKET_ENABLE		(1 << 16)
+
+#define DISP_MIX_CSI_REG				0x48
+#define DISP_MIX_CSI_REG_CFGFREQRANGE(x)		((x)  & 0x3f)
+#define DISP_MIX_CSI_REG_HSFREQRANGE(x)			(((x) & 0x7f) << 8)
+
+#define dwc_mipi_csi2h_write(__csi2h, __r, __v) writel(__v, __csi2h->base_regs + __r)
+#define dwc_mipi_csi2h_read(__csi2h, __r) readl(__csi2h->base_regs + __r)
+
+
+#define SW_CPHY_EN(x)		((x) << 0)
+#define SW_DSI_EN(x)		((x) << 4)
+#define SW_DATATYPE_FS(x)	((x) << 8)
+#define SW_DATATYPE_FE(x)	((x) << 14)
+#define SW_DATATYPE_LS(x)	((x) << 20)
+#define SW_DATATYPE_LE(x)	((x) << 26)
+
+#define ES_MAX_CSI_HW		(6)
+
+/*
+ * add new chip id in tail in time order
+ * by increasing to distinguish csi2 host version
+ */
+enum escsi2_chip_id {
+	CHIP_PX30_CSI2,
+	CHIP_ES1808_CSI2,
+	CHIP_ES3128_CSI2,
+	CHIP_ES3288_CSI2,
+	CHIP_RV1126_CSI2,
+	CHIP_ES3568_CSI2,
+	CHIP_ES3588_CSI2,
+	CHIP_RV1106_CSI2,
+	CHIP_ES3562_CSI2,
+};
+
+enum csi2_pads {
+	ES_CSI2_PAD_SINK = 0,
+	ES_CSI2X_PAD_SOURCE0,
+	ES_CSI2X_PAD_SOURCE1,
+	ES_CSI2X_PAD_SOURCE2,
+	ES_CSI2X_PAD_SOURCE3
+};
+
+enum csi2_err {
+	ES_CSI2_ERR_SOTSYN = 0x0,
+	ES_CSI2_ERR_FS_FE_MIS,
+	ES_CSI2_ERR_FRM_SEQ_ERR,
+	ES_CSI2_ERR_CRC_ONCE,
+	ES_CSI2_ERR_CRC,
+	ES_CSI2_ERR_ALL,
+	ES_CSI2_ERR_MAX
+};
+
+enum host_type_t {
+	ES_CSI_RXHOST,
+	ES_DSI_RXHOST
+};
+
+struct csi2_match_data {
+	int chip_id;
+	int num_pads;
+	int num_hw;
+};
+
+struct csi2_hw_match_data {
+	int chip_id;
+};
+
+struct csi2_sensor_info {
+	struct v4l2_subdev *sd;
+	struct v4l2_mbus_config mbus;
+	int lanes;
+};
+
+struct csi2_err_stats {
+	unsigned int cnt;
+};
+
+struct csi2_dev {
+	struct device		*dev;
+	struct v4l2_subdev	sd;
+	struct media_pad	pad[CSI2_NUM_PADS_MAX];
+	struct clk_bulk_data	*clks_bulk;
+	int			clks_num;
+	struct reset_control	*rsts_bulk;
+
+	void __iomem		*base;
+	struct v4l2_async_notifier	notifier;
+	struct v4l2_mbus_config_mipi_csi2	bus;
+
+	/* lock to protect all members below */
+	struct mutex lock;
+
+	struct v4l2_mbus_framefmt	format_mbus;
+	struct v4l2_rect	crop;
+	int			stream_count;
+	struct v4l2_subdev	*src_sd;
+	bool			sink_linked[CSI2_NUM_SRC_PADS];
+	bool			is_check_sot_sync;
+	struct csi2_sensor_info	sensors[MAX_CSI2_SENSORS];
+	const struct csi2_match_data	*match_data;
+	int			num_sensors;
+	atomic_t		frm_sync_seq;
+	struct csi2_err_stats	err_list[ES_CSI2_ERR_MAX];
+	struct csi2_hw		*csi2_hw[ES_MAX_CSI_HW];
+	int			irq1;
+	int			irq2;
+	int			dsi_input_en;
+	struct es_dvp2axi_csi_info	csi_info;
+	const char		*dev_name;
+};
+
+struct csi2_hw {
+	struct device		*dev;
+	struct clk_bulk_data	*clks_bulk;
+	int			clks_num;
+	struct reset_control	*rsts_bulk;
+	struct csi2_dev		*csi2;
+	const struct csi2_hw_match_data	*match_data;
+
+	void __iomem		*base;
+
+	/* lock to protect all members below */
+	struct mutex lock;
+
+	int			irq1;
+	int			irq2;
+	const char		*dev_name;
+
+	/* eic770x_mipi_csi2_init */
+	u32 num_lanes:4;
+    u32 ppi_width:2;
+	u32 phy_mode:1;
+
+    u32 vc:5;
+    u32 dt:6;	
+	u32 emb:1;
+	u32 frame_det:1;
+
+    u32 ipi_mode:1;
+    u32 ipi_color_com:1;
+    u32 ipi_auto_flush:1;
+    u32 ipi_cut_through:1;
+    u32 ipi_line_event;
+
+    u32 hsa;
+	u32 hbp;
+	u32 hsd;
+	u32 htotal;
+
+    u32 vsa;
+	u32 vbp;
+	u32 vfp;
+	u32 vactive;
+};
+
+u32 es_dvp2axi_csi2_get_sof(struct csi2_dev *csi2_dev);
+void es_dvp2axi_csi2_set_sof(struct csi2_dev *csi2_dev, u32 seq);
+void es_dvp2axi_csi2_event_inc_sof(struct csi2_dev *csi2_dev);
+int es_dvp2axi_csi2_plat_drv_init(void);
+void es_dvp2axi_csi2_plat_drv_exit(void);
+int es_dvp2axi_csi2_hw_plat_drv_init(void);
+void es_dvp2axi_csi2_hw_plat_drv_exit(void);
+int es_dvp2axi_csi2_register_notifier(struct notifier_block *nb);
+int es_dvp2axi_csi2_unregister_notifier(struct notifier_block *nb);
+void es_dvp2axi_csi2_event_reset_pipe(struct csi2_dev *csi2_dev, int reset_src);
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/phy-es-csi2-dphy-common.h b/drivers/media/platform/eswin/dvp2axi/phy-es-csi2-dphy-common.h
new file mode 100644
index 000000000000..a9c1a1051b0e
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/phy-es-csi2-dphy-common.h
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI csi-dphy-common driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+ #ifndef _PHY_ESWIN_CSI2_DPHY_COMMON_H_
+ #define _PHY_ESWIN_CSI2_DPHY_COMMON_H_
+ 
+ #include "../../../../phy/eswin/es-camera-module.h"
+ #include "../../../../phy/eswin/es-dvp2axi-config.h"
+ 
+ #define PHY_MAX 16
+ #define MAX_DEV_NAME_LEN 32
+ 
+ #define MAX_SAMSUNG_PHY_NUM 2
+ 
+ #define MAX_INNO_PHY_NUM 2
+ 
+ /* add new chip id in tail by time order */
+ enum csi2_dphy_chip_id {
+     CHIP_ID_ES3568 = 0x0,
+     CHIP_ID_ES3588 = 0x1,
+     CHIP_ID_ES3588_DCPHY = 0x2,
+     CHIP_ID_RV1106 = 0x3,
+     CHIP_ID_ES3562 = 0x4,
+ };
+ 
+ enum csi2_dphy_rx_pads {
+     CSI2_DPHY_RX_PAD_SINK = 0,
+     CSI2_DPHY_RX_PAD_SOURCE,
+     CSI2_DPHY_RX_PADS_NUM,
+ };
+ 
+ enum csi2_dphy_lane_mode {
+     LANE_MODE_UNDEF = 0x0,
+     LANE_MODE_FULL,
+     LANE_MODE_SPLIT,
+ };
+ 
+ struct grf_reg {
+     u32 offset;
+     u32 mask;
+     u32 shift;
+ };
+ 
+ struct csi2dphy_reg {
+     u32 offset;
+ };
+ 
+ #define MAX_DPHY_SENSORS	(2)
+ #define MAX_NUM_CSI2_DPHY	(0x2)
+ 
+ struct csi2_sensor {
+     struct v4l2_subdev *sd;
+     struct v4l2_mbus_config mbus;
+     struct v4l2_mbus_framefmt format;
+     int lanes;
+ };
+ 
+ struct csi2_dphy_hw;
+ struct samsung_mipi_dcphy;
+ 
+ struct dphy_drv_data {
+     const char dev_name[MAX_DEV_NAME_LEN];
+     enum csi2_dphy_chip_id chip_id;
+     char num_inno_phy;
+     char num_samsung_phy;
+ };
+ 
+ struct csi2_dphy {
+     struct device *dev;
+     struct list_head list;
+     struct csi2_dphy_hw *dphy_hw;
+     struct csi2_dphy_hw *dphy_hw_group[MAX_INNO_PHY_NUM];
+     struct samsung_mipi_dcphy *samsung_phy;
+     struct samsung_mipi_dcphy *samsung_phy_group[MAX_SAMSUNG_PHY_NUM];
+     struct v4l2_async_notifier notifier;
+     struct v4l2_subdev sd;
+     struct mutex mutex; /* lock for updating protection */
+     struct media_pad pads[CSI2_DPHY_RX_PADS_NUM];
+     struct csi2_sensor sensors[MAX_DPHY_SENSORS];
+     u64 data_rate_mbps;
+     int num_sensors;
+     int phy_index;
+     struct es_dvp2axi_csi_info csi_info;
+     void *phy_hw[ESMODULE_MULTI_DEV_NUM];
+     bool is_streaming;
+     int lane_mode;
+     const struct dphy_drv_data *drv_data;
+     struct esmodule_csi_dphy_param dphy_param;
+ };
+ 
+ struct dphy_hw_drv_data {
+     const struct hsfreq_range *hsfreq_ranges;
+     int num_hsfreq_ranges;
+     const struct hsfreq_range *hsfreq_ranges_cphy;
+     int num_hsfreq_ranges_cphy;
+     const struct grf_reg *grf_regs;
+     int num_grf_regs;
+     const struct csi2dphy_reg *csi2dphy_regs;
+     int num_csi2dphy_regs;
+     void (*individual_init)(struct csi2_dphy_hw *hw);
+     int (*stream_on)(struct csi2_dphy *dphy, struct v4l2_subdev *sd);
+     int (*stream_off)(struct csi2_dphy *dphy, struct v4l2_subdev *sd);
+     enum csi2_dphy_chip_id chip_id;
+ };
+ 
+ struct csi2_dphy_hw {
+     struct device *dev;
+     struct regmap *regmap_grf;
+     struct regmap *regmap_sys_grf;
+     const struct grf_reg *grf_regs;
+     const struct csi2dphy_reg *csi2dphy_regs;
+     const struct dphy_hw_drv_data *drv_data;
+     void __iomem *hw_base_addr;
+     struct clk_bulk_data	*clks_bulk;
+     struct reset_control	*rsts_bulk;
+     struct csi2_dphy *dphy_dev[MAX_NUM_CSI2_DPHY];
+     struct v4l2_subdev sd;
+     struct mutex mutex; /* lock for updating protection */
+     atomic_t stream_cnt;
+     int num_clks;
+     int num_sensors;
+     int dphy_dev_num;
+     enum csi2_dphy_lane_mode lane_mode;
+ 
+     int (*stream_on)(struct csi2_dphy *dphy, struct v4l2_subdev *sd);
+     int (*stream_off)(struct csi2_dphy *dphy, struct v4l2_subdev *sd);
+     int (*ttl_mode_enable)(struct csi2_dphy_hw *hw);
+     void (*ttl_mode_disable)(struct csi2_dphy_hw *hw);
+     int (*quick_stream_on)(struct csi2_dphy *dphy, struct v4l2_subdev *sd);
+     int (*quick_stream_off)(struct csi2_dphy *dphy, struct v4l2_subdev *sd);
+ };
+ 
+ int es_csi2_dphy_hw_init(void);
+ int es_csi2_dphy_init(void);
+ 
+ #endif
+ 
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/dvp2axi/procfs.c b/drivers/media/platform/eswin/dvp2axi/procfs.c
new file mode 100644
index 000000000000..f75c2ed77e3e
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/procfs.c
@@ -0,0 +1,445 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI procfs driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+#include <linux/clk.h>
+#include <linux/math64.h>
+#include <linux/proc_fs.h>
+#include <linux/sem.h>
+#include <linux/seq_file.h>
+#include <linux/spinlock.h>
+#include <linux/v4l2-mediabus.h>
+
+#include "dev.h"
+#include "procfs.h"
+
+#ifdef CONFIG_PROC_FS
+
+static const struct {
+	const char *name;
+	u32 mbus_code;
+} mbus_formats[] = {
+	/* media bus code */
+	{ "RGB444_1X12", MEDIA_BUS_FMT_RGB444_1X12 },
+	{ "RGB444_2X8_PADHI_BE", MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE },
+	{ "RGB444_2X8_PADHI_LE", MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE },
+	{ "RGB555_2X8_PADHI_BE", MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE },
+	{ "RGB555_2X8_PADHI_LE", MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE },
+	{ "RGB565_1X16", MEDIA_BUS_FMT_RGB565_1X16 },
+	{ "BGR565_2X8_BE", MEDIA_BUS_FMT_BGR565_2X8_BE },
+	{ "BGR565_2X8_LE", MEDIA_BUS_FMT_BGR565_2X8_LE },
+	{ "RGB565_2X8_BE", MEDIA_BUS_FMT_RGB565_2X8_BE },
+	{ "RGB565_2X8_LE", MEDIA_BUS_FMT_RGB565_2X8_LE },
+	{ "RGB666_1X18", MEDIA_BUS_FMT_RGB666_1X18 },
+	{ "RBG888_1X24", MEDIA_BUS_FMT_RBG888_1X24 },
+	{ "RGB666_1X24_CPADHI", MEDIA_BUS_FMT_RGB666_1X24_CPADHI },
+	{ "RGB666_1X7X3_SPWG", MEDIA_BUS_FMT_RGB666_1X7X3_SPWG },
+	{ "BGR888_1X24", MEDIA_BUS_FMT_BGR888_1X24 },
+	{ "GBR888_1X24", MEDIA_BUS_FMT_GBR888_1X24 },
+	{ "RGB888_1X24", MEDIA_BUS_FMT_RGB888_1X24 },
+	{ "RGB888_2X12_BE", MEDIA_BUS_FMT_RGB888_2X12_BE },
+	{ "RGB888_2X12_LE", MEDIA_BUS_FMT_RGB888_2X12_LE },
+	{ "RGB888_1X7X4_SPWG", MEDIA_BUS_FMT_RGB888_1X7X4_SPWG },
+	{ "RGB888_1X7X4_JEIDA", MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA },
+	{ "ARGB8888_1X32", MEDIA_BUS_FMT_ARGB8888_1X32 },
+	{ "RGB888_1X32_PADHI", MEDIA_BUS_FMT_RGB888_1X32_PADHI },
+	{ "RGB101010_1X30", MEDIA_BUS_FMT_RGB101010_1X30 },
+	{ "RGB121212_1X36", MEDIA_BUS_FMT_RGB121212_1X36 },
+	{ "RGB161616_1X48", MEDIA_BUS_FMT_RGB161616_1X48 },
+	{ "Y8_1X8", MEDIA_BUS_FMT_Y8_1X8 },
+	{ "UV8_1X8", MEDIA_BUS_FMT_UV8_1X8 },
+	{ "UYVY8_1_5X8", MEDIA_BUS_FMT_UYVY8_1_5X8 },
+	{ "VYUY8_1_5X8", MEDIA_BUS_FMT_VYUY8_1_5X8 },
+	{ "YUYV8_1_5X8", MEDIA_BUS_FMT_YUYV8_1_5X8 },
+	{ "YVYU8_1_5X8", MEDIA_BUS_FMT_YVYU8_1_5X8 },
+	{ "UYVY8_2X8", MEDIA_BUS_FMT_UYVY8_2X8 },
+	{ "VYUY8_2X8", MEDIA_BUS_FMT_VYUY8_2X8 },
+	{ "YUYV8_2X8", MEDIA_BUS_FMT_YUYV8_2X8 },
+	{ "YVYU8_2X8", MEDIA_BUS_FMT_YVYU8_2X8 },
+	{ "Y10_1X10", MEDIA_BUS_FMT_Y10_1X10 },
+	{ "Y10_2X8_PADHI_LE", MEDIA_BUS_FMT_Y10_2X8_PADHI_LE },
+	{ "UYVY10_2X10", MEDIA_BUS_FMT_UYVY10_2X10 },
+	{ "VYUY10_2X10", MEDIA_BUS_FMT_VYUY10_2X10 },
+	{ "YUYV10_2X10", MEDIA_BUS_FMT_YUYV10_2X10 },
+	{ "YVYU10_2X10", MEDIA_BUS_FMT_YVYU10_2X10 },
+	{ "Y12_1X12", MEDIA_BUS_FMT_Y12_1X12 },
+	{ "UYVY12_2X12", MEDIA_BUS_FMT_UYVY12_2X12 },
+	{ "VYUY12_2X12", MEDIA_BUS_FMT_VYUY12_2X12 },
+	{ "YUYV12_2X12", MEDIA_BUS_FMT_YUYV12_2X12 },
+	{ "YVYU12_2X12", MEDIA_BUS_FMT_YVYU12_2X12 },
+	{ "UYVY8_1X16", MEDIA_BUS_FMT_UYVY8_1X16 },
+	{ "VYUY8_1X16", MEDIA_BUS_FMT_VYUY8_1X16 },
+	{ "YUYV8_1X16", MEDIA_BUS_FMT_YUYV8_1X16 },
+	{ "YVYU8_1X16", MEDIA_BUS_FMT_YVYU8_1X16 },
+	{ "YDYUYDYV8_1X16", MEDIA_BUS_FMT_YDYUYDYV8_1X16 },
+	{ "UYVY10_1X20", MEDIA_BUS_FMT_UYVY10_1X20 },
+	{ "VYUY10_1X20", MEDIA_BUS_FMT_VYUY10_1X20 },
+	{ "YUYV10_1X20", MEDIA_BUS_FMT_YUYV10_1X20 },
+	{ "YVYU10_1X20", MEDIA_BUS_FMT_YVYU10_1X20 },
+	{ "VUY8_1X24", MEDIA_BUS_FMT_VUY8_1X24 },
+	{ "YUV8_1X24", MEDIA_BUS_FMT_YUV8_1X24 },
+	{ "UYYVYY8_0_5X24", MEDIA_BUS_FMT_UYYVYY8_0_5X24 },
+	{ "UYVY12_1X24", MEDIA_BUS_FMT_UYVY12_1X24 },
+	{ "VYUY12_1X24", MEDIA_BUS_FMT_VYUY12_1X24 },
+	{ "YUYV12_1X24", MEDIA_BUS_FMT_YUYV12_1X24 },
+	{ "YVYU12_1X24", MEDIA_BUS_FMT_YVYU12_1X24 },
+	{ "YUV10_1X30", MEDIA_BUS_FMT_YUV10_1X30 },
+	{ "UYYVYY10_0_5X30", MEDIA_BUS_FMT_UYYVYY10_0_5X30 },
+	{ "AYUV8_1X32", MEDIA_BUS_FMT_AYUV8_1X32 },
+	{ "UYYVYY12_0_5X36", MEDIA_BUS_FMT_UYYVYY12_0_5X36 },
+	{ "YUV12_1X36", MEDIA_BUS_FMT_YUV12_1X36 },
+	{ "YUV16_1X48", MEDIA_BUS_FMT_YUV16_1X48 },
+	{ "UYYVYY16_0_5X48", MEDIA_BUS_FMT_UYYVYY16_0_5X48 },
+	{ "SBGGR8_1X8", MEDIA_BUS_FMT_SBGGR8_1X8 },
+	{ "SGBRG8_1X8", MEDIA_BUS_FMT_SGBRG8_1X8 },
+	{ "SGRBG8_1X8", MEDIA_BUS_FMT_SGRBG8_1X8 },
+	{ "SRGGB8_1X8", MEDIA_BUS_FMT_SRGGB8_1X8 },
+	{ "SBGGR10_ALAW8_1X8", MEDIA_BUS_FMT_SBGGR10_ALAW8_1X8 },
+	{ "SGBRG10_ALAW8_1X8", MEDIA_BUS_FMT_SGBRG10_ALAW8_1X8 },
+	{ "SGRBG10_ALAW8_1X8", MEDIA_BUS_FMT_SGRBG10_ALAW8_1X8 },
+	{ "SRGGB10_ALAW8_1X8", MEDIA_BUS_FMT_SRGGB10_ALAW8_1X8 },
+	{ "SBGGR10_DPCM8_1X8", MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8 },
+	{ "SGBRG10_DPCM8_1X8", MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8 },
+	{ "SGRBG10_DPCM8_1X8", MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8 },
+	{ "SRGGB10_DPCM8_1X8", MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8 },
+	{ "SBGGR10_2X8_PADHI_BE", MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_BE },
+	{ "SBGGR10_2X8_PADHI_LE", MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_LE },
+	{ "SBGGR10_2X8_PADLO_BE", MEDIA_BUS_FMT_SBGGR10_2X8_PADLO_BE },
+	{ "SBGGR10_2X8_PADLO_LE", MEDIA_BUS_FMT_SBGGR10_2X8_PADLO_LE },
+	{ "SBGGR10_1X10", MEDIA_BUS_FMT_SBGGR10_1X10 },
+	{ "SGBRG10_1X10", MEDIA_BUS_FMT_SGBRG10_1X10 },
+	{ "SGRBG10_1X10", MEDIA_BUS_FMT_SGRBG10_1X10 },
+	{ "SRGGB10_1X10", MEDIA_BUS_FMT_SRGGB10_1X10 },
+	{ "SBGGR12_1X12", MEDIA_BUS_FMT_SBGGR12_1X12 },
+	{ "SGBRG12_1X12", MEDIA_BUS_FMT_SGBRG12_1X12 },
+	{ "SGRBG12_1X12", MEDIA_BUS_FMT_SGRBG12_1X12 },
+	{ "SRGGB12_1X12", MEDIA_BUS_FMT_SRGGB12_1X12 },
+	{ "SBGGR14_1X14", MEDIA_BUS_FMT_SBGGR14_1X14 },
+	{ "SGBRG14_1X14", MEDIA_BUS_FMT_SGBRG14_1X14 },
+	{ "SGRBG14_1X14", MEDIA_BUS_FMT_SGRBG14_1X14 },
+	{ "SRGGB14_1X14", MEDIA_BUS_FMT_SRGGB14_1X14 },
+	{ "SBGGR16_1X16", MEDIA_BUS_FMT_SBGGR16_1X16 },
+	{ "SGBRG16_1X16", MEDIA_BUS_FMT_SGBRG16_1X16 },
+	{ "SGRBG16_1X16", MEDIA_BUS_FMT_SGRBG16_1X16 },
+	{ "SRGGB16_1X16", MEDIA_BUS_FMT_SRGGB16_1X16 },
+	{ "JPEG_1X8", MEDIA_BUS_FMT_JPEG_1X8 },
+	{ "S5C_UYVY_JPEG_1X8", MEDIA_BUS_FMT_S5C_UYVY_JPEG_1X8 },
+	{ "AHSV8888_1X32", MEDIA_BUS_FMT_AHSV8888_1X32 },
+	{ "FIXED", MEDIA_BUS_FMT_FIXED },
+	{ "Y8", MEDIA_BUS_FMT_Y8_1X8 },
+	{ "Y10", MEDIA_BUS_FMT_Y10_1X10 },
+	{ "Y12", MEDIA_BUS_FMT_Y12_1X12 },
+	{ "YUYV", MEDIA_BUS_FMT_YUYV8_1X16 },
+	{ "YUYV1_5X8", MEDIA_BUS_FMT_YUYV8_1_5X8 },
+	{ "YUYV2X8", MEDIA_BUS_FMT_YUYV8_2X8 },
+	{ "UYVY", MEDIA_BUS_FMT_UYVY8_1X16 },
+	{ "UYVY1_5X8", MEDIA_BUS_FMT_UYVY8_1_5X8 },
+	{ "UYVY2X8", MEDIA_BUS_FMT_UYVY8_2X8 },
+	{ "VUY24", MEDIA_BUS_FMT_VUY8_1X24 },
+	{ "SBGGR8", MEDIA_BUS_FMT_SBGGR8_1X8 },
+	{ "SGBRG8", MEDIA_BUS_FMT_SGBRG8_1X8 },
+	{ "SGRBG8", MEDIA_BUS_FMT_SGRBG8_1X8 },
+	{ "SRGGB8", MEDIA_BUS_FMT_SRGGB8_1X8 },
+	{ "SBGGR10", MEDIA_BUS_FMT_SBGGR10_1X10 },
+	{ "SGBRG10", MEDIA_BUS_FMT_SGBRG10_1X10 },
+	{ "SGRBG10", MEDIA_BUS_FMT_SGRBG10_1X10 },
+	{ "SRGGB10", MEDIA_BUS_FMT_SRGGB10_1X10 },
+	{ "SBGGR10_DPCM8", MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8 },
+	{ "SGBRG10_DPCM8", MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8 },
+	{ "SGRBG10_DPCM8", MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8 },
+	{ "SRGGB10_DPCM8", MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8 },
+	{ "SBGGR12", MEDIA_BUS_FMT_SBGGR12_1X12 },
+	{ "SGBRG12", MEDIA_BUS_FMT_SGBRG12_1X12 },
+	{ "SGRBG12", MEDIA_BUS_FMT_SGRBG12_1X12 },
+	{ "SRGGB12", MEDIA_BUS_FMT_SRGGB12_1X12 },
+	{ "AYUV32", MEDIA_BUS_FMT_AYUV8_1X32 },
+	{ "RBG24", MEDIA_BUS_FMT_RBG888_1X24 },
+	{ "RGB32", MEDIA_BUS_FMT_RGB888_1X32_PADHI },
+	{ "ARGB32", MEDIA_BUS_FMT_ARGB8888_1X32 },
+
+	/* v4l2 fourcc code */
+	{ "NV16", V4L2_PIX_FMT_NV16},
+	{ "NV61", V4L2_PIX_FMT_NV61},
+	{ "NV12", V4L2_PIX_FMT_NV12},
+	{ "NV21", V4L2_PIX_FMT_NV21},
+	{ "YUYV", V4L2_PIX_FMT_YUYV},
+	{ "YVYU", V4L2_PIX_FMT_YVYU},
+	{ "UYVY", V4L2_PIX_FMT_UYVY},
+	{ "VYUY", V4L2_PIX_FMT_VYUY},
+	{ "RGB3", V4L2_PIX_FMT_RGB24},
+	{ "RGBP", V4L2_PIX_FMT_RGB565},
+	{ "BGRH", V4L2_PIX_FMT_BGR666},
+	{ "RGGB", V4L2_PIX_FMT_SRGGB8},
+	{ "GRBG", V4L2_PIX_FMT_SGRBG8},
+	{ "GBRG", V4L2_PIX_FMT_SGBRG8},
+	{ "BA81", V4L2_PIX_FMT_SBGGR8},
+	{ "RG10", V4L2_PIX_FMT_SRGGB10},
+	{ "BA10", V4L2_PIX_FMT_SGRBG10},
+	{ "GB10", V4L2_PIX_FMT_SGBRG10},
+	{ "BG10", V4L2_PIX_FMT_SBGGR10},
+	{ "RG12", V4L2_PIX_FMT_SRGGB12},
+	{ "BA12", V4L2_PIX_FMT_SGRBG12},
+	{ "GB12", V4L2_PIX_FMT_SGBRG12},
+	{ "BG12", V4L2_PIX_FMT_SBGGR12},
+	{ "BYR2", V4L2_PIX_FMT_SBGGR16},
+	{ "Y16 ", V4L2_PIX_FMT_Y16},
+};
+
+static const char *es_dvp2axi_pixelcode_to_string(u32 mbus_code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(mbus_formats); ++i) {
+		if (mbus_formats[i].mbus_code == mbus_code)
+			return mbus_formats[i].name;
+	}
+
+	return "unknown";
+}
+
+static const char *es_dvp2axi_get_monitor_mode(enum es_dvp2axi_monitor_mode monitor_mode)
+{
+	switch (monitor_mode) {
+	case ES_DVP2AXI_MONITOR_MODE_IDLE:
+		return "idle";
+	case ES_DVP2AXI_MONITOR_MODE_CONTINUE:
+		return "continue";
+	case ES_DVP2AXI_MONITOR_MODE_TRIGGER:
+		return "trigger";
+	case ES_DVP2AXI_MONITOR_MODE_HOTPLUG:
+		return "hotplug";
+	default:
+		return "unknown";
+	}
+}
+
+static void es_dvp2axi_show_mixed_info(struct es_dvp2axi_device *dev, struct seq_file *f)
+{
+	enum es_dvp2axi_monitor_mode monitor_mode;
+
+	seq_printf(f, "Driver Version:v%02x.%02x.%02x\n",
+		   ES_DVP2AXI_DRIVER_VERSION >> 16,
+		   (ES_DVP2AXI_DRIVER_VERSION & 0xff00) >> 8,
+		    ES_DVP2AXI_DRIVER_VERSION & 0x00ff);
+	seq_printf(f, "Work Mode:%s\n",
+		   dev->workmode == ES_DVP2AXI_WORKMODE_ONEFRAME ? "one frame" :
+		   dev->workmode == ES_DVP2AXI_WORKMODE_PINGPONG ? "ping pong" : "line loop");
+
+	monitor_mode = dev->reset_watchdog_timer.monitor_mode;
+	seq_printf(f, "Monitor Mode:%s\n",
+		   es_dvp2axi_get_monitor_mode(monitor_mode));
+}
+
+static void es_dvp2axi_show_clks(struct es_dvp2axi_device *dev, struct seq_file *f)
+{
+	int i;
+	struct es_dvp2axi_hw *hw = dev->hw_dev;
+
+	for (i = 0; i < hw->clk_size; i++) {
+		seq_printf(f, "%s:%ld\n",
+			   hw->match_data->clks[i],
+			   clk_get_rate(hw->clks[i]));
+	}
+}
+
+static void es_dvp2axi_show_format(struct es_dvp2axi_device *dev, struct seq_file *f)
+{
+	struct es_dvp2axi_stream *stream = &dev->stream[0];
+	struct es_dvp2axi_pipeline *pipe = &dev->pipe;
+	struct es_dvp2axi_sensor_info *sensor = &dev->terminal_sensor;
+	struct v4l2_rect *rect = &sensor->raw_rect;
+	struct v4l2_subdev_frame_interval *interval = &sensor->fi;
+	struct v4l2_subdev_selection *sel = &sensor->selection;
+	u32 mbus_flags;
+	u64 fps, timestamp0, timestamp1;
+	unsigned long flags;
+	u32 time_val = 0;
+
+	if (atomic_read(&pipe->stream_cnt) < 1)
+		return;
+
+	if (sensor) {
+		seq_puts(f, "Input Info:\n");
+
+		seq_printf(f, "\tsrc subdev:%s\n", sensor->sd->name);
+		if (sensor->mbus.type == V4L2_MBUS_PARALLEL ||
+		    sensor->mbus.type == V4L2_MBUS_BT656) {
+			mbus_flags = sensor->mbus.bus.parallel.flags;
+			seq_printf(f, "\tinterface:%s\n",
+				   sensor->mbus.type == V4L2_MBUS_PARALLEL ? "BT601" : "BT656/BT1120");
+			seq_printf(f, "\thref_pol:%s\n",
+				   mbus_flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH ? "high active" : "low active");
+			seq_printf(f, "\tvsync_pol:%s\n",
+				   mbus_flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH ? "high active" : "low active");
+		} else {
+			seq_printf(f, "\tinterface:%s\n",
+				   sensor->mbus.type == V4L2_MBUS_CSI2_DPHY ? "mipi csi2 dphy" :
+				   sensor->mbus.type == V4L2_MBUS_CSI2_CPHY ? "mipi csi2 cphy" :
+				   sensor->mbus.type == V4L2_MBUS_CCP2 ? "lvds" : "unknown");
+			seq_printf(f, "\tlanes:%d\n", sensor->lanes);
+		}
+
+		seq_printf(f, "\thdr mode: %s\n",
+			   dev->hdr.hdr_mode == NO_HDR ? "normal" :
+			   dev->hdr.hdr_mode == HDR_COMPR ? "hdr_compr" :
+			   dev->hdr.hdr_mode == HDR_X2 ? "hdr_x2" : "hdr_x3");
+
+		seq_printf(f, "\tformat:%s/%ux%u@%d\n",
+			   es_dvp2axi_pixelcode_to_string(stream->dvp2axi_fmt_in->mbus_code),
+			   rect->width, rect->height,
+			   interval->interval.denominator / interval->interval.numerator);
+
+		seq_printf(f, "\tcrop.bounds:(%u, %u)/%ux%u\n",
+			   sel->r.left, sel->r.top,
+			   sel->r.width, sel->r.height);
+
+		spin_lock_irqsave(&stream->fps_lock, flags);
+		timestamp0 = stream->fps_stats.frm0_timestamp;
+		timestamp1 = stream->fps_stats.frm1_timestamp;
+		spin_unlock_irqrestore(&stream->fps_lock, flags);
+		fps = timestamp0 > timestamp1 ?
+		      timestamp0 - timestamp1 : timestamp1 - timestamp0;
+		fps = div_u64(fps, 1000000);
+
+		seq_puts(f, "Output Info:\n");
+		seq_printf(f, "\tformat:%s/%ux%u(%u,%u)\n",
+			   es_dvp2axi_pixelcode_to_string(stream->dvp2axi_fmt_out->fourcc),
+			   dev->channels[0].width, dev->channels[0].height,
+			   dev->channels[0].crop_st_x, dev->channels[0].crop_st_y);
+		seq_printf(f, "\tcompact:%s\n", stream->is_compact ? "enable" : "disabled");
+		seq_printf(f, "\tframe amount:%d\n", stream->frame_idx - 1);
+		if (dev->inf_id == ES_DVP2AXI_MIPI_LVDS) {
+			time_val = div_u64(stream->readout.early_time, 1000000);
+			seq_printf(f, "\tearly:%u ms\n", time_val);
+			if (dev->hdr.hdr_mode == NO_HDR ||
+			    dev->hdr.hdr_mode == HDR_COMPR) {
+				time_val = div_u64(stream->readout.readout_time, 1000000);
+				seq_printf(f, "\treadout:%u ms\n", time_val);
+			} else {
+				time_val = div_u64(stream->readout.readout_time, 1000000);
+				seq_printf(f, "\tsingle readout:%u ms\n", time_val);
+				time_val = div_u64(stream->readout.total_time, 1000000);
+				seq_printf(f, "\ttotal readout:%u ms\n", time_val);
+
+			}
+		}
+		seq_printf(f, "\trate:%llu ms\n", fps);
+		fps = div_u64(1000, fps);
+		seq_printf(f, "\tfps:%llu\n", fps);
+		seq_puts(f, "\tirq statistics:\n");
+		seq_printf(f, "\t\t\ttotal:%llu\n",
+			   dev->irq_stats.frm_end_cnt[0] +
+			   dev->irq_stats.frm_end_cnt[1] +
+			   dev->irq_stats.frm_end_cnt[2] +
+			   dev->irq_stats.frm_end_cnt[3] +
+			   dev->irq_stats.all_err_cnt);
+		if (sensor->mbus.type == V4L2_MBUS_PARALLEL ||
+		    sensor->mbus.type == V4L2_MBUS_BT656) {
+			seq_printf(f, "\t\t\tdvp bus err:%llu\n", dev->irq_stats.dvp_bus_err_cnt);
+			seq_printf(f, "\t\t\tdvp pix err:%llu\n", dev->irq_stats.dvp_pix_err_cnt);
+			seq_printf(f, "\t\t\tdvp line err:%llu\n", dev->irq_stats.dvp_line_err_cnt);
+			seq_printf(f, "\t\t\tdvp over flow:%llu\n", dev->irq_stats.dvp_overflow_cnt);
+			seq_printf(f, "\t\t\tdvp bandwidth lack:%llu\n",
+				   dev->irq_stats.dvp_bwidth_lack_cnt);
+			seq_printf(f, "\t\t\tdvp size err:%llu\n", dev->irq_stats.dvp_size_err_cnt);
+		} else {
+			seq_printf(f, "\t\t\tcsi over flow:%llu\n", dev->irq_stats.csi_overflow_cnt);
+			seq_printf(f, "\t\t\tcsi bandwidth lack:%llu\n",
+				   dev->irq_stats.csi_bwidth_lack_cnt);
+			seq_printf(f, "\t\t\tcsi size err:%llu\n", dev->irq_stats.csi_size_err_cnt);
+		}
+		seq_printf(f, "\t\t\tnot active buf cnt:%llu %llu %llu %llu\n",
+			   dev->irq_stats.not_active_buf_cnt[0],
+			   dev->irq_stats.not_active_buf_cnt[1],
+			   dev->irq_stats.not_active_buf_cnt[2],
+			   dev->irq_stats.not_active_buf_cnt[3]);
+		seq_printf(f, "\t\t\tall err count:%llu\n", dev->irq_stats.all_err_cnt);
+		seq_printf(f, "\t\t\tframe dma end:%llu %llu %llu %llu\n",
+			   dev->irq_stats.frm_end_cnt[0],
+			   dev->irq_stats.frm_end_cnt[1],
+			   dev->irq_stats.frm_end_cnt[2],
+			   dev->irq_stats.frm_end_cnt[3]);
+		seq_printf(f, "irq time: %llu ns\n", dev->hw_dev->irq_time);
+		seq_printf(f, "dma enable: 0x%x 0x%x 0x%x 0x%x\n",
+			   dev->stream[0].dma_en, dev->stream[1].dma_en,
+			   dev->stream[2].dma_en, dev->stream[3].dma_en);
+		seq_printf(f, "buf_cnt in drv: %d %d %d %d\n",
+			   atomic_read(&dev->stream[0].buf_cnt),
+			   atomic_read(&dev->stream[1].buf_cnt),
+			   atomic_read(&dev->stream[2].buf_cnt),
+			   atomic_read(&dev->stream[3].buf_cnt));
+		seq_printf(f, "total buf_cnt: %d %d %d %d\n",
+			   dev->stream[0].total_buf_num,
+			   dev->stream[1].total_buf_num,
+			   dev->stream[2].total_buf_num,
+			   dev->stream[3].total_buf_num);
+	}
+}
+
+static int es_dvp2axi_proc_show(struct seq_file *f, void *v)
+{
+	struct es_dvp2axi_device *dev = f->private;
+
+	if (dev) {
+		es_dvp2axi_show_mixed_info(dev, f);
+		es_dvp2axi_show_clks(dev, f);
+		es_dvp2axi_show_format(dev, f);
+	} else {
+		seq_puts(f, "dev null\n");
+	}
+
+	return 0;
+}
+
+static int es_dvp2axi_proc_open(struct inode *inode, struct file *file)
+{
+	struct es_dvp2axi_device *data = pde_data(inode);
+
+	return single_open(file, es_dvp2axi_proc_show, data);
+}
+
+static const struct proc_ops es_dvp2axi_proc_fops = {
+	.proc_open = es_dvp2axi_proc_open,
+	.proc_release = single_release,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+};
+
+int es_dvp2axi_proc_init(struct es_dvp2axi_device *dev)
+{
+
+	dev->proc_dir = proc_create_data(dev_name(dev->dev), 0444,
+					 NULL, &es_dvp2axi_proc_fops,
+					 dev);
+	if (!dev->proc_dir) {
+		dev_err(dev->dev, "create proc/%s failed!\n",
+			dev_name(dev->dev));
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+void es_dvp2axi_proc_cleanup(struct es_dvp2axi_device *dev)
+{
+	remove_proc_entry(dev_name(dev->dev), NULL);
+}
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/procfs.h b/drivers/media/platform/eswin/dvp2axi/procfs.h
new file mode 100644
index 000000000000..02a0123730fc
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/procfs.h
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI procfs driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_PROCFS_H
+#define _ES_DVP2AXI_PROCFS_H
+
+#ifdef CONFIG_PROC_FS
+
+int es_dvp2axi_proc_init(struct es_dvp2axi_device *dev);
+void es_dvp2axi_proc_cleanup(struct es_dvp2axi_device *dev);
+
+#else
+
+static inline int es_dvp2axi_proc_init(struct esisp_device *dev)
+{
+	return 0;
+}
+static inline void es_dvp2axi_proc_cleanup(struct esisp_device *dev)
+{
+
+}
+
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/regs.h b/drivers/media/platform/eswin/dvp2axi/regs.h
new file mode 100644
index 000000000000..4e7a1cea81af
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/regs.h
@@ -0,0 +1,1070 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI regs driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_REGS_H
+#define _ES_DVP2AXI_REGS_H
+
+struct dvp2axi_reg {
+	u32 offset;
+};
+
+#define DVP2AXI_REG(_offset)		{ .offset = (_offset), }
+
+enum dvp2axi_reg_index {
+	/* dvp registers index */
+	DVP2AXI_REG_DVP_CTRL = 0x0,
+	DVP2AXI_REG_DVP_INTEN,
+	DVP2AXI_REG_DVP_INTSTAT,
+	DVP2AXI_REG_DVP_FOR,
+	DVP2AXI_REG_DVP_LINE_NUM_ADDR,
+	DVP2AXI_REG_DVP_DMA_IDLE_REQ,
+	DVP2AXI_REG_DVP_MULTI_ID,
+	DVP2AXI_REG_DVP_FRM0_ADDR_Y,
+	DVP2AXI_REG_DVP_FRM0_ADDR_UV,
+	DVP2AXI_REG_DVP_FRM1_ADDR_Y,
+	DVP2AXI_REG_DVP_FRM1_ADDR_UV,
+	DVP2AXI_REG_DVP_VIR_LINE_WIDTH,
+	DVP2AXI_REG_DVP_SET_SIZE,
+	DVP2AXI_REG_DVP_SCM_ADDR_Y,
+	DVP2AXI_REG_DVP_SCM_ADDR_U,
+	DVP2AXI_REG_DVP_SCM_ADDR_V,
+	DVP2AXI_REG_DVP_WB_UP_FILTER,
+	DVP2AXI_REG_DVP_WB_LOW_FILTER,
+	DVP2AXI_REG_DVP_WBC_CNT,
+	DVP2AXI_REG_DVP_LINE_INT_NUM,
+	DVP2AXI_REG_DVP_LINE_CNT,
+	DVP2AXI_REG_DVP_CROP,
+	DVP2AXI_REG_DVP_SCL_CTRL,
+	DVP2AXI_REG_DVP_SCL_DST,
+	DVP2AXI_REG_DVP_SCL_FCT,
+	DVP2AXI_REG_DVP_SCL_VALID_NUM,
+	DVP2AXI_REG_DVP_LINE_LOOP_CTRL,
+	DVP2AXI_REG_DVP_PATH_SEL,
+	DVP2AXI_REG_DVP_FIFO_ENTRY,
+	DVP2AXI_REG_DVP_FRAME_STATUS,
+	DVP2AXI_REG_DVP_CUR_DST,
+	DVP2AXI_REG_DVP_LAST_LINE,
+	DVP2AXI_REG_DVP_LAST_PIX,
+	DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID1,
+	DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID1,
+	DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID1,
+	DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID1,
+	DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID2,
+	DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID2,
+	DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID2,
+	DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID2,
+	DVP2AXI_REG_DVP_FRM0_ADDR_Y_ID3,
+	DVP2AXI_REG_DVP_FRM0_ADDR_UV_ID3,
+	DVP2AXI_REG_DVP_FRM1_ADDR_Y_ID3,
+	DVP2AXI_REG_DVP_FRM1_ADDR_UV_ID3,
+	DVP2AXI_REG_DVP_SAV_EAV,
+	DVP2AXI_REG_DVP_LINE_CNT1,
+	DVP2AXI_REG_DVP_LINE_INT_NUM1,
+	/* mipi & lvds registers index */
+	DVP2AXI_REG_MIPI_LVDS_ID0_CTRL0,
+	DVP2AXI_REG_MIPI_LVDS_ID0_CTRL1,
+	DVP2AXI_REG_MIPI_LVDS_ID1_CTRL0,
+	DVP2AXI_REG_MIPI_LVDS_ID1_CTRL1,
+	DVP2AXI_REG_MIPI_LVDS_ID2_CTRL0,
+	DVP2AXI_REG_MIPI_LVDS_ID2_CTRL1,
+	DVP2AXI_REG_MIPI_LVDS_ID3_CTRL0,
+	DVP2AXI_REG_MIPI_LVDS_ID3_CTRL1,
+	DVP2AXI_REG_MIPI_WATER_LINE,
+	DVP2AXI_REG_MIPI_LVDS_CTRL,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID0,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID1,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID2,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_Y_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_Y_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_ADDR_UV_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_ADDR_UV_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_Y_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_Y_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME0_VLW_UV_ID3,
+	DVP2AXI_REG_MIPI_LVDS_FRAME1_VLW_UV_ID3,
+	DVP2AXI_REG_MIPI_LVDS_INTEN,
+	DVP2AXI_REG_MIPI_LVDS_INTSTAT,
+	DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID0_1,
+	DVP2AXI_REG_MIPI_LVDS_LINE_INT_NUM_ID2_3,
+	DVP2AXI_REG_MIPI_LVDS_LINE_LINE_CNT_ID0_1,
+	DVP2AXI_REG_MIPI_LVDS_LINE_LINE_CNT_ID2_3,
+	DVP2AXI_REG_MIPI_LVDS_ID0_CROP_START,
+	DVP2AXI_REG_MIPI_LVDS_ID1_CROP_START,
+	DVP2AXI_REG_MIPI_LVDS_ID2_CROP_START,
+	DVP2AXI_REG_MIPI_LVDS_ID3_CROP_START,
+	DVP2AXI_REG_MIPI_FRAME_NUM_VC0,
+	DVP2AXI_REG_MIPI_FRAME_NUM_VC1,
+	DVP2AXI_REG_MIPI_FRAME_NUM_VC2,
+	DVP2AXI_REG_MIPI_FRAME_NUM_VC3,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID0,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID0,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID0,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID0,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID1,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID1,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID1,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID1,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID2,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID2,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID2,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID2,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT0_ID3,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK0_ID3,
+	DVP2AXI_REG_LVDS_SAV_EAV_ACT1_ID3,
+	DVP2AXI_REG_LVDS_SAV_EAV_BLK1_ID3,
+	DVP2AXI_REG_MIPI_EFFECT_CODE_ID0,
+	DVP2AXI_REG_MIPI_EFFECT_CODE_ID1,
+	DVP2AXI_REG_MIPI_EFFECT_CODE_ID2,
+	DVP2AXI_REG_MIPI_EFFECT_CODE_ID3,
+	DVP2AXI_REG_LVDS_ID0_CTRL0,
+	DVP2AXI_REG_LVDS_ID1_CTRL0,
+	DVP2AXI_REG_LVDS_ID2_CTRL0,
+	DVP2AXI_REG_LVDS_ID3_CTRL0,
+	DVP2AXI_REG_MIPI_FRAME_SIZE_ID0,
+	DVP2AXI_REG_MIPI_FRAME_SIZE_ID1,
+	DVP2AXI_REG_MIPI_FRAME_SIZE_ID2,
+	DVP2AXI_REG_MIPI_FRAME_SIZE_ID3,
+	DVP2AXI_REG_MIPI_ON_PAD,
+
+	DVP2AXI_REG_Y_STAT_CONTROL,
+	DVP2AXI_REG_Y_STAT_VALUE,
+	DVP2AXI_REG_MMU_DTE_ADDR,
+	DVP2AXI_REG_MMU_STATUS,
+	DVP2AXI_REG_MMU_COMMAND,
+	DVP2AXI_REG_MMU_PAGE_FAULT_ADDR,
+	DVP2AXI_REG_MMU_ZAP_ONE_LINE,
+	DVP2AXI_REG_MMU_INT_RAWSTAT,
+	DVP2AXI_REG_MMU_INT_CLEAR,
+	DVP2AXI_REG_MMU_INT_MASK,
+	DVP2AXI_REG_MMU_INT_STATUS,
+	DVP2AXI_REG_MMU_AUTO_GATING,
+	/* reg belowed is in grf */
+	DVP2AXI_REG_GRF_DVP2AXIIO_CON,
+	DVP2AXI_REG_GRF_DVP2AXIIO_CON1,
+	DVP2AXI_REG_GRF_DVP2AXIIO_VENC,
+	/* reg global control */
+	DVP2AXI_REG_GLB_CTRL,
+	DVP2AXI_REG_GLB_INTEN,
+	DVP2AXI_REG_GLB_INTST,
+	DVP2AXI_REG_SCL_CH_CTRL,
+	DVP2AXI_REG_SCL_CTRL,
+	DVP2AXI_REG_SCL_FRM0_ADDR_CH0,
+	DVP2AXI_REG_SCL_FRM1_ADDR_CH0,
+	DVP2AXI_REG_SCL_VLW_CH0,
+	DVP2AXI_REG_SCL_FRM0_ADDR_CH1,
+	DVP2AXI_REG_SCL_FRM1_ADDR_CH1,
+	DVP2AXI_REG_SCL_VLW_CH1,
+	DVP2AXI_REG_SCL_FRM0_ADDR_CH2,
+	DVP2AXI_REG_SCL_FRM1_ADDR_CH2,
+	DVP2AXI_REG_SCL_VLW_CH2,
+	DVP2AXI_REG_SCL_FRM0_ADDR_CH3,
+	DVP2AXI_REG_SCL_FRM1_ADDR_CH3,
+	DVP2AXI_REG_SCL_VLW_CH3,
+	DVP2AXI_REG_SCL_BLC_CH0,
+	DVP2AXI_REG_SCL_BLC_CH1,
+	DVP2AXI_REG_SCL_BLC_CH2,
+	DVP2AXI_REG_SCL_BLC_CH3,
+	DVP2AXI_REG_TOISP0_CTRL,
+	DVP2AXI_REG_TOISP0_SIZE,
+	DVP2AXI_REG_TOISP0_CROP,
+	DVP2AXI_REG_TOISP1_CTRL,
+	DVP2AXI_REG_TOISP1_SIZE,
+	DVP2AXI_REG_TOISP1_CROP,
+	DVP2AXI_REG_INDEX_MAX
+};
+
+/* DVP2AXI Reg Offset */
+#define DVP2AXI_CTRL			0x00
+#define DVP2AXI_INTEN			0x04
+#define DVP2AXI_INTSTAT			0x08
+#define DVP2AXI_FOR				0x0c
+#define DVP2AXI_LINE_NUM_ADDR		0x10
+#define DVP2AXI_DMA_IDLE_REQ		0x10
+#define DVP2AXI_FRM0_ADDR_Y			0x14
+#define DVP2AXI_FRM0_ADDR_UV		0x18
+#define DVP2AXI_FRM1_ADDR_Y			0x1c
+#define DVP2AXI_FRM1_ADDR_UV		0x20
+#define DVP2AXI_VIR_LINE_WIDTH		0x24
+#define DVP2AXI_SET_SIZE			0x28
+#define DVP2AXI_SCM_ADDR_Y			0x2c
+#define DVP2AXI_LINE_INT_NUM		0x2c
+#define DVP2AXI_SCM_ADDR_U			0x30
+#define DVP2AXI_LINE_CNT			0x30
+#define DVP2AXI_SCM_ADDR_V			0x34
+#define DVP2AXI_WB_UP_FILTER		0x38
+#define DVP2AXI_WB_LOW_FILTER		0x3c
+#define DVP2AXI_WBC_CNT			0x40
+#define DVP2AXI_CROP			0x44
+#define RV1126_DVP2AXI_CROP			0x34
+#define ES3568_DVP2AXI_FIFO_ENTRY		0x38
+#define DVP2AXI_SCL_CTRL			0x48
+#define DVP2AXI_PATH_SEL			0x48
+#define DVP2AXI_SCL_DST			0x4c
+#define DVP2AXI_SCL_FCT			0x50
+#define DVP2AXI_SCL_VALID_NUM		0x54
+#define DVP2AXI_FIFO_ENTRY			0x54
+#define DVP2AXI_LINE_LOOP_CTR		0x58
+#define DVP2AXI_FRAME_STATUS		0x60
+#define RV1126_DVP2AXI_FRAME_STATUS		0x3c
+#define DVP2AXI_CUR_DST			0x64
+#define RV1126_DVP2AXI_CUR_DST		0x40
+#define DVP2AXI_LAST_LINE			0x68
+#define RV1126_DVP2AXI_LAST_LINE		0x44
+#define DVP2AXI_LAST_PIX			0x6c
+#define RV1126_DVP2AXI_LAST_PIX		0x48
+#define DVP2AXI_MULTI_ID			0x10
+#define DVP2AXI_FRM0_ADDR_Y_ID1		0x50
+#define DVP2AXI_FRM0_ADDR_UV_ID1		0x54
+#define DVP2AXI_FRM1_ADDR_Y_ID1		0x58
+#define DVP2AXI_FRM1_ADDR_UV_ID1		0x5c
+#define DVP2AXI_FRM0_ADDR_Y_ID2		0x60
+#define DVP2AXI_FRM0_ADDR_UV_ID2		0x64
+#define DVP2AXI_FRM1_ADDR_Y_ID2		0x68
+#define DVP2AXI_FRM1_ADDR_UV_ID2		0x6c
+#define DVP2AXI_FRM0_ADDR_Y_ID3		0x70
+#define DVP2AXI_FRM0_ADDR_UV_ID3		0x74
+#define DVP2AXI_FRM1_ADDR_Y_ID3		0x78
+#define DVP2AXI_FRM1_ADDR_UV_ID3		0x7c
+
+#define DVP2AXI_FETCH_Y_LAST_LINE(val)	((val) & 0x1fff)
+/* Check if swap y and c in bt1120 mode */
+#define DVP2AXI_FETCH_IS_Y_FIRST(val)	((val >> 5) & 0x3)
+#define DVP2AXI_RAW_STORED_BIT_WIDTH	(16U)
+#define DVP2AXI_RAW_STORED_BIT_WIDTH_RV1126	(8U)
+#define DVP2AXI_YUV_STORED_BIT_WIDTH	(8U)
+
+/* ES1808 & RV1126 DVP2AXI CSI & LVDS Registers Offset */
+#define DVP2AXI_CSI_ID0_CTRL0		0x80
+#define DVP2AXI_CSI_ID0_CTRL1		0x84
+#define DVP2AXI_CSI_ID1_CTRL0		0x88
+#define DVP2AXI_CSI_ID1_CTRL1		0x8c
+#define DVP2AXI_CSI_ID2_CTRL0		0x90
+#define DVP2AXI_CSI_ID2_CTRL1		0x94
+#define DVP2AXI_CSI_ID3_CTRL0		0x98
+#define DVP2AXI_CSI_ID3_CTRL1		0x9c
+#define DVP2AXI_CSI_WATER_LINE		0xa0
+#define DVP2AXI_CSI_MIPI_LVDS_CTRL		0xa0
+#define DVP2AXI_CSI_FRM0_ADDR_Y_ID0		0xa4
+#define DVP2AXI_CSI_FRM1_ADDR_Y_ID0		0xa8
+#define DVP2AXI_CSI_FRM0_ADDR_UV_ID0	0xac
+#define DVP2AXI_CSI_FRM1_ADDR_UV_ID0	0xb0
+#define DVP2AXI_CSI_FRM0_VLW_Y_ID0		0xb4
+#define DVP2AXI_CSI_FRM1_VLW_Y_ID0		0xb8
+#define DVP2AXI_CSI_FRM0_VLW_UV_ID0		0xbc
+#define DVP2AXI_CSI_FRM1_VLW_UV_ID0		0xc0
+#define DVP2AXI_CSI_FRM0_ADDR_Y_ID1		0xc4
+#define DVP2AXI_CSI_FRM1_ADDR_Y_ID1		0xc8
+#define DVP2AXI_CSI_FRM0_ADDR_UV_ID1	0xcc
+#define DVP2AXI_CSI_FRM1_ADDR_UV_ID1	0xd0
+#define DVP2AXI_CSI_FRM0_VLW_Y_ID1		0xd4
+#define DVP2AXI_CSI_FRM1_VLW_Y_ID1		0xd8
+#define DVP2AXI_CSI_FRM0_VLW_UV_ID1		0xdc
+#define DVP2AXI_CSI_FRM1_VLW_UV_ID1		0xe0
+#define DVP2AXI_CSI_FRM0_ADDR_Y_ID2		0xe4
+#define DVP2AXI_CSI_FRM1_ADDR_Y_ID2		0xe8
+#define DVP2AXI_CSI_FRM0_ADDR_UV_ID2	0xec
+#define DVP2AXI_CSI_FRM1_ADDR_UV_ID2	0xf0
+#define DVP2AXI_CSI_FRM0_VLW_Y_ID2		0xf4
+#define DVP2AXI_CSI_FRM1_VLW_Y_ID2		0xf8
+#define DVP2AXI_CSI_FRM0_VLW_UV_ID2		0xfc
+#define DVP2AXI_CSI_FRM1_VLW_UV_ID2		0x100
+#define DVP2AXI_CSI_FRM0_ADDR_Y_ID3		0x104
+#define DVP2AXI_CSI_FRM1_ADDR_Y_ID3		0x108
+#define DVP2AXI_CSI_FRM0_ADDR_UV_ID3	0x10c
+#define DVP2AXI_CSI_FRM1_ADDR_UV_ID3	0x110
+#define DVP2AXI_CSI_FRM0_VLW_Y_ID3		0x114
+#define DVP2AXI_CSI_FRM1_VLW_Y_ID3		0x118
+#define DVP2AXI_CSI_FRM0_VLW_UV_ID3		0x11c
+#define DVP2AXI_CSI_FRM1_VLW_UV_ID3		0x120
+#define DVP2AXI_CSI_INTEN			0x124
+#define DVP2AXI_CSI_INTSTAT			0x128
+#define DVP2AXI_CSI_LINE_INT_NUM_ID0_1	0x12c
+#define DVP2AXI_CSI_LINE_INT_NUM_ID2_3	0x130
+#define DVP2AXI_CSI_LINE_CNT_ID0_1		0x134
+#define DVP2AXI_CSI_LINE_CNT_ID2_3		0x138
+#define DVP2AXI_CSI_ID0_CROP_START		0x13c
+#define DVP2AXI_CSI_ID1_CROP_START		0x140
+#define DVP2AXI_CSI_ID2_CROP_START		0x144
+#define DVP2AXI_CSI_ID3_CROP_START		0x148
+#define DVP2AXI_CSI_FRAME_NUM_VC0		0x14c
+#define DVP2AXI_CSI_FRAME_NUM_VC1		0x150
+#define DVP2AXI_CSI_FRAME_NUM_VC2		0x154
+#define DVP2AXI_CSI_FRAME_NUM_VC3		0x158
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID0	0x150
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID0	0x154
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID0	0x158
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID0	0x15c
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID1	0x160
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID1	0x164
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID1	0x168
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID1	0x16c
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID2	0x170
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID2	0x174
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID2	0x178
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID2	0x17c
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID3	0x180
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID3	0x184
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID3	0x188
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID3	0x18c
+#define DVP2AXI_Y_STAT_CONTROL		0x190
+#define DVP2AXI_Y_STAT_VALUE		0x194
+#define DVP2AXI_MMU_DTE_ADDR		0x800
+#define DVP2AXI_MMU_STATUS			0x804
+#define DVP2AXI_MMU_COMMAND			0x808
+#define DVP2AXI_MMU_PAGE_FAULT_ADDR		0x80c
+#define DVP2AXI_MMU_ZAP_ONE_LINE		0x810
+#define DVP2AXI_MMU_INT_RAWSTAT		0x814
+#define DVP2AXI_MMU_INT_CLEAR		0x818
+#define DVP2AXI_MMU_INT_MASK		0x81c
+#define DVP2AXI_MMU_INT_STATUS		0x820
+#define DVP2AXI_MMU_AUTO_GATING		0x824
+
+/* EIC770X DVP Registers Offset */
+#define DVP_CTRL			0x10
+#define DVP_INTEN			0x14
+#define DVP_INTSTAT			0x18
+#define DVP_FOR				0x1C
+#define DVP_MULTI_ID			0x20
+#define DVP_SAV_EAV			0x24
+#define DVP_CROP_SIZE			0x28
+#define DVP_CROP			0x2C
+#define DVP_FRM0_ADDR_Y_ID0		0x30
+#define DVP_FRM0_ADDR_UV_ID0		0x34
+#define DVP_FRM1_ADDR_Y_ID0		0x38
+#define DVP_FRM1_ADDR_UV_ID0		0x3C
+#define DVP_FRM0_ADDR_Y_ID1		0x40
+#define DVP_FRM0_ADDR_UV_ID1		0x44
+#define DVP_FRM1_ADDR_Y_ID1		0x48
+#define DVP_FRM1_ADDR_UV_ID1		0x4C
+#define DVP_FRM0_ADDR_Y_ID2		0x50
+#define DVP_FRM0_ADDR_UV_ID2		0x54
+#define DVP_FRM1_ADDR_Y_ID2		0x58
+#define DVP_FRM1_ADDR_UV_ID2		0x5C
+#define DVP_FRM0_ADDR_Y_ID3		0x60
+#define DVP_FRM0_ADDR_UV_ID3		0x64
+#define DVP_FRM1_ADDR_Y_ID3		0x68
+#define DVP_FRM1_ADDR_UV_ID3		0x6C
+#define DVP_VIR_LINE_WIDTH		0x70
+#define DVP_LINE_INT_NUM_01		0x74
+#define DVP_LINE_INT_NUM_23		0x78
+#define DVP_LINE_CNT_01			0x7C
+#define DVP_LINE_CNT_23			0x80
+
+/* EIC770X CSI Registers Offset */
+#define CSI_MIPI0_ID0_CTRL0		0x100
+#define CSI_MIPI0_ID0_CTRL1		0x104
+#define CSI_MIPI0_ID1_CTRL0		0x108
+#define CSI_MIPI0_ID1_CTRL1		0x10C
+#define CSI_MIPI0_ID2_CTRL0		0x110
+#define CSI_MIPI0_ID2_CTRL1		0x114
+#define CSI_MIPI0_ID3_CTRL0		0x118
+#define CSI_MIPI0_ID3_CTRL1		0x11C
+#define CSI_MIPI0_CTRL			0x120
+#define CSI_MIPI0_FRM0_ADDR_Y_ID0	0x124
+#define CSI_MIPI0_FRM1_ADDR_Y_ID0	0x128
+#define CSI_MIPI0_FRM0_ADDR_UV_ID0	0x12C
+#define CSI_MIPI0_FRM1_ADDR_UV_ID0	0x130
+#define CSI_MIPI0_VLW_ID0		0x134
+#define CSI_MIPI0_FRM0_ADDR_Y_ID1	0x138
+#define CSI_MIPI0_FRM1_ADDR_Y_ID1	0x13C
+#define CSI_MIPI0_FRM0_ADDR_UV_ID1	0x140
+#define CSI_MIPI0_FRM1_ADDR_UV_ID1	0x144
+#define CSI_MIPI0_VLW_ID1		0x148
+#define CSI_MIPI0_FRM0_ADDR_Y_ID2	0x14C
+#define CSI_MIPI0_FRM1_ADDR_Y_ID2	0x150
+#define CSI_MIPI0_FRM0_ADDR_UV_ID2	0x154
+#define CSI_MIPI0_FRM1_ADDR_UV_ID2	0x158
+#define CSI_MIPI0_VLW_ID2		0x15C
+#define CSI_MIPI0_FRM0_ADDR_Y_ID3	0x160
+#define CSI_MIPI0_FRM1_ADDR_Y_ID3	0x164
+#define CSI_MIPI0_FRM0_ADDR_UV_ID3	0x168
+#define CSI_MIPI0_FRM1_ADDR_UV_ID3	0x16C
+#define CSI_MIPI0_VLW_ID3		0x170
+#define CSI_MIPI0_INTEN			0x174
+#define CSI_MIPI0_INTSTAT		0x178
+#define CSI_MIPI0_LINE_INT_NUM_ID0_1	0x17C
+#define CSI_MIPI0_LINE_INT_NUM_ID2_3	0x180
+#define CSI_MIPI0_LINE_CNT_ID0_1	0x184
+#define CSI_MIPI0_LINE_CNT_ID2_3	0x188
+#define CSI_MIPI0_ID0_CROP_START	0x18C
+#define CSI_MIPI0_ID1_CROP_START	0x190
+#define CSI_MIPI0_ID2_CROP_START	0x194
+#define CSI_MIPI0_ID3_CROP_START	0x198
+#define CSI_MIPI0_FRAME_NUM_VC0		0x19C
+#define CSI_MIPI0_FRAME_NUM_VC1		0x1A0
+#define CSI_MIPI0_FRAME_NUM_VC2		0x1A4
+#define CSI_MIPI0_FRAME_NUM_VC3		0x1A8
+#define CSI_MIPI0_EFFECT_CODE_ID0	0x1AC
+#define CSI_MIPI0_EFFECT_CODE_ID1	0x1B0
+#define CSI_MIPI0_EFFECT_CODE_ID2	0x1B4
+#define CSI_MIPI0_EFFECT_CODE_ID3	0x1B8
+#define CSI_MIPI0_ON_PAD		0x1BC
+
+#define CSI_MIPI0_FRAME_SIZE_ID0	0x1C0
+#define CSI_MIPI0_FRAME_SIZE_ID1	0x1C4
+#define CSI_MIPI0_FRAME_SIZE_ID2	0x1C8
+#define CSI_MIPI0_FRAME_SIZE_ID3	0x1CC
+
+/* RV1106 CONTROL Registers Offset */
+#define DVP2AXI_LVDS0_ID0_CTRL0		0x1D0
+#define DVP2AXI_LVDS0_ID1_CTRL0		0x1D4
+#define DVP2AXI_LVDS0_ID2_CTRL0		0x1D8
+#define DVP2AXI_LVDS0_ID3_CTRL0		0x1DC
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID0_RV1106	0x1E0
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID0_RV1106	0x1E4
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID0_RV1106	0x1E8
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID0_RV1106	0x1EC
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID1_RV1106	0x1F0
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID1_RV1106	0x1F4
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID1_RV1106	0x1F8
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID1_RV1106	0x1FC
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID2_RV1106	0x200
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID2_RV1106	0x204
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID2_RV1106	0x208
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID2_RV1106	0x20C
+#define DVP2AXI_LVDS_SAV_EAV_ACT0_ID3_RV1106	0x210
+#define DVP2AXI_LVDS_SAV_EAV_BLK0_ID3_RV1106	0x214
+#define DVP2AXI_LVDS_SAV_EAV_ACT1_ID3_RV1106	0x218
+#define DVP2AXI_LVDS_SAV_EAV_BLK1_ID3_RV1106	0x21C
+
+/* EIC770X CONTROL Registers Offset */
+#define GLB_CTRL			0X000
+#define GLB_INTEN			0X004
+#define GLB_INTST			0X008
+#define SCL_CH_CTRL			0x700
+#define SCL_CTRL			0x704
+#define SCL_FRM0_ADDR_CH0		0x708
+#define SCL_FRM1_ADDR_CH0		0x70C
+#define SCL_VLW_CH0			0x710
+#define SCL_FRM0_ADDR_CH1		0x714
+#define SCL_FRM1_ADDR_CH1		0x718
+#define SCL_VLW_CH1			0x71C
+#define SCL_FRM0_ADDR_CH2		0x720
+#define SCL_FRM1_ADDR_CH2		0x724
+#define SCL_VLW_CH2			0x728
+#define SCL_FRM0_ADDR_CH3		0x72C
+#define SCL_FRM1_ADDR_CH3		0x730
+#define SCL_VLW_CH3			0x734
+#define SCL_BLC_CH0			0x738
+#define SCL_BLC_CH1			0x73C
+#define SCL_BLC_CH2			0x740
+#define SCL_BLC_CH3			0x744
+#define TOISP0_CH_CTRL			0x780
+#define TOISP0_CROP_SIZE		0x784
+#define TOISP0_CROP			0x788
+#define TOISP1_CH_CTRL			0x78C
+#define TOISP1_CROP_SIZE		0x790
+#define TOISP1_CROP			0x794
+
+/* The key register bit description */
+
+/* DVP2AXI_CTRL Reg */
+#define DISABLE_CAPTURE			(0x0 << 0)
+#define ENABLE_CAPTURE			(0x1 << 0)
+#define MODE_ONEFRAME			(0x0 << 1)
+#define MODE_PINGPONG			(0x1 << 1)
+#define MODE_LINELOOP			(0x2 << 1)
+#define AXI_BURST_16			(0xF << 12)
+#define DVP_PRESS_EN			(0x1 << 12)
+#define DVP_HURRY_EN			(0x1 << 8)
+#define DVP_DMA_EN			(0x1 << 1)
+#define DVP_SW_WATER_LINE_75		(0x0 << 5)
+#define DVP_SW_WATER_LINE_50		(0x1 << 5)
+#define DVP_SW_WATER_LINE_25		(0x2 << 5)
+#define DVP_SW_WATER_LINE_00		(0x3 << 5)
+
+/* DVP2AXI_INTEN */
+#define INTEN_DISABLE			(0x0 << 0)
+#define FRAME_END_EN			(0x1 << 0)
+#define BUS_ERR_EN			(0x1 << 6)
+#define SCL_ERR_EN			(0x1 << 7)
+#define PRE_INF_FRAME_END_EN		(0x1 << 8)
+#define PST_INF_FRAME_END_EN		(0x1 << 9)
+#define LINE_INT_EN			(0x1 << 10)
+#define DVP_CHANNEL1_FRM_END_EN		(0x1 << 11)
+#define DVP_CHANNEL2_FRM_END_EN		(0x1 << 12)
+#define DVP_CHANNEL3_FRM_END_EN		(0x1 << 13)
+
+/* DVP2AXI INTSTAT */
+#define INTSTAT_CLS			(0x3FF)
+#define FRAME_END			(0x01 << 0)
+#define LINE_ERR			(0x01 << 2)
+#define PIX_ERR				(0x01 << 3)
+#define IFIFO_OVERFLOW			(0x01 << 4)
+#define DFIFO_OVERFLOW			(0x01 << 5)
+#define BUS_ERR				(0x01 << 6)
+#define PRE_INF_FRAME_END		(0x01 << 8)
+#define PST_INF_FRAME_END		(0x01 << 9)
+#define LINE_INT_END			(0x01 << 10)
+#define FRAME_END_CLR			(0x01 << 0)
+#define PRE_INF_FRAME_END_CLR		(0x01 << 8)
+#define PST_INF_FRAME_END_CLR		(0x01 << 9)
+#define INTSTAT_ERR			(0xFC)
+#define INTSTAT_ERR_EIC770X		(DVP_SIZE_ERR |\
+					 DVP_FIFO_OVERFLOW |\
+					 DVP_BANDWIDTH_LACK)
+
+#define DVP_ALL_OVERFLOW		(IFIFO_OVERFLOW | DFIFO_OVERFLOW)
+
+#define DVP_FIFO_OVERFLOW		(0x01 << 16)
+#define DVP_BANDWIDTH_LACK		(0x01 << 17)
+
+#define DVP_SIZE_ERR_ID0		(0x1 << 22)
+#define DVP_SIZE_ERR_ID1		(0x1 << 23)
+#define DVP_SIZE_ERR_ID2		(0x1 << 24)
+#define DVP_SIZE_ERR_ID3		(0x1 << 25)
+
+#define DVP_SIZE_ERR			(DVP_SIZE_ERR_ID0 |\
+					 DVP_SIZE_ERR_ID1 |\
+					 DVP_SIZE_ERR_ID2 |\
+					 DVP_SIZE_ERR_ID3)
+
+#define DVP_SW_PRESS_VALUE(val)		(((val) & 0x7) << 13)
+#define DVP_SW_HURRY_VALUE(val)		(((val) & 0x7) << 9)
+#define DVP_SW_CAP_EN(ID)		(2 << ID)
+#define DVP_SW_DMA_EN(ID)		(0x100000 << ID)
+#define DVP_START_INTSTAT(ID)		(0x3 << ((ID) * 2))
+
+#define DVP_DMA_END_INTEN(id)	\
+	({ \
+	unsigned int mask; \
+	switch (id) { \
+	case 0: \
+		mask = 0x1 << 0; \
+		break; \
+	default: \
+		mask = 0x1 << (id  + 10); \
+		break; \
+	} \
+	mask; \
+	})
+
+#define DVP_LINE_INTEN			(0x01 << 10)
+
+#define DVP_DMA_END_INTSTAT(id)		\
+	({ \
+	unsigned int mask; \
+	switch (id) { \
+	case 0: \
+		mask = 0x1 << 0; \
+		break; \
+	default: \
+		mask = 0x1 << (id  + 10); \
+		break; \
+	} \
+	mask; \
+	})
+
+#define DVP_PST_INTSTAT			PST_INF_FRAME_END
+#define DVP_LINE_INTSTAT		(0x01 << 10)
+
+/* FRAME STATUS */
+#define FRAME_STAT_CLS			0x00
+/* write 0 to clear frame 0 */
+#define FRM0_STAT_CLS			0xfffffffe
+#define FRAME_NUM_SHIFT			(16U)
+#define FRAME_NUM_MASK			(0xffff << FRAME_NUM_SHIFT)
+#define DVP2AXI_GET_FRAME_ID(val)		(((val) & FRAME_NUM_MASK) >> FRAME_NUM_SHIFT)
+
+/* DVP2AXI FORMAT */
+#define VSY_HIGH_ACTIVE			(0x01 << 0)
+#define VSY_LOW_ACTIVE			(0x00 << 0)
+#define HSY_LOW_ACTIVE			(0x01 << 1)
+#define HSY_HIGH_ACTIVE			(0x00 << 1)
+#define INPUT_MODE_YUV			(0x00 << 2)
+#define INPUT_MODE_PAL			(0x02 << 2)
+#define INPUT_MODE_BT656_YUV422		(0x02 << 2)
+#define INPUT_MODE_NTSC			(0x03 << 2)
+#define INPUT_MODE_BT1120		(0x07 << 2)
+#define INPUT_MODE_RAW			(0x04 << 2)
+#define INPUT_MODE_JPEG			(0x05 << 2)
+#define INPUT_MODE_SONY_RAW		(0x05 << 2)
+#define INPUT_MODE_MIPI			(0x06 << 2)
+#define YUV_INPUT_ORDER_UYVY		(0x00 << 5)
+#define YUV_INPUT_ORDER_YVYU		(0x01 << 5)
+#define YUV_INPUT_ORDER_VYUY		(0x10 << 5)
+#define YUV_INPUT_ORDER_YUYV		(0x03 << 5)
+#define YUV_INPUT_422			(0x00 << 7)
+#define YUV_INPUT_420			(0x01 << 7)
+#define INPUT_420_ORDER_EVEN		(0x00 << 8)
+#define INPUT_420_ORDER_ODD		(0x01 << 8)
+#define CCIR_INPUT_ORDER_ODD		(0x00 << 9)
+#define CCIR_INPUT_ORDER_EVEN		(0x01 << 9)
+#define RAW_DATA_WIDTH_8		(0x00 << 11)
+#define RAW_DATA_WIDTH_10		(0x01 << 11)
+#define RAW_DATA_WIDTH_12		(0x02 << 11)
+#define MIPI_MODE_32BITS_BYPASS		(0x00 << 13)
+#define MIPI_MODE_RGB			(0x01 << 13)
+#define MIPI_MODE_YUV			(0x02 << 13)
+#define YUV_OUTPUT_422			(0x00 << 16)
+#define YUV_OUTPUT_420			(0x01 << 16)
+#define OUTPUT_420_ORDER_EVEN		(0x00 << 17)
+#define OUTPUT_420_ORDER_ODD		(0x01 << 17)
+#define RAWD_DATA_LITTLE_ENDIAN		(0x00 << 18)
+#define RAWD_DATA_BIG_ENDIAN		(0x01 << 18)
+#define UV_STORAGE_ORDER_UVUV		(0x00 << 19)
+#define UV_STORAGE_ORDER_VUVU		(0x01 << 19)
+#define BT1120_CLOCK_SINGLE_EDGES	(0x00 << 24)
+#define BT1120_CLOCK_DOUBLE_EDGES	(0x01 << 24)
+#define BT1120_TRANSMIT_INTERFACE	(0x00 << 25)
+#define BT1120_TRANSMIT_PROGRESS	(0x01 << 25)
+#define BT1120_YC_SWAP			(0x01 << 26)
+#define BT656_1120_MULTI_ID_DISABLE	(0x00 << 28)
+#define BT656_1120_MULTI_ID_ENABLE	(0x01 << 28)
+#define BT656_1120_MULTI_ID_SEL_MSB	(0x00 << 29)
+#define BT656_1120_MULTI_ID_SEL_LSB	(0x01 << 29)
+#define BT656_1120_MULTI_ID_MODE_1	(0x00 << 30)
+#define BT656_1120_MULTI_ID_MODE_2	(0x01 << 30)
+#define BT656_1120_MULTI_ID_MODE_4	(0x02 << 30)
+#define BT656_1120_MULTI_ID_0_MASK	~(0x03 << 4)
+#define BT656_1120_MULTI_ID_1_MASK	~(0x03 << 12)
+#define BT656_1120_MULTI_ID_2_MASK	~(0x03 << 20)
+#define BT656_1120_MULTI_ID_3_MASK	~(0x03 << 28)
+#define	DVP2AXI_HIGH_ALIGN			(0x01 << 18)
+#define	DVP2AXI_HIGH_ALIGN_EIC770X		(0x01 << 21)
+#define BT656_DETECT_SAV		(0X01 << 13)
+#define BT656_DETECT_SAV_EAV		(0X00 << 13)
+
+#define BT1120_CLOCK_SINGLE_EDGES_EIC770X	(0x00 << 11)
+#define BT1120_CLOCK_DOUBLE_EDGES_EIC770X	(0x01 << 11)
+#define TRANSMIT_INTERFACE_EIC770X		(0x01 << 9)
+#define TRANSMIT_PROGRESS_EIC770X		(0x00 << 9)
+#define BT1120_YC_SWAP_EIC770X			(0x01 << 12)
+#define INPUT_BT601_YUV422			(0x00 << 2)
+#define INPUT_BT601_RAW				(0x01 << 2)
+#define INPUT_BT656_YUV422			(0x02 << 2)
+#define INPUT_BT1120_YUV422			(0x03 << 2)
+#define INPUT_SONY_RAW				(0x04 << 2)
+
+/* DVP2AXI_SCL_CTRL */
+#define ENABLE_SCL_DOWN			(0x01 << 0)
+#define DISABLE_SCL_DOWN		(0x00 << 0)
+#define ENABLE_SCL_UP			(0x01 << 1)
+#define DISABLE_SCL_UP			(0x00 << 1)
+#define ENABLE_YUV_16BIT_BYPASS		(0x01 << 4)
+#define DISABLE_YUV_16BIT_BYPASS	(0x00 << 4)
+#define ENABLE_RAW_16BIT_BYPASS		(0x01 << 5)
+#define DISABLE_RAW_16BIT_BYPASS	(0x00 << 5)
+#define ENABLE_32BIT_BYPASS		(0x01 << 6)
+#define DISABLE_32BIT_BYPASS		(0x00 << 6)
+
+/* DVP2AXI_FRAME_INTSTAT */
+#define DVP2AXI_F0_READY			(0x01 << 0)
+#define DVP2AXI_F1_READY			(0x01 << 1)
+#define DVP_CHANNEL0_FRM_READ		(DVP2AXI_F0_READY | DVP2AXI_F1_READY)
+#define DVP_CHANNEL1_F0_READY		(0x01 << 4)
+#define DVP_CHANNEL1_F1_READY		(0x01 << 5)
+#define DVP_CHANNEL1_FRM_READ		(DVP_CHANNEL1_F0_READY | DVP_CHANNEL1_F1_READY)
+#define DVP_CHANNEL2_F0_READY		(0x01 << 8)
+#define DVP_CHANNEL2_F1_READY		(0x01 << 9)
+#define DVP_CHANNEL2_FRM_READ		(DVP_CHANNEL2_F0_READY | DVP_CHANNEL2_F1_READY)
+#define DVP_CHANNEL3_F0_READY		(0x01 << 12)
+#define DVP_CHANNEL3_F1_READY		(0x01 << 13)
+#define DVP_CHANNEL3_FRM_READ		(DVP_CHANNEL3_F0_READY | DVP_CHANNEL3_F1_READY)
+
+#define DVP_FRAME0_START_ID0		(0x1 << 0)
+#define DVP_FRAME1_START_ID0		(0x1 << 1)
+
+#define DVP_FRAME_END_ID0		(0x1 << 0)
+#define DVP_FRAME_END_ID1		(0x1 << 11)
+#define DVP_FRAME_END_ID2		(0x1 << 12)
+#define DVP_FRAME_END_ID3		(0x1 << 13)
+
+#define DVP_FRAME0_END_ID0		(0x1 << 8)
+#define DVP_FRAME1_END_ID0		(0x1 << 9)
+#define DVP_ALL_END_ID0			(DVP_FRAME0_END_ID0 | DVP_FRAME1_END_ID0)
+
+#define DVP_FRAME0_END_ID1		(0x1 << 10)
+#define DVP_FRAME1_END_ID1		(0x1 << 11)
+#define DVP_ALL_END_ID1			(DVP_FRAME0_END_ID1 | DVP_FRAME1_END_ID1)
+
+#define DVP_FRAME0_END_ID2		(0x1 << 12)
+#define DVP_FRAME1_END_ID2		(0x1 << 13)
+#define DVP_ALL_END_ID2			(DVP_FRAME0_END_ID2 | DVP_FRAME1_END_ID2)
+
+#define DVP_FRAME0_END_ID3		(0x1 << 14)
+#define DVP_FRAME1_END_ID3		(0x1 << 15)
+#define DVP_ALL_END_ID3			(DVP_FRAME0_END_ID3 | DVP_FRAME1_END_ID3)
+
+#define DVP_ALIGN_MSB			(0x01 << 21)
+#define DVP_ALIGN_LSB			(0x00 << 21)
+
+#define DVP_FRM_STS_ID0(x)		(((x) & (0x3 << 0)) >> 0)
+#define DVP_FRM_STS_ID1(x)		(((x) & (0x3 << 4)) >> 4)
+#define DVP_FRM_STS_ID2(x)		(((x) & (0x3 << 8)) >> 8)
+#define DVP_FRM_STS_ID3(x)		(((x) & (0x3 << 12)) >> 12)
+
+#define DVP_SW_MULTI_ID(channel, id, bits)	\
+	({ \
+		unsigned int mask; \
+		switch (channel) { \
+		case 0: \
+			mask = ((bits) << 4) | ((id) << 0); \
+			break; \
+		case 1: \
+			mask = ((bits) << 12) | ((id) << 8); \
+			break; \
+		case 2: \
+			mask = ((bits) << 20) | ((id) << 16); \
+			break; \
+		case 3: \
+			mask = ((bits) << 28) | ((id) << 24); \
+			break; \
+		default: \
+			mask = ((bits) << 4) | ((id) << 0); \
+			break; \
+		} \
+		mask; \
+	})
+
+/* DVP2AXI CROP */
+#define DVP2AXI_CROP_Y_SHIFT		16
+#define DVP2AXI_CROP_X_SHIFT		0
+
+/* DVP2AXI SCALE*/
+#define SCALE_END_INTSTAT(ch)		(0x3 << ((ch + 1) * 2))
+#define SCALE_FIFO_OVERFLOW(ch)		(1 << (10 + ch))
+#define SCALE_TOISP_AXI0_ERR		(1 << 0)
+#define SCALE_TOISP_AXI1_ERR		(1 << 1)
+#define DVP2AXI_SCALE_SW_PRESS_VALUE(val)	(((val) & 0x7) << 13)
+#define DVP2AXI_SCALE_SW_PRESS_ENABLE	(0x1 << 12)
+#define DVP2AXI_SCALE_SW_HURRY_VALUE(val)	(((val) & 0x7) << 5)
+#define DVP2AXI_SCALE_SW_HURRY_ENABLE	(0x1 << 4)
+#define DVP2AXI_SCALE_SW_WATER_LINE(val)	(val << 1)
+#define DVP2AXI_SCALE_SW_SRC_CH(val, ch)	((val & 0x1f) << (3 + ch * 8))
+#define DVP2AXI_SCALE_SW_MODE(val, ch)	((val & 0x3) << (1 + ch * 8))
+#define DVP2AXI_SCALE_EN(ch)		(1 << (ch * 8))
+#define SW_SCALE_END(intstat, ch)	((intstat >> ((ch + 1) * 2)) & 0x3)
+#define SCALE_SOFT_RESET(ch)		(0x1 << (ch + 16))
+
+/* DVP2AXI TOISP*/
+#define DVP2AXI_TOISP0_FS(ch)		(BIT(14) << ch)
+#define DVP2AXI_TOISP1_FS(ch)		(BIT(17) << ch)
+#define DVP2AXI_TOISP0_FE(ch)		(BIT(20) << ch)
+#define DVP2AXI_TOISP1_FE(ch)		(BIT(23) << ch)
+
+/* DVP2AXI_CSI_ID_CTRL0 */
+#define CSI_DISABLE_CAPTURE		(0x0 << 0)
+#define CSI_ENABLE_CAPTURE		(0x1 << 0)
+#define CSI_WRDDR_TYPE_RAW8		(0x0 << 1)
+#define CSI_WRDDR_TYPE_RAW10		(0x1 << 1)
+#define CSI_WRDDR_TYPE_RAW12		(0x2 << 1)
+#define CSI_WRDDR_TYPE_RGB888		(0x3 << 1)
+#define CSI_WRDDR_TYPE_YUV422		(0x4 << 1)
+#define CSI_WRDDR_TYPE_YUV420SP		(0x5 << 1)
+#define CSI_WRDDR_TYPE_YUV400		(0x6 << 1)
+#define CSI_WRDDR_TYPE_RGB565		(0x7 << 1)
+#define CSI_DISABLE_COMMAND_MODE	(0x0 << 4)
+#define CSI_ENABLE_COMMAND_MODE		(0x1 << 4)
+#define CSI_DISABLE_CROP		(0x0 << 5)
+#define CSI_ENABLE_CROP			(0x1 << 5)
+#define CSI_DISABLE_CROP_V1		(0x0 << 4)
+#define CSI_ENABLE_CROP_V1		(0x1 << 4)
+#define CSI_ENABLE_MIPI_COMPACT		(0x1 << 6)
+#define CSI_YUV_INPUT_ORDER_UYVY	(0x0 << 16)
+#define CSI_YUV_INPUT_ORDER_VYUY	(0x1 << 16)
+#define CSI_YUV_INPUT_ORDER_YUYV	(0x2 << 16)
+#define CSI_YUV_INPUT_ORDER_YVYU	(0x3 << 16)
+#define CSI_HIGH_ALIGN			(0x1 << 31)
+#define CSI_HIGH_ALIGN_EIC770X		(0x1 << 27)
+
+#define CSI_YUV_OUTPUT_ORDER_UYVY	(0x0 << 18)
+#define CSI_YUV_OUTPUT_ORDER_VYUY	(0x1 << 18)
+#define CSI_YUV_OUTPUT_ORDER_YUYV	(0x2 << 18)
+#define CSI_YUV_OUTPUT_ORDER_YVYU	(0x3 << 18)
+#define CSI_WRDDR_TYPE_RAW_COMPACT	(0x0 << 5)
+#define CSI_WRDDR_TYPE_RAW_UNCOMPACT	(0x1 << 5)
+#define CSI_WRDDR_TYPE_YUV_PACKET	(0x2 << 5)
+#define CSI_WRDDR_TYPE_YUV400_EIC770X	(0x3 << 5)
+#define CSI_WRDDR_TYPE_YUV422SP_EIC770X	(0x4 << 5)
+#define CSI_WRDDR_TYPE_YUV420SP_EIC770X	(0x5 << 5)
+#define CSI_ALIGN_MSB			(0x01 << 27)
+#define CSI_ALIGN_LSB			(0x0 << 27)
+#define CSI_DMA_ENABLE			(0x1 << 28)
+
+#define CSI_NO_HDR			(0X0 << 22)
+#define CSI_HDR2			(0X1 << 22)
+#define CSI_HDR3			(0X2 << 22)
+
+#define CSI_HDR_MODE_VC			(0x0 << 20)
+#define CSI_HDR_MODE_LINE_CNT		(0x1 << 20)
+#define CSI_HDR_MODE_LINE_INFO		(0x2 << 20)
+#define CSI_HDR_VC_MODE_PROTECT		(0x1 << 29)
+
+#define LVDS_ENABLE_CAPTURE		(0x1 << 16)
+#define LVDS_MODE(mode)			(((mode) & 0x7) << 17)
+#define LVDS_LANES_ENABLED(lanes)	\
+	({ \
+		unsigned int mask; \
+		switch (lanes) { \
+		case 1: \
+			mask = 0x1 << 20; \
+			break; \
+		case 2: \
+			mask = 0x3 << 20; \
+			break; \
+		case 3: \
+			mask = 0x7 << 20; \
+			break; \
+		case 4: \
+			mask = 0xf << 20; \
+			break; \
+		default: \
+			mask = 0x1 << 20; \
+			break; \
+		} \
+		mask; \
+	})
+
+#define LVDS_MAIN_LANE(index)		(((index) & 0x3) << 24)
+#define LVDS_FID(id)			(((id) & 0x3) << 26)
+#define LVDS_HDR_FRAME_X2		(0x0 << 28)
+#define LVDS_HDR_FRAME_X3		(0x1 << 28)
+#define LVDS_COMPACT			(0x1 << 29)
+
+#define LVDS_ENABLE_CAPTURE_RV1106		(0x1 << 0)
+#define LVDS_MODE_RV1106(mode)			(((mode) & 0x7) << 1)
+#define LVDS_LANES_ENABLED_RV1106(lanes)	\
+	({ \
+		unsigned int mask; \
+		switch (lanes) { \
+		case 1: \
+			mask = 0x1 << 4; \
+			break; \
+		case 2: \
+			mask = 0x3 << 4; \
+			break; \
+		case 3: \
+			mask = 0x7 << 4; \
+			break; \
+		case 4: \
+			mask = 0xf << 4; \
+			break; \
+		default: \
+			mask = 0x1 << 4; \
+			break; \
+		} \
+		mask; \
+	})
+
+/* DVP2AXI_CSI_INTEN */
+#define CSI_FRAME1_START_INTEN(id)	(0x1 << ((id) * 2 + 1))
+#define CSI_FRAME0_END_INTEN(id)	(0x1 << ((id) * 2 + 8))
+#define CSI_FRAME1_END_INTEN(id)	(0x1 << ((id) * 2 + 9))
+#define CSI_DMA_Y_FIFO_OVERFLOW_INTEN	(0x1 << 16)
+#define CSI_DMA_UV_FIFO_OVERFLOW_INTEN	(0x1 << 17)
+#define CSI_CONFIG_FIFO_OVERFLOW_INTEN	(0x1 << 18)
+#define CSI_BANDWIDTH_LACK_INTEN	(0x1 << 19)
+#define CSI_RX_FIFO_OVERFLOW_INTEN	(0x1 << 20)
+#define CSI_ALL_FRAME_START_INTEN	(0xff << 0)
+#define CSI_ALL_FRAME_END_INTEN		(0xff << 8)
+#define CSI_ALL_ERROR_INTEN		(0x1f << 16)
+#define CSI_ALL_ERROR_INTEN_V1		(0xf0f << 16)
+
+#define CSI_START_INTEN(id)		(0x3 << ((id) * 2))
+#define CSI_DMA_END_INTEN(id)		(0x3 << ((id) * 2 + 8))
+#define CSI_LINE_INTEN(id)		(0x1 << ((id) + 21))
+#define CSI_LINE_INTEN_EIC770X(id)	(0x1 << ((id) + 20))
+
+#define CSI_START_INTSTAT(id)		(0x3 << ((id) * 2))
+#define CSI_DMA_END_INTSTAT(id)		(0x3 << ((id) * 2 + 8))
+#define CSI_LINE_INTSTAT(id)		(0x1 << ((id) + 21))
+#define CSI_LINE_INTSTAT_V1(id)		(0x1 << ((id) + 20))
+
+/* DVP2AXI_CSI_INTSTAT */
+#define CSI_FRAME0_START_ID0		(0x1 << 0)
+#define CSI_FRAME1_START_ID0		(0x1 << 1)
+#define CSI_FRAME0_START_ID1		(0x1 << 2)
+#define CSI_FRAME1_START_ID1		(0x1 << 3)
+#define CSI_FRAME0_START_ID2		(0x1 << 4)
+#define CSI_FRAME1_START_ID2		(0x1 << 5)
+#define CSI_FRAME0_START_ID3		(0x1 << 6)
+#define CSI_FRAME1_START_ID3		(0x1 << 7)
+#define CSI_FRAME0_END_ID0		(0x1 << 8)
+#define CSI_FRAME1_END_ID0		(0x1 << 9)
+#define CSI_FRAME0_END_ID1		(0x1 << 10)
+#define CSI_FRAME1_END_ID1		(0x1 << 11)
+#define CSI_FRAME0_END_ID2		(0x1 << 12)
+#define CSI_FRAME1_END_ID2		(0x1 << 13)
+#define CSI_FRAME0_END_ID3		(0x1 << 14)
+#define CSI_FRAME1_END_ID3		(0x1 << 15)
+#define CSI_DMA_Y_FIFO_OVERFLOW		(0x1 << 16)
+#define CSI_DMA_UV_FIFO_OVERFLOW	(0x1 << 17)
+#define CSI_CONFIG_FIFO_OVERFLOW	(0x1 << 18)
+#define CSI_BANDWIDTH_LACK		(0x1 << 19)
+#define CSI_RX_FIFO_OVERFLOW		(0x1 << 20)
+#define CSI_LINE_ID0_INTST		(0x1 << 21)
+#define CSI_LINE_ID1_INTST		(0x1 << 22)
+#define CSI_LINE_ID2_INTST		(0x1 << 23)
+#define CSI_LINE_ID3_INTST		(0x1 << 24)
+#define CSI_DMA_LVDS_ID2_FIFO_OVERFLOW	(0x1 << 25)
+#define CSI_DMA_LVDS_ID3_FIFO_OVERFLOW	(0x1 << 26)
+#define CSI_SIZE_ERR_ID0		(0x1 << 24)
+#define CSI_SIZE_ERR_ID1		(0x1 << 25)
+#define CSI_SIZE_ERR_ID2		(0x1 << 26)
+#define CSI_SIZE_ERR_ID3		(0x1 << 27)
+
+#define CSI_FRAME_START_ID0		(CSI_FRAME0_START_ID0 |\
+					 CSI_FRAME1_START_ID0)
+#define CSI_FRAME_START_ID1		(CSI_FRAME0_START_ID1 |\
+					 CSI_FRAME1_START_ID1)
+#define CSI_FRAME_START_ID2		(CSI_FRAME0_START_ID2 |\
+					 CSI_FRAME1_START_ID2)
+#define CSI_FRAME_START_ID3		(CSI_FRAME0_START_ID3 |\
+					 CSI_FRAME1_START_ID3)
+#define CSI_FRAME_END_ID0		(CSI_FRAME0_END_ID0 |\
+					 CSI_FRAME1_END_ID0)
+#define CSI_FRAME_END_ID1		(CSI_FRAME0_END_ID1 |\
+					 CSI_FRAME1_END_ID1)
+#define CSI_FRAME_END_ID2		(CSI_FRAME0_END_ID2 |\
+					 CSI_FRAME1_END_ID2)
+#define CSI_FRAME_END_ID3		(CSI_FRAME0_END_ID3 |\
+					 CSI_FRAME1_END_ID3)
+#define CSI_FIFO_OVERFLOW		(CSI_DMA_Y_FIFO_OVERFLOW |\
+					 CSI_DMA_UV_FIFO_OVERFLOW |\
+					 CSI_CONFIG_FIFO_OVERFLOW |\
+					 CSI_RX_FIFO_OVERFLOW |\
+					 CSI_DMA_LVDS_ID2_FIFO_OVERFLOW |\
+					 CSI_DMA_LVDS_ID3_FIFO_OVERFLOW)
+
+/*mask for es3588*/
+#define CSI_RX_FIFO_OVERFLOW_V1		(0x1 << 19)
+#define CSI_BANDWIDTH_LACK_V1		(0x1 << 18)
+#define CSI_ALL_ERROR_INTEN_V1		(0xf0f << 16)
+
+
+#define CSI_FIFO_OVERFLOW_V1		(CSI_DMA_Y_FIFO_OVERFLOW |\
+					 CSI_DMA_UV_FIFO_OVERFLOW |\
+					 CSI_RX_FIFO_OVERFLOW_V1)
+#define CSI_SIZE_ERR			(CSI_SIZE_ERR_ID0 |\
+					 CSI_SIZE_ERR_ID1 |\
+					 CSI_SIZE_ERR_ID2 |\
+					 CSI_SIZE_ERR_ID3)
+
+/* DVP2AXI_MIPI_LVDS_CTRL */
+#define DVP2AXI_MIPI_LVDS_SW_DMA_IDLE			(0x1 << 16)
+#define DVP2AXI_MIPI_LVDS_SW_PRESS_VALUE(val)		(((val) & 0x3) << 13)
+#define DVP2AXI_MIPI_LVDS_SW_PRESS_VALUE_EIC770X(val)	(((val) & 0x7) << 13)
+#define DVP2AXI_MIPI_LVDS_SW_PRESS_ENABLE			(0x1 << 12)
+#define DVP2AXI_MIPI_LVDS_SW_LVDS_WIDTH_8BITS		(0x0 << 9)
+#define DVP2AXI_MIPI_LVDS_SW_LVDS_WIDTH_10BITS		(0x1 << 9)
+#define DVP2AXI_MIPI_LVDS_SW_LVDS_WIDTH_12BITS		(0x2 << 9)
+#define DVP2AXI_MIPI_LVDS_SW_SEL_LVDS			(0x1 << 8)
+#define DVP2AXI_MIPI_LVDS_SW_SEL_LVDS_RV1106		(0x1 << 3)
+#define DVP2AXI_MIPI_LVDS_SW_HURRY_VALUE(val)		(((val) & 0x3) << 5)
+#define DVP2AXI_MIPI_LVDS_SW_HURRY_VALUE_EIC770X(val)	(((val) & 0x7) << 5)
+#define DVP2AXI_MIPI_LVDS_SW_HURRY_ENABLE			(0x1 << 4)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_75			(0x0 << 1)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_50			(0x1 << 1)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_25			(0x2 << 1)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_00			(0x3 << 1)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_ENABLE		(0x1 << 0)
+#define DVP2AXI_MIPI_LVDS_SW_DMA_IDLE_ES1808		(0x1 << 24)
+#define DVP2AXI_MIPI_LVDS_SW_HURRY_VALUE_ES1808(val)	(((val) & 0x3) << 17)
+#define DVP2AXI_MIPI_LVDS_SW_HURRY_ENABLE_ES1808		(0x1 << 16)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_75_ES1808		(0x0 << 0)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_50_ES1808		(0x1 << 0)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_25_ES1808		(0x2 << 0)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_00_ES1808		(0x3 << 0)
+#define DVP2AXI_MIPI_LVDS_SW_WATER_LINE_ENABLE_ES1808	(0x1 << 4)
+
+/* CSI Host Registers Define */
+#define CSIHOST_N_LANES		0x04
+#define CSIHOST_PHY_RSTZ	0x0c
+#define CSIHOST_RESETN		0x10
+#define CSIHOST_ERR1		0x20
+#define CSIHOST_ERR2		0x24
+#define CSIHOST_MSK1		0x28
+#define CSIHOST_MSK2		0x2c
+#define CSIHOST_CONTROL		0x40
+
+#define SW_CPHY_EN(x)		((x) << 0)
+#define SW_DSI_EN(x)		((x) << 4)
+#define SW_DATATYPE_FS(x)	((x) << 8)
+#define SW_DATATYPE_FE(x)	((x) << 14)
+#define SW_DATATYPE_LS(x)	((x) << 20)
+#define SW_DATATYPE_LE(x)	((x) << 26)
+
+#define SW_FRM_END_ID0(x)	(((x) & CSI_FRAME_END_ID0) >> 8)
+#define SW_FRM_END_ID1(x)	(((x) & CSI_FRAME_END_ID1) >> 10)
+#define SW_FRM_END_ID2(x)	(((x) & CSI_FRAME_END_ID2) >> 12)
+#define SW_FRM_END_ID3(x)	(((x) & CSI_FRAME_END_ID3) >> 14)
+
+/*RV1106 SKIP FUNC*/
+#define ES_DVP2AXI_CAP_SHIFT		0x18
+#define ES_DVP2AXI_SKIP_SHIFT	0X15
+#define ES_DVP2AXI_SKIP_EN(x)	(0x1 << (8 + x))
+
+/* DVP2AXI LVDS SAV EAV Define */
+#define SW_LVDS_EAV_ACT(code)	(((code) & 0xfff) << 16)
+#define SW_LVDS_SAV_ACT(code)	(((code) & 0xfff) << 0)
+#define SW_LVDS_EAV_BLK(code)	(((code) & 0xfff) << 16)
+#define SW_LVDS_SAV_BLK(code)	(((code) & 0xfff) << 0)
+
+/* GRF related with DVP2AXI */
+#define DVP2AXI_GRF_DVP2AXIIO_CON		(0x10250)
+#define DVP2AXI_PCLK_SAMPLING_EDGE_RISING	(0x04000400)
+#define DVP2AXI_PCLK_SAMPLING_EDGE_FALLING	(0x04000000)
+#define DVP2AXI_PCLK_DELAY_ENABLE		(0x02000200)
+#define DVP2AXI_PCLK_DELAY_DISABLE		(0x02000000)
+#define DVP2AXI_SAMPLING_EDGE_DOUBLE	(0x01000100)
+#define DVP2AXI_SAMPLING_EDGE_SINGLE	(0x01000000)
+#define DVP2AXI_PCLK_DELAY_NUM(num)		(0x00ff0000 | ((num) & 0xff))
+#define DVP2AXI_GRF_VI_CON0			(0x340)
+#define DVP2AXI_GRF_VI_CON1			(0x344)
+#define DVP2AXI_GRF_SOC_CON2			(0x308)
+#define EIC770X_DVP2AXI_PCLK_SAMPLING_EDGE_RISING	(0x00100000)
+#define EIC770X_DVP2AXI_PCLK_SAMPLING_EDGE_FALLING	(0x00100010)
+#define EIC770X_DVP2AXI_PCLK_SINGLE_EDGE		(0x00200000)
+#define EIC770X_DVP2AXI_PCLK_DUAL_EDGE		(0x00200020)
+
+/*toisp*/
+#define TOISP_FS_CH0(index)		(0x1 << (14 + index * 3))
+#define TOISP_FS_CH1(index)		(0x1 << (15 + index * 3))
+#define TOISP_FS_CH2(index)		(0x1 << (16 + index * 3))
+
+#define TOISP_END_CH0(index)		(0x1 << (20 + index * 3))
+#define TOISP_END_CH1(index)		(0x1 << (21 + index * 3))
+#define TOISP_END_CH2(index)		(0x1 << (22 + index * 3))
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/subdev-itf.c b/drivers/media/platform/eswin/dvp2axi/subdev-itf.c
new file mode 100644
index 000000000000..e933957ee236
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/subdev-itf.c
@@ -0,0 +1,1416 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI subdev driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/reset.h>
+#include <linux/pm_runtime.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/regmap.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include "dev.h"
+#include <linux/regulator/consumer.h>
+#include "../../../../phy/eswin/es-camera-module.h"
+#include "common.h"
+
+static inline struct sditf_priv *to_sditf_priv(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct sditf_priv, sd);
+}
+
+static void sditf_buffree_work(struct work_struct *work)
+{
+	struct sditf_work_struct *buffree_work =
+		container_of(work, struct sditf_work_struct, work);
+	struct sditf_priv *priv =
+		container_of(buffree_work, struct sditf_priv, buffree_work);
+	struct es_dvp2axi_rx_buffer *rx_buf = NULL;
+	unsigned long flags;
+	LIST_HEAD(local_list);
+
+	spin_lock_irqsave(&priv->dvp2axi_dev->buffree_lock, flags);
+	list_replace_init(&priv->buf_free_list, &local_list);
+	while (!list_empty(&local_list)) {
+		rx_buf = list_first_entry(&local_list, struct es_dvp2axi_rx_buffer,
+					  list_free);
+		if (rx_buf) {
+			list_del(&rx_buf->list_free);
+			es_dvp2axi_free_reserved_mem_buf(priv->dvp2axi_dev, rx_buf);
+		}
+	}
+	spin_unlock_irqrestore(&priv->dvp2axi_dev->buffree_lock, flags);
+}
+
+static void sditf_get_hdr_mode(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct esmodule_hdr_cfg hdr_cfg;
+	int ret = 0;
+
+	if (!dvp2axi_dev->terminal_sensor.sd)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->terminal_sensor.sd) {
+		ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, core, ioctl,
+				       ESMODULE_GET_HDR_CFG, &hdr_cfg);
+		if (!ret)
+			priv->hdr_cfg = hdr_cfg;
+		else
+			priv->hdr_cfg.hdr_mode = NO_HDR;
+	} else {
+		priv->hdr_cfg.hdr_mode = NO_HDR;
+	}
+}
+
+static int sditf_g_frame_interval(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_frame_interval *fi)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_subdev *sensor_sd;
+
+	if (!dvp2axi_dev->terminal_sensor.sd)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->terminal_sensor.sd) {
+		sensor_sd = dvp2axi_dev->terminal_sensor.sd;
+		return v4l2_subdev_call(sensor_sd, video, g_frame_interval, fi);
+	}
+
+	return -EINVAL;
+}
+
+static int sditf_g_mbus_config(struct v4l2_subdev *sd, unsigned int pad_id,
+			       struct v4l2_mbus_config *config)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_subdev *sensor_sd;
+
+	if (!dvp2axi_dev->active_sensor)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->active_sensor) {
+		sensor_sd = dvp2axi_dev->active_sensor->sd;
+		return v4l2_subdev_call(sensor_sd, pad, get_mbus_config, 0,
+					config);
+	}
+
+	return -EINVAL;
+}
+
+static int sditf_get_set_fmt(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_state *sd_state,
+			     struct v4l2_subdev_format *fmt)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_subdev_selection input_sel;
+	struct v4l2_pix_format_mplane pixm;
+	const struct dvp2axi_output_fmt *out_fmt;
+	int ret = -EINVAL;
+	bool is_uncompact = false;
+
+	if (!dvp2axi_dev->terminal_sensor.sd)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->terminal_sensor.sd) {
+		sditf_get_hdr_mode(priv);
+		fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		fmt->pad = 0;
+		ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, pad,
+				       get_fmt, NULL, fmt);
+		if (ret) {
+			v4l2_err(&priv->sd, "%s: get sensor format failed\n",
+				 __func__);
+			return ret;
+		}
+
+		input_sel.target = V4L2_SEL_TGT_CROP_BOUNDS;
+		input_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		input_sel.pad = 0;
+		ret = v4l2_subdev_call(dvp2axi_dev->terminal_sensor.sd, pad,
+				       get_selection, NULL, &input_sel);
+		if (!ret) {
+			fmt->format.width = input_sel.r.width;
+			fmt->format.height = input_sel.r.height;
+		}
+		priv->cap_info.width = fmt->format.width;
+		priv->cap_info.height = fmt->format.height;
+		pixm.pixelformat =
+			es_dvp2axi_mbus_pixelcode_to_v4l2(fmt->format.code);
+		pixm.width = priv->cap_info.width;
+		pixm.height = priv->cap_info.height;
+
+		out_fmt = es_dvp2axi_find_output_fmt(NULL, pixm.pixelformat);
+		if (priv->toisp_inf.link_mode == TOISP_UNITE &&
+		    ((pixm.width / 2 - ESMOUDLE_UNITE_EXTEND_PIXEL) *
+		     out_fmt->raw_bpp / 8) &
+			    0xf)
+			is_uncompact = true;
+
+		v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "%s, width %d, height %d, hdr mode %d\n", __func__,
+			 fmt->format.width, fmt->format.height,
+			 priv->hdr_cfg.hdr_mode);
+		if (priv->hdr_cfg.hdr_mode == NO_HDR ||
+		    priv->hdr_cfg.hdr_mode == HDR_COMPR) {
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[0], &pixm, false);
+		} else if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+			if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE &&
+			    priv->toisp_inf.link_mode == TOISP_UNITE) {
+				if (is_uncompact) {
+					dvp2axi_dev->stream[0].is_compact = false;
+					dvp2axi_dev->stream[0].is_high_align = true;
+				} else {
+					dvp2axi_dev->stream[0].is_compact = true;
+				}
+			}
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[0], &pixm, false);
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[1], &pixm, false);
+		} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+			if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE &&
+			    priv->toisp_inf.link_mode == TOISP_UNITE) {
+				if (is_uncompact) {
+					dvp2axi_dev->stream[0].is_compact = false;
+					dvp2axi_dev->stream[0].is_high_align = true;
+					dvp2axi_dev->stream[1].is_compact = false;
+					dvp2axi_dev->stream[1].is_high_align = true;
+				} else {
+					dvp2axi_dev->stream[0].is_compact = true;
+					dvp2axi_dev->stream[1].is_compact = true;
+				}
+			}
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[0], &pixm, false);
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[1], &pixm, false);
+			es_dvp2axi_set_fmt(&dvp2axi_dev->stream[2], &pixm, false);
+		}
+	} else {
+		if (priv->sensor_sd) {
+			fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
+			fmt->pad = 0;
+			ret = v4l2_subdev_call(priv->sensor_sd, pad, get_fmt,
+					       NULL, fmt);
+			if (ret) {
+				v4l2_err(&priv->sd,
+					 "%s: get sensor format failed\n",
+					 __func__);
+				return ret;
+			}
+
+			input_sel.target = V4L2_SEL_TGT_CROP_BOUNDS;
+			input_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+			input_sel.pad = 0;
+			ret = v4l2_subdev_call(priv->sensor_sd, pad,
+					       get_selection, NULL, &input_sel);
+			if (!ret) {
+				fmt->format.width = input_sel.r.width;
+				fmt->format.height = input_sel.r.height;
+			}
+			priv->cap_info.width = fmt->format.width;
+			priv->cap_info.height = fmt->format.height;
+			pixm.pixelformat =
+				es_dvp2axi_mbus_pixelcode_to_v4l2(fmt->format.code);
+			pixm.width = priv->cap_info.width;
+			pixm.height = priv->cap_info.height;
+		} else {
+			fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
+			fmt->pad = 0;
+			fmt->format.code = MEDIA_BUS_FMT_SBGGR10_1X10;
+			fmt->format.width = 3280;
+			fmt->format.height = 2464;
+		}
+	}
+
+	return 0;
+}
+
+static int sditf_init_buf(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	int ret = 0;
+
+	if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+		if (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO) {
+			if (dvp2axi_dev->is_thunderboot)
+				dvp2axi_dev->resmem_size /= 2;
+			ret = es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[0],
+						priv->buf_num);
+			if (dvp2axi_dev->is_thunderboot)
+				dvp2axi_dev->resmem_pa += dvp2axi_dev->resmem_size;
+			ret |= es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[1],
+						 priv->buf_num);
+		} else {
+			ret = es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[0],
+						priv->buf_num);
+		}
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+		if (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO) {
+			if (dvp2axi_dev->is_thunderboot)
+				dvp2axi_dev->resmem_size /= 3;
+			ret = es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[0],
+						priv->buf_num);
+			if (dvp2axi_dev->is_thunderboot)
+				dvp2axi_dev->resmem_pa += dvp2axi_dev->resmem_size;
+			ret |= es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[1],
+						 priv->buf_num);
+			if (dvp2axi_dev->is_thunderboot)
+				dvp2axi_dev->resmem_pa += dvp2axi_dev->resmem_size;
+			ret |= es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[2],
+						 priv->buf_num);
+		} else {
+			ret = es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[0],
+						priv->buf_num);
+			ret |= es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[1],
+						 priv->buf_num);
+		}
+	} else {
+		if (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO)
+			ret = es_dvp2axi_init_rx_buf(&dvp2axi_dev->stream[0],
+						priv->buf_num);
+		else
+			ret = -EINVAL;
+	}
+	return ret;
+}
+
+static void sditf_free_buf(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+
+	if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+		es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[0],
+				  dvp2axi_dev->stream[0].rx_buf_num);
+		es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[1],
+				  dvp2axi_dev->stream[1].rx_buf_num);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+		es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[0],
+				  dvp2axi_dev->stream[0].rx_buf_num);
+		es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[1],
+				  dvp2axi_dev->stream[1].rx_buf_num);
+		es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[2],
+				  dvp2axi_dev->stream[2].rx_buf_num);
+	} else {
+		es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[0],
+				  dvp2axi_dev->stream[0].rx_buf_num);
+	}
+	if (dvp2axi_dev->is_thunderboot) {
+		dvp2axi_dev->wait_line_cache = 0;
+		dvp2axi_dev->wait_line = 0;
+		dvp2axi_dev->wait_line_bak = 0;
+		dvp2axi_dev->is_thunderboot = false;
+	}
+}
+
+static int sditf_get_selection(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_state *sd_state,
+			       struct v4l2_subdev_selection *sel)
+{
+	return -EINVAL;
+}
+
+static void sditf_reinit_mode(struct sditf_priv *priv,
+			      struct esisp_vicap_mode *mode)
+{
+	if (mode->rdbk_mode == ESISP_VICAP_RDBK_AIQ) {
+		priv->toisp_inf.link_mode = TOISP_NONE;
+	} else {
+		if (strstr(mode->name, ESISP0_DEVNAME))
+			priv->toisp_inf.link_mode = TOISP0;
+		else if (strstr(mode->name, ESISP1_DEVNAME))
+			priv->toisp_inf.link_mode = TOISP1;
+		else if (strstr(mode->name, ESISP_UNITE_DEVNAME))
+			priv->toisp_inf.link_mode = TOISP_UNITE;
+		else
+			priv->toisp_inf.link_mode = TOISP0;
+	}
+
+	v4l2_dbg(1, es_dvp2axi_debug, &priv->dvp2axi_dev->v4l2_dev,
+		 "%s, mode->rdbk_mode %d, mode->name %s, link_mode %d\n",
+		 __func__, mode->rdbk_mode, mode->name,
+		 priv->toisp_inf.link_mode);
+}
+
+static void sditf_channel_disable(struct sditf_priv *priv, int user);
+static long sditf_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct esisp_vicap_mode *mode;
+	struct v4l2_subdev_format fmt;
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_subdev *sensor_sd;
+	int *pbuf_num = NULL;
+	int ret = 0;
+	int *on = NULL;
+
+	switch (cmd) {
+	case ESISP_VICAP_CMD_MODE:
+		mode = (struct esisp_vicap_mode *)arg;
+		mutex_lock(&dvp2axi_dev->stream_lock);
+		memcpy(&priv->mode, mode, sizeof(*mode));
+		mutex_unlock(&dvp2axi_dev->stream_lock);
+		sditf_reinit_mode(priv, &priv->mode);
+		if (priv->is_combine_mode)
+			mode->input.merge_num = dvp2axi_dev->sditf_cnt;
+		else
+			mode->input.merge_num = 1;
+		mode->input.index = priv->combine_index;
+		return 0;
+	case ESISP_VICAP_CMD_INIT_BUF:
+		pbuf_num = (int *)arg;
+		priv->buf_num = *pbuf_num;
+		sditf_get_set_fmt(&priv->sd, NULL, &fmt);
+		ret = sditf_init_buf(priv);
+		return ret;
+	case ESMODULE_GET_HDR_CFG:
+		if (!dvp2axi_dev->terminal_sensor.sd)
+			es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+		if (dvp2axi_dev->terminal_sensor.sd) {
+			sensor_sd = dvp2axi_dev->terminal_sensor.sd;
+			return v4l2_subdev_call(sensor_sd, core, ioctl, cmd,
+						arg);
+		}
+		break;
+	case ESISP_VICAP_CMD_QUICK_STREAM:
+		on = (int *)arg;
+		if (*on) {
+			es_dvp2axi_stream_resume(dvp2axi_dev, ES_DVP2AXI_RESUME_ISP);
+		} else {
+			if (priv->toisp_inf.link_mode == TOISP0) {
+				sditf_channel_disable(priv, 0);
+			} else if (priv->toisp_inf.link_mode == TOISP1) {
+				sditf_channel_disable(priv, 1);
+			} else if (priv->toisp_inf.link_mode == TOISP_UNITE) {
+				sditf_channel_disable(priv, 0);
+				sditf_channel_disable(priv, 1);
+			}
+			es_dvp2axi_stream_suspend(dvp2axi_dev, ES_DVP2AXI_RESUME_ISP);
+		}
+		break;
+	case ESISP_VICAP_CMD_SET_RESET:
+		if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE) {
+			dvp2axi_dev->is_toisp_reset = true;
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+#ifdef CONFIG_COMPAT
+static long sditf_compat_ioctl32(struct v4l2_subdev *sd, unsigned int cmd,
+				 unsigned long arg)
+{
+	void __user *up = compat_ptr(arg);
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_subdev *sensor_sd;
+	struct esisp_vicap_mode *mode;
+	struct esmodule_hdr_cfg *hdr_cfg;
+	int buf_num;
+	int ret = 0;
+	int on;
+
+	switch (cmd) {
+	case ESISP_VICAP_CMD_MODE:
+		mode = kzalloc(sizeof(*mode), GFP_KERNEL);
+		if (!mode) {
+			ret = -ENOMEM;
+			return ret;
+		}
+		if (copy_from_user(mode, up, sizeof(*mode))) {
+			kfree(mode);
+			return -EFAULT;
+		}
+		ret = sditf_ioctl(sd, cmd, mode);
+		kfree(mode);
+		return ret;
+	case ESISP_VICAP_CMD_INIT_BUF:
+		if (copy_from_user(&buf_num, up, sizeof(int)))
+			return -EFAULT;
+		ret = sditf_ioctl(sd, cmd, &buf_num);
+		return ret;
+	case ESMODULE_GET_HDR_CFG:
+		hdr_cfg = kzalloc(sizeof(*hdr_cfg), GFP_KERNEL);
+		if (!hdr_cfg) {
+			ret = -ENOMEM;
+			return ret;
+		}
+		if (copy_from_user(hdr_cfg, up, sizeof(*hdr_cfg))) {
+			kfree(hdr_cfg);
+			return -EFAULT;
+		}
+		ret = sditf_ioctl(sd, cmd, hdr_cfg);
+		return ret;
+	case ESISP_VICAP_CMD_QUICK_STREAM:
+		if (copy_from_user(&on, up, sizeof(int)))
+			return -EFAULT;
+		ret = sditf_ioctl(sd, cmd, &on);
+		return ret;
+	case ESISP_VICAP_CMD_SET_RESET:
+		ret = sditf_ioctl(sd, cmd, NULL);
+		return ret;
+	default:
+		break;
+	}
+
+	if (!dvp2axi_dev->terminal_sensor.sd)
+		es_dvp2axi_update_sensor_info(&dvp2axi_dev->stream[0]);
+
+	if (dvp2axi_dev->terminal_sensor.sd) {
+		sensor_sd = dvp2axi_dev->terminal_sensor.sd;
+		return v4l2_subdev_call(sensor_sd, core, compat_ioctl32, cmd,
+					arg);
+	}
+
+	return -EINVAL;
+}
+#endif
+
+static int sditf_channel_enable(struct sditf_priv *priv, int user)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct esmodule_capture_info *capture_info =
+		&dvp2axi_dev->channels[0].capture_info;
+	unsigned int ch0 = 0, ch1 = 0, ch2 = 0;
+	unsigned int ctrl_val = 0;
+	unsigned int int_en = 0;
+	unsigned int offset_x = 0;
+	unsigned int offset_y = 0;
+	unsigned int width = priv->cap_info.width;
+	unsigned int height = priv->cap_info.height;
+	int csi_idx = dvp2axi_dev->csi_host_idx;
+
+	if (capture_info->mode == ESMODULE_MULTI_DEV_COMBINE_ONE &&
+	    priv->toisp_inf.link_mode == TOISP_UNITE) {
+		if (capture_info->multi_dev.dev_num != 2 ||
+		    capture_info->multi_dev.pixel_offset !=
+			    ESMOUDLE_UNITE_EXTEND_PIXEL) {
+			v4l2_err(
+				&dvp2axi_dev->v4l2_dev,
+				"param error of online mode, combine dev num %d, offset %d\n",
+				capture_info->multi_dev.dev_num,
+				capture_info->multi_dev.pixel_offset);
+			return -EINVAL;
+		}
+		csi_idx = capture_info->multi_dev.dev_idx[user];
+	}
+
+	if (priv->hdr_cfg.hdr_mode == NO_HDR ||
+	    priv->hdr_cfg.hdr_mode == HDR_COMPR) {
+		if (dvp2axi_dev->inf_id == ES_DVP2AXI_MIPI_LVDS)
+			ch0 = csi_idx * 4;
+		else
+			ch0 = 24; //dvp
+		ctrl_val = (ch0 << 3) | 0x1;
+		if (user == 0)
+			int_en = DVP2AXI_TOISP0_FS(0) | DVP2AXI_TOISP0_FE(0);
+		else
+			int_en = DVP2AXI_TOISP1_FS(0) | DVP2AXI_TOISP1_FE(0);
+		priv->toisp_inf.ch_info[0].is_valid = true;
+		priv->toisp_inf.ch_info[0].id = ch0;
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+		ch0 = dvp2axi_dev->csi_host_idx * 4 + 1;
+		ch1 = dvp2axi_dev->csi_host_idx * 4;
+		ctrl_val = (ch0 << 3) | 0x1;
+		ctrl_val |= (ch1 << 11) | 0x100;
+		if (user == 0)
+			int_en = DVP2AXI_TOISP0_FS(0) | DVP2AXI_TOISP0_FS(1) |
+				 DVP2AXI_TOISP0_FE(0) | DVP2AXI_TOISP0_FE(1);
+		else
+			int_en = DVP2AXI_TOISP1_FS(0) | DVP2AXI_TOISP1_FS(1) |
+				 DVP2AXI_TOISP1_FE(0) | DVP2AXI_TOISP1_FE(1);
+		priv->toisp_inf.ch_info[0].is_valid = true;
+		priv->toisp_inf.ch_info[0].id = ch0;
+		priv->toisp_inf.ch_info[1].is_valid = true;
+		priv->toisp_inf.ch_info[1].id = ch1;
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+		ch0 = dvp2axi_dev->csi_host_idx * 4 + 2;
+		ch1 = dvp2axi_dev->csi_host_idx * 4 + 1;
+		ch2 = dvp2axi_dev->csi_host_idx * 4;
+		ctrl_val = (ch0 << 3) | 0x1;
+		ctrl_val |= (ch1 << 11) | 0x100;
+		ctrl_val |= (ch2 << 19) | 0x10000;
+		if (user == 0)
+			int_en = DVP2AXI_TOISP0_FS(0) | DVP2AXI_TOISP0_FS(1) |
+				 DVP2AXI_TOISP0_FS(2) | DVP2AXI_TOISP0_FE(0) |
+				 DVP2AXI_TOISP0_FE(1) | DVP2AXI_TOISP0_FE(2);
+		else
+			int_en = DVP2AXI_TOISP1_FS(0) | DVP2AXI_TOISP1_FS(1) |
+				 DVP2AXI_TOISP1_FS(2) | DVP2AXI_TOISP1_FE(0) |
+				 DVP2AXI_TOISP1_FE(1) | DVP2AXI_TOISP1_FE(2);
+		priv->toisp_inf.ch_info[0].is_valid = true;
+		priv->toisp_inf.ch_info[0].id = ch0;
+		priv->toisp_inf.ch_info[1].is_valid = true;
+		priv->toisp_inf.ch_info[1].id = ch1;
+		priv->toisp_inf.ch_info[2].is_valid = true;
+		priv->toisp_inf.ch_info[2].id = ch2;
+	}
+	if (user == 0) {
+		if (priv->toisp_inf.link_mode == TOISP_UNITE)
+			width = priv->cap_info.width / 2 +
+				ESMOUDLE_UNITE_EXTEND_PIXEL;
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_TOISP0_CTRL, ctrl_val);
+		if (width && height) {
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_TOISP0_CROP,
+					     offset_x | (offset_y << 16));
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_TOISP0_SIZE,
+					     width | (height << 16));
+		} else {
+			return -EINVAL;
+		}
+	} else {
+		if (priv->toisp_inf.link_mode == TOISP_UNITE) {
+			if (capture_info->mode ==
+			    ESMODULE_MULTI_DEV_COMBINE_ONE)
+				offset_x = 0;
+			else
+				offset_x = priv->cap_info.width / 2 -
+					   ESMOUDLE_UNITE_EXTEND_PIXEL;
+			width = priv->cap_info.width / 2 +
+				ESMOUDLE_UNITE_EXTEND_PIXEL;
+		}
+		es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_TOISP1_CTRL, ctrl_val);
+		if (width && height) {
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_TOISP1_CROP,
+					     offset_x | (offset_y << 16));
+			es_dvp2axi_write_register(dvp2axi_dev, DVP2AXI_REG_TOISP1_SIZE,
+					     width | (height << 16));
+		} else {
+			return -EINVAL;
+		}
+	}
+
+	es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_GLB_INTEN, int_en);
+	return 0;
+}
+
+static void sditf_channel_disable(struct sditf_priv *priv, int user)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	unsigned int ctrl_val = 0;
+
+	if (priv->hdr_cfg.hdr_mode == NO_HDR ||
+	    priv->hdr_cfg.hdr_mode == HDR_COMPR) {
+		if (user == 0)
+			ctrl_val = DVP2AXI_TOISP0_FE(0);
+		else
+			ctrl_val = DVP2AXI_TOISP1_FE(0);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+		if (user == 0)
+			ctrl_val = DVP2AXI_TOISP0_FE(0) | DVP2AXI_TOISP0_FE(1);
+		else
+			ctrl_val = DVP2AXI_TOISP1_FE(0) | DVP2AXI_TOISP1_FE(1);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+		if (user == 0)
+			ctrl_val = DVP2AXI_TOISP0_FE(0) | DVP2AXI_TOISP0_FE(1) |
+				   DVP2AXI_TOISP0_FE(2);
+		else
+			ctrl_val = DVP2AXI_TOISP1_FE(0) | DVP2AXI_TOISP1_FE(1) |
+				   DVP2AXI_TOISP1_FE(2);
+	}
+
+	es_dvp2axi_write_register_or(dvp2axi_dev, DVP2AXI_REG_GLB_INTEN, ctrl_val);
+
+	priv->toisp_inf.ch_info[0].is_valid = false;
+	priv->toisp_inf.ch_info[1].is_valid = false;
+	priv->toisp_inf.ch_info[2].is_valid = false;
+}
+
+void sditf_change_to_online(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+
+	priv->mode.rdbk_mode = ESISP_VICAP_ONLINE;
+	if (priv->toisp_inf.link_mode == TOISP0) {
+		sditf_channel_enable(priv, 0);
+	} else if (priv->toisp_inf.link_mode == TOISP1) {
+		sditf_channel_enable(priv, 1);
+	} else if (priv->toisp_inf.link_mode == TOISP_UNITE) {
+		sditf_channel_enable(priv, 0);
+		sditf_channel_enable(priv, 1);
+	}
+
+	if (dvp2axi_dev->is_thunderboot) {
+		if (priv->hdr_cfg.hdr_mode == NO_HDR) {
+			es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[0],
+					  dvp2axi_dev->stream[0].rx_buf_num);
+			dvp2axi_dev->stream[0].is_line_wake_up = false;
+		} else if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+			es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[1],
+					  dvp2axi_dev->stream[1].rx_buf_num);
+			dvp2axi_dev->stream[0].is_line_wake_up = false;
+			dvp2axi_dev->stream[1].is_line_wake_up = false;
+		} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+			es_dvp2axi_free_rx_buf(&dvp2axi_dev->stream[2],
+					  dvp2axi_dev->stream[2].rx_buf_num);
+			dvp2axi_dev->stream[0].is_line_wake_up = false;
+			dvp2axi_dev->stream[1].is_line_wake_up = false;
+			dvp2axi_dev->stream[2].is_line_wake_up = false;
+		}
+		dvp2axi_dev->wait_line_cache = 0;
+		dvp2axi_dev->wait_line = 0;
+		dvp2axi_dev->wait_line_bak = 0;
+		dvp2axi_dev->is_thunderboot = false;
+	}
+}
+
+void sditf_disable_immediately(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	u32 ctrl_val = 0x10101;
+
+	if (priv->toisp_inf.link_mode == TOISP0) {
+		es_dvp2axi_write_register_and(dvp2axi_dev, DVP2AXI_REG_TOISP0_CTRL,
+					 ~ctrl_val);
+	} else if (priv->toisp_inf.link_mode == TOISP1) {
+		es_dvp2axi_write_register_and(dvp2axi_dev, DVP2AXI_REG_TOISP1_CTRL,
+					 ~ctrl_val);
+	} else if (priv->toisp_inf.link_mode == TOISP_UNITE) {
+		es_dvp2axi_write_register_and(dvp2axi_dev, DVP2AXI_REG_TOISP0_CTRL,
+					 ~ctrl_val);
+		es_dvp2axi_write_register_and(dvp2axi_dev, DVP2AXI_REG_TOISP1_CTRL,
+					 ~ctrl_val);
+	}
+}
+
+static void sditf_check_capture_mode(struct es_dvp2axi_device *dvp2axi_dev)
+{
+	struct es_dvp2axi_device *dev = NULL;
+	int i = 0;
+	int toisp_cnt = 0;
+
+	for (i = 0; i < dvp2axi_dev->hw_dev->dev_num; i++) {
+		dev = dvp2axi_dev->hw_dev->dvp2axi_dev[i];
+		if (dev && dev->sditf_cnt)
+			toisp_cnt++;
+	}
+	if (dvp2axi_dev->is_thunderboot && toisp_cnt == 1)
+		dvp2axi_dev->is_rdbk_to_online = true;
+	else
+		dvp2axi_dev->is_rdbk_to_online = false;
+}
+
+static int sditf_start_stream(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_subdev_format fmt;
+	unsigned int mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+
+	sditf_check_capture_mode(dvp2axi_dev);
+	sditf_get_set_fmt(&priv->sd, NULL, &fmt);
+	if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE) {
+		if (priv->toisp_inf.link_mode == TOISP0) {
+			sditf_channel_enable(priv, 0);
+		} else if (priv->toisp_inf.link_mode == TOISP1) {
+			sditf_channel_enable(priv, 1);
+		} else if (priv->toisp_inf.link_mode == TOISP_UNITE) {
+			sditf_channel_enable(priv, 0);
+			sditf_channel_enable(priv, 1);
+		}
+		mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+	} else if (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO) {
+		mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+	}
+
+	if (priv->hdr_cfg.hdr_mode == NO_HDR ||
+	    priv->hdr_cfg.hdr_mode == HDR_COMPR) {
+		es_dvp2axi_do_start_stream(&dvp2axi_dev->stream[0], mode);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+		es_dvp2axi_do_start_stream(&dvp2axi_dev->stream[0], mode);
+		es_dvp2axi_do_start_stream(&dvp2axi_dev->stream[1], mode);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+		es_dvp2axi_do_start_stream(&dvp2axi_dev->stream[0], mode);
+		es_dvp2axi_do_start_stream(&dvp2axi_dev->stream[1], mode);
+		es_dvp2axi_do_start_stream(&dvp2axi_dev->stream[2], mode);
+	}
+	INIT_LIST_HEAD(&priv->buf_free_list);
+	return 0;
+}
+
+static int sditf_stop_stream(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	unsigned int mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+
+	if (priv->toisp_inf.link_mode == TOISP0) {
+		sditf_channel_disable(priv, 0);
+	} else if (priv->toisp_inf.link_mode == TOISP1) {
+		sditf_channel_disable(priv, 1);
+	} else if (priv->toisp_inf.link_mode == TOISP_UNITE) {
+		sditf_channel_disable(priv, 0);
+		sditf_channel_disable(priv, 1);
+	}
+
+	if (priv->mode.rdbk_mode == ESISP_VICAP_ONLINE)
+		mode = ES_DVP2AXI_STREAM_MODE_TOISP;
+	else if (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AUTO)
+		mode = ES_DVP2AXI_STREAM_MODE_TOISP_RDBK;
+
+	if (priv->hdr_cfg.hdr_mode == NO_HDR ||
+	    priv->hdr_cfg.hdr_mode == HDR_COMPR) {
+		es_dvp2axi_do_stop_stream(&dvp2axi_dev->stream[0], mode);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X2) {
+		es_dvp2axi_do_stop_stream(&dvp2axi_dev->stream[0], mode);
+		es_dvp2axi_do_stop_stream(&dvp2axi_dev->stream[1], mode);
+	} else if (priv->hdr_cfg.hdr_mode == HDR_X3) {
+		es_dvp2axi_do_stop_stream(&dvp2axi_dev->stream[0], mode);
+		es_dvp2axi_do_stop_stream(&dvp2axi_dev->stream[1], mode);
+		es_dvp2axi_do_stop_stream(&dvp2axi_dev->stream[2], mode);
+	}
+	return 0;
+}
+
+static int sditf_s_stream(struct v4l2_subdev *sd, int on)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	int ret = 0;
+
+	if (!on && atomic_dec_return(&priv->stream_cnt))
+		return 0;
+
+	if (on && atomic_inc_return(&priv->stream_cnt) > 1)
+		return 0;
+
+	if (dvp2axi_dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+		if (priv->mode.rdbk_mode == ESISP_VICAP_RDBK_AIQ)
+			return 0;
+		v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "%s, toisp mode %d, hdr %d, stream on %d\n", __func__,
+			 priv->toisp_inf.link_mode, priv->hdr_cfg.hdr_mode, on);
+		if (on) {
+			ret = sditf_start_stream(priv);
+		} else {
+			ret = sditf_stop_stream(priv);
+			sditf_free_buf(priv);
+			priv->mode.rdbk_mode = ESISP_VICAP_RDBK_AIQ;
+		}
+	}
+	if (on && ret)
+		atomic_dec(&priv->stream_cnt);
+	return ret;
+}
+
+static int sditf_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct es_dvp2axi_vdev_node *node = &dvp2axi_dev->stream[0].vnode;
+	int ret = 0;
+
+	if (!on && atomic_dec_return(&priv->power_cnt))
+		return 0;
+
+	if (on && atomic_inc_return(&priv->power_cnt) > 1)
+		return 0;
+
+	if (dvp2axi_dev->chip_id >= CHIP_EIC770X_DVP2AXI) {
+		v4l2_dbg(1, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "%s, toisp mode %d, hdr %d, set power %d\n", __func__,
+			 priv->toisp_inf.link_mode, priv->hdr_cfg.hdr_mode, on);
+		mutex_lock(&dvp2axi_dev->stream_lock);
+		if (on) {
+			ret = pm_runtime_resume_and_get(dvp2axi_dev->dev);
+			ret |= v4l2_pipeline_pm_get(&node->vdev.entity);
+		} else {
+			v4l2_pipeline_pm_put(&node->vdev.entity);
+			pm_runtime_put_sync(dvp2axi_dev->dev);
+			priv->mode.rdbk_mode = ESISP_VICAP_RDBK_AIQ;
+		}
+		v4l2_info(&node->vdev, "s_power %d, entity use_count %d\n", on,
+			  node->vdev.entity.use_count);
+		mutex_unlock(&dvp2axi_dev->stream_lock);
+	}
+	return ret;
+}
+
+static int sditf_s_rx_buffer(struct v4l2_subdev *sd, void *buf,
+			     unsigned int *size)
+{
+	struct sditf_priv *priv = to_sditf_priv(sd);
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct es_dvp2axi_sensor_info *sensor = &dvp2axi_dev->terminal_sensor;
+	struct es_dvp2axi_stream *stream = NULL;
+	struct esisp_rx_buf *dbufs;
+	struct es_dvp2axi_rx_buffer *rx_buf = NULL;
+	unsigned long flags, buffree_flags;
+	u32 diff_time = 1000000;
+	u32 early_time = 0;
+	bool is_free = false;
+	bool is_single_dev = false;
+
+	if (!buf) {
+		v4l2_err(&dvp2axi_dev->v4l2_dev, "buf is NULL\n");
+		return -EINVAL;
+	}
+
+	dbufs = buf;
+	if (dvp2axi_dev->hdr.hdr_mode == NO_HDR) {
+		if (dbufs->type == BUF_SHORT)
+			stream = &dvp2axi_dev->stream[0];
+		else
+			return -EINVAL;
+	} else if (dvp2axi_dev->hdr.hdr_mode == HDR_X2) {
+		if (dbufs->type == BUF_SHORT)
+			stream = &dvp2axi_dev->stream[1];
+		else if (dbufs->type == BUF_MIDDLE)
+			stream = &dvp2axi_dev->stream[0];
+		else
+			return -EINVAL;
+	} else if (dvp2axi_dev->hdr.hdr_mode == HDR_X3) {
+		if (dbufs->type == BUF_SHORT)
+			stream = &dvp2axi_dev->stream[2];
+		else if (dbufs->type == BUF_MIDDLE)
+			stream = &dvp2axi_dev->stream[1];
+		else if (dbufs->type == BUF_LONG)
+			stream = &dvp2axi_dev->stream[0];
+		else
+			return -EINVAL;
+	}
+
+	if (!stream)
+		return -EINVAL;
+
+	rx_buf = to_dvp2axi_rx_buf(dbufs);
+	v4l2_dbg(3, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev, "buf back to vicap 0x%x\n",
+		 (u32)rx_buf->dummy.dma_addr);
+	spin_lock_irqsave(&stream->vbq_lock, flags);
+	stream->last_rx_buf_idx = dbufs->sequence + 1;
+	atomic_inc(&stream->buf_cnt);
+
+	is_single_dev = es_dvp2axi_check_single_dev_stream_on(dvp2axi_dev->hw_dev);
+	if (!list_empty(&stream->rx_buf_head) && dvp2axi_dev->is_thunderboot &&
+	    ((!dvp2axi_dev->is_rtt_suspend && !dvp2axi_dev->is_aov_reserved) ||
+	     !is_single_dev) &&
+	    (dbufs->type == BUF_SHORT ||
+	     (dbufs->type != BUF_SHORT && (!dbufs->is_switch)))) {
+		spin_lock_irqsave(&dvp2axi_dev->buffree_lock, buffree_flags);
+		list_add_tail(&rx_buf->list_free, &priv->buf_free_list);
+		spin_unlock_irqrestore(&dvp2axi_dev->buffree_lock, buffree_flags);
+		atomic_dec(&stream->buf_cnt);
+		stream->total_buf_num--;
+		schedule_work(&priv->buffree_work.work);
+		is_free = true;
+	}
+
+	if (!is_free && (!dbufs->is_switch)) {
+		list_add_tail(&rx_buf->list, &stream->rx_buf_head);
+		if (dvp2axi_dev->resume_mode != ESISP_RTT_MODE_ONE_FRAME) {
+			es_dvp2axi_assign_check_buffer_update_toisp(stream);
+			if (!stream->dma_en) {
+				stream->to_en_dma = ES_DVP2AXI_DMAEN_BY_ISP;
+				es_dvp2axi_enable_dma_capture(stream, true);
+				dvp2axi_dev->sensor_work.on = 1;
+				schedule_work(&dvp2axi_dev->sensor_work.work);
+			}
+		}
+		if (dvp2axi_dev->rdbk_debug) {
+			u32 offset = 0;
+
+			offset = rx_buf->dummy.size -
+				 stream->pixm.plane_fmt[0].bytesperline * 3;
+			memset(rx_buf->dummy.vaddr + offset, 0x00,
+			       stream->pixm.plane_fmt[0].bytesperline * 3);
+			if (dvp2axi_dev->is_thunderboot ||
+			    dvp2axi_dev->is_rtt_suspend || dvp2axi_dev->is_aov_reserved)
+				dma_sync_single_for_device(
+					dvp2axi_dev->dev,
+					rx_buf->dummy.dma_addr +
+						rx_buf->dummy.size -
+						stream->pixm.plane_fmt[0]
+								.bytesperline *
+							3,
+					stream->pixm.plane_fmt[0].bytesperline *
+						3,
+					DMA_FROM_DEVICE);
+			else
+				dvp2axi_dev->hw_dev->mem_ops->prepare(
+					rx_buf->dummy.mem_priv);
+		}
+	}
+
+	if (dbufs->is_switch && dbufs->type == BUF_SHORT) {
+		if (stream->is_in_vblank)
+			sditf_change_to_online(priv);
+		else
+			stream->is_change_toisp = true;
+		v4l2_dbg(3, es_dvp2axi_debug, &dvp2axi_dev->v4l2_dev,
+			 "switch to online mode\n");
+	}
+	spin_unlock_irqrestore(&stream->vbq_lock, flags);
+
+	if (!dvp2axi_dev->is_thunderboot || dvp2axi_dev->is_rdbk_to_online == false)
+		return 0;
+
+	if (dbufs->runtime_us && dvp2axi_dev->early_line == 0) {
+		if (!dvp2axi_dev->sensor_linetime)
+			dvp2axi_dev->sensor_linetime = es_dvp2axi_get_linetime(stream);
+		dvp2axi_dev->isp_runtime_max = dbufs->runtime_us;
+		if (dvp2axi_dev->is_thunderboot)
+			diff_time = 200000;
+		else
+			diff_time = 1000000;
+		if (dbufs->runtime_us * 1000 + dvp2axi_dev->sensor_linetime >
+		    diff_time)
+			early_time = dbufs->runtime_us * 1000 - diff_time;
+		else
+			early_time = diff_time;
+		dvp2axi_dev->early_line =
+			div_u64(early_time, dvp2axi_dev->sensor_linetime);
+		dvp2axi_dev->wait_line_cache =
+			sensor->raw_rect.height - dvp2axi_dev->early_line;
+		if (dvp2axi_dev->rdbk_debug && dbufs->sequence < 15)
+			v4l2_info(
+				&dvp2axi_dev->v4l2_dev,
+				"%s, isp runtime %d, line time %d, early_line %d, line_intr_cnt %d, seq %d, type %d, dma_addr %x\n",
+				__func__, dbufs->runtime_us,
+				dvp2axi_dev->sensor_linetime, dvp2axi_dev->early_line,
+				dvp2axi_dev->wait_line_cache, dbufs->sequence,
+				dbufs->type, (u32)rx_buf->dummy.dma_addr);
+	} else {
+		if (dbufs->runtime_us < dvp2axi_dev->isp_runtime_max) {
+			dvp2axi_dev->isp_runtime_max = dbufs->runtime_us;
+			if (dvp2axi_dev->is_thunderboot)
+				diff_time = 200000;
+			else
+				diff_time = 1000000;
+			if (dbufs->runtime_us * 1000 +
+				    dvp2axi_dev->sensor_linetime >
+			    diff_time)
+				early_time =
+					dbufs->runtime_us * 1000 - diff_time;
+			else
+				early_time = diff_time;
+			dvp2axi_dev->early_line =
+				div_u64(early_time, dvp2axi_dev->sensor_linetime);
+			dvp2axi_dev->wait_line_cache =
+				sensor->raw_rect.height - dvp2axi_dev->early_line;
+		}
+		if (dvp2axi_dev->rdbk_debug && dbufs->sequence < 15)
+			v4l2_info(
+				&dvp2axi_dev->v4l2_dev,
+				"isp runtime %d, seq %d, type %d, early_line %d, dma addr %x\n",
+				dbufs->runtime_us, dbufs->sequence, dbufs->type,
+				dvp2axi_dev->early_line,
+				(u32)rx_buf->dummy.dma_addr);
+	}
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops sditf_subdev_pad_ops = {
+	.set_fmt = sditf_get_set_fmt,
+	.get_fmt = sditf_get_set_fmt,
+	.get_selection = sditf_get_selection,
+	.get_mbus_config = sditf_g_mbus_config,
+};
+
+static const struct v4l2_subdev_video_ops sditf_video_ops = {
+	.g_frame_interval = sditf_g_frame_interval,
+	.s_stream = sditf_s_stream,
+	.s_rx_buffer = sditf_s_rx_buffer,
+};
+
+static const struct v4l2_subdev_core_ops sditf_core_ops = {
+	.ioctl = sditf_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl32 = sditf_compat_ioctl32,
+#endif
+	.s_power = sditf_s_power,
+};
+
+static const struct v4l2_subdev_ops sditf_subdev_ops = {
+	.core = &sditf_core_ops,
+	.video = &sditf_video_ops,
+	.pad = &sditf_subdev_pad_ops,
+};
+
+static int es_dvp2axi_sditf_attach_dvp2axidev(struct sditf_priv *sditf)
+{
+	struct device_node *np;
+	struct platform_device *pdev;
+	struct es_dvp2axi_device *dvp2axi_dev;
+
+	np = of_parse_phandle(sditf->dev->of_node, "eci770x,dvp2axi", 0);
+	if (!np || !of_device_is_available(np)) {
+		dev_err(sditf->dev, "failed to get dvp2axi dev node\n");
+		return -ENODEV;
+	}
+
+	pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!pdev) {
+		dev_err(sditf->dev, "failed to get dvp2axi dev from node\n");
+		return -ENODEV;
+	}
+
+	dvp2axi_dev = platform_get_drvdata(pdev);
+	if (!dvp2axi_dev) {
+		dev_err(sditf->dev, "failed attach dvp2axi dev\n");
+		return -EINVAL;
+	}
+
+	dvp2axi_dev->sditf[dvp2axi_dev->sditf_cnt] = sditf;
+	sditf->dvp2axi_dev = dvp2axi_dev;
+	dvp2axi_dev->sditf_cnt++;
+
+	return 0;
+}
+
+struct sensor_async_subdev {
+	struct v4l2_async_connection asd;
+	struct v4l2_mbus_config mbus;
+	int lanes;
+};
+
+// static int sditf_fwnode_parse(struct device *dev,
+// 					  struct v4l2_fwnode_endpoint *vep,
+// 					  struct v4l2_async_subdev *asd)
+// {
+// 	struct sensor_async_subdev *s_asd =
+// 			container_of(asd, struct sensor_async_subdev, asd);
+// 	struct v4l2_mbus_config *config = &s_asd->mbus;
+
+// 	if (vep->base.port != 0) {
+// 		dev_err(dev, "sditf has only port 0\n");
+// 		return -EINVAL;
+// 	}
+
+// 	if (vep->bus_type == V4L2_MBUS_CSI2_DPHY ||
+// 	    vep->bus_type == V4L2_MBUS_CSI2_CPHY) {
+// 		config->type = vep->bus_type;
+// 		config->bus.mipi_csi2.flags = vep->bus.mipi_csi2.flags;
+// 		s_asd->lanes = vep->bus.mipi_csi2.num_data_lanes;
+// 	} else if (vep->bus_type == V4L2_MBUS_CCP2) {
+// 		config->type = vep->bus_type;
+// 		s_asd->lanes = vep->bus.mipi_csi1.data_lane;
+// 	} else {
+// 		dev_err(dev, "type is not supported\n");
+// 		return -EINVAL;
+// 	}
+
+// 	return 0;
+// }
+
+static int sditf_fwnode_parse(struct sditf_priv *sditf)
+{
+	struct device *dev = sditf->dev;
+	struct fwnode_handle *ep = NULL;
+	struct v4l2_async_connection *s_asd = NULL;
+	struct fwnode_handle *remote_ep = NULL;
+	struct v4l2_fwnode_endpoint vep = { .bus_type = V4L2_MBUS_CSI2_DPHY };
+	int ret = 0;
+
+	fwnode_graph_for_each_endpoint(dev_fwnode(dev), ep) {
+		ret = v4l2_fwnode_endpoint_parse(ep, &vep);
+		if (ret)
+			goto err_parse;
+
+		/* only add fwnode form port 0 to notifier list */
+		if (vep.base.port != 0)
+			continue;
+
+		remote_ep = fwnode_graph_get_remote_port_parent(ep);
+
+		/* skip device dts status is disabled */
+		if (!fwnode_device_is_available(remote_ep)) {
+			fwnode_handle_put(remote_ep);
+			continue;
+		}
+
+		fwnode_handle_put(remote_ep);
+
+		v4l2_async_subdev_nf_init(&sditf->notifier, &sditf->sd);
+
+		s_asd = v4l2_async_nf_add_fwnode_remote(
+			&sditf->notifier, ep, struct v4l2_async_connection);
+		if (IS_ERR(s_asd)) {
+			ret = PTR_ERR(s_asd);
+			goto err_parse;
+		}
+
+		// if (vep->bus_type == V4L2_MBUS_CSI2_DPHY ||
+		//     vep->bus_type == V4L2_MBUS_CSI2_CPHY) {
+		// 	config->type = vep->bus_type;
+		// 	config->bus.mipi_csi2.flags = vep->bus.mipi_csi2.flags;
+		// 	s_asd->lanes = vep->bus.mipi_csi2.num_data_lanes;
+		// } else if (vep->bus_type == V4L2_MBUS_CCP2) {
+		// 	config->type = vep->bus_type;
+		// 	s_asd->lanes = vep->bus.mipi_csi1.data_lane;
+		// } else {
+		// 	dev_err(dev, "type is not supported\n");
+		// 	return -EINVAL;
+		// }
+	}
+	return 0;
+
+err_parse:
+	fwnode_handle_put(ep);
+	return ret;
+}
+
+static int es_dvp2axi_sditf_get_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct sditf_priv *priv =
+		container_of(ctrl->handler, struct sditf_priv, ctrl_handler);
+	struct v4l2_ctrl *sensor_ctrl = NULL;
+
+	switch (ctrl->id) {
+	case V4L2_CID_PIXEL_RATE:
+		if (priv->dvp2axi_dev->terminal_sensor.sd) {
+			sensor_ctrl = v4l2_ctrl_find(
+				priv->dvp2axi_dev->terminal_sensor.sd->ctrl_handler,
+				V4L2_CID_PIXEL_RATE);
+			if (sensor_ctrl) {
+				ctrl->val = v4l2_ctrl_g_ctrl_int64(sensor_ctrl);
+				__v4l2_ctrl_s_ctrl_int64(priv->pixel_rate,
+							 ctrl->val);
+				v4l2_dbg(
+					1, es_dvp2axi_debug,
+					&priv->dvp2axi_dev->v4l2_dev,
+					"%s, %s pixel rate %d\n", __func__,
+					priv->dvp2axi_dev->terminal_sensor.sd->name,
+					ctrl->val);
+				return 0;
+			} else {
+				return -EINVAL;
+			}
+		}
+		return -EINVAL;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct v4l2_ctrl_ops es_dvp2axi_sditf_ctrl_ops = {
+	.g_volatile_ctrl = es_dvp2axi_sditf_get_ctrl,
+};
+
+static int sditf_notifier_bound(struct v4l2_async_notifier *notifier,
+				struct v4l2_subdev *subdev,
+				struct v4l2_async_connection *asd)
+{
+	struct sditf_priv *sditf =
+		container_of(notifier, struct sditf_priv, notifier);
+	struct media_entity *source_entity, *sink_entity;
+	int ret = 0;
+
+	sditf->sensor_sd = subdev;
+
+	if (sditf->num_sensors == 1) {
+		v4l2_err(subdev, "%s: the num of subdev is beyond %d\n",
+			 __func__, sditf->num_sensors);
+		return -EBUSY;
+	}
+
+	if (sditf->sd.entity.pads[0].flags & MEDIA_PAD_FL_SINK) {
+		source_entity = &subdev->entity;
+		sink_entity = &sditf->sd.entity;
+
+		ret = media_create_pad_link(source_entity, 0, sink_entity, 0,
+					    MEDIA_LNK_FL_ENABLED);
+		if (ret)
+			v4l2_err(&sditf->sd, "failed to create link for %s\n",
+				 sditf->sensor_sd->name);
+	}
+	sditf->sensor_sd = subdev;
+	++sditf->num_sensors;
+
+	v4l2_err(subdev, "Async registered subdev\n");
+
+	return 0;
+}
+
+static void sditf_notifier_unbind(struct v4l2_async_notifier *notifier,
+				  struct v4l2_subdev *sd,
+				  struct v4l2_async_connection *asd)
+{
+	struct sditf_priv *sditf =
+		container_of(notifier, struct sditf_priv, notifier);
+
+	sditf->sensor_sd = NULL;
+}
+
+static const struct v4l2_async_notifier_operations sditf_notifier_ops = {
+	.bound = sditf_notifier_bound,
+	.unbind = sditf_notifier_unbind,
+};
+
+static int sditf_subdev_notifier(struct sditf_priv *sditf)
+{
+	struct v4l2_async_notifier *ntf = &sditf->notifier;
+	int ret;
+
+	ret = sditf_fwnode_parse(sditf);
+	if (ret < 0)
+		return ret;
+
+	sditf->sd.subdev_notifier = &sditf->notifier;
+	sditf->notifier.ops = &sditf_notifier_ops;
+
+	ret = v4l2_async_nf_register(ntf);
+	if (ret) {
+		dev_err(sditf->dev, "fail to register async notifier: %d\n",
+			ret);
+		v4l2_async_nf_cleanup(ntf);
+	}
+
+	return v4l2_async_register_subdev(&sditf->sd);
+}
+
+static int es_dvp2axi_subdev_media_init(struct sditf_priv *priv)
+{
+	struct es_dvp2axi_device *dvp2axi_dev = priv->dvp2axi_dev;
+	struct v4l2_ctrl_handler *handler = &priv->ctrl_handler;
+	unsigned long flags = V4L2_CTRL_FLAG_VOLATILE;
+	int ret;
+	int pad_num = 0;
+
+	if (priv->is_combine_mode) {
+		priv->pads[0].flags = MEDIA_PAD_FL_SINK;
+		priv->pads[1].flags = MEDIA_PAD_FL_SOURCE;
+		pad_num = 2;
+	} else {
+		priv->pads[0].flags = MEDIA_PAD_FL_SOURCE;
+		pad_num = 1;
+	}
+	priv->sd.entity.function = MEDIA_ENT_F_PROC_VIDEO_COMPOSER;
+	ret = media_entity_pads_init(&priv->sd.entity, pad_num, priv->pads);
+	if (ret < 0)
+		return ret;
+
+	ret = v4l2_ctrl_handler_init(handler, 1);
+	if (ret)
+		return ret;
+	priv->pixel_rate = v4l2_ctrl_new_std(handler, &es_dvp2axi_sditf_ctrl_ops,
+					     V4L2_CID_PIXEL_RATE, 0,
+					     SDITF_PIXEL_RATE_MAX, 1,
+					     SDITF_PIXEL_RATE_MAX);
+	if (priv->pixel_rate)
+		priv->pixel_rate->flags |= flags;
+	priv->sd.ctrl_handler = handler;
+	if (handler->error) {
+		v4l2_ctrl_handler_free(handler);
+		return handler->error;
+	}
+
+	strncpy(priv->sd.name, dev_name(dvp2axi_dev->dev), sizeof(priv->sd.name));
+	priv->cap_info.width = 0;
+	priv->cap_info.height = 0;
+	priv->mode.rdbk_mode = ESISP_VICAP_RDBK_AIQ;
+	priv->toisp_inf.link_mode = TOISP_NONE;
+	priv->toisp_inf.ch_info[0].is_valid = false;
+	priv->toisp_inf.ch_info[1].is_valid = false;
+	priv->toisp_inf.ch_info[2].is_valid = false;
+	if (priv->is_combine_mode)
+		sditf_subdev_notifier(priv);
+	atomic_set(&priv->power_cnt, 0);
+	atomic_set(&priv->stream_cnt, 0);
+	INIT_WORK(&priv->buffree_work.work, sditf_buffree_work);
+	INIT_LIST_HEAD(&priv->buf_free_list);
+	return 0;
+}
+
+static int es_dvp2axi_subdev_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct v4l2_subdev *sd;
+	struct sditf_priv *priv;
+	struct device_node *node = dev->of_node;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	priv->dev = dev;
+
+	sd = &priv->sd;
+	v4l2_subdev_init(sd, &sditf_subdev_ops);
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	snprintf(sd->name, sizeof(sd->name), "eic770x-dvp2axi-sditf");
+	sd->dev = dev;
+
+	platform_set_drvdata(pdev, &sd->entity);
+
+	ret = es_dvp2axi_sditf_attach_dvp2axidev(priv);
+	if (ret < 0)
+		return ret;
+
+	ret = of_property_read_u32(node, "eic770x,combine-index",
+				   &priv->combine_index);
+	if (ret) {
+		priv->is_combine_mode = false;
+		priv->combine_index = 0;
+	} else {
+		priv->is_combine_mode = true;
+	}
+	ret = es_dvp2axi_subdev_media_init(priv);
+	if (ret < 0)
+		return ret;
+
+	pm_runtime_enable(&pdev->dev);
+	return 0;
+}
+
+static int es_dvp2axi_subdev_remove(struct platform_device *pdev)
+{
+	struct media_entity *me = platform_get_drvdata(pdev);
+	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(me);
+
+	media_entity_cleanup(&sd->entity);
+
+	pm_runtime_disable(&pdev->dev);
+	return 0;
+}
+
+static const struct of_device_id es_dvp2axi_subdev_match_id[] = {
+	{
+		.compatible = "eic770x,es_dvp2axi-sditf",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, es_dvp2axi_subdev_match_id);
+
+struct platform_driver es_dvp2axi_subdev_driver = {
+	.probe = es_dvp2axi_subdev_probe,
+	.remove = es_dvp2axi_subdev_remove,
+	.driver = {
+		.name = "es_dvp2axi_sditf",
+		.of_match_table = es_dvp2axi_subdev_match_id,
+	},
+};
+EXPORT_SYMBOL(es_dvp2axi_subdev_driver);
+
+MODULE_AUTHOR("ESWIN VI team");
+MODULE_DESCRIPTION("ESWIN DVP2AXI platform driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/eswin/dvp2axi/subdev-itf.h b/drivers/media/platform/eswin/dvp2axi/subdev-itf.h
new file mode 100644
index 000000000000..b07acad787af
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/subdev-itf.h
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI subdev driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_SDITF_H
+#define _ES_DVP2AXI_SDITF_H
+
+#include <linux/mutex.h>
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-mc.h>
+#include "../../../../phy/eswin/es-camera-module.h"
+#include "hw.h"
+#include "isp_external.h"
+
+#define ESISP0_DEVNAME "esisp0"
+#define ESISP1_DEVNAME "esisp1"
+#define ESISP_UNITE_DEVNAME "esisp-unite"
+
+#define ES_DVP2AXI_TOISP_CH0	0
+#define ES_DVP2AXI_TOISP_CH1	1
+#define ES_DVP2AXI_TOISP_CH2	2
+#define TOISP_CH_MAX 3
+
+#define SDITF_PIXEL_RATE_MAX (1000000000)
+
+struct capture_info {
+	unsigned int offset_x;
+	unsigned int offset_y;
+	unsigned int width;
+	unsigned int height;
+};
+
+enum toisp_link_mode {
+	TOISP_NONE,
+	TOISP0,
+	TOISP1,
+	TOISP_UNITE,
+};
+
+struct toisp_ch_info {
+	bool is_valid;
+	int id;
+};
+
+struct toisp_info {
+	struct toisp_ch_info ch_info[TOISP_CH_MAX];
+	enum toisp_link_mode link_mode;
+};
+
+struct sditf_work_struct {
+	struct work_struct	work;
+	struct esisp_rx_buffer *buf;
+};
+
+struct sditf_priv {
+	struct device *dev;
+	struct v4l2_async_notifier notifier;
+	struct v4l2_subdev sd;
+	struct media_pad pads[2];
+	struct es_dvp2axi_device *dvp2axi_dev;
+	struct esmodule_hdr_cfg	hdr_cfg;
+	struct capture_info cap_info;
+	struct esisp_vicap_mode mode;
+	struct toisp_info toisp_inf;
+	struct v4l2_ctrl *pixel_rate;
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct v4l2_subdev *sensor_sd;
+	struct sditf_work_struct buffree_work;
+	struct list_head buf_free_list;
+	int buf_num;
+	int num_sensors;
+	int combine_index;
+	bool is_combine_mode;
+	atomic_t power_cnt;
+	atomic_t stream_cnt;
+};
+
+extern struct platform_driver es_dvp2axi_subdev_driver;
+void sditf_change_to_online(struct sditf_priv *priv);
+void sditf_disable_immediately(struct sditf_priv *priv);
+
+#endif
diff --git a/drivers/media/platform/eswin/dvp2axi/version.h b/drivers/media/platform/eswin/dvp2axi/version.h
new file mode 100644
index 000000000000..0ae6b27ac86d
--- /dev/null
+++ b/drivers/media/platform/eswin/dvp2axi/version.h
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DVP2AXI version driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+
+#ifndef _ES_DVP2AXI_VERSION_H
+#define _ES_DVP2AXI_VERSION_H
+#include <linux/version.h>
+#include "../../../../phy/eswin/es-dvp2axi-config.h"
+
+/*
+ *ES DVP2AXI DRIVER VERSION NOTE
+ *
+ *v0.1.0:
+ *1. First version;
+ *v0.1.1
+ *1. Support the mipi vc multi-channel input in dvp2axi driver for es1808
+ *v0.1.2
+ *1. support output yuyv fmt by setting the input mode to raw8
+ *2. Compatible with dvp2axi only have single dma mode in driver
+ *3. Support dvp2axi works with mipi channel for es3288
+ *4. Support switching between oneframe and pingpong for dvp2axi
+ *5. Support sampling raw data for dvp2axi
+ *6. fix the bug that dummpy buffer size is error
+ *7. Add framesizes and frmintervals callback
+ *8. fix dvp camera fails to link with dvp2axi on es1808
+ *9. add camera support hotplug for n4
+ *10. reconstruct register's reading and writing
+ *v0.1.3
+ *1. support kernel-4.19 and support vicap single dvp for rv1126
+ *2. support vicap + mipi(single) for rv1126
+ *3. support vicap + mipi hdr for rv1126
+ *4. add luma device node for rv1126 vicap
+ *v0.1.4
+ *1. support vicap-full lvds interface to work in linear and hdr mode for rv1126
+ *2. add vicap-lite device for rv1126
+ *v0.1.5
+ *1. support crop function
+ *2. fix compile error when config with module
+ *3. support mipi yuv
+ *4. support selection ioctl for cropping
+ *5. support dvp2axi compact mode(lvds & mipi) can be set from user space
+ *v0.1.6
+ *1. add dvp2axi self-defined ioctrl cmd:V4L2_CID_DVP2AXI_DATA_COMPACT
+ *v0.1.7
+ *1. support dvp and mipi/lvds run simultaneously
+ *2. add subdev as interface for isp
+ *3. support hdr_x3 mode
+ *4. support es1808 mipi interface in kernel-4.19
+ *v0.1.8
+ *1. add proc interface
+ *2. add reset mechanism to resume when csi crc err
+ *3. support bt1120 single path
+ *v0.1.9
+ *1. support es3568 dvp2axi
+ *2. support es3568 csi-host
+ *3. add dvp sof
+ *4. add extended lines to out image for normal & hdr short frame
+ *5. modify reset mechanism drivered by real-time frame rate
+ *6. support es356x iommu uses vb2 sg type
+ *7. register dvp2axi sd itf when pipeline completed
+ *v0.1.10
+ *1. rv1126/es356x support bt656/bt1120 multi channels function
+ *2. add dynamic cropping function
+ *3. optimize dts config of dvp2axi's pipeline
+ *4. register dvp2axi itf dev when clear unready subdev
+ *5. mipi csi host add cru rst
+ *6. support wake up mode with mipi
+ *7. add keepint time to csi2 err for resetting
+ *8. mipi supports pdaf/embedded data
+ *9. mipi supports interlaced capture
+ *v0.2.0
+ *1. vicap support combine multi mipi dev to one dev, this function is mainly used for es3588
+ */
+
+#define ES_DVP2AXI_DRIVER_VERSION ES_DVP2AXI_API_VERSION
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/Kconfig b/drivers/media/platform/eswin/isp/Kconfig
new file mode 100644
index 000000000000..580dcded3b67
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Configuration for the eswin isp driver
+#
+
+config VIDEO_ESWIN_ISP
+    tristate "ESWIN ISP driver support"
+    default m
+    depends on VIDEO_DEV
+    select VIDEOBUF2_CORE
+    help
+      Enable support for the ESWIN ISP video interface driver.
+      This driver is used for handling video input from ESWIN devices.
diff --git a/drivers/media/platform/eswin/isp/Makefile b/drivers/media/platform/eswin/isp/Makefile
new file mode 100644
index 000000000000..a53a584b81f1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-y += vvcam/isp/
+obj-y += vvcam/vb/
+obj-y += vvcam/v4l2/isp/
+obj-y += vvcam/v4l2/video/
diff --git a/drivers/media/platform/eswin/isp/vvcam/Makefile b/drivers/media/platform/eswin/isp/vvcam/Makefile
new file mode 100644
index 000000000000..960e2f67bfc1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/Makefile
@@ -0,0 +1,90 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+ccflags-$(CONFIG_WERROR) += -Wno-error
+all:
+#	@cd i2c;   make || exit $$?;
+#	@cd mipi;  make || exit $$?;
+	@cd isp;   make || exit $$?;
+	@cd vb;    make || exit $$?;
+#	@cd soc;   make || exit $$?;
+#	@cd vicap; make || exit $$?;
+#	@cd dwc; make || exit $$?;
+ifneq ($(VI200_VERSION), NONE)
+	@cd videoin;    make || exit $$?;
+endif
+ifdef MSS
+	@cd mi_subsystem;   make || exit $$?;
+endif
+#ifdef V4L2
+	@cd v4l2;    make || exit $$?;
+#endif
+
+clean:
+#	@cd i2c;   make clean;
+#	@cd mipi;  make clean;
+	@cd isp;   make clean;
+	@cd vb;    make clean;
+#	@cd soc;   make clean;
+#	@cd vicap; make clean;
+#	@cd dwc;   make clean;
+ifneq ($(VI200_VERSION), NONE)
+	@cd videoin;    make clean;
+endif
+ifdef MSS
+	@cd mi_subsystem;   make clean;
+endif
+#ifdef V4L2
+	@cd v4l2; make clean;
+#endif
+	@ rm install -rf
diff --git a/drivers/media/platform/eswin/isp/vvcam/Param.mk b/drivers/media/platform/eswin/isp/vvcam/Param.mk
new file mode 100644
index 000000000000..fbf35bdf8894
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/Param.mk
@@ -0,0 +1,286 @@
+KERNEL :=/lib/modules/$(shell uname -r)/build
+ISP_VERSION := ISP8000_V2107
+VI200_VERSION := NONE
+DW_VERSION := DW200_V40
+MIPI_VERSION := NWL
+INSTALL_DIR:= $(VVCAM_PATH)/install/$(ISP_VERSION)
+
+ifeq ($(VI200_VERSION), NONE)
+PCIE_BASE ?= 0xDE000000
+else
+PCIE_BASE ?= 0xDD000000
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2107)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2107.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2201)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2201.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2202)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2202.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2203)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2203.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200_V2204)
+include $(VVCAM_PATH)/isp_version/ISP8200_V2204.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200L_V2304)
+include $(VVCAM_PATH)/isp_version/ISP8200L_V2304.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200LC_V2403)
+include $(VVCAM_PATH)/isp_version/ISP8200LC_V2403.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200LCR_V2403)
+include $(VVCAM_PATH)/isp_version/ISP8200LCR_V2403.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8210L_V2302)
+include $(VVCAM_PATH)/isp_version/ISP8210L_V2302.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000L_V2305)
+include $(VVCAM_PATH)/isp_version/ISP8000L_V2305.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000L_V2306)
+include $(VVCAM_PATH)/isp_version/ISP8000L_V2306.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000L_V2310)
+include $(VVCAM_PATH)/isp_version/ISP8000L_V2310.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2011)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2011.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000L_V2002)
+include $(VVCAM_PATH)/isp_version/ISP8000L_V2002.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000L_V2103)
+include $(VVCAM_PATH)/isp_version/ISP8000L_V2103.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2104)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2104.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200_V2303)
+include $(VVCAM_PATH)/isp_version/ISP8200_V2303.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200_V2307)
+include $(VVCAM_PATH)/isp_version/ISP8200_V2307.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2308)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2308.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200_V2311)
+include $(VVCAM_PATH)/isp_version/ISP8200_V2311.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8210L_V2302)
+include $(VVCAM_PATH)/isp_version/ISP8210L_V2302.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200NANO_V2401)
+include $(VVCAM_PATH)/isp_version/ISP8200NANO_V2401.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200L_V2404)
+include $(VVCAM_PATH)/isp_version/ISP8200L_V2404.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000_V2405)
+include $(VVCAM_PATH)/isp_version/ISP8000_V2405.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8200_V2406)
+include $(VVCAM_PATH)/isp_version/ISP8200_V2406.mk
+endif
+
+ifeq ($(ISP_VERSION), ISP8000L_V2407)
+include $(VVCAM_PATH)/isp_version/ISP8000L_V2407.mk
+endif
+
+# Vi200 version
+ifeq ($(VI200_VERSION), VI200_V1)
+include $(VVCAM_PATH)/vi200_version/VI200_V1.mk
+else ifeq ($(VI200_VERSION), VI200_V2)
+include $(VVCAM_PATH)/vi200_version/VI200_V2.mk
+else ifeq ($(VI200_VERSION), VI200_V21)
+include $(VVCAM_PATH)/vi200_version/VI200_V21.mk
+else ifeq ($(VI200_VERSION), VI200_V22)
+include $(VVCAM_PATH)/vi200_version/VI200_V22.mk
+endif
+
+EXTRA_CFLAGS += -DM2_REGISTER_MAP
+
+EXTRA_CFLAGS += -DSCU_BASE=0x51828000
+EXTRA_CFLAGS += -DSCU_SIZE=0x00008000
+
+EXTRA_CFLAGS += -DPCIE_BASE=$(PCIE_BASE)
+
+# EXTRA_CFLAGS += -DVVCAM_I2C_SIZE=0x4
+# EXTRA_CFLAGS += -DVVCAM_I2C_BUS0=8
+# EXTRA_CFLAGS += -DVVCAM_I2C_BUS1=9
+# EXTRA_CFLAGS += -DVVCAM_I2C_BUS2=10
+# EXTRA_CFLAGS += -DVVCAM_I2C_BUS3=11
+
+ifeq ($(VI200_VERSION), NONE)
+EXTRA_CFLAGS += -DVVCAM_TPG0=0x00308300
+EXTRA_CFLAGS += -DVVCAM_TPG1=0x00308310
+EXTRA_CFLAGS += -DVVCAM_TPG2=0x00308320
+
+# EXTRA_CFLAGS += -DVVCAM_I2C0_BASE=0x003080f0
+# EXTRA_CFLAGS += -DVVCAM_I2C1_BASE=0x003080f4
+# EXTRA_CFLAGS += -DVVCAM_I2C2_BASE=0x003080f8
+# EXTRA_CFLAGS += -DVVCAM_I2C3_BASE=0x003080fc
+
+# EXTRA_CFLAGS += -DVVCAM_CSI0_BASE=0x00200000
+# EXTRA_CFLAGS += -DVVCAM_CSI1_BASE=0x00300000
+# EXTRA_CFLAGS += -DVVCAM_CSI2_BASE=0x00204000
+# EXTRA_CFLAGS += -DVVCAM_CSI3_BASE=0x00304000
+# EXTRA_CFLAGS += -DVVCAM_CSI_REG_SIZE=0x4000
+
+# EXTRA_CFLAGS += -DVVCAM_CSI0_CTRL=0x00308240
+# EXTRA_CFLAGS += -DVVCAM_CSI1_CTRL=0x00308244
+# EXTRA_CFLAGS += -DVVCAM_CSI2_CTRL=0x00308248
+# EXTRA_CFLAGS += -DVVCAM_CSI3_CTRL=0x0030824C
+# EXTRA_CFLAGS += -DVVCAM_CSI_CTRL_SIZE=0x4
+
+# EXTRA_CFLAGS += -DVVCAM_CSI0_MUX_CTRL=0x00208304
+# EXTRA_CFLAGS += -DVVCAM_CSI1_MUX_CTRL=0x00308304
+# EXTRA_CFLAGS += -DVVCAM_CSI2_MUX_CTRL=0x0020C304
+# EXTRA_CFLAGS += -DVVCAM_CSI3_MUX_CTRL=0x0030C304
+# EXTRA_CFLAGS += -DVVCAM_CSI_MUX_CTRL_SIZE=0x4
+
+EXTRA_CFLAGS += -DVVCAM_ISP_CTRL=0x00308270
+EXTRA_CFLAGS += -DVVCAM_ISP_CTRL_SIZE=0x4
+endif
+
+# ifeq ($(ISP_VERSION), ISP8000L_V2103)
+# EXTRA_CFLAGS += -DINPUT_CSI_MUX_8000L
+# endif
+
+EXTRA_CFLAGS += -DVVCAM_TPG3=0x00308330
+EXTRA_CFLAGS += -DVVCAM_TPG_SIZE=0x4
+
+ifeq ($(ISP_VERSION), ISP8000NANO_V2012)
+EXTRA_CFLAGS += -DVVCAM_CSI_BUS_WIDTH=12
+else
+EXTRA_CFLAGS += -DVVCAM_CSI_BUS_WIDTH=16
+endif
+
+ifeq ($(ISP_VERSION), $(findstring $(ISP_VERSION), ISP8000_V2107 ISP8000_V2108 ISP8000_V2202 ISP8000_V2203 ISP8200L_V2304 ISP8210L_V2302 ISP8000L_V2305 ISP8200_V2303 ISP8000_V2308 ISP8200_V2311 ISP8000L_V2310 ISP8000L_V2002 ISP8000NANO_V2302))
+
+EXTRA_CFLAGS += -DCSI_CTRL_MUX_T0T1=1
+EXTRA_CFLAGS += -DCSI_CTRL_MUX_T2T3=0
+else
+EXTRA_CFLAGS += -DCSI_CTRL_MUX_T0T1=1
+EXTRA_CFLAGS += -DCSI_CTRL_MUX_T2T3=0
+endif
+
+
+ifeq ($(ISP_VERSION), $(findstring $(ISP_VERSION), ISP8000_V2405 ISP8000_V2408 ISP8200_V2204 ISP8200_V2303 ISP8200_V2307 ISP8200_V2311 ISP8200_V2406 ISP8200L_V2304 ISP8200L_V2404 ISP8200LC_V2403 ISP8200LCR_V2403 ISP8210L_V2302))
+
+EXTRA_CFLAGS += -DAHB_WR_WITH_FE=1
+else
+EXTRA_CFLAGS += -DAHB_WR_WITH_FE=0
+endif
+
+ifeq ($(VI200_VERSION), NONE)
+
+ifeq ($(ISP_VERSION), ISP8000_V2009)
+EXTRA_CFLAGS += -DVVCAM_ISP0_BASE=0x00600000
+EXTRA_CFLAGS += -DVVCAM_ISP1_BASE=0x00700000
+EXTRA_CFLAGS += -DVVCAM_ISP2_BASE=0x00FFFFFF
+else
+EXTRA_CFLAGS += -DVVCAM_ISP0_BASE=0x00000000
+EXTRA_CFLAGS += -DVVCAM_ISP1_BASE=0x00100000
+EXTRA_CFLAGS += -DVVCAM_ISP2_BASE=0x00FFFFFF
+endif
+
+endif
+
+EXTRA_CFLAGS += -DVVCAM_ISP_REG_SIZE=0x10000
+
+EXTRA_CFLAGS += -DVVCAM_ISP0_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP0_MI_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP0_FE_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP0_FUSA_IRQ=16
+
+EXTRA_CFLAGS += -DVVCAM_ISP1_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP1_MI_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP1_FE_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP1_FUSA_IRQ=16
+
+EXTRA_CFLAGS += -DVVCAM_ISP2_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP2_MI_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP2_FE_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_ISP2_FUSA_IRQ=16
+
+EXTRA_CFLAGS += -DVVCAM_RMEM_BASE=0x10000000
+EXTRA_CFLAGS += -DVVCAM_RMEM_SIZE=0x20000000
+
+ifdef MSS
+EXTRA_CFLAGS += -DVVCAM_MSS_RESET_BASE=0x00308260
+EXTRA_CFLAGS += -DVVCAM_MSS_RESET_SIZE=0x4
+EXTRA_CFLAGS += -DVVCAM_MSS_BASE=0x00500000
+EXTRA_CFLAGS += -DVVCAM_MSS_REG_SIZE=0x10000
+endif
+
+ifeq ($(DW_VERSION), $(findstring $(DW_VERSION), DW200_V40 DW200_V41))
+EXTRA_CFLAGS += -DVVCAM_DWE_REG_BASE=0xDE381C00
+EXTRA_CFLAGS += -DVVCAM_DWE_REG_SIZE=0x00000800
+EXTRA_CFLAGS += -DVVCAM_VSE_REG_BASE=0xDE380000
+EXTRA_CFLAGS += -DVVCAM_VSE_REG_SIZE=0x00001C00
+endif
+
+ifeq ($(DW_VERSION), DW200_V42)
+EXTRA_CFLAGS += -DDW200_V42
+EXTRA_CFLAGS += -DVVCAM_DWE_REG_BASE=0xDE382C00
+EXTRA_CFLAGS += -DVVCAM_DWE_REG_SIZE=0x00000800
+EXTRA_CFLAGS += -DVVCAM_VSE_REG_BASE=0xDE380000
+EXTRA_CFLAGS += -DVVCAM_VSE_REG_SIZE=0x00002C00
+endif
+
+ifeq ($(DW_VERSION), DW200_V30)
+EXTRA_CFLAGS += -DVVCAM_DWE_REG_BASE=0xDE380C00
+EXTRA_CFLAGS += -DVVCAM_DWE_REG_SIZE=0x00000800
+EXTRA_CFLAGS += -DVVCAM_VSE_REG_BASE=0xDE380000
+EXTRA_CFLAGS += -DVVCAM_VSE_REG_SIZE=0x00000C00
+endif
+
+EXTRA_CFLAGS += -DVVCAM_DW_RST_REG_BASE=0xDE308250
+EXTRA_CFLAGS += -DVVCAM_DW_RST_REG_SIZE=0x4
+
+EXTRA_CFLAGS += -DVVCAM_DWE_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_VSE_IRQ=16
+EXTRA_CFLAGS += -DVVCAM_DW_FE_IRQ=16
+
+# EXTRA_CFLAGS += -O2 -Werror
+
+PLATFORM_DEVICE := 0
+
+ifeq ($(PLATFORM_DEVICE), 1)
+EXTRA_CFLAGS += -DVVCAM_PLATFORM_REGISTER
+endif
+
+ifeq ($(ISP_VERSION), ISP8000NANO_V2302)
+EXTRA_CFLAGS += -DISP8000NANO_V2302
+endif
+
+EXTRA_CFLAGS += -DVVCAM_ISP_DEV_MAX=2
diff --git a/drivers/media/platform/eswin/isp/vvcam/common/vvcam_event.c b/drivers/media/platform/eswin/isp/vvcam/common/vvcam_event.c
new file mode 100644
index 000000000000..5adb18e5800c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/common/vvcam_event.c
@@ -0,0 +1,299 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+
+#include "vvcam_event.h"
+
+void vvcam_event_dev_init(struct vvcam_event_dev *event_dev)
+{
+    spin_lock_init(&event_dev->fh_lock);
+    INIT_LIST_HEAD(&event_dev->fh_list);
+    return;
+}
+
+void vvcam_event_fh_init(struct vvcam_event_dev *event_dev,
+            struct vvcam_event_fh *event_fh)
+{
+    unsigned long flags;
+    INIT_LIST_HEAD(&event_fh->subscribed);
+    INIT_LIST_HEAD(&event_fh->available);
+    init_waitqueue_head(&event_fh->wait);
+    event_fh->event_dev = event_dev;
+
+    spin_lock_irqsave(&event_fh->event_dev->fh_lock, flags);
+    list_add(&event_fh->list, &event_dev->fh_list);
+    spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+
+    return;
+}
+
+void vvcam_event_fh_destroy(struct vvcam_event_dev *event_dev,
+            struct vvcam_event_fh *event_fh)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&event_fh->event_dev->fh_lock, flags);
+    list_del(&event_fh->list);
+    spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+
+    return;
+}
+
+static unsigned sev_pos(const struct vvcam_subscribed_event *sev, unsigned idx)
+{
+    idx += sev->first;
+    return idx >= sev->elems ? idx - sev->elems : idx;
+}
+
+static struct vvcam_subscribed_event *vvcam_event_subscribed(
+            struct vvcam_event_fh *event_fh, uint32_t type, uint32_t id)
+{
+    struct vvcam_subscribed_event *sev;
+
+    assert_spin_locked(&event_fh->event_dev->fh_lock);
+
+    if (list_empty(&event_fh->subscribed))
+        return NULL;
+
+    list_for_each_entry(sev, &event_fh->subscribed, list)
+        if (sev->type == type && sev->id == id)
+            return sev;
+
+    return NULL;
+}
+
+int vvcam_event_subscribe(struct vvcam_event_fh *event_fh,
+            const vvcam_subscription_t *sub, unsigned elems)
+{
+    struct vvcam_subscribed_event *sev, *found_ev;
+    unsigned long flags;
+    unsigned i;
+
+    if (elems < 1)
+        elems = 1;
+
+    sev = kzalloc(sizeof(*sev) + sizeof(struct vvcam_kevent) * elems, GFP_KERNEL);
+    if (!sev)
+        return -ENOMEM;
+
+    for (i = 0; i < elems; i++) {
+        sev->events[i].sev = sev;
+    }
+
+    sev->type = sub->type;
+    sev->id = sub->id;
+    sev->event_fh = event_fh;
+    sev->elems = elems;
+
+    spin_lock_irqsave(&event_fh->event_dev->fh_lock, flags);
+
+    found_ev = vvcam_event_subscribed(event_fh, sub->type, sub->id);
+    if (!found_ev)
+        list_add(&sev->list, &event_fh->subscribed);
+
+    spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+
+    if (found_ev) {
+        kfree(sev);
+        return 0;
+    }
+
+    return 0;
+}
+
+int vvcam_event_unsubscribe(struct vvcam_event_fh *event_fh,
+            const vvcam_subscription_t *sub)
+{
+    struct vvcam_subscribed_event *sev;
+    unsigned long flags;
+    int i;
+
+    spin_lock_irqsave(&event_fh->event_dev->fh_lock, flags);
+
+    sev = vvcam_event_subscribed(event_fh, sub->type, sub->id);
+    if (sev != NULL) {
+        for (i = 0; i < sev->in_use; i++) {
+            list_del(&sev->events[sev_pos(sev, i)].list);
+            event_fh->navailable--;
+        }
+
+        list_del(&sev->list);
+        kfree(sev);
+    }
+    spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+
+    return 0;
+}
+
+int vvcam_event_unsubscribe_all(struct vvcam_event_fh *event_fh)
+{
+    vvcam_subscription_t sub;
+    struct vvcam_subscribed_event *sev;
+    unsigned long flags;
+
+    do {
+        sev = NULL;
+
+        spin_lock_irqsave(&event_fh->event_dev->fh_lock, flags);
+
+        if (!list_empty(&event_fh->subscribed)) {
+            sev = list_first_entry(&event_fh->subscribed,
+                    struct vvcam_subscribed_event, list);
+            sub.type = sev->type;
+            sub.id = sev->id;
+        }
+        spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+
+        if (sev)
+            vvcam_event_unsubscribe(event_fh, &sub);
+
+    } while (sev);
+
+    return 0;
+}
+
+static void __vvcam_event_queue_fh(struct vvcam_event_fh *event_fh,
+            const vvcam_event_t *event)
+{
+    struct vvcam_subscribed_event *sev;
+    struct vvcam_kevent *kev;
+
+    sev = vvcam_event_subscribed(event_fh, event->type, event->id);
+    if (!sev)
+        return;
+
+    if (!sev->elems)
+        return;
+
+    if (sev->in_use == sev->elems) {
+        kev = sev->events + sev_pos(sev, 0);
+        list_del(&kev->list);
+        sev->in_use--;
+        sev->first = sev_pos(sev, 1);
+        event_fh->navailable--;
+    }
+
+    kev = sev->events + sev_pos(sev, sev->in_use);
+
+    kev->event = *event;
+    sev->in_use++;
+
+    list_add_tail(&kev->list, &event_fh->available);
+    event_fh->navailable++;
+
+    wake_up_all(&event_fh->wait);
+
+    return;
+}
+
+void vvcam_event_queue(struct vvcam_event_dev *event_dev,
+            const vvcam_event_t *event)
+{
+    struct vvcam_event_fh *event_fh;
+    unsigned long flags;
+
+    spin_lock_irqsave(&event_dev->fh_lock, flags);
+
+    if (list_empty(&event_dev->fh_list)) {
+        spin_unlock_irqrestore(&event_dev->fh_lock, flags);
+        return;
+    }
+
+    list_for_each_entry(event_fh, &event_dev->fh_list, list)
+        __vvcam_event_queue_fh(event_fh, event);
+
+    spin_unlock_irqrestore(&event_dev->fh_lock, flags);
+
+    return;
+}
+
+int vvcam_event_dequeue(struct vvcam_event_fh *event_fh, vvcam_event_t *event)
+{
+    struct vvcam_kevent *kev;
+    unsigned long flags;
+
+    spin_lock_irqsave(&event_fh->event_dev->fh_lock, flags);
+
+    if (list_empty(&event_fh->available)) {
+        spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+        return -ENOENT;
+    }
+
+    kev = list_first_entry(&event_fh->available, struct vvcam_kevent, list);
+    list_del(&kev->list);
+    event_fh->navailable--;
+
+    kev->sev->first = sev_pos(kev->sev, 1);
+    kev->sev->in_use--;
+    *event = kev->event;
+
+    spin_unlock_irqrestore(&event_fh->event_dev->fh_lock, flags);
+
+    return 0;
+}
+
+unsigned int vvcam_event_poll(struct file *file,
+            struct vvcam_event_fh *event_fh, poll_table *wait)
+{
+    poll_wait(file, &event_fh->wait, wait);
+
+    if (event_fh->navailable)
+        return POLLPRI;
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/common/vvcam_event.h b/drivers/media/platform/eswin/isp/vvcam/common/vvcam_event.h
new file mode 100644
index 000000000000..ff31569f4266
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/common/vvcam_event.h
@@ -0,0 +1,132 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_EVENT_H__
+#define __VVCAM_EVENT_H__
+
+#define VVCAM_EVENT_IRQ_TYPE                0x00000000
+#define VVCAM_VIDEO_IN_EVENT_IRQ_TYPE       0x00000001
+typedef struct {
+    uint32_t type;
+    uint32_t id;
+} vvcam_subscription_t;
+
+typedef struct {
+    uint32_t irq_value;
+} vvcam_irqevent;
+
+typedef struct {
+    uint32_t type;
+    uint32_t id;
+    union {
+        vvcam_irqevent irqevent;
+    };
+    uint64_t timestamp;
+} vvcam_event_t;
+
+#ifdef __KERNEL__
+#include <linux/poll.h>
+
+struct vvcam_event_dev {
+    spinlock_t fh_lock;
+    struct list_head fh_list;
+};
+
+struct vvcam_event_fh {
+    struct list_head list;
+    struct vvcam_event_dev *event_dev;
+    wait_queue_head_t wait;
+    unsigned int navailable;
+    struct list_head subscribed;
+    struct list_head available;
+};
+
+struct vvcam_kevent {
+    struct list_head list;
+    struct vvcam_subscribed_event *sev;
+    vvcam_event_t event;
+};
+
+struct vvcam_subscribed_event {
+    struct list_head list;
+    uint32_t type;
+    uint32_t id;
+
+    struct vvcam_event_fh *event_fh;
+    uint32_t elems;
+    uint32_t in_use;
+    uint32_t first;
+    struct vvcam_kevent events[];
+};
+
+void vvcam_event_dev_init(struct vvcam_event_dev *event_dev);
+void vvcam_event_fh_init(struct vvcam_event_dev *event_dev,
+            struct vvcam_event_fh *event_fh);
+void vvcam_event_fh_destroy(struct vvcam_event_dev *event_dev,
+            struct vvcam_event_fh *event_fh);
+int vvcam_event_subscribe(struct vvcam_event_fh *event_fh,
+            const vvcam_subscription_t *sub, unsigned elems);
+int vvcam_event_unsubscribe(struct vvcam_event_fh *event_fh,
+            const vvcam_subscription_t *sub);
+int vvcam_event_unsubscribe_all(struct vvcam_event_fh *event_fh);
+void vvcam_event_queue(struct vvcam_event_dev *event_dev,
+            const vvcam_event_t *event);
+int vvcam_event_dequeue(struct vvcam_event_fh *event_fh, vvcam_event_t *event);
+unsigned int vvcam_event_poll(struct file *file,
+            struct vvcam_event_fh *event_fh, poll_table *wait);
+
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/Makefile b/drivers/media/platform/eswin/isp/vvcam/dwc/Makefile
new file mode 100644
index 000000000000..d59dadfd11f1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/Makefile
@@ -0,0 +1,35 @@
+ifeq ($(VVCAM_PATH),)
+	VVCAM_PATH:=$(shell cd $(shell pwd)/../; pwd)
+	export VVCAM_PATH
+endif
+
+
+include $(PWD)/../Param.mk
+
+EXTRA_CFLAGS += -I$(PWD)/include
+# Module names
+obj-m := dw-csi.o dw-dphy.o es-csi-mipi.o
+
+# Source files for each module
+dw-csi-objs := dw-csi-plat.o dw-mipi-csi.o
+dw-dphy-objs := dw-dphy-rx.o dw-dphy-plat.o
+es-csi-mipi-objs := eswin-mipi-dma.o
+
+PWD :=$(shell pwd)
+
+all:
+	make -C $(KERNEL_DIR) M=$(PWD) modules
+	@[ -e $(INSTALL_DIR)/lib ] || mkdir $(INSTALL_DIR)/lib -p
+	@cp dw-csi.ko $(INSTALL_DIR)/lib
+	@cp dw-dphy.ko $(INSTALL_DIR)/lib
+	@cp es-csi-mipi.ko $(INSTALL_DIR)/lib
+modules_install:
+	make -C $(KERNEL_DIR) M=$(PWD) modules_install
+clean:
+	rm -rf $(dw-csi-objs)
+	rm -rf $(dw-dphy-objs)
+	rm -rf $(es-csi-mipi-objs)
+	make -C $(KERNEL) M=`pwd` clean
+
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-plat.c b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-plat.c
new file mode 100644
index 000000000000..66873f7b1188
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-plat.c
@@ -0,0 +1,817 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver.
+ * Platform driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include "include/dw-csi-data.h"
+#include "include/dw-dphy-data.h"
+
+#include "dw-csi-plat.h"
+
+
+void __iomem *csc;
+void __iomem *demo;
+
+#ifdef OYX_MOD
+static int init_dphy(void __iomem *base_addr){
+	writel(0x30  ,base_addr + 4*0xc10  ) ;//PPI_STARTUP_RW_COMMON_DPHY_10
+    writel(0x444 ,base_addr + 4*0x1cf2 ) ;//CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_2
+    writel(0x1444,base_addr + 4*0x1cf2 ) ;//CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_2
+    writel(0x1bfd,base_addr + 4*0x1cf0 ) ;//CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_0
+    writel(0x233 ,base_addr + 4*0xc11  ) ;//PPI_STARTUP_RW_COMMON_STARTUP_1_1
+    writel(0x27  ,base_addr + 4*0xc06  ) ;//PPI_STARTUP_RW_COMMON_DPHY_6
+    writel(0x1f4 ,base_addr + 4*0xc26  ) ;//PPI_CALIBCTRL_RW_COMMON_BG_0
+    writel(0x320 ,base_addr + 4*0xe02  ) ;//PPI_RW_LPDCOCAL_NREF
+    writel(0x1b  ,base_addr + 4*0xe03  ) ;//PPI_RW_LPDCOCAL_NREF_RANGE
+    writel(0xfec8,base_addr + 4*0xe05  ) ;//PPI_RW_LPDCOCAL_TWAIT_CONFIG
+    writel(0x646e,base_addr + 4*0xe06  ) ;//PPI_RW_LPDCOCAL_VT_CONFIG
+    writel(0x646e,base_addr + 4*0xe06  ) ;//PPI_RW_LPDCOCAL_VT_CONFIG
+    writel(0x646e,base_addr + 4*0xe06  ) ;//PPI_RW_LPDCOCAL_VT_CONFIG
+    writel(0x105 ,base_addr + 4*0xe08  ) ;//PPI_RW_LPDCOCAL_COARSE_CFG
+    writel(0x3   ,base_addr + 4*0xe36  ) ;//PPI_RW_COMMON_CFG
+    writel(0x5   ,base_addr + 4*0xc02  ) ;//PPI_STARTUP_RW_COMMON_DPHY_2
+    writel(0x17  ,base_addr + 4*0xe40  ) ;//PPI_RW_TERMCAL_CFG_0
+    writel(0x4   ,base_addr + 4*0xe50  ) ;//PPI_RW_OFFSETCAL_CFG_0
+    writel(0x5f  ,base_addr + 4*0xe01  ) ;//PPI_RW_LPDCOCAL_TIMEBASE
+    writel(0xfe1d,base_addr + 4*0xe05  ) ;//PPI_RW_LPDCOCAL_TWAIT_CONFIG
+    writel(0xeee ,base_addr + 4*0xe06  ) ;//PPI_RW_LPDCOCAL_VT_CONFIG
+    writel(0x0   ,base_addr + 4*0x1c20 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_0
+    writel(0x400 ,base_addr + 4*0x1c21 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_1
+    writel(0x400 ,base_addr + 4*0x1c21 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_1
+    writel(0x41f6,base_addr + 4*0x1c23 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_3
+    writel(0x0   ,base_addr + 4*0x1c20 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_0
+    writel(0x43f6,base_addr + 4*0x1c23 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_3
+    writel(0x2000,base_addr + 4*0x1c26 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_6
+    writel(0x0   ,base_addr + 4*0x1c27 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7
+    writel(0x3000,base_addr + 4*0x1c26 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_6
+    writel(0x0   ,base_addr + 4*0x1c27 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7
+    writel(0x7000,base_addr + 4*0x1c26 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_6
+    writel(0x0   ,base_addr + 4*0x1c27 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7
+    writel(0x4000,base_addr + 4*0x1c25 ) ;//CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_5
+    writel(0xf4  ,base_addr + 4*0x1c40 ) ;//CORE_DIG_RW_COMMON_0
+    writel(0xf4  ,base_addr + 4*0x1c40 ) ;//CORE_DIG_RW_COMMON_0
+    writel(0x14  ,base_addr + 4*0x1c47 ) ;//CORE_DIG_RW_COMMON_7
+    writel(0x10  ,base_addr + 4*0x1c47 ) ;//CORE_DIG_RW_COMMON_7
+    writel(0x0   ,base_addr + 4*0x1c47 ) ;//CORE_DIG_RW_COMMON_7
+    writel(0x50  ,base_addr + 4*0xc08  ) ;//PPI_STARTUP_RW_COMMON_DPHY_8
+    writel(0x68  ,base_addr + 4*0xc07  ) ;//PPI_STARTUP_RW_COMMON_DPHY_7
+    writel(0x473c,base_addr + 4*0x3040 ) ;//CORE_DIG_DLANE_0_RW_LP_0
+    writel(0x473c,base_addr + 4*0x3240 ) ;//CORE_DIG_DLANE_1_RW_LP_0
+    writel(0x0   ,base_addr + 4*0x1022 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_2
+    writel(0x1   ,base_addr + 4*0x1222 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_2
+    writel(0x0   ,base_addr + 4*0x1422 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_2
+    writel(0x9   ,base_addr + 4*0x1c46 ) ;//CORE_DIG_RW_COMMON_6
+    writel(0x9   ,base_addr + 4*0x1c46 ) ;//CORE_DIG_RW_COMMON_6
+    writel(0x802 ,base_addr + 4*0x102c ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_12
+    writel(0x802 ,base_addr + 4*0x122c ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_12
+    writel(0x802 ,base_addr + 4*0x142c ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_12
+    writel(0x2   ,base_addr + 4*0x102d ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_13
+    writel(0x2   ,base_addr + 4*0x122d ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_13
+    writel(0x2   ,base_addr + 4*0x142d ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_13
+    writel(0x802 ,base_addr + 4*0x102c ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_12
+    writel(0x802 ,base_addr + 4*0x122c ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_12
+    writel(0x802 ,base_addr + 4*0x142c ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_12
+    writel(0xa   ,base_addr + 4*0x102d ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_13
+    writel(0xa   ,base_addr + 4*0x122d ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_13
+    writel(0xa   ,base_addr + 4*0x142d ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_13
+    writel(0xa70 ,base_addr + 4*0x1229 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_9
+    writel(0x0   ,base_addr + 4*0x102a ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_10
+    writel(0x0   ,base_addr + 4*0x122a ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_10
+    writel(0x0   ,base_addr + 4*0x142a ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_10
+    writel(0x4   ,base_addr + 4*0x102f ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_15
+    writel(0x4   ,base_addr + 4*0x122f ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_15
+    writel(0x4   ,base_addr + 4*0x142f ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_15
+    writel(0x91c ,base_addr + 4*0x3880 ) ;//CORE_DIG_DLANE_CLK_RW_HS_RX_0
+    writel(0x3b06,base_addr + 4*0x3887 ) ;//CORE_DIG_DLANE_CLK_RW_HS_RX_7
+    writel(0xe1d ,base_addr + 4*0x3080 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_0
+    writel(0xe1d ,base_addr + 4*0x3280 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_0
+    writel(0x0   ,base_addr + 4*0x3001 ) ;//CORE_DIG_DLANE_0_RW_CFG_1
+    writel(0x0   ,base_addr + 4*0x3201 ) ;//CORE_DIG_DLANE_1_RW_CFG_1
+    writel(0x8   ,base_addr + 4*0x3001 ) ;//CORE_DIG_DLANE_0_RW_CFG_1
+    writel(0x8   ,base_addr + 4*0x3201 ) ;//CORE_DIG_DLANE_1_RW_CFG_1
+    writel(0xe69b,base_addr + 4*0x3082 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_2
+    writel(0xe69b,base_addr + 4*0x3282 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_2
+    writel(0x173c,base_addr + 4*0x3040 ) ;//CORE_DIG_DLANE_0_RW_LP_0
+    writel(0x173c,base_addr + 4*0x3240 ) ;//CORE_DIG_DLANE_1_RW_LP_0
+    writel(0x0   ,base_addr + 4*0x3042 ) ;//CORE_DIG_DLANE_0_RW_LP_2
+    writel(0x0   ,base_addr + 4*0x3242 ) ;//CORE_DIG_DLANE_1_RW_LP_2
+    writel(0x163c,base_addr + 4*0x3840 ) ;//CORE_DIG_DLANE_CLK_RW_LP_0
+    writel(0x0   ,base_addr + 4*0x3842 ) ;//CORE_DIG_DLANE_CLK_RW_LP_2
+    writel(0xe69b,base_addr + 4*0x3082 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_2
+    writel(0xe69b,base_addr + 4*0x3282 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_2
+    writel(0x4010,base_addr + 4*0x3081 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_1
+    writel(0x4010,base_addr + 4*0x3281 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_1
+    writel(0xe69b,base_addr + 4*0x3082 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_2
+    writel(0xe69b,base_addr + 4*0x3282 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_2
+    writel(0x9209,base_addr + 4*0x3083 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_3
+    writel(0x9209,base_addr + 4*0x3283 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_3
+    writel(0x96  ,base_addr + 4*0x3084 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_4
+    writel(0x96  ,base_addr + 4*0x3284 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_4
+    writel(0x100 ,base_addr + 4*0x3085 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_5
+    writel(0x100 ,base_addr + 4*0x3285 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_5
+    writel(0x100 ,base_addr + 4*0x3085 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_5
+    writel(0x100 ,base_addr + 4*0x3285 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_5
+    writel(0x2d02,base_addr + 4*0x3086 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_6
+    writel(0x2d02,base_addr + 4*0x3286 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_6
+    writel(0x1b06,base_addr + 4*0x3087 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_7
+    writel(0x1b06,base_addr + 4*0x3287 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_7
+    writel(0x1b06,base_addr + 4*0x3087 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_7
+    writel(0x1b06,base_addr + 4*0x3287 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_7
+    writel(0x9201,base_addr + 4*0x3083 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_3
+    writel(0x9201,base_addr + 4*0x3283 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_3
+    writel(0x0   ,base_addr + 4*0x3089 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_9
+    writel(0x0   ,base_addr + 4*0x3289 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_9
+    writel(0x2   ,base_addr + 4*0x3086 ) ;//CORE_DIG_DLANE_0_RW_HS_RX_6
+    writel(0x2   ,base_addr + 4*0x3286 ) ;//CORE_DIG_DLANE_1_RW_HS_RX_6
+    writel(0x404 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x40c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x414 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x41c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x423 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x429 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x430 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x43a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x445 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x44a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x450 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x45a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x465 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x469 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x472 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x47a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x485 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x489 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x490 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x49a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4a4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4ac ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4b4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4bc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4c4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4cc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4d4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4dc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4e4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4ec ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4f4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x4fc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x504 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x50c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x514 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x51c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x523 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x529 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x530 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x53a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x545 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x54a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x550 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x55a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x565 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x569 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x572 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x57a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x585 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x589 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x590 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x59a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5a4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5ac ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5b4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5bc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5c4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5cc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5d4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5dc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5e4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5ec ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5f4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x5fc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x604 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x60c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x614 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x61c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x623 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x629 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x632 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x63a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x645 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x64a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x650 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x65a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x665 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x669 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x672 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x67a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x685 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x689 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x690 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x69a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6a4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6ac ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6b4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6bc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6c4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6cc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6d4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6dc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6e4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6ec ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6f4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x6fc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x704 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x70c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x714 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x71c ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x723 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x72a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x730 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x73a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x745 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x74a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x750 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x75a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x765 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x769 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x772 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x77a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x785 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x789 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x790 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x79a ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7a4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7ac ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7b4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7bc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7c4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7cc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7d4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7dc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7e4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7ec ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7f4 ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x7fc ,base_addr + 4*0x1ff0 ) ;//CORE_DIG_COMMON_RW_DESKEW_FINE_MEM
+    writel(0x3   ,base_addr + 4*0x3800 ) ;//CORE_DIG_DLANE_CLK_RW_CFG_0
+    writel(0x3   ,base_addr + 4*0x3800 ) ;//CORE_DIG_DLANE_CLK_RW_CFG_0
+    writel(0x3   ,base_addr + 4*0x3000 ) ;//CORE_DIG_DLANE_0_RW_CFG_0
+    writel(0x3   ,base_addr + 4*0x3200 ) ;//CORE_DIG_DLANE_1_RW_CFG_0
+    writel(0x3   ,base_addr + 4*0x3000 ) ;//CORE_DIG_DLANE_0_RW_CFG_0
+    writel(0x3   ,base_addr + 4*0x3200 ) ;//CORE_DIG_DLANE_1_RW_CFG_0
+    writel(0xbf0 ,base_addr + 4*0x1029 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9
+    writel(0xb70 ,base_addr + 4*0x1229 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_9
+    writel(0xbf0 ,base_addr + 4*0x1429 ) ;//CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_9
+	udelay(20000);
+    return 0;
+}
+#endif
+
+static struct mipi_fmt *
+find_dw_mipi_csi_format(struct v4l2_mbus_framefmt *mf)
+{
+	unsigned int i;
+
+	pr_info("%s entered mbus: 0x%x\n", __func__, mf->code);
+
+	for (i = 0; i < ARRAY_SIZE(dw_mipi_csi_formats); i++)
+		if (mf->code == dw_mipi_csi_formats[i].mbus_code) {
+			pr_info("Found mbus 0x%x\n", dw_mipi_csi_formats[i].mbus_code);
+			return &dw_mipi_csi_formats[i];
+		}
+	return NULL;
+}
+
+static int dw_mipi_csi_enum_mbus_code(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_state *cfg,
+				      struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index != 0)
+		return -EINVAL;
+
+	code->code = dw_mipi_csi_formats[code->index].mbus_code;
+	return 0;
+}
+
+static struct mipi_fmt *
+dw_mipi_csi_try_format(struct v4l2_mbus_framefmt *mf)
+{
+	struct mipi_fmt *fmt;
+
+	fmt = find_dw_mipi_csi_format(mf);
+	if (!fmt)
+		fmt = &dw_mipi_csi_formats[0];
+
+	mf->code = fmt->mbus_code;
+
+	return fmt;
+}
+
+static struct v4l2_mbus_framefmt *
+dw_mipi_csi_get_format(struct dw_csi *dev, struct v4l2_subdev_state *cfg,
+		       enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return cfg ? v4l2_subdev_get_try_format(&dev->sd, cfg, 0) : NULL;
+	dev_info(dev->dev,
+		"%s got v4l2_mbus_pixelcode. 0x%x\n", __func__,
+		dev->format.code);
+	dev_info(dev->dev,
+		"%s got width. 0x%x\n", __func__,
+		dev->format.width);
+	dev_info(dev->dev,
+		"%s got height. 0x%x\n", __func__,
+		dev->format.height);
+	/*dev_dbg(dev->dev,
+		"%s got v4l2_mbus_pixelcode. 0x%x\n", __func__,
+		dev->format.code);
+	dev_dbg(dev->dev,
+		"%s got width. 0x%x\n", __func__,
+		dev->format.width);
+	dev_dbg(dev->dev,
+		"%s got height. 0x%x\n", __func__,
+		dev->format.height);*/
+	return &dev->format;
+}
+
+static int
+dw_mipi_csi_set_fmt(struct v4l2_subdev *sd,
+		    struct v4l2_subdev_state *cfg,
+		    struct v4l2_subdev_format *fmt)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+	struct mipi_fmt *dev_fmt = NULL;
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+
+	dev_info(dev->dev,
+		"%s got mbus_pixelcode. 0x%x\n", __func__,
+		fmt->format.code);
+
+	dev_fmt = dw_mipi_csi_try_format(&fmt->format);
+	dev_info(dev->dev,
+		"%s got v4l2_mbus_pixelcode. 0x%x\n", __func__,
+		dev_fmt->mbus_code);
+	if (!fmt)
+		return -EINVAL;
+
+	if (dev_fmt) {
+		if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE)
+			dev->fmt = dev_fmt;
+		dev->fmt->mbus_code = mf->code;
+		dw_mipi_csi_set_ipi_fmt(dev);
+	}
+
+	return 0;
+}
+
+static int
+dw_mipi_csi_get_fmt(struct v4l2_subdev *sd,
+		    struct v4l2_subdev_state *cfg,
+		    struct v4l2_subdev_format *fmt)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+	struct v4l2_mbus_framefmt *mf;
+
+	mf = dw_mipi_csi_get_format(dev, cfg, fmt->which);
+	if (!mf)
+		return -EINVAL;
+
+	mutex_lock(&dev->lock);
+	fmt->format = *mf;
+	mutex_unlock(&dev->lock);
+
+	return 0;
+}
+
+static int
+dw_mipi_csi_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+
+	dev_info(dev->dev, "%s: on=%d\n", __func__, on);
+	if (on) {
+		dw_mipi_csi_hw_stdby(dev);
+		dw_mipi_csi_start(dev);
+#ifdef OYX_MOD
+		void __iomem *base = ioremap(0x510d8000, 0x10);
+		writel(0x2c3f5, base + 0x0); // phy_cfg0
+		void __iomem *base1 = ioremap(0x510c0000, 0x20000);
+		init_dphy(base1);
+		dw_mipi_csi_recfg(dev);
+		unsigned int i ;
+		for (i = 10; i > 0; i--){
+			if (readl(base + 0x4) == 3)
+				break;
+			udelay(20000);
+		}
+
+		if (readl(base + 0x4) != 3) {
+			printk("===========err========phy lock===\n");
+			printk("===========err========phy lock===\n");
+			printk("===========err========phy lock===\n");
+			printk("===========err========phy lock===\n");
+		} else {
+			printk("===========ok========phy lock===\n");
+		}
+		writel(0x2c075, base + 0x0); // phy_cfg0
+#endif
+	} else {
+#ifdef DWC_PHY_USING
+		phy_power_off(dev->phy);
+#endif
+		dw_mipi_csi_mask_irq_power_off(dev);
+	}
+	return 0;
+}
+
+static int
+dw_mipi_csi_log_status(struct v4l2_subdev *sd)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+
+	dw_mipi_csi_dump(dev);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_VIDEO_ADV_DEBUG)
+static int
+dw_mipi_csi_g_register(struct v4l2_subdev *sd, struct v4l2_dbg_register *reg)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+
+	dev_vdbg(dev->dev, "%s: reg=%llu\n", __func__, reg->reg);
+	reg->val = dw_mipi_csi_read(dev, reg->reg);
+
+	return 0;
+}
+#endif
+static int dw_mipi_csi_init_cfg(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *cfg)
+{
+	struct v4l2_mbus_framefmt *format =
+	    v4l2_subdev_get_try_format(sd, cfg, 0);
+
+	format->colorspace = V4L2_COLORSPACE_SRGB;
+	format->code = MEDIA_BUS_FMT_RGB888_1X24;
+	format->field = V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static int dw_mipi_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	return dw_mipi_csi_s_power(sd, enable);
+}
+
+#define DW_MIPI_STREAM_CMD _IOWR('u', 0x20, int)
+
+long dw_mipi_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
+{
+	switch (cmd) {
+	case DW_MIPI_STREAM_CMD: {
+		int *enable = arg;
+		dw_mipi_csi_s_power(sd, *enable);
+		break;
+	}
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct v4l2_subdev_core_ops dw_mipi_csi_core_ops = {
+	.s_power = dw_mipi_csi_s_power,
+	.log_status = dw_mipi_csi_log_status,
+	.ioctl = dw_mipi_ioctl,
+#if IS_ENABLED(CONFIG_VIDEO_ADV_DEBUG)
+	.g_register = dw_mipi_csi_g_register,
+#endif
+};
+
+static struct v4l2_subdev_pad_ops dw_mipi_csi_pad_ops = {
+	.init_cfg = dw_mipi_csi_init_cfg,
+	.enum_mbus_code = dw_mipi_csi_enum_mbus_code,
+	.get_fmt = dw_mipi_csi_get_fmt,
+	.set_fmt = dw_mipi_csi_set_fmt,
+};
+
+static const struct v4l2_subdev_video_ops dw_mipi_video_ops = {
+	.s_stream = dw_mipi_set_stream,
+};
+
+static struct v4l2_subdev_ops dw_mipi_csi_subdev_ops = {
+	.core = &dw_mipi_csi_core_ops,
+	.pad = &dw_mipi_csi_pad_ops,
+	.video = &dw_mipi_video_ops,
+};
+
+static irqreturn_t dw_mipi_csi_irq1(int irq, void *dev_id)
+{
+	struct dw_csi *csi_dev = dev_id;
+
+	dw_mipi_csi_irq_handler(csi_dev);
+
+	return IRQ_HANDLED;
+}
+
+static int
+dw_mipi_csi_parse_dt(struct platform_device *pdev, struct dw_csi *dev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct v4l2_fwnode_endpoint ep = { .bus_type = V4L2_MBUS_CSI2_DPHY };
+	int ret = 0;
+
+	if (of_property_read_u32(node, "snps,output-type", &dev->hw.output))
+		dev->hw.output = 2;
+
+	if (of_property_read_u32(node, "snps,en-ppi-width", &dev->hw.ppi_width))
+		dev->hw.ppi_width = 0;
+
+	if (dev->hw.ppi_width > 1) {
+		dev_err(&pdev->dev, "Wrong ppi width(%d), valid value is '0'(ppi8) or '1'(ppi16)\n",
+				dev->hw.ppi_width);
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32(node, "snps,en-phy-mode", &dev->hw.phy_mode))
+		dev->hw.phy_mode = 0;
+
+	if (of_property_read_u32(node, "ipi2_en", &dev->hw.ipi2_en))
+		dev->hw.ipi2_en = 0;
+
+	if (of_property_read_u32(node, "ipi2_vcid", &dev->hw.ipi2_vcid))
+		dev->hw.ipi2_vcid = 0;
+
+	if (of_property_read_u32(node, "ipi3_en", &dev->hw.ipi3_en))
+		dev->hw.ipi3_en = 0;
+
+	if (of_property_read_u32(node, "ipi3_vcid", &dev->hw.ipi3_vcid))
+		dev->hw.ipi3_vcid = 0;
+
+	node = of_graph_get_next_endpoint(node, NULL);
+	if (!node) {
+		dev_err(&pdev->dev, "No port node at %pOF\n",
+			pdev->dev.of_node);
+		return -EINVAL;
+	}
+	/* Get port node and validate MIPI-CSI channel id. */
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(node), &ep);
+	if (ret)
+		goto err;
+
+	dev->index = ep.base.port - 1;
+	if (dev->index >= CSI_MAX_ENTITIES) {
+		ret = -ENXIO;
+		goto err;
+	}
+	dev->hw.num_lanes = ep.bus.mipi_csi2.num_data_lanes;
+err:
+	of_node_put(node);
+	return ret;
+}
+
+static const struct of_device_id dw_mipi_csi_of_match[];
+
+static int dw_csi_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *of_id = NULL;
+	struct dw_csih_pdata *pdata = NULL;
+	struct device *dev = &pdev->dev;
+	struct dw_csi *csi;
+	struct v4l2_subdev *sd;
+	int ret;
+
+	dev_info(dev, "DW MIPI CSI-2 Host Probe in. \n");
+
+	if (!IS_ENABLED(CONFIG_OF))
+		pdata = pdev->dev.platform_data;
+
+	dev_vdbg(dev, "Probing started\n");
+
+	/* Resource allocation */
+	csi = devm_kzalloc(dev, sizeof(*csi), GFP_KERNEL);
+	if (!csi)
+		return -ENOMEM;
+
+	mutex_init(&csi->lock);
+	spin_lock_init(&csi->slock);
+	csi->dev = dev;
+
+	if (dev->of_node) {
+		of_id = of_match_node(dw_mipi_csi_of_match, dev->of_node);
+		if (!of_id)
+			return -EINVAL;
+
+		ret = dw_mipi_csi_parse_dt(pdev, csi);
+		if (ret < 0)
+			return ret;
+
+#ifdef DWC_PHY_USING
+		csi->phy = devm_of_phy_get(dev, dev->of_node, NULL);
+		if (IS_ERR(csi->phy)) {
+			dev_err(dev, "No DPHY available\n");
+			return PTR_ERR(csi->phy);
+		}
+#endif
+	} else {
+#ifdef DWC_PHY_USING
+		csi->phy = devm_phy_get(dev, phys[pdata->id].name);
+		if (IS_ERR(csi->phy)) {
+			dev_err(dev, "No '%s' DPHY available\n",
+				phys[pdata->id].name);
+			return PTR_ERR(csi->phy);
+		}
+		dev_info(dev, "got D-PHY %s with id %d\n", phys[pdata->id].name,
+			 csi->phy->id);
+#endif
+	}
+
+	/* Registers mapping */
+	csi->base_address = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(csi->base_address)) {
+		dev_err(dev, "base address not set.\n");
+		return PTR_ERR(csi->base_address);
+	}
+
+	csi->ctrl_irq_number = platform_get_irq(pdev, 0);
+	if (csi->ctrl_irq_number < 0) {
+		dev_err(dev, "irq number %d not set.\n", csi->ctrl_irq_number);
+		ret = csi->ctrl_irq_number;
+		goto end;
+	}
+
+	csi->rst = devm_reset_control_get_optional_shared(dev, NULL);
+	if (IS_ERR(csi->rst)) {
+		dev_err(dev, "error getting reset control %d\n", ret);
+		return PTR_ERR(csi->rst);
+	}
+
+	ret = devm_request_irq(dev, csi->ctrl_irq_number, dw_mipi_csi_irq1,
+			       IRQF_SHARED, dev_name(dev), csi);
+	if (ret) {
+		if (dev->of_node)
+			dev_err(dev, "irq csi %s failed\n", of_id->name);
+		else
+			dev_err(dev, "irq csi %d failed\n", pdata->id);
+
+		goto end;
+	}
+
+	sd = &csi->sd;
+	v4l2_subdev_init(sd, &dw_mipi_csi_subdev_ops);
+	sd->dev = dev;
+	csi->sd.owner = THIS_MODULE;
+
+	if (dev->of_node) {
+		snprintf(sd->name, sizeof(sd->name), "%s.%d",
+			 "dw-csi", csi->index);
+
+		csi->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	} else {
+		strlcpy(sd->name, dev_name(dev), sizeof(sd->name));
+	}
+	csi->fmt = &dw_mipi_csi_formats[0];
+	csi->format.code = dw_mipi_csi_formats[0].mbus_code;
+
+	sd->entity.function = MEDIA_ENT_F_IO_V4L;
+
+	if (dev->of_node) {
+		csi->pads[CSI_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+		csi->pads[CSI_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+
+		ret = media_entity_pads_init(&csi->sd.entity, CSI_PADS_NUM, csi->pads);
+		if (ret < 0) {
+			dev_err(dev, "media entity init failed\n");
+			goto end;
+		}
+	} else {
+		csi->hw.num_lanes = pdata->lanes;
+		csi->hw.pclk = pdata->pclk;
+		csi->hw.fps = pdata->fps;
+		csi->hw.dphy_freq = pdata->hs_freq;
+	}
+
+	v4l2_set_subdevdata(&csi->sd, pdev);
+	platform_set_drvdata(pdev, &csi->sd);
+	dev_set_drvdata(dev, sd);
+
+	if (csi->rst)
+		reset_control_deassert(csi->rst);
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_csi_create_capabilities_sysfs(pdev);
+#endif
+	dw_mipi_csi_get_version(csi);
+	dw_mipi_csi_specific_mappings(csi);
+#ifdef OYX_MOD
+	void __iomem *base = ioremap(0x51828000, 0x500);
+	writel(0xffffffff, base + 0x200);
+	writel(0x3ff, base + 0x424);
+    udelay(20000);
+
+	writel(0x7, base + 0x470);
+	writel(0x1, base + 0x474);
+	writel(0x1, base + 0x478);
+	writel(0x1, base + 0x47c);
+	writel(0x1, base + 0x480);
+
+    udelay(20000);
+
+	writel(0x80000020, base + 0x184);
+	writel(0xc0000020, base + 0x188);
+	writel(0x80000020, base + 0x18c);
+	writel(0x80000020, base + 0x190);
+	writel(0x80000100, base + 0x194);
+	writel(0x80000100, base + 0x198);
+	writel(0x80000100, base + 0x19c);
+	writel(0x80000100, base + 0x1a0);
+	writel(0x80000100, base + 0x1a4);
+	writel(0x80000100, base + 0x1a8);
+	writel(0x3, base + 0x1ac);
+
+	void __iomem *base1 = ioremap(0x51030000, 0x100);
+	writel(0xffffffff, base1 + 0x40);
+    udelay(20000);
+#endif
+	dw_mipi_csi_mask_irq_power_off(csi);
+	dw_mipi_csi_fill_timings(csi);
+
+	printk("DW MIPI CSI-2 Host registered successfully HW v%u.%u\n",
+		 csi->hw_version_major, csi->hw_version_minor);
+
+#ifdef DWC_PHY_USING
+	ret = phy_init(csi->phy);
+	if (ret) {
+		dev_err(&csi->phy->dev, "phy init failed --> %d\n", ret);
+		goto end;
+	}
+#endif
+
+	ret = v4l2_async_register_subdev(sd);
+	if (ret < 0) {
+		dev_err(dev, "failed to register subdev\n");
+		goto end;
+	}
+
+	return 0;
+
+end:
+#if IS_ENABLED(CONFIG_OF)
+	media_entity_cleanup(&csi->sd.entity);
+	return ret;
+#endif
+	//v4l2_device_unregister(csi->vdev.v4l2_dev);
+	return ret;
+}
+
+static int dw_csi_remove(struct platform_device *pdev)
+{
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *mipi_csi = sd_to_mipi_csi_dev(sd);
+
+	if (mipi_csi->rst)
+		reset_control_assert(mipi_csi->rst);
+
+    v4l2_async_unregister_subdev(sd);
+
+#if IS_ENABLED(CONFIG_OF)
+	media_entity_cleanup(&mipi_csi->sd.entity);
+#else
+	v4l2_device_unregister(mipi_csi->vdev.v4l2_dev);
+#endif
+	dev_info(&pdev->dev, "DW MIPI CSI-2 Host module removed\n");
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id dw_mipi_csi_of_match[] = {
+	{ .compatible = "snps,dw-csi" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dw_mipi_csi_of_match);
+#endif
+
+static struct platform_driver dw_mipi_csi_driver = {
+	.remove = dw_csi_remove,
+	.probe = dw_csi_probe,
+	.driver = {
+		.name = "dw-csi",
+		.owner = THIS_MODULE,
+#if IS_ENABLED(CONFIG_OF)
+		.of_match_table = of_match_ptr(dw_mipi_csi_of_match),
+#endif
+	},
+};
+
+module_platform_driver(dw_mipi_csi_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Luis Oliveira <luis.oliveira@synopsys.com>");
+MODULE_DESCRIPTION("Synopsys DesignWare MIPI CSI-2 Host Platform driver");
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-plat.h b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-plat.h
new file mode 100644
index 000000000000..a4649383d73f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-plat.h
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Synopsys, Inc.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver.
+ * Supported bus formats
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#ifndef _DW_CSI_PLAT_H__
+#define _DW_CSI_PLAT_H__
+
+#include "dw-mipi-csi.h"
+
+/** Color Space Converter Block **/
+#define CSC_UNIT_NR 1
+#define CSC_COREID 0x0000
+#define CSC_CFG 0x0004
+#define CSC_COEF_A(x) (0x0008 + ((x) * 4))
+#define CSC_COEF_B(x) (0x0018 + ((x) * 4))
+#define CSC_COEF_C(x) (0x0028 + ((x) * 4))
+#define CSC_LIMIT_DN 0x0038
+#define CSC_LIMIT_UP 0x003C
+#define CSC_VID_CFG 0x0040
+
+/** Demosaic Block **/
+#define DEMO_CONTROL 0x0
+#define DEMO_GLOBAL_INT_EN 0x4
+#define DEMO_IP_INT_EN_REG 0x8
+#define DEMO_IP_INT_SATUS_REG 0xC
+#define DEMO_ACTIVE_WIDTH 0x10
+#define DEMO_ACTIVE_HEIGHT 0x18
+#define DEMO_BAYER_PHASE 0x28//0x20
+
+#define BAYER_RGGB 0x0
+#define BAYER_GRBG 0x1
+#define BAYER_GBRG 0x2
+#define BAYER_BGGR 0x3
+
+/* Video formats supported by the MIPI CSI-2 */
+struct mipi_fmt dw_mipi_csi_formats[] = {
+	{
+		/* RAW 8 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
+		.depth = 8,
+#if 0
+	}, {
+		/* RAW 6 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR6_1X8,
+		.depth = 8,
+	}, {
+		/* RAW 7 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR7_1X8,
+		.depth = 8,
+#endif
+	}, {
+		/* RAW 10 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10,
+		.depth = 10,
+	}, {
+		/* RAW 8 */
+		.mbus_code = 0x3001,
+		.depth = 8,
+	}, {
+		/* RAW 12 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12,
+		.depth = 12,
+	}, {
+		/* RAW 14 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR14_1X14,
+		.depth = 14,
+	}, {
+		/* RAW 16 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR16_1X16,
+		.depth = 16,
+	}, {
+		/* RGB 666 */
+		.mbus_code = MEDIA_BUS_FMT_RGB666_1X18,
+		.depth = 18,
+	}, {
+		/* RGB 565 */
+		.mbus_code = MEDIA_BUS_FMT_RGB565_2X8_BE,
+		.depth = 16,
+	}, {
+		/* BGR 565 */
+		.mbus_code = MEDIA_BUS_FMT_RGB565_2X8_LE,
+		.depth = 16,
+	}, {
+		/* RGB 555 */
+		.mbus_code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE,
+		.depth = 16,
+	}, {
+		/* BGR 555 */
+		.mbus_code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
+		.depth = 16,
+	}, {
+		/* RGB 444 */
+		.mbus_code = MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE,
+		.depth = 16,
+	}, {
+		/* RGB 444 */
+		.mbus_code = MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE,
+		.depth = 16,
+	}, {
+		/* RGB 888 */
+		.mbus_code = MEDIA_BUS_FMT_RGB888_2X12_LE,
+		.depth = 24,
+	}, {
+		/* BGR 888 */
+		.mbus_code = MEDIA_BUS_FMT_RGB888_2X12_BE,
+		.depth = 24,
+	}, {
+		/* BGR 888 */
+		.mbus_code = MEDIA_BUS_FMT_RGB888_1X24,
+		.depth = 24,
+	}, {
+		/* YUV 422 8-bit */
+		.mbus_code = MEDIA_BUS_FMT_VYUY8_1X16,
+		.depth = 16,
+	}, {
+		/* YUV 422 10-bit */
+		.mbus_code = MEDIA_BUS_FMT_UYVY10_1X20,
+		.depth = 24,
+	}, {
+		/* YUV 420 8-bit LEGACY */
+		.mbus_code = MEDIA_BUS_FMT_Y8_1X8,
+		.depth = 8,
+	}, {
+		/* YUV 420 8-bit LEGACY */
+		.mbus_code = MEDIA_BUS_FMT_YUYV8_1X16,
+		.depth = 24,
+	}, {
+		/* YUV 420 10-bit */
+		.mbus_code = MEDIA_BUS_FMT_VUY8_1X24,
+		.depth = 24,
+	}, {
+		/* YUV 420 8-bit */
+		.mbus_code = MEDIA_BUS_FMT_UYVY8_1X16,
+		.depth = 24,
+	}, {
+		/* YUV 420 10-bit */
+		.mbus_code = MEDIA_BUS_FMT_Y10_1X10,
+		.depth = 10,
+	},
+};
+#endif /* _DW_CSI_PLAT_H__ */
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-sysfs.c b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-sysfs.c
new file mode 100644
index 000000000000..e8d2bb9d0eab
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-csi-sysfs.c
@@ -0,0 +1,624 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver.
+ * SysFS components for the platform driver
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include "dw-mipi-csi.h"
+
+static ssize_t core_version_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "v.%d.%d*\n", csi_dev->hw_version_major,
+		 csi_dev->hw_version_minor);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t n_lanes_store(struct device *dev, struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	int ret;
+	unsigned long lanes;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &lanes);
+	if (ret < 0)
+		return ret;
+
+	if (lanes > 8) {
+		dev_err(dev, "Invalid number of lanes %lu\n", lanes);
+		return count;
+	}
+
+	dev_info(dev, "Lanes %lu\n", lanes);
+	csi_dev->hw.num_lanes = lanes;
+
+	return count;
+}
+
+static ssize_t n_lanes_show(struct device *dev,
+			    struct device_attribute *attr,
+			    char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.num_lanes);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t core_reset_show(struct device *dev,
+			       struct device_attribute *attr,
+			       char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	/* Reset Controller and DPHY */
+	phy_reset(csi_dev->phy);
+	dw_mipi_csi_reset(csi_dev);
+
+	snprintf(buffer, 10, "Reset\n");
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t data_type_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	int ret;
+	unsigned long dt;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &dt);
+	if (ret < 0)
+		return ret;
+
+	if (dt < 0x18 || dt > 0x2F) {
+		dev_err(dev, "Invalid data type %lx\n", dt);
+		return count;
+	}
+
+	dev_info(dev, "Data type 0x%lx\n", dt);
+	csi_dev->ipi_dt = dt;
+
+	return count;
+}
+
+static ssize_t data_type_show(struct device *dev,
+			      struct device_attribute *attr,
+			      char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->ipi_dt);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t hsa_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long hsa;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &hsa);
+	if (ret < 0)
+		return ret;
+
+	if (hsa > 0xFFF) {
+		dev_err(dev, "Invalid HSA time %lx\n", hsa);
+		return count;
+	}
+
+	dev_info(dev, "HSA time 0x%lx\n", hsa);
+	csi_dev->hw.hsa = hsa;
+
+	return count;
+}
+
+static ssize_t hsa_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.hsa);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t hbp_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long hbp;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &hbp);
+	if (ret < 0)
+		return ret;
+
+	if (hbp > 0xFFF) {
+		dev_err(dev, "Invalid HBP time %lx\n", hbp);
+		return count;
+	}
+
+	dev_info(dev, "HBP time 0x%lx\n", hbp);
+	csi_dev->hw.hbp = hbp;
+
+	return count;
+}
+
+static ssize_t hbp_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.hbp);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t hsd_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long hsd;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &hsd);
+	if (ret < 0)
+		return ret;
+
+	if (hsd > 0xFF) {
+		dev_err(dev, "Invalid HSD time %lx\n", hsd);
+		return count;
+	}
+
+	dev_info(dev, "HSD time 0x%lx\n", hsd);
+	csi_dev->hw.hsd = hsd;
+
+	return count;
+}
+
+static ssize_t hsd_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.hsd);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t vsa_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long vsa;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &vsa);
+	if (ret < 0)
+		return ret;
+
+	if (vsa > 0x3FF) {
+		dev_err(dev, "Invalid VSA period %lx\n", vsa);
+		return count;
+	}
+
+	dev_info(dev, "VSA period 0x%lx\n", vsa);
+	csi_dev->hw.vsa = vsa;
+
+	return count;
+}
+
+static ssize_t vsa_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.vsa);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t vbp_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long vbp;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &vbp);
+	if (ret < 0)
+		return ret;
+
+	if (vbp > 0x2FF) {
+		dev_err(dev, "Invalid VBP period %lx\n", vbp);
+		return count;
+	}
+
+	dev_info(dev, "VBP period 0x%lx\n", vbp);
+	csi_dev->hw.vbp = vbp;
+
+	return count;
+}
+
+static ssize_t vbp_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.vbp);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t vfp_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long vfp;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &vfp);
+	if (ret < 0)
+		return ret;
+
+	if (vfp > 0x3ff) {
+		dev_err(dev, "Invalid VFP period %lx\n", vfp);
+		return count;
+	}
+
+	dev_info(dev, "VFP period 0x%lx\n", vfp);
+	csi_dev->hw.vfp = vfp;
+
+	return count;
+}
+
+static ssize_t vfp_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.vfp);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t virtual_channel_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	int ret;
+	unsigned long virtual_ch;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &virtual_ch);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)virtual_ch < 0 || (signed int)virtual_ch > 8) {
+		dev_err(dev, "Invalid Virtual Channel %lu\n", virtual_ch);
+		return count;
+	}
+
+	dev_info(dev, "Virtual Channel %lu\n", virtual_ch);
+	csi_dev->hw.virtual_ch = virtual_ch;
+
+	return count;
+}
+
+static ssize_t virtual_channel_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.virtual_ch);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t ipi_color_mode_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int ret;
+	unsigned long ipi_color_mode;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &ipi_color_mode);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)ipi_color_mode < 0 || (signed int)ipi_color_mode > 1) {
+		dev_err(dev,
+			"Wrong Color Mode %lu, (48 bits -> 0 or 16 bits -> 1\n",
+			ipi_color_mode);
+		return count;
+	}
+
+	dev_info(dev, "IPI Color mode %lu\n", ipi_color_mode);
+	csi_dev->hw.ipi_color_mode = ipi_color_mode;
+
+	return count;
+}
+
+static ssize_t ipi_color_mode_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.ipi_color_mode);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t ipi_auto_flush_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int ret;
+	unsigned long ipi_auto_flush;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &ipi_auto_flush);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)ipi_auto_flush < 0 || (signed int)ipi_auto_flush > 1) {
+		dev_err(dev,
+			"Invalid Auto Flush Mode %lu, (No -> 0 or Yes -> 1\n",
+			ipi_auto_flush);
+		return count;
+	}
+
+	dev_info(dev, "IPI Auto Flush %lu\n", ipi_auto_flush);
+	csi_dev->hw.ipi_auto_flush = ipi_auto_flush;
+
+	return count;
+}
+
+static ssize_t ipi_auto_flush_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.ipi_auto_flush);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t ipi_timings_mode_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t count)
+{
+	int ret;
+	unsigned long ipi_mode;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &ipi_mode);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)ipi_mode < 0 || (signed int)ipi_mode > 1) {
+		dev_err(dev,
+			"Invalid Timing Source %lu (Camera:0|Controller:1)\n",
+			ipi_mode);
+		return count;
+	}
+
+	dev_info(dev, "IPI Color mode %lu\n", ipi_mode);
+	csi_dev->hw.ipi_mode = ipi_mode;
+
+	return count;
+}
+
+static ssize_t ipi_timings_mode_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.ipi_mode);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t output_type_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long output;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &output);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)output < 0 || (signed int)output > 1) {
+		dev_err(dev,
+			"Invalid Core output %lu to be used \
+			(IPI-> 0 or IDI->1 or BOTH- 2\n",
+			output);
+		return count;
+	}
+
+	dev_info(dev, "IPI Color mode %lu\n", output);
+	csi_dev->hw.output = output;
+
+	return count;
+}
+
+static ssize_t output_type_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.output);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static DEVICE_ATTR_RO(core_version);
+static DEVICE_ATTR_RO(core_reset);
+static DEVICE_ATTR_RW(n_lanes);
+static DEVICE_ATTR_RW(data_type);
+static DEVICE_ATTR_RW(hsa);
+static DEVICE_ATTR_RW(hbp);
+static DEVICE_ATTR_RW(hsd);
+static DEVICE_ATTR_RW(vsa);
+static DEVICE_ATTR_RW(vbp);
+static DEVICE_ATTR_RW(vfp);
+static DEVICE_ATTR_RW(virtual_channel);
+static DEVICE_ATTR_RW(ipi_color_mode);
+static DEVICE_ATTR_RW(ipi_auto_flush);
+static DEVICE_ATTR_RW(ipi_timings_mode);
+static DEVICE_ATTR_RW(output_type);
+
+int dw_csi_create_capabilities_sysfs(struct platform_device *pdev)
+{
+	device_create_file(&pdev->dev, &dev_attr_core_version);
+	device_create_file(&pdev->dev, &dev_attr_core_reset);
+	device_create_file(&pdev->dev, &dev_attr_n_lanes);
+	device_create_file(&pdev->dev, &dev_attr_data_type);
+	device_create_file(&pdev->dev, &dev_attr_hsa);
+	device_create_file(&pdev->dev, &dev_attr_hbp);
+	device_create_file(&pdev->dev, &dev_attr_hsd);
+	device_create_file(&pdev->dev, &dev_attr_vsa);
+	device_create_file(&pdev->dev, &dev_attr_vbp);
+	device_create_file(&pdev->dev, &dev_attr_vfp);
+	device_create_file(&pdev->dev, &dev_attr_virtual_channel);
+	device_create_file(&pdev->dev, &dev_attr_ipi_color_mode);
+	device_create_file(&pdev->dev, &dev_attr_ipi_auto_flush);
+	device_create_file(&pdev->dev, &dev_attr_ipi_timings_mode);
+	device_create_file(&pdev->dev, &dev_attr_output_type);
+
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-data.h b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-data.h
new file mode 100644
index 000000000000..0b489b17fbda
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-data.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY platform data
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include <linux/phy/phy.h>
+#include <linux/kernel.h>
+#include "include/dw-mipi-csi-pltfrm.h"
+
+struct dw_phy_pdata {
+	u32 dphy_frequency;
+	u8 dphy_te_len;
+	u32 config_8l;
+	u8 dphy_gen;
+	u8 phy_type;
+	u8 id;
+};
+
+static const struct pdata_names phys[] = {
+	{ .name = "phy-dw-dphy.0.0", },
+	{ .name = "phy-dw-dphy.1.1", },
+};
+
+struct dw_dphy_rx;
+
+struct plat_dw_dphy {
+	int (*get_resources)(struct device *dev, struct dw_dphy_rx *dphy);
+};
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-plat.c b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-plat.c
new file mode 100644
index 000000000000..538e34f3d1e2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-plat.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver.
+ * Platform driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include "include/dw-dphy-data.h"
+#include "include/dw-csi-data.h"
+
+#include "dw-dphy-rx.h"
+
+/* Global variable for compatibility mode, this could be override later */
+static int phy_type = 1; //Changed to fit single phy 0 - single | 1 - 8 lanes phy
+
+module_param(phy_type, int, 0664);
+MODULE_PARM_DESC(phy_type, "Disable compatibility mode for D-PHY G128");
+
+static struct phy_ops dw_dphy_ops = {
+	.init = dw_dphy_init,
+	.reset = dw_dphy_reset,
+	.power_on = dw_dphy_power_on,
+	.power_off = dw_dphy_power_off,
+	.owner = THIS_MODULE,
+};
+
+static struct phy_provider *phy_provider;
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static u8 get_config_8l(struct device *dev, struct dw_dphy_rx *dphy)
+{
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		dphy->config_8l = of_get_gpio(dev->of_node, 0);
+		if (!gpio_is_valid(dphy->config_8l)) {
+			dev_warn(dev,
+				 "failed to parse 8l config, default is 0\n");
+			dphy->config_8l = 0;
+		}
+	} else {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+
+		dphy->config_8l = pdata->config_8l;
+	}
+	return dphy->config_8l;
+}
+#endif
+
+static int get_resources(struct device *dev, struct dw_dphy_rx *dphy)
+{
+	int ret = 0;
+
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		if (of_property_read_u32(dev->of_node, "snps,dphy-frequency",
+					 &dphy->dphy_freq)) {
+			dev_err(dev, "failed to find dphy frequency\n");
+			ret = -EINVAL;
+		}
+		if (of_property_read_u32(dev->of_node, "bus-width",
+					 &dphy->dphy_te_len)) {
+			dev_err(dev, "failed to find dphy te length\n");
+			ret = -EINVAL;
+		}
+		if (of_property_read_u32(dev->of_node, "snps,phy_type",
+					 &dphy->phy_type)) {
+			dev_err(dev, "failed to find dphy type\n");
+			ret = -EINVAL;
+		}
+	} else {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+
+		dphy->dphy_freq = pdata->dphy_frequency;
+		dphy->dphy_te_len = pdata->dphy_te_len;
+		dphy->dphy_gen = pdata->dphy_gen;
+	}
+	dev_set_drvdata(dev, dphy);
+
+	return ret;
+}
+
+static int phy_register(struct device *dev)
+{
+	int ret = 0;
+
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		phy_provider = devm_of_phy_provider_register(dev,
+							     dw_dphy_xlate);
+		if (IS_ERR(phy_provider)) {
+			dev_err(dev, "error getting phy provider\n");
+			ret = PTR_ERR(phy_provider);
+		}
+	} else {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+		struct dw_dphy_rx *dphy = dev_get_drvdata(dev);
+
+		ret = phy_create_lookup(dphy->phy,
+					phys[pdata->id].name,
+					csis[pdata->id].name);
+		if (ret)
+			dev_err(dev, "Failed to create dphy lookup\n");
+		else
+			dev_warn(dev, "Created dphy lookup [%s] --> [%s]\n",
+				 phys[pdata->id].name, csis[pdata->id].name);
+	}
+	return ret;
+}
+
+static void phy_unregister(struct device *dev)
+{
+	if (!dev->of_node) {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+		struct dw_dphy_rx *dphy = dev_get_drvdata(dev);
+
+		phy_remove_lookup(dphy->phy,
+				  phys[pdata->id].name, csis[pdata->id].name);
+		dev_warn(dev, "Removed dphy lookup [%s] --> [%s]\n",
+			 phys[pdata->id].name, csis[pdata->id].name);
+	}
+}
+
+static int dw_dphy_rx_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dw_dphy_rx *dphy;
+	struct resource *res;
+
+	dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL);
+	if (!dphy)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dphy->base_address = devm_ioremap(&pdev->dev,
+					  res->start, resource_size(res));
+	if (IS_ERR(dphy->base_address)) {
+		dev_err(&pdev->dev, "error requesting base address\n");
+		return PTR_ERR(dphy->base_address);
+	}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+
+	dphy->dphy1_if_addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dphy->dphy1_if_addr)) {
+		dev_err(&pdev->dev, "error requesting dphy 1 if regbank\n");
+		return PTR_ERR(dphy->dphy1_if_addr);
+	}
+
+	dphy->max_lanes =
+		dw_dphy_if_read_msk(dphy, DPHYID, DPHY_ID_LANE_SUPPORT, 4);
+
+	dphy->dphy_gen = dw_dphy_if_read_msk(dphy, DPHYID, DPHY_ID_GEN, 4);
+
+	dev_info(&pdev->dev, "DPHY GEN %s with maximum %s lanes\n",
+		 dphy->dphy_gen == GEN3 ? "3" : "2",
+		 dphy->max_lanes == CTRL_8_LANES ? "8" : "4");
+
+	if (dphy->max_lanes == CTRL_8_LANES) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+		dphy->dphy2_if_addr =
+			devm_ioremap(&pdev->dev,
+				     res->start, resource_size(res));
+
+		if (IS_ERR(dphy->dphy2_if_addr)) {
+			dev_err(&pdev->dev,
+				"error requesting dphy 2 if regbank\n");
+			return PTR_ERR(dphy->dphy2_if_addr);
+		}
+		dphy->config_8l = get_config_8l(&pdev->dev, dphy);
+	}
+#endif
+	if (get_resources(dev, dphy)) {
+		dev_err(dev, "failed to parse PHY resources\n");
+		return -EINVAL;
+	}
+
+	dphy->phy = devm_phy_create(dev, NULL, &dw_dphy_ops);
+	if (IS_ERR(dphy->phy)) {
+		dev_err(dev, "failed to create PHY\n");
+		return PTR_ERR(dphy->phy);
+	}
+
+	platform_set_drvdata(pdev, dphy);
+	phy_set_drvdata(dphy->phy, dphy);
+
+	if (phy_register(dev)) {
+		dev_err(dev, "failed to register PHY\n");
+		return -EINVAL;
+	}
+
+	dphy->lp_time = 1000; /* 1000 ns */
+	dphy->lanes_config = dw_dphy_setup_config(dphy);
+
+	dev_info(&dphy->phy->dev, "Probing dphy finished\n");
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_create_capabilities_sysfs(pdev);
+#endif
+
+	return 0;
+}
+
+static int dw_dphy_rx_remove(struct platform_device *pdev)
+{
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+
+	dev_info(&dphy->phy->dev, "phy removed\n");
+	phy_unregister(&pdev->dev);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id dw_dphy_rx_of_match[] = {
+	{ .compatible = "snps,dw-dphy-rx" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dw_dphy_rx_of_match);
+#endif
+
+static struct platform_driver dw_dphy_rx_driver = {
+	.probe = dw_dphy_rx_probe,
+	.remove = dw_dphy_rx_remove,
+	.driver = {
+#if IS_ENABLED(CONFIG_OF)
+		.of_match_table = of_match_ptr(dw_dphy_rx_of_match),
+#endif
+		.name = "dw-dphy",
+		.owner = THIS_MODULE,
+	}
+};
+module_platform_driver(dw_dphy_rx_driver);
+
+MODULE_DESCRIPTION("Synopsys DesignWare MIPI DPHY Rx driver");
+MODULE_AUTHOR("Luis Oliveira <lolivei@synopsys.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-rx.c b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-rx.c
new file mode 100644
index 000000000000..970d2b572c02
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-rx.c
@@ -0,0 +1,613 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver
+ * Core functions
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include "dw-dphy-rx.h"
+
+struct range_dphy_gen2 {
+	u32 freq;
+	u8 hsfregrange;
+};
+
+struct range_dphy_gen2 range_gen2[] = {
+	{ 80, 0x00 }, { 90, 0x10 }, { 100, 0x20 }, { 110, 0x30 },
+	{ 120, 0x01 }, { 130, 0x11 }, { 140, 0x21 }, { 150, 0x31 },
+	{ 160, 0x02 }, { 170, 0x12 }, { 180, 0x22 }, { 190, 0x32 },
+	{ 205, 0x03 }, { 220, 0x13 }, { 235, 0x23 }, { 250, 0x33 },
+	{ 275, 0x04 }, { 300, 0x14 }, { 325, 0x05 }, { 350, 0x15 },
+	{ 400, 0x25 }, { 450, 0x06 }, { 500, 0x16 }, { 550, 0x07 },
+	{ 600, 0x17 }, { 650, 0x08 }, { 700, 0x18 }, { 750, 0x09 },
+	{ 800, 0x19 }, { 850, 0x29 }, { 900, 0x39 }, { 950, 0x0A },
+	{ 1000, 0x1A }, { 1050, 0x2A }, { 1100, 0x3A }, { 1150, 0x0B },
+	{ 1200, 0x1B }, { 1250, 0x2B }, { 1300, 0x3B }, { 1350, 0x0C },
+	{ 1400, 0x1C }, { 1450, 0x2C }, { 1500, 0x3C }, { 1550, 0x0D },
+	{ 1600, 0x1D }, { 1650, 0x2D }, { 1700, 0x0E }, { 1750, 0x1E },
+	{ 1800, 0x2E }, { 1850, 0x3E }, { 1900, 0x0F }, { 1950, 0x1F },
+	{ 2000, 0x2F },
+};
+
+struct range_dphy_gen3 {
+	u32 freq;
+	u8 hsfregrange;
+	u32 osc_freq_target;
+};
+
+struct range_dphy_gen3 range_gen3[] = {
+	{ 80, 0x00, 0x1B6 }, { 90, 0x10, 0x1B6 }, { 100, 0x20, 0x1B6 },
+	{ 110, 0x30, 0x1B6 }, { 120, 0x01, 0x1B6 }, { 130, 0x11, 0x1B6 },
+	{ 140, 0x21, 0x1B6 }, { 150, 0x31, 0x1B6 }, { 160, 0x02, 0x1B6 },
+	{ 170, 0x12, 0x1B6 }, { 180, 0x22, 0x1B6 }, { 190, 0x32, 0x1B6 },
+	{ 205, 0x03, 0x1B6 }, { 220, 0x13, 0x1B6 }, { 235, 0x23, 0x1B6 },
+	{ 250, 0x33, 0x1B6 }, { 275, 0x04, 0x1B6 }, { 300, 0x14, 0x1B6 },
+	{ 325, 0x25, 0x1B6 }, { 350, 0x35, 0x1B6 }, { 400, 0x05, 0x1B6 },
+	{ 450, 0x16, 0x1B6 }, { 500, 0x26, 0x1B6 }, { 550, 0x37, 0x1B6 },
+	{ 600, 0x07, 0x1B6 }, { 650, 0x18, 0x1B6 }, { 700, 0x28, 0x1B6 },
+	{ 750, 0x39, 0x1B6 }, { 800, 0x09, 0x1B6 }, { 850, 0x19, 0x1B6 },
+	{ 900, 0x29, 0x1B6 }, { 950, 0x3A, 0x1B6 }, { 1000, 0x0A, 0x1B6 },
+	{ 1050, 0x1A, 0x1B6 }, { 1100, 0x2A, 0x1B6 }, { 1150, 0x3B, 0x1B6 },
+	{ 1200, 0x0B, 0x1B6 }, { 1250, 0x1B, 0x1B6 }, { 1300, 0x2B, 0x1B6 },
+	{ 1350, 0x3C, 0x1B6 }, { 1400, 0x0C, 0x1B6 }, { 1450, 0x1C, 0x1B6 },
+	{ 1500, 0x2C, 0x1B6 }, { 1550, 0x3D, 0x10F }, { 1600, 0x0D, 0x118 },
+	{ 1650, 0x1D, 0x121 }, { 1700, 0x2E, 0x12A }, { 1750, 0x3E, 0x132 },
+	{ 1800, 0x0E, 0x13B }, { 1850, 0x1E, 0x144 }, { 1900, 0x2F, 0x14D },
+	{ 1950, 0x3F, 0x155 }, { 2000, 0x0F, 0x15E }, { 2050, 0x40, 0x167 },
+	{ 2100, 0x41, 0x170 }, { 2150, 0x42, 0x178 }, { 2200, 0x43, 0x181 },
+	{ 2250, 0x44, 0x18A }, { 2300, 0x45, 0x193 }, { 2350, 0x46, 0x19B },
+	{ 2400, 0x47, 0x1A4 }, { 2450, 0x48, 0x1AD }, { 2500, 0x49, 0x1B6 }
+};
+
+u8 dw_dphy_setup_config(struct dw_dphy_rx *dphy)
+{
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	int ret;
+
+	if (dphy->max_lanes == CTRL_4_LANES) {
+		dev_vdbg(&dphy->phy->dev, "CONFIG 4L\n");
+		return CTRL_4_LANES;
+	}
+	if (IS_ENABLED(CONFIG_OF)) {
+		ret = gpio_request(dphy->config_8l, "config");
+		if (ret < 0) {
+			dev_vdbg(&dphy->phy->dev,
+				 "could not acquire config (err=%d)\n", ret);
+			return ret;
+		}
+		ret = gpio_get_value(dphy->config_8l);
+		gpio_free(dphy->config_8l);
+	} else {
+		ret = dphy->config_8l;
+	}
+
+	dev_vdbg(&dphy->phy->dev,
+		 "Booting in [%s] mode\n",
+		 ret == CTRL_8_LANES ? "8L" : "4+4L");
+	return ret;
+
+#endif /* CONFIG_DWC_MIPI_TC_DPHY_GEN3 */
+	return CTRL_4_LANES;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+void dw_dphy_if_write(struct dw_dphy_rx *dphy, u32 address, u32 data)
+{
+	writel(data, dphy->dphy1_if_addr + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		return;
+
+	iowrite32(data, dphy->dphy2_if_addr + address);
+}
+
+u32 dw_dphy_if_read(struct dw_dphy_rx *dphy, u32 address)
+{
+	u32 if1 = 0, if2 = 0;
+
+	if1 = readl(dphy->dphy1_if_addr + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		goto end;
+
+	if (dphy->lanes_config == DPHYID)
+		goto end;
+
+	if2 = readl(dphy->dphy2_if_addr + address);
+
+	if (if1 != if2)
+		dev_vdbg(&dphy->phy->dev,
+			 "Values read different for each interface\n");
+end:
+	return if1;
+}
+#endif
+
+void dw_dphy_write(struct dw_dphy_rx *dphy, u32 address, u32 data)
+{
+	iowrite32(data, dphy->base_address + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		return;
+
+	if (address == R_CSI2_DPHY_TST_CTRL0)
+		iowrite32(data, dphy->base_address + R_CSI2_DPHY2_TST_CTRL0);
+	else if (address == R_CSI2_DPHY_TST_CTRL1)
+		iowrite32(data, dphy->base_address + R_CSI2_DPHY2_TST_CTRL1);
+}
+
+u32 dw_dphy_read(struct dw_dphy_rx *dphy, u32 address)
+{
+	int dphy1 = 0, dphy2 = 0;
+
+	dphy1 = ioread32(dphy->base_address + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		goto end;
+
+	if (address == R_CSI2_DPHY_TST_CTRL0)
+		dphy2 = ioread32(dphy->base_address + R_CSI2_DPHY2_TST_CTRL0);
+	else if (address == R_CSI2_DPHY_TST_CTRL1)
+		dphy2 = ioread32(dphy->base_address + R_CSI2_DPHY2_TST_CTRL1);
+	else
+		return -ENODEV;
+end:
+	return dphy1;
+}
+
+void dw_dphy_write_msk(struct dw_dphy_rx *dev, u32 address, u32 data, u8 shift,
+		       u8 width)
+{
+	u32 temp = dw_dphy_read(dev, address);
+	u32 mask = (1 << width) - 1;
+
+	temp &= ~(mask << shift);
+	temp |= (data & mask) << shift;
+	dw_dphy_write(dev, address, temp);
+}
+
+static void dw_dphy_te_12b_write(struct dw_dphy_rx *dphy, u16 addr, u8 data)
+{
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0x00, PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)(addr >> 8),
+			  PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)addr, PHY_TESTDIN,
+			  8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)data, PHY_TESTDIN,
+			  8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+}
+
+static void dw_dphy_te_8b_write(struct dw_dphy_rx *dphy, u8 addr, u8 data)
+{
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_TST_CTRL1, addr);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_TST_CTRL1, data);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+}
+
+static void dw_dphy_te_write(struct dw_dphy_rx *dphy, u16 addr, u8 data)
+{
+	if (dphy->dphy_te_len == BIT12)
+		dw_dphy_te_12b_write(dphy, addr, data);
+	else
+		dw_dphy_te_8b_write(dphy, addr, data);
+}
+
+static int dw_dphy_te_12b_read(struct dw_dphy_rx *dphy, u32 addr)
+{
+	u8 ret;
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0x00, PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)(addr >> 8),
+			  PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)addr, PHY_TESTDIN,
+			  8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0x00, 0, PHY_TESTDIN);
+	ret = dw_dphy_read_msk(dphy, R_CSI2_DPHY_TST_CTRL1, PHY_TESTDOUT, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 1);
+
+	return ret;
+}
+
+static int dw_dphy_te_8b_read(struct dw_dphy_rx *dphy, u32 addr)
+{
+	u8 ret;
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, addr, PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTDIN, 8);
+	ret = dw_dphy_read_msk(dphy, R_CSI2_DPHY_TST_CTRL1, PHY_TESTDOUT, 8);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 1);
+
+	return ret;
+}
+
+int dw_dphy_te_read(struct dw_dphy_rx *dphy, u32 addr)
+{
+	int ret;
+
+	if (dphy->dphy_te_len == BIT12)
+		ret = dw_dphy_te_12b_read(dphy, addr);
+	else
+		ret = dw_dphy_te_8b_read(dphy, addr);
+
+	return ret;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static void dw_dphy_if_init(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, TX_PHY);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 0);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 1);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, GLUELOGIC);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 0);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 1);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RX_PHY);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 0);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 1);
+}
+#endif
+
+static void dw_dphy_gen3_12bit_tc_power_up(struct dw_dphy_rx *dphy)
+{
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, GLUELOGIC);
+#endif
+	dw_dphy_te_write(dphy, CFGCLKFREQRANGE_TX, 0x1C);
+
+	/* CLKSEL | UPDATEPLL | SHADOW_CLEAR | SHADOW_CTRL | FORCEPLL */
+	dw_dphy_te_write(dphy, BYPASS, 0x3F);
+
+	/* IO_DS3 | IO_DS2 | IO_DS1 | IO_DS0 */
+	if (dphy->dphy_freq > 1500)
+		dw_dphy_te_write(dphy, IO_DS, 0x0F);
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RX_PHY);
+#endif
+}
+
+static void dw_dphy_gen3_8bit_tc_power_up(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq / 1000;
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, GLUELOGIC);
+	dw_dphy_te_write(dphy, CFGCLKFREQRANGE_RX, 0x1C);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RX_PHY);
+#endif
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX0_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX0_LSB, 0x02);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX1_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX1_LSB, 0x02);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX2_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX2_LSB, 0x02);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX3_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX3_LSB, 0x02);
+	dw_dphy_te_write(dphy, BANDGAP_CTRL, 0x80);
+
+	if (input_freq < 2000)
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE0, 0xC0);
+
+	if (input_freq < 1000) {
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE1, 0xC0);
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE2, 0xC0);
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE3, 0xC0);
+	}
+}
+
+int dw_dphy_g118_settle(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq, total_settle, settle_time, byte_clk, lp_time;
+
+	lp_time = dphy->lp_time;
+	input_freq = dphy->dphy_freq / 1000;
+
+	settle_time = (8 * (1000000 / (input_freq))) + 115000;
+	byte_clk = (8000000 / (input_freq));
+	total_settle = (settle_time + lp_time * 1000) / byte_clk;
+
+	if (total_settle > 0xFF)
+		total_settle = 0xFF;
+
+	return total_settle;
+}
+
+static void dw_dphy_pwr_down(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	if (dphy->lanes_config == CTRL_8_LANES)
+		dw_dphy_write_msk(dphy, R_CSI2_DPHY2_TST_CTRL0, 0, PHY_TESTCLK,
+				  1);
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+}
+
+static void dw_dphy_pwr_up(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	if (dphy->lanes_config == CTRL_8_LANES)
+		dw_dphy_write_msk(dphy, R_CSI2_DPHY2_TST_CTRL0, 1, PHY_TESTCLK,
+				  1);
+	dev_vdbg(&dphy->phy->dev, "DPHY power up.\n");
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+}
+
+static int dw_dphy_gen3_12bit_configure(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq;
+	u8 range = 0;
+
+	dev_vdbg(&dphy->phy->dev, "12bit: PHY GEN 3: Freq: %u\n", input_freq);
+	for (range = 0; (range < ARRAY_SIZE(range_gen3) - 1) &&
+	     ((input_freq / 1000) > range_gen3[range].freq);
+	     range++)
+		;
+
+	dw_dphy_gen3_12bit_tc_power_up(dphy);
+	dw_dphy_te_write(dphy, RX_SYS_1, range_gen3[range].hsfregrange);
+	dw_dphy_te_write(dphy, RX_SYS_0, 0x20);
+	dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_2,
+			 (u8)range_gen3[range].osc_freq_target);
+	dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_3,
+			 (u8)(range_gen3[range].osc_freq_target >> 8));
+	dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_4, 0x01);
+
+	if (dphy->phy_type) {
+		dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_1, 0x01);
+		dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_0, 0x80);
+	}
+
+	if (dphy->phy_type || input_freq <= 1500)
+		dw_dphy_te_write(dphy, RX_SYS_7, 0x38);
+
+	return 0;
+}
+
+static int dw_dphy_gen3_8bit_configure(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq;
+	u8 data;
+	u8 range = 0;
+
+	dev_vdbg(&dphy->phy->dev, "8bit: PHY GEN 3: Freq: %u\n", input_freq);
+	for (range = 0; (range < ARRAY_SIZE(range_gen3) - 1) &&
+	     ((input_freq / 1000) > range_gen3[range].freq);
+	     range++)
+		;
+
+	dw_dphy_te_write(dphy, RX_SKEW_CAL, dw_dphy_g118_settle(dphy));
+	data = 1 << 7 | range_gen3[range].hsfregrange;
+	dw_dphy_te_write(dphy, HSFREQRANGE_8BIT, data);
+	dw_dphy_gen3_8bit_tc_power_up(dphy);
+
+	return 0;
+}
+
+static int dw_dphy_gen2_configure(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq;
+	u8 data;
+	u8 range = 0;
+
+	/* provide an initial active-high test clear pulse in TESTCLR  */
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+
+	dev_vdbg(&dphy->phy->dev, "PHY GEN 2: Freq: %u\n", input_freq);
+	for (range = 0; (range < ARRAY_SIZE(range_gen2) - 1) &&
+	     ((input_freq / 1000) > range_gen2[range].freq); range++)
+		;
+
+	data = range_gen2[range].hsfregrange << 1;
+	dw_dphy_te_write(dphy, HSFREQRANGE_8BIT, data);
+
+	return 0;
+}
+
+static int dw_dphy_configure(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_pwr_down(dphy);
+
+	if (dphy->dphy_gen == GEN3) {
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+		dw_dphy_if_init(dphy);
+#endif
+		if (dphy->dphy_te_len == BIT12)
+			dw_dphy_gen3_12bit_configure(dphy);
+		else
+			dw_dphy_gen3_8bit_configure(dphy);
+	} else {
+		dw_dphy_gen2_configure(dphy);
+	}
+	dw_dphy_pwr_up(dphy);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+int dw_dphy_if_set_idelay(struct dw_dphy_rx *dphy, u8 dly, u8 cells)
+{
+	u32 val = 0;
+
+	dw_dphy_if_write(dphy, IDLYCFG, 0);
+	dw_dphy_if_write(dphy, IDLYSEL, cells);
+	dw_dphy_if_write(dphy, IDLYCNTINVAL, dly);
+
+	/* Pulse Value Set */
+	dw_dphy_if_write(dphy, IDLYCFG, 1);
+	usleep_range(10, 20);
+	dw_dphy_if_write(dphy, IDLYCFG, 0);
+
+	/* Pulse IDELAY CTRL Reset */
+	dw_dphy_if_write(dphy, DPHY1REGRSTN, 0);
+	usleep_range(10, 20);
+	dw_dphy_if_write(dphy, DPHY1REGRSTN, 1);
+
+	/* Get Value*/
+	val = dw_dphy_if_read(dphy, IDLYCNTOUTVAL);
+
+	if (val != dly) {
+		dev_vdbg(&dphy->phy->dev,
+			 "odelay config failed, set %d get %d", dly, val);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int dw_dphy_if_get_idelay(struct dw_dphy_rx *dphy)
+{
+	return dw_dphy_if_read(dphy, IDLYCNTOUTVAL);
+}
+
+int dw_dphy_if_set_idelay_lane(struct dw_dphy_rx *dphy, u8 dly, u8 lane)
+{
+	int cell;
+
+	switch (lane) {
+	case 0:
+		for (cell = 3; cell <= 10; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 1:
+		for (cell = 14; cell <= 21; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 2:
+		for (cell = 24; cell <= 31; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 3:
+		for (cell = 34; cell <= 41; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 4: /* ALL */
+		dw_dphy_if_set_idelay(dphy, dly, 0x7F);
+		break;
+	default:
+		dev_err(&dphy->phy->dev, "Lane Value not recognized\n");
+		return -1;
+	}
+	return 0;
+}
+#endif
+
+int dw_dphy_init(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	dev_warn(&dphy->phy->dev, "Init DPHY.\n");
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+
+	return 0;
+}
+
+static int dw_dphy_set_phy_state(struct dw_dphy_rx *dphy, u32 on)
+{
+	u8 hs_freq;
+
+	dphy->lanes_config = dw_dphy_setup_config(dphy);
+
+	if (dphy->dphy_te_len == BIT12)
+		hs_freq = RX_SYS_1;
+	else
+		hs_freq = HSFREQRANGE_8BIT;
+
+	if (on) {
+		dw_dphy_configure(dphy);
+		dev_info(&dphy->phy->dev,
+			 "HS Code: 0X%x\n", dw_dphy_te_read(dphy, hs_freq));
+	} else {
+		dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+		dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	}
+
+	return 0;
+}
+
+int dw_dphy_power_on(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	dev_info(&dphy->phy->dev, "DPHY Power ON\n");
+
+	return dw_dphy_set_phy_state(dphy, 1);
+}
+
+int dw_dphy_power_off(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	return dw_dphy_set_phy_state(dphy, 0);
+}
+
+int dw_dphy_reset(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	usleep_range(100, 200);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-rx.h b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-rx.h
new file mode 100644
index 000000000000..2e8d68c914c1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-rx.h
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#ifndef __PHY_SNPS_DPHY_RX_H__
+#define __PHY_SNPS_DPHY_RX_H__
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_gpio.h>
+#include <linux/phy/phy.h>
+#include <linux/phy/phy-mipi-dphy.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+/* DPHY interface register bank*/
+
+#define R_CSI2_DPHY_SHUTDOWNZ 0x0
+#define R_CSI2_DPHY_RSTZ 0x4
+#define R_CSI2_DPHY_RX 0x8
+#define	R_CSI2_DPHY_STOPSTATE 0xC
+#define R_CSI2_DPHY_TST_CTRL0 0x10
+#define R_CSI2_DPHY_TST_CTRL1 0x14
+#define R_CSI2_DPHY2_TST_CTRL0 0x18
+#define R_CSI2_DPHY2_TST_CTRL1 0x1C
+
+enum dphy_id_mask {
+	DPHY_ID_LANE_SUPPORT = 0,
+	DPHY_ID_IF = 4,
+	DPHY_ID_GEN = 8,
+};
+
+enum dphy_gen_values {
+	GEN1,
+	GEN2,
+	GEN3,
+};
+
+enum dphy_interface_length {
+	BIT8 = 8,
+	BIT12 = 12,
+};
+
+enum tst_ctrl0 {
+	PHY_TESTCLR,
+	PHY_TESTCLK,
+};
+
+enum tst_ctrl1 {
+	PHY_TESTDIN = 0,
+	PHY_TESTDOUT = 8,
+	PHY_TESTEN = 16,
+};
+
+enum lanes_config_values {
+	CTRL_4_LANES,
+	CTRL_8_LANES,
+};
+
+enum dphy_tc {
+	CFGCLKFREQRANGE_TX = 0x02,
+	CFGCLKFREQRANGE_RX = 0x05,
+	BYPASS = 0x20,
+	IO_DS = 0x30,
+};
+
+enum dphy_8bit_interface_addr {
+	BANDGAP_CTRL = 0x24,
+	HS_RX_CTRL_LANE0 = 0x42,
+	HSFREQRANGE_8BIT = 0x44,
+	OSC_FREQ_TARGET_RX0_LSB	= 0x4e,
+	OSC_FREQ_TARGET_RX0_MSB	= 0x4f,
+	HS_RX_CTRL_LANE1 = 0x52,
+	OSC_FREQ_TARGET_RX1_LSB	= 0x5e,
+	OSC_FREQ_TARGET_RX1_MSB	= 0x5f,
+	RX_SKEW_CAL = 0x7e,
+	HS_RX_CTRL_LANE2 = 0x82,
+	OSC_FREQ_TARGET_RX2_LSB	= 0x8e,
+	OSC_FREQ_TARGET_RX2_MSB	= 0x8f,
+	HS_RX_CTRL_LANE3 = 0x92,
+	OSC_FREQ_TARGET_RX3_LSB	= 0x9e,
+	OSC_FREQ_TARGET_RX3_MSB	= 0x9f,
+};
+
+enum dphy_12bit_interface_addr {
+	RX_SYS_0 = 0x01,
+	RX_SYS_1 = 0x02,
+	RX_SYS_7 = 0x08,
+	RX_RX_STARTUP_OVR_0 = 0xe0,
+	RX_RX_STARTUP_OVR_1 = 0xe1,
+	RX_RX_STARTUP_OVR_2 = 0xe2,
+	RX_RX_STARTUP_OVR_3 = 0xe3,
+	RX_RX_STARTUP_OVR_4 = 0xe4,
+};
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+/* Testchip interface register bank */
+#define IDLYCFG	0x00
+#define IDLYSEL	0x04
+#define IDLYCNTINVAL 0x08
+#define IDLYCNTOUTVAL 0x0c
+#define DPHY1REGRSTN 0x10
+#define DPHYZCALSTAT 0x14
+#define DPHYZCALCTRL 0x18
+#define DPHYLANE0STAT 0x1c
+#define DPHYLANE1STAT 0x20
+#define DPHYLANE2STAT 0x24
+#define DPHYLANE3STAT 0x28
+#define DPHYCLKSTAT 0x2c
+#define DPHYZCLKCTRL 0x30
+#define TCGENPURPOSOUT 0x34
+#define TCGENPURPOSIN 0x38
+#define DPHYGENERICOUT 0x3c
+#define DPHYGENERICIN 0x40
+#define DPHYGLUEIFTESTER 0x44
+#define DPHYID 0x100
+
+#define DPHY_DEFAULT_FREQ 300000
+
+enum glueiftester {
+	RESET = 0x0,
+	TX_PHY = 0x1,
+	RX_PHY = 0x2,
+	GLUELOGIC = 0x4,
+};
+#endif
+
+/**
+ * struct phy		specifies associated phy component
+ * struct cfg		to pass mipi dphy specific configurations
+ * @lanes_config	lanes configuration
+ * @dphy_freq		operating frequency of the d-phy (mbps)
+ * @phy_type		dphy can be of two types, passed here
+ * @dphy_gen		dphy can be of three generations, passed here
+ * @dphy_te_len		bus width
+ * @max_lanes		maximum number of lanes
+ * @lp_time		time in low-power
+ * @base_address	memmory address of dphy test interface
+ * @dphy1_if_addr	gluelogic dphy 1 memmory address of interface
+ * @dphy2_if_addr	gluelogic dphy 2 memmory address of interface
+ * @config_8l		eight lanes configuration
+ */
+
+struct dw_dphy_rx {
+	struct phy *phy;
+	struct phy_configure_opts_mipi_dphy *cfg;
+	u32 lanes_config;
+	u32 dphy_freq;
+	u32 phy_type;
+	u32 dphy_gen;
+	u32 dphy_te_len;
+	u32 max_lanes;
+	u32 lp_time;
+	void __iomem *base_address;
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	void __iomem *dphy1_if_addr;
+	void __iomem *dphy2_if_addr;
+	u8 config_8l;
+	u8 (*get_config_8l)(struct device *dev, struct dw_dphy_rx *dphy);
+#endif
+	u8 (*phy_register)(struct device *dev);
+	void (*phy_unregister)(struct device *dev);
+};
+
+int dw_dphy_init(struct phy *phy);
+int dw_dphy_reset(struct phy *phy);
+int dw_dphy_power_off(struct phy *phy);
+int dw_dphy_power_on(struct phy *phy);
+u8 dw_dphy_setup_config(struct dw_dphy_rx *dphy);
+void dw_dphy_write(struct dw_dphy_rx *dphy, u32 address, u32 data);
+u32 dw_dphy_read(struct dw_dphy_rx *dphy, u32 address);
+int dw_dphy_te_read(struct dw_dphy_rx *dphy, u32 addr);
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+u32 dw_dphy_if_read(struct dw_dphy_rx *dphy, u32 address);
+int dw_dphy_if_get_idelay(struct dw_dphy_rx *dphy);
+int dw_dphy_if_set_idelay_lane(struct dw_dphy_rx *dphy, u8 dly, u8 lane);
+int dw_dphy_create_capabilities_sysfs(struct platform_device *pdev);
+
+static inline
+u32 dw_dphy_if_read_msk(struct dw_dphy_rx *dphy,
+			u32 address, u8 shift, u8 width)
+{
+	return (dw_dphy_if_read(dphy, address) >> shift) & ((1 << width) - 1);
+}
+#endif /*IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)*/
+
+static inline struct phy *dw_dphy_xlate(struct device *dev,
+					struct of_phandle_args *args)
+{
+	struct dw_dphy_rx *dphy = dev_get_drvdata(dev);
+
+	return dphy->phy;
+}
+
+static inline
+u32 dw_dphy_read_msk(struct dw_dphy_rx *dev, u32 address, u8 shift,  u8 width)
+{
+	return (dw_dphy_read(dev, address) >> shift) & ((1 << width) - 1);
+}
+#endif /*__PHY_SNPS_DPHY_RX_H__*/
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-sysfs.c b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-sysfs.c
new file mode 100644
index 000000000000..d893e29f7adb
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-dphy-sysfs.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver.
+ * SysFS components for the platform driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include "dw-dphy-rx.h"
+
+static ssize_t dphy_reset_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[15];
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	usleep_range(100, 200);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t dphy_freq_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf,
+			       size_t count)
+{
+	int ret;
+	unsigned long freq;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+
+	ret = kstrtoul(buf, 10, &freq);
+	if (ret < 0)
+		return ret;
+
+	if (freq > 2500) {
+		dev_info(dev, "Freq must be under 2500 Mhz\n");
+		return count;
+	}
+	if (freq < 80) {
+		dev_info(dev, "Freq must be over 80 Mhz\n");
+		return count;
+	}
+
+	dev_vdbg(dev, "Data Rate %lu Mbps\n", freq);
+	dphy->dphy_freq = freq*1000;
+
+	return count;
+}
+
+static ssize_t dphy_freq_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[15];
+
+	snprintf(buffer,
+		 sizeof(buffer),
+		 "Freq %d\n", dphy->dphy_freq);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t dphy_addr_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf,
+			       size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long val;
+	u8 addr, payload;
+	int ret;
+
+	ret = kstrtoul(buf, 32, &val);
+	if (ret < 0)
+		return ret;
+
+	payload = (u16)val;
+	addr = (u16)(val >> 16);
+
+	dev_vdbg(dev, "addr 0x%lX\n", val);
+	dev_vdbg(dev, "payload: 0x%X\n", addr);
+	dev_vdbg(dev, "Addr [0x%x] -> 0x%x\n", (unsigned int)addr,
+		 dw_dphy_te_read(dphy, addr));
+
+	return count;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static ssize_t idelay_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[15];
+
+	snprintf(buffer,
+		 sizeof(buffer), "idelay %d\n", dw_dphy_if_get_idelay(dphy));
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t idelay_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long val;
+	u8 lane, delay;
+	int ret;
+
+	ret = kstrtoul(buf, 16, &val);
+	if (ret < 0)
+		return ret;
+
+	lane = (u8)val;
+	delay = (u8)(val >> 8);
+
+	dev_vdbg(dev, "Lanes %u\n", lane);
+	dev_vdbg(dev, "Delay %u\n", delay);
+
+	dw_dphy_if_set_idelay_lane(dphy, delay, lane);
+
+	return count;
+}
+#endif
+
+static ssize_t len_config_store(struct device *dev,
+				struct device_attribute *attr,
+				const char *buf,
+				size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long length;
+	int ret;
+
+	ret = kstrtoul(buf, 10, &length);
+	if (ret < 0)
+		return ret;
+
+	if (length == BIT8)
+		dev_vdbg(dev, "Configured for 8-bit interface\n");
+	else if (length == BIT12)
+		dev_vdbg(dev, "Configured for 12-bit interface\n");
+	else
+		return count;
+
+	dphy->dphy_te_len = length;
+
+	return count;
+}
+
+static ssize_t len_config_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[20];
+
+	snprintf(buffer, sizeof(buffer), "Length %d\n", dphy->dphy_te_len);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t dw_dphy_g118_settle_store(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long lp_time;
+	int ret;
+
+	ret = kstrtoul(buf, 10, &lp_time);
+	if (ret < 0)
+		return ret;
+
+	if (lp_time > 1 && lp_time < 10000) {
+		dphy->lp_time = lp_time;
+	} else {
+		dev_vdbg(dev, "Invalid Value configuring for 1000 ns\n");
+		dphy->lp_time = 1000;
+	}
+
+	dphy->lp_time = lp_time;
+
+	return count;
+}
+
+static ssize_t dw_dphy_g118_settle_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[10];
+
+	snprintf(buffer, sizeof(buffer), "Settle %d ns\n", dphy->lp_time);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static DEVICE_ATTR_RO(dphy_reset);
+static DEVICE_ATTR_RW(dphy_freq);
+static DEVICE_ATTR_WO(dphy_addr);
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static DEVICE_ATTR_RW(idelay);
+#endif
+static DEVICE_ATTR_RW(len_config);
+static DEVICE_ATTR_RW(dw_dphy_g118_settle);
+
+int dw_dphy_create_capabilities_sysfs(struct platform_device *pdev)
+{
+	device_create_file(&pdev->dev, &dev_attr_dphy_reset);
+	device_create_file(&pdev->dev, &dev_attr_dphy_freq);
+	device_create_file(&pdev->dev, &dev_attr_dphy_addr);
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	device_create_file(&pdev->dev, &dev_attr_idelay);
+#endif
+	device_create_file(&pdev->dev, &dev_attr_len_config);
+	device_create_file(&pdev->dev, &dev_attr_dw_dphy_g118_settle);
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-mipi-csi.c b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-mipi-csi.c
new file mode 100644
index 000000000000..50274446c816
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-mipi-csi.c
@@ -0,0 +1,691 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver
+ * Core MIPI CSI-2 functions
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include "dw-mipi-csi.h"
+
+#define DEBUG
+#define VERBOSE_DEBUG
+
+static struct R_CSI2 reg = {
+	.VERSION = 0x00,
+	.N_LANES = 0x04,
+	.CTRL_RESETN = 0x08,
+	.INTERRUPT = 0x0C,
+	.DATA_IDS_1 = 0x10,
+	.DATA_IDS_2 = 0x14,
+	.PHY_CFG = 0x18,
+	.PHY_MODE = 0x1c,
+	.IPI_MODE = 0x80,
+	.IPI_VCID = 0x84,
+	.IPI_DATA_TYPE = 0x88,
+	.IPI_MEM_FLUSH = 0x8C,
+	.IPI_HSA_TIME = 0x90,
+	.IPI_HBP_TIME = 0x94,
+	.IPI_HSD_TIME = 0x98,
+	.IPI_HLINE_TIME = 0x9C,
+	.IPI_SOFTRSTN = 0xA0,
+	.IPI_ADV_FEATURES = 0xAC,
+	.IPI_VSA_LINES = 0xB0,
+	.IPI_VBP_LINES = 0xB4,
+	.IPI_VFP_LINES = 0xB8,
+	.IPI_VACTIVE_LINES = 0xBC,
+
+	.IPI2_MODE = 0x200,
+	.IPI2_VCID = 0x204,
+	.IPI2_DATA_TYPE = 0x208,
+	.IPI2_MEM_FLUSH = 0x20C,
+	.IPI2_HSA_TIME = 0x210,
+	.IPI2_HBP_TIME = 0x214,
+	.IPI2_HSD_TIME = 0x218,
+	.IPI2_ADV_FEATURES = 0x21C,
+	.MASK_INT_IPI2 = 0x154,
+	.MASK_INT_AP_IPI2 = 0x1a4,
+
+	.IPI3_MODE = 0x220,
+	.IPI3_VCID = 0x224,
+	.IPI3_DATA_TYPE = 0x228,
+	.IPI3_MEM_FLUSH = 0x22C,
+	.IPI3_HSA_TIME = 0x230,
+	.IPI3_HBP_TIME = 0x234,
+	.IPI3_HSD_TIME = 0x238,
+	.IPI3_ADV_FEATURES = 0x23C,
+	.MASK_INT_IPI3 = 0x164,
+	.MASK_INT_AP_IPI3 = 0x1b4,
+
+	.INT_PHY_FATAL = 0xe0,
+	.MASK_INT_PHY_FATAL = 0xe4,
+	.FORCE_INT_PHY_FATAL = 0xe8,
+	.INT_PKT_FATAL = 0xf0,
+	.MASK_INT_PKT_FATAL = 0xf4,
+	.FORCE_INT_PKT_FATAL = 0xf8,
+	.INT_PHY = 0x110,
+	.MASK_INT_PHY = 0x114,
+	.FORCE_INT_PHY = 0x118,
+	.INT_LINE = 0x130,
+	.MASK_INT_LINE = 0x134,
+	.FORCE_INT_LINE = 0x138,
+	.INT_IPI = 0x140,
+	.MASK_INT_IPI = 0x144,
+	.FORCE_INT_IPI = 0x148,
+};
+
+struct interrupt_type csi_int = {
+	.PHY_FATAL = BIT(0),
+	.PKT_FATAL = BIT(1),
+	.PHY = BIT(16),
+};
+
+#define dw_print(VAR) \
+	dev_info(csi_dev->dev, "%s: 0x%x: %X\n", "##VAR##",\
+	VAR, dw_mipi_csi_read(csi_dev, VAR))
+
+void dw_mipi_csi_write_part(struct dw_csi *dev, u32 address, u32 data,
+			    u8 shift, u8 width)
+{
+	u32 mask = (1 << width) - 1;
+	u32 temp = dw_mipi_csi_read(dev, address);
+
+	temp &= ~(mask << shift);
+	temp |= (data & mask) << shift;
+	dw_mipi_csi_write(dev, address, temp);
+}
+
+void dw_mipi_csi_reset(struct dw_csi *csi_dev)
+{
+	dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 0);
+	usleep_range(100, 200);
+	dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 1);
+}
+
+int dw_mipi_csi_mask_irq_power_off(struct dw_csi *csi_dev)
+{
+	if (csi_dev->hw_version_major == 1) {
+		/* set only one lane (lane 0) as active (ON) */
+		dw_mipi_csi_write(csi_dev, reg.N_LANES, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY_FATAL, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PKT_FATAL, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_LINE, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI, 0);
+		if(csi_dev->hw.ipi2_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI2, 0);
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI2, 0);
+		}
+		if(csi_dev->hw.ipi3_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI3, 0);
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI3, 0);
+		}
+		/* only for version 1.30 */
+		if (csi_dev->hw_version_minor == 30)
+			dw_mipi_csi_write(csi_dev,
+					  reg.MASK_INT_FRAME_FATAL, 0);
+
+		dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 0);
+
+		/* only for version 1.40 */
+		if (csi_dev->hw_version_minor == 40) {
+			dw_mipi_csi_write(csi_dev, reg.MSK_BNDRY_FRAME_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_SEQ_FRAME_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_CRC_FRAME_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_PLD_CRC_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_DATA_ID, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_ECC_CORRECT, 0);
+		}
+	}
+
+	return 0;
+}
+
+#ifdef OYX_MOD
+int dw_mipi_csi_recfg(struct dw_csi *csi_dev) {
+	dw_mipi_csi_write(csi_dev, reg.PHY_CFG, 1);
+	dw_mipi_csi_write(csi_dev, 0x40, 1);
+	dw_mipi_csi_write(csi_dev, 0x44, 1);
+	dw_mipi_csi_write(csi_dev, reg.IPI_MODE, 0x1010100);
+	dw_mipi_csi_write(csi_dev, reg.IPI_DATA_TYPE, CSI_2_RAW10);
+	dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 1);
+	udelay(200);
+	return 0;
+}
+#endif
+
+int dw_mipi_csi_hw_stdby(struct dw_csi *csi_dev)
+{
+	if (csi_dev->hw_version_major == 1) {
+		/* set only one lane (lane 0) as active (ON) */
+		dw_mipi_csi_reset(csi_dev);
+		dw_mipi_csi_write(csi_dev, reg.N_LANES, 0);
+#ifdef DWC_PHY_USING
+		phy_init(csi_dev->phy);
+#endif
+
+		/* only for version 1.30 */
+		if (csi_dev->hw_version_minor == 30)
+			dw_mipi_csi_write(csi_dev,
+					  reg.MASK_INT_FRAME_FATAL,
+					  GENMASK(31, 0));
+
+		/* common */
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY_FATAL,
+				  GENMASK(8, 0));
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PKT_FATAL,
+				  GENMASK(1, 0));
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY, 0xff00ff);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_LINE, 0xff00ff);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI, GENMASK(6, 0));
+#ifdef OYX_MOD
+		dw_mipi_csi_write(csi_dev, 0x184, 0x3ffffff);
+		dw_mipi_csi_write(csi_dev, 0x194, 0x3f);
+		dw_mipi_csi_write(csi_dev, 0x2a4, 0xffffffff);
+#endif
+		if(csi_dev->hw.ipi2_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI2, GENMASK(5, 0));
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI2, GENMASK(5, 0));
+		}
+		if(csi_dev->hw.ipi3_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI3, GENMASK(5, 0));
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI3, GENMASK(5, 0));
+		}
+
+		/* only for version 1.40 */
+		if (csi_dev->hw_version_minor == 40) {
+			//dw_mipi_csi_write(csi_dev,reg.MSK_BNDRY_FRAME_FATAL,GENMASK(31, 0));
+			//dw_mipi_csi_write(csi_dev,reg.MSK_SEQ_FRAME_FATAL,GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_CRC_FRAME_FATAL,GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_PLD_CRC_FATAL,GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_DATA_ID, GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_ECC_CORRECT, GENMASK(31, 0));
+		}
+	}
+	return 0;
+}
+
+void dw_mipi_csi_set_ipi_fmt(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+	switch (csi_dev->fmt->mbus_code) {
+	case MEDIA_BUS_FMT_RGB666_1X18:
+		csi_dev->hw.ipi_dt = CSI_2_RGB666;
+		break;
+
+	case MEDIA_BUS_FMT_RGB565_2X8_BE:
+	case MEDIA_BUS_FMT_RGB565_2X8_LE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB565;
+		break;
+
+	case MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE:
+	case MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB555;
+		break;
+
+	case MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE:
+	case MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB444;
+		break;
+
+		break;
+	case MEDIA_BUS_FMT_RGB888_2X12_LE:
+	case MEDIA_BUS_FMT_RGB888_2X12_BE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB888;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_BE:
+		csi_dev->hw.ipi_dt = CSI_2_RAW10;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+		csi_dev->hw.ipi_dt = CSI_2_RAW12;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR14_1X14:
+		csi_dev->hw.ipi_dt = CSI_2_RAW14;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR16_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_RAW16;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+		csi_dev->hw.ipi_dt = CSI_2_RAW8;
+		break;
+
+	case MEDIA_BUS_FMT_YVYU8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV422_8;
+		break;
+
+	case MEDIA_BUS_FMT_VYUY8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV422_8;
+		break;
+
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		csi_dev->hw.ipi_dt = CSI_2_YUV422_10;
+		break;
+
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV420_8_LEG;
+		break;
+
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV420_8;
+		break;
+
+	case MEDIA_BUS_FMT_VUY8_1X24:
+		csi_dev->hw.ipi_dt = CSI_2_YUV420_10;
+		break;
+
+	case MEDIA_BUS_FMT_Y8_1X8:
+		csi_dev->hw.ipi_dt = CSI_2_RAW8;
+		break;
+
+	case MEDIA_BUS_FMT_Y10_1X10:
+		csi_dev->hw.ipi_dt = CSI_2_RAW8;
+		break;
+
+	default:
+		break;
+	}
+
+	dev_info(dev, "Selected IPI Data Type 0x%X\n", csi_dev->hw.ipi_dt);
+}
+
+void dw_mipi_csi_fill_timings(struct dw_csi *dev)
+{
+	dev->hw.ipi_vcid = 0;
+	dev->hw.ipi_dt = CSI_2_RAW12;
+	dev->hw.ipi_emb = 1;
+	dev->hw.ipi_color_mode = COLOR16;
+	dev->hw.ipi_auto_flush = 1;
+	dev->hw.ipi_mode = CAMERA_TIMING;
+	dev->hw.ipi_cut_through = CTACTIVE;
+#ifdef OYX_MOD
+    dev->hw.ipi_line_event = 0x130000 | LINE_EVENT_SELECTION(EVSELPROG);
+#else
+    dev->hw.ipi_line_event = LINE_EVENT_SELECTION(EVSELPROG) | EN_VIDEO |  EN_EMBEDDED;
+#endif
+	dev->hw.output = 0;
+#ifdef OYX_MOD
+	dev->hw.frame_det = 0;
+#else
+	dev->hw.frame_det = 1;
+#endif
+
+	dev->hw.hsa = 1;
+	dev->hw.hbp = 1;
+	dev->hw.hsd = 1;
+	dev->hw.htotal = 0;
+
+	dev->hw.vsa = 0;
+	dev->hw.vbp = 0;
+	dev->hw.vfp = 0;
+	dev->hw.vactive = 0;
+}
+
+void dw_mipi_csi_start(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+
+    /* Configure PHY mode  */
+    dw_mipi_csi_write(csi_dev, reg.PHY_MODE, csi_dev->hw.phy_mode);
+    /* Configure number of lanes */
+    dw_mipi_csi_write(csi_dev, reg.N_LANES, csi_dev->hw.num_lanes - 1);
+    /* Configure PPI width */
+    dw_mipi_csi_write(csi_dev, reg.PHY_CFG, csi_dev->hw.ppi_width);
+
+    /* select IPI virtual channal */
+    dw_mipi_csi_write(csi_dev, reg.IPI_VCID, csi_dev->hw.ipi_vcid);
+    /* select IPI data type */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_DATA_TYPE, csi_dev->hw.ipi_dt, 0, 6);
+    /* configure embedded data */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_DATA_TYPE, csi_dev->hw.ipi_emb, 8, 1);
+    /* enable IPI mode */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, 1, 24, 1);
+    /* Configure IPI MODE */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, csi_dev->hw.ipi_mode, 0, 1);
+    /* Configure IPI data interface */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, csi_dev->hw.ipi_color_mode, 8, 1);
+    /* Configure IPI cut through */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, csi_dev->hw.ipi_cut_through, 16, 1);
+    /* Configure IPI MEM flush */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MEM_FLUSH, csi_dev->hw.ipi_auto_flush, 8, 1);
+
+    if (csi_dev->hw.ipi_mode == CAMERA_TIMING) {
+        /* TODO: Configure line event selection */
+        dw_mipi_csi_write(csi_dev, reg.IPI_ADV_FEATURES, csi_dev->hw.ipi_line_event);
+        /* Configure ipi sync event mode */
+        dw_mipi_csi_write_part(csi_dev, reg.IPI_ADV_FEATURES, csi_dev->hw.frame_det, 24, 1);
+    }
+
+    // dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 0, 1);
+
+	dev_vdbg(dev, "*********** config *********\n");
+	dev_vdbg(dev, "IPI enable: %s\n",
+			csi_dev->hw.output ? "YES" : "NO");
+	dev_vdbg(dev, "video mode transmission type: %s timming\n",
+			csi_dev->hw.ipi_mode ? "controller" : "camera");
+	dev_vdbg(dev, "Color Mode: %s\n",
+			csi_dev->hw.ipi_color_mode ? "16 bits" : "48 bits");
+	dev_vdbg(dev, "Cut Through Mode: %s\n",
+			csi_dev->hw.ipi_cut_through ? "enable" : "disable");
+	dev_vdbg(dev, "Virtual Channel: %d\n",
+			csi_dev->hw.ipi_vcid);
+	dev_vdbg(dev, "Auto-flush: %d\n",
+			csi_dev->hw.ipi_auto_flush);
+
+	dw_mipi_csi_write(csi_dev, reg.IPI_SOFTRSTN, 1);
+
+#ifdef DWC_PHY_USING
+		if (csi_dev->hw.ipi_mode == AUTO_TIMING)
+			phy_power_on(csi_dev->phy);
+#endif
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HSA_TIME, csi_dev->hw.hsa);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HBP_TIME, csi_dev->hw.hbp);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HSD_TIME, csi_dev->hw.hsd);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HLINE_TIME, csi_dev->hw.htotal);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VSA_LINES, csi_dev->hw.vsa);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VBP_LINES, csi_dev->hw.vbp);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VFP_LINES, csi_dev->hw.vfp);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VACTIVE_LINES, csi_dev->hw.vactive);
+	if(csi_dev->hw.ipi2_en) {
+
+		/* select IPI virtual channal */
+		dw_mipi_csi_write(csi_dev, reg.IPI2_VCID, csi_dev->hw.ipi2_vcid);
+		/* select IPI data type */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_DATA_TYPE, csi_dev->hw.ipi_dt, 0, 6);
+		/* configure embedded data */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_DATA_TYPE, csi_dev->hw.ipi_emb, 8, 1);
+		/* enable IPI mode */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, 1, 24, 1);
+		/* Configure IPI MODE */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, csi_dev->hw.ipi_mode, 0, 1);
+		/* Configure IPI data interface */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, csi_dev->hw.ipi_color_mode, 8, 1);
+		/* Configure IPI cut through */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, csi_dev->hw.ipi_cut_through, 16, 1);
+		/* Configure IPI MEM flush */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MEM_FLUSH, csi_dev->hw.ipi_auto_flush, 8, 1);
+
+		if (csi_dev->hw.ipi_mode == CAMERA_TIMING) {
+			/* TODO: Configure line event selection */
+			dw_mipi_csi_write(csi_dev, reg.IPI2_ADV_FEATURES, csi_dev->hw.ipi_line_event);
+			/* Configure ipi sync event mode */
+			//dw_mipi_csi_write_part(csi_dev, reg.IPI2_ADV_FEATURES, csi_dev->hw.frame_det, 24, 1);
+		}
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI2_HSA_TIME, csi_dev->hw.hsa);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI2_HBP_TIME, csi_dev->hw.hbp);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI2_HSD_TIME, csi_dev->hw.hsd);
+		dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 4, 1);//bit4 IPI2_soft reset.
+		dev_vdbg(dev, "IPI2 enable: %s\n", csi_dev->hw.ipi2_en ? "YES" : "NO");
+		dev_vdbg(dev, "IPI2 Virtual Channel: %d\n", csi_dev->hw.ipi2_vcid);
+	}
+	if(csi_dev->hw.ipi3_en) {
+
+		/* select IPI virtual channal */
+		dw_mipi_csi_write(csi_dev, reg.IPI3_VCID, csi_dev->hw.ipi3_vcid);
+		/* select IPI data type */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_DATA_TYPE, csi_dev->hw.ipi_dt, 0, 6);
+		/* configure embedded data */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_DATA_TYPE, csi_dev->hw.ipi_emb, 8, 1);
+		/* enable IPI mode */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, 1, 24, 1);
+		/* Configure IPI MODE */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, csi_dev->hw.ipi_mode, 0, 1);
+		/* Configure IPI data interface */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, csi_dev->hw.ipi_color_mode, 8, 1);
+		/* Configure IPI cut through */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, csi_dev->hw.ipi_cut_through, 16, 1);
+		/* Configure IPI MEM flush */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MEM_FLUSH, csi_dev->hw.ipi_auto_flush, 8, 1);
+
+		if (csi_dev->hw.ipi_mode == CAMERA_TIMING) {
+			/* TODO: Configure line event selection */
+			dw_mipi_csi_write(csi_dev, reg.IPI3_ADV_FEATURES, csi_dev->hw.ipi_line_event);
+			/* Configure ipi sync event mode */
+			//dw_mipi_csi_write_part(csi_dev, reg.IPI3_ADV_FEATURES, csi_dev->hw.frame_det, 24, 1);
+		}
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI3_HSA_TIME, csi_dev->hw.hsa);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI3_HBP_TIME, csi_dev->hw.hbp);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI3_HSD_TIME, csi_dev->hw.hsd);
+		dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 8, 1);//bit8 IPI3_soft reset.
+		dev_vdbg(dev, "IPI3 enable: %s\n", csi_dev->hw.ipi3_en ? "YES" : "NO");
+		dev_vdbg(dev, "IPI3 Virtual Channel: %d\n", csi_dev->hw.ipi3_vcid);
+	}
+
+#ifdef DWC_PHY_USING
+	phy_power_on(csi_dev->phy);
+#endif
+
+	dw_mipi_csi_write(csi_dev, reg.IPI_SOFTRSTN, 1);
+    udelay(1000);
+}
+
+int dw_mipi_csi_irq_handler(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+	u32 global_int_status, i_sts;
+	unsigned long flags;
+
+	spin_lock_irqsave(&csi_dev->slock, flags);
+	global_int_status = dw_mipi_csi_read(csi_dev, reg.INTERRUPT);
+
+	if (global_int_status & csi_int.PHY_FATAL) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_PHY_FATAL);
+		dev_err_ratelimited(dev, "int %08X: PHY FATAL: %08X\n",
+				    reg.INT_PHY_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.PKT_FATAL) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_PKT_FATAL);
+		dev_err_ratelimited(dev, "int %08X: PKT FATAL: %08X\n",
+				    reg.INT_PKT_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.FRAME_FATAL &&
+	    csi_dev->hw_version_major == 1 &&
+	    csi_dev->hw_version_minor == 30) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_FRAME_FATAL);
+		dev_err_ratelimited(dev, "int %08X: FRAME FATAL: %08X\n",
+				    reg.INT_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.PHY) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_PHY);
+		dev_err_ratelimited(dev, "int %08X: PHY: %08X\n",
+				    reg.INT_PHY, i_sts);
+	}
+
+	if (global_int_status & csi_int.PKT &&
+	    csi_dev->hw_version_major == 1 &&
+	    csi_dev->hw_version_minor <= 30) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_PKT);
+		dev_err_ratelimited(dev, "int %08X: PKT: %08X\n",
+				    reg.INT_PKT, i_sts);
+	}
+
+	if (global_int_status & csi_int.LINE) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_LINE);
+		dev_err_ratelimited(dev, "int %08X: LINE: %08X\n",
+				    reg.INT_LINE, i_sts);
+	}
+
+	if (global_int_status & csi_int.IPI) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.INT_IPI);
+		dev_err_ratelimited(dev, "int %08X: IPI: %08X\n",
+				    reg.INT_IPI, i_sts);
+	}
+
+	if (global_int_status & csi_int.BNDRY_FRAME_FATAL) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.ST_BNDRY_FRAME_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_BNDRY_FRAME_FATAL: %08X\n",
+				    reg.ST_BNDRY_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.SEQ_FRAME_FATAL) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.ST_SEQ_FRAME_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_SEQ_FRAME_FATAL: %08X\n",
+				    reg.ST_SEQ_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.CRC_FRAME_FATAL) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.ST_CRC_FRAME_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_CRC_FRAME_FATAL: %08X\n",
+				    reg.ST_CRC_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.PLD_CRC_FATAL) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.ST_PLD_CRC_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_PLD_CRC_FATAL: %08X\n",
+				    reg.ST_PLD_CRC_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.DATA_ID) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.ST_DATA_ID);
+		dev_err_ratelimited(dev, "int %08X: ST_DATA_ID: %08X\n",
+				    reg.ST_DATA_ID, i_sts);
+	}
+
+	if (global_int_status & csi_int.ECC_CORRECTED) {
+		i_sts = dw_mipi_csi_read(csi_dev, reg.ST_ECC_CORRECT);
+		dev_err_ratelimited(dev, "int %08X: ST_ECC_CORRECT: %08X\n",
+				    reg.ST_ECC_CORRECT, i_sts);
+	}
+
+	spin_unlock_irqrestore(&csi_dev->slock, flags);
+
+	return 1;
+}
+
+void dw_mipi_csi_get_version(struct dw_csi *csi_dev)
+{
+	// u32 hw_version;
+
+	// hw_version = dw_mipi_csi_read(csi_dev, reg.VERSION);
+	// csi_dev->hw_version_major = (u8)((hw_version >> 24) - '0');
+	// csi_dev->hw_version_minor = (u8)((hw_version >> 16) - '0');
+	// csi_dev->hw_version_minor = csi_dev->hw_version_minor * 10;
+	// csi_dev->hw_version_minor += (u8)((hw_version >> 8) - '0');
+
+	csi_dev->hw_version_major = 1;
+	csi_dev->hw_version_minor = 40;
+}
+
+int dw_mipi_csi_specific_mappings(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+
+	if (csi_dev->hw_version_major == 1) {
+		if (csi_dev->hw_version_minor == 30) {
+			/*
+			 * Hardware registers that were
+			 * exclusive to version < 1.40
+			 */
+			reg.INT_FRAME_FATAL = 0x100;
+			reg.MASK_INT_FRAME_FATAL = 0x104;
+			reg.FORCE_INT_FRAME_FATAL = 0x108;
+			reg.INT_PKT = 0x120;
+			reg.MASK_INT_PKT = 0x124;
+			reg.FORCE_INT_PKT = 0x128;
+
+			/* interrupt source present until this release */
+			csi_int.PKT = BIT(17);
+			csi_int.LINE = BIT(18);
+			csi_int.IPI = BIT(19);
+			csi_int.FRAME_FATAL = BIT(2);
+
+		} else if (csi_dev->hw_version_minor == 40) {
+			/*
+			 * HW registers that were added
+			 * to version 1.40
+			 */
+			reg.ST_BNDRY_FRAME_FATAL = 0x280;
+			reg.MSK_BNDRY_FRAME_FATAL = 0x284;
+			reg.FORCE_BNDRY_FRAME_FATAL = 0x288;
+			reg.ST_SEQ_FRAME_FATAL = 0x290;
+			reg.MSK_SEQ_FRAME_FATAL	= 0x294;
+			reg.FORCE_SEQ_FRAME_FATAL = 0x298;
+			reg.ST_CRC_FRAME_FATAL = 0x2a0;
+			reg.MSK_CRC_FRAME_FATAL	= 0x2a4;
+			reg.FORCE_CRC_FRAME_FATAL = 0x2a8;
+			reg.ST_PLD_CRC_FATAL = 0x2b0;
+			reg.MSK_PLD_CRC_FATAL = 0x2b4;
+			reg.FORCE_PLD_CRC_FATAL = 0x2b8;
+			reg.ST_DATA_ID = 0x2c0;
+			reg.MSK_DATA_ID = 0x2c4;
+			reg.FORCE_DATA_ID = 0x2c8;
+			reg.ST_ECC_CORRECT = 0x2d0;
+			reg.MSK_ECC_CORRECT = 0x2d4;
+			reg.FORCE_ECC_CORRECT = 0x2d8;
+			reg.DATA_IDS_VC_1 = 0x0;
+			reg.DATA_IDS_VC_2 = 0x0;
+			reg.VC_EXTENSION = 0x0;
+
+			/* interrupts map were changed */
+			csi_int.LINE = BIT(17);
+			csi_int.IPI = BIT(18);
+			csi_int.BNDRY_FRAME_FATAL = BIT(2);
+			csi_int.SEQ_FRAME_FATAL	= BIT(3);
+			csi_int.CRC_FRAME_FATAL = BIT(4);
+			csi_int.PLD_CRC_FATAL = BIT(5);
+			csi_int.DATA_ID = BIT(6);
+			csi_int.ECC_CORRECTED = BIT(7);
+
+		} else {
+			dev_info(dev, "Version minor not supported.");
+		}
+	} else {
+		dev_info(dev, "Version major not supported.");
+	}
+	return 0;
+}
+
+void dw_mipi_csi_dump(struct dw_csi *csi_dev)
+{
+	dw_print(reg.VERSION);
+	dw_print(reg.N_LANES);
+	dw_print(reg.CTRL_RESETN);
+	dw_print(reg.INTERRUPT);
+	dw_print(reg.DATA_IDS_1);
+	dw_print(reg.DATA_IDS_2);
+	dw_print(reg.IPI_MODE);
+	dw_print(reg.IPI_VCID);
+	dw_print(reg.IPI_DATA_TYPE);
+	dw_print(reg.IPI_MEM_FLUSH);
+	dw_print(reg.IPI_HSA_TIME);
+	dw_print(reg.IPI_HBP_TIME);
+	dw_print(reg.IPI_HSD_TIME);
+	dw_print(reg.IPI_HLINE_TIME);
+	dw_print(reg.IPI_SOFTRSTN);
+	dw_print(reg.IPI_ADV_FEATURES);
+	dw_print(reg.IPI_VSA_LINES);
+	dw_print(reg.IPI_VBP_LINES);
+	dw_print(reg.IPI_VFP_LINES);
+	dw_print(reg.IPI_VACTIVE_LINES);
+	dw_print(reg.IPI_DATA_TYPE);
+	dw_print(reg.VERSION);
+	dw_print(reg.IPI_ADV_FEATURES);
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/dw-mipi-csi.h b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-mipi-csi.h
new file mode 100644
index 000000000000..698853802f82
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/dw-mipi-csi.h
@@ -0,0 +1,336 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#ifndef _DW_MIPI_CSI_H__
+#define _DW_MIPI_CSI_H__
+
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/phy/phy.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/ratelimit.h>
+#include <linux/reset.h>
+#include <linux/videodev2.h>
+#include <linux/wait.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include "include/dw-mipi-csi-pltfrm.h"
+
+
+#define OYX_MOD
+// #define DWC_PHY_USING
+#define REG_DUMMY_READ
+
+/* Advanced features */
+#define IPI_DT_OVERWRITE BIT(0)
+#define DATA_TYPE_OVERWRITE(dt) (((dt) & GENMASK(5, 0)) << 8)
+#define LINE_EVENT_SELECTION(n) ((n) << 16)
+
+enum line_event {
+	EVSELAUTO = 0,
+	EVSELPROG = 1,
+};
+
+#define EN_VIDEO BIT(17)
+#define EN_LINE_START BIT(18)
+#define EN_NULL BIT(19)
+#define EN_BLANKING BIT(20)
+#define EN_EMBEDDED BIT(21)
+#define IPI_SYNC_EVENT_MODE(n) ((n) << 24)
+
+enum sync_event {
+	SYNCEVFSN = 0,
+	SYNCEVFS = 1,
+};
+
+/* DW MIPI CSI-2 register addresses*/
+
+struct R_CSI2 {
+	u16 VERSION;
+	u16 N_LANES;
+	u16 CTRL_RESETN;
+	u16 INTERRUPT;
+	u16 DATA_IDS_1;
+	u16 DATA_IDS_2;
+	u16 DATA_IDS_VC_1;
+	u16 DATA_IDS_VC_2;
+	u16 PHY_CFG;
+	u16 PHY_MODE;
+	u16 IPI_MODE;
+	u16 IPI_VCID;
+	u16 IPI_DATA_TYPE;
+	u16 IPI_MEM_FLUSH;
+	u16 IPI_HSA_TIME;
+	u16 IPI_HBP_TIME;
+	u16 IPI_HSD_TIME;
+	u16 IPI_HLINE_TIME;
+	u16 IPI_SOFTRSTN;
+	u16 IPI_ADV_FEATURES;
+	u16 IPI_VSA_LINES;
+	u16 IPI_VBP_LINES;
+	u16 IPI_VFP_LINES;
+	u16 IPI_VACTIVE_LINES;
+	u16 IPI2_MODE;
+	u16 IPI2_VCID;
+	u16 IPI2_DATA_TYPE;
+	u16 IPI2_MEM_FLUSH;
+	u16 IPI2_HSA_TIME;
+	u16 IPI2_HBP_TIME;
+	u16 IPI2_HSD_TIME;
+	u16 IPI2_ADV_FEATURES;
+	u16 MASK_INT_IPI2;
+	u16 MASK_INT_AP_IPI2;
+	u16 IPI3_MODE;
+	u16 IPI3_VCID;
+	u16 IPI3_DATA_TYPE;
+	u16 IPI3_MEM_FLUSH;
+	u16 IPI3_HSA_TIME;
+	u16 IPI3_HBP_TIME;
+	u16 IPI3_HSD_TIME;
+	u16 IPI3_ADV_FEATURES;
+	u16 MASK_INT_IPI3;
+	u16 MASK_INT_AP_IPI3;
+	u16 VC_EXTENSION;
+	u16 INT_PHY_FATAL;
+	u16 MASK_INT_PHY_FATAL;
+	u16 FORCE_INT_PHY_FATAL;
+	u16 INT_PKT_FATAL;
+	u16 MASK_INT_PKT_FATAL;
+	u16 FORCE_INT_PKT_FATAL;
+	u16 INT_FRAME_FATAL;
+	u16 MASK_INT_FRAME_FATAL;
+	u16 FORCE_INT_FRAME_FATAL;
+	u16 INT_PHY;
+	u16 MASK_INT_PHY;
+	u16 FORCE_INT_PHY;
+	u16 INT_PKT;
+	u16 MASK_INT_PKT;
+	u16 FORCE_INT_PKT;
+	u16 INT_LINE;
+	u16 MASK_INT_LINE;
+	u16 FORCE_INT_LINE;
+	u16 INT_IPI;
+	u16 MASK_INT_IPI;
+	u16 FORCE_INT_IPI;
+	u16 ST_BNDRY_FRAME_FATAL;
+	u16 MSK_BNDRY_FRAME_FATAL;
+	u16 FORCE_BNDRY_FRAME_FATAL;
+	u16 ST_SEQ_FRAME_FATAL;
+	u16 MSK_SEQ_FRAME_FATAL;
+	u16 FORCE_SEQ_FRAME_FATAL;
+	u16 ST_CRC_FRAME_FATAL;
+	u16 MSK_CRC_FRAME_FATAL;
+	u16 FORCE_CRC_FRAME_FATAL;
+	u16 ST_PLD_CRC_FATAL;
+	u16 MSK_PLD_CRC_FATAL;
+	u16 FORCE_PLD_CRC_FATAL;
+	u16 ST_DATA_ID;
+	u16 MSK_DATA_ID;
+	u16 FORCE_DATA_ID;
+	u16 ST_ECC_CORRECT;
+	u16 MSK_ECC_CORRECT;
+	u16 FORCE_ECC_CORRECT;
+};
+
+/* Interrupt Masks */
+struct interrupt_type {
+	u32 PHY_FATAL;
+	u32 PKT_FATAL;
+	u32 FRAME_FATAL;
+	u32 PHY;
+	u32 PKT;
+	u32 LINE;
+	u32 IPI;
+	u32 BNDRY_FRAME_FATAL;
+	u32 SEQ_FRAME_FATAL;
+	u32 CRC_FRAME_FATAL;
+	u32 PLD_CRC_FATAL;
+	u32 DATA_ID;
+	u32 ECC_CORRECTED;
+};
+
+/* IPI Data Types */
+enum data_type {
+	CSI_2_YUV420_8 = 0x18,
+	CSI_2_YUV420_10 = 0x19,
+	CSI_2_YUV420_8_LEG = 0x1A,
+	CSI_2_YUV420_8_SHIFT = 0x1C,
+	CSI_2_YUV420_10_SHIFT = 0x1D,
+	CSI_2_YUV422_8 = 0x1E,
+	CSI_2_YUV422_10 = 0x1F,
+	CSI_2_RGB444 = 0x20,
+	CSI_2_RGB555 = 0x21,
+	CSI_2_RGB565 = 0x22,
+	CSI_2_RGB666 = 0x23,
+	CSI_2_RGB888 = 0x24,
+	CSI_2_RAW6 = 0x28,
+	CSI_2_RAW7 = 0x29,
+	CSI_2_RAW8 = 0x2A,
+	CSI_2_RAW10 = 0x2B,
+	CSI_2_RAW12 = 0x2C,
+	CSI_2_RAW14 = 0x2D,
+	CSI_2_RAW16 = 0x2E,
+	CSI_2_RAW20 = 0x2F,
+	USER_DEFINED_1 = 0x30,
+	USER_DEFINED_2 = 0x31,
+	USER_DEFINED_3 = 0x32,
+	USER_DEFINED_4 = 0x33,
+	USER_DEFINED_5 = 0x34,
+	USER_DEFINED_6 = 0x35,
+	USER_DEFINED_7 = 0x36,
+	USER_DEFINED_8 = 0x37,
+};
+
+/* DWC MIPI CSI-2 output types */
+enum output {
+	IPI_OUT = 0,
+	IDI_OUT = 1,
+	BOTH_OUT = 2
+};
+
+/* IPI color components */
+enum color_mode {
+	COLOR48 = 0,
+	COLOR16 = 1
+};
+
+/* IPI cut through */
+enum cut_through {
+	CTINACTIVE = 0,
+	CTACTIVE = 1
+};
+
+/* IPI output types */
+enum ipi_output {
+	CAMERA_TIMING = 0,
+	AUTO_TIMING = 1
+};
+
+/* Format template */
+struct mipi_fmt {
+	u32 mbus_code;
+	u8 depth;
+};
+
+/* CSI specific configuration */
+struct csi_data {
+	u32 num_lanes;
+	u32 dphy_freq;
+	u32 phy_mode;
+	u32 ppi_width;
+	u32 pclk;
+	u32 fps;
+	u32 bpp;
+	u32 output;
+
+	u32 ipi_dt;
+	u32 ipi_emb;
+	u32 ipi_mode;
+	u32 ipi_cut_through;
+	u32 ipi_color_mode;
+	u32 ipi_auto_flush;
+	u32 ipi_line_event;
+	u32 ipi_vcid;
+	u32 frame_det;
+
+	u32 ipi2_en;
+	u32 ipi2_vcid;
+	u32 ipi3_en;
+	u32 ipi3_vcid;
+
+	u32 hsa;
+	u32 hbp;
+	u32 hsd;
+	u32 htotal;
+	u32 vsa;
+	u32 vbp;
+	u32 vfp;
+	u32 vactive;
+};
+
+/* Structure to embed device driver information */
+struct dw_csi {
+	struct v4l2_subdev sd;
+	struct device *dev;
+	struct media_pad pads[CSI_PADS_NUM];
+	struct mipi_fmt *fmt;
+	struct v4l2_mbus_framefmt format;
+	void __iomem *base_address;
+	int ctrl_irq_number;
+	struct csi_data hw;
+	struct reset_control *rst;
+#ifdef DWC_PHY_USING
+	struct phy *phy;
+#endif
+	struct dw_csih_pdata *config;
+	struct mutex lock; /* protect resources sharing */
+	spinlock_t slock; /* interrupt handling lock */
+
+	u8 index;
+	u8 hw_version_major;
+	u16 hw_version_minor;
+};
+
+static inline struct dw_csi *sd_to_mipi_csi_dev(struct v4l2_subdev *sdev)
+{
+	return container_of(sdev, struct dw_csi, sd);
+}
+
+#ifdef OYX_MOD
+int dw_mipi_csi_recfg(struct dw_csi *csi_dev);
+#endif
+
+void dw_mipi_csi_reset(struct dw_csi *csi_dev);
+int dw_mipi_csi_mask_irq_power_off(struct dw_csi *csi_dev);
+int dw_mipi_csi_hw_stdby(struct dw_csi *csi_dev);
+void dw_mipi_csi_set_ipi_fmt(struct dw_csi *csi_dev);
+void dw_mipi_csi_start(struct dw_csi *csi_dev);
+int dw_mipi_csi_irq_handler(struct dw_csi *csi_dev);
+void dw_mipi_csi_get_version(struct dw_csi *csi_dev);
+int dw_mipi_csi_specific_mappings(struct dw_csi *csi_dev);
+void dw_mipi_csi_fill_timings(struct dw_csi *dev);
+void dw_mipi_csi_dump(struct dw_csi *csi_dev);
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+int dw_csi_create_capabilities_sysfs(struct platform_device *pdev);
+#endif
+
+static inline void dw_mipi_csi_write(struct dw_csi *dev,
+				     u32 address, u32 data)
+{
+	pr_debug("%s: [%08x]: %08x\n", __func__, address, data);
+	// printk("%s: [%08x]: %08x\n", __func__, dev->base_address + address, data);
+	writel(data, dev->base_address + address);
+}
+
+static inline u32 dw_mipi_csi_read(struct dw_csi *dev, u32 address)
+{
+    u32 ret =0;
+
+#ifdef REG_DUMMY_READ
+	readl(dev->base_address + address);
+#endif
+	ret = readl(dev->base_address + address);
+
+    /*当前的haps平台下axi总线上csi读取时延大，需要多读取几次保证读取到正确值*/
+	ret = readl(dev->base_address + address);
+	ret = readl(dev->base_address + address);
+	ret = readl(dev->base_address + address);
+	ret = readl(dev->base_address + address);
+	pr_debug("%s: [%08x]: %08x\n", __func__, address, ret);
+    return ret;
+}
+
+#endif /*_DW_MIPI_CSI_H__ */
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/eswin-mipi-dma.c b/drivers/media/platform/eswin/isp/vvcam/dwc/eswin-mipi-dma.c
new file mode 100644
index 000000000000..8c2598bbde3f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/eswin-mipi-dma.c
@@ -0,0 +1,298 @@
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+
+#include <media/v4l2-device.h>
+#include <media/v4l2-dev.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-subdev.h>
+#include <linux/dev_printk.h>
+
+struct csi_mipi_dma
+{
+	struct device *dev;
+
+	struct v4l2_device v4l2_dev;
+	struct media_device media_dev;
+
+	struct v4l2_async_notifier notifier;
+
+	struct video_device video;
+	struct media_pad pad;
+
+	struct vb2_queue queue;
+
+	struct mutex lock;
+
+	struct v4l2_subdev sd;
+	struct media_pad pads[8];
+	struct v4l2_subdev *src_sd;
+	struct v4l2_mbus_config_mipi_csi2 bus;
+};
+
+struct csi_dma_buffer
+{
+	struct vb2_v4l2_buffer buf;
+	struct list_head queue;
+	struct csi_mipi_dma *dma;
+};
+
+// static int csi_dma_fwnode_parse(struct device *dev,
+// 				struct v4l2_fwnode_endpoint *vep,
+// 				struct v4l2_async_subdev *asd)
+// {
+// 	switch (vep->bus_type) {
+// 	case V4L2_MBUS_CSI2_DPHY:
+// 		return 0;
+// 	default:
+// 		dev_err(dev, "Unsupported media bus type\n");
+// 		return -ENOTCONN;
+// 	}
+// }
+
+static int csi_dma_graph_notify_complete(struct v4l2_async_notifier *notifier)
+{
+	int ret;
+	struct csi_mipi_dma *dma =
+	    container_of(notifier, struct csi_mipi_dma, notifier);
+
+	ret = v4l2_device_register_subdev_nodes(&dma->v4l2_dev); /*create all subdev node*/
+	if (ret < 0)
+		dev_err(dma->dev, "failed to register subdev nodes\n");
+
+	return media_device_register(&dma->media_dev);
+}
+
+static int csi_dma_graph_notify_bound(struct v4l2_async_notifier *notifier,
+				      struct v4l2_subdev *sd,
+				      struct v4l2_async_connection *asd)
+{
+	printk("%s %d\n", __func__, __LINE__);
+	return 0;
+}
+
+static const struct v4l2_async_notifier_operations csi_dma_graph_notify_ops = {
+    .bound = csi_dma_graph_notify_bound,
+    .complete = csi_dma_graph_notify_complete,
+};
+
+static int csi_dma_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
+			       unsigned int *nplanes, unsigned int sizes[],
+			       struct device *alloc_devs[])
+{
+	return 0;
+}
+
+static void csi_dma_buffer_queue(struct vb2_buffer *vb)
+{
+}
+
+static const struct vb2_ops csi_dma_queue_qops = {
+    .queue_setup = csi_dma_queue_setup,
+    .buf_queue = csi_dma_buffer_queue,
+};
+
+static const struct v4l2_ioctl_ops csi_dma_ioctl_ops = {
+    .vidioc_reqbufs = vb2_ioctl_reqbufs,
+    .vidioc_querybuf = vb2_ioctl_querybuf,
+    .vidioc_qbuf = vb2_ioctl_qbuf,
+    .vidioc_dqbuf = vb2_ioctl_dqbuf,
+    .vidioc_create_bufs = vb2_ioctl_create_bufs,
+    .vidioc_expbuf = vb2_ioctl_expbuf,
+    .vidioc_streamon = vb2_ioctl_streamon,
+    .vidioc_streamoff = vb2_ioctl_streamoff,
+};
+
+static const struct v4l2_file_operations csi_dma_fops = {
+    .owner = THIS_MODULE,
+    .unlocked_ioctl = video_ioctl2,
+    .open = v4l2_fh_open,
+    .release = vb2_fop_release,
+    .poll = vb2_fop_poll,
+    .mmap = vb2_fop_mmap,
+};
+
+static int cio2_parse_firmware(struct csi_mipi_dma *cio2)
+{
+	struct device *dev = cio2->dev;
+	// unsigned int i;
+	int ret;
+
+	struct v4l2_fwnode_endpoint vep = {
+	    .bus_type = V4L2_MBUS_CSI2_DPHY};
+	struct v4l2_async_connection *asd;
+	struct fwnode_handle *ep;
+
+	ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), 0, 0,
+					     FWNODE_GRAPH_ENDPOINT_NEXT);
+	if (!ep)
+	{
+		dev_err(dev, "failed to fwnode_graph_get_endpoint_by_id\n");
+		return -ENOTCONN;
+	}
+
+	ret = v4l2_fwnode_endpoint_parse(ep, &vep);
+	if (ret)
+		dev_err(dev, "failed to v4l2_fwnode_endpoint_parse : %d\n", ret);
+
+	asd = v4l2_async_nf_add_fwnode_remote(&cio2->notifier, ep,
+					      struct v4l2_async_connection);
+	if (IS_ERR(asd))
+	{
+		ret = PTR_ERR(asd);
+	}
+
+	fwnode_handle_put(ep);
+
+	/*
+	 * Proceed even without sensors connected to allow the device to
+	 * suspend.
+	 */
+	cio2->notifier.ops = &csi_dma_graph_notify_ops;
+	ret = v4l2_async_nf_register(&cio2->notifier);
+	if (ret)
+		dev_err(dev, "failed to register async notifier : %d\n", ret);
+
+	return ret;
+}
+
+static int csi_composite_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct csi_mipi_dma *csi_dma;
+	// struct v4l2_async_subdev *asd;
+
+	int ret;
+
+	dev_info(dev, "Probing started\n");
+
+	csi_dma = devm_kzalloc(dev, sizeof(*csi_dma), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	csi_dma->dev = dev;
+	mutex_init(&csi_dma->lock);
+
+	/* media device init */
+	csi_dma->media_dev.dev = dev;
+	strscpy(csi_dma->media_dev.model, "eswin MIPI CSI DMA", sizeof(csi_dma->media_dev.model));
+	csi_dma->media_dev.hw_revision = 0;
+	media_device_init(&csi_dma->media_dev);
+
+	/* v4l2 device register */
+	csi_dma->v4l2_dev.mdev = &csi_dma->media_dev;
+	ret = v4l2_device_register(dev, &csi_dma->v4l2_dev);
+
+	if (ret < 0)
+	{
+		dev_err(dev, "V4L2 device registration failed (%d)\n", ret);
+		media_device_cleanup(&csi_dma->media_dev);
+		return ret;
+	}
+
+	/* video device init */
+	csi_dma->video.fops = &csi_dma_fops;
+	csi_dma->video.v4l2_dev = &csi_dma->v4l2_dev;
+	csi_dma->video.queue = &csi_dma->queue;
+	snprintf(csi_dma->video.name, sizeof(csi_dma->video.name), "eswin mipi csi dma");
+
+	csi_dma->video.vfl_type = VFL_TYPE_VIDEO;
+	csi_dma->video.vfl_dir = VFL_DIR_RX;
+	csi_dma->video.release = video_device_release_empty;
+	csi_dma->video.ioctl_ops = &csi_dma_ioctl_ops;
+	csi_dma->video.lock = &csi_dma->lock;
+	csi_dma->video.device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
+
+	video_set_drvdata(&csi_dma->video, csi_dma);
+
+	/* init vb queue */
+	csi_dma->queue.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	csi_dma->queue.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
+	csi_dma->queue.lock = &csi_dma->lock;
+	csi_dma->queue.drv_priv = csi_dma;
+	csi_dma->queue.buf_struct_size = sizeof(struct csi_dma_buffer);
+	csi_dma->queue.ops = &csi_dma_queue_qops;
+	csi_dma->queue.mem_ops = &vb2_dma_contig_memops;
+	csi_dma->queue.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC |
+					 V4L2_BUF_FLAG_TSTAMP_SRC_EOF;
+	csi_dma->queue.dev = dev;
+
+	ret = vb2_queue_init(&csi_dma->queue);
+	if (ret < 0)
+	{
+		dev_err(dev, "failed to initialize VB2 queue\n");
+		goto clean_out;
+	}
+
+	/* video device register */
+	ret = video_register_device(&csi_dma->video, VFL_TYPE_VIDEO, -1);
+	if (ret < 0)
+	{
+		dev_err(dev, "failed to register video device\n");
+		goto clean_out;
+	}
+
+	// 	/* Register the subdevices notifier. */
+	v4l2_async_nf_init(&csi_dma->notifier, &csi_dma->v4l2_dev);
+
+	ret = cio2_parse_firmware(csi_dma);
+	if (ret < 0)
+	{
+		dev_err(dev, "notifier registration failed\n");
+		goto clean_out;
+	}
+
+	platform_set_drvdata(pdev, csi_dma);
+
+	dev_info(dev, "Probing End\n");
+
+	return 0;
+
+clean_out:
+	v4l2_device_unregister(&csi_dma->v4l2_dev);
+	v4l2_async_nf_cleanup(&csi_dma->notifier);
+	media_device_unregister(&csi_dma->media_dev);
+	media_device_cleanup(&csi_dma->media_dev);
+
+	return ret;
+}
+
+static int csi_composite_remove(struct platform_device *pdev)
+{
+	struct csi_mipi_dma *csi_dma = platform_get_drvdata(pdev);
+
+	video_unregister_device(&csi_dma->video);
+	v4l2_async_nf_unregister(&csi_dma->notifier);
+	v4l2_async_nf_cleanup(&csi_dma->notifier);
+	v4l2_device_unregister(&csi_dma->v4l2_dev);
+	media_device_unregister(&csi_dma->media_dev);
+	media_device_cleanup(&csi_dma->media_dev);
+
+	return 0;
+}
+
+static const struct of_device_id eswin_composite_of_id_table[] = {
+    {.compatible = "eswin,csi-video"},
+    {}};
+MODULE_DEVICE_TABLE(of, eswin_composite_of_id_table);
+
+static struct platform_driver eswin_composite_driver = {
+    .driver = {
+	.name = "eswin-csi-video",
+	.of_match_table = eswin_composite_of_id_table,
+    },
+    .probe = csi_composite_probe,
+    .remove = csi_composite_remove,
+};
+
+module_platform_driver(eswin_composite_driver);
+
+MODULE_AUTHOR("Y o n g Y a n g, <yangyong@eswin.com>");
+MODULE_DESCRIPTION("A MIPI DMA CONTROLLER");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-csi-data.h b/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-csi-data.h
new file mode 100644
index 000000000000..08e7bacbae30
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-csi-data.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 platform data
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include <linux/kernel.h>
+#include "dw-mipi-csi-pltfrm.h"
+
+struct dw_csih_pdata {
+	u8 eotp_enabled;
+	u32 hs_freq;
+	u32 lanes;
+	u32 pclk;
+	u32 fps;
+	u32 bpp;
+	u8 id;
+};
+
+static const struct pdata_names csis[] = {
+	{ .name = "dw-csi.0", },
+	{ .name = "dw-csi.1", },
+};
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-dphy-data.h b/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-dphy-data.h
new file mode 100644
index 000000000000..afd27e64601b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-dphy-data.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY platform data
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include <linux/phy/phy.h>
+#include <linux/kernel.h>
+#include "dw-mipi-csi-pltfrm.h"
+
+struct dw_phy_pdata {
+	u32 dphy_frequency;
+	u8 dphy_te_len;
+	u32 config_8l;
+	u8 dphy_gen;
+	u8 phy_type;
+	u8 id;
+};
+
+static const struct pdata_names phys[] = {
+	{ .name = "phy-dw-dphy.0.0", },
+	{ .name = "phy-dw-dphy.1.1", },
+};
+
+struct dw_dphy_rx;
+
+struct plat_dw_dphy {
+	int (*get_resources)(struct device *dev, struct dw_dphy_rx *dphy);
+};
diff --git a/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-mipi-csi-pltfrm.h b/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-mipi-csi-pltfrm.h
new file mode 100644
index 000000000000..948db4e5397e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/dwc/include/dw-mipi-csi-pltfrm.h
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host media entities
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#ifndef __DW_MIPI_CSI_PLTFRM_INCLUDES_H_
+#define __DW_MIPI_CSI_PLTFRM_INCLUDES_H_
+
+#include <media/media-entity.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-subdev.h>
+
+#define MAX_WIDTH	3280
+#define MAX_HEIGHT	1852
+
+/* The subdevices' group IDs. */
+#define GRP_ID_SENSOR		(10)
+#define GRP_ID_CSI		(20)
+#define GRP_ID_VIF		(30)
+#define GRP_ID_VIDEODEV		(40)
+
+#define CSI_MAX_ENTITIES	(2)
+#define VIF_MAX_ENTITIES	(2)
+#define PLAT_MAX_SENSORS	(2)
+
+struct pdata_names {
+	char *name;
+};
+
+enum video_dev_pads {
+	VIDEO_DEV_SD_PAD_SINK_VIF1,
+	VIDEO_DEV_SD_PAD_SINK_VIF2,
+	VIDEO_DEV_SD_PAD_SOURCE_DMA,
+	VIDEO_DEV_SD_PADS_NUM,
+};
+
+enum vif_pads {
+	VIF_PAD_SINK_CSI,
+	VIF_PAD_SOURCE_DMA,
+	VIF_PADS_NUM,
+};
+
+enum mipi_csi_pads {
+	CSI_PAD_SINK,
+	CSI_PAD_SOURCE,
+	CSI_PADS_NUM,
+};
+
+struct plat_csi_source_info {
+	u16 flags;
+	u16 mux_id;
+};
+
+struct plat_csi_fmt {
+	char *name;
+	u32 mbus_code;
+	u32 fourcc;
+	u8 depth;
+};
+
+struct plat_csi_media_pipeline;
+
+/*
+ * Media pipeline operations to be called from within a video node,  i.e. the
+ * last entity within the pipeline. Implemented by related media device driver.
+ */
+struct plat_csi_media_pipeline_ops {
+	int (*prepare)(struct plat_csi_media_pipeline *p,
+		       struct media_entity *me);
+	int (*unprepare)(struct plat_csi_media_pipeline *p);
+	int (*open)(struct plat_csi_media_pipeline *p, struct media_entity *me,
+		    bool resume);
+	int (*close)(struct plat_csi_media_pipeline *p);
+	int (*set_stream)(struct plat_csi_media_pipeline *p, bool state);
+	int (*set_format)(struct plat_csi_media_pipeline *p,
+			  struct v4l2_subdev_format *fmt);
+};
+
+struct plat_csi_video_entity {
+	struct video_device vdev;
+	struct plat_csi_media_pipeline *pipe;
+};
+
+struct plat_csi_media_pipeline {
+	struct media_pipeline mp;
+	const struct plat_csi_media_pipeline_ops *ops;
+};
+
+static inline struct plat_csi_video_entity
+*vdev_to_plat_csi_video_entity(struct video_device *vdev)
+{
+	return container_of(vdev, struct plat_csi_video_entity, vdev);
+}
+
+#define plat_csi_pipeline_call(ent, op, args...)			  \
+	(!(ent) ? -ENOENT : (((ent)->pipe->ops && (ent)->pipe->ops->op) ? \
+	(ent)->pipe->ops->op(((ent)->pipe), ##args) : -ENOIOCTLCMD))	  \
+
+#endif /* __DW_MIPI_CSI_PLTFRM_INCLUDES_H_ */
diff --git a/drivers/media/platform/eswin/isp/vvcam/i2c/Makefile b/drivers/media/platform/eswin/isp/vvcam/i2c/Makefile
new file mode 100644
index 000000000000..2d9e1b9c19d3
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/i2c/Makefile
@@ -0,0 +1,85 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+ifeq ($(VVCAM_PATH),)
+	VVCAM_PATH:=$(shell cd $(shell pwd)/../; pwd)
+	export VVCAM_PATH
+endif
+
+include $(PWD)/../Param.mk
+TARGET = vvcam_i2c
+
+obj-m +=$(TARGET).o
+$(TARGET)-objs += soc_i2c_platform.o
+$(TARGET)-objs += soc_i2c_driver.o
+$(TARGET)-objs += soc_i2c_procfs.o
+
+PWD :=$(shell pwd)
+
+all:
+	make -C $(KERNEL_DIR) M=$(PWD) modules
+	@[ -e $(INSTALL_DIR)/lib ] || mkdir $(INSTALL_DIR)/lib -p
+	@cp $(TARGET).ko $(INSTALL_DIR)/lib
+modules_install:
+	make -C $(KERNEL_DIR) M=$(PWD) modules_install
+clean:
+	rm -rf $($(TARGET)-objs)
+	make -C $(KERNEL_DIR) M=$(PWD) clean
+	@rm $(INSTALL_DIR)/lib/$(TARGET).ko
+	@rm -rf $(OBJS)
+	@rm -rf modules.order Module.symvers .tmp_versions
+	@find $(AQROOT) -name ".dw_*.cmd" | xargs rm -f
+	@rm -f $(MODULE_NAME).o $(MODULE_NAME).mod.[co] .*.cmd
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_driver.c b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_driver.c
new file mode 100644
index 000000000000..c16a26f1d0e0
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_driver.c
@@ -0,0 +1,399 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <asm/io.h>
+#include <linux/gpio.h>
+#include <linux/delay.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/debugfs.h>
+#include <linux/i2c.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include "soc_i2c_platform.h"
+#include "soc_i2c_procfs.h"
+
+struct soc_gpio_i2c {
+	struct i2c_adapter adap;
+	void __iomem *virt_base_addr;
+	struct proc_dir_entry *pde;
+};
+
+static void i2c_gpio_setsda_val(void *source, int state)
+{
+	struct soc_gpio_i2c *i2c = source;
+	int reg_data;
+
+	reg_data = readl(i2c->virt_base_addr);
+	reg_data = (reg_data & (~0x1)) | (0x10 | (state & 0x01));
+
+	writel(reg_data, i2c->virt_base_addr);
+	return;
+}
+
+static void i2c_gpio_setscl_val(void *source, int state)
+{
+	struct soc_gpio_i2c *i2c = source;
+	int reg_data;
+
+	reg_data = readl(i2c->virt_base_addr);
+	reg_data = (reg_data & (~(0x1 << 8))) | ((0x10 | (state & 0x01)) << 8);
+
+	writel(reg_data, i2c->virt_base_addr);
+	return;
+}
+
+static int i2c_gpio_getsda(void *source)
+{
+	struct soc_gpio_i2c *i2c = source;
+	unsigned int reg_data;
+
+	reg_data = readl(i2c->virt_base_addr);
+	reg_data = (reg_data & (~0x10));
+	writel(reg_data, i2c->virt_base_addr);
+
+	return (readl(i2c->virt_base_addr) & 0x01);
+}
+
+// Start :When SCL is High ,SDA Change to low
+static void i2c_gpio_start(void *source)
+{
+	i2c_gpio_setscl_val(source, 1);
+	i2c_gpio_setsda_val(source, 1);
+	udelay(5);
+	i2c_gpio_setsda_val(source, 0);
+	udelay(1);
+	return;
+}
+
+//Send Bit : When SCL is LOW, sda change;When SCL is HIgh,SDA hold;
+static void i2c_gpio_send_byte(void *source, unsigned char val)
+{
+	int bit_idx;
+	for (bit_idx = 7; bit_idx >= 0; bit_idx--) {
+		i2c_gpio_setscl_val(source, 0);
+		i2c_gpio_setsda_val(source, (val >> bit_idx) & 0x01);
+		udelay(1);
+		i2c_gpio_setscl_val(source, 1);
+		udelay(1);
+	}
+	i2c_gpio_setscl_val(source, 0);
+	i2c_gpio_setsda_val(source, 1);
+	return;
+}
+
+//Read Bit: When SCL is High read data;
+static unsigned char i2c_gpio_read_byte(void *source)
+{
+	int bit_idx;
+	unsigned char data = 0;
+
+	i2c_gpio_getsda(source);	//SDA change to read
+
+	for (bit_idx = 7; bit_idx >= 0; bit_idx--) {
+
+		i2c_gpio_setscl_val(source, 1);
+		udelay(1);
+		data = (data << 1) | i2c_gpio_getsda(source);
+		i2c_gpio_setscl_val(source, 0);
+		udelay(1);
+	}
+	return data;
+}
+
+static int i2c_gpio_wait_ack(void *source)
+{
+	unsigned int i2c_retry_ack_cnt = 0;
+
+	i2c_gpio_getsda(source);	//SDA change to read
+	udelay(1);
+	i2c_gpio_setscl_val(source, 1);
+
+	while (i2c_gpio_getsda(source) == 1) {
+		udelay(1);
+		if (i2c_retry_ack_cnt++ > 20) {
+			return 0;
+		}
+	}
+	i2c_gpio_setscl_val(source, 0);
+	return 1;
+}
+
+static void i2c_gpio_ack(void *source)
+{
+	i2c_gpio_setsda_val(source, 0);
+	i2c_gpio_setscl_val(source, 1);
+	udelay(1);
+	i2c_gpio_setscl_val(source, 0);
+	udelay(1);
+}
+
+static void i2c_gpio_nack(void *source)
+{
+	i2c_gpio_setsda_val(source, 1);
+	i2c_gpio_setscl_val(source, 1);
+	udelay(1);
+	i2c_gpio_setscl_val(source, 0);
+	udelay(1);
+}
+
+static void i2c_gpio_stop(void *source)
+{
+	i2c_gpio_setsda_val(source, 0);
+	udelay(1);
+	i2c_gpio_setscl_val(source, 1);
+	udelay(1);
+	i2c_gpio_setsda_val(source, 1);
+	return;
+}
+
+static int i2c_gpio_write(void *source, struct i2c_msg *msg)
+{
+
+	unsigned char slave_address;
+	unsigned char *buf;
+	int i;
+	slave_address = (msg->addr) << 1;
+	buf = msg->buf;
+
+	i2c_gpio_start(source);
+	i2c_gpio_send_byte(source, slave_address);
+	if (i2c_gpio_wait_ack(source) == 0)
+		return -1;
+
+	for (i = 0; i < msg->len; i++) {
+		i2c_gpio_send_byte(source, buf[i]);
+		i2c_gpio_wait_ack(source);
+	}
+
+	i2c_gpio_stop(source);
+	return 0;
+}
+
+static int i2c_gpio_read(void *source, struct i2c_msg *msg)
+{
+	unsigned char slave_address;
+	unsigned char *buf;
+	int i;
+	slave_address = ((msg->addr) << 1) | 0x01;
+	buf = msg->buf;
+
+	i2c_gpio_start(source);
+	i2c_gpio_send_byte(source, slave_address);
+	if (i2c_gpio_wait_ack(source) == 0)
+		return -1;
+
+	for (i = 0; i < msg->len; i++) {
+		buf[i] = i2c_gpio_read_byte(source);
+		if (i < msg->len - 1) {
+			i2c_gpio_ack(source);
+		} else {
+			i2c_gpio_nack(source);
+		}
+	}
+
+	i2c_gpio_stop(source);
+
+	return 0;
+}
+
+static int i2c_gpio_xfer(struct i2c_adapter *adapter,
+						 struct i2c_msg *msgs, int num)
+{
+	int result = 0;
+	int i;
+	void *source;
+
+	source = i2c_get_adapdata(adapter);
+
+	for (i = 0; i < num; i++) {
+		if (msgs[i].flags & I2C_M_RD) {
+			result = i2c_gpio_read(source, &(msgs[i]));
+		} else {
+			result = i2c_gpio_write(source, &(msgs[i]));
+		}
+
+		if (result < 0)
+			break;
+	}
+
+	return (result < 0) ? result : num;
+}
+
+static u32 i2c_gpio_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C |
+		(I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) |
+		I2C_FUNC_PROTOCOL_MANGLING;
+}
+
+static const struct i2c_algorithm gpio_i2c_algo = {
+	.master_xfer   = i2c_gpio_xfer,
+	.functionality = i2c_gpio_func,
+};
+
+static int soc_gpio_i2c_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct soc_gpio_i2c *i2c;
+	struct i2c_adapter *adap;
+	struct resource *res;
+
+	i2c = devm_kzalloc(&pdev->dev, sizeof(struct soc_gpio_i2c), GFP_KERNEL);
+	if (!i2c) {
+		return -ENOMEM;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "can't fetch device resource info\n");
+		return -EIO;
+	}
+
+	i2c->virt_base_addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(i2c->virt_base_addr)) {
+		dev_err(&pdev->dev, "can't remap device resource info\n");
+		return PTR_ERR(i2c->virt_base_addr);
+	}
+
+	adap = &i2c->adap;
+	platform_set_drvdata(pdev, i2c);
+	i2c_set_adapdata(adap, i2c);
+
+	strlcpy(adap->name, "soc_gpio_i2c", sizeof(adap->name));
+	adap->owner       = THIS_MODULE;
+	adap->algo        = &gpio_i2c_algo;
+	adap->dev.parent  = &pdev->dev;
+	adap->nr          = pdev->id;
+	adap->dev.of_node = pdev->dev.of_node;
+	adap->timeout     = 2 * HZ;
+	adap->retries     = 0;
+	adap->class       = I2C_CLASS_HWMON | I2C_CLASS_SPD;
+
+	ret = i2c_add_numbered_adapter(adap);
+	if (ret) {
+		dev_err(&pdev->dev, "i2c_add_adapter failed\n");
+		return ret;
+	}
+
+	ret = soc_gpio_i2c_procfs_register(adap, &(i2c->pde));
+	if (ret) {
+		dev_err(&pdev->dev, "i2c register procfs failed\n");
+		return ret;
+	}
+
+	dev_info(&pdev->dev, "i2c bus probe success\n");
+
+	return 0;
+}
+
+static int soc_gpio_i2c_remove(struct platform_device *pdev)
+{
+	struct soc_gpio_i2c *i2c;
+
+	i2c = platform_get_drvdata(pdev);
+
+	soc_gpio_i2c_procfs_unregister(&i2c->pde);
+
+	i2c_del_adapter(&(i2c->adap));
+
+	return 0;
+}
+
+static struct platform_driver soc_gpio_i2c_driver = {
+	.probe  = soc_gpio_i2c_probe,
+	.remove = soc_gpio_i2c_remove,
+	.driver = {
+		.name = "soc_gpio_i2c",
+		.owner = THIS_MODULE,
+	},
+};
+
+static int __init soc_gpio_i2c_driver_init_module(void)
+{
+	int ret = 0;
+
+	ret = platform_driver_register(&soc_gpio_i2c_driver);
+	if (ret) {
+		printk(KERN_ERR "Failed to register gpio i2c driver\n");
+		return ret;
+	}
+
+	ret = soc_gpio_i2c_platform_device_register();
+	if (ret) {
+		platform_driver_unregister(&soc_gpio_i2c_driver);
+		printk(KERN_ERR "Failed to register gpio i2c platform devices\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static void __exit soc_gpio_i2c_driver_exit_module(void)
+{
+	platform_driver_unregister(&soc_gpio_i2c_driver);
+	soc_gpio_i2c_platform_device_unregister();
+	return;
+}
+
+late_initcall(soc_gpio_i2c_driver_init_module);
+module_exit(soc_gpio_i2c_driver_exit_module);
+
+MODULE_DESCRIPTION("Verisilicon soc_gpio_i2c driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_platform.c b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_platform.c
new file mode 100644
index 000000000000..4d5073010e0a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_platform.c
@@ -0,0 +1,201 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include <linux/i2c.h>
+
+#include "soc_i2c_platform.h"
+
+static int devices_mask = 0x01;
+MODULE_PARM_DESC(devices_mask, "I2C devices mask");
+
+static void soc_gpio_i2c_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+	return;
+}
+
+static struct resource soc_gpio_i2c0_resource[] = {
+    [0] = {
+        .start = VVCAM_I2C0_BASE + PCIE_BASE,
+        .end = VVCAM_I2C0_BASE + PCIE_BASE + VVCAM_I2C_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device soc_gpio_i2c0_pdev = {
+	.name = "soc_gpio_i2c",
+	.id = VVCAM_I2C_BUS0,
+	.resource = soc_gpio_i2c0_resource,
+	.num_resources = ARRAY_SIZE(soc_gpio_i2c0_resource),
+	.dev.release = soc_gpio_i2c_release,
+};
+
+static struct resource soc_gpio_i2c1_resource[] = {
+    [0] = {
+        .start = VVCAM_I2C1_BASE + PCIE_BASE,
+        .end = VVCAM_I2C1_BASE + PCIE_BASE + VVCAM_I2C_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device soc_gpio_i2c1_pdev = {
+	.name = "soc_gpio_i2c",
+	.id = VVCAM_I2C_BUS1,
+	.resource = soc_gpio_i2c1_resource,
+	.num_resources = ARRAY_SIZE(soc_gpio_i2c1_resource),
+	.dev.release = soc_gpio_i2c_release,
+};
+
+static struct resource soc_gpio_i2c2_resource[] = {
+    [0] = {
+        .start = VVCAM_I2C2_BASE + PCIE_BASE,
+        .end = VVCAM_I2C2_BASE + PCIE_BASE + VVCAM_I2C_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device soc_gpio_i2c2_pdev = {
+	.name = "soc_gpio_i2c",
+	.id = VVCAM_I2C_BUS2,
+	.resource = soc_gpio_i2c2_resource,
+	.num_resources = ARRAY_SIZE(soc_gpio_i2c2_resource),
+	.dev.release = soc_gpio_i2c_release,
+};
+
+static struct resource soc_gpio_i2c3_resource[] = {
+    [0] = {
+        .start = VVCAM_I2C3_BASE + PCIE_BASE,
+        .end = VVCAM_I2C3_BASE + PCIE_BASE + VVCAM_I2C_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device soc_gpio_i2c3_pdev = {
+	.name = "soc_gpio_i2c",
+	.id = VVCAM_I2C_BUS3,
+	.resource = soc_gpio_i2c3_resource,
+	.num_resources = ARRAY_SIZE(soc_gpio_i2c3_resource),
+	.dev.release = soc_gpio_i2c_release,
+};
+
+int soc_gpio_i2c_platform_device_register(void)
+{
+    int ret = 0;
+    int devs_mask = 0;
+    if (devices_mask & 0x01) {
+        ret = platform_device_register(&soc_gpio_i2c0_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x01;
+    }
+
+    if (devices_mask & 0x02) {
+        ret = platform_device_register(&soc_gpio_i2c1_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x02;
+    }
+
+    if (devices_mask & 0x04) {
+        ret = platform_device_register(&soc_gpio_i2c2_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x04;
+    }
+
+    if (devices_mask & 0x08) {
+        ret = platform_device_register(&soc_gpio_i2c3_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x08;
+    }
+
+    return ret;
+
+err_register:
+    if (devs_mask & 0x01)
+		platform_device_unregister(&soc_gpio_i2c0_pdev);
+	if (devs_mask & 0x02)
+		platform_device_unregister(&soc_gpio_i2c1_pdev);
+	if (devs_mask & 0x04)
+		platform_device_unregister(&soc_gpio_i2c2_pdev);
+	if (devs_mask & 0x08)
+		platform_device_unregister(&soc_gpio_i2c3_pdev);
+
+    return ret;
+}
+
+void soc_gpio_i2c_platform_device_unregister(void)
+{
+    if (devices_mask & 0x01)
+		platform_device_unregister(&soc_gpio_i2c0_pdev);
+	if (devices_mask & 0x02)
+		platform_device_unregister(&soc_gpio_i2c1_pdev);
+	if (devices_mask & 0x04)
+		platform_device_unregister(&soc_gpio_i2c2_pdev);
+	if (devices_mask & 0x08)
+		platform_device_unregister(&soc_gpio_i2c3_pdev);
+
+    return;
+}
+
+module_param(devices_mask, int, 0644);
diff --git a/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_platform.h b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_platform.h
new file mode 100644
index 000000000000..32a9a55e4b7f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __SOC_I2C_PLATFORM_H__
+#define __SOC_I2C_PLATFORM_H__
+
+int soc_gpio_i2c_platform_device_register(void);
+void soc_gpio_i2c_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_procfs.c b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_procfs.c
new file mode 100644
index 000000000000..8959295c53d8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_procfs.c
@@ -0,0 +1,546 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/i2c.h>
+#include <linux/ctype.h>
+#include "soc_i2c_procfs.h"
+
+#define GPIO_I2C_PROCFS_BUF_SIZE    128
+
+#define PROC_RW_ERROR           0x01
+#define PROC_PARSE_ERROR        0x02
+
+struct rw_cmd {
+    bool is_write;
+    int8_t flags;
+    uint8_t slave_addr;
+    uint8_t num_elems;
+    uint8_t *rw_buffer;
+    struct list_head list;
+};
+
+struct gpio_i2c_status {
+    int8_t flags;
+    struct list_head cmd_list;
+};
+
+struct gpio_i2c_procfs {
+    struct proc_dir_entry *pde;
+    struct i2c_adapter *adapter;
+    struct mutex lock;
+    struct gpio_i2c_status *status;
+};
+
+static int32_t gpio_i2c_scan_slave(struct i2c_adapter *adapter, uint8_t addr)
+{
+    int ret = 0;
+    char buf = 0;
+    struct i2c_msg msg = {.addr = 0x07, .flags = I2C_M_RD, .buf = &buf, .len=1};
+
+    msg.addr = addr;
+    ret = i2c_transfer(adapter, &msg, 1);
+    if (ret != msg.len)
+        return -1;
+    return 0;
+}
+
+
+static void release_cmd_list(struct list_head *cmd_list)
+{
+    struct rw_cmd *cmd = NULL, *old = NULL;
+
+    if (!list_empty( cmd_list)) {
+        list_for_each_entry(cmd, cmd_list, list) {
+            if (old) {
+                list_del(&old->list);
+                kfree(old);
+                old = NULL;
+            }
+            if (cmd->rw_buffer)
+                kfree(cmd->rw_buffer);
+            old = cmd;
+        }
+
+        if (old) {
+            list_del(&old->list);
+            kfree(old);
+            old = NULL;
+        }
+    }
+}
+
+static int gpio_i2c_procfs_info_show(struct seq_file *sfile, void *offset)
+{
+	struct gpio_i2c_procfs *i2c_proc;
+    struct i2c_adapter *adapter;
+    struct gpio_i2c_status *status;
+    struct rw_cmd *cmd = NULL;
+    uint32_t ret = 0;
+    uint8_t base = 0;
+    uint8_t shift = 0;
+    uint8_t i = 0;
+
+	i2c_proc = (struct gpio_i2c_procfs *) sfile->private;
+
+    mutex_lock(&i2c_proc->lock);
+    adapter = i2c_proc->adapter;
+    status = i2c_proc->status;
+
+    if (status && (status->flags & PROC_PARSE_ERROR)) {
+        seq_printf(sfile, "please check input!\n");
+        seq_printf(sfile, "Usage: echo w2@0x3c 0x30 0x0a r1>/proc/vsi/i2c8\n");
+        status->flags = 0;
+        mutex_unlock(&i2c_proc->lock);
+        return 0;
+    }
+
+    if (status && !list_empty(&status->cmd_list)) {
+        list_for_each_entry(cmd, &status->cmd_list, list) {
+            if (cmd->is_write) {
+                if (cmd->flags & PROC_RW_ERROR) {
+                    seq_printf(sfile, "write slave 0x%02x failed.\n",
+                        cmd->slave_addr);
+                } else {
+                    seq_printf(sfile, "write slave 0x%02x successed.\n",
+                        cmd->slave_addr);
+                }
+            } else {
+                if (cmd->flags & PROC_RW_ERROR) {
+                    seq_printf(sfile, "read slave 0x%02x failed.\n",
+                        cmd->slave_addr);
+                } else {
+                    seq_printf(sfile, "read %d bytes from slave 0x%02x:\n",
+                        cmd->num_elems, cmd->slave_addr);
+                    for (i = 0; i < cmd->num_elems; i++) {
+                        seq_printf(sfile, "value:%02x\n", cmd->rw_buffer[i]);
+                    }
+                }
+            }
+            cmd->flags = 0;
+        }
+        release_cmd_list(&status->cmd_list);
+        status->flags = 0;
+     } else {
+        seq_printf(sfile, "DeviceName:\t%s.%d\n", adapter->name, adapter->nr);
+        seq_printf(sfile, "Device:\t\ti2c-%d\n", adapter->nr);
+        seq_printf(sfile, "Slaves:\n");
+
+        seq_printf(sfile, "     ");
+        for (i = 0; i < 15; i++) {
+            seq_printf(sfile, "%2x  ", i);
+        }
+        seq_printf(sfile, "%2x\n", i);
+
+        for (base = 0x00; base < 0x80; base += 16) {
+            seq_printf(sfile, "%02x:  ", base);
+            for (shift = 0; shift < 16; shift++) {
+                ret = gpio_i2c_scan_slave(adapter, base + shift);
+                if (ret) {
+                    seq_printf(sfile, "--  ");
+                    continue;
+                }
+                seq_printf(sfile, "%02x  ", base + shift);
+            }
+            seq_printf(sfile, "\n");
+        }
+     }
+     mutex_unlock(&i2c_proc->lock);
+
+	return 0;
+}
+
+
+static int gpio_i2c_procfs_open(struct inode *inode, struct file *file)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	return single_open(file, gpio_i2c_procfs_info_show, PDE_DATA(inode));
+#else
+	return single_open(file, gpio_i2c_procfs_info_show, pde_data(inode));
+#endif
+}
+
+static char *gpio_i2c_next_arg(char *str_buf, char **param)
+{
+    int i = 0;
+    for (; str_buf[i]; i++) {
+        if (isspace(str_buf[i]))
+            break;
+    }
+
+    *param = str_buf;
+
+    if (str_buf[i]) {
+        str_buf[i] = '\0';
+        str_buf += i + 1;
+    } else 
+        str_buf += i;
+
+    return skip_spaces(str_buf);
+}
+
+static int gpio_i2c_parse_args(char *str_buf, struct list_head *cmd_list)
+{
+    char *param;
+    unsigned char address = 0x00, read_write = 0;
+    struct rw_cmd *cmd;
+    uint8_t  index = 0, num_elems = 0;
+    char *end;
+
+    while (*str_buf) {
+        param = NULL;
+        str_buf = gpio_i2c_next_arg(str_buf, &param);
+        if ((*param == 'r') || (*param == 'w')) {
+
+            if (index != 0)
+                goto error_i2c_msgs;
+
+            read_write = *param;
+            num_elems = (uint8_t)simple_strtoul((param+1), &end, 0);
+            if (num_elems != 0) {
+                param = end;
+                if (*end == '@')
+                    address = (unsigned char)simple_strtoul(param + 1, &end, 0);
+            } else {
+                num_elems = 1;
+            }
+
+            if (*end == '@') {
+                address = (uint8_t)simple_strtoul(param + 1, &end, 0);
+                if (((end - param - 1) > 4) || ((end - param - 1) <= 3)) {
+                    goto error_i2c_msgs;
+                }
+            }
+
+            if (*end != '\0')
+                goto error_i2c_msgs;
+
+            if ((address == 0x00) || (num_elems == 0) || (address > 0x7F))
+                goto error_i2c_msgs;
+
+            cmd = kzalloc(sizeof(struct rw_cmd), GFP_KERNEL);
+            if (!cmd)
+                return -ENOMEM;
+
+            cmd->is_write = (read_write == 'w' ? true : false);
+            cmd->slave_addr = address;
+            cmd->num_elems = num_elems;
+            cmd->rw_buffer =
+                kzalloc(sizeof(int32_t) * cmd->num_elems, GFP_KERNEL);
+            if (!cmd->rw_buffer)
+                goto error_i2c_msgs;
+
+            list_add_tail(&cmd->list, cmd_list);
+            if (read_write == 'w')
+                index = num_elems;
+
+            continue;
+        }
+
+        if (*param != '\0' && read_write != 0) {
+            if (read_write == 'r')
+                goto error_i2c_msgs;
+
+            if (index == 0)
+                goto error_i2c_msgs;
+
+            cmd->rw_buffer[num_elems - index] =
+                                (uint8_t)simple_strtoul(param, &end, 0);
+            
+            if ((*end != '\0') || ((end - param) > 4) ||
+                ((end - param) <= 3))
+                goto error_i2c_msgs;
+            index--;
+        }
+    }
+    return 0;
+
+error_i2c_msgs:
+    release_cmd_list(cmd_list);
+    return -EFAULT;;
+}
+
+static int32_t gpio_i2c_procfs_process(struct seq_file *sfile,
+                        struct gpio_i2c_procfs *i2c_proc, char *str_buf)
+{
+    struct i2c_msg msg;
+    struct gpio_i2c_status *status;
+    struct rw_cmd *cmd = NULL;
+    int32_t ret = 0;
+
+    mutex_lock(&i2c_proc->lock);
+    if (!i2c_proc->status) {
+        i2c_proc->status = devm_kzalloc(i2c_proc->adapter->dev.parent,
+                            sizeof(struct gpio_i2c_status), GFP_KERNEL);
+        if (!i2c_proc->status) {
+            mutex_unlock(&i2c_proc->lock);
+            return -ENOMEM;
+        }
+    } else {
+        release_cmd_list(&i2c_proc->status->cmd_list);
+    }
+
+    status = i2c_proc->status;
+    INIT_LIST_HEAD(&status->cmd_list);
+
+    ret = gpio_i2c_parse_args(str_buf, &status->cmd_list);
+    if (ret) {
+        status->flags = PROC_PARSE_ERROR;
+        mutex_unlock(&i2c_proc->lock);
+        return ret;
+    }
+
+    list_for_each_entry(cmd, &status->cmd_list, list) {
+        msg.addr = cmd->slave_addr;
+        msg.len = cmd->num_elems;
+        msg.buf = cmd->rw_buffer;
+        if (cmd->is_write) {
+            msg.flags |= 0; 
+        } else {
+            msg.flags |= I2C_M_RD;
+        }
+
+        ret = i2c_transfer(i2c_proc->adapter, &msg, 1);
+        if (ret != 1) {
+            cmd->flags = PROC_RW_ERROR;
+        }
+    }
+    mutex_unlock(&i2c_proc->lock);
+    return 0;
+}
+
+static ssize_t gpio_i2c_procfs_write(struct file *file,
+		const char __user *buffer, size_t count, loff_t *ppos)
+{
+    struct gpio_i2c_procfs *i2c_proc;
+    struct seq_file *sfile;
+    char *str_buf;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	i2c_proc = (struct gpio_i2c_procfs *) PDE_DATA(file_inode(file));
+#else
+	i2c_proc = (struct gpio_i2c_procfs *) pde_data(file_inode(file));
+#endif
+    sfile = file->private_data;
+
+    if (count > GPIO_I2C_PROCFS_BUF_SIZE)
+        count = GPIO_I2C_PROCFS_BUF_SIZE - 1;
+
+    str_buf = (char *)kzalloc(count, GFP_KERNEL);
+    if (!str_buf)
+        return -ENOMEM;
+    
+    if (copy_from_user(str_buf, buffer, count))
+        return -EFAULT;
+
+    *(str_buf + count) = '\0';
+
+    gpio_i2c_procfs_process(sfile, i2c_proc, str_buf);
+
+    kfree(str_buf);
+
+    return count;
+}
+
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
+static const struct file_operations gpio_i2c_procfs_ops = {
+    .open = gpio_i2c_procfs_open,
+    .release = seq_release,
+    .read = seq_read,
+    .write = gpio_i2c_procfs_write,
+    .llseek = seq_lseek,
+};
+#else
+static const struct proc_ops gpio_i2c_procfs_ops = {
+	.proc_open = gpio_i2c_procfs_open,
+	.proc_release = seq_release,
+	.proc_read = seq_read,
+	.proc_write = gpio_i2c_procfs_write,
+	.proc_lseek = seq_lseek,
+};
+#endif
+
+struct finddir_callback {
+    struct dir_context ctx;
+    const char *name;
+    int32_t files_cnt;
+    bool found;
+};
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)
+static int readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return 0;
+
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return 0;
+}
+
+#else
+static bool readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+
+    struct finddir_callback *fc = container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return true;
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return true;
+}
+#endif
+
+
+static int find_proc_dir_by_name(const char *root,
+                const char *name, bool *found, int32_t *files_cnt) {
+    struct file *pfile;
+    int ret = 0;
+    struct finddir_callback fc = {
+        .ctx.actor = readdir_callback,
+        .name = name,
+        .found = false,
+        .files_cnt = -2,
+    };
+
+    pfile = filp_open(root, O_RDONLY | O_DIRECTORY, 0);
+    if(IS_ERR(pfile)) {
+        pr_err("Failed to open %s\n", root);
+        return -1;
+    }
+    if (pfile->f_op->iterate_shared) {
+        ret = pfile->f_op->iterate_shared(pfile, &fc.ctx);
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 2, 0)
+    } else {
+        ret = pfile->f_op->iterate(pfile, &fc.ctx);
+#endif
+    }
+
+    if (ret == 0) {
+        *found = fc.found;
+    }
+
+    if (files_cnt != NULL) {
+        *files_cnt = fc.files_cnt;
+    }
+
+    filp_close(pfile, NULL);
+    return ret;
+}
+
+int soc_gpio_i2c_procfs_register(struct i2c_adapter *adapter,
+                                    struct proc_dir_entry **pde)
+{
+    struct gpio_i2c_procfs *i2c_proc;
+    char i2c_proc_name[32];
+    int ret = 0;
+    bool found = false;
+
+    if (!adapter)
+        return -1;
+    sprintf(i2c_proc_name, "vsi/i2c%d", adapter->nr);
+    i2c_proc = devm_kzalloc(adapter->dev.parent,
+                        sizeof(struct gpio_i2c_procfs), GFP_KERNEL);
+    if (!i2c_proc)
+        return -ENOMEM;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (!found)
+            proc_mkdir("vsi", NULL);    
+    } else {
+        return -EFAULT;
+    }
+
+    i2c_proc->adapter = adapter;
+    *pde = proc_create_data(i2c_proc_name, 0664, NULL,
+                                &gpio_i2c_procfs_ops, i2c_proc);
+    if (!*pde)
+        return -EFAULT;
+
+    i2c_proc->pde = *pde;
+    mutex_init(&(i2c_proc->lock));
+
+	return 0;
+}
+
+void soc_gpio_i2c_procfs_unregister(struct proc_dir_entry **pde)
+{
+    int ret = 0;
+    bool found = false;
+    int32_t files_cnt;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (found) {
+            proc_remove(*pde);
+            ret = find_proc_dir_by_name("/proc/vsi", "", &found, &files_cnt);
+            if (files_cnt == 0) {
+                remove_proc_subtree("vsi", NULL);
+            }
+        }
+    }
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_procfs.h b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_procfs.h
new file mode 100644
index 000000000000..14cab7a7e82c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/i2c/soc_i2c_procfs.h
@@ -0,0 +1,66 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __SOC_I2C_PROCFS_H__
+#define __SOC_I2C_PROCFS_H__
+
+
+
+int soc_gpio_i2c_procfs_register(struct i2c_adapter *adapter,
+                                    struct proc_dir_entry **pde);
+void soc_gpio_i2c_procfs_unregister(struct proc_dir_entry **pde);
+
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/insmod.sh b/drivers/media/platform/eswin/isp/vvcam/insmod.sh
new file mode 100644
index 000000000000..9336b0b403d2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/insmod.sh
@@ -0,0 +1,180 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+#!/bin/sh
+
+# Example: ./insmod.sh 4sensor 2isp vi mss
+
+param_sensor_num="2sensor" 
+param_isp_num="1isp"
+param_videoin="false"
+param_mss="false"
+param_rdc="false"
+
+if [ -n "$1" ];then
+    if [ "$1" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$1" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$1" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$1" = "vi" ];then
+        param_videoin="true"
+    elif [ "$1" = "mss" ];then
+        param_mss="true"
+    elif [ "$1" = "rdc" ];then
+        param_rdc="true"
+    fi
+fi
+
+if [ -n "$2" ];then
+    if [ "$2" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$2" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$2" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$2" = "vi" ];then
+        param_videoin="true"
+    elif [ "$2" = "mss" ];then
+        param_mss="true"
+    elif [ "$2" = "rdc" ];then
+        param_rdc="true"
+    fi
+fi
+
+if [ -n "$3" ];then
+    if [ "$3" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$3" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$3" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$3" = "vi" ];then
+        param_videoin="true"
+    elif [ "$3" = "mss" ];then
+        param_mss="true"
+    elif [ "$3" = "rdc" ];then
+        param_rdc="true"
+    fi
+fi
+
+if [ -n "$4" ];then
+    if [ "$4" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$4" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$4" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$4" = "vi" ];then
+        param_videoin="true"
+    elif [ "$4" = "mss" ];then
+        param_mss="true"
+    elif [ "$4" = "rdc" ];then
+        param_rdc="true"
+    fi
+fi
+
+if [ -n "$5" ];then
+    if [ "$5" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$5" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$5" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$5" = "vi" ];then
+        param_videoin="true"
+    elif [ "$5" = "mss" ];then
+        param_mss="true"
+    elif [ "$5" = "rdc" ];then
+        param_rdc="true"
+    fi
+fi
+
+insmod ./vb/vvcam_vb.ko
+
+if [ "$param_sensor_num" = "4sensor" ];then
+    insmod ./i2c/vvcam_i2c.ko devices_mask=0x0f
+    insmod ./mipi/vvcam_mipi.ko devices_mask=0x0f
+else
+    insmod ./i2c/vvcam_i2c.ko devices_mask=0x03
+    insmod ./mipi/vvcam_mipi.ko devices_mask=0x03
+fi
+
+if [ "$param_isp_num" = "3isp" ];then
+    devices_mask=0x07
+elif [ "$param_isp_num" = "2isp" ];then
+    devices_mask=0x03
+else
+    devices_mask=0x01
+fi
+
+if [ "$param_rdc" = "true" ];then
+    rdc_enable=1
+elif [ "$param_rdc" = "false" ];then
+    rdc_enable=0
+else
+    rdc_enable=0
+fi
+
+insmod ./isp/vvcam_isp.ko devices_mask=${devices_mask} rdc_enable=${rdc_enable}
+
+if [ "$param_videoin" = "true" ];then
+    insmod ./videoin/vvcam_vi.ko
+fi
+
+if [ "$param_mss" = "true" ];then
+    insmod ./mi_subsystem/vvcam_mss.ko
+fi
diff --git a/drivers/media/platform/eswin/isp/vvcam/insmod_isp_dw.sh b/drivers/media/platform/eswin/isp/vvcam/insmod_isp_dw.sh
new file mode 100644
index 000000000000..535d2075e601
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/insmod_isp_dw.sh
@@ -0,0 +1,161 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+#!/bin/sh
+
+# Example: ./insmod.sh 4sensor 2isp vi mss
+
+param_sensor_num="2sensor" 
+param_isp_num="1isp"
+param_videoin="false"
+param_mss="false"
+
+if [ -n "$1" ];then
+    if [ "$1" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$1" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$1" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$1" = "vi" ];then
+        param_videoin="true"
+    elif [ "$1" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+if [ -n "$2" ];then
+    if [ "$2" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$2" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$2" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$2" = "vi" ];then
+        param_videoin="true"
+    elif [ "$2" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+if [ -n "$3" ];then
+    if [ "$3" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$3" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$3" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$3" = "vi" ];then
+        param_videoin="true"
+    elif [ "$3" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+if [ -n "$4" ];then
+    if [ "$4" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$4" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$4" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$4" = "vi" ];then
+        param_videoin="true"
+    elif [ "$4" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+if [ -n "$5" ];then
+    if [ "$5" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$5" = "2isp" ];then
+        param_isp_num="2isp"
+    elif [ "$5" = "3isp" ];then
+        param_isp_num="3isp"
+    elif [ "$5" = "vi" ];then
+        param_videoin="true"
+    elif [ "$5" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+insmod ./vb/vvcam_vb.ko
+
+if [ "$param_sensor_num" = "4sensor" ];then
+    insmod ./i2c/vvcam_i2c.ko devices_mask=0x0f
+    insmod ./mipi/vvcam_mipi.ko devices_mask=0x0f
+else
+    insmod ./i2c/vvcam_i2c.ko devices_mask=0x03
+    insmod ./mipi/vvcam_mipi.ko devices_mask=0x03
+fi
+
+if [ "$param_isp_num" = "3isp" ];then
+    insmod ./isp/vvcam_isp.ko devices_mask=0x07
+elif [ "$param_isp_num" = "2isp" ];then
+    insmod ./isp/vvcam_isp.ko devices_mask=0x03
+else
+    insmod ./isp/vvcam_isp.ko
+fi
+
+if [ "$param_videoin" = "true" ];then
+    insmod ./videoin/vvcam_vi.ko
+fi
+
+if [ "$param_mss" = "true" ];then
+    insmod ./mi_subsystem/vvcam_mss.ko
+fi
+
+insmod ./dw200/vvcam_dw.ko devices_mask=0x07
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/Makefile b/drivers/media/platform/eswin/isp/vvcam/isp/Makefile
new file mode 100644
index 000000000000..d4f74c9b968c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/Makefile
@@ -0,0 +1,70 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+ifeq ($(VVCAM_PATH),)
+        VVCAM_PATH:=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))/../../vvcam
+        export VVCAM_PATH
+endif
+
+include $(VVCAM_PATH)/Param.mk
+TARGET = vvcam_isp
+
+obj-$(CONFIG_VIDEO_ESWIN_ISP) += $(TARGET).o
+
+#$(TARGET)-objs += vvcam_isp_platform.o
+$(TARGET)-objs += vvcam_isp_driver.o
+$(TARGET)-objs += vvcam_isp_hal.o
+$(TARGET)-objs += vvcam_isp_procfs.o
+$(TARGET)-objs += ../common/vvcam_event.o
+EXTRA_CFLAGS += -I$(VVCAM_PATH)/common/
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp.h b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp.h
new file mode 100644
index 000000000000..90171d943d18
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp.h
@@ -0,0 +1,104 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_H__
+#define __VVCAM_ISP_H__
+#include "vvcam_event.h"
+
+enum {
+    VVCAM_EID_ISP_MIS            = 0x00,
+    VVCAM_EID_MIV2_MIS           = 0x01,
+    VVCAM_EID_MIV2_MIS1          = 0x02,
+    VVCAM_EID_MIV2_MIS2          = 0x03,
+    VVCAM_EID_MIV2_MIS3          = 0x04,
+    VVCAM_EID_MI_MIS_HDR1        = 0x05,
+    VVCAM_EID_RDMA_MIS           = 0x06,
+    VVCAM_EID_MCM_WR_RAW0_MIS    = 0x07,
+    VVCAM_EID_MCM_WR_RAW1_MIS    = 0x08,
+    VVCAM_EID_MCM_WR_G2_RAW0_MIS = 0x09,
+    VVCAM_EID_MCM_WR_G2_RAW1_MIS = 0x10,
+    VVCAM_EID_FE_MIS             = 0x11,
+    VVCAM_EID_FE_BATCH_MODE_MIS  = 0x12,
+    VVCAM_EID_MIV2_MIS2_HDR      = 0x13,
+    VVCAM_EID_MIV2_JDP           = 0x14,
+    VVCAM_EID_FUSA_ECC_IMSC1     = 0x15,
+    VVCAM_EID_FUSA_ECC_IMSC2     = 0x16,
+    VVCAM_EID_FUSA_ECC_IMSC3     = 0x17,
+    VVCAM_EID_FUSA_ECC_IMSC4     = 0x18,
+    VVCAM_EID_FUSA_ECC_IMSC5     = 0x19,
+    VVCAM_EID_FUSA_ECC_IMSC6     = 0x1a,
+    VVCAM_EID_FUSA_DUP_IMSC      = 0x1b,
+    VVCAM_EID_FUSA_PARITY_IMSC   = 0x1c,
+    VVCAM_EID_FUSA_LV1_IMSC1     = 0x1d,
+    VVCAM_EID_RDCD_MI_MIS        = 0x1e,
+    VVCAM_EID_FUSA_RDCD_MIS      = 0x1f,
+    VVCAM_EID_MAX                = 0x20
+};
+
+
+typedef struct {
+    uint32_t addr;
+    uint32_t value;
+} vvcam_isp_reg_t;
+
+#define VVCAM_ISP_IOC_MAGIC 'v'
+#define VVCAM_ISP_RESET             _IOW(VVCAM_ISP_IOC_MAGIC, 0x01, uint32_t)
+#define VVCAM_ISP_READ_REG          _IOWR(VVCAM_ISP_IOC_MAGIC, 0x02, vvcam_isp_reg_t)
+#define VVCAM_ISP_WRITE_REG         _IOW(VVCAM_ISP_IOC_MAGIC, 0x03, vvcam_isp_reg_t)
+#define VVCAM_ISP_SUBSCRIBE_EVENT   _IOW(VVCAM_ISP_IOC_MAGIC, 0x04, vvcam_subscription_t)
+#define VVCAM_ISP_UNSUBSCRIBE_EVENT _IOW(VVCAM_ISP_IOC_MAGIC, 0x05, vvcam_subscription_t)
+#define VVCAM_ISP_DQEVENT           _IOR(VVCAM_ISP_IOC_MAGIC, 0x06, vvcam_event_t)
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_driver.c b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_driver.c
new file mode 100644
index 000000000000..18b4f1bf789f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_driver.c
@@ -0,0 +1,608 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/pm_runtime.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/property.h>
+#include <linux/i2c.h>
+#include <media/v4l2-subdev.h>
+#include <linux/delay.h>
+#include <linux/iommu.h>
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+#include "vvcam_isp_driver.h"
+//#include "vvcam_isp_platform.h"
+#include "vvcam_isp.h"
+#include "vvcam_event.h"
+#include "vvcam_isp_procfs.h"
+
+#define ISP_FUSA_DEBUG 0
+
+#define AWSMMUSID	GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID	GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID	GENMASK(15, 8)	// The sid of read operation
+#define ARSMMUSSID	GENMASK(7, 0)	// The ssid of read operation
+
+extern void vvcam_isp_irq_stat_tasklet(unsigned long);
+
+static irqreturn_t vvcam_isp_irq_handler(int irq, void *isp_dev)
+{
+    return vvcam_isp_irq_process(isp_dev);
+}
+
+static irqreturn_t vvcam_isp_mi_irq_handler(int irq, void *isp_dev)
+{
+    return vvcam_isp_mi_irq_process(isp_dev);
+}
+
+static irqreturn_t vvcam_isp_fe_irq_handler(int irq, void *isp_dev)
+{
+    return vvcam_isp_fe_irq_process(isp_dev);
+}
+
+#if ISP_FUSA_DEBUG
+static irqreturn_t vvcam_isp_fusa_irq_handler(int irq, void *isp_dev)
+{
+    return vvcam_isp_fusa_irq_process(isp_dev);
+}
+#endif
+
+static int vvcam_isp_open(struct inode *inode, struct file *file)
+{
+    struct miscdevice *pmisc_dev = file->private_data;
+    struct vvcam_isp_dev *isp_dev;
+    struct vvcam_isp_fh *isp_fh;
+
+    isp_dev = container_of(pmisc_dev, struct vvcam_isp_dev, miscdev);
+    if (!isp_dev)
+        return -ENOMEM;
+
+    isp_fh = kzalloc(sizeof(struct vvcam_isp_fh), GFP_KERNEL);
+    if (!isp_fh)
+        return -ENOMEM;
+
+    isp_fh->isp_dev = isp_dev;
+    vvcam_event_fh_init( &isp_dev->event_dev, &isp_fh->event_fh);
+    file->private_data = isp_fh;
+
+    mutex_lock(&isp_dev->mlock);
+    isp_dev->refcnt++;
+    pm_runtime_get_sync(isp_dev->dev);
+    mutex_unlock(&isp_dev->mlock);
+
+    return 0;
+}
+
+static int vvcam_isp_release(struct inode *inode, struct file *file)
+{
+    struct vvcam_isp_dev *isp_dev;
+    struct vvcam_isp_fh *isp_fh;
+
+    isp_fh = file->private_data;
+    isp_dev = isp_fh->isp_dev;
+
+    vvcam_event_unsubscribe_all(&isp_fh->event_fh);
+
+    vvcam_event_fh_destroy(&isp_dev->event_dev, &isp_fh->event_fh);
+
+    mutex_lock(&isp_dev->mlock);
+    isp_dev->refcnt--;
+    pm_runtime_put(isp_dev->dev);
+    mutex_unlock(&isp_dev->mlock);
+
+    kfree(isp_fh);
+
+    return 0;
+}
+
+static long vvcam_isp_ioctl(struct file *file,
+                        unsigned int cmd, unsigned long arg)
+{
+    struct vvcam_isp_dev *isp_dev;
+    struct vvcam_isp_fh *isp_fh;
+    uint32_t reset;
+    vvcam_isp_reg_t isp_reg;
+    vvcam_subscription_t sub;
+    vvcam_event_t event;
+    int ret = 0;
+
+    isp_fh = file->private_data;
+    isp_dev = isp_fh->isp_dev;
+
+    mutex_lock(&isp_dev->mlock);
+
+    switch(cmd) {
+    case VVCAM_ISP_RESET:
+        ret = copy_from_user(&reset, (void __user *)arg, sizeof(reset));
+        if (ret)
+            break;
+        ret = vvcam_isp_reset(isp_dev, reset);
+        break;
+    case VVCAM_ISP_READ_REG:
+        ret = copy_from_user(&isp_reg, (void __user *)arg, sizeof(isp_reg));
+        if (ret)
+            break;
+        ret = vvcam_isp_read_reg(isp_dev, &isp_reg);
+        if (ret)
+            break;
+        ret = copy_to_user((void __user *)arg, &isp_reg, sizeof(isp_reg));
+        break;
+    case VVCAM_ISP_WRITE_REG:
+        ret = copy_from_user(&isp_reg, (void __user *)arg, sizeof(isp_reg));
+        if (ret)
+            break;
+        ret = vvcam_isp_write_reg(isp_dev, isp_reg);
+        break;
+    case VVCAM_ISP_SUBSCRIBE_EVENT:
+        ret = copy_from_user(&sub, (void __user *)arg, sizeof(sub));
+        if (ret)
+            break;
+        ret = vvcam_event_subscribe(&isp_fh->event_fh,
+                    &sub, VVCAM_ISP_EVENT_ELEMS);
+        break;
+    case VVCAM_ISP_UNSUBSCRIBE_EVENT:
+        ret = copy_from_user(&sub, (void __user *)arg, sizeof(sub));
+        if (ret)
+            break;
+        ret = vvcam_event_unsubscribe(&isp_fh->event_fh, &sub);
+        break;
+    case VVCAM_ISP_DQEVENT:
+        ret = vvcam_event_dequeue(&isp_fh->event_fh, &event);
+        if (ret)
+            break;
+        ret = copy_to_user((void __user *)arg, &event, sizeof(event));
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    mutex_unlock(&isp_dev->mlock);
+
+    return ret;
+}
+
+static unsigned int vvcam_isp_poll(struct file *file, poll_table *wait)
+{
+    struct vvcam_isp_fh *isp_fh;
+
+    isp_fh = file->private_data;
+
+    return vvcam_event_poll(file, &isp_fh->event_fh, wait);
+}
+
+int vvcam_isp_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    struct vvcam_isp_dev *isp_dev;
+    struct vvcam_isp_fh *isp_fh;
+
+    isp_fh = file->private_data;
+    isp_dev = isp_fh->isp_dev;
+
+    vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+    return vm_iomap_memory(vma, isp_dev->paddr, isp_dev->regs_size);
+}
+
+static struct file_operations vvcam_isp_fops = {
+	.owner          = THIS_MODULE,
+	.open           = vvcam_isp_open,
+	.release        = vvcam_isp_release,
+	.unlocked_ioctl = vvcam_isp_ioctl,
+    .poll           = vvcam_isp_poll,
+    .mmap           = vvcam_isp_mmap,
+
+};
+
+static int vvcam_isp_parse_params(struct vvcam_isp_dev *isp_dev,
+                        struct platform_device *pdev)
+{
+	int ret;
+	struct resource *res;
+
+    ret = device_property_read_u32(&pdev->dev, "id", &pdev->id);
+    if (ret) {
+        pr_info("isp device id not found, use default\n");
+        pdev->id = 0;
+    }
+
+    res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+    if (!res) {
+		dev_err(&pdev->dev, "can't fetch device resource info\n");
+		return -EIO;
+	}
+    isp_dev->paddr = res->start;
+    isp_dev->regs_size = resource_size(res);
+    isp_dev->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(isp_dev->base)) {
+        dev_err(&pdev->dev, "can't remap device resource info\n");
+        return PTR_ERR(isp_dev->base);
+    }
+
+    isp_dev->isp_irq = platform_get_irq(pdev, 0);
+    if (isp_dev->isp_irq < 0) {
+		dev_err(&pdev->dev, "can't get irq resource\n");
+		return -ENXIO;
+	}
+
+    isp_dev->mi_irq = platform_get_irq(pdev, 1);
+    if (isp_dev->mi_irq < 0) {
+		dev_err(&pdev->dev, "can't get mi irq resource\n");
+		return -ENXIO;
+	}
+
+    isp_dev->fe_irq = platform_get_irq(pdev, 2);
+    if (isp_dev->fe_irq < 0) {
+		dev_err(&pdev->dev, "can't get fe irq resource\n");
+		return -ENXIO;
+	}
+#if 1 //ISP_FUSA_DEBUG
+    // isp_dev->fusa_irq = platform_get_irq(pdev, 3);
+    // if (isp_dev->fusa_irq < 0) {
+    //     dev_err(&pdev->dev, "can't get fusa irq resource\n");
+    //     return -ENXIO;
+    // }
+
+    // res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+    // if (!res) {
+	// 	dev_err(&pdev->dev, "can't fetch device reset reg\n");
+	// 	return -EIO;
+	// }
+    isp_dev->reset = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+    if (IS_ERR(isp_dev->reset)) {
+        dev_err(&pdev->dev, "can't remap device reset reg\n");
+        return PTR_ERR(isp_dev->reset);
+    }
+#endif
+    return 0;
+}
+
+static int isp_smmu_sid_cfg(struct device* dev)
+{
+    int ret = 0;
+    struct regmap* regmap = NULL;
+    int mmu_tbu0_vi_isp_reg = 0;
+    u32 rdwr_sid_ssid = 0;
+    u32 sid = 0;
+
+    struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+    pr_info("%s enter\n", __func__);
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		pr_info("isp is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,vi_top_csr");
+	if (IS_ERR(regmap)) {
+		pr_info("No vi_top_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,vi_top_csr", 1,
+					&mmu_tbu0_vi_isp_reg);
+	if (ret) {
+		pr_err("can't get isp sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, mmu_tbu0_vi_isp_reg, rdwr_sid_ssid);
+
+	ret = win2030_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		pr_err("failed to config isp streamID(%d)!\n", sid);
+	else
+		pr_info("success to config isp streamID(%d)!\n", sid);
+    
+    pr_info("%s exit\n", __func__);
+    return ret;
+
+}
+
+static int vvcam_isp_probe(struct platform_device *pdev)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev;
+    char *ispdev_name;
+    struct device *dev = &pdev->dev;
+    struct device_node *sensor_np;
+    struct device *sensor_dev;
+    struct v4l2_subdev *sd;
+    int wait_time = 0;
+
+    sensor_np = of_parse_phandle(dev->of_node, "depends-on", 0);
+    if (sensor_np)
+    {
+        while (wait_time < 20000) {
+            sensor_dev = bus_find_device_by_of_node(&i2c_bus_type, sensor_np);
+            of_node_put(sensor_np);
+            if (sensor_dev) {
+                sd = dev_get_drvdata(sensor_dev);
+                put_device(sensor_dev);
+                if (sd && sd->v4l2_dev) {
+                    dev_info(dev, "Sensor is ready.\n");
+                    break;
+                }
+            }
+            msleep(1000);
+            wait_time += 1000;
+        }
+    }
+
+    isp_dev = devm_kzalloc(&pdev->dev,
+                sizeof(struct vvcam_isp_dev), GFP_KERNEL);
+    if (!isp_dev)
+        return -ENOMEM;
+
+    (void)isp_smmu_sid_cfg(&pdev->dev);
+
+    ret = vvcam_isp_parse_params(isp_dev, pdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to parse params\n");
+        return -EINVAL;
+    }
+
+    mutex_init(&isp_dev->mlock);
+    spin_lock_init(&isp_dev->stat_lock);
+    vvcam_event_dev_init(&isp_dev->event_dev);
+	platform_set_drvdata(pdev, isp_dev);
+
+    isp_dev->dev = &pdev->dev;
+    isp_dev->id  = pdev->id;
+
+    ispdev_name = devm_kzalloc(&pdev->dev, 16, GFP_KERNEL);
+    if (!ispdev_name)
+        return -ENOMEM;
+    snprintf(ispdev_name, 16, "%s.%d", VVCAM_ISP_NAME, pdev->id);
+
+    isp_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
+    isp_dev->miscdev.name  = ispdev_name;
+    isp_dev->miscdev.fops  = &vvcam_isp_fops;
+
+    ret = misc_register(&isp_dev->miscdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to register device\n");
+        return -EINVAL;
+    }
+
+    ret = vvcam_isp_procfs_register(isp_dev, &isp_dev->pde);
+    if (ret) {
+        dev_err(&pdev->dev, "isp register procfs failed.\n");
+        goto error_register_procfs;
+    }
+
+    tasklet_init(&isp_dev->stat_tasklet,
+        vvcam_isp_irq_stat_tasklet, (unsigned long)isp_dev);
+
+    ret = devm_request_irq(&pdev->dev, isp_dev->isp_irq, vvcam_isp_irq_handler,
+                IRQF_TRIGGER_HIGH | IRQF_SHARED, dev_name(&pdev->dev), isp_dev);
+    if (ret) {
+        dev_err(&pdev->dev, "can't request isp irq\n");
+        goto error_request_isp_irq;
+    }
+
+    ret = devm_request_irq(&pdev->dev, isp_dev->mi_irq, vvcam_isp_mi_irq_handler,
+                IRQF_TRIGGER_HIGH | IRQF_SHARED, dev_name(&pdev->dev), isp_dev);
+    if (ret) {
+        dev_err(&pdev->dev, "can't request mi irq\n");
+        goto error_request_mi_irq;
+    }
+
+    ret = devm_request_irq(&pdev->dev, isp_dev->fe_irq, vvcam_isp_fe_irq_handler,
+                IRQF_TRIGGER_HIGH | IRQF_SHARED, dev_name(&pdev->dev), isp_dev);
+    if (ret) {
+        dev_err(&pdev->dev, "can't request fe irq\n");
+        goto error_request_fe_irq;
+    }
+
+#if ISP_FUSA_DEBUG
+    ret = devm_request_irq(&pdev->dev, isp_dev->fusa_irq, vvcam_isp_fusa_irq_handler,
+                IRQF_TRIGGER_HIGH | IRQF_SHARED, dev_name(&pdev->dev), isp_dev);
+    if (ret) {
+        dev_err(&pdev->dev, "can't request fusa irq\n");
+        goto error_request_fusa_irq;
+    }
+#endif
+    dev_info(&pdev->dev, "vvcam isp driver probe success\n");
+
+    return 0;
+
+#if ISP_FUSA_DEBUG
+error_request_fusa_irq:
+    devm_free_irq(&pdev->dev, isp_dev->fe_irq, isp_dev);
+#endif	
+error_request_fe_irq:
+    devm_free_irq(&pdev->dev, isp_dev->mi_irq, isp_dev);
+error_request_mi_irq:
+    devm_free_irq(&pdev->dev, isp_dev->isp_irq, isp_dev);
+error_request_isp_irq:
+    vvcam_isp_procfs_unregister(isp_dev->pde);
+error_register_procfs:
+    misc_deregister(&isp_dev->miscdev);
+
+    return ret;
+}
+
+static int vvcam_isp_remove(struct platform_device *pdev)
+{
+    struct vvcam_isp_dev *isp_dev;
+
+    isp_dev = platform_get_drvdata(pdev);
+
+    vvcam_isp_procfs_unregister(isp_dev->pde);
+
+    misc_deregister(&isp_dev->miscdev);
+    devm_free_irq(&pdev->dev, isp_dev->isp_irq, isp_dev);
+    devm_free_irq(&pdev->dev, isp_dev->mi_irq, isp_dev);
+    devm_free_irq(&pdev->dev, isp_dev->fe_irq, isp_dev);
+	
+#if ISP_FUSA_DEBUG
+    devm_free_irq(&pdev->dev, isp_dev->fusa_irq, isp_dev);
+#endif
+    tasklet_kill(&isp_dev->stat_tasklet);
+
+    return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int vvcam_isp_system_suspend(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_suspend(dev);
+    if (ret) {
+        dev_err(dev, "force suspend %s failed\n", dev_name(dev));
+        return ret;
+    }
+	return ret;
+}
+
+static int vvcam_isp_system_resume(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_resume(dev);
+    if (ret) {
+        dev_err(dev, "force resume %s failed\n", dev_name(dev));
+        return ret;
+    }
+	return ret;
+}
+
+static int vvcam_isp_runtime_suspend(struct device *dev)
+{
+    return 0;
+}
+
+static int vvcam_isp_runtime_resume(struct device *dev)
+{
+    return 0;
+}
+
+static const struct dev_pm_ops vvcam_isp_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(vvcam_isp_system_suspend, vvcam_isp_system_resume)
+	SET_RUNTIME_PM_OPS(vvcam_isp_runtime_suspend, vvcam_isp_runtime_resume, NULL)
+};
+#endif
+
+static const struct of_device_id vvcam_isp_of_match[] = {
+	{.compatible = "esw,win2030-isp",},
+	{ /* sentinel */ },
+};
+
+static struct platform_driver vvcam_isp_driver = {
+	.probe  = vvcam_isp_probe,
+	.remove = vvcam_isp_remove,
+	.driver = {
+		.name           = VVCAM_ISP_NAME,
+		.owner          = THIS_MODULE,
+        .of_match_table = vvcam_isp_of_match,
+#ifdef CONFIG_PM_SLEEP        
+        .pm             = &vvcam_isp_pm_ops,
+#endif
+	}
+};
+
+MODULE_DEVICE_TABLE(of, vvcam_isp_of_match);
+
+static int __init vvcam_isp_init_module(void)
+{
+    int ret;
+    ret = platform_driver_register(&vvcam_isp_driver);
+    if (ret) {
+        printk(KERN_ERR "Failed to register isp driver\n");
+        return ret;
+    }
+#if 0
+    ret = vvcam_isp_platform_device_register();
+    if (ret) {
+		platform_driver_unregister(&vvcam_isp_driver);
+		printk(KERN_ERR "Failed to register vvcam isp platform devices\n");
+		return ret;
+	}
+#endif
+    return ret;
+}
+
+static void __exit vvcam_isp_exit_module(void)
+{
+    platform_driver_unregister(&vvcam_isp_driver);
+//    vvcam_isp_platform_device_unregister();
+}
+
+late_initcall(vvcam_isp_init_module);
+module_exit(vvcam_isp_exit_module);
+
+MODULE_DESCRIPTION("Verisilicon isp driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_driver.h b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_driver.h
new file mode 100644
index 000000000000..39854eec287c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_driver.h
@@ -0,0 +1,104 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DRIVER_H__
+#define __VVCAM_ISP_DRIVER_H__
+
+#include <linux/miscdevice.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+
+#include "vvcam_isp.h"
+#include "vvcam_event.h"
+
+#define VVCAM_ISP_NAME "vvcam-isp"
+
+#define VVCAM_ISP_EVENT_ELEMS 10
+#define VVCAM_VI_EVENT_ELEMS 10
+struct vvcam_isp_dev {
+    phys_addr_t paddr;
+    uint32_t regs_size;
+    void __iomem *base;
+    void __iomem *reset;
+    int  id;
+    int isp_irq;
+    int fe_irq;
+    int mi_irq;
+    int fusa_irq;
+    struct miscdevice miscdev;
+    struct device *dev;
+    unsigned long pde;
+    struct mutex mlock;
+    struct vvcam_event_dev event_dev;
+    uint32_t irq_mis[VVCAM_EID_MAX];
+    struct tasklet_struct stat_tasklet;
+    spinlock_t stat_lock;
+    uint32_t refcnt;
+};
+
+struct vvcam_isp_fh {
+    struct vvcam_isp_dev *isp_dev;
+    struct vvcam_event_fh event_fh;
+};
+
+int vvcam_isp_reset(struct vvcam_isp_dev *isp_dev, uint32_t reset);
+int vvcam_isp_write_reg(struct vvcam_isp_dev *isp_dev, vvcam_isp_reg_t isp_reg);
+int vvcam_isp_read_reg(struct vvcam_isp_dev *isp_dev, vvcam_isp_reg_t *isp_reg);
+irqreturn_t vvcam_isp_irq_process(struct vvcam_isp_dev *isp_dev);
+irqreturn_t vvcam_isp_mi_irq_process(struct vvcam_isp_dev *isp_dev);
+irqreturn_t vvcam_isp_fe_irq_process(struct vvcam_isp_dev *isp_dev);
+irqreturn_t vvcam_isp_fusa_irq_process(struct vvcam_isp_dev *isp_dev);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_hal.c b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_hal.c
new file mode 100644
index 000000000000..e455dd8dd91b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_hal.c
@@ -0,0 +1,669 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/module.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_hal.h"
+
+static bool rdc_enable = false;
+MODULE_PARM_DESC(rdc_enable, "vvcam enable rdc");
+
+extern void vvcam_isp_proc_stat(unsigned long pde,
+                    const uint32_t *irq_mis, const int len);
+
+void vvcam_isp_irq_stat_tasklet(unsigned long dev)
+{
+    struct vvcam_isp_dev *isp_dev = (struct vvcam_isp_dev *)dev;
+    uint32_t irq_mis[VVCAM_EID_MAX];
+
+    memset(irq_mis, 0, sizeof(irq_mis));
+
+    memcpy(irq_mis, isp_dev->irq_mis, sizeof(irq_mis));
+    memset(isp_dev->irq_mis, 0, sizeof(isp_dev->irq_mis));
+
+    vvcam_isp_proc_stat(isp_dev->pde, irq_mis, VVCAM_EID_MAX);
+}
+
+static void vvcam_isp_hal_write_reg(void __iomem *base,
+			uint32_t addr, uint32_t value)
+{
+    writel(value, base + addr);
+}
+
+static int vvcam_isp_hal_read_reg(void __iomem *base, uint32_t addr)
+{
+    return readl(base + addr);
+}
+
+int vvcam_isp_reset(struct vvcam_isp_dev *isp_dev, uint32_t reset)
+{
+    uint32_t reset_reg = 0;
+
+    reset_reg = vvcam_isp_hal_read_reg(isp_dev->reset, 0);
+
+    if (reset) {
+        /* reset */
+        reset_reg &= ~TPG_ISP_RST_MASK;
+    } else {
+        /* unreset */
+        reset_reg |= TPG_ISP_RST_MASK;
+    }
+
+    vvcam_isp_hal_write_reg(isp_dev->reset, 0, reset_reg);
+
+    return 0;
+}
+
+int vvcam_isp_write_reg(struct vvcam_isp_dev *isp_dev, vvcam_isp_reg_t isp_reg)
+{
+	//printk("vvcam_isp_write_reg:base=%p,addr=%x,val=%x\n",isp_dev->base,isp_reg.addr,isp_reg.value);
+	vvcam_isp_hal_write_reg(isp_dev->base, isp_reg.addr, isp_reg.value);
+    return 0;
+}
+
+int vvcam_isp_read_reg(struct vvcam_isp_dev *isp_dev, vvcam_isp_reg_t *isp_reg)
+{
+    uint32_t reg_value = 0;
+
+    reg_value = vvcam_isp_hal_read_reg(isp_dev->base, isp_reg->addr);
+
+    isp_reg->value = reg_value;
+
+    return 0;
+}
+
+
+irqreturn_t vvcam_isp_irq_process(struct vvcam_isp_dev *isp_dev)
+{
+    uint32_t isp_mis = 0;
+#if AHB_WR_WITH_FE
+#else
+    uint32_t isp_fe_ctrl = 0;
+#endif
+    uint64_t timestamp;
+
+    vvcam_event_t event;
+
+    if (!isp_dev->refcnt)
+        return IRQ_NONE;
+#if AHB_WR_WITH_FE
+#else
+    isp_fe_ctrl = vvcam_isp_hal_read_reg(isp_dev->base, ISP_FE_CTL);
+    if ((isp_fe_ctrl & ISP_FE_CFG_SEL_MASK) == ISP_FE_SEL_CMDBUF) {
+        return IRQ_HANDLED;
+    }
+#endif
+
+    // printk("imsc = 0x%08x\n",vvcam_isp_hal_read_reg(isp_dev->base, 0x5bc));
+    // printk("ris = 0x%08x\n", vvcam_isp_hal_read_reg(isp_dev->base, 0x5c0));
+    // printk("mis = 0x%08x\n", vvcam_isp_hal_read_reg(isp_dev->base, 0x5c4));
+    // printk("icr = 0x%08x\n", vvcam_isp_hal_read_reg(isp_dev->base, 0x5c8));
+    // printk("isr = 0x%08x\n", vvcam_isp_hal_read_reg(isp_dev->base, 0x5cc));
+
+    isp_mis = vvcam_isp_hal_read_reg(isp_dev->base, ISP_MIS);
+    if (isp_mis) {
+        // printk("isp_mis mis 0x%08x\n", isp_mis);
+        vvcam_isp_hal_write_reg(isp_dev->base, ISP_ICR, isp_mis);
+    } else {
+        return IRQ_NONE;
+    }
+
+    timestamp = ktime_get_ns();
+
+    if (isp_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_ISP_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = isp_mis;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_ISP_MIS] = isp_mis;
+        tasklet_schedule(&isp_dev->stat_tasklet);
+    }
+
+    return IRQ_HANDLED;
+}
+
+irqreturn_t vvcam_isp_mi_irq_process(struct vvcam_isp_dev *isp_dev)
+{
+    uint32_t miv1_mis  = 0;
+    uint32_t miv2_mis  = 0;
+    uint32_t miv2_mis1 = 0;
+    uint32_t miv2_mis2 = 0;
+    uint32_t miv2_mis3 = 0;
+    uint32_t mi_mis_hdr1 = 0;
+#if AHB_WR_WITH_FE
+#else
+    uint32_t isp_fe_ctrl = 0;
+#endif
+    uint64_t timestamp;
+    uint64_t rdcd_mi_mis = 0;
+    vvcam_event_t event;
+    bool irq = false;
+
+    if (!isp_dev->refcnt)
+        return IRQ_NONE;
+#if AHB_WR_WITH_FE
+#else
+    isp_fe_ctrl = vvcam_isp_hal_read_reg(isp_dev->base, ISP_FE_CTL);
+    if ((isp_fe_ctrl & ISP_FE_CFG_SEL_MASK) == ISP_FE_SEL_CMDBUF) {
+        return IRQ_HANDLED;
+    }
+#endif
+
+#ifdef ISP8000NANO_V2302
+    miv1_mis = vvcam_isp_hal_read_reg(isp_dev->base, MIV1_MIS);
+    if (miv1_mis) {
+        // printk("miv1 mis 0x%08x\n", miv1_mis);
+        vvcam_isp_hal_write_reg(isp_dev->base, MIV1_ICR, miv1_mis);
+        irq = true;
+    }
+#endif
+
+    // printk("mi ris = 0x%0x8\n", vvcam_isp_hal_read_reg(isp_dev->base, 0x16e0));
+
+    miv2_mis = vvcam_isp_hal_read_reg(isp_dev->base, MIV2_MIS);
+    // printk("miv2 mis 0x%08x\n", miv2_mis);    
+    if (miv2_mis) {
+        vvcam_isp_hal_write_reg(isp_dev->base, MIV2_ICR, miv2_mis);
+        irq = true;
+    }
+
+    miv2_mis1 = vvcam_isp_hal_read_reg(isp_dev->base, MIV2_MIS1);
+    if (miv2_mis1) {
+        vvcam_isp_hal_write_reg(isp_dev->base, MIV2_ICR1, miv2_mis1);
+        irq = true;
+    }
+
+    miv2_mis2 = vvcam_isp_hal_read_reg(isp_dev->base, MIV2_MIS2);
+    if (miv2_mis2) {
+        vvcam_isp_hal_write_reg(isp_dev->base, MIV2_ICR2, miv2_mis2);
+        irq = true;
+    }
+
+    miv2_mis3 = vvcam_isp_hal_read_reg(isp_dev->base, MIV2_MIS3);
+    if (miv2_mis3) {
+        vvcam_isp_hal_write_reg(isp_dev->base, MIV2_ICR3, miv2_mis3);
+        irq = true;
+    }
+
+    mi_mis_hdr1 = vvcam_isp_hal_read_reg(isp_dev->base, MI_MIS_HDR1);
+    if (mi_mis_hdr1) {
+        vvcam_isp_hal_write_reg(isp_dev->base, MI_ICR_HDR1, mi_mis_hdr1);
+        irq = true;
+    }
+
+    rdcd_mi_mis = vvcam_isp_hal_read_reg(isp_dev->base, RDCD_MI_MIS);
+    if (rdcd_mi_mis) {
+        vvcam_isp_hal_write_reg(isp_dev->base, RDCD_MI_ICR, rdcd_mi_mis);
+        irq = true;
+    }
+
+    timestamp = ktime_get_ns();
+
+    if (miv2_mis & (MIV2_MIS_MCM_RAW_RADY_MASK | MIV2_MIS_FRAME_END_MASK)) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_RDMA_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis &
+            (MIV2_MIS_MCM_RAW_RADY_MASK | MIV2_MIS_FRAME_END_MASK);
+        miv2_mis &= ~MIV2_MIS_MCM_RAW_RADY_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_RDMA_MIS] = event.irqevent.irq_value;
+    }
+
+    if (miv2_mis & MIV2_MIS_JPD_FRAME_END_MASK) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_JDP;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis & MIV2_MIS_JPD_FRAME_END_MASK;
+        miv2_mis &= ~MIV2_MIS_JPD_FRAME_END_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_MIV2_JDP] = event.irqevent.irq_value;
+    }
+
+    if (miv2_mis & MIV2_MIS_MCM_RAW0_FRAME_END_MASK) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MCM_WR_RAW0_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis & MIV2_MIS_MCM_RAW0_FRAME_END_MASK;
+        miv2_mis &= ~MIV2_MIS_MCM_RAW0_FRAME_END_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_MCM_WR_RAW0_MIS] =  event.irqevent.irq_value;
+    }
+
+    if (miv2_mis & MIV2_MIS_MCM_RAW1_FRAME_END_MASK) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MCM_WR_RAW1_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis & MIV2_MIS_MCM_RAW1_FRAME_END_MASK;
+        miv2_mis &= ~MIV2_MIS_MCM_RAW1_FRAME_END_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_MCM_WR_RAW1_MIS] = event.irqevent.irq_value;
+    }
+
+
+    if (miv2_mis3 & MIV2_MIS3_MCM_G2RAW0_FRAME_END_MASK) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MCM_WR_G2_RAW0_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis3 & MIV2_MIS3_MCM_G2RAW0_FRAME_END_MASK;
+        miv2_mis3 &=  ~MIV2_MIS3_MCM_G2RAW0_FRAME_END_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_MCM_WR_G2_RAW0_MIS] = event.irqevent.irq_value;
+    }
+
+    if (miv2_mis3 & MIV2_MIS3_MCM_G2RAW1_FRAME_END_MASK) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MCM_WR_G2_RAW1_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis3 & MIV2_MIS3_MCM_G2RAW1_FRAME_END_MASK;
+        miv2_mis3 &=  ~MIV2_MIS3_MCM_G2RAW1_FRAME_END_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_MCM_WR_G2_RAW1_MIS] = event.irqevent.irq_value;
+    }
+
+    if (miv1_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv1_mis;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        // printk("MI V1 mi_irq_process.\n");
+    }
+
+    if (miv2_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if (miv2_mis1) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_MIS1;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis1;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if ((miv2_mis2 & MIV2_MIS2_HDR_RDMA_READY_MASK)
+       || (miv2_mis2 & MIV2_MIS2_HDR_FRAME_END_MASK)) {
+
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_MIS2_HDR;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = ((miv2_mis2 & MIV2_MIS2_HDR_RDMA_READY_MASK)
+                                   | (miv2_mis2 & MIV2_MIS2_HDR_FRAME_END_MASK));
+        miv2_mis2 &= ~MIV2_MIS2_HDR_RDMA_READY_MASK;
+        miv2_mis2 &= ~MIV2_MIS2_HDR_FRAME_END_MASK;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if (miv2_mis2) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_MIS2;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis2;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if (miv2_mis3) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MIV2_MIS3;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = miv2_mis3;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if (mi_mis_hdr1) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_MI_MIS_HDR1;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = mi_mis_hdr1;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if (rdcd_mi_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_RDCD_MI_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = rdcd_mi_mis;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+    }
+
+    if (irq) {
+        tasklet_schedule(&isp_dev->stat_tasklet);
+        return IRQ_HANDLED;
+    }
+
+    return IRQ_NONE;
+}
+
+irqreturn_t vvcam_isp_fe_irq_process(struct vvcam_isp_dev *isp_dev)
+{
+    uint32_t isp_fe_mis = 0;
+    uint32_t isp_fe_batch_mode_mis = 0;
+#if AHB_WR_WITH_FE
+#else
+    uint32_t isp_fe_ctrl = 0;
+#endif
+    vvcam_event_t event;
+    uint64_t timestamp;
+    bool irq = false;
+
+    if (!isp_dev->refcnt)
+        return IRQ_NONE;
+
+    isp_fe_mis = vvcam_isp_hal_read_reg(isp_dev->base, ISP_FE_MIS);
+    if (isp_fe_mis) {
+        vvcam_isp_hal_write_reg(isp_dev->base, ISP_FE_ICR, isp_fe_mis);
+#if AHB_WR_WITH_FE
+#else
+        isp_fe_ctrl = vvcam_isp_hal_read_reg(isp_dev->base, ISP_FE_CTL);
+        if ((isp_fe_ctrl & ISP_FE_CFG_SEL_MASK) == ISP_FE_SEL_CMDBUF) {
+            isp_fe_ctrl &= ~(ISP_FE_CFG_SEL_MASK | ISP_FE_AHB_WRITE_MASK);
+            isp_fe_ctrl |= (ISP_FE_SEL_AHBBUF) << ISP_FE_CFG_SEL_SHIFT;
+            isp_fe_ctrl |= (ISP_FE_AHB_WR_ENABLE) << ISP_FE_AHB_WRITE_SHIFT;
+            vvcam_isp_hal_write_reg(isp_dev->base, ISP_FE_CTL, isp_fe_ctrl);
+
+        }
+#endif
+        irq = true;
+    }
+
+    isp_fe_batch_mode_mis = vvcam_isp_hal_read_reg(isp_dev->base, ISP_FE_BATCH_MODE_MIS);
+    if (isp_fe_batch_mode_mis) {
+        vvcam_isp_hal_write_reg(isp_dev->base, ISP_FE_BATCH_MODE_ICR, isp_fe_batch_mode_mis);
+        irq = true;
+    }
+
+    timestamp = ktime_get_ns();
+
+    if (isp_fe_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FE_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = isp_fe_mis;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FE_MIS] = isp_fe_mis;
+    }
+
+    if (isp_fe_batch_mode_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FE_BATCH_MODE_MIS;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = isp_fe_batch_mode_mis;
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FE_BATCH_MODE_MIS] = isp_fe_batch_mode_mis;
+    }
+
+    if (irq) {
+        tasklet_schedule(&isp_dev->stat_tasklet);
+        return IRQ_HANDLED;
+    }
+
+    return IRQ_NONE;
+}
+
+irqreturn_t vvcam_isp_fusa_irq_process(struct vvcam_isp_dev *isp_dev)
+{
+    uint32_t fusa_ecc_mis1 = 0;
+    uint32_t fusa_ecc_mis2 = 0;
+    uint32_t fusa_ecc_mis3 = 0;
+    uint32_t fusa_ecc_mis4 = 0;
+    uint32_t fusa_ecc_mis5 = 0;
+    uint32_t fusa_ecc_mis6 = 0;
+    uint32_t fusa_dup_mis  = 0;
+    uint32_t fusa_parity_mis = 0;
+    uint32_t fusa_lv1_mis1 = 0;
+    uint32_t fusa_rdcd_mis = 0;
+    uint64_t timestamp;
+
+    vvcam_event_t event;
+    bool irq = false;
+
+    if (!isp_dev->refcnt)
+        return IRQ_NONE;
+
+    // printk("enter fusa_irq_process\n");
+
+    fusa_ecc_mis1 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_ECC_MIS1);
+    // printk("FUSA_ECC_MIS1 : 0x%08x \n", fusa_ecc_mis1);
+    if (fusa_ecc_mis1) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_ECC_ICR1, fusa_ecc_mis1);
+        irq = true;
+    }
+
+    fusa_ecc_mis2 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_ECC_MIS2);
+    // printk("FUSA_ECC_MIS2 : 0x%08x \n", fusa_ecc_mis2);
+    if (fusa_ecc_mis2) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_ECC_ICR2, fusa_ecc_mis2);
+        irq = true;
+    }
+
+    fusa_ecc_mis3 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_ECC_MIS3);
+    // printk("FUSA_ECC_MIS3 : 0x%08x \n", fusa_ecc_mis3);
+    if (fusa_ecc_mis3) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_ECC_ICR3, fusa_ecc_mis3);
+        irq = true;
+    }
+
+    fusa_ecc_mis4 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_ECC_MIS4);
+    // printk("FUSA_ECC_MIS4 : 0x%08x \n", fusa_ecc_mis4);
+    if (fusa_ecc_mis4) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_ECC_ICR4, fusa_ecc_mis4);
+        irq = true;
+    }
+
+    fusa_ecc_mis5 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_ECC_MIS5);
+    // printk("FUSA_ECC_MIS5 : 0x%08x \n", fusa_ecc_mis5);
+    if (fusa_ecc_mis5) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_ECC_ICR5, fusa_ecc_mis5);
+        irq = true;
+    }
+
+    fusa_ecc_mis6 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_ECC_MIS6);
+    // printk("FUSA_ECC_MIS6 : 0x%08x \n", fusa_ecc_mis6);
+    if (fusa_ecc_mis6) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_ECC_ICR6, fusa_ecc_mis6);
+        irq = true;
+    }
+
+    fusa_dup_mis = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_DUP_MIS);
+    // printk("FUSA_DUP_MIS : 0x%08x \n", fusa_dup_mis);
+    if (fusa_dup_mis) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_DUP_ICR, fusa_dup_mis);
+        irq = true;
+    }
+
+    fusa_parity_mis = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_PARITY_MIS);
+    // printk("FUSA_PARITY_MIS : 0x%08x \n", fusa_parity_mis);
+    if (fusa_parity_mis) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_PARITY_ICR, fusa_parity_mis);
+        irq = true;
+    }
+
+    fusa_lv1_mis1 = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_LV1_MIS1);
+    // printk("FUSA_LV1_MIS1 : 0x%08x \n", fusa_lv1_mis1);
+    if (fusa_lv1_mis1) {
+        vvcam_isp_hal_write_reg(isp_dev->base, FUSA_LV1_ICR1, fusa_lv1_mis1);
+        irq = true;
+    }
+
+    if (rdc_enable == true) {
+        fusa_rdcd_mis = vvcam_isp_hal_read_reg(isp_dev->base, FUSA_RDCD_MIS);
+        // printk("FUSA_RDCD_MIS : 0x%08x \n", fusa_rdcd_mis);
+        if (fusa_rdcd_mis) {
+            vvcam_isp_hal_write_reg(isp_dev->base, FUSA_RDCD_ICR, fusa_rdcd_mis);
+            irq = true;
+        }
+    }
+
+    timestamp = ktime_get_ns();
+
+    if (fusa_ecc_mis1) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_ECC_IMSC1;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_ecc_mis1;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_ECC_IMSC1] = event.irqevent.irq_value;
+    }
+
+    if (fusa_ecc_mis2) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_ECC_IMSC2;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_ecc_mis2;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_ECC_IMSC2] = event.irqevent.irq_value;
+    }
+
+    if (fusa_ecc_mis3) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_ECC_IMSC3;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_ecc_mis3;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_ECC_IMSC3] = event.irqevent.irq_value;
+    }
+
+    if (fusa_ecc_mis4) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_ECC_IMSC4;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_ecc_mis4;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_ECC_IMSC4] = event.irqevent.irq_value;
+    }
+
+    if (fusa_ecc_mis5) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_ECC_IMSC5;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_ecc_mis5;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_ECC_IMSC5] = event.irqevent.irq_value;
+    }
+
+    if (fusa_ecc_mis6) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_ECC_IMSC6;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_ecc_mis6;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_ECC_IMSC6] = event.irqevent.irq_value;
+    }
+
+    if (fusa_dup_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_DUP_IMSC;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_dup_mis;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_DUP_IMSC] = event.irqevent.irq_value;
+    }
+
+    if (fusa_parity_mis) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_PARITY_IMSC;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_parity_mis;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_PARITY_IMSC] = event.irqevent.irq_value;
+    }
+
+    if (fusa_lv1_mis1) {
+        event.type = VVCAM_EVENT_IRQ_TYPE;
+        event.id   = VVCAM_EID_FUSA_LV1_IMSC1;
+        event.timestamp = timestamp;
+        event.irqevent.irq_value = fusa_lv1_mis1;
+
+        vvcam_event_queue(&isp_dev->event_dev, &event);
+        isp_dev->irq_mis[VVCAM_EID_FUSA_LV1_IMSC1] = event.irqevent.irq_value;
+    }
+
+    if (rdc_enable == true) {
+        if (fusa_rdcd_mis) {
+            event.type = VVCAM_EVENT_IRQ_TYPE;
+            event.id   = VVCAM_EID_FUSA_RDCD_MIS;
+            event.timestamp = timestamp;
+            event.irqevent.irq_value = fusa_rdcd_mis;
+
+            vvcam_event_queue(&isp_dev->event_dev, &event);
+            isp_dev->irq_mis[VVCAM_EID_FUSA_RDCD_MIS] = event.irqevent.irq_value;
+        }
+    }
+
+    if (irq) {
+        tasklet_schedule(&isp_dev->stat_tasklet);
+        return IRQ_HANDLED;
+    }
+
+    return IRQ_HANDLED;
+}
+
+module_param(rdc_enable, bool, 0644);
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_hal.h b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_hal.h
new file mode 100644
index 000000000000..f3adfc7cb39b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_hal.h
@@ -0,0 +1,173 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HAL_H__
+#define __VVCAM_ISP_HAL_H__
+
+#define TPG_ISP_RST_SHIFT                    24
+#define TPG_ISP_RST_MASK                     0x01000000
+
+#define ISP_MIS                              0x000005c4
+#define ISP_ICR                              0x000005c8
+
+#define MIV1_CTRL                            0x00001400
+#define MIV1_MIS                             0x00001500
+#define MIV1_ICR                             0x00001504
+
+#define MIV2_CTRL                            0x00001300
+#define MIV2_MIS                             0x000016d0
+#define MIV2_MIS1                            0x000016d4
+#define MIV2_ICR                             0x000016d8
+#define MIV2_ICR1                            0x000016dc
+#define MIV2_MIS2                            0x000016f0
+#define MIV2_ICR2                            0x000016f4
+#define MIV2_MIS3                            0x000056d8
+#define MIV2_ICR3                            0x000056dc
+#define MI_MIS_HDR1                          0x000072c8
+#define MI_ICR_HDR1                          0x000072cc
+
+#define ISP_FE_MIS                           0x00003D74
+#define ISP_FE_ICR                           0x00003D78
+#define ISP_FE_CTL                           0x00003D60
+#define ISP_FE_BATCH_MODE_ICR                0x00003DAC
+#define ISP_FE_BATCH_MODE_MIS                0x00003DB8
+
+
+#define MIV1_MIS_FRAME_END_MASK              0x00000001
+
+#define MIV2_CTRL_MCM_RAW_RDMA_START_MASK    0x00008000
+
+#define MIV2_MIS_MCM_RAW_RADY_MASK           0x01000000
+#define MIV2_MIS_MCM_RAW1_FRAME_END_MASK     0x00000080
+#define MIV2_MIS_MCM_RAW0_FRAME_END_MASK     0x00000040
+#define MIV2_MIS_MCM_RAW_FRAME_END_MASK      0x000000c0
+#define MIV2_MIS_FRAME_END_MASK              0x0000003f
+#define MIV2_MIS_JPD_FRAME_END_MASK          0x00000004U
+
+#define MIV2_MIS2_HDR_RDMA_READY_MASK        0x20000700
+#define MIV2_MIS2_HDR_FRAME_END_MASK         0x0800001c
+
+#define MIV2_MIS3_MCM_G2RAW1_FRAME_END_MASK  0x00008000
+#define MIV2_MIS3_MCM_G2RAW0_FRAME_END_MASK  0x00004000
+#define MIV2_MIS3_MCM_G2RAW_FRAME_END_MASK   0x0000c000
+
+#define MIV2_MIS_MP_FRAME_END_MASK           0x00000003
+#define MIV2_MIS_SP_FRAME_END_MASK           0x00000008
+#define MIV2_MIS_SP2_FRAME_END_MASK          0x00000030
+
+#define ISP_MIS_FRAME_IN_MASK                0x00000020
+#define ISP_MIS_FRAME_OUT_MASK               0x00000002
+
+#define ISP_FE_CFG_SEL_MASK                  0x00000001
+#define ISP_FE_CFG_SEL_SHIFT                 0
+#define ISP_FE_AHB_WRITE_MASK                0x00000002
+#define ISP_FE_AHB_WRITE_SHIFT               1
+
+#define FUSA_ECC_IMSC1                       0x00005b54
+#define FUSA_ECC_RIS1                        0x00005b58
+#define FUSA_ECC_MIS1                        0x00005b5c
+#define FUSA_ECC_ICR1                        0x00005b64
+#define FUSA_ECC_IMSC2                       0x00005b68
+#define FUSA_ECC_RIS2                        0x00005b6C
+#define FUSA_ECC_MIS2                        0x00005b70
+#define FUSA_ECC_ICR2                        0x00005b78
+#define FUSA_ECC_IMSC3                       0x00005b7c
+#define FUSA_ECC_RIS3                        0x00005b80
+#define FUSA_ECC_MIS3                        0x00005b84
+#define FUSA_ECC_ICR3                        0x00005b8c
+#define FUSA_ECC_IMSC4                       0x00005b90
+#define FUSA_ECC_RIS4                        0x00005b94
+#define FUSA_ECC_MIS4                        0x00005b98
+#define FUSA_ECC_ICR4                        0x00005ba0
+#define FUSA_ECC_IMSC5                       0x00005ba4
+#define FUSA_ECC_RIS5                        0x00005ba8
+#define FUSA_ECC_MIS5                        0x00005bac
+#define FUSA_ECC_ICR5                        0x00005bb4
+#define FUSA_ECC_IMSC6                       0x00005c44
+#define FUSA_ECC_RIS6                        0x00005c48
+#define FUSA_ECC_MIS6                        0x00005c4c
+#define FUSA_ECC_ICR6                        0x00005c54
+
+#define FUSA_DUP_IMSC                        0x00005bb8
+#define FUSA_DUP_RIS                         0x00005bbc
+#define FUSA_DUP_MIS                         0x00005bc0
+#define FUSA_DUP_ICR                         0x00005bc8
+#define FUSA_PARITY_IMSC                     0x00005bcc
+#define FUSA_PARITY_RIS                      0x00005bd0
+#define FUSA_PARITY_MIS                      0x00005bd4
+#define FUSA_PARITY_ICR                      0x00005bdc
+#define FUSA_LV1_IMSC1                       0x00005b40
+#define FUSA_LV1_RIS1                        0x00005b44
+#define FUSA_LV1_MIS1                        0x00005b48
+#define FUSA_LV1_ICR1                        0x00005b50
+
+
+#define RDCD_MI_MIS                          0x000080c0
+#define RDCD_MI_ICR                          0x000080c4
+
+#define FUSA_RDCD_MIS                        0x00000bec
+#define FUSA_RDCD_ICR                        0x00000bf4
+
+enum isp_fe_cfg_sel_e {
+    ISP_FE_SEL_AHBBUF = 0,
+    ISP_FE_SEL_CMDBUF = 1,
+};
+
+enum isp_fe_ahb_wr_en_e {
+    ISP_FE_AHB_WR_DISABLE = 0,
+    ISP_FE_AHB_WR_ENABLE  = 1,
+};
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_platform.c b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_platform.c
new file mode 100644
index 000000000000..144f7997155f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_platform.c
@@ -0,0 +1,249 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+
+#include "vvcam_isp_platform.h"
+
+static int devices_mask = 0x01;
+MODULE_PARM_DESC(devices_mask, "vvcam isp devices mask");
+
+static void vvcam_isp_pdev_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+	return;
+}
+
+static struct resource vvcam_isp0_resource[] = {
+    [0] = {
+        .start = VVCAM_ISP0_BASE + PCIE_BASE,
+        .end   = VVCAM_ISP0_BASE + PCIE_BASE + VVCAM_ISP_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+    [1] = {
+        .start = VVCAM_TPG0 + PCIE_BASE,
+        .end   = VVCAM_TPG0 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+	},
+    [2] = {
+        .start = VVCAM_ISP0_IRQ,
+        .end   = VVCAM_ISP0_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [3] = {
+        .start = VVCAM_ISP0_MI_IRQ,
+        .end   = VVCAM_ISP0_MI_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [4] = {
+        .start = VVCAM_ISP0_FE_IRQ,
+        .end   = VVCAM_ISP0_FE_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [5] = {
+        .start = VVCAM_ISP0_FUSA_IRQ,
+        .end   = VVCAM_ISP0_FUSA_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+};
+
+static struct platform_device vvcam_isp0_pdev = {
+	.name          = "vvcam-isp",
+	.id            = 0,
+	.resource      = vvcam_isp0_resource,
+    .num_resources = ARRAY_SIZE(vvcam_isp0_resource),
+	.dev.release   = vvcam_isp_pdev_release,
+};
+
+
+static struct resource vvcam_isp1_resource[] = {
+    [0] = {
+        .start = VVCAM_ISP1_BASE + PCIE_BASE,
+        .end   = VVCAM_ISP1_BASE + PCIE_BASE + VVCAM_ISP_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+    [1] = {
+        .start = VVCAM_TPG1 + PCIE_BASE,
+        .end   = VVCAM_TPG1 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+	},
+    [2] = {
+        .start = VVCAM_ISP1_IRQ,
+        .end   = VVCAM_ISP1_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [3] = {
+        .start = VVCAM_ISP1_MI_IRQ,
+        .end   = VVCAM_ISP1_MI_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [4] = {
+        .start = VVCAM_ISP1_FE_IRQ,
+        .end   = VVCAM_ISP1_FE_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+        [5] = {
+        .start = VVCAM_ISP1_FUSA_IRQ,
+        .end   = VVCAM_ISP1_FUSA_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+};
+
+static struct platform_device vvcam_isp1_pdev = {
+	.name          = "vvcam-isp",
+	.id            = 1,
+	.resource      = vvcam_isp1_resource,
+    .num_resources = ARRAY_SIZE(vvcam_isp1_resource),
+	.dev.release   = vvcam_isp_pdev_release,
+};
+
+static struct resource vvcam_isp2_resource[] = {
+    [0] = {
+        .start = VVCAM_ISP2_BASE + PCIE_BASE,
+        .end   = VVCAM_ISP2_BASE + PCIE_BASE + VVCAM_ISP_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+    [1] = {
+        .start = VVCAM_TPG2 + PCIE_BASE,
+        .end   = VVCAM_TPG2 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+	},
+    [2] = {
+        .start = VVCAM_ISP2_IRQ,
+        .end   = VVCAM_ISP2_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [3] = {
+        .start = VVCAM_ISP2_MI_IRQ,
+        .end   = VVCAM_ISP2_MI_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+    [4] = {
+        .start = VVCAM_ISP2_FE_IRQ,
+        .end   = VVCAM_ISP2_FE_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+        [5] = {
+        .start = VVCAM_ISP2_FUSA_IRQ,
+        .end   = VVCAM_ISP2_FUSA_IRQ,
+        .flags = IORESOURCE_IRQ,
+	},
+};
+
+static struct platform_device vvcam_isp2_pdev = {
+	.name          = "vvcam-isp",
+	.id            = 2,
+	.resource      = vvcam_isp2_resource,
+    .num_resources = ARRAY_SIZE(vvcam_isp2_resource),
+	.dev.release   = vvcam_isp_pdev_release,
+};
+
+int vvcam_isp_platform_device_register(void)
+{
+    int ret = 0;
+    int devs_mask = 0;
+
+    if (devices_mask & 0x01) {
+        ret = platform_device_register(&vvcam_isp0_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x01;
+    }
+
+    if (devices_mask & 0x02) {
+        ret = platform_device_register(&vvcam_isp1_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x02;
+    }
+
+    if (devices_mask & 0x04) {
+        ret = platform_device_register(&vvcam_isp2_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x04;
+    }
+
+    return ret;
+
+err_register:
+    if (devs_mask & 0x01)
+		platform_device_unregister(&vvcam_isp0_pdev);
+	if (devs_mask & 0x02)
+		platform_device_unregister(&vvcam_isp1_pdev);
+	if (devs_mask & 0x04)
+		platform_device_unregister(&vvcam_isp2_pdev);
+
+    return ret;
+}
+
+void vvcam_isp_platform_device_unregister(void)
+{
+    if (devices_mask & 0x01)
+		platform_device_unregister(&vvcam_isp0_pdev);
+	if (devices_mask & 0x02)
+		platform_device_unregister(&vvcam_isp1_pdev);
+	if (devices_mask & 0x04)
+		platform_device_unregister(&vvcam_isp2_pdev);
+
+    return;
+}
+
+module_param(devices_mask, int, 0644);
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_platform.h b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_platform.h
new file mode 100644
index 000000000000..247a57fad731
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_PLATFORM_H__
+#define __VVCAM_ISP_PLATFORM_H__
+
+int vvcam_isp_platform_device_register(void);
+void vvcam_isp_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_procfs.c b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_procfs.c
new file mode 100644
index 000000000000..51ba3797e443
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_procfs.c
@@ -0,0 +1,718 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/i2c.h>
+#include <linux/ctype.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_procfs.h"
+#include "vvcam_isp_hal.h"
+
+#define VVCAM_ISP_PROCFS_BUF_SIZE    256
+
+#define PROC_OUT_OF_RANGE       0x01
+#define PROC_PARSE_ERROR        0x02
+
+#define PROC_PRINT_STAT(stat, sfile, name) \
+    seq_printf(sfile, "|%-32s|%-16d|\n", #name, stat.name); \
+    seq_printf(sfile, "|--------------------------------|----------------|\n");
+
+
+enum {
+    PROC_CMD_RW,
+    PROC_CMD_CLEAR
+};
+
+struct rw_cmd {
+    bool is_write;
+    int8_t flags;
+    uint32_t start_addr;
+    uint8_t num_elems;
+    uint32_t *rw_buffer;
+    struct list_head list;
+};
+
+struct vvcam_isp_irq_stat {
+    uint32_t isp_frame_in_cnt;
+    uint32_t isp_frame_out_cnt;
+
+    uint32_t mp_frame_out_cnt;
+    uint32_t sp_frame_out_cnt;
+    uint32_t sp2_frame_out_cnt;
+
+    uint32_t mcm_raw0_frame_cnt;
+    uint32_t mcm_raw1_frame_cnt;
+    uint32_t mcm_g2_raw0_frame_cnt;
+    uint32_t mcm_g2_raw1_frame_cnt;
+    uint32_t mcm_rdma_frame_cnt;
+
+    uint32_t fe_irq_cnt;
+
+    uint32_t fusa_ecc1_cnt;
+    uint32_t fusa_ecc2_cnt;
+    uint32_t fusa_ecc3_cnt;
+    uint32_t fusa_ecc4_cnt;
+    uint32_t fusa_ecc5_cnt;
+    uint32_t fusa_ecc6_cnt;
+    uint32_t fusa_dup_cnt;
+    uint32_t fusa_parity_cnt;
+    uint32_t fusa_lv1_cnt;
+
+    unsigned long long start;
+    unsigned long long end;
+};
+
+struct vvcam_isp_status {
+    int8_t flags;
+    int8_t type;
+    struct vvcam_isp_irq_stat  stat;
+    struct list_head cmd_list;
+};
+
+struct vvcam_isp_procfs {
+    struct proc_dir_entry *pde;
+    struct vvcam_isp_dev *isp_dev;
+    struct mutex lock;
+    struct vvcam_isp_status status;
+};
+
+void vvcam_isp_proc_stat(unsigned long pde,
+                    const uint32_t *irq_mis, const int len)
+{
+    struct vvcam_isp_procfs *isp_proc = (struct vvcam_isp_procfs*)pde;
+    struct vvcam_isp_irq_stat *stat = &isp_proc->status.stat;
+    int eid = 0;
+
+    //mutex_lock(&isp_proc->lock);
+    for (eid = 0; eid < len; eid++) {
+        if (!irq_mis[eid])
+            continue;
+
+        switch (eid)
+        {
+        case VVCAM_EID_ISP_MIS:
+            if (irq_mis[eid] & ISP_MIS_FRAME_IN_MASK)
+                stat->isp_frame_in_cnt++;
+
+            if (irq_mis[eid] & ISP_MIS_FRAME_OUT_MASK)
+                stat->isp_frame_out_cnt++;
+
+            break;
+
+        case VVCAM_EID_MIV2_MIS:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_MIV2_MIS1:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_MIV2_MIS2:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_MIV2_MIS3:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_RDMA_MIS:
+            if (irq_mis[eid] & MIV2_MIS_MCM_RAW_RADY_MASK)
+                stat->mcm_rdma_frame_cnt++;
+
+            if (irq_mis[eid] & MIV2_MIS_MP_FRAME_END_MASK)
+                stat->mp_frame_out_cnt++;
+
+            if (irq_mis[eid] & MIV2_MIS_SP_FRAME_END_MASK)
+                stat->sp_frame_out_cnt++;
+
+            if (irq_mis[eid] & MIV2_MIS_SP2_FRAME_END_MASK)
+                stat->sp2_frame_out_cnt++;
+
+            break;
+
+        case VVCAM_EID_MCM_WR_RAW0_MIS:
+            stat->mcm_raw0_frame_cnt++;
+            break;
+
+        case VVCAM_EID_MCM_WR_RAW1_MIS:
+            stat->mcm_raw1_frame_cnt++;
+            break;
+
+        case VVCAM_EID_MCM_WR_G2_RAW0_MIS:
+            stat->mcm_g2_raw0_frame_cnt++;
+            break;
+
+        case VVCAM_EID_MCM_WR_G2_RAW1_MIS:
+            stat->mcm_g2_raw1_frame_cnt++;
+            break;
+
+        case VVCAM_EID_FE_MIS:
+            stat->fe_irq_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_ECC_IMSC1:
+            stat->fusa_ecc1_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_ECC_IMSC2:
+            stat->fusa_ecc2_cnt++;
+            break;
+        
+        case VVCAM_EID_FUSA_ECC_IMSC3:
+            stat->fusa_ecc3_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_ECC_IMSC4:
+            stat->fusa_ecc4_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_ECC_IMSC5:
+            stat->fusa_ecc5_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_ECC_IMSC6:
+            stat->fusa_ecc6_cnt++;
+            break;
+        
+        case VVCAM_EID_FUSA_DUP_IMSC:
+            stat->fusa_dup_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_PARITY_IMSC:
+            stat->fusa_parity_cnt++;
+            break;
+
+        case VVCAM_EID_FUSA_LV1_IMSC1:
+            stat->fusa_lv1_cnt++;
+            break;
+        }
+    }
+    //mutex_unlock(&isp_proc->lock);
+}
+
+
+static void release_cmd_list(struct list_head *cmd_list)
+{
+    struct rw_cmd *cmd = NULL, *old = NULL;
+
+    if (!list_empty( cmd_list)) {
+        list_for_each_entry(cmd, cmd_list, list) {
+            if (old) {
+                list_del(&old->list);
+                kfree(old);
+                old = NULL;
+            }
+            if (cmd->rw_buffer)
+                kfree(cmd->rw_buffer);
+            old = cmd;
+        }
+
+        if (old) {
+            list_del(&old->list);
+            kfree(old);
+            old = NULL;
+        }
+    }
+}
+
+static int vvcam_isp_procfs_info_show(struct seq_file *sfile, void *offset)
+{
+	struct vvcam_isp_procfs *isp_proc;
+    struct vvcam_isp_dev *isp_dev;
+    struct vvcam_isp_status *status;
+    struct rw_cmd *cmd = NULL;
+    uint8_t i = 0;
+
+	isp_proc = (struct vvcam_isp_procfs *) sfile->private;
+
+    mutex_lock(&isp_proc->lock);
+    isp_dev = isp_proc->isp_dev;
+    status = &isp_proc->status;
+
+    if (status->flags & PROC_PARSE_ERROR) {
+        seq_printf(sfile, "please check input!\n");
+        seq_printf(sfile,
+        "Usage: echo [r|w][num_of_32bit_value]@[addr] [values]>[proc path]\n");
+        seq_printf(sfile,
+        "eg: echo w2@0x00005400 0x00412745 0x80252941 r2>/proc/vsi/vsiisp0 && cat /proc/vsi/vsiisp0\n");
+        status->flags = 0;
+        mutex_unlock(&isp_proc->lock);
+        return 0;
+    }
+
+    if (!list_empty(&status->cmd_list)) {
+        list_for_each_entry(cmd, &status->cmd_list, list) {
+            if (cmd->is_write) {
+                if (cmd->flags & PROC_OUT_OF_RANGE) {
+                    seq_printf(sfile, "write addr %08x ~ %08x out of range.\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                } else {
+                    seq_printf(sfile, "write addr %08x ~ %08x successed.\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                }
+            } else {
+                if (cmd->flags & PROC_OUT_OF_RANGE) {
+                    seq_printf(sfile, "read addr %08x ~ %08x out of range.\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                } else {
+                    seq_printf(sfile, "read addr %08x ~ %08x successed:\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                    for (i = 0; i < cmd->num_elems; i++) {
+                        seq_printf(sfile, "addr:0x%08x value:0x%08x\n",
+                        cmd->start_addr +
+                        (uint32_t)sizeof(cmd->start_addr) * i,
+                        cmd->rw_buffer[i]);
+                    }
+                }
+            }
+            cmd->flags = 0;
+        }
+        release_cmd_list(&status->cmd_list);
+        status->flags = 0;
+    } else {
+        status->stat.end = ktime_get_ns();
+        seq_printf(sfile,
+        "/***statistic for %s time(ns):%lld)***/\n",
+        isp_dev->miscdev.name, status->stat.end - status->stat.start);
+
+    seq_printf(sfile, "|--------------------------------|----------------|\n");
+        seq_printf(sfile, "|%-32s|%-16s|\n",
+        "              Name", "     Number"); \
+    seq_printf(sfile, "|--------------------------------|----------------|\n");
+        PROC_PRINT_STAT(status->stat, sfile, isp_frame_in_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, isp_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mp_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, sp_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, sp2_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_raw0_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_raw1_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_g2_raw0_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_g2_raw1_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_rdma_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fe_irq_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_ecc1_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_ecc2_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_ecc3_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_ecc4_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_ecc5_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_ecc6_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_dup_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_parity_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fusa_lv1_cnt);
+    }
+    mutex_unlock(&isp_proc->lock);
+
+	return 0;
+}
+
+static int vvcam_isp_procfs_open(struct inode *inode, struct file *file)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	return single_open(file, vvcam_isp_procfs_info_show, PDE_DATA(inode));
+#else
+	return single_open(file, vvcam_isp_procfs_info_show, pde_data(inode));
+#endif
+}
+
+static char *vvcam_isp_next_arg(char *str_buf, char **param)
+{
+    int i = 0;
+    for (; str_buf[i]; i++) {
+        if (isspace(str_buf[i]))
+            break;
+    }
+    *param = str_buf;
+
+    if (str_buf[i]) {
+        str_buf[i] = '\0';
+        str_buf += i + 1;
+    } else
+        str_buf += i;
+
+    return skip_spaces(str_buf);
+}
+
+static int vvcam_isp_proc_parse_args(char *str_buf,
+                struct vvcam_isp_status *status)
+{
+    char *param, *end = NULL;
+    int32_t address = -1;
+    uint8_t read_write = 0;
+    uint8_t num_elems = 0;
+    int8_t index= 0;
+    struct rw_cmd *cmd = NULL;
+
+    if ((strlen(str_buf) == 1) || (strcmp(str_buf, "clear\n") == 0)) {
+        status->type = PROC_CMD_CLEAR;
+        return 0;
+    }
+
+    status->type = PROC_CMD_RW;
+    while (*str_buf) {
+        param = NULL;
+        str_buf = vvcam_isp_next_arg(str_buf, &param);
+        if ((*param == 'r') || (*param == 'w')) {
+            if (index != 0)
+                goto error_isp_proc_cmd;
+
+            read_write = *param;
+            num_elems = (uint8_t)simple_strtoul((param + 1), &end, 0);
+            if (num_elems != 0)
+                param = end;
+            else
+                num_elems = 1;
+
+            if (*end == '@') {
+                address = (uint32_t)simple_strtoul(param + 1, &end, 0);
+                if (((end - param - 1) > 10) || ((end - param - 1) <= 6)) {
+                    goto error_isp_proc_cmd;
+                }
+            }
+
+            if (*end != '\0')
+                goto error_isp_proc_cmd;
+
+            if (address < 0) {
+                goto error_isp_proc_cmd;
+            }
+
+            cmd = kzalloc(sizeof(struct rw_cmd), GFP_KERNEL);
+            if (!cmd)
+                goto error_isp_proc_cmd;
+
+            cmd->is_write = (read_write == 'w' ? true : false);
+            cmd->start_addr = address;
+            cmd->num_elems = num_elems;
+            cmd->rw_buffer =
+                kzalloc(sizeof(int32_t) * cmd->num_elems, GFP_KERNEL);
+            if (!cmd->rw_buffer)
+                goto error_isp_proc_cmd;
+
+            list_add_tail(&cmd->list, &status->cmd_list);
+            if (read_write == 'w')
+                index = num_elems;
+            continue;
+        }
+
+         if (*param != '\0' && read_write != 0) {
+            if (read_write == 'r')
+                goto error_isp_proc_cmd;
+
+            if (index <= 0)
+                goto error_isp_proc_cmd;
+
+            cmd->rw_buffer[num_elems - index] =
+                                (uint32_t)simple_strtoul(param, &end, 0);
+
+            if ((*end != '\0') || ((end - param) > 10) ||
+                ((end - param) <= 6))
+                goto error_isp_proc_cmd;
+
+            index--;
+        }
+    }
+    return 0;
+
+error_isp_proc_cmd:
+    release_cmd_list(&status->cmd_list);
+    return -EFAULT;
+}
+
+static int32_t vvcam_isp_proc_process(struct seq_file *sfile,
+                        struct vvcam_isp_procfs *isp_proc, char *str_buf)
+{
+    struct vvcam_isp_status *status;
+    vvcam_isp_reg_t reg;
+    struct rw_cmd *cmd = NULL;
+    int32_t ret = 0;
+    uint8_t i = 0;
+
+    status = &isp_proc->status;
+
+    mutex_lock(&isp_proc->lock);
+    if (!list_empty(&status->cmd_list)) {
+        release_cmd_list(&status->cmd_list);
+    }
+
+    status->flags = 0;
+    status->type = -1;
+
+    ret = vvcam_isp_proc_parse_args(str_buf, status);
+    if (ret) {
+        status->flags = PROC_PARSE_ERROR;
+        mutex_unlock(&isp_proc->lock);
+        return ret;
+    }
+
+    if (status->type == PROC_CMD_CLEAR) {
+        memset(&status->stat, 0, sizeof(status->stat));
+        status->stat.start = ktime_get_ns();
+    } else {
+        mutex_lock(&isp_proc->isp_dev->mlock);
+        list_for_each_entry(cmd, &status->cmd_list, list) {
+            for (i = 0; i < cmd->num_elems; i++) {
+                reg.addr = cmd->start_addr + sizeof(cmd->start_addr) * i;
+                if (reg.addr >= isp_proc->isp_dev->regs_size) {
+                    cmd->flags |= PROC_OUT_OF_RANGE;
+                    continue;
+                }
+                if (cmd->is_write) {
+                    reg.value = cmd->rw_buffer[i];
+                    ret = vvcam_isp_write_reg(isp_proc->isp_dev, reg);
+                } else {
+                    ret = vvcam_isp_read_reg(isp_proc->isp_dev, &reg);
+                    cmd->rw_buffer[i] = reg.value;
+                }
+            }
+        }
+        mutex_unlock(&isp_proc->isp_dev->mlock);
+    }
+
+    mutex_unlock(&isp_proc->lock);
+
+    return 0;
+}
+
+static ssize_t vvcam_isp_procfs_write(struct file *file,
+		const char __user *buffer, size_t count, loff_t *ppos)
+{
+    struct vvcam_isp_procfs *isp_proc;
+    struct seq_file *sfile;
+    char *str_buf;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	isp_proc = (struct vvcam_isp_procfs *) PDE_DATA(file_inode(file));
+#else
+	isp_proc = (struct vvcam_isp_procfs *) pde_data(file_inode(file));
+#endif
+    sfile = file->private_data;
+
+    if (count > VVCAM_ISP_PROCFS_BUF_SIZE)
+        count = VVCAM_ISP_PROCFS_BUF_SIZE - 1;
+
+    str_buf = (char *)kzalloc(count, GFP_KERNEL);
+    if (!str_buf)
+        return -ENOMEM;
+
+    if (copy_from_user(str_buf, buffer, count))
+        return -EFAULT;
+
+    *(str_buf + count) = '\0';
+
+    vvcam_isp_proc_process(sfile, isp_proc, str_buf);
+
+    kfree(str_buf);
+
+    return count;
+}
+
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
+static const struct file_operations vvcam_isp_procfs_ops = {
+    .open = vvcam_isp_procfs_open,
+    .release = seq_release,
+    .read = seq_read,
+    .write = vvcam_isp_procfs_write,
+    .llseek = seq_lseek,
+};
+#else
+static const struct proc_ops vvcam_isp_procfs_ops = {
+	.proc_open = vvcam_isp_procfs_open,
+	.proc_release = seq_release,
+	.proc_read = seq_read,
+	.proc_write = vvcam_isp_procfs_write,
+	.proc_lseek = seq_lseek,
+};
+#endif
+
+struct finddir_callback {
+    struct dir_context ctx;
+    const char *name;
+    int32_t files_cnt;
+    bool found;
+};
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)
+static int readdir_callback(struct dir_context *ctx, const char *name,
+    int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return 0;
+
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return 0;
+}
+
+#else
+static bool readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return true;
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return true;
+}
+#endif
+
+
+static int find_proc_dir_by_name(const char *root,
+                const char *name, bool *found, int32_t *files_cnt) {
+    struct file *pfile;
+    int ret = 0;
+    struct finddir_callback fc = {
+        .ctx.actor = readdir_callback,
+        .name = name,
+        .found = false,
+        .files_cnt = -2,
+    };
+
+    pfile = filp_open(root, O_RDONLY | O_DIRECTORY, 0);
+    if(IS_ERR(pfile)) {
+        pr_err("Failed to open %s\n", root);
+        return -1;
+    }
+
+    if (pfile->f_op->iterate_shared) {
+        ret = pfile->f_op->iterate_shared(pfile, &fc.ctx);
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 2, 0)
+    } else {
+        ret = pfile->f_op->iterate(pfile, &fc.ctx);
+#endif
+    }
+
+    if (ret == 0) {
+        *found = fc.found;
+    }
+
+    if (files_cnt != NULL) {
+        *files_cnt = fc.files_cnt;
+    }
+
+    filp_close(pfile, NULL);
+    return ret;
+}
+
+int vvcam_isp_procfs_register(struct vvcam_isp_dev *isp_dev, unsigned long *pde)
+{
+    struct vvcam_isp_procfs *isp_proc;
+    char isp_proc_name[32];
+    int ret = 0;
+    bool found = false;
+
+    if (!isp_dev)
+        return -1;
+    sprintf(isp_proc_name, "vsi/isp%d", isp_dev->id);
+
+    isp_proc = devm_kzalloc(isp_dev->dev,
+                        sizeof(struct vvcam_isp_procfs), GFP_KERNEL);
+
+    if (!isp_proc)
+        return -ENOMEM;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (!found)
+            proc_mkdir("vsi", NULL);
+    } else {
+        return -EFAULT;
+    }
+
+    isp_proc->isp_dev = isp_dev;
+    isp_proc->pde = proc_create_data(isp_proc_name, 0666, NULL,
+                                &vvcam_isp_procfs_ops, isp_proc);
+    if (!isp_proc->pde)
+        return -EFAULT;
+    *pde = (unsigned long)&isp_proc->pde;
+
+    INIT_LIST_HEAD(&isp_proc->status.cmd_list);
+    mutex_init(&(isp_proc->lock));
+	return 0;
+}
+
+void vvcam_isp_procfs_unregister(unsigned long pde)
+{
+    int ret = 0;
+    bool found = false;
+    int32_t files_cnt;
+    struct vvcam_isp_procfs *isp_proc = (struct vvcam_isp_procfs*)pde;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+         if (found) {
+            proc_remove(isp_proc->pde);
+            ret = find_proc_dir_by_name("/proc/vsi", "", &found, &files_cnt);
+            if (files_cnt == 0) {
+                remove_proc_subtree("vsi", NULL);
+            }
+        }
+    }
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_procfs.h b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_procfs.h
new file mode 100644
index 000000000000..190bfc9a0a77
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp/vvcam_isp_procfs.h
@@ -0,0 +1,63 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_PROCFS_H__
+#define __VVCAM_ISP_PROCFS_H__
+
+int vvcam_isp_procfs_register(struct vvcam_isp_dev *isp_dev,
+                            unsigned long *pde);
+void vvcam_isp_procfs_unregister(unsigned long pde);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/isp_version/ISP8000_V2107.mk b/drivers/media/platform/eswin/isp/vvcam/isp_version/ISP8000_V2107.mk
new file mode 100644
index 000000000000..8b6f1050d179
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/isp_version/ISP8000_V2107.mk
@@ -0,0 +1,62 @@
+ISP_AE_V3 := 1
+EXTRA_CFLAGS += -DISP_AE_V3
+
+ISP_AWB_V3 := 1
+EXTRA_CFLAGS += -DISP_AWB_V3
+
+ISP_AF_V3 := 1
+EXTRA_CFLAGS += -DISP_AF_V3
+
+ISP_BLS := 1
+EXTRA_CFLAGS += -DISP_BLS
+
+ISP_CCM := 1
+EXTRA_CFLAGS += -DISP_CCM
+
+ISP_DPCC := 1
+EXTRA_CFLAGS += -DISP_DPCC
+
+ISP_GE := 1
+EXTRA_CFLAGS += -DISP_GE
+
+ISP_LSC_V3 := 1
+EXTRA_CFLAGS += -DISP_LSC_V3
+
+ISP_WB_V1_1 := 1
+EXTRA_CFLAGS += -DISP_WB_V1_1
+
+ISP_DG_V1 := 1
+EXTRA_CFLAGS += -DISP_DG_V1
+
+ISP_CMPD_20BIT := 1
+EXTRA_CFLAGS += -DISP_CMPD_20BIT
+
+ISP_CPROC := 1
+EXTRA_CFLAGS += -DISP_CPROC
+
+ISP_EE_V2 := 1
+EXTRA_CFLAGS += -DISP_EE_V2
+
+ISP_GC_V2 := 1
+EXTRA_CFLAGS += -DISP_GC_V2
+
+ISP_AFM_V1 := 1
+EXTRA_CFLAGS += -DISP_AFM_V1
+
+ISP_DMSC_V2 := 1
+EXTRA_CFLAGS += -DISP_DMSC_V2
+
+ISP_EXP_V2 := 1
+EXTRA_CFLAGS += -DISP_EXP_V2
+
+ISP_WDR_V4 := 1
+EXTRA_CFLAGS += -DISP_WDR_V4
+
+ISP_3DNR_V3_1 := 1
+EXTRA_CFLAGS += -DISP_3DNR_V3_1
+
+ISP_DPF_V1 := 1
+EXTRA_CFLAGS += -DISP_DPF_V1
+
+ISP_HDR_V2_1 := 1
+EXTRA_CFLAGS += -DISP_HDR_V2_1
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/Makefile b/drivers/media/platform/eswin/isp/vvcam/mipi/Makefile
new file mode 100644
index 000000000000..1aaced94db87
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/Makefile
@@ -0,0 +1,87 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+ifeq ($(VVCAM_PATH),)
+	VVCAM_PATH:=$(shell cd $(shell pwd)/../; pwd)
+	export VVCAM_PATH
+endif
+
+include $(PWD)/../Param.mk
+TARGET = vvcam_mipi
+
+obj-m +=$(TARGET).o
+$(TARGET)-objs += vvcam_mipi_platform.o
+$(TARGET)-objs += vvcam_mipi_driver.o
+ifeq ($(MIPI_VERSION), NWL)
+$(TARGET)-objs += vvcam_mipi_hal.o
+else
+$(TARGET)-objs += vvcam_mipi_dwc_hal.o
+endif
+
+CFLAGS += -Wno-error -Wno-unused-function
+
+PWD :=$(shell pwd)
+
+all:
+	make -C $(KERNEL_DIR) M=$(PWD) modules
+	@[ -e $(INSTALL_DIR)/lib ] || mkdir $(INSTALL_DIR)/lib -p
+	@cp $(TARGET).ko $(INSTALL_DIR)/lib
+modules_install:
+	make -C $(KERNEL_DIR) M=$(PWD) modules_install
+clean:
+	rm -rf $($(TARGET)-objs)
+	make -C $(KERNEL_DIR) M=$(PWD) clean
+	@rm $(INSTALL_DIR)/lib/$(TARGET).ko
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi.h b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi.h
new file mode 100644
index 000000000000..35299120efec
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi.h
@@ -0,0 +1,178 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_MIPI_H__
+#define __VVCAM_MIPI_H__
+
+#define MIPI_LANE_NUM_MAX 8
+#define MIPI_OUTPUT_INTERFACES_MAX 8
+
+typedef enum {
+    VVCAM_INPUT_MODE_MIPI    = 0x00,
+    VVCAM_INPUT_MODE_LVDS    = 0x01,
+    VVCAM_INPUT_MODE_BT1120  = 0x02,
+    VVCAM_INPUT_MODE_DVP     = 0x03
+} vvcam_input_mode_t;
+
+typedef struct {
+    unsigned int left;
+    unsigned int top;
+    unsigned int width;
+    unsigned int height;
+} vvcam_image_rect_t;
+
+typedef enum {
+    VVCAM_RAW_8BIT = 0,
+    VVCAM_RAW_10BIT,
+    VVCAM_RAW_12BIT,
+    VVCAM_RAW_14BIT,
+    VVCAM_RAW_16BIT,
+    VVCAM_RAW_20BIT,
+    VVCAM_RAW_24BIT,
+    VVCAM_RGB_444,
+    VVCAM_RGB_555,
+    VVCAM_RGB_565,
+    VVCAM_RGB_666,
+    VVCAM_RGB_888,
+    VVCAM_YUV420_8BIT,
+    VVCAM_YUV420_8BIT_LEG,
+    VVCAM_YUV420_8BIT_CSPS,
+    VVCAM_YUV420_10BIT,
+    VVCAM_YUV420_10BIT_CSPS,
+    VVCAM_YUV422_8BIT,
+    VVCAM_YUV422_10BIT,
+} vvcam_data_type_t;
+
+typedef enum {
+    VVCAM_ITF_VCID_0 = 0,
+    VVCAM_ITF_VCID_1,
+    VVCAM_ITF_VCID_2,
+    VVCAM_ITF_VCID_3,
+    VVCAM_ITF_VCID_4,
+    VVCAM_ITF_VCID_5,
+    VVCAM_ITF_VCID_6,
+    VVCAM_ITF_VCID_7,
+    VVCAM_ITF_VCID_MAX,
+} vvcam_itf_vcid_t;
+
+typedef enum {
+    VVCAM_ITF_BIT_MODE_48BIT = 0,
+    VVCAM_ITF_BIT_MODE_16BIT,
+} vvcam_itf_bit_mode_t;
+
+typedef enum {
+    VVCAM_SYNC_POL_HIGH_ACIVE = 0,
+    VVCAM_SYNC_POL_LOW_ACIVE
+}vvcam_sync_pol_t;
+
+typedef struct {
+    uint32_t hsa;   /* horizontal synchronism activate (pixel cycle) */
+    uint32_t hbp;   /* horizontal back porch (pixel cycle) */
+    uint32_t hfp;   /* horizontal front porch (pixel cycle) */
+    uint32_t hline;   /* overall time for each line (pixel cycle) */
+} vvcam_itf_horizontal_time_t;
+
+typedef struct {
+    uint32_t vsa;   /* vertical synchronism activate (hline) */
+    uint32_t vbp;   /* vertical back porch (hline) */
+    uint32_t vfp;   /* vertical front porch (hline) */
+    uint32_t vactivate_lines;   /* vertical resolution of video */
+} vvcam_itf_vertical_time_t;
+
+typedef struct {
+
+	vvcam_sync_pol_t pwdn;
+	vvcam_sync_pol_t rst;
+	vvcam_sync_pol_t fsin;
+}vvcam_power_attr_t;
+
+typedef struct {
+	vvcam_power_attr_t powerPin;
+	bool isPower;
+    uint32_t bitWidth;
+}vvcam_dvp_dev_attr_t;
+
+typedef struct {
+    bool enable;
+    bool embedded_data_en;
+    vvcam_itf_vcid_t vcid;
+    vvcam_itf_bit_mode_t bit_mode;
+    vvcam_data_type_t data_type;
+    vvcam_itf_horizontal_time_t h_time;
+    vvcam_itf_vertical_time_t v_time;
+} vvcam_mipi_output_itf_t;
+
+typedef struct {
+    vvcam_mipi_output_itf_t itf_cfg[MIPI_OUTPUT_INTERFACES_MAX];
+    short lane_id[MIPI_LANE_NUM_MAX];  /* lane id: -1 - disable */
+} vvcam_mipi_dev_attr_t;
+
+typedef struct {
+    vvcam_input_mode_t mode;       /* input mode: MIPI/LVDS/BT1120 */
+    vvcam_image_rect_t image_rect; /* Rx device area */
+
+    union {
+        vvcam_mipi_dev_attr_t mipi_attr;
+		vvcam_dvp_dev_attr_t  dvp_attr;
+        //vvcam_lvds_dev_attr_t lvds_attr;
+    };
+} vvcam_input_dev_attr_t;
+
+#define VVCAM_MIPI_IOC_MAGIC 'v'
+#define VVCAM_MIPI_RESET        _IOW(VVCAM_MIPI_IOC_MAGIC, 0x01, unsigned int) /* 1:reset 0:unreset */
+#define VVCAM_MIPI_SET_DEV_ATTR _IOW(VVCAM_MIPI_IOC_MAGIC, 0x02, vvcam_input_dev_attr_t)
+#define VVCAM_MIPI_SET_STREAM   _IOW(VVCAM_MIPI_IOC_MAGIC, 0x03, unsigned int)
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_driver.c b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_driver.c
new file mode 100644
index 000000000000..9d1aff02c652
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_driver.c
@@ -0,0 +1,329 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/pm_runtime.h>
+#include <linux/fs.h>
+#include "vvcam_mipi.h"
+#include "vvcam_mipi_driver.h"
+#include "vvcam_mipi_platform.h"
+
+static int vvcam_mipi_open(struct inode *inode, struct file *file)
+{
+    struct miscdevice *pmisc_dev = file->private_data;
+    struct vvcam_mipi_dev *mipi_dev;
+
+    mipi_dev = container_of(pmisc_dev, struct vvcam_mipi_dev, miscdev);
+    if (!mipi_dev)
+        return -ENOMEM;
+
+    file->private_data = mipi_dev;
+
+    mutex_lock(&mipi_dev->mlock);
+    pm_runtime_get_sync(mipi_dev->dev);
+    mutex_unlock(&mipi_dev->mlock);
+
+    return 0;
+}
+
+static int vvcam_mipi_release(struct inode *inode, struct file *file)
+{
+    struct vvcam_mipi_dev *mipi_dev;
+
+    mipi_dev = file->private_data;
+
+    mutex_lock(&mipi_dev->mlock);
+    pm_runtime_put(mipi_dev->dev);
+    mutex_unlock(&mipi_dev->mlock);
+
+    return 0;
+}
+
+static long vvcam_mipi_ioctl(struct file *file,
+                        unsigned int cmd, unsigned long arg)
+{
+    struct vvcam_mipi_dev *mipi_dev;
+    int ret = 0;
+    uint32_t reset;
+    uint32_t stream_enable;
+    vvcam_input_dev_attr_t mipi_cfg;
+
+    mipi_dev = file->private_data;
+
+    mutex_lock(&mipi_dev->mlock);
+
+    switch(cmd) {
+    case VVCAM_MIPI_RESET:
+        ret = copy_from_user(&reset, (void __user *)arg, sizeof(reset));
+        if (ret)
+            break;
+        ret = vvcam_mipi_reset(mipi_dev, reset);
+        break;
+    case VVCAM_MIPI_SET_DEV_ATTR:
+        ret = copy_from_user(&mipi_cfg, (void __user *)arg, sizeof(mipi_cfg));
+        if (ret)
+            break;
+        ret = vvcam_mipi_set_cfg(mipi_dev, mipi_cfg);
+        break;
+    case VVCAM_MIPI_SET_STREAM:
+        ret = copy_from_user(&stream_enable,
+                    (void __user *)arg, sizeof(stream_enable));
+        if (ret)
+            break;
+        ret = vvcam_mipi_set_stream(mipi_dev, stream_enable);
+        break;
+    default:
+        break;
+    }
+
+    mutex_unlock(&mipi_dev->mlock);
+
+    return ret;
+}
+
+static struct file_operations vvcam_mipi_fops = {
+	.owner          = THIS_MODULE,
+	.open           = vvcam_mipi_open,
+	.release        = vvcam_mipi_release,
+	.unlocked_ioctl = vvcam_mipi_ioctl,
+};
+
+static int vvcam_mipi_probe(struct platform_device *pdev)
+{
+    int ret = 0;
+    struct vvcam_mipi_dev *mipi_dev;
+    struct resource *res;
+    char *miscdev_name;
+
+    mipi_dev = devm_kzalloc(&pdev->dev,
+                sizeof(struct vvcam_mipi_dev), GFP_KERNEL);
+    if (!mipi_dev)
+        return -ENOMEM;
+
+    res =  platform_get_resource(pdev, IORESOURCE_MEM, 0);
+    if (!res) {
+		dev_err(&pdev->dev, "can't fetch device resource info\n");
+		return -EIO;
+	}
+    mipi_dev->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mipi_dev->base)) {
+        dev_err(&pdev->dev, "can't remap device resource info\n");
+        return PTR_ERR(mipi_dev->base);
+    }
+
+    res =  platform_get_resource(pdev, IORESOURCE_MEM, 1);
+    if (!res) {
+		dev_err(&pdev->dev, "can't fetch tpg resource info\n");
+		return -EIO;
+	}
+    mipi_dev->tpg_reg = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mipi_dev->tpg_reg)) {
+        dev_err(&pdev->dev, "can't remap tpg resource info\n");
+        return PTR_ERR(mipi_dev->tpg_reg);
+    }
+
+    res =  platform_get_resource(pdev, IORESOURCE_MEM, 2);
+    if (!res) {
+		dev_err(&pdev->dev, "can't fetch csi ctrl resource info\n");
+		return -EIO;
+	}
+    mipi_dev->csi_ctrl_reg = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mipi_dev->csi_ctrl_reg)) {
+        dev_err(&pdev->dev, "can't remap csi ctrl resource info\n");
+        return PTR_ERR(mipi_dev->csi_ctrl_reg);
+    }
+
+    res =  platform_get_resource(pdev, IORESOURCE_MEM, 3);
+    if (!res) {
+        dev_err(&pdev->dev, "can't fetch csi mux ctrl resource info\n");
+        return -EIO;
+    }
+    mipi_dev->csi_mux_reg = devm_ioremap_resource(&pdev->dev, res);
+    if (IS_ERR(mipi_dev->csi_mux_reg)) {
+        dev_err(&pdev->dev, "can't remap csi mux ctrl resource info\n");
+        return PTR_ERR(mipi_dev->csi_mux_reg);
+    }
+
+    mutex_init(&mipi_dev->mlock);
+	platform_set_drvdata(pdev, mipi_dev);
+
+    mipi_dev->dev = &pdev->dev;
+    mipi_dev->id  = pdev->id;
+
+    vvcam_mipi_default_cfg(mipi_dev);
+
+    miscdev_name = devm_kzalloc(&pdev->dev, 16, GFP_KERNEL);
+    if (!miscdev_name)
+        return -ENOMEM;
+    snprintf(miscdev_name, 16, "%s.%d", VVCAM_MIPI_NAME, pdev->id);
+
+    mipi_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
+    mipi_dev->miscdev.name  = miscdev_name;
+    mipi_dev->miscdev.fops  = &vvcam_mipi_fops;
+
+    ret = misc_register(&mipi_dev->miscdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to register device\n");
+        return -EINVAL;
+    }
+
+    dev_info(&pdev->dev, "vvcam mipi driver probe success\n");
+
+    return 0;
+}
+
+static int vvcam_mipi_remove(struct platform_device *pdev)
+{
+    struct vvcam_mipi_dev *mipi_dev;
+
+    mipi_dev = platform_get_drvdata(pdev);
+    misc_deregister(&mipi_dev->miscdev);
+    return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int vvcam_mipi_system_suspend(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_suspend(dev);
+    if (ret) {
+        dev_err(dev, "force suspend %s failed\n", dev_name(dev));
+        return ret;
+    }
+	return ret;
+}
+
+static int vvcam_mipi_system_resume(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_resume(dev);
+    if (ret) {
+        dev_err(dev, "force resume %s failed\n", dev_name(dev));
+        return ret;
+    }
+	return ret;
+}
+
+static int vvcam_mipi_runtime_suspend(struct device *dev)
+{
+    return 0;
+}
+
+static int vvcam_mipi_runtime_resume(struct device *dev)
+{
+    return 0;
+}
+
+static const struct dev_pm_ops vvcam_mipi_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(vvcam_mipi_system_suspend, vvcam_mipi_system_resume)
+	SET_RUNTIME_PM_OPS(vvcam_mipi_runtime_suspend, vvcam_mipi_runtime_resume, NULL)
+};
+#endif
+
+static const struct of_device_id vvcam_mipi_of_match[] = {
+	{.compatible = "verisilicon,mipi",},
+	{ /* sentinel */ },
+};
+
+static struct platform_driver vvcam_mipi_driver = {
+	.probe  = vvcam_mipi_probe,
+	.remove = vvcam_mipi_remove,
+	.driver = {
+		.name           = VVCAM_MIPI_NAME,
+		.owner          = THIS_MODULE,
+        .of_match_table = vvcam_mipi_of_match,
+#ifdef CONFIG_PM_SLEEP
+        .pm             = &vvcam_mipi_pm_ops,
+#endif
+	}
+};
+
+static int __init vvcam_mipi_init_module(void)
+{
+    int ret;
+    ret = platform_driver_register(&vvcam_mipi_driver);
+    if (ret) {
+        printk(KERN_ERR "Failed to register mipi driver\n");
+        return ret;
+    }
+
+    ret = vvcam_mipi_platform_device_register();
+    if (ret) {
+		platform_driver_unregister(&vvcam_mipi_driver);
+		printk(KERN_ERR "Failed to register vvcam mipi platform devices\n");
+		return ret;
+	}
+
+    return ret;
+}
+
+static void __exit vvcam_mipi_exit_module(void)
+{
+    platform_driver_unregister(&vvcam_mipi_driver);
+    vvcam_mipi_platform_device_unregister();
+    return;
+}
+
+late_initcall(vvcam_mipi_init_module);
+module_exit(vvcam_mipi_exit_module);
+
+MODULE_DESCRIPTION("Verisilicon mipi driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_driver.h b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_driver.h
new file mode 100644
index 000000000000..06ecfd83f757
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_driver.h
@@ -0,0 +1,83 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_MIPI_DRIVER_H__
+#define __VVCAM_MIPI_DRIVER_H__
+
+#include <linux/miscdevice.h>
+#include "vvcam_mipi.h"
+
+#define VVCAM_MIPI_NAME "vvcam-mipi"
+
+struct vvcam_mipi_dev {
+    void __iomem *base;
+	void __iomem *tpg_reg;
+	void __iomem *csi_ctrl_reg;
+    void __iomem *csi_mux_reg;
+    int  id;
+    uint8_t bus_width;
+    struct miscdevice miscdev;
+    struct device *dev;
+    struct mutex mlock;
+    vvcam_input_dev_attr_t mipi_cfg;
+};
+
+int vvcam_mipi_reset(struct vvcam_mipi_dev *mipi_dev, uint32_t val);
+int vvcam_mipi_default_cfg(struct vvcam_mipi_dev *mipi_dev);
+int vvcam_mipi_set_cfg(struct vvcam_mipi_dev *mipi_dev,
+                    vvcam_input_dev_attr_t mipi_cfg);
+int vvcam_mipi_set_stream(struct vvcam_mipi_dev *mipi_dev, uint32_t enable);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_dwc_hal.c b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_dwc_hal.c
new file mode 100644
index 000000000000..6d0455950672
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_dwc_hal.c
@@ -0,0 +1,863 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include "vvcam_mipi_driver.h"
+
+/**
+ * MRV_CSI2_NUM_LANES
+ * Config num lanes register [2:0] rw
+ * 0000b - 1 Lanes
+ * 0001b - 2 Lane
+ * 0010b - 3 Lanes
+ * 0011b - 4 Lanes (D-PHY only)
+ * 0100b - 5 Lanes (D-PHY only)
+ * 0101b - 6 Lanes (D-PHY only)
+ * 0110b - 7 Lanes (D-PHY only)
+ * 0111b - 8 Lanes (D-PHY only)
+ */
+#define MRV_CSI2_NUM_LANES 0x04
+
+/**
+ * CSI2 logical reset
+ * 0b : reset
+ * 1b : normal
+**/
+#define MRV_CSI2_RSTN 0x08
+
+
+/**
+ * CSI2 data ID Monitors
+**/
+#define MRV_CSI2_DATA_IDS1 0x10
+
+/**
+ * CSI2 data ID Monitors2
+**/
+#define MRV_CSI2_DATA_IDS2 0x14
+
+/**
+ * CSI2 virtual channel data ID Monitors1
+**/
+#define MRV_CSI2_DATA_IDS_VC1 0x30
+
+/**
+ * CSI2 virtual channel data ID Monitors2
+**/
+#define MRV_CSI2_DATA_IDS_VC2 0x34
+
+/**
+ * PHY SHUTDOWN, shutdown input
+ * 0b : OFF
+ * 1b : ON
+**/
+#define MRV_CSI2_PHY_SHUTDOWNZ 0x40
+
+/**
+ * D-PHY reset output
+ * 0b : reset
+ * 1b : normal
+**/
+#define MRV_CSI2_PHY_RSTZ 0x44
+
+/**
+ * Rx D-PHY status
+ * receiving high speed clock signal or ultra low power mode
+**/
+#define MRV_CSI2_PHY_RX 0x48
+
+
+/**
+ * D-PHY stop state
+**/
+#define MRV_CSI2_PHY_STOPSTATE 0x4C
+
+/**
+ IPI interface register
+**/
+
+/**
+ * IPI mode
+ * bit[0] : CAMERA mode(0), CTRL mode(1)
+ * bit[8] : 48bit(0), 16bit(1)
+ * bit[16]: CT inactive(0), CT active(1)
+ * bit[24]: IPI disable(0), IPI enable(1)
+**/
+#define MRV_CSI2_IPI_MODE 0x80
+
+/**
+ * IPI Virtual Channel
+ * select virtual channel processed by ipi
+ * 00b : vc 0
+ * 01b : vc 1
+ * 10b : vc 2
+ * 11b : vc 3
+ * [3:2] : virtual channel extension
+ * [4] : virtual channel extension extra bit
+**/
+#define MRV_CSI2_IPI_VCID 0x84
+
+/**
+ * MRV_CSI2_IPI_DATA_TYPE
+ * [5:0] : data type
+ * [8] : embedded data
+ * set ipi data format
+ * 0x18 - YUV420_8
+ * 0x19 - YUV420_10
+ * 0x1A - YUV420_8_LEG
+ * 0x1C - YUV420_8_SHIFT(NV12)
+ * 0x1D - YUV420_10_SHIFT
+ * 0x1E - YUV422_8(NV16)
+ * 0x1F - YUV422_10
+ * 0x20 - RGB444
+ * 0x21 - RGB555
+ * 0x22 - RGB565
+ * 0x23 - RGB666
+ * 0x24 - RGB888
+ * 0x28 - RAW6
+ * 0x29 - RAW7
+ * 0x2a - RAW8
+ * 0x2b - RAW10
+ * 0x2c - RAW12
+ * 0x2d - RAW14
+ * 0x2e - RAW16
+ * 0x2f - RAW20
+ * 0x27 - RAW24
+ */
+#define MRV_CSI2_IPI_DATA_TYPE 0x88
+
+/**
+ * IPI memory flush
+ * [0] : flush ipi memory
+ * [8] : memory is automatically flushed at each frame start
+**/
+#define MRV_CSI2_IPI_MEM_FLUSH 0x8C
+
+/**
+ * IPI horizontal synchronism active time
+**/
+#define MRV_CSI2_IPI_HSA_TIME 0x90
+
+/**
+ * IPI horizontal back porch time
+**/
+#define MRV_CSI2_IPI_HBP_TIME 0x94
+
+/**
+ * IPI horizontal sync delay time
+**/
+#define MRV_CSI2_IPI_HSD_TIME 0x98
+
+/**
+ * overall time for each video line
+**/
+#define MRV_CSI2_IPI_HLINE_TIME 0x9C
+
+/**
+ * IPI soft reset
+ * [0/4/8/12/16/20/24/28] : 1/2/3/4/5/6/7/8 ipi soft reset
+**/
+#define MRV_CSI2_IPI_SOFT_RSTN 0xA0
+
+/**
+ * IPI advance features
+**/
+#define MRV_CSI2_IPI_ADV_FEATURES 0xAC
+
+
+/**
+ * vertical synchronism active period in number of horizontal lines
+**/
+#define MRV_CSI2_IPI_VSA_LINES 0xB0
+
+/**
+ * vertical back porch period in number of horizontal lines
+**/
+#define MRV_CSI2_IPI_VBP_LINES 0xB4
+
+/**
+ * vertical front porch period in number of horizontal lines
+**/
+#define MRV_CSI2_IPI_VFP_LINES 0xB8
+
+/**
+ * vertical resolution of video
+**/
+#define MRV_CSI2_IPI_VACTIVATE_LINES 0xBC
+
+/**
+ * IPI2 interface register
+**/
+#define MRV_CSI2_IPI2_MODE 0x200
+#define MRV_CSI2_IPI2_VCID 0x204
+#define MRV_CSI2_IPI2_DATA_TYPE 0x208
+#define MRV_CSI2_IPI2_MEM_FLUSH 0x20C
+#define MRV_CSI2_IPI2_HSA_TIME 0x210
+#define MRV_CSI2_IPI2_HBP_TIME 0x214
+#define MRV_CSI2_IPI2_HSD_TIME 0x218
+#define MRV_CSI2_IPI2_ADV_FEATURES 0x21C
+
+/**
+ * IPI3 interface register
+**/
+#define MRV_CSI2_IPI3_MODE 0x220
+#define MRV_CSI2_IPI3_VCID 0x224
+#define MRV_CSI2_IPI3_DATA_TYPE 0x228
+#define MRV_CSI2_IPI3_MEM_FLUSH 0x22C
+#define MRV_CSI2_IPI3_HSA_TIME 0x230
+#define MRV_CSI2_IPI3_HBP_TIME 0x234
+#define MRV_CSI2_IPI3_HSD_TIME 0x238
+#define MRV_CSI2_IPI3_ADV_FEATURES 0x23C
+
+/**
+ * IPI4 interface register
+**/
+#define MRV_CSI2_IPI4_MODE 0x240
+#define MRV_CSI2_IPI4_VCID 0x244
+#define MRV_CSI2_IPI4_DATA_TYPE 0x248
+#define MRV_CSI2_IPI4_MEM_FLUSH 0x24C
+#define MRV_CSI2_IPI4_HSA_TIME 0x250
+#define MRV_CSI2_IPI4_HBP_TIME 0x254
+#define MRV_CSI2_IPI4_HSD_TIME 0x258
+#define MRV_CSI2_IPI4_ADV_FEATURES 0x25C
+
+/**
+ * IPI5 interface register
+**/
+#define MRV_CSI2_IPI5_MODE 0x5C0
+#define MRV_CSI2_IPI5_VCID 0x5C4
+#define MRV_CSI2_IPI5_DATA_TYPE 0x5C8
+#define MRV_CSI2_IPI5_MEM_FLUSH 0x5CC
+#define MRV_CSI2_IPI5_HSA_TIME 0x5D0
+#define MRV_CSI2_IPI5_HBP_TIME 0x5D4
+#define MRV_CSI2_IPI5_HSD_TIME 0x5D8
+#define MRV_CSI2_IPI5_ADV_FEATURES 0x5DC
+
+/**
+ * IPI6 interface register
+**/
+#define MRV_CSI2_IPI6_MODE 0x5E0
+#define MRV_CSI2_IPI6_VCID 0x5E4
+#define MRV_CSI2_IPI6_DATA_TYPE 0x5E8
+#define MRV_CSI2_IPI6_MEM_FLUSH 0x5EC
+#define MRV_CSI2_IPI6_HSA_TIME 0x5F0
+#define MRV_CSI2_IPI6_HBP_TIME 0x5F4
+#define MRV_CSI2_IPI6_HSD_TIME 0x5F8
+#define MRV_CSI2_IPI6_ADV_FEATURES 0x5FC
+
+/**
+ * IPI7 interface register
+**/
+#define MRV_CSI2_IPI7_MODE 0x600
+#define MRV_CSI2_IPI7_VCID 0x604
+#define MRV_CSI2_IPI7_DATA_TYPE 0x608
+#define MRV_CSI2_IPI7_MEM_FLUSH 0x60C
+#define MRV_CSI2_IPI7_HSA_TIME 0x610
+#define MRV_CSI2_IPI7_HBP_TIME 0x614
+#define MRV_CSI2_IPI7_HSD_TIME 0x618
+#define MRV_CSI2_IPI7_ADV_FEATURES 0x61C
+
+/**
+ * IPI8 interface register
+**/
+#define MRV_CSI2_IPI8_MODE 0x620
+#define MRV_CSI2_IPI8_VCID 0x624
+#define MRV_CSI2_IPI8_DATA_TYPE 0x628
+#define MRV_CSI2_IPI8_MEM_FLUSH 0x62C
+#define MRV_CSI2_IPI8_HSA_TIME 0x630
+#define MRV_CSI2_IPI8_HBP_TIME 0x634
+#define MRV_CSI2_IPI8_HSD_TIME 0x638
+#define MRV_CSI2_IPI8_ADV_FEATURES 0x63C
+
+/**
+ * macro values for register
+**/
+
+#define IPI_MODE_CAMERA             (0 << 0)
+#define IPI_MODE_CTRL               (1 << 0)
+#define IPI_MODE_48BIT_COLOR        (0 << 8)
+#define IPI_MODE_16BIT_COLOR        (1 << 8)
+#define IPI_MODE_CT_ACTIVATE        (1 << 16)
+#define IPI_MODE_CT_INACTIVATE      (0 << 16)
+#define IPI_MODE_ENABLE             (1 << 24)
+
+#define IPI_EMBEDDED_DATA_ENABLE    (1 << 8)
+
+#define IPI_DATA_TYPE_RAW_6BIT      0x28
+#define IPI_DATA_TYPE_RAW_7BIT      0x29
+#define IPI_DATA_TYPE_RAW_8BIT      0x2A
+#define IPI_DATA_TYPE_RAW_10BIT     0x2B
+#define IPI_DATA_TYPE_RAW_12BIT     0x2C
+#define IPI_DATA_TYPE_RAW_14BIT     0x2D
+#define IPI_DATA_TYPE_RAW_16BIT     0x2E
+#define IPI_DATA_TYPE_RAW_20BIT     0x2F
+#define IPI_DATA_TYPE_RAW_24BIT     0x27
+
+#define IPI_DATA_TYPE_RGB_444        0x20
+#define IPI_DATA_TYPE_RGB_555        0x21
+#define IPI_DATA_TYPE_RGB_565        0x22
+#define IPI_DATA_TYPE_RGB_666        0x23
+#define IPI_DATA_TYPE_RGB_888        0x24
+
+#define IPI_DATA_TYPE_YUV420_8BIT           0x18
+#define IPI_DATA_TYPE_YUV420_10BIT          0x19
+#define IPI_DATA_TYPE_YUV420_8BIT_LEG       0x1A
+#define IPI_DATA_TYPE_YUV420_8BIT_CSPS      0x1C
+#define IPI_DATA_TYPE_YUV420_10BIT_CSPS     0x1D
+#define IPI_DATA_TYPE_YUV422_8BIT           0x1E
+#define IPI_DATA_TYPE_YUV422_10BIT          0x1F
+
+#define IS_VALID_ID(id) (id != -1)
+
+static vvcam_input_dev_attr_t default_config = {
+    .mode = VVCAM_INPUT_MODE_MIPI,
+    .image_rect = {
+        .left = 0,
+        .top =0,
+        .width = 1920,
+        .height = 1080,
+    },
+    .mipi_attr = {
+        .itf_cfg[0] = {
+            .enable = true,
+            .embedded_data_en = false,
+            .vcid = VVCAM_ITF_VCID_0,
+            .bit_mode = VVCAM_ITF_BIT_MODE_16BIT,
+            .data_type = VVCAM_RAW_12BIT,
+            .h_time = {
+                .hsa = 80,
+                .hbp = 350,
+                .hfp = 300,
+                .hline = 2500,
+            },
+            .v_time = {
+                .vsa = 5,
+                .vbp = 25,
+                .vfp = 10,
+                .vactivate_lines = 1080,
+            }
+        },
+        .lane_id  = {0, 1, 2, 3, -1, -1, -1, -1},
+    },
+};
+
+static void vvcam_mipi_dwc_write_reg(void __iomem *base,
+			uint32_t addr, uint32_t value)
+{
+	writel(value, base + addr);
+}
+
+static uint32_t vvcam_mipi_dwc_read_reg(void __iomem *base, uint32_t addr)
+{
+	return readl(base + addr);
+}
+
+static int vvcam_mipi_dwc_set_stream(struct vvcam_mipi_dev *mipi_dev,
+                                    uint32_t enable)
+{
+    uint32_t lanes_stop_state;
+    uint32_t phy_rx;
+
+    if (enable) {
+        lanes_stop_state =
+            vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_PHY_STOPSTATE);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_PHY_SHUTDOWNZ, 1);
+    } else {
+        /* shutdown d-phy input */
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_PHY_SHUTDOWNZ, 0);
+    }
+
+    phy_rx = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_PHY_RX);
+
+    return 0;
+}
+
+static int vvcam_mipi_dwc_set_ipi_mode(struct vvcam_mipi_dev *mipi_dev,
+                                        uint8_t ipi,
+                                        vvcam_itf_bit_mode_t bit_mode,
+                                        vvcam_itf_vcid_t vcid)
+{
+    switch (ipi) {
+    case 0:
+        if ((vcid >= 0) && (vcid < VVCAM_ITF_VCID_MAX)) {
+            vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_IPI_VCID, vcid);
+        }
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_IPI_MODE,
+            (bit_mode << 8) | IPI_MODE_CT_ACTIVATE | IPI_MODE_ENABLE);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_IPI_ADV_FEATURES, 0x01000000);
+        break;
+
+    case 1:
+    case 2:
+    case 3:
+        if ((vcid >= 0) && (vcid < VVCAM_ITF_VCID_MAX)) {
+            vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_VCID + (ipi - 1) * 0x20, vcid);
+        }
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+            MRV_CSI2_IPI2_MODE + (ipi - 1) * 0x20,
+            (bit_mode << 8) | IPI_MODE_CT_ACTIVATE | IPI_MODE_ENABLE);
+        if (ipi == 1) {
+            vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_ADV_FEATURES + (ipi - 1) * 0x20, 0x01200000);
+        } else {
+            vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_ADV_FEATURES + (ipi - 1) * 0x20, 0x01000000);
+        }
+        break;
+
+    case 4:
+    case 5:
+    case 6:
+    case 7:
+        if ((vcid >= 0) && (vcid < VVCAM_ITF_VCID_MAX)) {
+            vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI5_VCID + (ipi - 4) * 0x20, vcid);
+        }
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+            MRV_CSI2_IPI5_MODE + (ipi - 4) * 0x20,
+            (bit_mode << 8) | IPI_MODE_CT_ACTIVATE | IPI_MODE_ENABLE);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+            MRV_CSI2_IPI5_ADV_FEATURES + (ipi - 4) * 0x20, 0x01000000);
+        break;
+
+    default:
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static int vvcam_mipi_dwc_set_ipi_format(struct vvcam_mipi_dev *mipi_dev,
+                    uint8_t ipi, uint32_t format, bool embedded_data_en)
+{
+    uint32_t fmt_code = 0;
+    uint32_t ids = 0;
+    uint32_t embedded_data_en_mask = 0;
+
+    if (embedded_data_en) {
+        embedded_data_en_mask = IPI_EMBEDDED_DATA_ENABLE;
+    }
+
+    switch (format) {
+    case VVCAM_RAW_8BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_8BIT;
+        break;
+
+    case VVCAM_RAW_10BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_10BIT;
+        break;
+
+    case VVCAM_RAW_12BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_12BIT;
+        break;
+
+    case VVCAM_RAW_14BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_14BIT;
+        break;
+
+    case VVCAM_RAW_16BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_16BIT;
+        break;
+
+    case VVCAM_RAW_20BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_20BIT;
+        break;
+
+    case VVCAM_RAW_24BIT:
+        fmt_code = IPI_DATA_TYPE_RAW_24BIT;
+        break;
+
+    case VVCAM_RGB_444:
+        fmt_code = IPI_DATA_TYPE_RGB_444;
+        break;
+
+    case VVCAM_RGB_555:
+        fmt_code = IPI_DATA_TYPE_RGB_555;
+        break;
+
+    case VVCAM_RGB_565:
+        fmt_code = IPI_DATA_TYPE_RGB_565;
+        break;
+
+    case VVCAM_RGB_666:
+        fmt_code = IPI_DATA_TYPE_RGB_666;
+        break;
+
+    case VVCAM_RGB_888:
+        fmt_code = IPI_DATA_TYPE_RGB_888;
+        break;
+
+    case VVCAM_YUV420_8BIT:
+        fmt_code = IPI_DATA_TYPE_YUV420_8BIT;
+        break;
+
+    case VVCAM_YUV420_10BIT:
+        fmt_code = IPI_DATA_TYPE_YUV420_10BIT;
+        break;
+
+    case VVCAM_YUV420_8BIT_CSPS:
+        fmt_code = IPI_DATA_TYPE_YUV420_8BIT_LEG;
+        break;
+
+    case VVCAM_YUV420_10BIT_CSPS:
+        fmt_code = IPI_DATA_TYPE_YUV420_10BIT_CSPS;
+        break;
+
+    case VVCAM_YUV422_8BIT:
+        fmt_code = IPI_DATA_TYPE_YUV422_8BIT;
+        break;
+
+    case VVCAM_YUV422_10BIT:
+        fmt_code = IPI_DATA_TYPE_YUV422_10BIT;
+        break;
+
+    default:
+        pr_err("format %d not support", format);
+        return -EINVAL;
+    }
+
+    switch (ipi) {
+    case 0:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1);
+        ids = fmt_code | (ids & 0xFFFFFFC0);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_DATA_IDS1, ids);
+        break;
+    case 1:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI2_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1);
+        ids = ((fmt_code << 8) | (ids & 0xFFFFC0FF));
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1, ids);
+        break;
+    case 2:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI3_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1);
+        ids = ((fmt_code << 16) | (ids & 0xFFC0FFFF));
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1, ids);
+        break;
+    case 3:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI4_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1);
+        ids = ((fmt_code << 24) | (ids & 0xC0FFFFFF));
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS1, ids);
+        break;
+    case 4:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI5_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2);
+        ids = fmt_code | (ids & 0xFFFFFFC0);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2, ids);
+        break;
+    case 5:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI6_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2);
+        ids = ((fmt_code << 8) | (ids & 0xFFFFC0FF));
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2, ids);
+        break;
+    case 6:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI7_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2);
+        ids = ((fmt_code << 16) | (ids & 0xFFC0FFFF));
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2, ids);
+        break;
+    case 7:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                    MRV_CSI2_IPI8_DATA_TYPE, fmt_code | embedded_data_en_mask);
+        ids = vvcam_mipi_dwc_read_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2);
+        ids = ((fmt_code << 24) | (ids & 0xC0FFFFFF));
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS2, ids);
+        break;
+    default:
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int vvcam_mipi_dwc_set_ipi_timing(struct vvcam_mipi_dev *mipi_dev,
+                                        uint8_t ipi,
+                                        vvcam_itf_horizontal_time_t *h_time,
+                                        vvcam_itf_vertical_time_t *v_time)
+{
+    switch (ipi) {
+    case 0:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_HSA_TIME, h_time->hsa);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_HBP_TIME, h_time->hbp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_HSD_TIME, h_time->hfp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_HLINE_TIME, h_time->hline);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_VSA_LINES, v_time->vsa);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_VBP_LINES, v_time->vbp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_VFP_LINES, v_time->vfp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_VACTIVATE_LINES, v_time->vactivate_lines);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI_MEM_FLUSH, 0x100);
+        break;
+
+    case 1:
+    case 2:
+    case 3:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_HSA_TIME + (ipi - 1) * 0x20, h_time->hsa);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_HBP_TIME + (ipi - 1) * 0x20, h_time->hbp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_HSD_TIME + (ipi - 1) * 0x20, h_time->hfp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI2_MEM_FLUSH + (ipi - 1) * 0x20, 0x100);
+        break;
+
+    case 4:
+    case 5:
+    case 6:
+    case 7:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI5_HSA_TIME + (ipi - 4) * 0x20, h_time->hsa);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI5_HBP_TIME + (ipi - 4) * 0x20, h_time->hbp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI5_HSD_TIME + (ipi - 4) * 0x20, h_time->hfp);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base,
+                MRV_CSI2_IPI5_MEM_FLUSH + (ipi - 4) * 0x20, 0x100);
+        break;
+
+    default:
+        pr_err("ipi %d is not support\n", ipi);
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int vvcam_mipi_dwc_cfg(struct vvcam_mipi_dev *mipi_dev,
+                            vvcam_input_dev_attr_t mipi_cfg)
+{
+    int i = 0;
+    int32_t ret;
+    uint8_t mipi_lanes = 0;
+    uint32_t ids_vc_1 = 0;
+    uint32_t ids_vc_2 = 0;
+
+    for (i = 0; i < MIPI_LANE_NUM_MAX; i++) {
+        if (IS_VALID_ID(mipi_cfg.mipi_attr.lane_id[i])){
+            mipi_lanes++;
+        }
+    }
+
+    vvcam_mipi_reset(mipi_dev, 1);
+    switch (mipi_lanes) {
+    case 1:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x00);
+        break;
+    case 2:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x01);
+        break;
+    case 3:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x02);
+        break;
+    case 4:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x03);
+        break;
+    case 5:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x04);
+        break;
+    case 6:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x05);
+        break;
+    case 7:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x06);
+        break;
+    case 8:
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x07);
+        break;
+    default:
+        return -EINVAL;
+    }
+
+    for (i = 0; i < MIPI_OUTPUT_INTERFACES_MAX; i++) {
+        if (!mipi_cfg.mipi_attr.itf_cfg[i].enable)
+            continue;
+
+        ret = vvcam_mipi_dwc_set_ipi_format(mipi_dev, i,
+                mipi_cfg.mipi_attr.itf_cfg[i].data_type,
+                mipi_cfg.mipi_attr.itf_cfg[i].embedded_data_en);
+        if (ret)
+            return ret;
+
+        ret = vvcam_mipi_dwc_set_ipi_timing(mipi_dev, i,
+                &mipi_cfg.mipi_attr.itf_cfg[i].h_time,
+                &mipi_cfg.mipi_attr.itf_cfg[i].v_time);
+        if (ret)
+            return ret;
+
+        ret = vvcam_mipi_dwc_set_ipi_mode(mipi_dev, i,
+                mipi_cfg.mipi_attr.itf_cfg[i].bit_mode,
+                mipi_cfg.mipi_attr.itf_cfg[i].vcid);
+        if (ret)
+            return ret;
+
+        if (i < 4) {
+            ids_vc_1 |= (mipi_cfg.mipi_attr.itf_cfg[i].vcid << (i * 8));
+        } else {
+            ids_vc_2 |= (mipi_cfg.mipi_attr.itf_cfg[i].vcid << ((i - 4) * 8));
+        }
+    }
+
+    vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS_VC1, ids_vc_1);
+    vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_DATA_IDS_VC2, ids_vc_2);
+    vvcam_mipi_reset(mipi_dev, 0);
+
+    return 0;
+}
+
+int vvcam_mipi_reset(struct vvcam_mipi_dev *mipi_dev, uint32_t val)
+{
+    if (val) {
+        /* reset */
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_PHY_RSTZ, 0);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_RSTN, 0);
+    } else {
+        /* unreset */
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_RSTN, 1);
+        vvcam_mipi_dwc_write_reg(mipi_dev->base, MRV_CSI2_PHY_RSTZ, 1);
+    }
+    return 0;
+}
+
+int vvcam_mipi_default_cfg(struct vvcam_mipi_dev *mipi_dev)
+{
+    memcpy(&mipi_dev->mipi_cfg,
+        &default_config, sizeof(vvcam_input_dev_attr_t));
+
+    vvcam_mipi_dwc_cfg(mipi_dev, mipi_dev->mipi_cfg);
+#if defined(ISP_VI200)
+    //hardware pin power on sensor
+    vvcam_mipi_dwc_write_reg(mipi_dev->csi_ctrl_reg-4, 0, 0x0);
+    vvcam_mipi_dwc_write_reg(mipi_dev->csi_ctrl_reg-4, 0, 0x3);
+ #elif defined(ISP_VI200_V2)
+    //sepearte reset bit
+    vvcam_mipi_dwc_write_reg(mipi_dev->csi_ctrl_reg-4, 0, 0x00000000);
+    vvcam_mipi_dwc_write_reg(mipi_dev->csi_ctrl_reg-4, 0, 0x00f0f000);
+#endif
+
+    return 0;
+}
+
+
+int vvcam_mipi_set_cfg(struct vvcam_mipi_dev *mipi_dev,
+                    vvcam_input_dev_attr_t mipi_cfg)
+{
+    int ret = 0;
+
+    switch (mipi_cfg.mode) {
+    case VVCAM_INPUT_MODE_MIPI:
+        ret = vvcam_mipi_dwc_cfg(mipi_dev, mipi_cfg);
+        break;
+    case VVCAM_INPUT_MODE_LVDS:
+        ret = -EINVAL;
+        break;
+    case VVCAM_INPUT_MODE_BT1120:
+        ret = -EINVAL;
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    if (ret == 0) {
+        memcpy(&mipi_dev->mipi_cfg, &mipi_cfg, sizeof(mipi_cfg));
+    }
+
+    return ret;
+}
+
+int vvcam_mipi_set_stream(struct vvcam_mipi_dev *mipi_dev, uint32_t enable)
+{
+    int ret = 0;
+
+    switch (mipi_dev->mipi_cfg.mode) {
+    case VVCAM_INPUT_MODE_MIPI:
+        ret = vvcam_mipi_dwc_set_stream(mipi_dev, enable);
+        break;
+    case VVCAM_INPUT_MODE_LVDS:
+        ret = -EINVAL;
+        break;
+    case VVCAM_INPUT_MODE_BT1120:
+        ret = -EINVAL;
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    return ret;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_hal.c b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_hal.c
new file mode 100644
index 000000000000..72f0e7fab74b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_hal.c
@@ -0,0 +1,495 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include "vvcam_mipi_driver.h"
+
+/**
+ *TPG_ISP_RST 0:reset 1:enable
+ *TPG_CSI_RST 0:reset 1:enable
+ *TPG_FRAME_GATE_MASK 0:ON 1:OFF
+ *TPG_MUX_S 0:TPG 1:SENSOR
+ */
+#define TPG_ISP_RST_SHIFT              24
+#define TPG_ISP_RST_MASK               0x01000000
+#define TPG_CSI_RST_SHIFT              28
+#define TPG_CSI_RST_MASK               0x10000210
+#define TPG_FRAME_GATE_SHIFIT          30
+#define TPG_FRAME_GATE_MASK            0x40000000
+#define TPG_MUX_S_SHIFT                29
+#define TPG_MUX_S_MASK                 0x20000000
+
+/**
+ *CSI_CTRL_MUX_T 0:self csi 1:other csi
+ *CSI_CTRL_SHIFT_MODE 0:independent mode 1:other csi dependent mode
+ */
+#define CSI_CTRL_MUX_T_SHIFT           31
+#define CSI_CTRL_MUX_T_MASK            0x80000000
+#define CSI_CTRL_SHIFT_MODE_SHIFT      28
+#define CSI_CTRL_SHIFT_MODE_MASK       0x10000000
+#define CSI_CTRL_VC0_CTRL_SHIFIT       0
+#define CSI_CTRL_VC0_CTRL_MASK         0x00000007
+#define CSI_CTRL_VC1_CTRL_SHIFIT       8
+#define CSI_CTRL_VC1_SHIFT_MASK        0x00000700
+#define CSI_CTRL_VC2_CTRL_SHIFIT       16
+#define CSI_CTRL_VC2_SHIFT_MASK        0x00070000
+#define CSI_CTRL_VC3_CTRL_SHIFIT       24
+#define CSI_CTRL_VC3_SHIFT_MASK        0x07000000
+
+/**
+*DVP CTRL
+*/
+#define DVP_CTRL_REG_CSI_BASE_SHIFT    4
+#define DVP_CTRL_RESET_MASK            0x00010000
+#define DVP_CTRL_FSIN_MASK             0x00020000
+#define DVP_CTRL_PWDN_MASK             0x00040000
+#define DVP_CTRL_VSYNC_EDGE_MASK       0x00000100
+#define DVP_CTRL_HSYNC_EDGE_MASK       0x00001000
+
+/**
+ * MRV_CSI2_NUM_LANES
+ * Config num lanes register [3:0] rw
+ * 0000b - controller off
+ * 0001b - 1 Lane
+ * 0010b - 2 Lanes
+ * 0011b - 3 Lanes
+ * 0100b - 4 Lanes
+ */
+#define MRV_CSI2_NUM_LANES 0x0
+
+/**
+ * MRV_CSI2_LANES_CLK_EN
+ * Configure lanes clock [0]
+ * 0b - disable
+ * 1b - enable
+ */
+#define MRV_CSI2_LANES_CLK_EN 0x4
+
+/**
+ * MRV_CSI2_LANES_DATA_EN
+ * enable/disable lanes data [7:0]
+ * setting bits to a '1' value enable data lane
+ *[0]-data lane 0
+ *[1]-data lane 1
+ *[2]-data lane 2
+ *[3]-data lane 3
+ *[4]-data lane 4
+ *[5]-data lane 5
+ *[6]-data lane 6
+ *[7]-data lane 7
+ */
+#define MRV_CSI2_LANES_DATA_EN 0x8
+
+/**
+ * MRV_CSI2_IGNORE_VC
+ * Set this input causes the interface to ignore the Virtual Channel
+ */
+#define MRV_CSI2_IGNORE_VC        0x80
+#define MRV_CSI2_FIFO_SEND_LEVEL  0x88
+#define MRV_CSI2_VID_VSYNC        0x8c
+#define MRV_CSI2_VID_HSYNC_FP     0x90
+#define MRV_CSI2_VID_HSYNC        0x94
+#define MRV_CSI2_VID_HSYNC_BP     0x98
+
+#define IS_VALID_ID(id) (id != -1)
+
+static vvcam_input_dev_attr_t default_config = {
+    .mode = VVCAM_INPUT_MODE_MIPI,
+    .image_rect = {
+        .left = 0,
+        .top =0,
+        .width = 1920,
+        .height = 1080,
+    },
+    .mipi_attr = {
+        .itf_cfg[0] = {
+            .enable = true,
+            .data_type = VVCAM_RAW_12BIT,
+        },
+        .lane_id  = {0, 1, 2, 3, -1, -1, -1, -1},
+    },
+};
+
+static void vvcam_mipi_write_reg(void __iomem *base,
+			uint32_t addr, uint32_t value)
+{
+	writel(value, base + addr);
+}
+
+static uint32_t vvcam_mipi_read_reg(void __iomem *base, uint32_t addr)
+{
+	return readl(base + addr);
+}
+
+static int vvcam_mipi_csi2_cfg(struct vvcam_mipi_dev *mipi_dev,
+                            vvcam_input_dev_attr_t mipi_cfg)
+{
+    int i = 0;
+    uint8_t mipi_lanes = 0;
+    uint32_t data_bits = 0;
+    uint32_t csi_ctrl_value = 0;
+    for (i = 0; i < MIPI_LANE_NUM_MAX; i++) {
+        if (IS_VALID_ID(mipi_cfg.mipi_attr.lane_id[i])){
+            mipi_lanes++;
+        }
+    }
+
+    switch (mipi_lanes) {
+    case 1:
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x01);
+        break;
+    case 2:
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x02);
+        break;
+    case 4:
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x04);
+        break;
+    case 8:
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_NUM_LANES, 0x04);
+        break;
+    default:
+        return -EINVAL;
+    }
+
+    switch (mipi_cfg.mipi_attr.itf_cfg[0].data_type) {
+    case VVCAM_RAW_8BIT:
+        data_bits =  8;
+        break;
+    case VVCAM_RAW_10BIT:
+        data_bits =  10;
+        break;
+    case VVCAM_RAW_12BIT:
+        data_bits =  12;
+        break;
+    case VVCAM_RAW_14BIT:
+        data_bits =  14;
+        break;
+    case VVCAM_RAW_16BIT:
+        data_bits =  16;
+        break;
+    case VVCAM_RAW_20BIT:
+        data_bits =  20;
+        break;
+    case VVCAM_RAW_24BIT:
+        data_bits =  24;
+        break;
+    default:
+        return -EINVAL;
+    }
+
+    if (mipi_dev->bus_width < data_bits) {
+        return -EINVAL;
+    }
+    csi_ctrl_value =  vvcam_mipi_read_reg(mipi_dev->csi_ctrl_reg, 0);
+    csi_ctrl_value &= (~0x07);
+
+    if ((mipi_dev->bus_width - data_bits) > 0x07)
+        csi_ctrl_value |= 0x07;
+    else {
+        csi_ctrl_value |= (mipi_dev->bus_width - data_bits);
+    }
+    vvcam_mipi_write_reg(mipi_dev->csi_ctrl_reg, 0, csi_ctrl_value);
+
+    return 0;
+}
+
+static int vvcam_mipi_csi2_set_stream(struct vvcam_mipi_dev *mipi_dev,
+                                    uint32_t enable)
+{
+    int i = 0;
+    uint32_t data_lane_en = 0;
+    if (enable) {
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_LANES_CLK_EN, 1);
+        for (i = 0; i < MIPI_LANE_NUM_MAX; i++) {
+            if (IS_VALID_ID(mipi_dev->mipi_cfg.mipi_attr.lane_id[i])) {
+                switch(mipi_dev->mipi_cfg.mipi_attr.lane_id[i]) {
+                case 0:
+                    data_lane_en |= (1 << 0);
+                    break;
+                case 1:
+                    data_lane_en |= (1 << 1);
+                    break;
+                case 2:
+                    data_lane_en |= (1 << 2);
+                    break;
+                case 3:
+                    data_lane_en |= (1 << 3);
+                    break;
+                case 4:
+                    data_lane_en |= (1 << 4);
+                    break;
+                case 5:
+                    data_lane_en |= (1 << 5);
+                    break;
+                case 6:
+                    data_lane_en |= (1 << 6);
+                    break;
+                case 7:
+                    data_lane_en |= (1 << 7);
+                    break;
+                default:
+                    break;
+                }
+            }
+        }
+        vvcam_mipi_write_reg(mipi_dev->base,
+                        MRV_CSI2_LANES_DATA_EN, data_lane_en);
+    } else {
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_LANES_CLK_EN, 0);
+        vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_LANES_DATA_EN, 0);
+    }
+    return 0;
+}
+
+int vvcam_mipi_default_cfg(struct vvcam_mipi_dev *mipi_dev)
+{
+    uint32_t csi_ctrl_value = 0;
+    uint32_t tpg_reg = 0;
+
+    memcpy(&mipi_dev->mipi_cfg, &default_config, sizeof(vvcam_input_dev_attr_t));
+
+    vvcam_mipi_reset(mipi_dev, 1);
+    msleep(10);
+    vvcam_mipi_reset(mipi_dev, 0);
+
+    vvcam_mipi_set_stream(mipi_dev, 0);
+
+    if ((((mipi_dev->id == 0) || (mipi_dev->id == 1)) && CSI_CTRL_MUX_T0T1) ||
+        (((mipi_dev->id == 2) || (mipi_dev->id == 3)) && CSI_CTRL_MUX_T2T3)) {
+        csi_ctrl_value = vvcam_mipi_read_reg(mipi_dev->csi_ctrl_reg, 0);
+        csi_ctrl_value |= CSI_CTRL_MUX_T_MASK;
+        vvcam_mipi_write_reg(mipi_dev->csi_ctrl_reg, 0, csi_ctrl_value);
+    } else {
+        csi_ctrl_value = vvcam_mipi_read_reg(mipi_dev->csi_ctrl_reg, 0);
+        csi_ctrl_value &= ~CSI_CTRL_MUX_T_MASK;
+        vvcam_mipi_write_reg(mipi_dev->csi_ctrl_reg, 0, csi_ctrl_value);
+    }
+
+#ifdef INPUT_CSI_MUX_8000L
+    vvcam_mipi_write_reg(mipi_dev->csi_mux_reg, 0, 0x80000000);
+#else
+    vvcam_mipi_write_reg(mipi_dev->csi_mux_reg, 0, 0x00000000);
+#endif
+
+    tpg_reg = vvcam_mipi_read_reg(mipi_dev->tpg_reg, 0);
+    tpg_reg  |= 0x210;
+    tpg_reg &= ~TPG_FRAME_GATE_MASK;
+    tpg_reg |= TPG_MUX_S_MASK;
+
+    vvcam_mipi_write_reg(mipi_dev->tpg_reg, 0, tpg_reg);
+
+    vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_IGNORE_VC, 0x01);
+    vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_FIFO_SEND_LEVEL, 0x41);
+    vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_VID_VSYNC, 0x20);
+    vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_VID_HSYNC_FP, 0x20);
+    vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_VID_HSYNC, 0x01);
+    vvcam_mipi_write_reg(mipi_dev->base, MRV_CSI2_VID_HSYNC_BP, 0x20);
+
+    mipi_dev->bus_width = VVCAM_CSI_BUS_WIDTH;
+    vvcam_mipi_set_cfg(mipi_dev, mipi_dev->mipi_cfg);
+
+    return 0;
+}
+
+static int vvcam_dvp_ctrl_cfg(struct vvcam_mipi_dev *mipi_dev,
+                        vvcam_input_dev_attr_t dvp_ctrl)
+{
+
+   u32 dvp_ctrl_value = 0;
+   u32 csi_ctrl_value = 0;
+   u32 tpg_reg = 0;
+   bool  pwdn, rst, fsin;
+
+    if(0 < dvp_ctrl.dvp_attr.bitWidth) {
+        //for ISP8000_V2405 dvp csi, data is cut from data[18:31], so the max bitwidth is raw14. For 8-bit sensor input, offset should be 6.
+        mipi_dev->bus_width = 14;
+        if ((mipi_dev->bus_width - dvp_ctrl.dvp_attr.bitWidth) > 0x07)
+            csi_ctrl_value |= 0x07;
+        else {
+            csi_ctrl_value |= (mipi_dev->bus_width - dvp_ctrl.dvp_attr.bitWidth);
+        }
+        vvcam_mipi_write_reg(mipi_dev->csi_ctrl_reg, 0, csi_ctrl_value);
+        return 0;
+    }
+
+   tpg_reg = vvcam_mipi_read_reg(mipi_dev->tpg_reg, 0);
+   dvp_ctrl_value = vvcam_mipi_read_reg(mipi_dev->base, DVP_CTRL_REG_CSI_BASE_SHIFT);
+
+   //power down sensor config
+   pwdn = !(dvp_ctrl.dvp_attr.powerPin.pwdn);
+   rst = !(dvp_ctrl.dvp_attr.powerPin.rst);
+   fsin = !(dvp_ctrl.dvp_attr.powerPin.fsin);
+   if (pwdn == VVCAM_SYNC_POL_HIGH_ACIVE) {
+	   dvp_ctrl_value &= (~DVP_CTRL_PWDN_MASK);
+   } else {
+	   dvp_ctrl_value |= DVP_CTRL_PWDN_MASK;
+   }
+
+   if (rst == VVCAM_SYNC_POL_HIGH_ACIVE) {
+	   dvp_ctrl_value &= (~DVP_CTRL_RESET_MASK);
+   } else {
+	   dvp_ctrl_value |= DVP_CTRL_RESET_MASK;
+   }
+
+   if (fsin == VVCAM_SYNC_POL_HIGH_ACIVE) {
+	   dvp_ctrl_value &= (~DVP_CTRL_FSIN_MASK);
+   } else {
+	   dvp_ctrl_value |= DVP_CTRL_FSIN_MASK;
+   }
+
+   if (dvp_ctrl.dvp_attr.isPower == true) {
+	   //enbable dvp ctrl
+	   tpg_reg |= TPG_MUX_S_MASK;
+	   vvcam_mipi_write_reg(mipi_dev->tpg_reg, 0, tpg_reg);
+	   //power on sensor
+	   vvcam_mipi_write_reg(mipi_dev->base, DVP_CTRL_REG_CSI_BASE_SHIFT, dvp_ctrl_value);
+   } else {
+        //disable dvp ctrl
+        tpg_reg &= (~TPG_MUX_S_MASK);
+        vvcam_mipi_write_reg(mipi_dev->tpg_reg, 0, tpg_reg);
+        //power down sensor
+        vvcam_mipi_write_reg(mipi_dev->base, DVP_CTRL_REG_CSI_BASE_SHIFT, dvp_ctrl_value);
+   }
+
+   return 0;
+}
+
+static int vvcam_dvp_ctrl_set_stream(struct vvcam_mipi_dev *mipi_dev,
+                                    uint32_t enable)
+{
+    u32 dvp_ctrl_value = 0;
+    u32 tpg_reg = 0;
+
+    tpg_reg = vvcam_mipi_read_reg(mipi_dev->tpg_reg, 0);
+    dvp_ctrl_value = vvcam_mipi_read_reg(mipi_dev->base, DVP_CTRL_REG_CSI_BASE_SHIFT);
+
+    return 0;
+}
+
+int vvcam_mipi_reset(struct vvcam_mipi_dev *mipi_dev, uint32_t val)
+{
+    uint32_t tpg_reg = 0;
+
+    tpg_reg = vvcam_mipi_read_reg(mipi_dev->tpg_reg, 0);
+
+    if (val) {
+        /* reset */
+        tpg_reg &= ~TPG_CSI_RST_MASK;
+    } else {
+        /* unreset */
+        tpg_reg |= TPG_CSI_RST_MASK;
+    }
+
+    vvcam_mipi_write_reg(mipi_dev->tpg_reg, 0, tpg_reg);
+    return 0;
+}
+
+int vvcam_mipi_set_cfg(struct vvcam_mipi_dev *mipi_dev,
+                    vvcam_input_dev_attr_t mipi_cfg)
+{
+    int ret = 0;
+
+    switch (mipi_cfg.mode) {
+    case VVCAM_INPUT_MODE_MIPI:
+        ret = vvcam_mipi_csi2_cfg(mipi_dev, mipi_cfg);
+        break;
+	case VVCAM_INPUT_MODE_DVP:
+		ret = vvcam_dvp_ctrl_cfg(mipi_dev, mipi_cfg);
+		break;
+    case VVCAM_INPUT_MODE_LVDS:
+        ret = -EINVAL;
+        break;
+    case VVCAM_INPUT_MODE_BT1120:
+        ret = -EINVAL;
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    if (ret == 0) {
+        memcpy(&mipi_dev->mipi_cfg, &mipi_cfg, sizeof(mipi_cfg));
+    }
+
+    return ret;
+}
+
+int vvcam_mipi_set_stream(struct vvcam_mipi_dev *mipi_dev, uint32_t enable)
+{
+    int ret = 0;
+
+    switch (mipi_dev->mipi_cfg.mode) {
+    case VVCAM_INPUT_MODE_MIPI:
+        ret = vvcam_mipi_csi2_set_stream(mipi_dev, enable);
+        break;
+	case VVCAM_INPUT_MODE_DVP:
+		ret = vvcam_dvp_ctrl_set_stream(mipi_dev, enable);
+		break;
+    case VVCAM_INPUT_MODE_LVDS:
+        ret = -EINVAL;
+        break;
+    case VVCAM_INPUT_MODE_BT1120:
+        ret = -EINVAL;
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    return ret;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_platform.c b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_platform.c
new file mode 100644
index 000000000000..59c7e5cbf6ef
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_platform.c
@@ -0,0 +1,281 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+
+#include "vvcam_mipi_platform.h"
+
+static int devices_mask = 0x01;
+MODULE_PARM_DESC(devices_mask, "vvcam mipi devices mask");
+
+static void vvcam_mipi_pdev_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+	return;
+}
+
+static struct resource vvcam_mipi0_resource[] = {
+	[0] = {
+        .start = VVCAM_CSI0_BASE + PCIE_BASE,
+        .end   = VVCAM_CSI0_BASE + PCIE_BASE + VVCAM_CSI_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[1] = {
+        .start = VVCAM_TPG0 + PCIE_BASE,
+        .end   = VVCAM_TPG0 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+
+	[2] = {
+#if CSI_CTRL_MUX_T0T1
+        .start = VVCAM_CSI1_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI1_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#else
+        .start = VVCAM_CSI0_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI0_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#endif
+        .flags = IORESOURCE_MEM,
+    },
+    [3] = {
+        .start = VVCAM_CSI0_MUX_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI0_MUX_CTRL + PCIE_BASE + VVCAM_CSI_MUX_CTRL_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device vvcam_mipi0_pdev = {
+	.name          = "vvcam-mipi",
+	.id            = 0,
+	.resource      = vvcam_mipi0_resource,
+    .num_resources = ARRAY_SIZE(vvcam_mipi0_resource),
+	.dev.release   = vvcam_mipi_pdev_release,
+};
+
+static struct resource vvcam_mipi1_resource[] = {
+	[0] = {
+        .start = VVCAM_CSI1_BASE + PCIE_BASE,
+        .end   = VVCAM_CSI1_BASE + PCIE_BASE + VVCAM_CSI_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[1] = {
+        .start = VVCAM_TPG1 + PCIE_BASE,
+        .end   = VVCAM_TPG1 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[2] = {
+#if CSI_CTRL_MUX_T0T1
+        .start = VVCAM_CSI0_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI0_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#else
+        .start = VVCAM_CSI1_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI1_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#endif
+        .flags = IORESOURCE_MEM,
+    },
+    [3] = {
+        .start = VVCAM_CSI1_MUX_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI1_MUX_CTRL + PCIE_BASE + VVCAM_CSI_MUX_CTRL_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device vvcam_mipi1_pdev = {
+	.name          = "vvcam-mipi",
+	.id            = 1,
+	.resource      = vvcam_mipi1_resource,
+    .num_resources = ARRAY_SIZE(vvcam_mipi1_resource),
+	.dev.release   = vvcam_mipi_pdev_release,
+};
+
+static struct resource vvcam_mipi2_resource[] = {
+	[0] = {
+        .start = VVCAM_CSI2_BASE + PCIE_BASE,
+        .end   = VVCAM_CSI2_BASE + PCIE_BASE + VVCAM_CSI_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[1] = {
+        .start = VVCAM_TPG2 + PCIE_BASE,
+        .end   = VVCAM_TPG2 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[2] = {
+#if CSI_CTRL_MUX_T2T3
+        .start = VVCAM_CSI3_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI3_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#else
+        .start = VVCAM_CSI2_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI2_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#endif
+        .flags = IORESOURCE_MEM,
+    },
+    [3] = {
+        .start = VVCAM_CSI2_MUX_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI2_MUX_CTRL + PCIE_BASE + VVCAM_CSI_MUX_CTRL_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device vvcam_mipi2_pdev = {
+	.name          = "vvcam-mipi",
+	.id            = 2,
+	.resource      = vvcam_mipi2_resource,
+    .num_resources = ARRAY_SIZE(vvcam_mipi2_resource),
+	.dev.release   = vvcam_mipi_pdev_release,
+};
+
+static struct resource vvcam_mipi3_resource[] = {
+	[0] = {
+        .start = VVCAM_CSI3_BASE + PCIE_BASE,
+        .end   = VVCAM_CSI3_BASE + PCIE_BASE + VVCAM_CSI_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[1] = {
+        .start = VVCAM_TPG3+ PCIE_BASE,
+        .end   = VVCAM_TPG3 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+	[2] = {
+#if CSI_CTRL_MUX_T2T3
+        .start = VVCAM_CSI2_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI2_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#else
+        .start = VVCAM_CSI3_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI3_CTRL + PCIE_BASE + VVCAM_CSI_CTRL_SIZE - 1,
+#endif
+        .flags = IORESOURCE_MEM,
+    },
+    [3] = {
+        .start = VVCAM_CSI3_MUX_CTRL + PCIE_BASE,
+        .end   = VVCAM_CSI3_MUX_CTRL + PCIE_BASE + VVCAM_CSI_MUX_CTRL_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+};
+
+static struct platform_device vvcam_mipi3_pdev = {
+	.name          = "vvcam-mipi",
+	.id            = 3,
+	.resource      = vvcam_mipi3_resource,
+    .num_resources = ARRAY_SIZE(vvcam_mipi3_resource),
+	.dev.release   = vvcam_mipi_pdev_release,
+};
+
+int vvcam_mipi_platform_device_register(void)
+{
+    int ret = 0;
+    int devs_mask = 0;
+    if (devices_mask & 0x01) {
+        ret = platform_device_register(&vvcam_mipi0_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x01;
+    }
+
+    if (devices_mask & 0x02) {
+        ret = platform_device_register(&vvcam_mipi1_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x02;
+    }
+
+    if (devices_mask & 0x04) {
+        ret = platform_device_register(&vvcam_mipi2_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x04;
+    }
+
+    if (devices_mask & 0x08) {
+        ret = platform_device_register(&vvcam_mipi3_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x08;
+    }
+
+    return ret;
+
+err_register:
+    if (devs_mask & 0x01)
+		platform_device_unregister(&vvcam_mipi0_pdev);
+	if (devs_mask & 0x02)
+		platform_device_unregister(&vvcam_mipi1_pdev);
+	if (devs_mask & 0x04)
+		platform_device_unregister(&vvcam_mipi2_pdev);
+	if (devs_mask & 0x08)
+		platform_device_unregister(&vvcam_mipi3_pdev);
+
+    return ret;
+}
+
+void vvcam_mipi_platform_device_unregister(void)
+{
+    if (devices_mask & 0x01)
+		platform_device_unregister(&vvcam_mipi0_pdev);
+	if (devices_mask & 0x02)
+		platform_device_unregister(&vvcam_mipi1_pdev);
+	if (devices_mask & 0x04)
+		platform_device_unregister(&vvcam_mipi2_pdev);
+	if (devices_mask & 0x08)
+		platform_device_unregister(&vvcam_mipi3_pdev);
+
+    return;
+}
+
+module_param(devices_mask, int, 0644);
diff --git a/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_platform.h b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_platform.h
new file mode 100644
index 000000000000..fff239a5fd5c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/mipi/vvcam_mipi_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_MIPI_PLATFORM_H__
+#define __VVCAM_MIPI_PLATFORM_H__
+
+int vvcam_mipi_platform_device_register(void);
+void vvcam_mipi_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/rmmod.sh b/drivers/media/platform/eswin/isp/vvcam/rmmod.sh
new file mode 100644
index 000000000000..547e75253c81
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/rmmod.sh
@@ -0,0 +1,90 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+#!/bin/sh
+
+param_mss="false"
+param_videoin="false"
+
+if [ -n "$1" ];then
+    if [ "$1" = "mss" ];then
+        param_mss="true"
+    elif [ "$1" = "vi" ];then
+        param_videoin="true"
+    fi
+fi
+
+if [ -n "$2" ];then
+    if [ "$2" = "mss" ];then
+        param_mss="true"
+    elif [ "$2" = "vi" ];then
+        param_videoin="true"
+    fi
+fi
+
+rmmod ./i2c/vvcam_i2c.ko
+rmmod ./mipi/vvcam_mipi.ko
+rmmod ./isp/vvcam_isp.ko
+rmmod ./vb/vvcam_vb.ko
+
+# remove mi_subsystem driver.
+if [ "$param_mss" = "true" ];then
+    rmmod ./mi_subsystem/vvcam_mss.ko
+fi
+
+# remove video in driver.
+if [ "$param_videoin" = "true" ];then
+    rmmod ./videoin/vvcam_vi.ko
+fi
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/Makefile b/drivers/media/platform/eswin/isp/vvcam/soc/Makefile
new file mode 100644
index 000000000000..2a96e7d42173
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/Makefile
@@ -0,0 +1,29 @@
+ifeq ($(VVCAM_PATH),)
+	VVCAM_PATH:=$(shell cd $(shell pwd)/../; pwd)
+	export VVCAM_PATH
+endif
+
+
+include $(PWD)/../Param.mk
+TARGET = vvcam_soc
+
+obj-m +=$(TARGET).o
+$(TARGET)-objs += vvcam_soc_driver.o
+$(TARGET)-objs += soc_ioctl.o
+$(TARGET)-objs += vivsoc_hub.o
+$(TARGET)-objs += vsi_core_gen6.o
+$(TARGET)-objs += vivsoc_dvp2axi.o
+
+PWD :=$(shell pwd)
+
+all:
+	make -C $(KERNEL_DIR) M=$(PWD) modules
+	@[ -e $(INSTALL_DIR)/lib ] || mkdir $(INSTALL_DIR)/lib -p
+	@cp $(TARGET).ko $(INSTALL_DIR)/lib
+modules_install:
+	make -C $(KERNEL_DIR) M=$(PWD) modules_install
+clean:
+	rm -rf $($(TARGET)-objs)
+	make -C $(KERNEL_DIR) M=$(PWD) clean
+	@rm $(INSTALL_DIR)/lib/$(TARGET).ko
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/soc_ioctl.c b/drivers/media/platform/eswin/isp/vvcam/soc/soc_ioctl.c
new file mode 100644
index 000000000000..8417b6ea185a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/soc_ioctl.c
@@ -0,0 +1,389 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+
+#ifndef __KERNEL__
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <errno.h>
+
+#define pr_info printf
+#define pr_err printf
+#define copy_from_user(a, b, c) soc_copy_data(a, b, c)
+#define copy_to_user(a, b, c) soc_copy_data(a, b, c)
+#define __user
+#define __iomem
+
+void soc_copy_data(void *dst, void *src, int size)
+{
+	if (dst != src)
+		memcpy(dst, src, size);
+}
+
+#else // __KERNEL__
+#include <linux/module.h>	/* Module support */
+#include <linux/uaccess.h>
+
+#endif
+
+#include "soc_ioctl.h"
+#include "vivsoc_hub.h"
+
+#define CHECK_COPY_RETVAL(retval) do { \
+	if (retval < 0) { \
+		pr_err("%s %d: copy from or to user failed\n", __func__, __LINE__); \
+		return -EFAULT; \
+	} \
+} while(0);
+
+long soc_priv_ioctl(struct visoc_pdata *pdata, unsigned int cmd,
+		    void __user *args)
+{
+	int ret = -1, retval;
+	struct soc_control_context soc_ctrl;
+	struct vvcam_soc_dev *dev;
+	struct vvcam_soc_driver_dev *pdriver_dev;
+
+	pdriver_dev = pdata->pdrv_dev;
+	dev = pdriver_dev->private;
+
+	if (!dev) {
+		return ret;
+	}
+
+	switch (cmd) {
+		/* ISP part */
+	case VVSOC_IOC_S_RESET_ISP:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_isp_reset(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_POWER_ISP:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_isp_s_power_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_POWER_ISP:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_isp_g_power_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	case VVSOC_IOC_S_CLOCK_ISP:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_isp_s_clock_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_CLOCK_ISP:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_isp_g_clock_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+		/* DWE part */
+	case VVSOC_IOC_S_RESET_DWE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_dwe_reset(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_POWER_DWE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_dwe_s_power_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_POWER_DWE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_dwe_g_power_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	case VVSOC_IOC_S_CLOCK_DWE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_dwe_s_clock_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_CLOCK_DWE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_isp_g_clock_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+		/* VSE part */
+	case VVSOC_IOC_S_RESET_VSE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_vse_reset(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_POWER_VSE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_vse_s_power_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_POWER_VSE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_vse_g_power_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	case VVSOC_IOC_S_CLOCK_VSE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_vse_s_clock_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_CLOCK_VSE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_vse_g_clock_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+		/* CSI part */
+	case VVSOC_IOC_S_RESET_CSI:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_csi_reset(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_POWER_CSI:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_csi_s_power_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_POWER_CSI:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_csi_g_power_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	case VVSOC_IOC_S_CLOCK_CSI:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_csi_s_clock_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_CLOCK_CSI:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_csi_g_clock_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+		/* sensor part */
+	case VVSOC_IOC_S_RESET_SENSOR:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_sensor_reset(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_POWER_SENSOR:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_sensor_s_power_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_POWER_SENSOR:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_sensor_g_power_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	case VVSOC_IOC_S_CLOCK_SENSOR:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_sensor_s_clock_ctrl(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_G_CLOCK_SENSOR:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_sensor_g_clock_ctrl(dev, &soc_ctrl);
+		retval = copy_to_user(args, &soc_ctrl, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	/* dvp part*/
+	case VVSOC_IOC_S_RESET_DVP:
+	case VVSOC_IOC_S_POWER_DVP:
+	case VVSOC_IOC_G_POWER_DVP:
+	case VVSOC_IOC_S_CLOCK_DVP:
+	case VVSOC_IOC_G_CLOCK_DVP:
+		ret = 0;
+		break;
+
+	/* vi top multi2isp */
+	case VVSOC_IOC_S_MULTI2ISP_SIZE:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_multi2isp_imgsize(dev, &soc_ctrl);
+		break;
+
+	/* vi top multi2isp */
+	case VVSOC_IOC_S_IMX327_HDR:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_imx327_hdr(dev, &soc_ctrl);
+		break;
+
+	/* dvp2axi */
+	case VVSOC_IOC_S_DVP2AXI_RESET:
+		ret = vivsoc_hub_s_reset_dvp2axi(pdriver_dev, dev);
+		break;
+
+	case VVSOC_IOC_S_DVP2AXI_IODVP_SEL:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		vivsoc_hub_s_dvp2axi_iodvp_sel(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_DVP2AXI_CHN_ATTR:
+		struct dvp2axi_chn_attr chn_attr;
+		retval = copy_from_user(&chn_attr, args, sizeof(chn_attr));
+		CHECK_COPY_RETVAL(retval);
+		pdata->dvp_idx = chn_attr.device_idx;
+		ret = vivsoc_hub_s_dvp2axi_chn_attr(dev, &chn_attr);
+		break;
+
+	case VVSOC_IOC_S_DVP2AXI_EMD_ATTR:
+		struct dvp2axi_emd_chn_attr emd_chn_attr;
+		retval = copy_from_user(&emd_chn_attr, args, sizeof(emd_chn_attr));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_dvp2axi_emd_chn_attr(dev, &emd_chn_attr);
+		break;
+
+	case VVSOC_IOC_S_DVP2AXI_IOVA_GET:
+		struct dvp2axi_dmabuf_attr dmabuf_attr;
+		retval = copy_from_user(&dmabuf_attr, args, sizeof(dmabuf_attr));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_dvp2axi_iova_get(pdata, &dmabuf_attr);
+		retval = copy_to_user(args, &dmabuf_attr, sizeof(dmabuf_attr));
+		CHECK_COPY_RETVAL(retval);
+		break;
+
+	case VVSOC_IOC_S_DVP2AXI_IOVA_PUT:
+		int dmabuf_fd;
+		retval = copy_from_user(&dmabuf_fd, args, sizeof(dmabuf_fd));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_dvp2axi_iova_put(pdata, dmabuf_fd);
+		break;
+
+	case VVSOC_IOC_S_PHY_MODE:
+		int phy_mode = 0;
+		retval = copy_from_user(&phy_mode, args, sizeof(phy_mode));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_phy_mode(dev, phy_mode);
+		break;
+
+	case VVSOC_IOC_S_CONTROLLER_SEL:
+		retval = copy_from_user(&soc_ctrl, args, sizeof(soc_ctrl));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_controller_sel(dev, &soc_ctrl);
+		break;
+
+	case VVSOC_IOC_S_ISP_DVP_SEL:
+		struct isp_dvp_attr dvp_attr;
+		retval = copy_from_user(&dvp_attr, args, sizeof(dvp_attr));
+		CHECK_COPY_RETVAL(retval);
+		ret = vivsoc_hub_s_isp_dvp_sel(dev, &dvp_attr);
+		break;
+
+	default:
+		pr_err("unsupported command %d", cmd);
+		break;
+	}
+
+	return ret;
+}
+
+extern struct vvcam_soc_function_s gen6_soc_function;
+
+int vvnative_soc_init(struct vvcam_soc_dev *dev)
+{
+	if (dev == NULL) {
+		pr_err("[%s] dev is NULL\n", __func__);
+		return -1;
+	}
+
+	vivsoc_register_hardware(dev, &gen6_soc_function);
+	return 0;
+}
+
+int vvnative_soc_deinit(struct vvcam_soc_dev *dev)
+{
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/soc_ioctl.h b/drivers/media/platform/eswin/isp/vvcam/soc/soc_ioctl.h
new file mode 100644
index 000000000000..f9d4bd066d9e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/soc_ioctl.h
@@ -0,0 +1,360 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+#ifndef _SOC_IOC_H_
+#define _SOC_IOC_H_
+
+#ifdef __KERNEL__
+#include <linux/cdev.h>
+#include <linux/dmabuf-heap-import-helper.h>
+#include "vivsoc_dvp2axi.h"
+#else
+#include <stdint.h>
+#endif
+
+#include <linux/ioctl.h>
+
+//vi top register,base address 0x5103_0000
+#define VI_REG_PHY_CONNECT_MODE		0x0000
+#define VI_REG_CONTROLLER_SEL		0x0004
+#define VI_REG_ISP_DVP_SEL			0x0008
+#define VI_REG_ISP0_DVP0_SIZE	 	0x000c
+#define VI_REG_ISP0_DVP1_SIZE	 	0x0010
+#define VI_REG_ISP1_DVP0_SIZE	 	0x0014
+#define VI_REG_ISP1_DVP1_SIZE	 	0x0018
+#define VI_REG_MULTI2ISP_BLANK	 	0x001c
+#define VI_REG_MULTI2ISP0_DVP0	 	0x0020
+#define VI_REG_MULTI2ISP0_DVP1	 	0x0024
+#define VI_REG_MULTI2ISP1_DVP0	 	0x0028
+#define VI_REG_MULTI2ISP1_DVP1	 	0x002c
+
+//VI_REG_PHY_CONNECT_MODE shift
+#define IMX327_HDR_RAW12_5			15
+#define IMX327_HDR_EN_5				14
+#define IMX327_HDR_RAW12_4			13
+#define IMX327_HDR_EN_4				12
+#define IMX327_HDR_RAW12_3			11
+#define IMX327_HDR_EN_3				10
+#define IMX327_HDR_RAW12_2			9
+#define IMX327_HDR_EN_2				8
+#define IMX327_HDR_RAW12_1			7
+#define IMX327_HDR_EN_1				6
+#define IMX327_HDR_RAW12_0			5
+#define IMX327_HDR_EN_0				4
+#define MIPI_CSI_TPG_CLOCK_MUX_EN 	3
+#define PHY_MODE					0
+/*  PHY_MODE bit0-2:
+phy connection mode settings , one phy for 2lanes:
+0: phy0,1,2,3 for 8lanes one sensor, phy4,5 for 4lanes one sensor
+1: phy0,1,2,3 for 8lanes sensor, phy4 for 2lanes sensor, phy5 for 2lanes sensor
+2: phy0,1 for 4lanes sensor, phy2,3 for 4lanes sensor, phy4,5 for 4lanes sensor
+3: phy0,1 for 4lanes senor, phy2,3 for 4lanes sensor, phy4 for 2lanes sensor, phy5 for 2lanes sensor
+4: phy0,1 for 4lanes sensor, phy2 for 2lanes sensor, phy3 for 2lanes sensor, phy4 for 2lanes sensor, phy5 for 2lanes sensor
+5. phy0 for 2lanes sensor, phy1 for 2lanes sensor, phy2 for 2lanes sensor phy3 for 2lanes sensor, phy4 for 2lanes sensor phy5 for 2lanes sensor
+[note] 8lanes just for sub lvds/slvs/hispi, mipi csi can't support 8lanes
+*/
+#define PHY_MODE_LANE_8_4					0
+#define PHY_MODE_LANE_8_2_2					1
+#define PHY_MODE_LANE_4_4_4					2
+#define PHY_MODE_LANE_4_4_2_2				3
+#define PHY_MODE_LANE_4_2_2_2_2				4
+#define PHY_MODE_LANE_2_2_2_2_2_2			5
+
+//VI_REG_CONTROLLER_SEL 0:mipi 1:others
+#define CSI5_VICAP5_SEL		5
+#define CSI4_VICAP4_SEL		4
+#define CSI3_VICAP3_SEL		3
+#define CSI2_VICAP2_SEL		2
+#define CSI1_VICAP1_SEL		1
+#define CSI0_VICAP0_SEL		0
+
+#define MIPI				0
+#define MULTI_PROTOCOL 		1
+
+//VI_REG_ISP_DVP_SEL
+#define ISP_DVP_SEL_BITS    3
+#define ISP0_DVP0_SEL_LSB   0
+#define ISP0_DVP1_SEL_LSB   3
+#define ISP0_DVP2_SEL_LSB   12
+#define ISP0_DVP3_SEL_LSB   15
+#define ISP1_DVP0_SEL_LSB   6
+#define ISP1_DVP1_SEL_LSB   9
+#define ISP1_DVP2_SEL_LSB   18
+#define ISP1_DVP3_SEL_LSB   21
+
+enum vi_cntl_type {
+	CONTROLLER0 = 0,
+	CONTROLLER1,
+	CONTROLLER2,
+	CONTROLLER3,
+	CONTROLLER4,
+	CONTROLLER5,
+	IO_DVP0,
+	IO_DVP1,
+	CONTROLLER_BUTT,
+};
+
+#define POL_HIGH_ACTVIE 	0
+#define POL_LOW_ACTIVE		1
+
+#define IO_DVP1_VALID_POL	29
+#define IO_DVP1_HSYNC_POL	28
+#define IO_DVP1_VSYNC_POL	27
+#define IO_DVP0_VALID_POL	26
+#define IO_DVP0_HSYNC_POL	25
+#define IO_DVP0_VSYNC_POL	24
+#define ISP1_DVP3_SEL  		21
+#define ISP1_DVP2_SEL  		18
+#define ISP0_DVP3_SEL  		15
+#define ISP0_DVP2_SEL  		12
+#define ISP1_DVP1_SEL  		9
+#define ISP1_DVP0_SEL  		6
+#define ISP0_DVP1_SEL  		3
+#define ISP0_DVP0_SEL  		0
+
+enum {
+	VVSOC_IOC_S_RESET_ISP = _IO('r', 0),
+	VVSOC_IOC_S_POWER_ISP,
+	VVSOC_IOC_G_POWER_ISP,
+	VVSOC_IOC_S_CLOCK_ISP,
+	VVSOC_IOC_G_CLOCK_ISP,
+
+	VVSOC_IOC_S_RESET_DWE,
+	VVSOC_IOC_S_POWER_DWE,
+	VVSOC_IOC_G_POWER_DWE,
+	VVSOC_IOC_S_CLOCK_DWE,
+	VVSOC_IOC_G_CLOCK_DWE,
+
+	VVSOC_IOC_S_RESET_VSE,
+	VVSOC_IOC_S_POWER_VSE,
+	VVSOC_IOC_G_POWER_VSE,
+	VVSOC_IOC_S_CLOCK_VSE,
+	VVSOC_IOC_G_CLOCK_VSE,
+
+	VVSOC_IOC_S_RESET_CSI,
+	VVSOC_IOC_S_POWER_CSI,
+	VVSOC_IOC_G_POWER_CSI,
+	VVSOC_IOC_S_CLOCK_CSI,
+	VVSOC_IOC_G_CLOCK_CSI,
+
+	VVSOC_IOC_S_RESET_SENSOR,
+	VVSOC_IOC_S_POWER_SENSOR,
+	VVSOC_IOC_G_POWER_SENSOR,
+	VVSOC_IOC_S_CLOCK_SENSOR,
+	VVSOC_IOC_G_CLOCK_SENSOR,
+
+	VVSOC_IOC_S_RESET_DVP,
+	VVSOC_IOC_S_POWER_DVP,
+	VVSOC_IOC_G_POWER_DVP,
+	VVSOC_IOC_S_CLOCK_DVP,
+	VVSOC_IOC_G_CLOCK_DVP,
+
+	VVSOC_IOC_S_MULTI2ISP_SIZE,
+	VVSOC_IOC_S_IMX327_HDR,
+
+	VVSOC_IOC_S_DVP2AXI_RESET,
+	VVSOC_IOC_S_DVP2AXI_IODVP_SEL,
+	VVSOC_IOC_S_DVP2AXI_CHN_ATTR,
+	VVSOC_IOC_S_DVP2AXI_EMD_ATTR,
+	VVSOC_IOC_S_DVP2AXI_IOVA_GET,
+	VVSOC_IOC_S_DVP2AXI_IOVA_PUT,
+
+	VVSOC_IOC_S_PHY_MODE,
+	VVSOC_IOC_S_CONTROLLER_SEL,
+	VVSOC_IOC_S_ISP_DVP_SEL,
+
+	VVSOC_IOC_MAX,
+};
+
+struct soc_control_context {
+	uint32_t device_idx;
+	uint32_t control_value;
+};
+
+struct dvp2axi_chn_attr {
+	uint32_t device_idx;
+	uint8_t enable;
+	uint8_t pixel_width;
+	uint32_t hs_num;
+	uint32_t vs_num;
+	uint32_t chn0_baddr;
+	uint32_t chn1_baddr;
+	uint32_t chn2_baddr;
+	uint16_t hs_stride;
+	uint8_t first_id;
+	uint8_t last_id;
+};
+
+struct dvp2axi_emd_chn_attr {
+	uint32_t device_idx;
+	uint32_t emd_baddr;
+	uint16_t emd_stride;
+	uint16_t emd_hsnum;
+};
+
+struct dvp2axi_dmabuf_attr {
+	int dmabuf_fd;
+	unsigned long iova;
+};
+
+struct isp_dvp_attr {
+	uint32_t devId; // 0 or 1
+	enum vi_cntl_type dvp_sel; // 0 ~ 7
+};
+
+struct vvcam_soc_func_s {
+	int (*set_power) (void *, unsigned int, unsigned int);
+	int (*get_power) (void *, unsigned int, unsigned int *);
+	int (*set_reset) (void *, unsigned int, unsigned int);
+	int (*set_clk) (void *, unsigned int, unsigned int);
+	int (*get_clk) (void *, unsigned int, unsigned int *);
+};
+
+struct vvcam_soc_function_s {
+	struct vvcam_soc_func_s isp_func;
+	struct vvcam_soc_func_s dwe_func;
+	struct vvcam_soc_func_s vse_func;
+	struct vvcam_soc_func_s csi_func;
+	struct vvcam_soc_func_s sensor_func;
+};
+
+struct vvcam_soc_access_s {
+	int (*write) (void *ctx, uint32_t address, uint32_t data);
+	int (*read) (void *ctx, uint32_t address, uint32_t *data);
+};
+
+#ifdef __KERNEL__
+
+struct vvcam_soc_dev {
+	void __iomem *base;
+	void __iomem *scu_base;
+	struct soc_control_context isp0;
+	struct soc_control_context isp1;
+	struct soc_control_context dwe;
+	struct soc_control_context vse;
+	struct vvcam_soc_function_s soc_func;
+	struct vvcam_soc_access_s soc_access;
+	void * csi_private;
+	struct dentry *soc_debug_dc;
+	struct dentry *soc_reset;
+	uint32_t dvp2axi_iodvp_en;
+	uint32_t multi2isp_iodvp_en;
+	wait_queue_head_t irq_wait;
+	atomic_t irqc_err;
+	atomic_t irqc[VI_DVP2AXI_DVP_CHNS][VI_DVP2AXI_VIRTUAL_CHNS]; // Record the irq times informed to userspace
+	uint32_t dvp2axi_framesize[VI_DVP2AXI_DVP_CHNS];
+	atomic_t dvp2axi_vchn_fidx[VI_DVP2AXI_DVP_CHNS][VI_DVP2AXI_VIRTUAL_CHNS]; // flush idx, to mask irq
+	atomic_t dvp2axi_vchn_didx[VI_DVP2AXI_DVP_CHNS][VI_DVP2AXI_VIRTUAL_CHNS]; // done idx, to mask irq
+	atomic_t dvp2axi_errirq[VI_DVP2AXI_ERRIRQ_NUM]; // to mask 9 types of err irq, otherwise the flooding irqs will come
+	uint32_t dvp2axi_irqnum[8];
+	uint32_t dvp2axi_intrid[8];
+};
+// internal functions
+
+typedef struct _isp_clk_rst {
+	struct clk            *aclk;
+	struct clk            *cfg_clk;
+	struct clk            *isp_aclk;
+	struct clk            *dvp_clk;
+	struct clk            *phy_cfg;
+	struct clk            *phy_escclk;
+	struct clk            *sht0;
+	struct clk            *sht1;
+	struct clk            *sht2;
+	struct clk            *sht3;
+	struct clk            *sht4;
+	struct clk            *sht5;
+	struct clk            *aclk_mux;
+	struct clk            *dvp_mux;
+	struct clk            *isp_mux;
+	struct clk            *spll0_fout1;
+	struct clk            *vpll_fout1;
+	struct reset_control  *rstc_axi;
+	struct reset_control  *rstc_cfg;
+	struct reset_control  *rstc_isp;
+	struct reset_control  *rstc_dvp;
+	struct reset_control  *rstc_sht0;
+	struct reset_control  *rstc_sht1;
+	struct reset_control  *rstc_sht2;
+	struct reset_control  *rstc_sht3;
+	struct reset_control  *rstc_sht4;
+	struct reset_control  *rstc_sht5;
+} isp_clk_rst_t;
+
+struct vvcam_soc_driver_dev {
+	struct cdev cdev;
+	dev_t devt;
+	struct class *class;
+	struct mutex vvmutex;
+	struct platform_device *pdev;
+	void *private;
+	isp_clk_rst_t isp_crg;
+};
+
+struct visoc_pdata {
+	struct vvcam_soc_driver_dev *pdrv_dev;
+	struct heap_root root;
+	int dvp_idx;
+};
+
+long soc_priv_ioctl(struct visoc_pdata *pdata, unsigned int cmd,
+		    void __user *args);
+int vvnative_soc_init(struct vvcam_soc_dev *dev);
+int vvnative_soc_deinit(struct vvcam_soc_dev *dev);
+int vvcam_soc_irq_init(struct platform_device *pdev, struct vvcam_soc_dev *psoc_dev);
+
+#else
+//User space connections
+
+#endif
+
+#endif // _SOC_IOC_H_
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_dvp2axi.c b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_dvp2axi.c
new file mode 100644
index 000000000000..8f0921fde35e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_dvp2axi.c
@@ -0,0 +1,203 @@
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+
+#include "soc_ioctl.h"
+#include "vivsoc_dvp2axi.h"
+#include "vivsoc_hub.h"
+
+/*
+ * Save frame number from sensor, buffer size should be +1 frame size
+ */
+#define SENSOR_FRAME_NUM 1
+
+struct dvp2axi_mem_attr {
+	unsigned int vchn_mask;
+	unsigned int chn0_reg;
+	unsigned int chn1_reg;
+	unsigned int chn2_reg;
+	unsigned int chn0_baddr;
+	unsigned int chn1_baddr;
+	unsigned int chn2_baddr;
+	unsigned int frame_size;
+	unsigned int intrid;
+};
+
+#define GET_VIRT_CHN(vchn_mask, vchn) do { \
+	if (vchn_mask & 0x1) { vchn = 0; } \
+	if (vchn_mask & 0x2) { vchn = 1; } \
+	if (vchn_mask & 0x4) { vchn = 2; } \
+} while(0)
+
+#define DVP2AXI_IRQ_HANDLE(intr, flush_mask, done_mask, csr0, csr1, csr2, mask_reg, flush_shift, done_shift) do { \
+		unsigned int vchn = 0; \
+		soc_dev->soc_access.read(soc_dev, intr, &int_status); \
+		if (int_status & flush_mask) { \
+			mem_attr.intrid = intrid; \
+			mem_attr.vchn_mask = (int_status & flush_mask) >> flush_shift; \
+			mem_attr.frame_size = soc_dev->dvp2axi_framesize[intrid]; \
+			mem_attr.chn0_reg = csr0; \
+			mem_attr.chn1_reg = csr1; \
+			mem_attr.chn2_reg = csr2; \
+			soc_dev->soc_access.read(soc_dev, csr0, &mem_attr.chn0_baddr); \
+			soc_dev->soc_access.read(soc_dev, csr1, &mem_attr.chn1_baddr); \
+			soc_dev->soc_access.read(soc_dev, csr2, &mem_attr.chn2_baddr); \
+			dvp2axi_update_mem_addr(soc_dev, &mem_attr); \
+			GET_VIRT_CHN(mem_attr.vchn_mask, vchn); \
+			if (atomic_read(&soc_dev->dvp2axi_vchn_fidx[intrid][vchn]) >= SENSOR_FRAME_NUM) \
+				vivsoc_hub_write_part(soc_dev, mask_reg, 1, flush_shift + vchn, 1); \
+		} \
+		if (int_status & done_mask) { \
+			mem_attr.vchn_mask = (int_status & done_mask) >> done_shift; \
+			GET_VIRT_CHN(mem_attr.vchn_mask, vchn); \
+			if (atomic_read(&soc_dev->dvp2axi_vchn_didx[intrid][vchn]) >= SENSOR_FRAME_NUM) \
+				vivsoc_hub_write_part(soc_dev, mask_reg, 1, done_shift + vchn, 1); \
+			else { \
+				wake_up_interruptible(&soc_dev->irq_wait); \
+			} \
+			atomic_inc(&soc_dev->dvp2axi_vchn_didx[intrid][vchn]); \
+		} \
+		soc_dev->soc_access.write(soc_dev, intr, int_status & (flush_mask | done_mask)); \
+	} while(0)
+
+static void dvp2axi_update_mem_addr(struct vvcam_soc_dev *dev, struct dvp2axi_mem_attr *mem_attr)
+{
+	if ((mem_attr->vchn_mask & 0x1) && mem_attr->chn0_baddr) {
+		atomic_inc(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][0]);
+		if (atomic_read(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][0]) <= SENSOR_FRAME_NUM) {
+			dev->soc_access.write(dev, mem_attr->chn0_reg, mem_attr->chn0_baddr +
+				    mem_attr->frame_size * atomic_read(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][0]));
+		}
+	}
+
+	if ((mem_attr->vchn_mask & 0x2) && mem_attr->chn1_baddr) {
+		atomic_inc(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][1]);
+		if (atomic_read(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][1]) <= SENSOR_FRAME_NUM) {
+			dev->soc_access.write(dev, mem_attr->chn1_reg, mem_attr->chn1_baddr +
+				    mem_attr->frame_size * atomic_read(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][1]));
+		}
+	}
+
+	if ((mem_attr->vchn_mask & 0x4) && mem_attr->chn2_baddr) {
+		atomic_inc(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][2]);
+		if (atomic_read(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][2]) <= SENSOR_FRAME_NUM) {
+			dev->soc_access.write(dev, mem_attr->chn2_reg, mem_attr->chn2_baddr +
+				    mem_attr->frame_size * atomic_read(&dev->dvp2axi_vchn_fidx[mem_attr->intrid][2]));
+		}
+	}
+}
+
+static irqreturn_t dvp2axi_irq_handler(int irq, void *dev_id)
+{
+	unsigned int int_status, i;
+	unsigned int intrid = *(int *)dev_id;
+	struct vvcam_soc_dev *soc_dev;
+	struct dvp2axi_mem_attr mem_attr;
+
+	if (intrid > 8) {
+		pr_err("%s: Error interrupt id\n", __func__);
+		return IRQ_NONE;
+	}
+
+	memset(&mem_attr, 0, sizeof(mem_attr));
+	soc_dev = (struct vvcam_soc_dev *)container_of(dev_id,
+			    struct vvcam_soc_dev, dvp2axi_intrid[intrid]);
+	switch(intrid) {
+	case 0:
+		DVP2AXI_IRQ_HANDLE(VI_DVP2AXI_INT0, DVP0_FLUSH_MASK, DVP0_DONE_MASK,
+			    VI_DVP2AXI_CTRL9_CSR, VI_DVP2AXI_CTRL10_CSR, VI_DVP2AXI_CTRL11_CSR, VI_DVP2AXI_INT_MASK0, 0, 9);
+		break;
+	case 1:
+		DVP2AXI_IRQ_HANDLE(VI_DVP2AXI_INT0, DVP1_FLUSH_MASK, DVP1_DONE_MASK,
+			    VI_DVP2AXI_CTRL12_CSR, VI_DVP2AXI_CTRL13_CSR, VI_DVP2AXI_CTRL14_CSR, VI_DVP2AXI_INT_MASK0, 3, 12);
+		break;
+	case 2:
+		DVP2AXI_IRQ_HANDLE(VI_DVP2AXI_INT0, DVP2_FLUSH_MASK, DVP2_DONE_MASK,
+			    VI_DVP2AXI_CTRL15_CSR, VI_DVP2AXI_CTRL16_CSR, VI_DVP2AXI_CTRL17_CSR, VI_DVP2AXI_INT_MASK0, 6, 15);
+		break;
+	case 3:
+		DVP2AXI_IRQ_HANDLE(VI_DVP2AXI_INT1, DVP3_FLUSH_MASK, DVP3_DONE_MASK,
+			    VI_DVP2AXI_CTRL18_CSR, VI_DVP2AXI_CTRL19_CSR, VI_DVP2AXI_CTRL20_CSR, VI_DVP2AXI_INT_MASK1, 0, 9);
+		break;
+	case 4:
+		DVP2AXI_IRQ_HANDLE(VI_DVP2AXI_INT1, DVP4_FLUSH_MASK, DVP4_DONE_MASK,
+			    VI_DVP2AXI_CTRL21_CSR, VI_DVP2AXI_CTRL22_CSR, VI_DVP2AXI_CTRL23_CSR, VI_DVP2AXI_INT_MASK1, 3, 12);
+		break;
+	case 5:
+		DVP2AXI_IRQ_HANDLE(VI_DVP2AXI_INT1, DVP3_FLUSH_MASK, DVP3_DONE_MASK,
+			    VI_DVP2AXI_CTRL24_CSR, VI_DVP2AXI_CTRL25_CSR, VI_DVP2AXI_CTRL26_CSR, VI_DVP2AXI_INT_MASK1, 6, 15);
+		break;
+	case 6:
+	case 7:
+		unsigned int err_status, err_flag = 0;
+
+		soc_dev->soc_access.read(soc_dev, VI_DVP2AXI_INT2, &err_status);
+		if (err_status & 0x1) {
+			err_flag = 1;
+			if (atomic_read(&soc_dev->dvp2axi_errirq[0]) > 0)
+				vivsoc_hub_write_part(soc_dev, VI_DVP2AXI_INT_MASK2, 1, 0, 1);
+			atomic_inc(&soc_dev->dvp2axi_errirq[0]);
+			pr_err("AXI id buffer full\n");
+		}
+
+		if (err_status & 0x2) {
+			err_flag = 1;
+			if (atomic_read(&soc_dev->dvp2axi_errirq[1]) > 0)
+				vivsoc_hub_write_part(soc_dev, VI_DVP2AXI_INT_MASK2, 1, 1, 1);
+			atomic_inc(&soc_dev->dvp2axi_errirq[1]);
+			pr_err("AXI response error\n");
+		}
+
+		for (i = 2; i < 8; i++) {
+			if (err_status & (0x1 << i)) {
+				err_flag = 1;
+				if (atomic_read(&soc_dev->dvp2axi_errirq[i]) > 0)
+					vivsoc_hub_write_part(soc_dev, VI_DVP2AXI_INT_MASK2, 1, i, 1);
+				atomic_inc(&soc_dev->dvp2axi_errirq[i]);
+				pr_err("DVP%d frame error\n", (i - 2));
+			}
+		}
+
+		if (err_flag) {
+			soc_dev->soc_access.write(soc_dev, VI_DVP2AXI_INT2, err_status);
+			atomic_inc(&soc_dev->irqc_err);
+			wake_up_interruptible(&soc_dev->irq_wait);
+		}
+
+		if (err_status & 0x100) {
+			pr_info("AXI id buffer is about to be full\n");
+			if (atomic_read(&soc_dev->dvp2axi_errirq[8]) > 0)
+				vivsoc_hub_write_part(soc_dev, VI_DVP2AXI_INT_MASK2, 1, 8, 1);
+			atomic_inc(&soc_dev->dvp2axi_errirq[8]);
+			soc_dev->soc_access.write(soc_dev, VI_DVP2AXI_INT2, err_status);
+			return IRQ_HANDLED;
+		}
+
+		break;
+	default:
+		return IRQ_NONE;
+	}
+
+	return IRQ_HANDLED;
+}
+
+int vvcam_soc_irq_init(struct platform_device *pdev, struct vvcam_soc_dev *soc_dev)
+{
+	int i;
+
+	for (i = 0; i < 8; i++) {
+		int ret;
+
+		soc_dev->dvp2axi_irqnum[i] = platform_get_irq(pdev, i);
+		soc_dev->dvp2axi_intrid[i] = i;
+
+		ret = devm_request_irq(&pdev->dev, soc_dev->dvp2axi_irqnum[i],
+			    dvp2axi_irq_handler, IRQF_SHARED | IRQF_ONESHOT, "dvp2axi_irq", (void *)&soc_dev->dvp2axi_intrid[i]);
+		if (ret < 0) {
+			pr_err("IRQ request failed for id %d\n", i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_dvp2axi.h b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_dvp2axi.h
new file mode 100644
index 000000000000..3915256f7149
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_dvp2axi.h
@@ -0,0 +1,150 @@
+#ifndef __VIVSOC_DVP2AXI_H__
+#define __VIVSOC_DVP2AXI_H__
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "soc_ioctl.h"
+
+// VI DVP2AXI registers and masks
+#define VI_DVP2AXI_CTRL0_CSR        0x4000
+#define VI_DVP2AXI_RST_DONE_MASK    BIT(14)
+#define VI_DVP2AXI_SOFT_RST_MASK    BIT(6)
+
+#define VI_DVP2AXI_CTRL1_CSR        0x4004
+#define IODVP_MUX_LSB               30
+#define IODVP_MUX_BITS              2
+#define PIX_WIDTH_BITS              5
+#define DVP0_PIX_WIDTH_LSB          20
+#define DVP1_PIX_WIDTH_LSB          25
+
+#define VI_DVP2AXI_CTRL2_CSR        0x4008
+#define DVP2_PIX_WIDTH_LSB          0
+#define DVP3_PIX_WIDTH_LSB          5
+#define DVP4_PIX_WIDTH_LSB          10
+#define DVP5_PIX_WIDTH_LSB          15
+
+#define VI_DVP2AXI_CTRL3_CSR        0x400c
+#define VI_DVP2AXI_CTRL4_CSR        0x4010
+#define VI_DVP2AXI_CTRL5_CSR        0x4014
+#define VI_DVP2AXI_CTRL6_CSR        0x4018
+#define VI_DVP2AXI_CTRL7_CSR        0x401c
+#define VI_DVP2AXI_CTRL8_CSR        0x4020
+
+#define VI_DVP2AXI_CTRL9_CSR        0x4024
+#define VI_DVP2AXI_CTRL10_CSR       0x4028
+#define VI_DVP2AXI_CTRL11_CSR       0x402c
+#define VI_DVP2AXI_CTRL12_CSR       0x4030
+#define VI_DVP2AXI_CTRL13_CSR       0x4034
+#define VI_DVP2AXI_CTRL14_CSR       0x4038
+#define VI_DVP2AXI_CTRL15_CSR       0x403c
+#define VI_DVP2AXI_CTRL16_CSR       0x4040
+#define VI_DVP2AXI_CTRL17_CSR       0x4044
+#define VI_DVP2AXI_CTRL18_CSR       0x4048
+#define VI_DVP2AXI_CTRL19_CSR       0x404c
+#define VI_DVP2AXI_CTRL20_CSR       0x4050
+#define VI_DVP2AXI_CTRL21_CSR       0x4054
+#define VI_DVP2AXI_CTRL22_CSR       0x4058
+#define VI_DVP2AXI_CTRL23_CSR       0x405c
+#define VI_DVP2AXI_CTRL24_CSR       0x4060
+#define VI_DVP2AXI_CTRL25_CSR       0x4064
+#define VI_DVP2AXI_CTRL26_CSR       0x4068
+#define VI_DVP2AXI_CTRL27_CSR       0x406c
+#define VI_DVP2AXI_CTRL28_CSR       0x4070
+#define VI_DVP2AXI_CTRL29_CSR       0x4074
+#define VI_DVP2AXI_CTRL30_CSR       0x4078
+#define VI_DVP2AXI_CTRL31_CSR       0x407c
+#define VI_DVP2AXI_CTRL32_CSR       0x4080
+
+#define STRIDE_BITS                 16
+#define VI_DVP2AXI_CTRL33_CSR       0x4084
+#define DVP0_HS_STRIDE_LSB          0
+#define DVP1_HS_STRIDE_LSB          16
+#define VI_DVP2AXI_CTRL34_CSR       0x4088
+#define DVP2_HS_STRIDE_LSB          0
+#define DVP3_HS_STRIDE_LSB          16
+#define VI_DVP2AXI_CTRL35_CSR       0x408c
+#define DVP4_HS_STRIDE_LSB          0
+#define DVP5_HS_STRIDE_LSB          16
+
+#define VI_DVP2AXI_CTRL36_CSR       0x4090
+#define HDR_ID_BITS                 2
+#define DVP0_FIRST_ID_LSB           0
+#define DVP1_FIRST_ID_LSB           2
+#define DVP2_FIRST_ID_LSB           4
+#define DVP3_FIRST_ID_LSB           6
+#define DVP4_FIRST_ID_LSB           8
+#define DVP5_FIRST_ID_LSB           10
+#define DVP0_LAST_ID_LSB            18
+#define DVP1_LAST_ID_LSB            20
+#define DVP2_LAST_ID_LSB            22
+#define DVP3_LAST_ID_LSB            24
+#define DVP4_LAST_ID_LSB            26
+#define DVP5_LAST_ID_LSB            28
+
+#define VI_DVP2AXI_INT0             0x40a0
+#define DVP0_FLUSH_MASK             0x7
+#define DVP1_FLUSH_MASK             0x38
+#define DVP2_FLUSH_MASK             0x1c0
+#define DVP0_DONE_MASK              0xe00
+#define DVP1_DONE_MASK              0x7000
+#define DVP2_DONE_MASK              0x38000
+
+#define VI_DVP2AXI_INT1             0x40a4
+#define DVP3_FLUSH_MASK             0x7
+#define DVP4_FLUSH_MASK             0x38
+#define DVP5_FLUSH_MASK             0x1c0
+#define DVP3_DONE_MASK              0xe00
+#define DVP4_DONE_MASK              0x7000
+#define DVP5_DONE_MASK              0x38000
+
+#define VI_DVP2AXI_INT2             0x40a8
+#define VI_DVP2AXI_INT_MASK0        0x40ac
+#define DVP0_DONE_MASK_LSB          9
+#define DVP0_FLUSH_MASK_LSB         0
+#define DVP1_DONE_MASK_LSB          12
+#define DVP1_FLUSH_MASK_LSB         3
+#define DVP2_DONE_MASK_LSB          15
+#define DVP2_FLUSH_MASK_LSB         6
+
+#define VI_DVP2AXI_INT_MASK1        0x40b0
+#define DVP3_DONE_MASK_LSB          9
+#define DVP3_FLUSH_MASK_LSB         0
+#define DVP4_DONE_MASK_LSB          12
+#define DVP4_FLUSH_MASK_LSB         3
+#define DVP5_DONE_MASK_LSB          15
+#define DVP5_FLUSH_MASK_LSB         6
+#define VI_DVP2AXI_INT_MASK2        0x40b4
+
+#define VI_DVP2AXI_CTRL37_CSR       0x4094
+#define DVP0_EMD_STRIDE_LSB         0
+#define DVP1_EMD_STRIDE_LSB         16
+#define VI_DVP2AXI_CTRL38_CSR       0x4098
+#define DVP2_EMD_STRIDE_LSB         0
+#define DVP3_EMD_STRIDE_LSB         16
+#define VI_DVP2AXI_CTRL39_CSR       0x409c
+#define DVP4_EMD_STRIDE_LSB         0
+#define DVP5_EMD_STRIDE_LSB         16
+
+#define EMD_HSNUM_BITS              16
+#define VI_DVP2AXI_CTRL40_CSR       0x40e0
+#define DVP0_EMD_HSNUM_LSB          0
+#define DVP1_EMD_HSNUM_LSB          16
+#define VI_DVP2AXI_CTRL41_CSR       0x40e4
+#define DVP2_EMD_HSNUM_LSB          0
+#define DVP3_EMD_HSNUM_LSB          16
+#define VI_DVP2AXI_CTRL42_CSR       0x40e8
+#define DVP4_EMD_HSNUM_LSB          0
+#define DVP5_EMD_HSNUM_LSB          16
+#define VI_DVP2AXI_CTRL43_CSR       0x40ec
+
+/*
+ * Save frame number from sensor, buffer size should be +1 frame size
+ */
+#define SENSOR_FRAME_NUM 1
+
+#define VI_DVP2AXI_ERRIRQ_NUM       9
+#define VI_DVP2AXI_DVP_CHNS         6
+#define VI_DVP2AXI_VIRTUAL_CHNS     4
+
+#endif /* __VIVSOC_DVP2AXI_H__ */
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_hub.c b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_hub.c
new file mode 100644
index 000000000000..77a321a48685
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_hub.c
@@ -0,0 +1,1041 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/delay.h>
+#include <linux/reset.h>
+
+#include "soc_ioctl.h"
+#include "vivsoc_hub.h"
+#include "vsi_core_gen6.h"
+
+int vivsoc_hub_isp_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.isp_func.set_reset == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.isp_func.set_reset(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_isp_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.isp_func.set_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.isp_func.set_power(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_isp_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.isp_func.get_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.isp_func.get_power(dev, soc_ctrl->device_idx,
+					     &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_isp_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.isp_func.set_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.isp_func.set_clk(dev, soc_ctrl->device_idx,
+					   soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_isp_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.isp_func.get_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.isp_func.get_clk(dev, soc_ctrl->device_idx,
+					   &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_dwe_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.dwe_func.set_reset == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.dwe_func.set_reset(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_dwe_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.dwe_func.set_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.dwe_func.set_power(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_dwe_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.dwe_func.get_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.dwe_func.get_power(dev, soc_ctrl->device_idx,
+					     &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_dwe_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.dwe_func.set_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.dwe_func.set_clk(dev, soc_ctrl->device_idx,
+					   soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_dwe_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.dwe_func.get_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.dwe_func.get_clk(dev, soc_ctrl->device_idx,
+					   &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_vse_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.vse_func.set_reset == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.vse_func.set_reset(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_vse_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.vse_func.set_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.vse_func.set_power(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_vse_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.vse_func.get_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.vse_func.get_power(dev, soc_ctrl->device_idx,
+					     &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_vse_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.vse_func.set_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.vse_func.set_clk(dev, soc_ctrl->device_idx,
+					   soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_vse_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.vse_func.get_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.vse_func.get_clk(dev, soc_ctrl->device_idx,
+					   &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_csi_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.csi_func.set_reset == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.csi_func.set_reset(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_csi_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.csi_func.set_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.csi_func.set_power(dev, soc_ctrl->device_idx,
+					     soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_csi_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.csi_func.get_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.csi_func.get_power(dev, soc_ctrl->device_idx,
+					     &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_csi_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.csi_func.set_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.csi_func.set_clk(dev, soc_ctrl->device_idx,
+					   soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_csi_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.csi_func.get_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.csi_func.get_clk(dev, soc_ctrl->device_idx,
+					   &soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_sensor_reset(struct vvcam_soc_dev *dev,
+			    struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.sensor_func.set_reset == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.sensor_func.set_reset(dev, soc_ctrl->device_idx,
+						soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_sensor_s_power_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.sensor_func.set_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.sensor_func.set_power(dev, soc_ctrl->device_idx,
+						soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_sensor_g_power_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.sensor_func.get_power == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.sensor_func.get_power(dev, soc_ctrl->device_idx,
+						&soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_sensor_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.sensor_func.set_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.sensor_func.set_clk(dev, soc_ctrl->device_idx,
+					      soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_sensor_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	if (dev->soc_func.sensor_func.get_clk == NULL) {
+		pr_err("[%s] function is null\n", __func__);
+		return -1;
+	}
+
+	ret =
+	    dev->soc_func.sensor_func.get_clk(dev, soc_ctrl->device_idx,
+					      &soc_ctrl->control_value);
+
+	return ret;
+}
+
+// OYX_TODO: choose reg base on the devId
+int vivsoc_hub_s_multi2isp_imgsize(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	dev->soc_access.write(dev, VI_REG_ISP0_DVP0_SIZE, soc_ctrl->control_value);
+	dev->soc_access.write(dev, VI_REG_ISP0_DVP1_SIZE, soc_ctrl->control_value);
+	dev->soc_access.write(dev, VI_REG_ISP1_DVP0_SIZE, soc_ctrl->control_value);
+	dev->soc_access.write(dev, VI_REG_ISP1_DVP1_SIZE, soc_ctrl->control_value);
+
+	return ret;
+}
+
+int vivsoc_hub_write_part(struct vvcam_soc_dev *dev, uint32_t address, uint32_t data, uint8_t shift, uint8_t width)
+{
+	uint32_t temp = 0;
+	uint32_t mask = (1 << width) - 1;
+
+	dev->soc_access.read(dev, address, &temp);
+//	pr_info("[%s]read 0x%x:0x%x\n", __func__, address, temp);
+
+	temp &= ~(mask << shift);
+	temp |= (data & mask) << shift;
+
+	dev->soc_access.write(dev, address, temp);
+//	pr_info("[%s]write 0x%x:0x%x\n", __func__, address, temp);
+
+	return 0;
+}
+
+int vivsoc_hub_s_imx327_hdr(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	int sensor_index = 0;
+	char hdr_en = 0;
+	uint32_t width = 0;
+	uint32_t shift = 0;
+
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	sensor_index = soc_ctrl->device_idx;
+	hdr_en = soc_ctrl->control_value &0x01;
+	width = 1;
+
+	shift = IMX327_HDR_EN_0 + sensor_index*2;
+	vivsoc_hub_write_part(dev, VI_REG_PHY_CONNECT_MODE, hdr_en, shift, width);
+
+	shift = IMX327_HDR_RAW12_0 + sensor_index*2;
+	vivsoc_hub_write_part(dev, VI_REG_PHY_CONNECT_MODE, hdr_en, shift, width);
+
+	return ret;
+}
+
+int vivsoc_hub_g_multi2isp_imgsize(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl)
+{
+	int ret = 0;
+	if ((dev == NULL) || (soc_ctrl == NULL)) {
+		pr_err("[%s]:dev is null\n", __func__);
+		return -1;
+	}
+
+	dev->soc_access.read(dev, 0x0c, &(soc_ctrl->control_value));
+
+	return ret;
+}
+
+int vivsoc_hub_s_reset_dvp2axi(struct vvcam_soc_driver_dev *pdriver_dev, struct vvcam_soc_dev *dev)
+{
+	int val, ret, rst_count = 0;
+
+	/*
+	 * Ensure no data from sensor before reseting dvp2axi
+	 */
+
+	dev->soc_access.read(dev, VI_DVP2AXI_CTRL0_CSR, &val);
+	val &= ~VI_DVP2AXI_SOFT_RST_MASK;
+	dev->soc_access.write(dev, VI_DVP2AXI_CTRL0_CSR, val);
+
+	do {
+		fsleep(10);
+		rst_count++;
+		dev->soc_access.read(dev, VI_DVP2AXI_CTRL0_CSR, &val);
+		pr_info("... dvp2axi reseting ...\n");
+		if (rst_count > 10) {
+			pr_info("... dvp2axi crg reseting ...\n");
+			ret = reset_control_reset(pdriver_dev->isp_crg.rstc_dvp);
+			WARN_ON(0 != ret);
+
+			if (rst_count > 20) {
+				pr_err("dvp2axi reset failed, stop sensor first\n");
+				return -1;
+			}
+		}
+	} while((val & VI_DVP2AXI_RST_DONE_MASK) != VI_DVP2AXI_RST_DONE_MASK);
+
+	dev->soc_access.read(dev, VI_DVP2AXI_CTRL0_CSR, &val);
+	val |= VI_DVP2AXI_SOFT_RST_MASK;
+	dev->soc_access.write(dev, VI_DVP2AXI_CTRL0_CSR, val);
+
+	return 0;
+}
+
+int vivsoc_hub_s_dvp2axi_iodvp_sel(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl)
+{
+	if (soc_ctrl->control_value) {
+		dev->dvp2axi_iodvp_en = 1;
+	} else {
+		dev->dvp2axi_iodvp_en = 0;
+	}
+
+	vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL1_CSR,
+		    dev->dvp2axi_iodvp_en, IODVP_MUX_LSB, IODVP_MUX_BITS);
+
+	return 0;
+}
+
+int vivsoc_hub_s_dvp2axi_chn_attr(struct vvcam_soc_dev *dev,
+					struct dvp2axi_chn_attr *chn_attr)
+{
+	int i;
+
+	dev->dvp2axi_framesize[chn_attr->device_idx] =
+		    chn_attr->hs_num * chn_attr->vs_num * (chn_attr->pixel_width / 8);
+	for (i = 0; i < VI_DVP2AXI_VIRTUAL_CHNS; i++) {
+		atomic_set(&dev->dvp2axi_vchn_fidx[chn_attr->device_idx][i], 0);
+		atomic_set(&dev->dvp2axi_vchn_didx[chn_attr->device_idx][i], 0);
+		atomic_set(&dev->irqc[chn_attr->device_idx][i], 0);
+	}
+
+	// error irq mask disable
+	atomic_set(&dev->dvp2axi_errirq[chn_attr->device_idx + 2], 0);
+	atomic_set(&dev->dvp2axi_errirq[0], 0); // axi full
+	atomic_set(&dev->dvp2axi_errirq[1], 0); // axi error
+	atomic_set(&dev->dvp2axi_errirq[8], 0); // axi afull
+	vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK2, 0, (chn_attr->device_idx + 2), 1);
+	vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK2, 0, 0, 1); // axi full
+	vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK2, 0, 1, 1); // axi error
+	vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK2, 0, 8, 1); // axi afull
+
+	/*
+	 * The way of blocking data coming from sensor is to
+	 * turn off the sensor, clearing dvp_chn_en is not working.
+	 *
+	 * Add flush idx and done idx to mask the related irq,
+	 * otherwise a lot of irq generated even if dvp_chn_en cleared.
+	 */
+	switch(chn_attr->device_idx) {
+	case 0:
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL1_CSR,
+			    chn_attr->pixel_width, DVP0_PIX_WIDTH_LSB, PIX_WIDTH_BITS);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL3_CSR, chn_attr->hs_num | (chn_attr->vs_num << 16));
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL9_CSR, chn_attr->chn0_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL10_CSR, chn_attr->chn1_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL11_CSR, chn_attr->chn2_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL33_CSR,
+			    chn_attr->hs_stride, DVP0_HS_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->first_id, DVP0_FIRST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->last_id, DVP0_LAST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK0, 0, DVP0_DONE_MASK_LSB, 3);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK0, 0, DVP0_FLUSH_MASK_LSB, 3);
+		break;
+	case 1:
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL1_CSR,
+			    chn_attr->pixel_width, DVP1_PIX_WIDTH_LSB, PIX_WIDTH_BITS);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL4_CSR, chn_attr->hs_num | (chn_attr->vs_num << 16));
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL12_CSR, chn_attr->chn0_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL13_CSR, chn_attr->chn1_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL14_CSR, chn_attr->chn2_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL33_CSR,
+			    chn_attr->hs_stride, DVP1_HS_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->first_id, DVP1_FIRST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->last_id, DVP1_LAST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK0, 0, DVP1_DONE_MASK_LSB, 3);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK0, 0, DVP1_FLUSH_MASK_LSB, 3);
+		break;
+	case 2:
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL2_CSR,
+			    chn_attr->pixel_width, DVP2_PIX_WIDTH_LSB, PIX_WIDTH_BITS);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL5_CSR, chn_attr->hs_num | (chn_attr->vs_num << 16));
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL15_CSR, chn_attr->chn0_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL16_CSR, chn_attr->chn1_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL17_CSR, chn_attr->chn2_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL34_CSR,
+			    chn_attr->hs_stride, DVP2_HS_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->first_id, DVP2_FIRST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->last_id, DVP2_LAST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK0, 0, DVP2_DONE_MASK_LSB, 3);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK0, 0, DVP2_FLUSH_MASK_LSB, 3);
+		break;
+	case 3:
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL2_CSR,
+			    chn_attr->pixel_width, DVP3_PIX_WIDTH_LSB, PIX_WIDTH_BITS);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL6_CSR, chn_attr->hs_num | (chn_attr->vs_num << 16));
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL18_CSR, chn_attr->chn0_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL19_CSR, chn_attr->chn1_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL20_CSR, chn_attr->chn2_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL34_CSR,
+			    chn_attr->hs_stride, DVP3_HS_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->first_id, DVP3_FIRST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->last_id, DVP3_LAST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK1, 0, DVP3_DONE_MASK_LSB, 3);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK1, 0, DVP3_FLUSH_MASK_LSB, 3);
+		break;
+	case 4:
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL2_CSR,
+			    chn_attr->pixel_width, DVP4_PIX_WIDTH_LSB, PIX_WIDTH_BITS);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL7_CSR, chn_attr->hs_num | (chn_attr->vs_num << 16));
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL21_CSR, chn_attr->chn0_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL22_CSR, chn_attr->chn1_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL23_CSR, chn_attr->chn2_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL35_CSR,
+			    chn_attr->hs_stride, DVP4_HS_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->first_id, DVP4_FIRST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->last_id, DVP4_LAST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK1, 0, DVP4_DONE_MASK_LSB, 3);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK1, 0, DVP4_FLUSH_MASK_LSB, 3);
+		break;
+	case 5:
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL2_CSR,
+			    chn_attr->pixel_width, DVP5_PIX_WIDTH_LSB, PIX_WIDTH_BITS);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL8_CSR, chn_attr->hs_num | (chn_attr->vs_num << 16));
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL24_CSR, chn_attr->chn0_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL25_CSR, chn_attr->chn1_baddr);
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL26_CSR, chn_attr->chn2_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL35_CSR,
+			    chn_attr->hs_stride, DVP5_HS_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->first_id, DVP5_FIRST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL36_CSR,
+			    chn_attr->last_id, DVP5_LAST_ID_LSB, HDR_ID_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK1, 0, DVP5_DONE_MASK_LSB, 3);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_INT_MASK1, 0, DVP5_FLUSH_MASK_LSB, 3);
+		break;
+	default:
+		pr_err("Invalid dvp port\n");
+		return -1;
+	}
+
+	if (chn_attr->enable)
+		chn_attr->enable = 1;
+
+	vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL0_CSR, chn_attr->enable, chn_attr->device_idx, 1);
+
+	return 0;
+}
+
+int vivsoc_hub_s_dvp2axi_emd_chn_attr(struct vvcam_soc_dev *dev,
+					struct dvp2axi_emd_chn_attr *chn_attr)
+{
+	switch(chn_attr->device_idx) {
+	case 0:
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL27_CSR, chn_attr->emd_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL37_CSR,
+			    chn_attr->emd_stride, DVP0_EMD_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL40_CSR,
+			    chn_attr->emd_hsnum, DVP0_EMD_HSNUM_LSB, EMD_HSNUM_BITS);
+		break;
+	case 1:
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL28_CSR, chn_attr->emd_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL37_CSR,
+			    chn_attr->emd_stride, DVP1_EMD_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL40_CSR,
+			    chn_attr->emd_hsnum, DVP1_EMD_HSNUM_LSB, EMD_HSNUM_BITS);
+		break;
+	case 2:
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL29_CSR, chn_attr->emd_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL38_CSR,
+			    chn_attr->emd_stride, DVP2_EMD_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL41_CSR,
+			    chn_attr->emd_hsnum, DVP2_EMD_HSNUM_LSB, EMD_HSNUM_BITS);
+		break;
+	case 3:
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL30_CSR, chn_attr->emd_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL38_CSR,
+			    chn_attr->emd_stride, DVP3_EMD_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL41_CSR,
+			    chn_attr->emd_hsnum, DVP3_EMD_HSNUM_LSB, EMD_HSNUM_BITS);
+		break;
+	case 4:
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL31_CSR, chn_attr->emd_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL39_CSR,
+			    chn_attr->emd_stride, DVP4_EMD_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL42_CSR,
+			    chn_attr->emd_hsnum, DVP4_EMD_HSNUM_LSB, EMD_HSNUM_BITS);
+		break;
+	case 5:
+		dev->soc_access.write(dev, VI_DVP2AXI_CTRL32_CSR, chn_attr->emd_baddr);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL39_CSR,
+			    chn_attr->emd_stride, DVP5_EMD_STRIDE_LSB, STRIDE_BITS);
+		vivsoc_hub_write_part(dev, VI_DVP2AXI_CTRL42_CSR,
+			    chn_attr->emd_hsnum, DVP5_EMD_HSNUM_LSB, EMD_HSNUM_BITS);
+		break;
+	default:
+		pr_err("Invalid dvp port\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int vivsoc_hub_s_dvp2axi_iova_get(struct visoc_pdata *pdata,
+					struct dvp2axi_dmabuf_attr *dmabuf_attr)
+{
+
+	size_t buf_size = 0;
+	struct heap_mem *hmem = NULL;
+
+	pr_info("import dmabuf_fd = %d\n", dmabuf_attr->dmabuf_fd);
+
+	//hmem = common_dmabuf_heap_import_from_user(&pdata->root, dmabuf_attr->dmabuf_fd, false);
+	if(IS_ERR(hmem)) {
+		printk(KERN_ERR "dmabuf-heap alloc from userspace failed\n");
+		return -ENOMEM;
+	}
+
+	buf_size = common_dmabuf_heap_get_size(hmem);
+	pr_info("dmabuf info: CPU VA:0x%lx, PA:0x%lx, DMA addr(iova):0x%lx, size=0x%lx\n",
+			(unsigned long)hmem->vaddr, (unsigned long)sg_phys(hmem->sgt->sgl), (unsigned long)sg_dma_address(hmem->sgt->sgl), (unsigned long)buf_size);
+
+	dmabuf_attr->iova = (unsigned long)sg_dma_address(hmem->sgt->sgl);
+
+	return 0;
+}
+
+int vivsoc_hub_s_dvp2axi_iova_put(struct visoc_pdata *pdata, int dmabuf_fd)
+{
+	struct heap_mem *hmem;
+
+	pr_info("release dmabuf_fd = %d\n", dmabuf_fd);
+
+	hmem = common_dmabuf_lookup_heapobj_by_fd(&pdata->root, dmabuf_fd);
+	if (IS_ERR(hmem)) {
+		pr_err("cannot find dmabuf-heap for dmabuf_fd %d\n", dmabuf_fd);
+		return -ENOMEM;
+	}
+
+	common_dmabuf_heap_release(hmem);
+
+	return 0;
+}
+
+int vivsoc_hub_s_phy_mode(struct vvcam_soc_dev *dev,int phy_mode)
+{
+	int shift = 0, width = 3;
+
+	if (phy_mode < 0 || phy_mode > PHY_MODE_LANE_2_2_2_2_2_2) {
+		pr_err("%s: Invalid phy mode\n", __func__);
+		return -EINVAL;
+	}
+
+	vivsoc_hub_write_part(dev, VI_REG_PHY_CONNECT_MODE, phy_mode, shift, width);
+
+	return 0;
+}
+
+
+int vivsoc_hub_s_controller_sel(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl)
+{
+	int width = 1;
+
+	if (soc_ctrl->device_idx > 5) {
+		pr_err("%s: Invalid device index\n", __func__);
+		return -EINVAL;
+	}
+
+	if (soc_ctrl->control_value) {
+		soc_ctrl->control_value = 1;
+	}
+
+	vivsoc_hub_write_part(dev, VI_REG_CONTROLLER_SEL,
+		    soc_ctrl->control_value, soc_ctrl->device_idx, width);
+
+	return 0;
+}
+
+int vivsoc_hub_s_isp_dvp_sel(struct vvcam_soc_dev *dev,
+					struct isp_dvp_attr *dvp_attr)
+{
+	int shift = 0;
+	if (dvp_attr->dvp_sel >= CONTROLLER_BUTT) {
+		pr_err("%s: Invalid dvp controller type\n", __func__);
+		return -EINVAL;
+	}
+
+	switch (dvp_attr->devId) {
+		case 0:
+			shift = ISP0_DVP0_SEL_LSB;
+			break;
+		case 1:
+			shift = ISP0_DVP1_SEL_LSB;
+			break;
+		case 2:
+			shift = ISP0_DVP2_SEL_LSB;
+			break;
+		case 3:
+			shift = ISP0_DVP3_SEL_LSB;
+			break;
+		case 4:
+			shift = ISP1_DVP0_SEL_LSB;
+			break;
+		case 5:
+			shift = ISP1_DVP1_SEL_LSB;
+			break;
+		case 6:
+			shift = ISP1_DVP2_SEL_LSB;
+			break;
+		case 7:
+			shift = ISP1_DVP3_SEL_LSB;
+			break;
+		default:
+			shift = 0;
+			return -EINVAL;
+	}
+	vivsoc_hub_write_part(dev, VI_REG_ISP_DVP_SEL,
+			dvp_attr->dvp_sel, shift, ISP_DVP_SEL_BITS);
+
+	return 0;
+}
+
+unsigned int vivsoc_register_hardware(struct vvcam_soc_dev *dev,
+				      struct vvcam_soc_function_s *func)
+{
+	if (func == NULL) {
+		return -1;
+	}
+
+	memcpy(&dev->soc_func, func, sizeof(struct vvcam_soc_function_s));
+
+	dev->soc_access.write = gen6_write_reg;
+	dev->soc_access.read = gen6_read_reg;
+
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_hub.h b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_hub.h
new file mode 100644
index 000000000000..eecd969ec093
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vivsoc_hub.h
@@ -0,0 +1,144 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+#ifndef _VIVSOC_HUB_H_
+#define _VIVSOC_HUB_H_
+
+#include "soc_ioctl.h"
+
+int vivsoc_hub_isp_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl);
+int vivsoc_hub_isp_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_isp_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_isp_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_isp_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+
+int vivsoc_hub_dwe_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl);
+int vivsoc_hub_dwe_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_dwe_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_dwe_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_dwe_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+
+int vivsoc_hub_vse_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl);
+int vivsoc_hub_vse_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_vse_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_vse_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_vse_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+
+int vivsoc_hub_csi_reset(struct vvcam_soc_dev *dev,
+			 struct soc_control_context *soc_ctrl);
+int vivsoc_hub_csi_s_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_csi_g_power_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_csi_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+int vivsoc_hub_csi_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				struct soc_control_context *soc_ctrl);
+
+int vivsoc_hub_sensor_reset(struct vvcam_soc_dev *dev,
+			    struct soc_control_context *soc_ctrl);
+int vivsoc_hub_sensor_s_power_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl);
+int vivsoc_hub_sensor_g_power_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl);
+int vivsoc_hub_sensor_s_clock_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl);
+int vivsoc_hub_sensor_g_clock_ctrl(struct vvcam_soc_dev *dev,
+				   struct soc_control_context *soc_ctrl);
+
+
+int vivsoc_hub_s_multi2isp_imgsize(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl);
+int vivsoc_hub_g_multi2isp_imgsize(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl);
+
+int vivsoc_hub_s_imx327_hdr(struct vvcam_soc_dev *dev,
+					struct soc_control_context *soc_ctrl);
+
+int vivsoc_hub_write_part(struct vvcam_soc_dev *dev,
+		    uint32_t address, uint32_t data, uint8_t shift, uint8_t width);
+int vivsoc_hub_s_reset_dvp2axi(struct vvcam_soc_driver_dev *pdriver_dev, struct vvcam_soc_dev *dev);
+int vivsoc_hub_s_dvp2axi_iodvp_sel(struct vvcam_soc_dev *dev,
+		    struct soc_control_context *soc_ctrl);
+int vivsoc_hub_s_dvp2axi_chn_attr(struct vvcam_soc_dev *dev,
+		    struct dvp2axi_chn_attr *chn_attr);
+int vivsoc_hub_s_dvp2axi_emd_chn_attr(struct vvcam_soc_dev *dev,
+		    struct dvp2axi_emd_chn_attr *chn_attr);
+int vivsoc_hub_s_dvp2axi_iova_get(struct visoc_pdata *pdata,
+		    struct dvp2axi_dmabuf_attr *dmabuf_attr);
+int vivsoc_hub_s_dvp2axi_iova_put(struct visoc_pdata *pdata, int dmabuf_fd);
+int vivsoc_hub_s_phy_mode(struct vvcam_soc_dev *dev,int phy_mode);
+int vivsoc_hub_s_controller_sel(struct vvcam_soc_dev *dev,
+			struct soc_control_context *soc_ctrl);
+int vivsoc_hub_s_isp_dvp_sel(struct vvcam_soc_dev *dev,
+			struct isp_dvp_attr *dvp_attr);
+
+
+unsigned int vivsoc_register_hardware(struct vvcam_soc_dev *dev,
+				      struct vvcam_soc_function_s *func);
+
+#endif /* _VIVSOC_HUB_H_ */
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vsi_core_gen6.c b/drivers/media/platform/eswin/isp/vvcam/soc/vsi_core_gen6.c
new file mode 100644
index 000000000000..1876cb1fff19
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vsi_core_gen6.c
@@ -0,0 +1,250 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#ifdef ISP8000L_V2008
+#include <linux/io.h>		//Fix thead compile error.
+#endif
+
+#include <linux/io.h> 
+#include "soc_ioctl.h"
+#include "vsi_core_gen6.h"
+
+int gen6_write_reg(void *dev, unsigned int addr, unsigned int val)
+{
+	struct vvcam_soc_dev *soc_dev;
+
+	soc_dev = (struct vvcam_soc_dev *)dev;
+
+	if (soc_dev == NULL)
+		return -1;
+
+	__raw_writel(val,soc_dev->base + addr);
+    
+	return 0;
+}
+
+int gen6_read_reg(void *dev, unsigned int addr, unsigned int *val)
+{
+	struct vvcam_soc_dev *soc_dev;
+
+	soc_dev = (struct vvcam_soc_dev *)dev;
+
+	if (soc_dev == NULL)
+		return -1;
+
+	*val = __raw_readl(soc_dev->base + addr);
+
+	return 0;
+}
+
+static int gen6_set_isp_power(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_get_isp_power(void *dev, unsigned int id, unsigned int *status)
+{
+	return 0;
+}
+
+static int gen6_set_isp_reset(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_set_isp_clk(void *dev, unsigned int id, unsigned int freq)
+{
+	return 0;
+}
+
+static int gen6_get_isp_clk(void *dev, unsigned int id, unsigned int *freq)
+{
+	return 0;
+}
+
+static int gen6_set_dwe_power(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_get_dwe_power(void *dev, unsigned int id, unsigned int *status)
+{
+	return 0;
+}
+
+static int gen6_set_dwe_reset(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_set_dwe_clk(void *dev, unsigned int id, unsigned int freq)
+{
+	return 0;
+}
+
+static int gen6_get_dwe_clk(void *dev, unsigned int id, unsigned int *freq)
+{
+	return 0;
+}
+
+static int gen6_set_vse_power(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_get_vse_power(void *dev, unsigned int id, unsigned int *status)
+{
+	return 0;
+}
+
+static int gen6_set_vse_reset(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_set_vse_clk(void *dev, unsigned int id, unsigned int freq)
+{
+	return 0;
+}
+
+static int gen6_get_vse_clk(void *dev, unsigned int id, unsigned int *freq)
+{
+	return 0;
+}
+
+static int gen6_set_csi_power(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_get_csi_power(void *dev, unsigned int id, unsigned int *status)
+{
+	return 0;
+}
+
+static int gen6_set_csi_reset(void *dev, unsigned int id, unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_set_csi_clk(void *dev, unsigned int id, unsigned int freq)
+{
+	return 0;
+}
+
+static int gen6_get_csi_clk(void *dev, unsigned int id, unsigned int *freq)
+{
+	return 0;
+}
+
+static int gen6_set_sensor_power(void *dev, unsigned int id,
+				 unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_get_sensor_power(void *dev, unsigned int id,
+				 unsigned int *status)
+{
+	return 0;
+}
+
+static int gen6_set_sensor_reset(void *dev, unsigned int id,
+				 unsigned int status)
+{
+	return 0;
+}
+
+static int gen6_set_sensor_clk(void *dev, unsigned int id, unsigned int freq)
+{
+	return 0;
+}
+
+static int gen6_get_sensor_clk(void *dev, unsigned int id, unsigned int *freq)
+{
+	return 0;
+}
+
+struct vvcam_soc_function_s gen6_soc_function = {
+	.isp_func.set_power = gen6_set_isp_power,
+	.isp_func.get_power = gen6_get_isp_power,
+	.isp_func.set_reset = gen6_set_isp_reset,
+	.isp_func.set_clk = gen6_set_isp_clk,
+	.isp_func.get_clk = gen6_get_isp_clk,
+
+	.dwe_func.set_power = gen6_set_dwe_power,
+	.dwe_func.get_power = gen6_get_dwe_power,
+	.dwe_func.set_reset = gen6_set_dwe_reset,
+	.dwe_func.set_clk = gen6_set_dwe_clk,
+	.dwe_func.get_clk = gen6_get_dwe_clk,
+
+	.vse_func.set_power = gen6_set_vse_power,
+	.vse_func.get_power = gen6_get_vse_power,
+	.vse_func.set_reset = gen6_set_vse_reset,
+	.vse_func.set_clk = gen6_set_vse_clk,
+	.vse_func.get_clk = gen6_get_vse_clk,
+
+	.csi_func.set_power = gen6_set_csi_power,
+	.csi_func.get_power = gen6_get_csi_power,
+	.csi_func.set_reset = gen6_set_csi_reset,
+	.csi_func.set_clk = gen6_set_csi_clk,
+	.csi_func.get_clk = gen6_get_csi_clk,
+
+	.sensor_func.set_power = gen6_set_sensor_power,
+	.sensor_func.get_power = gen6_get_sensor_power,
+	.sensor_func.set_reset = gen6_set_sensor_reset,
+	.sensor_func.set_clk = gen6_set_sensor_clk,
+	.sensor_func.get_clk = gen6_get_sensor_clk,
+};
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vsi_core_gen6.h b/drivers/media/platform/eswin/isp/vvcam/soc/vsi_core_gen6.h
new file mode 100644
index 000000000000..8b1fd21231cc
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vsi_core_gen6.h
@@ -0,0 +1,68 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+
+#ifndef _VSI_CORE_GEN6_H
+#define _VSI_CORE_GEN6_H
+
+#define REG_TPG0      0x300
+#define REG_TPG1      0x310
+#define REG_TPG2      0x320
+#define REG_TPG3      0x330
+#define REG_DWE_CTRL  0x250
+#define REG_VSE_CTRL  0x254
+
+extern struct vvcam_soc_function_s gen6_soc_function;
+int gen6_write_reg(void *dev, unsigned int addr, unsigned int val);
+int gen6_read_reg(void *dev, unsigned int addr, unsigned int *val);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/soc/vvcam_soc_driver.c b/drivers/media/platform/eswin/isp/vvcam/soc/vvcam_soc_driver.c
new file mode 100644
index 000000000000..a771a3ee58f1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/soc/vvcam_soc_driver.c
@@ -0,0 +1,855 @@
+/****************************************************************************
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************
+ *
+ * The GPL License (GPL)
+ *
+ * Copyright (c) 2020 VeriSilicon Holdings Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program;
+ *
+ *****************************************************************************
+ *
+ * Note: This software is released under dual MIT and GPL licenses. A
+ * recipient may use this file under the terms of either the MIT license or
+ * GPL License. If you wish to use only one license not the other, you can
+ * indicate your decision by deleting one of the above license notices in your
+ * version of this file.
+ *
+ *****************************************************************************/
+
+#include <asm/io.h>
+#include <linux/cdev.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/proc_fs.h>
+#include <linux/debugfs.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/poll.h>
+#include <linux/dmabuf-heap-import-helper.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+
+#include "soc_ioctl.h"
+
+#define VIVCAM_SOC_NAME "vivsoc"
+#define VIVCAM_SOC_MAXCNT 2
+
+#define VVCAM_AXI_CLK_HIGHEST 800000000
+#define VVCAM_DVP_CLK_HIGHEST 594000000
+#define VVCAM_ISP_CLK_HIGHEST 594000000
+
+static unsigned int imx_hdr = 0;
+static unsigned int phy_mode = PHY_MODE_LANE_2_2_2_2_2_2;
+
+static unsigned int vvcam_soc_major = 0;
+static unsigned int vvcam_soc_minor = 0;
+static struct class *vvcam_soc_class;
+static unsigned int devise_register_index;
+
+module_param(imx_hdr, int ,S_IRUGO);
+module_param(phy_mode, int ,S_IRUGO);
+
+// #define VI_SMMU_ENABLE
+
+static int vvcam_soc_open(struct inode *inode, struct file *file)
+{
+	struct visoc_pdata *pdata;
+	struct platform_device *pdev;
+	struct vvcam_soc_driver_dev *pdriver_dev;
+
+	pdata = kzalloc(sizeof(struct visoc_pdata), GFP_KERNEL);
+	if (!pdata) {
+		pr_err("%s: kzalloc failed for private data\n", __func__);
+		return -ENOMEM;
+	}
+
+	pdriver_dev =
+	    container_of(inode->i_cdev, struct vvcam_soc_driver_dev, cdev);
+
+	pdev = pdriver_dev->pdev;
+	pdata->pdrv_dev = pdriver_dev;
+	pdata->dvp_idx = -1;
+	common_dmabuf_heap_import_init(&pdata->root, &pdev->dev);
+	file->private_data = pdata;
+
+	return 0;
+};
+
+static long vvcam_soc_ioctl(struct file *file, unsigned int cmd,
+			    unsigned long arg)
+{
+	long ret = 0;
+	struct visoc_pdata *pdata;
+	struct vvcam_soc_driver_dev *pdriver_dev;
+
+	pdata = file->private_data;
+	pdriver_dev = pdata->pdrv_dev;
+	if (pdriver_dev == NULL) {
+		pr_err("%s:file private is null point error\n", __func__);
+		return -ENODEV;
+	}
+	ret = soc_priv_ioctl(pdata, cmd, (void __user *)arg);
+
+	return ret;
+};
+
+static unsigned int vvcam_soc_poll(struct file *file, poll_table *wait)
+{
+	unsigned int mask = 0, i;
+	struct visoc_pdata *pdata;
+	struct vvcam_soc_dev *psoc_dev;
+	struct vvcam_soc_driver_dev *pdriver_dev;
+
+	pdata = file->private_data;
+	pdriver_dev = pdata->pdrv_dev;
+	psoc_dev = pdriver_dev->private;
+
+	poll_wait(file, &psoc_dev->irq_wait, wait);
+
+	if (atomic_read(&psoc_dev->irqc_err)) {
+		pr_info("%s: err int recved\n", __func__);
+		mask |= POLLERR;
+		atomic_dec(&psoc_dev->irqc_err);
+		goto out;
+	}
+
+	for (i = 0; i < VI_DVP2AXI_VIRTUAL_CHNS; i++) {
+		if (atomic_read(&psoc_dev->dvp2axi_vchn_didx[pdata->dvp_idx][i]) &&
+				    atomic_read(&psoc_dev->irqc[pdata->dvp_idx][i]) < SENSOR_FRAME_NUM) {
+			pr_info("%s: frame recved, irqc %d\n", __func__, atomic_read(&psoc_dev->irqc[pdata->dvp_idx][i]));
+			atomic_inc(&psoc_dev->irqc[pdata->dvp_idx][i]);
+			mask |= POLLIN | POLLRDNORM;
+			goto out;
+		}
+	}
+
+out:
+	return mask;
+}
+
+static int vvcam_soc_release(struct inode *inode, struct file *file)
+{
+	struct visoc_pdata *pdata = file->private_data;
+
+	kfree(pdata);
+
+	return 0;
+};
+
+static int soc_debug_open(struct inode* inode, struct file* filp)
+{
+    filp->private_data = inode->i_private;
+
+    return 0;
+}
+
+static ssize_t soc_debug_read(struct file* filp, char __user *buffer, size_t count, loff_t* ppos)
+{
+    uint32_t reg_dvp0_io = 0; 
+    uint32_t reg_dvp1_io = 0; 
+    uint32_t reg_isp0_dvp0 = 0; 
+    uint32_t reg_isp0_dvp1 = 0; 
+    uint32_t reg_isp1_dvp0 = 0; 
+    uint32_t reg_isp1_dvp1 = 0; 
+	uint32_t reg_dvp0_cnt = 0;
+	uint32_t reg_dvp1_cnt = 0;
+	uint32_t reg_dvp2_cnt = 0;
+	uint32_t reg_dvp3_cnt = 0;
+	uint32_t reg_dvp4_cnt = 0;
+	uint32_t reg_dvp5_cnt = 0;
+	uint32_t reg_mmu_isp0 = 0;
+	uint32_t reg_mmu_isp1 = 0;
+	uint32_t reg_mmu_dw200 = 0;
+	uint32_t reg_mmu_dvp = 0;
+
+    char ret_buff[256];
+    struct vvcam_soc_dev * psoc_dev = NULL;
+    psoc_dev = filp->private_data;
+
+    psoc_dev->soc_access.read(psoc_dev, 0x6c, &reg_dvp0_cnt);
+    psoc_dev->soc_access.read(psoc_dev, 0x70, &reg_dvp1_cnt);
+    psoc_dev->soc_access.read(psoc_dev, 0x74, &reg_dvp2_cnt);
+    psoc_dev->soc_access.read(psoc_dev, 0x78, &reg_dvp3_cnt);
+    psoc_dev->soc_access.read(psoc_dev, 0x7c, &reg_dvp4_cnt);
+    psoc_dev->soc_access.read(psoc_dev, 0x80, &reg_dvp5_cnt);
+
+    psoc_dev->soc_access.read(psoc_dev, 0x84, &reg_dvp0_io);
+    psoc_dev->soc_access.read(psoc_dev, 0x88, &reg_dvp1_io);
+    psoc_dev->soc_access.read(psoc_dev, 0x8c, &reg_isp0_dvp0);
+    psoc_dev->soc_access.read(psoc_dev, 0x90, &reg_isp0_dvp1);
+    psoc_dev->soc_access.read(psoc_dev, 0x94, &reg_isp1_dvp0);
+    psoc_dev->soc_access.read(psoc_dev, 0x98, &reg_isp1_dvp1);
+
+    psoc_dev->soc_access.read(psoc_dev, 0x1000, &reg_mmu_isp0);
+    psoc_dev->soc_access.read(psoc_dev, 0x1004, &reg_mmu_isp1);
+    psoc_dev->soc_access.read(psoc_dev, 0x1008, &reg_mmu_dw200);
+    psoc_dev->soc_access.read(psoc_dev, 0x100c, &reg_mmu_dvp);
+
+    count = sprintf(ret_buff, "dvp0-5 cnt:%x %x %x %x %x %x, dvp0_io %d dvp1_io %d isp0_dvp0 %d isp0_dvp1 %d isp1_dvp0 %d isp1_dvp1 %d, smmu 0x%x 0x%x 0x%x 0x%x\n", 
+            reg_dvp0_cnt, reg_dvp1_cnt, reg_dvp2_cnt, reg_dvp3_cnt, reg_dvp4_cnt,reg_dvp5_cnt,
+			reg_dvp0_io, reg_dvp1_io, reg_isp0_dvp0, reg_isp0_dvp1, reg_isp1_dvp0, reg_isp1_dvp1,
+            reg_mmu_isp0, reg_mmu_isp1, reg_mmu_dw200, reg_mmu_dvp);
+
+    return simple_read_from_buffer(buffer, count, ppos, ret_buff, count);
+}
+
+static int soc_reset_open(struct inode* inode, struct file* filp)
+{
+    filp->private_data = inode->i_private;
+
+    return 0;
+}
+/*
+Writing the registers of smmu tbl may fail, 
+so we read back those value to ensure the writing was succeefully.
+*/
+
+#ifdef VI_SMMU_ENABLE
+static int soc_write_dynm_reg(struct vvcam_soc_dev  *psoc_dev, unsigned int addr, unsigned int val)
+{
+	int try_times = 100;
+	int read_val = 0;
+	int i = 0;
+	int ret = -1;
+
+	for(i = 0; i < try_times; i++)
+	{
+		psoc_dev->soc_access.write(psoc_dev, addr, val);
+		udelay(1);
+
+		psoc_dev->soc_access.read(psoc_dev, addr, &read_val);
+		if(read_val == val)
+		{
+			ret = 0;
+			break;
+		}
+	}
+	if(i >= try_times)
+	{
+		pr_err("%s failed!\n",__func__);
+	}
+
+    return ret;
+}
+#endif
+
+static ssize_t soc_reset_write(struct file* filp, const char __user *buffer, size_t count, loff_t* ppos)
+{
+    struct vvcam_soc_dev * psoc_dev = NULL;
+	char str_buff[64] = {0};
+
+    #ifdef VI_SMMU_ENABLE
+	uint32_t reg_mmu_isp0 = 0;
+	uint32_t reg_mmu_isp1 = 0;
+	uint32_t reg_mmu_dw200 = 0;
+	uint32_t reg_mmu_dvp = 0;
+    #endif
+
+    psoc_dev = filp->private_data;
+
+    if (*ppos >= 64) {
+			pr_err("%s: ppos out of range\n", __func__);
+        return 0;
+    }
+
+    if (*ppos + count > 64) {
+        count = 64 - *ppos;
+    }
+
+    if (copy_from_user(str_buff + *ppos, buffer, count) != 0) {
+		pr_err("%s: copy_from_user error\n", __func__);
+        return -EFAULT;
+    }
+    *ppos += count;
+
+    /*the val is reset register offset*/
+	//val = (uint32_t)simple_strtoul(str_buff, NULL, 16);
+	//pr_info("%s: Get isp_reg_addr: 0x%x\n", __func__, val);
+
+    #ifdef M2_REGISTER_MAP
+		/*带SMMU场景，复位前需要记录smmu stream id，复位后需要恢复smmu stream id*/
+		#ifdef VI_SMMU_ENABLE
+		psoc_dev->soc_access.read(psoc_dev, 0x1000, &reg_mmu_isp0);
+		psoc_dev->soc_access.read(psoc_dev, 0x1004, &reg_mmu_isp1);
+		psoc_dev->soc_access.read(psoc_dev, 0x1008, &reg_mmu_dw200);
+		psoc_dev->soc_access.read(psoc_dev, 0x100c, &reg_mmu_dvp);
+		#endif
+
+		__raw_writel(0x0, psoc_dev->scu_base + 0x470);
+		__raw_writel(0x7, psoc_dev->scu_base + 0x470);/*vi_rst_ctrl*/
+		__raw_writel(0x0, psoc_dev->scu_base + 0x474);
+		__raw_writel(0x1, psoc_dev->scu_base + 0x474);/*dvp_rst_ctrl*/
+		__raw_writel(0x0, psoc_dev->scu_base + 0x478);
+		__raw_writel(0x1, psoc_dev->scu_base + 0x478);/*isp0_rst_ctrl*/
+		__raw_writel(0x0, psoc_dev->scu_base + 0x47c);
+		__raw_writel(0x1, psoc_dev->scu_base + 0x47c);/*isp1_rst_ctrl*/
+
+		#ifdef VI_SMMU_ENABLE
+		soc_write_dynm_reg(psoc_dev, 0x1000, reg_mmu_isp0);
+		soc_write_dynm_reg(psoc_dev, 0x1004, reg_mmu_isp1);
+		soc_write_dynm_reg(psoc_dev, 0x1008, reg_mmu_dw200);
+		soc_write_dynm_reg(psoc_dev, 0x100c, reg_mmu_dvp);
+		#endif
+
+    #else
+	/*hardware reset the isp*/
+	__raw_writel(0x0, psoc_dev->scu_base + 0x0c);
+	__raw_writel(0x1, psoc_dev->scu_base + 0x0c);
+    #endif
+
+    return count;
+}
+
+static struct file_operations vvcam_soc_fops = {
+	.owner = THIS_MODULE,
+	.open = vvcam_soc_open,
+	.release = vvcam_soc_release,
+	.unlocked_ioctl = vvcam_soc_ioctl,
+	.poll           = vvcam_soc_poll,
+};
+
+struct file_operations soc_debug_fops = {
+    .owner = THIS_MODULE,
+    .open = soc_debug_open,
+    .read = soc_debug_read,
+};
+
+struct file_operations soc_reset_fops = {
+    .owner = THIS_MODULE,
+    .open = soc_reset_open,
+    .write = soc_reset_write,
+};
+
+#define VVCAM_RST_GET_SHARED(dev, rst_handle, rst_name) do { \
+	rst_handle = devm_reset_control_get_shared(dev, rst_name); \
+	if (IS_ERR_OR_NULL(rst_handle)) { \
+		dev_err(dev, "failed to get isp reset handle %s\n", rst_name); \
+		return -EFAULT; \
+	}} while(0)
+
+#define VVCAM_RST_GET_OPTIONAL(dev, rst_handle, rst_name) do { \
+	rst_handle = devm_reset_control_get_optional(dev, rst_name); \
+	if (IS_ERR_OR_NULL(rst_handle)) { \
+		dev_err(dev, "failed to get isp reset handle %s\n", rst_name); \
+		return -EFAULT; \
+	}} while(0)
+
+#define VVCAM_CLK_GET_HANDLE(dev, clk_handle, clk_name) do { \
+	clk_handle = devm_clk_get(dev, clk_name); \
+	if (IS_ERR(clk_handle)) { \
+		ret = PTR_ERR(clk_handle); \
+		dev_err(dev, "failed to get dw %s: %d\n", clk_name, ret); \
+		return ret; \
+	}} while(0)
+
+#define VVCAM_SYS_CLK_PREPARE(clk) do { \
+	ret = clk_prepare_enable(clk); \
+	if (ret) { \
+		pr_err("failed to enable clk %px\n", clk); \
+		return ret; \
+	}} while(0)
+
+static int vvcam_sys_reset_init(struct platform_device *pdev, isp_clk_rst_t *isp_crg)
+{
+	VVCAM_RST_GET_SHARED(&pdev->dev, isp_crg->rstc_axi, "axi");
+	VVCAM_RST_GET_SHARED(&pdev->dev, isp_crg->rstc_cfg, "cfg");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_dvp, "dvp");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_dvp, "isp0");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_dvp, "isp1");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_sht0, "sht0");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_sht1, "sht1");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_sht2, "sht2");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_sht3, "sht3");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_sht4, "sht4");
+	VVCAM_RST_GET_OPTIONAL(&pdev->dev, isp_crg->rstc_sht5, "sht5");
+
+	return 0;
+}
+
+static int vvcam_sys_clk_init(struct platform_device *pdev, isp_clk_rst_t *isp_crg)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->aclk, "aclk");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->cfg_clk, "cfg_clk");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->isp_aclk, "isp_aclk");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->dvp_clk, "dvp_clk");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->phy_cfg, "phy_cfg");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->phy_escclk, "phy_escclk");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->sht0, "sht0");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->sht1, "sht1");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->sht2, "sht2");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->sht3, "sht3");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->sht4, "sht4");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->sht4, "sht5");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->aclk_mux, "aclk_mux");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->dvp_mux, "dvp_mux");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->isp_mux, "isp_mux");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->spll0_fout1, "spll0_fout1");
+	VVCAM_CLK_GET_HANDLE(dev, isp_crg->vpll_fout1, "vpll_fout1");
+
+	return 0;
+}
+
+static int vvcam_sys_reset_release(isp_clk_rst_t *isp_crg)
+{
+	int ret;
+
+	ret = reset_control_deassert(isp_crg->rstc_cfg);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_deassert(isp_crg->rstc_axi);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_isp);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_dvp);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_sht0);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_sht1);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_sht2);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_sht3);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_sht4);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_reset(isp_crg->rstc_sht5);
+	WARN_ON(0 != ret);
+
+	return 0;
+}
+
+static int vvcam_sys_clk_prepare(isp_clk_rst_t *isp_crg)
+{
+	int ret;
+	long rate;
+
+	ret = clk_set_parent(isp_crg->aclk_mux, isp_crg->spll0_fout1);
+	if (ret < 0) {
+		pr_err("ISP: failed to set aclk_mux parent: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_set_parent(isp_crg->dvp_mux, isp_crg->vpll_fout1);
+	if (ret < 0) {
+		pr_err("ISP: failed to set dvp_mux parent: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_set_parent(isp_crg->isp_mux, isp_crg->vpll_fout1);
+	if (ret < 0) {
+		pr_err("ISP: failed to set isp_mux parent: %d\n", ret);
+		return ret;
+	}
+
+	rate = clk_round_rate(isp_crg->aclk, VVCAM_AXI_CLK_HIGHEST);
+	if (rate > 0) {
+		ret = clk_set_rate(isp_crg->aclk, rate);
+		if (ret) {
+			pr_err("ISP: failed to set aclk: %d\n", ret);
+			return ret;
+		}
+		pr_info("ISP set aclk to %ldHZ\n", rate);
+	}
+
+	rate = clk_round_rate(isp_crg->dvp_clk, VVCAM_DVP_CLK_HIGHEST);
+	if (rate > 0) {
+		ret = clk_set_rate(isp_crg->dvp_clk, rate);
+		if (ret) {
+			pr_err("ISP: failed to set dvp_clk: %d\n", ret);
+			return ret;
+		}
+		pr_info("ISP set dvp_clk to %ldHZ\n", rate);
+	}
+
+	rate = clk_round_rate(isp_crg->isp_aclk, VVCAM_ISP_CLK_HIGHEST);
+	if (rate > 0) {
+		ret = clk_set_rate(isp_crg->isp_aclk, rate);
+		if (ret) {
+			pr_err("ISP: failed to set isp_aclk: %d\n", ret);
+			return ret;
+		}
+		pr_info("ISP set vi_dig_isp clk to %ldHZ\n", rate);
+	}
+
+	VVCAM_SYS_CLK_PREPARE(isp_crg->aclk);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->dvp_clk);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->isp_aclk);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->cfg_clk);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->phy_cfg);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->phy_escclk);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->sht0);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->sht1);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->sht2);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->sht3);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->sht4);
+	VVCAM_SYS_CLK_PREPARE(isp_crg->sht5);
+
+	return 0;
+}
+
+static int vvcam_clk_reset_fini(isp_clk_rst_t *isp_crg)
+{
+	reset_control_assert(isp_crg->rstc_sht0);
+	reset_control_assert(isp_crg->rstc_sht1);
+	reset_control_assert(isp_crg->rstc_sht2);
+	reset_control_assert(isp_crg->rstc_sht3);
+	reset_control_assert(isp_crg->rstc_sht4);
+	reset_control_assert(isp_crg->rstc_sht5);
+	reset_control_assert(isp_crg->rstc_isp);
+	reset_control_assert(isp_crg->rstc_cfg);
+	reset_control_assert(isp_crg->rstc_axi);
+
+	clk_disable_unprepare(isp_crg->sht0);
+	clk_disable_unprepare(isp_crg->sht1);
+	clk_disable_unprepare(isp_crg->sht2);
+	clk_disable_unprepare(isp_crg->sht3);
+	clk_disable_unprepare(isp_crg->sht4);
+	clk_disable_unprepare(isp_crg->sht5);
+	clk_disable_unprepare(isp_crg->phy_escclk);
+	clk_disable_unprepare(isp_crg->phy_cfg);
+	clk_disable_unprepare(isp_crg->dvp_clk);
+	clk_disable_unprepare(isp_crg->isp_aclk);
+	clk_disable_unprepare(isp_crg->cfg_clk);
+	clk_disable_unprepare(isp_crg->aclk);
+
+	return 0;
+}
+
+static int vvcam_soc_probe(struct platform_device *pdev)
+{
+	int ret = 0, i, j;
+	uint32_t vi_top_phy_mode;
+	struct vvcam_soc_driver_dev *pdriver_dev;
+	struct vvcam_soc_dev *psoc_dev;
+
+    #ifdef VI_SMMU_ENABLE
+	uint32_t reg_mmu_isp0 = 0;
+	uint32_t reg_mmu_isp1 = 0;
+	uint32_t reg_mmu_dw200 = 0;
+	uint32_t reg_mmu_dvp = 0;
+    #endif
+	struct resource         *mem;
+	char debug_filename[64] = "soc_reg_dump";
+	char debug_soc_reset[64] = "soc_reset";
+
+	pr_info("enter %s\n", __func__);
+
+	if (device_property_read_u32(&pdev->dev, "id", &pdev->id)) {
+		dev_err(&pdev->dev, "Can not read device id!\n");
+		return -EINVAL;
+	}
+
+	if (pdev->id >= VIVCAM_SOC_MAXCNT) {
+		pr_err("%s:pdev id is %d error\n", __func__, pdev->id);
+		return -EINVAL;
+	}
+
+	pdriver_dev =
+	    devm_kzalloc(&pdev->dev, sizeof(struct vvcam_soc_driver_dev),
+			 GFP_KERNEL);
+	if (pdriver_dev == NULL) {
+		pr_err("%s:alloc struct vvcam_soc_driver_dev error\n",
+		       __func__);
+		return -ENOMEM;
+	}
+
+	psoc_dev =
+	    devm_kzalloc(&pdev->dev, sizeof(struct vvcam_soc_dev), GFP_KERNEL);
+	if (psoc_dev == NULL) {
+		pr_err("%s:alloc struct vvcam_soc_dev error\n", __func__);
+		return -ENOMEM;
+	}
+
+	ret = vvcam_sys_reset_init(pdev, &pdriver_dev->isp_crg);
+	if (ret) {
+		pr_err("%s: isp reset init failed\n", __func__);
+		return ret;
+	}
+
+	ret = vvcam_sys_clk_init(pdev, &pdriver_dev->isp_crg);
+	if (ret) {
+		pr_err("%s: isp clk init failed\n", __func__);
+		return ret;
+	}
+
+	ret = vvcam_sys_clk_prepare(&pdriver_dev->isp_crg);
+	if (ret) {
+		pr_err("%s: isp clk prepare failed\n", __func__);
+		return ret;
+	}
+
+	ret = vvcam_sys_reset_release(&pdriver_dev->isp_crg);
+	if (ret) {
+		pr_err("%s: isp reset release failed\n", __func__);
+		return ret;
+	}
+
+	ret = win2030_tbu_power(&pdev->dev, true);
+	if (ret) {
+		pr_err("%s: isp tbu power up failed\n", __func__);
+		return ret;
+	}
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	psoc_dev->base = devm_ioremap_resource(&pdev->dev, mem);
+	/*
+	psoc_dev->base = ioremap(VVCTRL_BASE, VVCTRL_SIZE);
+	if (IS_ERR(psoc_dev->base))
+		return PTR_ERR(psoc_dev->base);
+*/
+	psoc_dev->scu_base = ioremap(SCU_BASE, SCU_SIZE);
+	if (IS_ERR(psoc_dev->scu_base))
+		return PTR_ERR(psoc_dev->scu_base);
+
+	init_waitqueue_head(&psoc_dev->irq_wait);
+	for (i = 0; i < VI_DVP2AXI_DVP_CHNS; i++) {
+		for (j = 0; j < VI_DVP2AXI_VIRTUAL_CHNS; j++)
+			atomic_set(&psoc_dev->irqc[i][j], 0);
+	}
+
+	for (i = 0; i < VI_DVP2AXI_ERRIRQ_NUM; i++)
+		atomic_set(&psoc_dev->dvp2axi_errirq[i], 0);
+
+	atomic_set(&psoc_dev->irqc_err, 0);
+	pdriver_dev->private = psoc_dev;
+	pdriver_dev->pdev = pdev;
+	mutex_init(&pdriver_dev->vvmutex);
+	platform_set_drvdata(pdev, pdriver_dev);
+
+	vvnative_soc_init(psoc_dev);
+
+	if (devise_register_index == 0) {
+		if (vvcam_soc_major == 0) {
+			ret =
+			    alloc_chrdev_region(&pdriver_dev->devt, 0,
+						VIVCAM_SOC_MAXCNT,
+						VIVCAM_SOC_NAME);
+			if (ret != 0) {
+				pr_err("%s:alloc_chrdev_region error\n",
+				       __func__);
+				return ret;
+			}
+			vvcam_soc_major = MAJOR(pdriver_dev->devt);
+			vvcam_soc_minor = MINOR(pdriver_dev->devt);
+		} else {
+			pdriver_dev->devt =
+			    MKDEV(vvcam_soc_major, vvcam_soc_minor);
+			ret =
+			    register_chrdev_region(pdriver_dev->devt,
+						   VIVCAM_SOC_MAXCNT,
+						   VIVCAM_SOC_NAME);
+			if (ret) {
+				pr_err("%s:register_chrdev_region error\n",
+				       __func__);
+				return ret;
+			}
+		}
+// #if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0)
+		vvcam_soc_class = class_create(VIVCAM_SOC_NAME);
+// #else
+// 		vvcam_soc_class = class_create(THIS_MODULE, VIVCAM_SOC_NAME);
+// #endif
+
+		if (IS_ERR(vvcam_soc_class)) {
+			pr_err("%s[%d]:class_create error!\n", __func__,
+			       __LINE__);
+			return -EINVAL;
+		}
+	}
+	pdriver_dev->devt = MKDEV(vvcam_soc_major, vvcam_soc_minor + pdev->id); //将主设备号，次设备号转换为dev_t类型
+
+	cdev_init(&pdriver_dev->cdev, &vvcam_soc_fops);
+	ret = cdev_add(&pdriver_dev->cdev, pdriver_dev->devt, 1);
+	if (ret) {
+		pr_err("%s[%d]:cdev_add error!\n", __func__, __LINE__);
+		return ret;
+	}
+	pdriver_dev->class = vvcam_soc_class;
+	device_create(pdriver_dev->class, NULL, pdriver_dev->devt,
+		      pdriver_dev, "%s%d", VIVCAM_SOC_NAME, pdev->id);
+
+	devise_register_index++;
+
+    #ifndef M2_REGISTER_MAP //M1_REGISTER_MAP
+		/*hardware reset the isp*/
+		__raw_writel(0x0, psoc_dev->scu_base + 0x0c);
+		__raw_writel(0x1, psoc_dev->scu_base + 0x0c);
+
+		/*
+		* IMX327摄像头在hdr模式下需要配置此开关, 可配置的值包括 0x03(sensor 12bit width) 0x01(sensor 10bit width)
+		* 在M1平台offset为0x14，M2平台为vi_common_top phy_connect_mode寄存器的bit4～5
+		*/
+		//__raw_writel(0x03, psoc_dev->scu_base + 0x14);
+
+    #endif
+
+	ret = vvcam_soc_irq_init(pdev, psoc_dev);
+	if (ret < 0)
+		return ret;
+
+	vi_top_phy_mode = phy_mode << PHY_MODE;/* 2: 4lanes,  5: 2lanes*/
+	if(0 != imx_hdr){
+		for (i = 0; i <= 5; i++){
+			if(imx_hdr & (0x01<<i) ){
+				vi_top_phy_mode |= 1 << (IMX327_HDR_EN_0 + i*2);
+				vi_top_phy_mode |= 1 << (IMX327_HDR_RAW12_0 + i*2);
+			}
+		}
+	}
+	pr_err(" %s, vi_top_phy_mode:%x \n", __func__,vi_top_phy_mode);
+	psoc_dev->soc_access.write(psoc_dev, VI_REG_PHY_CONNECT_MODE,vi_top_phy_mode);
+
+	psoc_dev->soc_access.write(psoc_dev, 0x04, 0x0);  /*csix_vicapx_sel: all from MIPI-CSI*/
+	//psoc_dev->soc_access.write(psoc_dev, 0x04, 0x3f);  /*csix_vicapx_sel: all from vicap*/
+#ifdef MCM_DEBUG
+	/* isp0_dvp0_sel -> csi0, isp0_dvp1_sel -> csi1, isp1_dvp0_sel -> csi0, isp1_dvp1_sel -> csi1 */
+	psoc_dev->soc_access.write(psoc_dev, 0x08, (0 << 0) | (1 << 3) | (0 << 6) | (1 << 9));
+#else
+	/* isp0_dvp0_sel -> csi0, isp0_dvp1_sel -> csi0, isp1_dvp0_sel -> csi1, isp1_dvp1_sel -> csi1 */
+ 	psoc_dev->soc_access.write(psoc_dev, 0x08, (1 << 0) | (0 << 3) | (1 << 6) | (1 << 9));
+#endif
+	psoc_dev->soc_access.write(psoc_dev, 0x40, 0xffff);      /*ctrl0,isp0,isp1,isp0_dvp0,isp0_dvp1,isp1_dvp0,isp1_dvp1 clk en*/
+	psoc_dev->soc_access.write(psoc_dev, 0x1c, (0xe0 << 8) | 0xe0);
+	psoc_dev->soc_access.write(psoc_dev, 0x20, (4 << 9));
+	psoc_dev->soc_access.write(psoc_dev, 0x24, (4 << 9));
+	psoc_dev->soc_access.write(psoc_dev, 0x28, (4 << 9));
+	psoc_dev->soc_access.write(psoc_dev, 0x2c, (4 << 9));
+
+	psoc_dev->soc_debug_dc = debugfs_create_file(debug_filename, 0644, NULL, psoc_dev, &soc_debug_fops);
+	psoc_dev->soc_reset = debugfs_create_file(debug_soc_reset, 0644, NULL, psoc_dev, &soc_reset_fops);
+
+	pr_info("exit %s\n", __func__);
+	return ret;
+}
+
+static int vvcam_soc_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct vvcam_soc_driver_dev *pdriver_dev;
+	struct vvcam_soc_dev *psoc_dev;
+
+	pr_info("enter %s\n", __func__);
+	devise_register_index--;
+	pdriver_dev = platform_get_drvdata(pdev);
+
+	psoc_dev = pdriver_dev->private;
+	//iounmap(psoc_dev->base);
+	iounmap(psoc_dev->scu_base);
+    debugfs_remove(psoc_dev->soc_debug_dc);
+    debugfs_remove(psoc_dev->soc_reset);
+
+	cdev_del(&pdriver_dev->cdev);
+	device_destroy(pdriver_dev->class, pdriver_dev->devt);
+	unregister_chrdev_region(pdriver_dev->devt, VIVCAM_SOC_MAXCNT);
+	if (devise_register_index == 0) {
+		class_destroy(pdriver_dev->class);
+	}
+
+	ret = win2030_tbu_power(&pdev->dev, false);
+	if (ret) {
+		pr_err("isp tbu power down failed\n");
+		return -1;
+	}
+	vvcam_clk_reset_fini(&pdriver_dev->isp_crg);
+
+	return 0;
+}
+
+static const struct of_device_id visoc_of_match[] = {
+	{ .compatible = "esw,vi-common-csr" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, visoc_of_match);
+
+static struct platform_driver vvcam_soc_driver = {
+	.probe = vvcam_soc_probe,
+	.remove = vvcam_soc_remove,
+	.driver = {
+		   .name = VIVCAM_SOC_NAME,
+		   .owner = THIS_MODULE,
+		   .of_match_table = visoc_of_match,
+	}
+};
+
+static int __init vvcam_soc_init_module(void)
+{
+	int ret = 0;
+
+	pr_info("enter %s\n", __func__);
+
+	ret = platform_driver_register(&vvcam_soc_driver);
+	if (ret) {
+		pr_err("register platform driver failed.\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static void __exit vvcam_soc_exit_module(void)
+{
+	pr_info("enter %s\n", __func__);
+
+	platform_driver_unregister(&vvcam_soc_driver);
+}
+
+late_initcall(vvcam_soc_init_module);
+module_exit(vvcam_soc_exit_module);
+
+MODULE_DESCRIPTION("ISP");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/Makefile b/drivers/media/platform/eswin/isp/vvcam/v4l2/Makefile
new file mode 100644
index 000000000000..159663330962
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/Makefile
@@ -0,0 +1,61 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+all:
+	@cd isp;      make || exit $$?;
+	@cd video;    make || exit $$?;
+clean:
+	@cd isp;     make clean;
+	@cd video;   make clean;
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/common/vvcam_v4l2_common.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/common/vvcam_v4l2_common.h
new file mode 100644
index 000000000000..dd968ee22302
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/common/vvcam_v4l2_common.h
@@ -0,0 +1,160 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_V4L2_COMMON_H__
+#define __VVCAM_V4L2_COMMON_H__
+#include <linux/videodev2.h>
+
+struct vvcam_video_plane {
+	uint32_t dma_addr;
+	uint32_t size;
+};
+
+struct vvcam_vb2_buffer {
+    struct vb2_v4l2_buffer vb;
+	unsigned int num_planes;
+	struct vvcam_video_plane planes[VIDEO_MAX_PLANES];
+    struct list_head list;
+    uint32_t sequence;
+};
+
+struct vvcam_pad_reqbufs {
+    int pad;
+    uint32_t num_buffers;
+};
+
+struct vvcam_pad_buf {
+    uint32_t pad;
+    struct vvcam_vb2_buffer *buf;
+};
+
+struct vvcam_pad_stream_status {
+    uint32_t pad;
+    uint32_t status;
+};
+
+struct vvcam_pad_queryctrl {
+    uint32_t pad;
+    struct v4l2_queryctrl *query_ctrl;
+};
+
+struct vvcam_pad_query_ext_ctrl {
+    uint32_t pad;
+    struct v4l2_query_ext_ctrl *query_ext_ctrl;
+};
+
+struct vvcam_pad_control {
+    uint32_t pad;
+    struct v4l2_control *control;
+};
+
+struct vvcam_pad_ext_controls {
+    uint32_t pad;
+    struct v4l2_ext_controls *ext_controls;
+};
+
+struct vvcam_pad_querymenu {
+    uint32_t pad;
+    struct v4l2_querymenu *querymenu;
+};
+
+#define MBUS_CSI2_MAX_DATA_LANES		8
+
+struct mbus_config_mipi_csi2 {
+	unsigned int flags;
+	unsigned char data_lanes[MBUS_CSI2_MAX_DATA_LANES];
+	unsigned char clock_lane;
+	unsigned char num_data_lanes;
+	bool lane_polarities[1 + MBUS_CSI2_MAX_DATA_LANES];
+};
+
+enum mbus_type {
+    MBUS_UNKNOWN,
+	MBUS_PARALLEL,
+	MBUS_BT656,
+	MBUS_CSI1,
+	MBUS_CCP2,
+	MBUS_CSI2_DPHY,
+	MBUS_CSI2_CPHY,
+    MBUS_DPI,
+	MBUS_INVALID,
+};
+
+typedef struct {
+    enum mbus_type type;
+	union {
+		// struct v4l2_mbus_config_parallel parallel;
+		// struct v4l2_mbus_config_mipi_csi1 mipi_csi1;
+		struct mbus_config_mipi_csi2 mipi_csi2;
+	} bus;
+} sensor_mbus_config;
+
+#define VVCAM_PAD_REQUBUFS       _IOWR('V',  BASE_VIDIOC_PRIVATE + 0, struct vvcam_pad_reqbufs)
+#define VVCAM_PAD_BUF_DONE       _IOWR('V',  BASE_VIDIOC_PRIVATE + 1, struct vvcam_pad_buf)
+#define VVCAM_PAD_BUF_QUEUE      _IOWR('V',  BASE_VIDIOC_PRIVATE + 2, struct vvcam_pad_buf)
+#define VVCAM_PAD_S_STREAM       _IOWR('V',  BASE_VIDIOC_PRIVATE + 3, struct vvcam_pad_stream_status)
+
+#define VVCAM_PAD_QUERYCTRL      _IOWR('V',  BASE_VIDIOC_PRIVATE + 4, struct vvcam_pad_queryctrl)
+#define VVCAM_PAD_QUERY_EXT_CTRL _IOWR('V',  BASE_VIDIOC_PRIVATE + 5, struct vvcam_pad_query_ext_ctrl)
+#define VVCAM_PAD_G_CTRL         _IOWR('V',  BASE_VIDIOC_PRIVATE + 6, struct vvcam_pad_control)
+#define VVCAM_PAD_S_CTRL         _IOWR('V',  BASE_VIDIOC_PRIVATE + 7, struct vvcam_pad_control)
+#define VVCAM_PAD_G_EXT_CTRLS    _IOWR('V',  BASE_VIDIOC_PRIVATE + 8, struct vvcam_pad_ext_controls)
+#define VVCAM_PAD_S_EXT_CTRLS    _IOWR('V',  BASE_VIDIOC_PRIVATE + 9, struct vvcam_pad_ext_controls)
+#define VVCAM_PAD_TRY_EXT_CTRLS  _IOWR('V',  BASE_VIDIOC_PRIVATE + 10, struct vvcam_pad_ext_controls)
+#define VVCAM_PAD_QUERYMENU      _IOWR('V',  BASE_VIDIOC_PRIVATE + 11, struct vvcam_pad_querymenu)
+
+#define VVAM_CMD_GET_SENSOR_MBUS_CONFIG _IOR('V', BASE_VIDIOC_PRIVATE + 12, sensor_mbus_config)
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/common/vvcam_v4l2_std_exts.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/common/vvcam_v4l2_std_exts.h
new file mode 100644
index 000000000000..f665dbe291d8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/common/vvcam_v4l2_std_exts.h
@@ -0,0 +1,384 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_V4L2_STD_EXT_H__
+#define __VVCAM_V4L2_STD_EXT_H__
+
+#include <linux/videodev2.h>
+
+
+
+// 400-8bit 
+#ifndef V4L2_PIX_FMT_GREY
+#define V4L2_PIX_FMT_GREY v4l2_fourcc('G', 'R', 'E', 'Y')
+#endif
+
+// 400-10bit unalign 
+#ifndef V4L2_PIX_FMT_Y10BPACK
+#define V4L2_PIX_FMT_Y10BPACK v4l2_fourcc('Y', '1', '0', 'B')
+#endif
+
+// 400-10bit align mode0 
+#ifndef V4L2_PIX_FMT_Y10DWA
+#define V4L2_PIX_FMT_Y10DWA v4l2_fourcc('Y', '1', '0', 'D')
+#endif
+
+// 400-10bit align mode1
+#ifndef V4L2_PIX_FMT_Y10
+#define V4L2_PIX_FMT_Y10 v4l2_fourcc('Y', '1', '0', '')
+#endif
+
+/////////////////////////////////////////////////////////
+// 420sp-10bit unalign
+#ifndef V4L2_PIX_FMT_P00BPACK
+#define V4L2_PIX_FMT_P00BPACK v4l2_fourcc('P', '0', '0', 'B')
+#endif
+
+// 420sp-10bit double word align
+#ifndef V4L2_PIX_FMT_P00DWA
+#define V4L2_PIX_FMT_P00DWA v4l2_fourcc('P', '0', '0', 'D')
+#endif
+
+// 420sp-10bit align mode 1,every pixel 16bits
+#ifndef V4L2_PIX_FMT_P010
+#define V4L2_PIX_FMT_P010 v4l2_fourcc('P', '0', '1', '0')
+#endif
+
+// 420sp-12bit unalign
+#ifndef V4L2_PIX_FMT_P02BPACK
+#define V4L2_PIX_FMT_P02BPACK v4l2_fourcc('P', '0', '2', 'B')
+#endif
+
+/////////////////////////////////////////////////////////
+// 422sp-10bit unalign
+#ifndef V4L2_PIX_FMT_P20BPACK
+#define V4L2_PIX_FMT_P20BPACK v4l2_fourcc('P', '2', '0', 'B')
+#endif
+
+// 422sp-10bit double word align
+#ifndef V4L2_PIX_FMT_P20DWA
+#define V4L2_PIX_FMT_P20DWA v4l2_fourcc('P', '2', '0', 'D')
+#endif
+
+// 422sp-10bit mode 1
+#ifndef V4L2_PIX_FMT_P210
+#define V4L2_PIX_FMT_P210 v4l2_fourcc('P', '2', '1', '0')
+#endif
+
+// 422sp-12bit unalign
+#ifndef V4L2_PIX_FMT_P22BPACK
+#define V4L2_PIX_FMT_P22BPACK v4l2_fourcc('P', '2', '2', 'B')
+#endif
+
+// 422I-10bit unalign
+#ifndef V4L2_PIX_FMT_I20BPACK
+#define V4L2_PIX_FMT_I20BPACK v4l2_fourcc('I', '2', '0', 'B')
+#endif
+
+// 422I-10bit align mode 1, every pixel 16bits
+#ifndef V4L2_PIX_FMT_I210
+#define V4L2_PIX_FMT_I210 v4l2_fourcc('I', '2', '1', '0')
+#endif
+
+/////////////////////////////////////////////////////////
+
+// 444P-8bit unalign
+#ifndef V4L2_PIX_FMT_M48BPACK
+#define V4L2_PIX_FMT_M48BPACK v4l2_fourcc('M', '4', '8', 'B')
+#endif
+
+// 444I 8bit unalign
+#ifndef V4L2_PIX_FMT_I48BPACK
+#define V4L2_PIX_FMT_I48BPACK v4l2_fourcc('I', '4', '8', 'B')
+#endif
+
+// 444I 8bit align mode 0 
+#ifndef V4L2_PIX_FMT_I48DWA
+#define V4L2_PIX_FMT_I48DWA v4l2_fourcc('I', '4', '8', 'D')
+#endif
+
+// 444I 10bit mode0 
+#ifndef V4L2_PIX_FMT_I40DWA
+#define V4L2_PIX_FMT_I40DWA v4l2_fourcc('I', '4', '0', 'D')
+#endif
+
+
+/////////////////////////////////////////////////////////
+// RGB
+// RGB888
+#ifndef V4L2_PIX_FMT_RGB24
+#define V4L2_PIX_FMT_RGB24 v4l2_fourcc('R', 'G', 'B', '3')
+#endif
+
+// RGB888 align mode 0
+#ifndef V4L2_PIX_FMT_RGB24DWA
+#define V4L2_PIX_FMT_RGB24DWA v4l2_fourcc('R', 'G', '3', 'D')
+#endif
+
+// RGB888P
+#ifndef V4L2_PIX_FMT_RGB24P
+#define V4L2_PIX_FMT_RGB24P v4l2_fourcc('R', 'G', '3', 'P')
+#endif
+
+/////////////////////////////////////////////////////////
+// RAW
+// 10 bit bit-pack
+#ifndef V4L2_PIX_FMT_SBGGR10BPACK
+#define V4L2_PIX_FMT_SBGGR10BPACK v4l2_fourcc('B', 'G', '0', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG10BPACK
+#define V4L2_PIX_FMT_SGBRG10BPACK v4l2_fourcc('G', 'B', '0', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG10BPACK
+#define V4L2_PIX_FMT_SGRBG10BPACK v4l2_fourcc('G', 'R', '0', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB10BPACK
+#define V4L2_PIX_FMT_SRGGB10BPACK v4l2_fourcc('R', 'G', '0', 'B')
+#endif
+
+// 10 bit dwa
+#ifndef V4L2_PIX_FMT_SBGGR10DWA
+#define V4L2_PIX_FMT_SBGGR10DWA v4l2_fourcc('B', 'G', '0', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG10DWA
+#define V4L2_PIX_FMT_SGBRG10DWA v4l2_fourcc('G', 'B', '0', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG10DWA
+#define V4L2_PIX_FMT_SGRBG10DWA v4l2_fourcc('G', 'R', '0', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB10DWA
+#define V4L2_PIX_FMT_SRGGB10DWA v4l2_fourcc('R', 'G', '0', 'D')
+#endif
+
+// 12 bit bit-pack
+#ifndef V4L2_PIX_FMT_SBGGR12BPACK
+#define V4L2_PIX_FMT_SBGGR12BPACK v4l2_fourcc('B', 'G', '2', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG12BPACK
+#define V4L2_PIX_FMT_SGBRG12BPACK v4l2_fourcc('G', 'B', '2', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG12BPACK
+#define V4L2_PIX_FMT_SGRBG12BPACK v4l2_fourcc('G', 'R', '2', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB12BPACK
+#define V4L2_PIX_FMT_SRGGB12BPACK v4l2_fourcc('R', 'G', '2', 'B')
+#endif
+
+// 12 bit dwa
+#ifndef V4L2_PIX_FMT_SBGGR12DWA
+#define V4L2_PIX_FMT_SBGGR12DWA v4l2_fourcc('B', 'G', '2', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG12DWA
+#define V4L2_PIX_FMT_SGBRG12DWA v4l2_fourcc('G', 'B', '2', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG12DWA
+#define V4L2_PIX_FMT_SGRBG12DWA v4l2_fourcc('G', 'R', '2', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB12DWA
+#define V4L2_PIX_FMT_SRGGB12DWA v4l2_fourcc('R', 'G', '2', 'D')
+#endif
+
+// 14 bit bit-pack
+#ifndef V4L2_PIX_FMT_SBGGR14BPACK
+#define V4L2_PIX_FMT_SBGGR14BPACK v4l2_fourcc('B', 'G', '4', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG14BPACK
+#define V4L2_PIX_FMT_SGBRG14BPACK v4l2_fourcc('G', 'B', '4', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG14BPACK
+#define V4L2_PIX_FMT_SGRBG14BPACK v4l2_fourcc('G', 'R', '4', 'B')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB14BPACK
+#define V4L2_PIX_FMT_SRGGB14BPACK v4l2_fourcc('R', 'G', '4', 'B')
+#endif
+
+// 14 bit dwa
+#ifndef V4L2_PIX_FMT_SBGGR14DWA
+#define V4L2_PIX_FMT_SBGGR14DWA v4l2_fourcc('B', 'G', '4', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG14DWA
+#define V4L2_PIX_FMT_SGBRG14DWA v4l2_fourcc('G', 'B', '4', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG14DWA
+#define V4L2_PIX_FMT_SGRBG14DWA v4l2_fourcc('G', 'R', '4', 'D')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB14DWA
+#define V4L2_PIX_FMT_SRGGB14DWA v4l2_fourcc('R', 'G', '4', 'D')
+#endif
+
+// 14 bit align mode 1
+#ifndef V4L2_PIX_FMT_SBGGR14
+#define V4L2_PIX_FMT_SBGGR14 v4l2_fourcc('B', 'G', '1', '4')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG14
+#define V4L2_PIX_FMT_SGBRG14 v4l2_fourcc('G', 'B', '1', '4')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG14
+#define V4L2_PIX_FMT_SGRBG14 v4l2_fourcc('G', 'R', '1', '4')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB14
+#define V4L2_PIX_FMT_SRGGB14 v4l2_fourcc('R', 'G', '1', '4')
+#endif
+
+// 16 bit unalign
+#ifndef V4L2_PIX_FMT_SBGGR16
+#define V4L2_PIX_FMT_SBGGR16 v4l2_fourcc('B', 'Y', 'R', '2')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG16
+#define V4L2_PIX_FMT_SGBRG16 v4l2_fourcc('G', 'B', '1', '6')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG16
+#define V4L2_PIX_FMT_SGRBG16 v4l2_fourcc('G', 'R', '1', '6')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB16
+#define V4L2_PIX_FMT_SRGGB16 v4l2_fourcc('R', 'G', '1', '6')
+#endif
+
+// 24 bit unalign
+#ifndef V4L2_PIX_FMT_SBGGR24
+#define V4L2_PIX_FMT_SBGGR24 v4l2_fourcc('B', 'G', '2', '4')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGBRG24
+#define V4L2_PIX_FMT_SGBRG24 v4l2_fourcc('G', 'B', '2', '4')
+#endif
+
+#ifndef V4L2_PIX_FMT_SGRBG24
+#define V4L2_PIX_FMT_SGRBG24 v4l2_fourcc('G', 'R', '2', '4')
+#endif
+
+#ifndef V4L2_PIX_FMT_SRGGB24
+#define V4L2_PIX_FMT_SRGGB24 v4l2_fourcc('R', 'G', '2', '4')
+#endif
+
+
+/* media bus format */
+#ifndef MEDIA_BUS_FMT_RGB888_3X8
+#define MEDIA_BUS_FMT_RGB888_3X8        0x101c
+#endif
+
+#ifndef MEDIA_BUS_FMT_SBGGR14_1X14
+#define MEDIA_BUS_FMT_SBGGR14_1X14      0x3019
+#endif
+
+#ifndef MEDIA_BUS_FMT_SGBRG14_1X14
+#define MEDIA_BUS_FMT_SGBRG14_1X14      0x301a
+#endif
+
+#ifndef MEDIA_BUS_FMT_SGRBG14_1X14
+#define MEDIA_BUS_FMT_SGRBG14_1X14      0x301b
+#endif
+
+#ifndef MEDIA_BUS_FMT_SRGGB14_1X14
+#define MEDIA_BUS_FMT_SRGGB14_1X14      0x301c
+#endif
+
+#ifndef MEDIA_BUS_FMT_SBGGR16_1X16
+#define MEDIA_BUS_FMT_SBGGR16_1X16       0x301d
+#endif
+
+#ifndef MEDIA_BUS_FMT_SGBRG16_1X16
+#define MEDIA_BUS_FMT_SGBRG16_1X16       0x301e
+#endif
+
+#ifndef MEDIA_BUS_FMT_SGRBG16_1X16
+#define MEDIA_BUS_FMT_SGRBG16_1X16       0x301f
+#endif
+
+#ifndef MEDIA_BUS_FMT_SRGGB16_1X16
+#define MEDIA_BUS_FMT_SRGGB16_1X16       0x3020
+#endif
+
+#ifndef MEDIA_BUS_FMT_SBGGR24_1X24
+#define MEDIA_BUS_FMT_SBGGR24_1X24       0x3021
+#endif
+
+#ifndef MEDIA_BUS_FMT_SGBRG24_1X24
+#define MEDIA_BUS_FMT_SGBRG24_1X24       0x3022
+#endif
+
+#ifndef MEDIA_BUS_FMT_SGRBG24_1X24
+#define MEDIA_BUS_FMT_SGRBG24_1X24       0x3023
+#endif
+
+#ifndef MEDIA_BUS_FMT_SRGGB24_1X24
+#define MEDIA_BUS_FMT_SRGGB24_1X24       0x3024
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/Makefile b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/Makefile
new file mode 100644
index 000000000000..79632305489a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/Makefile
@@ -0,0 +1,514 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+ifeq ($(VVCAM_PATH),)
+        VVCAM_PATH:=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))/../../../vvcam
+        export VVCAM_PATH
+endif
+ISP_PATH := $(VVCAM_PATH)/v4l2/isp
+
+include $(VVCAM_PATH)/Param.mk
+TARGET = vvcam_isp_subdev
+
+obj-$(CONFIG_VIDEO_ESWIN_ISP) += $(TARGET).o
+$(TARGET)-objs += vvcam_isp_platform.o
+$(TARGET)-objs += vvcam_isp_driver.o
+$(TARGET)-objs += vvcam_isp_event.o
+$(TARGET)-objs += vvcam_isp_procfs.o
+$(TARGET)-objs += isp_ctrl/vvcam_isp_ctrl.o
+
+EXTRA_CFLAGS += -I$(ISP_PATH)
+EXTRA_CFLAGS += -I$(ISP_PATH)/../common/
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl
+
+USER_INC:=$(ISP_PATH)/isp_ctrl/vvcam_isp_ctrl.h
+
+ifeq ($(ISP_AE_V3), 1)
+$(TARGET)-objs += isp_ctrl/ae/ae_v3/vvcam_isp_ae.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ae/ae_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/ae/ae_v3/*.h
+endif
+
+ifeq ($(ISP_AE_V4), 1)
+$(TARGET)-objs += isp_ctrl/ae/ae_v4/vvcam_isp_ae.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ae/ae_v4/
+USER_INC += $(ISP_PATH)/isp_ctrl/ae/ae_v4/*.h
+endif
+
+ifeq ($(ISP_AE_V4_1), 1)
+$(TARGET)-objs += isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ae/ae_v4_1/
+USER_INC += $(ISP_PATH)/isp_ctrl/ae/ae_v4_1/*.h
+endif
+
+ifeq ($(ISP_AWB_V3), 1)
+$(TARGET)-objs += isp_ctrl/awb/awb_v3/vvcam_isp_awb.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/awb/awb_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/awb/awb_v3/*.h
+endif
+
+ifeq ($(ISP_AWB_V4), 1)
+$(TARGET)-objs += isp_ctrl/awb/awb_v4/vvcam_isp_awb.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/awb/awb_v4/
+USER_INC += $(ISP_PATH)/isp_ctrl/awb/awb_v4/*.h
+endif
+
+ifeq ($(ISP_AWB_V4_1), 1)
+$(TARGET)-objs += isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/awb/awb_v4_1/
+USER_INC += $(ISP_PATH)/isp_ctrl/awb/awb_v4_1/*.h
+endif
+
+ifeq ($(ISP_GC_V1), 1)
+$(TARGET)-objs += isp_ctrl/gc/gc_v1/vvcam_isp_gc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/gc/gc_v1/
+USER_INC += $(ISP_PATH)/isp_ctrl/gc/gc_v1/*.h
+endif
+
+ifeq ($(ISP_GC_V2), 1)
+$(TARGET)-objs += isp_ctrl/gc/gc_v2/vvcam_isp_gc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/gc/gc_v2/
+USER_INC += $(ISP_PATH)/isp_ctrl/gc/gc_v2/*.h
+endif
+
+ifeq ($(ISP_2DNR_V3), 1)
+$(TARGET)-objs += isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/2dnr/2dnr_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/2dnr/2dnr_v3/*.h
+endif
+
+ifeq ($(ISP_2DNR_V5_2), 1)
+$(TARGET)-objs += isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/2dnr/2dnr_v5_2/
+USER_INC += $(ISP_PATH)/isp_ctrl/2dnr/2dnr_v5_2/*.h
+endif
+
+ifeq ($(ISP_2DNR_V6), 1)
+$(TARGET)-objs += isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/2dnr/2dnr_v6/
+USER_INC += $(ISP_PATH)/isp_ctrl/2dnr/2dnr_v6/*.h
+endif
+
+ifeq ($(ISP_EE_V1), 1)
+$(TARGET)-objs += isp_ctrl/ee/ee_v1/vvcam_isp_ee.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ee/ee_v1/
+USER_INC += $(ISP_PATH)/isp_ctrl/ee/ee_v1/*.h
+endif
+
+ifeq ($(ISP_EE_V2), 1)
+$(TARGET)-objs += isp_ctrl/ee/ee_v2/vvcam_isp_ee.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ee/ee_v2/
+USER_INC += $(ISP_PATH)/isp_ctrl/ee/ee_v2/*.h
+endif
+
+ifeq ($(ISP_EE_V3), 1)
+$(TARGET)-objs += isp_ctrl/ee/ee_v3/vvcam_isp_ee.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ee/ee_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/ee/ee_v3/*.h
+endif
+
+ifeq ($(ISP_CPROC), 1)
+$(TARGET)-objs += isp_ctrl/cproc/cproc/vvcam_isp_cproc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/cproc/cproc/
+USER_INC += $(ISP_PATH)/isp_ctrl/cproc/cproc/*.h
+endif
+
+ifeq ($(ISP_WDR_V3), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v3/*.h
+endif
+
+ifeq ($(ISP_WDR_V4), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v4/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v4/*.h
+endif
+
+ifeq ($(ISP_WDR_V5), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v5/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v5/*.h
+endif
+
+ifeq ($(ISP_WDR_V5_1), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v5_1/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v5_1/*.h
+endif
+
+ifeq ($(ISP_WDR_V5_2), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v5_2/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v5_2/*.h
+endif
+
+ifeq ($(ISP_WDR_V5_2_1), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v5_2_1/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v5_2_1/*.h
+endif
+
+ifeq ($(ISP_WDR_V5_3), 1)
+$(TARGET)-objs += isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wdr/wdr_v5_3/
+USER_INC += $(ISP_PATH)/isp_ctrl/wdr/wdr_v5_3/*.h
+endif
+
+ifeq ($(ISP_BLS), 1)
+$(TARGET)-objs += isp_ctrl/bls/bls/vvcam_isp_bls.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/bls/bls/
+USER_INC += $(ISP_PATH)/isp_ctrl/bls/bls/*.h
+endif
+
+ifeq ($(ISP_CCM), 1)
+$(TARGET)-objs += isp_ctrl/ccm/ccm/vvcam_isp_ccm.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ccm/ccm/
+USER_INC += $(ISP_PATH)/isp_ctrl/ccm/ccm/*.h
+endif
+
+ifeq ($(ISP_DPCC), 1)
+$(TARGET)-objs += isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/dpcc/dpcc/
+USER_INC += $(ISP_PATH)/isp_ctrl/dpcc/dpcc/*.h
+endif
+
+ifeq ($(ISP_GE), 1)
+$(TARGET)-objs += isp_ctrl/ge/ge/vvcam_isp_ge.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ge/ge/
+USER_INC += $(ISP_PATH)/isp_ctrl/ge/ge/*.h
+endif
+
+ifeq ($(ISP_LSC_V1), 1)
+$(TARGET)-objs += isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/lsc/lsc_v1/
+endif
+
+ifeq ($(ISP_LSC_V2), 1)
+$(TARGET)-objs += isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/lsc/lsc_v2/
+USER_INC += $(ISP_PATH)/isp_ctrl/lsc/lsc_v2/*.h
+endif
+
+ifeq ($(ISP_LSC_V3), 1)
+$(TARGET)-objs += isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/lsc/lsc_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/lsc/lsc_v3/*.h
+endif
+
+ifeq ($(ISP_LSC_V4), 1)
+$(TARGET)-objs += isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/lsc/lsc_v4/
+USER_INC += $(ISP_PATH)/isp_ctrl/lsc/lsc_v4/*.h
+endif
+
+ifeq ($(ISP_WB_V1_1), 1)
+$(TARGET)-objs += isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wb/wb_v1_1/
+USER_INC += $(ISP_PATH)/isp_ctrl/wb/wb_v1_1/*.h
+endif
+
+ifeq ($(ISP_WB_V1), 1)
+$(TARGET)-objs += isp_ctrl/wb/wb_v1/vvcam_isp_wb.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/wb/wb_v1/
+USER_INC += $(ISP_PATH)/isp_ctrl/wb/wb_v1/*.h
+endif
+
+ifeq ($(ISP_3DNR_V1_2), 1)
+$(TARGET)-objs += isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/3dnr/3dnr_v1_2
+USER_INC += $(ISP_PATH)/isp_ctrl/3dnr/3dnr_v1_2/*.h
+endif
+
+ifeq ($(ISP_3DNR_V2_1), 1)
+$(TARGET)-objs += isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/3dnr/3dnr_v2_1
+USER_INC += $(ISP_PATH)/isp_ctrl/3dnr/3dnr_v2_1/*.h
+endif
+
+ifeq ($(ISP_3DNR_V2_4), 1)
+$(TARGET)-objs += isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/3dnr/3dnr_v2_4
+USER_INC += $(ISP_PATH)/isp_ctrl/3dnr/3dnr_v2_4/*.h
+endif
+
+ifeq ($(ISP_3DNR_V3), 1)
+$(TARGET)-objs += isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/3dnr/3dnr_v3
+USER_INC += $(ISP_PATH)/isp_ctrl/3dnr/3dnr_v3/*.h
+endif
+
+ifeq ($(ISP_3DNR_V3_1), 1)
+$(TARGET)-objs += isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/3dnr/3dnr_v3_1
+USER_INC += $(ISP_PATH)/isp_ctrl/3dnr/3dnr_v3_1/*.h
+endif
+
+ifeq ($(ISP_3DNR_V4), 1)
+$(TARGET)-objs += isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/3dnr/3dnr_v4
+USER_INC += $(ISP_PATH)/isp_ctrl/3dnr/3dnr_v4/*.h
+endif
+
+ifeq ($(ISP_AF_V3), 1)
+$(TARGET)-objs += isp_ctrl/af/af_v3/vvcam_isp_af.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/af/af_v3
+USER_INC += $(ISP_PATH)/isp_ctrl/af/af_v3/*.h
+endif
+
+ifeq ($(ISP_AF_V4), 1)
+$(TARGET)-objs += isp_ctrl/af/af_v4/vvcam_isp_af.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/af/af_v4
+USER_INC += $(ISP_PATH)/isp_ctrl/af/af_v4/*.h
+endif
+
+ifeq ($(ISP_AF_V4_1_1), 1)
+$(TARGET)-objs += isp_ctrl/af/af_v4_1_1/vvcam_isp_af.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/af/af_v4_1_1
+USER_INC += $(ISP_PATH)/isp_ctrl/af/af_v4_1_1/*.h
+endif
+
+ifeq ($(ISP_AF_V4_3), 1)
+$(TARGET)-objs += isp_ctrl/af/af_v4_3/vvcam_isp_af.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/af/af_v4_3
+USER_INC += $(ISP_PATH)/isp_ctrl/af/af_v4_3/*.h
+endif
+
+ifeq ($(ISP_PDAF_V1), 1)
+$(TARGET)-objs += isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/pdaf/pdaf_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/pdaf/pdaf_v1/*.h
+endif
+
+ifeq ($(ISP_AFM_V1), 1)
+$(TARGET)-objs += isp_ctrl/afm/afm_v1/vvcam_isp_afm.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/afm/afm_v1/
+USER_INC += $(ISP_PATH)/isp_ctrl/afm/afm_v1/*.h
+endif
+
+ifeq ($(ISP_AFM_V1_1), 1)
+$(TARGET)-objs += isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/afm/afm_v1_1/
+USER_INC += $(ISP_PATH)/isp_ctrl/afm/afm_v1_1/*.h
+endif
+
+ifeq ($(ISP_AFM_V3), 1)
+$(TARGET)-objs += isp_ctrl/afm/afm_v3/vvcam_isp_afm.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/afm/afm_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/afm/afm_v3/*.h
+endif
+
+ifeq ($(ISP_DMSC_V1), 1)
+$(TARGET)-objs += isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/dmsc/dmsc_v1/
+USER_INC += $(ISP_PATH)/isp_ctrl/dmsc/dmsc_v1/*.h
+endif
+
+ifeq ($(ISP_DMSC_V2), 1)
+$(TARGET)-objs += isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/dmsc/dmsc_v2/
+USER_INC += $(ISP_PATH)/isp_ctrl/dmsc/dmsc_v2/*.h
+endif
+
+ifeq ($(ISP_DMSC_V3), 1)
+$(TARGET)-objs += isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/dmsc/dmsc_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/dmsc/dmsc_v3/*.h
+endif
+
+ifeq ($(ISP_EXP_V2), 1)
+$(TARGET)-objs += isp_ctrl/exp/exp_v2/vvcam_isp_exp.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/exp/exp_v2/
+USER_INC += $(ISP_PATH)/isp_ctrl/exp/exp_v2/*.h
+endif
+
+ifeq ($(ISP_EXP_V3), 1)
+$(TARGET)-objs += isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/exp/exp_v3/
+USER_INC += $(ISP_PATH)/isp_ctrl/exp/exp_v3/*.h
+endif
+
+ifeq ($(ISP_GTM_V1), 1)
+$(TARGET)-objs += isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/gtm/gtm_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/gtm/gtm_v1/*.h
+endif
+
+ifeq ($(ISP_HDR_V1_3), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v1_3
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v1_3/*.h
+endif
+
+ifeq ($(ISP_HDR_V2), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v2
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v2/*.h
+endif
+
+ifeq ($(ISP_HDR_V2_1), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v2_1
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v2_1/*.h
+endif
+
+ifeq ($(ISP_HDR_V2_1_2DOL), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v2_1_2dol
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v2_1_2dol/*.h
+endif
+
+ifeq ($(ISP_HDR_V3), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v3
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v3/*.h
+endif
+
+ifeq ($(ISP_HDR_V3_2DOL), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v3_2dol
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v3_2dol/*.h
+endif
+
+ifeq ($(ISP_HDR_V3_1), 1)
+$(TARGET)-objs += isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/hdr/hdr_v3_1
+USER_INC += $(ISP_PATH)/isp_ctrl/hdr/hdr_v3_1/*.h
+endif
+
+ifeq ($(ISP_RGBIR_V1), 1)
+$(TARGET)-objs += isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/rgbir/rgbir_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/rgbir/rgbir_v1/*.h
+endif
+
+ifeq ($(ISP_RGBIR_V1_1), 1)
+$(TARGET)-objs += isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/rgbir/rgbir_v1_1
+USER_INC += $(ISP_PATH)/isp_ctrl/rgbir/rgbir_v1_1/*.h
+endif
+
+ifeq ($(ISP_RGBIR_V2), 1)
+$(TARGET)-objs += isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/rgbir/rgbir_v2
+USER_INC += $(ISP_PATH)/isp_ctrl/rgbir/rgbir_v2/*.h
+endif
+
+ifeq ($(ISP_RGBIR_V2_1), 1)
+$(TARGET)-objs += isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/rgbir/rgbir_v2_1
+USER_INC += $(ISP_PATH)/isp_ctrl/rgbir/rgbir_v2_1/*.h
+endif
+
+ifeq ($(ISP_DG_V1), 1)
+$(TARGET)-objs += isp_ctrl/dg/dg_v1/vvcam_isp_dg.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/dg/dg_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/dg/dg_v1/*.h
+endif
+
+ifeq ($(ISP_CMPD_20BIT), 1)
+$(TARGET)-objs += isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/cpd/cpd_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/cpd/cpd_v1/*.h
+endif
+
+ifeq ($(ISP_CMPD_24BIT), 1)
+$(TARGET)-objs += isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/cpd/cpd_v1_1
+USER_INC += $(ISP_PATH)/isp_ctrl/cpd/cpd_v1_1/*.h
+endif
+
+ifeq ($(ISP_DPF_V1), 1)
+$(TARGET)-objs += isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/dpf/dpf_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/dpf/dpf_v1/*.h
+endif
+
+ifeq ($(ISP_CNR_V1), 1)
+$(TARGET)-objs += isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/cnr/cnr_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/cnr/cnr_v1/*.h
+endif
+
+ifeq ($(ISP_CNR_V2_1), 1)
+$(TARGET)-objs += isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/cnr/cnr_v2_1
+USER_INC += $(ISP_PATH)/isp_ctrl/cnr/cnr_v2_1/*.h
+endif
+
+ifeq ($(ISP_CNR_V2_2), 1)
+$(TARGET)-objs += isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/cnr/cnr_v2_2
+USER_INC += $(ISP_PATH)/isp_ctrl/cnr/cnr_v2_2/*.h
+endif
+
+ifeq ($(ISP_GWDR_V1), 1)
+$(TARGET)-objs += isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/gwdr/gwdr_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/gwdr/gwdr_v1/*.h
+endif
+
+ifeq ($(ISP_YNR_V1), 1)
+$(TARGET)-objs += isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/ynr/ynr_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/ynr/ynr_v1/*.h
+endif
+
+ifeq ($(ISP_LUT3D_V1), 1)
+$(TARGET)-objs += isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.o
+EXTRA_CFLAGS += -I$(ISP_PATH)/isp_ctrl/lut3d/lut3d_v1
+USER_INC += $(ISP_PATH)/isp_ctrl/lut3d/lut3d_v1/*.h
+endif
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.c
new file mode 100644
index 000000000000..260f99ff17d6
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.c
@@ -0,0 +1,310 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_2dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_2dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_2DNR_ENABLE:
+        case VVCAM_ISP_CID_2DNR_RESET:
+        case VVCAM_ISP_CID_2DNR_MODE:
+        case VVCAM_ISP_CID_2DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_2DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_2DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_MANU_PRE_GC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_2dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_2DNR_ENABLE:
+        case VVCAM_ISP_CID_2DNR_RESET:
+        case VVCAM_ISP_CID_2DNR_MODE:
+        case VVCAM_ISP_CID_2DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_2DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_2DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_MANU_PRE_GC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_2DNR_STAT_PRE_GC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_2dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_2dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_2dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_2dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_SIGMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_pre_gc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_PRE_GC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_pre_gc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        /* float 0.1~100 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma",
+        .step = 1,
+        .min  = 1,
+        .max  = 1000,
+        .def  = 50,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_PRE_GC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_pre_gc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        /* float 0.1~100 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma",
+        .step = 1,
+        .min  = 1,
+        .max  = 1000,
+        .def  = 50,
+    },
+};
+
+int vvcam_isp_2dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_2dnr_ctrls);
+}
+
+int vvcam_isp_2dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_2dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_2dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 2dnr ctrl %s failed %d.\n",
+                vvcam_isp_2dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.h
new file mode 100644
index 000000000000..19001d2e9798
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v3/vvcam_isp_2dnr.h
@@ -0,0 +1,84 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_2DNR_H__
+#define __VVCAM_ISP_2DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_2DNR_ENABLE           (VVCAM_ISP_CID_2DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_2DNR_RESET            (VVCAM_ISP_CID_2DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_2DNR_MODE             (VVCAM_ISP_CID_2DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_2DNR_AUTO_LEVEL       (VVCAM_ISP_CID_2DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_2DNR_AUTO_GAIN        (VVCAM_ISP_CID_2DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_2DNR_AUTO_SIGMA       (VVCAM_ISP_CID_2DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_2DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_2DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_ENABLE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_2DNR_MANU_PRE_GC_ENABLE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_2DNR_MANU_STRENGTH    (VVCAM_ISP_CID_2DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA       (VVCAM_ISP_CID_2DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_2DNR_STAT_PRE_GC_ENABLE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_2DNR_STAT_STRENGTH    (VVCAM_ISP_CID_2DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA       (VVCAM_ISP_CID_2DNR_BASE + 0x000D)
+
+#ifdef __KERNEL__
+int vvcam_isp_2dnr_ctrl_count(void);
+int vvcam_isp_2dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.c
new file mode 100644
index 000000000000..7ccc794230d8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.c
@@ -0,0 +1,530 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_2dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_2dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_2DNR_ENABLE:
+        case VVCAM_ISP_CID_2DNR_RESET:
+        case VVCAM_ISP_CID_2DNR_ENABLE_LUMA:
+        case VVCAM_ISP_CID_2DNR_MODE:
+        case VVCAM_ISP_CID_2DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_2DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_2DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_PRE_GC_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_RANGE:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y:
+        // case VVCAM_ISP_CID_2DNR_BLEND_STATIC:
+        // case VVCAM_ISP_CID_2DNR_BLEND_MOTION:
+        // case VVCAM_ISP_CID_2DNR_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_SIGMA_OFFSET:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_2dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_2DNR_ENABLE:
+        case VVCAM_ISP_CID_2DNR_RESET:
+        case VVCAM_ISP_CID_2DNR_ENABLE_LUMA:
+        case VVCAM_ISP_CID_2DNR_MODE:
+        case VVCAM_ISP_CID_2DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_2DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_2DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_PRE_GC_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_RANGE:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y:
+        // case VVCAM_ISP_CID_2DNR_BLEND_STATIC:
+        // case VVCAM_ISP_CID_2DNR_BLEND_MOTION:
+        // case VVCAM_ISP_CID_2DNR_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_SIGMA_OFFSET:
+        case VVCAM_ISP_CID_2DNR_STAT_PRE_GC_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_RANGE:
+        case VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_Y:
+        // case VVCAM_ISP_CID_2DNR_STAT_BLEND_STATIC:
+        // case VVCAM_ISP_CID_2DNR_STAT_BLEND_MOTION:
+        // case VVCAM_ISP_CID_2DNR_STAT_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_OFFSET:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_2dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_2dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_2dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_2dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_ENABLE_LUMA,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_enable_luma",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_SIGMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {20},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_pre_gc_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20},
+    },
+    {
+        /* uint16_t array 8x16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 12},
+    },
+    {
+        /* uint16_t array 8x16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {20, 12},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_PRE_GC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_pre_gc_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        /* float 0.1~100. */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma",
+        .step = 1,
+        .min  = 1,
+        .max  = 1000,
+        .def  = 50,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA_RANGE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        /* uint16_t array 12x16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12},
+    },
+    {
+        /* uint16_t array 12x16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {12},
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~32.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_SIGMA_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_sigma_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_PRE_GC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_pre_gc_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        /* float 0.1~100. */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma",
+        .step = 1,
+        .min  = 1,
+        .max  = 1000,
+        .def  = 50,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_RANGE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        /* uint16_t array 12x16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12},
+    },
+    {
+        /* uint16_t array 12x16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {12},
+    },
+        {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~32.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+};
+
+int vvcam_isp_2dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_2dnr_ctrls);
+}
+
+int vvcam_isp_2dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_2dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_2dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 2dnr ctrl %s failed %d.\n",
+                vvcam_isp_2dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.h
new file mode 100644
index 000000000000..016007c74ad7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v5_2/vvcam_isp_2dnr.h
@@ -0,0 +1,111 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_2DNR_H__
+#define __VVCAM_ISP_2DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_2DNR_ENABLE           (VVCAM_ISP_CID_2DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_2DNR_RESET            (VVCAM_ISP_CID_2DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_2DNR_ENABLE_LUMA      (VVCAM_ISP_CID_2DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_2DNR_MODE             (VVCAM_ISP_CID_2DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_2DNR_AUTO_LEVEL       (VVCAM_ISP_CID_2DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_2DNR_AUTO_GAIN        (VVCAM_ISP_CID_2DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_2DNR_AUTO_SIGMA       (VVCAM_ISP_CID_2DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_2DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_2DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_2DNR_AUTO_PRE_GC_STRENGTH \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000B)
+
+#define VVCAM_ISP_CID_2DNR_MANU_PRE_GC_STRENGTH \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_2DNR_MANU_STRENGTH    (VVCAM_ISP_CID_2DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA       (VVCAM_ISP_CID_2DNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA_RANGE (VVCAM_ISP_CID_2DNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_2DNR_BLEND_STATIC     (VVCAM_ISP_CID_2DNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_2DNR_BLEND_MOTION     (VVCAM_ISP_CID_2DNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_2DNR_BLEND_SLOPE      (VVCAM_ISP_CID_2DNR_BASE + 0x0014)
+#define VVCAM_ISP_CID_2DNR_SIGMA_OFFSET     (VVCAM_ISP_CID_2DNR_BASE + 0x0015)
+#define VVCAM_ISP_CID_2DNR_STAT_PRE_GC_STRENGTH \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0016)
+#define VVCAM_ISP_CID_2DNR_STAT_STRENGTH    (VVCAM_ISP_CID_2DNR_BASE + 0x0017)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA       (VVCAM_ISP_CID_2DNR_BASE + 0x0018)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_RANGE (VVCAM_ISP_CID_2DNR_BASE + 0x0019)
+#define VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001A)
+#define VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001B)
+#define VVCAM_ISP_CID_2DNR_STAT_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001C)
+#define VVCAM_ISP_CID_2DNR_STAT_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001D)
+#define VVCAM_ISP_CID_2DNR_STAT_BLEND_SLOPE  \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001E)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_OFFSET \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001F)
+#ifdef __KERNEL__
+int vvcam_isp_2dnr_ctrl_count(void);
+int vvcam_isp_2dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.c
new file mode 100644
index 000000000000..e7acb6e57f9f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.c
@@ -0,0 +1,1340 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_2dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_2dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_2DNR_ENABLE:
+        case VVCAM_ISP_CID_2DNR_LSC_COMP_ENABLE:
+        case VVCAM_ISP_CID_2DNR_MOTION_FAC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_LUMA_ENABLE:
+        case VVCAM_ISP_CID_2DNR_RESET:
+        case VVCAM_ISP_CID_2DNR_MODE:
+        case VVCAM_ISP_CID_2DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_2DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_2DNR_AUTO_VST_FAC:
+        case VVCAM_ISP_CID_2DNR_AUTO_BLEND_STATIC:
+        case VVCAM_ISP_CID_2DNR_AUTO_BLEND_MOTION:
+        case VVCAM_ISP_CID_2DNR_AUTO_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_OFFSET:
+        case VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_AUTO_LSC_COMP_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOTION_FAC_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOTION_ANCHOR_X:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_SCALE:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_FAC:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MUL:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MOTION_MAX:
+        case VVCAM_ISP_CID_2DNR_MANU_BLEND_STATIC:
+        case VVCAM_ISP_CID_2DNR_MANU_BLEND_MOTION:
+        case VVCAM_ISP_CID_2DNR_MANU_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_MANU_VST_FAC:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_SCALE:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MUL:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MAX:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MIN:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_OFFSET:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_FAC:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_ANCHOR_X:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_INT_MODE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_2dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_2DNR_ENABLE:
+        case VVCAM_ISP_CID_2DNR_LSC_COMP_ENABLE:
+        case VVCAM_ISP_CID_2DNR_MOTION_FAC_ENABLE:
+        case VVCAM_ISP_CID_2DNR_LUMA_ENABLE:
+        case VVCAM_ISP_CID_2DNR_RESET:
+        case VVCAM_ISP_CID_2DNR_MODE:
+        case VVCAM_ISP_CID_2DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_2DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_2DNR_AUTO_VST_FAC:
+        case VVCAM_ISP_CID_2DNR_AUTO_BLEND_STATIC:
+        case VVCAM_ISP_CID_2DNR_AUTO_BLEND_MOTION:
+        case VVCAM_ISP_CID_2DNR_AUTO_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_OFFSET:
+        case VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_AUTO_LSC_COMP_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOTION_FAC_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOTION_ANCHOR_X:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_SCALE:
+        case VVCAM_ISP_CID_2DNR_AUTO_STATIC_FAC:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MUL:
+        case VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MOTION_MAX:
+        case VVCAM_ISP_CID_2DNR_MANU_BLEND_STATIC:
+        case VVCAM_ISP_CID_2DNR_MANU_BLEND_MOTION:
+        case VVCAM_ISP_CID_2DNR_MANU_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_MANU_VST_FAC:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_SCALE:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MUL:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MAX:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MIN:
+        case VVCAM_ISP_CID_2DNR_MANU_SIGMA_OFFSET:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_MANU_STATIC_FAC:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_ANCHOR_X:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_STAT_BLEND_STATIC:
+        case VVCAM_ISP_CID_2DNR_STAT_BLEND_MOTION:
+        case VVCAM_ISP_CID_2DNR_STAT_BLEND_SLOPE:
+        case VVCAM_ISP_CID_2DNR_STAT_VST_FAC:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_SCALE:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MUL:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MOTION_MAX:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MOTION_MIN:
+        case VVCAM_ISP_CID_2DNR_STAT_SIGMA_OFFSET:
+        case VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_THR:
+        case VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_BOOST_THR:
+        case VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_BOOST:
+        case VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_CLIP_THR:
+        case VVCAM_ISP_CID_2DNR_STAT_STATIC_FAC:
+        case VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_INT_MODE:
+        case VVCAM_ISP_CID_2DNR_STAT_MOTION_ANCHOR_X:
+        case VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_X:
+        case VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_Y:
+        case VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_PX:
+        case VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_INT_MODE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_2dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_2dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_2dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_2dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_LSC_COMP_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_lsc_comp_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MOTION_FAC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_mot_fac_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_LUMA_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_luma_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit 1.0~1000.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_VST_FAC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_vst_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit 0~100.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit 0~100.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20*32bit 0~32.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_SIGMA_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_sigma_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 12, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_LSC_COMP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_lsc_comp_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 12, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_MOTION_FAC_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_mot_fac_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 12, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_MOTION_ANCHOR_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_mot_anchor_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_stc_dtl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* uint16_t array 20*2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_stc_dtl_boost_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* float array 20*2*3*16bit 1.0~4.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_stc_dtl_boost",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* uint16_t array 20*2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_CLIP_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_stc_dtl_clip_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* uint16_t array 20*2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_mov_dtl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* uint16_t array 20*2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_mov_dtl_boost_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* float array 20*2*3*32bit 1.0~4.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_mov_dtl_boost",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* uint16_t array 20*2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_CLIP_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_mov_dtl_clip_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 3, 0},
+    },
+    {
+        /* float array 20*3*32bit 0.001~100.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_SIGMA_SCALE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_sigma_scale",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 3, 0, 0},
+    },
+    {
+        /* float array 20*3*32bit 0.01~1.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_STATIC_FAC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_stc_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 3, 0, 0},
+    },
+    {
+        /* float array 20*3*32bit 0.1~10.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MUL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_sigma_fac_mul",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MOTION_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_auto_sigma_fac_mot_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 1024,
+        .def  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~32.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        /* float 1.0~1000.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_VST_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_vst_fac",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10000,
+    },
+    {
+        /* float array 3*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA_SCALE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma_scale",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float array 3*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MUL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma_fac_mul",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma_fac_mot_max",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma_fac_mot_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_SIGMA_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_sigma_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_stc_dtl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_stc_dtl_boost_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* float array 2*3*32bit 1.0~4.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_stc_dtl_boost",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_CLIP_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_stc_dtl_clip_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mov_dtl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mov_dtl_boost_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* float array 2*3*32bit 1.0~4.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mov_dtl_boost",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_CLIP_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mov_dtl_clip_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* float array 3*32bit 0.01~1.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_STATIC_FAC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_stc_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_PX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_luma_curve_px",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_INT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_luma_curve_im",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_lsc_comp_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_lsc_comp_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_PX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_lsc_comp_curve_px",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_INT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_lsc_comp_curve_im",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t array 2*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOTION_ANCHOR_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mot_anchor_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mot_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mot_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_PX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mot_curve_px",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_INT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_manu_mot_curve_im",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~100.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0~32.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        /* float 1.0~1000.0 */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_VST_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_vst_fac",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10000,
+    },
+    {
+        /* float array 3*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_SCALE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_scale",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float array 3*32bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MUL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_fac_mul",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MOTION_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_fac_mot_max",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MOTION_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_fac_mot_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_SIGMA_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_sigma_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_stc_dtl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_BOOST_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_stc_dtl_boost_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* float array 2*3*32bit 1.0~4.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_BOOST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_stc_dtl_boost",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_CLIP_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_stc_dtl_clip_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mov_dtl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_BOOST_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mov_dtl_boost_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* float array 2*3*32bit 1.0~4.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_BOOST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mov_dtl_boost",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 2*3*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_CLIP_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mov_dtl_clip_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        /* float array 3*32bit 0.01~1.0*/
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_STATIC_FAC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_stc_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_PX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_luma_curve_px",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_INT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_luma_curve_im",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_lsc_comp_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_lsc_comp_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_PX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_lsc_comp_curve_px",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_INT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_lsc_comp_curve_im",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t array 2*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOTION_ANCHOR_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mot_anchor_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mot_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mot_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 12*16bit */
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_PX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mot_curve_px",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_2dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_INT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_2dnr_stat_mot_curve_im",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+};
+
+int vvcam_isp_2dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_2dnr_ctrls);
+}
+
+int vvcam_isp_2dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_2dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_2dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 2dnr ctrl %s failed %d.\n",
+                vvcam_isp_2dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.h
new file mode 100644
index 000000000000..8bc6d11a7f07
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/2dnr/2dnr_v6/vvcam_isp_2dnr.h
@@ -0,0 +1,233 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_2DNR_H__
+#define __VVCAM_ISP_2DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_2DNR_ENABLE           (VVCAM_ISP_CID_2DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_2DNR_LSC_COMP_ENABLE  (VVCAM_ISP_CID_2DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_2DNR_MOTION_FAC_ENABLE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_2DNR_LUMA_ENABLE      (VVCAM_ISP_CID_2DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_2DNR_RESET            (VVCAM_ISP_CID_2DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_2DNR_MODE             (VVCAM_ISP_CID_2DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_2DNR_AUTO_LEVEL       (VVCAM_ISP_CID_2DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_2DNR_AUTO_GAIN        (VVCAM_ISP_CID_2DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_2DNR_AUTO_VST_FAC     (VVCAM_ISP_CID_2DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_2DNR_AUTO_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_2DNR_AUTO_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_2DNR_AUTO_BLEND_SLOPE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_2DNR_AUTO_SIGMA_OFFSET \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_2DNR_AUTO_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_2DNR_AUTO_LSC_COMP_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_2DNR_AUTO_MOTION_FAC_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_2DNR_AUTO_MOTION_ANCHOR_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_BOOST \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_2DNR_AUTO_STATIC_DETAIL_CLIP_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0014)
+#define VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0015)
+#define VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0016)
+#define VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_BOOST \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0017)
+#define VVCAM_ISP_CID_2DNR_AUTO_MOVING_DETAIL_CLIP_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0018)
+#define VVCAM_ISP_CID_2DNR_AUTO_SIGMA_SCALE (VVCAM_ISP_CID_2DNR_BASE + 0x0019)
+#define VVCAM_ISP_CID_2DNR_AUTO_STATIC_FAC  (VVCAM_ISP_CID_2DNR_BASE + 0x001A)
+#define VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MUL \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001B)
+#define VVCAM_ISP_CID_2DNR_AUTO_SIGMA_FAC_MOTION_MAX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001C)
+#define VVCAM_ISP_CID_2DNR_MANU_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001D)
+#define VVCAM_ISP_CID_2DNR_MANU_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001E)
+#define VVCAM_ISP_CID_2DNR_MANU_BLEND_SLOPE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x001F)
+#define VVCAM_ISP_CID_2DNR_MANU_VST_FAC     (VVCAM_ISP_CID_2DNR_BASE + 0x0020)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA_SCALE (VVCAM_ISP_CID_2DNR_BASE + 0x0021)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MUL \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0022)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MAX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0023)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA_FAC_MOTION_MIN \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0024)
+#define VVCAM_ISP_CID_2DNR_MANU_SIGMA_OFFSET \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0025)
+#define VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0026)
+#define VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0027)
+#define VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_BOOST \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0028)
+#define VVCAM_ISP_CID_2DNR_MANU_STATIC_DETAIL_CLIP_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0029)
+#define VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x002A)
+#define VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x002B)
+#define VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_BOOST \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x002C)
+#define VVCAM_ISP_CID_2DNR_MANU_MOVING_DETAIL_CLIP_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x002D)
+#define VVCAM_ISP_CID_2DNR_MANU_STATIC_FAC  (VVCAM_ISP_CID_2DNR_BASE + 0x002E)
+#define VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x002F)
+#define VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0030)
+#define VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_PX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0031)
+#define VVCAM_ISP_CID_2DNR_MANU_LUMA_CURVE_INT_MODE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0032)
+#define VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0033)
+#define VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0034)
+#define VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_PX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0035)
+#define VVCAM_ISP_CID_2DNR_MANU_LSC_COMP_CURVE_INT_MODE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0036)
+#define VVCAM_ISP_CID_2DNR_MANU_MOTION_ANCHOR_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0037)
+#define VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0038)
+#define VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0039)
+#define VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_PX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x003A)
+#define VVCAM_ISP_CID_2DNR_MANU_MOTION_CURVE_INT_MODE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x003B)
+#define VVCAM_ISP_CID_2DNR_STAT_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x003C)
+#define VVCAM_ISP_CID_2DNR_STAT_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x003D)
+#define VVCAM_ISP_CID_2DNR_STAT_BLEND_SLOPE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x003E)
+#define VVCAM_ISP_CID_2DNR_STAT_VST_FAC     (VVCAM_ISP_CID_2DNR_BASE + 0x003F)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_SCALE (VVCAM_ISP_CID_2DNR_BASE + 0x0040)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MUL \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0041)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MOTION_MAX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0042)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_FAC_MOTION_MIN \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0043)
+#define VVCAM_ISP_CID_2DNR_STAT_SIGMA_OFFSET \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0044)
+#define VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0045)
+#define VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_BOOST_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0046)
+#define VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_BOOST \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0047)
+#define VVCAM_ISP_CID_2DNR_STAT_STATIC_DETAIL_CLIP_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0048)
+#define VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0049)
+#define VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_BOOST_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x004A)
+#define VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_BOOST \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x004B)
+#define VVCAM_ISP_CID_2DNR_STAT_MOVING_DETAIL_CLIP_THR \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x004C)
+#define VVCAM_ISP_CID_2DNR_STAT_STATIC_FAC  (VVCAM_ISP_CID_2DNR_BASE + 0x004D)
+#define VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x004E)
+#define VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x004F)
+#define VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_PX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0050)
+#define VVCAM_ISP_CID_2DNR_STAT_LUMA_CURVE_INT_MODE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0051)
+#define VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0052)
+#define VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0053)
+#define VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_PX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0054)
+#define VVCAM_ISP_CID_2DNR_STAT_LSC_COMP_CURVE_INT_MODE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0055)
+#define VVCAM_ISP_CID_2DNR_STAT_MOTION_ANCHOR_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0056)
+#define VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_X \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0057)
+#define VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_Y \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0058)
+#define VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_PX \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x0059)
+#define VVCAM_ISP_CID_2DNR_STAT_MOTION_CURVE_INT_MODE \
+                                            (VVCAM_ISP_CID_2DNR_BASE + 0x005A)
+
+#ifdef __KERNEL__
+int vvcam_isp_2dnr_ctrl_count(void);
+int vvcam_isp_2dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.c
new file mode 100644
index 000000000000..05431c34cb5e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.c
@@ -0,0 +1,308 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_3dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_3dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_AUTO_DELTA:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_MANU_DELTA:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_3dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_AUTO_DELTA:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_MANU_DELTA:
+        case VVCAM_ISP_CID_3DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_STAT_DELTA:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_3dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_3dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_3dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_3dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x  */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* int32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_3dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_3dnr_ctrls);
+}
+
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_3dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_3dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 3dnr ctrl %s failed %d.\n",
+                vvcam_isp_3dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.h
new file mode 100644
index 000000000000..72523aef59f7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v1_2/vvcam_isp_3dnr.h
@@ -0,0 +1,81 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_3DNR_H__
+#define __VVCAM_ISP_3DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_3DNR_ENABLE           (VVCAM_ISP_CID_3DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_3DNR_RESET            (VVCAM_ISP_CID_3DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_3DNR_MODE             (VVCAM_ISP_CID_3DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_3DNR_AUTO_LEVEL       (VVCAM_ISP_CID_3DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAIN        (VVCAM_ISP_CID_3DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_3DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_3DNR_AUTO_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_3DNR_MANU_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_3DNR_MANU_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_3DNR_STAT_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_3DNR_STAT_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x000D)
+
+#ifdef __KERNEL__
+int vvcam_isp_3dnr_ctrl_count(void);
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.c
new file mode 100644
index 000000000000..b6d2b9108c32
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.c
@@ -0,0 +1,584 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_3dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_3dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_AUTO_DELTA:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_UPDATE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_STRENGTH_CURVE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_CURVE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_RANGE_T:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_MANU_DELTA:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_UPDATE:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_STRENGTH_CURVE:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_CURVE:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_RANGE_T:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_NOISE_EN:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_INPUT_BIT:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_BLS_EXP:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_3dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_AUTO_DELTA:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_UPDATE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_STRENGTH_CURVE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_CURVE:
+        case VVCAM_ISP_CID_3DNR_AUTO_TEMP_RANGE_T:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_MANU_DELTA:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_UPDATE:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_STRENGTH_CURVE:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_CURVE:
+        case VVCAM_ISP_CID_3DNR_MANU_TEMP_RANGE_T:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_NOISE_EN:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_INPUT_BIT:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FEBE_BLS_EXP:
+        case VVCAM_ISP_CID_3DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_STAT_DELTA:
+        case VVCAM_ISP_CID_3DNR_STAT_TEMP_UPDATE:
+        case VVCAM_ISP_CID_3DNR_STAT_TEMP_STRENGTH_CURVE:
+        case VVCAM_ISP_CID_3DNR_STAT_TEMP_CURVE:
+        case VVCAM_ISP_CID_3DNR_STAT_TEMP_RANGE_T:
+        case VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_NOISE_EN:
+        case VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_INPUT_BIT:
+        case VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_BLS_EXP:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_3dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_3dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_3dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_3dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x  */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* int32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TEMP_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_temp_mot_dil_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TEMP_UPDATE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_temp_update",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TEMP_STRENGTH_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_temp_str_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x17 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TEMP_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_temp_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 17, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TEMP_RANGE_T,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_temp_range_t",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TEMP_UPDATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_temp_update",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TEMP_STRENGTH_CURVE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_temp_str_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+    },
+    {
+        /* uint16_t array 17x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TEMP_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_temp_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TEMP_RANGE_T,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_temp_range_t",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAMMA_FEBE_NOISE_EN,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gamma_febe_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 20 or 24 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAMMA_FEBE_INPUT_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gamma_febe_in_bit",
+        .step = 1,
+        .def  = 20,
+        .min  = 20,
+        .max  = 24,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAMMA_FEBE_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gamma_febe_fcs",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAMMA_FEBE_BLS_EXP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gamma_febe_bls",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TEMP_UPDATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_temp_update",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TEMP_STRENGTH_CURVE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_temp_str_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+    },
+    {
+        /* uint16_t array 17x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TEMP_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_temp_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TEMP_RANGE_T,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_temp_range_t",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_NOISE_EN,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_gamma_febe_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 20 or 24 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_INPUT_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_gamma_febe_in_bit",
+        .step = 1,
+        .def  = 20,
+        .min  = 20,
+        .max  = 24,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_gamma_febe_fcs",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_BLS_EXP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_gamma_febe_bls",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {4, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_3dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_3dnr_ctrls);
+}
+
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_3dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_3dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 3dnr ctrl %s failed %d.\n",
+                vvcam_isp_3dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.h
new file mode 100644
index 000000000000..e87d097fb467
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_1/vvcam_isp_3dnr.h
@@ -0,0 +1,119 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_3DNR_H__
+#define __VVCAM_ISP_3DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_3DNR_ENABLE           (VVCAM_ISP_CID_3DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_3DNR_RESET            (VVCAM_ISP_CID_3DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_3DNR_MODE             (VVCAM_ISP_CID_3DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_3DNR_AUTO_LEVEL       (VVCAM_ISP_CID_3DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAIN        (VVCAM_ISP_CID_3DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_3DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_3DNR_AUTO_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_3DNR_AUTO_TEMP_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_3DNR_AUTO_TEMP_UPDATE (VVCAM_ISP_CID_3DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_3DNR_AUTO_TEMP_STRENGTH_CURVE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_3DNR_AUTO_TEMP_CURVE  (VVCAM_ISP_CID_3DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_3DNR_AUTO_TEMP_RANGE_T \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_3DNR_MANU_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_3DNR_MANU_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_3DNR_MANU_TEMP_UPDATE (VVCAM_ISP_CID_3DNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_3DNR_MANU_TEMP_STRENGTH_CURVE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_3DNR_MANU_TEMP_CURVE  (VVCAM_ISP_CID_3DNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_3DNR_MANU_TEMP_RANGE_T \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0014)
+#define VVCAM_ISP_CID_3DNR_GAMMA_FEBE_NOISE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0015)
+#define VVCAM_ISP_CID_3DNR_GAMMA_FEBE_INPUT_BIT \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0016)                                            
+#define VVCAM_ISP_CID_3DNR_GAMMA_FEBE_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0017)
+#define VVCAM_ISP_CID_3DNR_GAMMA_FEBE_BLS_EXP \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0018)
+#define VVCAM_ISP_CID_3DNR_STAT_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0019)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x001A)
+#define VVCAM_ISP_CID_3DNR_STAT_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x001B)
+#define VVCAM_ISP_CID_3DNR_STAT_TEMP_UPDATE (VVCAM_ISP_CID_3DNR_BASE + 0x001C)
+#define VVCAM_ISP_CID_3DNR_STAT_TEMP_STRENGTH_CURVE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x001D)
+#define VVCAM_ISP_CID_3DNR_STAT_TEMP_CURVE  (VVCAM_ISP_CID_3DNR_BASE + 0x001E)
+#define VVCAM_ISP_CID_3DNR_STAT_TEMP_RANGE_T \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x001F)
+#define VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_NOISE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0020)
+#define VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_INPUT_BIT \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0021)                                            
+#define VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0022)
+#define VVCAM_ISP_CID_3DNR_STAT_GAMMA_FEBE_BLS_EXP \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0023)
+
+#ifdef __KERNEL__
+int vvcam_isp_3dnr_ctrl_count(void);
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.c
new file mode 100644
index 000000000000..efca62af9e62
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.c
@@ -0,0 +1,322 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_3dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_3dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_AUTO_DELTA:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_MANU_DELTA:
+        case VVCAM_ISP_CID_3DNR_MOTION_ROI:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_3dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_AUTO_DELTA:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_MANU_DELTA:
+        case VVCAM_ISP_CID_3DNR_MOTION_ROI:
+        case VVCAM_ISP_CID_3DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_INV:
+        case VVCAM_ISP_CID_3DNR_STAT_DELTA:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_3dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_3dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_3dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_3dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x  */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* int32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* 64x array 64x4x32bit */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_motion_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {64, 4, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_INV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_motion_inv",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DELTA,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_delta",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_3dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_3dnr_ctrls);
+}
+
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_3dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_3dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 3dnr ctrl %s failed %d.\n",
+                vvcam_isp_3dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.h
new file mode 100644
index 000000000000..444747e8f477
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v2_4/vvcam_isp_3dnr.h
@@ -0,0 +1,82 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_3DNR_H__
+#define __VVCAM_ISP_3DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_3DNR_ENABLE           (VVCAM_ISP_CID_3DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_3DNR_RESET            (VVCAM_ISP_CID_3DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_3DNR_MODE             (VVCAM_ISP_CID_3DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_3DNR_AUTO_LEVEL       (VVCAM_ISP_CID_3DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAIN        (VVCAM_ISP_CID_3DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_3DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_3DNR_AUTO_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_3DNR_MANU_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_3DNR_MANU_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_3DNR_MOTION_ROI       (VVCAM_ISP_CID_3DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_3DNR_STAT_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_INV  (VVCAM_ISP_CID_3DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_3DNR_STAT_DELTA       (VVCAM_ISP_CID_3DNR_BASE + 0x000E)
+
+#ifdef __KERNEL__
+int vvcam_isp_3dnr_ctrl_count(void);
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.c
new file mode 100644
index 000000000000..7a2391c8aa0a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.c
@@ -0,0 +1,929 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_3dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_3dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_PRE_GAMMA_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_WORK_MODE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_PRE_GAMMA_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_AUTO_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_3dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_PRE_GAMMA_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_WORK_MODE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_PRE_GAMMA_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_AUTO_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_STAT_WORK_MODE:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_ERODE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_STAT_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_STAT_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_STAT_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_STAT_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_STAT_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_STAT_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_STAT_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_STAT_SIGMA:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_3dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_3dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_3dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_3dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_PRE_GAMMA_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_pre_gamma_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mot_erode_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_WORK_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_work_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 1024.0*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_filter_len",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 1024.0*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_filter_len2",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_noise_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_slope_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_erode_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_PRE_GAMMA_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_pre_gamma_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_RANGE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_RANGE_V,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_range_v",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_DILATE_H,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_pre_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_SIGMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 3~7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DILATE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_filter_len",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_filter_len2",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 1 ~ 1024 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_level",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10240,
+    },
+    {
+        /* float 0 ~ 32 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_pre_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+    },
+    {
+        /* float 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_RANGE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+    },
+    {
+        /* float 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_RANGE_V,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_range_v",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+    },
+    {
+        /* float 0 ~ 16 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 1 ~ 1023 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_mot_slope_thr",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1023,
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0 ~ 32.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_sigma",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* float 0 ~ 1000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0 ~ 1000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_ERODE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_erode_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_WORK_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_work_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float 3~7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DILATE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_filter_len",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_filter_len2",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 1 ~ 1024 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_level",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10240,
+    },
+    {
+        /* float 0 ~ 32 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_PRE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_pre_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+    },
+    {
+        /* float 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_RANGE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+    },
+    {
+        /* float 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_RANGE_V,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_range_v",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+    },
+    {
+        /* float 0 ~ 16 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 1 ~ 1023 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_slope_thr",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1023,
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0 ~ 32.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_sigma",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* float 0 ~ 1000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0 ~ 1000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+};
+
+int vvcam_isp_3dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_3dnr_ctrls);
+}
+
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_3dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_3dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 3dnr ctrl %s failed %d.\n",
+                vvcam_isp_3dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.h
new file mode 100644
index 000000000000..b0cf35c06299
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3/vvcam_isp_3dnr.h
@@ -0,0 +1,148 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_3DNR_H__
+#define __VVCAM_ISP_3DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_3DNR_ENABLE           (VVCAM_ISP_CID_3DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_3DNR_PRE_GAMMA_ENABLE (VVCAM_ISP_CID_3DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_3DNR_WORK_MODE        (VVCAM_ISP_CID_3DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_3DNR_RESET            (VVCAM_ISP_CID_3DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_3DNR_MODE             (VVCAM_ISP_CID_3DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_3DNR_AUTO_LEVEL       (VVCAM_ISP_CID_3DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAIN        (VVCAM_ISP_CID_3DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_3DNR_AUTO_PRE_GAMMA_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_3DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_3DNR_AUTO_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_3DNR_AUTO_RANGE_V     (VVCAM_ISP_CID_3DNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_3DNR_AUTO_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x0014)
+#define VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0015)
+#define VVCAM_ISP_CID_3DNR_AUTO_SIGMA       (VVCAM_ISP_CID_3DNR_BASE + 0x0016)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0017)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0018)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE (VVCAM_ISP_CID_3DNR_BASE + 0x0019)
+#define VVCAM_ISP_CID_3DNR_MANU_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x001A)
+#define VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x001B)
+#define VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x001C)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x001D)
+#define VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x001E)
+#define VVCAM_ISP_CID_3DNR_MANU_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x001F)
+#define VVCAM_ISP_CID_3DNR_MANU_RANGE_V     (VVCAM_ISP_CID_3DNR_BASE + 0x0020)
+#define VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0021)
+#define VVCAM_ISP_CID_3DNR_MANU_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0022)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0023)
+#define VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0024)
+#define VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE (VVCAM_ISP_CID_3DNR_BASE + 0x0025)
+#define VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0026)
+#define VVCAM_ISP_CID_3DNR_MANU_SIGMA       (VVCAM_ISP_CID_3DNR_BASE + 0x0027)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0028)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0029)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002A)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_ERODE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002B)
+#define VVCAM_ISP_CID_3DNR_STAT_WORK_MODE   (VVCAM_ISP_CID_3DNR_BASE + 0x002C)
+#define VVCAM_ISP_CID_3DNR_STAT_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x002D)
+#define VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x002E)
+#define VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x002F)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x0030)
+#define VVCAM_ISP_CID_3DNR_STAT_PRE_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0031)
+#define VVCAM_ISP_CID_3DNR_STAT_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0032)
+#define VVCAM_ISP_CID_3DNR_STAT_RANGE_V     (VVCAM_ISP_CID_3DNR_BASE + 0x0033)
+#define VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0034)
+#define VVCAM_ISP_CID_3DNR_STAT_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0035)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0036)
+#define VVCAM_ISP_CID_3DNR_STAT_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0037)
+#define VVCAM_ISP_CID_3DNR_STAT_BLEND_SLOPE (VVCAM_ISP_CID_3DNR_BASE + 0x0038)
+#define VVCAM_ISP_CID_3DNR_STAT_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0039)
+#define VVCAM_ISP_CID_3DNR_STAT_SIGMA       (VVCAM_ISP_CID_3DNR_BASE + 0x003A)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003B)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003C)
+
+#ifdef __KERNEL__
+int vvcam_isp_3dnr_ctrl_count(void);
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.c
new file mode 100644
index 000000000000..316dfe1ba60f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.c
@@ -0,0 +1,973 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_3dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_3dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_WORK_MODE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAMMA_FE_BE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_AUTO_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_3dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_WORK_MODE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAMMA_FE_BE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_AUTO_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_ERODE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_STAT_WORK_MODE:
+        case VVCAM_ISP_CID_3DNR_STAT_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_STAT_PRE_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_STAT_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_STAT_RANGE_V:
+        case VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_STAT_BLEND_MOTION:
+        case VVCAM_ISP_CID_3DNR_STAT_BLEND_SLOPE:
+        case VVCAM_ISP_CID_3DNR_STAT_BLEND_STATIC:
+        case VVCAM_ISP_CID_3DNR_STAT_SIGMA:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_FIX_CURVE_START:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_3dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_3dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_3dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_3dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gamma_febe_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mot_erode_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_WORK_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_work_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 1024.0*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_filter_len",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x  0.0 ~ 1024.0*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_filter_len2",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x  1 ~ 1024*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_noise_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20x  1 ~ 1023*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_slope_thr",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_erode_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAMMA_FE_BE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gamma_febe_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_RANGE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_RANGE_V,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_range_v",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_DILATE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_pre_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x  0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_SIGMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x  0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 32.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3~7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DILATE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_filter_len",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_filter_len2",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 1 ~ 1024 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_level",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10240,
+    },
+    {
+        /* uint8_t 0 ~ 32 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_pre_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_RANGE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_RANGE_V,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_range_v",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 16 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t 1 ~ 1023 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_mot_slope_thr",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0 ~ 32.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_sigma",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_a",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 10000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_b",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 100000,
+    },
+    {
+        /* uint16_t  0 ~ 4096 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_fcs",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_ERODE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_erode_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_WORK_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_work_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* uint8_t 3~7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DILATE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_filter_len",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 0.0 ~ 1024.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_filter_len2",
+        .step = 1,
+        .min  = 0,
+        .max  = 10240,
+    },
+    {
+        /* float 1 ~ 1024 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_level",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10240,
+    },
+    {
+        /* uint8_t 0 ~ 32 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_PRE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_pre_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_RANGE_H,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3 ~ 7 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_RANGE_V,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_range_v",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 16 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t 1 ~ 1023 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_slope_thr",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_BLEND_MOTION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_blend_mot",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0 ~ 32.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_BLEND_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_blend_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 320,
+    },
+    {
+        /* float 0.0 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_BLEND_STATIC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_blend_stc",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_sigma",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_a",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* float 0.1 ~ 10000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_b",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 100000,
+    },
+    {
+        /* uint16_t  0 ~ 4096 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_fcs",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_3dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_3dnr_ctrls);
+}
+
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_3dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_3dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 3dnr ctrl %s failed %d.\n",
+                vvcam_isp_3dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.h
new file mode 100644
index 000000000000..fe1fab2c0c25
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v3_1/vvcam_isp_3dnr.h
@@ -0,0 +1,153 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_3DNR_H__
+#define __VVCAM_ISP_3DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_3DNR_ENABLE           (VVCAM_ISP_CID_3DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_3DNR_MOTION_ERODE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_3DNR_WORK_MODE        (VVCAM_ISP_CID_3DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_3DNR_RESET            (VVCAM_ISP_CID_3DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_3DNR_MODE             (VVCAM_ISP_CID_3DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_3DNR_AUTO_LEVEL       (VVCAM_ISP_CID_3DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAIN        (VVCAM_ISP_CID_3DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_3DNR_AUTO_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_ERODE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAMMA_FE_BE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_3DNR_AUTO_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_3DNR_AUTO_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_3DNR_AUTO_RANGE_V     (VVCAM_ISP_CID_3DNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_3DNR_AUTO_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x0014)
+#define VVCAM_ISP_CID_3DNR_AUTO_PRE_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0015)
+#define VVCAM_ISP_CID_3DNR_AUTO_SIGMA       (VVCAM_ISP_CID_3DNR_BASE + 0x0016)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0017)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0018)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLEND_SLOPE (VVCAM_ISP_CID_3DNR_BASE + 0x0019)
+#define VVCAM_ISP_CID_3DNR_MANU_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x001A)
+#define VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x001B)
+#define VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x001C)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x001D)
+#define VVCAM_ISP_CID_3DNR_MANU_PRE_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x001E)
+#define VVCAM_ISP_CID_3DNR_MANU_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x001F)
+#define VVCAM_ISP_CID_3DNR_MANU_RANGE_V     (VVCAM_ISP_CID_3DNR_BASE + 0x0020)
+#define VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0021)
+#define VVCAM_ISP_CID_3DNR_MANU_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0022)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0023)
+#define VVCAM_ISP_CID_3DNR_MANU_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0024)
+#define VVCAM_ISP_CID_3DNR_MANU_BLEND_SLOPE (VVCAM_ISP_CID_3DNR_BASE + 0x0025)
+#define VVCAM_ISP_CID_3DNR_MANU_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0026)
+#define VVCAM_ISP_CID_3DNR_MANU_SIGMA       (VVCAM_ISP_CID_3DNR_BASE + 0x0027)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0028)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0029)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002A)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002B)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_ERODE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002C)
+#define VVCAM_ISP_CID_3DNR_STAT_WORK_MODE   (VVCAM_ISP_CID_3DNR_BASE + 0x002D)
+#define VVCAM_ISP_CID_3DNR_STAT_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x002E)
+#define VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x002F)
+#define VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x0030)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x0031)
+#define VVCAM_ISP_CID_3DNR_STAT_PRE_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0032)
+#define VVCAM_ISP_CID_3DNR_STAT_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0033)
+#define VVCAM_ISP_CID_3DNR_STAT_RANGE_V     (VVCAM_ISP_CID_3DNR_BASE + 0x0034)
+#define VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0035)
+#define VVCAM_ISP_CID_3DNR_STAT_STRENGTH    (VVCAM_ISP_CID_3DNR_BASE + 0x0036)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0037)
+#define VVCAM_ISP_CID_3DNR_STAT_BLEND_MOTION \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0038)
+#define VVCAM_ISP_CID_3DNR_STAT_BLEND_SLOPE (VVCAM_ISP_CID_3DNR_BASE + 0x0039)
+#define VVCAM_ISP_CID_3DNR_STAT_BLEND_STATIC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003A)
+#define VVCAM_ISP_CID_3DNR_STAT_SIGMA       (VVCAM_ISP_CID_3DNR_BASE + 0x003B)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003C)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003D)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003E)
+
+#ifdef __KERNEL__
+int vvcam_isp_3dnr_ctrl_count(void);
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.c
new file mode 100644
index 000000000000..68df9346f346
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.c
@@ -0,0 +1,1089 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_3dnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_3dnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_GAUSS_BLUR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_NMK:
+        case VVCAM_ISP_CID_3DNR_AUTO_NMP:
+        case VVCAM_ISP_CID_3DNR_AUTO_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLS_EXP:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH2:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_FAC:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_SQR_DIFF_FAC:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_MOTION_SLOPE_Y:
+        case VVCAM_ISP_CID_3DNR_MANU_VST_FAC:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH2:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_FAC:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_DIFF_TYPE:
+        case VVCAM_ISP_CID_3DNR_MANU_SQR_DIFF_FAC:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_MOTION_SLOPE_Y:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_INPUT_BIT:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_BLS_EXP:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_3dnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_3DNR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE:
+        case VVCAM_ISP_CID_3DNR_GAUSS_BLUR_ENABLE:
+        case VVCAM_ISP_CID_3DNR_RESET:
+        case VVCAM_ISP_CID_3DNR_MODE:
+        case VVCAM_ISP_CID_3DNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_3DNR_AUTO_NMK:
+        case VVCAM_ISP_CID_3DNR_AUTO_NMP:
+        case VVCAM_ISP_CID_3DNR_AUTO_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_AUTO_BLS_EXP:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH2:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_FAC:
+        case VVCAM_ISP_CID_3DNR_AUTO_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_AUTO_SQR_DIFF_FAC:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_LEVEL:
+        case VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN:
+        case VVCAM_ISP_CID_3DNR_AUTO_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_3DNR_AUTO_TNR_MOTION_SLOPE_Y:
+        case VVCAM_ISP_CID_3DNR_MANU_VST_FAC:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH2:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_FAC:
+        case VVCAM_ISP_CID_3DNR_MANU_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_MANU_DIFF_TYPE:
+        case VVCAM_ISP_CID_3DNR_MANU_SQR_DIFF_FAC:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_3DNR_MANU_TNR_MOTION_SLOPE_Y:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_INPUT_BIT:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_BLS_EXP:
+        case VVCAM_ISP_CID_3DNR_STAT_VST_FAC:
+        case VVCAM_ISP_CID_3DNR_STAT_TNR_STRENGTH:
+        case VVCAM_ISP_CID_3DNR_STAT_TNR_STRENGTH2:
+        case VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN:
+        case VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_SMOOTH_FAC:
+        case VVCAM_ISP_CID_3DNR_STAT_RANGE_H:
+        case VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT:
+        case VVCAM_ISP_CID_3DNR_STAT_DIFF_TYPE:
+        case VVCAM_ISP_CID_3DNR_STAT_SQR_DIFF_FAC:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_SMOOTH_LEVEL:
+        case VVCAM_ISP_CID_3DNR_STAT_DILATE_H:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL:
+        case VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR:
+        case VVCAM_ISP_CID_3DNR_STAT_TNR_LUMA_CURVE_X:
+        case VVCAM_ISP_CID_3DNR_STAT_TNR_LUMA_CURVE_Y:
+        case VVCAM_ISP_CID_3DNR_STAT_TNR_MOTION_SLOPE_Y:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_INPUT_BIT:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_FIX_CURVE_START:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B:
+        case VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_BLS_EXP:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_3dnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_3dnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_3dnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_3dnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gamma_febe_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_GAUSS_BLUR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_gauss_blur_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 100*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_NMK,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_nmk",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0 ~ 100*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_NMP,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_nmp",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* uint16_t 0 ~ 4096 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_fix_curve_start",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* double array 20x  0.1~100*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_noise_mdl_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* double array 20x  0.1~10000*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_noise_mdl_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x4  */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_BLS_EXP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_bls_exp",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_tnr_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_tnr_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x 1~99 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_filter_len",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 99,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x 1~99 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_filter_len2",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 99,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* double array 20x  0.1 ~ 5.0*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_FAC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_smooth_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x  3 or 6 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_RANGE_H,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 6,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_SQR_DIFF_FAC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_sqr_diff_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_smooth_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_mot_dilate_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20x  3 or 6*/
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_DILATE_H,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 6,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TNR_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_tnr_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 12, 0, 0},
+    },
+    {
+        /* uint16_t array 20x */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_AUTO_TNR_MOTION_SLOPE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_auto_tnr_mot_slope_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 12, 0, 0},
+    },
+    {
+        /* double 1.0 ~  1000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_VST_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_vst_fac",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10000,
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_tnr_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_tnr_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 1 ~ 99 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_filter_len",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 99,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 1 ~ 99 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_filter_len2",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 99,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* double 0.1 ~ 5.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_mot_smooth_fac",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 50,
+    },
+    {
+        /* int 3 or 6 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_RANGE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 6,
+    },
+    {
+        /* uint8_t 0 ~ 16 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DIFF_TYPE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_diff_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_SQR_DIFF_FAC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_sqr_diff_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0~1 */ 
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_mot_smooth_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* 3 or 6 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_DILATE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 6,
+    },
+    {
+        /* uint16_t 0 ~ 65535 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 4095 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_mot_slope_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 4095 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_tnr_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 65535 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_tnr_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 4095 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_TNR_MOTION_SLOPE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_tnr_mot_slope_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* 20 or 24 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_INPUT_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_in_bit",
+        .step = 1,
+        .def  = 20,
+        .min  = 20,
+        .max  = 24,
+    },
+    {
+        /* uint16_t 0 ~ 4096 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_fcs",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* double 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_a",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* double 0.1 ~ 10000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_b",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 100000,
+    },
+    {
+        /* uint32_t  0 ~ 1048575 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_BLS_EXP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_manu_noise_mdl_bls_exp",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* double 1.0 ~  1000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_VST_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_vst_fac",
+        .step = 1,
+        .def  = 10,
+        .min  = 10,
+        .max  = 10000,
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TNR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_tnr_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 128 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TNR_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_tnr_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 1 ~ 99 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_filter_len",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 99,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 1 ~ 99 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_filter_len2",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 99,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* double 0.1 ~ 5.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_SMOOTH_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_smooth_fac",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 50,
+    },
+    {
+        /* int 3 or 6 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_RANGE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_range_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 6,
+    },
+    {
+        /* uint8_t 0 ~ 16 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_sad_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DIFF_TYPE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_diff_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_SQR_DIFF_FAC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_sqr_diff_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0~1 */ 
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_SMOOTH_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_smooth_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* 3 or 6 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_DILATE_H,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_dilate_h",
+        .step = 1,
+        .def  = 3,
+        .min  = 3,
+        .max  = 6,
+    },
+    {
+        /* uint16_t 0 ~ 65535 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 4095 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_mot_slope_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 4095 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TNR_LUMA_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_tnr_luma_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 65535 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TNR_LUMA_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_tnr_luma_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* uint16_t 0 ~ 4095 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_TNR_MOTION_SLOPE_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_tnr_mot_slope_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        /* 20 or 24 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_INPUT_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_in_bit",
+        .step = 1,
+        .def  = 20,
+        .min  = 20,
+        .max  = 24,
+    },
+    {
+        /* uint16_t 0 ~ 4096 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_FIX_CURVE_START,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_fcs",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* double 0.1 ~ 100.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_a",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 1000,
+    },
+    {
+        /* double 0.1 ~ 10000.0 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_b",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 100000,
+    },
+    {
+        /* uint32_t  0 ~ 1048575 */
+        .ops  = &vvcam_isp_3dnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_BLS_EXP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_3dnr_stat_noise_mdl_bls_exp",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {4, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_3dnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_3dnr_ctrls);
+}
+
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_3dnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_3dnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp 3dnr ctrl %s failed %d.\n",
+                vvcam_isp_3dnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.h
new file mode 100644
index 000000000000..520ebcf9aecb
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/3dnr/3dnr_v4/vvcam_isp_3dnr.h
@@ -0,0 +1,181 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_3DNR_H__
+#define __VVCAM_ISP_3DNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_3DNR_ENABLE           (VVCAM_ISP_CID_3DNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_3DNR_GAMMA_FE_BE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_3DNR_MOTION_DILATE_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_3DNR_GAUSS_BLUR_ENABLE \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_3DNR_RESET            (VVCAM_ISP_CID_3DNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_3DNR_MODE             (VVCAM_ISP_CID_3DNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_3DNR_AUTO_LEVEL       (VVCAM_ISP_CID_3DNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_3DNR_AUTO_GAIN        (VVCAM_ISP_CID_3DNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_3DNR_AUTO_NMK         (VVCAM_ISP_CID_3DNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_3DNR_AUTO_NMP         (VVCAM_ISP_CID_3DNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_3DNR_AUTO_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_3DNR_AUTO_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_3DNR_AUTO_BLS_EXP     (VVCAM_ISP_CID_3DNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_3DNR_AUTO_TNR_STRENGTH2 \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_3DNR_AUTO_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_FAC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_3DNR_AUTO_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_3DNR_AUTO_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0014)
+#define VVCAM_ISP_CID_3DNR_AUTO_SQR_DIFF_FAC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0015)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_SMOOTH_LEVEL \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0016)
+#define VVCAM_ISP_CID_3DNR_AUTO_MOTION_DILATE_EN \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0017)
+#define VVCAM_ISP_CID_3DNR_AUTO_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x0018)
+#define VVCAM_ISP_CID_3DNR_AUTO_TNR_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0019)
+#define VVCAM_ISP_CID_3DNR_AUTO_TNR_MOTION_SLOPE_Y \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x001A)
+
+#define VVCAM_ISP_CID_3DNR_MANU_VST_FAC     (VVCAM_ISP_CID_3DNR_BASE + 0x001B)
+#define VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x001C)
+#define VVCAM_ISP_CID_3DNR_MANU_TNR_STRENGTH2 \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x001D)
+#define VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x001E)
+#define VVCAM_ISP_CID_3DNR_MANU_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x001F)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_FAC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0020)
+#define VVCAM_ISP_CID_3DNR_MANU_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0021)
+#define VVCAM_ISP_CID_3DNR_MANU_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0022)
+#define VVCAM_ISP_CID_3DNR_MANU_DIFF_TYPE   (VVCAM_ISP_CID_3DNR_BASE + 0x0023)
+#define VVCAM_ISP_CID_3DNR_MANU_SQR_DIFF_FAC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0024)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_SMOOTH_LEVEL \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0025)
+#define VVCAM_ISP_CID_3DNR_MANU_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x0026)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x0027)
+#define VVCAM_ISP_CID_3DNR_MANU_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0028)
+#define VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0029)
+#define VVCAM_ISP_CID_3DNR_MANU_TNR_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002A)
+#define VVCAM_ISP_CID_3DNR_MANU_TNR_MOTION_SLOPE_Y \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002B)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_INPUT_BIT \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002C)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002D)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002E)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x002F)
+#define VVCAM_ISP_CID_3DNR_MANU_NOISE_MODEL_BLS_EXP \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0030)
+#define VVCAM_ISP_CID_3DNR_STAT_VST_FAC     (VVCAM_ISP_CID_3DNR_BASE + 0x0031)
+#define VVCAM_ISP_CID_3DNR_STAT_TNR_STRENGTH \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0032)
+#define VVCAM_ISP_CID_3DNR_STAT_TNR_STRENGTH2 \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0033)
+#define VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN  (VVCAM_ISP_CID_3DNR_BASE + 0x0034)
+#define VVCAM_ISP_CID_3DNR_STAT_FILTER_LEN2 (VVCAM_ISP_CID_3DNR_BASE + 0x0035)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_SMOOTH_FAC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0036)
+#define VVCAM_ISP_CID_3DNR_STAT_RANGE_H     (VVCAM_ISP_CID_3DNR_BASE + 0x0037)
+#define VVCAM_ISP_CID_3DNR_STAT_SAD_WEIGHT  (VVCAM_ISP_CID_3DNR_BASE + 0x0038)
+#define VVCAM_ISP_CID_3DNR_STAT_DIFF_TYPE   (VVCAM_ISP_CID_3DNR_BASE + 0x0039)
+#define VVCAM_ISP_CID_3DNR_STAT_SQR_DIFF_FAC \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003A)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_SMOOTH_LEVEL \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003B)
+#define VVCAM_ISP_CID_3DNR_STAT_DILATE_H    (VVCAM_ISP_CID_3DNR_BASE + 0x003C)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_LEVEL (VVCAM_ISP_CID_3DNR_BASE + 0x003D)
+#define VVCAM_ISP_CID_3DNR_STAT_MOTION_SLOPE_THR \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003E)
+#define VVCAM_ISP_CID_3DNR_STAT_TNR_LUMA_CURVE_X \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x003F)
+#define VVCAM_ISP_CID_3DNR_STAT_TNR_LUMA_CURVE_Y \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0040)
+#define VVCAM_ISP_CID_3DNR_STAT_TNR_MOTION_SLOPE_Y \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0041)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_INPUT_BIT \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0042)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_FIX_CURVE_START \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0043)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_A \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0044)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_B \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0045)
+#define VVCAM_ISP_CID_3DNR_STAT_NOISE_MODEL_BLS_EXP \
+                                            (VVCAM_ISP_CID_3DNR_BASE + 0x0046)
+
+#ifdef __KERNEL__
+int vvcam_isp_3dnr_ctrl_count(void);
+int vvcam_isp_3dnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v3/vvcam_isp_ae.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v3/vvcam_isp_ae.c
new file mode 100644
index 000000000000..6ce55c915362
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v3/vvcam_isp_ae.c
@@ -0,0 +1,716 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ae.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ae_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AE_ENABLE:
+        case VVCAM_ISP_CID_AE_RESET:
+        case VVCAM_ISP_CID_AE_STATE:
+        case VVCAM_ISP_CID_AE_SEM_MODE:
+        case VVCAM_ISP_CID_AE_FLICKER_PERIOD:
+        case VVCAM_ISP_CID_AE_SETPOINT:
+        case VVCAM_ISP_CID_AE_TOLORENCE:
+        case VVCAM_ISP_CID_AE_DAMPOVER:
+        case VVCAM_ISP_CID_AE_DAMPOVER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPOVER_GAIN:
+        case VVCAM_ISP_CID_AE_DAMPUNDER:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_GAIN:
+        case VVCAM_ISP_CID_AE_MOTION_FILTER:
+        case VVCAM_ISP_CID_AE_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_AE_TARGET_FILTER:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX:
+        case VVCAM_ISP_CID_AE_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_ROI:
+        case VVCAM_ISP_CID_AE_HIST:
+        case VVCAM_ISP_CID_AE_LUMA:
+        case VVCAM_ISP_CID_AE_OBJECT_REGION:
+        case VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE:
+        case VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_EC_AGAIN:
+        case VVCAM_ISP_CID_EC_AGAIN_MAX:
+        case VVCAM_ISP_CID_EC_AGAIN_MIN:
+        case VVCAM_ISP_CID_EC_AGAIN_STEP:
+        case VVCAM_ISP_CID_EC_DGAIN:
+        case VVCAM_ISP_CID_EC_DGAIN_MAX:
+        case VVCAM_ISP_CID_EC_DGAIN_MIN:
+        case VVCAM_ISP_CID_EC_DGAIN_STEP:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ae_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AE_ENABLE:
+        case VVCAM_ISP_CID_AE_RESET:
+        case VVCAM_ISP_CID_AE_STATE:
+        case VVCAM_ISP_CID_AE_SEM_MODE:
+        case VVCAM_ISP_CID_AE_FLICKER_PERIOD:
+        case VVCAM_ISP_CID_AE_SETPOINT:
+        case VVCAM_ISP_CID_AE_TOLORENCE:
+        case VVCAM_ISP_CID_AE_DAMPOVER:
+        case VVCAM_ISP_CID_AE_DAMPOVER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPOVER_GAIN:
+        case VVCAM_ISP_CID_AE_DAMPUNDER:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_GAIN:
+        case VVCAM_ISP_CID_AE_MOTION_FILTER:
+        case VVCAM_ISP_CID_AE_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_AE_TARGET_FILTER:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX:
+        case VVCAM_ISP_CID_AE_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_ROI:
+        case VVCAM_ISP_CID_AE_HIST:
+        case VVCAM_ISP_CID_AE_LUMA:
+        case VVCAM_ISP_CID_AE_OBJECT_REGION:
+        case VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE:
+        case VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_EC_AGAIN:
+        case VVCAM_ISP_CID_EC_AGAIN_MAX:
+        case VVCAM_ISP_CID_EC_AGAIN_MIN:
+        case VVCAM_ISP_CID_EC_AGAIN_STEP:
+        case VVCAM_ISP_CID_EC_DGAIN:
+        case VVCAM_ISP_CID_EC_DGAIN_MAX:
+        case VVCAM_ISP_CID_EC_DGAIN_MIN:
+        case VVCAM_ISP_CID_EC_DGAIN_STEP:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ae_ctrl_ops = {
+	.s_ctrl = vvcam_isp_ae_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ae_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ae_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_SEM_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_sem_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FLICKER_PERIOD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_flicker_period",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_SETPOINT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_setpoint",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_TOLORENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_tolerence",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (1,4) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover_ratio",
+        .step = 1,
+        .min  = 101,
+        .max  = 399,
+        .def  = 200,
+    },
+    {
+        /* float [1,128) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover_gain",
+        .step = 1,
+        .min  = 100,
+        .max  = 12799,
+        .def  = 100,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float [1,16) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder_gain",
+        .step = 1,
+        .min  = 100,
+        .max  = 1599,
+        .def  = 100,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_MOTION_FILTER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_motion_filter",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_TARGET_FILTER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_target_filter",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_repress",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_repress",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* int 0~16 */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_wdr_contrast_min",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_wdr_contrast_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float 25x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* int 25x4*16bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        /* uint32_t array 256*32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_HIST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_hist",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {256},
+    },
+    {
+        /* uint8_t array 32*32*8bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LUMA,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_luma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1024},
+    },
+    {
+        /* uint8_t array 32*32*8bit*/
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_OBJECT_REGION,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_object_region",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1024},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_frame_calc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 32x32x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_v2_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1024},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+};
+
+int vvcam_isp_ae_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ae_ctrls);
+}
+
+int vvcam_isp_ae_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ae_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ae_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ae ctrl %s failed %d.\n",
+                vvcam_isp_ae_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v3/vvcam_isp_ae.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v3/vvcam_isp_ae.h
new file mode 100644
index 000000000000..dfbd043c0e12
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v3/vvcam_isp_ae.h
@@ -0,0 +1,110 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AE_H__
+#define __VVCAM_ISP_AE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AE_ENABLE                     (VVCAM_ISP_CID_AE_BASE + 0x0000)
+#define VVCAM_ISP_CID_AE_RESET                      (VVCAM_ISP_CID_AE_BASE + 0x0001)
+#define VVCAM_ISP_CID_AE_STATE                      (VVCAM_ISP_CID_AE_BASE + 0x0002)
+#define VVCAM_ISP_CID_AE_SEM_MODE                   (VVCAM_ISP_CID_AE_BASE + 0x0004)
+#define VVCAM_ISP_CID_AE_FLICKER_PERIOD             (VVCAM_ISP_CID_AE_BASE + 0x0005)
+#define VVCAM_ISP_CID_AE_SETPOINT                   (VVCAM_ISP_CID_AE_BASE + 0x0006)
+#define VVCAM_ISP_CID_AE_TOLORENCE                  (VVCAM_ISP_CID_AE_BASE + 0x0007)
+#define VVCAM_ISP_CID_AE_DAMPOVER                   (VVCAM_ISP_CID_AE_BASE + 0x0008)
+#define VVCAM_ISP_CID_AE_DAMPOVER_RATIO             (VVCAM_ISP_CID_AE_BASE + 0x0009)
+#define VVCAM_ISP_CID_AE_DAMPOVER_GAIN              (VVCAM_ISP_CID_AE_BASE + 0x000A)
+#define VVCAM_ISP_CID_AE_DAMPUNDER                  (VVCAM_ISP_CID_AE_BASE + 0x000B)
+#define VVCAM_ISP_CID_AE_DAMPUNDER_RATIO            (VVCAM_ISP_CID_AE_BASE + 0x000C)
+#define VVCAM_ISP_CID_AE_DAMPUNDER_GAIN             (VVCAM_ISP_CID_AE_BASE + 0x000D)
+#define VVCAM_ISP_CID_AE_MOTION_FILTER              (VVCAM_ISP_CID_AE_BASE + 0x000E)
+#define VVCAM_ISP_CID_AE_MOTION_THRESHOLD           (VVCAM_ISP_CID_AE_BASE + 0x000F)
+#define VVCAM_ISP_CID_AE_TARGET_FILTER              (VVCAM_ISP_CID_AE_BASE + 0x0010)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS    (VVCAM_ISP_CID_AE_BASE + 0x0011)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN       (VVCAM_ISP_CID_AE_BASE + 0x0012)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL      (VVCAM_ISP_CID_AE_BASE + 0x0013)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS       (VVCAM_ISP_CID_AE_BASE + 0x0014)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN          (VVCAM_ISP_CID_AE_BASE + 0x0015)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL         (VVCAM_ISP_CID_AE_BASE + 0x0016)
+#define VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN           (VVCAM_ISP_CID_AE_BASE + 0x0017)
+#define VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX           (VVCAM_ISP_CID_AE_BASE + 0x0018)
+#define VVCAM_ISP_CID_AE_ROI_WEIGHT                 (VVCAM_ISP_CID_AE_BASE + 0x001A)
+#define VVCAM_ISP_CID_AE_ROI                        (VVCAM_ISP_CID_AE_BASE + 0x001B)
+#define VVCAM_ISP_CID_AE_HIST                       (VVCAM_ISP_CID_AE_BASE + 0x001C)
+#define VVCAM_ISP_CID_AE_LUMA                       (VVCAM_ISP_CID_AE_BASE + 0x001D)
+#define VVCAM_ISP_CID_AE_OBJECT_REGION              (VVCAM_ISP_CID_AE_BASE + 0x001E)
+#define VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE          (VVCAM_ISP_CID_AE_BASE + 0x001F)
+#define VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT        (VVCAM_ISP_CID_AE_BASE + 0x0020)
+#define VVCAM_ISP_CID_EC_AGAIN                      (VVCAM_ISP_CID_AE_BASE + 0x0021)
+#define VVCAM_ISP_CID_EC_AGAIN_MAX                  (VVCAM_ISP_CID_AE_BASE + 0x0022)
+#define VVCAM_ISP_CID_EC_AGAIN_MIN                  (VVCAM_ISP_CID_AE_BASE + 0x0023)
+#define VVCAM_ISP_CID_EC_AGAIN_STEP                 (VVCAM_ISP_CID_AE_BASE + 0x0024)
+#define VVCAM_ISP_CID_EC_DGAIN                      (VVCAM_ISP_CID_AE_BASE + 0x0025)
+#define VVCAM_ISP_CID_EC_DGAIN_MAX                  (VVCAM_ISP_CID_AE_BASE + 0x0026)
+#define VVCAM_ISP_CID_EC_DGAIN_MIN                  (VVCAM_ISP_CID_AE_BASE + 0x0027)
+#define VVCAM_ISP_CID_EC_DGAIN_STEP                 (VVCAM_ISP_CID_AE_BASE + 0x0028)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME           (VVCAM_ISP_CID_AE_BASE + 0x0029)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX       (VVCAM_ISP_CID_AE_BASE + 0x002A)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN       (VVCAM_ISP_CID_AE_BASE + 0x002B)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP      (VVCAM_ISP_CID_AE_BASE + 0x002C)
+
+#ifdef __KERNEL__
+int vvcam_isp_ae_ctrl_count(void);
+int vvcam_isp_ae_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4/vvcam_isp_ae.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4/vvcam_isp_ae.c
new file mode 100644
index 000000000000..cfe660acf6ff
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4/vvcam_isp_ae.c
@@ -0,0 +1,765 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ae.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ae_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AE_ENABLE:
+        case VVCAM_ISP_CID_AE_RESET:
+        case VVCAM_ISP_CID_AE_STATE:
+        case VVCAM_ISP_CID_AE_SEM_MODE:
+        case VVCAM_ISP_CID_AE_FLICKER_PERIOD:
+        case VVCAM_ISP_CID_AE_SETPOINT:
+        case VVCAM_ISP_CID_AE_TOLORENCE:
+        case VVCAM_ISP_CID_AE_DAMPOVER:
+        case VVCAM_ISP_CID_AE_DAMPOVER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPOVER_GAIN:
+        case VVCAM_ISP_CID_AE_DAMPUNDER:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_GAIN:
+        case VVCAM_ISP_CID_AE_MOTION_FILTER:
+        case VVCAM_ISP_CID_AE_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_AE_TARGET_FILTER:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX:
+        case VVCAM_ISP_CID_AE_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_ROI:
+        case VVCAM_ISP_CID_AE_HIST:
+        case VVCAM_ISP_CID_AE_LUMA:
+        case VVCAM_ISP_CID_AE_OBJECT_REGION:
+        case VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE:
+        case VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_EC_AGAIN:
+        case VVCAM_ISP_CID_EC_AGAIN_MAX:
+        case VVCAM_ISP_CID_EC_AGAIN_MIN:
+        case VVCAM_ISP_CID_EC_AGAIN_STEP:
+        case VVCAM_ISP_CID_EC_DGAIN:
+        case VVCAM_ISP_CID_EC_DGAIN_MAX:
+        case VVCAM_ISP_CID_EC_DGAIN_MIN:
+        case VVCAM_ISP_CID_EC_DGAIN_STEP:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP:
+        case VVCAM_ISP_CID_AE_EXP_TABLE:
+        case VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM:
+        case VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ae_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AE_ENABLE:
+        case VVCAM_ISP_CID_AE_RESET:
+        case VVCAM_ISP_CID_AE_STATE:
+        case VVCAM_ISP_CID_AE_SEM_MODE:
+        case VVCAM_ISP_CID_AE_FLICKER_PERIOD:
+        case VVCAM_ISP_CID_AE_SETPOINT:
+        case VVCAM_ISP_CID_AE_TOLORENCE:
+        case VVCAM_ISP_CID_AE_DAMPOVER:
+        case VVCAM_ISP_CID_AE_DAMPOVER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPOVER_GAIN:
+        case VVCAM_ISP_CID_AE_DAMPUNDER:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_GAIN:
+        case VVCAM_ISP_CID_AE_MOTION_FILTER:
+        case VVCAM_ISP_CID_AE_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_AE_TARGET_FILTER:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX:
+        case VVCAM_ISP_CID_AE_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_ROI:
+        case VVCAM_ISP_CID_AE_HIST:
+        case VVCAM_ISP_CID_AE_LUMA:
+        case VVCAM_ISP_CID_AE_OBJECT_REGION:
+        case VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE:
+        case VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_EC_AGAIN:
+        case VVCAM_ISP_CID_EC_AGAIN_MAX:
+        case VVCAM_ISP_CID_EC_AGAIN_MIN:
+        case VVCAM_ISP_CID_EC_AGAIN_STEP:
+        case VVCAM_ISP_CID_EC_DGAIN:
+        case VVCAM_ISP_CID_EC_DGAIN_MAX:
+        case VVCAM_ISP_CID_EC_DGAIN_MIN:
+        case VVCAM_ISP_CID_EC_DGAIN_STEP:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP:
+        case VVCAM_ISP_CID_AE_EXP_TABLE:
+        case VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM:
+        case VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE:
+        case VVCAM_ISP_CID_AE_RESULT_MEAN_LUMA:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ae_ctrl_ops = {
+    .s_ctrl = vvcam_isp_ae_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ae_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ae_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_SEM_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_sem_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FLICKER_PERIOD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_flicker_period",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_SETPOINT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_setpoint",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_TOLORENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_tolerence",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (1,4) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover_ratio",
+        .step = 1,
+        .min  = 101,
+        .max  = 399,
+        .def  = 200,
+    },
+    {
+        /* float [1,128) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover_gain",
+        .step = 1,
+        .min  = 100,
+        .max  = 12799,
+        .def  = 100,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float [1,16) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder_gain",
+        .step = 1,
+        .min  = 100,
+        .max  = 1599,
+        .def  = 100,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_MOTION_FILTER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_motion_filter",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_TARGET_FILTER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_target_filter",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_repress",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_repress",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* int 0~16 */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_wdr_contrast_min",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_wdr_contrast_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float 25x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* int 25x4*16bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        /* uint32_t array 256*32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_HIST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_hist",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {256},
+    },
+    {
+        /* uint8_t array 32*32*8bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LUMA,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_luma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1024},
+    },
+    {
+        /* uint8_t array 32*32*8bit*/
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_OBJECT_REGION,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_object_region",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1024},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_frame_calc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 32x32x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_v2_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1024},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 8x4*32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_TABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_table",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8, 4, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_decomp_custom",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_performance_opti_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_RESULT_MEAN_LUMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_result_mean_luma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+    },
+};
+
+int vvcam_isp_ae_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ae_ctrls);
+}
+
+int vvcam_isp_ae_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ae_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ae_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err(isp_dev->dev, "reigster isp ae ctrl %s failed %d.\n",
+                vvcam_isp_ae_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4/vvcam_isp_ae.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4/vvcam_isp_ae.h
new file mode 100644
index 000000000000..5db45a76b5e1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4/vvcam_isp_ae.h
@@ -0,0 +1,114 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AE_H__
+#define __VVCAM_ISP_AE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AE_ENABLE                     (VVCAM_ISP_CID_AE_BASE + 0x0000)
+#define VVCAM_ISP_CID_AE_RESET                      (VVCAM_ISP_CID_AE_BASE + 0x0001)
+#define VVCAM_ISP_CID_AE_STATE                      (VVCAM_ISP_CID_AE_BASE + 0x0002)
+#define VVCAM_ISP_CID_AE_SEM_MODE                   (VVCAM_ISP_CID_AE_BASE + 0x0004)
+#define VVCAM_ISP_CID_AE_FLICKER_PERIOD             (VVCAM_ISP_CID_AE_BASE + 0x0005)
+#define VVCAM_ISP_CID_AE_SETPOINT                   (VVCAM_ISP_CID_AE_BASE + 0x0006)
+#define VVCAM_ISP_CID_AE_TOLORENCE                  (VVCAM_ISP_CID_AE_BASE + 0x0007)
+#define VVCAM_ISP_CID_AE_DAMPOVER                   (VVCAM_ISP_CID_AE_BASE + 0x0008)
+#define VVCAM_ISP_CID_AE_DAMPOVER_RATIO             (VVCAM_ISP_CID_AE_BASE + 0x0009)
+#define VVCAM_ISP_CID_AE_DAMPOVER_GAIN              (VVCAM_ISP_CID_AE_BASE + 0x000A)
+#define VVCAM_ISP_CID_AE_DAMPUNDER                  (VVCAM_ISP_CID_AE_BASE + 0x000B)
+#define VVCAM_ISP_CID_AE_DAMPUNDER_RATIO            (VVCAM_ISP_CID_AE_BASE + 0x000C)
+#define VVCAM_ISP_CID_AE_DAMPUNDER_GAIN             (VVCAM_ISP_CID_AE_BASE + 0x000D)
+#define VVCAM_ISP_CID_AE_MOTION_FILTER              (VVCAM_ISP_CID_AE_BASE + 0x000E)
+#define VVCAM_ISP_CID_AE_MOTION_THRESHOLD           (VVCAM_ISP_CID_AE_BASE + 0x000F)
+#define VVCAM_ISP_CID_AE_TARGET_FILTER              (VVCAM_ISP_CID_AE_BASE + 0x0010)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS    (VVCAM_ISP_CID_AE_BASE + 0x0011)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN       (VVCAM_ISP_CID_AE_BASE + 0x0012)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL      (VVCAM_ISP_CID_AE_BASE + 0x0013)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS       (VVCAM_ISP_CID_AE_BASE + 0x0014)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN          (VVCAM_ISP_CID_AE_BASE + 0x0015)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL         (VVCAM_ISP_CID_AE_BASE + 0x0016)
+#define VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN           (VVCAM_ISP_CID_AE_BASE + 0x0017)
+#define VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX           (VVCAM_ISP_CID_AE_BASE + 0x0018)
+#define VVCAM_ISP_CID_AE_ROI_WEIGHT                 (VVCAM_ISP_CID_AE_BASE + 0x001A)
+#define VVCAM_ISP_CID_AE_ROI                        (VVCAM_ISP_CID_AE_BASE + 0x001B)
+#define VVCAM_ISP_CID_AE_HIST                       (VVCAM_ISP_CID_AE_BASE + 0x001C)
+#define VVCAM_ISP_CID_AE_LUMA                       (VVCAM_ISP_CID_AE_BASE + 0x001D)
+#define VVCAM_ISP_CID_AE_OBJECT_REGION              (VVCAM_ISP_CID_AE_BASE + 0x001E)
+#define VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE          (VVCAM_ISP_CID_AE_BASE + 0x001F)
+#define VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT        (VVCAM_ISP_CID_AE_BASE + 0x0020)
+#define VVCAM_ISP_CID_EC_AGAIN                      (VVCAM_ISP_CID_AE_BASE + 0x0021)
+#define VVCAM_ISP_CID_EC_AGAIN_MAX                  (VVCAM_ISP_CID_AE_BASE + 0x0022)
+#define VVCAM_ISP_CID_EC_AGAIN_MIN                  (VVCAM_ISP_CID_AE_BASE + 0x0023)
+#define VVCAM_ISP_CID_EC_AGAIN_STEP                 (VVCAM_ISP_CID_AE_BASE + 0x0024)
+#define VVCAM_ISP_CID_EC_DGAIN                      (VVCAM_ISP_CID_AE_BASE + 0x0025)
+#define VVCAM_ISP_CID_EC_DGAIN_MAX                  (VVCAM_ISP_CID_AE_BASE + 0x0026)
+#define VVCAM_ISP_CID_EC_DGAIN_MIN                  (VVCAM_ISP_CID_AE_BASE + 0x0027)
+#define VVCAM_ISP_CID_EC_DGAIN_STEP                 (VVCAM_ISP_CID_AE_BASE + 0x0028)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME           (VVCAM_ISP_CID_AE_BASE + 0x0029)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX       (VVCAM_ISP_CID_AE_BASE + 0x002A)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN       (VVCAM_ISP_CID_AE_BASE + 0x002B)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP      (VVCAM_ISP_CID_AE_BASE + 0x002C)
+#define VVCAM_ISP_CID_AE_EXP_TABLE                  (VVCAM_ISP_CID_AE_BASE + 0x002D)
+#define VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM          (VVCAM_ISP_CID_AE_BASE + 0x002E)
+#define VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE      (VVCAM_ISP_CID_AE_BASE + 0x002F)
+#define VVCAM_ISP_CID_AE_RESULT_MEAN_LUMA           (VVCAM_ISP_CID_AE_BASE + 0x0030)
+
+#ifdef __KERNEL__
+int vvcam_isp_ae_ctrl_count(void);
+int vvcam_isp_ae_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.c
new file mode 100644
index 000000000000..0678014bdc9b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.c
@@ -0,0 +1,847 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ae.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ae_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AE_ENABLE:
+        case VVCAM_ISP_CID_AE_RESET:
+        case VVCAM_ISP_CID_AE_STATE:
+        case VVCAM_ISP_CID_AE_SEM_MODE:
+        case VVCAM_ISP_CID_AE_FLICKER_PERIOD:
+        case VVCAM_ISP_CID_AE_SETPOINT:
+        case VVCAM_ISP_CID_AE_TOLORENCE:
+        case VVCAM_ISP_CID_AE_DAMPOVER:
+        case VVCAM_ISP_CID_AE_DAMPOVER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPOVER_GAIN:
+        case VVCAM_ISP_CID_AE_DAMPUNDER:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_GAIN:
+        case VVCAM_ISP_CID_AE_MOTION_FILTER:
+        case VVCAM_ISP_CID_AE_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_AE_TARGET_FILTER:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX:
+        case VVCAM_ISP_CID_AE_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_ROI:
+        case VVCAM_ISP_CID_AE_HIST:
+        case VVCAM_ISP_CID_AE_LUMA:
+        case VVCAM_ISP_CID_AE_OBJECT_REGION:
+        case VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE:
+        case VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_EC_AGAIN:
+        case VVCAM_ISP_CID_EC_AGAIN_MAX:
+        case VVCAM_ISP_CID_EC_AGAIN_MIN:
+        case VVCAM_ISP_CID_EC_AGAIN_STEP:
+        case VVCAM_ISP_CID_EC_DGAIN:
+        case VVCAM_ISP_CID_EC_DGAIN_MAX:
+        case VVCAM_ISP_CID_EC_DGAIN_MIN:
+        case VVCAM_ISP_CID_EC_DGAIN_STEP:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP:
+        case VVCAM_ISP_CID_AE_EXP_TABLE:
+        case VVCAM_ISP_CID_AE_EXP_V3_ROI:
+        case VVCAM_ISP_CID_AE_EXP_V3_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_EXP_V3_ROI_CALC_WEIGHT:
+        case VVCAM_ISP_CID_AE_FRONT_GROUND_TYPE:
+        case VVCAM_ISP_CID_AE_FRONT_GROUND_FACE_WEIGHT:
+        case VVCAM_ISP_CID_AE_FRONT_GROUND_TOUCH_WEIGHT:
+        case VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM:
+        case VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ae_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AE_ENABLE:
+        case VVCAM_ISP_CID_AE_RESET:
+        case VVCAM_ISP_CID_AE_STATE:
+        case VVCAM_ISP_CID_AE_SEM_MODE:
+        case VVCAM_ISP_CID_AE_FLICKER_PERIOD:
+        case VVCAM_ISP_CID_AE_SETPOINT:
+        case VVCAM_ISP_CID_AE_TOLORENCE:
+        case VVCAM_ISP_CID_AE_DAMPOVER:
+        case VVCAM_ISP_CID_AE_DAMPOVER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPOVER_GAIN:
+        case VVCAM_ISP_CID_AE_DAMPUNDER:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_RATIO:
+        case VVCAM_ISP_CID_AE_DAMPUNDER_GAIN:
+        case VVCAM_ISP_CID_AE_MOTION_FILTER:
+        case VVCAM_ISP_CID_AE_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_AE_TARGET_FILTER:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN:
+        case VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN:
+        case VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX:
+        case VVCAM_ISP_CID_AE_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_ROI:
+        case VVCAM_ISP_CID_AE_HIST:
+        case VVCAM_ISP_CID_AE_LUMA:
+        case VVCAM_ISP_CID_AE_OBJECT_REGION:
+        case VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE:
+        case VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_EC_AGAIN:
+        case VVCAM_ISP_CID_EC_AGAIN_MAX:
+        case VVCAM_ISP_CID_EC_AGAIN_MIN:
+        case VVCAM_ISP_CID_EC_AGAIN_STEP:
+        case VVCAM_ISP_CID_EC_DGAIN:
+        case VVCAM_ISP_CID_EC_DGAIN_MAX:
+        case VVCAM_ISP_CID_EC_DGAIN_MIN:
+        case VVCAM_ISP_CID_EC_DGAIN_STEP:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN:
+        case VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP:
+        case VVCAM_ISP_CID_AE_EXP_TABLE:
+        case VVCAM_ISP_CID_AE_EXP_V3_ROI:
+        case VVCAM_ISP_CID_AE_EXP_V3_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AE_EXP_V3_ROI_CALC_WEIGHT:
+        case VVCAM_ISP_CID_AE_FRONT_GROUND_TYPE:
+        case VVCAM_ISP_CID_AE_FRONT_GROUND_FACE_WEIGHT:
+        case VVCAM_ISP_CID_AE_FRONT_GROUND_TOUCH_WEIGHT:
+        case VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM:
+        case VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE:
+        case VVCAM_ISP_CID_AE_RESULT_MEAN_LUMA:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ae_ctrl_ops = {
+    .s_ctrl = vvcam_isp_ae_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ae_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ae_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_SEM_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_sem_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FLICKER_PERIOD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_flicker_period",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_SETPOINT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_setpoint",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_TOLORENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_tolerence",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (1,4) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover_ratio",
+        .step = 1,
+        .min  = 101,
+        .max  = 399,
+        .def  = 200,
+    },
+    {
+        /* float [1,128) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPOVER_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampover_gain",
+        .step = 1,
+        .min  = 100,
+        .max  = 12799,
+        .def  = 100,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float [1,16) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_DAMPUNDER_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_dampunder_gain",
+        .step = 1,
+        .min  = 100,
+        .max  = 1599,
+        .def  = 100,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_MOTION_FILTER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_motion_filter",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) 32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_TARGET_FILTER,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_target_filter",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_repress",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_linear_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_repress",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* float array 8x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8},
+    },
+    {
+        /* int 0~16 */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_lowlight_hdr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_wdr_contrast_min",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,255) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_wdr_contrast_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float 25x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* int 25x4*16bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        /* uint32_t array 256*32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_HIST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_hist",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {256},
+    },
+    {
+        /* uint8_t array 32*32*8bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_LUMA,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_luma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1024},
+    },
+    {
+        /* uint8_t array 32*32*8bit*/
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_OBJECT_REGION,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_object_region",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1024},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_frame_calc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 32x32x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_v2_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1024},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_AGAIN_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_again_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_DGAIN_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_dgain_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ec_integration_time_step",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 8x4*32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_TABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_table",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {8, 4, 0, 0},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_V3_ROI_CALC_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_v3_roi_calc_weight",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float 25x32bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_V3_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_v3_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* uint16_t 25x4x16bit */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_V3_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_v3_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FRONT_GROUND_TYPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_front_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float (0, 1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FRONT_GROUND_FACE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_front_face_weight",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0, 1) */
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_FRONT_GROUND_TOUCH_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_front_touch_weight",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_exp_decomp_custom",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_performance_opti_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_ae_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AE_RESULT_MEAN_LUMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ae_result_mean_luma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+    },
+};
+
+int vvcam_isp_ae_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ae_ctrls);
+}
+
+int vvcam_isp_ae_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ae_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ae_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err(isp_dev->dev, "reigster isp ae ctrl %s failed %d.\n",
+                vvcam_isp_ae_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.h
new file mode 100644
index 000000000000..4fd83e7fc178
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ae/ae_v4_1/vvcam_isp_ae.h
@@ -0,0 +1,120 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AE_H__
+#define __VVCAM_ISP_AE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AE_ENABLE                     (VVCAM_ISP_CID_AE_BASE + 0x0000)
+#define VVCAM_ISP_CID_AE_RESET                      (VVCAM_ISP_CID_AE_BASE + 0x0001)
+#define VVCAM_ISP_CID_AE_STATE                      (VVCAM_ISP_CID_AE_BASE + 0x0002)
+#define VVCAM_ISP_CID_AE_SEM_MODE                   (VVCAM_ISP_CID_AE_BASE + 0x0004)
+#define VVCAM_ISP_CID_AE_FLICKER_PERIOD             (VVCAM_ISP_CID_AE_BASE + 0x0005)
+#define VVCAM_ISP_CID_AE_SETPOINT                   (VVCAM_ISP_CID_AE_BASE + 0x0006)
+#define VVCAM_ISP_CID_AE_TOLORENCE                  (VVCAM_ISP_CID_AE_BASE + 0x0007)
+#define VVCAM_ISP_CID_AE_DAMPOVER                   (VVCAM_ISP_CID_AE_BASE + 0x0008)
+#define VVCAM_ISP_CID_AE_DAMPOVER_RATIO             (VVCAM_ISP_CID_AE_BASE + 0x0009)
+#define VVCAM_ISP_CID_AE_DAMPOVER_GAIN              (VVCAM_ISP_CID_AE_BASE + 0x000A)
+#define VVCAM_ISP_CID_AE_DAMPUNDER                  (VVCAM_ISP_CID_AE_BASE + 0x000B)
+#define VVCAM_ISP_CID_AE_DAMPUNDER_RATIO            (VVCAM_ISP_CID_AE_BASE + 0x000C)
+#define VVCAM_ISP_CID_AE_DAMPUNDER_GAIN             (VVCAM_ISP_CID_AE_BASE + 0x000D)
+#define VVCAM_ISP_CID_AE_MOTION_FILTER              (VVCAM_ISP_CID_AE_BASE + 0x000E)
+#define VVCAM_ISP_CID_AE_MOTION_THRESHOLD           (VVCAM_ISP_CID_AE_BASE + 0x000F)
+#define VVCAM_ISP_CID_AE_TARGET_FILTER              (VVCAM_ISP_CID_AE_BASE + 0x0010)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_REPRESS    (VVCAM_ISP_CID_AE_BASE + 0x0011)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_GAIN       (VVCAM_ISP_CID_AE_BASE + 0x0012)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_LINEAR_LEVEL      (VVCAM_ISP_CID_AE_BASE + 0x0013)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_REPRESS       (VVCAM_ISP_CID_AE_BASE + 0x0014)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_GAIN          (VVCAM_ISP_CID_AE_BASE + 0x0015)
+#define VVCAM_ISP_CID_AE_LOWLIGHT_HDR_LEVEL         (VVCAM_ISP_CID_AE_BASE + 0x0016)
+#define VVCAM_ISP_CID_AE_WDR_CONTRAST_MIN           (VVCAM_ISP_CID_AE_BASE + 0x0017)
+#define VVCAM_ISP_CID_AE_WDR_CONTRAST_MAX           (VVCAM_ISP_CID_AE_BASE + 0x0018)
+#define VVCAM_ISP_CID_AE_ROI_WEIGHT                 (VVCAM_ISP_CID_AE_BASE + 0x001A)
+#define VVCAM_ISP_CID_AE_ROI                        (VVCAM_ISP_CID_AE_BASE + 0x001B)
+#define VVCAM_ISP_CID_AE_HIST                       (VVCAM_ISP_CID_AE_BASE + 0x001C)
+#define VVCAM_ISP_CID_AE_LUMA                       (VVCAM_ISP_CID_AE_BASE + 0x001D)
+#define VVCAM_ISP_CID_AE_OBJECT_REGION              (VVCAM_ISP_CID_AE_BASE + 0x001E)
+#define VVCAM_ISP_CID_AE_FRAME_CALC_ENABLE          (VVCAM_ISP_CID_AE_BASE + 0x001F)
+#define VVCAM_ISP_CID_AE_EXPV2_WINDOW_WEIGHT        (VVCAM_ISP_CID_AE_BASE + 0x0020)
+#define VVCAM_ISP_CID_EC_AGAIN                      (VVCAM_ISP_CID_AE_BASE + 0x0021)
+#define VVCAM_ISP_CID_EC_AGAIN_MAX                  (VVCAM_ISP_CID_AE_BASE + 0x0022)
+#define VVCAM_ISP_CID_EC_AGAIN_MIN                  (VVCAM_ISP_CID_AE_BASE + 0x0023)
+#define VVCAM_ISP_CID_EC_AGAIN_STEP                 (VVCAM_ISP_CID_AE_BASE + 0x0024)
+#define VVCAM_ISP_CID_EC_DGAIN                      (VVCAM_ISP_CID_AE_BASE + 0x0025)
+#define VVCAM_ISP_CID_EC_DGAIN_MAX                  (VVCAM_ISP_CID_AE_BASE + 0x0026)
+#define VVCAM_ISP_CID_EC_DGAIN_MIN                  (VVCAM_ISP_CID_AE_BASE + 0x0027)
+#define VVCAM_ISP_CID_EC_DGAIN_STEP                 (VVCAM_ISP_CID_AE_BASE + 0x0028)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME           (VVCAM_ISP_CID_AE_BASE + 0x0029)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_MAX       (VVCAM_ISP_CID_AE_BASE + 0x002A)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_MIN       (VVCAM_ISP_CID_AE_BASE + 0x002B)
+#define VVCAM_ISP_CID_EC_INTEGRATION_TIME_STEP      (VVCAM_ISP_CID_AE_BASE + 0x002C)
+#define VVCAM_ISP_CID_AE_EXP_TABLE                  (VVCAM_ISP_CID_AE_BASE + 0x002D)
+#define VVCAM_ISP_CID_AE_EXP_V3_ROI                 (VVCAM_ISP_CID_AE_BASE + 0x002E)
+#define VVCAM_ISP_CID_AE_EXP_V3_ROI_WEIGHT          (VVCAM_ISP_CID_AE_BASE + 0x002F)
+#define VVCAM_ISP_CID_AE_EXP_V3_ROI_CALC_WEIGHT     (VVCAM_ISP_CID_AE_BASE + 0x0030)
+#define VVCAM_ISP_CID_AE_FRONT_GROUND_TYPE          (VVCAM_ISP_CID_AE_BASE + 0x0031)
+#define VVCAM_ISP_CID_AE_FRONT_GROUND_FACE_WEIGHT   (VVCAM_ISP_CID_AE_BASE + 0x0032)
+#define VVCAM_ISP_CID_AE_FRONT_GROUND_TOUCH_WEIGHT  (VVCAM_ISP_CID_AE_BASE + 0x0033)
+#define VVCAM_ISP_CID_AE_EXP_DECOMP_CUSTOM          (VVCAM_ISP_CID_AE_BASE + 0x0034)
+#define VVCAM_ISP_CID_AE_PERFORMANCE_OPTI_MODE      (VVCAM_ISP_CID_AE_BASE + 0x0035)
+#define VVCAM_ISP_CID_AE_RESULT_MEAN_LUMA           (VVCAM_ISP_CID_AE_BASE + 0x0036)
+
+#ifdef __KERNEL__
+int vvcam_isp_ae_ctrl_count(void);
+int vvcam_isp_ae_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v3/vvcam_isp_af.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v3/vvcam_isp_af.c
new file mode 100644
index 000000000000..0eac74aea856
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v3/vvcam_isp_af.c
@@ -0,0 +1,396 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_af.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_af_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_RESET:
+        case VVCAM_ISP_CID_AF_STATE:
+        case VVCAM_ISP_CID_AF_MODE:
+        case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_FOCAL_FILTER:
+        case VVCAM_ISP_CID_CDAF_SHAPE_FILTER:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_STABLE_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_MSE_TOLERENCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_FOCAL:
+        case VVCAM_ISP_CID_PDAF_PD_DISTANCE:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_af_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_RESET:
+        case VVCAM_ISP_CID_AF_STATE:
+        case VVCAM_ISP_CID_AF_MODE:
+        case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_FOCAL_FILTER:
+        case VVCAM_ISP_CID_CDAF_SHAPE_FILTER:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_STABLE_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_MSE_TOLERENCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_FOCAL:
+        case VVCAM_ISP_CID_PDAF_PD_DISTANCE:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_af_ctrl_ops = {
+    .s_ctrl = vvcam_isp_af_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_af_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_af_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 0,
+    },
+    {
+        /* float array 3x */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_stable_tolerance",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_points_of_curve",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        /* float 5x array 0~1023 */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_FOCAL_FILTER,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_focal_filter",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {5, 0, 0, 0},
+    },
+    {
+        /* float 5x array 0~1023 */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_SHAPE_FILTER,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_shape_filter",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {5, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MAX_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_max_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MIN_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_min_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 999,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_STABLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_stable_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 999,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_MSE_TOLERENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_mse_tolerence",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        /* float (0,10230) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_conf_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 10229,
+        .def  = 1,
+    },
+    {
+        /* int 48x array (-255,255)*/
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_FOCAL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {48, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_DISTANCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_distance",
+        .step = 1,
+        .min  = 1,
+        .max  = 254,
+        .def  = 1,
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_shift_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_stable_cnt_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 10,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_ROI_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_roi_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 48,
+    },
+};
+
+int vvcam_isp_af_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_af_ctrls);
+}
+
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_af_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_af_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp af ctrl %s failed %d.\n",
+                vvcam_isp_af_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v3/vvcam_isp_af.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v3/vvcam_isp_af.h
new file mode 100644
index 000000000000..21d1de26f1b7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v3/vvcam_isp_af.h
@@ -0,0 +1,90 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AF_H__
+#define __VVCAM_ISP_AF_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AF_ENABLE             (VVCAM_ISP_CID_AF_BASE + 0x0000)
+#define VVCAM_ISP_CID_AF_RESET              (VVCAM_ISP_CID_AF_BASE + 0x0001)
+#define VVCAM_ISP_CID_AF_STATE              (VVCAM_ISP_CID_AF_BASE + 0x0002)
+#define VVCAM_ISP_CID_AF_MODE               (VVCAM_ISP_CID_AF_BASE + 0x0003)
+#define VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT    (VVCAM_ISP_CID_AF_BASE + 0x0004)
+#define VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE (VVCAM_ISP_CID_AF_BASE + 0x0005)
+#define VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE  (VVCAM_ISP_CID_AF_BASE + 0x0006)
+#define VVCAM_ISP_CID_CDAF_FOCAL_FILTER     (VVCAM_ISP_CID_AF_BASE + 0x0007)
+#define VVCAM_ISP_CID_CDAF_SHAPE_FILTER     (VVCAM_ISP_CID_AF_BASE + 0x0008)
+#define VVCAM_ISP_CID_CDAF_MAX_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0009)
+#define VVCAM_ISP_CID_CDAF_MIN_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x000A)
+#define VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD (VVCAM_ISP_CID_AF_BASE + 0x000B)
+#define VVCAM_ISP_CID_CDAF_STABLE_THRESHOLD (VVCAM_ISP_CID_AF_BASE + 0x000C)
+#define VVCAM_ISP_CID_PDAF_MSE_TOLERENCE    (VVCAM_ISP_CID_AF_BASE + 0x000D)
+#define VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000E)
+#define VVCAM_ISP_CID_PDAF_PD_FOCAL         (VVCAM_ISP_CID_AF_BASE + 0x000F)
+#define VVCAM_ISP_CID_PDAF_PD_DISTANCE      (VVCAM_ISP_CID_AF_BASE + 0x0010)
+#define VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0011)
+#define VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0012)
+#define VVCAM_ISP_CID_PDAF_ROI_INDEX        (VVCAM_ISP_CID_AF_BASE + 0x0013)
+
+#ifdef __KERNEL__
+int vvcam_isp_af_ctrl_count(void);
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4/vvcam_isp_af.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4/vvcam_isp_af.c
new file mode 100644
index 000000000000..ed295e7b167b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4/vvcam_isp_af.c
@@ -0,0 +1,410 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_af.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_af_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_RESET:
+        case VVCAM_ISP_CID_AF_MODE:
+        // case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD:
+        case VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_af_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_STATE:
+        case VVCAM_ISP_CID_AF_MODE:
+        // case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD:
+        case VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_af_ctrl_ops = {
+    .s_ctrl = vvcam_isp_af_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_af_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_af_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    /* this control is related to afmv1_1, which is not supported in afv4 for now
+    {
+        // float array 9x
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {9, 0, 0, 0},
+    },
+    */
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_stable_tolerance",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_points_of_curve",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MAX_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_max_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MIN_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_min_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_up_hill_allowance",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_down_hill_allowance",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float (0,1023) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_conf_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 10229,
+        .def  = 1,
+    },
+    {
+        /* float [1,1023] */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_shift_threshold",
+        .step = 1,
+        .min  = 10,
+        .max  = 10230,
+        .def  = 10,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_stable_cnt_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 10,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float [1,64] */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_unlock_threshold",
+        .step = 1,
+        .min  = 100,
+        .max  = 6400,
+        .def  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_defocus_frame_num",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_loss_conf_frame_num",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_acc_focus_step",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_acc_focus_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_ROI_INDEX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_roi_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 48,
+        .dims = {1},
+    },
+};
+
+int vvcam_isp_af_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_af_ctrls);
+}
+
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_af_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_af_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp af ctrl %s failed %d.\n",
+                vvcam_isp_af_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4/vvcam_isp_af.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4/vvcam_isp_af.h
new file mode 100644
index 000000000000..5fab28aed574
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4/vvcam_isp_af.h
@@ -0,0 +1,97 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AF_H__
+#define __VVCAM_ISP_AF_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AF_ENABLE             (VVCAM_ISP_CID_AF_BASE + 0x0000)
+#define VVCAM_ISP_CID_AF_RESET              (VVCAM_ISP_CID_AF_BASE + 0x0001)
+#define VVCAM_ISP_CID_AF_STATE              (VVCAM_ISP_CID_AF_BASE + 0x0002)
+#define VVCAM_ISP_CID_AF_MODE               (VVCAM_ISP_CID_AF_BASE + 0x0003)
+#define VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT    (VVCAM_ISP_CID_AF_BASE + 0x0004)
+#define VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE (VVCAM_ISP_CID_AF_BASE + 0x0005)
+#define VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE  (VVCAM_ISP_CID_AF_BASE + 0x0006)
+#define VVCAM_ISP_CID_CDAF_MAX_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0007)
+#define VVCAM_ISP_CID_CDAF_MIN_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0008)
+#define VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD (VVCAM_ISP_CID_AF_BASE + 0x0009)
+#define VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000A)
+#define VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000B)
+#define VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000C)
+#define VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000D)
+#define VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000E)
+#define VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000F)
+#define VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0010)
+#define VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0011)
+#define VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP  (VVCAM_ISP_CID_AF_BASE + 0x0012)
+#define VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0013)
+#define VVCAM_ISP_CID_PDAF_ROI_INDEX        (VVCAM_ISP_CID_AF_BASE + 0x0014)
+
+#ifdef __KERNEL__
+int vvcam_isp_af_ctrl_count(void);
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_1_1/vvcam_isp_af.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_1_1/vvcam_isp_af.c
new file mode 100644
index 000000000000..8c1e34ad914b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_1_1/vvcam_isp_af.c
@@ -0,0 +1,435 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_af.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_af_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_RESET:
+        case VVCAM_ISP_CID_AF_MODE:
+        case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD:
+        case VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+        case VVCAM_ISP_CID_AFMV11_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AFMV11_ROI:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_af_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_STATE:
+        case VVCAM_ISP_CID_AF_MODE:
+        case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD:
+        case VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+        case VVCAM_ISP_CID_AFMV11_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AFMV11_ROI:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_af_ctrl_ops = {
+    .s_ctrl = vvcam_isp_af_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_af_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_af_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float array 9x */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {9, 0, 0, 0},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_stable_tolerance",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_points_of_curve",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MAX_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_max_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MIN_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_min_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_up_hill_allowance",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_down_hill_allowance",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float (0,1023) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_conf_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 10229,
+        .def  = 1,
+    },
+    {
+        /* float [1,1023] */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_shift_threshold",
+        .step = 1,
+        .min  = 10,
+        .max  = 10230,
+        .def  = 10,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_stable_cnt_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 10,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float [1,64] */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_unlock_threshold",
+        .step = 1,
+        .min  = 100,
+        .max  = 6400,
+        .def  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_defocus_frame_num",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_loss_conf_frame_num",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_acc_focus_step",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_acc_focus_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_ROI_INDEX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_roi_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 48,
+        .dims = {1},
+    },
+    {
+        /* float 25x array 0~1 */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFMV11_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afmv11_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFMV11_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afmv11_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    }
+};
+
+int vvcam_isp_af_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_af_ctrls);
+}
+
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_af_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_af_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp af ctrl %s failed %d.\n",
+                vvcam_isp_af_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_1_1/vvcam_isp_af.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_1_1/vvcam_isp_af.h
new file mode 100644
index 000000000000..5cebe09adc86
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_1_1/vvcam_isp_af.h
@@ -0,0 +1,99 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AF_H__
+#define __VVCAM_ISP_AF_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AF_ENABLE             (VVCAM_ISP_CID_AF_BASE + 0x0000)
+#define VVCAM_ISP_CID_AF_RESET              (VVCAM_ISP_CID_AF_BASE + 0x0001)
+#define VVCAM_ISP_CID_AF_STATE              (VVCAM_ISP_CID_AF_BASE + 0x0002)
+#define VVCAM_ISP_CID_AF_MODE               (VVCAM_ISP_CID_AF_BASE + 0x0003)
+#define VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT    (VVCAM_ISP_CID_AF_BASE + 0x0004)
+#define VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE (VVCAM_ISP_CID_AF_BASE + 0x0005)
+#define VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE  (VVCAM_ISP_CID_AF_BASE + 0x0006)
+#define VVCAM_ISP_CID_CDAF_MAX_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0007)
+#define VVCAM_ISP_CID_CDAF_MIN_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0008)
+#define VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD (VVCAM_ISP_CID_AF_BASE + 0x0009)
+#define VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000A)
+#define VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000B)
+#define VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000C)
+#define VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000D)
+#define VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000E)
+#define VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000F)
+#define VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0010)
+#define VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0011)
+#define VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP  (VVCAM_ISP_CID_AF_BASE + 0x0012)
+#define VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0013)
+#define VVCAM_ISP_CID_PDAF_ROI_INDEX        (VVCAM_ISP_CID_AF_BASE + 0x0014)
+#define VVCAM_ISP_CID_AFMV11_ROI_WEIGHT     (VVCAM_ISP_CID_AF_BASE + 0x0015)
+#define VVCAM_ISP_CID_AFMV11_ROI            (VVCAM_ISP_CID_AF_BASE + 0x0016)
+
+#ifdef __KERNEL__
+int vvcam_isp_af_ctrl_count(void);
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_3/vvcam_isp_af.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_3/vvcam_isp_af.c
new file mode 100644
index 000000000000..e09663eed7b8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_3/vvcam_isp_af.c
@@ -0,0 +1,424 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_af.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_af_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_RESET:
+        case VVCAM_ISP_CID_AF_MODE:
+        // case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD:
+        case VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+        case VVCAM_ISP_CID_AFMV3_ROI_WEIGHT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_af_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AF_ENABLE:
+        case VVCAM_ISP_CID_AF_STATE:
+        case VVCAM_ISP_CID_AF_MODE:
+        // case VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT:
+        case VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE:
+        case VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE:
+        case VVCAM_ISP_CID_CDAF_MAX_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MIN_FOCAL:
+        case VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD:
+        case VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE:
+        case VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX:
+        case VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD:
+        case VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP:
+        case VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE:
+        case VVCAM_ISP_CID_PDAF_ROI_INDEX:
+        case VVCAM_ISP_CID_AFMV3_ROI_WEIGHT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_af_ctrl_ops = {
+    .s_ctrl = vvcam_isp_af_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_af_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_af_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_af_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    /* this control is related to afmv1_1, which is not supported in afv4 for now
+    {
+        // float array 9x
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_window_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {9, 0, 0, 0},
+    },
+    */
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_stable_tolerance",
+        .step = 1,
+        .min  = 1,
+        .max  = 99,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_points_of_curve",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MAX_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_max_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MIN_FOCAL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_min_focal",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        /* float (0,1) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_motion_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_up_hill_allowance",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cdaf_down_hill_allowance",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float (0,1023) */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_conf_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 10229,
+        .def  = 1,
+    },
+    {
+        /* float [1,1023] */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_shift_threshold",
+        .step = 1,
+        .min  = 10,
+        .max  = 10230,
+        .def  = 10,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_pd_stable_cnt_max",
+        .step = 1,
+        .min  = 1,
+        .max  = 10,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float [1,64] */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_unlock_threshold",
+        .step = 1,
+        .min  = 100,
+        .max  = 6400,
+        .def  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_defocus_frame_num",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_loss_conf_frame_num",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_acc_focus_step",
+        .step = 1,
+        .min  = 3,
+        .max  = 20,
+        .def  = 3,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pcdaf_acc_focus_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_ROI_INDEX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_roi_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 48,
+        .dims = {1},
+    },
+    {
+        /* float 225x array 0~255 */
+        .ops  = &vvcam_isp_af_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFMV3_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afmv3_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {225, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_af_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_af_ctrls);
+}
+
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_af_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_af_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp af ctrl %s failed %d.\n",
+                vvcam_isp_af_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_3/vvcam_isp_af.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_3/vvcam_isp_af.h
new file mode 100644
index 000000000000..56143c0985ce
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/af/af_v4_3/vvcam_isp_af.h
@@ -0,0 +1,98 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AF_H__
+#define __VVCAM_ISP_AF_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AF_ENABLE             (VVCAM_ISP_CID_AF_BASE + 0x0000)
+#define VVCAM_ISP_CID_AF_RESET              (VVCAM_ISP_CID_AF_BASE + 0x0001)
+#define VVCAM_ISP_CID_AF_STATE              (VVCAM_ISP_CID_AF_BASE + 0x0002)
+#define VVCAM_ISP_CID_AF_MODE               (VVCAM_ISP_CID_AF_BASE + 0x0003)
+#define VVCAM_ISP_CID_CDAF_WINDOW_WEIGHT    (VVCAM_ISP_CID_AF_BASE + 0x0004)
+#define VVCAM_ISP_CID_CDAF_STABLE_TOLERENCE (VVCAM_ISP_CID_AF_BASE + 0x0005)
+#define VVCAM_ISP_CID_CDAF_POINTS_OF_CURVE  (VVCAM_ISP_CID_AF_BASE + 0x0006)
+#define VVCAM_ISP_CID_CDAF_MAX_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0007)
+#define VVCAM_ISP_CID_CDAF_MIN_FOCAL        (VVCAM_ISP_CID_AF_BASE + 0x0008)
+#define VVCAM_ISP_CID_CDAF_MOTION_THRESHOLD (VVCAM_ISP_CID_AF_BASE + 0x0009)
+#define VVCAM_ISP_CID_CDAF_UP_HILL_ALLOWANCE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000A)
+#define VVCAM_ISP_CID_CDAF_DOWN_HILL_ALLOWANCE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000B)
+#define VVCAM_ISP_CID_PDAF_PD_CONF_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000C)
+#define VVCAM_ISP_CID_PDAF_PD_SHIFT_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000D)
+#define VVCAM_ISP_CID_PDAF_PD_STABLE_CNT_MAX \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000E)
+#define VVCAM_ISP_CID_PDAF_UNLOCK_THRESHOLD \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x000F)
+#define VVCAM_ISP_CID_PCDAF_DEFOCUS_FRAME_NUM \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0010)
+#define VVCAM_ISP_CID_PCDAF_LOSS_CONF_FRAME_NUM \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0011)
+#define VVCAM_ISP_CID_PCDAF_ACC_FOCUS_STEP  (VVCAM_ISP_CID_AF_BASE + 0x0012)
+#define VVCAM_ISP_CID_PCDAF_ACC_FOCUS_ENABLE \
+                                            (VVCAM_ISP_CID_AF_BASE + 0x0013)
+#define VVCAM_ISP_CID_PDAF_ROI_INDEX        (VVCAM_ISP_CID_AF_BASE + 0x0014)
+#define VVCAM_ISP_CID_AFMV3_ROI_WEIGHT      (VVCAM_ISP_CID_AF_BASE + 0x0015)
+
+#ifdef __KERNEL__
+int vvcam_isp_af_ctrl_count(void);
+int vvcam_isp_af_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1/vvcam_isp_afm.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1/vvcam_isp_afm.c
new file mode 100644
index 000000000000..b2e1aedff62c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1/vvcam_isp_afm.c
@@ -0,0 +1,327 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_afm.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_afm_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AFM_ENABLE:
+        case VVCAM_ISP_CID_AFM_RESET:
+        case VVCAM_ISP_CID_AFM_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_WIN_C_PIXEL_CNT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_afm_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AFM_ENABLE:
+        case VVCAM_ISP_CID_AFM_RESET:
+        case VVCAM_ISP_CID_AFM_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_WIN_C_PIXEL_CNT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_afm_ctrl_ops = {
+    .s_ctrl = vvcam_isp_afm_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_afm_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_afm_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_MEASURE_WINDOW_A,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_measure_window_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_MEASURE_WINDOW_B,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_measure_window_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_MEASURE_WINDOW_C,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_measure_window_c",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 0~65535 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_A_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_a_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_B_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_b_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_C_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_c_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_A_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_a_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_B_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_b_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_C_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_c_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_A_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_a_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_B_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_b_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WIN_C_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_window_c_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+
+};
+
+int vvcam_isp_afm_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_afm_ctrls);
+}
+
+int vvcam_isp_afm_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_afm_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_afm_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp afm ctrl %s failed %d.\n",
+                vvcam_isp_afm_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1/vvcam_isp_afm.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1/vvcam_isp_afm.h
new file mode 100644
index 000000000000..cd35cb24762f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1/vvcam_isp_afm.h
@@ -0,0 +1,82 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AFM_H__
+#define __VVCAM_ISP_AFM_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AFM_ENABLE            (VVCAM_ISP_CID_AFM_BASE + 0x0000)
+#define VVCAM_ISP_CID_AFM_RESET             (VVCAM_ISP_CID_AFM_BASE + 0x0001)
+#define VVCAM_ISP_CID_AFM_MEASURE_WINDOW_A  (VVCAM_ISP_CID_AFM_BASE + 0x0002)
+#define VVCAM_ISP_CID_AFM_MEASURE_WINDOW_B  (VVCAM_ISP_CID_AFM_BASE + 0x0003)
+#define VVCAM_ISP_CID_AFM_MEASURE_WINDOW_C  (VVCAM_ISP_CID_AFM_BASE + 0x0004)
+#define VVCAM_ISP_CID_AFM_THRESHOLD         (VVCAM_ISP_CID_AFM_BASE + 0x0005)
+#define VVCAM_ISP_CID_AFM_WIN_A_SHARPNESS   (VVCAM_ISP_CID_AFM_BASE + 0x0006)
+#define VVCAM_ISP_CID_AFM_WIN_B_SHARPNESS   (VVCAM_ISP_CID_AFM_BASE + 0x0007)
+#define VVCAM_ISP_CID_AFM_WIN_C_SHARPNESS   (VVCAM_ISP_CID_AFM_BASE + 0x0008)
+#define VVCAM_ISP_CID_AFM_WIN_A_LUMINANCE   (VVCAM_ISP_CID_AFM_BASE + 0x0009)
+#define VVCAM_ISP_CID_AFM_WIN_B_LUMINANCE   (VVCAM_ISP_CID_AFM_BASE + 0x000A)
+#define VVCAM_ISP_CID_AFM_WIN_C_LUMINANCE   (VVCAM_ISP_CID_AFM_BASE + 0x000B)
+#define VVCAM_ISP_CID_AFM_WIN_A_PIXEL_CNT   (VVCAM_ISP_CID_AFM_BASE + 0x000C)
+#define VVCAM_ISP_CID_AFM_WIN_B_PIXEL_CNT   (VVCAM_ISP_CID_AFM_BASE + 0x000D)
+#define VVCAM_ISP_CID_AFM_WIN_C_PIXEL_CNT   (VVCAM_ISP_CID_AFM_BASE + 0x000E)
+
+#ifdef __KERNEL__
+int vvcam_isp_afm_ctrl_count(void);
+int vvcam_isp_afm_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.c
new file mode 100644
index 000000000000..126349efbec8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.c
@@ -0,0 +1,711 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_afm.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_afm_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AFM_INST0_ENABLE:
+        case VVCAM_ISP_CID_AFM_INST1_ENABLE:
+        case VVCAM_ISP_CID_AFM_INST2_ENABLE:
+        case VVCAM_ISP_CID_AFM_RESET:
+        case VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_INST0_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_INST1_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_INST2_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_C_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_C_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_C_PIXEL_CNT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_afm_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AFM_INST0_ENABLE:
+        case VVCAM_ISP_CID_AFM_INST1_ENABLE:
+        case VVCAM_ISP_CID_AFM_INST2_ENABLE:
+        case VVCAM_ISP_CID_AFM_WINDOW_ENABLE:
+        case VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_A:
+        case VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_B:
+        case VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_C:
+        case VVCAM_ISP_CID_AFM_INST0_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_INST1_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_INST2_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST0_WIN_C_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST1_WIN_C_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_A_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_B_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_C_SHARPNESS:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_A_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_B_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_C_LUMINANCE:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_A_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_B_PIXEL_CNT:
+        case VVCAM_ISP_CID_AFM_INST2_WIN_C_PIXEL_CNT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_afm_ctrl_ops = {
+    .s_ctrl = vvcam_isp_afm_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_afm_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_afm_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /*  bool 4x4 array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_WINDOW_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_win_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {4, 4, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_A,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_measure_win_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_B,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_measure_win_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_C,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_measure_win_c",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_A,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_measure_win_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_B,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_measure_win_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_C,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_measure_win_c",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_A,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_measure_win_a",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_B,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_measure_win_b",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_C,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_measure_win_c",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 0~255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .def  = 0,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* 0~255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .def  = 0,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* 0~255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .def  = 0,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_A_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_a_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_B_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_b_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_C_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_c_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_A_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_a_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_B_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_b_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_C_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_c_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_A_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_a_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_B_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_b_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST0_WIN_C_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst0_win_c_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_A_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_a_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_B_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_b_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_C_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_c_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_A_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_a_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_B_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_b_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_C_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_c_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_A_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_a_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_B_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_b_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST1_WIN_C_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst1_win_c_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_A_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_a_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_B_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_b_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_C_SHARPNESS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_c_sharpness",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_A_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_a_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_B_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_b_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_C_LUMINANCE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_c_luminance",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_A_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_a_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_B_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_b_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_INST2_WIN_C_PIXEL_CNT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_inst2_win_c_pixel_cnt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_afm_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_afm_ctrls);
+}
+
+int vvcam_isp_afm_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_afm_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_afm_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp afm ctrl %s failed %d.\n",
+                vvcam_isp_afm_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.h
new file mode 100644
index 000000000000..3d0e705fa8eb
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v1_1/vvcam_isp_afm.h
@@ -0,0 +1,148 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AFM_H__
+#define __VVCAM_ISP_AFM_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AFM_INST0_ENABLE      (VVCAM_ISP_CID_AFM_BASE + 0x0000)
+#define VVCAM_ISP_CID_AFM_INST1_ENABLE      (VVCAM_ISP_CID_AFM_BASE + 0x0001)
+#define VVCAM_ISP_CID_AFM_INST2_ENABLE      (VVCAM_ISP_CID_AFM_BASE + 0x0002)
+#define VVCAM_ISP_CID_AFM_WINDOW_ENABLE     (VVCAM_ISP_CID_AFM_BASE + 0x0003)
+#define VVCAM_ISP_CID_AFM_RESET             (VVCAM_ISP_CID_AFM_BASE + 0x0004)
+#define VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_A \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0005)
+#define VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_B \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0006)
+#define VVCAM_ISP_CID_AFM_INST0_MEASURE_WINDOW_C \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0007)
+#define VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_A \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0008)
+#define VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_B \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0009)
+#define VVCAM_ISP_CID_AFM_INST1_MEASURE_WINDOW_C \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000A)
+#define VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_A \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000B)
+#define VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_B \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000C)
+#define VVCAM_ISP_CID_AFM_INST2_MEASURE_WINDOW_C \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000D)
+#define VVCAM_ISP_CID_AFM_INST0_THRESHOLD   (VVCAM_ISP_CID_AFM_BASE + 0x000E)
+#define VVCAM_ISP_CID_AFM_INST1_THRESHOLD   (VVCAM_ISP_CID_AFM_BASE + 0x000F)
+#define VVCAM_ISP_CID_AFM_INST2_THRESHOLD   (VVCAM_ISP_CID_AFM_BASE + 0x0010)
+
+#define VVCAM_ISP_CID_AFM_INST0_WIN_A_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0011)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_B_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0012)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_C_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0013)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_A_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0014)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_B_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0015)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_C_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0016)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_A_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0017)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_B_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0018)
+#define VVCAM_ISP_CID_AFM_INST0_WIN_C_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0019)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_A_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001A)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_B_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001B)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_C_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001C)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_A_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001D)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_B_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001E)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_C_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001F)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_A_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0020)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_B_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0021)
+#define VVCAM_ISP_CID_AFM_INST1_WIN_C_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0022)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_A_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0023)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_B_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0024)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_C_SHARPNESS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0025)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_A_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0026)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_B_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0027)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_C_LUMINANCE \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0028)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_A_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0029)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_B_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x002A)
+#define VVCAM_ISP_CID_AFM_INST2_WIN_C_PIXEL_CNT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x002B)
+
+#ifdef __KERNEL__
+int vvcam_isp_afm_ctrl_count(void);
+int vvcam_isp_afm_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v3/vvcam_isp_afm.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v3/vvcam_isp_afm.c
new file mode 100644
index 000000000000..e607b9e59dda
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v3/vvcam_isp_afm.c
@@ -0,0 +1,551 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_afm.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_afm_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AFM_ENABLE:
+        case VVCAM_ISP_CID_AFM_GAMMA_ENABLE:
+        case VVCAM_ISP_CID_AFM_RESET:
+        case VVCAM_ISP_CID_AFM_BLS:
+        case VVCAM_ISP_CID_AFM_HIST_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_GAMMA_VAL:
+        case VVCAM_ISP_CID_AFM_HW_ROI:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_X:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_X:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_X:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_X:
+        case VVCAM_ISP_CID_AFM_CORING_IIR_HIGH_PASS:
+        case VVCAM_ISP_CID_AFM_CORING_IIR_LOW_PASS:
+        case VVCAM_ISP_CID_AFM_CORING_FIR_HIGH_PASS:
+        case VVCAM_ISP_CID_AFM_CORING_FIR_LOW_PASS:
+        case VVCAM_ISP_CID_AFM_IIR_DATA_SHIFT:
+        case VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_IIR_LOW_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_COEFF:
+        case VVCAM_ISP_CID_AFM_IIR_LOW_PASS_COEFF:
+        case VVCAM_ISP_CID_AFM_FIR_DATA_SHIFT:
+        case VVCAM_ISP_CID_AFM_FIR_HIGH_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_FIR_LOW_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_SHARPNESS_HIGH_PASS:
+        case VVCAM_ISP_CID_AFM_SHARPNESS_LOW_PASS:
+        case VVCAM_ISP_CID_AFM_HIST_LOW_DATA:
+        case VVCAM_ISP_CID_AFM_HIST_HIGH_DATA:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_afm_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AFM_ENABLE:
+        // case VVCAM_ISP_CID_AFM_GAMMA_ENABLE:
+        case VVCAM_ISP_CID_AFM_RESET:
+        case VVCAM_ISP_CID_AFM_BLS:
+        case VVCAM_ISP_CID_AFM_HIST_THRESHOLD:
+        case VVCAM_ISP_CID_AFM_GAMMA_VAL:
+        case VVCAM_ISP_CID_AFM_HW_ROI:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_X:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_X:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_X:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_Y:
+        case VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_X:
+        case VVCAM_ISP_CID_AFM_CORING_IIR_HIGH_PASS:
+        case VVCAM_ISP_CID_AFM_CORING_IIR_LOW_PASS:
+        case VVCAM_ISP_CID_AFM_CORING_FIR_HIGH_PASS:
+        case VVCAM_ISP_CID_AFM_CORING_FIR_LOW_PASS:
+        case VVCAM_ISP_CID_AFM_IIR_DATA_SHIFT:
+        case VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_IIR_LOW_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_COEFF:
+        case VVCAM_ISP_CID_AFM_IIR_LOW_PASS_COEFF:
+        case VVCAM_ISP_CID_AFM_FIR_DATA_SHIFT:
+        case VVCAM_ISP_CID_AFM_FIR_HIGH_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_FIR_LOW_PASS_WEIGHT:
+        case VVCAM_ISP_CID_AFM_SHARPNESS_HIGH_PASS:
+        case VVCAM_ISP_CID_AFM_SHARPNESS_LOW_PASS:
+        case VVCAM_ISP_CID_AFM_HIST_LOW_DATA:
+        case VVCAM_ISP_CID_AFM_HIST_HIGH_DATA:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_afm_ctrl_ops = {
+    .s_ctrl = vvcam_isp_afm_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_afm_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_afm_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_GAMMA_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_gamma_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_BLS,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_bls",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_HIST_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_hist_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float (0,4] */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_GAMMA_VAL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_gamma_val",
+        .step = 1,
+        .min  = 1,
+        .max  = 400,
+        .def  = 1,
+    },
+    {
+        /* uint16_t 4x array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_HW_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_hw_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_iir_high_pass_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_X,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_iir_high_pass_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_iir_low_pass_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_X,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_iir_low_pass_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_fir_high_pass_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_X,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_fir_high_pass_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_fir_low_pass_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_X,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_ldg_fir_low_pass_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_CORING_IIR_HIGH_PASS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_coring_iir_high_pass",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_CORING_IIR_LOW_PASS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_coring_iir_low_pass",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_CORING_FIR_HIGH_PASS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_coring_fir_high_pass",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 255 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_CORING_FIR_LOW_PASS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_coring_fir_low_pass",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 15 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_IIR_DATA_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_iir_data_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_iir_high_pass_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_IIR_LOW_PASS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_iir_low_pass_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float 3*4 -4095/256 ~ 4095/256 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_COEFF,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_iir_high_pass_coeff",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 4, 0, 0},
+    },
+    {
+        /* float 3*4 -4095/256 ~ 4095/256 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_IIR_LOW_PASS_COEFF,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_iir_low_pass_coeff",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 4, 0, 0},
+    },
+    {
+        /* uint8_t 0 ~ 15 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_FIR_DATA_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_fir_data_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_FIR_HIGH_PASS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_fir_high_pass_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float 0 ~ 15.9375 */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_FIR_LOW_PASS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_fir_low_pass_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* uint32_t 15x15 array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_SHARPNESS_HIGH_PASS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_sharpness_high_pass",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {225, 0, 0, 0},
+    },
+    {
+        /* uint32_t 15x15 array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_SHARPNESS_LOW_PASS,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_sharpness_low_pass",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {225, 0, 0, 0},
+    },
+    {
+        /* uint32_t 15x15 array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_HIST_LOW_DATA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_hist_low_data",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {225, 0, 0, 0},
+    },
+    {
+        /* uint32_t 15x15 array */
+        .ops  = &vvcam_isp_afm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AFM_HIST_HIGH_DATA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_afm_hist_high_data",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {225, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_afm_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_afm_ctrls);
+}
+
+int vvcam_isp_afm_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_afm_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_afm_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp afm ctrl %s failed %d.\n",
+                vvcam_isp_afm_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v3/vvcam_isp_afm.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v3/vvcam_isp_afm.h
new file mode 100644
index 000000000000..1dcca99ef276
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/afm/afm_v3/vvcam_isp_afm.h
@@ -0,0 +1,119 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AFM_H__
+#define __VVCAM_ISP_AFM_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AFM_ENABLE            (VVCAM_ISP_CID_AFM_BASE + 0x0000)
+#define VVCAM_ISP_CID_AFM_GAMMA_ENABLE      (VVCAM_ISP_CID_AFM_BASE + 0x0001)
+#define VVCAM_ISP_CID_AFM_RESET             (VVCAM_ISP_CID_AFM_BASE + 0x0002)
+#define VVCAM_ISP_CID_AFM_BLS               (VVCAM_ISP_CID_AFM_BASE + 0x0003)
+#define VVCAM_ISP_CID_AFM_HIST_THRESHOLD    (VVCAM_ISP_CID_AFM_BASE + 0x0004)
+#define VVCAM_ISP_CID_AFM_GAMMA_VAL         (VVCAM_ISP_CID_AFM_BASE + 0x0005)
+#define VVCAM_ISP_CID_AFM_HW_ROI            (VVCAM_ISP_CID_AFM_BASE + 0x0006)
+#define VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_Y \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0007)
+#define VVCAM_ISP_CID_AFM_LDG_IIR_HIGH_PASS_X \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0008)
+#define VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_Y \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0009)
+#define VVCAM_ISP_CID_AFM_LDG_IIR_LOW_PASS_X \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000A)
+#define VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_Y \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000B)
+#define VVCAM_ISP_CID_AFM_LDG_FIR_HIGH_PASS_X \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000C)
+#define VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_Y \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000D)
+#define VVCAM_ISP_CID_AFM_LDG_FIR_LOW_PASS_X \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000E)
+#define VVCAM_ISP_CID_AFM_CORING_IIR_HIGH_PASS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x000F)
+#define VVCAM_ISP_CID_AFM_CORING_IIR_LOW_PASS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0010)
+#define VVCAM_ISP_CID_AFM_CORING_FIR_HIGH_PASS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0011)
+#define VVCAM_ISP_CID_AFM_CORING_FIR_LOW_PASS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0012)
+#define VVCAM_ISP_CID_AFM_IIR_DATA_SHIFT    (VVCAM_ISP_CID_AFM_BASE + 0x0013)
+#define VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_WEIGHT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0014)
+#define VVCAM_ISP_CID_AFM_IIR_LOW_PASS_WEIGHT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0015)
+#define VVCAM_ISP_CID_AFM_IIR_HIGH_PASS_COEFF \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0016)
+#define VVCAM_ISP_CID_AFM_IIR_LOW_PASS_COEFF \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0017)
+#define VVCAM_ISP_CID_AFM_FIR_DATA_SHIFT    (VVCAM_ISP_CID_AFM_BASE + 0x0018)
+#define VVCAM_ISP_CID_AFM_FIR_HIGH_PASS_WEIGHT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x0019)
+#define VVCAM_ISP_CID_AFM_FIR_LOW_PASS_WEIGHT \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001A)
+#define VVCAM_ISP_CID_AFM_SHARPNESS_HIGH_PASS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001B)
+#define VVCAM_ISP_CID_AFM_SHARPNESS_LOW_PASS \
+                                            (VVCAM_ISP_CID_AFM_BASE + 0x001C)
+#define VVCAM_ISP_CID_AFM_HIST_LOW_DATA     (VVCAM_ISP_CID_AFM_BASE + 0x001D)
+#define VVCAM_ISP_CID_AFM_HIST_HIGH_DATA    (VVCAM_ISP_CID_AFM_BASE + 0x001E)
+
+#ifdef __KERNEL__
+int vvcam_isp_afm_ctrl_count(void);
+int vvcam_isp_afm_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v3/vvcam_isp_awb.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v3/vvcam_isp_awb.c
new file mode 100644
index 000000000000..c3778a6be66a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v3/vvcam_isp_awb.c
@@ -0,0 +1,216 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_awb.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_awb_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AWB_ENABLE:
+        case VVCAM_ISP_CID_AWB_RESET:
+        case VVCAM_ISP_CID_AWB_STATE:
+        case VVCAM_ISP_CID_AWB_MODE:
+        case VVCAM_ISP_CID_AWB_USE_DAMPING:
+        case VVCAM_ISP_CID_AWB_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AWB_ROI:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_awb_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AWB_ENABLE:
+        case VVCAM_ISP_CID_AWB_STATE:
+        case VVCAM_ISP_CID_AWB_MODE:
+        case VVCAM_ISP_CID_AWB_USE_DAMPING:
+        case VVCAM_ISP_CID_AWB_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AWB_ROI:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_awb_ctrl_ops = {
+	.s_ctrl = vvcam_isp_awb_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_awb_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_awb_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_USE_DAMPING,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_use_damping",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 25x32bit 0~1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* int array 25x4*16bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+};
+
+int vvcam_isp_awb_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_awb_ctrls);
+}
+
+int vvcam_isp_awb_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_awb_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_awb_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp awb ctrl %s failed %d.\n",
+                vvcam_isp_awb_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v3/vvcam_isp_awb.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v3/vvcam_isp_awb.h
new file mode 100644
index 000000000000..f745665e852d
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v3/vvcam_isp_awb.h
@@ -0,0 +1,74 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AWB_H__
+#define __VVCAM_ISP_AWB_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AWB_ENABLE            (VVCAM_ISP_CID_AWB_BASE + 0x0000)
+#define VVCAM_ISP_CID_AWB_RESET             (VVCAM_ISP_CID_AWB_BASE + 0x0001)
+#define VVCAM_ISP_CID_AWB_STATE             (VVCAM_ISP_CID_AWB_BASE + 0x0002)
+#define VVCAM_ISP_CID_AWB_MODE              (VVCAM_ISP_CID_AWB_BASE + 0x0003)
+#define VVCAM_ISP_CID_AWB_USE_DAMPING       (VVCAM_ISP_CID_AWB_BASE + 0x0004)
+#define VVCAM_ISP_CID_AWB_ROI_WEIGHT        (VVCAM_ISP_CID_AWB_BASE + 0x0005)
+#define VVCAM_ISP_CID_AWB_ROI               (VVCAM_ISP_CID_AWB_BASE + 0x0006)
+
+#ifdef __KERNEL__
+int vvcam_isp_awb_ctrl_count(void);
+int vvcam_isp_awb_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4/vvcam_isp_awb.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4/vvcam_isp_awb.c
new file mode 100644
index 000000000000..1796dc1d9b20
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4/vvcam_isp_awb.c
@@ -0,0 +1,713 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_awb.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_awb_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AWB_ENABLE:
+        case VVCAM_ISP_CID_AWB_RESET:
+        case VVCAM_ISP_CID_AWB_STATE:
+        case VVCAM_ISP_CID_AWB_MODE:
+        case VVCAM_ISP_CID_AWB_USE_DAMPING:
+        case VVCAM_ISP_CID_AWB_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AWB_ROI:
+        case VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_A:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_D65:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_R:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_B:
+        case VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE:
+        case VVCAM_ISP_CID_AWB_LOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_UNLOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_awb_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AWB_ENABLE:
+        case VVCAM_ISP_CID_AWB_STATE:
+        case VVCAM_ISP_CID_AWB_MODE:
+        case VVCAM_ISP_CID_AWB_USE_DAMPING:
+        case VVCAM_ISP_CID_AWB_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AWB_ROI:
+        case VVCAM_ISP_CID_AWB_COLOR_TEMP_WEIGHT:
+        case VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_A:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_D65:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_R:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_B:
+        case VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE:
+        case VVCAM_ISP_CID_AWB_LOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_UNLOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT:
+        case VVCAM_ISP_CID_AWB_RESULT_CCT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_awb_ctrl_ops = {
+	.s_ctrl = vvcam_isp_awb_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_awb_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_awb_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_USE_DAMPING,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_use_damping",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 25x32bit 0~1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* int array 25x4*16bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        /* float array 7x32bit [0, 1] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_COLOR_TEMP_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_color_temp_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {7},
+    },
+    {
+        /* float array 10x32bit [0, 3] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confidence_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10},
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_a",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_cwf",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_D65,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_d65",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,3) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_thresh",
+        .step = 1,
+        .min  = 1,
+        .max  = 29999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,3) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_thresh",
+        .step = 1,
+        .min  = 1,
+        .max  = 29999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,3) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_thresh",
+        .step = 1,
+        .min  = 1,
+        .max  = 29999,
+        .def  = 1,
+    },
+    {
+        /* bool array 10x32bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_light_weight_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {10},
+    },
+    {
+        /* float array 10x18x32bit [0, 17] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_light_weight_bright",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* float array 10x18x32bit [0, 1] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_light_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* bool array 10x32bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {10},
+    },
+    {
+        /* float array 10x18x32bit [0, 17] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_bright",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* uint16_t array 10x18x16bit (0, 512) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_R,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_r",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* uint16_t array 10x18x16bit (0, 512) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_B,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_b",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_use_manu_damp_coeff",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0, 1) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_manu_damp_coeff",
+        .step = 1,
+        .min  = 1,
+        .max  = 9999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_performance_opti_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float [0, 0.2] ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LOCK_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_lock_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float [0, 0.2] ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_UNLOCK_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_unlock_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* bool array 25x32bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_post_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit (0, 4) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_rg",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit (0, 4) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_bg",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit (0, 3) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit [0, 1] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_RESULT_CCT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_result_cct",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+};
+
+int vvcam_isp_awb_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_awb_ctrls);
+}
+
+int vvcam_isp_awb_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_awb_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_awb_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp awb ctrl %s failed %d.\n",
+                vvcam_isp_awb_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4/vvcam_isp_awb.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4/vvcam_isp_awb.h
new file mode 100644
index 000000000000..7aef6aa63a60
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4/vvcam_isp_awb.h
@@ -0,0 +1,111 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AWB_H__
+#define __VVCAM_ISP_AWB_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AWB_ENABLE                    (VVCAM_ISP_CID_AWB_BASE + 0x0000)
+#define VVCAM_ISP_CID_AWB_RESET                     (VVCAM_ISP_CID_AWB_BASE + 0x0001)
+#define VVCAM_ISP_CID_AWB_STATE                     (VVCAM_ISP_CID_AWB_BASE + 0x0002)
+#define VVCAM_ISP_CID_AWB_MODE                      (VVCAM_ISP_CID_AWB_BASE + 0x0003)
+#define VVCAM_ISP_CID_AWB_USE_DAMPING               (VVCAM_ISP_CID_AWB_BASE + 0x0004)
+#define VVCAM_ISP_CID_AWB_ROI_WEIGHT                (VVCAM_ISP_CID_AWB_BASE + 0x0005)
+#define VVCAM_ISP_CID_AWB_ROI                       (VVCAM_ISP_CID_AWB_BASE + 0x0006)
+#define VVCAM_ISP_CID_AWB_COLOR_TEMP_WEIGHT         (VVCAM_ISP_CID_AWB_BASE + 0x0007)
+#define VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH         (VVCAM_ISP_CID_AWB_BASE + 0x0008)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE        (VVCAM_ISP_CID_AWB_BASE + 0x0009)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_A             (VVCAM_ISP_CID_AWB_BASE + 0x000A)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF           (VVCAM_ISP_CID_AWB_BASE + 0x000B)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_D65           (VVCAM_ISP_CID_AWB_BASE + 0x000C)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE       (VVCAM_ISP_CID_AWB_BASE + 0x000D)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG           (VVCAM_ISP_CID_AWB_BASE + 0x000E)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG           (VVCAM_ISP_CID_AWB_BASE + 0x000F)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH       (VVCAM_ISP_CID_AWB_BASE + 0x0010)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE      (VVCAM_ISP_CID_AWB_BASE + 0x0011)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG          (VVCAM_ISP_CID_AWB_BASE + 0x0012)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG          (VVCAM_ISP_CID_AWB_BASE + 0x0013)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH      (VVCAM_ISP_CID_AWB_BASE + 0x0014)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE       (VVCAM_ISP_CID_AWB_BASE + 0x0015)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_RG           (VVCAM_ISP_CID_AWB_BASE + 0x0016)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_BG           (VVCAM_ISP_CID_AWB_BASE + 0x0017)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH       (VVCAM_ISP_CID_AWB_BASE + 0x0018)
+#define VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE       (VVCAM_ISP_CID_AWB_BASE + 0x0019)
+#define VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT       (VVCAM_ISP_CID_AWB_BASE + 0x001A)
+#define VVCAM_ISP_CID_AWB_LIGHT_WEIGHT              (VVCAM_ISP_CID_AWB_BASE + 0x001B)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE        (VVCAM_ISP_CID_AWB_BASE + 0x001C)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT        (VVCAM_ISP_CID_AWB_BASE + 0x001D)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_R             (VVCAM_ISP_CID_AWB_BASE + 0x001E)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_B             (VVCAM_ISP_CID_AWB_BASE + 0x001F)
+#define VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF       (VVCAM_ISP_CID_AWB_BASE + 0x0020)
+#define VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF           (VVCAM_ISP_CID_AWB_BASE + 0x0021)
+#define VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE     (VVCAM_ISP_CID_AWB_BASE + 0x0022)
+#define VVCAM_ISP_CID_AWB_LOCK_THRESH               (VVCAM_ISP_CID_AWB_BASE + 0x0023)
+#define VVCAM_ISP_CID_AWB_UNLOCK_THRESH             (VVCAM_ISP_CID_AWB_BASE + 0x0024)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE    (VVCAM_ISP_CID_AWB_BASE + 0x0025)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN   (VVCAM_ISP_CID_AWB_BASE + 0x0026)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG        (VVCAM_ISP_CID_AWB_BASE + 0x0027)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG        (VVCAM_ISP_CID_AWB_BASE + 0x0028)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH    (VVCAM_ISP_CID_AWB_BASE + 0x0029)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT    (VVCAM_ISP_CID_AWB_BASE + 0x002A)
+#define VVCAM_ISP_CID_AWB_RESULT_CCT                (VVCAM_ISP_CID_AWB_BASE + 0x002B)
+
+#ifdef __KERNEL__
+int vvcam_isp_awb_ctrl_count(void);
+int vvcam_isp_awb_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.c
new file mode 100644
index 000000000000..a74c47dfc8b2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.c
@@ -0,0 +1,779 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_awb.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_awb_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AWB_ENABLE:
+        case VVCAM_ISP_CID_AWB_RESET:
+        case VVCAM_ISP_CID_AWB_STATE:
+        case VVCAM_ISP_CID_AWB_MODE:
+        case VVCAM_ISP_CID_AWB_USE_DAMPING:
+        case VVCAM_ISP_CID_AWB_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AWB_ROI:
+        case VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_A:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_D65:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_R:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_B:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_ENABLE:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_ROI_NUM:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_WEIGHT:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_RG:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_BG:
+        case VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE:
+        case VVCAM_ISP_CID_AWB_LOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_UNLOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_awb_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_AWB_ENABLE:
+        case VVCAM_ISP_CID_AWB_RESET:
+        case VVCAM_ISP_CID_AWB_STATE:
+        case VVCAM_ISP_CID_AWB_MODE:
+        case VVCAM_ISP_CID_AWB_USE_DAMPING:
+        case VVCAM_ISP_CID_AWB_ROI_WEIGHT:
+        case VVCAM_ISP_CID_AWB_ROI:
+        case VVCAM_ISP_CID_AWB_COLOR_TEMP_WEIGHT:
+        case VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_A:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF:
+        case VVCAM_ISP_CID_AWB_TEMP_PREFER_D65:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT:
+        case VVCAM_ISP_CID_AWB_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_R:
+        case VVCAM_ISP_CID_AWB_GRAY_PREFER_B:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_ENABLE:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_ROI_NUM:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_WEIGHT:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_RG:
+        case VVCAM_ISP_CID_AWB_FRONT_GROUND_BG:
+        case VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF:
+        case VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE:
+        case VVCAM_ISP_CID_AWB_LOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_UNLOCK_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH:
+        case VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT:
+        case VVCAM_ISP_CID_AWB_RESULT_CCT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_awb_ctrl_ops = {
+	.s_ctrl = vvcam_isp_awb_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_awb_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_awb_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_STATE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_state",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_USE_DAMPING,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_use_damping",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 25x32bit 0~1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ROI_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_roi_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* int array 25x4*16bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        /* float array 7x32bit [0, 1] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_COLOR_TEMP_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_color_temp_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {7},
+    },
+    {
+        /* float array 10x32bit [0, 3] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confidence_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10},
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_A,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_a",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_cwf",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_TEMP_PREFER_D65,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_temp_prefer_d65",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,3) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_cwf_thresh",
+        .step = 1,
+        .min  = 1,
+        .max  = 29999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,3) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_tl84_thresh",
+        .step = 1,
+        .min  = 1,
+        .max  = 29999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0,3) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_d65_thresh",
+        .step = 1,
+        .min  = 1,
+        .max  = 29999,
+        .def  = 1,
+    },
+    {
+        /* bool array 10x32bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_light_weight_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {10},
+    },
+    {
+        /* float array 10x18x32bit [0, 17] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_light_weight_bright",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* float array 10x18x32bit [0, 1] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_light_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* bool array 10x32bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {10},
+    },
+    {
+        /* float array 10x18x32bit [0, 17] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_bright",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* uint16_t array 10x18x16bit (0, 512) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_R,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_r",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        /* uint16_t array 10x18x16bit (0, 512) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_GRAY_PREFER_B,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_gray_prefer_b",
+        .step = 1,
+        .min  = 1,
+        .max  = 511,
+        .def  = 1,
+        .dims = {10, 18, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_front_face_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_FRONT_GROUND_ROI_NUM,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_front_roi_num",
+        .step = 1,
+        .min  = 0,
+        .max  = 25,
+    },
+    {
+        /* float [0, 1] ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_front_face_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float (0, 4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_FRONT_GROUND_RG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_front_rg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        /* float (0, 4) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_FRONT_GROUND_BG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_front_bg",
+        .step = 1,
+        .min  = 1,
+        .max  = 39999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_use_manu_damp_coeff",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0, 1) ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_manu_damp_coeff",
+        .step = 1,
+        .min  = 1,
+        .max  = 9999,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_performance_opti_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float [0, 0.2] ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_LOCK_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_lock_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float [0, 0.2] ratio 10000:1 */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_UNLOCK_THRESH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_unlock_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* bool array 25x32bit */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_post_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit (0, 4) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_rg",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit (0, 4) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_bg",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit (0, 3) */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_thresh",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        /* float array 25x32bit [0, 1] */
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_confound_custom_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {25},
+    },
+    {
+        .ops  = &vvcam_isp_awb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_AWB_RESULT_CCT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_awb_result_cct",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+};
+
+int vvcam_isp_awb_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_awb_ctrls);
+}
+
+int vvcam_isp_awb_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_awb_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_awb_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp awb ctrl %s failed %d.\n",
+                vvcam_isp_awb_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.h
new file mode 100644
index 000000000000..4024cd990a63
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/awb/awb_v4_1/vvcam_isp_awb.h
@@ -0,0 +1,116 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_AWB_H__
+#define __VVCAM_ISP_AWB_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_AWB_ENABLE                    (VVCAM_ISP_CID_AWB_BASE + 0x0000)
+#define VVCAM_ISP_CID_AWB_RESET                     (VVCAM_ISP_CID_AWB_BASE + 0x0001)
+#define VVCAM_ISP_CID_AWB_STATE                     (VVCAM_ISP_CID_AWB_BASE + 0x0002)
+#define VVCAM_ISP_CID_AWB_MODE                      (VVCAM_ISP_CID_AWB_BASE + 0x0003)
+#define VVCAM_ISP_CID_AWB_USE_DAMPING               (VVCAM_ISP_CID_AWB_BASE + 0x0004)
+#define VVCAM_ISP_CID_AWB_ROI_WEIGHT                (VVCAM_ISP_CID_AWB_BASE + 0x0005)
+#define VVCAM_ISP_CID_AWB_ROI                       (VVCAM_ISP_CID_AWB_BASE + 0x0006)
+#define VVCAM_ISP_CID_AWB_COLOR_TEMP_WEIGHT         (VVCAM_ISP_CID_AWB_BASE + 0x0007)
+#define VVCAM_ISP_CID_AWB_CONFIDENCE_THRESH         (VVCAM_ISP_CID_AWB_BASE + 0x0008)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_ENABLE        (VVCAM_ISP_CID_AWB_BASE + 0x0009)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_A             (VVCAM_ISP_CID_AWB_BASE + 0x000A)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_CWF           (VVCAM_ISP_CID_AWB_BASE + 0x000B)
+#define VVCAM_ISP_CID_AWB_TEMP_PREFER_D65           (VVCAM_ISP_CID_AWB_BASE + 0x000C)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_ENABLE       (VVCAM_ISP_CID_AWB_BASE + 0x000D)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_RG           (VVCAM_ISP_CID_AWB_BASE + 0x000E)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_BG           (VVCAM_ISP_CID_AWB_BASE + 0x000F)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CWF_THRESH       (VVCAM_ISP_CID_AWB_BASE + 0x0010)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_ENABLE      (VVCAM_ISP_CID_AWB_BASE + 0x0011)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_RG          (VVCAM_ISP_CID_AWB_BASE + 0x0012)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_BG          (VVCAM_ISP_CID_AWB_BASE + 0x0013)
+#define VVCAM_ISP_CID_AWB_CONFOUND_TL84_THRESH      (VVCAM_ISP_CID_AWB_BASE + 0x0014)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_ENABLE       (VVCAM_ISP_CID_AWB_BASE + 0x0015)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_RG           (VVCAM_ISP_CID_AWB_BASE + 0x0016)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_BG           (VVCAM_ISP_CID_AWB_BASE + 0x0017)
+#define VVCAM_ISP_CID_AWB_CONFOUND_D65_THRESH       (VVCAM_ISP_CID_AWB_BASE + 0x0018)
+#define VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_ENABLE       (VVCAM_ISP_CID_AWB_BASE + 0x0019)
+#define VVCAM_ISP_CID_AWB_LIGHT_WEIGHT_BRIGHT       (VVCAM_ISP_CID_AWB_BASE + 0x001A)
+#define VVCAM_ISP_CID_AWB_LIGHT_WEIGHT              (VVCAM_ISP_CID_AWB_BASE + 0x001B)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_ENABLE        (VVCAM_ISP_CID_AWB_BASE + 0x001C)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_BRIGHT        (VVCAM_ISP_CID_AWB_BASE + 0x001D)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_R             (VVCAM_ISP_CID_AWB_BASE + 0x001E)
+#define VVCAM_ISP_CID_AWB_GRAY_PREFER_B             (VVCAM_ISP_CID_AWB_BASE + 0x001F)
+#define VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_ENABLE  (VVCAM_ISP_CID_AWB_BASE + 0x0020)
+#define VVCAM_ISP_CID_AWB_FRONT_GROUND_ROI_NUM      (VVCAM_ISP_CID_AWB_BASE + 0x0021)
+#define VVCAM_ISP_CID_AWB_FRONT_GROUND_FACE_WEIGHT  (VVCAM_ISP_CID_AWB_BASE + 0x0022)
+#define VVCAM_ISP_CID_AWB_FRONT_GROUND_RG           (VVCAM_ISP_CID_AWB_BASE + 0x0023)
+#define VVCAM_ISP_CID_AWB_FRONT_GROUND_BG           (VVCAM_ISP_CID_AWB_BASE + 0x0024)
+#define VVCAM_ISP_CID_AWB_USE_MANU_DAMP_COEFF       (VVCAM_ISP_CID_AWB_BASE + 0x0025)
+#define VVCAM_ISP_CID_AWB_MANU_DAMP_COEFF           (VVCAM_ISP_CID_AWB_BASE + 0x0026)
+#define VVCAM_ISP_CID_AWB_PERFORMANCE_OPTI_MODE     (VVCAM_ISP_CID_AWB_BASE + 0x0027)
+#define VVCAM_ISP_CID_AWB_LOCK_THRESH               (VVCAM_ISP_CID_AWB_BASE + 0x0028)
+#define VVCAM_ISP_CID_AWB_UNLOCK_THRESH             (VVCAM_ISP_CID_AWB_BASE + 0x0029)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_ENABLE    (VVCAM_ISP_CID_AWB_BASE + 0x002A)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_POST_EN   (VVCAM_ISP_CID_AWB_BASE + 0x002B)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_RG        (VVCAM_ISP_CID_AWB_BASE + 0x002C)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_BG        (VVCAM_ISP_CID_AWB_BASE + 0x002D)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_THRESH    (VVCAM_ISP_CID_AWB_BASE + 0x002E)
+#define VVCAM_ISP_CID_AWB_CONFOUND_CUSTOM_WEIGHT    (VVCAM_ISP_CID_AWB_BASE + 0x002F)
+#define VVCAM_ISP_CID_AWB_RESULT_CCT                (VVCAM_ISP_CID_AWB_BASE + 0x0030)
+
+#ifdef __KERNEL__
+int vvcam_isp_awb_ctrl_count(void);
+int vvcam_isp_awb_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/bls/bls/vvcam_isp_bls.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/bls/bls/vvcam_isp_bls.c
new file mode 100644
index 000000000000..09cf6306fb60
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/bls/bls/vvcam_isp_bls.c
@@ -0,0 +1,294 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_bls.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_bls_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_BLS_RESET:
+        case VVCAM_ISP_CID_BLS_MODE:
+        case VVCAM_ISP_CID_BLS_AUTO_LEVEL:
+        case VVCAM_ISP_CID_BLS_AUTO_GAIN:
+        case VVCAM_ISP_CID_BLS_AUTO_TABLE:
+        case VVCAM_ISP_CID_BLS_MANU_R_VALUE:
+        case VVCAM_ISP_CID_BLS_MANU_GR_VALUE:
+        case VVCAM_ISP_CID_BLS_MANU_GB_VALUE:
+        case VVCAM_ISP_CID_BLS_MANU_B_VALUE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_bls_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_BLS_RESET:
+        case VVCAM_ISP_CID_BLS_MODE:
+        case VVCAM_ISP_CID_BLS_AUTO_LEVEL:
+        case VVCAM_ISP_CID_BLS_AUTO_GAIN:
+        case VVCAM_ISP_CID_BLS_AUTO_TABLE:
+        case VVCAM_ISP_CID_BLS_MANU_R_VALUE:
+        case VVCAM_ISP_CID_BLS_MANU_GR_VALUE:
+        case VVCAM_ISP_CID_BLS_MANU_GB_VALUE:
+        case VVCAM_ISP_CID_BLS_MANU_B_VALUE:
+        case VVCAM_ISP_CID_BLS_STAT_R_VALUE:
+        case VVCAM_ISP_CID_BLS_STAT_GR_VALUE:
+        case VVCAM_ISP_CID_BLS_STAT_GB_VALUE:
+        case VVCAM_ISP_CID_BLS_STAT_B_VALUE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_bls_ctrl_ops = {
+    .s_ctrl = vvcam_isp_bls_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_bls_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_bls_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_AUTO_TABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_auto_table",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_MANU_R_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_manu_r_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_MANU_GR_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_manu_gr_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_MANU_GB_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_manu_gb_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_MANU_B_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_manu_b_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_STAT_R_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_stat_r_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_STAT_GR_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_stat_gr_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_STAT_GB_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_stat_gb_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_bls_ctrl_ops,
+        .id   = VVCAM_ISP_CID_BLS_STAT_B_VALUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_bls_stat_b_value",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1},
+    },
+};
+
+int vvcam_isp_bls_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_bls_ctrls);
+}
+
+int vvcam_isp_bls_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_bls_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_bls_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp bls ctrl %s failed %d.\n",
+                vvcam_isp_bls_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/bls/bls/vvcam_isp_bls.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/bls/bls/vvcam_isp_bls.h
new file mode 100644
index 000000000000..b5c7880cb4f2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/bls/bls/vvcam_isp_bls.h
@@ -0,0 +1,80 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_BLS_H__
+#define __VVCAM_ISP_BLS_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_BLS_RESET             (VVCAM_ISP_CID_BLS_BASE + 0x0000)
+#define VVCAM_ISP_CID_BLS_MODE              (VVCAM_ISP_CID_BLS_BASE + 0x0001)
+#define VVCAM_ISP_CID_BLS_AUTO_LEVEL        (VVCAM_ISP_CID_BLS_BASE + 0x0002)
+#define VVCAM_ISP_CID_BLS_AUTO_GAIN         (VVCAM_ISP_CID_BLS_BASE + 0x0003)
+#define VVCAM_ISP_CID_BLS_AUTO_TABLE        (VVCAM_ISP_CID_BLS_BASE + 0x0004)
+#define VVCAM_ISP_CID_BLS_MANU_R_VALUE      (VVCAM_ISP_CID_BLS_BASE + 0x0005)
+#define VVCAM_ISP_CID_BLS_MANU_GR_VALUE     (VVCAM_ISP_CID_BLS_BASE + 0x0006)
+#define VVCAM_ISP_CID_BLS_MANU_GB_VALUE     (VVCAM_ISP_CID_BLS_BASE + 0x0007)
+#define VVCAM_ISP_CID_BLS_MANU_B_VALUE      (VVCAM_ISP_CID_BLS_BASE + 0x0008)
+#define VVCAM_ISP_CID_BLS_STAT_R_VALUE      (VVCAM_ISP_CID_BLS_BASE + 0x0009)
+#define VVCAM_ISP_CID_BLS_STAT_GR_VALUE     (VVCAM_ISP_CID_BLS_BASE + 0x000A)
+#define VVCAM_ISP_CID_BLS_STAT_GB_VALUE     (VVCAM_ISP_CID_BLS_BASE + 0x000B)
+#define VVCAM_ISP_CID_BLS_STAT_B_VALUE      (VVCAM_ISP_CID_BLS_BASE + 0x000C)
+
+#ifdef __KERNEL__
+int vvcam_isp_bls_ctrl_count(void);
+int vvcam_isp_bls_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ccm/ccm/vvcam_isp_ccm.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ccm/ccm/vvcam_isp_ccm.c
new file mode 100644
index 000000000000..aa8314988a06
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ccm/ccm/vvcam_isp_ccm.c
@@ -0,0 +1,217 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ccm.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ccm_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CCM_ENABLE:
+        case VVCAM_ISP_CID_CCM_RESET:
+        case VVCAM_ISP_CID_CCM_MODE:
+        case VVCAM_ISP_CID_CCM_MATRIX:
+        case VVCAM_ISP_CID_CCM_OFFSET:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ccm_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CCM_ENABLE:
+        case VVCAM_ISP_CID_CCM_RESET:
+        case VVCAM_ISP_CID_CCM_MODE:
+        case VVCAM_ISP_CID_CCM_MATRIX:
+        case VVCAM_ISP_CID_CCM_OFFSET:
+        case VVCAM_ISP_CID_CCM_STAT_MATRIX:
+        case VVCAM_ISP_CID_CCM_STAT_OFFSET:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ccm_ctrl_ops = {
+    .s_ctrl = vvcam_isp_ccm_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ccm_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ccm_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 3, 0, 0},
+    },
+    {
+        /* float 3x array -2047~2047 */
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_OFFSET,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_STAT_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_stat_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 3, 0, 0},
+    },
+    {
+        /* float 3x array -2047~2047 */
+        .ops  = &vvcam_isp_ccm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CCM_STAT_OFFSET,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ccm_stat_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3},
+    },
+};
+
+int vvcam_isp_ccm_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ccm_ctrls);
+}
+
+int vvcam_isp_ccm_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ccm_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ccm_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ccm ctrl %s failed %d.\n",
+                vvcam_isp_ccm_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ccm/ccm/vvcam_isp_ccm.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ccm/ccm/vvcam_isp_ccm.h
new file mode 100644
index 000000000000..6d5ca4745fd3
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ccm/ccm/vvcam_isp_ccm.h
@@ -0,0 +1,74 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CCM_H__
+#define __VVCAM_ISP_CCM_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CCM_ENABLE            (VVCAM_ISP_CID_CCM_BASE + 0x0000)
+#define VVCAM_ISP_CID_CCM_RESET             (VVCAM_ISP_CID_CCM_BASE + 0x0001)
+#define VVCAM_ISP_CID_CCM_MODE              (VVCAM_ISP_CID_CCM_BASE + 0x0002)
+#define VVCAM_ISP_CID_CCM_MATRIX            (VVCAM_ISP_CID_CCM_BASE + 0x0003)
+#define VVCAM_ISP_CID_CCM_OFFSET            (VVCAM_ISP_CID_CCM_BASE + 0x0004)
+#define VVCAM_ISP_CID_CCM_STAT_MATRIX       (VVCAM_ISP_CID_CCM_BASE + 0x0005)
+#define VVCAM_ISP_CID_CCM_STAT_OFFSET       (VVCAM_ISP_CID_CCM_BASE + 0x0006)
+
+#ifdef __KERNEL__
+int vvcam_isp_ccm_ctrl_count(void);
+int vvcam_isp_ccm_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.c
new file mode 100644
index 000000000000..48c8755d12b2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.c
@@ -0,0 +1,227 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_cnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_cnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CNR_ENABLE:
+        case VVCAM_ISP_CID_CNR_RESET:
+        case VVCAM_ISP_CID_CNR_MANU_LINE_WIDTH:
+        case VVCAM_ISP_CID_CNR_MANU_THRESHOLD1:
+        case VVCAM_ISP_CID_CNR_MANU_THRESHOLD2:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_cnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CNR_ENABLE:
+        case VVCAM_ISP_CID_CNR_RESET:
+        case VVCAM_ISP_CID_CNR_MANU_LINE_WIDTH:
+        case VVCAM_ISP_CID_CNR_MANU_THRESHOLD1:
+        case VVCAM_ISP_CID_CNR_MANU_THRESHOLD2:
+        case VVCAM_ISP_CID_CNR_STAT_LINE_WIDTH:
+        case VVCAM_ISP_CID_CNR_STAT_THRESHOLD1:
+        case VVCAM_ISP_CID_CNR_STAT_THRESHOLD2:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_cnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_cnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_cnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_cnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_LINE_WIDTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_line_width",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_THRESHOLD1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_threshold1",
+        .step = 1,
+        .min  = 0,
+        .max  = 32767,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_THRESHOLD2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_threshold2",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_LINE_WIDTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_line_width",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_THRESHOLD1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_threshold1",
+        .step = 1,
+        .min  = 0,
+        .max  = 32767,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_THRESHOLD2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_threshold2",
+        .step = 1,
+        .min  = 0,
+        .max  = 16383,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_cnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_cnr_ctrls);
+}
+
+int vvcam_isp_cnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_cnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_cnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp cnr ctrl %s failed %d.\n",
+                vvcam_isp_cnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.h
new file mode 100644
index 000000000000..e8804d70a3f2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v1/vvcam_isp_cnr.h
@@ -0,0 +1,75 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CNR_H__
+#define __VVCAM_ISP_CNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CNR_ENABLE            (VVCAM_ISP_CID_CNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_CNR_RESET             (VVCAM_ISP_CID_CNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_CNR_MANU_LINE_WIDTH   (VVCAM_ISP_CID_CNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_CNR_MANU_THRESHOLD1   (VVCAM_ISP_CID_CNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_CNR_MANU_THRESHOLD2   (VVCAM_ISP_CID_CNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_CNR_STAT_LINE_WIDTH   (VVCAM_ISP_CID_CNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_CNR_STAT_THRESHOLD1   (VVCAM_ISP_CID_CNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_CNR_STAT_THRESHOLD2   (VVCAM_ISP_CID_CNR_BASE + 0x0007)
+
+#ifdef __KERNEL__
+int vvcam_isp_cnr_ctrl_count(void);
+int vvcam_isp_cnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.c
new file mode 100644
index 000000000000..439133285416
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.c
@@ -0,0 +1,412 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_cnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_cnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CNR_ENABLE:
+        case VVCAM_ISP_CID_CNR_RESET:
+        case VVCAM_ISP_CID_CNR_MODE:
+        case VVCAM_ISP_CID_CNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_CNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL:
+        case VVCAM_ISP_CID_CNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_CNR_AUTO_STRENGTH2:
+        case VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER3:
+        case VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER:
+        case VVCAM_ISP_CID_CNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_CNR_MANU_STRENGTH2:
+        case VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_cnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CNR_ENABLE:
+        case VVCAM_ISP_CID_CNR_RESET:
+        case VVCAM_ISP_CID_CNR_MODE:
+        case VVCAM_ISP_CID_CNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_CNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL:
+        case VVCAM_ISP_CID_CNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_CNR_AUTO_STRENGTH2:
+        case VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER3:
+        case VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER:
+        case VVCAM_ISP_CID_CNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_CNR_MANU_STRENGTH2:
+        case VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK:
+        case VVCAM_ISP_CID_CNR_STAT_SIGMA_LAYER:
+        case VVCAM_ISP_CID_CNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_CNR_STAT_STRENGTH2:
+        case VVCAM_ISP_CID_CNR_STAT_TEXTURE_MASK:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_cnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_cnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_cnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_cnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_texture_mask",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 48.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer0",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 48.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer1",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 48.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer2",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 48.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER3,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer3",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 4x array 1.0 ~ 48.0*/
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_sigma_layer",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_texture_mask",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 4x array 1.0 ~ 48.0*/
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_SIGMA_LAYER,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_sigma_layer",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_TEXTURE_MASK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_texture_mask",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_cnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_cnr_ctrls);
+}
+
+int vvcam_isp_cnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_cnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_cnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp cnr ctrl %s failed %d.\n",
+                vvcam_isp_cnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.h
new file mode 100644
index 000000000000..d6ede46141be
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_1/vvcam_isp_cnr.h
@@ -0,0 +1,89 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CNR_H__
+#define __VVCAM_ISP_CNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CNR_ENABLE            (VVCAM_ISP_CID_CNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_CNR_RESET             (VVCAM_ISP_CID_CNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_CNR_MODE              (VVCAM_ISP_CID_CNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_CNR_AUTO_LEVEL        (VVCAM_ISP_CID_CNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_CNR_AUTO_GAIN         (VVCAM_ISP_CID_CNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL   (VVCAM_ISP_CID_CNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_CNR_AUTO_STRENGTH     (VVCAM_ISP_CID_CNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_CNR_AUTO_STRENGTH2    (VVCAM_ISP_CID_CNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK (VVCAM_ISP_CID_CNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0 (VVCAM_ISP_CID_CNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1 (VVCAM_ISP_CID_CNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2 (VVCAM_ISP_CID_CNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER3 (VVCAM_ISP_CID_CNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER  (VVCAM_ISP_CID_CNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_CNR_MANU_STRENGTH     (VVCAM_ISP_CID_CNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_CNR_MANU_STRENGTH2    (VVCAM_ISP_CID_CNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK (VVCAM_ISP_CID_CNR_BASE + 0x0010)
+#define VVCAM_ISP_CID_CNR_STAT_SIGMA_LAYER  (VVCAM_ISP_CID_CNR_BASE + 0x0011)
+#define VVCAM_ISP_CID_CNR_STAT_STRENGTH     (VVCAM_ISP_CID_CNR_BASE + 0x0012)
+#define VVCAM_ISP_CID_CNR_STAT_STRENGTH2    (VVCAM_ISP_CID_CNR_BASE + 0x0013)
+#define VVCAM_ISP_CID_CNR_STAT_TEXTURE_MASK (VVCAM_ISP_CID_CNR_BASE + 0x0014)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_cnr_ctrl_count(void);
+int vvcam_isp_cnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.c
new file mode 100644
index 000000000000..f343d0c78fc1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.c
@@ -0,0 +1,357 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_cnr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_cnr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CNR_ENABLE:
+        case VVCAM_ISP_CID_CNR_RESET:
+        case VVCAM_ISP_CID_CNR_MODE:
+        case VVCAM_ISP_CID_CNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_CNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL:
+        case VVCAM_ISP_CID_CNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2:
+        case VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER:
+        case VVCAM_ISP_CID_CNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_cnr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CNR_ENABLE:
+        case VVCAM_ISP_CID_CNR_RESET:
+        case VVCAM_ISP_CID_CNR_MODE:
+        case VVCAM_ISP_CID_CNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_CNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL:
+        case VVCAM_ISP_CID_CNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1:
+        case VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2:
+        case VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER:
+        case VVCAM_ISP_CID_CNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK:
+        case VVCAM_ISP_CID_CNR_STAT_SIGMA_LAYER:
+        case VVCAM_ISP_CID_CNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_CNR_STAT_TEXTURE_MASK:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_cnr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_cnr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_cnr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_cnr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_texture_mask",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 96.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer0",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 96.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer1",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array 1.0 ~ 96.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_auto_sigma_layer2",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 3x array 1.0 ~ 96.0 */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_sigma_layer",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_manu_texture_mask",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 3x array 1.0 ~ 96.0*/
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_SIGMA_LAYER,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_sigma_layer",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t */
+        .ops  = &vvcam_isp_cnr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CNR_STAT_TEXTURE_MASK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cnr_stat_texture_mask",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_cnr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_cnr_ctrls);
+}
+
+int vvcam_isp_cnr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_cnr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_cnr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp cnr ctrl %s failed %d.\n",
+                vvcam_isp_cnr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.h
new file mode 100644
index 000000000000..61736c3c0947
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cnr/cnr_v2_2/vvcam_isp_cnr.h
@@ -0,0 +1,84 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CNR_H__
+#define __VVCAM_ISP_CNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CNR_ENABLE            (VVCAM_ISP_CID_CNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_CNR_RESET             (VVCAM_ISP_CID_CNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_CNR_MODE              (VVCAM_ISP_CID_CNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_CNR_AUTO_LEVEL        (VVCAM_ISP_CID_CNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_CNR_AUTO_GAIN         (VVCAM_ISP_CID_CNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_CNR_AUTO_ENABLE_TBL   (VVCAM_ISP_CID_CNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_CNR_AUTO_STRENGTH     (VVCAM_ISP_CID_CNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_CNR_AUTO_TEXTURE_MASK (VVCAM_ISP_CID_CNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER0 (VVCAM_ISP_CID_CNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER1 (VVCAM_ISP_CID_CNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_CNR_AUTO_SIGMA_LAYER2 (VVCAM_ISP_CID_CNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_CNR_MANU_SIGMA_LAYER  (VVCAM_ISP_CID_CNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_CNR_MANU_STRENGTH     (VVCAM_ISP_CID_CNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_CNR_MANU_TEXTURE_MASK (VVCAM_ISP_CID_CNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_CNR_STAT_SIGMA_LAYER  (VVCAM_ISP_CID_CNR_BASE + 0x000E)
+#define VVCAM_ISP_CID_CNR_STAT_STRENGTH     (VVCAM_ISP_CID_CNR_BASE + 0x000F)
+#define VVCAM_ISP_CID_CNR_STAT_TEXTURE_MASK (VVCAM_ISP_CID_CNR_BASE + 0x0010)
+
+#ifdef __KERNEL__
+int vvcam_isp_cnr_ctrl_count(void);
+int vvcam_isp_cnr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.c
new file mode 100644
index 000000000000..56bd7ace800f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.c
@@ -0,0 +1,225 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_cpd.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_cpd_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CPD_ENABLE:
+        case VVCAM_ISP_CID_CPD_RESET:
+        case VVCAM_ISP_CID_CPD_CURVE_X:
+        case VVCAM_ISP_CID_CPD_CURVE_Y:
+        case VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_cpd_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CPD_ENABLE:
+        case VVCAM_ISP_CID_CPD_RESET:
+        case VVCAM_ISP_CID_CPD_CURVE_X:
+        case VVCAM_ISP_CID_CPD_CURVE_Y:
+        case VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE:
+        case VVCAM_ISP_CID_CPD_STAT_CURVE_X:
+        case VVCAM_ISP_CID_CPD_STAT_CURVE_Y:
+        case VVCAM_ISP_CID_CPD_STAT_USE_OUT_Y_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_cpd_ctrl_ops = {
+    .s_ctrl = vvcam_isp_cpd_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_cpd_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_cpd_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 31,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_use_out_y_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_STAT_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_stat_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 31,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_STAT_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_stat_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_STAT_USE_OUT_Y_CURVE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_stat_use_out_y_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+};
+
+int vvcam_isp_cpd_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_cpd_ctrls);
+}
+
+int vvcam_isp_cpd_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_cpd_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_cpd_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp cpd ctrl %s failed %d.\n",
+                vvcam_isp_cpd_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.h
new file mode 100644
index 000000000000..4ee457cca2b5
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1/vvcam_isp_cpd.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CPD_H__
+#define __VVCAM_ISP_CPD_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CPD_ENABLE            (VVCAM_ISP_CID_CPD_BASE + 0x0000)
+#define VVCAM_ISP_CID_CPD_RESET             (VVCAM_ISP_CID_CPD_BASE + 0x0001)
+#define VVCAM_ISP_CID_CPD_CURVE_X           (VVCAM_ISP_CID_CPD_BASE + 0x0002)
+#define VVCAM_ISP_CID_CPD_CURVE_Y           (VVCAM_ISP_CID_CPD_BASE + 0x0003)
+#define VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE   (VVCAM_ISP_CID_CPD_BASE + 0x0004)
+#define VVCAM_ISP_CID_CPD_STAT_CURVE_X      (VVCAM_ISP_CID_CPD_BASE + 0x0005)
+#define VVCAM_ISP_CID_CPD_STAT_CURVE_Y      (VVCAM_ISP_CID_CPD_BASE + 0x0006)
+#define VVCAM_ISP_CID_CPD_STAT_USE_OUT_Y_CURVE \
+                                            (VVCAM_ISP_CID_CPD_BASE + 0x0007)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_cpd_ctrl_count(void);
+int vvcam_isp_cpd_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.c
new file mode 100644
index 000000000000..83b4e5805763
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.c
@@ -0,0 +1,225 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_cpd.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_cpd_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CPD_ENABLE:
+        case VVCAM_ISP_CID_CPD_RESET:
+        case VVCAM_ISP_CID_CPD_CURVE_X:
+        case VVCAM_ISP_CID_CPD_CURVE_Y:
+        case VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_cpd_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CPD_ENABLE:
+        case VVCAM_ISP_CID_CPD_RESET:
+        case VVCAM_ISP_CID_CPD_CURVE_X:
+        case VVCAM_ISP_CID_CPD_CURVE_Y:
+        case VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE:
+        case VVCAM_ISP_CID_CPD_STAT_CURVE_X:
+        case VVCAM_ISP_CID_CPD_STAT_CURVE_Y:
+        case VVCAM_ISP_CID_CPD_STAT_USE_OUT_Y_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_cpd_ctrl_ops = {
+    .s_ctrl = vvcam_isp_cpd_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_cpd_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_cpd_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 31,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_use_out_y_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_STAT_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_stat_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 31,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_STAT_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_stat_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_cpd_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPD_STAT_USE_OUT_Y_CURVE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cpd_stat_use_out_y_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+};
+
+int vvcam_isp_cpd_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_cpd_ctrls);
+}
+
+int vvcam_isp_cpd_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_cpd_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_cpd_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp cpd ctrl %s failed %d.\n",
+                vvcam_isp_cpd_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.h
new file mode 100644
index 000000000000..4ee457cca2b5
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cpd/cpd_v1_1/vvcam_isp_cpd.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CPD_H__
+#define __VVCAM_ISP_CPD_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CPD_ENABLE            (VVCAM_ISP_CID_CPD_BASE + 0x0000)
+#define VVCAM_ISP_CID_CPD_RESET             (VVCAM_ISP_CID_CPD_BASE + 0x0001)
+#define VVCAM_ISP_CID_CPD_CURVE_X           (VVCAM_ISP_CID_CPD_BASE + 0x0002)
+#define VVCAM_ISP_CID_CPD_CURVE_Y           (VVCAM_ISP_CID_CPD_BASE + 0x0003)
+#define VVCAM_ISP_CID_CPD_USE_OUT_Y_CURVE   (VVCAM_ISP_CID_CPD_BASE + 0x0004)
+#define VVCAM_ISP_CID_CPD_STAT_CURVE_X      (VVCAM_ISP_CID_CPD_BASE + 0x0005)
+#define VVCAM_ISP_CID_CPD_STAT_CURVE_Y      (VVCAM_ISP_CID_CPD_BASE + 0x0006)
+#define VVCAM_ISP_CID_CPD_STAT_USE_OUT_Y_CURVE \
+                                            (VVCAM_ISP_CID_CPD_BASE + 0x0007)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_cpd_ctrl_count(void);
+int vvcam_isp_cpd_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cproc/cproc/vvcam_isp_cproc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cproc/cproc/vvcam_isp_cproc.c
new file mode 100644
index 000000000000..692f16a8b306
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cproc/cproc/vvcam_isp_cproc.c
@@ -0,0 +1,366 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_cproc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_cproc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CPROC_ENABLE:
+        case VVCAM_ISP_CID_CPROC_RESET:
+        case VVCAM_ISP_CID_CPROC_CHROMA_OUT_TYPE:
+        case VVCAM_ISP_CID_CPROC_MODE:
+        case VVCAM_ISP_CID_CPROC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_CPROC_AUTO_GAIN:
+        case VVCAM_ISP_CID_CPROC_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_CPROC_AUTO_BRIGHT:
+        case VVCAM_ISP_CID_CPROC_AUTO_SATURATION:
+        case VVCAM_ISP_CID_CPROC_AUTO_HUE:
+        case VVCAM_ISP_CID_CPROC_MANU_CONTRAST:
+        case VVCAM_ISP_CID_CPROC_MANU_BRIGHT:
+        case VVCAM_ISP_CID_CPROC_MANU_SATURATION:
+        case VVCAM_ISP_CID_CPROC_MANU_HUE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_cproc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_CPROC_ENABLE:
+        case VVCAM_ISP_CID_CPROC_RESET:
+        case VVCAM_ISP_CID_CPROC_CHROMA_OUT_TYPE:
+        case VVCAM_ISP_CID_CPROC_MODE:
+        case VVCAM_ISP_CID_CPROC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_CPROC_AUTO_GAIN:
+        case VVCAM_ISP_CID_CPROC_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_CPROC_AUTO_BRIGHT:
+        case VVCAM_ISP_CID_CPROC_AUTO_SATURATION:
+        case VVCAM_ISP_CID_CPROC_AUTO_HUE:
+        case VVCAM_ISP_CID_CPROC_MANU_CONTRAST:
+        case VVCAM_ISP_CID_CPROC_MANU_BRIGHT:
+        case VVCAM_ISP_CID_CPROC_MANU_SATURATION:
+        case VVCAM_ISP_CID_CPROC_MANU_HUE:
+        case VVCAM_ISP_CID_CPROC_STAT_CONTRAST:
+        case VVCAM_ISP_CID_CPROC_STAT_BRIGHT:
+        case VVCAM_ISP_CID_CPROC_STAT_SATURATION:
+        case VVCAM_ISP_CID_CPROC_STAT_HUE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_cproc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_cproc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_cproc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_cproc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_CHROMA_OUT_TYPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_chroma_out_type",
+        .step = 1,
+        .min  = 1,
+        .max  = 2,
+        .def  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_AUTO_BRIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_auto_bright",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_AUTO_SATURATION,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_auto_saturation",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_AUTO_HUE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_auto_hue",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* float [0,1.992] ratio 128:1 */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_manu_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float [-128, 127] */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_MANU_BRIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_manu_bright",
+        .step = 1,
+        .min  = -128,
+        .max  = 127,
+        .def  = 0,
+    },
+    {
+        /* float [0,1.992] ratio 128:1 */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_MANU_SATURATION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_manu_saturation",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float [-90, 87] */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_MANU_HUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_manu_hue",
+        .step = 1,
+        .min  = -90,
+        .max  = 87,
+        .def  = 0,
+    },
+    {
+        /* float [0,1.992] ratio 128:1 */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_stat_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float [-128, 127] */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_STAT_BRIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_stat_bright",
+        .step = 1,
+        .min  = -128,
+        .max  = 127,
+        .def  = 0,
+    },
+    {
+        /* float [0,1.992] ratio 128:1 */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_STAT_SATURATION,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_stat_saturation",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+    },
+    {
+        /* float [-90, 87] */
+        .ops  = &vvcam_isp_cproc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_CPROC_STAT_HUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_cproc_stat_hue",
+        .step = 1,
+        .min  = -90,
+        .max  = 87,
+        .def  = 0,
+    },
+};
+
+int vvcam_isp_cproc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_cproc_ctrls);
+}
+
+int vvcam_isp_cproc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_cproc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_cproc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp cproc ctrl %s failed %d.\n",
+                vvcam_isp_cproc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cproc/cproc/vvcam_isp_cproc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cproc/cproc/vvcam_isp_cproc.h
new file mode 100644
index 000000000000..964f6173281a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/cproc/cproc/vvcam_isp_cproc.h
@@ -0,0 +1,86 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CPROC_H__
+#define __VVCAM_ISP_CPROC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_CPROC_ENABLE              (VVCAM_ISP_CID_CPROC_BASE + 0x0000)
+#define VVCAM_ISP_CID_CPROC_RESET               (VVCAM_ISP_CID_CPROC_BASE + 0x0001)
+#define VVCAM_ISP_CID_CPROC_CHROMA_OUT_TYPE     (VVCAM_ISP_CID_CPROC_BASE + 0x0002)
+#define VVCAM_ISP_CID_CPROC_MODE                (VVCAM_ISP_CID_CPROC_BASE + 0x0003)
+#define VVCAM_ISP_CID_CPROC_AUTO_LEVEL          (VVCAM_ISP_CID_CPROC_BASE + 0x0004)
+#define VVCAM_ISP_CID_CPROC_AUTO_GAIN           (VVCAM_ISP_CID_CPROC_BASE + 0x0005)
+#define VVCAM_ISP_CID_CPROC_AUTO_CONTRAST       (VVCAM_ISP_CID_CPROC_BASE + 0x0006)
+#define VVCAM_ISP_CID_CPROC_AUTO_BRIGHT         (VVCAM_ISP_CID_CPROC_BASE + 0x0007)
+#define VVCAM_ISP_CID_CPROC_AUTO_SATURATION     (VVCAM_ISP_CID_CPROC_BASE + 0x0008)
+#define VVCAM_ISP_CID_CPROC_AUTO_HUE            (VVCAM_ISP_CID_CPROC_BASE + 0x0009)
+#define VVCAM_ISP_CID_CPROC_MANU_CONTRAST       (VVCAM_ISP_CID_CPROC_BASE + 0x000A)
+#define VVCAM_ISP_CID_CPROC_MANU_BRIGHT         (VVCAM_ISP_CID_CPROC_BASE + 0x000B)
+#define VVCAM_ISP_CID_CPROC_MANU_SATURATION     (VVCAM_ISP_CID_CPROC_BASE + 0x000C)
+#define VVCAM_ISP_CID_CPROC_MANU_HUE            (VVCAM_ISP_CID_CPROC_BASE + 0x000D)
+#define VVCAM_ISP_CID_CPROC_STAT_CONTRAST       (VVCAM_ISP_CID_CPROC_BASE + 0x000E)
+#define VVCAM_ISP_CID_CPROC_STAT_BRIGHT         (VVCAM_ISP_CID_CPROC_BASE + 0x000F)
+#define VVCAM_ISP_CID_CPROC_STAT_SATURATION     (VVCAM_ISP_CID_CPROC_BASE + 0x0010)
+#define VVCAM_ISP_CID_CPROC_STAT_HUE            (VVCAM_ISP_CID_CPROC_BASE + 0x0011)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_cproc_ctrl_count(void);
+int vvcam_isp_cproc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dg/dg_v1/vvcam_isp_dg.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dg/dg_v1/vvcam_isp_dg.c
new file mode 100644
index 000000000000..aaf4c65ba2c8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dg/dg_v1/vvcam_isp_dg.c
@@ -0,0 +1,259 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_dg.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_dg_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DG_ENABLE:
+        case VVCAM_ISP_CID_DG_RESET:
+        case VVCAM_ISP_CID_DG_MANU_B_GAIN:
+        case VVCAM_ISP_CID_DG_MANU_GB_GAIN:
+        case VVCAM_ISP_CID_DG_MANU_GR_GAIN:
+        case VVCAM_ISP_CID_DG_MANU_R_GAIN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_dg_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DG_ENABLE:
+        case VVCAM_ISP_CID_DG_RESET:
+        case VVCAM_ISP_CID_DG_MANU_B_GAIN:
+        case VVCAM_ISP_CID_DG_MANU_GB_GAIN:
+        case VVCAM_ISP_CID_DG_MANU_GR_GAIN:
+        case VVCAM_ISP_CID_DG_MANU_R_GAIN:
+        case VVCAM_ISP_CID_DG_STAT_B_GAIN:
+        case VVCAM_ISP_CID_DG_STAT_GB_GAIN:
+        case VVCAM_ISP_CID_DG_STAT_GR_GAIN:
+        case VVCAM_ISP_CID_DG_STAT_R_GAIN:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_dg_ctrl_ops = {
+    .s_ctrl = vvcam_isp_dg_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_dg_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_dg_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_MANU_B_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_manu_b_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_MANU_GB_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_manu_gb_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_MANU_GR_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_manu_gr_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_MANU_R_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_manu_r_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_STAT_B_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_stat_b_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_STAT_GB_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_stat_gb_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_STAT_GR_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_stat_gr_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+    {
+        /* float [1.0,255.0] */
+        .ops  = &vvcam_isp_dg_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DG_STAT_R_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dg_stat_r_gain",
+        .step = 1,
+        .min  = 256,
+        .max  = 65280,
+        .def  = 256,
+    },
+};
+
+int vvcam_isp_dg_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_dg_ctrls);
+}
+
+int vvcam_isp_dg_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_dg_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_dg_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp dg ctrl %s failed %d.\n",
+                vvcam_isp_dg_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dg/dg_v1/vvcam_isp_dg.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dg/dg_v1/vvcam_isp_dg.h
new file mode 100644
index 000000000000..01c7d04699fa
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dg/dg_v1/vvcam_isp_dg.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DG_H__
+#define __VVCAM_ISP_DG_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_DG_ENABLE             (VVCAM_ISP_CID_DG_BASE + 0x0000)
+#define VVCAM_ISP_CID_DG_RESET              (VVCAM_ISP_CID_DG_BASE + 0x0001)
+#define VVCAM_ISP_CID_DG_MANU_B_GAIN        (VVCAM_ISP_CID_DG_BASE + 0x0002)
+#define VVCAM_ISP_CID_DG_MANU_GB_GAIN       (VVCAM_ISP_CID_DG_BASE + 0x0003)
+#define VVCAM_ISP_CID_DG_MANU_GR_GAIN       (VVCAM_ISP_CID_DG_BASE + 0x0004)
+#define VVCAM_ISP_CID_DG_MANU_R_GAIN        (VVCAM_ISP_CID_DG_BASE + 0x0005)
+#define VVCAM_ISP_CID_DG_STAT_B_GAIN        (VVCAM_ISP_CID_DG_BASE + 0x0006)
+#define VVCAM_ISP_CID_DG_STAT_GB_GAIN       (VVCAM_ISP_CID_DG_BASE + 0x0007)
+#define VVCAM_ISP_CID_DG_STAT_GR_GAIN       (VVCAM_ISP_CID_DG_BASE + 0x0008)
+#define VVCAM_ISP_CID_DG_STAT_R_GAIN        (VVCAM_ISP_CID_DG_BASE + 0x0009)
+
+#ifdef __KERNEL__
+int vvcam_isp_dg_ctrl_count(void);
+int vvcam_isp_dg_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.c
new file mode 100644
index 000000000000..05b07cbcd4bb
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.c
@@ -0,0 +1,497 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_dmsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_dmsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DMSC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_CAC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_FILTER_ENABLE:
+        case VVCAM_ISP_CID_DMSC_RESET:
+        case VVCAM_ISP_CID_DMSC_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_DENOISE_LEVEL:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_SHARPEN_LEVEL:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_V_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_H_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_dmsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DMSC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_CAC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_FILTER_ENABLE:
+        case VVCAM_ISP_CID_DMSC_RESET:
+        case VVCAM_ISP_CID_DMSC_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_DENOISE_LEVEL:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_SHARPEN_LEVEL:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_V_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_FILTER_H_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_FILTER_DENOISE_LEVEL:
+        case VVCAM_ISP_CID_DMSC_STAT_FILTER_SHARPEN_LEVEL:
+        case VVCAM_ISP_CID_DMSC_STAT_FILTER_V_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_FILTER_H_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_THRESHOLD:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_dmsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_dmsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_dmsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_dmsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_CAC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_cac_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_FILTER_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_filt_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_FILTER_DENOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_filt_denoise_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_FILTER_SHARPEN_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_filt_sharpen_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_FILTER_V_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_filt_v_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_FILTER_H_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_filt_h_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_a_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_a_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_b_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_b_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_c_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_c_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* according to image width */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_ctr_h_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        /* according to image height */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_ctr_v_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_FILTER_DENOISE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_filt_denoise_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_FILTER_SHARPEN_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_filt_sharpen_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_FILTER_V_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_filt_v_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_FILTER_H_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_filt_h_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_a_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_a_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_b_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_b_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_c_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_c_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* according to image width */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_ctr_h_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        /* according to image height */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_ctr_v_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+};
+
+int vvcam_isp_dmsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_dmsc_ctrls);
+}
+
+int vvcam_isp_dmsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_dmsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_dmsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp dmsc ctrl %s failed %d.\n",
+                vvcam_isp_dmsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.h
new file mode 100644
index 000000000000..7215fd9f720c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v1/vvcam_isp_dmsc.h
@@ -0,0 +1,109 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DMSC_H__
+#define __VVCAM_ISP_DMSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_DMSC_ENABLE           (VVCAM_ISP_CID_DMSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_DMSC_CAC_ENABLE       (VVCAM_ISP_CID_DMSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_DMSC_FILTER_ENABLE    (VVCAM_ISP_CID_DMSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_DMSC_RESET            (VVCAM_ISP_CID_DMSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_DMSC_THRESHOLD        (VVCAM_ISP_CID_DMSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_DMSC_MANU_FILTER_DENOISE_LEVEL \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_DMSC_MANU_FILTER_SHARPEN_LEVEL \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_DMSC_MANU_FILTER_V_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0007)
+#define VVCAM_ISP_CID_DMSC_MANU_FILTER_H_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0008)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0009)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x000A)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x000B)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x000C)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x000D)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x000E)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000F)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0010)
+#define VVCAM_ISP_CID_DMSC_STAT_FILTER_DENOISE_LEVEL \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0011)
+#define VVCAM_ISP_CID_DMSC_STAT_FILTER_SHARPEN_LEVEL \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0012)
+#define VVCAM_ISP_CID_DMSC_STAT_FILTER_V_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0013)
+#define VVCAM_ISP_CID_DMSC_STAT_FILTER_H_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0014)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0015)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0016)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0017)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0018)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0019)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x001A)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001B)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001C)
+#define VVCAM_ISP_CID_DMSC_STAT_THRESHOLD   (VVCAM_ISP_CID_DMSC_BASE + 0x001D)
+
+#ifdef __KERNEL__
+int vvcam_isp_dmsc_ctrl_count(void);
+int vvcam_isp_dmsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.c
new file mode 100644
index 000000000000..92147a9c5862
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.c
@@ -0,0 +1,1714 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_dmsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_dmsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DMSC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_CAC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SKIN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_RESET:
+        case VVCAM_ISP_CID_DMSC_MODE:
+        case VVCAM_ISP_CID_DMSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_DMSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_SIZE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R3:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_AREA_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R3:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T3:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_SIZE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THR_SHIFT1:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_dmsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DMSC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_CAC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SKIN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_RESET:
+        case VVCAM_ISP_CID_DMSC_MODE:
+        case VVCAM_ISP_CID_DMSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_DMSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_SIZE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R3:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_AREA_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R3:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T3:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_SIZE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THR_SHIFT1:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_STAT_DIR_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_STAT_DIR_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_T1:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_R1:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_R2:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_R1:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_R2:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_T1:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_AREA_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R2:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R3:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T3:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_SIZE:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R2:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_THR_SHIFT1:
+        case VVCAM_ISP_CID_DMSC_STAT_SKIN_CB_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_STAT_SKIN_CB_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_STAT_SKIN_CR_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_STAT_SKIN_CR_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_STAT_SKIN_Y_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_STAT_SKIN_Y_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_dmsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_dmsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_dmsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_dmsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_CAC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_cac_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_demoire_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_dpl_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_sharpen_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_sharpen_line_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_SKIN_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_skin_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 0: Manual 1: Auto */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* bool 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* bool 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_fac_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_fac_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_clip_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_clip_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_t4_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_denoise_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_cbcr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_threshold",
+        .step = 1,
+        .def  = 8,
+        .min  = 8,
+        .max  = 255,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dir_thr_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dir_thr_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T1,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_edge_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_edge_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_edge_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_edge_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_AREA_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_area_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_clip_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_clip_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_fac_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_fac_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_t4_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DENOISE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_denoise_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THR_SHIFT1,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_t_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* int16_t  */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_skin_cb_thr_max",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_skin_cb_thr_min",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_skin_cr_thr_max",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_skin_cr_thr_min",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_skin_y_thr_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_skin_y_thr_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_cbcr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_thr",
+        .step = 1,
+        .def  = 8,
+        .min  = 8,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_a_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_a_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_b_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_b_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_c_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_c_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* according to image width */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_ctr_h_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        /* according to image height */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_ctr_v_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DIR_THR_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dir_thr_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DIR_THR_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dir_thr_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_T1,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_edge_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_edge_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_edge_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_edge_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_AREA_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_area_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_clip_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_clip_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_fac_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_fac_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_t2_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_t4_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DENOISE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_denoise_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_THR_SHIFT1,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_t_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* int16_t  */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SKIN_CB_THR_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_skin_cb_thr_max",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SKIN_CB_THR_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_skin_cb_thr_min",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SKIN_CR_THR_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_skin_cr_thr_max",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SKIN_CR_THR_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_skin_cr_thr_min",
+        .step = 1,
+        .min  = -2047,
+        .max  = 2047,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SKIN_Y_THR_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_skin_y_thr_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SKIN_Y_THR_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_skin_y_thr_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CBCR_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_cbcr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_thr",
+        .step = 1,
+        .def  = 8,
+        .min  = 8,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_a_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_a_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_b_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_b_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_c_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_c_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* according to image width */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_ctr_h_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        /* according to image height */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_ctr_v_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+};
+
+int vvcam_isp_dmsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_dmsc_ctrls);
+}
+
+int vvcam_isp_dmsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_dmsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_dmsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp dmsc ctrl %s failed %d.\n",
+                vvcam_isp_dmsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.h
new file mode 100644
index 000000000000..67d0602de191
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v2/vvcam_isp_dmsc.h
@@ -0,0 +1,268 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DMSC_H__
+#define __VVCAM_ISP_DMSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_DMSC_ENABLE           (VVCAM_ISP_CID_DMSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_DMSC_CAC_ENABLE       (VVCAM_ISP_CID_DMSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE   (VVCAM_ISP_CID_DMSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE  (VVCAM_ISP_CID_DMSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE   (VVCAM_ISP_CID_DMSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_DMSC_SKIN_ENABLE      (VVCAM_ISP_CID_DMSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_DMSC_RESET            (VVCAM_ISP_CID_DMSC_BASE + 0x0007)
+#define VVCAM_ISP_CID_DMSC_MODE             (VVCAM_ISP_CID_DMSC_BASE + 0x0008)
+#define VVCAM_ISP_CID_DMSC_AUTO_LEVEL       (VVCAM_ISP_CID_DMSC_BASE + 0x0009)
+#define VVCAM_ISP_CID_DMSC_AUTO_GAIN        (VVCAM_ISP_CID_DMSC_BASE + 0x000A)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000B)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000C)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000D)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000E)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000F)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0010)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T1  (VVCAM_ISP_CID_DMSC_BASE + 0x0011)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0012)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T3  (VVCAM_ISP_CID_DMSC_BASE + 0x0013)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0014)
+#define VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0015)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_SIZE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0016)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R1  (VVCAM_ISP_CID_DMSC_BASE + 0x0017)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R2  (VVCAM_ISP_CID_DMSC_BASE + 0x0018)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_R3  (VVCAM_ISP_CID_DMSC_BASE + 0x0019)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001A)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001B)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001C)
+#define VVCAM_ISP_CID_DMSC_MANU_THRESHOLD   (VVCAM_ISP_CID_DMSC_BASE + 0x001D)
+#define VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MAX (VVCAM_ISP_CID_DMSC_BASE + 0x001E)
+#define VVCAM_ISP_CID_DMSC_MANU_DIR_THR_MIN (VVCAM_ISP_CID_DMSC_BASE + 0x001F)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T1  (VVCAM_ISP_CID_DMSC_BASE + 0x0020)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0021)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R1  (VVCAM_ISP_CID_DMSC_BASE + 0x0022)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_R2  (VVCAM_ISP_CID_DMSC_BASE + 0x0023)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0024)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0025)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0026)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_EDGE_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0027)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_AREA_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0028)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0029)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002A)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002B)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002C)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002D)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R1  (VVCAM_ISP_CID_DMSC_BASE + 0x002E)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R2  (VVCAM_ISP_CID_DMSC_BASE + 0x002F)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_R3  (VVCAM_ISP_CID_DMSC_BASE + 0x0030)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T1  (VVCAM_ISP_CID_DMSC_BASE + 0x0031)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T3  (VVCAM_ISP_CID_DMSC_BASE + 0x0032)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_SIZE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0033)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0034)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0035)
+#define VVCAM_ISP_CID_DMSC_MANU_DENOISE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0036)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0037)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0038)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0039)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THRESHOLD \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003A)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_THR_SHIFT1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003B)
+#define VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003C)
+#define VVCAM_ISP_CID_DMSC_MANU_SKIN_CB_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003D)
+#define VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003E)
+#define VVCAM_ISP_CID_DMSC_MANU_SKIN_CR_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003F)
+#define VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0040)
+#define VVCAM_ISP_CID_DMSC_MANU_SKIN_Y_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0041)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0042)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0043)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0044)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0045)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0046)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0047)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0048)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0049)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x004A)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004B)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004C)
+#define VVCAM_ISP_CID_DMSC_STAT_THRESHOLD   (VVCAM_ISP_CID_DMSC_BASE + 0x004D)
+#define VVCAM_ISP_CID_DMSC_STAT_DIR_THR_MAX (VVCAM_ISP_CID_DMSC_BASE + 0x004E)
+#define VVCAM_ISP_CID_DMSC_STAT_DIR_THR_MIN (VVCAM_ISP_CID_DMSC_BASE + 0x004F)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_T1  (VVCAM_ISP_CID_DMSC_BASE + 0x0050)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0051)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_R1  (VVCAM_ISP_CID_DMSC_BASE + 0x0052)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_R2  (VVCAM_ISP_CID_DMSC_BASE + 0x0053)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0054)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0055)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0056)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_EDGE_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0057)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_AREA_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0058)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0059)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005A)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005B)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005C)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005D)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R1  (VVCAM_ISP_CID_DMSC_BASE + 0x005E)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R2  (VVCAM_ISP_CID_DMSC_BASE + 0x005F)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_R3  (VVCAM_ISP_CID_DMSC_BASE + 0x0060)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T1  (VVCAM_ISP_CID_DMSC_BASE + 0x0061)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T3  (VVCAM_ISP_CID_DMSC_BASE + 0x0062)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_SIZE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0063)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0064)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0065)
+#define VVCAM_ISP_CID_DMSC_STAT_DENOISE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0066)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0067)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0068)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0069)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_THRESHOLD \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006A)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_THR_SHIFT1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006B)
+#define VVCAM_ISP_CID_DMSC_STAT_SKIN_CB_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006C)
+#define VVCAM_ISP_CID_DMSC_STAT_SKIN_CB_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006D)
+#define VVCAM_ISP_CID_DMSC_STAT_SKIN_CR_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006E)
+#define VVCAM_ISP_CID_DMSC_STAT_SKIN_CR_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006F)
+#define VVCAM_ISP_CID_DMSC_STAT_SKIN_Y_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0070)
+#define VVCAM_ISP_CID_DMSC_STAT_SKIN_Y_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0071)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CBCR_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0072)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0073)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0074)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0075)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0076)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0077)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0078)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0079)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x007A)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x007B)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x007C)
+
+#ifdef __KERNEL__
+int vvcam_isp_dmsc_ctrl_count(void);
+int vvcam_isp_dmsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.c
new file mode 100644
index 000000000000..cdfb5ed67c57
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.c
@@ -0,0 +1,2241 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_dmsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_dmsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DMSC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_CAC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEFALSE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_INTERP_CORNER_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DECLINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_RESET:
+        case VVCAM_ISP_CID_DMSC_MODE:
+        case VVCAM_ISP_CID_DMSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_DMSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_DMSC_AUTO_INTERP_DIR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_INTERP_LARGE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_DECLINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_HIGH_LIGHT_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_RANGE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R3:
+        case VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_DIFF_COLOR_COEF:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_LOW:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_DEBURST_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT_ISO:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT_ISO:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_REFINE_G_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_SHIFT1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T3:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R3:
+        case VVCAM_ISP_CID_DMSC_MANU_DEFALSE_COLOR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_DEFALSE_CBCR_CLASS_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEFALSE_GRAY_PROTECT_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_GRAD_CONTROL:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_THR_LOW:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_COLOR_RANGE_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_HIGH_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_HIGH_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_LARGE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR0_INTERP_TYPE:
+        case VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_SHIFT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_dmsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DMSC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_CAC_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DEFALSE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_INTERP_CORNER_ENABLE:
+        case VVCAM_ISP_CID_DMSC_DECLINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_RESET:
+        case VVCAM_ISP_CID_DMSC_MODE:
+        case VVCAM_ISP_CID_DMSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_DMSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_DMSC_AUTO_INTERP_DIR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_INTERP_LARGE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_DECLINE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_HIGH_LIGHT_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_RANGE:
+        case VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T3:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R1:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R2:
+        case VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R3:
+        case VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_DIFF_COLOR_COEF:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_LOW:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MIN:
+        case VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MAX:
+        case VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_DEBURST_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT_ISO:
+        case VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT_ISO:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET:
+        case VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_REFINE_G_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_SHIFT1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T3:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R1:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R2:
+        case VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R3:
+        case VVCAM_ISP_CID_DMSC_MANU_DEFALSE_COLOR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_DEFALSE_CBCR_CLASS_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEFALSE_GRAY_PROTECT_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_GRAD_CONTROL:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_THR_LOW:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_SHIFT:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_COLOR_RANGE_MODE:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_HIGH_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_HIGH_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_LARGE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR0_INTERP_TYPE:
+        case VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_GRAD_THR_MIN:
+        case VVCAM_ISP_CID_DMSC_STAT_GRAD_THR_MAX:
+        case VVCAM_ISP_CID_DMSC_STAT_DIFF_COLOR_COEF:
+        case VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_THR_LOW:
+        case VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_THR_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MIN:
+        case VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MAX:
+        case VVCAM_ISP_CID_DMSC_STAT_HIGH_LIGH_CONTROL_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_HIGH_LIGH_CONTROL_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_DEBURST_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_PURE_G_WEIGHT:
+        case VVCAM_ISP_CID_DMSC_STAT_PURE_RB_WEIGHT:
+        case VVCAM_ISP_CID_DMSC_STAT_PURE_G_WEIGHT_ISO:
+        case VVCAM_ISP_CID_DMSC_STAT_PURE_RB_WEIGHT_ISO:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET:
+        case VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_REFINE_G_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_T1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_SHIFT1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R2:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_DENOISE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_WHITE:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_BLACK:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_WHITE:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_BLACK:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T2_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T3:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T4_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R1:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R2:
+        case VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R3:
+        case VVCAM_ISP_CID_DMSC_STAT_DEFALSE_COLOR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_DEFALSE_CBCR_CLASS_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DEFALSE_GRAY_PROTECT_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_GRAD_CONTROL:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_PROTECT_THR_LOW:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_PROTECT_SHIFT:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CBCR_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_SAT_SHRINK:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_COLOR_RANGE_MODE:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CB_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CB_HIGH_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CR_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CR_HIGH_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_DIR_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_LARGE_STRENGTH:
+        case VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_DIR0_INTERP_TYPE:
+        case VVCAM_ISP_CID_DMSC_STAT_DECLINE_CORNER_LOW_THR:
+        case VVCAM_ISP_CID_DMSC_STAT_DECLINE_CORNER_SHIFT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_dmsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_dmsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_dmsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_dmsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_CAC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_cac_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_demoire_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_dpl_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_sharpen_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_sharpen_line_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_DEFALSE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_defalse_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_INTERP_CORNER_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_interp_corner_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_DECLINE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_decline_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 0: Manual 1: Auto */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_INTERP_DIR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_interp_dir_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_INTERP_LARGE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_interp_large_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* bool 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DECLINE_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_decline_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_HIGH_LIGHT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_high_light_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* bool 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x8x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_RANGE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_dpl_cbcr_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {20, 8, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_denoise_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* bool_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x6x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {20, 6, 0, 0},
+    },
+    {
+        /* uint16_t 20x6x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_clip",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 6, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_t2_s",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_t4_s",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 1, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv0_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_t2_s",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_t4_s",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 1, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv1_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv2_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv2_t2_s",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv2_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv2_t4_s",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv2_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 1, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv2_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_auto_sharpen_crv_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_grad_thr_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_grad_thr_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DIFF_COLOR_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_diff_color_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_adpt_coef_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_LOW,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_adpt_coef_thr_low",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_adpt_coef_thr_st",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_adpt_coef_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_adpt_coef_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_hl_ctrl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_hl_ctrl_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEBURST_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_deburst_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_pure_g_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_pure_rb_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT_ISO,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_pure_g_wt_iso",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT_ISO,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_pure_rb_wt_iso",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_a_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_a_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_b_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_b_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_c_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_c_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* according to image width */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_ctr_h_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        /* according to image height */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_cac_ctr_v_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_REFINE_G_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dmr_refine_g_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_SHIFT1,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_st1",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_line_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_DENOISE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_denoise_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_fac_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_fac_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_clip_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_clip_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_st2",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_st4",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_sharpen_curve_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEFALSE_COLOR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_defalse_color_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEFALSE_CBCR_CLASS_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_defalse_cbcr_ct",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEFALSE_GRAY_PROTECT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_defalse_gps",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_GRAD_CONTROL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_grad_ctrl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_THR_LOW,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_prot_thr_low",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_prot_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_cbcr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_COLOR_RANGE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_crm",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_LOW_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_cb_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_HIGH_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_cb_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_LOW_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_cr_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_HIGH_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dpl_cr_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_ic_dir_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_LARGE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_ic_large_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR0_INTERP_TYPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_ic_dir0_int_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_LOW_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dc_cor_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_manu_dc_cor_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_GRAD_THR_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_grad_thr_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_GRAD_THR_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_grad_thr_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DIFF_COLOR_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_diff_color_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_adpt_coef_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_THR_LOW,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_adpt_coef_thr_low",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_THR_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_adpt_coef_thr_st",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_adpt_coef_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_adpt_coef_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_HIGH_LIGH_CONTROL_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_hl_ctrl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_HIGH_LIGH_CONTROL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_hl_ctrl_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEBURST_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_deburst_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_PURE_G_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_pure_g_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_PURE_RB_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_pure_rb_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_PURE_G_WEIGHT_ISO,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_pure_g_wt_iso",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_PURE_RB_WEIGHT_ISO,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_pure_rb_wt_iso",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_a_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_a_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_b_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_b_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_c_blue",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* float -16.0~15.9375*/
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_c_red",
+        .step = 1,
+        .min  = -256,
+        .max  = 255,
+    },
+    {
+        /* according to image width */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_ctr_h_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        /* according to image height */
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_cac_ctr_v_offset",
+        .step = 1,
+        .min  = -32768,
+        .max  = 32767,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_REFINE_G_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dmr_refine_g_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_SHIFT1,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_st1",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_line_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_DENOISE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_denoise_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_fac_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_fac_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_WHITE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_clip_wht",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_BLACK,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_clip_blk",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_t1",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T2_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_st2",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_t3",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T4_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_st4",
+        .step = 1,
+        .min  = 0,
+        .max  = 11,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R1,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_r1",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_r2",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R3,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_sharpen_curve_r3",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEFALSE_COLOR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_defalse_color_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEFALSE_CBCR_CLASS_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_defalse_cbcr_ct",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEFALSE_GRAY_PROTECT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_defalse_gps",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_GRAD_CONTROL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_grad_ctrl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_PROTECT_THR_LOW,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_prot_thr_low",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_PROTECT_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_prot_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CBCR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_cbcr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_SAT_SHRINK,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_sat_shrink",
+        .step = 1,
+        .min  = 0,
+        .max  = 8,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_COLOR_RANGE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_crm",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CB_LOW_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_cb_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CB_HIGH_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_cb_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CR_LOW_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_cr_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CR_HIGH_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dpl_cr_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_DIR_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_ic_dir_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_LARGE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_ic_large_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_DIR0_INTERP_TYPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_ic_dir0_int_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DECLINE_CORNER_LOW_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dc_cor_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dmsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DMSC_STAT_DECLINE_CORNER_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dmsc_stat_dc_cor_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_dmsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_dmsc_ctrls);
+}
+
+int vvcam_isp_dmsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_dmsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_dmsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp dmsc ctrl %s failed %d.\n",
+                vvcam_isp_dmsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.h
new file mode 100644
index 000000000000..9f359808c21e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dmsc/dmsc_v3/vvcam_isp_dmsc.h
@@ -0,0 +1,376 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DMSC_H__
+#define __VVCAM_ISP_DMSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_DMSC_ENABLE           (VVCAM_ISP_CID_DMSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_DMSC_CAC_ENABLE       (VVCAM_ISP_CID_DMSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_DMSC_DEMOIRE_ENABLE   (VVCAM_ISP_CID_DMSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_DMSC_DEPURPLE_ENABLE  (VVCAM_ISP_CID_DMSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_DMSC_SHARPEN_ENABLE   (VVCAM_ISP_CID_DMSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_DMSC_SHARPEN_LINE_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_DMSC_DEFALSE_ENABLE   (VVCAM_ISP_CID_DMSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_DMSC_INTERP_CORNER_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0007)
+#define VVCAM_ISP_CID_DMSC_DECLINE_ENABLE   (VVCAM_ISP_CID_DMSC_BASE + 0x0008)
+#define VVCAM_ISP_CID_DMSC_RESET            (VVCAM_ISP_CID_DMSC_BASE + 0x0009)
+#define VVCAM_ISP_CID_DMSC_MODE             (VVCAM_ISP_CID_DMSC_BASE + 0x000A)
+#define VVCAM_ISP_CID_DMSC_AUTO_LEVEL       (VVCAM_ISP_CID_DMSC_BASE + 0x000B)
+#define VVCAM_ISP_CID_DMSC_AUTO_GAIN        (VVCAM_ISP_CID_DMSC_BASE + 0x000C)
+#define VVCAM_ISP_CID_DMSC_AUTO_INTERP_DIR_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000D)
+#define VVCAM_ISP_CID_DMSC_AUTO_INTERP_LARGE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000E)
+#define VVCAM_ISP_CID_DMSC_AUTO_DECLINE_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x000F)
+#define VVCAM_ISP_CID_DMSC_AUTO_HIGH_LIGHT_THRESHOLD \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0010)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0011)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0012)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0013)
+#define VVCAM_ISP_CID_DMSC_AUTO_DEPURPLE_CBCR_RANGE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0014)
+#define VVCAM_ISP_CID_DMSC_AUTO_DENOISE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0015)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_ENABLE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0016)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_FACTOR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0017)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CLIP \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0018)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0019)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001A)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001B)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001C)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001D)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001E)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE0_R3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x001F)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0020)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0021)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0022)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0023)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0024)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0025)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE1_R3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0026)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0027)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0028)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0029)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002A)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002B)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002C)
+#define VVCAM_ISP_CID_DMSC_AUTO_SHARPEN_CURVE2_R3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002D)
+#define VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002E)
+#define VVCAM_ISP_CID_DMSC_MANU_GRAD_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x002F)
+#define VVCAM_ISP_CID_DMSC_MANU_DIFF_COLOR_COEF \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0030)
+#define VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0031)
+#define VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_LOW \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0032)
+#define VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_THR_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0033)
+#define VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0034)
+#define VVCAM_ISP_CID_DMSC_MANU_ADAP_COEF_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0035)
+#define VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0036)
+#define VVCAM_ISP_CID_DMSC_MANU_HIGH_LIGH_CONTROL_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0037)
+#define VVCAM_ISP_CID_DMSC_MANU_DEBURST_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0038)
+#define VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0039)
+#define VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003A)
+#define VVCAM_ISP_CID_DMSC_MANU_PURE_G_WEIGHT_ISO \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003B)
+#define VVCAM_ISP_CID_DMSC_MANU_PURE_RB_WEIGHT_ISO \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x003C)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_A_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x003D)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_A_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x003E)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_B_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x003F)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_B_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0040)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_C_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0041)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_C_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x0042)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_H_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0043)
+#define VVCAM_ISP_CID_DMSC_MANU_CAC_CENTER_V_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0044)
+#define VVCAM_ISP_CID_DMSC_MANU_DEMOIRE_REFINE_G_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0045)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0046)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0047)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_SHIFT1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0048)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0049)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_LINE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004A)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_DENOISE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004B)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004C)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_FACTOR_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004D)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004E)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CLIP_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x004F)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0050)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0051)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0052)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0053)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0054)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0055)
+#define VVCAM_ISP_CID_DMSC_MANU_SHARPEN_CURVE_R3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0056)
+#define VVCAM_ISP_CID_DMSC_MANU_DEFALSE_COLOR_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0057)
+#define VVCAM_ISP_CID_DMSC_MANU_DEFALSE_CBCR_CLASS_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0058)
+#define VVCAM_ISP_CID_DMSC_MANU_DEFALSE_GRAY_PROTECT_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0059)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_GRAD_CONTROL \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005A)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_THR_LOW \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005B)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_PROTECT_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005C)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005D)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CBCR_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005E)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x005F)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_COLOR_RANGE_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0060)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_LOW_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0061)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CB_HIGH_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0062)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_LOW_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0063)
+#define VVCAM_ISP_CID_DMSC_MANU_DEPURPLE_CR_HIGH_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0064)
+#define VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0065)
+#define VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_LARGE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0066)
+#define VVCAM_ISP_CID_DMSC_MANU_INTERP_CORNER_DIR0_INTERP_TYPE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0067)
+#define VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_LOW_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0068)
+#define VVCAM_ISP_CID_DMSC_MANU_DECLINE_CORNER_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0069)
+#define VVCAM_ISP_CID_DMSC_STAT_GRAD_THR_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006A)
+#define VVCAM_ISP_CID_DMSC_STAT_GRAD_THR_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006B)
+#define VVCAM_ISP_CID_DMSC_STAT_DIFF_COLOR_COEF \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006C)
+#define VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006D)
+#define VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_THR_LOW \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006E)
+#define VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_THR_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x006F)
+#define VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MIN \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0070)
+#define VVCAM_ISP_CID_DMSC_STAT_ADAP_COEF_MAX \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0071)
+#define VVCAM_ISP_CID_DMSC_STAT_HIGH_LIGH_CONTROL_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0072)
+#define VVCAM_ISP_CID_DMSC_STAT_HIGH_LIGH_CONTROL_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0073)
+#define VVCAM_ISP_CID_DMSC_STAT_DEBURST_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0074)
+#define VVCAM_ISP_CID_DMSC_STAT_PURE_G_WEIGHT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0075)
+#define VVCAM_ISP_CID_DMSC_STAT_PURE_RB_WEIGHT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0076)
+#define VVCAM_ISP_CID_DMSC_STAT_PURE_G_WEIGHT_ISO \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0077)
+#define VVCAM_ISP_CID_DMSC_STAT_PURE_RB_WEIGHT_ISO \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0078)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_A_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x0079)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_A_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x007A)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_B_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x007B)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_B_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x007C)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_C_BLUE  (VVCAM_ISP_CID_DMSC_BASE + 0x007D)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_C_RED   (VVCAM_ISP_CID_DMSC_BASE + 0x007E)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_H_OFFSET\
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x007F)
+#define VVCAM_ISP_CID_DMSC_STAT_CAC_CENTER_V_OFFSET \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0080)
+#define VVCAM_ISP_CID_DMSC_STAT_DEMOIRE_REFINE_G_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0081)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0082)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0083)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_SHIFT1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0084)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0085)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_LINE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0086)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_DENOISE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0087)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0088)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_FACTOR_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0089)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_WHITE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x008A)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CLIP_BLACK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x008B)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x008C)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T2_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x008D)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x008E)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_T4_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x008F)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R1 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0090)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R2 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0091)
+#define VVCAM_ISP_CID_DMSC_STAT_SHARPEN_CURVE_R3 \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0092)
+#define VVCAM_ISP_CID_DMSC_STAT_DEFALSE_COLOR_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0093)
+#define VVCAM_ISP_CID_DMSC_STAT_DEFALSE_CBCR_CLASS_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0094)
+#define VVCAM_ISP_CID_DMSC_STAT_DEFALSE_GRAY_PROTECT_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0095)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_GRAD_CONTROL \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0096)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_PROTECT_THR_LOW \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0097)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_PROTECT_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0098)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x0099)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CBCR_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x009A)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_SAT_SHRINK \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x009B)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_COLOR_RANGE_MODE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x009C)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CB_LOW_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x009D)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CB_HIGH_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x009E)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CR_LOW_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x009F)
+#define VVCAM_ISP_CID_DMSC_STAT_DEPURPLE_CR_HIGH_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x00A0)
+#define VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_DIR_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x00A1)
+#define VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_LARGE_STRENGTH \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x00A2)
+#define VVCAM_ISP_CID_DMSC_STAT_INTERP_CORNER_DIR0_INTERP_TYPE \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x00A3)
+#define VVCAM_ISP_CID_DMSC_STAT_DECLINE_CORNER_LOW_THR \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x00A4)
+#define VVCAM_ISP_CID_DMSC_STAT_DECLINE_CORNER_SHIFT \
+                                            (VVCAM_ISP_CID_DMSC_BASE + 0x00A5)
+
+#ifdef __KERNEL__
+int vvcam_isp_dmsc_ctrl_count(void);
+int vvcam_isp_dmsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.c
new file mode 100644
index 000000000000..45e857a4208b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.c
@@ -0,0 +1,551 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_dpcc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_dpcc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DPCC_ENABLE:
+        case VVCAM_ISP_CID_DPCC_RESET:
+        case VVCAM_ISP_CID_DPCC_MODE:
+        case VVCAM_ISP_CID_DPCC_AUTO_MODE_SELECT:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_ENABLE:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_NUM:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_OUT_MODE:
+        case VVCAM_ISP_CID_DPCC_MANU_OUT_MODE:
+        case VVCAM_ISP_CID_DPCC_MANU_SET_USE:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_POS_X:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_POS_Y:
+        case VVCAM_ISP_CID_DPCC_MANU_METHODS_SET:
+        case VVCAM_ISP_CID_DPCC_MANU_LINE_MAD_FACTOR:
+        case VVCAM_ISP_CID_DPCC_MANU_LINE_THRESHOLD:
+        case VVCAM_ISP_CID_DPCC_MANU_PG_FACTOR:
+        case VVCAM_ISP_CID_DPCC_MANU_RG_FACTOR:
+        case VVCAM_ISP_CID_DPCC_MANU_RND_OFFSETS:
+        case VVCAM_ISP_CID_DPCC_MANU_RND_THRESHOLD:
+        case VVCAM_ISP_CID_DPCC_MANU_RO_LIMITS:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_dpcc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DPCC_ENABLE:
+        case VVCAM_ISP_CID_DPCC_RESET:
+        case VVCAM_ISP_CID_DPCC_MODE:
+        case VVCAM_ISP_CID_DPCC_AUTO_MODE_SELECT:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_ENABLE:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_NUM:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_OUT_MODE:
+        case VVCAM_ISP_CID_DPCC_MANU_OUT_MODE:
+        case VVCAM_ISP_CID_DPCC_MANU_SET_USE:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_POS_X:
+        case VVCAM_ISP_CID_DPCC_MANU_BPT_POS_Y:
+        case VVCAM_ISP_CID_DPCC_MANU_METHODS_SET:
+        case VVCAM_ISP_CID_DPCC_MANU_LINE_MAD_FACTOR:
+        case VVCAM_ISP_CID_DPCC_MANU_LINE_THRESHOLD:
+        case VVCAM_ISP_CID_DPCC_MANU_PG_FACTOR:
+        case VVCAM_ISP_CID_DPCC_MANU_RG_FACTOR:
+        case VVCAM_ISP_CID_DPCC_MANU_RND_OFFSETS:
+        case VVCAM_ISP_CID_DPCC_MANU_RND_THRESHOLD:
+        case VVCAM_ISP_CID_DPCC_MANU_RO_LIMITS:
+        case VVCAM_ISP_CID_DPCC_STAT_BPT_ENABLE:
+        case VVCAM_ISP_CID_DPCC_STAT_BPT_NUM:
+        case VVCAM_ISP_CID_DPCC_STAT_BPT_OUT_MODE:
+        case VVCAM_ISP_CID_DPCC_STAT_OUT_MODE:
+        case VVCAM_ISP_CID_DPCC_STAT_SET_USE:
+        case VVCAM_ISP_CID_DPCC_STAT_BPT_POS_X:
+        case VVCAM_ISP_CID_DPCC_STAT_BPT_POS_Y:
+        case VVCAM_ISP_CID_DPCC_STAT_METHODS_SET:
+        case VVCAM_ISP_CID_DPCC_STAT_LINE_MAD_FACTOR:
+        case VVCAM_ISP_CID_DPCC_STAT_LINE_THRESHOLD:
+        case VVCAM_ISP_CID_DPCC_STAT_PG_FACTOR:
+        case VVCAM_ISP_CID_DPCC_STAT_RG_FACTOR:
+        case VVCAM_ISP_CID_DPCC_STAT_RND_OFFSETS:
+        case VVCAM_ISP_CID_DPCC_STAT_RND_THRESHOLD:
+        case VVCAM_ISP_CID_DPCC_STAT_RO_LIMITS:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_dpcc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_dpcc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_dpcc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_dpcc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 0: Manual 1: Auto */
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_AUTO_MODE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_auto_mode_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_BPT_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_bpt_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_BPT_NUM,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_bpt_num",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_BPT_OUT_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_bpt_out_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_OUT_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_out_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_SET_USE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_set_use",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_BPT_POS_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_bpt_pos_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2048},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_BPT_POS_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_bpt_pos_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 8191,
+        .dims = {2048},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_METHODS_SET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_methods_set",
+        .step = 1,
+        .min  = 0,
+        .max  = 8191,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_LINE_MAD_FACTOR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_line_mad_factor",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_LINE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_line_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_PG_FACTOR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_pg_factor",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_RG_FACTOR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_rg_factor",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_RND_OFFSETS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_rnd_offsets",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_RND_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_rnd_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_MANU_RO_LIMITS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_manu_ro_limits",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_BPT_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_bpt_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_BPT_NUM,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_bpt_num",
+        .step = 1,
+        .min  = 0,
+        .max  = 2047,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_BPT_OUT_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_bpt_out_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_OUT_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_out_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_SET_USE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_set_use",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_BPT_POS_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_bpt_pos_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {2048},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_BPT_POS_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_bpt_pos_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 8191,
+        .dims = {2048},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_METHODS_SET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_methods_set",
+        .step = 1,
+        .min  = 0,
+        .max  = 8191,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_LINE_MAD_FACTOR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_line_mad_factor",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_LINE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_line_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_PG_FACTOR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_pg_factor",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_RG_FACTOR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_rg_factor",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_RND_OFFSETS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_rnd_offsets",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_RND_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_rnd_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {2, 3, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpcc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPCC_STAT_RO_LIMITS,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpcc_stat_ro_limits",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+        .dims = {2, 3, 0, 0},
+    },
+};
+
+int vvcam_isp_dpcc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_dpcc_ctrls);
+}
+
+int vvcam_isp_dpcc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_dpcc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_dpcc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp dpcc ctrl %s failed %d.\n",
+                vvcam_isp_dpcc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.h
new file mode 100644
index 000000000000..a5ea23f5248e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpcc/dpcc/vvcam_isp_dpcc.h
@@ -0,0 +1,111 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DPCC_H__
+#define __VVCAM_ISP_DPCC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_DPCC_ENABLE           (VVCAM_ISP_CID_DPCC_BASE + 0x0000)
+#define VVCAM_ISP_CID_DPCC_RESET            (VVCAM_ISP_CID_DPCC_BASE + 0x0001)
+#define VVCAM_ISP_CID_DPCC_MODE             (VVCAM_ISP_CID_DPCC_BASE + 0x0002)
+
+#define VVCAM_ISP_CID_DPCC_AUTO_MODE_SELECT (VVCAM_ISP_CID_DPCC_BASE + 0x0003)
+
+#define VVCAM_ISP_CID_DPCC_MANU_BPT_ENABLE  (VVCAM_ISP_CID_DPCC_BASE + 0x000F)
+#define VVCAM_ISP_CID_DPCC_MANU_BPT_NUM     (VVCAM_ISP_CID_DPCC_BASE + 0x0010)
+#define VVCAM_ISP_CID_DPCC_MANU_BPT_OUT_MODE \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x0011)
+#define VVCAM_ISP_CID_DPCC_MANU_OUT_MODE    (VVCAM_ISP_CID_DPCC_BASE + 0x0012)
+#define VVCAM_ISP_CID_DPCC_MANU_SET_USE     (VVCAM_ISP_CID_DPCC_BASE + 0x0013)
+#define VVCAM_ISP_CID_DPCC_MANU_BPT_POS_X   (VVCAM_ISP_CID_DPCC_BASE + 0x0014)
+#define VVCAM_ISP_CID_DPCC_MANU_BPT_POS_Y   (VVCAM_ISP_CID_DPCC_BASE + 0x0015)
+#define VVCAM_ISP_CID_DPCC_MANU_METHODS_SET (VVCAM_ISP_CID_DPCC_BASE + 0x0016)
+#define VVCAM_ISP_CID_DPCC_MANU_LINE_MAD_FACTOR \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x0017)
+#define VVCAM_ISP_CID_DPCC_MANU_LINE_THRESHOLD \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x0018)
+#define VVCAM_ISP_CID_DPCC_MANU_PG_FACTOR   (VVCAM_ISP_CID_DPCC_BASE + 0x0019)
+#define VVCAM_ISP_CID_DPCC_MANU_RG_FACTOR   (VVCAM_ISP_CID_DPCC_BASE + 0x001A)
+#define VVCAM_ISP_CID_DPCC_MANU_RND_OFFSETS (VVCAM_ISP_CID_DPCC_BASE + 0x001B)
+#define VVCAM_ISP_CID_DPCC_MANU_RND_THRESHOLD \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x001C)
+#define VVCAM_ISP_CID_DPCC_MANU_RO_LIMITS   (VVCAM_ISP_CID_DPCC_BASE + 0x001D)
+#define VVCAM_ISP_CID_DPCC_STAT_BPT_ENABLE  (VVCAM_ISP_CID_DPCC_BASE + 0x001E)
+#define VVCAM_ISP_CID_DPCC_STAT_BPT_NUM     (VVCAM_ISP_CID_DPCC_BASE + 0x001F)
+#define VVCAM_ISP_CID_DPCC_STAT_BPT_OUT_MODE \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x0020)
+#define VVCAM_ISP_CID_DPCC_STAT_OUT_MODE    (VVCAM_ISP_CID_DPCC_BASE + 0x0021)
+#define VVCAM_ISP_CID_DPCC_STAT_SET_USE     (VVCAM_ISP_CID_DPCC_BASE + 0x0022)
+#define VVCAM_ISP_CID_DPCC_STAT_BPT_POS_X   (VVCAM_ISP_CID_DPCC_BASE + 0x0023)
+#define VVCAM_ISP_CID_DPCC_STAT_BPT_POS_Y   (VVCAM_ISP_CID_DPCC_BASE + 0x0024)
+#define VVCAM_ISP_CID_DPCC_STAT_METHODS_SET (VVCAM_ISP_CID_DPCC_BASE + 0x0025)
+#define VVCAM_ISP_CID_DPCC_STAT_LINE_MAD_FACTOR \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x0026)
+#define VVCAM_ISP_CID_DPCC_STAT_LINE_THRESHOLD \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x0027)
+#define VVCAM_ISP_CID_DPCC_STAT_PG_FACTOR   (VVCAM_ISP_CID_DPCC_BASE + 0x0028)
+#define VVCAM_ISP_CID_DPCC_STAT_RG_FACTOR   (VVCAM_ISP_CID_DPCC_BASE + 0x0029)
+#define VVCAM_ISP_CID_DPCC_STAT_RND_OFFSETS (VVCAM_ISP_CID_DPCC_BASE + 0x002A)
+#define VVCAM_ISP_CID_DPCC_STAT_RND_THRESHOLD \
+                                            (VVCAM_ISP_CID_DPCC_BASE + 0x002B)
+#define VVCAM_ISP_CID_DPCC_STAT_RO_LIMITS   (VVCAM_ISP_CID_DPCC_BASE + 0x002C)
+
+#ifdef __KERNEL__
+int vvcam_isp_dpcc_ctrl_count(void);
+int vvcam_isp_dpcc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.c
new file mode 100644
index 000000000000..7b37fdd102cf
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.c
@@ -0,0 +1,503 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_dpf.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_dpf_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DPF_ENABLE:
+        case VVCAM_ISP_CID_DPF_RESET:
+        case VVCAM_ISP_CID_DPF_MODE:
+        case VVCAM_ISP_CID_DPF_AUTO_LEVEL:
+        case VVCAM_ISP_CID_DPF_AUTO_GAIN:
+        case VVCAM_ISP_CID_DPF_AUTO_GRADIENT:
+        case VVCAM_ISP_CID_DPF_AUTO_OFFSET:
+        case VVCAM_ISP_CID_DPF_AUTO_MIN:
+        case VVCAM_ISP_CID_DPF_AUTO_DIV:
+        case VVCAM_ISP_CID_DPF_AUTO_SIGMA_G:
+        case VVCAM_ISP_CID_DPF_AUTO_SIGMA_RB:
+        case VVCAM_ISP_CID_DPF_AUTO_NOISE_CURVE:
+        case VVCAM_ISP_CID_DPF_MANU_GAIN:
+        case VVCAM_ISP_CID_DPF_MANU_GRADIENT:
+        case VVCAM_ISP_CID_DPF_MANU_OFFSET:
+        case VVCAM_ISP_CID_DPF_MANU_MIN:
+        case VVCAM_ISP_CID_DPF_MANU_DIV:
+        case VVCAM_ISP_CID_DPF_MANU_SIGMA_G:
+        case VVCAM_ISP_CID_DPF_MANU_SIGMA_RB:
+        case VVCAM_ISP_CID_DPF_MANU_NOISE_CURVE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_dpf_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_DPF_ENABLE:
+        case VVCAM_ISP_CID_DPF_RESET:
+        case VVCAM_ISP_CID_DPF_MODE:
+        case VVCAM_ISP_CID_DPF_AUTO_LEVEL:
+        case VVCAM_ISP_CID_DPF_AUTO_GAIN:
+        case VVCAM_ISP_CID_DPF_AUTO_GRADIENT:
+        case VVCAM_ISP_CID_DPF_AUTO_OFFSET:
+        case VVCAM_ISP_CID_DPF_AUTO_MIN:
+        case VVCAM_ISP_CID_DPF_AUTO_DIV:
+        case VVCAM_ISP_CID_DPF_AUTO_SIGMA_G:
+        case VVCAM_ISP_CID_DPF_AUTO_SIGMA_RB:
+        case VVCAM_ISP_CID_DPF_AUTO_NOISE_CURVE:
+        case VVCAM_ISP_CID_DPF_MANU_GAIN:
+        case VVCAM_ISP_CID_DPF_MANU_GRADIENT:
+        case VVCAM_ISP_CID_DPF_MANU_OFFSET:
+        case VVCAM_ISP_CID_DPF_MANU_MIN:
+        case VVCAM_ISP_CID_DPF_MANU_DIV:
+        case VVCAM_ISP_CID_DPF_MANU_SIGMA_G:
+        case VVCAM_ISP_CID_DPF_MANU_SIGMA_RB:
+        case VVCAM_ISP_CID_DPF_MANU_NOISE_CURVE:
+        case VVCAM_ISP_CID_DPF_STAT_GAIN:
+        case VVCAM_ISP_CID_DPF_STAT_GRADIENT:
+        case VVCAM_ISP_CID_DPF_STAT_OFFSET:
+        case VVCAM_ISP_CID_DPF_STAT_MIN:
+        case VVCAM_ISP_CID_DPF_STAT_DIV:
+        case VVCAM_ISP_CID_DPF_STAT_SIGMA_G:
+        case VVCAM_ISP_CID_DPF_STAT_SIGMA_RB:
+        case VVCAM_ISP_CID_DPF_STAT_NOISE_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_dpf_ctrl_ops = {
+    .s_ctrl = vvcam_isp_dpf_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_dpf_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_dpf_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 0: Manual 1: Auto */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_GRADIENT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_gradient",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_OFFSET,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_DIV,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_div",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_sigma_g",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_sigma_rb",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x17 array */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_AUTO_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_auto_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 17, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_gain",
+        .step = 1,
+        .min  = 10,
+        .max  = 10000,
+        .def  = 10,
+    },
+    {
+        /* float 0.1 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_gradient",
+        .step = 1,
+        .min  = 1,
+        .max  = 1280,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1280,
+    },
+    {
+        /* float 1.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_min",
+        .step = 1,
+        .min  = 10,
+        .max  = 1280,
+        .def  = 10,
+    },
+    {
+        /* float 1.0 ~ 64.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_div",
+        .step = 1,
+        .min  = 10,
+        .max  = 650,
+        .def  = 10,
+    },
+    {
+        /* float 1.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_sigma_g",
+        .step = 1,
+        .min  = 10,
+        .max  = 1280,
+        .def  = 10,
+    },
+    {
+        /* float 1.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_sigma_rb",
+        .step = 1,
+        .min  = 10,
+        .max  = 1280,
+        .def  = 10,
+    },
+    {
+        /* float 17x array 0.0 ~ 4095.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_MANU_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_manu_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_gain",
+        .step = 1,
+        .min  = 10,
+        .max  = 10000,
+        .def  = 10,
+    },
+    {
+        /* float 0.1 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_gradient",
+        .step = 1,
+        .min  = 1,
+        .max  = 1280,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1280,
+    },
+    {
+        /* float 1.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_min",
+        .step = 1,
+        .min  = 10,
+        .max  = 1280,
+        .def  = 10,
+    },
+    {
+        /* float 1.0 ~ 64.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_div",
+        .step = 1,
+        .min  = 10,
+        .max  = 650,
+        .def  = 10,
+    },
+    {
+        /* float 1.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_sigma_g",
+        .step = 1,
+        .min  = 10,
+        .max  = 1280,
+        .def  = 10,
+    },
+    {
+        /* float 1.0 ~ 128.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_sigma_rb",
+        .step = 1,
+        .min  = 10,
+        .max  = 1280,
+        .def  = 10,
+    },
+    {
+        /* float 17x array 0.0 ~ 4095.0 */
+        .ops  = &vvcam_isp_dpf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_DPF_STAT_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_dpf_stat_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_dpf_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_dpf_ctrls);
+}
+
+int vvcam_isp_dpf_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_dpf_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_dpf_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp dpf ctrl %s failed %d.\n",
+                vvcam_isp_dpf_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.h
new file mode 100644
index 000000000000..7a04d108ec92
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/dpf/dpf_v1/vvcam_isp_dpf.h
@@ -0,0 +1,95 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DPF_H__
+#define __VVCAM_ISP_DPF_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_DPF_ENABLE            (VVCAM_ISP_CID_DPF_BASE + 0x0000)
+#define VVCAM_ISP_CID_DPF_RESET             (VVCAM_ISP_CID_DPF_BASE + 0x0001)
+#define VVCAM_ISP_CID_DPF_MODE              (VVCAM_ISP_CID_DPF_BASE + 0x0002)
+#define VVCAM_ISP_CID_DPF_AUTO_LEVEL        (VVCAM_ISP_CID_DPF_BASE + 0x0003)
+#define VVCAM_ISP_CID_DPF_AUTO_GAIN         (VVCAM_ISP_CID_DPF_BASE + 0x0004)
+#define VVCAM_ISP_CID_DPF_AUTO_GRADIENT     (VVCAM_ISP_CID_DPF_BASE + 0x0005)
+#define VVCAM_ISP_CID_DPF_AUTO_OFFSET       (VVCAM_ISP_CID_DPF_BASE + 0x0006)
+#define VVCAM_ISP_CID_DPF_AUTO_MIN          (VVCAM_ISP_CID_DPF_BASE + 0x0007)
+#define VVCAM_ISP_CID_DPF_AUTO_DIV          (VVCAM_ISP_CID_DPF_BASE + 0x0008)
+#define VVCAM_ISP_CID_DPF_AUTO_SIGMA_G      (VVCAM_ISP_CID_DPF_BASE + 0x0009)
+#define VVCAM_ISP_CID_DPF_AUTO_SIGMA_RB     (VVCAM_ISP_CID_DPF_BASE + 0x000A)
+#define VVCAM_ISP_CID_DPF_AUTO_NOISE_CURVE  (VVCAM_ISP_CID_DPF_BASE + 0x000B)
+#define VVCAM_ISP_CID_DPF_MANU_GAIN         (VVCAM_ISP_CID_DPF_BASE + 0x000C)
+#define VVCAM_ISP_CID_DPF_MANU_GRADIENT     (VVCAM_ISP_CID_DPF_BASE + 0x000D)
+#define VVCAM_ISP_CID_DPF_MANU_OFFSET       (VVCAM_ISP_CID_DPF_BASE + 0x000E)
+#define VVCAM_ISP_CID_DPF_MANU_MIN          (VVCAM_ISP_CID_DPF_BASE + 0x000F)
+#define VVCAM_ISP_CID_DPF_MANU_DIV          (VVCAM_ISP_CID_DPF_BASE + 0x0010)
+#define VVCAM_ISP_CID_DPF_MANU_SIGMA_G      (VVCAM_ISP_CID_DPF_BASE + 0x0011)
+#define VVCAM_ISP_CID_DPF_MANU_SIGMA_RB     (VVCAM_ISP_CID_DPF_BASE + 0x0012)
+#define VVCAM_ISP_CID_DPF_MANU_NOISE_CURVE  (VVCAM_ISP_CID_DPF_BASE + 0x0013)
+#define VVCAM_ISP_CID_DPF_STAT_GAIN         (VVCAM_ISP_CID_DPF_BASE + 0x0014)
+#define VVCAM_ISP_CID_DPF_STAT_GRADIENT     (VVCAM_ISP_CID_DPF_BASE + 0x0015)
+#define VVCAM_ISP_CID_DPF_STAT_OFFSET       (VVCAM_ISP_CID_DPF_BASE + 0x0016)
+#define VVCAM_ISP_CID_DPF_STAT_MIN          (VVCAM_ISP_CID_DPF_BASE + 0x0017)
+#define VVCAM_ISP_CID_DPF_STAT_DIV          (VVCAM_ISP_CID_DPF_BASE + 0x0018)
+#define VVCAM_ISP_CID_DPF_STAT_SIGMA_G      (VVCAM_ISP_CID_DPF_BASE + 0x0019)
+#define VVCAM_ISP_CID_DPF_STAT_SIGMA_RB     (VVCAM_ISP_CID_DPF_BASE + 0x001A)
+#define VVCAM_ISP_CID_DPF_STAT_NOISE_CURVE  (VVCAM_ISP_CID_DPF_BASE + 0x001B)
+
+#ifdef __KERNEL__
+int vvcam_isp_dpf_ctrl_count(void);
+int vvcam_isp_dpf_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v1/vvcam_isp_ee.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v1/vvcam_isp_ee.c
new file mode 100644
index 000000000000..f7834487a0ca
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v1/vvcam_isp_ee.c
@@ -0,0 +1,427 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ee.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ee_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EE_ENABLE:
+        case VVCAM_ISP_CID_EE_RESET:
+        case VVCAM_ISP_CID_EE_MODE:
+        case VVCAM_ISP_CID_EE_AUTO_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_AUTO_UV_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_UV_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ee_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EE_ENABLE:
+        case VVCAM_ISP_CID_EE_RESET:
+        case VVCAM_ISP_CID_EE_MODE:
+        case VVCAM_ISP_CID_EE_AUTO_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_AUTO_UV_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_UV_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_STAT_STRENGTH:
+        case VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_STAT_UV_GAIN:
+        case VVCAM_ISP_CID_EE_STAT_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_STAT_Y_GAIN_DOWN:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ee_ctrl_ops = {
+	.s_ctrl = vvcam_isp_ee_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ee_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ee_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20 * 32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_y_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_y_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_edge_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_EDGE_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_edge_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_y_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_y_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_EDGE_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_edge_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_Y_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_y_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_Y_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_y_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_ee_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ee_ctrls);
+}
+
+int vvcam_isp_ee_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ee_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ee_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ee ctrl %s failed %d.\n",
+                vvcam_isp_ee_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v1/vvcam_isp_ee.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v1/vvcam_isp_ee.h
new file mode 100644
index 000000000000..b2cb3711aa8b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v1/vvcam_isp_ee.h
@@ -0,0 +1,90 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_EE_H__
+#define __VVCAM_ISP_EE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_EE_ENABLE             (VVCAM_ISP_CID_EE_BASE + 0x0000)
+#define VVCAM_ISP_CID_EE_RESET              (VVCAM_ISP_CID_EE_BASE + 0x0001)
+#define VVCAM_ISP_CID_EE_MODE               (VVCAM_ISP_CID_EE_BASE + 0x0002)
+#define VVCAM_ISP_CID_EE_AUTO_LEVEL         (VVCAM_ISP_CID_EE_BASE + 0x0003)
+#define VVCAM_ISP_CID_EE_AUTO_GAIN          (VVCAM_ISP_CID_EE_BASE + 0x0004)
+#define VVCAM_ISP_CID_EE_AUTO_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x0005)
+#define VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0006)
+#define VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP     (VVCAM_ISP_CID_EE_BASE + 0x0007)
+#define VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN   (VVCAM_ISP_CID_EE_BASE + 0x0008)
+#define VVCAM_ISP_CID_EE_AUTO_UV_GAIN       (VVCAM_ISP_CID_EE_BASE + 0x0009)
+#define VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN     (VVCAM_ISP_CID_EE_BASE + 0x000A)
+#define VVCAM_ISP_CID_EE_MANU_EDGE_GAIN     (VVCAM_ISP_CID_EE_BASE + 0x000B)
+#define VVCAM_ISP_CID_EE_MANU_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x000C)
+#define VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x000D)
+#define VVCAM_ISP_CID_EE_MANU_UV_GAIN       (VVCAM_ISP_CID_EE_BASE + 0x000E)
+#define VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP     (VVCAM_ISP_CID_EE_BASE + 0x000F)
+#define VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN   (VVCAM_ISP_CID_EE_BASE + 0x0010)
+#define VVCAM_ISP_CID_EE_STAT_EDGE_GAIN     (VVCAM_ISP_CID_EE_BASE + 0x0011)
+#define VVCAM_ISP_CID_EE_STAT_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x0012)
+#define VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0013)
+#define VVCAM_ISP_CID_EE_STAT_UV_GAIN       (VVCAM_ISP_CID_EE_BASE + 0x0014)
+#define VVCAM_ISP_CID_EE_STAT_Y_GAIN_UP     (VVCAM_ISP_CID_EE_BASE + 0x0015)
+#define VVCAM_ISP_CID_EE_STAT_Y_GAIN_DOWN   (VVCAM_ISP_CID_EE_BASE + 0x0016)
+
+#ifdef __KERNEL__
+int vvcam_isp_ee_ctrl_count(void);
+int vvcam_isp_ee_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v2/vvcam_isp_ee.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v2/vvcam_isp_ee.c
new file mode 100644
index 000000000000..277ea4190e15
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v2/vvcam_isp_ee.c
@@ -0,0 +1,871 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ee.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ee_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EE_ENABLE:
+        case VVCAM_ISP_CID_EE_RESET:
+        case VVCAM_ISP_CID_EE_ENABLE_CURVE:
+        case VVCAM_ISP_CID_EE_ENABLE_CA:
+        case VVCAM_ISP_CID_EE_ENABLE_DCI:
+        case VVCAM_ISP_CID_EE_MODE:
+        case VVCAM_ISP_CID_EE_AUTO_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_AUTO_UV_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_CA_ENABLE_TBL:
+        case VVCAM_ISP_CID_EE_AUTO_CA_CURVE:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_ENABLE_TBL:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_UV_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_CA_MODE:
+        case VVCAM_ISP_CID_EE_MANU_CA_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MODE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ee_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EE_ENABLE:
+        case VVCAM_ISP_CID_EE_RESET:
+        case VVCAM_ISP_CID_EE_ENABLE_CURVE:
+        case VVCAM_ISP_CID_EE_ENABLE_CA:
+        case VVCAM_ISP_CID_EE_ENABLE_DCI:
+        case VVCAM_ISP_CID_EE_MODE:
+        case VVCAM_ISP_CID_EE_AUTO_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_AUTO_UV_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_CA_ENABLE_TBL:
+        case VVCAM_ISP_CID_EE_AUTO_CA_CURVE:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_ENABLE_TBL:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_UV_GAIN:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_CA_MODE:
+        case VVCAM_ISP_CID_EE_MANU_CA_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MODE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y:
+        case VVCAM_ISP_CID_EE_STAT_EDGE_GAIN:
+        case VVCAM_ISP_CID_EE_STAT_STRENGTH:
+        case VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_STAT_UV_GAIN:
+        case VVCAM_ISP_CID_EE_STAT_Y_GAIN_UP:
+        case VVCAM_ISP_CID_EE_STAT_Y_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_CA_MODE:
+        case VVCAM_ISP_CID_EE_STAT_CA_CURVE:
+        case VVCAM_ISP_CID_EE_STAT_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_STAT_DCI_DEGAMMA:
+        case VVCAM_ISP_CID_EE_STAT_DCI_MODE:
+        case VVCAM_ISP_CID_EE_STAT_DCI_LOW_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_STAT_DCI_MIDDLE_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_STAT_DCI_HIGH_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_START_Y:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_X:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_Y:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_X:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_Y:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_STOP_Y:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ee_ctrl_ops = {
+	.s_ctrl = vvcam_isp_ee_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ee_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ee_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_ENABLE_CURVE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_enable_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_ENABLE_CA,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_enable_ca",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_ENABLE_DCI,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_enable_dci",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 20 * 32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_y_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20}
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_y_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20}
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20}
+    },
+    {
+        /* uint32_t array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_edge_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {20}
+    },
+    {
+        /* uint32_t bool array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_CA_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_ca_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20},
+    },
+    {
+        // uint16_t array 20*16bit
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_CA_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_ca_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 65},
+    },
+    {
+        /* uint32_t bool array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DCI_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dci_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20},
+    },
+    {
+        // uint16_t array 20*16bit
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DCI_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dci_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 65},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_EDGE_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_edge_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_y_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_y_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CA_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_ca_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* uint16_t 65x array */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CA_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_ca_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65},
+    },
+    {
+        /* uint16_t 65x array */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65},
+    },
+    {
+        /* float [1.0, 4.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_degamma",
+        .step = 1,
+        .def  = 100,
+        .min  = 100,
+        .max  = 400,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_l_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_m_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_h_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_start_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_1_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_1_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_2_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_2_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_stop_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_EDGE_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_edge_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_Y_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_y_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_Y_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_y_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CA_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_ca_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* uint16_t 65x array */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CA_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_ca_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65},
+    },
+    {
+        /* uint16_t 65x array */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65},
+    },
+    {
+        /* float [1.0, 4.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_degamma",
+        .step = 1,
+        .def  = 100,
+        .min  = 100,
+        .max  = 400,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_LOW_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_l_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_MIDDLE_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_m_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_HIGH_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_h_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_START_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_start_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_1_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_1_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_2_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_2_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_STOP_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_stop_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+};
+
+int vvcam_isp_ee_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ee_ctrls);
+}
+
+int vvcam_isp_ee_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ee_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ee_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ee ctrl %s failed %d.\n",
+                vvcam_isp_ee_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v2/vvcam_isp_ee.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v2/vvcam_isp_ee.h
new file mode 100644
index 000000000000..b610b44b8dae
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v2/vvcam_isp_ee.h
@@ -0,0 +1,145 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_EE_H__
+#define __VVCAM_ISP_EE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_EE_ENABLE             (VVCAM_ISP_CID_EE_BASE + 0x0000)
+#define VVCAM_ISP_CID_EE_RESET              (VVCAM_ISP_CID_EE_BASE + 0x0001)
+#define VVCAM_ISP_CID_EE_ENABLE_CURVE       (VVCAM_ISP_CID_EE_BASE + 0x0002)
+#define VVCAM_ISP_CID_EE_ENABLE_CA          (VVCAM_ISP_CID_EE_BASE + 0x0003)
+#define VVCAM_ISP_CID_EE_ENABLE_DCI         (VVCAM_ISP_CID_EE_BASE + 0x0004)
+#define VVCAM_ISP_CID_EE_MODE               (VVCAM_ISP_CID_EE_BASE + 0x0005)
+#define VVCAM_ISP_CID_EE_AUTO_LEVEL         (VVCAM_ISP_CID_EE_BASE + 0x0006)
+#define VVCAM_ISP_CID_EE_AUTO_GAIN          (VVCAM_ISP_CID_EE_BASE + 0x0007)
+#define VVCAM_ISP_CID_EE_AUTO_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x0008)
+#define VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0009)
+#define VVCAM_ISP_CID_EE_AUTO_Y_GAIN_UP     (VVCAM_ISP_CID_EE_BASE + 0x000A)
+#define VVCAM_ISP_CID_EE_AUTO_Y_GAIN_DOWN   (VVCAM_ISP_CID_EE_BASE + 0x000B)
+#define VVCAM_ISP_CID_EE_AUTO_UV_GAIN       (VVCAM_ISP_CID_EE_BASE + 0x000C)
+#define VVCAM_ISP_CID_EE_AUTO_EDGE_GAIN     (VVCAM_ISP_CID_EE_BASE + 0x000D)
+#define VVCAM_ISP_CID_EE_AUTO_CA_ENABLE_TBL (VVCAM_ISP_CID_EE_BASE + 0x000E)
+#define VVCAM_ISP_CID_EE_AUTO_CA_CURVE      (VVCAM_ISP_CID_EE_BASE + 0x000F)
+#define VVCAM_ISP_CID_EE_AUTO_DCI_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0010)
+#define VVCAM_ISP_CID_EE_AUTO_DCI_CURVE     (VVCAM_ISP_CID_EE_BASE + 0x0011)
+#define VVCAM_ISP_CID_EE_MANU_EDGE_GAIN     (VVCAM_ISP_CID_EE_BASE + 0x0012)
+#define VVCAM_ISP_CID_EE_MANU_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x0013)
+#define VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0014)
+#define VVCAM_ISP_CID_EE_MANU_UV_GAIN       (VVCAM_ISP_CID_EE_BASE + 0x0015)
+#define VVCAM_ISP_CID_EE_MANU_Y_GAIN_UP     (VVCAM_ISP_CID_EE_BASE + 0x0016)
+#define VVCAM_ISP_CID_EE_MANU_Y_GAIN_DOWN   (VVCAM_ISP_CID_EE_BASE + 0x0017)
+#define VVCAM_ISP_CID_EE_MANU_CA_MODE       (VVCAM_ISP_CID_EE_BASE + 0x0018)
+#define VVCAM_ISP_CID_EE_MANU_CA_CURVE      (VVCAM_ISP_CID_EE_BASE + 0x0019)
+#define VVCAM_ISP_CID_EE_MANU_DCI_CURVE     (VVCAM_ISP_CID_EE_BASE + 0x001A)
+#define VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA   (VVCAM_ISP_CID_EE_BASE + 0x001B)
+#define VVCAM_ISP_CID_EE_MANU_DCI_MODE      (VVCAM_ISP_CID_EE_BASE + 0x001C)
+#define VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x001D)
+#define VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x001E)
+#define VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x001F)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0020)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0021)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0022)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0023)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0024)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0025)
+
+#define VVCAM_ISP_CID_EE_STAT_EDGE_GAIN     (VVCAM_ISP_CID_EE_BASE + 0x0026)
+#define VVCAM_ISP_CID_EE_STAT_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x0027)
+#define VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0028)
+#define VVCAM_ISP_CID_EE_STAT_UV_GAIN       (VVCAM_ISP_CID_EE_BASE + 0x0029)
+#define VVCAM_ISP_CID_EE_STAT_Y_GAIN_UP     (VVCAM_ISP_CID_EE_BASE + 0x002A)
+#define VVCAM_ISP_CID_EE_STAT_Y_GAIN_DOWN   (VVCAM_ISP_CID_EE_BASE + 0x002B)
+#define VVCAM_ISP_CID_EE_STAT_CA_MODE       (VVCAM_ISP_CID_EE_BASE + 0x002C)
+#define VVCAM_ISP_CID_EE_STAT_CA_CURVE      (VVCAM_ISP_CID_EE_BASE + 0x002D)
+#define VVCAM_ISP_CID_EE_STAT_DCI_CURVE     (VVCAM_ISP_CID_EE_BASE + 0x002E)
+#define VVCAM_ISP_CID_EE_STAT_DCI_DEGAMMA   (VVCAM_ISP_CID_EE_BASE + 0x002F)
+#define VVCAM_ISP_CID_EE_STAT_DCI_MODE      (VVCAM_ISP_CID_EE_BASE + 0x0030)
+#define VVCAM_ISP_CID_EE_STAT_DCI_LOW_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0031)
+#define VVCAM_ISP_CID_EE_STAT_DCI_MIDDLE_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0032)
+#define VVCAM_ISP_CID_EE_STAT_DCI_HIGH_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0033)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_START_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0034)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0035)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0036)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0037)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0038)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_STOP_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0039)
+
+#ifdef __KERNEL__
+int vvcam_isp_ee_ctrl_count(void);
+int vvcam_isp_ee_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v3/vvcam_isp_ee.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v3/vvcam_isp_ee.c
new file mode 100644
index 000000000000..3d0dae482e13
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v3/vvcam_isp_ee.c
@@ -0,0 +1,2327 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ee.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ee_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EE_ENABLE:
+        case VVCAM_ISP_CID_EE_RESET:
+        case VVCAM_ISP_CID_EE_CURVE_ENABLE:
+        case VVCAM_ISP_CID_EE_CA_ENABLE:
+        case VVCAM_ISP_CID_EE_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_EE_DCI_ENABLE:
+        case VVCAM_ISP_CID_EE_YUV_DOMAIN_ENABLE:
+        case VVCAM_ISP_CID_EE_SHARPEN_LIMIT_ENABLE:
+        case VVCAM_ISP_CID_EE_SKIN_PROC_ENABLE:
+        case VVCAM_ISP_CID_EE_MODE:
+        case VVCAM_ISP_CID_EE_AUTO_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_EE_EN:
+        case VVCAM_ISP_CID_EE_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH_SKIN:
+        case VVCAM_ISP_CID_EE_AUTO_EDGE_NR_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_DETAIL_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_DETAIL_PRE_ENHANCE_STR:
+        case VVCAM_ISP_CID_EE_AUTO_SKIN_PROC_EN:
+        case VVCAM_ISP_CID_EE_AUTO_SKIN_DETECT_STR:
+        case VVCAM_ISP_CID_EE_AUTO_SKIN_THRESHOLD:
+        case VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE:
+        case VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE_SKIN:
+        case VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD:
+        case VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD_SKIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_UV_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_LIMIT:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_LIMIT:
+        case VVCAM_ISP_CID_EE_AUTO_DEPURPLE_EN:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_LIMIT:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_THR:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_COMP_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_SAT_STR:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_FIX_STR:
+        case VVCAM_ISP_CID_EE_AUTO_CA_ENABLE:
+        case VVCAM_ISP_CID_EE_AUTO_CA_LUT_CHROMA_TBL:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_EN:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH_SKIN:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_NR_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_SCALER:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_USE_AUXI_DIR:
+        case VVCAM_ISP_CID_EE_MANU_DETAIL_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_DETAIL_SCALER:
+        case VVCAM_ISP_CID_EE_MANU_DETAIL_PRE_ENHANCE_STR:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_DETECT_STR:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MIN:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MAX:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MIN:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MAX:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MIN:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MAX:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T0:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T1_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T2:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T3_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T0:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T1_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T2:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T3_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE_SKIN:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL_SKIN:
+        case VVCAM_ISP_CID_EE_MANU_ENHANCE_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_SKIN_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UV:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_TYPE:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_DPL_DETECT_RANGE:
+        case VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_THR:
+        case VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_MANU_DPL_COMP_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_MANU_DPL_SAT_STR:
+        case VVCAM_ISP_CID_EE_MANU_DPL_FIX_STR:
+        case VVCAM_ISP_CID_EE_MANU_CA_LUT_CHROMA_TBL:
+        case VVCAM_ISP_CID_EE_MANU_CA_MODE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA:
+        case VVCAM_ISP_CID_EE_MANU_DCI_LUT_LUMA_TBL:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MODE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ee_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EE_ENABLE:
+        case VVCAM_ISP_CID_EE_RESET:
+        case VVCAM_ISP_CID_EE_CURVE_ENABLE:
+        case VVCAM_ISP_CID_EE_CA_ENABLE:
+        case VVCAM_ISP_CID_EE_DEPURPLE_ENABLE:
+        case VVCAM_ISP_CID_EE_DCI_ENABLE:
+        case VVCAM_ISP_CID_EE_YUV_DOMAIN_ENABLE:
+        case VVCAM_ISP_CID_EE_SHARPEN_LIMIT_ENABLE:
+        case VVCAM_ISP_CID_EE_SKIN_PROC_ENABLE:
+        case VVCAM_ISP_CID_EE_MODE:
+        case VVCAM_ISP_CID_EE_AUTO_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_EE_EN:
+        case VVCAM_ISP_CID_EE_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH_SKIN:
+        case VVCAM_ISP_CID_EE_AUTO_EDGE_NR_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_DETAIL_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_DETAIL_PRE_ENHANCE_STR:
+        case VVCAM_ISP_CID_EE_AUTO_SKIN_PROC_EN:
+        case VVCAM_ISP_CID_EE_AUTO_SKIN_DETECT_STR:
+        case VVCAM_ISP_CID_EE_AUTO_SKIN_THRESHOLD:
+        case VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE:
+        case VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE_SKIN:
+        case VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD:
+        case VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD_SKIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_UV_GAIN:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_LIMIT:
+        case VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_LIMIT:
+        case VVCAM_ISP_CID_EE_AUTO_DEPURPLE_EN:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_LIMIT:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_THR:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_COMP_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_SAT_STR:
+        case VVCAM_ISP_CID_EE_AUTO_DPL_FIX_STR:
+        case VVCAM_ISP_CID_EE_AUTO_CA_ENABLE:
+        case VVCAM_ISP_CID_EE_AUTO_CA_LUT_CHROMA_TBL:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_EN:
+        case VVCAM_ISP_CID_EE_AUTO_DCI_CURVE:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH_SKIN:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_NR_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_SCALER:
+        case VVCAM_ISP_CID_EE_MANU_EDGE_USE_AUXI_DIR:
+        case VVCAM_ISP_CID_EE_MANU_DETAIL_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_DETAIL_SCALER:
+        case VVCAM_ISP_CID_EE_MANU_DETAIL_PRE_ENHANCE_STR:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_DETECT_STR:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MIN:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MAX:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MIN:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MAX:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MIN:
+        case VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MAX:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T0:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T1_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T2:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T3_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T0:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T1_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T2:
+        case VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T3_SHIFT:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE_SKIN:
+        case VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL_SKIN:
+        case VVCAM_ISP_CID_EE_MANU_ENHANCE_STRENGTH:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_SKIN_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UV:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_TYPE:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_UP:
+        case VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_DPL_DETECT_RANGE:
+        case VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_THR:
+        case VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_MANU_DPL_COMP_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_MANU_DPL_SAT_STR:
+        case VVCAM_ISP_CID_EE_MANU_DPL_FIX_STR:
+        case VVCAM_ISP_CID_EE_MANU_CA_LUT_CHROMA_TBL:
+        case VVCAM_ISP_CID_EE_MANU_CA_MODE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA:
+        case VVCAM_ISP_CID_EE_MANU_DCI_LUT_LUMA_TBL:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MODE:
+        case VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y:
+        case VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_PROC_ENABLE:
+        case VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH:
+        case VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH_SKIN:
+        case VVCAM_ISP_CID_EE_STAT_EDGE_NR_LEVEL:
+        case VVCAM_ISP_CID_EE_STAT_EDGE_SCALER:
+        case VVCAM_ISP_CID_EE_STAT_EDGE_USE_AUXI_DIR:
+        case VVCAM_ISP_CID_EE_STAT_DETAIL_LEVEL:
+        case VVCAM_ISP_CID_EE_STAT_DETAIL_SCALER:
+        case VVCAM_ISP_CID_EE_STAT_DETAIL_PRE_ENHANCE_STR:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_DETECT_STR:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_Y_MIN:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_Y_MAX:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CR_MIN:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CR_MAX:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CB_MIN:
+        case VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CB_MAX:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T0:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T1_SHIFT:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T2:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T3_SHIFT:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T0:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T1_SHIFT:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T2:
+        case VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T3_SHIFT:
+        case VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_EDGE:
+        case VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_DETAIL:
+        case VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_EDGE_SKIN:
+        case VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_DETAIL_SKIN:
+        case VVCAM_ISP_CID_EE_STAT_ENHANCE_STRENGTH:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_SKIN_CURVE_LEVEL:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_UP:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_SKIN_UP:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_SKIN_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_UV:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_TYPE:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_SKIN_UP:
+        case VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_SKIN_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_DPL_DETECT_RANGE:
+        case VVCAM_ISP_CID_EE_STAT_DPL_U_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_STAT_DPL_U_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_DPL_V_LIMIT_UP:
+        case VVCAM_ISP_CID_EE_STAT_DPL_V_LIMIT_DOWN:
+        case VVCAM_ISP_CID_EE_STAT_DPL_DETECT_LUMA_THR:
+        case VVCAM_ISP_CID_EE_STAT_DPL_DETECT_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_STAT_DPL_COMP_LUMA_DIFF:
+        case VVCAM_ISP_CID_EE_STAT_DPL_SAT_STR:
+        case VVCAM_ISP_CID_EE_STAT_DPL_FIX_STR:
+        case VVCAM_ISP_CID_EE_STAT_CA_LUT_CHROMA_TBL:
+        case VVCAM_ISP_CID_EE_STAT_CA_MODE:
+        case VVCAM_ISP_CID_EE_STAT_DCI_DEGAMMA:
+        case VVCAM_ISP_CID_EE_STAT_DCI_LUT_LUMA_TBL:
+        case VVCAM_ISP_CID_EE_STAT_DCI_MODE:
+        case VVCAM_ISP_CID_EE_STAT_DCI_LOW_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_STAT_DCI_MIDDLE_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_STAT_DCI_HIGH_LUMA_INDEX:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_START_Y:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_X:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_Y:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_X:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_Y:
+        case VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_STOP_Y:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ee_ctrl_ops = {
+	.s_ctrl = vvcam_isp_ee_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ee_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ee_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_CURVE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_curve_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_CA_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_ca_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_DEPURPLE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_depurple_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_DCI_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_dci_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_YUV_DOMAIN_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_yuv_domain_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_SHARPEN_LIMIT_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_sharpen_lmt_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_SKIN_PROC_ENABLE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_skin_proc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20 * 32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* bool 20 * 32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_EE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_ee_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit  0 ~ 256 */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit 0 ~ 256 */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_src_strength_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_EDGE_NR_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_edge_nr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DETAIL_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dtl_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DETAIL_PRE_ENHANCE_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dtl_pre_enhance_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SKIN_PROC_EN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_skin_proc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SKIN_DETECT_STR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_skin_detect_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* int16_t array 20*6*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SKIN_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_skin_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20, 6, 0, 0},
+    },
+    {
+        /* uint16_t array 20*4*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_hf_merge_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+        /* uint16_t array 20*4*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_hf_merge_curve_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_grad_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_grad_threshold_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_CURVE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_crv_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_CURVE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_skin_crv_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_skin_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* uint8_t array 20*8bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_UV_GAIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_uv_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_LIMIT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_lmt",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* uint16_t array 20*2*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_LIMIT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_sharp_skin_limit",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {20, 2, 0, 0},
+    },
+    {
+        /* bool array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DEPURPLE_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*4*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DPL_LIMIT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_lmt",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_det_luma_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_DIFF,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_det_luma_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DPL_COMP_LUMA_DIFF,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_comp_luma_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DPL_SAT_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_sat_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DPL_FIX_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dpl_fix_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* bool array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_CA_ENABLE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_ca_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*65*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_CA_LUT_CHROMA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_ca_lut_chroma_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 65, 0, 0},
+    },
+    {
+        /* bool array 20*32bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DCI_EN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dci_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 20*65*16bit */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_AUTO_DCI_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_auto_dci_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 65, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_src_strength_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_EDGE_NR_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_edge_nr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_EDGE_SCALER,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_edge_scaler",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_EDGE_USE_AUXI_DIR,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_edge_use_auxi_dir",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DETAIL_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dtl_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DETAIL_SCALER,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dtl_scaler",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DETAIL_PRE_ENHANCE_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dtl_pre_enhance_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_DETECT_STR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_detect_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_h_thr_y_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_h_thr_y_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_h_thr_cr_min",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_h_thr_cr_max",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_h_thr_cb_min",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_skin_h_thr_cb_max",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T0,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_t0",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T1_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_t1_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_t2",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T3_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_t3_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T0,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_skin_t0",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T1_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_skin_t1_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_skin_t2",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T3_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_hf_mc_skin_t3_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_grad_thr_edge",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_grad_thr_dtl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_grad_thr_edge_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_grad_thr_dtl_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_ENHANCE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_enhance_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_CURVE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_crv_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_SKIN_CURVE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_skin_crv_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_skin_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_skin_gain_dwn",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UV,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_gain_uv",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_TYPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_lmt_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_lmt_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_lmt_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_lmt_skin_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_sharp_lmt_skin_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_DETECT_RANGE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_detect_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_u_lmt_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_u_lmt_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_v_lmt_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_v_lmt_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_detect_luma_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_DIFF,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_detect_luma_dif",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_COMP_LUMA_DIFF,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_comp_luma_dif",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_SAT_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_sat_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DPL_FIX_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dpl_fix_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CA_LUT_CHROMA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_ca_lut_chroma_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_CA_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_ca_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_degamma",
+        .step = 1,
+        .def  = 100,
+        .min  = 100,
+        .max  = 400,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_LUT_LUMA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_lut_luma_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_l_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_m_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_h_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_start_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_1_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_1_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_2_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_2_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_manu_dci_inflct_stop_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_PROC_ENABLE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_proc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_src_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_src_strength_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_EDGE_NR_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_edge_nr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 5,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_EDGE_SCALER,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_edge_scaler",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_EDGE_USE_AUXI_DIR,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_edge_use_auxi_dir",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DETAIL_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dtl_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DETAIL_SCALER,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dtl_scaler",
+        .step = 1,
+        .min  = 0,
+        .max  = 32,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DETAIL_PRE_ENHANCE_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dtl_pre_enhance_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_DETECT_STR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_detect_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_Y_MIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_h_thr_y_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_Y_MAX,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_h_thr_y_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CR_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_h_thr_cr_min",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CR_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_h_thr_cr_max",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CB_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_h_thr_cb_min",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CB_MAX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_skin_h_thr_cb_max",
+        .step = 1,
+        .min  = -1024,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T0,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_t0",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T1_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_t1_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_t2",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T3_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_t3_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T0,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_skin_t0",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T1_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_skin_t1_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T2,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_skin_t2",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T3_SHIFT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_hf_mc_skin_t3_shift",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_EDGE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_grad_thr_edge",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_DETAIL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_grad_thr_dtl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_EDGE_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_grad_thr_edge_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_DETAIL_SKIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_grad_thr_dtl_skin",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_ENHANCE_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_enhance_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_CURVE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_crv_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_SKIN_CURVE_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_skin_crv_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_SKIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_skin_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_SKIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_skin_gain_dwn",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_UV,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_gain_uv",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_TYPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_lmt_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_lmt_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_lmt_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_SKIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_lmt_skin_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_SKIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_sharp_lmt_skin_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 512,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_DETECT_RANGE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_detect_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 7,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_U_LIMIT_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_u_lmt_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_U_LIMIT_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_u_lmt_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_V_LIMIT_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_v_lmt_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_V_LIMIT_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_v_lmt_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_DETECT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_detect_luma_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_DETECT_LUMA_DIFF,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_detect_luma_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_COMP_LUMA_DIFF,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_comp_luma_dif",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_SAT_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_sat_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DPL_FIX_STR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dpl_fix_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CA_LUT_CHROMA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_ca_lut_chroma_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_CA_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_ca_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_degamma",
+        .step = 1,
+        .def  = 100,
+        .min  = 100,
+        .max  = 400,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_LUT_LUMA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_lut_luma_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {65, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_LOW_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_l_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_MIDDLE_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_m_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* float [0, 2.0] */
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_HIGH_LUMA_INDEX,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_h_luma_index",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_START_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_start_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_1_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_1_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_X,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_2_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_2_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ee_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_STOP_Y,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ee_stat_dci_inflct_stop_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_ee_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ee_ctrls);
+}
+
+int vvcam_isp_ee_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ee_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ee_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ee ctrl %s failed %d.\n",
+                vvcam_isp_ee_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v3/vvcam_isp_ee.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v3/vvcam_isp_ee.h
new file mode 100644
index 000000000000..3f70886b91d8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ee/ee_v3/vvcam_isp_ee.h
@@ -0,0 +1,362 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_EE_H__
+#define __VVCAM_ISP_EE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_EE_ENABLE             (VVCAM_ISP_CID_EE_BASE + 0x0000)
+#define VVCAM_ISP_CID_EE_RESET              (VVCAM_ISP_CID_EE_BASE + 0x0001)
+#define VVCAM_ISP_CID_EE_CURVE_ENABLE       (VVCAM_ISP_CID_EE_BASE + 0x0002)
+#define VVCAM_ISP_CID_EE_CA_ENABLE          (VVCAM_ISP_CID_EE_BASE + 0x0003)
+#define VVCAM_ISP_CID_EE_DEPURPLE_ENABLE    (VVCAM_ISP_CID_EE_BASE + 0x0004)
+#define VVCAM_ISP_CID_EE_DCI_ENABLE         (VVCAM_ISP_CID_EE_BASE + 0x0005)
+#define VVCAM_ISP_CID_EE_YUV_DOMAIN_ENABLE  (VVCAM_ISP_CID_EE_BASE + 0x0006)
+#define VVCAM_ISP_CID_EE_SHARPEN_LIMIT_ENABLE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0007)
+#define VVCAM_ISP_CID_EE_SKIN_PROC_ENABLE   (VVCAM_ISP_CID_EE_BASE + 0x0008)
+#define VVCAM_ISP_CID_EE_MODE               (VVCAM_ISP_CID_EE_BASE + 0x0009)
+#define VVCAM_ISP_CID_EE_AUTO_LEVEL         (VVCAM_ISP_CID_EE_BASE + 0x000A)
+#define VVCAM_ISP_CID_EE_AUTO_GAIN          (VVCAM_ISP_CID_EE_BASE + 0x000B)
+#define VVCAM_ISP_CID_EE_AUTO_EE_EN         (VVCAM_ISP_CID_EE_BASE + 0x000C)
+#define VVCAM_ISP_CID_EE_AUTO_STRENGTH      (VVCAM_ISP_CID_EE_BASE + 0x000D)
+#define VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x000E)
+#define VVCAM_ISP_CID_EE_AUTO_SRC_STRENGTH_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x000F)
+#define VVCAM_ISP_CID_EE_AUTO_EDGE_NR_LEVEL (VVCAM_ISP_CID_EE_BASE + 0x0010)
+#define VVCAM_ISP_CID_EE_AUTO_DETAIL_LEVEL  (VVCAM_ISP_CID_EE_BASE + 0x0011)
+#define VVCAM_ISP_CID_EE_AUTO_DETAIL_PRE_ENHANCE_STR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0012)
+#define VVCAM_ISP_CID_EE_AUTO_SKIN_PROC_EN  (VVCAM_ISP_CID_EE_BASE + 0x0013)
+#define VVCAM_ISP_CID_EE_AUTO_SKIN_DETECT_STR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0014)
+#define VVCAM_ISP_CID_EE_AUTO_SKIN_THRESHOLD \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0015)
+#define VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0016)
+#define VVCAM_ISP_CID_EE_AUTO_HF_MERGE_CURVE_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0017)
+#define VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0018)
+#define VVCAM_ISP_CID_EE_AUTO_GRAD_THRESHOLD_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0019)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_CURVE_LEVEL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x001A)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_CURVE_LEVEL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x001B)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_GAIN    (VVCAM_ISP_CID_EE_BASE + 0x001C)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_GAIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x001D)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_UV_GAIN (VVCAM_ISP_CID_EE_BASE + 0x001E)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_LIMIT   (VVCAM_ISP_CID_EE_BASE + 0x001F)
+#define VVCAM_ISP_CID_EE_AUTO_SHARP_SKIN_LIMIT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0020)
+#define VVCAM_ISP_CID_EE_AUTO_DEPURPLE_EN   (VVCAM_ISP_CID_EE_BASE + 0x0021)
+#define VVCAM_ISP_CID_EE_AUTO_DPL_LIMIT     (VVCAM_ISP_CID_EE_BASE + 0x0022)
+#define VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_THR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0023)
+#define VVCAM_ISP_CID_EE_AUTO_DPL_DETECT_LUMA_DIFF \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0024)
+#define VVCAM_ISP_CID_EE_AUTO_DPL_COMP_LUMA_DIFF \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0025)
+#define VVCAM_ISP_CID_EE_AUTO_DPL_SAT_STR   (VVCAM_ISP_CID_EE_BASE + 0x0026)
+#define VVCAM_ISP_CID_EE_AUTO_DPL_FIX_STR   (VVCAM_ISP_CID_EE_BASE + 0x0027)
+#define VVCAM_ISP_CID_EE_AUTO_CA_ENABLE     (VVCAM_ISP_CID_EE_BASE + 0x0028)
+#define VVCAM_ISP_CID_EE_AUTO_CA_LUT_CHROMA_TBL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0029)
+#define VVCAM_ISP_CID_EE_AUTO_DCI_EN        (VVCAM_ISP_CID_EE_BASE + 0x002A)
+#define VVCAM_ISP_CID_EE_AUTO_DCI_CURVE     (VVCAM_ISP_CID_EE_BASE + 0x002B)
+#define VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0040)
+#define VVCAM_ISP_CID_EE_MANU_SRC_STRENGTH_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0041)
+#define VVCAM_ISP_CID_EE_MANU_EDGE_NR_LEVEL (VVCAM_ISP_CID_EE_BASE + 0x0042)
+#define VVCAM_ISP_CID_EE_MANU_EDGE_SCALER   (VVCAM_ISP_CID_EE_BASE + 0x0043)
+#define VVCAM_ISP_CID_EE_MANU_EDGE_USE_AUXI_DIR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0044)
+#define VVCAM_ISP_CID_EE_MANU_DETAIL_LEVEL  (VVCAM_ISP_CID_EE_BASE + 0x0045)
+#define VVCAM_ISP_CID_EE_MANU_DETAIL_SCALER (VVCAM_ISP_CID_EE_BASE + 0x0046)
+#define VVCAM_ISP_CID_EE_MANU_DETAIL_PRE_ENHANCE_STR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0047)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_DETECT_STR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0048)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0049)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_Y_MAX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x004A)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x004B)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CR_MAX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x004C)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x004D)
+#define VVCAM_ISP_CID_EE_MANU_SKIN_H_THR_CB_MAX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x004E)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T0 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x004F)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T1_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0050)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T2 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0051)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_T3_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0052)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T0 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0053)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T1_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0054)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T2 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0055)
+#define VVCAM_ISP_CID_EE_MANU_HF_MERGE_CURVE_SKIN_T3_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0056)
+#define VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0057)
+#define VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0058)
+#define VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_EDGE_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0059)
+#define VVCAM_ISP_CID_EE_MANU_CUTOFF_GRAD_THR_DETAIL_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x005A)
+#define VVCAM_ISP_CID_EE_MANU_ENHANCE_STRENGTH \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x005B)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_CURVE_LEVEL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x005C)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_SKIN_CURVE_LEVEL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x005D)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UP (VVCAM_ISP_CID_EE_BASE + 0x005E)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x005F)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0060)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_SKIN_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0061)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_GAIN_UV (VVCAM_ISP_CID_EE_BASE + 0x0062)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_TYPE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0063)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0064)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0065)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0066)
+#define VVCAM_ISP_CID_EE_MANU_SHARP_LIMIT_SKIN_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0067)
+#define VVCAM_ISP_CID_EE_MANU_DPL_DETECT_RANGE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0068)
+#define VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0069)
+#define VVCAM_ISP_CID_EE_MANU_DPL_U_LIMIT_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x006A)
+#define VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x006B)
+#define VVCAM_ISP_CID_EE_MANU_DPL_V_LIMIT_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x006C)
+#define VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_THR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x006D)
+#define VVCAM_ISP_CID_EE_MANU_DPL_DETECT_LUMA_DIFF \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x006E)
+#define VVCAM_ISP_CID_EE_MANU_DPL_COMP_LUMA_DIFF \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x006F)
+#define VVCAM_ISP_CID_EE_MANU_DPL_SAT_STR   (VVCAM_ISP_CID_EE_BASE + 0x0070)
+#define VVCAM_ISP_CID_EE_MANU_DPL_FIX_STR   (VVCAM_ISP_CID_EE_BASE + 0x0071)
+#define VVCAM_ISP_CID_EE_MANU_CA_LUT_CHROMA_TBL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0072)
+#define VVCAM_ISP_CID_EE_MANU_CA_MODE       (VVCAM_ISP_CID_EE_BASE + 0x0073)
+#define VVCAM_ISP_CID_EE_MANU_DCI_DEGAMMA   (VVCAM_ISP_CID_EE_BASE + 0x0074)
+#define VVCAM_ISP_CID_EE_MANU_DCI_LUT_LUMA_TBL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0075)
+#define VVCAM_ISP_CID_EE_MANU_DCI_MODE      (VVCAM_ISP_CID_EE_BASE + 0x0076)
+#define VVCAM_ISP_CID_EE_MANU_DCI_LOW_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0077)
+#define VVCAM_ISP_CID_EE_MANU_DCI_MIDDLE_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0078)
+#define VVCAM_ISP_CID_EE_MANU_DCI_HIGH_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0079)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_START_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x007A)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x007B)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_1_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x007C)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x007D)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_2_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x007E)
+#define VVCAM_ISP_CID_EE_MANU_DCI_INFLCT_STOP_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x007F)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_PROC_ENABLE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0080)
+#define VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH  (VVCAM_ISP_CID_EE_BASE + 0x0081)
+#define VVCAM_ISP_CID_EE_STAT_SRC_STRENGTH_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0082)
+#define VVCAM_ISP_CID_EE_STAT_EDGE_NR_LEVEL (VVCAM_ISP_CID_EE_BASE + 0x0083)
+#define VVCAM_ISP_CID_EE_STAT_EDGE_SCALER   (VVCAM_ISP_CID_EE_BASE + 0x0084)
+#define VVCAM_ISP_CID_EE_STAT_EDGE_USE_AUXI_DIR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0085)
+#define VVCAM_ISP_CID_EE_STAT_DETAIL_LEVEL  (VVCAM_ISP_CID_EE_BASE + 0x0086)
+#define VVCAM_ISP_CID_EE_STAT_DETAIL_SCALER (VVCAM_ISP_CID_EE_BASE + 0x0087)
+#define VVCAM_ISP_CID_EE_STAT_DETAIL_PRE_ENHANCE_STR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0088)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_DETECT_STR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0089)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_Y_MIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x008A)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_Y_MAX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x008B)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CR_MIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x008C)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CR_MAX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x008D)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CB_MIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x008E)
+#define VVCAM_ISP_CID_EE_STAT_SKIN_H_THR_CB_MAX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x008F)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T0 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0090)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T1_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0091)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T2 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0092)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_T3_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0093)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T0 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0094)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T1_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0095)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T2 \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0096)
+#define VVCAM_ISP_CID_EE_STAT_HF_MERGE_CURVE_SKIN_T3_SHIFT \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0097)
+#define VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_EDGE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0098)
+#define VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_DETAIL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x0099)
+#define VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_EDGE_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x009A)
+#define VVCAM_ISP_CID_EE_STAT_CUTOFF_GRAD_THR_DETAIL_SKIN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x009B)
+#define VVCAM_ISP_CID_EE_STAT_ENHANCE_STRENGTH \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x009C)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_CURVE_LEVEL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x009D)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_SKIN_CURVE_LEVEL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x009E)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_UP (VVCAM_ISP_CID_EE_BASE + 0x009F)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A0)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_SKIN_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A1)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_SKIN_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A2)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_GAIN_UV (VVCAM_ISP_CID_EE_BASE + 0x00A3)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_TYPE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A4)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A5)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A6)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_SKIN_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A7)
+#define VVCAM_ISP_CID_EE_STAT_SHARP_LIMIT_SKIN_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A8)
+#define VVCAM_ISP_CID_EE_STAT_DPL_DETECT_RANGE \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00A9)
+#define VVCAM_ISP_CID_EE_STAT_DPL_U_LIMIT_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00AA)
+#define VVCAM_ISP_CID_EE_STAT_DPL_U_LIMIT_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00AB)
+#define VVCAM_ISP_CID_EE_STAT_DPL_V_LIMIT_UP \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00AC)
+#define VVCAM_ISP_CID_EE_STAT_DPL_V_LIMIT_DOWN \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00AD)
+#define VVCAM_ISP_CID_EE_STAT_DPL_DETECT_LUMA_THR \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00AE)
+#define VVCAM_ISP_CID_EE_STAT_DPL_DETECT_LUMA_DIFF \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00AF)
+#define VVCAM_ISP_CID_EE_STAT_DPL_COMP_LUMA_DIFF \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00B0)
+#define VVCAM_ISP_CID_EE_STAT_DPL_SAT_STR   (VVCAM_ISP_CID_EE_BASE + 0x00B1)
+#define VVCAM_ISP_CID_EE_STAT_DPL_FIX_STR   (VVCAM_ISP_CID_EE_BASE + 0x00B2)
+#define VVCAM_ISP_CID_EE_STAT_CA_LUT_CHROMA_TBL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00B3)
+#define VVCAM_ISP_CID_EE_STAT_CA_MODE       (VVCAM_ISP_CID_EE_BASE + 0x00B4)
+#define VVCAM_ISP_CID_EE_STAT_DCI_DEGAMMA   (VVCAM_ISP_CID_EE_BASE + 0x00B5)
+#define VVCAM_ISP_CID_EE_STAT_DCI_LUT_LUMA_TBL \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00B6)
+#define VVCAM_ISP_CID_EE_STAT_DCI_MODE      (VVCAM_ISP_CID_EE_BASE + 0x00B7)
+#define VVCAM_ISP_CID_EE_STAT_DCI_LOW_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00B8)
+#define VVCAM_ISP_CID_EE_STAT_DCI_MIDDLE_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00B9)
+#define VVCAM_ISP_CID_EE_STAT_DCI_HIGH_LUMA_INDEX \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00BA)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_START_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00BB)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00BC)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_1_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00BD)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_X \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00BE)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_2_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00BF)
+#define VVCAM_ISP_CID_EE_STAT_DCI_INFLCT_STOP_Y \
+                                            (VVCAM_ISP_CID_EE_BASE + 0x00C0)
+
+#ifdef __KERNEL__
+int vvcam_isp_ee_ctrl_count(void);
+int vvcam_isp_ee_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v2/vvcam_isp_exp.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v2/vvcam_isp_exp.c
new file mode 100644
index 000000000000..0b820d46547f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v2/vvcam_isp_exp.c
@@ -0,0 +1,269 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_exp.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_exp_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EXP_ENABLE:
+        case VVCAM_ISP_CID_EXP_RESET:
+        case VVCAM_ISP_CID_EXP_R_WEIGHT:
+        case VVCAM_ISP_CID_EXP_GR_WEIGHT:
+        case VVCAM_ISP_CID_EXP_GB_WEIGHT:
+        case VVCAM_ISP_CID_EXP_B_WEIGHT:
+        // case VVCAM_ISP_CID_EXP_SIZE_RATIO:
+        case VVCAM_ISP_CID_EXP_INPUT_SELECT:
+        case VVCAM_ISP_CID_EXP_MEASURE_WINDOW:
+        case VVCAM_ISP_CID_EXP_STATISTIC:
+        case VVCAM_ISP_CID_EXP_STATISTIC_TYPE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_exp_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EXP_ENABLE:
+        case VVCAM_ISP_CID_EXP_RESET:
+        case VVCAM_ISP_CID_EXP_R_WEIGHT:
+        case VVCAM_ISP_CID_EXP_GR_WEIGHT:
+        case VVCAM_ISP_CID_EXP_GB_WEIGHT:
+        case VVCAM_ISP_CID_EXP_B_WEIGHT:
+        // case VVCAM_ISP_CID_EXP_SIZE_RATIO:
+        case VVCAM_ISP_CID_EXP_INPUT_SELECT:
+        case VVCAM_ISP_CID_EXP_MEASURE_WINDOW:
+        case VVCAM_ISP_CID_EXP_STATISTIC:
+        case VVCAM_ISP_CID_EXP_STATISTIC_TYPE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_exp_ctrl_ops = {
+    .s_ctrl = vvcam_isp_exp_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_exp_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_exp_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_R_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_r_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_GR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_gr_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_GB_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_gb_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_B_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_b_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 63,
+        .dims = {1, 0, 0, 0},
+    },
+    /*{
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_SIZE_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_size_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 100000,
+    },*/
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_INPUT_SELECT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_input_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* uint16_t array 4x16bit */
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_MEASURE_WINDOW,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_measure_window",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        // uint32_t array 32*32*4*32bit
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_STATISTIC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_statistic",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {32, 32, 4, 0},
+    },
+    {
+        .ops  = &vvcam_isp_exp_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_STATISTIC_TYPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_statistic_type",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+};
+
+int vvcam_isp_exp_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_exp_ctrls);
+}
+
+int vvcam_isp_exp_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_exp_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_exp_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp exp ctrl %s failed %d.\n",
+                vvcam_isp_exp_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v2/vvcam_isp_exp.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v2/vvcam_isp_exp.h
new file mode 100644
index 000000000000..0a4fa18d57c8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v2/vvcam_isp_exp.h
@@ -0,0 +1,78 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_EXP_H__
+#define __VVCAM_ISP_EXP_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_EXP_ENABLE            (VVCAM_ISP_CID_EXP_BASE + 0x0000)
+#define VVCAM_ISP_CID_EXP_RESET             (VVCAM_ISP_CID_EXP_BASE + 0x0001)
+#define VVCAM_ISP_CID_EXP_R_WEIGHT          (VVCAM_ISP_CID_EXP_BASE + 0x0002)
+#define VVCAM_ISP_CID_EXP_GR_WEIGHT         (VVCAM_ISP_CID_EXP_BASE + 0x0003)
+#define VVCAM_ISP_CID_EXP_GB_WEIGHT         (VVCAM_ISP_CID_EXP_BASE + 0x0004)
+#define VVCAM_ISP_CID_EXP_B_WEIGHT          (VVCAM_ISP_CID_EXP_BASE + 0x0005)
+// #define VVCAM_ISP_CID_EXP_SIZE_RATIO     (VVCAM_ISP_CID_EXP_BASE + 0x0006)
+#define VVCAM_ISP_CID_EXP_INPUT_SELECT      (VVCAM_ISP_CID_EXP_BASE + 0x0007)
+#define VVCAM_ISP_CID_EXP_MEASURE_WINDOW    (VVCAM_ISP_CID_EXP_BASE + 0x0008)
+#define VVCAM_ISP_CID_EXP_STATISTIC         (VVCAM_ISP_CID_EXP_BASE + 0x0009)
+#define VVCAM_ISP_CID_EXP_STATISTIC_TYPE    (VVCAM_ISP_CID_EXP_BASE + 0x000A)
+
+#ifdef __KERNEL__
+int vvcam_isp_exp_ctrl_count(void);
+int vvcam_isp_exp_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.c
new file mode 100644
index 000000000000..fd186ae6a412
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.c
@@ -0,0 +1,193 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_exp_v3.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_exp_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EXP_V3_ENABLE:
+        case VVCAM_ISP_CID_EXP_V3_AUTO_STOP:
+        case VVCAM_ISP_CID_EXP_V3_INPUT_SELECT:
+        case VVCAM_ISP_CID_EXP_V3_MEASURE_WINDOW:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_exp_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_EXP_V3_ENABLE:
+        case VVCAM_ISP_CID_EXP_V3_AUTO_STOP:
+        case VVCAM_ISP_CID_EXP_V3_INPUT_SELECT:
+        case VVCAM_ISP_CID_EXP_V3_MEASURE_WINDOW:
+        case VVCAM_ISP_CID_EXP_V3_STATISTIC:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_exp_v3_ctrl_ops = {
+    .s_ctrl = vvcam_isp_exp_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_exp_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_exp_v3_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_exp_v3_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_V3_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_v3_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_exp_v3_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_V3_INPUT_SELECT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_v3_input_select",
+        .step = 1,
+        .min  = 1,
+        .max  = 4,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_exp_v3_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_V3_AUTO_STOP,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_v3_auto_stop",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint16_t array 25x4x16bit */
+        .ops  = &vvcam_isp_exp_v3_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_V3_MEASURE_WINDOW,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_v3_measure_window",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {25, 4, 0, 0},
+    },
+    {
+        /* uint32_t array 25*2*32bit */
+        .ops  = &vvcam_isp_exp_v3_ctrl_ops,
+        .id   = VVCAM_ISP_CID_EXP_V3_STATISTIC,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_exp_v3_statistic",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {50, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_exp_v3_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_exp_v3_ctrls);
+}
+
+int vvcam_isp_exp_v3_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_exp_v3_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_exp_v3_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp exp ctrl %s failed %d.\n",
+                vvcam_isp_exp_v3_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.h
new file mode 100644
index 000000000000..4a6a89c844d9
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/exp/exp_v3/vvcam_isp_exp_v3.h
@@ -0,0 +1,72 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_EXP_V3_H__
+#define __VVCAM_ISP_EXP_V3_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_EXP_V3_ENABLE         (VVCAM_ISP_CID_EXP_V3_BASE + 0x0000)
+#define VVCAM_ISP_CID_EXP_V3_AUTO_STOP      (VVCAM_ISP_CID_EXP_V3_BASE + 0x0001)
+#define VVCAM_ISP_CID_EXP_V3_INPUT_SELECT   (VVCAM_ISP_CID_EXP_V3_BASE + 0x0002)
+#define VVCAM_ISP_CID_EXP_V3_MEASURE_WINDOW (VVCAM_ISP_CID_EXP_V3_BASE + 0x0003)
+#define VVCAM_ISP_CID_EXP_V3_STATISTIC      (VVCAM_ISP_CID_EXP_V3_BASE + 0x0004)
+
+#ifdef __KERNEL__
+int vvcam_isp_exp_v3_ctrl_count(void);
+int vvcam_isp_exp_v3_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v1/vvcam_isp_gc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v1/vvcam_isp_gc.c
new file mode 100644
index 000000000000..ad1396cdd43c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v1/vvcam_isp_gc.c
@@ -0,0 +1,254 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_gc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_gc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GC_ENABLE:
+        case VVCAM_ISP_CID_GC_RESET:
+        case VVCAM_ISP_CID_GC_MANU_USE_STD:
+        case VVCAM_ISP_CID_GC_MANU_STD_VAL:
+        case VVCAM_ISP_CID_GC_MANU_CURVE:
+        case VVCAM_ISP_CID_GC_MANU_CURVE_MODE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_gc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GC_ENABLE:
+        case VVCAM_ISP_CID_GC_RESET:
+        case VVCAM_ISP_CID_GC_MANU_USE_STD:
+        case VVCAM_ISP_CID_GC_MANU_STD_VAL:
+        case VVCAM_ISP_CID_GC_MANU_CURVE:
+        case VVCAM_ISP_CID_GC_MANU_CURVE_MODE:
+        case VVCAM_ISP_CID_GC_STAT_USE_STD:
+        case VVCAM_ISP_CID_GC_STAT_STD_VAL:
+        case VVCAM_ISP_CID_GC_STAT_CURVE:
+        case VVCAM_ISP_CID_GC_STAT_CURVE_MODE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_gc_ctrl_ops = {
+	.s_ctrl = vvcam_isp_gc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_gc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_gc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_USE_STD,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_use_std",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 1.0 ~ 4.0 */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_STD_VAL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_std_val",
+        .step = 1,
+        .min  = 100,
+        .max  = 400,
+        .def  = 220,
+    },
+    {
+        /* uint16_t 17*16bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 64*32bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_CURVE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_curve_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_USE_STD,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_use_std",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 1.0 ~ 4.0 */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_STD_VAL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_std_val",
+        .step = 1,
+        .min  = 100,
+        .max  = 400,
+        .def  = 220,
+    },
+    {
+        /* uint16_t 17*16bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 64*32bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_CURVE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_curve_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+};
+
+int vvcam_isp_gc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_gc_ctrls);
+}
+
+int vvcam_isp_gc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_gc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_gc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp gc ctrl %s failed %d.\n",
+                vvcam_isp_gc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v1/vvcam_isp_gc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v1/vvcam_isp_gc.h
new file mode 100644
index 000000000000..d7bc5dcb7a45
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v1/vvcam_isp_gc.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_GC_H__
+#define __VVCAM_ISP_GC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_GC_ENABLE             (VVCAM_ISP_CID_GC_BASE + 0x0000)
+#define VVCAM_ISP_CID_GC_RESET              (VVCAM_ISP_CID_GC_BASE + 0x0001)
+#define VVCAM_ISP_CID_GC_MANU_USE_STD       (VVCAM_ISP_CID_GC_BASE + 0x0002)
+#define VVCAM_ISP_CID_GC_MANU_STD_VAL       (VVCAM_ISP_CID_GC_BASE + 0x0003)
+#define VVCAM_ISP_CID_GC_MANU_CURVE         (VVCAM_ISP_CID_GC_BASE + 0x0004)
+#define VVCAM_ISP_CID_GC_MANU_CURVE_MODE    (VVCAM_ISP_CID_GC_BASE + 0x0005)
+#define VVCAM_ISP_CID_GC_STAT_USE_STD       (VVCAM_ISP_CID_GC_BASE + 0x0006)
+#define VVCAM_ISP_CID_GC_STAT_STD_VAL       (VVCAM_ISP_CID_GC_BASE + 0x0007)
+#define VVCAM_ISP_CID_GC_STAT_CURVE         (VVCAM_ISP_CID_GC_BASE + 0x0008)
+#define VVCAM_ISP_CID_GC_STAT_CURVE_MODE    (VVCAM_ISP_CID_GC_BASE + 0x0009)
+
+#ifdef __KERNEL__
+int vvcam_isp_gc_ctrl_count(void);
+int vvcam_isp_gc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v2/vvcam_isp_gc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v2/vvcam_isp_gc.c
new file mode 100644
index 000000000000..69bf97ba9356
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v2/vvcam_isp_gc.c
@@ -0,0 +1,333 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_gc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_gc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GC_ENABLE:
+        case VVCAM_ISP_CID_GC_RESET:
+        case VVCAM_ISP_CID_GC_MODE:
+        case VVCAM_ISP_CID_GC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_GC_AUTO_GAIN:
+        case VVCAM_ISP_CID_GC_AUTO_CURVE:
+        case VVCAM_ISP_CID_GC_MANU_USE_STD:
+        case VVCAM_ISP_CID_GC_MANU_STD_VAL:
+        case VVCAM_ISP_CID_GC_MANU_CURVE:
+        case VVCAM_ISP_CID_GC_MANU_USER_X:
+        case VVCAM_ISP_CID_GC_MANU_CURVE_X:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_gc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GC_ENABLE:
+        case VVCAM_ISP_CID_GC_MODE:
+        case VVCAM_ISP_CID_GC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_GC_AUTO_GAIN:
+        case VVCAM_ISP_CID_GC_AUTO_CURVE:
+        case VVCAM_ISP_CID_GC_MANU_USE_STD:
+        case VVCAM_ISP_CID_GC_MANU_STD_VAL:
+        case VVCAM_ISP_CID_GC_MANU_CURVE:
+        case VVCAM_ISP_CID_GC_MANU_USER_X:
+        case VVCAM_ISP_CID_GC_MANU_CURVE_X:
+        case VVCAM_ISP_CID_GC_STAT_USE_STD:
+        case VVCAM_ISP_CID_GC_STAT_STD_VAL:
+        case VVCAM_ISP_CID_GC_STAT_CURVE:
+        case VVCAM_ISP_CID_GC_STAT_USER_X:
+        case VVCAM_ISP_CID_GC_STAT_CURVE_X:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_gc_ctrl_ops = {
+	.s_ctrl = vvcam_isp_gc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_gc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_gc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        // uint16_t  20 * 64 * 16bit
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_AUTO_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_auto_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 64, 0, 0}
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_USE_STD,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_use_std",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 1.0 ~ 4.0 */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_STD_VAL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_std_val",
+        .step = 1,
+        .min  = 100,
+        .max  = 400,
+        .def  = 220,
+    },
+    {
+        /* uint16_t 64*16bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_USER_X,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_use_user_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint32_t array 64*32bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_MANU_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_manu_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_USE_STD,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_use_std",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 1.0 ~ 4.0 */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_STD_VAL,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_std_val",
+        .step = 1,
+        .min  = 100,
+        .max  = 400,
+        .def  = 220,
+    },
+    {
+        /* uint16_t 64*16bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {64},
+    },
+    {
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_USER_X,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_use_user_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint32_t array 64*32bit */
+        .ops  = &vvcam_isp_gc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GC_STAT_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gc_stat_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 12,
+        .dims = {64},
+    },
+};
+
+int vvcam_isp_gc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_gc_ctrls);
+}
+
+int vvcam_isp_gc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_gc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_gc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp gc ctrl %s failed %d.\n",
+                vvcam_isp_gc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v2/vvcam_isp_gc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v2/vvcam_isp_gc.h
new file mode 100644
index 000000000000..3cbe8aa45d05
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gc/gc_v2/vvcam_isp_gc.h
@@ -0,0 +1,83 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_GC_H__
+#define __VVCAM_ISP_GC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_GC_ENABLE             (VVCAM_ISP_CID_GC_BASE + 0x0000)
+#define VVCAM_ISP_CID_GC_RESET              (VVCAM_ISP_CID_GC_BASE + 0x0001)
+#define VVCAM_ISP_CID_GC_MODE               (VVCAM_ISP_CID_GC_BASE + 0x0002)
+#define VVCAM_ISP_CID_GC_AUTO_LEVEL         (VVCAM_ISP_CID_GC_BASE + 0x0003)
+#define VVCAM_ISP_CID_GC_AUTO_GAIN          (VVCAM_ISP_CID_GC_BASE + 0x0004)
+#define VVCAM_ISP_CID_GC_AUTO_CURVE         (VVCAM_ISP_CID_GC_BASE + 0x0005)
+#define VVCAM_ISP_CID_GC_MANU_USE_STD       (VVCAM_ISP_CID_GC_BASE + 0x0006)
+#define VVCAM_ISP_CID_GC_MANU_STD_VAL       (VVCAM_ISP_CID_GC_BASE + 0x0007)
+#define VVCAM_ISP_CID_GC_MANU_CURVE         (VVCAM_ISP_CID_GC_BASE + 0x0008)
+#define VVCAM_ISP_CID_GC_MANU_USER_X        (VVCAM_ISP_CID_GC_BASE + 0x0009)
+#define VVCAM_ISP_CID_GC_MANU_CURVE_X       (VVCAM_ISP_CID_GC_BASE + 0x000A)
+#define VVCAM_ISP_CID_GC_STAT_USE_STD       (VVCAM_ISP_CID_GC_BASE + 0x000B)
+#define VVCAM_ISP_CID_GC_STAT_STD_VAL       (VVCAM_ISP_CID_GC_BASE + 0x000C)
+#define VVCAM_ISP_CID_GC_STAT_CURVE         (VVCAM_ISP_CID_GC_BASE + 0x000D)
+#define VVCAM_ISP_CID_GC_STAT_USER_X        (VVCAM_ISP_CID_GC_BASE + 0x000E)
+#define VVCAM_ISP_CID_GC_STAT_CURVE_X       (VVCAM_ISP_CID_GC_BASE + 0x000F)
+
+#ifdef __KERNEL__
+int vvcam_isp_gc_ctrl_count(void);
+int vvcam_isp_gc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ge/ge/vvcam_isp_ge.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ge/ge/vvcam_isp_ge.c
new file mode 100644
index 000000000000..0d482d5eaabf
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ge/ge/vvcam_isp_ge.c
@@ -0,0 +1,176 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ge.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ge_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GE_ENABLE:
+        case VVCAM_ISP_CID_GE_RESET:
+        case VVCAM_ISP_CID_GE_THRESHOLD:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ge_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GE_ENABLE:
+        case VVCAM_ISP_CID_GE_RESET:
+        case VVCAM_ISP_CID_GE_THRESHOLD:
+        case VVCAM_ISP_CID_GE_STAT_THRESHOLD:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ge_ctrl_ops = {
+	.s_ctrl = vvcam_isp_ge_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ge_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ge_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ge_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ge_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ge_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GE_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ge_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float 0.0 ~ 511.992 */
+        .ops  = &vvcam_isp_ge_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ge_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+    {
+        /* float 0.0 ~ 511.992 */
+        .ops  = &vvcam_isp_ge_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GE_STAT_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ge_stat_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+    },
+};
+
+int vvcam_isp_ge_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ge_ctrls);
+}
+
+int vvcam_isp_ge_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ge_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ge_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ge ctrl %s failed %d.\n",
+                vvcam_isp_ge_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ge/ge/vvcam_isp_ge.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ge/ge/vvcam_isp_ge.h
new file mode 100644
index 000000000000..f445eb55d595
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ge/ge/vvcam_isp_ge.h
@@ -0,0 +1,72 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_GE_H__
+#define __VVCAM_ISP_GE_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_GE_ENABLE             (VVCAM_ISP_CID_GE_BASE + 0x0000)
+#define VVCAM_ISP_CID_GE_RESET              (VVCAM_ISP_CID_GE_BASE + 0x0001)
+#define VVCAM_ISP_CID_GE_THRESHOLD          (VVCAM_ISP_CID_GE_BASE + 0x0002)
+#define VVCAM_ISP_CID_GE_STAT_THRESHOLD     (VVCAM_ISP_CID_GE_BASE + 0x0003)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_ge_ctrl_count(void);
+int vvcam_isp_ge_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.c
new file mode 100644
index 000000000000..3b93204f992c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.c
@@ -0,0 +1,983 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_gtm.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_gtm_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GTM_ENABLE:
+        case VVCAM_ISP_CID_GTM_BW_CORRECTION_ENABLE:
+        case VVCAM_ISP_CID_GTM_RESET:
+        case VVCAM_ISP_CID_GTM_MODE:
+        case VVCAM_ISP_CID_GTM_AUTO_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_USER_CURVE_ENABLE:
+        case VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_CONTRAST_LIMIT_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_HLC_FACTOR:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_PRESERVE_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_MAX_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_MIN_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_STRENGTH:
+        case VVCAM_ISP_CID_GTM_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_USER_CURVE:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_MODE:
+        case VVCAM_ISP_CID_GTM_AUTO_EDR_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_CONTRAST_LIMIT_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_THR_LOG:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_HLC_FACTOR:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_PRESERVE_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_COMPRESS_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MAX_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MIN_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_STRENGTH:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_DAMP_COEF:
+        case VVCAM_ISP_CID_GTM_AUTO_EDR:
+        case VVCAM_ISP_CID_GTM_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_GTM_MANU_LIGHTNESS_WEIGHT:
+        case VVCAM_ISP_CID_GTM_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_MODE:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_DAMP_COEF:
+        case VVCAM_ISP_CID_GTM_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_GTM_MANU_CURVE_SHIFT_BIT:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_X_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_Y_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_X_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_SLOPE:
+        case VVCAM_ISP_CID_GTM_MANU_USER_CURVE:
+        case VVCAM_ISP_CID_GTM_HIST_STATISTIC_MIN:
+        case VVCAM_ISP_CID_GTM_HIST_STATISTIC_MAX:
+        case VVCAM_ISP_CID_GTM_HIST_STATISTIC_DATA:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_gtm_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GTM_ENABLE:
+        case VVCAM_ISP_CID_GTM_BW_CORRECTION_ENABLE:
+        case VVCAM_ISP_CID_GTM_RESET:
+        case VVCAM_ISP_CID_GTM_MODE:
+        case VVCAM_ISP_CID_GTM_AUTO_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_USER_CURVE_ENABLE:
+        case VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_CONTRAST_LIMIT_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_HLC_FACTOR:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_PRESERVE_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_MAX_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_MIN_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_STRENGTH:
+        case VVCAM_ISP_CID_GTM_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_USER_CURVE:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_MODE:
+        case VVCAM_ISP_CID_GTM_AUTO_EDR_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_CONTRAST_LIMIT_LEVEL:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_THR_LOG:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_HLC_FACTOR:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_PRESERVE_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_COMPRESS_WEIGHT:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MAX_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MIN_GAIN:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_STRENGTH:
+        case VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_DAMP_COEF:
+        case VVCAM_ISP_CID_GTM_AUTO_EDR:
+        case VVCAM_ISP_CID_GTM_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_GTM_MANU_LIGHTNESS_WEIGHT:
+        case VVCAM_ISP_CID_GTM_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_MODE:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_BW_COR_DAMP_COEF:
+        case VVCAM_ISP_CID_GTM_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_GTM_MANU_CURVE_SHIFT_BIT:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_X_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_Y_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_PWL_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_X_LOG:
+        case VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_SLOPE:
+        case VVCAM_ISP_CID_GTM_MANU_USER_CURVE:
+        case VVCAM_ISP_CID_GTM_HIST_STATISTIC_MIN:
+        case VVCAM_ISP_CID_GTM_HIST_STATISTIC_MAX:
+        case VVCAM_ISP_CID_GTM_HIST_STATISTIC_DATA:
+        case VVCAM_ISP_CID_GTM_STAT_RGB_COEF:
+        case VVCAM_ISP_CID_GTM_STAT_LIGHTNESS_WEIGHT:
+        case VVCAM_ISP_CID_GTM_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_GTM_STAT_BW_COR_MODE:
+        case VVCAM_ISP_CID_GTM_STAT_BW_COR_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_BW_COR_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_BW_COR_DAMP_COEF:
+        case VVCAM_ISP_CID_GTM_STAT_CURVE_SELECT:
+        case VVCAM_ISP_CID_GTM_STAT_CURVE_SHIFT_BIT:
+        case VVCAM_ISP_CID_GTM_STAT_PWL_KNEE_X_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_PWL_KNEE_Y_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_PWL_MAX_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_PWL_MIN_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_LOG_KNEE_X_LOG:
+        case VVCAM_ISP_CID_GTM_STAT_LOG_KNEE_SLOPE:
+        case VVCAM_ISP_CID_GTM_STAT_USER_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_gtm_ctrl_ops = {
+    .s_ctrl = vvcam_isp_gtm_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_gtm_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_gtm_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_BW_CORRECTION_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_bw_cor_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* 0: Manual 1: Auto */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float array 20x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MIN_LOG,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_bw_cor_min_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 24,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MAX_LOG,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_bw_cor_max_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 24,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 20x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_CONTRAST_LIMIT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_con_lmt_lvl",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 10.0*/
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_HLC_FACTOR,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_hlc_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 1.0*/
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_PRESERVE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_luma_pres_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 1048575.0*/
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x 0.0 ~ 1048575.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float array 20x20 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 20, 0, 0},
+    },
+    {
+        /* uint8_t array 20x3 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 3, 0, 0},
+    },
+    {
+        /* uint16_t array 20x4 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_hist_luma_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 4, 0, 0},
+    },
+    {
+         // uint32_t array 20x129 0~1048575
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_USER_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_user_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {20, 129, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_USER_CURVE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_user_curve_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_CONTRAST_LIMIT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_cll",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t array 4x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_ltl",
+        .step = 1,
+        .min  = 0,
+        .max  = 24,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* uint16_t array 4x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_luma_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 10.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_HLC_FACTOR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_hlc_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_PRESERVE_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_lpw",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_COMPRESS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_comp_wt",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1048575.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_max_g",
+        .step = 1,
+        .min  = 0,
+        .max  = 104857500,
+    },
+    {
+        /* float 0.0 ~ 1048575.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_min_g",
+        .step = 1,
+        .min  = 0,
+        .max  = 104857500,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_STRENGTH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_str",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_DAMP_COEF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_curve_hist_damp_c",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_EDR_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_edr_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 20,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x  */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_AUTO_EDR,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_auto_edr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_LIGHTNESS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_BW_COR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_bw_cor_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* double  0.0 ~ 24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_BW_COR_MIN_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_bw_cor_min_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double  0.0 ~ 24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_BW_COR_MAX_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_bw_cor_max_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double  0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_BW_COR_DAMP_COEF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_bw_cor_damp_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_CURVE_SHIFT_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_curve_shift_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        /* double 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_pwl_knee_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double 0~20.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_Y_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_pwl_knee_y_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* double 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_PWL_MAX_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_pwl_max_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_PWL_MIN_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_pwl_min_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* float 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_log_knee_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* float 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_log_knee_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* uint32_t array 129x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_MANU_USER_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_manu_user_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {129, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_HIST_STATISTIC_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_hist_statistic_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_HIST_STATISTIC_MAX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_hist_statistic_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint32_t array 128x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_HIST_STATISTIC_DATA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_hist_statistic_data",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {128, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_LIGHTNESS_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_BW_COR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_bw_cor_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        /* double  0.0 ~ 24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_BW_COR_MIN_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_bw_cor_min_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double  0.0 ~ 24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_BW_COR_MAX_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_bw_cor_max_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double  0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_BW_COR_DAMP_COEF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_bw_cor_damp_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_CURVE_SHIFT_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_curve_shift_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 4,
+    },
+    {
+        /* double 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_PWL_KNEE_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_pwl_knee_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double 0~20.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_PWL_KNEE_Y_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_pwl_knee_y_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 200,
+    },
+    {
+        /* double 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_PWL_MAX_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_pwl_max_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* double 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_PWL_MIN_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_pwl_min_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* float 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_LOG_KNEE_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_log_knee_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* float 0~24.0 */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_LOG_KNEE_SLOPE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_log_knee_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 240,
+    },
+    {
+        /* uint32_t array 129x */
+        .ops  = &vvcam_isp_gtm_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GTM_STAT_USER_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gtm_stat_user_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {129, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_gtm_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_gtm_ctrls);
+}
+
+int vvcam_isp_gtm_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_gtm_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_gtm_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp gtm ctrl %s failed %d.\n",
+                vvcam_isp_gtm_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.h
new file mode 100644
index 000000000000..463b6465c25f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gtm/gtm_v1/vvcam_isp_gtm.h
@@ -0,0 +1,185 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_GTM_H__
+#define __VVCAM_ISP_GTM_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_GTM_ENABLE            (VVCAM_ISP_CID_GTM_BASE + 0x0000)
+#define VVCAM_ISP_CID_GTM_BW_CORRECTION_ENABLE \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0001)
+#define VVCAM_ISP_CID_GTM_RESET             (VVCAM_ISP_CID_GTM_BASE + 0x0002)
+#define VVCAM_ISP_CID_GTM_MODE              (VVCAM_ISP_CID_GTM_BASE + 0x0003)
+#define VVCAM_ISP_CID_GTM_AUTO_LEVEL        (VVCAM_ISP_CID_GTM_BASE + 0x0004)
+#define VVCAM_ISP_CID_GTM_AUTO_GAIN         (VVCAM_ISP_CID_GTM_BASE + 0x0005)
+#define VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MIN_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0006)
+#define VVCAM_ISP_CID_GTM_AUTO_BW_CORRECTION_MAX_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0007)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_CONTRAST_LIMIT_LEVEL \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0008)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_HLC_FACTOR \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0009)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_PRESERVE_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x000A)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_MAX_GAIN \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x000B)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_MIN_GAIN \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x000C)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_STRENGTH \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x000D)
+#define VVCAM_ISP_CID_GTM_AUTO_COLOR_WEIGHT (VVCAM_ISP_CID_GTM_BASE + 0x000E)
+#define VVCAM_ISP_CID_GTM_AUTO_HIST_LUMA_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x000F)
+#define VVCAM_ISP_CID_GTM_AUTO_USER_CURVE   (VVCAM_ISP_CID_GTM_BASE + 0x0010)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_MODE   (VVCAM_ISP_CID_GTM_BASE + 0x0011)
+#define VVCAM_ISP_CID_GTM_AUTO_USER_CURVE_ENABLE \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0012)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_CONTRAST_LIMIT_LEVEL \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0013)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_THR_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0014)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0015)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_HLC_FACTOR \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0016)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_LUMA_PRESERVE_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0017)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_COMPRESS_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0018)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MAX_GAIN \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0019)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_MIN_GAIN \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x001A)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_STRENGTH \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x001B)
+#define VVCAM_ISP_CID_GTM_AUTO_CURVE_HIST_DAMP_COEF \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x001C)
+#define VVCAM_ISP_CID_GTM_AUTO_EDR_LEVEL    (VVCAM_ISP_CID_GTM_BASE + 0x001D)
+#define VVCAM_ISP_CID_GTM_AUTO_EDR          (VVCAM_ISP_CID_GTM_BASE + 0x001E)
+#define VVCAM_ISP_CID_GTM_MANU_RGB_COEF     (VVCAM_ISP_CID_GTM_BASE + 0x001F)
+#define VVCAM_ISP_CID_GTM_MANU_LIGHTNESS_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0020)
+#define VVCAM_ISP_CID_GTM_MANU_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0021)
+#define VVCAM_ISP_CID_GTM_MANU_BW_COR_MODE \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0022)
+#define VVCAM_ISP_CID_GTM_MANU_BW_COR_MIN_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0023)
+#define VVCAM_ISP_CID_GTM_MANU_BW_COR_MAX_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0024)
+#define VVCAM_ISP_CID_GTM_MANU_BW_COR_DAMP_COEF \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0025)
+#define VVCAM_ISP_CID_GTM_MANU_CURVE_SELECT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0026)
+#define VVCAM_ISP_CID_GTM_MANU_CURVE_SHIFT_BIT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0027)
+#define VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_X_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0028)
+#define VVCAM_ISP_CID_GTM_MANU_PWL_KNEE_Y_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0029)
+#define VVCAM_ISP_CID_GTM_MANU_PWL_MAX_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x002A)
+#define VVCAM_ISP_CID_GTM_MANU_PWL_MIN_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x002B)
+#define VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_X_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x002C)
+#define VVCAM_ISP_CID_GTM_MANU_LOG_KNEE_SLOPE \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x002D)
+#define VVCAM_ISP_CID_GTM_MANU_USER_CURVE   (VVCAM_ISP_CID_GTM_BASE + 0x002E)
+#define VVCAM_ISP_CID_GTM_HIST_STATISTIC_MIN \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x002F)
+#define VVCAM_ISP_CID_GTM_HIST_STATISTIC_MAX \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0030)
+#define VVCAM_ISP_CID_GTM_HIST_STATISTIC_DATA \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0031)
+#define VVCAM_ISP_CID_GTM_STAT_RGB_COEF     (VVCAM_ISP_CID_GTM_BASE + 0x0032)
+#define VVCAM_ISP_CID_GTM_STAT_LIGHTNESS_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0033)
+#define VVCAM_ISP_CID_GTM_STAT_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0034)
+#define VVCAM_ISP_CID_GTM_STAT_BW_COR_MODE \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0035)
+#define VVCAM_ISP_CID_GTM_STAT_BW_COR_MIN_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0036)
+#define VVCAM_ISP_CID_GTM_STAT_BW_COR_MAX_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0037)
+#define VVCAM_ISP_CID_GTM_STAT_BW_COR_DAMP_COEF \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0038)
+#define VVCAM_ISP_CID_GTM_STAT_CURVE_SELECT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0039)
+#define VVCAM_ISP_CID_GTM_STAT_CURVE_SHIFT_BIT \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x003A)
+#define VVCAM_ISP_CID_GTM_STAT_PWL_KNEE_X_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x003B)
+#define VVCAM_ISP_CID_GTM_STAT_PWL_KNEE_Y_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x003C)
+#define VVCAM_ISP_CID_GTM_STAT_PWL_MAX_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x003D)
+#define VVCAM_ISP_CID_GTM_STAT_PWL_MIN_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x003E)
+#define VVCAM_ISP_CID_GTM_STAT_LOG_KNEE_X_LOG \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x003F)
+#define VVCAM_ISP_CID_GTM_STAT_LOG_KNEE_SLOPE \
+                                            (VVCAM_ISP_CID_GTM_BASE + 0x0040)
+#define VVCAM_ISP_CID_GTM_STAT_USER_CURVE   (VVCAM_ISP_CID_GTM_BASE + 0x0041)
+
+#ifdef __KERNEL__
+int vvcam_isp_gtm_ctrl_count(void);
+int vvcam_isp_gtm_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.c
new file mode 100644
index 000000000000..6630d75101a1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.c
@@ -0,0 +1,279 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_gwdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_gwdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GWDR_ENABLE:
+        case VVCAM_ISP_CID_GWDR_RESET:
+        case VVCAM_ISP_CID_GWDR_MANU_MODE:
+        case VVCAM_ISP_CID_GWDR_MANU_RGB_OFFSET:
+        case VVCAM_ISP_CID_GWDR_MANU_LUM_OFFSET:
+        case VVCAM_ISP_CID_GWDR_MANU_GAMMA:
+        case VVCAM_ISP_CID_GWDR_MANU_CURVE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_gwdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_GWDR_ENABLE:
+        case VVCAM_ISP_CID_GWDR_RESET:
+        case VVCAM_ISP_CID_GWDR_MANU_MODE:
+        case VVCAM_ISP_CID_GWDR_MANU_RGB_OFFSET:
+        case VVCAM_ISP_CID_GWDR_MANU_LUM_OFFSET:
+        case VVCAM_ISP_CID_GWDR_MANU_GAMMA:
+        case VVCAM_ISP_CID_GWDR_MANU_CURVE:
+        case VVCAM_ISP_CID_GWDR_STAT_MODE:
+        case VVCAM_ISP_CID_GWDR_STAT_RGB_OFFSET:
+        case VVCAM_ISP_CID_GWDR_STAT_LUM_OFFSET:
+        case VVCAM_ISP_CID_GWDR_STAT_GAMMA:
+        case VVCAM_ISP_CID_GWDR_STAT_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_gwdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_gwdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_gwdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_gwdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_MANU_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_manu_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_MANU_RGB_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_manu_rgb_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_MANU_LUM_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_manu_lum_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 4.0 */
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_MANU_GAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_manu_gamma",
+        .step = 1,
+        .min  = 100,
+        .max  = 400,
+        .def  = 220,
+    },
+    {
+        /* uint16_t 33x array */
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_MANU_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_manu_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {33, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_STAT_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_stat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_STAT_RGB_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_stat_rgb_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_STAT_LUM_OFFSET,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_stat_lum_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 4.0 */
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_STAT_GAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_stat_gamma",
+        .step = 1,
+        .min  = 100,
+        .max  = 400,
+        .def  = 220,
+    },
+    {
+        /* uint16_t 33x array */
+        .ops  = &vvcam_isp_gwdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_GWDR_STAT_CURVE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_gwdr_stat_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {33, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_gwdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_gwdr_ctrls);
+}
+
+int vvcam_isp_gwdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_gwdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_gwdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp gwdr ctrl %s failed %d.\n",
+                vvcam_isp_gwdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.h
new file mode 100644
index 000000000000..4592433a1fb9
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/gwdr/gwdr_v1/vvcam_isp_gwdr.h
@@ -0,0 +1,79 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_GWDR_H__
+#define __VVCAM_ISP_GWDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_GWDR_ENABLE           (VVCAM_ISP_CID_GWDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_GWDR_RESET            (VVCAM_ISP_CID_GWDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_GWDR_MANU_MODE        (VVCAM_ISP_CID_GWDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_GWDR_MANU_RGB_OFFSET  (VVCAM_ISP_CID_GWDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_GWDR_MANU_LUM_OFFSET  (VVCAM_ISP_CID_GWDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_GWDR_MANU_GAMMA       (VVCAM_ISP_CID_GWDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_GWDR_MANU_CURVE       (VVCAM_ISP_CID_GWDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_GWDR_STAT_MODE        (VVCAM_ISP_CID_GWDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_GWDR_STAT_RGB_OFFSET  (VVCAM_ISP_CID_GWDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_GWDR_STAT_LUM_OFFSET  (VVCAM_ISP_CID_GWDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_GWDR_STAT_GAMMA       (VVCAM_ISP_CID_GWDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_GWDR_STAT_CURVE       (VVCAM_ISP_CID_GWDR_BASE + 0x000B)
+
+#ifdef __KERNEL__
+int vvcam_isp_gwdr_ctrl_count(void);
+int vvcam_isp_gwdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..94b7e24c12c1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.c
@@ -0,0 +1,260 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1, 256] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 2x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 2x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..7e6f88d97798
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v1_3/vvcam_isp_hdr.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_EXTEND_BIT        (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT   (VVCAM_ISP_CID_HDR_BASE + 0x0009)
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..1c174a92d045
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.c
@@ -0,0 +1,1178 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_DPF_L_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_S_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE:
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DPF_L_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_L_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_L_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_S_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_S_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_S_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_VS_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_VS_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_L_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_S_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DPF_L_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_L_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_L_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_S_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_S_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_S_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_VS_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_VS_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_GAIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_GRADIENT:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_OFFSET:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_MIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_DIV:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_GAIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_GRADIENT:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_OFFSET:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_MIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_DIV:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_GAIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_GRADIENT:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_OFFSET:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_MIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_DIV:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_NOISE_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_deghost_ls_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_deghost_lsvs_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t 2x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_motion_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_motion_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_motion_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_motion_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_motion_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t 2x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_mo_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_mo_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_mo_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_mo_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_mo_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_dk_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_dk_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_dk_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_dk_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..bfe77d1315e6
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2/vvcam_isp_hdr.h
@@ -0,0 +1,185 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_DPF_L_ENABLE      (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_DPF_S_ENABLE      (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_DPF_VS_ENABLE     (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_HDR_EXTEND_BIT        (VVCAM_ISP_CID_HDR_BASE + 0x000A)
+
+#define VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_HDR_DPF_L_GAIN        (VVCAM_ISP_CID_HDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_HDR_DPF_L_GRADIENT    (VVCAM_ISP_CID_HDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_HDR_DPF_L_OFFSET      (VVCAM_ISP_CID_HDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_HDR_DPF_L_MIN         (VVCAM_ISP_CID_HDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_HDR_DPF_L_DIV         (VVCAM_ISP_CID_HDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G     (VVCAM_ISP_CID_HDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB    (VVCAM_ISP_CID_HDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE (VVCAM_ISP_CID_HDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_HDR_DPF_S_GAIN        (VVCAM_ISP_CID_HDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_HDR_DPF_S_GRADIENT    (VVCAM_ISP_CID_HDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_HDR_DPF_S_OFFSET      (VVCAM_ISP_CID_HDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_HDR_DPF_S_MIN         (VVCAM_ISP_CID_HDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_HDR_DPF_S_DIV         (VVCAM_ISP_CID_HDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G     (VVCAM_ISP_CID_HDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB    (VVCAM_ISP_CID_HDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE (VVCAM_ISP_CID_HDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_HDR_DPF_VS_GAIN       (VVCAM_ISP_CID_HDR_BASE + 0x0024)
+#define VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT   (VVCAM_ISP_CID_HDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_HDR_DPF_VS_OFFSET     (VVCAM_ISP_CID_HDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_HDR_DPF_VS_MIN        (VVCAM_ISP_CID_HDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_HDR_DPF_VS_DIV        (VVCAM_ISP_CID_HDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G    (VVCAM_ISP_CID_HDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB   (VVCAM_ISP_CID_HDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x002B)
+
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT   (VVCAM_ISP_CID_HDR_BASE + 0x002F)
+
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0038)
+
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_GAIN   (VVCAM_ISP_CID_HDR_BASE + 0x0039)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_GRADIENT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_OFFSET (VVCAM_ISP_CID_HDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_MIN    (VVCAM_ISP_CID_HDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_DIV    (VVCAM_ISP_CID_HDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_G \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_RB \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_GAIN   (VVCAM_ISP_CID_HDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_GRADIENT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0042)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_OFFSET (VVCAM_ISP_CID_HDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_MIN    (VVCAM_ISP_CID_HDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_DIV    (VVCAM_ISP_CID_HDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_G \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0046)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_RB \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_GAIN  (VVCAM_ISP_CID_HDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_GRADIENT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_OFFSET \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_MIN   (VVCAM_ISP_CID_HDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_DIV   (VVCAM_ISP_CID_HDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_G \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_RB \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004F)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0050)
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..3191a3bea767
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.c
@@ -0,0 +1,259 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..eb9dcde4048f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1/vvcam_isp_hdr.h
@@ -0,0 +1,78 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_EXTEND_BIT        (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT   (VVCAM_ISP_CID_HDR_BASE + 0x0009)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..d207af8d7a87
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.c
@@ -0,0 +1,257 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float [1.0, 256.0] ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 10000,
+        .max  = 2560000,
+        .def  = 10000,
+    },
+    {
+        /* float 2x2 array [0, 1.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 2, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_extend_bit",
+        .step = 1,
+        .min  = -1,
+        .max  = 8,
+        .def  = 0,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float [1.0, 256.0] ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 10000,
+        .max  = 2560000,
+        .def  = 10000,
+    },
+    {
+        /* float 2x2 array [0, 1.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 2, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_extend_bit",
+        .step = 1,
+        .min  = -1,
+        .max  = 8,
+        .def  = 0,
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..7e6f88d97798
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v2_1_2dol/vvcam_isp_hdr.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_EXTEND_BIT        (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT   (VVCAM_ISP_CID_HDR_BASE + 0x0009)
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..3a4bf8a67b5e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.c
@@ -0,0 +1,1499 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_DPF_L_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_S_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE:
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_MODE:
+        case VVCAM_ISP_CID_HDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_HDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_ENABLE_TBL:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_ENABLE_TBL:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_ENABLE_TBL:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_GRADIENT:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_OFFSET:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_MIN:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_DIV:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DPF_L_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_L_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_L_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_S_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_S_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_S_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_VS_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_VS_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_L_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_S_ENABLE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE:
+        case VVCAM_ISP_CID_HDR_MODE:
+        case VVCAM_ISP_CID_HDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_HDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_ENABLE_TBL:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_ENABLE_TBL:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_ENABLE_TBL:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_GRADIENT:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_OFFSET:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_MIN:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_DIV:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_AUTO_DPF_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DPF_L_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_L_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_L_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_L_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_S_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_S_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_S_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_S_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GAIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT:
+        case VVCAM_ISP_CID_HDR_DPF_VS_OFFSET:
+        case VVCAM_ISP_CID_HDR_DPF_VS_MIN:
+        case VVCAM_ISP_CID_HDR_DPF_VS_DIV:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_GAIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_GRADIENT:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_OFFSET:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_MIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_DIV:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_L_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_GAIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_GRADIENT:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_OFFSET:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_MIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_DIV:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_S_NOISE_CURVE:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_GAIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_GRADIENT:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_OFFSET:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_MIN:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_DIV:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_G:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_RB:
+        case VVCAM_ISP_CID_HDR_STAT_DPF_VS_NOISE_CURVE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_deghost_ls_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_deghost_lsvs_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        /* float array 20*32bit */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint32_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_ls_mot_en_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_lsvs_mot_en_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_ls_mot_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_lsvs_mot_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_ls_mot_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_ls_mot_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_lsvs_mot_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_lsvs_mot_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_ls_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_ls_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_lsvs_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_lsvs_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint32_t 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_GRADIENT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_gradient",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_OFFSET,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_offset",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_MIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_DIV,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_div",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_sigma_g",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_sigma_rb",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 20, 0, 0},
+    },
+    {
+        /* float 3x17x20 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_AUTO_DPF_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_auto_dpf_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 17, 20, 0},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t 2x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_motion_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_motion_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_motion_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_motion_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_motion_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_l_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_s_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dpf_vs_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t 2x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_mo_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_mo_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_mo_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_mo_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_mo_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_dk_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_dk_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_dk_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_dk_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_L_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_l_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_S_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_s_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+    {
+        /* float 1.0 ~ 1000.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_gain",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 10000000,
+    },
+    {
+        /* float 0.1 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_GRADIENT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_gradient",
+        .step = 1,
+        .def  = 1000,
+        .min  = 1000,
+        .max  = 1280000,
+    },
+    {
+        /* float 0.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_OFFSET,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_offset",
+        .step = 1,
+        .def  = 0,
+        .min  = 0,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_MIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_min",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 64.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_DIV,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_div",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 640000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_G,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_sigma_g",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 1.0 ~ 128.0, ratio 10000:1 */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_RB,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_sigma_rb",
+        .step = 1,
+        .def  = 10000,
+        .min  = 10000,
+        .max  = 1280000,
+    },
+    {
+        /* float 17x array [0, 4095] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DPF_VS_NOISE_CURVE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dpf_vs_noise_curve",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {17, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..1fa8f986f461
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3/vvcam_isp_hdr.h
@@ -0,0 +1,222 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_DPF_L_ENABLE      (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_DPF_S_ENABLE      (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_DPF_VS_ENABLE     (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_MODE              (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_HDR_AUTO_LEVEL        (VVCAM_ISP_CID_HDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_HDR_AUTO_GAIN         (VVCAM_ISP_CID_HDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_HDR_AUTO_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_HDR_AUTO_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_HDR_AUTO_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_HDR_AUTO_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_HDR_AUTO_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_GRADIENT (VVCAM_ISP_CID_HDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_OFFSET   (VVCAM_ISP_CID_HDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_MIN      (VVCAM_ISP_CID_HDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_DIV      (VVCAM_ISP_CID_HDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_G  (VVCAM_ISP_CID_HDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_SIGMA_RB (VVCAM_ISP_CID_HDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_HDR_AUTO_DPF_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_HDR_EXTEND_BIT        (VVCAM_ISP_CID_HDR_BASE + 0x0021)
+
+#define VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0024)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_HDR_DPF_L_GAIN        (VVCAM_ISP_CID_HDR_BASE + 0x002B)
+#define VVCAM_ISP_CID_HDR_DPF_L_GRADIENT    (VVCAM_ISP_CID_HDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_HDR_DPF_L_OFFSET      (VVCAM_ISP_CID_HDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_HDR_DPF_L_MIN         (VVCAM_ISP_CID_HDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_HDR_DPF_L_DIV         (VVCAM_ISP_CID_HDR_BASE + 0x002F)
+#define VVCAM_ISP_CID_HDR_DPF_L_SIGMA_G     (VVCAM_ISP_CID_HDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_HDR_DPF_L_SIGMA_RB    (VVCAM_ISP_CID_HDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_HDR_DPF_L_NOISE_CURVE (VVCAM_ISP_CID_HDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_HDR_DPF_S_GAIN        (VVCAM_ISP_CID_HDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_HDR_DPF_S_GRADIENT    (VVCAM_ISP_CID_HDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_HDR_DPF_S_OFFSET      (VVCAM_ISP_CID_HDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_HDR_DPF_S_MIN         (VVCAM_ISP_CID_HDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_HDR_DPF_S_DIV         (VVCAM_ISP_CID_HDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_HDR_DPF_S_SIGMA_G     (VVCAM_ISP_CID_HDR_BASE + 0x0038)
+#define VVCAM_ISP_CID_HDR_DPF_S_SIGMA_RB    (VVCAM_ISP_CID_HDR_BASE + 0x0039)
+#define VVCAM_ISP_CID_HDR_DPF_S_NOISE_CURVE (VVCAM_ISP_CID_HDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_HDR_DPF_VS_GAIN       (VVCAM_ISP_CID_HDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_HDR_DPF_VS_GRADIENT   (VVCAM_ISP_CID_HDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_HDR_DPF_VS_OFFSET     (VVCAM_ISP_CID_HDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_HDR_DPF_VS_MIN        (VVCAM_ISP_CID_HDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_HDR_DPF_VS_DIV        (VVCAM_ISP_CID_HDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_G    (VVCAM_ISP_CID_HDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_HDR_DPF_VS_SIGMA_RB   (VVCAM_ISP_CID_HDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_HDR_DPF_VS_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0042)
+
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT   (VVCAM_ISP_CID_HDR_BASE + 0x0046)
+
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x004F)
+
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_GAIN   (VVCAM_ISP_CID_HDR_BASE + 0x0050)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_GRADIENT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0051)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_OFFSET (VVCAM_ISP_CID_HDR_BASE + 0x0052)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_MIN    (VVCAM_ISP_CID_HDR_BASE + 0x0053)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_DIV    (VVCAM_ISP_CID_HDR_BASE + 0x0054)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_G \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0055)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_SIGMA_RB \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0056)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_L_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0057)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_GAIN   (VVCAM_ISP_CID_HDR_BASE + 0x0058)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_GRADIENT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0059)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_OFFSET (VVCAM_ISP_CID_HDR_BASE + 0x005A)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_MIN    (VVCAM_ISP_CID_HDR_BASE + 0x005B)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_DIV    (VVCAM_ISP_CID_HDR_BASE + 0x005C)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_G \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x005D)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_SIGMA_RB \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x005E)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_S_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x005F)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_GAIN  (VVCAM_ISP_CID_HDR_BASE + 0x0060)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_GRADIENT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0061)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_OFFSET \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0062)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_MIN   (VVCAM_ISP_CID_HDR_BASE + 0x0063)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_DIV   (VVCAM_ISP_CID_HDR_BASE + 0x0064)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_G \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0065)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_SIGMA_RB \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0066)
+#define VVCAM_ISP_CID_HDR_STAT_DPF_VS_NOISE_CURVE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0067)
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..bab6a3ad4db8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.c
@@ -0,0 +1,232 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 6x2 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {6, 2, 0, 0},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 6x2 array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {6, 2, 0, 0},
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..7d7da39d6d59
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_1/vvcam_isp_hdr.h
@@ -0,0 +1,74 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.c
new file mode 100644
index 000000000000..9450dcb69997
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.c
@@ -0,0 +1,494 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_hdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_hdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE:
+        case VVCAM_ISP_CID_HDR_RESET:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_hdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_HDR_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE:
+        case VVCAM_ISP_CID_HDR_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_RATIO:
+        case VVCAM_ISP_CID_HDR_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_RATIO:
+        case VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE:
+        case VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR:
+        case VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_hdr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_hdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_hdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_hdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_deghost_ls_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_deghost_lsvs_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t 2x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_motion_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+        {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_motion_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_motion_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_motion_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_motion_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_ls_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_dark_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_dg_lsvs_dark_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {3, 0, 0, 0},
+    },
+    {
+        /* float 2x array [1.0, 256.0] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_ratio",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* float 4x2 array [0, 1.1] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_trans_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4, 2, 0, 0},
+    },
+    {
+        /* int8_t 2x array [-1, 8] */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_extend_bit",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        /* uint16_t 2x array */
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_mo_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1024,
+        .dims = {2, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_mo_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_mo_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_mo_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_mo_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_dk_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_ls_dk_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_dk_lwr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+    {
+        .ops  = &vvcam_isp_hdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_hdr_stat_dg_lsvs_dk_upr_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+    },
+};
+
+int vvcam_isp_hdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_hdr_ctrls);
+}
+
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_hdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_hdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp hdr ctrl %s failed %d.\n",
+                vvcam_isp_hdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.h
new file mode 100644
index 000000000000..a57052d3e510
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/hdr/hdr_v3_2dol/vvcam_isp_hdr.h
@@ -0,0 +1,119 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_HDR_H__
+#define __VVCAM_ISP_HDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_HDR_ENABLE            (VVCAM_ISP_CID_HDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_ENABLE (VVCAM_ISP_CID_HDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_ENABLE \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_HDR_RESET             (VVCAM_ISP_CID_HDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_HDR_COLOR_WEIGHT      (VVCAM_ISP_CID_HDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_HDR_RATIO             (VVCAM_ISP_CID_HDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_HDR_TRANS_RANGE       (VVCAM_ISP_CID_HDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_HDR_EXTEND_BIT        (VVCAM_ISP_CID_HDR_BASE + 0x0007)
+
+#define VVCAM_ISP_CID_HDR_DEGHOST_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_HDR_DEGHOST_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0010)
+
+#define VVCAM_ISP_CID_HDR_STAT_COLOR_WEIGHT (VVCAM_ISP_CID_HDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_HDR_STAT_RATIO        (VVCAM_ISP_CID_HDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_HDR_STAT_TRANS_RANGE  (VVCAM_ISP_CID_HDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_HDR_STAT_EXTEND_BIT   (VVCAM_ISP_CID_HDR_BASE + 0x0014)
+
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_MOTION_WEIGHT \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_MOTION_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_LOWER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_HDR_STAT_DEGHOST_LSVS_DARK_UPPER_THR \
+                                            (VVCAM_ISP_CID_HDR_BASE + 0x001D)
+
+#ifdef __KERNEL__
+int vvcam_isp_hdr_ctrl_count(void);
+int vvcam_isp_hdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.c
new file mode 100644
index 000000000000..220177526854
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.c
@@ -0,0 +1,232 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_lsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_lsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_lsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_DATA_TBL:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_lsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_lsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_lsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_lsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_x_size_tbl",
+        .step = 1,
+        .min  = 10,
+        .max  = 0xFFFF,
+        .def  = 10,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_y_size_tbl",
+        .step = 1,
+        .min  = 8,
+        .max  = 0xFFFF,
+        .def  = 8,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 17, 17, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_x_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_y_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {8},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 17, 17, 0},
+    },
+};
+
+int vvcam_isp_lsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_lsc_ctrls);
+}
+
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_lsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_lsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp lsc ctrl %s failed %d.\n",
+                vvcam_isp_lsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.h
new file mode 100644
index 000000000000..d460a0f7e539
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v1/vvcam_isp_lsc.h
@@ -0,0 +1,75 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_LSC_H__
+#define __VVCAM_ISP_LSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_LSC_ENABLE            (VVCAM_ISP_CID_LSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_LSC_RESET             (VVCAM_ISP_CID_LSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_LSC_MANU_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_LSC_STAT_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x0007)
+
+#ifdef __KERNEL__
+int vvcam_isp_lsc_ctrl_count(void);
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.c
new file mode 100644
index 000000000000..05780df9719a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.c
@@ -0,0 +1,311 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_lsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_lsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MODE:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+        case VVCAM_ISP_CID_LSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_LSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_LSC_AUTO_DAMPING:
+        case VVCAM_ISP_CID_LSC_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_LSC_AUTO_INTER_MODE:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_lsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MODE:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+        case VVCAM_ISP_CID_LSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_LSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_LSC_AUTO_DAMPING:
+        case VVCAM_ISP_CID_LSC_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_LSC_AUTO_INTER_MODE:
+        case VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_DATA_TBL:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_lsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_lsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_lsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_lsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_x_size_tbl",
+        .step = 1,
+        .min  = 10,
+        .max  = 0xFFFF,
+        .def  = 10,
+        .dims = {8, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_y_size_tbl",
+        .step = 1,
+        .min  = 8,
+        .max  = 0xFFFF,
+        .def  = 8,
+        .dims = {8, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 17, 17, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 1.0 */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_DAMPING,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_damping",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 20x array 0 ~ 1*/
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_INTER_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_inter_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_x_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {8, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_y_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {8, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 17, 17, 0},
+    },
+};
+
+int vvcam_isp_lsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_lsc_ctrls);
+}
+
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_lsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_lsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp lsc ctrl %s failed %d.\n",
+                vvcam_isp_lsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.h
new file mode 100644
index 000000000000..c1486ed4632d
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v2/vvcam_isp_lsc.h
@@ -0,0 +1,81 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_LSC_H__
+#define __VVCAM_ISP_LSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_LSC_ENABLE            (VVCAM_ISP_CID_LSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_LSC_RESET             (VVCAM_ISP_CID_LSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_LSC_MODE              (VVCAM_ISP_CID_LSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_LSC_MANU_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_LSC_AUTO_LEVEL        (VVCAM_ISP_CID_LSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_LSC_AUTO_GAIN         (VVCAM_ISP_CID_LSC_BASE + 0x0007)
+#define VVCAM_ISP_CID_LSC_AUTO_DAMPING      (VVCAM_ISP_CID_LSC_BASE + 0x0008)
+#define VVCAM_ISP_CID_LSC_AUTO_STRENGTH     (VVCAM_ISP_CID_LSC_BASE + 0x0009)
+#define VVCAM_ISP_CID_LSC_AUTO_INTER_MODE   (VVCAM_ISP_CID_LSC_BASE + 0x000A)
+#define VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x000B)
+#define VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x000C)
+#define VVCAM_ISP_CID_LSC_STAT_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x000D)
+
+#ifdef __KERNEL__
+int vvcam_isp_lsc_ctrl_count(void);
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.c
new file mode 100644
index 000000000000..8b7d67745a49
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.c
@@ -0,0 +1,312 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_lsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_lsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MODE:
+        case VVCAM_ISP_CID_LSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_LSC_AUTO_DAMPING:
+        case VVCAM_ISP_CID_LSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_LSC_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_LSC_AUTO_INTER_MODE:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_lsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MODE:
+        case VVCAM_ISP_CID_LSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_LSC_AUTO_DAMPING:
+        case VVCAM_ISP_CID_LSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_LSC_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_LSC_AUTO_INTER_MODE:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_DATA_TBL:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_lsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_lsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_lsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_lsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_DAMPING,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_damping",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 20x */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* float 20x 0~1 */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_INTER_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_inter_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_x_size_tbl",
+        .step = 1,
+        .min  = 10,
+        .max  = 0xFFFF,
+        .def  = 10,
+        .dims = {32},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_y_size_tbl",
+        .step = 1,
+        .min  = 8,
+        .max  = 0xFFFF,
+        .def  = 8,
+        .dims = {16},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 33, 33, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_x_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {32},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_y_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {16},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 33, 33, 0},
+    },
+};
+
+int vvcam_isp_lsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_lsc_ctrls);
+}
+
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_lsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_lsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp lsc ctrl %s failed %d.\n",
+                vvcam_isp_lsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.h
new file mode 100644
index 000000000000..753fc1f9eeb2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v3/vvcam_isp_lsc.h
@@ -0,0 +1,81 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_LSC_H__
+#define __VVCAM_ISP_LSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_LSC_ENABLE            (VVCAM_ISP_CID_LSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_LSC_RESET             (VVCAM_ISP_CID_LSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_LSC_MODE              (VVCAM_ISP_CID_LSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_LSC_AUTO_LEVEL        (VVCAM_ISP_CID_LSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_LSC_AUTO_DAMPING      (VVCAM_ISP_CID_LSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_LSC_AUTO_GAIN         (VVCAM_ISP_CID_LSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_LSC_AUTO_STRENGTH     (VVCAM_ISP_CID_LSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_LSC_AUTO_INTER_MODE   (VVCAM_ISP_CID_LSC_BASE + 0x0007)
+#define VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0008)
+#define VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0009)
+#define VVCAM_ISP_CID_LSC_MANU_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x000A)
+#define VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x000B)
+#define VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x000C)
+#define VVCAM_ISP_CID_LSC_STAT_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x000D)
+
+#ifdef __KERNEL__
+int vvcam_isp_lsc_ctrl_count(void);
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.c
new file mode 100644
index 000000000000..9ad33ce8db4c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.c
@@ -0,0 +1,312 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_lsc.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_lsc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MODE:
+        case VVCAM_ISP_CID_LSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_LSC_AUTO_DAMPING:
+        case VVCAM_ISP_CID_LSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_LSC_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_LSC_AUTO_INTER_MODE:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_lsc_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LSC_ENABLE:
+        case VVCAM_ISP_CID_LSC_RESET:
+        case VVCAM_ISP_CID_LSC_MODE:
+        case VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_MANU_DATA_TBL:
+        case VVCAM_ISP_CID_LSC_AUTO_LEVEL:
+        case VVCAM_ISP_CID_LSC_AUTO_DAMPING:
+        case VVCAM_ISP_CID_LSC_AUTO_GAIN:
+        case VVCAM_ISP_CID_LSC_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_LSC_AUTO_INTER_MODE:
+        case VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL:
+        case VVCAM_ISP_CID_LSC_STAT_DATA_TBL:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_lsc_ctrl_ops = {
+    .s_ctrl = vvcam_isp_lsc_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_lsc_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_lsc_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* manual/auto */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_x_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {32, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_y_size_tbl",
+        .step = 1,
+        .min  = 1,
+        .max  = 0xFFFF,
+        .def  = 1,
+        .dims = {32, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_MANU_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_manu_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 33, 33, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_DAMPING,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_damping",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 20x */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x 0~1 */
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_AUTO_INTER_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_auto_inter_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_x_size_tbl",
+        .step = 1,
+        .min  = 10,
+        .max  = 0xFFFF,
+        .def  = 10,
+        .dims = {32, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_y_size_tbl",
+        .step = 1,
+        .min  = 8,
+        .max  = 0xFFFF,
+        .def  = 8,
+        .dims = {32, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_lsc_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LSC_STAT_DATA_TBL,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lsc_stat_data_tbl",
+        .step = 1,
+        .min  = 1024,
+        .max  = 16383,
+        .def  = 1024,
+        .dims = {4, 33, 33, 0},
+    },
+};
+
+int vvcam_isp_lsc_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_lsc_ctrls);
+}
+
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_lsc_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_lsc_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp lsc ctrl %s failed %d.\n",
+                vvcam_isp_lsc_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.h
new file mode 100644
index 000000000000..2b22ec5a6e7f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lsc/lsc_v4/vvcam_isp_lsc.h
@@ -0,0 +1,81 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_LSC_H__
+#define __VVCAM_ISP_LSC_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_LSC_ENABLE            (VVCAM_ISP_CID_LSC_BASE + 0x0000)
+#define VVCAM_ISP_CID_LSC_RESET             (VVCAM_ISP_CID_LSC_BASE + 0x0001)
+#define VVCAM_ISP_CID_LSC_MODE              (VVCAM_ISP_CID_LSC_BASE + 0x0002)
+#define VVCAM_ISP_CID_LSC_MANU_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0003)
+#define VVCAM_ISP_CID_LSC_MANU_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x0004)
+#define VVCAM_ISP_CID_LSC_MANU_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x0005)
+#define VVCAM_ISP_CID_LSC_AUTO_LEVEL        (VVCAM_ISP_CID_LSC_BASE + 0x0006)
+#define VVCAM_ISP_CID_LSC_AUTO_DAMPING      (VVCAM_ISP_CID_LSC_BASE + 0x0007)
+#define VVCAM_ISP_CID_LSC_AUTO_GAIN         (VVCAM_ISP_CID_LSC_BASE + 0x0008)
+#define VVCAM_ISP_CID_LSC_AUTO_STRENGTH     (VVCAM_ISP_CID_LSC_BASE + 0x0009)
+#define VVCAM_ISP_CID_LSC_AUTO_INTER_MODE   (VVCAM_ISP_CID_LSC_BASE + 0x000A)
+#define VVCAM_ISP_CID_LSC_STAT_X_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x000B)
+#define VVCAM_ISP_CID_LSC_STAT_Y_SIZE_TBL   (VVCAM_ISP_CID_LSC_BASE + 0x000C)
+#define VVCAM_ISP_CID_LSC_STAT_DATA_TBL     (VVCAM_ISP_CID_LSC_BASE + 0x000D)
+
+#ifdef __KERNEL__
+int vvcam_isp_lsc_ctrl_count(void);
+int vvcam_isp_lsc_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.c
new file mode 100644
index 000000000000..9d30ffd0f28e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.c
@@ -0,0 +1,247 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_lut3d.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_lut3d_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LUT3D_ENABLE:
+        case VVCAM_ISP_CID_LUT3D_RESET:
+        // case VVCAM_ISP_CID_LUT3D_MODE:
+        case VVCAM_ISP_CID_LUT3D_MANU_R_TBL:
+        case VVCAM_ISP_CID_LUT3D_MANU_G_TBL:
+        case VVCAM_ISP_CID_LUT3D_MANU_B_TBL:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_lut3d_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_LUT3D_ENABLE:
+        case VVCAM_ISP_CID_LUT3D_RESET:
+        // case VVCAM_ISP_CID_LUT3D_MODE:
+        case VVCAM_ISP_CID_LUT3D_MANU_R_TBL:
+        case VVCAM_ISP_CID_LUT3D_MANU_G_TBL:
+        case VVCAM_ISP_CID_LUT3D_MANU_B_TBL:
+        case VVCAM_ISP_CID_LUT3D_STAT_R_TBL:
+        case VVCAM_ISP_CID_LUT3D_STAT_G_TBL:
+        case VVCAM_ISP_CID_LUT3D_STAT_B_TBL:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_lut3d_ctrl_ops = {
+    .s_ctrl = vvcam_isp_lut3d_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_lut3d_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_lut3d_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    /*
+    {
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    */
+    {
+        /* uint32 17x17x17 array */
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_MANU_R_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_manu_r_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 17, 17, 0},
+    },
+    {
+        /* uint32 17x17x17 array */
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_MANU_G_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_manu_g_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 17, 17, 0},
+    },
+    {
+        /* uint32 17x17x17 array */
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_MANU_B_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_manu_b_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 17, 17, 0},
+    },
+    {
+        /* uint32 17x17x17 array */
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_STAT_R_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_stat_r_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 17, 17, 0},
+    },
+    {
+        /* uint32 17x17x17 array */
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_STAT_G_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_stat_g_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 17, 17, 0},
+    },
+    {
+        /* uint32 17x17x17 array */
+        .ops  = &vvcam_isp_lut3d_ctrl_ops,
+        .id   = VVCAM_ISP_CID_LUT3D_STAT_B_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_lut3d_stat_b_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {17, 17, 17, 0},
+    },
+};
+
+int vvcam_isp_lut3d_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_lut3d_ctrls);
+}
+
+int vvcam_isp_lut3d_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_lut3d_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_lut3d_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ynr ctrl %s failed %d.\n",
+                vvcam_isp_lut3d_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.h
new file mode 100644
index 000000000000..21347ed2de11
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/lut3d/lut3d_v1/vvcam_isp_lut3d.h
@@ -0,0 +1,76 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_LUT3D_H__
+#define __VVCAM_ISP_LUT3D_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_LUT3D_ENABLE          (VVCAM_ISP_CID_LUT3D_BASE + 0x0000)
+#define VVCAM_ISP_CID_LUT3D_RESET           (VVCAM_ISP_CID_LUT3D_BASE + 0x0001)
+// #define VVCAM_ISP_CID_LUT3D_MODE            (VVCAM_ISP_CID_LUT3D_BASE + 0x0002)
+#define VVCAM_ISP_CID_LUT3D_MANU_R_TBL      (VVCAM_ISP_CID_LUT3D_BASE + 0x0003)
+#define VVCAM_ISP_CID_LUT3D_MANU_G_TBL      (VVCAM_ISP_CID_LUT3D_BASE + 0x0004)
+#define VVCAM_ISP_CID_LUT3D_MANU_B_TBL      (VVCAM_ISP_CID_LUT3D_BASE + 0x0005)
+#define VVCAM_ISP_CID_LUT3D_STAT_R_TBL      (VVCAM_ISP_CID_LUT3D_BASE + 0x0006)
+#define VVCAM_ISP_CID_LUT3D_STAT_G_TBL      (VVCAM_ISP_CID_LUT3D_BASE + 0x0007)
+#define VVCAM_ISP_CID_LUT3D_STAT_B_TBL      (VVCAM_ISP_CID_LUT3D_BASE + 0x0008)
+
+#ifdef __KERNEL__
+int vvcam_isp_lut3d_ctrl_count(void);
+int vvcam_isp_lut3d_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.c
new file mode 100644
index 000000000000..e7b652f1d058
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.c
@@ -0,0 +1,438 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_pdaf.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_pdaf_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_PDAF_ENABLE:
+        case VVCAM_ISP_CID_PDAF_COR_ENABLE:
+        case VVCAM_ISP_CID_PDAF_RESET:
+        case VVCAM_ISP_CID_PDAF_COR_L_CHANGE_DOWN:
+        case VVCAM_ISP_CID_PDAF_COR_L_CHANGE_UP:
+        case VVCAM_ISP_CID_PDAF_COR_L_CHANNEL:
+        case VVCAM_ISP_CID_PDAF_COR_L_LIMIT_BASE:
+        case VVCAM_ISP_CID_PDAF_COR_R_CHANGE_DOWN:
+        case VVCAM_ISP_CID_PDAF_COR_R_CHANGE_UP:
+        case VVCAM_ISP_CID_PDAF_COR_R_CHANNEL:
+        case VVCAM_ISP_CID_PDAF_COR_R_LIMIT_BASE:
+        case VVCAM_ISP_CID_PDAF_COR_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_DUMMY_LINE_HW:
+        case VVCAM_ISP_CID_PDAF_ROI:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_pdaf_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_PDAF_ENABLE:
+        case VVCAM_ISP_CID_PDAF_COR_ENABLE:
+        case VVCAM_ISP_CID_PDAF_COR_L_CHANGE_DOWN:
+        case VVCAM_ISP_CID_PDAF_COR_L_CHANGE_UP:
+        case VVCAM_ISP_CID_PDAF_COR_L_CHANNEL:
+        case VVCAM_ISP_CID_PDAF_COR_L_LIMIT_BASE:
+        case VVCAM_ISP_CID_PDAF_COR_R_CHANGE_DOWN:
+        case VVCAM_ISP_CID_PDAF_COR_R_CHANGE_UP:
+        case VVCAM_ISP_CID_PDAF_COR_R_CHANNEL:
+        case VVCAM_ISP_CID_PDAF_COR_R_LIMIT_BASE:
+        case VVCAM_ISP_CID_PDAF_COR_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_DUMMY_LINE_HW:
+        case VVCAM_ISP_CID_PDAF_ROI:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANGE_DOWN:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANGE_UP:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANNEL:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_L_LIMIT_BASE:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANGE_DOWN:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANGE_UP:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANNEL:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_R_LIMIT_BASE:
+        case VVCAM_ISP_CID_PDAF_STAT_COR_THRESHOLD:
+        case VVCAM_ISP_CID_PDAF_STAT_DUMMY_LINE_HW:
+        case VVCAM_ISP_CID_PDAF_STAT_ROI:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_pdaf_ctrl_ops = {
+    .s_ctrl = vvcam_isp_pdaf_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_pdaf_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_pdaf_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_L_CHANGE_DOWN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_l_change_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_L_CHANGE_UP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_l_change_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_L_CHANNEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_l_channel",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_L_LIMIT_BASE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_l_limit_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_R_CHANGE_DOWN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_r_change_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_R_CHANGE_UP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_r_change_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_R_CHANNEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_r_channel",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_R_LIMIT_BASE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_r_limit_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_COR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_cor_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_DUMMY_LINE_HW,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_dummy_line_hw",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {4},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANGE_DOWN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_l_change_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANGE_UP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_l_change_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANNEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_l_channel",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_L_LIMIT_BASE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_l_limit_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANGE_DOWN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_r_change_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANGE_UP,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_r_change_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANNEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_r_channel",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_R_LIMIT_BASE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_r_limit_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_COR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_cor_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_DUMMY_LINE_HW,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_dummy_line_hw",
+        .step = 1,
+        .min  = 0,
+        .max  = 1000,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_pdaf_ctrl_ops,
+        .id   = VVCAM_ISP_CID_PDAF_STAT_ROI,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_pdaf_stat_roi",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {4},
+    },
+};
+
+int vvcam_isp_pdaf_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_pdaf_ctrls);
+}
+
+int vvcam_isp_pdaf_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_pdaf_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_pdaf_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ynr ctrl %s failed %d.\n",
+                vvcam_isp_pdaf_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.h
new file mode 100644
index 000000000000..43b9bdba0bc1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/pdaf/pdaf_v1/vvcam_isp_pdaf.h
@@ -0,0 +1,105 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_PDAF_H__
+#define __VVCAM_ISP_PDAF_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_PDAF_ENABLE           (VVCAM_ISP_CID_PDAF_BASE + 0x0000)
+#define VVCAM_ISP_CID_PDAF_COR_ENABLE       (VVCAM_ISP_CID_PDAF_BASE + 0x0001)
+#define VVCAM_ISP_CID_PDAF_RESET            (VVCAM_ISP_CID_PDAF_BASE + 0x0002)
+#define VVCAM_ISP_CID_PDAF_COR_L_CHANGE_DOWN \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0003)
+#define VVCAM_ISP_CID_PDAF_COR_L_CHANGE_UP  (VVCAM_ISP_CID_PDAF_BASE + 0x0004)
+#define VVCAM_ISP_CID_PDAF_COR_L_CHANNEL    (VVCAM_ISP_CID_PDAF_BASE + 0x0005)
+#define VVCAM_ISP_CID_PDAF_COR_L_LIMIT_BASE (VVCAM_ISP_CID_PDAF_BASE + 0x0006)
+#define VVCAM_ISP_CID_PDAF_COR_R_CHANGE_DOWN \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0007)
+#define VVCAM_ISP_CID_PDAF_COR_R_CHANGE_UP  (VVCAM_ISP_CID_PDAF_BASE + 0x0008)
+#define VVCAM_ISP_CID_PDAF_COR_R_CHANNEL    (VVCAM_ISP_CID_PDAF_BASE + 0x0009)
+#define VVCAM_ISP_CID_PDAF_COR_R_LIMIT_BASE (VVCAM_ISP_CID_PDAF_BASE + 0x000A)
+#define VVCAM_ISP_CID_PDAF_COR_THRESHOLD    (VVCAM_ISP_CID_PDAF_BASE + 0x000B)
+#define VVCAM_ISP_CID_PDAF_DUMMY_LINE_HW    (VVCAM_ISP_CID_PDAF_BASE + 0x000C)
+#define VVCAM_ISP_CID_PDAF_ROI              (VVCAM_ISP_CID_PDAF_BASE + 0x000D)
+
+#define VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANGE_DOWN \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x000E)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANGE_UP \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x000F)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_L_CHANNEL \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0010)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_L_LIMIT_BASE \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0011)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANGE_DOWN \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0012)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANGE_UP \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0013)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_R_CHANNEL \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0014)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_R_LIMIT_BASE \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0015)
+#define VVCAM_ISP_CID_PDAF_STAT_COR_THRESHOLD \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0016)
+#define VVCAM_ISP_CID_PDAF_STAT_DUMMY_LINE_HW \
+                                            (VVCAM_ISP_CID_PDAF_BASE + 0x0017)
+#define VVCAM_ISP_CID_PDAF_STAT_ROI         (VVCAM_ISP_CID_PDAF_BASE + 0x0018)
+
+#ifdef __KERNEL__
+int vvcam_isp_pdaf_ctrl_count(void);
+int vvcam_isp_pdaf_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.c
new file mode 100644
index 000000000000..f96d008e3851
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.c
@@ -0,0 +1,294 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_rgbir.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_rgbir_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_rgbir_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+        case VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_rgbir_ctrl_ops = {
+    .s_ctrl = vvcam_isp_rgbir_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_rgbir_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_rgbir_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_middle_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 12bit: 4095 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_OUT_PATTERN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_out_pattern",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_mid_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 12bit: 4095 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_rgbir_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_rgbir_ctrls);
+}
+
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_rgbir_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_rgbir_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp rgbir ctrl %s failed %d.\n",
+                vvcam_isp_rgbir_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.h
new file mode 100644
index 000000000000..27233d7f54a1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1/vvcam_isp_rgbir.h
@@ -0,0 +1,86 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_RGBIR_H__
+#define __VVCAM_ISP_RGBIR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_RGBIR_ENABLE          (VVCAM_ISP_CID_RGBIR_BASE + 0x0000)
+#define VVCAM_ISP_CID_RGBIR_RESET           (VVCAM_ISP_CID_RGBIR_BASE + 0x0001)
+#define VVCAM_ISP_CID_RGBIR_CC_MATRIX       (VVCAM_ISP_CID_RGBIR_BASE + 0x0002)
+#define VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0003)
+#define VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD  (VVCAM_ISP_CID_RGBIR_BASE + 0x0004)
+#define VVCAM_ISP_CID_RGBIR_IR_THRESHOLD    (VVCAM_ISP_CID_RGBIR_BASE + 0x0005)
+#define VVCAM_ISP_CID_RGBIR_L_THRESHOLD     (VVCAM_ISP_CID_RGBIR_BASE + 0x0006)
+#define VVCAM_ISP_CID_RGBIR_OUT_PATTERN     (VVCAM_ISP_CID_RGBIR_BASE + 0x0007)
+#define VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX  (VVCAM_ISP_CID_RGBIR_BASE + 0x0008)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0009)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000A)
+#define VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000B)
+#define VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000C)
+
+
+#ifdef __KERNEL__
+int vvcam_isp_rgbir_ctrl_count(void);
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.c
new file mode 100644
index 000000000000..ade62ffe8953
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.c
@@ -0,0 +1,306 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_rgbir.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_rgbir_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_rgbir_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        // case VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+        case VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_rgbir_ctrl_ops = {
+    .s_ctrl = vvcam_isp_rgbir_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_rgbir_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_rgbir_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_raw_out_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_middle_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 12bit: 4095 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_OUT_PATTERN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_out_pattern",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_mid_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 12bit: 4095 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 4095,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_rgbir_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_rgbir_ctrls);
+}
+
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_rgbir_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_rgbir_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp rgbir ctrl %s failed %d.\n",
+                vvcam_isp_rgbir_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.h
new file mode 100644
index 000000000000..8a2aad20a6a5
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v1_1/vvcam_isp_rgbir.h
@@ -0,0 +1,87 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_RGBIR_H__
+#define __VVCAM_ISP_RGBIR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_RGBIR_ENABLE          (VVCAM_ISP_CID_RGBIR_BASE + 0x0000)
+#define VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0001)
+#define VVCAM_ISP_CID_RGBIR_RESET           (VVCAM_ISP_CID_RGBIR_BASE + 0x0002)
+#define VVCAM_ISP_CID_RGBIR_CC_MATRIX       (VVCAM_ISP_CID_RGBIR_BASE + 0x0003)
+#define VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0004)
+#define VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD  (VVCAM_ISP_CID_RGBIR_BASE + 0x0005)
+#define VVCAM_ISP_CID_RGBIR_IR_THRESHOLD    (VVCAM_ISP_CID_RGBIR_BASE + 0x0006)
+#define VVCAM_ISP_CID_RGBIR_L_THRESHOLD     (VVCAM_ISP_CID_RGBIR_BASE + 0x0007)
+#define VVCAM_ISP_CID_RGBIR_OUT_PATTERN     (VVCAM_ISP_CID_RGBIR_BASE + 0x0008)
+#define VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX  (VVCAM_ISP_CID_RGBIR_BASE + 0x0009)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000A)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000B)
+#define VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000C)
+#define VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000D)
+
+#ifdef __KERNEL__
+int vvcam_isp_rgbir_ctrl_count(void);
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.c
new file mode 100644
index 000000000000..2ff25d3bbfa7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.c
@@ -0,0 +1,318 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_rgbir.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_rgbir_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RCCC_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_rgbir_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RCCC_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+        case VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_rgbir_ctrl_ops = {
+    .s_ctrl = vvcam_isp_rgbir_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_rgbir_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_rgbir_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_RCCC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_rccc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_raw_out_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_middle_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 24bit: 16777215 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_OUT_PATTERN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_out_pattern",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_mid_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 24bit: 16777215 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_rgbir_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_rgbir_ctrls);
+}
+
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_rgbir_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_rgbir_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp rgbir ctrl %s failed %d.\n",
+                vvcam_isp_rgbir_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.h
new file mode 100644
index 000000000000..f2c49915358b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2/vvcam_isp_rgbir.h
@@ -0,0 +1,88 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_RGBIR_H__
+#define __VVCAM_ISP_RGBIR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_RGBIR_ENABLE          (VVCAM_ISP_CID_RGBIR_BASE + 0x0000)
+#define VVCAM_ISP_CID_RGBIR_RCCC_ENABLE     (VVCAM_ISP_CID_RGBIR_BASE + 0x0001)
+#define VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0002)
+#define VVCAM_ISP_CID_RGBIR_RESET           (VVCAM_ISP_CID_RGBIR_BASE + 0x0003)
+#define VVCAM_ISP_CID_RGBIR_CC_MATRIX       (VVCAM_ISP_CID_RGBIR_BASE + 0x0004)
+#define VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0005)
+#define VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD  (VVCAM_ISP_CID_RGBIR_BASE + 0x0006)
+#define VVCAM_ISP_CID_RGBIR_IR_THRESHOLD    (VVCAM_ISP_CID_RGBIR_BASE + 0x0007)
+#define VVCAM_ISP_CID_RGBIR_L_THRESHOLD     (VVCAM_ISP_CID_RGBIR_BASE + 0x0008)
+#define VVCAM_ISP_CID_RGBIR_OUT_PATTERN     (VVCAM_ISP_CID_RGBIR_BASE + 0x0009)
+#define VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX  (VVCAM_ISP_CID_RGBIR_BASE + 0x000A)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000B)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000C)
+#define VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000D)
+#define VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000E)
+
+#ifdef __KERNEL__
+int vvcam_isp_rgbir_ctrl_count(void);
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.c
new file mode 100644
index 000000000000..00377b6988d1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.c
@@ -0,0 +1,330 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_rgbir.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_rgbir_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RCCC_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+        case VVCAM_ISP_CID_RGBIR_IR_PATH_SELECT:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_rgbir_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_RGBIR_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RCCC_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE:
+        case VVCAM_ISP_CID_RGBIR_RESET:
+        case VVCAM_ISP_CID_RGBIR_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_L_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_OUT_PATTERN:
+        case VVCAM_ISP_CID_RGBIR_IR_PATH_SELECT:
+        case VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD:
+        case VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_rgbir_ctrl_ops = {
+    .s_ctrl = vvcam_isp_rgbir_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_rgbir_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_rgbir_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_RCCC_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_rccc_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_raw_out_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_middle_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 24bit: 16777215 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_OUT_PATTERN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_out_pattern",
+        .step = 1,
+        .min  = 0,
+        .max  = 3,
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_IR_PATH_SELECT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_ir_path_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float array 12x (-4,4)*/
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_cc_matrix",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {12, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_mid_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_dpcc_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 65535,
+        .dims = {4, 0, 0, 0},
+    },
+    {
+        /* 24bit: 16777215 */
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_ir_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_rgbir_ctrl_ops,
+        .id   = VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_rgbir_stat_l_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 16777215,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_rgbir_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_rgbir_ctrls);
+}
+
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_rgbir_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_rgbir_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp rgbir ctrl %s failed %d.\n",
+                vvcam_isp_rgbir_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.h
new file mode 100644
index 000000000000..543f9d87356b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/rgbir/rgbir_v2_1/vvcam_isp_rgbir.h
@@ -0,0 +1,89 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_RGBIR_H__
+#define __VVCAM_ISP_RGBIR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_RGBIR_ENABLE          (VVCAM_ISP_CID_RGBIR_BASE + 0x0000)
+#define VVCAM_ISP_CID_RGBIR_RCCC_ENABLE     (VVCAM_ISP_CID_RGBIR_BASE + 0x0001)
+#define VVCAM_ISP_CID_RGBIR_IR_RAW_OUT_ENABLE \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0002)
+#define VVCAM_ISP_CID_RGBIR_RESET           (VVCAM_ISP_CID_RGBIR_BASE + 0x0003)
+#define VVCAM_ISP_CID_RGBIR_CC_MATRIX       (VVCAM_ISP_CID_RGBIR_BASE + 0x0004)
+#define VVCAM_ISP_CID_RGBIR_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x0005)
+#define VVCAM_ISP_CID_RGBIR_DPCC_THRESHOLD  (VVCAM_ISP_CID_RGBIR_BASE + 0x0006)
+#define VVCAM_ISP_CID_RGBIR_IR_THRESHOLD    (VVCAM_ISP_CID_RGBIR_BASE + 0x0007)
+#define VVCAM_ISP_CID_RGBIR_L_THRESHOLD     (VVCAM_ISP_CID_RGBIR_BASE + 0x0008)
+#define VVCAM_ISP_CID_RGBIR_OUT_PATTERN     (VVCAM_ISP_CID_RGBIR_BASE + 0x0009)
+#define VVCAM_ISP_CID_RGBIR_IR_PATH_SELECT  (VVCAM_ISP_CID_RGBIR_BASE + 0x000A)
+#define VVCAM_ISP_CID_RGBIR_STAT_CC_MATRIX  (VVCAM_ISP_CID_RGBIR_BASE + 0x000B)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_MIDDLE_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000C)
+#define VVCAM_ISP_CID_RGBIR_STAT_DPCC_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000D)
+#define VVCAM_ISP_CID_RGBIR_STAT_IR_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000E)
+#define VVCAM_ISP_CID_RGBIR_STAT_L_THRESHOLD \
+                                            (VVCAM_ISP_CID_RGBIR_BASE + 0x000F)
+
+#ifdef __KERNEL__
+int vvcam_isp_rgbir_ctrl_count(void);
+int vvcam_isp_rgbir_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/vvcam_isp_ctrl.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/vvcam_isp_ctrl.c
new file mode 100644
index 000000000000..6404a5528fed
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/vvcam_isp_ctrl.c
@@ -0,0 +1,461 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+
+#if defined(ISP_AE_V3) || defined(ISP_AE_V4) || defined(ISP_AE_V4_1)
+#include "vvcam_isp_ae.h"
+#endif
+
+#if defined(ISP_AWB_V3) || defined(ISP_AWB_V4) || defined(ISP_AWB_V4_1)
+#include "vvcam_isp_awb.h"
+#endif
+
+#if defined(ISP_GC_V1) || defined(ISP_GC_V2)
+#include "vvcam_isp_gc.h"
+#endif
+
+#if defined(ISP_2DNR_V3) || defined(ISP_2DNR_V5_2) || defined(ISP_2DNR_V6)
+#include "vvcam_isp_2dnr.h"
+#endif
+
+#if defined(ISP_EE_V1) || defined(ISP_EE_V2) || defined(ISP_EE_V3)
+#include "vvcam_isp_ee.h"
+#endif
+
+#if defined(ISP_CPROC)
+#include "vvcam_isp_cproc.h"
+#endif
+
+#if defined(ISP_WDR_V3) || defined(ISP_WDR_V4) || defined(ISP_WDR_V5) \
+    || defined(ISP_WDR_V5_1) || defined(ISP_WDR_V5_2) || defined(ISP_WDR_V5_2_1) \
+    || defined(ISP_WDR_V5_3)
+#include "vvcam_isp_wdr.h"
+#endif
+
+#if defined(ISP_BLS)
+#include "vvcam_isp_bls.h"
+#endif
+
+#if defined(ISP_CCM)
+#include "vvcam_isp_ccm.h"
+#endif
+
+#if defined(ISP_DPCC)
+#include "vvcam_isp_dpcc.h"
+#endif
+
+#if defined(ISP_GE)
+#include "vvcam_isp_ge.h"
+#endif
+
+#if defined(ISP_LSC_V1) || defined(ISP_LSC_V2) || defined(ISP_LSC_V3) \
+    || defined(ISP_LSC_V4)
+#include "vvcam_isp_lsc.h"
+#endif
+
+#if defined(ISP_WB_V1) || defined(ISP_WB_V1_1)
+#include "vvcam_isp_wb.h"
+#endif
+
+#if defined(ISP_3DNR_V1_2) || defined(ISP_3DNR_V2_1) || defined(ISP_3DNR_V2_4) \
+    || defined(ISP_3DNR_V3) || defined(ISP_3DNR_V3_1) || defined(ISP_3DNR_V4)
+#include "vvcam_isp_3dnr.h"
+#endif
+
+#if defined(ISP_AF_V3) || defined(ISP_AF_V4) || defined(ISP_AF_V4_1_1) \
+    || defined(ISP_AF_V4_3)
+#include "vvcam_isp_af.h"
+#endif
+
+#if defined(ISP_PDAF_V1)
+#include "vvcam_isp_pdaf.h"
+#endif
+
+#if defined(ISP_AFM_V1) || defined(ISP_AFM_V1_1) || defined(ISP_AFM_V3)
+#include "vvcam_isp_afm.h"
+#endif
+
+#if defined(ISP_DMSC_V1) || defined(ISP_DMSC_V2) || defined(ISP_DMSC_V3)
+#include "vvcam_isp_dmsc.h"
+#endif
+
+#if defined(ISP_EXP_V2)
+#include "vvcam_isp_exp.h"
+#endif
+
+#if defined(ISP_EXP_V3)
+#include "vvcam_isp_exp_v3.h"
+#endif
+
+#if defined(ISP_GTM_V1)
+#include "vvcam_isp_gtm.h"
+#endif
+
+#if defined(ISP_HDR_V1_3) || defined(ISP_HDR_V2) || defined(ISP_HDR_V2_1) \
+    || defined(ISP_HDR_V2_1_2DOL) || defined(ISP_HDR_V3) \
+    || defined(ISP_HDR_V3_2DOL) || defined(ISP_HDR_V3_1)
+#include "vvcam_isp_hdr.h"
+#endif
+
+#if defined(ISP_RGBIR_V1) || defined(ISP_RGBIR_V1_1) || defined(ISP_RGBIR_V2) \
+    || defined(ISP_RGBIR_V2_1)
+#include "vvcam_isp_rgbir.h"
+#endif
+
+#if defined(ISP_DG_V1)
+#include "vvcam_isp_dg.h"
+#endif
+
+#if defined(ISP_CMPD_20BIT) || defined(ISP_CMPD_24BIT)
+#include "vvcam_isp_cpd.h"
+#endif
+
+#if defined(ISP_DPF_V1)
+#include "vvcam_isp_dpf.h"
+#endif
+
+#if defined(ISP_CNR_V1) || defined(ISP_CNR_V2_1) || defined(ISP_CNR_V2_2)
+#include "vvcam_isp_cnr.h"
+#endif
+
+#if defined(ISP_GWDR_V1)
+#include "vvcam_isp_gwdr.h"
+#endif
+
+#if defined(ISP_YNR_V1)
+#include "vvcam_isp_ynr.h"
+#endif
+
+#if defined(ISP_LUT3D_V1)
+#include "vvcam_isp_lut3d.h"
+#endif
+
+int vvcam_isp_ctrl_init(struct vvcam_isp_dev *isp_dev)
+{
+    uint32_t ctrl_count = 0;
+
+#if defined(ISP_AE_V3) || defined(ISP_AE_V4) || defined(ISP_AE_V4_1)
+    ctrl_count += vvcam_isp_ae_ctrl_count();
+#endif
+
+#if defined(ISP_AWB_V3) || defined(ISP_AWB_V4) || defined(ISP_AWB_V4_1)
+    ctrl_count += vvcam_isp_awb_ctrl_count();
+#endif
+
+#if defined(ISP_GC_V1) || defined(ISP_GC_V2)
+    ctrl_count += vvcam_isp_gc_ctrl_count();
+#endif
+
+#if defined(ISP_2DNR_V3) || defined(ISP_2DNR_V5_2) || defined(ISP_2DNR_V6)
+    ctrl_count += vvcam_isp_2dnr_ctrl_count();
+#endif
+
+#if defined(ISP_EE_V1) || defined(ISP_EE_V2) || defined(ISP_EE_V3)
+    ctrl_count += vvcam_isp_ee_ctrl_count();
+#endif
+
+#if defined(ISP_CPROC)
+    ctrl_count += vvcam_isp_cproc_ctrl_count();
+#endif
+
+#if defined(ISP_WDR_V3) || defined(ISP_WDR_V4) || defined(ISP_WDR_V5) \
+    || defined(ISP_WDR_V5_1) || defined(ISP_WDR_V5_2) || defined(ISP_WDR_V5_2_1) \
+    || defined(ISP_WDR_V5_3)
+    ctrl_count += vvcam_isp_wdr_ctrl_count();
+#endif
+
+#if defined(ISP_BLS)
+    ctrl_count += vvcam_isp_bls_ctrl_count();
+#endif
+
+#if defined(ISP_CCM)
+    ctrl_count += vvcam_isp_ccm_ctrl_count();
+#endif
+
+#if defined(ISP_DPCC)
+    ctrl_count += vvcam_isp_dpcc_ctrl_count();
+#endif
+
+#if defined(ISP_GE)
+    ctrl_count += vvcam_isp_ge_ctrl_count();
+#endif
+
+#if defined(ISP_LSC_V1) || defined(ISP_LSC_V2) || defined(ISP_LSC_V3) \
+    || defined(ISP_LSC_V4)
+    ctrl_count += vvcam_isp_lsc_ctrl_count();
+#endif
+
+#if defined(ISP_WB_V1) || defined(ISP_WB_V1_1)
+    ctrl_count += vvcam_isp_wb_ctrl_count();
+#endif
+
+#if defined(ISP_3DNR_V1_2) || defined(ISP_3DNR_V2_1) || defined(ISP_3DNR_V2_4) \
+    || defined(ISP_3DNR_V3) || defined(ISP_3DNR_V3_1) || defined(ISP_3DNR_V4)
+    ctrl_count += vvcam_isp_3dnr_ctrl_count();
+#endif
+
+#if defined(ISP_AF_V3) || defined(ISP_AF_V4) || defined(ISP_AF_V4_1_1) \
+    || defined(ISP_AF_V4_3)
+    ctrl_count += vvcam_isp_af_ctrl_count();
+#endif
+
+#if defined(ISP_PDAF_V1)
+    ctrl_count += vvcam_isp_pdaf_ctrl_count();
+#endif
+
+#if defined(ISP_AFM_V1) || defined(ISP_AFM_V1_1) || defined(ISP_AFM_V3)
+    ctrl_count += vvcam_isp_afm_ctrl_count();
+#endif
+
+#if defined(ISP_DMSC_V1) || defined(ISP_DMSC_V2) || defined(ISP_DMSC_V3)
+    ctrl_count += vvcam_isp_dmsc_ctrl_count();
+#endif
+
+#if defined(ISP_EXP_V2)
+    ctrl_count += vvcam_isp_exp_ctrl_count();
+#endif
+
+#if defined(ISP_EXP_V3)
+    ctrl_count += vvcam_isp_exp_v3_ctrl_count();
+#endif
+
+#if defined(ISP_GTM_V1)
+    ctrl_count += vvcam_isp_gtm_ctrl_count();
+#endif
+
+#if defined(ISP_HDR_V1_3) || defined(ISP_HDR_V2) || defined(ISP_HDR_V2_1) \
+    || defined(ISP_HDR_V2_1_2DOL) || defined(ISP_HDR_V3) \
+    || defined(ISP_HDR_V3_2DOL) || defined(ISP_HDR_V3_1)
+    ctrl_count += vvcam_isp_hdr_ctrl_count();
+#endif
+
+#if defined(ISP_RGBIR_V1) || defined(ISP_RGBIR_V1_1) || defined(ISP_RGBIR_V2) \
+    || defined(ISP_RGBIR_V2_1)
+    ctrl_count += vvcam_isp_rgbir_ctrl_count();
+#endif
+
+#if defined(ISP_DG_V1)
+    ctrl_count += vvcam_isp_dg_ctrl_count();
+#endif
+
+#if defined(ISP_CMPD_20BIT) || defined(ISP_CMPD_24BIT)
+    ctrl_count += vvcam_isp_cpd_ctrl_count();
+#endif
+
+#if defined(ISP_DPF_V1)
+    ctrl_count += vvcam_isp_dpf_ctrl_count();
+#endif
+
+#if defined(ISP_CNR_V1) || defined(ISP_CNR_V2_1) || defined(ISP_CNR_V2_2)
+    ctrl_count += vvcam_isp_cnr_ctrl_count();
+#endif
+
+#if defined(ISP_GWDR_V1)
+    ctrl_count += vvcam_isp_gwdr_ctrl_count();
+#endif
+
+#if defined(ISP_YNR_V1)
+    ctrl_count += vvcam_isp_ynr_ctrl_count();
+#endif
+
+#if defined(ISP_LUT3D_V1)
+    ctrl_count += vvcam_isp_lut3d_ctrl_count();
+#endif
+
+    v4l2_ctrl_handler_init(&isp_dev->ctrl_handler,  ctrl_count);
+
+#if defined(ISP_AE_V3) || defined (ISP_AE_V4) || defined(ISP_AE_V4_1)
+    vvcam_isp_ae_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_AWB_V3) || defined(ISP_AWB_V4) || defined(ISP_AWB_V4_1)
+   vvcam_isp_awb_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_GC_V1) || defined(ISP_GC_V2)
+   vvcam_isp_gc_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_2DNR_V3) || defined(ISP_2DNR_V5_2) || defined(ISP_2DNR_V6)
+   vvcam_isp_2dnr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_EE_V1) || defined(ISP_EE_V2) || defined(ISP_EE_V3)
+   vvcam_isp_ee_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_CPROC)
+    vvcam_isp_cproc_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_WDR_V3) || defined(ISP_WDR_V4) || defined(ISP_WDR_V5) \
+    || defined(ISP_WDR_V5_1) || defined(ISP_WDR_V5_2) || defined(ISP_WDR_V5_2_1) \
+    || defined(ISP_WDR_V5_3)
+    vvcam_isp_wdr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_BLS)
+    vvcam_isp_bls_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_CCM)
+    vvcam_isp_ccm_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_DPCC)
+    vvcam_isp_dpcc_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_GE)
+    vvcam_isp_ge_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_LSC_V1) || defined(ISP_LSC_V2) || defined(ISP_LSC_V3) \
+    || defined(ISP_LSC_V4)
+    vvcam_isp_lsc_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_WB_V1) || defined(ISP_WB_V1_1)
+    vvcam_isp_wb_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_3DNR_V1_2) || defined(ISP_3DNR_V2_1) || defined(ISP_3DNR_V2_4) \
+    || defined(ISP_3DNR_V3) || defined(ISP_3DNR_V3_1) || defined(ISP_3DNR_V4)
+    vvcam_isp_3dnr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_AF_V3) || defined(ISP_AF_V4) || defined(ISP_AF_V4_1_1) \
+    || defined(ISP_AF_V4_3)
+    vvcam_isp_af_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_PDAF_V1)
+    vvcam_isp_pdaf_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_AFM_V1) || defined(ISP_AFM_V1_1) || defined(ISP_AFM_V3)
+    vvcam_isp_afm_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_DMSC_V1) || defined(ISP_DMSC_V2) || defined(ISP_DMSC_V3)
+    vvcam_isp_dmsc_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_EXP_V2)
+    vvcam_isp_exp_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_EXP_V3)
+    vvcam_isp_exp_v3_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_GTM_V1)
+     vvcam_isp_gtm_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_HDR_V1_3) || defined(ISP_HDR_V2) || defined(ISP_HDR_V2_1) \
+    || defined(ISP_HDR_V2_1_2DOL) || defined(ISP_HDR_V3) \
+    || defined(ISP_HDR_V3_2DOL) || defined(ISP_HDR_V3_1)
+    vvcam_isp_hdr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_RGBIR_V1) || defined(ISP_RGBIR_V1_1) || defined(ISP_RGBIR_V2) \
+    || defined(ISP_RGBIR_V2_1)
+    vvcam_isp_rgbir_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_DG_V1)
+    vvcam_isp_dg_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_CMPD_20BIT) || defined(ISP_CMPD_24BIT)
+    vvcam_isp_cpd_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_DPF_V1)
+    vvcam_isp_dpf_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_CNR_V1) || defined(ISP_CNR_V2_1) || defined(ISP_CNR_V2_2)
+    vvcam_isp_cnr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_GWDR_V1)
+    vvcam_isp_gwdr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_YNR_V1)
+    vvcam_isp_ynr_ctrl_create(isp_dev);
+#endif
+
+#if defined(ISP_LUT3D_V1)
+    vvcam_isp_lut3d_ctrl_create(isp_dev);
+#endif
+
+    isp_dev->sd.ctrl_handler = &isp_dev->ctrl_handler;
+
+    return 0;
+}
+
+int vvcam_isp_ctrl_destroy(struct vvcam_isp_dev *isp_dev)
+{
+    v4l2_ctrl_handler_free(&isp_dev->ctrl_handler);
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/vvcam_isp_ctrl.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/vvcam_isp_ctrl.h
new file mode 100644
index 000000000000..0a704498f21b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/vvcam_isp_ctrl.h
@@ -0,0 +1,101 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_CTRL_H__
+#define  __VVCAM_ISP_CTRL_H__
+
+#ifdef __KERNEL__
+#include "vvcam_isp_driver.h"
+#else
+#include <linux/videodev2.h>
+#endif
+
+#define VVCAM_ISP_CID_AE_BASE       (V4L2_CID_USER_BASE + 0x2000)
+#define VVCAM_ISP_CID_AWB_BASE      (V4L2_CID_USER_BASE + 0x2100)
+#define VVCAM_ISP_CID_GC_BASE       (V4L2_CID_USER_BASE + 0x2200)
+#define VVCAM_ISP_CID_2DNR_BASE     (V4L2_CID_USER_BASE + 0x2300)
+#define VVCAM_ISP_CID_3DNR_BASE     (V4L2_CID_USER_BASE + 0x2400)
+#define VVCAM_ISP_CID_BLS_BASE      (V4L2_CID_USER_BASE + 0x2500)
+#define VVCAM_ISP_CID_CCM_BASE      (V4L2_CID_USER_BASE + 0x2600)
+#define VVCAM_ISP_CID_CNR_BASE      (V4L2_CID_USER_BASE + 0x2700)
+#define VVCAM_ISP_CID_CPD_BASE      (V4L2_CID_USER_BASE + 0x2800)
+#define VVCAM_ISP_CID_CPROC_BASE    (V4L2_CID_USER_BASE + 0x2900)
+#define VVCAM_ISP_CID_DG_BASE       (V4L2_CID_USER_BASE + 0x2A00)
+#define VVCAM_ISP_CID_DMSC_BASE     (V4L2_CID_USER_BASE + 0x2B00)
+#define VVCAM_ISP_CID_DPCC_BASE     (V4L2_CID_USER_BASE + 0x2C00)
+#define VVCAM_ISP_CID_DPF_BASE      (V4L2_CID_USER_BASE + 0x2D00)
+#define VVCAM_ISP_CID_EE_BASE       (V4L2_CID_USER_BASE + 0x2E00)
+#define VVCAM_ISP_CID_GE_BASE       (V4L2_CID_USER_BASE + 0x2F00)
+#define VVCAM_ISP_CID_GTM_BASE      (V4L2_CID_USER_BASE + 0x3000)
+#define VVCAM_ISP_CID_HDR_BASE      (V4L2_CID_USER_BASE + 0x3100)
+#define VVCAM_ISP_CID_LSC_BASE      (V4L2_CID_USER_BASE + 0x3200)
+#define VVCAM_ISP_CID_LUT3D_BASE    (V4L2_CID_USER_BASE + 0x3300)
+#define VVCAM_ISP_CID_PDAF_BASE     (V4L2_CID_USER_BASE + 0x3400)
+#define VVCAM_ISP_CID_AF_BASE       (V4L2_CID_USER_BASE + 0x3500)
+#define VVCAM_ISP_CID_RGBIR_BASE    (V4L2_CID_USER_BASE + 0x3600)
+#define VVCAM_ISP_CID_WB_BASE       (V4L2_CID_USER_BASE + 0x3700)
+#define VVCAM_ISP_CID_WDR_BASE      (V4L2_CID_USER_BASE + 0x3800)
+#define VVCAM_ISP_CID_YNR_BASE      (V4L2_CID_USER_BASE + 0x3900)
+#define VVCAM_ISP_CID_AFM_BASE      (V4L2_CID_USER_BASE + 0x3A00)
+#define VVCAM_ISP_CID_EXP_BASE      (V4L2_CID_USER_BASE + 0x3B00)
+#define VVCAM_ISP_CID_GWDR_BASE     (V4L2_CID_USER_BASE + 0x3C00)
+#define VVCAM_ISP_CID_EXP_V3_BASE   (V4L2_CID_USER_BASE + 0x3D00)
+
+#ifdef __KERNEL__
+int vvcam_isp_ctrl_init(struct vvcam_isp_dev *isp_dev);
+int vvcam_isp_ctrl_destroy(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1/vvcam_isp_wb.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1/vvcam_isp_wb.c
new file mode 100644
index 000000000000..77cb6d7a7229
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1/vvcam_isp_wb.c
@@ -0,0 +1,258 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wb.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wb_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WB_ENABLE:
+        case VVCAM_ISP_CID_WB_RESET:
+        case VVCAM_ISP_CID_WB_R_GAIN:
+        case VVCAM_ISP_CID_WB_GR_GAIN:
+        case VVCAM_ISP_CID_WB_GB_GAIN:
+        case VVCAM_ISP_CID_WB_B_GAIN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wb_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WB_ENABLE:
+        case VVCAM_ISP_CID_WB_R_GAIN:
+        case VVCAM_ISP_CID_WB_GR_GAIN:
+        case VVCAM_ISP_CID_WB_GB_GAIN:
+        case VVCAM_ISP_CID_WB_B_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_R_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_GR_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_GB_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_B_GAIN:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wb_ctrl_ops = {
+    .s_ctrl = vvcam_isp_wb_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wb_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wb_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_R_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_r_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_GR_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_gr_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_GB_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_gb_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_B_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_b_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_R_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_r_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_GR_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_gr_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_GB_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_gb_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+    {
+        /* float (0, 3.996], ratio 256:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_B_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_b_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 1022,
+        .def  = 1,
+    },
+};
+
+int vvcam_isp_wb_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wb_ctrls);
+}
+
+int vvcam_isp_wb_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wb_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wb_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wb ctrl %s failed %d.\n",
+                vvcam_isp_wb_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1/vvcam_isp_wb.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1/vvcam_isp_wb.h
new file mode 100644
index 000000000000..9093618b6f09
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1/vvcam_isp_wb.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WB_H__
+#define __VVCAM_ISP_WB_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WB_ENABLE             (VVCAM_ISP_CID_WB_BASE + 0x0000)
+#define VVCAM_ISP_CID_WB_RESET              (VVCAM_ISP_CID_WB_BASE + 0x0001)
+#define VVCAM_ISP_CID_WB_R_GAIN             (VVCAM_ISP_CID_WB_BASE + 0x0002)
+#define VVCAM_ISP_CID_WB_GR_GAIN            (VVCAM_ISP_CID_WB_BASE + 0x0003)
+#define VVCAM_ISP_CID_WB_GB_GAIN            (VVCAM_ISP_CID_WB_BASE + 0x0004)
+#define VVCAM_ISP_CID_WB_B_GAIN             (VVCAM_ISP_CID_WB_BASE + 0x0005)
+#define VVCAM_ISP_CID_WB_STAT_R_GAIN        (VVCAM_ISP_CID_WB_BASE + 0x0006)
+#define VVCAM_ISP_CID_WB_STAT_GR_GAIN       (VVCAM_ISP_CID_WB_BASE + 0x0007)
+#define VVCAM_ISP_CID_WB_STAT_GB_GAIN       (VVCAM_ISP_CID_WB_BASE + 0x0008)
+#define VVCAM_ISP_CID_WB_STAT_B_GAIN        (VVCAM_ISP_CID_WB_BASE + 0x0009)
+
+#ifdef __KERNEL__
+int vvcam_isp_wb_ctrl_count(void);
+int vvcam_isp_wb_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.c
new file mode 100644
index 000000000000..ee209ce5e08e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.c
@@ -0,0 +1,258 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wb.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wb_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WB_ENABLE:
+        case VVCAM_ISP_CID_WB_RESET:
+        case VVCAM_ISP_CID_WB_R_GAIN:
+        case VVCAM_ISP_CID_WB_GR_GAIN:
+        case VVCAM_ISP_CID_WB_GB_GAIN:
+        case VVCAM_ISP_CID_WB_B_GAIN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wb_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WB_ENABLE:
+        case VVCAM_ISP_CID_WB_R_GAIN:
+        case VVCAM_ISP_CID_WB_GR_GAIN:
+        case VVCAM_ISP_CID_WB_GB_GAIN:
+        case VVCAM_ISP_CID_WB_B_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_R_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_GR_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_GB_GAIN:
+        case VVCAM_ISP_CID_WB_STAT_B_GAIN:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wb_ctrl_ops = {
+    .s_ctrl = vvcam_isp_wb_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wb_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wb_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_R_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_r_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_GR_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_gr_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_GB_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_gb_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_B_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_b_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_R_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_r_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_GR_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_gr_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_GB_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_gb_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+    {
+        /* float (0, 15.999], ratio 1024:1 */
+        .ops  = &vvcam_isp_wb_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WB_STAT_B_GAIN,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wb_stat_b_gain",
+        .step = 1,
+        .min  = 1,
+        .max  = 16382,
+        .def  = 1,
+    },
+};
+
+int vvcam_isp_wb_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wb_ctrls);
+}
+
+int vvcam_isp_wb_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wb_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wb_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wb ctrl %s failed %d.\n",
+                vvcam_isp_wb_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.h
new file mode 100644
index 000000000000..9093618b6f09
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wb/wb_v1_1/vvcam_isp_wb.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WB_H__
+#define __VVCAM_ISP_WB_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WB_ENABLE             (VVCAM_ISP_CID_WB_BASE + 0x0000)
+#define VVCAM_ISP_CID_WB_RESET              (VVCAM_ISP_CID_WB_BASE + 0x0001)
+#define VVCAM_ISP_CID_WB_R_GAIN             (VVCAM_ISP_CID_WB_BASE + 0x0002)
+#define VVCAM_ISP_CID_WB_GR_GAIN            (VVCAM_ISP_CID_WB_BASE + 0x0003)
+#define VVCAM_ISP_CID_WB_GB_GAIN            (VVCAM_ISP_CID_WB_BASE + 0x0004)
+#define VVCAM_ISP_CID_WB_B_GAIN             (VVCAM_ISP_CID_WB_BASE + 0x0005)
+#define VVCAM_ISP_CID_WB_STAT_R_GAIN        (VVCAM_ISP_CID_WB_BASE + 0x0006)
+#define VVCAM_ISP_CID_WB_STAT_GR_GAIN       (VVCAM_ISP_CID_WB_BASE + 0x0007)
+#define VVCAM_ISP_CID_WB_STAT_GB_GAIN       (VVCAM_ISP_CID_WB_BASE + 0x0008)
+#define VVCAM_ISP_CID_WB_STAT_B_GAIN        (VVCAM_ISP_CID_WB_BASE + 0x0009)
+
+#ifdef __KERNEL__
+int vvcam_isp_wb_ctrl_count(void);
+int vvcam_isp_wb_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..714503824fca
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.c
@@ -0,0 +1,227 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_MAX_GAIN:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..04786474ff69
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v3/vvcam_isp_wdr.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_MANU_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_STAT_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..ee565cff36cd
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.c
@@ -0,0 +1,646 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_HIGH_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_LOW_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_HIGH_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_LOW_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_THR:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_PRE:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_HIGH_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_LOW_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_HIGH_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_LOW_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_THR:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_PRE:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP:
+        case VVCAM_ISP_CID_WDR_STAT_CONTRAST:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_HIGH_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_LOW_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_THR:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_DOWN:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_PRE:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_level",
+        .step = 1,
+        .def  = 1,
+        .min  = 1,
+        .max  = 20,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HIGH_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LOW_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GLOBAL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_global_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* int16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 19,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 20,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+
+    /*
+    manual control type is difference between cam_device and cam_engine
+    and some code is not match with doc, need to double check.
+    */
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HIGH_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LOW_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 19,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 20,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_PRE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_pre",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HIGH_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LOW_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 255,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 19,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 20,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy",
+        .step = 1,
+        .min  = 0,
+        .max  = 511,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_DOWN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_PRE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_pre",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {20, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..f2a5f3159930
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v4/vvcam_isp_wdr.h
@@ -0,0 +1,120 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_MODE              (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_AUTO_LEVEL        (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_AUTO_GAIN         (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_AUTO_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_AUTO_HIGH_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_AUTO_LOW_STRENGTH (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_WDR_AUTO_GLOBAL_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_WDR_AUTO_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_THR     (VVCAM_ISP_CID_WDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x000D)
+
+#define VVCAM_ISP_CID_WDR_MANU_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_WDR_MANU_HIGH_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_WDR_MANU_LOW_STRENGTH (VVCAM_ISP_CID_WDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY      (VVCAM_ISP_CID_WDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_DOWN   (VVCAM_ISP_CID_WDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_PRE    (VVCAM_ISP_CID_WDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP     (VVCAM_ISP_CID_WDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_WDR_STAT_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_WDR_STAT_HIGH_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_WDR_STAT_LOW_STRENGTH (VVCAM_ISP_CID_WDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY      (VVCAM_ISP_CID_WDR_BASE + 0x0024)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_DOWN   (VVCAM_ISP_CID_WDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_PRE    (VVCAM_ISP_CID_WDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP     (VVCAM_ISP_CID_WDR_BASE + 0x0027)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..6ddb50dd6c80
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.c
@@ -0,0 +1,1170 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_CONTRAST:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_RATIO:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_STAT_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_H_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_L_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_global_curve_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20}
+    },
+        {
+        /* int16_t 20x array [-1023, 1023]*/
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array [0.0 ~ 1.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array [0.0 ~ 3.0]*/
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4, 17, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_high_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4, 17, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_high_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..3b0be4f38007
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5/vvcam_isp_wdr.h
@@ -0,0 +1,170 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_MODE              (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_AUTO_LEVEL        (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_AUTO_GAIN         (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_WDR_AUTO_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_WDR_AUTO_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_WDR_MANU_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_WDR_MANU_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_WDR_MANU_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_WDR_MANU_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x001A)
+
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE (VVCAM_ISP_CID_WDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_WDR_MANU_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0024)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT (VVCAM_ISP_CID_WDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_WDR_MANU_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_WDR_MANU_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x002B)
+#define VVCAM_ISP_CID_WDR_MANU_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_WDR_MANU_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_WDR_MANU_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x002F)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_WDR_STAT_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_WDR_STAT_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_WDR_STAT_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0038)
+#define VVCAM_ISP_CID_WDR_STAT_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0039)
+
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE (VVCAM_ISP_CID_WDR_BASE + 0x0042)
+#define VVCAM_ISP_CID_WDR_STAT_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT (VVCAM_ISP_CID_WDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0046)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_WDR_STAT_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_WDR_STAT_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_WDR_STAT_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_WDR_STAT_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_WDR_STAT_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004F)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0050)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..6ddb50dd6c80
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.c
@@ -0,0 +1,1170 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_CONTRAST:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_RATIO:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_STAT_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_H_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_L_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_global_curve_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20}
+    },
+        {
+        /* int16_t 20x array [-1023, 1023]*/
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array [0.0 ~ 1.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array [0.0 ~ 3.0]*/
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4, 17, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_high_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4, 17, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        /* float (0.0, 2.2] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_high_fac",
+        .step = 1,
+        .min  = 1,
+        .max  = 220,
+        .def  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1, 0, 0, 0},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..3b0be4f38007
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_1/vvcam_isp_wdr.h
@@ -0,0 +1,170 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_GLOBAL_CURVE_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_MODE              (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_AUTO_LEVEL        (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_AUTO_GAIN         (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_WDR_AUTO_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_WDR_AUTO_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_WDR_MANU_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_WDR_MANU_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_WDR_MANU_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_WDR_MANU_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x001A)
+
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE (VVCAM_ISP_CID_WDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_WDR_MANU_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0024)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT (VVCAM_ISP_CID_WDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_WDR_MANU_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_WDR_MANU_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x002B)
+#define VVCAM_ISP_CID_WDR_MANU_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_WDR_MANU_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_WDR_MANU_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x002F)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_WDR_STAT_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_WDR_STAT_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_WDR_STAT_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0038)
+#define VVCAM_ISP_CID_WDR_STAT_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0039)
+
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE (VVCAM_ISP_CID_WDR_BASE + 0x0042)
+#define VVCAM_ISP_CID_WDR_STAT_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT (VVCAM_ISP_CID_WDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0046)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC (VVCAM_ISP_CID_WDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_WDR_STAT_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_WDR_STAT_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_WDR_STAT_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_WDR_STAT_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_WDR_STAT_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004F)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0050)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..f332e80be307
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.c
@@ -0,0 +1,2113 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL:
+        case VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL:
+        case VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_CONTRAST:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_RATIO:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_STAT_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_H_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_L_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_halo_color_fading_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_luma_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_sat_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20}
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_sat_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4},
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_edr_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_EDR,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_edr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* int16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20, 20},
+    },
+    {
+        /* uint16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 20},
+    },
+    {
+        /* uint8_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {20, 20},
+    },
+    {
+        /* float 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 20},
+    },
+    {
+        /* uint16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 20},
+    },
+    {
+        /* uint16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 20},
+    },
+    {
+        /* uint8_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20, 20},
+    },
+    {
+        /* bool 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 20},
+    },
+    {
+        /* uint8_t 20x20x3 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 20, 3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4,17},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_high_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_sat_l_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_sat_h_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 1900,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_coef_dec_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_coef_inc_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_filter_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1},
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_h_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_l_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4,17},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_high_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_sat_l_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_sat_h_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 1900,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_coef_dec_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_coef_inc_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_filter_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1},
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_h_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_l_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..349b763e19ec
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2/vvcam_isp_wdr.h
@@ -0,0 +1,319 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_MODE              (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_AUTO_LEVEL        (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_AUTO_GAIN         (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_WDR_AUTO_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_WDR_AUTO_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL    (VVCAM_ISP_CID_WDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_WDR_AUTO_EDR          (VVCAM_ISP_CID_WDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_WDR_AUTO_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE    (VVCAM_ISP_CID_WDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x0024)
+
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_WDR_MANU_H_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_WDR_MANU_L_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_WDR_MANU_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002B)
+#define VVCAM_ISP_CID_WDR_MANU_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002F)
+#define VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_WDR_MANU_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0038)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0039)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_WDR_MANU_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_WDR_MANU_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_WDR_MANU_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_WDR_MANU_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_WDR_MANU_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0042)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0046)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004F)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0050)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0051)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0052)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0053)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0054)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0055)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0056)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0057)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0058)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0059)
+#define VVCAM_ISP_CID_WDR_MANU_RGB_COEF     (VVCAM_ISP_CID_WDR_BASE + 0x005A)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005B)
+#define VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005C)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005D)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005E)
+
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0070)
+#define VVCAM_ISP_CID_WDR_STAT_H_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0071)
+#define VVCAM_ISP_CID_WDR_STAT_L_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0072)
+#define VVCAM_ISP_CID_WDR_STAT_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0073)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0074)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0075)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0076)
+#define VVCAM_ISP_CID_WDR_STAT_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0077)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x0078)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x0079)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007A)
+#define VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007B)
+#define VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007C)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x007D)
+#define VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x007E)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007F)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0080)
+#define VVCAM_ISP_CID_WDR_STAT_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0081)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0082)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0083)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0084)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0085)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0086)
+#define VVCAM_ISP_CID_WDR_STAT_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0087)
+#define VVCAM_ISP_CID_WDR_STAT_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0088)
+#define VVCAM_ISP_CID_WDR_STAT_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0089)
+#define VVCAM_ISP_CID_WDR_STAT_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x008A)
+#define VVCAM_ISP_CID_WDR_STAT_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x008B)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x008C)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008D)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008E)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008F)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0090)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0091)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0092)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0093)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0094)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0095)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0096)
+#define VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0097)
+#define VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0098)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x0099)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009A)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009B)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009C)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009D)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009E)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009F)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A0)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A1)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A2)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A3)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A4)
+#define VVCAM_ISP_CID_WDR_STAT_RGB_COEF     (VVCAM_ISP_CID_WDR_BASE + 0x00A5)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A6)
+#define VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A7)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A8)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A9)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..5025d141c1f9
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.c
@@ -0,0 +1,2085 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL:
+        case VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL:
+        case VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_CONTRAST:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_RATIO:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_STAT_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_H_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_L_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_halo_color_fading_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_luma_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_sat_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20}
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_sat_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4},
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4},
+    },
+    {
+        /* int16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20},
+    },
+    {
+        /* bool 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x3 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4,17},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_high_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_sat_l_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_sat_h_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 1900,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_coef_dec_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_coef_inc_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_filter_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1},
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_h_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_l_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4,17},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_high_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_sat_l_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_sat_h_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 1900,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_coef_dec_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_coef_inc_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_filter_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1},
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_h_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_l_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..6057b95079f4
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_2_1/vvcam_isp_wdr.h
@@ -0,0 +1,317 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_MODE              (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_AUTO_LEVEL        (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_AUTO_GAIN         (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_WDR_AUTO_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_WDR_AUTO_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_WDR_AUTO_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE    (VVCAM_ISP_CID_WDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x0022)
+
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_WDR_MANU_H_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0024)
+#define VVCAM_ISP_CID_WDR_MANU_L_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_WDR_MANU_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_WDR_MANU_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x002B)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002F)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_WDR_MANU_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0038)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0039)
+#define VVCAM_ISP_CID_WDR_MANU_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_WDR_MANU_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_WDR_MANU_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_WDR_MANU_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_WDR_MANU_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0042)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0046)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004F)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0050)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0051)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0052)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0053)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0054)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0055)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0056)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0057)
+#define VVCAM_ISP_CID_WDR_MANU_RGB_COEF     (VVCAM_ISP_CID_WDR_BASE + 0x0058)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0059)
+#define VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005A)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005B)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005C)
+
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0070)
+#define VVCAM_ISP_CID_WDR_STAT_H_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0071)
+#define VVCAM_ISP_CID_WDR_STAT_L_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0072)
+#define VVCAM_ISP_CID_WDR_STAT_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0073)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0074)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0075)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0076)
+#define VVCAM_ISP_CID_WDR_STAT_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0077)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x0078)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x0079)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007A)
+#define VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007B)
+#define VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007C)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x007D)
+#define VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x007E)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007F)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0080)
+#define VVCAM_ISP_CID_WDR_STAT_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0081)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0082)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0083)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0084)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0085)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0086)
+#define VVCAM_ISP_CID_WDR_STAT_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0087)
+#define VVCAM_ISP_CID_WDR_STAT_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0088)
+#define VVCAM_ISP_CID_WDR_STAT_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0089)
+#define VVCAM_ISP_CID_WDR_STAT_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x008A)
+#define VVCAM_ISP_CID_WDR_STAT_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x008B)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x008C)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008D)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008E)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008F)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0090)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0091)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0092)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0093)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0094)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0095)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0096)
+#define VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0097)
+#define VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0098)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x0099)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009A)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009B)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009C)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009D)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009E)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009F)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A0)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A1)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A2)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A3)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A4)
+#define VVCAM_ISP_CID_WDR_STAT_RGB_COEF     (VVCAM_ISP_CID_WDR_BASE + 0x00A5)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A6)
+#define VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A7)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A8)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A9)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.c
new file mode 100644
index 000000000000..f332e80be307
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.c
@@ -0,0 +1,2113 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_wdr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_wdr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL:
+        case VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_wdr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_WDR_ENABLE:
+        case VVCAM_ISP_CID_WDR_RESET:
+        case VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE:
+        case VVCAM_ISP_CID_WDR_MODE:
+        case VVCAM_ISP_CID_WDR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_AUTO_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_AUTO_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL:
+        case VVCAM_ISP_CID_WDR_AUTO_EDR:
+        case VVCAM_ISP_CID_WDR_AUTO_CONTRAST:
+        case VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL:
+        case VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_MANU_CONTRAST:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_RATIO:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_MANU_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_MANU_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_MANU_H_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_L_DIFF:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_MANU_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y:
+        case VVCAM_ISP_CID_WDR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_H_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_L_STRENGTH:
+        case VVCAM_ISP_CID_WDR_STAT_CONTRAST:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE:
+        case VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_LUMA_THR:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC:
+        case VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT:
+        case VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_RATIO:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_THR:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC:
+        case VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL:
+        case VVCAM_ISP_CID_WDR_STAT_DEGAMMA:
+        case VVCAM_ISP_CID_WDR_STAT_MAX_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_MIN_GAIN:
+        case VVCAM_ISP_CID_WDR_STAT_H_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_L_DIFF:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_RANGE:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN:
+        case VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_MODE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG:
+        case VVCAM_ISP_CID_WDR_STAT_RGB_COEF:
+        case VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X:
+        case VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_wdr_ctrl_ops = {
+	.s_ctrl = vvcam_isp_wdr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_wdr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_wdr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_halo_color_fading_en",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20}
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_luma_threshold",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {20}
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20},
+    },
+    {
+        /* uint16_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_sat_low_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20}
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_sat_high_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4}
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4},
+    },
+    {
+        /* float 20x4 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 4},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_edr_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_EDR,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_edr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20},
+    },
+    {
+        /* int16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_CONTRAST,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_contrast",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFF,
+        .dims = {20, 20},
+    },
+    {
+        /* uint16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {20, 20},
+    },
+    {
+        /* uint8_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {20, 20},
+    },
+    {
+        /* float 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 20},
+    },
+    {
+        /* uint16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 20},
+    },
+    {
+        /* uint16_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {20, 20},
+    },
+    {
+        /* uint8_t 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {20, 20},
+    },
+    {
+        /* bool 20x20 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_light_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 20},
+    },
+    {
+        /* uint8_t 20x20x3 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_auto_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 20, 3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4,17},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_low_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_curve2_high_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_sat_l_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_sat_h_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 1900,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_curve_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_avg_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_coef_dec_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_coef_inc_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_filter_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1},
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_h_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_damp_l_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_manu_gamma_up_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CONTRAST,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_contrast",
+        .step = 1,
+        .min  = -1023,
+        .max  = 1023,
+        .def  = 500,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_base",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_entropy_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LUMA_THR,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_luma_thr",
+        .step = 1,
+        .min  = 1,
+        .max  = 1023,
+        .def  = 1,
+        .dims = {1}
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_MODE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 4x17 array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_flat_level_inc",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {4,17},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_dark_atten_level",
+        .step = 1,
+        .min  = 0,
+        .max  = 14,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_fixed_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 1023,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve_select",
+        .step = 1,
+        .min  = 0,
+        .max  = 2,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_log_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 10,
+        .dims = {1},
+    },
+    {
+        /* float 0.0 ~ 20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_x_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_anchor_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .def  = 16,
+        .dims = {1},
+    },
+    {
+        /* float (0.0, 2.0] */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RATIO,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_actual_ratio",
+        .step = 1,
+        .min  = 1,
+        .max  = 200,
+        .def  = 1,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_THR,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_threshold",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 1.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_sat",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_low_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        /* float 0.0 ~ 2.2 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_curve2_high_fac",
+        .step = 1,
+        .min  = 0,
+        .max  = 220,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_global_flat_lv",
+        .step = 1,
+        .min  = 0,
+        .max  = 15,
+        .dims = {1},
+    },
+    {
+        /* float 0~3.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DEGAMMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_degamma",
+        .step = 1,
+        .min  = 0,
+        .max  = 300,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MAX_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_max_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_MIN_GAIN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_min_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 4096,
+        .dims = {1},
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_H_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_high_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~100.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_L_DIFF,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_low_diff",
+        .step = 1,
+        .min  = 0,
+        .max  = 10000,
+    },
+    {
+        /* float 0~1 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_RANGE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_range",
+        .step = 1,
+        .min  = 0,
+        .max  = 100,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_down",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP,
+        .type = V4L2_CTRL_TYPE_U16,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_sat_thr_gain_up",
+        .step = 1,
+        .min  = 0,
+        .max  = 256,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_sat_l_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_sat_h_thr",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_r_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_g_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_b_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_y_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_c_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float 4x array 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_light_m_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {4},
+    },
+    {
+        /* float */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_hlc_base_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 1900,
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_hlc_slope",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFF,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_MODE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_curve_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_max",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_avg_min",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_coef_dec_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_coef_inc_lt",
+        .step = 1,
+        .min  = 0,
+        .max  = 127,
+        .dims = {1},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_filter_size",
+        .step = 1,
+        .min  = 0,
+        .max  = 16,
+        .dims = {1},
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_h_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* float 0~20.0 */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_damp_l_thr_log",
+        .step = 1,
+        .min  = 0,
+        .max  = 2000,
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_RGB_COEF,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_rgb_coef",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_lightness_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {1},
+    },
+    {
+        /* uint8_t 3x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_color_weight",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {3},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up_curve_x",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+    {
+        /* uint32_t 65x array */
+        .ops  = &vvcam_isp_wdr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_wdr_stat_gamma_up_curve_y",
+        .step = 1,
+        .min  = 0,
+        .max  = 1048575,
+        .dims = {65},
+    },
+};
+
+int vvcam_isp_wdr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_wdr_ctrls);
+}
+
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_wdr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_wdr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp wdr ctrl %s failed %d.\n",
+                vvcam_isp_wdr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.h
new file mode 100644
index 000000000000..349b763e19ec
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/wdr/wdr_v5_3/vvcam_isp_wdr.h
@@ -0,0 +1,319 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_WDR_H__
+#define __VVCAM_ISP_WDR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_WDR_ENABLE            (VVCAM_ISP_CID_WDR_BASE + 0x0000)
+#define VVCAM_ISP_CID_WDR_RESET             (VVCAM_ISP_CID_WDR_BASE + 0x0001)
+#define VVCAM_ISP_CID_WDR_HALO_COLOR_FADING_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0002)
+#define VVCAM_ISP_CID_WDR_MODE              (VVCAM_ISP_CID_WDR_BASE + 0x0003)
+#define VVCAM_ISP_CID_WDR_AUTO_LEVEL        (VVCAM_ISP_CID_WDR_BASE + 0x0004)
+#define VVCAM_ISP_CID_WDR_AUTO_GAIN         (VVCAM_ISP_CID_WDR_BASE + 0x0005)
+#define VVCAM_ISP_CID_WDR_AUTO_H_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0006)
+#define VVCAM_ISP_CID_WDR_AUTO_L_STRENGTH   (VVCAM_ISP_CID_WDR_BASE + 0x0007)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_BASE (VVCAM_ISP_CID_WDR_BASE + 0x0008)
+#define VVCAM_ISP_CID_WDR_AUTO_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0009)
+#define VVCAM_ISP_CID_WDR_AUTO_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000A)
+#define VVCAM_ISP_CID_WDR_AUTO_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x000B)
+#define VVCAM_ISP_CID_WDR_AUTO_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000C)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x000D)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000E)
+#define VVCAM_ISP_CID_WDR_AUTO_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x000F)
+#define VVCAM_ISP_CID_WDR_AUTO_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0010)
+#define VVCAM_ISP_CID_WDR_AUTO_HLC_BASE_LOG (VVCAM_ISP_CID_WDR_BASE + 0x0011)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0012)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0013)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0014)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0015)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0016)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0017)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0018)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0019)
+#define VVCAM_ISP_CID_WDR_AUTO_EDR_LEVEL    (VVCAM_ISP_CID_WDR_BASE + 0x001A)
+#define VVCAM_ISP_CID_WDR_AUTO_EDR          (VVCAM_ISP_CID_WDR_BASE + 0x001B)
+#define VVCAM_ISP_CID_WDR_AUTO_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x001C)
+#define VVCAM_ISP_CID_WDR_AUTO_FIXED_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x001D)
+#define VVCAM_ISP_CID_WDR_AUTO_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x001E)
+#define VVCAM_ISP_CID_WDR_AUTO_ANCHOR_X_LOG (VVCAM_ISP_CID_WDR_BASE + 0x001F)
+#define VVCAM_ISP_CID_WDR_AUTO_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0020)
+#define VVCAM_ISP_CID_WDR_AUTO_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0021)
+#define VVCAM_ISP_CID_WDR_AUTO_HLC_SLOPE    (VVCAM_ISP_CID_WDR_BASE + 0x0022)
+#define VVCAM_ISP_CID_WDR_AUTO_LIGHT_ENABLE_TBL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0023)
+#define VVCAM_ISP_CID_WDR_AUTO_COLOR_WEIGHT (VVCAM_ISP_CID_WDR_BASE + 0x0024)
+
+#define VVCAM_ISP_CID_WDR_MANU_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0025)
+#define VVCAM_ISP_CID_WDR_MANU_H_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0026)
+#define VVCAM_ISP_CID_WDR_MANU_L_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0027)
+#define VVCAM_ISP_CID_WDR_MANU_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0028)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0029)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002A)
+#define VVCAM_ISP_CID_WDR_MANU_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002B)
+#define VVCAM_ISP_CID_WDR_MANU_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x002C)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x002D)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x002E)
+#define VVCAM_ISP_CID_WDR_MANU_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x002F)
+#define VVCAM_ISP_CID_WDR_MANU_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0030)
+#define VVCAM_ISP_CID_WDR_MANU_FIXED_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0031)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x0032)
+#define VVCAM_ISP_CID_WDR_MANU_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x0033)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_X_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0034)
+#define VVCAM_ISP_CID_WDR_MANU_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0035)
+#define VVCAM_ISP_CID_WDR_MANU_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0036)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0037)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_SAT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0038)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_L_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0039)
+#define VVCAM_ISP_CID_WDR_MANU_CURVE2_H_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003A)
+#define VVCAM_ISP_CID_WDR_MANU_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x003B)
+#define VVCAM_ISP_CID_WDR_MANU_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x003C)
+#define VVCAM_ISP_CID_WDR_MANU_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x003D)
+#define VVCAM_ISP_CID_WDR_MANU_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x003E)
+#define VVCAM_ISP_CID_WDR_MANU_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x003F)
+#define VVCAM_ISP_CID_WDR_MANU_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x0040)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x0041)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0042)
+#define VVCAM_ISP_CID_WDR_MANU_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0043)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0044)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0045)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0046)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0047)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0048)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0049)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004A)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004B)
+#define VVCAM_ISP_CID_WDR_MANU_HLC_BASE_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004C)
+#define VVCAM_ISP_CID_WDR_MANU_HLC_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004D)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x004E)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x004F)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0050)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_CURVE_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0051)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0052)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0053)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_AVG_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0054)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_DEC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0055)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_COEF_INC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0056)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_FILTER_SIZE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0057)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_H_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0058)
+#define VVCAM_ISP_CID_WDR_MANU_DAMP_L_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0059)
+#define VVCAM_ISP_CID_WDR_MANU_RGB_COEF     (VVCAM_ISP_CID_WDR_BASE + 0x005A)
+#define VVCAM_ISP_CID_WDR_MANU_LIGHT_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005B)
+#define VVCAM_ISP_CID_WDR_MANU_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005C)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_X \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005D)
+#define VVCAM_ISP_CID_WDR_MANU_GAMMA_UP_CURVE_Y \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x005E)
+
+#define VVCAM_ISP_CID_WDR_STAT_STRENGTH     (VVCAM_ISP_CID_WDR_BASE + 0x0070)
+#define VVCAM_ISP_CID_WDR_STAT_H_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0071)
+#define VVCAM_ISP_CID_WDR_STAT_L_STRENGTH \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0072)
+#define VVCAM_ISP_CID_WDR_STAT_CONTRAST     (VVCAM_ISP_CID_WDR_BASE + 0x0073)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_ENABLE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0074)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_BASE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0075)
+#define VVCAM_ISP_CID_WDR_STAT_ENTROPY_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0076)
+#define VVCAM_ISP_CID_WDR_STAT_LUMA_THR     (VVCAM_ISP_CID_WDR_BASE + 0x0077)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x0078)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL   (VVCAM_ISP_CID_WDR_BASE + 0x0079)
+#define VVCAM_ISP_CID_WDR_STAT_FLAT_LEVEL_INC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007A)
+#define VVCAM_ISP_CID_WDR_STAT_DARK_ATT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007B)
+#define VVCAM_ISP_CID_WDR_STAT_FIXED_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007C)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE_SELECT (VVCAM_ISP_CID_WDR_BASE + 0x007D)
+#define VVCAM_ISP_CID_WDR_STAT_LOG_WEIGHT   (VVCAM_ISP_CID_WDR_BASE + 0x007E)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_X_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x007F)
+#define VVCAM_ISP_CID_WDR_STAT_ANCHOR_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0080)
+#define VVCAM_ISP_CID_WDR_STAT_RATIO        (VVCAM_ISP_CID_WDR_BASE + 0x0081)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_THR   (VVCAM_ISP_CID_WDR_BASE + 0x0082)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_SAT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0083)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_L_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0084)
+#define VVCAM_ISP_CID_WDR_STAT_CURVE2_H_FAC \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0085)
+#define VVCAM_ISP_CID_WDR_STAT_GLOBAL_FLAT_LEVEL \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0086)
+#define VVCAM_ISP_CID_WDR_STAT_DEGAMMA      (VVCAM_ISP_CID_WDR_BASE + 0x0087)
+#define VVCAM_ISP_CID_WDR_STAT_MAX_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0088)
+#define VVCAM_ISP_CID_WDR_STAT_MIN_GAIN     (VVCAM_ISP_CID_WDR_BASE + 0x0089)
+#define VVCAM_ISP_CID_WDR_STAT_H_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x008A)
+#define VVCAM_ISP_CID_WDR_STAT_L_DIFF       (VVCAM_ISP_CID_WDR_BASE + 0x008B)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_RANGE    (VVCAM_ISP_CID_WDR_BASE + 0x008C)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_DOWN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008D)
+#define VVCAM_ISP_CID_WDR_STAT_SAT_THR_GAIN_UP \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008E)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_L_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x008F)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_SAT_H_THR \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0090)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_R_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0091)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_G_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0092)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_B_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0093)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_Y_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0094)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_C_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0095)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_M_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0096)
+#define VVCAM_ISP_CID_WDR_STAT_HLC_BASE_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0097)
+#define VVCAM_ISP_CID_WDR_STAT_HLC_SLOPE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x0098)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_MODE    (VVCAM_ISP_CID_WDR_BASE + 0x0099)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009A)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009B)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_CURVE_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009C)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_COEF \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009D)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MAX \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009E)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_AVG_MIN \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x009F)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_DEC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A0)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_COEF_INC_LIMIT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A1)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_FILTER_SIZE \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A2)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_H_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A3)
+#define VVCAM_ISP_CID_WDR_STAT_DAMP_L_THR_LOG \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A4)
+#define VVCAM_ISP_CID_WDR_STAT_RGB_COEF     (VVCAM_ISP_CID_WDR_BASE + 0x00A5)
+#define VVCAM_ISP_CID_WDR_STAT_LIGHT_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A6)
+#define VVCAM_ISP_CID_WDR_STAT_COLOR_WEIGHT \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A7)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_X \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A8)
+#define VVCAM_ISP_CID_WDR_STAT_GAMMA_UP_CURVE_Y \
+                                            (VVCAM_ISP_CID_WDR_BASE + 0x00A9)
+
+#ifdef __KERNEL__
+int vvcam_isp_wdr_ctrl_count(void);
+int vvcam_isp_wdr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.c
new file mode 100644
index 000000000000..f2277e89042c
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.c
@@ -0,0 +1,318 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-ioctl.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_ynr.h"
+#include "vvcam_isp_event.h"
+
+static int vvcam_isp_ynr_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_YNR_ENABLE:
+        case VVCAM_ISP_CID_YNR_RESET:
+        case VVCAM_ISP_CID_YNR_MODE:
+        case VVCAM_ISP_CID_YNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_YNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_YNR_AUTO_ENABLE_TBL:
+        case VVCAM_ISP_CID_YNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_YNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_YNR_AUTO_STRENGTH2:
+        case VVCAM_ISP_CID_YNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_YNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_YNR_MANU_STRENGTH2:
+            ret = vvcam_isp_s_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_ynr_g_ctrl(struct v4l2_ctrl *ctrl)
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev =
+        container_of(ctrl->handler, struct vvcam_isp_dev, ctrl_handler);
+
+    switch (ctrl->id)
+    {
+        case VVCAM_ISP_CID_YNR_ENABLE:
+        case VVCAM_ISP_CID_YNR_RESET:
+        case VVCAM_ISP_CID_YNR_MODE:
+        case VVCAM_ISP_CID_YNR_AUTO_LEVEL:
+        case VVCAM_ISP_CID_YNR_AUTO_GAIN:
+        case VVCAM_ISP_CID_YNR_AUTO_ENABLE_TBL:
+        case VVCAM_ISP_CID_YNR_AUTO_SIGMA:
+        case VVCAM_ISP_CID_YNR_AUTO_STRENGTH:
+        case VVCAM_ISP_CID_YNR_AUTO_STRENGTH2:
+        case VVCAM_ISP_CID_YNR_MANU_SIGMA:
+        case VVCAM_ISP_CID_YNR_MANU_STRENGTH:
+        case VVCAM_ISP_CID_YNR_MANU_STRENGTH2:
+        case VVCAM_ISP_CID_YNR_STAT_SIGMA:
+        case VVCAM_ISP_CID_YNR_STAT_STRENGTH:
+        case VVCAM_ISP_CID_YNR_STAT_STRENGTH2:
+            ret = vvcam_isp_g_ctrl_event(isp_dev, isp_dev->ctrl_pad, ctrl);
+            break;
+
+        default:
+            dev_err(isp_dev->dev, "unknow v4l2 ctrl id %d\n", ctrl->id);
+            return -EACCES;
+    }
+
+    return ret;
+}
+
+static const struct v4l2_ctrl_ops vvcam_isp_ynr_ctrl_ops = {
+    .s_ctrl = vvcam_isp_ynr_s_ctrl,
+    .g_volatile_ctrl = vvcam_isp_ynr_g_ctrl,
+};
+
+const struct v4l2_ctrl_config vvcam_isp_ynr_ctrls[] = {
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_ENABLE,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_enable",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_RESET,
+        .type = V4L2_CTRL_TYPE_BOOLEAN,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_reset",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_MODE,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_mode",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_AUTO_LEVEL,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_auto_level",
+        .step = 1,
+        .min  = 1,
+        .max  = 20,
+        .def  = 1,
+        .dims = {1, 0, 0, 0},
+    },
+    {
+        /* float 20x array */
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_AUTO_GAIN,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_auto_gain",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_AUTO_ENABLE_TBL,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_auto_enable_tbl",
+        .step = 1,
+        .min  = 0,
+        .max  = 1,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 20x array [0.0, 128.0] */
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_AUTO_SIGMA,
+        .type = V4L2_CTRL_TYPE_U32,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_auto_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 0xFFFFFFFF,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_AUTO_STRENGTH,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_auto_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* uint8_t 20x array */
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_AUTO_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_U8,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_auto_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+        .dims = {20, 0, 0, 0},
+    },
+    {
+        /* float 0 ~ 128.0 */
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_MANU_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_manu_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 12800,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_MANU_STRENGTH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_manu_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_MANU_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_manu_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+    },
+    {
+        /* float 0 ~ 128.0 */
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_STAT_SIGMA,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_stat_sigma",
+        .step = 1,
+        .min  = 0,
+        .max  = 12800,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_STAT_STRENGTH,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_stat_strength",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+    },
+    {
+        .ops  = &vvcam_isp_ynr_ctrl_ops,
+        .id   = VVCAM_ISP_CID_YNR_STAT_STRENGTH2,
+        .type = V4L2_CTRL_TYPE_INTEGER,
+        .flags= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_EXECUTE_ON_WRITE,
+        .name = "isp_ynr_stat_strength2",
+        .step = 1,
+        .min  = 0,
+        .max  = 128,
+    },
+};
+
+int vvcam_isp_ynr_ctrl_count(void)
+{
+    return ARRAY_SIZE(vvcam_isp_ynr_ctrls);
+}
+
+int vvcam_isp_ynr_ctrl_create(struct vvcam_isp_dev *isp_dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_isp_ynr_ctrls); i++) {
+        v4l2_ctrl_new_custom(&isp_dev->ctrl_handler,
+                            &vvcam_isp_ynr_ctrls[i], NULL);
+        if (isp_dev->ctrl_handler.error) {
+            dev_err( isp_dev->dev, "reigster isp ynr ctrl %s failed %d.\n",
+                vvcam_isp_ynr_ctrls[i].name, isp_dev->ctrl_handler.error);
+        }
+    }
+
+    return 0;
+
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.h
new file mode 100644
index 000000000000..65a7d262d271
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/isp_ctrl/ynr/ynr_v1/vvcam_isp_ynr.h
@@ -0,0 +1,82 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_YNR_H__
+#define __VVCAM_ISP_YNR_H__
+
+#include "vvcam_isp_ctrl.h"
+
+#define VVCAM_ISP_CID_YNR_ENABLE            (VVCAM_ISP_CID_YNR_BASE + 0x0000)
+#define VVCAM_ISP_CID_YNR_RESET             (VVCAM_ISP_CID_YNR_BASE + 0x0001)
+#define VVCAM_ISP_CID_YNR_MODE              (VVCAM_ISP_CID_YNR_BASE + 0x0002)
+#define VVCAM_ISP_CID_YNR_AUTO_LEVEL        (VVCAM_ISP_CID_YNR_BASE + 0x0003)
+#define VVCAM_ISP_CID_YNR_AUTO_GAIN         (VVCAM_ISP_CID_YNR_BASE + 0x0004)
+#define VVCAM_ISP_CID_YNR_AUTO_ENABLE_TBL   (VVCAM_ISP_CID_YNR_BASE + 0x0005)
+#define VVCAM_ISP_CID_YNR_AUTO_SIGMA        (VVCAM_ISP_CID_YNR_BASE + 0x0006)
+#define VVCAM_ISP_CID_YNR_AUTO_STRENGTH     (VVCAM_ISP_CID_YNR_BASE + 0x0007)
+#define VVCAM_ISP_CID_YNR_AUTO_STRENGTH2    (VVCAM_ISP_CID_YNR_BASE + 0x0008)
+#define VVCAM_ISP_CID_YNR_MANU_SIGMA        (VVCAM_ISP_CID_YNR_BASE + 0x0009)
+#define VVCAM_ISP_CID_YNR_MANU_STRENGTH     (VVCAM_ISP_CID_YNR_BASE + 0x000A)
+#define VVCAM_ISP_CID_YNR_MANU_STRENGTH2    (VVCAM_ISP_CID_YNR_BASE + 0x000B)
+#define VVCAM_ISP_CID_YNR_STAT_SIGMA        (VVCAM_ISP_CID_YNR_BASE + 0x000C)
+#define VVCAM_ISP_CID_YNR_STAT_STRENGTH     (VVCAM_ISP_CID_YNR_BASE + 0x000D)
+#define VVCAM_ISP_CID_YNR_STAT_STRENGTH2    (VVCAM_ISP_CID_YNR_BASE + 0x000E)
+
+#ifdef __KERNEL__
+int vvcam_isp_ynr_ctrl_count(void);
+int vvcam_isp_ynr_ctrl_create(struct vvcam_isp_dev *isp_dev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_driver.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_driver.c
new file mode 100644
index 000000000000..107cf344a61d
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_driver.c
@@ -0,0 +1,1787 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/of_graph.h>
+#include <linux/vmalloc.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-ctrls.h>
+#include <linux/i2c.h>
+#include <linux/delay.h>
+#include <media/v4l2-subdev.h>
+#include "vvcam_v4l2_common.h"
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_event.h"
+#include "vvcam_isp_ctrl.h"
+#include "vvcam_isp_procfs.h"
+#include "vvcam_v4l2_std_exts.h"
+#ifdef VVCAM_PLATFORM_REGISTER
+#include "vvcam_isp_platform.h"
+#endif
+
+#ifdef VVCAM_PLATFORM_REGISTER
+#define VVCAM_ISP_DEFAULT_SENSOR_LIB          "libos08a20.so"
+#define VVCAM_ISP_DEFAULT_SENSOR_ISI_SYM      "OS08a20_IsiCamDrvConfig"
+#define VVCAM_ISP_DEFAULT_SENSOR_MODE   0
+#define VVCAM_ISP_DEFAULT_SENSOR_XML    "OS08a20_8M_02_1080p.xml"
+#define VVCAM_ISP_DEFAULT_SENSOR_MANU_JSON    "vvbcfg/project_json_file/manual_ext.json"
+#define VVCAM_ISP_DEFAULT_SENSOR_AUTO_JSON    "vvbcfg/project_json_file/auto.json"
+#endif
+
+struct vvcam_isp_format vvcam_isp_mp_fmts[] = {
+    {
+        .fourcc    = V4L2_PIX_FMT_NV16,
+        .code      = MEDIA_BUS_FMT_YUYV8_2X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_NV12,
+        .code      = MEDIA_BUS_FMT_YUYV8_1_5X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_YUYV,
+        .code      = MEDIA_BUS_FMT_YUYV8_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR8,
+        .code      = MEDIA_BUS_FMT_SBGGR8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG8,
+        .code      = MEDIA_BUS_FMT_SGBRG8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG8,
+        .code      = MEDIA_BUS_FMT_SGRBG8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB8,
+        .code      = MEDIA_BUS_FMT_SRGGB8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10,
+        .code      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10,
+        .code      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10,
+        .code      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10,
+        .code      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12,
+        .code      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12,
+        .code      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12,
+        .code      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12,
+        .code      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P010,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_GREY,
+        .code      = MEDIA_BUS_FMT_Y8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10BPACK,
+        .code      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10DWA,
+        .code      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10,
+        .code      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P00BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P00DWA,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P02BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV12_2X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P20BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P20DWA,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P210,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P22BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV12_2X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I20BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I210,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_M48BPACK,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I48BPACK,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I48DWA,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I40DWA,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24,
+        .code      = MEDIA_BUS_FMT_RGB888_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24DWA,
+        .code      = MEDIA_BUS_FMT_RGB888_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24P,
+        .code      = MEDIA_BUS_FMT_RGB888_3X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10BPACK,
+        .code      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10BPACK,
+        .code      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10BPACK,
+        .code      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10BPACK,
+        .code      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10DWA,
+        .code      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10DWA,
+        .code      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10DWA,
+        .code      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10DWA,
+        .code      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12BPACK,
+        .code      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12BPACK,
+        .code      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12BPACK,
+        .code      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12BPACK,
+        .code      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12DWA,
+        .code      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12DWA,
+        .code      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12DWA,
+        .code      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12DWA,
+        .code      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14BPACK,
+        .code      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14BPACK,
+        .code      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14BPACK,
+        .code      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14BPACK,
+        .code      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14DWA,
+        .code      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14DWA,
+        .code      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14DWA,
+        .code      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14DWA,
+        .code      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14,
+        .code      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14,
+        .code      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14,
+        .code      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14,
+        .code      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR16,
+        .code      = MEDIA_BUS_FMT_SBGGR16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG16,
+        .code      = MEDIA_BUS_FMT_SGBRG16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG16,
+        .code      = MEDIA_BUS_FMT_SGRBG16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB16,
+        .code      = MEDIA_BUS_FMT_SRGGB16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR24,
+        .code      = MEDIA_BUS_FMT_SBGGR24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG24,
+        .code      = MEDIA_BUS_FMT_SGBRG24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG24,
+        .code      = MEDIA_BUS_FMT_SGRBG24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB24,
+        .code      = MEDIA_BUS_FMT_SRGGB24_1X24,
+    },
+};
+
+struct vvcam_isp_format vvcam_isp_sp_fmts[] = {
+    {
+        .fourcc    = V4L2_PIX_FMT_NV16,
+        .code      = MEDIA_BUS_FMT_YUYV8_2X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_NV12,
+        .code      = MEDIA_BUS_FMT_YUYV8_1_5X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_YUYV,
+        .code      = MEDIA_BUS_FMT_YUYV8_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P010,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_GREY,
+        .code      = MEDIA_BUS_FMT_Y8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10BPACK,
+        .code      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10DWA,
+        .code      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10,
+        .code      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P00BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P00DWA,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P02BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV12_2X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P20BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P20DWA,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P210,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P22BPACK,
+        .code      = MEDIA_BUS_FMT_YUYV12_2X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I210,
+        .code      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_M48BPACK,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I48BPACK,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I48DWA,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I40DWA,
+        .code      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24,
+        .code      = MEDIA_BUS_FMT_RGB888_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24DWA,
+        .code      = MEDIA_BUS_FMT_RGB888_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24P,
+        .code      = MEDIA_BUS_FMT_RGB888_3X8,
+    },
+};
+
+// main path
+struct vvcam_isp_format vvcam_isp_raw_fmts[] = {
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR8,
+        .code      = MEDIA_BUS_FMT_SBGGR8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG8,
+        .code      = MEDIA_BUS_FMT_SGBRG8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG8,
+        .code      = MEDIA_BUS_FMT_SGRBG8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB8,
+        .code      = MEDIA_BUS_FMT_SRGGB8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10,
+        .code      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10,
+        .code      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10,
+        .code      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10,
+        .code      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12,
+        .code      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12,
+        .code      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12,
+        .code      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12,
+        .code      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10BPACK,
+        .code      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10BPACK,
+        .code      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10BPACK,
+        .code      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10BPACK,
+        .code      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10DWA,
+        .code      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10DWA,
+        .code      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10DWA,
+        .code      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10DWA,
+        .code      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12BPACK,
+        .code      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12BPACK,
+        .code      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12BPACK,
+        .code      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12BPACK,
+        .code      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12DWA,
+        .code      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12DWA,
+        .code      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12DWA,
+        .code      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12DWA,
+        .code      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14BPACK,
+        .code      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14BPACK,
+        .code      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14BPACK,
+        .code      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14BPACK,
+        .code      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14DWA,
+        .code      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14DWA,
+        .code      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14DWA,
+        .code      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14DWA,
+        .code      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14,
+        .code      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14,
+        .code      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14,
+        .code      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14,
+        .code      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR16,
+        .code      = MEDIA_BUS_FMT_SBGGR16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG16,
+        .code      = MEDIA_BUS_FMT_SGBRG16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG16,
+        .code      = MEDIA_BUS_FMT_SGRBG16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB16,
+        .code      = MEDIA_BUS_FMT_SRGGB16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR24,
+        .code      = MEDIA_BUS_FMT_SBGGR24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG24,
+        .code      = MEDIA_BUS_FMT_SGBRG24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG24,
+        .code      = MEDIA_BUS_FMT_SGRBG24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB24,
+        .code      = MEDIA_BUS_FMT_SRGGB24_1X24,
+    },
+
+};
+
+static int vvcam_isp_querycap(struct v4l2_subdev *sd, void *arg)
+{
+	struct v4l2_capability *cap = (struct v4l2_capability *)arg;
+
+    strlcpy(cap->driver, sd->name, sizeof(cap->driver));
+	strlcpy(cap->card, sd->name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+            "platform:%s", sd->name);
+
+	return 0;
+}
+
+static int vvcam_isp_pad_requbufs(struct v4l2_subdev *sd, void *arg)
+{
+    struct vvcam_pad_reqbufs *pad_requbufs = (struct vvcam_pad_reqbufs *)arg;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+
+    return vvcam_isp_requebus_event(isp_dev, pad_requbufs->pad, pad_requbufs->num_buffers);
+}
+
+static int vvcam_isp_pad_buf_queue(struct v4l2_subdev *sd, void *arg)
+{
+    struct vvcam_pad_buf *pad_buf = (struct vvcam_pad_buf *)arg;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    int ret;
+    unsigned long flags;
+    struct vvcam_isp_pad_data *cur_pad;
+
+    cur_pad = &isp_dev->pad_data[pad_buf->pad];
+
+    spin_lock_irqsave(&cur_pad->qlock, flags);
+
+    list_add_tail(&pad_buf->buf->list, &cur_pad->queue);
+
+	spin_unlock_irqrestore(&cur_pad->qlock, flags);
+
+    ret = vvcam_isp_qbuf_event(isp_dev, pad_buf->pad, pad_buf->buf);
+
+    return ret;
+}
+// static struct v4l2_subdev *find_remote_sensor(struct media_entity *entity)
+// {
+//     struct v4l2_subdev *subdev;
+// 	struct media_pad *pad;
+
+// 	while (1) {
+// 		pad = &entity->pads[0];
+// 		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+// 			return NULL;
+
+// 		pad = media_pad_remote_pad_first(pad);
+// 		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+// 			return NULL;
+
+// 		entity = pad->entity;
+// 		if (entity->function == MEDIA_ENT_F_CAM_SENSOR) {
+//             subdev = media_entity_to_v4l2_subdev(entity);
+//             if (subdev) {
+//                 //printk("Found remote sensor subdev: %s\n", subdev->name);
+//                 return subdev;
+//             }
+//         }
+// 	}
+//     return NULL;
+// }
+static int vvcam_isp_pad_s_stream(struct v4l2_subdev *sd, void *arg)
+{
+    struct vvcam_pad_stream_status *pad_stream = (struct vvcam_pad_stream_status *)arg;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+
+    isp_dev->pad_data[pad_stream->pad].stream = pad_stream->status;
+
+    if (pad_stream->status == 0 ) {
+        INIT_LIST_HEAD(&isp_dev->pad_data[pad_stream->pad].queue);
+    }
+
+    return vvcam_isp_s_stream_event(isp_dev, pad_stream->pad, pad_stream->status);
+}
+
+static int vvcam_isp_buf_done(struct v4l2_subdev *sd, void *arg)
+{
+    struct vvcam_isp_buf ubuf;
+    struct vvcam_isp_pad_data *cur_pad;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    unsigned long flags;
+    struct vvcam_vb2_buffer *pos, *next;
+    struct vvcam_vb2_buffer *buf = NULL;
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct video_device *video;
+    struct vvcam_pad_buf pad_buf;
+    int ret;
+
+    memcpy(&ubuf, arg, sizeof(struct vvcam_isp_buf));
+    cur_pad = &isp_dev->pad_data[ubuf.pad];
+
+    if (list_empty(&cur_pad->queue) || (cur_pad->stream == 0))
+        return -EINVAL;
+    spin_lock_irqsave(&cur_pad->qlock, flags);
+    list_for_each_entry_safe(pos, next, &cur_pad->queue, list) {
+        if (pos && (pos->sequence == ubuf.index)) {
+           buf = pos;
+           list_del(&pos->list);
+           break;
+        }
+    }
+    spin_unlock_irqrestore(&cur_pad->qlock, flags);
+    if (buf) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&isp_dev->pads[ubuf.pad]);
+#else
+        pad = media_entity_remote_pad(&isp_dev->pads[ubuf.pad]);
+#endif
+        if (!pad)
+            return -EINVAL;
+        if (is_media_entity_v4l2_subdev(pad->entity)) {
+            subdev = media_entity_to_v4l2_subdev(pad->entity);
+            memset(&pad_buf, 0, sizeof(pad_buf));
+            pad_buf.pad = pad->index;
+            pad_buf.buf = buf;
+            ret = v4l2_subdev_call(subdev, core, ioctl, VVCAM_PAD_BUF_DONE, &pad_buf);
+            if (ret)
+                return ret;
+        } else if (is_media_entity_v4l2_video_device(pad->entity)){
+            video = media_entity_to_video_device(pad->entity);
+            if (buf->sequence < video->queue->num_buffers) {
+                if (buf->vb.vb2_buf.state == VB2_BUF_STATE_ACTIVE) {
+                    vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+                    //printk("vb done addr = 0x%x\n", buf->vb.vb2_buf.planes[0].dma_addr + buf->vb.vb2_buf.planes[0].offset);
+                }
+            }
+        }
+    }
+
+    return 0;
+}
+
+static int vvcam_isp_queryctrl(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_queryctrl *pad_querctrl =
+                                    (struct vvcam_pad_queryctrl *)arg;
+    ret = v4l2_queryctrl(&isp_dev->ctrl_handler, pad_querctrl->query_ctrl);
+
+    return ret;
+}
+
+static int vvcam_isp_query_ext_ctrl(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_query_ext_ctrl *pad_quer_ext_ctrl =
+                                    (struct vvcam_pad_query_ext_ctrl *)arg;
+    ret = v4l2_query_ext_ctrl(&isp_dev->ctrl_handler,
+                        pad_quer_ext_ctrl->query_ext_ctrl);
+
+    return ret;
+}
+
+static int vvcam_isp_querymenu(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_querymenu *pad_quermenu =
+                                    (struct vvcam_pad_querymenu *)arg;
+    ret = v4l2_querymenu(&isp_dev->ctrl_handler,
+                        pad_quermenu->querymenu);
+
+    return ret;
+}
+
+static int vvcam_isp_g_ctrl(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_control *pad_ctrl = (struct vvcam_pad_control *)arg;
+
+    mutex_lock(&isp_dev->ctrl_lock);
+    isp_dev->ctrl_pad = pad_ctrl->pad;
+    ret = v4l2_g_ctrl(&isp_dev->ctrl_handler, pad_ctrl->control);
+    mutex_unlock(&isp_dev->ctrl_lock);
+
+    return ret;
+}
+
+static int vvcam_isp_s_ctrl(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_control *pad_ctrl = (struct vvcam_pad_control *)arg;
+
+    mutex_lock(&isp_dev->ctrl_lock);
+    isp_dev->ctrl_pad = pad_ctrl->pad;
+    ret = v4l2_s_ctrl(NULL, &isp_dev->ctrl_handler, pad_ctrl->control);
+    mutex_unlock(&isp_dev->ctrl_lock);
+
+    return ret;
+}
+
+static int vvcam_isp_g_ext_ctrls(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_ext_controls *pad_ext_ctrls =
+                            (struct vvcam_pad_ext_controls *)arg;
+
+    mutex_lock(&isp_dev->ctrl_lock);
+    isp_dev->ctrl_pad = pad_ext_ctrls->pad;
+    ret = v4l2_g_ext_ctrls(&isp_dev->ctrl_handler, sd->devnode,
+                            sd->v4l2_dev->mdev,
+                            pad_ext_ctrls->ext_controls);
+    mutex_unlock(&isp_dev->ctrl_lock);
+
+    return ret;
+}
+
+static int vvcam_isp_s_ext_ctrls(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_ext_controls *pad_ext_ctrls =
+                            (struct vvcam_pad_ext_controls *)arg;
+
+    mutex_lock(&isp_dev->ctrl_lock);
+    isp_dev->ctrl_pad = pad_ext_ctrls->pad;
+    ret = v4l2_s_ext_ctrls(NULL, &isp_dev->ctrl_handler, sd->devnode,
+                            sd->v4l2_dev->mdev,
+                            pad_ext_ctrls->ext_controls);
+    mutex_unlock(&isp_dev->ctrl_lock);
+
+    return ret;
+}
+
+static int vvcam_isp_try_ext_ctrls(struct v4l2_subdev *sd,void *arg)
+{
+    int ret;
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_pad_ext_controls *pad_ext_ctrls =
+                            (struct vvcam_pad_ext_controls *)arg;
+    ret = v4l2_try_ext_ctrls(&isp_dev->ctrl_handler, sd->devnode,
+                            sd->v4l2_dev->mdev,
+                            pad_ext_ctrls->ext_controls);
+
+    return ret;
+}
+
+static long vvcam_isp_priv_ioctl(struct v4l2_subdev *sd,
+                                unsigned int cmd, void *arg)
+{
+    int ret = -EINVAL;
+    switch (cmd) {
+        case VIDIOC_QUERYCAP:
+            ret = vvcam_isp_querycap(sd, arg);
+            break;
+        case VVCAM_PAD_REQUBUFS:
+            ret = vvcam_isp_pad_requbufs(sd, arg);
+            break;
+        case VVCAM_PAD_BUF_QUEUE:
+            ret = vvcam_isp_pad_buf_queue(sd, arg);
+            break;
+        case VVCAM_PAD_S_STREAM:
+            ret = vvcam_isp_pad_s_stream(sd, arg);
+            break;
+        case VVCAM_ISP_IOC_BUFDONE:
+            ret = vvcam_isp_buf_done(sd, arg);
+            break;
+        case VVCAM_PAD_QUERYCTRL:
+            ret = vvcam_isp_queryctrl(sd, arg);
+            break;
+        case VVCAM_PAD_QUERY_EXT_CTRL:
+            ret = vvcam_isp_query_ext_ctrl(sd, arg);
+            break;
+        case VVCAM_PAD_G_CTRL:
+            ret = vvcam_isp_g_ctrl(sd, arg);
+            break;
+        case VVCAM_PAD_S_CTRL:
+            ret = vvcam_isp_s_ctrl(sd, arg);
+            break;
+        case VVCAM_PAD_G_EXT_CTRLS:
+            ret = vvcam_isp_g_ext_ctrls(sd, arg);
+            break;
+        case VVCAM_PAD_S_EXT_CTRLS:
+            ret = vvcam_isp_s_ext_ctrls(sd, arg);
+            break;
+        case VVCAM_PAD_TRY_EXT_CTRLS:
+            ret = vvcam_isp_try_ext_ctrls(sd, arg);
+            break;
+        case VVCAM_PAD_QUERYMENU:
+            ret = vvcam_isp_querymenu(sd, arg);
+            break;
+        default:
+            break;
+    }
+
+    return ret;
+}
+
+int vvcam_isp_subscribe_event(struct v4l2_subdev *sd,
+			                struct v4l2_fh *fh,
+			                struct v4l2_event_subscription *sub)
+{
+    switch (sub->type) {
+        case V4L2_EVENT_CTRL:
+            return v4l2_ctrl_subdev_subscribe_event(sd, fh, sub);
+        case VVCAM_ISP_DEAMON_EVENT:
+            return v4l2_event_subscribe(fh, sub, 2, NULL);
+        default:
+            return -EINVAL;
+    }
+
+}
+
+int vvcam_isp_s_stream(struct v4l2_subdev *sd, int enable)
+{
+    return 0;
+}
+
+static struct v4l2_subdev_core_ops vvcam_isp_core_ops = {
+	.ioctl             = vvcam_isp_priv_ioctl,
+	.subscribe_event   = vvcam_isp_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static struct v4l2_subdev_video_ops vvcam_isp_video_ops = {
+	.s_stream = vvcam_isp_s_stream,
+};
+
+static int vvcam_isp_set_fmt(struct v4l2_subdev *sd,
+			struct v4l2_subdev_state *sd_state,
+			struct v4l2_subdev_format *format)
+{
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    uint32_t w, h;
+    uint32_t sink_pad_index;
+    struct vvcam_isp_pad_data *cur_pad = &isp_dev->pad_data[format->pad];
+    struct vvcam_isp_pad_data *sink_pad;
+    struct vvcam_isp_pad_data *source_pad;
+    int i;
+    int ret;
+    uint32_t fourcc_code = 0;
+
+    sink_pad_index = format->pad - (format->pad % VVCAM_ISP_PORT_PAD_NR);
+    sink_pad = &isp_dev->pad_data[sink_pad_index];
+
+    if (sink_pad == cur_pad) {
+        cur_pad->sink_detected = 1;
+        cur_pad->format = format->format;
+        for (i = 1; i < VVCAM_ISP_PORT_PAD_NR; i++) {
+            source_pad = &isp_dev->pad_data[sink_pad_index + i];
+            source_pad->sink_detected = 1;
+            source_pad->crop.left = 0;
+            source_pad->crop.top = 0;
+            source_pad->crop.width = format->format.width;
+            source_pad->crop.height = format->format.height;
+            source_pad->compose.left = 0;
+            source_pad->compose.top = 0;
+            source_pad->compose.width = format->format.width;
+            source_pad->compose.height = format->format.height;
+
+            switch (i) {
+                case VVCAM_ISP_PORT_PAD_SOURCE_MP:
+                case VVCAM_ISP_PORT_PAD_SOURCE_SP1:
+                case VVCAM_ISP_PORT_PAD_SOURCE_SP2:
+                case VVCAM_ISP_PORT_PAD_SOURCE_RAW:
+                    source_pad->format = format->format;
+                    source_pad->format.code = source_pad->fmts[0].code;
+                    memcpy(source_pad->format.reserved, &source_pad->fmts[0].fourcc, sizeof(uint32_t));
+                    source_pad->format.field = V4L2_FIELD_NONE;
+                    source_pad->format.quantization = V4L2_QUANTIZATION_DEFAULT;
+                    source_pad->format.colorspace = V4L2_COLORSPACE_DEFAULT;
+                    break;
+                default:
+                    break;
+            }
+        }
+        return 0;
+    }
+
+    w = ALIGN(format->format.width, VVCAM_ISP_WIDTH_ALIGN);
+    h = ALIGN(format->format.height, VVCAM_ISP_HEIGHT_ALIGN);
+    w = clamp_t(uint32_t, w, VVCAM_ISP_WIDTH_MIN, sink_pad->format.width);
+    h = clamp_t(uint32_t, h, VVCAM_ISP_HEIGHT_MIN, sink_pad->format.height);
+
+    format->format.width = w;
+    format->format.height = h;
+
+    memcpy(&fourcc_code, format->format.reserved, sizeof(uint32_t));
+
+    for (i = 0; i < cur_pad->num_formats; i++) {
+        if (format->format.code == cur_pad->fmts[i].code && fourcc_code == cur_pad->fmts[i].fourcc) {
+            break;
+        }
+    }
+
+    if (i >= cur_pad->num_formats) {
+        format->format.code = cur_pad->fmts[0].code;
+        memcpy(format->format.reserved, &cur_pad->fmts[0].fourcc, sizeof(uint32_t));
+    }
+
+    ret = vvcam_isp_set_fmt_event(isp_dev, format->pad, &format->format);
+    if (ret) {
+        printk("isp set fmt event failed %d\n", ret);
+        return ret;
+    }
+
+
+    cur_pad->compose.left = 0;
+    cur_pad->compose.top = 0;
+    cur_pad->compose.width = w;
+    cur_pad->compose.height = h;
+
+    cur_pad->format = format->format;
+
+    return 0;
+}
+
+static int vvcam_isp_get_fmt(struct v4l2_subdev *sd,
+			struct v4l2_subdev_state *sd_state,
+			struct v4l2_subdev_format *format)
+{
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_isp_pad_data *pad_data = &isp_dev->pad_data[format->pad];
+
+    if (pad_data->sink_detected) {
+        format->format = pad_data->format;
+    } else {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int vvcam_isp_enum_mbus_code(struct v4l2_subdev *sd,
+			struct v4l2_subdev_state *sd_state,
+            struct v4l2_subdev_mbus_code_enum *code)
+{
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_isp_pad_data *pad_data = &isp_dev->pad_data[code->pad];
+
+    if (code->index >= pad_data->num_formats) {
+        return -EINVAL;
+    }
+
+    code->code = pad_data->fmts[code->index].code;
+    code->reserved[0] = pad_data->fmts[code->index].fourcc;
+
+    return 0;
+}
+
+static int vvcam_isp_set_selection(struct v4l2_subdev *sd,
+            struct v4l2_subdev_state *sd_state,
+            struct v4l2_subdev_selection *sd_sel)
+{
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_isp_pad_data *pad_data = &isp_dev->pad_data[sd_sel->pad];
+    struct vvcam_isp_pad_data *sink_pad;
+    struct vvcam_isp_selection isp_sel;
+    uint32_t sink_pad_index;
+    int ret = -EINVAL;
+
+    sink_pad_index = sd_sel->pad - (sd_sel->pad % VVCAM_ISP_PORT_PAD_NR);
+    sink_pad = &isp_dev->pad_data[sink_pad_index];
+
+    sd_sel->r.width = ALIGN(sd_sel->r.width, VVCAM_ISP_WIDTH_ALIGN);
+    sd_sel->r.height = ALIGN(sd_sel->r.height, VVCAM_ISP_HEIGHT_ALIGN);
+
+    switch (sd_sel->target)
+    {
+        case V4L2_SEL_TGT_CROP:
+            if (((sd_sel->r.left + sd_sel->r.width) < VVCAM_ISP_WIDTH_MIN) ||
+                ((sd_sel->r.left + sd_sel->r.width) > sink_pad->format.width) ||
+                ((sd_sel->r.top + sd_sel->r.height) < VVCAM_ISP_HEIGHT_MIN) ||
+                ((sd_sel->r.top + sd_sel->r.height) > sink_pad->format.height)) {
+                return -EINVAL;
+            }
+            break;
+
+        case V4L2_SEL_TGT_COMPOSE:
+            if (((sd_sel->r.left + sd_sel->r.width) < VVCAM_ISP_WIDTH_MIN) ||
+                ((sd_sel->r.top + sd_sel->r.height) < VVCAM_ISP_HEIGHT_MIN)) {
+                return -EINVAL;
+            }
+            break;
+
+        default:
+            return -EINVAL;
+    }
+
+    isp_sel.rect = sd_sel->r;
+    if (sd_sel->target == V4L2_SEL_TGT_CROP) {
+        isp_sel.type = VVCAM_ISP_SEL_TYPE_CROP;
+    } else {
+        isp_sel.type = VVCAM_ISP_SEL_TYPE_COMPOSE;
+    }
+
+    ret = vvcam_isp_s_selection_event(isp_dev, sd_sel->pad, &isp_sel);
+    if (ret == 0) {
+        if (sd_sel->target == V4L2_SEL_TGT_CROP) {
+            pad_data->crop = isp_sel.rect;
+        } else {
+            pad_data->compose = isp_sel.rect;
+        }
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_get_selection(struct v4l2_subdev *sd,
+            struct v4l2_subdev_state *sd_state,
+            struct v4l2_subdev_selection *sel)
+{
+    struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+    struct vvcam_isp_pad_data *pad_data = &isp_dev->pad_data[sel->pad];
+    struct vvcam_isp_pad_data *sink_pad;
+    uint32_t sink_pad_index;
+
+    sink_pad_index = sel->pad - (sel->pad % VVCAM_ISP_PORT_PAD_NR);
+    sink_pad = &isp_dev->pad_data[sink_pad_index];
+
+    switch (sel->target)
+    {
+        case V4L2_SEL_TGT_CROP_DEFAULT:
+        case V4L2_SEL_TGT_CROP_BOUNDS:
+            sel->r.left = 0;
+            sel->r.top = 0;
+            sel->r.width = sink_pad->format.width;
+            sel->r.height = sink_pad->format.height;
+            break;
+
+        case V4L2_SEL_TGT_CROP:
+            sel->r = pad_data->crop;
+            break;
+
+        case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+        case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+            sel->r.left = pad_data->crop.left;
+            sel->r.top = pad_data->crop.top;
+            sel->r.width = pad_data->crop.width;
+            sel->r.height = pad_data->crop.height;
+            break;
+
+        case V4L2_SEL_TGT_COMPOSE:
+            sel->r = pad_data->compose;
+            break;
+
+        default:
+            return -EINVAL;
+            break;
+    }
+    return 0;
+}
+
+static const struct v4l2_subdev_pad_ops vvcam_isp_pad_ops = {
+	.set_fmt        = vvcam_isp_set_fmt,
+    .get_fmt        = vvcam_isp_get_fmt,
+    .enum_mbus_code = vvcam_isp_enum_mbus_code,
+    .set_selection  = vvcam_isp_set_selection,
+    .get_selection  = vvcam_isp_get_selection,
+};
+
+struct v4l2_subdev_ops vvcam_isp_subdev_ops = {
+	.core  = &vvcam_isp_core_ops,
+	.video = &vvcam_isp_video_ops,
+	.pad   = &vvcam_isp_pad_ops,
+};
+
+static int vvcam_isp_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+
+	mutex_lock(&isp_dev->mlock);
+	isp_dev->refcnt++;
+	pm_runtime_get_sync(sd->dev);
+
+	mutex_unlock(&isp_dev->mlock);
+	return 0;
+}
+
+static int vvcam_isp_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct vvcam_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
+
+	mutex_lock(&isp_dev->mlock);
+	isp_dev->refcnt--;
+	pm_runtime_put_sync(sd->dev);
+
+	mutex_unlock(&isp_dev->mlock);
+
+	return 0;
+}
+
+
+static struct v4l2_subdev_internal_ops vvcam_isp_internal_ops = {
+	.open  = vvcam_isp_open,
+	.close = vvcam_isp_close,
+};
+
+static int vvcam_isp_link_setup(struct media_entity *entity,
+		const struct media_pad *local,
+		const struct media_pad *remote, u32 flags)
+{
+    printk(KERN_ERR "%s %d\n", __func__, __LINE__);
+	return 0;
+}
+
+static const struct media_entity_operations vvcam_isp_entity_ops = {
+	.link_setup     = vvcam_isp_link_setup,
+	.link_validate  = v4l2_subdev_link_validate,
+	.get_fwnode_pad = v4l2_subdev_get_fwnode_pad_1_to_1,
+
+};
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+static int vvcam_isp_notifier_bound(struct v4l2_async_notifier *notifier,
+		                            struct v4l2_subdev *sd,
+		                            struct v4l2_async_connection *asc)
+#else
+static int vvcam_isp_notifier_bound(struct v4l2_async_notifier *notifier,
+		                            struct v4l2_subdev *sd,
+		                            struct v4l2_async_subdev *asd)
+#endif
+{
+    int ret = 0;
+    struct vvcam_isp_dev *isp_dev = container_of(notifier,
+			struct vvcam_isp_dev, notifier);
+    struct device *dev =  isp_dev->dev;
+
+    struct fwnode_handle *ep = NULL;
+	struct v4l2_fwnode_link link;
+	struct media_entity *source, *sink;
+	unsigned int source_pad, sink_pad;
+
+    while(1) {
+        ep = fwnode_graph_get_next_endpoint(sd->fwnode, ep);
+        if (!ep)
+			break;
+
+        ret = v4l2_fwnode_parse_link(ep, &link);
+		if (ret < 0) {
+			dev_err(dev, "failed to parse link for %pOF: %d\n",
+                    to_of_node(ep), ret);
+			continue;
+		}
+
+        if (sd->entity.pads[link.local_port].flags == MEDIA_PAD_FL_SINK)
+			continue;
+
+        source     = &sd->entity;
+		source_pad = 1;//link.remote_port; // 1
+		sink       = &isp_dev->sd.entity;
+		sink_pad   = 0;//link.local_port; //0
+		v4l2_fwnode_put_link(&link);
+		ret = media_create_pad_link(source, source_pad,
+				sink, sink_pad, MEDIA_LNK_FL_ENABLED);
+        if (ret) {
+			dev_err(dev, "failed to create %s:%u -> %s:%u flags = %ld link\n",
+				source->name, source_pad,
+				sink->name, sink_pad, sd->entity.pads[source_pad].flags);
+			break;
+		}
+    }
+
+    fwnode_handle_put(ep);
+
+	return ret;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+static void vvcam_isp_notifier_unbound(struct v4l2_async_notifier *notifier,
+		                            struct v4l2_subdev *sd,
+		                            struct v4l2_async_connection *asc)
+{
+	return;
+}
+#else
+static void vvcam_isp_notifier_unbound(struct v4l2_async_notifier *notifier,
+		                            struct v4l2_subdev *sd,
+		                            struct v4l2_async_subdev *asd)
+{
+	return;
+}
+#endif
+
+static const struct v4l2_async_notifier_operations vvcam_isp_notify_ops = {
+	.bound    = vvcam_isp_notifier_bound,
+	.unbind   = vvcam_isp_notifier_unbound,
+};
+
+static int vvcam_isp_async_notifier(struct vvcam_isp_dev *isp_dev)
+{
+    struct fwnode_handle *ep;
+    struct fwnode_handle *remote_ep;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    struct v4l2_async_connection *asc;
+#else
+    struct v4l2_async_subdev *asd;
+#endif
+    struct device *dev = isp_dev->dev;
+    int ret = 0;
+    int pad = 0;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    v4l2_async_subdev_nf_init(&isp_dev->notifier, &isp_dev->sd);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    v4l2_async_nf_init(&isp_dev->notifier);
+#else
+    v4l2_async_notifier_init(&isp_dev->notifier);
+#endif
+
+    isp_dev->notifier.ops = &vvcam_isp_notify_ops;
+
+    if (dev_fwnode(isp_dev->dev) == NULL)
+        return 0;
+
+    for (pad = 0; pad < VVCAM_ISP_PAD_NR; pad++) {
+
+        //pr_info("%s, %d pad:%d, isp_dev->pads[pad].flags :%ld\n", __func__, __LINE__, pad, isp_dev->pads[pad].flags);
+
+        if (isp_dev->pads[pad].flags != MEDIA_PAD_FL_SINK)
+            continue;
+
+        //pr_info("%s, %d pad:%d, subdev name = %s, fwnode name = %s\n", __func__, __LINE__, pad, isp_dev->sd.name, fwnode_get_name(dev_fwnode(dev)));
+
+        ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev),
+                                        pad, 0, FWNODE_GRAPH_ENDPOINT_NEXT);
+        if (!ep)
+            continue;
+
+        remote_ep = fwnode_graph_get_remote_endpoint(ep);
+        if (!remote_ep) {
+            fwnode_handle_put(ep);
+            continue;
+        }
+		pr_info("%s, %d remote_ep:%s\n", __func__, __LINE__,
+		       fwnode_get_name(remote_ep));
+
+		pr_info("%s, %d ep:%s\n", __func__, __LINE__,
+		       fwnode_get_name(ep));
+
+        fwnode_handle_put(remote_ep);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+        asc = v4l2_async_nf_add_fwnode_remote(&isp_dev->notifier,
+                                            ep, struct v4l2_async_connection);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+        asd = v4l2_async_nf_add_fwnode_remote(&isp_dev->notifier,
+                                            ep, struct v4l2_async_subdev);
+#else
+        asd = v4l2_async_notifier_add_fwnode_remote_subdev(&isp_dev->notifier,
+                                                ep, struct v4l2_async_subdev);
+#endif
+
+        fwnode_handle_put(ep);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+        if (IS_ERR(asc)) {
+            ret = PTR_ERR(asc);
+#else
+        if (IS_ERR(asd)) {
+            ret = PTR_ERR(asd);
+#endif
+			if (ret != -EEXIST) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+                v4l2_async_nf_cleanup(&isp_dev->notifier);
+#else
+				v4l2_async_notifier_cleanup(&isp_dev->notifier);
+#endif
+				return ret;
+			}
+        }
+    }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    ret = v4l2_async_nf_register(&isp_dev->notifier);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    ret = v4l2_async_subdev_nf_register(&isp_dev->sd,
+						  &isp_dev->notifier);
+#else
+    ret = v4l2_async_subdev_notifier_register(&isp_dev->sd,
+						  &isp_dev->notifier);
+#endif
+    if (ret) {
+        dev_err(isp_dev->dev, "Async notifier register error\n");
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+        v4l2_async_nf_cleanup(&isp_dev->notifier);
+#else
+		v4l2_async_notifier_cleanup(&isp_dev->notifier);
+#endif
+    }
+
+    return ret;
+}
+
+static int vvcam_isp_pads_init(struct vvcam_isp_dev *isp_dev)
+{
+    int pad = 0;
+
+    for (pad = 0; pad < VVCAM_ISP_PAD_NR; pad++) {
+        if ((pad % VVCAM_ISP_PORT_PAD_NR) == VVCAM_ISP_PORT_PAD_SINK) {
+            isp_dev->pads[pad].flags = MEDIA_PAD_FL_SINK;
+        } else {
+            isp_dev->pads[pad].flags = MEDIA_PAD_FL_SOURCE;
+        }
+
+        switch (pad % VVCAM_ISP_PORT_PAD_NR) {
+            case VVCAM_ISP_PORT_PAD_SINK:
+                break;
+            case VVCAM_ISP_PORT_PAD_SOURCE_MP:
+                isp_dev->pad_data[pad].num_formats = ARRAY_SIZE(vvcam_isp_mp_fmts);
+                isp_dev->pad_data[pad].fmts = vvcam_isp_mp_fmts;
+                break;
+            case VVCAM_ISP_PORT_PAD_SOURCE_SP1:
+                isp_dev->pad_data[pad].num_formats = ARRAY_SIZE(vvcam_isp_sp_fmts);
+                isp_dev->pad_data[pad].fmts = vvcam_isp_sp_fmts;
+                break;
+            case VVCAM_ISP_PORT_PAD_SOURCE_SP2:
+                isp_dev->pad_data[pad].num_formats = ARRAY_SIZE(vvcam_isp_sp_fmts);
+                isp_dev->pad_data[pad].fmts = vvcam_isp_sp_fmts;
+                break;
+            case VVCAM_ISP_PORT_PAD_SOURCE_RAW:
+                isp_dev->pad_data[pad].num_formats = ARRAY_SIZE(vvcam_isp_raw_fmts);
+                isp_dev->pad_data[pad].fmts = vvcam_isp_raw_fmts;
+                break;
+            default:
+                break;
+        }
+
+        INIT_LIST_HEAD(&isp_dev->pad_data[pad].queue);
+        spin_lock_init(&isp_dev->pad_data[pad].qlock);
+    }
+
+    return 0;
+}
+
+static int vvcam_isp_parse_params(struct vvcam_isp_dev *isp_dev,
+                        struct platform_device *pdev)
+{
+
+#ifdef VVCAM_PLATFORM_REGISTER
+    int port = 0;
+    isp_dev->id  = pdev->id;
+    for (port = 0; port < VVCAM_ISP_PORT_NR; port++) {
+        strncpy(isp_dev->sensor_info[port].lib, VVCAM_ISP_DEFAULT_SENSOR_LIB,
+            strlen(VVCAM_ISP_DEFAULT_SENSOR_LIB));
+        strncpy(isp_dev->sensor_info[port].isi_sym, VVCAM_ISP_DEFAULT_SENSOR_ISI_SYM,
+            strlen(VVCAM_ISP_DEFAULT_SENSOR_ISI_SYM));
+        strncpy(isp_dev->sensor_info[port].xml, VVCAM_ISP_DEFAULT_SENSOR_XML,
+            strlen(VVCAM_ISP_DEFAULT_SENSOR_XML));
+        isp_dev->sensor_info[port].mode = VVCAM_ISP_DEFAULT_SENSOR_MODE;
+        strncpy(isp_dev->sensor_info[port].manu_json, VVCAM_ISP_DEFAULT_SENSOR_MANU_JSON,
+            strlen(VVCAM_ISP_DEFAULT_SENSOR_MANU_JSON));
+        strncpy(isp_dev->sensor_info[port].auto_json, VVCAM_ISP_DEFAULT_SENSOR_AUTO_JSON,
+            strlen(VVCAM_ISP_DEFAULT_SENSOR_AUTO_JSON));
+    }
+#else
+    fwnode_property_read_u32(of_fwnode_handle(pdev->dev.of_node),
+			"id", &isp_dev->id);
+#endif
+    return 0;
+}
+
+#ifdef VVCAM_PLATFORM_REGISTER
+struct v4l2_subdev *g_vvcam_isp_subdev[VVCAM_ISP_DEV_MAX] = {NULL};
+EXPORT_SYMBOL_GPL(g_vvcam_isp_subdev);
+#endif
+
+static int vvcam_isp_probe(struct platform_device *pdev)
+{
+    struct device *dev = &pdev->dev;
+    struct vvcam_isp_dev *isp_dev;
+    struct device_node *sensor_np;
+    struct device *sensor_dev;
+    struct v4l2_subdev *sd;
+    int ret;
+    int wait_time = 0;
+
+    sensor_np = of_parse_phandle(dev->of_node, "depends-on", 0);
+    if (sensor_np)
+    {
+        while (wait_time < 20000) {
+            sensor_dev = bus_find_device_by_of_node(&i2c_bus_type, sensor_np);
+            of_node_put(sensor_np);
+            if (sensor_dev) {
+                sd = dev_get_drvdata(sensor_dev);
+                put_device(sensor_dev);
+                if (sd && sd->v4l2_dev) {
+                    dev_info(dev, "Sensor is ready.\n");
+                    break;
+                }
+            }
+            msleep(1000);
+            wait_time += 1000;
+        }
+    }
+
+    isp_dev = devm_kzalloc(&pdev->dev,
+		        sizeof(struct vvcam_isp_dev), GFP_KERNEL);
+    if (!isp_dev)
+        return -ENOMEM;
+
+    mutex_init(&isp_dev->mlock);
+    mutex_init(&isp_dev->ctrl_lock);
+    isp_dev->dev = &pdev->dev;
+    platform_set_drvdata(pdev, isp_dev);
+
+    ret = vvcam_isp_parse_params(isp_dev, pdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to parse params\n");
+        return -EINVAL;
+    }
+
+    v4l2_subdev_init(&isp_dev->sd, &vvcam_isp_subdev_ops);
+	snprintf(isp_dev->sd.name, V4L2_SUBDEV_NAME_SIZE,
+		"%s.%d",VVCAM_ISP_NAME, isp_dev->id);
+
+    isp_dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+    isp_dev->sd.flags |= V4L2_SUBDEV_FL_HAS_EVENTS;
+    isp_dev->sd.dev =  &pdev->dev;
+    isp_dev->sd.owner = THIS_MODULE;
+    isp_dev->sd.internal_ops = &vvcam_isp_internal_ops;
+    isp_dev->sd.entity.ops = &vvcam_isp_entity_ops;
+    isp_dev->sd.entity.function = MEDIA_ENT_F_IO_V4L;
+    isp_dev->sd.entity.obj_type = MEDIA_ENTITY_TYPE_V4L2_SUBDEV;
+    isp_dev->sd.entity.name = isp_dev->sd.name;
+    v4l2_set_subdevdata(&isp_dev->sd, isp_dev);
+    vvcam_isp_pads_init(isp_dev);
+    ret = media_entity_pads_init(&isp_dev->sd.entity,
+                                VVCAM_ISP_PAD_NR, isp_dev->pads);
+    if (ret)
+       return ret;
+
+    ret = vvcam_isp_async_notifier(isp_dev);
+    if (ret)
+        goto err_async_notifier;
+#ifdef VVCAM_PLATFORM_REGISTER
+	isp_dev->sd.fwnode = &isp_dev->fwnode;
+    g_vvcam_isp_subdev[isp_dev->id] = &isp_dev->sd;
+#endif
+    ret = v4l2_async_register_subdev(&isp_dev->sd);
+    if (ret) {
+		dev_err(dev, "register subdev error\n");
+		goto error_regiter_subdev;
+	}
+
+    ret = vvcam_v4l2_isp_procfs_register(isp_dev, &isp_dev->pde);
+    if (ret) {
+        dev_err(dev, "register procfs failed.\n");
+        goto err_register_procfs;
+    }
+
+    printk("isp media device register ok, entity name = %s, pad num = %d major = %d, minor = %d\n", 
+    isp_dev->sd.entity.name, VVCAM_ISP_PAD_NR, isp_dev->sd.entity.info.dev.major, isp_dev->sd.entity.info.dev.minor);
+
+    isp_dev->event_shm.virt_addr = (void *)__get_free_pages(GFP_KERNEL, 3);
+    isp_dev->event_shm.size = PAGE_SIZE * 8;
+    memset(isp_dev->event_shm.virt_addr, 0, isp_dev->event_shm.size);
+    isp_dev->event_shm.phy_addr = virt_to_phys(isp_dev->event_shm.virt_addr);
+    mutex_init(&isp_dev->event_shm.event_lock);
+
+    pm_runtime_enable(&pdev->dev);
+    vvcam_isp_ctrl_init(isp_dev);
+
+    dev_info(&pdev->dev, "vvcam isp subdev driver probe success\n");
+
+    return 0;
+
+err_register_procfs:
+    v4l2_async_unregister_subdev(&isp_dev->sd);
+
+error_regiter_subdev:
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    v4l2_async_nf_cleanup(&isp_dev->notifier);
+    v4l2_async_nf_unregister(&isp_dev->notifier);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    v4l2_async_nf_unregister(&isp_dev->notifier);
+    v4l2_async_nf_cleanup(&isp_dev->notifier);
+#else
+    v4l2_async_notifier_unregister(&isp_dev->notifier);
+    v4l2_async_notifier_cleanup(&isp_dev->notifier);
+#endif
+err_async_notifier:
+    media_entity_cleanup(&isp_dev->sd.entity);
+
+    return ret;
+}
+
+static int vvcam_isp_remove(struct platform_device *pdev)
+{
+    struct vvcam_isp_dev *isp_dev;
+
+    isp_dev = platform_get_drvdata(pdev);
+
+    vvcam_v4l2_isp_procfs_unregister(isp_dev->pde);
+    v4l2_async_unregister_subdev(&isp_dev->sd);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    v4l2_async_nf_cleanup(&isp_dev->notifier);
+    v4l2_async_nf_unregister(&isp_dev->notifier);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    v4l2_async_nf_unregister(&isp_dev->notifier);
+    v4l2_async_nf_cleanup(&isp_dev->notifier);
+#else
+    v4l2_async_notifier_unregister(&isp_dev->notifier);
+    v4l2_async_notifier_cleanup(&isp_dev->notifier);
+#endif
+    media_entity_cleanup(&isp_dev->sd.entity);
+    pm_runtime_disable(&pdev->dev);
+    free_pages((unsigned long)isp_dev->event_shm.virt_addr, 3);
+    vvcam_isp_ctrl_destroy(isp_dev);
+    dev_info(&pdev->dev, "vvcam isp driver remove\n");
+
+    return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int vvcam_isp_system_suspend(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_suspend(dev);
+    if (ret) {
+        dev_err(dev, "force suspend %s failed\n", dev_name(dev));
+        return ret;
+    }
+	return ret;
+}
+
+static int vvcam_isp_system_resume(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_resume(dev);
+    if (ret) {
+        dev_err(dev, "force resume %s failed\n", dev_name(dev));
+        return ret;
+    }
+	return ret;
+}
+
+static int vvcam_isp_runtime_suspend(struct device *dev)
+{
+    return 0;
+}
+
+static int vvcam_isp_runtime_resume(struct device *dev)
+{
+    return 0;
+}
+
+static const struct dev_pm_ops vvcam_isp_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(vvcam_isp_system_suspend, vvcam_isp_system_resume)
+	SET_RUNTIME_PM_OPS(vvcam_isp_runtime_suspend, vvcam_isp_runtime_resume, NULL)
+};
+#endif
+
+static const struct of_device_id vvcam_isp_of_match[] = {
+	{.compatible = "verisilicon,isp-v4l2",},
+	{ /* sentinel */ },
+};
+
+static struct platform_driver vvcam_isp_driver = {
+	.probe  = vvcam_isp_probe,
+	.remove = vvcam_isp_remove,
+	.driver = {
+		.name           = VVCAM_ISP_NAME,
+		.owner          = THIS_MODULE,
+        .of_match_table = vvcam_isp_of_match,
+#ifdef CONFIG_PM_SLEEP
+        .pm             = &vvcam_isp_pm_ops,
+#endif
+	}
+};
+
+MODULE_DEVICE_TABLE(of, vvcam_isp_of_match);
+
+static int __init vvcam_isp_init_module(void)
+{
+    int ret;
+    ret = platform_driver_register(&vvcam_isp_driver);
+    if (ret) {
+        printk(KERN_ERR "Failed to register isp driver\n");
+        return ret;
+    }
+
+#ifdef VVCAM_PLATFORM_REGISTER
+    printk(KERN_ERR "%s %d\n", __func__, __LINE__);
+    ret = vvcam_isp_platform_device_register();
+    if (ret) {
+		platform_driver_unregister(&vvcam_isp_driver);
+		printk(KERN_ERR "Failed to register vvcam isp platform devices\n");
+		return ret;
+	}
+#endif
+
+    return ret;
+}
+
+static void __exit vvcam_isp_exit_module(void)
+{
+    platform_driver_unregister(&vvcam_isp_driver);
+#ifdef VVCAM_PLATFORM_REGISTER
+    vvcam_isp_platform_device_unregister();
+#endif
+}
+
+late_initcall(vvcam_isp_init_module);
+module_exit(vvcam_isp_exit_module);
+
+MODULE_DESCRIPTION("Verisilicon isp v4l2 driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_driver.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_driver.h
new file mode 100644
index 000000000000..72f041472919
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_driver.h
@@ -0,0 +1,160 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_DRIVER_H__
+#define __VVCAM_ISP_DRIVER_H__
+#include <linux/list.h>
+#include <linux/videodev2.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/v4l2-ctrls.h>
+#include <media/videobuf2-dma-contig.h>
+#include "vvcam_v4l2_common.h"
+
+#define VVCAM_ISP_NAME "vvcam-isp-subdev"
+
+#define VVCAM_ISP_WIDTH_ALIGN 16
+#define VVCAM_ISP_HEIGHT_ALIGN 8
+#define VVCAM_ISP_WIDTH_MIN 32
+#define VVCAM_ISP_HEIGHT_MIN 16
+
+#define VVCAM_ISP_PORT_NR  4
+enum vvcam_isp_port_pad_e {
+	VVCAM_ISP_PORT_PAD_SINK = 0,
+	VVCAM_ISP_PORT_PAD_SOURCE_MP,
+	VVCAM_ISP_PORT_PAD_SOURCE_SP1,
+	VVCAM_ISP_PORT_PAD_SOURCE_SP2,
+	VVCAM_ISP_PORT_PAD_SOURCE_RAW,
+	VVCAM_ISP_PORT_PAD_NR,
+};
+
+#define VVCAM_ISP_PAD_NR (VVCAM_ISP_PORT_NR * VVCAM_ISP_PORT_PAD_NR)
+
+enum vvcam_isp_input_type_e {
+    VVCAM_ISP_INPUT_TYPE_SENSOR,         /**< Sensor input */
+    VVCAM_ISP_INPUT_TYPE_IMAGE,          /**< Image buffer input */
+    VVCAM_ISP_INPUT_TYPE_TPG,            /**< ISP TPG buffer input */
+    VVCAM_ISP_INPUT_TYPE_MAX             /**< Maximum input type. */
+};
+
+struct vvcam_isp_format {
+	uint32_t code;
+	uint32_t fourcc;
+};
+
+struct vvcam_isp_pad_data {
+	uint32_t sink_detected;
+	struct v4l2_mbus_framefmt format;
+	struct v4l2_fract frmival_min;
+    struct v4l2_fract frmival_max;
+	uint32_t num_formats;
+	struct vvcam_isp_format *fmts;
+	struct list_head queue;
+	spinlock_t qlock;
+	uint32_t stream;
+	struct v4l2_rect crop;
+	struct v4l2_rect compose;
+};
+
+struct vvcam_isp_event_shm {
+	struct mutex event_lock;
+	uint64_t phy_addr;
+	void *virt_addr;
+	uint32_t size;
+};
+
+struct vvcam_isp_sensor_info {
+	char lib[128];
+	char isi_sym[32];
+	uint8_t mode;
+	char xml[64];
+	char manu_json[128];
+	char auto_json[128];
+	char one_json[128];
+};
+
+struct vvcam_isp_dev {
+    phys_addr_t paddr;
+    uint32_t regs_size;
+    void __iomem *base;
+    void __iomem *reset;
+    int  id;
+    int fe_irq;
+    int isp_irq;
+    int mi_irq;
+    struct device *dev;
+    struct mutex mlock;
+    uint32_t refcnt;
+    struct v4l2_subdev sd;
+	struct media_pad pads[VVCAM_ISP_PAD_NR];
+	struct v4l2_async_notifier notifier;
+#ifdef VVCAM_PLATFORM_REGISTER
+	struct fwnode_handle fwnode;
+#endif
+	struct vvcam_isp_pad_data pad_data[VVCAM_ISP_PAD_NR];
+	struct vvcam_isp_event_shm event_shm;
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct mutex ctrl_lock;
+	uint32_t ctrl_pad;
+
+	unsigned long pde;
+	struct vvcam_isp_sensor_info sensor_info[VVCAM_ISP_PORT_NR];
+	enum vvcam_isp_input_type_e input_type[VVCAM_ISP_PORT_NR];
+};
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_event.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_event.c
new file mode 100644
index 000000000000..9aa488f42add
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_event.c
@@ -0,0 +1,327 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <linux/delay.h>
+#include "vvcam_isp_event.h"
+#include "vvcam_isp_driver.h"
+
+static bool vvcam_isp_event_subscribed(struct v4l2_subdev *sd,
+                                uint32_t type, uint32_t id)
+{
+    struct v4l2_fh *fh;
+    unsigned long flags;
+    struct v4l2_subscribed_event *sev;
+    bool subscribed = false;
+
+    spin_lock_irqsave(&sd->devnode->fh_lock, flags);
+
+    list_for_each_entry(fh, &sd->devnode->fh_list, list) {
+        list_for_each_entry(sev, &fh->subscribed, list) {
+            if (sev->type == type && sev->id == id) {
+                subscribed = true;
+                break;
+            }
+        }
+        if (subscribed)
+            break;
+    }
+
+    spin_unlock_irqrestore(&sd->devnode->fh_lock, flags);
+
+    return subscribed;
+}
+
+int vvcam_isp_post_event(struct v4l2_subdev *sd, struct vvcam_isp_event_pkg *event_pkg)
+{
+    struct v4l2_event event;
+    int timeout_ms = 200000;
+    int i = 0;
+
+    if (!sd || !sd->dev || !sd->devnode) {
+        pr_err("%s: invalid sd/dev/devnode\n", __func__);
+        return -ENODEV;
+    }
+
+    memset(&event, 0, sizeof(event));
+
+    event.type   = VVCAM_ISP_DEAMON_EVENT;
+	event.id     = event_pkg->head.eid;
+    memcpy(event.u.data, &event_pkg->head, sizeof(event_pkg->head));
+
+    if (!vvcam_isp_event_subscribed(sd, event.type, event.id)) {
+        dev_err(sd->dev, "post event %d not subscribed\n", event.id);
+        return -EINVAL;
+    }
+
+    v4l2_event_queue(sd->devnode, &event);
+
+    for (i = 0; i < timeout_ms; i++) {
+        if (event_pkg->ack) {
+            break;
+        }
+        usleep_range(5, 10);
+    }
+
+    if (event_pkg->ack == 0) {
+        dev_err(sd->dev, "post event %d time out\n", event.id);
+        return -EIO;
+    }
+
+    if (event_pkg->result) {
+        dev_err(sd->dev, "post event %d return error\n", event.id);
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+int vvcam_isp_set_fmt_event(struct vvcam_isp_dev *isp_dev, int pad, struct v4l2_mbus_framefmt *format)
+{
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret;
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    event_pkg->head.eid = VVCAM_ISP_EVENT_SET_FMT;
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = sizeof(struct v4l2_mbus_framefmt);
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+    memcpy(event_pkg->data, format, sizeof(struct v4l2_mbus_framefmt));
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+
+    return ret;
+}
+
+int vvcam_isp_requebus_event(struct vvcam_isp_dev *isp_dev, int pad, uint32_t num_buffers)
+{
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret;
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    event_pkg->head.eid = VVCAM_ISP_EVENT_REQBUFS;
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = sizeof(num_buffers);
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+    memcpy(event_pkg->data, &num_buffers, sizeof(num_buffers));
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+
+    return ret;
+}
+
+int vvcam_isp_qbuf_event(struct vvcam_isp_dev *isp_dev, int pad, struct vvcam_vb2_buffer *buf)
+{
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret = 0;
+    struct vvcam_isp_buf user_buf;
+    int i;
+
+    user_buf.pad = pad;
+    user_buf.index = buf->sequence;
+    user_buf.num_planes = buf->num_planes;
+    for (i = 0; i < user_buf.num_planes; i++) {
+        user_buf.planes[i].dma_addr = buf->planes[i].dma_addr;
+        user_buf.planes[i].size = buf->planes[i].size;
+    }
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    event_pkg->head.eid = VVCAM_ISP_EVENT_QBUF;
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = sizeof(struct vvcam_isp_buf);
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+    memcpy(event_pkg->data, &user_buf, sizeof(struct vvcam_isp_buf));
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+    return ret;
+}
+
+int vvcam_isp_s_stream_event(struct vvcam_isp_dev *isp_dev, int pad, uint32_t status)
+{
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret = 0;
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    if (status) {
+        event_pkg->head.eid = VVCAM_ISP_EVENT_STREAMON;
+    } else {
+        event_pkg->head.eid = VVCAM_ISP_EVENT_STREAMOFF;
+    }
+
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = 0;
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+    return ret;
+}
+
+int vvcam_isp_s_ctrl_event(struct vvcam_isp_dev *isp_dev,
+            int pad, struct v4l2_ctrl *ctrl)
+{
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret;
+    struct vvcam_isp_ctrl *isp_ctrl;
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+
+    isp_ctrl = (struct vvcam_isp_ctrl *)event_pkg->data;
+    isp_ctrl->cid = ctrl->id;
+    isp_ctrl->size = ctrl->elem_size * ctrl->elems;
+    memcpy(isp_ctrl->data, ctrl->p_new.p_u8, isp_ctrl->size);
+
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    event_pkg->head.eid = VVCAM_ISP_EVENT_S_CTRL;
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = sizeof(isp_ctrl) + isp_ctrl->size;
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+
+    return ret;
+}
+
+int vvcam_isp_g_ctrl_event(struct vvcam_isp_dev *isp_dev,
+            int pad, struct v4l2_ctrl *ctrl)
+{
+    if (!isp_dev || !isp_dev->sd.dev || !isp_dev->sd.devnode) {
+        pr_err("%s: invalid isp_dev or sd not registered\n", __func__);
+        return -ENODEV;
+    }
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret = 0;
+    struct vvcam_isp_ctrl *isp_ctrl;
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+
+    isp_ctrl = (struct vvcam_isp_ctrl *)event_pkg->data;
+    isp_ctrl->cid = ctrl->id;
+    isp_ctrl->size = ctrl->elem_size * ctrl->elems;
+
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    event_pkg->head.eid = VVCAM_ISP_EVENT_G_CTRL;
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = sizeof(isp_ctrl) + isp_ctrl->size;
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    if (ret == 0) {
+         memcpy(ctrl->p_new.p_u8, event_pkg->data + sizeof(isp_ctrl),
+            isp_ctrl->size);
+    }
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+
+    return ret;
+}
+
+int vvcam_isp_s_selection_event(struct vvcam_isp_dev *isp_dev,
+            int pad, struct vvcam_isp_selection *sel)
+{
+    struct vvcam_isp_event_pkg *event_pkg = isp_dev->event_shm.virt_addr;
+    int ret = 0;
+
+    mutex_lock(&isp_dev->event_shm.event_lock);
+
+    event_pkg->head.pad = pad;
+    event_pkg->head.dev = isp_dev->id;
+    event_pkg->head.eid = VVCAM_ISP_EVENT_S_SELECTION;
+    event_pkg->head.shm_addr = isp_dev->event_shm.phy_addr;
+    event_pkg->head.shm_size = isp_dev->event_shm.size;
+    event_pkg->head.data_size = sizeof(struct vvcam_isp_selection);
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+    memcpy(event_pkg->data, sel, sizeof(struct vvcam_isp_selection));
+
+    ret = vvcam_isp_post_event(&isp_dev->sd, event_pkg);
+
+    mutex_unlock(&isp_dev->event_shm.event_lock);
+
+    return ret;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_event.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_event.h
new file mode 100644
index 000000000000..37711f951960
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_event.h
@@ -0,0 +1,137 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_EVENT_H__
+#define __VVCAM_ISP_EVENT_H__
+
+#define	 VVCAM_ISP_DEAMON_EVENT (V4L2_EVENT_PRIVATE_START + 2000)
+
+enum vvcam_isp_vevent_id {
+    VVCAM_ISP_EVENT_SET_FMT,
+    VVCAM_ISP_EVENT_REQBUFS,
+    VVCAM_ISP_EVENT_QBUF,
+    VVCAM_ISP_EVENT_BUF_DONE,
+    VVCAM_ISP_EVENT_STREAMON,
+    VVCAM_ISP_EVENT_STREAMOFF,
+    VVCAM_ISP_EVENT_S_CTRL,
+    VVCAM_ISP_EVENT_G_CTRL,
+    VVCAM_ISP_EVENT_S_SELECTION,
+    VVCAM_ISP_EVENT_MAX,
+};
+
+enum vvcam_isp_selection_type {
+    VVCAM_ISP_SEL_TYPE_CROP,
+    VVCAM_ISP_SEL_TYPE_COMPOSE,
+};
+
+
+struct vvcam_isp_plane {
+    uint32_t dma_addr;
+	uint32_t size;
+};
+
+struct vvcam_isp_buf {
+    uint32_t pad;
+    uint32_t index;
+    uint32_t num_planes;
+	struct vvcam_isp_plane planes[VIDEO_MAX_PLANES];
+};
+
+struct vvcam_isp_ctrl {
+    uint32_t cid;
+    uint32_t size;
+#ifdef __KERNEL__
+    uint8_t data[0];
+#endif
+};
+
+struct vvcam_isp_selection {
+    uint32_t type;
+    struct v4l2_rect rect;
+};
+
+struct vvcam_isp_event_pkg_head {
+    uint32_t pad;
+    uint8_t  dev;
+    uint32_t eid;
+    uint64_t shm_addr;
+    uint32_t shm_size;
+    uint32_t data_size;
+};
+
+struct vvcam_isp_event_pkg {
+    struct vvcam_isp_event_pkg_head head;
+    uint8_t  ack;
+    int32_t  result;
+    uint8_t data[2048];
+};
+
+#define VVCAM_ISP_IOC_BUFDONE    _IOWR('I',  BASE_VIDIOC_PRIVATE + 0, struct vvcam_isp_buf)
+
+#ifdef __KERNEL__
+#include "vvcam_isp_driver.h"
+
+int vvcam_isp_set_fmt_event(struct vvcam_isp_dev *isp_dev, int pad, struct v4l2_mbus_framefmt *format);
+int vvcam_isp_requebus_event(struct vvcam_isp_dev *isp_dev, int pad, uint32_t num_buffers);
+int vvcam_isp_qbuf_event(struct vvcam_isp_dev *isp_dev, int pad, struct vvcam_vb2_buffer *buf);
+int vvcam_isp_s_stream_event(struct vvcam_isp_dev *isp_dev, int pad, uint32_t status);
+int vvcam_isp_s_ctrl_event(struct vvcam_isp_dev *isp_dev, int pad, struct v4l2_ctrl *ctrl);
+int vvcam_isp_g_ctrl_event(struct vvcam_isp_dev *isp_dev, int pad, struct v4l2_ctrl *ctrl);
+int vvcam_isp_s_selection_event(struct vvcam_isp_dev *isp_dev, int pad, struct vvcam_isp_selection *sel);
+
+#endif
+
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_platform.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_platform.c
new file mode 100644
index 000000000000..32b84cb2edec
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_platform.c
@@ -0,0 +1,206 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+
+#include "vvcam_isp_platform.h"
+
+static int devices_mask = 0x01;
+MODULE_PARM_DESC(devices_mask, "vvcam isp devices mask");
+
+// #define VVCAM_ISP0_BASE 0x51000000
+// #define VVCAM_ISP1_BASE 0x51010000
+// #define VVCAM_ISP_REG_SIZE 0x10000
+
+// #define VVCAM_TPG0 (VVCAM_ISP0_BASE + 0x700)
+// #define VVCAM_TPG1 (VVCAM_ISP1_BASE + 0x700)
+// #define VVCAM_TPG_SIZE 0x24
+
+// #define VVCAM_ISP0_IRQ (VVCAM_ISP0_BASE + 0x5c4)
+// #define VVCAM_ISP1_IRQ (VVCAM_ISP1_BASE + 0x5c4)
+
+// #define VVCAM_ISP0_MI_IRQ 0x0
+// #define VVCAM_ISP1_MI_IRQ 0x0
+
+// #define VVCAM_ISP0_FE_IRQ 0x0
+// #define VVCAM_ISP1_FE_IRQ 0x0
+
+static void vvcam_isp_pdev_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+	return;
+}
+
+static struct resource vvcam_isp0_resource[] = {
+    [0] = {
+        .start = VVCAM_ISP0_BASE + PCIE_BASE,
+        .end   = VVCAM_ISP0_BASE + PCIE_BASE + VVCAM_ISP_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+//    [1] = {
+//        .start = VVCAM_TPG0 + PCIE_BASE,
+//        .end   = VVCAM_TPG0 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+//        .flags = IORESOURCE_MEM,
+//	},
+//    [2] = {
+//        .start = VVCAM_ISP0_IRQ,
+//        .end   = VVCAM_ISP0_IRQ,
+//        .flags = IORESOURCE_IRQ,
+//	},
+//    [3] = {
+//        .start = VVCAM_ISP0_MI_IRQ,
+//        .end   = VVCAM_ISP0_MI_IRQ,
+//        .flags = IORESOURCE_IRQ,
+//	},
+//    [4] = {
+//        .start = VVCAM_ISP0_FE_IRQ,
+//        .end   = VVCAM_ISP0_FE_IRQ,
+//        .flags = IORESOURCE_IRQ,
+//	},
+};
+
+static struct platform_device vvcam_isp0_pdev = {
+	.name          = "vvcam-isp-subdev",
+	.id            = 0,
+	.resource      = vvcam_isp0_resource,
+    .num_resources = ARRAY_SIZE(vvcam_isp0_resource),
+	.dev.release   = vvcam_isp_pdev_release,
+};
+
+
+static struct resource vvcam_isp1_resource[] = {
+    [0] = {
+        .start = VVCAM_ISP1_BASE + PCIE_BASE,
+        .end   = VVCAM_ISP1_BASE + PCIE_BASE + VVCAM_ISP_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+//    [1] = {
+//        .start = VVCAM_TPG1 + PCIE_BASE,
+//        .end   = VVCAM_TPG1 + PCIE_BASE + VVCAM_TPG_SIZE - 1,
+//        .flags = IORESOURCE_MEM,
+//	},
+//    [2] = {
+//        .start = VVCAM_ISP1_IRQ,
+//        .end   = VVCAM_ISP1_IRQ,
+//        .flags = IORESOURCE_IRQ,
+//	},
+//    [3] = {
+//        .start = VVCAM_ISP1_MI_IRQ,
+//        .end   = VVCAM_ISP1_MI_IRQ,
+//        .flags = IORESOURCE_IRQ,
+//	},
+//    [4] = {
+//        .start = VVCAM_ISP1_FE_IRQ,
+//        .end   = VVCAM_ISP1_FE_IRQ,
+//        .flags = IORESOURCE_IRQ,
+//	},
+};
+
+static struct platform_device vvcam_isp1_pdev = {
+	.name          = "vvcam-isp-subdev",
+	.id            = 1,
+	.resource      = vvcam_isp1_resource,
+    .num_resources = ARRAY_SIZE(vvcam_isp1_resource),
+	.dev.release   = vvcam_isp_pdev_release,
+};
+
+int vvcam_isp_platform_device_register(void)
+{
+    int ret = 0;
+    int devs_mask = 0;
+
+    if (devices_mask & 0x01) {
+        ret = platform_device_register(&vvcam_isp0_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x01;
+    }
+
+    if (devices_mask & 0x02) {
+        ret = platform_device_register(&vvcam_isp1_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x02;
+    }
+
+    return ret;
+
+err_register:
+    if (devs_mask & 0x01)
+		platform_device_unregister(&vvcam_isp0_pdev);
+	if (devs_mask & 0x02)
+		platform_device_unregister(&vvcam_isp1_pdev);
+
+    return ret;
+}
+
+void vvcam_isp_platform_device_unregister(void)
+{
+    if (devices_mask & 0x01)
+		platform_device_unregister(&vvcam_isp0_pdev);
+	if (devices_mask & 0x02)
+		platform_device_unregister(&vvcam_isp1_pdev);
+
+    return;
+}
+
+module_param(devices_mask, int, 0644);
+
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_platform.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_platform.h
new file mode 100644
index 000000000000..247a57fad731
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_PLATFORM_H__
+#define __VVCAM_ISP_PLATFORM_H__
+
+int vvcam_isp_platform_device_register(void);
+void vvcam_isp_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_procfs.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_procfs.c
new file mode 100644
index 000000000000..4c08d643c33e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_procfs.c
@@ -0,0 +1,403 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/ctype.h>
+#include "vvcam_isp_driver.h"
+#include "vvcam_isp_procfs.h"
+
+struct vvcam_isp_procfs {
+    struct proc_dir_entry *pde;
+    struct vvcam_isp_dev *isp_dev;
+    struct mutex lock;
+};
+
+static char* type2str(int type)
+{
+    switch (type)
+    {
+    case VVCAM_ISP_INPUT_TYPE_SENSOR:
+        return "sensor";
+
+    // temporarily not supported
+    // case VVCAM_ISP_INPUT_TYPE_IMAGE:
+    //     return "image";
+
+    case VVCAM_ISP_INPUT_TYPE_TPG:
+        return "tpg";
+
+    default:
+        return "sensor";
+    }
+}
+
+static int vvcam_isp_procfs_info_show(struct seq_file *sfile, void *offset)
+{
+	struct vvcam_isp_procfs *isp_proc;
+    struct vvcam_isp_dev *isp_dev;
+    struct media_pad *pad = NULL;
+    int pad_idx = 0;
+    int port = -1;
+
+    isp_proc = (struct vvcam_isp_procfs *) sfile->private;
+
+    mutex_lock(&isp_proc->lock);
+    isp_dev = isp_proc->isp_dev;
+
+    seq_printf(sfile, "/******sensor configuration******/\n");
+    for (pad_idx = 0; pad_idx < VVCAM_ISP_PAD_NR; pad_idx++) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&isp_dev->pads[pad_idx]);
+#else
+        pad = media_entity_remote_pad(&isp_dev->pads[pad_idx]);
+#endif
+
+        if (pad && (port != (pad_idx / VVCAM_ISP_PORT_PAD_NR))) {
+            port = pad_idx / VVCAM_ISP_PORT_PAD_NR;
+            seq_printf(sfile, "isp%d port%d:\n", isp_dev->id, port);
+            seq_printf(sfile, "input_type: %s\n", type2str(isp_dev->input_type[port]));
+            seq_printf(sfile, "lib      : %s\n", isp_dev->sensor_info[port].lib);
+            seq_printf(sfile, "isi_sym  : %s\n", isp_dev->sensor_info[port].isi_sym);
+            seq_printf(sfile, "mode     : %d\n", isp_dev->sensor_info[port].mode);
+            seq_printf(sfile, "xml      : %s\n", isp_dev->sensor_info[port].xml);
+            seq_printf(sfile, "manu_json: %s\n", isp_dev->sensor_info[port].manu_json);
+            seq_printf(sfile, "auto_json: %s\n", isp_dev->sensor_info[port].auto_json);
+            seq_printf(sfile, "one_json : %s\n", isp_dev->sensor_info[port].one_json);
+            seq_printf(sfile, "*********************************\n");
+        }
+    }
+
+    mutex_unlock(&isp_proc->lock);
+
+	return 0;
+}
+
+static int vvcam_isp_procfs_open(struct inode *inode, struct file *file)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	return single_open(file, vvcam_isp_procfs_info_show, PDE_DATA(inode));
+#else
+	return single_open(file, vvcam_isp_procfs_info_show, pde_data(inode));
+#endif
+}
+
+static int32_t vvcam_isp_proc_process(struct seq_file *sfile,
+                        struct vvcam_isp_procfs *isp_proc, char *str_buf)
+{
+    struct vvcam_isp_dev *isp_dev;
+    char *token = NULL, *cur = str_buf;
+    char *val = NULL, *kv_cur = NULL, *end = NULL;
+    char *const delim = " ";
+    char *const kv_delim = "=";
+    int port = 0;
+
+    isp_dev = isp_proc->isp_dev;
+
+    mutex_lock(&isp_proc->lock);
+
+    while((token = strsep(&cur, delim))) {
+        if (isdigit(*token)) {
+            port = *token - '0';
+            continue;
+        }
+
+        if ((port > VVCAM_ISP_PORT_NR) || (port < 0)) {
+            continue;
+        }
+
+        kv_cur = token;
+        val = strsep(&kv_cur, kv_delim);
+        if (val) {
+            if (strcmp(val, "lib") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val) {
+                    memset(isp_dev->sensor_info[port].lib, 0, \
+                                sizeof(isp_dev->sensor_info[port].lib));
+                    strncpy(isp_dev->sensor_info[port].lib, val, strlen(val));
+                }
+            } else if (strcmp(val, "isi_sym") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val) {
+                    memset(isp_dev->sensor_info[port].isi_sym, 0, \
+                                sizeof(isp_dev->sensor_info[port].isi_sym));
+                    strncpy(isp_dev->sensor_info[port].isi_sym, val, strlen(val));
+                }
+            } else if (strcmp(val, "mode") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val && isdigit(*val)) {
+                    isp_dev->sensor_info[port].mode = (uint32_t)simple_strtoul(val, &end, 0);
+                }
+            } else if (strcmp(val, "xml") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val) {
+                    memset(isp_dev->sensor_info[port].xml, 0, \
+                                sizeof(isp_dev->sensor_info[port].xml));
+                    strncpy(isp_dev->sensor_info[port].xml, val, strlen(val));
+                }
+            } else if (strcmp(val, "manu_json") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val) {
+                    memset(isp_dev->sensor_info[port].manu_json, 0, \
+                                sizeof(isp_dev->sensor_info[port].manu_json));
+                    strncpy(isp_dev->sensor_info[port].manu_json, val, strlen(val));
+                }
+            } else if (strcmp(val, "auto_json") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val) {
+                    memset(isp_dev->sensor_info[port].auto_json, 0, \
+                                sizeof(isp_dev->sensor_info[port].auto_json));
+                    strncpy(isp_dev->sensor_info[port].auto_json, val, strlen(val));
+                }
+            } else if (strcmp(val, "one_json") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val) {
+                    memset(isp_dev->sensor_info[port].one_json, 0, \
+                                sizeof(isp_dev->sensor_info[port].one_json));
+                    strncpy(isp_dev->sensor_info[port].one_json, val, strlen(val));
+                }
+            } else if (strcmp(val, "input_type") == 0) {
+                val = strsep(&kv_cur, kv_delim);
+                if (val && isdigit(*val)) {
+                    isp_dev->input_type[port] = (uint32_t)simple_strtoul(val, &end, 0);
+                }
+            }
+        }
+    }
+
+    mutex_unlock(&isp_proc->lock);
+
+    return 0;
+}
+
+static ssize_t vvcam_isp_procfs_write(struct file *file,
+		const char __user *buffer, size_t count, loff_t *ppos)
+{
+    struct vvcam_isp_procfs *isp_proc;
+    struct seq_file *sfile;
+    char *str_buf;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	isp_proc = (struct vvcam_isp_procfs *) PDE_DATA(file_inode(file));
+#else
+	isp_proc = (struct vvcam_isp_procfs *) pde_data(file_inode(file));
+#endif
+    sfile = file->private_data;
+
+    str_buf = (char *)kzalloc(count, GFP_KERNEL);
+    if (!str_buf)
+        return -ENOMEM;
+
+    if (copy_from_user(str_buf, buffer, count))
+        return -EFAULT;
+
+    *(str_buf + count - 1) = '\0';
+
+    vvcam_isp_proc_process(sfile, isp_proc, str_buf);
+
+    kfree(str_buf);
+
+    return count;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
+static const struct file_operations vvcam_isp_procfs_ops = {
+    .open = vvcam_isp_procfs_open,
+    .release = seq_release,
+    .read = seq_read,
+    .write = vvcam_isp_procfs_write,
+    .llseek = seq_lseek,
+};
+#else
+static const struct proc_ops vvcam_isp_procfs_ops = {
+	.proc_open = vvcam_isp_procfs_open,
+	.proc_release = seq_release,
+	.proc_read = seq_read,
+	.proc_write = vvcam_isp_procfs_write,
+	.proc_lseek = seq_lseek,
+};
+#endif
+
+struct finddir_callback {
+    struct dir_context ctx;
+    const char *name;
+    int32_t files_cnt;
+    bool found;
+};
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)
+static int readdir_callback(struct dir_context *ctx, const char *name,
+    int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return 0;
+
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return 0;
+}
+
+#else
+static bool readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return true;
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return true;
+}
+#endif
+
+static int find_proc_dir_by_name(const char *root,
+                const char *name, bool *found, int32_t *files_cnt) {
+    struct file *pfile;
+    int ret = 0;
+    struct finddir_callback fc = {
+        .ctx.actor = readdir_callback,
+        .name = name,
+        .found = false,
+        .files_cnt = -2,
+    };
+
+    pfile = filp_open(root, O_RDONLY | O_DIRECTORY, 0);
+    if(IS_ERR(pfile)) {
+        pr_err("Failed to open %s\n", root);
+        return -1;
+    }
+    if (pfile->f_op->iterate_shared) {
+        ret = pfile->f_op->iterate_shared(pfile, &fc.ctx);
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 2, 0)
+    } else {
+        ret = pfile->f_op->iterate(pfile, &fc.ctx);
+#endif
+    }
+
+    if (ret == 0) {
+        *found = fc.found;
+    }
+
+    if (files_cnt != NULL) {
+        *files_cnt = fc.files_cnt;
+    }
+
+    filp_close(pfile, NULL);
+    return ret;
+}
+
+int vvcam_v4l2_isp_procfs_register(struct vvcam_isp_dev *isp_dev, unsigned long *pde)
+{
+    struct vvcam_isp_procfs *isp_proc;
+    char isp_proc_name[32];
+    int ret = 0;
+    bool found = false;
+
+    if (!isp_dev)
+        return -1;
+    sprintf(isp_proc_name, "vsi/isp_subdev%d", isp_dev->id);
+
+    isp_proc = devm_kzalloc(isp_dev->dev,
+                        sizeof(struct vvcam_isp_procfs), GFP_KERNEL);
+
+    if (!isp_proc)
+        return -ENOMEM;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (!found)
+            proc_mkdir("vsi", NULL);
+    } else {
+        return -EFAULT;
+    }
+
+    isp_proc->isp_dev = isp_dev;
+    isp_proc->pde = proc_create_data(isp_proc_name, 0666, NULL,
+                                &vvcam_isp_procfs_ops, isp_proc);
+    if (!isp_proc->pde)
+        return -EFAULT;
+    *pde = (unsigned long)&isp_proc->pde;
+
+    mutex_init(&(isp_proc->lock));
+	return 0;
+}
+
+void vvcam_v4l2_isp_procfs_unregister(unsigned long pde)
+{
+    int ret = 0;
+    bool found = false;
+    int32_t files_cnt;
+    struct vvcam_isp_procfs *isp_proc = (struct vvcam_isp_procfs*)pde;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (found) {
+            proc_remove(isp_proc->pde);
+            ret = find_proc_dir_by_name("/proc/vsi", "", &found, &files_cnt);
+            if (files_cnt == 0) {
+                remove_proc_subtree("vsi", NULL);
+            }
+        }
+    }
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_procfs.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_procfs.h
new file mode 100644
index 000000000000..d175cf967b39
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/isp/vvcam_isp_procfs.h
@@ -0,0 +1,64 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_ISP_PROCFS_H__
+#define __VVCAM_ISP_PROCFS_H__
+
+int vvcam_v4l2_isp_procfs_register(struct vvcam_isp_dev *isp_dev,
+                            unsigned long *pde);
+void vvcam_v4l2_isp_procfs_unregister(unsigned long pde);
+
+#endif
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/Makefile b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/Makefile
new file mode 100644
index 000000000000..5da1c7d9c535
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/Makefile
@@ -0,0 +1,70 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+ifeq ($(VVCAM_PATH),)
+        VVCAM_PATH:=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))/../../../vvcam
+        export VVCAM_PATH
+endif
+
+include $(VVCAM_PATH)/Param.mk
+TARGET = vvcam_video
+
+obj-$(CONFIG_VIDEO_ESWIN_ISP) += $(TARGET).o
+$(TARGET)-objs += vvcam_video_driver.o
+$(TARGET)-objs += vvcam_video_platform.o
+$(TARGET)-objs += vvcam_video_register.o
+$(TARGET)-objs += vvcam_video_m2m_register.o
+$(TARGET)-objs += vvcam_video_event.o
+EXTRA_CFLAGS += -I$(VVCAM_PATH)/v4l2/common/
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_pipeline_link.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_pipeline_link.h
new file mode 100644
index 000000000000..d687b80a06e2
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_pipeline_link.h
@@ -0,0 +1,235 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+ #ifndef __VVCAM_PIPELINE_LINK_H__
+ #define __VVCAM_PIPELINE_LINK_H__
+
+extern struct v4l2_subdev *g_vvcam_isp_subdev[VVCAM_ISP_DEV_MAX];
+
+static struct vvcam_v4l2_link pipeline0[] = {
+    {
+        .local_is_video = true,
+        .video_index = 0,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 1,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 1,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 2,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 2,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 3,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 3,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 4,
+    },
+
+};
+
+static struct vvcam_v4l2_link pipeline1[] = {
+    {
+        .local_is_video = true,
+        .video_index = 0,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 1,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 1,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 2,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 2,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 3,
+    },
+};
+
+static struct vvcam_v4l2_link mcm_pipeline0[] = {
+    {
+        .local_is_video = true,
+        .video_index = 0,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 1,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 1,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 6,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 2,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 11,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 3,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 16,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 4,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 4,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 5,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 9,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 6,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 14,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 7,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[0],
+        .remote_pad = 19,
+    },
+};
+
+static struct vvcam_v4l2_link mcm_pipeline1[] = {
+    {
+        .local_is_video = true,
+        .video_index = 0,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 1,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 1,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 6,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 2,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 11,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 3,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 16,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 4,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 4,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 5,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 9,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 6,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 14,
+    },
+    {
+        .local_is_video = true,
+        .video_index = 7,
+        .local_pad = 0,
+        .remote_subdev = &g_vvcam_isp_subdev[1],
+        .remote_pad = 19,
+    },
+};
+
+ #endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video.h
new file mode 100644
index 000000000000..fc21ed0e96cf
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video.h
@@ -0,0 +1,59 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VIDEO_HAL_H__
+#define __VVCAM_VIDEO_HAL_H__
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_driver.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_driver.c
new file mode 100644
index 000000000000..b48cd7f5e2a8
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_driver.c
@@ -0,0 +1,787 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/of_graph.h>
+#include <linux/vmalloc.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include <linux/i2c.h>
+#include <linux/delay.h>
+#include <media/v4l2-subdev.h>
+#include "vvcam_video_driver.h"
+#include "vvcam_video_register.h"
+#ifdef VVCAM_PLATFORM_REGISTER
+#include "vvcam_video_platform.h"
+#include "vvcam_pipeline_link.h"
+#endif
+
+#ifdef VVCAM_PLATFORM_REGISTER
+static int mcm_mask = 0x00;
+MODULE_PARM_DESC(mcm_mask, "vvcam video mcm mask");
+#endif
+
+static int vvcam_video_register_ports(struct vvcam_media_dev *vvcam_mdev)
+{
+    int i = 0;
+    int ret = 0;
+
+    for (i = 0; i < vvcam_mdev->ports; i++) {
+        if (vvcam_mdev->video_params[i].m2m == false) {
+            ret = vvcam_video_register(vvcam_mdev, i);
+            if (ret) {
+                goto err_register_video;
+            }
+        }
+    }
+    return 0;
+
+err_register_video:
+    for (i = 0; i < vvcam_mdev->ports; i++) {
+        if (vvcam_mdev->video_params[i].m2m == false)
+            vvcam_video_unregister(vvcam_mdev, i);
+    }
+
+    return ret;
+}
+
+static int vvcam_video_unregister_ports(struct vvcam_media_dev *vvcam_mdev)
+{
+    int i;
+
+    for (i = 0; i < vvcam_mdev->ports; i++) {
+        if (vvcam_mdev->video_params[i].m2m == false)
+            vvcam_video_unregister(vvcam_mdev, i);
+    }
+
+    return 0;
+}
+
+#ifdef VVCAM_PLATFORM_REGISTER
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+static int vvcam_video_notifier_bound(struct v4l2_async_notifier *notifier,
+				struct v4l2_subdev *sd, struct v4l2_async_connection *asc)
+#else
+static int vvcam_video_notifier_bound(struct v4l2_async_notifier *notifier,
+				struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
+#endif
+{
+    struct vvcam_media_dev *vvcam_mdev = container_of(notifier,
+							   struct vvcam_media_dev, notifier);
+    struct device *dev = vvcam_mdev->dev;
+	struct v4l2_subdev *remote_subdev;
+    struct v4l2_subdev *local_subdev;
+    struct video_device *vdev;
+    uint32_t port;
+    struct media_entity *source, *sink;
+	unsigned int source_pad, sink_pad;
+    int i = 0;
+    int ret;
+
+    for (i = 0; i < vvcam_mdev->pipeline_link_size; i++) {
+        if (vvcam_mdev->pipeline_link[i].remote_subdev) {
+            remote_subdev = *vvcam_mdev->pipeline_link[i].remote_subdev;
+            if (remote_subdev) {
+                if (sd == remote_subdev) {
+                    if (vvcam_mdev->pipeline_link[i].local_is_video) {
+                        port = vvcam_mdev->pipeline_link[i].video_index;
+                        vdev = vvcam_mdev->video_devs[port]->video;
+                        source = &sd->entity;
+                        source_pad = vvcam_mdev->pipeline_link[i].remote_pad;
+                        sink =&vdev->entity;
+                        sink_pad = 0;
+
+                    } else {
+                        local_subdev = *vvcam_mdev->pipeline_link[i].local_subdev;
+                        source = &sd->entity;
+                        source_pad = vvcam_mdev->pipeline_link[i].remote_pad;
+                        sink =&local_subdev->entity;
+                        sink_pad = vvcam_mdev->pipeline_link[i].local_pad;
+                    }
+
+                    ret = media_create_pad_link(source, source_pad, sink, sink_pad, MEDIA_LNK_FL_ENABLED);
+                    if (ret) {
+                        dev_err(dev, "failed to create %s:%u -> %s:%u link\n",
+                            source->name, source_pad, sink->name, sink_pad);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    return 0;
+}
+
+#else
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+static int vvcam_video_notifier_bound(struct v4l2_async_notifier *notifier,
+				struct v4l2_subdev *sd, struct v4l2_async_connection *asc)
+#else
+static int vvcam_video_notifier_bound(struct v4l2_async_notifier *notifier,
+				struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
+#endif
+{
+    struct vvcam_media_dev *vvcam_mdev = container_of(notifier,
+							   struct vvcam_media_dev, notifier);
+	struct device *dev = vvcam_mdev->dev;
+    struct fwnode_handle *ep = NULL;
+	struct v4l2_fwnode_link link;
+	struct media_entity *source, *sink;
+	unsigned int source_pad, sink_pad;
+    struct vvcam_video_dev *vvcam_vdev;
+	struct video_device *vdev;
+	int ret;
+
+	while (1) {
+		ep = fwnode_graph_get_next_endpoint(sd->fwnode, ep);
+		if (!ep)
+			break;
+		ret = v4l2_fwnode_parse_link(ep, &link);
+		if (ret < 0) {
+			dev_err(dev, "failed to parse link for %pOF: %d\n", to_of_node(ep), ret);
+			continue;
+		}
+
+		if (sd->entity.pads[link.local_port].flags == MEDIA_PAD_FL_SINK)
+			continue;
+
+        vvcam_vdev = vvcam_mdev->video_devs[link.remote_port];
+		vdev = vvcam_vdev->video;
+		source = &sd->entity;
+		source_pad = link.local_port;
+		sink = &vdev->entity;
+		sink_pad = 0;
+		v4l2_fwnode_put_link(&link);
+		ret = media_create_pad_link(source, source_pad, sink, sink_pad, MEDIA_LNK_FL_ENABLED);        
+		if (ret) {
+			dev_err(dev, "failed to create %s:%u -> %s:%u link\n", source->name, source_pad, sink->name, sink_pad);
+			break;
+		}
+        dev_err(dev, "success to create %s:%u -> %s:%u link\n", source->name, source_pad, sink->name, sink_pad);
+	}
+	fwnode_handle_put(ep);
+
+    return 0;
+}
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+static void vvcam_video_notifier_unbound(struct v4l2_async_notifier *notifier,
+					struct v4l2_subdev *sd, struct v4l2_async_connection *asc)
+{
+	return;
+}
+#else
+static void vvcam_video_notifier_unbound(struct v4l2_async_notifier *notifier,
+					struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
+{
+	return;
+}
+#endif
+
+static int vvcam_video_notifier_complete(struct v4l2_async_notifier *notifier)
+{
+    struct vvcam_media_dev *vvcam_mdev = container_of(notifier,
+							   struct vvcam_media_dev, notifier);
+
+    printk("complete vvcam video register subdev nodes\n");
+    return v4l2_device_register_subdev_nodes(&vvcam_mdev->v4l2_dev);
+}
+
+static const struct v4l2_async_notifier_operations vvcam_video_async_nf_ops = {
+	.bound    = vvcam_video_notifier_bound,
+	.unbind   = vvcam_video_notifier_unbound,
+	.complete = vvcam_video_notifier_complete,
+};
+
+#ifdef VVCAM_PLATFORM_REGISTER
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+static int vvcam_video_async_register_subdev(struct vvcam_media_dev *vvcam_mdev)
+{
+    int ret;
+    int i = 0;
+    bool repeat = false;
+    struct v4l2_subdev *remote_subdev = NULL;
+    struct v4l2_subdev *last_subdev = NULL;
+    struct v4l2_async_connection *asc;
+
+    v4l2_async_nf_init(&vvcam_mdev->notifier, &vvcam_mdev->v4l2_dev);
+    vvcam_mdev->notifier.ops = &vvcam_video_async_nf_ops;
+
+    for (i = 0; i < vvcam_mdev->pipeline_link_size; i++) {
+        if (vvcam_mdev->pipeline_link[i].remote_subdev) {
+            remote_subdev = *vvcam_mdev->pipeline_link[i].remote_subdev;
+            /* not allow repeat add same fwnode */
+            for (int j = i - 1; j >= 0; j--) {
+                last_subdev = *vvcam_mdev->pipeline_link[j].remote_subdev;
+                if (remote_subdev && last_subdev &&
+                    remote_subdev == last_subdev) {
+                    repeat = true;
+                    break;
+                }
+            }
+
+            if (remote_subdev && !repeat) {
+                asc = v4l2_async_nf_add_fwnode(&vvcam_mdev->notifier,
+                                        remote_subdev->fwnode,
+                                        struct v4l2_async_connection);
+                if (IS_ERR(asc)) {
+                    ret = PTR_ERR(asc);
+                    if (ret != -EEXIST) {
+                        v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+                        return ret;
+                    }
+                }
+            }
+            repeat = false;
+        }
+    }
+
+    ret = v4l2_async_nf_register(&vvcam_mdev->notifier);
+    if (ret) {
+        v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+        return ret;
+    }
+
+    return 0;
+}
+
+#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) */
+static int vvcam_video_async_register_subdev(struct vvcam_media_dev *vvcam_mdev)
+{
+    int ret;
+    int i = 0;
+    struct v4l2_subdev *remote_subdev;
+    struct v4l2_async_subdev *asd;
+
+    vvcam_mdev->notifier.ops = &vvcam_video_async_nf_ops;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    v4l2_async_nf_init(&vvcam_mdev->notifier);
+#else
+    v4l2_async_notifier_init(&vvcam_mdev->notifier);
+#endif
+
+    for (i = 0; i < vvcam_mdev->pipeline_link_size; i++) {
+        if (vvcam_mdev->pipeline_link[i].remote_subdev) {
+            remote_subdev = *vvcam_mdev->pipeline_link[i].remote_subdev;
+            if (remote_subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+                asd = v4l2_async_nf_add_fwnode(&vvcam_mdev->notifier,
+                                        remote_subdev->fwnode,
+                                        struct v4l2_async_subdev);
+#else
+                asd = v4l2_async_notifier_add_fwnode_subdev(
+                                        &vvcam_mdev->notifier,
+                                        remote_subdev->fwnode,
+                                        struct v4l2_async_subdev);
+#endif
+                 if (IS_ERR(asd)) {
+			        ret = PTR_ERR(asd);
+			        if (ret != -EEXIST) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+                        v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+#else
+				        v4l2_async_notifier_cleanup(&vvcam_mdev->notifier);
+#endif
+				        return ret;
+			        }
+		        }
+            }
+        }
+    }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    ret = v4l2_async_nf_register(&vvcam_mdev->v4l2_dev, &vvcam_mdev->notifier);
+#else
+    ret = v4l2_async_notifier_register(&vvcam_mdev->v4l2_dev, &vvcam_mdev->notifier);
+#endif
+	if (ret) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+        v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+#else
+		v4l2_async_notifier_cleanup(&vvcam_mdev->notifier);
+#endif
+		dev_err(vvcam_mdev->dev, "v4l2 async notifier register error\n");
+        return ret;
+    }
+
+    return 0;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) */
+
+#else /* VVCAM_PLATFORM_REGISTER */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+struct v4l2_async_connection *
+vvcam_video_async_nf_add_fwnode_remote(struct v4l2_async_notifier *notif,
+				  struct fwnode_handle *endpoint,
+				  unsigned int asc_struct_size)
+{
+    struct v4l2_async_connection *asc;
+    struct fwnode_handle *remote;
+
+    remote = fwnode_graph_get_remote_port_parent(endpoint);
+    if (!remote) {
+        printk("get remote name = failed\n");
+	    return ERR_PTR(-ENOTCONN);
+    }
+
+    asc = __v4l2_async_nf_add_fwnode(notif, remote, asc_struct_size);
+    fwnode_handle_put(remote);
+
+    return asc;
+}
+
+static int vvcam_video_async_register_subdev(struct vvcam_media_dev *vvcam_mdev)
+{
+    int ret = 0;
+    struct fwnode_handle *ep;
+    struct v4l2_async_connection *asc;
+    unsigned int port_id = 0;
+
+    vvcam_mdev->notifier.ops = &vvcam_video_async_nf_ops;
+
+    printk(KERN_ERR "%s %d\n", __func__, __LINE__);
+    v4l2_async_nf_init(&vvcam_mdev->notifier, &vvcam_mdev->v4l2_dev);
+
+    while (1) {
+        struct fwnode_handle *remote_ep = NULL;
+        ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(vvcam_mdev->dev),
+                port_id, 0, FWNODE_GRAPH_ENDPOINT_NEXT);
+        if (!ep)
+            break;
+
+		remote_ep = fwnode_graph_get_remote_port_parent(ep);
+
+		pr_info("%s, %d remote_ep:%s\n", __func__, __LINE__,
+		       fwnode_get_name(remote_ep));
+
+		pr_info("%s, %d ep:%s\n", __func__, __LINE__,
+		       fwnode_get_name(ep));
+    
+        // asc = vvcam_video_async_nf_add_fwnode_remote(&vvcam_mdev->notifier,
+        //         ep, sizeof(struct v4l2_async_connection));
+        asc = v4l2_async_nf_add_fwnode_remote(
+			&vvcam_mdev->notifier, ep, struct v4l2_async_connection);
+
+        fwnode_handle_put(ep);
+
+        if (IS_ERR(asc)) {
+            ret = PTR_ERR(asc);
+            if (ret != -EEXIST) {
+                v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+                return ret;
+            }
+        }
+        port_id++;
+    }
+
+    ret = v4l2_async_nf_register(&vvcam_mdev->notifier);
+    if (ret) {
+        v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+		dev_err(vvcam_mdev->dev, "v4l2 async notifier register error ret:%d\n",ret);
+        return ret;
+    }
+    printk("v4l2_async_nf_register ok\n");
+    return 0;
+}
+
+#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) */
+static struct v4l2_async_subdev *
+vvcam_video_async_nf_add_fwnode_remote(struct v4l2_async_notifier *notif,
+				  struct fwnode_handle *endpoint,
+				  unsigned int asd_struct_size)
+{
+	struct v4l2_async_subdev *asd;
+	struct fwnode_handle *remote;
+
+	remote = fwnode_graph_get_remote_port_parent(endpoint);
+	if (!remote) {
+        printk("get remote name failed\n");
+		return ERR_PTR(-ENOTCONN);
+    }
+
+	asd = __v4l2_async_nf_add_fwnode(notif, remote, asd_struct_size);
+	fwnode_handle_put(remote);
+
+	return asd;
+}
+
+static int vvcam_video_async_register_subdev(struct vvcam_media_dev *vvcam_mdev)
+{
+    int ret = 0;
+	struct fwnode_handle *ep;
+	struct v4l2_async_subdev *asd;
+    unsigned int port_id = 0;
+
+	vvcam_mdev->notifier.ops = &vvcam_video_async_nf_ops;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+	v4l2_async_nf_init(&vvcam_mdev->notifier);
+#else
+    v4l2_async_notifier_init(&vvcam_mdev->notifier);
+#endif
+
+    while (1) {
+        ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(vvcam_mdev->dev),
+                port_id, 0, FWNODE_GRAPH_ENDPOINT_NEXT);
+        if (!ep)
+            break;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+        asd = vvcam_video_async_nf_add_fwnode_remote(&vvcam_mdev->notifier,
+                ep, sizeof(struct v4l2_async_subdev));
+#else
+        asd = v4l2_async_notifier_add_fwnode_remote_subdev(&vvcam_mdev->notifier,
+                ep, struct v4l2_async_subdev);
+#endif
+        fwnode_handle_put(ep);
+
+        if (IS_ERR(asd)) {
+			ret = PTR_ERR(asd);
+			if (ret != -EEXIST) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+                v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+#else
+				v4l2_async_notifier_cleanup(&vvcam_mdev->notifier);
+#endif
+				return ret;
+			}
+		}
+        port_id++;
+    }
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    ret = v4l2_async_nf_register(&vvcam_mdev->v4l2_dev, &vvcam_mdev->notifier);
+#else
+    ret = v4l2_async_notifier_register(&vvcam_mdev->v4l2_dev, &vvcam_mdev->notifier);
+#endif
+	if (ret) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+        v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+#else
+		v4l2_async_notifier_cleanup(&vvcam_mdev->notifier);
+#endif
+		dev_err(vvcam_mdev->dev, "v4l2 async notifier register error\n");
+        return ret;
+    }
+	
+    dev_err(vvcam_mdev->dev, "v4l2 async notifier register ok\n");
+    return 0;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) */
+#endif /* VVCAM_PLATFORM_REGISTER */
+
+static int vvcam_video_async_unregister_subdev(struct vvcam_media_dev *vvcam_mdev)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+    v4l2_async_nf_unregister(&vvcam_mdev->notifier);
+    v4l2_async_nf_cleanup(&vvcam_mdev->notifier);
+#else
+    v4l2_async_notifier_unregister(&vvcam_mdev->notifier);
+    v4l2_async_notifier_cleanup(&vvcam_mdev->notifier);
+#endif
+
+    return 0;
+}
+
+static const struct media_device_ops vvcam_video_media_ops = {
+    .link_notify = v4l2_pipeline_link_notify,
+};
+
+static int vvcam_video_parse_params(struct vvcam_media_dev *vvcam_mdev,
+                                struct platform_device *pdev)
+{
+#ifdef VVCAM_PLATFORM_REGISTER
+    int ports = 0;
+    int i;
+    vvcam_mdev->id = pdev->id;
+    switch(vvcam_mdev->id) {
+        case 0:
+            if (!mcm_mask) {
+                vvcam_mdev->pipeline_link = pipeline0;
+                vvcam_mdev->pipeline_link_size = ARRAY_SIZE(pipeline0);
+            } else {
+                vvcam_mdev->pipeline_link = mcm_pipeline0;
+                vvcam_mdev->pipeline_link_size = ARRAY_SIZE(mcm_pipeline0);
+            }
+            break;
+        case 1:
+            if (!mcm_mask) {
+                vvcam_mdev->pipeline_link = pipeline1;
+                vvcam_mdev->pipeline_link_size = ARRAY_SIZE(pipeline1);
+            } else {
+                vvcam_mdev->pipeline_link = mcm_pipeline1;
+                vvcam_mdev->pipeline_link_size = ARRAY_SIZE(mcm_pipeline1);
+            }
+            break;
+        default:
+            break;
+    }
+
+    for (i = 0; i < vvcam_mdev->pipeline_link_size; i++) {
+        if (vvcam_mdev->pipeline_link[i].local_is_video) {
+            if ((vvcam_mdev->pipeline_link[i].video_index + 1) > ports) {
+                ports = vvcam_mdev->pipeline_link[i].video_index + 1;
+            }
+        }
+    }
+    vvcam_mdev->ports = ports;
+#else
+    unsigned int port_id = 0;
+    struct fwnode_handle *ep;
+    fwnode_property_read_u32(of_fwnode_handle(pdev->dev.of_node),
+			"id", &vvcam_mdev->id);
+
+     while (1) {
+        ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(vvcam_mdev->dev),
+                port_id, 0, FWNODE_GRAPH_ENDPOINT_NEXT);
+        if (!ep)
+            break;
+        pr_info("Found endpoint: %s port_id:%d\n", fwnode_get_name(ep), port_id);
+        port_id++;
+     }
+
+    vvcam_mdev->ports = port_id;
+#endif
+    return 0;
+}
+
+static int vvcam_video_probe(struct platform_device *pdev)
+{
+    int ret = 0;
+    // struct media_device *mdev;
+	struct device *dev = &pdev->dev;
+    struct vvcam_media_dev *vvcam_mdev;
+    struct device *parent = pdev->dev.parent;
+    struct eswin_vi_device* es_vi_device;
+    struct device_node *sensor_np;
+    struct device *sensor_dev;
+    struct v4l2_subdev *sd;
+    int wait_time = 0;
+
+    sensor_np = of_parse_phandle(dev->of_node, "depends-on", 0);
+    if (sensor_np)
+    {
+        while (wait_time < 20000) {
+            sensor_dev = bus_find_device_by_of_node(&i2c_bus_type, sensor_np);
+            of_node_put(sensor_np);
+            if (sensor_dev) {
+                sd = dev_get_drvdata(sensor_dev);
+                put_device(sensor_dev);
+                if (sd && sd->v4l2_dev) {
+                    dev_info(dev, "Sensor is ready.\n");
+                    break;
+                }
+            }
+            msleep(1000);
+            wait_time += 1000;
+        }
+    }
+
+    vvcam_mdev = devm_kzalloc(dev, sizeof(struct vvcam_media_dev), GFP_KERNEL);
+	if (!vvcam_mdev)
+		return -ENOMEM;
+
+    vvcam_mdev->dev = dev;
+    platform_set_drvdata(pdev, vvcam_mdev);
+    es_vi_device = dev_get_drvdata(parent);
+    if (!es_vi_device) {
+        printk("es_vi_devic is null\n");
+        return -1;
+    }
+    pr_info("%s:%d es_vi_device:%p\n", __func__, __LINE__, es_vi_device);
+
+    ret = vvcam_video_parse_params(vvcam_mdev, pdev);
+    if (ret) {
+        dev_err(dev, "parse device params error\n");
+		return ret;
+    }
+
+    ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret < 0) {
+		dev_err(dev, "%s:%d: dma_set_mask_and_coherent failed: %d\n",
+			__func__, __LINE__, ret);
+		return ret;
+	}    
+    // mdev = &vvcam_mdev->mdev;
+    // mdev->dev = dev;
+    // mdev->ops = &vvcam_video_media_ops;
+    // strscpy(mdev->model, "verisilicon_media", sizeof(mdev->model));
+	// media_device_init(mdev);
+
+    vvcam_mdev->mdev = es_vi_device->media_dev;
+    vvcam_mdev->v4l2_dev.mdev = es_vi_device->media_dev;
+
+    // vvcam_mdev->v4l2_dev.mdev = mdev;
+    ret = v4l2_device_register(dev, &vvcam_mdev->v4l2_dev);
+	if (ret) {
+		dev_err(dev, "register v4l2 device error\n");
+		return ret;
+	}
+
+    ret = vvcam_video_register_ports(vvcam_mdev);
+    if (ret) {
+		dev_err(dev, "register video device nodes error\n");
+		goto err_unregister_v4l2_device;
+	}
+
+    ret = vvcam_video_async_register_subdev(vvcam_mdev);
+    if (ret) {
+        dev_err(dev, "register v4l2 async notifier error\n");
+		goto err_unregister_video_ports;
+    }
+
+    // ret = media_device_register(mdev);
+    // if (ret) {
+    //     dev_err(dev, "register media device error\n");
+    //     goto err_unregister_subdev;
+    // }
+    dev_info(&pdev->dev, "vvcam video driver probe success\n");
+
+    return 0;
+// err_unregister_subdev:
+    // vvcam_video_async_unregister_subdev(vvcam_mdev);
+err_unregister_video_ports:
+    vvcam_video_unregister_ports(vvcam_mdev);
+err_unregister_v4l2_device:
+    v4l2_device_unregister(&vvcam_mdev->v4l2_dev);
+
+    return ret;
+}
+
+static int vvcam_video_remove(struct platform_device *pdev)
+{
+    struct vvcam_media_dev *vvcam_mdev;
+
+    vvcam_mdev = platform_get_drvdata(pdev);
+
+    // media_device_unregister(&vvcam_mdev->mdev);
+    vvcam_video_async_unregister_subdev(vvcam_mdev);
+    vvcam_video_unregister_ports(vvcam_mdev);
+    v4l2_device_unregister(&vvcam_mdev->v4l2_dev);
+    dev_info(&pdev->dev, "vvcam video driver remove\n");
+
+    return 0;
+}
+
+static const struct of_device_id vvcam_video_of_match[] = {
+	{.compatible = "verisilicon,video",},
+	{ /* sentinel */ },
+};
+
+MODULE_DEVICE_TABLE(of, vvcam_video_of_match);
+
+static struct platform_driver vvcam_video_driver = {
+	.probe  = vvcam_video_probe,
+	.remove = vvcam_video_remove,
+	.driver = {
+        .name  = VVCAM_VIDEO_NAME,
+        .owner = THIS_MODULE,
+        .of_match_table = vvcam_video_of_match,
+    },
+};
+
+static int __init vvcam_video_init_module(void)
+{
+    int ret;
+    ret = platform_driver_register(&vvcam_video_driver);
+    if (ret) {
+        printk(KERN_ERR "Failed to register video driver\n");
+        return ret;
+    }
+
+#ifdef VVCAM_PLATFORM_REGISTER
+    ret = vvcam_video_platform_device_register();
+    if (ret) {
+        platform_driver_unregister(&vvcam_video_driver);
+		printk(KERN_ERR "Failed to register vvcam video platform devices\n");
+		return ret;
+    }
+#endif
+
+    return ret;
+}
+
+static void __exit vvcam_video_exit_module(void)
+{
+    platform_driver_unregister(&vvcam_video_driver);
+#ifdef VVCAM_PLATFORM_REGISTER
+    vvcam_video_platform_device_unregister();
+#endif
+
+    return;
+}
+
+#ifdef VVCAM_PLATFORM_REGISTER
+module_param(mcm_mask, int, 0644);
+#endif
+
+late_initcall(vvcam_video_init_module);
+module_exit(vvcam_video_exit_module);
+
+MODULE_DESCRIPTION("Verisilicon isp driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_driver.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_driver.h
new file mode 100644
index 000000000000..6def1bf11d94
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_driver.h
@@ -0,0 +1,133 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VIDEO_DRIVER_H__
+#define __VVCAM_VIDEO_DRIVER_H__
+
+#include <linux/list.h>
+#include <linux/videodev2.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/v4l2-ctrls.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/eswin/eswin_vi.h>
+
+#ifdef VVCAM_PLATFORM_REGISTER
+ struct vvcam_v4l2_link {
+    struct v4l2_subdev **local_subdev;
+    bool local_is_video;
+    uint32_t video_index;
+	int local_pad;
+	struct v4l2_subdev **remote_subdev;
+	int remote_pad;
+ };
+#endif
+
+#define VVCAM_VIDEO_NAME "vvcam-video"
+#define VVCAM_VIDEO_PORT_MAX 64
+
+#define VVCAM_VIDEO_WIDTH_ALIGN  16
+#define VVCAM_VIDEO_HEIGHT_ALIGN  8
+
+#define VVCAM_VIDEO_MIN_WIDTH 32
+#define VVCAM_VIDEO_MIN_HEIGHT 16
+
+struct vvcam_video_params {
+	bool m2m;
+};
+
+struct vvcam_video_event_shm {
+	struct mutex event_lock;
+	uint64_t phy_addr;
+	void *virt_addr;
+	uint32_t size;
+};
+
+struct vvcam_video_dev {
+	struct vvcam_video_params video_params;
+	struct vvcam_media_dev *vvcam_mdev;
+	struct video_device *video;
+	struct media_pad pad;
+	struct vb2_queue queue;
+	struct mutex video_lock;
+	struct v4l2_format format;
+	uint32_t pipeline;
+	struct vvcam_video_event_shm event_shm;
+};
+
+struct vvcam_media_dev {
+	int id;
+    struct device *dev;
+	struct media_device *mdev;
+	struct v4l2_device v4l2_dev;
+	struct v4l2_async_notifier notifier;
+	int ports;
+	struct vvcam_video_params video_params[VVCAM_VIDEO_PORT_MAX];
+	struct vvcam_video_dev *video_devs[VVCAM_VIDEO_PORT_MAX];
+#ifdef VVCAM_PLATFORM_REGISTER
+	struct vvcam_v4l2_link *pipeline_link;
+	uint32_t pipeline_link_size;
+#endif
+};
+
+struct vvcam_video_fmt_info {
+	uint32_t fourcc;
+	uint32_t mbus;
+};
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_event.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_event.c
new file mode 100644
index 000000000000..85f137f4176a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_event.c
@@ -0,0 +1,173 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/videodev2.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <linux/delay.h>
+#include "vvcam_video_event.h"
+
+static bool vvcam_video_event_subscribed(struct vvcam_video_dev *vvcam_vdev,
+                                uint32_t type, uint32_t id)
+{
+    struct v4l2_fh *fh;
+    unsigned long flags;
+    struct v4l2_subscribed_event *sev;
+    bool subscribed = false;
+
+    spin_lock_irqsave(&vvcam_vdev->video->fh_lock, flags);
+
+    list_for_each_entry(fh, &vvcam_vdev->video->fh_list, list) {
+        list_for_each_entry(sev, &fh->subscribed, list) {
+            if (sev->type == type && sev->id == id) {
+                subscribed = true;
+                break;
+            }
+        }
+        if (subscribed)
+            break;
+    }
+
+    spin_unlock_irqrestore(&vvcam_vdev->video->fh_lock, flags);
+
+    return subscribed;
+}
+
+int vvcam_video_post_event(struct vvcam_video_dev *vvcam_vdev,
+                        struct vvcam_video_event_pkg *event_pkg)
+{
+    struct v4l2_event event;
+    int timeout_ms = 200000;
+    int i = 0;
+
+    memset(&event, 0, sizeof(event));
+
+    event.type   = VVCAM_VIDEO_DEAMON_EVENT;
+	event.id     = event_pkg->head.eid;
+    memcpy(event.u.data, &event_pkg->head, sizeof(event_pkg->head));
+
+    if (!vvcam_video_event_subscribed(vvcam_vdev, event.type, event.id)) {
+        return -EINVAL;
+    }
+
+
+    v4l2_event_queue(vvcam_vdev->video, &event);
+
+    for (i = 0; i < timeout_ms; i++) {
+        if (event_pkg->ack) {
+            break;
+        }
+        usleep_range(5, 10);
+    }
+
+    if (event_pkg->ack == 0) {
+        dev_err(vvcam_vdev->vvcam_mdev->dev,
+            "%s post event %d time out\n",
+            vvcam_vdev->video->name, event.id);
+        return -EIO;
+    }
+
+    if (event_pkg->result) {
+        printk("event failed, event_pkg res = %d\n", event_pkg->result);
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+int vvcam_video_create_pipeline_event(struct vvcam_video_dev *vvcam_vdev)
+{
+    struct vvcam_video_event_pkg *event_pkg = vvcam_vdev->event_shm.virt_addr;
+    int ret;
+
+    mutex_lock(&vvcam_vdev->event_shm.event_lock);
+
+    event_pkg->head.eid = VVCAM_VEVENT_CREATE_PIPELINE;
+    event_pkg->head.shm_addr = vvcam_vdev->event_shm.phy_addr;
+    event_pkg->head.shm_size = vvcam_vdev->event_shm.size;
+    event_pkg->head.data_size = 0;
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+
+    ret = vvcam_video_post_event(vvcam_vdev, event_pkg);
+
+    mutex_unlock(&vvcam_vdev->event_shm.event_lock);
+
+    return ret;
+}
+
+int vvcam_video_destroy_pipeline_event(struct vvcam_video_dev *vvcam_vdev)
+{
+    struct vvcam_video_event_pkg *event_pkg = vvcam_vdev->event_shm.virt_addr;
+    int ret;
+
+    mutex_lock(&vvcam_vdev->event_shm.event_lock);
+
+    event_pkg->head.eid = VVCAM_VEVENT_DESTROY_PIPELINE;
+    event_pkg->head.shm_addr = vvcam_vdev->event_shm.phy_addr;
+    event_pkg->head.shm_size = vvcam_vdev->event_shm.size;
+    event_pkg->head.data_size = 0;
+    event_pkg->ack = 0;
+    event_pkg->result = 0;
+
+    ret = vvcam_video_post_event(vvcam_vdev, event_pkg);
+
+    mutex_unlock(&vvcam_vdev->event_shm.event_lock);
+
+    return ret;
+}
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_event.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_event.h
new file mode 100644
index 000000000000..2a4c49c5ffc9
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_event.h
@@ -0,0 +1,87 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VIDEO_EVENT_H__
+#define __VVCAM_VIDEO_EVENT_H__
+
+#define	 VVCAM_VIDEO_DEAMON_EVENT (V4L2_EVENT_PRIVATE_START + 1000)
+
+enum vvcam_video_event_id {
+    VVCAM_VEVENT_CREATE_PIPELINE = 0,
+    VVCAM_VEVENT_DESTROY_PIPELINE,
+    VVCAM_VEVENT_MAX,
+};
+
+struct vvcam_video_event_pkg_head {
+    uint32_t eid;
+    uint64_t shm_addr;
+    uint32_t shm_size;
+    uint32_t data_size;
+};
+
+struct vvcam_video_event_pkg {
+    struct vvcam_video_event_pkg_head head;
+    uint8_t  ack;
+    int32_t  result;
+    uint8_t data[2048];
+};
+
+#ifdef __KERNEL__
+#include "vvcam_video_driver.h"
+int vvcam_video_create_pipeline_event(struct vvcam_video_dev *vvcam_vdev);
+int vvcam_video_destroy_pipeline_event(struct vvcam_video_dev *vvcam_vdev);
+#endif
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_m2m_register.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_m2m_register.c
new file mode 100644
index 000000000000..36f89b491bf7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_m2m_register.c
@@ -0,0 +1,55 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_m2m_register.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_m2m_register.h
new file mode 100644
index 000000000000..36f89b491bf7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_m2m_register.h
@@ -0,0 +1,55 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_platform.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_platform.c
new file mode 100644
index 000000000000..6a71181c69b4
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_platform.c
@@ -0,0 +1,125 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include "vvcam_video_platform.h"
+
+static int devices_mask = 0x01;
+MODULE_PARM_DESC(devices_mask, "vvcam video devices mask");
+
+static void vvcam_video_pdev_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+	return;
+}
+
+static struct platform_device vvcam_video0_pdev = {
+	.name          = "vvcam-video",
+	.id            = 0,
+	.dev.release   = vvcam_video_pdev_release,
+};
+
+static struct platform_device vvcam_video1_pdev = {
+	.name          = "vvcam-video",
+	.id            = 1,
+	.dev.release   = vvcam_video_pdev_release,
+};
+
+int vvcam_video_platform_device_register(void)
+{
+    int ret = 0;
+    int devs_mask = 0;
+
+    if (devices_mask & 0x01) {
+        ret = platform_device_register(&vvcam_video0_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x01;
+    }
+
+    if (devices_mask & 0x02) {
+        ret = platform_device_register(&vvcam_video1_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x02;
+    }
+
+    return ret;
+
+err_register:
+    if (devs_mask & 0x01)
+		platform_device_unregister(&vvcam_video0_pdev);
+	if (devs_mask & 0x02)
+		platform_device_unregister(&vvcam_video1_pdev);
+
+    return ret;
+}
+
+void vvcam_video_platform_device_unregister(void)
+{
+    if (devices_mask & 0x01)
+        platform_device_unregister(&vvcam_video0_pdev);
+
+    if (devices_mask & 0x02)
+        platform_device_unregister(&vvcam_video1_pdev);
+
+    return;
+}
+
+module_param(devices_mask, int, 0644);
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_platform.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_platform.h
new file mode 100644
index 000000000000..ccc6afdee0a1
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VIDEO_PLATFORM_H__
+#define __VVCAM_VIDEO_PLATFORM_H__
+
+int vvcam_video_platform_device_register(void);
+void vvcam_video_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_register.c b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_register.c
new file mode 100644
index 000000000000..d926dc00e8a4
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_register.c
@@ -0,0 +1,2376 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/of_graph.h>
+#include <linux/vmalloc.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include "vvcam_video_register.h"
+#include "vvcam_v4l2_common.h"
+#include "vvcam_video_event.h"
+#include "vvcam_v4l2_std_exts.h"
+
+static struct vvcam_video_fmt_info vvcam_formats_info[] = {
+    {
+        .fourcc    = V4L2_PIX_FMT_NV16,
+        .mbus      = MEDIA_BUS_FMT_YUYV8_2X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_NV12,
+        .mbus      = MEDIA_BUS_FMT_YUYV8_1_5X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_YUYV,
+        .mbus      = MEDIA_BUS_FMT_YUYV8_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR8,
+        .mbus      = MEDIA_BUS_FMT_SBGGR8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG8,
+        .mbus      = MEDIA_BUS_FMT_SGBRG8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG8,
+        .mbus      = MEDIA_BUS_FMT_SGRBG8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB8,
+        .mbus      = MEDIA_BUS_FMT_SRGGB8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10,
+        .mbus      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10,
+        .mbus      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10,
+        .mbus      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10,
+        .mbus      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12,
+        .mbus      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12,
+        .mbus      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12,
+        .mbus      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12,
+        .mbus      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P010,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_GREY,
+        .mbus      = MEDIA_BUS_FMT_Y8_1X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10BPACK,
+        .mbus      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10DWA,
+        .mbus      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_Y10,
+        .mbus      = MEDIA_BUS_FMT_Y10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P00BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P00DWA,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P02BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUYV12_2X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P20BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P20DWA,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P210,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_P22BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUYV12_2X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I20BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I210,
+        .mbus      = MEDIA_BUS_FMT_YUYV10_2X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_M48BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I48BPACK,
+        .mbus      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I48DWA,
+        .mbus      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_I40DWA,
+        .mbus      = MEDIA_BUS_FMT_YUV8_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24,
+        .mbus      = MEDIA_BUS_FMT_RGB888_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24DWA,
+        .mbus      = MEDIA_BUS_FMT_RGB888_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_RGB24P,
+        .mbus      = MEDIA_BUS_FMT_RGB888_3X8,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10BPACK,
+        .mbus      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10BPACK,
+        .mbus      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10BPACK,
+        .mbus      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10BPACK,
+        .mbus      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR10DWA,
+        .mbus      = MEDIA_BUS_FMT_SBGGR10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG10DWA,
+        .mbus      = MEDIA_BUS_FMT_SGBRG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG10DWA,
+        .mbus      = MEDIA_BUS_FMT_SGRBG10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB10DWA,
+        .mbus      = MEDIA_BUS_FMT_SRGGB10_1X10,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12BPACK,
+        .mbus      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12BPACK,
+        .mbus      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12BPACK,
+        .mbus      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12BPACK,
+        .mbus      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR12DWA,
+        .mbus      = MEDIA_BUS_FMT_SBGGR12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG12DWA,
+        .mbus      = MEDIA_BUS_FMT_SGBRG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG12DWA,
+        .mbus      = MEDIA_BUS_FMT_SGRBG12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB12DWA,
+        .mbus      = MEDIA_BUS_FMT_SRGGB12_1X12,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14BPACK,
+        .mbus      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14BPACK,
+        .mbus      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14BPACK,
+        .mbus      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14BPACK,
+        .mbus      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14DWA,
+        .mbus      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14DWA,
+        .mbus      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14DWA,
+        .mbus      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14DWA,
+        .mbus      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR14,
+        .mbus      = MEDIA_BUS_FMT_SBGGR14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG14,
+        .mbus      = MEDIA_BUS_FMT_SGBRG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG14,
+        .mbus      = MEDIA_BUS_FMT_SGRBG14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB14,
+        .mbus      = MEDIA_BUS_FMT_SRGGB14_1X14,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR16,
+        .mbus      = MEDIA_BUS_FMT_SBGGR16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG16,
+        .mbus      = MEDIA_BUS_FMT_SGBRG16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG16,
+        .mbus      = MEDIA_BUS_FMT_SGRBG16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB16,
+        .mbus      = MEDIA_BUS_FMT_SRGGB16_1X16,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SBGGR24,
+        .mbus      = MEDIA_BUS_FMT_SBGGR24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGBRG24,
+        .mbus      = MEDIA_BUS_FMT_SGBRG24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SGRBG24,
+        .mbus      = MEDIA_BUS_FMT_SGRBG24_1X24,
+    },
+    {
+        .fourcc    = V4L2_PIX_FMT_SRGGB24,
+        .mbus      = MEDIA_BUS_FMT_SRGGB24_1X24,
+    },
+};
+
+static int vvcam_video_mbus_to_fourcc(uint32_t mbus, uint32_t *fourcc, uint32_t fmt)
+{
+    int i = 0;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_formats_info); i++) {
+        if ((vvcam_formats_info[i].mbus == mbus) && (vvcam_formats_info[i].fourcc == fmt)) {
+            *fourcc = vvcam_formats_info[i].fourcc;
+            return 0;
+        }
+    }
+
+    return -EINVAL;
+}
+
+static int vvcam_video_fourcc_to_mbus(uint32_t fourcc, uint32_t *mbus)
+{
+    int i = 0;
+
+    for (i = 0; i < ARRAY_SIZE(vvcam_formats_info); i++) {
+        if (vvcam_formats_info[i].fourcc == fourcc) {
+            *mbus = vvcam_formats_info[i].mbus;
+            return 0;
+        }
+    }
+
+    return -EINVAL;
+}
+
+static int vvcam_video_vfmt_to_mfmt(struct v4l2_format *f, struct v4l2_subdev_format *mfmt)
+{
+    int ret;
+
+    mfmt->format.width        = f->fmt.pix.width;
+    mfmt->format.height       = f->fmt.pix.height;
+    mfmt->format.field        = f->fmt.pix.field;
+    mfmt->format.colorspace   = f->fmt.pix.colorspace;
+    mfmt->format.quantization = f->fmt.pix.quantization;
+
+    ret = vvcam_video_fourcc_to_mbus(f->fmt.pix.pixelformat, &mfmt->format.code);
+    memcpy(mfmt->format.reserved, &f->fmt.pix.pixelformat, sizeof(f->fmt.pix.pixelformat));
+
+    return ret;
+
+}
+
+static const struct v4l2_format_info *vvcam_video_vfmt_info(u32 format)
+{
+    /* format info for user define or supported by later versions */
+    static const struct v4l2_format_info formats[] = {
+        {
+            .format = V4L2_PIX_FMT_GREY, .pixel_enc = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1, .comp_planes = 1,
+            .hdiv       = 1, .vdiv        = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp = {1, 0, 0, 0},    .bpp_div = {1, 1, 1, 1},
+#else
+            .bpp = {1, 0, 0, 0},
+#endif
+        },
+        {
+            .format = V4L2_PIX_FMT_Y10BPACK, .pixel_enc = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1, .comp_planes = 1,
+            .hdiv       = 1, .vdiv        = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp = {5, 0, 0, 0},    .bpp_div = {4, 1, 1, 1},
+#else
+            .bpp = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format = V4L2_PIX_FMT_Y10DWA, .pixel_enc = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1, .comp_planes = 1,
+            .hdiv       = 1, .vdiv        = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp = {4, 0, 0, 0},    .bpp_div = {3, 1, 1, 1},
+#else
+            .bpp = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format = V4L2_PIX_FMT_Y10, .pixel_enc = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1, .comp_planes = 1,
+            .hdiv       = 1, .vdiv        = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp = {2, 0, 0, 0},     .bpp_div = {1, 1, 1, 1},
+#else
+            .bpp = {2, 0, 0, 0},
+#endif
+        },
+        // 420
+        {
+            .format     = V4L2_PIX_FMT_P00BPACK,.pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 2,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {5, 10, 0, 0},               .bpp_div    = {4, 4, 1, 1},
+#else
+            .bpp = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_P00DWA,  .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 2,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 8, 0, 0},         .bpp_div = {3, 3, 1, 1},
+#else
+            .bpp        = {2, 2, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_P010,    .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 1,                    .vdiv       = 2,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+             .bpp       = {2, 2, 0, 0},         .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 2, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_P02BPACK,.pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 2,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 6, 0, 0},         .bpp_div = {2, 2, 1, 1},
+#else
+            .bpp        = {2, 2, 0, 0},
+#endif
+        },
+        // 422
+        {
+            .format     = V4L2_PIX_FMT_P20BPACK,.pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 1,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {5, 10, 0, 0},        .bpp_div = {4, 4, 1, 1},
+#else
+            .bpp        = {2, 2, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_P20DWA,  .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 1,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 8, 0, 0},         .bpp_div = {3, 3, 1, 1},
+#else
+            .bpp = {2, 2, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_P210,    .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 1,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 4, 0, 0},         .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 2, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_P22BPACK,.pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 2,
+            .hdiv       = 1,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 6, 0, 0},         .bpp_div    = {2, 2, 1, 1},
+#else
+            .bpp        = {2, 2, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_I20BPACK,.pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 1,
+            .hdiv       = 1,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {15, 0, 0, 0},        .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {4, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_I210,    .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                    .comp_planes= 1,
+            .hdiv       = 1,                    .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},         .bpp_div = {1, 1, 1, 1},
+#else
+            .bpp = {4, 0, 0, 0},
+#endif
+        },
+        // 444
+        {
+            .format     = V4L2_PIX_FMT_M48BPACK,    .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                        .comp_planes= 3,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {1, 1, 1, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {1, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_I48BPACK,    .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {3, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_I48DWA,      .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {4, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_I40DWA,      .pixel_enc  = V4L2_PIXEL_ENC_YUV,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {4, 0, 0, 0},
+#endif
+        },
+        // RGB
+        {
+            .format     = V4L2_PIX_FMT_RGB24,       .pixel_enc  = V4L2_PIXEL_ENC_RGB,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {3, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_RGB24DWA,    .pixel_enc  = V4L2_PIXEL_ENC_RGB,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {4, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_RGB24P,      .pixel_enc  = V4L2_PIXEL_ENC_RGB,
+            .mem_planes = 1,                        .comp_planes= 3,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {1, 1, 1, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {1, 1, 1, 0},
+#endif
+        },
+        // RAW
+        {
+            .format     = V4L2_PIX_FMT_SBGGR10BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {5, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG10BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+             .bpp       = {5, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG10BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {5, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB10BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {5, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR10DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div = {3, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG10DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div = {3, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG10DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div = {3, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB10DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {4, 0, 0, 0},             .bpp_div = {3, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR12BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 0, 0, 0},            .bpp_div = {2, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG12BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 0, 0, 0},            .bpp_div     = {2, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG12BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 0, 0, 0},            .bpp_div     = {2, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB12BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {3, 0, 0, 0},            .bpp_div     = {2, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR12DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {8, 0, 0, 0},             .bpp_div = {5, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG12DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {8, 0, 0, 0},             .bpp_div    = {5, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG12DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {8, 0, 0, 0},             .bpp_div    = {5, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB12DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {8, 0, 0, 0},             .bpp_div    = {5, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR14BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {7, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG14BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {7, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG14BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {7, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB14BPACK,.pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {7, 0, 0, 0},             .bpp_div    = {4, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR14DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {16, 0, 0, 0},            .bpp_div = {9, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG14DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {16, 0, 0, 0},            .bpp_div    = {9, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG14DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {16, 0, 0, 0},            .bpp_div    = {9, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB14DWA,  .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {16, 0, 0, 0},            .bpp_div    = {9, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR14,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG14,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG14,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB14,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR16,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG16,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG16,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB16,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {2, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SBGGR24,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {3, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGBRG24,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {3, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SGRBG24,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {3, 0, 0, 0},
+#endif
+        },
+        {
+            .format     = V4L2_PIX_FMT_SRGGB24,     .pixel_enc  = V4L2_PIXEL_ENC_BAYER,
+            .mem_planes = 1,                        .comp_planes= 1,
+            .hdiv       = 1,                        .vdiv       = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
+            .bpp        = {2, 0, 0, 0},             .bpp_div    = {1, 1, 1, 1},
+#else
+            .bpp        = {3, 0, 0, 0},
+#endif
+        },
+
+    };
+
+    unsigned int i;
+
+    for (i = 0; i < ARRAY_SIZE(formats); i++) {
+        if (formats[i].format == format)
+            return &formats[i];
+    }
+
+    return NULL;
+}
+
+static int vvcam_video_mfmt_to_vfmt( struct v4l2_subdev_format *mfmt, struct v4l2_format *f)
+{
+    int ret;
+    const struct v4l2_format_info *info = NULL;
+    uint32_t bytesperline;
+    uint32_t sizeimage = 0;
+    uint32_t width;
+    uint32_t height;
+    int i;
+    int private_fmt = 0;
+    uint32_t fourcc_code = 0;
+
+    memcpy(&fourcc_code, mfmt->format.reserved, sizeof(fourcc_code));
+    f->fmt.pix.width       = mfmt->format.width;
+    f->fmt.pix.height      = mfmt->format.height;
+    f->fmt.pix.field       = mfmt->format.field;
+    f->fmt.pix.colorspace  = mfmt->format.colorspace;
+    f->fmt.pix.quantization = mfmt->format.quantization;
+    ret = vvcam_video_mbus_to_fourcc(mfmt->format.code, &f->fmt.pix.pixelformat, fourcc_code);
+    if (ret) {
+        return ret;
+    }
+
+    width  = f->fmt.pix.width;
+    height = f->fmt.pix.height;
+    info   = v4l2_format_info(f->fmt.pix.pixelformat);
+    if (info != NULL) {
+        bytesperline = info->bpp[0] * width;
+    } else {
+        private_fmt = 1;
+        info = vvcam_video_vfmt_info(f->fmt.pix.pixelformat);
+        if (info == NULL) {
+            return -EINVAL;
+        }
+
+        // calculate size information here for private format
+        switch (info->format)
+        {
+        /* YUV format */
+        // 8bit unalign
+        case V4L2_PIX_FMT_GREY:
+        case V4L2_PIX_FMT_M48BPACK:
+        case V4L2_PIX_FMT_I48BPACK:
+            bytesperline = width;
+            if (info->format == V4L2_PIX_FMT_I48BPACK) {
+                bytesperline *= 3;
+            }
+            break;
+
+        // 8bit dwa
+        case V4L2_PIX_FMT_I48DWA:
+        case V4L2_PIX_FMT_I40DWA:
+            bytesperline = ((width * 4) / 3) * 3;
+            break;
+
+        // 10bit unalign
+        case V4L2_PIX_FMT_Y10BPACK:
+        case V4L2_PIX_FMT_P00BPACK:
+        case V4L2_PIX_FMT_P20BPACK:
+        case V4L2_PIX_FMT_I20BPACK:
+            bytesperline = DIV_ROUND_UP(width*10, 128) * 16;
+            if (info->format == V4L2_PIX_FMT_I20BPACK) {
+                bytesperline *= 4;
+            }
+            break;
+
+        // 10 bit double align
+        case V4L2_PIX_FMT_Y10DWA:
+        case V4L2_PIX_FMT_P00DWA:
+        case V4L2_PIX_FMT_P20DWA:
+            bytesperline = width * 4 / 3;
+            break;
+
+        // 10 bit mode 1, per pixel 16bits
+        case V4L2_PIX_FMT_Y10:
+        case V4L2_PIX_FMT_P010:
+        case V4L2_PIX_FMT_P210:
+        case V4L2_PIX_FMT_I210:
+            bytesperline = DIV_ROUND_UP(width, 8) * 16;
+            if (info->format == V4L2_PIX_FMT_I210) {
+                bytesperline *= 2;
+            }
+            break;
+
+        // 12bit unalign
+        case V4L2_PIX_FMT_P02BPACK:
+        case V4L2_PIX_FMT_P22BPACK:
+            bytesperline = DIV_ROUND_UP(width*12, 128) * 16;
+            break;
+
+        // RGB
+        case V4L2_PIX_FMT_RGB24:
+            bytesperline = width * 3;
+            break;
+
+        case V4L2_PIX_FMT_RGB24P:
+            bytesperline = width;
+            break;
+
+        case V4L2_PIX_FMT_RGB24DWA:
+            bytesperline = (width * 4 / 3) * 3;
+            break;
+
+        // RAW
+        // 10 bit-unalign
+        case V4L2_PIX_FMT_SBGGR10BPACK:
+        case V4L2_PIX_FMT_SGBRG10BPACK:
+        case V4L2_PIX_FMT_SGRBG10BPACK:
+        case V4L2_PIX_FMT_SRGGB10BPACK:
+            bytesperline = DIV_ROUND_UP(width * 10, 128) * 16;
+            break;
+
+        case V4L2_PIX_FMT_SBGGR10DWA:
+        case V4L2_PIX_FMT_SGBRG10DWA:
+        case V4L2_PIX_FMT_SGRBG10DWA:
+        case V4L2_PIX_FMT_SRGGB10DWA:
+            bytesperline = DIV_ROUND_UP(width, 12) * 16;
+            break;
+
+        // 12bit
+        case V4L2_PIX_FMT_SBGGR12BPACK:
+        case V4L2_PIX_FMT_SGBRG12BPACK:
+        case V4L2_PIX_FMT_SGRBG12BPACK:
+        case V4L2_PIX_FMT_SRGGB12BPACK:
+            bytesperline = DIV_ROUND_UP(width * 12, 128) * 16;
+            break;
+
+        case V4L2_PIX_FMT_SBGGR12DWA:
+        case V4L2_PIX_FMT_SGBRG12DWA:
+        case V4L2_PIX_FMT_SGRBG12DWA:
+        case V4L2_PIX_FMT_SRGGB12DWA:
+            bytesperline = DIV_ROUND_UP(width, 10) * 16;
+            break;
+
+        // 14 bit
+        case V4L2_PIX_FMT_SBGGR14BPACK:
+        case V4L2_PIX_FMT_SGBRG14BPACK:
+        case V4L2_PIX_FMT_SGRBG14BPACK:
+        case V4L2_PIX_FMT_SRGGB14BPACK:
+            bytesperline = DIV_ROUND_UP(width * 14, 128) * 16;
+            break;
+
+        case V4L2_PIX_FMT_SBGGR14DWA:
+        case V4L2_PIX_FMT_SGBRG14DWA:
+        case V4L2_PIX_FMT_SGRBG14DWA:
+        case V4L2_PIX_FMT_SRGGB14DWA:
+            bytesperline = DIV_ROUND_UP(width, 9) * 16;
+            break;
+
+        case V4L2_PIX_FMT_SBGGR14:
+        case V4L2_PIX_FMT_SGBRG14:
+        case V4L2_PIX_FMT_SGRBG14:
+        case V4L2_PIX_FMT_SRGGB14:
+            bytesperline = DIV_ROUND_UP(width, 8) * 16;
+            break;
+
+        case V4L2_PIX_FMT_SBGGR16:
+        case V4L2_PIX_FMT_SGBRG16:
+        case V4L2_PIX_FMT_SGRBG16:
+        case V4L2_PIX_FMT_SRGGB16:
+            bytesperline = DIV_ROUND_UP(width * 16, 128) * 16;
+            break;
+
+        case V4L2_PIX_FMT_SBGGR24:
+        case V4L2_PIX_FMT_SGBRG24:
+        case V4L2_PIX_FMT_SGRBG24:
+        case V4L2_PIX_FMT_SRGGB24:
+            bytesperline = DIV_ROUND_UP(width * 24, 128) * 16;
+            break;
+
+        default:
+            break;
+        }
+    }
+
+    sizeimage = bytesperline * height;
+
+    if (info->comp_planes == 1) {
+        f->fmt.pix.bytesperline = bytesperline;
+        f->fmt.pix.sizeimage = sizeimage;
+        return 0;
+    }
+
+    f->fmt.pix.bytesperline = bytesperline;
+    f->fmt.pix.sizeimage = sizeimage;
+
+    for (i = 1; i < info->comp_planes; i++) {
+        if (private_fmt) {
+            bytesperline = DIV_ROUND_UP(bytesperline, info->hdiv);
+        } else {
+            bytesperline = info->bpp[i] * DIV_ROUND_UP(width, info->hdiv);
+        }
+        sizeimage = bytesperline * DIV_ROUND_UP(height, info->vdiv);
+        f->fmt.pix.sizeimage += sizeimage;
+    }
+
+    return 0;
+}
+
+static struct v4l2_subdev *vvcam_video_remote_subdev(struct vvcam_video_dev *vvcam_vdev)
+{
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+    pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+    pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+	if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+		return NULL;
+
+    subdev = media_entity_to_v4l2_subdev(pad->entity);
+
+    return subdev;
+}
+
+static int vvcam_video_try_create_pipeline(struct vvcam_video_dev *vvcam_vdev)
+{
+    int ret;
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct v4l2_subdev_format sd_fmt;
+    struct v4l2_subdev_pad_config pad_cfg;
+    struct v4l2_subdev_state sd_state = {
+        .pads = &pad_cfg,
+    };
+
+    if (vvcam_vdev->pipeline) {
+        return 0;
+    }
+
+    ret = vvcam_video_create_pipeline_event(vvcam_vdev);
+    if (ret) {
+        return ret;
+    }
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (!subdev) {
+        return -EINVAL;
+    }
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+    pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+    pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+
+    memset(&sd_fmt, 0, sizeof(sd_fmt));
+    sd_fmt.pad = pad->index;
+    sd_fmt.which = V4L2_SUBDEV_FORMAT_TRY;
+
+    ret = v4l2_subdev_call(subdev, pad, get_fmt, &sd_state, &sd_fmt);
+    if (ret) {
+        return ret;
+    }
+
+    ret = vvcam_video_mfmt_to_vfmt(&sd_fmt, &vvcam_vdev->format);
+    if (ret) {
+        return ret;
+    }
+
+    vvcam_vdev->pipeline = 1;
+
+    return 0;
+}
+
+static int vvcam_video_destroy_pipeline(struct vvcam_video_dev *vvcam_vdev)
+{
+    vvcam_vdev->pipeline = 0;
+    vvcam_video_destroy_pipeline_event(vvcam_vdev);
+    return 0;
+}
+
+static int vvcam_videoc_querycap(struct file *file, void *priv,
+                                struct v4l2_capability *cap)
+{
+	struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+	strlcpy(cap->driver, vvcam_vdev->video->name, sizeof(cap->driver));
+	strlcpy(cap->card, vvcam_vdev->video->name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+            "platform:%s", vvcam_vdev->video->name);
+
+	return 0;
+}
+
+static int vvcam_videoc_enum_fmt_vid_cap(struct file *file, void *priv,
+                                        struct v4l2_fmtdesc *f)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct v4l2_subdev_mbus_code_enum mbus_code;
+    struct v4l2_subdev_pad_config pad_cfg;
+    struct v4l2_subdev_state sd_state = {
+        .pads = &pad_cfg,
+    };
+    int ret = -EINVAL;
+
+    ret = vvcam_video_try_create_pipeline(vvcam_vdev);
+    if (ret) {
+        return ret;
+    }
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&mbus_code, 0, sizeof(mbus_code));
+        mbus_code.pad = pad->index;
+        mbus_code.index = f->index;
+        ret = v4l2_subdev_call(subdev, pad, enum_mbus_code, &sd_state, &mbus_code);
+        if (ret)
+            return ret;
+
+        ret = vvcam_video_mbus_to_fourcc(mbus_code.code, &f->pixelformat, (uint32_t)mbus_code.reserved[0]);
+        if (ret)
+            return ret;
+        memcpy(f->description,  &f->pixelformat, sizeof(f->pixelformat));
+    }
+
+    return ret;
+}
+
+static int vvcam_videoc_try_fmt_vid_cap(struct file *file, void *priv,
+                                        struct v4l2_format *f)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct v4l2_subdev_format sd_fmt;
+    struct v4l2_subdev_pad_config pad_cfg;
+    struct v4l2_subdev_state sd_state = {
+        .pads = &pad_cfg,
+    };
+    int ret;
+
+    if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+    ret = vvcam_video_try_create_pipeline(vvcam_vdev);
+    if (ret) {
+        return ret;
+    }
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (!subdev)
+        return -EINVAL;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+    pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+    pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+
+    memset(&sd_fmt, 0, sizeof(sd_fmt));
+    sd_fmt.pad = pad->index;
+    sd_fmt.which = V4L2_SUBDEV_FORMAT_TRY;
+
+    vvcam_video_vfmt_to_mfmt(f, &sd_fmt);
+
+    ret = v4l2_subdev_call(subdev, pad, set_fmt, &sd_state, &sd_fmt);
+    if (ret) {
+        return ret;
+    }
+
+    ret = vvcam_video_mfmt_to_vfmt(&sd_fmt, f);
+    return ret;
+}
+
+static int vvcam_videoc_s_fmt_vid_cap(struct file *file, void *priv,
+                                    struct v4l2_format *f)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct vb2_queue *queue = &vvcam_vdev->queue;
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct v4l2_subdev_format sd_fmt;
+    struct v4l2_subdev_pad_config pad_cfg;
+    struct v4l2_subdev_state sd_state = {
+        .pads = &pad_cfg,
+    };
+    int ret;
+
+    if (vb2_is_busy(queue))
+        return -EBUSY;
+    ret = vvcam_videoc_try_fmt_vid_cap(file, priv, f);
+    if (ret)
+        return ret;
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (!subdev)
+        return -EINVAL;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+    pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+    pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+
+    memset(&sd_fmt, 0, sizeof(sd_fmt));
+    sd_fmt.pad = pad->index;
+    sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+
+    vvcam_video_vfmt_to_mfmt(f, &sd_fmt);
+
+    ret = v4l2_subdev_call(subdev, pad, set_fmt, &sd_state, &sd_fmt);
+    if (ret) {
+        return ret;
+    }
+
+    vvcam_vdev->format = *f;
+    return 0;
+}
+
+static int vvcam_videoc_g_fmt_vid_cap(struct file *file, void *fh,
+                                struct v4l2_format *f)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+
+    int ret;
+
+    if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+    ret = vvcam_video_try_create_pipeline(vvcam_vdev);
+    if (ret) {
+        return ret;
+    }
+
+    *f = vvcam_vdev->format;
+
+    return 0;
+}
+
+static int vvcam_videoc_reqbufs(struct file *file, void *priv,
+			            struct v4l2_requestbuffers *p)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_reqbufs pad_requbufs;
+    int ret;
+
+    ret = vvcam_video_try_create_pipeline(vvcam_vdev);
+    if (ret)
+        return ret;
+
+    ret = vb2_ioctl_reqbufs(file, priv, p);
+    if (ret) {
+        printk("vb2 reqbufs failed\n");
+        return ret;
+    }
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_requbufs, 0, sizeof(pad_requbufs));
+        pad_requbufs.pad = pad->index;
+        pad_requbufs.num_buffers = p->count;
+        v4l2_subdev_call(subdev, core, ioctl, VVCAM_PAD_REQUBUFS, &pad_requbufs);
+    }
+    return ret;
+}
+
+static int vvcam_videoc_enum_input(struct file *file, void *fh,
+                                struct v4l2_input *input)
+{
+    if (input->index > 0)
+        return -EINVAL;
+
+    strscpy(input->name, "camera", sizeof(input->name));
+    input->type = V4L2_INPUT_TYPE_CAMERA;
+
+	return 0;
+}
+
+static int vvcam_videoc_g_input(struct file *file, void *fh, unsigned int *input)
+{
+    *input = 0;
+	return 0;
+}
+
+static int vvcam_videoc_s_input(struct file *file, void *fh, unsigned int input)
+{
+	return input == 0 ? 0 : -EINVAL;
+}
+
+static int vvcam_videoc_g_selection(struct file *file, void *fh,
+				    struct v4l2_selection *s)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct v4l2_subdev_pad_config pad_cfg;
+    struct v4l2_subdev_selection sd_sel;
+    struct v4l2_subdev_state sd_state = {
+        .pads = &pad_cfg,
+    };
+    int ret = -EINVAL;
+
+
+    if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+        return ret;
+    }
+
+    ret = vvcam_video_try_create_pipeline(vvcam_vdev);
+    if (ret)
+        return ret;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (!subdev)
+        return -EINVAL;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+    pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+    pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+
+    memset(&sd_sel, 0, sizeof(sd_sel));
+    sd_sel.pad = pad->index;
+    sd_sel.which = V4L2_SUBDEV_FORMAT_TRY;
+    sd_sel.target = s->target;
+
+    ret = v4l2_subdev_call(subdev, pad, get_selection, &sd_state, &sd_sel);
+    if (ret != 0) {
+        return ret;
+    }
+
+    s->r = sd_sel.r;
+
+    return ret;
+}
+
+static int vvcam_videoc_s_selection(struct file *file, void *fh,
+				    struct v4l2_selection *s)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct v4l2_subdev_pad_config pad_cfg;
+    struct v4l2_subdev_selection sd_sel;
+    struct v4l2_subdev_state sd_state = {
+        .pads = &pad_cfg,
+    };
+    int ret = -EINVAL;
+
+
+    if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+        return ret;
+    }
+
+    ret = vvcam_video_try_create_pipeline(vvcam_vdev);
+    if (ret)
+        return ret;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (!subdev)
+        return -EINVAL;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+    pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+    pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+
+    memset(&sd_sel, 0, sizeof(sd_sel));
+    sd_sel.pad = pad->index;
+    sd_sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+    sd_sel.target = s->target;
+    sd_sel.r = s->r;
+
+    ret = v4l2_subdev_call(subdev, pad, set_selection, &sd_state, &sd_sel);
+
+    return ret;
+}
+
+static int vvcam_videoc_queryctrl(struct file *file, void *fh,
+				struct v4l2_queryctrl *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_queryctrl pad_query_ctrl;
+    int ret;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_query_ctrl, 0, sizeof(pad_query_ctrl));
+        pad_query_ctrl.pad = pad->index;
+        pad_query_ctrl.query_ctrl = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_QUERYCTRL, &pad_query_ctrl);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_videoc_query_ext_ctrl(struct file *file, void *fh,
+				     struct v4l2_query_ext_ctrl *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_query_ext_ctrl pad_query_ext_ctrl;
+    int ret;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_query_ext_ctrl, 0, sizeof(pad_query_ext_ctrl));
+        pad_query_ext_ctrl.pad = pad->index;
+        pad_query_ext_ctrl.query_ext_ctrl = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_QUERY_EXT_CTRL, &pad_query_ext_ctrl);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_vidioc_g_ctrl(struct file *file, void *fh,
+			     struct v4l2_control *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_control pad_control;
+    int ret;
+    printk("%s %d\n", __func__, __LINE__);
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_control, 0, sizeof(pad_control));
+        pad_control.pad = pad->index;
+        pad_control.control = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_G_CTRL, &pad_control);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_vidioc_s_ctrl(struct file *file, void *fh,
+			     struct v4l2_control *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_control pad_control;
+    int ret;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_control, 0, sizeof(pad_control));
+        pad_control.pad = pad->index;
+        pad_control.control = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_S_CTRL, &pad_control);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_vidioc_g_ext_ctrls(struct file *file, void *fh,
+				  struct v4l2_ext_controls *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_ext_controls pad_ext_controls;
+    int ret;
+    printk("%s %d\n", __func__, __LINE__);
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_ext_controls, 0, sizeof(pad_ext_controls));
+        pad_ext_controls.pad = pad->index;
+        pad_ext_controls.ext_controls = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_G_EXT_CTRLS, &pad_ext_controls);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_vidioc_s_ext_ctrls(struct file *file, void *fh,
+				  struct v4l2_ext_controls *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_ext_controls pad_ext_controls;
+    int ret;
+    printk("%s %d\n", __func__, __LINE__);
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_ext_controls, 0, sizeof(pad_ext_controls));
+        pad_ext_controls.pad = pad->index;
+        pad_ext_controls.ext_controls = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_S_EXT_CTRLS, &pad_ext_controls);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_vidioc_try_ext_ctrls(struct file *file, void *fh,
+				  struct v4l2_ext_controls *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_ext_controls pad_ext_controls;
+    int ret;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_ext_controls, 0, sizeof(pad_ext_controls));
+        pad_ext_controls.pad = pad->index;
+        pad_ext_controls.ext_controls = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_TRY_EXT_CTRLS, &pad_ext_controls);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_vidioc_querymenu(struct file *file, void *fh,
+				struct v4l2_querymenu *a)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_querymenu pad_querymenu;
+    int ret;
+    printk("%s %d\n", __func__, __LINE__);
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_querymenu, 0, sizeof(pad_querymenu));
+        pad_querymenu.pad = pad->index;
+        pad_querymenu.querymenu = a;
+        ret = v4l2_subdev_call(subdev, core, ioctl,
+                        VVCAM_PAD_QUERYMENU, &pad_querymenu);
+
+    } else {
+        return -ENOTTY;
+    }
+
+    return ret;
+}
+
+static int vvcam_videoc_subscribe_event(struct v4l2_fh *fh,
+                                const struct v4l2_event_subscription *sub)
+{
+    int ret;
+    switch (sub->type) {
+	case V4L2_EVENT_CTRL:
+		ret = v4l2_ctrl_subscribe_event(fh, sub);
+        break;
+    case VVCAM_VIDEO_DEAMON_EVENT:
+        ret = v4l2_event_subscribe(fh, sub, 2, NULL);
+        break;
+	default:
+		ret = -EINVAL;
+        break;
+	}
+    return ret;
+}
+
+// static struct v4l2_subdev *find_remote_sensor(struct media_entity *entity)
+// {
+//     struct v4l2_subdev *subdev;
+// 	struct media_pad *pad;
+
+// 	while (1) {
+// 		pad = &entity->pads[0];
+// 		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+// 			return NULL;
+
+// 		pad = media_pad_remote_pad_first(pad);
+// 		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+// 			return NULL;
+
+// 		entity = pad->entity;
+// 		if (entity->function == MEDIA_ENT_F_CAM_SENSOR) {
+//             subdev = media_entity_to_v4l2_subdev(entity);
+//             if (subdev) {
+//                 printk("Found remote sensor subdev: %s\n", subdev->name);
+//                 return subdev;
+//             }
+//         }
+// 	}
+//     return NULL;
+// }
+
+// static int vvcam_get_sensor_mbus_config(struct vvcam_video_dev *vvcam_vdev, void *arg)
+// {
+// 	int ret;
+//     struct v4l2_mbus_config mbus;
+//     sensor_mbus_config  *mbus_config = (sensor_mbus_config*)arg;
+
+//     struct v4l2_subdev *subdev = find_remote_sensor(vvcam_vdev->pad.entity);
+//     if (subdev) {
+//         ret = v4l2_subdev_call(subdev, pad, get_mbus_config, 0, &mbus);
+//         if (ret) {
+//             return ret;
+//         }
+
+//         switch (mbus.type) {
+//             case V4L2_MBUS_PARALLEL:
+//                 mbus_config->type = MBUS_PARALLEL;
+//                 break;
+//             case V4L2_MBUS_BT656:
+//                 mbus_config->type = MBUS_BT656;
+//                 break;            
+//             case V4L2_MBUS_CSI1:
+//                 mbus_config->type = MBUS_CSI1;
+//                 break; 
+//             case V4L2_MBUS_CCP2:
+//                 mbus_config->type = MBUS_CCP2;
+//                 break; 
+//             case V4L2_MBUS_CSI2_DPHY:
+//                 mbus_config->type = MBUS_CSI2_DPHY;
+//                 break; 
+//             case V4L2_MBUS_CSI2_CPHY:
+//                 mbus_config->type = MBUS_CSI2_CPHY;
+//                 break; 
+//             case V4L2_MBUS_DPI:
+//                 mbus_config->type = MBUS_DPI;
+//                 break;                 
+//         }
+//         mbus_config->bus.mipi_csi2.num_data_lanes = mbus.bus.mipi_csi2.num_data_lanes;
+//         return 0;
+//     }
+//     return -1;
+// }
+
+// static long vvcam_ioctl_default(struct file *file, void *fh, bool valid_prio,
+// 				unsigned int cmd, void *arg)
+// {
+// 	struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+// 	bool is_single_dev = false;
+// 	struct v4l2_subdev *sd;
+// 	int ret = -EINVAL;
+
+// 	switch (cmd) {
+// 	    case VVAM_CMD_GET_SENSOR_MBUS_CONFIG:
+//             return vvcam_get_sensor_mbus_config(vvcam_vdev, arg);
+//     }
+
+//     return 0;
+// }
+
+static const struct v4l2_ioctl_ops vvcam_video_ioctl_ops = {
+    .vidioc_querycap            = vvcam_videoc_querycap,
+    .vidioc_enum_fmt_vid_cap    = vvcam_videoc_enum_fmt_vid_cap,
+    .vidioc_try_fmt_vid_cap     = vvcam_videoc_try_fmt_vid_cap,
+    .vidioc_g_fmt_vid_cap       = vvcam_videoc_g_fmt_vid_cap,
+    .vidioc_s_fmt_vid_cap       = vvcam_videoc_s_fmt_vid_cap,
+
+    .vidioc_reqbufs             = vvcam_videoc_reqbufs,
+    .vidioc_querybuf            = vb2_ioctl_querybuf,
+    .vidioc_create_bufs         = vb2_ioctl_create_bufs,
+    .vidioc_qbuf                = vb2_ioctl_qbuf,
+    .vidioc_expbuf              = vb2_ioctl_expbuf,
+    .vidioc_dqbuf               = vb2_ioctl_dqbuf,
+    .vidioc_prepare_buf         = vb2_ioctl_prepare_buf,
+    .vidioc_streamon            = vb2_ioctl_streamon,
+    .vidioc_streamoff           = vb2_ioctl_streamoff,
+
+    .vidioc_enum_input          = vvcam_videoc_enum_input,
+    .vidioc_g_input             = vvcam_videoc_g_input,
+    .vidioc_s_input             = vvcam_videoc_s_input,
+    .vidioc_g_selection         = vvcam_videoc_g_selection,
+    .vidioc_s_selection         = vvcam_videoc_s_selection,
+    // .vidioc_g_parm              = vvcam_videoc_g_parm,
+    // .vidioc_s_parm              = vvcam_videoc_s_parm,
+    // .vidioc_enum_framesizes     = vvcam_videoc_enum_framesizes,
+    // .vidioc_enum_frameintervals = vvcam_videoc_enum_frmaeintervals,
+    .vidioc_queryctrl           = vvcam_videoc_queryctrl,
+    .vidioc_query_ext_ctrl      = vvcam_videoc_query_ext_ctrl,
+    .vidioc_g_ctrl              = vvcam_vidioc_g_ctrl,
+    .vidioc_s_ctrl              = vvcam_vidioc_s_ctrl,
+    .vidioc_g_ext_ctrls         = vvcam_vidioc_g_ext_ctrls,
+    .vidioc_s_ext_ctrls         = vvcam_vidioc_s_ext_ctrls,
+    .vidioc_try_ext_ctrls       = vvcam_vidioc_try_ext_ctrls,
+    .vidioc_querymenu           = vvcam_vidioc_querymenu,
+    .vidioc_subscribe_event     = vvcam_videoc_subscribe_event,
+    .vidioc_unsubscribe_event   = v4l2_event_unsubscribe,
+    // .vidioc_default             = vvcam_ioctl_default,
+};
+
+static __poll_t vvcam_video_poll(struct file *file,
+                        struct poll_table_struct *wait)
+{
+    struct v4l2_fh *fh = file->private_data;
+   
+    if (!list_empty(&fh->subscribed)) {
+        return v4l2_ctrl_poll(file, wait);
+    } else {
+        return vb2_fop_poll(file, wait);
+    }
+}
+
+static int vvcam_video_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    struct v4l2_fh *fh = file->private_data;
+    int ret;
+
+    if (vvcam_vdev->video->queue->owner &&
+        (vvcam_vdev->video->queue->owner == fh)) {
+       return vb2_fop_mmap(file, vma);
+    } else {
+        ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+                            vma->vm_end - vma->vm_start,
+                            vma->vm_page_prot);
+        return ret;
+    }
+}
+
+static int vvcam_video_open(struct file *file)
+{
+    return v4l2_fh_open(file);
+}
+
+static int vvcam_video_release(struct file *file)
+{
+    struct vvcam_video_dev *vvcam_vdev = video_drvdata(file);
+    int ret;
+
+    ret = vb2_fop_release(file);
+    if (vvcam_vdev->video->queue->owner == NULL) {
+        if (vvcam_vdev->pipeline) {
+            vvcam_video_destroy_pipeline(vvcam_vdev);
+        }
+    }
+
+    return ret;
+}
+
+static const struct v4l2_file_operations vvcam_video_fops = {
+	.owner          = THIS_MODULE,
+	.open           = vvcam_video_open,
+	.release        = vvcam_video_release,
+	.poll           = vvcam_video_poll,
+	.unlocked_ioctl = video_ioctl2,
+	.mmap           = vvcam_video_mmap,
+};
+
+static int vvcam_video_vb2_queue_setup(struct vb2_queue *queue,
+                                    unsigned int *num_buffers,
+                                    unsigned int *num_planes,
+                                    unsigned int sizes[],
+                                    struct device *alloc_devs[])
+{
+    struct vvcam_video_dev *vvcam_vdev = queue->drv_priv;
+    struct v4l2_format *format = &vvcam_vdev->format;
+    unsigned int i;
+
+    if (format->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+        if (*num_planes) {
+            if (*num_planes != 1)
+                return -EINVAL;
+            if (sizes[0] < format->fmt.pix.sizeimage)
+                return -EINVAL;
+        } else {
+            *num_planes = 1;
+            sizes[0] = format->fmt.pix.sizeimage;
+        }
+    } else if (format->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+        if (*num_planes) {
+            if (*num_planes != format->fmt.pix_mp.num_planes)
+                return -EINVAL;
+            for (i = 0; i < format->fmt.pix_mp.num_planes; i++) {
+                if (sizes[i] < format->fmt.pix_mp.plane_fmt[i].sizeimage)
+                    return -EINVAL;
+            }
+        } else {
+            *num_planes = format->fmt.pix_mp.num_planes;
+            for (i = 0; i < format->fmt.pix_mp.num_planes; i++) {
+                sizes[i] = format->fmt.pix_mp.plane_fmt[i].sizeimage;
+            }
+        }
+    } else {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int vvcam_video_vb2_buf_prepare(struct vb2_buffer *vb)
+{
+    struct vvcam_video_dev *vvcam_vdev = vb->vb2_queue->drv_priv;
+    struct v4l2_format *format = &vvcam_vdev->format;
+    struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct vvcam_vb2_buffer *buf = container_of(vbuf,
+						  struct vvcam_vb2_buffer, vb);
+    int i;
+
+    if (format->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+        if (vb2_plane_size(vb, 0) < format->fmt.pix.sizeimage)
+            return -EINVAL;
+
+        buf->num_planes = 1;
+        buf->planes[0].dma_addr = vb2_dma_contig_plane_dma_addr(vb, 0);
+        buf->planes[0].size     = format->fmt.pix.sizeimage;
+        buf->sequence           = vb->index;
+
+        vb2_set_plane_payload(vb, 0, buf->planes[0].size);
+
+    } else if (format->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+        buf->num_planes = format->fmt.pix_mp.num_planes;
+        buf->sequence   = vb->index;
+        for (i = 0; i < format->fmt.pix_mp.num_planes; i++) {
+            if (vb2_plane_size(vb, i) < format->fmt.pix_mp.plane_fmt[i].sizeimage)
+                return -EINVAL;
+
+            buf->planes[i].dma_addr = vb2_dma_contig_plane_dma_addr(vb, i);
+            buf->planes[i].size     = format->fmt.pix_mp.plane_fmt[i].sizeimage;
+
+            vb2_set_plane_payload(vb, i, buf->planes[i].size);
+        }
+    } else {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static void vvcam_video_vb2_buf_queue(struct vb2_buffer *vb)
+{
+    struct vvcam_video_dev *vvcam_vdev = vb->vb2_queue->drv_priv;
+    struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct vvcam_vb2_buffer *buf = container_of(vbuf,
+						  struct vvcam_vb2_buffer, vb);
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_buf pad_buf;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&pad_buf, 0, sizeof(pad_buf));
+        pad_buf.pad = pad->index;
+        pad_buf.buf = buf;
+        v4l2_subdev_call(subdev, core, ioctl, VVCAM_PAD_BUF_QUEUE, &pad_buf);
+    }
+
+    return;
+}
+
+static int isp_pipeline_start(struct media_entity *entity, int on)
+{
+    struct v4l2_subdev *subdev;
+	struct media_pad *pad;
+    int ret;
+
+	while (1) {
+		pad = &entity->pads[0];
+
+        if (!pad) {
+            break;
+        }
+
+		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+			return -1;
+            
+		pad = media_pad_remote_pad_first(pad);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+			return -1; 
+
+		entity = pad->entity;
+        subdev = media_entity_to_v4l2_subdev(entity);
+
+        if (subdev) {
+            ret = v4l2_subdev_call(subdev, video, s_stream, on);
+            if(on)
+                printk("%s stream on\n", subdev->name);
+            else
+                printk("%s stream off\n", subdev->name);
+            if (ret != 0) {
+                printk("isp pipeline start failed\n");
+                return ret;
+            }
+        }
+    }
+	return 0;    
+}
+
+static int vvcam_video_vb2_start_streaming(struct vb2_queue *queue,
+                                        unsigned int count)
+{
+   struct vvcam_video_dev *vvcam_vdev = queue->drv_priv;
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_stream_status stream_status;
+    int ret = -EINVAL;
+    // struct media_pipeline pipe;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&stream_status, 0, sizeof(stream_status));
+        stream_status.pad = pad->index;
+        stream_status.status = 1;
+        ret = v4l2_subdev_call(subdev, core, ioctl, VVCAM_PAD_S_STREAM, &stream_status);
+        if(ret) {
+            printk("vvcam_video_vb2_start_streaming failed\n");
+            return ret;
+        }
+        printk("video strat stream ret = %d, name = %s\n", ret, subdev->name);
+        //video_device_pipeline_start(vvcam_vdev->video, &pipe);
+        isp_pipeline_start(&subdev->entity, 1);
+    }
+    return ret;
+}
+
+static void vvcam_video_vb2_stop_streaming(struct vb2_queue *queue)
+{
+    struct vvcam_video_dev *vvcam_vdev = queue->drv_priv;
+    struct media_pad *pad;
+    struct v4l2_subdev *subdev;
+    struct vvcam_pad_stream_status stream_status;
+    int i;
+
+    subdev = vvcam_video_remote_subdev(vvcam_vdev);
+    if (subdev) {
+        isp_pipeline_start(&subdev->entity, 0);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+        pad = media_pad_remote_pad_first(&vvcam_vdev->pad);
+#else
+        pad = media_entity_remote_pad(&vvcam_vdev->pad);
+#endif
+        memset(&stream_status, 0, sizeof(stream_status));
+        stream_status.pad = pad->index;
+        stream_status.status = 0;
+        v4l2_subdev_call(subdev, core, ioctl, VVCAM_PAD_S_STREAM, &stream_status);
+    }
+
+    for(i = 0; i < queue->num_buffers; i++) {
+		if(queue->bufs[i]->state == VB2_BUF_STATE_ACTIVE)
+			vb2_buffer_done(queue->bufs[i], VB2_BUF_STATE_ERROR);
+	}
+
+    
+    return;
+}
+
+static const struct vb2_ops vvcam_video_queue_ops = {
+	.queue_setup     = vvcam_video_vb2_queue_setup,
+	.buf_prepare     = vvcam_video_vb2_buf_prepare,
+	.buf_queue       = vvcam_video_vb2_buf_queue,
+    .wait_prepare    = vb2_ops_wait_prepare,
+    .wait_finish     = vb2_ops_wait_finish,
+	.start_streaming = vvcam_video_vb2_start_streaming,
+	.stop_streaming  = vvcam_video_vb2_stop_streaming,
+};
+
+static int vvcam_video_queue_init(struct vvcam_video_dev *vvcam_vdev)
+{
+    int ret = 0;
+    struct vb2_queue *queue;
+
+    queue = &vvcam_vdev->queue;
+    queue->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    vvcam_vdev->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    queue->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
+    queue->drv_priv = vvcam_vdev;
+    queue->ops = &vvcam_video_queue_ops;
+    queue->mem_ops = &vb2_dma_contig_memops;
+    queue->buf_struct_size = sizeof(struct vvcam_vb2_buffer);
+	queue->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+    queue->lock = &vvcam_vdev->video_lock;
+    queue->dev = vvcam_vdev->vvcam_mdev->dev;
+
+    ret = vb2_queue_init(queue);
+    if (ret) {
+        dev_err(vvcam_vdev->vvcam_mdev->dev, "vb2 queue init failed\n");
+        return ret;
+    }
+    vvcam_vdev->video->queue = queue;
+
+    return 0;
+}
+
+static int vvcam_video_link_setup(struct media_entity *entity,
+                const struct media_pad *local,
+				const struct media_pad *remote, u32 flags)
+{
+	return 0;
+}
+
+
+static const struct media_entity_operations vvcam_video_entity_ops = {
+	.link_setup     = vvcam_video_link_setup,
+	.link_validate  = v4l2_subdev_link_validate,
+};
+
+int 
+vvcam_video_register(struct vvcam_media_dev *vvcam_mdev, int port)
+{
+    int ret = 0;
+    struct vvcam_video_dev *vvcam_vdev;
+
+    vvcam_vdev = devm_kzalloc(vvcam_mdev->dev,
+                sizeof(struct vvcam_video_dev), GFP_KERNEL);
+    if (!vvcam_vdev)
+        return -ENOMEM;
+
+    mutex_init(&vvcam_vdev->video_lock);
+    vvcam_vdev->vvcam_mdev = vvcam_mdev;
+    vvcam_vdev->video_params = vvcam_mdev->video_params[port];
+
+    vvcam_vdev->video = video_device_alloc();
+    if (!vvcam_vdev->video) {
+        dev_err(vvcam_mdev->dev, "could not alloc video device\n");
+        ret = -ENOMEM;
+        goto error_free_vvcam_vdev;
+    }
+
+    snprintf(vvcam_vdev->video->name, sizeof(vvcam_vdev->video->name),
+                "%s.%d.%d", VVCAM_VIDEO_NAME, vvcam_mdev->id, port);
+
+    vvcam_vdev->video->fops          = &vvcam_video_fops;
+    vvcam_vdev->video->ioctl_ops     = &vvcam_video_ioctl_ops;
+    vvcam_vdev->video->release       = video_device_release_empty;
+    vvcam_vdev->video->v4l2_dev      = &vvcam_mdev->v4l2_dev;
+    //vvcam_vdev->video->lock          = &vvcam_vdev->video_lock;
+    vvcam_vdev->video->device_caps   = V4L2_CAP_VIDEO_CAPTURE |
+                                       V4L2_CAP_STREAMING;
+    vvcam_vdev->video->minor         = -1;
+
+    video_set_drvdata(vvcam_vdev->video, vvcam_vdev);
+
+    vvcam_vdev->video->entity.name     = vvcam_vdev->video->name;
+    vvcam_vdev->video->entity.obj_type = MEDIA_ENTITY_TYPE_VIDEO_DEVICE;
+	vvcam_vdev->video->entity.function = MEDIA_ENT_F_IO_V4L;
+	vvcam_vdev->video->entity.ops      = &vvcam_video_entity_ops;
+	vvcam_vdev->pad.flags              = MEDIA_PAD_FL_SINK;
+
+    ret = media_entity_pads_init(&vvcam_vdev->video->entity, 1, &vvcam_vdev->pad);
+	if (ret) {
+		dev_err(vvcam_mdev->dev, "entity pad init error\n");
+		goto error_video_device_release;
+	}
+
+    ret = vvcam_video_queue_init(vvcam_vdev);
+    if (ret) {
+        dev_err(vvcam_mdev->dev, "queue init error\n");
+		goto err_media_entity_cleanup;
+    }
+
+	ret = video_register_device(vvcam_vdev->video, VFL_TYPE_VIDEO, -1);
+    if (ret) {
+		dev_err(vvcam_mdev->dev, "video register device error\n");
+		goto err_media_entity_cleanup;
+    }
+    
+    vvcam_vdev->event_shm.virt_addr = (void *)__get_free_pages(GFP_KERNEL, 0);
+    vvcam_vdev->event_shm.size = PAGE_SIZE;
+    memset(vvcam_vdev->event_shm.virt_addr, 0, vvcam_vdev->event_shm.size);
+    vvcam_vdev->event_shm.phy_addr = virt_to_phys(vvcam_vdev->event_shm.virt_addr);
+    mutex_init(&vvcam_vdev->event_shm.event_lock);
+
+    vvcam_mdev->video_devs[port] = vvcam_vdev;
+
+    return 0;
+
+err_media_entity_cleanup:
+    media_entity_cleanup(&vvcam_vdev->video->entity);
+
+error_video_device_release:
+    video_device_release(vvcam_vdev->video);
+    vvcam_vdev->video = NULL;
+
+error_free_vvcam_vdev:
+    devm_kfree(vvcam_mdev->dev,vvcam_vdev);
+
+    return ret;
+}
+
+int vvcam_video_unregister(struct vvcam_media_dev *vvcam_mdev, int port)
+{
+    struct vvcam_video_dev *vvcam_vdev = vvcam_mdev->video_devs[port];
+
+    if (vvcam_vdev == NULL)
+        return 0;
+
+    video_unregister_device(vvcam_vdev->video);
+    media_entity_cleanup(&vvcam_vdev->video->entity);
+    video_device_release(vvcam_vdev->video);
+    devm_kfree(vvcam_mdev->dev,vvcam_vdev);
+    vvcam_mdev->video_devs[port] = NULL;
+
+    free_pages((unsigned long)vvcam_vdev->event_shm.virt_addr, 0);
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_register.h b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_register.h
new file mode 100644
index 000000000000..89b87f59d6a5
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2/video/vvcam_video_register.h
@@ -0,0 +1,63 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VIDEO_REGISTER_H__
+#define __VVCAM_VIDEO_REGISTER_H__
+#include "vvcam_video_driver.h"
+
+int vvcam_video_register(struct vvcam_media_dev *vvcam_mdev, int port);
+int vvcam_video_unregister(struct vvcam_media_dev *vvcam_mdev, int port);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2_insmod.sh b/drivers/media/platform/eswin/isp/vvcam/v4l2_insmod.sh
new file mode 100644
index 000000000000..e8c73ed5a415
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2_insmod.sh
@@ -0,0 +1,121 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+#!/bin/sh
+
+param_sensor_num="2sensor"
+param_mcm_mask="false"
+param_mss="false"
+
+if [ -n "$1" ];then
+    if [ "$1" = "mcm" ];then
+        param_mcm_mask="true"
+    elif [ "$1" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$1" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+if [ -n "$2" ];then
+    if [ "$2" = "mcm" ];then
+        param_mcm_mask="true"
+    elif [ "$2" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$2" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+if [ -n "$3" ];then
+    if [ "$3" = "mcm" ];then
+        param_mcm_mask="true"
+    elif [ "$3" = "4sensor" ];then
+        param_sensor_num="4sensor"
+    elif [ "$3" = "mss" ];then
+        param_mss="true"
+    fi
+fi
+
+modprobe i2c-dev
+
+if [ "$param_sensor_num" = "4sensor" ];then
+    insmod ./i2c/vvcam_i2c.ko devices_mask=0x0f
+    insmod ./mipi/vvcam_mipi.ko devices_mask=0x0f
+else
+    insmod ./i2c/vvcam_i2c.ko devices_mask=0x03
+    insmod ./mipi/vvcam_mipi.ko devices_mask=0x03
+fi
+
+insmod ./isp/vvcam_isp.ko
+insmod ./vb/vvcam_vb.ko
+modprobe videobuf2-v4l2
+modprobe videodev
+modprobe mc
+modprobe v4l2-async
+modprobe videobuf2-dma-contig
+modprobe v4l2-fwnode
+modprobe videobuf2-common
+insmod ./v4l2/isp/vvcam_isp_subdev.ko
+
+if [ "$param_mcm_mask" = "true" ];then
+    insmod ./v4l2/video/vvcam_video.ko mcm_mask=0x01
+else
+    insmod ./v4l2/video/vvcam_video.ko
+fi
+
+if [ "$param_mss" = "true" ];then
+    insmod ./mi_subsystem/vvcam_mss.ko
+fi
diff --git a/drivers/media/platform/eswin/isp/vvcam/v4l2_rmmod.sh b/drivers/media/platform/eswin/isp/vvcam/v4l2_rmmod.sh
new file mode 100644
index 000000000000..1a8e882d14f7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/v4l2_rmmod.sh
@@ -0,0 +1,64 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+#!/bin/sh
+
+rmmod ./i2c/vvcam_i2c.ko
+rmmod ./mipi/vvcam_mipi.ko
+rmmod ./isp/vvcam_isp.ko
+rmmod ./vb/vvcam_vb.ko
+rmmod ./v4l2/video/vvcam_video.ko
+rmmod ./v4l2/isp/vvcam_isp_subdev.ko
+rmmod ./mi_subsystem/vvcam_mss.ko
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/Makefile b/drivers/media/platform/eswin/isp/vvcam/vb/Makefile
new file mode 100644
index 000000000000..6783d10a87ee
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/Makefile
@@ -0,0 +1,69 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+ifeq ($(VVCAM_PATH),)
+        VVCAM_PATH:=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))/../../vvcam
+        export VVCAM_PATH
+endif
+
+include $(VVCAM_PATH)/Param.mk
+TARGET = vvcam_vb
+
+obj-$(CONFIG_VIDEO_ESWIN_ISP) += $(TARGET).o
+$(TARGET)-objs += vvcam_vb_platform.o
+$(TARGET)-objs += vvcam_vb_driver.o
+$(TARGET)-objs += vvcam_mmz.o
+$(TARGET)-objs += vvcam_cma.o
+$(TARGET)-objs += vvcam_vb_procfs.o
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_cma.c b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_cma.c
new file mode 100644
index 000000000000..6709e22df0ae
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_cma.c
@@ -0,0 +1,237 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/pm_runtime.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+
+#include "vvcam_cma.h"
+#include "vvcam_vb_driver.h"
+
+static int vvcam_cma_alloc(struct file *file,
+            unsigned long *paddr, unsigned long size, char *name);
+static int vvcam_cma_mmap(struct file *file, struct vm_area_struct *vma);
+static int vvcam_cma_free(struct file *file, unsigned long paddr);
+static int vvcam_cma_free_all(struct file *file);
+
+static void *vvcam_alloc_cma_buf_attr(struct vvcam_cma *cma)
+{
+    return devm_kzalloc(cma->dev, sizeof(struct vvcam_cma_buf), GFP_KERNEL);
+}
+
+static void vvcam_free_cma_buf_attr(struct vvcam_cma *cma,
+            struct vvcam_cma_buf *cma_buf)
+{
+    devm_kfree(cma->dev, cma_buf);
+}
+
+int vvcam_cma_create(void *allocator, struct device *dev)
+{
+    struct vvcam_cma *cma;
+    struct vvcam_vb_mm_allocator *pallocator =
+                (struct vvcam_vb_mm_allocator *)allocator;
+    int ret = 0;
+
+    cma = (struct vvcam_cma *)devm_kzalloc(dev, sizeof(struct vvcam_cma), GFP_KERNEL);
+    if (!cma)
+        return -ENOMEM;
+    cma->dev = dev;
+
+    if (!dev->dma_mask)
+        dev->dma_mask = &dev->coherent_dma_mask;
+    ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
+    if (ret) {
+        return ret;
+    }
+    
+#if IS_ENABLED(CONFIG_ARCH_ESWIN_EIC770X_SOC_FAMILY)
+    dev->dma_coherent = false;
+#endif
+
+    INIT_LIST_HEAD(&cma->buf_list);
+
+    pallocator->mm_dev = cma;
+    pallocator->alloc = vvcam_cma_alloc;
+    pallocator->mmap = vvcam_cma_mmap;
+    pallocator->free = vvcam_cma_free;
+    pallocator->free_all = vvcam_cma_free_all;
+    return 0;
+}
+
+int vvcam_cma_alloc(struct file *file,
+             unsigned long *paddr, unsigned long size, char *name)
+{
+    struct vvcam_cma *cma;
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_cma_buf *cma_buf = NULL;
+
+    vb_dev = file->private_data;
+    cma = vb_dev->allocator.mm_dev;
+
+    cma_buf = vvcam_alloc_cma_buf_attr(cma);
+    if (!cma_buf)
+        return -ENOMEM;
+
+    cma_buf->virt_addr =
+        dma_alloc_coherent(cma->dev, size, &(cma_buf->phys_addr), GFP_KERNEL);
+      
+    if (!cma_buf->virt_addr) {
+        vvcam_free_cma_buf_attr(cma, cma_buf);
+        return -ENOMEM;
+    }
+    cma_buf->size = size;
+    strncpy(cma_buf->name, name, sizeof(cma_buf->name));
+    cma_buf->private = file;
+    list_add_tail(&cma_buf->list, &cma->buf_list);
+
+    *paddr = cma_buf->phys_addr;
+
+    return 0;
+}
+
+static int vvcam_cma_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    unsigned long size = 0;
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_cma *cma;
+    struct vvcam_cma_buf *cma_buf = NULL;
+
+    vb_dev = file->private_data;
+    cma = vb_dev->allocator.mm_dev;
+
+    if (list_empty(&cma->buf_list))
+        return -1;
+
+    size = vma->vm_end - vma->vm_start;
+
+    list_for_each_entry(cma_buf, &cma->buf_list, list) {
+        if (!cma_buf) {
+            continue;
+        }
+        if ((cma_buf->phys_addr >> PAGE_SHIFT) == vma->vm_pgoff) {
+            break;
+        }
+    }
+
+    if (!cma_buf)
+        return -EAGAIN;
+
+    vma->vm_pgoff = 0;
+    return dma_mmap_coherent(cma->dev, vma, cma_buf->virt_addr,
+            cma_buf->phys_addr, size);
+}
+
+static int vvcam_cma_free(struct file *file, unsigned long paddr)
+{
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_cma *cma;
+    struct vvcam_cma_buf *cma_buf = NULL;
+
+    vb_dev = file->private_data;
+    cma = vb_dev->allocator.mm_dev;
+    if (list_empty(&cma->buf_list))
+        return -1;
+
+    list_for_each_entry(cma_buf, &cma->buf_list, list) {
+        if (cma_buf->phys_addr == paddr) {
+            break;
+        }
+    }
+
+    if (cma_buf == NULL)
+        return -1;
+
+    list_del(&cma_buf->list);
+    dma_free_coherent(cma->dev, cma_buf->size,
+                cma_buf->virt_addr, cma_buf->phys_addr);
+    vvcam_free_cma_buf_attr(cma, cma_buf);
+
+    return 0;
+}
+
+int vvcam_cma_free_all(struct file *file)
+{
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_cma *cma;
+    struct vvcam_cma_buf *cma_buf = NULL, *tmp_cma_buf;
+
+    vb_dev = file->private_data;
+    cma = vb_dev->allocator.mm_dev;
+
+    if (list_empty(&cma->buf_list))
+        return 0;
+
+    list_for_each_entry_safe(cma_buf, tmp_cma_buf, &cma->buf_list, list) {
+        if (!cma)
+            continue;
+        if (cma_buf->private == file) {
+            list_del(&cma_buf->list);
+            dma_free_coherent(cma->dev, cma_buf->size,
+                        cma_buf->virt_addr, cma_buf->phys_addr);
+            vvcam_free_cma_buf_attr(cma, cma_buf);
+        }
+    }
+
+    return 0;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_cma.h b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_cma.h
new file mode 100644
index 000000000000..cf5272797020
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_cma.h
@@ -0,0 +1,77 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_CMA_H__
+#define __VVCAM_CMA_H__
+
+#define VVCAM_CMA_BUF_NAME_LEN 32
+
+struct vvcam_cma_buf {
+    char name[VVCAM_CMA_BUF_NAME_LEN];
+    struct list_head list;
+    dma_addr_t phys_addr;
+    unsigned long size;
+    void *virt_addr;
+    void *private;
+};
+
+struct vvcam_cma {
+    struct device *dev;
+    struct list_head buf_list;
+    spinlock_t lock;
+};
+
+int vvcam_cma_create(void *allocator, struct device *dev);
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_mmz.c b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_mmz.c
new file mode 100644
index 000000000000..b8bc9950370d
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_mmz.c
@@ -0,0 +1,378 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/pm_runtime.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+
+#include "vvcam_mmz.h"
+#include "vvcam_vb_driver.h"
+
+static int vvcam_mmz_alloc(struct file *file,
+            unsigned long *paddr, unsigned long size, char *name);
+static int vvcam_mmz_mmap(struct file *file, struct vm_area_struct *vma);
+static int vvcam_mmz_free(struct file *file, unsigned long paddr);
+static int vvcam_mmz_free_all(struct file *file);
+
+static void *vvcam_alloc_mmb_attr(struct vvcam_mmz *mmz)
+{
+    return devm_kzalloc(mmz->dev, sizeof(struct vvcam_mmb), GFP_KERNEL);
+}
+
+static void vvcam_free_mmb_attr(struct vvcam_mmz *mmz, struct vvcam_mmb *mmb)
+{
+    devm_kfree(mmz->dev, mmb);
+}
+
+static int vvcam_get_order(unsigned long size)
+{
+    int order;
+    unsigned long pfn  = (size >> PAGE_SHIFT);
+
+    order = -1;
+    do {
+        pfn >>= 1;
+        order++;
+    } while(pfn);
+
+    return order;
+}
+
+int vvcam_mmz_create(void *allocator, struct device *dev,
+                    const unsigned long phys_addr, const unsigned long size)
+{
+    int i;
+    unsigned int order;
+    struct vvcam_mmb *mmb;
+    struct vvcam_mmz *mmz;
+    struct vvcam_vb_mm_allocator *pmm_agent =
+                    (struct vvcam_vb_mm_allocator *)allocator;
+
+    mmz =(struct vvcam_mmz *)devm_kzalloc(dev,
+                sizeof(struct vvcam_mmz), GFP_KERNEL);
+    if (!mmz)
+        return -ENOMEM;
+    mmz->dev = dev;
+
+    mmz->size = size;
+    mmz->paddr = phys_addr;
+
+    mmz->zone_start_pfn = (phys_addr >> PAGE_SHIFT);
+    mmz->spanned_pages  = (size >> PAGE_SHIFT);
+
+    order = vvcam_get_order(size);
+    if (order >= VVCAM_MAX_ORDER)
+        return -1;
+
+    mmz->managed_pages = (1UL << order);
+    for (i = 0; i < VVCAM_MAX_ORDER; i++) {
+        INIT_LIST_HEAD(&mmz->free_area[i].free_list);
+        mmz->free_area[i].nr_free = 0;
+    }
+
+    INIT_LIST_HEAD(&mmz->mmb_list);
+    spin_lock_init(&mmz->lock);
+
+    mmb = vvcam_alloc_mmb_attr(mmz);
+    if (mmb == NULL)
+        return -ENOMEM;
+
+    mmb->mmz = mmz;
+    mmb->phys_addr = phys_addr;
+    mmb->size = (mmz->managed_pages << PAGE_SHIFT);
+    mmb->order  = order;
+
+    list_add_tail(&mmb->list, &mmz->free_area[mmb->order].free_list);
+    mmz->free_area[mmb->order].nr_free += 1;
+
+    pmm_agent->mm_dev = mmz;
+    pmm_agent->alloc = vvcam_mmz_alloc;
+    pmm_agent->mmap = vvcam_mmz_mmap;
+    pmm_agent->free = vvcam_mmz_free;
+    pmm_agent->free_all = vvcam_mmz_free_all;
+    return 0;
+}
+
+static int vvcam_mmz_split_block(struct vvcam_mmz *mmz, struct vvcam_mmb *mmb)
+{
+    unsigned int order;
+    unsigned long size;
+    struct vvcam_mmb *block;
+
+    if (mmb->order == VVCAM_MIN_ORDER)
+        return -1;
+
+    order = mmb->order - 1;
+    size  = mmb->size / 2;
+
+    block = vvcam_alloc_mmb_attr(mmz);
+    if (block == NULL)
+        return -ENOMEM;
+
+    block->mmz       = mmz;
+    block->phys_addr = mmb->phys_addr + size;
+    block->size      = size;
+    block->order     = order;
+
+    mmb->size        = size;
+    mmb->order       = order;
+
+    list_add_tail(&mmb->list, &mmz->free_area[mmb->order].free_list);
+    mmz->free_area[mmb->order].nr_free += 1;
+
+    list_add_tail(&block->list, &mmz->free_area[block->order].free_list);
+    mmz->free_area[block->order].nr_free += 1;
+
+    return 0;
+}
+
+static struct vvcam_mmb *vvcam_mmz_alloc_block(struct vvcam_mmz *mmz, int order)
+{
+    struct vvcam_mmb *mmb = NULL;
+    struct vvcam_mmb *block = NULL;
+    int i = 0;
+
+    if (order >= VVCAM_MAX_ORDER)
+        return NULL;
+
+    if (mmz->free_area[order].nr_free) {
+        mmb = list_first_entry(&mmz->free_area[order].free_list,
+                            struct vvcam_mmb, list);
+        list_del(&mmb->list);
+        mmz->free_area[order].nr_free--;
+        return mmb;
+    } else {
+        block = NULL;
+
+        for (i = order; i < VVCAM_MAX_ORDER; i++) {
+            if (mmz->free_area[i].nr_free) {
+                block = list_first_entry(&mmz->free_area[i].free_list,
+                            struct vvcam_mmb, list);
+                list_del(&block->list);
+                mmz->free_area[i].nr_free--;
+                break;
+            }
+        }
+        if (block == NULL)
+            return NULL;
+
+        vvcam_mmz_split_block(mmz, block);
+        mmb = vvcam_mmz_alloc_block(mmz, order);
+        return mmb;
+    }
+}
+
+struct vvcam_mmb * vvcam_mmz_get_buddy(struct vvcam_mmb *mmb)
+{
+    struct vvcam_mmz *mmz = mmb->mmz;
+    unsigned int block_index;
+    unsigned int buddy_index;
+    unsigned long buddy_phys_addr;
+    struct vvcam_mmb *block = NULL, *buddy_block = NULL;
+
+    block_index = ((mmb->phys_addr >> PAGE_SHIFT) - mmz->zone_start_pfn) / (1 << mmb->order);
+    buddy_index = block_index ^ 0x01;
+
+    buddy_phys_addr = ((buddy_index * (1 << mmb->order) + mmz->zone_start_pfn) << PAGE_SHIFT);
+
+    if (mmz->free_area[mmb->order].nr_free) {
+        list_for_each_entry(block, &mmz->free_area[mmb->order].free_list, list) {
+            if (!block) {
+                break;
+            }
+            if (block->phys_addr == buddy_phys_addr) {
+                buddy_block = block;
+                break;
+            }
+        }
+    }
+
+    return buddy_block;
+}
+
+static void vvcam_mmz_free_block(struct vvcam_mmz *mmz, struct vvcam_mmb *mmb)
+{
+    struct vvcam_mmb *buddy_block = NULL;
+
+    buddy_block = vvcam_mmz_get_buddy(mmb);
+
+    if (buddy_block == NULL) {
+        list_add_tail(&mmb->list, &mmz->free_area[mmb->order].free_list);
+        mmz->free_area[mmb->order].nr_free++;
+        return;
+    }
+
+    list_del(&buddy_block->list);
+    mmz->free_area[buddy_block->order].nr_free--;
+
+    //merge block;
+    mmb->phys_addr =  (buddy_block->phys_addr > mmb->phys_addr) ?
+                        mmb->phys_addr : buddy_block->phys_addr;
+    mmb->size = mmb->size * 2;
+    mmb->order += 1;
+    vvcam_free_mmb_attr(mmz, buddy_block);
+
+    vvcam_mmz_free_block(mmz, mmb);
+    return;
+}
+
+static int vvcam_mmz_alloc(struct file *file,
+            unsigned long *paddr, unsigned long size, char *name)
+{
+    struct vvcam_mmz *mmz;
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_mmb *mmb;
+    int order;
+
+    vb_dev = file->private_data;
+    mmz = vb_dev->allocator.mm_dev;
+
+    if (size < (1UL << PAGE_SHIFT))
+        size = (1UL << PAGE_SHIFT);
+
+    order = vvcam_get_order(size);
+
+    if (size > (1 << order) << PAGE_SHIFT) {
+        order += 1;
+    }
+
+    mmb = vvcam_mmz_alloc_block(mmz, order);
+    if (mmb == NULL)
+        return -1;
+
+    strncpy(mmb->name, name, sizeof(mmb->name));
+    mmb->private = file;
+    list_add_tail(&mmb->list, &mmz->mmb_list);
+
+    *paddr = mmb->phys_addr;
+    return 0;
+}
+
+static int vvcam_mmz_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    unsigned long start = 0;
+    unsigned long pfn = 0;
+    unsigned long vm_len = 0;
+
+    pfn = (start >> PAGE_SHIFT) + vma->vm_pgoff;
+    vm_len =  vma->vm_end - vma->vm_start;
+    vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+
+    return remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
+}
+
+static int vvcam_mmz_free(struct file *file, unsigned long paddr)
+{
+    struct vvcam_mmz *mmz;
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_mmb *mmb = NULL, *block = NULL;
+
+    vb_dev = file->private_data;
+    mmz = vb_dev->allocator.mm_dev;
+
+    if (list_empty(&mmz->mmb_list))
+        return -1;
+
+    list_for_each_entry(block, &mmz->mmb_list, list) {
+        if (block->phys_addr == paddr) {
+            mmb = block;
+            break;
+        }
+    }
+
+    if (mmb == NULL)
+        return -1;
+
+    list_del(&mmb->list);
+
+    vvcam_mmz_free_block(mmz, mmb);
+
+    return 0;
+}
+
+int vvcam_mmz_free_all(struct file *file)
+{
+    struct vvcam_mmz *mmz;
+    struct vvcam_vb_dev *vb_dev;
+    struct vvcam_mmb *mmb = NULL, *block;
+
+    vb_dev = file->private_data;
+    mmz = vb_dev->allocator.mm_dev;
+
+    if (list_empty(&mmz->mmb_list))
+        return 0;
+
+    list_for_each_entry_safe(mmb, block, &mmz->mmb_list, list) {
+        if (mmb->private == file) {
+            list_del(&mmb->list);
+            vvcam_mmz_free_block(mmz, mmb);
+        }
+    }
+
+    return 0;
+}
+
+
+
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_mmz.h b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_mmz.h
new file mode 100644
index 000000000000..5c8b33389394
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_mmz.h
@@ -0,0 +1,98 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_MMZ_H__
+#define __VVCAM_MMZ_H__
+
+#define VVCAM_MMB_NAME_LEN 32
+#define VVCAM_MAX_ORDER 18
+#define VVCAM_MIN_ORDER 0
+
+struct vvcam_free_area {
+    struct list_head free_list;
+    unsigned long nr_free;
+};
+
+struct vvcam_mmb {
+    char name[VVCAM_MMB_NAME_LEN];
+    struct vvcam_mmz *mmz;
+    struct list_head list;
+    unsigned long phys_addr;
+    unsigned long size;
+    unsigned int order;
+
+    void *private;
+};
+
+struct vvcam_mmz {
+    struct device *dev;
+    unsigned long paddr;
+    unsigned long size;
+    /*
+     * zone_start_pfn = zone_start_pfn >> PAGE_SHIFT
+     * managed_pages = zone_end_pfn - zone_start_pfn
+     */
+    unsigned long zone_start_pfn;
+    unsigned long spanned_pages;
+    unsigned long managed_pages;
+
+    struct vvcam_free_area free_area[VVCAM_MAX_ORDER];
+    struct list_head mmb_list;
+    spinlock_t lock;
+};
+
+int vvcam_mmz_create(void *allocator, struct device *dev,
+            const unsigned long phys_addr, const unsigned long size);
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb.h b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb.h
new file mode 100644
index 000000000000..651f0659fc41
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb.h
@@ -0,0 +1,74 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VB_H__
+#define __VVCAM_VB_H__
+
+#define VVCAM_VB_NAME_LEN 32
+
+typedef struct {
+    unsigned long paddr;
+    unsigned long size;
+    void *mapped;
+    char name[VVCAM_VB_NAME_LEN];
+} vvcam_vb_t;
+
+#define VVCAM_VB_IOC_MAGIC 'v'
+#define VVCAM_VB_ALLOC  _IOWR(VVCAM_VB_IOC_MAGIC, 0x01, vvcam_vb_t)
+#define VVCAM_VB_FREE   _IOWR(VVCAM_VB_IOC_MAGIC, 0x02, vvcam_vb_t)
+#define VVCAM_VB_CACHE_FLUSH   _IOWR(VVCAM_VB_IOC_MAGIC, 0x03, vvcam_vb_t)
+#define VVCAM_VB_CACHE_INVALID _IOWR(VVCAM_VB_IOC_MAGIC, 0x04, vvcam_vb_t)
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_driver.c b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_driver.c
new file mode 100644
index 000000000000..cf1b1877c08f
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_driver.c
@@ -0,0 +1,427 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/pm_runtime.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/i2c.h>
+#include <media/v4l2-subdev.h>
+#include "vvcam_vb_driver.h"
+#include "vvcam_vb.h"
+#include "vvcam_vb_platform.h"
+#include "vvcam_mmz.h"
+#include "vvcam_cma.h"
+#include "vvcam_vb_procfs.h"
+
+static int enable_cma = 1;
+
+static int vvcam_vb_alloc(struct file *file, vvcam_vb_t *vb_mem)
+{
+    struct vvcam_vb_dev *vb_dev;
+
+    vb_dev = file->private_data;
+    if (!vb_dev->allocator.mm_dev || !vb_dev->allocator.alloc)
+        return -EPERM;
+
+    return vb_dev->allocator.alloc(file,
+            &vb_mem->paddr, vb_mem->size, vb_mem->name);
+}
+
+static int vvcam_vb_free(struct file *file, vvcam_vb_t *vb_mem)
+{
+    struct vvcam_vb_dev *vb_dev;
+
+    vb_dev = file->private_data;
+    if (!vb_dev->allocator.mm_dev || !vb_dev->allocator.free)
+        return -EPERM;
+
+    return vb_dev->allocator.free(file, vb_mem->paddr);
+}
+
+static int vvcam_vb_cache_flush(struct file *file, vvcam_vb_t *vb_mem)
+{
+    struct vvcam_vb_dev *vb_dev = file->private_data;
+    unsigned long pfn = __phys_to_pfn(vb_mem->paddr);
+    struct page *page = pfn_to_page(pfn);
+    unsigned long offset = offset_in_page(vb_mem->paddr);
+    size_t size = PAGE_ALIGN(vb_mem->size);
+    dma_addr_t dma_addr = 0;
+
+    dma_addr = dma_map_page(vb_dev->dev, page, offset, size, DMA_TO_DEVICE);
+    if (dma_mapping_error(vb_dev->dev, dma_addr)) {
+        dev_err(vb_dev->dev, "ma map page failed.\n");
+        return -EINVAL;
+    }
+
+    dma_sync_single_for_device(vb_dev->dev, dma_addr, size, DMA_TO_DEVICE);
+
+    dma_unmap_page(vb_dev->dev, dma_addr, size, DMA_TO_DEVICE);
+
+    return 0;
+}
+
+static int vvcam_vb_cache_invalid(struct file *file, vvcam_vb_t *vb_mem)
+{
+    struct vvcam_vb_dev *vb_dev = file->private_data;
+    unsigned long pfn = __phys_to_pfn(vb_mem->paddr);
+    struct page *page = pfn_to_page(pfn);
+    unsigned long offset = offset_in_page(vb_mem->paddr);
+    size_t size = PAGE_ALIGN(vb_mem->size);
+    dma_addr_t dma_addr = 0;
+
+    dma_addr = dma_map_page(vb_dev->dev, page, offset, size, DMA_FROM_DEVICE);
+    if (dma_mapping_error(vb_dev->dev, dma_addr)) {
+        dev_err(vb_dev->dev, "ma map page failed.\n");
+        return -EINVAL;
+    }
+
+    dma_sync_single_for_cpu(vb_dev->dev, dma_addr, size, DMA_FROM_DEVICE);
+
+    dma_unmap_page(vb_dev->dev, dma_addr, size, DMA_FROM_DEVICE);
+
+    return 0;
+}
+
+static int vvcam_vb_open(struct inode *inode, struct file *file)
+{
+    struct vvcam_vb_dev *vb_dev;
+    struct miscdevice *pmisc_dev = file->private_data;
+
+    vb_dev = container_of(pmisc_dev, struct vvcam_vb_dev, miscdev);
+    if (!vb_dev)
+        return -ENOMEM;
+
+    file->private_data = vb_dev;
+
+    return 0;
+}
+
+static int vvcam_vb_release(struct inode *inode, struct file *file)
+{
+    struct vvcam_vb_dev *vb_dev;
+
+    vb_dev = file->private_data;
+    if (!vb_dev->allocator.mm_dev || !vb_dev->allocator.free_all)
+        return -EPERM;
+
+    mutex_lock(&vb_dev->mlock);
+    vb_dev->allocator.free_all(file);
+    mutex_unlock(&vb_dev->mlock);
+
+    return 0;
+}
+
+static long vvcam_vb_ioctl(struct file *file,
+                        unsigned int cmd, unsigned long arg)
+{
+    struct vvcam_vb_dev *vb_dev;
+    vvcam_vb_t vb_mem;
+    int ret = 0;
+
+    vb_dev = file->private_data;
+
+    mutex_lock(&vb_dev->mlock);
+
+    switch(cmd) {
+    case VVCAM_VB_ALLOC:
+        ret = copy_from_user(&vb_mem, (void __user *)arg, sizeof(vb_mem));
+        if (ret)
+            break;
+        ret = vvcam_vb_alloc(file, &vb_mem);
+        if (ret)
+            break;
+        ret = copy_to_user((void __user *)arg, &vb_mem, sizeof(vb_mem));
+        break;
+    case VVCAM_VB_FREE:
+        ret = copy_from_user(&vb_mem, (void __user *)arg, sizeof(vb_mem));
+        if (ret)
+            break;
+        ret = vvcam_vb_free(file, &vb_mem);
+        break;
+    case VVCAM_VB_CACHE_FLUSH:
+        ret = copy_from_user(&vb_mem, (void __user *)arg, sizeof(vb_mem));
+        if (ret)
+            break;
+        ret = vvcam_vb_cache_flush(file, &vb_mem);
+        break;
+    case VVCAM_VB_CACHE_INVALID:
+        ret = copy_from_user(&vb_mem, (void __user *)arg, sizeof(vb_mem));
+        if (ret)
+            break;
+        ret = vvcam_vb_cache_invalid(file, &vb_mem);
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    mutex_unlock(&vb_dev->mlock);
+
+    return ret;
+}
+
+int vvcam_vb_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    struct vvcam_vb_dev *vb_dev;
+    int ret = 0;
+
+    vb_dev = file->private_data;
+    if (!vb_dev->allocator.mm_dev || !vb_dev->allocator.mmap)
+        return -EPERM;
+
+    mutex_lock(&vb_dev->mlock);
+    ret = vb_dev->allocator.mmap(file, vma);
+    mutex_unlock(&vb_dev->mlock);
+
+    return ret;
+}
+
+static struct file_operations vvcam_vb_fops = {
+    .owner          = THIS_MODULE,
+	.open           = vvcam_vb_open,
+	.release        = vvcam_vb_release,
+	.unlocked_ioctl = vvcam_vb_ioctl,
+    .mmap           = vvcam_vb_mmap,
+};
+
+static int vvcam_vb_create_mm_allocator(struct platform_device *pdev,
+        struct vvcam_vb_mm_allocator *allocator, const int type)
+{
+    int ret = 0;
+    struct resource *res;
+
+    switch (type)
+    {
+    case VVCAM_VB_MMZ:
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        if (!res) {
+            dev_err(&pdev->dev, "can't fetch device resource info\n");
+            return -EIO;
+        }
+
+        ret = vvcam_mmz_create(allocator, &pdev->dev,
+                                res->start, resource_size(res));
+        if (ret) {
+            dev_err(&pdev->dev, "failed to create mmz\n");
+            return ret;
+        }
+        break;
+
+    case VVCAM_VB_CMA:
+        ret = vvcam_cma_create(allocator, &pdev->dev);
+        if (ret) {
+            dev_err(&pdev->dev, "failed to create mmz\n");
+            return ret;
+        }
+        break;
+
+    default:
+        break;
+    }
+    return 0;
+}
+
+static int vvcam_vb_probe(struct platform_device *pdev)
+{
+    struct vvcam_vb_dev *vb_dev;
+    struct device *dev = &pdev->dev;
+    struct device_node *sensor_np;
+    struct device *sensor_dev;
+    struct v4l2_subdev *sd;
+    int ret = 0;
+    int wait_time = 0;
+
+    sensor_np = of_parse_phandle(dev->of_node, "depends-on", 0);
+    if (sensor_np)
+    {
+        while (wait_time < 20000) {
+            sensor_dev = bus_find_device_by_of_node(&i2c_bus_type, sensor_np);
+            of_node_put(sensor_np);
+            if (sensor_dev) {
+                sd = dev_get_drvdata(sensor_dev);
+                put_device(sensor_dev);
+                if (sd && sd->v4l2_dev) {
+                    dev_info(dev, "Sensor is ready.\n");
+                    break;
+                }
+            }
+            msleep(1000);
+            wait_time += 1000;
+        }
+    }
+
+    vb_dev = devm_kzalloc(&pdev->dev,
+                sizeof(struct vvcam_vb_dev), GFP_KERNEL);
+    if (!vb_dev)
+        return -ENOMEM;
+
+    if (!enable_cma) {
+        vb_dev->type = VVCAM_VB_MMZ;
+        ret = vvcam_vb_create_mm_allocator(pdev,
+                &vb_dev->allocator, VVCAM_VB_MMZ);
+        if (ret) {
+            dev_err(&pdev->dev, "create vb mmz failed.\n");
+            return ret;
+        }
+    } else {
+        vb_dev->type = VVCAM_VB_CMA;
+        ret = vvcam_vb_create_mm_allocator(pdev,
+                &vb_dev->allocator, VVCAM_VB_CMA);
+        if (ret) {
+            dev_err(&pdev->dev, "create vb cma failed.\n");
+            return ret;
+        }
+    }
+
+    mutex_init(&vb_dev->mlock);
+    vb_dev->dev = &pdev->dev;
+    platform_set_drvdata(pdev, vb_dev);
+
+    ret = vvcam_vb_procfs_register(vb_dev, &vb_dev->pde);
+    if (ret) {
+        dev_err(&pdev->dev, "vb failed to register procfs\n");
+        return -EINVAL;
+    }
+
+    vb_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
+    vb_dev->miscdev.name  = VVCAM_VB_NAME;
+    vb_dev->miscdev.fops  = &vvcam_vb_fops;
+
+    ret = misc_register(&vb_dev->miscdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to register device\n");
+        return -EINVAL;
+    }
+
+    dev_info(&pdev->dev, "vvcam vb probe success\n");
+    return 0;
+
+}
+
+static int vvcam_vb_remove(struct platform_device *pdev)
+{
+    struct vvcam_vb_dev *vb_dev;
+    vb_dev = platform_get_drvdata(pdev);
+
+    vvcam_vb_procfs_unregister(&vb_dev->pde);
+
+    misc_deregister(&vb_dev->miscdev);
+
+    return 0;
+}
+
+static const struct of_device_id vvcam_vb_of_match[] = {
+	{.compatible = "verisilicon,vb",},
+	{ /* sentinel */ },
+};
+
+static struct platform_driver vvcam_vb_driver = {
+	.probe  = vvcam_vb_probe,
+	.remove = vvcam_vb_remove,
+	.driver = {
+		.name           = VVCAM_VB_NAME,
+		.owner          = THIS_MODULE,
+        .of_match_table = vvcam_vb_of_match,
+	}
+};
+
+MODULE_DEVICE_TABLE(of, vvcam_vb_of_match);
+
+static int __init vvcam_vb_init_module(void)
+{
+    int ret;
+
+    ret = platform_driver_register(&vvcam_vb_driver);
+    if (ret) {
+        printk(KERN_ERR "Failed to register vb driver\n");
+        return ret;
+    }
+
+    // ret = vvcam_vb_platform_device_register();
+    // if (ret) {
+	// 	platform_driver_unregister(&vvcam_vb_driver);
+	// 	printk(KERN_ERR "Failed to register vvcam vb platform devices\n");
+	// 	return ret;
+	// }
+
+    return ret;
+}
+
+static void __exit vvcam_vb_exit_module(void)
+{
+    platform_driver_unregister(&vvcam_vb_driver);
+    vvcam_vb_platform_device_unregister();
+}
+
+late_initcall(vvcam_vb_init_module);
+module_exit(vvcam_vb_exit_module);
+
+module_param(enable_cma, int, 0644);
+
+MODULE_DESCRIPTION("Verisilicon video buffer driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_driver.h b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_driver.h
new file mode 100644
index 000000000000..fe95efa6986a
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_driver.h
@@ -0,0 +1,86 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VB_DRIVER_H__
+#define __VVCAM_VB_DRIVER_H__
+
+#define VVCAM_VB_NAME "vvcam-vb"
+
+enum {
+    VVCAM_VB_MMZ,
+    VVCAM_VB_CMA
+};
+
+struct vvcam_vb_mm_allocator {
+    void *mm_dev;
+    int (*alloc)(struct file *file, unsigned long *phys_addr,
+        const unsigned long size, char *name);
+    int (*free)(struct file *file, const unsigned long phys_addr);
+    int (*mmap)(struct file *file, struct vm_area_struct *vma);
+    int (*free_all)(struct file *file);
+};
+
+struct vvcam_vb_dev {
+    phys_addr_t paddr;
+    uint32_t    size;
+    struct device *dev;
+    struct miscdevice miscdev;
+    struct mutex mlock;
+    int32_t type;
+    struct vvcam_vb_mm_allocator allocator;
+    struct proc_dir_entry *pde;
+};
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_platform.c b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_platform.c
new file mode 100644
index 000000000000..87d19acf7dc7
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_platform.c
@@ -0,0 +1,94 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+
+#include "vvcam_vb_platform.h"
+
+static void vvcam_vb_pdev_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+	return;
+}
+
+// static struct resource vvcam_vb_resource[] = {
+//     [0] = {
+//         .start = VVCAM_RMEM_BASE,
+//         .end   = VVCAM_RMEM_BASE + VVCAM_RMEM_SIZE - 1,
+//         .flags = IORESOURCE_MEM,
+//     },
+// };
+
+static struct platform_device vvcam_vb_pdev = {
+    .name          = "vvcam-vb",
+	.id            = 0,
+	// .resource      = vvcam_vb_resource,
+    // .num_resources = ARRAY_SIZE(vvcam_vb_resource),
+	.dev.release   = vvcam_vb_pdev_release,
+};
+
+int vvcam_vb_platform_device_register(void)
+{
+    return platform_device_register(&vvcam_vb_pdev);
+}
+
+void vvcam_vb_platform_device_unregister(void)
+{
+    platform_device_unregister(&vvcam_vb_pdev);
+
+    return;
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_platform.h b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_platform.h
new file mode 100644
index 000000000000..064693630344
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VB_PLATFORM_H__
+#define __VVCAM_VB_PLATFORM_H__
+
+int vvcam_vb_platform_device_register(void);
+void vvcam_vb_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_procfs.c b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_procfs.c
new file mode 100644
index 000000000000..619c3b621762
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_procfs.c
@@ -0,0 +1,455 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/ctype.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+
+#include "vvcam_mmz.h"
+#include "vvcam_cma.h"
+#include "vvcam_vb_driver.h"
+#include "vvcam_vb_procfs.h"
+
+struct mmb_info {
+    char name[32];
+    unsigned long size;
+    unsigned long paddr;
+    struct list_head list;
+};
+
+struct filed_mmb {
+    void *private;
+    struct list_head mbi_list; 
+    struct list_head list;
+};
+
+struct vvcam_vb_proc {
+    struct proc_dir_entry *pde;
+    struct vvcam_vb_dev *vb_dev;
+};
+
+
+static void release_mmlist(struct list_head *mm_list)
+{
+    struct mmb_info *mbi = NULL, *old_mbi = NULL;
+    struct filed_mmb *fmb = NULL, *old_fmb = NULL;
+    if (!list_empty(mm_list)) {
+        list_for_each_entry(fmb, mm_list, list) {
+            list_for_each_entry(mbi, &fmb->mbi_list, list) {
+                if (old_mbi) {
+                    list_del(&old_mbi->list);
+                    kfree(old_mbi);
+                    old_mbi = NULL;
+                }
+                old_mbi = mbi;
+            }
+            
+            if (old_fmb) {
+                list_del(&old_fmb->list);
+                kfree(old_fmb);
+                old_fmb = NULL;
+            }
+            old_fmb = fmb;
+        }
+
+        if (old_mbi) {
+            list_del(&old_mbi->list);
+            kfree(old_mbi);
+            old_mbi = NULL;
+        }
+
+        if (old_fmb) {
+            list_del(&old_fmb->list);
+            kfree(old_fmb);
+            old_fmb = NULL;
+        }
+    }
+}
+
+static int vvcam_vb_get_mmz_info(struct vvcam_mmz *mmz,
+                    struct list_head *mm_list, unsigned long *base,
+                    unsigned long *size, unsigned long *used)
+{
+    struct vvcam_mmb *mmb = NULL;
+    struct mmb_info *mbi = NULL;
+    struct filed_mmb *fmb = NULL, *t_fmb = NULL;
+
+    *size = mmz->size;
+    *base = mmz->paddr;
+
+    list_for_each_entry(mmb, &mmz->mmb_list, list) {
+        fmb = NULL;
+        if (!mmb) {
+            break;
+        }
+        list_for_each_entry(t_fmb, mm_list, list) {
+            if (!t_fmb) {
+                break;
+            }
+            if (t_fmb->private == mmb->private) {
+                fmb = t_fmb;
+                t_fmb = NULL;
+                break;
+            }
+        }
+
+        if (!fmb) {
+            fmb = kzalloc(sizeof(struct filed_mmb), GFP_KERNEL);
+            if (!fmb) {
+                release_mmlist(mm_list);
+                return -ENOMEM;
+            }
+            INIT_LIST_HEAD(&fmb->mbi_list);
+            list_add_tail(&fmb->list, mm_list);
+        }
+
+        mbi = kzalloc(sizeof(struct mmb_info), GFP_KERNEL);
+        if (!mbi) {
+            release_mmlist(mm_list);
+            return -ENOMEM;
+        }
+
+        fmb->private = mmb->private;
+        memcpy(mbi->name, mmb->name, sizeof(mmb->name));
+        mbi->paddr = mmb->phys_addr;
+        mbi->size = mmb->size;
+        *used += mbi->size;
+
+        list_add_tail(&mbi->list, &fmb->mbi_list);
+    }
+    return 0;
+}
+
+static int vvcam_vb_get_cma_info(struct vvcam_cma *cma,
+                struct list_head *mm_list, unsigned long *used)
+{
+    struct vvcam_cma_buf *cma_buf = NULL;
+    struct mmb_info *mbi = NULL;
+    struct filed_mmb *fmb = NULL, *tmp = NULL;
+
+    list_for_each_entry(cma_buf, &cma->buf_list, list) {
+        fmb = NULL;
+        if (!cma_buf) {
+            break;
+        }
+        list_for_each_entry(tmp, mm_list, list) {
+            if (!tmp) {
+                break;
+            }
+            if (tmp->private == cma_buf->private) {
+                fmb = tmp;
+                tmp = NULL;
+                break;
+            }
+        }
+
+        if (!fmb) {
+            fmb = kzalloc(sizeof(struct filed_mmb), GFP_KERNEL);
+            if (!fmb) {
+                release_mmlist(mm_list);
+                return -ENOMEM;
+            }
+            INIT_LIST_HEAD(&fmb->mbi_list);
+            list_add_tail(&fmb->list, mm_list);
+        }
+
+        mbi = kzalloc(sizeof(struct mmb_info), GFP_KERNEL);
+        if (!mbi) {
+            release_mmlist(mm_list);
+            return -ENOMEM;
+        }
+
+        fmb->private = cma_buf->private;
+        memcpy(mbi->name, cma_buf->name, sizeof(mbi->name));
+        mbi->paddr = cma_buf->phys_addr;
+        mbi->size = cma_buf->size;
+
+        *used += cma_buf->size;
+
+        list_add_tail(&mbi->list, &fmb->mbi_list);
+    }
+    return 0;
+}
+
+static int vvcam_vb_proc_info_show(struct seq_file *sfile, void *offset)
+{
+	struct vvcam_vb_proc *vb_proc;
+    struct list_head mm_list;
+    struct filed_mmb *fmb = NULL;
+    struct mmb_info *mbi = NULL;
+
+    unsigned long size = 0;
+    unsigned long used = 0;
+    unsigned long base = 0;
+    int ret = 0;
+
+	vb_proc = (struct vvcam_vb_proc *) sfile->private;
+
+    INIT_LIST_HEAD(&mm_list);
+
+    switch (vb_proc->vb_dev->type)
+    {
+    case VVCAM_VB_MMZ:
+        ret = vvcam_vb_get_mmz_info(
+            (struct vvcam_mmz*)vb_proc->vb_dev->allocator.mm_dev,
+            &mm_list, &base, &size, &used);
+        break;
+    
+    case VVCAM_VB_CMA:
+        ret = vvcam_vb_get_cma_info(
+            (struct vvcam_cma*)vb_proc->vb_dev->allocator.mm_dev,
+            &mm_list, &used);
+        break;
+    
+    default:
+        ret = -EINVAL; 
+        break;
+    }
+
+    if (ret) {
+        seq_printf(sfile, "get mm info failed\n");
+        return 0;
+    }
+
+    seq_printf(sfile, "DeviceName:\t%s\n", vb_proc->vb_dev->miscdev.name);
+    seq_printf(sfile, "VB Type:\t%s\n",
+        vb_proc->vb_dev->type == VVCAM_VB_MMZ ? "reserved mem" : "cma");
+    if (vb_proc->vb_dev->type == VVCAM_VB_MMZ) {
+        seq_printf(sfile, "Base:\t\t0x%-8lx\n", base);
+        seq_printf(sfile, "Size:\t\t0x%-8lx\n", size);
+        seq_printf(sfile, "Free:\t\t0x%-8lx\n", size - used);
+    }
+    seq_printf(sfile, "Used:\t\t0x%-8lx\n", used);
+
+    seq_printf(sfile,
+    "|------------------|----------------|---------------------|----------|\n");
+    seq_printf(sfile, "|%-18s|%-16s|%-21s|%-10s|\n",
+        "       File", "      Name", "        Range", "   Size");
+    seq_printf(sfile,
+    "|------------------|----------------|---------------------|----------|\n");
+    if (!list_empty(&mm_list)) {
+         list_for_each_entry(fmb, &mm_list, list) {
+            list_for_each_entry(mbi, &fmb->mbi_list, list) {
+                seq_printf(sfile, "|0x%-16p|%-16s|0x%08lx~0x%08lx|0x%-8lx|\n",
+                fmb->private, mbi->name,
+                mbi->paddr, mbi->paddr + mbi->size - 1, mbi->size);
+            }
+            seq_printf(sfile,
+    "|------------------|----------------|---------------------|----------|\n");
+         }
+    }
+
+    release_mmlist(&mm_list);
+
+	return 0;
+}
+
+
+static int vvcam_vb_proc_open(struct inode *inode, struct file *file)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	return single_open(file, vvcam_vb_proc_info_show, PDE_DATA(inode));
+#else
+	return single_open(file, vvcam_vb_proc_info_show, pde_data(inode));
+#endif
+}
+
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
+static const struct file_operations vvcam_vb_proc_ops = {
+    .open = vvcam_vb_proc_open,
+    .release = seq_release,
+    .read = seq_read,
+    .llseek = seq_lseek,
+};
+#else
+static const struct proc_ops vvcam_vb_proc_ops = {
+	.proc_open = vvcam_vb_proc_open,
+	.proc_release = seq_release,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+};
+#endif
+
+struct finddir_callback {
+    struct dir_context ctx;
+    const char *name;
+    int32_t files_cnt;
+    bool found;
+};
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)
+static int readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return 0;
+
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return 0;
+}
+
+#else
+static bool readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return true;
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    fc->files_cnt++;
+    return true;
+}
+#endif
+
+static int find_proc_dir_by_name(const char *root,
+                const char *name, bool *found, int32_t *files_cnt) {
+    struct file *pfile;
+    int ret = 0;
+    struct finddir_callback fc = {
+        .ctx.actor = readdir_callback,
+        .name = name,
+        .found = false,
+        .files_cnt = -2,
+    };
+
+    pfile = filp_open(root, O_RDONLY | O_DIRECTORY, 0);
+    if(IS_ERR(pfile)) {
+        pr_err("Failed to open %s\n", root);
+        return -1;
+    }
+    if (pfile->f_op->iterate_shared) {
+        ret = pfile->f_op->iterate_shared(pfile, &fc.ctx);
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 2, 0)
+    } else {
+        ret = pfile->f_op->iterate(pfile, &fc.ctx);
+#endif
+    }
+
+    if (ret == 0) {
+        *found = fc.found;
+    }
+
+    if (files_cnt != NULL) {
+        *files_cnt = fc.files_cnt;
+    }
+
+    filp_close(pfile, NULL);
+    return ret;
+}
+
+int vvcam_vb_procfs_register(struct vvcam_vb_dev *vb_dev,
+                                struct proc_dir_entry **pde)
+{
+    struct vvcam_vb_proc *vb_proc;
+    struct platform_device *pdev = to_platform_device(vb_dev->dev);
+    char vb_proc_name[32];
+    int ret = 0;
+    bool found = false;
+
+    sprintf(vb_proc_name, "vsi/vb%d", pdev->id);
+    vb_proc = devm_kzalloc(vb_dev->dev, 
+                        sizeof(struct vvcam_vb_proc), GFP_KERNEL);
+    if (!vb_proc)
+        return -ENOMEM;
+
+    vb_proc->vb_dev = vb_dev;
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (!found)
+            proc_mkdir("vsi", NULL);    
+    } else {
+        return -EFAULT;
+    }
+
+    vb_proc->vb_dev = vb_dev;
+    *pde = proc_create_data(vb_proc_name, 0666, NULL,
+                                &vvcam_vb_proc_ops, vb_proc);
+    if (!*pde)
+        return -EFAULT;
+
+    vb_proc->pde = *pde;
+	return 0;
+}
+
+void vvcam_vb_procfs_unregister(struct proc_dir_entry **pde)
+{
+    int ret = 0;
+    bool found = false;
+    int32_t files_cnt;
+
+    ret = find_proc_dir_by_name("/proc", "vsi", &found, NULL);
+    if (ret == 0) {
+        if (found) {
+            proc_remove(*pde);
+            ret = find_proc_dir_by_name("/proc/vsi", "", &found, &files_cnt);
+            if (files_cnt == 0) {
+                remove_proc_subtree("vsi", NULL);
+            }
+        }
+    }
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_procfs.h b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_procfs.h
new file mode 100644
index 000000000000..a7181ad708af
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vb/vvcam_vb_procfs.h
@@ -0,0 +1,64 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VB_PROCFS_H__
+#define __VVCAM_VB_PROCFS_H__
+
+int vvcam_vb_procfs_register(struct vvcam_vb_dev *vb_dev,
+                                    struct proc_dir_entry **pde);
+void vvcam_vb_procfs_unregister(struct proc_dir_entry **pde);
+
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/vicap/Makefile b/drivers/media/platform/eswin/isp/vvcam/vicap/Makefile
new file mode 100644
index 000000000000..4ec13fd0481e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vicap/Makefile
@@ -0,0 +1,25 @@
+ifeq ($(VVCAM_PATH),)
+	VVCAM_PATH:=$(shell cd $(shell pwd)/../; pwd)
+	export VVCAM_PATH
+endif
+
+
+include $(PWD)/../Param.mk
+TARGET = vicap
+
+obj-m +=$(TARGET).o
+$(TARGET)-objs += vicap_driver.o
+$(TARGET)-objs += vicap_ioctl.o
+
+PWD :=$(shell pwd)
+
+all:
+	make -C $(KERNEL_DIR) M=$(PWD) modules
+	@[ -e $(INSTALL_DIR)/lib ] || mkdir $(INSTALL_DIR)/lib -p
+	@cp $(TARGET).ko $(INSTALL_DIR)/lib
+modules_install:
+	make -C $(KERNEL_DIR) M=$(PWD) modules_install
+clean:
+	rm -rf $($(TARGET)-objs)
+	make -C $(KERNEL_DIR) M=$(PWD) clean
+	@rm $(INSTALL_DIR)/lib/$(TARGET).ko
diff --git a/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_driver.c b/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_driver.c
new file mode 100644
index 000000000000..70c9ccdd0a8e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_driver.c
@@ -0,0 +1,227 @@
+#include <asm/io.h>
+#include <linux/cdev.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/proc_fs.h>
+#include <linux/debugfs.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/poll.h>
+
+#include "vicap_ioctl.h"
+
+#define VICAP_NAME "vicap"
+
+static unsigned int vvcam_vicap_major = 0;
+static unsigned int vvcam_vicap_minor = 0;
+static struct class *vvcam_vicap_class = NULL;
+
+vicap_dev_info_t *vicap_dev_info[VICAP_DEV_MAXCNT] = {NULL,NULL,NULL,NULL,NULL,NULL};
+
+static int vicap_open(struct inode *inode, struct file *file)
+{
+	struct vicap_device *vicap_dev = NULL;
+
+	vicap_dev = container_of(inode->i_cdev, struct vicap_device, cdev);
+	if (!vicap_dev) {
+		pr_err("vicap dev cannot found\n");
+		return -ENODEV;
+	}
+	file->private_data = vicap_dev;
+
+	return 0;
+}
+
+static long vicap_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long ret;
+	struct vicap_device *vicap_dev = (struct vicap_device *)file->private_data;
+
+	if (!vicap_dev) {
+		pr_err("vicap dev cannot found\n");
+		return -ENODEV;
+	}
+
+	ret = vicap_priv_ioctl(vicap_dev, cmd, (void __user *)arg);
+
+	return ret;
+}
+
+static unsigned int vicap_poll(struct file *file, poll_table *wait)
+{
+	int mask = 0;
+
+	return mask;
+}
+
+static int vicap_release(struct inode *inode, struct file *file)
+{
+	return 0;
+}
+
+static struct file_operations vicap_fops = {
+	.owner          = THIS_MODULE,
+	.open           = vicap_open,
+	.release        = vicap_release,
+	.unlocked_ioctl = vicap_ioctl,
+	.poll           = vicap_poll,
+};
+
+static int vvcam_vicap_probe(struct platform_device *pdev)
+{
+	int ret = 0;
+	struct resource *mem;
+	struct vicap_device *vicap_dev;
+
+	pr_info("enter %s\n", __func__);
+
+	if (device_property_read_u32(&pdev->dev, "id", &pdev->id)) {
+		dev_err(&pdev->dev, "No device id found\n");
+		return -EINVAL;
+	}
+
+	if (pdev->id >= VICAP_DEV_MAXCNT) {
+		dev_err(&pdev->dev, "Device id %d error!\n", pdev->id);
+		return -EINVAL;
+	}
+
+	vicap_dev = devm_kzalloc(&pdev->dev, sizeof(struct vicap_device), GFP_KERNEL);
+	if (!vicap_dev) {
+		pr_err("%s: kzalloc failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	if (!vicap_dev_info[pdev->id]) {
+		vicap_dev_info[pdev->id] = devm_kzalloc(&pdev->dev,
+				    sizeof(vicap_dev_info_t), GFP_KERNEL);
+		if (!vicap_dev_info[pdev->id]) {
+			pr_err("%s: kzalloc failed\n", __func__);
+			return -ENOMEM;
+		}
+
+		vicap_dev_info[1] = devm_kzalloc(&pdev->dev,
+				    sizeof(vicap_dev_info_t), GFP_KERNEL);
+	} else {
+		pr_err("vicap dev info already been alloced\n");
+	}
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	vicap_dev->base = devm_ioremap_resource(&pdev->dev, mem);
+	vicap_dev->id = pdev->id;
+	vicap_dev_info[pdev->id]->id = pdev->id;
+	mutex_init(&vicap_dev->vicap_mutex);
+
+	platform_set_drvdata(pdev, vicap_dev);
+
+	if (vvcam_vicap_major == 0) {
+		ret = alloc_chrdev_region(&vicap_dev->devt, 0, VICAP_DEV_MAXCNT, VICAP_NAME);
+		if (ret != 0) {
+			pr_err("%s: alloc_chrdev_region error\n", __func__);
+			return ret;
+		}
+		vvcam_vicap_major = MAJOR(vicap_dev->devt);
+		vvcam_vicap_minor = MINOR(vicap_dev->devt);
+	} else {
+		vicap_dev->devt = MKDEV(vvcam_vicap_major, vvcam_vicap_minor);
+		ret = register_chrdev_region(vicap_dev->devt, VICAP_DEV_MAXCNT,	VICAP_NAME);
+		if (ret) {
+			pr_err("%s:register_chrdev_region error\n",	__func__);
+			return ret;
+		}
+	}
+
+// #if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0)
+	vvcam_vicap_class = class_create(VICAP_NAME);
+// #else
+// 	vvcam_vicap_class = class_create(THIS_MODULE, VICAP_NAME);
+// #endif
+	if (IS_ERR(vvcam_vicap_class)) {
+		pr_err("%s[%d]:class_create error!\n", __func__, __LINE__);
+		return -EINVAL;
+	}
+
+	vicap_dev->devt = MKDEV(vvcam_vicap_major, vvcam_vicap_minor + pdev->id);
+
+	cdev_init(&vicap_dev->cdev, &vicap_fops);
+	ret = cdev_add(&vicap_dev->cdev, vicap_dev->devt, 1);
+	if (ret) {
+		pr_err("%s[%d]:cdev_add error!\n", __func__, __LINE__);
+		return ret;
+	}
+	vicap_dev->class = vvcam_vicap_class;
+	device_create(vicap_dev->class, NULL, vicap_dev->devt,
+		      vicap_dev, "%s%d", VICAP_NAME, pdev->id);
+
+	pr_info("exit %s\n", __func__);
+
+	return ret;
+}
+
+static int vvcam_vicap_remove(struct platform_device *pdev)
+{
+    struct vicap_device *vicap_dev;
+
+    vicap_dev = platform_get_drvdata(pdev);
+
+    cdev_del(&vicap_dev->cdev);
+    device_destroy(vicap_dev->class, vicap_dev->devt);
+    unregister_chrdev_region(vicap_dev->devt, VICAP_DEV_MAXCNT);
+    class_destroy(vicap_dev->class);
+
+	return 0;
+}
+
+static const struct of_device_id vicap_of_match[] = {
+	{ .compatible = "eswin,vicap" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, vicap_of_match);
+
+static struct platform_driver vicap_driver = {
+	.probe = vvcam_vicap_probe,
+	.remove = vvcam_vicap_remove,
+	.driver = {
+		   .name = VICAP_NAME,
+		   .owner = THIS_MODULE,
+		   .of_match_table = vicap_of_match,
+	}
+};
+
+static int __init vicap_init_module(void)
+{
+	int ret = 0;
+
+	pr_info("enter %s\n", __func__);
+
+	ret = platform_driver_register(&vicap_driver);
+	if (ret) {
+		pr_err("register platform driver failed.\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static void __exit vicap_exit_module(void)
+{
+	pr_info("enter %s\n", __func__);
+
+	platform_driver_unregister(&vicap_driver);
+}
+
+late_initcall(vicap_init_module);
+module_exit(vicap_exit_module);
+
+MODULE_DESCRIPTION("VICAP");
+MODULE_LICENSE("GPL");
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_ioctl.c b/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_ioctl.c
new file mode 100644
index 000000000000..6969e71f7c66
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_ioctl.c
@@ -0,0 +1,352 @@
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+
+#include "vicap_ioctl.h"
+
+extern vicap_dev_info_t *vicap_dev_info[VICAP_DEV_MAXCNT];
+
+static int vicap_reg_write_part(struct vicap_device *dev, uint32_t addr, uint32_t data, uint8_t shift, uint8_t width)
+{
+    uint32_t temp = 0;
+    uint32_t mask = (1 << width) - 1;
+
+    // TODO: Remove 2 times reading on soc
+    readl(dev->base + addr);
+    temp = readl(dev->base + addr);
+
+    temp &= ~(mask << shift);
+    temp |= (data & mask) << shift;
+
+    writel(temp, dev->base + addr);
+
+    return 0;
+}
+
+static long vicap_sw_reset_ctrl(struct vicap_device *dev)
+{
+	if (!vicap_dev_info[dev->id]) {
+		pr_err("%s: vicap dev info not allocated\n", __func__);
+		return -ENODEV;
+	}
+
+	memset(vicap_dev_info[dev->id], 0, sizeof(vicap_dev_info_t));
+
+	writel(0, dev->base + VICAP_SW_RESET);
+	fsleep(100);
+	writel(1, dev->base + VICAP_SW_RESET);
+
+	vicap_reg_write_part(dev, VICAP_PHY_CTRL_CFG8, 0, VIN_UNLOCK_RST_FIFO_LSB, 1);
+	vicap_reg_write_part(dev, VICAP_PHY_CTRL_CFG8, 0, VIN_SOFT_RST_LSB, 1);
+	fsleep(100);
+	vicap_reg_write_part(dev, VICAP_PHY_CTRL_CFG8, 1, VIN_SOFT_RST_LSB, 1);
+	vicap_reg_write_part(dev, VICAP_PHY_CTRL_CFG8, 1, VIN_UNLOCK_RST_FIFO_LSB, 1);
+
+	return 0;
+}
+
+static long check_phy_mode_4_lane(struct vicap_device *dev)
+{
+	vicap_dev_info[dev->id]->phy_num = 2;
+
+	if (!vicap_dev_info[dev->id + 1]) { 
+		pr_err("%s: vicap dev info not allocated\n", __func__);
+		return -ENODEV;
+	} else {
+		vicap_dev_info[dev->id + 1]->dev_mode = DEV_MODE_SLAVE;
+		vicap_dev_info[dev->id + 1]->phy_num = 2;
+	}
+
+	return 0;
+}
+
+static long check_phy_mode_8_lane(struct vicap_device *dev)
+{
+	vicap_dev_info[dev->id]->phy_num = 4;
+
+	if (!vicap_dev_info[dev->id + 1] || 
+		    !vicap_dev_info[dev->id + 2] ||
+		    !vicap_dev_info[dev->id + 3]) {
+		pr_err("vicap dev info not allocated\n");
+		return -ENODEV;
+	} else {
+		vicap_dev_info[dev->id + 1]->dev_mode = DEV_MODE_SLAVE;
+		vicap_dev_info[dev->id + 2]->dev_mode = DEV_MODE_SLAVE;
+		vicap_dev_info[dev->id + 3]->dev_mode = DEV_MODE_SLAVE;
+		vicap_dev_info[dev->id + 1]->phy_num = 4;
+		vicap_dev_info[dev->id + 2]->phy_num = 4;
+		vicap_dev_info[dev->id + 3]->phy_num = 4;
+	}
+
+	return 0;
+}
+
+static long vicap_ioc_dev_attr(struct vicap_device *dev, vicap_dev_attr_t *dev_attr)
+{
+	long ret;
+
+	if (!vicap_dev_info[dev->id]) {
+		pr_err("%s: vicap dev info not allocated\n", __func__);
+		return -ENODEV;
+	}
+
+	if (vicap_dev_info[dev->id]->dev_mode == DEV_MODE_SLAVE) {
+		pr_err("vicap dev already configured ");
+		return -EINVAL;
+	}
+
+	memcpy(&vicap_dev_info[dev->id]->dev_attr, dev_attr, sizeof(vicap_dev_attr_t));
+	vicap_dev_info[dev->id]->dev_mode = DEV_MODE_MASTER;
+	vicap_dev_info[dev->id]->phy_num = 1;
+
+	switch(dev_attr->phy_mode) {
+	case PHY_MODE_LANE_8_4:
+		if (dev->id == 0) {
+			ret = check_phy_mode_8_lane(dev);
+			if (ret)
+				return ret;
+		} else if(dev->id == 4) {
+			ret = check_phy_mode_4_lane(dev);
+			if (ret)
+				return ret;
+		} else {
+			goto DEV_ERR;
+		}
+		break;
+	case PHY_MODE_LANE_8_2_2:
+		if (dev->id == 0) {
+			ret = check_phy_mode_8_lane(dev);
+			if (ret)
+				return ret;
+		} else if (dev->id == 1 || dev->id == 2 || dev->id == 3) {
+			goto DEV_ERR;
+		}
+		break;
+	case PHY_MODE_LANE_4_4_4:
+		if (dev->id == 0 || dev->id == 2 || dev->id == 4) {
+			ret = check_phy_mode_4_lane(dev);
+			if (ret)
+				return ret;
+		} else {
+			goto DEV_ERR;
+		}
+		break;
+	case PHY_MODE_LANE_4_4_2_2:
+		if (dev->id == 0 || dev->id == 2) {
+			ret = check_phy_mode_4_lane(dev);
+			if (ret)
+				return ret;
+		} else if (dev->id == 1 || dev->id == 3) {
+			goto DEV_ERR;
+		}
+		break;
+	case PHY_MODE_LANE_4_2_2_2_2:
+		if (dev->id == 0) {
+			ret = check_phy_mode_4_lane(dev);
+			if (ret)
+				return ret;
+		} else if (dev->id == 1) {
+			goto DEV_ERR;
+		}
+		break;
+	case PHY_MODE_LANE_2_2_2_2_2_2:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+DEV_ERR:
+	pr_err("Dev %d cannot support current mode\n", dev->id);
+	return -EINVAL;
+}
+
+static void vicap_set_common_attr(struct vicap_device *dev)
+{
+	uint32_t pic_res, lane_en;
+	vicap_dev_attr_t *dev_attr = &vicap_dev_info[dev->id]->dev_attr;
+
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, 2, LANE_NUM_LSB, BIT_WIDTH_2); // We only support 2 lane per phy
+	//vicap_reg_write_part(dev, VICAP_COMMON_SET, vicap_dev_info[dev->id]->phy_num / 2, PHY_NUM_LSB, BIT_WIDTH_2); // phy_num support 1, 2, 4, lvds ok
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, 0, PHY_NUM_LSB, BIT_WIDTH_2); // phy_num support 1, 2, 4
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, dev_attr->combo_mode, COMBO_MODE_LSB, BIT_WIDTH_1);
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, dev_attr->data_type, L_BIT_LSB, BIT_WIDTH_3);
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, dev_attr->data_endian, BIT_REORDER_LSB, BIT_WIDTH_1);
+	
+	pic_res = ((dev_attr->frame_info.width + dev_attr->frame_info.h_pad) << 16) | (dev_attr->frame_info.height + dev_attr->frame_info.v_pad);
+//	pic_res = (dev_attr->frame_info.width << 16) | dev_attr->frame_info.height;
+	writel(pic_res, dev->base + VICAP_PIC_RES);
+	vicap_reg_write_part(dev, VICAP_HDR_ARG1,
+		    (dev_attr->frame_info.height + dev_attr->frame_info.v_pad), FRAME_HEIGHT_LSB, FRAME_HEIGHT_BITS);
+
+	if (dev_attr->crop_info.crop_en) {
+		uint32_t crop_x = (dev_attr->crop_info.crop_x1 << 16) | dev_attr->crop_info.crop_x2;
+		uint32_t crop_y = (dev_attr->crop_info.crop_y1 << 16) | dev_attr->crop_info.crop_y2;
+
+		vicap_reg_write_part(dev, VICAP_COMMON_SET, 1, CROP_EN_LSB, BIT_WIDTH_1);
+		writel(crop_x, dev->base + VICAP_CROP_X);
+		writel(crop_y, dev->base + VICAP_CROP_Y);
+	}
+
+    // 2 lane per phy
+	lane_en = (1 << (vicap_dev_info[dev->id]->phy_num * 2)) - 1;
+	vicap_reg_write_part(dev, VICAP_PHY_CTRL_CFG8, lane_en, VIN_LANE_ENABLE_LSB, VIN_LANE_EN_BITS);
+
+	if (vicap_dev_info[dev->id]->dev_mode == DEV_MODE_MASTER) {
+		mutex_lock(&dev->vicap_mutex);
+		vicap_dev_info[dev->id]->enable = 1;
+		mutex_unlock(&dev->vicap_mutex);
+	}
+}
+
+static void vicap_set_slvs_attr(struct vicap_device *dev)
+{
+	slvs_dev_attr_t *slvs_attr = &vicap_dev_info[dev->id]->dev_attr.v.slvs_attr;
+
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, 0, SLVS_MAP_LSB, BIT_WIDTH_1);
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, 0, SLVS_5WORD_LSB, BIT_WIDTH_1); // TODO: Add 5 word sync code support
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, slvs_attr->sync_mode, SLVS_MODE_LSB, BIT_WIDTH_1);
+
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, slvs_attr->ob_line_en, OB_LINE_EN_LSB, LINE_EN_BITS);
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, slvs_attr->info_line_en, INFO_LINE_EN_LSB, LINE_EN_BITS);
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, slvs_attr->ob_line_num, OB_LINE_NUM_LSB, LINE_NUM_BITS);
+	vicap_reg_write_part(dev, VICAP_SLVS_ARG, slvs_attr->info_line_num, INFO_LINE_NUM_LSB, LINE_NUM_BITS);
+
+	// 4 word sync code configuration
+	if (slvs_attr->sync_mode == SLVS_SYNC_MODE_SAV) {
+		vicap_reg_write_part(dev, VICAP_REG_SC1, slvs_attr->sync_code[0], SC_W1_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC4, slvs_attr->sync_code[1], SC_W7_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC7, slvs_attr->sync_code[2], SC_W13_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC7, slvs_attr->sync_code[3], SC_W14_LSB, SYNC_CODE_BITS);
+	} else if (slvs_attr->sync_mode == SLVS_SYNC_MODE_SOF) {
+		// sol: sc2.w4, eol: sc5.w10, sof: sc1.w1, eof: sc4.w7
+		vicap_reg_write_part(dev, VICAP_REG_SC2, slvs_attr->sync_code[0], SC_W4_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC5, slvs_attr->sync_code[1], SC_W10_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC1, slvs_attr->sync_code[2], SC_W1_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC4, slvs_attr->sync_code[3], SC_W7_LSB, SYNC_CODE_BITS);
+	}
+}
+
+static void vicap_set_hispi_attr(struct vicap_device *dev)
+{
+	uint16_t *sync_code;
+	hispi_dev_attr_t *hispi_attr = &vicap_dev_info[dev->id]->dev_attr.v.hispi_attr;
+
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, 1, CAP_EN_LSB, BIT_WIDTH_1);
+	vicap_reg_write_part(dev, VICAP_COMMON_SET, hispi_attr->ail_endian, AIL_ENDIAN_LSB, BIT_WIDTH_1);
+	// flr and crc are optional for all hispi modes
+	vicap_reg_write_part(dev, VICAP_HISPI_ARG, hispi_attr->flr_en, FLR_EN_LSB, BIT_WIDTH_1);
+	vicap_reg_write_part(dev, VICAP_HISPI_ARG, hispi_attr->crc_en, CRC_EN_LSB, BIT_WIDTH_1);
+	vicap_reg_write_part(dev, VICAP_HISPI_ARG, hispi_attr->hispi_mode, HISPI_MODE_LSB, HISPI_MODE_BITS);
+
+	switch(hispi_attr->hispi_mode) {
+	case HISPI_MODE_PACKETIZED_SP:
+		// SOF: sc1.w1, SOL: sc1.w2, EOF: sc2.w3, EOL: sc2.w4
+		sync_code = hispi_attr->sc.pack_sp_sc;
+		vicap_reg_write_part(dev, VICAP_REG_SC1, sync_code[0], SC_W1_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC1, sync_code[1], SC_W2_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC2, sync_code[2], SC_W3_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC2, sync_code[3], SC_W4_LSB, SYNC_CODE_BITS);
+		break;
+	case HISPI_MODE_STREAMING_SP:
+		// SOF: sc1.w1, SOL: sc1.w2, SOV: sc3.w5
+		sync_code = hispi_attr->sc.stream_sp_sc;
+		vicap_reg_write_part(dev, VICAP_REG_SC1, sync_code[0], SC_W1_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC1, sync_code[1], SC_W2_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC3, sync_code[2], SC_W5_LSB, SYNC_CODE_BITS);
+		break;
+	case HISPI_MODE_STREAMING_S:
+		break;
+	case HISPI_MODE_ACTIVESTART_SP8:
+		sync_code = hispi_attr->sc.act_sp_sc;
+		vicap_reg_write_part(dev, VICAP_REG_SC1, sync_code[0], SC_W1_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC1, sync_code[1], SC_W2_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC2, sync_code[2], SC_W3_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC2, sync_code[3], SC_W4_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC3, sync_code[4], SC_W5_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC3, sync_code[5], SC_W6_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC4, sync_code[6], SC_W7_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC4, sync_code[7], SC_W8_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC5, sync_code[8], SC_W9_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC5, sync_code[9], SC_W10_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC6, sync_code[10], SC_W11_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC6, sync_code[11], SC_W12_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC7, sync_code[12], SC_W13_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC7, sync_code[13], SC_W14_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC8, sync_code[14], SC_W15_LSB, SYNC_CODE_BITS);
+		vicap_reg_write_part(dev, VICAP_REG_SC8, sync_code[15], SC_W16_LSB, SYNC_CODE_BITS);
+		break;
+	default:
+		break;
+	}
+}
+
+static long vicap_ioc_dev_enable(struct vicap_device *dev)
+{
+	vicap_dev_attr_t *dev_attr;
+
+	if (!vicap_dev_info[dev->id]) {
+		pr_err("%s: vicap dev info not allocated\n", __func__);
+		return -ENODEV;
+	}
+
+	dev_attr = &vicap_dev_info[dev->id]->dev_attr;
+	if (dev_attr->combo_mode == MODE_SLVS_SUBLVDS) {
+		vicap_set_slvs_attr(dev);
+	} else if (dev_attr->combo_mode == MODE_HISPI) {
+		vicap_set_hispi_attr(dev);
+	}
+
+	vicap_set_common_attr(dev);
+
+	return 0;
+}
+
+static long vicap_ioc_dev_disable(struct vicap_device *dev)
+{
+	if (!vicap_dev_info[dev->id]) {
+		pr_err("%s: vicap dev info not allocated\n", __func__);
+		return -ENODEV;
+	}
+
+	if (vicap_dev_info[dev->id]->dev_mode == DEV_MODE_MASTER) {
+		mutex_lock(&dev->vicap_mutex);
+		vicap_dev_info[dev->id]->enable = 0;
+		mutex_unlock(&dev->vicap_mutex);
+	}
+
+	vicap_reg_write_part(dev, VICAP_PHY_CTRL_CFG8, 0, VIN_LANE_ENABLE_LSB, VIN_LANE_EN_BITS);
+
+	return 0;
+}
+
+long vicap_priv_ioctl(struct vicap_device *dev, unsigned int cmd, void __user *args)
+{
+	long ret = 0, retval;
+
+	switch(cmd) {
+	case VICAP_IOC_S_RESET_CTRL:
+		ret = vicap_sw_reset_ctrl(dev);
+		break;
+	case VICAP_IOC_S_DEV_ATTR:
+		vicap_dev_attr_t dev_attr;
+		retval = copy_from_user(&dev_attr, args, sizeof(vicap_dev_attr_t));
+		if (retval < 0) {
+			pr_err("Copy from user failed\n");
+			return -EFAULT;
+		}
+		ret = vicap_ioc_dev_attr(dev, &dev_attr);
+		break;
+	case VICAP_IOC_S_DEV_ENABLE:
+		ret = vicap_ioc_dev_enable(dev);
+		break;
+	case VICAP_IOC_S_DEV_DISABLE:
+		ret = vicap_ioc_dev_disable(dev);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_ioctl.h b/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_ioctl.h
new file mode 100644
index 000000000000..5f08ae8f9bfe
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/vicap/vicap_ioctl.h
@@ -0,0 +1,235 @@
+#ifndef __VICAP_IOCTL_H__
+#define __VICAP_IOCTL_H__
+
+#ifdef __KERNEL__
+#include <linux/cdev.h>
+#else
+#include <stdint.h>
+#endif
+
+#include <linux/ioctl.h>
+
+#ifdef __KERNEL__
+#define VICAP_DEV_MAXCNT 6
+
+#define BIT_WIDTH_1                1
+#define BIT_WIDTH_2                2
+#define BIT_WIDTH_3                3
+
+#define VICAP_COMMON_SET           0x00
+#define LANE_NUM_LSB               8
+#define PHY_NUM_LSB                10
+#define COMBO_MODE_LSB             12
+#define CROP_EN_LSB                13
+#define AIL_ENDIAN_LSB             19
+#define BIT_REORDER_LSB            20
+#define L_BIT_LSB                  21
+#define CAP_EN_LSB                 25
+
+#define VICAP_PIC_RES              0x04
+#define VICAP_HDR_ARG1             0x08
+#define FRAME_HEIGHT_LSB           16
+#define FRAME_HEIGHT_BITS          16
+
+#define VICAP_CROP_X               0x10
+#define VICAP_CROP_Y               0x14
+
+#define VICAP_REG_SC1              0x18
+#define SC_W1_LSB                  16
+#define SC_W2_LSB                  0
+#define VICAP_REG_SC2              0x1c
+#define SC_W3_LSB                  16
+#define SC_W4_LSB                  0
+#define VICAP_REG_SC3              0x20
+#define SC_W5_LSB                  16
+#define SC_W6_LSB                  0
+#define VICAP_REG_SC4              0x24
+#define SC_W7_LSB                  16
+#define SC_W8_LSB                  0
+#define VICAP_REG_SC5              0x28
+#define SC_W9_LSB                  16
+#define SC_W10_LSB                 0
+#define VICAP_REG_SC6              0x2c
+#define SC_W11_LSB                 16
+#define SC_W12_LSB                 0
+#define VICAP_REG_SC7              0x30
+#define SC_W13_LSB                 16
+#define SC_W14_LSB                 0
+#define VICAP_REG_SC8              0x34
+#define SC_W15_LSB                 16
+#define SC_W16_LSB                 0
+#define SYNC_CODE_BITS             16
+
+#define VICAP_SLVS_ARG             0x3c
+#define OB_LINE_NUM_LSB            0
+#define INFO_LINE_NUM_LSB          8
+#define OB_LINE_EN_LSB             16
+#define INFO_LINE_EN_LSB           20
+#define SLVS_MODE_LSB              24
+#define SLVS_5WORD_LSB             25
+#define SLVS_MAP_LSB               26
+#define LINE_NUM_BITS              8
+#define LINE_EN_BITS               3
+
+#define VICAP_HISPI_ARG            0x40
+#define CRC_EN_LSB                 4
+#define FLR_EN_LSB                 5 
+#define HISPI_MODE_LSB             0
+#define HISPI_MODE_BITS            4
+
+
+#define VICAP_PHY_CTRL_CFG8        0xa0
+#define VIN_SOFT_RST_LSB           25
+#define VIN_UNLOCK_RST_FIFO_LSB    24
+#define VIN_LANE_ENABLE_LSB        16
+#define VIN_SC_BYTE_ORDOR_LSB      15
+#define VIN_LANE_EN_BITS           8
+
+#define VICAP_SW_RESET             0xc4
+#endif
+
+#define SLVS_SYNC_CODE_WORD 4
+
+enum {
+	VICAP_IOC_S_RESET_CTRL = _IO('v', 0), // Control reset, no register
+	VICAP_IOC_S_DEV_ATTR,
+	VICAP_IOC_S_DEV_ENABLE,
+	VICAP_IOC_S_DEV_DISABLE,
+
+	VICAP_IOC_MAX,
+};
+
+typedef enum _dev_mode {
+	DEV_MODE_MASTER,
+	DEV_MODE_SLAVE,
+	DEV_MODE_BUTT,
+} dev_mode_t;
+
+typedef enum _phy_mode {
+	PHY_MODE_LANE_8_4,
+	PHY_MODE_LANE_8_2_2,
+	PHY_MODE_LANE_4_4_4,
+	PHY_MODE_LANE_4_4_2_2,
+	PHY_MODE_LANE_4_2_2_2_2,
+	PHY_MODE_LANE_2_2_2_2_2_2,
+	PHY_MODE_BUTT,
+} phy_mode_t;
+
+typedef enum _combo_mode {
+	MODE_HISPI = 0,
+	MODE_SLVS_SUBLVDS,
+	MODE_BUTT,
+} combo_mode_t;
+
+typedef struct _frame_info {
+	uint32_t width;
+	uint32_t height;
+	uint32_t h_pad;
+	uint32_t v_pad;
+} frame_info_t;
+
+typedef struct _crop_info {
+	uint32_t crop_en;
+	uint32_t crop_x1;
+	uint32_t crop_x2;
+	uint32_t crop_y1;
+	uint32_t crop_y2;
+} crop_info_t;
+
+typedef enum _data_type {
+	DATA_TYPE_RAW_8BIT = 0,
+	DATA_TYPE_RAW_10BIT,
+	DATA_TYPE_RAW_12BIT,
+	DATA_TYPE_RAW_14BIT,
+	DATA_TYPE_RAW_16BIT,
+	DATA_TYPE_BUTT,
+} data_type_t;
+
+typedef enum _data_endian {
+	DATA_ENDIAN_LSB = 0,
+	DATA_ENDIAN_MSB,
+	DATA_ENDIAN_BUTT,
+} data_endian_t;
+
+typedef enum _slvs_sync_mode {
+	SLVS_SYNC_MODE_SAV = 0, // SAV, EAV
+	SLVS_SYNC_MODE_SOF, // SOL, EOL, SOF, EOF
+	SLVS_SYNC_MODE_BUTT,
+} slvs_sync_mode_t;
+
+typedef struct {
+	slvs_sync_mode_t sync_mode; // sync mode: SOF, SAV
+	uint16_t info_line_en;
+	uint16_t info_line_num;
+	uint16_t ob_line_en;
+	uint16_t ob_line_num;
+	// 0: sav 1: eav 2: sav(invalid) 3: eav(invalid)
+	// 0: sol 1: eol 2: sof 3: eof
+	uint16_t sync_code[SLVS_SYNC_CODE_WORD];
+} slvs_dev_attr_t;
+
+typedef enum {
+	HISPI_MODE_PACKETIZED_SP = 0x1,
+	HISPI_MODE_STREAMING_SP = 0x2,
+	HISPI_MODE_STREAMING_S = 0x4,
+	HISPI_MODE_ACTIVESTART_SP8 = 0x8,
+	HISPI_MODE_BUTT,
+} hispi_mode_t;
+
+typedef struct {
+	hispi_mode_t hispi_mode;
+	uint16_t flr_en;
+	uint16_t crc_en;
+	data_endian_t ail_endian;
+	// Packetized SP Mode: 0: SOF 1: SOL 2: EOF 3: EOL
+	// HISPI_MODE_STREAMING_SP: 0: SOF 1: SOL 2: SOV
+	// HISPI_MODE_ACTIVESTART_SP8: 0~7: SOF word1~8, 8~15: SOL word1~8
+	union {
+		uint16_t pack_sp_sc[4];
+		uint16_t stream_sp_sc[3];
+		uint16_t act_sp_sc[16];
+	} sc;
+} hispi_dev_attr_t;
+
+typedef struct vicap_dev_attr {
+	combo_mode_t combo_mode;
+	data_type_t data_type;
+	data_endian_t data_endian;
+	frame_info_t frame_info;
+	crop_info_t crop_info;
+	phy_mode_t phy_mode;
+
+	union {
+		slvs_dev_attr_t slvs_attr; // slvs or sub-lvds
+		hispi_dev_attr_t hispi_attr; // hispi mode
+	} v;
+} vicap_dev_attr_t;
+
+#ifdef __KERNEL__
+
+typedef struct _vicap_dev_info {
+	uint32_t id;
+	uint32_t enable;
+	uint32_t phy_num;
+	dev_mode_t dev_mode;
+	vicap_dev_attr_t dev_attr;
+} vicap_dev_info_t;
+
+struct vicap_device {
+	void __iomem *base;
+	struct cdev cdev;
+	dev_t devt;
+	uint32_t id;
+	struct mutex vicap_mutex;
+	struct class *class;
+	void *private;
+};
+
+long vicap_priv_ioctl(struct vicap_device *dev, unsigned int cmd, void __user *args);
+
+#else
+//User space connections
+
+#endif
+
+#endif // __VICAP_IOCTL_H__
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/Makefile b/drivers/media/platform/eswin/isp/vvcam/videoin/Makefile
new file mode 100644
index 000000000000..d9a4be8324a3
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/Makefile
@@ -0,0 +1,82 @@
+##############################################################################
+#
+#    The MIT License (MIT)
+#
+#    Copyright (c) 2014 - 2024 Vivante Corporation
+#
+#    Permission is hereby granted, free of charge, to any person obtaining a
+#    copy of this software and associated documentation files (the "Software"),
+#    to deal in the Software without restriction, including without limitation
+#    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#    and/or sell copies of the Software, and to permit persons to whom the
+#    Software is furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#    DEALINGS IN THE SOFTWARE.
+#
+##############################################################################
+#
+#    The GPL License (GPL)
+#
+#    Copyright (C) 2014 - 2024 Vivante Corporation
+#
+#    This program is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License
+#    as published by the Free Software Foundation; either version 2
+#    of the License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software Foundation,
+#    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+#
+#    Note: This software is released under dual MIT and GPL licenses. A
+#    recipient may use this file under the terms of either the MIT license or
+#    GPL License. If you wish to use only one license not the other, you can
+#    indicate your decision by deleting one of the above license notices in your
+#    version of this file.
+#
+##############################################################################
+
+
+ifeq ($(VVCAM_PATH),)
+	VVCAM_PATH:=$(shell cd $(shell pwd)/../; pwd)
+	export VVCAM_PATH
+endif
+
+include $(PWD)/../Param.mk
+TARGET = vvcam_vi
+ifeq ($(or $(ISP_VI200),$(ISP_VI200_V2)),1)
+obj-m +=$(TARGET).o
+$(TARGET)-objs += vvcam_vi_platform.o
+$(TARGET)-objs += vvcam_vi_driver.o
+$(TARGET)-objs += vvcam_vi_hal.o
+#$(TARGET)-objs += vvcam_vi_procfs.o
+$(TARGET)-objs += ../common/vvcam_event.o
+EXTRA_CFLAGS += -I$(PWD)/../common/
+
+PWD :=$(shell pwd)
+endif
+all:
+	make -C $(KERNEL_DIR) M=$(PWD) modules
+modules_install:
+	make -C $(KERNEL_DIR) M=$(PWD) modules_install
+clean:
+	rm -rf $($(TARGET)-objs)
+	make -C $(KERNEL_DIR) M=$(PWD) clean
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi.h b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi.h
new file mode 100644
index 000000000000..2836d2758acc
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi.h
@@ -0,0 +1,113 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VI_H__
+#define __VVCAM_VI_H__
+#include "vvcam_event.h"
+
+enum {
+    VVCAM_EID_VIDEO_IN_MCM_PATH0        = 0x00,
+    VVCAM_EID_VIDEO_IN_MCM_PATH1,
+    VVCAM_EID_VIDEO_IN_MCM_PATH2,
+    VVCAM_EID_VIDEO_IN_MCM_PATH3,
+    VVCAM_EID_VIDEO_IN_MCM_PATH4,
+    VVCAM_EID_VIDEO_IN_MCM_PATH5,
+    VVCAM_EID_VIDEO_IN_MCM_PATH6,
+    VVCAM_EID_VIDEO_IN_MCM_PATH7,
+    VVCAM_EID_VIDEO_IN_MCM_PATH8,
+    VVCAM_EID_VIDEO_IN_MCM_PATH9,
+    VVCAM_EID_VIDEO_IN_MCM_PATH10,
+    VVCAM_EID_VIDEO_IN_MCM_PATH11,
+    VVCAM_EID_VIDEO_IN_MCM_PATH12,
+    VVCAM_EID_VIDEO_IN_MCM_PATH13,
+    VVCAM_EID_VIDEO_IN_MCM_PATH14,
+    VVCAM_EID_VIDEO_IN_MCM_PATH15,
+    VVCAM_EID_VIDEO_IN_MCM_MAX                      ,
+};
+
+enum {
+    VVCAM_EID_VIDEO_IN_ADAPT_0  = VVCAM_EID_VIDEO_IN_MCM_PATH15 + 1,
+    VVCAM_EID_VIDEO_IN_ADAPT_1,
+    VVCAM_EID_VIDEO_IN_ADAPT_2,
+    VVCAM_EID_VIDEO_IN_ADAPT_3,
+    VVCAM_EID_VIDEO_IN_ADAPT_MAX                      ,
+};
+
+enum {
+    VVCAM_EID_VIDEO_IN_FUSA_LV1 = VVCAM_EID_VIDEO_IN_ADAPT_3 + 1,
+    VVCAM_EID_VIDEO_IN_FUSA_ECC,
+    VVCAM_EID_VIDEO_IN_FUSA_PARITY_1,
+    VVCAM_EID_VIDEO_IN_FUSA_PARITY_2,
+    VVCAM_EID_VIDEO_IN_FUSA_ADAP_PIXCNT,
+    VVCAM_EID_VIDEO_IN_FUSA_RDC_PIXCNT,
+    VVCAM_EID_VIDEO_IN_FUSA_RDC_BIST_1,
+    VVCAM_EID_VIDEO_IN_FUSA_RDC_BIST_2,
+    VVCAM_EID_VIDEO_IN_FUSA_MAX
+};
+
+typedef struct {
+    uint32_t addr;
+    uint32_t value;
+} vvcam_vi_reg_t;
+
+#define VVCAM_VI_IOC_MAGIC 'I'          //video in magic
+#define VVCAM_VI_RESET             _IOW(VVCAM_VI_IOC_MAGIC, 0x01, uint32_t)
+#define VVCAM_VI_READ_REG          _IOWR(VVCAM_VI_IOC_MAGIC, 0x02, vvcam_vi_reg_t)
+#define VVCAM_VI_WRITE_REG         _IOW(VVCAM_VI_IOC_MAGIC, 0x03, vvcam_vi_reg_t)
+#define VVCAM_VI_SUBSCRIBE_EVENT   _IOW(VVCAM_VI_IOC_MAGIC, 0x04, vvcam_subscription_t)
+#define VVCAM_VI_UNSUBSCRIBE_EVENT _IOW(VVCAM_VI_IOC_MAGIC, 0x05, vvcam_subscription_t)
+#define VVCAM_VI_DQEVENT           _IOR(VVCAM_VI_IOC_MAGIC, 0x06, vvcam_event_t)
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_driver.c b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_driver.c
new file mode 100644
index 000000000000..ca001ca7f5ab
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_driver.c
@@ -0,0 +1,438 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/pm_runtime.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+
+#include "vvcam_vi_driver.h"
+#include "vvcam_vi_platform.h"
+#include "vvcam_vi.h"
+#include "vvcam_event.h"
+#include "vvcam_vi_procfs.h"
+
+extern void vvcam_vi_irq_stat_tasklet(unsigned long);
+
+
+static irqreturn_t vvcam_vi_irq_handler(int irq, void *vi_dev)
+{
+
+    return vvcam_vi_irq_process(vi_dev);
+}
+
+static int vvcam_vi_open(struct inode *inode, struct file *file)
+{
+    struct miscdevice *pmisc_dev = file->private_data;
+    struct vvcam_vi_dev *vi_dev;
+    struct vvcam_vi_fh *vi_fh;
+    printk("enter %s\n", __func__);
+    vi_dev = container_of(pmisc_dev, struct vvcam_vi_dev, miscdev);
+    if (!vi_dev)
+        return -ENOMEM;
+
+    vi_fh = kzalloc(sizeof(struct vvcam_vi_dev), GFP_KERNEL);
+    if (!vi_fh)
+        return -ENOMEM;
+
+    vi_fh->vi_dev = vi_dev;
+    vvcam_event_fh_init( &vi_dev->event_dev, &vi_fh->event_fh);
+    file->private_data = vi_fh;
+
+    mutex_lock(&vi_dev->mlock);
+
+    if (vi_dev->refcnt == 0) {
+        enable_irq(vi_dev->vi_irq);
+    }
+    vi_dev->refcnt++;
+    printk("%s vi_dev->refcnt %d\n ", __func__, vi_dev->refcnt);
+    pm_runtime_get_sync(vi_dev->dev);
+    mutex_unlock(&vi_dev->mlock);
+    printk("exit %s\n", __func__);
+
+    return 0;
+}
+
+static int vvcam_vi_release(struct inode *inode, struct file *file)
+{
+    struct vvcam_vi_dev *vi_dev;
+    struct vvcam_vi_fh *vi_fh;
+    printk("enter %s\n", __func__);
+
+    vi_fh = file->private_data;
+    vi_dev = vi_fh->vi_dev;
+
+    vvcam_event_unsubscribe_all(&vi_fh->event_fh);
+
+    vvcam_event_fh_destroy(&vi_dev->event_dev, &vi_fh->event_fh);
+
+    mutex_lock(&vi_dev->mlock);
+    vi_dev->refcnt--;
+
+    if (vi_dev->refcnt == 0) {
+        disable_irq(vi_dev->vi_irq);
+    }
+
+    printk("%s vi_dev->refcnt %d\n ", __func__, vi_dev->refcnt);
+
+    pm_runtime_put(vi_dev->dev);
+    mutex_unlock(&vi_dev->mlock);
+
+    kfree(vi_fh);
+    printk("exit %s\n", __func__);
+
+    return 0;
+}
+
+static long vvcam_vi_ioctl(struct file *file,
+                        unsigned int cmd, unsigned long arg)
+{
+    struct vvcam_vi_dev *vi_dev;
+    struct vvcam_vi_fh *vi_fh;
+    uint32_t reset;
+    vvcam_vi_reg_t vi_reg;
+    vvcam_subscription_t sub;
+    vvcam_event_t event;
+    int ret = 0;
+
+    vi_fh = file->private_data;
+    vi_dev = vi_fh->vi_dev;
+
+    mutex_lock(&vi_dev->mlock);
+    switch(cmd) {
+    case VVCAM_VI_RESET:
+        ret = copy_from_user(&reset, (void __user *)arg, sizeof(reset));
+        if (ret)
+            break;
+        ret = vvcam_vi_reset(vi_dev, reset);
+        break;
+    case VVCAM_VI_READ_REG:
+        ret = copy_from_user(&vi_reg, (void __user *)arg, sizeof(vi_reg));
+        if (ret)
+            break;
+        ret = vvcam_vi_read_reg(vi_dev, &vi_reg);
+        if (ret)
+            break;
+        ret = copy_to_user((void __user *)arg, &vi_reg, sizeof(vi_reg));
+        break;
+    case VVCAM_VI_WRITE_REG:
+        ret = copy_from_user(&vi_reg, (void __user *)arg, sizeof(vi_reg));
+        if (ret)
+            break;
+        ret = vvcam_vi_write_reg(vi_dev, vi_reg);
+        break;
+    case VVCAM_VI_SUBSCRIBE_EVENT:
+        ret = copy_from_user(&sub, (void __user *)arg, sizeof(sub));
+        if (ret)
+            break;
+        
+        ret = vvcam_event_subscribe(&vi_fh->event_fh,
+                    &sub, VVCAM_VI_EVENT_ELEMS);
+        break;
+    case VVCAM_VI_UNSUBSCRIBE_EVENT:
+        ret = copy_from_user(&sub, (void __user *)arg, sizeof(sub));
+        if (ret)
+            break;
+        ret = vvcam_event_unsubscribe(&vi_fh->event_fh, &sub);
+        break;
+    case VVCAM_VI_DQEVENT:
+        ret = vvcam_event_dequeue(&vi_fh->event_fh, &event);
+        if (ret)
+            break;
+        ret = copy_to_user((void __user *)arg, &event, sizeof(event));
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    mutex_unlock(&vi_dev->mlock);
+
+    return ret;
+}
+
+static unsigned int vvcam_vi_poll(struct file *file, poll_table *wait)
+{
+    struct vvcam_vi_fh *vi_fh;
+    vi_fh = file->private_data;
+
+    return vvcam_event_poll(file, &vi_fh->event_fh, wait);
+}
+
+int vvcam_vi_mmap(struct file *file, struct vm_area_struct *vma)
+{
+    struct vvcam_vi_dev *vi_dev;
+    struct vvcam_vi_fh *vi_fh;
+
+    vi_fh = file->private_data;
+    vi_dev = vi_fh->vi_dev;
+
+    vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+    return vm_iomap_memory(vma, vi_dev->paddr, vi_dev->regs_size);
+}
+
+static struct file_operations vvcam_vi_fops = {
+    .owner          = THIS_MODULE,
+    .open           = vvcam_vi_open,
+    .release        = vvcam_vi_release,
+    .unlocked_ioctl = vvcam_vi_ioctl,
+    .poll           = vvcam_vi_poll,
+    .mmap           = vvcam_vi_mmap,
+
+};
+
+static int vvcam_vi_parse_params(struct vvcam_vi_dev *vi_dev,
+                        struct platform_device *pdev)
+{
+    struct resource *res;
+
+    res =  platform_get_resource(pdev, IORESOURCE_MEM, 0);
+    if (!res) {
+        dev_err(&pdev->dev, "can't fetch device resource info\n");
+        return -EIO;
+    }
+    vi_dev->paddr = res->start;
+    vi_dev->regs_size = resource_size(res);
+    vi_dev->base = devm_ioremap_resource(&pdev->dev, res);
+    if (IS_ERR(vi_dev->base)) {
+        dev_err(&pdev->dev, "can't remap device resource info\n");
+        return PTR_ERR(vi_dev->base);
+    }
+
+    vi_dev->vi_irq = platform_get_irq(pdev, 0);
+    if (vi_dev->vi_irq < 0) {
+        dev_err(&pdev->dev, "can't get irq resource\n");
+        return -ENXIO;
+    }
+
+    return 0;
+}
+
+static int vvcam_vi_probe(struct platform_device *pdev)
+{
+    int ret = 0;
+    struct vvcam_vi_dev *vi_dev;
+    char *videv_name;
+
+    vi_dev = devm_kzalloc(&pdev->dev,
+                sizeof(struct vvcam_vi_dev), GFP_KERNEL);
+    if (!vi_dev)
+        return -ENOMEM;
+
+    ret = vvcam_vi_parse_params(vi_dev, pdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to parse params\n");
+        return -EINVAL;
+    }
+
+    mutex_init(&vi_dev->mlock);
+    spin_lock_init(&vi_dev->stat_lock);
+    vvcam_event_dev_init(&vi_dev->event_dev);
+    platform_set_drvdata(pdev, vi_dev);
+
+    vi_dev->dev = &pdev->dev;
+    vi_dev->id = pdev->id;
+
+    videv_name = devm_kzalloc(&pdev->dev, 16, GFP_KERNEL);
+    if (!videv_name)
+        return -ENOMEM;
+    snprintf(videv_name, 16, "%s.%d", VVCAM_VI_NAME, pdev->id);
+
+    vi_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
+    vi_dev->miscdev.name  = videv_name;
+    vi_dev->miscdev.fops  = &vvcam_vi_fops;
+
+    ret = misc_register(&vi_dev->miscdev);
+    if (ret) {
+        dev_err(&pdev->dev, "failed to register device\n");
+        return -EINVAL;
+    }
+
+    // ret = vvcam_vi_procfs_register(vi_dev, &vi_dev->pde);
+    // if (ret) {
+    //     dev_err(&pdev->dev, "vi register procfs failed.\n");
+    //     goto error_request_vi_irq;
+    // }
+
+    // tasklet_init(&vi_dev->stat_tasklet,
+    //     vvcam_vi_irq_stat_tasklet, (unsigned long)vi_dev);
+
+    ret = devm_request_irq(&pdev->dev, vi_dev->vi_irq, vvcam_vi_irq_handler,
+                IRQF_TRIGGER_HIGH | IRQF_SHARED, dev_name(&pdev->dev), vi_dev);
+    if (ret) {
+        dev_err(&pdev->dev, "can't request vi irq\n");
+        goto error_request_vi_irq;
+    }
+
+    disable_irq(vi_dev->vi_irq);
+    dev_info(&pdev->dev, "vvcam vi driver probe success\n");
+
+    return 0;
+
+
+error_request_vi_irq:
+    misc_deregister(&vi_dev->miscdev);
+    return ret;
+}
+
+static int vvcam_vi_remove(struct platform_device *pdev)
+{
+    struct vvcam_vi_dev *vi_dev;
+
+    vi_dev = platform_get_drvdata(pdev);
+
+    // vvcam_vi_procfs_unregister(vi_dev->pde);
+
+    misc_deregister(&vi_dev->miscdev);
+    enable_irq(vi_dev->vi_irq);
+
+    devm_free_irq(&pdev->dev, vi_dev->vi_irq, vi_dev);
+
+    return 0;
+}
+
+static int vvcam_vi_system_suspend(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_suspend(dev);
+    if (ret) {
+        dev_err(dev, "force suspend %s failed\n", dev_name(dev));
+        return ret;
+    }
+    return ret;
+}
+
+static int vvcam_vi_system_resume(struct device *dev)
+{
+    int ret = 0;
+    ret = pm_runtime_force_resume(dev);
+    if (ret) {
+        dev_err(dev, "force resume %s failed\n", dev_name(dev));
+        return ret;
+    }
+    return ret;
+}
+
+static int vvcam_vi_runtime_suspend(struct device *dev)
+{
+    return 0;
+}
+
+static int vvcam_vi_runtime_resume(struct device *dev)
+{
+    return 0;
+}
+
+static const struct dev_pm_ops vvcam_vi_pm_ops = {
+    SET_SYSTEM_SLEEP_PM_OPS(vvcam_vi_system_suspend, vvcam_vi_system_resume)
+    SET_RUNTIME_PM_OPS(vvcam_vi_runtime_suspend, vvcam_vi_runtime_resume, NULL)
+};
+
+static const struct of_device_id vvcam_vi_of_match[] = {
+    {.compatible = "verisilicon,vi",},
+    { /* sentinel */ },
+};
+
+static struct platform_driver vvcam_vi_driver = {
+    .probe  = vvcam_vi_probe,
+    .remove = vvcam_vi_remove,
+    .driver = {
+        .name           = VVCAM_VI_NAME,
+        .owner          = THIS_MODULE,
+        .of_match_table = vvcam_vi_of_match,
+        .pm             = &vvcam_vi_pm_ops,
+    }
+};
+
+static int __init vvcam_vi_init_module(void)
+{
+    int ret;
+    ret = platform_driver_register(&vvcam_vi_driver);
+    if (ret) {
+        printk(KERN_ERR "Failed to register vi driver\n");
+        return ret;
+    }
+
+    ret = vvcam_vi_platform_device_register();
+    if (ret) {
+        platform_driver_unregister(&vvcam_vi_driver);
+        printk(KERN_ERR "Failed to register vvcam vi platform devices\n");
+        return ret;
+    }
+
+    return ret;
+}
+
+static void __exit vvcam_vi_exit_module(void)
+{
+    platform_driver_unregister(&vvcam_vi_driver);
+    vvcam_vi_platform_device_unregister();
+}
+
+late_initcall(vvcam_vi_init_module);
+module_exit(vvcam_vi_exit_module);
+
+MODULE_DESCRIPTION("Verisilicon video in driver");
+MODULE_AUTHOR("Verisilicon ISP SW Team");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_driver.h b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_driver.h
new file mode 100644
index 000000000000..abf590d8a7ae
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_driver.h
@@ -0,0 +1,100 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VI_DRIVER_H__
+#define __VVCAM_VI_DRIVER_H__
+
+#include <linux/miscdevice.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+
+#include "vvcam_vi.h"
+#include "vvcam_event.h"
+
+#define VVCAM_VI_NAME "vvcam-vi"
+
+#define VVCAM_VI_EVENT_ELEMS 10
+
+struct vvcam_vi_dev {
+    phys_addr_t paddr;
+    uint32_t regs_size;
+    void __iomem *base;
+    void __iomem *reset;
+    int  id;
+    int vi_irq;
+    // int fe_irq;
+    // int mi_irq;
+    struct miscdevice miscdev;
+    struct device *dev;
+    unsigned long pde;
+    struct mutex mlock;
+    struct vvcam_event_dev event_dev;
+    // uint32_t irq_mis[VVCAM_EID_MAX];
+    struct tasklet_struct stat_tasklet;
+    spinlock_t stat_lock;
+    uint32_t refcnt;
+};
+
+struct vvcam_vi_fh {
+    struct vvcam_vi_dev *vi_dev;
+    struct vvcam_event_fh event_fh;
+};
+
+int vvcam_vi_reset(struct vvcam_vi_dev *vi_dev, uint32_t reset);
+int vvcam_vi_write_reg(struct vvcam_vi_dev *vi_dev, vvcam_vi_reg_t vi_reg);
+int vvcam_vi_read_reg(struct vvcam_vi_dev *vi_dev, vvcam_vi_reg_t *vi_reg);
+irqreturn_t vvcam_vi_irq_process(struct vvcam_vi_dev *vi_dev);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_hal.c b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_hal.c
new file mode 100644
index 000000000000..4fc483e5cc8b
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_hal.c
@@ -0,0 +1,304 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include "vvcam_vi_driver.h"
+#include "vvcam_vi_hal.h"
+
+extern void vvcam_vi_proc_stat(unsigned long pde,
+                    const uint32_t *irq_mis, const int len);
+
+void vvcam_vi_irq_stat_tasklet(unsigned long vi_dev)
+{
+    // struct vvcam_vi_dev *vi_dev = (struct vvcam_vi_dev *)vi_dev;
+    // uint32_t irq_mis[VVCAM_EID_MAX];
+    //     printk("enter %s\n", __func__);
+
+    // memset(irq_mis, 0, sizeof(irq_mis));
+
+    // memcpy(irq_mis, vi_dev->irq_mis, sizeof(irq_mis));
+    // memset(vi_dev->irq_mis, 0, sizeof(vi_dev->irq_mis));
+
+    // vvcam_vi_proc_stat(vi_dev->pde, irq_mis, VVCAM_EID_MAX);
+    printk("exit %s\n", __func__);
+}
+
+static void vvcam_vi_hal_write_reg(void __iomem *base,
+			uint32_t addr, uint32_t value)
+{
+    writel(value, base + addr);
+}
+
+static int vvcam_vi_hal_read_reg(void __iomem *base, uint32_t addr)
+{
+    return readl(base + addr);
+}
+
+int vvcam_vi_reset(struct vvcam_vi_dev *vi_dev, uint32_t reset)
+{
+    // uint32_t reset_reg = 0;
+
+    // reset_reg = vvcam_vi_hal_read_reg(vi_dev->reset, 0);
+
+    // if (reset) {
+    //     /* reset */
+    //     reset_reg &= ~TPG_ISP_RST_MASK;
+    // } else {
+    //     /* unreset */
+    //     reset_reg |= TPG_ISP_RST_MASK;
+    // }
+
+    // vvcam_vi_hal_write_reg(vi_dev->reset, 0, reset_reg);
+
+    return 0;
+}
+
+int vvcam_vi_write_reg(struct vvcam_vi_dev *vi_dev, vvcam_vi_reg_t vi_reg)
+{
+    vvcam_vi_hal_write_reg(vi_dev->base, vi_reg.addr, vi_reg.value);
+    return 0;
+}
+
+int vvcam_vi_read_reg(struct vvcam_vi_dev *vi_dev, vvcam_vi_reg_t *vi_reg)
+{
+    uint32_t reg_value = 0;
+
+    reg_value = vvcam_vi_hal_read_reg(vi_dev->base, vi_reg->addr);
+
+    vi_reg->value = reg_value;
+
+    return 0;
+}
+
+
+irqreturn_t vvcam_vi_irq_process(struct vvcam_vi_dev *vi_dev)
+{
+    uint32_t mcm_mis[4] = {0};
+    uint32_t fusa_mis[8] = {0};
+    uint32_t adapt_mis[4] = {0};
+    uint64_t timestamp;
+    uint32_t i = 0;
+    vvcam_event_t event;
+    bool irq = false;
+    // printk("enter vvcam_vi_irq_process \n");
+
+    mcm_mis[0] = vvcam_vi_hal_read_reg(vi_dev->base, MCM0_MI_MIS2);
+    if (mcm_mis[0]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, MCM0_MI_ICR2, mcm_mis[0]);
+        irq = true;
+    }
+    mcm_mis[1] = vvcam_vi_hal_read_reg(vi_dev->base, MCM1_MI_MIS2);
+    if ( mcm_mis[1] ) {
+        vvcam_vi_hal_write_reg(vi_dev->base, MCM1_MI_ICR2, mcm_mis[1]);
+        irq = true;
+    }
+    mcm_mis[2] = vvcam_vi_hal_read_reg(vi_dev->base, MCM2_MI_MIS2);
+    if ( mcm_mis[2] ) {
+        vvcam_vi_hal_write_reg(vi_dev->base, MCM2_MI_ICR2, mcm_mis[2]);
+        irq = true;
+    }
+
+    mcm_mis[3] = vvcam_vi_hal_read_reg(vi_dev->base, MCM3_MI_MIS2);
+    if (mcm_mis[3]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, MCM3_MI_ICR2, mcm_mis[3]);
+        irq = true;
+    }
+    //fusa
+    fusa_mis[0] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_LV1_MIS1);
+    if (fusa_mis[0]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_LV1_ICR1, fusa_mis[0]);
+        irq = true;
+    }
+    fusa_mis[1] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_ECC_MIS1);
+    if (fusa_mis[1]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_ECC_ICR1, fusa_mis[1]);
+        irq = true;
+    }
+    fusa_mis[2] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_PARITY_MIS1);
+    if (fusa_mis[2]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_PARITY_ICR1, fusa_mis[2]);
+        irq = true;
+    }
+    fusa_mis[3] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_PARITY_MIS2);
+    if (fusa_mis[3]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_PARITY_ICR2, fusa_mis[3]);
+        irq = true;
+    }
+    fusa_mis[4] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_ADAP_PIX_LOSS_MIS1);
+    if (fusa_mis[4]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_ADAP_PIX_LOSS_ICR1, fusa_mis[4]);
+        irq = true;
+    }
+    fusa_mis[5] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_RDC_PIX_LOSS_MIS1);
+    if (fusa_mis[5]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_RDC_PIX_LOSS_ICR1, fusa_mis[5]);
+        irq = true;
+    }
+    fusa_mis[6] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_RDC_TPG_BIST_MIS1);
+    if (fusa_mis[6]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_RDC_TPG_BIST_ICR1, fusa_mis[6]);
+        irq = true;
+    }
+    fusa_mis[7] = vvcam_vi_hal_read_reg(vi_dev->base, VI_FUSA_RDC_TPG_BIST_MIS2);
+    if (fusa_mis[7]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, VI_FUSA_RDC_TPG_BIST_ICR2, fusa_mis[7]);
+        irq = true;
+    }
+
+    adapt_mis[0] = vvcam_vi_hal_read_reg(vi_dev->base, ADAPT0_MIS);
+    if (adapt_mis[0]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, ADAPT0_ICR, adapt_mis[0]);
+        irq = true;
+    }
+
+    adapt_mis[1] = vvcam_vi_hal_read_reg(vi_dev->base, ADAPT1_MIS);
+    if (adapt_mis[1]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, ADAPT1_ICR, adapt_mis[1]);
+        irq = true;
+    }
+
+    adapt_mis[2] = vvcam_vi_hal_read_reg(vi_dev->base, ADAPT2_MIS);
+    if (adapt_mis[2]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, ADAPT2_ICR, adapt_mis[2]);
+        irq = true;
+    }
+
+    adapt_mis[3] = vvcam_vi_hal_read_reg(vi_dev->base, ADAPT3_MIS);
+    if (adapt_mis[3]) {
+        vvcam_vi_hal_write_reg(vi_dev->base, ADAPT3_ICR, adapt_mis[3]);
+        irq = true;
+    }
+
+    timestamp = ktime_get_ns();
+
+    for (i = VVCAM_EID_VIDEO_IN_FUSA_LV1; i < VVCAM_EID_VIDEO_IN_FUSA_MAX; i += 1){
+        if(fusa_mis[i - VVCAM_EID_VIDEO_IN_FUSA_LV1]){
+            // printk("vi200 fusa %d mis : 0x%08x \n", i, fusa_mis[i - VVCAM_EID_VIDEO_IN_FUSA_LV1]);
+
+            event.type = VVCAM_VIDEO_IN_EVENT_IRQ_TYPE;
+            event.timestamp = timestamp;
+            event.id   = i ;
+            event.irqevent.irq_value = fusa_mis[i - VVCAM_EID_VIDEO_IN_FUSA_LV1];
+            vvcam_event_queue(&vi_dev->event_dev, &event);
+        }
+    }
+
+    for (i = VVCAM_EID_VIDEO_IN_ADAPT_0; i < VVCAM_EID_VIDEO_IN_ADAPT_MAX; i += 1){
+        if(adapt_mis[i - VVCAM_EID_VIDEO_IN_ADAPT_0]){
+            // printk("vi200 adaptor %d mis : 0x%08x \n", i - VVCAM_EID_VIDEO_IN_ADAPT_0, adapt_mis[i - VVCAM_EID_VIDEO_IN_ADAPT_0]);
+            event.type = VVCAM_VIDEO_IN_EVENT_IRQ_TYPE;
+            event.timestamp = timestamp;
+            event.id   = i;
+            event.irqevent.irq_value = adapt_mis[i - VVCAM_EID_VIDEO_IN_ADAPT_0];
+            vvcam_event_queue(&vi_dev->event_dev, &event);
+        }
+    }
+
+    for (i = 0; i < VVCAM_EID_VIDEO_IN_MCM_MAX; i += 4) {
+        if (mcm_mis[i /4]) {
+            // printk("vi200 mcm mis2 : 0x%08x, i:%d \n", mcm_mis[i /4], i);
+            event.type = VVCAM_VIDEO_IN_EVENT_IRQ_TYPE;
+            event.timestamp = timestamp;
+
+            if (mcm_mis[i /4] & (CH0_L_FRAME_END_MASK | CH0_S_FRAME_END_MASK |
+                                 CH0_VS_FRAME_END_MASK | CH0_XS_FRAME_END_MASK)) {
+                event.id   = i;
+                event.irqevent.irq_value = (mcm_mis[i /4] & (CH0_L_FRAME_END_MASK |
+                                            CH0_S_FRAME_END_MASK |
+                                            CH0_VS_FRAME_END_MASK |
+                                            CH0_XS_FRAME_END_MASK));
+                vvcam_event_queue(&vi_dev->event_dev, &event);
+            }
+            if (mcm_mis[i /4] &(CH1_L_FRAME_END_MASK | CH1_S_FRAME_END_MASK |
+                                 CH1_VS_FRAME_END_MASK | CH1_XS_FRAME_END_MASK)) {
+                event.id   = i + 1;
+                event.irqevent.irq_value = (mcm_mis[i /4] & (CH1_L_FRAME_END_MASK |
+                                            CH1_S_FRAME_END_MASK |
+                                            CH1_VS_FRAME_END_MASK |
+                                            CH1_XS_FRAME_END_MASK));
+                vvcam_event_queue(&vi_dev->event_dev, &event);
+            }
+            if (mcm_mis[i /4] & (CH2_L_FRAME_END_MASK | CH2_S_FRAME_END_MASK |
+                                 CH2_VS_FRAME_END_MASK | CH2_XS_FRAME_END_MASK)) {
+                event.id   = i + 2;
+                event.irqevent.irq_value = (mcm_mis[i /4] & (CH2_L_FRAME_END_MASK |
+                                            CH2_S_FRAME_END_MASK |
+                                            CH2_VS_FRAME_END_MASK |
+                                            CH2_XS_FRAME_END_MASK));
+                vvcam_event_queue(&vi_dev->event_dev, &event);
+            }
+            if (mcm_mis[i /4] & (CH3_L_FRAME_END_MASK | CH3_S_FRAME_END_MASK |
+                                 CH3_VS_FRAME_END_MASK | CH3_XS_FRAME_END_MASK)) {
+                event.id   = i + 3;
+                event.irqevent.irq_value = (mcm_mis[i /4] & (CH3_L_FRAME_END_MASK |
+                                            CH3_S_FRAME_END_MASK |
+                                            CH3_VS_FRAME_END_MASK |
+                                            CH3_XS_FRAME_END_MASK));
+                vvcam_event_queue(&vi_dev->event_dev, &event);
+            }
+
+        }
+
+    }
+    if (irq) {
+        // vi_dev->irq_mis[VVCAM_EID_mcm_mis] = mcm0_mis;
+        // tasklet_schedule(&vi_dev->stat_tasklet);
+        return IRQ_HANDLED;
+    }
+    return IRQ_HANDLED;
+}
+
+
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_hal.h b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_hal.h
new file mode 100644
index 000000000000..7d33c79c6ddf
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_hal.h
@@ -0,0 +1,115 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VI_HAL_H__
+#define __VVCAM_VI_HAL_H__
+
+
+#define VI_FUSA_LV1_MIS1  0x2190
+#define VI_FUSA_ECC_MIS1  0x21a4
+#define VI_FUSA_PARITY_MIS1  0x21b8
+#define VI_FUSA_PARITY_MIS2  0x21cc
+#define VI_FUSA_ADAP_PIX_LOSS_MIS1  0x21e0
+#define VI_FUSA_RDC_PIX_LOSS_MIS1  0x21f8
+#define VI_FUSA_RDC_TPG_BIST_MIS1  0x2210
+#define VI_FUSA_RDC_TPG_BIST_MIS2  0x2224
+
+#define VI_FUSA_LV1_ICR1  0x2198
+#define VI_FUSA_ECC_ICR1  0x21ac
+#define VI_FUSA_PARITY_ICR1  0x21c0
+#define VI_FUSA_PARITY_ICR2  0x21d4
+#define VI_FUSA_ADAP_PIX_LOSS_ICR1  0x21e8
+#define VI_FUSA_RDC_PIX_LOSS_ICR1  0x2200
+#define VI_FUSA_RDC_TPG_BIST_ICR1  0x2218
+#define VI_FUSA_RDC_TPG_BIST_ICR2  0x222c
+
+#define  MCM0_MI_MIS2  0x1224
+#define  MCM1_MI_MIS2  0x1624
+#define  MCM2_MI_MIS2  0x1a24
+#define  MCM3_MI_MIS2  0x1e24
+
+#define  MCM0_MI_ICR2  0x122c
+#define  MCM1_MI_ICR2  0x162c
+#define  MCM2_MI_ICR2  0x1a2c
+#define  MCM3_MI_ICR2  0x1e2c
+
+#define  ADAPT0_MIS   0x0134
+#define  ADAPT1_MIS   0x0534
+#define  ADAPT2_MIS   0x0934
+#define  ADAPT3_MIS   0x0D34
+
+#define  ADAPT0_ICR   0x012C
+#define  ADAPT1_ICR   0x052C
+#define  ADAPT2_ICR   0x092C
+#define  ADAPT3_ICR   0x0D2C
+
+#define  CH0_XS_FRAME_END_MASK  (1U << 31)
+#define  CH0_VS_FRAME_END_MASK  (1U << 30)
+#define  CH0_S_FRAME_END_MASK   (1U << 29)
+#define  CH0_L_FRAME_END_MASK   (1U << 28)
+#define  CH1_XS_FRAME_END_MASK  (1U << 27)
+#define  CH1_VS_FRAME_END_MASK  (1U << 26)
+#define  CH1_S_FRAME_END_MASK   (1U << 25)
+#define  CH1_L_FRAME_END_MASK   (1U << 24)
+#define  CH2_XS_FRAME_END_MASK  (1U << 23)
+#define  CH2_VS_FRAME_END_MASK  (1U << 22)
+#define  CH2_S_FRAME_END_MASK   (1U << 21)
+#define  CH2_L_FRAME_END_MASK   (1U << 20)
+#define  CH3_XS_FRAME_END_MASK  (1U << 19)
+#define  CH3_VS_FRAME_END_MASK  (1U << 18)
+#define  CH3_S_FRAME_END_MASK   (1U << 17)
+#define  CH3_L_FRAME_END_MASK   (1U << 16)
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_platform.c b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_platform.c
new file mode 100644
index 000000000000..8617d3976014
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_platform.c
@@ -0,0 +1,151 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+
+#include "vvcam_vi_platform.h"
+
+static int devices_mask = 0x01;
+MODULE_PARM_DESC(devices_mask, "vvcam videoin devices mask");
+
+static void vvcam_vi_pdev_release(struct device *dev)
+{
+    dev_info(dev, "%s\n", __func__);
+    return;
+}
+
+static struct resource vvcam_vi1_resource[] = {
+    [0] = {
+        .start = VVCAM_VI1_BASE,
+        .end   = VVCAM_VI1_BASE + VVCAM_VI_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+    [1] = {
+        .start = VVCAM_VI1_IRQ,
+        .end   = VVCAM_VI1_IRQ,
+        .flags = IORESOURCE_IRQ,
+    },
+};
+
+static struct platform_device vvcam_vi1_pdev = {
+    .name          = "vvcam-vi",
+    .id            = 1,
+    .resource      = vvcam_vi1_resource,
+    .num_resources = ARRAY_SIZE(vvcam_vi1_resource),
+    .dev.release   = vvcam_vi_pdev_release,
+};
+
+static struct resource vvcam_vi0_resource[] = {
+    [0] = {
+        .start = VVCAM_VI0_BASE + PCIE_BASE,
+        .end   = VVCAM_VI0_BASE + PCIE_BASE + VVCAM_VI_REG_SIZE - 1,
+        .flags = IORESOURCE_MEM,
+    },
+    [1] = {
+        .start = VVCAM_VI0_IRQ,
+        .end   = VVCAM_VI0_IRQ,
+        .flags = IORESOURCE_IRQ,
+    },
+};
+
+static struct platform_device vvcam_vi0_pdev = {
+    .name          = "vvcam-vi",
+    .id            = 0,
+    .resource      = vvcam_vi0_resource,
+    .num_resources = ARRAY_SIZE(vvcam_vi0_resource),
+    .dev.release   = vvcam_vi_pdev_release,
+};
+
+int vvcam_vi_platform_device_register(void)
+{
+    int ret = 0;
+    int devs_mask = 0;
+
+    if (devices_mask & 0x01) {
+
+        ret = platform_device_register(&vvcam_vi0_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x01;
+    }
+    if (devices_mask & 0x02) {
+        ret = platform_device_register(&vvcam_vi1_pdev);
+        if (ret) {
+            goto err_register;
+        }
+        devs_mask |= 0x02;
+    }
+    return ret;
+
+err_register:
+    if (devs_mask & 0x01)
+        platform_device_unregister(&vvcam_vi0_pdev);
+    if (devs_mask & 0x02)
+        platform_device_unregister(&vvcam_vi1_pdev);
+    return ret;
+}
+
+void vvcam_vi_platform_device_unregister(void)
+{
+    if (devices_mask & 0x01)
+        platform_device_unregister(&vvcam_vi0_pdev);
+    if (devices_mask & 0x02)
+        platform_device_unregister(&vvcam_vi1_pdev);
+    return;
+}
+module_param(devices_mask, int, 0644);
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_platform.h b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_platform.h
new file mode 100644
index 000000000000..3281c4f4c403
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_platform.h
@@ -0,0 +1,62 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VI_PLATFORM_H__
+#define __VVCAM_VI_PLATFORM_H__
+
+int vvcam_vi_platform_device_register(void);
+void vvcam_vi_platform_device_unregister(void);
+
+#endif
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_procfs.c b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_procfs.c
new file mode 100644
index 000000000000..89ce352d71e9
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_procfs.c
@@ -0,0 +1,645 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/i2c.h>
+#include <linux/ctype.h>
+#include "vvcam_vi_driver.h"
+#include "vvcam_vi_procfs.h"
+#include "vvcam_vi_hal.h"
+
+#define VVCAM_VI_PROCFS_BUF_SIZE    256
+
+#define PROC_OUT_OF_RANGE       0x01
+#define PROC_PARSE_ERROR        0x02
+
+#define PROC_PRINT_STAT(stat, sfile, name) \
+    seq_printf(sfile, "|%-32s|%-16d|\n", #name, stat.name); \
+    seq_printf(sfile, "|--------------------------------|----------------|\n");
+
+
+enum {
+    PROC_CMD_RW,
+    PROC_CMD_CLEAR
+};
+
+struct rw_cmd {
+    bool is_write;
+    int8_t flags;
+    uint32_t start_addr;
+    uint8_t num_elems;
+    uint32_t *rw_buffer;
+    struct list_head list;
+};
+
+struct vvcam_vi_irq_stat {
+    uint32_t isp_frame_in_cnt;
+    uint32_t isp_frame_out_cnt;
+
+    uint32_t mp_frame_out_cnt;
+    uint32_t sp_frame_out_cnt;
+    uint32_t sp2_frame_out_cnt;
+
+    uint32_t mcm_raw0_frame_cnt;
+    uint32_t mcm_raw1_frame_cnt;
+    uint32_t mcm_g2_raw0_frame_cnt;
+    uint32_t mcm_g2_raw1_frame_cnt;
+    uint32_t mcm_rdma_frame_cnt;
+
+    uint32_t fe_irq_cnt;
+
+    unsigned long long start;
+    unsigned long long end;
+};
+
+struct vvcam_vi_status {
+    int8_t flags;
+    int8_t type;
+    struct vvcam_vi_irq_stat  stat;
+    struct list_head cmd_list;
+};
+
+struct vvcam_vi_procfs {
+    struct proc_dir_entry *pde;
+    struct vvcam_vi_dev *vi_dev;
+    struct mutex lock;
+    struct vvcam_vi_status status;
+};
+
+void vvcam_vi_proc_stat(unsigned long pde,
+                    const uint32_t *irq_mis, const int len)
+{
+    struct vvcam_vi_procfs *vi_proc = (struct vvcam_vi_procfs*)pde;
+    struct vvcam_vi_irq_stat *stat = &vi_proc->status.stat;
+    int eid = 0;
+
+    //mutex_lock(&vi_proc->lock);
+    for (eid = 0; eid < len; eid++) {
+        if (!irq_mis[eid])
+            continue;
+
+        switch (eid)
+        {
+        case VVCAM_EID_ISP_MIS:
+            if (irq_mis[eid] & ISP_MIS_FRAME_IN_MASK)
+                stat->isp_frame_in_cnt++;
+
+            if (irq_mis[eid] & ISP_MIS_FRAME_OUT_MASK)
+                stat->isp_frame_out_cnt++;
+
+            break;
+
+        case VVCAM_EID_MIV2_MIS:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_MIV2_MIS1:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_MIV2_MIS2:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_MIV2_MIS3:
+            /* reserve */
+            break;
+
+        case VVCAM_EID_RDMA_MIS:
+            if (irq_mis[eid] & MIV2_MIS_MCM_RAW_RADY_MASK)
+                stat->mcm_rdma_frame_cnt++;
+
+            if (irq_mis[eid] & MIV2_MIS_MP_FRAME_END_MASK)
+                stat->mp_frame_out_cnt++;
+
+            if (irq_mis[eid] & MIV2_MIS_SP_FRAME_END_MASK)
+                stat->sp_frame_out_cnt++;
+
+            if (irq_mis[eid] & MIV2_MIS_SP2_FRAME_END_MASK)
+                stat->sp2_frame_out_cnt++;
+
+            break;
+
+        case VVCAM_EID_MCM_WR_RAW0_MIS:
+            stat->mcm_raw0_frame_cnt++;
+            break;
+
+        case VVCAM_EID_MCM_WR_RAW1_MIS:
+            stat->mcm_raw1_frame_cnt++;
+            break;
+
+        case VVCAM_EID_MCM_WR_G2_RAW0_MIS:
+            stat->mcm_g2_raw0_frame_cnt++;
+            break;
+
+        case VVCAM_EID_MCM_WR_G2_RAW1_MIS:
+            stat->mcm_g2_raw1_frame_cnt++;
+            break;
+
+        case VVCAM_EID_FE_MIS:
+            stat->fe_irq_cnt++;
+            break;
+        }
+    }
+    //mutex_unlock(&vi_proc->lock);
+}
+
+
+static void release_cmd_list(struct list_head *cmd_list)
+{
+    struct rw_cmd *cmd = NULL, *old = NULL;
+
+    if (!list_empty( cmd_list)) {
+        list_for_each_entry(cmd, cmd_list, list) {
+            if (old) {
+                list_del(&old->list);
+                kfree(old);
+                old = NULL;
+            }
+            if (cmd->rw_buffer)
+                kfree(cmd->rw_buffer);
+            old = cmd;
+        }
+
+        if (old) {
+            list_del(&old->list);
+            kfree(old);
+            old = NULL;
+        }
+    }
+}
+
+static int vvcam_vi_procfs_info_show(struct seq_file *sfile, void *offset)
+{
+	struct vvcam_vi_procfs *vi_proc;
+    struct vvcam_vi_dev *vi_dev;
+    struct vvcam_vi_status *status;
+    struct rw_cmd *cmd = NULL;
+    uint8_t i = 0;
+
+	vi_proc = (struct vvcam_vi_procfs *) sfile->private;
+
+    mutex_lock(&vi_proc->lock);
+    vi_dev = vi_proc->vi_dev;
+    status = &vi_proc->status;
+
+    if (status && (status->flags & PROC_PARSE_ERROR)) {
+        seq_printf(sfile, "please check input!\n");
+        seq_printf(sfile,
+        "Usage: echo [r|w][num_of_32bit_value]@[addr] [values]>[proc path]\n");
+        seq_printf(sfile,
+        "eg: echo w2@0x00005400 0x00412745 0x80252941 r2>/proc/vsi/vsiisp0 && cat /proc/vsi/vsiisp0\n");
+        status->flags = 0;
+        mutex_unlock(&vi_proc->lock);
+        return 0;
+    }
+
+    if (status && !list_empty(&status->cmd_list)) {
+        list_for_each_entry(cmd, &status->cmd_list, list) {
+            if (cmd->is_write) {
+                if (cmd->flags & PROC_OUT_OF_RANGE) {
+                    seq_printf(sfile, "write addr %08x ~ %08x out of range.\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                } else {
+                    seq_printf(sfile, "write addr %08x ~ %08x successed.\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                }
+            } else {
+                if (cmd->flags & PROC_OUT_OF_RANGE) {
+                    seq_printf(sfile, "read addr %08x ~ %08x out of range.\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                } else {
+                    seq_printf(sfile, "read addr %08x ~ %08x successed:\n",
+                    cmd->start_addr, cmd->start_addr +
+                    (uint32_t)sizeof(cmd->start_addr) * (cmd->num_elems - 1));
+                    for (i = 0; i < cmd->num_elems; i++) {
+                        seq_printf(sfile, "addr:0x%08x value:0x%08x\n",
+                        cmd->start_addr +
+                        (uint32_t)sizeof(cmd->start_addr) * i,
+                        cmd->rw_buffer[i]);
+                    }
+                }
+            }
+            cmd->flags = 0;
+        }
+        release_cmd_list(&status->cmd_list);
+        status->flags = 0;
+    } else {
+        status->stat.end = ktime_get_ns();
+        seq_printf(sfile,
+        "/***statistic for %s time(ns):%lld)***/\n",
+        vi_dev->miscdev.name, status->stat.end - status->stat.start);
+
+    seq_printf(sfile, "|--------------------------------|----------------|\n");
+        seq_printf(sfile, "|%-32s|%-16s|\n",
+        "              Name", "     Number"); \
+    seq_printf(sfile, "|--------------------------------|----------------|\n");
+        PROC_PRINT_STAT(status->stat, sfile, isp_frame_in_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, isp_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mp_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, sp_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, sp2_frame_out_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_raw0_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_raw1_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_g2_raw0_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_g2_raw1_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, mcm_rdma_frame_cnt);
+        PROC_PRINT_STAT(status->stat, sfile, fe_irq_cnt);
+    }
+    mutex_unlock(&vi_proc->lock);
+
+	return 0;
+}
+
+static int vvcam_vi_procfs_open(struct inode *inode, struct file *file)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	return single_open(file, vvcam_vi_procfs_info_show, PDE_DATA(inode));
+#else
+	return single_open(file, vvcam_vi_procfs_info_show, pde_data(inode));
+#endif
+}
+
+static char *vvcam_vi_next_arg(char *str_buf, char **param)
+{
+    int i = 0;
+    for (; str_buf[i]; i++) {
+        if (isspace(str_buf[i]))
+            break;
+    }
+    *param = str_buf;
+
+    if (str_buf[i]) {
+        str_buf[i] = '\0';
+        str_buf += i + 1;
+    } else
+        str_buf += i;
+
+    return skip_spaces(str_buf);
+}
+
+static int vvcam_vi_proc_parse_args(char *str_buf,
+                struct vvcam_vi_status *status)
+{
+    char *param, *end = NULL;
+    int32_t address = -1;
+    uint8_t read_write = 0;
+    uint8_t num_elems = 0;
+    int8_t index= 0;
+    struct rw_cmd *cmd = NULL;
+
+    if ((strlen(str_buf) == 1) || (strcmp(str_buf, "clear\n") == 0)) {
+        status->type = PROC_CMD_CLEAR;
+        return 0;
+    }
+
+    status->type = PROC_CMD_RW;
+    while (*str_buf) {
+        param = NULL;
+        str_buf = vvcam_vi_next_arg(str_buf, &param);
+        if ((*param == 'r') || (*param == 'w')) {
+            if (index != 0)
+                goto error_vi_proc_cmd;
+
+            read_write = *param;
+            num_elems = (uint8_t)simple_strtoul((param + 1), &end, 0);
+            if (num_elems != 0)
+                param = end;
+            else
+                num_elems = 1;
+
+            if (*end == '@') {
+                address = (uint32_t)simple_strtoul(param + 1, &end, 0);
+                if (((end - param - 1) > 10) || ((end - param - 1) <= 6)) {
+                    goto error_vi_proc_cmd;
+                }
+            }
+
+            if (*end != '\0')
+                goto error_vi_proc_cmd;
+
+            if (address < 0) {
+                goto error_vi_proc_cmd;
+            }
+
+            cmd = kzalloc(sizeof(struct rw_cmd), GFP_KERNEL);
+            if (!cmd)
+                goto error_vi_proc_cmd;
+
+            cmd->is_write = (read_write == 'w' ? true : false);
+            cmd->start_addr = address;
+            cmd->num_elems = num_elems;
+            cmd->rw_buffer =
+                kzalloc(sizeof(int32_t) * cmd->num_elems, GFP_KERNEL);
+            if (!cmd->rw_buffer)
+                goto error_vi_proc_cmd;
+
+            list_add_tail(&cmd->list, &status->cmd_list);
+            if (read_write == 'w')
+                index = num_elems;
+            continue;
+        }
+
+         if (*param != '\0' && read_write != 0) {
+            if (read_write == 'r')
+                goto error_vi_proc_cmd;
+
+            if (index <= 0)
+                goto error_vi_proc_cmd;
+
+            cmd->rw_buffer[num_elems - index] =
+                                (uint32_t)simple_strtoul(param, &end, 0);
+
+            if ((*end != '\0') || ((end - param) > 10) ||
+                ((end - param) <= 6))
+                goto error_vi_proc_cmd;
+
+            index--;
+        }
+    }
+    return 0;
+
+error_vi_proc_cmd:
+    release_cmd_list(&status->cmd_list);
+    return -EFAULT;
+}
+
+static int32_t vvcam_vi_proc_process(struct seq_file *sfile,
+                        struct vvcam_vi_procfs *vi_proc, char *str_buf)
+{
+    struct vvcam_vi_status *status;
+    vvcam_vi_reg_t reg;
+    struct rw_cmd *cmd = NULL;
+    int32_t ret = 0;
+    uint8_t i = 0;
+
+    status = &vi_proc->status;
+
+    mutex_lock(&vi_proc->lock);
+    if (!list_empty(&status->cmd_list)) {
+        release_cmd_list(&status->cmd_list);
+    }
+
+    status->flags = 0;
+    status->type = -1;
+
+    ret = vvcam_vi_proc_parse_args(str_buf, status);
+    if (ret) {
+        status->flags = PROC_PARSE_ERROR;
+        mutex_unlock(&vi_proc->lock);
+        return ret;
+    }
+
+    if (status->type == PROC_CMD_CLEAR) {
+        memset(&status->stat, 0, sizeof(status->stat));
+        status->stat.start = ktime_get_ns();
+    } else {
+        mutex_lock(&vi_proc->vi_dev->mlock);
+        list_for_each_entry(cmd, &status->cmd_list, list) {
+            for (i = 0; i < cmd->num_elems; i++) {
+                reg.addr = cmd->start_addr + sizeof(cmd->start_addr) * i;
+                if (reg.addr >= vi_proc->vi_dev->regs_size) {
+                    cmd->flags |= PROC_OUT_OF_RANGE;
+                    continue;
+                }
+                if (cmd->is_write) {
+                    reg.value = cmd->rw_buffer[i];
+                    ret = vvcam_vi_write_reg(vi_proc->vi_dev, reg);
+                } else {
+                    ret = vvcam_vi_read_reg(vi_proc->vi_dev, &reg);
+                    cmd->rw_buffer[i] = reg.value;
+                }
+            }
+        }
+        mutex_unlock(&vi_proc->vi_dev->mlock);
+    }
+
+    mutex_unlock(&vi_proc->lock);
+
+    return 0;
+}
+
+static ssize_t vvcam_vi_procfs_write(struct file *file,
+		const char __user *buffer, size_t count, loff_t *ppos)
+{
+    struct vvcam_vi_procfs *vi_proc;
+    struct seq_file *sfile;
+    char *str_buf;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	vi_proc = (struct vvcam_vi_procfs *) PDE_DATA(file_inode(file));
+#else
+	vi_proc = (struct vvcam_vi_procfs *) pde_data(file_inode(file));
+#endif
+    sfile = file->private_data;
+
+    if (count > VVCAM_VI_PROCFS_BUF_SIZE)
+        count = VVCAM_VI_PROCFS_BUF_SIZE - 1;
+
+    str_buf = (char *)kzalloc(count, GFP_KERNEL);
+    if (!str_buf)
+        return -ENOMEM;
+
+    if (copy_from_user(str_buf, buffer, count))
+        return -EFAULT;
+
+    *(str_buf + count) = '\0';
+
+    vvcam_vi_proc_process(sfile, vi_proc, str_buf);
+
+    kfree(str_buf);
+
+    return count;
+}
+
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
+static const struct file_operations vvcam_vi_procfs_ops = {
+    .open = vvcam_vi_procfs_open,
+    .release = seq_release,
+    .read = seq_read,
+    .write = vvcam_vi_procfs_write,
+    .llseek = seq_lseek,
+};
+#else
+static const struct proc_ops vvcam_vi_procfs_ops = {
+	.proc_open = vvcam_vi_procfs_open,
+	.proc_release = seq_release,
+	.proc_read = seq_read,
+	.proc_write = vvcam_vi_procfs_write,
+	.proc_lseek = seq_lseek,
+};
+#endif
+
+struct finddir_callback {
+    struct dir_context ctx;
+    const char *name;
+    bool found;
+};
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)
+static int readdir_callback(struct dir_context *ctx, const char *name,
+    int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return 0;
+
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    return 0;
+}
+
+#else
+static bool readdir_callback(struct dir_context *ctx, const char *name,
+        int namelen, loff_t offset, u64 ino, unsigned int d_type) {
+
+    struct finddir_callback *fc =
+        container_of(ctx, struct finddir_callback, ctx);
+    if (fc->found)
+        return true;
+    if(strcmp(name, fc->name) == 0) {
+        fc->found = true;
+    }
+    return true;
+}
+#endif
+
+
+static int find_proc_dir_by_name(const char *name, bool *found) {
+    struct file *pfile;
+    int ret = 0;
+    struct finddir_callback fc = {
+        .ctx.actor = readdir_callback,
+        .name = name,
+        .found = false
+    };
+
+    pfile = filp_open("/proc", O_RDONLY | O_DIRECTORY, 0);
+    if(IS_ERR(pfile)) {
+        pr_err("Failed to open %s\n", root);
+        return -1;
+    }
+    if (pfile->f_op->iterate_shared) {
+        ret = pfile->f_op->iterate_shared(pfile, &fc.ctx);
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 2, 0)
+    } else {
+        ret = pfile->f_op->iterate(pfile, &fc.ctx);
+#endif
+    }
+
+    if (ret == 0) {
+        *found = fc.found;
+    }
+
+    filp_close(pfile, NULL);
+    return ret;
+}
+
+int vvcam_vi_procfs_register(struct vvcam_vi_dev *vi_dev, unsigned long *pde)
+{
+    struct vvcam_vi_procfs *vi_proc;
+    char vi_proc_name[32];
+    int ret = 0;
+    bool found = false;
+
+    if (!vi_dev)
+        return -1;
+    sprintf(vi_proc_name, "vsi/vi");
+
+    vi_proc = devm_kzalloc(vi_dev->dev,
+                        sizeof(struct vvcam_vi_procfs), GFP_KERNEL);
+
+    if (!vi_proc)
+        return -ENOMEM;
+
+    ret = find_proc_dir_by_name("vsi", &found);
+    if (ret == 0) {
+        if (!found)
+            proc_mkdir("vsi", NULL);
+    } else {
+        return -EFAULT;
+    }
+
+    vi_proc->vi_dev = vi_dev;
+    vi_proc->pde = proc_create_data(vi_proc_name, 0444, NULL,
+                                &vvcam_vi_procfs_ops, vi_proc);
+    if (!vi_proc->pde)
+        return -EFAULT;
+    *pde = (unsigned long)&vi_proc->pde;
+
+    INIT_LIST_HEAD(&vi_proc->status.cmd_list);
+    mutex_init(&(vi_proc->lock));
+	return 0;
+}
+
+void vvcam_vi_procfs_unregister(unsigned long pde)
+{
+    int ret = 0;
+    bool found = false;
+    ret = find_proc_dir_by_name("vsi", &found);
+    if (ret == 0) {
+        if (found)
+            remove_proc_subtree("vsi", NULL);
+    }
+}
diff --git a/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_procfs.h b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_procfs.h
new file mode 100644
index 000000000000..d1112a4e7a2e
--- /dev/null
+++ b/drivers/media/platform/eswin/isp/vvcam/videoin/vvcam_vi_procfs.h
@@ -0,0 +1,63 @@
+/****************************************************************************
+*
+*    The MIT License (MIT)
+*
+*    Copyright (c) 2014 - 2024 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************
+*
+*    The GPL License (GPL)
+*
+*    Copyright (C) 2014 - 2024 Vivante Corporation
+*
+*    This program is free software; you can redistribute it and/or
+*    modify it under the terms of the GNU General Public License
+*    as published by the Free Software Foundation; either version 2
+*    of the License, or (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software Foundation,
+*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*****************************************************************************
+*
+*    Note: This software is released under dual MIT and GPL licenses. A
+*    recipient may use this file under the terms of either the MIT license or
+*    GPL License. If you wish to use only one license not the other, you can
+*    indicate your decision by deleting one of the above license notices in your
+*    version of this file.
+*
+*****************************************************************************/
+
+
+#ifndef __VVCAM_VI_PROCFS__
+#define __VVCAM_VI_PROCFS__
+
+int vvcam_vi_procfs_register(struct vvcam_vi_dev *vi_dev,
+                            unsigned long *pde);
+void vvcam_vi_procfs_unregister(unsigned long pde);
+
+#endif
diff --git a/drivers/media/platform/eswin/mipi-csi2/Kconfig b/drivers/media/platform/eswin/mipi-csi2/Kconfig
new file mode 100755
index 000000000000..20bbab487fb6
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/Kconfig
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+#  Synopsys DWC Platform drivers
+#	Drivers here are currently for MIPI CSI-2 and MIPI DPHY support
+
+config DWC_MIPI_CSI2_HOST
+	tristate "Synopsys DesignWare CSI-2 Host Controller support"
+	select VIDEO_DEV
+	select VIDEO_V4L2
+	select VIDEO_V4L2_SUBDEV_API
+	select V4L2_FWNODE
+    select VIDEOBUF2_CORE
+    select VIDEOBUF2_V4L2
+    select VIDEOBUF2_MEMOPS
+    select VIDEOBUF2_DMA_CONTIG
+	help
+	  This selects the DesignWare MIPI CSI-2 host controller support. This
+	  controller gives access to control a CSI-2 receiver acting as a V4L2
+	  subdevice.
+
+	  If you have a controller with this interface, say Y.
+
+	   If unsure, say N.
+
+config DWC_MIPI_DPHY_GEN3
+	default n
+	tristate "DesignWare platform support using a Gen3 D-PHY"
+	select GENERIC_PHY
+	help
+	 Synopsys MIPI D-PHY Generation 3 reference driver. This driver supports
+	 all Generation 3 D-PHYs. Choose Y or M if you have a platform with this
+	 block.
+
+	  If unsure, say N.
+
+if DWC_MIPI_DPHY_GEN3
+
+config DWC_MIPI_TC_DPHY_GEN3
+	tristate "Platform support using a Synopsys Test Chip"
+	default n
+	help
+	 Synopsys Test Chip is for prototyping purposes. This enables extra
+	 features that exist only in prototyping and/or for debug purposes.
+
+	  If unsure, say N.
+
+endif # DWC_MIPI_DPHY_GEN3
+
+config ESWIN_MIPI_DMA
+	tristate "A MIPI CSI2 DMA written controller"
+	default n
+	select DWC_MIPI_CSI2_HOST
+	help
+	 If unsure, say N.
diff --git a/drivers/media/platform/eswin/mipi-csi2/Makefile b/drivers/media/platform/eswin/mipi-csi2/Makefile
new file mode 100755
index 000000000000..60d981dc940b
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/Makefile
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for Synopsys DWC Platform drivers
+#
+# ifeq ($(CONFIG_DWC_MIPI_TC_DPHY_GEN3),y)#
+# 	dw-csi-objs += dw-csi-sysfs.o
+# endif
+obj-$(CONFIG_DWC_MIPI_CSI2_HOST) += dw-csi.o
+dw-csi-objs := dw-csi-plat.o dw-mipi-csi.o
+
+# dw-dphy-objs := dw-dphy-plat.o dw-dphy-rx.o
+# ifeq ($(CONFIG_DWC_MIPI_TC_DPHY_GEN3),y)
+# 	dw-dphy-objs += dw-dphy-sysfs.o
+# endif
+# obj-$(CONFIG_DWC_MIPI_DPHY_GEN3) += dw-dphy.o
+#obj-$(CONFIG_ESWIN_MIPI_DMA) += eswin-mipi-dma.o
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-csi-plat.c b/drivers/media/platform/eswin/mipi-csi2/dw-csi-plat.c
new file mode 100755
index 000000000000..973ee3ef97bf
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-csi-plat.c
@@ -0,0 +1,855 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver.
+ * Platform driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include <media/eswin/dw-csi-data.h>
+#include <media/eswin/dw-dphy-data.h>
+
+#include "dw-csi-plat.h"
+
+
+void __iomem *csc;
+void __iomem *demo;
+
+static inline struct dw_csi *sd_to_dwc_mipi_csi2h(
+						struct v4l2_subdev *sdev)
+{
+	return container_of(sdev, struct dw_csi, sd);
+}
+
+static inline struct dw_csi *sd_to_dev(struct v4l2_subdev *sdev)
+{
+	return container_of(sdev, struct dw_csi, sd);
+}
+
+static struct csi2_sensor_info *sd_to_sensor(struct dw_csi *csi2,
+					     struct v4l2_subdev *sd)
+{
+	int i;
+
+	for (i = 0; i < csi2->num_sensors; ++i)
+		if (csi2->sensors[i].sd == sd)
+			return &csi2->sensors[i];
+
+	return NULL;
+}
+
+static struct mipi_fmt *
+find_dw_mipi_csi_format(struct v4l2_mbus_framefmt *mf)
+{
+	unsigned int i;
+
+	pr_info("%s entered mbus: 0x%x\n", __func__, mf->code);
+
+	for (i = 0; i < ARRAY_SIZE(dw_mipi_csi_formats); i++)
+		if (mf->code == dw_mipi_csi_formats[i].mbus_code) {
+			pr_info("Found mbus 0x%x\n", dw_mipi_csi_formats[i].mbus_code);
+			return &dw_mipi_csi_formats[i];
+		}
+	return NULL;
+}
+
+static int dw_mipi_csi_enum_mbus_code(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_state *cfg,
+				      struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index != 0)
+		return -EINVAL;
+
+	code->code = dw_mipi_csi_formats[code->index].mbus_code;
+	return 0;
+}
+
+static struct mipi_fmt *
+dw_mipi_csi_try_format(struct v4l2_mbus_framefmt *mf)
+{
+	struct mipi_fmt *fmt;
+
+	fmt = find_dw_mipi_csi_format(mf);
+	if (!fmt)
+		fmt = &dw_mipi_csi_formats[0];
+
+	mf->code = fmt->mbus_code;
+
+	return fmt;
+}
+
+static struct v4l2_subdev *get_remote_sensor(struct v4l2_subdev *sd)
+{
+	struct media_pad *local, *remote;
+	struct media_entity *sensor_me;
+
+	local = &sd->entity.pads[DWC_CSI2_PAD_SINK];
+	remote = media_pad_remote_pad_first(local);
+	if (!remote) {
+		v4l2_warn(sd, "No link between dphy and sensor\n");
+		return NULL;
+	}
+
+	sensor_me = media_pad_remote_pad_first(local)->entity;
+	return media_entity_to_v4l2_subdev(sensor_me);
+}
+
+static struct v4l2_mbus_framefmt *
+dw_mipi_csi_get_format(struct dw_csi *dev, struct v4l2_subdev_state *cfg,
+		       enum v4l2_subdev_format_whence which)
+{
+	pr_info("%s entered\n", __func__);
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return cfg ? v4l2_subdev_get_try_format(&dev->sd, cfg, 0) : NULL;
+	dev_info(dev->dev,
+		"%s got v4l2_mbus_pixelcode. 0x%x\n", __func__,
+		dev->format.code);
+	dev_info(dev->dev,
+		"%s got width. 0x%x\n", __func__,
+		dev->format.width);
+	dev_info(dev->dev,
+		"%s got height. 0x%x\n", __func__,
+		dev->format.height);
+	return &dev->format;
+}
+
+static int
+dw_mipi_csi_set_fmt(struct v4l2_subdev *sd,
+		    struct v4l2_subdev_state *cfg,
+		    struct v4l2_subdev_format *fmt)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+	struct mipi_fmt *dev_fmt = NULL;
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+	pr_info("%s entered\n", __func__);
+
+	dev_info(dev->dev,
+		"%s got mbus_pixelcode. 0x%x\n", __func__,
+		fmt->format.code);
+
+	dev_fmt = dw_mipi_csi_try_format(&fmt->format);
+	dev_info(dev->dev,
+		"%s got v4l2_mbus_pixelcode. 0x%x\n", __func__,
+		dev_fmt->mbus_code);
+	if (!fmt)
+		return -EINVAL;
+
+	if (dev_fmt) {
+		if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE)
+			dev->fmt = dev_fmt;
+		dev->fmt->mbus_code = mf->code;
+		dw_mipi_csi_set_ipi_fmt(dev);
+	}
+
+	return 0;
+}
+
+static int
+dw_mipi_csi_get_fmt(struct v4l2_subdev *sd,
+		    struct v4l2_subdev_state *cfg,
+		    struct v4l2_subdev_format *fmt)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+	struct v4l2_mbus_framefmt *mf;
+
+	mf = dw_mipi_csi_get_format(dev, cfg, fmt->which);
+	if (!mf)
+		return -EINVAL;
+
+	mutex_lock(&dev->lock);
+	fmt->format = *mf;
+	mutex_unlock(&dev->lock);
+
+	return 0;
+}
+
+static int dwc_mipi_csi2_g_mbus_config(struct v4l2_subdev *sd, unsigned int pad_id,
+			      struct v4l2_mbus_config *mbus)
+{
+	struct dw_csi *csi2 = sd_to_dwc_mipi_csi2h(sd);
+	struct v4l2_subdev *sensor_sd = get_remote_sensor(sd);
+	int ret;
+
+	ret = v4l2_subdev_call(sensor_sd, pad, get_mbus_config, 0, mbus);
+	if (ret) {
+		mbus->type = V4L2_MBUS_CSI2_DPHY;
+		mbus->bus.mipi_csi2.flags = csi2->bus.flags;
+		mbus->bus.mipi_csi2.flags |= BIT(csi2->bus.num_data_lanes - 1);
+	}
+
+	return 0;
+}
+
+static struct v4l2_rect *
+mipi_csi2_get_crop(struct dw_csi *csi2h, struct v4l2_subdev_state *sd_state,
+		   enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_crop(&csi2h->sd, sd_state,
+						DWC_CSI2_PAD_SINK);
+	else
+		return &csi2h->crop;
+}
+static int dwc_mipi_csi2_get_selection(struct v4l2_subdev *sd,
+			      struct v4l2_subdev_state *sd_state,
+			      struct v4l2_subdev_selection *sel)
+{
+	struct dw_csi *csi2h = sd_to_dwc_mipi_csi2h(sd);
+	struct v4l2_subdev *sensor = get_remote_sensor(sd);
+	struct v4l2_subdev_format fmt;
+	int ret = 0;
+
+	if (!sel) {
+		pr_err("sel is null\n");
+		goto err;
+	}
+
+	if (sel->pad > DWC_CSI2X_PAD_SOURCE3) {
+		pr_err("pad[%d] isn't matched\n",
+			 sel->pad);
+		goto err;
+	}
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+			sel->pad = 0;
+			ret = v4l2_subdev_call(sensor, pad, get_selection,
+					       sd_state, sel);
+			if (ret) {
+				fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+				fmt.pad = 0;
+				ret = v4l2_subdev_call(sensor, pad, get_fmt,
+						       NULL, &fmt);
+				if (!ret) {
+					csi2h->format_mbus = fmt.format;
+					sel->r.top = 0;
+					sel->r.left = 0;
+					sel->r.width = csi2h->format_mbus.width;
+					sel->r.height =
+						csi2h->format_mbus.height;
+					csi2h->crop = sel->r;
+				} else {
+					sel->r = csi2h->crop;
+				}
+			} else {
+				csi2h->crop = sel->r;
+			}
+		} else {
+			sel->r = *v4l2_subdev_get_try_crop(&csi2h->sd, sd_state,
+							   sel->pad);
+		}
+		break;
+
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *mipi_csi2_get_crop(csi2h, sd_state, sel->which);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+err:
+	return -EINVAL;
+}
+
+static int dwc_mipi_csi2_set_selection(struct v4l2_subdev *sd,
+			      struct v4l2_subdev_state *sd_state,
+			      struct v4l2_subdev_selection *sel)
+{
+	struct dw_csi *csi2h = sd_to_dwc_mipi_csi2h(sd);
+	struct v4l2_subdev *sensor = get_remote_sensor(sd);
+	int ret = 0;
+
+	ret = v4l2_subdev_call(sensor, pad, set_selection, sd_state, sel);
+	if (!ret)
+		csi2h->crop = sel->r;
+
+	return ret;
+}
+
+static int
+dw_mipi_csi_log_status(struct v4l2_subdev *sd)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+
+	dw_mipi_csi_dump(dev);
+
+	return 0;
+}
+
+#if 0
+static void dw_mipi_csi_start_phy(struct dw_csi *csi_dev)
+{
+	int ret = 0;
+	pr_info("%s:%d sensor name %s\n", __func__, __LINE__, csi_dev->sensors[0].sd->name);
+	ret = v4l2_subdev_call(csi_dev->sensors[0].sd, video, s_stream, 1);
+	if (ret) {
+		dev_err(csi_dev->dev, "Failed to start dphy: %d\n", ret);
+	}
+}
+#endif
+
+static int
+dw_mipi_csi_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+
+	dev_info(dev->dev, "%s: on=%d\n", __func__, on);
+
+	if (on) {
+		dw_mipi_csi_hw_stdby(dev);
+		dw_mipi_csi_start(dev);
+		dw_mipi_csi_log_status(sd);
+	} else {
+#ifdef DWC_PHY_USING
+		phy_power_off(dev->phy);
+#endif
+		dw_mipi_csi_mask_irq_power_off(dev);
+	}
+	return 0;
+}
+
+
+
+#if IS_ENABLED(CONFIG_VIDEO_ADV_DEBUG)
+static int
+dw_mipi_csi_g_register(struct v4l2_subdev *sd, struct v4l2_dbg_register *reg)
+{
+	struct dw_csi *dev = sd_to_mipi_csi_dev(sd);
+
+	dev_vdbg(dev->dev, "%s: reg=%llu\n", __func__, reg->reg);
+	reg->val = dw_mipi_csi_read(dev, reg->reg);
+
+	return 0;
+}
+#endif
+static int dw_mipi_csi_init_cfg(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *cfg)
+{
+	struct v4l2_mbus_framefmt *format =
+	    v4l2_subdev_get_try_format(sd, cfg, 0);
+
+	format->colorspace = V4L2_COLORSPACE_SRGB;
+	format->code = MEDIA_BUS_FMT_RGB888_1X24;
+	format->field = V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static int dw_mipi_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	return dw_mipi_csi_s_power(sd, enable);
+}
+
+#define DW_MIPI_STREAM_CMD _IOWR('u', 0x20, int)
+
+long dw_mipi_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
+{
+	switch (cmd) {
+	case DW_MIPI_STREAM_CMD: {
+		int *enable = arg;
+		dw_mipi_csi_s_power(sd, *enable);
+		break;
+	}
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct v4l2_subdev_core_ops dw_mipi_csi_core_ops = {
+	.s_power = dw_mipi_csi_s_power,
+	.log_status = dw_mipi_csi_log_status,
+	.ioctl = dw_mipi_ioctl,
+#if IS_ENABLED(CONFIG_VIDEO_ADV_DEBUG)
+	.g_register = dw_mipi_csi_g_register,
+#endif
+};
+
+static struct v4l2_subdev_pad_ops dw_mipi_csi_pad_ops = {
+	.init_cfg = dw_mipi_csi_init_cfg,
+	.enum_mbus_code = dw_mipi_csi_enum_mbus_code,
+	.get_fmt = dw_mipi_csi_get_fmt,
+	.set_fmt = dw_mipi_csi_set_fmt,
+	.get_mbus_config 	 = dwc_mipi_csi2_g_mbus_config,
+	.get_selection 		 = dwc_mipi_csi2_get_selection,
+	.set_selection 		 = dwc_mipi_csi2_set_selection,
+};
+
+static const struct v4l2_subdev_video_ops dw_mipi_video_ops = {
+	.s_stream = dw_mipi_set_stream,
+};
+
+static struct v4l2_subdev_ops dw_mipi_csi_subdev_ops = {
+	.core = &dw_mipi_csi_core_ops,
+	.pad = &dw_mipi_csi_pad_ops,
+	.video = &dw_mipi_video_ops,
+};
+
+static irqreturn_t dw_mipi_csi_irq1(int irq, void *dev_id)
+{
+	struct dw_csi *csi_dev = dev_id;
+
+	dw_mipi_csi_irq_handler(csi_dev);
+
+	return IRQ_HANDLED;
+}
+
+static int
+dw_mipi_csi_parse_dt(struct platform_device *pdev, struct dw_csi *dev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	// struct v4l2_fwnode_endpoint ep = { .bus_type = V4L2_MBUS_CSI2_DPHY };
+	int ret = 0;
+
+	// struct fwnode_handle *ep = NULL;
+	// struct v4l2_async_connection *s_asd = NULL;
+	// struct fwnode_handle *remote_ep = NULL;
+	// struct v4l2_fwnode_endpoint vep = { .bus_type = V4L2_MBUS_CSI2_DPHY };
+
+	if (of_property_read_u32(node, "snps,output-type", &dev->hw.output))
+		dev->hw.output = 2;
+
+	if (of_property_read_u32(node, "snps,en-ppi-width", &dev->hw.ppi_width))
+		dev->hw.ppi_width = 0;
+
+	if (dev->hw.ppi_width > 1) {
+		dev_err(&pdev->dev, "Wrong ppi width(%d), valid value is '0'(ppi8) or '1'(ppi16)\n",
+				dev->hw.ppi_width);
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32(node, "snps,en-phy-mode", &dev->hw.phy_mode))
+		dev->hw.phy_mode = 0;
+
+	if (of_property_read_u32(node, "ipi2_en", &dev->hw.ipi2_en))
+		dev->hw.ipi2_en = 0;
+
+	if (of_property_read_u32(node, "ipi2_vcid", &dev->hw.ipi2_vcid))
+		dev->hw.ipi2_vcid = 0;
+
+	if (of_property_read_u32(node, "ipi3_en", &dev->hw.ipi3_en))
+		dev->hw.ipi3_en = 0;
+
+	if (of_property_read_u32(node, "ipi3_vcid", &dev->hw.ipi3_vcid))
+		dev->hw.ipi3_vcid = 0;
+
+	if (of_property_read_u32(node, "num_lanes", &dev->hw.num_lanes))
+		dev->hw.num_lanes = 2;
+
+	if (of_property_read_u32(node, "index", &dev->index))
+		dev->index = 0;
+
+	node = of_graph_get_next_endpoint(node, NULL);
+	if (!node) {
+		dev_err(&pdev->dev, "No port node at %pOF\n",
+			pdev->dev.of_node);
+		return -EINVAL;
+	}
+
+	/* Get port node and validate MIPI-CSI channel id. */
+	// ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(node), &ep);
+	// pr_info("%s:%d (node)->name %s \n", __func__, __LINE__, node->name);
+
+
+	// if (ret)
+	// 	goto err;
+	// dev->index = vep.base.port - 1;
+	// // dev->index = ep.base.port - 1;
+	// pr_info("%s:%d dev->index %d \n", __func__, __LINE__, dev->index);
+	// if (dev->index >= CSI_MAX_ENTITIES) {
+	// 	ret = -ENXIO;
+	// 	goto err;
+	// }
+	dev->hw.num_lanes = 2;//vep.bus.mipi_csi2.num_data_lanes;
+
+// err:
+// 	pr_info("%s:%d \n", __func__, __LINE__);
+
+
+	of_node_put(node);
+	return ret;
+}
+
+static const struct of_device_id dw_mipi_csi_of_match[];
+
+static int csi2_notifier_bound(struct v4l2_async_notifier *notifier,
+			       struct v4l2_subdev *sd,
+			       struct v4l2_async_connection *asd)
+{
+	struct dw_csi *csi2 =
+		container_of(notifier, struct dw_csi, notifier);
+	struct csi2_sensor_info *sensor;
+	struct media_link *link;
+	unsigned int pad, ret;
+
+	if (csi2->num_sensors == ARRAY_SIZE(csi2->sensors)) {
+		v4l2_err(&csi2->sd, "%s: the num of sd is beyond:%d\n",
+			 __func__, csi2->num_sensors);
+		return -EBUSY;
+	}
+	pr_debug("%s:%d csi2->num_sensors %d\n", __func__, __LINE__, csi2->num_sensors);
+	sensor = &csi2->sensors[csi2->num_sensors++];
+	sensor->sd = sd;
+
+	for (pad = 0; pad < sd->entity.num_pads; pad++)
+		if (sensor->sd->entity.pads[pad].flags & MEDIA_PAD_FL_SOURCE)
+			break;
+	pr_debug("%s:%d sensor->sd->name %s\n", __func__, __LINE__, sensor->sd->name);
+	pr_debug("%s:%d sensor->sd->entity->name %s\n", __func__, __LINE__, sensor->sd->entity.name);
+	
+	if (pad == sensor->sd->entity.num_pads) {
+		dev_err(csi2->dev, "failed to find src pad for %s\n", sd->name);
+
+		return -ENXIO;
+	}
+
+	ret = media_create_pad_link(
+		&sensor->sd->entity, pad, &csi2->sd.entity, DWC_CSI2_PAD_SINK,
+		0 /* csi2->num_sensors != 1 ? 0 : MEDIA_LNK_FL_ENABLED */);
+	if (ret) {
+		dev_err(csi2->dev, "failed to create link for %s\n", sd->name);
+		return ret;
+	}
+
+	link = list_first_entry(&csi2->sd.entity.links, struct media_link,
+				list);
+	ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);
+	if (ret) {
+		dev_err(csi2->dev, "failed to create link for %s\n",
+			sensor->sd->name);
+		return ret;
+	}
+
+	return 0;
+}
+
+/* The .unbind callback */
+static void csi2_notifier_unbind(struct v4l2_async_notifier *notifier,
+				 struct v4l2_subdev *sd,
+				 struct v4l2_async_connection *asd)
+{
+	struct dw_csi *csi2 =
+		container_of(notifier, struct dw_csi, notifier);
+	struct csi2_sensor_info *sensor = sd_to_sensor(csi2, sd);
+
+	if (sensor)
+		sensor->sd = NULL;
+	pr_info("t1 %s, %d succsess\n", __func__, __LINE__);
+}
+
+static const struct v4l2_async_notifier_operations csi2_async_ops = {
+	.bound = csi2_notifier_bound,
+	.unbind = csi2_notifier_unbind,
+};
+
+// /* Parse fwnode with port0, if an empty function is used, each node will parse
+//  * all ports, causing the device to repeatedly join the link and unable to
+//  * complete the link
+//  */
+static int csi2_fwnode_parse(struct dw_csi *csi2)
+{
+	struct device *dev = csi2->dev;
+	struct fwnode_handle *ep = NULL;
+	struct v4l2_async_connection *s_asd = NULL;
+	struct fwnode_handle *remote_ep = NULL;
+	struct v4l2_fwnode_endpoint vep = { .bus_type = V4L2_MBUS_CSI2_DPHY };
+	int ret = 0;
+
+	fwnode_graph_for_each_endpoint(dev_fwnode(dev), ep) {
+		ret = v4l2_fwnode_endpoint_parse(ep, &vep);
+		if (ret)
+			goto err_parse;
+
+		/* only add fwnode form port 0 to notifier list */
+		if (vep.base.port != 0)
+			continue;
+
+		remote_ep = fwnode_graph_get_remote_port_parent(ep);
+
+		/* skip device dts status is disabled */
+		if (!fwnode_device_is_available(remote_ep)) {
+			fwnode_handle_put(remote_ep);
+			continue;
+		}
+
+		fwnode_handle_put(remote_ep);
+
+		v4l2_async_subdev_nf_init(&csi2->notifier, &csi2->sd);
+
+		s_asd = v4l2_async_nf_add_fwnode_remote(
+			&csi2->notifier, ep, struct v4l2_async_connection);
+		if (IS_ERR(s_asd)) {
+			ret = PTR_ERR(s_asd);
+			goto err_parse;
+		}
+	}
+	return 0;
+
+err_parse:
+	fwnode_handle_put(ep);
+	return ret;
+}
+
+static int csi2_notifier(struct dw_csi *csi2)
+{
+	// struct v4l2_async_notifier *ntf = &csi2->notifier;
+	int ret;
+
+	ret = csi2_fwnode_parse(csi2);
+	if (ret < 0)
+		return ret;
+	csi2->sd.subdev_notifier = &csi2->notifier;
+	csi2->notifier.ops = &csi2_async_ops;
+	ret = v4l2_async_nf_register(&csi2->notifier);
+	if (ret) {
+		dev_err(csi2->dev, "fail to register async notifier: %d\n",
+			ret);
+		v4l2_async_nf_cleanup(&csi2->notifier);
+	}
+	ret = v4l2_async_register_subdev(&csi2->sd);
+	return ret;
+}
+
+
+static int dw_csi_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *of_id = NULL;
+	struct dw_csih_pdata *pdata = NULL;
+	struct device *dev = &pdev->dev;
+	struct dw_csi *csi;
+	struct v4l2_subdev *sd;
+	struct device *parent = pdev->dev.parent;
+	struct eswin_vi_device* es_vi_dev;
+
+	int ret;
+	
+	es_vi_dev = dev_get_drvdata(parent);
+
+	dev_info(dev, "DW MIPI CSI-2 Host Probe in. \n");
+
+	if (!IS_ENABLED(CONFIG_OF))
+		pdata = pdev->dev.platform_data;
+
+	dev_vdbg(dev, "Probing started\n");
+
+	/* Resource allocation */
+	csi = devm_kzalloc(dev, sizeof(*csi), GFP_KERNEL);
+	if (!csi)
+		return -ENOMEM;
+
+	mutex_init(&csi->lock);
+	spin_lock_init(&csi->slock);
+	csi->dev = dev;
+
+	if (dev->of_node) {
+		of_id = of_match_node(dw_mipi_csi_of_match, dev->of_node);
+		if (!of_id)
+			return -EINVAL;
+
+		ret = dw_mipi_csi_parse_dt(pdev, csi);
+		if (ret < 0)
+			return ret;
+
+#ifdef DWC_PHY_USING
+		csi->phy = devm_of_phy_get(dev, dev->of_node, NULL);
+		if (IS_ERR(csi->phy)) {
+			dev_err(dev, "No DPHY available\n");
+			return PTR_ERR(csi->phy);
+		}
+#endif
+	} else {
+#ifdef DWC_PHY_USING
+		csi->phy = devm_phy_get(dev, phys[pdata->id].name);
+		if (IS_ERR(csi->phy)) {
+			dev_err(dev, "No '%s' DPHY available\n",
+				phys[pdata->id].name);
+			return PTR_ERR(csi->phy);
+		}
+		dev_info(dev, "got D-PHY %s with id %d\n", phys[pdata->id].name,
+			 csi->phy->id);
+#endif
+	}
+
+
+	/* Registers mapping */
+	csi->base_address = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(csi->base_address)) {
+		dev_err(dev, "base address not set.\n");
+		return PTR_ERR(csi->base_address);
+	}
+
+	csi->ctrl_irq_number = platform_get_irq(pdev, 0);
+	if (csi->ctrl_irq_number < 0) {
+		dev_err(dev, "irq number %d not set.\n", csi->ctrl_irq_number);
+		ret = csi->ctrl_irq_number;
+		goto end;
+	}
+
+	csi->rst = devm_reset_control_get_optional_shared(dev, NULL);
+	if (IS_ERR(csi->rst)) {
+		dev_err(dev, "error getting reset control %d\n", ret);
+		return PTR_ERR(csi->rst);
+	}
+
+	ret = devm_request_irq(dev, csi->ctrl_irq_number, dw_mipi_csi_irq1,
+			       IRQF_SHARED, dev_name(dev), csi);
+	if (ret) {
+		if (dev->of_node)
+			dev_err(dev, "irq csi %s failed\n", of_id->name);
+		else
+			dev_err(dev, "irq csi %d failed\n", pdata->id);
+
+		goto end;
+	}
+
+	csi->v4l2_dev.mdev = es_vi_dev->media_dev;
+	ret = v4l2_device_register(dev, &csi->v4l2_dev);
+
+	sd = &csi->sd;
+	v4l2_subdev_init(sd, &dw_mipi_csi_subdev_ops);
+	sd->dev = dev;
+	csi->sd.owner = THIS_MODULE;
+
+	if (dev->of_node) {
+		snprintf(sd->name, sizeof(sd->name), "%s.%d",
+			 "dw-csi", csi->index);
+	} else {
+		strlcpy(sd->name, dev_name(dev), sizeof(sd->name));
+	}
+	csi->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
+	csi->fmt = &dw_mipi_csi_formats[0];
+	csi->format.code = dw_mipi_csi_formats[0].mbus_code;
+
+	sd->entity.function = MEDIA_ENT_F_IO_V4L;
+
+	if (dev->of_node) {
+		csi->pads[CSI_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+		csi->pads[CSI_PAD_SOURCE0].flags = MEDIA_PAD_FL_SOURCE;
+		csi->pads[CSI_PAD_SOURCE1].flags = MEDIA_PAD_FL_SOURCE;
+
+		ret = media_entity_pads_init(&csi->sd.entity, CSI_PADS_NUM, csi->pads);
+		if (ret < 0) {
+			dev_err(dev, "media entity init failed\n");
+			goto end;
+		}
+	} else {
+		csi->hw.num_lanes = pdata->lanes;
+		csi->hw.pclk = pdata->pclk;
+		csi->hw.fps = pdata->fps;
+		csi->hw.dphy_freq = pdata->hs_freq;
+	}
+
+	v4l2_set_subdevdata(&csi->sd, pdev);
+	platform_set_drvdata(pdev, &csi->sd);
+	dev_set_drvdata(dev, sd);
+
+	if (csi->rst)
+		reset_control_deassert(csi->rst);
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_csi_create_capabilities_sysfs(pdev);
+#endif
+
+	dw_mipi_csi_get_version(csi);
+	dw_mipi_csi_specific_mappings(csi);
+	dw_mipi_csi_mask_irq_power_off(csi);
+	dw_mipi_csi_fill_timings(csi);
+
+	dev_info(dev, "DW MIPI CSI-2 Host registered successfully HW v%u.%u\n",
+		 csi->hw_version_major, csi->hw_version_minor);
+
+#ifdef DWC_PHY_USING
+	ret = phy_init(csi->phy);
+	if (ret) {
+		dev_err(&csi->phy->dev, "phy init failed --> %d\n", ret);
+		goto end;
+	}
+#endif
+
+	ret = csi2_notifier(csi);
+
+	// ret = v4l2_async_register_subdev(sd);
+	if (ret < 0) {
+		dev_err(dev, "failed to register subdev\n");
+		goto end;
+	}
+
+	return 0;
+
+end:
+#if IS_ENABLED(CONFIG_OF)
+	media_entity_cleanup(&csi->sd.entity);
+	return ret;
+#endif
+	//v4l2_device_unregister(csi->vdev.v4l2_dev);
+	return ret;
+}
+
+static int dw_csi_remove(struct platform_device *pdev)
+{
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *mipi_csi = sd_to_mipi_csi_dev(sd);
+
+	if (mipi_csi->rst)
+		reset_control_assert(mipi_csi->rst);
+
+    v4l2_async_unregister_subdev(sd);
+
+#if IS_ENABLED(CONFIG_OF)
+	media_entity_cleanup(&mipi_csi->sd.entity);
+#else
+	v4l2_device_unregister(mipi_csi->vdev.v4l2_dev);
+#endif
+	dev_info(&pdev->dev, "DW MIPI CSI-2 Host module removed\n");
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id dw_mipi_csi_of_match[] = {
+	{ .compatible = "eswin,eic770x-mipi-csi2" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dw_mipi_csi_of_match);
+#endif
+
+static struct platform_driver dw_mipi_csi_driver = {
+	.remove = dw_csi_remove,
+	.probe = dw_csi_probe,
+	.driver = {
+		.name = "dw-csi",
+		.owner = THIS_MODULE,
+#if IS_ENABLED(CONFIG_OF)
+		.of_match_table = of_match_ptr(dw_mipi_csi_of_match),
+#endif
+	},
+};
+
+int dw_mipi_csi_driver_init(void)
+{
+	
+	return platform_driver_register(&dw_mipi_csi_driver);
+
+
+}
+
+static void __exit dw_mipi_csi_driver_exit(void)
+{
+	platform_driver_unregister(&dw_mipi_csi_driver);
+
+}
+
+
+late_initcall(dw_mipi_csi_driver_init);
+module_exit(dw_mipi_csi_driver_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Luis Oliveira <luis.oliveira@synopsys.com>");
+MODULE_DESCRIPTION("Synopsys DesignWare MIPI CSI-2 Host Platform driver");
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-csi-plat.h b/drivers/media/platform/eswin/mipi-csi2/dw-csi-plat.h
new file mode 100755
index 000000000000..257cbfbd044b
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-csi-plat.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Synopsys, Inc.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver.
+ * Supported bus formats
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#ifndef _DW_CSI_PLAT_H__
+#define _DW_CSI_PLAT_H__
+
+#include <media/eswin/eswin_vi.h>
+#include "dw-mipi-csi.h"
+
+/** Color Space Converter Block **/
+#define CSC_UNIT_NR 1
+#define CSC_COREID 0x0000
+#define CSC_CFG 0x0004
+#define CSC_COEF_A(x) (0x0008 + ((x) * 4))
+#define CSC_COEF_B(x) (0x0018 + ((x) * 4))
+#define CSC_COEF_C(x) (0x0028 + ((x) * 4))
+#define CSC_LIMIT_DN 0x0038
+#define CSC_LIMIT_UP 0x003C
+#define CSC_VID_CFG 0x0040
+
+/** Demosaic Block **/
+#define DEMO_CONTROL 0x0
+#define DEMO_GLOBAL_INT_EN 0x4
+#define DEMO_IP_INT_EN_REG 0x8
+#define DEMO_IP_INT_SATUS_REG 0xC
+#define DEMO_ACTIVE_WIDTH 0x10
+#define DEMO_ACTIVE_HEIGHT 0x18
+#define DEMO_BAYER_PHASE 0x28//0x20
+
+#define BAYER_RGGB 0x0
+#define BAYER_GRBG 0x1
+#define BAYER_GBRG 0x2
+#define BAYER_BGGR 0x3
+
+/* Video formats supported by the MIPI CSI-2 */
+struct mipi_fmt dw_mipi_csi_formats[] = {
+	{
+		/* RAW 8 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
+		.depth = 8,
+#if 0
+	}, {
+		/* RAW 6 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR6_1X8,
+		.depth = 8,
+	}, {
+		/* RAW 7 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR7_1X8,
+		.depth = 8,
+#endif
+	}, {
+		/* RAW 10 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10,
+		.depth = 10,
+	}, {
+		/* RAW 8 */
+		.mbus_code = 0x3001,
+		.depth = 8,
+	}, {
+		/* RAW 12 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12,
+		.depth = 12,
+	}, {
+		/* RAW 14 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR14_1X14,
+		.depth = 14,
+	}, {
+		/* RAW 16 */
+		.mbus_code = MEDIA_BUS_FMT_SBGGR16_1X16,
+		.depth = 16,
+	}, {
+		/* RGB 666 */
+		.mbus_code = MEDIA_BUS_FMT_RGB666_1X18,
+		.depth = 18,
+	}, {
+		/* RGB 565 */
+		.mbus_code = MEDIA_BUS_FMT_RGB565_2X8_BE,
+		.depth = 16,
+	}, {
+		/* BGR 565 */
+		.mbus_code = MEDIA_BUS_FMT_RGB565_2X8_LE,
+		.depth = 16,
+	}, {
+		/* RGB 555 */
+		.mbus_code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE,
+		.depth = 16,
+	}, {
+		/* BGR 555 */
+		.mbus_code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
+		.depth = 16,
+	}, {
+		/* RGB 444 */
+		.mbus_code = MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE,
+		.depth = 16,
+	}, {
+		/* RGB 444 */
+		.mbus_code = MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE,
+		.depth = 16,
+	}, {
+		/* RGB 888 */
+		.mbus_code = MEDIA_BUS_FMT_RGB888_2X12_LE,
+		.depth = 24,
+	}, {
+		/* BGR 888 */
+		.mbus_code = MEDIA_BUS_FMT_RGB888_2X12_BE,
+		.depth = 24,
+	}, {
+		/* BGR 888 */
+		.mbus_code = MEDIA_BUS_FMT_RGB888_1X24,
+		.depth = 24,
+	}, {
+		/* YUV 422 8-bit */
+		.mbus_code = MEDIA_BUS_FMT_VYUY8_1X16,
+		.depth = 16,
+	}, {
+		/* YUV 422 10-bit */
+		.mbus_code = MEDIA_BUS_FMT_UYVY10_1X20,
+		.depth = 24,
+	}, {
+		/* YUV 420 8-bit LEGACY */
+		.mbus_code = MEDIA_BUS_FMT_Y8_1X8,
+		.depth = 8,
+	}, {
+		/* YUV 420 8-bit LEGACY */
+		.mbus_code = MEDIA_BUS_FMT_YUYV8_1X16,
+		.depth = 24,
+	}, {
+		/* YUV 420 10-bit */
+		.mbus_code = MEDIA_BUS_FMT_VUY8_1X24,
+		.depth = 24,
+	}, {
+		/* YUV 420 8-bit */
+		.mbus_code = MEDIA_BUS_FMT_UYVY8_1X16,
+		.depth = 24,
+	}, {
+		/* YUV 420 10-bit */
+		.mbus_code = MEDIA_BUS_FMT_Y10_1X10,
+		.depth = 10,
+	},
+};
+#endif /* _DW_CSI_PLAT_H__ */
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-csi-sysfs.c b/drivers/media/platform/eswin/mipi-csi2/dw-csi-sysfs.c
new file mode 100755
index 000000000000..e8d2bb9d0eab
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-csi-sysfs.c
@@ -0,0 +1,624 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver.
+ * SysFS components for the platform driver
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include "dw-mipi-csi.h"
+
+static ssize_t core_version_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "v.%d.%d*\n", csi_dev->hw_version_major,
+		 csi_dev->hw_version_minor);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t n_lanes_store(struct device *dev, struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	int ret;
+	unsigned long lanes;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &lanes);
+	if (ret < 0)
+		return ret;
+
+	if (lanes > 8) {
+		dev_err(dev, "Invalid number of lanes %lu\n", lanes);
+		return count;
+	}
+
+	dev_info(dev, "Lanes %lu\n", lanes);
+	csi_dev->hw.num_lanes = lanes;
+
+	return count;
+}
+
+static ssize_t n_lanes_show(struct device *dev,
+			    struct device_attribute *attr,
+			    char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.num_lanes);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t core_reset_show(struct device *dev,
+			       struct device_attribute *attr,
+			       char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	/* Reset Controller and DPHY */
+	phy_reset(csi_dev->phy);
+	dw_mipi_csi_reset(csi_dev);
+
+	snprintf(buffer, 10, "Reset\n");
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t data_type_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	int ret;
+	unsigned long dt;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &dt);
+	if (ret < 0)
+		return ret;
+
+	if (dt < 0x18 || dt > 0x2F) {
+		dev_err(dev, "Invalid data type %lx\n", dt);
+		return count;
+	}
+
+	dev_info(dev, "Data type 0x%lx\n", dt);
+	csi_dev->ipi_dt = dt;
+
+	return count;
+}
+
+static ssize_t data_type_show(struct device *dev,
+			      struct device_attribute *attr,
+			      char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->ipi_dt);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t hsa_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long hsa;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &hsa);
+	if (ret < 0)
+		return ret;
+
+	if (hsa > 0xFFF) {
+		dev_err(dev, "Invalid HSA time %lx\n", hsa);
+		return count;
+	}
+
+	dev_info(dev, "HSA time 0x%lx\n", hsa);
+	csi_dev->hw.hsa = hsa;
+
+	return count;
+}
+
+static ssize_t hsa_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.hsa);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t hbp_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long hbp;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &hbp);
+	if (ret < 0)
+		return ret;
+
+	if (hbp > 0xFFF) {
+		dev_err(dev, "Invalid HBP time %lx\n", hbp);
+		return count;
+	}
+
+	dev_info(dev, "HBP time 0x%lx\n", hbp);
+	csi_dev->hw.hbp = hbp;
+
+	return count;
+}
+
+static ssize_t hbp_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.hbp);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t hsd_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long hsd;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &hsd);
+	if (ret < 0)
+		return ret;
+
+	if (hsd > 0xFF) {
+		dev_err(dev, "Invalid HSD time %lx\n", hsd);
+		return count;
+	}
+
+	dev_info(dev, "HSD time 0x%lx\n", hsd);
+	csi_dev->hw.hsd = hsd;
+
+	return count;
+}
+
+static ssize_t hsd_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.hsd);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t vsa_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long vsa;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &vsa);
+	if (ret < 0)
+		return ret;
+
+	if (vsa > 0x3FF) {
+		dev_err(dev, "Invalid VSA period %lx\n", vsa);
+		return count;
+	}
+
+	dev_info(dev, "VSA period 0x%lx\n", vsa);
+	csi_dev->hw.vsa = vsa;
+
+	return count;
+}
+
+static ssize_t vsa_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.vsa);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t vbp_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long vbp;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &vbp);
+	if (ret < 0)
+		return ret;
+
+	if (vbp > 0x2FF) {
+		dev_err(dev, "Invalid VBP period %lx\n", vbp);
+		return count;
+	}
+
+	dev_info(dev, "VBP period 0x%lx\n", vbp);
+	csi_dev->hw.vbp = vbp;
+
+	return count;
+}
+
+static ssize_t vbp_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.vbp);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t vfp_store(struct device *dev,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long vfp;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 16, &vfp);
+	if (ret < 0)
+		return ret;
+
+	if (vfp > 0x3ff) {
+		dev_err(dev, "Invalid VFP period %lx\n", vfp);
+		return count;
+	}
+
+	dev_info(dev, "VFP period 0x%lx\n", vfp);
+	csi_dev->hw.vfp = vfp;
+
+	return count;
+}
+
+static ssize_t vfp_show(struct device *dev,
+			struct device_attribute *attr,
+			char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%x\n", csi_dev->hw.vfp);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t virtual_channel_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	int ret;
+	unsigned long virtual_ch;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &virtual_ch);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)virtual_ch < 0 || (signed int)virtual_ch > 8) {
+		dev_err(dev, "Invalid Virtual Channel %lu\n", virtual_ch);
+		return count;
+	}
+
+	dev_info(dev, "Virtual Channel %lu\n", virtual_ch);
+	csi_dev->hw.virtual_ch = virtual_ch;
+
+	return count;
+}
+
+static ssize_t virtual_channel_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.virtual_ch);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t ipi_color_mode_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int ret;
+	unsigned long ipi_color_mode;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &ipi_color_mode);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)ipi_color_mode < 0 || (signed int)ipi_color_mode > 1) {
+		dev_err(dev,
+			"Wrong Color Mode %lu, (48 bits -> 0 or 16 bits -> 1\n",
+			ipi_color_mode);
+		return count;
+	}
+
+	dev_info(dev, "IPI Color mode %lu\n", ipi_color_mode);
+	csi_dev->hw.ipi_color_mode = ipi_color_mode;
+
+	return count;
+}
+
+static ssize_t ipi_color_mode_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.ipi_color_mode);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t ipi_auto_flush_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int ret;
+	unsigned long ipi_auto_flush;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &ipi_auto_flush);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)ipi_auto_flush < 0 || (signed int)ipi_auto_flush > 1) {
+		dev_err(dev,
+			"Invalid Auto Flush Mode %lu, (No -> 0 or Yes -> 1\n",
+			ipi_auto_flush);
+		return count;
+	}
+
+	dev_info(dev, "IPI Auto Flush %lu\n", ipi_auto_flush);
+	csi_dev->hw.ipi_auto_flush = ipi_auto_flush;
+
+	return count;
+}
+
+static ssize_t ipi_auto_flush_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.ipi_auto_flush);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t ipi_timings_mode_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t count)
+{
+	int ret;
+	unsigned long ipi_mode;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &ipi_mode);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)ipi_mode < 0 || (signed int)ipi_mode > 1) {
+		dev_err(dev,
+			"Invalid Timing Source %lu (Camera:0|Controller:1)\n",
+			ipi_mode);
+		return count;
+	}
+
+	dev_info(dev, "IPI Color mode %lu\n", ipi_mode);
+	csi_dev->hw.ipi_mode = ipi_mode;
+
+	return count;
+}
+
+static ssize_t ipi_timings_mode_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.ipi_mode);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t output_type_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	int ret;
+	unsigned long output;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	ret = kstrtoul(buf, 10, &output);
+	if (ret < 0)
+		return ret;
+
+	if ((signed int)output < 0 || (signed int)output > 1) {
+		dev_err(dev,
+			"Invalid Core output %lu to be used \
+			(IPI-> 0 or IDI->1 or BOTH- 2\n",
+			output);
+		return count;
+	}
+
+	dev_info(dev, "IPI Color mode %lu\n", output);
+	csi_dev->hw.output = output;
+
+	return count;
+}
+
+static ssize_t output_type_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
+	struct dw_csi *csi_dev = sd_to_mipi_csi_dev(sd);
+
+	char buffer[10];
+
+	snprintf(buffer, 10, "%d\n", csi_dev->hw.output);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static DEVICE_ATTR_RO(core_version);
+static DEVICE_ATTR_RO(core_reset);
+static DEVICE_ATTR_RW(n_lanes);
+static DEVICE_ATTR_RW(data_type);
+static DEVICE_ATTR_RW(hsa);
+static DEVICE_ATTR_RW(hbp);
+static DEVICE_ATTR_RW(hsd);
+static DEVICE_ATTR_RW(vsa);
+static DEVICE_ATTR_RW(vbp);
+static DEVICE_ATTR_RW(vfp);
+static DEVICE_ATTR_RW(virtual_channel);
+static DEVICE_ATTR_RW(ipi_color_mode);
+static DEVICE_ATTR_RW(ipi_auto_flush);
+static DEVICE_ATTR_RW(ipi_timings_mode);
+static DEVICE_ATTR_RW(output_type);
+
+int dw_csi_create_capabilities_sysfs(struct platform_device *pdev)
+{
+	device_create_file(&pdev->dev, &dev_attr_core_version);
+	device_create_file(&pdev->dev, &dev_attr_core_reset);
+	device_create_file(&pdev->dev, &dev_attr_n_lanes);
+	device_create_file(&pdev->dev, &dev_attr_data_type);
+	device_create_file(&pdev->dev, &dev_attr_hsa);
+	device_create_file(&pdev->dev, &dev_attr_hbp);
+	device_create_file(&pdev->dev, &dev_attr_hsd);
+	device_create_file(&pdev->dev, &dev_attr_vsa);
+	device_create_file(&pdev->dev, &dev_attr_vbp);
+	device_create_file(&pdev->dev, &dev_attr_vfp);
+	device_create_file(&pdev->dev, &dev_attr_virtual_channel);
+	device_create_file(&pdev->dev, &dev_attr_ipi_color_mode);
+	device_create_file(&pdev->dev, &dev_attr_ipi_auto_flush);
+	device_create_file(&pdev->dev, &dev_attr_ipi_timings_mode);
+	device_create_file(&pdev->dev, &dev_attr_output_type);
+
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-dphy-plat.c b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-plat.c
new file mode 100755
index 000000000000..f0628844e09b
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-plat.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver.
+ * Platform driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+// #include <media/dwc/dw-dphy-data.h>
+// #include <media/dwc/dw-csi-data.h>
+
+#include "dw-dphy-rx.h"
+
+/* Global variable for compatibility mode, this could be override later */
+static int phy_type = 1; //Changed to fit single phy 0 - single | 1 - 8 lanes phy
+
+module_param(phy_type, int, 0664);
+MODULE_PARM_DESC(phy_type, "Disable compatibility mode for D-PHY G128");
+
+static struct phy_ops dw_dphy_ops = {
+	.init = dw_dphy_init,
+	.reset = dw_dphy_reset,
+	.power_on = dw_dphy_power_on,
+	.power_off = dw_dphy_power_off,
+	.owner = THIS_MODULE,
+};
+
+static struct phy_provider *phy_provider;
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static u8 get_config_8l(struct device *dev, struct dw_dphy_rx *dphy)
+{
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		dphy->config_8l = of_get_gpio(dev->of_node, 0);
+		if (!gpio_is_valid(dphy->config_8l)) {
+			dev_warn(dev,
+				 "failed to parse 8l config, default is 0\n");
+			dphy->config_8l = 0;
+		}
+	} else {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+
+		dphy->config_8l = pdata->config_8l;
+	}
+	return dphy->config_8l;
+}
+#endif
+
+static int get_resources(struct device *dev, struct dw_dphy_rx *dphy)
+{
+	int ret = 0;
+
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		if (of_property_read_u32(dev->of_node, "snps,dphy-frequency",
+					 &dphy->dphy_freq)) {
+			dev_err(dev, "failed to find dphy frequency\n");
+			ret = -EINVAL;
+		}
+		if (of_property_read_u32(dev->of_node, "bus-width",
+					 &dphy->dphy_te_len)) {
+			dev_err(dev, "failed to find dphy te length\n");
+			ret = -EINVAL;
+		}
+		if (of_property_read_u32(dev->of_node, "snps,phy_type",
+					 &dphy->phy_type)) {
+			dev_err(dev, "failed to find dphy type\n");
+			ret = -EINVAL;
+		}
+	} else {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+
+		dphy->dphy_freq = pdata->dphy_frequency;
+		dphy->dphy_te_len = pdata->dphy_te_len;
+		dphy->dphy_gen = pdata->dphy_gen;
+	}
+	dev_set_drvdata(dev, dphy);
+
+	return ret;
+}
+
+static int phy_register(struct device *dev)
+{
+	int ret = 0;
+
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		phy_provider = devm_of_phy_provider_register(dev,
+							     dw_dphy_xlate);
+		if (IS_ERR(phy_provider)) {
+			dev_err(dev, "error getting phy provider\n");
+			ret = PTR_ERR(phy_provider);
+		}
+	} else {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+		struct dw_dphy_rx *dphy = dev_get_drvdata(dev);
+
+		ret = phy_create_lookup(dphy->phy,
+					phys[pdata->id].name,
+					csis[pdata->id].name);
+		if (ret)
+			dev_err(dev, "Failed to create dphy lookup\n");
+		else
+			dev_warn(dev, "Created dphy lookup [%s] --> [%s]\n",
+				 phys[pdata->id].name, csis[pdata->id].name);
+	}
+	return ret;
+}
+
+static void phy_unregister(struct device *dev)
+{
+	if (!dev->of_node) {
+		struct dw_phy_pdata *pdata = dev->platform_data;
+		struct dw_dphy_rx *dphy = dev_get_drvdata(dev);
+
+		phy_remove_lookup(dphy->phy,
+				  phys[pdata->id].name, csis[pdata->id].name);
+		dev_warn(dev, "Removed dphy lookup [%s] --> [%s]\n",
+			 phys[pdata->id].name, csis[pdata->id].name);
+	}
+}
+
+static int dw_dphy_rx_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dw_dphy_rx *dphy;
+	struct resource *res;
+
+	dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL);
+	if (!dphy)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dphy->base_address = devm_ioremap(&pdev->dev,
+					  res->start, resource_size(res));
+	if (IS_ERR(dphy->base_address)) {
+		dev_err(&pdev->dev, "error requesting base address\n");
+		return PTR_ERR(dphy->base_address);
+	}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+
+	dphy->dphy1_if_addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dphy->dphy1_if_addr)) {
+		dev_err(&pdev->dev, "error requesting dphy 1 if regbank\n");
+		return PTR_ERR(dphy->dphy1_if_addr);
+	}
+
+	dphy->max_lanes =
+		dw_dphy_if_read_msk(dphy, DPHYID, DPHY_ID_LANE_SUPPORT, 4);
+
+	dphy->dphy_gen = dw_dphy_if_read_msk(dphy, DPHYID, DPHY_ID_GEN, 4);
+
+	dev_info(&pdev->dev, "DPHY GEN %s with maximum %s lanes\n",
+		 dphy->dphy_gen == GEN3 ? "3" : "2",
+		 dphy->max_lanes == CTRL_8_LANES ? "8" : "4");
+
+	if (dphy->max_lanes == CTRL_8_LANES) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+		dphy->dphy2_if_addr =
+			devm_ioremap(&pdev->dev,
+				     res->start, resource_size(res));
+
+		if (IS_ERR(dphy->dphy2_if_addr)) {
+			dev_err(&pdev->dev,
+				"error requesting dphy 2 if regbank\n");
+			return PTR_ERR(dphy->dphy2_if_addr);
+		}
+		dphy->config_8l = get_config_8l(&pdev->dev, dphy);
+	}
+#endif
+	if (get_resources(dev, dphy)) {
+		dev_err(dev, "failed to parse PHY resources\n");
+		return -EINVAL;
+	}
+
+	dphy->phy = devm_phy_create(dev, NULL, &dw_dphy_ops);
+	if (IS_ERR(dphy->phy)) {
+		dev_err(dev, "failed to create PHY\n");
+		return PTR_ERR(dphy->phy);
+	}
+
+	platform_set_drvdata(pdev, dphy);
+	phy_set_drvdata(dphy->phy, dphy);
+
+	if (phy_register(dev)) {
+		dev_err(dev, "failed to register PHY\n");
+		return -EINVAL;
+	}
+
+	dphy->lp_time = 1000; /* 1000 ns */
+	dphy->lanes_config = dw_dphy_setup_config(dphy);
+
+	dev_info(&dphy->phy->dev, "Probing dphy finished\n");
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_create_capabilities_sysfs(pdev);
+#endif
+
+	return 0;
+}
+
+static int dw_dphy_rx_remove(struct platform_device *pdev)
+{
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+
+	dev_info(&dphy->phy->dev, "phy removed\n");
+	phy_unregister(&pdev->dev);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id dw_dphy_rx_of_match[] = {
+	{ .compatible = "eswin,eic770x-csi2-dphy-hw" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dw_dphy_rx_of_match);
+#endif
+
+static struct platform_driver dw_dphy_rx_driver = {
+	.probe = dw_dphy_rx_probe,
+	.remove = dw_dphy_rx_remove,
+	.driver = {
+#if IS_ENABLED(CONFIG_OF)
+		.of_match_table = of_match_ptr(dw_dphy_rx_of_match),
+#endif
+		.name = "dw-dphy",
+		.owner = THIS_MODULE,
+	}
+};
+module_platform_driver(dw_dphy_rx_driver);
+
+MODULE_DESCRIPTION("Synopsys DesignWare MIPI DPHY Rx driver");
+MODULE_AUTHOR("Luis Oliveira <lolivei@synopsys.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-dphy-rx.c b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-rx.c
new file mode 100755
index 000000000000..970d2b572c02
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-rx.c
@@ -0,0 +1,613 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver
+ * Core functions
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include "dw-dphy-rx.h"
+
+struct range_dphy_gen2 {
+	u32 freq;
+	u8 hsfregrange;
+};
+
+struct range_dphy_gen2 range_gen2[] = {
+	{ 80, 0x00 }, { 90, 0x10 }, { 100, 0x20 }, { 110, 0x30 },
+	{ 120, 0x01 }, { 130, 0x11 }, { 140, 0x21 }, { 150, 0x31 },
+	{ 160, 0x02 }, { 170, 0x12 }, { 180, 0x22 }, { 190, 0x32 },
+	{ 205, 0x03 }, { 220, 0x13 }, { 235, 0x23 }, { 250, 0x33 },
+	{ 275, 0x04 }, { 300, 0x14 }, { 325, 0x05 }, { 350, 0x15 },
+	{ 400, 0x25 }, { 450, 0x06 }, { 500, 0x16 }, { 550, 0x07 },
+	{ 600, 0x17 }, { 650, 0x08 }, { 700, 0x18 }, { 750, 0x09 },
+	{ 800, 0x19 }, { 850, 0x29 }, { 900, 0x39 }, { 950, 0x0A },
+	{ 1000, 0x1A }, { 1050, 0x2A }, { 1100, 0x3A }, { 1150, 0x0B },
+	{ 1200, 0x1B }, { 1250, 0x2B }, { 1300, 0x3B }, { 1350, 0x0C },
+	{ 1400, 0x1C }, { 1450, 0x2C }, { 1500, 0x3C }, { 1550, 0x0D },
+	{ 1600, 0x1D }, { 1650, 0x2D }, { 1700, 0x0E }, { 1750, 0x1E },
+	{ 1800, 0x2E }, { 1850, 0x3E }, { 1900, 0x0F }, { 1950, 0x1F },
+	{ 2000, 0x2F },
+};
+
+struct range_dphy_gen3 {
+	u32 freq;
+	u8 hsfregrange;
+	u32 osc_freq_target;
+};
+
+struct range_dphy_gen3 range_gen3[] = {
+	{ 80, 0x00, 0x1B6 }, { 90, 0x10, 0x1B6 }, { 100, 0x20, 0x1B6 },
+	{ 110, 0x30, 0x1B6 }, { 120, 0x01, 0x1B6 }, { 130, 0x11, 0x1B6 },
+	{ 140, 0x21, 0x1B6 }, { 150, 0x31, 0x1B6 }, { 160, 0x02, 0x1B6 },
+	{ 170, 0x12, 0x1B6 }, { 180, 0x22, 0x1B6 }, { 190, 0x32, 0x1B6 },
+	{ 205, 0x03, 0x1B6 }, { 220, 0x13, 0x1B6 }, { 235, 0x23, 0x1B6 },
+	{ 250, 0x33, 0x1B6 }, { 275, 0x04, 0x1B6 }, { 300, 0x14, 0x1B6 },
+	{ 325, 0x25, 0x1B6 }, { 350, 0x35, 0x1B6 }, { 400, 0x05, 0x1B6 },
+	{ 450, 0x16, 0x1B6 }, { 500, 0x26, 0x1B6 }, { 550, 0x37, 0x1B6 },
+	{ 600, 0x07, 0x1B6 }, { 650, 0x18, 0x1B6 }, { 700, 0x28, 0x1B6 },
+	{ 750, 0x39, 0x1B6 }, { 800, 0x09, 0x1B6 }, { 850, 0x19, 0x1B6 },
+	{ 900, 0x29, 0x1B6 }, { 950, 0x3A, 0x1B6 }, { 1000, 0x0A, 0x1B6 },
+	{ 1050, 0x1A, 0x1B6 }, { 1100, 0x2A, 0x1B6 }, { 1150, 0x3B, 0x1B6 },
+	{ 1200, 0x0B, 0x1B6 }, { 1250, 0x1B, 0x1B6 }, { 1300, 0x2B, 0x1B6 },
+	{ 1350, 0x3C, 0x1B6 }, { 1400, 0x0C, 0x1B6 }, { 1450, 0x1C, 0x1B6 },
+	{ 1500, 0x2C, 0x1B6 }, { 1550, 0x3D, 0x10F }, { 1600, 0x0D, 0x118 },
+	{ 1650, 0x1D, 0x121 }, { 1700, 0x2E, 0x12A }, { 1750, 0x3E, 0x132 },
+	{ 1800, 0x0E, 0x13B }, { 1850, 0x1E, 0x144 }, { 1900, 0x2F, 0x14D },
+	{ 1950, 0x3F, 0x155 }, { 2000, 0x0F, 0x15E }, { 2050, 0x40, 0x167 },
+	{ 2100, 0x41, 0x170 }, { 2150, 0x42, 0x178 }, { 2200, 0x43, 0x181 },
+	{ 2250, 0x44, 0x18A }, { 2300, 0x45, 0x193 }, { 2350, 0x46, 0x19B },
+	{ 2400, 0x47, 0x1A4 }, { 2450, 0x48, 0x1AD }, { 2500, 0x49, 0x1B6 }
+};
+
+u8 dw_dphy_setup_config(struct dw_dphy_rx *dphy)
+{
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	int ret;
+
+	if (dphy->max_lanes == CTRL_4_LANES) {
+		dev_vdbg(&dphy->phy->dev, "CONFIG 4L\n");
+		return CTRL_4_LANES;
+	}
+	if (IS_ENABLED(CONFIG_OF)) {
+		ret = gpio_request(dphy->config_8l, "config");
+		if (ret < 0) {
+			dev_vdbg(&dphy->phy->dev,
+				 "could not acquire config (err=%d)\n", ret);
+			return ret;
+		}
+		ret = gpio_get_value(dphy->config_8l);
+		gpio_free(dphy->config_8l);
+	} else {
+		ret = dphy->config_8l;
+	}
+
+	dev_vdbg(&dphy->phy->dev,
+		 "Booting in [%s] mode\n",
+		 ret == CTRL_8_LANES ? "8L" : "4+4L");
+	return ret;
+
+#endif /* CONFIG_DWC_MIPI_TC_DPHY_GEN3 */
+	return CTRL_4_LANES;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+void dw_dphy_if_write(struct dw_dphy_rx *dphy, u32 address, u32 data)
+{
+	writel(data, dphy->dphy1_if_addr + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		return;
+
+	iowrite32(data, dphy->dphy2_if_addr + address);
+}
+
+u32 dw_dphy_if_read(struct dw_dphy_rx *dphy, u32 address)
+{
+	u32 if1 = 0, if2 = 0;
+
+	if1 = readl(dphy->dphy1_if_addr + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		goto end;
+
+	if (dphy->lanes_config == DPHYID)
+		goto end;
+
+	if2 = readl(dphy->dphy2_if_addr + address);
+
+	if (if1 != if2)
+		dev_vdbg(&dphy->phy->dev,
+			 "Values read different for each interface\n");
+end:
+	return if1;
+}
+#endif
+
+void dw_dphy_write(struct dw_dphy_rx *dphy, u32 address, u32 data)
+{
+	iowrite32(data, dphy->base_address + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		return;
+
+	if (address == R_CSI2_DPHY_TST_CTRL0)
+		iowrite32(data, dphy->base_address + R_CSI2_DPHY2_TST_CTRL0);
+	else if (address == R_CSI2_DPHY_TST_CTRL1)
+		iowrite32(data, dphy->base_address + R_CSI2_DPHY2_TST_CTRL1);
+}
+
+u32 dw_dphy_read(struct dw_dphy_rx *dphy, u32 address)
+{
+	int dphy1 = 0, dphy2 = 0;
+
+	dphy1 = ioread32(dphy->base_address + address);
+
+	if (dphy->lanes_config == CTRL_4_LANES)
+		goto end;
+
+	if (address == R_CSI2_DPHY_TST_CTRL0)
+		dphy2 = ioread32(dphy->base_address + R_CSI2_DPHY2_TST_CTRL0);
+	else if (address == R_CSI2_DPHY_TST_CTRL1)
+		dphy2 = ioread32(dphy->base_address + R_CSI2_DPHY2_TST_CTRL1);
+	else
+		return -ENODEV;
+end:
+	return dphy1;
+}
+
+void dw_dphy_write_msk(struct dw_dphy_rx *dev, u32 address, u32 data, u8 shift,
+		       u8 width)
+{
+	u32 temp = dw_dphy_read(dev, address);
+	u32 mask = (1 << width) - 1;
+
+	temp &= ~(mask << shift);
+	temp |= (data & mask) << shift;
+	dw_dphy_write(dev, address, temp);
+}
+
+static void dw_dphy_te_12b_write(struct dw_dphy_rx *dphy, u16 addr, u8 data)
+{
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0x00, PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)(addr >> 8),
+			  PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)addr, PHY_TESTDIN,
+			  8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)data, PHY_TESTDIN,
+			  8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+}
+
+static void dw_dphy_te_8b_write(struct dw_dphy_rx *dphy, u8 addr, u8 data)
+{
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_TST_CTRL1, addr);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_TST_CTRL1, data);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+}
+
+static void dw_dphy_te_write(struct dw_dphy_rx *dphy, u16 addr, u8 data)
+{
+	if (dphy->dphy_te_len == BIT12)
+		dw_dphy_te_12b_write(dphy, addr, data);
+	else
+		dw_dphy_te_8b_write(dphy, addr, data);
+}
+
+static int dw_dphy_te_12b_read(struct dw_dphy_rx *dphy, u32 addr)
+{
+	u8 ret;
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0x00, PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)(addr >> 8),
+			  PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, (u8)addr, PHY_TESTDIN,
+			  8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0x00, 0, PHY_TESTDIN);
+	ret = dw_dphy_read_msk(dphy, R_CSI2_DPHY_TST_CTRL1, PHY_TESTDOUT, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 1);
+
+	return ret;
+}
+
+static int dw_dphy_te_8b_read(struct dw_dphy_rx *dphy, u32 addr)
+{
+	u8 ret;
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 1, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, addr, PHY_TESTDIN, 8);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTEN, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL1, 0, PHY_TESTDIN, 8);
+	ret = dw_dphy_read_msk(dphy, R_CSI2_DPHY_TST_CTRL1, PHY_TESTDOUT, 8);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 1);
+
+	return ret;
+}
+
+int dw_dphy_te_read(struct dw_dphy_rx *dphy, u32 addr)
+{
+	int ret;
+
+	if (dphy->dphy_te_len == BIT12)
+		ret = dw_dphy_te_12b_read(dphy, addr);
+	else
+		ret = dw_dphy_te_8b_read(dphy, addr);
+
+	return ret;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static void dw_dphy_if_init(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, TX_PHY);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 0);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 1);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, GLUELOGIC);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 0);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 1);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RX_PHY);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 0);
+	dw_dphy_if_write(dphy, DPHYZCALCTRL, 1);
+}
+#endif
+
+static void dw_dphy_gen3_12bit_tc_power_up(struct dw_dphy_rx *dphy)
+{
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, GLUELOGIC);
+#endif
+	dw_dphy_te_write(dphy, CFGCLKFREQRANGE_TX, 0x1C);
+
+	/* CLKSEL | UPDATEPLL | SHADOW_CLEAR | SHADOW_CTRL | FORCEPLL */
+	dw_dphy_te_write(dphy, BYPASS, 0x3F);
+
+	/* IO_DS3 | IO_DS2 | IO_DS1 | IO_DS0 */
+	if (dphy->dphy_freq > 1500)
+		dw_dphy_te_write(dphy, IO_DS, 0x0F);
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RX_PHY);
+#endif
+}
+
+static void dw_dphy_gen3_8bit_tc_power_up(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq / 1000;
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, GLUELOGIC);
+	dw_dphy_te_write(dphy, CFGCLKFREQRANGE_RX, 0x1C);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RESET);
+	dw_dphy_if_write(dphy, DPHYGLUEIFTESTER, RX_PHY);
+#endif
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX0_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX0_LSB, 0x02);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX1_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX1_LSB, 0x02);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX2_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX2_LSB, 0x02);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX3_MSB, 0x03);
+	dw_dphy_te_write(dphy, OSC_FREQ_TARGET_RX3_LSB, 0x02);
+	dw_dphy_te_write(dphy, BANDGAP_CTRL, 0x80);
+
+	if (input_freq < 2000)
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE0, 0xC0);
+
+	if (input_freq < 1000) {
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE1, 0xC0);
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE2, 0xC0);
+		dw_dphy_te_write(dphy, HS_RX_CTRL_LANE3, 0xC0);
+	}
+}
+
+int dw_dphy_g118_settle(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq, total_settle, settle_time, byte_clk, lp_time;
+
+	lp_time = dphy->lp_time;
+	input_freq = dphy->dphy_freq / 1000;
+
+	settle_time = (8 * (1000000 / (input_freq))) + 115000;
+	byte_clk = (8000000 / (input_freq));
+	total_settle = (settle_time + lp_time * 1000) / byte_clk;
+
+	if (total_settle > 0xFF)
+		total_settle = 0xFF;
+
+	return total_settle;
+}
+
+static void dw_dphy_pwr_down(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLK, 1);
+	if (dphy->lanes_config == CTRL_8_LANES)
+		dw_dphy_write_msk(dphy, R_CSI2_DPHY2_TST_CTRL0, 0, PHY_TESTCLK,
+				  1);
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+}
+
+static void dw_dphy_pwr_up(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLK, 1);
+	if (dphy->lanes_config == CTRL_8_LANES)
+		dw_dphy_write_msk(dphy, R_CSI2_DPHY2_TST_CTRL0, 1, PHY_TESTCLK,
+				  1);
+	dev_vdbg(&dphy->phy->dev, "DPHY power up.\n");
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 1);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+}
+
+static int dw_dphy_gen3_12bit_configure(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq;
+	u8 range = 0;
+
+	dev_vdbg(&dphy->phy->dev, "12bit: PHY GEN 3: Freq: %u\n", input_freq);
+	for (range = 0; (range < ARRAY_SIZE(range_gen3) - 1) &&
+	     ((input_freq / 1000) > range_gen3[range].freq);
+	     range++)
+		;
+
+	dw_dphy_gen3_12bit_tc_power_up(dphy);
+	dw_dphy_te_write(dphy, RX_SYS_1, range_gen3[range].hsfregrange);
+	dw_dphy_te_write(dphy, RX_SYS_0, 0x20);
+	dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_2,
+			 (u8)range_gen3[range].osc_freq_target);
+	dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_3,
+			 (u8)(range_gen3[range].osc_freq_target >> 8));
+	dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_4, 0x01);
+
+	if (dphy->phy_type) {
+		dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_1, 0x01);
+		dw_dphy_te_write(dphy, RX_RX_STARTUP_OVR_0, 0x80);
+	}
+
+	if (dphy->phy_type || input_freq <= 1500)
+		dw_dphy_te_write(dphy, RX_SYS_7, 0x38);
+
+	return 0;
+}
+
+static int dw_dphy_gen3_8bit_configure(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq;
+	u8 data;
+	u8 range = 0;
+
+	dev_vdbg(&dphy->phy->dev, "8bit: PHY GEN 3: Freq: %u\n", input_freq);
+	for (range = 0; (range < ARRAY_SIZE(range_gen3) - 1) &&
+	     ((input_freq / 1000) > range_gen3[range].freq);
+	     range++)
+		;
+
+	dw_dphy_te_write(dphy, RX_SKEW_CAL, dw_dphy_g118_settle(dphy));
+	data = 1 << 7 | range_gen3[range].hsfregrange;
+	dw_dphy_te_write(dphy, HSFREQRANGE_8BIT, data);
+	dw_dphy_gen3_8bit_tc_power_up(dphy);
+
+	return 0;
+}
+
+static int dw_dphy_gen2_configure(struct dw_dphy_rx *dphy)
+{
+	u32 input_freq = dphy->dphy_freq;
+	u8 data;
+	u8 range = 0;
+
+	/* provide an initial active-high test clear pulse in TESTCLR  */
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 1, PHY_TESTCLR, 1);
+	dw_dphy_write_msk(dphy, R_CSI2_DPHY_TST_CTRL0, 0, PHY_TESTCLR, 1);
+
+	dev_vdbg(&dphy->phy->dev, "PHY GEN 2: Freq: %u\n", input_freq);
+	for (range = 0; (range < ARRAY_SIZE(range_gen2) - 1) &&
+	     ((input_freq / 1000) > range_gen2[range].freq); range++)
+		;
+
+	data = range_gen2[range].hsfregrange << 1;
+	dw_dphy_te_write(dphy, HSFREQRANGE_8BIT, data);
+
+	return 0;
+}
+
+static int dw_dphy_configure(struct dw_dphy_rx *dphy)
+{
+	dw_dphy_pwr_down(dphy);
+
+	if (dphy->dphy_gen == GEN3) {
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+		dw_dphy_if_init(dphy);
+#endif
+		if (dphy->dphy_te_len == BIT12)
+			dw_dphy_gen3_12bit_configure(dphy);
+		else
+			dw_dphy_gen3_8bit_configure(dphy);
+	} else {
+		dw_dphy_gen2_configure(dphy);
+	}
+	dw_dphy_pwr_up(dphy);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+int dw_dphy_if_set_idelay(struct dw_dphy_rx *dphy, u8 dly, u8 cells)
+{
+	u32 val = 0;
+
+	dw_dphy_if_write(dphy, IDLYCFG, 0);
+	dw_dphy_if_write(dphy, IDLYSEL, cells);
+	dw_dphy_if_write(dphy, IDLYCNTINVAL, dly);
+
+	/* Pulse Value Set */
+	dw_dphy_if_write(dphy, IDLYCFG, 1);
+	usleep_range(10, 20);
+	dw_dphy_if_write(dphy, IDLYCFG, 0);
+
+	/* Pulse IDELAY CTRL Reset */
+	dw_dphy_if_write(dphy, DPHY1REGRSTN, 0);
+	usleep_range(10, 20);
+	dw_dphy_if_write(dphy, DPHY1REGRSTN, 1);
+
+	/* Get Value*/
+	val = dw_dphy_if_read(dphy, IDLYCNTOUTVAL);
+
+	if (val != dly) {
+		dev_vdbg(&dphy->phy->dev,
+			 "odelay config failed, set %d get %d", dly, val);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int dw_dphy_if_get_idelay(struct dw_dphy_rx *dphy)
+{
+	return dw_dphy_if_read(dphy, IDLYCNTOUTVAL);
+}
+
+int dw_dphy_if_set_idelay_lane(struct dw_dphy_rx *dphy, u8 dly, u8 lane)
+{
+	int cell;
+
+	switch (lane) {
+	case 0:
+		for (cell = 3; cell <= 10; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 1:
+		for (cell = 14; cell <= 21; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 2:
+		for (cell = 24; cell <= 31; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 3:
+		for (cell = 34; cell <= 41; cell++)
+			dw_dphy_if_set_idelay(dphy, dly, cell);
+		break;
+	case 4: /* ALL */
+		dw_dphy_if_set_idelay(dphy, dly, 0x7F);
+		break;
+	default:
+		dev_err(&dphy->phy->dev, "Lane Value not recognized\n");
+		return -1;
+	}
+	return 0;
+}
+#endif
+
+int dw_dphy_init(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	dev_warn(&dphy->phy->dev, "Init DPHY.\n");
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+
+	return 0;
+}
+
+static int dw_dphy_set_phy_state(struct dw_dphy_rx *dphy, u32 on)
+{
+	u8 hs_freq;
+
+	dphy->lanes_config = dw_dphy_setup_config(dphy);
+
+	if (dphy->dphy_te_len == BIT12)
+		hs_freq = RX_SYS_1;
+	else
+		hs_freq = HSFREQRANGE_8BIT;
+
+	if (on) {
+		dw_dphy_configure(dphy);
+		dev_info(&dphy->phy->dev,
+			 "HS Code: 0X%x\n", dw_dphy_te_read(dphy, hs_freq));
+	} else {
+		dw_dphy_write(dphy, R_CSI2_DPHY_SHUTDOWNZ, 0);
+		dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	}
+
+	return 0;
+}
+
+int dw_dphy_power_on(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	dev_info(&dphy->phy->dev, "DPHY Power ON\n");
+
+	return dw_dphy_set_phy_state(dphy, 1);
+}
+
+int dw_dphy_power_off(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	return dw_dphy_set_phy_state(dphy, 0);
+}
+
+int dw_dphy_reset(struct phy *phy)
+{
+	struct dw_dphy_rx *dphy = phy_get_drvdata(phy);
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	usleep_range(100, 200);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-dphy-rx.h b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-rx.h
new file mode 100755
index 000000000000..2e8d68c914c1
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-rx.h
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#ifndef __PHY_SNPS_DPHY_RX_H__
+#define __PHY_SNPS_DPHY_RX_H__
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_gpio.h>
+#include <linux/phy/phy.h>
+#include <linux/phy/phy-mipi-dphy.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+/* DPHY interface register bank*/
+
+#define R_CSI2_DPHY_SHUTDOWNZ 0x0
+#define R_CSI2_DPHY_RSTZ 0x4
+#define R_CSI2_DPHY_RX 0x8
+#define	R_CSI2_DPHY_STOPSTATE 0xC
+#define R_CSI2_DPHY_TST_CTRL0 0x10
+#define R_CSI2_DPHY_TST_CTRL1 0x14
+#define R_CSI2_DPHY2_TST_CTRL0 0x18
+#define R_CSI2_DPHY2_TST_CTRL1 0x1C
+
+enum dphy_id_mask {
+	DPHY_ID_LANE_SUPPORT = 0,
+	DPHY_ID_IF = 4,
+	DPHY_ID_GEN = 8,
+};
+
+enum dphy_gen_values {
+	GEN1,
+	GEN2,
+	GEN3,
+};
+
+enum dphy_interface_length {
+	BIT8 = 8,
+	BIT12 = 12,
+};
+
+enum tst_ctrl0 {
+	PHY_TESTCLR,
+	PHY_TESTCLK,
+};
+
+enum tst_ctrl1 {
+	PHY_TESTDIN = 0,
+	PHY_TESTDOUT = 8,
+	PHY_TESTEN = 16,
+};
+
+enum lanes_config_values {
+	CTRL_4_LANES,
+	CTRL_8_LANES,
+};
+
+enum dphy_tc {
+	CFGCLKFREQRANGE_TX = 0x02,
+	CFGCLKFREQRANGE_RX = 0x05,
+	BYPASS = 0x20,
+	IO_DS = 0x30,
+};
+
+enum dphy_8bit_interface_addr {
+	BANDGAP_CTRL = 0x24,
+	HS_RX_CTRL_LANE0 = 0x42,
+	HSFREQRANGE_8BIT = 0x44,
+	OSC_FREQ_TARGET_RX0_LSB	= 0x4e,
+	OSC_FREQ_TARGET_RX0_MSB	= 0x4f,
+	HS_RX_CTRL_LANE1 = 0x52,
+	OSC_FREQ_TARGET_RX1_LSB	= 0x5e,
+	OSC_FREQ_TARGET_RX1_MSB	= 0x5f,
+	RX_SKEW_CAL = 0x7e,
+	HS_RX_CTRL_LANE2 = 0x82,
+	OSC_FREQ_TARGET_RX2_LSB	= 0x8e,
+	OSC_FREQ_TARGET_RX2_MSB	= 0x8f,
+	HS_RX_CTRL_LANE3 = 0x92,
+	OSC_FREQ_TARGET_RX3_LSB	= 0x9e,
+	OSC_FREQ_TARGET_RX3_MSB	= 0x9f,
+};
+
+enum dphy_12bit_interface_addr {
+	RX_SYS_0 = 0x01,
+	RX_SYS_1 = 0x02,
+	RX_SYS_7 = 0x08,
+	RX_RX_STARTUP_OVR_0 = 0xe0,
+	RX_RX_STARTUP_OVR_1 = 0xe1,
+	RX_RX_STARTUP_OVR_2 = 0xe2,
+	RX_RX_STARTUP_OVR_3 = 0xe3,
+	RX_RX_STARTUP_OVR_4 = 0xe4,
+};
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+/* Testchip interface register bank */
+#define IDLYCFG	0x00
+#define IDLYSEL	0x04
+#define IDLYCNTINVAL 0x08
+#define IDLYCNTOUTVAL 0x0c
+#define DPHY1REGRSTN 0x10
+#define DPHYZCALSTAT 0x14
+#define DPHYZCALCTRL 0x18
+#define DPHYLANE0STAT 0x1c
+#define DPHYLANE1STAT 0x20
+#define DPHYLANE2STAT 0x24
+#define DPHYLANE3STAT 0x28
+#define DPHYCLKSTAT 0x2c
+#define DPHYZCLKCTRL 0x30
+#define TCGENPURPOSOUT 0x34
+#define TCGENPURPOSIN 0x38
+#define DPHYGENERICOUT 0x3c
+#define DPHYGENERICIN 0x40
+#define DPHYGLUEIFTESTER 0x44
+#define DPHYID 0x100
+
+#define DPHY_DEFAULT_FREQ 300000
+
+enum glueiftester {
+	RESET = 0x0,
+	TX_PHY = 0x1,
+	RX_PHY = 0x2,
+	GLUELOGIC = 0x4,
+};
+#endif
+
+/**
+ * struct phy		specifies associated phy component
+ * struct cfg		to pass mipi dphy specific configurations
+ * @lanes_config	lanes configuration
+ * @dphy_freq		operating frequency of the d-phy (mbps)
+ * @phy_type		dphy can be of two types, passed here
+ * @dphy_gen		dphy can be of three generations, passed here
+ * @dphy_te_len		bus width
+ * @max_lanes		maximum number of lanes
+ * @lp_time		time in low-power
+ * @base_address	memmory address of dphy test interface
+ * @dphy1_if_addr	gluelogic dphy 1 memmory address of interface
+ * @dphy2_if_addr	gluelogic dphy 2 memmory address of interface
+ * @config_8l		eight lanes configuration
+ */
+
+struct dw_dphy_rx {
+	struct phy *phy;
+	struct phy_configure_opts_mipi_dphy *cfg;
+	u32 lanes_config;
+	u32 dphy_freq;
+	u32 phy_type;
+	u32 dphy_gen;
+	u32 dphy_te_len;
+	u32 max_lanes;
+	u32 lp_time;
+	void __iomem *base_address;
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	void __iomem *dphy1_if_addr;
+	void __iomem *dphy2_if_addr;
+	u8 config_8l;
+	u8 (*get_config_8l)(struct device *dev, struct dw_dphy_rx *dphy);
+#endif
+	u8 (*phy_register)(struct device *dev);
+	void (*phy_unregister)(struct device *dev);
+};
+
+int dw_dphy_init(struct phy *phy);
+int dw_dphy_reset(struct phy *phy);
+int dw_dphy_power_off(struct phy *phy);
+int dw_dphy_power_on(struct phy *phy);
+u8 dw_dphy_setup_config(struct dw_dphy_rx *dphy);
+void dw_dphy_write(struct dw_dphy_rx *dphy, u32 address, u32 data);
+u32 dw_dphy_read(struct dw_dphy_rx *dphy, u32 address);
+int dw_dphy_te_read(struct dw_dphy_rx *dphy, u32 addr);
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+u32 dw_dphy_if_read(struct dw_dphy_rx *dphy, u32 address);
+int dw_dphy_if_get_idelay(struct dw_dphy_rx *dphy);
+int dw_dphy_if_set_idelay_lane(struct dw_dphy_rx *dphy, u8 dly, u8 lane);
+int dw_dphy_create_capabilities_sysfs(struct platform_device *pdev);
+
+static inline
+u32 dw_dphy_if_read_msk(struct dw_dphy_rx *dphy,
+			u32 address, u8 shift, u8 width)
+{
+	return (dw_dphy_if_read(dphy, address) >> shift) & ((1 << width) - 1);
+}
+#endif /*IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)*/
+
+static inline struct phy *dw_dphy_xlate(struct device *dev,
+					struct of_phandle_args *args)
+{
+	struct dw_dphy_rx *dphy = dev_get_drvdata(dev);
+
+	return dphy->phy;
+}
+
+static inline
+u32 dw_dphy_read_msk(struct dw_dphy_rx *dev, u32 address, u8 shift,  u8 width)
+{
+	return (dw_dphy_read(dev, address) >> shift) & ((1 << width) - 1);
+}
+#endif /*__PHY_SNPS_DPHY_RX_H__*/
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-dphy-sysfs.c b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-sysfs.c
new file mode 100755
index 000000000000..d893e29f7adb
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-dphy-sysfs.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI D-PHY controller driver.
+ * SysFS components for the platform driver
+ *
+ * Author: Luis Oliveira <luis.oliveira@synopsys.com>
+ */
+
+#include "dw-dphy-rx.h"
+
+static ssize_t dphy_reset_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[15];
+
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 0);
+	usleep_range(100, 200);
+	dw_dphy_write(dphy, R_CSI2_DPHY_RSTZ, 1);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t dphy_freq_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf,
+			       size_t count)
+{
+	int ret;
+	unsigned long freq;
+
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+
+	ret = kstrtoul(buf, 10, &freq);
+	if (ret < 0)
+		return ret;
+
+	if (freq > 2500) {
+		dev_info(dev, "Freq must be under 2500 Mhz\n");
+		return count;
+	}
+	if (freq < 80) {
+		dev_info(dev, "Freq must be over 80 Mhz\n");
+		return count;
+	}
+
+	dev_vdbg(dev, "Data Rate %lu Mbps\n", freq);
+	dphy->dphy_freq = freq*1000;
+
+	return count;
+}
+
+static ssize_t dphy_freq_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[15];
+
+	snprintf(buffer,
+		 sizeof(buffer),
+		 "Freq %d\n", dphy->dphy_freq);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t dphy_addr_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf,
+			       size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long val;
+	u8 addr, payload;
+	int ret;
+
+	ret = kstrtoul(buf, 32, &val);
+	if (ret < 0)
+		return ret;
+
+	payload = (u16)val;
+	addr = (u16)(val >> 16);
+
+	dev_vdbg(dev, "addr 0x%lX\n", val);
+	dev_vdbg(dev, "payload: 0x%X\n", addr);
+	dev_vdbg(dev, "Addr [0x%x] -> 0x%x\n", (unsigned int)addr,
+		 dw_dphy_te_read(dphy, addr));
+
+	return count;
+}
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static ssize_t idelay_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[15];
+
+	snprintf(buffer,
+		 sizeof(buffer), "idelay %d\n", dw_dphy_if_get_idelay(dphy));
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t idelay_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long val;
+	u8 lane, delay;
+	int ret;
+
+	ret = kstrtoul(buf, 16, &val);
+	if (ret < 0)
+		return ret;
+
+	lane = (u8)val;
+	delay = (u8)(val >> 8);
+
+	dev_vdbg(dev, "Lanes %u\n", lane);
+	dev_vdbg(dev, "Delay %u\n", delay);
+
+	dw_dphy_if_set_idelay_lane(dphy, delay, lane);
+
+	return count;
+}
+#endif
+
+static ssize_t len_config_store(struct device *dev,
+				struct device_attribute *attr,
+				const char *buf,
+				size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long length;
+	int ret;
+
+	ret = kstrtoul(buf, 10, &length);
+	if (ret < 0)
+		return ret;
+
+	if (length == BIT8)
+		dev_vdbg(dev, "Configured for 8-bit interface\n");
+	else if (length == BIT12)
+		dev_vdbg(dev, "Configured for 12-bit interface\n");
+	else
+		return count;
+
+	dphy->dphy_te_len = length;
+
+	return count;
+}
+
+static ssize_t len_config_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[20];
+
+	snprintf(buffer, sizeof(buffer), "Length %d\n", dphy->dphy_te_len);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static ssize_t dw_dphy_g118_settle_store(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	unsigned long lp_time;
+	int ret;
+
+	ret = kstrtoul(buf, 10, &lp_time);
+	if (ret < 0)
+		return ret;
+
+	if (lp_time > 1 && lp_time < 10000) {
+		dphy->lp_time = lp_time;
+	} else {
+		dev_vdbg(dev, "Invalid Value configuring for 1000 ns\n");
+		dphy->lp_time = 1000;
+	}
+
+	dphy->lp_time = lp_time;
+
+	return count;
+}
+
+static ssize_t dw_dphy_g118_settle_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_dphy_rx *dphy = platform_get_drvdata(pdev);
+	char buffer[10];
+
+	snprintf(buffer, sizeof(buffer), "Settle %d ns\n", dphy->lp_time);
+
+	return strlcpy(buf, buffer, PAGE_SIZE);
+}
+
+static DEVICE_ATTR_RO(dphy_reset);
+static DEVICE_ATTR_RW(dphy_freq);
+static DEVICE_ATTR_WO(dphy_addr);
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+static DEVICE_ATTR_RW(idelay);
+#endif
+static DEVICE_ATTR_RW(len_config);
+static DEVICE_ATTR_RW(dw_dphy_g118_settle);
+
+int dw_dphy_create_capabilities_sysfs(struct platform_device *pdev)
+{
+	device_create_file(&pdev->dev, &dev_attr_dphy_reset);
+	device_create_file(&pdev->dev, &dev_attr_dphy_freq);
+	device_create_file(&pdev->dev, &dev_attr_dphy_addr);
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+	device_create_file(&pdev->dev, &dev_attr_idelay);
+#endif
+	device_create_file(&pdev->dev, &dev_attr_len_config);
+	device_create_file(&pdev->dev, &dev_attr_dw_dphy_g118_settle);
+	return 0;
+}
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-mipi-csi.c b/drivers/media/platform/eswin/mipi-csi2/dw-mipi-csi.c
new file mode 100755
index 000000000000..fd94144747be
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-mipi-csi.c
@@ -0,0 +1,668 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver
+ * Core MIPI CSI-2 functions
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#include "dw-mipi-csi.h"
+
+#define DEBUG
+#define VERBOSE_DEBUG
+
+static struct R_CSI2 reg = {
+	.VERSION = 0x00,
+	.N_LANES = 0x04,
+	.CTRL_RESETN = 0x08,
+	.INTERRUPT = 0x0C,
+	.DATA_IDS_1 = 0x10,
+	.DATA_IDS_2 = 0x14,
+	.PHY_CFG = 0x18,
+	.PHY_MODE = 0x1c,
+	.AP_INTERRUPT = 0x2c,
+	.IPI_MODE = 0x80,
+	.IPI_VCID = 0x84,
+	.IPI_DATA_TYPE = 0x88,
+	.IPI_MEM_FLUSH = 0x8C,
+	.IPI_HSA_TIME = 0x90,
+	.IPI_HBP_TIME = 0x94,
+	.IPI_HSD_TIME = 0x98,
+	.IPI_HLINE_TIME = 0x9C,
+	.IPI_SOFTRSTN = 0xA0,
+	.IPI_ADV_FEATURES = 0xAC,
+	.IPI_VSA_LINES = 0xB0,
+	.IPI_VBP_LINES = 0xB4,
+	.IPI_VFP_LINES = 0xB8,
+	.IPI_VACTIVE_LINES = 0xBC,
+
+	.IPI2_MODE = 0x200,
+	.IPI2_VCID = 0x204,
+	.IPI2_DATA_TYPE = 0x208,
+	.IPI2_MEM_FLUSH = 0x20C,
+	.IPI2_HSA_TIME = 0x210,
+	.IPI2_HBP_TIME = 0x214,
+	.IPI2_HSD_TIME = 0x218,
+	.IPI2_ADV_FEATURES = 0x21C,
+	.MASK_INT_IPI2 = 0x154,
+	.MASK_INT_AP_IPI2 = 0x1a4,
+
+	.IPI3_MODE = 0x220,
+	.IPI3_VCID = 0x224,
+	.IPI3_DATA_TYPE = 0x228,
+	.IPI3_MEM_FLUSH = 0x22C,
+	.IPI3_HSA_TIME = 0x230,
+	.IPI3_HBP_TIME = 0x234,
+	.IPI3_HSD_TIME = 0x238,
+	.IPI3_ADV_FEATURES = 0x23C,
+	.MASK_INT_IPI3 = 0x164,
+	.MASK_INT_AP_IPI3 = 0x1b4,
+
+	.INT_PHY_FATAL = 0xe0,
+	.MASK_INT_PHY_FATAL = 0xe4,
+	.FORCE_INT_PHY_FATAL = 0xe8,
+	.INT_PKT_FATAL = 0xf0,
+	.MASK_INT_PKT_FATAL = 0xf4,
+	.FORCE_INT_PKT_FATAL = 0xf8,
+	.INT_PHY = 0x110,
+	.MASK_INT_PHY = 0x114,
+	.FORCE_INT_PHY = 0x118,
+	.INT_LINE = 0x130,
+	.MASK_INT_LINE = 0x134,
+	.FORCE_INT_LINE = 0x138,
+	.INT_IPI = 0x140,
+	.MASK_INT_IPI = 0x144,
+	.FORCE_INT_IPI = 0x148,
+};
+
+struct interrupt_type csi_int = {
+	.PHY_FATAL = BIT(0),
+	.PKT_FATAL = BIT(1),
+	.PHY = BIT(16),
+};
+
+#define dw_print(VAR) \
+	dev_info(csi_dev->dev, "%s: 0x%x: %X\n", "##VAR##",\
+	VAR, dw_mipi_csi_read(csi_dev, VAR))
+
+void dw_mipi_csi_write_part(struct dw_csi *dev, u32 address, u32 data,
+			    u8 shift, u8 width)
+{
+	u32 mask = (1 << width) - 1;
+	u32 temp = dw_mipi_csi_read(dev, address);
+
+	temp &= ~(mask << shift);
+	temp |= (data & mask) << shift;
+	dw_mipi_csi_write(dev, address, temp);
+}
+
+void dw_mipi_csi_reset(struct dw_csi *csi_dev)
+{
+	dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 0);
+	usleep_range(100, 200);
+	dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 1);
+}
+
+int dw_mipi_csi_mask_irq_power_off(struct dw_csi *csi_dev)
+{
+	if (csi_dev->hw_version_major == 1) {
+		/* set only one lane (lane 0) as active (ON) */
+		dw_mipi_csi_write(csi_dev, reg.N_LANES, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY_FATAL, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PKT_FATAL, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_LINE, 0);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI, 0);
+		if(csi_dev->hw.ipi2_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI2, 0);
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI2, 0);
+		}
+		if(csi_dev->hw.ipi3_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI3, 0);
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI3, 0);
+		}
+		/* only for version 1.30 */
+		if (csi_dev->hw_version_minor == 30)
+			dw_mipi_csi_write(csi_dev,
+					  reg.MASK_INT_FRAME_FATAL, 0);
+
+		dw_mipi_csi_write(csi_dev, reg.CTRL_RESETN, 0);
+
+		/* only for version 1.40 */
+		if (csi_dev->hw_version_minor == 40) {
+			dw_mipi_csi_write(csi_dev, reg.MSK_BNDRY_FRAME_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_SEQ_FRAME_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_CRC_FRAME_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_PLD_CRC_FATAL, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_DATA_ID, 0);
+			dw_mipi_csi_write(csi_dev, reg.MSK_ECC_CORRECT, 0);
+		}
+	}
+
+	return 0;
+}
+
+int dw_mipi_csi_hw_stdby(struct dw_csi *csi_dev)
+{
+	if (csi_dev->hw_version_major == 1) {
+		/* set only one lane (lane 0) as active (ON) */
+		dw_mipi_csi_reset(csi_dev);
+		dw_mipi_csi_write(csi_dev, reg.N_LANES, 0);
+#ifdef DWC_PHY_USING
+		phy_init(csi_dev->phy);
+#endif
+
+		/* only for version 1.30 */
+		if (csi_dev->hw_version_minor == 30)
+			dw_mipi_csi_write(csi_dev,
+					  reg.MASK_INT_FRAME_FATAL,
+					  GENMASK(31, 0));
+
+		/* common */
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY_FATAL,
+				  GENMASK(8, 0));
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PKT_FATAL,
+				  GENMASK(1, 0));
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_PHY, 0xff00ff);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_LINE, 0xff00ff);
+		dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI, GENMASK(6, 0));
+		if(csi_dev->hw.ipi2_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI2, GENMASK(5, 0));
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI2, GENMASK(5, 0));
+		}
+		if(csi_dev->hw.ipi3_en) {
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_IPI3, GENMASK(5, 0));
+			dw_mipi_csi_write(csi_dev, reg.MASK_INT_AP_IPI3, GENMASK(5, 0));
+		}
+
+		/* only for version 1.40 */
+		if (csi_dev->hw_version_minor == 40) {
+			//dw_mipi_csi_write(csi_dev,reg.MSK_BNDRY_FRAME_FATAL,GENMASK(31, 0));
+			//dw_mipi_csi_write(csi_dev,reg.MSK_SEQ_FRAME_FATAL,GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_CRC_FRAME_FATAL,GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_PLD_CRC_FATAL,GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_DATA_ID, GENMASK(31, 0));
+			dw_mipi_csi_write(csi_dev,reg.MSK_ECC_CORRECT, GENMASK(31, 0));
+		}
+	}
+	return 0;
+}
+
+void dw_mipi_csi_set_ipi_fmt(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+
+	switch (csi_dev->fmt->mbus_code) {
+	case MEDIA_BUS_FMT_RGB666_1X18:
+		csi_dev->hw.ipi_dt = CSI_2_RGB666;
+		break;
+
+	case MEDIA_BUS_FMT_RGB565_2X8_BE:
+	case MEDIA_BUS_FMT_RGB565_2X8_LE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB565;
+		break;
+
+	case MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE:
+	case MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB555;
+		break;
+
+	case MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE:
+	case MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB444;
+		break;
+
+		break;
+	case MEDIA_BUS_FMT_RGB888_2X12_LE:
+	case MEDIA_BUS_FMT_RGB888_2X12_BE:
+		csi_dev->hw.ipi_dt = CSI_2_RGB888;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_BE:
+		csi_dev->hw.ipi_dt = CSI_2_RAW10;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+		csi_dev->hw.ipi_dt = CSI_2_RAW12;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR14_1X14:
+		csi_dev->hw.ipi_dt = CSI_2_RAW14;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR16_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_RAW16;
+		break;
+
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+		csi_dev->hw.ipi_dt = CSI_2_RAW8;
+		break;
+
+	case MEDIA_BUS_FMT_YVYU8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV422_8;
+		break;
+
+	case MEDIA_BUS_FMT_VYUY8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV422_8;
+		break;
+
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		csi_dev->hw.ipi_dt = CSI_2_YUV422_10;
+		break;
+
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV420_8_LEG;
+		break;
+
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		csi_dev->hw.ipi_dt = CSI_2_YUV420_8;
+		break;
+
+	case MEDIA_BUS_FMT_VUY8_1X24:
+		csi_dev->hw.ipi_dt = CSI_2_YUV420_10;
+		break;
+
+	case MEDIA_BUS_FMT_Y8_1X8:
+		csi_dev->hw.ipi_dt = CSI_2_RAW8;
+		break;
+
+	case MEDIA_BUS_FMT_Y10_1X10:
+		csi_dev->hw.ipi_dt = CSI_2_RAW8;
+		break;
+
+	default:
+		break;
+	}
+
+	dev_info(dev, "Selected IPI Data Type 0x%X\n", csi_dev->hw.ipi_dt);
+}
+
+void dw_mipi_csi_fill_timings(struct dw_csi *dev)
+{
+	dev->hw.ipi_vcid = 0;
+	dev->hw.ipi_dt = CSI_2_RAW10;
+	dev->hw.ipi_emb = 1;
+	dev->hw.ipi_color_mode = COLOR16;
+	dev->hw.ipi_auto_flush = 0;
+	dev->hw.ipi_mode = CAMERA_TIMING;
+	dev->hw.ipi_cut_through = CTACTIVE;
+	dev->hw.ipi_line_event = LINE_EVENT_SELECTION(EVSELPROG) | EN_VIDEO |  EN_EMBEDDED;
+	dev->hw.output = 0;
+	dev->hw.frame_det = 1;
+
+	dev->hw.hsa = 1;
+	dev->hw.hbp = 1;
+	dev->hw.hsd = 1;
+	dev->hw.htotal = 0;
+
+	dev->hw.vsa = 0;
+	dev->hw.vbp = 0;
+	dev->hw.vfp = 0;
+	dev->hw.vactive = 0;
+}
+
+void dw_mipi_csi_start(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+
+    /* Configure PHY mode  */
+    dw_mipi_csi_write(csi_dev, reg.PHY_MODE, csi_dev->hw.phy_mode);
+    /* Configure number of lanes */
+    dw_mipi_csi_write(csi_dev, reg.N_LANES, csi_dev->hw.num_lanes - 1);
+    /* Configure PPI width */
+    dw_mipi_csi_write(csi_dev, reg.PHY_CFG, csi_dev->hw.ppi_width);
+
+    /* select IPI virtual channal */
+    dw_mipi_csi_write(csi_dev, reg.IPI_VCID, csi_dev->hw.ipi_vcid);
+    /* select IPI data type */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_DATA_TYPE, csi_dev->hw.ipi_dt, 0, 6);
+    /* configure embedded data */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_DATA_TYPE, csi_dev->hw.ipi_emb, 8, 1);
+    /* enable IPI mode */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, 1, 24, 1);
+    /* Configure IPI MODE */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, csi_dev->hw.ipi_mode, 0, 1);
+    /* Configure IPI data interface */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, csi_dev->hw.ipi_color_mode, 8, 1);
+    /* Configure IPI cut through */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MODE, csi_dev->hw.ipi_cut_through, 16, 1);
+    /* Configure IPI MEM flush */
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_MEM_FLUSH, csi_dev->hw.ipi_auto_flush, 8, 1);
+
+    if (csi_dev->hw.ipi_mode == CAMERA_TIMING) {
+        /* TODO: Configure line event selection */
+        dw_mipi_csi_write(csi_dev, reg.IPI_ADV_FEATURES, csi_dev->hw.ipi_line_event);
+        /* Configure ipi sync event mode */
+        //dw_mipi_csi_write_part(csi_dev, reg.IPI_ADV_FEATURES, csi_dev->hw.frame_det, 24, 1);
+    }
+
+    // dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 0, 1);
+
+	dev_vdbg(dev, "*********** config *********\n");
+	dev_vdbg(dev, "IPI enable: %s\n",
+			csi_dev->hw.output ? "YES" : "NO");
+	dev_vdbg(dev, "video mode transmission type: %s timming\n",
+			csi_dev->hw.ipi_mode ? "controller" : "camera");
+	dev_vdbg(dev, "Color Mode: %s\n",
+			csi_dev->hw.ipi_color_mode ? "16 bits" : "48 bits");
+	dev_vdbg(dev, "Cut Through Mode: %s\n",
+			csi_dev->hw.ipi_cut_through ? "enable" : "disable");
+	dev_vdbg(dev, "Virtual Channel: %d\n",
+			csi_dev->hw.ipi_vcid);
+	dev_vdbg(dev, "Auto-flush: %d\n",
+			csi_dev->hw.ipi_auto_flush);
+	//dw_mipi_csi_write(csi_dev, reg.IPI_SOFTRSTN, 1);
+
+#ifdef DWC_PHY_USING
+		if (csi_dev->hw.ipi_mode == AUTO_TIMING)
+			phy_power_on(csi_dev->phy);
+#endif
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HSA_TIME, csi_dev->hw.hsa);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HBP_TIME, csi_dev->hw.hbp);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HSD_TIME, csi_dev->hw.hsd);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_HLINE_TIME, csi_dev->hw.htotal);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VSA_LINES, csi_dev->hw.vsa);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VBP_LINES, csi_dev->hw.vbp);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VFP_LINES, csi_dev->hw.vfp);
+	dw_mipi_csi_write(csi_dev,
+				reg.IPI_VACTIVE_LINES, csi_dev->hw.vactive);
+	if(csi_dev->hw.ipi2_en) {
+
+		/* select IPI virtual channal */
+		dw_mipi_csi_write(csi_dev, reg.IPI2_VCID, csi_dev->hw.ipi2_vcid);
+		/* select IPI data type */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_DATA_TYPE, csi_dev->hw.ipi_dt, 0, 6);
+		/* configure embedded data */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_DATA_TYPE, csi_dev->hw.ipi_emb, 8, 1);
+		/* enable IPI mode */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, 1, 24, 1);
+		/* Configure IPI MODE */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, csi_dev->hw.ipi_mode, 0, 1);
+		/* Configure IPI data interface */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, csi_dev->hw.ipi_color_mode, 8, 1);
+		/* Configure IPI cut through */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MODE, csi_dev->hw.ipi_cut_through, 16, 1);
+		/* Configure IPI MEM flush */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI2_MEM_FLUSH, csi_dev->hw.ipi_auto_flush, 8, 1);
+
+		if (csi_dev->hw.ipi_mode == CAMERA_TIMING) {
+			/* TODO: Configure line event selection */
+			dw_mipi_csi_write(csi_dev, reg.IPI2_ADV_FEATURES, csi_dev->hw.ipi_line_event);
+			/* Configure ipi sync event mode */
+			//dw_mipi_csi_write_part(csi_dev, reg.IPI2_ADV_FEATURES, csi_dev->hw.frame_det, 24, 1);
+		}
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI2_HSA_TIME, csi_dev->hw.hsa);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI2_HBP_TIME, csi_dev->hw.hbp);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI2_HSD_TIME, csi_dev->hw.hsd);
+		dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 4, 1);//bit4 IPI2_soft reset.
+		dev_vdbg(dev, "IPI2 enable: %s\n", csi_dev->hw.ipi2_en ? "YES" : "NO");
+		dev_vdbg(dev, "IPI2 Virtual Channel: %d\n", csi_dev->hw.ipi2_vcid);
+	}
+	if(csi_dev->hw.ipi3_en) {
+
+		/* select IPI virtual channal */
+		dw_mipi_csi_write(csi_dev, reg.IPI3_VCID, csi_dev->hw.ipi3_vcid);
+		/* select IPI data type */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_DATA_TYPE, csi_dev->hw.ipi_dt, 0, 6);
+		/* configure embedded data */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_DATA_TYPE, csi_dev->hw.ipi_emb, 8, 1);
+		/* enable IPI mode */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, 1, 24, 1);
+		/* Configure IPI MODE */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, csi_dev->hw.ipi_mode, 0, 1);
+		/* Configure IPI data interface */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, csi_dev->hw.ipi_color_mode, 8, 1);
+		/* Configure IPI cut through */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MODE, csi_dev->hw.ipi_cut_through, 16, 1);
+		/* Configure IPI MEM flush */
+		dw_mipi_csi_write_part(csi_dev, reg.IPI3_MEM_FLUSH, csi_dev->hw.ipi_auto_flush, 8, 1);
+
+		if (csi_dev->hw.ipi_mode == CAMERA_TIMING) {
+			/* TODO: Configure line event selection */
+			dw_mipi_csi_write(csi_dev, reg.IPI3_ADV_FEATURES, csi_dev->hw.ipi_line_event);
+			/* Configure ipi sync event mode */
+			//dw_mipi_csi_write_part(csi_dev, reg.IPI3_ADV_FEATURES, csi_dev->hw.frame_det, 24, 1);
+		}
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI3_HSA_TIME, csi_dev->hw.hsa);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI3_HBP_TIME, csi_dev->hw.hbp);
+		dw_mipi_csi_write(csi_dev,
+					reg.IPI3_HSD_TIME, csi_dev->hw.hsd);
+		dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 8, 1);//bit8 IPI3_soft reset.
+		dev_vdbg(dev, "IPI3 enable: %s\n", csi_dev->hw.ipi3_en ? "YES" : "NO");
+		dev_vdbg(dev, "IPI3 Virtual Channel: %d\n", csi_dev->hw.ipi3_vcid);
+	}
+    dw_mipi_csi_write_part(csi_dev, reg.IPI_SOFTRSTN, 1, 0, 1);
+    udelay(1000);
+	// int ret = v4l2_subdev_call(&csi_dev->sd, video, s_stream, 1);
+	// if (ret) {
+	// 	dev_err(dev, "Failed to start streaming: %d\n", ret);
+	// }
+#ifdef DWC_PHY_USING
+	phy_power_on(csi_dev->phy);
+#endif
+}
+
+int dw_mipi_csi_irq_handler(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+	u32 global_int_status, i_sts;
+	unsigned long flags;
+
+	spin_lock_irqsave(&csi_dev->slock, flags);
+	global_int_status = dw_mipi_csi_read_int_status(csi_dev, reg.INTERRUPT);
+
+	if (global_int_status & csi_int.PHY_FATAL) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_PHY_FATAL);
+		dev_err_ratelimited(dev, "int %08X: PHY FATAL: %08X\n",
+				    reg.INT_PHY_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.PKT_FATAL) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_PKT_FATAL);
+		dev_err_ratelimited(dev, "int %08X: PKT FATAL: %08X\n",
+				    reg.INT_PKT_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.FRAME_FATAL &&
+	    csi_dev->hw_version_major == 1 &&
+	    csi_dev->hw_version_minor == 30) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_FRAME_FATAL);
+		dev_err_ratelimited(dev, "int %08X: FRAME FATAL: %08X\n",
+				    reg.INT_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.PHY) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_PHY);
+		dev_err_ratelimited(dev, "int %08X: PHY: %08X\n",
+				    reg.INT_PHY, i_sts);
+	}
+
+	if (global_int_status & csi_int.PKT &&
+	    csi_dev->hw_version_major == 1 &&
+	    csi_dev->hw_version_minor <= 30) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_PKT);
+		dev_err_ratelimited(dev, "int %08X: PKT: %08X\n",
+				    reg.INT_PKT, i_sts);
+	}
+
+	if (global_int_status & csi_int.LINE) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_LINE);
+		dev_err_ratelimited(dev, "int %08X: LINE: %08X\n",
+				    reg.INT_LINE, i_sts);
+	}
+
+	if (global_int_status & csi_int.IPI) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.INT_IPI);
+		dev_err_ratelimited(dev, "int %08X: IPI: %08X\n",
+				    reg.INT_IPI, i_sts);
+	}
+
+	if (global_int_status & csi_int.BNDRY_FRAME_FATAL) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.ST_BNDRY_FRAME_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_BNDRY_FRAME_FATAL: %08X\n",
+				    reg.ST_BNDRY_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.SEQ_FRAME_FATAL) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.ST_SEQ_FRAME_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_SEQ_FRAME_FATAL: %08X\n",
+				    reg.ST_SEQ_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.CRC_FRAME_FATAL) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.ST_CRC_FRAME_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_CRC_FRAME_FATAL: %08X\n",
+				    reg.ST_CRC_FRAME_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.PLD_CRC_FATAL) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.ST_PLD_CRC_FATAL);
+		dev_err_ratelimited(dev,
+				    "int %08X: ST_PLD_CRC_FATAL: %08X\n",
+				    reg.ST_PLD_CRC_FATAL, i_sts);
+	}
+
+	if (global_int_status & csi_int.DATA_ID) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.ST_DATA_ID);
+		dev_err_ratelimited(dev, "int %08X: ST_DATA_ID: %08X\n",
+				    reg.ST_DATA_ID, i_sts);
+	}
+
+	if (global_int_status & csi_int.ECC_CORRECTED) {
+		i_sts = dw_mipi_csi_read_int_status(csi_dev, reg.ST_ECC_CORRECT);
+		dev_err_ratelimited(dev, "int %08X: ST_ECC_CORRECT: %08X\n",
+				    reg.ST_ECC_CORRECT, i_sts);
+	}
+
+	spin_unlock_irqrestore(&csi_dev->slock, flags);
+
+	return 1;
+}
+
+void dw_mipi_csi_get_version(struct dw_csi *csi_dev)
+{
+	// u32 hw_version;
+
+	// hw_version = dw_mipi_csi_read(csi_dev, reg.VERSION);
+	// csi_dev->hw_version_major = (u8)((hw_version >> 24) - '0');
+	// csi_dev->hw_version_minor = (u8)((hw_version >> 16) - '0');
+	// csi_dev->hw_version_minor = csi_dev->hw_version_minor * 10;
+	// csi_dev->hw_version_minor += (u8)((hw_version >> 8) - '0');
+
+	csi_dev->hw_version_major = 1;
+	csi_dev->hw_version_minor = 40;
+}
+
+int dw_mipi_csi_specific_mappings(struct dw_csi *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+
+	if (csi_dev->hw_version_major == 1) {
+		if (csi_dev->hw_version_minor == 30) {
+			/*
+			 * Hardware registers that were
+			 * exclusive to version < 1.40
+			 */
+			reg.INT_FRAME_FATAL = 0x100;
+			reg.MASK_INT_FRAME_FATAL = 0x104;
+			reg.FORCE_INT_FRAME_FATAL = 0x108;
+			reg.INT_PKT = 0x120;
+			reg.MASK_INT_PKT = 0x124;
+			reg.FORCE_INT_PKT = 0x128;
+
+			/* interrupt source present until this release */
+			csi_int.PKT = BIT(17);
+			csi_int.LINE = BIT(18);
+			csi_int.IPI = BIT(19);
+			csi_int.FRAME_FATAL = BIT(2);
+
+		} else if (csi_dev->hw_version_minor == 40) {
+			/*
+			 * HW registers that were added
+			 * to version 1.40
+			 */
+			reg.ST_BNDRY_FRAME_FATAL = 0x280;
+			reg.MSK_BNDRY_FRAME_FATAL = 0x284;
+			reg.FORCE_BNDRY_FRAME_FATAL = 0x288;
+			reg.ST_SEQ_FRAME_FATAL = 0x290;
+			reg.MSK_SEQ_FRAME_FATAL	= 0x294;
+			reg.FORCE_SEQ_FRAME_FATAL = 0x298;
+			reg.ST_CRC_FRAME_FATAL = 0x2a0;
+			reg.MSK_CRC_FRAME_FATAL	= 0x2a4;
+			reg.FORCE_CRC_FRAME_FATAL = 0x2a8;
+			reg.ST_PLD_CRC_FATAL = 0x2b0;
+			reg.MSK_PLD_CRC_FATAL = 0x2b4;
+			reg.FORCE_PLD_CRC_FATAL = 0x2b8;
+			reg.ST_DATA_ID = 0x2c0;
+			reg.MSK_DATA_ID = 0x2c4;
+			reg.FORCE_DATA_ID = 0x2c8;
+			reg.ST_ECC_CORRECT = 0x2d0;
+			reg.MSK_ECC_CORRECT = 0x2d4;
+			reg.FORCE_ECC_CORRECT = 0x2d8;
+			reg.DATA_IDS_VC_1 = 0x0;
+			reg.DATA_IDS_VC_2 = 0x0;
+			reg.VC_EXTENSION = 0x0;
+
+			/* interrupts map were changed */
+			csi_int.LINE = BIT(17);
+			csi_int.IPI = BIT(18);
+			csi_int.BNDRY_FRAME_FATAL = BIT(2);
+			csi_int.SEQ_FRAME_FATAL	= BIT(3);
+			csi_int.CRC_FRAME_FATAL = BIT(4);
+			csi_int.PLD_CRC_FATAL = BIT(5);
+			csi_int.DATA_ID = BIT(6);
+			csi_int.ECC_CORRECTED = BIT(7);
+
+		} else {
+			dev_info(dev, "Version minor not supported.");
+		}
+	} else {
+		dev_info(dev, "Version major not supported.");
+	}
+	return 0;
+}
+
+void dw_mipi_csi_dump(struct dw_csi *csi_dev)
+{
+	dw_print(reg.VERSION);
+	dw_print(reg.N_LANES);
+	dw_print(reg.CTRL_RESETN);
+	dw_print(reg.INTERRUPT);
+	dw_print(reg.DATA_IDS_1);
+	dw_print(reg.DATA_IDS_2);
+	dw_print(reg.IPI_MODE);
+	dw_print(reg.IPI_VCID);
+	dw_print(reg.IPI_DATA_TYPE);
+	dw_print(reg.IPI_MEM_FLUSH);
+	dw_print(reg.IPI_HSA_TIME);
+	dw_print(reg.IPI_HBP_TIME);
+	dw_print(reg.IPI_HSD_TIME);
+	dw_print(reg.IPI_HLINE_TIME);
+	dw_print(reg.IPI_SOFTRSTN);
+	dw_print(reg.IPI_ADV_FEATURES);
+	dw_print(reg.IPI_VSA_LINES);
+	dw_print(reg.IPI_VBP_LINES);
+	dw_print(reg.IPI_VFP_LINES);
+	dw_print(reg.IPI_VACTIVE_LINES);
+	dw_print(reg.IPI_DATA_TYPE);
+	dw_print(reg.VERSION);
+	dw_print(reg.IPI_ADV_FEATURES);
+}
diff --git a/drivers/media/platform/eswin/mipi-csi2/dw-mipi-csi.h b/drivers/media/platform/eswin/mipi-csi2/dw-mipi-csi.h
new file mode 100755
index 000000000000..e041c739519f
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/dw-mipi-csi.h
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ *
+ * Synopsys DesignWare MIPI CSI-2 Host controller driver
+ *
+ * Author: Luis Oliveira <Luis.Oliveira@synopsys.com>
+ */
+
+#ifndef _DW_MIPI_CSI_H__
+#define _DW_MIPI_CSI_H__
+
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/phy/phy.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/ratelimit.h>
+#include <linux/reset.h>
+#include <linux/videodev2.h>
+#include <linux/wait.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/eswin/dw-mipi-csi-pltfrm.h>
+
+// #define DWC_PHY_USING
+#define REG_DUMMY_READ
+
+/* Advanced features */
+#define IPI_DT_OVERWRITE BIT(0)
+#define DATA_TYPE_OVERWRITE(dt) (((dt) & GENMASK(5, 0)) << 8)
+#define LINE_EVENT_SELECTION(n) ((n) << 16)
+
+enum line_event {
+	EVSELAUTO = 0,
+	EVSELPROG = 1,
+};
+
+#define EN_VIDEO BIT(17)
+#define EN_LINE_START BIT(18)
+#define EN_NULL BIT(19)
+#define EN_BLANKING BIT(20)
+#define EN_EMBEDDED BIT(21)
+#define IPI_SYNC_EVENT_MODE(n) ((n) << 24)
+
+enum sync_event {
+	SYNCEVFSN = 0,
+	SYNCEVFS = 1,
+};
+
+/* DW MIPI CSI-2 register addresses*/
+
+struct R_CSI2 {
+	u16 VERSION;
+	u16 N_LANES;
+	u16 CTRL_RESETN;
+	u16 INTERRUPT;
+	u16 DATA_IDS_1;
+	u16 DATA_IDS_2;
+	u16 DATA_IDS_VC_1;
+	u16 DATA_IDS_VC_2;
+	u16 PHY_CFG;
+	u16 PHY_MODE;
+	u16 AP_INTERRUPT;
+	u16 IPI_MODE;
+	u16 IPI_VCID;
+	u16 IPI_DATA_TYPE;
+	u16 IPI_MEM_FLUSH;
+	u16 IPI_HSA_TIME;
+	u16 IPI_HBP_TIME;
+	u16 IPI_HSD_TIME;
+	u16 IPI_HLINE_TIME;
+	u16 IPI_SOFTRSTN;
+	u16 IPI_ADV_FEATURES;
+	u16 IPI_VSA_LINES;
+	u16 IPI_VBP_LINES;
+	u16 IPI_VFP_LINES;
+	u16 IPI_VACTIVE_LINES;
+	u16 IPI2_MODE;
+	u16 IPI2_VCID;
+	u16 IPI2_DATA_TYPE;
+	u16 IPI2_MEM_FLUSH;
+	u16 IPI2_HSA_TIME;
+	u16 IPI2_HBP_TIME;
+	u16 IPI2_HSD_TIME;
+	u16 IPI2_ADV_FEATURES;
+	u16 MASK_INT_IPI2;
+	u16 MASK_INT_AP_IPI2;
+	u16 IPI3_MODE;
+	u16 IPI3_VCID;
+	u16 IPI3_DATA_TYPE;
+	u16 IPI3_MEM_FLUSH;
+	u16 IPI3_HSA_TIME;
+	u16 IPI3_HBP_TIME;
+	u16 IPI3_HSD_TIME;
+	u16 IPI3_ADV_FEATURES;
+	u16 MASK_INT_IPI3;
+	u16 MASK_INT_AP_IPI3;
+	u16 VC_EXTENSION;
+	u16 INT_PHY_FATAL;
+	u16 MASK_INT_PHY_FATAL;
+	u16 FORCE_INT_PHY_FATAL;
+	u16 INT_PKT_FATAL;
+	u16 MASK_INT_PKT_FATAL;
+	u16 FORCE_INT_PKT_FATAL;
+	u16 INT_FRAME_FATAL;
+	u16 MASK_INT_FRAME_FATAL;
+	u16 FORCE_INT_FRAME_FATAL;
+	u16 INT_PHY;
+	u16 MASK_INT_PHY;
+	u16 FORCE_INT_PHY;
+	u16 INT_PKT;
+	u16 MASK_INT_PKT;
+	u16 FORCE_INT_PKT;
+	u16 INT_LINE;
+	u16 MASK_INT_LINE;
+	u16 FORCE_INT_LINE;
+	u16 INT_IPI;
+	u16 MASK_INT_IPI;
+	u16 FORCE_INT_IPI;
+	u16 ST_BNDRY_FRAME_FATAL;
+	u16 MSK_BNDRY_FRAME_FATAL;
+	u16 FORCE_BNDRY_FRAME_FATAL;
+	u16 ST_SEQ_FRAME_FATAL;
+	u16 MSK_SEQ_FRAME_FATAL;
+	u16 FORCE_SEQ_FRAME_FATAL;
+	u16 ST_CRC_FRAME_FATAL;
+	u16 MSK_CRC_FRAME_FATAL;
+	u16 FORCE_CRC_FRAME_FATAL;
+	u16 ST_PLD_CRC_FATAL;
+	u16 MSK_PLD_CRC_FATAL;
+	u16 FORCE_PLD_CRC_FATAL;
+	u16 ST_DATA_ID;
+	u16 MSK_DATA_ID;
+	u16 FORCE_DATA_ID;
+	u16 ST_ECC_CORRECT;
+	u16 MSK_ECC_CORRECT;
+	u16 FORCE_ECC_CORRECT;
+};
+
+/* Interrupt Masks */
+struct interrupt_type {
+	u32 PHY_FATAL;
+	u32 PKT_FATAL;
+	u32 FRAME_FATAL;
+	u32 PHY;
+	u32 PKT;
+	u32 LINE;
+	u32 IPI;
+	u32 BNDRY_FRAME_FATAL;
+	u32 SEQ_FRAME_FATAL;
+	u32 CRC_FRAME_FATAL;
+	u32 PLD_CRC_FATAL;
+	u32 DATA_ID;
+	u32 ECC_CORRECTED;
+};
+
+/* IPI Data Types */
+enum data_type {
+	CSI_2_YUV420_8 = 0x18,
+	CSI_2_YUV420_10 = 0x19,
+	CSI_2_YUV420_8_LEG = 0x1A,
+	CSI_2_YUV420_8_SHIFT = 0x1C,
+	CSI_2_YUV420_10_SHIFT = 0x1D,
+	CSI_2_YUV422_8 = 0x1E,
+	CSI_2_YUV422_10 = 0x1F,
+	CSI_2_RGB444 = 0x20,
+	CSI_2_RGB555 = 0x21,
+	CSI_2_RGB565 = 0x22,
+	CSI_2_RGB666 = 0x23,
+	CSI_2_RGB888 = 0x24,
+	CSI_2_RAW6 = 0x28,
+	CSI_2_RAW7 = 0x29,
+	CSI_2_RAW8 = 0x2A,
+	CSI_2_RAW10 = 0x2B,
+	CSI_2_RAW12 = 0x2C,
+	CSI_2_RAW14 = 0x2D,
+	CSI_2_RAW16 = 0x2E,
+	CSI_2_RAW20 = 0x2F,
+	USER_DEFINED_1 = 0x30,
+	USER_DEFINED_2 = 0x31,
+	USER_DEFINED_3 = 0x32,
+	USER_DEFINED_4 = 0x33,
+	USER_DEFINED_5 = 0x34,
+	USER_DEFINED_6 = 0x35,
+	USER_DEFINED_7 = 0x36,
+	USER_DEFINED_8 = 0x37,
+};
+
+/* DWC MIPI CSI-2 output types */
+enum output {
+	IPI_OUT = 0,
+	IDI_OUT = 1,
+	BOTH_OUT = 2
+};
+
+/* IPI color components */
+enum color_mode {
+	COLOR48 = 0,
+	COLOR16 = 1
+};
+
+/* IPI cut through */
+enum cut_through {
+	CTINACTIVE = 0,
+	CTACTIVE = 1
+};
+
+/* IPI output types */
+enum ipi_output {
+	CAMERA_TIMING = 0,
+	AUTO_TIMING = 1
+};
+
+/* Format template */
+struct mipi_fmt {
+	u32 mbus_code;
+	u8 depth;
+};
+
+/* CSI specific configuration */
+struct csi_data {
+	u32 num_lanes;
+	u32 dphy_freq;
+	u32 phy_mode;
+	u32 ppi_width;
+	u32 pclk;
+	u32 fps;
+	u32 bpp;
+	u32 output;
+
+	u32 ipi_dt;
+	u32 ipi_emb;
+	u32 ipi_mode;
+	u32 ipi_cut_through;
+	u32 ipi_color_mode;
+	u32 ipi_auto_flush;
+	u32 ipi_line_event;
+	u32 ipi_vcid;
+	u32 frame_det;
+
+	u32 ipi2_en;
+	u32 ipi2_vcid; 
+	u32 ipi3_en;
+	u32 ipi3_vcid;
+
+	u32 hsa;
+	u32 hbp;
+	u32 hsd;
+	u32 htotal;
+	u32 vsa;
+	u32 vbp;
+	u32 vfp;
+	u32 vactive;
+};
+#define CSI2_NUM_PADS_MAX 12
+#define CSI2_NUM_SRC_PADS 4
+#define MAX_CSI2_SENSORS 2
+
+enum csi2_pads {
+	DWC_CSI2_PAD_SINK = 0,
+	DWC_CSI2X_PAD_SOURCE0,
+	DWC_CSI2X_PAD_SOURCE1,
+	DWC_CSI2X_PAD_SOURCE2,
+	DWC_CSI2X_PAD_SOURCE3
+};
+
+struct csi2_sensor_info {
+	struct v4l2_subdev *sd;
+	struct v4l2_mbus_config mbus;
+	int lanes;
+};
+/* Structure to embed device driver information */
+struct dw_csi {
+	struct v4l2_subdev sd;
+	struct device *dev;
+	struct media_pad pads[CSI_PADS_NUM];
+	struct mipi_fmt *fmt;
+	struct v4l2_mbus_framefmt format;
+	struct v4l2_device    v4l2_dev;
+	struct v4l2_async_notifier	notifier;
+
+	void __iomem *base_address;
+	int ctrl_irq_number;
+	struct csi_data hw;
+	struct reset_control *rst;
+#ifdef DWC_PHY_USING
+	struct phy *phy;
+#endif
+	struct dw_csih_pdata *config;
+	struct mutex lock; /* protect resources sharing */
+	spinlock_t slock; /* interrupt handling lock */
+
+	u32 index;
+	u8 hw_version_major;
+	u16 hw_version_minor;
+
+	bool sink_linked[CSI2_NUM_SRC_PADS];
+	struct csi2_sensor_info	sensors[MAX_CSI2_SENSORS];
+	int			num_sensors;
+	struct v4l2_mbus_config_mipi_csi2	bus;
+	struct v4l2_mbus_framefmt	format_mbus;
+	struct v4l2_rect	crop;
+
+
+};
+
+static inline struct dw_csi *sd_to_mipi_csi_dev(struct v4l2_subdev *sdev)
+{
+	return container_of(sdev, struct dw_csi, sd);
+}
+
+void dw_mipi_csi_reset(struct dw_csi *csi_dev);
+int dw_mipi_csi_mask_irq_power_off(struct dw_csi *csi_dev);
+int dw_mipi_csi_hw_stdby(struct dw_csi *csi_dev);
+void dw_mipi_csi_set_ipi_fmt(struct dw_csi *csi_dev);
+void dw_mipi_csi_start(struct dw_csi *csi_dev);
+int dw_mipi_csi_irq_handler(struct dw_csi *csi_dev);
+void dw_mipi_csi_get_version(struct dw_csi *csi_dev);
+int dw_mipi_csi_specific_mappings(struct dw_csi *csi_dev);
+void dw_mipi_csi_fill_timings(struct dw_csi *dev);
+void dw_mipi_csi_dump(struct dw_csi *csi_dev);
+
+#if IS_ENABLED(CONFIG_DWC_MIPI_TC_DPHY_GEN3)
+int dw_csi_create_capabilities_sysfs(struct platform_device *pdev);
+#endif
+
+static inline void dw_mipi_csi_write(struct dw_csi *dev,
+				     u32 address, u32 data)
+{
+	dev_dbg(dev->dev, "%s: [%08x]: %08x\n", __func__, address, data);
+	writel(data, dev->base_address + address);
+}
+
+static inline u32 dw_mipi_csi_read(struct dw_csi *dev, u32 address)
+{
+    u32 ret =0;
+
+#ifdef REG_DUMMY_READ
+	readl(dev->base_address + address);
+#endif
+	ret = readl(dev->base_address + address);
+    
+    /*当前的haps平台下axi总线上csi读取时延大，需要多读取几次保证读取到正确值*/
+	ret = readl(dev->base_address + address);
+	ret = readl(dev->base_address + address);
+	ret = readl(dev->base_address + address);
+	ret = readl(dev->base_address + address);
+	dev_dbg(dev->dev, "%s: [%08x]: %08x\n", __func__, address, ret);
+    return ret;
+}
+
+static inline u32 dw_mipi_csi_read_int_status(struct dw_csi *dev, u32 address)
+{
+    u32 ret =0;
+	ret = readl(dev->base_address + address);
+	dev_dbg_ratelimited(dev->dev, "%s: [%08x]: %08x\n", __func__, address, ret);
+    return ret;
+}
+
+
+#endif /*_DW_MIPI_CSI_H__ */
diff --git a/drivers/media/platform/eswin/mipi-csi2/eswin-mipi-dma.c b/drivers/media/platform/eswin/mipi-csi2/eswin-mipi-dma.c
new file mode 100755
index 000000000000..8755e2b5f552
--- /dev/null
+++ b/drivers/media/platform/eswin/mipi-csi2/eswin-mipi-dma.c
@@ -0,0 +1,270 @@
+/*
+dvp2axi driver
+*/
+
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+
+#include <media/v4l2-device.h>
+#include <media/v4l2-dev.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-async.h>
+#include <linux/dev_printk.h>
+
+struct csi_mipi_dma {
+	struct device *dev;
+
+	struct v4l2_device v4l2_dev;
+	struct media_device media_dev;
+
+	struct v4l2_async_notifier notifier;
+
+	struct video_device video;
+	struct media_pad pad;
+
+	struct vb2_queue queue;
+
+	struct mutex lock;
+};
+
+struct csi_dma_buffer {
+	struct vb2_v4l2_buffer buf;
+	struct list_head queue;
+	struct csi_mipi_dma *dma;
+};
+
+static int csi_dma_fwnode_parse(struct device *dev,
+				struct v4l2_fwnode_endpoint *vep,
+				struct v4l2_async_subdev *asd)
+{
+	switch (vep->bus_type) {
+	case V4L2_MBUS_CSI2_DPHY:
+		return 0;
+	default:
+		dev_err(dev, "Unsupported media bus type\n");
+		return -ENOTCONN;
+	}
+}
+
+static int csi_dma_graph_notify_complete(struct v4l2_async_notifier *notifier)
+{
+	int ret;
+	struct csi_mipi_dma *dma =
+		container_of(notifier, struct csi_mipi_dma, notifier);
+
+	ret = v4l2_device_register_subdev_nodes(&dma->v4l2_dev);/*create all subdev node*/
+	if (ret < 0)
+		dev_err(dma->dev, "failed to register subdev nodes\n");
+
+	return media_device_register(&dma->media_dev);
+}
+
+static int csi_dma_graph_notify_bound(struct v4l2_async_notifier *notifier,
+				      struct v4l2_subdev *subdev,
+				      struct v4l2_async_subdev *unused)
+{
+	printk("%s %d\n", __func__, __LINE__);
+	return 0;
+}
+
+static const struct v4l2_async_notifier_operations csi_dma_graph_notify_ops = {
+	.bound = csi_dma_graph_notify_bound,
+	.complete = csi_dma_graph_notify_complete,
+};
+
+static int csi_dma_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
+			       unsigned int *nplanes, unsigned int sizes[],
+			       struct device *alloc_devs[])
+{
+	return 0;
+}
+
+static void csi_dma_buffer_queue(struct vb2_buffer *vb)
+{
+}
+
+static const struct vb2_ops csi_dma_queue_qops = {
+	.queue_setup = csi_dma_queue_setup,
+	.buf_queue = csi_dma_buffer_queue,
+};
+
+static const struct v4l2_ioctl_ops csi_dma_ioctl_ops = {
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+};
+
+static const struct v4l2_file_operations csi_dma_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = video_ioctl2,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.poll = vb2_fop_poll,
+	.mmap = vb2_fop_mmap,
+};
+
+static int csi_composite_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct csi_mipi_dma *csi_dma;
+	struct v4l2_async_subdev *asd;
+
+	int ret;
+
+	dev_info(dev, "Probing started\n");
+
+	csi_dma = devm_kzalloc(dev, sizeof(*csi_dma), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	csi_dma->dev = dev;
+	mutex_init(&csi_dma->lock);
+
+	/* media device init */
+	csi_dma->media_dev.dev = dev;
+	strscpy(csi_dma->media_dev.model, "eswin MIPI CSI DMA", sizeof(csi_dma->media_dev.model));
+	csi_dma->media_dev.hw_revision = 0;
+	media_device_init(&csi_dma->media_dev);
+
+	/* v4l2 device register */
+	csi_dma->v4l2_dev.mdev = &csi_dma->media_dev;
+	ret = v4l2_device_register(dev, &csi_dma->v4l2_dev);
+
+	if (ret < 0) {
+		dev_err(dev, "V4L2 device registration failed (%d)\n", ret);
+		media_device_cleanup(&csi_dma->media_dev);
+		return ret;
+	}
+
+	/* video device init */
+	csi_dma->video.fops = &csi_dma_fops;
+	csi_dma->video.v4l2_dev = &csi_dma->v4l2_dev;
+	csi_dma->video.queue = &csi_dma->queue;
+	snprintf(csi_dma->video.name, sizeof(csi_dma->video.name), "eswin mipi csi dma");
+
+	csi_dma->video.vfl_type = VFL_TYPE_VIDEO;
+	csi_dma->video.vfl_dir = VFL_DIR_RX;
+	csi_dma->video.release = video_device_release_empty;
+	csi_dma->video.ioctl_ops = &csi_dma_ioctl_ops;
+	csi_dma->video.lock = &csi_dma->lock;
+	csi_dma->video.device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
+
+	video_set_drvdata(&csi_dma->video, csi_dma);
+
+	/* init vb queue */
+	csi_dma->queue.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	csi_dma->queue.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
+	csi_dma->queue.lock = &csi_dma->lock;
+	csi_dma->queue.drv_priv = csi_dma;
+	csi_dma->queue.buf_struct_size = sizeof(struct csi_dma_buffer);
+	csi_dma->queue.ops = &csi_dma_queue_qops;
+	csi_dma->queue.mem_ops = &vb2_dma_contig_memops;
+	csi_dma->queue.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC |
+					 V4L2_BUF_FLAG_TSTAMP_SRC_EOF;
+	csi_dma->queue.dev = dev;
+
+	ret = vb2_queue_init(&csi_dma->queue);
+	if (ret < 0) {
+		dev_err(dev, "failed to initialize VB2 queue\n");
+		goto clean_out;
+	}
+
+	/* video device register */
+	ret = video_register_device(&csi_dma->video, VFL_TYPE_VIDEO, -1);
+	if (ret < 0) {
+		dev_err(dev, "failed to register video device\n");
+		goto clean_out;
+	}
+
+	/* Register the subdevices notifier. */
+	v4l2_async_nf_init(&csi_dma->notifier);
+
+
+    /*get the first fwnode*/
+	ret = v4l2_async_nf_parse_fwnode_endpoints(
+		csi_dma->dev, &csi_dma->notifier,
+		sizeof(struct v4l2_async_subdev), csi_dma_fwnode_parse);
+	if (ret)
+		goto clean_out;
+
+    /* get every fwnode on device tree with "remote-endpoint" field,
+     * and and it to notifier list
+     */
+	list_for_each_entry (asd, &csi_dma->notifier.asd_list, asd_list) {
+		if (asd->match.fwnode->dev == dev)
+			continue;
+		ret = v4l2_async_nf_parse_fwnode_endpoints(
+			asd->match.fwnode->dev, &csi_dma->notifier,
+			sizeof(struct v4l2_async_subdev), csi_dma_fwnode_parse);
+		if (ret)
+			goto clean_out;
+	}
+
+	csi_dma->notifier.ops = &csi_dma_graph_notify_ops;
+	ret = v4l2_async_nf_register(&csi_dma->v4l2_dev, &csi_dma->notifier);
+
+	if (ret < 0) {
+		dev_err(dev, "notifier registration failed\n");
+		goto clean_out;
+	}
+
+	platform_set_drvdata(pdev, csi_dma);
+
+	dev_info(dev, "Probing End\n");
+
+	return 0;
+
+clean_out:
+	v4l2_device_unregister(&csi_dma->v4l2_dev);
+    v4l2_async_nf_cleanup(&csi_dma->notifier);
+	media_device_unregister(&csi_dma->media_dev);
+	media_device_cleanup(&csi_dma->media_dev);
+
+	return ret;
+}
+
+static int csi_composite_remove(struct platform_device *pdev)
+{
+	struct csi_mipi_dma *csi_dma = platform_get_drvdata(pdev);
+
+    video_unregister_device(&csi_dma->video);
+    v4l2_async_nf_unregister(&csi_dma->notifier);
+    v4l2_async_nf_cleanup(&csi_dma->notifier);
+	v4l2_device_unregister(&csi_dma->v4l2_dev);
+	media_device_unregister(&csi_dma->media_dev);
+	media_device_cleanup(&csi_dma->media_dev);
+
+	return 0;
+}
+
+static const struct of_device_id eswin_composite_of_id_table[] = {
+	{ .compatible = "eswin,csi-video" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, eswin_composite_of_id_table);
+
+static struct platform_driver eswin_composite_driver = {
+	.driver = {
+		.name = "eswin-csi-video",
+		.of_match_table = eswin_composite_of_id_table,
+	},
+	.probe = csi_composite_probe,
+	.remove = csi_composite_remove,
+};
+
+module_platform_driver(eswin_composite_driver);
+
+MODULE_AUTHOR("Y o n g Y a n g, <yangyong@eswin.com>");
+MODULE_DESCRIPTION("A MIPI DMA CONTROLLER");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/eswin/vi_top/Kconfig b/drivers/media/platform/eswin/vi_top/Kconfig
new file mode 100644
index 000000000000..092b8515bfee
--- /dev/null
+++ b/drivers/media/platform/eswin/vi_top/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Configuration for the eswin vi_top driver
+#
+
+config VIDEO_ESWIN_VI_TOP
+    tristate "ESWIN VI_TOP driver support"
+    default y
+    depends on VIDEO_DEV
+    select VIDEOBUF2_CORE
+    help
+      Enable support for the ESWIN VI_TOP video interface driver.
+      This driver is used for handling video input from ESWIN devices.
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/vi_top/Makefile b/drivers/media/platform/eswin/vi_top/Makefile
new file mode 100644
index 000000000000..b2491689b4c8
--- /dev/null
+++ b/drivers/media/platform/eswin/vi_top/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VIDEO_ESWIN_VI_TOP) += es_vi_top.o
+es_vi_top-objs += eswin_vi.o
\ No newline at end of file
diff --git a/drivers/media/platform/eswin/vi_top/eswin_vi.c b/drivers/media/platform/eswin/vi_top/eswin_vi.c
new file mode 100644
index 000000000000..cbf611921d6c
--- /dev/null
+++ b/drivers/media/platform/eswin/vi_top/eswin_vi.c
@@ -0,0 +1,655 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN vi top driver
+ *
+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Eswin VI team
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/uaccess.h>
+#include <linux/media.h>
+#include <linux/platform_device.h>
+#include <linux/mdev.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+
+#include <media/eswin/eswin_vi.h>
+#include <media/eswin/common-def.h>
+
+#include "vitop.h"
+#include <linux/eswin-win2030-sid-cfg.h>
+
+#define DRIVER_NAME "eswin-vi"
+#define DEVICE_NAME "eswin_vi"
+#define CLASS_NAME "eswin_vi_class"
+
+static int major_number;
+
+#define ES_VITOP_REG_OFFSET 0x30000
+#define ES_VITOP_WRITE_REG(base, offset, val) writel(val, base + offset+ES_VITOP_REG_OFFSET)
+#define ES_VITOP_READ_REG(base, offset) readl(base + offset+ES_VITOP_REG_OFFSET)
+
+#define eswin_vi_DW_CLK_HIGHEST 594000000
+#define eswin_vi_AXI_CLK_HIGHEST 800000000
+
+#if defined(__GNUC__) || defined(__clang__)
+    #define UNUSED_FUNC __attribute__((unused))
+#else
+    #define UNUSED_FUNC
+#endif
+
+#define CHECK_COPY_RETVAL(retval) do { \
+	if (retval < 0) { \
+		pr_err("%s %d: copy from or to user failed\n", __func__, __LINE__); \
+		return -EFAULT; \
+	} \
+} while(0);
+
+
+
+static int viscu_cfg(struct eswin_vi_device *es_vi_dev)
+{
+	struct regmap *regmap = es_vi_dev->syscrg_regmap;
+    
+	regmap_write(regmap, VI_SUBSYSTEM_SCU_ISP_RESET, 0x0);///isp reset(sys_crg)
+	pr_debug("SCU: ISP Reset ...\n");
+	regmap_write(regmap, VI_SUBSYSTEM_SCU_ISP_RESET, 0x1);
+
+	// CSI controller 0 HDR mode
+	pr_debug("SCU: HDR Disabled ...\n");
+    // regmap_write(regmap, VI_SUBSYSTEM_SCU_SONY_HDR_MODE, 0x0);///i2c_rst_ctl(sys_crg)
+	return 0;
+}
+
+static void vi_top_register_write(struct eswin_vi_device *es_vi_dev, u32 reg, u32 val)
+{
+    writel(val, es_vi_dev->base + reg);
+}
+static u32 vi_top_register_read(struct eswin_vi_device *es_vi_dev, u32 reg)
+{
+    return readl(es_vi_dev->base + reg);
+}
+
+UNUSED_FUNC static void syscrg_register_write(struct eswin_vi_device *es_vi_dev, u32 reg, u32 val)
+{
+    u32 reg_value;
+    pr_debug("%s reg:0x%x, val:0x%x", __func__, reg, val);
+    regmap_write(es_vi_dev->syscrg_regmap, reg, val);
+    regmap_read(es_vi_dev->syscrg_regmap, reg, &reg_value);
+    pr_debug("read reg:0x%x, val:0x%x", reg, reg_value);
+}
+UNUSED_FUNC static u32 syscrg_register_read(struct eswin_vi_device *es_vi_dev, u32 reg)
+{
+    u32 reg_value;
+    regmap_read(es_vi_dev->syscrg_regmap, reg, &reg_value);
+    pr_debug("%s reg:0x%x, val:0x%x", __func__, reg, reg_value);
+    return reg_value;
+}
+
+static int vitop_intf_cfg(struct eswin_vi_device *es_vi_dev)
+{
+	u32 val = 0;
+	unsigned int reg_value;
+
+    pr_debug("ISP Top Setting ...\n");
+    
+    vi_top_register_write(es_vi_dev, VI_TOP_PHY_CONNECT_MODE, es_vi_dev->phy_mode);
+
+    vi_top_register_write(es_vi_dev, VI_TOP_CONTROLLER_SELECT, 0);
+	
+	val = (CSI_CONTROLLER_ID << VI_TOP_ISP0_DVP0_SEL_OFFSET) | (CSI_CONTROLLER_ID << VI_TOP_ISP0_DVP1_SEL_OFFSET);
+    val |= (CSI_CONTROLLER_ID << VI_TOP_ISP0_DVP2_SEL_OFFSET) | (CSI_CONTROLLER_ID << VI_TOP_ISP0_DVP3_SEL_OFFSET);
+    val |= (CSI_CONTROLLER_ID << VI_TOP_ISP1_DVP0_SEL_OFFSET) | (CSI_CONTROLLER_ID << VI_TOP_ISP1_DVP1_SEL_OFFSET);
+    val |= (CSI_CONTROLLER_ID << VI_TOP_ISP1_DVP2_SEL_OFFSET) | (CSI_CONTROLLER_ID << VI_TOP_ISP1_DVP3_SEL_OFFSET);
+
+    vi_top_register_write(es_vi_dev, VI_TOP_ISP_DVP_SEL, val);
+    vi_top_register_write(es_vi_dev, VI_TOP_ISP0_DVP0_SIZE, (es_vi_dev->isp_dvp0_ver << 16) | es_vi_dev->isp_dvp0_hor);
+    vi_top_register_write(es_vi_dev, VI_TOP_ISP0_DVP1_SIZE, (es_vi_dev->isp_dvp0_ver << 16) | es_vi_dev->isp_dvp0_hor);
+    vi_top_register_write(es_vi_dev, VI_TOP_ISP1_DVP0_SIZE, (es_vi_dev->isp_dvp0_ver << 16) | es_vi_dev->isp_dvp0_hor);
+    vi_top_register_write(es_vi_dev, VI_TOP_ISP1_DVP1_SIZE, (es_vi_dev->isp_dvp0_ver << 16) | es_vi_dev->isp_dvp0_hor);
+    vi_top_register_write(es_vi_dev, VI_TOP_MULTI2ISP_BLANK, (0xff << 8) | 0xff);
+    vi_top_register_write(es_vi_dev, VI_TOP_MULTI2ISP0_DVP0, (4 << 9));
+    vi_top_register_write(es_vi_dev, VI_TOP_MULTI2ISP0_DVP1, (4 << 9));
+    vi_top_register_write(es_vi_dev, VI_TOP_MULTI2ISP1_DVP0, (4 << 9));
+    vi_top_register_write(es_vi_dev, VI_TOP_MULTI2ISP1_DVP1, (4 << 9));
+
+
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_PHY_CONNECT_MODE);
+    pr_debug("t2 VI_TOP_PHY_CONNECT_MODE[0x51030000] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_ISP_DVP_SEL);
+    pr_debug("ISP0_DVP_SEL[0x51030008] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_ISP0_DVP0_SIZE);
+    pr_debug("ISP0_DVP0 size[0x5103000c] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_ISP0_DVP1_SIZE);
+    pr_debug("ISP0_DVP1 size[0x51030010] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_ISP1_DVP0_SIZE);
+    pr_debug("ISP1_DVP0 size[0x51030014] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_ISP1_DVP1_SIZE);
+    pr_debug("ISP1_DVP1 size[0x51030018] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_MULTI2ISP_BLANK);
+    pr_debug("ISP_MUL2ISP_BLANK[0x5103001c] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_MULTI2ISP0_DVP0);
+    pr_debug("ISP0_DVP0 multi[0x51030020] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_MULTI2ISP0_DVP1);
+    pr_debug("ISP0_DVP1 multi[0x51030024] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_MULTI2ISP1_DVP0);
+    pr_debug("ISP1_DVP0 multi[0x51030028] = %x\n", reg_value);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_MULTI2ISP1_DVP1);
+    pr_debug("ISP1_DVP1 multi[0x5103002c] = %x\n", reg_value);
+
+	return 0;
+}
+
+
+static int eic770x_vi_init(struct eswin_vi_device *es_vi_dev)
+{
+    struct eswin_vi_device *regmap = es_vi_dev;
+    unsigned int reg_value;
+
+    syscrg_register_write(regmap, 0x200, 0xffffffff);///lsp_clk_en0 enable(sys_crg)
+    syscrg_register_write(regmap, 0x424, 0x3ff);///i2c_rst_ctl(sys_crg)
+    
+    udelay(1000);
+
+
+    syscrg_register_write(regmap, 0x470, 0x7);///vi_rst_ctl
+    syscrg_register_write(regmap, 0x474, 0x1);///dvp_rst_ctl
+    syscrg_register_write(regmap, 0x478, 0x1);///isp0_rst_ctl
+    syscrg_register_write(regmap, 0x47c, 0x1);///isp1_rst_ctl
+    syscrg_register_write(regmap, 0x480, 0x1);///shutter_rst_ctl
+    udelay(20000);
+
+
+    syscrg_register_write(regmap, 0x184, 0x80000020);///vi_dwclk_ctl
+    syscrg_register_write(regmap, 0x188, 0xc0000020);///vi_aclk_ctl
+    syscrg_register_write(regmap, 0x18c, 0x80000020);///vi_dig_isp_clk_ctl
+
+    if(es_vi_dev->board_compat == EIC7700_EVB_VI_COMPAT)
+        syscrg_register_write(regmap, 0x190, 0x80000020);///vi_dvp_clk_ctl
+	else
+        syscrg_register_write(regmap, 0x190, 0x80000020);///vi_dvp_clk_ctl
+    
+    if(es_vi_dev->board_compat == EIC7700_EVB_VI_COMPAT) {
+        syscrg_register_write(regmap, 0x194, 0x80000100);///vi_shutter0
+        syscrg_register_write(regmap, 0x198, 0x80000100);///vi_shutter1
+    } else {
+	    syscrg_register_write(regmap, 0x194, 0x80000180);///vi_shutter0
+	    syscrg_register_write(regmap, 0x198, 0x80000180);///vi_shutter1
+	}
+
+    syscrg_register_write(regmap, 0x19c, 0x80000100);///vi_shutter2
+    syscrg_register_write(regmap, 0x1a0, 0x80000100);///vi_shutter3
+    syscrg_register_write(regmap, 0x1a4, 0x80000100);///vi_shutter4
+    syscrg_register_write(regmap, 0x1a8, 0x80000100);///vi_shutter5
+    syscrg_register_write(regmap, 0x1ac, 0x3);///vi_phy_clk_ctl
+
+    // Enable Clocks from TOP CSR
+    vi_top_register_write(es_vi_dev, VI_TOP_CLOCK_ENABLE, 0xffffffff);
+    reg_value = vi_top_register_read(es_vi_dev, VI_TOP_CLOCK_ENABLE);
+    pr_debug("ISP_TOP_CLOCK_EN[0x51030040] = %x\n", reg_value);
+    
+    udelay(200000);
+
+	viscu_cfg(es_vi_dev);///isp_rst(sys_crg)
+
+    vitop_intf_cfg(es_vi_dev);///(vi_top_cfg)
+   
+    
+    return 0;
+}
+
+
+
+UNUSED_FUNC static int eswin_vi_sys_reset_init(struct platform_device *pdev,
+				struct eswin_vi_clk_rst *vi_crg)
+{
+	vi_crg->rstc_axi = devm_reset_control_get_shared(&pdev->dev, "axi");
+	if (IS_ERR_OR_NULL(vi_crg->rstc_axi)) {
+		dev_err(&pdev->dev, "Failed to get vi axi reset handle\n");
+		return -EFAULT;
+	}
+
+	vi_crg->rstc_cfg = devm_reset_control_get_shared(&pdev->dev, "cfg");
+	if (IS_ERR_OR_NULL(vi_crg->rstc_cfg)) {
+		dev_err(&pdev->dev, "Failed to get vi cfg reset handle\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+#define eswin_vi_CLK_GET_HANDLE(dev, clk_handle, clk_name)                     \
+	{                                                                   \
+		clk_handle = devm_clk_get(dev, clk_name);                   \
+		if (IS_ERR(clk_handle)) {                                   \
+			ret = PTR_ERR(clk_handle);                          \
+			dev_err(dev, "failed to get dw %s: %d\n", clk_name, \
+				ret);                                       \
+			return ret;                                         \
+		}                                                           \
+	}
+
+UNUSED_FUNC static int eswin_vi_sys_clk_init(struct platform_device *pdev,
+			      struct eswin_vi_clk_rst *vi_crg)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+
+	eswin_vi_CLK_GET_HANDLE(dev, vi_crg->aclk, "aclk");
+	eswin_vi_CLK_GET_HANDLE(dev, vi_crg->cfg_clk, "cfg_clk");
+	eswin_vi_CLK_GET_HANDLE(dev, vi_crg->aclk_mux, "aclk_mux");
+	eswin_vi_CLK_GET_HANDLE(dev, vi_crg->spll0_fout1, "spll0_fout1");
+	eswin_vi_CLK_GET_HANDLE(dev, vi_crg->vpll_fout1, "vpll_fout1");
+
+	return 0;
+}
+
+
+#define eswin_vi_SYS_CLK_PREPARE(clk)                                 \
+	do {                                                       \
+		if (clk_prepare_enable(clk)) {                     \
+			pr_err("Failed to enable clk %px\n", clk); \
+		}                                                  \
+	} while (0)
+
+
+UNUSED_FUNC static int eswin_vi_sys_clk_config(struct eswin_vi_clk_rst *vi_crg)
+{
+	int ret = 0;
+	long rate;
+
+	ret = clk_set_parent(vi_crg->aclk_mux, vi_crg->spll0_fout1);
+	if (ret < 0) {
+		pr_err("DW: failed to set aclk_mux parent: %d\n", ret);
+		return ret;
+	}
+
+	rate = clk_round_rate(vi_crg->aclk, eswin_vi_AXI_CLK_HIGHEST);
+	if (rate > 0) {
+		ret = clk_set_rate(vi_crg->aclk, rate);
+		if (ret) {
+			pr_err("DW: failed to set aclk: %d\n", ret);
+			return ret;
+		}
+		pr_info("DW set aclk to %ldHZ\n", rate);
+	}
+
+	return 0;
+}
+
+UNUSED_FUNC static int eswin_vi_sys_clk_prepare(struct eswin_vi_clk_rst *vi_crg)
+{
+	int ret = 0;
+	eswin_vi_SYS_CLK_PREPARE(vi_crg->aclk);
+	eswin_vi_SYS_CLK_PREPARE(vi_crg->cfg_clk);
+	ret = win2030_tbu_power(vi_crg->dev, true);
+	if (ret) {
+		pr_err("%s: DW tbu power up failed\n", __func__);
+		return ret;
+	}
+	return 0;
+}
+
+UNUSED_FUNC static int eswin_vi_sys_clk_unprepare(struct eswin_vi_clk_rst *vi_crg)
+{
+	int ret = 0;
+	//  tbu power down need enanle clk
+	ret = win2030_tbu_power(vi_crg->dev, false);
+	if (ret) {
+		pr_err("dw tbu power down failed\n");
+		return ret;
+	}
+	clk_disable_unprepare(vi_crg->cfg_clk);
+	clk_disable_unprepare(vi_crg->aclk);
+
+	return 0;
+}
+
+UNUSED_FUNC static int eswin_vi_reset_fini(struct eswin_vi_clk_rst *vi_crg)
+{
+	reset_control_assert(vi_crg->rstc_cfg);
+	reset_control_assert(vi_crg->rstc_axi);
+	return 0;
+}
+
+UNUSED_FUNC static int eswin_vi_sys_reset_release(struct eswin_vi_clk_rst *vi_crg)
+{
+	int ret;
+
+	ret = reset_control_deassert(vi_crg->rstc_cfg);
+	WARN_ON(0 != ret);
+
+	ret = reset_control_deassert(vi_crg->rstc_axi);
+	WARN_ON(0 != ret);
+
+	return 0;
+}
+
+static int eswin_open(struct inode *inode, struct file *file)
+{
+    struct eswin_vi_device *es_vi_dev = container_of(inode->i_cdev, struct eswin_vi_device, es_vi_cdev);
+    file->private_data = es_vi_dev;
+    return 0;
+}
+
+static int eswin_release(struct inode *inode, struct file *file)
+{
+    return 0;
+}
+
+static ssize_t eswin_read(struct file *file, char __user *buffer, size_t len, loff_t *offset)
+{
+    return 0;
+}
+
+static ssize_t eswin_write(struct file *file, const char __user *buffer, size_t len, loff_t *offset)
+{
+    return len;
+}
+
+static int eswin_vi_reset(struct eswin_vi_device *es_vi_dev, struct soc_control_context* soc_ctrl)
+{
+    //TODO template for reset
+    eic770x_vi_init(es_vi_dev);
+    return 0;
+}
+
+long eswin_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+    int retval, ret;
+    struct eswin_vi_device *es_vi_dev = file->private_data;
+    struct soc_control_context soc_ctrl;
+    struct isp_control_HxV isp_control_h_v;
+
+    switch (cmd) {
+        case VI_IOCTL_RESET:
+            pr_info(DRIVER_NAME ": VI_IOCTL_RESET\n");
+            retval = copy_from_user(&soc_ctrl, (int __user *)arg, sizeof(soc_ctrl));
+		    CHECK_COPY_RETVAL(retval);
+            ret = eswin_vi_reset(es_vi_dev, &soc_ctrl);
+            break;
+        case VI_IOCTL_ISP_H_V:
+            pr_info(DRIVER_NAME ": VI_IOCTL_ISP_H_V\n");
+            retval = copy_from_user(&isp_control_h_v, (int __user *)arg, sizeof(isp_control_h_v));
+            es_vi_dev->isp_dvp0_hor = isp_control_h_v.horizontal;
+            es_vi_dev->isp_dvp0_ver = isp_control_h_v.vertical;
+            break;
+        default:
+            pr_err(DRIVER_NAME ": Invalid IOCTL command\n");
+            return -EINVAL;
+    }
+
+    return ret;
+}
+
+
+static const struct media_device_ops es_mdev_ops = {
+    .link_notify = v4l2_pipeline_link_notify,
+};
+
+static const struct file_operations eswin_fops = {
+    .owner = THIS_MODULE,
+    .open = eswin_open,
+    .release = eswin_release,
+    .read = eswin_read,
+    .write = eswin_write,
+    .unlocked_ioctl = eswin_ioctl,
+};
+
+static int eswin_vi_probe(struct platform_device *pdev)
+{
+    int ret;
+    struct device *dev = &pdev->dev;
+    struct eswin_vi_device *es_vi_dev;
+    struct media_device *media_dev;
+    struct device_node *np = pdev->dev.of_node;
+    struct resource* res;
+    const char *dts_board_compat;
+    int numa_id = 0;
+    char class_name[32];
+    __maybe_unused struct eswin_vi_clk_rst *vi_clk_rst;
+
+    es_vi_dev = devm_kzalloc(dev, sizeof(*es_vi_dev), GFP_KERNEL);
+
+#if 1
+    //TODO: next step is to get the vi clk and rst from the device tree
+    //and remove clk and rst in dewarp driver
+    vi_clk_rst = &es_vi_dev->clk_rst;
+	ret = eswin_vi_sys_reset_init(pdev, &es_vi_dev->clk_rst);
+	if (ret) {
+		pr_err("%s: DW reset init failed\n", __func__);
+		return ret;
+	}
+
+	ret = eswin_vi_sys_clk_init(pdev,&es_vi_dev->clk_rst);
+	if (ret) {
+		pr_err("%s: DW clk init failed\n", __func__);
+		return ret;
+	}
+
+	ret = eswin_vi_sys_clk_config(&es_vi_dev->clk_rst);
+	if (ret) {
+		pr_err("%s: DW clk prepare failed\n", __func__);
+		return ret;
+	}
+
+    es_vi_dev->clk_rst.dev = &pdev->dev;
+
+	ret = eswin_vi_sys_clk_prepare(&es_vi_dev->clk_rst);
+	if (ret) {
+		pr_err("%s: DW clk prepare failed\n", __func__);
+		return ret;
+	}
+
+	ret = eswin_vi_sys_reset_release(&es_vi_dev->clk_rst);
+	if (ret) {
+		pr_err("%s: DW reset release failed\n", __func__);
+		return ret;
+	}
+#endif
+    es_vi_dev->media_dev = devm_kzalloc(dev, sizeof(*media_dev), GFP_KERNEL);
+
+    media_dev = es_vi_dev->media_dev;
+    if (!es_vi_dev) {
+        pr_err(DRIVER_NAME ": Failed to allocate device\n");
+        return -ENOMEM;
+    }
+    es_vi_dev->dev = dev;
+    dev_set_drvdata(dev, es_vi_dev);
+
+    ret = alloc_chrdev_region(&major_number, 0, 1, "eswin_vi");
+    if (ret < 0) {
+        pr_err("Failed to allocate major number\n");
+        return ret;
+    }
+
+    // Initialize cdev
+    cdev_init(&es_vi_dev->es_vi_cdev, &eswin_fops);
+    ret = cdev_add(&es_vi_dev->es_vi_cdev, major_number, 1);
+    if (ret < 0) {
+        pr_err(DRIVER_NAME ": Failed to add cdev\n");
+        return ret;
+    }
+
+    ret = of_property_read_u32(dev->of_node, "numa-node-id", &numa_id);
+    if(ret) {
+        dev_warn(dev, "Could not get numa-node-id, use default 0\n");
+        numa_id = 0;
+    }
+
+    if(numa_id == 0) {
+        dev_info(dev, "create device class node eswin_vi_class\n");
+        snprintf(class_name, sizeof(class_name), "eswin_vi_class");
+    } else {
+        dev_info(dev, "create device class node eswin_vi_class_d%d\n", numa_id);
+        snprintf(class_name, sizeof(class_name), "eswin_vi_class_d%d", numa_id);
+    }
+    es_vi_dev->es_vi_class = class_create(class_name);
+    if (IS_ERR(es_vi_dev->es_vi_class)) {
+        cdev_del(&es_vi_dev->es_vi_cdev);
+        unregister_chrdev_region(major_number, 1);
+        return PTR_ERR(es_vi_dev->es_vi_class);
+    }
+
+    // 创建设备节点
+    if(numa_id == 0) {
+        dev_info(dev, "create device node eswin_vi\n");
+        device_create(es_vi_dev->es_vi_class, NULL, major_number, NULL, "eswin_vi");
+    } else {
+        dev_info(dev, "create device node eswin_vi_d%d\n", numa_id);
+        device_create(es_vi_dev->es_vi_class, NULL, major_number, NULL, "eswin_vi_d%d", numa_id);
+    }
+    media_device_init(media_dev);
+    media_dev->dev = &pdev->dev;
+    media_dev->ops = &es_mdev_ops;
+    strscpy(media_dev->model, "verisilicon_media", sizeof(media_dev->model));
+
+    ret = media_device_register(media_dev);
+    if (ret < 0) {
+        media_device_cleanup(media_dev);
+        kfree(media_dev);
+        cdev_del(&es_vi_dev->es_vi_cdev);
+        device_destroy(es_vi_dev->es_vi_class, MKDEV(major_number, 0));
+        class_destroy(es_vi_dev->es_vi_class);
+        unregister_chrdev(major_number, DEVICE_NAME);
+        pr_err(DRIVER_NAME ": Failed to register media device\n");
+        return ret;
+    }
+    of_platform_populate(np, NULL, NULL, &pdev->dev);
+
+    //TODO add parser es_vi_top dts
+
+    res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	es_vi_dev->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(es_vi_dev->base)) {
+		resource_size_t offset = res->start;
+		resource_size_t size = resource_size(res);
+		dev_warn(&pdev->dev, "avoid secondary mipi resource check!\n");
+		es_vi_dev->base = devm_ioremap(&pdev->dev, offset, size);
+		if (IS_ERR(es_vi_dev->base)) {
+			dev_err(&pdev->dev, "Failed to ioremap resource\n");
+			return PTR_ERR(es_vi_dev->base);
+		}
+	}
+
+    ret = of_property_read_string(np, "board_compat", &dts_board_compat);
+    if (ret) {
+        pr_warn(DRIVER_NAME ": Failed to read board_compat property! Use Default EVB board_compat!\n");
+        dts_board_compat = "eic7700,evb_vi";
+    }
+    dev_dbg(dev, ": board_compat: %s\n", dts_board_compat);
+
+    if(!strcmp(dts_board_compat, "eic7700,evb_vi")) {
+        es_vi_dev->board_compat = EIC7700_EVB_VI_COMPAT;
+    } else if(!strcmp(dts_board_compat, "eic7700,dvb_vi")) {
+        es_vi_dev->board_compat = EIC7700_DVB_VI_COMPAT;
+    } else {
+        pr_warn(DRIVER_NAME ": Unknown board_compat! Use Default EVB board_compat!\n");
+        es_vi_dev->board_compat = EIC7700_EVB_VI_COMPAT;
+    }
+
+    ret = of_property_read_u32(np, "eswin,isp_dvp0_hor", &es_vi_dev->isp_dvp0_hor);
+    if (ret) {
+        pr_warn(DRIVER_NAME ": Failed to read isp_dvp0_hor property! Use Default 3280!\n");
+        es_vi_dev->isp_dvp0_hor = 3280;
+    }
+    ret = of_property_read_u32(np, "eswin,isp_dvp0_ver", &es_vi_dev->isp_dvp0_ver);
+    if (ret) {
+        pr_warn(DRIVER_NAME ": Failed to read isp_dvp0_ver property! Use Default 2464!\n");
+        es_vi_dev->isp_dvp0_ver = 2464;
+    }
+
+    dev_dbg(dev, "isp_dvp0_hor: %d\n", es_vi_dev->isp_dvp0_hor);
+    dev_dbg(dev,  "isp_dvp0_ver: %d\n", es_vi_dev->isp_dvp0_ver);
+
+    ret = of_property_read_u32(np, "phy_mode", &es_vi_dev->phy_mode);
+    if (ret) {
+        pr_warn(DRIVER_NAME ": Failed to read phy_mode property! Use Default 5!\n");
+        es_vi_dev->phy_mode = 5;
+    }
+
+    es_vi_dev->syscrg_regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscrg_csr");
+    if (IS_ERR(es_vi_dev->syscrg_regmap)) {
+        dev_err(dev, "No syscrg_csr phandle specified\n");
+        return PTR_ERR(es_vi_dev->syscrg_regmap);
+    }
+
+
+    eic770x_vi_init(es_vi_dev);
+    dev_info(dev, "Probe successful\n");
+
+
+    return 0;
+}
+
+static int eswin_remove(struct platform_device *pdev)
+{
+    struct eswin_vi_device *es_vi_dev = platform_get_drvdata(pdev);
+    media_device_unregister(es_vi_dev->media_dev);
+    media_device_cleanup(es_vi_dev->media_dev);
+    kfree(es_vi_dev->media_dev);
+    cdev_del(&es_vi_dev->es_vi_cdev);
+    device_destroy(es_vi_dev->es_vi_class, MKDEV(major_number, 0));
+    class_destroy(es_vi_dev->es_vi_class);
+    unregister_chrdev(major_number, DEVICE_NAME);
+    of_platform_depopulate(&pdev->dev);
+    pr_info(DRIVER_NAME ": Removed\n");
+    return 0;
+}
+
+static const struct of_device_id eswin_id_table[] = {
+    { 
+        .compatible = "esw,vi_subsys", 
+        .data = 0 
+    },
+    { }
+};
+
+static struct platform_driver eswin_vi_driver = {
+    .driver = {
+        .name = DRIVER_NAME,
+        .of_match_table = of_match_ptr(eswin_id_table),
+    },
+    .probe = eswin_vi_probe,
+    .remove = eswin_remove,
+};
+
+int es_vi_drv_init(void)
+{
+	platform_driver_register(&eswin_vi_driver);
+	return 0;
+}
+
+static void __exit es_vi_drv_exit(void)
+{
+	platform_driver_unregister(&eswin_vi_driver);
+}
+
+module_init(es_vi_drv_init);
+module_exit(es_vi_drv_exit);
+
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("yufangxian@eswincomputing.com");
+MODULE_DESCRIPTION("ESWIN VI TOP Driver");
+MODULE_VERSION("0.1");
diff --git a/drivers/media/platform/eswin/vi_top/vitop.h b/drivers/media/platform/eswin/vi_top/vitop.h
new file mode 100644
index 000000000000..31879702e0a1
--- /dev/null
+++ b/drivers/media/platform/eswin/vi_top/vitop.h
@@ -0,0 +1,241 @@
+#ifndef __M1_VITOP_HEADER__
+#define __M1_VITOP_HEADER__
+
+#define M2_VI_TOP
+
+#ifdef M1_VI_TOP
+#define VI_SUBSYSTEM_REGISTER_BASE_ADDRESS      0x41000000
+#else //M2 Final Design
+#define VI_SUBSYSTEM_REGISTER_BASE_ADDRESS      0x51000000
+#endif
+
+enum {
+	VI_IOCTL_RESET = _IO('r', 0),
+	VI_IOCTL_ISP_H_V = _IO('r', 1),
+
+};
+
+struct soc_control_context {
+	uint32_t device_idx;
+	uint32_t control_value;
+};
+
+struct isp_control_HxV {
+    uint32_t horizontal;
+    uint32_t vertical;
+};
+
+// VI Submodule Base Address
+#define VI_ISP0_REGISTER_BASE_ADDRESS           (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x00000)
+#define VI_ISP1_REGISTER_BASE_ADDRESS           (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x10000)
+#define VI_DEWARP_REGISTER_BASE_ADDRESS         (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x20000)
+#define VI_TOP_REGISTER_BASE_ADDRESS            (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x30000)
+#define VI_SHUTTER_CTRL_REGISTER_BASE_ADDRESS   (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x32000)
+#define VI_DVP2AXI_REGISTER_BASE_ADDRESS        (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x34000)
+#define VI_SLVS_CTRL0_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x38000) //0x38000 FPGA use csi_base
+#define VI_SLVS_CTRL1_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x3c000)
+#define VI_SLVS_CTRL2_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x40000)
+#define VI_SLVS_CTRL3_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x44000)
+#define VI_SLVS_CTRL4_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x48000)
+#define VI_SLVS_CTRL5_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x4c000)
+
+#define VI_CSI2_CTRL0_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x50000)
+#define VI_CSI2_CTRL1_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x60000)
+#define VI_CSI2_CTRL2_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x70000)
+#define VI_CSI2_CTRL3_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x80000)
+#define VI_CSI2_CTRL4_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x90000)
+#define VI_CSI2_CTRL5_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0xa0000)
+
+#define VI_COMBO_PHY0_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0xc0000)
+#define VI_COMBO_PHY1_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0xe0000)
+#define VI_COMBO_PHY2_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x100000)
+#define VI_COMBO_PHY3_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x120000)
+#define VI_COMBO_PHY4_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x140000)
+#define VI_COMBO_PHY5_REGISTER_BASE_ADDRESS     (VI_SUBSYSTEM_REGISTER_BASE_ADDRESS + 0x160000)
+
+// Should be confirmed in M2
+#define VI_TOP_PHY_CONNECT_MODE                 0x00
+#define VI_TOP_PHY_MODE_OFFSET                  0x0
+// #define VI_TOP_PHY_MODE0_8LANE_4LANE            0x0
+// #define VI_TOP_PHY_MODE1_8LANE_2LANEX2          0x1
+// #define VI_TOP_PHY_MODE2_4LANEX3                0x2
+// #define VI_TOP_PHY_MODE3_4LANEX2_2LANEX2        0x3
+// #define VI_TOP_PHY_MODE4_4LANE_2LANEX4          0x4
+// #define VI_TOP_PHY_MODE1_2LANEX6                0x5
+#define VI_TOP_PHY_TPG_CLOCK_MUX_EN_OFFSET      0x3
+
+#define VI_TOP_CONTROLLER_SELECT                0x04
+#define VI_TOP_CONTROLLER_CSI0_SEL_OFFSET       0x0
+#define VI_TOP_CONTROLLER_CSI1_SEL_OFFSET       0x1
+#define VI_TOP_CONTROLLER_CSI2_SEL_OFFSET       0x2
+#define VI_TOP_CONTROLLER_CSI3_SEL_OFFSET       0x3
+#define VI_TOP_CONTROLLER_CSI4_SEL_OFFSET       0x4
+#define VI_TOP_CONTROLLER_CSI5_SEL_OFFSET       0x5
+// #define VI_TOP_CONTROLLER_FROM_CSI              0x0
+// #define VI_TOP_CONTROLLER_FROM_MULTI            0x1
+
+
+#define VI_TOP_ISP_DVP_SEL                      0x08
+#define VI_TOP_ISP0_DVP0_SEL_OFFSET             0x0
+#define VI_TOP_ISP0_DVP1_SEL_OFFSET             0x3
+#define VI_TOP_ISP1_DVP0_SEL_OFFSET             0x6
+#define VI_TOP_ISP1_DVP1_SEL_OFFSET             0x9
+#define VI_TOP_ISP0_DVP2_SEL_OFFSET             0xc
+#define VI_TOP_ISP0_DVP3_SEL_OFFSET             0xf
+#define VI_TOP_ISP1_DVP2_SEL_OFFSET             0x12
+#define VI_TOP_ISP1_DVP3_SEL_OFFSET             0x15
+// #define VI_TOP_ISP_DVP_INPUT_CONTROLER0         0x0
+// #define VI_TOP_ISP_DVP_INPUT_CONTROLER1         0x1
+// #define VI_TOP_ISP_DVP_INPUT_CONTROLER2         0x2
+// #define VI_TOP_ISP_DVP_INPUT_CONTROLER3         0x3
+// #define VI_TOP_ISP_DVP_INPUT_CONTROLER4         0x4
+// #define VI_TOP_ISP_DVP_INPUT_CONTROLER5         0x5
+// #define VI_TOP_ISP_DVP_INPUT_DVP0               0x6
+// #define VI_TOP_ISP_DVP_INPUT_DVP1               0x7
+
+#define VI_TOP_ISP0_DVP0_SIZE                   0x0c
+#define VI_TOP_ISP0_DVP1_SIZE                   0x10
+#define VI_TOP_ISP1_DVP0_SIZE                   0x14
+#define VI_TOP_ISP1_DVP1_SIZE                   0x18
+#define VI_TOP_ISP0_DVP2_SIZE                   0xa4
+#define VI_TOP_ISP0_DVP3_SIZE                   0xa8
+#define VI_TOP_ISP1_DVP2_SIZE                   0xac
+#define VI_TOP_ISP1_DVP3_SIZE                   0xb0
+// #define VI_TOP_ISP_DVP_WIDTH_OFFSET           0x0
+// #define VI_TOP_ISP_DVP_HEIGHT_OFFSET          0x10
+
+#define VI_TOP_MULTI2ISP_BLANK                  0x1c
+
+#define VI_TOP_MULTI2ISP0_DVP0                  0x20
+#define VI_TOP_MULTI2ISP0_DVP1                  0x24
+#define VI_TOP_MULTI2ISP1_DVP0                  0x28
+#define VI_TOP_MULTI2ISP1_DVP1                  0x2c
+#define VI_TOP_MULTI2ISP0_DVP2                  0xb4
+#define VI_TOP_MULTI2ISP0_DVP3                  0xb8
+#define VI_TOP_MULTI2ISP1_DVP2                  0xbc
+#define VI_TOP_MULTI2ISP1_DVP3                  0xc0
+// #define VI_TOP_MULTI2ISP_SHIFT_BIT_OFFSET       0x0
+// #define VI_TOP_MULTI2ISP_INV_OFFSET             0x8
+// #define VI_TOP_MULTI2ISP_VH_GAP_OFFSET          0x9
+
+#define VI_TOP_DISABLE_ISP0                    0x30
+#define VI_TOP_DISABLE_ISP1                    0x34
+#define VI_TOP_DEWARP_START                    0x38
+#define VI_TOP_ON_STREAM_SEL                   0x3c
+
+#define VI_TOP_CLOCK_ENABLE                    0x40
+#define VI_TOP_CLOCK_ENABLE_MASKS              0x1fff
+#define VI_TOP_DVP2AXI_CLOCK_ENABLED   (1 << 17)
+#define VI_TOP_ISP1_DVP3_CLOCK_ENABLED (1 << 16)
+#define VI_TOP_ISP1_DVP2_CLOCK_ENABLED (1 << 15)
+#define VI_TOP_ISP0_DVP3_CLOCK_ENABLED (1 << 14)
+#define VI_TOP_ISP0_DVP2_CLOCK_ENABLED (1 << 13)
+#define VI_TOP_CTRL5_DVP_CLOCK_ENABLED (1 << 12)
+#define VI_TOP_CTRL4_DVP_CLOCK_ENABLED (1 << 11)
+#define VI_TOP_CTRL3_DVP_CLOCK_ENABLED (1 << 10)
+#define VI_TOP_CTRL2_DVP_CLOCK_ENABLED (1 << 9)
+#define VI_TOP_CTRL1_DVP_CLOCK_ENABLED (1 << 8)
+#define VI_TOP_CTRL0_DVP_CLOCK_ENABLED (1 << 7)
+#define VI_TOP_ISP1_DVP1_CLOCK_ENABLED (1 << 6)
+#define VI_TOP_ISP1_DVP0_CLOCK_ENABLED (1 << 5)
+#define VI_TOP_ISP0_DVP1_CLOCK_ENABLED (1 << 4)
+#define VI_TOP_ISP0_DVP0_CLOCK_ENABLED (1 << 3)
+#define VI_TOP_DW200_CLOCK_ENABLED     (1 << 2)
+#define VI_TOP_ISP1_CLOCK_ENABLED      (1 << 1)
+#define VI_TOP_ISP0_CLOCK_ENABLED      (1 << 0)
+
+
+#define VI_TOP_CSI_COMBO_PHY_IDERR              0x44
+#define VI_TOP_CMN_INT_EN                       0x48
+#define VI_TOP_CMN_INT_MASK                     0x4c
+#define VI_TOP_CMN_INT_CLEAR                    0x50
+#define VI_TOP_CMN_INT_STATUS                   0x54
+
+
+#define VI_TOP_MULTI2ISP0_DVP0_STATUS           0x58
+#define VI_TOP_MULTI2ISP0_DVP1_STATUS           0x5c
+#define VI_TOP_MULTI2ISP1_DVP0_STATUS           0x60
+#define VI_TOP_MULTI2ISP1_DVP1_STATUS           0x64
+#define VI_TOP_MULTI2ISP0_DVP2_STATUS           0xb4
+#define VI_TOP_MULTI2ISP0_DVP3_STATUS           0xb8
+#define VI_TOP_MULTI2ISP1_DVP2_STATUS           0xbc
+#define VI_TOP_MULTI2ISP1_DVP3_STATUS           0xc0
+// #define VI_TOP_MULTI2ISP_VBLANK_ERR_OFFSET      0x0
+// #define VI_TOP_MULTI2ISP_HBLANK_ERR_OFFSET      0x10
+
+#define VI_TOP_PHY_PCLK_CFG                     0x68
+#define VI_TOP_DVP0_CNT_STATUS                  0x6c
+#define VI_TOP_DVP1_CNT_STATUS                  0x70
+#define VI_TOP_DVP2_CNT_STATUS                  0x74
+#define VI_TOP_DVP3_CNT_STATUS                  0x78
+#define VI_TOP_DVP4_CNT_STATUS                  0x7c
+#define VI_TOP_DVP5_CNT_STATUS                  0x80
+
+#define VI_TOP_IO_DVP0_CNT_STATUS               0x84
+#define VI_TOP_IO_DVP1_CNT_STATUS               0x88
+
+#define VI_TOP_TOISP0_DVP0_CNT_STATUS           0x8c
+#define VI_TOP_TOISP0_DVP1_CNT_STATUS           0x90
+#define VI_TOP_TOISP1_DVP0_CNT_STATUS           0x94
+#define VI_TOP_TOISP1_DVP1_CNT_STATUS           0x98
+#define VI_TOP_TOISP0_DVP2_CNT_STATUS           0xc4
+#define VI_TOP_TOISP0_DVP3_CNT_STATUS           0xc8
+#define VI_TOP_TOISP1_DVP2_CNT_STATUS           0xcc
+#define VI_TOP_TOISP1_DVP3_CNT_STATUS           0xd0
+
+#define VI_TOP_TRIG_IO_SEL                      0xa0
+
+#define VI_TOP_ISP_INPUT_LINESIZE_ERR           0xd4
+#define VI_TOP_ISP0_DVP0_LINE_SIZE_ERR_OFFSET   0x0
+#define VI_TOP_ISP0_DVP1_LINE_SIZE_ERR_OFFSET   0x1
+#define VI_TOP_ISP0_DVP2_LINE_SIZE_ERR_OFFSET   0x2
+#define VI_TOP_ISP0_DVP3_LINE_SIZE_ERR_OFFSET   0x3
+#define VI_TOP_ISP1_DVP0_LINE_SIZE_ERR_OFFSET   0x4
+#define VI_TOP_ISP1_DVP1_LINE_SIZE_ERR_OFFSET   0x5
+#define VI_TOP_ISP1_DVP2_LINE_SIZE_ERR_OFFSET   0x6
+#define VI_TOP_ISP1_DVP3_LINE_SIZE_ERR_OFFSET   0x7
+
+
+#ifdef M1_VI_TOP
+//M1
+#define VI_SUBSYSTEM_SCU_BASE_REGISTER          0x42020000
+#define VI_SUBSYSTEM_SCU_ISP_RESET              0x0c
+#define VI_SUBSYSTEM_SCU_SONY_HDR_MODE          0x14
+#define VI_SLVS_CTRL_0_IRQ_NUM (14 + 1)
+#define VI_SLVS_CTRL_1_IRQ_NUM (15 + 1) 
+#define VI_CSI2_CTRL0_IRQ_NUM (14 + 1)
+
+#define VI_DVP2AXI_DVP0_IRQ_NUM     29
+#define VI_DVP2AXI_DVP1_IRQ_NUM     30
+#define VI_DVP2AXI_DVP2_IRQ_NUM     31
+#define VI_DVP2AXI_DVP3_IRQ_NUM     32
+#define VI_DVP2AXI_DVP4_IRQ_NUM     33
+#define VI_DVP2AXI_DVP5_IRQ_NUM     34
+#define VI_DVP2AXI_ERROR_IRQ_NUM    35
+#define VI_DVP2AXI_AFULL_IRQ_NUM    36
+
+#else
+#define VI_SUBSYSTEM_SCU_BASE_REGISTER          0x51828000
+#define VI_SUBSYSTEM_SCU_ISP_RESET              0x478
+// #define VI_SUBSYSTEM_SCU_SONY_HDR_MODE          0x14
+#define VI_SLVS_CTRL_0_IRQ_NUM (24 + 1)//(30 + 1) FPGA use csi_irq
+#define VI_SLVS_CTRL_1_IRQ_NUM (25 + 1)//(31 + 1)  
+
+#define VI_CSI2_CTRL0_IRQ_NUM (24 + 1)
+#define VI_CSI2_CTRL1_IRQ_NUM (25 + 1)
+#define VI_CSI2_CTRL2_IRQ_NUM (26 + 1)
+#define VI_CSI2_CTRL3_IRQ_NUM (27 + 1)
+#define VI_CSI2_CTRL4_IRQ_NUM (28 + 1)
+#define VI_CSI2_CTRL5_IRQ_NUM (29 + 1)
+
+#define VI_DVP2AXI_DVP0_IRQ_NUM     (363 + 1)
+#define VI_DVP2AXI_DVP1_IRQ_NUM     (364 + 1)
+#define VI_DVP2AXI_DVP2_IRQ_NUM     (365 + 1)
+#define VI_DVP2AXI_DVP3_IRQ_NUM     (366 + 1)
+#define VI_DVP2AXI_DVP4_IRQ_NUM     (367 + 1)
+#define VI_DVP2AXI_DVP5_IRQ_NUM     (368 + 1)
+#define VI_DVP2AXI_ERROR_IRQ_NUM    (369 + 1)
+#define VI_DVP2AXI_AFULL_IRQ_NUM    (370 + 1)
+#endif
+
+#endif
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index cb3ad72a3e54..eb905c565fe1 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -662,9 +662,10 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 
 		if (!client_supports_streams)
 			format->stream = 0;
-
+		printk("set fmt,, sd name = %s, %d",sd->name, *format->format.reserved);
 		memset(format->reserved, 0, sizeof(format->reserved));
 		memset(format->format.reserved, 0, sizeof(format->format.reserved));
+		
 		return v4l2_subdev_call(sd, pad, set_fmt, state, format);
 	}
 
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index c82d8d8a16ea..24faad30a79f 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -227,5 +227,6 @@ config STM32_FMC2_EBI
 
 source "drivers/memory/samsung/Kconfig"
 source "drivers/memory/tegra/Kconfig"
+source "drivers/memory/eswin/Kconfig"
 
 endif
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index d2e6ca9abbe0..e1459a3760a3 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_STM32_FMC2_EBI)	+= stm32-fmc2-ebi.o
 
 obj-$(CONFIG_SAMSUNG_MC)	+= samsung/
 obj-$(CONFIG_TEGRA_MC)		+= tegra/
+obj-$(CONFIG_ESWIN_MC)          += eswin/
 obj-$(CONFIG_TI_EMIF_SRAM)	+= ti-emif-sram.o
 obj-$(CONFIG_FPGA_DFL_EMIF)	+= dfl-emif.o
 
diff --git a/drivers/memory/eswin/Kconfig b/drivers/memory/eswin/Kconfig
new file mode 100644
index 000000000000..81477c0c29b5
--- /dev/null
+++ b/drivers/memory/eswin/Kconfig
@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: GPL-2.0
+config ESWIN_MC
+	bool "ESWIN Memory Controller and reserved memory block support"
+	default y
+	depends on ARCH_ESWIN_EIC770X_SOC_FAMILY
+	select SG_SPLIT
+	help
+	  This driver supports the Memory Controller (MC) hardware, especialy for 
+	  LLC SPRAM controller, found on ESWIN SoCs. And the memblock of the reserved
+	  memory declared with the compatible = "eswin-reserve-memory" property in dts
+	  is also initialized.
+
+if ESWIN_MC
+
+config ESWIN_RSV_MEMBLOCK
+	bool "ESWIN reserved memory block support"
+	depends on MMU
+	help
+	  If reserved-memory has compatible = "eswin-reserve-memory" and
+	  no-map properties, this memory block will be initialized and freed in
+	  Eswin private buddy system. Using es_alloc_pages(memblock, order) to
+	  alloc pages from this memory block and es_free_pages(memblock, page) to
+	  free the pages to this memory block.
+
+	  If unsure, say "n".
+
+source "drivers/memory/eswin/codacache/Kconfig"
+source "drivers/memory/eswin/es_buddy/Kconfig"
+source "drivers/memory/eswin/es_proc/Kconfig"
+source "drivers/memory/eswin/es_rsvmem_heap/Kconfig"
+source "drivers/memory/eswin/es_mmz_vb/Kconfig"
+source "drivers/memory/eswin/es_dev_buf/Kconfig"
+source "drivers/memory/eswin/es_iommu_rsv/Kconfig"
+source "drivers/memory/eswin/es_dma_memcp/Kconfig"
+source "drivers/memory/eswin/es_malloc_dmabuf/Kconfig"
+
+endif
diff --git a/drivers/memory/eswin/Makefile b/drivers/memory/eswin/Makefile
new file mode 100644
index 000000000000..c7b09cc3491b
--- /dev/null
+++ b/drivers/memory/eswin/Makefile
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ESWIN_MC)				+= eswin_cpuid_hartid_convert.o
+obj-$(CONFIG_ESWIN_RSV_MEMBLOCK)		+= eswin_memblock.o
+obj-$(CONFIG_ESWIN_CODACACHE_CONTROLLER)	+= codacache/
+obj-$(CONFIG_ESWIN_BUDDY)			+= es_buddy/
+obj-$(CONFIG_ESWIN_PROC)			+= es_proc/
+obj-$(CONFIG_ESWIN_RSVMEM_HEAP)			+= es_rsvmem_heap/
+obj-$(CONFIG_ESWIN_RSVMEM_HEAP)			+= es_mmz_vb/
+obj-$(CONFIG_ESWIN_DEV_DMA_BUF)			+= es_dev_buf/
+obj-$(CONFIG_ESWIN_IOMMU_RSV)			+= es_iommu_rsv/
+obj-$(CONFIG_ESWIN_DMA_MEMCP)           += es_dma_memcp/
+obj-$(CONFIG_ESWIN_MALLOC_DMABUF)       += es_malloc_dmabuf/
diff --git a/drivers/memory/eswin/buddy.h b/drivers/memory/eswin/buddy.h
new file mode 100644
index 000000000000..ff47325f2d27
--- /dev/null
+++ b/drivers/memory/eswin/buddy.h
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Header file of ESWIN internal buddy allocator
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#ifndef __BUDDY_H__
+#define __BUDDY_H__
+
+#define RUN_IN_KERNEL 1
+
+#ifdef RUN_IN_KERNEL
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/bug.h>
+#include <linux/list.h>
+#include <linux/string.h>
+#include <linux/spinlock.h>
+
+// #define buddy_print(fmt...) printk(fmt)
+#define buddy_print(fmt...)
+#define BUDDY_BUG_ON(condition) WARN_ON(condition)
+
+#define es_spin_lock_init(esLock)	spin_lock_init(esLock)
+#define es_spin_lock(esLock)		spin_lock(esLock)
+#define es_spin_unlock(esLock)		spin_unlock(esLock)
+#define buddy_spin_lock_init(lock)
+#define buddy_spin_lock(lock)
+#define buddy_spin_unlock(lock)
+#else
+#include "list.h"
+#include <stdio.h>  //printf
+#include <string.h> //memset
+#include <assert.h> //assert
+
+#define buddy_print(fmt...) printf(fmt)
+#define BUDDY_BUG_ON(condition) assert(condition)
+
+#define buddy_spin_lock_init(lock)	do {}while(0)
+#define buddy_spin_lock(lock)		do {}while(0)
+#define buddy_spin_unlock(lock)		do {}while(0)
+
+#define es_spin_lock_init(esLock)
+#define es_spin_lock(esLock)
+#define es_spin_unlock(esLock)
+#endif
+
+enum esPageflags_e{
+	enPG_head,
+	enPG_tail,
+	enPG_buddy,
+};
+
+#define BUDDY_PAGE_SHIFT    PAGE_SHIFT//(12UL)
+#define BUDDY_PAGE_SIZE     (1UL << BUDDY_PAGE_SHIFT)
+#define BUDDY_MAX_ORDER     MAX_ORDER // (10UL)//(9UL)
+
+struct esPage_s
+{
+	// spin_lock        lock;
+	struct list_head    lru;
+	unsigned long       flags;
+	union {
+		unsigned long   order;
+		struct esPage_s     *first_page;
+	};
+};
+
+struct es_free_area_s
+{
+	struct list_head    free_list;
+	unsigned long       nr_free;
+};
+
+struct mem_zone
+{
+#ifdef RUN_IN_KERNEL
+	spinlock_t          lock;
+#endif
+	unsigned long       page_num;
+	unsigned long       page_size;
+	struct esPage_s     *first_page;
+	unsigned long       start_addr;
+	unsigned long       end_addr;
+	struct es_free_area_s    free_area[BUDDY_MAX_ORDER];
+};
+
+#ifdef RUN_IN_KERNEL
+#define BLOCK_MAX_NAME 64
+struct mem_block {
+#ifdef RUN_IN_KERNEL
+	spinlock_t		esLock;
+#endif
+	struct mem_zone		zone;
+	unsigned long		page_num;
+	struct esPage_s		*esPagesStart;
+	struct page		*kPageStart;
+	char name[BLOCK_MAX_NAME];
+};
+#else
+struct mem_block {
+	struct mem_zone     zone;
+	struct esPage_s        *pages;
+};
+#endif
+
+void         buddy_system_init(struct mem_block *memblock,
+							   struct esPage_s *start_page,
+							   unsigned long start_addr,
+							   unsigned long page_num);
+struct esPage_s *buddy_get_pages(struct mem_zone *zone,
+							 unsigned long order);
+void         buddy_free_pages(struct mem_zone *zone,
+							  struct esPage_s *page);
+unsigned long buddy_num_free_page(struct mem_zone *zone);
+
+
+static inline void __esSetPageHead(struct esPage_s *page)
+{
+	page->flags |= (1UL<<enPG_head);
+}
+
+static inline void __SetPageTail(struct esPage_s *page)
+{
+	page->flags |= (1UL<<enPG_tail);
+}
+
+static inline void __esSetPageBuddy(struct esPage_s *page)
+{
+	page->flags |= (1UL<<enPG_buddy);
+}
+/**/
+static inline void __esClearPageHead(struct esPage_s *page)
+{
+	page->flags &= ~(1UL<<enPG_head);
+}
+
+static inline void __ClearPageTail(struct esPage_s *page)
+{
+	page->flags &= ~(1UL<<enPG_tail);
+}
+
+static inline void __esClearPageBuddy(struct esPage_s *page)
+{
+	page->flags &= ~(1UL<<enPG_buddy);
+}
+/**/
+static inline int esPageHead(struct esPage_s *page)
+{
+	return (page->flags & (1UL<<enPG_head));
+}
+
+static inline int esPageTail(struct esPage_s *page)
+{
+	return (page->flags & (1UL<<enPG_tail));
+}
+
+static inline int esPageBuddy(struct esPage_s *page)
+{
+	return (page->flags & (1UL<<enPG_buddy));
+}
+
+
+static inline void set_page_order_buddy(struct esPage_s *page, unsigned long order)
+{
+	page->order = order;
+	__esSetPageBuddy(page);
+}
+
+static inline void rmv_page_order_buddy(struct esPage_s *page)
+{
+	page->order = 0;
+	__esClearPageBuddy(page);
+}
+
+
+static inline unsigned long
+__find_buddy_index(unsigned long page_idx, unsigned int order)
+{
+	return (page_idx ^ (1 << order));
+}
+
+static inline unsigned long
+__find_combined_index(unsigned long page_idx, unsigned int order)
+{
+	return (page_idx & ~(1 << order));
+}
+
+
+static inline unsigned long esCompound_order(struct esPage_s *page)
+{
+	if (!esPageHead(page))
+		return 0;
+
+	return page->order;
+}
+
+static inline void esSet_compound_order(struct esPage_s *page, unsigned long order)
+{
+	page->order = order;
+}
+
+static inline void BUDDY_BUG(const char *f, int line)
+{
+	buddy_print("BUDDY_BUG in %s, %d.\n", f, line);
+	BUDDY_BUG_ON(1);
+}
+
+// print buddy system status
+void dump_print(struct mem_zone *zone);
+void dump_print_dot(struct mem_zone *zone);
+
+#endif
diff --git a/drivers/memory/eswin/codacache/Kconfig b/drivers/memory/eswin/codacache/Kconfig
new file mode 100644
index 000000000000..104fdb4d08a2
--- /dev/null
+++ b/drivers/memory/eswin/codacache/Kconfig
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+config ESWIN_CODACACHE_CONTROLLER
+	bool "ESWIN codacache controller support"
+	help
+	  The codacache, 4MB in total, could be entirely intialized as SPRAM, or partialy as
+	  SPRAM and partilay as cache(i.e. Last Level Cache, LLC), or entirely as cache.
+	  For example, if 2MB is initialized as SPRAM, then the reset of the codacache, 2MB
+	  will be initialized as cache. The size of the SPRAM is determined by the reg size of
+	  the spram node in dts. The below example shows that the codacache of 4MB is entirely
+	  initialized as SPRAM.
+	  npu0_reserved: sprammemory@59000000 {
+		reg = <0x0 0x59000000 0x0 0x400000>;
+	  };
+
+	  If unsure, say "n".
+
diff --git a/drivers/memory/eswin/codacache/Makefile b/drivers/memory/eswin/codacache/Makefile
new file mode 100644
index 000000000000..f992e9a5eb89
--- /dev/null
+++ b/drivers/memory/eswin/codacache/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ESWIN_CODACACHE_CONTROLLER)	+= llc_spram.o
diff --git a/drivers/memory/eswin/codacache/llc_spram.c b/drivers/memory/eswin/codacache/llc_spram.c
new file mode 100644
index 000000000000..7b188b69f773
--- /dev/null
+++ b/drivers/memory/eswin/codacache/llc_spram.c
@@ -0,0 +1,1790 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN LLC_SPRAM on-chip SRAM allocation driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Min Lin <linmin@eswincomputing.com>
+ *          Huawei Dong <donghuawei@eswincomputing.com>
+ *          Wei Yang <yangwei1@eswincomputing.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/delay.h>
+#include <linux/genalloc.h>
+#include <linux/regmap.h>
+#include <linux/io.h>
+#include <linux/list_sort.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-heap.h>
+#include <uapi/linux/dma-heap.h>
+#include <linux/dma-map-ops.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/mfd/syscon.h>
+#include <linux/memblock.h>
+#include <linux/version.h>
+#include <linux/clk-provider.h>
+
+#include <linux/eswin_npu.h>
+#include <linux/regulator/consumer.h>
+#include <linux/pm_runtime.h>
+#include "llc_spram.h"
+
+#define HAVE_LLC_HARDWARE	1
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+#define DEVICE_NAME	"llc_spram"
+
+#define SPRAM_GRANULARITY	PAGE_SIZE
+
+static int npu_spram_size = 4 * 0x100000; //4M
+module_param(npu_spram_size, int, 0644);
+MODULE_PARM_DESC(npu_spram_size, "npu spram size");
+
+struct platform_device *pdevs[2] = {NULL, NULL};
+const static uint32_t npu_llc_offset[2] = {NPU_LLC0_OFFSET, NPU_LLC1_OFFSET};
+
+static const struct of_device_id spram_dt_ids[] = {
+	{ .compatible = "eswin,llc" },
+	{}
+};
+
+static struct llc_cache_ops llc_ops = {
+	.llc_flush_all = NULL,
+};
+
+struct llc_user {
+	struct list_head node;
+	const char *name;
+};
+
+struct llc_user_list {
+	struct device *dev;
+	atomic_t refcount;
+	struct mutex ref_lock;
+	struct list_head head;
+};
+
+static struct llc_user_list llc_user_lists[2];
+
+struct spram_dev {
+	struct device *dev;
+	char *name;
+	void __iomem *virt_base;
+	phys_addr_t phys_addr;
+	struct gen_pool *pool;
+	void __iomem *npu_base; // The vaddr of the npu
+	int nid;
+	struct dma_heap *heap;
+	struct clk *aclk;
+	struct clk *cfg_clk;
+	struct clk *llc_clk;
+	struct clk *core_clk;
+	struct clk *mux_u_npu_core_3mux1_gfree;
+	struct clk *fixed_rate_clk_spll2_fout2;
+	struct clk *fixed_rate_clk_spll1_fout1;
+	struct reset_control *rstc_axi;
+	struct reset_control *rstc_cfg;
+	struct reset_control *rstc_core;
+	struct reset_control *rstc_llc;
+	struct regulator *npu_regulator;
+	u8 is_low_freq;
+};
+
+#define dma_buf_map		iosys_map
+#define dma_buf_map_set_vaddr	iosys_map_set_vaddr
+#define dma_buf_map_clear	iosys_map_clear
+
+#define spram_phys_to_virt(spram, phys) \
+	 (spram->virt_base + phys - spram->phys_addr)
+
+static struct llc_reg_base  llc_base[MAX_LLC_CACHE_NODE_NUM];
+
+
+static void llc_reg_write(struct spram_dev *spram, uint32_t addr, uint32_t value)
+{
+	writel(value, spram->npu_base + addr);
+}
+
+static uint32_t llc_reg_read(struct spram_dev *spram, uint32_t addr)
+{
+	return readl(spram->npu_base + addr);
+}
+
+static void llc_write(struct spram_dev *spram, uint32_t device, uint32_t addr, uint32_t value)
+{
+	llc_reg_write(spram, npu_llc_offset[device] + addr, value);
+	dev_dbg(spram->dev, "%s:addr(0x%x)=0x%x\n", __func__, npu_llc_offset[device] + addr, value);
+}
+
+static uint32_t llc_read(struct spram_dev *spram, uint32_t device, uint32_t addr)
+{
+	return llc_reg_read(spram, npu_llc_offset[device] + addr);
+}
+
+static int llc_interleave_enable(struct spram_dev *spram)
+{
+	uint32_t regval, reg;
+	struct regmap *regmap;
+	struct device *dev = spram->dev;
+	int ret = 0;
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "No syscfg phandle specified\n");
+		return PTR_ERR(regmap);
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", 1, &reg);
+	if (ret) {
+		dev_err(dev, "can't get llc interleave enable reg offset(%d)\n", ret);
+		return ret;
+	}
+
+	ret = regmap_read(regmap, reg, &regval);
+	if (ret) {
+		dev_err(dev, "failed to read llc interleave enable reg(%d)\n", ret);
+		return -EIO;
+	}
+	regval |= (1 << 0);
+	ret = regmap_write(regmap, reg, regval);
+	if (ret) {
+		dev_err(dev, "failed to write llc interleave enable reg(%d)\n", ret);
+		return -EIO;
+	}
+
+	ret = regmap_read(regmap, reg, &regval);
+	if (!ret) {
+		dev_dbg(dev, "write, interleave reg_offset=0x%x, val=0x%x\n", reg, regval);
+	}
+
+	return 0;
+
+}
+
+static int do_llc_spram_init(struct spram_dev *spram, uint32_t spram_size, uint32_t device)
+{
+	uint32_t val = 0;
+	uint32_t num_of_ways = 0;
+	uint32_t spram_num_of_ways = 0;
+
+	dev_dbg(spram->dev, "---%s\n", __func__);
+
+	//spramSzie must be intergral multiple of SIZE_OF_PER_WAY
+	if((spram_size > MAX_CACHE_SIZE)   || (spram_size % SIZE_OF_PER_WAY)) {
+		dev_err(spram->dev,"Invalid spramSize\n");
+		return -1;
+	}
+
+	num_of_ways = MAX_CACHE_SIZE/SIZE_OF_PER_WAY;
+	spram_num_of_ways = spram_size / SIZE_OF_PER_WAY;
+
+	llc_write(spram, device, CODA_CACHE_REG_CCUCMWVR, GENMASK_ULL(num_of_ways - 1, 0));
+	llc_write(spram, device, CODA_CACHE_REG_CCUCMCR, 0x0);
+	do {
+		val = llc_read(spram, device, CODA_CACHE_REG_CCUCMAR);
+		msleep(1);
+	}while(val & 0x1);
+
+	llc_write(spram, device, CODA_CACHE_REG_CCUCMCR, 0x10000);
+	do {
+		val = llc_read(spram, device, CODA_CACHE_REG_CCUCMAR);
+		msleep(1);
+	}while(val & 0x1);
+
+	llc_write(spram, device, CODA_CACHE_REG_CCUSPBR0, 0);
+	llc_write(spram, device, CODA_CACHE_REG_CCUSPBR1, 0);
+
+	if (spram_num_of_ways) {
+		llc_write(spram, device, CODA_CACHE_REG_CCUSPCR0, (spram_num_of_ways - 1) << 16); // num of  ways are used as spram
+		/*number of cachelines, taking 2048 sets as an example*/
+		llc_write(spram, device, CODA_CACHE_REG_CCUSPCR1, MAX_SETS * spram_num_of_ways - 1);
+		val = llc_read(spram, device, CODA_CACHE_REG_CCUSPCR0);
+		val |= 0x1;      //enable Spram
+		llc_write(spram, device, CODA_CACHE_REG_CCUSPCR0, val);
+	}
+
+	llc_write(spram, device, CODA_CACHE_REG_CCUCTCR, 0x3);     // enable codacache ip lookups and fill
+	llc_write(spram, device, CODA_CACHE_REG_CCUUEDR, 0x3);     // enable codacache ip error detection
+
+	/*  0xbf0007:enable codacache ip write allocation partial
+	    0xff0007:enable codacache ip write-back Read and Write-allocate
+	*/
+	llc_write(spram, device, CODA_CACHE_REG_CCUCAOR, 0xff0007);
+
+	return 0;
+}
+
+static int llc_spram_init(struct spram_dev *spram)
+{
+	dev_dbg(spram->dev, "---%s\n", __func__);
+
+	if (llc_interleave_enable(spram) < 0)
+	{
+		dev_err(spram->dev, "llc_interleave_enable error\n");
+		return -1;
+	}
+
+	if (do_llc_spram_init(spram, npu_spram_size / 2 , 0) < 0)
+	{
+		dev_err(spram->dev, "do_llc_spram_init0 error\n");
+		return -1;
+	}
+
+	if (do_llc_spram_init(spram, npu_spram_size / 2, 1) < 0)
+	{
+		dev_err(spram->dev, "do_llc_spram_init1 error\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int llc_flush_operation(unsigned long start, unsigned long len)
+{
+	if (unlikely(!llc_ops.llc_flush_all)) {
+		pr_err("LLC cache ops is NULL!!!\n");
+		return -1;
+	}
+
+	llc_ops.llc_flush_all(start, len);
+
+	return 0;
+}
+EXPORT_SYMBOL(llc_flush_operation);
+
+static void devm_llc_ops_unregister(struct device *dev, void *res)
+{
+	llc_ops.llc_flush_all = NULL;
+	dev_info(dev, "%s done!\n", __func__);
+}
+
+static int devm_llc_ops_register(struct device *dev, llc_flush_all_t llc_flush_all)
+{
+	void *ptr;
+
+	if (unlikely(!llc_flush_all))
+		return -1;
+
+	ptr = devres_alloc(devm_llc_ops_unregister, 0, GFP_KERNEL);
+	if (!ptr)
+		return -1;
+
+	llc_ops.llc_flush_all = llc_flush_all;
+
+	dev_info(dev, "%s, done sucessfully!\n", __func__);
+	devres_add(dev, ptr);
+
+	return 0;
+}
+
+#if defined(HAVE_LLC_HARDWARE)
+static bool llc_maint_is_activity(onwhich_die_t node_id)
+{
+	if (((readl(llc_base[node_id].CodaCache0_RegBase + CODA_CACHE_REG_CCUCMAR) & 0x1) == 1) || (readl(llc_base[node_id].CodaCache0_RegBase + CODA_CACHE_REG_CCUCTAR) & 0x3))
+		return true;
+
+	if (((readl(llc_base[node_id].CodaCache1_RegBase + CODA_CACHE_REG_CCUCMAR) & 0x1) == 1) || (readl(llc_base[node_id].CodaCache1_RegBase + CODA_CACHE_REG_CCUCTAR) & 0x3))
+		return true;
+
+	return false;
+}
+
+
+void llc_flush_all(unsigned long start, unsigned long len)
+{
+	int node_id = 0;
+
+	// mb();	/* sync */
+	if (start >= CONFIG_RISCV_DIE0_CACHED_OFFSET && (start + len) <= (CONFIG_RISCV_DIE0_CACHED_OFFSET + CONFIG_RISCV_DIE0_MEM_MAX_SIZE)) {
+		node_id = LLC_CACHE_NODE_0;
+	}else if (start >= CONFIG_RISCV_DIE1_CACHED_OFFSET && (start + len) <= (CONFIG_RISCV_DIE1_CACHED_OFFSET + CONFIG_RISCV_DIE1_MEM_MAX_SIZE)) {
+		node_id = LLC_CACHE_NODE_1;
+	}
+	else if (start >= CONFIG_RISCV_INTERLEAVE_CACHED_OFFSET && (start + len) <= (CONFIG_RISCV_INTERLEAVE_CACHED_OFFSET + CONFIG_RISCV_INTERLEAVE_MEM_MAX_SIZE)) {
+		node_id = -1;;
+	}
+	else {
+		WARN(1, "llc: out of range: %lx(%lx), skip flush\n",
+		     start, len);
+		return;
+	}
+
+	if (node_id == -1) {
+		writeq(0x4, (llc_base[LLC_CACHE_NODE_0].CodaCache0_RegBase + CODA_CACHE_REG_CCUCMCR));
+		writeq(0x4, (llc_base[LLC_CACHE_NODE_0].CodaCache1_RegBase + CODA_CACHE_REG_CCUCMCR));
+		writeq(0x4, (llc_base[LLC_CACHE_NODE_1].CodaCache0_RegBase + CODA_CACHE_REG_CCUCMCR));
+		writeq(0x4, (llc_base[LLC_CACHE_NODE_1].CodaCache1_RegBase + CODA_CACHE_REG_CCUCMCR));
+		while (llc_maint_is_activity(LLC_CACHE_NODE_0)) {
+			ndelay(100);
+		}
+		while (llc_maint_is_activity(LLC_CACHE_NODE_1)) {
+			ndelay(100);
+		}
+	}
+	else {
+		writeq(0x4, (llc_base[node_id].CodaCache0_RegBase + CODA_CACHE_REG_CCUCMCR));
+		writeq(0x4, (llc_base[node_id].CodaCache1_RegBase + CODA_CACHE_REG_CCUCMCR));
+		while (llc_maint_is_activity(node_id)) {
+			ndelay(100);
+		}
+	}
+	pr_debug("---%s, node_id %d, start 0x%lx, len 0x%lx\n", __func__, node_id, start, len);
+}
+#else
+static void llc_flush_all(unsigned long start, unsigned long len)
+{
+	pr_info("%s\n", __func__);
+	return;
+}
+#endif
+
+static void llc_user_init(struct spram_dev *spram)
+{
+	struct llc_user_list *llc_user_list = &llc_user_lists[spram->nid];
+
+	llc_user_list->dev = spram->dev;
+	atomic_set(&llc_user_list->refcount, 0);
+	mutex_init(&llc_user_list->ref_lock);
+	INIT_LIST_HEAD(&llc_user_list->head);
+
+	dev_dbg(spram->dev, "%s\n", __func__);
+}
+
+static void llc_user_unregister(struct device *user_dev, void *res)
+{
+	int nid = dev_to_node(user_dev);
+	struct llc_user *user, *tmp;
+	struct llc_user_list *llc_user_list = NULL;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		dev_err(user_dev, "%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return;
+	#else
+		dev_dbg(user_dev, "%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+	llc_user_list = &llc_user_lists[nid];
+
+	mutex_lock(&llc_user_list->ref_lock);
+	list_for_each_entry_safe(user, tmp, &llc_user_list->head, node) {
+		if (0 == strcmp(user->name, dev_name(user_dev))) {
+			list_del(&user->node);
+			kfree(user);
+			atomic_sub(1, &llc_user_list->refcount);
+			dev_dbg(user_dev, "llc_user_unregistered!\n");
+			break;
+		}
+	}
+	mutex_unlock(&llc_user_list->ref_lock);
+
+	dev_dbg(user_dev, "%s done!\n", __func__);
+}
+
+int llc_user_register(struct device *user_dev)
+{
+	struct device *dev;
+	int nid = dev_to_node(user_dev);
+	struct llc_user_list *llc_user_list = NULL;
+	struct llc_user *new_user = NULL;
+	int ret = 0;
+	void *ptr;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		dev_err(user_dev, "%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		dev_dbg(user_dev, "%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	llc_user_list = &llc_user_lists[nid];
+	dev = llc_user_list->dev;
+
+	mutex_lock(&llc_user_list->ref_lock);
+	new_user = kzalloc(sizeof(*new_user), GFP_KERNEL);
+	if (!new_user) {
+		dev_err(dev, "%s failed to alloc user for %s\n", __func__, dev_name(user_dev));
+		ret = -ENOMEM;
+		goto llc_user_unlock;
+	}
+	new_user->name = dev_name(user_dev);
+	list_add_tail(&new_user->node, &llc_user_list->head);
+	atomic_add(1, &llc_user_list->refcount);
+
+	ptr = devres_alloc(llc_user_unregister, 0, GFP_KERNEL);
+	if (!ptr) {
+		dev_err(dev, "%s devres_alloc failed for %s\n", __func__, dev_name(user_dev));
+		kfree(new_user);
+		ret = -ENOMEM;
+		goto llc_user_unlock;
+	}
+
+	devres_add(user_dev, ptr);
+
+llc_user_unlock:
+	mutex_unlock(&llc_user_list->ref_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(llc_user_register);
+
+int npu_cfg_rst(int nid, bool enable)
+{
+	struct platform_device *pdev = pdevs[nid];
+	struct spram_dev *spram;
+	int ret;
+
+	if (NULL == pdev) {
+		pr_err("%s, Invalid node id:%d\n", __func__, nid);
+		return -EINVAL;
+	}
+	spram = platform_get_drvdata(pdev);
+	if (spram == NULL)
+		return -EINVAL;
+
+	if (true == enable) {
+		/*reset npu cfg*/
+		ret = reset_control_deassert(spram->rstc_cfg);
+		WARN_ON(0 != ret);
+	}
+	else
+		reset_control_assert(spram->rstc_cfg);
+
+	return 0;
+}
+EXPORT_SYMBOL(npu_cfg_rst);
+
+static int npu_clk_enable(struct platform_device *pdev, struct spram_dev *spram)
+{
+	int ret;
+
+	if (!__clk_is_enabled(spram->cfg_clk)) {
+		ret = clk_prepare_enable(spram->cfg_clk);
+		if (ret) {
+			dev_err(&pdev->dev, "failed to enable cfg_clk: %d\n", ret);
+			return ret;
+		}
+	}
+
+	if (!__clk_is_enabled(spram->core_clk)) {
+		ret = clk_prepare_enable(spram->core_clk);
+		if (ret) {
+			dev_err(&pdev->dev, "failed to enable core_clk: %d\n", ret);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static int npu_clk_disable(struct platform_device *pdev, struct spram_dev *spram)
+{
+	clk_disable_unprepare(spram->core_clk);
+	clk_disable_unprepare(spram->cfg_clk);
+	return 0;
+}
+
+int npu_clk_gate_set(int nid, bool enable)
+{
+	struct platform_device *pdev = pdevs[nid];
+	struct spram_dev *spram;
+
+	if (NULL == pdev) {
+		pr_err("%s, Invalid node id:%d\n", __func__, nid);
+		return -EINVAL;
+	}
+
+	spram = platform_get_drvdata(pdev);
+	if (spram == NULL)
+		return -EINVAL;
+
+	if (enable == true) {
+		return npu_clk_enable(pdev, spram);
+	} else if (enable == false) {
+		return npu_clk_disable(pdev, spram);
+	} else {
+		pr_err("param enable=%d error.\n", enable);
+		return -EINVAL;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(npu_clk_gate_set);
+
+int npu_core_rst(int nid, bool enable)
+{
+	struct platform_device *pdev = pdevs[nid];
+	struct spram_dev *spram;
+	int ret;
+
+	if (NULL == pdev) {
+		pr_err("%s, Invalid node id:%d\n", __func__, nid);
+		return -EINVAL;
+	}
+	spram = platform_get_drvdata(pdev);
+	if (spram == NULL)
+		return -EINVAL;
+
+	if (true == enable) {
+		/*reset npu cfg*/
+		ret = reset_control_deassert(spram->rstc_core);
+		WARN_ON(0 != ret);
+	}
+	else
+		reset_control_assert(spram->rstc_core);
+
+	return 0;
+}
+EXPORT_SYMBOL(npu_core_rst);
+
+int llc_spram_avail_size(int nid, uint32_t *pSpramSize)
+{
+	struct platform_device *pdev = pdevs[nid];
+	struct spram_dev *spram;
+
+	if (NULL == pdev) {
+		pr_err("%s, Invalid node id:%d\n", __func__, nid);
+		return -EINVAL;
+	}
+	spram = platform_get_drvdata(pdev);
+	if (spram == NULL)
+		return -EINVAL;
+
+	*pSpramSize = gen_pool_avail(spram->pool);
+
+	return 0;
+}
+EXPORT_SYMBOL(llc_spram_avail_size);
+
+static int spram_proc_show(struct seq_file *m, void *v)
+{
+	struct spram_dev *spram = m->private;
+	int nid = spram->nid;
+	struct llc_user_list *llc_user_list = &llc_user_lists[nid];
+	struct llc_user *user;
+
+	seq_printf(m, "SRAM pool: %zu KiB, Available: %zu KiB\n",
+		gen_pool_size(spram->pool) / 1024,
+		gen_pool_avail(spram->pool) / 1024);
+#if 1
+	seq_printf(m, "LLC Users(%d): \n", atomic_read(&llc_user_list->refcount));
+	list_for_each_entry(user, &llc_user_list->head, node) {
+		seq_printf(m, "%s\n", user->name);
+	}
+#endif
+	return 0;
+}
+
+static int __init proc_spram_init(struct spram_dev *spram)
+{
+	char proc_name[64];
+
+	sprintf(proc_name, "%s_info", spram->name);
+	dev_dbg(spram->dev, "%s, proc_name:%s\n", __func__, proc_name);
+	if (NULL == proc_create_single_data(proc_name, 0, NULL, spram_proc_show, spram)) {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int llc_clk_init(struct platform_device *pdev)
+{
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+	int ret = 0;
+
+	if (spram == NULL)
+		return -EINVAL;
+
+	spram->aclk = devm_clk_get(&pdev->dev, "aclk");
+	if (IS_ERR(spram->aclk)) {
+		ret = PTR_ERR(spram->aclk);
+		spram->aclk = NULL;
+		dev_err(&pdev->dev, "failed to get aclk: %d\n", ret);
+		return ret;
+	}
+
+	spram->cfg_clk = devm_clk_get(&pdev->dev, "cfg_clk");
+	if (IS_ERR(spram->cfg_clk)) {
+		ret = PTR_ERR(spram->cfg_clk);
+		spram->cfg_clk = NULL;
+		dev_err(&pdev->dev, "failed to get cfg_clk: %d\n", ret);
+		return ret;
+	}
+
+	spram->llc_clk = devm_clk_get(&pdev->dev, "llc_clk");
+	if (IS_ERR(spram->llc_clk)) {
+		ret = PTR_ERR(spram->llc_clk);
+		spram->llc_clk = NULL;
+		dev_err(&pdev->dev, "failed to get llc_clk: %d\n", ret);
+		return ret;
+	}
+
+
+
+	spram->core_clk = devm_clk_get(&pdev->dev, "core_clk");
+	if (IS_ERR(spram->core_clk)) {
+		ret = PTR_ERR(spram->core_clk);
+		spram->core_clk = NULL;
+		dev_err(&pdev->dev, "failed to get core_clk: %d\n", ret);
+		return ret;
+	}
+
+	spram->mux_u_npu_core_3mux1_gfree = devm_clk_get(&pdev->dev, "mux_u_npu_core_3mux1_gfree");
+	if (IS_ERR(spram->mux_u_npu_core_3mux1_gfree)) {
+		ret = PTR_ERR(spram->mux_u_npu_core_3mux1_gfree);
+		dev_err(&pdev->dev, "failed to get mux_u_npu_core_3mux1_gfree: %d\n", ret);
+		return ret;
+	}
+
+	spram->fixed_rate_clk_spll2_fout2 = devm_clk_get(&pdev->dev, "fixed_rate_clk_spll2_fout2");
+	if (IS_ERR(spram->fixed_rate_clk_spll2_fout2)) {
+		ret = PTR_ERR(spram->fixed_rate_clk_spll2_fout2);
+		dev_err(&pdev->dev, "failed to get fixed_rate_clk_spll2_fout2: %d\n", ret);
+		return ret;
+	}
+	spram->fixed_rate_clk_spll1_fout1 =
+		devm_clk_get(&pdev->dev, "fixed_rate_clk_spll1_fout1");
+	if (IS_ERR(spram->fixed_rate_clk_spll1_fout1))
+	{
+		ret = PTR_ERR(spram->fixed_rate_clk_spll1_fout1);
+		dev_err(&pdev->dev, "failed to get fixed_rate_clk_spll1_fout1: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int llc_clk_enable(struct spram_dev *spram)
+{
+	int ret = 0;
+
+	if (spram == NULL)
+		return -EINVAL;
+
+	/*enable clk*/
+	ret = clk_prepare_enable(spram->aclk);
+	if (ret) {
+		dev_err(spram->dev, "failed to enable aclk: %d\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(spram->cfg_clk);
+	if (ret) {
+		dev_err(spram->dev, "failed to enable cfg_clk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(spram->llc_clk);
+	if (ret) {
+		dev_err(spram->dev, "failed to enable llc_clk: %d\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(spram->core_clk);
+	if (ret) {
+		dev_err(spram->dev, "failed to enable core_clk: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int llc_clk_disable(struct spram_dev *spram)
+{
+	if (spram == NULL)
+		return -EINVAL;
+	clk_disable_unprepare(spram->aclk);
+	clk_disable_unprepare(spram->cfg_clk);
+	clk_disable_unprepare(spram->llc_clk);
+	clk_disable_unprepare(spram->core_clk);
+
+	return 0;
+}
+
+static int llc_rst_init(struct platform_device *pdev)
+{
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+
+	if (spram == NULL)
+		return -EINVAL;
+
+	spram->rstc_axi = devm_reset_control_get_optional(&pdev->dev, "axi");
+	if (IS_ERR_OR_NULL(spram->rstc_axi)) {
+		dev_err(&pdev->dev, "Failed to get cfg reset handle\n");
+		return -EFAULT;
+	}
+
+	spram->rstc_cfg = devm_reset_control_get_optional(&pdev->dev, "cfg");
+	if (IS_ERR_OR_NULL(spram->rstc_cfg)) {
+		dev_err(&pdev->dev, "Failed to get llc reset handle\n");
+		return -EFAULT;
+	}
+
+	spram->rstc_core = devm_reset_control_get_optional(&pdev->dev, "core");
+	if (IS_ERR_OR_NULL(spram->rstc_core)) {
+		dev_err(&pdev->dev, "Failed to get cfg reset handle\n");
+		return -EFAULT;
+	}
+
+	spram->rstc_llc = devm_reset_control_get_optional(&pdev->dev, "llc");
+	if (IS_ERR_OR_NULL(spram->rstc_llc)) {
+		dev_err(&pdev->dev, "Failed to get llc reset handle\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int llc_clk_set_parent(struct platform_device *pdev)
+{
+	int ret;
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+	struct device_node *np;
+	struct device *dev = &pdev->dev;
+
+	spram->is_low_freq = 0;
+	if (spram == NULL)
+		return -EINVAL;
+
+	np = of_node_get(dev->of_node);
+
+	spram->npu_regulator = devm_regulator_get(dev, "npu");
+	if (IS_ERR_OR_NULL(spram->npu_regulator)) {
+		dev_err(dev, "failed to get npu regulator!\n");
+		return -ENODEV;
+	}
+
+	ret = regulator_get_voltage(spram->npu_regulator);
+	if (ret < 0) {
+		dev_warn(dev, "failed to get npu regulator,the npu freq will set to 1G\n");
+		spram->is_low_freq = 1;
+	} else {
+		spram->is_low_freq = (of_property_read_bool(np, "apply_npu_1G_freq"));
+		dev_dbg(dev, "success to get npu regulator,apply_npu_1G_freq:%d\n",
+				 spram->is_low_freq);
+		ret = regulator_enable(spram->npu_regulator);
+		if (ret < 0) {
+			return ret;
+		}
+	}
+	return 0;
+}
+static int llc_clk_set_frq(struct platform_device *pdev)
+{
+	int ret;
+	unsigned long rate = 0;
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+
+	if (spram == NULL)
+		return -EINVAL;
+
+	rate = clk_round_rate(spram->aclk, NPU_ACLK_RATE);
+	ret = clk_set_rate(spram->aclk, rate);
+	if (ret != 0)
+	{
+		dev_err(&pdev->dev, "failed to set aclk: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int llc_rst_deassert(struct spram_dev *spram)
+{
+	int ret = 0;
+
+	if (spram == NULL)
+		return -EINVAL;
+
+	/*reset npu bus*/
+	ret = reset_control_deassert(spram->rstc_axi);
+	WARN_ON(0 != ret);
+
+	/*reset npu core*/
+	ret = reset_control_deassert(spram->rstc_core);
+	WARN_ON(0 != ret);
+
+	/*reset npu llc*/
+	ret = reset_control_deassert(spram->rstc_llc);
+	WARN_ON(0 != ret);
+
+	/*reset npu cfg*/
+	ret = reset_control_deassert(spram->rstc_cfg);
+	WARN_ON(0 != ret);
+
+	return 0;
+}
+
+
+static int llc_rst_assert(struct spram_dev *spram)
+{
+	int ret = 0;
+
+	if (spram == NULL)
+		return -EINVAL;
+	ret = reset_control_assert(spram->rstc_axi);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(spram->rstc_core);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(spram->rstc_llc);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(spram->rstc_cfg);
+	WARN_ON(0 != ret);
+
+	return 0;
+}
+
+static int llc_clk_rst_print(struct platform_device *pdev)
+{
+	uint32_t regval[5];
+	struct regmap *regmap;
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+	struct device *dev;
+	int ret = 0;
+
+	if (spram == NULL)
+		return -EINVAL;
+
+	dev = spram->dev;
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscrg_csr");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "No syscrg phandle specified\n");
+		return PTR_ERR(regmap);
+	}
+
+	ret = regmap_read(regmap, 0x178, &regval[0]);
+	if (ret) {
+		dev_err(dev, "%s:%d, failed to read reg(%d)\n", __func__, __LINE__, ret);
+		return -EIO;
+	}
+
+	/* llc clken*/
+	ret = regmap_read(regmap, 0x17c, &regval[1]);
+	if (ret) {
+		dev_err(dev, "%s:%d, failed to read reg(%d)\n", __func__, __LINE__, ret);
+		return -EIO;
+	}
+
+	ret = regmap_read(regmap, 0x180, &regval[2]);
+	if (ret) {
+		dev_err(dev, "%s:%d, failed to read reg(%d)\n", __func__, __LINE__, ret);
+		return -EIO;
+	}
+
+	ret = regmap_read(regmap, 0x184, &regval[3]);
+	if (ret) {
+		dev_err(dev, "%s:%d, failed to read reg(%d)\n", __func__, __LINE__, ret);
+		return -EIO;
+	}
+
+	ret = regmap_read(regmap, 0x418, &regval[4]);
+	if (ret) {
+		dev_err(dev, "%s:%d, failed to read reg(%d)\n", __func__, __LINE__, ret);
+		return -EIO;
+	}
+
+	dev_info(dev, "[0x178]=0x%08x [0x17c]=0x%08x [0x180]=0x%08x [0x184]=0x%08x [0x418]=0x%08x\n",
+			regval[0], regval[1], regval[2], regval[3],regval[4]);
+
+	return 0;
+}
+
+/* Release the reset signal to let llc run */
+static int llc_clk_rst_init(struct platform_device *pdev)
+{
+	int ret = 0;
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+
+	spram->is_low_freq = 0;
+	dev_dbg(&pdev->dev, "---%s\n", __func__);
+
+	ret = llc_clk_init(pdev);
+	if(ret != 0){
+		dev_err(&pdev->dev, "llc_clk_init error: %d\n", ret);
+		return ret;
+	}
+
+	ret = llc_clk_set_parent(pdev);
+	if(ret != 0){
+		dev_err(&pdev->dev, "llc_clk_set_parent error: %d\n", ret);
+		return ret;
+	}
+
+	ret = llc_rst_init(pdev);
+	if(ret != 0){
+		dev_err(&pdev->dev, "llc_rst_init error: %d\n", ret);
+		return ret;
+	}
+
+	ret = llc_clk_set_frq(pdev);
+	if(ret != 0){
+		dev_err(&pdev->dev, "llc_clk_set_frq error: %d\n", ret);
+		return ret;
+	}
+
+	ret = llc_clk_enable(spram);
+	if(ret != 0){
+		dev_err(&pdev->dev, "llc_clk_enable error: %d\n", ret);
+		return ret;
+	}
+
+	llc_rst_deassert(spram);
+
+	llc_clk_rst_print(pdev);
+	dev_dbg(&pdev->dev, "%s done successfully!\n", __func__);
+
+	return ret;
+}
+
+static int llc_resource_parse(struct platform_device *pdev)
+{
+	struct spram_dev *spram = platform_get_drvdata(pdev);
+	struct device* dev = &pdev->dev;
+	struct resource *res;
+	struct resource res_spram;
+	struct device_node *spram_node;
+	int nid = dev_to_node(dev);
+	void __iomem *npu_base;
+	struct page *page = NULL;
+	phys_addr_t phys;
+	int ret = 0;
+
+	dev_dbg(dev, "----%s:%d\n", __func__, __LINE__);
+
+	#ifdef CONFIG_NUMA
+	nid = dev_to_node(dev);
+	if (nid == NUMA_NO_NODE) {
+		dev_err(dev, "%s:%d, numa-node-id was not defined!\n", __func__, __LINE__);
+		return -EFAULT;
+	}
+	#else
+	if (of_property_read_s32(dev->of_node, "numa-node-id", &nid)) {
+		dev_err(dev, "%s:%d, numa-node-id was not defined!\n", __func__, __LINE__);
+		return -EFAULT;
+	}
+	#endif
+	spram->nid = nid;
+
+	spram_node = of_parse_phandle(dev->of_node, "spram-region", 0);
+	of_address_to_resource(spram_node, 0, &res_spram);
+
+	if (npu_spram_size > resource_size(&res_spram)) {
+		dev_err(dev, "Invalid spram size(0x%x), max spram size is 0x%x\n",
+			npu_spram_size, (unsigned int)resource_size(&res_spram));
+		ret = -EINVAL;
+		goto out_spram_region;
+	}
+
+	dev_info(dev, "res_spram.start=0x%x, max_size=0x%x, configured size=0x%x\n",
+		(unsigned int)res_spram.start,
+		(unsigned int)resource_size(&res_spram),
+		npu_spram_size);
+
+	spram->phys_addr = res_spram.start;
+	spram->virt_base = devm_ioremap(&pdev->dev, spram->phys_addr, npu_spram_size);
+	if (spram->virt_base == NULL) {
+		dev_err(dev, "failed to ioremap() spram-region\n");
+		ret = -ENODEV;
+		goto out_spram_region;
+	}
+	dev_dbg(dev, "---%s:%d, spram->virt_base=0x%px\n",
+		__func__, __LINE__, spram->virt_base);
+
+	page = phys_to_page(spram->phys_addr);
+	phys = page_to_phys(page);
+	dev_dbg(dev, "---%s:%d, spram->phys_addr=0x%x, phys_to_page->page_to_phys:0x%x\n",
+		__func__, __LINE__, (unsigned int)spram->phys_addr, (unsigned int)phys);
+
+	dev_dbg(dev, "%s:%d,---page:0x%px\n", __func__, __LINE__, page);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dev_dbg(&pdev->dev, "npu_base resource phys_start=0x%llx, size=0x%llx\n", res->start, resource_size(res));
+	npu_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	if (IS_ERR(npu_base)) {
+		dev_err(&pdev->dev, "could not map NPU registers\n");
+		ret = PTR_ERR(npu_base);
+		goto out_spram_region;
+	}
+	spram->npu_base = npu_base;
+
+	llc_base[nid].CodaCache0_RegBase = npu_base + NPU_LLC0_OFFSET;
+	llc_base[nid].CodaCache1_RegBase = npu_base + NPU_LLC1_OFFSET;
+
+out_spram_region:
+	of_node_put(spram_node);
+
+	return ret;
+}
+struct spram_heap {
+	struct dma_heap *heap;
+	struct spram_dev *spram;
+};
+
+struct spram_heap_buffer {
+	struct spram_heap *heap;
+	struct list_head attachments;
+	struct mutex lock;
+	unsigned long len;
+	struct sg_table sg_table;
+	int vmap_cnt;
+	void *vaddr;
+};
+
+struct dma_heap_attachment {
+	struct device *dev;
+	/* For spram, sgt is useless infact, just to adapt to the dma-buf.c */
+	struct sg_table *table;
+	phys_addr_t phys_addr;
+	struct list_head list;
+	bool mapped;
+};
+
+static struct sg_table *dup_sg_table(struct sg_table *table)
+{
+	struct sg_table *new_table;
+	int ret, i;
+	struct scatterlist *sg, *new_sg;
+
+	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
+	if (!new_table)
+		return ERR_PTR(-ENOMEM);
+
+	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
+	if (ret) {
+		kfree(new_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_sg = new_table->sgl;
+	for_each_sgtable_sg(table, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+	return new_table;
+}
+
+static int spram_heap_attach(struct dma_buf *dmabuf,
+			   struct dma_buf_attachment *attachment)
+{
+	struct spram_heap_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a;
+	struct sg_table *table;
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a)
+		return -ENOMEM;
+
+	table = dup_sg_table(&buffer->sg_table);
+	if (IS_ERR(table)) {
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	a->table = table;
+	a->dev = attachment->dev;
+	INIT_LIST_HEAD(&a->list);
+	a->mapped = false;
+
+	attachment->priv = a;
+
+	mutex_lock(&buffer->lock);
+	list_add(&a->list, &buffer->attachments);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static void spram_heap_detach(struct dma_buf *dmabuf,
+			    struct dma_buf_attachment *attachment)
+{
+	struct spram_heap_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a = attachment->priv;
+
+	mutex_lock(&buffer->lock);
+	list_del(&a->list);
+	mutex_unlock(&buffer->lock);
+
+	sg_free_table(a->table);
+	kfree(a->table);
+	kfree(a);
+}
+
+static struct sg_table *spram_heap_map_dma_buf(struct dma_buf_attachment *attachment,
+					     enum dma_data_direction direction)
+{
+	struct dma_heap_attachment *a = attachment->priv;
+	struct sg_table *table = a->table;
+	int ret;
+
+	ret = dma_map_sgtable(attachment->dev, table, direction, DMA_ATTR_SKIP_CPU_SYNC);
+	if (ret)
+		return ERR_PTR(-ENOMEM);
+
+
+	a->mapped = true;
+	return table;
+}
+
+static void spram_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
+				      struct sg_table *table,
+				      enum dma_data_direction direction)
+{
+	struct dma_heap_attachment *a = attachment->priv;
+
+	a->mapped = false;
+	dma_unmap_sgtable(attachment->dev, table, direction, DMA_ATTR_SKIP_CPU_SYNC);
+}
+
+static int spram_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+						enum dma_data_direction direction)
+{
+	/* Do Nothing, because spram is uncached memory space */
+	return 0;
+}
+
+static int spram_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
+					      enum dma_data_direction direction)
+{
+	/* Do Nothing, because spram is uncached memory space */
+	return 0;
+}
+
+static int spram_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct spram_heap_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	unsigned long addr = vma->vm_start;
+	struct sg_page_iter piter;
+	int ret;
+
+	/* vm_private_data will be used by eswin-ipc-scpu.c to get iova and
+	   by mmz_vb.c to retrieve mem node
+	*/
+	vma->vm_private_data = dmabuf;
+
+	for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
+		struct page *page = sg_page_iter_page(&piter);
+		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
+				      vma->vm_page_prot);
+		if (ret)
+			return ret;
+		addr += PAGE_SIZE;
+		if (addr >= vma->vm_end)
+			return 0;
+	}
+	return 0;
+}
+
+static void *spram_heap_do_vmap(struct spram_heap_buffer *buffer)
+{
+	struct sg_table *table = &buffer->sg_table;
+	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
+	struct page **pages = vmalloc(sizeof(struct page *) * npages);
+	struct page **tmp = pages;
+	struct page *page;
+	struct sg_page_iter piter;
+	void *vaddr;
+
+	if (!pages)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_sgtable_page(table, &piter, 0) {
+		WARN_ON(tmp - pages >= npages);
+		*tmp++ = sg_page_iter_page(&piter);
+		page = sg_page_iter_page(&piter);
+	}
+
+	vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
+	vfree(pages);
+
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+static int spram_heap_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct spram_heap_buffer *buffer = dmabuf->priv;
+	void *vaddr;
+	int ret = 0;
+
+	mutex_lock(&buffer->lock);
+	if (buffer->vmap_cnt) {
+		buffer->vmap_cnt++;
+		dma_buf_map_set_vaddr(map, buffer->vaddr);
+		goto out;
+	}
+
+	vaddr = spram_heap_do_vmap(buffer);
+
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto out;
+	}
+
+	buffer->vaddr = vaddr;
+	buffer->vmap_cnt++;
+	dma_buf_map_set_vaddr(map, buffer->vaddr);
+out:
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static void spram_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct spram_heap_buffer *buffer = dmabuf->priv;
+
+	mutex_lock(&buffer->lock);
+	if (!--buffer->vmap_cnt) {
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+	mutex_unlock(&buffer->lock);
+	dma_buf_map_clear(map);
+}
+
+static void spram_heap_dma_buf_release(struct dma_buf *dmabuf)
+{
+	struct spram_heap_buffer *buffer = dmabuf->priv;
+	struct sg_table *table;
+	struct spram_dev *spram = buffer->heap->spram;
+	struct scatterlist *sg;
+	int i;
+
+	table = &buffer->sg_table;
+	for_each_sgtable_sg(table, sg, i) {
+		struct page *page = sg_page(sg);
+		#ifdef WANT_PAGE_VIRTUAL
+		void *vaddr = page_address(page);
+		#else
+		void *vaddr = spram_phys_to_virt(spram, page_to_phys(page));
+		#endif
+		gen_pool_free(spram->pool, (unsigned long)vaddr, page_size(page));
+	}
+	sg_free_table(table);
+	kfree(buffer);
+}
+
+static const struct dma_buf_ops spram_heap_buf_ops = {
+	.attach = spram_heap_attach,
+	.detach = spram_heap_detach,
+	.map_dma_buf = spram_heap_map_dma_buf,
+	.unmap_dma_buf = spram_heap_unmap_dma_buf,
+	.begin_cpu_access = spram_heap_dma_buf_begin_cpu_access,
+	.end_cpu_access = spram_heap_dma_buf_end_cpu_access,
+	.mmap = spram_heap_mmap,
+	.vmap = spram_heap_vmap,
+	.vunmap = spram_heap_vunmap,
+	.release = spram_heap_dma_buf_release,
+};
+
+static int spram_noncontiguous_alloc(struct spram_dev *spram, size_t len, struct sg_table *table)
+{
+	struct gen_pool *pool = spram->pool;
+	struct list_head pages;
+	struct page *page, *tmp_page;
+	struct scatterlist *sg;
+	unsigned long size_remaining = len;
+	phys_addr_t phys_addr;
+	void *vaddr;
+	int i, ret = -ENOMEM;
+
+	INIT_LIST_HEAD(&pages);
+	i = 0;
+	while (size_remaining > 0) {
+		/*
+		 * Avoid trying to allocate memory if the process
+		 * has been killed by SIGKILL
+		 */
+		if (fatal_signal_pending(current)) {
+			ret = -EINTR;
+			goto free_spram;
+		}
+
+		vaddr = gen_pool_dma_alloc(pool, PAGE_SIZE, &phys_addr);
+		if (!vaddr)
+			goto free_spram;
+
+		page = phys_to_page(phys_addr);
+		pr_debug("---%s:%d, phys_to_page->page_to_phys:0x%x,page:0x%px\n",
+			__func__, __LINE__, (unsigned int)page_to_phys(page), page);
+		/* page->virtual is used for recording the gen pool vaddr which is needed when releasing spram memory */
+		#ifdef WANT_PAGE_VIRTUAL
+		page->virtual = vaddr;
+		set_page_address(page, vaddr);
+		#endif
+
+		list_add_tail(&page->lru, &pages);
+		size_remaining -= PAGE_SIZE;
+		i++;
+	}
+
+	if (sg_alloc_table(table, i, GFP_KERNEL))
+		goto free_spram;
+
+	sg = table->sgl;
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		sg_set_page(sg, page, PAGE_SIZE, 0);
+		sg = sg_next(sg);
+		list_del(&page->lru);
+	}
+
+	return 0;
+free_spram:
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		#ifdef WANT_PAGE_VIRTUAL
+		vaddr = page_address(page);
+		#else
+		vaddr = spram_phys_to_virt(spram, page_to_phys(page));
+		#endif
+		gen_pool_free(pool, (unsigned long)vaddr, PAGE_SIZE);
+	}
+	return ret;
+}
+
+#if 0
+static int spram_contiguous_alloc(struct spram_dev *spram, size_t len, struct sg_table *table)
+{
+	struct gen_pool *pool = spram->pool;
+	int npages = PAGE_ALIGN(len) / PAGE_SIZE;
+	void *vaddr, *page_vaddr;
+	phys_addr_t phys_addr;
+	struct page *page;
+	struct scatterlist *sg;
+	int i, ret = -ENOMEM;
+
+	vaddr =  gen_pool_dma_alloc(pool, len, &phys_addr);
+	if (!vaddr) {
+		return ret;
+	}
+
+	if (sg_alloc_table(table, npages, GFP_KERNEL))
+		goto free_spram;
+
+	page_vaddr = vaddr;
+	sg = table->sgl;
+	for(i = 0; i < npages; i++) {
+		page = phys_to_page(phys_addr);
+		/* page->virtual is used for recording the gen pool vaddr which is needed when
+		* releasing spram memory
+		*/
+		#ifdef WANT_PAGE_VIRTUAL
+		set_page_address(page, page_vaddr);
+		#endif
+		sg_set_page(sg, page, PAGE_SIZE, 0);
+		sg = sg_next(sg);
+		page_vaddr += PAGE_SIZE;
+		phys_addr += PAGE_SIZE;
+	}
+
+	return 0;
+
+free_spram:
+	gen_pool_free(pool, (unsigned long)vaddr, len);
+	return ret;
+}
+#endif
+
+#ifdef CONFIG_PM
+static int __maybe_unused llc_suspend(struct device *dev)
+{
+
+	struct spram_dev *spram = dev_get_drvdata(dev);
+
+	llc_rst_assert(spram);
+	if (!pm_runtime_status_suspended(dev))
+	{
+		llc_clk_disable(spram);
+	}
+	if ((NULL != spram->npu_regulator) && (!IS_ERR(spram->npu_regulator)))
+	{
+		regulator_disable(spram->npu_regulator);
+	}
+
+	return 0;
+}
+
+static int __maybe_unused llc_resume(struct device *dev)
+{
+	int ret = 0;
+	int is_enable = 0;
+	struct spram_dev *spram = dev_get_drvdata(dev);
+
+	if (!IS_ERR_OR_NULL(spram->npu_regulator)) {
+		is_enable = regulator_is_enabled(spram->npu_regulator);
+		if (0 == is_enable) {
+			mdelay(20);
+		}
+		ret = regulator_enable(spram->npu_regulator);
+		if (ret < 0) {
+			dev_err(spram->dev, "regulator_enable error: %d\n", ret);
+			return ret;
+		}
+		if (0 == is_enable) {
+			mdelay(20);
+		}
+	}
+
+	ret = llc_clk_enable(spram);
+	if (ret != 0) {
+		dev_err(spram->dev, "llc_clk_enable error: %d\n", ret);
+		return ret;
+	}
+	ret = llc_rst_deassert(spram);
+	if (ret)
+		return ret;
+
+	ret = llc_spram_init(spram);
+	if (ret) {
+		return ret;
+	}
+
+	return ret;
+}
+
+
+static int __maybe_unused llc_runtime_suspend(struct device *dev)
+{
+	struct spram_dev *spram = dev_get_drvdata(dev);
+
+	llc_clk_disable(spram);
+
+	return 0;
+}
+
+static int __maybe_unused llc_runtime_resume(struct device *dev)
+{
+	struct spram_dev *spram = dev_get_drvdata(dev);
+	int ret = 0;
+
+	ret = llc_clk_enable(spram);
+	if(ret != 0){
+		dev_err(spram->dev, "llc_clk_enable error: %d\n", ret);
+	}
+
+	return ret;
+}
+
+static const struct dev_pm_ops llc_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(llc_suspend, llc_resume)
+	SET_RUNTIME_PM_OPS(llc_runtime_suspend,
+				   llc_runtime_resume, NULL)
+};
+
+#define DEV_PM_OPS (&llc_dev_pm_ops)
+#else
+#define DEV_PM_OPS NULL
+#endif /* CONFIG_PM */
+
+static struct dma_buf *spram_heap_allocate(struct dma_heap *heap,
+					 unsigned long len,
+					 unsigned long fd_flags,
+					 unsigned long heap_flags)
+{
+	struct spram_heap *spram_heap = dma_heap_get_drvdata(heap);
+	struct spram_dev *spram = spram_heap->spram;
+	struct spram_heap_buffer *buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct dma_buf *dmabuf;
+	struct sg_table *table;
+	struct scatterlist *sg;
+	int i, ret = -ENOMEM;
+
+	if (!PAGE_ALIGNED(len)) {
+		dev_err(spram->dev, "Err, request len is NOT aligned with PAGE_SIZE\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&buffer->attachments);
+	mutex_init(&buffer->lock);
+	buffer->heap = spram_heap;
+	buffer->len = len;
+
+	table = &buffer->sg_table;
+
+	/* Todo: contiguous alloc flag is not permitted by dma-heap. */
+#if 0
+	if ((heap_flags & DMA_HEAP_VALID_HEAP_FLAGS) == HEAP_FLAGS_SPRAM_FORCE_CONTIGUOUS) {
+		dev_dbg(spram->dev, "force contiguous allocation!\n");
+		ret = spram_contiguous_alloc(spram, len, table);
+	}
+	else
+#endif
+	{
+		dev_dbg(spram->dev, "non-contiguous allocation!\n");
+		ret = spram_noncontiguous_alloc(spram, len, table);
+	}
+
+	if (ret) {
+		dev_err(spram->dev, "failed to alloc spram, Errcode:%d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	/* create the dmabuf */
+	exp_info.exp_name = dma_heap_get_name(heap);
+	exp_info.ops = &spram_heap_buf_ops;
+	exp_info.size = buffer->len;
+	exp_info.flags = fd_flags;
+	exp_info.priv = buffer;
+	dmabuf = dma_buf_export(&exp_info);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto free_pages;
+	}
+
+	dev_dbg(spram->dev, "%s successfully!\n", __func__);
+
+	return dmabuf;
+
+free_pages:
+	for_each_sgtable_sg(table, sg, i) {
+		struct page *page = sg_page(sg);
+		#ifdef WANT_PAGE_VIRTUAL
+		void *vaddr = page_address(page);
+		#else
+		void *vaddr = spram_phys_to_virt(spram, page_to_phys(page));
+		#endif
+		gen_pool_free(spram->pool, (unsigned long)vaddr, PAGE_SIZE);
+	}
+	sg_free_table(table);
+	kfree(buffer);
+
+	dev_err(spram->dev, "%s:%d, failed to alloc spram buffer, ret=%d\n", __func__, __LINE__, ret);
+
+	return ERR_PTR(ret);
+}
+
+static const struct dma_heap_ops spram_heap_ops = {
+	.allocate = spram_heap_allocate,
+};
+
+static int __add_spram_heap(struct spram_dev *spram, void *data)
+{
+	struct spram_heap *spram_heap;
+	struct dma_heap_export_info exp_info;
+
+	spram_heap = devm_kzalloc(spram->dev, sizeof(*spram_heap), GFP_KERNEL);
+	if (!spram_heap)
+		return -ENOMEM;
+	spram_heap->spram = spram;
+
+	exp_info.name = spram->name;
+	exp_info.ops = &spram_heap_ops;
+	exp_info.priv = spram_heap;
+
+	dev_dbg(spram->dev, "%s:%d, spram->name:%s\n", __func__, __LINE__, spram->name);
+
+	spram_heap->heap = dma_heap_add(&exp_info);
+	if (IS_ERR(spram_heap->heap)) {
+		int ret = PTR_ERR(spram_heap->heap);
+
+		return ret;
+	}
+
+	/* spram->heap is only for the purpose of deleting heap when driver removed */
+	spram->heap = spram_heap->heap;
+
+	dev_info(spram->dev, "%s, Done adding spram heap name:%s\n", __func__, exp_info.name);
+
+	return 0;
+}
+
+static ssize_t npu_regulator_show(struct device *device,
+			      struct device_attribute *attr, char *buf)
+{
+	struct spram_dev *spram = dev_get_drvdata(device);
+
+	return sprintf(buf, "%d\n",regulator_is_enabled(spram->npu_regulator));
+
+}
+
+static ssize_t npu_regulator_store(struct device *device,
+			       struct device_attribute *attr, const char *buf,
+			       size_t count)
+{
+	int ret = 0;
+	struct spram_dev *spram = dev_get_drvdata(device);
+
+	if (!strncmp(buf, "0", 1))
+	{
+		regulator_disable(spram->npu_regulator);
+	}
+	else
+	{
+		ret = regulator_enable(spram->npu_regulator);
+		if (ret < 0)
+		{
+			return ret;
+		}
+
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(npu_regulator);
+
+static struct attribute *llc_attrs[] = {
+	&dev_attr_npu_regulator.attr,
+	NULL,
+};
+
+static struct attribute_group llc_attr_group = {
+	.name = NULL, /* we want them in the same directory */
+	.attrs = llc_attrs,
+};
+
+static int llc_probe(struct platform_device *pdev)
+{
+	struct spram_dev *spram;
+	int ret = 0;
+
+	dev_info(&pdev->dev, "%s start!\n", __func__);
+
+	spram = devm_kzalloc(&pdev->dev, sizeof(*spram), GFP_KERNEL);
+	if (!spram)
+		return -ENOMEM;
+
+	spram->dev = &pdev->dev;
+
+	platform_set_drvdata(pdev, spram);
+
+	ret = llc_resource_parse(pdev);
+	if (ret) {
+		return ret;
+	}
+
+	#if defined(CONFIG_RISCV) && defined(HAVE_LLC_HARDWARE)
+	/* Init llc controller */
+	ret = llc_clk_rst_init(pdev);
+	if (ret)
+		return ret;
+
+	ret = llc_spram_init(spram);
+	if (ret) {
+		return ret;
+	}
+	#endif
+	if (devm_llc_ops_register(&pdev->dev, llc_flush_all)) {
+		dev_err(&pdev->dev, "register llc ops failed!!!\n");
+		return -EFAULT;
+	}
+
+	/* Create spram pool */
+
+	spram->pool = devm_gen_pool_create(spram->dev, ilog2(SPRAM_GRANULARITY),
+						dev_to_node(spram->dev), NULL);
+	if (IS_ERR(spram->pool)) {
+		dev_err(spram->dev, "devm_gen_pool_create() failed!!!\n");
+		return PTR_ERR(spram->pool);
+	}
+	ret = gen_pool_add_virt(spram->pool, (unsigned long)spram->virt_base,
+				spram->phys_addr, npu_spram_size, dev_to_node(spram->dev));
+	if (ret < 0) {
+		dev_err(spram->dev, "gen_pool_add_virt failed with %d\n", ret);
+		return ret;
+	}
+
+	if (spram->pool)
+		dev_info(spram->dev, "SRAM pool: %zu KiB @ vaddr 0x%px, phys 0x%x\n",
+			gen_pool_size(spram->pool) / 1024, spram->virt_base, (unsigned int)spram->phys_addr);
+
+	// vaddr = gen_pool_dma_alloc(spram->pool, PAGE_SIZE, &dma);
+	// dev_info(spram->dev, "gen_pool_alloc, vaddr=0x%px, dma=0x%x\n", vaddr, dma);
+
+	spram->name = devm_kasprintf(spram->dev, GFP_KERNEL, "%s%d", DEVICE_NAME, spram->nid);
+	ret = __add_spram_heap(spram, NULL);
+	if (ret) {
+		dev_err(spram->dev, "failed to add spram heap\n");
+		return ret;
+	}
+
+	llc_user_init(spram);
+
+	proc_spram_init(spram);
+
+	#ifdef WANT_PAGE_VIRTUAL
+	dev_dbg(&pdev->dev, "WANT_PAGE_VIRTUAL is defined!\n");
+	#else
+	dev_dbg(&pdev->dev, "WANT_PAGE_VIRTUAL is NOT defined!\n");
+	#endif
+
+	#if defined(HAVE_LLC_HARDWARE)
+	dev_dbg(&pdev->dev, "HAVE_LLC_HARDWARE is defined!\n");
+	#else
+	dev_dbg(&pdev->dev, "HAVE_LLC_HARDWARE is NOT defined!\n");
+	#endif
+
+	#if defined(CONFIG_RISCV)
+	dev_dbg(&pdev->dev, "CONFIG_RISCV is defined!\n");
+	#else
+	dev_dbg(&pdev->dev, "CONFIG_RISCV is NOT defined!\n");
+	#endif
+
+	#if defined(CONFIG_RISCV) && defined(HAVE_LLC_HARDWARE)
+	dev_dbg(&pdev->dev, "CONFIG_RISCV && HAVE_LLC_HARDWARE is defined!\n");
+	#else
+	dev_dbg(&pdev->dev, "CONFIG_RISCV && HAVE_LLC_HARDWARE is NOT defined!\n");
+	#endif
+
+	dev_info(&pdev->dev, "%s Done!\n", __func__);
+
+	pdevs[spram->nid] = pdev;
+	ret = sysfs_create_group(&pdev->dev.kobj, &llc_attr_group);
+	if (ret)
+		dev_err(&pdev->dev, "failed to create sysfs group: %d\n", ret);
+	return 0;
+}
+
+static const struct of_device_id llc_dt_ids[] = {
+	{ .compatible = "eswin,llc" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, llc_dt_ids);
+
+// static struct platform_driver llc_driver __initdata = {
+static struct platform_driver llc_driver = {
+	.driver = {
+		.name = DEVICE_NAME,
+		.of_match_table = llc_dt_ids,
+		.pm	= DEV_PM_OPS,
+	},
+	.probe = llc_probe,
+};
+
+builtin_platform_driver(llc_driver);
+MODULE_DESCRIPTION("ESWIN LLC driver");
+MODULE_AUTHOR("Lin MIn <linmin@eswincomputing.com>");
+MODULE_LICENSE("GPL");
+
diff --git a/drivers/memory/eswin/codacache/llc_spram.h b/drivers/memory/eswin/codacache/llc_spram.h
new file mode 100644
index 000000000000..de54d3ba6a7f
--- /dev/null
+++ b/drivers/memory/eswin/codacache/llc_spram.h
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Defines for the LLC SPRAM driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Min Lin <linmin@eswincomputing.com>
+ *          Huawei Dong <donghuawei@eswincomputing.com>
+ */
+
+#ifndef __LLC_SPRAM_H
+#define __LLC_SPRAM_H
+#include <linux/io.h>
+
+// At least one way must be reserved for cache
+#define MAX_CACHE_SIZE  (2 * 0x100000)  //2MB
+#define MAX_NUM_OF_WAY_SET_ASSOCIATIVE  16  //16-way set associative, i.e. 16 lines per set
+#define SIZE_OF_PER_WAY (MAX_CACHE_SIZE / MAX_NUM_OF_WAY_SET_ASSOCIATIVE)
+#define CACHE_LINE_SIZE 128
+
+// 8MB/16way/256 = 2048 sets
+// 4MB/16way/128 = 2048 sets
+#define MAX_SETS  (MAX_CACHE_SIZE / MAX_NUM_OF_WAY_SET_ASSOCIATIVE/CACHE_LINE_SIZE)
+
+//At least one way must be reserved for cache
+#define SPRAM_WAYS    (MAX_NUM_OF_WAY_SET_ASSOCIATIVE - 1)
+#define SPRAM_SIZE    (SPRAM_WAYS * MAX_CACHE_SIZE / MAX_NUM_OF_WAY_SET_ASSOCIATIVE)  //4MB of MAX_CACHE_SIZE is used as spram
+
+#define NPU_LLC0_OFFSET    0x188000
+#define NPU_LLC1_OFFSET    0x189000
+
+
+enum coda_cache_reg
+{
+	CODA_CACHE_REG_CCUTCR       = 0x0000,
+	CODA_CACHE_REG_CCUTAR       = 0x0004,
+	CODA_CACHE_REG_CCUCTCR      = 0x0010,
+	CODA_CACHE_REG_CCUCTAR      = 0x0014,
+	CODA_CACHE_REG_CCUCAOR      = 0x0018,
+	CODA_CACHE_REG_CCUSPCR0     = 0x0020,
+	CODA_CACHE_REG_CCUSPCR1     = 0x0024,
+	CODA_CACHE_REG_CCUSPBR0     = 0x0028,
+	CODA_CACHE_REG_CCUSPBR1     = 0x002C,
+	CODA_CACHE_REG_CCUWPCR00    = 0x0040,
+	CODA_CACHE_REG_CCUWPCR10    = 0x0044,
+	CODA_CACHE_REG_CCUWPCR01    = 0x0048,
+	CODA_CACHE_REG_CCUWPCR11    = 0x004c,
+	CODA_CACHE_REG_CCUCMCR      = 0x0100,
+	CODA_CACHE_REG_CCUCMAR      = 0x0104,
+	CODA_CACHE_REG_CCUCMLR0     = 0x0108,
+	CODA_CACHE_REG_CCUCMLR1     = 0x010c,
+	CODA_CACHE_REG_CCUCMLR2     = 0x0110,
+	CODA_CACHE_REG_CCUCMDR      = 0x0114,
+	CODA_CACHE_REG_CCUCMWVR     = 0x0118,
+	CODA_CACHE_REG_CCUCECR      = 0x0140,
+	CODA_CACHE_REG_CCUCESR      = 0x0144,
+	CODA_CACHE_REG_CCUCESAR     = 0x0148,
+	CODA_CACHE_REG_CCUCELR0     = 0x014c,
+	CODA_CACHE_REG_CCUCELR1     = 0x0150,
+	CODA_CACHE_REG_CCUUEDR      = 0x0154,
+	CODA_CACHE_REG_CCUUEIR      = 0x0158,
+	CODA_CACHE_REG_CCUUESR      = 0x015c,
+	CODA_CACHE_REG_CCUUESAR     = 0x0160,
+	CODA_CACHE_REG_CCUUELR0     = 0x0164,
+	CODA_CACHE_REG_CCUUELR1     = 0x0168,
+	CODA_CACHE_REG_CCUIDR       = 0x01c0,
+	CODA_CACHE_REG_CCUCRTR      = 0x01c4,
+	CODA_CACHE_REG_CCUESR       = 0x01c8,
+	CODA_CACHE_REG_CCUEMR       = 0x01cc,
+	CODA_CACHE_REG_CCUEAR       = 0x01d0,
+};
+typedef enum {
+	LLC_CACHE_NODE_0 = 0,
+	LLC_CACHE_NODE_1,
+	MAX_LLC_CACHE_NODE_NUM,
+}onwhich_die_t;
+
+struct llc_reg_base {
+	void __iomem *CodaCache0_RegBase;
+	void __iomem *CodaCache1_RegBase;
+};
+
+/* llc cache operation */
+typedef  void (*llc_flush_all_t)(unsigned long start, unsigned long len);
+struct llc_cache_ops {
+		llc_flush_all_t llc_flush_all;
+};
+
+int llc_user_register(struct device *user_dev);
+int npu_cfg_rst(int nid, bool enable);
+int npu_core_rst(int nid, bool enable);
+int npu_clk_gate_set(int nid, bool enable);
+int llc_spram_avail_size(int nid, uint32_t *pSpramSize);
+
+int llc_flush_operation(unsigned long start, unsigned long len);
+#endif
diff --git a/drivers/memory/eswin/es_buddy/Kconfig b/drivers/memory/eswin/es_buddy/Kconfig
new file mode 100644
index 000000000000..94d755f4ba86
--- /dev/null
+++ b/drivers/memory/eswin/es_buddy/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_BUDDY
+	tristate "ESWIN buddy allocator for reserved memory"
+	help
+	  buddy allocator api.
diff --git a/drivers/memory/eswin/es_buddy/Makefile b/drivers/memory/eswin/es_buddy/Makefile
new file mode 100644
index 000000000000..4115fd0901d7
--- /dev/null
+++ b/drivers/memory/eswin/es_buddy/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ESWIN_BUDDY)	+= es_buddy.o buddy_allocator/buddy.o
+
diff --git a/drivers/memory/eswin/es_buddy/buddy_allocator/buddy.c b/drivers/memory/eswin/es_buddy/buddy_allocator/buddy.c
new file mode 100644
index 000000000000..56f084d0fc46
--- /dev/null
+++ b/drivers/memory/eswin/es_buddy/buddy_allocator/buddy.c
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Internal APIs for ESWIN buddy allocator
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#include "../../buddy.h"
+
+#define SIZE_4GB    0x100000000
+static int is_4G_boundary_page(struct mem_block *memblock, unsigned long page_idx)
+{
+	struct page *kpage;
+
+    kpage = memblock->kPageStart + page_idx;
+    if (!((page_to_phys(kpage) + BUDDY_PAGE_SIZE) % SIZE_4GB))
+        return 1;
+
+    return 0;
+}
+
+void buddy_system_init(struct mem_block *memblock,
+                       struct esPage_s *start_page,
+                       unsigned long start_addr,
+                       unsigned long page_num)
+{
+    unsigned long i;
+    struct esPage_s *page = NULL;
+    struct es_free_area_s *area = NULL;
+    // init memory zone
+    struct mem_zone *zone = &memblock->zone;
+    zone->page_num = page_num;
+    zone->page_size = BUDDY_PAGE_SIZE;
+    zone->first_page = start_page;
+    zone->start_addr = start_addr;
+    zone->end_addr = start_addr + page_num * BUDDY_PAGE_SIZE;
+    // TODO: init zone->lock
+    #ifdef RUN_IN_KERNEL
+    buddy_spin_lock_init(&zone->lock);
+    #endif
+    // init each area
+    for (i = 0; i < BUDDY_MAX_ORDER; i++)
+    {
+        area = zone->free_area + i;
+        INIT_LIST_HEAD(&area->free_list);
+        area->nr_free = 0;
+    }
+    memset(start_page, 0, page_num * sizeof(struct esPage_s));
+    // init and free each page
+    for (i = 0; i < page_num; i++)
+    {
+        page = zone->first_page + i;
+        INIT_LIST_HEAD(&page->lru);
+        /* Reserve 4kB at (4GB-4k) alignment address boundary. This is a workaround for g2d.
+           The g2d hardware has problem with accessing the 4GB alignment address boundray,
+           such as the address at 4GB, 8GB, 12GB and 16GB.
+         */
+        if (is_4G_boundary_page(memblock, i)) {
+            memblock->page_num--;
+            continue;
+        }
+
+        buddy_free_pages(zone, page);
+    }
+}
+
+static void prepare_compound_pages(struct esPage_s *page, unsigned long order)
+{
+    unsigned long i;
+    unsigned long nr_pages = (1UL<<order);
+
+    esSet_compound_order(page, order);
+    __esSetPageHead(page);
+    for(i = 1; i < nr_pages; i++)
+    {
+        struct esPage_s *p = page + i;
+        __SetPageTail(p);
+        p->first_page = page;
+    }
+}
+
+static void expand(struct mem_zone *zone, struct esPage_s *page,
+                   unsigned long low_order, unsigned long high_order,
+                   struct es_free_area_s *area)
+{
+    unsigned long size = (1U << high_order);
+    while (high_order > low_order)
+    {
+        area--;
+        high_order--;
+        size >>= 1;
+        list_add(&page[size].lru, &area->free_list);
+        area->nr_free++;
+        // set page order
+        set_page_order_buddy(&page[size], high_order);
+    }
+}
+
+static struct esPage_s *__alloc_page(unsigned long order,
+                                 struct mem_zone *zone)
+{
+    struct esPage_s *page = NULL;
+    struct es_free_area_s *area = NULL;
+    unsigned long current_order = 0;
+
+    for (current_order = order;
+         current_order < BUDDY_MAX_ORDER; current_order++)
+    {
+        area = zone->free_area + current_order;
+        if (list_empty(&area->free_list)) {
+            continue;
+        }
+        // remove closest size page
+        page = list_entry(area->free_list.next, struct esPage_s, lru);
+        list_del(&page->lru);
+        rmv_page_order_buddy(page);
+        area->nr_free--;
+        // expand to lower order
+        expand(zone, page, order, current_order, area);
+        // compound page
+        if (order > 0)
+            prepare_compound_pages(page, order);
+        else // single page
+            page->order = 0;
+        return page;
+    }
+    return NULL;
+}
+
+struct esPage_s *buddy_get_pages(struct mem_zone *zone,
+                             unsigned long order)
+{
+    struct esPage_s *page = NULL;
+
+    if (order >= BUDDY_MAX_ORDER)
+    {
+        BUDDY_BUG(__FILE__, __LINE__);
+        return NULL;
+    }
+    //TODO: lock zone->lock
+    buddy_spin_lock(&zone->lock);
+    page = __alloc_page(order, zone);
+    //TODO: unlock zone->lock
+    buddy_spin_unlock(&zone->lock);
+    return page;
+}
+
+static int destroy_compound_pages(struct esPage_s *page, unsigned long order)
+{
+    int bad = 0;
+    unsigned long i;
+    unsigned long nr_pages = (1UL<<order);
+
+    __esClearPageHead(page);
+    for(i = 1; i < nr_pages; i++)
+    {
+        struct esPage_s *p = page + i;
+        if( !esPageTail(p) || p->first_page != page )
+        {
+            bad++;
+            BUDDY_BUG(__FILE__, __LINE__);
+        }
+        __ClearPageTail(p);
+    }
+    return bad;
+}
+
+#define PageCompound(page) \
+        (page->flags & ((1UL<<enPG_head)|(1UL<<enPG_tail)))
+
+#define page_is_buddy(page,order) \
+        (esPageBuddy(page) && (page->order == order))
+
+void buddy_free_pages(struct mem_zone *zone,
+                      struct esPage_s *page)
+{
+    unsigned long order = esCompound_order(page);
+    unsigned long buddy_idx = 0, combinded_idx = 0;
+    unsigned long page_idx = page - zone->first_page;
+    //TODO: lock zone->lock
+    buddy_spin_lock(&zone->lock);
+    if (PageCompound(page))
+        if (destroy_compound_pages(page, order))
+            BUDDY_BUG(__FILE__, __LINE__);
+
+    while (order < BUDDY_MAX_ORDER-1)
+    {
+        struct esPage_s *buddy;
+        // find and delete buddy to combine
+        buddy_idx = __find_buddy_index(page_idx, order);
+        buddy = page + (buddy_idx - page_idx);
+        if (!page_is_buddy(buddy, order))
+            break;
+        list_del(&buddy->lru);
+        zone->free_area[order].nr_free--;
+        // remove buddy's flag and order
+        rmv_page_order_buddy(buddy);
+        // update page and page_idx after combined
+        combinded_idx = __find_combined_index(page_idx, order);
+        page = page + (combinded_idx - page_idx);
+        page_idx = combinded_idx;
+        order++;
+    }
+    set_page_order_buddy(page, order);
+    list_add(&page->lru, &zone->free_area[order].free_list);
+    zone->free_area[order].nr_free++;
+    //TODO: unlock zone->lock
+     buddy_spin_unlock(&zone->lock);
+}
+
+unsigned long buddy_num_free_page(struct mem_zone *zone)
+{
+    unsigned long i, ret;
+    for (i = 0, ret = 0; i < BUDDY_MAX_ORDER; i++)
+    {
+        ret += zone->free_area[i].nr_free * (1UL<<i);
+    }
+    return ret;
+}
diff --git a/drivers/memory/eswin/es_buddy/es_buddy.c b/drivers/memory/eswin/es_buddy/es_buddy.c
new file mode 100644
index 000000000000..bc05cf473af6
--- /dev/null
+++ b/drivers/memory/eswin/es_buddy/es_buddy.c
@@ -0,0 +1,186 @@
+/*
+ * ESWIN buddy allocator.
+ * eswin_rsvmem initializes the reserved memory in dst that has compatible = "eswin-reserve-memory" and
+ * no-map property. Each of these memory region will be treated as one memory block and managed by eswin
+ * buddy system. Users can allocate/frree pages from/to these memory blocks via es_alloc_pages/es_free_pages.
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#define pr_fmt(fmt) "eswin_buddy: " fmt
+
+#include <linux/memblock.h>
+#include <linux/err.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/log2.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/kmemleak.h>
+
+#include "../eswin_memblock.h"
+#include "es_buddy.h"
+
+
+static void es_buddy_system_init(struct mem_block *memblock,
+                       unsigned long start_addr,
+                       unsigned long page_num)
+{
+	struct esPage_s *start_page = memblock->esPagesStart;
+
+	es_spin_lock_init(&memblock->esLock);
+	buddy_system_init(memblock, start_page, start_addr, page_num);
+}
+
+struct page *es_alloc_pages(struct mem_block *memblock,
+					unsigned long order)
+{
+	struct esPage_s *page;
+	struct page *kpage;
+	struct mem_zone *zone = &memblock->zone;
+	unsigned long page_idx;
+
+	es_spin_lock(&memblock->esLock);
+	page = buddy_get_pages(zone, order);
+	if (NULL == page) {
+		es_spin_unlock(&memblock->esLock);
+		return NULL;
+	}
+
+	page_idx = page - zone->first_page;
+	kpage = memblock->kPageStart + page_idx;
+
+	if (order > 0) {
+		__SetPageHead(kpage);
+		set_compound_order(kpage, order);
+	}
+	es_spin_unlock(&memblock->esLock);
+
+	buddy_print("%s:input order=%ld, esCompound_order(page)=%ld, kCompound_order(kpage)=%d, page_size(kpage)=0x%lx, phys_addr=0x%llx\n",
+			__func__, order, esCompound_order(page), compound_order(kpage), page_size(kpage), page_to_phys(kpage));
+	return kpage;
+}
+EXPORT_SYMBOL(es_alloc_pages);
+
+void es_free_pages(struct mem_block *memblock,
+					struct page *kpage)
+{
+	struct mem_zone *zone = &memblock->zone;
+	unsigned long page_idx = kpage - memblock->kPageStart;
+	struct esPage_s *page = zone->first_page + page_idx;
+	unsigned long order = esCompound_order(page);
+
+	buddy_print("%s:esCompound_order(page)=%ld, kCompound_order(kpage)=%d, page_idx=0x%lx, page_size(kpage)=0x%lx, phys_addr=0x%llx\n",
+			__func__, esCompound_order(page), compound_order(kpage), page_idx, page_size(kpage), page_to_phys(kpage));
+	es_spin_lock(&memblock->esLock);
+	buddy_free_pages(zone, page);
+
+	if (order > 0) {
+		ClearPageHead(kpage);
+	}
+	es_spin_unlock(&memblock->esLock);
+
+}
+EXPORT_SYMBOL(es_free_pages);
+
+unsigned long es_num_free_pages(struct mem_block *memblock)
+{
+	struct mem_zone *zone = &memblock->zone;
+
+	return buddy_num_free_page(zone);
+}
+EXPORT_SYMBOL(es_num_free_pages);
+
+void *es_page_to_virt(struct mem_zone *zone,
+                   struct esPage_s *page)
+{
+    unsigned long page_idx = 0;
+    unsigned long address = 0;
+
+    page_idx = page - zone->first_page;
+    address = zone->start_addr + page_idx * BUDDY_PAGE_SIZE;
+
+    return (void *)address;
+}
+
+struct esPage_s *es_virt_to_page(struct mem_zone *zone, void *ptr)
+{
+    unsigned long page_idx = 0;
+    struct esPage_s *page = NULL;
+    unsigned long address = (unsigned long)ptr;
+
+    if((address<zone->start_addr)||(address>zone->end_addr))
+    {
+        buddy_print("start_addr=0x%lx, end_addr=0x%lx, address=0x%lx\n",
+                zone->start_addr, zone->end_addr, address);
+        BUDDY_BUG(__FILE__, __LINE__);
+        return NULL;
+    }
+    page_idx = (address - zone->start_addr)>>BUDDY_PAGE_SHIFT;
+
+    page = zone->first_page + page_idx;
+    return page;
+}
+
+static int do_rsvmem_buddy_init(struct mem_block *memblock, void *data)
+{
+	int pages_size;
+
+	pr_debug("eswin buddy init for %s\n", memblock->name);
+	/* alloc esPage_s for all the pages to manage the pages*/
+	pages_size = memblock->page_num * sizeof(struct esPage_s);
+	memblock->esPagesStart = (struct esPage_s*)vmalloc(pages_size);
+	if (!memblock->esPagesStart) {
+		pr_err("%s:%d, failed to buddy init for %s\n",
+			__func__, __LINE__, memblock->name);
+		return -ENOMEM;
+	}
+	es_buddy_system_init(memblock, 0, memblock->page_num);
+
+	return 0;
+}
+static int eswin_rsvmem_buddy_init(void)
+{
+	int ret = 0;
+
+	ret = eswin_rsvmem_for_each_block(do_rsvmem_buddy_init, NULL);
+
+	return ret;
+}
+
+static int do_rsvmem_buddy_uninit(struct mem_block *memblock, void *data)
+{
+	unsigned long numFreePages = 0;
+
+	if (NULL == memblock->esPagesStart)
+		return 0;
+
+	numFreePages = es_num_free_pages(memblock);
+	pr_debug("%s: free mem=0x%lx\n",
+		memblock->name, numFreePages<<PAGE_SHIFT);
+	if (numFreePages == memblock->zone.page_num) {
+		vfree((void*)memblock->esPagesStart);
+	}
+	else {
+		pr_err("%s: %ld outof %ld pages still in use, skip destroy memblock!\n",
+			memblock->name, numFreePages, memblock->zone.page_num);
+
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static void __exit eswin_rsvmem_buddy_uninit(void)
+{
+	eswin_rsvmem_for_each_block(do_rsvmem_buddy_uninit, NULL);
+}
+
+subsys_initcall(eswin_rsvmem_buddy_init);
+module_exit(eswin_rsvmem_buddy_uninit);
+MODULE_LICENSE("GPL v2");
\ No newline at end of file
diff --git a/drivers/memory/eswin/es_buddy/es_buddy.h b/drivers/memory/eswin/es_buddy/es_buddy.h
new file mode 100644
index 000000000000..7861a6cbcab8
--- /dev/null
+++ b/drivers/memory/eswin/es_buddy/es_buddy.h
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Header file of ESWIN buddy allocator
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#ifndef __ESWIN_BUDDY_H__
+#define __ESWIN_BUDDY_H__
+
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/version.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0)
+#include <linux/mm_types.h>
+#include <asm/atomic.h>
+#endif
+#include <linux/numa.h>
+#include "../buddy.h"
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0)
+static inline void folio_set_order(struct folio *folio, unsigned int order)
+{
+	if (WARN_ON_ONCE(!order || !folio_test_large(folio)))
+		return;
+
+	folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order;
+#ifdef CONFIG_64BIT
+	folio->_folio_nr_pages = 1U << order;
+#endif
+}
+
+static inline void prep_compound_head(struct page *page, unsigned int order)
+{
+	struct folio *folio = (struct folio *)page;
+
+	folio_set_order(folio, order);
+	atomic_set(&folio->_entire_mapcount, -1);
+	atomic_set(&folio->_nr_pages_mapped, 0);
+	atomic_set(&folio->_pincount, 0);
+}
+#define set_compound_order(kpage, order) prep_compound_head(kpage, order)
+#endif
+
+extern struct page *es_alloc_pages(struct mem_block *memblock, unsigned long order);
+extern void es_free_pages(struct mem_block *memblock, struct page *kpage);
+extern unsigned long es_num_free_pages(struct mem_block *memblock);
+
+#endif
diff --git a/drivers/memory/eswin/es_dev_buf/Kconfig b/drivers/memory/eswin/es_dev_buf/Kconfig
new file mode 100644
index 000000000000..78e59f570ac9
--- /dev/null
+++ b/drivers/memory/eswin/es_dev_buf/Kconfig
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_DEV_DMA_BUF
+	tristate "ESWIN dev dma buf interface"
+	depends on ESWIN_RSVMEM_HEAP
+	help
+	  eswin device dma buf 
+
diff --git a/drivers/memory/eswin/es_dev_buf/Makefile b/drivers/memory/eswin/es_dev_buf/Makefile
new file mode 100644
index 000000000000..391af838aada
--- /dev/null
+++ b/drivers/memory/eswin/es_dev_buf/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_ESWIN_DEV_DMA_BUF) += es_dev_buf.o
diff --git a/drivers/memory/eswin/es_dev_buf/es_dev_buf.c b/drivers/memory/eswin/es_dev_buf/es_dev_buf.c
new file mode 100644
index 000000000000..02bb8e2de2da
--- /dev/null
+++ b/drivers/memory/eswin/es_dev_buf/es_dev_buf.c
@@ -0,0 +1,194 @@
+// Copyright © 2023 ESWIN. All rights reserved.
+//
+// Beijing ESWIN Computing Technology Co., Ltd and its affiliated companies ("ESWIN") retain
+// all intellectual property and proprietary rights in and to this software. Except as expressly
+// authorized by ESWIN, no part of the software may be released, copied, distributed, reproduced,
+// modified, adapted, translated, or created derivative work of, in whole or in part.
+
+#include <linux/dma-heap.h>
+#include <linux/dma-direct.h>
+#include <linux/dsp_dma_buf.h>
+#include <linux/eswin_rsvmem_common.h>
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+dma_addr_t dev_mem_attach(struct dma_buf *buf, struct device *dev,
+			  enum dma_data_direction direc,
+			  struct dma_buf_attachment **attachment)
+{
+	struct sg_table *table;
+	dma_addr_t reg;
+	unsigned int size;
+	struct dma_buf_attachment *attach;
+	struct scatterlist *sg = NULL;
+	u64 addr;
+	int len;
+	int i = 0;
+
+	attach = dma_buf_attach(buf, dev);
+	if (IS_ERR(attach)) {
+		pr_err("%s, dma buf attach error.\n", __func__);
+		return 0;
+	}
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+	table = dma_buf_map_attachment_unlocked(attach, direc);
+#else
+	table = dma_buf_map_attachment(attach, direc);
+#endif
+	if (IS_ERR(table)) {
+		pr_err("%s, dma buf map attachment error.\n", __func__);
+		dma_buf_detach(buf, attach);
+		return 0;
+	}
+
+	/*
+	 * workaround: caching sgt table in attachment.
+	 */
+	if (!attach->dmabuf->ops->cache_sgt_mapping) {
+		attach->sgt = table;
+	}
+
+	for_each_sgtable_dma_sg(table, sg, i) {
+		addr = sg_dma_address(sg);
+		len = sg_dma_len(sg);
+		pr_debug("%s, i=%d, sg addr=0x%llx, len=0x%x, phyaddr=0x%llx.\n",
+		       __func__, i, addr, len, dma_to_phys(dev, addr));
+	}
+
+	reg = sg_dma_address(table->sgl);
+	size = sg_dma_len(table->sgl);
+	if (attachment != NULL) {
+		*attachment = attach;
+	}
+
+	return reg;
+}
+EXPORT_SYMBOL(dev_mem_attach);
+
+int dev_mem_detach(struct dma_buf_attachment *attach,
+		   enum dma_data_direction direction)
+{
+	struct sg_table *table;
+
+	if (!attach->dmabuf->ops->cache_sgt_mapping) {
+		table = attach->sgt;
+		attach->sgt = NULL;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+	dma_buf_unmap_attachment_unlocked(attach, table, DMA_BIDIRECTIONAL);
+#else
+	dma_buf_unmap_attachment(attach, table, DMA_BIDIRECTIONAL);
+#endif
+	dma_buf_detach(attach->dmabuf, attach);
+	return 0;
+}
+EXPORT_SYMBOL(dev_mem_detach);
+
+// this interface only for  x86 emulation
+int dev_mem_alloc(size_t size, enum es_malloc_policy policy,
+		  struct dma_buf **buf)
+{
+	int fd;
+	struct dma_buf *dma_buf;
+
+	switch (policy) {
+	case ES_MEM_ALLOC_RSV:
+		fd = dma_heap_kalloc("reserved", size, O_RDWR, 0);
+		break;
+	case ES_MEM_ALLOC_NORMAL:
+		fd = dma_heap_kalloc("system", size, O_RDWR, 0);
+		break;
+	case ES_MEM_ALLOC_NORMAL_COHERENT:
+		fd = dma_heap_kalloc("system", size, O_RDWR | O_DSYNC, 0);
+		break;
+	case ES_MEM_ALLOC_CMA:
+		fd = dma_heap_kalloc("linux,cma", size, O_RDWR, 0);
+		break;
+	case ES_MEM_ALLOC_CMA_COHERENT:
+		fd = dma_heap_kalloc("linux,cma", size, O_RDWR  | O_DSYNC, 0);
+		break;
+	case ES_MEM_ALLOC_CMA_LLC:
+		fd = dma_heap_kalloc("linux,cma", size, O_RDWR | O_SYNC, 0);
+		break;
+	case ES_MEM_ALLOC_SPRAM_DIE0:
+		fd = dma_heap_kalloc("llc_spram0", size, O_RDWR, 1);
+		break;
+	case ES_MEM_ALLOC_SPRAM_DIE1:
+		fd = dma_heap_kalloc("llc_spram1", size, O_RDWR, 1);
+		break;
+	default:
+		pr_err("policy = %d not supported.\n", policy);
+		return -EINVAL;
+	}
+	if (fd < 0) {
+		return fd;
+	}
+	dma_buf = dma_buf_get(fd);
+	if (IS_ERR(dma_buf)) {
+		return -EINVAL;
+	}
+	if (buf != NULL) {
+		*buf = dma_buf;
+	}
+
+	return fd;
+}
+EXPORT_SYMBOL(dev_mem_alloc);
+
+int dev_mem_free(struct dma_buf *buf)
+{
+	if (buf != NULL) {
+		dma_buf_put(buf);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(dev_mem_free);
+
+void *dev_mem_vmap(struct dev_buffer_desc *buffer)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    struct iosys_map map;
+#else
+	struct dma_buf_map map;
+#endif
+	int ret;
+
+	if (!buffer) {
+		return NULL;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+	ret = dma_buf_vmap_unlocked(buffer->buf, &map);
+#else
+	ret = dma_buf_vmap(buffer->buf, &map);
+#endif
+	if (ret) {
+		return NULL;
+	}
+	return map.vaddr;
+}
+EXPORT_SYMBOL(dev_mem_vmap);
+
+void dev_mem_vunmap(struct dev_buffer_desc *buffer, void *virt)
+{
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    struct iosys_map map;
+#else
+	struct dma_buf_map map;
+#endif
+	if (!buffer || !virt) {
+		return;
+	}
+
+	map.vaddr = virt;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+	dma_buf_vunmap_unlocked(buffer->buf, &map);
+#else
+	dma_buf_vunmap(buffer->buf, &map);
+#endif
+}
+EXPORT_SYMBOL(dev_mem_vunmap);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h b/drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h
new file mode 100644
index 000000000000..ab6962ad1ccf
--- /dev/null
+++ b/drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h
@@ -0,0 +1,44 @@
+// Copyright © 2023 ESWIN. All rights reserved.
+//
+// Beijing ESWIN Computing Technology Co., Ltd and its affiliated companies ("ESWIN") retain
+// all intellectual property and proprietary rights in and to this software. Except as expressly
+// authorized by ESWIN, no part of the software may be released, copied, distributed, reproduced,
+// modified, adapted, translated, or created derivative work of, in whole or in part.
+
+#ifndef __DSP_DMA_BUF_H_
+#define __DSP_DMA_BUF_H_
+#include <linux/dma-buf.h>
+
+struct dev_buffer_desc {
+	struct dma_buf *buf;
+	u32 offset;
+	u32 len;
+};
+
+enum es_malloc_policy {
+	ES_MEM_ALLOC_RSV,
+	ES_MEM_ALLOC_NORMAL,
+	ES_MEM_ALLOC_NORMAL_COHERENT,
+	ES_MEM_ALLOC_CMA,
+	ES_MEM_ALLOC_CMA_COHERENT,
+	ES_MEM_ALLOC_CMA_LLC,
+	ES_MEM_ALLOC_SPRAM_DIE0,
+	ES_MEM_ALLOC_SPRAM_DIE1,
+};
+
+int dev_mem_alloc(size_t size, enum es_malloc_policy policy,
+		   struct dma_buf **buf);
+int dev_mem_free(struct dma_buf *buf);
+
+int dev_mem_alloc_pool(size_t size, enum es_malloc_policy policy, char *mmb,
+		       char *zone, struct dma_buf **buf);
+
+dma_addr_t dev_mem_attach(struct dma_buf *buf, struct device *dev,
+			  enum dma_data_direction direc,
+			  struct dma_buf_attachment **attachment);
+int dev_mem_detach(struct dma_buf_attachment *attach,
+		   enum dma_data_direction direction);
+void *dev_mem_vmap(struct dev_buffer_desc *dev_buffer);
+void dev_mem_vunmap(struct dev_buffer_desc *dev_buf, void *vaddr);
+
+#endif
diff --git a/drivers/memory/eswin/es_dma_memcp/Kconfig b/drivers/memory/eswin/es_dma_memcp/Kconfig
new file mode 100644
index 000000000000..deff4abf37de
--- /dev/null
+++ b/drivers/memory/eswin/es_dma_memcp/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_DMA_MEMCP
+	tristate "ESWIN DMA memory copy"
+	help
+	  ESWIN DMA memory copy device.
diff --git a/drivers/memory/eswin/es_dma_memcp/Makefile b/drivers/memory/eswin/es_dma_memcp/Makefile
new file mode 100644
index 000000000000..615375e6b476
--- /dev/null
+++ b/drivers/memory/eswin/es_dma_memcp/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_ESWIN_DMA_MEMCP) += es_dma_memcp.o
diff --git a/drivers/memory/eswin/es_dma_memcp/es_dma_memcp.c b/drivers/memory/eswin/es_dma_memcp/es_dma_memcp.c
new file mode 100644
index 000000000000..1c2096270595
--- /dev/null
+++ b/drivers/memory/eswin/es_dma_memcp/es_dma_memcp.c
@@ -0,0 +1,687 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DMA MEMCP Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Zonglin Geng <gengzonglin@eswincomputing.com>
+ *          Yuyang Cong <congyuyang@eswincomputing.com>
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/delay.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <asm/uaccess.h>
+#include <linux/version.h>
+#include <linux/dmaengine.h>
+#include <linux/uaccess.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-buf.h>
+#include <linux/workqueue.h>
+#include <linux/poll.h>
+#include <uapi/linux/dma_memcp.h>
+
+#define DRIVER_NAME 		"es_memcp"
+
+#define MAX_NUM_USED_DMA_CH       (4)
+
+static struct device *esw_memcp_dev;
+
+struct esw_memcp_dma_buf_info {
+    struct dma_buf *dma_buf;
+    int mem_nid;
+    struct dma_buf_attachment *attach;
+    struct sg_table *sgt;
+};
+
+struct cmdq {
+    struct workqueue_struct *wq;
+    struct mutex lock;
+    atomic_t total_tasks;
+    atomic_t completed_tasks;
+    atomic_t last_error;
+    wait_queue_head_t wait;
+};
+
+struct esw_cmdq_task {
+    struct cmdq *cmdq;
+    struct esw_memcp_f2f_cmd f2f_cmd;
+    struct esw_memcp_dma_buf_info src_buf;
+    struct esw_memcp_dma_buf_info dst_buf;
+    struct work_struct work;
+    struct dma_chan *dma_ch; /* pointer to the dma channel. */
+    struct completion dma_finished;
+};
+
+struct dma_filter_params {
+    unsigned int nid;
+    char dev_name[32];
+};
+
+#ifdef CONFIG_NUMA
+static int memcp_attach_dma_buf(struct device *dma_dev, struct esw_memcp_dma_buf_info *buf_info);
+static int memcp_detach_dma_buf(struct dma_buf_attachment *attach, struct sg_table *sgt);
+#define PAGE_IN_SPRAM_DIE0(page) ((page_to_phys(page)>=0x59000000) && (page_to_phys(page)<0x59400000))
+#define PAGE_IN_SPRAM_DIE1(page) ((page_to_phys(page)>=0x79000000) && (page_to_phys(page)<0x79400000))
+static int esw_memcp_get_mem_nid(struct esw_memcp_dma_buf_info *buf_info)
+{
+    int ret = 0;
+    struct page *page = NULL;
+    int nid = -1;
+
+    ret = memcp_attach_dma_buf(esw_memcp_dev, buf_info);
+    if(ret) {
+        pr_err("Failed to attach DMA buffer, ret=%d\n", ret);
+        return ret;
+    }
+
+    page = sg_page(buf_info->sgt->sgl);
+    if (unlikely(PAGE_IN_SPRAM_DIE0(page))) {
+        nid = 0;
+    }
+    else if(unlikely(PAGE_IN_SPRAM_DIE1(page))) {
+        nid = 1;
+    }
+    else {
+        nid = page_to_nid(page);
+    }
+
+    ret = memcp_detach_dma_buf(buf_info->attach, buf_info->sgt);
+    if(ret) {
+        pr_err("Failed to detach DMA buffer, , ret=%d\n", ret);
+        return ret;
+    }
+
+    buf_info->mem_nid = nid;
+
+    return ret;
+}
+#else
+static int esw_memcp_get_mem_nid(struct esw_memcp_dma_buf_info *buf_info)
+{
+    buf_info->mem_nid = 0;
+    return 0;
+}
+#endif
+
+static bool numa_filter(struct dma_chan *chan, unsigned int param)
+{
+#ifdef CONFIG_NUMA
+    return param == dev_to_node(chan->device->dev);
+#else
+    return true;
+#endif
+}
+
+static bool device_name_filter(struct dma_chan *chan, char *param)
+{
+    if (param[0] == '\0')
+        return true;
+
+    const char *dev_name_trimmed = strchr(dev_name(chan->device->dev), '.');
+    if (dev_name_trimmed) {
+        dev_name_trimmed++;
+    } else {
+        return false;
+    }
+
+    return strcmp(dev_name_trimmed, param) == 0;
+}
+
+static bool filter(struct dma_chan *chan, void *param)
+{
+    struct dma_filter_params *params = param;
+    return numa_filter(chan, params->nid) && device_name_filter(chan, params->dev_name);
+}
+
+int esw_memcp_alloc_dma(struct esw_cmdq_task *task)
+{
+    int ret = 0;
+    dma_cap_mask_t mask;
+    struct dma_chan *dma_ch = NULL;
+    struct dma_filter_params params;
+
+    ret = esw_memcp_get_mem_nid(&task->src_buf);
+    if(ret) {
+        return ret;
+    }
+
+    strcpy(params.dev_name, "dma-controller-aon");
+    params.nid = task->src_buf.mem_nid;
+
+    dma_cap_zero(mask);
+    dma_cap_set(DMA_MEMCPY, mask);
+
+    dma_ch = dma_request_channel(mask, filter, &params);
+    if (!dma_ch) {
+        pr_warn("dma request channel failed, Try using any of them.\n");
+        dma_ch = dma_request_channel(mask, NULL, NULL);
+    }
+
+    if (!dma_ch) {
+        pr_err("dma request channel failed\n");
+        return -ENODEV;
+    }
+
+    task->dma_ch = dma_ch;
+    return 0;
+}
+
+static int memcp_attach_dma_buf(struct device *dma_dev, struct esw_memcp_dma_buf_info *buf_info)
+{
+    int ret = 0;
+    struct dma_buf *dma_buf;
+    struct dma_buf_attachment *attach;
+    struct sg_table *sgt;
+
+    if (!buf_info || !dma_dev) {
+        pr_err("Invalid parameters: buf_info or dma_dev is NULL\n");
+        return -EINVAL;
+    }
+
+    dma_buf = buf_info->dma_buf;
+    if (IS_ERR(dma_buf)) {
+        return PTR_ERR(dma_buf);
+    }
+    /* Ref + 1 */
+    attach = dma_buf_attach(dma_buf, dma_dev);
+    if (IS_ERR(attach)) {
+        ret = PTR_ERR(attach);
+        return ret;
+    }
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
+#else
+    sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+#endif
+    if (IS_ERR(sgt)) {
+        ret = PTR_ERR(sgt);
+        dma_buf_detach(dma_buf, attach);
+        return ret;
+    }
+#ifdef DMA_MEMCP_DEBUG_EN
+    struct scatterlist *sg = NULL;
+    u64 addr;
+    int len;
+    int i = 0;
+    for_each_sgtable_dma_sg(sgt, sg, i) {
+        addr = sg_dma_address(sg);
+        len = sg_dma_len(sg);
+    }
+#endif
+
+    buf_info->attach = attach;
+    buf_info->sgt = sgt;
+    return ret;
+}
+
+static int memcp_detach_dma_buf(struct dma_buf_attachment *attach, struct sg_table *sgt)
+{
+    int ret = 0;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
+#else
+    dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
+#endif
+    /* detach attach->dma_buf*/
+    dma_buf_detach(attach->dmabuf, attach);
+    return ret;
+}
+
+static int memcp_detach_dma_buf_check(struct esw_memcp_dma_buf_info *buf_info)
+{
+    int ret = 0;
+    if(buf_info->attach) {
+        ret = memcp_detach_dma_buf(buf_info->attach, buf_info->sgt);
+        buf_info->attach = NULL;
+    }
+    return ret;
+}
+
+static int esw_memcp_release_cmdq_task(struct esw_cmdq_task  *cmdq_task)
+{
+    memcp_detach_dma_buf_check(&cmdq_task->src_buf);
+    memcp_detach_dma_buf_check(&cmdq_task->dst_buf);
+
+    if (cmdq_task->src_buf.dma_buf) {
+        dma_buf_put(cmdq_task->src_buf.dma_buf);
+        cmdq_task->src_buf.dma_buf = NULL;
+    }
+    if (cmdq_task->dst_buf.dma_buf) {
+        dma_buf_put(cmdq_task->dst_buf.dma_buf);
+        cmdq_task->dst_buf.dma_buf = NULL;
+    }
+
+    if (cmdq_task->dma_ch) {
+        dma_release_channel(cmdq_task->dma_ch);
+        cmdq_task->dma_ch = NULL;
+    }
+
+    kfree(cmdq_task);
+    return 0;
+}
+
+
+static void esw_memcp_complete_func(void *cb_param)
+{
+    struct esw_cmdq_task *cmdq_task = (struct esw_cmdq_task *)cb_param;
+
+    complete(&cmdq_task->dma_finished);
+}
+
+static int esw_memcp_start_dma_f2f_trans(struct esw_cmdq_task  *cmdq_task)
+{
+    int ret = 0;
+    struct esw_memcp_f2f_cmd *f2f_cmd;
+    struct dma_buf_attachment *attach;
+    struct sg_table *sgt;
+    struct dma_buf *dma_buf;
+    struct dma_chan *dma_ch;
+    struct device *dma_dev;
+    enum dma_ctrl_flags flags;
+    dma_cookie_t cookie;  //used for judge whether trans has been completed.
+    struct dma_async_tx_descriptor *tx = NULL;
+    dma_addr_t src_buf_addr, dst_buf_addr;
+    size_t src_size, dst_size;
+
+    ret = esw_memcp_alloc_dma(cmdq_task);
+    if(ret) {
+        goto release_cmdq_task;
+    }
+
+    f2f_cmd = &cmdq_task->f2f_cmd;
+    dma_ch = cmdq_task->dma_ch;
+    dma_dev = dmaengine_get_dma_device(dma_ch);
+
+    ret = memcp_attach_dma_buf(dma_dev, &cmdq_task->src_buf);
+    if(ret) {
+        goto release_cmdq_task;
+    }
+    attach = cmdq_task->src_buf.attach;
+    sgt = cmdq_task->src_buf.sgt;
+    src_buf_addr = sg_dma_address(sgt->sgl) + f2f_cmd->src_offset;
+    dma_buf = attach->dmabuf;
+    src_size = dma_buf->size;
+
+    ret = memcp_attach_dma_buf(dma_dev, &cmdq_task->dst_buf);
+    if(ret) {
+        goto release_cmdq_task;
+    }
+    attach = cmdq_task->dst_buf.attach;
+    sgt = cmdq_task->dst_buf.sgt;
+    dst_buf_addr = sg_dma_address(sgt->sgl) + f2f_cmd->dst_offset;
+    dma_buf = attach->dmabuf;
+    dst_size = dma_buf->size;
+
+    init_completion(&cmdq_task->dma_finished);
+    dma_ch = cmdq_task->dma_ch;
+    flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+    tx = dmaengine_prep_dma_memcpy(dma_ch, dst_buf_addr, src_buf_addr, f2f_cmd->len, flags);
+    if(!tx){
+        pr_err("Failed to prepare DMA memcp\n");
+        ret = -EFAULT;
+        goto release_cmdq_task;
+    }
+
+    tx->callback = esw_memcp_complete_func;
+    tx->callback_param = cmdq_task;
+
+    cookie = dmaengine_submit(tx);
+    if(dma_submit_error(cookie)){
+        pr_err("Failed to dma tx_submit\n");
+        ret = -EFAULT;
+        goto release_cmdq_task;
+    }
+
+    dma_async_issue_pending(dma_ch);
+    if (wait_for_completion_interruptible_timeout(&cmdq_task->dma_finished,
+        msecs_to_jiffies(cmdq_task->f2f_cmd.timeout)) == 0) {
+        pr_err("DMA transfer timeout.\n");
+        ret = -ETIMEDOUT;
+        dmaengine_terminate_sync(dma_ch);
+        complete(&cmdq_task->dma_finished);
+        goto release_cmdq_task;
+    }
+
+release_cmdq_task:
+    esw_memcp_release_cmdq_task(cmdq_task);
+    return ret;
+}
+
+
+static void cmdq_task_worker(struct work_struct *work) {
+    struct esw_cmdq_task *task = container_of(work, struct esw_cmdq_task, work);
+    struct cmdq *q = task->cmdq;
+    int ret;
+
+    if (!task) {
+        pr_err("cmdq_task_worker: Invalid task pointer\n");
+        return;
+    }
+
+    // Start DMA Transfer
+    ret = esw_memcp_start_dma_f2f_trans(task);
+
+    if (ret) {
+        pr_err("cmdq_task_worker: DMA transfer failed with error code %d\n", ret);
+        atomic_set(&q->last_error, ret);
+    }
+    wake_up_interruptible(&q->wait);
+    atomic_inc(&q->completed_tasks);
+
+}
+
+static int esw_cmdq_add_task(struct file *filp, void __user *user_arg) {
+    struct cmdq *q = (struct cmdq *)filp->private_data;
+    struct esw_cmdq_task *cmdq_task;
+    struct esw_memcp_f2f_cmd memcp_f2f_cmd;
+    int ret;
+
+    if (!q || !q->wq) {
+        pr_err("CMDQ or workqueue is NULL\n");
+        return -EINVAL;
+    }
+
+    cmdq_task = kzalloc(sizeof(struct esw_cmdq_task), GFP_KERNEL);
+    if (!cmdq_task) {
+        pr_err("Failed to allocate cmdq_task\n");
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(&memcp_f2f_cmd, user_arg, sizeof(struct esw_memcp_f2f_cmd))) {
+        pr_err("Failed to copy data from user space\n");
+        kfree(cmdq_task);
+        return -EFAULT;
+    }
+
+    if (memcp_f2f_cmd.len == 0) {
+        pr_err("Copy length is zero\n");
+        kfree(cmdq_task);
+        return -EINVAL;
+    }
+
+    if ((size_t)memcp_f2f_cmd.src_offset > SIZE_MAX - memcp_f2f_cmd.len) {
+        pr_err("Overflow detected in (src_offset + len), src_offset=%d, len=%zu\n",
+               memcp_f2f_cmd.src_offset, memcp_f2f_cmd.len);
+        kfree(cmdq_task);
+        return -EINVAL;
+    }
+
+    if ((size_t)memcp_f2f_cmd.dst_offset > SIZE_MAX - memcp_f2f_cmd.len) {
+        pr_err("Overflow detected in (dst_offset + len), dst_offset=%d, len=%zu\n",
+               memcp_f2f_cmd.dst_offset, memcp_f2f_cmd.len);
+        kfree(cmdq_task);
+        return -EINVAL;
+    }
+
+    cmdq_task->cmdq = q;
+    cmdq_task->f2f_cmd = memcp_f2f_cmd;
+
+    cmdq_task->src_buf.dma_buf = dma_buf_get(memcp_f2f_cmd.src_fd);
+    if (IS_ERR(cmdq_task->src_buf.dma_buf)) {
+        pr_err("Failed to get src dma_buf, src_fd=%d, err=%ld\n",
+                memcp_f2f_cmd.src_fd, PTR_ERR(cmdq_task->src_buf.dma_buf));
+        kfree(cmdq_task);
+        return PTR_ERR(cmdq_task->src_buf.dma_buf);
+    }
+
+    if ((memcp_f2f_cmd.src_offset + memcp_f2f_cmd.len) > cmdq_task->src_buf.dma_buf->size) {
+        pr_err("Source buffer overflow: src_offset=%d, len=%lu, buf_size=%lu\n",
+                memcp_f2f_cmd.src_offset, memcp_f2f_cmd.len, cmdq_task->src_buf.dma_buf->size);
+        dma_buf_put(cmdq_task->src_buf.dma_buf);
+        kfree(cmdq_task);
+        return -EINVAL;
+    }
+
+    cmdq_task->dst_buf.dma_buf = dma_buf_get(memcp_f2f_cmd.dst_fd);
+    if (IS_ERR(cmdq_task->dst_buf.dma_buf)) {
+        pr_err("Failed to get dst dma_buf, dst_fd=%d, err=%ld\n",
+                memcp_f2f_cmd.dst_fd, PTR_ERR(cmdq_task->dst_buf.dma_buf));
+        dma_buf_put(cmdq_task->src_buf.dma_buf);
+        kfree(cmdq_task);
+        return PTR_ERR(cmdq_task->dst_buf.dma_buf);
+    }
+
+    if ((memcp_f2f_cmd.dst_offset + memcp_f2f_cmd.len) > cmdq_task->dst_buf.dma_buf->size) {
+        pr_err("Destination buffer overflow: dst_offset=%d, len=%lu, buf_size=%lu\n",
+                memcp_f2f_cmd.dst_offset, memcp_f2f_cmd.len, cmdq_task->dst_buf.dma_buf->size);
+        dma_buf_put(cmdq_task->src_buf.dma_buf);
+        dma_buf_put(cmdq_task->dst_buf.dma_buf);
+        kfree(cmdq_task);
+        return -EINVAL;
+    }
+
+    atomic_set(&q->last_error, 0);
+    INIT_WORK(&cmdq_task->work, cmdq_task_worker);
+    ret = queue_work(cmdq_task->cmdq->wq, &cmdq_task->work);
+    if (!ret) {
+        pr_err("Failed to queue work\n");
+        dma_buf_put(cmdq_task->src_buf.dma_buf);
+        dma_buf_put(cmdq_task->dst_buf.dma_buf);
+        kfree(cmdq_task);
+        return -EFAULT;
+    }
+
+    if (atomic_read(&q->last_error)) {
+        pr_err("Task failed with error code %d\n", atomic_read(&q->last_error));
+        return -EFAULT;
+    }
+
+    atomic_inc(&q->total_tasks);
+
+    return 0;
+}
+
+static int esw_cmdq_sync(struct file *filp) {
+    struct cmdq *q = (struct cmdq *)filp->private_data;
+
+    if (!q) {
+        pr_err("esw_cmdq_sync: Invalid cmdq\n");
+        return -EINVAL;
+    }
+
+    flush_workqueue(q->wq);
+
+    if (atomic_read(&q->last_error) != 0) {
+        pr_err("Last task error: %d\n", atomic_read(&q->last_error));
+        return atomic_read(&q->last_error);
+    }
+
+    return 0;
+}
+
+
+
+static int esw_cmdq_query(struct file *file, void __user *user_arg)
+{
+    struct cmdq *q;
+    struct esw_cmdq_query query;
+
+    if (!file || !user_arg) {
+        pr_err("esw_cmdq_query: Invalid arguments\n");
+        return -EINVAL;
+    }
+
+    q = file->private_data;
+    if (!q) {
+        pr_err("esw_cmdq_query: No cmdq associated with this file descriptor\n");
+        return -EINVAL;
+    }
+
+    int total = atomic_read(&q->total_tasks);
+    int completed = atomic_read(&q->completed_tasks);
+
+    query.status = (total == completed) ? 0 : 1; // 0 FREE，1 BUSY
+    query.task_count = total - completed;
+    query.last_error = atomic_read(&q->last_error);
+
+    if (copy_to_user(user_arg, &query, sizeof(query))) {
+        pr_err("esw_cmdq_query: Failed to copy data to user space\n");
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+static long esw_memcp_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+    int ret = 0;
+
+    switch (cmd) {
+    case (ESW_CMDQ_ADD_TASK):
+        ret = esw_cmdq_add_task(filp, (void *)arg);
+        break;
+
+    case (ESW_CMDQ_SYNC):
+        ret = esw_cmdq_sync(filp);
+        break;
+
+    case (ESW_CMDQ_QUERY):
+        ret = esw_cmdq_query(filp, (void *)arg);
+        break;
+
+    default:
+        dev_err(esw_memcp_dev, "invalid cmd! cmd=0x%x, arg=0x%lx\n", cmd, arg);
+        ret = -EINVAL;
+        break;
+    }
+
+    return ret;
+}
+
+static ssize_t esw_memcp_write(struct file *file, const char __user *data, size_t len, loff_t *ppos)
+{
+    dev_info(esw_memcp_dev, "Write: operation not supported\n");
+    return -EINVAL;
+}
+
+static int esw_memcp_open(struct inode *inode, struct file *filp)
+{
+    struct cmdq *q;
+
+    q = kzalloc(sizeof(struct cmdq), GFP_KERNEL);
+    if (!q) {
+        pr_err("Failed to allocate cmdq\n");
+        return -ENOMEM;
+    }
+
+    init_waitqueue_head(&q->wait);
+
+    q->wq = alloc_workqueue("cmdq_wq", WQ_UNBOUND | WQ_FREEZABLE | WQ_MEM_RECLAIM, 0);
+    if (!q->wq) {
+        pr_err("Failed to allocate workqueue\n");
+        kfree(q);
+        return -ENOMEM;
+    }
+
+    atomic_set(&q->total_tasks, 0);
+    atomic_set(&q->completed_tasks, 0);
+
+    filp->private_data = q;
+
+    return 0;
+}
+
+
+
+static int esw_memcp_release(struct inode *inode, struct file *filp)
+{
+    struct cmdq *q = (struct cmdq *)filp->private_data;
+
+    if (q) {
+        if (q->wq) {
+            destroy_workqueue(q->wq);
+        }
+
+        kfree(q);
+    }
+
+    return 0;
+}
+
+static unsigned int esw_memcp_poll(struct file *filp, poll_table *wait)
+{
+    struct cmdq *q = filp->private_data;
+    unsigned int mask = 0;
+    int total = atomic_read(&q->total_tasks);
+    int completed = atomic_read(&q->completed_tasks);
+
+    poll_wait(filp, &q->wait, wait);
+
+    if (atomic_read(&q->last_error) != 0 || total == completed) {
+        mask |= POLLIN | POLLRDNORM;
+    }
+
+    return mask;
+}
+
+
+static struct file_operations esw_memcp_fops = {
+    .owner          = THIS_MODULE,
+    .llseek         = no_llseek,
+    .write          = esw_memcp_write,
+    .unlocked_ioctl = esw_memcp_ioctl,
+    .open           = esw_memcp_open,
+    .release        = esw_memcp_release,
+    .poll           = esw_memcp_poll,
+};
+
+static struct miscdevice esw_memcp_miscdev = {
+    .minor         = MISC_DYNAMIC_MINOR,
+    .name          = DRIVER_NAME,
+    .mode          = 0666,
+    .fops          = &esw_memcp_fops,
+};
+
+static int __init esw_memcp_init(void)
+{
+    int ret = 0;
+
+    ret = misc_register(&esw_memcp_miscdev);
+    if (ret) {
+        pr_err("%s: Failed to register misc device, err=%d\n", __func__, ret);
+        return ret;
+    }
+
+    esw_memcp_dev = esw_memcp_miscdev.this_device;
+
+    if (!esw_memcp_dev->dma_mask) {
+        esw_memcp_dev->dma_mask = &esw_memcp_dev->coherent_dma_mask;
+    }
+    ret = dma_set_mask_and_coherent(esw_memcp_dev, DMA_BIT_MASK(64));
+    if (ret)
+        dev_err(esw_memcp_dev, "Failed to set coherent mask.\n");
+    return ret;
+}
+
+static void __exit esw_memcp_exit(void)
+{
+    misc_deregister(&esw_memcp_miscdev);
+}
+module_init(esw_memcp_init);
+module_exit(esw_memcp_exit);
+
+MODULE_AUTHOR("Geng Zonglin <gengzonglin@eswincomputing.com>");
+MODULE_AUTHOR("Yuyang Cong <congyuyang@eswincomputing.com>");
+MODULE_DESCRIPTION("ESW DMA MEMCP Driver");
+MODULE_LICENSE("GPL v2");
\ No newline at end of file
diff --git a/drivers/memory/eswin/es_iommu_rsv/Kconfig b/drivers/memory/eswin/es_iommu_rsv/Kconfig
new file mode 100644
index 000000000000..f04159dd39df
--- /dev/null
+++ b/drivers/memory/eswin/es_iommu_rsv/Kconfig
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_IOMMU_RSV
+	tristate "ESWIN IOMMU reserved for device interface"
+	help
+	  eswin iommu rsv intf for device
+
diff --git a/drivers/memory/eswin/es_iommu_rsv/Makefile b/drivers/memory/eswin/es_iommu_rsv/Makefile
new file mode 100644
index 000000000000..361a05e9f900
--- /dev/null
+++ b/drivers/memory/eswin/es_iommu_rsv/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_ESWIN_IOMMU_RSV) += iommu_rsv.o
diff --git a/drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c b/drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c
new file mode 100644
index 000000000000..9bcbec3fa4e7
--- /dev/null
+++ b/drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c
@@ -0,0 +1,561 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * A fairly generic DMA-API to IOMMU-API glue layer.
+ *
+ * Copyright (C) 2014-2015 ARM Ltd.
+ *
+ * based in part on arch/arm/mm/dma-mapping.c:
+ * Copyright (C) 2000-2004 Russell King
+ */
+#include <linux/version.h>
+#include <linux/acpi_iort.h>
+#include <linux/atomic.h>
+#include <linux/crash_dump.h>
+#include <linux/device.h>
+#include <linux/dma-direct.h>
+#include <linux/dma-map-ops.h>
+#include <linux/gfp.h>
+#include <linux/huge_mm.h>
+#include <linux/iommu.h>
+#include <linux/iova.h>
+#include <linux/irq.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+#include <linux/swiotlb.h>
+#include <linux/vmalloc.h>
+#include <linux/fs.h>
+#include <linux/namei.h>
+#include <linux/kallsyms.h>
+#include <linux/sched/mm.h>
+
+#define BUF_SIZE 0x300000
+
+#define IOVA_FQ_SIZE 256
+
+/* Timeout (in ms) after which entries are flushed from the queue */
+#define IOVA_FQ_TIMEOUT 10
+
+/* Flush queue entry for deferred flushing */
+struct iova_fq_entry
+{
+	unsigned long iova_pfn;
+	unsigned long pages;
+	struct list_head freelist;
+	u64 counter; /* Flush counter when this entry was added */
+};
+
+/* Per-CPU flush queue structure */
+struct iova_fq
+{
+	struct iova_fq_entry entries[IOVA_FQ_SIZE];
+	unsigned int head, tail;
+	spinlock_t lock;
+};
+
+enum iommu_dma_cookie_type
+{
+	IOMMU_DMA_IOVA_COOKIE,
+	IOMMU_DMA_MSI_COOKIE,
+};
+
+struct iommu_dma_cookie
+{
+	enum iommu_dma_cookie_type type;
+	union
+	{
+		struct
+		{
+			struct iova_domain iovad;
+
+			struct iova_fq __percpu *fq; /* Flush queue */
+			/* Number of TLB flushes that have been started */
+			atomic64_t fq_flush_start_cnt;
+			/* Number of TLB flushes that have been finished */
+			atomic64_t fq_flush_finish_cnt;
+			/* Timer to regularily empty the flush queues */
+			struct timer_list fq_timer;
+			/* 1 when timer is active, 0 when not */
+			atomic_t fq_timer_on;
+		};
+		/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
+		dma_addr_t msi_iova;
+	};
+	struct list_head msi_page_list;
+
+	/* Domain for flush queue callback; NULL if flush queue not in use */
+	struct iommu_domain *fq_domain;
+};
+
+static void *(*es_common_pages_remap)(struct page **pages, size_t size, pgprot_t prot, const void *caller) = NULL;
+static void (*__iommu_dma_unmap)(struct device *dev, dma_addr_t dma_addr, size_t size) = NULL;
+
+static void (*__iommu_dma_free)(struct device *dev, size_t size, void *cpu_add) = NULL;
+static void (*dma_prep_coherent)(struct page *page, size_t size) = NULL;
+static unsigned long (*klp_name)(const char *name) = NULL;
+
+static struct iova_domain *es_get_iovad(struct iommu_domain *domain)
+{
+	struct iova_domain *iovad;
+
+	iovad = (struct iova_domain *)(&(domain->iova_cookie->iovad));
+	return iovad;
+}
+
+static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
+							unsigned long attrs)
+{
+	int prot = coherent ? IOMMU_CACHE : 0;
+
+	if (attrs & DMA_ATTR_PRIVILEGED)
+		prot |= IOMMU_PRIV;
+
+	switch (dir)
+	{
+	case DMA_BIDIRECTIONAL:
+		return prot | IOMMU_READ | IOMMU_WRITE;
+	case DMA_TO_DEVICE:
+		return prot | IOMMU_READ;
+	case DMA_FROM_DEVICE:
+		return prot | IOMMU_WRITE;
+	default:
+		return 0;
+	}
+}
+
+static void __iommu_dma_free_pages(struct page **pages, int count)
+{
+	while (count--)
+		__free_page(pages[count]);
+	kvfree(pages);
+}
+
+static struct page **__iommu_dma_alloc_pages(struct device *dev,
+											 unsigned int count, unsigned long order_mask, gfp_t gfp)
+{
+	struct page **pages;
+	unsigned int i = 0, nid = dev_to_node(dev);
+
+	order_mask &= (2U << MAX_ORDER) - 1;
+	if (!order_mask)
+		return NULL;
+
+	pages = kvcalloc(count, sizeof(*pages), GFP_KERNEL);
+	if (!pages)
+		return NULL;
+
+	/* IOMMU can map any pages, so himem can also be used here */
+	gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
+
+	/* It makes no sense to muck about with huge pages */
+	gfp &= ~__GFP_COMP;
+
+	while (count)
+	{
+		struct page *page = NULL;
+		unsigned int order_size;
+
+		/*
+		 * Higher-order allocations are a convenience rather
+		 * than a necessity, hence using __GFP_NORETRY until
+		 * falling back to minimum-order allocations.
+		 */
+		// 2U << __fls(count): 得到count的mask， 比如count为0b1000, mask就是0b1111.
+		// mask和order_mask相与，取出最最高bit，就是针对当前count可以分配内存的最大block size。
+		for (order_mask &= (2U << __fls(count)) - 1;
+			 order_mask; order_mask &= ~order_size)
+		{
+			unsigned int order = __fls(order_mask);
+			gfp_t alloc_flags = gfp;
+
+			order_size = 1U << order;
+			if (order_mask > order_size)
+				alloc_flags |= __GFP_NORETRY;
+			page = alloc_pages_node(nid, alloc_flags, order);
+			if (!page)
+				continue;
+			if (order)
+				split_page(page, order);
+			break;
+		}
+		if (!page)
+		{
+			__iommu_dma_free_pages(pages, i);
+			return NULL;
+		}
+		count -= order_size;
+		while (order_size--)
+			pages[i++] = page++;
+	}
+	return pages;
+}
+
+static struct page **iommu_rsv_iova_alloc_pages(struct device *dev, unsigned long iova, unsigned long size, struct sg_table *sgt, gfp_t gfp, unsigned long attrs)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = es_get_iovad(domain); //&domain->iova_cookie->iovad;
+	bool coherent = dev_is_dma_coherent(dev);
+	int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
+	struct page **pages = NULL;
+	int ret;
+	unsigned int alloc_sizes = domain->pgsize_bitmap;
+	unsigned int count, min_size;
+
+	min_size = alloc_sizes & -alloc_sizes;
+	if (min_size < PAGE_SIZE)
+	{
+		min_size = PAGE_SIZE;
+		alloc_sizes |= PAGE_SIZE;
+	}
+	else
+	{
+		size = ALIGN(size, min_size);
+	}
+	count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT, gfp);
+	if (pages == NULL)
+	{
+		dev_err(dev, "alloc pages failed.\n");
+		return NULL;
+	}
+	size = iova_align(iovad, size);
+
+	if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL))
+	{
+		goto out_free_iova;
+	}
+
+	if (!(ioprot & IOMMU_CACHE))
+	{
+		struct scatterlist *sg;
+		int i;
+
+		for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
+			dma_prep_coherent(sg_page(sg), sg->length);
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot, GFP_KERNEL);
+#else
+	ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot);
+#endif
+	if (ret < size)
+	{
+		dev_err(dev, "%s %d fail %d\n", __func__, __LINE__, ret);
+		goto out_free_sg;
+	}
+	return pages;
+
+out_free_sg:
+	sg_free_table(sgt);
+out_free_iova:
+	__iommu_dma_free_pages(pages, count);
+	return NULL;
+}
+
+static void *iommu_rsv_iova_alloc_remap(struct device *dev, dma_addr_t iova, unsigned long size, gfp_t gfp, unsigned long attrs)
+{
+
+	struct page **pages = NULL;
+	struct sg_table sgt;
+	void *cpu_addr;
+
+	pages = iommu_rsv_iova_alloc_pages(dev, iova, size, &sgt, gfp, attrs);
+	if (pages == NULL)
+	{
+		dev_err(dev, "alloc pages failed.\n");
+		return NULL;
+	}
+
+	sg_free_table(&sgt);
+	cpu_addr = es_common_pages_remap(pages, size, pgprot_dmacoherent(PAGE_KERNEL), __builtin_return_address(0));
+	if (!cpu_addr)
+		goto out_unmap;
+	return cpu_addr;
+
+out_unmap:
+	__iommu_dma_unmap(dev, iova, size);
+	__iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
+	return NULL;
+}
+
+void *iommu_map_rsv_iova(struct device *dev, dma_addr_t iova, unsigned long size, gfp_t gfp, unsigned long attrs)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	phys_addr_t phys;
+
+	if (!size)
+	{
+		dev_err(dev, "%s, error, size is 0.\n", __func__);
+		return NULL;
+	}
+
+	phys = iommu_iova_to_phys(domain, iova);
+	if (phys != 0)
+	{
+		dev_err(dev, "%s, iova=0x%llx, have exits phys=0x%llx\n", __func__, iova, phys);
+		return NULL;
+	}
+
+	gfp &= ~(__GFP_DMA | GFP_DMA32 | __GFP_HIGHMEM);
+	gfp |= __GFP_ZERO;
+
+	if (gfpflags_allow_blocking(gfp))
+	{
+		return iommu_rsv_iova_alloc_remap(dev, iova, size, gfp, attrs);
+	}
+	return NULL;
+}
+EXPORT_SYMBOL(iommu_map_rsv_iova);
+
+int iommu_map_rsv_iova_with_phys(struct device *dev, dma_addr_t iova, unsigned long size, phys_addr_t paddr, unsigned long attrs)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = es_get_iovad(domain);
+	bool coherent = dev_is_dma_coherent(dev);
+	int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
+	size_t iova_off;
+	unsigned int min_pagesz;
+	phys_addr_t phys;
+
+	if (!size)
+	{
+		dev_err(dev, "%s, error, size is 0.\n", __func__);
+		return -EINVAL;
+	}
+	phys = iommu_iova_to_phys(domain, iova);
+	if (phys != 0)
+	{
+		dev_err(dev, "%s, iova=0x%llx, have exits phys=0x%llx\n", __func__, iova, paddr);
+		return -EINVAL;
+	}
+	/* find out the minimum page size supported */
+	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
+
+	/*
+	 *	 * both the virtual address and the physical one, as well as
+	 *		 * the size of the mapping, must be aligned (at least) to the
+	 *		 * size of the smallest page supported by the hardware
+	 */
+
+	if (!IS_ALIGNED(iova | paddr | size, min_pagesz))
+	{
+		pr_err("unaligned: iova 0x%llx pa %pa size 0x%zx min_pagesz 0x%x\n", iova, &paddr, size, min_pagesz);
+		return -EINVAL;
+	}
+
+	iova_off = iova_offset(iovad, phys);
+	size = iova_align(iovad, size + iova_off);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+    if (iommu_map(domain, iova, paddr - iova_off, size, ioprot, GFP_ATOMIC))
+#else
+	if (iommu_map_atomic(domain, iova, paddr - iova_off, size, ioprot))
+#endif
+	{
+		free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad));
+		return -EIO;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(iommu_map_rsv_iova_with_phys);
+
+int iommu_unmap_rsv_iova(struct device *dev, void *cpu_addr, dma_addr_t iova, unsigned long size)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = es_get_iovad(domain);
+	phys_addr_t phys;
+
+	phys = iommu_iova_to_phys(domain, iova);
+	if (phys == 0)
+	{
+		dev_dbg(dev, "%s, iova=0x%llx, have not maps\n", __func__, iova);
+		return -EFAULT;
+	}
+
+	if (!size)
+	{
+		return -EINVAL;
+	}
+	size = iova_align(iovad, size);
+	iommu_unmap(domain, iova, size);
+
+	if (cpu_addr != NULL && __iommu_dma_free)
+	{
+		__iommu_dma_free(dev, size, cpu_addr);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(iommu_unmap_rsv_iova);
+
+ssize_t iommu_rsv_iova_map_sgt(struct device *dev, dma_addr_t iova, struct sg_table *sgt, unsigned long attrs, size_t buf_size)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	bool coherent = dev_is_dma_coherent(dev);
+	int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
+	struct scatterlist *sg;
+	phys_addr_t phys;
+	int i;
+	size_t size = 0;
+	ssize_t ret = 0;
+
+	phys = iommu_iova_to_phys(domain, iova);
+	if (phys != 0) {
+		if (phys != sg_phys(sgt->sgl)) {
+			dev_err(dev, "%s, iova=0x%llx, have exits phys=0x%llx\n", __func__, iova, phys);
+			return -EINVAL;
+		}
+		else {
+			dev_dbg(dev, "%s, iova=0x%llx mapped!\n", __func__, iova);
+			return ret;
+		}
+	}
+#if 0
+	if (!(ioprot & IOMMU_CACHE))
+	{
+		int i;
+
+		for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
+			dma_prep_coherent(sg_page(sg), sg->length);
+	}
+#endif
+	for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
+		size += sg->length;
+
+	if (size != buf_size) {
+		dev_err(dev, "%s %d fail, sg size 0x%lx != buf_size 0x%lx\n", __func__, __LINE__, size, buf_size);
+		return -EINVAL;
+	}
+
+	ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot, GFP_KERNEL);
+	if (ret < 0 || ret < size)
+	{
+		dev_err(dev, "%s %d fail, ret %ld, size %ld\n", __func__, __LINE__, ret, size);
+		goto out;
+	}
+
+	return ret;
+out:
+	if (ret != -ENOMEM && ret != -EREMOTEIO)
+		return -EINVAL;
+
+	return ret;
+}
+EXPORT_SYMBOL(iommu_rsv_iova_map_sgt);
+
+static void eswin_iommu_put_func(void)
+{
+	if (__iommu_dma_free)
+	{
+		__iommu_dma_free = NULL;
+	}
+
+	if (es_common_pages_remap)
+	{
+		es_common_pages_remap = NULL;
+	}
+
+	if (__iommu_dma_unmap)
+	{
+		__iommu_dma_unmap = NULL;
+	}
+
+	if (dma_prep_coherent)
+	{
+		dma_prep_coherent = NULL;
+	}
+}
+
+void *kaddr_lookup_name(const char *fname_raw)
+{
+    int i;
+    unsigned long kaddr;
+    char *fname;
+    char fname_lookup[KSYM_SYMBOL_LEN];
+
+    fname = kzalloc(strlen(fname_raw)+5, GFP_KERNEL);
+    if (!fname) {
+        return NULL;
+    }
+    strcpy(fname, fname_raw);
+    strcat(fname, "+0x0");
+    fname[strlen(fname_raw) + 4] = '\0';
+
+    kaddr = (unsigned long) &sprint_symbol;
+    kaddr &= 0xfffffffffffff000;
+
+    for (i = 0x0 ; i < 0x1000000; i++ )
+    {
+        sprint_symbol(fname_lookup, kaddr);
+
+        if (strncmp(fname_lookup, fname, strlen(fname)) == 0)
+        {
+            kfree(fname);
+            return (void *)kaddr;
+        }
+        kaddr += 0x2;
+    }
+
+    kaddr = (unsigned long) &sprint_symbol;
+    kaddr &= 0xfffffffffffff000;
+
+    for (i = 0x1000000; i >= 0 && kaddr; i--) {
+        sprint_symbol(fname_lookup, kaddr);
+        if (strncmp(fname_lookup, fname, strlen(fname)) == 0)
+        {
+            kfree(fname);
+            return (void *)kaddr;
+        }
+        kaddr -= 0x2;
+    }
+    kfree(fname);
+    return NULL;
+}
+
+static int __init eswin_iommu_rsv_init(void)
+{
+
+    klp_name = kaddr_lookup_name("kallsyms_lookup_name");
+    if (klp_name == NULL) {
+        printk("cannot lookup name.\n");
+        return -EINVAL;
+    }
+    __iommu_dma_free = (void *)klp_name("__iommu_dma_free");
+	if (!__iommu_dma_free)
+	{
+		goto err;
+	}
+
+	es_common_pages_remap = (void *)klp_name("dma_common_pages_remap");
+	if (!es_common_pages_remap)
+	{
+		goto err;
+	}
+
+	__iommu_dma_unmap = (void *)klp_name("__iommu_dma_unmap");
+	if (!__iommu_dma_unmap)
+	{
+		goto err;
+	}
+
+	dma_prep_coherent = (void *)klp_name("arch_dma_prep_coherent");
+	if (!dma_prep_coherent) {
+        goto err;
+    }
+
+	return 0;
+err:
+	eswin_iommu_put_func();
+	return -EINVAL;
+}
+
+static void __exit eswin_iommu_rsv_exit(void)
+{
+	eswin_iommu_put_func();
+}
+
+module_init(eswin_iommu_rsv_init);
+module_exit(eswin_iommu_rsv_exit);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/memory/eswin/es_malloc_dmabuf/Kconfig b/drivers/memory/eswin/es_malloc_dmabuf/Kconfig
new file mode 100644
index 000000000000..a3f7d1ca0da6
--- /dev/null
+++ b/drivers/memory/eswin/es_malloc_dmabuf/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_MALLOC_DMABUF
+	tristate "ESWIN Malloc to DMA-BUF"
+	help
+	  ESWIN Malloc to DMA-BUF.
diff --git a/drivers/memory/eswin/es_malloc_dmabuf/Makefile b/drivers/memory/eswin/es_malloc_dmabuf/Makefile
new file mode 100644
index 000000000000..6a8e2cecc2cc
--- /dev/null
+++ b/drivers/memory/eswin/es_malloc_dmabuf/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_ESWIN_MALLOC_DMABUF) += es_malloc_dmabuf.o
\ No newline at end of file
diff --git a/drivers/memory/eswin/es_malloc_dmabuf/es_malloc_dmabuf.c b/drivers/memory/eswin/es_malloc_dmabuf/es_malloc_dmabuf.c
new file mode 100644
index 000000000000..f573f7f9891f
--- /dev/null
+++ b/drivers/memory/eswin/es_malloc_dmabuf/es_malloc_dmabuf.c
@@ -0,0 +1,616 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Malloc To DMA-BUF Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yuyang Cong <congyuyang@eswincomputing.com>
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/uaccess.h>
+#include <linux/pagemap.h>
+#include <linux/highmem.h>
+#include <linux/dma-buf.h>
+#include <uapi/linux/dma-heap.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/types.h>
+#include <linux/sched/mm.h>
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <linux/dma-map-ops.h>
+#include <linux/eswin_rsvmem_common.h>
+#include <linux/scatterlist.h>
+#include <linux/vmalloc.h>
+
+#include <uapi/linux/malloc_dmabuf.h>
+
+#define DRIVER_NAME "malloc_dmabuf"
+
+struct malloc_dmabuf_attachment {
+    struct device *dev;
+    struct sg_table *table;
+    struct list_head list;
+    bool mapped;
+};
+struct malloc_dmabuf_priv {
+    struct page **pages;
+    unsigned int nr_pages;
+    unsigned int offset_in_first_page;
+    size_t size;
+    struct mutex lock;
+    struct sg_table sg_table;
+    bool sg_init;
+    struct list_head attachments;
+    struct list_head list;
+    void *vaddr;
+    int vmap_cnt;
+    unsigned long fd_flags;  /* Used to determine vmap properties, e.g., O_DSYNC for uncached */
+};
+
+static struct sg_table *dup_sg_table(struct sg_table *orig)
+{
+    struct sg_table *new_table;
+    int ret, i;
+    struct scatterlist *sg, *new_sg;
+
+    new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
+    if (!new_table)
+        return ERR_PTR(-ENOMEM);
+
+    ret = sg_alloc_table(new_table, orig->nents, GFP_KERNEL);
+    if (ret) {
+        kfree(new_table);
+        return ERR_PTR(-ENOMEM);
+    }
+
+    new_sg = new_table->sgl;
+	for_each_sgtable_sg(orig, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+    return new_table;
+}
+
+
+static int malloc_dmabuf_attach(struct dma_buf *dmabuf,
+                                struct dma_buf_attachment *attachment)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    struct malloc_dmabuf_attachment *a;
+    struct sg_table *table;
+
+    if (!buffer->sg_init) {
+        pr_err("No sg_table in buffer priv when attach!\n");
+        return -EINVAL;
+    }
+
+    a = kzalloc(sizeof(*a), GFP_KERNEL);
+    if (!a)
+        return -ENOMEM;
+
+    table = dup_sg_table(&buffer->sg_table);
+    if (IS_ERR(table)) {
+        kfree(a);
+        return PTR_ERR(table);
+    }
+
+    a->table = table;
+    a->dev = attachment->dev;
+    a->mapped = false;
+    INIT_LIST_HEAD(&a->list);
+
+    attachment->priv = a;
+
+    mutex_lock(&buffer->lock);
+    list_add(&a->list, &buffer->attachments);
+    mutex_unlock(&buffer->lock);
+
+    pr_debug("%s: device %s attached\n", __func__, dev_name(attachment->dev));
+
+    return 0;
+}
+
+static void malloc_dmabuf_detach(struct dma_buf *dmabuf,
+                                 struct dma_buf_attachment *attachment)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    struct malloc_dmabuf_attachment *a = attachment->priv;
+
+    mutex_lock(&buffer->lock);
+    list_del(&a->list);
+    mutex_unlock(&buffer->lock);
+
+    sg_free_table(a->table);
+    kfree(a->table);
+    kfree(a);
+
+    pr_debug("%s: device %s detached\n", __func__, dev_name(attachment->dev));
+}
+
+static struct sg_table *malloc_dmabuf_map_dma_buf(struct dma_buf_attachment *attach,
+                                                  enum dma_data_direction dir)
+{
+    struct malloc_dmabuf_attachment *a = attach->priv;
+    struct sg_table *table = a->table;
+    int ret;
+
+    ret = dma_map_sgtable(attach->dev, table, dir, 0);
+    if (ret)
+        return ERR_PTR(-ENOMEM);
+
+    a->mapped = true;
+    return table;
+}
+
+static void malloc_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach,
+                                        struct sg_table *sgt,
+                                        enum dma_data_direction dir)
+{
+    struct malloc_dmabuf_attachment *a = attach->priv;
+
+    a->mapped = false;
+
+    dma_unmap_sgtable(attach->dev, sgt, dir, 0);
+}
+
+static void malloc_dmabuf_release(struct dma_buf *dmabuf)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+
+    if (buffer->pages && buffer->nr_pages > 0) {
+        unpin_user_pages(buffer->pages, buffer->nr_pages);
+    }
+    
+    if (buffer->sg_init) {
+        sg_free_table(&buffer->sg_table);
+        buffer->sg_init = false;
+    }
+
+    kfree(buffer->pages);
+    kfree(buffer);
+}
+
+static int malloc_dmabuf_begin_cpu_access(struct dma_buf *dmabuf,
+                                          enum dma_data_direction direction)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    struct sg_table *table = &buffer->sg_table;
+    struct scatterlist *sg;
+    int i;
+
+    mutex_lock(&buffer->lock);
+
+    if (buffer->vmap_cnt) {
+        invalidate_kernel_vmap_range(buffer->vaddr, buffer->size);
+    }
+        
+#ifndef QEMU_DEBUG
+    /* Cache sync since we used DMA_ATTR_SKIP_CPU_SYNC */
+    for_each_sg(table->sgl, sg, table->nents, i)
+        arch_sync_dma_for_cpu(sg_phys(sg), sg->length, direction);
+#endif
+    mutex_unlock(&buffer->lock);
+    return 0;
+}
+
+static int malloc_dmabuf_end_cpu_access(struct dma_buf *dmabuf,
+                                        enum dma_data_direction direction)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    struct sg_table *table = &buffer->sg_table;
+    struct scatterlist *sg;
+    int i;
+
+    mutex_lock(&buffer->lock);
+
+    if (buffer->vmap_cnt){
+        flush_kernel_vmap_range(buffer->vaddr, buffer->size);
+    }
+
+#ifndef QEMU_DEBUG
+    /* Sync back for device after CPU access */
+    for_each_sg(table->sgl, sg, table->nents, i) {
+        arch_sync_dma_for_device(sg_phys(sg), sg->length, direction);
+    }
+#endif
+    mutex_unlock(&buffer->lock);
+    return 0;
+}
+
+
+static void *malloc_dmabuf_do_vmap(struct dma_buf *dmabuf)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    pgprot_t prot = PAGE_KERNEL;
+    struct sg_table *table = &buffer->sg_table;
+    int npages = buffer->nr_pages;
+    struct page **pages = vmalloc(sizeof(struct page *) * npages);
+    struct page **tmp = pages;
+    struct sg_page_iter piter;
+    void *vaddr;
+
+    if (!pages)
+        return ERR_PTR(-ENOMEM);
+
+    for_each_sgtable_page(table, &piter, 0) {
+		WARN_ON(tmp - pages >= npages);
+		*tmp++ = sg_page_iter_page(&piter);
+	}
+
+    if (buffer->fd_flags & O_DSYNC) {
+#ifndef QEMU_DEBUG
+        prot = pgprot_dmacoherent(PAGE_KERNEL);
+#endif
+        pr_debug("%s: using uncached mapping, prot=0x%x\n", __func__, (unsigned int)pgprot_val(prot));
+    }
+    else {
+        pr_debug("%s: using cached mapping, prot=0x%x\n", __func__, (unsigned int)pgprot_val(prot));
+    }
+
+    vaddr = vmap(pages, npages, VM_MAP, prot);
+    vfree(pages);
+
+    if (!vaddr)
+        return ERR_PTR(-ENOMEM);
+
+    return vaddr;
+}
+
+static int malloc_dmabuf_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    void *vaddr;
+    int ret = 0;
+
+    mutex_lock(&buffer->lock);
+    if (buffer->vmap_cnt > 0) {
+        buffer->vmap_cnt++;
+        dma_buf_map_set_vaddr(map, buffer->vaddr + buffer->offset_in_first_page);
+        goto out;
+    }
+
+    vaddr = malloc_dmabuf_do_vmap(dmabuf);
+    if (IS_ERR(vaddr)) {
+        ret = PTR_ERR(vaddr);
+        goto out;
+    }
+    buffer->vaddr = vaddr;
+    buffer->vmap_cnt++;
+    dma_buf_map_set_vaddr(map, buffer->vaddr + buffer->offset_in_first_page);
+
+out:
+    mutex_unlock(&buffer->lock);
+    return ret;
+}
+
+
+static void malloc_dmabuf_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    void *vaddr;
+    vaddr = buffer->vaddr;
+    mutex_lock(&buffer->lock);
+    if (--buffer->vmap_cnt == 0) {
+        vunmap(vaddr);
+        buffer->vaddr = NULL;
+    }
+    mutex_unlock(&buffer->lock);
+    dma_buf_map_clear(map);
+}
+
+static int malloc_dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+    struct malloc_dmabuf_priv *buffer = dmabuf->priv;
+    struct sg_table *sgt = &buffer->sg_table;
+    unsigned long addr = vma->vm_start;
+    struct scatterlist *sg;
+    struct page *page = NULL;
+    unsigned long pgoff = vma->vm_pgoff, mapsize = 0;
+    unsigned long size_remaining = vma->vm_end - vma->vm_start;//vma_pages(vma);
+    unsigned int nents = 0;
+    int i, ret = 0;
+
+    if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
+        return -EINVAL;
+
+    vma->vm_private_data = dmabuf;
+    /* support mman flag MAP_SHARED_VALIDATE | VM_NORESERVE, used to map uncached memory to user space.
+        Users should guarantee this buffer has been flushed to cache already.
+        */
+    if (vma->vm_flags & VM_NORESERVE) {
+        vm_flags_clear(vma, VM_NORESERVE);
+        #ifndef QEMU_DEBUG
+        vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+        #endif
+        /* skip sync cache, users should guarantee the cache is clean after done using it in
+            cached mode(i.e, ES_SYS_Mmap(SYS_CACHE_MODE_CACHED))
+        */
+    }
+
+    for_each_sg(sgt->sgl, sg, sgt->orig_nents, i) {
+        pr_debug("sgl:%d, phys:0x%llx, sg->length:%d\n", i, sg_phys(sg), sg->length);
+
+        page = sg_page(sg);
+        if (nents == 0) {
+            mapsize = sg->length - (pgoff << PAGE_SHIFT);
+            mapsize = min(size_remaining, mapsize);
+            ret = remap_pfn_range(vma, addr, page_to_pfn(page) + pgoff, mapsize,
+                    vma->vm_page_prot);
+            pr_debug("nents:%d, sgl:%d, pgoff:0x%lx, mapsize:0x%lx, phys:0x%llx\n",
+                nents, i, pgoff, mapsize, pfn_to_phys(page_to_pfn(page) + pgoff));
+        }
+        else {
+            mapsize = min((unsigned int)size_remaining, (sg->length));
+            ret = remap_pfn_range(vma, addr, page_to_pfn(page), mapsize,
+                    vma->vm_page_prot);
+            pr_debug("nents:%d, sgl:%d, mapsize:0x%lx, phys:0x%llx\n", nents, i, mapsize, page_to_phys(page));
+        }
+        pgoff = 0;
+        nents++;
+        if (ret)
+            return ret;
+
+        addr += PAGE_ALIGN(mapsize);
+        size_remaining -= PAGE_ALIGN(mapsize);
+        if (size_remaining == 0){
+            return 0;
+        }
+    }
+
+    return 0;
+}
+
+static const struct dma_buf_ops malloc_dmabuf_ops = {
+    .attach = malloc_dmabuf_attach,
+    .detach = malloc_dmabuf_detach,
+    .map_dma_buf = malloc_dmabuf_map_dma_buf,
+    .unmap_dma_buf = malloc_dmabuf_unmap_dma_buf,
+    .release = malloc_dmabuf_release,
+    .vmap = malloc_dmabuf_vmap,
+    .vunmap = malloc_dmabuf_vunmap,
+    .begin_cpu_access = malloc_dmabuf_begin_cpu_access,
+    .end_cpu_access = malloc_dmabuf_end_cpu_access,
+    .mmap = malloc_dmabuf_mmap,
+};
+
+static int build_sg_table_for_priv(struct malloc_dmabuf_priv *buffer)
+{
+    int ret;
+
+    ret = sg_alloc_table_from_pages(&buffer->sg_table,
+                                    buffer->pages, buffer->nr_pages,
+                                    buffer->offset_in_first_page, buffer->size,
+                                    GFP_KERNEL);
+    if (ret == 0) {
+        buffer->sg_init = true;
+        return 0;
+    }
+
+    pr_err("Failed to build sg_table, ret=%d\n", ret);
+    return ret;
+}
+
+static int esw_convert_userptr_to_dmabuf(unsigned long arg)
+{
+    struct esw_userptr u;
+    struct malloc_dmabuf_priv *buffer;
+    struct dma_buf *dmabuf;
+    struct mm_struct *mm = current->mm;
+    DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+    unsigned long start, end;
+    int nr_pages;
+    int pinned, fd, ret;
+
+    if (copy_from_user(&u, (void __user *)arg, sizeof(u)))
+        return -EFAULT;
+
+    if (!access_ok((void __user *)(uintptr_t)u.user_ptr, u.length))
+        return -EFAULT;
+
+    if (u.length == 0)
+        return -EINVAL;
+
+    start = (u.user_ptr & PAGE_MASK);
+    end = ((u.user_ptr + u.length - 1) & PAGE_MASK);
+    nr_pages = (int)((end - start) / PAGE_SIZE + 1);
+    u.aligned_length = nr_pages << PAGE_SHIFT;
+
+    buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+    if (!buffer)
+        return -ENOMEM;
+
+    buffer->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
+    if (!buffer->pages) {
+        kfree(buffer);
+        return -ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&buffer->attachments);
+    mutex_init(&buffer->lock);
+
+    mmap_read_lock(mm);
+    pinned = pin_user_pages(start, nr_pages, FOLL_WRITE | FOLL_LONGTERM, buffer->pages);
+    mmap_read_unlock(mm);
+
+    if (pinned < nr_pages) {
+        if (pinned > 0) {
+            unpin_user_pages(buffer->pages, pinned);
+        }
+        kfree(buffer->pages);
+        kfree(buffer);
+        return -EFAULT;
+    }
+
+    buffer->nr_pages = nr_pages;
+    buffer->offset_in_first_page = offset_in_page(u.user_ptr);
+    buffer->size = u.aligned_length;
+    buffer->fd_flags = u.fd_flags;
+
+    ret = build_sg_table_for_priv(buffer);
+    if (ret) {
+        unpin_user_pages(buffer->pages, buffer->nr_pages);
+        kfree(buffer->pages);
+        kfree(buffer);
+        return ret;
+    }
+
+    exp_info.ops = &malloc_dmabuf_ops;
+    exp_info.size = buffer->size;
+    exp_info.flags = O_RDWR | O_CLOEXEC;
+    exp_info.priv = buffer;
+
+    dmabuf = dma_buf_export(&exp_info);
+    if (IS_ERR(dmabuf)) {
+        ret = PTR_ERR(dmabuf);
+        sg_free_table(&buffer->sg_table);
+        unpin_user_pages(buffer->pages, buffer->nr_pages);
+        kfree(buffer->pages);
+        kfree(buffer);
+        return ret;
+    }
+
+    fd = dma_buf_fd(dmabuf, O_CLOEXEC);
+    if (fd < 0) {
+        dma_buf_put(dmabuf);
+        return fd;
+    }
+
+    u.dma_fd = fd;
+    u.offset_in_first_page = buffer->offset_in_first_page;
+    if (copy_to_user((void __user *)arg, &u, sizeof(u))) {
+        dma_buf_put(dmabuf);
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+static int esw_get_dmabuf_info(unsigned long arg)
+{
+    struct esw_userptr u;
+    struct dma_buf *dmabuf;
+    struct malloc_dmabuf_priv *buffer;
+    int ret = 0;
+
+    if (copy_from_user(&u, (void __user *)arg, sizeof(u)))
+        return -EFAULT;
+
+    dmabuf = dma_buf_get(u.dma_fd);
+    if (IS_ERR(dmabuf)) {
+        ret = PTR_ERR(dmabuf);
+        return ret;
+    }
+
+    if (dmabuf->ops == &malloc_dmabuf_ops) {
+        buffer = (struct malloc_dmabuf_priv *)dmabuf->priv;
+        u.mem_type = true;
+        u.offset_in_first_page = buffer->offset_in_first_page;
+    } else {
+        u.mem_type = false;
+        u.offset_in_first_page = 0;
+    }
+
+    if (copy_to_user((void __user *)arg, &u, sizeof(u))) {
+        dma_buf_put(dmabuf);
+        return -EFAULT;
+    }
+
+    dma_buf_put(dmabuf);
+    return 0;
+}
+
+static long malloc_dmabuf_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+    int ret = 0;
+
+    switch (cmd) {
+    case ESW_CONVERT_USERPTR_TO_DMABUF:
+        ret = esw_convert_userptr_to_dmabuf(arg);
+        break;
+    case ESW_GET_DMABUF_INFO:
+        ret = esw_get_dmabuf_info(arg);
+        break;
+    default:
+        pr_err(DRIVER_NAME ": invalid cmd=0x%x\n", cmd);
+        ret = -EINVAL;
+        break;
+    }
+
+    return ret;
+}
+
+static int malloc_dmabuf_open(struct inode *inode, struct file *file)
+{
+    pr_debug("%s:%d, success!\n", __func__, __LINE__);
+    return 0;
+}
+
+static int malloc_dmabuf_release_fops(struct inode *inode, struct file *file)
+{
+    pr_debug("%s:%d, success!\n", __func__, __LINE__);
+    return 0;
+}
+
+static const struct file_operations malloc_dmabuf_fops = {
+    .owner          = THIS_MODULE,
+    .open           = malloc_dmabuf_open,
+    .release        = malloc_dmabuf_release_fops,
+    .unlocked_ioctl = malloc_dmabuf_ioctl,
+    .llseek         = no_llseek,
+};
+
+static struct miscdevice malloc_dmabuf_miscdev = {
+    .minor = MISC_DYNAMIC_MINOR,
+    .name  = DRIVER_NAME,
+    .mode = 0666,
+    .fops  = &malloc_dmabuf_fops,
+};
+
+static int __init malloc_dmabuf_init(void)
+{
+    int ret;
+
+    ret = misc_register(&malloc_dmabuf_miscdev);
+    if (ret) {
+        pr_err(DRIVER_NAME ": failed to register misc device: %d\n", ret);
+        return ret;
+    }
+
+    return ret;
+}
+
+static void __exit malloc_dmabuf_exit(void)
+{
+    misc_deregister(&malloc_dmabuf_miscdev);
+}
+
+module_init(malloc_dmabuf_init);
+module_exit(malloc_dmabuf_exit);
+
+MODULE_AUTHOR("Yuyang Cong <congyuyang@eswincomputing.com>");
+MODULE_DESCRIPTION("Malloc To DMA-BUF Driver");
+MODULE_LICENSE("GPL v2");
\ No newline at end of file
diff --git a/drivers/memory/eswin/es_mmz_vb/Kconfig b/drivers/memory/eswin/es_mmz_vb/Kconfig
new file mode 100644
index 000000000000..e290be281abe
--- /dev/null
+++ b/drivers/memory/eswin/es_mmz_vb/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_MMZ_VB
+	tristate "ESWIN MMZ reserved memory VB"
+	help
+	  ESWIN MMZ reserved memory VB device.
diff --git a/drivers/memory/eswin/es_mmz_vb/Makefile b/drivers/memory/eswin/es_mmz_vb/Makefile
new file mode 100644
index 000000000000..fe2bfe6291b1
--- /dev/null
+++ b/drivers/memory/eswin/es_mmz_vb/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ESWIN_MMZ_VB)	+=  mmz_vb.o
+
+
+
diff --git a/drivers/memory/eswin/es_mmz_vb/include/linux/mmz_vb.h b/drivers/memory/eswin/es_mmz_vb/include/linux/mmz_vb.h
new file mode 100644
index 000000000000..e167311a238d
--- /dev/null
+++ b/drivers/memory/eswin/es_mmz_vb/include/linux/mmz_vb.h
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#ifndef __MMZ_VB_H__
+#define __MMZ_VB_H__
+
+#include <linux/types.h>
+#include "../../../eswin_memblock.h"
+#include "../../../es_buddy/es_buddy.h"
+
+
+/**
+ *  select whether the block will be memset to 0 while exporting it as dmabuf.
+ *  1: memset
+ *  0: do NOT memset
+*/
+// #define MMZ_VB_DMABUF_MEMSET	1
+
+#define MMZ_VB_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE)
+
+#define VB_K_POOL_MAX_ID	INT_MAX
+
+/* one block info organized in kernel */
+typedef struct esVB_K_BLOCK_INFO_S {
+	struct page *cma_pages;
+	struct sg_table sg_table; // for buddy allocator
+	struct esVB_K_POOL_INFO_S *pool;
+	int nr;
+}VB_K_BLOCK_INFO_S;
+
+/* one pool info organized in kernel */
+typedef struct esVB_K_POOL_INFO_S {
+	s32 poolId;
+	struct esVB_POOL_CONFIG_S poolCfg;
+	unsigned long *bitmap; // used for block get/release managment
+	struct esVB_K_BLOCK_INFO_S *blocks; // point to the block array
+	struct esVB_K_MMZ_S *partitions; // poiont to the partitions
+	struct hlist_node node;
+	spinlock_t lock;
+	enum esVB_UID_E enVbUid;
+	unsigned long flag;
+}VB_K_POOL_INFO_S;
+
+/* MMZs info in kernel */
+typedef struct esVB_K_MMZ_S {
+	u32 partCnt;
+	// pool is found via idr api
+	struct idr pool_idr;
+	struct rw_semaphore idr_lock;
+	struct mem_block *mem_blocks[ES_VB_MAX_MMZs];
+}VB_K_MMZ_S;
+
+#endif
diff --git a/drivers/memory/eswin/es_mmz_vb/include/linux/mmz_vb_type.h b/drivers/memory/eswin/es_mmz_vb/include/linux/mmz_vb_type.h
new file mode 100644
index 000000000000..8adecbbc4d6b
--- /dev/null
+++ b/drivers/memory/eswin/es_mmz_vb/include/linux/mmz_vb_type.h
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#ifndef __MMZ_VB_TYPE_H__
+#define __MMZ_VB_TYPE_H__
+
+typedef unsigned int ES_U32;
+typedef unsigned long long ES_U64;
+typedef char ES_CHAR;
+typedef ES_U32 VB_POOL;
+
+#endif
diff --git a/drivers/memory/eswin/es_mmz_vb/mmz_vb.c b/drivers/memory/eswin/es_mmz_vb/mmz_vb.c
new file mode 100644
index 000000000000..4e45d6bf4ffd
--- /dev/null
+++ b/drivers/memory/eswin/es_mmz_vb/mmz_vb.c
@@ -0,0 +1,2401 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN MMZ VB driver. MMZ VB stands for Media Memory Zone Video Buffer.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Min Lin <linmin@eswincomputing.com>
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-heap.h>
+#include <linux/dma-map-ops.h>
+#include <linux/highmem.h>
+#include <linux/bitfield.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <asm/uaccess.h>
+#include <linux/dmaengine.h>
+#include <linux/uaccess.h>	/* For copy_to_user/put_user/... */
+#include <linux/of_address.h>
+#include <crypto/hash.h>
+#include <linux/delay.h>
+#include <linux/hashtable.h>
+#include <linux/es_proc.h>
+#include <linux/dma-resv.h>
+#include <linux/dmabuf-heap-import-helper.h>
+#include "include/linux/mmz_vb_type.h" /*must include before es_vb_user.h*/
+#include <uapi/linux/es_vb_user.h>
+#include <uapi/linux/mmz_vb.h>
+#include "include/linux/mmz_vb.h"
+#include <asm/dma-noncoherent.h>
+
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+#define DRIVER_NAME 	"mmz_vb"
+#define MMZ_VB_DMABUF_NAME		"mmz_vb_dmabuf"
+#define MMZ_VB_DMABUF_SPLITTED_NAME	"mmz_vb_dmabuf_splitted"
+
+#define vb_fmt(fmt)	"[%s-MMZ_VB]: " fmt
+#define info_fmt(fmt)	vb_fmt("%s[%d]: " fmt), "INFO",	\
+		        __func__, __LINE__
+#define dbg_fmt(fmt)	vb_fmt("%s[%d]: " fmt), "DEBUG",	\
+		        __func__, __LINE__
+#define err_fmt(fmt)	vb_fmt("%s[%d]: " fmt), "ERROR",	\
+		        __func__, __LINE__
+
+#define vb_info(fmt, args...) \
+	do {							\
+		printk(KERN_INFO info_fmt(fmt), ##args);	\
+	} while (0)
+
+#define vb_debug(fmt, args...) \
+	do {							\
+		printk(KERN_DEBUG dbg_fmt(fmt), ##args);	\
+	} while (0)
+#define vb_err(fmt, args...) \
+	do {							\
+		printk(KERN_ERR err_fmt(fmt), ##args);		\
+	} while (0)
+
+static struct device *mmz_vb_dev;
+static struct mmz_vb_priv *g_mmz_vb_priv = NULL;
+
+struct mmz_vb_priv {
+	struct device *dev;
+	struct esVB_K_MMZ_S partitions;
+	atomic_t allocBlkcnt;   /*total block allocated*/
+	struct hlist_head ht[VB_UID_MAX][ES_VB_MAX_MOD_POOL];
+	struct rw_semaphore pool_lock[VB_UID_MAX];
+	unsigned long cfg_flag[VB_UID_MAX];	/*flag for pVbConfig*/
+	struct esVB_CONFIG_S *pVbConfig[VB_UID_MAX];
+	struct mutex cfg_lock[VB_UID_MAX];
+};
+
+#define do_vb_pool_size(pPool)	(pPool->poolCfg.blkCnt * pPool->poolCfg.blkSize)
+static int vb_find_pool_by_id_unlock(VB_POOL poolId, struct esVB_K_POOL_INFO_S **ppPool);
+static int vb_find_pool_by_id(VB_POOL poolId, struct esVB_K_POOL_INFO_S **ppPool);
+static int vb_pool_size(VB_POOL poolId, u64 *pPoolSize);
+// static int vb_flush_pool(struct esVB_FLUSH_POOL_CMD_S *flushPoolCmd);
+static int vb_get_block(struct esVB_GET_BLOCK_CMD_S *getBlkCmd, struct esVB_K_BLOCK_INFO_S **ppBlk);
+static void vb_release_block(struct esVB_K_BLOCK_INFO_S *pBlk);
+static int vb_pool_get_free_block_cnt_unlock(struct esVB_K_POOL_INFO_S *pool);
+static int vb_is_splitted_blk(int fd, bool *isSplittedBlk);
+static int vb_blk_to_pool(struct esVB_BLOCK_TO_POOL_CMD_S *blkToPoolCmd);
+static int vb_get_blk_offset(struct esVB_GET_BLOCKOFFSET_CMD_S *getBlkOffsetCmd);
+static int vb_split_dmabuf(struct esVB_SPLIT_DMABUF_CMD_S *splitDmabufCmd);
+static int vb_get_dmabuf_refcnt(struct esVB_DMABUF_REFCOUNT_CMD_S *getDmabufRefCntCmd);
+static int vb_retrieve_mem_node(struct esVB_RETRIEVE_MEM_NODE_CMD_S *retrieveMemNodeCmd,
+	eic770x_memory_type_t *p_mem_type);
+static int vb_get_dmabuf_size(struct esVB_DMABUF_SIZE_CMD_S *getDmabufSizeCmd);
+static int mmz_vb_pool_exit(void);
+static int mmz_vb_init_memory_region(void);
+
+/**
+ * vb dmabuf releated struct
+ *
+ */
+struct mmz_vb_buffer {
+	struct esVB_K_BLOCK_INFO_S *pBlk;
+	struct list_head attachments;
+	struct mutex lock;
+	unsigned long len;
+	struct sg_table *table; // for buddy allocator
+	struct page **pages;
+	int vmap_cnt;
+	void *vaddr;
+};
+
+struct mmz_vb_attachment {
+	struct device *dev;
+	struct sg_table *table;
+	struct list_head list;
+	bool mapped;
+};
+
+static struct sg_table *dup_sg_table(struct sg_table *table)
+{
+	struct sg_table *new_table;
+	int ret, i;
+	struct scatterlist *sg, *new_sg;
+
+	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
+	if (!new_table)
+		return ERR_PTR(-ENOMEM);
+
+	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
+	if (ret) {
+		kfree(new_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_sg = new_table->sgl;
+	for_each_sgtable_sg(table, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+	return new_table;
+}
+
+static int mmz_vb_attach(struct dma_buf *dmabuf,
+			   struct dma_buf_attachment *attachment)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct mmz_vb_attachment *a;
+	struct sg_table *table;
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a)
+		return -ENOMEM;
+
+	table = dup_sg_table(buffer->table);
+	if (IS_ERR(table)) {
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	a->table = table;
+	a->dev = attachment->dev;
+	INIT_LIST_HEAD(&a->list);
+	a->mapped = false;
+
+	attachment->priv = a;
+
+	mutex_lock(&buffer->lock);
+	list_add(&a->list, &buffer->attachments);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static void mmz_vb_detach(struct dma_buf *dmabuf,
+			    struct dma_buf_attachment *attachment)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct mmz_vb_attachment *a = attachment->priv;
+
+	mutex_lock(&buffer->lock);
+	list_del(&a->list);
+	mutex_unlock(&buffer->lock);
+
+	sg_free_table(a->table);
+	kfree(a->table);
+	kfree(a);
+}
+
+static struct sg_table *mmz_vb_map_dma_buf(struct dma_buf_attachment *attachment,
+					     enum dma_data_direction direction)
+{
+	struct mmz_vb_attachment *a = attachment->priv;
+	struct sg_table *table =a->table;
+	int ret;
+
+	/* Skipt cache sync, since it takes a lot of time when import to device.
+	*  It's the user's responsibility for guaranteeing the cache coherency by
+	   flusing cache explicitly before importing to device.
+	*/
+	ret = dma_map_sgtable(attachment->dev, table, direction, DMA_ATTR_SKIP_CPU_SYNC);
+
+	if (ret)
+		return ERR_PTR(-ENOMEM);
+	a->mapped = true;
+	return table;
+}
+
+static void mmz_vb_unmap_dma_buf(struct dma_buf_attachment *attachment,
+				   struct sg_table *table,
+				   enum dma_data_direction direction)
+{
+	struct mmz_vb_attachment *a = attachment->priv;
+
+	a->mapped = false;
+
+	/* Skipt cache sync, since it takes a lot of time when unmap from device.
+	*  It's the user's responsibility for guaranteeing the cache coherency after
+	   the device has done processing the data.(For example, CPU do NOT read untill
+	   the device has done)
+	*/
+	dma_unmap_sgtable(attachment->dev, table, direction, DMA_ATTR_SKIP_CPU_SYNC);
+}
+
+static int mmz_vb_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+					     enum dma_data_direction direction)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = buffer->table;
+	struct scatterlist *sg;
+	int i;
+
+	mutex_lock(&buffer->lock);
+
+	if (buffer->vmap_cnt)
+		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	/* Since the cache sync was skipped when mmz_vb_map_dma_buf/mmz_vb_unmap_dma_buf,
+	   So force cache sync here when user call ES_SYS_MemFlushCache, eventhough there
+	   is no device attached to this dmabuf.
+	*/
+	#ifndef QEMU_DEBUG
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_cpu(sg_phys(sg), sg->length, direction);
+	#endif
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static int mmz_vb_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
+					   enum dma_data_direction direction)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = buffer->table;
+	struct scatterlist *sg;
+	int i;
+
+	mutex_lock(&buffer->lock);
+
+	if (buffer->vmap_cnt)
+		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	/* Since the cache sync was skipped while mmz_vb_map_dma_buf/mmz_vb_unmap_dma_buf,
+	   So force cache sync here when user call ES_SYS_MemFlushCache, eventhough there
+	   is no device attached to this dmabuf.
+	*/
+	#ifndef QEMU_DEBUG
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_device(sg_phys(sg), sg->length, direction);
+	#endif
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+#if 0
+static int mmz_vb_sync_cache_internal(struct dma_buf *dmabuf, enum dma_data_direction direction)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = buffer->table;
+	struct scatterlist *sg;
+	int i;
+
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_device(sg_phys(sg), sg->length, direction);
+
+
+	return 0;
+}
+#endif
+
+static int mmz_vb_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = buffer->table;
+	unsigned long addr = vma->vm_start;
+	unsigned long pgoff = vma->vm_pgoff, mapsize = 0;
+	unsigned long size_remaining = vma->vm_end - vma->vm_start;
+	struct scatterlist *sg;
+	struct page *page = NULL;
+	unsigned int nents = 0;
+	int i;
+	int ret;
+
+	if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
+		return -EINVAL;
+
+	/* vm_private_data will be used by eswin-ipc-scpu.c.
+	    ipc will import this dmabuf to get iova.
+	*/
+	vma->vm_private_data = dmabuf;
+
+	/* support mman flag MAP_SHARED_VALIDATE | VM_NORESERVE, used to map uncached memory to user space.
+	   The cache needs to be flush first since there might be dirty data in cache.
+	 */
+	if (vma->vm_flags & VM_NORESERVE) {
+		vm_flags_clear(vma, VM_NORESERVE);
+		#ifndef QEMU_DEBUG
+		vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+		#endif
+		/* skip sync cache, users should guarantee the cache is clean after done using it in
+		   cached mode(i.e, ES_SYS_Mmap(SYS_CACHE_MODE_CACHED))
+		*/
+		#if 0
+		pr_debug("%s uncached user memory, flush cache firstly!\n", __func__);
+		if (mmz_vb_sync_cache_internal(dmabuf, DMA_TO_DEVICE)) {
+			vb_err("%s, failed to flush cache!\n",__func__);
+			return -EINVAL;
+		}
+		#endif
+	}
+	pr_debug("%s, size_remaining:0x%lx, pgoff:0x%lx, dmabuf->size:0x%lx, start_phys:0x%llx\n",
+		__func__, size_remaining, pgoff, dmabuf->size, sg_phys(table->sgl));
+	for_each_sg(table->sgl, sg, table->orig_nents, i) {
+		pr_debug("sgl:%d, phys:0x%llx, length:0x%x\n", i, sg_phys(sg), sg->length);
+		if (pgoff >= (sg->length >> PAGE_SHIFT)) {
+			pgoff -= (sg->length >> PAGE_SHIFT);
+			continue;
+		}
+
+		page = sg_page(sg);
+		if (nents == 0) {
+			mapsize = sg->length - (pgoff << PAGE_SHIFT);
+			mapsize = min(size_remaining, mapsize);
+			ret = remap_pfn_range(vma, addr, page_to_pfn(page) + pgoff, mapsize,
+					vma->vm_page_prot);
+			pr_debug("nents:%d, sgl:%d, pgoff:0x%lx, mapsize:0x%lx, phys:0x%llx\n",
+				nents, i, pgoff, mapsize, pfn_to_phys(page_to_pfn(page) + pgoff));
+		}
+		else {
+			mapsize = min((unsigned int)size_remaining, (sg->length));
+			ret = remap_pfn_range(vma, addr, page_to_pfn(page), mapsize,
+					vma->vm_page_prot);
+			pr_debug("nents:%d, sgl:%d, mapsize:0x%lx, phys:0x%llx\n", nents, i, mapsize, page_to_phys(page));
+		}
+		pgoff = 0;
+		nents++;
+
+		if (ret)
+			return ret;
+
+		addr += mapsize;
+		size_remaining -= mapsize;
+		if (size_remaining == 0)
+			return 0;
+	}
+
+	return 0;
+}
+
+static void *mmz_vb_do_vmap(struct dma_buf *dmabuf)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	struct esVB_K_POOL_INFO_S *pool = buffer->pBlk->pool;
+	pgprot_t prot = PAGE_KERNEL;
+	struct sg_table *table = buffer->table;
+	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
+	struct page **pages = vmalloc(sizeof(struct page *) * npages);
+	struct page **tmp = pages;
+	struct sg_page_iter piter;
+	void *vaddr;
+
+	if (!pages)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_sgtable_page(table, &piter, 0) {
+		WARN_ON(tmp - pages >= npages);
+		*tmp++ = sg_page_iter_page(&piter);
+	}
+
+	/* The property of this dmabuf in kernel space is determined by SYS_CACHE_MODE_E of the pool . */
+	if (pool->poolCfg.enRemapMode == SYS_CACHE_MODE_NOCACHE) {
+		pr_debug("%s uncached kernel buffer!\n", __func__);
+		#ifndef QEMU_DEBUG
+		prot = pgprot_dmacoherent(PAGE_KERNEL);
+		#endif
+	}
+	else {
+		pr_debug("%s cached kernel buffer!\n", __func__);
+	}
+
+	vaddr = vmap(pages, npages, VM_MAP, prot);
+	vfree(pages);
+
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+static int mmz_vb_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+	void *vaddr;
+	int ret = 0;
+
+	mutex_lock(&buffer->lock);
+	if (buffer->vmap_cnt) {
+		buffer->vmap_cnt++;
+		dma_buf_map_set_vaddr(map, buffer->vaddr);
+		goto out;
+	}
+
+	vaddr = mmz_vb_do_vmap(dmabuf);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto out;
+	}
+	buffer->vaddr = vaddr;
+	buffer->vmap_cnt++;
+	dma_buf_map_set_vaddr(map, buffer->vaddr);
+out:
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static void mmz_vb_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+
+	mutex_lock(&buffer->lock);
+	if (!--buffer->vmap_cnt) {
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+	mutex_unlock(&buffer->lock);
+	dma_buf_map_clear(map);
+}
+
+static void mmz_vb_dma_buf_release(struct dma_buf *dmabuf)
+{
+	struct mmz_vb_buffer *buffer = dmabuf->priv;
+
+	if (buffer->vmap_cnt > 0) {
+		WARN(1, "%s: buffer still mapped in the kernel\n", __func__);
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+
+	/* release block. In fact, release block to pool */
+	vb_release_block(buffer->pBlk);
+
+	kfree(buffer);
+}
+
+static const struct dma_buf_ops mmz_vb_buf_ops = {
+	.attach = mmz_vb_attach,
+	.detach = mmz_vb_detach,
+	.map_dma_buf = mmz_vb_map_dma_buf,
+	.unmap_dma_buf = mmz_vb_unmap_dma_buf,
+	.begin_cpu_access = mmz_vb_dma_buf_begin_cpu_access,
+	.end_cpu_access = mmz_vb_dma_buf_end_cpu_access,
+	.mmap = mmz_vb_mmap,
+	.vmap = mmz_vb_vmap,
+	.vunmap = mmz_vb_vunmap,
+	.release = mmz_vb_dma_buf_release,
+};
+
+static const unsigned int orders[] = {MAX_ORDER-1, 9, 0};
+#define NUM_ORDERS ARRAY_SIZE(orders)
+
+static struct page *alloc_largest_available(struct mem_block *memblock,
+						unsigned long size,
+						unsigned int max_order)
+{
+	struct page *page;
+	int i;
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		if (size <  (PAGE_SIZE << orders[i]))
+			continue;
+		if (max_order < orders[i])
+			continue;
+
+		page = es_alloc_pages(memblock, orders[i]);
+		if (!page)
+			continue;
+		return page;
+	}
+	return NULL;
+}
+
+static int vb_blk_pages_allocate(struct mem_block *memblock, struct esVB_K_BLOCK_INFO_S *blocks, unsigned long len)
+{
+	unsigned long size_remaining = len;
+	unsigned int max_order = orders[0];
+	struct sg_table *table;
+	struct scatterlist *sg;
+	struct list_head pages;
+	struct page *page, *tmp_page;
+	int i, ret = -ENOMEM;
+
+	INIT_LIST_HEAD(&pages);
+	i = 0;
+	while (size_remaining > 0) {
+		/*
+		 * Avoid trying to allocate memory if the process
+		 * has been killed by SIGKILL
+		 */
+		if (fatal_signal_pending(current)) {
+			ret = -EINTR;
+			goto free_buffer;
+		}
+
+		page = alloc_largest_available(memblock, size_remaining, max_order);
+		if (!page)
+			goto free_buffer;
+
+		list_add_tail(&page->lru, &pages);
+		size_remaining -= page_size(page);
+		max_order = compound_order(page);
+		i++;
+		// pr_debug("page_size(page)=0x%lx, phys_addr=0x%llx, max_order=%d\n",
+			// page_size(page), page_to_phys(page), max_order);
+
+	}
+
+	table = &blocks->sg_table;
+	if (sg_alloc_table(table, i, GFP_KERNEL))
+		goto free_buffer;
+
+	sg = table->sgl;
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		sg_set_page(sg, page, page_size(page), 0);
+		sg = sg_next(sg);
+		list_del(&page->lru);
+	}
+
+	return 0;
+
+free_buffer:
+	list_for_each_entry_safe(page, tmp_page, &pages, lru)
+		es_free_pages(memblock, page);
+
+
+	return ret;
+}
+
+static void vb_blk_pages_release(struct mem_block *memblock, struct esVB_K_BLOCK_INFO_S *blocks)
+{
+	struct sg_table *table;
+	struct scatterlist *sg;
+	int i;
+
+	table = &blocks->sg_table;
+	for_each_sgtable_sg(table, sg, i) {
+		struct page *page = sg_page(sg);
+		// pr_debug("%s:%d,page_size(page)=0x%lx, phys_addr=0x%llx\n",
+			// __func__, __LINE__, page_size(page), page_to_phys(page));
+		es_free_pages(memblock, page);
+	}
+	sg_free_table(table);
+
+}
+
+static int vb_blk_dmabuf_alloc(struct esVB_GET_BLOCK_CMD_S *getBlkCmd)
+{
+	struct mmz_vb_buffer *buffer;
+	struct esVB_K_BLOCK_INFO_S *pBlk;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	size_t size;
+	struct dma_buf *dmabuf;
+	#ifdef MMZ_VB_DMABUF_MEMSET
+	struct sg_table *table;
+	struct scatterlist *sg;
+	int i;
+	#endif
+	int fd;
+	int ret = -ENOMEM;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&buffer->attachments);
+	mutex_init(&buffer->lock);
+
+	ret = vb_get_block(getBlkCmd, &pBlk);
+	/* try to get a required block from pool */
+	if (ret) {
+		vb_err("failed to get block from pool!!!\n");
+		goto free_buffer;
+	}
+
+	size = pBlk->pool->poolCfg.blkSize;
+	buffer->len = size;
+	buffer->table = &pBlk->sg_table;
+	#ifdef MMZ_VB_DMABUF_MEMSET
+	/*TODO: Clear the pages, sg_virt dose not work because vb memory is reserved as no-map!!!!*/
+	#if 0
+	{
+		table = buffer->table;
+		for_each_sg(table->sgl, sg, table->orig_nents, i)
+			memset(sg_virt(sg), 0, sg->length);
+	}
+	#endif
+	#endif
+	buffer->pBlk = pBlk;
+
+	/* create the dmabuf */
+	exp_info.exp_name = MMZ_VB_DMABUF_NAME;
+	exp_info.ops = &mmz_vb_buf_ops;
+	exp_info.size = buffer->len;
+	exp_info.flags = O_RDWR | O_CLOEXEC;
+	exp_info.priv = buffer;
+	dmabuf = dma_buf_export(&exp_info);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto release_block;
+	}
+
+	fd = dma_buf_fd(dmabuf, MMZ_VB_VALID_FD_FLAGS);
+	if (fd < 0) {
+		dma_buf_put(dmabuf);
+		/* just return, as put will call release and that will free */
+		return fd;
+	}
+
+	getBlkCmd->getBlkResp.fd = fd;
+	getBlkCmd->getBlkResp.actualBlkSize = size;
+	getBlkCmd->getBlkResp.nr = pBlk->nr;
+	return 0;
+
+release_block:
+	vb_release_block(pBlk);
+free_buffer:
+	kfree(buffer);
+
+	return ret;
+}
+
+static int vb_ioctl_get_blk(void __user *user_getBlkCmd)
+{
+	int ret = 0;
+	struct esVB_GET_BLOCK_CMD_S *getBlkCmd;
+
+	getBlkCmd = kzalloc(sizeof(*getBlkCmd), GFP_KERNEL);
+	if (!getBlkCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(getBlkCmd, user_getBlkCmd, sizeof(*getBlkCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_blk_dmabuf_alloc(getBlkCmd);
+	if (ret) {
+		goto out_free;
+	}
+
+	if (copy_to_user(user_getBlkCmd, getBlkCmd, sizeof(*getBlkCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(getBlkCmd);
+	return ret;
+}
+
+static int vb_ioctl_pool_size(void __user *user_getPoolSizeCmd)
+{
+	int ret = 0;
+	struct esVB_GET_POOLSIZE_CMD_S *getPoolSizeCmd;
+
+	getPoolSizeCmd = kzalloc(sizeof(*getPoolSizeCmd), GFP_KERNEL);
+	if (!getPoolSizeCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(getPoolSizeCmd, user_getPoolSizeCmd, sizeof(*getPoolSizeCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_pool_size(getPoolSizeCmd->poolId, &getPoolSizeCmd->poolSize);
+	if (ret) {
+		goto out_free;
+	}
+
+	if (copy_to_user(user_getPoolSizeCmd, getPoolSizeCmd, sizeof(*getPoolSizeCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(getPoolSizeCmd);
+	return ret;
+}
+#if 0
+static int vb_ioctl_flush_pool(void __user *user_flushPoolCmd)
+{
+	int ret = 0;
+	struct esVB_FLUSH_POOL_CMD_S *flushPoolCmd;
+
+	flushPoolCmd = kzalloc(sizeof(*flushPoolCmd), GFP_KERNEL);
+	if (!flushPoolCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(flushPoolCmd, user_flushPoolCmd, sizeof(*flushPoolCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_flush_pool(flushPoolCmd);
+	if (ret) {
+		goto out_free;
+	}
+
+	if (copy_to_user(user_flushPoolCmd, flushPoolCmd, sizeof(*flushPoolCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(flushPoolCmd);
+	return ret;
+}
+#endif
+
+static int vb_ioctl_blk_to_pool(void __user *user_blkToPoolCmd)
+{
+	int ret = 0;
+	struct esVB_BLOCK_TO_POOL_CMD_S *blkToPoolCmd;
+
+	blkToPoolCmd = kzalloc(sizeof(*blkToPoolCmd), GFP_KERNEL);
+	if (!blkToPoolCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(blkToPoolCmd, user_blkToPoolCmd, sizeof(*blkToPoolCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_blk_to_pool(blkToPoolCmd);
+	if (ret)
+		goto out_free;
+
+	if (copy_to_user(user_blkToPoolCmd, blkToPoolCmd, sizeof(*blkToPoolCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(blkToPoolCmd);
+	return ret;
+}
+
+static int vb_ioctl_get_blk_offset(void __user *user_getBlkOffsetCmd)
+{
+	int ret = 0;
+	struct esVB_GET_BLOCKOFFSET_CMD_S *getBlkOffsetCmd;
+
+	getBlkOffsetCmd = kzalloc(sizeof(*getBlkOffsetCmd), GFP_KERNEL);
+	if (!getBlkOffsetCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(getBlkOffsetCmd, user_getBlkOffsetCmd, sizeof(*getBlkOffsetCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_get_blk_offset(getBlkOffsetCmd);
+	if (ret) {
+		goto out_free;
+	}
+
+	if (copy_to_user(user_getBlkOffsetCmd, getBlkOffsetCmd, sizeof(*getBlkOffsetCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(getBlkOffsetCmd);
+	return ret;
+}
+
+static int vb_ioctl_split_dmabuf(void __user *user_splitDmabufCmd)
+{
+	int ret = 0;
+	struct esVB_SPLIT_DMABUF_CMD_S *splitDmabufCmd;
+
+	splitDmabufCmd = kzalloc(sizeof(*splitDmabufCmd), GFP_KERNEL);
+	if (!splitDmabufCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(splitDmabufCmd, user_splitDmabufCmd, sizeof(*splitDmabufCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_split_dmabuf(splitDmabufCmd);
+	if (ret) {
+		goto out_free;
+	}
+
+	if (copy_to_user(user_splitDmabufCmd, splitDmabufCmd, sizeof(*splitDmabufCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(splitDmabufCmd);
+	return ret;
+}
+
+static int vb_ioctl_get_dmabuf_refcnt(void __user *user_getDmabufRefCntCmd)
+{
+	int ret = 0;
+	struct esVB_DMABUF_REFCOUNT_CMD_S *getDmabufRefCntCmd;
+
+	getDmabufRefCntCmd = kzalloc(sizeof(*getDmabufRefCntCmd), GFP_KERNEL);
+	if (!getDmabufRefCntCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(getDmabufRefCntCmd, user_getDmabufRefCntCmd, sizeof(*getDmabufRefCntCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_get_dmabuf_refcnt(getDmabufRefCntCmd);
+	if (ret)
+		goto out_free;
+
+	if (copy_to_user(user_getDmabufRefCntCmd, getDmabufRefCntCmd, sizeof(*getDmabufRefCntCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(getDmabufRefCntCmd);
+	return ret;
+}
+
+static int vb_ioctl_retrieve_mem_node(void __user *user_retrieveMemNodeCmd)
+{
+	int ret = 0;
+	eic770x_memory_type_t mem_type;
+	struct esVB_RETRIEVE_MEM_NODE_CMD_S *retrieveMemNodeCmd;
+
+	retrieveMemNodeCmd = kzalloc(sizeof(*retrieveMemNodeCmd), GFP_KERNEL);
+	if (!retrieveMemNodeCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(retrieveMemNodeCmd, user_retrieveMemNodeCmd, sizeof(*retrieveMemNodeCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_retrieve_mem_node(retrieveMemNodeCmd, &mem_type);
+	if (ret)
+		goto out_free;
+
+	if (copy_to_user(user_retrieveMemNodeCmd, retrieveMemNodeCmd, sizeof(*retrieveMemNodeCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(retrieveMemNodeCmd);
+	return ret;
+}
+
+static int vb_ioctl_get_dmabuf_size(void __user *user_getDmabufSizeCmd)
+{
+	int ret = 0;
+	struct esVB_DMABUF_SIZE_CMD_S *getDmabufSizeCmd;
+
+	getDmabufSizeCmd = kzalloc(sizeof(*getDmabufSizeCmd), GFP_KERNEL);
+	if (!getDmabufSizeCmd) {
+		return -ENOMEM;
+	}
+	if (copy_from_user(getDmabufSizeCmd, user_getDmabufSizeCmd, sizeof(*getDmabufSizeCmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+
+	ret = vb_get_dmabuf_size(getDmabufSizeCmd);
+	if (ret)
+		goto out_free;
+
+	if (copy_to_user(user_getDmabufSizeCmd, getDmabufSizeCmd, sizeof(*getDmabufSizeCmd)))
+		ret = -EFAULT;
+
+out_free:
+	kfree(getDmabufSizeCmd);
+	return ret;
+}
+
+static int mmz_vb_assign_pool_id(struct esVB_K_POOL_INFO_S *pool)
+{
+	int ret = 0;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+
+	down_write(&partitions->idr_lock);
+	ret = idr_alloc(&partitions->pool_idr, pool, 0, VB_K_POOL_MAX_ID,
+			GFP_KERNEL);
+	if (ret >= 0) {
+		pool->poolId = ret;
+	}
+	up_write(&partitions->idr_lock);
+
+	return ret < 0 ? ret : 0;
+}
+
+static int mmz_vb_remove_pool_id(struct esVB_K_POOL_INFO_S *pool, bool is_lock)
+{
+	int ret = 0;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+
+	if (is_lock) {
+		down_write(&partitions->idr_lock);
+	}
+
+	idr_remove(&partitions->pool_idr, pool->poolId);
+
+	if (is_lock) {
+		up_write(&partitions->idr_lock);
+	}
+	return ret < 0 ? ret : 0;
+}
+
+static int mmz_pool_insert_list(struct esVB_K_POOL_INFO_S *pool, enum esVB_UID_E uid)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+
+	if (uid <= VB_UID_PRIVATE || uid >= VB_UID_MAX) {
+		dev_err(mmz_vb_dev, "%s %d, invalid uid %d\n",__func__,__LINE__, uid);
+		return -EINVAL;
+	}
+	down_write(&vb_priv->pool_lock[uid]);
+	hash_add(vb_priv->ht[uid], &pool->node, pool->poolCfg.blkSize);
+	up_write(&vb_priv->pool_lock[uid]);
+	return 0;
+}
+
+static struct mem_block *vb_get_memblock(const char *memBlkName)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	struct mem_block *memblock = NULL, *rsvmem_block = NULL;
+	int i;
+
+	for (i = 0; i < partitions->partCnt; i++) {
+		rsvmem_block = partitions->mem_blocks[i];
+		if (!strcmp(memBlkName, rsvmem_block->name)){
+			memblock = rsvmem_block;
+			break;
+		}
+	}
+
+	return memblock;
+}
+
+static int mmz_vb_do_create_pool(struct esVB_POOL_CONFIG_S *pool_cfg,
+	struct esVB_K_POOL_INFO_S **pool_out)
+{
+	int i;
+	int ret = 0;
+	struct esVB_K_POOL_INFO_S *pool;
+	struct esVB_K_BLOCK_INFO_S *blocks;
+	struct mem_block *memblock = NULL;
+	const char *memBlkName = pool_cfg->mmzName;
+	size_t size;
+
+	// 0.find the memblock
+	memblock = vb_get_memblock(memBlkName);
+	if (NULL == memblock) {
+		vb_err("%s NOT found!\n", memBlkName);
+		return -EINVAL;
+	}
+
+	// 1.init pool
+	pool = devm_kzalloc(mmz_vb_dev, sizeof(struct esVB_K_POOL_INFO_S), GFP_KERNEL);
+	if (!pool) {
+		dev_err(mmz_vb_dev, "%s %d, faild to alloc pool cb\n",__func__,__LINE__);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	pool->blocks = vmalloc(sizeof(struct esVB_K_BLOCK_INFO_S) * pool_cfg->blkCnt);
+	if (!pool->blocks) {
+		dev_err(mmz_vb_dev, "%s %d, faild to alloc blocks cb\n",__func__,__LINE__);
+		ret = -ENOMEM;
+		goto out_free_pool;
+	}
+
+	spin_lock_init(&pool->lock);
+	memcpy(&pool->poolCfg, pool_cfg, sizeof(struct esVB_POOL_CONFIG_S));
+
+	pool->bitmap = bitmap_zalloc(pool_cfg->blkCnt, GFP_KERNEL);
+	if (!pool->bitmap) {
+		dev_err(mmz_vb_dev, "%s %d, faild to alloc bitmap\n",__func__,__LINE__);
+		ret = -ENOMEM;
+		goto out_free_block_arrays;
+	}
+
+	// 2. make blkSize align
+	size = PAGE_ALIGN(pool_cfg->blkSize);
+	/* If len >= 1MB, align len with 2M to improve performance of SMMU */
+	if (size/(PAGE_SIZE << 8)) {
+		size = ALIGN(size, (PAGE_SIZE << 9));
+	}
+	pool_cfg->blkSize = size;
+	pool->poolCfg.blkSize = pool_cfg->blkSize;
+	dev_dbg(mmz_vb_dev, "blkSize(0x%llx) from pool creation is "
+		"aligned to 0x%lx to improve performance.\n",
+		pool_cfg->blkSize, size);
+
+	// 3. alloc pages for blocks
+	for (i = 0; i < pool_cfg->blkCnt; i++) {
+		blocks = &pool->blocks[i];
+		blocks->nr = i;
+		blocks->pool = pool;
+		ret = vb_blk_pages_allocate(memblock, blocks, pool_cfg->blkSize);
+		if (ret) {
+			while (--i >= 0) {
+				vb_blk_pages_release(memblock, &pool->blocks[i]);
+			}
+			dev_err(mmz_vb_dev, "%s %d, faild to alloc block page!\n", __func__,__LINE__);
+			ret = -ENOMEM;
+			goto out_free_bitmap;
+		}
+	}
+	// 4. everthing is ok, add pool to idr
+	ret = mmz_vb_assign_pool_id(pool);
+	if (0 != ret) {
+		dev_err(mmz_vb_dev, "%s %d, faild to assign pool id\n",__func__,__LINE__);
+		ret = -EINVAL;
+		goto out_free_block_pages;
+	}
+	*pool_out = pool;
+	return ret;
+
+out_free_block_pages:
+	for (i = 0; i < pool_cfg->blkCnt; i++) {
+		vb_blk_pages_release(memblock, &pool->blocks[i]);
+	}
+out_free_bitmap:
+	bitmap_free(pool->bitmap);;
+out_free_block_arrays:
+	vfree(pool->blocks);
+out_free_pool:
+	devm_kfree(mmz_vb_dev, pool);
+out:
+	return ret;
+}
+
+static int vb_pool_config_check(struct esVB_POOL_CONFIG_S *pool_cfg)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct mem_block *memblock = NULL;
+	const char *memBlkName = pool_cfg->mmzName;
+	unsigned long numFreePages = 0;
+	u64 req_size;
+
+	if (NULL == vb_priv) {
+		return 0;
+	}
+
+	// find the memblock
+	memblock = vb_get_memblock(memBlkName);
+	if (NULL == memblock) {
+		vb_err("%s NOT found!\n", memBlkName);
+		return -EINVAL;
+	}
+
+	req_size = pool_cfg->blkCnt * PAGE_ALIGN(pool_cfg->blkSize);
+	numFreePages = es_num_free_pages(memblock);
+	if (numFreePages < (req_size >> PAGE_SHIFT)) {
+		dev_err(mmz_vb_dev, "%s %d, (%s)out of memory, request pool size %llu "
+			"free %ld!\n",
+			__func__,__LINE__,
+			memBlkName, req_size, (numFreePages << PAGE_SHIFT));
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int vb_ioctl_create_pool(void __user *user_cmd)
+{
+	int ret = 0;
+	struct esVB_CREATE_POOL_CMD_S cmd;
+	struct esVB_CREATE_POOL_REQ_S *req;
+	struct esVB_CREATE_POOL_RESP_S *rsp;
+	struct esVB_POOL_CONFIG_S *pool_cfg;
+	struct esVB_K_POOL_INFO_S *pool = NULL;
+
+	if (copy_from_user(&cmd, user_cmd, sizeof(cmd))) {
+		ret = -EFAULT;
+		goto out_free;
+	}
+	req = &cmd.PoolReq;
+	pool_cfg = &req->req;
+	ret = vb_pool_config_check(pool_cfg);
+	if (ret) {
+		goto out_free;
+	}
+	ret = mmz_vb_do_create_pool(pool_cfg, &pool);
+	if (ret) {
+		goto out_free;
+	}
+	pool->enVbUid = VB_UID_PRIVATE;
+	rsp = &cmd.PoolResp;
+	rsp->PoolId = pool->poolId;
+	dev_dbg(mmz_vb_dev, "[%s %d]:create pool, PoolId %d!\n",__func__,__LINE__, rsp->PoolId);
+	if (copy_to_user(user_cmd, &cmd, sizeof(cmd)))
+		ret = -EFAULT;
+	else
+		ret = 0;
+
+out_free:
+	return ret;
+}
+
+/**
+ * mmz_vb_do_destory_pool - do the pool destory operation
+ * @pool:	The pool
+ * @is_lock:	when set true, will lock the idr when remove the idr id.
+ * @is_force:	when set true, will still destory the bool even the bitmap is not empty.
+ */
+static int mmz_vb_do_destory_pool(struct esVB_K_POOL_INFO_S *pool, bool is_lock, bool is_force)
+{
+	struct esVB_POOL_CONFIG_S *poolCfg = &pool->poolCfg;
+	const char *memBlkName = poolCfg->mmzName;
+	struct mem_block *memblock = NULL;
+	struct esVB_K_BLOCK_INFO_S *blocks = NULL;
+	int ret = 0;
+	int i;
+
+	// find the memblock
+	memblock = vb_get_memblock(memBlkName);
+	if (NULL == memblock) {
+		vb_err("%s NOT found!\n", memBlkName);
+		return -EINVAL;
+	}
+
+	if (!bitmap_empty(pool->bitmap, pool->poolCfg.blkCnt)) {
+		if (true == is_force) {
+			dev_info(mmz_vb_dev, "%s %d, non-empty pool, still destory it!\n",__func__,__LINE__);
+		} else {
+			dev_dbg(mmz_vb_dev, "%s %d, non-empty pool, can not destory!\n",__func__,__LINE__);
+			ret = -ENOTEMPTY;
+			goto out;
+		}
+	}
+
+
+	blocks = pool->blocks;
+	for (i = 0; i < poolCfg->blkCnt; i++) {
+		vb_blk_pages_release(memblock, &blocks[i]);
+	}
+	mmz_vb_remove_pool_id(pool, is_lock);
+	if (pool->enVbUid >= VB_UID_COMMON && pool->enVbUid < VB_UID_MAX) {
+		hash_del(&pool->node);
+	}
+	bitmap_free(pool->bitmap);
+	vfree(pool->blocks);
+out:
+	if (0 == ret) {
+		devm_kfree(mmz_vb_dev, pool);
+	}
+	return ret;
+}
+
+static int vb_ioctl_destory_pool(void __user *user_cmd)
+{
+	int ret = 0;
+	struct esVB_K_POOL_INFO_S *pool = NULL;
+	struct esVB_DESTORY_POOL_CMD_S cmd;
+	struct esVB_DESTORY_POOL_REQ_S *req = NULL;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+
+	if (copy_from_user(&cmd, user_cmd, sizeof(cmd))) {
+		return -EFAULT;
+	}
+	req = &cmd.req;
+	dev_dbg(mmz_vb_dev, "[%s %d]:destory pool, PoolId %d!\n",__func__,__LINE__, req->PoolId);
+	down_write(&partitions->idr_lock);
+	ret = vb_find_pool_by_id_unlock(req->PoolId, &pool);
+	if (ret) {
+		up_write(&partitions->idr_lock);
+		dev_err(mmz_vb_dev, "%s %d, faild to find pool, PoolId %d\n",
+			__func__,__LINE__, req->PoolId);
+		return ret;
+	}
+	ret = mmz_vb_do_destory_pool(pool, false, false);
+	if (-ENOTEMPTY == ret) {
+		set_bit(MMZ_VB_POOL_FLAG_DESTORY, &pool->flag);
+		up_write(&partitions->idr_lock);
+		dev_dbg(mmz_vb_dev, "%s %d, pool %d not empty, waiting to destory\n",
+			__func__,__LINE__, req->PoolId);
+		return 0;
+	} else if (ret) {
+		up_write(&partitions->idr_lock);
+		dev_err(mmz_vb_dev, "%s %d, faild to destory pool, PoolId %d\n",
+			__func__,__LINE__, req->PoolId);
+		return ret;
+	}
+	up_write(&partitions->idr_lock);
+	return 0;
+}
+
+/*check whether the VbConfig is legal*/
+static int vb_config_check(struct esVB_CONFIG_S *pVbConfig)
+{
+	int i;
+	struct esVB_POOL_CONFIG_S *pool_cfg = NULL;
+	int ret;
+
+	if (pVbConfig->poolCnt > ES_VB_MAX_MOD_POOL) {
+		dev_err(mmz_vb_dev, "%s %d, poolCnt %d exceed the limit %d!\n",
+			__func__,__LINE__, pVbConfig->poolCnt, ES_VB_MAX_MOD_POOL);
+		return -EINVAL;
+	}
+	for (i = 0; i < pVbConfig->poolCnt; i++) {
+		pool_cfg = &pVbConfig->poolCfgs[i];
+		ret = vb_pool_config_check(pool_cfg);
+		if (0 != ret) {
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static int vb_ioctl_set_config(void __user *user_cmd)
+{
+	struct esVB_SET_CFG_CMD_S *cmd;
+	struct esVB_SET_CFG_REQ_S *req;
+	enum esVB_UID_E enVbUid;
+	struct esVB_CONFIG_S *pVbConfig = NULL;
+	struct esVB_CONFIG_S *vb_cfg = NULL;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	int ret = 0;
+
+	cmd = devm_kzalloc(mmz_vb_dev, sizeof(*cmd), GFP_KERNEL);
+	if (!cmd) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d, failed to alloc memory!\n",
+			__func__,__LINE__, enVbUid);
+		ret = -ENOMEM;
+		goto out;
+	}
+	if (copy_from_user(cmd, user_cmd, sizeof(*cmd))) {
+		ret = -EFAULT;
+		goto out_free_cmd;
+	}
+	req = &cmd->CfgReq;
+	enVbUid = req->uid;
+	pVbConfig = &req->cfg;
+	if (enVbUid <= VB_UID_PRIVATE || enVbUid >= VB_UID_MAX) {
+		dev_err(mmz_vb_dev, "%s %d, invaild uid %d!\n", __func__,__LINE__, enVbUid);
+		ret = -EFAULT;
+		goto out_free_cmd;
+	}
+	ret = vb_config_check(pVbConfig);
+	if (ret) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d, vbConfig check fail!\n",
+			__func__,__LINE__, enVbUid);
+		goto out_free_cmd;
+	}
+	mutex_lock(&vb_priv->cfg_lock[enVbUid]);
+	if (NULL != vb_priv->pVbConfig[enVbUid]) {
+		if (test_bit(MMZ_VB_CFG_FLAG_INIT, &vb_priv->cfg_flag[enVbUid])) {
+			dev_err(mmz_vb_dev, "%s %d, uid %d cfg already exist and init!\n",
+				__func__,__LINE__, enVbUid);
+			ret = -EFAULT;
+			goto out_unlock;
+		} else {
+			/*release the old config*/
+			devm_kfree(mmz_vb_dev, vb_priv->pVbConfig[enVbUid]);
+			vb_priv->pVbConfig[enVbUid] = NULL;
+		}
+	}
+	vb_cfg = devm_kzalloc(mmz_vb_dev, sizeof(struct esVB_CONFIG_S), GFP_KERNEL);
+	if (!vb_cfg) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d, failed to alloc memory!\n",
+			__func__,__LINE__, enVbUid);
+		ret =  -ENOMEM;
+		goto out_unlock;
+	}
+	memcpy(vb_cfg, pVbConfig, sizeof(struct esVB_CONFIG_S));
+	vb_priv->pVbConfig[enVbUid] = vb_cfg;
+out_unlock:
+	mutex_unlock(&vb_priv->cfg_lock[enVbUid]);
+out_free_cmd:
+	devm_kfree(mmz_vb_dev, cmd);
+out:
+	return ret;
+}
+
+static int vb_ioctl_get_config(void __user *user_cmd)
+{
+	struct esVB_GET_CFG_CMD_S *cmd;
+	struct esVB_GET_CFG_REQ_S *req;
+	struct esVB_GET_CFG_RSP_S *rsp;
+	enum esVB_UID_E enVbUid;
+	struct esVB_CONFIG_S *vb_cfg = NULL;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	int ret = 0;
+
+	cmd = devm_kzalloc(mmz_vb_dev, sizeof(*cmd), GFP_KERNEL);
+	if (!cmd) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d, failed to alloc memory!\n",
+			__func__,__LINE__, enVbUid);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	if (copy_from_user(cmd, user_cmd, sizeof(*cmd))) {
+		ret = -EFAULT;
+		goto out_free_cmd;
+	}
+	req = &cmd->req;
+	enVbUid = req->uid;
+	if (enVbUid <= VB_UID_PRIVATE || enVbUid >= VB_UID_MAX) {
+		dev_err(mmz_vb_dev, "%s %d, invaild uid %d!\n",__func__,__LINE__, enVbUid);
+		ret = -EFAULT;
+		goto out_free_cmd;
+	}
+	mutex_lock(&vb_priv->cfg_lock[enVbUid]);
+	vb_cfg = vb_priv->pVbConfig[enVbUid];
+	if (NULL == vb_cfg) {
+		dev_dbg(mmz_vb_dev, "%s %d, uid %d cfg not exist!\n", __func__,__LINE__, enVbUid);
+		ret = -EFAULT;
+		goto out_unlock;
+	}
+	rsp = &cmd->rsp;
+	memcpy(&rsp->cfg, vb_cfg, sizeof(struct esVB_CONFIG_S));
+	if (copy_to_user(user_cmd, cmd, sizeof(*cmd))) {
+		ret = -EFAULT;
+		goto out_unlock;
+	}
+out_unlock:
+	mutex_unlock(&vb_priv->cfg_lock[enVbUid]);
+out_free_cmd:
+	devm_kfree(mmz_vb_dev, cmd);
+out:
+	return ret;
+}
+
+static int vb_ioctl_init_config(void __user *user_cmd)
+{
+	VB_INIT_CFG_CMD_S cmd;
+	VB_INIT_CFG_REQ_S *req = NULL;
+	enum esVB_UID_E enVbUid;
+	struct esVB_CONFIG_S *vb_cfg = NULL;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	int i;
+	int ret = 0;
+	struct esVB_POOL_CONFIG_S *pool_cfg;
+	struct esVB_K_POOL_INFO_S *pool[ES_VB_MAX_MOD_POOL] = {NULL};
+
+	if (copy_from_user(&cmd, user_cmd, sizeof(cmd))) {
+		ret = -EFAULT;
+		goto out;
+	}
+	req = &cmd.req;
+	enVbUid = req->uid;
+
+	if (enVbUid < VB_UID_COMMON || enVbUid >= VB_UID_MAX) {
+		dev_err(mmz_vb_dev, "%s %d, invaild uid %d!\n",__func__,__LINE__, enVbUid);
+		ret = -EFAULT;
+		goto out;
+	}
+	mutex_lock(&vb_priv->cfg_lock[enVbUid]);
+	vb_cfg = vb_priv->pVbConfig[enVbUid];
+	if (NULL == vb_cfg) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d cfg not exist!\n", __func__,__LINE__, enVbUid);
+		ret = -EFAULT;
+		goto out_unlock;
+	}
+	if (test_bit(MMZ_VB_CFG_FLAG_INIT, &vb_priv->cfg_flag[enVbUid])) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d cfg already initialized!\n", __func__,__LINE__, enVbUid);
+		ret = -EPERM;
+		goto out_unlock;
+	}
+
+	for (i = 0; i < vb_cfg->poolCnt; i++) {
+		pool_cfg = &vb_cfg->poolCfgs[i];
+		ret = mmz_vb_do_create_pool(pool_cfg, &pool[i]);
+		if (0 != ret) {
+			while(--i >= 0) {
+				ret = mmz_vb_do_destory_pool(pool[i], true, false);
+				if (ret) {
+					dev_err(mmz_vb_dev, "%s %d, faild to destory pool!\n",
+						__func__,__LINE__);
+				}
+			}
+			dev_err(mmz_vb_dev, "%s %d, faild to create pool!\n",__func__, __LINE__);
+			goto out_unlock;
+		}
+		mmz_pool_insert_list(pool[i], enVbUid);
+		pool[i]->enVbUid = enVbUid;
+	}
+	set_bit(MMZ_VB_CFG_FLAG_INIT, &vb_priv->cfg_flag[enVbUid]);
+out_unlock:
+	mutex_unlock(&vb_priv->cfg_lock[enVbUid]);
+out:
+	return ret;
+}
+
+static int vb_ioctl_uninit_config(void __user *user_cmd)
+{
+	struct esVB_UNINIT_CFG_CMD_S cmd;
+	enum esVB_UID_E enVbUid;
+	int ret;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_POOL_INFO_S *pool = NULL;
+	unsigned long bkt = 0;
+	struct hlist_node *tmp_node = NULL;
+
+	if (copy_from_user(&cmd, user_cmd, sizeof(cmd))) {
+		return -EFAULT;
+	}
+	enVbUid = cmd.req.uid;
+
+	if (enVbUid <= VB_UID_PRIVATE || enVbUid >= VB_UID_MAX) {
+		dev_err(mmz_vb_dev, "%s %d, invaild uid %d!\n",__func__,__LINE__, enVbUid);
+		return -EFAULT;
+	}
+	mutex_lock(&vb_priv->cfg_lock[enVbUid]);
+	if (!test_bit(MMZ_VB_CFG_FLAG_INIT, &vb_priv->cfg_flag[enVbUid])) {
+		dev_err(mmz_vb_dev, "%s %d, uid %d cfg not initialized!\n", __func__,__LINE__, enVbUid);
+		mutex_unlock(&vb_priv->cfg_lock[enVbUid]);
+		return -EINVAL;
+	}
+	mutex_unlock(&vb_priv->cfg_lock[enVbUid]);
+
+	down_write(&vb_priv->pool_lock[enVbUid]);
+	hash_for_each_safe(vb_priv->ht[enVbUid], bkt, tmp_node, pool, node) {
+		ret = mmz_vb_do_destory_pool(pool, true, false);
+		if (ret) {
+			dev_err(mmz_vb_dev, "%s %d, faild to destory pool, PoolId %d, enVbUid %d\n",
+				__func__,__LINE__, pool->poolId, enVbUid);
+			up_write(&vb_priv->pool_lock[enVbUid]);
+			return ret;
+		}
+	}
+	up_write(&vb_priv->pool_lock[enVbUid]);
+
+	mutex_lock(&vb_priv->cfg_lock[enVbUid]);
+	devm_kfree(mmz_vb_dev, vb_priv->pVbConfig[enVbUid]);
+	vb_priv->pVbConfig[enVbUid] = NULL;
+	clear_bit(MMZ_VB_CFG_FLAG_INIT, &vb_priv->cfg_flag[enVbUid]);
+	mutex_unlock(&vb_priv->cfg_lock[enVbUid]);
+	return 0;
+}
+
+static int vb_ioctl_flush_all(void __user *user_cmd)
+{
+	EIC770X_LOGICAL_MEM_NODE_E nid;
+
+	if (copy_from_user(&nid, user_cmd, sizeof(EIC770X_LOGICAL_MEM_NODE_E))) {
+		return -EFAULT;
+	}
+
+	_do_arch_sync_cache_all(nid);
+
+	return 0;
+}
+
+static long mmz_vb_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	int ret = 0;
+	void __user *argp;
+
+	argp = (void __user *)arg;
+	switch (cmd) {
+	case MMZ_VB_IOCTL_GET_BLOCK:
+		return vb_ioctl_get_blk(argp);
+	case MMZ_VB_IOCTL_CREATE_POOL:
+		return vb_ioctl_create_pool(argp);
+	case MMZ_VB_IOCTL_DESTORY_POOL:
+		return vb_ioctl_destory_pool(argp);
+	case MMZ_VB_IOCTL_SET_CFG:
+		return vb_ioctl_set_config(argp);
+	case MMZ_VB_IOCTL_GET_CFG:
+		return vb_ioctl_get_config(argp);
+	case MMZ_VB_IOCTL_INIT_CFG:
+		return vb_ioctl_init_config(argp);
+	case MMZ_VB_IOCTL_UNINIT_CFG:
+		return vb_ioctl_uninit_config(argp);
+	case MMZ_VB_IOCTL_POOL_SIZE:
+		return vb_ioctl_pool_size(argp);
+#if 0
+	case MMZ_VB_IOCTL_FLUSH_POOL:
+		return vb_ioctl_flush_pool(argp);
+#endif
+	case MMZ_VB_IOCTL_BLOCK_TO_POOL:
+		return vb_ioctl_blk_to_pool(argp);
+	case MMZ_VB_IOCTL_GET_BLOCK_OFFSET:
+		return vb_ioctl_get_blk_offset(argp);
+	case MMZ_VB_IOCTL_SPLIT_DMABUF:
+		return vb_ioctl_split_dmabuf(argp);
+	case MMZ_VB_IOCTL_DMABUF_REFCOUNT:
+		return vb_ioctl_get_dmabuf_refcnt(argp);
+	case MMZ_VB_IOCTL_RETRIEVE_MEM_NODE:
+		return vb_ioctl_retrieve_mem_node(argp);
+	case MMZ_VB_IOCTL_DMABUF_SIZE:
+		return vb_ioctl_get_dmabuf_size(argp);
+	case MMZ_VB_IOCTL_FLUSH_ALL:
+		return vb_ioctl_flush_all(argp);
+	default:
+		pr_debug("Invalid IOCTL CMD!!!\n");
+		return -EINVAL;
+	}
+	pr_debug("%s:%d, success!\n", __func__, __LINE__);
+	return ret;
+}
+
+static int mmz_vb_open(struct inode *inode, struct file *file)
+{
+
+	pr_debug("%s:%d, success!\n", __func__, __LINE__);
+
+	return 0;
+}
+
+static int mmz_vb_release(struct inode *inode, struct file *file)
+{
+	pr_debug("%s:%d, success!\n", __func__, __LINE__);
+
+	return 0;
+}
+
+/* mem = mmap(0, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, addr_pha);
+*  vma->vm_pgoff indicats the pool ID
+*/
+static int mmz_vb_mmap_pool(struct file *file, struct vm_area_struct *vma)
+{
+	int ret = 0;
+	size_t size = vma->vm_end - vma->vm_start;
+	VB_POOL poolId = (VB_POOL)vma->vm_pgoff;
+	unsigned long addr = vma->vm_start;
+	struct esVB_K_POOL_INFO_S *pPool = NULL;
+	struct esVB_K_BLOCK_INFO_S *pBlk = NULL;
+	u64 poolSize, blkSize;
+	u32 i;
+
+	ret = vb_find_pool_by_id(poolId, &pPool);
+	if (ret)
+		return ret;
+
+	poolSize = do_vb_pool_size(pPool);
+	/* is the mmap size equal to poolSize? */
+	if (size != poolSize)
+		return -EINVAL;
+
+	/* pool is mmapped as uncached memory */
+	#ifndef QEMU_DEBUG
+	vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+	#endif
+
+	blkSize = pPool->poolCfg.blkSize;
+	pBlk = pPool->blocks;
+	for (i = 0; i < pPool->poolCfg.blkCnt; i++) {
+		struct sg_table *table = &pBlk->sg_table;
+
+		/* mmap for one block */
+		struct scatterlist *sg;
+		int j;
+
+		for_each_sg(table->sgl, sg, table->orig_nents, j) {
+			struct page *page = sg_page(sg);
+			ret = remap_pfn_range(vma, addr, page_to_pfn(page), sg->length,
+					vma->vm_page_prot);
+			if (ret)
+				return ret;
+			addr += sg->length;
+			if (addr >= vma->vm_end)
+				return 0;
+		}
+		pBlk++;
+	}
+
+	pr_debug("%s:%d, success!\n", __func__, __LINE__);
+
+	return ret;
+}
+
+static struct file_operations mmz_vb_fops = {
+	.owner        = THIS_MODULE,
+	.llseek        = no_llseek,
+	.unlocked_ioctl = mmz_vb_unlocked_ioctl,
+	.open        = mmz_vb_open,
+	.release    = mmz_vb_release,
+	.mmap	= mmz_vb_mmap_pool,
+};
+
+static struct miscdevice mmz_vb_miscdev = {
+	.minor	= MISC_DYNAMIC_MINOR,
+	.name	= DRIVER_NAME,
+	.fops	= &mmz_vb_fops,
+};
+
+static char es_mmz_name_prefix[] = "mmz_nid_";
+static int mmz_vb_init_partitions(void)
+{
+	int ret = 0;
+	struct mmz_vb_priv *mmz_vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions;
+
+	if (NULL == mmz_vb_priv)
+		return -EFAULT;
+
+	partitions = &mmz_vb_priv->partitions;;
+	init_rwsem(&partitions->idr_lock);
+	idr_init(&partitions->pool_idr);
+
+	partitions->partCnt = mmz_vb_init_memory_region();
+	if (partitions->partCnt == 0) {
+		vb_err("No VB memory block was found or correctly initialized!\n");
+		ret = -EFAULT;
+	}
+
+	return ret;
+}
+
+static int mmz_vb_idr_iterate_show(int id, void *p, void *data)
+{
+	struct esVB_K_POOL_INFO_S *pool = (struct esVB_K_POOL_INFO_S *)p;
+	struct esVB_POOL_CONFIG_S *pool_cfg;
+	es_proc_entry_t *s = (es_proc_entry_t *)data;
+
+	spin_lock(&pool->lock);
+	pool_cfg = &pool->poolCfg;
+	es_seq_printf(s, "\t Uid %-2d, PoolId %-4d, blkSize 0x%08llx, blkCnt %-8d, "
+		"RemapMode %d, mmzName %s, allocated blkCnt %-8d\n\r", pool->enVbUid,
+		pool->poolId, pool_cfg->blkSize, pool_cfg->blkCnt,
+		pool_cfg->enRemapMode, pool_cfg->mmzName,
+		pool_cfg->blkCnt - vb_pool_get_free_block_cnt_unlock(pool));
+	spin_unlock(&pool->lock);
+	return 0;
+}
+
+static int mmz_vb_proc_show(es_proc_entry_t *s)
+{
+	int i;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	unsigned long numFreePages = 0;
+	struct mem_block *memblock = NULL, *rsvmem_block = NULL;
+	int ret;
+
+	es_seq_printf(s, "\nModule: [VB], Build Time[xx]\n");
+	/*
+	   use es_seq_printf to show more debug info
+	*/
+	es_seq_printf(s, "-----MMZ REGION CONFIG-----\n\r");
+	for (i = 0; i < partitions->partCnt; i++) {
+		rsvmem_block = partitions->mem_blocks[i];
+		memblock = vb_get_memblock(rsvmem_block->name);
+		if (NULL == memblock) {
+			vb_err("%s NOT found!\n", rsvmem_block->name);
+			return -EINVAL;
+		}
+		numFreePages = es_num_free_pages(memblock);
+		es_seq_printf(s, "\tmemblock: %s, total size(0x%lx), free mem size(0x%lx)\n\r",
+				rsvmem_block->name,
+				memblock->page_num << PAGE_SHIFT,
+				numFreePages << PAGE_SHIFT);
+	}
+	es_seq_printf(s, "-----POOL CONFIG-----\n\r");
+	ret = idr_for_each(&partitions->pool_idr, mmz_vb_idr_iterate_show, s);
+	if (ret) {
+		dev_err(mmz_vb_dev, "%s %d, failed to iterate vb pool ret %d\n",
+			__func__,__LINE__, ret);
+		return ret;
+	}
+	return 0;
+}
+
+int mmz_vb_proc_store(struct es_proc_dir_entry *entry, const char *buf,
+		int count, long long *ppos)
+{
+	int ret;
+
+	ret = mmz_vb_pool_exit();
+	if (0 != ret) {
+		dev_err(mmz_vb_dev, "%s %d, failed to release vb pool "
+			"when exit, ret %d\n", __func__,__LINE__, ret);
+	}
+	return count;
+}
+
+void mmz_vb_vb_dbg_init(void)
+{
+	es_proc_entry_t *proc = NULL;
+
+	proc = es_create_proc_entry(PROC_ENTRY_VB, 0666, NULL);
+
+	if (proc == NULL) {
+		vb_err("Kernel: Register vb proc failed!\n");
+		return;
+	}
+	proc->read = mmz_vb_proc_show;
+	/*NULL means use the default routine*/
+	proc->write = mmz_vb_proc_store;
+	proc->open = NULL;
+}
+
+static int __init mmz_vb_init(void)
+{
+	int i;
+	int ret = 0;
+	struct device *dev;
+
+	g_mmz_vb_priv = kzalloc(sizeof(struct mmz_vb_priv), GFP_KERNEL);
+	if (!g_mmz_vb_priv) {
+		vb_err("Failed to alloc priv data for mmz_vb driver!!!\n");
+		return -ENOMEM;
+	}
+	ret = misc_register(&mmz_vb_miscdev);
+	if(ret) {
+		vb_err ("cannot register miscdev (err=%d)\n", ret);
+		goto free_vb_priv;
+	}
+	mmz_vb_dev = mmz_vb_miscdev.this_device;
+	g_mmz_vb_priv->dev = mmz_vb_dev;
+	for (i = 0; i < VB_UID_MAX; i++) {
+		hash_init(g_mmz_vb_priv->ht[i]);
+		mutex_init(&g_mmz_vb_priv->cfg_lock[i]);
+		init_rwsem(&g_mmz_vb_priv->pool_lock[i]);
+	}
+
+	ret = mmz_vb_init_partitions();
+	if (ret) {
+		goto deregister_vb;
+	}
+	atomic_set(&g_mmz_vb_priv->allocBlkcnt, 0);
+
+	mmz_vb_vb_dbg_init();
+
+	dev = mmz_vb_dev;
+	if (!dev->dma_mask) {
+		dev->dma_mask = &dev->coherent_dma_mask;
+	}
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret)
+		vb_err("Unable to set coherent mask\n");
+	return 0;
+
+deregister_vb:
+	misc_deregister(&mmz_vb_miscdev);
+free_vb_priv:
+	kfree(g_mmz_vb_priv);
+	return ret;
+}
+
+static int mmz_vb_pool_exit(void)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	struct esVB_K_POOL_INFO_S *pool = NULL;
+	int ret = 0;
+	u32 id = 0;
+	int i;
+
+	down_write(&partitions->idr_lock);
+	idr_for_each_entry(&partitions->pool_idr, pool, id) {
+		ret = mmz_vb_do_destory_pool(pool, false, true);
+		if (ret) {
+			dev_err(mmz_vb_dev, "%s %d, failed to destory vb pool, ret %d\n",
+				__func__,__LINE__, ret);
+			continue;
+		}
+	}
+
+	up_write(&partitions->idr_lock);
+
+	atomic_set(&vb_priv->allocBlkcnt, 0);
+	for (i = 0; i < VB_UID_MAX; i++) {
+		if (NULL != vb_priv->pVbConfig[i]) {
+			devm_kfree(mmz_vb_dev, vb_priv->pVbConfig[i]);
+			vb_priv->pVbConfig[i] = NULL;
+		}
+	}
+	memset(vb_priv->cfg_flag, 0, sizeof(unsigned long) * VB_UID_MAX);
+	return 0;
+}
+
+static void __exit mmz_vb_exit(void)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	int ret = 0;
+
+	ret = mmz_vb_pool_exit();
+	if (0 != ret) {
+		dev_err(mmz_vb_dev, "%s %d, failed to release vb pool "
+			"when exit, ret %d\n", __func__,__LINE__, ret);
+	}
+	es_remove_proc_entry(PROC_ENTRY_VB, NULL);
+	misc_deregister(&mmz_vb_miscdev);
+	kfree(vb_priv);
+}
+
+module_init(mmz_vb_init);
+module_exit(mmz_vb_exit);
+
+MODULE_DESCRIPTION("MMZ VB Driver");
+MODULE_AUTHOR("Lin MIn <linmin@eswincomputing.com>");
+MODULE_LICENSE("GPL v2");
+
+
+static int vb_find_pool_by_id_unlock(VB_POOL poolId, struct esVB_K_POOL_INFO_S **ppPool)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	struct esVB_K_POOL_INFO_S *pool = NULL;
+
+	pool = idr_find(&partitions->pool_idr, poolId);
+	if (!pool) {
+		dev_err(mmz_vb_dev, "%s %d, faild to find pool by id %d\n",
+			__func__,__LINE__, poolId);
+		return -EINVAL;
+	}
+	*ppPool = pool;
+	return 0;
+}
+
+static int vb_find_pool_by_id(VB_POOL poolId, struct esVB_K_POOL_INFO_S **ppPool)
+{
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	int ret;
+
+	down_read(&partitions->idr_lock);
+	ret = vb_find_pool_by_id_unlock(poolId, ppPool);
+	up_read(&partitions->idr_lock);
+	return ret;
+}
+
+static int vb_pool_size(VB_POOL poolId, u64 *pPoolSize)
+{
+	int ret = 0;
+	struct esVB_K_POOL_INFO_S *pPool;
+
+	ret = vb_find_pool_by_id(poolId, &pPool);
+	if (ret) {
+		vb_info("failed to find pool %d\n", poolId);
+		return ret;
+	}
+
+	*pPoolSize = do_vb_pool_size(pPool);
+
+	return ret;
+}
+#if 0
+static int vb_flush_pool(struct esVB_FLUSH_POOL_CMD_S *flushPoolCmd)
+{
+	int ret = 0;
+	struct esVB_K_POOL_INFO_S *pPool = NULL;
+	struct esVB_K_BLOCK_INFO_S *pBlk = NULL;
+	u64 blkSize, poolSize = 0;
+	u64 offset_inPool = 0, offset_inBlk = 0, size, left_size = 0;
+	u64 phy_addr;
+	u32 i;
+
+	ret = vb_find_pool_by_id(flushPoolCmd->poolId, &pPool);
+	if (ret) {
+		vb_info("%s,failed to find pool %d\n", __func__, flushPoolCmd->poolId);
+		return ret;
+	}
+
+	poolSize = do_vb_pool_size(pPool);
+	if ((flushPoolCmd->offset + flushPoolCmd->size - 1) >= poolSize)
+		return -EINVAL;
+
+	// find the block according to the offset
+	blkSize = pPool->poolCfg.blkSize;
+	pBlk = pPool->blocks;
+	left_size = flushPoolCmd->size;
+	for (i = 0; i < pPool->poolCfg.blkCnt; i++) {
+		if ((offset_inPool + blkSize -1) >= flushPoolCmd->offset)
+			break;
+		offset_inPool += blkSize;
+		pBlk++;
+	}
+	offset_inBlk = flushPoolCmd->offset - offset_inPool;
+	for (; i < pPool->poolCfg.blkCnt; i++) {
+		struct page *page = pBlk->cma_pages;
+		size = min(left_size, (blkSize - offset_inBlk));
+		phy_addr = page_to_phys(page) + offset_inBlk;
+		arch_sync_dma_for_device(phy_addr, size, DMA_TO_DEVICE);
+		left_size -= size;
+		if (left_size == 0)
+			break;
+		pBlk++;
+		offset_inBlk = 0;
+	}
+
+	return ret;
+}
+#endif
+static int vb_pool_get_free_block_cnt_unlock(struct esVB_K_POOL_INFO_S *pool)
+{
+	int count = 0;
+	int start = 0;
+	struct esVB_POOL_CONFIG_S *pool_cfg = &pool->poolCfg;
+	int nr;
+
+	while (true) {
+		nr = find_next_zero_bit(pool->bitmap, pool_cfg->blkCnt, start);
+		if (likely(nr < pool_cfg->blkCnt)) {
+			count++;
+			start = nr + 1;
+		} else {
+			break;
+		}
+	}
+	return count;
+}
+
+static int vb_pool_get_block(struct esVB_K_POOL_INFO_S *pool,
+	struct esVB_K_BLOCK_INFO_S **ppBlk)
+{
+	unsigned int nr = -1U;
+	struct esVB_POOL_CONFIG_S *pool_cfg;
+	int ret = -EINVAL;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+
+	spin_lock(&pool->lock);
+	pool_cfg = &pool->poolCfg;
+	nr = find_next_zero_bit(pool->bitmap, pool_cfg->blkCnt, 0);
+	if (likely(nr < pool_cfg->blkCnt)) {
+		ret = 0;
+		*ppBlk = &pool->blocks[nr];
+		bitmap_set(pool->bitmap, nr, 1);
+		if (atomic_inc_return(&vb_priv->allocBlkcnt) == 1) {
+			__module_get(THIS_MODULE);
+		}
+	} else {
+		dev_warn(mmz_vb_dev, "%s %d, pool %d used up, blkSize 0x%llx,"
+			"blkCnt 0x%x\n",__func__,__LINE__, pool->poolId,
+			pool_cfg->blkSize, pool_cfg->blkCnt);
+	}
+	spin_unlock(&pool->lock);
+	return ret;
+}
+
+static int vb_get_block(struct esVB_GET_BLOCK_CMD_S *getBlkCmd,
+	struct esVB_K_BLOCK_INFO_S **ppBlk)
+{
+	int ret = -EINVAL;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	struct esVB_GET_BLOCK_REQ_S *req = &getBlkCmd->getBlkReq;
+	struct esVB_K_POOL_INFO_S *pool = NULL, *pool_tmp = NULL;
+	struct esVB_POOL_CONFIG_S *pool_cfg;
+	unsigned long bkt = 0;
+
+	if (VB_UID_PRIVATE == req->uid) {
+		down_read(&partitions->idr_lock);
+		ret = vb_find_pool_by_id_unlock(req->poolId, &pool);
+		if (ret) {
+			up_read(&partitions->idr_lock);
+			dev_err(mmz_vb_dev, "%s %d, failed to find pool by id %d!\n",__func__,__LINE__, req->poolId);
+			return -EINVAL;
+		}
+		if (test_bit(MMZ_VB_POOL_FLAG_DESTORY, &pool->flag)) {
+			up_read(&partitions->idr_lock);
+			dev_err(mmz_vb_dev, "%s %d, pool %d is in destory state, not allow "
+				"to alloc block!\n",__func__,__LINE__, req->poolId);
+			return -ENOTSUPP;
+		}
+		pool_cfg = &pool->poolCfg;
+		if (req->blkSize > pool_cfg->blkSize) {
+			up_read(&partitions->idr_lock);
+			dev_err(mmz_vb_dev, "%s %d, pool blkSize 0x%llx is "
+				"smaller than request size 0x%llx\n",__func__,__LINE__,
+				pool_cfg->blkSize, req->blkSize);
+			return -EINVAL;
+		}
+		ret = vb_pool_get_block(pool, ppBlk);
+		up_read(&partitions->idr_lock);
+	} else if (req->uid >= VB_UID_COMMON && req->uid < VB_UID_MAX) {
+		down_read(&vb_priv->pool_lock[req->uid]);
+		/*try to get block for the exact block size */
+		hash_for_each_possible(vb_priv->ht[req->uid], pool, node, PAGE_ALIGN(req->blkSize)) {
+			pool_cfg = &pool->poolCfg;
+			if (PAGE_ALIGN(req->blkSize) == pool_cfg->blkSize &&
+				!strcmp(req->mmzName, pool_cfg->mmzName)) {
+					ret = vb_pool_get_block(pool, ppBlk);
+					if (0 == ret) {
+						break;
+					}
+			}
+		}
+		/*try to get block from the pool whose block size > req->blkSize*/
+		if (0 != ret) {
+			hash_for_each(vb_priv->ht[req->uid], bkt, pool, node) {
+				pool_cfg = &pool->poolCfg;
+				if (req->blkSize < pool_cfg->blkSize &&
+					!strcmp(req->mmzName, pool_cfg->mmzName)) {
+						/*get the pool size which is closest to the req->blkSize*/
+						if ((NULL == pool_tmp || (pool_tmp->poolCfg.blkSize > pool->poolCfg.blkSize))
+							&& vb_pool_get_free_block_cnt_unlock(pool)) {
+								pool_tmp = pool;
+						}
+				}
+			}
+			if (NULL != pool_tmp) {
+				ret = vb_pool_get_block(pool_tmp, ppBlk);
+			}
+		}
+		up_read(&vb_priv->pool_lock[req->uid]);
+	} else {
+        dev_err(mmz_vb_dev, "%s %d, invaild uid %d\n",__func__,__LINE__, req->uid);
+    }
+	return ret;
+}
+
+static void vb_release_block(struct esVB_K_BLOCK_INFO_S *pBlk)
+{
+	struct esVB_K_POOL_INFO_S *pool;
+	struct mmz_vb_priv *vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &vb_priv->partitions;
+	bool need_destory = false;
+	struct rw_semaphore *lock;
+	int ret;
+
+	pool = pBlk->pool;
+
+	lock = VB_UID_PRIVATE == pool->enVbUid ? \
+		&partitions->idr_lock : &vb_priv->pool_lock[pool->enVbUid];
+	/*
+	  usually we don't need to destory pool here.
+	  so just get read lock first.
+	*/
+	down_read(lock);
+	spin_lock(&pool->lock);
+	bitmap_clear(pool->bitmap, pBlk->nr, 1);
+	if (bitmap_empty(pool->bitmap, pool->poolCfg.blkCnt) &&
+		test_bit(MMZ_VB_POOL_FLAG_DESTORY, &pool->flag)) {
+			need_destory = true;
+	}
+	spin_unlock(&pool->lock);
+	up_read(lock);
+	if (atomic_dec_return(&vb_priv->allocBlkcnt) == 0) {
+		module_put(THIS_MODULE);
+	}
+
+	if (true == need_destory) {
+		down_write(lock);
+		ret = mmz_vb_do_destory_pool(pool, false, false);
+		if (ret) {
+			dev_err(mmz_vb_dev, "%s %d, faild to destory pool, enVbUid %d, PoolId %d, ret %d\n",
+				__func__,__LINE__, pool->enVbUid, pool->poolId, ret);
+		}
+		up_write(lock);
+	}
+}
+
+static int vb_is_splitted_blk(int fd, bool *isSplittedBlk)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+
+	/* get dmabuf handle */
+	dmabuf = dma_buf_get(fd);
+	if (IS_ERR(dmabuf)) {
+		return -EINVAL;
+	}
+
+	if (strncmp(dmabuf->exp_name, MMZ_VB_DMABUF_NAME, sizeof(MMZ_VB_DMABUF_NAME))) {
+		vb_err("It's NOT a mmz_vb buffer!!!\n");
+		dma_buf_put(dmabuf);
+		return -EINVAL;
+	}
+
+	if (!strncmp(dmabuf->exp_name, MMZ_VB_DMABUF_SPLITTED_NAME, sizeof(MMZ_VB_DMABUF_SPLITTED_NAME)))
+		*isSplittedBlk = true;
+	else
+		*isSplittedBlk = false;
+
+	dma_buf_put(dmabuf);
+	return ret;
+}
+
+static int vb_blk_to_pool(struct esVB_BLOCK_TO_POOL_CMD_S *blkToPoolCmd)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+	struct mmz_vb_buffer *buffer;
+	struct eswin_split_buffer *splittedBuffer;
+	struct dma_buf *blkDmabuf;
+	bool isSplittedBlk;
+
+	/* get dmabuf handle */
+	dmabuf = dma_buf_get(blkToPoolCmd->fd);
+	if (IS_ERR(dmabuf)) {
+		return -EINVAL;
+	}
+
+	ret = vb_is_splitted_blk(blkToPoolCmd->fd, &isSplittedBlk);
+	if (ret) {
+		dma_buf_put(dmabuf);
+		ret = -EINVAL;
+		goto out_put_dmabuf;
+	}
+
+	if (true == isSplittedBlk) { // This is a splitted block
+		splittedBuffer = dmabuf->priv;
+		blkDmabuf = dma_buf_get(splittedBuffer->dbuf_fd);
+		if (IS_ERR(blkDmabuf)) {
+			ret = -EINVAL;
+			goto out_put_dmabuf;
+		}
+		buffer = blkDmabuf->priv;
+		blkToPoolCmd->poolId = buffer->pBlk->pool->poolId;
+		dma_buf_put(blkDmabuf);
+	}
+	else {	// This is a real block
+		buffer = dmabuf->priv;
+		blkToPoolCmd->poolId = buffer->pBlk->pool->poolId;
+	}
+
+out_put_dmabuf:
+	dma_buf_put(dmabuf);
+	return ret;
+}
+
+static int vb_get_blk_offset(struct esVB_GET_BLOCKOFFSET_CMD_S *getBlkOffsetCmd)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+	struct mmz_vb_buffer *buffer;
+	struct eswin_split_buffer *splittedBuffer;
+	struct dma_buf *blkDmabuf;
+	__u64 blkSize, offsetInPool;
+
+	bool isSplittedBlk;
+
+	dmabuf = dma_buf_get(getBlkOffsetCmd->fd);
+	if (IS_ERR(dmabuf)) {
+		return -EINVAL;
+	}
+
+	ret = vb_is_splitted_blk(getBlkOffsetCmd->fd, &isSplittedBlk);
+	if (ret) {
+		ret = -EINVAL;
+		goto out_put_dmabuf;
+	}
+
+	if (true == isSplittedBlk) { // It's a splitted block
+		splittedBuffer = dmabuf->priv;
+		blkDmabuf = dma_buf_get(splittedBuffer->dbuf_fd);
+		if (IS_ERR(blkDmabuf)) {
+			ret = -EINVAL;
+			goto out_put_dmabuf;
+		}
+		buffer = blkDmabuf->priv;
+		blkSize = buffer->len;
+		offsetInPool = blkSize * buffer->pBlk->nr + splittedBuffer->slice.offset;
+		dma_buf_put(blkDmabuf);
+	}
+	else { // It's a real block
+		buffer = dmabuf->priv;
+		blkSize = buffer->len;
+		offsetInPool = blkSize * buffer->pBlk->nr;
+	}
+	getBlkOffsetCmd->offset = offsetInPool;
+
+out_put_dmabuf:
+	dma_buf_put(dmabuf);
+
+	return ret;
+}
+
+static int vb_split_dmabuf(struct esVB_SPLIT_DMABUF_CMD_S *splitDmabufCmd)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+	char splittedBuffer_ExpName[ES_MAX_MMZ_NAME_LEN];
+	int i;
+
+	if (!splitDmabufCmd->len)
+		return -EINVAL;
+
+	/* get dmabuf handle */
+	dmabuf = dma_buf_get(splitDmabufCmd->fd);
+	if (IS_ERR(dmabuf)) {
+		return -EINVAL;
+	}
+
+	if (strstr(dmabuf->exp_name, "_splitted")) { // It's a splitted dmabuf already, can't be splitted further
+		vb_err("Can't split a splitted buffer!!!\n");
+		ret = -EINVAL;
+		goto out_put_dmabuf;
+	}
+
+	/* offset and len must be paged aligned */
+	if (!PAGE_ALIGNED(splitDmabufCmd->offset) || !PAGE_ALIGNED(splitDmabufCmd->len)) {
+		vb_err("splitted offset or len is not page aligned!!!\n");
+		ret = -EINVAL;
+		goto out_put_dmabuf;
+	}
+
+	if (splitDmabufCmd->offset + splitDmabufCmd->len > dmabuf->size) {
+		vb_err("Splitted offset(0x%llx)+len(0x%llx) exceed the size(0x%llx) of the original buffer!!!\n",
+			splitDmabufCmd->offset, splitDmabufCmd->len, (__u64)dmabuf->size);
+		ret = -EINVAL;
+		goto out_put_dmabuf;
+	}
+
+	/* Apend "_splitted" to the splitted buffer expname, so that it is identified by the suffix */
+	i = snprintf(splittedBuffer_ExpName, sizeof(splittedBuffer_ExpName), "%s_splitted", dmabuf->exp_name);
+	if (i > sizeof(splittedBuffer_ExpName)) {
+		vb_err("Length of name(%d) for the the splitted buffer exceed the max name len(%ld)!!!\n",
+			i, sizeof(splittedBuffer_ExpName));
+		ret = -EINVAL;
+		goto out_put_dmabuf;
+	}
+
+	splitDmabufCmd->slice_fd = esw_common_dmabuf_split_export(splitDmabufCmd->fd, splitDmabufCmd->offset,
+						splitDmabufCmd->len, dmabuf->file->f_flags, splittedBuffer_ExpName);
+	if (splitDmabufCmd->slice_fd < 0) {
+		vb_err("Failed to split buffer, errVal %d\n", splitDmabufCmd->slice_fd);
+		ret = -EFAULT;
+		goto out_put_dmabuf;
+	}
+
+out_put_dmabuf:
+	dma_buf_put(dmabuf);
+
+	return ret;
+}
+
+static int vb_get_dmabuf_refcnt(struct esVB_DMABUF_REFCOUNT_CMD_S *getDmabufRefCntCmd)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+
+	/* get dmabuf handle */
+	dmabuf = dma_buf_get(getDmabufRefCntCmd->fd);
+	if (IS_ERR(dmabuf)) {
+		return -EINVAL;
+	}
+
+	/* minus 1 because it was +1 by dma_buf_get */
+	getDmabufRefCntCmd->refCnt = file_count(dmabuf->file) - 1;
+
+	dma_buf_put(dmabuf);
+	return ret;
+}
+
+static int do_vb_retrive_mem_node(struct dma_buf *dmabuf,
+	EIC770X_LOGICAL_MEM_NODE_E *p_nid, eic770x_memory_type_t *p_mem_type)
+{
+	int ret = 0;
+	struct dma_buf_attachment *attach;
+	struct sg_table *sgt;
+	struct page *page = NULL;
+	unsigned long pfn;
+
+	get_dma_buf(dmabuf);
+	attach = dma_buf_attach(dmabuf, mmz_vb_dev);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+		/* put dmabuf back */
+		dma_buf_put(dmabuf);
+		return ret;
+	}
+
+	sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		dma_buf_detach(dmabuf, attach);
+		dma_buf_put(dmabuf);
+		return ret;
+	}
+
+	page = sg_page(sgt->sgl);
+	pfn = page_to_pfn(page);
+	arch_get_mem_node_and_type(pfn, p_nid, p_mem_type);
+
+	dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
+	/* detach */
+	dma_buf_detach(dmabuf, attach);
+	/* put dmabuf back */
+	dma_buf_put(dmabuf);
+
+	pr_debug("%s, mem node is %d\n", __func__, *p_nid);
+	return ret;
+}
+
+static int vb_retrieve_mem_node(struct esVB_RETRIEVE_MEM_NODE_CMD_S *retrieveMemNodeCmd,
+	eic770x_memory_type_t *p_mem_type)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+	struct vm_area_struct *vma = NULL;
+	vm_flags_t vm_flags;
+	struct mm_struct *mm = current->mm;
+	u64 vaddr;
+
+	/* If cpu_vaddr is NULL, then try to retrieve mem node id by fd */
+	if (retrieveMemNodeCmd->cpu_vaddr == NULL) {
+		/* get dmabuf handle */
+		dmabuf = dma_buf_get(retrieveMemNodeCmd->fd);
+		if (IS_ERR(dmabuf)) {
+			return PTR_ERR(dmabuf);
+		}
+
+		ret = do_vb_retrive_mem_node(dmabuf, &retrieveMemNodeCmd->numa_node, p_mem_type);
+		/* put dmabuf back */
+		dma_buf_put(dmabuf);
+	}
+	else {
+		vaddr = (u64)retrieveMemNodeCmd->cpu_vaddr;
+		mmap_read_lock(mm);
+		vma = vma_lookup(mm, vaddr & PAGE_MASK);
+		if (!vma) {
+			pr_err("Failed to vma_lookup!\n");
+			return -EFAULT;
+		}
+		vm_flags = vma->vm_flags;
+		mmap_read_unlock(mm);
+
+		if (!(vm_flags & (VM_IO | VM_PFNMAP)) || (NULL == vma->vm_private_data)) {
+			pr_debug("This vaddr is NOT mmapped with VM_PFNMAP!\n");
+			return -EFAULT;
+		}
+		dmabuf = vma->vm_private_data;
+		ret = do_vb_retrive_mem_node(dmabuf, &retrieveMemNodeCmd->numa_node, p_mem_type);
+	}
+
+	return ret;
+}
+
+static int vb_get_dmabuf_size(struct esVB_DMABUF_SIZE_CMD_S *getDmabufSizeCmd)
+{
+	int ret = 0;
+	struct dma_buf *dmabuf;
+
+	/* get dmabuf handle */
+	dmabuf = dma_buf_get(getDmabufSizeCmd->fd);
+	if (IS_ERR(dmabuf)) {
+		return -EINVAL;
+	}
+
+	/* minus 1 because it was +1 by dma_buf_get */
+	getDmabufSizeCmd->size = dmabuf->size;
+
+	dma_buf_put(dmabuf);
+
+	return ret;
+}
+
+static int mmz_vb_init_memory_region(void)
+{
+	struct mmz_vb_priv *mmz_vb_priv = g_mmz_vb_priv;
+	struct esVB_K_MMZ_S *partitions = &mmz_vb_priv->partitions;
+	int nid, part = 0;
+	int partitionID = 0;
+	char blkName[BLOCK_MAX_NAME];
+	struct mem_block *memblock = NULL;
+
+	for (nid = 0; nid < 2; nid++) {
+		for (part = 0; part < 2; part++) {
+			snprintf(blkName, sizeof(blkName), "%s%d_part_%d", es_mmz_name_prefix, nid, part);
+			memblock = eswin_rsvmem_get_memblock(blkName);
+			if (memblock) {
+				partitions->mem_blocks[partitionID] = memblock;
+				dev_info(mmz_vb_dev, "%s was found successfully\n", blkName);
+				partitionID++;
+			}
+			else {
+				dev_dbg(mmz_vb_dev, "%s was NOT found\n", blkName);
+			}
+		}
+	}
+
+	/* Indicate how many VB memory block have been correctly initialized */
+	return partitionID;
+}
diff --git a/drivers/memory/eswin/es_proc/Kconfig b/drivers/memory/eswin/es_proc/Kconfig
new file mode 100644
index 000000000000..d1a2f8220e9b
--- /dev/null
+++ b/drivers/memory/eswin/es_proc/Kconfig
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_PROC
+	tristate "ESWIN MMZ proc interface"
+	help
+	  MMZ proc interface for user space.
+
diff --git a/drivers/memory/eswin/es_proc/Makefile b/drivers/memory/eswin/es_proc/Makefile
new file mode 100644
index 000000000000..09e2bc7767a3
--- /dev/null
+++ b/drivers/memory/eswin/es_proc/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ESWIN_PROC)	+= es_proc.o
+ccflags-y := -DDEBUG
diff --git a/drivers/memory/eswin/es_proc/es_proc.c b/drivers/memory/eswin/es_proc/es_proc.c
new file mode 100644
index 000000000000..0e955525f082
--- /dev/null
+++ b/drivers/memory/eswin/es_proc/es_proc.c
@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ESWIN proc APIs for MMZ_VB
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    HuangYiFeng<huangyifeng@eswincomputing.com>
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/fs.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/slab.h>
+#include <asm/uaccess.h>
+#include <linux/es_proc.h>
+
+static LIST_HEAD(list);
+static es_proc_entry_t *proc_entry = NULL;
+static DEFINE_MUTEX(proc_lock);
+static DEFINE_MUTEX(list_lock);
+
+es_proc_entry_t *_es_proc_mkdir(const char *name, umode_t mode, es_proc_entry_t *parent);
+static int es_seq_show(struct seq_file *s, void *p)
+{
+	es_proc_entry_t *oldsentry = s->private;
+	es_proc_entry_t sentry;
+
+	if (oldsentry == NULL) {
+		pr_err("parameter invalid!\n");
+		return -1;
+	}
+	memset(&sentry, 0, sizeof(es_proc_entry_t));
+	/* only these two parameters are used */
+	sentry.seqfile = s;
+	sentry.private = oldsentry->private;
+	oldsentry->read(&sentry);
+	return 0;
+}
+
+static ssize_t es_procwrite(struct file *file, const char __user *buf,
+			      size_t count, loff_t *ppos)
+{
+	es_proc_entry_t *item = pde_data(file_inode(file));
+
+	if ((item != NULL) && (item->write != NULL)) {
+		return item->write(item, buf, count, (long long *)ppos);
+	}
+
+	return -ENOSYS;
+}
+
+static int es_procopen(struct inode *inode, struct file *file)
+{
+	es_proc_entry_t *sentry = pde_data(inode);
+
+	if ((sentry != NULL) && (sentry->open != NULL)) {
+		sentry->open(sentry);
+	}
+	return single_open(file, es_seq_show, sentry);
+}
+
+static const struct proc_ops es_proc_ops = {
+	.proc_open = es_procopen,
+	.proc_read = seq_read,
+	.proc_write = es_procwrite,
+	.proc_lseek = seq_lseek,
+	.proc_release = single_release
+};
+
+es_proc_entry_t *es_create_proc(const char *name, umode_t mode, es_proc_entry_t *parent)
+{
+	struct proc_dir_entry *entry = NULL;
+	es_proc_entry_t *sentry = NULL;
+
+	sentry = kzalloc(sizeof(struct es_proc_dir_entry), GFP_KERNEL);
+	if (sentry == NULL) {
+		pr_err("kmalloc failed!\n");
+		return NULL;
+	}
+
+	strncpy(sentry->name, name, sizeof(sentry->name) - 1);
+	pr_debug("create file %s with mode %o\n", name, (unsigned)mode);
+	if (parent == NULL) {
+		entry = proc_create_data(name, mode, NULL, &es_proc_ops, sentry);
+	} else {
+		entry = proc_create_data(name, mode, parent->proc_dir_entry, &es_proc_ops, sentry);
+	}
+	if (entry == NULL) {
+		pr_err("create_proc_entry failed!\n");
+		kfree(sentry);
+		sentry = NULL;
+		return NULL;
+	}
+	sentry->proc_dir_entry = entry;
+	sentry->open = NULL;
+	sentry->parent = parent;
+
+	list_add_tail(&(sentry->node), &list);
+	return sentry;
+}
+
+es_proc_entry_t *es_create_proc_entry(const char *name, umode_t mode,
+					  es_proc_entry_t *parent)
+{
+	mutex_lock(&proc_lock);
+	if (!proc_entry )
+		proc_entry = _es_proc_mkdir("eswin", 0755, NULL);
+	mutex_unlock(&proc_lock);
+	if (!proc_entry )
+		return NULL;
+
+	/* If NULL, create entry under the eswin dir, otherwise under the specified parent*/
+	if (parent == NULL)
+		parent = proc_entry;
+
+	return es_create_proc(name, mode, parent);
+}
+EXPORT_SYMBOL(es_create_proc_entry);
+
+void _es_remove_proc_entry(const char *name, es_proc_entry_t *parent)
+{
+	struct es_proc_dir_entry *sproc, *tmp;
+	struct es_proc_dir_entry *child;
+	bool found = false;
+
+	if (name == NULL) {
+		pr_err("parameter invalid!\n");
+		return;
+	}
+	mutex_lock(&list_lock);
+	list_for_each_entry_safe(sproc, tmp, &list, node) {
+		if (strncmp(sproc->name, name, sizeof(sproc->name)) != 0)
+			continue;
+		// got target dir, then checking if it's contains any child node
+		list_for_each_entry(child, &list, node) {
+			if (child->parent == sproc) {
+				// some node's parent point to target entry, do not remove.
+				pr_err("Error: directory %s is not empty!\n", name);
+				mutex_unlock(&list_lock);
+				return;
+			}
+		}
+
+		// remove
+		found = true;
+		remove_proc_entry(name, parent == NULL ? NULL : parent->proc_dir_entry);
+		list_del(&sproc->node);
+		kfree(sproc);
+		break;
+	}
+
+	mutex_unlock(&list_lock);
+	if (found)
+		pr_debug("entry %s with parent=%s removed\n", name, parent ? parent->name : "eswin");
+	else
+		pr_err("entry %s with parent=%s not found\n", name, parent ? parent->name : "eswin");
+}
+
+void es_remove_proc_entry(const char *name, es_proc_entry_t *parent)
+{
+	mutex_lock(&proc_lock);
+	if (WARN_ON(!proc_entry))
+		pr_err("remove %s before es_proc initialize\n", name);
+	mutex_unlock(&proc_lock);
+
+	/* If NULL, remove entry under the eswin dir, otherwise under the specified parent*/
+	if (parent == NULL)
+		parent = proc_entry;
+
+	return _es_remove_proc_entry(name, parent);
+}
+EXPORT_SYMBOL(es_remove_proc_entry);
+
+es_proc_entry_t *_es_proc_mkdir(const char *name, umode_t mode, es_proc_entry_t *parent)
+{
+	struct proc_dir_entry *proc = NULL;
+	struct es_proc_dir_entry *sproc = NULL;
+
+	sproc = kzalloc(sizeof(struct es_proc_dir_entry), GFP_KERNEL);
+	if (sproc == NULL) {
+		pr_err("kmalloc failed!\n");
+		return NULL;
+	}
+
+	strncpy(sproc->name, name, sizeof(sproc->name) - 1);
+
+	if (parent != NULL) {
+		proc = proc_mkdir_data(name, mode, parent->proc_dir_entry, sproc);
+	} else {
+		proc = proc_mkdir_data(name, mode, NULL, sproc);
+	}
+	if (proc == NULL) {
+		pr_err("proc_mkdir failed!\n");
+		kfree(sproc);
+		sproc = NULL;
+		return NULL;
+	}
+	sproc->proc_dir_entry = proc;
+	sproc->parent = parent;
+	INIT_LIST_HEAD(&sproc->node);
+
+	mutex_lock(&list_lock);
+	list_add_tail(&(sproc->node), &list);
+	mutex_unlock(&list_lock);
+	return sproc;
+}
+
+es_proc_entry_t *es_proc_mkdir(const char *name, umode_t mode, es_proc_entry_t *parent)
+{
+	mutex_lock(&proc_lock);
+	if (!proc_entry )
+		proc_entry = _es_proc_mkdir("eswin", 0755, NULL);
+	mutex_unlock(&proc_lock);
+	if (!proc_entry )
+		return NULL;
+
+	/* If NULL, create entry under the eswin dir, otherwise under the specified parent*/
+	if (parent == NULL)
+		parent = proc_entry;
+
+	return  _es_proc_mkdir(name, mode, parent);
+}
+EXPORT_SYMBOL(es_proc_mkdir);
+
+int es_seq_printf(es_proc_entry_t *entry, const char *fmt, ...)
+{
+	struct seq_file *s = (struct seq_file *)(entry->seqfile);
+	va_list args;
+	int r = 0;
+
+	va_start(args, fmt);
+	seq_vprintf(s, fmt, args);
+	va_end(args);
+
+	return r;
+}
+EXPORT_SYMBOL(es_seq_printf);
+
diff --git a/drivers/memory/eswin/es_rsvmem_heap/Kconfig b/drivers/memory/eswin/es_rsvmem_heap/Kconfig
new file mode 100644
index 000000000000..3b425da42e2f
--- /dev/null
+++ b/drivers/memory/eswin/es_rsvmem_heap/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config ESWIN_RSVMEM_HEAP
+	tristate "ESWIN reserved memory heap"
+	help
+	  ESWIN reserved memory heap device.
diff --git a/drivers/memory/eswin/es_rsvmem_heap/Makefile b/drivers/memory/eswin/es_rsvmem_heap/Makefile
new file mode 100644
index 000000000000..293a7b51148d
--- /dev/null
+++ b/drivers/memory/eswin/es_rsvmem_heap/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ESWIN_RSVMEM_HEAP)	+= eswin_rsvmem_heap.o eswin_rsvmem_common.o dmabuf-heap-import-helper.o
+
+
+ES_RSVMEM_HEADER := drivers/memory/eswin/es_rsvmem_heap/include/linux
diff --git a/drivers/memory/eswin/es_rsvmem_heap/dmabuf-heap-import-helper.c b/drivers/memory/eswin/es_rsvmem_heap/dmabuf-heap-import-helper.c
new file mode 100644
index 000000000000..c634da8df58e
--- /dev/null
+++ b/drivers/memory/eswin/es_rsvmem_heap/dmabuf-heap-import-helper.c
@@ -0,0 +1,1379 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN DMABUF heap helper APIs
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Min Lin <linmin@eswincomputing.com>
+ */
+
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/errno.h>
+#include <linux/highmem.h>
+#include <linux/fcntl.h>
+#include <linux/scatterlist.h>
+#include <linux/miscdevice.h>
+#include <linux/dma-map-ops.h>
+#include <linux/dma-heap.h>
+#include <linux/dma-resv.h>
+#include <linux/dmabuf-heap-import-helper.h>
+
+static struct device *split_dmabuf_dev;
+
+struct drm_prime_member {
+	struct dma_buf *dma_buf;
+	uint64_t handle;
+
+	struct rb_node dmabuf_rb;
+	struct rb_node handle_rb;
+};
+
+struct dma_heap_attachment {
+	struct device *dev;
+	struct sg_table *table;
+	struct list_head list;
+	bool mapped;
+};
+
+struct dma_heap_attachment_cma {
+	struct device *dev;
+	struct sg_table table;
+	struct list_head list;
+	bool mapped;
+};
+
+static void __common_dmabuf_heap_release(struct kref *kref);
+
+static int dmabuf_heap_add_buf_handle(struct dmaheap_file_private *prime_fpriv,
+				    struct dma_buf *dma_buf, uint64_t handle)
+{
+	struct drm_prime_member *member;
+	struct rb_node **p, *rb;
+
+	member = kmalloc(sizeof(*member), GFP_KERNEL);
+	if (!member)
+		return -ENOMEM;
+
+	get_dma_buf(dma_buf);
+	member->dma_buf = dma_buf;
+	member->handle = handle;
+
+	rb = NULL;
+	p = &prime_fpriv->dmabufs.rb_node;
+	while (*p) {
+		struct drm_prime_member *pos;
+
+		rb = *p;
+		pos = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
+		if (dma_buf > pos->dma_buf)
+			p = &rb->rb_right;
+		else
+			p = &rb->rb_left;
+	}
+	rb_link_node(&member->dmabuf_rb, rb, p);
+	rb_insert_color(&member->dmabuf_rb, &prime_fpriv->dmabufs);
+
+	rb = NULL;
+	p = &prime_fpriv->handles.rb_node;
+	while (*p) {
+		struct drm_prime_member *pos;
+
+		rb = *p;
+		pos = rb_entry(rb, struct drm_prime_member, handle_rb);
+		if (handle > pos->handle)
+			p = &rb->rb_right;
+		else
+			p = &rb->rb_left;
+	}
+	rb_link_node(&member->handle_rb, rb, p);
+	rb_insert_color(&member->handle_rb, &prime_fpriv->handles);
+
+	return 0;
+}
+
+static int dmabuf_heap_lookup_buf_handle(struct dmaheap_file_private *prime_fpriv,
+				       struct dma_buf *dma_buf,
+				       uint64_t *handle)
+{
+	struct rb_node *rb;
+
+	rb = prime_fpriv->dmabufs.rb_node;
+	while (rb) {
+		struct drm_prime_member *member;
+
+		member = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
+		if (member->dma_buf == dma_buf) {
+			*handle = member->handle;
+			return 0;
+		} else if (member->dma_buf < dma_buf) {
+			rb = rb->rb_right;
+		} else {
+			rb = rb->rb_left;
+		}
+	}
+
+	return -ENOENT;
+}
+
+static void _dmabuf_heap_remove_buf_handle(struct dmaheap_file_private *prime_fpriv,
+					struct dma_buf *dma_buf)
+{
+	struct rb_node *rb;
+
+	rb = prime_fpriv->dmabufs.rb_node;
+	while (rb) {
+		struct drm_prime_member *member;
+
+		member = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
+		if (member->dma_buf == dma_buf) {
+			rb_erase(&member->handle_rb, &prime_fpriv->handles);
+			rb_erase(&member->dmabuf_rb, &prime_fpriv->dmabufs);
+
+			dma_buf_put(dma_buf);
+			kfree(member);
+			return;
+		} else if (member->dma_buf < dma_buf) {
+			rb = rb->rb_right;
+		} else {
+			rb = rb->rb_left;
+		}
+	}
+}
+
+void common_dmabuf_heap_import_init(struct heap_root *root, struct device *dev)
+{
+	memset(root, 0, sizeof(*root));
+
+	mutex_init(&root->lock);
+	INIT_LIST_HEAD(&root->header);
+
+	root->dev = dev;
+}
+EXPORT_SYMBOL(common_dmabuf_heap_import_init);
+
+void common_dmabuf_heap_import_uninit(struct heap_root *root)
+{
+	struct heap_mem *h, *tmp;
+
+	mutex_lock(&root->lock);
+	list_for_each_entry_safe(h, tmp, &root->header, list) {
+		kref_put(&h->refcount, __common_dmabuf_heap_release);
+	}
+	mutex_unlock(&root->lock);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_import_uninit);
+
+static struct heap_mem *dmabuf_heap_import(struct heap_root *root, int fd)
+{
+    struct dma_buf *dma_buf;
+    struct dma_buf_attachment *attach;
+    struct sg_table *sgt;
+
+    uint64_t handle;
+    struct heap_mem *heap_obj;
+    int ret;
+
+    /* get dmabuf handle */
+    dma_buf = dma_buf_get(fd);
+	if (IS_ERR(dma_buf))
+		return ERR_CAST(dma_buf);
+
+    mutex_lock(&root->lock);
+
+    ret = dmabuf_heap_lookup_buf_handle(&root->fp, dma_buf, &handle);
+    if (ret == 0) {
+        heap_obj = (struct heap_mem *)handle;
+        dma_buf_put(dma_buf);
+        kref_get(&heap_obj->refcount);
+        mutex_unlock(&root->lock);
+	return heap_obj;
+    }
+
+    heap_obj = kzalloc(sizeof(*heap_obj), GFP_KERNEL);
+    if (!heap_obj) {
+        mutex_unlock(&root->lock);
+        dma_buf_put(dma_buf);
+        return ERR_PTR(-ENOMEM);
+    }
+
+    attach = dma_buf_attach(dma_buf, root->dev);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+        goto clean_up;
+    }
+
+    sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		goto fail_detach;
+	}
+
+    heap_obj->dbuf_fd = fd;
+    heap_obj->dbuf = dma_buf;
+
+    heap_obj->import_attach = attach;
+    heap_obj->sgt = sgt;
+
+	heap_obj->root = root;
+	heap_obj->vaddr = NULL;
+	heap_obj->dir = DMA_BIDIRECTIONAL;
+
+	/* get_dma_buf was called in dmabuf_heap_add_buf_handle()*/
+    ret = dmabuf_heap_add_buf_handle(&root->fp, dma_buf, (uint64_t)heap_obj);
+    if (ret) {
+        goto fail_add_handle;
+    }
+
+    kref_init(&heap_obj->refcount);
+
+    INIT_LIST_HEAD(&heap_obj->list);
+    list_add(&heap_obj->list, &root->header);
+
+    mutex_unlock(&root->lock);
+
+    dma_buf_put(dma_buf);
+
+    return heap_obj;
+
+fail_add_handle:
+	dma_buf_unmap_attachment_unlocked(heap_obj->import_attach, heap_obj->sgt, heap_obj->dir);
+fail_detach:
+    dma_buf_detach(dma_buf, attach);
+clean_up:
+    kfree(heap_obj);
+	mutex_unlock(&root->lock);
+    dma_buf_put(dma_buf);
+
+    return ERR_PTR(ret);
+}
+
+static struct heap_mem *dmabuf_heap_import_with_dma_buf_st(struct heap_root *root, struct dma_buf *dma_buf)
+{
+    struct dma_buf_attachment *attach;
+    struct sg_table *sgt;
+
+    uint64_t handle;
+    struct heap_mem *heap_obj;
+    int ret;
+
+    mutex_lock(&root->lock);
+
+    ret = dmabuf_heap_lookup_buf_handle(&root->fp, dma_buf, &handle);
+	if (ret == 0) {
+        heap_obj = (struct heap_mem *)handle;
+        kref_get(&heap_obj->refcount);
+        mutex_unlock(&root->lock);
+	return heap_obj;
+    }
+
+    heap_obj = kzalloc(sizeof(*heap_obj), GFP_KERNEL);
+    if (!heap_obj) {
+        mutex_unlock(&root->lock);
+        return ERR_PTR(-ENOMEM);
+    }
+
+    attach = dma_buf_attach(dma_buf, root->dev);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+        goto clean_up;
+    }
+
+    sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		goto fail_detach;
+	}
+
+    heap_obj->dbuf_fd = -1;
+    heap_obj->dbuf = dma_buf;
+
+    heap_obj->import_attach = attach;
+    heap_obj->sgt = sgt;
+
+	heap_obj->root = root;
+	heap_obj->vaddr = NULL;
+	heap_obj->dir = DMA_BIDIRECTIONAL;
+
+	/* get_dma_buf was called in dmabuf_heap_add_buf_handle()*/
+    ret = dmabuf_heap_add_buf_handle(&root->fp, dma_buf, (uint64_t)heap_obj);
+    if (ret) {
+        goto fail_add_handle;
+    }
+
+    kref_init(&heap_obj->refcount);
+
+    INIT_LIST_HEAD(&heap_obj->list);
+    list_add(&heap_obj->list, &root->header);
+
+    mutex_unlock(&root->lock);
+
+    return heap_obj;
+
+fail_add_handle:
+	dma_buf_unmap_attachment_unlocked(heap_obj->import_attach, heap_obj->sgt, heap_obj->dir);
+fail_detach:
+    dma_buf_detach(dma_buf, attach);
+clean_up:
+    kfree(heap_obj);
+    mutex_unlock(&root->lock);
+
+    return ERR_PTR(ret);
+}
+
+struct heap_mem *common_dmabuf_lookup_heapobj_by_fd(struct heap_root *root, int fd)
+{
+	int ret = 0;
+	struct dma_buf *dma_buf;
+	struct heap_mem *heap_obj;
+
+	/* get dmabuf handle */
+	dma_buf = dma_buf_get(fd);
+	if (IS_ERR(dma_buf))
+		return ERR_CAST(dma_buf);
+
+	mutex_lock(&root->lock);
+	ret = dmabuf_heap_lookup_buf_handle(&root->fp, dma_buf, (uint64_t *)&heap_obj);
+	mutex_unlock(&root->lock);
+
+	dma_buf_put(dma_buf);
+	if (0 == ret)
+		return heap_obj;
+	else
+		return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(common_dmabuf_lookup_heapobj_by_fd);
+
+struct heap_mem *common_dmabuf_lookup_heapobj_by_dma_buf_st(struct heap_root *root, struct dma_buf *dma_buf)
+{
+	int ret = 0;
+	struct heap_mem *heap_obj;
+
+	pr_debug("%s:dma_buf=0x%px, file_count=%ld\n",
+		__func__, dma_buf, file_count(dma_buf->file));
+	mutex_lock(&root->lock);
+	ret = dmabuf_heap_lookup_buf_handle(&root->fp, dma_buf, (uint64_t *)&heap_obj);
+	mutex_unlock(&root->lock);
+
+	if (0 == ret)
+		return heap_obj;
+	else
+		return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(common_dmabuf_lookup_heapobj_by_dma_buf_st);
+
+struct heap_mem *common_dmabuf_heap_import_from_user(struct heap_root *root, int fd)
+{
+	return dmabuf_heap_import(root, fd);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_import_from_user);
+
+struct heap_mem *common_dmabuf_heap_import_from_user_with_dma_buf_st(struct heap_root *root, struct dma_buf *dma_buf)
+{
+	return dmabuf_heap_import_with_dma_buf_st(root, dma_buf);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_import_from_user_with_dma_buf_st);
+
+static void __common_dmabuf_heap_release(struct kref *kref)
+{
+	struct heap_root *root;
+	struct heap_mem *heap_obj = container_of(kref, struct heap_mem, refcount);
+
+	WARN_ON(!heap_obj);
+	if (!heap_obj)
+		return;
+
+	root = heap_obj->root;
+	WARN_ON(!mutex_is_locked(&root->lock));
+	list_del(&heap_obj->list);
+
+	common_dmabuf_heap_umap_vaddr(heap_obj);
+
+	dma_buf_unmap_attachment_unlocked(heap_obj->import_attach, heap_obj->sgt, heap_obj->dir);
+
+	dma_buf_detach(heap_obj->dbuf, heap_obj->import_attach);
+
+	/* dma_buf_put was called in _dmabuf_heap_remove_buf_handle()*/
+	_dmabuf_heap_remove_buf_handle(&root->fp, heap_obj->dbuf);
+
+	kfree(heap_obj);
+}
+
+void common_dmabuf_heap_release(struct heap_mem *heap_obj)
+{
+	struct heap_root *root;
+
+	if (WARN_ON(IS_ERR(heap_obj)))
+		return;
+
+	root = heap_obj->root;
+	mutex_lock(&root->lock);
+	kref_put(&heap_obj->refcount, __common_dmabuf_heap_release);
+	mutex_unlock(&root->lock);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_release);
+
+void *common_dmabuf_heap_map_vaddr(struct heap_mem *heap_obj)
+{
+    struct dma_buf_map map;
+    int ret;
+
+	if (WARN_ON(IS_ERR(heap_obj)))
+		return NULL;
+
+    if (heap_obj->vaddr)
+        return heap_obj->vaddr;
+
+    ret = dma_buf_vmap_unlocked(heap_obj->dbuf, &map);
+    if (ret)
+        return NULL;
+
+    WARN_ON_ONCE(map.is_iomem);
+    heap_obj->vaddr = map.vaddr;
+
+    return heap_obj->vaddr;
+}
+EXPORT_SYMBOL(common_dmabuf_heap_map_vaddr);
+
+void common_dmabuf_heap_umap_vaddr(struct heap_mem *heap_obj)
+{
+	struct dma_buf_map map;
+
+	if (WARN_ON(IS_ERR(heap_obj)))
+		return;
+
+	if (heap_obj && heap_obj->vaddr) {
+        map.vaddr = heap_obj->vaddr;
+        map.is_iomem = 0;
+        dma_buf_vunmap_unlocked(heap_obj->dbuf, &map);
+        heap_obj->vaddr = NULL;
+    }
+}
+EXPORT_SYMBOL(common_dmabuf_heap_umap_vaddr);
+
+struct heap_mem *
+common_dmabuf_heap_import_from_kernel(struct heap_root *root, char *name, size_t len, unsigned int fd_flags)
+{
+	int dbuf_fd;
+
+	dbuf_fd = eswin_heap_kalloc(name, len, O_RDWR | fd_flags, 0);
+	if (dbuf_fd < 0) {
+        return ERR_PTR(dbuf_fd);
+	}
+
+	return dmabuf_heap_import(root, dbuf_fd);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_import_from_kernel);
+
+struct heap_mem *common_dmabuf_heap_rsv_iova_map(struct heap_root *root, int fd, dma_addr_t iova, size_t size)
+{
+	struct dma_buf *dma_buf;
+	struct dma_buf_attachment *attach;
+	struct dma_heap_attachment *a;
+
+	uint64_t handle;
+	struct heap_mem *heap_obj;
+	ssize_t ret = 0;
+
+	/* get dmabuf handle */
+	dma_buf = dma_buf_get(fd);
+	if (IS_ERR(dma_buf)) {
+		return ERR_CAST(dma_buf);
+	}
+
+	mutex_lock(&root->lock);
+	dev_dbg(root->dev, "%s, fd=%d, iova=0x%llx, size=0x%lx\n", __func__, fd, iova, size);
+
+	ret = dmabuf_heap_lookup_buf_handle(&root->fp, dma_buf, &handle);
+	if (ret == 0) {
+		heap_obj = (struct heap_mem *)handle;
+		dma_buf_put(dma_buf);
+		kref_get(&heap_obj->refcount);
+		mutex_unlock(&root->lock);
+		return heap_obj;
+	}
+
+	heap_obj = kzalloc(sizeof(*heap_obj), GFP_KERNEL);
+	if (!heap_obj) {
+		mutex_unlock(&root->lock);
+		dma_buf_put(dma_buf);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	attach = dma_buf_attach(dma_buf, root->dev);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+		goto clean_up;
+	}
+
+	a = (struct dma_heap_attachment *)attach->priv;
+	ret = iommu_rsv_iova_map_sgt(root->dev, iova, a->table, 0, size);
+	dma_buf_detach(dma_buf, attach);
+	if (ret < 0)
+		goto clean_up;
+
+	heap_obj->dbuf_fd = fd;
+	heap_obj->dbuf = dma_buf;
+	heap_obj->root = root;
+	heap_obj->vaddr = NULL;
+	heap_obj->dir = DMA_BIDIRECTIONAL;
+	heap_obj->iova = iova;
+	heap_obj->size = size;
+
+	/* get_dma_buf was called in dmabuf_heap_add_buf_handle()*/
+	ret = dmabuf_heap_add_buf_handle(&root->fp, dma_buf, (uint64_t)heap_obj);
+		if (ret) {
+		goto fail_add_handle;
+	}
+	/* get_dma_buf was called in dmabuf_heap_add_buf_handle(), need to put back since
+	 * we don't want to hold the dma_buf in this API
+	*/
+	dma_buf_put(dma_buf);
+
+	kref_init(&heap_obj->refcount);
+
+	INIT_LIST_HEAD(&heap_obj->list);
+	list_add(&heap_obj->list, &root->header);
+
+	mutex_unlock(&root->lock);
+
+	dma_buf_put(dma_buf);
+
+	return heap_obj;
+
+fail_add_handle:
+	iommu_unmap_rsv_iova(root->dev, 0, iova, size);
+clean_up:
+	kfree(heap_obj);
+	mutex_unlock(&root->lock);
+	dma_buf_put(dma_buf);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_rsv_iova_map);
+
+static void __common_dmabuf_heap_rsv_iova_unmap(struct kref *kref)
+{
+	struct heap_root *root;
+	struct heap_mem *heap_obj = container_of(kref, struct heap_mem, refcount);
+
+	WARN_ON(!heap_obj);
+	if (!heap_obj)
+		return;
+
+	root = heap_obj->root;
+	WARN_ON(!mutex_is_locked(&root->lock));
+	list_del(&heap_obj->list);
+
+
+	iommu_unmap_rsv_iova(root->dev, 0, heap_obj->iova, heap_obj->size);
+	/* dma_buf_put will be  called in _dmabuf_heap_remove_buf_handle(),
+	 * so, call get_dma_buf first
+	*/
+	get_dma_buf(heap_obj->dbuf);
+	_dmabuf_heap_remove_buf_handle(&root->fp, heap_obj->dbuf);
+	kfree(heap_obj);
+}
+
+void common_dmabuf_heap_rsv_iova_unmap(struct heap_mem *heap_obj)
+{
+	struct heap_root *root;
+
+	if (WARN_ON(IS_ERR(heap_obj)))
+		return;
+
+	root = heap_obj->root;
+	mutex_lock(&root->lock);
+	kref_put(&heap_obj->refcount, __common_dmabuf_heap_rsv_iova_unmap);
+	mutex_unlock(&root->lock);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_rsv_iova_unmap);
+
+void common_dmabuf_heap_rsv_iova_uninit(struct heap_root *root)
+{
+	struct heap_mem *h, *tmp;
+
+	mutex_lock(&root->lock);
+	list_for_each_entry_safe(h, tmp, &root->header, list) {
+		kref_put(&h->refcount, __common_dmabuf_heap_rsv_iova_unmap);
+	}
+	mutex_unlock(&root->lock);
+}
+EXPORT_SYMBOL(common_dmabuf_heap_rsv_iova_uninit);
+
+
+struct eswin_split_attachment {
+	struct device *dev;
+	struct sg_table *table;
+	struct list_head list;
+	bool mapped;
+};
+
+static struct sg_table *dup_sg_table(struct sg_table *table)
+{
+	struct sg_table *new_table;
+	int ret, i;
+	struct scatterlist *sg, *new_sg;
+
+	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
+	if (!new_table)
+		return ERR_PTR(-ENOMEM);
+
+	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
+	if (ret) {
+		kfree(new_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_sg = new_table->sgl;
+	for_each_sgtable_sg(table, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+	return new_table;
+}
+
+// #define PRINT_ORIGINAL_SPLITTERS 1
+static int eswin_split_heap_attach(struct dma_buf *dmabuf,
+			   struct dma_buf_attachment *attachment)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	struct eswin_split_attachment *a;
+	struct sg_table *table;
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a)
+		return -ENOMEM;
+
+	table = dup_sg_table(&buffer->sg_table);
+	if (IS_ERR(table)) {
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	a->table = table;
+	a->dev = attachment->dev;
+	INIT_LIST_HEAD(&a->list);
+	a->mapped = false;
+
+	attachment->priv = a;
+
+	mutex_lock(&buffer->lock);
+	list_add(&a->list, &buffer->attachments);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static void eswin_split_heap_detach(struct dma_buf *dmabuf,
+			    struct dma_buf_attachment *attachment)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	struct eswin_split_attachment *a = attachment->priv;
+
+	mutex_lock(&buffer->lock);
+	list_del(&a->list);
+	mutex_unlock(&buffer->lock);
+
+	sg_free_table(a->table);
+	kfree(a->table);
+	kfree(a);
+}
+
+static struct sg_table *eswin_split_map_dma_buf(struct dma_buf_attachment *attachment,
+					     enum dma_data_direction direction)
+{
+	struct eswin_split_attachment *a = attachment->priv;
+	struct sg_table *table =a->table;
+	int ret;
+	unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
+
+	/* Skipt cache sync, since it takes a lot of time when import to device.
+	*  It's the user's responsibility for guaranteeing the cache coherency by
+	   flusing cache explicitly before importing to device.
+	*/
+	ret = dma_map_sgtable(attachment->dev, table, direction, attrs);
+
+	if (ret)
+		return ERR_PTR(-ENOMEM);
+	a->mapped = true;
+	return table;
+}
+
+static void eswin_spilt_unmap_dma_buf(struct dma_buf_attachment *attachment,
+				   struct sg_table *table,
+				   enum dma_data_direction direction)
+{
+	struct eswin_split_attachment *a = attachment->priv;
+	unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
+
+	a->mapped = false;
+
+	/* Skipt cache sync, since it takes a lot of time when unmap from device.
+	*  It's the user's responsibility for guaranteeing the cache coherency after
+	   the device has done processing the data.(For example, CPU do NOT read untill
+	   the device has done)
+	*/
+	dma_unmap_sgtable(attachment->dev, table, direction, attrs);
+}
+
+static int eswin_split_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+					     enum dma_data_direction direction)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	struct scatterlist *sg;
+	int i;
+
+	mutex_lock(&buffer->lock);
+
+	if (buffer->vmap_cnt)
+		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	/* Since the cache sync was skipped when eswin_rsvmem_heap_map_dma_buf/eswin_rsvmem_heap_unmap_dma_buf,
+	   So force cache sync here when user call ES_SYS_MemFlushCache, eventhough there
+	   is no device attached to this dmabuf.
+	*/
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_cpu(sg_phys(sg), sg->length, direction);
+
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static int eswin_split_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
+					   enum dma_data_direction direction)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	struct scatterlist *sg;
+	int i;
+
+	mutex_lock(&buffer->lock);
+
+	if (buffer->vmap_cnt)
+		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	/* Since the cache sync was skipped while eswin_rsvmem_heap_map_dma_buf/eswin_rsvmem_heap_unmap_dma_buf,
+	   So force cache sync here when user call ES_SYS_MemFlushCache, eventhough there
+	   is no device attached to this dmabuf.
+	*/
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_device(sg_phys(sg), sg->length, direction);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static int eswin_split_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	unsigned long addr = vma->vm_start;
+	unsigned long pgoff = (table->sgl->offset >> PAGE_SHIFT);
+	// unsigned long pgoff = (buffer->slice.offset >> PAGE_SHIFT);
+	unsigned long size_remaining = vma->vm_end - vma->vm_start;//vma_pages(vma);
+	struct sg_page_iter piter;
+	int ret;
+
+	/* Mapping secure_memory with cached proprty to user space for CPU is NOT permitted */
+	if (unlikely(!strncmp("secure_memory", dmabuf->exp_name, 13))) {
+		if (!(vma->vm_flags & VM_NORESERVE))
+			return -EPERM;
+	}
+
+	if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
+		return -EINVAL;
+
+	/* vm_private_data will be used by eswin-ipc-scpu.c.
+	    ipc will import this dmabuf to get iova.
+	*/
+	vma->vm_private_data = dmabuf;
+
+	/* support mman flag MAP_SHARED_VALIDATE | VM_NORESERVE, used to map uncached memory to user space.
+	   Users should guarantee this buffer has been flushed to cache already.
+	 */
+	if (vma->vm_flags & VM_NORESERVE) {
+		vm_flags_clear(vma, VM_NORESERVE);
+		vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+		/* skip sync cache, users should guarantee the cache is clean after done using it in
+		   cached mode(i.e, ES_SYS_Mmap(SYS_CACHE_MODE_CACHED))
+		*/
+	}
+	pr_debug("%s, vma->vm_start:0x%lx, vma->vm_end:0x%lx\n", __func__, vma->vm_start, vma->vm_end);
+	pr_debug("%s, size_remaining:0x%lx, pgoff:0x%lx, vma->vm_pgoff:0x%lx, dmabuf->size:0x%lx, start_phys:0x%llx, pfn sg_page(sg):0x%lx,sg->length=0x%x\n",
+		__func__, size_remaining, pgoff, vma->vm_pgoff, dmabuf->size, sg_phys(table->sgl),page_to_pfn(sg_page(table->sgl)), table->sgl->length);
+
+	for_each_sgtable_page(table, &piter, pgoff) {
+		struct page *page = sg_page_iter_page(&piter);
+		// pr_debug("addr:0x%lx, page_to_pfn:0x%lx\n", addr, page_to_pfn(page));
+		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
+				      vma->vm_page_prot);
+		if (ret)
+			return ret;
+		addr += PAGE_SIZE;
+		if (addr >= vma->vm_end)
+			return 0;
+	}
+
+	return 0;
+}
+
+static void *eswin_split_heap_do_vmap(struct dma_buf *dmabuf)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	pgprot_t prot = PAGE_KERNEL;
+	struct sg_table *table = &buffer->sg_table;
+	int npages = PAGE_ALIGN(buffer->slice.len) / PAGE_SIZE;
+	struct page **pages = vmalloc(sizeof(struct page *) * npages);
+	struct page **tmp = pages;
+	struct sg_page_iter piter;
+	void *vaddr;
+	unsigned long pgoff = (table->sgl->offset >> PAGE_SHIFT);
+
+	if (!pages)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_sgtable_page(table, &piter, pgoff) {
+		WARN_ON(tmp - pages >= npages);
+		*tmp++ = sg_page_iter_page(&piter);
+	}
+
+	/* The property of this dmabuf in kernel space is determined by heap alloc with fd_flag. */
+	if (buffer->fd_flags & O_DSYNC) {
+		prot = pgprot_dmacoherent(PAGE_KERNEL);
+		pr_debug("%s syport uncached kernel dmabuf!, prot=0x%x\n", __func__, (unsigned int)pgprot_val(prot));
+	}
+	else {
+		pr_debug("%s memport cached kernel dmabuf!\n", __func__);
+	}
+
+	vaddr = vmap(pages, npages, VM_MAP, prot);
+	vfree(pages);
+
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+static int eswin_split_heap_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+	void *vaddr;
+	int ret = 0;
+
+	mutex_lock(&buffer->lock);
+	if (buffer->vmap_cnt) {
+		buffer->vmap_cnt++;
+		dma_buf_map_set_vaddr(map, buffer->vaddr);
+		goto out;
+	}
+
+	vaddr = eswin_split_heap_do_vmap(dmabuf);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto out;
+	}
+	buffer->vaddr = vaddr;
+	buffer->vmap_cnt++;
+	dma_buf_map_set_vaddr(map, buffer->vaddr);
+out:
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static void eswin_split_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct eswin_split_buffer *buffer = dmabuf->priv;
+
+	mutex_lock(&buffer->lock);
+	if (!--buffer->vmap_cnt) {
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+	mutex_unlock(&buffer->lock);
+	dma_buf_map_clear(map);
+}
+
+/* split parent dmabuf, and generate a new child sgt */
+static int eswin_get_split_dmabuf(struct eswin_split_buffer *split_buffer)
+{
+	struct dma_buf *par_dma_buf = split_buffer->dmabuf;
+	u64 offset = split_buffer->slice.offset;
+	size_t size = split_buffer->slice.len;
+	 struct sg_table *splitted_sgt = &split_buffer->sg_table;
+	 struct sg_table *orig_splitted_sgt = &split_buffer->orig_sg_table;
+	struct dma_buf_attachment *attach;
+	struct dma_heap_attachment *a;
+	struct dma_heap_attachment_cma *cma_a;
+	struct sg_table *par_origin_table;
+	int out_nents[1];
+	struct scatterlist *sg;
+	struct scatterlist *split_sg;
+	size_t size_remaining, len;
+	int i;
+	int ret = 0;
+
+	if (split_dmabuf_dev == NULL) {
+		pr_err("split_dmabuf_dev has not been created!!!\n");
+		return -EINVAL;
+	}
+
+	get_dma_buf(par_dma_buf);
+	attach = dma_buf_attach(par_dma_buf, split_dmabuf_dev);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+		dma_buf_put(par_dma_buf);
+		return ret;
+	}
+
+	if (unlikely(!strcmp("linux,cma", par_dma_buf->exp_name)))
+	{
+		cma_a = (struct dma_heap_attachment_cma *)attach->priv;
+		par_origin_table = &cma_a->table;
+	}
+	else {
+		a = (struct dma_heap_attachment *)attach->priv;
+		par_origin_table = a->table;
+	}
+
+	#ifdef PRINT_ORIGINAL_SPLITTERS
+	{
+		pr_debug("%s:parent[0x%px]:sgt->orig_nents=%d, nents=%d, sg_nents(in) %d\n",
+			__func__, par_dma_buf,
+			par_origin_table->orig_nents, par_origin_table->nents,
+			sg_nents(par_origin_table->sgl));
+		for_each_sg(par_origin_table->sgl, sg, par_origin_table->orig_nents, i) {
+			pr_debug("parent[0x%px]:orig[%d]:sg->offset=0x%x ,sg->length=0x%x, sg_dma_len=0x%x, sg_phys=0x%lx\n",
+				par_dma_buf, i, sg->offset, sg->length, sg_dma_len(sg), (unsigned long)sg_phys(sg));
+		}
+	}
+	#endif
+
+	/* split unmaped/original sgt of parent */
+	ret = sg_split(par_origin_table->sgl, 0, offset, 1, &size,
+					&split_sg, &out_nents[0], GFP_KERNEL);
+	if (ret) {
+		pr_err("Failed to split from parents's sgt\n");
+	}
+
+	orig_splitted_sgt->orig_nents = out_nents[0];
+	orig_splitted_sgt->nents = out_nents[0];
+	orig_splitted_sgt->sgl = split_sg;
+
+	splitted_sgt->orig_nents = out_nents[0];
+	splitted_sgt->nents = out_nents[0];
+	/* skip the first sgl if it's length is 0 */
+	if (split_sg->length == 0) {
+		splitted_sgt->sgl = sg_next(split_sg);
+		splitted_sgt->orig_nents--;
+		splitted_sgt->nents--;
+	}
+	else {
+		splitted_sgt->sgl = split_sg;
+	}
+	/* Re-format the splitted sg list in the actual slice len */
+	pr_debug("%s:%d, split[0x%px]:out_nents[0]=%d, Re-format the splitted sgl.\n", __func__, __LINE__, par_dma_buf, out_nents[0]);
+	size_remaining = size;
+
+	for_each_sgtable_sg(splitted_sgt, sg, i) {
+		len = min_t(size_t, size_remaining, sg->length);
+		size_remaining -= len;
+		if (size_remaining == 0) {
+			sg->length = len;
+			sg_mark_end(sg);
+			pr_debug("split[0x%px]:orig[%d]:sg->offset=0x%x ,sg->length=0x%x, sg_dma_len=0x%x, sg_phys=0x%lx\n",
+				par_dma_buf, i, sg->offset, sg->length, sg_dma_len(sg), (unsigned long)sg_phys(sg));
+			break;
+		}
+			pr_debug("split[0x%px]:orig[%d]:sg->offset=0x%x ,sg->length=0x%x, sg_dma_len=0x%x, sg_phys=0x%lx\n",
+				par_dma_buf, i, sg->offset, sg->length, sg_dma_len(sg), (unsigned long)sg_phys(sg));
+	}
+	splitted_sgt->orig_nents = splitted_sgt->nents = i + 1;
+
+	pr_debug("%s:split[0x%px]reformatted, splitted:sgt->orig_nents=%d, nents=%d, out_nents[0]=%d\n",
+		__func__, par_dma_buf, splitted_sgt->orig_nents, splitted_sgt->nents, out_nents[0]);
+	/* parent's dupped sgt table is useless*/
+	dma_buf_detach(par_dma_buf, attach);
+	dma_buf_put(par_dma_buf);
+
+	return ret;
+}
+
+static void eswin_put_split_dmabuf(struct sg_table *splitted_sgt)
+{
+	kfree(splitted_sgt->sgl);
+}
+
+static void eswin_split_dma_buf_release(struct dma_buf *dmabuf)
+{
+	struct eswin_split_buffer *split_buffer = dmabuf->priv;
+	struct sg_table *table = &split_buffer->orig_sg_table;;
+
+	pr_debug("%s %d\n", __func__, __LINE__);
+	/* free splitted sgt->sgl which was allocated during esw_common_dmabuf_split_export()*/
+	eswin_put_split_dmabuf(table);
+
+	/* put parent's dmabuf which was got during esw_common_dmabuf_split_export()*/
+	dma_buf_put(split_buffer->dmabuf);
+
+	/* free split buffer which was allocated during esw_common_dmabuf_split_export()*/
+	kfree(split_buffer);
+}
+
+static const struct dma_buf_ops eswin_common_buf_ops = {
+	.attach = eswin_split_heap_attach,
+	.detach = eswin_split_heap_detach,
+	.map_dma_buf = eswin_split_map_dma_buf,
+	.unmap_dma_buf = eswin_spilt_unmap_dma_buf,
+	.begin_cpu_access = eswin_split_dma_buf_begin_cpu_access,
+	.end_cpu_access = eswin_split_dma_buf_end_cpu_access,
+	.mmap = eswin_split_mmap,
+	.vmap = eswin_split_heap_vmap,
+	.vunmap = eswin_split_heap_vunmap,
+	.release = eswin_split_dma_buf_release,
+};
+
+
+int esw_common_dmabuf_split_export(int par_dmabuf_fd, unsigned int offset, size_t len, int fd_flags, char *name)
+{
+	struct eswin_split_buffer *split_buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct sg_table *table;
+	int fd;
+	struct dma_buf *dmabuf;
+	int ret = 0;
+
+	split_buffer = kzalloc(sizeof(*split_buffer), GFP_KERNEL);
+	if (!split_buffer)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&split_buffer->attachments);
+	mutex_init(&split_buffer->lock);
+
+	split_buffer->dbuf_fd = par_dmabuf_fd;
+	split_buffer->fd_flags = fd_flags;
+	split_buffer->slice.offset = offset;
+	split_buffer->slice.len = len;
+	snprintf(split_buffer->name, sizeof(split_buffer->name), "%s", name);
+
+	split_buffer->dmabuf = dma_buf_get(split_buffer->dbuf_fd);
+	if (IS_ERR(split_buffer->dmabuf)) {
+		kfree(split_buffer);
+		return PTR_ERR(split_buffer->dmabuf);
+	}
+
+	pr_debug("%s:%d, par_dbuf_fd %d, input slice: oft=0x%llx, len=0x%lx\n", __func__, __LINE__,
+		split_buffer->dbuf_fd, split_buffer->slice.offset, split_buffer->slice.len);
+
+	split_buffer->slice.offset = PAGE_ALIGN(split_buffer->slice.offset);
+	split_buffer->slice.len = PAGE_ALIGN(split_buffer->slice.len);
+
+
+	table = &split_buffer->sg_table;
+	ret = eswin_get_split_dmabuf(split_buffer);
+	if (ret) {
+		/* put back parent's dmabuf*/
+		dma_buf_put(split_buffer->dmabuf);
+		/* free split buffer */
+		kfree(split_buffer);
+		return ret;
+	}
+	/* create the dmabuf */
+	exp_info.exp_name = split_buffer->name;
+	exp_info.ops = &eswin_common_buf_ops;
+	exp_info.size = split_buffer->slice.len;
+	exp_info.flags = O_RDWR | O_CLOEXEC;
+	exp_info.priv = split_buffer;
+
+	dmabuf = dma_buf_export(&exp_info);
+	if (IS_ERR(dmabuf)) {
+		eswin_put_split_dmabuf(table);
+		/* put back parent's dmabuf*/
+		dma_buf_put(split_buffer->dmabuf);
+		/* free split buffer */
+		kfree(split_buffer);
+		return PTR_ERR(dmabuf);
+	}
+
+	fd = dma_buf_fd(dmabuf, split_buffer->fd_flags);
+	if (fd < 0) {
+		dma_buf_put(dmabuf);
+		/* put the splitted dmabuf, the eswin_split_dma_buf_release will be called,
+		   the parent dmabuf will be put and the split_buffer will be free at that time */
+	}
+	return fd;
+}
+
+EXPORT_SYMBOL(esw_common_dmabuf_split_export);
+
+/* create a misc device for attaching the parent dmabuf to get the parent's original sgt*/
+#define DRIVER_NAME 	"split_dmabuf"
+
+static int split_dmabuf_open(struct inode *inode, struct file *file)
+{
+
+	pr_debug("%s:%d, success!\n", __func__, __LINE__);
+
+	return 0;
+}
+
+static int split_dmabuf_release(struct inode *inode, struct file *file)
+{
+	pr_debug("%s:%d, success!\n", __func__, __LINE__);
+
+	return 0;
+}
+
+static struct file_operations split_dmabuf_fops = {
+	.owner        = THIS_MODULE,
+	.llseek        = no_llseek,
+	.open        = split_dmabuf_open,
+	.release    = split_dmabuf_release,
+};
+
+static struct miscdevice split_dmabuf_miscdev = {
+	.minor	= MISC_DYNAMIC_MINOR,
+	.name	= DRIVER_NAME,
+	.fops	= &split_dmabuf_fops,
+};
+
+static int __init split_dmabuf_init(void)
+{
+	int ret = 0;
+	struct device *dev;
+
+	ret = misc_register(&split_dmabuf_miscdev);
+	if(ret) {
+		pr_err ("cannot register miscdev (err=%d)\n", ret);
+		return ret;
+	}
+	split_dmabuf_dev = split_dmabuf_miscdev.this_device;
+	dev = split_dmabuf_dev;
+	if (!dev->dma_mask) {
+		dev->dma_mask = &dev->coherent_dma_mask;
+	}
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret) {
+		pr_err("Unable to set coherent mask\n");
+		goto deregister_dev;
+	}
+
+	return 0;
+
+deregister_dev:
+	misc_deregister(&split_dmabuf_miscdev);
+	split_dmabuf_dev = NULL;
+	return ret;
+}
+
+static void __exit split_dmabuf_exit(void)
+{
+	misc_deregister(&split_dmabuf_miscdev);
+}
+
+module_init(split_dmabuf_init);
+module_exit(split_dmabuf_exit);
+
+
+static int vmf_replace_pages(struct vm_area_struct *vma, unsigned long addr,
+				struct page **pages, unsigned long num, pgprot_t prot)
+{
+	struct mm_struct *const mm = vma->vm_mm;
+	unsigned long remaining_pages_total = num;
+	unsigned long pfn;
+	pte_t *pte, entry;
+	spinlock_t *ptl;
+	unsigned long newprot_val = pgprot_val(prot);
+	pgprot_t new_prot;
+	u32 i = 0;
+
+	while (remaining_pages_total) {
+		pte = get_locked_pte(mm, addr, &ptl);
+		if (!pte)
+			return VM_FAULT_OOM;
+
+		entry = ptep_get(pte);
+		pfn = page_to_pfn(pages[i]);
+		pr_debug("page_to_pfn(pages[%d])=0x%lx, pte_pfn(entry)=0x%lx, pte_val(entry)=0x%lx\n",
+			i, pfn, pte_pfn(entry), pte_val(entry));
+
+		newprot_val = (pte_val(entry) & (~_PAGE_PFN_MASK)) | newprot_val;
+		if (newprot_val == (pte_val(entry) & (~_PAGE_PFN_MASK)))
+			goto SKIP_PAGE;
+
+		new_prot = __pgprot(newprot_val);
+		entry = mk_pte(pages[i], new_prot);
+		pr_debug("page_to_pfn(pages[%d])=0x%lx, pte_pfn(entry)=0x%lx, modified pte_val(entry)=0x%lx\n",
+			i, page_to_pfn(pages[i]), pte_pfn(entry), pte_val(entry));
+		set_pte_at(vma->vm_mm, addr, pte, entry);
+		update_mmu_cache(vma, addr, pte);
+
+SKIP_PAGE:
+		addr += PAGE_SIZE;
+		pte_unmap_unlock(pte, ptl);
+		remaining_pages_total--;
+		i++;
+	}
+
+	return 0;
+}
+
+static int zap_and_replace_pages(struct vm_area_struct *vma, u64 addr, size_t len, pgprot_t prot)
+{
+	struct page **pages;
+	u32 offset;
+	unsigned long nr_pages;
+	u64 first, last;
+	u64 addr_aligned = ALIGN_DOWN(addr, PAGE_SIZE);
+	u32 i;
+	int ret = -ENOMEM;
+
+	if (!len) {
+		pr_err("invalid userptr size.\n");
+		return -EINVAL;
+	}
+	/* offset into first page */
+	offset = offset_in_page(addr);
+
+	/* Calculate number of pages */
+	first = (addr & PAGE_MASK) >> PAGE_SHIFT;
+	last  = ((addr + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
+	nr_pages = last - first + 1;
+	pr_debug("%s:%d, addr=0x%llx(addr_aligned=0x%llx), len=0x%lx, nr_pages=0x%lx(fist:0x%llx,last:0x%llx)\n",
+		__func__, __LINE__, addr, addr_aligned, len, nr_pages, first, last);
+
+	/* alloc array to storing the pages */
+	pages = kvmalloc_array(nr_pages,
+				   sizeof(struct page *),
+				   GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	ret = get_user_pages_fast(addr_aligned,
+				nr_pages,
+				FOLL_FORCE | FOLL_WRITE,
+				pages);
+
+	if (ret != nr_pages) {
+		nr_pages = (ret >= 0) ? ret : 0;
+		pr_err("get_user_pages_fast, err=%d [0x%lx]\n",
+			ret, nr_pages);
+		ret = ret < 0 ? ret : -EINVAL;
+		goto free_pages_list;
+	}
+	#if 0
+	for (i = 0; i < nr_pages; i++) {
+		pr_debug("page_to_pfn(pages[%i])=0x%lx\n", i, page_to_pfn(pages[i]));
+	}
+	#endif
+
+	pr_debug("%s, vma->vm_start 0x%lx, vma->vm_end 0x%lx, (vm_end - vm_start) 0x%lx, vma->vm_pgoff 0x%lx, user_va 0x%llx, len 0x%lx, nr_pages 0x%lx\n",
+		__func__, vma->vm_start, vma->vm_end,
+		(vma->vm_end - vma->vm_start), vma->vm_pgoff, addr, len, nr_pages);
+
+	/* construct new page table entry for the pages*/
+	ret = vmf_replace_pages(vma, addr_aligned,
+				pages, nr_pages, prot);
+	if (ret) {
+		pr_err("err %d, failed to vmf_replace_pages!!!\n", ret);
+		ret = -EFAULT;
+		goto free_user_pages;
+	}
+
+	/* Flush cache if the access to the user virtual address is uncached. */
+	if (pgprot_val(prot) & _PAGE_UNCACHE) {
+		for (i = 0; i < nr_pages; i++) {
+			/* flush cache*/
+			arch_sync_dma_for_device(page_to_phys(pages[i]), PAGE_SIZE, DMA_BIDIRECTIONAL);
+			/* put page back */
+			put_page(pages[i]);
+		}
+	}
+	else {
+		for (i = 0; i < nr_pages; i++) {
+			/* put page back */
+			put_page(pages[i]);
+		}
+	}
+
+	kvfree(pages);
+
+	return 0;
+
+free_user_pages:
+	for (i = 0; i < nr_pages; i++) {
+		/* put page back */
+		put_page(pages[i]);
+	}
+free_pages_list:
+	kvfree(pages);
+
+	return ret;
+}
+
+/**
+ * remap_malloc_buf - remap a range of memory allocated by malloc() API from user space.
+ * Normally, the CPU access to the user virtual address which is allocated by mallc() API is
+ * through cache. This remap_malloc_buf() API is to re-construct the pte table entry for the
+ * corresponding pages of the user virtual address as uncached memory, so that CPU access to
+ * the virtual address is uncached.
+ * @addr: virtual address which is got by malloc API from user space
+ * @len: the length of the memory allocated by malloc API
+ * @uncaced: if true, remap the memory as uncached, otherwise cached
+ *
+ * Return 0 if success.
+ *
+ * If uncached flag is true, the memory range of this virtual address will be flushed to make
+ * sure all the dirty data is evicted.
+ *
+ */
+int remap_malloc_buf(unsigned long addr, size_t len, bool uncaced)
+{
+	struct vm_area_struct *vma = NULL;
+	struct mm_struct *mm = current->mm;
+	pgprot_t prot;
+	int ret = 0;
+
+	if (!len) {
+		pr_err("Invalid userptr size!!!\n");
+		return -EINVAL;
+	}
+
+	mmap_read_lock(mm);
+	vma = vma_lookup(mm, addr);
+	if (!vma) {
+		pr_err("%s, vma_lookup failed!\n", __func__);
+		mmap_read_unlock(mm);
+		return -EFAULT;
+	}
+
+	pgprot_val(prot) = 0;
+	/* If true, add uncached property so that pfn_pte will use the pfn of system port to
+	   constructs the page table entry.
+	   Be carefull, do NOT change the value of the original vma->vm_page_prot*/
+	if (uncaced)
+		prot = pgprot_dmacoherent(prot);
+
+	ret = zap_and_replace_pages(vma, addr, len, prot);
+	mmap_read_unlock(mm);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(remap_malloc_buf);
\ No newline at end of file
diff --git a/drivers/memory/eswin/es_rsvmem_heap/eswin_rsvmem_common.c b/drivers/memory/eswin/es_rsvmem_heap/eswin_rsvmem_common.c
new file mode 100644
index 000000000000..067f30627a46
--- /dev/null
+++ b/drivers/memory/eswin/es_rsvmem_heap/eswin_rsvmem_common.c
@@ -0,0 +1,447 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ESWIN heap APIs
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#include <linux/cdev.h>
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/err.h>
+#include <linux/xarray.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/nospec.h>
+#include <linux/uaccess.h>
+#include <linux/syscalls.h>
+#include <linux/eswin_rsvmem_common.h>
+#include "include/uapi/linux/eswin_rsvmem_common.h"
+
+#define DEVNAME "eswin_heap"
+
+#define NUM_HEAP_MINORS 128
+
+/**
+ * struct eswin_heap - represents a dmabuf heap in the system
+ * @name:		used for debugging/device-node name
+ * @ops:		ops struct for this heap
+ * @heap_devt		heap device node
+ * @list		list head connecting to list of heaps
+ * @heap_cdev		heap char device
+ *
+ * Represents a heap of memory from which buffers can be made.
+ */
+struct eswin_heap {
+	const char *name;
+	const struct eswin_heap_ops *ops;
+	void *priv;
+	dev_t heap_devt;
+	struct list_head list;
+	struct cdev heap_cdev;
+};
+
+static LIST_HEAD(heap_list);
+static DEFINE_MUTEX(heap_list_lock);
+static dev_t eswin_heap_devt;
+static struct class *eswin_heap_class;
+static DEFINE_XARRAY_ALLOC(eswin_heap_minors);
+
+static int eswin_heap_buffer_alloc(struct eswin_heap *heap, size_t len,
+				 unsigned int fd_flags,
+				 unsigned int heap_flags)
+{
+	struct dma_buf *dmabuf;
+	int fd;
+
+	/*
+	 * Allocations from all heaps have to begin
+	 * and end on page boundaries.
+	 */
+	len = PAGE_ALIGN(len);
+	if (!len)
+		return -EINVAL;
+
+	dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags);
+	if (IS_ERR(dmabuf))
+		return PTR_ERR(dmabuf);
+
+	fd = dma_buf_fd(dmabuf, fd_flags);
+	if (fd < 0) {
+		dma_buf_put(dmabuf);
+		/* just return, as put will call release and that will free */
+	}
+	return fd;
+}
+
+static int eswin_heap_open(struct inode *inode, struct file *file)
+{
+	struct eswin_heap *heap;
+
+	heap = xa_load(&eswin_heap_minors, iminor(inode));
+	if (!heap) {
+		pr_err("eswin_heap: minor %d unknown.\n", iminor(inode));
+		return -ENODEV;
+	}
+
+	/* instance data as context */
+	file->private_data = heap;
+	nonseekable_open(inode, file);
+
+	return 0;
+}
+
+static long eswin_heap_ioctl_allocate(struct file *file, void *data)
+{
+	struct eswin_heap_allocation_data *heap_allocation = data;
+	struct eswin_heap *heap = file->private_data;
+	int fd;
+
+	if (heap_allocation->fd)
+		return -EINVAL;
+
+	if (heap_allocation->fd_flags & ~ESWIN_HEAP_VALID_FD_FLAGS)
+		return -EINVAL;
+
+	if (heap_allocation->heap_flags & ~ESWIN_HEAP_VALID_HEAP_FLAGS)
+		return -EINVAL;
+
+	fd = eswin_heap_buffer_alloc(heap, heap_allocation->len,
+				   heap_allocation->fd_flags,
+				   heap_allocation->heap_flags);
+	if (fd < 0)
+		return fd;
+
+	heap_allocation->fd = fd;
+
+	return 0;
+}
+
+static unsigned int eswin_heap_ioctl_cmds[] = {
+	ESWIN_HEAP_IOCTL_ALLOC,
+};
+
+static long eswin_heap_ioctl(struct file *file, unsigned int ucmd,
+			   unsigned long arg)
+{
+	char stack_kdata[128];
+	char *kdata = stack_kdata;
+	unsigned int kcmd;
+	unsigned int in_size, out_size, drv_size, ksize;
+	int nr = _IOC_NR(ucmd);
+	int ret = 0;
+
+	if (nr >= ARRAY_SIZE(eswin_heap_ioctl_cmds))
+		return -EINVAL;
+
+	nr = array_index_nospec(nr, ARRAY_SIZE(eswin_heap_ioctl_cmds));
+	/* Get the kernel ioctl cmd that matches */
+	kcmd = eswin_heap_ioctl_cmds[nr];
+
+	/* Figure out the delta between user cmd size and kernel cmd size */
+	drv_size = _IOC_SIZE(kcmd);
+	out_size = _IOC_SIZE(ucmd);
+	in_size = out_size;
+	if ((ucmd & kcmd & IOC_IN) == 0)
+		in_size = 0;
+	if ((ucmd & kcmd & IOC_OUT) == 0)
+		out_size = 0;
+	ksize = max(max(in_size, out_size), drv_size);
+
+	/* If necessary, allocate buffer for ioctl argument */
+	if (ksize > sizeof(stack_kdata)) {
+		kdata = kmalloc(ksize, GFP_KERNEL);
+		if (!kdata)
+			return -ENOMEM;
+	}
+
+	if (copy_from_user(kdata, (void __user *)arg, in_size) != 0) {
+		ret = -EFAULT;
+		goto err;
+	}
+
+	/* zero out any difference between the kernel/user structure size */
+	if (ksize > in_size)
+		memset(kdata + in_size, 0, ksize - in_size);
+
+	switch (kcmd) {
+	case ESWIN_HEAP_IOCTL_ALLOC:
+		ret = eswin_heap_ioctl_allocate(file, kdata);
+		break;
+	default:
+		ret = -ENOTTY;
+		goto err;
+	}
+
+	if (copy_to_user((void __user *)arg, kdata, out_size) != 0)
+		ret = -EFAULT;
+err:
+	if (kdata != stack_kdata)
+		kfree(kdata);
+	return ret;
+}
+
+static const struct file_operations eswin_heap_fops = {
+	.owner          = THIS_MODULE,
+	.open		= eswin_heap_open,
+	.unlocked_ioctl = eswin_heap_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= eswin_heap_ioctl,
+#endif
+};
+
+/**
+ * eswin_heap_get_drvdata() - get per-subdriver data for the heap
+ * @heap: DMA-Heap to retrieve private data for
+ *
+ * Returns:
+ * The per-subdriver data for the heap.
+ */
+void *eswin_heap_get_drvdata(struct eswin_heap *heap)
+{
+	return heap->priv;
+}
+
+/**
+ * eswin_heap_get_name() - get heap name
+ * @heap: DMA-Heap to retrieve private data for
+ *
+ * Returns:
+ * The char* for the heap name.
+ */
+const char *eswin_heap_get_name(struct eswin_heap *heap)
+{
+	return heap->name;
+}
+
+struct eswin_heap *eswin_heap_add(const struct eswin_heap_export_info *exp_info)
+{
+	struct eswin_heap *heap, *h = NULL, *err_ret;
+	struct device *dev_ret;
+	unsigned int minor;
+	int ret;
+
+	if (!exp_info->name || !strcmp(exp_info->name, "")) {
+		pr_err("eswin_heap: Cannot add heap without a name\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (!exp_info->ops || !exp_info->ops->allocate) {
+		pr_err("eswin_heap: Cannot add heap with invalid ops struct\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* check the name is unique */
+	mutex_lock(&heap_list_lock);
+	list_for_each_entry(h, &heap_list, list) {
+		if (!strcmp(h->name, exp_info->name)) {
+			mutex_unlock(&heap_list_lock);
+			pr_err("eswin_heap: Already registered heap named %s\n",
+			       exp_info->name);
+			return ERR_PTR(-EINVAL);
+		}
+	}
+	mutex_unlock(&heap_list_lock);
+
+	heap = kzalloc(sizeof(*heap), GFP_KERNEL);
+	if (!heap)
+		return ERR_PTR(-ENOMEM);
+
+	heap->name = exp_info->name;
+	heap->ops = exp_info->ops;
+	heap->priv = exp_info->priv;
+
+	/* Find unused minor number */
+	ret = xa_alloc(&eswin_heap_minors, &minor, heap,
+		       XA_LIMIT(0, NUM_HEAP_MINORS - 1), GFP_KERNEL);
+	if (ret < 0) {
+		pr_err("eswin_heap: Unable to get minor number for heap\n");
+		err_ret = ERR_PTR(ret);
+		goto err0;
+	}
+
+	/* Create device */
+	heap->heap_devt = MKDEV(MAJOR(eswin_heap_devt), minor);
+
+	cdev_init(&heap->heap_cdev, &eswin_heap_fops);
+	ret = cdev_add(&heap->heap_cdev, heap->heap_devt, 1);
+	if (ret < 0) {
+		pr_err("eswin_heap: Unable to add char device\n");
+		err_ret = ERR_PTR(ret);
+		goto err1;
+	}
+
+	dev_ret = device_create(eswin_heap_class,
+				NULL,
+				heap->heap_devt,
+				NULL,
+				heap->name);
+	if (IS_ERR(dev_ret)) {
+		pr_err("eswin_heap: Unable to create device\n");
+		err_ret = ERR_CAST(dev_ret);
+		goto err2;
+	}
+	/* Add heap to the list */
+	mutex_lock(&heap_list_lock);
+	list_add(&heap->list, &heap_list);
+	mutex_unlock(&heap_list_lock);
+
+	return heap;
+
+err2:
+	cdev_del(&heap->heap_cdev);
+err1:
+	xa_erase(&eswin_heap_minors, minor);
+err0:
+	kfree(heap);
+	return err_ret;
+}
+
+int eswin_heap_delete(struct eswin_heap *heap)
+{
+	struct eswin_heap *h, *tmp;
+	int ret = -1;
+
+	if (!heap->name || !strcmp(heap->name, "")) {
+		pr_err("eswin_heap: Cannot delet heap without a name\n");
+		return -EINVAL;
+	}
+
+	/* find the heaplist by the heap name */
+	mutex_lock(&heap_list_lock);
+	list_for_each_entry_safe(h, tmp, &heap_list, list) {
+		if (!strcmp(h->name, heap->name)) {
+			pr_info("eswin_heap: deleted heap %s\n",
+			       heap->name);
+			device_destroy(eswin_heap_class, h->heap_devt);
+			cdev_del(&h->heap_cdev);
+			xa_erase(&eswin_heap_minors, MINOR(h->heap_devt));
+			list_del(&h->list);
+			kfree(h);
+			ret = 0;
+		}
+	}
+	mutex_unlock(&heap_list_lock);
+
+	if (ret) {
+		pr_err("eswin_heap: heap named %s NOT found!\n", heap->name);
+	}
+
+	return ret;
+
+}
+
+int eswin_heap_delete_by_name(const char *name)
+{
+	struct eswin_heap *h, *tmp;
+	int ret = -1;
+
+	if (!name || !strcmp(name, "")) {
+		pr_err("eswin_heap: Cannot delet heap without a name\n");
+		return -EINVAL;
+	}
+
+	/* find the heaplist by the heap name */
+	mutex_lock(&heap_list_lock);
+	list_for_each_entry_safe(h, tmp, &heap_list, list) {
+		if (!strcmp(h->name, name)) {
+			pr_info("eswin_heap: deleted heap %s\n",
+			       name);
+			device_destroy(eswin_heap_class, h->heap_devt);
+			cdev_del(&h->heap_cdev);
+			xa_erase(&eswin_heap_minors, MINOR(h->heap_devt));
+			list_del(&h->list);
+			kfree(h);
+			ret = 0;
+		}
+	}
+	mutex_unlock(&heap_list_lock);
+
+	if (ret) {
+		pr_err("eswin_heap: heap named %s NOT found!\n", name);
+	}
+
+	return ret;
+}
+
+int eswin_heap_kalloc(char *name, size_t len, unsigned int fd_flags, unsigned int heap_flags)
+{
+	struct eswin_heap *heap = NULL;
+#if 0
+	struct eswin_heap *h = NULL;
+	/* check the name is unique */
+	mutex_lock(&heap_list_lock);
+	list_for_each_entry(h, &heap_list, list) {
+		if (!strcmp(h->name, name)) {
+			heap = h;
+			break;
+		}
+	}
+	mutex_unlock(&heap_list_lock);
+#else
+	char *dev_path = NULL;
+	struct file *file;
+	int ret;
+
+	dev_path = kasprintf(GFP_KERNEL, "/dev/dma_heap/%s", name);
+	file = filp_open(dev_path, O_RDWR, 0);
+	if (IS_ERR(file)) {
+		ret = PTR_ERR(file);
+		pr_err("failed to open file %s: (%d)\n",
+			dev_path, ret);
+		return ret;
+	}
+	heap = file->private_data;
+#endif
+	if (!heap) {
+		printk("ERROR: Can't find this heap %s\n", name);
+		return -ENODEV;
+	}
+
+	return eswin_heap_buffer_alloc(heap, len, fd_flags, heap_flags);
+}
+EXPORT_SYMBOL(eswin_heap_kalloc);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0)
+static char *eswin_heap_devnode(const struct device *dev, umode_t *mode)
+#else
+static char *eswin_heap_devnode(struct device *dev, umode_t *mode)
+#endif
+
+{
+	// return kasprintf(GFP_KERNEL, "eswin_heap/%s", dev_name(dev));
+	/* create device node under dma_heap instead of eswin_heap, so that memory lib can
+	   avoid the diverseness.
+	*/
+	return kasprintf(GFP_KERNEL, "dma_heap/%s", dev_name(dev));
+}
+
+int eswin_heap_init(void)
+{
+	int ret;
+
+	ret = alloc_chrdev_region(&eswin_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME);
+	if (ret)
+		return ret;
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0)
+	eswin_heap_class = class_create(DEVNAME);
+	#else
+	eswin_heap_class = class_create(THIS_MODULE, DEVNAME);
+	#endif
+	if (IS_ERR(eswin_heap_class)) {
+		unregister_chrdev_region(eswin_heap_devt, NUM_HEAP_MINORS);
+		return PTR_ERR(eswin_heap_class);
+	}
+	eswin_heap_class->devnode = eswin_heap_devnode;
+
+	return 0;
+}
+
+void eswin_heap_uninit(void)
+{
+	class_destroy(eswin_heap_class);
+}
diff --git a/drivers/memory/eswin/es_rsvmem_heap/eswin_rsvmem_heap.c b/drivers/memory/eswin/es_rsvmem_heap/eswin_rsvmem_heap.c
new file mode 100644
index 000000000000..804258f5ec74
--- /dev/null
+++ b/drivers/memory/eswin/es_rsvmem_heap/eswin_rsvmem_heap.c
@@ -0,0 +1,634 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ESWIN rerserved memory heap.
+ * eswin_rsvmem_heap creates heap for the reserved memory that has compatible = "eswin-reserve-memory";
+ * property and no-map property.
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/dma-map-ops.h>
+#include <linux/eswin_rsvmem_common.h>
+#include "../eswin_memblock.h"
+#include "../es_buddy/es_buddy.h"
+#include "include/uapi/linux/eswin_rsvmem_common.h"
+
+static const unsigned int orders[] = {MAX_ORDER-1, 9, 0};
+#define NUM_ORDERS ARRAY_SIZE(orders)
+
+struct eswin_rsvmem_heap {
+	struct eswin_heap *heap;
+	struct mem_block *memblock;
+};
+
+struct eswin_rsvmem_heap_buffer {
+	struct eswin_rsvmem_heap *heap;
+	struct list_head attachments;
+	struct mutex lock;
+	unsigned long len;
+	struct sg_table sg_table; // for buddy allocator
+	struct page **pages;
+	int vmap_cnt;
+	void *vaddr;
+	unsigned long fd_flags; // for vmap to determin the cache or non-cached mapping
+};
+
+struct eswin_heap_attachment {
+	struct device *dev;
+	struct sg_table *table;
+	struct list_head list;
+	bool mapped;
+};
+
+static struct sg_table *dup_sg_table(struct sg_table *table)
+{
+	struct sg_table *new_table;
+	int ret, i;
+	struct scatterlist *sg, *new_sg;
+
+	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
+	if (!new_table)
+		return ERR_PTR(-ENOMEM);
+
+	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
+	if (ret) {
+		kfree(new_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_sg = new_table->sgl;
+	for_each_sgtable_sg(table, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+	return new_table;
+}
+
+static int eswin_rsvmem_heap_attach(struct dma_buf *dmabuf,
+			   struct dma_buf_attachment *attachment)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct eswin_heap_attachment *a;
+	struct sg_table *table;
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a)
+		return -ENOMEM;
+
+	table = dup_sg_table(&buffer->sg_table);
+	if (IS_ERR(table)) {
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	a->table = table;
+	a->dev = attachment->dev;
+	INIT_LIST_HEAD(&a->list);
+	a->mapped = false;
+
+	attachment->priv = a;
+
+	mutex_lock(&buffer->lock);
+	list_add(&a->list, &buffer->attachments);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static void eswin_rsvmem_heap_detach(struct dma_buf *dmabuf,
+			    struct dma_buf_attachment *attachment)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct eswin_heap_attachment *a = attachment->priv;
+
+	mutex_lock(&buffer->lock);
+	list_del(&a->list);
+	mutex_unlock(&buffer->lock);
+
+	sg_free_table(a->table);
+	kfree(a->table);
+	kfree(a);
+}
+
+static struct sg_table *eswin_rsvmem_heap_map_dma_buf(struct dma_buf_attachment *attachment,
+					     enum dma_data_direction direction)
+{
+	struct eswin_heap_attachment *a = attachment->priv;
+	struct sg_table *table =a->table;
+	int ret;
+	unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
+
+	/* Skipt cache sync, since it takes a lot of time when import to device.
+	*  It's the user's responsibility for guaranteeing the cache coherency by
+	   flusing cache explicitly before importing to device.
+	*/
+	ret = dma_map_sgtable(attachment->dev, table, direction, attrs);
+
+	if (ret)
+		return ERR_PTR(-ENOMEM);
+	a->mapped = true;
+	return table;
+}
+
+static void eswin_rsvmem_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
+				   struct sg_table *table,
+				   enum dma_data_direction direction)
+{
+	struct eswin_heap_attachment *a = attachment->priv;
+	unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
+
+	a->mapped = false;
+
+	/* Skipt cache sync, since it takes a lot of time when unmap from device.
+	*  It's the user's responsibility for guaranteeing the cache coherency after
+	   the device has done processing the data.(For example, CPU do NOT read untill
+	   the device has done)
+	*/
+	dma_unmap_sgtable(attachment->dev, table, direction, attrs);
+}
+
+static int eswin_rsvmem_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+					     enum dma_data_direction direction)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	struct scatterlist *sg;
+	int i;
+
+	mutex_lock(&buffer->lock);
+
+	if (buffer->vmap_cnt)
+		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	/* Since the cache sync was skipped when eswin_rsvmem_heap_map_dma_buf/eswin_rsvmem_heap_unmap_dma_buf,
+	   So force cache sync here when user call ES_SYS_MemFlushCache, eventhough there
+	   is no device attached to this dmabuf.
+	*/
+	#ifndef QEMU_DEBUG
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_cpu(sg_phys(sg), sg->length, direction);
+
+	#endif
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static int eswin_rsvmem_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
+					   enum dma_data_direction direction)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	struct scatterlist *sg;
+	int i;
+
+	mutex_lock(&buffer->lock);
+
+	if (buffer->vmap_cnt)
+		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	/* Since the cache sync was skipped while eswin_rsvmem_heap_map_dma_buf/eswin_rsvmem_heap_unmap_dma_buf,
+	   So force cache sync here when user call ES_SYS_MemFlushCache, eventhough there
+	   is no device attached to this dmabuf.
+	*/
+	#ifndef QEMU_DEBUG
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_device(sg_phys(sg), sg->length, direction);
+	#endif
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+#if 0
+static int eswin_rsvmem_sync_cache_internal(struct dma_buf *dmabuf, enum dma_data_direction direction)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	struct scatterlist *sg;
+	int i;
+
+	for_each_sg(table->sgl, sg, table->orig_nents, i)
+		arch_sync_dma_for_device(sg_phys(sg), sg->length, direction);
+
+
+	return 0;
+}
+#endif
+
+static int eswin_rsvmem_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct eswin_heap *heap = buffer->heap->heap;
+	struct sg_table *table = &buffer->sg_table;
+	unsigned long addr = vma->vm_start;
+	unsigned long pgoff = vma->vm_pgoff, mapsize = 0;
+	unsigned long size_remaining = vma->vm_end - vma->vm_start;//vma_pages(vma);
+	struct scatterlist *sg;
+	struct page *page = NULL;
+	unsigned int nents = 0;
+	int i;
+	int ret;
+	const char *heap_name = NULL;
+
+	/* Mapping secure_memory with cached proprty to user space for CPU is NOT permitted */
+	heap_name = eswin_heap_get_name(heap);
+	if (unlikely(!strncmp("secure_memory", heap_name, 13))) {
+		if (!(vma->vm_flags & VM_NORESERVE))
+			return -EPERM;
+	}
+
+	if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
+		return -EINVAL;
+
+	/* vm_private_data will be used by eswin-ipc-scpu.c.
+	    ipc will import this dmabuf to get iova.
+	*/
+	vma->vm_private_data = dmabuf;
+
+	/* support mman flag MAP_SHARED_VALIDATE | VM_NORESERVE, used to map uncached memory to user space.
+	   Users should guarantee this buffer has been flushed to cache already.
+	 */
+	if (vma->vm_flags & VM_NORESERVE) {
+		vm_flags_clear(vma, VM_NORESERVE);
+		#ifndef QEMU_DEBUG
+		vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+		#endif
+		/* skip sync cache, users should guarantee the cache is clean after done using it in
+		   cached mode(i.e, ES_SYS_Mmap(SYS_CACHE_MODE_CACHED))
+		*/
+	}
+	pr_debug("%s, size_remaining:0x%lx, pgoff:0x%lx, dmabuf->size:0x%lx, start_phys:0x%llx\n",
+		__func__, size_remaining, pgoff, dmabuf->size, sg_phys(table->sgl));
+	for_each_sg(table->sgl, sg, table->orig_nents, i) {
+		pr_debug("sgl:%d, phys:0x%llx\n", i, sg_phys(sg));
+		if (pgoff >= (sg->length >> PAGE_SHIFT)) {
+			pgoff -= (sg->length >> PAGE_SHIFT);
+			continue;
+		}
+
+		page = sg_page(sg);
+		if (nents == 0) {
+			mapsize = sg->length - (pgoff << PAGE_SHIFT);
+			mapsize = min(size_remaining, mapsize);
+			ret = remap_pfn_range(vma, addr, page_to_pfn(page) + pgoff, mapsize,
+					vma->vm_page_prot);
+			pr_debug("nents:%d, sgl:%d, pgoff:0x%lx, mapsize:0x%lx, phys:0x%llx\n",
+				nents, i, pgoff, mapsize, pfn_to_phys(page_to_pfn(page) + pgoff));
+		}
+		else {
+			mapsize = min((unsigned int)size_remaining, (sg->length));
+			ret = remap_pfn_range(vma, addr, page_to_pfn(page), mapsize,
+					vma->vm_page_prot);
+			pr_debug("nents:%d, sgl:%d, mapsize:0x%lx, phys:0x%llx\n", nents, i, mapsize, page_to_phys(page));
+		}
+		pgoff = 0;
+		nents++;
+
+		if (ret)
+			return ret;
+
+		addr += mapsize;
+		size_remaining -= mapsize;
+		if (size_remaining == 0)
+			return 0;
+	}
+
+	return 0;
+}
+
+static void *eswin_rsvmem_heap_do_vmap(struct dma_buf *dmabuf)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	pgprot_t prot = PAGE_KERNEL;
+	struct sg_table *table = &buffer->sg_table;
+	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
+	struct page **pages = vmalloc(sizeof(struct page *) * npages);
+	struct page **tmp = pages;
+	struct sg_page_iter piter;
+	void *vaddr;
+
+	if (!pages)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_sgtable_page(table, &piter, 0) {
+		WARN_ON(tmp - pages >= npages);
+		*tmp++ = sg_page_iter_page(&piter);
+	}
+
+	/* The property of this dmabuf in kernel space is determined by heap alloc with fd_flag. */
+	if (buffer->fd_flags & O_DSYNC) {
+		#ifndef QEMU_DEBUG
+		prot = pgprot_dmacoherent(PAGE_KERNEL);
+		#endif
+		pr_debug("%s syport uncached kernel dmabuf!, prot=0x%x\n", __func__, (unsigned int)pgprot_val(prot));
+	}
+	else {
+		pr_debug("%s memport cached kernel dmabuf!\n", __func__);
+	}
+
+	vaddr = vmap(pages, npages, VM_MAP, prot);
+	vfree(pages);
+
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+static int eswin_rsvmem_heap_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	void *vaddr;
+	int ret = 0;
+
+	mutex_lock(&buffer->lock);
+	if (buffer->vmap_cnt) {
+		buffer->vmap_cnt++;
+		dma_buf_map_set_vaddr(map, buffer->vaddr);
+		goto out;
+	}
+
+	vaddr = eswin_rsvmem_heap_do_vmap(dmabuf);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto out;
+	}
+	buffer->vaddr = vaddr;
+	buffer->vmap_cnt++;
+	dma_buf_map_set_vaddr(map, buffer->vaddr);
+out:
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static void eswin_rsvmem_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+
+	mutex_lock(&buffer->lock);
+	if (!--buffer->vmap_cnt) {
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+	mutex_unlock(&buffer->lock);
+	dma_buf_map_clear(map);
+}
+
+static void eswin_rsvmem_heap_dma_buf_release(struct dma_buf *dmabuf)
+{
+	struct eswin_rsvmem_heap_buffer *buffer = dmabuf->priv;
+	struct sg_table *table;
+	struct scatterlist *sg;
+	int i;
+
+	table = &buffer->sg_table;
+	if (buffer->vmap_cnt > 0) {
+		WARN(1, "%s: buffer still mapped in the kernel\n", __func__);
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+
+	for_each_sgtable_sg(table, sg, i) {
+		struct page *page = sg_page(sg);
+		// pr_debug("%s:%d,page_size(page)=0x%lx, phys_addr=0x%llx\n",
+			// __func__, __LINE__, page_size(page), page_to_phys(page));
+		es_free_pages(buffer->heap->memblock, page);
+	}
+	sg_free_table(table);
+
+	kfree(buffer);
+}
+
+static const struct dma_buf_ops eswin_rsvmem_heap_buf_ops = {
+	.attach = eswin_rsvmem_heap_attach,
+	.detach = eswin_rsvmem_heap_detach,
+	.map_dma_buf = eswin_rsvmem_heap_map_dma_buf,
+	.unmap_dma_buf = eswin_rsvmem_heap_unmap_dma_buf,
+	.begin_cpu_access = eswin_rsvmem_dma_buf_begin_cpu_access,
+	.end_cpu_access = eswin_rsvmem_dma_buf_end_cpu_access,
+	.mmap = eswin_rsvmem_heap_mmap,
+	.vmap = eswin_rsvmem_heap_vmap,
+	.vunmap = eswin_rsvmem_heap_vunmap,
+	.release = eswin_rsvmem_heap_dma_buf_release,
+};
+
+static struct page *alloc_largest_available(struct mem_block *memblock,
+						unsigned long size,
+						unsigned int max_order)
+{
+	struct page *page;
+	int i;
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		if (size <  (PAGE_SIZE << orders[i]))
+			continue;
+		if (max_order < orders[i])
+			continue;
+
+		page = es_alloc_pages(memblock, orders[i]);
+		if (!page)
+			continue;
+		return page;
+	}
+	return NULL;
+}
+
+static struct dma_buf *eswin_rsvmem_heap_allocate(struct eswin_heap *heap,
+					    unsigned long len,
+					    unsigned long fd_flags,
+					    unsigned long heap_flags)
+{
+	struct eswin_rsvmem_heap *rsvmem_heap = eswin_heap_get_drvdata(heap);
+	struct eswin_rsvmem_heap_buffer *buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	unsigned long size_remaining = len;
+	unsigned int max_order = orders[0];
+	struct dma_buf *dmabuf;
+	struct sg_table *table;
+	struct scatterlist *sg;
+	struct list_head pages;
+	struct page *page, *tmp_page;
+	int i, ret = -ENOMEM;
+	const char *heap_name = NULL;
+
+	/* Mapping secure_memory with cached proprty to kernel space for CPU is NOT permitted */
+	heap_name = eswin_heap_get_name(rsvmem_heap->heap);
+	if (unlikely(!strncmp("secure_memory", heap_name, 13))) {
+		if (!(fd_flags & O_DSYNC))
+			return ERR_PTR(-EINVAL);
+	}
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&buffer->attachments);
+	mutex_init(&buffer->lock);
+	buffer->heap = rsvmem_heap;
+	buffer->len = len;
+	buffer->fd_flags = fd_flags;
+
+	INIT_LIST_HEAD(&pages);
+	i = 0;
+	while (size_remaining > 0) {
+		/*
+		 * Avoid trying to allocate memory if the process
+		 * has been killed by SIGKILL
+		 */
+		if (fatal_signal_pending(current)) {
+			ret = -EINTR;
+			goto free_buffer;
+		}
+
+		page = alloc_largest_available(rsvmem_heap->memblock, size_remaining, max_order);
+		if (!page)
+			goto free_buffer;
+
+		list_add_tail(&page->lru, &pages);
+		size_remaining -= page_size(page);
+		max_order = compound_order(page);
+		i++;
+	}
+
+	table = &buffer->sg_table;
+	if (sg_alloc_table(table, i, GFP_KERNEL))
+		goto free_buffer;
+
+	sg = table->sgl;
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		sg_set_page(sg, page, page_size(page), 0);
+		sg = sg_next(sg);
+		list_del(&page->lru);
+	}
+
+	/* create the dmabuf */
+	exp_info.exp_name = eswin_heap_get_name(heap);
+	exp_info.ops = &eswin_rsvmem_heap_buf_ops;
+	exp_info.size = buffer->len;
+	exp_info.flags = O_RDWR | O_CLOEXEC;
+	exp_info.priv = buffer;
+	dmabuf = dma_buf_export(&exp_info);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto free_pages;
+	}
+	return dmabuf;
+
+free_pages:
+	for_each_sgtable_sg(table, sg, i) {
+		struct page *p = sg_page(sg);
+
+		es_free_pages(rsvmem_heap->memblock, p);
+	}
+	sg_free_table(table);
+free_buffer:
+	list_for_each_entry_safe(page, tmp_page, &pages, lru)
+		es_free_pages(rsvmem_heap->memblock, page);
+	kfree(buffer);
+
+	return ERR_PTR(ret);
+}
+
+static const struct eswin_heap_ops eswin_rsvmem_heap_ops = {
+	.allocate = eswin_rsvmem_heap_allocate,
+};
+
+static int __add_eswin_rsvmem_heap(struct mem_block *memblock, void *data)
+{
+	struct eswin_rsvmem_heap *rsvmem_heap;
+	struct eswin_heap_export_info exp_info;
+
+	rsvmem_heap = kzalloc(sizeof(*rsvmem_heap), GFP_KERNEL);
+	if (!rsvmem_heap)
+		return -ENOMEM;
+	rsvmem_heap->memblock = memblock;
+
+	exp_info.name = eswin_rsvmem_get_name(memblock);
+	exp_info.ops = &eswin_rsvmem_heap_ops;
+	exp_info.priv = rsvmem_heap;
+
+	rsvmem_heap->heap = eswin_heap_add(&exp_info);
+	if (IS_ERR(rsvmem_heap->heap)) {
+		int ret = PTR_ERR(rsvmem_heap->heap);
+
+		kfree(rsvmem_heap);
+		return ret;
+	}
+
+	pr_info("%s for %s successfully!\n", __func__, exp_info.name);
+
+	return 0;
+}
+
+static char *es_heap_name_prefix[] = {
+						"mmz_nid_",
+						"secure_memory"
+};
+#define NUM_ESWIN_RSVMEM_HEAPS ARRAY_SIZE(es_heap_name_prefix)
+
+static int do_add_eswin_rsvmem_heap(struct mem_block *memblock, void *data)
+{
+	int ret = 0;
+	char *prefix = data;
+	const char *rsvmem_name = eswin_rsvmem_get_name(memblock);
+
+	if (strncmp(rsvmem_name, prefix, strlen(prefix)) == 0)
+		ret = __add_eswin_rsvmem_heap(memblock, NULL);
+
+	return ret;
+}
+static int add_eswin_rsvmem_heaps(void)
+{
+	int i;
+	int ret;
+
+	ret = eswin_heap_init();
+	if (ret)
+		return ret;
+
+	for (i = 0; i < NUM_ESWIN_RSVMEM_HEAPS; i++) {
+		eswin_rsvmem_for_each_block(do_add_eswin_rsvmem_heap, es_heap_name_prefix[i]);
+	}
+
+	return 0;
+}
+
+static int do_delete_eswin_rsvmem_heap(struct mem_block *memblock, void *data)
+{
+	int ret = 0;
+	char *prefix = data;
+	const char *rsvmem_name = eswin_rsvmem_get_name(memblock);
+
+	if (strncmp(rsvmem_name, prefix, strlen(prefix)) == 0)
+		ret = eswin_heap_delete_by_name(rsvmem_name);
+
+	return ret;
+}
+static void __exit delete_eswin_rsvmem_heaps(void)
+{
+	int i;
+
+	for (i = 0; i < NUM_ESWIN_RSVMEM_HEAPS; i++) {
+		eswin_rsvmem_for_each_block(do_delete_eswin_rsvmem_heap, es_heap_name_prefix[i]);
+	}
+	eswin_heap_uninit();
+}
+module_init(add_eswin_rsvmem_heaps);
+module_exit(delete_eswin_rsvmem_heaps);
+MODULE_IMPORT_NS(DMA_BUF);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/eswin/es_rsvmem_heap/include/uapi/linux/eswin_rsvmem_common.h b/drivers/memory/eswin/es_rsvmem_heap/include/uapi/linux/eswin_rsvmem_common.h
new file mode 100644
index 000000000000..7d863001cfc3
--- /dev/null
+++ b/drivers/memory/eswin/es_rsvmem_heap/include/uapi/linux/eswin_rsvmem_common.h
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ESWIN Heaps Userspace API
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *   Authors:
+ *    LinMin<linmin@eswincomputing.com>
+ *
+ */
+
+#ifndef _UAPI_ESWIN_HEAPS_H
+#define _UAPI_ESWIN_HEAPS_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/**
+ * DOC: DMABUF Heaps Userspace API
+ */
+/* Valid FD_FLAGS are O_CLOEXEC, O_RDONLY, O_WRONLY, O_RDWR, O_SYNC */
+#define ESWIN_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE | O_SYNC)
+
+/* Add HEAP_SPRAM_FORCE_CONTIGUOUS heap flags for ESWIN SPRAM HEAP */
+#define HEAP_FLAGS_SPRAM_FORCE_CONTIGUOUS	(1 << 0)
+#define ESWIN_HEAP_VALID_HEAP_FLAGS (HEAP_FLAGS_SPRAM_FORCE_CONTIGUOUS)
+
+/**
+ * struct eswin_heap_allocation_data - metadata passed from userspace for
+ *                                      allocations
+ * @len:		size of the allocation
+ * @fd:			will be populated with a fd which provides the
+ *			handle to the allocated dma-buf
+ * @fd_flags:		file descriptor flags used when allocating
+ * @heap_flags:		flags passed to heap
+ *
+ * Provided by userspace as an argument to the ioctl
+ */
+struct eswin_heap_allocation_data {
+	__u64 len;
+	__u32 fd;
+	__u32 fd_flags;
+	__u64 heap_flags;
+};
+
+#define ESWIN_HEAP_IOC_MAGIC		'H'
+
+/**
+ * DOC: ESWIN_HEAP_IOCTL_ALLOC - allocate memory from pool
+ *
+ * Takes a eswin_heap_allocation_data struct and returns it with the fd field
+ * populated with the dmabuf handle of the allocation.
+ */
+#define ESWIN_HEAP_IOCTL_ALLOC	_IOWR(ESWIN_HEAP_IOC_MAGIC, 0x0,\
+				      struct eswin_heap_allocation_data)
+
+#endif /* _UAPI_ESWIN_HEAPS_H */
diff --git a/drivers/memory/eswin/eswin_cpuid_hartid_convert.c b/drivers/memory/eswin/eswin_cpuid_hartid_convert.c
new file mode 100644
index 000000000000..71fd4da20a04
--- /dev/null
+++ b/drivers/memory/eswin/eswin_cpuid_hartid_convert.c
@@ -0,0 +1,39 @@
+/*
+ * ESWIN eic770x cpu_nid and hart_id convert
+ * Copyright 2023, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#include <linux/cpu.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/eswin_cpuid_hartid_convert.h>
+
+unsigned long __eswin_cpuid_to_hartid_map[NR_CPUS] = {
+	[0 ... NR_CPUS-1] = INVALID_HARTID
+};
+EXPORT_SYMBOL_GPL(__eswin_cpuid_to_hartid_map);
+
+static int __init eswin_cpuid_hartid_convert_init(void)
+{
+	int i;
+	
+	for (i = 0; i < NR_CPUS; i++)
+		eswin_cpuid_to_hartid_map(i) = cpuid_to_hartid_map(i);
+	
+	return 0;
+}
+arch_initcall(eswin_cpuid_hartid_convert_init);
\ No newline at end of file
diff --git a/drivers/memory/eswin/eswin_memblock.c b/drivers/memory/eswin/eswin_memblock.c
new file mode 100644
index 000000000000..06c8635c6a60
--- /dev/null
+++ b/drivers/memory/eswin/eswin_memblock.c
@@ -0,0 +1,178 @@
+/*
+ * eswin_rsvmem initializes the reserved memory in dst that has compatible = "eswin-reserve-memory" and
+ * no-map property. Each of these memory region will be treated as one memory block and managed by eswin
+ * buddy system. Users can allocate/frree pages from/to these memory blocks via es_alloc_pages/es_free_pages.
+ *
+ * Copyright 2023, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#define pr_fmt(fmt) "eswin_rsvmem: " fmt
+
+#include <linux/memblock.h>
+#include <linux/err.h>
+#include <linux/mm.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/log2.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/kmemleak.h>
+#include "eswin_memblock.h"
+
+
+struct mem_block eswin_rsvmem_blocks[MAX_ESWIN_RSVMEM_AREAS] = {0};
+unsigned eswin_rsvmem_block_count = 0;
+
+phys_addr_t eswin_rsvmem_get_base(const struct mem_block *memblock)
+{
+	return page_to_phys(memblock->kPageStart);
+}
+
+unsigned long eswin_rsvmem_get_size(const struct mem_block *memblock)
+{
+	return memblock->page_num << PAGE_SHIFT;
+}
+EXPORT_SYMBOL(eswin_rsvmem_get_size);
+
+const char *eswin_rsvmem_get_name(const struct mem_block *memblock)
+{
+	return memblock->name;
+}
+EXPORT_SYMBOL(eswin_rsvmem_get_name);
+
+struct mem_block *eswin_rsvmem_get_memblock(const char *memBlkName)
+{
+	struct mem_block *memblock = NULL;
+	int i;
+
+	for (i = 0; i < eswin_rsvmem_block_count; i++) {
+		if ((!strcmp(memBlkName, eswin_rsvmem_blocks[i].name)) &&
+		    (NULL != eswin_rsvmem_blocks[i].esPagesStart)){
+			memblock = &eswin_rsvmem_blocks[i];
+			break;
+		}
+	}
+
+	return memblock;
+}
+EXPORT_SYMBOL(eswin_rsvmem_get_memblock);
+
+/**
+ * eswin_rsvmem_init_reserved_mem() - create eswin reserve memory from reserved memory
+ * @base: Base address of the reserved area
+ * @size: Size of the reserved area (in bytes),
+ * @name: The name of the area.
+ *
+ * This function creates eswin reserve memory block and manage it with eswin buddy allocator.
+ */
+static int __init eswin_rsvmem_init_reserved_mem(phys_addr_t base, phys_addr_t size, const char *name)
+{
+	struct mem_block *memblock;
+	phys_addr_t alignment;
+	char *temp;
+	int bname_len;
+
+	/* Sanity checks */
+	if (eswin_rsvmem_block_count == ARRAY_SIZE(eswin_rsvmem_blocks)) {
+		pr_err("Not enough slots for eswin-reserve-memory!\n");
+		return -ENOSPC;
+	}
+
+	if (!size) {
+		pr_err("Invalid size 0x%llx\n", size);
+		return -EINVAL;
+	}
+
+	/* ensure minimal alignment */
+	alignment = PAGE_SIZE <<
+			max_t(unsigned long, MAX_ORDER - 1, pageblock_order);
+
+	if (ALIGN(base, alignment) != base || ALIGN(size, alignment) != size) {
+		pr_err("Alignment Err! base:0x%llx, size:0x%llx, alignment:0x%llx\n", base, size, alignment);
+		return -EINVAL;
+	}
+
+	if (!name) {
+		pr_err("rsvmem block name is NULL!\n");
+		return -EINVAL;
+	}
+
+	memblock = &eswin_rsvmem_blocks[eswin_rsvmem_block_count];
+	memblock->page_num = size >> PAGE_SHIFT;
+
+	/* Set esPagesStart = NULL here, it will be allocated later by fs_initcall*/
+	memblock->esPagesStart = NULL;
+	memblock->kPageStart = phys_to_page(base);
+
+	snprintf(memblock->name, BLOCK_MAX_NAME, name);
+	temp = strchr(memblock->name, '@');
+	if (temp) {
+		bname_len = strnlen(memblock->name, BLOCK_MAX_NAME) - strnlen(temp,
+								BLOCK_MAX_NAME);
+		*(memblock->name + bname_len) = '\0';
+	}
+
+	eswin_rsvmem_block_count++;
+
+	return 0;
+}
+
+int eswin_rsvmem_for_each_block(int (*it)(struct mem_block *rsvmem_block, void *data), void *data)
+{
+	int i;
+
+	for (i = 0; i < eswin_rsvmem_block_count; i++) {
+		int ret = it(&eswin_rsvmem_blocks[i], data);
+
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(eswin_rsvmem_for_each_block);
+
+#ifdef CONFIG_OF_RESERVED_MEM
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/of_reserved_mem.h>
+
+#undef pr_fmt
+#define pr_fmt(fmt) fmt
+
+static int __init rmem_eswin_setup(struct reserved_mem *rmem)
+{
+	unsigned long node = rmem->fdt_node;
+	int err;
+
+	if (of_get_flat_dt_prop(node, "reusable", NULL) ||
+	    !of_get_flat_dt_prop(node, "no-map", NULL))
+		return -EINVAL;
+
+
+	err = eswin_rsvmem_init_reserved_mem(rmem->base, rmem->size, rmem->name);
+	if (err) {
+		pr_err("Reserved memory: unable to setup eswin reserve region, err=%d\n", err);
+		return err;
+	}
+
+	pr_info("Reserved memory: created %s eswin reserve memory at %pa, size %ld MiB\n",
+		rmem->name, &rmem->base, (unsigned long)rmem->size / SZ_1M);
+
+	return 0;
+}
+RESERVEDMEM_OF_DECLARE(eswin_rsvmem, "eswin-reserve-memory", rmem_eswin_setup);
+#endif
\ No newline at end of file
diff --git a/drivers/memory/eswin/eswin_memblock.h b/drivers/memory/eswin/eswin_memblock.h
new file mode 100644
index 000000000000..6c18444599de
--- /dev/null
+++ b/drivers/memory/eswin/eswin_memblock.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ESWIN_RSVMEM_H__
+#define __ESWIN_RSVMEM_H__
+
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/numa.h>
+#include "buddy.h"
+
+// #define QEMU_DEBUG 1
+
+#define MAX_ESWIN_RSVMEM_AREAS	(64)
+
+struct mem_block *eswin_rsvmem_get_memblock(const char *memBlkName);
+int eswin_rsvmem_for_each_block(int (*it)(struct mem_block *rsvmem_block, void *data), void *data);
+const char *eswin_rsvmem_get_name(const struct mem_block *memblock);
+
+#endif
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index bc7e2ad37002..2fb029a80648 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -150,6 +150,34 @@ config MMC_SDHCI_PLTFM
 
 	  If unsure, say N.
 
+config MMC_SDHCI_OF_ESWIN
+        tristate "SDHCI OF support for the Eswin SDHCI controllers"
+        depends on MMC_SDHCI_PLTFM
+        depends on OF
+        depends on COMMON_CLK
+        select MMC_CQHCI
+        help
+          This selects the Eswin Secure Digital Host Controller Interface
+          (SDHCI).
+
+          If you have a controller with this interface, say Y or M here.
+
+          If unsure, say N.
+
+config MMC_SDHCI_OF_SDIO_ESWIN
+   tristate "SDHCI OF SDIO support for the Eswin SDHCI controllers"
+    depends on MMC_SDHCI_PLTFM
+    depends on OF
+    depends on COMMON_CLK
+    select MMC_CQHCI
+    help
+      This selects the Eswin Secure Digital Host Controller Interface
+      (SDHCI-SDIO).
+
+      If you have a controller with this interface, say Y or M here.
+
+      If unsure, say N.
+
 config MMC_SDHCI_OF_ARASAN
 	tristate "SDHCI OF support for the Arasan SDHCI controllers"
 	depends on MMC_SDHCI_PLTFM
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index a693fa3d3f1c..bcab602d3ea5 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -81,6 +81,10 @@ obj-$(CONFIG_MMC_SDHCI_ESDHC_IMX)	+= sdhci-esdhc-imx.o
 obj-$(CONFIG_MMC_SDHCI_DOVE)		+= sdhci-dove.o
 obj-$(CONFIG_MMC_SDHCI_TEGRA)		+= sdhci-tegra.o
 obj-$(CONFIG_MMC_SDHCI_OF_ARASAN)	+= sdhci-of-arasan.o
+eswin-sdhci-emmc-objs                   := sdhci-of-eswin.o sdhci-eswin.o
+obj-$(CONFIG_MMC_SDHCI_OF_ESWIN)	+= eswin-sdhci-emmc.o
+eswin-sdhci-sdio-objs                   := sdhci-of-eswin-sdio.o sdhci-eswin.o
+obj-$(CONFIG_MMC_SDHCI_OF_SDIO_ESWIN)   += eswin-sdhci-sdio.o
 obj-$(CONFIG_MMC_SDHCI_OF_ASPEED)	+= sdhci-of-aspeed.o
 obj-$(CONFIG_MMC_SDHCI_OF_AT91)		+= sdhci-of-at91.o
 obj-$(CONFIG_MMC_SDHCI_OF_ESDHC)	+= sdhci-of-esdhc.o
diff --git a/drivers/mmc/host/sdhci-eswin.c b/drivers/mmc/host/sdhci-eswin.c
new file mode 100644
index 000000000000..861a1467623b
--- /dev/null
+++ b/drivers/mmc/host/sdhci-eswin.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Emmc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswincomputing.com>
+ */
+#include <linux/delay.h>
+#include <linux/reset.h>
+#include <linux/regmap.h>
+#include "sdhci-eswin.h"
+
+static void eswin_mshc_coreclk_config(struct sdhci_host *host, uint16_t divisor,
+					unsigned int flag_sel)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	struct eswin_sdhci_data *eswin_sdhci;
+	u32 val = 0;
+	u32 delay = 0xfffff;
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	regmap_read(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, &val);
+	val &= ~MSHC_CORE_CLK_ENABLE;
+	regmap_write(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, val);
+	while (delay--)
+		;
+	val &= ~(MSHC_CORE_CLK_FREQ_BIT_MASK << MSHC_CORE_CLK_FREQ_BIT_SHIFT);
+	val |= (divisor & MSHC_CORE_CLK_FREQ_BIT_MASK)
+	       << MSHC_CORE_CLK_FREQ_BIT_SHIFT;
+	val &= ~(MSHC_CORE_CLK_SEL_BIT);
+	val |= flag_sel;
+	regmap_write(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, val);
+
+	udelay(100);
+	val |= MSHC_CORE_CLK_ENABLE;
+	regmap_write(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, val);
+	mdelay(1);
+}
+
+static void eswin_mshc_coreclk_disable(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	struct eswin_sdhci_data *eswin_sdhci;
+	u32 val = 0;
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	regmap_read(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, &val);
+	val &= ~MSHC_CORE_CLK_ENABLE;
+	regmap_write(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, val);
+}
+
+void eswin_sdhci_disable_card_clk(struct sdhci_host *host)
+{
+	unsigned int clk;
+
+	/* Reset SD Clock Enable */
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+	clk &= ~SDHCI_CLOCK_CARD_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+}
+
+void eswin_sdhci_enable_card_clk(struct sdhci_host *host)
+{
+	ktime_t timeout;
+	unsigned int clk;
+
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+
+	clk |= SDHCI_CLOCK_INT_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	/* Wait max 150 ms */
+	timeout = ktime_add_ms(ktime_get(), 150);
+	while (1) {
+		bool timedout = ktime_after(ktime_get(), timeout);
+		clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+		if (clk & SDHCI_CLOCK_INT_STABLE)
+			break;
+		if (timedout) {
+			pr_err("%s: Internal clock never stabilised.\n",
+					mmc_hostname(host->mmc));
+			return;
+		}
+		udelay(10);
+	}
+
+	clk |= SDHCI_CLOCK_CARD_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+	mdelay(1);
+}
+
+void eswin_sdhci_set_core_clock(struct sdhci_host *host,
+				       unsigned int clock)
+{
+	unsigned int div, divide;
+	unsigned int flag_sel, max_clk;
+
+	host->mmc->actual_clock = clock;
+
+	if (clock == 0) {
+		eswin_mshc_coreclk_disable(host);
+		return;
+	}
+
+	if (SDHCI_CLK_208M % clock == 0) {
+		flag_sel = 1;
+		max_clk = SDHCI_CLK_208M;
+	} else {
+		flag_sel = 0;
+		max_clk = SDHCI_CLK_200M;
+	}
+
+	for (div = 1; div <= MAX_CORE_CLK_DIV; div++) {
+		if ((max_clk / div) <= clock)
+			break;
+	}
+	div--;
+
+	if (div == 0 || div == 1) {
+		divide = 2;
+	} else {
+		divide = (div + 1) * 2;
+	}
+	pr_debug("%s: clock:%d timing:%d\n", mmc_hostname(host->mmc), clock, host->timing);
+
+	eswin_sdhci_disable_card_clk(host);
+	eswin_mshc_coreclk_config(host, divide, flag_sel);
+	eswin_sdhci_enable_card_clk(host);
+	mdelay(2);
+}
+
+static void eswin_sdhci_set_clk_delays(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+
+	clk_set_phase(clk_data->sampleclk,
+			clk_data->clk_phase_in[host->timing]);
+	clk_set_phase(clk_data->sdcardclk,
+			clk_data->clk_phase_out[host->timing]);
+}
+
+static void eswin_sdhci_dt_read_clk_phase(struct device *dev,
+					struct eswin_sdhci_clk_data *clk_data,
+					unsigned int timing, const char *prop)
+{
+	struct device_node *np = dev->of_node;
+
+	int clk_phase[2] = { 0 };
+
+	/*
+	 * Read Tap Delay values from DT, if the DT does not contain the
+	 * Tap Values then use the pre-defined values.
+	 */
+	if (of_property_read_variable_u32_array(np, prop, &clk_phase[0], 2,
+						0)) {
+		dev_dbg(dev, "Using predefined clock phase for %s = %d %d\n",
+			prop, clk_data->clk_phase_in[timing],
+			clk_data->clk_phase_out[timing]);
+		return;
+	}
+
+	/* The values read are Input and Output Clock Delays in order */
+	clk_data->clk_phase_in[timing] = clk_phase[0];
+	clk_data->clk_phase_out[timing] = clk_phase[1];
+}
+
+/**
+ * eswin_dt_parse_clk_phases - Read Clock Delay values from DT
+ *
+ * @dev:        Pointer to our struct device.
+ * @clk_data:       Pointer to the Clock Data structure
+ *
+ * Called at initialization to parse the values of Clock Delays.
+ */
+void eswin_sdhci_dt_parse_clk_phases(struct device *dev,
+				struct eswin_sdhci_clk_data *clk_data)
+{
+	clk_data->set_clk_delays = eswin_sdhci_set_clk_delays;
+
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_LEGACY,
+				      "clk-phase-legacy");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS,
+				      "clk-phase-mmc-hs");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_SD_HS,
+				      "clk-phase-sd-hs");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR12,
+				      "clk-phase-uhs-sdr12");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR25,
+				      "clk-phase-uhs-sdr25");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR50,
+				      "clk-phase-uhs-sdr50");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR104,
+				      "clk-phase-uhs-sdr104");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_DDR50,
+				      "clk-phase-uhs-ddr50");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_DDR52,
+				      "clk-phase-mmc-ddr52");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS200,
+				      "clk-phase-mmc-hs200");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS400,
+				      "clk-phase-mmc-hs400");
+}
+
+unsigned int eswin_convert_drive_impedance_ohm(struct platform_device *pdev,
+					     unsigned int dr_ohm)
+{
+	switch (dr_ohm) {
+	case 100:
+		return PHYCTRL_DR_100OHM;
+	case 66:
+		return PHYCTRL_DR_66OHM;
+	case 50:
+		return PHYCTRL_DR_50OHM;
+	case 40:
+		return PHYCTRL_DR_40OHM;
+	case 33:
+		return PHYCTRL_DR_33OHM;
+	}
+
+	dev_warn(&pdev->dev, "Invalid value %u for drive-impedance-ohm.\n",
+		 dr_ohm);
+	return PHYCTRL_DR_50OHM;
+}
+
+static void eswin_sdhci_do_reset(struct eswin_sdhci_data *eswin_sdhci)
+{
+	int ret;
+
+	ret = reset_control_assert(eswin_sdhci->txrx_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(eswin_sdhci->phy_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(eswin_sdhci->prstn);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(eswin_sdhci->arstn);
+	WARN_ON(0 != ret);
+
+	mdelay(2);
+
+	ret = reset_control_deassert(eswin_sdhci->txrx_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(eswin_sdhci->phy_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(eswin_sdhci->prstn);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(eswin_sdhci->arstn);
+	WARN_ON(0 != ret);
+}
+
+int eswin_sdhci_reset_init(struct device *dev,
+				 struct eswin_sdhci_data *eswin_sdhci)
+{
+	int ret = 0;
+	eswin_sdhci->txrx_rst = devm_reset_control_get_optional(dev, "txrx_rst");
+	if (IS_ERR_OR_NULL(eswin_sdhci->txrx_rst)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->txrx_rst),
+			      "txrx_rst reset not found.\n");
+		return -EFAULT;
+	}
+
+	eswin_sdhci->phy_rst = devm_reset_control_get_optional(dev, "phy_rst");
+	if (IS_ERR_OR_NULL(eswin_sdhci->phy_rst)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->phy_rst),
+			      "phy_rst reset not found.\n");
+		return -EFAULT;
+	}
+
+	eswin_sdhci->prstn = devm_reset_control_get_optional(dev, "prstn");
+	if (IS_ERR_OR_NULL(eswin_sdhci->prstn)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->prstn),
+			      "prstn reset not found.\n");
+		return -EFAULT;
+	}
+
+	eswin_sdhci->arstn = devm_reset_control_get_optional(dev, "arstn");
+	if (IS_ERR_OR_NULL(eswin_sdhci->arstn)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->arstn),
+			      "arstn reset not found.\n");
+		return -EFAULT;
+	}
+	eswin_sdhci_do_reset(eswin_sdhci);
+
+	return ret;
+}
+
+#define DRIVER_NAME "sdhci_esw"
+#define SDHCI_ESW_DUMP(f, x...) \
+	pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
+
+void eswin_sdhci_dump_vendor_regs(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+	u32 val = 0, val1 = 0;
+
+	SDHCI_ESW_DUMP("----------- VENDOR REGISTER DUMP -----------\n");
+
+	ret = regmap_read(eswin_sdhci->crg_regmap, eswin_sdhci->crg_core_clk, &val);
+	if (ret) {
+		pr_err("%s: read crg_core_clk failed, ret:%d\n",
+					mmc_hostname(host->mmc), ret);
+		return;
+	}
+
+	SDHCI_ESW_DUMP("CORE CLK: 0x%08x | IRQ FLAG:  0x%016lx\n",
+		val, arch_local_save_flags());
+
+	ret = regmap_read(eswin_sdhci->crg_regmap, eswin_sdhci->crg_aclk_ctrl, &val);
+	if (ret) {
+		pr_err("%s: read crg_aclk_ctrl failed, ret:%d\n",
+					mmc_hostname(host->mmc), ret);
+		return;
+	}
+	ret = regmap_read(eswin_sdhci->crg_regmap, eswin_sdhci->crg_cfg_ctrl, &val1);
+	if (ret) {
+		pr_err("%s: read crg_cfg_ctrl failed, ret:%d\n",
+					mmc_hostname(host->mmc), ret);
+		return;
+	}
+	SDHCI_ESW_DUMP("HSP ACLK: 0x%08x | HSP CFG:  0x%08x\n", val, val1);
+
+	ret = regmap_read(eswin_sdhci->hsp_regmap, eswin_sdhci->hsp_int_status, &val);
+	if (ret) {
+		pr_err("%s: read hsp_int_status failed, ret:%d\n",
+					mmc_hostname(host->mmc), ret);
+		return;
+	}
+	ret = regmap_read(eswin_sdhci->hsp_regmap, eswin_sdhci->hsp_pwr_ctrl, &val1);
+	if (ret) {
+		pr_err("%s: read hsp_pwr_ctrl failed, ret:%d\n",
+					mmc_hostname(host->mmc), ret);
+		return;
+	}
+	SDHCI_ESW_DUMP("HSP STA: 0x%08x | PWR CTRL:  0x%08x\n", val, val1);
+
+	return;
+}
+
diff --git a/drivers/mmc/host/sdhci-eswin.h b/drivers/mmc/host/sdhci-eswin.h
new file mode 100644
index 000000000000..65b0b69bfda7
--- /dev/null
+++ b/drivers/mmc/host/sdhci-eswin.h
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN SDHCI Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswincomputing.com>
+ */
+#ifndef _DRIVERS_MMC_SDHCI_ESWIN_H
+#define _DRIVERS_MMC_SDHCI_ESWIN_H
+
+#include <linux/reset.h>
+#include <linux/clk-provider.h>
+#include "sdhci-pltfm.h"
+
+#define MSHC_CARD_CLK_STABLE BIT(28)
+#define MSHC_INT_BCLK_STABLE BIT(16)
+#define MSHC_INT_ACLK_STABLE BIT(8)
+#define MSHC_INT_TMCLK_STABLE BIT(0)
+#define MSHC_INT_CLK_STABLE                             \
+	(MSHC_CARD_CLK_STABLE | MSHC_INT_ACLK_STABLE | \
+	 MSHC_INT_BCLK_STABLE | MSHC_INT_TMCLK_STABLE)
+#define MSHC_HOST_VAL_STABLE BIT(0)
+#define EMMC0_CARD_DETECT BIT(9)
+#define EMMC0_CARD_WRITE_PROT BIT(8)
+
+#define MSHC_CORE_CLK_ENABLE BIT(16)
+#define MSHC_CORE_CLK_FREQ_BIT_SHIFT 4
+#define MSHC_CORE_CLK_FREQ_BIT_MASK 0xfffu
+#define MSHC_CORE_CLK_SEL_BIT BIT(0)
+
+/* Controller does not have CD wired and will not function normally without */
+#define SDHCI_ESWIN_QUIRK_FORCE_CDTEST BIT(0)
+/* Controller immediately reports SDHCI_CLOCK_INT_STABLE after enabling the
+ * internal clock even when the clock isn't stable */
+#define SDHCI_ESWIN_QUIRK_CLOCK_UNSTABLE BIT(1)
+
+#define ESWIN_SDHCI_CTRL_HS400 0x0007 // Non-standard, for eswin,these bits are 0x7
+
+#define SDHCI_CLK_208M   208000000
+#define SDHCI_CLK_200M   200000000
+
+#define AWSMMUSID GENMASK(31, 24)  // The sid of write operation
+#define AWSMMUSSID GENMASK(23, 16)  // The ssid of write operation
+#define ARSMMUSID GENMASK(15, 8)  // The sid of read operation
+#define ARSMMUSSID GENMASK(7, 0)  // The ssid of read operation
+
+/* DWC_mshc_map/DWC_mshc_phy_block register */
+#define DWC_MSHC_PTR_PHY_R 0x300
+#define PHY_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x00)
+#define PHY_CMDPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x04)
+#define PHY_DATAPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x06)
+#define PHY_CLKPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x08)
+#define PHY_STBPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x0a)
+#define PHY_RSTNPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x0c)
+#define PHY_PADTEST_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x0e)
+#define PHY_PADTEST_OUT_R (DWC_MSHC_PTR_PHY_R + 0x10)
+#define PHY_PADTEST_IN_R (DWC_MSHC_PTR_PHY_R + 0x12)
+#define PHY_PRBS_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x18)
+#define PHY_PHYLBK_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x1a)
+#define PHY_COMMDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x1c)
+#define PHY_SDCLKDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x1d)
+#define PHY_SDCLKDL_DC_R (DWC_MSHC_PTR_PHY_R + 0x1e)
+#define PHY_SMPLDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x20)
+#define PHY_ATDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x21)
+#define PHY_DLL_CTRL_R (DWC_MSHC_PTR_PHY_R + 0x24)
+#define PHY_DLL_CNFG1_R (DWC_MSHC_PTR_PHY_R + 0x25)
+#define PHY_DLL_CNFG2_R (DWC_MSHC_PTR_PHY_R + 0x26)
+#define PHY_DLLDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x28)
+#define PHY_DLL_OFFST_R (DWC_MSHC_PTR_PHY_R + 0x29)
+#define PHY_DLLMST_TSTDC_R (DWC_MSHC_PTR_PHY_R + 0x2a)
+#define PHY_DLLBT_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x2c)
+#define PHY_DLL_STATUS_R (DWC_MSHC_PTR_PHY_R + 0x2e)
+#define PHY_DLLDBG_MLKDC_R (DWC_MSHC_PTR_PHY_R + 0x30)
+#define PHY_DLLDBG_SLKDC_R (DWC_MSHC_PTR_PHY_R + 0x32)
+
+#define ENABLE 1
+#define DISABLE 0
+/* strength definition */
+#define PHYCTRL_DR_33OHM 0xee
+#define PHYCTRL_DR_40OHM 0xcc
+#define PHYCTRL_DR_50OHM 0x88
+#define PHYCTRL_DR_66OHM 0x44
+#define PHYCTRL_DR_100OHM 0x00
+
+#define PHY_PAD_MAX_DRIVE_STRENGTH 0xf
+#define PHY_CLK_MAX_DELAY_MASK 0x7f
+#define PHY_PAD_SP_DRIVE_SHIF 16
+#define PHY_PAD_SN_DRIVE_SHIF 20
+
+#define PHY_RSTN BIT(0)
+#define PHY_UPDATE_DELAY_CODE BIT(4)
+
+#define VENDOR_EMMC_CTRL_R 0x52c
+#define EMMC_CRAD_PRESENT BIT(0)
+#define EMMC_RST_N_OE BIT(3)
+#define EMMC_RST_N  BIT(2)
+
+#define PHY_SLEW_0 0x0
+#define PHY_SLEW_1 0x1
+#define PHY_SLEW_2 0x2
+#define PHY_SLEW_3 0x3
+#define PHY_TX_SLEW_CTRL_P_BIT_SHIFT 5
+#define PHY_TX_SLEW_CTRL_N_BIT_SHIFT 9
+
+#define PHY_PULL_BIT_SHIF 0x3
+#define PHY_PULL_DISABLED 0x0
+#define PHY_PULL_UP 0x1
+#define PHY_PULL_DOWN 0x2
+#define PHY_PULL_MASK 0x3
+
+#define PHY_PAD_RXSEL_0 0x0
+#define PHY_PAD_RXSEL_1 0x1
+
+#define VENDOR_AT_CTRL_R 0x540
+#define LATENCY_LT_BIT_OFFSET 19
+#define LATENCY_LT_MASK 0x3
+
+#define LATENCY_LT_1 0x0
+#define LATENCY_LT_2 0x1
+#define LATENCY_LT_3 0x2
+#define LATENCY_LT_4 0x3
+#define SW_TUNE_ENABLE BIT(4)
+
+#define VENDOR_AT_SATA_R 0x544
+#define MAX_PHASE_CODE 0xff
+
+#define DLL_ENABEL BIT(0)
+#define DLL_LOCK_STS BIT(0)
+#define DLL_ERROR_STS BIT(1)
+#define PHY_DELAY_CODE_MASK 0x7f
+#define PHY_DELAY_CODE_MAX 0x7f
+
+#define MAX_CORE_CLK_DIV 0xfff
+
+/**
+ * struct eswin_sdhci_clk_ops - Clock Operations for eswin SD controller
+ *
+ * @sdcardclk_ops:  The output clock related operations
+ * @sampleclk_ops:  The sample clock related operations
+ */
+struct eswin_sdhci_clk_ops {
+	const struct clk_ops *sdcardclk_ops;
+	const struct clk_ops *sampleclk_ops;
+};
+
+/**
+ * struct eswin_sdhci_clk_data - ESWIN Controller Clock Data.
+ *
+ * @sdcardclk_hw:   Struct for the clock we might provide to a PHY.
+ * @sdcardclk:      Pointer to normal 'struct clock' for sdcardclk_hw.
+ * @sampleclk_hw:   Struct for the clock we might provide to a PHY.
+ * @sampleclk:      Pointer to normal 'struct clock' for sampleclk_hw.
+ * @clk_phase_in:   Array of Input Clock Phase Delays for all speed modes
+ * @clk_phase_out:  Array of Output Clock Phase Delays for all speed modes
+ * @set_clk_delays: Function pointer for setting Clock Delays
+ * @clk_of_data:    Platform specific runtime clock data storage pointer
+ */
+struct eswin_sdhci_clk_data {
+	struct clk_hw sdcardclk_hw;
+	struct clk *sdcardclk;
+	struct clk_hw sampleclk_hw;
+	struct clk *sampleclk;
+	int clk_phase_in[MMC_TIMING_MMC_HS400 + 1];
+	int clk_phase_out[MMC_TIMING_MMC_HS400 + 1];
+	void (*set_clk_delays)(struct sdhci_host *host);
+	void *clk_of_data;
+};
+
+struct eswin_sdhci_phy_data {
+	unsigned int drive_impedance;
+	unsigned int enable_strobe_pulldown;
+	unsigned int enable_data_pullup;
+	unsigned int enable_cmd_pullup;
+	unsigned int delay_code;
+};
+
+/**
+ * struct eswin_sdhci_data - ESWIN Controller Data
+ *
+ * @host:       Pointer to the main SDHCI host structure.
+ * @clk_ahb:        Pointer to the AHB clock
+ * @has_cqe:        True if controller has command queuing engine.
+ * @clk_data:       Struct for the ESWIN Controller Clock Data.
+ * @clk_ops:        Struct for the ESWIN Controller Clock Operations.
+ * @soc_ctl_base:   Pointer to regmap for syscon for soc_ctl registers.
+ * @soc_ctl_map:    Map to get offsets into soc_ctl registers.
+ * @quirks:     ESWIN deviations from spec.
+ * @phy:        ESWIN sdhci phy configs.
+ * @private:    private for spec driver.
+ */
+struct eswin_sdhci_data {
+	struct sdhci_host *host;
+	struct clk *clk_ahb;
+	bool has_cqe;
+	struct eswin_sdhci_clk_data clk_data;
+	const struct eswin_sdhci_clk_ops *clk_ops;
+	unsigned int quirks;
+
+	struct regmap *crg_regmap;
+	unsigned int crg_core_clk;
+	unsigned int crg_aclk_ctrl;
+	unsigned int crg_cfg_ctrl;
+
+	struct regmap *hsp_regmap;
+	unsigned int hsp_int_status;
+	unsigned int hsp_pwr_ctrl;
+
+	struct reset_control *txrx_rst;
+	struct reset_control *phy_rst;
+	struct reset_control *prstn;
+	struct reset_control *arstn;
+	struct eswin_sdhci_phy_data phy;
+	unsigned long private[] ____cacheline_aligned;
+};
+
+struct eswin_sdhci_of_data {
+	const struct sdhci_pltfm_data *pdata;
+	const struct eswin_sdhci_clk_ops *clk_ops;
+};
+
+void eswin_sdhci_set_core_clock(struct sdhci_host *host,
+				       unsigned int clock);
+void eswin_sdhci_disable_card_clk(struct sdhci_host *host);
+void eswin_sdhci_enable_card_clk(struct sdhci_host *host);
+void eswin_sdhci_dt_parse_clk_phases(struct device *dev,
+				struct eswin_sdhci_clk_data *clk_data);
+unsigned int eswin_convert_drive_impedance_ohm(struct platform_device *pdev,
+					     unsigned int dr_ohm);
+int eswin_sdhci_reset_init(struct device *dev,
+				 struct eswin_sdhci_data *eswin_sdhci);
+void eswin_sdhci_dump_vendor_regs(struct sdhci_host *host);
+
+#endif /* _DRIVERS_MMC_SDHCI_ESWIN_H */
diff --git a/drivers/mmc/host/sdhci-of-eswin-sdio.c b/drivers/mmc/host/sdhci-of-eswin-sdio.c
new file mode 100644
index 000000000000..17cc92dd4cce
--- /dev/null
+++ b/drivers/mmc/host/sdhci-of-eswin-sdio.c
@@ -0,0 +1,1089 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN sdio Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswincomputing.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include <linux/reset.h>
+#include "cqhci.h"
+#include "sdhci-pltfm.h"
+
+#include <linux/eswin-win2030-sid-cfg.h>
+#include <linux/bitfield.h>
+#include <linux/iommu.h>
+#include "sdhci-eswin.h"
+
+#define ESWIN_SDHCI_SD_CQE_BASE_ADDR 0x180
+#define TUNING_RANGE_THRESHOLD   40
+
+static inline void *sdhci_sdio_priv(struct eswin_sdhci_data *sdio)
+{
+	return sdio->private;
+}
+
+static void eswin_sdhci_sdio_set_clock(struct sdhci_host *host,
+				       unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_clk_data *clk_data =
+		&eswin_sdhci_sdio->clk_data;
+
+	/* Set the Input and Output Clock Phase Delays */
+	if (clk_data->set_clk_delays) {
+		clk_data->set_clk_delays(host);
+	}
+
+	eswin_sdhci_set_core_clock(host, clock);
+
+	if (eswin_sdhci_sdio->quirks & SDHCI_ESWIN_QUIRK_CLOCK_UNSTABLE)
+		/*
+         * Some controllers immediately report SDHCI_CLOCK_INT_STABLE
+         * after enabling the clock even though the clock is not
+         * stable. Trying to use a clock without waiting here results
+         * in EILSEQ while detecting some older/slower cards. The
+         * chosen delay is the maximum delay from sdhci_set_clock.
+         */
+		msleep(20);
+}
+
+static void eswin_sdhci_sdio_config_phy_delay(struct sdhci_host *host,
+					      int delay)
+{
+	delay &= PHY_CLK_MAX_DELAY_MASK;
+
+	/*phy clk delay line config*/
+	sdhci_writeb(host, PHY_UPDATE_DELAY_CODE, PHY_SDCLKDL_CNFG_R);
+	sdhci_writeb(host, delay, PHY_SDCLKDL_DC_R);
+	sdhci_writeb(host, 0x0, PHY_SDCLKDL_CNFG_R);
+}
+
+static void eswin_sdhci_sdio_config_phy(struct sdhci_host *host)
+{
+	unsigned int val = 0;
+	unsigned int drv = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci =
+		sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_phy_data *phy = &eswin_sdhci->phy;
+
+	drv = phy->drive_impedance << PHY_PAD_SP_DRIVE_SHIF;
+	pr_debug("%s: phy drv=0x%x \n",mmc_hostname(host->mmc), drv);
+
+	eswin_sdhci_disable_card_clk(host);
+
+	/* reset phy,config phy's pad */
+	sdhci_writel(host, drv | (~PHY_RSTN), PHY_CNFG_R);
+	/*CMDPAD_CNFS*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+	      (phy->enable_cmd_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_CMDPAD_CNFG_R);
+	pr_debug("%s: phy cmd=0x%x\n",mmc_hostname(host->mmc), val);
+
+	/*DATA PAD CNFG*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+	      (phy->enable_data_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_DATAPAD_CNFG_R);
+	pr_debug("%s: phy data=0x%x\n",mmc_hostname(host->mmc), val);
+
+	/*Clock PAD Setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) | PHY_PAD_RXSEL_0;
+	sdhci_writew(host, val, PHY_CLKPAD_CNFG_R);
+	pr_debug("%s: phy clk=0x%x\n",mmc_hostname(host->mmc), val);
+	mdelay(2);
+
+	/*PHY RSTN PAD setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+	      (PHY_PULL_UP << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_RSTNPAD_CNFG_R);
+
+	sdhci_writel(host, drv | PHY_RSTN, PHY_CNFG_R);
+
+	eswin_sdhci_sdio_config_phy_delay(host, phy->delay_code);
+
+	eswin_sdhci_enable_card_clk(host);
+}
+
+static void eswin_sdhci_sdio_reset(struct sdhci_host *host, u8 mask)
+{
+	u8 ctrl;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+
+	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
+	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
+
+	if (mask & SDHCI_RESET_ALL) {
+		sdhci_reset(host, SDHCI_RESET_ALL);
+	}
+	if (mask & SDHCI_RESET_DATA) {
+		sdhci_reset(host, SDHCI_RESET_DATA);
+	}
+
+	if (mask & SDHCI_RESET_CMD) {
+		sdhci_reset(host, SDHCI_RESET_CMD);
+	}
+
+	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
+	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+
+	if (eswin_sdhci_sdio->quirks & SDHCI_ESWIN_QUIRK_FORCE_CDTEST) {
+		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
+		ctrl |= SDHCI_CTRL_CDTEST_INS | SDHCI_CTRL_CDTEST_EN;
+		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
+	}
+	if (mask == SDHCI_RESET_ALL) {	// after reset all,the phy`s config will be clear.
+		eswin_sdhci_sdio_config_phy(host);
+	}
+}
+
+static int eswin_sdhci_sdio_phase_code_tuning(struct sdhci_host *host,
+					      u32 opcode)
+{
+	int cmd_error = 0;
+	int ret = 0;
+	int phase_code = -1;
+	int code_min = -1;
+	int code_max = -1;
+	int code_range = -1;
+	int i = 0;
+
+	for (i = 0; i <= MAX_PHASE_CODE; i++) {
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, i, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+		if (ret) {
+			udelay(200);
+			pr_debug("%s: bad phase_code:0x%x!\n", mmc_hostname(host->mmc), i);
+			if (code_min != -1 && code_max != -1) {
+				if (code_max - code_min > code_range) {
+					code_range = code_max - code_min;
+					phase_code = (code_min + code_max) / 2;
+					if (code_range > TUNING_RANGE_THRESHOLD)
+						break;
+				}
+				code_min = -1;
+				code_max = -1;
+			}
+		} else {
+			pr_debug("%s: ok phase_code:0x%x\n", mmc_hostname(host->mmc), i);
+			if (code_min == -1) {
+				code_min = i;
+			}
+			code_max = i;
+			if (i == MAX_PHASE_CODE) {
+				if (code_max - code_min > code_range) {
+					code_range = code_max - code_min;
+					phase_code = (code_min + code_max) / 2;
+				}
+			}
+			continue;
+		}
+	}
+
+	if (phase_code == -1) {
+		pr_debug("%s: phase code tuning failed!\n",
+		       mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+		return -EIO;
+	}
+
+	pr_debug("%s: set phase_code:0x%x\n", mmc_hostname(host->mmc), phase_code);
+
+	eswin_sdhci_disable_card_clk(host);
+	sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+	eswin_sdhci_enable_card_clk(host);
+
+	ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+	host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+	if (ret) {
+		pr_err("%s: phase_code code(0x%x) not work, tuning failed!\n",
+		       mmc_hostname(host->mmc), phase_code);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_sdhci_sdio_executing_tuning(struct sdhci_host *host,
+					     u32 opcode)
+{
+	u32 ctrl;
+	u32 val;
+	int ret = 0;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct eswin_sdhci_data *eswin_sdhci_sdio;
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci_sdio = sdhci_pltfm_priv(pltfm_host);
+
+	eswin_sdhci_disable_card_clk(host);
+
+	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
+	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
+
+	val = sdhci_readl(host, VENDOR_AT_CTRL_R);
+	val |= SW_TUNE_ENABLE;
+	sdhci_writew(host, val, VENDOR_AT_CTRL_R);
+	sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+
+	eswin_sdhci_enable_card_clk(host);
+
+	sdhci_writew(host, 0x0, SDHCI_CMD_DATA);
+
+	ret = eswin_sdhci_sdio_phase_code_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+
+	return 0;
+}
+
+static u32 eswin_sdhci_sdio_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+static void eswin_sdhci_sdio_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static void eswin_sdhci_sdio_cqe_enable(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 reg;
+
+	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	while (reg & SDHCI_DATA_AVAILABLE) {
+		sdhci_readl(host, SDHCI_BUFFER);
+		reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	}
+
+	sdhci_cqe_enable(mmc);
+}
+
+static const struct cqhci_host_ops eswin_sdhci_sdio_cqhci_ops = {
+	.enable = eswin_sdhci_sdio_cqe_enable,
+	.disable = sdhci_cqe_disable,
+	.dumpregs = eswin_sdhci_sdio_dumpregs,
+};
+
+static const struct sdhci_ops eswin_sdhci_sdio_cqe_ops = {
+	.set_clock = eswin_sdhci_sdio_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = eswin_sdhci_sdio_reset,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_power = sdhci_set_power_and_bus_voltage,
+	.irq = eswin_sdhci_sdio_cqhci_irq,
+	.platform_execute_tuning = eswin_sdhci_sdio_executing_tuning,
+	.dump_vendor_regs = eswin_sdhci_dump_vendor_regs,
+};
+
+static const struct sdhci_pltfm_data eswin_sdhci_sdio_cqe_pdata = {
+	.ops = &eswin_sdhci_sdio_cqe_ops,
+	.quirks = SDHCI_QUIRK_BROKEN_CQE |
+		SDHCI_QUIRK_SINGLE_POWER_WRITE |
+		SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+		SDHCI_QUIRK_BROKEN_TIMEOUT_VAL,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+		SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
+};
+
+#ifdef CONFIG_PM_SLEEP
+/**
+ * eswin_sdhci_sdio_suspend- Suspend method for the driver
+ * @dev:        Address of the device structure
+ *
+ * Put the device in a low power state.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	pm_runtime_get_sync(dev);
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	ret = sdhci_suspend_host(host);
+	if (ret)
+		return ret;
+
+	win2030_tbu_power(dev, false);
+	clk_disable_unprepare(pltfm_host->clk);
+	clk_disable_unprepare(eswin_sdhci_sdio->clk_ahb);
+
+	return 0;
+}
+
+/**
+ * eswin_sdhci_sdio_resume- Resume method for the driver
+ * @dev:        Address of the device structure
+ *
+ * Resume operation after suspend
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	pm_runtime_put_sync(dev);
+	ret = clk_prepare_enable(eswin_sdhci_sdio->clk_ahb);
+	if (ret) {
+		dev_err(dev, "can't enable clk_ahb.\n");
+		return ret;
+	}
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret) {
+		dev_err(dev, "can't enable mainck.\n");
+		goto clk_ahb_disable;
+	}
+	win2030_tbu_power(dev, true);
+
+	ret = sdhci_resume_host(host);
+	if (ret) {
+		dev_err(dev, "runtime resume failed!\n");
+		goto clk_disable;
+	}
+
+	return 0;
+clk_disable:
+	clk_disable_unprepare(pltfm_host->clk);
+clk_ahb_disable:
+	clk_disable_unprepare(eswin_sdhci_sdio->clk_ahb);
+
+	return ret;
+}
+
+static int eswin_sdhci_sdio_runtime_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = sdhci_runtime_suspend_host(host);
+	if (ret)
+		return ret;
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	win2030_tbu_power(dev, false);
+	clk_disable_unprepare(pltfm_host->clk);
+	clk_disable_unprepare(eswin_sdhci_sdio->clk_ahb);
+
+	return 0;
+}
+
+static int eswin_sdhci_sdio_runtime_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_prepare_enable(eswin_sdhci_sdio->clk_ahb);
+	if (ret) {
+		dev_err(dev, "can't enable clk_ahb\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret) {
+		dev_err(dev, "can't enable mainck\n");
+		goto clk_ahb_disable;
+	}
+	win2030_tbu_power(dev, true);
+
+	ret = sdhci_runtime_resume_host(host, 1);
+	if (ret) {
+		dev_err(dev, "runtime resume failed!\n");
+		goto clk_disable;
+	}
+
+	return 0;
+clk_disable:
+	clk_disable_unprepare(pltfm_host->clk);
+clk_ahb_disable:
+	clk_disable_unprepare(eswin_sdhci_sdio->clk_ahb);
+
+	return ret;
+}
+
+#endif /* ! CONFIG_PM_SLEEP */
+
+/**
+ * eswin_sdhci_sdio_sdcardclk_recalc_rate- Return the card clock rate
+ *
+ * @hw:                 Pointer to the hardware clock structure.
+ * @parent_rate:                The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the SD card clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The card clock rate.
+ */
+static unsigned long
+eswin_sdhci_sdio_sdcardclk_recalc_rate(struct clk_hw *hw,
+				       unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data = container_of(
+		hw, struct eswin_sdhci_clk_data, sdcardclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci_sdio->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sdio_sdcardclk_ops = {
+	.recalc_rate = eswin_sdhci_sdio_sdcardclk_recalc_rate,
+};
+
+/**
+ * eswin_sdhci_sdio_sampleclk_recalc_rate- Return the sampling clock rate
+ *
+ * @hw:                 Pointer to the hardware clock structure.
+ * @parent_rate:                The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the sampling clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The sample clock rate.
+ */
+static unsigned long
+eswin_sdhci_sdio_sampleclk_recalc_rate(struct clk_hw *hw,
+				       unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data = container_of(
+		hw, struct eswin_sdhci_clk_data, sampleclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci_sdio->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sdio_sampleclk_ops = {
+	.recalc_rate = eswin_sdhci_sdio_sampleclk_recalc_rate,
+};
+
+static const struct eswin_sdhci_clk_ops eswin_sdio_clk_ops = {
+	.sdcardclk_ops = &eswin_sdio_sdcardclk_ops,
+	.sampleclk_ops = &eswin_sdio_sampleclk_ops,
+};
+
+static struct eswin_sdhci_of_data eswin_sdhci_fu800_sdio_data = {
+	.pdata = &eswin_sdhci_sdio_cqe_pdata,
+	.clk_ops = &eswin_sdio_clk_ops,
+};
+
+static const struct of_device_id eswin_sdhci_sdio_of_match[] = {
+	/* SoC-specific compatible strings*/
+	{
+		.compatible = "eswin,sdhci-sdio",
+		.data = &eswin_sdhci_fu800_sdio_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, eswin_sdhci_sdio_of_match);
+
+/**
+ * eswin_sdhci_sdio_register_sdcardclk- Register the sdcardclk for a PHY to use
+ *
+ * @sdhci_arasan:       Our private data structure.
+ * @clk_xin:            Pointer to the functional clock
+ * @dev:                Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_register_sdcardclk(
+	struct eswin_sdhci_data *eswin_sdhci_sdio, struct clk *clk_xin,
+	struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		&eswin_sdhci_sdio->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sdcardclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 0,
+					    &sdcardclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sdcardclk_init.parent_names = &parent_clk_name;
+	sdcardclk_init.num_parents = 1;
+	sdcardclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sdcardclk_init.ops = eswin_sdhci_sdio->clk_ops->sdcardclk_ops;
+
+	clk_data->sdcardclk_hw.init = &sdcardclk_init;
+	clk_data->sdcardclk = devm_clk_register(dev, &clk_data->sdcardclk_hw);
+	if (IS_ERR(clk_data->sdcardclk))
+		return PTR_ERR(clk_data->sdcardclk);
+
+	clk_data->sdcardclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+				  clk_data->sdcardclk);
+	if (ret)
+		dev_err(dev, "Failed to add sdcard clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_sdio_register_sampleclk - Register the sampleclk for a PHY to use
+ *
+ * @sdhci_arasan:       Our private data structure.
+ * @clk_xin:            Pointer to the functional clock
+ * @dev:                Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_register_sampleclk(
+	struct eswin_sdhci_data *eswin_sdhci_sdio, struct clk *clk_xin,
+	struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		&eswin_sdhci_sdio->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sampleclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 1,
+					    &sampleclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sampleclk_init.parent_names = &parent_clk_name;
+	sampleclk_init.num_parents = 1;
+	sampleclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sampleclk_init.ops = eswin_sdhci_sdio->clk_ops->sampleclk_ops;
+
+	clk_data->sampleclk_hw.init = &sampleclk_init;
+	clk_data->sampleclk = devm_clk_register(dev, &clk_data->sampleclk_hw);
+	if (IS_ERR(clk_data->sampleclk))
+		return PTR_ERR(clk_data->sampleclk);
+	clk_data->sampleclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+				  clk_data->sampleclk);
+	if (ret)
+		dev_err(dev, "Failed to add sample clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_sdio_unregister_sdclk- Undoes sdhci_arasan_register_sdclk()
+ *
+ * @dev:                Pointer to our struct device.
+ *
+ * Should be called any time we're exiting and sdhci_arasan_register_sdclk()
+ * returned success.
+ */
+static void eswin_sdhci_sdio_unregister_sdclk(struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+
+	if (!of_find_property(np, "#clock-cells", NULL))
+		return;
+
+	of_clk_del_provider(dev->of_node);
+}
+
+/**
+ * eswin_sdhci_sdio_register_sdclk- Register the sdcardclk for a PHY to use
+ *
+ * @eswin_sdhci_sdio:   Our private data structure.
+ * @clk_xin:            Pointer to the functional clock
+ * @dev:                Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Note: without seriously re-architecting SDHCI's clock code and testing on
+ * all platforms, there's no way to create a totally beautiful clock here
+ * with all clock ops implemented.      Instead, we'll just create a clock that can
+ * be queried and set the CLK_GET_RATE_NOCACHE attribute to tell common clock
+ * framework that we're doing things behind its back.  This should be sufficient
+ * to create nice clean device tree bindings and later (if needed) we can try
+ * re-architecting SDHCI if we see some benefit to it.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int
+eswin_sdhci_sdio_register_sdclk(struct eswin_sdhci_data *eswin_sdhci_sdio,
+				struct clk *clk_xin, struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	u32 num_clks = 0;
+	int ret;
+
+	/* Providing a clock to the PHY is optional; no error if missing */
+	if (of_property_read_u32(np, "#clock-cells", &num_clks) < 0)
+		return 0;
+
+	ret = eswin_sdhci_sdio_register_sdcardclk(eswin_sdhci_sdio, clk_xin,
+						  dev);
+	if (ret)
+		return ret;
+
+	if (num_clks) {
+		ret = eswin_sdhci_sdio_register_sampleclk(eswin_sdhci_sdio,
+							  clk_xin, dev);
+		if (ret) {
+			eswin_sdhci_sdio_unregister_sdclk(dev);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+eswin_sdhci_sdio_add_host(struct eswin_sdhci_data *eswin_sdhci_sdio)
+{
+	struct sdhci_host *host = eswin_sdhci_sdio->host;
+	struct cqhci_host *cq_host;
+	bool dma64;
+	int ret;
+
+	if (!eswin_sdhci_sdio->has_cqe)
+		return sdhci_add_host(host);
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	cq_host = devm_kzalloc(host->mmc->parent, sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	cq_host->mmio = host->ioaddr + ESWIN_SDHCI_SD_CQE_BASE_ADDR;
+	cq_host->ops = &eswin_sdhci_sdio_cqhci_ops;
+
+	dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+	if (dma64)
+		cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+	ret = cqhci_init(cq_host, host->mmc, dma64);
+	if (ret)
+		goto cleanup;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
+static int eswin_sdhci_sdio_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_sdio_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev,
+			"dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+					 &hsp_mmu_sdio_reg);
+	if (ret) {
+		dev_err(dev, "can't get sdio sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_sdio_reg, rdwr_sid_ssid);
+
+	ret = win2030_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config sdio streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config sdio streamID(%d)!\n", sid);
+
+	return ret;
+}
+
+static int eswin_sdhci_sdio_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct clk *clk_xin;
+	struct clk *clk_spll2_fout3;
+	struct clk *clk_mux;
+	struct sdhci_host *host;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct eswin_sdhci_data *eswin_sdhci_sdio;
+	const struct eswin_sdhci_of_data *data;
+	unsigned int val = 0;
+
+	data = of_device_get_match_data(dev);
+	host = sdhci_pltfm_init(pdev, data->pdata, sizeof(*eswin_sdhci_sdio));
+
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci_sdio = sdhci_pltfm_priv(pltfm_host);
+	eswin_sdhci_sdio->host = host;
+	eswin_sdhci_sdio->has_cqe = false;
+
+	sdhci_get_of_property(pdev);
+
+	eswin_sdhci_sdio->clk_ops = data->clk_ops;
+	eswin_sdhci_sdio->clk_ahb = devm_clk_get(dev, "clk_ahb");
+	if (IS_ERR(eswin_sdhci_sdio->clk_ahb)) {
+		ret = dev_err_probe(dev, PTR_ERR(eswin_sdhci_sdio->clk_ahb),
+				    "clk_ahb clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	clk_xin = devm_clk_get(dev, "clk_xin");
+	if (IS_ERR(clk_xin)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_xin),
+				    "clk_xin clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	clk_spll2_fout3 = devm_clk_get(dev, "clk_spll2_fout3");
+
+	if (IS_ERR(clk_spll2_fout3)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_spll2_fout3),
+				    "clk_spll2_fout3 clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	if (of_device_is_compatible(np, "eswin,sdhci-sdio")) {
+		clk_mux = devm_clk_get(dev, "clk_mux1_1");
+		if (IS_ERR(clk_mux)) {
+			ret = dev_err_probe(dev, PTR_ERR(clk_mux),
+					    "clk_mux1_1 clock not found.\n");
+			goto err_pltfm_free;
+		}
+		/*switch the core clk source*/
+		clk_set_parent(clk_mux, clk_spll2_fout3);
+	}
+
+	ret = clk_prepare_enable(eswin_sdhci_sdio->clk_ahb);
+	if (ret) {
+		dev_err(dev, "Unable to enable AHB clock.\n");
+		goto err_pltfm_free;
+	}
+	/* If clock-frequency property is set, use the provided value */
+	if (pltfm_host->clock && pltfm_host->clock != clk_get_rate(clk_xin)) {
+		ret = clk_set_rate(clk_xin, pltfm_host->clock);
+		if (ret) {
+			dev_err(&pdev->dev, "Failed to set SD clock rate\n");
+			goto clk_dis_ahb;
+		}
+	}
+
+	ret = clk_prepare_enable(clk_xin);
+	if (ret) {
+		dev_err(dev, "Unable to enable SD clock.\n");
+		goto clk_dis_ahb;
+	}
+
+	pltfm_host->clk = clk_xin;
+	ret = eswin_sdhci_sdio_register_sdclk(eswin_sdhci_sdio, clk_xin, dev);
+	if (ret)
+		goto clk_disable_all;
+
+	ret = eswin_sdhci_reset_init(dev, eswin_sdhci_sdio);
+	if (ret < 0) {
+		dev_err(dev, "failed to reset\n");
+		goto clk_disable_all;
+	}
+
+	eswin_sdhci_sdio->crg_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,syscrg_csr");
+	if (IS_ERR(eswin_sdhci_sdio->crg_regmap)){
+		dev_dbg(&pdev->dev, "No syscrg_csr phandle specified\n");
+		goto clk_disable_all;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 1,
+                                    &eswin_sdhci_sdio->crg_core_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get crg_core_clk (%d)\n", ret);
+		goto clk_disable_all;
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 2,
+                                    &eswin_sdhci_sdio->crg_aclk_ctrl);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get crg_aclk_ctrl (%d)\n", ret);
+		goto clk_disable_all;
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 3,
+                                    &eswin_sdhci_sdio->crg_cfg_ctrl);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get crg_cfg_ctrl (%d)\n", ret);
+		goto clk_disable_all;
+	}
+
+	eswin_sdhci_sdio->hsp_regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(eswin_sdhci_sdio->hsp_regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		goto clk_disable_all;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 2,
+                                    &eswin_sdhci_sdio->hsp_int_status);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_int_status (%d)\n", ret);
+		goto clk_disable_all;
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 3,
+                                    &eswin_sdhci_sdio->hsp_pwr_ctrl);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_pwr_ctrl (%d)\n", ret);
+		goto clk_disable_all;
+	}
+
+	regmap_write(eswin_sdhci_sdio->hsp_regmap, eswin_sdhci_sdio->hsp_int_status, MSHC_INT_CLK_STABLE);
+	regmap_write(eswin_sdhci_sdio->hsp_regmap, eswin_sdhci_sdio->hsp_pwr_ctrl, MSHC_HOST_VAL_STABLE);
+
+	ret = eswin_sdhci_sdio_sid_cfg(dev);
+	if (ret < 0) {
+		dev_err(dev, "failed to use smmu\n");
+		goto clk_disable_all;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "delay_code", &val)) {
+		eswin_sdhci_sdio->phy.delay_code = val;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val))
+		eswin_sdhci_sdio->phy.drive_impedance =
+			eswin_convert_drive_impedance_ohm(pdev, val);
+
+	if (of_property_read_bool(dev->of_node, "enable-cmd-pullup"))
+		eswin_sdhci_sdio->phy.enable_cmd_pullup = ENABLE;
+	else
+		eswin_sdhci_sdio->phy.enable_cmd_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable-data-pullup"))
+		eswin_sdhci_sdio->phy.enable_data_pullup = ENABLE;
+	else
+		eswin_sdhci_sdio->phy.enable_data_pullup = DISABLE;
+
+	eswin_sdhci_dt_parse_clk_phases(dev, &eswin_sdhci_sdio->clk_data);
+	ret = mmc_of_parse(host->mmc);
+	if (ret) {
+		ret = dev_err_probe(dev, ret, "parsing dt failed.\n");
+		goto unreg_clk;
+	}
+
+	win2030_tbu_power(&pdev->dev, true);
+
+	ret = eswin_sdhci_sdio_add_host(eswin_sdhci_sdio);
+	if (ret)
+		goto unreg_clk;
+
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 50);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_suspend_ignore_children(&pdev->dev, 1);
+	pm_runtime_enable(&pdev->dev);
+
+	return 0;
+
+unreg_clk:
+	eswin_sdhci_sdio_unregister_sdclk(dev);
+clk_disable_all:
+	clk_disable_unprepare(clk_xin);
+clk_dis_ahb:
+	clk_disable_unprepare(eswin_sdhci_sdio->clk_ahb);
+err_pltfm_free:
+	sdhci_pltfm_free(pdev);
+	return ret;
+}
+
+static int eswin_sdhci_sdio_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	struct clk *clk_ahb = eswin_sdhci_sdio->clk_ahb;
+
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
+	sdhci_pltfm_remove(pdev);
+	win2030_tbu_power(&pdev->dev, false);
+
+	if (eswin_sdhci_sdio->txrx_rst) {
+		ret = reset_control_assert(eswin_sdhci_sdio->txrx_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci_sdio->phy_rst) {
+		ret = reset_control_assert(eswin_sdhci_sdio->phy_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci_sdio->prstn) {
+		ret = reset_control_assert(eswin_sdhci_sdio->prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci_sdio->arstn) {
+		ret = reset_control_assert(eswin_sdhci_sdio->arstn);
+		WARN_ON(0 != ret);
+	}
+
+	eswin_sdhci_sdio_unregister_sdclk(&pdev->dev);
+	clk_disable_unprepare(clk_ahb);
+
+	return 0;
+}
+
+static const struct dev_pm_ops eswin_sdhci_sdio_pmops = {
+	SET_SYSTEM_SLEEP_PM_OPS(eswin_sdhci_sdio_suspend, eswin_sdhci_sdio_resume)
+	SET_RUNTIME_PM_OPS(eswin_sdhci_sdio_runtime_suspend,
+			   eswin_sdhci_sdio_runtime_resume, NULL)
+};
+
+static struct platform_driver eswin_sdhci_sdio_driver = {
+	.driver = {
+		.name = "eswin-sdhci-sdio",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = eswin_sdhci_sdio_of_match,
+		.pm = &eswin_sdhci_sdio_pmops,
+	},
+	.probe = eswin_sdhci_sdio_probe,
+	.remove = eswin_sdhci_sdio_remove,
+};
+
+static __init int eswin_sdhci_sdio_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&eswin_sdhci_sdio_driver);
+	if (ret) {
+		pr_err("%s: failed to register platform driver\n",
+			__func__);
+	}
+
+	return ret;
+}
+
+static void __exit eswin_sdhci_sdio_exit(void)
+{
+	platform_driver_unregister(&eswin_sdhci_sdio_driver);
+}
+
+/*Cause EMMC is often used as a system disk(mmc0), we need the SD driver to run later than the EMMC driver*/
+late_initcall(eswin_sdhci_sdio_init);
+module_exit(eswin_sdhci_sdio_exit);
+
+MODULE_DESCRIPTION("Driver for the Eswin SDHCI Controller");
+MODULE_AUTHOR("Eswin");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/sdhci-of-eswin.c b/drivers/mmc/host/sdhci-of-eswin.c
new file mode 100644
index 000000000000..94ab3392cffe
--- /dev/null
+++ b/drivers/mmc/host/sdhci-of-eswin.c
@@ -0,0 +1,1122 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Emmc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswincomputing.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include "cqhci.h"
+#include "sdhci-pltfm.h"
+#include <linux/reset.h>
+#include <linux/iommu.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+#include "sdhci-eswin.h"
+
+//EMMC_DWC_MSHC_CRYPTO_CFG_PTR 8 -- parameter
+#define eswin_sdhci_VENDOR_REGISTER_BASEADDR 0x800
+#define eswin_sdhci_VENDOR_EMMC_CTRL_REGISTER 0x2c
+#define VENDOR_ENHANCED_STROBE BIT(8)
+
+#define eswin_sdhci_CQE_BASE_ADDR eswin_sdhci_VENDOR_REGISTER_BASEADDR
+
+/* Controller does not have CD wired and will not function normally without */
+#define eswin_sdhci_QUIRK_FORCE_CDTEST BIT(0)
+/* Controller immediately reports SDHCI_CLOCK_INT_STABLE after enabling the
+ * internal clock even when the clock isn't stable */
+#define eswin_sdhci_QUIRK_CLOCK_UNSTABLE BIT(1)
+
+/*
+ * On some SoCs the syscon area has a feature where the upper 16-bits of
+ * each 32-bit register act as a write mask for the lower 16-bits.  This allows
+ * atomic updates of the register without locking.  This macro is used on SoCs
+ * that have that feature.
+ */
+#define HIWORD_UPDATE(val, mask, shift) \
+	((val) << (shift) | (mask) << ((shift) + 16))
+
+static void eswin_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+
+	/* Set the Input and Output Clock Phase Delays */
+	if (clk_data->set_clk_delays)
+		clk_data->set_clk_delays(host);
+
+	eswin_sdhci_set_core_clock(host, clock);
+
+	/*
+	 * Some controllers immediately report SDHCI_CLOCK_INT_STABLE
+	 * after enabling the clock even though the clock is not
+	 * stable. Trying to use a clock without waiting here results
+	 * in EILSEQ while detecting some older/slower cards. The
+	 * chosen delay is the maximum delay from sdhci_set_clock.
+	 */
+	if (eswin_sdhci->quirks & SDHCI_ESWIN_QUIRK_CLOCK_UNSTABLE)
+		msleep(20);
+}
+
+static void eswin_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
+					      struct mmc_ios *ios)
+{
+	u32 vendor;
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	vendor = sdhci_readl(host, eswin_sdhci_VENDOR_EMMC_CTRL_REGISTER);
+	if (ios->enhanced_strobe)
+		vendor |= VENDOR_ENHANCED_STROBE;
+	else
+		vendor &= ~VENDOR_ENHANCED_STROBE;
+
+	sdhci_writel(host, vendor, eswin_sdhci_VENDOR_EMMC_CTRL_REGISTER);
+}
+
+static void eswin_sdhci_config_phy_delay(struct sdhci_host *host, int delay)
+{
+	delay &= PHY_CLK_MAX_DELAY_MASK;
+
+	/*phy clk delay line config*/
+	sdhci_writeb(host, PHY_UPDATE_DELAY_CODE, PHY_SDCLKDL_CNFG_R);
+	sdhci_writeb(host, delay, PHY_SDCLKDL_DC_R);
+	sdhci_writeb(host, 0x0, PHY_SDCLKDL_CNFG_R);
+}
+
+static void eswin_sdhci_config_phy(struct sdhci_host *host)
+{
+	unsigned int val = 0;
+	unsigned int drv = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_phy_data *phy = &eswin_sdhci->phy;
+
+	drv = phy->drive_impedance << PHY_PAD_SP_DRIVE_SHIF;
+
+	pr_debug("%s: phy drv=0x%x\n", mmc_hostname(host->mmc), drv);
+
+	val = sdhci_readw(host, VENDOR_EMMC_CTRL_R);
+	val |= EMMC_CRAD_PRESENT;  // emmc card
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+
+	eswin_sdhci_disable_card_clk(host);
+
+	/* reset phy,config phy's pad */
+	sdhci_writel(host, drv | (~PHY_RSTN), PHY_CNFG_R);
+	/*CMDPAD_CNFS*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		(phy->enable_cmd_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_CMDPAD_CNFG_R);
+	pr_debug("%s: phy cmd=0x%x\n", mmc_hostname(host->mmc), val);
+
+	/*DATA PAD CNFG*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		(phy->enable_data_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_DATAPAD_CNFG_R);
+	pr_debug("%s: phy data=0x%x\n", mmc_hostname(host->mmc), val);
+
+	/*Clock PAD Setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) | PHY_PAD_RXSEL_0;
+	sdhci_writew(host, val, PHY_CLKPAD_CNFG_R);
+	pr_debug("%s: phy clk=0x%x\n", mmc_hostname(host->mmc), val);
+
+	/*PHY strobe PAD setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		((phy->enable_strobe_pulldown * PHY_PULL_DOWN) << PHY_PULL_BIT_SHIF) |
+		PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_STBPAD_CNFG_R);
+	pr_debug("%s: phy strobe=0x%x\n", mmc_hostname(host->mmc), val);
+	mdelay(2);
+
+	/*PHY RSTN PAD setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		(PHY_PULL_UP << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_RSTNPAD_CNFG_R);
+	pr_debug("%s: phy rstn=0x%x\n", mmc_hostname(host->mmc), val);
+
+	sdhci_writel(host, drv | PHY_RSTN, PHY_CNFG_R);
+
+	eswin_sdhci_config_phy_delay(host, phy->delay_code);
+
+	eswin_sdhci_enable_card_clk(host);
+}
+
+static void eswin_sdhci_reset(struct sdhci_host *host, u8 mask)
+{
+	u8 ctrl;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
+	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
+	sdhci_reset(host, mask);
+	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
+	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+
+	if (eswin_sdhci->quirks & SDHCI_ESWIN_QUIRK_FORCE_CDTEST) {
+		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
+		ctrl |= SDHCI_CTRL_CDTEST_INS | SDHCI_CTRL_CDTEST_EN;
+		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
+	}
+
+	if (mask == SDHCI_RESET_ALL) { // after reset all,the phy`s config will be clear.
+		eswin_sdhci_config_phy(host);
+	}
+}
+
+static u32 eswin_sdhci_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+static void eswin_sdhci_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static void eswin_sdhci_cqe_enable(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 reg;
+
+	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	while (reg & SDHCI_DATA_AVAILABLE) {
+		sdhci_readl(host, SDHCI_BUFFER);
+		reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	}
+
+	sdhci_cqe_enable(mmc);
+}
+
+static int eswin_sdhci_delay_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int ret;
+	int delay = 0;
+	int i = 0;
+	int delay_min = -1;
+	int delay_max = -1;
+	int cmd_error = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	for (i = 0; i <= PHY_DELAY_CODE_MAX; i++) {
+		eswin_sdhci_disable_card_clk(host);
+		eswin_sdhci_config_phy_delay(host, i);
+		eswin_sdhci_enable_card_clk(host);
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		if (ret) {
+			host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+			udelay(200);
+			if (delay_min != -1 && delay_max != -1) {
+				if (delay_max < 15 && (delay_max - delay_min) < 6) {
+					/* Dropping initial small bogus window */
+					delay_min = delay_max = -1;
+				} else {
+					break;
+				}
+			}
+		} else {
+			if (delay_min == -1) {
+				delay_min = i;
+				continue;
+			} else {
+				delay_max = i;
+				continue;
+			}
+		}
+	}
+	if (delay_min == -1 && delay_max == -1) {
+		pr_err("%s: delay code tuning failed!\n",
+					mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		eswin_sdhci_config_phy_delay(host, eswin_sdhci->phy.delay_code);
+		eswin_sdhci_enable_card_clk(host);
+
+		return ret;
+	}
+
+	delay = (delay_min + delay_max) / 2;
+	pr_debug("%s: set delay:0x%x\n", mmc_hostname(host->mmc), delay);
+	eswin_sdhci_disable_card_clk(host);
+	eswin_sdhci_config_phy_delay(host, delay);
+	eswin_sdhci_enable_card_clk(host);
+
+	return 0;
+}
+
+static int eswin_sdhci_phase_code_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int cmd_error = 0;
+	int ret = 0;
+	int phase_code = 0;
+	int code_min = -1;
+	int code_max = -1;
+
+	for (phase_code = 0; phase_code <= MAX_PHASE_CODE; phase_code++) {
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		if (ret) {
+			host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+			udelay(200);
+			if (code_min != -1 && code_max != -1)
+				break;
+		} else {
+			if (code_min == -1) {
+				code_min = phase_code;
+				continue;
+			} else {
+				code_max = phase_code;
+				continue;
+			}
+		}
+	}
+	if (code_min == -1 && code_max == -1) {
+		pr_err("%s: phase code tuning failed!\n",
+					mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+		return -EIO;
+	}
+
+	phase_code = (code_min + code_max) / 2;
+	pr_debug("%s: set phase_code:0x%x\n", mmc_hostname(host->mmc), phase_code);
+
+	eswin_sdhci_disable_card_clk(host);
+	sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+	eswin_sdhci_enable_card_clk(host);
+
+	return 0;
+}
+
+static int eswin_sdhci_executing_tuning(struct sdhci_host *host, u32 opcode)
+{
+	u32 ctrl;
+	u32 val;
+	int ret = 0;
+
+	eswin_sdhci_disable_card_clk(host);
+
+	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
+	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
+
+	val = sdhci_readl(host, VENDOR_AT_CTRL_R);
+	val |= SW_TUNE_ENABLE;
+	sdhci_writew(host, val, VENDOR_AT_CTRL_R);
+	sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+
+	eswin_sdhci_enable_card_clk(host);
+
+	sdhci_writew(host, 0x0, SDHCI_CMD_DATA);
+
+	ret = eswin_sdhci_delay_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+	ret = eswin_sdhci_phase_code_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+
+	return 0;
+}
+
+void eswin_sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
+{
+	u32 val;
+	u32 status;
+	u32 timeout = 0;
+	u16 ctrl_2;
+
+	ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	/* Select Bus Speed Mode for host */
+	ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
+	if ((timing == MMC_TIMING_MMC_HS200) ||
+	    (timing == MMC_TIMING_UHS_SDR104))
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
+	else if (timing == MMC_TIMING_UHS_SDR12)
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
+	else if (timing == MMC_TIMING_UHS_SDR25)
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
+	else if (timing == MMC_TIMING_UHS_SDR50)
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
+	else if ((timing == MMC_TIMING_UHS_DDR50) ||
+		 (timing == MMC_TIMING_MMC_DDR52))
+		ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
+	else if (timing == MMC_TIMING_MMC_HS400)
+		ctrl_2 |= ESWIN_SDHCI_CTRL_HS400; /* Non-standard */
+	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
+
+	/*
+	 * here need make dll locked when in hs400 at 200MHz
+	 */
+	if ((timing == MMC_TIMING_MMC_HS400) && (host->clock == 200000000)) {
+		eswin_sdhci_disable_card_clk(host);
+
+		val = sdhci_readl(host, VENDOR_AT_CTRL_R);
+		val &= ~(LATENCY_LT_MASK << LATENCY_LT_BIT_OFFSET);
+		val |= (LATENCY_LT_3 << LATENCY_LT_MASK);
+		sdhci_writel(host, val, VENDOR_AT_CTRL_R);
+
+		sdhci_writeb(host, 0x23, PHY_DLL_CNFG1_R);
+		sdhci_writeb(host, 0x02, PHY_DLL_CNFG2_R);
+		sdhci_writeb(host, 0x60, PHY_DLLDL_CNFG_R);
+		sdhci_writeb(host, 0x00, PHY_DLL_OFFST_R);
+		sdhci_writew(host, 0xffff, PHY_DLLBT_CNFG_R);
+
+		eswin_sdhci_enable_card_clk(host);
+		sdhci_writeb(host, DLL_ENABEL, PHY_DLL_CTRL_R);
+		udelay(100);
+
+		while (1) {
+			status = sdhci_readb(host, PHY_DLL_STATUS_R);
+			if (status & DLL_LOCK_STS) {
+				pr_debug("%s: locked status:0x%x\n", mmc_hostname(host->mmc), status);
+				break;
+			}
+			timeout++;
+			udelay(100);
+			if (timeout > 10000) {
+				pr_err("%s: DLL lock failed!status:0x%x\n",
+					mmc_hostname(host->mmc), status);
+				return;
+			}
+		}
+
+		status = sdhci_readb(host, PHY_DLL_STATUS_R);
+		if (status & DLL_ERROR_STS) {
+			pr_err("%s: DLL lock failed!err_status:0x%x\n",
+				mmc_hostname(host->mmc), status);
+		} else {
+			pr_debug("%s: DLL lock is success\n", mmc_hostname(host->mmc));
+		}
+	}
+}
+
+static const struct cqhci_host_ops eswin_sdhci_cqhci_ops = {
+	.enable = eswin_sdhci_cqe_enable,
+	.disable = sdhci_cqe_disable,
+	.dumpregs = eswin_sdhci_dumpregs,
+};
+
+static const struct sdhci_ops eswin_sdhci_cqe_ops = {
+	.set_clock = eswin_sdhci_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = eswin_sdhci_reset,
+	.set_uhs_signaling = eswin_sdhci_set_uhs_signaling,
+	.set_power = sdhci_set_power_and_bus_voltage,
+	.irq = eswin_sdhci_cqhci_irq,
+	.platform_execute_tuning = eswin_sdhci_executing_tuning,
+	.dump_vendor_regs = eswin_sdhci_dump_vendor_regs,
+};
+
+static const struct sdhci_pltfm_data eswin_sdhci_cqe_pdata = {
+	.ops = &eswin_sdhci_cqe_ops,
+	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+		SDHCI_QUIRK_BROKEN_TIMEOUT_VAL,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+#if defined(__DISABLE_HS200)
+		SDHCI_QUIRK2_BROKEN_HS200 |
+#endif
+		SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
+};
+
+#ifdef CONFIG_PM_SLEEP
+/**
+ * eswin_sdhci_suspend - Suspend method for the driver
+ * @dev:    Address of the device structure
+ *
+ * Put the device in a low power state.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	ret = sdhci_suspend_host(host);
+	if (ret)
+		return ret;
+
+	win2030_tbu_power(dev, false);
+	clk_disable_unprepare(pltfm_host->clk);
+	clk_disable_unprepare(eswin_sdhci->clk_ahb);
+
+	return 0;
+}
+
+/**
+ * eswin_sdhci_resume - Resume method for the driver
+ * @dev:    Address of the device structure
+ *
+ * Resume operation after suspend
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_prepare_enable(eswin_sdhci->clk_ahb);
+	if (ret) {
+		dev_err(dev, "can't enable clk_ahb\n");
+		return ret;
+	}
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret) {
+		dev_err(dev, "can't enable mainck\n");
+		goto clk_ahb_disable;
+	}
+	win2030_tbu_power(dev, true);
+
+	ret = sdhci_resume_host(host);
+	if (ret) {
+		dev_err(dev, "runtime resume failed!\n");
+		goto clk_disable;
+	}
+
+	return 0;
+clk_disable:
+	clk_disable_unprepare(pltfm_host->clk);
+clk_ahb_disable:
+	clk_disable_unprepare(eswin_sdhci->clk_ahb);
+
+	return ret;
+}
+
+#endif /* ! CONFIG_PM_SLEEP */
+
+/**
+ * eswin_sdhci_sdcardclk_recalc_rate - Return the card clock rate
+ *
+ * @hw:         Pointer to the hardware clock structure.
+ * @parent_rate:        The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the SD card clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The card clock rate.
+ */
+static unsigned long
+eswin_sdhci_sdcardclk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		container_of(hw, struct eswin_sdhci_clk_data, sdcardclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sdcardclk_ops = {
+	.recalc_rate = eswin_sdhci_sdcardclk_recalc_rate,
+};
+
+/**
+ * eswin_sdhci_sampleclk_recalc_rate - Return the sampling clock rate
+ *
+ * @hw:         Pointer to the hardware clock structure.
+ * @parent_rate:        The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the sampling clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The sample clock rate.
+ */
+static unsigned long
+eswin_sdhci_sampleclk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		container_of(hw, struct eswin_sdhci_clk_data, sampleclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sampleclk_ops = {
+	.recalc_rate = eswin_sdhci_sampleclk_recalc_rate,
+};
+
+static const struct eswin_sdhci_clk_ops eswin_clk_ops = {
+	.sdcardclk_ops = &eswin_sdcardclk_ops,
+	.sampleclk_ops = &eswin_sampleclk_ops,
+};
+
+static struct eswin_sdhci_of_data eswin_sdhci_fu800_data = {
+	.pdata = &eswin_sdhci_cqe_pdata,
+	.clk_ops = &eswin_clk_ops,
+};
+
+static const struct of_device_id eswin_sdhci_of_match[] = {
+	{
+		.compatible = "eswin,emmc-sdhci-5.1",
+		.data = &eswin_sdhci_fu800_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, eswin_sdhci_of_match);
+
+/**
+ * eswin_sdhci_register_sdcardclk - Register the sdcardclk for a PHY to use
+ *
+ * @eswin_sdhci:    Our private data structure.
+ * @clk_xin:        Pointer to the functional clock
+ * @dev:        Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_register_sdcardclk(struct eswin_sdhci_data *eswin_sdhci,
+									struct clk *clk_xin, struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sdcardclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 0,
+						&sdcardclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sdcardclk_init.parent_names = &parent_clk_name;
+	sdcardclk_init.num_parents = 1;
+	sdcardclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sdcardclk_init.ops = eswin_sdhci->clk_ops->sdcardclk_ops;
+
+	clk_data->sdcardclk_hw.init = &sdcardclk_init;
+	clk_data->sdcardclk = devm_clk_register(dev, &clk_data->sdcardclk_hw);
+	if (IS_ERR(clk_data->sdcardclk))
+		return PTR_ERR(clk_data->sdcardclk);
+	clk_data->sdcardclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+							clk_data->sdcardclk);
+	if (ret)
+		dev_err(dev, "Failed to add sdcard clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_register_sampleclk - Register the sampleclk for a PHY to use
+ *
+ * @eswin_sdhci:    Our private data structure.
+ * @clk_xin:        Pointer to the functional clock
+ * @dev:        Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_register_sampleclk(struct eswin_sdhci_data *eswin_sdhci,
+						struct clk *clk_xin, struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sampleclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 1,
+						&sampleclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sampleclk_init.parent_names = &parent_clk_name;
+	sampleclk_init.num_parents = 1;
+	sampleclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sampleclk_init.ops = eswin_sdhci->clk_ops->sampleclk_ops;
+
+	clk_data->sampleclk_hw.init = &sampleclk_init;
+	clk_data->sampleclk = devm_clk_register(dev, &clk_data->sampleclk_hw);
+	if (IS_ERR(clk_data->sampleclk))
+		return PTR_ERR(clk_data->sampleclk);
+	clk_data->sampleclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+				  clk_data->sampleclk);
+	if (ret)
+		dev_err(dev, "Failed to add sample clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_unregister_sdclk - Undoes eswin_sdhci_register_sdclk()
+ *
+ * @dev:        Pointer to our struct device.
+ *
+ * Should be called any time we're exiting and eswin_sdhci_register_sdclk()
+ * returned success.
+ */
+static void eswin_sdhci_unregister_sdclk(struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+
+	if (!of_find_property(np, "#clock-cells", NULL))
+		return;
+
+	of_clk_del_provider(dev->of_node);
+}
+
+/**
+ * eswin_sdhci_register_sdclk - Register the sdcardclk for a PHY to use
+ *
+ * @eswin_sdhci:    Our private data structure.
+ * @clk_xin:        Pointer to the functional clock
+ * @dev:        Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Note: without seriously re-architecting SDHCI's clock code and testing on
+ * all platforms, there's no way to create a totally beautiful clock here
+ * with all clock ops implemented.  Instead, we'll just create a clock that can
+ * be queried and set the CLK_GET_RATE_NOCACHE attribute to tell common clock
+ * framework that we're doing things behind its back.  This should be sufficient
+ * to create nice clean device tree bindings and later (if needed) we can try
+ * re-architecting SDHCI if we see some benefit to it.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_register_sdclk(struct eswin_sdhci_data *eswin_sdhci,
+				      struct clk *clk_xin, struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	u32 num_clks = 0;
+	int ret;
+
+	/* Providing a clock to the PHY is optional; no error if missing */
+	if (of_property_read_u32(np, "#clock-cells", &num_clks) < 0)
+		return 0;
+
+	ret = eswin_sdhci_register_sdcardclk(eswin_sdhci, clk_xin, dev);
+	if (ret)
+		return ret;
+
+	if (num_clks) {
+		ret = eswin_sdhci_register_sampleclk(eswin_sdhci, clk_xin, dev);
+		if (ret) {
+			eswin_sdhci_unregister_sdclk(dev);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int eswin_sdhci_add_host(struct eswin_sdhci_data *eswin_sdhci)
+{
+	struct sdhci_host *host = eswin_sdhci->host;
+	struct cqhci_host *cq_host;
+	bool dma64;
+	int ret;
+
+	if (!eswin_sdhci->has_cqe)
+		return sdhci_add_host(host);
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	cq_host = devm_kzalloc(host->mmc->parent, sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	cq_host->mmio = host->ioaddr + eswin_sdhci_CQE_BASE_ADDR;
+	cq_host->ops = &eswin_sdhci_cqhci_ops;
+
+	dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+	if (dma64)
+		cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+	ret = cqhci_init(cq_host, host->mmc, dma64);
+	if (ret)
+		goto cleanup;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
+static int eswin_emmc_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_emmc_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev,
+			"dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+					 &hsp_mmu_emmc_reg);
+	if (ret) {
+		dev_err(dev, "can't get emmc sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_emmc_reg, rdwr_sid_ssid);
+
+	ret = win2030_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config emmc streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config emmc streamID(%d)!\n", sid);
+
+	return ret;
+}
+
+static int eswin_sdhci_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct clk *clk_xin;
+	struct sdhci_host *host;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct device *dev = &pdev->dev;
+	struct eswin_sdhci_data *eswin_sdhci;
+	const struct eswin_sdhci_of_data *data;
+	unsigned int val = 0;
+
+	data = of_device_get_match_data(dev);
+	host = sdhci_pltfm_init(pdev, data->pdata, sizeof(*eswin_sdhci));
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	eswin_sdhci->host = host;
+	eswin_sdhci->clk_ops = data->clk_ops;
+
+	eswin_sdhci->clk_ahb = devm_clk_get(dev, "clk_ahb");
+	if (IS_ERR(eswin_sdhci->clk_ahb)) {
+		ret = dev_err_probe(dev, PTR_ERR(eswin_sdhci->clk_ahb),
+				    "clk_ahb clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	clk_xin = devm_clk_get(dev, "clk_xin");
+	if (IS_ERR(clk_xin)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_xin),
+				    "clk_xin clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	ret = clk_prepare_enable(eswin_sdhci->clk_ahb);
+	if (ret) {
+		dev_err(dev, "Unable to enable AHB clock.\n");
+		goto err_pltfm_free;
+	}
+
+	ret = clk_prepare_enable(clk_xin);
+	if (ret) {
+		dev_err(dev, "Unable to enable SD clock.\n");
+		goto clk_dis_ahb;
+	}
+
+	ret = eswin_sdhci_reset_init(dev, eswin_sdhci);
+	if (ret < 0) {
+		dev_err(dev, "failed to reset\n");
+		goto clk_disable_all;
+	}
+
+	win2030_tbu_power(dev, true);
+
+	eswin_sdhci->crg_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,syscrg_csr");
+	if (IS_ERR(eswin_sdhci->crg_regmap)){
+		dev_dbg(&pdev->dev, "No syscrg_csr phandle specified\n");
+		goto clk_disable_all;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 1,
+                                    &eswin_sdhci->crg_core_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get crg_core_clk (%d)\n", ret);
+		goto clk_disable_all;
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 2,
+                                    &eswin_sdhci->crg_aclk_ctrl);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get crg_aclk_ctrl (%d)\n", ret);
+		goto clk_disable_all;
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 3,
+                                    &eswin_sdhci->crg_cfg_ctrl);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get crg_cfg_ctrl (%d)\n", ret);
+		goto clk_disable_all;
+	}
+
+	eswin_sdhci->hsp_regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(eswin_sdhci->hsp_regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		goto clk_disable_all;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 2,
+                                    &eswin_sdhci->hsp_int_status);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_int_status (%d)\n", ret);
+		goto clk_disable_all;
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 3,
+                                    &eswin_sdhci->hsp_pwr_ctrl);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_pwr_ctrl (%d)\n", ret);
+		goto clk_disable_all;
+	}
+
+	regmap_write(eswin_sdhci->hsp_regmap, eswin_sdhci->hsp_int_status, MSHC_INT_CLK_STABLE);
+	regmap_write(eswin_sdhci->hsp_regmap, eswin_sdhci->hsp_pwr_ctrl, MSHC_HOST_VAL_STABLE);
+
+	/* smmu */
+	eswin_emmc_sid_cfg(dev);
+
+	if (!of_property_read_u32(dev->of_node, "delay_code", &val)) {
+		eswin_sdhci->phy.delay_code = val;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val))
+		eswin_sdhci->phy.drive_impedance =
+			eswin_convert_drive_impedance_ohm(pdev, val);
+
+	if (of_property_read_bool(dev->of_node, "enable-cmd-pullup"))
+		eswin_sdhci->phy.enable_cmd_pullup = ENABLE;
+	else
+		eswin_sdhci->phy.enable_cmd_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable-data-pullup"))
+		eswin_sdhci->phy.enable_data_pullup = ENABLE;
+	else
+		eswin_sdhci->phy.enable_data_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable-strobe-pulldown"))
+		eswin_sdhci->phy.enable_strobe_pulldown = ENABLE;
+	else
+		eswin_sdhci->phy.enable_strobe_pulldown = DISABLE;
+
+	sdhci_get_of_property(pdev);
+
+	pltfm_host->clk = clk_xin;
+
+	ret = eswin_sdhci_register_sdclk(eswin_sdhci, clk_xin, dev);
+	if (ret)
+		goto clk_disable_all;
+
+	eswin_sdhci_dt_parse_clk_phases(dev, &eswin_sdhci->clk_data);
+
+	ret = mmc_of_parse(host->mmc);
+	if (ret) {
+		ret = dev_err_probe(dev, ret, "parsing dt failed.\n");
+		goto unreg_clk;
+	}
+
+	if (of_device_is_compatible(dev->of_node, "eswin,sdhci-5.1")) {
+		host->mmc_host_ops.hs400_enhanced_strobe =
+			eswin_sdhci_hs400_enhanced_strobe;
+		eswin_sdhci->has_cqe = true;
+		host->mmc->caps2 |= MMC_CAP2_CQE;
+
+		if (!of_property_read_bool(dev->of_node, "disable-cqe-dcmd"))
+			host->mmc->caps2 |= MMC_CAP2_CQE_DCMD;
+	}
+
+	sdhci_enable_v4_mode(eswin_sdhci->host);
+
+	ret = eswin_sdhci_add_host(eswin_sdhci);
+	if (ret)
+		goto unreg_clk;
+
+	return 0;
+
+unreg_clk:
+	eswin_sdhci_unregister_sdclk(dev);
+clk_disable_all:
+	clk_disable_unprepare(clk_xin);
+clk_dis_ahb:
+	clk_disable_unprepare(eswin_sdhci->clk_ahb);
+err_pltfm_free:
+	sdhci_pltfm_free(pdev);
+	return ret;
+}
+
+static int eswin_sdhci_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct clk *clk_ahb = eswin_sdhci->clk_ahb;
+
+	sdhci_pltfm_remove(pdev);
+	win2030_tbu_power(&pdev->dev, false);
+
+	if (eswin_sdhci->txrx_rst) {
+		ret = reset_control_assert(eswin_sdhci->txrx_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci->phy_rst) {
+		ret = reset_control_assert(eswin_sdhci->phy_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci->prstn) {
+		ret = reset_control_assert(eswin_sdhci->prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci->arstn) {
+		ret = reset_control_assert(eswin_sdhci->arstn);
+		WARN_ON(0 != ret);
+	}
+	eswin_sdhci_unregister_sdclk(&pdev->dev);
+	clk_disable_unprepare(clk_ahb);
+
+	return 0;
+}
+
+static void emmc_hard_reset(struct sdhci_host *host)
+{
+	unsigned int val;
+
+	val = sdhci_readw(host, VENDOR_EMMC_CTRL_R);
+	val |= EMMC_RST_N_OE;
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+	val &= ~EMMC_RST_N;
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+	mdelay(20);
+	val |= EMMC_RST_N;
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+}
+
+static void eswin_sdhci_shutdown(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+
+	if (!host)
+		return;
+
+	emmc_hard_reset(host);
+	host->ops->reset(host, SDHCI_RESET_ALL);
+	platform_set_drvdata(pdev, NULL);
+}
+
+static const struct dev_pm_ops eswin_sdhci_pmops = {
+	SET_SYSTEM_SLEEP_PM_OPS(eswin_sdhci_suspend, eswin_sdhci_resume)
+};
+
+static struct platform_driver eswin_sdhci_driver =
+{
+	.driver = {
+		.name = "sdhci-eswin",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = eswin_sdhci_of_match,
+		.pm = &eswin_sdhci_pmops,
+	},
+	.probe = eswin_sdhci_probe,
+	.remove = eswin_sdhci_remove,
+	.shutdown = eswin_sdhci_shutdown,
+};
+
+module_platform_driver(eswin_sdhci_driver);
+
+MODULE_DESCRIPTION("Driver for the ESWIN SDHCI Controller");
+MODULE_AUTHOR("liangshuang@eswin.com");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 92d7d5a00b84..7b5ff1129461 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -49,6 +49,14 @@ config DWMAC_DWC_QOS_ETH
 	help
 	  Support for chips using the snps,dwc-qos-ethernet.txt DT binding.
 
+config DWMAC_WIN2030
+	tristate "Support for Eswin WIN2030 ethernet driver"
+	select CRC32
+	select MII
+	depends on OF && HAS_DMA && ARCH_ESWIN_EIC770X_SOC_FAMILY
+	help
+	  Support for Eswin WIN2030 ethernet driver.
+
 config DWMAC_GENERIC
 	tristate "Generic driver for DWMAC"
 	default STMMAC_PLATFORM
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index 5b57aee19267..4b22312c5ea8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -33,6 +33,7 @@ obj-$(CONFIG_DWMAC_GENERIC)	+= dwmac-generic.o
 obj-$(CONFIG_DWMAC_IMX8)	+= dwmac-imx.o
 obj-$(CONFIG_DWMAC_TEGRA)	+= dwmac-tegra.o
 obj-$(CONFIG_DWMAC_VISCONTI)	+= dwmac-visconti.o
+obj-$(CONFIG_DWMAC_WIN2030)	+= dwmac-win2030.o
 stmmac-platform-objs:= stmmac_platform.o
 dwmac-altr-socfpga-objs := dwmac-socfpga.o
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-win2030.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-win2030.c
new file mode 100644
index 000000000000..54fb17a89b9f
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-win2030.c
@@ -0,0 +1,647 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Eswin DWC Ethernet linux driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: iangshuang <liangshuang@eswincomputing.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/ethtool.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_net.h>
+#include <linux/mfd/syscon.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/stmmac.h>
+#include <linux/iommu.h>
+#include "stmmac_platform.h"
+#include "dwmac4.h"
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eswin-win2030-sid-cfg.h>
+#include <linux/gpio/consumer.h>
+
+/* eth_phy_ctrl_offset eth0:0x100; eth1:0x200 */
+#define ETH_TX_CLK_SEL          BIT(16)
+#define ETH_PHY_INTF_SELI       BIT(0)
+
+/* eth_axi_lp_ctrl_offset eth0:0x108; eth1:0x208 */
+#define ETH_CSYSREQ_VAL         BIT(0)
+
+/* hsp_aclk_ctrl_offset (0x148) */
+#define HSP_ACLK_CLKEN           BIT(31)
+#define HSP_ACLK_DIVSOR          (0x2 << 4)
+
+/* hsp_cfg_ctrl_offset (0x14c) */
+#define HSP_CFG_CLKEN            BIT(31)
+#define SCU_HSP_PCLK_EN          BIT(30)
+#define HSP_CFG_CTRL_REGSET      (HSP_CFG_CLKEN | SCU_HSP_PCLK_EN)
+
+/* RTL8211F PHY Configurations for LEDs */
+#define PHY_ADDR                 0
+#define PHY_PAGE_SWITCH_REG      31
+#define PHY_LED_CFG_REG          16
+#define PHY_LED_PAGE_CFG         0xd04
+
+#define AWSMMUSID   GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID  GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID   GENMASK(15, 8)  // The sid of read operation
+#define ARSMMUSSID  GENMASK(7, 0)   // The ssid of read operation
+
+struct dwc_qos_priv {
+	struct device *dev;
+	int dev_id;
+	struct regmap *crg_regmap;
+	struct regmap *hsp_regmap;
+	struct reset_control *rst;
+	struct clk *clk_app;
+	struct clk *clk_tx;
+	struct regmap *rgmii_sel;
+	struct gpio_desc *phy_reset;
+	struct stmmac_priv *stmpriv;
+	int phyaddr;
+	int phyled_cfgs[3];
+	unsigned int dly_hsp_reg[3];
+	unsigned int dly_param_1000m[3];
+	unsigned int dly_param_100m[3];
+	unsigned int dly_param_10m[3];
+};
+
+static int dwc_eth_dwmac_config_dt(struct platform_device *pdev,
+				   struct plat_stmmacenet_data *plat_dat)
+{
+	struct device *dev = &pdev->dev;
+	u32 burst_map = 0;
+	u32 bit_index = 0;
+	u32 a_index = 0;
+
+	if (!plat_dat->axi) {
+		plat_dat->axi = kzalloc(sizeof(struct stmmac_axi), GFP_KERNEL);
+
+		if (!plat_dat->axi)
+			return -ENOMEM;
+	}
+
+	plat_dat->axi->axi_lpi_en = device_property_read_bool(dev,
+							      "snps,en-lpi");
+	if (device_property_read_u32(dev, "snps,write-requests",
+				     &plat_dat->axi->axi_wr_osr_lmt)) {
+		/**
+		 * Since the register has a reset value of 1, if property
+		 * is missing, default to 1.
+		 */
+		plat_dat->axi->axi_wr_osr_lmt = 1;
+	} else {
+		/**
+		 * If property exists, to keep the behavior from dwc_eth_qos,
+		 * subtract one after parsing.
+		 */
+		plat_dat->axi->axi_wr_osr_lmt--;
+	}
+
+	if (device_property_read_u32(dev, "snps,read-requests",
+				     &plat_dat->axi->axi_rd_osr_lmt)) {
+		/**
+		 * Since the register has a reset value of 1, if property
+		 * is missing, default to 1.
+		 */
+		plat_dat->axi->axi_rd_osr_lmt = 1;
+	} else {
+		/**
+		 * If property exists, to keep the behavior from dwc_eth_qos,
+		 * subtract one after parsing.
+		 */
+		plat_dat->axi->axi_rd_osr_lmt--;
+	}
+	device_property_read_u32(dev, "snps,burst-map", &burst_map);
+
+	/* converts burst-map bitmask to burst array */
+	for (bit_index = 0; bit_index < 7; bit_index++) {
+		if (burst_map & (1 << bit_index)) {
+			switch (bit_index) {
+			case 0:
+			plat_dat->axi->axi_blen[a_index] = 4; break;
+			case 1:
+			plat_dat->axi->axi_blen[a_index] = 8; break;
+			case 2:
+			plat_dat->axi->axi_blen[a_index] = 16; break;
+			case 3:
+			plat_dat->axi->axi_blen[a_index] = 32; break;
+			case 4:
+			plat_dat->axi->axi_blen[a_index] = 64; break;
+			case 5:
+			plat_dat->axi->axi_blen[a_index] = 128; break;
+			case 6:
+			plat_dat->axi->axi_blen[a_index] = 256; break;
+			default:
+			break;
+			}
+			a_index++;
+		}
+	}
+
+	/* dwc-qos needs GMAC4, AAL, TSO and PMT */
+	plat_dat->has_gmac4 = 1;
+	plat_dat->dma_cfg->aal = 1;
+    plat_dat->flags |= STMMAC_FLAG_TSO_EN;
+	plat_dat->pmt = 1;
+
+	return 0;
+}
+
+static int eswin_eth_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_eth_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+				&hsp_mmu_eth_reg);
+	if (ret) {
+		dev_err(dev, "can't get eth sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_eth_reg, rdwr_sid_ssid);
+
+	ret = win2030_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config eth streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config eth streamID(%d)!\n", sid);
+
+	return ret;
+}
+
+static void dwc_qos_fix_speed(void *priv, unsigned int speed, unsigned int mode)
+{
+	unsigned long rate = 125000000;
+	int i, err, data = 0;
+	struct dwc_qos_priv *dwc_priv = (struct dwc_qos_priv *)priv;
+
+	switch (speed) {
+	case SPEED_1000:
+		rate = 125000000;
+
+		for (i = 0; i < 3; i++) {
+			regmap_write(dwc_priv->hsp_regmap, dwc_priv->dly_hsp_reg[i], dwc_priv->dly_param_1000m[i]);
+		}
+
+		if (dwc_priv->stmpriv) {
+			data = mdiobus_read(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, PHY_LED_PAGE_CFG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_LED_CFG_REG, dwc_priv->phyled_cfgs[0]);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, data);
+		}
+
+		break;
+	case SPEED_100:
+		rate = 25000000;
+
+		for (i = 0; i < 3; i++) {
+			regmap_write(dwc_priv->hsp_regmap, dwc_priv->dly_hsp_reg[i], dwc_priv->dly_param_100m[i]);
+		}
+
+		if (dwc_priv->stmpriv) {
+			data = mdiobus_read(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, PHY_LED_PAGE_CFG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_LED_CFG_REG, dwc_priv->phyled_cfgs[1]);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, data);
+		}
+
+		break;
+	case SPEED_10:
+		rate = 2500000;
+
+		for (i = 0; i < 3; i++) {
+			regmap_write(dwc_priv->hsp_regmap, dwc_priv->dly_hsp_reg[i], dwc_priv->dly_param_10m[i]);
+		}
+
+		if (dwc_priv->stmpriv) {
+			data = mdiobus_read(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, PHY_LED_PAGE_CFG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_LED_CFG_REG, dwc_priv->phyled_cfgs[2]);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, data);
+		}
+
+		break;
+	default:
+		dev_err(dwc_priv->dev, "invalid speed %u\n", speed);
+		break;
+	}
+
+	err = clk_set_rate(dwc_priv->clk_tx, rate);
+	if (err < 0)
+	{
+		dev_err(dwc_priv->dev, "failed to set TX rate: %d\n", err);
+	}
+}
+
+static int dwc_clks_config(void *priv, bool enabled)
+{
+	int ret = 0;
+	struct dwc_qos_priv *dwc_priv = (struct dwc_qos_priv *)priv;
+
+	if (enabled) {
+		ret = clk_prepare_enable(dwc_priv->clk_app);
+		if (ret) {
+			dev_err(dwc_priv->dev, "failed to enable app clk, err = %d\n", ret);
+			return ret;
+		}
+
+		ret = clk_prepare_enable(dwc_priv->clk_tx);
+		if (ret < 0) {
+			dev_err(dwc_priv->dev, "failed to enable tx clock: %d\n", ret);
+			return ret;
+		}
+
+		ret = win2030_tbu_power(dwc_priv->dev, true);
+		if (ret) {
+			dev_err(dwc_priv->dev, "failed to power up tbu\n");
+			return ret;
+		}
+	} else {
+
+		ret = win2030_tbu_power(dwc_priv->dev, false);
+		if (ret) {
+			dev_err(dwc_priv->dev, "failed to power down tbu\n");
+			return ret;
+		}
+
+		clk_disable_unprepare(dwc_priv->clk_tx);
+		clk_disable_unprepare(dwc_priv->clk_app);
+	}
+
+	return ret;
+}
+
+static int dwc_qos_probe(struct platform_device *pdev,
+			 struct plat_stmmacenet_data *plat_dat,
+			 struct stmmac_resources *stmmac_res)
+{
+	struct dwc_qos_priv *dwc_priv;
+	int ret;
+	u32 hsp_aclk_ctrl_offset;
+	u32 hsp_aclk_ctrl_regset;
+	u32 hsp_cfg_ctrl_offset;
+	u32 eth_axi_lp_ctrl_offset;
+	u32 eth_phy_ctrl_offset;
+	u32 eth_phy_ctrl_regset;
+	u32 rgmiisel_offset;
+	u32 rgmiisel_regset;
+
+	dwc_priv = devm_kzalloc(&pdev->dev, sizeof(*dwc_priv), GFP_KERNEL);
+	if (!dwc_priv)
+		return -ENOMEM;
+
+	if (device_property_read_u32(&pdev->dev, "id", &dwc_priv->dev_id)) {
+		dev_err(&pdev->dev, "Can not read device id!\n");
+		return -EINVAL;
+	}
+
+	dwc_priv->dev = &pdev->dev;
+	dwc_priv->phy_reset = devm_gpiod_get(&pdev->dev, "rst", GPIOD_OUT_LOW);
+	if (IS_ERR(dwc_priv->phy_reset)) {
+		dev_info(&pdev->dev, "Reset gpio not specified\n");
+	}
+
+	if (dwc_priv->phy_reset)
+		gpiod_set_value(dwc_priv->phy_reset, 0);
+
+	dwc_priv->rgmii_sel = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,rgmiisel");
+	if (IS_ERR(dwc_priv->rgmii_sel)){
+		dev_dbg(&pdev->dev, "rgmiisel not specified\n");
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,phyaddr", 0, &dwc_priv->phyaddr);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get phyaddr (%d)\n", ret);
+	}
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,led-cfgs", 0, &dwc_priv->phyled_cfgs[0]);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get led cfgs for 1Gbps mode (%d)\n", ret);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,led-cfgs", 1, &dwc_priv->phyled_cfgs[1]);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get led cfgs for 100Mbps mode (%d)\n", ret);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,led-cfgs", 2, &dwc_priv->phyled_cfgs[2]);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get led cfgs for 10Mbps mode (%d)\n", ret);
+	}
+
+	ret = of_property_read_variable_u32_array(pdev->dev.of_node, "eswin,dly_hsp_reg", &dwc_priv->dly_hsp_reg[0], 3, 0);
+	if (ret != 3) {
+		dev_err(&pdev->dev, "can't get delay hsp reg.ret(%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_variable_u32_array(pdev->dev.of_node, "dly-param-1000m", &dwc_priv->dly_param_1000m[0], 3, 0);
+	if (ret != 3) {
+		dev_err(&pdev->dev, "can't get delay param for 1Gbps mode (%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_variable_u32_array(pdev->dev.of_node, "dly-param-100m", &dwc_priv->dly_param_100m[0], 3, 0);
+	if (ret != 3) {
+		dev_err(&pdev->dev, "can't get delay param for 100Mbps mode (%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_variable_u32_array(pdev->dev.of_node, "dly-param-10m", &dwc_priv->dly_param_10m[0], 3, 0);
+	if (ret != 3) {
+		dev_err(&pdev->dev, "can't get delay param for 10Mbps mode (%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,rgmiisel", 1, &rgmiisel_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get rgmiisel_offset (%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,rgmiisel", 2, &rgmiisel_regset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get rgmiisel_regset (%d)\n", ret);
+		return ret;
+	}
+
+	regmap_write(dwc_priv->rgmii_sel, rgmiisel_offset, rgmiisel_regset);
+
+	dwc_priv->crg_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,syscrg_csr");
+	if (IS_ERR(dwc_priv->crg_regmap)){
+		dev_dbg(&pdev->dev, "No syscrg_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 1,
+                                    &hsp_aclk_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_aclk_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_read(dwc_priv->crg_regmap, hsp_aclk_ctrl_offset, &hsp_aclk_ctrl_regset);
+	hsp_aclk_ctrl_regset |= (HSP_ACLK_CLKEN | HSP_ACLK_DIVSOR);
+	regmap_write(dwc_priv->crg_regmap, hsp_aclk_ctrl_offset, hsp_aclk_ctrl_regset);
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 2,
+                                    &hsp_cfg_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_cfg_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_write(dwc_priv->crg_regmap, hsp_cfg_ctrl_offset, HSP_CFG_CTRL_REGSET);
+
+	dwc_priv->hsp_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(dwc_priv->hsp_regmap)){
+		dev_dbg(&pdev->dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 2,
+                                    &eth_phy_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get eth_phy_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_read(dwc_priv->hsp_regmap, eth_phy_ctrl_offset, &eth_phy_ctrl_regset);
+	eth_phy_ctrl_regset |= (ETH_TX_CLK_SEL | ETH_PHY_INTF_SELI);
+	regmap_write(dwc_priv->hsp_regmap, eth_phy_ctrl_offset, eth_phy_ctrl_regset);
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 3,
+                                    &eth_axi_lp_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get eth_axi_lp_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_write(dwc_priv->hsp_regmap, eth_axi_lp_ctrl_offset, ETH_CSYSREQ_VAL);
+
+	dwc_priv->clk_app = devm_clk_get(&pdev->dev, "app");
+	if (IS_ERR(dwc_priv->clk_app)) {
+		dev_err(&pdev->dev, "app clock not found.\n");
+		return PTR_ERR(dwc_priv->clk_app);
+	}
+
+	dwc_priv->clk_tx = devm_clk_get(&pdev->dev, "tx");
+	if (IS_ERR(dwc_priv->clk_tx)) {
+		dev_err(&pdev->dev, "tx clock not found.\n");
+		return PTR_ERR(dwc_priv->clk_tx);
+	}
+
+	ret = dwc_clks_config(dwc_priv, true);
+	if (ret) {
+		return ret;
+	}
+
+	dwc_priv->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "ethrst");
+	if (IS_ERR(dwc_priv->rst)) {
+		return PTR_ERR(dwc_priv->rst);
+	}
+
+	ret = reset_control_assert(dwc_priv->rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(dwc_priv->rst);
+	WARN_ON(0 != ret);
+
+	ret = win2030_tbu_power(&pdev->dev, true);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to power on tbu\n");
+		return ret;
+	}
+
+	plat_dat->fix_mac_speed = dwc_qos_fix_speed;
+	plat_dat->bsp_priv = dwc_priv;
+	plat_dat->phy_addr = PHY_ADDR;
+	plat_dat->clks_config = dwc_clks_config;
+	plat_dat->bus_id = dwc_priv->dev_id;
+
+	return 0;
+}
+
+static int dwc_qos_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct dwc_qos_priv *dwc_priv = get_stmmac_bsp_priv(&pdev->dev);
+
+	ret = win2030_tbu_power(&pdev->dev, false);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to power down tbu\n");
+		return ret;
+	}
+
+	reset_control_assert(dwc_priv->rst);
+	dwc_clks_config(dwc_priv, false);
+
+	if (dwc_priv->phy_reset)
+		devm_gpiod_put(&pdev->dev, dwc_priv->phy_reset);
+
+	return 0;
+}
+
+struct dwc_eth_dwmac_data {
+	int (*probe)(struct platform_device *pdev,
+		     struct plat_stmmacenet_data *data,
+		     struct stmmac_resources *res);
+	int (*remove)(struct platform_device *pdev);
+};
+
+static const struct dwc_eth_dwmac_data dwc_qos_data = {
+	.probe = dwc_qos_probe,
+	.remove = dwc_qos_remove,
+};
+
+static int dwc_eth_dwmac_probe(struct platform_device *pdev)
+{
+	const struct dwc_eth_dwmac_data *data;
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct net_device *ndev = NULL;
+	struct stmmac_priv *stmpriv = NULL;
+	struct dwc_qos_priv *dwc_priv = NULL;
+	int ret;
+
+	data = device_get_match_data(&pdev->dev);
+
+	memset(&stmmac_res, 0, sizeof(struct stmmac_resources));
+
+	/**
+	 * Since stmmac_platform supports name IRQ only, basic platform
+	 * resource initialization is done in the glue logic.
+	 */
+	stmmac_res.irq = platform_get_irq(pdev, 0);
+	if (stmmac_res.irq < 0)
+		return stmmac_res.irq;
+	stmmac_res.wol_irq = stmmac_res.irq;
+	stmmac_res.addr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(stmmac_res.addr))
+		return PTR_ERR(stmmac_res.addr);
+
+	plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	ret = data->probe(pdev, plat_dat, &stmmac_res);
+	if (ret < 0) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to probe subdriver: %d\n",
+				ret);
+
+		goto remove_config;
+	}
+
+	ret = dwc_eth_dwmac_config_dt(pdev, plat_dat);
+	if (ret)
+		goto remove;
+
+    ret =  eswin_eth_sid_cfg(&pdev->dev);
+	if (ret)
+		goto remove;
+
+	dwc_priv = (struct dwc_qos_priv *)plat_dat->bsp_priv;
+		plat_dat->phy_addr = dwc_priv->phyaddr;
+
+	ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
+	if (ret)
+		goto remove;
+
+	ndev = dev_get_drvdata(&pdev->dev);
+	stmpriv = netdev_priv(ndev);
+
+	dwc_priv->stmpriv = stmpriv;
+
+	return ret;
+
+remove:
+	data->remove(pdev);
+remove_config:
+	stmmac_remove_config_dt(pdev, plat_dat);
+
+	return ret;
+}
+
+static int dwc_eth_dwmac_remove(struct platform_device *pdev)
+{
+	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	const struct dwc_eth_dwmac_data *data;
+	int err;
+
+	data = device_get_match_data(&pdev->dev);
+
+	stmmac_dvr_remove(&pdev->dev);
+
+	err = data->remove(pdev);
+	if (err < 0)
+		dev_err(&pdev->dev, "failed to remove subdriver: %d\n", err);
+
+	stmmac_remove_config_dt(pdev, priv->plat);
+
+	return err;
+}
+
+static const struct of_device_id dwc_eth_dwmac_match[] = {
+	{ .compatible = "eswin,win2030-qos-eth", .data = &dwc_qos_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, dwc_eth_dwmac_match);
+
+static struct platform_driver win2030_eth_dwmac_driver = {
+	.probe  = dwc_eth_dwmac_probe,
+	.remove = dwc_eth_dwmac_remove,
+	.driver = {
+		.name           = "win2030-eth-dwmac",
+		.pm             = &stmmac_pltfr_pm_ops,
+		.of_match_table = dwc_eth_dwmac_match,
+	},
+};
+module_platform_driver(win2030_eth_dwmac_driver);
+
+MODULE_AUTHOR("Eswin");
+MODULE_DESCRIPTION("Eswin win2030 qos ethernet driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig
index 7555af5195ec..1890822520d1 100644
--- a/drivers/net/wireless/Kconfig
+++ b/drivers/net/wireless/Kconfig
@@ -37,6 +37,7 @@ source "drivers/net/wireless/st/Kconfig"
 source "drivers/net/wireless/ti/Kconfig"
 source "drivers/net/wireless/zydas/Kconfig"
 source "drivers/net/wireless/quantenna/Kconfig"
+source "drivers/net/wireless/ap12275/Kconfig"
 
 source "drivers/net/wireless/legacy/Kconfig"
 
diff --git a/drivers/net/wireless/Makefile b/drivers/net/wireless/Makefile
index 4d7374d567d1..17277ec302c4 100644
--- a/drivers/net/wireless/Makefile
+++ b/drivers/net/wireless/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_WLAN_VENDOR_SILABS) += silabs/
 obj-$(CONFIG_WLAN_VENDOR_ST) += st/
 obj-$(CONFIG_WLAN_VENDOR_TI) += ti/
 obj-$(CONFIG_WLAN_VENDOR_ZYDAS) += zydas/
+obj-$(CONFIG_BCMDHD) += ap12275/
 
 obj-$(CONFIG_WLAN) += legacy/
 obj-$(CONFIG_WLAN) += virtual/
diff --git a/drivers/net/wireless/ap12275/Kconfig b/drivers/net/wireless/ap12275/Kconfig
new file mode 100644
index 000000000000..3ae12151a66c
--- /dev/null
+++ b/drivers/net/wireless/ap12275/Kconfig
@@ -0,0 +1,89 @@
+#
+# Copyright (C) 2023, Broadcom.
+#
+#      Unless you and Broadcom execute a separate written software license
+# agreement governing use of this software, this software is licensed to you
+# under the terms of the GNU General Public License version 2 (the "GPL"),
+# available at http://www.broadcom.com/licenses/GPLv2.php, with the
+# following added to such license:
+#
+#      As a special exception, the copyright holders of this software give you
+# permission to link this software with independent modules, and to copy and
+# distribute the resulting executable under terms of your choice, provided that
+# you also meet, for each linked independent module, the terms and conditions of
+# the license of that module.  An independent module is a module which is not
+# derived from this software.  The special exception does not apply to any
+# modifications of the software.
+#
+#
+# <<Broadcom-WL-IPTag/Open:>>
+#
+
+config BCMDHD
+	tristate "Broadcom FullMAC wireless cards support"
+	default m
+	help
+	  This module adds support for wireless adapters based on
+	  Broadcom FullMAC chipset.
+
+config BCMDHD_FW_PATH
+	depends on BCMDHD
+	string "Firmware path"
+	default "/lib/firmware/fw_bcm43752a2_ag.bin"
+	help
+	  Path to the firmware file.
+
+config BCMDHD_NVRAM_PATH
+	depends on BCMDHD
+	string "NVRAM path"
+	default "/lib/firmware/nvram_ap12275_m2.txt"
+	help
+	  Path to the calibration file.
+
+config BCMDHD_CLM_PATH
+	  depends on BCMDHD
+	  string "CLM path"
+	  default "/lib/firmware/clm_bcm43752a2_ag.blob"
+	  help
+		Path to the CLM file.
+
+config BCMDHD_WEXT
+	bool "Enable WEXT support"
+	depends on BCMDHD && CFG80211 = n
+	select WIRELESS_EXT
+	select WEXT_PRIV
+	help
+	  Enables WEXT support
+
+choice
+	prompt "Enable Chip Interface"
+	depends on BCMDHD
+	help
+		Enable Chip Interface.
+config BCMDHD_SDIO
+		bool "SDIO bus interface support"
+		depends on BCMDHD && MMC
+config BCMDHD_PCIE
+		bool "PCIe bus interface support"
+		depends on BCMDHD && PCI
+config BCMDHD_USB
+		bool "USB bus interface support"
+		depends on BCMDHD && USB
+endchoice
+
+choice
+	depends on BCMDHD && BCMDHD_SDIO
+	prompt "Interrupt type"
+	help
+		Interrupt type
+config BCMDHD_OOB
+	depends on BCMDHD && BCMDHD_SDIO
+	bool "Out-of-Band Interrupt"
+	help
+		Interrupt from WL_HOST_WAKE.
+config BCMDHD_SDIO_IRQ
+	depends on BCMDHD && BCMDHD_SDIO
+	bool "In-Band Interrupt"
+	help
+	  Interrupt from SDIO DAT[1]
+endchoice
diff --git a/drivers/net/wireless/ap12275/Makefile b/drivers/net/wireless/ap12275/Makefile
new file mode 100644
index 000000000000..5107db727388
--- /dev/null
+++ b/drivers/net/wireless/ap12275/Makefile
@@ -0,0 +1,492 @@
+# bcmdhd
+#
+# Copyright (C) 2023, Broadcom.
+#
+#      Unless you and Broadcom execute a separate written software license
+# agreement governing use of this software, this software is licensed to you
+# under the terms of the GNU General Public License version 2 (the "GPL"),
+# available at http://www.broadcom.com/licenses/GPLv2.php, with the
+# following added to such license:
+#
+#      As a special exception, the copyright holders of this software give you
+# permission to link this software with independent modules, and to copy and
+# distribute the resulting executable under terms of your choice, provided that
+# you also meet, for each linked independent module, the terms and conditions of
+# the license of that module.  An independent module is a module which is not
+# derived from this software.  The special exception does not apply to any
+# modifications of the software.
+#
+#
+# <<Broadcom-WL-IPTag/Open:>>
+#
+
+MODULE_NAME := bcmdhd
+CONFIG_BCMDHD ?= m
+
+#CONFIG_BCMDHD_SDIO := y
+#CONFIG_BCMDHD_PCIE := y
+#CONFIG_BCMDHD_USB := y
+
+#CONFIG_BCMDHD_OOB := y
+#CONFIG_BCMDHD_CUSB := y
+#CONFIG_BCMDHD_NO_POWER_OFF := y
+CONFIG_BCMDHD_PROPTXSTATUS := y
+CONFIG_DHD_USE_STATIC_BUF := y
+CONFIG_BCMDHD_STATIC_BUF_IN_DHD := y
+#CONFIG_BCMDHD_ANDROID_VERSION := 12
+CONFIG_BCMDHD_AUTO_SELECT := y
+CONFIG_BCMDHD_DEBUG := y
+#CONFIG_BCMDHD_RECONNECT := y
+#CONFIG_BCMDHD_TIMESTAMP := y
+#CONFIG_BCMDHD_WAPI := y
+#CONFIG_BCMDHD_RANDOM_MAC := y
+#CONFIG_BCMDHD_REQUEST_FW := y
+#CONFIG_BCMDHD_FW_SIGNATURE := y
+#CONFIG_BCMDHD_DWDS := y
+
+#CONFIG_BCMDHD_MULTIPLE_DRIVER := y
+#CONFIG_BCMDHD_ADAPTER_INDEX := 0
+
+CONFIG_MACH_PLATFORM := y
+CONFIG_BCMDHD_DTS := y
+
+DHDCFLAGS = -Wall -Wstrict-prototypes -Wno-date-time                      \
+	-Wno-maybe-uninitialized                                              \
+	-Dlinux -DLINUX -DBCMDRIVER -DBCMUTILS_ERR_CODES                      \
+	-DBCMDONGLEHOST -DBCMDMA32 -DBCMFILEIMAGE                             \
+	-DDHDTHREAD -DDHD_DEBUG -DSHOW_EVENTS -DGET_OTP_MAC_ENABLE            \
+	-DWIFI_ACT_FRAME -DARP_OFFLOAD_SUPPORT -DSUPPORT_PM2_ONLY             \
+	-DPKTPRIO_OVERRIDE                                                    \
+	-DKEEP_ALIVE -DPKT_FILTER_SUPPORT -DDHDTCPACK_SUPPRESS                \
+	-DDHD_DONOT_FORWARD_BCMEVENT_AS_NETWORK_PKT -DOEM_ANDROID             \
+	-DMULTIPLE_SUPPLICANT -DTSQ_MULTIPLIER -DMFP -DDHD_8021X_DUMP         \
+	-DPOWERUP_MAX_RETRY=0 -DIFACE_HANG_FORCE_DEV_CLOSE -DWAIT_DEQUEUE     \
+	-DUSE_NEW_RSPEC_DEFS                                                  \
+	-DWL_EXT_IAPSTA -DWL_ESCAN -DCCODE_LIST -DSUSPEND_EVENT               \
+	-DKEY_INSTALL_CHECK                                                   \
+	-DENABLE_INSMOD_NO_FW_LOAD -DENODEBUG
+
+DHDOFILES = aiutils.o siutils.o sbutils.o                                 \
+	bcmutils.o bcmwifi_channels.o bcmxtlv.o bcm_app_utils.o bcmstdlib_s.o \
+	dhd_linux.o dhd_linux_platdev.o dhd_linux_sched.o dhd_pno.o           \
+	dhd_common.o dhd_ip.o dhd_linux_wq.o dhd_custom_gpio.o                \
+	bcmevent.o hndpmu.o linux_osl.o wldev_common.o wl_android.o           \
+	dhd_debug_linux.o dhd_debug.o dhd_mschdbg.o dhd_dbg_ring.o            \
+	hnd_pktq.o hnd_pktpool.o linux_pkt.o frag.o                           \
+	dhd_linux_exportfs.o dhd_linux_pktdump.o dhd_mschdbg.o linuxerrmap.o  \
+	dhd_linux_tx.o dhd_linux_rx.o fwpkg_utils.o                           \
+	dhd_config.o dhd_ccode.o wl_event.o wl_android_ext.o                  \
+	wl_iapsta.o wl_escan.o wl_timer.o
+
+ifneq ($(CONFIG_WIRELESS_EXT),)
+	DHDOFILES += wl_iw.o
+	DHDCFLAGS += -DSOFTAP -DWL_WIRELESS_EXT -DUSE_IW
+endif
+ifneq ($(CONFIG_CFG80211),)
+	DHDOFILES += wl_cfg80211.o wl_cfgscan.o wl_cfgp2p.o
+	DHDOFILES += wl_linux_mon.o wl_cfg_btcoex.o wl_cfgvendor.o
+	DHDOFILES += dhd_cfg80211.o wl_cfgvif.o wl_roam.o
+	DHDCFLAGS += -DWL_CFG80211 -DWLP2P -DWL_CFG80211_STA_EVENT
+	DHDCFLAGS += -DWL_CFG80211_GON_COLLISION
+	DHDCFLAGS += -DCONFIG_CFG80211_INTERNAL_REGDB #-DEXT_REG_INFO
+	DHDCFLAGS += -DWL_CAP_HE -DWL_6G_BAND -DWL_5P9G -DWL_P2P_6G
+	DHDCFLAGS += -DWL11U
+#	DHDCFLAGS += -DCONFIG_6GHZ_BKPORT
+	DHDCFLAGS += -DWL_IFACE_COMB_NUM_CHANNELS
+	DHDCFLAGS += -DCUSTOM_PNO_EVENT_LOCK_xTIME=10
+	DHDCFLAGS += -DWL_SUPPORT_AUTO_CHANNEL
+	DHDCFLAGS += -DWL_SUPPORT_BACKPORTED_KPATCHES
+	DHDCFLAGS += -DESCAN_RESULT_PATCH -DESCAN_BUF_OVERFLOW_MGMT
+	DHDCFLAGS += -DVSDB -DWL_CFG80211_VSDB_PRIORITIZE_SCAN_REQUEST
+	DHDCFLAGS += -DWLTDLS -DMIRACAST_AMPDU_SIZE=8
+#	DHDCFLAGS += -DHOSTAPD_BW_SUPPORT
+	DHDCFLAGS += -DWL_VIRTUAL_APSTA -DSTA_MGMT -DSOFTAP_UAPSD_OFF
+	DHDCFLAGS += -DWL_SOFTAP_ACS #-DACS_MONITOR
+	DHDCFLAGS += -DNUM_SCB_MAX_PROBE=3
+	DHDCFLAGS += -DWL_SCB_TIMEOUT=10
+#	DHDCFLAGS += -DWL_BEACON_RATE
+	DHDCFLAGS += -DEXPLICIT_DISCIF_CLEANUP
+	DHDCFLAGS += -DDHD_USE_SCAN_WAKELOCK
+	DHDCFLAGS += -DSPECIFIC_MAC_GEN_SCHEME
+	DHDCFLAGS += -DWL_IFACE_MGMT
+	DHDCFLAGS += -DSUPPORT_RSSI_SUM_REPORT
+	DHDCFLAGS += -DWLFBT -DWL_GCMP_SUPPORT -DWL_OWE -DWL_SAE_FT
+	DHDCFLAGS += -DROAM_CHANNEL_CACHE -DDHD_LOSSLESS_ROAMING
+#	DHDCFLAGS += -DWL_CFGVENDOR_SEND_HANG_EVENT
+	DHDCFLAGS += -DGTK_OFFLOAD_SUPPORT
+#	DHDCFLAGS += -DWL_STATIC_IF #-DDHD_MAX_STATIC_IFS=2
+#	DHDCFLAGS += -DWL_CLIENT_SAE
+	DHDCFLAGS += -DCONNECT_INFO_WAR -DWL_ROAM_WAR
+	DHDCFLAGS += -DVNDR_IE_WAR
+	DHDCFLAGS += -DRESTART_AP_WAR -DRXF0OVFL_REINIT_WAR
+endif
+
+#BCMDHD_SDIO
+ifneq ($(CONFIG_BCMDHD_SDIO),)
+BUS_TYPE := sdio
+DHDCFLAGS += -DBCMSDIO -DMMC_SDIO_ABORT -DUSE_SDIOFIFO_IOVAR -DBCMLXSDMMC \
+	-DSDTEST -DBDC -DDHD_USE_IDLECOUNT -DCUSTOM_SDIO_F2_BLKSIZE=256       \
+	-DBCMSDIOH_TXGLOM -DBCMSDIOH_TXGLOM_EXT -DBCMSDIOH_STATIC_COPY_BUF    \
+	-DRXFRAME_THREAD -DDHDENABLE_TAILPAD -DSUPPORT_P2P_GO_PS              \
+	-DBCMSDIO_RXLIM_POST -DBCMSDIO_TXSEQ_SYNC -DCONSOLE_DPC               \
+	-DBCMSDIO_INTSTATUS_WAR
+DHDCFLAGS += -DMMC_HW_RESET -DMMC_SW_RESET #-DBUS_POWER_RESTORE
+ifeq ($(CONFIG_BCMDHD_OOB),y)
+	DHDCFLAGS += -DOOB_INTR_ONLY -DCUSTOMER_OOB -DHW_OOB
+ifeq ($(CONFIG_BCMDHD_DISABLE_WOWLAN),y)
+	DHDCFLAGS += -DDISABLE_WOWLAN
+endif
+else
+	DHDCFLAGS += -DSDIO_ISR_THREAD
+endif
+DHDOFILES += bcmsdh.o bcmsdh_linux.o bcmsdh_sdmmc.o bcmsdh_sdmmc_linux.o  \
+	dhd_sdio.o dhd_cdc.o dhd_wlfc.o
+endif
+
+#BCMDHD_PCIE
+ifneq ($(CONFIG_BCMDHD_PCIE),)
+BUS_TYPE := pcie
+	DHDCFLAGS += -DPCIE_FULL_DONGLE -DBCMPCIE -DCUSTOM_DPC_PRIO_SETTING=-1
+	DHDCFLAGS += -DDONGLE_ENABLE_ISOLATION
+	DHDCFLAGS += -DDHD_LB -DDHD_LB_RXP -DDHD_LB_STATS -DDHD_LB_TXP
+	DHDCFLAGS += -DDHD_LB_PRIMARY_CPUS=0xF0 -DDHD_LB_SECONDARY_CPUS=0x0E
+	DHDCFLAGS += -DDHD_PKTID_AUDIT_ENABLED
+	DHDCFLAGS += -DEAPOL_PKT_PRIO -DENABLE_DHD_GRO
+	DHDCFLAGS += -DDHD_SKIP_DONGLE_RESET_IN_ATTACH
+	DHDCFLAGS += -DDHD_DONGLE_TRAP_IN_DETACH
+	DHDCFLAGS += -DFORCE_DONGLE_RESET_IN_DEVRESET_ON
+	DHDCFLAGS += -DDHD_USE_COHERENT_MEM_FOR_RING
+	DHDCFLAGS += -DCHECK_TRAP_ROT
+	DHDCFLAGS += -DINSMOD_FW_LOAD
+	DHDCFLAGS += -DCONFIG_HAS_WAKELOCK #-DDHD_DEBUG_WAKE_LOCK
+	DHDCFLAGS += -DDHD_PACKET_TIMEOUT_MS=50 -DMAX_TX_TIMEOUT=50
+ifeq ($(CONFIG_BCMDHD),y)
+	DHDCFLAGS += -DWAKEUP_KSOFTIRQD_POST_NAPI_SCHEDULE
+endif
+ifeq ($(CONFIG_BCMDHD_OOB),y)
+	DHDCFLAGS += -DCUSTOMER_OOB -DBCMPCIE_OOB_HOST_WAKE -DHW_OOB
+endif
+ifneq ($(CONFIG_PCI_MSI),)
+	DHDCFLAGS += -DDHD_MSI_SUPPORT
+endif
+DHDOFILES += dhd_pcie.o dhd_pcie_linux.o pcie_core.o dhd_flowring.o       \
+	dhd_msgbuf.o dhd_linux_lb.o
+endif
+
+#BCMDHD_USB
+ifneq ($(CONFIG_BCMDHD_USB),)
+BUS_TYPE := usb
+DHDCFLAGS += -DUSBOS_TX_THREAD -DBCMDBUS -DBCMTRXV2 -DDBUS_USB_LOOPBACK   \
+	-DBDC
+DHDCFLAGS += -DINSMOD_FW_LOAD
+DHDCFLAGS += -DBCM_REQUEST_FW
+DHDCFLAGS += -DSHOW_LOGTRACE
+DHDCFLAGS += -DWL_EXT_WOWL
+ifneq ($(CONFIG_BCMDHD_REQUEST_FW),y)
+	DHDCFLAGS += -DEXTERNAL_FW_PATH
+endif
+ifneq ($(CONFIG_BCMDHD_CUSB),)
+	DHDCFLAGS += -DBCMUSBDEV_COMPOSITE
+	CONFIG_BCMDHD_NO_POWER_OFF := y
+endif
+DHDOFILES += dbus.o dbus_usb.o dbus_usb_linux.o dhd_cdc.o dhd_wlfc.o
+endif
+
+ifeq ($(CONFIG_BCMDHD_NO_POWER_OFF),y)
+	DHDCFLAGS += -DENABLE_INSMOD_NO_FW_LOAD
+	DHDCFLAGS += -DENABLE_INSMOD_NO_POWER_OFF -DNO_POWER_OFF_AFTER_OPEN
+endif
+
+ifeq ($(CONFIG_BCMDHD_MULTIPLE_DRIVER),y)
+	DHDCFLAGS += -DBCMDHD_MDRIVER
+ifneq ($(CONFIG_BCMDHD_ADAPTER_INDEX),)
+	CONFIG_BCMDHD_STATIC_BUF_IN_DHD := y
+	MODULE_NAME := dhd$(BUS_TYPE)_$(CONFIG_BCMDHD_ADAPTER_INDEX)
+	DHDCFLAGS += -DADAPTER_IDX=$(CONFIG_BCMDHD_ADAPTER_INDEX)
+	DHDCFLAGS += -DBUS_TYPE=\"-$(BUS_TYPE)-$(CONFIG_BCMDHD_ADAPTER_INDEX)\"
+	DHDCFLAGS += -DDHD_LOG_PREFIX=\"[dhd-$(BUS_TYPE)-$(CONFIG_BCMDHD_ADAPTER_INDEX)]\"
+else
+	MODULE_NAME := dhd$(BUS_TYPE)
+	DHDCFLAGS += -DBUS_TYPE=\"-$(BUS_TYPE)\"
+	DHDCFLAGS += -DDHD_LOG_PREFIX=\"[dhd-$(BUS_TYPE)]\"
+endif
+else
+	DHDCFLAGS += -DBUS_TYPE=\"\"
+endif
+
+ifeq ($(CONFIG_BCMDHD_TIMESTAMP),y)
+	DHDCFLAGS += -DKERNEL_TIMESTAMP
+	DHDCFLAGS += -DSYSTEM_TIMESTAMP
+endif
+
+#PROPTXSTATUS
+ifeq ($(CONFIG_BCMDHD_PROPTXSTATUS),y)
+ifneq ($(CONFIG_BCMDHD_USB),)
+	DHDCFLAGS += -DPROP_TXSTATUS
+	DHDCFLAGS += -DLIMIT_BORROW -DBULK_DEQUEUE
+endif
+ifneq ($(CONFIG_BCMDHD_SDIO),)
+	DHDCFLAGS += -DPROP_TXSTATUS -DPROPTX_MAXCOUNT
+	DHDCFLAGS += -DLIMIT_BORROW -DBULK_DEQUEUE
+endif
+ifneq ($(CONFIG_CFG80211),)
+	DHDCFLAGS += -DPROP_TXSTATUS_VSDB
+endif
+endif
+
+ifeq ($(CONFIG_64BIT),y)
+	DHDCFLAGS := $(filter-out -DBCMDMA32,$(DHDCFLAGS))
+	DHDCFLAGS += -DBCMDMA64OSL
+endif
+
+# For Android VTS
+ifneq ($(CONFIG_BCMDHD_ANDROID_VERSION),)
+	DHDCFLAGS += -DANDROID_VERSION=$(CONFIG_BCMDHD_ANDROID_VERSION)
+	DHDCFLAGS += -DDHD_NOTIFY_MAC_CHANGED
+	DHDCFLAGS += -DANDROID_BKPORT
+ifneq ($(CONFIG_CFG80211),)
+	DHDCFLAGS += -DPNO_SUPPORT
+#	DHDCFLAGS += -DWL_SCHED_SCAN
+#	DHDCFLAGS += -DWL_P2P_RAND
+	DHDCFLAGS += -DGSCAN_SUPPORT -DRTT_SUPPORT
+	DHDCFLAGS += -DLINKSTAT_SUPPORT -DLINKSTAT_EXT_SUPPORT
+	DHDCFLAGS += -DCUSTOM_COUNTRY_CODE -DDHD_GET_VALID_CHANNELS
+	DHDCFLAGS += -DDEBUGABILITY -DDEBUGABILITY_DISABLE_MEMDUMP -DDBG_PKT_MON
+	DHDCFLAGS += -DDHD_LOG_DUMP -DDHD_FW_COREDUMP
+	DHDCFLAGS += -DDHD_PKT_LOGGING_DBGRING -DDHD_PKT_LOGGING
+	DHDCFLAGS += -DAPF -DNDO_CONFIG_SUPPORT -DRSSI_MONITOR_SUPPORT
+	DHDCFLAGS += -DDHD_WAKE_STATUS
+	DHDCFLAGS += -DWL_LATENCY_MODE
+	DHDOFILES += dhd_rtt.o
+	DHDOFILES += dhd_log_dump.o
+	DHDOFILES += dhd_pktlog.o
+endif
+else
+	DHDCFLAGS += -DANDROID_VERSION=0
+endif
+
+# For Debug
+ifeq ($(CONFIG_BCMDHD_DEBUG),y)
+	DHDCFLAGS += -DDHD_ARP_DUMP -DDHD_DHCP_DUMP -DDHD_ICMP_DUMP
+	DHDCFLAGS += -DDHD_DNS_DUMP -DDHD_TRX_DUMP
+	DHDCFLAGS += -DTPUT_MONITOR
+#	DHDCFLAGS += -DSCAN_SUPPRESS -DBSSCACHE
+	DHDCFLAGS += -DCHECK_DOWNLOAD_FW
+	DHDCFLAGS += -DPKT_STATICS
+	DHDCFLAGS += -DKSO_DEBUG
+#	DHDCFLAGS += -DDHD_PKTDUMP_TOFW
+#	DHDCFLAGS += -DDHD_DUMP_FILE_WRITE_FROM_KERNEL
+endif
+
+# For Debug2
+ifeq ($(CONFIG_BCMDHD_DEBUG2),y)
+	DHDCFLAGS += -DDEBUGFS_CFG80211
+	DHDCFLAGS += -DSHOW_LOGTRACE -DDHD_LOG_DUMP -DDHD_FW_COREDUMP
+	DHDCFLAGS += -DBCMASSERT_LOG -DSI_ERROR_ENFORCE
+ifneq ($(CONFIG_BCMDHD_PCIE),)
+	DHDCFLAGS += -DEWP_EDL
+	DHDCFLAGS += -DDNGL_EVENT_SUPPORT
+	DHDCFLAGS += -DDHD_SSSR_DUMP
+	DHDCFLAGS += -DDHD_MEM_STATS
+	DHDCFLAGS += -DDHD_FLOW_RING_STATUS_TRACE
+	DHDCFLAGS += -DDNGL_AXI_ERROR_LOGGING
+endif
+	DHDOFILES += dhd_log_dump.o
+endif
+
+# MESH support for kernel 3.10 later
+ifeq ($(CONFIG_BCMDHD_WLMESH),y)
+	DHDCFLAGS += -DWLMESH
+ifneq ($(CONFIG_CFG80211),)
+	DHDCFLAGS += -DWLMESH_CFG80211
+endif
+ifneq ($(CONFIG_BCMDHD_PCIE),)
+	DHDCFLAGS += -DBCM_HOST_BUF -DDMA_HOST_BUFFER_LEN=0x80000
+endif
+	DHDCFLAGS += -DDHD_UPDATE_INTF_MAC
+	DHDCFLAGS :=$(filter-out -DDHD_FW_COREDUMP,$(DHDCFLAGS))
+	DHDCFLAGS :=$(filter-out -DWL_STATIC_IF,$(DHDCFLAGS))
+endif
+
+# EasyMesh
+ifeq ($(CONFIG_BCMDHD_EASYMESH),y)
+	DHDCFLAGS :=$(filter-out -DDHD_FW_COREDUMP,$(DHDCFLAGS))
+	DHDCFLAGS :=$(filter-out -DDHD_LOG_DUMP,$(DHDCFLAGS))
+	DHDCFLAGS += -DWLEASYMESH
+	CONFIG_BCMDHD_DWDS := y
+endif
+
+# DWDS
+ifeq ($(CONFIG_BCMDHD_DWDS),y)
+ifneq ($(CONFIG_CFG80211),)
+	DHDCFLAGS += -DWLDWDS -DFOURADDR_AUTO_BRG
+ifneq ($(CONFIG_BCMDHD_SDIO),)
+	DHDCFLAGS += -DRXF_DEQUEUE_ON_BUSY
+endif
+	DHDCFLAGS += -DWL_STATIC_IF
+endif
+endif
+
+# CSI_SUPPORT
+ifeq ($(CONFIG_CSI_SUPPORT),y)
+	DHDCFLAGS += -DCSI_SUPPORT
+	DHDOFILES += dhd_csi.o
+endif
+
+# For CONNECTION_IMPROVE
+ifeq ($(CONFIG_BCMDHD_RECONNECT),y)
+	DHDCFLAGS += -DEAPOL_RESEND -DEAPOL_RESEND_M4
+ifneq ($(CONFIG_CFG80211),)
+	DHDCFLAGS += -DWL_EXT_RECONNECT -DWL_REASSOC_BCAST
+	DHDCFLAGS += -DWL_EXT_DISCONNECT_RECONNECT
+endif
+endif
+
+# For TPUT_IMPROVE
+ifeq ($(CONFIG_BCMDHD_TPUT),y)
+	DHDCFLAGS += -DDHD_TPUT_PATCH
+	DHDCFLAGS += -DTCPACK_INFO_MAXNUM=40 -DTCPDATA_INFO_MAXNUM=40
+ifneq ($(CONFIG_BCMDHD_SDIO),)
+	DHDCFLAGS += -DDYNAMIC_MAX_HDR_READ
+	DHDCFLAGS :=$(filter-out -DSDTEST,$(DHDCFLAGS))
+endif
+ifneq ($(CONFIG_BCMDHD_PCIE),)
+	DHDCFLAGS += -DDHD_LB_TXP_DEFAULT_ENAB
+	DHDCFLAGS += -DSET_RPS_CPUS -DSET_XPS_CPUS
+	DHDCFLAGS += -DDHD_LB_PRIMARY_CPUS=0xF0 -DDHD_LB_SECONDARY_CPUS=0x0E
+endif
+endif
+
+# For Zero configure
+ifeq ($(CONFIG_BCMDHD_ZEROCONFIG),y)
+	DHDCFLAGS += -DWL_EXT_GENL -DSENDPROB
+	DHDOFILES += wl_ext_genl.o
+endif
+
+# For WAPI
+ifeq ($(CONFIG_BCMDHD_WAPI),y)
+	DHDCFLAGS += -DBCMWAPI_WPI -DBCMWAPI_WAI
+ifeq ($(CONFIG_BCMDHD_ANDROID_VERSION),11)
+	DHDCFLAGS += -DCFG80211_WAPI_BKPORT
+endif
+endif
+
+# For scan random mac
+ifneq ($(CONFIG_BCMDHD_RANDOM_MAC),)
+ifneq ($(CONFIG_CFG80211),)
+	DHDCFLAGS += -DSUPPORT_RANDOM_MAC_SCAN -DWL_USE_RANDOMIZED_SCAN
+endif
+endif
+
+# For NAN
+ifneq ($(CONFIG_BCMDHD_NAN),)
+	DHDCFLAGS += -DWL_NAN -DWL_NAN_DISC_CACHE -DWL_NANP2P -DWL_NAN_ENABLE_MERGE
+	DHDOFILES += wl_cfgnan.o bcmbloom.o
+endif
+
+# For Module auto-selection
+ifeq ($(CONFIG_BCMDHD_AUTO_SELECT),y)
+	DHDCFLAGS += -DUPDATE_MODULE_NAME
+ifeq ($(CONFIG_BCMDHD_REQUEST_FW),y)
+#	DHDCFLAGS += -DFW_AMPAK_PATH="\"ampak\""
+#	DHDCFLAGS += -DMODULE_PATH
+endif
+ifneq ($(CONFIG_BCMDHD_SDIO),)
+	DHDCFLAGS += -DGET_OTP_MODULE_NAME -DCOMPAT_OLD_MODULE
+endif
+endif
+
+ifeq ($(CONFIG_BCMDHD),y)
+	DHDCFLAGS += -DUSE_LATE_INITCALL_SYNC
+	DHDCFLAGS += -DBCM_USE_PLATFORM_STRLCPY
+endif
+ifeq ($(CONFIG_BCMDHD),m)
+	DHDCFLAGS += -DBCMDHD_MODULAR
+	DHDCFLAGS += -DBOARD_MODULAR_INIT
+endif
+
+ifeq ($(CONFIG_MACH_PLATFORM),y)
+	DHDOFILES += dhd_gpio.o
+ifeq ($(CONFIG_BCMDHD_DTS),y)
+	DHDCFLAGS += -DBCMDHD_DTS
+endif
+	DHDCFLAGS += -DCUSTOMER_HW -DDHD_OF_SUPPORT
+endif
+
+ifeq ($(CONFIG_BCMDHD_REQUEST_FW),y)
+	DHDCFLAGS += -DDHD_LINUX_STD_FW_API
+	DHDCFLAGS += -DDHD_REQUEST_FW_PATH
+	DHDCFLAGS += -DDHD_FW_NAME="\"fw_bcmdhd.bin\""
+	DHDCFLAGS += -DDHD_NVRAM_NAME="\"nvram.txt\""
+	DHDCFLAGS += -DDHD_CLM_NAME="\"clm_bcmdhd.blob\""
+	DHDCFLAGS += -DDHD_CONFIG_NAME="\"config.txt\""
+else
+	DHDCFLAGS += -DDHD_SUPPORT_VFS_CALL
+ifeq ($(CONFIG_BCMDHD_FW_PATH),)
+	DHDCFLAGS += -DCONFIG_BCMDHD_FW_PATH="\"/lib/firmware/fw_bcm43752a2_ag.bin\""
+	DHDCFLAGS += -DCONFIG_BCMDHD_NVRAM_PATH="\"/lib/firmware/nvram_ap12275_m2.txt\""
+	DHDCFLAGS += -DCONFIG_BCMDHD_CLM_PATH="\"/lib/firmware/clm_bcm43752a2_ag.blob\""
+endif
+	DHDCFLAGS += -DCONFIG_BCMDHD_CONFIG_PATH="\"/vendor/etc/firmware/config.txt\""
+endif
+
+ifeq ($(CONFIG_BCMDHD_FW_SIGNATURE),y)
+	DHDCFLAGS += -DFW_SIGNATURE
+	DHDCFLAGS += -DBL_HEAP_START_GAP_SIZE=0x1000 -DBL_HEAP_SIZE=0x10000
+endif
+
+ifeq ($(CONFIG_BCMDHD_AG),y)
+	DHDCFLAGS += -DBAND_AG
+endif
+
+ifeq ($(CONFIG_DHD_USE_STATIC_BUF),y)
+ifeq  ($(CONFIG_BCMDHD_STATIC_BUF_IN_DHD),y)
+	DHDOFILES += dhd_static_buf.o
+	DHDCFLAGS += -DDHD_STATIC_IN_DRIVER
+else
+	obj-m += dhd_static_buf.o
+endif
+	DHDCFLAGS += -DSTATIC_WL_PRIV_STRUCT -DENHANCED_STATIC_BUF
+	DHDCFLAGS += -DCONFIG_DHD_USE_STATIC_BUF
+ifneq ($(filter -DDHD_LOG_DUMP, $(DHDCFLAGS)),)
+	DHDCFLAGS += -DDHD_USE_STATIC_MEMDUMP
+endif
+ifneq ($(filter -DDHD_FW_COREDUMP, $(DHDCFLAGS)),)
+	DHDCFLAGS += -DDHD_USE_STATIC_MEMDUMP
+endif
+ifneq ($(CONFIG_BCMDHD_PCIE),)
+	DHDCFLAGS += -DDHD_USE_STATIC_CTRLBUF
+endif
+endif
+
+ARCH ?= riscv
+BCMDHD_ROOT = $(src)
+#$(warning "BCMDHD_ROOT=$(BCMDHD_ROOT)")
+EXTRA_CFLAGS = $(DHDCFLAGS)
+EXTRA_CFLAGS += -DDHD_COMPILED=\"$(BCMDHD_ROOT)\"
+EXTRA_CFLAGS += -I$(BCMDHD_ROOT)/include/ -I$(BCMDHD_ROOT)/
+ifeq ($(CONFIG_BCMDHD),m)
+EXTRA_LDFLAGS += --strip-debug
+endif
+
+obj-$(CONFIG_BCMDHD) += $(MODULE_NAME).o
+$(MODULE_NAME)-objs += $(DHDOFILES)
+ccflags-y := $(EXTRA_CFLAGS)
+
+all: bcmdhd_pcie bcmdhd_sdio bcmdhd_usb
+
+bcmdhd_pcie:
+	$(warning "building BCMDHD_PCIE..........")
+	$(MAKE) -C $(LINUXDIR) M=$(PWD) ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) modules CONFIG_BCMDHD=m CONFIG_BCMDHD_PCIE=y
+
+bcmdhd_sdio:
+	$(warning "building BCMDHD_SDIO..........")
+	$(MAKE) -C $(LINUXDIR) M=$(PWD) ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) modules CONFIG_BCMDHD=m CONFIG_BCMDHD_SDIO=y
+
+bcmdhd_usb:
+	$(warning "building BCMDHD_USB..........")
+	$(MAKE) -C $(LINUXDIR) M=$(PWD) ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) modules CONFIG_BCMDHD=m CONFIG_BCMDHD_USB=y
+
+clean:
+	rm -rf *.o *.ko *.mod.c *~ .*.cmd *.o.cmd .*.o.cmd *.mod \
+	Module.symvers modules.order .tmp_versions modules.builtin
diff --git a/drivers/net/wireless/ap12275/aiutils.c b/drivers/net/wireless/ap12275/aiutils.c
new file mode 100644
index 000000000000..12f204401c57
--- /dev/null
+++ b/drivers/net/wireless/ap12275/aiutils.c
@@ -0,0 +1,2622 @@
+/*
+ * Misc utility routines for accessing chip-specific features
+ * of the SiliconBackplane-based Broadcom chips.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmdefs.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <siutils.h>
+#include <hndsoc.h>
+#include <sbchipc.h>
+#include <pcicfg.h>
+#include <pcie_core.h>
+
+#include "siutils_priv.h"
+#include <bcmdevs.h>
+
+#if defined(ETD)
+#include <etd.h>
+#endif
+
+#if !defined(BCMDONGLEHOST)
+#define PMU_DMP()  (cores_info->coreid[sii->curidx] == PMU_CORE_ID)
+#define GCI_DMP()  (cores_info->coreid[sii->curidx] == GCI_CORE_ID)
+#else
+#define PMU_DMP() (0)
+#define GCI_DMP() (0)
+#endif /* !defined(BCMDONGLEHOST) */
+
+#if defined(AXI_TIMEOUTS_NIC)
+static bool ai_get_apb_bridge(const si_t *sih, uint32 coreidx, uint32 *apb_id,
+	uint32 *apb_coreunit);
+#endif /* AXI_TIMEOUTS_NIC */
+
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+static void ai_reset_axi_to(const si_info_t *sii, aidmp_t *ai);
+#endif	/* defined (AXI_TIMEOUTS) || defined (AXI_TIMEOUTS_NIC) */
+
+#ifdef DONGLEBUILD
+static uint32 ai_get_sizeof_wrapper_offsets_to_dump(void);
+static uint32 ai_get_wrapper_base_addr(uint32 **offset);
+#endif /* DONGLEBUILD */
+
+/* AXI ID to CoreID + unit mappings */
+typedef struct axi_to_coreidx {
+	uint coreid;
+	uint coreunit;
+} axi_to_coreidx_t;
+
+static const axi_to_coreidx_t axi2coreidx_4369[] = {
+	{CC_CORE_ID, 0},	/* 00 Chipcommon */
+	{PCIE2_CORE_ID, 0},	/* 01 PCIe */
+	{D11_CORE_ID, 0},	/* 02 D11 Main */
+	{ARMCR4_CORE_ID, 0},	/* 03 ARM */
+	{BT_CORE_ID, 0},	/* 04 BT AHB */
+	{D11_CORE_ID, 1},	/* 05 D11 Aux */
+	{D11_CORE_ID, 0},	/* 06 D11 Main l1 */
+	{D11_CORE_ID, 1},	/* 07 D11 Aux  l1 */
+	{D11_CORE_ID, 0},	/* 08 D11 Main l2 */
+	{D11_CORE_ID, 1},	/* 09 D11 Aux  l2 */
+	{NODEV_CORE_ID, 0},	/* 10 M2M DMA */
+	{NODEV_CORE_ID, 0},	/* 11 unused */
+	{NODEV_CORE_ID, 0},	/* 12 unused */
+	{NODEV_CORE_ID, 0},	/* 13 unused */
+	{NODEV_CORE_ID, 0},	/* 14 unused */
+	{NODEV_CORE_ID, 0}	/* 15 unused */
+};
+
+/* EROM parsing */
+
+static uint32
+get_erom_ent(const si_t *sih, uint32 **eromptr, uint32 mask, uint32 match)
+{
+	uint32 ent;
+	uint inv = 0, nom = 0;
+	uint32 size = 0;
+
+	while (TRUE) {
+		ent = R_REG(SI_INFO(sih)->osh, *eromptr);
+		(*eromptr)++;
+
+		if (mask == 0)
+			break;
+
+		if ((ent & ER_VALID) == 0) {
+			inv++;
+			continue;
+		}
+
+		if (ent == (ER_END | ER_VALID))
+			break;
+
+		if ((ent & mask) == match)
+			break;
+
+		/* escape condition related EROM size if it has invalid values */
+		size += sizeof(*eromptr);
+		if (size >= ER_SZ_MAX) {
+			SI_ERROR(("Failed to find end of EROM marker\n"));
+			break;
+		}
+
+		nom++;
+	}
+
+	SI_VMSG(("get_erom_ent: Returning ent 0x%08x\n", ent));
+	if (inv + nom) {
+		SI_VMSG(("  after %d invalid and %d non-matching entries\n", inv, nom));
+	}
+	return ent;
+}
+
+static uint32
+get_asd(const si_t *sih, uint32 **eromptr, uint sp, uint ad, uint st, uint32 *addrl, uint32 *addrh,
+	uint32 *sizel, uint32 *sizeh)
+{
+	uint32 asd, sz, szd;
+
+	BCM_REFERENCE(ad);
+
+	asd = get_erom_ent(sih, eromptr, ER_VALID, ER_VALID);
+	if (((asd & ER_TAG1) != ER_ADD) ||
+	    (((asd & AD_SP_MASK) >> AD_SP_SHIFT) != sp) ||
+	    ((asd & AD_ST_MASK) != st)) {
+		/* This is not what we want, "push" it back */
+		(*eromptr)--;
+		return 0;
+	}
+	*addrl = asd & AD_ADDR_MASK;
+	if (asd & AD_AG32)
+		*addrh = get_erom_ent(sih, eromptr, 0, 0);
+	else
+		*addrh = 0;
+	*sizeh = 0;
+	sz = asd & AD_SZ_MASK;
+	if (sz == AD_SZ_SZD) {
+		szd = get_erom_ent(sih, eromptr, 0, 0);
+		*sizel = szd & SD_SZ_MASK;
+		if (szd & SD_SG32)
+			*sizeh = get_erom_ent(sih, eromptr, 0, 0);
+	} else
+		*sizel = AD_SZ_BASE << (sz >> AD_SZ_SHIFT);
+
+	SI_VMSG(("  SP %d, ad %d: st = %d, 0x%08x_0x%08x @ 0x%08x_0x%08x\n",
+	        sp, ad, st, *sizeh, *sizel, *addrh, *addrl));
+
+	return asd;
+}
+
+/* Parse the enumeration rom to identify all cores */
+void
+ai_scan(si_t *sih, void *regs, uint devid)
+{
+	si_info_t *sii = SI_INFO(sih);
+	si_cores_info_t *cores_info = (si_cores_info_t *)sii->cores_info;
+	chipcregs_t *cc = (chipcregs_t *)regs;
+	uint32 erombase, *eromptr, *eromlim;
+	axi_wrapper_t * axi_wrapper = sii->axi_wrapper;
+
+	SI_MSG_DBG_REG(("%s: Enter\n", __FUNCTION__));
+	BCM_REFERENCE(devid);
+
+	erombase = R_REG(sii->osh, &cc->eromptr);
+
+	switch (BUSTYPE(sih->bustype)) {
+	case SI_BUS:
+		eromptr = (uint32 *)REG_MAP(erombase, SI_CORE_SIZE);
+		break;
+
+	case PCI_BUS:
+		/* Set wrappers address */
+		sii->curwrap = (void *)((uintptr)regs + SI_CORE_SIZE);
+
+		/* Now point the window at the erom */
+		OSL_PCI_WRITE_CONFIG(sii->osh, PCI_BAR0_WIN, 4, erombase);
+		eromptr = regs;
+		break;
+
+#ifdef BCMSDIO
+	case SPI_BUS:
+	case SDIO_BUS:
+		eromptr = (uint32 *)(uintptr)erombase;
+		break;
+#endif	/* BCMSDIO */
+
+	default:
+		SI_ERROR(("Don't know how to do AXI enumeration on bus %d\n", sih->bustype));
+		ASSERT(0);
+		return;
+	}
+	eromlim = eromptr + (ER_REMAPCONTROL / sizeof(uint32));
+	sii->axi_num_wrappers = 0;
+
+	SI_VMSG(("ai_scan: regs = 0x%p, erombase = 0x%08x, eromptr = 0x%p, eromlim = 0x%p\n",
+	         OSL_OBFUSCATE_BUF(regs), erombase,
+		OSL_OBFUSCATE_BUF(eromptr), OSL_OBFUSCATE_BUF(eromlim)));
+	while (eromptr < eromlim) {
+		uint32 cia, cib, cid, mfg, crev, nmw, nsw, nmp, nsp;
+		uint32 mpd, asd, addrl, addrh, sizel, sizeh;
+		uint i, j, idx;
+		bool br;
+
+		br = FALSE;
+
+		/* Grok a component */
+		cia = get_erom_ent(sih, &eromptr, ER_TAG, ER_CI);
+		if (cia == (ER_END | ER_VALID)) {
+			SI_VMSG(("Found END of erom after %d cores\n", sii->numcores));
+			SI_MSG_DBG_REG(("%s: Exit\n", __FUNCTION__));
+			return;
+		}
+
+		cib = get_erom_ent(sih, &eromptr, 0, 0);
+
+		if ((cib & ER_TAG) != ER_CI) {
+			SI_ERROR(("CIA not followed by CIB\n"));
+			goto error;
+		}
+
+		cid = (cia & CIA_CID_MASK) >> CIA_CID_SHIFT;
+		mfg = (cia & CIA_MFG_MASK) >> CIA_MFG_SHIFT;
+		crev = (cib & CIB_REV_MASK) >> CIB_REV_SHIFT;
+		nmw = (cib & CIB_NMW_MASK) >> CIB_NMW_SHIFT;
+		nsw = (cib & CIB_NSW_MASK) >> CIB_NSW_SHIFT;
+		nmp = (cib & CIB_NMP_MASK) >> CIB_NMP_SHIFT;
+		nsp = (cib & CIB_NSP_MASK) >> CIB_NSP_SHIFT;
+
+#ifdef BCMDBG_SI
+		SI_VMSG(("Found component 0x%04x/0x%04x rev %d at erom addr 0x%p, with nmw = %d, "
+		         "nsw = %d, nmp = %d & nsp = %d\n",
+		         mfg, cid, crev, OSL_OBFUSCATE_BUF(eromptr - 1), nmw, nsw, nmp, nsp));
+#else
+		BCM_REFERENCE(crev);
+#endif
+
+		/* Include Default slave wrapper for timeout monitoring */
+		if ((nsp == 0 && nsw == 0) ||
+#if !defined(AXI_TIMEOUTS) && !defined(AXI_TIMEOUTS_NIC)
+			((mfg == MFGID_ARM) && (cid == DEF_AI_COMP)) ||
+#else
+			((CHIPTYPE(sii->pub.socitype) == SOCI_NAI) &&
+			(mfg == MFGID_ARM) && (cid == DEF_AI_COMP)) ||
+#endif /* !defined(AXI_TIMEOUTS) && !defined(AXI_TIMEOUTS_NIC) */
+			FALSE) {
+			continue;
+		}
+
+		if ((nmw + nsw == 0)) {
+			/* A component which is not a core */
+			/* Should record some info */
+			if (cid == OOB_ROUTER_CORE_ID) {
+				asd = get_asd(sih, &eromptr, 0, 0, AD_ST_SLAVE,
+					&addrl, &addrh, &sizel, &sizeh);
+				if (asd != 0) {
+					if ((sii->oob_router != 0) && (sii->oob_router != addrl)) {
+						sii->oob_router1 = addrl;
+					} else {
+						sii->oob_router = addrl;
+					}
+				}
+			}
+			if ((cid != NS_CCB_CORE_ID) && (cid != PMU_CORE_ID) &&
+				(cid != GCI_CORE_ID) && (cid != SR_CORE_ID) &&
+				(cid != HUB_CORE_ID) && (cid != HND_OOBR_CORE_ID) &&
+				(cid != CCI400_CORE_ID) && (cid != SPMI_SLAVE_CORE_ID)) {
+				continue;
+			}
+		}
+
+		idx = sii->numcores;
+
+		cores_info->cia[idx] = cia;
+		cores_info->cib[idx] = cib;
+		cores_info->coreid[idx] = cid;
+
+		/* workaround the fact the variable buscoretype is used in _ai_set_coreidx()
+		 * when checking PCIE_GEN2() for PCI_BUS case before it is setup later...,
+		 * both use and setup happen in si_buscore_setup().
+		 */
+		if (BUSTYPE(sih->bustype) == PCI_BUS &&
+		    (cid == PCI_CORE_ID || cid == PCIE_CORE_ID || cid == PCIE2_CORE_ID)) {
+			sii->pub.buscoretype = (uint16)cid;
+		}
+
+		for (i = 0; i < nmp; i++) {
+			mpd = get_erom_ent(sih, &eromptr, ER_VALID, ER_VALID);
+			if ((mpd & ER_TAG) != ER_MP) {
+				SI_ERROR(("Not enough MP entries for component 0x%x\n", cid));
+				goto error;
+			}
+			/* Record something? */
+			SI_VMSG(("  Master port %d, mp: %d id: %d\n", i,
+			         (mpd & MPD_MP_MASK) >> MPD_MP_SHIFT,
+			         (mpd & MPD_MUI_MASK) >> MPD_MUI_SHIFT));
+		}
+
+		/* First Slave Address Descriptor should be port 0:
+		 * the main register space for the core
+		 */
+		asd = get_asd(sih, &eromptr, 0, 0, AD_ST_SLAVE, &addrl, &addrh, &sizel, &sizeh);
+		if (asd == 0) {
+			do {
+			/* Try again to see if it is a bridge */
+			asd = get_asd(sih, &eromptr, 0, 0, AD_ST_BRIDGE, &addrl, &addrh,
+			              &sizel, &sizeh);
+			if (asd != 0)
+				br = TRUE;
+			else {
+					break;
+				}
+			} while (1);
+		} else {
+			if (addrl == 0 || sizel == 0) {
+				SI_ERROR((" Invalid ASD %x for slave port \n", asd));
+				goto error;
+			}
+			cores_info->coresba[idx] = addrl;
+			cores_info->coresba_size[idx] = sizel;
+		}
+
+		/* Get any more ASDs in first port */
+		j = 1;
+		do {
+			asd = get_asd(sih, &eromptr, 0, j, AD_ST_SLAVE, &addrl, &addrh,
+			              &sizel, &sizeh);
+			/* Support ARM debug core ASD with address space > 4K */
+			if ((asd != 0) && (j == 1)) {
+				SI_VMSG(("Warning: sizel > 0x1000\n"));
+				cores_info->coresba2[idx] = addrl;
+				cores_info->coresba2_size[idx] = sizel;
+			}
+			j++;
+		} while (asd != 0);
+
+		/* Go through the ASDs for other slave ports */
+		for (i = 1; i < nsp; i++) {
+			j = 0;
+			do {
+				asd = get_asd(sih, &eromptr, i, j, AD_ST_SLAVE, &addrl, &addrh,
+				              &sizel, &sizeh);
+				/* To get the first base address of second slave port */
+				if ((asd != 0) && (i == 1) && (j == 0)) {
+					cores_info->csp2ba[idx] = addrl;
+					cores_info->csp2ba_size[idx] = sizel;
+				}
+				if (asd == 0)
+					break;
+				j++;
+			} while (1);
+			if (j == 0) {
+				SI_ERROR((" SP %d has no address descriptors\n", i));
+				goto error;
+			}
+		}
+
+		/* Now get master wrappers */
+		for (i = 0; i < nmw; i++) {
+			asd = get_asd(sih, &eromptr, i, 0, AD_ST_MWRAP, &addrl, &addrh,
+			              &sizel, &sizeh);
+			if (asd == 0) {
+				SI_ERROR(("Missing descriptor for MW %d\n", i));
+				goto error;
+			}
+			if ((sizeh != 0) || (sizel != SI_CORE_SIZE)) {
+				SI_ERROR(("Master wrapper %d is not 4KB\n", i));
+				goto error;
+			}
+			if (i == 0) {
+				cores_info->wrapba[idx] = addrl;
+			} else if (i == 1) {
+				cores_info->wrapba2[idx] = addrl;
+			} else if (i == 2) {
+				cores_info->wrapba3[idx] = addrl;
+			}
+
+			if (axi_wrapper && (sii->axi_num_wrappers < SI_MAX_AXI_WRAPPERS)) {
+				axi_wrapper[sii->axi_num_wrappers].mfg = mfg;
+				axi_wrapper[sii->axi_num_wrappers].cid = cid;
+				axi_wrapper[sii->axi_num_wrappers].rev = crev;
+				axi_wrapper[sii->axi_num_wrappers].wrapper_type = AI_MASTER_WRAPPER;
+				axi_wrapper[sii->axi_num_wrappers].wrapper_addr = addrl;
+				sii->axi_num_wrappers++;
+				SI_VMSG(("MASTER WRAPPER: %d, mfg:%x, cid:%x,"
+					"rev:%x, addr:%x, size:%x\n",
+					sii->axi_num_wrappers, mfg, cid, crev, addrl, sizel));
+			}
+		}
+
+		/* And finally slave wrappers */
+		for (i = 0; i < nsw; i++) {
+			uint fwp = (nsp <= 1) ? 0 : 1;
+			asd = get_asd(sih, &eromptr, fwp + i, 0, AD_ST_SWRAP, &addrl, &addrh,
+			              &sizel, &sizeh);
+			if (asd == 0) {
+				SI_ERROR(("Missing descriptor for SW %d cid %x eromp %p fwp %d \n",
+					i, cid, eromptr, fwp));
+				goto error;
+			}
+
+			if ((sizeh != 0) || (sizel != SI_CORE_SIZE)) {
+				SI_ERROR(("Slave wrapper %d is not 4KB\n", i));
+				goto error;
+			}
+
+			/* cache APB bridge wrapper address for set/clear timeout */
+			if ((mfg == MFGID_ARM) && (cid == APB_BRIDGE_ID)) {
+				ASSERT(sii->num_br < SI_MAXBR);
+				if (sii->num_br >= SI_MAXBR) {
+					SI_ERROR(("bridge number %d is overflowed\n", sii->num_br));
+					goto error;
+				}
+
+				sii->br_wrapba[sii->num_br++] = addrl;
+			}
+
+			if ((mfg == MFGID_ARM) && (cid == ADB_BRIDGE_ID)) {
+				br = TRUE;
+			}
+
+			BCM_REFERENCE(br);
+
+			if ((nmw == 0) && (i == 0)) {
+				cores_info->wrapba[idx] = addrl;
+			} else if ((nmw == 0) && (i == 1)) {
+				cores_info->wrapba2[idx] = addrl;
+			} else if ((nmw == 0) && (i == 2)) {
+				cores_info->wrapba3[idx] = addrl;
+			}
+
+			/* Include all slave wrappers to the list to
+			 * enable and monitor watchdog timeouts
+			 */
+
+			if (axi_wrapper && (sii->axi_num_wrappers < SI_MAX_AXI_WRAPPERS)) {
+				axi_wrapper[sii->axi_num_wrappers].mfg = mfg;
+				axi_wrapper[sii->axi_num_wrappers].cid = cid;
+				axi_wrapper[sii->axi_num_wrappers].rev = crev;
+				axi_wrapper[sii->axi_num_wrappers].wrapper_type = AI_SLAVE_WRAPPER;
+				axi_wrapper[sii->axi_num_wrappers].wrapper_addr = addrl;
+
+				sii->axi_num_wrappers++;
+
+				SI_VMSG(("SLAVE WRAPPER: %d,  mfg:%x, cid:%x,"
+					"rev:%x, addr:%x, size:%x\n",
+					sii->axi_num_wrappers,  mfg, cid, crev, addrl, sizel));
+			}
+		}
+
+#ifndef AXI_TIMEOUTS_NIC
+		/* Don't record bridges and core with 0 slave ports */
+		if (br || (nsp == 0)) {
+			continue;
+		}
+#endif
+
+		/* Done with core */
+		sii->numcores++;
+	}
+
+	SI_ERROR(("Reached end of erom without finding END\n"));
+
+error:
+	sii->numcores = 0;
+	SI_MSG_DBG_REG(("%s: Exit\n", __FUNCTION__));
+	return;
+}
+
+#define AI_SETCOREIDX_MAPSIZE(coreid) \
+	(((coreid) == NS_CCB_CORE_ID) ? 15 * SI_CORE_SIZE : SI_CORE_SIZE)
+
+/* This function changes the logical "focus" to the indicated core.
+ * Return the current core's virtual address.
+ */
+static volatile void *
+BCMPOSTTRAPFN(_ai_setcoreidx)(si_t *sih, uint coreidx, uint use_wrapn)
+{
+	si_info_t *sii = SI_INFO(sih);
+	si_cores_info_t *cores_info = (si_cores_info_t *)sii->cores_info;
+	uint32 addr, wrap, wrap2, wrap3;
+	volatile void *regs;
+
+	if (coreidx >= MIN(sii->numcores, SI_MAXCORES))
+		return (NULL);
+
+	addr = cores_info->coresba[coreidx];
+	wrap = cores_info->wrapba[coreidx];
+	wrap2 = cores_info->wrapba2[coreidx];
+	wrap3 = cores_info->wrapba3[coreidx];
+
+#ifdef AXI_TIMEOUTS_NIC
+	/* No need to disable interrupts while entering/exiting APB bridge core */
+	if ((cores_info->coreid[coreidx] != APB_BRIDGE_CORE_ID) &&
+		(cores_info->coreid[sii->curidx] != APB_BRIDGE_CORE_ID))
+#endif /* AXI_TIMEOUTS_NIC */
+	{
+		/*
+		 * If the user has provided an interrupt mask enabled function,
+		 * then assert interrupts are disabled before switching the core.
+		 */
+		ASSERT((sii->intrsenabled_fn == NULL) ||
+			!(*(sii)->intrsenabled_fn)((sii)->intr_arg));
+	}
+
+	switch (BUSTYPE(sih->bustype)) {
+	case SI_BUS:
+		/* map new one */
+		if (!cores_info->regs[coreidx]) {
+			cores_info->regs[coreidx] = REG_MAP(addr,
+				AI_SETCOREIDX_MAPSIZE(cores_info->coreid[coreidx]));
+			ASSERT(GOODREGS(cores_info->regs[coreidx]));
+		}
+		sii->curmap = regs = cores_info->regs[coreidx];
+		if (!cores_info->wrappers[coreidx] && (wrap != 0)) {
+			cores_info->wrappers[coreidx] = REG_MAP(wrap, SI_CORE_SIZE);
+			ASSERT(GOODREGS(cores_info->wrappers[coreidx]));
+		}
+		if (!cores_info->wrappers2[coreidx] && (wrap2 != 0)) {
+			cores_info->wrappers2[coreidx] = REG_MAP(wrap2, SI_CORE_SIZE);
+			ASSERT(GOODREGS(cores_info->wrappers2[coreidx]));
+		}
+		if (!cores_info->wrappers3[coreidx] && (wrap3 != 0)) {
+			cores_info->wrappers3[coreidx] = REG_MAP(wrap3, SI_CORE_SIZE);
+			ASSERT(GOODREGS(cores_info->wrappers3[coreidx]));
+		}
+
+		if (use_wrapn == 2) {
+			sii->curwrap = cores_info->wrappers3[coreidx];
+		} else if (use_wrapn == 1) {
+			sii->curwrap = cores_info->wrappers2[coreidx];
+		} else {
+			sii->curwrap = cores_info->wrappers[coreidx];
+		}
+		break;
+
+	case PCI_BUS:
+		regs = sii->curmap;
+
+		/* point bar0 2nd 4KB window to the primary wrapper */
+		if (use_wrapn == 2) {
+			wrap = wrap3;
+		} else if (use_wrapn == 1) {
+			wrap = wrap2;
+		}
+
+		/* Use BAR0 Window to support dual mac chips... */
+
+		/* TODO: the other mac unit can't be supportd by the current BAR0 window.
+		 * need to find other ways to access these cores.
+		 */
+
+		switch (sii->slice) {
+		case 0: /* main/first slice */
+#ifdef AXI_TIMEOUTS_NIC
+			/* No need to set the BAR0 if core is APB Bridge.
+			 * This is to reduce 2 PCI writes while checkng for errlog
+			 */
+			if (cores_info->coreid[coreidx] != APB_BRIDGE_CORE_ID)
+#endif /* AXI_TIMEOUTS_NIC */
+			{
+				/* point bar0 window */
+				OSL_PCI_WRITE_CONFIG(sii->osh, PCI_BAR0_WIN, 4, addr);
+			}
+
+			if (PCIE_GEN2(sii))
+				OSL_PCI_WRITE_CONFIG(sii->osh, PCIE2_BAR0_WIN2, 4, wrap);
+			else
+				OSL_PCI_WRITE_CONFIG(sii->osh, PCI_BAR0_WIN2, 4, wrap);
+
+			break;
+
+		case 1: /* aux/second slice */
+			/* PCIE GEN2 only for other slices */
+			if (!PCIE_GEN2(sii)) {
+				/* other slices not supported */
+				SI_ERROR(("PCI GEN not supported for slice %d\n", sii->slice));
+				ASSERT(0);
+				break;
+			}
+
+			/* 0x4000 - 0x4fff: enum space 0x5000 - 0x5fff: wrapper space */
+			regs = (volatile uint8 *)regs + PCI_SEC_BAR0_WIN_OFFSET;
+			sii->curwrap = (void *)((uintptr)regs + SI_CORE_SIZE);
+
+			/* point bar0 window */
+			OSL_PCI_WRITE_CONFIG(sii->osh, PCIE2_BAR0_CORE2_WIN, 4, addr);
+			OSL_PCI_WRITE_CONFIG(sii->osh, PCIE2_BAR0_CORE2_WIN2, 4, wrap);
+			break;
+
+		case 2: /* scan/third slice */
+			/* PCIE GEN2 only for other slices */
+			if (!PCIE_GEN2(sii)) {
+				/* other slices not supported */
+				SI_ERROR(("PCI GEN not supported for slice %d\n", sii->slice));
+				ASSERT(0);
+				break;
+			}
+
+			/* 0x9000 - 0x9fff: enum space 0xa000 - 0xafff: wrapper space */
+			regs = (volatile uint8 *)regs + PCI_TER_BAR0_WIN_OFFSET;
+			sii->curwrap = (void *)((uintptr)regs + SI_CORE_SIZE);
+
+			/* point bar0 window */
+			ai_corereg(sih, sih->buscoreidx,
+			            PCIE_TER_BAR0_WIN_REG(sih->buscorerev), ~0, addr);
+			ai_corereg(sih, sih->buscoreidx,
+			            PCIE_TER_BAR0_WRAPPER_REG(sih->buscorerev), ~0, wrap);
+			break;
+
+		default: /* other slices */
+			SI_ERROR(("BAR0 Window not supported for slice %d\n", sii->slice));
+			ASSERT(0);
+			break;
+		}
+
+		break;
+
+#ifdef BCMSDIO
+	case SPI_BUS:
+	case SDIO_BUS:
+		sii->curmap = regs = (void *)((uintptr)addr);
+		if (use_wrapn)
+			sii->curwrap = (void *)((uintptr)wrap2);
+		else
+			sii->curwrap = (void *)((uintptr)wrap);
+		break;
+#endif	/* BCMSDIO */
+
+	default:
+		ASSERT(0);
+		sii->curmap = regs = NULL;
+		break;
+	}
+
+	sii->curidx = coreidx;
+
+	if (regs) {
+		SI_MSG_DBG_REG(("%s: %d\n", __FUNCTION__, coreidx));
+	}
+
+	return regs;
+}
+
+volatile void *
+BCMPOSTTRAPFN(ai_setcoreidx)(si_t *sih, uint coreidx)
+{
+	return _ai_setcoreidx(sih, coreidx, 0);
+}
+
+volatile void *
+BCMPOSTTRAPFN(ai_setcoreidx_2ndwrap)(si_t *sih, uint coreidx)
+{
+	return _ai_setcoreidx(sih, coreidx, 1);
+}
+
+volatile void *
+BCMPOSTTRAPFN(ai_setcoreidx_3rdwrap)(si_t *sih, uint coreidx)
+{
+	return _ai_setcoreidx(sih, coreidx, 2);
+}
+
+void
+ai_coreaddrspaceX(const si_t *sih, uint asidx, uint32 *addr, uint32 *size)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	chipcregs_t *cc = NULL;
+	uint32 erombase, *eromptr, *eromlim;
+	uint i, j, cidx;
+	uint32 cia, cib, nmp, nsp;
+	uint32 asd, addrl, addrh, sizel, sizeh;
+
+	for (i = 0; i < sii->numcores; i++) {
+		if (cores_info->coreid[i] == CC_CORE_ID) {
+			cc = (chipcregs_t *)cores_info->regs[i];
+			break;
+		}
+	}
+	if (cc == NULL)
+		goto error;
+
+	BCM_REFERENCE(erombase);
+	erombase = R_REG(sii->osh, &cc->eromptr);
+	eromptr = (uint32 *)REG_MAP(erombase, SI_CORE_SIZE);
+	eromlim = eromptr + (ER_REMAPCONTROL / sizeof(uint32));
+
+	cidx = sii->curidx;
+	cia = cores_info->cia[cidx];
+	cib = cores_info->cib[cidx];
+
+	nmp = (cib & CIB_NMP_MASK) >> CIB_NMP_SHIFT;
+	nsp = (cib & CIB_NSP_MASK) >> CIB_NSP_SHIFT;
+
+	/* scan for cores */
+	while (eromptr < eromlim) {
+		if ((get_erom_ent(sih, &eromptr, ER_TAG, ER_CI) == cia) &&
+			(get_erom_ent(sih, &eromptr, 0, 0) == cib)) {
+			break;
+		}
+	}
+
+	/* skip master ports */
+	for (i = 0; i < nmp; i++)
+		get_erom_ent(sih, &eromptr, ER_VALID, ER_VALID);
+
+	/* Skip ASDs in port 0 */
+	asd = get_asd(sih, &eromptr, 0, 0, AD_ST_SLAVE, &addrl, &addrh, &sizel, &sizeh);
+	if (asd == 0) {
+		/* Try again to see if it is a bridge */
+		asd = get_asd(sih, &eromptr, 0, 0, AD_ST_BRIDGE, &addrl, &addrh,
+		              &sizel, &sizeh);
+	}
+
+	j = 1;
+	do {
+		asd = get_asd(sih, &eromptr, 0, j, AD_ST_SLAVE, &addrl, &addrh,
+		              &sizel, &sizeh);
+		j++;
+	} while (asd != 0);
+
+	/* Go through the ASDs for other slave ports */
+	for (i = 1; i < nsp; i++) {
+		j = 0;
+		do {
+			asd = get_asd(sih, &eromptr, i, j, AD_ST_SLAVE, &addrl, &addrh,
+				&sizel, &sizeh);
+			if (asd == 0)
+				break;
+
+			if (!asidx--) {
+				*addr = addrl;
+				*size = sizel;
+				return;
+			}
+			j++;
+		} while (1);
+
+		if (j == 0) {
+			SI_ERROR((" SP %d has no address descriptors\n", i));
+			break;
+		}
+	}
+
+error:
+	*size = 0;
+	return;
+}
+
+/* Return the number of address spaces in current core */
+int
+ai_numaddrspaces(const si_t *sih)
+{
+	/* TODO: Either save it or parse the EROM on demand, currently hardcode 2 */
+	BCM_REFERENCE(sih);
+
+	return 2;
+}
+
+/* Return the address of the nth address space in the current core
+ * Arguments:
+ * sih : Pointer to struct si_t
+ * spidx : slave port index
+ * baidx : base address index
+ */
+uint32
+ai_addrspace(const si_t *sih, uint spidx, uint baidx)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	uint cidx;
+
+	cidx = sii->curidx;
+
+	if (spidx == CORE_SLAVE_PORT_0) {
+		if (baidx == CORE_BASE_ADDR_0)
+			return cores_info->coresba[cidx];
+		else if (baidx == CORE_BASE_ADDR_1)
+			return cores_info->coresba2[cidx];
+	}
+	else if (spidx == CORE_SLAVE_PORT_1) {
+		if (baidx == CORE_BASE_ADDR_0)
+			return cores_info->csp2ba[cidx];
+	}
+
+	SI_ERROR(("ai_addrspace: Need to parse the erom again to find %d base addr"
+		" in %d slave port\n",
+		baidx, spidx));
+
+	return 0;
+
+}
+
+/* Return the size of the nth address space in the current core
+* Arguments:
+* sih : Pointer to struct si_t
+* spidx : slave port index
+* baidx : base address index
+*/
+uint32
+ai_addrspacesize(const si_t *sih, uint spidx, uint baidx)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	uint cidx;
+
+	cidx = sii->curidx;
+	if (spidx == CORE_SLAVE_PORT_0) {
+		if (baidx == CORE_BASE_ADDR_0)
+			return cores_info->coresba_size[cidx];
+		else if (baidx == CORE_BASE_ADDR_1)
+			return cores_info->coresba2_size[cidx];
+	}
+	else if (spidx == CORE_SLAVE_PORT_1) {
+		if (baidx == CORE_BASE_ADDR_0)
+			return cores_info->csp2ba_size[cidx];
+	}
+
+	SI_ERROR(("ai_addrspacesize: Need to parse the erom again to find %d"
+		" base addr in %d slave port\n",
+		baidx, spidx));
+
+	return 0;
+}
+
+uint
+ai_flag(si_t *sih)
+{
+	const si_info_t *sii = SI_INFO(sih);
+#if !defined(BCMDONGLEHOST)
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+#endif
+	aidmp_t *ai;
+
+	if (PMU_DMP()) {
+		uint idx, flag;
+		idx = sii->curidx;
+		ai_setcoreidx(sih, SI_CC_IDX);
+		flag = ai_flag_alt(sih);
+		ai_setcoreidx(sih, idx);
+		return flag;
+	}
+
+	ai = sii->curwrap;
+	ASSERT(ai != NULL);
+
+	return (R_REG(sii->osh, &ai->oobselouta30) & 0x1f);
+}
+
+uint
+ai_flag_alt(const si_t *sih)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	aidmp_t *ai = sii->curwrap;
+
+	return ((R_REG(sii->osh, &ai->oobselouta30) >> AI_OOBSEL_1_SHIFT) & AI_OOBSEL_MASK);
+}
+
+void
+ai_setint(const si_t *sih, int siflag)
+{
+	BCM_REFERENCE(sih);
+	BCM_REFERENCE(siflag);
+
+	/* TODO: Figure out how to set interrupt mask in ai */
+}
+
+uint
+BCMPOSTTRAPFN(ai_wrap_reg)(const si_t *sih, uint32 offset, uint32 mask, uint32 val)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	uint32 *addr = (uint32 *) ((uchar *)(sii->curwrap) + offset);
+
+	if (mask || val) {
+		uint32 w = R_REG(sii->osh, addr);
+		w &= ~mask;
+		w |= val;
+		W_REG(sii->osh, addr, w);
+	}
+	return (R_REG(sii->osh, addr));
+}
+
+uint
+ai_corevendor(const si_t *sih)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	uint32 cia;
+
+	cia = cores_info->cia[sii->curidx];
+	return ((cia & CIA_MFG_MASK) >> CIA_MFG_SHIFT);
+}
+
+uint
+BCMPOSTTRAPFN(ai_corerev)(const si_t *sih)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	uint32 cib;
+
+	cib = cores_info->cib[sii->curidx];
+	return ((cib & CIB_REV_MASK) >> CIB_REV_SHIFT);
+}
+
+uint
+ai_corerev_minor(const si_t *sih)
+{
+	return (ai_core_sflags(sih, 0, 0) >> SISF_MINORREV_D11_SHIFT) &
+			SISF_MINORREV_D11_MASK;
+}
+
+bool
+BCMPOSTTRAPFN(ai_iscoreup)(const si_t *sih)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	aidmp_t *ai = sii->curwrap;
+
+	return (((R_REG(sii->osh, &ai->ioctrl) & (SICF_FGC | SICF_CLOCK_EN)) == SICF_CLOCK_EN) &&
+	        ((R_REG(sii->osh, &ai->resetctrl) & AIRC_RESET) == 0));
+}
+
+/*
+ * Switch to 'coreidx', issue a single arbitrary 32bit register mask&set operation,
+ * switch back to the original core, and return the new value.
+ *
+ * When using the silicon backplane, no fiddling with interrupts or core switches is needed.
+ *
+ * Also, when using pci/pcie, we can optimize away the core switching for pci registers
+ * and (on newer pci cores) chipcommon registers.
+ */
+uint
+BCMPOSTTRAPFN(ai_corereg)(si_t *sih, uint coreidx, uint regoff, uint mask, uint val)
+{
+	uint origidx = 0;
+	volatile uint32 *r = NULL;
+	uint w;
+	bcm_int_bitmask_t intr_val;
+	bool fast = FALSE;
+	si_info_t *sii = SI_INFO(sih);
+	si_cores_info_t *cores_info = (si_cores_info_t *)sii->cores_info;
+
+	ASSERT(GOODIDX(coreidx, sii->numcores));
+	ASSERT(regoff < SI_CORE_SIZE);
+	ASSERT((val & ~mask) == 0);
+
+	if (coreidx >= SI_MAXCORES)
+		return 0;
+
+	if (BUSTYPE(sih->bustype) == SI_BUS) {
+		/* If internal bus, we can always get at everything */
+		fast = TRUE;
+		/* map if does not exist */
+		if (!cores_info->regs[coreidx]) {
+			cores_info->regs[coreidx] = REG_MAP(cores_info->coresba[coreidx],
+			                            SI_CORE_SIZE);
+			ASSERT(GOODREGS(cores_info->regs[coreidx]));
+		}
+		r = (volatile uint32 *)((volatile uchar *)cores_info->regs[coreidx] + regoff);
+	} else if (BUSTYPE(sih->bustype) == PCI_BUS) {
+		/* If pci/pcie, we can get at pci/pcie regs and on newer cores to chipc */
+
+		if ((cores_info->coreid[coreidx] == CC_CORE_ID) && SI_FAST(sii)) {
+			/* Chipc registers are mapped at 12KB */
+
+			fast = TRUE;
+			r = (volatile uint32 *)((volatile char *)sii->curmap +
+			               PCI_16KB0_CCREGS_OFFSET + regoff);
+		} else if (sii->pub.buscoreidx == coreidx) {
+			/* pci registers are at either in the last 2KB of an 8KB window
+			 * or, in pcie and pci rev 13 at 8KB
+			 */
+			fast = TRUE;
+			if (SI_FAST(sii))
+				r = (volatile uint32 *)((volatile char *)sii->curmap +
+				               PCI_16KB0_PCIREGS_OFFSET + regoff);
+			else
+				r = (volatile uint32 *)((volatile char *)sii->curmap +
+				               ((regoff >= SBCONFIGOFF) ?
+				                PCI_BAR0_PCISBR_OFFSET : PCI_BAR0_PCIREGS_OFFSET) +
+				               regoff);
+		}
+	}
+
+	if (!fast) {
+		INTR_OFF(sii, &intr_val);
+
+		/* save current core index */
+		origidx = si_coreidx(&sii->pub);
+
+		/* switch core */
+		r = (volatile uint32*) ((volatile uchar*) ai_setcoreidx(&sii->pub, coreidx) +
+		               regoff);
+	}
+	ASSERT(r != NULL);
+
+	/* mask and set */
+	if (mask || val) {
+		w = (R_REG(sii->osh, r) & ~mask) | val;
+		W_REG(sii->osh, r, w);
+	}
+
+	/* readback */
+	w = R_REG(sii->osh, r);
+
+	if (!fast) {
+		/* restore core index */
+		if (origidx != coreidx)
+			ai_setcoreidx(&sii->pub, origidx);
+
+		INTR_RESTORE(sii, &intr_val);
+	}
+
+	return (w);
+}
+
+/*
+ * Switch to 'coreidx', issue a single arbitrary 32bit register mask&set operation,
+ * switch back to the original core, and return the new value.
+ *
+ * When using the silicon backplane, no fiddling with interrupts or core switches is needed.
+ *
+ * Also, when using pci/pcie, we can optimize away the core switching for pci registers
+ * and (on newer pci cores) chipcommon registers.
+ */
+uint
+ai_corereg_writeonly(si_t *sih, uint coreidx, uint regoff, uint mask, uint val)
+{
+	uint origidx = 0;
+	volatile uint32 *r = NULL;
+	uint w = 0;
+	bcm_int_bitmask_t intr_val;
+	bool fast = FALSE;
+	si_info_t *sii = SI_INFO(sih);
+	si_cores_info_t *cores_info = (si_cores_info_t *)sii->cores_info;
+
+	ASSERT(GOODIDX(coreidx, sii->numcores));
+	ASSERT(regoff < SI_CORE_SIZE);
+	ASSERT((val & ~mask) == 0);
+
+	if (coreidx >= SI_MAXCORES)
+		return 0;
+
+	if (BUSTYPE(sih->bustype) == SI_BUS) {
+		/* If internal bus, we can always get at everything */
+		fast = TRUE;
+		/* map if does not exist */
+		if (!cores_info->regs[coreidx]) {
+			cores_info->regs[coreidx] = REG_MAP(cores_info->coresba[coreidx],
+			                            SI_CORE_SIZE);
+			ASSERT(GOODREGS(cores_info->regs[coreidx]));
+		}
+		r = (volatile uint32 *)((volatile uchar *)cores_info->regs[coreidx] + regoff);
+	} else if (BUSTYPE(sih->bustype) == PCI_BUS) {
+		/* If pci/pcie, we can get at pci/pcie regs and on newer cores to chipc */
+
+		if ((cores_info->coreid[coreidx] == CC_CORE_ID) && SI_FAST(sii)) {
+			/* Chipc registers are mapped at 12KB */
+
+			fast = TRUE;
+			r = (volatile uint32 *)((volatile char *)sii->curmap +
+			               PCI_16KB0_CCREGS_OFFSET + regoff);
+		} else if (sii->pub.buscoreidx == coreidx) {
+			/* pci registers are at either in the last 2KB of an 8KB window
+			 * or, in pcie and pci rev 13 at 8KB
+			 */
+			fast = TRUE;
+			if (SI_FAST(sii))
+				r = (volatile uint32 *)((volatile char *)sii->curmap +
+				               PCI_16KB0_PCIREGS_OFFSET + regoff);
+			else
+				r = (volatile uint32 *)((volatile char *)sii->curmap +
+				               ((regoff >= SBCONFIGOFF) ?
+				                PCI_BAR0_PCISBR_OFFSET : PCI_BAR0_PCIREGS_OFFSET) +
+				               regoff);
+		}
+	}
+
+	if (!fast) {
+		INTR_OFF(sii, &intr_val);
+
+		/* save current core index */
+		origidx = si_coreidx(&sii->pub);
+
+		/* switch core */
+		r = (volatile uint32*) ((volatile uchar*) ai_setcoreidx(&sii->pub, coreidx) +
+		               regoff);
+	}
+	ASSERT(r != NULL);
+
+	/* mask and set */
+	if (mask || val) {
+		w = (R_REG(sii->osh, r) & ~mask) | val;
+		W_REG(sii->osh, r, w);
+	}
+
+	if (!fast) {
+		/* restore core index */
+		if (origidx != coreidx)
+			ai_setcoreidx(&sii->pub, origidx);
+
+		INTR_RESTORE(sii, &intr_val);
+	}
+
+	return (w);
+}
+
+/*
+ * If there is no need for fiddling with interrupts or core switches (typically silicon
+ * back plane registers, pci registers and chipcommon registers), this function
+ * returns the register offset on this core to a mapped address. This address can
+ * be used for W_REG/R_REG directly.
+ *
+ * For accessing registers that would need a core switch, this function will return
+ * NULL.
+ */
+volatile uint32 *
+BCMPOSTTRAPFN(ai_corereg_addr)(si_t *sih, uint coreidx, uint regoff)
+{
+	volatile uint32 *r = NULL;
+	bool fast = FALSE;
+	si_info_t *sii = SI_INFO(sih);
+	si_cores_info_t *cores_info = (si_cores_info_t *)sii->cores_info;
+
+	ASSERT(GOODIDX(coreidx, sii->numcores));
+	ASSERT(regoff < SI_CORE_SIZE);
+
+	if (coreidx >= SI_MAXCORES)
+		return 0;
+
+	if (BUSTYPE(sih->bustype) == SI_BUS) {
+		/* If internal bus, we can always get at everything */
+		fast = TRUE;
+		/* map if does not exist */
+		if (!cores_info->regs[coreidx]) {
+			cores_info->regs[coreidx] = REG_MAP(cores_info->coresba[coreidx],
+			                            SI_CORE_SIZE);
+			ASSERT(GOODREGS(cores_info->regs[coreidx]));
+		}
+		r = (volatile uint32 *)((volatile uchar *)cores_info->regs[coreidx] + regoff);
+	} else if (BUSTYPE(sih->bustype) == PCI_BUS) {
+		/* If pci/pcie, we can get at pci/pcie regs and on newer cores to chipc */
+
+		if ((cores_info->coreid[coreidx] == CC_CORE_ID) && SI_FAST(sii)) {
+			/* Chipc registers are mapped at 12KB */
+
+			fast = TRUE;
+			r = (volatile uint32 *)((volatile char *)sii->curmap +
+			               PCI_16KB0_CCREGS_OFFSET + regoff);
+		} else if (sii->pub.buscoreidx == coreidx) {
+			/* pci registers are at either in the last 2KB of an 8KB window
+			 * or, in pcie and pci rev 13 at 8KB
+			 */
+			fast = TRUE;
+			if (SI_FAST(sii))
+				r = (volatile uint32 *)((volatile char *)sii->curmap +
+				               PCI_16KB0_PCIREGS_OFFSET + regoff);
+			else
+				r = (volatile uint32 *)((volatile char *)sii->curmap +
+				               ((regoff >= SBCONFIGOFF) ?
+				                PCI_BAR0_PCISBR_OFFSET : PCI_BAR0_PCIREGS_OFFSET) +
+				               regoff);
+		}
+	}
+
+	if (!fast) {
+		ASSERT(sii->curidx == coreidx);
+		r = (volatile uint32*) ((volatile uchar*)sii->curmap + regoff);
+	}
+
+	return (r);
+}
+
+void
+ai_core_disable(const si_t *sih, uint32 bits)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	volatile uint32 dummy;
+	uint32 status;
+	aidmp_t *ai;
+
+	ASSERT(GOODREGS(sii->curwrap));
+	ai = sii->curwrap;
+
+	/* if core is already in reset, just return */
+	if (R_REG(sii->osh, &ai->resetctrl) & AIRC_RESET) {
+		return;
+	}
+
+	/* ensure there are no pending backplane operations */
+	SPINWAIT(((status = R_REG(sii->osh, &ai->resetstatus)) != 0), 300);
+
+	/* if pending backplane ops still, try waiting longer */
+	if (status != 0) {
+		/* 300usecs was sufficient to allow backplane ops to clear for big hammer */
+		/* during driver load we may need more time */
+		SPINWAIT(((status = R_REG(sii->osh, &ai->resetstatus)) != 0), 10000);
+		/* if still pending ops, continue on and try disable anyway */
+		/* this is in big hammer path, so don't call wl_reinit in this case... */
+#ifdef BCMDBG
+		if (status != 0) {
+			SI_ERROR(("ai_core_disable: WARN: resetstatus=%0x on core disable\n",
+				status));
+		}
+#endif
+	}
+
+	W_REG(sii->osh, &ai->resetctrl, AIRC_RESET);
+	dummy = R_REG(sii->osh, &ai->resetctrl);
+	BCM_REFERENCE(dummy);
+	OSL_DELAY(1);
+
+	W_REG(sii->osh, &ai->ioctrl, bits);
+	dummy = R_REG(sii->osh, &ai->ioctrl);
+	BCM_REFERENCE(dummy);
+	OSL_DELAY(10);
+}
+
+/* reset and re-enable a core
+ * inputs:
+ * bits - core specific bits that are set during and after reset sequence
+ * resetbits - core specific bits that are set only during reset sequence
+ */
+static bool
+_ai_core_reset(const si_t *sih, uint32 bits, uint32 resetbits)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	aidmp_t *ai;
+	volatile uint32 dummy;
+	uint loop_counter = 10;
+
+	ASSERT(GOODREGS(sii->curwrap));
+	ai = sii->curwrap;
+
+	if (R_REG(sii->osh, &ai->resetstatus) == 0xffffffff &&
+		R_REG(sii->osh, &ai->ioctrl) == 0xffffffff &&
+		R_REG(sii->osh, &ai->resetctrl) == 0xffffffff) {
+		SI_ERROR(("%s: fail, resetstatus&ioctrl&resetctrl is 0xffffffff\n", __func__));
+		return FALSE;
+	}
+	/* ensure there are no pending backplane operations */
+	SPINWAIT(((dummy = R_REG(sii->osh, &ai->resetstatus)) != 0), 300);
+
+#ifdef BCMDBG_ERR
+	if (dummy != 0) {
+		SI_ERROR(("_ai_core_reset: WARN1: resetstatus=0x%0x\n", dummy));
+	}
+#endif /* BCMDBG_ERR */
+
+	SI_ERROR(("%s: &ai->ioctrl = 0x%x, &ai->resetctrl = 0x%x, &ai->resetstatus = 0x%x\n",
+			__func__,
+			R_REG(sii->osh, &ai->ioctrl),
+			R_REG(sii->osh, &ai->resetctrl),
+			R_REG(sii->osh, &ai->resetstatus)));
+
+	/* put core into reset state */
+	W_REG(sii->osh, &ai->resetctrl, AIRC_RESET);
+	OSL_DELAY(10);
+
+	/* ensure there are no pending backplane operations */
+	SPINWAIT((R_REG(sii->osh, &ai->resetstatus) != 0), 300);
+
+	W_REG(sii->osh, &ai->ioctrl, (bits | resetbits | SICF_FGC | SICF_CLOCK_EN));
+	dummy = R_REG(sii->osh, &ai->ioctrl);
+	BCM_REFERENCE(dummy);
+#ifdef UCM_CORRUPTION_WAR
+	if (si_coreid(sih) == D11_CORE_ID) {
+		/* Reset FGC */
+		OSL_DELAY(1);
+		W_REG(sii->osh, &ai->ioctrl, (dummy & (~SICF_FGC)));
+	}
+#endif /* UCM_CORRUPTION_WAR */
+	/* ensure there are no pending backplane operations */
+	SPINWAIT(((dummy = R_REG(sii->osh, &ai->resetstatus)) != 0), 300);
+
+#ifdef BCMDBG_ERR
+	if (dummy != 0)
+		SI_ERROR(("_ai_core_reset: WARN2: resetstatus=0x%0x\n", dummy));
+#endif
+
+	while (R_REG(sii->osh, &ai->resetctrl) != 0 && --loop_counter != 0) {
+		/* ensure there are no pending backplane operations */
+		SPINWAIT(((dummy = R_REG(sii->osh, &ai->resetstatus)) != 0), 300);
+
+#ifdef BCMDBG_ERR
+		if (dummy != 0)
+			SI_ERROR(("_ai_core_reset: WARN3 resetstatus=0x%0x\n", dummy));
+#endif
+
+		/* take core out of reset */
+		W_REG(sii->osh, &ai->resetctrl, 0);
+
+		/* ensure there are no pending backplane operations */
+		SPINWAIT((R_REG(sii->osh, &ai->resetstatus) != 0), 300);
+	}
+
+#ifdef BCMDBG_ERR
+	if (loop_counter == 0) {
+		SI_ERROR(("_ai_core_reset: Failed to take core 0x%x out of reset\n",
+			si_coreid(sih)));
+	}
+#endif
+
+#ifdef UCM_CORRUPTION_WAR
+	/* Pulse FGC after lifting Reset */
+	W_REG(sii->osh, &ai->ioctrl, (bits | SICF_FGC | SICF_CLOCK_EN));
+#else
+	W_REG(sii->osh, &ai->ioctrl, (bits | SICF_CLOCK_EN));
+#endif /* UCM_CORRUPTION_WAR */
+	dummy = R_REG(sii->osh, &ai->ioctrl);
+	BCM_REFERENCE(dummy);
+#ifdef UCM_CORRUPTION_WAR
+	if (si_coreid(sih) == D11_CORE_ID) {
+		/* Reset FGC */
+		OSL_DELAY(1);
+		W_REG(sii->osh, &ai->ioctrl, (dummy & (~SICF_FGC)));
+	}
+#endif /* UCM_CORRUPTION_WAR */
+	OSL_DELAY(1);
+	return TRUE;
+}
+
+bool
+ai_core_reset(si_t *sih, uint32 bits, uint32 resetbits)
+{
+	si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	uint idx = sii->curidx;
+	bool ret = TRUE;
+
+	if (cores_info->wrapba3[idx] != 0) {
+		ai_setcoreidx_3rdwrap(sih, idx);
+		ret = _ai_core_reset(sih, bits, resetbits);
+		ai_setcoreidx(sih, idx);
+	}
+
+	if (cores_info->wrapba2[idx] != 0) {
+		ai_setcoreidx_2ndwrap(sih, idx);
+		ret = _ai_core_reset(sih, bits, resetbits);
+		ai_setcoreidx(sih, idx);
+	}
+
+	ret = _ai_core_reset(sih, bits, resetbits);
+	return ret;
+}
+
+#ifdef BOOKER_NIC400_INF
+void
+BCMPOSTTRAPFN(ai_core_reset_ext)(const si_t *sih, uint32 bits, uint32 resetbits)
+{
+	_ai_core_reset(sih, bits, resetbits);
+}
+#endif /* BOOKER_NIC400_INF */
+
+void
+ai_core_cflags_wo(const si_t *sih, uint32 mask, uint32 val)
+{
+	const si_info_t *sii = SI_INFO(sih);
+#if !defined(BCMDONGLEHOST)
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+#endif
+	aidmp_t *ai;
+	uint32 w;
+
+	if (PMU_DMP()) {
+		SI_ERROR(("ai_core_cflags_wo: Accessing PMU DMP register (ioctrl)\n"));
+		return;
+	}
+
+	ASSERT(GOODREGS(sii->curwrap));
+	ai = sii->curwrap;
+
+	ASSERT((val & ~mask) == 0);
+
+	if (mask || val) {
+		w = ((R_REG(sii->osh, &ai->ioctrl) & ~mask) | val);
+		W_REG(sii->osh, &ai->ioctrl, w);
+	}
+}
+
+uint32
+BCMPOSTTRAPFN(ai_core_cflags)(const si_t *sih, uint32 mask, uint32 val)
+{
+	const si_info_t *sii = SI_INFO(sih);
+#if !defined(BCMDONGLEHOST)
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+#endif
+	aidmp_t *ai;
+	uint32 w;
+
+	if (PMU_DMP()) {
+		SI_ERROR(("ai_core_cflags: Accessing PMU DMP register (ioctrl)\n"));
+		return 0;
+	}
+	ASSERT(GOODREGS(sii->curwrap));
+	ai = sii->curwrap;
+
+	ASSERT((val & ~mask) == 0);
+
+	if (mask || val) {
+		w = ((R_REG(sii->osh, &ai->ioctrl) & ~mask) | val);
+		W_REG(sii->osh, &ai->ioctrl, w);
+	}
+
+	return R_REG(sii->osh, &ai->ioctrl);
+}
+
+uint32
+ai_core_sflags(const si_t *sih, uint32 mask, uint32 val)
+{
+	const si_info_t *sii = SI_INFO(sih);
+#if !defined(BCMDONGLEHOST)
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+#endif
+	aidmp_t *ai;
+	uint32 w;
+
+	if (PMU_DMP()) {
+		SI_ERROR(("ai_core_sflags: Accessing PMU DMP register (ioctrl)\n"));
+		return 0;
+	}
+
+	ASSERT(GOODREGS(sii->curwrap));
+	ai = sii->curwrap;
+
+	ASSERT((val & ~mask) == 0);
+	ASSERT((mask & ~SISF_CORE_BITS) == 0);
+
+	if (mask || val) {
+		w = ((R_REG(sii->osh, &ai->iostatus) & ~mask) | val);
+		W_REG(sii->osh, &ai->iostatus, w);
+	}
+
+	return R_REG(sii->osh, &ai->iostatus);
+}
+
+#if defined(BCMDBG) || defined(BCMDBG_DUMP) || defined(BCMDBG_PHYDUMP)
+/* print interesting aidmp registers */
+void
+ai_dumpregs(const si_t *sih, struct bcmstrbuf *b)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	osl_t *osh;
+	aidmp_t *ai;
+	uint i;
+	uint32 prev_value = 0;
+	const axi_wrapper_t * axi_wrapper = sii->axi_wrapper;
+	uint32 cfg_reg = 0;
+	uint bar0_win_offset = 0;
+
+	osh = sii->osh;
+
+	/* Save and restore wrapper access window */
+	if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+		if (PCIE_GEN2(sii)) {
+			cfg_reg = PCIE2_BAR0_CORE2_WIN2;
+			bar0_win_offset = PCIE2_BAR0_CORE2_WIN2_OFFSET;
+		} else {
+			cfg_reg = PCI_BAR0_WIN2;
+			bar0_win_offset = PCI_BAR0_WIN2_OFFSET;
+		}
+
+		prev_value = OSL_PCI_READ_CONFIG(osh, cfg_reg, 4);
+
+		if (prev_value == ID32_INVALID) {
+			SI_PRINT(("ai_dumpregs, PCI_BAR0_WIN2 - %x\n", prev_value));
+			return;
+		}
+	}
+
+	bcm_bprintf(b, "ChipNum:%x, ChipRev;%x, BusType:%x, BoardType:%x, BoardVendor:%x\n\n",
+		sih->chip, sih->chiprev, sih->bustype, sih->boardtype, sih->boardvendor);
+
+	for (i = 0; i < sii->axi_num_wrappers; i++) {
+
+		if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+			/* Set BAR0 window to bridge wapper base address */
+			OSL_PCI_WRITE_CONFIG(osh,
+				cfg_reg, 4, axi_wrapper[i].wrapper_addr);
+
+			ai = (aidmp_t *) ((volatile uint8*)sii->curmap + bar0_win_offset);
+		} else {
+			ai = (aidmp_t *)(uintptr) axi_wrapper[i].wrapper_addr;
+		}
+
+		bcm_bprintf(b, "core 0x%x: core_rev:%d, %s_WR ADDR:%x \n", axi_wrapper[i].cid,
+			axi_wrapper[i].rev,
+			axi_wrapper[i].wrapper_type == AI_SLAVE_WRAPPER ? "SLAVE" : "MASTER",
+			axi_wrapper[i].wrapper_addr);
+
+		bcm_bprintf(b, "ioctrlset 0x%x ioctrlclear 0x%x ioctrl 0x%x iostatus 0x%x "
+			    "ioctrlwidth 0x%x iostatuswidth 0x%x\n"
+			    "resetctrl 0x%x resetstatus 0x%x resetreadid 0x%x resetwriteid 0x%x\n"
+			    "errlogctrl 0x%x errlogdone 0x%x errlogstatus 0x%x "
+			    "errlogaddrlo 0x%x errlogaddrhi 0x%x\n"
+			    "errlogid 0x%x errloguser 0x%x errlogflags 0x%x\n"
+			    "intstatus 0x%x config 0x%x itcr 0x%x\n\n",
+			    R_REG(osh, &ai->ioctrlset),
+			    R_REG(osh, &ai->ioctrlclear),
+			    R_REG(osh, &ai->ioctrl),
+			    R_REG(osh, &ai->iostatus),
+			    R_REG(osh, &ai->ioctrlwidth),
+			    R_REG(osh, &ai->iostatuswidth),
+			    R_REG(osh, &ai->resetctrl),
+			    R_REG(osh, &ai->resetstatus),
+			    R_REG(osh, &ai->resetreadid),
+			    R_REG(osh, &ai->resetwriteid),
+			    R_REG(osh, &ai->errlogctrl),
+			    R_REG(osh, &ai->errlogdone),
+			    R_REG(osh, &ai->errlogstatus),
+			    R_REG(osh, &ai->errlogaddrlo),
+			    R_REG(osh, &ai->errlogaddrhi),
+			    R_REG(osh, &ai->errlogid),
+			    R_REG(osh, &ai->errloguser),
+			    R_REG(osh, &ai->errlogflags),
+			    R_REG(osh, &ai->intstatus),
+			    R_REG(osh, &ai->config),
+			    R_REG(osh, &ai->itcr));
+	}
+
+	/* Restore the initial wrapper space */
+	if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+		if (prev_value && cfg_reg) {
+			OSL_PCI_WRITE_CONFIG(osh, cfg_reg, 4, prev_value);
+		}
+	}
+}
+#endif	/* BCMDBG || BCMDBG_DUMP || BCMDBG_PHYDUMP */
+
+#ifdef BCMDBG
+static void
+_ai_view(osl_t *osh, aidmp_t *ai, uint32 cid, uint32 addr, bool verbose)
+{
+	uint32 config;
+
+	config = R_REG(osh, &ai->config);
+	SI_PRINT(("\nCore ID: 0x%x, addr 0x%x, config 0x%x\n", cid, addr, config));
+
+	if (config & AICFG_RST)
+		SI_PRINT(("resetctrl 0x%x, resetstatus 0x%x, resetreadid 0x%x, resetwriteid 0x%x\n",
+		          R_REG(osh, &ai->resetctrl), R_REG(osh, &ai->resetstatus),
+		          R_REG(osh, &ai->resetreadid), R_REG(osh, &ai->resetwriteid)));
+
+	if (config & AICFG_IOC)
+		SI_PRINT(("ioctrl 0x%x, width %d\n", R_REG(osh, &ai->ioctrl),
+		          R_REG(osh, &ai->ioctrlwidth)));
+
+	if (config & AICFG_IOS)
+		SI_PRINT(("iostatus 0x%x, width %d\n", R_REG(osh, &ai->iostatus),
+		          R_REG(osh, &ai->iostatuswidth)));
+
+	if (config & AICFG_ERRL) {
+		SI_PRINT(("errlogctrl 0x%x, errlogdone 0x%x, errlogstatus 0x%x, intstatus 0x%x\n",
+		          R_REG(osh, &ai->errlogctrl), R_REG(osh, &ai->errlogdone),
+		          R_REG(osh, &ai->errlogstatus), R_REG(osh, &ai->intstatus)));
+		SI_PRINT(("errlogid 0x%x, errloguser 0x%x, errlogflags 0x%x, errlogaddr "
+		          "0x%x/0x%x\n",
+		          R_REG(osh, &ai->errlogid), R_REG(osh, &ai->errloguser),
+		          R_REG(osh, &ai->errlogflags), R_REG(osh, &ai->errlogaddrhi),
+		          R_REG(osh, &ai->errlogaddrlo)));
+	}
+
+	if (verbose && (config & AICFG_OOB)) {
+		SI_PRINT(("oobselina30 0x%x, oobselina74 0x%x\n",
+		          R_REG(osh, &ai->oobselina30), R_REG(osh, &ai->oobselina74)));
+		SI_PRINT(("oobselinb30 0x%x, oobselinb74 0x%x\n",
+		          R_REG(osh, &ai->oobselinb30), R_REG(osh, &ai->oobselinb74)));
+		SI_PRINT(("oobselinc30 0x%x, oobselinc74 0x%x\n",
+		          R_REG(osh, &ai->oobselinc30), R_REG(osh, &ai->oobselinc74)));
+		SI_PRINT(("oobselind30 0x%x, oobselind74 0x%x\n",
+		          R_REG(osh, &ai->oobselind30), R_REG(osh, &ai->oobselind74)));
+		SI_PRINT(("oobselouta30 0x%x, oobselouta74 0x%x\n",
+		          R_REG(osh, &ai->oobselouta30), R_REG(osh, &ai->oobselouta74)));
+		SI_PRINT(("oobseloutb30 0x%x, oobseloutb74 0x%x\n",
+		          R_REG(osh, &ai->oobseloutb30), R_REG(osh, &ai->oobseloutb74)));
+		SI_PRINT(("oobseloutc30 0x%x, oobseloutc74 0x%x\n",
+		          R_REG(osh, &ai->oobseloutc30), R_REG(osh, &ai->oobseloutc74)));
+		SI_PRINT(("oobseloutd30 0x%x, oobseloutd74 0x%x\n",
+		          R_REG(osh, &ai->oobseloutd30), R_REG(osh, &ai->oobseloutd74)));
+		SI_PRINT(("oobsynca 0x%x, oobseloutaen 0x%x\n",
+		          R_REG(osh, &ai->oobsynca), R_REG(osh, &ai->oobseloutaen)));
+		SI_PRINT(("oobsyncb 0x%x, oobseloutben 0x%x\n",
+		          R_REG(osh, &ai->oobsyncb), R_REG(osh, &ai->oobseloutben)));
+		SI_PRINT(("oobsyncc 0x%x, oobseloutcen 0x%x\n",
+		          R_REG(osh, &ai->oobsyncc), R_REG(osh, &ai->oobseloutcen)));
+		SI_PRINT(("oobsyncd 0x%x, oobseloutden 0x%x\n",
+		          R_REG(osh, &ai->oobsyncd), R_REG(osh, &ai->oobseloutden)));
+		SI_PRINT(("oobaextwidth 0x%x, oobainwidth 0x%x, oobaoutwidth 0x%x\n",
+		          R_REG(osh, &ai->oobaextwidth), R_REG(osh, &ai->oobainwidth),
+		          R_REG(osh, &ai->oobaoutwidth)));
+		SI_PRINT(("oobbextwidth 0x%x, oobbinwidth 0x%x, oobboutwidth 0x%x\n",
+		          R_REG(osh, &ai->oobbextwidth), R_REG(osh, &ai->oobbinwidth),
+		          R_REG(osh, &ai->oobboutwidth)));
+		SI_PRINT(("oobcextwidth 0x%x, oobcinwidth 0x%x, oobcoutwidth 0x%x\n",
+		          R_REG(osh, &ai->oobcextwidth), R_REG(osh, &ai->oobcinwidth),
+		          R_REG(osh, &ai->oobcoutwidth)));
+		SI_PRINT(("oobdextwidth 0x%x, oobdinwidth 0x%x, oobdoutwidth 0x%x\n",
+		          R_REG(osh, &ai->oobdextwidth), R_REG(osh, &ai->oobdinwidth),
+		          R_REG(osh, &ai->oobdoutwidth)));
+	}
+}
+
+void
+ai_view(const si_t *sih, bool verbose)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	osl_t *osh;
+	aidmp_t *ai;
+	uint32 cid, addr;
+
+	ai = sii->curwrap;
+	osh = sii->osh;
+
+	if (PMU_DMP()) {
+		SI_ERROR(("Cannot access pmu DMP\n"));
+		return;
+	}
+	cid = cores_info->coreid[sii->curidx];
+	addr = cores_info->wrapba[sii->curidx];
+	_ai_view(osh, ai, cid, addr, verbose);
+}
+
+void
+ai_viewall(si_t *sih, bool verbose)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	osl_t *osh;
+	aidmp_t *ai;
+	uint32 cid, addr;
+	uint i;
+
+	osh = sii->osh;
+	for (i = 0; i < sii->numcores; i++) {
+		si_setcoreidx(sih, i);
+
+		if (PMU_DMP()) {
+			SI_ERROR(("Skipping pmu DMP\n"));
+			continue;
+		}
+		ai = sii->curwrap;
+		cid = cores_info->coreid[sii->curidx];
+		addr = cores_info->wrapba[sii->curidx];
+		_ai_view(osh, ai, cid, addr, verbose);
+	}
+}
+#endif	/* BCMDBG */
+
+void
+ai_update_backplane_timeouts(const si_t *sih, bool enable, uint32 timeout_exp, uint32 cid)
+{
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+	const si_info_t *sii = SI_INFO(sih);
+	aidmp_t *ai;
+	uint32 i;
+	axi_wrapper_t * axi_wrapper = sii->axi_wrapper;
+	uint32 errlogctrl = (enable << AIELC_TO_ENAB_SHIFT) |
+		((timeout_exp << AIELC_TO_EXP_SHIFT) & AIELC_TO_EXP_MASK);
+
+#ifdef AXI_TIMEOUTS_NIC
+	uint32 prev_value = 0;
+	osl_t *osh = sii->osh;
+	uint32 cfg_reg = 0;
+	uint32 offset = 0;
+#endif /* AXI_TIMEOUTS_NIC */
+
+	if ((sii->axi_num_wrappers == 0) ||
+#ifdef AXI_TIMEOUTS_NIC
+		(!PCIE(sii)) ||
+#endif /* AXI_TIMEOUTS_NIC */
+		FALSE) {
+		SI_VMSG((" iai_update_backplane_timeouts, axi_num_wrappers:%d, Is_PCIE:%d,"
+			" BUS_TYPE:%d, ID:%x\n",
+			sii->axi_num_wrappers, PCIE(sii),
+			BUSTYPE(sii->pub.bustype), sii->pub.buscoretype));
+		return;
+	}
+
+#ifdef AXI_TIMEOUTS_NIC
+	/* Save and restore the wrapper access window */
+	if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+		if (PCIE_GEN1(sii)) {
+			cfg_reg = PCI_BAR0_WIN2;
+			offset = PCI_BAR0_WIN2_OFFSET;
+		} else if (PCIE_GEN2(sii)) {
+			cfg_reg = PCIE2_BAR0_CORE2_WIN2;
+			offset = PCIE2_BAR0_CORE2_WIN2_OFFSET;
+		}
+		else {
+			ASSERT(!"!PCIE_GEN1 && !PCIE_GEN2");
+		}
+
+		prev_value = OSL_PCI_READ_CONFIG(osh, cfg_reg, 4);
+		if (prev_value == ID32_INVALID) {
+			SI_PRINT(("ai_update_backplane_timeouts, PCI_BAR0_WIN2 - %x\n",
+				prev_value));
+			return;
+		}
+	}
+#endif /* AXI_TIMEOUTS_NIC */
+
+	for (i = 0; i < sii->axi_num_wrappers; ++i) {
+		/* WAR for wrong EROM entries w.r.t slave and master wrapper
+		 * for ADB bridge core...so checking actual wrapper config to determine type
+		 * http://jira.broadcom.com/browse/HW4388-905
+		*/
+		if ((cid == 0 || cid == ADB_BRIDGE_ID) &&
+				(axi_wrapper[i].cid == ADB_BRIDGE_ID)) {
+			/* WAR is applicable only to 89B0 and 89C0 */
+			if (CCREV(sih->ccrev) == 70) {
+				ai = (aidmp_t *)(uintptr)axi_wrapper[i].wrapper_addr;
+				if (R_REG(sii->osh, &ai->config) & WRAPPER_TIMEOUT_CONFIG) {
+					axi_wrapper[i].wrapper_type  = AI_SLAVE_WRAPPER;
+				} else {
+					axi_wrapper[i].wrapper_type  = AI_MASTER_WRAPPER;
+				}
+			}
+		}
+		if (axi_wrapper[i].wrapper_type != AI_SLAVE_WRAPPER || ((BCM4389_CHIP(sih->chip) ||
+				BCM4388_CHIP(sih->chip)) &&
+				(axi_wrapper[i].wrapper_addr == WL_BRIDGE1_S ||
+				axi_wrapper[i].wrapper_addr == WL_BRIDGE2_S))) {
+			SI_VMSG(("SKIP ENABLE BPT: MFG:%x, CID:%x, ADDR:%x\n",
+				axi_wrapper[i].mfg,
+				axi_wrapper[i].cid,
+				axi_wrapper[i].wrapper_addr));
+			continue;
+		}
+
+		/* Update only given core if requested */
+		if ((cid != 0) && (axi_wrapper[i].cid != cid)) {
+			continue;
+		}
+
+#ifdef AXI_TIMEOUTS_NIC
+		if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+			/* Set BAR0_CORE2_WIN2 to bridge wapper base address */
+			OSL_PCI_WRITE_CONFIG(osh,
+				cfg_reg, 4, axi_wrapper[i].wrapper_addr);
+
+			/* set AI to BAR0 + Offset corresponding to Gen1 or gen2 */
+			ai = (aidmp_t *) (DISCARD_QUAL(sii->curmap, uint8) + offset);
+		}
+		else
+#endif /* AXI_TIMEOUTS_NIC */
+		{
+			ai = (aidmp_t *)(uintptr) axi_wrapper[i].wrapper_addr;
+		}
+
+		W_REG(sii->osh, &ai->errlogctrl, errlogctrl);
+
+		SI_VMSG(("ENABLED BPT: MFG:%x, CID:%x, ADDR:%x, ERR_CTRL:%x\n",
+			axi_wrapper[i].mfg,
+			axi_wrapper[i].cid,
+			axi_wrapper[i].wrapper_addr,
+			R_REG(sii->osh, &ai->errlogctrl)));
+	}
+
+#ifdef AXI_TIMEOUTS_NIC
+	/* Restore the initial wrapper space */
+	if (prev_value) {
+		OSL_PCI_WRITE_CONFIG(osh, cfg_reg, 4, prev_value);
+	}
+#endif /* AXI_TIMEOUTS_NIC */
+
+#endif /* AXI_TIMEOUTS || AXI_TIMEOUTS_NIC */
+}
+
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+
+/* slave error is ignored, so account for those cases */
+static uint32 si_ignore_errlog_cnt = 0;
+
+static bool
+BCMPOSTTRAPFN(ai_ignore_errlog)(const si_info_t *sii, const aidmp_t *ai,
+	uint32 lo_addr, uint32 hi_addr, uint32 err_axi_id, uint32 errsts)
+{
+	uint32 ignore_errsts = AIELS_SLAVE_ERR;
+	uint32 ignore_errsts_2 = 0;
+	uint32 ignore_hi = BT_CC_SPROM_BADREG_HI;
+	uint32 ignore_lo = BT_CC_SPROM_BADREG_LO;
+	uint32 ignore_size = BT_CC_SPROM_BADREG_SIZE;
+	bool address_check = TRUE;
+	uint32 axi_id = 0;
+	uint32 axi_id2 = 0;
+	bool extd_axi_id_mask = FALSE;
+	uint32 axi_id_mask;
+
+	SI_PRINT(("err check: core %p, error %d, axi id 0x%04x, addr(0x%08x:%08x)\n",
+		ai, errsts, err_axi_id, hi_addr, lo_addr));
+
+	/* ignore the BT slave errors if the errlog is to chipcommon addr 0x190 */
+	switch (CHIPID(sii->pub.chip)) {
+#if defined(BT_WLAN_REG_ON_WAR)
+		/*
+		 * 4389B0/C0 - WL and BT turn on WAR, ignore AXI error originating from
+		 * AHB-AXI bridge i.e, any slave error or timeout from BT access
+		 */
+		case BCM4389_CHIP_GRPID:
+			axi_id = BCM4389_BT_AXI_ID;
+			ignore_errsts = AIELS_SLAVE_ERR;
+			axi_id2 = BCM4389_BT_AXI_ID;
+			ignore_errsts_2 = AIELS_TIMEOUT;
+			address_check = FALSE;
+			extd_axi_id_mask = TRUE;
+			break;
+#endif /* BT_WLAN_REG_ON_WAR */
+#ifdef BTOVERPCIE
+		case BCM4388_CHIP_GRPID:
+			axi_id = BCM4388_BT_AXI_ID;
+		/* For BT over PCIE, ignore any slave error from BT. */
+		/* No need to check any address range */
+			address_check = FALSE;
+			ignore_errsts_2 = AIELS_DECODE;
+			break;
+		case BCM4369_CHIP_GRPID:
+			axi_id = BCM4369_BT_AXI_ID;
+		/* For BT over PCIE, ignore any slave error from BT. */
+		/* No need to check any address range */
+			address_check = FALSE;
+			ignore_errsts_2 = AIELS_DECODE;
+			break;
+#endif /* BTOVERPCIE */
+		case BCM4376_CHIP_GRPID:
+		case BCM4378_CHIP_GRPID:
+		case BCM4385_CHIP_GRPID:
+		case BCM4387_CHIP_GRPID:
+#ifdef BTOVERPCIE
+			axi_id = BCM4378_BT_AXI_ID;
+			/* For BT over PCIE, ignore any slave error from BT. */
+			/* No need to check any address range */
+			address_check = FALSE;
+#endif /* BTOVERPCIE */
+			axi_id2 = BCM4378_ARM_PREFETCH_AXI_ID;
+			extd_axi_id_mask = TRUE;
+			ignore_errsts_2 = AIELS_DECODE;
+			break;
+		default:
+			return FALSE;
+	}
+
+	axi_id_mask = extd_axi_id_mask ? AI_ERRLOGID_AXI_ID_MASK_EXTD : AI_ERRLOGID_AXI_ID_MASK;
+
+	/* AXI ID check */
+	err_axi_id &= axi_id_mask;
+	errsts &=  AIELS_ERROR_MASK;
+
+	/* check the ignore error cases. 2 checks */
+	if (!(((err_axi_id == axi_id) && (errsts == ignore_errsts)) ||
+		((err_axi_id == axi_id2) && (errsts == ignore_errsts_2)))) {
+		/* not the error ignore cases */
+		return FALSE;
+
+	}
+
+	/* check the specific address checks now, if specified */
+	if (address_check) {
+		/* address range check */
+		if ((hi_addr != ignore_hi) ||
+		    (lo_addr < ignore_lo) || (lo_addr >= (ignore_lo + ignore_size))) {
+			return FALSE;
+		}
+	}
+
+	SI_PRINT(("err check: ignored\n"));
+	return TRUE;
+}
+#endif /* defined (AXI_TIMEOUTS) || defined (AXI_TIMEOUTS_NIC) */
+
+#ifdef AXI_TIMEOUTS_NIC
+
+/* Function to return the APB bridge details corresponding to the core */
+static bool
+ai_get_apb_bridge(const si_t * sih, uint32 coreidx, uint32 *apb_id, uint32 * apb_coreunit)
+{
+	uint i;
+	uint32 core_base, core_end;
+	const si_info_t *sii = SI_INFO(sih);
+	static uint32 coreidx_cached = 0, apb_id_cached = 0, apb_coreunit_cached = 0;
+	uint32 tmp_coreunit = 0;
+	const si_cores_info_t *cores_info = (si_cores_info_t *)sii->cores_info;
+
+	if (coreidx >= MIN(sii->numcores, SI_MAXCORES))
+		return FALSE;
+
+	/* Most of the time apb bridge query will be for d11 core.
+	 * Maintain the last cache and return if found rather than iterating the table
+	 */
+	if (coreidx_cached == coreidx) {
+		*apb_id = apb_id_cached;
+		*apb_coreunit = apb_coreunit_cached;
+		return TRUE;
+	}
+
+	core_base = cores_info->coresba[coreidx];
+	core_end = core_base + cores_info->coresba_size[coreidx];
+
+	for (i = 0; i < sii->numcores; i++) {
+		if (cores_info->coreid[i] == APB_BRIDGE_ID) {
+			uint32 apb_base;
+			uint32 apb_end;
+
+			apb_base = cores_info->coresba[i];
+			apb_end = apb_base + cores_info->coresba_size[i];
+
+			if ((core_base >= apb_base) &&
+				(core_end <= apb_end)) {
+				/* Current core is attached to this APB bridge */
+				*apb_id = apb_id_cached = APB_BRIDGE_ID;
+				*apb_coreunit = apb_coreunit_cached = tmp_coreunit;
+				coreidx_cached = coreidx;
+				return TRUE;
+			}
+			/* Increment the coreunit */
+			tmp_coreunit++;
+		}
+	}
+
+	return FALSE;
+}
+
+uint32
+ai_clear_backplane_to_fast(si_t *sih, void *addr)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	volatile const void *curmap = sii->curmap;
+	bool core_reg = FALSE;
+
+	/* Use fast path only for core register access */
+	if (((uintptr)addr >= (uintptr)curmap) &&
+		((uintptr)addr < ((uintptr)curmap + SI_CORE_SIZE))) {
+		/* address being accessed is within current core reg map */
+		core_reg = TRUE;
+	}
+
+	if (core_reg) {
+		uint32 apb_id, apb_coreunit;
+
+		if (ai_get_apb_bridge(sih, si_coreidx(&sii->pub),
+			&apb_id, &apb_coreunit) == TRUE) {
+			/* Found the APB bridge corresponding to current core,
+			 * Check for bus errors in APB wrapper
+			 */
+			return ai_clear_backplane_to_per_core(sih,
+				apb_id, apb_coreunit, NULL);
+		}
+	}
+
+	/* Default is to poll for errors on all slave wrappers */
+	return si_clear_backplane_to(sih);
+}
+#endif /* AXI_TIMEOUTS_NIC */
+
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+static bool g_disable_backplane_logs = FALSE;
+
+static uint32 last_axi_error = AXI_WRAP_STS_NONE;
+static uint32 last_axi_error_log_status = 0;
+static uint32 last_axi_error_core = 0;
+static uint32 last_axi_error_wrap = 0;
+static uint32 last_axi_errlog_lo = 0;
+static uint32 last_axi_errlog_hi = 0;
+static uint32 last_axi_errlog_id = 0;
+
+/*
+ * API to clear the back plane timeout per core.
+ * Caller may pass optional wrapper address. If present this will be used as
+ * the wrapper base address. If wrapper base address is provided then caller
+ * must provide the coreid also.
+ * If both coreid and wrapper is zero, then err status of current bridge
+ * will be verified.
+ */
+uint32
+BCMPOSTTRAPFN(ai_clear_backplane_to_per_core)(si_t *sih, uint coreid, uint coreunit, void *wrap)
+{
+	int ret = AXI_WRAP_STS_NONE;
+	aidmp_t *ai = NULL;
+	uint32 errlog_status = 0;
+	const si_info_t *sii = SI_INFO(sih);
+	uint32 errlog_lo = 0, errlog_hi = 0, errlog_id = 0, errlog_flags = 0;
+	uint32 current_coreidx = si_coreidx(sih);
+	uint32 target_coreidx = si_findcoreidx(sih, coreid, coreunit);
+
+#if defined(AXI_TIMEOUTS_NIC)
+	si_axi_error_t * axi_error = sih->err_info ?
+		&sih->err_info->axi_error[sih->err_info->count] : NULL;
+#endif /* AXI_TIMEOUTS_NIC */
+	bool restore_core = FALSE;
+
+	if ((sii->axi_num_wrappers == 0) ||
+#ifdef AXI_TIMEOUTS_NIC
+		(!PCIE(sii)) ||
+#endif /* AXI_TIMEOUTS_NIC */
+		FALSE) {
+		SI_VMSG(("ai_clear_backplane_to_per_core, axi_num_wrappers:%d, Is_PCIE:%d,"
+			" BUS_TYPE:%d, ID:%x\n",
+			sii->axi_num_wrappers, PCIE(sii),
+			BUSTYPE(sii->pub.bustype), sii->pub.buscoretype));
+		return AXI_WRAP_STS_NONE;
+	}
+
+	if (wrap != NULL) {
+		ai = (aidmp_t *)wrap;
+	} else if (coreid && (target_coreidx != current_coreidx)) {
+
+		if (ai_setcoreidx(sih, target_coreidx) == NULL) {
+			/* Unable to set the core */
+			SI_PRINT(("Set Code Failed: coreid:%x, unit:%d, target_coreidx:%d\n",
+				coreid, coreunit, target_coreidx));
+			errlog_lo = target_coreidx;
+			ret = AXI_WRAP_STS_SET_CORE_FAIL;
+			goto end;
+		}
+
+		restore_core = TRUE;
+		ai = (aidmp_t *)si_wrapperregs(sih);
+	} else {
+		/* Read error status of current wrapper */
+		ai = (aidmp_t *)si_wrapperregs(sih);
+
+		/* Update CoreID to current Code ID */
+		coreid = si_coreid(sih);
+	}
+
+	/* read error log status */
+	errlog_status = R_REG(sii->osh, &ai->errlogstatus);
+
+	if (errlog_status == ID32_INVALID) {
+		/* Do not try to peek further */
+		SI_PRINT(("ai_clear_backplane_to_per_core, errlogstatus:%x - Slave Wrapper:%x\n",
+			errlog_status, coreid));
+		ret = AXI_WRAP_STS_WRAP_RD_ERR;
+		errlog_lo = (uint32)(uintptr)&ai->errlogstatus;
+		goto end;
+	}
+
+	if ((errlog_status & AIELS_ERROR_MASK) != 0) {
+		uint32 tmp;
+		uint32 count = 0;
+		/* set ErrDone to clear the condition */
+		W_REG(sii->osh, &ai->errlogdone, AIELD_ERRDONE_MASK);
+
+		/* SPINWAIT on errlogstatus timeout status bits */
+		while ((tmp = R_REG(sii->osh, &ai->errlogstatus)) & AIELS_ERROR_MASK) {
+
+			if (tmp == ID32_INVALID) {
+				SI_PRINT(("ai_clear_backplane_to_per_core: prev errlogstatus:%x,"
+					" errlogstatus:%x\n",
+					errlog_status, tmp));
+				ret = AXI_WRAP_STS_WRAP_RD_ERR;
+				errlog_lo = (uint32)(uintptr)&ai->errlogstatus;
+				goto end;
+			}
+			/*
+			 * Clear again, to avoid getting stuck in the loop, if a new error
+			 * is logged after we cleared the first timeout
+			 */
+			W_REG(sii->osh, &ai->errlogdone, AIELD_ERRDONE_MASK);
+
+			count++;
+			OSL_DELAY(10);
+			if ((10 * count) > AI_REG_READ_TIMEOUT) {
+				errlog_status = tmp;
+				break;
+			}
+		}
+
+		errlog_lo = R_REG(sii->osh, &ai->errlogaddrlo);
+		errlog_hi = R_REG(sii->osh, &ai->errlogaddrhi);
+		errlog_id = R_REG(sii->osh, &ai->errlogid);
+		errlog_flags = R_REG(sii->osh, &ai->errlogflags);
+
+		/* we are already in the error path, so OK to check for the  slave error */
+		if (ai_ignore_errlog(sii, ai, errlog_lo, errlog_hi, errlog_id,
+			errlog_status)) {
+			si_ignore_errlog_cnt++;
+			goto end;
+		}
+
+		/* only reset APB Bridge on timeout (not slave error, or dec error) */
+		switch (errlog_status & AIELS_ERROR_MASK) {
+			case AIELS_SLAVE_ERR:
+				SI_PRINT(("AXI slave error\n"));
+				ret |= AXI_WRAP_STS_SLAVE_ERR;
+				break;
+
+			case AIELS_TIMEOUT:
+				ai_reset_axi_to(sii, ai);
+				ret |= AXI_WRAP_STS_TIMEOUT;
+				break;
+
+			case AIELS_DECODE:
+				SI_PRINT(("AXI decode error\n"));
+				{
+					ret |= AXI_WRAP_STS_DECODE_ERR;
+				}
+				break;
+			default:
+				ASSERT(0);	/* should be impossible */
+		}
+
+		if (errlog_status & AIELS_MULTIPLE_ERRORS) {
+			SI_PRINT(("Multiple AXI Errors\n"));
+			/* Set multiple errors bit only if actual error is not ignored */
+			if (ret) {
+				ret |= AXI_WRAP_STS_MULTIPLE_ERRORS;
+			}
+		}
+
+		SI_PRINT(("\tCoreID: %x\n", coreid));
+		SI_PRINT(("\t errlog: lo 0x%08x, hi 0x%08x, id 0x%08x, flags 0x%08x"
+			", status 0x%08x\n",
+			errlog_lo, errlog_hi, errlog_id, errlog_flags,
+			errlog_status));
+	}
+
+end:
+	if (ret != AXI_WRAP_STS_NONE) {
+		last_axi_error = ret;
+		last_axi_error_log_status = errlog_status;
+		last_axi_error_core = coreid;
+		last_axi_error_wrap = (uint32)ai;
+		last_axi_errlog_lo = errlog_lo;
+		last_axi_errlog_hi = errlog_hi;
+		last_axi_errlog_id = errlog_id;
+	}
+
+#if defined(AXI_TIMEOUTS_NIC)
+	if (axi_error && (ret != AXI_WRAP_STS_NONE)) {
+		axi_error->error = ret;
+		axi_error->coreid = coreid;
+		axi_error->errlog_lo = errlog_lo;
+		axi_error->errlog_hi = errlog_hi;
+		axi_error->errlog_id = errlog_id;
+		axi_error->errlog_flags = errlog_flags;
+		axi_error->errlog_status = errlog_status;
+		sih->err_info->count++;
+
+		if (sih->err_info->count == SI_MAX_ERRLOG_SIZE) {
+			sih->err_info->count = SI_MAX_ERRLOG_SIZE - 1;
+			SI_PRINT(("AXI Error log overflow\n"));
+		}
+	}
+#endif /* AXI_TIMEOUTS_NIC */
+
+	if (restore_core) {
+		if (ai_setcoreidx(sih, current_coreidx) == NULL) {
+			/* Unable to set the core */
+			return ID32_INVALID;
+		}
+	}
+
+	return ret;
+}
+
+/* reset AXI timeout */
+static void
+BCMPOSTTRAPFN(ai_reset_axi_to)(const si_info_t *sii, aidmp_t *ai)
+{
+	/* reset APB Bridge */
+	OR_REG(sii->osh, &ai->resetctrl, AIRC_RESET);
+	/* sync write */
+	(void)R_REG(sii->osh, &ai->resetctrl);
+	/* clear Reset bit */
+	AND_REG(sii->osh, &ai->resetctrl, ~(AIRC_RESET));
+	/* sync write */
+	(void)R_REG(sii->osh, &ai->resetctrl);
+	SI_PRINT(("AXI timeout\n"));
+	if (R_REG(sii->osh, &ai->resetctrl) & AIRC_RESET) {
+		SI_PRINT(("reset failed on wrapper %p\n", ai));
+		g_disable_backplane_logs = TRUE;
+	}
+}
+
+void
+BCMPOSTTRAPFN(ai_wrapper_get_last_error)(const si_t *sih, uint32 *error_status, uint32 *core,
+	uint32 *lo, uint32 *hi, uint32 *id)
+{
+	*error_status = last_axi_error_log_status;
+	*core = last_axi_error_core;
+	*lo = last_axi_errlog_lo;
+	*hi = last_axi_errlog_hi;
+	*id = last_axi_errlog_id;
+}
+
+/* Function to check whether AXI timeout has been registered on a core */
+uint32
+ai_get_axi_timeout_reg(void)
+{
+	return (GOODREGS(last_axi_errlog_lo) ? last_axi_errlog_lo : 0);
+}
+#endif /* AXI_TIMEOUTS || AXI_TIMEOUTS_NIC */
+
+uint32
+BCMPOSTTRAPFN(ai_findcoreidx_by_axiid)(const si_t *sih, uint32 axiid)
+{
+	uint coreid = 0;
+	uint coreunit = 0;
+	const axi_to_coreidx_t *axi2coreidx = NULL;
+	switch (CHIPID(sih->chip)) {
+		case BCM4369_CHIP_GRPID:
+			axi2coreidx = axi2coreidx_4369;
+			break;
+		default:
+			SI_PRINT(("Chipid mapping not found\n"));
+			break;
+	}
+
+	if (!axi2coreidx)
+		return (BADIDX);
+
+	coreid = axi2coreidx[axiid].coreid;
+	coreunit = axi2coreidx[axiid].coreunit;
+
+	return si_findcoreidx(sih, coreid, coreunit);
+
+}
+
+/*
+ * This API polls all slave wrappers for errors and returns bit map of
+ * all reported errors.
+ * return - bit map of
+ *	AXI_WRAP_STS_NONE
+ *	AXI_WRAP_STS_TIMEOUT
+ *	AXI_WRAP_STS_SLAVE_ERR
+ *	AXI_WRAP_STS_DECODE_ERR
+ *	AXI_WRAP_STS_PCI_RD_ERR
+ *	AXI_WRAP_STS_WRAP_RD_ERR
+ *	AXI_WRAP_STS_SET_CORE_FAIL
+ * On timeout detection, correspondign bridge will be reset to
+ * unblock the bus.
+ * Error reported in each wrapper can be retrieved using the API
+ * si_get_axi_errlog_info()
+ */
+uint32
+BCMPOSTTRAPFN(ai_clear_backplane_to)(si_t *sih)
+{
+	uint32 ret = 0;
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+	const si_info_t *sii = SI_INFO(sih);
+	aidmp_t *ai;
+	uint32 i;
+	axi_wrapper_t * axi_wrapper = sii->axi_wrapper;
+
+#ifdef AXI_TIMEOUTS_NIC
+	uint32 prev_value = 0;
+	osl_t *osh = sii->osh;
+	uint32 cfg_reg = 0;
+	uint32 offset = 0;
+
+	if ((sii->axi_num_wrappers == 0) || (!PCIE(sii)))
+#else
+	if (sii->axi_num_wrappers == 0)
+#endif
+	{
+		SI_VMSG(("ai_clear_backplane_to, axi_num_wrappers:%d, Is_PCIE:%d, BUS_TYPE:%d,"
+			" ID:%x\n",
+			sii->axi_num_wrappers, PCIE(sii),
+			BUSTYPE(sii->pub.bustype), sii->pub.buscoretype));
+		return AXI_WRAP_STS_NONE;
+	}
+
+#ifdef AXI_TIMEOUTS_NIC
+	/* Save and restore wrapper access window */
+	if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+		if (PCIE_GEN1(sii)) {
+			cfg_reg = PCI_BAR0_WIN2;
+			offset = PCI_BAR0_WIN2_OFFSET;
+		} else if (PCIE_GEN2(sii)) {
+			cfg_reg = PCIE2_BAR0_CORE2_WIN2;
+			offset = PCIE2_BAR0_CORE2_WIN2_OFFSET;
+		}
+		else {
+			ASSERT(!"!PCIE_GEN1 && !PCIE_GEN2");
+		}
+
+		prev_value = OSL_PCI_READ_CONFIG(osh, cfg_reg, 4);
+
+		if (prev_value == ID32_INVALID) {
+			si_axi_error_t * axi_error =
+				sih->err_info ?
+					&sih->err_info->axi_error[sih->err_info->count] :
+					NULL;
+
+			SI_PRINT(("ai_clear_backplane_to, PCI_BAR0_WIN2 - %x\n", prev_value));
+			if (axi_error) {
+				axi_error->error = ret = AXI_WRAP_STS_PCI_RD_ERR;
+				axi_error->errlog_lo = cfg_reg;
+				sih->err_info->count++;
+
+				if (sih->err_info->count == SI_MAX_ERRLOG_SIZE) {
+					sih->err_info->count = SI_MAX_ERRLOG_SIZE - 1;
+					SI_PRINT(("AXI Error log overflow\n"));
+				}
+			}
+
+			return ret;
+		}
+	}
+#endif /* AXI_TIMEOUTS_NIC */
+
+	for (i = 0; i < sii->axi_num_wrappers; ++i) {
+		uint32 tmp;
+
+		if (axi_wrapper[i].wrapper_type != AI_SLAVE_WRAPPER) {
+			continue;
+		}
+
+#ifdef AXI_TIMEOUTS_NIC
+		if (BUSTYPE(sii->pub.bustype) == PCI_BUS) {
+			/* Set BAR0_CORE2_WIN2 to bridge wapper base address */
+			OSL_PCI_WRITE_CONFIG(osh,
+				cfg_reg, 4, axi_wrapper[i].wrapper_addr);
+
+			/* set AI to BAR0 + Offset corresponding to Gen1 or gen2 */
+			ai = (aidmp_t *) (DISCARD_QUAL(sii->curmap, uint8) + offset);
+		}
+		else
+#endif /* AXI_TIMEOUTS_NIC */
+		{
+			ai = (aidmp_t *)(uintptr) axi_wrapper[i].wrapper_addr;
+		}
+
+		tmp = ai_clear_backplane_to_per_core(sih, axi_wrapper[i].cid, 0,
+			DISCARD_QUAL(ai, void));
+
+		ret |= tmp;
+	}
+
+#ifdef AXI_TIMEOUTS_NIC
+	/* Restore the initial wrapper space */
+	if (prev_value) {
+		OSL_PCI_WRITE_CONFIG(osh, cfg_reg, 4, prev_value);
+	}
+#endif /* AXI_TIMEOUTS_NIC */
+
+#endif /* AXI_TIMEOUTS || AXI_TIMEOUTS_NIC */
+
+	return ret;
+}
+
+uint
+ai_num_slaveports(const si_t *sih, uint coreidx)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+	uint32 cib;
+
+	cib = cores_info->cib[coreidx];
+	return ((cib & CIB_NSP_MASK) >> CIB_NSP_SHIFT);
+}
+
+#ifdef UART_TRAP_DBG
+void
+ai_dump_APB_Bridge_registers(const si_t *sih)
+{
+	aidmp_t *ai;
+	const si_info_t *sii = SI_INFO(sih);
+
+	ai = (aidmp_t *)sii->br_wrapba[0];
+	printf("APB Bridge 0\n");
+	printf("lo 0x%08x, hi 0x%08x, id 0x%08x, flags 0x%08x",
+		R_REG(sii->osh, &ai->errlogaddrlo),
+		R_REG(sii->osh, &ai->errlogaddrhi),
+		R_REG(sii->osh, &ai->errlogid),
+		R_REG(sii->osh, &ai->errlogflags));
+	printf("\n status 0x%08x\n", R_REG(sii->osh, &ai->errlogstatus));
+}
+#endif /* UART_TRAP_DBG */
+
+void
+ai_force_clocks(const si_t *sih, uint clock_state)
+{
+	const si_info_t *sii = SI_INFO(sih);
+	aidmp_t *ai, *ai_sec = NULL;
+	volatile uint32 dummy;
+	uint32 ioctrl;
+	const si_cores_info_t *cores_info = (const si_cores_info_t *)sii->cores_info;
+
+	ASSERT(GOODREGS(sii->curwrap));
+	ai = sii->curwrap;
+	if (cores_info->wrapba2[sii->curidx])
+		ai_sec = REG_MAP(cores_info->wrapba2[sii->curidx], SI_CORE_SIZE);
+
+	/* ensure there are no pending backplane operations */
+	SPINWAIT((R_REG(sii->osh, &ai->resetstatus) != 0), 300);
+
+	if (clock_state == FORCE_CLK_ON) {
+		ioctrl = R_REG(sii->osh, &ai->ioctrl);
+		W_REG(sii->osh, &ai->ioctrl, (ioctrl | SICF_FGC));
+		dummy = R_REG(sii->osh, &ai->ioctrl);
+		BCM_REFERENCE(dummy);
+		if (ai_sec) {
+			ioctrl = R_REG(sii->osh, &ai_sec->ioctrl);
+			W_REG(sii->osh, &ai_sec->ioctrl, (ioctrl | SICF_FGC));
+			dummy = R_REG(sii->osh, &ai_sec->ioctrl);
+			BCM_REFERENCE(dummy);
+		}
+	} else {
+		ioctrl = R_REG(sii->osh, &ai->ioctrl);
+		W_REG(sii->osh, &ai->ioctrl, (ioctrl & (~SICF_FGC)));
+		dummy = R_REG(sii->osh, &ai->ioctrl);
+		BCM_REFERENCE(dummy);
+		if (ai_sec) {
+			ioctrl = R_REG(sii->osh, &ai_sec->ioctrl);
+			W_REG(sii->osh, &ai_sec->ioctrl, (ioctrl & (~SICF_FGC)));
+			dummy = R_REG(sii->osh, &ai_sec->ioctrl);
+			BCM_REFERENCE(dummy);
+		}
+	}
+	/* ensure there are no pending backplane operations */
+	SPINWAIT((R_REG(sii->osh, &ai->resetstatus) != 0), 300);
+}
+
+#ifdef DONGLEBUILD
+/*
+ * this is not declared as static const, although that is the right thing to do
+ * reason being if declared as static const, compile/link process would that in
+ * read only section...
+ * currently this code/array is used to identify the registers which are dumped
+ * during trap processing
+ * and usually for the trap buffer, .rodata buffer is reused,  so for now just static
+*/
+static uint32 BCMPOST_TRAP_RODATA(wrapper_offsets_to_dump)[] = {
+	OFFSETOF(aidmp_t, ioctrlset),
+	OFFSETOF(aidmp_t, ioctrlclear),
+	OFFSETOF(aidmp_t, ioctrl),
+	OFFSETOF(aidmp_t, iostatus),
+	OFFSETOF(aidmp_t, ioctrlwidth),
+	OFFSETOF(aidmp_t, iostatuswidth),
+	OFFSETOF(aidmp_t, resetctrl),
+	OFFSETOF(aidmp_t, resetstatus),
+	OFFSETOF(aidmp_t, resetreadid),
+	OFFSETOF(aidmp_t, resetwriteid),
+	OFFSETOF(aidmp_t, errlogctrl),
+	OFFSETOF(aidmp_t, errlogdone),
+	OFFSETOF(aidmp_t, errlogstatus),
+	OFFSETOF(aidmp_t, errlogaddrlo),
+	OFFSETOF(aidmp_t, errlogaddrhi),
+	OFFSETOF(aidmp_t, errlogid),
+	OFFSETOF(aidmp_t, errloguser),
+	OFFSETOF(aidmp_t, errlogflags),
+	OFFSETOF(aidmp_t, intstatus),
+	OFFSETOF(aidmp_t, config),
+	OFFSETOF(aidmp_t, itipoobaout),
+	OFFSETOF(aidmp_t, itipoobbout),
+	OFFSETOF(aidmp_t, itipoobcout),
+	OFFSETOF(aidmp_t, itipoobdout)};
+
+#ifdef ETD
+
+/* This is used for dumping wrapper registers for etd when axierror happens.
+ * This should match with the structure hnd_ext_trap_bp_err_t
+ */
+static uint32 BCMPOST_TRAP_RODATA(etd_wrapper_offsets_axierr)[] = {
+	OFFSETOF(aidmp_t, ioctrl),
+	OFFSETOF(aidmp_t, iostatus),
+	OFFSETOF(aidmp_t, resetctrl),
+	OFFSETOF(aidmp_t, resetstatus),
+	OFFSETOF(aidmp_t, resetreadid),
+	OFFSETOF(aidmp_t, resetwriteid),
+	OFFSETOF(aidmp_t, errlogctrl),
+	OFFSETOF(aidmp_t, errlogdone),
+	OFFSETOF(aidmp_t, errlogstatus),
+	OFFSETOF(aidmp_t, errlogaddrlo),
+	OFFSETOF(aidmp_t, errlogaddrhi),
+	OFFSETOF(aidmp_t, errlogid),
+	OFFSETOF(aidmp_t, errloguser),
+	OFFSETOF(aidmp_t, errlogflags),
+	OFFSETOF(aidmp_t, itipoobaout),
+	OFFSETOF(aidmp_t, itipoobbout),
+	OFFSETOF(aidmp_t, itipoobcout),
+	OFFSETOF(aidmp_t, itipoobdout)};
+#endif /* ETD */
+
+/* wrapper function to access the global array wrapper_offsets_to_dump */
+static uint32
+BCMRAMFN(ai_get_sizeof_wrapper_offsets_to_dump)(void)
+{
+	return (sizeof(wrapper_offsets_to_dump));
+}
+
+static uint32
+BCMPOSTTRAPRAMFN(ai_get_wrapper_base_addr)(uint32 **offset)
+{
+	uint32 arr_size = ARRAYSIZE(wrapper_offsets_to_dump);
+
+	*offset = &wrapper_offsets_to_dump[0];
+	return arr_size;
+}
+
+uint32
+ai_wrapper_dump_buf_size(const si_t *sih)
+{
+	uint32 buf_size = 0;
+	uint32 wrapper_count = 0;
+	const si_info_t *sii = SI_INFO(sih);
+
+	wrapper_count = sii->axi_num_wrappers;
+	if (wrapper_count == 0)
+		return 0;
+
+	/* cnt indicates how many registers, tag_id 0 will say these are address/value */
+	/* address/value pairs */
+	buf_size += 2 * (ai_get_sizeof_wrapper_offsets_to_dump() * wrapper_count);
+
+	return buf_size;
+}
+
+static uint32*
+BCMPOSTTRAPFN(ai_wrapper_dump_binary_one)(const si_info_t *sii, uint32 *p32, uint32 wrap_ba)
+{
+	uint i;
+	uint32 *addr;
+	uint32 arr_size;
+	uint32 *offset_base;
+
+	arr_size = ai_get_wrapper_base_addr(&offset_base);
+
+	for (i = 0; i < arr_size; i++) {
+		addr = (uint32 *)(wrap_ba + *(offset_base + i));
+		*p32++ = (uint32)addr;
+		*p32++ = R_REG(sii->osh, addr);
+	}
+	return p32;
+}
+
+#if defined(ETD)
+static uint32
+BCMPOSTTRAPRAMFN(ai_get_wrapper_base_addr_etd_axierr)(uint32 **offset)
+{
+	uint32 arr_size = ARRAYSIZE(etd_wrapper_offsets_axierr);
+
+	*offset = &etd_wrapper_offsets_axierr[0];
+	return arr_size;
+}
+
+uint32
+BCMPOSTTRAPFN(ai_wrapper_dump_last_timeout)(const si_t *sih, uint32 *error, uint32 *core,
+	uint32 *ba, uchar *p)
+{
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+	uint32 *p32;
+	uint32 wrap_ba = last_axi_error_wrap;
+	uint i;
+	uint32 *addr;
+
+	const si_info_t *sii = SI_INFO(sih);
+
+	if (last_axi_error != AXI_WRAP_STS_NONE)
+	{
+		if (wrap_ba)
+		{
+			p32 = (uint32 *)p;
+			uint32 arr_size;
+			uint32 *offset_base;
+
+			arr_size = ai_get_wrapper_base_addr_etd_axierr(&offset_base);
+			for (i = 0; i < arr_size; i++) {
+				addr = (uint32 *)(wrap_ba + *(offset_base + i));
+				*p32++ = R_REG(sii->osh, addr);
+			}
+		}
+		*error = last_axi_error;
+		*core = last_axi_error_core;
+		*ba = wrap_ba;
+	}
+#else
+	*error = 0;
+	*core = 0;
+	*ba = 0;
+#endif /* AXI_TIMEOUTS || AXI_TIMEOUTS_NIC */
+	return 0;
+}
+#endif /* ETD */
+
+uint32
+BCMPOSTTRAPFN(ai_wrapper_dump_binary)(const si_t *sih, uchar *p)
+{
+	uint32 *p32 = (uint32 *)p;
+	uint32 i;
+	const si_info_t *sii = SI_INFO(sih);
+
+	for (i = 0; i < sii->axi_num_wrappers; i++) {
+		p32 = ai_wrapper_dump_binary_one(sii, p32, sii->axi_wrapper[i].wrapper_addr);
+	}
+	return 0;
+}
+
+bool
+BCMPOSTTRAPFN(ai_check_enable_backplane_log)(const si_t *sih)
+{
+#if defined(AXI_TIMEOUTS) || defined(AXI_TIMEOUTS_NIC)
+	if (g_disable_backplane_logs) {
+		return FALSE;
+	}
+	else {
+		return TRUE;
+	}
+#else /*  (AXI_TIMEOUTS) || defined (AXI_TIMEOUTS_NIC) */
+	return FALSE;
+#endif /*  (AXI_TIMEOUTS) || defined (AXI_TIMEOUTS_NIC) */
+}
+#endif /* DONGLEBUILD */
diff --git a/drivers/net/wireless/ap12275/bcm_app_utils.c b/drivers/net/wireless/ap12275/bcm_app_utils.c
new file mode 100644
index 000000000000..7f8d553b9988
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcm_app_utils.c
@@ -0,0 +1,1252 @@
+/*
+ * Misc utility routines used by kernel or app-level.
+ * Contents are wifi-specific, used by any kernel or app-level
+ * software that might want wifi things as it grows.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+
+#ifdef BCMDRIVER
+#include <osl.h>
+#define strtoul(nptr, endptr, base) bcm_strtoul((nptr), (endptr), (base))
+#ifndef tolower
+#define tolower(c) (bcm_isupper((c)) ? ((c) + 'a' - 'A') : (c))
+#endif /* tolower */
+#else /* BCMDRIVER */
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#ifndef ASSERT
+#define ASSERT(exp)
+#endif
+#endif /* BCMDRIVER */
+#include <bcmwifi_channels.h>
+
+#if defined(WIN32) && (defined(BCMDLL) || defined(WLMDLL) || defined(_CONSOLE))
+#include <bcmstdlib.h>	/* For wlexe/Makefile.wlm_dll */
+#endif
+
+#include <bcmutils.h>
+#include <wlioctl.h>
+#include <wlioctl_utils.h>
+
+#ifndef BCMDRIVER
+/*	Take an array of measurments representing a single channel over time and return
+	a summary. Currently implemented as a simple average but could easily evolve
+	into more cpomplex alogrithms.
+*/
+cca_congest_channel_req_t *
+cca_per_chan_summary(cca_congest_channel_req_t *input, cca_congest_channel_req_t *avg, bool percent)
+{
+	int sec;
+	cca_congest_t totals;
+
+	totals.duration  = 0;
+	totals.congest_ibss  = 0;
+	totals.congest_obss  = 0;
+	totals.interference  = 0;
+	avg->num_secs = 0;
+
+	for (sec = 0; sec < input->num_secs; sec++) {
+		if (input->secs[sec].duration) {
+			totals.duration += input->secs[sec].duration;
+			totals.congest_ibss += input->secs[sec].congest_ibss;
+			totals.congest_obss += input->secs[sec].congest_obss;
+			totals.interference += input->secs[sec].interference;
+			avg->num_secs++;
+		}
+	}
+	avg->chanspec = input->chanspec;
+
+	if (!avg->num_secs || !totals.duration)
+		return (avg);
+
+	if (percent) {
+		avg->secs[0].duration = totals.duration / avg->num_secs;
+		avg->secs[0].congest_ibss = totals.congest_ibss * 100/totals.duration;
+		avg->secs[0].congest_obss = totals.congest_obss * 100/totals.duration;
+		avg->secs[0].interference = totals.interference * 100/totals.duration;
+	} else {
+		avg->secs[0].duration = totals.duration / avg->num_secs;
+		avg->secs[0].congest_ibss = totals.congest_ibss / avg->num_secs;
+		avg->secs[0].congest_obss = totals.congest_obss / avg->num_secs;
+		avg->secs[0].interference = totals.interference / avg->num_secs;
+	}
+
+	return (avg);
+}
+
+static void
+cca_info(uint8 *bitmap, int num_bits, int *left, int *bit_pos)
+{
+	int i;
+	for (*left = 0, i = 0; i < num_bits; i++) {
+		if (isset(bitmap, i)) {
+			(*left)++;
+			*bit_pos = i;
+		}
+	}
+}
+
+static uint8
+spec_to_chan(chanspec_t chspec)
+{
+	return wf_chspec_primary20_chan(chspec);
+}
+
+/*
+	Take an array of measumrements representing summaries of different channels.
+	Return a recomended channel.
+	Interference is evil, get rid of that first.
+	Then hunt for lowest Other bss traffic.
+	Don't forget that channels with low duration times may not have accurate readings.
+	For the moment, do not overwrite input array.
+*/
+int
+cca_analyze(cca_congest_channel_req_t *input[], int num_chans, uint flags, chanspec_t *answer)
+{
+	uint8 *bitmap = NULL;	/* 38 Max channels needs 5 bytes  = 40 */
+	int i, left, winner, ret_val = 0;
+	uint32 min_obss = 1 << 30;
+	uint bitmap_sz;
+
+	bitmap_sz = CEIL(num_chans, NBBY);
+	bitmap = (uint8 *)malloc(bitmap_sz);
+	if (bitmap == NULL) {
+		printf("unable to allocate memory\n");
+		return BCME_NOMEM;
+	}
+
+	bzero(bitmap, bitmap_sz);
+	/* Initially, all channels are up for consideration */
+	for (i = 0; i < num_chans; i++) {
+		if (input[i]->chanspec)
+			setbit(bitmap, i);
+	}
+	cca_info(bitmap, num_chans, &left, &i);
+	if (!left) {
+		ret_val = CCA_ERRNO_TOO_FEW;
+		goto f_exit;
+	}
+
+	/* Filter for 2.4 GHz Band */
+	if (flags & CCA_FLAG_2G_ONLY) {
+		for (i = 0; i < num_chans; i++) {
+			if (!CHSPEC_IS2G(input[i]->chanspec))
+				clrbit(bitmap, i);
+		}
+	}
+	cca_info(bitmap, num_chans, &left, &i);
+	if (!left) {
+		ret_val = CCA_ERRNO_BAND;
+		goto f_exit;
+	}
+
+	/* Filter for 5 GHz Band */
+	if (flags & CCA_FLAG_5G_ONLY) {
+		for (i = 0; i < num_chans; i++) {
+			if (!CHSPEC_IS5G(input[i]->chanspec))
+				clrbit(bitmap, i);
+		}
+	}
+	cca_info(bitmap, num_chans, &left, &i);
+	if (!left) {
+		ret_val = CCA_ERRNO_BAND;
+		goto f_exit;
+	}
+
+	/* Filter for Duration */
+	if (!(flags & CCA_FLAG_IGNORE_DURATION)) {
+		for (i = 0; i < num_chans; i++) {
+			if (input[i]->secs[0].duration < CCA_THRESH_MILLI)
+				clrbit(bitmap, i);
+		}
+	}
+	cca_info(bitmap, num_chans, &left, &i);
+	if (!left) {
+		ret_val = CCA_ERRNO_DURATION;
+		goto f_exit;
+	}
+
+	/* Filter for 1 6 11 on 2.4 Band */
+	if (flags &  CCA_FLAGS_PREFER_1_6_11) {
+		int tmp_channel = spec_to_chan(input[i]->chanspec);
+		int is2g = CHSPEC_IS2G(input[i]->chanspec);
+		for (i = 0; i < num_chans; i++) {
+			if (is2g && tmp_channel != 1 && tmp_channel != 6 && tmp_channel != 11)
+				clrbit(bitmap, i);
+		}
+	}
+	cca_info(bitmap, num_chans, &left, &i);
+	if (!left) {
+		ret_val = CCA_ERRNO_PREF_CHAN;
+		goto f_exit;
+	}
+
+	/* Toss high interference interference */
+	if (!(flags & CCA_FLAG_IGNORE_INTERFER)) {
+		for (i = 0; i < num_chans; i++) {
+			if (input[i]->secs[0].interference > CCA_THRESH_INTERFERE)
+				clrbit(bitmap, i);
+		}
+		cca_info(bitmap, num_chans, &left, &i);
+		if (!left) {
+			ret_val = CCA_ERRNO_INTERFER;
+			goto f_exit;
+		}
+	}
+
+	/* Now find lowest obss */
+	winner = 0;
+	for (i = 0; i < num_chans; i++) {
+		if (isset(bitmap, i) && input[i]->secs[0].congest_obss < min_obss) {
+			winner = i;
+			min_obss = input[i]->secs[0].congest_obss;
+		}
+	}
+	*answer = input[winner]->chanspec;
+	f_exit:
+	free(bitmap);	/* free the allocated memory for bitmap */
+	return ret_val;
+}
+#endif /* !BCMDRIVER */
+
+/* offset of cntmember by sizeof(uint32) from the first cnt variable, txframe. */
+#define IDX_IN_WL_CNT_VER_6_T(cntmember)		\
+	((OFFSETOF(wl_cnt_ver_6_t, cntmember) - OFFSETOF(wl_cnt_ver_6_t, txframe)) / sizeof(uint32))
+
+#define IDX_IN_WL_CNT_VER_7_T(cntmember)		\
+	((OFFSETOF(wl_cnt_ver_7_t, cntmember) - OFFSETOF(wl_cnt_ver_7_t, txframe)) / sizeof(uint32))
+
+#define IDX_IN_WL_CNT_VER_11_T(cntmember)		\
+	((OFFSETOF(wl_cnt_ver_11_t, cntmember) - OFFSETOF(wl_cnt_ver_11_t, txframe))	\
+	/ sizeof(uint32))
+
+/* Exclude version and length fields */
+#define NUM_OF_CNT_IN_WL_CNT_VER_6_T	\
+	((sizeof(wl_cnt_ver_6_t) - 2 * sizeof(uint16)) / sizeof(uint32))
+/* Exclude macstat cnt variables. wl_cnt_ver_6_t only has 62 macstat cnt variables. */
+#define NUM_OF_WLCCNT_IN_WL_CNT_VER_6_T			\
+	(NUM_OF_CNT_IN_WL_CNT_VER_6_T - (WL_CNT_MCST_VAR_NUM - 2))
+
+/* Exclude version and length fields */
+#define NUM_OF_CNT_IN_WL_CNT_VER_7_T	\
+	((sizeof(wl_cnt_ver_7_t) - 2 * sizeof(uint16)) / sizeof(uint32))
+
+/* Exclude version and length fields */
+#define NUM_OF_CNT_IN_WL_CNT_VER_11_T	\
+	((sizeof(wl_cnt_ver_11_t) - 2 * sizeof(uint16)) / sizeof(uint32))
+/* Exclude 64 macstat cnt variables. */
+#define NUM_OF_WLCCNT_IN_WL_CNT_VER_11_T	\
+	((sizeof(wl_cnt_wlc_t)) / sizeof(uint32))
+
+/* Index conversion table from wl_cnt_ver_6_t to wl_cnt_wlc_t */
+static const uint8 wlcntver6t_to_wlcntwlct[NUM_OF_WLCCNT_IN_WL_CNT_VER_6_T] = {
+	IDX_IN_WL_CNT_VER_6_T(txframe),
+	IDX_IN_WL_CNT_VER_6_T(txbyte),
+	IDX_IN_WL_CNT_VER_6_T(txretrans),
+	IDX_IN_WL_CNT_VER_6_T(txerror),
+	IDX_IN_WL_CNT_VER_6_T(txctl),
+	IDX_IN_WL_CNT_VER_6_T(txprshort),
+	IDX_IN_WL_CNT_VER_6_T(txserr),
+	IDX_IN_WL_CNT_VER_6_T(txnobuf),
+	IDX_IN_WL_CNT_VER_6_T(txnoassoc),
+	IDX_IN_WL_CNT_VER_6_T(txrunt),
+	IDX_IN_WL_CNT_VER_6_T(txchit),
+	IDX_IN_WL_CNT_VER_6_T(txcmiss),
+	IDX_IN_WL_CNT_VER_6_T(txuflo),
+	IDX_IN_WL_CNT_VER_6_T(txphyerr),
+	IDX_IN_WL_CNT_VER_6_T(txphycrs),
+	IDX_IN_WL_CNT_VER_6_T(rxframe),
+	IDX_IN_WL_CNT_VER_6_T(rxbyte),
+	IDX_IN_WL_CNT_VER_6_T(rxerror),
+	IDX_IN_WL_CNT_VER_6_T(rxctl),
+	IDX_IN_WL_CNT_VER_6_T(rxnobuf),
+	IDX_IN_WL_CNT_VER_6_T(rxnondata),
+	IDX_IN_WL_CNT_VER_6_T(rxbadds),
+	IDX_IN_WL_CNT_VER_6_T(rxbadcm),
+	IDX_IN_WL_CNT_VER_6_T(rxfragerr),
+	IDX_IN_WL_CNT_VER_6_T(rxrunt),
+	IDX_IN_WL_CNT_VER_6_T(rxgiant),
+	IDX_IN_WL_CNT_VER_6_T(rxnoscb),
+	IDX_IN_WL_CNT_VER_6_T(rxbadproto),
+	IDX_IN_WL_CNT_VER_6_T(rxbadsrcmac),
+	IDX_IN_WL_CNT_VER_6_T(rxbadda),
+	IDX_IN_WL_CNT_VER_6_T(rxfilter),
+	IDX_IN_WL_CNT_VER_6_T(rxoflo),
+	IDX_IN_WL_CNT_VER_6_T(rxuflo),
+	IDX_IN_WL_CNT_VER_6_T(rxuflo) + 1,
+	IDX_IN_WL_CNT_VER_6_T(rxuflo) + 2,
+	IDX_IN_WL_CNT_VER_6_T(rxuflo) + 3,
+	IDX_IN_WL_CNT_VER_6_T(rxuflo) + 4,
+	IDX_IN_WL_CNT_VER_6_T(rxuflo) + 5,
+	IDX_IN_WL_CNT_VER_6_T(d11cnt_txrts_off),
+	IDX_IN_WL_CNT_VER_6_T(d11cnt_rxcrc_off),
+	IDX_IN_WL_CNT_VER_6_T(d11cnt_txnocts_off),
+	IDX_IN_WL_CNT_VER_6_T(dmade),
+	IDX_IN_WL_CNT_VER_6_T(dmada),
+	IDX_IN_WL_CNT_VER_6_T(dmape),
+	IDX_IN_WL_CNT_VER_6_T(reset),
+	IDX_IN_WL_CNT_VER_6_T(tbtt),
+	IDX_IN_WL_CNT_VER_6_T(txdmawar),
+	IDX_IN_WL_CNT_VER_6_T(pkt_callback_reg_fail),
+	IDX_IN_WL_CNT_VER_6_T(txfrag),
+	IDX_IN_WL_CNT_VER_6_T(txmulti),
+	IDX_IN_WL_CNT_VER_6_T(txfail),
+	IDX_IN_WL_CNT_VER_6_T(txretry),
+	IDX_IN_WL_CNT_VER_6_T(txretrie),
+	IDX_IN_WL_CNT_VER_6_T(rxdup),
+	IDX_IN_WL_CNT_VER_6_T(txrts),
+	IDX_IN_WL_CNT_VER_6_T(txnocts),
+	IDX_IN_WL_CNT_VER_6_T(txnoack),
+	IDX_IN_WL_CNT_VER_6_T(rxfrag),
+	IDX_IN_WL_CNT_VER_6_T(rxmulti),
+	IDX_IN_WL_CNT_VER_6_T(rxcrc),
+	IDX_IN_WL_CNT_VER_6_T(txfrmsnt),
+	IDX_IN_WL_CNT_VER_6_T(rxundec),
+	IDX_IN_WL_CNT_VER_6_T(tkipmicfaill),
+	IDX_IN_WL_CNT_VER_6_T(tkipcntrmsr),
+	IDX_IN_WL_CNT_VER_6_T(tkipreplay),
+	IDX_IN_WL_CNT_VER_6_T(ccmpfmterr),
+	IDX_IN_WL_CNT_VER_6_T(ccmpreplay),
+	IDX_IN_WL_CNT_VER_6_T(ccmpundec),
+	IDX_IN_WL_CNT_VER_6_T(fourwayfail),
+	IDX_IN_WL_CNT_VER_6_T(wepundec),
+	IDX_IN_WL_CNT_VER_6_T(wepicverr),
+	IDX_IN_WL_CNT_VER_6_T(decsuccess),
+	IDX_IN_WL_CNT_VER_6_T(tkipicverr),
+	IDX_IN_WL_CNT_VER_6_T(wepexcluded),
+	IDX_IN_WL_CNT_VER_6_T(txchanrej),
+	IDX_IN_WL_CNT_VER_6_T(psmwds),
+	IDX_IN_WL_CNT_VER_6_T(phywatchdog),
+	IDX_IN_WL_CNT_VER_6_T(prq_entries_handled),
+	IDX_IN_WL_CNT_VER_6_T(prq_undirected_entries),
+	IDX_IN_WL_CNT_VER_6_T(prq_bad_entries),
+	IDX_IN_WL_CNT_VER_6_T(atim_suppress_count),
+	IDX_IN_WL_CNT_VER_6_T(bcn_template_not_ready),
+	IDX_IN_WL_CNT_VER_6_T(bcn_template_not_ready_done),
+	IDX_IN_WL_CNT_VER_6_T(late_tbtt_dpc),
+	IDX_IN_WL_CNT_VER_6_T(rx1mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx2mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx5mbps5),
+	IDX_IN_WL_CNT_VER_6_T(rx6mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx9mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx11mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx12mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx18mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx24mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx36mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx48mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx54mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx108mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx162mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx216mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx270mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx324mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx378mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx432mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx486mbps),
+	IDX_IN_WL_CNT_VER_6_T(rx540mbps),
+	IDX_IN_WL_CNT_VER_6_T(rfdisable),
+	IDX_IN_WL_CNT_VER_6_T(txexptime),
+	IDX_IN_WL_CNT_VER_6_T(txmpdu_sgi),
+	IDX_IN_WL_CNT_VER_6_T(rxmpdu_sgi),
+	IDX_IN_WL_CNT_VER_6_T(txmpdu_stbc),
+	IDX_IN_WL_CNT_VER_6_T(rxmpdu_stbc),
+	IDX_IN_WL_CNT_VER_6_T(rxundec_mcst),
+	IDX_IN_WL_CNT_VER_6_T(tkipmicfaill_mcst),
+	IDX_IN_WL_CNT_VER_6_T(tkipcntrmsr_mcst),
+	IDX_IN_WL_CNT_VER_6_T(tkipreplay_mcst),
+	IDX_IN_WL_CNT_VER_6_T(ccmpfmterr_mcst),
+	IDX_IN_WL_CNT_VER_6_T(ccmpreplay_mcst),
+	IDX_IN_WL_CNT_VER_6_T(ccmpundec_mcst),
+	IDX_IN_WL_CNT_VER_6_T(fourwayfail_mcst),
+	IDX_IN_WL_CNT_VER_6_T(wepundec_mcst),
+	IDX_IN_WL_CNT_VER_6_T(wepicverr_mcst),
+	IDX_IN_WL_CNT_VER_6_T(decsuccess_mcst),
+	IDX_IN_WL_CNT_VER_6_T(tkipicverr_mcst),
+	IDX_IN_WL_CNT_VER_6_T(wepexcluded_mcst)
+};
+
+#define INVALID_IDX ((uint8)(-1))
+
+/* Index conversion table from wl_cnt_ver_7_t to wl_cnt_wlc_t */
+static const uint8 wlcntver7t_to_wlcntwlct[] = {
+	IDX_IN_WL_CNT_VER_7_T(txframe),
+	IDX_IN_WL_CNT_VER_7_T(txbyte),
+	IDX_IN_WL_CNT_VER_7_T(txretrans),
+	IDX_IN_WL_CNT_VER_7_T(txerror),
+	IDX_IN_WL_CNT_VER_7_T(txctl),
+	IDX_IN_WL_CNT_VER_7_T(txprshort),
+	IDX_IN_WL_CNT_VER_7_T(txserr),
+	IDX_IN_WL_CNT_VER_7_T(txnobuf),
+	IDX_IN_WL_CNT_VER_7_T(txnoassoc),
+	IDX_IN_WL_CNT_VER_7_T(txrunt),
+	IDX_IN_WL_CNT_VER_7_T(txchit),
+	IDX_IN_WL_CNT_VER_7_T(txcmiss),
+	IDX_IN_WL_CNT_VER_7_T(txuflo),
+	IDX_IN_WL_CNT_VER_7_T(txphyerr),
+	IDX_IN_WL_CNT_VER_7_T(txphycrs),
+	IDX_IN_WL_CNT_VER_7_T(rxframe),
+	IDX_IN_WL_CNT_VER_7_T(rxbyte),
+	IDX_IN_WL_CNT_VER_7_T(rxerror),
+	IDX_IN_WL_CNT_VER_7_T(rxctl),
+	IDX_IN_WL_CNT_VER_7_T(rxnobuf),
+	IDX_IN_WL_CNT_VER_7_T(rxnondata),
+	IDX_IN_WL_CNT_VER_7_T(rxbadds),
+	IDX_IN_WL_CNT_VER_7_T(rxbadcm),
+	IDX_IN_WL_CNT_VER_7_T(rxfragerr),
+	IDX_IN_WL_CNT_VER_7_T(rxrunt),
+	IDX_IN_WL_CNT_VER_7_T(rxgiant),
+	IDX_IN_WL_CNT_VER_7_T(rxnoscb),
+	IDX_IN_WL_CNT_VER_7_T(rxbadproto),
+	IDX_IN_WL_CNT_VER_7_T(rxbadsrcmac),
+	IDX_IN_WL_CNT_VER_7_T(rxbadda),
+	IDX_IN_WL_CNT_VER_7_T(rxfilter),
+	IDX_IN_WL_CNT_VER_7_T(rxoflo),
+	IDX_IN_WL_CNT_VER_7_T(rxuflo),
+	IDX_IN_WL_CNT_VER_7_T(rxuflo) + 1,
+	IDX_IN_WL_CNT_VER_7_T(rxuflo) + 2,
+	IDX_IN_WL_CNT_VER_7_T(rxuflo) + 3,
+	IDX_IN_WL_CNT_VER_7_T(rxuflo) + 4,
+	IDX_IN_WL_CNT_VER_7_T(rxuflo) + 5,
+	IDX_IN_WL_CNT_VER_7_T(d11cnt_txrts_off),
+	IDX_IN_WL_CNT_VER_7_T(d11cnt_rxcrc_off),
+	IDX_IN_WL_CNT_VER_7_T(d11cnt_txnocts_off),
+	IDX_IN_WL_CNT_VER_7_T(dmade),
+	IDX_IN_WL_CNT_VER_7_T(dmada),
+	IDX_IN_WL_CNT_VER_7_T(dmape),
+	IDX_IN_WL_CNT_VER_7_T(reset),
+	IDX_IN_WL_CNT_VER_7_T(tbtt),
+	IDX_IN_WL_CNT_VER_7_T(txdmawar),
+	IDX_IN_WL_CNT_VER_7_T(pkt_callback_reg_fail),
+	IDX_IN_WL_CNT_VER_7_T(txfrag),
+	IDX_IN_WL_CNT_VER_7_T(txmulti),
+	IDX_IN_WL_CNT_VER_7_T(txfail),
+	IDX_IN_WL_CNT_VER_7_T(txretry),
+	IDX_IN_WL_CNT_VER_7_T(txretrie),
+	IDX_IN_WL_CNT_VER_7_T(rxdup),
+	IDX_IN_WL_CNT_VER_7_T(txrts),
+	IDX_IN_WL_CNT_VER_7_T(txnocts),
+	IDX_IN_WL_CNT_VER_7_T(txnoack),
+	IDX_IN_WL_CNT_VER_7_T(rxfrag),
+	IDX_IN_WL_CNT_VER_7_T(rxmulti),
+	IDX_IN_WL_CNT_VER_7_T(rxcrc),
+	IDX_IN_WL_CNT_VER_7_T(txfrmsnt),
+	IDX_IN_WL_CNT_VER_7_T(rxundec),
+	IDX_IN_WL_CNT_VER_7_T(tkipmicfaill),
+	IDX_IN_WL_CNT_VER_7_T(tkipcntrmsr),
+	IDX_IN_WL_CNT_VER_7_T(tkipreplay),
+	IDX_IN_WL_CNT_VER_7_T(ccmpfmterr),
+	IDX_IN_WL_CNT_VER_7_T(ccmpreplay),
+	IDX_IN_WL_CNT_VER_7_T(ccmpundec),
+	IDX_IN_WL_CNT_VER_7_T(fourwayfail),
+	IDX_IN_WL_CNT_VER_7_T(wepundec),
+	IDX_IN_WL_CNT_VER_7_T(wepicverr),
+	IDX_IN_WL_CNT_VER_7_T(decsuccess),
+	IDX_IN_WL_CNT_VER_7_T(tkipicverr),
+	IDX_IN_WL_CNT_VER_7_T(wepexcluded),
+	IDX_IN_WL_CNT_VER_7_T(txchanrej),
+	IDX_IN_WL_CNT_VER_7_T(psmwds),
+	IDX_IN_WL_CNT_VER_7_T(phywatchdog),
+	IDX_IN_WL_CNT_VER_7_T(prq_entries_handled),
+	IDX_IN_WL_CNT_VER_7_T(prq_undirected_entries),
+	IDX_IN_WL_CNT_VER_7_T(prq_bad_entries),
+	IDX_IN_WL_CNT_VER_7_T(atim_suppress_count),
+	IDX_IN_WL_CNT_VER_7_T(bcn_template_not_ready),
+	IDX_IN_WL_CNT_VER_7_T(bcn_template_not_ready_done),
+	IDX_IN_WL_CNT_VER_7_T(late_tbtt_dpc),
+	IDX_IN_WL_CNT_VER_7_T(rx1mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx2mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx5mbps5),
+	IDX_IN_WL_CNT_VER_7_T(rx6mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx9mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx11mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx12mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx18mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx24mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx36mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx48mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx54mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx108mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx162mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx216mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx270mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx324mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx378mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx432mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx486mbps),
+	IDX_IN_WL_CNT_VER_7_T(rx540mbps),
+	IDX_IN_WL_CNT_VER_7_T(rfdisable),
+	IDX_IN_WL_CNT_VER_7_T(txexptime),
+	IDX_IN_WL_CNT_VER_7_T(txmpdu_sgi),
+	IDX_IN_WL_CNT_VER_7_T(rxmpdu_sgi),
+	IDX_IN_WL_CNT_VER_7_T(txmpdu_stbc),
+	IDX_IN_WL_CNT_VER_7_T(rxmpdu_stbc),
+	IDX_IN_WL_CNT_VER_7_T(rxundec_mcst),
+	IDX_IN_WL_CNT_VER_7_T(tkipmicfaill_mcst),
+	IDX_IN_WL_CNT_VER_7_T(tkipcntrmsr_mcst),
+	IDX_IN_WL_CNT_VER_7_T(tkipreplay_mcst),
+	IDX_IN_WL_CNT_VER_7_T(ccmpfmterr_mcst),
+	IDX_IN_WL_CNT_VER_7_T(ccmpreplay_mcst),
+	IDX_IN_WL_CNT_VER_7_T(ccmpundec_mcst),
+	IDX_IN_WL_CNT_VER_7_T(fourwayfail_mcst),
+	IDX_IN_WL_CNT_VER_7_T(wepundec_mcst),
+	IDX_IN_WL_CNT_VER_7_T(wepicverr_mcst),
+	IDX_IN_WL_CNT_VER_7_T(decsuccess_mcst),
+	IDX_IN_WL_CNT_VER_7_T(tkipicverr_mcst),
+	IDX_IN_WL_CNT_VER_7_T(wepexcluded_mcst),
+	IDX_IN_WL_CNT_VER_7_T(dma_hang),
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	IDX_IN_WL_CNT_VER_7_T(rxrtry)
+};
+
+/* Max wl_cnt_wlc_t fields including rxrtry */
+#define NUM_OF_WLCCNT_IN_WL_CNT_VER_7_T				\
+	(sizeof(wlcntver7t_to_wlcntwlct) / sizeof(uint8))
+
+/* Index conversion table from wl_cnt_ver_11_t to wl_cnt_wlc_t */
+static const uint8 wlcntver11t_to_wlcntwlct[NUM_OF_WLCCNT_IN_WL_CNT_VER_11_T] = {
+	IDX_IN_WL_CNT_VER_11_T(txframe),
+	IDX_IN_WL_CNT_VER_11_T(txbyte),
+	IDX_IN_WL_CNT_VER_11_T(txretrans),
+	IDX_IN_WL_CNT_VER_11_T(txerror),
+	IDX_IN_WL_CNT_VER_11_T(txctl),
+	IDX_IN_WL_CNT_VER_11_T(txprshort),
+	IDX_IN_WL_CNT_VER_11_T(txserr),
+	IDX_IN_WL_CNT_VER_11_T(txnobuf),
+	IDX_IN_WL_CNT_VER_11_T(txnoassoc),
+	IDX_IN_WL_CNT_VER_11_T(txrunt),
+	IDX_IN_WL_CNT_VER_11_T(txchit),
+	IDX_IN_WL_CNT_VER_11_T(txcmiss),
+	IDX_IN_WL_CNT_VER_11_T(txuflo),
+	IDX_IN_WL_CNT_VER_11_T(txphyerr),
+	IDX_IN_WL_CNT_VER_11_T(txphycrs),
+	IDX_IN_WL_CNT_VER_11_T(rxframe),
+	IDX_IN_WL_CNT_VER_11_T(rxbyte),
+	IDX_IN_WL_CNT_VER_11_T(rxerror),
+	IDX_IN_WL_CNT_VER_11_T(rxctl),
+	IDX_IN_WL_CNT_VER_11_T(rxnobuf),
+	IDX_IN_WL_CNT_VER_11_T(rxnondata),
+	IDX_IN_WL_CNT_VER_11_T(rxbadds),
+	IDX_IN_WL_CNT_VER_11_T(rxbadcm),
+	IDX_IN_WL_CNT_VER_11_T(rxfragerr),
+	IDX_IN_WL_CNT_VER_11_T(rxrunt),
+	IDX_IN_WL_CNT_VER_11_T(rxgiant),
+	IDX_IN_WL_CNT_VER_11_T(rxnoscb),
+	IDX_IN_WL_CNT_VER_11_T(rxbadproto),
+	IDX_IN_WL_CNT_VER_11_T(rxbadsrcmac),
+	IDX_IN_WL_CNT_VER_11_T(rxbadda),
+	IDX_IN_WL_CNT_VER_11_T(rxfilter),
+	IDX_IN_WL_CNT_VER_11_T(rxoflo),
+	IDX_IN_WL_CNT_VER_11_T(rxuflo),
+	IDX_IN_WL_CNT_VER_11_T(rxuflo) + 1,
+	IDX_IN_WL_CNT_VER_11_T(rxuflo) + 2,
+	IDX_IN_WL_CNT_VER_11_T(rxuflo) + 3,
+	IDX_IN_WL_CNT_VER_11_T(rxuflo) + 4,
+	IDX_IN_WL_CNT_VER_11_T(rxuflo) + 5,
+	IDX_IN_WL_CNT_VER_11_T(d11cnt_txrts_off),
+	IDX_IN_WL_CNT_VER_11_T(d11cnt_rxcrc_off),
+	IDX_IN_WL_CNT_VER_11_T(d11cnt_txnocts_off),
+	IDX_IN_WL_CNT_VER_11_T(dmade),
+	IDX_IN_WL_CNT_VER_11_T(dmada),
+	IDX_IN_WL_CNT_VER_11_T(dmape),
+	IDX_IN_WL_CNT_VER_11_T(reset),
+	IDX_IN_WL_CNT_VER_11_T(tbtt),
+	IDX_IN_WL_CNT_VER_11_T(txdmawar),
+	IDX_IN_WL_CNT_VER_11_T(pkt_callback_reg_fail),
+	IDX_IN_WL_CNT_VER_11_T(txfrag),
+	IDX_IN_WL_CNT_VER_11_T(txmulti),
+	IDX_IN_WL_CNT_VER_11_T(txfail),
+	IDX_IN_WL_CNT_VER_11_T(txretry),
+	IDX_IN_WL_CNT_VER_11_T(txretrie),
+	IDX_IN_WL_CNT_VER_11_T(rxdup),
+	IDX_IN_WL_CNT_VER_11_T(txrts),
+	IDX_IN_WL_CNT_VER_11_T(txnocts),
+	IDX_IN_WL_CNT_VER_11_T(txnoack),
+	IDX_IN_WL_CNT_VER_11_T(rxfrag),
+	IDX_IN_WL_CNT_VER_11_T(rxmulti),
+	IDX_IN_WL_CNT_VER_11_T(rxcrc),
+	IDX_IN_WL_CNT_VER_11_T(txfrmsnt),
+	IDX_IN_WL_CNT_VER_11_T(rxundec),
+	IDX_IN_WL_CNT_VER_11_T(tkipmicfaill),
+	IDX_IN_WL_CNT_VER_11_T(tkipcntrmsr),
+	IDX_IN_WL_CNT_VER_11_T(tkipreplay),
+	IDX_IN_WL_CNT_VER_11_T(ccmpfmterr),
+	IDX_IN_WL_CNT_VER_11_T(ccmpreplay),
+	IDX_IN_WL_CNT_VER_11_T(ccmpundec),
+	IDX_IN_WL_CNT_VER_11_T(fourwayfail),
+	IDX_IN_WL_CNT_VER_11_T(wepundec),
+	IDX_IN_WL_CNT_VER_11_T(wepicverr),
+	IDX_IN_WL_CNT_VER_11_T(decsuccess),
+	IDX_IN_WL_CNT_VER_11_T(tkipicverr),
+	IDX_IN_WL_CNT_VER_11_T(wepexcluded),
+	IDX_IN_WL_CNT_VER_11_T(txchanrej),
+	IDX_IN_WL_CNT_VER_11_T(psmwds),
+	IDX_IN_WL_CNT_VER_11_T(phywatchdog),
+	IDX_IN_WL_CNT_VER_11_T(prq_entries_handled),
+	IDX_IN_WL_CNT_VER_11_T(prq_undirected_entries),
+	IDX_IN_WL_CNT_VER_11_T(prq_bad_entries),
+	IDX_IN_WL_CNT_VER_11_T(atim_suppress_count),
+	IDX_IN_WL_CNT_VER_11_T(bcn_template_not_ready),
+	IDX_IN_WL_CNT_VER_11_T(bcn_template_not_ready_done),
+	IDX_IN_WL_CNT_VER_11_T(late_tbtt_dpc),
+	IDX_IN_WL_CNT_VER_11_T(rx1mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx2mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx5mbps5),
+	IDX_IN_WL_CNT_VER_11_T(rx6mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx9mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx11mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx12mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx18mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx24mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx36mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx48mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx54mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx108mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx162mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx216mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx270mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx324mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx378mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx432mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx486mbps),
+	IDX_IN_WL_CNT_VER_11_T(rx540mbps),
+	IDX_IN_WL_CNT_VER_11_T(rfdisable),
+	IDX_IN_WL_CNT_VER_11_T(txexptime),
+	IDX_IN_WL_CNT_VER_11_T(txmpdu_sgi),
+	IDX_IN_WL_CNT_VER_11_T(rxmpdu_sgi),
+	IDX_IN_WL_CNT_VER_11_T(txmpdu_stbc),
+	IDX_IN_WL_CNT_VER_11_T(rxmpdu_stbc),
+	IDX_IN_WL_CNT_VER_11_T(rxundec_mcst),
+	IDX_IN_WL_CNT_VER_11_T(tkipmicfaill_mcst),
+	IDX_IN_WL_CNT_VER_11_T(tkipcntrmsr_mcst),
+	IDX_IN_WL_CNT_VER_11_T(tkipreplay_mcst),
+	IDX_IN_WL_CNT_VER_11_T(ccmpfmterr_mcst),
+	IDX_IN_WL_CNT_VER_11_T(ccmpreplay_mcst),
+	IDX_IN_WL_CNT_VER_11_T(ccmpundec_mcst),
+	IDX_IN_WL_CNT_VER_11_T(fourwayfail_mcst),
+	IDX_IN_WL_CNT_VER_11_T(wepundec_mcst),
+	IDX_IN_WL_CNT_VER_11_T(wepicverr_mcst),
+	IDX_IN_WL_CNT_VER_11_T(decsuccess_mcst),
+	IDX_IN_WL_CNT_VER_11_T(tkipicverr_mcst),
+	IDX_IN_WL_CNT_VER_11_T(wepexcluded_mcst),
+	IDX_IN_WL_CNT_VER_11_T(dma_hang),
+	IDX_IN_WL_CNT_VER_11_T(reinit),
+	IDX_IN_WL_CNT_VER_11_T(pstatxucast),
+	IDX_IN_WL_CNT_VER_11_T(pstatxnoassoc),
+	IDX_IN_WL_CNT_VER_11_T(pstarxucast),
+	IDX_IN_WL_CNT_VER_11_T(pstarxbcmc),
+	IDX_IN_WL_CNT_VER_11_T(pstatxbcmc),
+	IDX_IN_WL_CNT_VER_11_T(cso_passthrough),
+	IDX_IN_WL_CNT_VER_11_T(cso_normal),
+	IDX_IN_WL_CNT_VER_11_T(chained),
+	IDX_IN_WL_CNT_VER_11_T(chainedsz1),
+	IDX_IN_WL_CNT_VER_11_T(unchained),
+	IDX_IN_WL_CNT_VER_11_T(maxchainsz),
+	IDX_IN_WL_CNT_VER_11_T(currchainsz),
+	IDX_IN_WL_CNT_VER_11_T(pciereset),
+	IDX_IN_WL_CNT_VER_11_T(cfgrestore),
+	IDX_IN_WL_CNT_VER_11_T(reinitreason),
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 1,
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 2,
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 3,
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 4,
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 5,
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 6,
+	IDX_IN_WL_CNT_VER_11_T(reinitreason) + 7,
+	IDX_IN_WL_CNT_VER_11_T(rxrtry),
+	IDX_IN_WL_CNT_VER_11_T(rxmpdu_mu),
+	IDX_IN_WL_CNT_VER_11_T(txbar),
+	IDX_IN_WL_CNT_VER_11_T(rxbar),
+	IDX_IN_WL_CNT_VER_11_T(txpspoll),
+	IDX_IN_WL_CNT_VER_11_T(rxpspoll),
+	IDX_IN_WL_CNT_VER_11_T(txnull),
+	IDX_IN_WL_CNT_VER_11_T(rxnull),
+	IDX_IN_WL_CNT_VER_11_T(txqosnull),
+	IDX_IN_WL_CNT_VER_11_T(rxqosnull),
+	IDX_IN_WL_CNT_VER_11_T(txassocreq),
+	IDX_IN_WL_CNT_VER_11_T(rxassocreq),
+	IDX_IN_WL_CNT_VER_11_T(txreassocreq),
+	IDX_IN_WL_CNT_VER_11_T(rxreassocreq),
+	IDX_IN_WL_CNT_VER_11_T(txdisassoc),
+	IDX_IN_WL_CNT_VER_11_T(rxdisassoc),
+	IDX_IN_WL_CNT_VER_11_T(txassocrsp),
+	IDX_IN_WL_CNT_VER_11_T(rxassocrsp),
+	IDX_IN_WL_CNT_VER_11_T(txreassocrsp),
+	IDX_IN_WL_CNT_VER_11_T(rxreassocrsp),
+	IDX_IN_WL_CNT_VER_11_T(txauth),
+	IDX_IN_WL_CNT_VER_11_T(rxauth),
+	IDX_IN_WL_CNT_VER_11_T(txdeauth),
+	IDX_IN_WL_CNT_VER_11_T(rxdeauth),
+	IDX_IN_WL_CNT_VER_11_T(txprobereq),
+	IDX_IN_WL_CNT_VER_11_T(rxprobereq),
+	IDX_IN_WL_CNT_VER_11_T(txprobersp),
+	IDX_IN_WL_CNT_VER_11_T(rxprobersp),
+	IDX_IN_WL_CNT_VER_11_T(txaction),
+	IDX_IN_WL_CNT_VER_11_T(rxaction),
+	IDX_IN_WL_CNT_VER_11_T(ampdu_wds),
+	IDX_IN_WL_CNT_VER_11_T(txlost),
+	IDX_IN_WL_CNT_VER_11_T(txdatamcast),
+	IDX_IN_WL_CNT_VER_11_T(txdatabcast),
+	INVALID_IDX,
+	IDX_IN_WL_CNT_VER_11_T(rxback),
+	IDX_IN_WL_CNT_VER_11_T(txback),
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	INVALID_IDX,
+	IDX_IN_WL_CNT_VER_11_T(txbcast),
+	IDX_IN_WL_CNT_VER_11_T(txdropped),
+	IDX_IN_WL_CNT_VER_11_T(rxbcast),
+	IDX_IN_WL_CNT_VER_11_T(rxdropped)
+};
+
+/* Index conversion table from wl_cnt_ver_11_t to
+ * either wl_cnt_ge40mcst_v1_t or wl_cnt_lt40mcst_v1_t
+ */
+static const uint8 wlcntver11t_to_wlcntXX40mcstv1t[WL_CNT_MCST_VAR_NUM] = {
+	IDX_IN_WL_CNT_VER_11_T(txallfrm),
+	IDX_IN_WL_CNT_VER_11_T(txrtsfrm),
+	IDX_IN_WL_CNT_VER_11_T(txctsfrm),
+	IDX_IN_WL_CNT_VER_11_T(txackfrm),
+	IDX_IN_WL_CNT_VER_11_T(txdnlfrm),
+	IDX_IN_WL_CNT_VER_11_T(txbcnfrm),
+	IDX_IN_WL_CNT_VER_11_T(txfunfl),
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 1,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 2,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 3,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 4,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 5,
+	IDX_IN_WL_CNT_VER_11_T(txfbw),
+	IDX_IN_WL_CNT_VER_11_T(txmpdu),
+	IDX_IN_WL_CNT_VER_11_T(txtplunfl),
+	IDX_IN_WL_CNT_VER_11_T(txphyerror),
+	IDX_IN_WL_CNT_VER_11_T(pktengrxducast),
+	IDX_IN_WL_CNT_VER_11_T(pktengrxdmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxfrmtoolong),
+	IDX_IN_WL_CNT_VER_11_T(rxfrmtooshrt),
+	IDX_IN_WL_CNT_VER_11_T(rxinvmachdr),
+	IDX_IN_WL_CNT_VER_11_T(rxbadfcs),
+	IDX_IN_WL_CNT_VER_11_T(rxbadplcp),
+	IDX_IN_WL_CNT_VER_11_T(rxcrsglitch),
+	IDX_IN_WL_CNT_VER_11_T(rxstrt),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmucastmbss),
+	IDX_IN_WL_CNT_VER_11_T(rxmfrmucastmbss),
+	IDX_IN_WL_CNT_VER_11_T(rxcfrmucast),
+	IDX_IN_WL_CNT_VER_11_T(rxrtsucast),
+	IDX_IN_WL_CNT_VER_11_T(rxctsucast),
+	IDX_IN_WL_CNT_VER_11_T(rxackucast),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmocast),
+	IDX_IN_WL_CNT_VER_11_T(rxmfrmocast),
+	IDX_IN_WL_CNT_VER_11_T(rxcfrmocast),
+	IDX_IN_WL_CNT_VER_11_T(rxrtsocast),
+	IDX_IN_WL_CNT_VER_11_T(rxctsocast),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxmfrmmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxcfrmmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxbeaconmbss),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmucastobss),
+	IDX_IN_WL_CNT_VER_11_T(rxbeaconobss),
+	IDX_IN_WL_CNT_VER_11_T(rxrsptmout),
+	IDX_IN_WL_CNT_VER_11_T(bcntxcancl),
+	IDX_IN_WL_CNT_VER_11_T(rxnodelim),
+	IDX_IN_WL_CNT_VER_11_T(rxf0ovfl),
+	IDX_IN_WL_CNT_VER_11_T(rxf1ovfl),
+	IDX_IN_WL_CNT_VER_11_T(rxf2ovfl),
+	IDX_IN_WL_CNT_VER_11_T(txsfovfl),
+	IDX_IN_WL_CNT_VER_11_T(pmqovfl),
+	IDX_IN_WL_CNT_VER_11_T(rxcgprqfrm),
+	IDX_IN_WL_CNT_VER_11_T(rxcgprsqovfl),
+	IDX_IN_WL_CNT_VER_11_T(txcgprsfail),
+	IDX_IN_WL_CNT_VER_11_T(txcgprssuc),
+	IDX_IN_WL_CNT_VER_11_T(prs_timeout),
+	IDX_IN_WL_CNT_VER_11_T(rxnack),
+	IDX_IN_WL_CNT_VER_11_T(frmscons),
+	IDX_IN_WL_CNT_VER_11_T(txnack),
+	IDX_IN_WL_CNT_VER_11_T(rxback),
+	IDX_IN_WL_CNT_VER_11_T(txback),
+	IDX_IN_WL_CNT_VER_11_T(bphy_rxcrsglitch),
+	IDX_IN_WL_CNT_VER_11_T(rxdrop20s),
+	IDX_IN_WL_CNT_VER_11_T(rxtoolate),
+	IDX_IN_WL_CNT_VER_11_T(bphy_badplcp)
+};
+
+/* For mcst offsets that were not used. (2 Pads) */
+#define INVALID_MCST_IDX ((uint8)(-1))
+/* Index conversion table from wl_cnt_ver_11_t to wl_cnt_v_le10_mcst_t */
+static const uint8 wlcntver11t_to_wlcntvle10mcstt[WL_CNT_MCST_VAR_NUM] = {
+	IDX_IN_WL_CNT_VER_11_T(txallfrm),
+	IDX_IN_WL_CNT_VER_11_T(txrtsfrm),
+	IDX_IN_WL_CNT_VER_11_T(txctsfrm),
+	IDX_IN_WL_CNT_VER_11_T(txackfrm),
+	IDX_IN_WL_CNT_VER_11_T(txdnlfrm),
+	IDX_IN_WL_CNT_VER_11_T(txbcnfrm),
+	IDX_IN_WL_CNT_VER_11_T(txfunfl),
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 1,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 2,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 3,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 4,
+	IDX_IN_WL_CNT_VER_11_T(txfunfl) + 5,
+	IDX_IN_WL_CNT_VER_11_T(txfbw),
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_11_T(txtplunfl),
+	IDX_IN_WL_CNT_VER_11_T(txphyerror),
+	IDX_IN_WL_CNT_VER_11_T(pktengrxducast),
+	IDX_IN_WL_CNT_VER_11_T(pktengrxdmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxfrmtoolong),
+	IDX_IN_WL_CNT_VER_11_T(rxfrmtooshrt),
+	IDX_IN_WL_CNT_VER_11_T(rxinvmachdr),
+	IDX_IN_WL_CNT_VER_11_T(rxbadfcs),
+	IDX_IN_WL_CNT_VER_11_T(rxbadplcp),
+	IDX_IN_WL_CNT_VER_11_T(rxcrsglitch),
+	IDX_IN_WL_CNT_VER_11_T(rxstrt),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmucastmbss),
+	IDX_IN_WL_CNT_VER_11_T(rxmfrmucastmbss),
+	IDX_IN_WL_CNT_VER_11_T(rxcfrmucast),
+	IDX_IN_WL_CNT_VER_11_T(rxrtsucast),
+	IDX_IN_WL_CNT_VER_11_T(rxctsucast),
+	IDX_IN_WL_CNT_VER_11_T(rxackucast),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmocast),
+	IDX_IN_WL_CNT_VER_11_T(rxmfrmocast),
+	IDX_IN_WL_CNT_VER_11_T(rxcfrmocast),
+	IDX_IN_WL_CNT_VER_11_T(rxrtsocast),
+	IDX_IN_WL_CNT_VER_11_T(rxctsocast),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxmfrmmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxcfrmmcast),
+	IDX_IN_WL_CNT_VER_11_T(rxbeaconmbss),
+	IDX_IN_WL_CNT_VER_11_T(rxdfrmucastobss),
+	IDX_IN_WL_CNT_VER_11_T(rxbeaconobss),
+	IDX_IN_WL_CNT_VER_11_T(rxrsptmout),
+	IDX_IN_WL_CNT_VER_11_T(bcntxcancl),
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_11_T(rxf0ovfl),
+	IDX_IN_WL_CNT_VER_11_T(rxf1ovfl),
+	IDX_IN_WL_CNT_VER_11_T(rxf2ovfl),
+	IDX_IN_WL_CNT_VER_11_T(txsfovfl),
+	IDX_IN_WL_CNT_VER_11_T(pmqovfl),
+	IDX_IN_WL_CNT_VER_11_T(rxcgprqfrm),
+	IDX_IN_WL_CNT_VER_11_T(rxcgprsqovfl),
+	IDX_IN_WL_CNT_VER_11_T(txcgprsfail),
+	IDX_IN_WL_CNT_VER_11_T(txcgprssuc),
+	IDX_IN_WL_CNT_VER_11_T(prs_timeout),
+	IDX_IN_WL_CNT_VER_11_T(rxnack),
+	IDX_IN_WL_CNT_VER_11_T(frmscons),
+	IDX_IN_WL_CNT_VER_11_T(txnack),
+	IDX_IN_WL_CNT_VER_11_T(rxback),
+	IDX_IN_WL_CNT_VER_11_T(txback),
+	IDX_IN_WL_CNT_VER_11_T(bphy_rxcrsglitch),
+	IDX_IN_WL_CNT_VER_11_T(rxdrop20s),
+	IDX_IN_WL_CNT_VER_11_T(rxtoolate),
+	IDX_IN_WL_CNT_VER_11_T(bphy_badplcp)
+};
+
+/* Index conversion table from wl_cnt_ver_6_t to wl_cnt_v_le10_mcst_t */
+static const uint8 wlcntver6t_to_wlcntvle10mcstt[WL_CNT_MCST_VAR_NUM] = {
+	IDX_IN_WL_CNT_VER_6_T(txallfrm),
+	IDX_IN_WL_CNT_VER_6_T(txrtsfrm),
+	IDX_IN_WL_CNT_VER_6_T(txctsfrm),
+	IDX_IN_WL_CNT_VER_6_T(txackfrm),
+	IDX_IN_WL_CNT_VER_6_T(txdnlfrm),
+	IDX_IN_WL_CNT_VER_6_T(txbcnfrm),
+	IDX_IN_WL_CNT_VER_6_T(txfunfl),
+	IDX_IN_WL_CNT_VER_6_T(txfunfl) + 1,
+	IDX_IN_WL_CNT_VER_6_T(txfunfl) + 2,
+	IDX_IN_WL_CNT_VER_6_T(txfunfl) + 3,
+	IDX_IN_WL_CNT_VER_6_T(txfunfl) + 4,
+	IDX_IN_WL_CNT_VER_6_T(txfunfl) + 5,
+	IDX_IN_WL_CNT_VER_6_T(txfbw),
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_6_T(txtplunfl),
+	IDX_IN_WL_CNT_VER_6_T(txphyerror),
+	IDX_IN_WL_CNT_VER_6_T(pktengrxducast),
+	IDX_IN_WL_CNT_VER_6_T(pktengrxdmcast),
+	IDX_IN_WL_CNT_VER_6_T(rxfrmtoolong),
+	IDX_IN_WL_CNT_VER_6_T(rxfrmtooshrt),
+	IDX_IN_WL_CNT_VER_6_T(rxinvmachdr),
+	IDX_IN_WL_CNT_VER_6_T(rxbadfcs),
+	IDX_IN_WL_CNT_VER_6_T(rxbadplcp),
+	IDX_IN_WL_CNT_VER_6_T(rxcrsglitch),
+	IDX_IN_WL_CNT_VER_6_T(rxstrt),
+	IDX_IN_WL_CNT_VER_6_T(rxdfrmucastmbss),
+	IDX_IN_WL_CNT_VER_6_T(rxmfrmucastmbss),
+	IDX_IN_WL_CNT_VER_6_T(rxcfrmucast),
+	IDX_IN_WL_CNT_VER_6_T(rxrtsucast),
+	IDX_IN_WL_CNT_VER_6_T(rxctsucast),
+	IDX_IN_WL_CNT_VER_6_T(rxackucast),
+	IDX_IN_WL_CNT_VER_6_T(rxdfrmocast),
+	IDX_IN_WL_CNT_VER_6_T(rxmfrmocast),
+	IDX_IN_WL_CNT_VER_6_T(rxcfrmocast),
+	IDX_IN_WL_CNT_VER_6_T(rxrtsocast),
+	IDX_IN_WL_CNT_VER_6_T(rxctsocast),
+	IDX_IN_WL_CNT_VER_6_T(rxdfrmmcast),
+	IDX_IN_WL_CNT_VER_6_T(rxmfrmmcast),
+	IDX_IN_WL_CNT_VER_6_T(rxcfrmmcast),
+	IDX_IN_WL_CNT_VER_6_T(rxbeaconmbss),
+	IDX_IN_WL_CNT_VER_6_T(rxdfrmucastobss),
+	IDX_IN_WL_CNT_VER_6_T(rxbeaconobss),
+	IDX_IN_WL_CNT_VER_6_T(rxrsptmout),
+	IDX_IN_WL_CNT_VER_6_T(bcntxcancl),
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_6_T(rxf0ovfl),
+	IDX_IN_WL_CNT_VER_6_T(rxf1ovfl),
+	IDX_IN_WL_CNT_VER_6_T(rxf2ovfl),
+	IDX_IN_WL_CNT_VER_6_T(txsfovfl),
+	IDX_IN_WL_CNT_VER_6_T(pmqovfl),
+	IDX_IN_WL_CNT_VER_6_T(rxcgprqfrm),
+	IDX_IN_WL_CNT_VER_6_T(rxcgprsqovfl),
+	IDX_IN_WL_CNT_VER_6_T(txcgprsfail),
+	IDX_IN_WL_CNT_VER_6_T(txcgprssuc),
+	IDX_IN_WL_CNT_VER_6_T(prs_timeout),
+	IDX_IN_WL_CNT_VER_6_T(rxnack),
+	IDX_IN_WL_CNT_VER_6_T(frmscons),
+	IDX_IN_WL_CNT_VER_6_T(txnack),
+	IDX_IN_WL_CNT_VER_6_T(rxback),
+	IDX_IN_WL_CNT_VER_6_T(txback),
+	IDX_IN_WL_CNT_VER_6_T(bphy_rxcrsglitch),
+	IDX_IN_WL_CNT_VER_6_T(rxdrop20s),
+	IDX_IN_WL_CNT_VER_6_T(rxtoolate),
+	IDX_IN_WL_CNT_VER_6_T(bphy_badplcp)
+};
+
+/* Index conversion table from wl_cnt_ver_7_t to wl_cnt_v_le10_mcst_t */
+static const uint8 wlcntver7t_to_wlcntvle10mcstt[WL_CNT_MCST_VAR_NUM] = {
+	IDX_IN_WL_CNT_VER_7_T(txallfrm),
+	IDX_IN_WL_CNT_VER_7_T(txrtsfrm),
+	IDX_IN_WL_CNT_VER_7_T(txctsfrm),
+	IDX_IN_WL_CNT_VER_7_T(txackfrm),
+	IDX_IN_WL_CNT_VER_7_T(txdnlfrm),
+	IDX_IN_WL_CNT_VER_7_T(txbcnfrm),
+	IDX_IN_WL_CNT_VER_7_T(txfunfl),
+	IDX_IN_WL_CNT_VER_7_T(txfunfl) + 1,
+	IDX_IN_WL_CNT_VER_7_T(txfunfl) + 2,
+	IDX_IN_WL_CNT_VER_7_T(txfunfl) + 3,
+	IDX_IN_WL_CNT_VER_7_T(txfunfl) + 4,
+	IDX_IN_WL_CNT_VER_7_T(txfunfl) + 5,
+	INVALID_MCST_IDX,
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_7_T(txtplunfl),
+	IDX_IN_WL_CNT_VER_7_T(txphyerror),
+	IDX_IN_WL_CNT_VER_7_T(pktengrxducast),
+	IDX_IN_WL_CNT_VER_7_T(pktengrxdmcast),
+	IDX_IN_WL_CNT_VER_7_T(rxfrmtoolong),
+	IDX_IN_WL_CNT_VER_7_T(rxfrmtooshrt),
+	IDX_IN_WL_CNT_VER_7_T(rxinvmachdr),
+	IDX_IN_WL_CNT_VER_7_T(rxbadfcs),
+	IDX_IN_WL_CNT_VER_7_T(rxbadplcp),
+	IDX_IN_WL_CNT_VER_7_T(rxcrsglitch),
+	IDX_IN_WL_CNT_VER_7_T(rxstrt),
+	IDX_IN_WL_CNT_VER_7_T(rxdfrmucastmbss),
+	IDX_IN_WL_CNT_VER_7_T(rxmfrmucastmbss),
+	IDX_IN_WL_CNT_VER_7_T(rxcfrmucast),
+	IDX_IN_WL_CNT_VER_7_T(rxrtsucast),
+	IDX_IN_WL_CNT_VER_7_T(rxctsucast),
+	IDX_IN_WL_CNT_VER_7_T(rxackucast),
+	IDX_IN_WL_CNT_VER_7_T(rxdfrmocast),
+	IDX_IN_WL_CNT_VER_7_T(rxmfrmocast),
+	IDX_IN_WL_CNT_VER_7_T(rxcfrmocast),
+	IDX_IN_WL_CNT_VER_7_T(rxrtsocast),
+	IDX_IN_WL_CNT_VER_7_T(rxctsocast),
+	IDX_IN_WL_CNT_VER_7_T(rxdfrmmcast),
+	IDX_IN_WL_CNT_VER_7_T(rxmfrmmcast),
+	IDX_IN_WL_CNT_VER_7_T(rxcfrmmcast),
+	IDX_IN_WL_CNT_VER_7_T(rxbeaconmbss),
+	IDX_IN_WL_CNT_VER_7_T(rxdfrmucastobss),
+	IDX_IN_WL_CNT_VER_7_T(rxbeaconobss),
+	IDX_IN_WL_CNT_VER_7_T(rxrsptmout),
+	IDX_IN_WL_CNT_VER_7_T(bcntxcancl),
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_7_T(rxf0ovfl),
+	IDX_IN_WL_CNT_VER_7_T(rxf1ovfl),
+	IDX_IN_WL_CNT_VER_7_T(rxf2ovfl),
+	IDX_IN_WL_CNT_VER_7_T(txsfovfl),
+	IDX_IN_WL_CNT_VER_7_T(pmqovfl),
+	IDX_IN_WL_CNT_VER_7_T(rxcgprqfrm),
+	IDX_IN_WL_CNT_VER_7_T(rxcgprsqovfl),
+	IDX_IN_WL_CNT_VER_7_T(txcgprsfail),
+	IDX_IN_WL_CNT_VER_7_T(txcgprssuc),
+	IDX_IN_WL_CNT_VER_7_T(prs_timeout),
+	IDX_IN_WL_CNT_VER_7_T(rxnack),
+	IDX_IN_WL_CNT_VER_7_T(frmscons),
+	IDX_IN_WL_CNT_VER_7_T(txnack),
+	INVALID_MCST_IDX,
+	INVALID_MCST_IDX,
+	IDX_IN_WL_CNT_VER_7_T(bphy_rxcrsglitch),
+	INVALID_MCST_IDX,
+	INVALID_MCST_IDX,
+	INVALID_MCST_IDX
+};
+
+/* copy wlc layer counters from old type cntbuf to wl_cnt_wlc_t type. */
+static int
+wl_copy_wlccnt(uint16 cntver, uint32 *dst, uint32 *src, uint8 src_max_idx)
+{
+	uint i;
+	if (dst == NULL || src == NULL) {
+		return BCME_ERROR;
+	}
+
+	/* Init wlccnt with invalid value. Unchanged value will not be printed out */
+	for (i = 0; i < (sizeof(wl_cnt_wlc_t) / sizeof(uint32)); i++) {
+		dst[i] = INVALID_CNT_VAL;
+	}
+
+	if (cntver == WL_CNT_VERSION_6) {
+		for (i = 0; i < NUM_OF_WLCCNT_IN_WL_CNT_VER_6_T; i++) {
+			if (wlcntver6t_to_wlcntwlct[i] >= src_max_idx) {
+				/* src buffer does not have counters from here */
+				break;
+			}
+			dst[i] = src[wlcntver6t_to_wlcntwlct[i]];
+		}
+	} else if (cntver == WL_CNT_VERSION_7) {
+		for (i = 0; i < NUM_OF_WLCCNT_IN_WL_CNT_VER_7_T; i++) {
+			if (wlcntver7t_to_wlcntwlct[i] >= src_max_idx ||
+				wlcntver7t_to_wlcntwlct[i] == INVALID_IDX) {
+				continue;
+			}
+			dst[i] = src[wlcntver7t_to_wlcntwlct[i]];
+		}
+	} else {
+		for (i = 0; i < NUM_OF_WLCCNT_IN_WL_CNT_VER_11_T; i++) {
+			if (wlcntver11t_to_wlcntwlct[i] >= src_max_idx) {
+				if (wlcntver11t_to_wlcntwlct[i] == INVALID_IDX) {
+					continue;
+				}
+				else {
+					/* src buffer does not have counters from here */
+					break;
+				}
+			}
+			dst[i] = src[wlcntver11t_to_wlcntwlct[i]];
+		}
+	}
+	return BCME_OK;
+}
+
+/* copy macstat counters from old type cntbuf to wl_cnt_v_le10_mcst_t type. */
+static int
+wl_copy_macstat_upto_ver10(uint16 cntver, uint32 *dst, uint32 *src)
+{
+	uint i;
+
+	if (dst == NULL || src == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (cntver == WL_CNT_VERSION_6) {
+		for (i = 0; i < WL_CNT_MCST_VAR_NUM; i++) {
+			if (wlcntver6t_to_wlcntvle10mcstt[i] == INVALID_MCST_IDX) {
+				/* This mcst counter does not exist in wl_cnt_ver_6_t */
+				dst[i] = INVALID_CNT_VAL;
+			} else {
+				dst[i] = src[wlcntver6t_to_wlcntvle10mcstt[i]];
+			}
+		}
+	} else if (cntver == WL_CNT_VERSION_7) {
+		for (i = 0; i < WL_CNT_MCST_VAR_NUM; i++) {
+			if (wlcntver7t_to_wlcntvle10mcstt[i] == INVALID_MCST_IDX) {
+				/* This mcst counter does not exist in wl_cnt_ver_7_t */
+				dst[i] = INVALID_CNT_VAL;
+			} else {
+				dst[i] = src[wlcntver7t_to_wlcntvle10mcstt[i]];
+			}
+		}
+	} else {
+		for (i = 0; i < WL_CNT_MCST_VAR_NUM; i++) {
+			if (wlcntver11t_to_wlcntvle10mcstt[i] == INVALID_MCST_IDX) {
+				/* This mcst counter does not exist in wl_cnt_ver_11_t */
+				dst[i] = INVALID_CNT_VAL;
+			} else {
+				dst[i] = src[wlcntver11t_to_wlcntvle10mcstt[i]];
+			}
+		}
+	}
+	return BCME_OK;
+}
+
+static int
+wl_copy_macstat_ver11(uint32 *dst, uint32 *src)
+{
+	uint i;
+
+	if (dst == NULL || src == NULL) {
+		return BCME_ERROR;
+	}
+
+	for (i = 0; i < WL_CNT_MCST_VAR_NUM; i++) {
+		dst[i] = src[wlcntver11t_to_wlcntXX40mcstv1t[i]];
+	}
+	return BCME_OK;
+}
+
+/**
+ * Translate non-xtlv 'wl counters' IOVar buffer received by old driver/FW to xtlv format.
+ * Parameters:
+ *	cntbuf: pointer to non-xtlv 'wl counters' IOVar buffer received by old driver/FW.
+ *		Newly translated xtlv format is written to this pointer.
+ *	buflen: length of the "cntbuf" without any padding.
+ *	corerev: chip core revision of the driver/FW.
+ */
+int
+wl_cntbuf_to_xtlv_format(void *ctx, void *cntbuf, int buflen, uint32 corerev)
+{
+	wl_cnt_wlc_t *wlccnt = NULL;
+	uint32 *macstat = NULL;
+	xtlv_desc_t xtlv_desc[3];
+	uint16 mcst_xtlv_id;
+	int res = BCME_OK;
+	wl_cnt_info_t *cntinfo = cntbuf;
+	uint8 *xtlvbuf_p = cntinfo->data;
+	uint16 ver = cntinfo->version;
+	uint16 xtlvbuflen = (uint16)buflen;
+	uint16 src_max_idx;
+#ifdef BCMDRIVER
+	osl_t *osh = ctx;
+#else
+	BCM_REFERENCE(ctx);
+#endif
+
+	if (ver >= WL_CNT_VERSION_XTLV) {
+		/* Already in xtlv format. */
+		goto exit;
+	}
+
+#ifdef BCMDRIVER
+	wlccnt = MALLOCZ(osh, sizeof(*wlccnt));
+	macstat = MALLOCZ(osh, WL_CNT_MCST_STRUCT_SZ);
+#else
+	wlccnt = (wl_cnt_wlc_t *)malloc(sizeof(*wlccnt));
+	macstat = (uint32 *)malloc(WL_CNT_MCST_STRUCT_SZ);
+#endif
+	if (!wlccnt || !macstat) {
+		printf("wl_cntbuf_to_xtlv_format: malloc fail!\n");
+		res = BCME_NOMEM;
+		goto exit;
+	}
+
+	/* Check if the max idx in the struct exceeds the boundary of uint8 */
+	if (NUM_OF_CNT_IN_WL_CNT_VER_6_T > ((uint8)(-1) + 1) ||
+		NUM_OF_CNT_IN_WL_CNT_VER_7_T > ((uint8)(-1) + 1) ||
+		NUM_OF_CNT_IN_WL_CNT_VER_11_T > ((uint8)(-1) + 1)) {
+		printf("wlcntverXXt_to_wlcntwlct and src_max_idx need"
+			" to be of uint16 instead of uint8\n");
+		res = BCME_ERROR;
+		goto exit;
+	}
+
+	/* Exclude version and length fields in either wlc_cnt_ver_6_t or wlc_cnt_ver_11_t */
+	src_max_idx = (cntinfo->datalen - OFFSETOF(wl_cnt_info_t, data)) / sizeof(uint32);
+	if (src_max_idx > (uint8)(-1)) {
+		printf("wlcntverXXt_to_wlcntwlct and src_max_idx need"
+			" to be of uint16 instead of uint8\n"
+			"Try updating wl utility to the latest.\n");
+		src_max_idx = (uint8)(-1);
+	}
+
+	/* Copy wlc layer counters to wl_cnt_wlc_t */
+	res = wl_copy_wlccnt(ver, (uint32 *)wlccnt, (uint32 *)cntinfo->data, (uint8)src_max_idx);
+	if (res != BCME_OK) {
+		printf("wl_copy_wlccnt fail!\n");
+		goto exit;
+	}
+
+	/* Copy macstat counters to wl_cnt_wlc_t */
+	if (ver == WL_CNT_VERSION_11) {
+		res = wl_copy_macstat_ver11(macstat, (uint32 *)cntinfo->data);
+		if (res != BCME_OK) {
+			printf("wl_copy_macstat_ver11 fail!\n");
+			goto exit;
+		}
+		if (corerev >= 40) {
+			mcst_xtlv_id = WL_CNT_XTLV_GE40_UCODE_V1;
+		} else {
+			mcst_xtlv_id = WL_CNT_XTLV_LT40_UCODE_V1;
+		}
+	} else {
+		res = wl_copy_macstat_upto_ver10(ver, macstat, (uint32 *)cntinfo->data);
+		if (res != BCME_OK) {
+			printf("wl_copy_macstat_upto_ver10 fail!\n");
+			goto exit;
+		}
+		mcst_xtlv_id = WL_CNT_XTLV_CNTV_LE10_UCODE;
+	}
+
+	xtlv_desc[0].type = WL_CNT_XTLV_WLC;
+	xtlv_desc[0].len = sizeof(*wlccnt);
+	xtlv_desc[0].ptr = wlccnt;
+
+	xtlv_desc[1].type = mcst_xtlv_id;
+	xtlv_desc[1].len = WL_CNT_MCST_STRUCT_SZ;
+	xtlv_desc[1].ptr = macstat;
+
+	xtlv_desc[2].type = 0;
+	xtlv_desc[2].len = 0;
+	xtlv_desc[2].ptr = NULL;
+
+	bzero(cntbuf, buflen);
+
+	res = bcm_pack_xtlv_buf_from_mem(&xtlvbuf_p, &xtlvbuflen,
+		xtlv_desc, BCM_XTLV_OPTION_ALIGN32);
+	cntinfo->datalen = (buflen - xtlvbuflen);
+exit:
+#ifdef BCMDRIVER
+	if (wlccnt) {
+		MFREE(osh, wlccnt, sizeof(*wlccnt));
+	}
+	if (macstat) {
+		MFREE(osh, macstat, WL_CNT_MCST_STRUCT_SZ);
+	}
+#else
+	if (wlccnt) {
+		free(wlccnt);
+	}
+	if (macstat) {
+		free(macstat);
+	}
+#endif
+	return res;
+}
diff --git a/drivers/net/wireless/ap12275/bcmbloom.c b/drivers/net/wireless/ap12275/bcmbloom.c
new file mode 100644
index 000000000000..a062531413b9
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmbloom.c
@@ -0,0 +1,232 @@
+/*
+ * Bloom filter support
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmdefs.h>
+
+#ifdef BCMDRIVER
+#include <osl.h>
+#include <bcmutils.h>
+#else /* !BCMDRIVER */
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#ifndef ASSERT
+#define ASSERT(exp)
+#endif
+#endif /* !BCMDRIVER */
+#include <bcmutils.h>
+
+#include <bcmbloom.h>
+
+#define BLOOM_BIT_LEN(_x) ((_x) << 3)
+
+struct bcm_bloom_filter {
+	void *cb_ctx;
+	uint max_hash;
+	bcm_bloom_hash_t *hash;	/* array of hash functions */
+	uint filter_size; 		/* in bytes */
+	uint8 *filter; 			/* can be NULL for validate only */
+};
+
+/* public interface */
+int
+bcm_bloom_create(bcm_bloom_alloc_t alloc_cb,
+	bcm_bloom_free_t free_cb, void *cb_ctx, uint max_hash,
+	uint filter_size, bcm_bloom_filter_t **bloom)
+{
+	int err = BCME_OK;
+	bcm_bloom_filter_t *bp = NULL;
+
+	if (!bloom || !alloc_cb || (max_hash == 0)) {
+		err = BCME_BADARG;
+		goto done;
+	}
+
+	bp = (*alloc_cb)(cb_ctx, sizeof(*bp));
+	if (!bp) {
+		err = BCME_NOMEM;
+		goto done;
+	}
+	bzero(bp, sizeof(*bp));
+
+	bp->cb_ctx = cb_ctx;
+	bp->max_hash = max_hash;
+	bp->hash = (*alloc_cb)(cb_ctx, sizeof(*bp->hash) * max_hash);
+	if (!bp->hash) {
+		err = BCME_NOMEM;
+		goto done;
+	}
+	bzero(bp->hash, sizeof(*bp->hash) * max_hash);
+
+	if (filter_size > 0) {
+		bp->filter = (*alloc_cb)(cb_ctx, filter_size);
+		if (!bp->filter) {
+			err = BCME_NOMEM;
+			goto done;
+		}
+		bp->filter_size = filter_size;
+		bzero(bp->filter, filter_size);
+	}
+
+	*bloom = bp;
+
+done:
+	if (err != BCME_OK)
+		bcm_bloom_destroy(&bp, free_cb);
+
+	return err;
+}
+
+int
+bcm_bloom_destroy(bcm_bloom_filter_t **bloom, bcm_bloom_free_t free_cb)
+{
+	int err = BCME_OK;
+	bcm_bloom_filter_t *bp;
+
+	if (!bloom || !*bloom || !free_cb)
+		goto done;
+
+	bp = *bloom;
+	*bloom = NULL;
+
+	if (bp->filter)
+		(*free_cb)(bp->cb_ctx, bp->filter, bp->filter_size);
+	if (bp->hash)
+		(*free_cb)(bp->cb_ctx, bp->hash,
+			sizeof(*bp->hash) * bp->max_hash);
+	(*free_cb)(bp->cb_ctx, bp, sizeof(*bp));
+
+done:
+	return err;
+}
+
+int
+bcm_bloom_add_hash(bcm_bloom_filter_t *bp, bcm_bloom_hash_t hash, uint *idx)
+{
+	uint i;
+
+	if (!bp || !hash || !idx)
+		return BCME_BADARG;
+
+	for (i = 0; i < bp->max_hash; ++i) {
+		if (bp->hash[i] == NULL)
+			break;
+	}
+
+	if (i >= bp->max_hash)
+		return BCME_NORESOURCE;
+
+	bp->hash[i] = hash;
+	*idx = i;
+	return BCME_OK;
+}
+
+int
+bcm_bloom_remove_hash(bcm_bloom_filter_t *bp, uint idx)
+{
+	if (!bp)
+		return BCME_BADARG;
+
+	if (idx >= bp->max_hash)
+		return BCME_NOTFOUND;
+
+	bp->hash[idx] = NULL;
+	return BCME_OK;
+}
+
+bool
+bcm_bloom_is_member(bcm_bloom_filter_t *bp,
+	const uint8 *tag, uint tag_len, const uint8 *buf, uint buf_len)
+{
+	uint i;
+	int err = BCME_OK;
+
+	if (!tag || (tag_len == 0)) /* empty tag is always a member */
+		goto done;
+
+	/* use internal buffer if none was specified */
+	if (!buf || (buf_len == 0)) {
+		if (!bp->filter)	/* every one is a member of empty filter */
+			goto done;
+
+		buf = bp->filter;
+		buf_len = bp->filter_size;
+	}
+
+	for (i = 0; i < bp->max_hash; ++i) {
+		uint pos;
+		if (!bp->hash[i])
+			continue;
+		pos = (*bp->hash[i])(bp->cb_ctx, i, tag, tag_len);
+
+		/* all bits must be set for a match */
+		if (isclr(buf, pos % BLOOM_BIT_LEN(buf_len))) {
+			err = BCME_NOTFOUND;
+			break;
+		}
+	}
+
+done:
+	return err;
+}
+
+int
+bcm_bloom_add_member(bcm_bloom_filter_t *bp, const uint8 *tag, uint tag_len)
+{
+	uint i;
+
+	if (!bp || !tag || (tag_len == 0))
+		return BCME_BADARG;
+
+	if (!bp->filter)		/* validate only */
+		return BCME_UNSUPPORTED;
+
+	for (i = 0; i < bp->max_hash; ++i) {
+		uint pos;
+		if (!bp->hash[i])
+			continue;
+		pos = (*bp->hash[i])(bp->cb_ctx, i, tag, tag_len);
+		setbit(bp->filter, pos % BLOOM_BIT_LEN(bp->filter_size));
+	}
+
+	return BCME_OK;
+}
+
+int bcm_bloom_get_filter_data(bcm_bloom_filter_t *bp,
+	uint buf_size, uint8 *buf, uint *buf_len)
+{
+	if (!bp)
+		return BCME_BADARG;
+
+	if (buf_len)
+		*buf_len = bp->filter_size;
+
+	if (buf_size < bp->filter_size)
+		return BCME_BUFTOOSHORT;
+
+	if (bp->filter && bp->filter_size)
+		memcpy(buf, bp->filter, bp->filter_size);
+
+	return BCME_OK;
+}
diff --git a/drivers/net/wireless/ap12275/bcmevent.c b/drivers/net/wireless/ap12275/bcmevent.c
new file mode 100644
index 000000000000..25054456f58e
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmevent.c
@@ -0,0 +1,449 @@
+/*
+ * bcmevent read-only data shared by kernel or app layers
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <ethernet.h>
+#include <bcmeth.h>
+#include <bcmevent.h>
+#include <802.11.h>
+#include <802.11brcm.h>
+
+/* Table of event name strings for UIs and debugging dumps */
+typedef struct {
+	uint event;
+	const char *name;
+} bcmevent_name_str_t;
+
+/* Use the actual name for event tracing */
+#define BCMEVENT_NAME(_event) {(_event), #_event}
+
+/* this becomes static data when all code is changed to use
+ * the bcmevent_get_name() API
+ */
+static const bcmevent_name_str_t bcmevent_names[] = {
+	BCMEVENT_NAME(WLC_E_SET_SSID),
+	BCMEVENT_NAME(WLC_E_JOIN),
+	BCMEVENT_NAME(WLC_E_START),
+	BCMEVENT_NAME(WLC_E_AUTH),
+	BCMEVENT_NAME(WLC_E_AUTH_IND),
+	BCMEVENT_NAME(WLC_E_DEAUTH),
+	BCMEVENT_NAME(WLC_E_DEAUTH_IND),
+	BCMEVENT_NAME(WLC_E_ASSOC),
+	BCMEVENT_NAME(WLC_E_ASSOC_IND),
+	BCMEVENT_NAME(WLC_E_REASSOC),
+	BCMEVENT_NAME(WLC_E_REASSOC_IND),
+	BCMEVENT_NAME(WLC_E_DISASSOC),
+	BCMEVENT_NAME(WLC_E_DISASSOC_IND),
+	BCMEVENT_NAME(WLC_E_QUIET_START),
+	BCMEVENT_NAME(WLC_E_QUIET_END),
+	BCMEVENT_NAME(WLC_E_BEACON_RX),
+	BCMEVENT_NAME(WLC_E_LINK),
+	BCMEVENT_NAME(WLC_E_MIC_ERROR),
+	BCMEVENT_NAME(WLC_E_NDIS_LINK),
+	BCMEVENT_NAME(WLC_E_ROAM),
+	BCMEVENT_NAME(WLC_E_TXFAIL),
+	BCMEVENT_NAME(WLC_E_PMKID_CACHE),
+	BCMEVENT_NAME(WLC_E_RETROGRADE_TSF),
+	BCMEVENT_NAME(WLC_E_PRUNE),
+	BCMEVENT_NAME(WLC_E_AUTOAUTH),
+	BCMEVENT_NAME(WLC_E_EAPOL_MSG),
+	BCMEVENT_NAME(WLC_E_SCAN_COMPLETE),
+	BCMEVENT_NAME(WLC_E_ADDTS_IND),
+	BCMEVENT_NAME(WLC_E_DELTS_IND),
+	BCMEVENT_NAME(WLC_E_BCNSENT_IND),
+	BCMEVENT_NAME(WLC_E_BCNRX_MSG),
+	BCMEVENT_NAME(WLC_E_BCNLOST_MSG),
+	BCMEVENT_NAME(WLC_E_ROAM_PREP),
+	BCMEVENT_NAME(WLC_E_PFN_NET_FOUND),
+	BCMEVENT_NAME(WLC_E_PFN_SCAN_ALLGONE),
+	BCMEVENT_NAME(WLC_E_PFN_NET_LOST),
+	BCMEVENT_NAME(WLC_E_JOIN_START),
+	BCMEVENT_NAME(WLC_E_ROAM_START),
+	BCMEVENT_NAME(WLC_E_ASSOC_START),
+#ifdef EXT_STA
+	BCMEVENT_NAME(WLC_E_RESET_COMPLETE),
+	BCMEVENT_NAME(WLC_E_JOIN_START),
+	BCMEVENT_NAME(WLC_E_ROAM_START),
+	BCMEVENT_NAME(WLC_E_ASSOC_START),
+	BCMEVENT_NAME(WLC_E_ASSOC_RECREATED),
+	BCMEVENT_NAME(WLC_E_SPEEDY_RECREATE_FAIL),
+#endif /* EXT_STA */
+#if defined(IBSS_PEER_DISCOVERY_EVENT)
+	BCMEVENT_NAME(WLC_E_IBSS_ASSOC),
+#endif /* defined(IBSS_PEER_DISCOVERY_EVENT) */
+	BCMEVENT_NAME(WLC_E_RADIO),
+	BCMEVENT_NAME(WLC_E_PSM_WATCHDOG),
+	BCMEVENT_NAME(WLC_E_PROBREQ_MSG),
+	BCMEVENT_NAME(WLC_E_SCAN_CONFIRM_IND),
+	BCMEVENT_NAME(WLC_E_PSK_SUP),
+	BCMEVENT_NAME(WLC_E_COUNTRY_CODE_CHANGED),
+	BCMEVENT_NAME(WLC_E_EXCEEDED_MEDIUM_TIME),
+	BCMEVENT_NAME(WLC_E_ICV_ERROR),
+	BCMEVENT_NAME(WLC_E_UNICAST_DECODE_ERROR),
+	BCMEVENT_NAME(WLC_E_MULTICAST_DECODE_ERROR),
+	BCMEVENT_NAME(WLC_E_TRACE),
+	BCMEVENT_NAME(WLC_E_IF),
+#ifdef WLP2P
+	BCMEVENT_NAME(WLC_E_P2P_DISC_LISTEN_COMPLETE),
+#endif
+	BCMEVENT_NAME(WLC_E_RSSI),
+	BCMEVENT_NAME(WLC_E_PFN_SCAN_COMPLETE),
+	BCMEVENT_NAME(WLC_E_ACTION_FRAME),
+	BCMEVENT_NAME(WLC_E_ACTION_FRAME_RX),
+	BCMEVENT_NAME(WLC_E_ACTION_FRAME_COMPLETE),
+#if defined(NDIS)
+	BCMEVENT_NAME(WLC_E_PRE_ASSOC_IND),
+	BCMEVENT_NAME(WLC_E_PRE_REASSOC_IND),
+	BCMEVENT_NAME(WLC_E_CHANNEL_ADOPTED),
+	BCMEVENT_NAME(WLC_E_AP_STARTED),
+	BCMEVENT_NAME(WLC_E_DFS_AP_STOP),
+	BCMEVENT_NAME(WLC_E_DFS_AP_RESUME),
+	BCMEVENT_NAME(WLC_E_ASSOC_IND_NDIS),
+	BCMEVENT_NAME(WLC_E_REASSOC_IND_NDIS),
+	BCMEVENT_NAME(WLC_E_ACTION_FRAME_RX_NDIS),
+	BCMEVENT_NAME(WLC_E_AUTH_REQ),
+	BCMEVENT_NAME(WLC_E_IBSS_COALESCE),
+#endif /* #if defined(NDIS) */
+
+#ifdef BCMWAPI_WAI
+	BCMEVENT_NAME(WLC_E_WAI_STA_EVENT),
+	BCMEVENT_NAME(WLC_E_WAI_MSG),
+#endif /* BCMWAPI_WAI */
+
+	BCMEVENT_NAME(WLC_E_ESCAN_RESULT),
+	BCMEVENT_NAME(WLC_E_ACTION_FRAME_OFF_CHAN_COMPLETE),
+#ifdef WLP2P
+	BCMEVENT_NAME(WLC_E_PROBRESP_MSG),
+	BCMEVENT_NAME(WLC_E_P2P_PROBREQ_MSG),
+#endif
+#ifdef PROP_TXSTATUS
+	BCMEVENT_NAME(WLC_E_FIFO_CREDIT_MAP),
+#endif
+	BCMEVENT_NAME(WLC_E_WAKE_EVENT),
+	BCMEVENT_NAME(WLC_E_DCS_REQUEST),
+	BCMEVENT_NAME(WLC_E_RM_COMPLETE),
+	BCMEVENT_NAME(WLC_E_OVERLAY_REQ),
+	BCMEVENT_NAME(WLC_E_CSA_COMPLETE_IND),
+	BCMEVENT_NAME(WLC_E_EXCESS_PM_WAKE_EVENT),
+	BCMEVENT_NAME(WLC_E_PFN_SCAN_NONE),
+	BCMEVENT_NAME(WLC_E_PFN_SCAN_ALLGONE),
+#ifdef SOFTAP
+	BCMEVENT_NAME(WLC_E_GTK_PLUMBED),
+#endif
+	BCMEVENT_NAME(WLC_E_ASSOC_REQ_IE),
+	BCMEVENT_NAME(WLC_E_ASSOC_RESP_IE),
+	BCMEVENT_NAME(WLC_E_BEACON_FRAME_RX),
+#ifdef WLTDLS
+	BCMEVENT_NAME(WLC_E_TDLS_PEER_EVENT),
+#endif /* WLTDLS */
+	BCMEVENT_NAME(WLC_E_NATIVE),
+#ifdef WLPKTDLYSTAT
+	BCMEVENT_NAME(WLC_E_PKTDELAY_IND),
+#endif /* WLPKTDLYSTAT */
+	BCMEVENT_NAME(WLC_E_SERVICE_FOUND),
+	BCMEVENT_NAME(WLC_E_GAS_FRAGMENT_RX),
+	BCMEVENT_NAME(WLC_E_GAS_COMPLETE),
+	BCMEVENT_NAME(WLC_E_P2PO_ADD_DEVICE),
+	BCMEVENT_NAME(WLC_E_P2PO_DEL_DEVICE),
+#ifdef WLWNM
+	BCMEVENT_NAME(WLC_E_WNM_STA_SLEEP),
+#endif /* WLWNM */
+#if defined(WL_PROXDETECT) || defined(RTT_SUPPORT)
+	BCMEVENT_NAME(WLC_E_PROXD),
+#endif
+	BCMEVENT_NAME(WLC_E_CCA_CHAN_QUAL),
+	BCMEVENT_NAME(WLC_E_BSSID),
+#ifdef PROP_TXSTATUS
+	BCMEVENT_NAME(WLC_E_BCMC_CREDIT_SUPPORT),
+#endif
+	BCMEVENT_NAME(WLC_E_PSTA_PRIMARY_INTF_IND),
+	BCMEVENT_NAME(WLC_E_TXFAIL_THRESH),
+#ifdef GSCAN_SUPPORT
+	BCMEVENT_NAME(WLC_E_PFN_GSCAN_FULL_RESULT),
+	BCMEVENT_NAME(WLC_E_PFN_SSID_EXT),
+#endif /* GSCAN_SUPPORT */
+#ifdef WLBSSLOAD_REPORT
+	BCMEVENT_NAME(WLC_E_BSS_LOAD),
+#endif
+#ifdef WLFBT
+	BCMEVENT_NAME(WLC_E_FBT),
+#endif /* WLFBT */
+	BCMEVENT_NAME(WLC_E_AUTHORIZED),
+	BCMEVENT_NAME(WLC_E_PROBREQ_MSG_RX),
+
+	BCMEVENT_NAME(WLC_E_CSA_START_IND),
+	BCMEVENT_NAME(WLC_E_CSA_DONE_IND),
+	BCMEVENT_NAME(WLC_E_CSA_FAILURE_IND),
+	BCMEVENT_NAME(WLC_E_RMC_EVENT),
+	BCMEVENT_NAME(WLC_E_DPSTA_INTF_IND),
+	BCMEVENT_NAME(WLC_E_ALLOW_CREDIT_BORROW),
+	BCMEVENT_NAME(WLC_E_MSCH),
+	BCMEVENT_NAME(WLC_E_ULP),
+	BCMEVENT_NAME(WLC_E_NAN),
+	BCMEVENT_NAME(WLC_E_PKT_FILTER),
+	BCMEVENT_NAME(WLC_E_DMA_TXFLUSH_COMPLETE),
+	BCMEVENT_NAME(WLC_E_PSK_AUTH),
+	BCMEVENT_NAME(WLC_E_SDB_TRANSITION),
+	BCMEVENT_NAME(WLC_E_PFN_SCAN_BACKOFF),
+	BCMEVENT_NAME(WLC_E_PFN_BSSID_SCAN_BACKOFF),
+	BCMEVENT_NAME(WLC_E_AGGR_EVENT),
+	BCMEVENT_NAME(WLC_E_TVPM_MITIGATION),
+	BCMEVENT_NAME(WLC_E_SCAN),
+	BCMEVENT_NAME(WLC_E_SLOTTED_BSS_PEER_OP),
+	BCMEVENT_NAME(WLC_E_PHY_CAL),
+#ifdef WL_NAN
+	BCMEVENT_NAME(WLC_E_NAN_CRITICAL),
+	BCMEVENT_NAME(WLC_E_NAN_NON_CRITICAL),
+	BCMEVENT_NAME(WLC_E_NAN),
+#endif /* WL_NAN */
+	BCMEVENT_NAME(WLC_E_RPSNOA),
+	BCMEVENT_NAME(WLC_E_WA_LQM),
+	BCMEVENT_NAME(WLC_E_OBSS_DETECTION),
+	BCMEVENT_NAME(WLC_E_SC_CHAN_QUAL),
+	BCMEVENT_NAME(WLC_E_DYNSAR),
+	BCMEVENT_NAME(WLC_E_ROAM_CACHE_UPDATE),
+	BCMEVENT_NAME(WLC_E_AP_BCN_DRIFT),
+	BCMEVENT_NAME(WLC_E_PFN_SCAN_ALLGONE_EXT),
+#ifdef WL_TWT
+	BCMEVENT_NAME(WLC_E_TWT),
+#endif /* WL_TWT */
+	BCMEVENT_NAME(WLC_E_AMT),
+	BCMEVENT_NAME(WLC_E_ROAM_SCAN_RESULT),
+#if defined(XRAPI)
+	BCMEVENT_NAME(WLC_E_XR_SOFTAP_PSMODE),
+#endif /* XRAPI */
+#ifdef WL_SIB_COEX
+	BCMEVENT_NAME(WLC_E_SIB),
+#endif /* WL_SIB_COEX */
+	BCMEVENT_NAME(WLC_E_MSCS),
+	BCMEVENT_NAME(WLC_E_RXDMA_RECOVERY_ATMPT),
+#ifdef WL_SCHED_SCAN
+	BCMEVENT_NAME(WLC_E_PFN_PARTIAL_RESULT),
+#endif /* WL_SCHED_SCAN */
+	BCMEVENT_NAME(WLC_E_MLO_LINK_INFO),
+	BCMEVENT_NAME(WLC_E_C2C),
+	BCMEVENT_NAME(WLC_E_BCN_TSF),
+	BCMEVENT_NAME(WLC_E_OWE_INFO),
+	BCMEVENT_NAME(WLC_E_ULMU_DISABLED_REASON_UPD),
+};
+
+const char *bcmevent_get_name(uint event_type)
+{
+	/* note:  first coded this as a static const but some
+	 * ROMs already have something called event_name so
+	 * changed it so we don't have a variable for the
+	 * 'unknown string
+	 */
+	const char *event_name = NULL;
+
+	uint idx;
+	for (idx = 0; idx < (uint)ARRAYSIZE(bcmevent_names); idx++) {
+
+		if (bcmevent_names[idx].event == event_type) {
+			event_name = bcmevent_names[idx].name;
+			break;
+		}
+	}
+
+	/* if we find an event name in the array, return it.
+	 * otherwise return unknown string.
+	 */
+	return ((event_name) ? event_name : "Unknown Event");
+}
+
+void
+wl_event_to_host_order(wl_event_msg_t * evt)
+{
+	/* Event struct members passed from dongle to host are stored in network
+	* byte order. Convert all members to host-order.
+	*/
+	evt->event_type = ntoh32(evt->event_type);
+	evt->flags = ntoh16(evt->flags);
+	evt->status = ntoh32(evt->status);
+	evt->reason = ntoh32(evt->reason);
+	evt->auth_type = ntoh32(evt->auth_type);
+	evt->datalen = ntoh32(evt->datalen);
+	evt->version = ntoh16(evt->version);
+}
+
+void
+wl_event_to_network_order(wl_event_msg_t * evt)
+{
+	/* Event struct members passed from dongle to host are stored in network
+	* byte order. Convert all members to host-order.
+	*/
+	evt->event_type = hton32(evt->event_type);
+	evt->flags = hton16(evt->flags);
+	evt->status = hton32(evt->status);
+	evt->reason = hton32(evt->reason);
+	evt->auth_type = hton32(evt->auth_type);
+	evt->datalen = hton32(evt->datalen);
+	evt->version = hton16(evt->version);
+}
+
+/*
+ * Validate if the event is proper and if valid copy event header to event.
+ * If proper event pointer is passed, to just validate, pass NULL to event.
+ *
+ * Return values are
+ *	BCME_OK - It is a BRCM event or BRCM dongle event
+ *	BCME_NOTFOUND - Not BRCM, not an event, may be okay
+ *	BCME_BADLEN - Bad length, should not process, just drop
+ */
+int
+is_wlc_event_frame(void *pktdata, uint pktlen, uint16 exp_usr_subtype,
+	bcm_event_msg_u_t *out_event)
+{
+	uint16 evlen = 0;	/* length in bcmeth_hdr */
+	uint16 subtype;
+	uint16 usr_subtype;
+	bcm_event_t *bcm_event;
+	uint8 *pktend;
+	uint8 *evend;
+	int err = BCME_OK;
+	uint32 data_len = 0; /* data length in bcm_event */
+
+	pktend = (uint8 *)pktdata + pktlen;
+	bcm_event = (bcm_event_t *)pktdata;
+
+	/* only care about 16-bit subtype / length versions */
+	if ((uint8 *)&bcm_event->bcm_hdr < pktend) {
+		uint8 short_subtype = *(uint8 *)&bcm_event->bcm_hdr;
+		if (!(short_subtype & 0x80)) {
+			err = BCME_NOTFOUND;
+			goto done;
+		}
+	}
+
+	/* must have both ether_header and bcmeth_hdr */
+	if (pktlen < OFFSETOF(bcm_event_t, event)) {
+		err = BCME_BADLEN;
+		goto done;
+	}
+
+	/* check length in bcmeth_hdr */
+
+#ifdef BCMDONGLEHOST
+	/* temporary - header length not always set properly. When the below
+	 * !BCMDONGLEHOST is in all branches that use trunk DHD, the code
+	 * under BCMDONGLEHOST can be removed.
+	 */
+	evlen = (uint16)(pktend - (uint8 *)&bcm_event->bcm_hdr.version);
+#else
+	evlen = ntoh16_ua((void *)&bcm_event->bcm_hdr.length);
+#endif /* BCMDONGLEHOST */
+	evend = (uint8 *)&bcm_event->bcm_hdr.version + evlen;
+	if (evend != pktend) {
+		err = BCME_BADLEN;
+		goto done;
+	}
+
+	/* match on subtype, oui and usr subtype for BRCM events */
+	subtype = ntoh16_ua((void *)&bcm_event->bcm_hdr.subtype);
+	if (subtype != BCMILCP_SUBTYPE_VENDOR_LONG) {
+		err = BCME_NOTFOUND;
+		goto done;
+	}
+
+	if (bcmp(BRCM_OUI, &bcm_event->bcm_hdr.oui[0], DOT11_OUI_LEN)) {
+		err = BCME_NOTFOUND;
+		goto done;
+	}
+
+	/* if it is a bcm_event or bcm_dngl_event_t, validate it */
+	usr_subtype = ntoh16_ua((void *)&bcm_event->bcm_hdr.usr_subtype);
+	switch (usr_subtype) {
+	case BCMILCP_BCM_SUBTYPE_EVENT:
+		/* check that header length and pkt length are sufficient */
+		if ((pktlen < sizeof(bcm_event_t)) ||
+			(evend < ((uint8 *)bcm_event + sizeof(bcm_event_t)))) {
+			err = BCME_BADLEN;
+			goto done;
+		}
+
+		/* ensure data length in event is not beyond the packet. */
+		data_len = ntoh32_ua((void *)&bcm_event->event.datalen);
+		if ((sizeof(bcm_event_t) + data_len +
+			BCMILCP_BCM_SUBTYPE_EVENT_DATA_PAD) != pktlen) {
+			err = BCME_BADLEN;
+			goto done;
+		}
+
+		if (exp_usr_subtype && (exp_usr_subtype != usr_subtype)) {
+			err = BCME_NOTFOUND;
+			goto done;
+		}
+
+		if (out_event) {
+			/* ensure BRCM event pkt aligned */
+			(void)memcpy_s(&out_event->event, sizeof(out_event->event),
+				&bcm_event->event, sizeof(out_event->event));
+		}
+
+		break;
+
+	case BCMILCP_BCM_SUBTYPE_DNGLEVENT:
+#if defined(HEALTH_CHECK) || defined(DNGL_EVENT_SUPPORT)
+		if ((pktlen < sizeof(bcm_dngl_event_t)) ||
+			(evend < ((uint8 *)bcm_event + sizeof(bcm_dngl_event_t)))) {
+			err = BCME_BADLEN;
+			goto done;
+		}
+
+		/* ensure data length in event is not beyond the packet. */
+		data_len = ntoh16_ua((void *)&((bcm_dngl_event_t *)pktdata)->dngl_event.datalen);
+		if ((sizeof(bcm_dngl_event_t) + data_len +
+			BCMILCP_BCM_SUBTYPE_EVENT_DATA_PAD) != pktlen) {
+			err = BCME_BADLEN;
+			goto done;
+		}
+
+		if (exp_usr_subtype && (exp_usr_subtype != usr_subtype)) {
+			err = BCME_NOTFOUND;
+			goto done;
+		}
+
+		if (out_event) {
+			/* ensure BRCM dngl event pkt aligned */
+			(void)memcpy_s(&out_event->dngl_event, sizeof(out_event->dngl_event),
+				&((bcm_dngl_event_t *)pktdata)->dngl_event,
+				sizeof(out_event->dngl_event));
+		}
+
+		break;
+#else
+		err = BCME_UNSUPPORTED;
+		break;
+#endif /* HEALTH_CHECK  || DNGL_EVENT_SUPPORT */
+
+	default:
+		err = BCME_NOTFOUND;
+		goto done;
+	}
+
+	BCM_REFERENCE(data_len);
+done:
+	return err;
+}
diff --git a/drivers/net/wireless/ap12275/bcmsdh.c b/drivers/net/wireless/ap12275/bcmsdh.c
new file mode 100644
index 000000000000..c564690aa6a5
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdh.c
@@ -0,0 +1,947 @@
+/*
+ *  BCMSDH interface glue
+ *  implement bcmsdh API for SDIOH driver
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/**
+ * @file bcmsdh.c
+ */
+
+/* ****************** BCMSDH Interface Functions *************************** */
+
+#include <typedefs.h>
+#include <bcmdevs.h>
+#include <bcmendian.h>
+#include <bcmutils.h>
+#include <hndsoc.h>
+#include <siutils.h>
+#if !defined(BCMDONGLEHOST)
+#include <bcmsrom.h>
+#endif /* !defined(BCMDONGLEHOST) */
+#include <osl.h>
+
+#include <bcmsdh.h>	/* BRCM API for SDIO clients (such as wl, dhd) */
+#include <bcmsdbus.h>	/* common SDIO/controller interface */
+#include <sbsdio.h>	/* SDIO device core hardware definitions. */
+#include <sdio.h>	/* SDIO Device and Protocol Specs */
+
+#if defined(BT_OVER_SDIO)
+#include <dhd_bt_interface.h>
+#endif /* defined (BT_OVER_SDIO) */
+
+#define SDIOH_API_ACCESS_RETRY_LIMIT	2
+const uint bcmsdh_msglevel = BCMSDH_ERROR_VAL;
+
+/* local copy of bcm sd handler */
+bcmsdh_info_t * l_bcmsdh = NULL;
+
+#if defined(BT_OVER_SDIO)
+struct sdio_func *func_f3 = NULL;
+static f3intr_handler processf3intr = NULL;
+static dhd_hang_notification process_dhd_hang_notification = NULL;
+static dhd_hang_state_t g_dhd_hang_state = NO_HANG_STATE;
+#endif /* defined (BT_OVER_SDIO) */
+
+#if defined(NDIS) && (NDISVER < 0x0630)
+extern SDIOH_API_RC sdioh_detach(osl_t *osh, sdioh_info_t *sd);
+#endif
+
+#if defined(OOB_INTR_ONLY) && defined(HW_OOB) || defined(FORCE_WOWLAN)
+extern int
+sdioh_enable_hw_oob_intr(void *sdioh, bool enable);
+
+void
+bcmsdh_enable_hw_oob_intr(bcmsdh_info_t *sdh, bool enable)
+{
+	sdioh_enable_hw_oob_intr(sdh->sdioh, enable);
+}
+#endif
+
+#if defined(BT_OVER_SDIO)
+void bcmsdh_btsdio_process_hang_state(dhd_hang_state_t new_state)
+{
+	bool state_change = false;
+
+	BCMSDH_ERROR(("%s: DHD hang state changed - [%d] -> [%d]\n",
+		__FUNCTION__, g_dhd_hang_state, new_state));
+
+	if (g_dhd_hang_state == new_state)
+		return;
+
+	switch (g_dhd_hang_state) {
+		case NO_HANG_STATE:
+			if (HANG_START_STATE == new_state)
+				state_change = true;
+		break;
+
+		case HANG_START_STATE:
+			if (HANG_RECOVERY_STATE == new_state ||
+				NO_HANG_STATE == new_state)
+				state_change = true;
+
+		break;
+
+		case HANG_RECOVERY_STATE:
+			if (NO_HANG_STATE == new_state)
+				state_change = true;
+		break;
+
+		default:
+			BCMSDH_ERROR(("%s: Unhandled Hang state\n", __FUNCTION__));
+		break;
+	}
+
+	if (!state_change) {
+		BCMSDH_ERROR(("%s: Hang state cannot be changed\n", __FUNCTION__));
+		return;
+	}
+
+	g_dhd_hang_state = new_state;
+}
+
+void bcmsdh_btsdio_process_f3_intr(void)
+{
+	if (processf3intr && (g_dhd_hang_state == NO_HANG_STATE))
+		processf3intr(func_f3);
+}
+
+void bcmsdh_btsdio_process_dhd_hang_notification(bool wifi_recovery_completed)
+{
+	bcmsdh_btsdio_process_hang_state(HANG_START_STATE);
+
+	if (process_dhd_hang_notification)
+		process_dhd_hang_notification(func_f3, wifi_recovery_completed);
+
+	/* WiFi was off, so HANG_RECOVERY_STATE is not needed */
+	if (wifi_recovery_completed)
+		bcmsdh_btsdio_process_hang_state(NO_HANG_STATE);
+	else {
+		bcmsdh_btsdio_process_hang_state(HANG_RECOVERY_STATE);
+	}
+}
+
+void bcmsdh_btsdio_interface_init(struct sdio_func *func,
+	f3intr_handler f3intr_fun, dhd_hang_notification hang_notification)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)l_bcmsdh;
+	BCMSDH_INFO(("%s: func %p \n", __FUNCTION__, func));
+	func_f3 = func;
+	processf3intr = f3intr_fun;
+	sdioh_sdmmc_card_enable_func_f3(bcmsdh->sdioh, func);
+	process_dhd_hang_notification = hang_notification;
+
+} EXPORT_SYMBOL(bcmsdh_btsdio_interface_init);
+#endif /* defined (BT_OVER_SDIO) */
+
+/* Attach BCMSDH layer to SDIO Host Controller Driver
+ *
+ * @param osh OSL Handle.
+ * @param cfghdl Configuration Handle.
+ * @param regsva Virtual address of controller registers.
+ * @param irq Interrupt number of SDIO controller.
+ *
+ * @return bcmsdh_info_t Handle to BCMSDH context.
+ */
+bcmsdh_info_t *
+bcmsdh_attach(osl_t *osh, void *sdioh, ulong *regsva)
+{
+	bcmsdh_info_t *bcmsdh;
+
+	if ((bcmsdh = (bcmsdh_info_t *)MALLOC(osh, sizeof(bcmsdh_info_t))) == NULL) {
+		BCMSDH_ERROR(("bcmsdh_attach: out of memory, malloced %d bytes\n", MALLOCED(osh)));
+		return NULL;
+	}
+	bzero((char *)bcmsdh, sizeof(bcmsdh_info_t));
+	bcmsdh->sdioh = sdioh;
+	bcmsdh->osh = osh;
+	bcmsdh->init_success = TRUE;
+	*regsva = si_enum_base(0);
+
+	bcmsdh_force_sbwad_calc(bcmsdh, FALSE);
+
+	/* Report the BAR, to fix if needed */
+	bcmsdh->sbwad = si_enum_base(0);
+
+	/* save the handler locally */
+	l_bcmsdh = bcmsdh;
+
+	return bcmsdh;
+}
+
+int
+bcmsdh_detach(osl_t *osh, void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	if (bcmsdh != NULL) {
+#if defined(NDIS) && (NDISVER < 0x0630)
+		if (bcmsdh->sdioh)
+			sdioh_detach(osh, bcmsdh->sdioh);
+#endif
+		MFREE(osh, bcmsdh, sizeof(bcmsdh_info_t));
+	}
+
+	l_bcmsdh = NULL;
+
+	return 0;
+}
+
+int
+bcmsdh_iovar_op(void *sdh, const char *name,
+                void *params, uint plen, void *arg, uint len, bool set)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	return sdioh_iovar_op(bcmsdh->sdioh, name, params, plen, arg, len, set);
+}
+
+bool
+bcmsdh_intr_query(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+	bool on;
+
+	ASSERT(bcmsdh);
+	status = sdioh_interrupt_query(bcmsdh->sdioh, &on);
+	if (SDIOH_API_SUCCESS(status))
+		return FALSE;
+	else
+		return on;
+}
+
+int
+bcmsdh_intr_enable(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+#ifdef BCMSPI_ANDROID
+	uint32 data;
+#endif /* BCMSPI_ANDROID */
+	ASSERT(bcmsdh);
+
+	status = sdioh_interrupt_set(bcmsdh->sdioh, TRUE);
+#ifdef BCMSPI_ANDROID
+	data = bcmsdh_cfg_read_word(sdh, 0, 4, NULL);
+	data |= 0xE0E70000;
+	bcmsdh_cfg_write_word(sdh, 0, 4, data, NULL);
+#endif /* BCMSPI_ANDROID */
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdh_intr_disable(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+#ifdef BCMSPI_ANDROID
+	uint32 data;
+#endif /* BCMSPI_ANDROID */
+	ASSERT(bcmsdh);
+
+	status = sdioh_interrupt_set(bcmsdh->sdioh, FALSE);
+#ifdef BCMSPI_ANDROID
+	data = bcmsdh_cfg_read_word(sdh, 0, 4, NULL);
+	data &= ~0xE0E70000;
+	bcmsdh_cfg_write_word(sdh, 0, 4, data, NULL);
+#endif /* BCMSPI_ANDROID */
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdh_intr_reg(void *sdh, bcmsdh_cb_fn_t fn, void *argh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh);
+
+	status = sdioh_interrupt_register(bcmsdh->sdioh, fn, argh);
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdh_intr_dereg(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh);
+
+	status = sdioh_interrupt_deregister(bcmsdh->sdioh);
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+#if defined(DHD_DEBUG) || defined(BCMDBG)
+bool
+bcmsdh_intr_pending(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	ASSERT(sdh);
+	return sdioh_interrupt_pending(bcmsdh->sdioh);
+}
+#endif
+
+int
+bcmsdh_devremove_reg(void *sdh, bcmsdh_cb_fn_t fn, void *argh)
+{
+	ASSERT(sdh);
+
+	/* don't support yet */
+	return BCME_UNSUPPORTED;
+}
+
+/**
+ * Read from SDIO Configuration Space
+ * @param sdh SDIO Host context.
+ * @param func_num Function number to read from.
+ * @param addr Address to read from.
+ * @param err Error return.
+ * @return value read from SDIO configuration space.
+ */
+uint8
+bcmsdh_cfg_read(void *sdh, uint fnc_num, uint32 addr, int *err)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+#ifdef SDIOH_API_ACCESS_RETRY_LIMIT
+	int32 retry = 0;
+#endif
+	uint8 data = 0;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+
+#ifdef SDIOH_API_ACCESS_RETRY_LIMIT
+	do {
+		if (retry)	/* wait for 1 ms till bus get settled down */
+			OSL_DELAY(1000);
+#endif
+	status = sdioh_cfg_read(bcmsdh->sdioh, fnc_num, addr, (uint8 *)&data);
+#ifdef SDIOH_API_ACCESS_RETRY_LIMIT
+	} while (!SDIOH_API_SUCCESS(status) && (retry++ < SDIOH_API_ACCESS_RETRY_LIMIT));
+#endif
+	if (err)
+		*err = (SDIOH_API_SUCCESS(status) ? 0 : BCME_SDIO_ERROR);
+
+	BCMSDH_INFO(("%s:fun = %d, addr = 0x%x, uint8data = 0x%x\n", __FUNCTION__,
+	            fnc_num, addr, data));
+
+	return data;
+}
+
+void
+bcmsdh_cfg_write(void *sdh, uint fnc_num, uint32 addr, uint8 data, int *err)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+#ifdef SDIOH_API_ACCESS_RETRY_LIMIT
+	int32 retry = 0;
+#endif
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+
+#ifdef SDIOH_API_ACCESS_RETRY_LIMIT
+	do {
+		if (retry)	/* wait for 1 ms till bus get settled down */
+			OSL_DELAY(1000);
+#endif
+	status = sdioh_cfg_write(bcmsdh->sdioh, fnc_num, addr, (uint8 *)&data);
+#ifdef SDIOH_API_ACCESS_RETRY_LIMIT
+	} while (!SDIOH_API_SUCCESS(status) && (retry++ < SDIOH_API_ACCESS_RETRY_LIMIT));
+#endif
+	if (err)
+		*err = SDIOH_API_SUCCESS(status) ? 0 : BCME_SDIO_ERROR;
+
+	BCMSDH_INFO(("%s:fun = %d, addr = 0x%x, uint8data = 0x%x\n", __FUNCTION__,
+	            fnc_num, addr, data));
+}
+
+uint32
+bcmsdh_cfg_read_word(void *sdh, uint fnc_num, uint32 addr, int *err)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+	uint32 data = 0;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+
+	status = sdioh_request_word(bcmsdh->sdioh, SDIOH_CMD_TYPE_NORMAL, SDIOH_READ, fnc_num,
+	                            addr, &data, 4);
+
+	if (err)
+		*err = (SDIOH_API_SUCCESS(status) ? 0 : BCME_SDIO_ERROR);
+
+	BCMSDH_INFO(("%s:fun = %d, addr = 0x%x, uint32data = 0x%x\n", __FUNCTION__,
+	            fnc_num, addr, data));
+
+	return data;
+}
+
+void
+bcmsdh_cfg_write_word(void *sdh, uint fnc_num, uint32 addr, uint32 data, int *err)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+
+	status = sdioh_request_word(bcmsdh->sdioh, SDIOH_CMD_TYPE_NORMAL, SDIOH_WRITE, fnc_num,
+	                            addr, &data, 4);
+
+	if (err)
+		*err = (SDIOH_API_SUCCESS(status) ? 0 : BCME_SDIO_ERROR);
+
+	BCMSDH_INFO(("%s:fun = %d, addr = 0x%x, uint32data = 0x%x\n", __FUNCTION__, fnc_num,
+	             addr, data));
+}
+
+int
+bcmsdh_cis_read(void *sdh, uint func, uint8 *cis, uint length)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+
+	uint8 *tmp_buf, *tmp_ptr;
+	uint8 *ptr;
+	bool ascii = func & ~0xf;
+	func &= 0x7;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+	ASSERT(cis);
+	ASSERT(length <= SBSDIO_CIS_SIZE_LIMIT);
+
+	status = sdioh_cis_read(bcmsdh->sdioh, func, cis, length);
+
+	if (ascii) {
+		/* Move binary bits to tmp and format them into the provided buffer. */
+		if ((tmp_buf = (uint8 *)MALLOC(bcmsdh->osh, length)) == NULL) {
+			BCMSDH_ERROR(("%s: out of memory\n", __FUNCTION__));
+			return BCME_NOMEM;
+		}
+		bcopy(cis, tmp_buf, length);
+		for (tmp_ptr = tmp_buf, ptr = cis; ptr < (cis + length - 4); tmp_ptr++) {
+			ptr += snprintf((char*)ptr, (cis + length - ptr - 4),
+				"%.2x ", *tmp_ptr & 0xff);
+			if ((((tmp_ptr - tmp_buf) + 1) & 0xf) == 0)
+				ptr += snprintf((char *)ptr, (cis + length - ptr -4), "\n");
+		}
+		MFREE(bcmsdh->osh, tmp_buf, length);
+	}
+
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdh_cisaddr_read(void *sdh, uint func, uint8 *cisd, uint32 offset)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+
+	func &= 0x7;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+	ASSERT(cisd);
+
+	status = sdioh_cisaddr_read(bcmsdh->sdioh, func, cisd, offset);
+
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdhsdio_set_sbaddr_window(void *sdh, uint32 address, bool force_set)
+{
+	int err = 0;
+	uint bar0 = address & ~SBSDIO_SB_OFT_ADDR_MASK;
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	if (bar0 != bcmsdh->sbwad || force_set) {
+		bcmsdh_cfg_write(bcmsdh, SDIO_FUNC_1, SBSDIO_FUNC1_SBADDRLOW,
+			(address >> 8) & SBSDIO_SBADDRLOW_MASK, &err);
+		if (!err)
+			bcmsdh_cfg_write(bcmsdh, SDIO_FUNC_1, SBSDIO_FUNC1_SBADDRMID,
+				(address >> 16) & SBSDIO_SBADDRMID_MASK, &err);
+		if (!err)
+			bcmsdh_cfg_write(bcmsdh, SDIO_FUNC_1, SBSDIO_FUNC1_SBADDRHIGH,
+				(address >> 24) & SBSDIO_SBADDRHIGH_MASK, &err);
+
+		if (!err)
+			bcmsdh->sbwad = bar0;
+		else
+			/* invalidate cached window var */
+			bcmsdh->sbwad = 0;
+
+#ifdef BCMDBG
+		if (err)
+			BCMSDH_ERROR(("%s: error setting address window %08x\n",
+				__FUNCTION__, address));
+#endif /* BCMDBG */
+	}
+
+	return err;
+}
+
+uint32
+bcmsdh_reg_read(void *sdh, uintptr addr, uint size)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+	uint32 word = 0;
+
+	BCMSDH_INFO(("%s:fun = 1, addr = 0x%x\n", __FUNCTION__, (unsigned int)addr));
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+
+	if (bcmsdhsdio_set_sbaddr_window(bcmsdh, addr, bcmsdh->force_sbwad_calc)) {
+		bcmsdh->regfail = TRUE; // terence 20130621: prevent dhd_dpc in dead lock
+		return 0xFFFFFFFF;
+	}
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	if (size == 4)
+		addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	status = sdioh_request_word(bcmsdh->sdioh, SDIOH_CMD_TYPE_NORMAL,
+		SDIOH_READ, SDIO_FUNC_1, addr, &word, size);
+
+	bcmsdh->regfail = !(SDIOH_API_SUCCESS(status));
+
+	BCMSDH_INFO(("uint32data = 0x%x\n", word));
+
+	/* if ok, return appropriately masked word */
+	/* XXX Masking was put in for NDIS port, remove if not needed */
+	if (SDIOH_API_SUCCESS(status)) {
+		switch (size) {
+			case sizeof(uint8):
+				return (word & 0xff);
+			case sizeof(uint16):
+				return (word & 0xffff);
+			case sizeof(uint32):
+				return word;
+			default:
+				bcmsdh->regfail = TRUE;
+
+		}
+	}
+
+	/* otherwise, bad sdio access or invalid size */
+	BCMSDH_ERROR(("%s: error reading addr 0x%x size %d\n",
+		__FUNCTION__, (unsigned int)addr, size));
+	return 0xFFFFFFFF;
+}
+
+uint32
+bcmsdh_reg_write(void *sdh, uintptr addr, uint size, uint32 data)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+	int err = 0;
+
+	BCMSDH_INFO(("%s:fun = 1, addr = 0x%x, uint%ddata = 0x%x\n",
+	             __FUNCTION__, (unsigned int)addr, size*8, data));
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	ASSERT(bcmsdh->init_success);
+
+	if ((err = bcmsdhsdio_set_sbaddr_window(bcmsdh, addr, bcmsdh->force_sbwad_calc))) {
+		bcmsdh->regfail = TRUE; // terence 20130621:
+		return err;
+	}
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	if (size == 4)
+		addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+	status = sdioh_request_word(bcmsdh->sdioh, SDIOH_CMD_TYPE_NORMAL, SDIOH_WRITE, SDIO_FUNC_1,
+	                            addr, &data, size);
+	bcmsdh->regfail = !(SDIOH_API_SUCCESS(status));
+
+	if (SDIOH_API_SUCCESS(status))
+		return 0;
+
+	BCMSDH_ERROR(("%s: error writing 0x%08x to addr 0x%04x size %d\n",
+	              __FUNCTION__, data, (unsigned int)addr, size));
+	return 0xFFFFFFFF;
+}
+
+bool
+bcmsdh_regfail(void *sdh)
+{
+	return ((bcmsdh_info_t *)sdh)->regfail;
+}
+
+int
+bcmsdh_recv_buf(void *sdh, uint32 addr, uint fn, uint flags,
+                uint8 *buf, uint nbytes, void *pkt,
+                bcmsdh_cmplt_fn_t complete_fn, void *handle)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+	uint incr_fix;
+	uint width;
+	int err = 0;
+
+	ASSERT(bcmsdh);
+	ASSERT(bcmsdh->init_success);
+
+	BCMSDH_INFO(("%s:fun = %d, addr = 0x%x, size = %d\n",
+	             __FUNCTION__, fn, addr, nbytes));
+
+	/* Async not implemented yet */
+	ASSERT(!(flags & SDIO_REQ_ASYNC));
+	if (flags & SDIO_REQ_ASYNC)
+		return BCME_UNSUPPORTED;
+
+	if ((err = bcmsdhsdio_set_sbaddr_window(bcmsdh, addr, FALSE)))
+		return err;
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+
+	incr_fix = (flags & SDIO_REQ_FIXED) ? SDIOH_DATA_FIX : SDIOH_DATA_INC;
+	width = (flags & SDIO_REQ_4BYTE) ? 4 : 2;
+	if (width == 4)
+		addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	status = sdioh_request_buffer(bcmsdh->sdioh, SDIOH_DATA_PIO, incr_fix,
+	                              SDIOH_READ, fn, addr, width, nbytes, buf, pkt);
+
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_SDIO_ERROR);
+}
+
+int
+bcmsdh_send_buf(void *sdh, uint32 addr, uint fn, uint flags,
+                uint8 *buf, uint nbytes, void *pkt,
+                bcmsdh_cmplt_fn_t complete_fn, void *handle)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+	uint incr_fix;
+	uint width;
+	int err = 0;
+
+	ASSERT(bcmsdh);
+	ASSERT(bcmsdh->init_success);
+
+	BCMSDH_INFO(("%s:fun = %d, addr = 0x%x, size = %d\n",
+	            __FUNCTION__, fn, addr, nbytes));
+
+	/* Async not implemented yet */
+	ASSERT(!(flags & SDIO_REQ_ASYNC));
+	if (flags & SDIO_REQ_ASYNC)
+		return BCME_UNSUPPORTED;
+
+	if ((err = bcmsdhsdio_set_sbaddr_window(bcmsdh, addr, FALSE)))
+		return err;
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+
+	incr_fix = (flags & SDIO_REQ_FIXED) ? SDIOH_DATA_FIX : SDIOH_DATA_INC;
+	width = (flags & SDIO_REQ_4BYTE) ? 4 : 2;
+	if (width == 4)
+		addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	status = sdioh_request_buffer(bcmsdh->sdioh, SDIOH_DATA_PIO, incr_fix,
+	                              SDIOH_WRITE, fn, addr, width, nbytes, buf, pkt);
+
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdh_rwdata(void *sdh, uint rw, uint32 addr, uint8 *buf, uint nbytes)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	SDIOH_API_RC status;
+
+	ASSERT(bcmsdh);
+	ASSERT(bcmsdh->init_success);
+	ASSERT((addr & SBSDIO_SBWINDOW_MASK) == 0);
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	status = sdioh_request_buffer(bcmsdh->sdioh, SDIOH_DATA_PIO, SDIOH_DATA_INC,
+	                              (rw ? SDIOH_WRITE : SDIOH_READ), SDIO_FUNC_1,
+	                              addr, 4, nbytes, buf, NULL);
+
+	return (SDIOH_API_SUCCESS(status) ? 0 : BCME_ERROR);
+}
+
+int
+bcmsdh_abort(void *sdh, uint fn)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	return sdioh_abort(bcmsdh->sdioh, fn);
+}
+
+int
+bcmsdh_start(void *sdh, int stage)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	return sdioh_start(bcmsdh->sdioh, stage);
+}
+
+int
+bcmsdh_stop(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	return sdioh_stop(bcmsdh->sdioh);
+}
+
+int
+bcmsdh_waitlockfree(void *sdh)
+{
+#ifdef LINUX
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	return sdioh_waitlockfree(bcmsdh->sdioh);
+#else
+	return 0;
+#endif
+}
+
+int
+bcmsdh_query_device(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+#if defined(BCMDONGLEHOST)
+	bcmsdh->vendevid = (VENDOR_BROADCOM << 16) | 0;
+#else
+	uint8 *fn0cis[1];
+	int err;
+	char *vars;
+	uint varsz;
+	osl_t *osh = bcmsdh->osh;
+
+	bcmsdh->vendevid = ~(0);
+
+	if (!(fn0cis[0] = MALLOC(osh, SBSDIO_CIS_SIZE_LIMIT))) {
+		BCMSDH_ERROR(("%s: CIS malloc failed\n", __FUNCTION__));
+		return (bcmsdh->vendevid);
+	}
+
+	bzero(fn0cis[0], SBSDIO_CIS_SIZE_LIMIT);
+
+	if ((err = bcmsdh_cis_read(sdh, 0, fn0cis[0], SBSDIO_CIS_SIZE_LIMIT))) {
+		BCMSDH_ERROR(("%s: CIS read err %d, report unknown BRCM device\n",
+		              __FUNCTION__, err));
+		bcmsdh->vendevid = (VENDOR_BROADCOM << 16) | 0;
+		MFREE(osh, fn0cis[0], SBSDIO_CIS_SIZE_LIMIT);
+		return (bcmsdh->vendevid);
+	}
+
+	if (!err) {
+		if ((err = srom_parsecis(NULL, osh, fn0cis, 1, &vars, &varsz))) {
+			BCMSDH_ERROR(("%s: Error parsing CIS = %d\n", __FUNCTION__, err));
+		} else {
+			bcmsdh->vendevid = (getintvar(vars, "vendid") << 16) |
+			                    getintvar(vars, "devid");
+			MFREE(osh, vars, varsz);
+		}
+	}
+
+	MFREE(osh, fn0cis[0], SBSDIO_CIS_SIZE_LIMIT);
+#endif /* BCMDONGLEHOST */
+	return (bcmsdh->vendevid);
+}
+
+uint
+bcmsdh_query_iofnum(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	return (sdioh_query_iofnum(bcmsdh->sdioh));
+}
+
+int
+bcmsdh_reset(bcmsdh_info_t *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	return sdioh_sdio_reset(bcmsdh->sdioh);
+}
+
+/* XXX For use by NDIS port, remove if not needed. */
+void *bcmsdh_get_sdioh(bcmsdh_info_t *sdh)
+{
+	ASSERT(sdh);
+	return sdh->sdioh;
+}
+
+/* Function to pass device-status bits to DHD. */
+uint32
+bcmsdh_get_dstatus(void *sdh)
+{
+#ifdef BCMSPI
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+	return sdioh_get_dstatus(sd);
+#else
+	return 0;
+#endif /* BCMSPI */
+}
+uint32
+bcmsdh_cur_sbwad(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+
+	return (bcmsdh->sbwad);
+}
+
+/* example usage: if force is TRUE, forces the bcmsdhsdio_set_sbaddr_window to
+ * calculate sbwad always instead of caching.
+ */
+void
+bcmsdh_force_sbwad_calc(void *sdh, bool force)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	if (!bcmsdh)
+		bcmsdh = l_bcmsdh;
+	bcmsdh->force_sbwad_calc = force;
+}
+
+void
+bcmsdh_chipinfo(void *sdh, uint32 chip, uint32 chiprev)
+{
+#ifdef BCMSPI
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+	sdioh_chipinfo(sd, chip, chiprev);
+#else
+	return;
+#endif /* BCMSPI */
+}
+
+#ifdef BCMSPI
+void
+bcmsdh_dwordmode(void *sdh, bool set)
+{
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+	sdioh_dwordmode(sd, set);
+	return;
+}
+#endif /* BCMSPI */
+
+int
+bcmsdh_sleep(void *sdh, bool enab)
+{
+#ifdef SDIOH_SLEEP_ENABLED
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+
+	return sdioh_sleep(sd, enab);
+#else
+	return BCME_UNSUPPORTED;
+#endif
+}
+
+int
+bcmsdh_gpio_init(void *sdh)
+{
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+
+	return sdioh_gpio_init(sd);
+}
+
+bool
+bcmsdh_gpioin(void *sdh, uint32 gpio)
+{
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+
+	return sdioh_gpioin(sd, gpio);
+}
+
+int
+bcmsdh_gpioouten(void *sdh, uint32 gpio)
+{
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+
+	return sdioh_gpioouten(sd, gpio);
+}
+
+int
+bcmsdh_gpioout(void *sdh, uint32 gpio, bool enab)
+{
+	bcmsdh_info_t *p = (bcmsdh_info_t *)sdh;
+	sdioh_info_t *sd = (sdioh_info_t *)(p->sdioh);
+
+	return sdioh_gpioout(sd, gpio, enab);
+}
+
+uint
+bcmsdh_set_mode(void *sdh, uint mode)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+
+	return (sdioh_set_mode(bcmsdh->sdioh, mode));
+}
+
+#ifdef PKT_STATICS
+uint32
+bcmsdh_get_spend_time(void *sdh)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)sdh;
+	return (sdioh_get_spend_time(bcmsdh->sdioh));
+}
+#endif
diff --git a/drivers/net/wireless/ap12275/bcmsdh_linux.c b/drivers/net/wireless/ap12275/bcmsdh_linux.c
new file mode 100644
index 000000000000..521141b1999d
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdh_linux.c
@@ -0,0 +1,586 @@
+/*
+ * SDIO access interface for drivers - linux specific (pci only)
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/**
+ * @file bcmsdh_linux.c
+ */
+
+#define __UNDEF_NO_VERSION__
+
+#include <typedefs.h>
+#include <linuxver.h>
+#include <linux/pci.h>
+#include <linux/completion.h>
+
+#include <osl.h>
+#include <pcicfg.h>
+#include <bcmdefs.h>
+#include <bcmdevs.h>
+#include <linux/irq.h>
+extern void dhdsdio_isr(void * args);
+#include <bcmutils.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#if defined(CONFIG_ARCH_ODIN)
+#include <linux/platform_data/gpio-odin.h>
+#endif /* defined(CONFIG_ARCH_ODIN) */
+#include <dhd_linux.h>
+
+/* driver info, initialized when bcmsdh_register is called */
+static bcmsdh_driver_t drvinfo = {NULL, NULL, NULL, NULL};
+
+typedef enum {
+	DHD_INTR_INVALID = 0,
+	DHD_INTR_INBAND,
+	DHD_INTR_HWOOB,
+	DHD_INTR_SWOOB
+} DHD_HOST_INTR_TYPE;
+
+/* the BCMSDH module comprises the generic part (bcmsdh.c) and OS specific layer (e.g.
+ * bcmsdh_linux.c). Put all OS specific variables (e.g. irq number and flags) here rather
+ * than in the common structure bcmsdh_info. bcmsdh_info only keeps a handle (os_ctx) to this
+ * structure.
+ */
+typedef struct bcmsdh_os_info {
+	DHD_HOST_INTR_TYPE	intr_type;
+	int			oob_irq_num;	/* valid when hardware or software oob in use */
+	unsigned long		oob_irq_flags;	/* valid when hardware or software oob in use */
+	bool			oob_irq_registered;
+	bool			oob_irq_enabled;
+	bool			oob_irq_wake_enabled;
+	spinlock_t		oob_irq_spinlock;
+	bcmsdh_cb_fn_t		oob_irq_handler;
+	void			*oob_irq_handler_context;
+	void			*context;	/* context returned from upper layer */
+	void			*sdioh;		/* handle to lower layer (sdioh) */
+	void			*dev;		/* handle to the underlying device */
+	bool			dev_wake_enabled;
+} bcmsdh_os_info_t;
+
+#ifndef ENODEBUG
+/* debugging macros */
+#ifdef BCMDBG_ERR
+#define SDLX_ERR(x) printf x
+#define SDLX_MSG(x)	printf x
+#else
+#define SDLX_ERR(x) printf x
+#define SDLX_MSG(x)	printf x
+#endif /* BCMDBG_ERR */
+#else
+#define SDLX_MSG(x)
+#define SDLX_ERR(x) printf x
+#endif
+/**
+ * Checks to see if vendor and device IDs match a supported SDIO Host Controller.
+ */
+bool
+bcmsdh_chipmatch(uint16 vendor, uint16 device)
+{
+	/* Add other vendors and devices as required */
+
+#ifdef BCMSDIOH_STD
+	/* Check for Arasan host controller */
+	if (vendor == VENDOR_SI_IMAGE) {
+		return (TRUE);
+	}
+	/* Check for BRCM 27XX Standard host controller */
+	if (device == BCM27XX_SDIOH_ID && vendor == VENDOR_BROADCOM) {
+		return (TRUE);
+	}
+	/* Check for BRCM Standard host controller */
+	if (device == SDIOH_FPGA_ID && vendor == VENDOR_BROADCOM) {
+		return (TRUE);
+	}
+	/* Check for TI PCIxx21 Standard host controller */
+	if (device == PCIXX21_SDIOH_ID && vendor == VENDOR_TI) {
+		return (TRUE);
+	}
+	if (device == PCIXX21_SDIOH0_ID && vendor == VENDOR_TI) {
+		return (TRUE);
+	}
+	/* Ricoh R5C822 Standard SDIO Host */
+	if (device == R5C822_SDIOH_ID && vendor == VENDOR_RICOH) {
+		return (TRUE);
+	}
+	/* JMicron Standard SDIO Host */
+	if (device == JMICRON_SDIOH_ID && vendor == VENDOR_JMICRON) {
+		return (TRUE);
+	}
+
+#endif /* BCMSDIOH_STD */
+#ifdef BCMSDIOH_SPI
+	/* This is the PciSpiHost. */
+	if (device == SPIH_FPGA_ID && vendor == VENDOR_BROADCOM) {
+		printf("Found PCI SPI Host Controller\n");
+		return (TRUE);
+	}
+
+#endif /* BCMSDIOH_SPI */
+
+	return (FALSE);
+}
+
+void* bcmsdh_probe(osl_t *osh, void *dev, void *sdioh, void *adapter_info, uint bus_type,
+	uint bus_num, uint slot_num)
+{
+	ulong regs;
+	bcmsdh_info_t *bcmsdh;
+	uint32 vendevid;
+	bcmsdh_os_info_t *bcmsdh_osinfo = NULL;
+
+	bcmsdh = bcmsdh_attach(osh, sdioh, &regs);
+	if (bcmsdh == NULL) {
+		SDLX_ERR(("%s: bcmsdh_attach failed\n", __FUNCTION__));
+		goto err;
+	}
+	bcmsdh_osinfo = MALLOC(osh, sizeof(bcmsdh_os_info_t));
+	if (bcmsdh_osinfo == NULL) {
+		SDLX_ERR(("%s: failed to allocate bcmsdh_os_info_t\n", __FUNCTION__));
+		goto err;
+	}
+	bzero((char *)bcmsdh_osinfo, sizeof(bcmsdh_os_info_t));
+	bcmsdh->os_cxt = bcmsdh_osinfo;
+	bcmsdh_osinfo->sdioh = sdioh;
+	bcmsdh_osinfo->dev = dev;
+	osl_set_bus_handle(osh, bcmsdh);
+
+#if !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36))
+	if (dev && device_init_wakeup(dev, true) == 0)
+		bcmsdh_osinfo->dev_wake_enabled = TRUE;
+#endif /* !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36)) */
+
+#if defined(OOB_INTR_ONLY)
+	spin_lock_init(&bcmsdh_osinfo->oob_irq_spinlock);
+	/* Get customer specific OOB IRQ parametres: IRQ number as IRQ type */
+	bcmsdh_osinfo->oob_irq_num = wifi_platform_get_irq_number(adapter_info,
+		&bcmsdh_osinfo->oob_irq_flags);
+	if  (bcmsdh_osinfo->oob_irq_num < 0) {
+		SDLX_ERR(("%s: Host OOB irq is not defined\n", __FUNCTION__));
+		goto err;
+	}
+#endif /* defined(BCMLXSDMMC) */
+
+	/* Read the vendor/device ID from the CIS */
+	vendevid = bcmsdh_query_device(bcmsdh);
+	/* try to attach to the target device */
+	bcmsdh_osinfo->context = drvinfo.probe((vendevid >> 16), (vendevid & 0xFFFF), bus_num,
+		slot_num, 0, bus_type, (void *)regs, osh, bcmsdh);
+	if (bcmsdh_osinfo->context == NULL) {
+		SDLX_ERR(("%s: device attach failed\n", __FUNCTION__));
+		goto err;
+	}
+
+	return bcmsdh;
+
+	/* error handling */
+err:
+	if (bcmsdh != NULL)
+		bcmsdh_detach(osh, bcmsdh);
+	if (bcmsdh_osinfo != NULL)
+		MFREE(osh, bcmsdh_osinfo, sizeof(bcmsdh_os_info_t));
+	return NULL;
+}
+
+int bcmsdh_remove(bcmsdh_info_t *bcmsdh)
+{
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+#if !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36))
+	if (bcmsdh_osinfo->dev)
+		device_init_wakeup(bcmsdh_osinfo->dev, false);
+	bcmsdh_osinfo->dev_wake_enabled = FALSE;
+#endif /* !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36)) */
+
+	drvinfo.remove(bcmsdh_osinfo->context);
+	MFREE(bcmsdh->osh, bcmsdh->os_cxt, sizeof(bcmsdh_os_info_t));
+	bcmsdh_detach(bcmsdh->osh, bcmsdh);
+
+	return 0;
+}
+
+#ifdef DHD_WAKE_STATUS
+int bcmsdh_get_total_wake(bcmsdh_info_t *bcmsdh)
+{
+	return bcmsdh->total_wake_count;
+}
+
+int bcmsdh_set_get_wake(bcmsdh_info_t *bcmsdh, int flag)
+{
+#if defined(OOB_INTR_ONLY)
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+	unsigned long flags;
+#endif
+	int ret;
+
+#if defined(OOB_INTR_ONLY)
+	spin_lock_irqsave(&bcmsdh_osinfo->oob_irq_spinlock, flags);
+#endif
+
+	ret = bcmsdh->pkt_wake;
+	bcmsdh->total_wake_count += flag;
+	bcmsdh->pkt_wake = flag;
+
+#if defined(OOB_INTR_ONLY)
+	spin_unlock_irqrestore(&bcmsdh_osinfo->oob_irq_spinlock, flags);
+#endif
+	return ret;
+}
+
+int bcmsdh_get_wake(bcmsdh_info_t *bcmsdh)
+{
+#if defined(OOB_INTR_ONLY)
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+	unsigned long flags;
+#endif
+	int ret;
+
+#if defined(OOB_INTR_ONLY)
+	spin_lock_irqsave(&bcmsdh_osinfo->oob_irq_spinlock, flags);
+#endif
+
+	ret = bcmsdh->pkt_wake;
+
+#if defined(OOB_INTR_ONLY)
+	spin_unlock_irqrestore(&bcmsdh_osinfo->oob_irq_spinlock, flags);
+#endif
+	return ret;
+}
+#endif /* DHD_WAKE_STATUS */
+
+int bcmsdh_suspend(bcmsdh_info_t *bcmsdh)
+{
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+	if (drvinfo.suspend && drvinfo.suspend(bcmsdh_osinfo->context))
+		return -EBUSY;
+	return 0;
+}
+
+int bcmsdh_resume(bcmsdh_info_t *bcmsdh)
+{
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+	if (drvinfo.resume)
+		return drvinfo.resume(bcmsdh_osinfo->context);
+	return 0;
+}
+
+extern int bcmsdh_register_client_driver(void);
+extern void bcmsdh_unregister_client_driver(void);
+extern int sdio_func_reg_notify(void* semaphore);
+extern void sdio_func_unreg_notify(void);
+
+#if defined(BCMLXSDMMC)
+int bcmsdh_reg_sdio_notify(void* semaphore)
+{
+	return sdio_func_reg_notify(semaphore);
+}
+
+void bcmsdh_unreg_sdio_notify(void)
+{
+	sdio_func_unreg_notify();
+}
+#endif /* defined(BCMLXSDMMC) */
+
+int
+bcmsdh_register(bcmsdh_driver_t *driver)
+{
+	int error = 0;
+
+	drvinfo = *driver;
+	SDLX_MSG(("%s: register client driver\n", __FUNCTION__));
+	error = bcmsdh_register_client_driver();
+	if (error) {
+		SDLX_MSG(("%s: failed %d\n", __FUNCTION__, error));
+		bcmsdh_unregister_client_driver();
+		memset(&drvinfo, 0, sizeof(drvinfo));
+	}
+
+	return error;
+}
+
+void
+bcmsdh_unregister(void)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0))
+		if (bcmsdh_pci_driver.node.next == NULL)
+			return;
+#endif
+
+	bcmsdh_unregister_client_driver();
+}
+
+void *bcmsdh_get_dev(bcmsdh_info_t *sdh)
+{
+	bcmsdh_os_info_t *bcmsdh_osinfo = sdh->os_cxt;
+	return bcmsdh_osinfo->dev;
+}
+
+void bcmsdh_dev_pm_stay_awake(bcmsdh_info_t *bcmsdh)
+{
+#if !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36))
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+	pm_stay_awake(bcmsdh_osinfo->dev);
+#endif /* !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36)) */
+}
+
+void bcmsdh_dev_relax(bcmsdh_info_t *bcmsdh)
+{
+#if !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36))
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+	pm_relax(bcmsdh_osinfo->dev);
+#endif /* !defined(CONFIG_HAS_WAKELOCK) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36)) */
+}
+
+bool bcmsdh_dev_pm_enabled(bcmsdh_info_t *bcmsdh)
+{
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+	return bcmsdh_osinfo->dev_wake_enabled;
+}
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+int bcmsdh_get_oob_intr_num(bcmsdh_info_t *bcmsdh)
+{
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+	return bcmsdh_osinfo->oob_irq_num;
+}
+
+void bcmsdh_oob_intr_set(bcmsdh_info_t *bcmsdh, bool enable)
+{
+	unsigned long flags;
+	bcmsdh_os_info_t *bcmsdh_osinfo;
+
+	if (!bcmsdh)
+		return;
+
+	bcmsdh_osinfo = bcmsdh->os_cxt;
+	spin_lock_irqsave(&bcmsdh_osinfo->oob_irq_spinlock, flags);
+	if (bcmsdh_osinfo->oob_irq_enabled != enable) {
+		if (enable)
+			enable_irq(bcmsdh_osinfo->oob_irq_num);
+		else
+			disable_irq_nosync(bcmsdh_osinfo->oob_irq_num);
+		bcmsdh_osinfo->oob_irq_enabled = enable;
+	}
+	spin_unlock_irqrestore(&bcmsdh_osinfo->oob_irq_spinlock, flags);
+}
+
+#ifdef ENABLE_WAKEUP_PKT_DUMP
+extern volatile bool dhd_mmc_suspend;
+extern volatile bool dhd_mmc_wake;
+#endif /* ENABLE_WAKEUP_PKT_DUMP */
+
+static irqreturn_t wlan_oob_irq(int irq, void *dev_id)
+{
+	bcmsdh_info_t *bcmsdh = (bcmsdh_info_t *)dev_id;
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+#ifndef BCMSPI_ANDROID
+	bcmsdh_oob_intr_set(bcmsdh, FALSE);
+#endif /* !BCMSPI_ANDROID */
+	bcmsdh_osinfo->oob_irq_handler(bcmsdh_osinfo->oob_irq_handler_context);
+
+#ifdef ENABLE_WAKEUP_PKT_DUMP
+	if (dhd_mmc_suspend) {
+		dhd_mmc_wake = TRUE;
+	}
+#endif /* ENABLE_WAKEUP_PKT_DUMP */
+
+	return IRQ_HANDLED;
+}
+
+int bcmsdh_oob_intr_register(bcmsdh_info_t *bcmsdh, bcmsdh_cb_fn_t oob_irq_handler,
+	void* oob_irq_handler_context)
+{
+	int err = 0;
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+	if (bcmsdh_osinfo->oob_irq_registered) {
+		SDLX_ERR(("%s: irq is already registered\n", __FUNCTION__));
+		return -EBUSY;
+	}
+#ifdef HW_OOB
+	SDLX_MSG(("%s: HW_OOB irq=%d flags=0x%X\n", __FUNCTION__,
+		(int)bcmsdh_osinfo->oob_irq_num, (int)bcmsdh_osinfo->oob_irq_flags));
+#else
+	SDLX_MSG(("%s: SW_OOB irq=%d flags=0x%X\n", __FUNCTION__,
+		(int)bcmsdh_osinfo->oob_irq_num, (int)bcmsdh_osinfo->oob_irq_flags));
+#endif
+	bcmsdh_osinfo->oob_irq_handler = oob_irq_handler;
+	bcmsdh_osinfo->oob_irq_handler_context = oob_irq_handler_context;
+	bcmsdh_osinfo->oob_irq_enabled = TRUE;
+	bcmsdh_osinfo->oob_irq_registered = TRUE;
+#if defined(CONFIG_ARCH_ODIN)
+	err = odin_gpio_sms_request_irq(bcmsdh_osinfo->oob_irq_num, wlan_oob_irq,
+		bcmsdh_osinfo->oob_irq_flags, "bcmsdh_sdmmc", bcmsdh);
+#else
+	err = request_irq(bcmsdh_osinfo->oob_irq_num, wlan_oob_irq,
+		bcmsdh_osinfo->oob_irq_flags, "bcmsdh_sdmmc", bcmsdh);
+#endif /* defined(CONFIG_ARCH_ODIN) */
+	if (err) {
+		SDLX_ERR(("%s: request_irq failed with %d\n", __FUNCTION__, err));
+		bcmsdh_osinfo->oob_irq_enabled = FALSE;
+		bcmsdh_osinfo->oob_irq_registered = FALSE;
+		return err;
+	}
+
+#if defined(DISABLE_WOWLAN)
+	SDLX_MSG(("%s: disable_irq_wake\n", __FUNCTION__));
+	bcmsdh_osinfo->oob_irq_wake_enabled = FALSE;
+#else
+#if defined(CONFIG_ARCH_RHEA) || defined(CONFIG_ARCH_CAPRI)
+	if (device_may_wakeup(bcmsdh_osinfo->dev)) {
+#endif /* CONFIG_ARCH_RHEA || CONFIG_ARCH_CAPRI */
+		err = enable_irq_wake(bcmsdh_osinfo->oob_irq_num);
+		if (err)
+			SDLX_ERR(("%s: enable_irq_wake failed with %d\n", __FUNCTION__, err));
+		else
+			bcmsdh_osinfo->oob_irq_wake_enabled = TRUE;
+#if defined(CONFIG_ARCH_RHEA) || defined(CONFIG_ARCH_CAPRI)
+	}
+#endif /* CONFIG_ARCH_RHEA || CONFIG_ARCH_CAPRI */
+#endif
+
+	return 0;
+}
+
+void bcmsdh_oob_intr_unregister(bcmsdh_info_t *bcmsdh)
+{
+	int err = 0;
+	bcmsdh_os_info_t *bcmsdh_osinfo = bcmsdh->os_cxt;
+
+	SDLX_MSG(("%s: Enter\n", __FUNCTION__));
+	if (!bcmsdh_osinfo->oob_irq_registered) {
+		SDLX_MSG(("%s: irq is not registered\n", __FUNCTION__));
+		return;
+	}
+	if (bcmsdh_osinfo->oob_irq_wake_enabled) {
+#if defined(CONFIG_ARCH_RHEA) || defined(CONFIG_ARCH_CAPRI)
+		if (device_may_wakeup(bcmsdh_osinfo->dev)) {
+#endif /* CONFIG_ARCH_RHEA || CONFIG_ARCH_CAPRI */
+			err = disable_irq_wake(bcmsdh_osinfo->oob_irq_num);
+			if (!err)
+				bcmsdh_osinfo->oob_irq_wake_enabled = FALSE;
+#if defined(CONFIG_ARCH_RHEA) || defined(CONFIG_ARCH_CAPRI)
+		}
+#endif /* CONFIG_ARCH_RHEA || CONFIG_ARCH_CAPRI */
+	}
+	if (bcmsdh_osinfo->oob_irq_enabled) {
+		disable_irq(bcmsdh_osinfo->oob_irq_num);
+		bcmsdh_osinfo->oob_irq_enabled = FALSE;
+	}
+	free_irq(bcmsdh_osinfo->oob_irq_num, bcmsdh);
+	bcmsdh_osinfo->oob_irq_registered = FALSE;
+}
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+
+/* Module parameters specific to each host-controller driver */
+/* XXX Need to move these to where they really belong! */
+
+extern uint sd_msglevel;	/* Debug message level */
+module_param(sd_msglevel, uint, 0);
+
+extern uint sd_power;	/* 0 = SD Power OFF, 1 = SD Power ON. */
+module_param(sd_power, uint, 0);
+
+extern uint sd_clock;	/* SD Clock Control, 0 = SD Clock OFF, 1 = SD Clock ON */
+module_param(sd_clock, uint, 0);
+
+extern uint sd_divisor;	/* Divisor (-1 means external clock) */
+module_param(sd_divisor, uint, 0);
+
+extern uint sd_sdmode;	/* Default is SD4, 0=SPI, 1=SD1, 2=SD4 */
+module_param(sd_sdmode, uint, 0);
+
+extern uint sd_hiok;	/* Ok to use hi-speed mode */
+module_param(sd_hiok, uint, 0);
+
+extern uint sd_f2_blocksize;
+module_param(sd_f2_blocksize, int, 0);
+
+extern uint sd_f1_blocksize;
+module_param(sd_f1_blocksize, int, 0);
+
+#ifdef BCMSDIOH_STD
+extern int sd_uhsimode;
+module_param(sd_uhsimode, int, 0);
+extern uint sd_tuning_period;
+module_param(sd_tuning_period, uint, 0);
+extern int sd_delay_value;
+module_param(sd_delay_value, uint, 0);
+
+/* SDIO Drive Strength for UHSI mode specific to SDIO3.0 */
+extern char dhd_sdiod_uhsi_ds_override[2];
+module_param_string(dhd_sdiod_uhsi_ds_override, dhd_sdiod_uhsi_ds_override, 2, 0);
+
+#endif
+
+#ifdef BCMSDH_MODULE
+EXPORT_SYMBOL(bcmsdh_attach);
+EXPORT_SYMBOL(bcmsdh_detach);
+EXPORT_SYMBOL(bcmsdh_intr_query);
+EXPORT_SYMBOL(bcmsdh_intr_enable);
+EXPORT_SYMBOL(bcmsdh_intr_disable);
+EXPORT_SYMBOL(bcmsdh_intr_reg);
+EXPORT_SYMBOL(bcmsdh_intr_dereg);
+
+#if defined(DHD_DEBUG) || defined(BCMDBG)
+EXPORT_SYMBOL(bcmsdh_intr_pending);
+#endif
+
+#if defined(BT_OVER_SDIO)
+EXPORT_SYMBOL(bcmsdh_btsdio_interface_init);
+#endif /* defined (BT_OVER_SDIO) */
+
+EXPORT_SYMBOL(bcmsdh_devremove_reg);
+EXPORT_SYMBOL(bcmsdh_cfg_read);
+EXPORT_SYMBOL(bcmsdh_cfg_write);
+EXPORT_SYMBOL(bcmsdh_cis_read);
+EXPORT_SYMBOL(bcmsdh_reg_read);
+EXPORT_SYMBOL(bcmsdh_reg_write);
+EXPORT_SYMBOL(bcmsdh_regfail);
+EXPORT_SYMBOL(bcmsdh_send_buf);
+EXPORT_SYMBOL(bcmsdh_recv_buf);
+
+EXPORT_SYMBOL(bcmsdh_rwdata);
+EXPORT_SYMBOL(bcmsdh_abort);
+EXPORT_SYMBOL(bcmsdh_query_device);
+EXPORT_SYMBOL(bcmsdh_query_iofnum);
+EXPORT_SYMBOL(bcmsdh_iovar_op);
+EXPORT_SYMBOL(bcmsdh_register);
+EXPORT_SYMBOL(bcmsdh_unregister);
+EXPORT_SYMBOL(bcmsdh_chipmatch);
+EXPORT_SYMBOL(bcmsdh_reset);
+EXPORT_SYMBOL(bcmsdh_waitlockfree);
+
+EXPORT_SYMBOL(bcmsdh_get_dstatus);
+EXPORT_SYMBOL(bcmsdh_cfg_read_word);
+EXPORT_SYMBOL(bcmsdh_cfg_write_word);
+EXPORT_SYMBOL(bcmsdh_cur_sbwad);
+EXPORT_SYMBOL(bcmsdh_chipinfo);
+
+#endif /* BCMSDH_MODULE */
diff --git a/drivers/net/wireless/ap12275/bcmsdh_sdmmc.c b/drivers/net/wireless/ap12275/bcmsdh_sdmmc.c
new file mode 100644
index 000000000000..4acc1c25a57d
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdh_sdmmc.c
@@ -0,0 +1,2079 @@
+/*
+ * BCMSDH Function Driver for the native SDIO/MMC driver in the Linux Kernel
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <typedefs.h>
+
+#include <bcmdevs.h>
+#include <bcmendian.h>
+#include <bcmutils.h>
+#include <osl.h>
+#include <sdio.h>	/* SDIO Device and Protocol Specs */
+#include <sdioh.h>	/* Standard SDIO Host Controller Specification */
+#include <bcmsdbus.h>	/* bcmsdh to/from specific controller APIs */
+#include <sdiovar.h>	/* ioctl/iovars */
+
+#include <linux/mmc/core.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/card.h>
+#include <linux/mmc/sdio_func.h>
+#include <linux/mmc/sdio_ids.h>
+
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && defined(CONFIG_PM_SLEEP)
+#include <linux/suspend.h>
+extern volatile bool dhd_mmc_suspend;
+#endif
+#include "bcmsdh_sdmmc.h"
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)) || \
+	(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
+static inline void
+mmc_host_clk_hold(struct mmc_host *host)
+{
+	BCM_REFERENCE(host);
+	return;
+}
+
+static inline void
+mmc_host_clk_release(struct mmc_host *host)
+{
+	BCM_REFERENCE(host);
+	return;
+}
+
+static inline unsigned int
+mmc_host_clk_rate(struct mmc_host *host)
+{
+	return host->ios.clock;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0) */
+
+#ifndef BCMSDH_MODULE
+extern int sdio_function_init(void);
+extern void sdio_function_cleanup(void);
+#endif /* BCMSDH_MODULE */
+
+#if !defined(OOB_INTR_ONLY)
+static void IRQHandler(struct sdio_func *func);
+static void IRQHandlerF2(struct sdio_func *func);
+#endif /* !defined(OOB_INTR_ONLY) */
+static int sdioh_sdmmc_get_cisaddr(sdioh_info_t *sd, uint32 regaddr);
+#if defined(ENABLE_INSMOD_NO_FW_LOAD)
+#if defined(MMC_SW_RESET) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
+extern int mmc_sw_reset(struct mmc_card *card);
+#else
+extern int mmc_sw_reset(struct mmc_host *host);
+#endif
+#elif defined(MMC_HW_RESET) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+extern int mmc_hw_reset(struct mmc_card *card);
+#else
+extern int mmc_hw_reset(struct mmc_host *host);
+#endif
+#elif defined(BUS_POWER_RESTORE) && \
+LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
+#undef MMC_SW_RESET
+#undef MMC_HW_RESET
+#else
+extern int sdio_reset_comm(struct mmc_card *card);
+#endif
+#endif
+#ifdef GLOBAL_SDMMC_INSTANCE
+extern PBCMSDH_SDMMC_INSTANCE gInstance;
+#endif
+
+#define DEFAULT_SDIO_F2_BLKSIZE		512
+#ifndef CUSTOM_SDIO_F2_BLKSIZE
+#define CUSTOM_SDIO_F2_BLKSIZE		DEFAULT_SDIO_F2_BLKSIZE
+#endif
+
+#define DEFAULT_SDIO_F1_BLKSIZE		64
+#ifndef CUSTOM_SDIO_F1_BLKSIZE
+#define CUSTOM_SDIO_F1_BLKSIZE		DEFAULT_SDIO_F1_BLKSIZE
+#endif
+
+#define COPY_BUF_SIZE	(SDPCM_MAXGLOM_SIZE * 1600)
+
+#define MAX_IO_RW_EXTENDED_BLK		511
+
+uint sd_sdmode = SDIOH_MODE_SD4;	/* Use SD4 mode by default */
+uint sd_f2_blocksize = CUSTOM_SDIO_F2_BLKSIZE;
+uint sd_f1_blocksize = CUSTOM_SDIO_F1_BLKSIZE;
+
+#if defined(BT_OVER_SDIO)
+uint sd_f3_blocksize = 64;
+#endif /* defined (BT_OVER_SDIO) */
+
+uint sd_divisor = 2;			/* Default 48MHz/2 = 24MHz */
+
+uint sd_power = 1;		/* Default to SD Slot powered ON */
+uint sd_clock = 1;		/* Default to SD Clock turned ON */
+uint sd_hiok = FALSE;	/* Don't use hi-speed mode by default */
+uint sd_msglevel = SDH_ERROR_VAL;
+uint sd_use_dma = TRUE;
+
+#ifndef CUSTOM_RXCHAIN
+#define CUSTOM_RXCHAIN 0
+#endif
+
+DHD_PM_RESUME_WAIT_INIT(sdioh_request_byte_wait);
+DHD_PM_RESUME_WAIT_INIT(sdioh_request_word_wait);
+DHD_PM_RESUME_WAIT_INIT(sdioh_request_packet_wait);
+DHD_PM_RESUME_WAIT_INIT(sdioh_request_buffer_wait);
+
+#define DMA_ALIGN_MASK	0x03
+#define MMC_SDIO_ABORT_RETRY_LIMIT 5
+
+int sdioh_sdmmc_card_regread(sdioh_info_t *sd, int func, uint32 regaddr, int regsize, uint32 *data);
+#ifdef NOTYET
+static int
+sdioh_sdmmc_card_regwrite(sdioh_info_t *sd, int func, uint32 regaddr, int regsize, uint32 data);
+#endif /* NOTYET */
+
+#if defined(BT_OVER_SDIO)
+extern
+void sdioh_sdmmc_card_enable_func_f3(sdioh_info_t *sd, struct sdio_func *func)
+{
+	sd->func[3] = func;
+	sd_info(("%s sd->func[3] %p\n", __FUNCTION__, sd->func[3]));
+}
+#endif /* defined (BT_OVER_SDIO) */
+
+void  sdmmc_set_timing(sdioh_info_t *sd, uint8 timing);
+uint8 sdmmc_get_timing(sdioh_info_t *sd);
+void  sdmmc_set_clock_rate(sdioh_info_t *sd, uint hz);
+uint  sdmmc_get_clock_rate(sdioh_info_t *sd);
+void  sdmmc_set_clock_divisor(sdioh_info_t *sd, uint sd_div);
+
+static int
+sdioh_sdmmc_card_enablefuncs(sdioh_info_t *sd)
+{
+	int err_ret;
+	uint32 fbraddr;
+	uint8 func;
+
+	sd_trace(("%s\n", __FUNCTION__));
+
+	/* Get the Card's common CIS address */
+	sd->com_cis_ptr = sdioh_sdmmc_get_cisaddr(sd, SDIOD_CCCR_CISPTR_0);
+	sd->func_cis_ptr[0] = sd->com_cis_ptr;
+	sd_info(("%s: Card's Common CIS Ptr = 0x%x\n", __FUNCTION__, sd->com_cis_ptr));
+
+	/* Get the Card's function CIS (for each function) */
+	for (fbraddr = SDIOD_FBR_STARTADDR, func = 1;
+	     func <= sd->num_funcs; func++, fbraddr += SDIOD_FBR_SIZE) {
+		sd->func_cis_ptr[func] = sdioh_sdmmc_get_cisaddr(sd, SDIOD_FBR_CISPTR_0 + fbraddr);
+		sd_info(("%s: Function %d CIS Ptr = 0x%x\n",
+		         __FUNCTION__, func, sd->func_cis_ptr[func]));
+	}
+
+	sd->func_cis_ptr[0] = sd->com_cis_ptr;
+	sd_info(("%s: Card's Common CIS Ptr = 0x%x\n", __FUNCTION__, sd->com_cis_ptr));
+
+	/* Enable Function 1 */
+	sdio_claim_host(sd->func[1]);
+	err_ret = sdio_enable_func(sd->func[1]);
+	sdio_release_host(sd->func[1]);
+	if (err_ret) {
+		sd_err(("bcmsdh_sdmmc: Failed to enable F1 Err: %d\n", err_ret));
+	}
+
+	return FALSE;
+}
+
+/*
+ *  Set standard CCCR driver strength. Default value is type B.
+ *  Type: Value, driver strength
+ *     A:  0x10, x1.5
+ *     B:  0x00, x1
+ *     C:  0x20, x0.75
+ *     D:  0x30, x0.5
+ */
+int
+sdioh_set_driver_strength(struct sdio_func *func, uint8 level)
+{
+	int reg;
+	int err = 0;
+
+	sdio_claim_host(func);
+	reg = sdio_readb(func, SDIOD_CCCR_DRIVER_STRENGTH, &err);
+	if (err) {
+		sd_err(("sd_ds error for read SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n", err));
+		goto done;
+	} else {
+		sd_info(("SYNA: sd_ds get cccr driver strength 0x%x\n", reg));
+	}
+
+	reg = ((reg & 0xf) | level);
+	sdio_writeb(func, reg, SDIOD_CCCR_DRIVER_STRENGTH, &err);
+
+	if (err) {
+		sd_err(("sd_ds error for write SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n", err));
+		goto done;
+	} else {
+		sd_info(("SYNA: sd_ds set cccr driver strength 0x%x\n", reg));
+	}
+done:
+	sdio_release_host(func);
+	return err;
+}
+
+/*
+ *	Public entry points & extern's
+ */
+extern sdioh_info_t *
+sdioh_attach(osl_t *osh, struct sdio_func *func)
+{
+	sdioh_info_t *sd = NULL;
+	int err_ret;
+
+	sd_trace(("%s\n", __FUNCTION__));
+
+	if (func == NULL) {
+		sd_err(("%s: sdio function device is NULL\n", __FUNCTION__));
+		return NULL;
+	}
+
+	if ((sd = (sdioh_info_t *)MALLOC(osh, sizeof(sdioh_info_t))) == NULL) {
+		sd_err(("sdioh_attach: out of memory, malloced %d bytes\n", MALLOCED(osh)));
+		return NULL;
+	}
+	bzero((char *)sd, sizeof(sdioh_info_t));
+#if defined(BCMSDIOH_TXGLOM) && defined(BCMSDIOH_STATIC_COPY_BUF)
+	sd->copy_buf = MALLOC(osh, COPY_BUF_SIZE);
+	if (sd->copy_buf == NULL) {
+		sd_err(("%s: MALLOC of %d-byte copy_buf failed\n",
+			__FUNCTION__, COPY_BUF_SIZE));
+		goto fail;
+	}
+#endif
+	sd->osh = osh;
+	sd->fake_func0.num = 0;
+	sd->fake_func0.card = func->card;
+	sd->func[0] = &sd->fake_func0;
+#ifdef GLOBAL_SDMMC_INSTANCE
+	if (func->num == 2)
+		sd->func[1] = gInstance->func[1];
+#else
+	sd->func[1] = func->card->sdio_func[0];
+#endif
+	sd->func[2] = func->card->sdio_func[1];
+#ifdef GLOBAL_SDMMC_INSTANCE
+	sd->func[func->num] = func;
+#endif
+
+#if defined(BT_OVER_SDIO)
+	sd->func[3] = NULL;
+#endif /* defined (BT_OVER_SDIO) */
+
+	sd->num_funcs = 2;
+	sd->sd_blockmode = TRUE;
+	sd->use_client_ints = TRUE;
+	sd->client_block_size[0] = 64;
+	sd->use_rxchain = CUSTOM_RXCHAIN;
+	if (sd->func[1] == NULL || sd->func[2] == NULL) {
+		sd_err(("%s: func 1 or 2 is null \n", __FUNCTION__));
+		goto fail;
+	}
+	sdio_set_drvdata(sd->func[1], sd);
+
+	sdio_claim_host(sd->func[1]);
+	sd->client_block_size[1] = sd_f1_blocksize;
+	err_ret = sdio_set_block_size(sd->func[1], sd_f1_blocksize);
+	sdio_release_host(sd->func[1]);
+	if (err_ret) {
+		sd_err(("bcmsdh_sdmmc: Failed to set F1 blocksize(%d)\n", err_ret));
+		goto fail;
+	}
+
+	sdio_claim_host(sd->func[2]);
+	if ((func->device == BCM43362_CHIP_ID || func->device == BCM4330_CHIP_ID) &&
+			sd_f2_blocksize > 128)
+		sd_f2_blocksize = 128;
+	sd->client_block_size[2] = sd_f2_blocksize;
+	sd_info(("%s: set sd_f2_blocksize %d\n", __FUNCTION__, sd_f2_blocksize));
+	err_ret = sdio_set_block_size(sd->func[2], sd_f2_blocksize);
+	sdio_release_host(sd->func[2]);
+	if (err_ret) {
+		sd_err(("bcmsdh_sdmmc: Failed to set F2 blocksize to %d(%d)\n",
+			sd_f2_blocksize, err_ret));
+		goto fail;
+	}
+
+	sd->sd_clk_rate = sdmmc_get_clock_rate(sd);
+	sd_info(("%s: sd clock rate = %u\n", __FUNCTION__, sd->sd_clk_rate));
+	sdioh_sdmmc_card_enablefuncs(sd);
+#if !defined(OOB_INTR_ONLY)
+	mutex_init(&sd->claim_host_mutex); // terence 20140926: fix for claim host issue
+#endif
+
+	sd_trace(("%s: Done\n", __FUNCTION__));
+	return sd;
+
+fail:
+#if defined(BCMSDIOH_TXGLOM) && defined(BCMSDIOH_STATIC_COPY_BUF)
+	MFREE(sd->osh, sd->copy_buf, COPY_BUF_SIZE);
+#endif
+	MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+	return NULL;
+}
+
+extern SDIOH_API_RC
+sdioh_detach(osl_t *osh, sdioh_info_t *sd)
+{
+	sd_trace(("%s\n", __FUNCTION__));
+
+	if (sd) {
+
+		/* Disable Function 2 */
+		if (sd->func[2]) {
+			sdio_claim_host(sd->func[2]);
+			sdio_disable_func(sd->func[2]);
+			sdio_release_host(sd->func[2]);
+		}
+
+		/* Disable Function 1 */
+		if (sd->func[1]) {
+			sdio_claim_host(sd->func[1]);
+			sdio_disable_func(sd->func[1]);
+			sdio_release_host(sd->func[1]);
+		}
+
+		sd->func[1] = NULL;
+		sd->func[2] = NULL;
+
+#if defined(BCMSDIOH_TXGLOM) && defined(BCMSDIOH_STATIC_COPY_BUF)
+		MFREE(sd->osh, sd->copy_buf, COPY_BUF_SIZE);
+#endif
+		MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+	}
+	return SDIOH_API_RC_SUCCESS;
+}
+
+#if defined(OOB_INTR_ONLY) && defined(HW_OOB)
+
+extern SDIOH_API_RC
+sdioh_enable_func_intr(sdioh_info_t *sd)
+{
+	uint8 reg;
+	int err;
+
+	if (sd->func[0] == NULL) {
+		sd_err(("%s: function 0 pointer is NULL\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	sdio_claim_host(sd->func[0]);
+	reg = sdio_readb(sd->func[0], SDIOD_CCCR_INTEN, &err);
+	if (err) {
+		sd_err(("%s: error for read SDIO_CCCR_IENx : 0x%x\n", __FUNCTION__, err));
+		sdio_release_host(sd->func[0]);
+		return SDIOH_API_RC_FAIL;
+	}
+	/* Enable F1 and F2 interrupts, clear master enable */
+	reg &= ~INTR_CTL_MASTER_EN;
+	reg |= (INTR_CTL_FUNC1_EN | INTR_CTL_FUNC2_EN);
+#if defined(BT_OVER_SDIO)
+	reg |= (INTR_CTL_FUNC3_EN);
+#endif /* defined (BT_OVER_SDIO) */
+	sdio_writeb(sd->func[0], reg, SDIOD_CCCR_INTEN, &err);
+	sdio_release_host(sd->func[0]);
+
+	if (err) {
+		sd_err(("%s: error for write SDIO_CCCR_IENx : 0x%x\n", __FUNCTION__, err));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_disable_func_intr(sdioh_info_t *sd)
+{
+	uint8 reg;
+	int err;
+
+	if (sd->func[0] == NULL) {
+		sd_err(("%s: function 0 pointer is NULL\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	sdio_claim_host(sd->func[0]);
+	reg = sdio_readb(sd->func[0], SDIOD_CCCR_INTEN, &err);
+	if (err) {
+		sd_err(("%s: error for read SDIO_CCCR_IENx : 0x%x (ignore when remove)\n",
+			__FUNCTION__, err));
+		sdio_release_host(sd->func[0]);
+		return SDIOH_API_RC_FAIL;
+	}
+	reg &= ~(INTR_CTL_FUNC1_EN | INTR_CTL_FUNC2_EN);
+#if defined(BT_OVER_SDIO)
+	reg &= ~INTR_CTL_FUNC3_EN;
+#endif
+	/* Disable master interrupt with the last function interrupt */
+	if (!(reg & 0xFE))
+		reg = 0;
+	sdio_writeb(sd->func[0], reg, SDIOD_CCCR_INTEN, &err);
+	sdio_release_host(sd->func[0]);
+
+	if (err) {
+		sd_err(("%s: error for write SDIO_CCCR_IENx : 0x%x\n", __FUNCTION__, err));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	return SDIOH_API_RC_SUCCESS;
+}
+#endif /* defined(OOB_INTR_ONLY) && defined(HW_OOB) */
+
+/* Configure callback to client when we recieve client interrupt */
+extern SDIOH_API_RC
+sdioh_interrupt_register(sdioh_info_t *sd, sdioh_cb_fn_t fn, void *argh)
+{
+	sd_trace(("%s: Entering\n", __FUNCTION__));
+	if (fn == NULL) {
+		sd_err(("%s: interrupt handler is NULL, not registering\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+#if !defined(OOB_INTR_ONLY)
+	sd->intr_handler = fn;
+	sd->intr_handler_arg = argh;
+	sd->intr_handler_valid = TRUE;
+
+	/* register and unmask irq */
+	if (sd->func[2]) {
+		sdio_claim_host(sd->func[2]);
+		sdio_claim_irq(sd->func[2], IRQHandlerF2);
+		sdio_release_host(sd->func[2]);
+	}
+
+	if (sd->func[1]) {
+		sdio_claim_host(sd->func[1]);
+		sdio_claim_irq(sd->func[1], IRQHandler);
+		sdio_release_host(sd->func[1]);
+	}
+#elif defined(HW_OOB)
+	sdioh_enable_func_intr(sd);
+#endif /* !defined(OOB_INTR_ONLY) */
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_interrupt_deregister(sdioh_info_t *sd)
+{
+	sd_trace(("%s: Entering\n", __FUNCTION__));
+
+#if !defined(OOB_INTR_ONLY)
+	if (sd->func[1]) {
+		/* register and unmask irq */
+		sdio_claim_host(sd->func[1]);
+		sdio_release_irq(sd->func[1]);
+		sdio_release_host(sd->func[1]);
+	}
+
+	if (sd->func[2]) {
+		/* Claim host controller F2 */
+		sdio_claim_host(sd->func[2]);
+		sdio_release_irq(sd->func[2]);
+		/* Release host controller F2 */
+		sdio_release_host(sd->func[2]);
+	}
+
+	sd->intr_handler_valid = FALSE;
+	sd->intr_handler = NULL;
+	sd->intr_handler_arg = NULL;
+#elif defined(HW_OOB)
+	if (dhd_download_fw_on_driverload)
+		sdioh_disable_func_intr(sd);
+#endif /* !defined(OOB_INTR_ONLY) */
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_interrupt_query(sdioh_info_t *sd, bool *onoff)
+{
+	sd_trace(("%s: Entering\n", __FUNCTION__));
+	*onoff = sd->client_intr_enabled;
+	return SDIOH_API_RC_SUCCESS;
+}
+
+#if defined(DHD_DEBUG) || defined(BCMDBG)
+extern bool
+sdioh_interrupt_pending(sdioh_info_t *sd)
+{
+	return (0);
+}
+#endif
+
+uint
+sdioh_query_iofnum(sdioh_info_t *sd)
+{
+	return sd->num_funcs;
+}
+
+/* IOVar table */
+enum {
+	IOV_MSGLEVEL = 1,
+	IOV_BLOCKMODE,
+	IOV_BLOCKSIZE,
+	IOV_DMA,
+	IOV_USEINTS,
+	IOV_NUMINTS,
+	IOV_NUMLOCALINTS,
+	IOV_HOSTREG,
+	IOV_DEVREG,
+	IOV_DIVISOR,
+	IOV_SDMODE,
+	IOV_HISPEED,
+	IOV_HCIREGS,
+	IOV_POWER,
+	IOV_CLOCK,
+	IOV_RXCHAIN,
+	IOV_DS
+};
+
+const bcm_iovar_t sdioh_iovars[] = {
+	{"sd_msglevel", IOV_MSGLEVEL,	0, 0,	IOVT_UINT32,	0 },
+	{"sd_blockmode", IOV_BLOCKMODE, 0, 0,	IOVT_BOOL,	0 },
+	{"sd_blocksize", IOV_BLOCKSIZE, 0, 0,	IOVT_UINT32,	0 }, /* ((fn << 16) | size) */
+	{"sd_dma",	IOV_DMA,	0, 0,	IOVT_BOOL,	0 },
+	{"sd_ints",	IOV_USEINTS,	0, 0,	IOVT_BOOL,	0 },
+	{"sd_numints",	IOV_NUMINTS,	0, 0,	IOVT_UINT32,	0 },
+	{"sd_numlocalints", IOV_NUMLOCALINTS, 0, 0, IOVT_UINT32,	0 },
+	{"sd_divisor",	IOV_DIVISOR,	0, 0,	IOVT_UINT32,	0 },
+	{"sd_power",	IOV_POWER,	0, 0,	IOVT_UINT32,	0 },
+	{"sd_clock",	IOV_CLOCK,	0, 0,	IOVT_UINT32,	0 },
+	{"sd_mode",	IOV_SDMODE,	0, 0,	IOVT_UINT32,	100},
+	{"sd_highspeed", IOV_HISPEED,	0, 0,	IOVT_UINT32,	0 },
+	{"sd_rxchain",  IOV_RXCHAIN,    0, 0, 	IOVT_BOOL,	0 },
+#ifdef BCMDBG
+	{"sd_hciregs",	IOV_HCIREGS,	0, 0,	IOVT_BUFFER,	0 },
+#endif
+	{"sd_ds",	IOV_DS,		0, 0,	IOVT_UINT32,	0 },
+	{NULL, 0, 0, 0, 0, 0 }
+};
+
+int
+sdioh_iovar_op(sdioh_info_t *si, const char *name,
+                           void *params, int plen, void *arg, uint len, bool set)
+{
+	const bcm_iovar_t *vi = NULL;
+	int bcmerror = 0;
+	uint val_size;
+	int32 int_val = 0;
+	bool bool_val;
+	uint32 actionid;
+
+	ASSERT(name);
+
+	/* Get must have return space; Set does not take qualifiers */
+	ASSERT(set || (arg && len));
+	ASSERT(!set || (!params && !plen));
+
+	sd_trace(("%s: Enter (%s %s)\n", __FUNCTION__, (set ? "set" : "get"), name));
+
+	if ((vi = bcm_iovar_lookup(sdioh_iovars, name)) == NULL) {
+		bcmerror = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	if ((bcmerror = bcm_iovar_lencheck(vi, arg, len, set)) != 0)
+		goto exit;
+
+	/* XXX Copied from dhd, copied from wl; certainly overkill here? */
+	/* Set up params so get and set can share the convenience variables */
+	if (params == NULL) {
+		params = arg;
+		plen = len;
+	}
+
+	if (vi->type == IOVT_VOID)
+		val_size = 0;
+	else if (vi->type == IOVT_BUFFER)
+		val_size = len;
+	else
+		val_size = sizeof(int);
+
+	if (plen >= (int)sizeof(int_val))
+		bcopy(params, &int_val, sizeof(int_val));
+
+	bool_val = (int_val != 0) ? TRUE : FALSE;
+	BCM_REFERENCE(bool_val);
+
+	actionid = set ? IOV_SVAL(vi->varid) : IOV_GVAL(vi->varid);
+	switch (actionid) {
+	case IOV_GVAL(IOV_MSGLEVEL):
+		int_val = (int32)sd_msglevel;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_MSGLEVEL):
+		sd_msglevel = int_val;
+		break;
+
+	case IOV_GVAL(IOV_BLOCKMODE):
+		int_val = (int32)si->sd_blockmode;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_BLOCKMODE):
+		si->sd_blockmode = (bool)int_val;
+		/* Haven't figured out how to make non-block mode with DMA */
+		break;
+
+	case IOV_GVAL(IOV_BLOCKSIZE):
+		if ((uint32)int_val > si->num_funcs) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = (int32)si->client_block_size[int_val];
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_BLOCKSIZE):
+	{
+		uint func = ((uint32)int_val >> 16);
+		uint blksize = (uint16)int_val;
+		uint maxsize;
+
+		if (func > si->num_funcs) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		/* XXX These hardcoded sizes are a hack, remove after proper CIS parsing. */
+		switch (func) {
+		case 0: maxsize = 32; break;
+		case 1: maxsize = BLOCK_SIZE_4318; break;
+		case 2: maxsize = BLOCK_SIZE_4328; break;
+		default: maxsize = 0;
+		}
+		if (blksize > maxsize) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		if (!blksize) {
+			blksize = maxsize;
+		}
+
+		/* Now set it */
+		si->client_block_size[func] = blksize;
+
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+		if (si->func[func] == NULL) {
+			sd_err(("%s: SDIO Device not present\n", __FUNCTION__));
+			bcmerror = BCME_NORESOURCE;
+			break;
+		}
+		sdio_claim_host(si->func[func]);
+		bcmerror = sdio_set_block_size(si->func[func], blksize);
+		if (bcmerror)
+			sd_err(("%s: Failed to set F%d blocksize to %d(%d)\n",
+				__FUNCTION__, func, blksize, bcmerror));
+		sdio_release_host(si->func[func]);
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+		break;
+	}
+
+	case IOV_GVAL(IOV_RXCHAIN):
+		int_val = (int32)si->use_rxchain;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_DMA):
+		int_val = (int32)si->sd_use_dma;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DMA):
+		si->sd_use_dma = (bool)int_val;
+		break;
+
+	case IOV_GVAL(IOV_USEINTS):
+		int_val = (int32)si->use_client_ints;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_USEINTS):
+		si->use_client_ints = (bool)int_val;
+		if (si->use_client_ints)
+			si->intmask |= CLIENT_INTR;
+		else
+			si->intmask &= ~CLIENT_INTR;
+
+		break;
+
+	case IOV_GVAL(IOV_DIVISOR):
+		int_val = (uint32)sd_divisor;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DIVISOR):
+		/* set the clock to divisor, if value is non-zero & power of 2 */
+		if (int_val && !(int_val & (int_val - 1))) {
+			sd_divisor = int_val;
+			sdmmc_set_clock_divisor(si, sd_divisor);
+		} else {
+			DHD_ERROR(("%s: Invalid sd_divisor value, should be power of 2!\n",
+				__FUNCTION__));
+		}
+		break;
+
+	case IOV_GVAL(IOV_POWER):
+		int_val = (uint32)sd_power;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_POWER):
+		sd_power = int_val;
+		break;
+
+	case IOV_GVAL(IOV_CLOCK):
+		int_val = (uint32)sd_clock;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_CLOCK):
+		sd_clock = int_val;
+		break;
+
+	case IOV_GVAL(IOV_SDMODE):
+		int_val = (uint32)sd_sdmode;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SDMODE):
+		sd_sdmode = int_val;
+		break;
+
+	case IOV_GVAL(IOV_HISPEED):
+		int_val = (uint32)sd_hiok;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_HISPEED):
+		sd_hiok = int_val;
+		break;
+
+	case IOV_GVAL(IOV_NUMINTS):
+		int_val = (int32)si->intrcount;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_NUMLOCALINTS):
+		int_val = (int32)0;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_DS):
+	{
+		uint8 reg2;
+		int err;
+
+		sdio_claim_host(si->func[0]);
+		reg2 = sdio_readb(si->func[0], SDIOD_CCCR_DRIVER_STRENGTH, &err);
+		if (err) {
+			sd_err(("sd_ds error for read SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n", err));
+			bcmerror = BCME_SDIO_ERROR;
+		} else {
+			sd_trace(("sd_ds get cccr driver strength 0x%x\n", reg2));
+		}
+		sdio_release_host(si->func[0]);
+
+		int_val = (int)reg2;
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+
+	case IOV_SVAL(IOV_DS):
+	{
+		uint8 type;
+		int err;
+
+		if (int_val == 0) {
+			type = 0x0;
+		} else if (int_val == 1) {
+			type = 0x10;
+		} else if (int_val == 2) {
+			type = 0x20;
+		} else if (int_val == 3) {
+			type = 0x30;
+		} else {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		err = sdioh_set_driver_strength(si->func[0], type);
+		if (err) {
+			sd_err(("sd_ds error write SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n", err));
+		}
+		break;
+	}
+
+	default:
+		bcmerror = BCME_UNSUPPORTED;
+		break;
+	}
+exit:
+
+	/* XXX Remove protective lock after clients all clean... */
+	return bcmerror;
+}
+
+#if (defined(OOB_INTR_ONLY) && defined(HW_OOB)) || defined(FORCE_WOWLAN)
+/*
+ * XXX dhd -i eth0 sd_devreg 0 0xf2 0x3
+ */
+
+SDIOH_API_RC
+sdioh_enable_hw_oob_intr(sdioh_info_t *sd, bool enable)
+{
+	SDIOH_API_RC status;
+	uint8 data;
+
+	if (enable)
+		data = SDIO_SEPINT_MASK | SDIO_SEPINT_OE | SDIO_SEPINT_ACT_HI;
+	else
+		data = SDIO_SEPINT_ACT_HI;	/* disable hw oob interrupt */
+
+	status = sdioh_request_byte(sd, SDIOH_WRITE, 0, SDIOD_CCCR_BRCM_SEPINT, &data);
+	return status;
+}
+#endif /* defined(OOB_INTR_ONLY) && defined(HW_OOB) */
+
+extern SDIOH_API_RC
+sdioh_cfg_read(sdioh_info_t *sd, uint fnc_num, uint32 addr, uint8 *data)
+{
+	SDIOH_API_RC status;
+	/* No lock needed since sdioh_request_byte does locking */
+	status = sdioh_request_byte(sd, SDIOH_READ, fnc_num, addr, data);
+	return status;
+}
+
+extern SDIOH_API_RC
+sdioh_cfg_write(sdioh_info_t *sd, uint fnc_num, uint32 addr, uint8 *data)
+{
+	/* No lock needed since sdioh_request_byte does locking */
+	SDIOH_API_RC status;
+	status = sdioh_request_byte(sd, SDIOH_WRITE, fnc_num, addr, data);
+	return status;
+}
+
+static int
+sdioh_sdmmc_get_cisaddr(sdioh_info_t *sd, uint32 regaddr)
+{
+	/* read 24 bits and return valid 17 bit addr */
+	int i;
+	uint32 scratch, regdata;
+	uint8 *ptr = (uint8 *)&scratch;
+	for (i = 0; i < 3; i++) {
+		if ((sdioh_sdmmc_card_regread (sd, 0, regaddr, 1, &regdata)) != SUCCESS)
+			sd_err(("%s: Can't read!\n", __FUNCTION__));
+
+		*ptr++ = (uint8) regdata;
+		regaddr++;
+	}
+
+	/* Only the lower 17-bits are valid */
+	scratch = ltoh32(scratch);
+	scratch &= 0x0001FFFF;
+	return (scratch);
+}
+
+extern SDIOH_API_RC
+sdioh_cis_read(sdioh_info_t *sd, uint func, uint8 *cisd, uint32 length)
+{
+	uint32 count;
+	int offset;
+	uint32 foo;
+	uint8 *cis = cisd;
+
+	sd_trace(("%s: Func = %d\n", __FUNCTION__, func));
+
+	if (!sd->func_cis_ptr[func]) {
+		bzero(cis, length);
+		sd_err(("%s: no func_cis_ptr[%d]\n", __FUNCTION__, func));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	sd_info(("%s: func_cis_ptr[%d]=0x%04x\n", __FUNCTION__, func, sd->func_cis_ptr[func]));
+
+	for (count = 0; count < length; count++) {
+		offset =  sd->func_cis_ptr[func] + count;
+		if (sdioh_sdmmc_card_regread (sd, 0, offset, 1, &foo) < 0) {
+			sd_err(("%s: regread failed: Can't read CIS\n", __FUNCTION__));
+			return SDIOH_API_RC_FAIL;
+		}
+
+		*cis = (uint8)(foo & 0xff);
+		cis++;
+	}
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_cisaddr_read(sdioh_info_t *sd, uint func, uint8 *cisd, uint32 offset)
+{
+	uint32 foo;
+
+	sd_trace(("%s: Func = %d\n", __FUNCTION__, func));
+
+	if (!sd->func_cis_ptr[func]) {
+		sd_err(("%s: no func_cis_ptr[%d]\n", __FUNCTION__, func));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	if (sdioh_sdmmc_card_regread (sd, 0, sd->func_cis_ptr[func]+offset, 1, &foo) < 0) {
+		sd_err(("%s: regread failed: Can't read CIS\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	*cisd = (uint8)(foo & 0xff);
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_request_byte(sdioh_info_t *sd, uint rw, uint func, uint regaddr, uint8 *byte)
+{
+	int err_ret = 0;
+#if defined(MMC_SDIO_ABORT)
+	int sdio_abort_retry = MMC_SDIO_ABORT_RETRY_LIMIT;
+#endif
+	struct osl_timespec now, before;
+
+	if (sd_msglevel & SDH_COST_VAL)
+		osl_do_gettimeofday(&before);
+
+	sd_info(("%s: rw=%d, func=%d, addr=0x%05x\n", __FUNCTION__, rw, func, regaddr));
+
+	DHD_PM_RESUME_WAIT(sdioh_request_byte_wait);
+	DHD_PM_RESUME_RETURN_ERROR(SDIOH_API_RC_FAIL);
+	if(rw) { /* CMD52 Write */
+		if (func == 0) {
+			/* Can only directly write to some F0 registers.  Handle F2 enable
+			 * as a special case.
+			 */
+			if (regaddr == SDIOD_CCCR_IOEN) {
+#if defined(BT_OVER_SDIO)
+				do {
+				if (sd->func[3]) {
+					sd_info(("bcmsdh_sdmmc F3: *byte 0x%x\n", *byte));
+
+					if (*byte & SDIO_FUNC_ENABLE_3) {
+						sdio_claim_host(sd->func[3]);
+
+						/* Set Function 3 Block Size */
+						err_ret = sdio_set_block_size(sd->func[3],
+						sd_f3_blocksize);
+						if (err_ret) {
+							sd_err(("F3 blocksize set err%d\n",
+								err_ret));
+						}
+
+						/* Enable Function 3 */
+						sd_info(("bcmsdh_sdmmc F3: enable F3 fn %p\n",
+						sd->func[3]));
+						err_ret = sdio_enable_func(sd->func[3]);
+						if (err_ret) {
+							sd_err(("bcmsdh_sdmmc: enable F3 err:%d\n",
+								err_ret));
+						}
+
+						sdio_release_host(sd->func[3]);
+
+						break;
+					} else if (*byte & SDIO_FUNC_DISABLE_3) {
+						sdio_claim_host(sd->func[3]);
+
+						/* Disable Function 3 */
+						sd_info(("bcmsdh_sdmmc F3: disable F3 fn %p\n",
+						sd->func[3]));
+						err_ret = sdio_disable_func(sd->func[3]);
+						if (err_ret) {
+							sd_err(("bcmsdh_sdmmc: Disable F3 err:%d\n",
+								err_ret));
+						}
+						sdio_release_host(sd->func[3]);
+						sd->func[3] = NULL;
+
+						break;
+					}
+				}
+#endif /* defined (BT_OVER_SDIO) */
+				if (sd->func[2]) {
+					sdio_claim_host(sd->func[2]);
+					if (*byte & SDIO_FUNC_ENABLE_2) {
+						/* Enable Function 2 */
+						err_ret = sdio_enable_func(sd->func[2]);
+						if (err_ret) {
+							sd_err(("bcmsdh_sdmmc: enable F2 failed:%d\n",
+								err_ret));
+						}
+					} else {
+						/* Disable Function 2 */
+						err_ret = sdio_disable_func(sd->func[2]);
+						if (err_ret) {
+							sd_err(("bcmsdh_sdmmc: Disab F2 failed:%d\n",
+								err_ret));
+						}
+					}
+					sdio_release_host(sd->func[2]);
+				}
+#if defined(BT_OVER_SDIO)
+			} while (0);
+#endif /* defined (BT_OVER_SDIO) */
+		}
+#if defined(MMC_SDIO_ABORT)
+			/* to allow abort command through F1 */
+			else if (regaddr == SDIOD_CCCR_IOABORT) {
+				/* XXX Because of SDIO3.0 host issue on Manta,
+				 * sometimes the abort fails.
+				 * Retrying again will fix this issue.
+				 */
+				while (sdio_abort_retry--) {
+					if (sd->func[func]) {
+						sdio_claim_host(sd->func[func]);
+						/*
+						 * this sdio_f0_writeb() can be replaced with
+						 * another api depending upon MMC driver change.
+						 * As of this time, this is temporaray one
+						 */
+						sdio_writeb(sd->func[func],
+							*byte, regaddr, &err_ret);
+						sdio_release_host(sd->func[func]);
+					}
+					if (!err_ret)
+						break;
+				}
+			}
+#endif /* MMC_SDIO_ABORT */
+#if defined(SDIO_ISR_THREAD)
+			else if (regaddr == SDIOD_CCCR_INTR_EXTN) {
+				while (sdio_abort_retry--) {
+					if (sd->func[func]) {
+						sdio_claim_host(sd->func[func]);
+						/*
+						 * this sdio_f0_writeb() can be replaced with
+						 * another api depending upon MMC driver change.
+						 * As of this time, this is temporaray one
+						 */
+						sdio_writeb(sd->func[func],
+							*byte, regaddr, &err_ret);
+						sdio_release_host(sd->func[func]);
+					}
+					if (!err_ret)
+						break;
+				}
+			}
+#endif
+			else if (regaddr < 0xF0) {
+				sd_err(("bcmsdh_sdmmc: F0 Wr:0x%02x: write disallowed\n", regaddr));
+			} else {
+				/* Claim host controller, perform F0 write, and release */
+				if (sd->func[func]) {
+					sdio_claim_host(sd->func[func]);
+					sdio_f0_writeb(sd->func[func],
+						*byte, regaddr, &err_ret);
+					sdio_release_host(sd->func[func]);
+				}
+			}
+		} else {
+			/* Claim host controller, perform Fn write, and release */
+			if (sd->func[func]) {
+				sdio_claim_host(sd->func[func]);
+				sdio_writeb(sd->func[func], *byte, regaddr, &err_ret);
+				sdio_release_host(sd->func[func]);
+			}
+		}
+	} else { /* CMD52 Read */
+		/* Claim host controller, perform Fn read, and release */
+		if (sd->func[func]) {
+			sdio_claim_host(sd->func[func]);
+			if (func == 0) {
+				*byte = sdio_f0_readb(sd->func[func], regaddr, &err_ret);
+			} else {
+				*byte = sdio_readb(sd->func[func], regaddr, &err_ret);
+			}
+			sdio_release_host(sd->func[func]);
+		}
+	}
+
+	if (err_ret) {
+		if (regaddr == 0x1001F) {
+			/* XXX: Read/Write to SBSDIO_FUNC1_SLEEPCSR could return -110(timeout)
+			 * 	or -84(CRC) error in case the host tries to wake the device up.
+			 *	Skip error log message if err code is -110 or -84 when accessing
+			 *	to SBSDIO_FUNC1_SLEEPCSR to avoid QA misunderstand and DHD shoul
+			 *	print error log message if retry count over the MAX_KSO_ATTEMPTS.
+			 */
+		} else {
+			sd_err(("bcmsdh_sdmmc: Failed to %s byte F%d:@0x%05x=%02x, Err: %d\n",
+				rw ? "Write" : "Read", func, regaddr, *byte, err_ret));
+		}
+	}
+
+	if (sd_msglevel & SDH_COST_VAL) {
+		uint32 diff_us;
+		osl_do_gettimeofday(&now);
+		diff_us = osl_do_gettimediff(&now, &before);
+		sd_cost(("%s: rw=%d len=1 cost = %3dms %3dus\n", __FUNCTION__,
+			rw, diff_us/1000, diff_us%1000));
+	}
+
+	return ((err_ret == 0) ? SDIOH_API_RC_SUCCESS : SDIOH_API_RC_FAIL);
+}
+
+#ifdef PKT_STATICS
+uint32
+sdioh_get_spend_time(sdioh_info_t *sd)
+{
+	return (sd->sdio_spent_time_us);
+}
+#endif
+
+extern SDIOH_API_RC
+sdioh_request_word(sdioh_info_t *sd, uint cmd_type, uint rw, uint func, uint addr,
+                                   uint32 *word, uint nbytes)
+{
+	int err_ret = SDIOH_API_RC_FAIL;
+	int err_ret2 = SDIOH_API_RC_SUCCESS; // terence 20130621: prevent dhd_dpc in dead lock
+#if defined(MMC_SDIO_ABORT)
+	int sdio_abort_retry = MMC_SDIO_ABORT_RETRY_LIMIT;
+#endif
+	struct osl_timespec now, before;
+
+	if (sd_msglevel & SDH_COST_VAL)
+		osl_do_gettimeofday(&before);
+
+	if (func == 0) {
+		sd_err(("%s: Only CMD52 allowed to F0.\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	sd_info(("%s: cmd_type=%d, rw=%d, func=%d, addr=0x%05x, nbytes=%d\n",
+	         __FUNCTION__, cmd_type, rw, func, addr, nbytes));
+
+	DHD_PM_RESUME_WAIT(sdioh_request_word_wait);
+	DHD_PM_RESUME_RETURN_ERROR(SDIOH_API_RC_FAIL);
+	/* Claim host controller */
+	sdio_claim_host(sd->func[func]);
+
+	if(rw) { /* CMD52 Write */
+		if (nbytes == 4) {
+			sdio_writel(sd->func[func], *word, addr, &err_ret);
+		} else if (nbytes == 2) {
+			sdio_writew(sd->func[func], (*word & 0xFFFF), addr, &err_ret);
+		} else {
+			sd_err(("%s: Invalid nbytes: %d\n", __FUNCTION__, nbytes));
+		}
+	} else { /* CMD52 Read */
+		if (nbytes == 4) {
+			*word = sdio_readl(sd->func[func], addr, &err_ret);
+		} else if (nbytes == 2) {
+			*word = sdio_readw(sd->func[func], addr, &err_ret) & 0xFFFF;
+		} else {
+			sd_err(("%s: Invalid nbytes: %d\n", __FUNCTION__, nbytes));
+		}
+	}
+
+	/* Release host controller */
+	sdio_release_host(sd->func[func]);
+
+	if (err_ret) {
+#if defined(MMC_SDIO_ABORT)
+		/* Any error on CMD53 transaction should abort that function using function 0. */
+		while (sdio_abort_retry--) {
+			if (sd->func[0]) {
+				sdio_claim_host(sd->func[0]);
+				/*
+				 * this sdio_f0_writeb() can be replaced with another api
+				 * depending upon MMC driver change.
+				 * As of this time, this is temporaray one
+				 */
+				sdio_writeb(sd->func[0],
+					func, SDIOD_CCCR_IOABORT, &err_ret2);
+				sdio_release_host(sd->func[0]);
+			}
+			if (!err_ret2)
+				break;
+		}
+		if (err_ret)
+#endif /* MMC_SDIO_ABORT */
+		{
+			sd_err(("bcmsdh_sdmmc: Failed to %s word F%d:@0x%05x=%02x, Err: %d\n",
+				rw ? "Write" : "Read", func, addr, *word, err_ret));
+		}
+	}
+
+	if (sd_msglevel & SDH_COST_VAL) {
+		uint32 diff_us;
+		osl_do_gettimeofday(&now);
+		diff_us = osl_do_gettimediff(&now, &before);
+		sd_cost(("%s: rw=%d, len=%d cost = %3dms %3dus\n", __FUNCTION__,
+			rw, nbytes, diff_us/1000, diff_us%1000));
+	}
+
+	return (((err_ret == 0)&&(err_ret2 == 0)) ? SDIOH_API_RC_SUCCESS : SDIOH_API_RC_FAIL);
+}
+
+#ifdef BCMSDIOH_TXGLOM
+static SDIOH_API_RC
+sdioh_request_packet_chain(sdioh_info_t *sd, uint fix_inc, uint write, uint func,
+                     uint addr, void *pkt)
+{
+	bool fifo = (fix_inc == SDIOH_DATA_FIX);
+	int err_ret = 0;
+	void *pnext;
+	uint ttl_len = 0, pkt_offset;
+	uint blk_num;
+	uint blk_size;
+	uint max_blk_count;
+	uint max_req_size;
+	struct mmc_request mmc_req;
+	struct mmc_command mmc_cmd;
+	struct mmc_data mmc_dat;
+	uint32 sg_count = 0;
+	struct sdio_func *sdio_func = sd->func[func];
+	struct mmc_host *host = sdio_func->card->host;
+	uint8 *localbuf = NULL;
+	uint local_plen = 0;
+	uint pkt_len = 0;
+	struct osl_timespec now, before;
+
+	sd_trace(("%s: Enter\n", __FUNCTION__));
+	ASSERT(pkt);
+	DHD_PM_RESUME_WAIT(sdioh_request_packet_wait);
+	DHD_PM_RESUME_RETURN_ERROR(SDIOH_API_RC_FAIL);
+
+#ifndef PKT_STATICS
+	if (sd_msglevel & SDH_COST_VAL)
+#endif
+		osl_do_gettimeofday(&before);
+
+	blk_size = sd->client_block_size[func];
+	max_blk_count = min(host->max_blk_count, (uint)MAX_IO_RW_EXTENDED_BLK);
+	max_req_size = min(max_blk_count * blk_size, host->max_req_size);
+
+	pkt_offset = 0;
+	pnext = pkt;
+
+	ttl_len = 0;
+	sg_count = 0;
+	if(sd->txglom_mode == SDPCM_TXGLOM_MDESC) {
+		while (pnext != NULL) {
+			ttl_len = 0;
+			sg_count = 0;
+			memset(&mmc_req, 0, sizeof(struct mmc_request));
+			memset(&mmc_cmd, 0, sizeof(struct mmc_command));
+			memset(&mmc_dat, 0, sizeof(struct mmc_data));
+			sg_init_table(sd->sg_list, ARRAYSIZE(sd->sg_list));
+
+			/* Set up scatter-gather DMA descriptors. this loop is to find out the max
+			 * data we can transfer with one command 53. blocks per command is limited by
+			 * host max_req_size and 9-bit max block number. when the total length of this
+			 * packet chain is bigger than max_req_size, use multiple SD_IO_RW_EXTENDED
+			 * commands (each transfer is still block aligned)
+			 */
+			while (pnext != NULL && ttl_len < max_req_size) {
+				int pkt_len;
+				int sg_data_size;
+				uint8 *pdata = (uint8*)PKTDATA(sd->osh, pnext);
+
+				ASSERT(pdata != NULL);
+				pkt_len = PKTLEN(sd->osh, pnext);
+				sd_trace(("%s[%d] data=%p, len=%d\n", __FUNCTION__, write, pdata, pkt_len));
+				/* sg_count is unlikely larger than the array size, and this is
+				 * NOT something we can handle here, but in case it happens, PLEASE put
+				 * a restriction on max tx/glom count (based on host->max_segs).
+				 */
+				if (sg_count >= ARRAYSIZE(sd->sg_list)) {
+					sd_err(("%s: sg list entries(%u) exceed limit(%zu),"
+						" sd blk_size=%u\n",
+						__FUNCTION__, sg_count, (size_t)ARRAYSIZE(sd->sg_list), blk_size));
+					return (SDIOH_API_RC_FAIL);
+				}
+				pdata += pkt_offset;
+
+				sg_data_size = pkt_len - pkt_offset;
+				if (sg_data_size > max_req_size - ttl_len)
+					sg_data_size = max_req_size - ttl_len;
+				/* some platforms put a restriction on the data size of each scatter-gather
+				 * DMA descriptor, use multiple sg buffers when xfer_size is bigger than
+				 * max_seg_size
+				 */
+				if (sg_data_size > host->max_seg_size) {
+					sg_data_size = host->max_seg_size;
+				}
+				sg_set_buf(&sd->sg_list[sg_count++], pdata, sg_data_size);
+
+				ttl_len += sg_data_size;
+				pkt_offset += sg_data_size;
+				if (pkt_offset == pkt_len) {
+					pnext = PKTNEXT(sd->osh, pnext);
+					pkt_offset = 0;
+				}
+			}
+
+			if (ttl_len % blk_size != 0) {
+				sd_err(("%s, data length %d not aligned to block size %d\n",
+					__FUNCTION__,  ttl_len, blk_size));
+				return SDIOH_API_RC_FAIL;
+			}
+			blk_num = ttl_len / blk_size;
+			mmc_dat.sg = sd->sg_list;
+			mmc_dat.sg_len = sg_count;
+			mmc_dat.blksz = blk_size;
+			mmc_dat.blocks = blk_num;
+			mmc_dat.flags = write ? MMC_DATA_WRITE : MMC_DATA_READ;
+			mmc_cmd.opcode = 53; /* SD_IO_RW_EXTENDED */
+			mmc_cmd.arg = write ? 1<<31 : 0;
+			mmc_cmd.arg |= (func & 0x7) << 28;
+			mmc_cmd.arg |= 1<<27;
+			mmc_cmd.arg |= fifo ? 0 : 1<<26;
+			mmc_cmd.arg |= (addr & 0x1FFFF) << 9;
+			mmc_cmd.arg |= blk_num & 0x1FF;
+			mmc_cmd.flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_ADTC;
+			mmc_req.cmd = &mmc_cmd;
+			mmc_req.data = &mmc_dat;
+			if (!fifo)
+				addr += ttl_len;
+
+			sdio_claim_host(sdio_func);
+			mmc_set_data_timeout(&mmc_dat, sdio_func->card);
+			mmc_wait_for_req(host, &mmc_req);
+			sdio_release_host(sdio_func);
+
+			err_ret = mmc_cmd.error? mmc_cmd.error : mmc_dat.error;
+			if (0 != err_ret) {
+				sd_err(("%s:CMD53 %s failed with code %d\n",
+					__FUNCTION__, write ? "write" : "read", err_ret));
+				return SDIOH_API_RC_FAIL;
+			}
+		}
+	}
+	else if(sd->txglom_mode == SDPCM_TXGLOM_CPY) {
+		for (pnext = pkt; pnext; pnext = PKTNEXT(sd->osh, pnext)) {
+			ttl_len += PKTLEN(sd->osh, pnext);
+		}
+		/* Claim host controller */
+		sdio_claim_host(sd->func[func]);
+		for (pnext = pkt; pnext; pnext = PKTNEXT(sd->osh, pnext)) {
+			uint8 *buf = (uint8*)PKTDATA(sd->osh, pnext);
+			pkt_len = PKTLEN(sd->osh, pnext);
+			if (!localbuf) {
+#ifdef BCMSDIOH_STATIC_COPY_BUF
+				if (ttl_len <= COPY_BUF_SIZE)
+					localbuf = sd->copy_buf;
+#else
+				localbuf = (uint8 *)MALLOC(sd->osh, ttl_len);
+#endif
+				if (localbuf == NULL) {
+					sd_err(("%s: %s localbuf malloc FAILED ttl_len=%d\n",
+						__FUNCTION__, (write) ? "TX" : "RX", ttl_len));
+					ttl_len -= pkt_len;
+					goto txglomfail;
+				}
+			}
+			bcopy(buf, (localbuf + local_plen), pkt_len);
+			local_plen += pkt_len;
+			if (PKTNEXT(sd->osh, pnext))
+				continue;
+
+			buf = localbuf;
+			pkt_len = local_plen;
+txglomfail:
+			/* Align Patch */
+			if (!write || pkt_len < 32)
+				pkt_len = (pkt_len + 3) & 0xFFFFFFFC;
+			else if (pkt_len % blk_size)
+				pkt_len += blk_size - (pkt_len % blk_size);
+
+			if ((write) && (!fifo))
+				err_ret = sdio_memcpy_toio(sd->func[func], addr, buf, pkt_len);
+			else if (write)
+				err_ret = sdio_memcpy_toio(sd->func[func], addr, buf, pkt_len);
+			else if (fifo)
+				err_ret = sdio_readsb(sd->func[func], buf, addr, pkt_len);
+			else
+				err_ret = sdio_memcpy_fromio(sd->func[func], buf, addr, pkt_len);
+
+			if (err_ret)
+				sd_err(("%s: %s FAILED %p[%d], addr=0x%05x, pkt_len=%d, ERR=%d\n",
+				       __FUNCTION__,
+				       (write) ? "TX" : "RX",
+				       pnext, sg_count, addr, pkt_len, err_ret));
+			else
+				sd_trace(("%s: %s xfr'd %p[%d], addr=0x%05x, len=%d\n",
+					__FUNCTION__,
+					(write) ? "TX" : "RX",
+					pnext, sg_count, addr, pkt_len));
+
+			if (!fifo)
+				addr += pkt_len;
+			sg_count ++;
+		}
+		sdio_release_host(sd->func[func]);
+	} else {
+		sd_err(("%s: set to wrong glom mode %d\n", __FUNCTION__, sd->txglom_mode));
+		return SDIOH_API_RC_FAIL;
+	}
+
+#ifndef BCMSDIOH_STATIC_COPY_BUF
+	if (localbuf)
+		MFREE(sd->osh, localbuf, ttl_len);
+#endif
+
+#ifndef PKT_STATICS
+	if (sd_msglevel & SDH_COST_VAL)
+#endif
+	{
+		uint32 diff_us;
+		osl_do_gettimeofday(&now);
+		diff_us = osl_do_gettimediff(&now, &before);
+		sd_cost(("%s: rw=%d, ttl_len=%4d cost = %3dms %3dus\n", __FUNCTION__,
+			write, ttl_len, diff_us/1000, diff_us%1000));
+#ifdef PKT_STATICS
+		if (write && (func == 2))
+			sd->sdio_spent_time_us = diff_us;
+#endif
+	}
+
+	sd_trace(("%s: Exit\n", __FUNCTION__));
+	return SDIOH_API_RC_SUCCESS;
+}
+#endif /* BCMSDIOH_TXGLOM */
+
+static SDIOH_API_RC
+sdioh_buffer_tofrom_bus(sdioh_info_t *sd, uint fix_inc, uint write, uint func,
+                     uint addr, uint8 *buf, uint len)
+{
+	bool fifo = (fix_inc == SDIOH_DATA_FIX);
+	int err_ret = 0;
+	struct osl_timespec now, before;
+
+	sd_trace(("%s: Enter\n", __FUNCTION__));
+	ASSERT(buf);
+
+	if (sd_msglevel & SDH_COST_VAL)
+		osl_do_gettimeofday(&before);
+
+	/* NOTE:
+	 * For all writes, each packet length is aligned to 32 (or 4)
+	 * bytes in dhdsdio_txpkt_preprocess, and for glom the last packet length
+	 * is aligned to block boundary. If you want to align each packet to
+	 * a custom size, please do it in dhdsdio_txpkt_preprocess, NOT here
+	 *
+	 * For reads, the alignment is doen in sdioh_request_buffer.
+	 *
+	 */
+	sdio_claim_host(sd->func[func]);
+
+	if ((write) && (!fifo))
+		err_ret = sdio_memcpy_toio(sd->func[func], addr, buf, len);
+	else if (write)
+		err_ret = sdio_memcpy_toio(sd->func[func], addr, buf, len);
+	else if (fifo)
+		err_ret = sdio_readsb(sd->func[func], buf, addr, len);
+	else
+		err_ret = sdio_memcpy_fromio(sd->func[func], buf, addr, len);
+
+	sdio_release_host(sd->func[func]);
+
+	if (err_ret)
+		sd_err(("%s: %s FAILED %p, addr=0x%05x, pkt_len=%d, ERR=%d\n", __FUNCTION__,
+		       (write) ? "TX" : "RX", buf, addr, len, err_ret));
+	else
+		sd_trace(("%s: %s xfr'd %p, addr=0x%05x, len=%d\n", __FUNCTION__,
+			(write) ? "TX" : "RX", buf, addr, len));
+
+	sd_trace(("%s: Exit\n", __FUNCTION__));
+
+	if (sd_msglevel & SDH_COST_VAL) {
+		uint32 diff_us;
+		osl_do_gettimeofday(&now);
+		diff_us = osl_do_gettimediff(&now, &before);
+		sd_cost(("%s: rw=%d, len=%4d cost = %3dms %3dus\n", __FUNCTION__,
+			write, len, diff_us/1000, diff_us%1000));
+	}
+
+	return ((err_ret == 0) ? SDIOH_API_RC_SUCCESS : SDIOH_API_RC_FAIL);
+}
+
+/*
+ * This function takes a buffer or packet, and fixes everything up so that in the
+ * end, a DMA-able packet is created.
+ *
+ * A buffer does not have an associated packet pointer, and may or may not be aligned.
+ * A packet may consist of a single packet, or a packet chain.  If it is a packet chain,
+ * then all the packets in the chain must be properly aligned.  If the packet data is not
+ * aligned, then there may only be one packet, and in this case, it is copied to a new
+ * aligned packet.
+ *
+ */
+extern SDIOH_API_RC
+sdioh_request_buffer(sdioh_info_t *sd, uint pio_dma, uint fix_inc, uint write, uint func,
+	uint addr, uint reg_width, uint buf_len, uint8 *buffer, void *pkt)
+{
+	SDIOH_API_RC status;
+	void *tmppkt;
+	int is_vmalloc = FALSE;
+	struct osl_timespec now, before;
+
+	sd_trace(("%s: Enter\n", __FUNCTION__));
+	DHD_PM_RESUME_WAIT(sdioh_request_buffer_wait);
+	DHD_PM_RESUME_RETURN_ERROR(SDIOH_API_RC_FAIL);
+
+	if (sd_msglevel & SDH_COST_VAL)
+		osl_do_gettimeofday(&before);
+
+	if (pkt) {
+#ifdef BCMSDIOH_TXGLOM
+		/* packet chain, only used for tx/rx glom, all packets length
+		 * are aligned, total length is a block multiple
+		 */
+		if (PKTNEXT(sd->osh, pkt))
+			return sdioh_request_packet_chain(sd, fix_inc, write, func, addr, pkt);
+#endif /* BCMSDIOH_TXGLOM */
+		/* non-glom mode, ignore the buffer parameter and use the packet pointer
+		 * (this shouldn't happen)
+		 */
+		buffer = PKTDATA(sd->osh, pkt);
+		buf_len = PKTLEN(sd->osh, pkt);
+	}
+
+	ASSERT(buffer);
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 24)
+	is_vmalloc = is_vmalloc_addr(buffer);
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 24) */
+
+	/* buffer and length are aligned, use it directly so we can avoid memory copy */
+	if ((((ulong)buffer & DMA_ALIGN_MASK) == 0) && ((buf_len & DMA_ALIGN_MASK) == 0) &&
+			(!is_vmalloc)) {
+		return sdioh_buffer_tofrom_bus(sd, fix_inc, write, func, addr, buffer, buf_len);
+	}
+
+	if (is_vmalloc) {
+		sd_trace(("%s: Need to memory copy due to virtual memory address.\n",
+			__FUNCTION__));
+	}
+
+	sd_trace(("%s: [%d] doing memory copy buf=%p, len=%d\n",
+		__FUNCTION__, write, buffer, buf_len));
+
+	/* otherwise, a memory copy is needed as the input buffer is not aligned */
+	tmppkt = PKTGET_STATIC(sd->osh, buf_len + DEFAULT_SDIO_F2_BLKSIZE, write ? TRUE : FALSE);
+	if (tmppkt == NULL) {
+		sd_err(("%s: PKTGET failed: len %d\n", __FUNCTION__, buf_len));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	if (write)
+		bcopy(buffer, PKTDATA(sd->osh, tmppkt), buf_len);
+
+	status = sdioh_buffer_tofrom_bus(sd, fix_inc, write, func, addr,
+		PKTDATA(sd->osh, tmppkt), ROUNDUP(buf_len, (DMA_ALIGN_MASK+1)));
+
+	if (!write)
+		bcopy(PKTDATA(sd->osh, tmppkt), buffer, buf_len);
+
+	PKTFREE_STATIC(sd->osh, tmppkt, write ? TRUE : FALSE);
+
+	if (sd_msglevel & SDH_COST_VAL) {
+		uint32 diff_us;
+		osl_do_gettimeofday(&now);
+		diff_us = osl_do_gettimediff(&now, &before);
+		sd_cost(("%s: rw=%d, len=%d cost = %3dms %3dus\n", __FUNCTION__,
+			write, buf_len, diff_us/1000, diff_us%1000));
+	}
+
+	return status;
+}
+
+/* this function performs "abort" for both of host & device */
+extern int
+sdioh_abort(sdioh_info_t *sd, uint func)
+{
+#if defined(MMC_SDIO_ABORT)
+	char t_func = (char) func;
+#endif /* defined(MMC_SDIO_ABORT) */
+	sd_trace(("%s: Enter\n", __FUNCTION__));
+
+	/* XXX Standard Linux SDIO Stack cannot perform an abort. */
+#if defined(MMC_SDIO_ABORT)
+	/* issue abort cmd52 command through F1 */
+	sdioh_request_byte(sd, SD_IO_OP_WRITE, SDIO_FUNC_0, SDIOD_CCCR_IOABORT, &t_func);
+#endif /* defined(MMC_SDIO_ABORT) */
+
+	sd_trace(("%s: Exit\n", __FUNCTION__));
+	return SDIOH_API_RC_SUCCESS;
+}
+
+/* Reset and re-initialize the device */
+int sdioh_sdio_reset(sdioh_info_t *si)
+{
+	sd_trace(("%s: Enter\n", __FUNCTION__));
+	sd_trace(("%s: Exit\n", __FUNCTION__));
+	return SDIOH_API_RC_SUCCESS;
+}
+
+/* Disable device interrupt */
+void
+sdioh_sdmmc_devintr_off(sdioh_info_t *sd)
+{
+	sd_trace(("%s: %d\n", __FUNCTION__, sd->use_client_ints));
+	sd->intmask &= ~CLIENT_INTR;
+}
+
+/* Enable device interrupt */
+void
+sdioh_sdmmc_devintr_on(sdioh_info_t *sd)
+{
+	sd_trace(("%s: %d\n", __FUNCTION__, sd->use_client_ints));
+	sd->intmask |= CLIENT_INTR;
+}
+
+/* Read client card reg */
+int
+sdioh_sdmmc_card_regread(sdioh_info_t *sd, int func, uint32 regaddr, int regsize, uint32 *data)
+{
+
+	if ((func == 0) || (regsize == 1)) {
+		uint8 temp = 0;
+
+		sdioh_request_byte(sd, SDIOH_READ, func, regaddr, &temp);
+		*data = temp;
+		*data &= 0xff;
+		sd_data(("%s: byte read data=0x%02x\n",
+		         __FUNCTION__, *data));
+	} else {
+		if (sdioh_request_word(sd, 0, SDIOH_READ, func, regaddr, data, regsize)) {
+			return BCME_SDIO_ERROR;
+		}
+		if (regsize == 2)
+			*data &= 0xffff;
+
+		sd_data(("%s: word read data=0x%08x\n",
+		         __FUNCTION__, *data));
+	}
+
+	return SUCCESS;
+}
+
+#if !defined(OOB_INTR_ONLY)
+void sdio_claim_host_lock_local(sdioh_info_t *sd) // terence 20140926: fix for claim host issue
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25))
+	if (sd)
+		mutex_lock(&sd->claim_host_mutex);
+#endif
+}
+
+void sdio_claim_host_unlock_local(sdioh_info_t *sd) // terence 20140926: fix for claim host issue
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25))
+	if (sd)
+		mutex_unlock(&sd->claim_host_mutex);
+#endif
+}
+
+/* bcmsdh_sdmmc interrupt handler */
+static void IRQHandler(struct sdio_func *func)
+{
+	sdioh_info_t *sd;
+
+	sd = sdio_get_drvdata(func);
+
+	ASSERT(sd != NULL);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25))
+	if (mutex_is_locked(&sd->claim_host_mutex)) {
+		printf("%s: muxtex is locked and return\n", __FUNCTION__);
+		return;
+	}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)) */
+
+	sdio_claim_host_lock_local(sd);
+	sdio_release_host(sd->func[0]);
+
+	if (sd->use_client_ints) {
+		sd->intrcount++;
+		ASSERT(sd->intr_handler);
+		ASSERT(sd->intr_handler_arg);
+		(sd->intr_handler)(sd->intr_handler_arg);
+	} else { /* XXX - Do not remove these sd_err messages.  Need to figure
+			          out how to keep interrupts disabled until DHD registers
+			          a handler.
+			  */
+		sd_err(("bcmsdh_sdmmc: ***IRQHandler\n"));
+
+		sd_err(("%s: Not ready for intr: enabled %d, handler %p\n",
+		        __FUNCTION__, sd->client_intr_enabled, sd->intr_handler));
+	}
+
+	sdio_claim_host(sd->func[0]);
+	sdio_claim_host_unlock_local(sd);
+}
+
+/* bcmsdh_sdmmc interrupt handler for F2 (dummy handler) */
+static void IRQHandlerF2(struct sdio_func *func)
+{
+	sd_trace(("bcmsdh_sdmmc: ***IRQHandlerF2\n"));
+}
+#endif /* !defined(OOB_INTR_ONLY) */
+
+#ifdef NOTUSED
+/* Write client card reg */
+static int
+sdioh_sdmmc_card_regwrite(sdioh_info_t *sd, int func, uint32 regaddr, int regsize, uint32 data)
+{
+
+	if ((func == 0) || (regsize == 1)) {
+		uint8 temp;
+
+		temp = data & 0xff;
+		sdioh_request_byte(sd, SDIOH_READ, func, regaddr, &temp);
+		sd_data(("%s: byte write data=0x%02x\n",
+		         __FUNCTION__, data));
+	} else {
+		if (regsize == 2)
+			data &= 0xffff;
+
+		sdioh_request_word(sd, 0, SDIOH_READ, func, regaddr, &data, regsize);
+
+		sd_data(("%s: word write data=0x%08x\n",
+		         __FUNCTION__, data));
+	}
+
+	return SUCCESS;
+}
+#endif /* NOTUSED */
+
+#if defined(ENABLE_INSMOD_NO_FW_LOAD)
+static int sdio_sw_reset(sdioh_info_t *sd)
+{
+	struct mmc_card *card = sd->func[0]->card;
+	int err = 0;
+
+#if defined(MMC_SW_RESET) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+	/* MMC_SW_RESET */
+	sd_info(("%s: call mmc_sw_reset\n", __FUNCTION__));
+	sdio_claim_host(sd->func[0]);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
+	err = mmc_sw_reset(card);
+#else
+	err = mmc_sw_reset(card->host);
+#endif
+	sdio_release_host(sd->func[0]);
+#elif defined(MMC_HW_RESET) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)
+	/* MMC_HW_RESET */
+	sd_info(("%s: call mmc_hw_reset\n", __FUNCTION__));
+	sdio_claim_host(sd->func[0]);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
+	while (atomic_read(&card->sdio_funcs_probed) > 1) {
+		atomic_dec(&card->sdio_funcs_probed);
+	}
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+	err = mmc_hw_reset(card);
+#else
+	err = mmc_hw_reset(card->host);
+#endif
+	sdio_release_host(sd->func[0]);
+#elif defined(BUS_POWER_RESTORE) && \
+LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
+	sd_info(("%s: call mmc_power_restore_host\n", __FUNCTION__));
+	mmc_power_restore_host(card->host);
+#else
+	/* sdio_reset_comm */
+	err = sdio_reset_comm(card);
+#endif
+
+	if (err)
+		sd_err(("%s Failed, error = %d\n", __FUNCTION__, err));
+
+	return err;
+}
+#endif
+
+int
+sdioh_start(sdioh_info_t *sd, int stage)
+{
+#if defined(OEM_ANDROID)
+	int ret = 0;
+
+	if (!sd) {
+		sd_err(("%s Failed, sd is NULL\n", __FUNCTION__));
+		return (0);
+	}
+
+	/* Need to do this stages as we can't enable the interrupt till
+		downloading of the firmware is complete, other wise polling
+		sdio access will come in way
+	*/
+	if (sd->func[0]) {
+		if (stage == 0) {
+			/* Since the power to the chip is killed, we will have
+			   re enumerate the device again. Set the block size
+			   and enable the fucntion 1 for in preparation for
+			   downloading the code
+			*/
+			/* sdio_reset_comm() - has been fixed in latest kernel/msm.git for Linux
+			   2.6.27. The implementation prior to that is buggy, and needs broadcom's
+			   patch for it
+			*/
+#if defined(ENABLE_INSMOD_NO_FW_LOAD)
+			if ((ret = sdio_sw_reset(sd))) {
+				sd_err(("%s Failed, error = %d\n", __FUNCTION__, ret));
+				return ret;
+			} else
+#endif
+			{
+				sd->num_funcs = 2;
+				sd->sd_blockmode = TRUE;
+				sd->use_client_ints = TRUE;
+				sd->client_block_size[0] = 64;
+
+				if (sd->func[1]) {
+					/* Claim host controller */
+					sdio_claim_host(sd->func[1]);
+
+					sd->client_block_size[1] = 64;
+					ret = sdio_set_block_size(sd->func[1], 64);
+					if (ret) {
+						sd_err(("bcmsdh_sdmmc: Failed to set F1 "
+							"blocksize(%d)\n", ret));
+					}
+
+					/* Release host controller F1 */
+					sdio_release_host(sd->func[1]);
+				}
+
+				if (sd->func[2]) {
+					/* Claim host controller F2 */
+					sdio_claim_host(sd->func[2]);
+
+					sd->client_block_size[2] = sd_f2_blocksize;
+					sd_info(("%s: set sd_f2_blocksize %d\n", __FUNCTION__, sd_f2_blocksize));
+					ret = sdio_set_block_size(sd->func[2], sd_f2_blocksize);
+					if (ret) {
+						sd_err(("bcmsdh_sdmmc: Failed to set F2 blocksize"
+							" to %d(%d)\n", sd_f2_blocksize, ret));
+					}
+
+					/* Release host controller F2 */
+					sdio_release_host(sd->func[2]);
+				}
+
+				sdioh_sdmmc_card_enablefuncs(sd);
+			}
+		} else {
+#if !defined(OOB_INTR_ONLY)
+			sdio_claim_host(sd->func[0]);
+			if (sd->func[2])
+				sdio_claim_irq(sd->func[2], IRQHandlerF2);
+			if (sd->func[1])
+				sdio_claim_irq(sd->func[1], IRQHandler);
+			sdio_release_host(sd->func[0]);
+#else /* defined(OOB_INTR_ONLY) */
+#if defined(HW_OOB)
+			sdioh_enable_func_intr(sd);
+#endif
+			bcmsdh_oob_intr_set(sd->bcmsdh, TRUE);
+#endif /* !defined(OOB_INTR_ONLY) */
+		}
+	} else
+		sd_err(("%s Failed\n", __FUNCTION__));
+#endif /* defined(OEM_ANDROID) */
+
+	return (ret);
+}
+
+int
+sdioh_stop(sdioh_info_t *sd)
+{
+#if defined(OEM_ANDROID)
+	/* MSM7201A Android sdio stack has bug with interrupt
+		So internaly within SDIO stack they are polling
+		which cause issue when device is turned off. So
+		unregister interrupt with SDIO stack to stop the
+		polling
+	*/
+#if !defined(OOB_INTR_ONLY)
+	sdio_claim_host_lock_local(sd);
+#endif
+	if (sd->func[0]) {
+#if !defined(OOB_INTR_ONLY)
+		sdio_claim_host(sd->func[0]);
+		if (sd->func[1])
+			sdio_release_irq(sd->func[1]);
+		if (sd->func[2])
+			sdio_release_irq(sd->func[2]);
+		sdio_release_host(sd->func[0]);
+#else /* defined(OOB_INTR_ONLY) */
+#if defined(HW_OOB)
+		sdioh_disable_func_intr(sd);
+#endif
+		bcmsdh_oob_intr_set(sd->bcmsdh, FALSE);
+#endif /* !defined(OOB_INTR_ONLY) */
+	}
+	else
+		sd_err(("%s Failed\n", __FUNCTION__));
+#if !defined(MMC_SW_RESET) && !defined(MMC_HW_RESET)
+#if defined(BUS_POWER_RESTORE) && \
+LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
+	if (sd->func[0]) {
+		struct mmc_card *card = sd->func[0]->card;
+		printf("%s: call mmc_power_save_host\n", __FUNCTION__);
+		mmc_power_save_host(card->host);
+	}
+#endif
+#endif
+#if !defined(OOB_INTR_ONLY)
+	sdio_claim_host_unlock_local(sd);
+#endif
+#endif /* defined(OEM_ANDROID) */
+	return (0);
+}
+
+int
+sdioh_waitlockfree(sdioh_info_t *sd)
+{
+	return (1);
+}
+
+SDIOH_API_RC
+sdioh_gpioouten(sdioh_info_t *sd, uint32 gpio)
+{
+	return SDIOH_API_RC_FAIL;
+}
+
+SDIOH_API_RC
+sdioh_gpioout(sdioh_info_t *sd, uint32 gpio, bool enab)
+{
+	return SDIOH_API_RC_FAIL;
+}
+
+bool
+sdioh_gpioin(sdioh_info_t *sd, uint32 gpio)
+{
+	return FALSE;
+}
+
+SDIOH_API_RC
+sdioh_gpio_init(sdioh_info_t *sd)
+{
+	return SDIOH_API_RC_FAIL;
+}
+
+uint8
+sdmmc_get_timing(sdioh_info_t *sd)
+{
+	struct sdio_func *sdio_func = sd->func[0];
+	struct mmc_host *host = sdio_func->card->host;
+	return host->ios.timing;
+}
+
+void
+sdmmc_set_timing(sdioh_info_t *sd, uint8 timing)
+{
+	struct sdio_func *sdio_func = sd->func[0];
+	struct mmc_host *host = sdio_func->card->host;
+	struct mmc_ios *ios = &host->ios;
+
+	mmc_host_clk_hold(host);
+	DHD_INFO(("%s: Before change: sd current timing is %u\n", __FUNCTION__, ios->timing));
+	ios->timing = timing;
+	host->ops->set_ios(host, ios);
+	DHD_ERROR(("%s: After change: sd timing is %u\n", __FUNCTION__, ios->timing));
+	mmc_host_clk_release(host);
+}
+
+uint
+sdmmc_get_clock_rate(sdioh_info_t *sd)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
+	struct sdio_func *sdio_func = sd->func[0];
+	struct mmc_host *host = sdio_func->card->host;
+	return mmc_host_clk_rate(host);
+#else
+	return 0;
+#endif
+}
+
+void
+sdmmc_set_clock_rate(sdioh_info_t *sd, uint hz)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
+	struct sdio_func *sdio_func = sd->func[0];
+	struct mmc_host *host = sdio_func->card->host;
+	struct mmc_ios *ios = &host->ios;
+
+	mmc_host_clk_hold(host);
+	DHD_INFO(("%s: Before change: sd clock rate is %u\n", __FUNCTION__, ios->clock));
+	if (hz < host->f_min) {
+		DHD_ERROR(("%s: Intended rate is below min rate, setting to min\n", __FUNCTION__));
+		hz = host->f_min;
+	}
+
+	if (hz > host->f_max) {
+		DHD_ERROR(("%s: Intended rate exceeds max rate, setting to max\n", __FUNCTION__));
+		hz = host->f_max;
+	}
+	ios->clock = hz;
+	host->ops->set_ios(host, ios);
+	DHD_ERROR(("%s: After change: sd clock rate is %u\n", __FUNCTION__, ios->clock));
+	mmc_host_clk_release(host);
+#else
+	return;
+#endif
+}
+
+void
+sdmmc_set_clock_divisor(sdioh_info_t *sd, uint sd_div)
+{
+	uint hz;
+	uint old_div = sdmmc_get_clock_rate(sd);
+	if (old_div == sd_div) {
+		return;
+	}
+
+	hz = sd->sd_clk_rate / sd_div;
+	sdmmc_set_clock_rate(sd, hz);
+}
+
+uint
+sdioh_set_mode(sdioh_info_t *sd, uint mode)
+{
+	if (mode == SDPCM_TXGLOM_CPY)
+		sd->txglom_mode = mode;
+	else if (mode == SDPCM_TXGLOM_MDESC)
+		sd->txglom_mode = mode;
+
+	return (sd->txglom_mode);
+}
diff --git a/drivers/net/wireless/ap12275/bcmsdh_sdmmc_linux.c b/drivers/net/wireless/ap12275/bcmsdh_sdmmc_linux.c
new file mode 100644
index 000000000000..78d0067413c9
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdh_sdmmc_linux.c
@@ -0,0 +1,421 @@
+/*
+ * BCMSDH Function Driver for the native SDIO/MMC driver in the Linux Kernel
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <bcmutils.h>
+#include <sdio.h>	/* SDIO Device and Protocol Specs */
+#include <bcmsdbus.h>	/* bcmsdh to/from specific controller APIs */
+#include <sdiovar.h>	/* to get msglevel bit values */
+
+#include <linux/sched.h>	/* request_irq() */
+
+#include <linux/mmc/core.h>
+#include <linux/mmc/card.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/sdio_func.h>
+#include <linux/mmc/sdio_ids.h>
+#include <dhd_linux.h>
+#include <bcmsdh_sdmmc.h>
+#include <dhd_dbg.h>
+#include <bcmdevs.h>
+
+#if !defined(SDIO_VENDOR_ID_BROADCOM)
+#define SDIO_VENDOR_ID_BROADCOM		0x02d0
+#endif /* !defined(SDIO_VENDOR_ID_BROADCOM) */
+
+#if !defined(SDIO_VENDOR_ID_SYNAPTICS)
+#define SDIO_VENDOR_ID_SYNAPTICS	0x06cb
+#endif /* !defined(SDIO_VENDOR_ID_SYNAPTICS) */
+
+#define SDIO_DEVICE_ID_BROADCOM_DEFAULT	0x0000
+
+extern void wl_cfg80211_set_parent_dev(void *dev);
+extern void sdioh_sdmmc_devintr_off(sdioh_info_t *sd);
+extern void sdioh_sdmmc_devintr_on(sdioh_info_t *sd);
+extern void* bcmsdh_probe(osl_t *osh, void *dev, void *sdioh, void *adapter_info, uint bus_type,
+	uint bus_num, uint slot_num);
+extern int bcmsdh_remove(bcmsdh_info_t *bcmsdh);
+
+int sdio_function_init(void);
+void sdio_function_cleanup(void);
+
+#define DESCRIPTION "bcmsdh_sdmmc Driver"
+#define AUTHOR "Broadcom Corporation"
+
+/* module param defaults */
+static int clockoverride = 0;
+
+module_param(clockoverride, int, 0644);
+MODULE_PARM_DESC(clockoverride, "SDIO card clock override");
+
+#ifdef GLOBAL_SDMMC_INSTANCE
+PBCMSDH_SDMMC_INSTANCE gInstance;
+#endif
+
+/* Maximum number of bcmsdh_sdmmc devices supported by driver */
+#define BCMSDH_SDMMC_MAX_DEVICES 1
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM_SLEEP)
+extern volatile bool dhd_mmc_suspend;
+#endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM_SLEEP) */
+
+static int sdioh_probe(struct sdio_func *func)
+{
+	int host_idx = func->card->host->index;
+	uint32 rca = func->card->rca;
+	wifi_adapter_info_t *adapter;
+	osl_t *osh = NULL;
+	sdioh_info_t *sdioh = NULL;
+
+	sd_info(("bus num (host idx)=%d, slot num (rca)=%d, caps=0x%x\n",
+		host_idx, rca, func->card->host->caps));
+	adapter = dhd_wifi_platform_get_adapter(SDIO_BUS, host_idx, rca);
+	if (adapter != NULL) {
+		sd_info(("found adapter info '%s'\n", adapter->name));
+		adapter->bus_type = SDIO_BUS;
+		adapter->bus_num = host_idx;
+		adapter->slot_num = rca;
+		adapter->sdio_func = func;
+	} else {
+		sd_err(("can't find adapter info for this chip\n"));
+#ifdef ADAPTER_IDX
+		goto fail;
+#endif
+	}
+
+#ifdef WL_CFG80211
+	wl_cfg80211_set_parent_dev(&func->dev);
+#endif
+
+	/* allocate SDIO Host Controller state info */
+	osh = osl_attach(&func->dev, SDIO_BUS, TRUE);
+	if (osh == NULL) {
+		sd_err(("%s: osl_attach failed\n", __FUNCTION__));
+		goto fail;
+	}
+	osl_static_mem_init(osh, adapter);
+	sdioh = sdioh_attach(osh, func);
+	if (sdioh == NULL) {
+		sd_err(("%s: sdioh_attach failed\n", __FUNCTION__));
+		goto fail;
+	}
+	if (!(func->card->host->caps & MMC_CAP_NONREMOVABLE)) {
+		sd_info(("%s: MMC_CAP_NONREMOVABLE not enabled in SDIO driver\n", __FUNCTION__));
+//		func->card->host->caps |= MMC_CAP_NONREMOVABLE;
+	}
+	if ((func->card->host->caps & MMC_CAP_NEEDS_POLL)) {
+		sd_info(("%s: MMC_CAP_NEEDS_POLL enabled in SDIO driver\n", __FUNCTION__));
+//		func->card->host->caps &= ~MMC_CAP_NEEDS_POLL;
+	}
+	sdioh->bcmsdh = bcmsdh_probe(osh, &func->dev, sdioh, adapter, SDIO_BUS, host_idx, rca);
+	if (sdioh->bcmsdh == NULL) {
+		sd_err(("%s: bcmsdh_probe failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	sdio_set_drvdata(func, sdioh);
+	return 0;
+
+fail:
+	if (sdioh != NULL)
+		sdioh_detach(osh, sdioh);
+	if (osh != NULL)
+		osl_detach(osh);
+	return -ENOMEM;
+}
+
+static void sdioh_remove(struct sdio_func *func)
+{
+	sdioh_info_t *sdioh;
+	osl_t *osh;
+
+	sdioh = sdio_get_drvdata(func);
+	if (sdioh == NULL) {
+		sd_err(("%s: error, no sdioh handler found\n", __FUNCTION__));
+		return;
+	}
+	sd_err(("%s: Enter\n", __FUNCTION__));
+
+	osh = sdioh->osh;
+	bcmsdh_remove(sdioh->bcmsdh);
+	sdioh_detach(osh, sdioh);
+	osl_detach(osh);
+}
+
+static int bcmsdh_sdmmc_probe(struct sdio_func *func,
+                              const struct sdio_device_id *id)
+{
+	int ret = 0;
+
+	if (func == NULL)
+		return -EINVAL;
+
+	sd_info(("%s: Enter num=%d\n", __FUNCTION__, func->num));
+	sd_info(("sdio_bcmsdh: func->class=%x\n", func->class));
+	sd_info(("sdio_vendor: 0x%04x\n", func->vendor));
+	sd_info(("sdio_device: 0x%04x\n", func->device));
+	sd_info(("Function#: 0x%04x\n", func->num));
+
+#ifdef GLOBAL_SDMMC_INSTANCE
+	gInstance->func[func->num] = func;
+#endif
+
+	/* 4318 doesn't have function 2 */
+	if ((func->num == 2) || (func->num == 1 && func->device == 0x4))
+		ret = sdioh_probe(func);
+
+	return ret;
+}
+
+static void bcmsdh_sdmmc_remove(struct sdio_func *func)
+{
+	if (func == NULL) {
+		sd_err(("%s is called with NULL SDIO function pointer\n", __FUNCTION__));
+		return;
+	}
+
+	sd_trace(("bcmsdh_sdmmc: %s Enter\n", __FUNCTION__));
+	sd_info(("sdio_bcmsdh: func->class=%x\n", func->class));
+	sd_info(("sdio_vendor: 0x%04x\n", func->vendor));
+	sd_info(("sdio_device: 0x%04x\n", func->device));
+	sd_info(("Function#: 0x%04x\n", func->num));
+
+	if ((func->num == 2) || (func->num == 1 && func->device == 0x4))
+		sdioh_remove(func);
+}
+
+/* devices we support, null terminated */
+static const struct sdio_device_id bcmsdh_sdmmc_ids[] = {
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, SDIO_DEVICE_ID_BROADCOM_DEFAULT) },
+	/* XXX This should not be in the external release, as it will attach to any SDIO
+	 * device, even non-WLAN devices.
+	 * Need to add IDs for the FALCON-based chips and put this under BCMINTERNAL
+	 { SDIO_DEVICE_CLASS(SDIO_CLASS_NONE) },
+	 */
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM4362_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43751_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43752_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43756_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43012_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43014_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43014_D11N_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43014_D11N2G_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43014_D11N5G_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43013_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43013_D11N_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43013_D11N2G_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM43013_D11N5G_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, SDIO_ANY_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM4381_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_BROADCOM, BCM4382_CHIP_ID) },
+	{ SDIO_DEVICE(SDIO_VENDOR_ID_SYNAPTICS, BCM43711_CHIP_ID) },
+	{ SDIO_DEVICE_CLASS(SDIO_CLASS_NONE)		},
+	{ 0, 0, 0, 0 /* end: all zeroes */
+	},
+};
+
+MODULE_DEVICE_TABLE(sdio, bcmsdh_sdmmc_ids);
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM_SLEEP)
+static int bcmsdh_sdmmc_suspend(struct device *pdev)
+{
+	int err;
+	sdioh_info_t *sdioh;
+	struct sdio_func *func = dev_to_sdio_func(pdev);
+	mmc_pm_flag_t sdio_flags;
+
+	printf("%s Enter func->num=%d\n", __FUNCTION__, func->num);
+	if (func->num != 2)
+		return 0;
+
+	dhd_mmc_suspend = TRUE;
+	sdioh = sdio_get_drvdata(func);
+	err = bcmsdh_suspend(sdioh->bcmsdh);
+	if (err) {
+		printf("%s bcmsdh_suspend err=%d\n", __FUNCTION__, err);
+		dhd_mmc_suspend = FALSE;
+		return err;
+	}
+
+	sdio_flags = sdio_get_host_pm_caps(func);
+	if (!(sdio_flags & MMC_PM_KEEP_POWER)) {
+		sd_err(("%s: can't keep power while host is suspended\n", __FUNCTION__));
+		dhd_mmc_suspend = FALSE;
+		return  -EINVAL;
+	}
+
+	/* keep power while host suspended */
+	err = sdio_set_host_pm_flags(func, MMC_PM_KEEP_POWER);
+	if (err) {
+		sd_err(("%s: error while trying to keep power\n", __FUNCTION__));
+		dhd_mmc_suspend = FALSE;
+		return err;
+	}
+	smp_mb();
+
+	printf("%s Exit\n", __FUNCTION__);
+	return 0;
+}
+
+static int bcmsdh_sdmmc_resume(struct device *pdev)
+{
+	sdioh_info_t *sdioh;
+	struct sdio_func *func = dev_to_sdio_func(pdev);
+
+	printf("%s Enter func->num=%d\n", __FUNCTION__, func->num);
+	if (func->num != 2)
+		return 0;
+
+	dhd_mmc_suspend = FALSE;
+	sdioh = sdio_get_drvdata(func);
+	bcmsdh_resume(sdioh->bcmsdh);
+
+	smp_mb();
+	printf("%s Exit\n", __FUNCTION__);
+	return 0;
+}
+
+static const struct dev_pm_ops bcmsdh_sdmmc_pm_ops = {
+	.suspend	= bcmsdh_sdmmc_suspend,
+	.resume		= bcmsdh_sdmmc_resume,
+};
+#endif  /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM_SLEEP) */
+
+#if defined(BCMLXSDMMC)
+static struct semaphore *notify_semaphore = NULL;
+
+static int dummy_probe(struct sdio_func *func,
+                              const struct sdio_device_id *id)
+{
+	if (func)
+		sd_info(("%s: func->num=0x%x; \n", __FUNCTION__, func->num));
+	if (id) {
+		sd_info(("%s: class=0x%x; vendor=0x%x; device=0x%x\n", __FUNCTION__,
+			id->class, id->vendor, id->device));
+		if ((id->vendor != SDIO_VENDOR_ID_BROADCOM) &&
+			(id->vendor != SDIO_VENDOR_ID_SYNAPTICS))
+				return -ENODEV;
+	}
+
+	if (func && (func->num != 2)) {
+		return 0;
+	}
+
+	if (notify_semaphore)
+		up(notify_semaphore);
+	return 0;
+}
+
+static void dummy_remove(struct sdio_func *func)
+{
+}
+
+static struct sdio_driver dummy_sdmmc_driver = {
+	.probe		= dummy_probe,
+	.remove		= dummy_remove,
+	.name		= "dummy_sdmmc",
+	.id_table	= bcmsdh_sdmmc_ids,
+	};
+
+int sdio_func_reg_notify(void* semaphore)
+{
+	notify_semaphore = semaphore;
+	return sdio_register_driver(&dummy_sdmmc_driver);
+}
+
+void sdio_func_unreg_notify(void)
+{
+	OSL_SLEEP(15);
+	sdio_unregister_driver(&dummy_sdmmc_driver);
+}
+
+#endif /* defined(BCMLXSDMMC) */
+
+static struct sdio_driver bcmsdh_sdmmc_driver = {
+	.probe		= bcmsdh_sdmmc_probe,
+	.remove		= bcmsdh_sdmmc_remove,
+	.name		= "bcmsdh_sdmmc",
+	.id_table	= bcmsdh_sdmmc_ids,
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM_SLEEP)
+	.drv = {
+	.pm	= &bcmsdh_sdmmc_pm_ops,
+	},
+#endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM_SLEEP) */
+	};
+
+struct sdos_info {
+	sdioh_info_t *sd;
+	spinlock_t lock;
+};
+
+/* Interrupt enable/disable */
+SDIOH_API_RC
+sdioh_interrupt_set(sdioh_info_t *sd, bool enable)
+{
+	if (!sd)
+		return BCME_BADARG;
+
+	sd_trace(("%s: %s\n", __FUNCTION__, enable ? "Enabling" : "Disabling"));
+	return SDIOH_API_RC_SUCCESS;
+}
+
+#ifdef BCMSDH_MODULE
+static int __init
+bcmsdh_module_init(void)
+{
+	int error = 0;
+	error = sdio_function_init();
+	return error;
+}
+
+static void __exit
+bcmsdh_module_cleanup(void)
+{
+	sdio_function_cleanup();
+}
+
+module_init(bcmsdh_module_init);
+module_exit(bcmsdh_module_cleanup);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION(DESCRIPTION);
+MODULE_AUTHOR(AUTHOR);
+
+#endif /* BCMSDH_MODULE */
+/*
+ * module init
+*/
+int bcmsdh_register_client_driver(void)
+{
+	return sdio_register_driver(&bcmsdh_sdmmc_driver);
+}
+
+/*
+ * module cleanup
+*/
+void bcmsdh_unregister_client_driver(void)
+{
+	sdio_unregister_driver(&bcmsdh_sdmmc_driver);
+}
diff --git a/drivers/net/wireless/ap12275/bcmsdstd.c b/drivers/net/wireless/ap12275/bcmsdstd.c
new file mode 100644
index 000000000000..e7dfab4fee3b
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdstd.c
@@ -0,0 +1,5308 @@
+/*
+ *  'Standard' SDIO HOST CONTROLLER driver
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+
+#include <bcmdevs.h>
+#include <bcmendian.h>
+#include <bcmutils.h>
+#include <osl.h>
+#include <siutils.h>
+#include <sdio.h>	/* SDIO Device and Protocol Specs */
+#include <sdioh.h>	/* Standard SDIO Host Controller Specification */
+#include <bcmsdbus.h>	/* bcmsdh to/from specific controller APIs */
+#include <sdiovar.h>	/* ioctl/iovars */
+#include <pcicfg.h>
+#include <bcmsdstd.h>
+/* XXX Quick NDIS hack */
+#ifdef NDIS
+#define inline  __inline
+#define	PCI_CFG_VID		0
+#define	PCI_CFG_BAR0	0x10
+#endif
+
+#define SD_PAGE_BITS	12
+#define SD_PAGE 	(1 << SD_PAGE_BITS)
+#define SDSTD_MAX_TUNING_PHASE	5
+
+/*
+ * Upper GPIO 16 - 31 are available on J22
+ *   J22.pin3 == gpio16, J22.pin5 == gpio17, etc.
+ * Lower GPIO 0 - 15 are available on J15 (WL_GPIO)
+ */
+#define SDH_GPIO16		16
+#define SDH_GPIO_ENABLE		0xffff
+
+#include <bcmsdstd.h>
+#include <sbsdio.h> /* SDIOH (host controller) core hardware definitions */
+
+/* Globals */
+uint sd_msglevel = SDH_ERROR_VAL;
+
+uint sd_hiok = TRUE;			/* Use hi-speed mode if available? */
+uint sd_sdmode = SDIOH_MODE_SD4;	/* Use SD4 mode by default */
+uint sd_f2_blocksize = 256;		/* Default blocksize */
+uint sd_f1_blocksize = BLOCK_SIZE_4318;		/* Default blocksize */
+
+#define sd3_trace(x)
+
+/*	sd3ClkMode: 	0-SDR12 [25MHz]
+ *				1-SDR25 [50MHz]+SHS=1
+ *				2-SDR50 [100MHz]+SSDR50=1
+ *				3-SDR104 [208MHz]+SSDR104=1
+ *				4-DDR50 [50MHz]+SDDR50=1
+ */
+#define SD3CLKMODE_0_SDR12	(0)
+#define SD3CLKMODE_1_SDR25	(1)
+#define SD3CLKMODE_2_SDR50	(2)
+#define SD3CLKMODE_3_SDR104	(3)
+#define SD3CLKMODE_4_DDR50	(4)
+#define SD3CLKMODE_DISABLED	(-1)
+#define SD3CLKMODE_AUTO		(99)
+
+/* values for global_UHSI_Supp : Means host and card caps match. */
+#define HOST_SDR_UNSUPP			(0)
+#define HOST_SDR_12_25			(1)
+#define HOST_SDR_50_104_DDR		(2)
+
+/* depends-on/affects sd3_autoselect_uhsi_max.
+ *	see sd3_autoselect_uhsi_max
+ */
+int sd_uhsimode = SD3CLKMODE_DISABLED;
+uint sd_tuning_period = CAP3_RETUNING_TC_OTHER;
+uint sd_delay_value = 500000;
+/* Enables host to dongle glomming. Also increases the
+ * dma buffer size. This will increase the rx throughput
+ * as there will be lesser CMD53 transactions
+ */
+#ifdef BCMSDIOH_TXGLOM
+uint sd_txglom;
+#ifdef LINUX
+module_param(sd_txglom, uint, 0);
+#endif
+#endif /* BCMSDIOH_TXGLOM */
+
+char dhd_sdiod_uhsi_ds_override[2] = {' '};
+
+#define MAX_DTS_INDEX		(3)
+#define DRVSTRN_MAX_CHAR	('D')
+#define DRVSTRN_IGNORE_CHAR	(' ')
+
+char DTS_vals[MAX_DTS_INDEX + 1] = {
+	0x1, /* Driver Strength Type-A */
+	0x0, /* Driver Strength Type-B */
+	0x2, /* Driver Strength Type-C */
+	0x3, /* Driver Strength Type-D */
+	};
+
+/* depends-on/affects sd_uhsimode.
+	select MAX speed automatically based on caps of host and card.
+	If this is 1, sd_uhsimode will be ignored. If the sd_uhsimode is set
+	by the user specifically, this var becomes 0. default value: 0. [XXX:TBD: for future]
+	*/
+uint32 sd3_autoselect_uhsi_max = 0;
+
+#define MAX_TUNING_ITERS		(40)
+/* (150+10)millisecs total time; so dividing it for per-loop */
+#define PER_TRY_TUNING_DELAY_MS		(160/MAX_TUNING_ITERS)
+#define CLKTUNING_MAX_BRR_RETRIES	(1000)	/* 1 ms: 1000 retries with 1 us delay per loop */
+
+/* table analogous to preset value register.
+*	This is bcos current HC doesn't have preset value reg support.
+*	All has DrvStr as 'B' [val:0] and CLKGEN as 0.
+*/
+static unsigned short presetval_sw_table[] = {
+	0x0520, /* initialization:	DrvStr:'B' [0]; CLKGen:0;
+			* SDCLKFreqSel: 520 [division: 320*2 = 640: ~400 KHz]
+			*/
+	0x0008, /* default speed:DrvStr:'B' [0]; CLKGen:0;
+			* SDCLKFreqSel: 8 [division: 6*2 = 12: ~25 MHz]
+			*/
+	0x0004, /* High speed: DrvStr:'B' [0]; CLKGen:0;
+			* SDCLKFreqSel: 4 [division: 3*2 = 6: ~50 MHz]
+			*/
+	0x0008, /* SDR12:		DrvStr:'B' [0]; CLKGen:0;
+			* SDCLKFreqSel: 8 [division: 6*2 = 12: ~25 MHz]
+			*/
+	0x0004, /* SDR25:		DrvStr:'B' [0]; CLKGen:0;
+			* SDCLKFreqSel: 4 [division: 3*2 = 6: ~50 MHz]
+			*/
+	0x0001, /* SDR50:		DrvStr:'B' [0]; CLKGen:0;
+			* SDCLKFreqSel: 2 [division: 1*2 = 2: ~100 MHz]
+			*/
+	0x0001, /* SDR104:		DrvStr:'B' [0]; CLKGen:0;
+			SDCLKFreqSel: 1 [no division: ~255/~208 MHz]
+			*/
+	0x0002  /* DDR50:		DrvStr:'B' [0]; CLKGen:0;
+			SDCLKFreqSel: 4 [division: 3*2 = 6: ~50 MHz]
+			*/
+};
+
+#ifdef BCMFPGA_HW
+/*
+ * Voltage switch, clock tuning and preset are not supported in
+ * FPGA which operates at 6.25Mhz.
+ */
+bool sd3_sw_override1 = TRUE;
+#else
+/* This is to have software overrides to the hardware. Info follows:
+	For override [1]:	Preset registers: not supported
+	Voltage switch:  not supported
+	Clock Tuning: not supported
+*/
+bool sd3_sw_override1 = FALSE;
+#endif /* BCMFPGA_HW */
+bool sd3_sw_read_magic_bytes = FALSE;
+
+#define SD3_TUNING_REQD(sd, sd_uhsimode) ((sd_uhsimode != SD3CLKMODE_DISABLED) && \
+			(sd->version == HOST_CONTR_VER_3) &&	\
+			((sd_uhsimode == SD3CLKMODE_3_SDR104) ||	\
+			((sd_uhsimode == SD3CLKMODE_2_SDR50) &&	\
+			(GFIELD(sd->caps3, CAP3_TUNING_SDR50)))))
+
+/* find next power of 2 */
+#define NEXT_POW2(n)  {n--; n |= n>>1; n |= n>>2; n |= n>>4; n++;}
+
+#ifdef BCMSDYIELD
+bool sd_yieldcpu = TRUE;		/* Allow CPU yielding for buffer requests */
+uint sd_minyield = 0;			/* Minimum xfer size to allow CPU yield */
+bool sd_forcerb = FALSE;		/* Force sync readback in intrs_on/off */
+#endif
+
+/* XXX: Issues with CMD14 enter/exit sleep
+ * XXX: Temp fix for special CMD14 handling
+ */
+#define F1_SLEEPCSR_ADDR	0x1001F
+
+uint sd_divisor = 2;	/* Default 48MHz/2 = 24MHz
+							:might get changed in code for 208
+							*/
+
+uint sd_power = 1;		/* Default to SD Slot powered ON */
+uint sd_3_power_save = 1;		/* Default to SDIO 3.0 power save */
+uint sd_clock = 1;		/* Default to SD Clock turned ON */
+uint sd_pci_slot = 0xFFFFffff; /* Used to force selection of a particular PCI slot */
+uint8 sd_dma_mode = DMA_MODE_AUTO; /* Default to AUTO & program based on capability */
+
+/* XXX Base timeout counter value on 48MHz (2^20 @ 48MHz => 21845us)
+ * Could adjust by adding sd_divisor (to maintain bit count) but really
+ * need something more elaborate to do that right.  Still allows xfer
+ * of about 1000 bytes at 400KHz, so constant is ok.
+ * Timeout control N produces 2^(13+N) counter.
+ */
+uint sd_toctl = 7;
+static bool trap_errs = FALSE;
+
+static const char *dma_mode_description[] = { "PIO", "SDMA", "ADMA1", "32b ADMA2", "64b ADMA2" };
+
+/* Prototypes */
+static bool sdstd_start_clock(sdioh_info_t *sd, uint16 divisor);
+static uint16 sdstd_start_power(sdioh_info_t *sd, int volts_req);
+static bool sdstd_bus_width(sdioh_info_t *sd, int width);
+static int sdstd_set_highspeed_mode(sdioh_info_t *sd, bool HSMode);
+static int sdstd_set_dma_mode(sdioh_info_t *sd, int8 dma_mode);
+static int sdstd_card_enablefuncs(sdioh_info_t *sd);
+static void sdstd_cmd_getrsp(sdioh_info_t *sd, uint32 *rsp_buffer, int count);
+static int sdstd_cmd_issue(sdioh_info_t *sd, bool use_dma, uint32 cmd, uint32 arg);
+static int sdstd_card_regread(sdioh_info_t *sd, int func, uint32 regaddr,
+                              int regsize, uint32 *data);
+static int sdstd_card_regwrite(sdioh_info_t *sd, int func, uint32 regaddr,
+                               int regsize, uint32 data);
+static int sdstd_driver_init(sdioh_info_t *sd);
+static bool sdstd_reset(sdioh_info_t *sd, bool host_reset, bool client_reset);
+static int sdstd_card_buf(sdioh_info_t *sd, int rw, int func, bool fifo,
+                          uint32 addr, int nbytes, uint32 *data);
+static int sdstd_abort(sdioh_info_t *sd, uint func);
+static int sdstd_check_errs(sdioh_info_t *sdioh_info, uint32 cmd, uint32 arg);
+static int set_client_block_size(sdioh_info_t *sd, int func, int blocksize);
+static void sd_map_dma(sdioh_info_t * sd);
+static void sd_unmap_dma(sdioh_info_t * sd);
+static void sd_clear_adma_dscr_buf(sdioh_info_t *sd);
+static void sd_fill_dma_data_buf(sdioh_info_t *sd, uint8 data);
+static void sd_create_adma_descriptor(sdioh_info_t *sd,
+                                      uint32 index, uint32 addr_phys,
+                                      uint16 length, uint16 flags);
+static void sd_dump_adma_dscr(sdioh_info_t *sd);
+static void sdstd_dumpregs(sdioh_info_t *sd);
+
+static int sdstd_3_set_highspeed_uhsi_mode(sdioh_info_t *sd, int sd3ClkMode);
+static int sdstd_3_sigvoltswitch_proc(sdioh_info_t *sd);
+static int sdstd_3_get_matching_uhsi_clkmode(sdioh_info_t *sd,
+	int sd3_requested_clkmode);
+static bool sdstd_3_get_matching_drvstrn(sdioh_info_t *sd,
+	int sd3_requested_clkmode, uint32 *drvstrn, uint16 *presetval);
+static int sdstd_3_clock_wrapper(sdioh_info_t *sd);
+static int sdstd_clock_wrapper(sdioh_info_t *sd);
+
+#ifdef BCMDBG
+static void print_regs(sdioh_info_t *sd);
+#endif
+
+/*
+ * Private register access routines.
+ */
+
+/* 16 bit PCI regs */
+
+/* XXX This is a hack to satisfy the -Wmissing-prototypes warning */
+extern uint16 sdstd_rreg16(sdioh_info_t *sd, uint reg);
+uint16
+sdstd_rreg16(sdioh_info_t *sd, uint reg)
+{
+
+	volatile uint16 data = *(volatile uint16 *)(sd->mem_space + reg);
+	sd_ctrl(("16: R Reg 0x%02x, Data 0x%x\n", reg, data));
+	return data;
+}
+
+/* XXX This is a hack to satisfy the -Wmissing-prototypes warning */
+extern void sdstd_wreg16(sdioh_info_t *sd, uint reg, uint16 data);
+void
+sdstd_wreg16(sdioh_info_t *sd, uint reg, uint16 data)
+{
+	*(volatile uint16 *)(sd->mem_space + reg) = (uint16) data;
+	sd_ctrl(("16: W Reg 0x%02x, Data 0x%x\n", reg, data));
+}
+
+static void
+sdstd_or_reg16(sdioh_info_t *sd, uint reg, uint16 val)
+{
+	volatile uint16 data = *(volatile uint16 *)(sd->mem_space + reg);
+	sd_ctrl(("16: OR Reg 0x%02x, Val 0x%x\n", reg, val));
+	data |= val;
+	*(volatile uint16 *)(sd->mem_space + reg) = (uint16)data;
+
+}
+static void
+sdstd_mod_reg16(sdioh_info_t *sd, uint reg, int16 mask, uint16 val)
+{
+
+	volatile uint16 data = *(volatile uint16 *)(sd->mem_space + reg);
+	sd_ctrl(("16: MOD Reg 0x%02x, Mask 0x%x, Val 0x%x\n", reg, mask, val));
+	data &= ~mask;
+	data |= (val & mask);
+	*(volatile uint16 *)(sd->mem_space + reg) = (uint16)data;
+}
+
+/* 32 bit PCI regs */
+static uint32
+sdstd_rreg(sdioh_info_t *sd, uint reg)
+{
+	volatile uint32 data = *(volatile uint32 *)(sd->mem_space + reg);
+	sd_ctrl(("32: R Reg 0x%02x, Data 0x%x\n", reg, data));
+	return data;
+}
+static inline void
+sdstd_wreg(sdioh_info_t *sd, uint reg, uint32 data)
+{
+	*(volatile uint32 *)(sd->mem_space + reg) = (uint32)data;
+	sd_ctrl(("32: W Reg 0x%02x, Data 0x%x\n", reg, data));
+
+}
+
+/* 8 bit PCI regs */
+static inline void
+sdstd_wreg8(sdioh_info_t *sd, uint reg, uint8 data)
+{
+	*(volatile uint8 *)(sd->mem_space + reg) = (uint8)data;
+	sd_ctrl(("08: W Reg 0x%02x, Data 0x%x\n", reg, data));
+}
+static uint8
+sdstd_rreg8(sdioh_info_t *sd, uint reg)
+{
+	volatile uint8 data = *(volatile uint8 *)(sd->mem_space + reg);
+	sd_ctrl(("08: R Reg 0x%02x, Data 0x%x\n", reg, data));
+	return data;
+}
+
+/*
+ * Private work routines
+ */
+
+sdioh_info_t *glob_sd;
+
+/*
+ *  Public entry points & extern's
+ */
+extern sdioh_info_t *
+sdioh_attach(osl_t *osh, void *bar0, uint irq)
+{
+	sdioh_info_t *sd;
+
+	sd_trace(("%s\n", __FUNCTION__));
+	if ((sd = (sdioh_info_t *)MALLOC(osh, sizeof(sdioh_info_t))) == NULL) {
+		sd_err(("sdioh_attach: out of memory, malloced %d bytes\n", MALLOCED(osh)));
+		return NULL;
+	}
+	bzero((char *)sd, sizeof(sdioh_info_t));
+	glob_sd = sd;
+	sd->osh = osh;
+	if (sdstd_osinit(sd) != 0) {
+		sd_err(("%s:sdstd_osinit() failed\n", __FUNCTION__));
+		MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+		return NULL;
+	}
+	sd->mem_space = (volatile char *)sdstd_reg_map(osh, (ulong)bar0, SDIOH_REG_WINSZ);
+	sd_init_dma(sd);
+	sd->irq = irq;
+	if (sd->mem_space == NULL) {
+		sd_err(("%s:ioremap() failed\n", __FUNCTION__));
+		sdstd_osfree(sd);
+		MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+		return NULL;
+	}
+	sd_info(("%s:sd->mem_space = %p\n", __FUNCTION__, sd->mem_space));
+	sd->intr_handler = NULL;
+	sd->intr_handler_arg = NULL;
+	sd->intr_handler_valid = FALSE;
+
+	/* Set defaults */
+	sd->sd_blockmode = TRUE;
+	sd->use_client_ints = TRUE;
+	sd->sd_dma_mode = sd_dma_mode;
+
+	/* XXX Haven't figured out how to make bytemode work with dma */
+	if (!sd->sd_blockmode)
+		sd->sd_dma_mode = DMA_MODE_NONE;
+
+	if (sdstd_driver_init(sd) != SUCCESS) {
+		/* If host CPU was reset without resetting SD bus or
+		   SD device, the device will still have its RCA but
+		   driver no longer knows what it is (since driver has been restarted).
+		   go through once to clear the RCA and a gain reassign it.
+		 */
+		sd_info(("driver_init failed - Reset RCA and try again\n"));
+		if (sdstd_driver_init(sd) != SUCCESS) {
+			sd_err(("%s:driver_init() failed()\n", __FUNCTION__));
+			if (sd->mem_space) {
+				sdstd_reg_unmap(osh, (ulong)sd->mem_space, SDIOH_REG_WINSZ);
+				sd->mem_space = NULL;
+			}
+			sdstd_osfree(sd);
+			MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+			return (NULL);
+		}
+	}
+
+	/* XXX Needed for NDIS as its OSL checks for correct dma address width
+	 * This value is normally set by wlc_attach() which has yet to run
+	 */
+	OSL_DMADDRWIDTH(osh, 32);
+
+	/* Always map DMA buffers, so we can switch between DMA modes. */
+	sd_map_dma(sd);
+
+	if (sdstd_register_irq(sd, irq) != SUCCESS) {
+		sd_err(("%s: sdstd_register_irq() failed for irq = %d\n", __FUNCTION__, irq));
+		sdstd_free_irq(sd->irq, sd);
+		if (sd->mem_space) {
+			sdstd_reg_unmap(osh, (ulong)sd->mem_space, SDIOH_REG_WINSZ);
+			sd->mem_space = NULL;
+		}
+
+		sdstd_osfree(sd);
+		MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+		return (NULL);
+	}
+
+	sd_trace(("%s: Done\n", __FUNCTION__));
+	return sd;
+}
+
+extern SDIOH_API_RC
+sdioh_detach(osl_t *osh, sdioh_info_t *sd)
+{
+	sd_trace(("%s\n", __FUNCTION__));
+	if (sd) {
+		sd_unmap_dma(sd);
+		sdstd_wreg16(sd, SD_IntrSignalEnable, 0);
+		if (sd->sd3_tuning_reqd == TRUE) {
+			sdstd_3_osclean_tuning(sd);
+			sd->sd3_tuning_reqd = FALSE;
+		}
+		sd->sd3_tuning_disable = FALSE;
+		sd_trace(("%s: freeing irq %d\n", __FUNCTION__, sd->irq));
+		sdstd_free_irq(sd->irq, sd);
+		if (sd->card_init_done)
+			sdstd_reset(sd, 1, 1);
+		if (sd->mem_space) {
+			sdstd_reg_unmap(osh, (ulong)sd->mem_space, SDIOH_REG_WINSZ);
+			sd->mem_space = NULL;
+		}
+
+		sdstd_osfree(sd);
+		MFREE(sd->osh, sd, sizeof(sdioh_info_t));
+	}
+	return SDIOH_API_RC_SUCCESS;
+}
+
+/* Configure callback to client when we receive client interrupt */
+extern SDIOH_API_RC
+sdioh_interrupt_register(sdioh_info_t *sd, sdioh_cb_fn_t fn, void *argh)
+{
+	sd_trace(("%s: Entering\n", __FUNCTION__));
+	sd->intr_handler = fn;
+	sd->intr_handler_arg = argh;
+	sd->intr_handler_valid = TRUE;
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_interrupt_deregister(sdioh_info_t *sd)
+{
+	sd_trace(("%s: Entering\n", __FUNCTION__));
+	sd->intr_handler_valid = FALSE;
+	sd->intr_handler = NULL;
+	sd->intr_handler_arg = NULL;
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_interrupt_query(sdioh_info_t *sd, bool *onoff)
+{
+	sd_trace(("%s: Entering\n", __FUNCTION__));
+	*onoff = sd->client_intr_enabled;
+	return SDIOH_API_RC_SUCCESS;
+}
+
+#if defined(DHD_DEBUG) || defined(BCMDBG)
+extern bool
+sdioh_interrupt_pending(sdioh_info_t *sd)
+{
+	uint16 intrstatus;
+	intrstatus = sdstd_rreg16(sd, SD_IntrStatus);
+	return !!(intrstatus & CLIENT_INTR);
+}
+#endif
+
+uint
+sdioh_query_iofnum(sdioh_info_t *sd)
+{
+	return sd->num_funcs;
+}
+
+/* IOVar table */
+enum {
+	IOV_MSGLEVEL = 1,
+	IOV_BLOCKMODE,
+	IOV_BLOCKSIZE,
+	IOV_DMA,
+	IOV_USEINTS,
+	IOV_NUMINTS,
+	IOV_NUMLOCALINTS,
+	IOV_HOSTREG,
+	IOV_DEVREG,
+	IOV_DIVISOR,
+	IOV_SDMODE,
+	IOV_HISPEED,
+	IOV_HCIREGS,
+	IOV_POWER,
+	IOV_POWER_SAVE,
+	IOV_YIELDCPU,
+	IOV_MINYIELD,
+	IOV_FORCERB,
+	IOV_CLOCK,
+	IOV_UHSIMOD,
+	IOV_TUNEMOD,
+	IOV_TUNEDIS,
+	IOV_DS
+};
+
+const bcm_iovar_t sdioh_iovars[] = {
+	{"sd_msglevel",	IOV_MSGLEVEL,	0,	0, IOVT_UINT32,	0 },
+	{"sd_blockmode", IOV_BLOCKMODE,	0,	0, IOVT_BOOL,	0 },
+	{"sd_blocksize", IOV_BLOCKSIZE, 0,	0, IOVT_UINT32,	0 }, /* ((fn << 16) | size) */
+	{"sd_dma",	IOV_DMA,	0,	0, IOVT_UINT32,	0 },
+#ifdef BCMSDYIELD
+	{"sd_yieldcpu",	IOV_YIELDCPU,	0,	0, IOVT_BOOL,	0 },
+	{"sd_minyield",	IOV_MINYIELD,	0,	0, IOVT_UINT32,	0 },
+	{"sd_forcerb",	IOV_FORCERB,	0,	0, IOVT_BOOL,	0 },
+#endif
+	{"sd_ints",	IOV_USEINTS,	0,	0, IOVT_BOOL,	0 },
+	{"sd_numints",	IOV_NUMINTS,	0,	0, IOVT_UINT32,	0 },
+	{"sd_numlocalints", IOV_NUMLOCALINTS, 0, 0, IOVT_UINT32,	0 },
+	{"sd_hostreg",	IOV_HOSTREG,	0,	0, IOVT_BUFFER,	sizeof(sdreg_t) },
+	{"sd_devreg",	IOV_DEVREG,	0,	0, IOVT_BUFFER,	sizeof(sdreg_t)	},
+	{"sd_divisor",	IOV_DIVISOR,	0,	0, IOVT_UINT32,	0 },
+	{"sd_power",	IOV_POWER,	0,	0, IOVT_UINT32,	0 },
+	{"sd_power_save",	IOV_POWER_SAVE,	0,	0, IOVT_UINT32,	0 },
+	{"sd_clock",	IOV_CLOCK,	0,	0, IOVT_UINT32,	0 },
+	{"sd_mode",	IOV_SDMODE,	0,	0, IOVT_UINT32,	100},
+	{"sd_highspeed",	IOV_HISPEED,	0,	0, IOVT_UINT32,	0},
+	{"sd_uhsimode",	IOV_UHSIMOD,	0,	0, IOVT_UINT32,	0},
+#ifdef BCMDBG
+	{"sd_hciregs",	IOV_HCIREGS,	0,	0, IOVT_BUFFER,	0 },
+#endif
+	{"tuning_mode", IOV_TUNEMOD,	0,	0, IOVT_UINT32,	0},
+	{"sd3_tuning_disable", IOV_TUNEDIS,	0,	0, IOVT_BOOL,	0},
+	{"sd_ds",	IOV_DS,		0,	0, IOVT_UINT32,	0},
+
+	{NULL, 0, 0, 0, 0, 0 }
+};
+uint8 sdstd_turn_on_clock(sdioh_info_t *sd)
+{
+	sdstd_or_reg16(sd, SD_ClockCntrl, 0x4);
+	return 0;
+}
+
+uint8 sdstd_turn_off_clock(sdioh_info_t *sd)
+{
+	sdstd_wreg16(sd, SD_ClockCntrl, sdstd_rreg16(sd, SD_ClockCntrl) & ~((uint16)0x4));
+	return 0;
+}
+
+int
+sdioh_iovar_op(sdioh_info_t *si, const char *name,
+               void *params, int plen, void *arg, uint len, bool set)
+{
+	const bcm_iovar_t *vi = NULL;
+	int bcmerror = 0;
+	uint val_size;
+	int32 int_val = 0;
+	bool bool_val;
+	uint32 actionid;
+
+	ASSERT(name);
+
+	/* Get must have return space; Set does not take qualifiers */
+	ASSERT(set || (arg && len));
+	ASSERT(!set || (!params && !plen));
+
+	sd_trace(("%s: Enter (%s %s)\n", __FUNCTION__, (set ? "set" : "get"), name));
+
+	if ((vi = bcm_iovar_lookup(sdioh_iovars, name)) == NULL) {
+		bcmerror = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	if ((bcmerror = bcm_iovar_lencheck(vi, arg, len, set)) != 0)
+		goto exit;
+
+	/* XXX Copied from dhd, copied from wl; certainly overkill here? */
+	/* Set up params so get and set can share the convenience variables */
+	if (params == NULL) {
+		params = arg;
+		plen = len;
+	}
+
+	if (vi->type == IOVT_VOID)
+		val_size = 0;
+	else if (vi->type == IOVT_BUFFER)
+		val_size = len;
+	else
+		val_size = sizeof(int);
+
+	if (plen >= (int)sizeof(int_val))
+		bcopy(params, &int_val, sizeof(int_val));
+
+	bool_val = (int_val != 0) ? TRUE : FALSE;
+	BCM_REFERENCE(bool_val);
+
+	actionid = set ? IOV_SVAL(vi->varid) : IOV_GVAL(vi->varid);
+	switch (actionid) {
+	case IOV_GVAL(IOV_MSGLEVEL):
+		int_val = (int32)sd_msglevel;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_MSGLEVEL):
+		sd_msglevel = int_val;
+		break;
+
+	case IOV_GVAL(IOV_BLOCKMODE):
+		int_val = (int32)si->sd_blockmode;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_BLOCKMODE):
+		si->sd_blockmode = (bool)int_val;
+		/* Haven't figured out how to make non-block mode with DMA */
+		if (!si->sd_blockmode)
+			si->sd_dma_mode = DMA_MODE_NONE;
+		break;
+
+#ifdef BCMSDYIELD
+	case IOV_GVAL(IOV_YIELDCPU):
+		int_val = sd_yieldcpu;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_YIELDCPU):
+		sd_yieldcpu = (bool)int_val;
+		break;
+
+	case IOV_GVAL(IOV_MINYIELD):
+		int_val = sd_minyield;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_MINYIELD):
+		sd_minyield = (bool)int_val;
+		break;
+
+	case IOV_GVAL(IOV_FORCERB):
+		int_val = sd_forcerb;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_FORCERB):
+		sd_forcerb = (bool)int_val;
+		break;
+#endif /* BCMSDYIELD */
+
+	case IOV_GVAL(IOV_BLOCKSIZE):
+		if ((uint32)int_val > si->num_funcs) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = (int32)si->client_block_size[int_val];
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_BLOCKSIZE):
+	{
+		uint func = ((uint32)int_val >> 16);
+		uint blksize = (uint16)int_val;
+		uint maxsize;
+
+		if (func > si->num_funcs) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		/* XXX These hardcoded sizes are a hack, remove after proper CIS parsing. */
+		switch (func) {
+		case 0: maxsize = 32; break;
+		case 1: maxsize = BLOCK_SIZE_4318; break;
+		case 2: maxsize = BLOCK_SIZE_4328; break;
+		default: maxsize = 0;
+		}
+		if (blksize > maxsize) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		if (!blksize) {
+			blksize = maxsize;
+		}
+
+		/* Now set it */
+		sdstd_lock(si);
+		bcmerror = set_client_block_size(si, func, blksize);
+		sdstd_unlock(si);
+		break;
+	}
+
+	case IOV_GVAL(IOV_DMA):
+		int_val = (int32)si->sd_dma_mode;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DMA):
+		si->sd_dma_mode = (char)int_val;
+		sdstd_set_dma_mode(si, si->sd_dma_mode);
+		break;
+
+	case IOV_GVAL(IOV_USEINTS):
+		int_val = (int32)si->use_client_ints;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_USEINTS):
+		si->use_client_ints = (bool)int_val;
+		if (si->use_client_ints)
+			si->intmask |= CLIENT_INTR;
+		else
+			si->intmask &= ~CLIENT_INTR;
+		break;
+
+	case IOV_GVAL(IOV_DIVISOR):
+		int_val = (uint32)sd_divisor;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DIVISOR):
+		sd_divisor = int_val;
+		if (!sdstd_start_clock(si, (uint16)sd_divisor)) {
+			sd_err(("set clock failed!\n"));
+			bcmerror = BCME_ERROR;
+		}
+		break;
+
+	case IOV_GVAL(IOV_POWER):
+		int_val = (uint32)sd_power;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_POWER_SAVE):
+		int_val = (uint32)sd_3_power_save;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_POWER):
+		sd_power = int_val;
+		if (sd_power == 1) {
+			if (sdstd_driver_init(si) != SUCCESS) {
+				sd_err(("set SD Slot power failed!\n"));
+				bcmerror = BCME_ERROR;
+			} else {
+				sd_err(("SD Slot Powered ON.\n"));
+			}
+		} else {
+			uint8 pwr = 0;
+
+			pwr = SFIELD(pwr, PWR_BUS_EN, 0);
+			sdstd_wreg8(si, SD_PwrCntrl, pwr); /* Set Voltage level */
+			sd_err(("SD Slot Powered OFF.\n"));
+		}
+		break;
+
+	case IOV_SVAL(IOV_POWER_SAVE):
+		sd_3_power_save = int_val;
+		break;
+
+	case IOV_GVAL(IOV_CLOCK):
+		int_val = (uint32)sd_clock;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_CLOCK):
+		sd_clock = int_val;
+		if (sd_clock == 1) {
+			sd_info(("SD Clock turned ON.\n"));
+			if (!sdstd_start_clock(si, (uint16)sd_divisor)) {
+				sd_err(("sdstd_start_clock failed\n"));
+				bcmerror = BCME_ERROR;
+			}
+		} else {
+			/* turn off HC clock */
+			sdstd_wreg16(si, SD_ClockCntrl,
+			             sdstd_rreg16(si, SD_ClockCntrl) & ~((uint16)0x4));
+
+			sd_info(("SD Clock turned OFF.\n"));
+		}
+		break;
+
+	case IOV_GVAL(IOV_SDMODE):
+		int_val = (uint32)sd_sdmode;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SDMODE):
+		sd_sdmode = int_val;
+
+		if (!sdstd_bus_width(si, sd_sdmode)) {
+			sd_err(("sdstd_bus_width failed\n"));
+			bcmerror = BCME_ERROR;
+		}
+		break;
+
+	case IOV_GVAL(IOV_HISPEED):
+		int_val = (uint32)sd_hiok;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_HISPEED):
+		sd_hiok = int_val;
+		bcmerror = sdstd_set_highspeed_mode(si, (bool)sd_hiok);
+		break;
+
+	case IOV_GVAL(IOV_UHSIMOD):
+		sd3_trace(("%s: Get UHSI: \n", __FUNCTION__));
+		int_val = (int)sd_uhsimode;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_UHSIMOD):
+		{
+			int oldval = sd_uhsimode; /* save old, working value */
+			sd3_trace(("%s: Set UHSI: \n", __FUNCTION__));
+			/* check if UHSI is supported by card/host */
+			if (!(si->card_UHSI_voltage_Supported && si->host_UHSISupported)) {
+				sd_err(("%s:UHSI not suppoted!\n", __FUNCTION__));
+				bcmerror = BCME_UNSUPPORTED;
+				break;
+			}
+			/* check for valid values */
+			if (!((int_val == SD3CLKMODE_AUTO) ||
+				(int_val == SD3CLKMODE_DISABLED) ||
+				((int_val >= SD3CLKMODE_0_SDR12) &&
+				(int_val <= SD3CLKMODE_4_DDR50)))) {
+				sd_err(("%s:CLK: bad arg!\n", __FUNCTION__));
+				bcmerror = BCME_BADARG;
+				break;
+			}
+
+			sd_uhsimode = int_val;
+			if (SUCCESS != sdstd_3_clock_wrapper(si)) {
+				sd_err(("%s:Error in setting uhsi clkmode:%d,"
+					"restoring back to %d\n", __FUNCTION__,
+					sd_uhsimode, oldval));
+				/* try to set back the old one */
+				sd_uhsimode = oldval;
+				if (SUCCESS != sdstd_3_clock_wrapper(si)) {
+					sd_err(("%s:Error in setting uhsi to old mode;"
+						"ignoring:\n", __FUNCTION__));
+				}
+			}
+			break;
+		}
+#ifdef DHD_DEBUG
+	case IOV_SVAL(IOV_TUNEMOD):
+	{
+
+		if( int_val == SD_DHD_DISABLE_PERIODIC_TUNING) { /* do tuning single time */
+			sd3_trace(("Start tuning from Iovar\n"));
+			si->sd3_tuning_reqd = TRUE;
+			sdstd_enable_disable_periodic_timer(si, int_val);
+			sdstd_lock(si);
+			sdstd_3_clk_tuning(si, sdstd_3_get_uhsi_clkmode(si));
+			sdstd_unlock(si);
+			si->sd3_tuning_reqd = FALSE;
+		}
+		if (int_val == SD_DHD_ENABLE_PERIODIC_TUNING) {
+			sd3_trace(("Enabling  automatic tuning\n"));
+			si->sd3_tuning_reqd = TRUE;
+		    sdstd_enable_disable_periodic_timer(si, int_val);
+		}
+		break;
+	}
+#endif /* debugging purpose */
+	case IOV_GVAL(IOV_NUMINTS):
+		int_val = (int32)si->intrcount;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_NUMLOCALINTS):
+		int_val = (int32)si->local_intrcount;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_HOSTREG):
+	{
+		/* XXX Should copy for alignment reasons */
+		sdreg_t *sd_ptr = (sdreg_t *)params;
+
+		if (sd_ptr->offset < SD_SysAddr || sd_ptr->offset > SD3_WL_BT_reset_register) {
+			sd_err(("%s: bad offset 0x%x\n", __FUNCTION__, sd_ptr->offset));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		sd_trace(("%s: rreg%d at offset %d\n", __FUNCTION__,
+		          (sd_ptr->offset & 1) ? 8 : ((sd_ptr->offset & 2) ? 16 : 32),
+		          sd_ptr->offset));
+		if (sd_ptr->offset & 1)
+			int_val = sdstd_rreg8(si, sd_ptr->offset);
+		else if (sd_ptr->offset & 2)
+			int_val = sdstd_rreg16(si, sd_ptr->offset);
+		else
+			int_val = sdstd_rreg(si, sd_ptr->offset);
+
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+
+	case IOV_SVAL(IOV_HOSTREG):
+	{
+		/* XXX Should copy for alignment reasons */
+		sdreg_t *sd_ptr = (sdreg_t *)params;
+
+		if (sd_ptr->offset < SD_SysAddr || sd_ptr->offset > SD3_WL_BT_reset_register) {
+			sd_err(("%s: bad offset 0x%x\n", __FUNCTION__, sd_ptr->offset));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		sd_trace(("%s: wreg%d value 0x%08x at offset %d\n", __FUNCTION__, sd_ptr->value,
+		          (sd_ptr->offset & 1) ? 8 : ((sd_ptr->offset & 2) ? 16 : 32),
+		          sd_ptr->offset));
+		if (sd_ptr->offset & 1)
+			sdstd_wreg8(si, sd_ptr->offset, (uint8)sd_ptr->value);
+		else if (sd_ptr->offset & 2)
+			sdstd_wreg16(si, sd_ptr->offset, (uint16)sd_ptr->value);
+		else
+			sdstd_wreg(si, sd_ptr->offset, (uint32)sd_ptr->value);
+
+		break;
+	}
+
+	case IOV_GVAL(IOV_DEVREG):
+	{
+		/* XXX Should copy for alignment reasons */
+		sdreg_t *sd_ptr = (sdreg_t *)params;
+		uint8 data;
+
+		if (sdioh_cfg_read(si, sd_ptr->func, sd_ptr->offset, &data)) {
+			bcmerror = BCME_SDIO_ERROR;
+			break;
+		}
+
+		int_val = (int)data;
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+
+	case IOV_SVAL(IOV_DEVREG):
+	{
+		/* XXX Should copy for alignment reasons */
+		sdreg_t *sd_ptr = (sdreg_t *)params;
+		uint8 data = (uint8)sd_ptr->value;
+
+		if (sdioh_cfg_write(si, sd_ptr->func, sd_ptr->offset, &data)) {
+			bcmerror = BCME_SDIO_ERROR;
+			break;
+		}
+		break;
+	}
+
+#ifdef BCMDBG
+	case IOV_GVAL(IOV_HCIREGS):
+	{
+		struct bcmstrbuf b;
+		bcm_binit(&b, arg, len);
+
+		sdstd_lock(si);
+		bcm_bprintf(&b, "IntrStatus:       0x%04x ErrorIntrStatus       0x%04x\n",
+		            sdstd_rreg16(si, SD_IntrStatus),
+		            sdstd_rreg16(si, SD_ErrorIntrStatus));
+		bcm_bprintf(&b, "IntrStatusEnable: 0x%04x ErrorIntrStatusEnable 0x%04x\n",
+		            sdstd_rreg16(si, SD_IntrStatusEnable),
+		            sdstd_rreg16(si, SD_ErrorIntrStatusEnable));
+		bcm_bprintf(&b, "IntrSignalEnable: 0x%04x ErrorIntrSignalEnable 0x%04x\n",
+		            sdstd_rreg16(si, SD_IntrSignalEnable),
+		            sdstd_rreg16(si, SD_ErrorIntrSignalEnable));
+		print_regs(si);
+
+		sdstd_unlock(si);
+
+		if (!b.size)
+			bcmerror = BCME_BUFTOOSHORT;
+		break;
+	}
+#endif /* BCMDBG */
+
+	case IOV_SVAL(IOV_TUNEDIS):
+		si->sd3_tuning_disable = (bool)int_val;
+		break;
+
+	case IOV_GVAL(IOV_DS):
+	{
+		uint32 drvstrn;
+		int status;
+
+		status = sdstd_card_regread(si, 0, SDIOD_CCCR_DRIVER_STRENGTH, 1, &drvstrn);
+		if (status != BCME_OK) {
+			sd_err(("sd_ds error for read SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n",
+				status));
+			bcmerror = BCME_SDIO_ERROR;
+		} else {
+			sd_err(("sd_ds get cccr driver strength 0x%x\n", drvstrn));
+		}
+		int_val = (uint32)drvstrn;
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+
+	case IOV_SVAL(IOV_DS):
+	{
+		uint32 drvstrn;
+		int status;
+
+		status = sdstd_card_regread(si, 0, SDIOD_CCCR_DRIVER_STRENGTH, 1, &drvstrn);
+		if (status != BCME_OK) {
+			sd_err(("sd_ds error for read SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n",
+				status));
+			bcmerror = BCME_SDIO_ERROR;
+		} else {
+			sd_err(("sd_ds get cccr driver strength 0x%x\n", drvstrn));
+		}
+
+		if (int_val == 0) {
+			drvstrn = ((drvstrn & 0xf) | 0x0);
+		} else if (int_val == 1) {
+			drvstrn = ((drvstrn & 0xf) | 0x10);
+		} else if (int_val == 2) {
+			drvstrn = ((drvstrn & 0xf) | 0x20);
+		} else if (int_val == 3) {
+			drvstrn = ((drvstrn & 0xf) | 0x30);
+		} else {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		status = sdstd_card_regwrite(si, 0, SDIOD_CCCR_DRIVER_STRENGTH, 1, drvstrn);
+		if (status != BCME_OK) {
+			sd_err(("sd_ds error write SDIOD_CCCR_DRIVER_STRENGTH : 0x%x\n", status));
+		}
+		break;
+	}
+
+	default:
+		bcmerror = BCME_UNSUPPORTED;
+		break;
+	}
+exit:
+
+	/* XXX Remove protective lock after clients all clean... */
+	return bcmerror;
+}
+
+extern SDIOH_API_RC
+sdioh_cfg_read(sdioh_info_t *sd, uint fnc_num, uint32 addr, uint8 *data)
+{
+	SDIOH_API_RC status;
+	/* No lock needed since sdioh_request_byte does locking */
+	status = sdioh_request_byte(sd, SDIOH_READ, fnc_num, addr, data);
+	return status;
+}
+
+extern SDIOH_API_RC
+sdioh_cfg_write(sdioh_info_t *sd, uint fnc_num, uint32 addr, uint8 *data)
+{
+	/* No lock needed since sdioh_request_byte does locking */
+	SDIOH_API_RC status;
+	status = sdioh_request_byte(sd, SDIOH_WRITE, fnc_num, addr, data);
+	return status;
+}
+
+extern SDIOH_API_RC
+sdioh_cis_read(sdioh_info_t *sd, uint func, uint8 *cisd, uint32 length)
+{
+	uint32 count;
+	int offset;
+	uint32 foo;
+	uint8 *cis = cisd;
+
+	sd_trace(("%s: Func = %d\n", __FUNCTION__, func));
+
+	if (!sd->func_cis_ptr[func]) {
+		bzero(cis, length);
+		return SDIOH_API_RC_FAIL;
+	}
+
+	sdstd_lock(sd);
+	*cis = 0;
+	for (count = 0; count < length; count++) {
+		offset =  sd->func_cis_ptr[func] + count;
+		if (sdstd_card_regread(sd, 0, offset, 1, &foo)) {
+			sd_err(("%s: regread failed: Can't read CIS\n", __FUNCTION__));
+			sdstd_unlock(sd);
+			return SDIOH_API_RC_FAIL;
+		}
+		*cis = (uint8)(foo & 0xff);
+		cis++;
+	}
+	sdstd_unlock(sd);
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_request_byte(sdioh_info_t *sd, uint rw, uint func, uint regaddr, uint8 *byte)
+{
+	int status = SDIOH_API_RC_SUCCESS;
+	uint32 cmd_arg;
+	uint32 rsp5;
+
+	sdstd_lock(sd);
+	if (rw == SDIOH_READ)
+	sdstd_3_check_and_do_tuning(sd, CHECK_TUNING_PRE_DATA);
+
+	/* Change to DATA_TRANSFER_ONGOING , protection against tuning tasklet */
+	sdstd_3_set_data_state(sd, DATA_TRANSFER_ONGOING);
+
+#ifdef BCMDBG
+	if (sdstd_rreg16 (sd, SD_ErrorIntrStatus) != 0) {
+		sd_err(("%s: Entering: ErririntrStatus 0x%x, intstat = 0x%x\n",
+		        __FUNCTION__, sdstd_rreg16(sd, SD_ErrorIntrStatus),
+		        sdstd_rreg16(sd, SD_IntrStatus)));
+	}
+#endif
+	cmd_arg = 0;
+	cmd_arg = SFIELD(cmd_arg, CMD52_FUNCTION, func);
+	cmd_arg = SFIELD(cmd_arg, CMD52_REG_ADDR, regaddr);
+	cmd_arg = SFIELD(cmd_arg, CMD52_RW_FLAG, rw == SDIOH_READ ? 0 : 1);
+	cmd_arg = SFIELD(cmd_arg, CMD52_RAW, 0);
+	cmd_arg = SFIELD(cmd_arg, CMD52_DATA, rw == SDIOH_READ ? 0 : *byte);
+
+	if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_52, cmd_arg)) != SUCCESS) {
+		/* Change to DATA_TRANSFER_IDLE */
+		sdstd_3_set_data_state(sd, DATA_TRANSFER_IDLE);
+		sdstd_unlock(sd);
+		return status;
+	}
+
+	sdstd_cmd_getrsp(sd, &rsp5, 1);
+	if (sdstd_rreg16 (sd, SD_ErrorIntrStatus) != 0) {
+		sd_err(("%s: 1: ErrorintrStatus 0x%x\n",
+		        __FUNCTION__, sdstd_rreg16(sd, SD_ErrorIntrStatus)));
+		status = SDIOH_API_RC_FAIL;
+	}
+	if (GFIELD(rsp5, RSP5_FLAGS) != 0x10) {
+		/* PR 101351: sdiod_aos sleep followed by immediate wakeup
+		 *    before sdiod_aos takes over has a problem.
+		 * While exiting sleep with CMD14, device returning 0x00
+		 * Don't flag as error for now for 0x1001f.
+		 */
+		if (GFIELD(cmd_arg, CMD52_REG_ADDR) != F1_SLEEPCSR_ADDR) {
+			sd_err(("%s: rsp5 flags is 0x%x\t %d \n",
+				__FUNCTION__, GFIELD(rsp5, RSP5_FLAGS), func));
+		}
+		status = SDIOH_API_RC_FAIL;
+	}
+
+	if (GFIELD(rsp5, RSP5_STUFF)) {
+		sd_err(("%s: rsp5 stuff is 0x%x: should be 0\n",
+		        __FUNCTION__, GFIELD(rsp5, RSP5_STUFF)));
+		status = SDIOH_API_RC_FAIL;
+	}
+
+	if (rw == SDIOH_READ)
+		*byte = GFIELD(rsp5, RSP5_DATA);
+
+	/* Change to DATA_TRANSFER_IDLE */
+	sdstd_3_set_data_state(sd, DATA_TRANSFER_IDLE);
+
+	/* check if we have to do tuning; if so, start */
+	sdstd_3_check_and_do_tuning(sd, CHECK_TUNING_POST_DATA);
+
+	sdstd_unlock(sd);
+	return status;
+}
+
+extern SDIOH_API_RC
+sdioh_request_word(sdioh_info_t *sd, uint cmd_type, uint rw, uint func, uint addr,
+                   uint32 *word, uint nbytes)
+{
+	int status;
+
+	sdstd_lock(sd);
+
+	sdstd_3_check_and_do_tuning(sd, CHECK_TUNING_PRE_DATA);
+
+	/* Change to DATA_TRANSFER_ONGOING , protection against tuning tasklet */
+	sdstd_3_set_data_state(sd, DATA_TRANSFER_ONGOING);
+
+	if (rw == SDIOH_READ) {
+		status = sdstd_card_regread(sd, func, addr, nbytes, word);
+	} else {
+		status = sdstd_card_regwrite(sd, func, addr, nbytes, *word);
+	}
+
+	/* Change to DATA_TRANSFER_IDLE */
+	sdstd_3_set_data_state(sd, DATA_TRANSFER_IDLE);
+
+	/* check if we have to do tuning; if so, start */
+	sdstd_3_check_and_do_tuning(sd, CHECK_TUNING_POST_DATA);
+
+	sdstd_unlock(sd);
+	return (status == SUCCESS ?  SDIOH_API_RC_SUCCESS : SDIOH_API_RC_FAIL);
+}
+
+#ifdef BCMSDIOH_TXGLOM
+void
+sdioh_glom_post(sdioh_info_t *sd, uint8 *frame, void *pkt, uint len)
+{
+	BCM_REFERENCE(pkt);
+	sd->glom_info.dma_buf_arr[sd->glom_info.count] = frame;
+	sd->glom_info.nbytes[sd->glom_info.count] = len;
+	/* Convert the frame addr to phy addr for DMA in case of host controller version3 */
+	if (sd->txglom_mode == SDPCM_TXGLOM_MDESC) {
+		sd->glom_info.dma_phys_arr[sd->glom_info.count] = DMA_MAP(sd->osh,
+			frame,
+			len,
+			DMA_TX, 0, 0);
+	}
+	sd->glom_info.count++;
+}
+
+void
+sdioh_glom_clear(sdioh_info_t *sd)
+{
+	int i;
+	/* DMA_MAP is done per frame only if host controller version is 3 */
+	if (sd->txglom_mode == SDPCM_TXGLOM_MDESC) {
+		for (i = 0; i < sd->glom_info.count; i++) {
+			DMA_UNMAP(sd->osh,
+				sd->glom_info.dma_phys_arr[i],
+				sd->glom_info.nbytes[i],
+				DMA_TX, 0, 0);
+		}
+	}
+	sd->glom_info.count = 0;
+}
+
+uint
+sdioh_set_mode(sdioh_info_t *sd, uint mode)
+{
+	if (mode == SDPCM_TXGLOM_CPY)
+		sd->txglom_mode = mode;
+	else if ((mode == SDPCM_TXGLOM_MDESC) && (sd->version == HOST_CONTR_VER_3))
+		sd->txglom_mode = mode;
+
+	return (sd->txglom_mode);
+}
+
+bool
+sdioh_glom_enabled(void)
+{
+	return sd_txglom;
+}
+#endif /* BCMSDIOH_TXGLOM */
+
+extern SDIOH_API_RC
+sdioh_request_buffer(sdioh_info_t *sd, uint pio_dma, uint fix_inc, uint rw, uint func,
+                     uint addr, uint reg_width, uint buflen_u, uint8 *buffer, void *pkt)
+{
+	uint8 is_ddr50 = FALSE;
+	int len;
+	int buflen = (int)buflen_u;
+	bool fifo = (fix_inc == SDIOH_DATA_FIX);
+	uint8 *localbuf = NULL, *tmpbuf = NULL;
+	bool local_blockmode = sd->sd_blockmode;
+	SDIOH_API_RC status = SDIOH_API_RC_SUCCESS;
+
+	sdstd_lock(sd);
+
+	is_ddr50 = (sd_uhsimode == SD3CLKMODE_4_DDR50) ? TRUE : FALSE;
+
+	sdstd_3_check_and_do_tuning(sd, CHECK_TUNING_PRE_DATA);
+
+	/* Change to DATA_TRANSFER_ONGOING , protection against tuning tasklet */
+	sdstd_3_set_data_state(sd, DATA_TRANSFER_ONGOING);
+
+	ASSERT(reg_width == 4);
+	ASSERT(buflen_u < (1 << 30));
+	ASSERT(sd->client_block_size[func]);
+
+#ifdef BCMSDIOH_TXGLOM
+	if (sd_txglom) {
+		while (pkt) {
+			sdioh_glom_post(sd, PKTDATA(sd->osh, pkt), pkt, PKTLEN(sd->osh, pkt));
+			pkt = PKTNEXT(sd->osh, pkt);
+		}
+	}
+#endif
+	sd_data(("%s: %c len %d r_cnt %d t_cnt %d, pkt @0x%p\n",
+	         __FUNCTION__, rw == SDIOH_READ ? 'R' : 'W',
+	         buflen_u, sd->r_cnt, sd->t_cnt, pkt));
+
+	/* Break buffer down into blocksize chunks:
+	 * Bytemode: 1 block at a time.
+	 * Blockmode: Multiples of blocksizes at a time w/ max of SD_PAGE.
+	 * Both: leftovers are handled last (will be sent via bytemode).
+	 */
+	while (buflen > 0) {
+		if (local_blockmode) {
+			int max_tran_size = SD_PAGE;
+#ifdef BCMSDIOH_TXGLOM
+			/* There is no alignment requirement for HC3 */
+			if ((sd->version == HOST_CONTR_VER_3) && sd_txglom)
+				max_tran_size = SD_PAGE * 4;
+#endif
+			/* Max xfer is Page size */
+			len = MIN(max_tran_size, buflen);
+
+			/* Round down to a block boundry */
+			if (buflen > sd->client_block_size[func])
+				len = (len/sd->client_block_size[func]) *
+				        sd->client_block_size[func];
+			/* XXX Arasan trashes 3-byte transfers, WAR to add one byte extra. */
+			/* XXX In Case of SDIO3.0 DDR50 mode if no of bytes to be
+			 * transferred is odd append one more byte to make it even.
+			 * Check If odd bytes can come for SDIO_FUNC_2 also.
+			 */
+			if ((func == SDIO_FUNC_1) && (((len % 4) == 3) || (((len % 2) == 1) &&
+			    (is_ddr50))) && ((rw == SDIOH_WRITE) || (rw == SDIOH_READ))) {
+				sd_err(("%s: Rounding up buffer to mod4 length.\n", __FUNCTION__));
+				len++;
+				tmpbuf = buffer;
+				if ((localbuf = (uint8 *)MALLOC(sd->osh, len)) == NULL) {
+					sd_err(("out of memory, malloced %d bytes\n",
+					        MALLOCED(sd->osh)));
+					status = SDIOH_API_RC_FAIL;
+					goto done;
+				}
+				bcopy(buffer, localbuf, len);
+				buffer = localbuf;
+			}
+		} else {
+			/* Byte mode: One block at a time */
+			len = MIN(sd->client_block_size[func], buflen);
+		}
+
+		if (sdstd_card_buf(sd, rw, func, fifo, addr, len, (uint32 *)buffer) != SUCCESS) {
+			status = SDIOH_API_RC_FAIL;
+		}
+
+		/* XXX Restore len and buffer pointer WAR'ed for Arasan 3-byte transfer problem */
+		/* XXX WAR for SDIO3.0 DDR50 mode. */
+		if (local_blockmode && localbuf) {
+			MFREE(sd->osh, localbuf, len);
+			len--;
+			buffer = tmpbuf;
+			sd_err(("%s: Restoring back buffer ptr and len.\n", __FUNCTION__));
+		}
+
+		if (status == SDIOH_API_RC_FAIL) {
+			goto done;
+		}
+
+		buffer += len;
+		buflen -= len;
+		if (!fifo)
+			addr += len;
+#ifdef BCMSDIOH_TXGLOM
+		/* This loop should not come in case of glommed pkts as it is send in
+		 * multiple of blocks or total pkt size less than a block
+		 */
+		if (sd->glom_info.count != 0)
+			buflen = 0;
+#endif
+	}
+done:
+
+	/* Change to DATA_TRANSFER_IDLE */
+	sdstd_3_set_data_state(sd, DATA_TRANSFER_IDLE);
+
+	/* check if we have to do tuning; if so, start */
+	sdstd_3_check_and_do_tuning(sd, CHECK_TUNING_POST_DATA);
+
+	sdstd_unlock(sd);
+
+#ifdef BCMSDIOH_TXGLOM
+	if (sd_txglom)
+		sdioh_glom_clear(sd);
+#endif
+
+	return status;
+}
+
+extern SDIOH_API_RC
+sdioh_gpioouten(sdioh_info_t *sd, uint32 gpio)
+{
+	uint offset = 0;
+	uint16 val;
+
+	/* check if upper bank */
+	if (gpio >= SDH_GPIO16) {
+		gpio -= SDH_GPIO16;
+		offset = 2;
+	}
+
+	val = sdstd_rreg16(sd, SD_GPIO_OE + offset);
+	val |= (1 << gpio);
+	sdstd_wreg16(sd, SD_GPIO_OE + offset, val);
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_gpioout(sdioh_info_t *sd, uint32 gpio, bool enab)
+{
+	uint offset = 0;
+	uint16 val;
+
+	/* check if upper bank */
+	if (gpio >= SDH_GPIO16) {
+		gpio -= SDH_GPIO16;
+		offset = 2;
+	}
+
+	val = sdstd_rreg16(sd, SD_GPIO_Reg + offset);
+	if (enab == TRUE)
+		val |= (1 << gpio);
+	else
+		val &= ~(1 << gpio);
+	sdstd_wreg16(sd, SD_GPIO_Reg + offset, val);
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern bool
+sdioh_gpioin(sdioh_info_t *sd, uint32 gpio)
+{
+	uint offset = 0;
+	uint16 val;
+
+	/* check if upper bank */
+	if (gpio >= SDH_GPIO16) {
+		gpio -= SDH_GPIO16;
+		offset = 2;
+	}
+
+	val = sdstd_rreg16(sd, SD_GPIO_Reg + offset);
+	val = (val >> gpio) & 1;
+
+	return (val == 1);
+}
+
+extern SDIOH_API_RC
+sdioh_gpio_init(sdioh_info_t *sd)
+{
+	uint rev;
+
+	rev = sdstd_rreg16(sd, SD_HostControllerVersion) >> 8;
+
+	/* Only P206 (fpga rev >= 16) supports gpio */
+	if (rev < 16) {
+		sd_err(("%s: gpio not supported in rev %d \n", __FUNCTION__, rev));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	sdstd_wreg16(sd, SD_GPIO_Enable, SDH_GPIO_ENABLE);
+	sdstd_wreg16(sd, SD_GPIO_Enable + 2, SDH_GPIO_ENABLE);
+
+	/* Default to input */
+	sdstd_wreg16(sd, SD_GPIO_OE, 0);
+	sdstd_wreg16(sd, SD_GPIO_OE + 2, 0);
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+extern SDIOH_API_RC
+sdioh_sleep(sdioh_info_t *sd, bool enab)
+{
+	SDIOH_API_RC status;
+	uint32 cmd_arg = 0, rsp1 = 0;
+	int retry = 100;
+
+	sdstd_lock(sd);
+
+	cmd_arg = SFIELD(cmd_arg, CMD14_RCA, sd->card_rca);
+	cmd_arg = SFIELD(cmd_arg, CMD14_SLEEP, enab);
+
+	/*
+	 * For ExitSleep:
+	 *  1) Repeat CMD14 until R1 is received
+	 *  2) Send CMD7
+	 */
+	status = SDIOH_API_RC_FAIL;
+	while (retry-- > 0) {
+		if ((sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_14, cmd_arg)) == SUCCESS) {
+			status = SDIOH_API_RC_SUCCESS;
+			break;
+		}
+		OSL_DELAY(1400);
+	}
+
+	if (status == SDIOH_API_RC_FAIL) {
+		sd_err(("%s: CMD14: failed! enable:%d\n", __FUNCTION__, enab));
+		goto exit;
+	}
+
+	sdstd_cmd_getrsp(sd, &rsp1, 1);
+	sd_info(("%s: CMD14 OK: cmd_resp:0x%x\n", __FUNCTION__, rsp1));
+
+	/* ExitSleep: Send CMD7 After R1 */
+	if (enab == FALSE) {
+		/* Select the card */
+		cmd_arg = SFIELD(0, CMD7_RCA, sd->card_rca);
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_7, cmd_arg)) != SUCCESS) {
+			sd_err(("%s: CMD14 send CMD7 failed!\n", __FUNCTION__));
+			status = SDIOH_API_RC_FAIL;
+			goto exit;
+		}
+
+		sdstd_cmd_getrsp(sd, &rsp1, 1);
+		if (rsp1 != SDIOH_CMD7_EXP_STATUS) {
+			sd_err(("%s: CMD7 response error. Response = 0x%x!\n",
+			        __FUNCTION__, rsp1));
+			status = SDIOH_API_RC_FAIL;
+			goto exit;
+		}
+	}
+
+exit:
+	sdstd_unlock(sd);
+
+	return status;
+}
+
+/* XXX Copied guts of request_byte and cmd_issue.  Might make sense to fold this into
+ * those by passing another parameter indicating command type (abort).  [But maybe
+ * keeping it separate is better -- if called internally on command failure it's less
+ * recursion to wrap your head around?]
+ */
+static int
+sdstd_abort(sdioh_info_t *sd, uint func)
+{
+	int err = 0;
+	int retries;
+
+	uint16 cmd_reg;
+	uint32 cmd_arg;
+	uint32 rsp5;
+	uint8 rflags;
+
+	uint16 int_reg = 0;
+	uint16 plain_intstatus;
+
+	/* Argument is write to F0 (CCCR) IOAbort with function number */
+	cmd_arg = 0;
+	cmd_arg = SFIELD(cmd_arg, CMD52_FUNCTION, SDIO_FUNC_0);
+	cmd_arg = SFIELD(cmd_arg, CMD52_REG_ADDR, SDIOD_CCCR_IOABORT);
+	cmd_arg = SFIELD(cmd_arg, CMD52_RW_FLAG, SD_IO_OP_WRITE);
+	cmd_arg = SFIELD(cmd_arg, CMD52_RAW, 0);
+	cmd_arg = SFIELD(cmd_arg, CMD52_DATA, func);
+
+	/* Command is CMD52 write */
+	cmd_reg = 0;
+	cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48_BUSY);
+	cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 1);
+	cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 1);
+	cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+	cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_ABORT);
+	cmd_reg = SFIELD(cmd_reg, CMD_INDEX, SDIOH_CMD_52);
+
+	/* XXX Copied from cmd_issue(), but no SPI response handling! */
+	if (sd->sd_mode == SDIOH_MODE_SPI) {
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 0);
+	}
+
+	/* Wait for CMD_INHIBIT to go away as per spec section 3.6.1.1 */
+	/* XXX For a single-threaded driver, what circumstances would result
+	 * in cmd_inhibit being on but going off in a short time?  Experiment
+	 * shows a HW command timeout doesn't leave inhibit on, so maybe a SW
+	 * timeout?  Then that command should be responsible for clearing...
+	 */
+	retries = RETRIES_SMALL;
+	while (GFIELD(sdstd_rreg(sd, SD_PresentState), PRES_CMD_INHIBIT)) {
+		if (retries == RETRIES_SMALL)
+			sd_err(("%s: Waiting for Command Inhibit, state 0x%08x\n",
+			        __FUNCTION__, sdstd_rreg(sd, SD_PresentState)));
+		if (!--retries) {
+			sd_err(("%s: Command Inhibit timeout, state 0x%08x\n",
+			        __FUNCTION__, sdstd_rreg(sd, SD_PresentState)));
+			if (trap_errs)
+				ASSERT(0);
+			err = BCME_SDIO_ERROR;
+			goto done;
+		}
+	}
+
+	/* Clear errors from any previous commands */
+	if ((plain_intstatus = sdstd_rreg16(sd, SD_ErrorIntrStatus)) != 0) {
+		sd_err(("abort: clearing errstat 0x%04x\n", plain_intstatus));
+		sdstd_wreg16(sd, SD_ErrorIntrStatus, plain_intstatus);
+	}
+	plain_intstatus = sdstd_rreg16(sd, SD_IntrStatus);
+	if (plain_intstatus & ~(SFIELD(0, INTSTAT_CARD_INT, 1))) {
+		sd_err(("abort: intstatus 0x%04x\n", plain_intstatus));
+		if (GFIELD(plain_intstatus, INTSTAT_CMD_COMPLETE)) {
+			sd_err(("SDSTD_ABORT: CMD COMPLETE SET BEFORE COMMAND GIVEN!!!\n"));
+		}
+		if (GFIELD(plain_intstatus, INTSTAT_CARD_REMOVAL)) {
+			sd_err(("SDSTD_ABORT: INTSTAT_CARD_REMOVAL\n"));
+			err = BCME_NODEVICE;
+			goto done;
+		}
+	}
+
+	/* Issue the command */
+	sdstd_wreg(sd, SD_Arg0, cmd_arg);
+	sdstd_wreg16(sd, SD_Command, cmd_reg);
+
+	/* In interrupt mode return, expect later CMD_COMPLETE interrupt */
+	if (!sd->polled_mode)
+		return err;
+
+	/* Otherwise, wait for the command to complete */
+	retries = RETRIES_LARGE;
+	do {
+		int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+	} while (--retries &&
+	         (GFIELD(int_reg, INTSTAT_ERROR_INT) == 0) &&
+	         (GFIELD(int_reg, INTSTAT_CMD_COMPLETE) == 0));
+
+	/* If command completion fails, do a cmd reset and note the error */
+	if (!retries) {
+		sd_err(("%s: CMD_COMPLETE timeout: intr 0x%04x err 0x%04x state 0x%08x\n",
+		        __FUNCTION__, int_reg,
+		        sdstd_rreg16(sd, SD_ErrorIntrStatus),
+		        sdstd_rreg(sd, SD_PresentState)));
+
+		sdstd_wreg8(sd, SD_SoftwareReset, SFIELD(0, SW_RESET_CMD, 1));
+		retries = RETRIES_LARGE;
+		do {
+			sd_trace(("%s: waiting for CMD line reset\n", __FUNCTION__));
+		} while ((GFIELD(sdstd_rreg8(sd, SD_SoftwareReset),
+		                 SW_RESET_CMD)) && retries--);
+
+		if (!retries) {
+			sd_err(("%s: Timeout waiting for CMD line reset\n", __FUNCTION__));
+		}
+
+		if (trap_errs)
+			ASSERT(0);
+
+		err = BCME_SDIO_ERROR;
+	}
+
+	/* Clear Command Complete interrupt */
+	int_reg = SFIELD(0, INTSTAT_CMD_COMPLETE, 1);
+	sdstd_wreg16(sd, SD_IntrStatus, int_reg);
+
+	/* Check for Errors */
+	if ((plain_intstatus = sdstd_rreg16 (sd, SD_ErrorIntrStatus)) != 0) {
+		sd_err(("%s: ErrorintrStatus: 0x%x, "
+		        "(intrstatus = 0x%x, present state 0x%x) clearing\n",
+		        __FUNCTION__, plain_intstatus,
+		        sdstd_rreg16(sd, SD_IntrStatus),
+		        sdstd_rreg(sd, SD_PresentState)));
+
+		sdstd_wreg16(sd, SD_ErrorIntrStatus, plain_intstatus);
+
+		sdstd_wreg8(sd, SD_SoftwareReset, SFIELD(0, SW_RESET_DAT, 1));
+		retries = RETRIES_LARGE;
+		do {
+			sd_trace(("%s: waiting for DAT line reset\n", __FUNCTION__));
+		} while ((GFIELD(sdstd_rreg8(sd, SD_SoftwareReset),
+		                 SW_RESET_DAT)) && retries--);
+
+		if (!retries) {
+			sd_err(("%s: Timeout waiting for DAT line reset\n", __FUNCTION__));
+		}
+
+		if (trap_errs)
+			ASSERT(0);
+
+		/* ABORT is dataless, only cmd errs count */
+		/* XXX But what about busy timeout?  Response valid? */
+		if (plain_intstatus & ERRINT_CMD_ERRS)
+			err = BCME_SDIO_ERROR;
+	}
+
+	/* If command failed don't bother looking at response */
+	if (err)
+		goto done;
+
+	/* Otherwise, check the response */
+	sdstd_cmd_getrsp(sd, &rsp5, 1);
+	rflags = GFIELD(rsp5, RSP5_FLAGS);
+
+	if (rflags & SD_RSP_R5_ERRBITS) {
+		sd_err(("%s: R5 flags include errbits: 0x%02x\n", __FUNCTION__, rflags));
+
+		/* The CRC error flag applies to the previous command */
+		if (rflags & (SD_RSP_R5_ERRBITS & ~SD_RSP_R5_COM_CRC_ERROR)) {
+			err = BCME_SDIO_ERROR;
+			goto done;
+		}
+	}
+
+	if (((rflags & (SD_RSP_R5_IO_CURRENTSTATE0 | SD_RSP_R5_IO_CURRENTSTATE1)) != 0x10) &&
+	    ((rflags & (SD_RSP_R5_IO_CURRENTSTATE0 | SD_RSP_R5_IO_CURRENTSTATE1)) != 0x20)) {
+		sd_err(("%s: R5 flags has bad state: 0x%02x\n", __FUNCTION__, rflags));
+		err = BCME_SDIO_ERROR;
+		goto done;
+	}
+
+	if (GFIELD(rsp5, RSP5_STUFF)) {
+		sd_err(("%s: rsp5 stuff is 0x%x: should be 0\n",
+		        __FUNCTION__, GFIELD(rsp5, RSP5_STUFF)));
+		err = BCME_SDIO_ERROR;
+		goto done;
+	}
+
+done:
+	if (err == BCME_NODEVICE)
+		return err;
+
+	/* XXX As per spec 3.7.1 (and to be safe) do the resets here */
+	sdstd_wreg8(sd, SD_SoftwareReset,
+	            SFIELD(SFIELD(0, SW_RESET_DAT, 1), SW_RESET_CMD, 1));
+
+	retries = RETRIES_LARGE;
+	do {
+		rflags = sdstd_rreg8(sd, SD_SoftwareReset);
+		if (!GFIELD(rflags, SW_RESET_DAT) && !GFIELD(rflags, SW_RESET_CMD))
+			break;
+	} while (--retries);
+
+	if (!retries) {
+		sd_err(("%s: Timeout waiting for DAT/CMD reset: 0x%02x\n",
+		        __FUNCTION__, rflags));
+		err = BCME_SDIO_ERROR;
+	}
+
+	return err;
+}
+
+extern int
+sdioh_abort(sdioh_info_t *sd, uint fnum)
+{
+	int ret;
+
+	sdstd_lock(sd);
+	ret = sdstd_abort(sd, fnum);
+	sdstd_unlock(sd);
+
+	return ret;
+}
+
+int
+sdioh_start(sdioh_info_t *sd, int stage)
+{
+	return SUCCESS;
+}
+
+int
+sdioh_stop(sdioh_info_t *sd)
+{
+	return SUCCESS;
+}
+
+int
+sdioh_waitlockfree(sdioh_info_t *sd)
+{
+	sdstd_waitlockfree(sd);
+	return SUCCESS;
+}
+
+static int
+sdstd_check_errs(sdioh_info_t *sdioh_info, uint32 cmd, uint32 arg)
+{
+	uint16 regval;
+	uint retries;
+	uint function = 0;
+
+	/* If no errors, we're done */
+	if ((regval = sdstd_rreg16(sdioh_info, SD_ErrorIntrStatus)) == 0)
+		return SUCCESS;
+
+#ifdef BCMQT
+	if (regval == 0xFFFF) {
+		/* XXX - Getting bogus errors under QT
+		 * XXX - Not sure why; Just ignore for now
+		 */
+		sd_err(("%s: Bogus SD_ErrorIntrStatus: 0x%x????\n", __FUNCTION__, regval));
+		sdstd_wreg16(sdioh_info, SD_ErrorIntrStatus, regval);
+		return SUCCESS;
+	}
+#endif
+
+	sd_info(("%s: ErrorIntrStatus 0x%04x (clearing), IntrStatus 0x%04x PresentState 0x%08x\n",
+	        __FUNCTION__, regval, sdstd_rreg16(sdioh_info, SD_IntrStatus),
+	        sdstd_rreg(sdioh_info, SD_PresentState)));
+	sdstd_wreg16(sdioh_info, SD_ErrorIntrStatus, regval);
+
+	if (cmd == SDIOH_CMD_14) {
+		if (regval & ERRINT_CMD_TIMEOUT_BIT) {
+			/* PR 101351: sdiod_aos sleep followed by immediate wakeup
+			 *    before sdiod_aos takes over has a problem.
+			 * Getting command timeouts while exiting sleep
+			 * with CMD14. Ignore this error due to this PR.
+			 */
+			regval &= ~ERRINT_CMD_TIMEOUT_BIT;
+		}
+	}
+
+	/* On command error, issue CMD reset */
+	if (regval & ERRINT_CMD_ERRS) {
+		sd_trace(("%s: issuing CMD reset\n", __FUNCTION__));
+		sdstd_wreg8(sdioh_info, SD_SoftwareReset, SFIELD(0, SW_RESET_CMD, 1));
+		for (retries = RETRIES_LARGE; retries; retries--)
+			if (!(GFIELD(sdstd_rreg8(sdioh_info, SD_SoftwareReset), SW_RESET_CMD)))
+				break;
+		if (!retries) {
+			sd_err(("%s: Timeout waiting for CMD line reset\n", __FUNCTION__));
+		}
+	}
+
+	/* On data error, issue DAT reset */
+	if (regval & ERRINT_DATA_ERRS) {
+		if (regval & ERRINT_ADMA_BIT)
+			sd_err(("%s:ADMAError: status:0x%x\n",
+				__FUNCTION__, sdstd_rreg(sdioh_info, SD_ADMA_ErrStatus)));
+		sd_trace(("%s: issuing DAT reset\n", __FUNCTION__));
+		sdstd_wreg8(sdioh_info, SD_SoftwareReset, SFIELD(0, SW_RESET_DAT, 1));
+		for (retries = RETRIES_LARGE; retries; retries--)
+			if (!(GFIELD(sdstd_rreg8(sdioh_info, SD_SoftwareReset), SW_RESET_DAT)))
+				break;
+		if (!retries) {
+			sd_err(("%s: Timeout waiting for DAT line reset\n", __FUNCTION__));
+		}
+	}
+
+	/* For an IO command (CMD52 or CMD53) issue an abort to the appropriate function */
+	if (cmd == SDIOH_CMD_53)
+		function = GFIELD(arg, CMD53_FUNCTION);
+	else if (cmd == SDIOH_CMD_52) {
+		/* PR 101351: sdiod_aos sleep followed by immediate wakeup
+		 *    before sdiod_aos takes over has a problem.
+		 */
+		if (GFIELD(arg, CMD52_REG_ADDR) != F1_SLEEPCSR_ADDR)
+			function = GFIELD(arg, CMD52_FUNCTION);
+	}
+	if (function) {
+		sd_trace(("%s: requesting abort for function %d after cmd %d\n",
+		          __FUNCTION__, function, cmd));
+		sdstd_abort(sdioh_info, function);
+	}
+
+	if (trap_errs)
+		ASSERT(0);
+
+	return ERROR;
+}
+
+/*
+ * Private/Static work routines
+ */
+static bool
+sdstd_reset(sdioh_info_t *sd, bool host_reset, bool client_reset)
+{
+	int retries = RETRIES_LARGE;
+	uchar regval;
+
+	if (!sd)
+		return TRUE;
+
+	sdstd_lock(sd);
+	/* Reset client card */
+	if (client_reset && (sd->adapter_slot != -1)) {
+		if (sdstd_card_regwrite(sd, 0, SDIOD_CCCR_IOABORT, 1, 0x8) != SUCCESS)
+			sd_err(("%s: Cannot write to card reg 0x%x\n",
+			        __FUNCTION__, SDIOD_CCCR_IOABORT));
+		else
+			sd->card_rca = 0;
+	}
+
+	/* Reset host controller */
+	if (host_reset) {
+		regval = SFIELD(0, SW_RESET_ALL, 1);
+		sdstd_wreg8(sd, SD_SoftwareReset, regval);
+		do {
+			sd_trace(("%s: waiting for reset\n", __FUNCTION__));
+		} while ((sdstd_rreg8(sd, SD_SoftwareReset) & regval) && retries--);
+
+		if (!retries) {
+			sd_err(("%s: Timeout waiting for host reset\n", __FUNCTION__));
+			sdstd_unlock(sd);
+			return (FALSE);
+		}
+
+		/* A reset should reset bus back to 1 bit mode */
+		sd->sd_mode = SDIOH_MODE_SD1;
+		sdstd_set_dma_mode(sd, sd->sd_dma_mode);
+	}
+	sdstd_unlock(sd);
+	return TRUE;
+}
+
+/* Disable device interrupt */
+void
+sdstd_devintr_off(sdioh_info_t *sd)
+{
+	sd_trace(("%s: %d\n", __FUNCTION__, sd->use_client_ints));
+	if (sd->use_client_ints) {
+		sd->intmask &= ~CLIENT_INTR;
+		sdstd_wreg16(sd, SD_IntrSignalEnable, sd->intmask);
+		sdstd_rreg16(sd, SD_IntrSignalEnable); /* Sync readback */
+	}
+}
+
+/* Enable device interrupt */
+void
+sdstd_devintr_on(sdioh_info_t *sd)
+{
+	ASSERT(sd->lockcount == 0);
+	sd_trace(("%s: %d\n", __FUNCTION__, sd->use_client_ints));
+	if (sd->use_client_ints) {
+		if (sd->version < HOST_CONTR_VER_3) {
+			uint16 status = sdstd_rreg16(sd, SD_IntrStatusEnable);
+			sdstd_wreg16(sd, SD_IntrStatusEnable, SFIELD(status, INTSTAT_CARD_INT, 0));
+			sdstd_wreg16(sd, SD_IntrStatusEnable, status);
+		}
+
+		sd->intmask |= CLIENT_INTR;
+		sdstd_wreg16(sd, SD_IntrSignalEnable, sd->intmask);
+		sdstd_rreg16(sd, SD_IntrSignalEnable); /* Sync readback */
+	}
+}
+
+#ifdef BCMSDYIELD
+/* Enable/disable other interrupts */
+void
+sdstd_intrs_on(sdioh_info_t *sd, uint16 norm, uint16 err)
+{
+	if (err) {
+		norm = SFIELD(norm, INTSTAT_ERROR_INT, 1);
+		sdstd_wreg16(sd, SD_ErrorIntrSignalEnable, err);
+	}
+
+	sd->intmask |= norm;
+	sdstd_wreg16(sd, SD_IntrSignalEnable, sd->intmask);
+	if (sd_forcerb)
+		sdstd_rreg16(sd, SD_IntrSignalEnable); /* Sync readback */
+}
+
+void
+sdstd_intrs_off(sdioh_info_t *sd, uint16 norm, uint16 err)
+{
+	if (err) {
+		norm = SFIELD(norm, INTSTAT_ERROR_INT, 1);
+		sdstd_wreg16(sd, SD_ErrorIntrSignalEnable, 0);
+	}
+
+	sd->intmask &= ~norm;
+	sdstd_wreg16(sd, SD_IntrSignalEnable, sd->intmask);
+	if (sd_forcerb)
+		sdstd_rreg16(sd, SD_IntrSignalEnable); /* Sync readback */
+}
+#endif /* BCMSDYIELD */
+
+static int
+sdstd_host_init(sdioh_info_t *sd)
+{
+	int 		num_slots, full_slot;
+	uint8		reg8;
+	uint32		card_ins;
+	int		slot, first_bar = 0;
+	bool		detect_slots = FALSE;
+#ifdef _WIN32
+	NDIS_PHYSICAL_ADDRESS	bar;
+#else
+	uint		bar;
+#endif
+
+	/* Check for Arasan ID */
+	if ((OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_VID, 4) & 0xFFFF) == VENDOR_SI_IMAGE) {
+		sd_info(("%s: Found Arasan Standard SDIO Host Controller\n", __FUNCTION__));
+		sd->controller_type = SDIOH_TYPE_ARASAN_HDK;
+		detect_slots = TRUE;
+		/* Controller supports SDMA, so turn it on here. */
+		sd->sd_dma_mode = DMA_MODE_SDMA;
+	} else if ((OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_VID, 4) & 0xFFFF) == VENDOR_BROADCOM) {
+		sd_info(("%s: Found Broadcom 27xx Standard SDIO Host Controller\n", __FUNCTION__));
+		sd->controller_type = SDIOH_TYPE_BCM27XX;
+		detect_slots = FALSE;
+	} else if ((OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_VID, 4) & 0xFFFF) == VENDOR_TI) {
+		sd_info(("%s: Found TI PCIxx21 Standard SDIO Host Controller\n", __FUNCTION__));
+		sd->controller_type = SDIOH_TYPE_TI_PCIXX21;
+		detect_slots = TRUE;
+	} else if ((OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_VID, 4) & 0xFFFF) == VENDOR_RICOH) {
+		sd_info(("%s: Ricoh Co Ltd R5C822 SD/SDIO/MMC/MS/MSPro Host Adapter\n",
+			__FUNCTION__));
+		sd->controller_type = SDIOH_TYPE_RICOH_R5C822;
+		detect_slots = TRUE;
+	} else if ((OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_VID, 4) & 0xFFFF) == VENDOR_JMICRON) {
+		sd_info(("%s: JMicron Standard SDIO Host Controller\n",
+			__FUNCTION__));
+		sd->controller_type = SDIOH_TYPE_JMICRON;
+		detect_slots = TRUE;
+	} else {
+		return ERROR;
+	}
+
+	/*
+	 * Determine num of slots
+	 * Search each slot
+	 */
+
+	first_bar = OSL_PCI_READ_CONFIG(sd->osh, SD_SlotInfo, 4) & 0x7;
+	num_slots = (OSL_PCI_READ_CONFIG(sd->osh, SD_SlotInfo, 4) & 0xff) >> 4;
+	num_slots &= 7;
+	num_slots++;   	/* map bits to num slots according to spec */
+
+	/* XXX Since the sdio20h core does not present the proper SD_SlotInfo
+	 * register at PCI config space offset 0x40, fake it here.  Also,
+	 * set the BAR0 window to point to the sdio20h core.
+	 */
+	if (OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_VID, 4) ==
+	    ((SDIOH_FPGA_ID << 16) | VENDOR_BROADCOM)) {
+		sd_err(("%s: Found Broadcom Standard SDIO Host Controller FPGA\n", __FUNCTION__));
+		/* Set BAR0 Window to SDIOSTH core */
+		OSL_PCI_WRITE_CONFIG(sd->osh, PCI_BAR0_WIN, 4, 0x18001000);
+
+		/* Set defaults particular to this controller. */
+		detect_slots = TRUE;
+		num_slots = 1;
+		first_bar = 0;
+
+		/* Controller supports ADMA2, so turn it on here. */
+		sd->sd_dma_mode = DMA_MODE_ADMA2;
+	}
+
+	/* Map in each slot on the board and query it to see if a
+	 * card is inserted.  Use the first populated slot found.
+	 */
+	if (sd->mem_space) {
+		sdstd_reg_unmap(sd->osh, (ulong)sd->mem_space, SDIOH_REG_WINSZ);
+		sd->mem_space = NULL;
+	}
+
+	full_slot = -1;
+
+	for (slot = 0; slot < num_slots; slot++) {
+/* XXX :Ugly define, is there a better way */
+#ifdef _WIN32
+		bar.HighPart = 0;
+		bar.LowPart = OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_BAR0
+		              + (4*(slot + first_bar)), 4);
+		sd->mem_space = (volatile char *)sdstd_reg_map(sd->osh,
+		                                               (int32)&bar, SDIOH_REG_WINSZ);
+#else
+		bar = OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_BAR0 + (4*(slot + first_bar)), 4);
+		sd->mem_space = (volatile char *)sdstd_reg_map(sd->osh,
+		                                               (uintptr)bar, SDIOH_REG_WINSZ);
+#endif
+
+		sd->adapter_slot = -1;
+
+		if (detect_slots) {
+			card_ins = GFIELD(sdstd_rreg(sd, SD_PresentState), PRES_CARD_PRESENT);
+		} else {
+			card_ins = TRUE;
+		}
+
+		if (card_ins) {
+			sd_info(("%s: SDIO slot %d: Full\n", __FUNCTION__, slot));
+			if (full_slot < 0)
+				full_slot = slot;
+		} else {
+			sd_info(("%s: SDIO slot %d: Empty\n", __FUNCTION__, slot));
+		}
+
+		if (sd->mem_space) {
+			sdstd_reg_unmap(sd->osh, (ulong)sd->mem_space, SDIOH_REG_WINSZ);
+			sd->mem_space = NULL;
+		}
+	}
+
+	if (full_slot < 0) {
+		sd_err(("No slots on SDIO controller are populated\n"));
+		return -1;
+	}
+
+/* XXX :Ugly define, is there a better way */
+#ifdef _WIN32
+	bar.HighPart = 0;
+	bar.LowPart = OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_BAR0 + (4*(full_slot + first_bar)), 4);
+	sd->mem_space = (volatile char *)sdstd_reg_map(sd->osh, (int32)&bar, SDIOH_REG_WINSZ);
+#else
+	bar = OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_BAR0 + (4*(full_slot + first_bar)), 4);
+	sd->mem_space = (volatile char *)sdstd_reg_map(sd->osh, (uintptr)bar, SDIOH_REG_WINSZ);
+#endif
+
+	sd_err(("Using slot %d at BAR%d [0x%08x] mem_space 0x%p\n",
+		full_slot,
+		(full_slot + first_bar),
+		OSL_PCI_READ_CONFIG(sd->osh, PCI_CFG_BAR0 + (4*(full_slot + first_bar)), 4),
+		sd->mem_space));
+
+	sd->adapter_slot = full_slot;
+
+	sd->version = sdstd_rreg16(sd, SD_HostControllerVersion) & 0xFF;
+	switch (sd->version) {
+		case 0:
+			sd_err(("Host Controller version 1.0, Vendor Revision: 0x%02x\n",
+				sdstd_rreg16(sd, SD_HostControllerVersion) >> 8));
+			break;
+		case 1:
+			sd_err(("Host Controller version 2.0, Vendor Revision: 0x%02x\n",
+				sdstd_rreg16(sd, SD_HostControllerVersion) >> 8));
+			break;
+		case 2:
+			sd_err(("Host Controller version 3.0, Vendor Revision: 0x%02x\n",
+				sdstd_rreg16(sd, SD_HostControllerVersion) >> 8));
+			break;
+		default:
+			sd_err(("%s: Host Controller version 0x%02x not supported.\n",
+			    __FUNCTION__, sd->version));
+			break;
+	}
+
+	sd->caps = sdstd_rreg(sd, SD_Capabilities);	/* Cache this for later use */
+	/* MSB 32 bits of caps supported in sdio 3.0 */
+	sd->caps3 = sdstd_rreg(sd, SD_Capabilities3); /* Cache this for later use */
+	sd3_trace(("sd3: %s: caps: 0x%x; MCCap:0x%x\n", __FUNCTION__, sd->caps, sd->curr_caps));
+	sd3_trace(("sd3: %s: caps3: 0x%x\n", __FUNCTION__, sd->caps3));
+	sd->curr_caps = sdstd_rreg(sd, SD_MaxCurCap);
+
+	sd_info(("%s: caps: 0x%x; MCCap:0x%x\n", __FUNCTION__, sd->caps, sd->curr_caps));
+
+	sdstd_set_dma_mode(sd, sd->sd_dma_mode);
+
+	sdstd_reset(sd, 1, 0);
+
+	/* Read SD4/SD1 mode */
+	if ((reg8 = sdstd_rreg8(sd, SD_HostCntrl))) {
+		if (reg8 & SD4_MODE) {
+			sd_err(("%s: Host cntrlr already in 4 bit mode: 0x%x\n",
+			        __FUNCTION__,  reg8));
+		}
+	}
+
+	/* Default power on mode is SD1 */
+	sd->sd_mode = SDIOH_MODE_SD1;
+	sd->polled_mode = TRUE;
+	sd->host_init_done = TRUE;
+	sd->card_init_done = FALSE;
+	sd->adapter_slot = full_slot;
+
+	/* XXX: If sd_uhsimode is disabled, which means that, use the HC in SDIO 2.0 mode. */
+	if (sd_uhsimode == SD3CLKMODE_DISABLED) {
+		sd->version = HOST_CONTR_VER_2;
+		sd3_trace(("%s:forcing to SDIO HC 2.0\n", __FUNCTION__));
+	}
+
+	if (sd->version == HOST_CONTR_VER_3) {
+		/* read host ctrl 2 */
+		uint16 reg16 = 0;
+		sd3_trace(("sd3: %s: HC3: reading additional regs\n", __FUNCTION__));
+
+		reg16 = sdstd_rreg16(sd, SD3_HostCntrl2);
+
+		sd_info(("%s: HCtrl: 0x%x; HCtrl2:0x%x\n", __FUNCTION__, reg8, reg16));
+		BCM_REFERENCE(reg16);
+
+		/* if HC supports 1.8V and one of the SDR/DDR modes, hc uhci support is PRESENT */
+		if ((GFIELD(sd->caps, CAP_VOLT_1_8)) &&
+			(GFIELD(sd->caps3, CAP3_SDR50_SUP) ||
+			GFIELD(sd->caps3, CAP3_SDR104_SUP) ||
+			GFIELD(sd->caps3, CAP3_DDR50_SUP)))
+			sd->host_UHSISupported = 1;
+	}
+
+#ifdef BCMQT
+	{
+		uint32 intmask;
+
+		/* FIX: force interrupts with QT sdio20 host */
+		/* pci cw [expr $def(configbase) +0x95] 1 2 */
+		intmask = OSL_PCI_READ_CONFIG(sd->osh, PCI_INT_MASK, 4);
+		intmask |= 0x0200;
+		OSL_PCI_WRITE_CONFIG(sd->osh, PCI_INT_MASK, 4, intmask);
+	}
+#endif
+	return (SUCCESS);
+}
+#define CMD5_RETRIES 200
+static int
+get_ocr(sdioh_info_t *sd, uint32 *cmd_arg, uint32 *cmd_rsp)
+{
+	int retries, status;
+
+	/* Get the Card's Operation Condition.  Occasionally the board
+	 * takes a while to become ready
+	 */
+	retries = CMD5_RETRIES;
+	do {
+		*cmd_rsp = 0;
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_5, *cmd_arg))
+		    != SUCCESS) {
+			sd_err(("%s: CMD5 failed\n", __FUNCTION__));
+			return status;
+		}
+		sdstd_cmd_getrsp(sd, cmd_rsp, 1);
+		if (!GFIELD(*cmd_rsp, RSP4_CARD_READY))
+			sd_trace(("%s: Waiting for card to become ready\n", __FUNCTION__));
+	} while ((!GFIELD(*cmd_rsp, RSP4_CARD_READY)) && --retries);
+	if (!retries)
+		return ERROR;
+
+	return (SUCCESS);
+}
+
+static int
+sdstd_client_init(sdioh_info_t *sd)
+{
+	uint32 cmd_arg, cmd_rsp;
+	int status;
+	uint8 fn_ints;
+	uint32 regdata;
+	uint16 powerstat = 0;
+
+	sd_trace(("%s: Powering up slot %d\n", __FUNCTION__, sd->adapter_slot));
+
+	/* Clear any pending ints */
+	sdstd_wreg16(sd, SD_IntrStatus, 0x1fff);
+	sdstd_wreg16(sd, SD_ErrorIntrStatus, 0x0fff);
+
+	/* Enable both Normal and Error Status.  This does not enable
+	 * interrupts, it only enables the status bits to
+	 * become 'live'
+	 */
+
+	if (!sd->host_UHSISupported)
+		sdstd_wreg16(sd, SD_IntrStatusEnable, 0x1ff);
+	else
+	{
+		/* INT_x interrupts, but DO NOT enable signalling [enable retuning
+		 * will happen later]
+		 */
+		sdstd_wreg16(sd, SD_IntrStatusEnable, 0x0fff);
+	}
+	sdstd_wreg16(sd, SD_ErrorIntrStatusEnable, 0xffff);
+
+	sdstd_wreg16(sd, SD_IntrSignalEnable, 0);	  /* Disable ints for now. */
+
+	if (sd->host_UHSISupported) {
+		/* when HC is started for SDIO 3.0 mode, start in lowest voltage mode first. */
+		powerstat = sdstd_start_power(sd, 1);
+		if (SDIO_OCR_READ_FAIL == powerstat) {
+			/* This could be because the device is 3.3V, and possible does
+			 * not have sdio3.0 support. So, try in highest voltage
+			 */
+			sd_err(("sdstd_start_power: legacy device: trying highest voltage\n"));
+			sd_err(("%s failed\n", __FUNCTION__));
+			return ERROR;
+		} else if (TRUE != powerstat) {
+			sd_err(("sdstd_start_power failed\n"));
+			return ERROR;
+		}
+	} else
+	/* XXX legacy driver: start in highest voltage mode first.
+	 * CAUTION: trying to start a legacy dhd with sdio3.0HC and sdio3.0 device could
+	 * burn the sdio3.0device if the device has started in 1.8V.
+	 */
+	if (TRUE != sdstd_start_power(sd, 0)) {
+		sd_err(("sdstd_start_power failed\n"));
+		return ERROR;
+	}
+
+	if (sd->num_funcs == 0) {
+		sd_err(("%s: No IO funcs!\n", __FUNCTION__));
+		return ERROR;
+	}
+
+	/* In SPI mode, issue CMD0 first */
+	if (sd->sd_mode == SDIOH_MODE_SPI) {
+		cmd_arg = 0;
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_0, cmd_arg))
+		    != SUCCESS) {
+			sd_err(("BCMSDIOH: cardinit: CMD0 failed!\n"));
+			return status;
+		}
+	}
+
+	if (sd->sd_mode != SDIOH_MODE_SPI) {
+		uint16 rsp6_status;
+
+		/* Card is operational. Ask it to send an RCA */
+		cmd_arg = 0;
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_3, cmd_arg))
+		    != SUCCESS) {
+			sd_err(("%s: CMD3 failed!\n", __FUNCTION__));
+			return status;
+		}
+
+		/* Verify the card status returned with the cmd response */
+		sdstd_cmd_getrsp(sd, &cmd_rsp, 1);
+		rsp6_status = GFIELD(cmd_rsp, RSP6_STATUS);
+		if (GFIELD(rsp6_status, RSP6STAT_COM_CRC_ERROR) ||
+		    GFIELD(rsp6_status, RSP6STAT_ILLEGAL_CMD) ||
+		    GFIELD(rsp6_status, RSP6STAT_ERROR)) {
+			sd_err(("%s: CMD3 response error. Response = 0x%x!\n",
+			        __FUNCTION__, rsp6_status));
+			return ERROR;
+		}
+
+		/* Save the Card's RCA */
+		sd->card_rca = GFIELD(cmd_rsp, RSP6_IO_RCA);
+		sd_info(("RCA is 0x%x\n", sd->card_rca));
+
+		if (rsp6_status)
+			sd_err(("raw status is 0x%x\n", rsp6_status));
+
+		/* Select the card */
+		cmd_arg = SFIELD(0, CMD7_RCA, sd->card_rca);
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_7, cmd_arg))
+		    != SUCCESS) {
+			sd_err(("%s: CMD7 failed!\n", __FUNCTION__));
+			return status;
+		}
+		sdstd_cmd_getrsp(sd, &cmd_rsp, 1);
+		if (cmd_rsp != SDIOH_CMD7_EXP_STATUS) {
+			sd_err(("%s: CMD7 response error. Response = 0x%x!\n",
+			        __FUNCTION__, cmd_rsp));
+			return ERROR;
+		}
+	}
+
+	/* Disable default/power-up device Card Detect (CD) pull up resistor on DAT3
+	 * via CCCR bus interface control register.  Set CD disable bit while leaving
+	 * others alone.
+	 */
+	if (sdstd_card_regread (sd, 0, SDIOD_CCCR_BICTRL, 1, &regdata) != SUCCESS) {
+		sd_err(("Disabling card detect: read of device CCCR BICTRL register failed\n"));
+		return ERROR;
+	}
+	regdata |= BUS_CARD_DETECT_DIS;
+
+	if (sdstd_card_regwrite (sd, 0, SDIOD_CCCR_BICTRL, 1, regdata) != SUCCESS) {
+		sd_err(("Disabling card detect: write of device CCCR BICTRL register failed\n"));
+		return ERROR;
+	}
+
+	sdstd_card_enablefuncs(sd);
+
+	if (!sdstd_bus_width(sd, sd_sdmode)) {
+		sd_err(("sdstd_bus_width failed\n"));
+		return ERROR;
+	}
+
+	set_client_block_size(sd, 1, sd_f1_blocksize);
+	fn_ints = INTR_CTL_FUNC1_EN;
+
+	if (sd->num_funcs >= 2) {
+		/* XXX Device side can't handle 512 yet */
+		set_client_block_size(sd, 2, sd_f2_blocksize /* BLOCK_SIZE_4328 */);
+		fn_ints |= INTR_CTL_FUNC2_EN;
+	}
+
+	/* Enable/Disable Client interrupts */
+	/* Turn on here but disable at host controller? */
+	if (sdstd_card_regwrite(sd, 0, SDIOD_CCCR_INTEN, 1,
+	                        (fn_ints | INTR_CTL_MASTER_EN)) != SUCCESS) {
+		sd_err(("%s: Could not enable ints in CCCR\n", __FUNCTION__));
+		return ERROR;
+	}
+
+	if (sd_uhsimode != SD3CLKMODE_DISABLED) {
+	/* Switch to High-speed clocking mode if both host and device support it */
+		if (sdstd_3_clock_wrapper(sd) != SUCCESS) {
+				sd_err(("sdstd_3_clock_wrapper failed\n"));
+				return ERROR;
+		}
+	} else
+	{
+		if (sdstd_clock_wrapper(sd)) {
+			sd_err(("sdstd_start_clock failed\n"));
+			return ERROR;
+		}
+	}
+	sd->card_init_done = TRUE;
+
+	return SUCCESS;
+}
+
+static int
+sdstd_clock_wrapper(sdioh_info_t *sd)
+{
+	uint ret = 0;
+	sd_trace(("%s:Enter\n", __FUNCTION__));
+	/* After configuring for High-Speed mode, set the desired clock rate. */
+	sdstd_set_highspeed_mode(sd, (bool)sd_hiok);
+
+#ifdef BCMFPGA_HW
+	/*
+	 * To make the SDIO clock 6.25Mhz for FPGA divide the
+	 * standard clock with 8 insted of default 2. 50Mhz/8 = 6.25Mhz
+	 * sdstd_clock_wrapper() is used to start clock for default and
+	 * high speed only.
+	 */
+	sd_err(("%s set 6.25Mhz clock speed for FPGA default or high speed\n", __FUNCTION__));
+	ret = sdstd_start_clock(sd, ((uint16)sd_divisor * 4));
+#else
+	ret = sdstd_start_clock(sd, (uint16)sd_divisor);
+#endif /* BCMFPGA_HW */
+	if (ret != TRUE) {
+		sd_err(("sdstd_start_clock failed\n"));
+		return ERROR;
+	}
+	return SUCCESS;
+}
+
+static int
+sdstd_3_clock_wrapper(sdioh_info_t *sd)
+{
+	int retclk = 0;
+	sd_info(("%s: Enter\n", __FUNCTION__));
+	if (sd->card_UHSI_voltage_Supported) {
+		/* check if clk config requested is supported by both host and target. */
+		retclk = sdstd_3_get_matching_uhsi_clkmode(sd, sd_uhsimode);
+
+		/* if no match for requested caps, try to get the max match possible */
+		if (retclk == -1) {
+			/* if auto enabled */
+			if (sd3_autoselect_uhsi_max == 1) {
+				retclk = sdstd_3_get_matching_uhsi_clkmode(sd, SD3CLKMODE_AUTO);
+				/* still NO match */
+				if (retclk == -1) {
+					/* NO match with HC and card capabilities. Now try the
+					 * High speed/legacy mode if possible.
+					 */
+
+					sd_err(("%s: Not able to set requested clock\n",
+					        __FUNCTION__));
+					return ERROR;
+				}
+			} else {
+				/* means user doesn't want auto clock. So return ERROR */
+				sd_err(("%s: Not able to set requested clock, Try"
+				"auto mode\n", __FUNCTION__));
+				return ERROR;
+			}
+		}
+
+		if (retclk != -1) {
+			/* set the current clk to be selected clock */
+			sd_uhsimode = retclk;
+
+			if (BCME_OK != sdstd_3_set_highspeed_uhsi_mode(sd, sd_uhsimode)) {
+				sd_err(("%s: Not able to set requested clock\n", __FUNCTION__));
+				return ERROR;
+			}
+		} else {
+			/* try legacy mode */
+			if (SUCCESS != sdstd_clock_wrapper(sd)) {
+				sd_err(("sdstd_start_clock failed\n"));
+				return ERROR;
+			}
+		}
+	} else {
+		sd_info(("%s: Legacy Mode Clock\n", __FUNCTION__));
+		/* try legacy mode */
+		if (SUCCESS != sdstd_clock_wrapper(sd)) {
+			sd_err(("%s sdstd_clock_wrapper failed\n", __FUNCTION__));
+			return ERROR;
+		}
+	}
+	return SUCCESS;
+}
+
+int
+sdstd_3_clk_tuning(sdioh_info_t *sd, uint32 sd3ClkMode)
+{
+	int status, lcount = 0, brr_count = 0;
+	uint16 val1 = 0, bufready = 0;
+	uint32 val2 = 0;
+	uint8 phase_info_local = 0;
+
+	sd3_trace(("sd3: %s: Enter\n", __FUNCTION__));
+	/* if (NOT SDR104) OR
+	 *	(SDR_50 AND sdr50_tuning_reqd is NOT enabled)
+	 * return success, as tuning not reqd.
+	 */
+	if (!sd->sd3_tuning_reqd) {
+		sd_info(("%s: Tuning NOT reqd!\n", __FUNCTION__));
+		return SUCCESS;
+	}
+
+	/* execute tuning procedure */
+
+	/* enable Buffer ready  status. [donot enable the interrupt right now] */
+	/* Execute tuning */
+	sd_trace(("sd3: %s: Execute tuning\n", __FUNCTION__));
+	val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+	val1 = SFIELD(val1, HOSTCtrl2_EXEC_TUNING, 1);
+	sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+
+	do {
+		sd3_trace(("sd3: %s: cmd19 issue\n", __FUNCTION__));
+		/* Issue cmd19 */
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_19, 0))
+			!= SUCCESS) {
+			sd_err(("%s: CMD19 failed\n", __FUNCTION__));
+			val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+			val1 = SFIELD(val1, HOSTCtrl2_EXEC_TUNING, 0);
+			val1 = SFIELD(val1, HOSTCtrl2_SAMPCLK_SEL, 0);
+			sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+			return status;
+		}
+
+		/* wait for buffer read ready */
+		brr_count = 0;
+		do {
+			bufready = sdstd_rreg16(sd, SD_IntrStatus);
+
+			if (GFIELD(bufready, INTSTAT_BUF_READ_READY))
+				break;
+
+			/* delay after checking bufready becuase INTSTAT_BUF_READ_READY
+			   might have been most likely set already in the first check
+			*/
+			OSL_DELAY(1);
+		} while (++brr_count < CLKTUNING_MAX_BRR_RETRIES);
+
+		/* buffer read ready timedout */
+		if (brr_count == CLKTUNING_MAX_BRR_RETRIES) {
+			sd_err(("%s: TUNINGFAILED: BRR response timedout!\n",
+				__FUNCTION__));
+			val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+			val1 = SFIELD(val1, HOSTCtrl2_EXEC_TUNING, 0);
+			val1 = SFIELD(val1, HOSTCtrl2_SAMPCLK_SEL, 0);
+			sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+			return ERROR;
+		}
+
+		/* In response to CMD19 card will send 64 magic bytes.
+		 * Current Aizyc HC h/w doesn't auto clear those bytes.
+		 * So read 64 bytes send by card.
+		 * Aizyc need to implement in hw to do an auto clear.
+		*/
+		if (sd3_sw_read_magic_bytes == TRUE)
+		{
+			uint8 l_cnt_1 = 0;
+			uint32 l_val_1 = 0;
+			for (l_cnt_1 = 0; l_cnt_1 < 16; l_cnt_1++) {
+				l_val_1 = sdstd_rreg(sd, SD_BufferDataPort0);
+				sd_trace(("%s:l_val_1 = 0x%x", __FUNCTION__, l_val_1));
+			}
+			BCM_REFERENCE(l_val_1);
+		}
+
+		/* clear BuffReadReady int */
+		bufready = SFIELD(bufready, INTSTAT_BUF_READ_READY, 1);
+		sdstd_wreg16(sd, SD_IntrStatus, bufready);
+
+		/* wait before continuing */
+		/* OSL_DELAY(PER_TRY_TUNING_DELAY_MS * 1000); */ /* Not required */
+
+		/* check execute tuning bit */
+		val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+		if (!GFIELD(val1, HOSTCtrl2_EXEC_TUNING)) {
+			/* done tuning, break from loop */
+			break;
+		}
+
+		/* max tuning iterations exceeded */
+		if (lcount++ > MAX_TUNING_ITERS) {
+			sd_err(("%s: TUNINGFAILED: Max tuning iterations"
+				"exceeded!\n", __FUNCTION__));
+			val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+			val1 = SFIELD(val1, HOSTCtrl2_EXEC_TUNING, 0);
+			val1 = SFIELD(val1, HOSTCtrl2_SAMPCLK_SEL, 0);
+			sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+			return ERROR;
+		}
+	} while (1);
+
+	val2 = sdstd_rreg(sd, SD3_Tuning_Info_Register);
+	phase_info_local = ((val2>>15)& 0x7);
+	sd_info(("Phase passed info: 0x%x\n", (val2>>8)& 0x3F));
+	sd_info(("Phase selected post tune: 0x%x\n", phase_info_local));
+
+	if (phase_info_local > SDSTD_MAX_TUNING_PHASE) {
+		sd_err(("!!Phase selected:%x\n", phase_info_local));
+	}
+
+	/* check sampling clk select */
+	val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+	if (!GFIELD(val1, HOSTCtrl2_SAMPCLK_SEL)) {
+		/* error in selecting clk */
+		sd_err(("%s: TUNINGFAILED: SamplClkSel failed!\n", __FUNCTION__));
+		val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+		val1 = SFIELD(val1, HOSTCtrl2_EXEC_TUNING, 0);
+		val1 = SFIELD(val1, HOSTCtrl2_SAMPCLK_SEL, 0);
+		sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+		return ERROR;
+	}
+/* done: */
+	sd_info(("%s: TUNING Success!\n", __FUNCTION__));
+	return SUCCESS;
+}
+
+void
+sdstd_3_enable_retuning_int(sdioh_info_t *sd)
+{
+	uint16 raw_int;
+	unsigned long flags;
+
+	sdstd_os_lock_irqsave(sd, &flags);
+	raw_int = sdstd_rreg16(sd, SD_IntrSignalEnable);
+	sdstd_wreg16(sd, SD_IntrSignalEnable, (raw_int | HC_INTR_RETUNING));
+	/* Enable retuning status */
+	raw_int = sdstd_rreg16(sd, SD_IntrStatusEnable);
+	sdstd_wreg16(sd, SD_IntrStatusEnable, (raw_int | HC_INTR_RETUNING));
+	sdstd_os_unlock_irqrestore(sd, &flags);
+}
+
+void
+sdstd_3_disable_retuning_int(sdioh_info_t *sd)
+{
+	uint16 raw_int;
+	unsigned long flags;
+
+	sdstd_os_lock_irqsave(sd, &flags);
+	sd->intmask &= ~HC_INTR_RETUNING;
+	raw_int = sdstd_rreg16(sd, SD_IntrSignalEnable);
+	sdstd_wreg16(sd, SD_IntrSignalEnable, (raw_int & (~HC_INTR_RETUNING)));
+	/* Disable retuning status */
+	raw_int = sdstd_rreg16(sd, SD_IntrStatusEnable);
+	sdstd_wreg16(sd, SD_IntrStatusEnable, (raw_int & (~HC_INTR_RETUNING)));
+	sdstd_os_unlock_irqrestore(sd, &flags);
+}
+
+bool
+sdstd_3_is_retuning_int_set(sdioh_info_t *sd)
+{
+	uint16 raw_int;
+
+	raw_int = sdstd_rreg16(sd, SD_IntrStatus);
+
+	if (GFIELD(raw_int, INTSTAT_RETUNING_INT))
+		return TRUE;
+
+	return FALSE;
+}
+
+/*
+	Assumption: sd3ClkMode is checked to be present in both host/card
+	capabilities before entering this function. VALID values for sd3ClkMode
+	in this function: SD3CLKMODE_2, 3, 4 [0 and 1 NOT supported as
+	they are legacy] For that, need to call
+	sdstd_3_get_matching_uhsi_clkmode()
+*/
+static int
+sdstd_3_set_highspeed_uhsi_mode(sdioh_info_t *sd, int sd3ClkMode)
+{
+	uint32 drvstrn;
+	int status;
+	uint8 hc_reg8;
+	uint16 val1 = 0, presetval = 0;
+	uint32 regdata;
+	uint32 ret = 0;
+
+	sd3_trace(("sd3: %s:enter:clkmode:%d\n", __FUNCTION__, sd3ClkMode));
+
+	hc_reg8 = sdstd_rreg8(sd, SD_HostCntrl);
+
+	if (HOST_SDR_UNSUPP == sd->global_UHSI_Supp) {
+		sd_err(("%s:Trying to set clk with unsupported global support\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* get [double check, as this is already done in
+		sdstd_3_get_matching_uhsi_clkmode] drvstrn
+	*/
+	if (!sdstd_3_get_matching_drvstrn(sd, sd3ClkMode, &drvstrn, &presetval)) {
+		sd_err(("%s:DRVStrn mismatch!: card strn:0x%x; HC preset"
+			"val:0x%x\n", __FUNCTION__, drvstrn, presetval));
+		return BCME_SDIO_ERROR;
+	}
+
+	/* also set driver type select in CCCR */
+	if ((status = sdstd_card_regwrite(sd, 0, SDIOD_CCCR_DRIVER_STRENGTH,
+		1, drvstrn)) != BCME_OK) {
+		sd_err(("%s:Setting SDIOD_CCCR_DRIVER_STRENGTH in card Failed!\n", __FUNCTION__));
+		return BCME_SDIO_ERROR;
+	}
+
+	/* ********** change Bus speed select in device */
+	if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+	                                 1, &regdata)) != SUCCESS) {
+		sd_err(("%s:FAILED 1\n", __FUNCTION__));
+		return BCME_SDIO_ERROR;
+	}
+	sd_info(("Attempting to change BSS.current val:0x%x\n", regdata));
+
+	if (regdata & SDIO_SPEED_SHS) {
+		sd_info(("Device supports High-Speed mode.\n"));
+		/* clear existing BSS */
+		regdata &= ~0xE;
+
+		regdata |= (sd3ClkMode << 1);
+
+		sd_info(("Writing %08x to Card at %08x\n",
+		         regdata, SDIOD_CCCR_SPEED_CONTROL));
+		if ((status = sdstd_card_regwrite(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+		                                  1, regdata)) != BCME_OK) {
+			sd_err(("%s:FAILED 2\n", __FUNCTION__));
+			return BCME_SDIO_ERROR;
+		}
+
+		if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+		                                 1, &regdata)) != BCME_OK) {
+			sd_err(("%s:FAILED 3\n", __FUNCTION__));
+			return BCME_SDIO_ERROR;
+		}
+
+		sd_info(("Read %08x from Card at %08x\n", regdata, SDIOD_CCCR_SPEED_CONTROL));
+	}
+	else {
+		sd_err(("Device does not support High-Speed Mode.\n"));
+	}
+
+	/* SD Clock Enable = 0 */
+	sdstd_wreg16(sd, SD_ClockCntrl,
+		sdstd_rreg16(sd, SD_ClockCntrl) & ~((uint16)0x4));
+
+	/* set to HighSpeed mode */
+	/* TBD: is these to change SD_HostCntrl reqd for UHSI? */
+	hc_reg8 = SFIELD(hc_reg8, HOST_HI_SPEED_EN, 1);
+	sdstd_wreg8(sd, SD_HostCntrl, hc_reg8);
+
+	/* set UHS Mode select in HC2 and also set preset */
+	val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+	val1 = SFIELD(val1, HOSTCtrl2_UHSMODE_SEL, sd3ClkMode);
+	if (TRUE != sd3_sw_override1) {
+		val1 = SFIELD(val1, HOSTCtrl2_PRESVAL_EN, 1);
+	} else {
+		/* set hC registers manually using the retreived values */
+		/* *set drvstrn */
+		val1 = SFIELD(val1, HOSTCtrl2_DRIVSTRENGTH_SEL,
+			GFIELD(presetval, PRESET_DRIVR_SELECT));
+		val1 = SFIELD(val1, HOSTCtrl2_PRESVAL_EN, 0);
+	}
+
+	/* finally write Hcontrol2 */
+	sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+
+	sd_err(("%s:HostCtrl2 final value:0x%x\n", __FUNCTION__, val1));
+#ifdef BCMFPGA_HW
+	/*
+	 * To make the SDIO clock 6.25Mhz for FPGA divide the
+	 * standard clock with 0x20. 208Mhz/32 = 6.25Mhz
+	 * this function is invoked to start clock for UHSI mode only
+	 */
+	sd_err(("%s set 6.25Mhz clock speed for UHSI speed\n", __FUNCTION__));
+	ret = sdstd_start_clock(sd, 0x20);
+#else
+	ret = sdstd_start_clock(sd, GFIELD(presetval, PRESET_CLK_DIV));
+#endif /* BCMFPGA_HW */
+	if (ret != TRUE) {
+	/* start clock : clk will be enabled inside. */
+		sd_err(("sdstd_start_clock failed\n"));
+		return ERROR;
+	}
+
+	/* execute first tuning procedure */
+	if (!sd3_sw_override1) {
+		if (SD3_TUNING_REQD(sd, sd3ClkMode)) {
+			sd_err(("%s: Tuning start..\n", __FUNCTION__));
+			sd->sd3_tuning_reqd = TRUE;
+			/* TBD: first time: enabling INT's could be problem? */
+			sdstd_3_start_tuning(sd);
+		}
+		else
+			sd->sd3_tuning_reqd = FALSE;
+	}
+
+	return BCME_OK;
+}
+
+/* Check & do tuning if required */
+void sdstd_3_check_and_do_tuning(sdioh_info_t *sd, int tuning_param)
+{
+	int retries = 0;
+
+	if (!sd->sd3_tuning_disable && sd->sd3_tuning_reqd) {
+		sd3_trace(("sd3: %s: tuning reqd\n", __FUNCTION__));
+			if (tuning_param == CHECK_TUNING_PRE_DATA) {
+			if (sd->sd3_tun_state == TUNING_ONGOING) {
+				retries = RETRIES_SMALL;
+				/* check if tuning is already going on */
+				while ((GFIELD(sdstd_rreg(sd, SD3_HostCntrl2),
+					HOSTCtrl2_EXEC_TUNING)) && retries--) {
+					if (retries == RETRIES_SMALL)
+						sd_err(("%s: Waiting for Tuning to complete\n",
+						__FUNCTION__));
+				}
+
+				if (!retries) {
+					sd_err(("%s: Tuning wait timeout\n", __FUNCTION__));
+					if (trap_errs)
+						ASSERT(0);
+				}
+			} else if (sd->sd3_tun_state == TUNING_START) {
+				/* check and start tuning if required. */
+				sd3_trace(("sd3 : %s : Doing Tuning before Data Transfer\n",
+					__FUNCTION__));
+				sdstd_3_start_tuning(sd);
+			}
+		} else if (tuning_param == CHECK_TUNING_POST_DATA) {
+			if (sd->sd3_tun_state == TUNING_START_AFTER_DAT) {
+				sd3_trace(("sd3: %s: tuning start\n", __FUNCTION__));
+				/* check and start tuning if required. */
+				sdstd_3_start_tuning(sd);
+			}
+		}
+	}
+}
+/* Need to run this function in interrupt-disabled context */
+bool sdstd_3_check_and_set_retuning(sdioh_info_t *sd)
+{
+	sd3_trace(("sd3: %s:\n", __FUNCTION__));
+
+	/* if already initiated, just return without anything */
+	if ((sd->sd3_tun_state == TUNING_START) ||
+		(sd->sd3_tun_state == TUNING_ONGOING) ||
+		(sd->sd3_tun_state == TUNING_START_AFTER_DAT)) {
+		/* do nothing */
+		return FALSE;
+	}
+
+	if (sd->sd3_dat_state == DATA_TRANSFER_IDLE) {
+		sd->sd3_tun_state = TUNING_START; /* tuning to be started by the tasklet */
+		return TRUE;
+	} else {
+		/* tuning to be started after finishing the existing data transfer */
+		sd->sd3_tun_state = TUNING_START_AFTER_DAT;
+	}
+	return FALSE;
+}
+
+int sdstd_3_get_data_state(sdioh_info_t *sd)
+{
+	return sd->sd3_dat_state;
+}
+
+void sdstd_3_set_data_state(sdioh_info_t *sd, int state)
+{
+	sd->sd3_dat_state = state;
+}
+
+int sdstd_3_get_tune_state(sdioh_info_t *sd)
+{
+	return sd->sd3_tun_state;
+}
+
+void sdstd_3_set_tune_state(sdioh_info_t *sd, int state)
+{
+	sd->sd3_tun_state = state;
+}
+
+uint8 sdstd_3_get_tuning_exp(sdioh_info_t *sd)
+{
+	if (sd_tuning_period == CAP3_RETUNING_TC_OTHER) {
+		return GFIELD(sd->caps3, CAP3_RETUNING_TC);
+	} else {
+		return (uint8)sd_tuning_period;
+	}
+}
+
+uint32 sdstd_3_get_uhsi_clkmode(sdioh_info_t *sd)
+{
+	return sd_uhsimode;
+}
+
+/* check, to see if the card supports driver_type corr to the driver_type
+	in preset value, which will be selected by requested UHSI mode
+    input:
+	clk mode: valid values: SD3CLKMODE_2_SDR50, SD3CLKMODE_3_SDR104,
+			SD3CLKMODE_4_DDR50, SD3CLKMODE_AUTO
+    outputs:
+	return_val: 	TRUE; if a matching drvstrn for the given clkmode is
+		found in both HC and card. otherwise, FALSE.
+		[other outputs below valid ONLY if return_val is TRUE]
+	drvstrn	:      driver strength read from CCCR.
+	presetval: 	value of preset reg, corr to the clkmode.
+ */
+static bool
+sdstd_3_get_matching_drvstrn(sdioh_info_t *sd, int sd3_requested_clkmode,
+	uint32 *drvstrn, uint16 *presetval)
+{
+	int status;
+	uint8 presetreg;
+	uint8 cccr_reqd_dtype_mask = 1;
+
+	sd3_trace(("sd3: %s:\n", __FUNCTION__));
+
+	if (sd3_requested_clkmode != SD3CLKMODE_AUTO) {
+		/* CARD: get the card driver strength from cccr */
+		if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_DRIVER_STRENGTH,
+			1, drvstrn)) != BCME_OK) {
+			sd_err(("%s:Reading SDIOD_CCCR_DRIVER_STRENGTH from card"
+				"Failed!\n", __FUNCTION__));
+			return FALSE;
+		}
+		if (TRUE != sd3_sw_override1) {
+		/* HOSTC: get the addr of preset register indexed by the clkmode */
+		presetreg = SD3_PresetValStart +
+			(2*sd3_requested_clkmode + 6);
+		*presetval = sdstd_rreg16(sd, presetreg);
+		} else {
+			/* Note: +3 for mapping between SD3CLKMODE_xxx and presetval_sw_table */
+			*presetval = presetval_sw_table[sd3_requested_clkmode + 3];
+		}
+		sd_err(("%s:reqCLK: %d, presetval: 0x%x\n",
+			__FUNCTION__, sd3_requested_clkmode, *presetval));
+
+		cccr_reqd_dtype_mask <<= GFIELD(*presetval, PRESET_DRIVR_SELECT);
+
+		/* compare/match */
+		if (!(cccr_reqd_dtype_mask & GFIELD(*drvstrn, SDIO_BUS_DRVR_TYPE_CAP))) {
+			sd_err(("%s:cccr_reqd_dtype_mask and SDIO_BUS_DRVR_TYPE_CAP"
+				"not matching!:reqd:0x%x, cap:0x%x\n", __FUNCTION__,
+				cccr_reqd_dtype_mask, GFIELD(*drvstrn, SDIO_BUS_DRVR_TYPE_CAP)));
+			return FALSE;
+		} else {
+			/* check if drive strength override is required. If so, first setit */
+			if (*dhd_sdiod_uhsi_ds_override != DRVSTRN_IGNORE_CHAR) {
+				int ds_offset = 0;
+				uint32 temp = 0;
+
+				/* drvstrn to reflect the preset val: this is default */
+				*drvstrn = GFIELD(*presetval, PRESET_DRIVR_SELECT);
+
+				/* now check override */
+				ds_offset = (((int)DRVSTRN_MAX_CHAR -
+					(int)(*dhd_sdiod_uhsi_ds_override)));
+				if ((ds_offset >= 0) && (ds_offset <= MAX_DTS_INDEX)) {
+					ds_offset = MAX_DTS_INDEX - ds_offset;
+					sd_err(("%s:Drive strength override: %c, offset: "
+						"%d, val: %d\n", __FUNCTION__,
+						*dhd_sdiod_uhsi_ds_override,
+						ds_offset, DTS_vals[ds_offset]));
+					temp = SFIELD(*drvstrn, SDIO_BUS_DRVR_TYPE_SEL,
+						DTS_vals[ds_offset]);
+					sd_err(("%s:DrvStrn orig: 0x%x, modif: 0x%x\n",
+						__FUNCTION__, *drvstrn, temp));
+					*drvstrn = temp;
+				} else {
+					/* else case is default: use preset val */
+					sd_err(("%s:override invalid: DrvStrn is from "
+						"preset: 0x%x\n",
+						__FUNCTION__, *drvstrn));
+				}
+			} else {
+				sd_err(("%s:DrvStrn is from preset: 0x%x\n",
+					__FUNCTION__, *drvstrn));
+			}
+		}
+	} else {
+		/* TBD check for sd3_requested_clkmode : -1 also.  */
+		sd_err(("%s: Automode not supported!\n", __FUNCTION__));
+		return FALSE;
+	}
+	return TRUE;
+}
+
+/* Returns a matching UHSI clk speed is found. If not, returns -1.
+	Also, if  sd3_requested_clkmode is -1, finds the closest max match clk and returns.
+ */
+static int
+sdstd_3_get_matching_uhsi_clkmode(sdioh_info_t *sd, int sd3_requested_clkmode)
+{
+	uint32 card_val_uhsisupp;
+	uint8 speedmask = 1;
+	uint32 drvstrn;
+	uint16 presetval;
+	int status;
+
+	sd3_trace(("sd3: %s:\n", __FUNCTION__));
+
+	sd->global_UHSI_Supp = HOST_SDR_UNSUPP;
+
+	/* for legacy/25MHz/50MHz bus speeds, no checks done here */
+	if ((sd3_requested_clkmode == SD3CLKMODE_0_SDR12) ||
+		(sd3_requested_clkmode == SD3CLKMODE_1_SDR25)) {
+		sd->global_UHSI_Supp = HOST_SDR_12_25;
+		return sd3_requested_clkmode;
+	}
+	/* get cap of card */
+	if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_UHSI_SUPPORT,
+	                                 1, &card_val_uhsisupp)) != BCME_OK) {
+		sd_err(("%s:SDIOD_CCCR_UHSI_SUPPORT query failed!\n", __FUNCTION__));
+		return -1;
+	}
+	sd_info(("%s:Read %08x from Card at %08x\n", __FUNCTION__,
+		card_val_uhsisupp, SDIOD_CCCR_UHSI_SUPPORT));
+
+	if (sd3_requested_clkmode != SD3CLKMODE_AUTO) {
+		/* Note: it is assumed that, following are executed when (sd3ClkMode >= 2) */
+		speedmask <<= (sd3_requested_clkmode - SD3CLKMODE_2_SDR50);
+
+		/* check first about 3.0 HS CLK modes */
+		if (!(GFIELD(sd->caps3, CAP3_30CLKCAP) & speedmask)) {
+			sd_err(("%s:HC does not support req 3.0 UHSI mode."
+				"requested:%d; capable:0x%x\n", __FUNCTION__,
+				sd3_requested_clkmode, GFIELD(sd->caps3, CAP3_30CLKCAP)));
+			return -1;
+		}
+
+		/* check first about 3.0 CARD CLK modes */
+		if (!(GFIELD(card_val_uhsisupp, SDIO_BUS_SPEED_UHSICAP) & speedmask)) {
+			sd_err(("%s:Card does not support req 3.0 UHSI mode. requested:%d;"
+				"capable:0x%x\n", __FUNCTION__, sd3_requested_clkmode,
+				GFIELD(card_val_uhsisupp, SDIO_BUS_SPEED_UHSICAP)));
+			return -1;
+		}
+
+		/* check, to see if the card supports driver_type corr to the
+			driver_type in preset value, which will be selected by
+			requested UHSI mode
+		*/
+		if (!sdstd_3_get_matching_drvstrn(sd, sd3_requested_clkmode,
+			&drvstrn, &presetval)) {
+			sd_err(("%s:DRVStrn mismatch!: card strn:0x%x; HC preset"
+				"val:0x%x\n", __FUNCTION__, drvstrn, presetval));
+			return -1;
+		}
+		/* success path. change the support variable accordingly */
+		sd->global_UHSI_Supp = HOST_SDR_50_104_DDR;
+		return sd3_requested_clkmode;
+	} else {
+		/* auto clk selection: get the highest clock capable by both card and HC */
+/* TBD		TOBE DONE */
+/* 		sd->global_UHSI_Supp = TRUE; on success */
+		return -1;
+	}
+}
+
+static int
+sdstd_3_sigvoltswitch_proc(sdioh_info_t *sd)
+{
+	int status;
+	uint32 cmd_rsp = 0, presst;
+	uint16 val1 = 0;
+
+	sd3_trace(("sd3: %s:\n", __FUNCTION__));
+
+	/* Issue cmd11 */
+	if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_11, 0))
+		!= SUCCESS) {
+		sd_err(("%s: CMD11 failed\n", __FUNCTION__));
+		return status;
+	}
+
+	/* check response */
+	sdstd_cmd_getrsp(sd, &cmd_rsp, 1);
+	if (
+		GFIELD(cmd_rsp, RSP1_ERROR) ||	/*  bit 19 */
+		GFIELD(cmd_rsp, RSP1_ILLEGAL_CMD) ||	/*  bit 22 */
+		GFIELD(cmd_rsp, RSP1_COM_CRC_ERROR) ||	/*  bit 23 */
+		GFIELD(cmd_rsp, RSP1_CARD_LOCKED)	/*  bit 25 */	) {
+		sd_err(("%s: FAIL:CMD11: cmd_resp:0x%x\n", __FUNCTION__, cmd_rsp));
+		return ERROR;
+	}
+
+	/* SD Clock Enable = 0 */
+	sdstd_wreg16(sd, SD_ClockCntrl,
+	             sdstd_rreg16(sd, SD_ClockCntrl) & ~((uint16)0x4));
+
+	/* check DAT[3..0] using Present State Reg. If not 0, error */
+	presst = sdstd_rreg(sd, SD_PresentState);
+	if (0 != GFIELD(presst, PRES_DAT_SIGNAL)) {
+		sd_err(("%s: FAIL: PRESTT:0x%x\n", __FUNCTION__, presst));
+		return ERROR;
+	}
+
+	/* turn 1.8V sig enable in HC2 */
+	val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+	val1 = SFIELD(val1, HOSTCtrl2_1_8SIG_EN, 1);
+	sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+
+	/* wait 5ms */
+	OSL_DELAY(5000);
+
+	/* check 1.8V sig enable in HC2. if cleared, error */
+	val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+
+	if (!GFIELD(val1, HOSTCtrl2_1_8SIG_EN)) {
+		sd_err(("%s: FAIL: HC2:1.8V_En:0x%x\n", __FUNCTION__, val1));
+		return ERROR;
+	}
+
+	/* SD Clock Enable = 1 */
+	val1 = sdstd_rreg16(sd, SD_ClockCntrl);
+	sdstd_wreg16(sd, SD_ClockCntrl, val1 | 0x4);
+
+	/* wait 1ms */
+	OSL_DELAY(1000);
+
+	/* check DAT[3..0] using Present State Reg. If not 0b1111, error */
+	presst = sdstd_rreg(sd, SD_PresentState);
+	if (0xf != GFIELD(presst, PRES_DAT_SIGNAL)) {
+		sd_err(("%s: FAIL: PRESTT_FINAL:0x%x\n", __FUNCTION__, presst));
+		return ERROR;
+	}
+
+	return (SUCCESS);
+}
+
+static int
+sdstd_set_highspeed_mode(sdioh_info_t *sd, bool HSMode)
+{
+	uint32 regdata;
+	int status;
+	uint8 reg8;
+
+	uint32 drvstrn;
+
+	reg8 = sdstd_rreg8(sd, SD_HostCntrl);
+
+	if (HSMode == TRUE) {
+		if (sd_hiok && (GFIELD(sd->caps, CAP_HIGHSPEED)) == 0) {
+			sd_err(("Host Controller does not support hi-speed mode.\n"));
+			return BCME_ERROR;
+		}
+
+		sd_info(("Attempting to enable High-Speed mode.\n"));
+
+		if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+		                                 1, &regdata)) != SUCCESS) {
+			return BCME_SDIO_ERROR;
+		}
+		if (regdata & SDIO_SPEED_SHS) {
+			sd_info(("Device supports High-Speed mode.\n"));
+
+			regdata |= SDIO_SPEED_EHS;
+
+			sd_info(("Writing %08x to Card at %08x\n",
+			         regdata, SDIOD_CCCR_SPEED_CONTROL));
+			if ((status = sdstd_card_regwrite(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+			                                  1, regdata)) != BCME_OK) {
+				return BCME_SDIO_ERROR;
+			}
+
+			if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+			                                 1, &regdata)) != BCME_OK) {
+				return BCME_SDIO_ERROR;
+			}
+
+			sd_info(("Read %08x to Card at %08x\n", regdata, SDIOD_CCCR_SPEED_CONTROL));
+
+			reg8 = SFIELD(reg8, HOST_HI_SPEED_EN, 1);
+
+			sd_err(("High-speed clocking mode enabled.\n"));
+		}
+		else {
+			sd_err(("Device does not support High-Speed Mode.\n"));
+			reg8 = SFIELD(reg8, HOST_HI_SPEED_EN, 0);
+		}
+	} else {
+		/* Force off device bit */
+		if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+		                                 1, &regdata)) != BCME_OK) {
+			return status;
+		}
+		if (regdata & SDIO_SPEED_EHS) {
+			regdata &= ~SDIO_SPEED_EHS;
+			if ((status = sdstd_card_regwrite(sd, 0, SDIOD_CCCR_SPEED_CONTROL,
+			                                  1, regdata)) != BCME_OK) {
+				return status;
+			}
+		}
+
+		sd_err(("High-speed clocking mode disabled.\n"));
+		reg8 = SFIELD(reg8, HOST_HI_SPEED_EN, 0);
+	}
+
+	if ((sd->host_UHSISupported) && (sd->card_UHSI_voltage_Supported)) {
+		/* also set the default driver strength in the card/HC [this is reqd because,
+			if earlier we selected any other drv_strn, we need to reset it]
+		*/
+		/* get the card driver strength from cccr */
+		if ((status = sdstd_card_regread(sd, 0, SDIOD_CCCR_DRIVER_STRENGTH,
+			1, &drvstrn)) != BCME_OK) {
+			sd_err(("%s:Reading SDIOD_CCCR_DRIVER_STRENGTH from card"
+				"Failed!\n", __FUNCTION__));
+			return BCME_SDIO_ERROR;
+		}
+
+		/* reset card drv strn */
+		drvstrn = SFIELD(drvstrn, SDIO_BUS_DRVR_TYPE_SEL, 0);
+
+		/* set card drv strn */
+		if ((status = sdstd_card_regwrite(sd, 0, SDIOD_CCCR_DRIVER_STRENGTH,
+			1, drvstrn)) != BCME_OK) {
+			sd_err(("%s:Setting SDIOD_CCCR_DRIVER_STRENGTH in"
+				"card Failed!\n", __FUNCTION__));
+			return BCME_SDIO_ERROR;
+		}
+	}
+
+	sdstd_wreg8(sd, SD_HostCntrl, reg8);
+
+	return BCME_OK;
+}
+
+/* Select DMA Mode:
+ * If dma_mode == DMA_MODE_AUTO, pick the "best" mode.
+ * Otherwise, pick the selected mode if supported.
+ * If not supported, use PIO mode.
+ */
+static int
+sdstd_set_dma_mode(sdioh_info_t *sd, int8 dma_mode)
+{
+	uint8 reg8, dma_sel_bits = SDIOH_SDMA_MODE;
+	int8 prev_dma_mode = sd->sd_dma_mode;
+
+	switch (prev_dma_mode) {
+		case DMA_MODE_AUTO:
+			sd_dma(("%s: Selecting best DMA mode supported by controller.\n",
+			          __FUNCTION__));
+			if (GFIELD(sd->caps, CAP_ADMA2)) {
+				sd->sd_dma_mode = DMA_MODE_ADMA2;
+				dma_sel_bits = SDIOH_ADMA2_MODE;
+			} else if (GFIELD(sd->caps, CAP_ADMA1)) {
+				sd->sd_dma_mode = DMA_MODE_ADMA1;
+				dma_sel_bits = SDIOH_ADMA1_MODE;
+			} else if (GFIELD(sd->caps, CAP_DMA)) {
+				sd->sd_dma_mode = DMA_MODE_SDMA;
+			} else {
+				sd->sd_dma_mode = DMA_MODE_NONE;
+			}
+			break;
+		case DMA_MODE_NONE:
+			sd->sd_dma_mode = DMA_MODE_NONE;
+			break;
+		case DMA_MODE_SDMA:
+			if (GFIELD(sd->caps, CAP_DMA)) {
+				sd->sd_dma_mode = DMA_MODE_SDMA;
+			} else {
+				sd_err(("%s: SDMA not supported by controller.\n", __FUNCTION__));
+				sd->sd_dma_mode = DMA_MODE_NONE;
+			}
+			break;
+		case DMA_MODE_ADMA1:
+			if (GFIELD(sd->caps, CAP_ADMA1)) {
+				sd->sd_dma_mode = DMA_MODE_ADMA1;
+				dma_sel_bits = SDIOH_ADMA1_MODE;
+			} else {
+				sd_err(("%s: ADMA1 not supported by controller.\n", __FUNCTION__));
+				sd->sd_dma_mode = DMA_MODE_NONE;
+			}
+			break;
+		case DMA_MODE_ADMA2:
+			if (GFIELD(sd->caps, CAP_ADMA2)) {
+				sd->sd_dma_mode = DMA_MODE_ADMA2;
+				dma_sel_bits = SDIOH_ADMA2_MODE;
+			} else {
+				sd_err(("%s: ADMA2 not supported by controller.\n", __FUNCTION__));
+				sd->sd_dma_mode = DMA_MODE_NONE;
+			}
+			break;
+		case DMA_MODE_ADMA2_64:
+			sd_err(("%s: 64b ADMA2 not supported by driver.\n", __FUNCTION__));
+			sd->sd_dma_mode = DMA_MODE_NONE;
+			break;
+		default:
+			sd_err(("%s: Unsupported DMA Mode %d requested.\n", __FUNCTION__,
+			        prev_dma_mode));
+			sd->sd_dma_mode = DMA_MODE_NONE;
+			break;
+	}
+
+	/* clear SysAddr, only used for SDMA */
+	sdstd_wreg(sd, SD_SysAddr, 0);
+
+	sd_err(("%s: %s mode selected.\n", __FUNCTION__, dma_mode_description[sd->sd_dma_mode]));
+
+	reg8 = sdstd_rreg8(sd, SD_HostCntrl);
+	reg8 = SFIELD(reg8, HOST_DMA_SEL, dma_sel_bits);
+	sdstd_wreg8(sd, SD_HostCntrl, reg8);
+	sd_dma(("%s: SD_HostCntrl=0x%02x\n", __FUNCTION__, reg8));
+
+	return BCME_OK;
+}
+
+#ifdef BCMDBG
+void
+print_regs(sdioh_info_t *sd)
+{
+	uint8 reg8 = 0;
+	uint16 reg16 = 0;
+	uint32 reg32 = 0;
+	uint8 presetreg;
+	int i;
+
+	reg8 = sdstd_rreg8(sd, SD_BlockSize);
+	printf("REGS: SD_BlockSize [004h]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_BlockCount);
+	printf("REGS: SD_BlockCount [006h]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_BlockSize);
+	printf("REGS: SD_BlockSize [004h]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_TransferMode);
+	printf("REGS: SD_TransferMode [00Ch]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_HostCntrl);
+	printf("REGS: SD_HostCntrl [028h]:0x%x\n", reg8);
+
+	reg32 = sdstd_rreg(sd, SD_PresentState);
+	printf("REGS: SD_PresentState [024h]:0x%x\n", reg32);
+
+	reg8 = sdstd_rreg8(sd, SD_PwrCntrl);
+	printf("REGS: SD_PwrCntrl [029h]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_BlockGapCntrl);
+	printf("REGS: SD_BlockGapCntrl [02Ah]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_WakeupCntrl);
+	printf("REGS: SD_WakeupCntrl [02Bh]:0x%x\n", reg8);
+
+	reg16 = sdstd_rreg16(sd, SD_ClockCntrl);
+	printf("REGS: SD_ClockCntrl [02Ch]:0x%x\n", reg16);
+
+	reg8 = sdstd_rreg8(sd, SD_TimeoutCntrl);
+	printf("REGS: SD_TimeoutCntrl [02Eh]:0x%x\n", reg8);
+
+	reg8 = sdstd_rreg8(sd, SD_SoftwareReset);
+	printf("REGS: SD_SoftwareReset [02Fh]:0x%x\n", reg8);
+
+	reg16 = sdstd_rreg16(sd, SD_IntrStatus);
+	printf("REGS: SD_IntrStatus [030h]:0x%x\n", reg16);
+
+	reg16 = sdstd_rreg16(sd, SD_ErrorIntrStatus);
+	printf("REGS: SD_ErrorIntrStatus [032h]:0x%x\n", reg16);
+
+	reg16 = sdstd_rreg16(sd, SD_IntrStatusEnable);
+	printf("REGS: SD_IntrStatusEnable [034h]:0x%x\n", reg16);
+
+	reg16 = sdstd_rreg16(sd, SD_ErrorIntrStatusEnable);
+	printf("REGS: SD_ErrorIntrStatusEnable [036h]:0x%x\n", reg16);
+
+	reg16 = sdstd_rreg16(sd, SD_IntrSignalEnable);
+	printf("REGS: SD_IntrSignalEnable [038h]:0x%x\n", reg16);
+
+	reg16 = sdstd_rreg16(sd, SD_ErrorIntrSignalEnable);
+	printf("REGS: SD_ErrorIntrSignalEnable [03Ah]:0x%x\n", reg16);
+
+	reg32 = sdstd_rreg(sd, SD_Capabilities);
+	printf("REGS: SD_Capabilities [040h]:0x%x\n", reg32);
+
+	reg32 = sdstd_rreg(sd, SD_MaxCurCap);
+	printf("REGS: SD_MaxCurCap [04Ah]:0x%x\n", reg32);
+
+	reg32 = sdstd_rreg(sd, SD_Capabilities3);
+	printf("REGS: SD_Capabilities3 [044h]:0x%x\n", reg32);
+
+	reg16 = sdstd_rreg16(sd, SD3_HostCntrl2);
+	printf("REGS: SD3_HostCntrl2 [03Eh]:0x%x\n", reg16);
+
+	for (i = 0; i < 8; i++) {
+		presetreg = SD3_PresetValStart + i*2;
+		printf("REGS: Presetvalreg:ix[%d]:0x%x, val=0x%x\n", i,
+		       presetreg, sdstd_rreg16(sd, presetreg));
+	}
+
+	reg16 = sdstd_rreg16(sd, SD_SlotInterruptStatus);
+	printf("REGS: SD_SlotInterruptStatus [0FCh]:0x%x\n", reg16);
+
+	reg16 = sdstd_rreg16(sd, SD_HostControllerVersion);
+	printf("REGS: SD_HostControllerVersion [0FEh]:0x%x\n", reg16);
+}
+#endif /* BCMDBG */
+
+/* XXX Per SDIO Host Controller Spec section 3.2.1
+	Note: for 2.x HC, new_sd_divisor should be a power of 2, but for 3.0
+	HC, new_sd_divisor should be a multiple of 2.
+*/
+bool
+sdstd_start_clock(sdioh_info_t *sd, uint16 new_sd_divisor)
+{
+	uint rc, count;
+	uint16 divisor;
+	uint16 regdata;
+	uint16 val1;
+
+	sd3_trace(("%s: starting clk\n", __FUNCTION__));
+	/* turn off HC clock */
+	sdstd_wreg16(sd, SD_ClockCntrl,
+	             sdstd_rreg16(sd, SD_ClockCntrl) & ~((uint16)0x4)); /*  Disable the HC clock */
+
+	/* Set divisor */
+	if (sd->host_UHSISupported) {
+#ifdef BCMDBG
+		if ((new_sd_divisor != 1) &&		/* 1 is a valid value */
+		    ((new_sd_divisor & (0x1)) ||	/* check for multiple of 2 */
+		     (new_sd_divisor == 0) ||
+		     (new_sd_divisor > 0x3ff))) {
+			sd_err(("3.0: Invalid clock divisor target: %d\n", new_sd_divisor));
+			return FALSE;
+		}
+#endif
+		divisor = (new_sd_divisor >> 1);
+	} else
+	{
+#ifdef BCMDBG
+		if ((new_sd_divisor & (new_sd_divisor-1)) ||
+		    (new_sd_divisor == 0)) {
+			sd_err(("Invalid clock divisor target: %d\n", new_sd_divisor));
+			return FALSE;
+		}
+#endif
+		/* new logic: if divisor > 256, restrict to 256 */
+		if (new_sd_divisor > 256)
+			new_sd_divisor = 256;
+		divisor = (new_sd_divisor >> 1) << 8;
+	}
+
+	sd_info(("Clock control is 0x%x\n", sdstd_rreg16(sd, SD_ClockCntrl)));
+	if (sd->host_UHSISupported) {
+		/* *get preset value and shift so that.
+		*	bits 0-7 are in 15-8 and 9-8 are in 7-6 of clkctrl
+		*/
+		val1 = divisor << 2;
+		val1 &= 0x0ffc;
+		val1 |= divisor >> 8;
+		val1 <<= 6;
+		printf("divisor:%x;val1:%x\n", divisor, val1);
+		sdstd_mod_reg16(sd, SD_ClockCntrl, 0xffC0, val1);
+	} else
+	{
+		sdstd_mod_reg16(sd, SD_ClockCntrl, 0xff00, divisor);
+	}
+
+	sd_err(("%s: Using clock divisor of %d (regval 0x%04x)\n", __FUNCTION__,
+	         new_sd_divisor, divisor));
+	if (new_sd_divisor > 0)
+		sd_err(("%s:now, divided clk is: %d Hz\n",
+		  __FUNCTION__, GFIELD(sd->caps, CAP_BASECLK)*1000000/new_sd_divisor));
+	else
+		sd_err(("Using Primary Clock Freq of %d MHz\n", GFIELD(sd->caps, CAP_BASECLK)));
+	sd_info(("Primary Clock Freq = %d MHz\n", GFIELD(sd->caps, CAP_BASECLK)));
+	if (GFIELD(sd->caps, CAP_TO_CLKFREQ) == 50) {
+		sd_info(("%s: Resulting SDIO clock is %d %s\n", __FUNCTION__,
+		        ((50 % new_sd_divisor) ? (50000 / new_sd_divisor) : (50 / new_sd_divisor)),
+		        ((50 % new_sd_divisor) ? "KHz" : "MHz")));
+	} else if (GFIELD(sd->caps, CAP_TO_CLKFREQ) == 48) {
+		sd_info(("%s: Resulting SDIO clock is %d %s\n", __FUNCTION__,
+		        ((48 % new_sd_divisor) ? (48000 / new_sd_divisor) : (48 / new_sd_divisor)),
+		        ((48 % new_sd_divisor) ? "KHz" : "MHz")));
+	} else if (GFIELD(sd->caps, CAP_TO_CLKFREQ) == 33) {
+		sd_info(("%s: Resulting SDIO clock is %d %s\n", __FUNCTION__,
+		        ((33 % new_sd_divisor) ? (33000 / new_sd_divisor) : (33 / new_sd_divisor)),
+		        ((33 % new_sd_divisor) ? "KHz" : "MHz")));
+	} else if (GFIELD(sd->caps, CAP_TO_CLKFREQ) == 31) {
+		sd_info(("%s: Resulting SDIO clock is %d %s\n", __FUNCTION__,
+		        ((31 % new_sd_divisor) ? (31000 / new_sd_divisor) : (31 / new_sd_divisor)),
+		        ((31 % new_sd_divisor) ? "KHz" : "MHz")));
+	} else if (GFIELD(sd->caps, CAP_TO_CLKFREQ) == 8) {
+		sd_info(("%s: Resulting SDIO clock is %d %s\n", __FUNCTION__,
+		        ((8 % new_sd_divisor) ? (8000 / new_sd_divisor) : (8 / new_sd_divisor)),
+		        ((8 % new_sd_divisor) ? "KHz" : "MHz")));
+	} else if (sd->controller_type == SDIOH_TYPE_BCM27XX) {
+		/* XXX - BCM 27XX Standard Host Controller returns 0 for CLKFREQ */
+	} else {
+		sd_err(("Need to determine divisor for %d MHz clocks\n",
+		        GFIELD(sd->caps, CAP_BASECLK)));
+		sd_err(("Consult SD Host Controller Spec: Clock Control Register\n"));
+		return (FALSE);
+	}
+
+	sdstd_or_reg16(sd, SD_ClockCntrl, 0x1); /*  Enable the clock */
+
+	/* Wait for clock to stabilize */
+	rc = (sdstd_rreg16(sd, SD_ClockCntrl) & 2);
+	count = 0;
+	while (!rc) {
+		OSL_DELAY(1);
+		sd_info(("Waiting for clock to become stable 0x%x\n", rc));
+		rc = (sdstd_rreg16(sd, SD_ClockCntrl) & 2);
+		count++;
+		if (count > 10000) {
+			sd_err(("%s:Clocks failed to stabilize after %u attempts\n",
+			        __FUNCTION__, count));
+			return (FALSE);
+		}
+	}
+	/* Turn on clock */
+	sdstd_or_reg16(sd, SD_ClockCntrl, 0x4);
+
+	OSL_DELAY(20);
+
+	/* Set timeout control (adjust default value based on divisor).
+	 * Disabling timeout interrupts during setting is advised by host spec.
+	 */
+#ifdef BCMQT
+	if (GFIELD(sd->caps, CAP_BASECLK) < 50)
+#endif
+	{
+		uint toval;
+
+		toval = sd_toctl;
+		divisor = new_sd_divisor;
+
+		while (toval && !(divisor & 1)) {
+			toval -= 1;
+			divisor >>= 1;
+		}
+
+		regdata = sdstd_rreg16(sd, SD_ErrorIntrStatusEnable);
+		sdstd_wreg16(sd, SD_ErrorIntrStatusEnable, (regdata & ~ERRINT_DATA_TIMEOUT_BIT));
+		sdstd_wreg8(sd, SD_TimeoutCntrl, (uint8)toval);
+		sdstd_wreg16(sd, SD_ErrorIntrStatusEnable, regdata);
+	}
+#ifdef BCMQT
+	else {
+		sd_info(("%s: REsetting err int control\n", __FUNCTION__));
+		/* XXX: turn off timeout INT, it resets clk ctrl bit */
+		regdata = sdstd_rreg16(sd, SD_ErrorIntrStatusEnable);
+		sdstd_wreg16(sd, SD_ErrorIntrStatusEnable, (regdata & ~ERRINT_DATA_TIMEOUT_BIT));
+	}
+#endif
+	OSL_DELAY(2);
+
+	sd_info(("Final Clock control is 0x%x\n", sdstd_rreg16(sd, SD_ClockCntrl)));
+
+	return TRUE;
+}
+
+/* XXX Per SDIO Host Controller Spec 3.3
+ * volts_req:
+ *		0 means default: select highest voltage.
+ *		1 means 1.8V
+ *		2 means 3.0V
+ *		3 means 3.3V
+ * returns
+ *	TRUE: no error
+ *	FALSE: general error
+ *	SDIO_OCR_READ_FAIL: ocr reading failure. Now the HC has to try in other available voltages.
+*/
+uint16
+sdstd_start_power(sdioh_info_t *sd, int volts_req)
+{
+	char *s;
+	uint32 cmd_arg;
+	uint32 cmd_rsp;
+	uint8 pwr = 0;
+	int volts = 0;
+	uint16 val1;
+	uint16 init_divider = 0;
+	uint8 baseclk = 0;
+	bool selhighest = (volts_req == 0) ? TRUE : FALSE;
+
+	/* reset the card uhsi volt support to false */
+	sd->card_UHSI_voltage_Supported = FALSE;
+
+	/* Ensure a power on reset by turning off bus power in case it happened to
+	 * be on already. (This might happen if driver doesn't unload/clean up correctly,
+	 * crash, etc.)  Leave off for 100ms to make sure the power off isn't
+	 * ignored/filtered by the device.  Note we can't skip this step if the power is
+	 * off already since we don't know how long it has been off before starting
+	 * the driver.
+	 */
+	sdstd_wreg8(sd, SD_PwrCntrl, 0);
+	sd_info(("Turning off VDD/bus power briefly (100ms) to ensure reset\n"));
+	OSL_DELAY(100000);
+
+	/* For selecting highest available voltage, start from lowest and iterate */
+	if (!volts_req)
+		volts_req = 1;
+
+	s = NULL;
+
+	if (volts_req == 1) {
+		if (GFIELD(sd->caps, CAP_VOLT_1_8)) {
+			volts = 5;
+			s = "1.8";
+			if (FALSE == selhighest)
+				goto voltsel;
+			else
+				volts_req++;
+		} else {
+			sd_err(("HC doesn't support voltage! trying higher voltage: %d\n", volts));
+			volts_req++;
+		}
+	}
+
+	if (volts_req == 2) {
+		if (GFIELD(sd->caps, CAP_VOLT_3_0)) {
+			volts = 6;
+			s = "3.0";
+			if (FALSE == selhighest)
+				goto voltsel;
+			else volts_req++;
+		} else {
+			sd_err(("HC doesn't support voltage! trying higher voltage: %d\n", volts));
+			volts_req++;
+		}
+	}
+
+	if (volts_req == 3) {
+		if (GFIELD(sd->caps, CAP_VOLT_3_3)) {
+			volts = 7;
+			s = "3.3";
+		} else {
+			if ((FALSE == selhighest) || (volts == 0)) {
+				sd_err(("HC doesn't support any voltage! error!\n"));
+				return FALSE;
+			}
+		}
+	}
+
+	/* XXX
+	 * if UHSI is NOT supported, check for other voltages also. This is a safety measure
+	 * for embedded devices also, so that HC starts in lower power first. If this
+	 * function fails, the caller may disable UHSISupported
+	 * and call start power again to check support in higher voltages.
+	 */
+
+voltsel:
+	pwr = SFIELD(pwr, PWR_VOLTS, volts);
+	pwr = SFIELD(pwr, PWR_BUS_EN, 1);
+	sdstd_wreg8(sd, SD_PwrCntrl, pwr); /* Set Voltage level */
+	sd_info(("Setting Bus Power to %s Volts\n", s));
+	BCM_REFERENCE(s);
+
+	/*
+	 * PR101766 : BRCM SDIO3.0 card is an embedded SD device. It is not a SD card.
+	 * VDDIO signalling will be tied to 1.8v level on all SDIO3.0 based boards.
+	 * So program the HC to drive VDDIO at 1.8v level.
+	 */
+	if ((sd->version == HOST_CONTR_VER_3) && (volts == 5)) {
+		val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+		val1 = SFIELD(val1, HOSTCtrl2_1_8SIG_EN, 1);
+		sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+	}
+
+	/* Wait for 500ms for power to stabilize.  Some designs have reset IC's
+	 * which can hold reset low for close to 300ms.  In addition there can
+	 * be ramp time for VDD and/or VDDIO which might be provided from a LDO.
+	 * For these reasons we need a pretty conservative delay here to have
+	 * predictable reset behavior in the face of an unknown design.
+	 */
+	OSL_DELAY(500000);
+
+	baseclk = GFIELD(sd->caps, CAP_BASECLK);
+	sd_info(("%s:baseclk: %d MHz\n",	__FUNCTION__, baseclk));
+	/* for 3.0, find divisor */
+	if (sd->host_UHSISupported) {
+		/* ToDo : Dynamic modification of preset value table based on base clk */
+		sd3_trace(("sd3: %s: checking divisor\n", __FUNCTION__));
+		if (GFIELD(sd->caps3, CAP3_CLK_MULT) != 0)	{
+			sd_err(("%s:Possible error: CLK Mul 1 CLOCKING NOT supported!\n",
+				__FUNCTION__));
+			return FALSE;
+		} else {
+			/*  calculate dividor, which leads to 400KHz.  */
+			init_divider = baseclk*10/4; /* baseclk*1000000/(400000); */
+			/* make it a multiple of 2. */
+			init_divider += (init_divider & 0x1);
+			sd_err(("%s:divider used for init:%d\n",
+				__FUNCTION__, init_divider));
+		}
+	} else {
+		/* Note: sd_divisor assumes that SDIO Base CLK is 50MHz. */
+		int final_freq_based_on_div = 50/sd_divisor;
+	    if (baseclk > 50)
+			sd_divisor = baseclk/final_freq_based_on_div;
+		/* TBD: merge both SDIO 2.0 and 3.0 to share same divider logic */
+		init_divider = baseclk*10/4; /* baseclk*1000000/(400000); */
+		/* find next power of 2 */
+		NEXT_POW2(init_divider);
+		sd_err(("%s:NONUHSI: divider used for init:%d\n",
+			__FUNCTION__, init_divider));
+	}
+
+	/* Start at ~400KHz clock rate for initialization */
+	if (!sdstd_start_clock(sd, init_divider)) {
+		sd_err(("%s: sdstd_start_clock failed\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	/* Get the Card's Operation Condition.  Occasionally the board
+	 * takes a while to become ready
+	 */
+	cmd_arg = 0;
+	cmd_rsp = 0;
+	if (get_ocr(sd, &cmd_arg, &cmd_rsp) != SUCCESS) {
+		sd_err(("%s: Failed to get OCR bailing\n", __FUNCTION__));
+		/* No need to reset as not sure in what state the card is. */
+		return SDIO_OCR_READ_FAIL;
+	}
+
+	sd_info(("cmd_rsp = 0x%x\n", cmd_rsp));
+	sd_info(("mem_present = %d\n", GFIELD(cmd_rsp, RSP4_MEM_PRESENT)));
+	sd_info(("num_funcs = %d\n", GFIELD(cmd_rsp, RSP4_NUM_FUNCS)));
+	sd_info(("card_ready = %d\n", GFIELD(cmd_rsp, RSP4_CARD_READY)));
+	sd_info(("OCR = 0x%x\n", GFIELD(cmd_rsp, RSP4_IO_OCR)));
+
+	/* Verify that the card supports I/O mode */
+	if (GFIELD(cmd_rsp, RSP4_NUM_FUNCS) == 0) {
+		sd_err(("%s: Card does not support I/O\n", __FUNCTION__));
+		return ERROR;
+	}
+	sd->num_funcs = GFIELD(cmd_rsp, RSP4_NUM_FUNCS);
+
+	/* Examine voltage: Arasan only supports 3.3 volts,
+	 * so look for 3.2-3.3 Volts and also 3.3-3.4 volts.
+	 */
+
+	/* XXX Pg 10 SDIO spec v1.10 */
+	if ((GFIELD(cmd_rsp, RSP4_IO_OCR) & (0x3 << 20)) == 0) {
+		sd_err(("This client does not support 3.3 volts!\n"));
+		return ERROR;
+	}
+	sd_info(("Leaving bus power at 3.3 Volts\n"));
+
+	cmd_arg = SFIELD(0, CMD5_OCR, 0xfff000);
+	/* if HC uhsi supported and card voltage set is 3.3V then switch to 1.8V */
+	if ((sd->host_UHSISupported) && (volts == 5)) {
+		/* set S18R also */
+		cmd_arg = SFIELD(cmd_arg, CMD5_S18R, 1);
+	}
+	cmd_rsp = 0;
+	get_ocr(sd, &cmd_arg, &cmd_rsp);
+	sd_info(("OCR = 0x%x\n", GFIELD(cmd_rsp, RSP4_IO_OCR)));
+
+	if ((sd->host_UHSISupported)) {
+		/* card responded with s18A => card supports sdio3.0,do tuning proc */
+		if (GFIELD(cmd_rsp, RSP4_S18A) == 1) {
+			if (sdstd_3_sigvoltswitch_proc(sd)) {
+				/* continue with legacy way of working */
+				sd_err(("%s: voltage switch not done. error, stopping\n",
+					__FUNCTION__));
+				/* How to gracefully proceced here? */
+				return FALSE;
+			} else {
+				sd->card_UHSI_voltage_Supported = TRUE;
+				sd_err(("%s: voltage switch SUCCESS!\n", __FUNCTION__));
+			}
+		} else {
+				/* This could happen for 2 cases.
+				 * 1) means card is NOT sdio3.0 . Note that
+				 *    card_UHSI_voltage_Supported is already false.
+				 * 2) card is sdio3.0 but it is already in 1.8V.
+				 *    But now, how to change host controller's voltage?
+				 * In this case we need to do the following.
+				 * sd->card_UHSI_voltage_Supported = TRUE;
+				 * turn 1.8V sig enable in HC2
+				 * val1 = sdstd_rreg16(sd, SD3_HostCntrl2);
+				 * val1 = SFIELD(val1, HOSTCtrl2_1_8SIG_EN, 1);
+				 * sdstd_wreg16(sd, SD3_HostCntrl2, val1);
+				 */
+
+				sd_info(("%s: Not sdio3.0: host_UHSISupported: %d; HC volts=%d\n",
+					__FUNCTION__, sd->host_UHSISupported, volts));
+		}
+	} else {
+		sd_info(("%s: Legacy [non sdio3.0] HC\n", __FUNCTION__));
+	}
+
+	return TRUE;
+}
+
+bool
+sdstd_bus_width(sdioh_info_t *sd, int new_mode)
+{
+	uint32 regdata;
+	int status;
+	uint8 reg8;
+
+	sd_trace(("%s\n", __FUNCTION__));
+	if (sd->sd_mode == new_mode) {
+		sd_info(("%s: Already at width %d\n", __FUNCTION__, new_mode));
+		/* Could exit, but continue just in case... */
+	}
+
+	/* Set client side via reg 0x7 in CCCR */
+	if ((status = sdstd_card_regread (sd, 0, SDIOD_CCCR_BICTRL, 1, &regdata)) != SUCCESS)
+		return (bool)status;
+	regdata &= ~BUS_SD_DATA_WIDTH_MASK;
+	if (new_mode == SDIOH_MODE_SD4) {
+		sd_info(("Changing to SD4 Mode\n"));
+		regdata |= SD4_MODE;
+	} else if (new_mode == SDIOH_MODE_SD1) {
+		sd_info(("Changing to SD1 Mode\n"));
+	} else {
+		sd_err(("SPI Mode not supported by Standard Host Controller\n"));
+	}
+
+	if ((status = sdstd_card_regwrite (sd, 0, SDIOD_CCCR_BICTRL, 1, regdata)) != SUCCESS)
+		return (bool)status;
+
+	if (sd->host_UHSISupported) {
+		uint32 card_asyncint = 0;
+		uint16 host_asyncint = 0;
+
+		if ((status = sdstd_card_regread (sd, 0, SDIOD_CCCR_INTR_EXTN, 1,
+			&card_asyncint)) != SUCCESS) {
+			sd_err(("%s:INTR EXT getting failed!, ignoring\n", __FUNCTION__));
+		} else {
+			host_asyncint = sdstd_rreg16(sd, SD3_HostCntrl2);
+
+			/* check if supported by host and card */
+			if ((regdata & SD4_MODE) &&
+				(GFIELD(card_asyncint, SDIO_BUS_ASYNCINT_CAP)) &&
+				(GFIELD(sd->caps, CAP_ASYNCINT_SUP))) {
+				/* set enable async int in card */
+				card_asyncint = SFIELD(card_asyncint, SDIO_BUS_ASYNCINT_SEL, 1);
+
+				if ((status = sdstd_card_regwrite (sd, 0,
+					SDIOD_CCCR_INTR_EXTN, 1, card_asyncint)) != SUCCESS)
+					sd_err(("%s:INTR EXT setting failed!, ignoring\n",
+					__FUNCTION__));
+				else {
+					/* set enable async int in host */
+					host_asyncint = SFIELD(host_asyncint,
+						HOSTCtrl2_ASYINT_EN, 1);
+					sdstd_wreg16(sd, SD3_HostCntrl2, host_asyncint);
+				}
+			} else {
+				sd_err(("%s:INTR EXT NOT supported by either host or"
+					"card!, ignoring\n", __FUNCTION__));
+			}
+		}
+	}
+
+	/* Set host side via Host reg */
+	reg8 = sdstd_rreg8(sd, SD_HostCntrl) & ~SD4_MODE;
+	if (new_mode == SDIOH_MODE_SD4)
+		reg8 |= SD4_MODE;
+	sdstd_wreg8(sd, SD_HostCntrl, reg8);
+
+	sd->sd_mode = new_mode;
+
+	return TRUE;
+}
+
+static int
+sdstd_driver_init(sdioh_info_t *sd)
+{
+	sd_trace(("%s\n", __FUNCTION__));
+	sd->sd3_tuning_reqd = FALSE;
+	sd->sd3_tuning_disable = FALSE;
+	if ((sdstd_host_init(sd)) != SUCCESS) {
+		return ERROR;
+	}
+
+	/* Give WL_reset before sending CMD5 to dongle for Revx SDIO3 HC's */
+	if ((sd->controller_type == SDIOH_TYPE_RICOH_R5C822) && (sd->version == HOST_CONTR_VER_3))
+	{
+		sdstd_wreg16(sd, SD3_WL_BT_reset_register, 0x8);
+		OSL_DELAY(sd_delay_value);
+		sdstd_wreg16(sd, SD3_WL_BT_reset_register, 0x0);
+		OSL_DELAY(500000);
+	}
+
+	if (sdstd_client_init(sd) != SUCCESS) {
+		return ERROR;
+	}
+
+	/* if the global cap matched and is SDR 104/50 [if 50 it is reqd] enable tuning. */
+	if ((TRUE != sd3_sw_override1) && SD3_TUNING_REQD(sd, sd_uhsimode)) {
+			sd->sd3_tuning_reqd = TRUE;
+
+			/* init OS structs for tuning */
+			sdstd_3_osinit_tuning(sd);
+
+			/* enable HC tuning interrupt OR timer based on tuning method */
+			if (GFIELD(sd->caps3, CAP3_RETUNING_MODES)) {
+				/* enable both RTReq and timer */
+				sd->intmask |= HC_INTR_RETUNING;
+				sdstd_wreg16(sd, SD_IntrSignalEnable, sd->intmask);
+#ifdef BCMSDYIELD
+				if (sd_forcerb)
+					sdstd_rreg16(sd, SD_IntrSignalEnable); /* Sync readback */
+#endif /* BCMSDYIELD */
+			}
+		}
+
+	return SUCCESS;
+}
+
+static int
+sdstd_get_cisaddr(sdioh_info_t *sd, uint32 regaddr)
+{
+	/* read 24 bits and return valid 17 bit addr */
+	int i;
+	uint32 scratch, regdata;
+	uint8 *ptr = (uint8 *)&scratch;
+	for (i = 0; i < 3; i++) {
+		if ((sdstd_card_regread (sd, 0, regaddr, 1, &regdata)) != SUCCESS)
+			sd_err(("%s: Can't read!\n", __FUNCTION__));
+
+		*ptr++ = (uint8) regdata;
+		regaddr++;
+	}
+	/* Only the lower 17-bits are valid */
+	scratch = ltoh32(scratch);
+	scratch &= 0x0001FFFF;
+	return (scratch);
+}
+
+static int
+sdstd_card_enablefuncs(sdioh_info_t *sd)
+{
+	int status;
+	uint32 regdata;
+	uint32 fbraddr;
+	uint8 func;
+
+	sd_trace(("%s\n", __FUNCTION__));
+
+	/* Get the Card's common CIS address */
+	sd->com_cis_ptr = sdstd_get_cisaddr(sd, SDIOD_CCCR_CISPTR_0);
+	sd->func_cis_ptr[0] = sd->com_cis_ptr;
+	sd_info(("%s: Card's Common CIS Ptr = 0x%x\n", __FUNCTION__, sd->com_cis_ptr));
+
+	/* Get the Card's function CIS (for each function) */
+	for (fbraddr = SDIOD_FBR_STARTADDR, func = 1;
+	     func <= sd->num_funcs; func++, fbraddr += SDIOD_FBR_SIZE) {
+		sd->func_cis_ptr[func] = sdstd_get_cisaddr(sd, SDIOD_FBR_CISPTR_0 + fbraddr);
+		sd_info(("%s: Function %d CIS Ptr = 0x%x\n",
+		         __FUNCTION__, func, sd->func_cis_ptr[func]));
+	}
+
+	/* Enable function 1 on the card */
+	regdata = SDIO_FUNC_ENABLE_1;
+	if ((status = sdstd_card_regwrite(sd, 0, SDIOD_CCCR_IOEN, 1, regdata)) != SUCCESS)
+		return status;
+
+	return SUCCESS;
+}
+
+/* Read client card reg */
+static int
+sdstd_card_regread(sdioh_info_t *sd, int func, uint32 regaddr, int regsize, uint32 *data)
+{
+	int status;
+	uint32 cmd_arg;
+	uint32 rsp5;
+
+#ifdef BCMDBG
+	if (sdstd_rreg16 (sd, SD_ErrorIntrStatus) != 0) {
+		sd_err(("%s: Entering: ErrorintrStatus 0x%x, intstat = 0x%x\n",
+		        __FUNCTION__, sdstd_rreg16(sd, SD_ErrorIntrStatus),
+		        sdstd_rreg16(sd, SD_IntrStatus)));
+	}
+#endif
+
+	cmd_arg = 0;
+
+	if ((func == 0) || (regsize == 1)) {
+		cmd_arg = SFIELD(cmd_arg, CMD52_FUNCTION, func);
+		cmd_arg = SFIELD(cmd_arg, CMD52_REG_ADDR, regaddr);
+		cmd_arg = SFIELD(cmd_arg, CMD52_RW_FLAG, SDIOH_XFER_TYPE_READ);
+		cmd_arg = SFIELD(cmd_arg, CMD52_RAW, 0);
+		cmd_arg = SFIELD(cmd_arg, CMD52_DATA, 0);
+
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_52, cmd_arg))
+		    != SUCCESS)
+			return status;
+
+		sdstd_cmd_getrsp(sd, &rsp5, 1);
+		if (sdstd_rreg16(sd, SD_ErrorIntrStatus) != 0) {
+			sd_err(("%s: 1: ErrorintrStatus 0x%x\n",
+			        __FUNCTION__, sdstd_rreg16(sd, SD_ErrorIntrStatus)));
+		}
+
+		if (GFIELD(rsp5, RSP5_FLAGS) != 0x10)
+			sd_err(("%s: rsp5 flags is 0x%x\t %d\n",
+			        __FUNCTION__, GFIELD(rsp5, RSP5_FLAGS), func));
+
+		if (GFIELD(rsp5, RSP5_STUFF))
+			sd_err(("%s: rsp5 stuff is 0x%x: should be 0\n",
+			        __FUNCTION__, GFIELD(rsp5, RSP5_STUFF)));
+		*data = GFIELD(rsp5, RSP5_DATA);
+
+		sd_data(("%s: Resp data(0x%x)\n", __FUNCTION__, *data));
+	} else {
+		cmd_arg = SFIELD(cmd_arg, CMD53_BYTE_BLK_CNT, regsize);
+		cmd_arg = SFIELD(cmd_arg, CMD53_OP_CODE, 1);	/* XXX SDIO spec v 1.10, Sec 5.3 */
+		cmd_arg = SFIELD(cmd_arg, CMD53_BLK_MODE, 0);
+		cmd_arg = SFIELD(cmd_arg, CMD53_FUNCTION, func);
+		cmd_arg = SFIELD(cmd_arg, CMD53_REG_ADDR, regaddr);
+		cmd_arg = SFIELD(cmd_arg, CMD53_RW_FLAG, SDIOH_XFER_TYPE_READ);
+
+		sd->data_xfer_count = regsize;
+
+		/* sdstd_cmd_issue() returns with the command complete bit
+		 * in the ISR already cleared
+		 */
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_53, cmd_arg))
+		    != SUCCESS)
+			return status;
+
+		sdstd_cmd_getrsp(sd, &rsp5, 1);
+
+		if (GFIELD(rsp5, RSP5_FLAGS) != 0x10)
+			sd_err(("%s: rsp5 flags is 0x%x\t %d\n",
+			        __FUNCTION__, GFIELD(rsp5, RSP5_FLAGS), func));
+
+		if (GFIELD(rsp5, RSP5_STUFF))
+			sd_err(("%s: rsp5 stuff is 0x%x: should be 0\n",
+			        __FUNCTION__, GFIELD(rsp5, RSP5_STUFF)));
+
+		if (sd->polled_mode) {
+			volatile uint16 int_reg;
+			int retries = RETRIES_LARGE;
+
+			/* Wait for Read Buffer to become ready */
+			do {
+				sdstd_os_yield(sd);
+				int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+			} while (--retries && (GFIELD(int_reg, INTSTAT_BUF_READ_READY) == 0));
+
+			if (!retries) {
+				sd_err(("%s: Timeout on Buf_Read_Ready: "
+				        "intStat: 0x%x errint: 0x%x PresentState 0x%x\n",
+				        __FUNCTION__, int_reg,
+				        sdstd_rreg16(sd, SD_ErrorIntrStatus),
+				        sdstd_rreg(sd, SD_PresentState)));
+				sdstd_check_errs(sd, SDIOH_CMD_53, cmd_arg);
+				return (ERROR);
+			}
+
+			/* Have Buffer Ready, so clear it and read the data */
+			sdstd_wreg16(sd, SD_IntrStatus, SFIELD(0, INTSTAT_BUF_READ_READY, 1));
+			if (regsize == 2)
+				*data = sdstd_rreg16(sd, SD_BufferDataPort0);
+			else
+				*data = sdstd_rreg(sd, SD_BufferDataPort0);
+
+			sd_data(("%s: Resp data(0x%x)\n", __FUNCTION__, *data));
+			/* Check Status.
+			 * After the data is read, the Transfer Complete bit should be on
+			 */
+			retries = RETRIES_LARGE;
+			do {
+				int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+			} while (--retries && (GFIELD(int_reg, INTSTAT_XFER_COMPLETE) == 0));
+
+			/* Check for any errors from the data phase */
+			if (sdstd_check_errs(sd, SDIOH_CMD_53, cmd_arg))
+				return ERROR;
+
+			if (!retries) {
+				sd_err(("%s: Timeout on xfer complete: "
+				        "intr 0x%04x err 0x%04x state 0x%08x\n",
+				        __FUNCTION__, int_reg,
+				        sdstd_rreg16(sd, SD_ErrorIntrStatus),
+				        sdstd_rreg(sd, SD_PresentState)));
+				return (ERROR);
+			}
+
+			sdstd_wreg16(sd, SD_IntrStatus, SFIELD(0, INTSTAT_XFER_COMPLETE, 1));
+		}
+	}
+	if (sd->polled_mode) {
+		if (regsize == 2)
+			*data &= 0xffff;
+	}
+	return SUCCESS;
+}
+
+bool
+check_client_intr(sdioh_info_t *sd)
+{
+	uint16 raw_int, cur_int, old_int;
+
+	raw_int = sdstd_rreg16(sd, SD_IntrStatus);
+	cur_int = raw_int & sd->intmask;
+
+	if (!cur_int) {
+		/* Not an error -- might share interrupts... */
+		return FALSE;
+	}
+
+	if (GFIELD(cur_int, INTSTAT_CARD_INT)) {
+		unsigned long flags;
+
+		sdstd_os_lock_irqsave(sd, &flags);
+		old_int = sdstd_rreg16(sd, SD_IntrStatusEnable);
+		sdstd_wreg16(sd, SD_IntrStatusEnable, SFIELD(old_int, INTSTAT_CARD_INT, 0));
+		sdstd_os_unlock_irqrestore(sd, &flags);
+
+		if (sd->client_intr_enabled && sd->use_client_ints) {
+			sd->intrcount++;
+			ASSERT(sd->intr_handler);
+			ASSERT(sd->intr_handler_arg);
+			(sd->intr_handler)(sd->intr_handler_arg);
+		} else {
+			sd_err(("%s: Not ready for intr: enabled %d, handler %p\n",
+			        __FUNCTION__, sd->client_intr_enabled, sd->intr_handler));
+		}
+		sdstd_os_lock_irqsave(sd, &flags);
+		old_int = sdstd_rreg16(sd, SD_IntrStatusEnable);
+		sdstd_wreg16(sd, SD_IntrStatusEnable, SFIELD(old_int, INTSTAT_CARD_INT, 1));
+		sdstd_os_unlock_irqrestore(sd, &flags);
+	} else {
+		/* Local interrupt: disable, set flag, and save intrstatus */
+		sdstd_wreg16(sd, SD_IntrSignalEnable, 0);
+		sdstd_wreg16(sd, SD_ErrorIntrSignalEnable, 0);
+		sd->local_intrcount++;
+		sd->got_hcint = TRUE;
+		sd->last_intrstatus = cur_int;
+	}
+
+	return TRUE;
+}
+
+void
+sdstd_spinbits(sdioh_info_t *sd, uint16 norm, uint16 err)
+{
+	uint16 int_reg, err_reg;
+	int retries = RETRIES_LARGE;
+
+	do {
+		int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+		err_reg = sdstd_rreg16(sd, SD_ErrorIntrStatus);
+	} while (--retries && !(int_reg & norm) && !(err_reg & err));
+
+	norm |= sd->intmask;
+	if (err_reg & err)
+		norm = SFIELD(norm, INTSTAT_ERROR_INT, 1);
+	sd->last_intrstatus = int_reg & norm;
+}
+
+/* write a client register */
+static int
+sdstd_card_regwrite(sdioh_info_t *sd, int func, uint32 regaddr, int regsize, uint32 data)
+{
+	int status;
+	uint32 cmd_arg, rsp5, flags;
+
+	cmd_arg = 0;
+
+	if ((func == 0) || (regsize == 1)) {
+		cmd_arg = SFIELD(cmd_arg, CMD52_FUNCTION, func);
+		cmd_arg = SFIELD(cmd_arg, CMD52_REG_ADDR, regaddr);
+		cmd_arg = SFIELD(cmd_arg, CMD52_RW_FLAG, SDIOH_XFER_TYPE_WRITE);
+		cmd_arg = SFIELD(cmd_arg, CMD52_RAW, 0);
+		cmd_arg = SFIELD(cmd_arg, CMD52_DATA, data & 0xff);
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_52, cmd_arg))
+		    != SUCCESS)
+			return status;
+
+		sdstd_cmd_getrsp(sd, &rsp5, 1);
+		flags = GFIELD(rsp5, RSP5_FLAGS);
+		if (flags && (flags != 0x10))
+			sd_err(("%s: rsp5.rsp5.flags = 0x%x, expecting 0x10\n",
+			        __FUNCTION__,  flags));
+	}
+	else {
+		cmd_arg = SFIELD(cmd_arg, CMD53_BYTE_BLK_CNT, regsize);
+		/* XXX SDIO spec v 1.10, Sec 5.3  Not FIFO */
+		cmd_arg = SFIELD(cmd_arg, CMD53_OP_CODE, 1);
+		cmd_arg = SFIELD(cmd_arg, CMD53_BLK_MODE, 0);
+		cmd_arg = SFIELD(cmd_arg, CMD53_FUNCTION, func);
+		cmd_arg = SFIELD(cmd_arg, CMD53_REG_ADDR, regaddr);
+		cmd_arg = SFIELD(cmd_arg, CMD53_RW_FLAG, SDIOH_XFER_TYPE_WRITE);
+
+		sd->data_xfer_count = regsize;
+
+		/* sdstd_cmd_issue() returns with the command complete bit
+		 * in the ISR already cleared
+		 */
+		if ((status = sdstd_cmd_issue(sd, USE_DMA(sd), SDIOH_CMD_53, cmd_arg))
+		    != SUCCESS)
+			return status;
+
+		sdstd_cmd_getrsp(sd, &rsp5, 1);
+
+		if (GFIELD(rsp5, RSP5_FLAGS) != 0x10)
+			sd_err(("%s: rsp5 flags = 0x%x, expecting 0x10\n",
+			        __FUNCTION__,  GFIELD(rsp5, RSP5_FLAGS)));
+		if (GFIELD(rsp5, RSP5_STUFF))
+			sd_err(("%s: rsp5 stuff is 0x%x: expecting 0\n",
+			        __FUNCTION__, GFIELD(rsp5, RSP5_STUFF)));
+
+		if (sd->polled_mode) {
+			uint16 int_reg;
+			int retries = RETRIES_LARGE;
+
+			/* Wait for Write Buffer to become ready */
+			do {
+				int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+			} while (--retries && (GFIELD(int_reg, INTSTAT_BUF_WRITE_READY) == 0));
+
+			if (!retries) {
+				sd_err(("%s: Timeout on Buf_Write_Ready: intStat: 0x%x "
+				        "errint: 0x%x PresentState 0x%x\n",
+				        __FUNCTION__, int_reg,
+				        sdstd_rreg16(sd, SD_ErrorIntrStatus),
+				        sdstd_rreg(sd, SD_PresentState)));
+				sdstd_check_errs(sd, SDIOH_CMD_53, cmd_arg);
+				return (ERROR);
+			}
+			/* Clear Write Buf Ready bit */
+			int_reg = 0;
+			int_reg = SFIELD(int_reg, INTSTAT_BUF_WRITE_READY, 1);
+			sdstd_wreg16(sd, SD_IntrStatus, int_reg);
+
+			/* At this point we have Buffer Ready, so write the data */
+			if (regsize == 2)
+				sdstd_wreg16(sd, SD_BufferDataPort0, (uint16) data);
+			else
+				sdstd_wreg(sd, SD_BufferDataPort0, data);
+
+			/* Wait for Transfer Complete */
+			retries = RETRIES_LARGE;
+			do {
+				int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+			} while (--retries && (GFIELD(int_reg, INTSTAT_XFER_COMPLETE) == 0));
+
+			/* Check for any errors from the data phase */
+			if (sdstd_check_errs(sd, SDIOH_CMD_53, cmd_arg))
+				return ERROR;
+
+			if (retries == 0) {
+				sd_err(("%s: Timeout for xfer complete; State = 0x%x, "
+				        "intr state=0x%x, Errintstatus 0x%x rcnt %d, tcnt %d\n",
+				        __FUNCTION__, sdstd_rreg(sd, SD_PresentState),
+				        int_reg, sdstd_rreg16(sd, SD_ErrorIntrStatus),
+				        sd->r_cnt, sd->t_cnt));
+			}
+			/* Clear the status bits */
+			sdstd_wreg16(sd, SD_IntrStatus, SFIELD(int_reg, INTSTAT_CARD_INT, 0));
+		}
+	}
+	return SUCCESS;
+}
+
+void
+sdstd_cmd_getrsp(sdioh_info_t *sd, uint32 *rsp_buffer, int count /* num 32 bit words */)
+{
+	int rsp_count;
+	int respaddr = SD_Response0;
+
+	if (count > 4)
+		count = 4;
+
+	for (rsp_count = 0; rsp_count < count; rsp_count++) {
+		*rsp_buffer++ = sdstd_rreg(sd, respaddr);
+		respaddr += 4;
+	}
+}
+
+/*
+	Note: options: 0 - default
+				 1 - tuning option: Means that, this cmd issue is as a part
+					of tuning. So no need to check the start tuning function.
+*/
+static int
+sdstd_cmd_issue(sdioh_info_t *sdioh_info, bool use_dma, uint32 cmd, uint32 arg)
+{
+	uint16 cmd_reg;
+	int retries;
+	uint32 cmd_arg;
+	uint16 xfer_reg = 0;
+
+#ifdef BCMDBG
+	if (sdstd_rreg16(sdioh_info, SD_ErrorIntrStatus) != 0) {
+		sd_err(("%s: Entering: ErrorIntrStatus 0x%x, Expecting 0\n",
+		        __FUNCTION__, sdstd_rreg16(sdioh_info, SD_ErrorIntrStatus)));
+	}
+#endif
+
+	if ((sdioh_info->sd_mode == SDIOH_MODE_SPI) &&
+	    ((cmd == SDIOH_CMD_3) || (cmd == SDIOH_CMD_7) || (cmd == SDIOH_CMD_15))) {
+		sd_err(("%s: Cmd %d is not for SPI\n", __FUNCTION__, cmd));
+		return ERROR;
+	}
+
+	retries = RETRIES_SMALL;
+	while ((GFIELD(sdstd_rreg(sdioh_info, SD_PresentState), PRES_CMD_INHIBIT)) && --retries) {
+		if (retries == RETRIES_SMALL)
+			sd_err(("%s: Waiting for Command Inhibit cmd = %d 0x%x\n",
+			        __FUNCTION__, cmd, sdstd_rreg(sdioh_info, SD_PresentState)));
+	}
+	if (!retries) {
+		sd_err(("%s: Command Inhibit timeout\n", __FUNCTION__));
+		if (trap_errs)
+			ASSERT(0);
+		return ERROR;
+	}
+
+	cmd_reg = 0;
+	switch (cmd) {
+	case SDIOH_CMD_0:       /* Set Card to Idle State - No Response */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_NONE);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_3:	/* Ask card to send RCA - Response R6 */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_5:	/* Send Operation condition - Response R4 */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_7:	/* Select card - Response R1 */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_14:	/* eSD Sleep - Response R1 */
+	case SDIOH_CMD_11:	/* Select card - Response R1 */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_15:	/* Set card to inactive state - Response None */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_NONE);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_19:	/* clock tuning - Response R1 */
+		sd_data(("%s: CMD%d\n", __FUNCTION__, cmd));
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		/* Host controller reads 64 byte magic pattern from card
+		*  Hence Direction = 1 ( READ )
+		*/
+		xfer_reg = SFIELD(xfer_reg, XFER_DATA_DIRECTION, 1);
+		break;
+
+	case SDIOH_CMD_52:	/* IO R/W Direct (single byte) - Response R5 */
+
+		sd_data(("%s: CMD52 func(%d) addr(0x%x) %s data(0x%x)\n",
+			__FUNCTION__,
+			GFIELD(arg, CMD52_FUNCTION),
+			GFIELD(arg, CMD52_REG_ADDR),
+			GFIELD(arg, CMD52_RW_FLAG) ? "W" : "R",
+			GFIELD(arg, CMD52_DATA)));
+
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+		break;
+
+	case SDIOH_CMD_53:	/* IO R/W Extended (multiple bytes/blocks) */
+
+		sd_data(("%s: CMD53 func(%d) addr(0x%x) %s mode(%s) cnt(%d), %s\n",
+			__FUNCTION__,
+			GFIELD(arg, CMD53_FUNCTION),
+			GFIELD(arg, CMD53_REG_ADDR),
+			GFIELD(arg, CMD53_RW_FLAG) ? "W" : "R",
+			GFIELD(arg, CMD53_BLK_MODE) ? "Block" : "Byte",
+			GFIELD(arg, CMD53_BYTE_BLK_CNT),
+			GFIELD(arg, CMD53_OP_CODE) ? "Incrementing addr" : "Single addr"));
+
+		cmd_arg = arg;
+		xfer_reg = 0;
+
+		cmd_reg = SFIELD(cmd_reg, CMD_RESP_TYPE, RESP_TYPE_48);
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_DATA_EN, 1);
+		cmd_reg = SFIELD(cmd_reg, CMD_TYPE, CMD_TYPE_NORMAL);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX, cmd);
+
+		use_dma = USE_DMA(sdioh_info) && GFIELD(cmd_arg, CMD53_BLK_MODE);
+
+		if (GFIELD(cmd_arg, CMD53_BLK_MODE)) {
+			uint16 blocksize;
+			uint16 blockcount;
+			int func;
+
+			ASSERT(sdioh_info->sd_blockmode);
+
+			func = GFIELD(cmd_arg, CMD53_FUNCTION);
+			blocksize = MIN((int)sdioh_info->data_xfer_count,
+			                sdioh_info->client_block_size[func]);
+			blockcount = GFIELD(cmd_arg, CMD53_BYTE_BLK_CNT);
+
+			/* data_xfer_cnt is already setup so that for multiblock mode,
+			 * it is the entire buffer length.  For non-block or single block,
+			 * it is < 64 bytes
+			 */
+			if (use_dma) {
+				switch (sdioh_info->sd_dma_mode) {
+				case DMA_MODE_SDMA:
+					sd_dma(("%s: SDMA: SysAddr reg was 0x%x now 0x%x\n",
+					      __FUNCTION__, sdstd_rreg(sdioh_info, SD_SysAddr),
+					     (uint32)sdioh_info->dma_phys));
+				sdstd_wreg(sdioh_info, SD_SysAddr, sdioh_info->dma_phys);
+					break;
+				case DMA_MODE_ADMA1:
+				case DMA_MODE_ADMA2:
+					sd_dma(("%s: ADMA: Using ADMA\n", __FUNCTION__));
+#ifdef BCMSDIOH_TXGLOM
+					/* multi-descriptor is currently used only for hc3 */
+					if ((sdioh_info->glom_info.count != 0) &&
+						(sdioh_info->txglom_mode == SDPCM_TXGLOM_MDESC)) {
+						uint32 i = 0;
+						for (i = 0;
+							i < sdioh_info->glom_info.count-1;
+							i++) {
+							glom_buf_t *glom_info;
+							glom_info = &(sdioh_info->glom_info);
+							sd_create_adma_descriptor(sdioh_info,
+								i,
+								glom_info->dma_phys_arr[i],
+								glom_info->nbytes[i],
+								ADMA2_ATTRIBUTE_VALID |
+								ADMA2_ATTRIBUTE_ACT_TRAN);
+						}
+
+						sd_create_adma_descriptor(sdioh_info,
+							i,
+							sdioh_info->glom_info.dma_phys_arr[i],
+							sdioh_info->glom_info.nbytes[i],
+							ADMA2_ATTRIBUTE_VALID |
+							ADMA2_ATTRIBUTE_END |
+							ADMA2_ATTRIBUTE_INT |
+							ADMA2_ATTRIBUTE_ACT_TRAN);
+					} else
+#endif /* BCMSDIOH_TXGLOM */
+					{
+						sd_create_adma_descriptor(sdioh_info, 0,
+						sdioh_info->dma_phys, blockcount*blocksize,
+						ADMA2_ATTRIBUTE_VALID | ADMA2_ATTRIBUTE_END |
+						ADMA2_ATTRIBUTE_INT | ADMA2_ATTRIBUTE_ACT_TRAN);
+					}
+					/* Dump descriptor if DMA debugging is enabled. */
+					if (sd_msglevel & SDH_DMA_VAL) {
+						sd_dump_adma_dscr(sdioh_info);
+					}
+
+					sdstd_wreg(sdioh_info, SD_ADMA_SysAddr,
+					           sdioh_info->adma2_dscr_phys);
+					break;
+				default:
+					sd_err(("%s: unsupported DMA mode %d.\n",
+						__FUNCTION__, sdioh_info->sd_dma_mode));
+					break;
+				}
+			}
+
+			sd_trace(("%s: Setting block count %d, block size %d bytes\n",
+			          __FUNCTION__, blockcount, blocksize));
+			sdstd_wreg16(sdioh_info, SD_BlockSize, blocksize);
+			sdstd_wreg16(sdioh_info, SD_BlockCount, blockcount);
+
+			xfer_reg = SFIELD(xfer_reg, XFER_DMA_ENABLE, use_dma);
+
+			if (sdioh_info->client_block_size[func] != blocksize)
+				set_client_block_size(sdioh_info, 1, blocksize);
+
+			if (blockcount > 1) {
+				xfer_reg = SFIELD(xfer_reg, XFER_MULTI_BLOCK, 1);
+				xfer_reg = SFIELD(xfer_reg, XFER_BLK_COUNT_EN, 1);
+				xfer_reg = SFIELD(xfer_reg, XFER_CMD_12_EN, 0);
+			} else {
+				xfer_reg = SFIELD(xfer_reg, XFER_MULTI_BLOCK, 0);
+				xfer_reg = SFIELD(xfer_reg, XFER_BLK_COUNT_EN, 0);
+				xfer_reg = SFIELD(xfer_reg, XFER_CMD_12_EN, 0);
+			}
+
+			if (GFIELD(cmd_arg, CMD53_RW_FLAG) == SDIOH_XFER_TYPE_READ)
+				xfer_reg = SFIELD(xfer_reg, XFER_DATA_DIRECTION, 1);
+			else
+				xfer_reg = SFIELD(xfer_reg, XFER_DATA_DIRECTION, 0);
+
+			retries = RETRIES_SMALL;
+			while (GFIELD(sdstd_rreg(sdioh_info, SD_PresentState),
+			              PRES_DAT_INHIBIT) && --retries)
+				sd_err(("%s: Waiting for Data Inhibit cmd = %d\n",
+				        __FUNCTION__, cmd));
+			if (!retries) {
+				sd_err(("%s: Data Inhibit timeout\n", __FUNCTION__));
+				if (trap_errs)
+					ASSERT(0);
+				return ERROR;
+			}
+
+			/* Consider deferring this write to the comment below "Deferred Write" */
+			sdstd_wreg16(sdioh_info, SD_TransferMode, xfer_reg);
+
+		} else {	/* Non block mode */
+			uint16 bytes = GFIELD(cmd_arg, CMD53_BYTE_BLK_CNT);
+			/* The byte/block count field only has 9 bits,
+			 * so, to do a 512-byte bytemode transfer, this
+			 * field will contain 0, but we need to tell the
+			 * controller we're transferring 512 bytes.
+			 */
+			if (bytes == 0) bytes = 512;
+
+			if (use_dma)
+				sdstd_wreg(sdioh_info, SD_SysAddr, sdioh_info->dma_phys);
+
+			/* PCI: Transfer Mode register 0x0c */
+			xfer_reg = SFIELD(xfer_reg, XFER_DMA_ENABLE, bytes <= 4 ? 0 : use_dma);
+			xfer_reg = SFIELD(xfer_reg, XFER_CMD_12_EN, 0);
+			if (GFIELD(cmd_arg, CMD53_RW_FLAG) == SDIOH_XFER_TYPE_READ)
+				xfer_reg = SFIELD(xfer_reg, XFER_DATA_DIRECTION, 1);
+			else
+				xfer_reg = SFIELD(xfer_reg, XFER_DATA_DIRECTION, 0);
+			/* See table 2-8 Host Controller spec ver 1.00 */
+			xfer_reg = SFIELD(xfer_reg, XFER_BLK_COUNT_EN, 0); /* Dont care */
+			xfer_reg = SFIELD(xfer_reg, XFER_MULTI_BLOCK, 0);
+
+			sdstd_wreg16(sdioh_info, SD_BlockSize,  bytes);
+
+			/* XXX This should be a don't care but Arasan needs it
+			 * to be one.  Its fixed in later versions (but they
+			 * don't have version numbers, sigh).
+			 */
+			sdstd_wreg16(sdioh_info, SD_BlockCount, 1);
+
+			retries = RETRIES_SMALL;
+			while (GFIELD(sdstd_rreg(sdioh_info, SD_PresentState),
+			              PRES_DAT_INHIBIT) && --retries)
+				sd_err(("%s: Waiting for Data Inhibit cmd = %d\n",
+				        __FUNCTION__, cmd));
+			if (!retries) {
+				sd_err(("%s: Data Inhibit timeout\n", __FUNCTION__));
+				if (trap_errs)
+					ASSERT(0);
+				return ERROR;
+			}
+
+			/* Consider deferring this write to the comment below "Deferred Write" */
+			sdstd_wreg16(sdioh_info, SD_TransferMode, xfer_reg);
+		}
+		break;
+
+	default:
+		sd_err(("%s: Unknown command\n", __FUNCTION__));
+		return ERROR;
+	}
+
+	if (sdioh_info->sd_mode == SDIOH_MODE_SPI) {
+		cmd_reg = SFIELD(cmd_reg, CMD_CRC_EN, 0);
+		cmd_reg = SFIELD(cmd_reg, CMD_INDEX_EN, 0);
+	}
+
+	/* Setup and issue the SDIO command */
+	sdstd_wreg(sdioh_info, SD_Arg0, arg);
+
+	/* Deferred Write
+	 * Consider deferring the two writes above until this point in the code.
+	 * The following would do one 32 bit write.
+	 *
+	 * {
+	 *	uint32 tmp32 = cmd_reg << 16;
+	 *	tmp32 |= xfer_reg;
+	 *	sdstd_wreg(sdioh_info, SD_TransferMode, tmp32);
+	 * }
+	 */
+
+	/* Alternate to Deferred Write START */
+
+	/* In response to CMD19 card sends 64 byte magic pattern.
+	 * So SD_BlockSize = 64 & SD_BlockCount = 1
+	*/
+	if (GFIELD(cmd_reg, CMD_INDEX) == SDIOH_CMD_19) {
+		sdstd_wreg16(sdioh_info, SD_TransferMode, xfer_reg);
+		sdstd_wreg16(sdioh_info, SD_BlockSize, 64);
+		sdstd_wreg16(sdioh_info, SD_BlockCount, 1);
+	}
+	sdstd_wreg16(sdioh_info, SD_Command, cmd_reg);
+
+	/* Alternate to Deferred Write END */
+
+	/* If we are in polled mode, wait for the command to complete.
+	 * In interrupt mode, return immediately. The calling function will
+	 * know that the command has completed when the CMDATDONE interrupt
+	 * is asserted
+	 */
+	if (sdioh_info->polled_mode) {
+		uint16 int_reg = 0;
+		retries = RETRIES_LARGE;
+
+		/* For CMD19 no need to wait for cmd completion */
+		if (GFIELD(cmd_reg, CMD_INDEX) == SDIOH_CMD_19)
+			return SUCCESS;
+
+		do {
+			int_reg = sdstd_rreg16(sdioh_info, SD_IntrStatus);
+			sdstd_os_yield(sdioh_info);
+		} while (--retries &&
+		         (GFIELD(int_reg, INTSTAT_ERROR_INT) == 0) &&
+		         (GFIELD(int_reg, INTSTAT_CMD_COMPLETE) == 0));
+
+		if (!retries) {
+			sd_err(("%s: CMD_COMPLETE timeout: intrStatus: 0x%x "
+			        "error stat 0x%x state 0x%x\n",
+			        __FUNCTION__, int_reg,
+			        sdstd_rreg16(sdioh_info, SD_ErrorIntrStatus),
+			        sdstd_rreg(sdioh_info, SD_PresentState)));
+
+			/* Attempt to reset CMD line when we get a CMD timeout */
+			sdstd_wreg8(sdioh_info, SD_SoftwareReset, SFIELD(0, SW_RESET_CMD, 1));
+			retries = RETRIES_LARGE;
+			do {
+				sd_trace(("%s: waiting for CMD line reset\n", __FUNCTION__));
+			} while ((GFIELD(sdstd_rreg8(sdioh_info, SD_SoftwareReset),
+			                 SW_RESET_CMD)) && retries--);
+
+			if (!retries) {
+				sd_err(("%s: Timeout waiting for CMD line reset\n", __FUNCTION__));
+			}
+
+			if (trap_errs)
+				ASSERT(0);
+			return (ERROR);
+		}
+
+		/* Clear Command Complete interrupt */
+		int_reg = SFIELD(0, INTSTAT_CMD_COMPLETE, 1);
+		sdstd_wreg16(sdioh_info, SD_IntrStatus, int_reg);
+
+		/* Check for Errors */
+		if (sdstd_check_errs(sdioh_info, cmd, arg)) {
+			if (trap_errs)
+				ASSERT(0);
+			return ERROR;
+		}
+	}
+	return SUCCESS;
+}
+
+/*
+ * XXX On entry: If single block or non-block, buffersize <= blocksize.
+ * If Mulitblock, buffersize is unlimited.
+ * Question is how to handle the leftovers in either single or multiblock.
+ * I think the caller should break the buffer up so this routine will always
+ * use blocksize == buffersize to handle the end piece of the buffer
+ */
+
+static int
+sdstd_card_buf(sdioh_info_t *sd, int rw, int func, bool fifo, uint32 addr, int nbytes, uint32 *data)
+{
+	int retval = SUCCESS;
+	int status;
+	uint32 cmd_arg;
+	uint32 rsp5;
+	uint16 int_reg, int_bit;
+	uint flags;
+	int num_blocks, blocksize;
+	bool local_blockmode, local_dma;
+	bool read = rw == SDIOH_READ ? 1 : 0;
+	bool local_yield = FALSE;
+#ifdef BCMSDIOH_TXGLOM
+	uint32 i;
+	uint8 *localbuf = NULL;
+#endif
+
+	ASSERT(nbytes);
+
+	cmd_arg = 0;
+
+	sd_data(("%s: %s 53 addr 0x%x, len %d bytes, r_cnt %d t_cnt %d\n",
+	         __FUNCTION__, read ? "Rd" : "Wr", addr, nbytes, sd->r_cnt, sd->t_cnt));
+
+	if (read) sd->r_cnt++; else sd->t_cnt++;
+
+	local_blockmode = sd->sd_blockmode;
+	local_dma = USE_DMA(sd);
+
+#ifdef BCMSDIOH_TXGLOM
+	/* If multiple buffers are there, then calculate the nbytes from that */
+	if (!read && (func == SDIO_FUNC_2) && (sd->glom_info.count != 0)) {
+		uint32 ii;
+		nbytes = 0;
+		for (ii = 0; ii < sd->glom_info.count; ii++) {
+			nbytes += sd->glom_info.nbytes[ii];
+		}
+		ASSERT(nbytes <= sd->alloced_dma_size);
+	}
+#endif
+
+	/* Don't bother with block mode on small xfers */
+	if (nbytes < sd->client_block_size[func]) {
+		sd_data(("setting local blockmode to false: nbytes (%d) != block_size (%d)\n",
+		         nbytes, sd->client_block_size[func]));
+		local_blockmode = FALSE;
+		local_dma = FALSE;
+#ifdef BCMSDIOH_TXGLOM
+		/* In glommed case, create a single pkt from multiple pkts */
+		if (!read && (func == SDIO_FUNC_2) && (sd->glom_info.count != 0)) {
+			uint32 offset = 0;
+			localbuf = (uint8 *)MALLOC(sd->osh, nbytes);
+			data = (uint32 *)localbuf;
+			for (i = 0; i < sd->glom_info.count; i++) {
+				bcopy(sd->glom_info.dma_buf_arr[i],
+					((uint8 *)data + offset),
+					sd->glom_info.nbytes[i]);
+				offset += sd->glom_info.nbytes[i];
+			}
+		}
+#endif
+	}
+
+	if (local_blockmode) {
+		blocksize = MIN(sd->client_block_size[func], nbytes);
+		num_blocks = nbytes/blocksize;
+		cmd_arg = SFIELD(cmd_arg, CMD53_BYTE_BLK_CNT, num_blocks);
+		cmd_arg = SFIELD(cmd_arg, CMD53_BLK_MODE, 1);
+	} else {
+		num_blocks =  1;
+		blocksize = nbytes;
+		cmd_arg = SFIELD(cmd_arg, CMD53_BYTE_BLK_CNT, nbytes);
+		cmd_arg = SFIELD(cmd_arg, CMD53_BLK_MODE, 0);
+	}
+
+	if (local_dma && !read) {
+#ifdef BCMSDIOH_TXGLOM
+		if ((func == SDIO_FUNC_2) && (sd->glom_info.count != 0)) {
+			/* In case of hc ver 2 DMA_MAP may not work properly due to 4K alignment
+			 * requirements. So copying pkt to 4K aligned pre-allocated pkt.
+			 * Total length should not cross the pre-alloced memory size
+			 */
+			if (sd->txglom_mode == SDPCM_TXGLOM_CPY) {
+				uint32 total_bytes = 0;
+				for (i = 0; i < sd->glom_info.count; i++) {
+					bcopy(sd->glom_info.dma_buf_arr[i],
+						(uint8 *)sd->dma_buf + total_bytes,
+						sd->glom_info.nbytes[i]);
+					total_bytes += sd->glom_info.nbytes[i];
+				}
+				sd_sync_dma(sd, read, total_bytes);
+			}
+		} else
+#endif /* BCMSDIOH_TXGLOM */
+		{
+			bcopy(data, sd->dma_buf, nbytes);
+			sd_sync_dma(sd, read, nbytes);
+		}
+	}
+
+	if (fifo)
+		cmd_arg = SFIELD(cmd_arg, CMD53_OP_CODE, 0);	/* XXX SDIO spec v 1.10, Sec 5.3  */
+	else
+		cmd_arg = SFIELD(cmd_arg, CMD53_OP_CODE, 1);	/* XXX SDIO spec v 1.10, Sec 5.3  */
+
+	cmd_arg = SFIELD(cmd_arg, CMD53_FUNCTION, func);
+	cmd_arg = SFIELD(cmd_arg, CMD53_REG_ADDR, addr);
+	if (read)
+		cmd_arg = SFIELD(cmd_arg, CMD53_RW_FLAG, SDIOH_XFER_TYPE_READ);
+	else
+		cmd_arg = SFIELD(cmd_arg, CMD53_RW_FLAG, SDIOH_XFER_TYPE_WRITE);
+
+	sd->data_xfer_count = nbytes;
+
+	/* sdstd_cmd_issue() returns with the command complete bit
+	 * in the ISR already cleared
+	 */
+	if ((status = sdstd_cmd_issue(sd, local_dma, SDIOH_CMD_53, cmd_arg)) != SUCCESS) {
+		sd_err(("%s: cmd_issue failed for %s\n", __FUNCTION__, (read ? "read" : "write")));
+		retval = status;
+		goto done;
+	}
+
+	sdstd_cmd_getrsp(sd, &rsp5, 1);
+
+	if ((flags = GFIELD(rsp5, RSP5_FLAGS)) != 0x10) {
+		sd_err(("%s: Rsp5: nbytes %d, dma %d blockmode %d, read %d "
+		        "numblocks %d, blocksize %d\n",
+		        __FUNCTION__, nbytes, local_dma, local_dma, read, num_blocks, blocksize));
+
+		if (flags & 1)
+			sd_err(("%s: rsp5: Command not accepted: arg out of range 0x%x, "
+			        "bytes %d dma %d\n",
+			        __FUNCTION__, flags, GFIELD(cmd_arg, CMD53_BYTE_BLK_CNT),
+			        GFIELD(cmd_arg, CMD53_BLK_MODE)));
+		if (flags & 0x8)
+			sd_err(("%s: Rsp5: General Error\n", __FUNCTION__));
+
+		sd_err(("%s: rsp5 flags = 0x%x, expecting 0x10 returning error\n",
+		        __FUNCTION__,  flags));
+		if (trap_errs)
+			ASSERT(0);
+		retval = ERROR;
+		goto done;
+	}
+
+	if (GFIELD(rsp5, RSP5_STUFF))
+		sd_err(("%s: rsp5 stuff is 0x%x: expecting 0\n",
+		        __FUNCTION__, GFIELD(rsp5, RSP5_STUFF)));
+
+#ifdef BCMSDYIELD
+	local_yield = sd_yieldcpu && ((uint)nbytes >= sd_minyield);
+#endif
+
+	if (!local_dma) {
+		int bytes, ii;
+		uint32 tmp;
+
+		for (ii = 0; ii < num_blocks; ii++) {
+			int words;
+
+			/* Decide which status bit we're waiting for */
+			if (read)
+				int_bit = SFIELD(0, INTSTAT_BUF_READ_READY, 1);
+			else
+				int_bit = SFIELD(0, INTSTAT_BUF_WRITE_READY, 1);
+
+			/* If not on, wait for it (or for xfer error) */
+			int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+			if (!(int_reg & int_bit)) {
+				status = sdstd_waitbits(sd, int_bit, ERRINT_TRANSFER_ERRS,
+				                        local_yield, &int_reg);
+				switch (status) {
+				case -1:
+					sd_err(("%s: pio interrupted\n", __FUNCTION__));
+					retval = ERROR;
+					goto done;
+				case -2:
+					sd_err(("%s: pio timeout waiting for interrupt\n",
+					        __FUNCTION__));
+					retval = ERROR;
+					goto done;
+				}
+			}
+#ifdef BCMSLTGT
+		/*	int_reg = sdstd_rreg16(sd, SD_IntrStatus); */
+#endif
+			/* Confirm we got the bit w/o error */
+			if (!(int_reg & int_bit) || GFIELD(int_reg, INTSTAT_ERROR_INT)) {
+				sd_err(("%s: Error or timeout for Buf_%s_Ready: intStat: 0x%x "
+				        "errint: 0x%x PresentState 0x%x\n",
+				        __FUNCTION__, read ? "Read" : "Write", int_reg,
+				        sdstd_rreg16(sd, SD_ErrorIntrStatus),
+				        sdstd_rreg(sd, SD_PresentState)));
+				sdstd_dumpregs(sd);
+				sdstd_check_errs(sd, SDIOH_CMD_53, cmd_arg);
+				retval = ERROR;
+				goto done;
+			}
+
+			/* Clear Buf Ready bit */
+			sdstd_wreg16(sd, SD_IntrStatus, int_bit);
+
+			/* At this point we have Buffer Ready, write the data 4 bytes at a time */
+			for (words = blocksize/4; words; words--) {
+				if (read)
+					*data = sdstd_rreg(sd, SD_BufferDataPort0);
+				else
+					sdstd_wreg(sd, SD_BufferDataPort0, *data);
+				data++;
+			}
+
+			/* XXX
+			 * Handle < 4 bytes.  wlc_pio.c currently (as of 12/20/05) truncates buflen
+			 * to be evenly divisible by 4.  However dongle passes arbitrary lengths,
+			 * so handle it here
+			 */
+			bytes = blocksize % 4;
+
+			/* If no leftover bytes, go to next block */
+			if (!bytes)
+				continue;
+
+			switch (bytes) {
+			case 1:
+				/* R/W 8 bits */
+				if (read)
+					*(data++) = (uint32)(sdstd_rreg8(sd, SD_BufferDataPort0));
+				else
+					sdstd_wreg8(sd, SD_BufferDataPort0,
+					            (uint8)(*(data++) & 0xff));
+				break;
+			case 2:
+				/* R/W 16 bits */
+				if (read)
+					*(data++) = (uint32)sdstd_rreg16(sd, SD_BufferDataPort0);
+				else
+					sdstd_wreg16(sd, SD_BufferDataPort0, (uint16)(*(data++)));
+				break;
+			case 3:
+				/* R/W 24 bits:
+				 * SD_BufferDataPort0[0-15] | SD_BufferDataPort1[16-23]
+				 */
+				if (read) {
+					tmp = (uint32)sdstd_rreg16(sd, SD_BufferDataPort0);
+					tmp |= ((uint32)(sdstd_rreg8(sd,
+					                             SD_BufferDataPort1)) << 16);
+					*(data++) = tmp;
+				} else {
+					tmp = *(data++);
+					sdstd_wreg16(sd, SD_BufferDataPort0, (uint16)tmp & 0xffff);
+					sdstd_wreg8(sd, SD_BufferDataPort1,
+					            (uint8)((tmp >> 16) & 0xff));
+				}
+				break;
+			default:
+				sd_err(("%s: Unexpected bytes leftover %d\n",
+				        __FUNCTION__, bytes));
+				ASSERT(0);
+				break;
+			}
+		}
+	}	/* End PIO processing */
+
+	/* Wait for Transfer Complete or Transfer Error */
+	int_bit = SFIELD(0, INTSTAT_XFER_COMPLETE, 1);
+
+	/* If not on, wait for it (or for xfer error) */
+	int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+	if (!(int_reg & int_bit)) {
+		status = sdstd_waitbits(sd, int_bit, ERRINT_TRANSFER_ERRS, local_yield, &int_reg);
+		switch (status) {
+		case -1:
+			sd_err(("%s: interrupted\n", __FUNCTION__));
+			retval = ERROR;
+			goto done;
+		case -2:
+			sd_err(("%s: timeout waiting for interrupt\n", __FUNCTION__));
+			retval = ERROR;
+			goto done;
+		}
+	}
+
+	/* Check for any errors from the data phase */
+	if (sdstd_check_errs(sd, SDIOH_CMD_53, cmd_arg)) {
+		retval = ERROR;
+		goto done;
+	}
+
+	/* May have gotten a software timeout if not blocking? */
+	int_reg = sdstd_rreg16(sd, SD_IntrStatus);
+	if (!(int_reg & int_bit)) {
+		sd_err(("%s: Error or Timeout for xfer complete; %s, dma %d, State 0x%08x, "
+		        "intr 0x%04x, Err 0x%04x, len = %d, rcnt %d, tcnt %d\n",
+		        __FUNCTION__, read ? "R" : "W", local_dma,
+		        sdstd_rreg(sd, SD_PresentState), int_reg,
+		        sdstd_rreg16(sd, SD_ErrorIntrStatus), nbytes,
+		        sd->r_cnt, sd->t_cnt));
+		sdstd_dumpregs(sd);
+		retval = ERROR;
+		goto done;
+	}
+
+	/* Clear the status bits */
+	int_reg = int_bit;
+	if (local_dma) {
+		/* DMA Complete */
+		/* XXX Step 14, Section 3.6.2.2 Stnd Cntrlr Spec */
+		/* Reads in particular don't have DMA_COMPLETE set */
+		int_reg = SFIELD(int_reg, INTSTAT_DMA_INT, 1);
+	}
+	sdstd_wreg16(sd, SD_IntrStatus, int_reg);
+
+	/* Fetch data */
+	if (local_dma && read) {
+		sd_sync_dma(sd, read, nbytes);
+		bcopy(sd->dma_buf, data, nbytes);
+	}
+
+done:
+#ifdef BCMSDIOH_TXGLOM
+	if (localbuf)
+		MFREE(sd->osh, localbuf, nbytes);
+#endif
+	return retval;
+}
+
+static int
+set_client_block_size(sdioh_info_t *sd, int func, int block_size)
+{
+	int base;
+	int err = 0;
+
+	sd_err(("%s: Setting block size %d, func %d\n", __FUNCTION__, block_size, func));
+	sd->client_block_size[func] = block_size;
+
+	/* Set the block size in the SDIO Card register */
+	base = func * SDIOD_FBR_SIZE;
+	err = sdstd_card_regwrite(sd, 0, base+SDIOD_CCCR_BLKSIZE_0, 1, block_size & 0xff);
+	if (!err) {
+		err = sdstd_card_regwrite(sd, 0, base+SDIOD_CCCR_BLKSIZE_1, 1,
+		                          (block_size >> 8) & 0xff);
+	}
+
+	/* Do not set the block size in the SDIO Host register, that
+	 * is func dependent and will get done on an individual
+	 * transaction basis
+	 */
+
+	return (err ? BCME_SDIO_ERROR : 0);
+}
+
+/* Reset and re-initialize the device */
+int
+sdioh_sdio_reset(sdioh_info_t *si)
+{
+	uint8 hreg;
+
+	/* Reset the attached device (use slower clock for safety) */
+	if (!sdstd_start_clock(si, 128)) {
+			sd_err(("set clock failed!\n"));
+			return ERROR;
+			}
+	sdstd_reset(si, 0, 1);
+
+	/* Reset portions of the host state accordingly */
+	hreg = sdstd_rreg8(si, SD_HostCntrl);
+	hreg = SFIELD(hreg, HOST_HI_SPEED_EN, 0);
+	hreg = SFIELD(hreg, HOST_DATA_WIDTH, 0);
+	si->sd_mode = SDIOH_MODE_SD1;
+
+	/* Reinitialize the card */
+	si->card_init_done = FALSE;
+	return sdstd_client_init(si);
+}
+
+static void
+sd_map_dma(sdioh_info_t * sd)
+{
+
+	int alloced;
+	void *va;
+	uint dma_buf_size = SD_PAGE;
+
+#ifdef BCMSDIOH_TXGLOM
+	/* There is no alignment requirement for HC3 */
+	if ((sd->version == HOST_CONTR_VER_3) && sd_txglom) {
+		/* Max glom packet length is 64KB */
+		dma_buf_size = SD_PAGE * 16;
+	}
+#endif
+
+	alloced = 0;
+	if ((va = DMA_ALLOC_CONSISTENT(sd->osh, dma_buf_size, SD_PAGE_BITS, &alloced,
+		&sd->dma_start_phys, 0x12)) == NULL) {
+		sd->sd_dma_mode = DMA_MODE_NONE;
+		sd->dma_start_buf = 0;
+		sd->dma_buf = (void *)0;
+		sd->dma_phys = 0;
+		sd->alloced_dma_size = 0;
+		sd_err(("%s: DMA_ALLOC failed. Disabling DMA support.\n", __FUNCTION__));
+	} else {
+		sd->dma_start_buf = va;
+		sd->dma_buf = (void *)ROUNDUP((uintptr)va, SD_PAGE);
+		sd->dma_phys = ROUNDUP((sd->dma_start_phys), SD_PAGE);
+		sd->alloced_dma_size = alloced;
+		sd_err(("%s: Mapped DMA Buffer %dbytes @virt/phys: %p/0x%x-0x%x\n",
+		        __FUNCTION__, sd->alloced_dma_size, sd->dma_buf,
+		        (uint)PHYSADDRHI(sd->dma_phys), (uint)PHYSADDRLO(sd->dma_phys)));
+		sd_fill_dma_data_buf(sd, 0xA5);
+	}
+
+	if ((va = DMA_ALLOC_CONSISTENT(sd->osh, SD_PAGE, SD_PAGE_BITS, &alloced,
+		&sd->adma2_dscr_start_phys, 0x12)) == NULL) {
+		sd->sd_dma_mode = DMA_MODE_NONE;
+		sd->adma2_dscr_start_buf = 0;
+		sd->adma2_dscr_buf = (void *)0;
+		sd->adma2_dscr_phys = 0;
+		sd->alloced_adma2_dscr_size = 0;
+		sd_err(("%s: DMA_ALLOC failed for descriptor buffer. "
+		        "Disabling DMA support.\n", __FUNCTION__));
+	} else {
+		sd->adma2_dscr_start_buf = va;
+		sd->adma2_dscr_buf = (void *)ROUNDUP((uintptr)va, SD_PAGE);
+		sd->adma2_dscr_phys = ROUNDUP((sd->adma2_dscr_start_phys), SD_PAGE);
+		sd->alloced_adma2_dscr_size = alloced;
+		sd_err(("%s: Mapped ADMA2 Descriptor Buffer %dbytes @virt/phys: %p/0x%x-0x%x\n",
+		        __FUNCTION__, sd->alloced_adma2_dscr_size, sd->adma2_dscr_buf,
+		        (uint)PHYSADDRHI(sd->adma2_dscr_phys),
+			(uint)PHYSADDRLO(sd->adma2_dscr_phys)));
+		sd_clear_adma_dscr_buf(sd);
+	}
+}
+
+static void
+sd_unmap_dma(sdioh_info_t * sd)
+{
+	if (sd->dma_start_buf) {
+		DMA_FREE_CONSISTENT(sd->osh, sd->dma_start_buf, sd->alloced_dma_size,
+			sd->dma_start_phys, 0x12);
+	}
+
+	if (sd->adma2_dscr_start_buf) {
+		DMA_FREE_CONSISTENT(sd->osh, sd->adma2_dscr_start_buf, sd->alloced_adma2_dscr_size,
+		                    sd->adma2_dscr_start_phys, 0x12);
+	}
+}
+
+static void
+sd_clear_adma_dscr_buf(sdioh_info_t *sd)
+{
+	bzero((char *)sd->adma2_dscr_buf, SD_PAGE);
+	sd_dump_adma_dscr(sd);
+}
+
+static void
+sd_fill_dma_data_buf(sdioh_info_t *sd, uint8 data)
+{
+	memset((char *)sd->dma_buf, data, SD_PAGE);
+}
+
+static void
+sd_create_adma_descriptor(sdioh_info_t *sd, uint32 index,
+                          uint32 addr_phys, uint16 length, uint16 flags)
+{
+	adma2_dscr_32b_t *adma2_dscr_table;
+	adma1_dscr_t *adma1_dscr_table;
+
+	adma2_dscr_table = sd->adma2_dscr_buf;
+	adma1_dscr_table = sd->adma2_dscr_buf;
+
+	switch (sd->sd_dma_mode) {
+		case DMA_MODE_ADMA2:
+			sd_dma(("%s: creating ADMA2 descriptor for index %d\n",
+				__FUNCTION__, index));
+
+			adma2_dscr_table[index].phys_addr = addr_phys;
+			adma2_dscr_table[index].len_attr = length << 16;
+			adma2_dscr_table[index].len_attr |= flags;
+			break;
+		case DMA_MODE_ADMA1:
+			/* ADMA1 requires two descriptors, one for len
+			 * and the other for data transfer
+			 */
+			index <<= 1;
+
+			sd_dma(("%s: creating ADMA1 descriptor for index %d\n",
+				__FUNCTION__, index));
+
+			adma1_dscr_table[index].phys_addr_attr = length << 12;
+			adma1_dscr_table[index].phys_addr_attr |= (ADMA1_ATTRIBUTE_ACT_SET |
+			                                           ADMA2_ATTRIBUTE_VALID);
+			adma1_dscr_table[index+1].phys_addr_attr = addr_phys & 0xFFFFF000;
+			adma1_dscr_table[index+1].phys_addr_attr |= (flags & 0x3f);
+			break;
+		default:
+			sd_err(("%s: cannot create ADMA descriptor for DMA mode %d\n",
+				__FUNCTION__, sd->sd_dma_mode));
+			break;
+	}
+}
+
+static void
+sd_dump_adma_dscr(sdioh_info_t *sd)
+{
+	adma2_dscr_32b_t *adma2_dscr_table;
+	adma1_dscr_t *adma1_dscr_table;
+	uint32 i = 0;
+	uint16 flags;
+	char flags_str[32];
+
+	ASSERT(sd->adma2_dscr_buf != NULL);
+
+	adma2_dscr_table = sd->adma2_dscr_buf;
+	adma1_dscr_table = sd->adma2_dscr_buf;
+
+	switch (sd->sd_dma_mode) {
+		case DMA_MODE_ADMA2:
+			sd_err(("ADMA2 Descriptor Table (%dbytes) @virt/phys: %p/0x%x-0x%x\n",
+				SD_PAGE, sd->adma2_dscr_buf,
+			        (uint)PHYSADDRHI(sd->adma2_dscr_phys),
+			        (uint)PHYSADDRLO(sd->adma2_dscr_phys)));
+			sd_err((" #[Descr VA  ]  Buffer PA  | Len    | Flags  (5:4  2   1   0)"
+			        "     |\n"));
+			while (adma2_dscr_table->len_attr & ADMA2_ATTRIBUTE_VALID) {
+				flags = adma2_dscr_table->len_attr & 0xFFFF;
+				sprintf(flags_str, "%s%s%s%s",
+					((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+					ADMA2_ATTRIBUTE_ACT_LINK) ? "LINK " :
+					((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+					ADMA2_ATTRIBUTE_ACT_TRAN) ? "TRAN " :
+					((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+					ADMA2_ATTRIBUTE_ACT_NOP) ? "NOP  " : "RSV  ",
+					(flags & ADMA2_ATTRIBUTE_INT ? "INT " : "    "),
+					(flags & ADMA2_ATTRIBUTE_END ? "END " : "    "),
+					(flags & ADMA2_ATTRIBUTE_VALID ? "VALID" : ""));
+				sd_err(("%2d[0x%p]: 0x%08x | 0x%04x | 0x%04x (%s) |\n",
+				        i, adma2_dscr_table, adma2_dscr_table->phys_addr,
+				        adma2_dscr_table->len_attr >> 16, flags, flags_str));
+				i++;
+
+#ifdef linux
+				/* Follow LINK descriptors or skip to next. */
+				if ((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+				     ADMA2_ATTRIBUTE_ACT_LINK) {
+					adma2_dscr_table = phys_to_virt(
+					    adma2_dscr_table->phys_addr);
+				} else {
+					adma2_dscr_table++;
+				}
+#else
+				adma2_dscr_table++;
+#endif /* linux */
+
+			}
+			break;
+		case DMA_MODE_ADMA1:
+			sd_err(("ADMA1 Descriptor Table (%dbytes) @virt/phys: %p/0x%x-0x%x\n",
+			         SD_PAGE, sd->adma2_dscr_buf,
+			        (uint)PHYSADDRHI(sd->adma2_dscr_phys),
+			        (uint)PHYSADDRLO(sd->adma2_dscr_phys)));
+			sd_err((" #[Descr VA  ]  Buffer PA  | Flags  (5:4  2   1   0)     |\n"));
+
+			for (i = 0; adma1_dscr_table->phys_addr_attr & ADMA2_ATTRIBUTE_VALID; i++) {
+				flags = adma1_dscr_table->phys_addr_attr & 0x3F;
+				sprintf(flags_str, "%s%s%s%s",
+					((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+					ADMA2_ATTRIBUTE_ACT_LINK) ? "LINK " :
+					((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+					ADMA2_ATTRIBUTE_ACT_TRAN) ? "TRAN " :
+					((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+					ADMA2_ATTRIBUTE_ACT_NOP) ? "NOP  " : "SET  ",
+					(flags & ADMA2_ATTRIBUTE_INT ? "INT " : "    "),
+					(flags & ADMA2_ATTRIBUTE_END ? "END " : "    "),
+					(flags & ADMA2_ATTRIBUTE_VALID ? "VALID" : ""));
+				sd_err(("%2d[0x%p]: 0x%08x | 0x%04x | (%s) |\n",
+				        i, adma1_dscr_table,
+				        adma1_dscr_table->phys_addr_attr & 0xFFFFF000,
+				        flags, flags_str));
+
+#ifdef linux
+				/* Follow LINK descriptors or skip to next. */
+				if ((flags & ADMA2_ATTRIBUTE_ACT_LINK) ==
+				     ADMA2_ATTRIBUTE_ACT_LINK) {
+					adma1_dscr_table = phys_to_virt(
+						adma1_dscr_table->phys_addr_attr & 0xFFFFF000);
+				} else {
+					adma1_dscr_table++;
+				}
+#else
+				adma2_dscr_table++;
+#endif /* linux */
+			}
+			break;
+		default:
+			sd_err(("Unknown DMA Descriptor Table Format.\n"));
+			break;
+	}
+}
+
+static void
+sdstd_dumpregs(sdioh_info_t *sd)
+{
+	sd_err(("IntrStatus:       0x%04x ErrorIntrStatus       0x%04x\n",
+	            sdstd_rreg16(sd, SD_IntrStatus),
+	            sdstd_rreg16(sd, SD_ErrorIntrStatus)));
+	sd_err(("IntrStatusEnable: 0x%04x ErrorIntrStatusEnable 0x%04x\n",
+	            sdstd_rreg16(sd, SD_IntrStatusEnable),
+	            sdstd_rreg16(sd, SD_ErrorIntrStatusEnable)));
+	sd_err(("IntrSignalEnable: 0x%04x ErrorIntrSignalEnable 0x%04x\n",
+	            sdstd_rreg16(sd, SD_IntrSignalEnable),
+	            sdstd_rreg16(sd, SD_ErrorIntrSignalEnable)));
+}
diff --git a/drivers/net/wireless/ap12275/bcmsdstd.h b/drivers/net/wireless/ap12275/bcmsdstd.h
new file mode 100644
index 000000000000..de742ed544a9
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdstd.h
@@ -0,0 +1,298 @@
+/*
+ *  'Standard' SDIO HOST CONTROLLER driver
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id: bcmsdstd.h 833030 2019-08-02 17:22:42Z $
+ */
+#ifndef	_BCM_SD_STD_H
+#define	_BCM_SD_STD_H
+
+/* global msglevel for debug messages - bitvals come from sdiovar.h */
+#ifdef BCMDBG
+#define sd_err(x)	do { if (sd_msglevel & SDH_ERROR_VAL) printf x; } while (0)
+#define sd_trace(x)	do { if (sd_msglevel & SDH_TRACE_VAL) printf x; } while (0)
+#define sd_info(x)	do { if (sd_msglevel & SDH_INFO_VAL)  printf x; } while (0)
+#define sd_debug(x)	do { if (sd_msglevel & SDH_DEBUG_VAL) printf x; } while (0)
+#define sd_data(x)	do { if (sd_msglevel & SDH_DATA_VAL)  printf x; } while (0)
+#define sd_ctrl(x)	do { if (sd_msglevel & SDH_CTRL_VAL)  printf x; } while (0)
+#define sd_dma(x)	do { if (sd_msglevel & SDH_DMA_VAL)  printf x; } while (0)
+#else
+#define sd_err(x)	do { if (sd_msglevel & SDH_ERROR_VAL) printf x; } while (0)
+#define sd_trace(x)
+#define sd_info(x)
+#define sd_debug(x)
+#define sd_data(x)
+#define sd_ctrl(x)
+#define sd_dma(x)
+#endif /* BCMDBG */
+
+#define sd_sync_dma(sd, read, nbytes)
+#define sd_init_dma(sd)
+#define sd_ack_intr(sd)
+#define sd_wakeup(sd);
+/* Allocate/init/free per-OS private data */
+extern int sdstd_osinit(sdioh_info_t *sd);
+extern void sdstd_osfree(sdioh_info_t *sd);
+
+#ifdef BCMPERFSTATS
+#define sd_log(x)	do { if (sd_msglevel & SDH_LOG_VAL)	 bcmlog x; } while (0)
+#else
+#define sd_log(x)
+#endif
+
+#define SDIOH_ASSERT(exp) \
+	do { if (!(exp)) \
+		printf("!!!ASSERT fail: file %s lines %d", __FILE__, __LINE__); \
+	} while (0)
+
+#define BLOCK_SIZE_4318 64
+#define BLOCK_SIZE_4328 512
+
+/* internal return code */
+#define SUCCESS	0
+#define ERROR	1
+
+/* private bus modes */
+#define SDIOH_MODE_SPI		0
+#define SDIOH_MODE_SD1		1
+#define SDIOH_MODE_SD4		2
+
+#define MAX_SLOTS 6 	/* For PCI: Only 6 BAR entries => 6 slots */
+#define SDIOH_REG_WINSZ	0x100 /* Number of registers in Standard Host Controller */
+
+#define SDIOH_TYPE_ARASAN_HDK	1
+#define SDIOH_TYPE_BCM27XX	2
+#define SDIOH_TYPE_TI_PCIXX21	4	/* TI PCIxx21 Standard Host Controller */
+#define SDIOH_TYPE_RICOH_R5C822	5	/* Ricoh Co Ltd R5C822 SD/SDIO/MMC/MS/MSPro Host Adapter */
+#define SDIOH_TYPE_JMICRON	6	/* JMicron Standard SDIO Host Controller */
+
+/* For linux, allow yielding for dongle */
+#if defined(linux) && defined(BCMDONGLEHOST)
+#define BCMSDYIELD
+#endif
+
+/* Expected card status value for CMD7 */
+#define SDIOH_CMD7_EXP_STATUS   0x00001E00
+
+#define RETRIES_LARGE 100000
+#ifdef BCMQT
+extern void sdstd_os_yield(sdioh_info_t *sd);
+#define RETRIES_SMALL 10000
+#else
+#define sdstd_os_yield(sd)	do {} while (0)
+#define RETRIES_SMALL 100
+#endif
+
+#define USE_BLOCKMODE		0x2	/* Block mode can be single block or multi */
+#define USE_MULTIBLOCK		0x4
+
+#define USE_FIFO		0x8	/* Fifo vs non-fifo */
+
+#define CLIENT_INTR 		0x100	/* Get rid of this! */
+
+#define HC_INTR_RETUNING	0x1000
+
+#ifdef BCMSDIOH_TXGLOM
+/* Total glom pkt can not exceed 64K
+ * need one more slot for glom padding packet
+ */
+#define SDIOH_MAXGLOM_SIZE	(40+1)
+
+typedef struct glom_buf {
+	uint32 count;				/* Total number of pkts queued */
+	void *dma_buf_arr[SDIOH_MAXGLOM_SIZE];	/* Frame address */
+	dmaaddr_t dma_phys_arr[SDIOH_MAXGLOM_SIZE]; /* DMA_MAPed address of frames */
+	uint16 nbytes[SDIOH_MAXGLOM_SIZE];	/* Size of each frame */
+} glom_buf_t;
+#endif
+
+struct sdioh_info {
+	uint cfg_bar;				/* pci cfg address for bar */
+	uint32 caps;				/* cached value of capabilities reg */
+	uint32 curr_caps;			/* max current capabilities reg */
+
+	osl_t		*osh;			/* osh handler */
+	volatile char	*mem_space;		/* pci device memory va */
+	uint		lockcount;		/* nest count of sdstd_lock() calls */
+	bool		client_intr_enabled;	/* interrupt connnected flag */
+	bool		intr_handler_valid;	/* client driver interrupt handler valid */
+	sdioh_cb_fn_t	intr_handler;		/* registered interrupt handler */
+	void		*intr_handler_arg;	/* argument to call interrupt handler */
+	bool		initialized;		/* card initialized */
+	uint		target_dev;		/* Target device ID */
+	uint16		intmask;		/* Current active interrupts */
+	void		*sdos_info;		/* Pointer to per-OS private data */
+	void		*bcmsdh;		/* handler to upper layer stack (bcmsdh) */
+
+	uint32		controller_type;	/* Host controller type */
+	uint8		version;		/* Host Controller Spec Compliance Version */
+	uint		irq;			/* Client irq */
+	int		intrcount;		/* Client interrupts */
+	int		local_intrcount;	/* Controller interrupts */
+	bool		host_init_done;		/* Controller initted */
+	bool		card_init_done;		/* Client SDIO interface initted */
+	bool		polled_mode;		/* polling for command completion */
+
+	bool		sd_blockmode;		/* sd_blockmode == FALSE => 64 Byte Cmd 53s. */
+						/*  Must be on for sd_multiblock to be effective */
+	bool		use_client_ints;	/* If this is false, make sure to restore */
+						/*  polling hack in wl_linux.c:wl_timer() */
+	int		adapter_slot;		/* Maybe dealing with multiple slots/controllers */
+	int		sd_mode;		/* SD1/SD4/SPI */
+	int		client_block_size[SDIOD_MAX_IOFUNCS];		/* Blocksize */
+	uint32		data_xfer_count;	/* Current transfer */
+	uint16		card_rca;		/* Current Address */
+	int8		sd_dma_mode;		/* DMA Mode (PIO, SDMA, ... ADMA2) on CMD53 */
+	uint8		num_funcs;		/* Supported funcs on client */
+	uint32		com_cis_ptr;
+	uint32		func_cis_ptr[SDIOD_MAX_IOFUNCS];
+	void		*dma_buf;		/* DMA Buffer virtual address */
+	dmaaddr_t	dma_phys;		/* DMA Buffer physical address */
+	void		*adma2_dscr_buf;	/* ADMA2 Descriptor Buffer virtual address */
+	dmaaddr_t	adma2_dscr_phys;	/* ADMA2 Descriptor Buffer physical address */
+
+	/* adjustments needed to make the dma align properly */
+	void		*dma_start_buf;
+	dmaaddr_t	dma_start_phys;
+	uint		alloced_dma_size;
+	void		*adma2_dscr_start_buf;
+	dmaaddr_t	adma2_dscr_start_phys;
+	uint		alloced_adma2_dscr_size;
+
+	int		r_cnt;			/* rx count */
+	int		t_cnt;			/* tx_count */
+	bool		got_hcint;		/* local interrupt flag */
+	uint16		last_intrstatus;	/* to cache intrstatus */
+	int	host_UHSISupported;		/* whether UHSI is supported for HC. */
+	int	card_UHSI_voltage_Supported;	/* whether UHSI is supported for
+						 * Card in terms of Voltage [1.8 or 3.3].
+						 */
+	int	global_UHSI_Supp;	/* type of UHSI support in both host and card.
+					 * HOST_SDR_UNSUPP: capabilities not supported/matched
+					 * HOST_SDR_12_25: SDR12 and SDR25 supported
+					 * HOST_SDR_50_104_DDR: one of SDR50/SDR104 or DDR50 supptd
+					 */
+	volatile int	sd3_dat_state;		/* data transfer state used for retuning check */
+	volatile int	sd3_tun_state;		/* tuning state used for retuning check */
+	bool	sd3_tuning_reqd;	/* tuning requirement parameter */
+	bool	sd3_tuning_disable;	/* tuning disable due to bus sleeping */
+	uint32	caps3;			/* cached value of 32 MSbits capabilities reg (SDIO 3.0) */
+#ifdef BCMSDIOH_TXGLOM
+	glom_buf_t glom_info;		/* pkt information used for glomming */
+	uint	txglom_mode;		/* Txglom mode: 0 - copy, 1 - multi-descriptor */
+#endif
+};
+
+#define DMA_MODE_NONE	0
+#define DMA_MODE_SDMA	1
+#define DMA_MODE_ADMA1	2
+#define DMA_MODE_ADMA2	3
+#define DMA_MODE_ADMA2_64 4
+#define DMA_MODE_AUTO	-1
+
+#define USE_DMA(sd)		((bool)((sd->sd_dma_mode > 0) ? TRUE : FALSE))
+
+/* States for Tuning and corr data */
+#define TUNING_IDLE 			0
+#define TUNING_START 			1
+#define TUNING_START_AFTER_DAT 	2
+#define TUNING_ONGOING 			3
+
+#define DATA_TRANSFER_IDLE 		0
+#define DATA_TRANSFER_ONGOING	1
+
+#define CHECK_TUNING_PRE_DATA	1
+#define CHECK_TUNING_POST_DATA	2
+
+#ifdef DHD_DEBUG
+#define SD_DHD_DISABLE_PERIODIC_TUNING 0x01
+#define SD_DHD_ENABLE_PERIODIC_TUNING  0x00
+#endif
+
+/************************************************************
+ * Internal interfaces: per-port references into bcmsdstd.c
+ */
+
+/* Global message bits */
+extern uint sd_msglevel;
+
+/* OS-independent interrupt handler */
+extern bool check_client_intr(sdioh_info_t *sd);
+
+/* Core interrupt enable/disable of device interrupts */
+extern void sdstd_devintr_on(sdioh_info_t *sd);
+extern void sdstd_devintr_off(sdioh_info_t *sd);
+
+/* Enable/disable interrupts for local controller events */
+extern void sdstd_intrs_on(sdioh_info_t *sd, uint16 norm, uint16 err);
+extern void sdstd_intrs_off(sdioh_info_t *sd, uint16 norm, uint16 err);
+
+/* Wait for specified interrupt and error bits to be set */
+extern void sdstd_spinbits(sdioh_info_t *sd, uint16 norm, uint16 err);
+
+/**************************************************************
+ * Internal interfaces: bcmsdstd.c references to per-port code
+ */
+
+/* Register mapping routines */
+extern uint32 *sdstd_reg_map(osl_t *osh, dmaaddr_t addr, int size);
+extern void sdstd_reg_unmap(osl_t *osh, dmaaddr_t addr, int size);
+
+/* Interrupt (de)registration routines */
+extern int sdstd_register_irq(sdioh_info_t *sd, uint irq);
+extern void sdstd_free_irq(uint irq, sdioh_info_t *sd);
+
+/* OS-specific interrupt wrappers (atomic interrupt enable/disable) */
+extern void sdstd_lock(sdioh_info_t *sd);
+extern void sdstd_unlock(sdioh_info_t *sd);
+extern void sdstd_waitlockfree(sdioh_info_t *sd);
+
+/* OS-specific wrappers for safe concurrent register access */
+extern void sdstd_os_lock_irqsave(sdioh_info_t *sd, ulong* flags);
+extern void sdstd_os_unlock_irqrestore(sdioh_info_t *sd, ulong* flags);
+
+/* OS-specific wait-for-interrupt-or-status */
+extern int sdstd_waitbits(sdioh_info_t *sd, uint16 norm, uint16 err, bool yield, uint16 *bits);
+
+/* used by bcmsdstd_linux [implemented in sdstd] */
+extern void sdstd_3_enable_retuning_int(sdioh_info_t *sd);
+extern void sdstd_3_disable_retuning_int(sdioh_info_t *sd);
+extern bool sdstd_3_is_retuning_int_set(sdioh_info_t *sd);
+extern void sdstd_3_check_and_do_tuning(sdioh_info_t *sd, int tuning_param);
+extern bool sdstd_3_check_and_set_retuning(sdioh_info_t *sd);
+extern int sdstd_3_get_tune_state(sdioh_info_t *sd);
+extern int sdstd_3_get_data_state(sdioh_info_t *sd);
+extern void sdstd_3_set_tune_state(sdioh_info_t *sd, int state);
+extern void sdstd_3_set_data_state(sdioh_info_t *sd, int state);
+extern uint8 sdstd_3_get_tuning_exp(sdioh_info_t *sd);
+extern uint32 sdstd_3_get_uhsi_clkmode(sdioh_info_t *sd);
+extern int sdstd_3_clk_tuning(sdioh_info_t *sd, uint32 sd3ClkMode);
+
+/* used by sdstd [implemented in bcmsdstd_linux/ndis] */
+extern void sdstd_3_start_tuning(sdioh_info_t *sd);
+extern void sdstd_3_osinit_tuning(sdioh_info_t *sd);
+extern void sdstd_3_osclean_tuning(sdioh_info_t *sd);
+
+extern void sdstd_enable_disable_periodic_timer(sdioh_info_t * sd, uint val);
+
+extern sdioh_info_t *sdioh_attach(osl_t *osh, void *bar0, uint irq);
+extern SDIOH_API_RC sdioh_detach(osl_t *osh, sdioh_info_t *sd);
+#endif /* _BCM_SD_STD_H */
diff --git a/drivers/net/wireless/ap12275/bcmsdstd_linux.c b/drivers/net/wireless/ap12275/bcmsdstd_linux.c
new file mode 100644
index 000000000000..11d3f09f37d6
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmsdstd_linux.c
@@ -0,0 +1,697 @@
+/*
+ *  'Standard' SDIO HOST CONTROLLER driver - linux portion
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <linux/sched.h>	/* request_irq() */
+#include <typedefs.h>
+#include <pcicfg.h>
+#include <bcmutils.h>
+#include <sdio.h>	/* SDIO Device and Protocol Specs */
+#include <sdioh.h> /* SDIO Host Controller Spec header file */
+#include <bcmsdbus.h>	/* bcmsdh to/from specific controller APIs */
+#include <sdiovar.h>	/* to get msglevel bit values */
+#include <bcmsdstd.h>
+#include <bcmdevs.h>
+
+extern void* bcmsdh_probe(osl_t *osh, void *dev, void *sdioh, void *adapter_info, uint bus_type,
+	uint bus_num, uint slot_num);
+extern int bcmsdh_remove(bcmsdh_info_t *bcmsdh);
+
+/* Extern functions for sdio power save */
+extern uint8 sdstd_turn_on_clock(sdioh_info_t *sd);
+extern uint8 sdstd_turn_off_clock(sdioh_info_t *sd);
+/* Extern variable for sdio power save. This is enabled or disabled using the IOCTL call */
+extern uint sd_3_power_save;
+
+struct sdos_info {
+	sdioh_info_t *sd;
+	spinlock_t lock;
+	wait_queue_head_t intr_wait_queue;
+	timer_list_compat_t tuning_timer;
+	int tuning_timer_exp;
+	atomic_t timer_enab;
+	struct tasklet_struct tuning_tasklet;
+};
+
+#define SDSTD_WAITBITS_TIMEOUT		(5 * HZ)	/* seconds * HZ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0))
+#define BLOCKABLE()	(!in_atomic())
+#else
+#define BLOCKABLE()	(!in_interrupt()) /* XXX Doesn't handle CONFIG_PREEMPT? */
+#endif
+
+static void
+sdstd_3_ostasklet(ulong data);
+static void
+sdstd_3_tuning_timer(ulong data);
+
+/* Interrupt handler */
+static irqreturn_t
+sdstd_isr(int irq, void *dev_id
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
+, struct pt_regs *ptregs
+#endif
+)
+{
+	sdioh_info_t *sd;
+	struct sdos_info *sdos;
+	bool ours;
+
+	unsigned long flags;
+	sd = (sdioh_info_t *)dev_id;
+	sdos = (struct sdos_info *)sd->sdos_info;
+
+	if (!sd->card_init_done) {
+		sd_err(("%s: Hey Bogus intr...not even initted: irq %d\n", __FUNCTION__, irq));
+		return IRQ_RETVAL(FALSE);
+	} else {
+		if (sdstd_3_is_retuning_int_set(sd)) {
+			/* for 3.0 host, retuning request might come in this path */
+			/* * disable ISR's */
+			local_irq_save(flags);
+
+			if (sdstd_3_check_and_set_retuning(sd))
+				tasklet_schedule(&sdos->tuning_tasklet);
+
+			/* * enable back ISR's */
+			local_irq_restore(flags);
+
+			/* * disable tuning isr signaling */
+			sdstd_3_disable_retuning_int(sd);
+			/* * note: check_client_intr() checks for  intmask also to
+				wakeup. so be careful to use sd->intmask to disable
+				re-tuning ISR.
+				*/
+		}
+		ours = check_client_intr(sd);
+
+		/* For local interrupts, wake the waiting process */
+		if (ours && sd->got_hcint) {
+			sd_trace(("INTR->WAKE\n"));
+/* 			sdos = (struct sdos_info *)sd->sdos_info; */
+			wake_up_interruptible(&sdos->intr_wait_queue);
+		}
+		return IRQ_RETVAL(ours);
+	}
+}
+
+/* Register with Linux for interrupts */
+int
+sdstd_register_irq(sdioh_info_t *sd, uint irq)
+{
+	sd_trace(("Entering %s: irq == %d\n", __FUNCTION__, irq));
+	if (request_irq(irq, sdstd_isr, IRQF_SHARED, "bcmsdstd", sd) < 0) {
+		sd_err(("%s: request_irq() failed\n", __FUNCTION__));
+		return ERROR;
+	}
+	return SUCCESS;
+}
+
+/* Free Linux irq */
+void
+sdstd_free_irq(uint irq, sdioh_info_t *sd)
+{
+	free_irq(irq, sd);
+}
+
+/* Map Host controller registers */
+
+uint32 *
+sdstd_reg_map(osl_t *osh, dmaaddr_t addr, int size)
+{
+	return (uint32 *)REG_MAP(addr, size);
+}
+
+void
+sdstd_reg_unmap(osl_t *osh, dmaaddr_t addr, int size)
+{
+	REG_UNMAP((void*)(uintptr)addr);
+}
+
+int
+sdstd_osinit(sdioh_info_t *sd)
+{
+	struct sdos_info *sdos;
+
+	sdos = (struct sdos_info*)MALLOC(sd->osh, sizeof(struct sdos_info));
+	sd->sdos_info = (void*)sdos;
+	if (sdos == NULL)
+		return BCME_NOMEM;
+
+	sdos->sd = sd;
+	spin_lock_init(&sdos->lock);
+	atomic_set(&sdos->timer_enab, FALSE);
+	init_waitqueue_head(&sdos->intr_wait_queue);
+	return BCME_OK;
+}
+
+/* initilize tuning related OS structures */
+void
+sdstd_3_osinit_tuning(sdioh_info_t *sd)
+{
+	struct sdos_info *sdos = (struct sdos_info *)sd->sdos_info;
+	uint8 timer_count = sdstd_3_get_tuning_exp(sdos->sd);
+
+	sd_trace(("%s Enter\n", __FUNCTION__));
+
+	init_timer_compat(&sdos->tuning_timer, sdstd_3_tuning_timer, sdos);
+	if (timer_count == CAP3_RETUNING_TC_DISABLED || timer_count > CAP3_RETUNING_TC_1024S) {
+		sdos->tuning_timer_exp = 0;
+	} else {
+		sdos->tuning_timer_exp = 1 << (timer_count - 1);
+	}
+	tasklet_init(&sdos->tuning_tasklet, sdstd_3_ostasklet, (ulong)sdos);
+	if (sdos->tuning_timer_exp) {
+		timer_expires(&sdos->tuning_timer) = jiffies + sdos->tuning_timer_exp * HZ;
+		add_timer(&sdos->tuning_timer);
+		atomic_set(&sdos->timer_enab, TRUE);
+	}
+}
+
+/* finalize tuning related OS structures */
+void
+sdstd_3_osclean_tuning(sdioh_info_t *sd)
+{
+	struct sdos_info *sdos = (struct sdos_info *)sd->sdos_info;
+	if (atomic_read(&sdos->timer_enab) == TRUE) {
+		/* disable timer if it was running */
+		del_timer_sync(&sdos->tuning_timer);
+		atomic_set(&sdos->timer_enab, FALSE);
+	}
+	tasklet_kill(&sdos->tuning_tasklet);
+}
+
+static void
+sdstd_3_ostasklet(ulong data)
+{
+	struct sdos_info *sdos = (struct sdos_info *)data;
+	int tune_state = sdstd_3_get_tune_state(sdos->sd);
+	int data_state = sdstd_3_get_data_state(sdos->sd);
+	if ((tune_state == TUNING_START) || (tune_state == TUNING_ONGOING) ||
+		(tune_state == TUNING_START_AFTER_DAT)) {
+		return;
+	}
+	else if (data_state == DATA_TRANSFER_IDLE)
+		sdstd_3_set_tune_state(sdos->sd, TUNING_START);
+	else if (data_state == DATA_TRANSFER_ONGOING)
+		sdstd_3_set_tune_state(sdos->sd, TUNING_START_AFTER_DAT);
+}
+
+static void
+sdstd_3_tuning_timer(ulong data)
+{
+	struct sdos_info *sdos = (struct sdos_info *)data;
+/* 	uint8 timeout = 0; */
+	unsigned long int_flags;
+
+	sd_trace(("%s: enter\n", __FUNCTION__));
+	/* schedule tasklet */
+	/* * disable ISR's */
+	local_irq_save(int_flags);
+	if (sdstd_3_check_and_set_retuning(sdos->sd))
+		tasklet_schedule(&sdos->tuning_tasklet);
+
+	/* * enable back ISR's */
+	local_irq_restore(int_flags);
+}
+
+void sdstd_3_start_tuning(sdioh_info_t *sd)
+{
+	int tune_state;
+	unsigned long int_flags = 0;
+	unsigned int timer_enab;
+	struct sdos_info *sdos = (struct sdos_info *)sd->sdos_info;
+	sd_trace(("%s: enter\n", __FUNCTION__));
+	/* * disable ISR's */
+	local_irq_save(int_flags);
+	timer_enab = atomic_read(&sdos->timer_enab);
+
+	tune_state = sdstd_3_get_tune_state(sd);
+
+	if (tune_state == TUNING_ONGOING) {
+		/* do nothing */
+		local_irq_restore(int_flags);
+		goto exit;
+	}
+	/* change state */
+	sdstd_3_set_tune_state(sd, TUNING_ONGOING);
+	/* * enable ISR's */
+	local_irq_restore(int_flags);
+	sdstd_3_clk_tuning(sd, sdstd_3_get_uhsi_clkmode(sd));
+#ifdef BCMSDIOH_STD_TUNING_WAR
+	/*
+	 * Observed intermittent SDIO command error after re-tuning done
+	 * successfully.  Re-tuning twice is giving much reliable results.
+	 */
+	sdstd_3_clk_tuning(sd, sdstd_3_get_uhsi_clkmode(sd));
+#endif /* BCMSDIOH_STD_TUNING_WAR */
+	/* * disable ISR's */
+	local_irq_save(int_flags);
+	sdstd_3_set_tune_state(sd, TUNING_IDLE);
+	/* * enable ISR's */
+	local_irq_restore(int_flags);
+
+	/* enable retuning intrrupt */
+	sdstd_3_enable_retuning_int(sd);
+
+	/* start retuning timer if enabled */
+	if ((sdos->tuning_timer_exp) && (timer_enab)) {
+		if (sd->sd3_tuning_reqd) {
+			timer_expires(&sdos->tuning_timer) = jiffies + sdos->tuning_timer_exp * HZ;
+			mod_timer(&sdos->tuning_timer, timer_expires(&sdos->tuning_timer));
+		}
+	}
+exit:
+	return;
+
+}
+
+void
+sdstd_osfree(sdioh_info_t *sd)
+{
+	struct sdos_info *sdos;
+	ASSERT(sd && sd->sdos_info);
+
+	sdos = (struct sdos_info *)sd->sdos_info;
+	MFREE(sd->osh, sdos, sizeof(struct sdos_info));
+}
+
+/* Interrupt enable/disable */
+SDIOH_API_RC
+sdioh_interrupt_set(sdioh_info_t *sd, bool enable)
+{
+	ulong flags;
+	struct sdos_info *sdos;
+
+	sd_trace(("%s: %s\n", __FUNCTION__, enable ? "Enabling" : "Disabling"));
+
+	sdos = (struct sdos_info *)sd->sdos_info;
+	ASSERT(sdos);
+
+	if (!(sd->host_init_done && sd->card_init_done)) {
+		sd_err(("%s: Card & Host are not initted - bailing\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	if (enable && !(sd->intr_handler && sd->intr_handler_arg)) {
+		sd_err(("%s: no handler registered, will not enable\n", __FUNCTION__));
+		return SDIOH_API_RC_FAIL;
+	}
+
+	/* Ensure atomicity for enable/disable calls */
+	spin_lock_irqsave(&sdos->lock, flags);
+
+	sd->client_intr_enabled = enable;
+	if (enable && !sd->lockcount)
+		sdstd_devintr_on(sd);
+	else
+		sdstd_devintr_off(sd);
+
+	spin_unlock_irqrestore(&sdos->lock, flags);
+
+	return SDIOH_API_RC_SUCCESS;
+}
+
+/* Protect against reentrancy (disable device interrupts while executing) */
+void
+sdstd_lock(sdioh_info_t *sd)
+{
+	ulong flags;
+	struct sdos_info *sdos;
+	int    wait_count = 0;
+
+	sdos = (struct sdos_info *)sd->sdos_info;
+	ASSERT(sdos);
+
+	sd_trace(("%s: %d\n", __FUNCTION__, sd->lockcount));
+
+	spin_lock_irqsave(&sdos->lock, flags);
+	while (sd->lockcount)
+	{
+	    spin_unlock_irqrestore(&sdos->lock, flags);
+	    yield();
+		spin_lock_irqsave(&sdos->lock, flags);
+		if (++wait_count == 25000) {
+		    if (!(sd->lockcount == 0)) {
+			sd_err(("%s: ERROR: sd->lockcount == 0\n", __FUNCTION__));
+		    }
+		}
+	}
+	/* PR86684: Add temporary debugging print */
+	if (wait_count)
+		printk("sdstd_lock: wait count = %d\n", wait_count);
+	sdstd_devintr_off(sd);
+	sd->lockcount++;
+	spin_unlock_irqrestore(&sdos->lock, flags);
+	if ((sd->controller_type == SDIOH_TYPE_RICOH_R5C822) && (sd->version == HOST_CONTR_VER_3))
+		sdstd_turn_on_clock(sd);
+}
+
+/* Enable client interrupt */
+void
+sdstd_unlock(sdioh_info_t *sd)
+{
+	ulong flags;
+	struct sdos_info *sdos;
+
+	sd_trace(("%s: %d, %d\n", __FUNCTION__, sd->lockcount, sd->client_intr_enabled));
+	ASSERT(sd->lockcount > 0);
+
+	sdos = (struct sdos_info *)sd->sdos_info;
+	ASSERT(sdos);
+
+	spin_lock_irqsave(&sdos->lock, flags);
+	if (--sd->lockcount == 0 && sd->client_intr_enabled) {
+		sdstd_devintr_on(sd);
+	}
+	spin_unlock_irqrestore(&sdos->lock, flags);
+	if (sd_3_power_save)
+	{
+		if ((sd->controller_type == SDIOH_TYPE_RICOH_R5C822) &&
+			(sd->version == HOST_CONTR_VER_3))
+			sdstd_turn_off_clock(sd);
+	}
+}
+
+void
+sdstd_os_lock_irqsave(sdioh_info_t *sd, ulong* flags)
+{
+	struct sdos_info *sdos = (struct sdos_info *)sd->sdos_info;
+	spin_lock_irqsave(&sdos->lock, *flags);
+}
+void
+sdstd_os_unlock_irqrestore(sdioh_info_t *sd, ulong* flags)
+{
+	struct sdos_info *sdos = (struct sdos_info *)sd->sdos_info;
+	spin_unlock_irqrestore(&sdos->lock, *flags);
+}
+
+void
+sdstd_waitlockfree(sdioh_info_t *sd)
+{
+	if (sd->lockcount) {
+		printk("wait lock free\n");
+		while (sd->lockcount)
+		{
+		    yield();
+		}
+	}
+}
+
+#ifdef BCMQT
+void
+sdstd_os_yield(sdioh_info_t *sd)
+{
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 29))
+/*
+ * FC4/11 issue on QT if driver hogs > 10s of CPU causing:
+ *    BUG: soft lockup detected on CPU#0!
+ *
+ * XXX Hack: For now, interleave yielding of CPU when we're spinning waiting for
+ * XXX register status
+ */
+	yield();
+#endif
+}
+#endif /* BCMQT */
+
+/* Returns 0 for success, -1 for interrupted, -2 for timeout */
+int
+sdstd_waitbits(sdioh_info_t *sd, uint16 norm, uint16 err, bool local_yield, uint16 *bits)
+{
+	struct sdos_info *sdos;
+	int rc = 0;
+
+	sdos = (struct sdos_info *)sd->sdos_info;
+
+#ifndef BCMSDYIELD
+	ASSERT(!local_yield);
+#endif
+	sd_trace(("%s: int 0x%02x err 0x%02x yield %d canblock %d\n",
+	          __FUNCTION__, norm, err, local_yield, BLOCKABLE()));
+
+	/* Clear the "interrupt happened" flag and last intrstatus */
+	sd->got_hcint = FALSE;
+	sd->last_intrstatus = 0;
+
+#ifdef BCMSDYIELD
+	if (local_yield && BLOCKABLE()) {
+		/* Enable interrupts, wait for the indication, then disable */
+		sdstd_intrs_on(sd, norm, err);
+		rc = wait_event_interruptible_timeout(sdos->intr_wait_queue,
+		                                      (sd->got_hcint),
+		                                      SDSTD_WAITBITS_TIMEOUT);
+		if (rc < 0)
+			rc = -1;	/* interrupted */
+		else if (rc == 0)
+			rc = -2;	/* timeout */
+		else
+			rc = 0;		/* success */
+		sdstd_intrs_off(sd, norm, err);
+	} else
+#endif /* BCMSDYIELD */
+	{
+		sdstd_spinbits(sd, norm, err);
+	}
+
+	sd_trace(("%s: last_intrstatus 0x%04x\n", __FUNCTION__, sd->last_intrstatus));
+
+	*bits = sd->last_intrstatus;
+
+	return rc;
+}
+
+#ifdef DHD_DEBUG
+void sdstd_enable_disable_periodic_timer(sdioh_info_t *sd, uint val)
+{
+	struct sdos_info *sdos = (struct sdos_info *)sd->sdos_info;
+
+	    if (val == SD_DHD_ENABLE_PERIODIC_TUNING) {
+			/* start of tuning timer */
+			timer_expires(&sdos->tuning_timer) = jiffies +  sdos->tuning_timer_exp * HZ;
+			mod_timer(&sdos->tuning_timer, timer_expires(&sdos->tuning_timer));
+		}
+	    if (val == SD_DHD_DISABLE_PERIODIC_TUNING) {
+			/* stop periodic timer */
+		   del_timer_sync(&sdos->tuning_timer);
+		}
+}
+#endif /* debugging purpose */
+
+/* forward declarations for PCI probe and remove functions. */
+static int __devinit bcmsdh_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
+static void __devexit bcmsdh_pci_remove(struct pci_dev *pdev);
+
+/**
+ * pci id table
+ */
+static struct pci_device_id bcmsdh_pci_devid[] __devinitdata = {
+	{ vendor: PCI_ANY_ID,
+	device: PCI_ANY_ID,
+	subvendor: PCI_ANY_ID,
+	subdevice: PCI_ANY_ID,
+	class: 0,
+	class_mask: 0,
+	driver_data: 0,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+	override_only: 0,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) */
+	},
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+	{ 0, 0, 0, 0, 0, 0, 0, 0}
+#else
+	{ 0, 0, 0, 0, 0, 0, 0}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) */
+};
+MODULE_DEVICE_TABLE(pci, bcmsdh_pci_devid);
+
+/**
+ * SDIO Host Controller pci driver info
+ */
+static struct pci_driver bcmsdh_pci_driver = {
+	node:		{&(bcmsdh_pci_driver.node), &(bcmsdh_pci_driver.node)},
+	name:		"bcmsdh",
+	id_table:	bcmsdh_pci_devid,
+	probe:		bcmsdh_pci_probe,
+	remove:		bcmsdh_pci_remove,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0))
+	save_state:	NULL,
+#endif
+	suspend:	NULL,
+	resume:		NULL,
+	};
+
+extern uint sd_pci_slot;	/* Force detection to a particular PCI */
+							/* slot only . Allows for having multiple */
+							/* WL devices at once in a PC */
+							/* Only one instance of dhd will be */
+							/* usable at a time */
+							/* Upper word is bus number, */
+							/* lower word is slot number */
+							/* Default value of 0xffffffff turns this */
+							/* off */
+module_param(sd_pci_slot, uint, 0);
+
+/**
+ * Detect supported SDIO Host Controller and attach if found.
+ *
+ * Determine if the device described by pdev is a supported SDIO Host
+ * Controller.  If so, attach to it and attach to the target device.
+ */
+static int __devinit
+bcmsdh_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	osl_t *osh = NULL;
+	sdioh_info_t *sdioh = NULL;
+	int rc;
+
+	if (sd_pci_slot != 0xFFFFffff) {
+		if (pdev->bus->number != (sd_pci_slot>>16) ||
+			PCI_SLOT(pdev->devfn) != (sd_pci_slot&0xffff)) {
+			sd_err(("%s: %s: bus %X, slot %X, vend %X, dev %X\n",
+				__FUNCTION__,
+				bcmsdh_chipmatch(pdev->vendor, pdev->device)
+				?"Found compatible SDIOHC"
+				:"Probing unknown device",
+				pdev->bus->number, PCI_SLOT(pdev->devfn), pdev->vendor,
+				pdev->device));
+			return -ENODEV;
+		}
+		sd_err(("%s: %s: bus %X, slot %X, vendor %X, device %X (good PCI location)\n",
+			__FUNCTION__,
+			bcmsdh_chipmatch(pdev->vendor, pdev->device)
+			?"Using compatible SDIOHC"
+			:"WARNING, forced use of unkown device",
+			pdev->bus->number, PCI_SLOT(pdev->devfn), pdev->vendor, pdev->device));
+	}
+
+	if ((pdev->vendor == VENDOR_TI) && ((pdev->device == PCIXX21_FLASHMEDIA_ID) ||
+	    (pdev->device == PCIXX21_FLASHMEDIA0_ID))) {
+		uint32 config_reg;
+
+		sd_err(("%s: Disabling TI FlashMedia Controller.\n", __FUNCTION__));
+		if (!(osh = osl_attach(pdev, SDIO_BUS, TRUE))) {
+			sd_err(("%s: osl_attach failed\n", __FUNCTION__));
+			goto err;
+		}
+
+		config_reg = OSL_PCI_READ_CONFIG(osh, 0x4c, 4);
+
+		/*
+		 * Set MMC_SD_DIS bit in FlashMedia Controller.
+		 * Disbling the SD/MMC Controller in the FlashMedia Controller
+		 * allows the Standard SD Host Controller to take over control
+		 * of the SD Slot.
+		 */
+		config_reg |= 0x02;
+		OSL_PCI_WRITE_CONFIG(osh, 0x4c, 4, config_reg);
+		osl_detach(osh);
+	}
+	/* match this pci device with what we support */
+	/* we can't solely rely on this to believe it is our SDIO Host Controller! */
+	if (!bcmsdh_chipmatch(pdev->vendor, pdev->device)) {
+		if (pdev->vendor == VENDOR_BROADCOM) {
+			sd_err(("%s: Unknown Broadcom device (vendor: %#x, device: %#x).\n",
+				__FUNCTION__, pdev->vendor, pdev->device));
+		}
+		return -ENODEV;
+	}
+
+	/* this is a pci device we might support */
+	sd_err(("%s: Found possible SDIO Host Controller: bus %d slot %d func %d irq %d\n",
+		__FUNCTION__,
+		pdev->bus->number, PCI_SLOT(pdev->devfn),
+		PCI_FUNC(pdev->devfn), pdev->irq));
+
+	/* use bcmsdh_query_device() to get the vendor ID of the target device so
+	 * it will eventually appear in the Broadcom string on the console
+	 */
+
+	/* allocate SDIO Host Controller state info */
+	if (!(osh = osl_attach(pdev, SDIO_BUS, TRUE))) {
+		sd_err(("%s: osl_attach failed\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* map to address where host can access */
+	pci_set_master(pdev);
+	rc = pci_enable_device(pdev);
+	if (rc) {
+		sd_err(("%s: Cannot enable PCI device\n", __FUNCTION__));
+		goto err;
+	}
+
+	sdioh = sdioh_attach(osh, (void *)(ulong)pci_resource_start(pdev, 0), pdev->irq);
+	if (sdioh == NULL) {
+		sd_err(("%s: sdioh_attach failed\n", __FUNCTION__));
+		goto err;
+	}
+	sdioh->bcmsdh = bcmsdh_probe(osh, &pdev->dev, sdioh, NULL, PCI_BUS, -1, -1);
+	if (sdioh->bcmsdh == NULL) {
+		sd_err(("%s: bcmsdh_probe failed\n", __FUNCTION__));
+		goto err;
+	}
+
+	pci_set_drvdata(pdev, sdioh);
+	return 0;
+
+err:
+	if (sdioh != NULL)
+		sdioh_detach(osh, sdioh);
+	if (osh != NULL)
+		osl_detach(osh);
+	return -ENOMEM;
+}
+
+/**
+ * Detach from target devices and SDIO Host Controller
+ */
+static void __devexit
+bcmsdh_pci_remove(struct pci_dev *pdev)
+{
+	sdioh_info_t *sdioh;
+	osl_t *osh;
+
+	sdioh = pci_get_drvdata(pdev);
+	if (sdioh == NULL) {
+		sd_err(("%s: error, no sdioh handler found\n", __FUNCTION__));
+		return;
+	}
+
+	osh = sdioh->osh;
+	bcmsdh_remove(sdioh->bcmsdh);
+	sdioh_detach(osh, sdioh);
+	osl_detach(osh);
+}
+
+int bcmsdh_register_client_driver(void)
+{
+	return pci_module_init(&bcmsdh_pci_driver);
+}
+
+void bcmsdh_unregister_client_driver(void)
+{
+	pci_unregister_driver(&bcmsdh_pci_driver);
+}
diff --git a/drivers/net/wireless/ap12275/bcmstdlib_s.c b/drivers/net/wireless/ap12275/bcmstdlib_s.c
new file mode 100644
index 000000000000..c540349ac503
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmstdlib_s.c
@@ -0,0 +1,299 @@
+/*
+ * Broadcom Secure Standard Library.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmdefs.h>
+#ifdef BCMDRIVER
+#include <osl.h>
+#else /* BCMDRIVER */
+#include <stddef.h>
+#include <string.h>
+#include <stdlib.h>
+#endif /* else BCMDRIVER */
+
+#include <bcmstdlib_s.h>
+#include <bcmutils.h>
+
+/* Don't use compiler builtins for stdlib APIs within the implementation of the stdlib itself. */
+#if defined(BCM_STDLIB_S_BUILTINS_TEST)
+	#undef memmove_s
+	#undef memcpy_s
+	#undef memset_s
+	#undef strlcpy
+	#undef strlcat_s
+#endif /* BCM_STDLIB_S_BUILTINS_TEST */
+
+/*
+ * __SIZE_MAX__ value is depending on platform:
+ * Firmware Dongle: RAMSIZE (Dongle Specific Limit).
+ * LINUX NIC/Windows/MACOSX/Application: OS Native or
+ * 0xFFFFFFFFu if not defined.
+ */
+#ifndef SIZE_MAX
+#ifndef __SIZE_MAX__
+#ifdef DONGLEBUILD
+#define __SIZE_MAX__ RAMSIZE
+#else
+#define __SIZE_MAX__ 0xFFFFFFFFu
+#endif /* DONGLEBUILD */
+#endif /* __SIZE_MAX__ */
+#define SIZE_MAX __SIZE_MAX__
+#endif /* SIZE_MAX */
+#define RSIZE_MAX (SIZE_MAX >> 1u)
+
+#if !defined(__STDC_WANT_SECURE_LIB__) && !(defined(__STDC_LIB_EXT1__) && \
+	defined(__STDC_WANT_LIB_EXT1__))
+/*
+ * memmove_s - secure memmove
+ * dest : pointer to the object to copy to
+ * destsz : size of the destination buffer
+ * src : pointer to the object to copy from
+ * n : number of bytes to copy
+ * Return Value : zero on success and non-zero on error
+ * Also on error, if dest is not a null pointer and destsz not greater
+ * than RSIZE_MAX, writes destsz zero bytes into the dest object.
+ */
+int
+BCMPOSTTRAPFN(memmove_s)(void *dest, size_t destsz, const void *src, size_t n)
+{
+	int err = BCME_OK;
+
+	if ((!dest) || (((char *)dest + destsz) < (char *)dest)) {
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	if (destsz > RSIZE_MAX) {
+		err = BCME_BADLEN;
+		goto exit;
+	}
+
+	if (destsz < n) {
+		bzero(dest, destsz);
+		err = BCME_BADLEN;
+		goto exit;
+	}
+
+	if ((!src) || (((const char *)src + n) < (const char *)src)) {
+		bzero(dest, destsz);
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	memmove(dest, src, n);
+exit:
+	return err;
+}
+
+/*
+ * memcpy_s - secure memcpy
+ * dest : pointer to the object to copy to
+ * destsz : size of the destination buffer
+ * src : pointer to the object to copy from
+ * n : number of bytes to copy
+ * Return Value : zero on success and non-zero on error
+ * Also on error, if dest is not a null pointer and destsz not greater
+ * than RSIZE_MAX, writes destsz zero bytes into the dest object.
+ */
+int
+BCMPOSTTRAPFN(memcpy_s)(void *dest, size_t destsz, const void *src, size_t n)
+{
+	int err = BCME_OK;
+	char *d = dest;
+	const char *s = src;
+
+	if ((!d) || ((d + destsz) < d)) {
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	if (destsz > RSIZE_MAX) {
+		err = BCME_BADLEN;
+		goto exit;
+	}
+
+	if (destsz < n) {
+		bzero(dest, destsz);
+		err = BCME_BADLEN;
+		goto exit;
+	}
+
+	if ((!s) || ((s + n) < s)) {
+		bzero(dest, destsz);
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	/* overlap checking between dest and src */
+	if (!(((d + destsz) <= s) || (d >= (s + n)))) {
+		bzero(dest, destsz);
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	(void)memcpy(dest, src, n);
+exit:
+	return err;
+}
+
+/*
+ * memset_s - secure memset
+ * dest : pointer to the object to be set
+ * destsz : size of the destination buffer
+ * c : byte value
+ * n : number of bytes to be set
+ * Return Value : zero on success and non-zero on error
+ * Also on error, if dest is not a null pointer and destsz not greater
+ * than RSIZE_MAX, writes destsz bytes with value c into the dest object.
+ */
+int
+BCMPOSTTRAPFN(memset_s)(void *dest, size_t destsz, int c, size_t n)
+{
+	int err = BCME_OK;
+	if ((!dest) || (((char *)dest + destsz) < (char *)dest)) {
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	if (destsz > RSIZE_MAX) {
+		err = BCME_BADLEN;
+		goto exit;
+	}
+
+	if (destsz < n) {
+		(void)memset(dest, c, destsz);
+		err = BCME_BADLEN;
+		goto exit;
+	}
+
+	(void)memset(dest, c, n);
+exit:
+	return err;
+}
+#endif /* !__STDC_WANT_SECURE_LIB__ && !(__STDC_LIB_EXT1__ && __STDC_WANT_LIB_EXT1__) */
+
+#if !defined(FREEBSD) && !defined(MACOSX) && !defined(BCM_USE_PLATFORM_STRLCPY)
+/**
+ * strlcpy - Copy a %NUL terminated string into a sized buffer
+ * @dest: Where to copy the string to
+ * @src: Where to copy the string from
+ * @size: size of destination buffer 0 if input parameters are NOK
+ * return: string leng of src (which is always < size) on success or size on failure
+ *
+ * Compatible with *BSD: the result is always a valid
+ * NUL-terminated string that fits in the buffer (unless,
+ * of course, the buffer size is zero). It does not pad
+ * out the result like strncpy() does.
+ */
+size_t BCMPOSTTRAPFN(strlcpy)(char *dest, const char *src, size_t size)
+{
+	size_t i;
+
+	if (dest == NULL || size == 0) {
+		return 0;
+	}
+
+	if (src == NULL) {
+		*dest = '\0';
+		return 0;
+	}
+
+	for (i = 0; i < size; i++) {
+		dest[i] = src[i];
+		if (dest[i] == '\0') {
+			/* success - src string copied */
+			return i;
+		}
+	}
+
+	/* NULL terminate since not found in src */
+	dest[size - 1u] = '\0';
+
+	/* fail - src string truncated */
+	return size;
+}
+#endif /* !defined(FREEBSD) && !defined(MACOSX) && !defined(BCM_USE_PLATFORM_STRLCPY) */
+
+/**
+ * strlcat_s - Concatenate a %NUL terminated string with a sized buffer
+ * @dest: Where to concatenate the string to
+ * @src: Where to copy the string from
+ * @size: size of destination buffer
+ * return: string length of created string (i.e. the initial length of dest plus the length of src)
+ *         not including the NUL char, up until size
+ *
+ * Unlike strncat(), strlcat() take the full size of the buffer (not just the number of bytes to
+ * copy) and guarantee to NUL-terminate the result (even when there's nothing to concat).
+ * If the length of dest string concatinated with the src string >= size, truncation occurs.
+ *
+ * Compatible with *BSD: the result is always a valid NUL-terminated string that fits in the buffer
+ * (unless, of course, the buffer size is zero).
+ *
+ * If either src or dest is not NUL-terminated, dest[size-1] will be set to NUL.
+ * If size < strlen(dest) + strlen(src), dest[size-1] will be set to NUL.
+ * If size == 0, dest[0] will be set to NUL.
+ */
+size_t
+strlcat_s(char *dest, const char *src, size_t size)
+{
+	char *d = dest;
+	const char *s = src;	/* point to the start of the src string */
+	size_t n = size;
+	size_t dlen;
+	size_t bytes_to_copy = 0;
+
+	if (dest == NULL) {
+		return 0;
+	}
+
+	/* set d to point to the end of dest string (up to size) */
+	while (n != 0 && *d != '\0') {
+		d++;
+		n--;
+	}
+	dlen = (size_t)(d - dest);
+
+	if (s != NULL) {
+		size_t slen = 0;
+
+		/* calculate src len in case it's not null-terminated */
+		n = size;
+		while (n-- != 0 && *(s + slen) != '\0') {
+			++slen;
+		}
+
+		n = size - dlen;	/* maximum num of chars to copy */
+		if (n != 0) {
+			/* copy relevant chars (until end of src buf or given size is reached) */
+			bytes_to_copy = MIN(slen - (size_t)(s - src), n - 1);
+			(void)memcpy(d, s, bytes_to_copy);
+			d += bytes_to_copy;
+		}
+	}
+	if (n == 0 && dlen != 0) {
+		--d;	/* nothing to copy, but NUL-terminate dest anyway */
+	}
+	*d = '\0';	/* NUL-terminate dest */
+
+	return (dlen + bytes_to_copy);
+}
diff --git a/drivers/net/wireless/ap12275/bcmutils.c b/drivers/net/wireless/ap12275/bcmutils.c
new file mode 100644
index 000000000000..cf2c0b8d8dbe
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmutils.c
@@ -0,0 +1,6516 @@
+/*
+ * Driver O/S-independent utility routines
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmdefs.h>
+
+#ifdef BCMDRIVER
+#include <osl.h>
+#include <bcmutils.h>
+#if !defined(BCMDONGLEHOST) || defined(BCMNVRAM)
+#include <bcmnvram.h>
+#endif
+
+#else /* !BCMDRIVER */
+
+#include <stdio.h>
+#include <string.h>
+#include <bcmutils.h>
+#include <stdlib.h>
+
+#if defined(BCMEXTSUP)
+#include <bcm_osl.h>
+#endif
+
+#ifndef ASSERT
+#define ASSERT(exp)
+#endif
+#ifndef ASSERT_FP
+#define ASSERT_FP(exp)
+#endif
+
+#endif /* !BCMDRIVER */
+
+#if defined(_WIN32) || defined(NDIS) || defined(DONGLEBUILD)
+/* Debatable */
+#include <bcmstdlib.h>
+#endif
+#include <bcmstdlib_s.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+#include <ethernet.h>
+#include <vlan.h>
+#include <bcmip.h>
+#include <802.1d.h>
+#include <802.11.h>
+#include <bcmip.h>
+#include <bcmipv6.h>
+#include <bcmtcp.h>
+#ifdef BCMPERFSTATS
+#include <bcmperf.h>
+#endif
+#include <wl_android.h>
+
+#include <event_log_payload.h>
+
+#define NUMBER_OF_BITS_BYTE	8u
+
+/* TX_HISTOGRAM enable/disable state flag
+ * Updated in histogram.c
+ */
+#if defined(TX_HISTOGRAM) && !defined(TX_HISTOGRAM_DISABLED)
+/* initialized to TRUE in tx_histogram_init() */
+bool _tx_histogram_enabled = FALSE;
+#else
+bool _tx_histogram_enabled = FALSE;
+#endif /* TX_HISTOGRAM && !TX_HISTOGRAM_DISABLED */
+
+#ifdef PRIVACY_MASK
+struct ether_addr privacy_addrmask;
+
+/* RAM accessor function to avoid 'privacy_addrmask' in ROM/RAM shared data section. */
+static struct ether_addr *
+BCMRAMFN(privacy_addrmask_get)(void)
+{
+	return &privacy_addrmask;
+}
+#endif /* PRIVACY_MASK */
+
+#ifdef BCMDRIVER
+
+#ifndef BCM_ARM_BACKTRACE
+/* function pointers for firmware stack backtrace utility */
+void (*const BCMPOST_TRAP_RODATA(print_btrace_int_fn))(int depth, uint32 pc, uint32 lr, uint32 sp) =
+	NULL;
+void (*const BCMPOST_TRAP_RODATA(print_btrace_fn))(int depth) = NULL;
+#else
+void print_backtrace(int depth);
+void (*const BCMPOST_TRAP_RODATA(print_btrace_fn))(int depth) = print_backtrace;
+void print_backtrace_int(int depth, uint32 pc, uint32 lr, uint32 sp);
+void (*const BCMPOST_TRAP_RODATA(print_btrace_int_fn))(int depth, uint32 pc, uint32 lr, uint32 sp) =
+	print_backtrace_int;
+#endif
+
+#if !defined(BCMDONGLEHOST)
+
+#ifndef BCM_BOOTLOADER
+/* Forward declarations */
+static int getintvararray_internal(char *vars, const char *name, int index);
+static int getintvararraysize_internal(char *vars, const char *name);
+static
+#ifndef ATE_BUILD
+const
+#endif
+char * getvar_internal(char *vars, const char *name);
+static int getintvar_internal(char *vars, const char *name);
+
+/*
+ * Search the name=value vars for a specific one and return its value.
+ * Returns NULL if not found.
+ */
+#ifndef ATE_BUILD
+const
+#endif
+char *
+getvar(char *vars, const char *name)
+{
+	NVRAM_RECLAIM_CHECK(name);
+	return getvar_internal(vars, name);
+}
+
+static
+#ifndef ATE_BUILD
+const
+#endif
+char *
+getvar_internal(char *vars, const char *name)
+{
+	char *s;
+	uint len;
+
+	if (!name)
+		return NULL;
+
+	len = strlen(name);
+	if (len == 0u) {
+		return NULL;
+	}
+
+	/* first look in vars[] */
+	for (s = vars; s && *s;) {
+		if ((bcmp(s, name, len) == 0) && (s[len] == '=')) {
+			return (&s[len+1u]);
+		}
+		while (*s++)
+			;
+	}
+
+	/* then query nvram */
+	return (nvram_get(name));
+}
+
+/*
+ * Search the vars for a specific one and return its value as
+ * an integer. Returns 0 if not found.
+ */
+int
+getintvar(char *vars, const char *name)
+{
+	NVRAM_RECLAIM_CHECK(name);
+	return getintvar_internal(vars, name);
+}
+
+static int
+getintvar_internal(char *vars, const char *name)
+{
+	const char *val;
+
+	if ((val = getvar_internal(vars, name)) == NULL)
+		return (0);
+
+	return (int)(bcm_strtoul(val, NULL, 0));
+}
+
+int
+getintvararray(char *vars, const char *name, int index)
+{
+	NVRAM_RECLAIM_CHECK(name);
+	return getintvararray_internal(vars, name, index);
+}
+
+static int
+getintvararray_internal(char *vars, const char *name, int index)
+{
+	const char *buf;
+	char *endp;
+	int i = 0;
+	int val = 0;
+
+	if ((buf = getvar_internal(vars, name)) == NULL) {
+		return (0);
+	}
+
+	/* table values are always separated by "," or " " */
+	while (*buf != '\0') {
+		val = (int)bcm_strtoul(buf, &endp, 0);
+		if (i == index) {
+			return val;
+		}
+		buf = endp;
+		/* delimiter is ',' */
+		if (*buf == ',')
+			buf++;
+		i++;
+	}
+	return (0);
+}
+
+int
+getintvararraysize(char *vars, const char *name)
+{
+	NVRAM_RECLAIM_CHECK(name);
+	return getintvararraysize_internal(vars, name);
+}
+
+static int
+getintvararraysize_internal(char *vars, const char *name)
+{
+	const char *buf;
+	char *endp;
+	int count = 0;
+	int val = 0;
+
+	if ((buf = getvar_internal(vars, name)) == NULL) {
+		return (0);
+	}
+
+	/* table values are always separated by "," or " " */
+	while (*buf != '\0') {
+		val = (int)bcm_strtoul(buf, &endp, 0);
+		buf = endp;
+		/* delimiter is ',' */
+		if (*buf == ',')
+			buf++;
+		count++;
+	}
+	BCM_REFERENCE(val);
+	return count;
+}
+
+/* Read an array of values from a possibly slice-specific nvram string
+ * Store the values in either the uint8 dest_array1 or in the int16 dest_array2.
+ * Pass in NULL for the dest_array[12] that is not to be used.
+ */
+static int
+getintvararray_slicespecific(osl_t *osh, char *vars, char *vars_table_accessor,
+	const char* name, uint8* dest_array1, int16* dest_array2, uint dest_size)
+{
+	uint i;
+	uint array_size = 0;
+	int err = BCME_OK;
+	uint prefixed_name_sz;
+	char *prefixed_name = NULL;
+	const char *new_name;
+	int val;
+
+	prefixed_name_sz = get_slicespecific_var_name(osh, vars_table_accessor,
+		name, &prefixed_name);
+	if (prefixed_name_sz == 0) {
+		return BCME_NOMEM;
+	}
+
+	new_name = prefixed_name;
+	(void) new_name;
+	if (getvar(vars, new_name) == NULL) {
+		/* Try again without the slice prefix in the name */
+		new_name = name;
+		if (getvar(vars, name) == NULL) {
+			err = BCME_NOTFOUND;
+			goto done;
+		}
+	}
+
+	array_size = (uint)getintvararraysize(vars, new_name);
+	if (array_size > dest_size) {
+		err = BCME_BUFTOOSHORT;
+		ASSERT(array_size <= dest_size);
+		goto done;
+	}
+
+	/* limit the initialization to the size of the nvram array */
+	array_size = MIN(array_size, dest_size);
+
+	/* load the destination array with the nvram array values */
+	for (i = 0; i < array_size; i++) {
+		val = getintvararray(vars, new_name, (int)i);
+		if (dest_array1) {
+			dest_array1[i] = (uint8)val;
+		} else if (dest_array2) {
+			dest_array2[i] = (int16)val;
+		}
+	}
+done:
+	MFREE(osh, prefixed_name, prefixed_name_sz);
+	return (err < 0) ? err : (int)array_size;
+}
+
+int
+get_uint8_vararray_slicespecific(osl_t *osh, char *vars, char *vars_table_accessor,
+	const char* name, uint8* dest_array, uint dest_size)
+{
+	int ret;
+
+	ret = getintvararray_slicespecific(osh, vars, vars_table_accessor,
+		name, dest_array, NULL, dest_size);
+	return ret;
+}
+
+int
+get_int16_vararray_slicespecific(osl_t *osh, char *vars, char *vars_table_accessor,
+	const char* name, int16* dest_array, uint dest_size)
+{
+	return getintvararray_slicespecific(osh, vars, vars_table_accessor,
+		name, NULL, dest_array, dest_size);
+}
+
+/* Prepend a slice-specific accessor to an nvram string name.
+ * Sets name_out to the allocated string. Returns the allocated size of the name string.
+ * Caller is responsible for freeing the resulting name string with MFREE.
+ */
+uint
+get_slicespecific_var_name(osl_t *osh, char *vars_table_accessor, const char *name,
+	char **name_out)
+{
+	char *name_with_prefix = NULL;
+	uint sz;
+	uint max_copy_size;
+
+	sz = strlen(name) + strlen(vars_table_accessor) + 1;
+	name_with_prefix = (char *) MALLOC_NOPERSIST(osh, sz);
+	if (name_with_prefix == NULL) {
+		sz = 0;
+		goto end;
+	}
+	name_with_prefix[0] = 0;
+	name_with_prefix[sz - 1] = 0;
+	max_copy_size = sz - 1;
+
+	/* if accessor contains a "slice/N/" string */
+	if (vars_table_accessor[0] != 0) {
+		/* prepend accessor to the vars-name */
+		bcmstrncat(name_with_prefix, vars_table_accessor, max_copy_size);
+		max_copy_size -= strlen(name_with_prefix);
+	}
+
+	/* Append vars-name */
+	bcmstrncat(name_with_prefix, name, max_copy_size);
+end:
+	*name_out = name_with_prefix;
+	return sz;
+}
+#endif /* BCM_BOOTLOADER */
+
+#if defined(BCMNVRAMR) || defined(BCMNVRAMW)
+/* Search for token in comma separated token-string */
+static int
+findmatch(const char *string, const char *name)
+{
+	uint len;
+	char *c;
+
+	len = strlen(name);
+	while ((c = strchr(string, ',')) != NULL) {
+		if (len == (uint)(c - string) && !strncmp(string, name, len))
+			return 1;
+		string = c + 1;
+	}
+
+	return (!strcmp(string, name));
+}
+
+/* Return gpio pin number assigned to the named pin
+ *
+ * Variable should be in format:
+ *
+ *	gpio<N>=pin_name,pin_name
+ *
+ * This format allows multiple features to share the gpio with mutual
+ * understanding.
+ *
+ * 'def_pin' is returned if a specific gpio is not defined for the requested functionality
+ * and if def_pin is not used by others.
+ */
+uint
+getgpiopin(char *vars, char *pin_name, uint def_pin)
+{
+	char name[] = "gpioXXXX";
+	const char *val;
+	uint pin;
+
+	/* Go thru all possibilities till a match in pin name */
+	for (pin = 0; pin < GPIO_NUMPINS; pin ++) {
+		snprintf(name, sizeof(name), "gpio%d", pin);
+		val = getvar(vars, name);
+		if (val && findmatch(val, pin_name))
+			return pin;
+	}
+
+	if (def_pin != GPIO_PIN_NOTDEFINED) {
+		/* make sure the default pin is not used by someone else */
+		snprintf(name, sizeof(name), "gpio%d", def_pin);
+		if (getvar(vars, name)) {
+			def_pin =  GPIO_PIN_NOTDEFINED;
+		}
+	}
+	return def_pin;
+}
+#endif /* BCMNVRAMR || BCMNVRAMW */
+#endif /* !BCMDONGLEHOST */
+
+/* return total length of buffer chain. In case of CSO, submsdu may have extra tsohdr and if
+ * pktotlen should not include submsdu tso header, use the API pkttotlen_no_sfhtoe_hdr.
+ */
+uint
+BCMFASTPATH(pkttotlen)(osl_t *osh, void *p)
+{
+	uint total = 0;
+
+	for (; p; p = PKTNEXT(osh, p)) {
+		total += PKTLEN(osh, p);
+
+		if (BCMLFRAG_ENAB() && PKTISFRAG(osh, p)) {
+			total += PKTFRAGTOTLEN(osh, p);
+		}
+	}
+
+	return (total);
+}
+
+#ifdef WLCSO
+/* return total length of buffer chain, but exclude tso hdr of submsdu if its added  */
+uint
+BCMFASTPATH(pkttotlen_no_sfhtoe_hdr)(osl_t *osh, void *p, uint toe_hdr_len)
+{
+	uint total = 0;
+
+	for (; p; p = PKTNEXT(osh, p)) {
+		total += PKTLEN(osh, p);
+
+		/* exclude toe_hdr_len if its part of PKTLEN()  */
+		if (PKTISSUBMSDUTOEHDR(osh, p)) {
+			total -= toe_hdr_len;
+		}
+
+		if (BCMLFRAG_ENAB() && PKTISFRAG(osh, p)) {
+			total += PKTFRAGTOTLEN(osh, p);
+		}
+	}
+
+	return (total);
+}
+#endif /* WLCSO */
+
+/* return total length of buffer chain */
+uint
+BCMFASTPATH(pkttotcnt)(osl_t *osh, void *p)
+{
+	uint cnt = 0;
+
+	for (; p; p = PKTNEXT(osh, p)) {
+		cnt++;
+	}
+
+	return (cnt);
+}
+
+/* return the last buffer of chained pkt */
+void *
+BCMFASTPATH(pktlast)(osl_t *osh, void *p)
+{
+	for (; PKTNEXT(osh, p); p = PKTNEXT(osh, p))
+		;
+
+	return (p);
+}
+
+/* count segments of a chained packet */
+uint
+BCMFASTPATH(pktsegcnt)(osl_t *osh, void *p)
+{
+	uint cnt;
+
+	for (cnt = 0; p; p = PKTNEXT(osh, p)) {
+		if (PKTLEN(osh, p)) {
+			cnt++;
+		}
+#ifdef BCMLFRAG
+		if (BCMLFRAG_ENAB() && PKTISFRAG(osh, p)) {
+				cnt += PKTFRAGTOTNUM(osh, p);
+		}
+#endif /* BCMLFRAG */
+	}
+
+	return cnt;
+}
+
+#ifdef DONGLEBUILD
+/**
+ * Takes in a lbuf/lfrag and no of bytes to be trimmed from tail.
+ * trim bytes  could be spread out in below 3 formats
+ *     1. entirely in dongle
+ *     2. entirely in host
+ *     3. split between host-dongle
+ */
+void
+BCMFASTPATH(pktfrag_trim_tailbytes)(osl_t * osh, void* p, uint16 trim_len, uint8 type)
+{
+	uint tcmseg_len = PKTLEN(osh, p);	/* TCM segment length */
+	uint hostseg_len = PKTFRAGUSEDLEN(osh, p);	/* HOST segment length */
+
+	/* return if zero trim length- Nothing to do */
+	if (trim_len == 0)
+		return;
+
+	/* if header conv is on, there is no fcs at the end */
+	/* JIRA:SW4349-318 */
+	if (PKTISHDRCONVTD(osh, p))
+		return;
+
+	/* if pktfetched, then its already trimmed */
+	if (PKTISPKTFETCHED(osh, p))
+		return;
+
+	if (PKTFRAGUSEDLEN(osh, p) >= trim_len) {
+		/* TRIM bytes entirely in host */
+		ASSERT_FP(PKTISRXFRAG(osh, p));
+
+		PKTSETFRAGUSEDLEN(osh, p, (hostseg_len - trim_len));
+	} else {
+		/* trim bytes either in dongle or split between dongle-host */
+		PKTSETLEN(osh, p, (tcmseg_len - (trim_len - hostseg_len)));
+
+		/* No more contents in host; reset length to zero */
+		if (PKTFRAGUSEDLEN(osh, p))
+			PKTSETFRAGUSEDLEN(osh, p, 0);
+	}
+}
+#endif /* DONGLEBUILD */
+
+/* copy a pkt buffer chain into a buffer */
+uint
+BCMPOSTTRAPFN(pktcopy)(osl_t *osh, void *p, uint offset, uint len, uchar *buf)
+{
+	uint n, ret = 0;
+
+	/* skip 'offset' bytes */
+	for (; p && offset; p = PKTNEXT(osh, p)) {
+		if (offset < PKTLEN(osh, p))
+			break;
+		offset -= PKTLEN(osh, p);
+	}
+
+	if (!p)
+		return 0;
+
+	/* copy the data */
+	for (; p && len; p = PKTNEXT(osh, p)) {
+		n = MIN(PKTLEN(osh, p) - offset, len);
+		bcopy(PKTDATA(osh, p) + offset, buf, n);
+		buf += n;
+		len -= n;
+		ret += n;
+		offset = 0;
+	}
+
+	return ret;
+}
+
+/* copy a buffer into a pkt buffer chain */
+uint
+pktfrombuf(osl_t *osh, void *p, uint offset, uint len, uchar *buf)
+{
+	uint n, ret = 0;
+
+	/* skip 'offset' bytes */
+	for (; p && offset; p = PKTNEXT(osh, p)) {
+		if (offset < PKTLEN(osh, p))
+			break;
+		offset -= PKTLEN(osh, p);
+	}
+
+	if (!p)
+		return 0;
+
+	/* copy the data */
+	for (; p && len; p = PKTNEXT(osh, p)) {
+		n = MIN(PKTLEN(osh, p) - offset, len);
+		bcopy(buf, PKTDATA(osh, p) + offset, n);
+		buf += n;
+		len -= n;
+		ret += n;
+		offset = 0;
+	}
+
+	return ret;
+}
+
+uint8 *
+BCMFASTPATH(pktdataoffset)(osl_t *osh, void *p,  uint offset)
+{
+	uint total = pkttotlen(osh, p);
+	uint pkt_off = 0, len = 0;
+	uint8 *pdata = (uint8 *) PKTDATA(osh, p);
+
+	if (offset > total)
+		return NULL;
+
+	for (; p; p = PKTNEXT(osh, p)) {
+		pdata = (uint8 *) PKTDATA(osh, p);
+		pkt_off = offset - len;
+		len += PKTLEN(osh, p);
+		if (len > offset)
+			break;
+	}
+	return (uint8*) (pdata+pkt_off);
+}
+
+/* given a offset in pdata, find the pkt seg hdr */
+void *
+pktoffset(osl_t *osh, void *p,  uint offset)
+{
+	uint total = pkttotlen(osh, p);
+	uint len = 0;
+
+	if (offset > total)
+		return NULL;
+
+	for (; p; p = PKTNEXT(osh, p)) {
+		len += PKTLEN(osh, p);
+		if (len > offset)
+			break;
+	}
+	return p;
+}
+
+void
+bcm_mdelay(uint ms)
+{
+	uint i;
+
+	for (i = 0; i < ms; i++) {
+		OSL_DELAY(1000);
+	}
+}
+
+#if defined(BCMPERFSTATS) || defined(BCMTSTAMPEDLOGS)
+
+#if defined(__ARM_ARCH_7R__)
+#define BCMLOG_CYCLE_OVERHEAD	54	/* Number of CPU cycle overhead due to bcmlog().
+					 * This is to compensate CPU cycle incurred by
+					 * added bcmlog() function call for profiling.
+					 */
+#else
+#define BCMLOG_CYCLE_OVERHEAD	0
+#endif
+
+#define	LOGSIZE	256			/* should be power of 2 to avoid div below */
+static struct {
+	uint	cycles;
+	const char	*fmt;
+	uint	a1;
+	uint	a2;
+	uchar   indent;		/* track indent level for nice printing */
+} logtab[LOGSIZE];
+
+/* last entry logged  */
+static uint logi = 0;
+/* next entry to read */
+static uint volatile readi = 0;
+#endif	/* defined(BCMPERFSTATS) || defined(BCMTSTAMPEDLOGS) */
+
+#ifdef BCMPERFSTATS
+/* TODO: make the utility configurable (choose between icache, dcache, hits, misses ...)  */
+void
+bcm_perf_enable()
+{
+	BCMPERF_ENABLE_INSTRCOUNT();
+	BCMPERF_ENABLE_ICACHE_MISS();
+	BCMPERF_ENABLE_ICACHE_HIT();
+}
+
+/* WARNING:  This routine uses OSL_GETCYCLES(), which can give unexpected results on
+ * modern speed stepping CPUs.  Use bcmtslog() instead in combination with TSF counter.
+ */
+void
+bcmlog(char *fmt, uint a1, uint a2)
+{
+	static uint last = 0;
+	uint cycles, i, elapsed;
+	OSL_GETCYCLES(cycles);
+
+	i = logi;
+
+	elapsed = cycles - last;
+	if (elapsed > BCMLOG_CYCLE_OVERHEAD)
+		logtab[i].cycles = elapsed - BCMLOG_CYCLE_OVERHEAD;
+	else
+		logtab[i].cycles = 0;
+	logtab[i].fmt = fmt;
+	logtab[i].a1 = a1;
+	logtab[i].a2 = a2;
+
+	logi = (i + 1) % LOGSIZE;
+	last = cycles;
+
+	/* if log buffer is overflowing, readi should be advanced.
+	 * Otherwise logi and readi will become out of sync.
+	 */
+	if (logi == readi) {
+		readi = (readi + 1) % LOGSIZE;
+	} else {
+		/* This redundant else is to make CPU cycles of bcmlog() function to be uniform,
+		 * so that the cycle compensation with BCMLOG_CYCLE_OVERHEAD is more accurate.
+		 */
+		readi = readi % LOGSIZE;
+	}
+}
+
+/* Same as bcmlog but specializes the use of a1 and a2 to
+ * store icache misses and instruction count.
+ *  TODO : make this use a configuration array to decide what counter to read.
+ * We are limited to 2 numbers but it seems it is the most we can get anyway
+ * since dcache and icache cannot be enabled at the same time. Recording
+ * both the hits and misses at the same time for a given cache is not that useful either.
+*/
+
+void
+bcmstats(char *fmt)
+{
+	static uint last = 0;
+	static uint32 ic_miss = 0;
+	static uint32 instr_count = 0;
+	uint32 ic_miss_cur;
+	uint32 instr_count_cur;
+	uint cycles, i;
+
+	OSL_GETCYCLES(cycles);
+	BCMPERF_GETICACHE_MISS(ic_miss_cur);
+	BCMPERF_GETINSTRCOUNT(instr_count_cur);
+
+	i = logi;
+
+	logtab[i].cycles = cycles - last;
+	logtab[i].a1 = ic_miss_cur - ic_miss;
+	logtab[i].a2 = instr_count_cur - instr_count;
+	logtab[i].fmt = fmt;
+
+	logi = (i + 1) % LOGSIZE;
+
+	last = cycles;
+	instr_count = instr_count_cur;
+	ic_miss = ic_miss_cur;
+
+	/* if log buffer is overflowing, readi should be advanced.
+	 * Otherwise logi and readi will become out of sync.
+	 */
+	if (logi == readi) {
+		readi = (readi + 1) % LOGSIZE;
+	} else {
+		/* This redundant else is to make CPU cycles of bcmstats() function to be uniform
+		 */
+		readi = readi % LOGSIZE;
+	}
+}
+
+/*
+ * TODO (linux version): a "proc" version where the log would be dumped
+ * on the proc file directly.
+ */
+
+void
+bcmdumplog(char *buf, int size)
+{
+	char *limit;
+	int j = 0;
+	int num;
+	struct bcmstrbuf strubuf;
+	struct bcmstrbuf *b = &strubuf;
+
+	bcm_binit(b, buf, size);
+
+	limit = BCMSTRBUF_BUF(b) + BCMSTRBUF_LEN(b) - 80;
+
+	num = logi - readi;
+
+	if (num < 0)
+		num += LOGSIZE;
+
+	/* print in chronological order */
+
+	for (j = 0; j < num && (BCMSTRBUF_BUF(b) < limit); readi = (readi + 1) % LOGSIZE, j++) {
+		if (logtab[readi].fmt == NULL)
+		    continue;
+		bcm_bprintf(b, "%d\t", logtab[readi].cycles);
+		bcm_bprintf(b, logtab[readi].fmt, logtab[readi].a1, logtab[readi].a2);
+		bcm_bprintf(b, "\n");
+	}
+
+}
+
+/*
+ * Dump one log entry at a time.
+ * Return index of next entry or -1 when no more .
+ */
+int
+bcmdumplogent(char *buf, uint i)
+{
+	bool hit;
+	struct bcmstrbuf strubuf;
+	struct bcmstrbuf *b = &strubuf;
+
+	/*
+	 * If buf is NULL, return the starting index,
+	 * interpreting i as the indicator of last 'i' entries to dump.
+	 */
+	if (buf == NULL) {
+		i = ((i > 0) && (i < (LOGSIZE - 1))) ? i : (LOGSIZE - 1);
+		return ((logi - i) % LOGSIZE);
+	}
+
+	*buf = '\0';
+
+	ASSERT(i < LOGSIZE);
+
+	if (i == logi)
+		return (-1);
+
+	bcm_binit(b, buf, i);
+	hit = FALSE;
+	for (; (i != logi) && !hit; i = (i + 1) % LOGSIZE) {
+		if (logtab[i].fmt == NULL)
+			continue;
+		bcm_bprintf(b, "%d: %d\t", i, logtab[i].cycles);
+		bcm_bprintf(b, logtab[i].fmt, logtab[i].a1, logtab[i].a2);
+		bcm_bprintf(b, "\n");
+		hit = TRUE;
+	}
+
+	return (i);
+}
+
+#endif	/* BCMPERFSTATS */
+
+#if defined(BCMTSTAMPEDLOGS)
+/* Store a TSF timestamp and a log line in the log buffer */
+/*
+	a1 is used to signify entering/exiting a routine.  When entering
+	the indent level is increased.  When exiting, the delta since entering
+	is printed and the indent level is bumped back out.
+	Nesting can go up to level MAX_TS_INDENTS deep.
+*/
+#define MAX_TS_INDENTS 20
+void
+bcmtslog(uint32 tstamp, const char *fmt, uint a1, uint a2)
+{
+	uint i = logi;
+	bool use_delta = TRUE;
+	static uint32 last = 0;	/* used only when use_delta is true */
+	static uchar indent = 0;
+	static uint32 indents[MAX_TS_INDENTS];
+
+	logtab[i].cycles = tstamp;
+	if (use_delta)
+		logtab[i].cycles -= last;
+
+	logtab[i].a2 = a2;
+
+	if (a1 == TS_EXIT && indent) {
+		indent--;
+		logtab[i].a2 = tstamp - indents[indent];
+	}
+
+	logtab[i].fmt = fmt;
+	logtab[i].a1 = a1;
+	logtab[i].indent = indent;
+
+	if (a1 == TS_ENTER) {
+		indents[indent] = tstamp;
+		if (indent < MAX_TS_INDENTS - 1)
+			indent++;
+	}
+
+	if (use_delta)
+		last = tstamp;
+	logi = (i + 1) % LOGSIZE;
+}
+
+/* Print out a microsecond timestamp as "sec.ms.us " */
+void
+bcmprinttstamp(uint32 ticks)
+{
+	uint us, ms, sec;
+
+	us = (ticks % TSF_TICKS_PER_MS) * 1000 / TSF_TICKS_PER_MS;
+	ms = ticks / TSF_TICKS_PER_MS;
+	sec = ms / 1000;
+	ms -= sec * 1000;
+	printf("%04u.%03u.%03u ", sec, ms, us);
+}
+
+/* Print out the log buffer with timestamps */
+void
+bcmprinttslogs(void)
+{
+	int j = 0;
+	int num;
+
+	num = logi - readi;
+	if (num < 0)
+		num += LOGSIZE;
+
+	/* Format and print the log entries directly in chronological order */
+	for (j = 0; j < num; readi = (readi + 1) % LOGSIZE, j++) {
+		if (logtab[readi].fmt == NULL)
+		    continue;
+		bcmprinttstamp(logtab[readi].cycles);
+		printf(logtab[readi].fmt, logtab[readi].a1, logtab[readi].a2);
+		printf("\n");
+	}
+}
+
+/*
+	Identical to bcmdumplog, but output is based on tsf instead of cycles.
+
+	a1 is used to signify entering/exiting a routine.  When entering
+	the indent level is increased.  When exiting, the delta since entering
+	is printed and the indent level is bumped back out.
+*/
+void
+bcmdumptslog(struct bcmstrbuf *b)
+{
+	char *limit;
+	int j = 0;
+	int num;
+	uint us, ms, sec;
+	int skip;
+	char *lines = "| | | | | | | | | | | | | | | | | | | |";
+
+	limit = BCMSTRBUF_BUF(b) + BCMSTRBUF_LEN(b) - 80;
+
+	num = logi - readi;
+
+	if (num < 0)
+		num += LOGSIZE;
+
+	/* print in chronological order */
+	for (j = 0; j < num && (BCMSTRBUF_BUF(b) < limit); readi = (readi + 1) % LOGSIZE, j++) {
+		char *last_buf = BCMSTRBUF_BUF(b);
+		if (logtab[readi].fmt == NULL)
+			continue;
+
+		us = (logtab[readi].cycles % TSF_TICKS_PER_MS) * 1000 / TSF_TICKS_PER_MS;
+		ms = logtab[readi].cycles / TSF_TICKS_PER_MS;
+		sec = ms / 1000;
+		ms -= sec * 1000;
+
+		bcm_bprintf(b, "%04u.%03u.%03u ", sec, ms, us);
+
+		/* 2 spaces for each indent level */
+		bcm_bprintf(b, "%.*s", logtab[readi].indent * 2, lines);
+
+		/*
+		 * The following call to snprintf generates a compiler warning
+		 * due to -Wformat-security. However, the format string is coming
+		 * from internal callers rather than external data input, and is a
+		 * useful debugging tool serving a variety of diagnostics. Rather
+		 * than expand code size by replicating multiple functions with different
+		 * argument lists, or disabling the warning globally, let's consider
+		 * if we can just disable the warning for this one instance.
+		 */
+		bcm_bprintf(b, logtab[readi].fmt);
+
+		/* If a1 is ENTER or EXIT, print the + or - */
+		skip = 0;
+		if (logtab[readi].a1 == TS_ENTER) {
+			bcm_bprintf(b, " +");
+			skip++;
+		}
+		if (logtab[readi].a1 == TS_EXIT) {
+			bcm_bprintf(b, " -");
+			skip++;
+		}
+
+		/* else print the real a1 */
+		if (logtab[readi].a1 && !skip)
+			bcm_bprintf(b, "   %d", logtab[readi].a1);
+
+		/*
+		   If exiting routine, print a nicely formatted delta since entering.
+		   Otherwise, just print a2 normally.
+		*/
+		if (logtab[readi].a2) {
+			if (logtab[readi].a1 == TS_EXIT) {
+				int num_space = 75 - (BCMSTRBUF_BUF(b) - last_buf);
+				bcm_bprintf(b, "%*.s", num_space, "");
+				bcm_bprintf(b, "%5d usecs", logtab[readi].a2);
+			} else
+				bcm_bprintf(b, "  %d", logtab[readi].a2);
+		}
+		bcm_bprintf(b, "\n");
+		last_buf = BCMSTRBUF_BUF(b);
+	}
+}
+
+#endif	/* BCMTSTAMPEDLOGS */
+
+#if defined(BCMDBG) || defined(DHD_DEBUG)
+/* pretty hex print a pkt buffer chain */
+void
+prpkt(const char *msg, osl_t *osh, void *p0)
+{
+	void *p;
+
+	if (msg && (msg[0] != '\0'))
+		printf("%s:\n", msg);
+
+	for (p = p0; p; p = PKTNEXT(osh, p))
+		prhex(NULL, PKTDATA(osh, p), PKTLEN(osh, p));
+}
+#endif	/* BCMDBG || DHD_DEBUG */
+
+/* Takes an Ethernet frame and sets out-of-bound PKTPRIO.
+ * Also updates the inplace vlan tag if requested.
+ * For debugging, it returns an indication of what it did.
+ */
+uint
+BCMFASTPATH(pktsetprio)(void *pkt, bool update_vtag)
+{
+	struct ether_header *eh;
+	struct ethervlan_header *evh;
+	uint8 *pktdata;
+	uint priority = 0;
+	uint rc = 0;
+
+	pktdata = (uint8 *)PKTDATA(OSH_NULL, pkt);
+	ASSERT_FP(ISALIGNED((uintptr)pktdata, sizeof(uint16)));
+
+	eh = (struct ether_header *) pktdata;
+
+	if (eh->ether_type == hton16(ETHER_TYPE_8021Q)) {
+		uint16 vlan_tag;
+		uint vlan_prio, dscp_prio = 0;
+
+		evh = (struct ethervlan_header *)eh;
+
+		vlan_tag = ntoh16(evh->vlan_tag);
+		vlan_prio = (vlan_tag >> VLAN_PRI_SHIFT) & VLAN_PRI_MASK;
+
+		if ((evh->ether_type == hton16(ETHER_TYPE_IP)) ||
+			(evh->ether_type == hton16(ETHER_TYPE_IPV6))) {
+			uint8 *ip_body = pktdata + sizeof(struct ethervlan_header);
+			uint8 tos_tc = IP_TOS46(ip_body);
+			dscp_prio = tos_tc >> IPV4_TOS_PREC_SHIFT;
+		}
+
+		/* DSCP priority gets precedence over 802.1P (vlan tag) */
+		if (dscp_prio != 0) {
+			priority = dscp_prio;
+			rc |= PKTPRIO_VDSCP;
+		} else {
+			priority = vlan_prio;
+			rc |= PKTPRIO_VLAN;
+		}
+		/*
+		 * If the DSCP priority is not the same as the VLAN priority,
+		 * then overwrite the priority field in the vlan tag, with the
+		 * DSCP priority value. This is required for Linux APs because
+		 * the VLAN driver on Linux, overwrites the skb->priority field
+		 * with the priority value in the vlan tag
+		 */
+		if (update_vtag && (priority != vlan_prio)) {
+			vlan_tag &= ~(VLAN_PRI_MASK << VLAN_PRI_SHIFT);
+			vlan_tag |= priority << VLAN_PRI_SHIFT;
+			evh->vlan_tag = hton16(vlan_tag);
+			rc |= PKTPRIO_UPD;
+		}
+#if defined(EAPOL_PKT_PRIO) || defined(DHD_LOSSLESS_ROAMING)
+	} else if (eh->ether_type == hton16(ETHER_TYPE_802_1X)) {
+		priority = PRIO_8021D_NC;
+		rc = PKTPRIO_DSCP;
+#endif /* EAPOL_PKT_PRIO || DHD_LOSSLESS_ROAMING */
+#if defined(PRIORITIZE_ARP)
+	} else if (eh->ether_type == hton16(ETHER_TYPE_ARP)) {
+		/* Certain Host stacks attempt disconnection on continous ARP timeouts. In
+		 * congested scenarios with traffic, ARP packets may not get chance
+		 * for transmission leading to disconnection. so prioritize it.
+		 */
+		priority = PRIO_8021D_NC;
+		rc = PKTPRIO_DSCP;
+#endif /* PRIORITIZE_ARP */
+#if defined(WLTDLS)
+	} else if (eh->ether_type == hton16(ETHER_TYPE_89_0D)) {
+		/* Bump up the priority for TDLS frames */
+		priority = PRIO_8021D_VI;
+		rc = PKTPRIO_DSCP;
+#endif /* WLTDLS */
+	} else if ((eh->ether_type == hton16(ETHER_TYPE_IP)) ||
+		(eh->ether_type == hton16(ETHER_TYPE_IPV6))) {
+		uint8 *ip_body = pktdata + sizeof(struct ether_header);
+		uint8 tos_tc = IP_TOS46(ip_body);
+		uint8 dscp = tos_tc >> IPV4_TOS_DSCP_SHIFT;
+		switch (dscp) {
+		case DSCP_EF:
+		case DSCP_VA:
+			priority = PRIO_8021D_VO;
+			break;
+		case DSCP_AF31:
+		case DSCP_AF32:
+		case DSCP_AF33:
+		case DSCP_CS3:
+			priority = PRIO_8021D_CL;
+			break;
+		case DSCP_AF21:
+		case DSCP_AF22:
+		case DSCP_AF23:
+			priority = PRIO_8021D_EE;
+			break;
+		case DSCP_AF11:
+		case DSCP_AF12:
+		case DSCP_AF13:
+		case DSCP_CS2:
+			priority = PRIO_8021D_BE;
+			break;
+#ifdef RFC8325_DSCP_CS6_TO_UP7
+		case DSCP_CS6:
+#endif /* RFC8325_DSCP_CS6_TO_UP7 */
+		case DSCP_CS7:
+			priority = PRIO_8021D_NC;
+			break;
+		default:
+			priority = tos_tc >> IPV4_TOS_PREC_SHIFT;
+			break;
+		}
+
+		rc |= PKTPRIO_DSCP;
+	}
+
+	ASSERT_FP(priority <= MAXPRIO);
+	PKTSETPRIO(pkt, priority);
+	return (rc | priority);
+}
+
+/* lookup user priority for specified DSCP */
+static uint8
+dscp2up(uint8 *up_table, uint8 dscp)
+{
+	uint8 user_priority = 255;
+
+	/* lookup up from table if parameters valid */
+	if (up_table != NULL && dscp < UP_TABLE_MAX) {
+		user_priority = up_table[dscp];
+	}
+
+	if (user_priority == 255) {	/* unknown user priority */
+
+		/* If the user_priority from the QoS Map table is unknown(i.e., 255), then
+		 * set the default user priority as PRIO_8021D_BE(0); Reference RFC 8325.
+		 */
+		user_priority = PRIO_8021D_BE; /* default priority */
+	}
+
+	return user_priority;
+}
+
+/* set user priority by QoS Map Set table (UP table), table size is UP_TABLE_MAX */
+uint
+BCMFASTPATH(pktsetprio_qms)(void *pkt, uint8* up_table, bool update_vtag)
+{
+	if (up_table) {
+		uint8 *pktdata;
+		uint pktlen;
+		uint8 dscp;
+		uint user_priority = 0;
+		uint rc = 0;
+
+		pktdata = (uint8 *)PKTDATA(OSH_NULL, pkt);
+		pktlen = PKTLEN(OSH_NULL, pkt);
+		if (pktgetdscp(pktdata, pktlen, &dscp)) {
+			rc = PKTPRIO_DSCP;
+			user_priority = dscp2up(up_table, dscp);
+			PKTSETPRIO(pkt, user_priority);
+		}
+#ifdef WL_CUSTOM_MAPPING_OF_DSCP
+		else {
+			return pktsetprio(pkt, update_vtag);
+		}
+#endif /* WL_CUSTOM_MAPPING_OF_DSCP */
+
+		return (rc | user_priority);
+	} else {
+		return pktsetprio(pkt, update_vtag);
+	}
+}
+
+/* Returns TRUE and DSCP if IP header found, FALSE otherwise.
+ */
+bool
+BCMFASTPATH(pktgetdscp)(uint8 *pktdata, uint pktlen, uint8 *dscp)
+{
+	struct ether_header *eh;
+	struct ethervlan_header *evh;
+	uint8 *ip_body;
+	bool rc = FALSE;
+
+	/* minimum length is ether header and IP header */
+	if (pktlen < (sizeof(struct ether_header) + IPV4_MIN_HEADER_LEN)) {
+		return FALSE;
+	}
+
+	eh = (struct ether_header *) pktdata;
+
+	if ((eh->ether_type == HTON16(ETHER_TYPE_IP)) ||
+		(eh->ether_type == HTON16(ETHER_TYPE_IPV6))) {
+		ip_body = pktdata + sizeof(struct ether_header);
+		*dscp = IP_DSCP46(ip_body);
+		rc = TRUE;
+	}
+	else if (eh->ether_type == HTON16(ETHER_TYPE_8021Q)) {
+		evh = (struct ethervlan_header *)eh;
+
+		/* minimum length is ethervlan header and IP header */
+		if (pktlen >= sizeof(struct ethervlan_header) + IPV4_MIN_HEADER_LEN &&
+			evh->ether_type == HTON16(ETHER_TYPE_IP)) {
+			ip_body = pktdata + sizeof(struct ethervlan_header);
+			*dscp = IP_DSCP46(ip_body);
+			rc = TRUE;
+		}
+	}
+
+	return rc;
+}
+
+/* usr_prio range from low to high with usr_prio value */
+static bool
+up_table_set(uint8 *up_table, uint8 usr_prio, uint8 low, uint8 high)
+{
+	int i;
+
+	if (usr_prio > 7 || low > high || low >= UP_TABLE_MAX || high >= UP_TABLE_MAX) {
+		return FALSE;
+	}
+
+	for (i = low; i <= high; i++) {
+		up_table[i] = usr_prio;
+	}
+
+	return TRUE;
+}
+
+/* set user priority table */
+int
+BCMFASTPATH(wl_set_up_table)(uint8 *up_table, bcm_tlv_t *qos_map_ie)
+{
+	uint8 len;
+
+	if (up_table == NULL || qos_map_ie == NULL) {
+		return BCME_ERROR;
+	}
+
+	/* clear table to check table was set or not */
+	memset(up_table, 0xff, UP_TABLE_MAX);
+
+	/* length of QoS Map IE must be 16+n*2, n is number of exceptions */
+	if (qos_map_ie != NULL && qos_map_ie->id == DOT11_MNG_QOS_MAP_ID &&
+			(len = qos_map_ie->len) >= QOS_MAP_FIXED_LENGTH &&
+			(len % 2) == 0) {
+		uint8 *except_ptr = (uint8 *)qos_map_ie->data;
+		uint8 except_len = len - QOS_MAP_FIXED_LENGTH;
+		uint8 *range_ptr = except_ptr + except_len;
+		int i;
+
+		/* fill in ranges */
+		for (i = 0; i < QOS_MAP_FIXED_LENGTH; i += 2) {
+			uint8 low = range_ptr[i];
+			uint8 high = range_ptr[i + 1];
+			if (low == 255 && high == 255) {
+				continue;
+			}
+
+			if (!up_table_set(up_table, i / 2, low, high)) {
+				/* clear the table on failure */
+				memset(up_table, 0xff, UP_TABLE_MAX);
+				return BCME_ERROR;
+			}
+		}
+
+		/* update exceptions */
+		for (i = 0; i < except_len; i += 2) {
+			uint8 dscp = except_ptr[i];
+			uint8 usr_prio = except_ptr[i+1];
+
+			/* exceptions with invalid dscp/usr_prio are ignored */
+			up_table_set(up_table, usr_prio, dscp, dscp);
+		}
+	}
+
+	return BCME_OK;
+}
+
+#ifndef BCM_BOOTLOADER
+/* The 0.5KB string table is not removed by compiler even though it's unused */
+
+static char bcm_undeferrstr[32];
+static const char *bcmerrorstrtable[] = BCMERRSTRINGTABLE;
+
+/* Convert the error codes into related error strings  */
+/* BCMRAMFN for BCME_LAST usage */
+const char *
+BCMRAMFN(bcmerrorstr)(int bcmerror)
+{
+	/* check if someone added a bcmerror code but forgot to add errorstring */
+	ASSERT(ABS(BCME_LAST) == (ARRAYSIZE(bcmerrorstrtable) - 1));
+
+	if (bcmerror > 0 || bcmerror < BCME_LAST) {
+		snprintf(bcm_undeferrstr, sizeof(bcm_undeferrstr), "Undefined error %d", bcmerror);
+		return bcm_undeferrstr;
+	}
+
+	ASSERT(strlen(bcmerrorstrtable[-bcmerror]) < BCME_STRLEN);
+
+	return bcmerrorstrtable[-bcmerror];
+}
+
+#endif /* !BCM_BOOTLOADER */
+
+#ifdef BCMDBG_PKT   /* pkt logging for debugging */
+/* Add a packet to the pktlist */
+static void
+_pktlist_add(pktlist_info_t *pktlist, void *pkt, int line, char *file)
+{
+	uint16 i;
+	char *basename;
+#ifdef BCMDBG_PTRACE
+	uint16 *idx = PKTLIST_IDX(pkt);
+#endif /* BCMDBG_PTRACE */
+
+	ASSERT(pktlist->count < PKTLIST_SIZE);
+
+	/* Verify the packet is not already part of the list */
+	for (i = 0; i < pktlist->count; i++) {
+		if (pktlist->list[i].pkt == pkt)
+			ASSERT(0);
+	}
+	pktlist->list[pktlist->count].pkt = pkt;
+	pktlist->list[pktlist->count].line = line;
+
+	basename = strrchr(file, '/');
+	if (basename)
+		basename++;
+	else
+		basename = file;
+	pktlist->list[pktlist->count].file = basename;
+#ifdef BCMDBG_PTRACE
+	*idx = pktlist->count;
+	bzero(pktlist->list[pktlist->count].pkt_trace, PKTTRACE_MAX_BYTES);
+#endif /* BCMDBG_PTRACE */
+	pktlist->count++;
+
+	return;
+}
+
+void
+pktlist_add(pktlist_info_t *pktlist, void *pkt, int line, char *file)
+{
+	void *p;
+	for (p = pkt; p != NULL; p = PKTCLINK(p))
+		_pktlist_add(pktlist, p, line, file);
+}
+
+/* Remove a packet from the pktlist */
+static void
+_pktlist_remove(pktlist_info_t *pktlist, void *pkt)
+{
+	uint16 i;
+	uint16 num = pktlist->count;
+#ifdef BCMDBG_PTRACE
+	uint16 *idx = PKTLIST_IDX(pkt);
+
+	ASSERT((*idx) < pktlist->count);
+#endif /* BCMDBG_PTRACE */
+
+	/* find the index where pkt exists */
+	for (i = 0; i < num; i++) {
+		/* check for the existence of pkt in the list */
+		if (pktlist->list[i].pkt == pkt) {
+#ifdef BCMDBG_PTRACE
+			ASSERT((*idx) == i);
+#endif /* BCMDBG_PTRACE */
+			/* replace with the last element */
+			pktlist->list[i].pkt = pktlist->list[num-1].pkt;
+			pktlist->list[i].line = pktlist->list[num-1].line;
+			pktlist->list[i].file = pktlist->list[num-1].file;
+#ifdef BCMDBG_PTRACE
+			memcpy(pktlist->list[i].pkt_trace, pktlist->list[num-1].pkt_trace,
+				PKTTRACE_MAX_BYTES);
+			idx = PKTLIST_IDX(pktlist->list[i].pkt);
+			*idx = i;
+#endif /* BCMDBG_PTRACE */
+			pktlist->count--;
+			return;
+		}
+	}
+	ASSERT(0);
+}
+
+void
+pktlist_remove(pktlist_info_t *pktlist, void *pkt)
+{
+	void *p;
+	for (p = pkt; p != NULL; p = PKTCLINK(p))
+		_pktlist_remove(pktlist, p);
+}
+
+#ifdef BCMDBG_PTRACE
+static void
+_pktlist_trace(pktlist_info_t *pktlist, void *pkt, uint16 bit)
+{
+	uint16 *idx = PKTLIST_IDX(pkt);
+
+	ASSERT(((*idx) < pktlist->count) && (bit < PKTTRACE_MAX_BITS));
+	ASSERT(pktlist->list[(*idx)].pkt == pkt);
+
+	pktlist->list[(*idx)].pkt_trace[bit/NBBY] |= (1 << ((bit)%NBBY));
+
+}
+
+void
+pktlist_trace(pktlist_info_t *pktlist, void *pkt, uint16 bit)
+{
+	void *p;
+	for (p = pkt; p != NULL; p = PKTCLINK(p))
+		_pktlist_trace(pktlist, p, bit);
+}
+#endif /* BCMDBG_PTRACE */
+
+/* Dump the pktlist (and the contents of each packet if 'data'
+ * is set). 'buf' should be large enough
+ */
+
+char *
+pktlist_dump(pktlist_info_t *pktlist, char *buf)
+{
+	char *obuf = buf;
+	uint16 i;
+
+	if (buf != NULL)
+		buf += sprintf(buf, "Packet list dump:\n");
+	else
+		printf("Packet list dump:\n");
+
+	for (i = 0; i < (pktlist->count); i++) {
+		if (buf != NULL)
+			buf += sprintf(buf, "Pkt_addr: 0x%p Line: %d File: %s\t",
+				OSL_OBFUSCATE_BUF(pktlist->list[i].pkt), pktlist->list[i].line,
+				pktlist->list[i].file);
+		else
+			printf("Pkt_addr: 0x%p Line: %d File: %s\t",
+				OSL_OBFUSCATE_BUF(pktlist->list[i].pkt),
+				pktlist->list[i].line, pktlist->list[i].file);
+
+/* #ifdef NOTDEF  Remove this ifdef to print pkttag and pktdata */
+		if (buf != NULL) {
+			if (PKTTAG(pktlist->list[i].pkt)) {
+				/* Print pkttag */
+				buf += sprintf(buf, "Pkttag(in hex): ");
+				buf += bcm_format_hex(buf, PKTTAG(pktlist->list[i].pkt),
+					OSL_PKTTAG_SZ);
+			}
+			buf += sprintf(buf, "Pktdata(in hex): ");
+			buf += bcm_format_hex(buf, PKTDATA(OSH_NULL, pktlist->list[i].pkt),
+			                      PKTLEN(OSH_NULL, pktlist->list[i].pkt));
+		} else {
+			void *pkt = pktlist->list[i].pkt, *npkt;
+
+			printf("Pkt[%d] Dump:\n", i);
+			while (pkt) {
+				int hroom;
+				uint pktlen;
+				uchar *src;
+#ifdef BCMDBG_PTRACE
+				uint16 *idx = PKTLIST_IDX(pkt);
+
+				ASSERT((*idx) < pktlist->count);
+				prhex("Pkt Trace (in hex):", pktlist->list[(*idx)].pkt_trace,
+					PKTTRACE_MAX_BYTES);
+#endif /* BCMDBG_PTRACE */
+				npkt = (void *)PKTNEXT(OSH_NULL, pkt);
+				PKTSETNEXT(OSH_NULL, pkt, NULL);
+
+				src = (uchar *)(PKTTAG(pkt));
+				pktlen = PKTLEN(OSH_NULL, pkt);
+				hroom = PKTHEADROOM(OSH_NULL, pkt);
+
+				printf("Pkttag_addr: %p\n", OSL_OBFUSCATE_BUF(src));
+				if (src)
+					prhex("Pkttag(in hex): ", src, OSL_PKTTAG_SZ);
+				src = (uchar *) (PKTDATA(OSH_NULL, pkt));
+				printf("Pkthead_addr: %p len: %d\n",
+					OSL_OBFUSCATE_BUF(src - hroom), hroom);
+				prhex("Pkt headroom content(in hex): ", src - hroom, hroom);
+				printf("Pktdata_addr: %p len: %d\n",
+					OSL_OBFUSCATE_BUF(src), pktlen);
+				prhex("Pktdata(in hex): ", src, pktlen);
+
+				pkt = npkt;
+			}
+		}
+/* #endif  NOTDEF */
+
+		if (buf != NULL)
+			buf += sprintf(buf, "\n");
+		else
+			printf("\n");
+	}
+	return obuf;
+}
+#endif  /* BCMDBG_PKT */
+
+/* iovar table lookup */
+/* could mandate sorted tables and do a binary search */
+const bcm_iovar_t*
+bcm_iovar_lookup(const bcm_iovar_t *table, const char *name)
+{
+	const bcm_iovar_t *vi;
+	const char *lookup_name;
+
+	/* skip any ':' delimited option prefixes */
+	lookup_name = strrchr(name, ':');
+	if (lookup_name != NULL)
+		lookup_name++;
+	else
+		lookup_name = name;
+
+	ASSERT(table != NULL);
+
+	for (vi = table; vi->name; vi++) {
+		if (!strcmp(vi->name, lookup_name))
+			return vi;
+	}
+	/* ran to end of table */
+
+	return NULL; /* var name not found */
+}
+
+int
+bcm_iovar_lencheck(const bcm_iovar_t *vi, void *arg, uint len, bool set)
+{
+	int bcmerror = 0;
+	BCM_REFERENCE(arg);
+
+	/* length check on io buf */
+	switch (vi->type) {
+	case IOVT_BOOL:
+	case IOVT_INT8:
+	case IOVT_INT16:
+	case IOVT_INT32:
+	case IOVT_UINT8:
+	case IOVT_UINT16:
+	case IOVT_UINT32:
+		/* all integers are int32 sized args at the ioctl interface */
+		if (len < sizeof(int)) {
+			bcmerror = BCME_BUFTOOSHORT;
+		}
+		break;
+
+	case IOVT_BUFFER:
+		/* buffer must meet minimum length requirement */
+		if (len < vi->minlen) {
+			bcmerror = BCME_BUFTOOSHORT;
+		}
+		break;
+
+	case IOVT_VOID:
+		if (!set) {
+			/* Cannot return nil... */
+			bcmerror = BCME_UNSUPPORTED;
+		}
+		break;
+
+	default:
+		/* unknown type for length check in iovar info */
+		ASSERT(0);
+		bcmerror = BCME_UNSUPPORTED;
+	}
+
+	return bcmerror;
+}
+
+/*
+ * Hierarchical Multiword bitmap based small id allocator.
+ *
+ * Multilevel hierarchy bitmap. (maximum 2 levels)
+ * First hierarchy uses a multiword bitmap to identify 32bit words in the
+ * second hierarchy that have at least a single bit set. Each bit in a word of
+ * the second hierarchy represents a unique ID that may be allocated.
+ *
+ * BCM_MWBMAP_ITEMS_MAX: Maximum number of IDs managed.
+ * BCM_MWBMAP_BITS_WORD: Number of bits in a bitmap word word
+ * BCM_MWBMAP_WORDS_MAX: Maximum number of bitmap words needed for free IDs.
+ * BCM_MWBMAP_WDMAP_MAX: Maximum number of bitmap wordss identifying first non
+ *                       non-zero bitmap word carrying at least one free ID.
+ * BCM_MWBMAP_SHIFT_OP:  Used in MOD, DIV and MUL operations.
+ * BCM_MWBMAP_INVALID_IDX: Value ~0U is treated as an invalid ID
+ *
+ * Design Notes:
+ * BCM_MWBMAP_USE_CNTSETBITS trades CPU for memory. A runtime count of how many
+ * bits are computed each time on allocation and deallocation, requiring 4
+ * array indexed access and 3 arithmetic operations. When not defined, a runtime
+ * count of set bits state is maintained. Upto 32 Bytes per 1024 IDs is needed.
+ * In a 4K max ID allocator, up to 128Bytes are hence used per instantiation.
+ * In a memory limited system e.g. dongle builds, a CPU for memory tradeoff may
+ * be used by defining BCM_MWBMAP_USE_CNTSETBITS.
+ *
+ * Note: wd_bitmap[] is statically declared and is not ROM friendly ... array
+ * size is fixed. No intention to support larger than 4K indice allocation. ID
+ * allocators for ranges smaller than 4K will have a wastage of only 12Bytes
+ * with savings in not having to use an indirect access, had it been dynamically
+ * allocated.
+ */
+#if defined(DONGLEBUILD)
+#define BCM_MWBMAP_USE_CNTSETBITS		/* runtime count set bits */
+#if defined(PCIEDEV_HOST_PKTID_AUDIT_ENABLED)
+#define BCM_MWBMAP_ITEMS_MAX	(38 * 1024)
+#else /* ! PCIEDEV_HOST_PKTID_AUDIT_ENABLED */
+#define BCM_MWBMAP_ITEMS_MAX	(7 * 1024)
+#endif /* PCIEDEV_HOST_PKTID_AUDIT_ENABLED */
+#else  /* ! DONGLEBUILD */
+#define BCM_MWBMAP_ITEMS_MAX    (64 * 1024)  /* May increase to 64K */
+#endif /*   DONGLEBUILD */
+
+#define BCM_MWBMAP_BITS_WORD    (NBITS(uint32))
+#define BCM_MWBMAP_WORDS_MAX    (BCM_MWBMAP_ITEMS_MAX / BCM_MWBMAP_BITS_WORD)
+#define BCM_MWBMAP_WDMAP_MAX    (BCM_MWBMAP_WORDS_MAX / BCM_MWBMAP_BITS_WORD)
+#define BCM_MWBMAP_SHIFT_OP     (5)
+#define BCM_MWBMAP_MODOP(ix)    ((ix) & (BCM_MWBMAP_BITS_WORD - 1))
+#define BCM_MWBMAP_DIVOP(ix)    ((ix) >> BCM_MWBMAP_SHIFT_OP)
+#define BCM_MWBMAP_MULOP(ix)    ((ix) << BCM_MWBMAP_SHIFT_OP)
+
+/* Redefine PTR() and/or HDL() conversion to invoke audit for debugging */
+#define BCM_MWBMAP_PTR(hdl)		((struct bcm_mwbmap *)(hdl))
+#define BCM_MWBMAP_HDL(ptr)		((void *)(ptr))
+
+#if defined(BCM_MWBMAP_DEBUG)
+#define BCM_MWBMAP_AUDIT(mwb) \
+	do { \
+		ASSERT((mwb != NULL) && \
+		       (((struct bcm_mwbmap *)(mwb))->magic == (void *)(mwb))); \
+		bcm_mwbmap_audit(mwb); \
+	} while (0)
+#define MWBMAP_ASSERT(exp)		ASSERT(exp)
+#define MWBMAP_DBG(x)           printf x
+#else   /* !BCM_MWBMAP_DEBUG */
+#define BCM_MWBMAP_AUDIT(mwb)   do {} while (0)
+#define MWBMAP_ASSERT(exp)		do {} while (0)
+#define MWBMAP_DBG(x)
+#endif  /* !BCM_MWBMAP_DEBUG */
+
+#ifndef ENODEBUG
+#define EDBG_DUMP(x)           printf x
+#else
+#define EDBG_DUMP(x)
+#endif
+
+typedef struct bcm_mwbmap {     /* Hierarchical multiword bitmap allocator    */
+	uint16 wmaps;               /* Total number of words in free wd bitmap    */
+	uint16 imaps;               /* Total number of words in free id bitmap    */
+	int32  ifree;               /* Count of free indices. Used only in audits */
+	uint16 total;               /* Total indices managed by multiword bitmap  */
+
+	void * magic;               /* Audit handle parameter from user           */
+
+	uint32 wd_bitmap[BCM_MWBMAP_WDMAP_MAX]; /* 1st level bitmap of            */
+#if !defined(BCM_MWBMAP_USE_CNTSETBITS)
+	int8   wd_count[BCM_MWBMAP_WORDS_MAX];  /* free id running count, 1st lvl */
+#endif /*  ! BCM_MWBMAP_USE_CNTSETBITS */
+
+	uint32 id_bitmap[0];        /* Second level bitmap                        */
+} bcm_mwbmap_t;
+
+/* Incarnate a hierarchical multiword bitmap based small index allocator. */
+struct bcm_mwbmap *
+bcm_mwbmap_init(osl_t *osh, uint32 items_max)
+{
+	struct bcm_mwbmap * mwbmap_p;
+	uint32 wordix, size, words, extra;
+
+	/* Implementation Constraint: Uses 32bit word bitmap */
+	MWBMAP_ASSERT(BCM_MWBMAP_BITS_WORD == 32U);
+	MWBMAP_ASSERT(BCM_MWBMAP_SHIFT_OP == 5U);
+	MWBMAP_ASSERT(ISPOWEROF2(BCM_MWBMAP_ITEMS_MAX));
+	MWBMAP_ASSERT((BCM_MWBMAP_ITEMS_MAX % BCM_MWBMAP_BITS_WORD) == 0U);
+
+	ASSERT(items_max <= BCM_MWBMAP_ITEMS_MAX);
+
+	/* Determine the number of words needed in the multiword bitmap */
+	extra = BCM_MWBMAP_MODOP(items_max);
+	words = BCM_MWBMAP_DIVOP(items_max) + ((extra != 0U) ? 1U : 0U);
+
+	/* Allocate runtime state of multiword bitmap */
+	/* Note: wd_count[] or wd_bitmap[] are not dynamically allocated */
+	size = sizeof(bcm_mwbmap_t) + (sizeof(uint32) * words);
+	mwbmap_p = (bcm_mwbmap_t *)MALLOCZ(osh, size);
+	if (mwbmap_p == (bcm_mwbmap_t *)NULL) {
+		ASSERT(0);
+		goto error1;
+	}
+
+	/* Initialize runtime multiword bitmap state */
+	mwbmap_p->imaps = (uint16)words;
+	mwbmap_p->ifree = (int32)items_max;
+	mwbmap_p->total = (uint16)items_max;
+
+	/* Setup magic, for use in audit of handle */
+	mwbmap_p->magic = BCM_MWBMAP_HDL(mwbmap_p);
+
+	/* Setup the second level bitmap of free indices */
+	/* Mark all indices as available */
+	for (wordix = 0U; wordix < mwbmap_p->imaps; wordix++) {
+		mwbmap_p->id_bitmap[wordix] = (uint32)(~0U);
+#if !defined(BCM_MWBMAP_USE_CNTSETBITS)
+		mwbmap_p->wd_count[wordix] = BCM_MWBMAP_BITS_WORD;
+#endif /*  ! BCM_MWBMAP_USE_CNTSETBITS */
+	}
+
+	/* Ensure that extra indices are tagged as un-available */
+	if (extra) { /* fixup the free ids in last bitmap and wd_count */
+		uint32 * bmap_p = &mwbmap_p->id_bitmap[mwbmap_p->imaps - 1];
+		*bmap_p ^= (uint32)(~0U << extra); /* fixup bitmap */
+#if !defined(BCM_MWBMAP_USE_CNTSETBITS)
+		mwbmap_p->wd_count[mwbmap_p->imaps - 1] = (int8)extra; /* fixup count */
+#endif /*  ! BCM_MWBMAP_USE_CNTSETBITS */
+	}
+
+	/* Setup the first level bitmap hierarchy */
+	extra = BCM_MWBMAP_MODOP(mwbmap_p->imaps);
+	words = BCM_MWBMAP_DIVOP(mwbmap_p->imaps) + ((extra != 0U) ? 1U : 0U);
+
+	mwbmap_p->wmaps = (uint16)words;
+
+	for (wordix = 0U; wordix < mwbmap_p->wmaps; wordix++)
+		mwbmap_p->wd_bitmap[wordix] = (uint32)(~0U);
+	if (extra) {
+		uint32 * bmap_p = &mwbmap_p->wd_bitmap[mwbmap_p->wmaps - 1];
+		*bmap_p ^= (uint32)(~0U << extra); /* fixup bitmap */
+	}
+
+	return mwbmap_p;
+
+error1:
+	return BCM_MWBMAP_INVALID_HDL;
+}
+
+/* Release resources used by multiword bitmap based small index allocator. */
+void
+bcm_mwbmap_fini(osl_t * osh, struct bcm_mwbmap * mwbmap_hdl)
+{
+	bcm_mwbmap_t * mwbmap_p;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	MFREE(osh, mwbmap_p, sizeof(struct bcm_mwbmap)
+			     + (sizeof(uint32) * mwbmap_p->imaps));
+	return;
+}
+
+/* Allocate a unique small index using a multiword bitmap index allocator.    */
+uint32
+BCMFASTPATH(bcm_mwbmap_alloc)(struct bcm_mwbmap * mwbmap_hdl)
+{
+	bcm_mwbmap_t * mwbmap_p;
+	uint32 wordix, bitmap;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	/* Start with the first hierarchy */
+	for (wordix = 0; wordix < mwbmap_p->wmaps; ++wordix) {
+
+		bitmap = mwbmap_p->wd_bitmap[wordix]; /* get the word bitmap */
+
+		if (bitmap != 0U) {
+
+			uint32 count, bitix, *bitmap_p;
+
+			bitmap_p = &mwbmap_p->wd_bitmap[wordix];
+
+			/* clear all except trailing 1 */
+			if (bitmap != (1u << 31u)) {
+				bitmap = (uint32)(((int)(bitmap)) & (-((int)(bitmap))));
+			}
+			MWBMAP_ASSERT(C_bcm_count_leading_zeros(bitmap) ==
+			              bcm_count_leading_zeros(bitmap));
+			bitix = bcm_count_leading_zeros(bitmap); /* use asm clz */
+			bitix = (bitix <= (BCM_MWBMAP_BITS_WORD - 1)) ?
+					((BCM_MWBMAP_BITS_WORD - 1) - bitix) : 0;
+			wordix = BCM_MWBMAP_MULOP(wordix) + bitix;
+
+			/* Clear bit if wd count is 0, without conditional branch */
+#if defined(BCM_MWBMAP_USE_CNTSETBITS)
+			count = bcm_cntsetbits(mwbmap_p->id_bitmap[wordix]) - 1;
+#else  /* ! BCM_MWBMAP_USE_CNTSETBITS */
+			mwbmap_p->wd_count[wordix]--;
+			count = mwbmap_p->wd_count[wordix];
+			MWBMAP_ASSERT(count ==
+			              (bcm_cntsetbits(mwbmap_p->id_bitmap[wordix]) - 1));
+#endif /* ! BCM_MWBMAP_USE_CNTSETBITS */
+			MWBMAP_ASSERT(count >= 0);
+
+			/* clear wd_bitmap bit if id_map count is 0 */
+			bitmap = (uint32)(count == 0) << bitix;
+
+			MWBMAP_DBG((
+			    "Lvl1: bitix<%02u> wordix<%02u>: %08x ^ %08x = %08x wfree %d",
+			    bitix, wordix, *bitmap_p, bitmap, (*bitmap_p) ^ bitmap, count));
+
+			*bitmap_p ^= bitmap;
+
+			/* Use bitix in the second hierarchy */
+			bitmap_p = &mwbmap_p->id_bitmap[wordix];
+
+			bitmap = mwbmap_p->id_bitmap[wordix]; /* get the id bitmap */
+			MWBMAP_ASSERT(bitmap != 0U);
+
+			/* clear all except trailing 1 */
+			if (bitmap != (1u << 31u)) {
+				bitmap = (uint32)(((int)(bitmap)) & (-((int)(bitmap))));
+			}
+			MWBMAP_ASSERT(C_bcm_count_leading_zeros(bitmap) ==
+			              bcm_count_leading_zeros(bitmap));
+			bitix    = BCM_MWBMAP_MULOP(wordix)
+				 + (BCM_MWBMAP_BITS_WORD - 1)
+				 - bcm_count_leading_zeros(bitmap); /* use asm clz */
+
+			mwbmap_p->ifree--; /* decrement system wide free count */
+			MWBMAP_ASSERT(mwbmap_p->ifree >= 0);
+
+			MWBMAP_DBG((
+			    "Lvl2: bitix<%02u> wordix<%02u>: %08x ^ %08x = %08x ifree %d",
+			    bitix, wordix, *bitmap_p, bitmap, (*bitmap_p) ^ bitmap,
+			    mwbmap_p->ifree));
+
+			*bitmap_p ^= bitmap; /* mark as allocated = 1b0 */
+
+			return bitix;
+		}
+	}
+
+	ASSERT(mwbmap_p->ifree == 0);
+
+	return BCM_MWBMAP_INVALID_IDX;
+}
+
+/* Force an index at a specified position to be in use */
+void
+bcm_mwbmap_force(struct bcm_mwbmap * mwbmap_hdl, uint32 bitix)
+{
+	bcm_mwbmap_t * mwbmap_p;
+	uint32 count, wordix, bitmap, *bitmap_p;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	ASSERT(bitix < mwbmap_p->total);
+
+	/* Start with second hierarchy */
+	wordix   = BCM_MWBMAP_DIVOP(bitix);
+	bitmap   = (uint32)(1U << BCM_MWBMAP_MODOP(bitix));
+	bitmap_p = &mwbmap_p->id_bitmap[wordix];
+
+	ASSERT((*bitmap_p & bitmap) == bitmap);
+
+	mwbmap_p->ifree--; /* update free count */
+	ASSERT(mwbmap_p->ifree >= 0);
+
+	MWBMAP_DBG(("Lvl2: bitix<%u> wordix<%u>: %08x ^ %08x = %08x ifree %d",
+	            bitix, wordix, *bitmap_p, bitmap, (*bitmap_p) ^ bitmap,
+	            mwbmap_p->ifree));
+
+	*bitmap_p ^= bitmap; /* mark as in use */
+
+	/* Update first hierarchy */
+	bitix    = wordix;
+
+	wordix   = BCM_MWBMAP_DIVOP(bitix);
+	bitmap_p = &mwbmap_p->wd_bitmap[wordix];
+
+#if defined(BCM_MWBMAP_USE_CNTSETBITS)
+	count = bcm_cntsetbits(mwbmap_p->id_bitmap[bitix]);
+#else  /* ! BCM_MWBMAP_USE_CNTSETBITS */
+	mwbmap_p->wd_count[bitix]--;
+	count = mwbmap_p->wd_count[bitix];
+	MWBMAP_ASSERT(count == bcm_cntsetbits(mwbmap_p->id_bitmap[bitix]));
+#endif /* ! BCM_MWBMAP_USE_CNTSETBITS */
+	MWBMAP_ASSERT(count >= 0);
+
+	bitmap   = (uint32)(count == 0) << BCM_MWBMAP_MODOP(bitix);
+
+	MWBMAP_DBG(("Lvl1: bitix<%02lu> wordix<%02u>: %08x ^ %08x = %08x wfree %d",
+	            BCM_MWBMAP_MODOP(bitix), wordix, *bitmap_p, bitmap,
+	            (*bitmap_p) ^ bitmap, count));
+
+	*bitmap_p ^= bitmap; /* mark as in use */
+
+	return;
+}
+
+/* Free a previously allocated index back into the multiword bitmap allocator */
+void
+BCMPOSTTRAPFASTPATH(bcm_mwbmap_free)(struct bcm_mwbmap * mwbmap_hdl, uint32 bitix)
+{
+	bcm_mwbmap_t * mwbmap_p;
+	uint32 wordix, bitmap, *bitmap_p;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	ASSERT_FP(bitix < mwbmap_p->total);
+
+	/* Start with second level hierarchy */
+	wordix   = BCM_MWBMAP_DIVOP(bitix);
+	bitmap   = (1U << BCM_MWBMAP_MODOP(bitix));
+	bitmap_p = &mwbmap_p->id_bitmap[wordix];
+
+	ASSERT_FP((*bitmap_p & bitmap) == 0U);	/* ASSERT not a double free */
+
+	mwbmap_p->ifree++; /* update free count */
+	ASSERT_FP(mwbmap_p->ifree <= mwbmap_p->total);
+
+	MWBMAP_DBG(("Lvl2: bitix<%02u> wordix<%02u>: %08x | %08x = %08x ifree %d",
+	            bitix, wordix, *bitmap_p, bitmap, (*bitmap_p) | bitmap,
+	            mwbmap_p->ifree));
+
+	*bitmap_p |= bitmap; /* mark as available */
+
+	/* Now update first level hierarchy */
+
+	bitix    = wordix;
+
+	wordix   = BCM_MWBMAP_DIVOP(bitix); /* first level's word index */
+	bitmap   = (1U << BCM_MWBMAP_MODOP(bitix));
+	bitmap_p = &mwbmap_p->wd_bitmap[wordix];
+
+#if !defined(BCM_MWBMAP_USE_CNTSETBITS)
+	mwbmap_p->wd_count[bitix]++;
+#endif
+
+#if defined(BCM_MWBMAP_DEBUG)
+	{
+		uint32 count;
+#if defined(BCM_MWBMAP_USE_CNTSETBITS)
+		count = bcm_cntsetbits(mwbmap_p->id_bitmap[bitix]);
+#else  /*  ! BCM_MWBMAP_USE_CNTSETBITS */
+		count = mwbmap_p->wd_count[bitix];
+		MWBMAP_ASSERT(count == bcm_cntsetbits(mwbmap_p->id_bitmap[bitix]));
+#endif /*  ! BCM_MWBMAP_USE_CNTSETBITS */
+
+		MWBMAP_ASSERT(count <= BCM_MWBMAP_BITS_WORD);
+
+		MWBMAP_DBG(("Lvl1: bitix<%02u> wordix<%02u>: %08x | %08x = %08x wfree %d",
+		            bitix, wordix, *bitmap_p, bitmap, (*bitmap_p) | bitmap, count));
+	}
+#endif /* BCM_MWBMAP_DEBUG */
+
+	*bitmap_p |= bitmap;
+
+	return;
+}
+
+/* Fetch the toal number of free indices in the multiword bitmap allocator */
+uint32
+bcm_mwbmap_free_cnt(struct bcm_mwbmap * mwbmap_hdl)
+{
+	bcm_mwbmap_t * mwbmap_p;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	ASSERT_FP(mwbmap_p->ifree >= 0);
+
+	return (uint32)mwbmap_p->ifree;
+}
+
+/* Determine whether an index is inuse or free */
+bool
+bcm_mwbmap_isfree(struct bcm_mwbmap * mwbmap_hdl, uint32 bitix)
+{
+	bcm_mwbmap_t * mwbmap_p;
+	uint32 wordix, bitmap;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	ASSERT(bitix < mwbmap_p->total);
+
+	wordix   = BCM_MWBMAP_DIVOP(bitix);
+	bitmap   = (1U << BCM_MWBMAP_MODOP(bitix));
+
+	return ((mwbmap_p->id_bitmap[wordix] & bitmap) != 0U);
+}
+
+/* Debug dump a multiword bitmap allocator */
+void
+bcm_mwbmap_show(struct bcm_mwbmap * mwbmap_hdl)
+{
+	uint32 ix, count;
+	bcm_mwbmap_t * mwbmap_p;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	printf("mwbmap_p %p wmaps %u imaps %u ifree %d total %u\n",
+		OSL_OBFUSCATE_BUF((void *)mwbmap_p),
+	       mwbmap_p->wmaps, mwbmap_p->imaps, mwbmap_p->ifree, mwbmap_p->total);
+	for (ix = 0U; ix < mwbmap_p->wmaps; ix++) {
+		printf("\tWDMAP:%2u. 0x%08x\t", ix, mwbmap_p->wd_bitmap[ix]);
+		bcm_bitprint32(mwbmap_p->wd_bitmap[ix]);
+		printf("\n");
+	}
+	for (ix = 0U; ix < mwbmap_p->imaps; ix++) {
+#if defined(BCM_MWBMAP_USE_CNTSETBITS)
+		count = bcm_cntsetbits(mwbmap_p->id_bitmap[ix]);
+#else  /* ! BCM_MWBMAP_USE_CNTSETBITS */
+		count = mwbmap_p->wd_count[ix];
+		MWBMAP_ASSERT(count == bcm_cntsetbits(mwbmap_p->id_bitmap[ix]));
+#endif /* ! BCM_MWBMAP_USE_CNTSETBITS */
+		printf("\tIDMAP:%2u. 0x%08x %02u\t", ix, mwbmap_p->id_bitmap[ix], count);
+		bcm_bitprint32(mwbmap_p->id_bitmap[ix]);
+		printf("\n");
+	}
+
+	return;
+}
+
+/* Audit a hierarchical multiword bitmap */
+void
+bcm_mwbmap_audit(struct bcm_mwbmap * mwbmap_hdl)
+{
+	bcm_mwbmap_t * mwbmap_p;
+	uint32 count, free_cnt = 0U, wordix, idmap_ix, bitix, *bitmap_p;
+
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	for (wordix = 0U; wordix < mwbmap_p->wmaps; ++wordix) {
+
+		bitmap_p = &mwbmap_p->wd_bitmap[wordix];
+
+		for (bitix = 0U; bitix < BCM_MWBMAP_BITS_WORD; bitix++) {
+			if ((*bitmap_p) & (1u << bitix)) {
+				idmap_ix = BCM_MWBMAP_MULOP(wordix) + bitix;
+#if defined(BCM_MWBMAP_USE_CNTSETBITS)
+				count = bcm_cntsetbits(mwbmap_p->id_bitmap[idmap_ix]);
+#else  /* ! BCM_MWBMAP_USE_CNTSETBITS */
+				count = mwbmap_p->wd_count[idmap_ix];
+				ASSERT(count == bcm_cntsetbits(mwbmap_p->id_bitmap[idmap_ix]));
+#endif /* ! BCM_MWBMAP_USE_CNTSETBITS */
+				ASSERT(count != 0U);
+				free_cnt += count;
+			}
+		}
+	}
+
+	ASSERT((int)free_cnt == mwbmap_p->ifree);
+	if (free_cnt != mwbmap_p->ifree) {
+		printf("Error: free_cnt != mwbmap_p->ifree\n");
+	}
+}
+
+/* END : Multiword bitmap based 64bit to Unique 32bit Id allocator. */
+
+/* Simple 16bit Id allocator using a stack implementation. */
+typedef struct id16_map {
+	uint32  failures;  /* count of failures */
+	void    *dbg;      /* debug placeholder */
+	uint16  total;     /* total number of ids managed by allocator */
+	uint16  start;     /* start value of 16bit ids to be managed */
+	int     stack_idx; /* index into stack of available ids */
+	uint16  stack[0];  /* stack of 16 bit ids */
+} id16_map_t;
+
+#define ID16_MAP_SZ(items)      (sizeof(id16_map_t) + \
+				     (sizeof(uint16) * (items)))
+
+#if defined(BCM_DBG)
+
+/* Uncomment BCM_DBG_ID16 to debug double free */
+/* #define BCM_DBG_ID16 */
+
+typedef struct id16_map_dbg {
+	uint16  total;
+	bool    avail[0];
+} id16_map_dbg_t;
+#define ID16_MAP_DBG_SZ(items)  (sizeof(id16_map_dbg_t) + \
+				     (sizeof(bool) * (items)))
+#define ID16_MAP_MSG(x)         print x
+#else
+#define ID16_MAP_MSG(x)
+#endif /* BCM_DBG */
+
+void * /* Construct an id16 allocator: [start_val16 .. start_val16+total_ids) */
+id16_map_init(osl_t *osh, uint16 total_ids, uint16 start_val16)
+{
+	uint16 idx, val16;
+	id16_map_t * id16_map;
+
+	ASSERT(total_ids > 0);
+
+	/* A start_val16 of ID16_UNDEFINED, allows the caller to fill the id16 map
+	 * with random values.
+	 */
+	ASSERT((start_val16 == ID16_UNDEFINED) ||
+	       (start_val16 + total_ids) < ID16_INVALID);
+
+	id16_map = (id16_map_t *) MALLOCZ(osh, ID16_MAP_SZ(total_ids));
+	if (id16_map == NULL) {
+		return NULL;
+	}
+
+	id16_map->total = total_ids;
+	id16_map->start = start_val16;
+	id16_map->failures = 0;
+	id16_map->dbg = NULL;
+
+	/*
+	 * Populate stack with 16bit id values, commencing with start_val16.
+	 * if start_val16 is ID16_UNDEFINED, then do not populate the id16 map.
+	 */
+	id16_map->stack_idx = -1;
+
+	if (id16_map->start != ID16_UNDEFINED) {
+		val16 = start_val16;
+
+		for (idx = 0; idx < total_ids; idx++, val16++) {
+			id16_map->stack_idx = idx;
+			id16_map->stack[id16_map->stack_idx] = val16;
+		}
+	}
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+	if (id16_map->start != ID16_UNDEFINED) {
+		id16_map->dbg = MALLOCZ(osh, ID16_MAP_DBG_SZ(total_ids));
+
+		if (id16_map->dbg) {
+			id16_map_dbg_t *id16_map_dbg = (id16_map_dbg_t *)id16_map->dbg;
+
+			id16_map_dbg->total = total_ids;
+			for (idx = 0; idx < total_ids; idx++) {
+				id16_map_dbg->avail[idx] = TRUE;
+			}
+		}
+	}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+
+	return (void *)id16_map;
+}
+
+void * /* Destruct an id16 allocator instance */
+id16_map_fini(osl_t *osh, void * id16_map_hndl)
+{
+	uint16 total_ids;
+	id16_map_t * id16_map;
+
+	if (id16_map_hndl == NULL)
+		return NULL;
+
+	id16_map = (id16_map_t *)id16_map_hndl;
+
+	total_ids = id16_map->total;
+	ASSERT(total_ids > 0);
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+	if (id16_map->dbg) {
+		MFREE(osh, id16_map->dbg, ID16_MAP_DBG_SZ(total_ids));
+	}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+
+	id16_map->total = 0;
+	MFREE(osh, id16_map, ID16_MAP_SZ(total_ids));
+
+	return NULL;
+}
+
+void
+id16_map_clear(void * id16_map_hndl, uint16 total_ids, uint16 start_val16)
+{
+	uint16 idx, val16;
+	id16_map_t * id16_map;
+
+	ASSERT(total_ids > 0);
+	/* A start_val16 of ID16_UNDEFINED, allows the caller to fill the id16 map
+	 * with random values.
+	 */
+	ASSERT((start_val16 == ID16_UNDEFINED) ||
+	       (start_val16 + total_ids) < ID16_INVALID);
+
+	id16_map = (id16_map_t *)id16_map_hndl;
+	if (id16_map == NULL) {
+		return;
+	}
+
+	id16_map->total = total_ids;
+	id16_map->start = start_val16;
+	id16_map->failures = 0;
+
+	/* Populate stack with 16bit id values, commencing with start_val16 */
+	id16_map->stack_idx = -1;
+
+	if (id16_map->start != ID16_UNDEFINED) {
+		val16 = start_val16;
+
+		for (idx = 0; idx < total_ids; idx++, val16++) {
+			id16_map->stack_idx = idx;
+			id16_map->stack[id16_map->stack_idx] = val16;
+		}
+	}
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+	if (id16_map->start != ID16_UNDEFINED) {
+		if (id16_map->dbg) {
+			id16_map_dbg_t *id16_map_dbg = (id16_map_dbg_t *)id16_map->dbg;
+
+			id16_map_dbg->total = total_ids;
+			for (idx = 0; idx < total_ids; idx++) {
+				id16_map_dbg->avail[idx] = TRUE;
+			}
+		}
+	}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+}
+
+uint16 /* Allocate a unique 16bit id */
+BCMFASTPATH(id16_map_alloc)(void * id16_map_hndl)
+{
+	uint16 val16;
+	id16_map_t * id16_map;
+
+	ASSERT_FP(id16_map_hndl != NULL);
+
+	id16_map = (id16_map_t *)id16_map_hndl;
+
+	ASSERT_FP(id16_map->total > 0);
+
+	if (id16_map->stack_idx < 0) {
+		id16_map->failures++;
+		return ID16_INVALID;
+	}
+
+	val16 = id16_map->stack[id16_map->stack_idx];
+	id16_map->stack_idx--;
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+	ASSERT_FP((id16_map->start == ID16_UNDEFINED) ||
+	       (val16 < (id16_map->start + id16_map->total)));
+
+	if (id16_map->dbg) { /* Validate val16 */
+		id16_map_dbg_t *id16_map_dbg = (id16_map_dbg_t *)id16_map->dbg;
+
+		ASSERT_FP(id16_map_dbg->avail[val16 - id16_map->start] == TRUE);
+		id16_map_dbg->avail[val16 - id16_map->start] = FALSE;
+	}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+
+	return val16;
+}
+
+void /* Free a 16bit id value into the id16 allocator */
+BCMFASTPATH(id16_map_free)(void * id16_map_hndl, uint16 val16)
+{
+	id16_map_t * id16_map;
+
+	ASSERT_FP(id16_map_hndl != NULL);
+
+	id16_map = (id16_map_t *)id16_map_hndl;
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+	ASSERT_FP((id16_map->start == ID16_UNDEFINED) ||
+	       (val16 < (id16_map->start + id16_map->total)));
+
+	if (id16_map->dbg) { /* Validate val16 */
+		id16_map_dbg_t *id16_map_dbg = (id16_map_dbg_t *)id16_map->dbg;
+
+		ASSERT_FP(id16_map_dbg->avail[val16 - id16_map->start] == FALSE);
+		id16_map_dbg->avail[val16 - id16_map->start] = TRUE;
+	}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+
+	id16_map->stack_idx++;
+	id16_map->stack[id16_map->stack_idx] = val16;
+}
+
+uint32 /* Returns number of failures to allocate an unique id16 */
+id16_map_failures(void * id16_map_hndl)
+{
+	ASSERT(id16_map_hndl != NULL);
+	return ((id16_map_t *)id16_map_hndl)->failures;
+}
+
+bool
+id16_map_audit(void * id16_map_hndl)
+{
+	int idx;
+	int insane = 0;
+	id16_map_t * id16_map;
+
+	ASSERT(id16_map_hndl != NULL);
+
+	id16_map = (id16_map_t *)id16_map_hndl;
+
+	ASSERT(id16_map->stack_idx >= -1);
+	ASSERT(id16_map->stack_idx < (int)id16_map->total);
+
+	if (id16_map->start == ID16_UNDEFINED)
+		goto done;
+
+	for (idx = 0; idx <= id16_map->stack_idx; idx++) {
+		ASSERT(id16_map->stack[idx] >= id16_map->start);
+		ASSERT(id16_map->stack[idx] < (id16_map->start + id16_map->total));
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+		if (id16_map->dbg) {
+			uint16 val16 = id16_map->stack[idx];
+			if (((id16_map_dbg_t *)(id16_map->dbg))->avail[val16] != TRUE) {
+				insane |= 1;
+				ID16_MAP_MSG(("id16_map<%p>: stack_idx %u invalid val16 %u\n",
+				              OSL_OBFUSATE_BUF(id16_map_hndl), idx, val16));
+			}
+		}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+	}
+
+#if defined(BCM_DBG) && defined(BCM_DBG_ID16)
+	if (id16_map->dbg) {
+		uint16 avail = 0; /* Audit available ids counts */
+		for (idx = 0; idx < id16_map_dbg->total; idx++) {
+			if (((id16_map_dbg_t *)(id16_map->dbg))->avail[idx16] == TRUE)
+				avail++;
+		}
+		if (avail && (avail != (id16_map->stack_idx + 1))) {
+			insane |= 1;
+			ID16_MAP_MSG(("id16_map<%p>: avail %u stack_idx %u\n",
+			              OSL_OBFUSCATE_BUF(id16_map_hndl),
+			              avail, id16_map->stack_idx));
+		}
+	}
+#endif /* BCM_DBG && BCM_DBG_ID16 */
+
+done:
+	/* invoke any other system audits */
+	return (!!insane);
+}
+
+/* END: Simple id16 allocator */
+
+void
+dll_pool_detach(void * osh, dll_pool_t * pool, uint16 elems_max, uint16 elem_size)
+{
+	uint32 mem_size;
+	mem_size = (uint32)sizeof(dll_pool_t) + ((uint32)elems_max * (uint32)elem_size);
+	if (pool) {
+		MFREE(osh, pool, mem_size);
+	}
+}
+
+dll_pool_t *
+dll_pool_init(void * osh, uint16 elems_max, uint16 elem_size)
+{
+	uint32 mem_size, i;
+	dll_pool_t * dll_pool_p;
+	dll_t * elem_p;
+
+	ASSERT(elem_size > sizeof(dll_t));
+
+	mem_size = (uint32)sizeof(dll_pool_t) + ((uint32)elems_max * (uint32)elem_size);
+
+	if ((dll_pool_p = (dll_pool_t *)MALLOCZ(osh, mem_size)) == NULL) {
+		ASSERT(0);
+		return dll_pool_p;
+	}
+
+	dll_init(&dll_pool_p->free_list);
+	dll_pool_p->elems_max = elems_max;
+	dll_pool_p->elem_size = elem_size;
+
+	elem_p = dll_pool_p->elements;
+	for (i = 0; i < elems_max; i++) {
+		dll_append(&dll_pool_p->free_list, elem_p);
+		elem_p = (dll_t *)((uintptr)elem_p + elem_size);
+	}
+
+	dll_pool_p->free_count = elems_max;
+
+	return dll_pool_p;
+}
+
+void *
+dll_pool_alloc(dll_pool_t * dll_pool_p)
+{
+	dll_t * elem_p;
+
+	if (dll_pool_p->free_count == 0) {
+		ASSERT(dll_empty(&dll_pool_p->free_list));
+		return NULL;
+	}
+
+	elem_p = dll_head_p(&dll_pool_p->free_list);
+	dll_delete(elem_p);
+	dll_pool_p->free_count -= 1;
+
+	return (void *)elem_p;
+}
+
+void
+BCMPOSTTRAPFN(dll_pool_free)(dll_pool_t * dll_pool_p, void * elem_p)
+{
+	dll_t * node_p = (dll_t *)elem_p;
+	dll_prepend(&dll_pool_p->free_list, node_p);
+	dll_pool_p->free_count += 1;
+}
+
+void
+dll_pool_free_tail(dll_pool_t * dll_pool_p, void * elem_p)
+{
+	dll_t * node_p = (dll_t *)elem_p;
+	dll_append(&dll_pool_p->free_list, node_p);
+	dll_pool_p->free_count += 1;
+}
+
+#ifdef BCMDBG
+void
+dll_pool_dump(dll_pool_t * dll_pool_p, dll_elem_dump elem_dump)
+{
+	dll_t * elem_p;
+	dll_t * next_p;
+	printf("dll_pool<%p> free_count<%u> elems_max<%u> elem_size<%u>\n",
+		OSL_OBFUSCATE_BUF(dll_pool_p), dll_pool_p->free_count,
+		dll_pool_p->elems_max, dll_pool_p->elem_size);
+
+	for (elem_p = dll_head_p(&dll_pool_p->free_list);
+		 !dll_end(&dll_pool_p->free_list, elem_p); elem_p = next_p) {
+
+		next_p = dll_next_p(elem_p);
+		printf("\telem<%p>\n", OSL_OBFUSCATE_BUF(elem_p));
+		if (elem_dump != NULL)
+			elem_dump((void *)elem_p);
+	}
+}
+#endif /* BCMDBG */
+
+#endif /* BCMDRIVER */
+
+#if defined(BCMDRIVER) || defined(WL_UNITTEST)
+
+#ifndef DONGLEBUILD
+/* triggers bcm_bprintf to print to kernel log */
+bool bcm_bprintf_bypass = FALSE;
+#endif /* !DONGLEBUILD */
+
+#if !(defined(_WIN32) || defined(NDIS))
+/* TODO: add vprintf to ndis OSL and remove this conditional */
+#define BCM_BPRINTF_ALLOW_NULL_B
+#endif
+
+/* Initialization of bcmstrbuf structure */
+void
+BCMPOSTTRAPFN(bcm_binit)(struct bcmstrbuf *b, char *buf, uint size)
+{
+#ifdef BCM_BPRINTF_ALLOW_NULL_B
+	/* pass NULL to struct bcmstrbuf *b to indicate the output is console */
+	if (b == NULL) {
+		return;
+	}
+#endif /* BCM_BPRINTF_ALLOW_NULL_B */
+
+	b->origsize = b->size = size;
+	b->origbuf = b->buf = buf;
+	if (size > 0) {
+		buf[0] = '\0';
+	}
+}
+
+/* Buffer sprintf wrapper to guard against buffer overflow */
+int
+BCMPOSTTRAPFN(bcm_bprintf)(struct bcmstrbuf *b, const char *fmt, ...)
+{
+	va_list ap;
+	int r;
+
+	va_start(ap, fmt);
+
+#ifdef BCM_BPRINTF_ALLOW_NULL_B
+	/* pass NULL to struct bcmstrbuf *b to indicate the output is console */
+	if (b == NULL) {
+		r = vprintf(fmt, ap);
+		goto exit;
+	}
+#endif /* BCM_BPRINTF_ALLOW_NULL_B */
+
+	r = vsnprintf(b->buf, b->size, fmt, ap);
+
+#ifndef DONGLEBUILD
+	if (bcm_bprintf_bypass == TRUE) {
+		printf("%s", b->buf);
+		goto exit;
+	}
+#endif /* !DONGLEBUILD */
+
+	/* Non Ansi C99 compliant returns -1,
+	 * Ansi compliant return r >= b->size,
+	 * bcmstdlib returns 0, handle all
+	 */
+	/* r == 0 is also the case when strlen(fmt) is zero.
+	 * typically the case when "" is passed as argument.
+	 */
+	if ((r == -1) || (r >= (int)b->size)) {
+		b->size = 0;
+	} else {
+		b->size -= (uint)r;
+		b->buf += r;
+	}
+
+exit:
+	va_end(ap);
+
+	return r;
+}
+
+void
+bcm_bprhex(struct bcmstrbuf *b, const char *msg, bool newline, const uint8 *buf, uint len)
+{
+	uint i;
+
+	if (msg != NULL && msg[0] != '\0')
+		bcm_bprintf(b, "%s", msg);
+	for (i = 0u; i < len; i ++)
+		bcm_bprintf(b, "%02X", buf[i]);
+	if (newline)
+		bcm_bprintf(b, "\n");
+}
+
+/* print the buffer in hex string format with the most significant byte first */
+void
+bcm_bprhex_msb(struct bcmstrbuf *b, const char *msg, bool newline,
+	const uint8 *buf, uint len)
+{
+	int i;
+
+	if (msg != NULL && msg[0] != '\0') {
+		bcm_bprintf(b, "%s", msg);
+	}
+	for (i = (int)len - 1; i >= 0; i --) {
+		bcm_bprintf(b, "%02X", buf[i]);
+	}
+	if (newline) {
+		bcm_bprintf(b, "\n");
+	}
+}
+
+void
+bcm_inc_bytes(uchar *num, int num_bytes, uint8 amount)
+{
+	int i;
+
+	for (i = 0; i < num_bytes; i++) {
+		num[i] += amount;
+		if (num[i] >= amount)
+			break;
+		amount = 1;
+	}
+}
+
+int
+bcm_cmp_bytes(const uchar *arg1, const uchar *arg2, uint8 nbytes)
+{
+	int i;
+
+	for (i = nbytes - 1; i >= 0; i--) {
+		if (arg1[i] != arg2[i])
+			return (arg1[i] - arg2[i]);
+	}
+	return 0;
+}
+
+void
+bcm_print_bytes(const char *name, const uchar *data, uint len)
+{
+	uint i;
+	int per_line = 0;
+
+	printf("%s: %d \n", name ? name : "", len);
+	for (i = 0u; i < len; i++) {
+		printf("%02x ", *data++);
+		per_line++;
+		if (per_line == 16) {
+			per_line = 0;
+			printf("\n");
+		}
+	}
+	printf("\n");
+}
+
+/* Search for an IE having a specific tag and an OUI type from a buffer.
+ * tlvs: buffer to search for IE
+ * tlvs_len: buffer length
+ * tag: IE tag
+ * oui: Specific OUI to match
+ * oui_len: length of the OUI
+ * type: OUI type
+ * Return the matched IE, else return null.
+*/
+bcm_tlv_t *
+bcm_find_ie(const uint8* tlvs, uint tlvs_len, uint8 tag, uint8 oui_len,
+	const char *oui, uint8 type)
+{
+	const bcm_tlv_t *ie;
+
+	COV_TAINTED_DATA_SINK(tlvs_len);
+	COV_NEG_SINK(tlvs_len);
+
+	/* Walk through the IEs looking for an OUI match */
+	while ((ie = bcm_parse_tlvs_advance(&tlvs, &tlvs_len, tag,
+		BCM_TLV_ADVANCE_TO))) {
+		if ((ie->len > oui_len) &&
+		    !bcmp(ie->data, oui, oui_len) &&
+			ie->data[oui_len] == type) {
+
+			COV_TAINTED_DATA_ARG(ie);
+
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			return (bcm_tlv_t *)(ie);		/* a match */
+			GCC_DIAGNOSTIC_POP();
+		}
+		/* Point to the next IE */
+		bcm_tlv_buffer_advance_past(ie, &tlvs, &tlvs_len);
+	}
+
+	return NULL;
+}
+
+/* Look for vendor-specific IE with specified OUI and optional type */
+bcm_tlv_t *
+bcm_find_vendor_ie(const  void *tlvs, uint tlvs_len, const char *voui, uint8 *type, uint type_len)
+{
+	const  bcm_tlv_t *ie;
+	uint8 ie_len;
+
+	COV_TAINTED_DATA_SINK(tlvs_len);
+	COV_NEG_SINK(tlvs_len);
+
+	ie = (const  bcm_tlv_t*)tlvs;
+
+	/* make sure we are looking at a valid IE */
+	if (ie == NULL || !bcm_valid_tlv(ie, tlvs_len)) {
+		return NULL;
+	}
+
+	/* Walk through the IEs looking for an OUI match */
+	do {
+		ie_len = ie->len;
+		if ((ie->id == DOT11_MNG_VS_ID) &&
+		    (ie_len >= (DOT11_OUI_LEN + type_len)) &&
+		    !bcmp(ie->data, voui, DOT11_OUI_LEN))
+		{
+			/* compare optional type */
+			if (type_len == 0 ||
+			    !bcmp(((const char *)ie->data) + DOT11_OUI_LEN, type, type_len)) {
+
+				COV_TAINTED_DATA_ARG(ie);
+
+				GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+				return (bcm_tlv_t *)(ie);		/* a match */
+				GCC_DIAGNOSTIC_POP();
+			}
+		}
+	} while ((ie = bcm_next_tlv(ie, &tlvs_len)) != NULL);
+
+	return NULL;
+}
+
+#if defined(WLTINYDUMP) || defined(BCMDBG) || defined(WLMSG_INFORM) || \
+	defined(WLMSG_ASSOC) || defined(WLMSG_PRPKT) || defined(WLMSG_WSEC)
+#define SSID_FMT_BUF_LEN	((4 * DOT11_MAX_SSID_LEN) + 1)
+
+int
+bcm_format_ssid(char* buf, const uchar ssid[], uint ssid_len)
+{
+	uint i, c;
+	char *p = buf;
+	char *endp = buf + SSID_FMT_BUF_LEN;
+
+	if (ssid_len > DOT11_MAX_SSID_LEN) ssid_len = DOT11_MAX_SSID_LEN;
+
+	for (i = 0; i < ssid_len; i++) {
+		c = (uint)ssid[i];
+		if (c == '\\') {
+			*p++ = '\\';
+			*p++ = '\\';
+		} else if (bcm_isprint((uchar)c)) {
+			*p++ = (char)c;
+		} else {
+			p += snprintf(p, (size_t)(endp - p), "\\x%02X", c);
+		}
+	}
+	*p = '\0';
+	ASSERT(p < endp);
+
+	return (int)(p - buf);
+}
+#endif /* WLTINYDUMP || BCMDBG || WLMSG_INFORM || WLMSG_ASSOC || WLMSG_PRPKT */
+
+#endif /* BCMDRIVER || WL_UNITTEST */
+
+/* Masking few bytes of MAC address per customer in all prints/eventlogs. */
+int
+BCMRAMFN(bcm_addrmask_set)(int enable)
+{
+#ifdef PRIVACY_MASK
+	struct ether_addr *privacy = privacy_addrmask_get();
+	if (enable) {
+		/* apply mask as (For SS)
+		 * orig		: 12:34:56:78:90:ab
+		 * masked	: 12:xx:xx:xx:x0:ab
+		 */
+		privacy->octet[1] = privacy->octet[2] =
+			privacy->octet[3] = 0;
+		privacy->octet[0] = privacy->octet[5] = 0xff;
+		privacy->octet[4] = 0x0f;
+	} else
+	{
+		/* No masking. All are 0xff. */
+		memcpy(privacy, &ether_bcast, sizeof(struct ether_addr));
+	}
+
+	return BCME_OK;
+#else
+	BCM_REFERENCE(enable);
+	return BCME_UNSUPPORTED;
+#endif /* PRIVACY_MASK */
+
+}
+
+int
+bcm_addrmask_get(int *val)
+{
+#ifdef PRIVACY_MASK
+	struct ether_addr *privacy = privacy_addrmask_get();
+	if (!eacmp(&ether_bcast, privacy)) {
+		*val = FALSE;
+	} else {
+		*val = TRUE;
+	}
+
+	return BCME_OK;
+#else
+	BCM_REFERENCE(val);
+	return BCME_UNSUPPORTED;
+#endif
+}
+
+uint64
+BCMRAMFN(bcm_ether_ntou64)(const struct ether_addr *ea)
+{
+	uint64 mac;
+	struct ether_addr addr;
+
+	memcpy(&addr, ea, sizeof(struct ether_addr));
+
+#ifdef PRIVACY_MASK
+	struct ether_addr *privacy = privacy_addrmask_get();
+	if (!ETHER_ISMULTI(ea)) {
+		*(uint32*)(&addr.octet[0]) &= *((uint32*)&privacy->octet[0]);
+		*(uint16*)(&addr.octet[4]) &= *((uint16*)&privacy->octet[4]);
+	}
+#endif /*  PRIVACY_MASK */
+
+	mac = ((uint64)HTON16(*((const uint16*)&addr.octet[4]))) << 32 |
+		HTON32(*((const uint32*)&addr.octet[0]));
+	return (mac);
+}
+
+char *
+bcm_ether_ntoa(const struct ether_addr *ea, char *buf)
+{
+	static const char hex[] =
+	  {
+		  '0', '1', '2', '3', '4', '5', '6', '7',
+		  '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
+	  };
+	const uint8 *octet = ea->octet;
+	char *p = buf;
+	int i;
+
+	for (i = 0; i < 6; i++, octet++) {
+		*p++ = hex[(*octet >> 4) & 0xf];
+		*p++ = hex[*octet & 0xf];
+		*p++ = ':';
+	}
+
+	*(p-1) = '\0';
+
+	return (buf);
+}
+
+/* Find the position of first bit set
+ * in the given number.
+ */
+int
+bcm_find_fsb(uint32 num)
+{
+	uint8 pos = 0;
+	if (!num)
+		return pos;
+	while (!(num & 1)) {
+		num >>= 1;
+		pos++;
+	}
+	return (pos+1);
+}
+
+/* TODO: need to pass in the buffer length for validation check */
+char *
+bcm_ip_ntoa(struct ipv4_addr *ia, char *buf)
+{
+	snprintf(buf, 16, "%d.%d.%d.%d",
+	         ia->addr[0], ia->addr[1], ia->addr[2], ia->addr[3]);
+	return (buf);
+}
+
+/* TODO: need to pass in the buffer length for validation check */
+char *
+bcm_ipv6_ntoa(void *ipv6, char *buf)
+{
+	/* Implementing RFC 5952 Sections 4 + 5 */
+	/* Not thoroughly tested */
+	uint16 tmp[8];
+	uint16 *a = &tmp[0];
+	char *p = buf;
+	int i, i_max = -1, cnt = 0, cnt_max = 1;
+	uint8 *a4 = NULL;
+	memcpy((uint8 *)&tmp[0], (uint8 *)ipv6, IPV6_ADDR_LEN);
+
+	for (i = 0; i < IPV6_ADDR_LEN/2; i++) {
+		if (a[i]) {
+			if (cnt > cnt_max) {
+				cnt_max = cnt;
+				i_max = i - cnt;
+			}
+			cnt = 0;
+		} else
+			cnt++;
+	}
+	if (cnt > cnt_max) {
+		cnt_max = cnt;
+		i_max = i - cnt;
+	}
+	if (i_max == 0 &&
+		/* IPv4-translated: ::ffff:0:a.b.c.d */
+		((cnt_max == 4 && a[4] == 0xffff && a[5] == 0) ||
+		/* IPv4-mapped: ::ffff:a.b.c.d */
+		(cnt_max == 5 && a[5] == 0xffff)))
+		a4 = (uint8*) (a + 6);
+
+	for (i = 0; i < IPV6_ADDR_LEN/2; i++) {
+		if ((uint8*) (a + i) == a4) {
+			snprintf(p, 17, ":%u.%u.%u.%u", a4[0], a4[1], a4[2], a4[3]);
+			break;
+		} else if (i == i_max) {
+			*p++ = ':';
+			i += cnt_max - 1;
+			p[0] = ':';
+			p[1] = '\0';
+		} else {
+			if (i)
+				*p++ = ':';
+			p += snprintf(p, 8, "%x", ntoh16(a[i]));
+		}
+	}
+
+	return buf;
+}
+
+#if !defined(BCMROMOFFLOAD_EXCLUDE_BCMUTILS_FUNCS)
+const unsigned char BCMPOST_TRAP_RODATA(bcm_ctype)[256] = {
+
+	_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,			/* 0-7 */
+	_BCM_C, _BCM_C|_BCM_S, _BCM_C|_BCM_S, _BCM_C|_BCM_S, _BCM_C|_BCM_S, _BCM_C|_BCM_S, _BCM_C,
+	_BCM_C,	/* 8-15 */
+	_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,			/* 16-23 */
+	_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,_BCM_C,			/* 24-31 */
+	_BCM_S|_BCM_SP,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,		/* 32-39 */
+	_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,			/* 40-47 */
+	_BCM_D,_BCM_D,_BCM_D,_BCM_D,_BCM_D,_BCM_D,_BCM_D,_BCM_D,			/* 48-55 */
+	_BCM_D,_BCM_D,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,			/* 56-63 */
+	_BCM_P, _BCM_U|_BCM_X, _BCM_U|_BCM_X, _BCM_U|_BCM_X, _BCM_U|_BCM_X, _BCM_U|_BCM_X,
+	_BCM_U|_BCM_X, _BCM_U, /* 64-71 */
+	_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,			/* 72-79 */
+	_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,_BCM_U,			/* 80-87 */
+	_BCM_U,_BCM_U,_BCM_U,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_P,			/* 88-95 */
+	_BCM_P, _BCM_L|_BCM_X, _BCM_L|_BCM_X, _BCM_L|_BCM_X, _BCM_L|_BCM_X, _BCM_L|_BCM_X,
+	_BCM_L|_BCM_X, _BCM_L, /* 96-103 */
+	_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L, /* 104-111 */
+	_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L,_BCM_L, /* 112-119 */
+	_BCM_L,_BCM_L,_BCM_L,_BCM_P,_BCM_P,_BCM_P,_BCM_P,_BCM_C, /* 120-127 */
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,		/* 128-143 */
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,		/* 144-159 */
+	_BCM_S|_BCM_SP, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P,
+	_BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P,	/* 160-175 */
+	_BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P,
+	_BCM_P, _BCM_P, _BCM_P, _BCM_P, _BCM_P,	/* 176-191 */
+	_BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U,
+	_BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U,	/* 192-207 */
+	_BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_P, _BCM_U, _BCM_U, _BCM_U,
+	_BCM_U, _BCM_U, _BCM_U, _BCM_U, _BCM_L,	/* 208-223 */
+	_BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L,
+	_BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L,	/* 224-239 */
+	_BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_P, _BCM_L, _BCM_L, _BCM_L,
+	_BCM_L, _BCM_L, _BCM_L, _BCM_L, _BCM_L /* 240-255 */
+};
+
+uint64
+bcm_strtoull(const char *cp, char **endp, uint base)
+{
+	uint64 result, last_result = 0, value;
+	bool minus;
+
+	minus = FALSE;
+
+	while (bcm_isspace(*cp))
+		cp++;
+
+	if (cp[0] == '+')
+		cp++;
+	else if (cp[0] == '-') {
+		minus = TRUE;
+		cp++;
+	}
+
+	if (base == 0) {
+		if (cp[0] == '0') {
+			if ((cp[1] == 'x') || (cp[1] == 'X')) {
+				base = 16;
+				cp = &cp[2];
+			} else {
+				base = 8;
+				cp = &cp[1];
+			}
+		} else
+			base = 10;
+	} else if (base == 16 && (cp[0] == '0') && ((cp[1] == 'x') || (cp[1] == 'X'))) {
+		cp = &cp[2];
+	}
+
+	result = 0;
+
+	while (bcm_isxdigit(*cp) &&
+	       (value = (uint64)(bcm_isdigit(*cp) ? *cp-'0' : bcm_toupper(*cp)-'A'+10)) < base) {
+		result = result*base + value;
+		/* Detected overflow */
+		if (result < last_result && !minus) {
+			if (endp) {
+				/* Go to the end of current number */
+				while (bcm_isxdigit(*cp)) {
+					cp++;
+				}
+				*endp = DISCARD_QUAL(cp, char);
+			}
+			return (ulong)-1;
+		}
+		last_result = result;
+		cp++;
+	}
+
+	if (minus)
+		result = (ulong)(-(long)result);
+
+	if (endp)
+		*endp = DISCARD_QUAL(cp, char);
+
+	return (result);
+}
+
+ulong
+bcm_strtoul(const char *cp, char **endp, uint base)
+{
+	return (ulong) bcm_strtoull(cp, endp, base);
+}
+
+int
+bcm_atoi(const char *s)
+{
+	return (int)bcm_strtoul(s, NULL, 10);
+}
+
+/* return pointer to location of substring 'needle' in 'haystack' */
+char *
+bcmstrstr(const char *haystack, const char *needle)
+{
+	uint len, nlen;
+	uint i;
+
+	if ((haystack == NULL) || (needle == NULL))
+		return DISCARD_QUAL(haystack, char);
+
+	nlen = (uint)strlen(needle);
+	if (strlen(haystack) < nlen) {
+		return NULL;
+	}
+	len = (uint)strlen(haystack) - nlen + 1u;
+
+	for (i = 0u; i < len; i++)
+		if (memcmp(needle, &haystack[i], nlen) == 0)
+			return DISCARD_QUAL(&haystack[i], char);
+	return (NULL);
+}
+
+char *
+bcmstrnstr(const char *s, uint s_len, const char *substr, uint substr_len)
+{
+	for (; s_len >= substr_len; s++, s_len--)
+		if (strncmp(s, substr, substr_len) == 0)
+			return DISCARD_QUAL(s, char);
+
+	return NULL;
+}
+
+char *
+bcmstrcat(char *dest, const char *src)
+{
+	char *p;
+
+	p = dest + strlen(dest);
+
+	while ((*p++ = *src++) != '\0')
+		;
+
+	return (dest);
+}
+
+char *
+bcmstrncat(char *dest, const char *src, uint size)
+{
+	char *endp;
+	char *p;
+
+	p = dest + strlen(dest);
+	endp = p + size;
+
+	while (p != endp && (*p++ = *src++) != '\0')
+		;
+
+	return (dest);
+}
+
+/****************************************************************************
+* Function:   bcmstrtok
+*
+* Purpose:
+*  Tokenizes a string. This function is conceptually similiar to ANSI C strtok(),
+*  but allows strToken() to be used by different strings or callers at the same
+*  time. Each call modifies '*string' by substituting a NULL character for the
+*  first delimiter that is encountered, and updates 'string' to point to the char
+*  after the delimiter. Leading delimiters are skipped.
+*
+* Parameters:
+*  string      (mod) Ptr to string ptr, updated by token.
+*  delimiters  (in)  Set of delimiter characters.
+*  tokdelim    (out) Character that delimits the returned token. (May
+*                    be set to NULL if token delimiter is not required).
+*
+* Returns:  Pointer to the next token found. NULL when no more tokens are found.
+*****************************************************************************
+*/
+char *
+bcmstrtok(char **string, const char *delimiters, char *tokdelim)
+{
+	unsigned char *str;
+	unsigned long map[8];
+	int count;
+	char *nextoken;
+
+	if (tokdelim != NULL) {
+		/* Prime the token delimiter */
+		*tokdelim = '\0';
+	}
+
+	/* Clear control map */
+	for (count = 0; count < 8; count++) {
+		map[count] = 0;
+	}
+
+	/* Set bits in delimiter table */
+	do {
+		map[*delimiters >> 5] |= (1UL << (*delimiters & 31));
+	}
+	while (*delimiters++);
+
+	str = (unsigned char*)*string;
+
+	/* Find beginning of token (skip over leading delimiters). Note that
+	 * there is no token iff this loop sets str to point to the terminal
+	 * null (*str == '\0')
+	 */
+	while (((map[*str >> 5] & (1UL << (*str & 31))) && *str) || (*str == ' ')) {
+		str++;
+	}
+
+	nextoken = (char*)str;
+
+	/* Find the end of the token. If it is not the end of the string,
+	 * put a null there.
+	 */
+	for (; *str; str++) {
+		if (map[*str >> 5] & (1UL << (*str & 31))) {
+			if (tokdelim != NULL) {
+				*tokdelim = *str;
+			}
+
+			*str++ = '\0';
+			break;
+		}
+	}
+
+	*string = (char*)str;
+
+	/* Determine if a token has been found. */
+	if (nextoken == (char *) str) {
+		return NULL;
+	}
+	else {
+		return nextoken;
+	}
+}
+
+#define xToLower(C) \
+	((C >= 'A' && C <= 'Z') ? (char)((int)C - (int)'A' + (int)'a') : C)
+
+/****************************************************************************
+* Function:   bcmstricmp
+*
+* Purpose:    Compare to strings case insensitively.
+*
+* Parameters: s1 (in) First string to compare.
+*             s2 (in) Second string to compare.
+*
+* Returns:    Return 0 if the two strings are equal, -1 if t1 < t2 and 1 if
+*             t1 > t2, when ignoring case sensitivity.
+*****************************************************************************
+*/
+int
+bcmstricmp(const char *s1, const char *s2)
+{
+	char dc, sc;
+
+	while (*s2 && *s1) {
+		dc = xToLower(*s1);
+		sc = xToLower(*s2);
+		if (dc < sc) return -1;
+		if (dc > sc) return 1;
+		s1++;
+		s2++;
+	}
+
+	if (*s1 && !*s2) return 1;
+	if (!*s1 && *s2) return -1;
+	return 0;
+}
+
+/****************************************************************************
+* Function:   bcmstrnicmp
+*
+* Purpose:    Compare to strings case insensitively, upto a max of 'cnt'
+*             characters.
+*
+* Parameters: s1  (in) First string to compare.
+*             s2  (in) Second string to compare.
+*             cnt (in) Max characters to compare.
+*
+* Returns:    Return 0 if the two strings are equal, -1 if t1 < t2 and 1 if
+*             t1 > t2, when ignoring case sensitivity.
+*****************************************************************************
+*/
+int
+bcmstrnicmp(const char* s1, const char* s2, int cnt)
+{
+	char dc, sc;
+
+	while (*s2 && *s1 && cnt) {
+		dc = xToLower(*s1);
+		sc = xToLower(*s2);
+		if (dc < sc) return -1;
+		if (dc > sc) return 1;
+		s1++;
+		s2++;
+		cnt--;
+	}
+
+	if (!cnt) return 0;
+	if (*s1 && !*s2) return 1;
+	if (!*s1 && *s2) return -1;
+	return 0;
+}
+
+/* parse a xx:xx:xx:xx:xx:xx format ethernet address */
+int
+bcm_ether_atoe(const char *p, struct ether_addr *ea)
+{
+	int i = 0;
+	char *ep;
+
+	for (;;) {
+		ea->octet[i++] = (char) bcm_strtoul(p, &ep, 16);
+		p = ep;
+		if (!*p++ || i == 6)
+			break;
+	}
+
+	return (i == 6);
+}
+
+/* parse a nnn.nnn.nnn.nnn format IPV4 address */
+int
+bcm_atoipv4(const char *p, struct ipv4_addr *ip)
+{
+
+	int i = 0;
+	char *c;
+	for (;;) {
+		ip->addr[i++] = (uint8)bcm_strtoul(p, &c, 0);
+		if (*c++ != '.' || i == IPV4_ADDR_LEN)
+			break;
+		p = c;
+	}
+	return (i == IPV4_ADDR_LEN);
+}
+#endif	/* !BCMROMOFFLOAD_EXCLUDE_BCMUTILS_FUNCS */
+
+const struct ether_addr ether_bcast = {{255, 255, 255, 255, 255, 255}};
+const struct ether_addr BCMPOST_TRAP_RODATA(ether_null) = {{0, 0, 0, 0, 0, 0}};
+const struct ether_addr ether_ipv6_mcast = {{0x33, 0x33, 0x00, 0x00, 0x00, 0x01}};
+
+int
+ether_isbcast(const void *ea)
+{
+	return (memcmp(ea, &ether_bcast, sizeof(struct ether_addr)) == 0);
+}
+
+int
+BCMPOSTTRAPFN(ether_isnulladdr)(const void *ea)
+{
+	const uint8 *ea8 = (const uint8 *)ea;
+	return !(ea8[5] || ea8[4] || ea8[3] || ea8[2] || ea8[1] || ea8[0]);
+}
+
+#if defined(CONFIG_USBRNDIS_RETAIL) || defined(NDIS_MINIPORT_DRIVER)
+/* registry routine buffer preparation utility functions:
+ * parameter order is like strlcpy, but returns count
+ * of bytes copied. Minimum bytes copied is null char(1)/wchar(2)
+ */
+ulong
+wchar2ascii(char *abuf, ushort *wbuf, ushort wbuflen, ulong abuflen)
+{
+	ulong copyct = 1;
+	ushort i;
+
+	if (abuflen == 0)
+		return 0;
+
+	/* wbuflen is in bytes */
+	wbuflen /= sizeof(ushort);
+
+	for (i = 0; i < wbuflen; ++i) {
+		if (--abuflen == 0)
+			break;
+		*abuf++ = (char) *wbuf++;
+		++copyct;
+	}
+	*abuf = '\0';
+
+	return copyct;
+}
+#endif /* CONFIG_USBRNDIS_RETAIL || NDIS_MINIPORT_DRIVER */
+
+#ifdef BCM_OBJECT_TRACE
+
+#define BCM_OBJECT_MERGE_SAME_OBJ	0
+
+/* some place may add / remove the object to trace list for Linux: */
+/* add:    osl_alloc_skb dev_alloc_skb skb_realloc_headroom dhd_start_xmit */
+/* remove: osl_pktfree dev_kfree_skb netif_rx */
+
+#if defined(__linux__)
+#define BCM_OBJDBG_COUNT          (1024 * 100)
+static spinlock_t dbgobj_lock;
+#define	BCM_OBJDBG_LOCK_INIT()    spin_lock_init(&dbgobj_lock)
+#define	BCM_OBJDBG_LOCK_DESTROY()
+#define	BCM_OBJDBG_LOCK           spin_lock_irqsave
+#define	BCM_OBJDBG_UNLOCK         spin_unlock_irqrestore
+#else
+#define BCM_OBJDBG_COUNT          (256)
+#define BCM_OBJDBG_LOCK_INIT()
+#define	BCM_OBJDBG_LOCK_DESTROY()
+#define BCM_OBJDBG_LOCK(x, y)
+#define BCM_OBJDBG_UNLOCK(x, y)
+#endif /* else OS */
+
+#define BCM_OBJDBG_ADDTOHEAD      0
+#define BCM_OBJDBG_ADDTOTAIL      1
+
+#define BCM_OBJDBG_CALLER_LEN     32
+struct bcm_dbgobj {
+	struct bcm_dbgobj *prior;
+	struct bcm_dbgobj *next;
+	uint32 flag;
+	void   *obj;
+	uint32 obj_sn;
+	uint32 obj_state;
+	uint32 line;
+	char   caller[BCM_OBJDBG_CALLER_LEN];
+};
+
+static struct bcm_dbgobj *dbgobj_freehead = NULL;
+static struct bcm_dbgobj *dbgobj_freetail = NULL;
+static struct bcm_dbgobj *dbgobj_objhead = NULL;
+static struct bcm_dbgobj *dbgobj_objtail = NULL;
+
+static uint32 dbgobj_sn = 0;
+static int dbgobj_count = 0;
+static struct bcm_dbgobj bcm_dbg_objs[BCM_OBJDBG_COUNT];
+
+void
+bcm_object_trace_init(void)
+{
+	int i = 0;
+	BCM_OBJDBG_LOCK_INIT();
+	bzero(&bcm_dbg_objs, sizeof(struct bcm_dbgobj) * BCM_OBJDBG_COUNT);
+	dbgobj_freehead = &bcm_dbg_objs[0];
+	dbgobj_freetail = &bcm_dbg_objs[BCM_OBJDBG_COUNT - 1];
+
+	for (i = 0; i < BCM_OBJDBG_COUNT; ++i) {
+		bcm_dbg_objs[i].next = (i == (BCM_OBJDBG_COUNT - 1)) ?
+			dbgobj_freehead : &bcm_dbg_objs[i + 1];
+		bcm_dbg_objs[i].prior = (i == 0) ?
+			dbgobj_freetail : &bcm_dbg_objs[i - 1];
+	}
+}
+
+void
+bcm_object_trace_deinit(void)
+{
+	if (dbgobj_objhead || dbgobj_objtail) {
+		printf("bcm_object_trace_deinit: not all objects are released\n");
+		ASSERT(0);
+	}
+	BCM_OBJDBG_LOCK_DESTROY();
+}
+
+static void
+bcm_object_rm_list(struct bcm_dbgobj **head, struct bcm_dbgobj **tail,
+	struct bcm_dbgobj *dbgobj)
+{
+	if ((dbgobj == *head) && (dbgobj == *tail)) {
+		*head = NULL;
+		*tail = NULL;
+	} else if (dbgobj == *head) {
+		*head = (*head)->next;
+	} else if (dbgobj == *tail) {
+		*tail = (*tail)->prior;
+	}
+	dbgobj->next->prior = dbgobj->prior;
+	dbgobj->prior->next = dbgobj->next;
+}
+
+static void
+bcm_object_add_list(struct bcm_dbgobj **head, struct bcm_dbgobj **tail,
+	struct bcm_dbgobj *dbgobj, int addtotail)
+{
+	if (!(*head) && !(*tail)) {
+		*head = dbgobj;
+		*tail = dbgobj;
+		dbgobj->next = dbgobj;
+		dbgobj->prior = dbgobj;
+	} else if ((*head) && (*tail)) {
+		(*tail)->next = dbgobj;
+		(*head)->prior = dbgobj;
+		dbgobj->next = *head;
+		dbgobj->prior = *tail;
+		if (addtotail == BCM_OBJDBG_ADDTOTAIL)
+			*tail = dbgobj;
+		else
+			*head = dbgobj;
+	} else {
+		ASSERT(0); /* can't be this case */
+	}
+}
+
+static INLINE void
+bcm_object_movetoend(struct bcm_dbgobj **head, struct bcm_dbgobj **tail,
+	struct bcm_dbgobj *dbgobj, int movetotail)
+{
+	if ((*head) && (*tail)) {
+		if (movetotail == BCM_OBJDBG_ADDTOTAIL) {
+			if (dbgobj != (*tail)) {
+				bcm_object_rm_list(head, tail, dbgobj);
+				bcm_object_add_list(head, tail, dbgobj, movetotail);
+			}
+		} else {
+			if (dbgobj != (*head)) {
+				bcm_object_rm_list(head, tail, dbgobj);
+				bcm_object_add_list(head, tail, dbgobj, movetotail);
+			}
+		}
+	} else {
+		ASSERT(0); /* can't be this case */
+	}
+}
+
+void
+bcm_object_trace_opr(void *obj, uint32 opt, const char *caller, int line)
+{
+	struct bcm_dbgobj *dbgobj;
+	unsigned long flags;
+
+	BCM_REFERENCE(flags);
+	BCM_OBJDBG_LOCK(&dbgobj_lock, flags);
+
+	if (opt == BCM_OBJDBG_ADD_PKT ||
+		opt == BCM_OBJDBG_ADD) {
+		dbgobj = dbgobj_objtail;
+		while (dbgobj) {
+			if (dbgobj->obj == obj) {
+				printf("bcm_object_trace_opr: obj %p allocated from %s(%d),"
+					" allocate again from %s(%d)\n",
+					dbgobj->obj,
+					dbgobj->caller, dbgobj->line,
+					caller, line);
+				ASSERT(0);
+				goto EXIT;
+			}
+			dbgobj = dbgobj->prior;
+			if (dbgobj == dbgobj_objtail)
+				break;
+		}
+
+#if BCM_OBJECT_MERGE_SAME_OBJ
+		dbgobj = dbgobj_freetail;
+		while (dbgobj) {
+			if (dbgobj->obj == obj) {
+				goto FREED_ENTRY_FOUND;
+			}
+			dbgobj = dbgobj->prior;
+			if (dbgobj == dbgobj_freetail)
+				break;
+		}
+#endif /* BCM_OBJECT_MERGE_SAME_OBJ */
+
+		dbgobj = dbgobj_freehead;
+#if BCM_OBJECT_MERGE_SAME_OBJ
+FREED_ENTRY_FOUND:
+#endif /* BCM_OBJECT_MERGE_SAME_OBJ */
+		if (!dbgobj) {
+			printf("bcm_object_trace_opr: already got %d objects ?????????????????\n",
+				BCM_OBJDBG_COUNT);
+			ASSERT(0);
+			goto EXIT;
+		}
+
+		bcm_object_rm_list(&dbgobj_freehead, &dbgobj_freetail, dbgobj);
+		dbgobj->obj = obj;
+		strlcpy(dbgobj->caller, caller, sizeof(dbgobj->caller));
+		dbgobj->line = line;
+		dbgobj->flag = 0;
+		if (opt == BCM_OBJDBG_ADD_PKT) {
+			dbgobj->obj_sn = dbgobj_sn++;
+			dbgobj->obj_state = 0;
+			/* first 4 bytes is pkt sn */
+			if (((unsigned long)PKTTAG(obj)) & 0x3)
+				printf("pkt tag address not aligned by 4: %p\n", PKTTAG(obj));
+			*(uint32*)PKTTAG(obj) = dbgobj->obj_sn;
+		}
+		bcm_object_add_list(&dbgobj_objhead, &dbgobj_objtail, dbgobj,
+			BCM_OBJDBG_ADDTOTAIL);
+
+		dbgobj_count++;
+
+	} else if (opt == BCM_OBJDBG_REMOVE) {
+		dbgobj = dbgobj_objtail;
+		while (dbgobj) {
+			if (dbgobj->obj == obj) {
+				if (dbgobj->flag) {
+					printf("bcm_object_trace_opr: rm flagged obj %p"
+						" flag 0x%08x from %s(%d)\n",
+						obj, dbgobj->flag, caller, line);
+				}
+				bcm_object_rm_list(&dbgobj_objhead, &dbgobj_objtail, dbgobj);
+				bzero(dbgobj->caller, sizeof(dbgobj->caller));
+				strlcpy(dbgobj->caller, caller, sizeof(dbgobj->caller));
+				dbgobj->line = line;
+				bcm_object_add_list(&dbgobj_freehead, &dbgobj_freetail, dbgobj,
+					BCM_OBJDBG_ADDTOTAIL);
+				dbgobj_count--;
+				goto EXIT;
+			}
+			dbgobj = dbgobj->prior;
+			if (dbgobj == dbgobj_objtail)
+				break;
+		}
+
+		dbgobj = dbgobj_freetail;
+		while (dbgobj && dbgobj->obj) {
+			if (dbgobj->obj == obj) {
+				printf("bcm_object_trace_opr: obj %p already freed"
+					" from from %s(%d),"
+					" try free again from %s(%d)\n",
+					obj,
+					dbgobj->caller, dbgobj->line,
+					caller, line);
+				//ASSERT(0); /* release same obj more than one time? */
+				goto EXIT;
+			}
+			dbgobj = dbgobj->prior;
+			if (dbgobj == dbgobj_freetail)
+				break;
+		}
+
+		printf("bcm_object_trace_opr: ################### release none-existing"
+			" obj %p from %s(%d)\n",
+			obj, caller, line);
+		//ASSERT(0); /* release same obj more than one time? */
+
+	}
+
+EXIT:
+	BCM_OBJDBG_UNLOCK(&dbgobj_lock, flags);
+	return;
+}
+
+void
+bcm_object_trace_upd(void *obj, void *obj_new)
+{
+	struct bcm_dbgobj *dbgobj;
+	unsigned long flags;
+
+	BCM_REFERENCE(flags);
+	BCM_OBJDBG_LOCK(&dbgobj_lock, flags);
+
+	dbgobj = dbgobj_objtail;
+	while (dbgobj) {
+		if (dbgobj->obj == obj) {
+			dbgobj->obj = obj_new;
+			if (dbgobj != dbgobj_objtail) {
+				bcm_object_movetoend(&dbgobj_objhead, &dbgobj_objtail,
+					dbgobj, BCM_OBJDBG_ADDTOTAIL);
+			}
+			goto EXIT;
+		}
+		dbgobj = dbgobj->prior;
+		if (dbgobj == dbgobj_objtail)
+			break;
+	}
+
+EXIT:
+	BCM_OBJDBG_UNLOCK(&dbgobj_lock, flags);
+	return;
+}
+
+void
+bcm_object_trace_chk(void *obj, uint32 chksn, uint32 sn,
+	const char *caller, int line)
+{
+	struct bcm_dbgobj *dbgobj;
+	unsigned long flags;
+
+	BCM_REFERENCE(flags);
+	BCM_OBJDBG_LOCK(&dbgobj_lock, flags);
+
+	dbgobj = dbgobj_objtail;
+	while (dbgobj) {
+		if ((dbgobj->obj == obj) &&
+			((!chksn) || (dbgobj->obj_sn == sn))) {
+			if (dbgobj != dbgobj_objtail) {
+				bcm_object_movetoend(&dbgobj_objhead, &dbgobj_objtail,
+					dbgobj, BCM_OBJDBG_ADDTOTAIL);
+			}
+			goto EXIT;
+		}
+		dbgobj = dbgobj->prior;
+		if (dbgobj == dbgobj_objtail)
+			break;
+	}
+
+	dbgobj = dbgobj_freetail;
+	while (dbgobj) {
+		if ((dbgobj->obj == obj) &&
+			((!chksn) || (dbgobj->obj_sn == sn))) {
+			printf("bcm_object_trace_chk: (%s:%d) obj %p (sn %d state %d)"
+				" was freed from %s(%d)\n",
+				caller, line,
+				dbgobj->obj, dbgobj->obj_sn, dbgobj->obj_state,
+				dbgobj->caller, dbgobj->line);
+			goto EXIT;
+		}
+		else if (dbgobj->obj == NULL) {
+			break;
+		}
+		dbgobj = dbgobj->prior;
+		if (dbgobj == dbgobj_freetail)
+			break;
+	}
+
+	printf("bcm_object_trace_chk: obj %p not found, check from %s(%d), chksn %s, sn %d\n",
+		obj, caller, line, chksn ? "yes" : "no", sn);
+	dbgobj = dbgobj_objtail;
+	while (dbgobj) {
+		printf("bcm_object_trace_chk: (%s:%d) obj %p sn %d was allocated from %s(%d)\n",
+				caller, line,
+				dbgobj->obj, dbgobj->obj_sn, dbgobj->caller, dbgobj->line);
+		dbgobj = dbgobj->prior;
+		if (dbgobj == dbgobj_objtail)
+			break;
+	}
+
+EXIT:
+	BCM_OBJDBG_UNLOCK(&dbgobj_lock, flags);
+	return;
+}
+
+void
+bcm_object_feature_set(void *obj, uint32 type, uint32 value)
+{
+	struct bcm_dbgobj *dbgobj;
+	unsigned long flags;
+
+	BCM_REFERENCE(flags);
+	BCM_OBJDBG_LOCK(&dbgobj_lock, flags);
+
+	dbgobj = dbgobj_objtail;
+	while (dbgobj) {
+		if (dbgobj->obj == obj) {
+			if (type == BCM_OBJECT_FEATURE_FLAG) {
+				if (value & BCM_OBJECT_FEATURE_CLEAR)
+					dbgobj->flag &= ~(value);
+				else
+					dbgobj->flag |= (value);
+			} else if (type == BCM_OBJECT_FEATURE_PKT_STATE) {
+				dbgobj->obj_state = value;
+			}
+			if (dbgobj != dbgobj_objtail) {
+				bcm_object_movetoend(&dbgobj_objhead, &dbgobj_objtail,
+					dbgobj, BCM_OBJDBG_ADDTOTAIL);
+			}
+			goto EXIT;
+		}
+		dbgobj = dbgobj->prior;
+		if (dbgobj == dbgobj_objtail)
+			break;
+	}
+
+	printf("bcm_object_feature_set: obj %p not found in active list\n", obj);
+	ASSERT(0);
+
+EXIT:
+	BCM_OBJDBG_UNLOCK(&dbgobj_lock, flags);
+	return;
+}
+
+int
+bcm_object_feature_get(void *obj, uint32 type, uint32 value)
+{
+	int rtn = 0;
+	struct bcm_dbgobj *dbgobj;
+	unsigned long flags;
+
+	BCM_REFERENCE(flags);
+	BCM_OBJDBG_LOCK(&dbgobj_lock, flags);
+
+	dbgobj = dbgobj_objtail;
+	while (dbgobj) {
+		if (dbgobj->obj == obj) {
+			if (type == BCM_OBJECT_FEATURE_FLAG) {
+				rtn = (dbgobj->flag & value) & (~BCM_OBJECT_FEATURE_CLEAR);
+			}
+			if (dbgobj != dbgobj_objtail) {
+				bcm_object_movetoend(&dbgobj_objhead, &dbgobj_objtail,
+					dbgobj, BCM_OBJDBG_ADDTOTAIL);
+			}
+			goto EXIT;
+		}
+		dbgobj = dbgobj->prior;
+		if (dbgobj == dbgobj_objtail)
+			break;
+	}
+
+	printf("bcm_object_feature_get: obj %p not found in active list\n", obj);
+	ASSERT(0);
+
+EXIT:
+	BCM_OBJDBG_UNLOCK(&dbgobj_lock, flags);
+	return rtn;
+}
+
+#endif /* BCM_OBJECT_TRACE */
+
+uint8 *
+BCMPOSTTRAPFN(bcm_write_tlv)(int type, const void *data, uint datalen, uint8 *dst)
+{
+	uint8 *new_dst = dst;
+	bcm_tlv_t *dst_tlv = (bcm_tlv_t *)dst;
+
+	/* dst buffer should always be valid */
+	ASSERT(dst);
+
+	/* data len must be within valid range */
+	ASSERT((datalen <= BCM_TLV_MAX_DATA_SIZE));
+
+	/* source data buffer pointer should be valid, unless datalen is 0
+	 * meaning no data with this TLV
+	 */
+	ASSERT((data != NULL) || (datalen == 0));
+
+	/* only do work if the inputs are valid
+	 * - must have a dst to write to AND
+	 * - datalen must be within range AND
+	 * - the source data pointer must be non-NULL if datalen is non-zero
+	 * (this last condition detects datalen > 0 with a NULL data pointer)
+	 */
+	if ((dst != NULL) &&
+	    ((datalen <= BCM_TLV_MAX_DATA_SIZE)) &&
+	    ((data != NULL) || (datalen == 0u))) {
+
+		/* write type, len fields */
+		dst_tlv->id = (uint8)type;
+		dst_tlv->len = (uint8)datalen;
+
+		/* if data is present, copy to the output buffer and update
+		 * pointer to output buffer
+		 */
+		if (datalen > 0u) {
+
+			memcpy(dst_tlv->data, data, datalen);
+		}
+
+		/* update the output destination poitner to point past
+		 * the TLV written
+		 */
+		new_dst = dst + BCM_TLV_HDR_SIZE + datalen;
+	}
+
+	return (new_dst);
+}
+
+uint8 *
+bcm_write_tlv_ext(uint8 type, uint8 ext, const void *data, uint8 datalen, uint8 *dst)
+{
+	uint8 *new_dst = dst;
+	bcm_tlv_ext_t *dst_tlv = (bcm_tlv_ext_t *)dst;
+
+	/* dst buffer should always be valid */
+	ASSERT(dst);
+
+	/* data len must be within valid range */
+	ASSERT(datalen <= BCM_TLV_EXT_MAX_DATA_SIZE);
+
+	/* source data buffer pointer should be valid, unless datalen is 0
+	 * meaning no data with this TLV
+	 */
+	ASSERT((data != NULL) || (datalen == 0));
+
+	/* only do work if the inputs are valid
+	 * - must have a dst to write to AND
+	 * - datalen must be within range AND
+	 * - the source data pointer must be non-NULL if datalen is non-zero
+	 * (this last condition detects datalen > 0 with a NULL data pointer)
+	 */
+	if ((dst != NULL) &&
+	    (datalen <= BCM_TLV_EXT_MAX_DATA_SIZE) &&
+	    ((data != NULL) || (datalen == 0))) {
+
+		/* write type, len fields */
+		dst_tlv->id = (uint8)type;
+		dst_tlv->ext = ext;
+		dst_tlv->len = 1 + (uint8)datalen;
+
+		/* if data is present, copy to the output buffer and update
+		 * pointer to output buffer
+		 */
+		if (datalen > 0) {
+			memcpy(dst_tlv->data, data, datalen);
+		}
+
+		/* update the output destination poitner to point past
+		 * the TLV written
+		 */
+		new_dst = dst + BCM_TLV_EXT_HDR_SIZE + datalen;
+	}
+
+	return (new_dst);
+}
+
+uint8 *
+BCMPOSTTRAPFN(bcm_write_tlv_safe)(int type, const void *data, uint datalen, uint8 *dst,
+	uint dst_maxlen)
+{
+	uint8 *new_dst = dst;
+
+	if ((datalen <= BCM_TLV_MAX_DATA_SIZE)) {
+
+		/* if len + tlv hdr len is more than destlen, don't do anything
+		 * just return the buffer untouched
+		 */
+		if ((datalen + BCM_TLV_HDR_SIZE) <= dst_maxlen) {
+
+			new_dst = bcm_write_tlv(type, data, datalen, dst);
+		}
+	}
+
+	return (new_dst);
+}
+
+uint8 *
+bcm_copy_tlv(const void *src, uint8 *dst)
+{
+	uint8 *new_dst = dst;
+	const bcm_tlv_t *src_tlv = (const bcm_tlv_t *)src;
+	uint totlen;
+
+	ASSERT(dst && src);
+	if (dst && src) {
+
+		totlen = BCM_TLV_HDR_SIZE + src_tlv->len;
+		memcpy(dst, src_tlv, totlen);
+		new_dst = dst + totlen;
+	}
+
+	return (new_dst);
+}
+
+uint8 *
+bcm_copy_tlv_safe(const void *src, uint8 *dst, uint dst_maxlen)
+{
+	uint8 *new_dst = dst;
+	const bcm_tlv_t *src_tlv = (const bcm_tlv_t *)src;
+
+	ASSERT(src);
+	if (bcm_valid_tlv(src_tlv, dst_maxlen)) {
+		new_dst = bcm_copy_tlv(src, dst);
+	}
+
+	return (new_dst);
+}
+
+#if !defined(BCMROMOFFLOAD_EXCLUDE_BCMUTILS_FUNCS)
+/*******************************************************************************
+ * crc8
+ *
+ * Computes a crc8 over the input data using the polynomial:
+ *
+ *       x^8 + x^7 +x^6 + x^4 + x^2 + 1
+ *
+ * The caller provides the initial value (either CRC8_INIT_VALUE
+ * or the previous returned value) to allow for processing of
+ * discontiguous blocks of data.  When generating the CRC the
+ * caller is responsible for complementing the final return value
+ * and inserting it into the byte stream.  When checking, a final
+ * return value of CRC8_GOOD_VALUE indicates a valid CRC.
+ *
+ * Reference: Dallas Semiconductor Application Note 27
+ *   Williams, Ross N., "A Painless Guide to CRC Error Detection Algorithms",
+ *     ver 3, Aug 1993, ross@guest.adelaide.edu.au, Rocksoft Pty Ltd.,
+ *     ftp://ftp.rocksoft.com/clients/rocksoft/papers/crc_v3.txt
+ *
+ * ****************************************************************************
+ */
+
+static const uint8 crc8_table[256] = {
+    0x00, 0xF7, 0xB9, 0x4E, 0x25, 0xD2, 0x9C, 0x6B,
+    0x4A, 0xBD, 0xF3, 0x04, 0x6F, 0x98, 0xD6, 0x21,
+    0x94, 0x63, 0x2D, 0xDA, 0xB1, 0x46, 0x08, 0xFF,
+    0xDE, 0x29, 0x67, 0x90, 0xFB, 0x0C, 0x42, 0xB5,
+    0x7F, 0x88, 0xC6, 0x31, 0x5A, 0xAD, 0xE3, 0x14,
+    0x35, 0xC2, 0x8C, 0x7B, 0x10, 0xE7, 0xA9, 0x5E,
+    0xEB, 0x1C, 0x52, 0xA5, 0xCE, 0x39, 0x77, 0x80,
+    0xA1, 0x56, 0x18, 0xEF, 0x84, 0x73, 0x3D, 0xCA,
+    0xFE, 0x09, 0x47, 0xB0, 0xDB, 0x2C, 0x62, 0x95,
+    0xB4, 0x43, 0x0D, 0xFA, 0x91, 0x66, 0x28, 0xDF,
+    0x6A, 0x9D, 0xD3, 0x24, 0x4F, 0xB8, 0xF6, 0x01,
+    0x20, 0xD7, 0x99, 0x6E, 0x05, 0xF2, 0xBC, 0x4B,
+    0x81, 0x76, 0x38, 0xCF, 0xA4, 0x53, 0x1D, 0xEA,
+    0xCB, 0x3C, 0x72, 0x85, 0xEE, 0x19, 0x57, 0xA0,
+    0x15, 0xE2, 0xAC, 0x5B, 0x30, 0xC7, 0x89, 0x7E,
+    0x5F, 0xA8, 0xE6, 0x11, 0x7A, 0x8D, 0xC3, 0x34,
+    0xAB, 0x5C, 0x12, 0xE5, 0x8E, 0x79, 0x37, 0xC0,
+    0xE1, 0x16, 0x58, 0xAF, 0xC4, 0x33, 0x7D, 0x8A,
+    0x3F, 0xC8, 0x86, 0x71, 0x1A, 0xED, 0xA3, 0x54,
+    0x75, 0x82, 0xCC, 0x3B, 0x50, 0xA7, 0xE9, 0x1E,
+    0xD4, 0x23, 0x6D, 0x9A, 0xF1, 0x06, 0x48, 0xBF,
+    0x9E, 0x69, 0x27, 0xD0, 0xBB, 0x4C, 0x02, 0xF5,
+    0x40, 0xB7, 0xF9, 0x0E, 0x65, 0x92, 0xDC, 0x2B,
+    0x0A, 0xFD, 0xB3, 0x44, 0x2F, 0xD8, 0x96, 0x61,
+    0x55, 0xA2, 0xEC, 0x1B, 0x70, 0x87, 0xC9, 0x3E,
+    0x1F, 0xE8, 0xA6, 0x51, 0x3A, 0xCD, 0x83, 0x74,
+    0xC1, 0x36, 0x78, 0x8F, 0xE4, 0x13, 0x5D, 0xAA,
+    0x8B, 0x7C, 0x32, 0xC5, 0xAE, 0x59, 0x17, 0xE0,
+    0x2A, 0xDD, 0x93, 0x64, 0x0F, 0xF8, 0xB6, 0x41,
+    0x60, 0x97, 0xD9, 0x2E, 0x45, 0xB2, 0xFC, 0x0B,
+    0xBE, 0x49, 0x07, 0xF0, 0x9B, 0x6C, 0x22, 0xD5,
+    0xF4, 0x03, 0x4D, 0xBA, 0xD1, 0x26, 0x68, 0x9F
+};
+
+#define CRC_INNER_LOOP(n, c, x) \
+	(c) = ((c) >> 8) ^ crc##n##_table[((c) ^ (x)) & 0xff]
+
+uint8
+hndcrc8(
+	const uint8 *pdata,	/* pointer to array of data to process */
+	uint  nbytes,	/* number of input data bytes to process */
+	uint8 crc	/* either CRC8_INIT_VALUE or previous return value */
+)
+{
+	/* hard code the crc loop instead of using CRC_INNER_LOOP macro
+	 * to avoid the undefined and unnecessary (uint8 >> 8) operation.
+	 */
+	while (nbytes-- > 0)
+		crc = crc8_table[(crc ^ *pdata++) & 0xff];
+
+	return crc;
+}
+
+/*******************************************************************************
+ * crc16
+ *
+ * Computes a crc16 over the input data using the polynomial:
+ *
+ *       x^16 + x^12 +x^5 + 1
+ *
+ * The caller provides the initial value (either CRC16_INIT_VALUE
+ * or the previous returned value) to allow for processing of
+ * discontiguous blocks of data.  When generating the CRC the
+ * caller is responsible for complementing the final return value
+ * and inserting it into the byte stream.  When checking, a final
+ * return value of CRC16_GOOD_VALUE indicates a valid CRC.
+ *
+ * Reference: Dallas Semiconductor Application Note 27
+ *   Williams, Ross N., "A Painless Guide to CRC Error Detection Algorithms",
+ *     ver 3, Aug 1993, ross@guest.adelaide.edu.au, Rocksoft Pty Ltd.,
+ *     ftp://ftp.rocksoft.com/clients/rocksoft/papers/crc_v3.txt
+ *
+ * ****************************************************************************
+ */
+
+static const uint16 crc16_table[256] = {
+    0x0000, 0x1189, 0x2312, 0x329B, 0x4624, 0x57AD, 0x6536, 0x74BF,
+    0x8C48, 0x9DC1, 0xAF5A, 0xBED3, 0xCA6C, 0xDBE5, 0xE97E, 0xF8F7,
+    0x1081, 0x0108, 0x3393, 0x221A, 0x56A5, 0x472C, 0x75B7, 0x643E,
+    0x9CC9, 0x8D40, 0xBFDB, 0xAE52, 0xDAED, 0xCB64, 0xF9FF, 0xE876,
+    0x2102, 0x308B, 0x0210, 0x1399, 0x6726, 0x76AF, 0x4434, 0x55BD,
+    0xAD4A, 0xBCC3, 0x8E58, 0x9FD1, 0xEB6E, 0xFAE7, 0xC87C, 0xD9F5,
+    0x3183, 0x200A, 0x1291, 0x0318, 0x77A7, 0x662E, 0x54B5, 0x453C,
+    0xBDCB, 0xAC42, 0x9ED9, 0x8F50, 0xFBEF, 0xEA66, 0xD8FD, 0xC974,
+    0x4204, 0x538D, 0x6116, 0x709F, 0x0420, 0x15A9, 0x2732, 0x36BB,
+    0xCE4C, 0xDFC5, 0xED5E, 0xFCD7, 0x8868, 0x99E1, 0xAB7A, 0xBAF3,
+    0x5285, 0x430C, 0x7197, 0x601E, 0x14A1, 0x0528, 0x37B3, 0x263A,
+    0xDECD, 0xCF44, 0xFDDF, 0xEC56, 0x98E9, 0x8960, 0xBBFB, 0xAA72,
+    0x6306, 0x728F, 0x4014, 0x519D, 0x2522, 0x34AB, 0x0630, 0x17B9,
+    0xEF4E, 0xFEC7, 0xCC5C, 0xDDD5, 0xA96A, 0xB8E3, 0x8A78, 0x9BF1,
+    0x7387, 0x620E, 0x5095, 0x411C, 0x35A3, 0x242A, 0x16B1, 0x0738,
+    0xFFCF, 0xEE46, 0xDCDD, 0xCD54, 0xB9EB, 0xA862, 0x9AF9, 0x8B70,
+    0x8408, 0x9581, 0xA71A, 0xB693, 0xC22C, 0xD3A5, 0xE13E, 0xF0B7,
+    0x0840, 0x19C9, 0x2B52, 0x3ADB, 0x4E64, 0x5FED, 0x6D76, 0x7CFF,
+    0x9489, 0x8500, 0xB79B, 0xA612, 0xD2AD, 0xC324, 0xF1BF, 0xE036,
+    0x18C1, 0x0948, 0x3BD3, 0x2A5A, 0x5EE5, 0x4F6C, 0x7DF7, 0x6C7E,
+    0xA50A, 0xB483, 0x8618, 0x9791, 0xE32E, 0xF2A7, 0xC03C, 0xD1B5,
+    0x2942, 0x38CB, 0x0A50, 0x1BD9, 0x6F66, 0x7EEF, 0x4C74, 0x5DFD,
+    0xB58B, 0xA402, 0x9699, 0x8710, 0xF3AF, 0xE226, 0xD0BD, 0xC134,
+    0x39C3, 0x284A, 0x1AD1, 0x0B58, 0x7FE7, 0x6E6E, 0x5CF5, 0x4D7C,
+    0xC60C, 0xD785, 0xE51E, 0xF497, 0x8028, 0x91A1, 0xA33A, 0xB2B3,
+    0x4A44, 0x5BCD, 0x6956, 0x78DF, 0x0C60, 0x1DE9, 0x2F72, 0x3EFB,
+    0xD68D, 0xC704, 0xF59F, 0xE416, 0x90A9, 0x8120, 0xB3BB, 0xA232,
+    0x5AC5, 0x4B4C, 0x79D7, 0x685E, 0x1CE1, 0x0D68, 0x3FF3, 0x2E7A,
+    0xE70E, 0xF687, 0xC41C, 0xD595, 0xA12A, 0xB0A3, 0x8238, 0x93B1,
+    0x6B46, 0x7ACF, 0x4854, 0x59DD, 0x2D62, 0x3CEB, 0x0E70, 0x1FF9,
+    0xF78F, 0xE606, 0xD49D, 0xC514, 0xB1AB, 0xA022, 0x92B9, 0x8330,
+    0x7BC7, 0x6A4E, 0x58D5, 0x495C, 0x3DE3, 0x2C6A, 0x1EF1, 0x0F78
+};
+
+uint16
+hndcrc16(
+    const uint8 *pdata,  /* pointer to array of data to process */
+    uint nbytes, /* number of input data bytes to process */
+    uint16 crc     /* either CRC16_INIT_VALUE or previous return value */
+)
+{
+	while (nbytes-- > 0)
+		CRC_INNER_LOOP(16, crc, *pdata++);
+	return crc;
+}
+
+static const uint32 BCMPOST_TRAP_RODATA(crc32_table)[256] = {
+    0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
+    0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
+    0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
+    0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
+    0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
+    0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
+    0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
+    0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
+    0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
+    0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
+    0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940,
+    0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
+    0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
+    0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
+    0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
+    0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
+    0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
+    0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
+    0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
+    0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
+    0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
+    0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
+    0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
+    0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
+    0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
+    0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
+    0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
+    0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
+    0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
+    0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
+    0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
+    0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
+    0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
+    0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
+    0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
+    0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
+    0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
+    0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
+    0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
+    0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
+    0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
+    0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
+    0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
+    0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
+    0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
+    0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
+    0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04,
+    0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
+    0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
+    0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
+    0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
+    0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
+    0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
+    0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
+    0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
+    0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
+    0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
+    0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
+    0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
+    0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
+    0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
+    0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
+    0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
+    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
+};
+
+/*
+ * crc input is CRC32_INIT_VALUE for a fresh start, or previous return value if
+ * accumulating over multiple pieces.
+ */
+uint32
+BCMPOSTTRAPRAMFN(hndcrc32)(const uint8 *pdata, uint nbytes, uint32 crc)
+{
+	const uint8 *pend;
+	pend = pdata + nbytes;
+	while (pdata < pend)
+		CRC_INNER_LOOP(32, crc, *pdata++);
+
+	return crc;
+}
+
+/*
+ * Advance from the current 1-byte tag/1-byte length/variable-length value
+ * triple, to the next, returning a pointer to the next.
+ * If the current or next TLV is invalid (does not fit in given buffer length),
+ * NULL is returned.
+ * *buflen is not modified if the TLV elt parameter is invalid, or is decremented
+ * by the TLV parameter's length if it is valid.
+ */
+bcm_tlv_t *
+bcm_next_tlv(const bcm_tlv_t *elt, uint *buflen)
+{
+	uint len;
+
+	COV_TAINTED_DATA_SINK(buflen);
+	COV_NEG_SINK(buflen);
+
+	/* validate current elt */
+	if (!bcm_valid_tlv(elt, *buflen)) {
+		return NULL;
+	}
+
+	/* advance to next elt */
+	len = TLV_HDR_LEN + elt->len;
+	elt = (const  bcm_tlv_t*)((const uint8 *)elt + len);
+
+#if defined(__COVERITY__)
+	/* The 'len' value is tainted in Coverity because it is read from the tainted data pointed
+	 * to by 'elt'.  However, bcm_valid_tlv() verifies that the elt pointer is a valid element,
+	 * so its length, len = (TLV_HDR_LEN + elt->len), is in the bounds of the buffer.
+	 * Clearing the tainted attribute of 'len' for Coverity.
+	 */
+	__coverity_tainted_data_sanitize__(len);
+	if (len > *buflen) {
+		return NULL;
+	}
+#endif /* __COVERITY__ */
+
+	*buflen -= len;
+
+	/* validate next elt */
+	if (!bcm_valid_tlv(elt, *buflen)) {
+		return NULL;
+	}
+
+	COV_TAINTED_DATA_ARG(elt);
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	return (bcm_tlv_t *)(elt);
+	GCC_DIAGNOSTIC_POP();
+}
+
+/**
+ * Advance a const tlv buffer pointer and length up to the given tlv element pointer
+ * 'elt'.  The function checks that elt is a valid tlv; the elt pointer and data
+ * are all in the range of the buffer/length.
+ *
+ * @param elt      pointer to a valid bcm_tlv_t in the buffer
+ * @param buffer   pointer to a tlv buffer
+ * @param buflen   length of the buffer in bytes
+ *
+ * On return, if elt is not a tlv in the buffer bounds, the *buffer parameter
+ * will be set to NULL and *buflen parameter will be set to zero.  Otherwise,
+ * *buffer will point to elt, and *buflen will have been adjusted by the the
+ * difference between *buffer and elt.
+ */
+void
+bcm_tlv_buffer_advance_to(const bcm_tlv_t *elt, const uint8 **buffer, uint *buflen)
+{
+	uint new_buflen;
+	const uint8 *new_buffer;
+
+	/* model the input length value as a tainted and negative sink so
+	 * Coverity will complain about unvalidated or possibly length values
+	 */
+	COV_TAINTED_DATA_SINK(*buflen);
+	COV_NEG_SINK(*buflen);
+
+	new_buffer = (const uint8*)elt;
+
+	/* make sure the input buffer pointer is non-null, that (buffer + buflen) does not wrap,
+	 * and that the elt pointer is in the range of [buffer, buffer + buflen]
+	 */
+	if ((*buffer != NULL) &&
+	    ((uintptr)*buffer < ((uintptr)*buffer + *buflen)) &&
+	    (new_buffer >= *buffer) &&
+	    (new_buffer < (*buffer + *buflen))) {
+		/* delta between buffer and new_buffer is <= *buflen, so truncating cast to uint
+		 * from ptrdiff is ok
+		 */
+		uint delta = (uint)(new_buffer - *buffer);
+
+		/* New buffer length is old len minus the delta from the buffer start to elt.
+		 * The check just above guarantees that the subtractions does not underflow.
+		 */
+		new_buflen = *buflen - delta;
+
+		/* validate current elt */
+		if (bcm_valid_tlv(elt, new_buflen)) {
+			/* All good, so update the input/output parameters */
+			*buffer = new_buffer;
+			*buflen = new_buflen;
+			return;
+		}
+	}
+
+	/* something did not check out, clear out the buffer info */
+	*buffer = NULL;
+	*buflen = 0;
+
+	return;
+}
+
+/**
+ * Advance a const tlv buffer pointer and length past the given tlv element pointer
+ * 'elt'.  The function checks that elt is a valid tlv; the elt pointer and data
+ * are all in the range of the buffer/length.  The function also checks that the
+ * remaining buffer starts with a valid tlv.
+ *
+ * @param elt      pointer to a valid bcm_tlv_t in the buffer
+ * @param buffer   pointer to a tlv buffer
+ * @param buflen   length of the buffer in bytes
+ *
+ * On return, if elt is not a tlv in the buffer bounds, or the remaining buffer
+ * following the elt does not begin with a tlv in the buffer bounds, the *buffer
+ * parameter will be set to NULL and *buflen parameter will be set to zero.
+ * Otherwise, *buffer will point to the first byte past elt, and *buflen will
+ * have the remaining buffer length.
+ */
+void
+bcm_tlv_buffer_advance_past(const bcm_tlv_t *elt, const uint8 **buffer, uint *buflen)
+{
+	/* Start by advancing the buffer up to the given elt */
+	bcm_tlv_buffer_advance_to(elt, buffer, buflen);
+
+	/* if that did not work, bail out */
+	if (*buflen == 0) {
+		return;
+	}
+
+#if defined(__COVERITY__)
+	/* The elt has been verified by bcm_tlv_buffer_advance_to() to be a valid element,
+	 * so its elt->len is in the bounds of the buffer. The following check prevents
+	 * Coverity from flagging the (elt->data + elt->len) statement below as using a
+	 * tainted elt->len to index into array 'elt->data'.
+	 */
+	if (elt->len > *buflen) {
+		return;
+	}
+#endif /* __COVERITY__ */
+
+	/* We know we are advanced up to a good tlv.
+	 * Now just advance to the following tlv.
+	 */
+	elt = (const bcm_tlv_t*)(elt->data + elt->len);
+
+	bcm_tlv_buffer_advance_to(elt, buffer, buflen);
+
+	return;
+}
+
+/*
+ * Traverse a string of 1-byte tag/1-byte length/variable-length value
+ * triples, returning a pointer to the substring whose first element
+ * matches tag
+ */
+bcm_tlv_t *
+bcm_parse_tlvs(const void *buf, uint buflen, uint key)
+{
+	const bcm_tlv_t *elt;
+	uint totlen;
+
+	COV_TAINTED_DATA_SINK(buflen);
+	COV_NEG_SINK(buflen);
+
+	if ((elt = (const bcm_tlv_t*)buf) == NULL) {
+		return NULL;
+	}
+	totlen = buflen;
+
+	/* find tagged parameter */
+	while (totlen >= TLV_HDR_LEN) {
+		uint len = elt->len;
+
+		/* check if elt overruns buffer */
+		if (totlen < (len + TLV_HDR_LEN)) {
+			break;
+		}
+		/* did we find the ID? */
+		if ((elt->id == key) ||
+		    (elt->id == DOT11_MNG_ID_EXT_ID && len > 0 &&
+		     elt->data[0] + (uint)DOT11_MNG_ID_EXT_ID == key)) {
+			COV_TAINTED_DATA_ARG(elt);
+
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			return (bcm_tlv_t *)(elt);
+			GCC_DIAGNOSTIC_POP();
+		}
+		elt = (const bcm_tlv_t*)((const uint8*)elt + (len + TLV_HDR_LEN));
+		totlen -= (len + TLV_HDR_LEN);
+	}
+
+	return NULL;
+}
+
+/*
+ * Traverse a string of 1-byte tag/1-byte length/variable-length value
+ * triples, returning a pointer to the substring whose first element
+ * matches tag.
+ * The 'advance' parmeter specifies what to do to the parse buf/buflen values if a
+ * matching tlv is found:
+ *	BCM_TLV_ADVANCE_NONE - do nothing
+ *	BCM_TLV_ADVANCE_TO - move the buf up to the discovered tlv, and adjust buflen.
+ *	BCM_TLV_ADVANCE_PAST - move the buf past the discovered tlb, and adjust buflen.
+ * If a tlv is not found, no changes are made to buf/buflen
+ *
+ */
+const bcm_tlv_t *
+bcm_parse_tlvs_advance(const uint8 **buf, uint *buflen, uint key, bcm_tlv_advance_mode_t advance)
+{
+	const bcm_tlv_t *elt;
+
+	elt = bcm_parse_tlvs(*buf, *buflen, key);
+
+	if (elt == NULL) {
+		return elt;
+	}
+
+	if (advance == BCM_TLV_ADVANCE_TO) {
+		bcm_tlv_buffer_advance_to(elt, buf, buflen);
+	} else if (advance == BCM_TLV_ADVANCE_PAST) {
+		bcm_tlv_buffer_advance_past(elt, buf, buflen);
+	} else if (advance == BCM_TLV_ADVANCE_NONE) {
+		/* nothing to do */
+	} else {
+		/* there are only 3 modes, but just in case, zero the parse buffer pointer and
+		 * length to prevent infinite loops in callers that expect progress.
+		 */
+		ASSERT(0);
+		*buf = NULL;
+		*buflen = 0;
+	}
+
+	return elt;
+}
+
+bcm_tlv_t *
+bcm_parse_tlvs_dot11(const void *buf, uint buflen, uint key, bool id_ext)
+{
+	bcm_tlv_t *elt;
+	uint totlen;
+
+	COV_TAINTED_DATA_SINK(buflen);
+	COV_NEG_SINK(buflen);
+
+	/*
+	   ideally, we don't want to do that, but returning a const pointer
+	   from these parse function spreads casting everywhere in the code
+	*/
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	elt = (bcm_tlv_t*)buf;
+	GCC_DIAGNOSTIC_POP();
+
+	totlen = buflen;
+
+	/* find tagged parameter */
+	while (totlen >= TLV_HDR_LEN) {
+		uint len = elt->len;
+
+		/* validate remaining totlen */
+		if (totlen < (len + TLV_HDR_LEN)) {
+			break;
+		}
+
+		do {
+			if (id_ext) {
+				if (!DOT11_MNG_IE_ID_EXT_MATCH(elt, key))
+					break;
+			} else if (elt->id != key) {
+				break;
+			}
+
+			COV_TAINTED_DATA_ARG(elt);
+
+			return (bcm_tlv_t *)(elt);		/* a match */
+		} while (0);
+
+		elt = (bcm_tlv_t*)((uint8*)elt + (len + TLV_HDR_LEN));
+		totlen -= (len + TLV_HDR_LEN);
+	}
+
+	return NULL;
+}
+
+/*
+ * Traverse a string of 1-byte tag/1-byte length/variable-length value
+ * triples, returning a pointer to the substring whose first element
+ * matches tag
+ * return NULL if not found or length field < min_varlen
+ */
+bcm_tlv_t *
+bcm_parse_tlvs_min_bodylen(const  void *buf, uint buflen, uint key, uint min_bodylen)
+{
+	bcm_tlv_t * ret;
+	ret = bcm_parse_tlvs(buf, buflen, key);
+	if (ret == NULL || ret->len < min_bodylen) {
+		return NULL;
+	}
+	return ret;
+}
+
+/*
+ * Traverse a string of 1-byte tag/1-byte length/variable-length value
+ * triples, returning a pointer to the substring whose first element
+ * matches tag
+ * return NULL if not found or tlv size > max_len or < min_len
+ */
+bcm_tlv_t *
+bcm_parse_tlvs_minmax_len(const  void *buf, uint buflen, uint key,
+	uint min_len, uint max_len)
+{
+	bcm_tlv_t * ret;
+	ret = bcm_parse_tlvs(buf, buflen, key);
+	if (ret == NULL ||
+		(BCM_TLV_SIZE(ret) > max_len) ||
+		(BCM_TLV_SIZE(ret) < min_len)) {
+		return NULL;
+	}
+	return ret;
+}
+
+/*
+ * Traverse a string of 1-byte tag/1-byte length/variable-length value
+ * triples, returning a pointer to the substring whose first element
+ * matches tag.  Stop parsing when we see an element whose ID is greater
+ * than the target key.
+ */
+const  bcm_tlv_t *
+bcm_parse_ordered_tlvs(const  void *buf, uint buflen, uint key)
+{
+	const  bcm_tlv_t *elt;
+	uint totlen;
+
+	COV_TAINTED_DATA_SINK(buflen);
+	COV_NEG_SINK(buflen);
+
+	elt = (const  bcm_tlv_t*)buf;
+	totlen = buflen;
+
+	/* find tagged parameter */
+	while (totlen >= TLV_HDR_LEN) {
+		uint id = elt->id;
+		uint len = elt->len;
+
+		/* Punt if we start seeing IDs > than target key */
+		if (id > key) {
+			return (NULL);
+		}
+
+		/* validate remaining totlen */
+		if (totlen < (len + TLV_HDR_LEN)) {
+			break;
+		}
+		if (id == key) {
+			COV_TAINTED_DATA_ARG(elt);
+			return (elt);
+		}
+
+		elt = (const  bcm_tlv_t*)((const  uint8*)elt + (len + TLV_HDR_LEN));
+		totlen -= (len + TLV_HDR_LEN);
+	}
+	return NULL;
+}
+
+/**
+ * Return a const tlv buffer pointer and length representing the sub-buffer contained
+ * inside the given 'elt' starting at the given 'body_offset'.
+ * The function assumes that elt is a valid tlv; the elt pointer and data
+ * are all in the range of its parent buffer/length.
+ *
+ * @param elt         pointer to a valid bcm_tlv_t
+ * @param body_offset offset into the data body of elt
+ * @param buffer      on return, pointer to an offset in elt
+ * @param buflen      on return, length of the buffer to the end of elt
+ *
+ * On return, if body_offset is not less than the elt's body length, the *buffer parameter
+ * will be set to NULL and *buflen parameter will be set to zero.  Otherwise,
+ * *buffer will point to the sub-buffer at the body_offset, and *buflen be the remaining
+ * bytes in the body.
+ * E.g. Given a TLV with elt->len == 10, the call
+ * bcm_tlv_sub_buffer(elt, 4, &buffer, &buflen)
+ * will result with buffer = elt->data + 4, and buflen = 6.
+ */
+void
+bcm_tlv_sub_buffer(const bcm_tlv_t *elt, uint body_offset, const uint8 **buffer, uint8 *buflen)
+{
+	if (elt->len > body_offset) {
+		uint8 body_len = elt->len;
+
+#if defined(__COVERITY__)
+		/* The 'body_len' value is tainted in Coverity because it is read from
+		 * the tainted data pointed to by 'elt'. However, bcm_parse_tlvs() verifies
+		 * that the elt pointer is a valid element, so its body length is in the bounds
+		 * of the buffer.
+		 * Clearing the tainted attribute of 'body_len' for Coverity.
+		 */
+		__coverity_tainted_data_sanitize__(body_len);
+#endif /* __COVERITY__ */
+		*buflen = body_len - body_offset;
+		*buffer = elt->data + body_offset;
+	} else {
+		*buflen = 0u;
+		*buffer = NULL;
+	}
+}
+#endif	/* !BCMROMOFFLOAD_EXCLUDE_BCMUTILS_FUNCS */
+
+uint
+bcm_format_field(const bcm_bit_desc_ex_t *bd, uint32 flags, char* buf, uint len)
+{
+	uint i, slen = 0;
+	uint32 bit, mask;
+	const char *name;
+	mask = bd->mask;
+	if (len < 2 || !buf)
+		return 0;
+
+	buf[0] = '\0';
+
+	for (i = 0;  (name = bd->bitfield[i].name) != NULL; i++) {
+		bit = bd->bitfield[i].bit;
+		if ((flags & mask) == bit) {
+			slen = strlen(name);
+			if (memcpy_s(buf, len, name, slen + 1) != BCME_OK) {
+				slen = 0;
+			}
+			break;
+		}
+	}
+	return slen;
+}
+
+int
+bcm_format_flags(const bcm_bit_desc_t *bd, uint32 flags, char* buf, uint len)
+{
+	uint i;
+	char *p = buf;
+	char *end =  (buf + len);
+	char hexstr[16];
+	uint32 bit;
+	const char* name;
+	bool err = FALSE;
+
+	if (len < 2 || !buf)
+		return 0;
+
+	buf[0] = '\0';
+
+	for (i = 0; flags != 0; i++) {
+		bit = bd[i].bit;
+		name = bd[i].name;
+		if (bit == 0 && flags != 0) {
+			/* print any unnamed bits */
+			snprintf(hexstr, sizeof(hexstr), "0x%X", flags);
+			name = hexstr;
+			flags = 0;	/* exit loop */
+		} else if ((flags & bit) == 0) {
+			continue;
+		}
+		flags &= ~bit;
+
+		/* Print named bit. */
+		p += strlcpy(p, name, (size_t)(end - p));
+		if (p == end) {
+			/* Truncation error. */
+			err = TRUE;
+			break;
+		}
+
+		/* Add space delimiter if there are more bits. */
+		if (flags != 0) {
+			p += strlcpy(p, " ", (size_t)(end - p));
+			if (p == end) {
+				/* Truncation error. */
+				err = TRUE;
+				break;
+			}
+		}
+	}
+
+	/* indicate the str was too short */
+	if (err) {
+		ASSERT(len >= 2u);
+		buf[len - 2u] = '>';
+	}
+
+	return (int)(p - buf);
+}
+
+/* print out whcih bits in octet array 'addr' are set. bcm_bit_desc_t:bit is a bit offset. */
+int
+bcm_format_octets(const bcm_bit_desc_t *bd, uint bdsz,
+	const uint8 *addr, uint size, char *buf, uint len)
+{
+	uint i;
+	char *p = buf;
+	uint slen = 0, nlen = 0;
+	uint32 bit;
+	const char* name;
+	bool more = FALSE;
+
+	BCM_REFERENCE(size);
+
+	if (len < 2 || !buf)
+		return 0;
+
+	buf[0] = '\0';
+
+	for (i = 0; i < bdsz; i++) {
+		bit = bd[i].bit;
+		name = bd[i].name;
+		if (isset(addr, bit)) {
+			nlen = strlen(name);
+			slen += nlen;
+			/* need SPACE - for simplicity */
+			slen += 1;
+			/* need NULL as well */
+			if (len < slen + 1) {
+				more = TRUE;
+				break;
+			}
+			memcpy(p, name, nlen);
+			p += nlen;
+			p[0] = ' ';
+			p += 1;
+			p[0] = '\0';
+		}
+	}
+
+	if (more) {
+		p[0] = '>';
+		p += 1;
+		p[0] = '\0';
+	}
+
+	return (int)(p - buf);
+}
+
+/* Transform an hexadecimal string into binary.
+ * Output is limited to 64K.
+ * hex : string
+ * hex_len : string length
+ * buf : allocated output buffer
+ * buf_len : allocated size
+ * return : copied length, if successfull, 0 if error.
+ */
+uint16
+bcmhex2bin(const uint8* hex, uint hex_len, uint8 *buf, uint buf_len)
+{
+		uint i = 0;
+		uint16 out_len;
+		char tmp[] = "XX";
+		if (hex_len % 2) {
+			/* hex_len not even */
+			return 0;
+		}
+		/* check for hex radix */
+		if ((hex[0] == '0') && ((hex[1] == 'x') || (hex[1] == 'X'))) {
+			hex += 2;
+			hex_len -= 2;
+		}
+		if (hex_len/2 > 0xFFFF) {
+			/* exceed 64K buffer capacity */
+			return 0;
+		}
+		if ((out_len = hex_len/2) > buf_len) {
+			/* buf too short */
+			return 0;
+		}
+		do {
+			tmp[0] = *hex++;
+			tmp[1] = *hex++;
+			if (!bcm_isxdigit(tmp[0]) || !bcm_isxdigit(tmp[1])) {
+				/* char is not a 256-bit hex number	*/
+				return 0;
+			}
+			/* okay so far; make this piece a number */
+			buf[i] = (uint8) bcm_strtoul(tmp, NULL, 16);
+		} while (++i < out_len);
+		return out_len;
+}
+
+/* print bytes formatted as hex to a string. return the resulting string length */
+int
+bcm_format_hex(char *str, const void *bytes, uint len)
+{
+	uint i;
+	char *p = str;
+	const uint8 *src = (const uint8*)bytes;
+
+	for (i = 0; i < len; i++) {
+		p += snprintf(p, 3, "%02X", *src);
+		src++;
+	}
+	return (int)(p - str);
+}
+
+/* pretty hex print a contiguous buffer */
+void
+prhex(const char *msg, const uchar *buf, uint nbytes)
+{
+	char line[128], *p;
+	uint len = sizeof(line);
+	int nchar;
+	uint i;
+
+	if (msg && (msg[0] != '\0'))
+		printf("%s:\n", msg);
+
+	p = line;
+	for (i = 0; i < nbytes; i++) {
+		if (i % 16 == 0) {
+			nchar = snprintf(p, len, "  %04x: ", i);	/* line prefix */
+			p += nchar;
+			len -= (uint)nchar;
+		}
+		if (len > 0) {
+			nchar = snprintf(p, len, "%02x ", buf[i]);
+			p += nchar;
+			len -= (uint)nchar;
+		}
+
+		if (i % 16 == 15) {
+			printf("%s\n", line);		/* flush line */
+			p = line;
+			len = sizeof(line);
+		}
+	}
+
+	/* flush last partial line */
+	if (p != line)
+		printf("%s\n", line);
+}
+
+static const char *crypto_algo_names[] = {
+	"NONE",
+	"WEP1",
+	"TKIP",
+	"WEP128",
+	"AES_CCM",
+	"AES_OCB_MSDU",
+	"AES_OCB_MPDU",
+	"NALG",
+	"UNDEF",
+	"UNDEF",
+	"UNDEF",
+
+#ifdef BCMWAPI_WAI
+	"WAPI",
+#endif /* BCMWAPI_WAI */
+
+#ifndef BCMWAPI_WAI
+	"UNDEF",
+#endif
+	"PMK",
+	"BIP",
+	"AES_GCM",
+	"AES_CCM256",
+	"AES_GCM256",
+	"BIP_CMAC256",
+	"BIP_GMAC",
+	"BIP_GMAC256",
+	"UNDEF"
+};
+
+const char *
+bcm_crypto_algo_name(uint algo)
+{
+	return (algo < ARRAYSIZE(crypto_algo_names)) ? crypto_algo_names[algo] : "ERR";
+}
+
+#ifdef BCMDBG
+void
+deadbeef(void *p, uint len)
+{
+	static uint8 meat[] = { 0xde, 0xad, 0xbe, 0xef };
+
+	while (len-- > 0) {
+		*(uint8*)p = meat[((uintptr)p) & 3];
+		p = (uint8*)p + 1;
+	}
+}
+#endif /* BCMDBG */
+
+char *
+bcm_chipname(uint chipid, char *buf, uint len)
+{
+	const char *fmt;
+
+	fmt = ((chipid > 0xa000) || (chipid < 0x4000)) ? "%d" : "%x";
+	snprintf(buf, len, fmt, chipid);
+	return buf;
+}
+
+/* Produce a human-readable string for boardrev */
+char *
+bcm_brev_str(uint32 brev, char *buf)
+{
+	if (brev < 0x100)
+		snprintf(buf, 8, "%d.%d", (brev & 0xf0) >> 4, brev & 0xf);
+	else
+		snprintf(buf, 8, "%c%03x", ((brev & 0xf000) == 0x1000) ? 'P' : 'A', brev & 0xfff);
+
+	return (buf);
+}
+
+#define BUFSIZE_TODUMP_ATONCE 128 /* Buffer size */
+
+/* dump large strings to console */
+void
+printbig(char *buf)
+{
+	uint len, max_len;
+	char c;
+
+	len = (uint)strlen(buf);
+
+	max_len = BUFSIZE_TODUMP_ATONCE;
+
+	while (len > max_len) {
+		c = buf[max_len];
+		buf[max_len] = '\0';
+		printf("%s", buf);
+		buf[max_len] = c;
+
+		buf += max_len;
+		len -= max_len;
+	}
+	/* print the remaining string */
+	printf("%s\n", buf);
+	return;
+}
+
+/* routine to dump fields in a fileddesc structure */
+uint
+bcmdumpfields(bcmutl_rdreg_rtn read_rtn, void *arg0, uint arg1, struct fielddesc *fielddesc_array,
+	char *buf, uint32 bufsize)
+{
+	int ret;
+	uint filled_len;
+	uint len;
+	struct fielddesc *cur_ptr;
+
+	filled_len = 0;
+	cur_ptr = fielddesc_array;
+
+	while (bufsize > 1) {
+		if (cur_ptr->nameandfmt == NULL)
+			break;
+		/* check for snprintf overflow or error */
+		ret = snprintf(buf, bufsize, cur_ptr->nameandfmt,
+		               read_rtn(arg0, arg1, cur_ptr->offset));
+		if (ret < 0 || ret >= (int)bufsize) {
+			/* encoding error from snprintf */
+			len = bufsize - 1u;
+		} else {
+			len = (uint32)ret;
+		}
+
+		buf += len;
+		bufsize -= len;
+		filled_len += len;
+		cur_ptr++;
+	}
+	return filled_len;
+}
+
+uint
+bcm_mkiovar(const char *name, const char *data, uint datalen, char *buf, uint buflen)
+{
+	uint len;
+
+	len = (uint)strlen(name) + 1;
+
+	if ((len + datalen) > buflen)
+		return 0;
+
+	strlcpy(buf, name, buflen);
+
+	/* append data onto the end of the name string */
+	if (data && datalen != 0) {
+		memcpy(&buf[len], data, datalen);
+		len += datalen;
+	}
+
+	return len;
+}
+
+/* Quarter dBm units to mW
+ * Table starts at QDBM_OFFSET, so the first entry is mW for qdBm=153
+ * Table is offset so the last entry is largest mW value that fits in
+ * a uint16.
+ */
+
+#define QDBM_OFFSET 153		/* Offset for first entry */
+#define QDBM_TABLE_LEN 40	/* Table size */
+
+/* Smallest mW value that will round up to the first table entry, QDBM_OFFSET.
+ * Value is ( mW(QDBM_OFFSET - 1) + mW(QDBM_OFFSET) ) / 2
+ */
+#define QDBM_TABLE_LOW_BOUND 6493 /* Low bound */
+
+/* Largest mW value that will round down to the last table entry,
+ * QDBM_OFFSET + QDBM_TABLE_LEN-1.
+ * Value is ( mW(QDBM_OFFSET + QDBM_TABLE_LEN - 1) + mW(QDBM_OFFSET + QDBM_TABLE_LEN) ) / 2.
+ */
+#define QDBM_TABLE_HIGH_BOUND 64938 /* High bound */
+
+static const uint16 nqdBm_to_mW_map[QDBM_TABLE_LEN] = {
+/* qdBm:	+0	+1	+2	+3	+4	+5	+6	+7 */
+/* 153: */      6683,	7079,	7499,	7943,	8414,	8913,	9441,	10000,
+/* 161: */      10593,	11220,	11885,	12589,	13335,	14125,	14962,	15849,
+/* 169: */      16788,	17783,	18836,	19953,	21135,	22387,	23714,	25119,
+/* 177: */      26607,	28184,	29854,	31623,	33497,	35481,	37584,	39811,
+/* 185: */      42170,	44668,	47315,	50119,	53088,	56234,	59566,	63096
+};
+
+uint16
+bcm_qdbm_to_mw(uint8 qdbm)
+{
+	uint factor = 1;
+	int idx = qdbm - QDBM_OFFSET;
+
+	if (idx >= QDBM_TABLE_LEN) {
+		/* clamp to max uint16 mW value */
+		return 0xFFFF;
+	}
+
+	/* scale the qdBm index up to the range of the table 0-40
+	 * where an offset of 40 qdBm equals a factor of 10 mW.
+	 */
+	while (idx < 0) {
+		idx += 40;
+		factor *= 10;
+	}
+
+	/* return the mW value scaled down to the correct factor of 10,
+	 * adding in factor/2 to get proper rounding.
+	 */
+	return ((nqdBm_to_mW_map[idx] + factor/2) / factor);
+}
+
+uint8
+bcm_mw_to_qdbm(uint16 mw)
+{
+	uint8 qdbm;
+	int offset;
+	uint16 mw_uint = mw;
+	uint16 boundary;
+
+	/* handle boundary case */
+	if (mw_uint <= 1)
+		return 0;
+
+	offset = QDBM_OFFSET;
+
+	/* move mw into the range of the table */
+	while (mw_uint < QDBM_TABLE_LOW_BOUND) {
+		mw_uint *= 10;
+		offset -= 40;
+	}
+
+	for (qdbm = 0; qdbm < QDBM_TABLE_LEN-1; qdbm++) {
+		boundary = nqdBm_to_mW_map[qdbm] + (nqdBm_to_mW_map[qdbm+1] -
+		                                    nqdBm_to_mW_map[qdbm])/2;
+		if (mw_uint < boundary) break;
+	}
+
+	qdbm += (uint8)offset;
+
+	return (qdbm);
+}
+
+uint
+BCMPOSTTRAPFN(bcm_bitcount)(const uint8 *bitmap, uint length)
+{
+	uint bitcount = 0, i;
+	uint8 tmp;
+	for (i = 0; i < length; i++) {
+		tmp = bitmap[i];
+		while (tmp) {
+			bitcount++;
+			tmp &= (tmp - 1);
+		}
+	}
+	return bitcount;
+}
+
+/* Count 0s or 1s in an octets list from 'from_bit' to 'to_bit' inclusively */
+uint
+bcm_count_bits(const uint8 *buf, uint buf_len, uint from_bit, uint to_bit, bool val_1)
+{
+	uint from;
+	uint to;
+	uint len;
+	uint8 tmp;
+	uint bits;
+
+	/* sanity check */
+	if (to_bit < from_bit) {
+		return 0u;
+	}
+
+	/* byte offset */
+	from = from_bit >> 3u;
+	to = to_bit >> 3u;
+	if (from >= buf_len || to >= buf_len) {
+		return 0u;
+	}
+
+	/* count 1s always */
+	len = to - from + 1u;
+	if (len == 1u) {
+		/* the only octet */
+		ASSERT(from == to);
+		tmp = buf[from];
+		tmp &= (0xffu << (from_bit & 7u));
+		tmp &= ~(0xffu << ((to_bit & 7u) + 1u));
+		bits = bcm_bitcount(&tmp, 1u);
+	} else {
+		ASSERT(len >= 2u);
+		/* the first octet */
+		tmp = buf[from];
+		tmp &= (0xffu << (from_bit & 7u));
+		bits = bcm_bitcount(&tmp, 1u);
+		/* the middle octet(s) */
+		if (len > 2u) {
+			bits += bcm_bitcount(&buf[from + 1u], len - 2u);
+		}
+		/* the last octet */
+		tmp = buf[to];
+		tmp &= ~(0xffu << ((to_bit & 7u) + 1u));
+		bits += bcm_bitcount(&tmp, 1u);
+	}
+
+	/* calc 0s if requested */
+	if (!val_1) {
+		bits = to_bit - from_bit + 1u - bits;
+	}
+
+	return bits;
+}
+
+static void
+dump_nvram(char *varbuf, int column, unsigned int n, unsigned int len)
+{
+	unsigned int m;
+	char vars[128];
+
+	if (((n==0) && (varbuf[0]=='#')) ||
+			((column==0) && (android_msg_level & ANDROID_INFO_LEVEL))) {
+		memset(vars, 0x00, sizeof(vars));
+		for (m=n; m<len && (m-n)<(sizeof(vars)-1); m++) {
+			if (varbuf[m] == '\n')
+				break;
+			vars[m-n] = varbuf[m];
+		}
+		EDBG_DUMP(("%s\n", vars));
+	}
+}
+
+/*
+ * ProcessVars:Takes a buffer of "<var>=<value>\n" lines read from a file and ending in a NUL.
+ * also accepts nvram files which are already in the format of <var1>=<value>\0\<var2>=<value2>\0
+ * Removes carriage returns, empty lines, comment lines, and converts newlines to NULs.
+ * Shortens buffer as needed and pads with NULs.  End of buffer is marked by two NULs.
+*/
+
+unsigned int
+process_nvram_vars(char *varbuf, unsigned int len)
+{
+	char *dp;
+	bool findNewline;
+	int column;
+	unsigned int buf_len, n;
+	unsigned int pad = 0;
+
+	dp = varbuf;
+
+	findNewline = FALSE;
+	column = 0;
+
+	dump_nvram(varbuf, 0, 0, len);
+	for (n = 0; n < len; n++) {
+		if (varbuf[n] == '\r')
+			continue;
+		if (findNewline && varbuf[n] != '\n')
+			continue;
+		findNewline = FALSE;
+		if (varbuf[n] == '#') {
+			findNewline = TRUE;
+			continue;
+		}
+		if (varbuf[n] == '\n') {
+			if (column == 0)
+				continue;
+			*dp++ = 0;
+			column = 0;
+			continue;
+		}
+		dump_nvram(varbuf, column, n, len);
+		*dp++ = varbuf[n];
+		column++;
+	}
+	buf_len = (unsigned int)(dp - varbuf);
+	if (buf_len % 4) {
+		pad = 4 - buf_len % 4;
+		if (pad && (buf_len + pad <= len)) {
+			buf_len += pad;
+		}
+	}
+
+	while (dp < varbuf + n)
+		*dp++ = 0;
+
+	return buf_len;
+}
+
+#ifndef setbit /* As in the header file */
+#ifdef BCMUTILS_BIT_MACROS_USE_FUNCS
+/* Set bit in byte array. */
+void
+setbit(void *array, uint bit)
+{
+	((uint8 *)array)[bit / NBBY] |= 1 << (bit % NBBY);
+}
+
+/* Clear bit in byte array. */
+void
+clrbit(void *array, uint bit)
+{
+	((uint8 *)array)[bit / NBBY] &= ~(1 << (bit % NBBY));
+}
+
+/* Test if bit is set in byte array. */
+bool
+isset(const void *array, uint bit)
+{
+	return (((const uint8 *)array)[bit / NBBY] & (1 << (bit % NBBY)));
+}
+
+/* Test if bit is clear in byte array. */
+bool
+isclr(const void *array, uint bit)
+{
+	return ((((const uint8 *)array)[bit / NBBY] & (1 << (bit % NBBY))) == 0);
+}
+#endif /* BCMUTILS_BIT_MACROS_USE_FUNCS */
+#endif /* setbit */
+
+void
+BCMPOSTTRAPFN(set_bitrange)(void *array, uint start, uint end, uint maxbit)
+{
+	uint startbyte = start/NBBY;
+	uint endbyte = end/NBBY;
+	uint i, startbytelastbit, endbytestartbit;
+
+	if (end >= start) {
+		if (endbyte - startbyte > 1) {
+			startbytelastbit = ((startbyte + 1) * NBBY) - 1;
+			endbytestartbit = endbyte * NBBY;
+			for (i = startbyte + 1; i < endbyte; i++)
+				((uint8 *)array)[i] = 0xFF;
+			for (i = start; i <= startbytelastbit; i++)
+				setbit(array, i);
+			for (i = endbytestartbit; i <= end; i++)
+				setbit(array, i);
+		} else {
+			for (i = start; i <= end; i++)
+				setbit(array, i);
+		}
+	} else {
+		set_bitrange(array, start, maxbit, maxbit);
+		set_bitrange(array, 0, end, maxbit);
+	}
+}
+
+void
+clr_bitrange(void *array, uint start, uint end, uint maxbit)
+{
+	uint startbyte = start/NBBY;
+	uint endbyte = end/NBBY;
+	uint i, startbytelastbit, endbytestartbit;
+
+	if (end >= start) {
+		if (endbyte - startbyte > 1) {
+			startbytelastbit = ((startbyte + 1) * NBBY) - 1;
+			endbytestartbit = endbyte * NBBY;
+			for (i = startbyte + 1; i < endbyte; i++)
+				((uint8 *)array)[i] = 0x0;
+			for (i = start; i <= startbytelastbit; i++)
+				clrbit(array, i);
+			for (i = endbytestartbit; i <= end; i++)
+				clrbit(array, i);
+		} else {
+			for (i = start; i <= end; i++)
+				clrbit(array, i);
+		}
+	} else {
+		clr_bitrange(array, start, maxbit, maxbit);
+		clr_bitrange(array, 0, end, maxbit);
+	}
+}
+
+/*
+ * This api (set_bitrange_int_access) as same as set_bitrange but uses int32 operation
+ * This api can be used in the place of set_bitrange but array should be word (32bit) alligned.
+ * This api has to be used when the memory being accessed has restrictions of
+ * not using them in 8bit (byte) mode and needing 32bit (word) mode.
+ */
+void
+set_bitrange_u32(void *array, uint start, uint end, uint maxbit)
+{
+	uint startword = start/SIZE_BITS32(uint32);
+	uint endword = end/SIZE_BITS32(uint32);
+	uint startwordstartbit = start % SIZE_BITS32(uint32);
+	uint endwordlastbit = end % SIZE_BITS32(uint32);
+	/* Used to caluculate bit number from MSB */
+	uint u32msbnum = SIZE_BITS32(uint32) - 1U;
+	uint i;
+	uint32 setbitsword;
+	uint32 u32max = ~0U;
+
+	ASSERT(ISALIGNED(array, sizeof(uint32))); /* array should be alligned for this API */
+
+	if (start > end) {
+		set_bitrange_u32(array, start, maxbit, maxbit);
+		set_bitrange_u32(array, 0U, end, maxbit);
+		return;
+	}
+
+	if (endword - startword) {
+		/* Setting MSB bits including startwordstartbit */
+		setbitsword = u32max << startwordstartbit;
+		((uint32 *)array)[startword] |= setbitsword;
+
+		/* Setting all bits in 'startword + 1' to 'endword - 1' */
+		for (i = startword + 1U; i <= endword - 1U; i++) {
+			((uint32 *)array)[i] = u32max;
+		}
+
+		/* Setting LSB bits including endwordlastbit */
+		setbitsword = u32max >> (u32msbnum - endwordlastbit);
+		((uint32 *)array)[endword] |= setbitsword;
+	} else { /* start and end are in same word */
+		/* Setting start bit to end bit including start and end bits */
+		setbitsword =
+			(u32max << startwordstartbit) & (u32max >> (u32msbnum - endwordlastbit));
+		((uint32 *)array)[startword] |= setbitsword;
+	}
+}
+
+/*
+ * This api (clr_bitrange_u32) as same as clr_bitrange but uses int32 operation
+ * This api can be used in the place of clr_bitrange but array should be word (32bit) alligned.
+ * This api has to be used when the memory being accessed has restrictions of
+ * not using them in 8bit (byte) mode and needing 32bit (word) mode.
+ */
+void
+clr_bitrange_u32(void *array, uint start, uint end, uint maxbit)
+{
+	uint startword = start/SIZE_BITS32(uint32);
+	uint endword = end/SIZE_BITS32(uint32);
+	uint startwordstartbit = start % SIZE_BITS32(uint32);
+	uint endwordlastbit = end % SIZE_BITS32(uint32);
+	/* Used to caluculate bit number from MSB */
+	uint u32msbnum = SIZE_BITS32(uint32) - 1U;
+	uint i;
+	uint32 clrbitsword;
+	uint32 u32max = ~0U;
+
+	ASSERT(ISALIGNED(array, sizeof(uint32))); /* array should be alligned for this API */
+
+	if (start > end) {
+		clr_bitrange_u32(array, start, maxbit, maxbit);
+		clr_bitrange_u32(array, 0U, end, maxbit);
+		return;
+	}
+
+	if (endword - startword) {
+		/* Clearing MSB bits including startwordstartbit */
+		clrbitsword = ~(u32max << startwordstartbit);
+		((uint32 *)array)[startword] &= clrbitsword;
+
+		/* Clearing all bits in 'startword + 1' to 'endword - 1' */
+		for (i = startword + 1U; i <= endword - 1U; i++) {
+			((uint32 *)array)[i] = 0U;
+		}
+
+		/* Clearing LSB bits including endwordlastbit */
+		clrbitsword = ~(u32max >> (u32msbnum - endwordlastbit));
+		((uint32 *)array)[endword] &= clrbitsword;
+	} else { /* start and end are in same word */
+		/* Clearing start bit to end bit including start and end bits */
+		clrbitsword =
+			~(u32max << startwordstartbit) | ~(u32max >> (u32msbnum - endwordlastbit));
+		((uint32 *)array)[startword] &= clrbitsword;
+	}
+}
+
+void
+bcm_bitprint32(const uint32 u32arg)
+{
+	int i;
+	for (i = NBITS(uint32) - 1; i >= 0; i--) {
+		if (isbitset(u32arg, i)) {
+			printf("1");
+		} else {
+			printf("0");
+		}
+
+		if ((i % NBBY) == 0) printf(" ");
+	}
+	printf("\n");
+}
+
+/* calculate checksum for ip header, tcp / udp header / data */
+uint16
+bcm_ip_cksum(uint8 *buf, uint32 len, uint32 sum)
+{
+	while (len > 1) {
+		sum += ((uint32)buf[0] << 8) | buf[1];
+		buf += 2;
+		len -= 2;
+	}
+
+	if (len > 0) {
+		sum += ((uint32)*buf) << 8;
+	}
+
+	while (sum >> 16) {
+		sum = (sum & 0xffff) + (sum >> 16);
+	}
+
+	return ((uint16)~sum);
+}
+
+/* calculate a + b where a is a 64 bit number and b is a 32 bit number */
+void
+bcm_add_64(uint32* r_hi, uint32* r_lo, uint32 offset)
+{
+	uint32 r1_lo = *r_lo;
+	(*r_lo) += offset;
+	if (*r_lo < r1_lo)
+		(*r_hi) ++;
+}
+
+/* calculate a - b where a is a 64 bit number and b is a 32 bit number */
+void
+bcm_sub_64(uint32* r_hi, uint32* r_lo, uint32 offset)
+{
+	uint32 r1_lo = *r_lo;
+	(*r_lo) -= offset;
+	if (*r_lo > r1_lo)
+		(*r_hi) --;
+}
+
+int
+BCMRAMFN(valid_bcmerror)(int e)
+{
+	return ((e <= 0) && (e >= BCME_LAST));
+}
+
+#ifdef DEBUG_COUNTER
+#if (OSL_SYSUPTIME_SUPPORT == TRUE)
+void counter_printlog(counter_tbl_t *ctr_tbl)
+{
+	uint32 now;
+
+	if (!ctr_tbl->enabled)
+		return;
+
+	now = OSL_SYSUPTIME();
+
+	if (now - ctr_tbl->prev_log_print > ctr_tbl->log_print_interval) {
+		uint8 i = 0;
+		printf("counter_print(%s %d):", ctr_tbl->name, now - ctr_tbl->prev_log_print);
+
+		for (i = 0; i < ctr_tbl->needed_cnt; i++) {
+			printf(" %u", ctr_tbl->cnt[i]);
+		}
+		printf("\n");
+
+		ctr_tbl->prev_log_print = now;
+		bzero(ctr_tbl->cnt, CNTR_TBL_MAX * sizeof(uint));
+	}
+}
+#else
+/* OSL_SYSUPTIME is not supported so no way to get time */
+#define counter_printlog(a) do {} while (0)
+#endif /* OSL_SYSUPTIME_SUPPORT == TRUE */
+#endif /* DEBUG_COUNTER */
+
+/* calculate partial checksum */
+static uint32
+ip_cksum_partial(uint32 sum, uint8 *val8, uint32 count)
+{
+	uint32 i;
+	uint16 *val16 = (uint16 *)val8;
+
+	ASSERT(val8 != NULL);
+	/* partial chksum calculated on 16-bit values */
+	ASSERT((count % 2) == 0);
+
+	count /= 2;
+
+	for (i = 0; i < count; i++) {
+		sum += *val16++;
+	}
+	return sum;
+}
+
+/* calculate IP checksum */
+static uint16
+ip_cksum(uint32 sum, uint8 *val8, uint32 count)
+{
+	uint16 *val16 = (uint16 *)val8;
+
+	ASSERT(val8 != NULL);
+
+	while (count > 1) {
+		sum += *val16++;
+		count -= 2;
+	}
+	/*  add left-over byte, if any */
+	if (count > 0) {
+		sum += (*(uint8 *)val16);
+	}
+
+	/*  fold 32-bit sum to 16 bits */
+	sum = (sum >> 16) + (sum & 0xffff);
+	sum += (sum >> 16);
+	return ((uint16)~sum);
+}
+
+/* calculate IPv4 header checksum
+ * - input ip points to IP header in network order
+ * - output cksum is in network order
+ */
+uint16
+ipv4_hdr_cksum(uint8 *ip, uint ip_len)
+{
+	uint32 sum = 0;
+	uint8 *ptr = ip;
+
+	ASSERT(ip != NULL);
+	ASSERT(ip_len >= IPV4_MIN_HEADER_LEN);
+
+	if (ip_len < IPV4_MIN_HEADER_LEN) {
+		return 0;
+	}
+
+	/* partial cksum skipping the hdr_chksum field */
+	sum = ip_cksum_partial(sum, ptr, OFFSETOF(struct ipv4_hdr, hdr_chksum));
+	ptr += OFFSETOF(struct ipv4_hdr, hdr_chksum) + 2;
+
+	/* return calculated chksum */
+	return ip_cksum(sum, ptr, ip_len - OFFSETOF(struct ipv4_hdr, src_ip));
+}
+
+/* calculate TCP header checksum using partial sum */
+static uint16
+tcp_hdr_chksum(uint32 sum, uint8 *tcp_hdr, uint16 tcp_len)
+{
+	uint8 *ptr = tcp_hdr;
+
+	ASSERT(tcp_hdr != NULL);
+	ASSERT(tcp_len >= TCP_MIN_HEADER_LEN);
+
+	/* partial TCP cksum skipping the chksum field */
+	sum = ip_cksum_partial(sum, ptr, OFFSETOF(struct bcmtcp_hdr, chksum));
+	ptr += OFFSETOF(struct bcmtcp_hdr, chksum) + 2;
+
+	/* return calculated chksum */
+	return ip_cksum(sum, ptr, tcp_len - OFFSETOF(struct bcmtcp_hdr, urg_ptr));
+}
+
+struct tcp_pseudo_hdr {
+	uint8   src_ip[IPV4_ADDR_LEN];  /* Source IP Address */
+	uint8   dst_ip[IPV4_ADDR_LEN];  /* Destination IP Address */
+	uint8	zero;
+	uint8	prot;
+	uint16	tcp_size;
+};
+
+/* calculate IPv4 TCP header checksum
+ * - input ip and tcp points to IP and TCP header in network order
+ * - output cksum is in network order
+ */
+uint16
+ipv4_tcp_hdr_cksum(uint8 *ip, uint8 *tcp, uint16 tcp_len)
+{
+	struct ipv4_hdr *ip_hdr = (struct ipv4_hdr *)ip;
+	struct tcp_pseudo_hdr tcp_ps;
+	uint32 sum = 0;
+
+	ASSERT(ip != NULL);
+	ASSERT(tcp != NULL);
+	ASSERT(tcp_len >= TCP_MIN_HEADER_LEN);
+
+	/* pseudo header cksum */
+	bzero(&tcp_ps, sizeof(tcp_ps));
+	memcpy(&tcp_ps.dst_ip, ip_hdr->dst_ip, IPV4_ADDR_LEN);
+	memcpy(&tcp_ps.src_ip, ip_hdr->src_ip, IPV4_ADDR_LEN);
+	tcp_ps.zero = 0;
+	tcp_ps.prot = ip_hdr->prot;
+	tcp_ps.tcp_size = hton16(tcp_len);
+	sum = ip_cksum_partial(sum, (uint8 *)&tcp_ps, sizeof(tcp_ps));
+
+	/* return calculated TCP header chksum */
+	return tcp_hdr_chksum(sum, tcp, tcp_len);
+}
+
+struct ipv6_pseudo_hdr {
+	uint8  saddr[IPV6_ADDR_LEN];
+	uint8  daddr[IPV6_ADDR_LEN];
+	uint16 payload_len;
+	uint8  zero;
+	uint8  next_hdr;
+};
+
+/* calculate IPv6 TCP header checksum
+ * - input ipv6 and tcp points to IPv6 and TCP header in network order
+ * - output cksum is in network order
+ */
+uint16
+ipv6_tcp_hdr_cksum(uint8 *ipv6, uint8 *tcp, uint16 tcp_len)
+{
+	struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)ipv6;
+	struct ipv6_pseudo_hdr ipv6_pseudo;
+	uint32 sum = 0;
+
+	ASSERT(ipv6 != NULL);
+	ASSERT(tcp != NULL);
+	ASSERT(tcp_len >= TCP_MIN_HEADER_LEN);
+
+	/* pseudo header cksum */
+	bzero((char *)&ipv6_pseudo, sizeof(ipv6_pseudo));
+	memcpy((char *)ipv6_pseudo.saddr, (char *)ipv6_hdr->saddr.addr,
+		sizeof(ipv6_pseudo.saddr));
+	memcpy((char *)ipv6_pseudo.daddr, (char *)ipv6_hdr->daddr.addr,
+		sizeof(ipv6_pseudo.daddr));
+	ipv6_pseudo.payload_len = ipv6_hdr->payload_len;
+	ipv6_pseudo.next_hdr = ipv6_hdr->nexthdr;
+	sum = ip_cksum_partial(sum, (uint8 *)&ipv6_pseudo, sizeof(ipv6_pseudo));
+
+	/* return calculated TCP header chksum */
+	return tcp_hdr_chksum(sum, tcp, tcp_len);
+}
+
+void *_bcmutils_dummy_fn = NULL;
+
+/* GROUP 1 --- start
+ * These function under GROUP 1 are general purpose functions to do complex number
+ * calculations and square root calculation.
+ */
+
+uint32 sqrt_int(uint32 value)
+{
+	uint32 root = 0, shift = 0;
+
+	/* Compute integer nearest to square root of input integer value */
+	for (shift = 0; shift < 32; shift += 2) {
+		if (((0x40000000u >> shift) + root) <= value) {
+			value -= ((0x40000000u >> shift) + root);
+			root = (root >> 1) | (0x40000000u >> shift);
+		}
+		else {
+			root = root >> 1;
+		}
+	}
+
+	/* round to the nearest integer */
+	if (root < value) ++root;
+
+	return root;
+}
+
+/* GROUP 1 --- end */
+
+/* read/write field in a consecutive bits in an octet array.
+ * 'addr' is the octet array's start byte address
+ * 'size' is the octet array's byte size
+ * 'stbit' is the value's start bit offset
+ * 'nbits' is the value's bit size
+ * This set of utilities are for convenience. Don't use them
+ * in time critical/data path as there's a great overhead in them.
+ */
+void
+setbits(uint8 *addr, uint size, uint stbit, uint nbits, uint32 val)
+{
+	uint fbyte = stbit >> 3;		/* first byte */
+	uint lbyte = (stbit + nbits - 1) >> 3;	/* last byte */
+	uint fbit = stbit & 7;			/* first bit in the first byte */
+	uint rbits = (nbits > 8 - fbit ?
+	              nbits - (8 - fbit) :
+	              0) & 7;			/* remaining bits of the last byte when not 0 */
+	uint8 mask;
+	uint byte;
+
+	BCM_REFERENCE(size);
+
+	ASSERT(fbyte < size);
+	ASSERT(lbyte < size);
+	ASSERT(nbits <= (sizeof(val) << 3));
+
+	/* all bits are in the same byte */
+	if (fbyte == lbyte) {
+		mask = ((1 << nbits) - 1) << fbit;
+		addr[fbyte] &= ~mask;
+		addr[fbyte] |= (uint8)(val << fbit);
+		return;
+	}
+
+	/* first partial byte */
+	if (fbit > 0) {
+		mask = (0xff << fbit);
+		addr[fbyte] &= ~mask;
+		addr[fbyte] |= (uint8)(val << fbit);
+		val >>= (8 - fbit);
+		nbits -= (8 - fbit);
+		fbyte ++;	/* first full byte */
+	}
+
+	/* last partial byte */
+	if (rbits > 0) {
+		mask = (1 << rbits) - 1;
+		addr[lbyte] &= ~mask;
+		addr[lbyte] |= (uint8)(val >> (nbits - rbits));
+		lbyte --;	/* last full byte */
+	}
+
+	/* remaining full byte(s) */
+	for (byte = fbyte; byte <= lbyte; byte ++) {
+		addr[byte] = (uint8)val;
+		val >>= 8;
+	}
+}
+
+uint32
+getbits(const uint8 *addr, uint size, uint stbit, uint nbits)
+{
+	uint fbyte = stbit >> 3u;		/* first byte */
+	uint lbyte = (stbit + nbits - 1u) >> 3u;	/* last byte */
+	uint fbit = stbit & 7u;			/* first bit in the first byte */
+	uint rbits = (nbits > 8u - fbit ?
+	              nbits - (8u - fbit) :
+	              0) & 7u;			/* remaining bits of the last byte when not 0 */
+	uint32 val = 0;
+	uint bits = 0;				/* bits in first partial byte */
+	uint8 mask;
+	uint byte;
+
+	BCM_REFERENCE(size);
+
+	ASSERT_FP(fbyte < size);
+	ASSERT_FP(lbyte < size);
+	ASSERT_FP(nbits <= (sizeof(val) << 3u));
+
+	/* all bits are in the same byte */
+	if (fbyte == lbyte) {
+		mask = ((1u << nbits) - 1u) << fbit;
+		val = ((uint32)addr[fbyte] & mask) >> fbit;
+		return val;
+	}
+
+	/* first partial byte */
+	if (fbit > 0) {
+		bits = 8u - fbit;
+		mask = (0xffu << fbit);
+		val |= ((uint32)addr[fbyte] & mask) >> fbit;
+		fbyte ++;	/* first full byte */
+	}
+
+	/* last partial byte */
+	if (rbits > 0) {
+		mask = (1u << rbits) - 1u;
+		val |= ((uint32)addr[lbyte] & mask) << (nbits - rbits);
+		lbyte --;	/* last full byte */
+	}
+
+	/* remaining full byte(s) */
+	for (byte = fbyte; byte <= lbyte; byte ++) {
+		val |= ((uint32)addr[byte] << (((byte - fbyte) << 3u) + bits));
+	}
+
+	return val;
+}
+
+#if defined(BCMDBG) || defined(WLMSG_ASSOC)
+/* support for getting 802.11 frame type/name based on frame kind */
+#define FK_NAME_DECL(x) {FC_##x, #x}
+static const struct {
+    uint fk;
+    const char *name;
+} bcm_80211_fk_names[] =  {
+	FK_NAME_DECL(ASSOC_REQ),
+	FK_NAME_DECL(ASSOC_RESP),
+	FK_NAME_DECL(REASSOC_REQ),
+	FK_NAME_DECL(REASSOC_RESP),
+	FK_NAME_DECL(PROBE_REQ),
+	FK_NAME_DECL(PROBE_RESP),
+	FK_NAME_DECL(BEACON),
+	FK_NAME_DECL(ATIM),
+	FK_NAME_DECL(DISASSOC),
+	FK_NAME_DECL(AUTH),
+	FK_NAME_DECL(DEAUTH),
+	FK_NAME_DECL(ACTION),
+	FK_NAME_DECL(ACTION_NOACK),
+	FK_NAME_DECL(CTL_TRIGGER),
+	FK_NAME_DECL(CTL_WRAPPER),
+	FK_NAME_DECL(BLOCKACK_REQ),
+	FK_NAME_DECL(BLOCKACK),
+	FK_NAME_DECL(PS_POLL),
+	FK_NAME_DECL(RTS),
+	FK_NAME_DECL(CTS),
+	FK_NAME_DECL(ACK),
+	FK_NAME_DECL(CF_END),
+	FK_NAME_DECL(CF_END_ACK),
+	FK_NAME_DECL(DATA),
+	FK_NAME_DECL(NULL_DATA),
+	FK_NAME_DECL(DATA_CF_ACK),
+	FK_NAME_DECL(QOS_DATA),
+	FK_NAME_DECL(QOS_NULL)
+};
+static const uint n_bcm_80211_fk_names = ARRAYSIZE(bcm_80211_fk_names);
+
+const char *bcm_80211_fk_name(uint fk)
+{
+	uint i;
+	for (i = 0; i < n_bcm_80211_fk_names; ++i) {
+		if (bcm_80211_fk_names[i].fk == fk) {
+			return bcm_80211_fk_names[i].name;
+		}
+	}
+	return "unknown";
+}
+#endif /* BCMDBG || WLMSG_ASSOC */
+
+#ifdef BCMDRIVER
+
+/** allocate variable sized data with 'size' bytes. note: vld should NOT be null.
+ */
+int
+bcm_vdata_alloc(osl_t *osh, var_len_data_t *vld, uint32 size)
+{
+	int ret = BCME_ERROR;
+	uint8 *dat = NULL;
+
+	if (vld == NULL) {
+		ASSERT(0);
+		goto done;
+	}
+
+	/* trying to allocate twice? */
+	if (vld->vdata != NULL) {
+		ASSERT(0);
+		goto done;
+	}
+
+	/* trying to allocate 0 size? */
+	if (size == 0) {
+		ASSERT(0);
+		ret = BCME_BADARG;
+		goto done;
+	}
+
+	dat = MALLOCZ(osh, size);
+	if (dat == NULL) {
+		ret = BCME_NOMEM;
+		goto done;
+	}
+	vld->vlen = size;
+	vld->vdata = dat;
+	ret = BCME_OK;
+done:
+	return ret;
+}
+
+/** free memory associated with variable sized data. note: vld should NOT be null.
+ */
+int
+bcm_vdata_free(osl_t *osh, var_len_data_t *vld)
+{
+	int ret = BCME_ERROR;
+
+	if (vld == NULL) {
+		ASSERT(0);
+		goto done;
+	}
+
+	if (vld->vdata) {
+		MFREE(osh, vld->vdata, vld->vlen);
+		vld->vlen = 0;
+		ret = BCME_OK;
+	}
+done:
+	return ret;
+}
+
+/* return TRUE if :
+ * - both buffers are of length 0
+ * OR
+ * - both buffers are NULL
+ * OR
+ * lengths and contents are the same.
+ */
+bool
+bcm_match_buffers(const uint8 *b1, uint b1_len, const uint8 *b2, uint b2_len)
+
+{
+	if (b1_len == 0 && b2_len == 0) {
+		return TRUE;
+	}
+
+	if (b1 == NULL && b2 == NULL) {
+		return TRUE;
+	}
+
+	/* If they are not both NULL, neither can be */
+	if (b1 == NULL || b2 == NULL) {
+		return FALSE;
+	}
+
+	if ((b1_len == b2_len) && !memcmp(b1, b2, b1_len)) {
+		return TRUE;
+	}
+	return FALSE;
+}
+
+#ifdef PRIVACY_MASK
+/* applies privacy mask on the input address itself */
+void
+BCMRAMFN(bcm_ether_privacy_mask)(struct ether_addr *addr)
+{
+	struct ether_addr *privacy = privacy_addrmask_get();
+	if (addr && !ETHER_ISMULTI(addr)) {
+		*(uint32*)(&(addr->octet[0])) &= *((uint32*)&privacy->octet[0]);
+		*(uint16*)(&(addr->octet[4])) &= *((uint16*)&privacy->octet[4]);
+	}
+}
+#endif /* PRIVACY_MASK */
+#endif /* BCMDRIVER */
+
+/* Count the number of elements not matching a given value in a null terminated array */
+int
+array_value_mismatch_count(uint8 value, uint8 *array, int array_size)
+{
+	int i;
+	int count = 0;
+
+	for (i = 0; i < array_size; i++) {
+		/* exit if a null terminator is found */
+		if (array[i] == 0) {
+			break;
+		}
+		if (array[i] != value) {
+			count++;
+		}
+	}
+	return count;
+}
+
+/* Count the number of non-zero elements in an uint8 array */
+int
+array_nonzero_count(uint8 *array, int array_size)
+{
+	return array_value_mismatch_count(0, array, array_size);
+}
+
+/* Count the number of non-zero elements in an int16 array */
+int
+array_nonzero_count_int16(int16 *array, int array_size)
+{
+	int i;
+	int count = 0;
+
+	for (i = 0; i < array_size; i++) {
+		if (array[i] != 0) {
+			count++;
+		}
+	}
+	return count;
+}
+
+/* Count the number of zero elements in an uint8 array */
+int
+array_zero_count(uint8 *array, int array_size)
+{
+	int i;
+	int count = 0;
+
+	for (i = 0; i < array_size; i++) {
+		if (array[i] == 0) {
+			count++;
+		}
+	}
+	return count;
+}
+
+/* Validate an array that can be 1 of 2 data types.
+ * One of array1 or array2 should be non-NULL.  The other should be NULL.
+ */
+static int
+verify_ordered_array(uint8 *array1, int16 *array2, int array_size,
+	int range_lo, int range_hi, bool err_if_no_zero_term, bool is_ordered)
+{
+	int ret;
+	int i;
+	int val = 0;
+	int prev_val = 0;
+
+	ret = err_if_no_zero_term ? BCME_NOTFOUND : BCME_OK;
+
+	/* Check that:
+	 * - values are in descending order.
+	 * - values are within the valid range.
+	 */
+	for (i = 0; i < array_size; i++) {
+		if (array1) {
+			val = (int)array1[i];
+		} else if (array2) {
+			val = (int)array2[i];
+		} else {
+			/* both array parameters are NULL */
+			return BCME_NOTFOUND;
+		}
+		if (val == 0) {
+			/* array is zero-terminated */
+			ret = BCME_OK;
+			break;
+		}
+
+		if (is_ordered && i > 0 && val > prev_val) {
+			/* array is not in descending order */
+			ret = BCME_BADOPTION;
+			break;
+		}
+		prev_val = val;
+
+		if (val < range_lo || val > range_hi) {
+			/* array value out of range */
+			ret = BCME_RANGE;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+/* Validate an ordered uint8 configuration array */
+int
+verify_ordered_array_uint8(uint8 *array, int array_size,
+	uint8 range_lo, uint8 range_hi)
+{
+	return verify_ordered_array(array, NULL, array_size, (int)range_lo, (int)range_hi,
+		TRUE, TRUE);
+}
+
+/* Validate an ordered int16 non-zero-terminated configuration array */
+int
+verify_ordered_array_int16(int16 *array, int array_size,
+	int16 range_lo, int16 range_hi)
+{
+	return verify_ordered_array(NULL, array, array_size, (int)range_lo, (int)range_hi,
+		FALSE, TRUE);
+}
+
+/* Validate all values in an array are in range */
+int
+verify_array_values(uint8 *array, int array_size,
+	int range_lo, int range_hi, bool zero_terminated)
+{
+	int ret = BCME_OK;
+	int i;
+	int val = 0;
+
+	/* Check that:
+	 * - values are in strict descending order.
+	 * - values are within the valid range.
+	 */
+	for (i = 0; i < array_size; i++) {
+		val = (int)array[i];
+		if (val == 0 && zero_terminated) {
+			ret = BCME_OK;
+			break;
+		}
+		if (val < range_lo || val > range_hi) {
+			/* array value out of range */
+			ret = BCME_RANGE;
+			break;
+		}
+	}
+	return ret;
+}
+
+/* Adds/replaces NVRAM variable with given value
+ * varbuf[in,out]   - Buffer with NVRAM variables (sequence of zero-terminated 'name=value' records,
+ *                    terminated with additional zero)
+ * buflen[in]       - Length of buffer (may, even should, have some unused space)
+ * variable[in]     - Variable to add/replace in 'name=value' form
+ * datalen[out,opt] - Optional output parameter - resulting length of data in buffer
+ * Returns TRUE on success, FALSE if buffer too short or variable specified incorrectly
+ */
+bool
+replace_nvram_variable(char *varbuf, unsigned int buflen, const char *variable,
+	unsigned int *datalen)
+{
+	char *p;
+	uint variable_heading_len, record_len, variable_record_len = strlen(variable) + 1;
+	char *buf_end = varbuf + buflen;
+	p = strchr(variable, '=');
+	if (!p) {
+		return FALSE;
+	}
+	/* Length of given variable name, followed by '=' */
+	variable_heading_len = (uint)((const char *)(p + 1) - variable);
+	/* Scanning NVRAM, record by record up to trailing 0 */
+	for (p = varbuf; *p; p += strlen(p) + 1) {
+		/* If given variable found - remove it */
+		if (!strncmp(p, variable, variable_heading_len)) {
+			record_len = strlen(p) + 1;
+			memmove_s(p, (size_t)(buf_end - p), p + record_len,
+				(size_t)(buf_end - (p + record_len)));
+		}
+	}
+	/* If buffer does not have space for given variable - return FALSE */
+	if ((p + variable_record_len + 1) > buf_end) {
+		return FALSE;
+	}
+	/* Copy given variable to end of buffer */
+	memmove_s(p, (size_t)(buf_end - p), variable, variable_record_len);
+	/* Adding trailing 0 */
+	p[variable_record_len] = 0;
+	/* Setting optional output parameter - length of data in buffer */
+	if (datalen) {
+		*datalen = (unsigned int)(p + variable_record_len + 1  - varbuf);
+	}
+	return TRUE;
+}
+
+/*
+ * Gets the ceil bit set to the nearest power of 2
+ * val[in]	- value for which nearest power of 2 bit set to be returned
+ * bitpos[out]	- the position of the nearest power of 2 bit set
+ */
+uint8
+bcm_get_ceil_pow_2(uint val)
+{
+	uint8 bitpos = 0;
+	ASSERT(val);
+	if (val & (val-1)) {
+		/* val is not powers of 2.
+		 * pad it, so that allocation will be aligned to
+		 * next immediate powers of 2.
+		 */
+		bitpos = 1;
+	}
+	while (val >>= 1) {
+		bitpos ++;
+	}
+	return (bitpos);
+}
+
+#if !defined(BCMDONGLEHOST)
+/** Initialization of varbuf structure */
+void
+varbuf_init(varbuf_t *b, char *buf, uint size)
+{
+	b->size = size;
+	b->base = b->buf = buf;
+}
+
+/** append a null terminated var=value string */
+int
+varbuf_append(varbuf_t *b, const char *fmt, ...)
+{
+	va_list ap;
+	int r;
+	size_t len;
+	char *s;
+
+	if (b->size < 2)
+	  return 0;
+
+	va_start(ap, fmt);
+	r = vsnprintf(b->buf, b->size, fmt, ap);
+	va_end(ap);
+
+	/* C99 snprintf behavior returns r >= size on overflow,
+	 * others return -1 on overflow.
+	 * All return -1 on format error.
+	 * We need to leave room for 2 null terminations, one for the current var
+	 * string, and one for final null of the var table. So check that the
+	 * strlen written, r, leaves room for 2 chars.
+	 */
+	if ((r == -1) || (r > (int)(b->size - 2))) {
+		b->size = 0;
+		return 0;
+	}
+
+	/* Remove any earlier occurrence of the same variable */
+	if ((s = strchr(b->buf, '=')) != NULL) {
+		len = (size_t)(s - b->buf);
+		for (s = b->base; s < b->buf;) {
+			if ((memcmp(s, b->buf, len) == 0) && s[len] == '=') {
+				len = strlen(s) + 1;
+				memmove(s, (s + len), (size_t)((b->buf + r + 1) - (s + len)));
+				b->buf -= len;
+				b->size += (unsigned int)len;
+				break;
+			}
+
+			while (*s++)
+				;
+		}
+	}
+
+	/* skip over this string's null termination */
+	r++;
+	b->size -= (uint)r;
+	b->buf += r;
+
+	return r;
+}
+
+#if defined(BCMDRIVER)
+/**
+ * Create variable table from memory.
+ * Return 0 on success, nonzero on error.
+ */
+int
+initvars_table(osl_t *osh, char *start, char *end, char **vars,
+	uint *count)
+{
+	uint c = (uint)(end - start);
+
+	/* do it only when there is more than just the null string */
+	if (c > 1) {
+		char *vp = MALLOCZ(osh, c);
+		ASSERT(vp != NULL);
+		if (!vp)
+			return BCME_NOMEM;
+		bcopy(start, vp, c);
+		*vars = vp;
+		*count = c;
+	}
+	else {
+		*vars = NULL;
+		*count = 0;
+	}
+
+	return 0;
+}
+#endif /* BCMDRIVER */
+
+#endif /* !BCMDONGLEHOST */
+
+/* bit shift operation in serialized buffer taking input bits % 8 */
+int buf_shift_right(uint8 *buf, uint16 len, uint8 bits)
+{
+	uint16 i;
+
+	if (len == 0 || (bits == 0) || (bits >= NBBY)) {
+		return BCME_BADARG;
+	}
+
+	for (i = len - 1u; i > 0; i--) {
+		buf[i] = (buf[i - 1u] << (NBBY - bits)) | (buf[i] >> bits);
+	}
+	buf[0] >>= bits;
+
+	return BCME_OK;
+}
+
+/* print the content of the 'buf' in hex string format */
+void
+prhexstr(const char *prefix, const uint8 *buf, uint len, bool newline)
+{
+	if (len > 0) {
+		uint i;
+
+		if (prefix != NULL) {
+			printf("%s", prefix);
+		}
+		for (i = 0; i < len; i ++) {
+			printf("%02X", buf[i]);
+		}
+		if (newline) {
+			printf("\n");
+		}
+	}
+}
+
+/* print the buffer in hex string format with the most significant byte first */
+void
+prhexstr_msb(const char *prefix, const uint8 *buf, uint len, bool newline)
+{
+	if (len > 0) {
+		int i;
+
+		if (prefix != NULL) {
+			printf("%s", prefix);
+		}
+		for (i = (int)len - 1; i >= 0; i --) {
+			printf("%02X", buf[i]);
+		}
+		if (newline) {
+			printf("\n");
+		}
+	}
+}
+
+#ifdef DONGLEBUILD
+static const char BCMPOST_TRAP_RODATA(bcm_print_string)[] = "%s";
+
+int
+BCMPOSTTRAPFN(print_string)(const char *str)
+{
+	return printf(bcm_print_string, str);
+}
+#endif /* DONGLEBUILD */
+
+#ifdef DBG_SEQ_LOG
+#define DBG_SEQ_SEQ_LOG_MAX 10000u
+
+#define DBG_SEQ_SEQ_START (1u << 12u)
+#define DBG_SEQ_SEQ_END   (1u << 13u)
+#define DBG_SEQ_SEQ_MASK   0xFFFu
+
+#define DBG_SEQ_SEQ_INC_UPDATE(id, seq, flags) \
+	do { \
+		g_dbg_seq_seq_idx[id]++; \
+		g_dbg_seq_seq_idx[id] %= DBG_SEQ_SEQ_LOG_MAX; \
+		g_dbg_seq_seq[id][g_dbg_seq_seq_idx[id]] = (seq & DBG_SEQ_SEQ_MASK) | flags; \
+	} while (0)
+
+uint16 g_dbg_seq_last_seq[DBG_SEQ_LOG_ID_MAX] = {0};
+uint16 g_dbg_seq_seq_exp[DBG_SEQ_LOG_ID_MAX] = {0};
+uint16 g_dbg_seq_seq_idx[DBG_SEQ_LOG_ID_MAX] = {0};
+uint16 g_dbg_seq_seq[DBG_SEQ_LOG_ID_MAX][DBG_SEQ_SEQ_LOG_MAX] = {0};
+uint32 g_dbg_seq_pkt[DBG_SEQ_SEQ_LOG_MAX] = {0};
+uint32 g_dbg_seq_pkt_prev_free[DBG_SEQ_SEQ_LOG_MAX] = {0};
+
+/**
+ * @brief Function to log the sequence numbers in a compreesed or uncompressed format.
+ *
+ * @param[in] id    Debug module ID. Refer to dbg_seq_log_id_t for possible values.
+ * @param[in] seq   Sequence number to log
+ * @param[in] arg1  User specific argument 1 to log
+ * @param[in] arg2  User specific argument 2 to log
+ */
+void dbg_seq_log_seq(dbg_seq_log_id_t id, uint16 seq, void *arg1, void *arg2)
+{
+#if DBG_SEQ_LOG_COMPRESSED
+	/* Storing the sequence numbers in compressed format */
+	if ((g_dbg_seq_seq[id][g_dbg_seq_seq_idx[id]] & DBG_SEQ_SEQ_START) &&
+			!(g_dbg_seq_seq[id][g_dbg_seq_seq_idx[id]] & DBG_SEQ_SEQ_END)) {
+		/* Continuation of the previous contigous logging */
+		if ((seq != g_dbg_seq_seq_exp[id]) || (seq == 0)) {
+			/* If we have received a dicontigous seq number or seq number restart then
+			 * close the logging and start new logging. If the seq numbers are
+			 * contigous, then we don't come here.
+			 */
+			if ((g_dbg_seq_seq[id][g_dbg_seq_seq_idx[id]] & DBG_SEQ_SEQ_MASK) ==
+					g_dbg_seq_last_seq[id]) {
+				/* If the contigous seq number count is just 1, then close the
+				 * current logging in the same index and start a new one.
+				 */
+				g_dbg_seq_seq[id][g_dbg_seq_seq_idx[id]] |= DBG_SEQ_SEQ_END;
+			} else {
+				/* If the contigous seq number count is > 1, then close the
+				 * current logging in the next index and start a new one.
+				 */
+				DBG_SEQ_SEQ_INC_UPDATE(id, g_dbg_seq_last_seq[id], DBG_SEQ_SEQ_END);
+			}
+			DBG_SEQ_SEQ_INC_UPDATE(id, seq, DBG_SEQ_SEQ_START);
+		}
+	} else {
+		/* Starting a fresh logging */
+		DBG_SEQ_SEQ_INC_UPDATE(id, seq, DBG_SEQ_SEQ_START);
+	}
+
+	g_dbg_seq_last_seq[id] = seq;
+	g_dbg_seq_seq_exp[id] = MODINC_POW2(seq, SEQNUM_MAX);
+#else
+	/* Storing the sequence numbers in un-compressed format */
+	DBG_SEQ_SEQ_INC_UPDATE(id, seq, (DBG_SEQ_SEQ_START | DBG_SEQ_SEQ_END));
+	g_dbg_seq_last_seq[id] = seq;
+
+	if (id == DBG_SEQ_LOG_ID_RX) {
+		g_dbg_seq_pkt[g_dbg_seq_seq_idx[id]] = (uint32)arg1;
+		g_dbg_seq_pkt_prev_free[g_dbg_seq_seq_idx[id]] = (uint32)arg2;
+	}
+#endif /* DBG_SEQ_LOG_COMPRESSED */
+}
+#endif /* DBG_SEQ_LOG */
+
+/* To be used in computing timestamps on log records. Applicable to dumps carrying
+ * enhanced timestamp records
+ * inputs:
+ * The latest Enhanced timestamp versioned message
+ * Log record time as a 64-bit value
+ * output:
+ * FW time of log record in ns
+ */
+
+#define BCMUTILS_EVENT_LOG_TS_CPU_FREQUENCY_MULTIPLE 1000u
+
+/* Adapted from version 1 timestamp computation in logprint.c:process_event_log_data() */
+static uint64
+bcmutils_event_log_compute_current_time_v1(ets_msg_t *ets_msg, uint32 log_record_time)
+{
+	uint32 base_cycles;
+	uint32 cpu_freq;
+	uint64 base_time;
+	int64 cycle_count_offset;
+
+	ets_msg_v1_t *ets_msg_v1 = (ets_msg_v1_t *)ets_msg;
+
+	/* ets_msg_v1->cyclecount is the ARM cycle count */
+	base_cycles = ets_msg_v1->cyclecount;
+	cpu_freq = ets_msg_v1->cpu_freq * BCMUTILS_EVENT_LOG_TS_CPU_FREQUENCY_MULTIPLE;
+
+	/* ets_msg_v1->timestamp is in ms and holds sys up time */
+	base_time = ets_msg_v1->timestamp;
+
+	cycle_count_offset = log_record_time - base_cycles;
+	if (cycle_count_offset >= 0) {
+		base_time += (log_record_time - base_cycles)/cpu_freq;
+	} else {
+		/* Negative cycle_count_offset == counter wrapped */
+		base_time += (log_record_time +
+				(0xffffffff - base_cycles))/cpu_freq;
+	}
+
+	/* basetime is in ms. convert to uint64 and return */
+	return base_time;
+}
+
+static uint64
+bcmutils_event_log_compute_current_time_v2(ets_msg_t *ets_msg, uint64 q4_log_record_time)
+{
+	ets_msg_v2_t *ets_msg_v2 = (ets_msg_v2_t *)ets_msg->data;
+	uint64 ets_msg_40bit_ptm_time = ets_msg_v2->ets_write_ptm_time & 0xFFFFFFFFFFULL;
+	uint64 el_40bit_log_record_time = q4_log_record_time << 8;
+
+	uint64 delta_time;
+
+	/* Corner case. The top 40 bits match indicating logging happened in the same time unit
+	 * as recorded RAW PTM time in the ETS message.
+	 */
+	if (q4_log_record_time == (ets_msg_40bit_ptm_time >> 8)) {
+		delta_time = 0;
+	} else {
+		delta_time = (el_40bit_log_record_time - ets_msg_40bit_ptm_time) &
+			0xFFFFFFFFFFULL;
+	}
+	return ets_msg_v2->sysuptime_ns + delta_time;
+}
+
+int
+bcmutils_event_log_compute_current_time(void *ets_msg, uint64 log_record_time, uint64 *current_time)
+{
+	ets_msg_t *ets_msg_ptr = (ets_msg_t *)ets_msg;
+
+	if (ets_msg_ptr == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (ets_msg_ptr->version == ENHANCED_TS_MSG_VERSION_1) {
+		/* log record time is a 32-bit ARM cycle count */
+		*current_time = bcmutils_event_log_compute_current_time_v1(ets_msg_ptr,
+			(uint32) log_record_time);
+	} else if (ets_msg_ptr->version == ENHANCED_TS_MSG_VERSION_2) {
+		*current_time = bcmutils_event_log_compute_current_time_v2(ets_msg_ptr,
+			log_record_time);
+	} else {
+		return BCME_VERSION;
+	}
+
+	return BCME_OK;
+}
+
+#ifdef DBG_PKTID_POOLPKT_PROCESS
+uint16
+bcm_mwbmap_total_pktid(struct bcm_mwbmap * mwbmap_hdl)
+{
+	bcm_mwbmap_t * mwbmap_p;
+
+	BCM_MWBMAP_AUDIT(mwbmap_hdl);
+	mwbmap_p = BCM_MWBMAP_PTR(mwbmap_hdl);
+
+	return mwbmap_p->total;
+}
+#endif /* DBG_PKTID_POOLPKT_PROCESS */
+
+/**
+ * @brief Divide the uint64 dividend 'q' by 1,000
+ *
+ * @param[in] dividend
+ *
+ * @return    quotient
+ */
+uint64
+BCMPOSTTRAPFASTPATH(div_1K)(uint64 q)
+{
+	/** 1/1000 fractional bits: 0x004189374BC6A7EF(9DB)
+	 * shift until no leading zeros: 0x83126E97'8D4FDF3B S=9
+	 */
+
+	const uint64 d1 = 0x83126E97llu; // first 32 fractional bits
+	const uint64 d2 = 0x8D4FDF3Bllu; // second 32 fractional bits
+	const uint64 S  = 9llu;
+
+	uint64 a1;
+	uint64 a;  // answer
+	int carry;
+	uint32 a_low;
+	uint32 lo; // low 32 bits of quotient
+	uint32 hi; // high 32 bits of quotient
+
+	lo = (uint32)q;
+	hi = (uint32)(q >> 32u);
+
+	// fractional d1/d2 values truncate all lower order bits, so
+	// add a low order bit to make the fraction slightly higher than the
+	// exact 1/1000. This allows the final truncation of the answer to
+	// no come up short.
+	a1 = lo * (d1 + 1);
+	a  = hi * (d2 + 1) + a1;
+
+	// Detect carry
+	carry = (a < a1);
+
+	a_low = (uint32)(a >> 32u);
+
+	a = (hi * d1) + a_low;
+	if (carry) {
+		a += 0x100000000llu;
+	}
+	a = a >> S;
+
+	return a;
+}
+
+/**
+ * @brief Divide the uint64 dividend 'q' by 1,000,000
+ *
+ * @param[in] dividend
+ *
+ * @return    quotient
+ */
+uint64
+BCMPOSTTRAPFASTPATH(div_1M)(uint64 q)
+{
+	/** 1/1000000 fractional bits: 0x000010C6F7A0B5ED8D36B
+	 * shift until no leading zeros: 0x8637BD05'AF6C69B5 S=19
+	 */
+
+	const uint64 d1 = 0x8637BD05llu; // first 32 fractional bits
+	const uint64 d2 = 0xAF6C69B5llu; // second 32 fractional bits
+	const uint64 S = 19llu;
+
+	uint64 a1;
+	uint64 a;  // answer
+	int carry;
+	uint32 a_low;
+	uint32 lo; // low 32 bits of quotent
+	uint32 hi; // high 32 bits of quotent
+
+	lo = (uint32)q;
+	hi = (uint32)(q >> 32u);
+
+	// fractional d1/d2 values truncate all lower order bits, so
+	// add a low order bit to make the fraction slightly higher than the
+	// exact 1/1000. This allows the final truncation of the answer to
+	// no come up short.
+	a1 = lo * (d1 + 1);
+	a = hi * (d2 + 1) + a1;
+
+	// Detect carry
+	carry = (a < a1);
+
+	a_low = (uint32)(a >> 32u);
+
+	a = (hi * d1) + a_low;
+	if (carry) {
+		a += 0x100000000llu;
+	}
+	a = a >> S;
+
+	return a;
+}
+
+/* Add to adjust the 802.1x priority */
+void
+pktset8021xprio(void *pkt, int prio)
+{
+	struct ether_header *eh;
+	uint8 *pktdata;
+	if(prio == PKTPRIO(pkt))
+		return;
+	pktdata = (uint8 *)PKTDATA(OSH_NULL, pkt);
+	ASSERT(ISALIGNED((uintptr)pktdata, sizeof(uint16)));
+	eh = (struct ether_header *) pktdata;
+	if (eh->ether_type == hton16(ETHER_TYPE_802_1X)) {
+		ASSERT(prio >= 0 && prio <= MAXPRIO);
+		PKTSETPRIO(pkt, prio);
+	}
+}
diff --git a/drivers/net/wireless/ap12275/bcmwifi_channels.c b/drivers/net/wireless/ap12275/bcmwifi_channels.c
new file mode 100644
index 000000000000..9333915ea703
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmwifi_channels.c
@@ -0,0 +1,3333 @@
+/*
+ * Misc utility routines used by kernel or app-level.
+ * Contents are wifi-specific, used by any kernel or app-level
+ * software that might want wifi things as it grows.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmutils.h>
+
+#ifdef BCMDRIVER
+#include <osl.h>
+#define strtoul(nptr, endptr, base) bcm_strtoul((nptr), (endptr), (base))
+#ifndef tolower
+#define tolower(c) (bcm_isupper((c)) ? ((c) + 'a' - 'A') : (c))
+#endif
+#else
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#ifndef ASSERT
+#define ASSERT(exp)
+#endif
+#endif /* BCMDRIVER */
+
+#include <bcmwifi_channels.h>
+
+#if defined(WIN32) && (defined(BCMDLL) || defined(WLMDLL))
+#include <bcmstdlib.h>	/* For wlexe/Makefile.wlm_dll */
+#endif
+
+#include <802.11.h>
+#include <802.11ac.h>
+
+/* Definitions for D11AC capable (80MHz+) Chanspec type */
+
+/* Chanspec ASCII representation:
+ *
+ * [<band>'g']<channel>['/'<bandwidth>[<primary-sideband>]
+ *    ['/'<1st-channel-segment>'-'<2nd-channel-segment>]]
+ *
+ * <band>:
+ *      (optional) 2, 4, 5, 6 for 2.4GHz, 4GHz, 5GHz, and 6GHz respectively.
+ *      Default value is 2g if channel <= 14, otherwise 5g.
+ * <channel>:
+ *      channel number of the 20MHz channel,
+ *      or primary 20 MHz channel of 40MHz, 80MHz, 160MHz, 80+80MHz,
+ *      240MHz, 320MHz, or 160+160MHz channels.
+ * <bandwidth>:
+ *      (optional) 20, 40, 80, 160, 80+80, 240, 320, or 160+160. Default value is 20.
+ * <primary-sideband>:
+ *      'u' or 'l' (only for 2.4GHz band 40MHz)
+ *
+ *       For 2.4GHz band 40MHz channels, the same primary channel may be the
+ *       upper sideband for one 40MHz channel, and the lower sideband for an
+ *       overlapping 40MHz channel. The {u: upper, l: lower} primary sideband
+ *       indication disambiguates which 40MHz channel is being specified.
+ *
+ *       For 40MHz in the 5GHz or 6GHz band and all channel bandwidths greater than
+ *       40MHz, the U/L specification is not necessary or allowed since the channels are
+ *       non-overlapping and the primary 20MHz channel position is derived from its
+ *       position in the wide bandwidth channel.
+ * <1st-channel-segment>
+ * <2nd-channel-segment>:
+ *       Required for 80+80 or 160+160, otherwise not allowed.
+ *       These fields specify the center channel of the first and the second 80MHz
+ *       or 160MHz channels.
+ *
+ * In its simplest form, it is a 20MHz channel number, with the implied band
+ * of 2.4GHz if channel number <= 14, and 5GHz otherwise.
+ *
+ * To allow for backward compatibility with scripts, the old form for
+ * 40MHz channels is also allowed: <channel><primary-sideband>
+ *
+ * <channel>:
+ *	primary channel of 40MHz, channel <= 14 is 2GHz, otherwise 5GHz
+ * <primary-sideband>:
+ *	"U" for upper, "L" for lower (or lower case "u" "l")
+ *
+ * 5 GHz Examples:
+ *      Chanspec        BW        Center Ch  Channel Range  Primary Ch
+ *      5g8             20MHz     8          -              -
+ *      52              20MHz     52         -              -
+ *      52/40           40MHz     54         52-56          52
+ *      56/40           40MHz     54         52-56          56
+ *      52/80           80MHz     58         52-64          52
+ *      56/80           80MHz     58         52-64          56
+ *      60/80           80MHz     58         52-64          60
+ *      64/80           80MHz     58         52-64          64
+ *      52/160          160MHz    50         36-64          52
+ *      36/160          160MGz    50         36-64          36
+ *      36/80+80/42-106 80+80MHz  42,106     36-48,100-112  36
+ *
+ * 2 GHz Examples:
+ *      Chanspec        BW        Center Ch  Channel Range  Primary Ch
+ *      2g8             20MHz     8          -              -
+ *      8               20MHz     8          -              -
+ *      6               20MHz     6          -              -
+ *      6/40l           40MHz     8          6-10           6
+ *      6l              40MHz     8          6-10           6
+ *      6/40u           40MHz     4          2-6            6
+ *      6u              40MHz     4          2-6            6
+ */
+
+/* bandwidth ASCII string */
+static const char *wf_chspec_bw_str[] =
+{
+	"320",
+	"160+160",
+	"20",
+	"40",
+	"80",
+	"160",
+	"80+80",
+	"240"
+};
+
+static const uint16 wf_chspec_bw_mhz[] = {
+	320, 320, 20, 40, 80, 160, 160, 240
+};
+#define WF_NUM_BW ARRAYSIZE(wf_chspec_bw_mhz)
+
+/* 40MHz channels in 2.4GHz band */
+static const uint8 wf_2g_40m_chans[] = {
+	3, 4, 5, 6, 7, 8, 9, 10, 11
+};
+#define WF_NUM_2G_40M_CHANS ARRAYSIZE(wf_2g_40m_chans)
+
+/* 40MHz channels in 5GHz band */
+static const uint8 wf_5g_40m_chans[] = {
+	38, 46, 54, 62, 102, 110, 118, 126, 134, 142, 151, 159, 167, 175
+};
+#define WF_NUM_5G_40M_CHANS ARRAYSIZE(wf_5g_40m_chans)
+
+/* 80MHz channels in 5GHz band */
+static const uint8 wf_5g_80m_chans[] = {
+	42, 58, 106, 122, 138, 155, 171
+};
+#define WF_NUM_5G_80M_CHANS ARRAYSIZE(wf_5g_80m_chans)
+
+/* 160MHz channels in 5GHz band */
+static const uint8 wf_5g_160m_chans[] = {
+	50, 114, 163
+};
+#define WF_NUM_5G_160M_CHANS ARRAYSIZE(wf_5g_160m_chans)
+
+/** 80MHz channels in 6GHz band */
+#define WF_NUM_6G_80M_CHANS 14
+
+/** 160MHz channels in 6GHz band */
+#define WF_NUM_6G_160M_CHANS 7	/* TBD */
+
+/** 240MHz channels in 6GHz band */
+#define WF_NUM_6G_240M_CHANS 4 /* TBD */
+
+/** 320MHz channels in 6GHz band */
+#define WF_NUM_6G_320M_CHANS 3	/* TBD */
+
+/* Define the conditional macro to help with reducing the code size bloat
+ * in other branches and in trunk targets that don't need 11BE features...
+ */
+#define WFC_2VALS_EQ(var, val)	((var) == (val))
+
+/* compare bandwidth unconditionally for 11be related stuff */
+#ifdef WL11BE
+#define WFC_BW_EQ(bw, val)	WFC_2VALS_EQ(bw, val)
+#else
+#define WFC_BW_EQ(bw, val)	(FALSE)
+#endif
+
+/* compare bandwidth based on WFC_NON_CONT_CHAN */
+#ifdef WFC_NON_CONT_CHAN
+#define WFC_NCBW_EQ(bw, val)	WFC_2VALS_EQ(bw, val)
+#else
+#define WFC_NCBW_EQ(bw, val)	(FALSE)
+#endif
+
+static void wf_chanspec_iter_firstchan(wf_chanspec_iter_t *iter);
+static chanspec_bw_t wf_iter_next_bw(chanspec_bw_t bw);
+static bool wf_chanspec_iter_next_2g(wf_chanspec_iter_t *iter);
+static bool wf_chanspec_iter_next_5g(wf_chanspec_iter_t *iter);
+static int wf_chanspec_iter_next_5g_range(wf_chanspec_iter_t *iter, chanspec_bw_t bw);
+static void wf_chanspec_iter_6g_range_init(wf_chanspec_iter_t *iter, chanspec_bw_t bw);
+static bool wf_chanspec_iter_next_6g(wf_chanspec_iter_t *iter);
+
+/**
+ * Return the chanspec bandwidth in MHz
+ * Bandwidth of 160 MHz will be returned for 80+80MHz chanspecs.
+ *
+ * @param	chspec		chanspec_t
+ *
+ * @return	bandwidth of chspec in MHz units
+ */
+uint
+wf_bw_chspec_to_mhz(chanspec_t chspec)
+{
+	uint bw;
+
+	bw = (chspec & WL_CHANSPEC_BW_MASK) >> WL_CHANSPEC_BW_SHIFT;
+	return (bw >= WF_NUM_BW ? 0 : wf_chspec_bw_mhz[bw]);
+}
+
+/* bw in MHz, return the channel count from the center channel to the
+ * the channel at the edge of the band
+ */
+static uint
+center_chan_to_edge(chanspec_bw_t bw)
+{
+	uint delta = 0;
+
+	/* edge channels separated by BW - 10MHz on each side
+	 * delta from cf to edge is half of that,
+	 */
+	if (bw == WL_CHANSPEC_BW_40) {
+		/* 10 MHz */
+		delta = 2;
+	} else if (bw == WL_CHANSPEC_BW_80) {
+		/* 30 MHz */
+		delta = 6;
+	} else if (bw == WL_CHANSPEC_BW_160) {
+		/* 70 MHz */
+		delta = 14;
+	} else if (WFC_BW_EQ(bw, WL_CHANSPEC_BW_240)) {
+		/* 110 MHz */
+		delta = 22;
+	} else if (WFC_BW_EQ(bw, WL_CHANSPEC_BW_320)) {
+		/* 150 MHz */
+		delta = 30;
+	}
+	return delta;
+}
+
+/* return channel number of the low edge of the band
+ * given the center channel and BW
+ */
+static uint
+channel_low_edge(uint center_ch, chanspec_bw_t bw)
+{
+	return (center_ch - center_chan_to_edge(bw));
+}
+
+/* return side band number given center channel and primary20 channel
+ * return -1 on error
+ */
+static int
+channel_to_sb(uint center_ch, uint primary_ch, chanspec_bw_t bw)
+{
+	uint lowest = channel_low_edge(center_ch, bw);
+	uint sb;
+
+	if (primary_ch < lowest ||
+	    (primary_ch - lowest) % 4) {
+		/* bad primary channel lower than the low edge of the channel,
+		 * or not mult 4.
+		 */
+		return -1;
+	}
+
+	sb = ((primary_ch - lowest) / 4);
+
+	/* sb must be a index to a 20MHz channel in range */
+	if ((bw == WL_CHANSPEC_BW_20 && sb >= 1) ||
+	    (bw == WL_CHANSPEC_BW_40 && sb >= 2) ||
+	    (bw == WL_CHANSPEC_BW_80 && sb >= 4) ||
+	    (bw == WL_CHANSPEC_BW_160 && sb >= 8) ||
+	    (WFC_BW_EQ(bw, WL_CHANSPEC_BW_240) && sb >= 12) ||
+	    (WFC_BW_EQ(bw, WL_CHANSPEC_BW_320) && sb >= 16)) {
+		/* primary_ch must have been too high for the center_ch */
+		return -1;
+	}
+
+	return sb;
+}
+
+/* return primary20 channel given center channel and side band */
+static uint
+channel_to_primary20_chan(uint center_ch, chanspec_bw_t bw, uint sb)
+{
+	return (channel_low_edge(center_ch, bw) + sb * 4);
+}
+
+/* return index of 80MHz channel from channel number
+ * return -1 on error
+ */
+static int
+channel_80mhz_to_id(uint ch)
+{
+	uint i;
+	for (i = 0; i < WF_NUM_5G_80M_CHANS; i ++) {
+		if (ch == wf_5g_80m_chans[i])
+			return i;
+	}
+
+	return -1;
+}
+
+/* return index of the 6G 80MHz channel from channel number
+ * return -1 on error
+ */
+static int
+channel_6g_80mhz_to_id(uint ch)
+{
+	/* The 6GHz center channels start at 7, and have a spacing of 16 */
+	if (ch >= CH_MIN_6G_80M_CHANNEL &&
+	    ch <= CH_MAX_6G_80M_CHANNEL &&
+	    ((ch - CH_MIN_6G_80M_CHANNEL) % 16) == 0) {  // even multiple of 16
+		return (ch - CH_MIN_6G_80M_CHANNEL) / 16;
+	}
+
+	return -1;
+}
+
+/* return index of the 5G 160MHz channel from channel number
+ * return -1 on error
+ */
+static int
+channel_5g_160mhz_to_id(uint ch)
+{
+	uint i;
+	for (i = 0; i < WF_NUM_5G_160M_CHANS; i ++) {
+		if (ch == wf_5g_160m_chans[i]) {
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+/* return index of the 6G 160MHz channel from channel number
+ * return -1 on error
+ */
+static int
+channel_6g_160mhz_to_id(uint ch)
+{
+	/* The 6GHz center channels start at 15, and have a spacing of 32 */
+	if (ch >= CH_MIN_6G_160M_CHANNEL &&
+	    ch <= CH_MAX_6G_160M_CHANNEL &&
+	    ((ch - CH_MIN_6G_160M_CHANNEL) % 32) == 0) {
+		return (ch - CH_MIN_6G_160M_CHANNEL) / 32;
+	}
+
+	return -1;
+}
+
+/* return index of the 6G 240MHz channel from channel number
+ * return -1 on error
+ */
+static int
+channel_6g_240mhz_to_id(uint ch)
+{
+	/* The 6GHz center channels start at 23, and have a spacing of 48 */
+	if (ch >= CH_MIN_6G_240M_CHANNEL &&
+	    ch <= CH_MAX_6G_240M_CHANNEL &&
+	    ((ch - CH_MIN_6G_240M_CHANNEL) % 48) == 0) {
+		return (ch - CH_MIN_6G_240M_CHANNEL) / 48;
+	}
+
+	return -1;
+}
+
+/* return index of the 6G 320MHz channel from channel number
+ * return -1 on error
+ */
+static int
+channel_6g_320mhz_to_id(uint ch)
+{
+	/* The 6GHz center channels start at 31, and have a spacing of 64 */
+	if (ch >= CH_MIN_6G_320M_CHANNEL &&
+	    ch <= CH_MAX_6G_320M_CHANNEL &&
+	    ((ch - CH_MIN_6G_320M_CHANNEL) % 64) == 0) {
+		return (ch - CH_MIN_6G_320M_CHANNEL) / 64;
+	}
+
+	return -1;
+}
+
+/**
+ * This function returns the the 5GHz 80MHz center channel for the given chanspec 80MHz ID
+ *
+ * @param    chan_80MHz_id    80MHz chanspec ID
+ *
+ * @return   Return the center channel number, or 0 on error.
+ *
+ */
+static uint8
+wf_chspec_5G_id80_to_ch(uint8 chan_80MHz_id)
+{
+	if (chan_80MHz_id < WF_NUM_5G_80M_CHANS) {
+		return wf_5g_80m_chans[chan_80MHz_id];
+	}
+
+	return 0;
+}
+
+/**
+ * This function returns the the 6GHz 80MHz center channel for the given chanspec 80MHz ID
+ *
+ * @param    chan_80MHz_id    80MHz chanspec ID
+ *
+ * @return   Return the center channel number, or 0 on error.
+ *
+ */
+static uint8
+wf_chspec_6G_id80_to_ch(uint8 chan_80MHz_id)
+{
+	uint8 ch = 0;
+
+	if (chan_80MHz_id < WF_NUM_6G_80M_CHANS) {
+		/* The 6GHz center channels have a spacing of 16
+		 * starting from the first 80MHz center
+		 */
+		ch = CH_MIN_6G_80M_CHANNEL + (chan_80MHz_id * 16);
+	}
+
+	return ch;
+}
+
+/**
+ * This function returns the the 5GHz 160MHz center channel for the given chanspec 160MHz ID
+ *
+ * @param    chan_160MHz_id    160MHz chanspec ID
+ *
+ * @return   Return the center channel number, or 0 on error.
+ *
+ */
+static uint8
+wf_chspec_5G_id160_to_ch(uint8 chan_160MHz_id)
+{
+	if (chan_160MHz_id < WF_NUM_5G_160M_CHANS) {
+		return wf_5g_160m_chans[chan_160MHz_id];
+	}
+
+	return 0;
+}
+
+/**
+ * This function returns the the 6GHz 160MHz center channel for the given chanspec 160MHz ID
+ *
+ * @param    chan_160MHz_id    160MHz chanspec ID
+ *
+ * @return   Return the center channel number, or 0 on error.
+ *
+ */
+static uint8
+wf_chspec_6G_id160_to_ch(uint8 chan_160MHz_id)
+{
+	uint8 ch = 0;
+
+	if (chan_160MHz_id < WF_NUM_6G_160M_CHANS) {
+		/* The 6GHz center channels have a spacing of 32
+		 * starting from the first 160MHz center
+		 */
+		ch = CH_MIN_6G_160M_CHANNEL + (chan_160MHz_id * 32);
+	}
+
+	return ch;
+}
+
+/**
+ * This function returns the the 6GHz 240MHz center channel for the given chanspec 240MHz ID
+ *
+ * @param    chan_240MHz_id    240MHz chanspec ID
+ *
+ * @return   Return the center channel number, or 0 on error.
+ *
+ */
+static uint8
+wf_chspec_6G_id240_to_ch(uint8 chan_240MHz_id)
+{
+	uint8 ch = 0;
+
+	if (chan_240MHz_id < WF_NUM_6G_240M_CHANS) {
+		/* The 6GHz center channels have a spacing of 48
+		 * starting from the first 240MHz center
+		 */
+		ch = CH_MIN_6G_240M_CHANNEL + (chan_240MHz_id * 48);
+	}
+
+	return ch;
+}
+
+/* Retrive the chan_id and convert it to center channel */
+uint8
+wf_chspec_240_id2cch(chanspec_t chanspec)
+{
+	if (CHSPEC_BAND(chanspec) == WL_CHANSPEC_BAND_6G &&
+	    CHSPEC_BW(chanspec) == WL_CHANSPEC_BW_240) {
+		uint8 ch_id = CHSPEC_GE240_CHAN(chanspec);
+
+		return wf_chspec_6G_id240_to_ch(ch_id);
+	}
+	return 0;
+}
+
+/**
+ * This function returns the the 6GHz 320MHz center channel for the given chanspec 320MHz ID
+ *
+ * @param    chan_320MHz_id    320MHz chanspec ID
+ *
+ * @return   Return the center channel number, or 0 on error.
+ *
+ */
+static uint8
+wf_chspec_6G_id320_to_ch(uint8 chan_320MHz_id)
+{
+	uint8 ch = 0;
+
+	if (chan_320MHz_id < WF_NUM_6G_320M_CHANS) {
+		/* The 6GHz center channels have a spacing of 64
+		 * starting from the first 320MHz center
+		 */
+		ch = CH_MIN_6G_320M_CHANNEL + (chan_320MHz_id * 64);
+	}
+
+	return ch;
+}
+
+/* Retrive the chan_id and convert it to center channel */
+uint8
+wf_chspec_320_id2cch(chanspec_t chanspec)
+{
+	if (CHSPEC_BAND(chanspec) == WL_CHANSPEC_BAND_6G &&
+	    CHSPEC_BW(chanspec) == WL_CHANSPEC_BW_320) {
+		uint8 ch_id = CHSPEC_GE240_CHAN(chanspec);
+
+		return wf_chspec_6G_id320_to_ch(ch_id);
+	}
+	return 0;
+}
+
+/**
+ * Convert chanspec to ascii string, or formats hex of an invalid chanspec.
+ *
+ * @param	chspec   chanspec to format
+ * @param	buf      pointer to buf with room for at least CHANSPEC_STR_LEN bytes
+ *
+ * @return      Returns pointer to passed in buf. The buffer will have the ascii
+ *              representation of the given chspec, or "invalid 0xHHHH" where
+ *              0xHHHH is the hex representation of the invalid chanspec.
+ *
+ * @see		CHANSPEC_STR_LEN
+ *
+ * Wrapper function for wf_chspec_ntoa. In case of an error it puts
+ * the original chanspec in the output buffer, prepended with "invalid".
+ * Can be directly used in print routines as it takes care of null
+ */
+char *
+wf_chspec_ntoa_ex(chanspec_t chspec, char *buf)
+{
+	if (wf_chspec_ntoa(chspec, buf) == NULL)
+		snprintf(buf, CHANSPEC_STR_LEN, "invalid 0x%04x", chspec);
+	return buf;
+}
+
+/**
+ * Convert chanspec to ascii string, or return NULL on error.
+ *
+ * @param	chspec   chanspec to format
+ * @param	buf      pointer to buf with room for at least CHANSPEC_STR_LEN bytes
+ *
+ * @return      Returns pointer to passed in buf or NULL on error. On sucess, the buffer
+ *              will have the ascii representation of the given chspec.
+ *
+ * @see		CHANSPEC_STR_LEN
+ *
+ * Given a chanspec and a string buffer, format the chanspec as a
+ * string, and return the original pointer buf.
+ * Min buffer length must be CHANSPEC_STR_LEN.
+ * On error return NULL.
+ */
+char *
+wf_chspec_ntoa(chanspec_t chspec, char *buf)
+{
+	const char *band;
+	uint pri_chan;
+
+	if (wf_chspec_malformed(chspec))
+		return NULL;
+
+	band = "";
+
+	/* check for non-default band spec */
+	if (CHSPEC_IS2G(chspec) && CHSPEC_CHANNEL(chspec) > CH_MAX_2G_CHANNEL) {
+		band = "2g";
+	} else if (CHSPEC_IS5G(chspec) && CHSPEC_CHANNEL(chspec) <= CH_MAX_2G_CHANNEL) {
+		band = "5g";
+	} else if (CHSPEC_IS6G(chspec)) {
+		band = "6g";
+	}
+
+	/* primary20 channel */
+	pri_chan = wf_chspec_primary20_chan(chspec);
+
+	/* bandwidth and primary20 sideband */
+	if (CHSPEC_IS20(chspec)) {
+		snprintf(buf, CHANSPEC_STR_LEN, "%s%d", band, pri_chan);
+	} else if (CHSPEC_IS8080(chspec)) {
+		/* 80+80 */
+		uint ch0;
+		uint ch1;
+
+		/* get the center channels for each frequency segment */
+		if (CHSPEC_IS5G(chspec)) {
+			ch0 = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN0(chspec));
+			ch1 = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN1(chspec));
+		} else if (CHSPEC_IS6G(chspec)) {
+			ch0 = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN0(chspec));
+			ch1 = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN1(chspec));
+		} else {
+			return NULL;
+		}
+
+		/* Outputs a max of CHANSPEC_STR_LEN chars including '\0'  */
+		snprintf(buf, CHANSPEC_STR_LEN, "%s%d/80+80/%d-%d", band, pri_chan, ch0, ch1);
+	} else if (CHSPEC_IS240(chspec)) {
+		/* 240 */
+		const char *bw;
+
+		bw = wf_chspec_to_bw_str(chspec);
+
+		snprintf(buf, CHANSPEC_STR_LEN, "%s%d/%s", band, pri_chan, bw);
+	} else if (CHSPEC_IS320(chspec)) {
+		/* 320 */
+		const char *bw;
+
+		bw = wf_chspec_to_bw_str(chspec);
+
+		snprintf(buf, CHANSPEC_STR_LEN, "%s%d/%s", band, pri_chan, bw);
+	} else if (CHSPEC_IS160160(chspec)) {
+		/* 160+160 */
+		uint ch0;
+		uint ch1;
+
+		/* get the center channels for each frequency segment */
+		if (CHSPEC_IS5G(chspec)) {
+			ch0 = wf_chspec_5G_id160_to_ch(CHSPEC_CHAN0(chspec));
+			ch1 = wf_chspec_5G_id160_to_ch(CHSPEC_CHAN1(chspec));
+		} else if (CHSPEC_IS6G(chspec)) {
+			ch0 = wf_chspec_6G_id160_to_ch(CHSPEC_CHAN0(chspec));
+			ch1 = wf_chspec_6G_id160_to_ch(CHSPEC_CHAN1(chspec));
+		} else {
+			return NULL;
+		}
+
+		/* Outputs a max of CHANSPEC_STR_LEN chars including '\0'  */
+		snprintf(buf, CHANSPEC_STR_LEN, "%s%d/160+160/%d-%d", band, pri_chan, ch0, ch1);
+	} else {
+		const char *bw;
+		const char *sb = "";
+
+		bw = wf_chspec_to_bw_str(chspec);
+
+#ifdef CHANSPEC_NEW_40MHZ_FORMAT
+		/* primary20 sideband string if needed for 2g 40MHz */
+		if (CHSPEC_IS40(chspec) && CHSPEC_IS2G(chspec)) {
+			sb = CHSPEC_SB_UPPER(chspec) ? "u" : "l";
+		}
+
+		snprintf(buf, CHANSPEC_STR_LEN, "%s%d/%s%s", band, pri_chan, bw, sb);
+#else
+		/* primary20 sideband string instead of BW for 40MHz */
+		if (CHSPEC_IS40(chspec) && !CHSPEC_IS6G(chspec)) {
+			sb = CHSPEC_SB_UPPER(chspec) ? "u" : "l";
+			snprintf(buf, CHANSPEC_STR_LEN, "%s%d%s", band, pri_chan, sb);
+		} else {
+			snprintf(buf, CHANSPEC_STR_LEN, "%s%d/%s", band, pri_chan, bw);
+		}
+#endif /* CHANSPEC_NEW_40MHZ_FORMAT */
+	}
+
+	return (buf);
+}
+
+static int
+read_uint(const char **p, unsigned int *num)
+{
+	unsigned long val;
+	char *endp = NULL;
+
+	val = strtoul(*p, &endp, 10);
+	/* if endp is the initial pointer value, then a number was not read */
+	if (endp == *p)
+		return 0;
+
+	/* advance the buffer pointer to the end of the integer string */
+	*p = endp;
+	/* return the parsed integer */
+	*num = (unsigned int)val;
+
+	return 1;
+}
+
+/**
+ * Convert ascii string to chanspec
+ *
+ * @param	a     pointer to input string
+ *
+ * @return	Return > 0 if successful or 0 otherwise
+ */
+chanspec_t
+wf_chspec_aton(const char *a)
+{
+	chanspec_t chspec;
+	chanspec_band_t chspec_band;
+	chanspec_subband_t chspec_sb;
+	chanspec_bw_t chspec_bw;
+	uint bw;
+	uint num, pri_ch;
+	char c, sb_ul = '\0';
+
+	bw = 20;
+	chspec_sb = 0;
+
+	/* parse channel num or band */
+	if (!read_uint(&a, &num))
+		return 0;
+	/* if we are looking at a 'g', then the first number was a band */
+	c = tolower((int)a[0]);
+	if (c == 'g') {
+		a++; /* consume the char */
+
+		/* band must be "2", "5", or "6" */
+		if (num == 2)
+			chspec_band = WL_CHANSPEC_BAND_2G;
+		else if (num == 5)
+			chspec_band = WL_CHANSPEC_BAND_5G;
+		else if (num == 6)
+			chspec_band = WL_CHANSPEC_BAND_6G;
+		else
+			return 0;
+
+		/* read the channel number */
+		if (!read_uint(&a, &pri_ch))
+			return 0;
+
+		c = tolower((int)a[0]);
+	} else {
+		/* first number is channel, use default for band */
+		pri_ch = num;
+		chspec_band = ((pri_ch <= CH_MAX_2G_CHANNEL) ?
+		               WL_CHANSPEC_BAND_2G : WL_CHANSPEC_BAND_5G);
+	}
+
+	if (c == '\0') {
+		/* default BW of 20MHz */
+		chspec_bw = WL_CHANSPEC_BW_20;
+		goto done_read;
+	}
+
+	a ++; /* consume the 'u','l', or '/' */
+
+	/* check 'u'/'l' */
+	if (c == 'u' || c == 'l') {
+		sb_ul = c;
+		chspec_bw = WL_CHANSPEC_BW_40;
+		goto done_read;
+	}
+
+	/* next letter must be '/' */
+	if (c != '/')
+		return 0;
+
+	/* read bandwidth */
+	if (!read_uint(&a, &bw))
+		return 0;
+
+	/* convert to chspec value */
+	if (bw == 20) {
+		chspec_bw = WL_CHANSPEC_BW_20;
+	} else if (bw == 40) {
+		chspec_bw = WL_CHANSPEC_BW_40;
+	} else if (bw == 80) {
+		chspec_bw = WL_CHANSPEC_BW_80;
+	} else if (bw == 160) {
+		chspec_bw = WL_CHANSPEC_BW_160;
+	} else if (WFC_BW_EQ(bw, 240)) {
+		chspec_bw = WL_CHANSPEC_BW_240;
+	} else if (WFC_BW_EQ(bw, 320)) {
+		chspec_bw = WL_CHANSPEC_BW_320;
+	} else {
+		return 0;
+	}
+
+	/* So far we have <band>g<chan>/<bw>
+	 * Can now be followed by u/l if bw = 40,
+	 */
+
+	c = tolower((int)a[0]);
+
+	/* if we have a 2g/40 channel, we should have a l/u spec now */
+	if (chspec_band == WL_CHANSPEC_BAND_2G && bw == 40) {
+		if (c == 'u' || c == 'l') {
+			a ++; /* consume the u/l char */
+			sb_ul = c;
+			goto done_read;
+		}
+	}
+
+	/* check for 80+80 or 160+160 */
+	if (c == '+') {
+		return 0;
+	}
+
+done_read:
+	/* skip trailing white space */
+	while (a[0] == ' ') {
+		a ++;
+	}
+
+	/* must be end of string */
+	if (a[0] != '\0')
+		return 0;
+
+	/* Now have all the chanspec string parts read;
+	 * chspec_band, pri_ch, chspec_bw, sb_ul.
+	 * chspec_band and chspec_bw are chanspec values.
+	 * Need to convert pri_ch, and sb_ul into
+	 * a center channel (or two) and sideband.
+	 */
+
+	/* if a sb u/l string was given, just use that,
+	 * guaranteed to be bw = 40 by string parse.
+	 */
+	if (sb_ul != '\0') {
+		if (sb_ul == 'l') {
+			chspec_sb = WL_CHANSPEC_CTL_SB_LLL;
+		} else if (sb_ul == 'u') {
+			chspec_sb = WL_CHANSPEC_CTL_SB_LLU;
+		}
+		chspec = wf_create_40MHz_chspec_primary_sb(pri_ch, chspec_sb, chspec_band);
+	} else if (chspec_bw == WL_CHANSPEC_BW_20) {
+		/* if the bw is 20, only need the primary channel and band */
+		chspec = wf_create_20MHz_chspec(pri_ch, chspec_band);
+	} else {
+		/* If the bw is 40/80/160/240/320 (and not 40MHz 2G), the channels are
+		 * non-overlapping in 5G or 6G bands. Each primary channel is contained
+		 * in only one higher bandwidth channel. The wf_create_chspec_from_primary()
+		 * will create the chanspec. 2G 40MHz is handled just above, assuming a {u,l}
+		 * sub-band spec was given.
+		 */
+		chspec = wf_create_chspec_from_primary(pri_ch, chspec_bw, chspec_band);
+	}
+
+	if (wf_chspec_malformed(chspec))
+		return 0;
+
+	return chspec;
+}
+
+/**
+ * Verify the chanspec is using a legal set of parameters, i.e. that the
+ * chanspec specified a band, bw, pri_sb and channel and that the
+ * combination could be legal given any set of circumstances.
+ *
+ * @param  chanspec   the chanspec to check
+ *
+ * @return Returns TRUE if the chanspec is malformed, FALSE if it looks good.
+ */
+bool
+#ifdef BCMPOSTTRAPFN
+BCMPOSTTRAPFN(wf_chspec_malformed)(chanspec_t chanspec)
+#else
+wf_chspec_malformed(chanspec_t chanspec)
+#endif
+{
+	uint chspec_bw = CHSPEC_BW(chanspec);
+	uint chspec_sb;
+
+	if (CHSPEC_IS2G(chanspec)) {
+		/* must be valid bandwidth for 2G */
+		if (!BW_LE40(chspec_bw)) {
+			return TRUE;
+		}
+
+		/* check for invalid channel number */
+		if (CHSPEC_CHANNEL(chanspec) == INVCHANNEL) {
+			return TRUE;
+		}
+	} else if (CHSPEC_IS5G(chanspec) || CHSPEC_IS6G(chanspec)) {
+		if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_240)) {
+			uint ch_id;
+
+			ch_id = CHSPEC_GE240_CHAN(chanspec);
+
+			/* channel IDs in 240 must be in range */
+			if (CHSPEC_IS6G(chanspec)) {
+				if (ch_id >= WF_NUM_6G_240M_CHANS) {
+					/* bad 240MHz channel ID for the band */
+					return TRUE;
+				}
+			} else {
+				return TRUE;
+			}
+		} else if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_320)) {
+			uint ch_id;
+
+			ch_id = CHSPEC_GE240_CHAN(chanspec);
+
+			/* channel IDs in 320 must be in range */
+			if (CHSPEC_IS6G(chanspec)) {
+				if (ch_id >= WF_NUM_6G_320M_CHANS) {
+					/* bad 320MHz channel ID for the band */
+					return TRUE;
+				}
+			} else {
+				return TRUE;
+			}
+		} else if (WFC_NCBW_EQ(chspec_bw, WL_CHANSPEC_BW_160160)) {
+			uint ch0_id, ch1_id;
+
+			ch0_id = CHSPEC_CHAN0(chanspec);
+			ch1_id = CHSPEC_CHAN1(chanspec);
+
+			/* channel IDs in 160+160 must be in range */
+			if (CHSPEC_IS5G(chanspec) &&
+			    (ch0_id >= WF_NUM_5G_160M_CHANS || ch1_id >= WF_NUM_5G_160M_CHANS)) {
+				/* bad 160MHz channel ID for the band */
+				return TRUE;
+			}
+			if (CHSPEC_IS6G(chanspec) &&
+			    (ch0_id >= WF_NUM_6G_160M_CHANS || ch1_id >= WF_NUM_6G_160M_CHANS)) {
+				/* bad 160MHz channel ID for the band */
+				return TRUE;
+			}
+		} else if (WFC_NCBW_EQ(chspec_bw, WL_CHANSPEC_BW_8080)) {
+			uint ch0_id, ch1_id;
+
+			ch0_id = CHSPEC_CHAN0(chanspec);
+			ch1_id = CHSPEC_CHAN1(chanspec);
+
+			/* channel IDs in 80+80 must be in range */
+			if (CHSPEC_IS5G(chanspec) &&
+			    (ch0_id >= WF_NUM_5G_80M_CHANS || ch1_id >= WF_NUM_5G_80M_CHANS)) {
+				/* bad 80MHz channel ID for the band */
+				return TRUE;
+			}
+			if (CHSPEC_IS6G(chanspec) &&
+			    (ch0_id >= WF_NUM_6G_80M_CHANS || ch1_id >= WF_NUM_6G_80M_CHANS)) {
+				/* bad 80MHz channel ID for the band */
+				return TRUE;
+			}
+		} else if (chspec_bw == WL_CHANSPEC_BW_20 || chspec_bw == WL_CHANSPEC_BW_40 ||
+		           chspec_bw == WL_CHANSPEC_BW_80 || chspec_bw == WL_CHANSPEC_BW_160) {
+
+			/* check for invalid channel number */
+			if (CHSPEC_CHANNEL(chanspec) == INVCHANNEL) {
+				return TRUE;
+			}
+		} else {
+			/* invalid bandwidth */
+			return TRUE;
+		}
+	} else {
+		/* must be a valid band */
+		return TRUE;
+	}
+
+	/* retrive sideband */
+	if ((WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_240)) ||
+			(WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_320))) {
+		chspec_sb = CHSPEC_GE240_SB(chanspec);
+	} else {
+		chspec_sb = CHSPEC_CTL_SB(chanspec);
+	}
+
+	/* side band needs to be consistent with bandwidth */
+	if (chspec_bw == WL_CHANSPEC_BW_20) {
+		if (chspec_sb != WL_CHANSPEC_CTL_SB_LLL)
+			return TRUE;
+	} else if (chspec_bw == WL_CHANSPEC_BW_40) {
+		if (chspec_sb > WL_CHANSPEC_CTL_SB_LLU)
+			return TRUE;
+	} else if (chspec_bw == WL_CHANSPEC_BW_80 ||
+	           WFC_NCBW_EQ(chspec_bw, WL_CHANSPEC_BW_8080)) {
+		/* both 80MHz and 80+80MHz use 80MHz side bands.
+		 * 80+80 SB info is relative to the primary 80MHz sub-band.
+		 */
+		if (chspec_sb > WL_CHANSPEC_CTL_SB_LUU)
+			return TRUE;
+	} else if (chspec_bw == WL_CHANSPEC_BW_160 ||
+	           WFC_NCBW_EQ(chspec_bw, WL_CHANSPEC_BW_160160)) {
+		ASSERT(chspec_sb <= WL_CHANSPEC_CTL_SB_UUU);
+	} else if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_240)) {
+		/* FIXME: define the max sideband index */
+		ASSERT((chspec_sb >> WL_CHANSPEC_GE240_SB_SHIFT) <= 11);
+	} else if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_320)) {
+		/* FIXME: define the max sideband index */
+		ASSERT((chspec_sb >> WL_CHANSPEC_GE240_SB_SHIFT) <= 15);
+	}
+
+	return FALSE;
+}
+
+/**
+ * Verify the chanspec specifies a valid channel according to 802.11.
+ *
+ * @param   chanspec     the chanspec to check
+ *
+ * @return  Returns TRUE if the chanspec is a valid 802.11 channel
+ */
+bool
+wf_chspec_valid(chanspec_t chanspec)
+{
+	chanspec_band_t chspec_band = CHSPEC_BAND(chanspec);
+	chanspec_bw_t chspec_bw = CHSPEC_BW(chanspec);
+	uint chspec_ch = -1;
+
+	if (wf_chspec_malformed(chanspec)) {
+		return FALSE;
+	}
+
+	if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_240)) {
+		if (CHSPEC_IS6G(chanspec)) {
+			chspec_ch = wf_chspec_6G_id240_to_ch(CHSPEC_GE240_CHAN(chanspec));
+		} else {
+			return FALSE;
+		}
+	} else if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_320)) {
+		if (CHSPEC_IS6G(chanspec)) {
+			chspec_ch = wf_chspec_6G_id320_to_ch(CHSPEC_GE240_CHAN(chanspec));
+		} else {
+			return FALSE;
+		}
+	} else {
+		chspec_ch = CHSPEC_CHANNEL(chanspec);
+	}
+
+	/* After the malformed check, we know that we have
+	 * a valid band field,
+	 * a valid bandwidth for the band,
+	 * and a valid sub-band value for the bandwidth.
+	 *
+	 * Since all sub-band specs are valid for any channel, the only thing remaining to
+	 * check is that
+	 *   the 20MHz channel,
+	 *   or the center channel for higher BW,
+	 *   or both center channels for an 80+80MHz channel,
+	 * are valid for the specified band.
+	 * Also, 80+80MHz channels need to be non-contiguous.
+	 */
+
+	if (chspec_bw == WL_CHANSPEC_BW_20) {
+
+		return wf_valid_20MHz_chan(chspec_ch, chspec_band);
+
+	} else if (chspec_bw == WL_CHANSPEC_BW_40) {
+
+		return wf_valid_40MHz_center_chan(chspec_ch, chspec_band);
+
+	} else if (chspec_bw == WL_CHANSPEC_BW_80) {
+
+		return wf_valid_80MHz_center_chan(chspec_ch, chspec_band);
+
+	} else if (chspec_bw == WL_CHANSPEC_BW_160) {
+
+		return wf_valid_160MHz_center_chan(chspec_ch, chspec_band);
+
+	} else if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_240)) {
+
+		return wf_valid_240MHz_center_chan(chspec_ch, chspec_band);
+
+	} else if (WFC_BW_EQ(chspec_bw, WL_CHANSPEC_BW_320)) {
+
+		return wf_valid_320MHz_center_chan(chspec_ch, chspec_band);
+
+	} else if (WFC_NCBW_EQ(chspec_bw, WL_CHANSPEC_BW_8080)) {
+		uint16 ch0 = 0;
+		uint16 ch1 = 0;
+
+		/* get the center channels for each frequency segment */
+		if (CHSPEC_IS5G(chanspec)) {
+			ch0 = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN0(chanspec));
+			ch1 = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN1(chanspec));
+		} else if (CHSPEC_IS6G(chanspec)) {
+			ch0 = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN0(chanspec));
+			ch1 = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN1(chanspec));
+		} else {
+			return FALSE;
+		}
+
+		/* the two channels must be separated by more than 80MHz by VHT req */
+		if ((ch1 > ch0 + CH_80MHZ_APART) ||
+		    (ch0 > ch1 + CH_80MHZ_APART))
+			return TRUE;
+	} else if (WFC_NCBW_EQ(chspec_bw, WL_CHANSPEC_BW_160160)) {
+		uint16 ch0 = 0;
+		uint16 ch1 = 0;
+
+		/* get the center channels for each frequency segment */
+		if (CHSPEC_IS5G(chanspec)) {
+			ch0 = wf_chspec_5G_id160_to_ch(CHSPEC_CHAN0(chanspec));
+			ch1 = wf_chspec_5G_id160_to_ch(CHSPEC_CHAN1(chanspec));
+		} else if (CHSPEC_IS6G(chanspec)) {
+			ch0 = wf_chspec_6G_id160_to_ch(CHSPEC_CHAN0(chanspec));
+			ch1 = wf_chspec_6G_id160_to_ch(CHSPEC_CHAN1(chanspec));
+		} else {
+			return FALSE;
+		}
+
+		/* the two channels must be separated by more than 160MHz by EHT req */
+		if ((ch1 > ch0 + CH_160MHZ_APART) ||
+		    (ch0 > ch1 + CH_160MHZ_APART)) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/* 5G band 20MHz channel ranges with even (+4) channel spacing */
+static const struct wf_iter_range wf_5g_iter_ranges[] = {
+	{36, 64},
+	{100, 144},
+	{149, 165}
+};
+
+#define RANGE_ID_INVAL 0xFFu
+enum wf_iter_state {
+	WF_ITER_INIT = 0,
+	WF_ITER_RUN  = 1,
+	WF_ITER_DONE = 2
+};
+
+/**
+ * @brief  Initialize a chanspec iteration structure.
+ */
+bool
+wf_chanspec_iter_init(wf_chanspec_iter_t *iter, chanspec_band_t band, chanspec_bw_t bw)
+{
+	if (iter == NULL) {
+		return FALSE;
+	}
+
+	/* Initialize the iter structure to the "DONE" state
+	 * in case the parameter validation fails.
+	 * If the validation fails then the iterator will return INVCHANSPEC as the current
+	 * chanspec, and wf_chanspec_iter_next() will return FALSE.
+	 */
+	memset(iter, 0, sizeof(*iter));
+	iter->state = WF_ITER_DONE;
+	iter->chanspec = INVCHANSPEC;
+
+	if (band != WL_CHANSPEC_BAND_2G &&
+	    band != WL_CHANSPEC_BAND_5G &&
+	    band != WL_CHANSPEC_BAND_6G) {
+		ASSERT(0);
+		return FALSE;
+	}
+
+	/* make sure the BW is unspecified (INVCHANSPEC), 20/40,
+	 * or (not 2g and 80/160)
+	 */
+	if (!(bw == INVCHANSPEC ||
+	      bw == WL_CHANSPEC_BW_20 ||
+	      bw == WL_CHANSPEC_BW_40 ||
+	      (band != WL_CHANSPEC_BAND_2G &&
+	       (bw == WL_CHANSPEC_BW_80 ||
+	        bw == WL_CHANSPEC_BW_160 ||
+		WFC_BW_EQ(bw, WL_CHANSPEC_BW_240) ||
+	        WFC_BW_EQ(bw, WL_CHANSPEC_BW_320))))) {
+
+		ASSERT(0);
+		return FALSE;
+	}
+
+	/* Validation of the params is successful so move to the "INIT" state to
+	 * allow the first wf_chanspec_iter_next() move the iteration to the first
+	 * chanspec in the set.
+	 */
+	iter->state = WF_ITER_INIT;
+	iter->band = band;
+	iter->bw = bw;
+	iter->range_id = RANGE_ID_INVAL;
+
+	return TRUE;
+}
+
+/**
+ * Start the iterator off from the 'init' state.
+ * The internal state is set up and advanced to the first chanspec.
+ */
+static void
+wf_chanspec_iter_firstchan(wf_chanspec_iter_t *iter)
+{
+	chanspec_band_t band = iter->band;
+	chanspec_bw_t bw = iter->bw;
+	chanspec_t chspec;
+
+	/* if BW unspecified (INVCHANSPEC), start with 20 MHz */
+	if (bw == INVCHANSPEC) {
+		bw = WL_CHANSPEC_BW_20;
+	}
+
+	/* calc the initial channel based on band */
+	if (band == WL_CHANSPEC_BAND_2G) {
+		/* 2g has overlapping 40MHz channels, so cannot just use the
+		 * wf_create_chspec_from_primary() fn.
+		 */
+		if (bw == WL_CHANSPEC_BW_20) {
+			chspec = wf_create_20MHz_chspec(CH_MIN_2G_CHANNEL, band);
+		} else {
+			chspec = (WL_CHANSPEC_BAND_2G | bw | WL_CHANSPEC_CTL_SB_L |
+			          CH_MIN_2G_40M_CHANNEL);
+		}
+	} else {
+		if (band == WL_CHANSPEC_BAND_5G) {
+			wf_chanspec_iter_next_5g_range(iter, bw);
+		} else {
+			wf_chanspec_iter_6g_range_init(iter, bw);
+		}
+		chspec = wf_create_chspec_from_primary(iter->range.start, bw, band);
+	}
+
+	iter->chanspec = chspec;
+}
+
+/**
+ * @brief Return the current chanspec of the iteration.
+ */
+chanspec_t
+wf_chanspec_iter_current(wf_chanspec_iter_t *iter)
+{
+	return iter->chanspec;
+}
+
+/**
+ * @brief Advance the iteration to the next chanspec in the set.
+ */
+bool
+wf_chanspec_iter_next(wf_chanspec_iter_t *iter, chanspec_t *chspec)
+{
+	bool ok = FALSE;
+	chanspec_band_t band = iter->band;
+
+	/* Handle the INIT and DONE states. Otherwise, we are in the RUN state
+	 * and will dispatch to the 'next' function for the appropriate band.
+	 */
+	if (iter->state == WF_ITER_INIT) {
+		iter->state = WF_ITER_RUN;
+		wf_chanspec_iter_firstchan(iter);
+		ok = TRUE;
+	} else if (iter->state == WF_ITER_DONE) {
+		ok = FALSE;
+	} else if (band == WL_CHANSPEC_BAND_2G) {
+		ok = wf_chanspec_iter_next_2g(iter);
+	} else if (band == WL_CHANSPEC_BAND_5G) {
+		ok = wf_chanspec_iter_next_5g(iter);
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		ok = wf_chanspec_iter_next_6g(iter);
+	}
+
+	/* Return the new chanspec if a pointer was provided.
+	 * In case the iteration is done, the return will be INVCHANSPEC.
+	 */
+	if (chspec != NULL) {
+		*chspec = iter->chanspec;
+	}
+
+	return ok;
+}
+
+/**
+ * When the iterator completes a particular bandwidth, this function
+ * returns the next BW, or INVCHANSPEC when done.
+ *
+ * Internal iterator helper.
+ */
+static chanspec_bw_t
+wf_iter_next_bw(chanspec_bw_t bw)
+{
+	switch (bw) {
+	case WL_CHANSPEC_BW_20:
+		bw = WL_CHANSPEC_BW_40;
+		break;
+	case WL_CHANSPEC_BW_40:
+		bw = WL_CHANSPEC_BW_80;
+		break;
+	case WL_CHANSPEC_BW_80:
+		bw = WL_CHANSPEC_BW_160;
+		break;
+#ifdef WL11BE
+	case WL_CHANSPEC_BW_160:
+		bw = WL_CHANSPEC_BW_240;
+		break;
+	case WL_CHANSPEC_BW_240:
+		bw = WL_CHANSPEC_BW_320;
+		break;
+#endif
+	default:
+		bw = INVCHANSPEC;
+		break;
+	}
+	return bw;
+}
+
+/**
+ * This is the _iter_next() helper for 2g band chanspec iteration.
+ */
+static bool
+wf_chanspec_iter_next_2g(wf_chanspec_iter_t *iter)
+{
+	chanspec_t chspec = iter->chanspec;
+	uint8 ch = CHSPEC_CHANNEL(chspec);
+
+	if (CHSPEC_IS20(chspec)) {
+		if (ch < CH_MAX_2G_CHANNEL) {
+			ch++;
+			chspec = wf_create_20MHz_chspec(ch, WL_CHANSPEC_BAND_2G);
+		} else if (iter->bw == INVCHANSPEC) {
+			/* hit the end of 20M channels, go to 40M if bw was unspecified */
+			ch = CH_MIN_2G_40M_CHANNEL;
+			chspec = wf_create_40MHz_chspec(LOWER_20_SB(ch), ch, WL_CHANSPEC_BAND_2G);
+		} else {
+			/* done */
+			iter->state = WF_ITER_DONE;
+			chspec = INVCHANSPEC;
+		}
+	} else {
+		/* step through low then high primary sideband, then next 40 center channel */
+		if (CHSPEC_SB_LOWER(iter->chanspec)) {
+			/* move from lower primary 20 to upper */
+			chspec = wf_create_40MHz_chspec(UPPER_20_SB(ch),
+			                                ch, WL_CHANSPEC_BAND_2G);
+		} else if (ch < CH_MAX_2G_40M_CHANNEL) {
+			/* move to next 40M center and lower primary 20 */
+			ch++;
+			chspec = wf_create_40MHz_chspec(LOWER_20_SB(ch),
+			                                ch, WL_CHANSPEC_BAND_2G);
+		} else {
+			/* done */
+			iter->state = WF_ITER_DONE;
+			chspec = INVCHANSPEC;
+		}
+	}
+
+	iter->chanspec = chspec;
+
+	return (chspec != INVCHANSPEC);
+}
+
+/**
+ * This is the _iter_next() helper for 5g band chanspec iteration.
+ * The 5g iterator uses ranges of primary 20MHz channels, and the current BW, to create
+ * each chanspec in the set.
+ * When a 5g range is exhausted, wf_chanspec_iter_next_5g_range() is called to get the next
+ * range appropriate to the current BW.
+ */
+static bool
+wf_chanspec_iter_next_5g(wf_chanspec_iter_t *iter)
+{
+	chanspec_t chspec = iter->chanspec;
+	chanspec_bw_t bw = CHSPEC_BW(chspec);
+	uint8 ch = wf_chspec_primary20_chan(chspec);
+	uint8 end = iter->range.end;
+
+	if (ch < end) {
+		/* not at the end of the current range, so
+		 * step to the next 20MHz channel and create the current BW
+		 * channel with that new primary 20MHz.
+		 */
+		ch += CH_20MHZ_APART;
+	} else if (wf_chanspec_iter_next_5g_range(iter, bw)) {
+		/* there was a new range in the current BW, so start at the beginning */
+		ch = iter->range.start;
+	} else if (iter->bw == INVCHANSPEC) {
+		/* hit the end of current bw, so move to the next bw */
+		bw = wf_iter_next_bw(bw);
+		if (bw != INVCHANSPEC) {
+			/* initialize the first range */
+			iter->range_id = RANGE_ID_INVAL;
+			wf_chanspec_iter_next_5g_range(iter, bw);
+			ch = iter->range.start;
+		} else {
+			/* no more BWs */
+			chspec = INVCHANSPEC;
+		}
+	} else {
+		/* no more channels, ranges, or BWs */
+		chspec = INVCHANSPEC;
+	}
+
+	/* if we are not at the end of the iteration, calc the next chanspec from components */
+	if (chspec != INVCHANSPEC) {
+		chspec = wf_create_chspec_from_primary(ch, bw, WL_CHANSPEC_BAND_5G);
+	}
+
+	iter->chanspec = chspec;
+	if (chspec != INVCHANSPEC) {
+		return TRUE;
+	} else {
+		iter->state = WF_ITER_DONE;
+		return FALSE;
+	}
+}
+
+/**
+ * Helper function to set up the next range of primary 20MHz channels to
+ * iterate over for the current BW. This will advance
+ *    iter->range_id
+ * and set up
+ *    iter->range.start
+ *    iter->range.end
+ * for the new range.
+ * Returns FALSE if there are no more ranges in the current BW.
+ */
+static int
+wf_chanspec_iter_next_5g_range(wf_chanspec_iter_t *iter, chanspec_bw_t bw)
+{
+	uint8 range_id = iter->range_id;
+	const uint8 *channels;
+	uint count;
+
+	if (bw == WL_CHANSPEC_BW_20) {
+		if (range_id == RANGE_ID_INVAL) {
+			range_id = 0;
+		} else {
+			range_id++;
+		}
+
+		if (range_id < ARRAYSIZE(wf_5g_iter_ranges)) {
+			iter->range_id = range_id;
+			iter->range = wf_5g_iter_ranges[range_id];
+			return TRUE;
+		}
+
+		return FALSE;
+	}
+
+	if (bw == WL_CHANSPEC_BW_40) {
+		channels = wf_5g_40m_chans;
+		count = WF_NUM_5G_40M_CHANS;
+	} else if (bw == WL_CHANSPEC_BW_80) {
+		channels = wf_5g_80m_chans;
+		count = WF_NUM_5G_80M_CHANS;
+	} else if (bw == WL_CHANSPEC_BW_160) {
+		channels = wf_5g_160m_chans;
+		count = WF_NUM_5G_160M_CHANS;
+	} else {
+		return FALSE;
+	}
+
+	if (range_id == RANGE_ID_INVAL) {
+		range_id = 0;
+	} else {
+		range_id++;
+	}
+	if (range_id < count) {
+		uint8 ch = channels[range_id];
+		uint offset = center_chan_to_edge(bw);
+
+		iter->range_id = range_id;
+		iter->range.start = ch - offset;
+		iter->range.end = ch + offset;
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+/**
+ * This is the _iter_next() helper for 6g band chanspec iteration.
+ * The 6g iterator uses ranges of primary 20MHz channels, and the current BW, to create
+ * each chanspec in the set.
+ * Each BW in 6g has one contiguous range of primary 20MHz channels. When a range is
+ * exhausted, the iterator moves to the next BW.
+ */
+static bool
+wf_chanspec_iter_next_6g(wf_chanspec_iter_t *iter)
+{
+	chanspec_t chspec = iter->chanspec;
+	chanspec_bw_t bw = CHSPEC_BW(chspec);
+	uint8 ch = wf_chspec_primary20_chan(chspec);
+	uint8 end = iter->range.end;
+
+	if (ch < end) {
+		/* not at the end of the current range, so
+		 * step to the next 20MHz channel and create the current BW
+		 * channel with that new primary 20MHz.
+		 */
+		ch += CH_20MHZ_APART;
+
+		/* try to create a valid channel of the current BW
+		 * with a primary20 'ch'
+		 */
+		chspec = wf_create_chspec_from_primary(ch, bw, WL_CHANSPEC_BAND_6G);
+
+		/* if chspec is INVCHANSPEC, then we hit the end
+		 * of the valid channels in the range.
+		 */
+	} else {
+		/* hit the end of the current range */
+		chspec = INVCHANSPEC;
+	}
+
+	/* if we are at the end of the current channel range
+	 * check if there is another BW to iterate
+	 * Note: (iter->bw == INVCHANSPEC) indicates an unspecified BW for the interation,
+	 * so it will iterate over all BWs.
+	 */
+	if (chspec == INVCHANSPEC &&
+	    iter->bw == INVCHANSPEC &&
+	    (bw = wf_iter_next_bw(bw)) != INVCHANSPEC) {
+		/* start the new bw with the first primary20 */
+		ch = iter->range.start;
+		chspec = wf_create_chspec_from_primary(ch, bw, WL_CHANSPEC_BAND_6G);
+	}
+
+	iter->chanspec = chspec;
+	if (chspec != INVCHANSPEC) {
+		return TRUE;
+	} else {
+		iter->state = WF_ITER_DONE;
+		return FALSE;
+	}
+}
+
+/**
+ * Helper used by wf_chanspec_iter_firstchan() to set up the first range of
+ * primary channels for the 6g band and for the BW being iterated.
+ */
+static void
+wf_chanspec_iter_6g_range_init(wf_chanspec_iter_t *iter, chanspec_bw_t bw)
+{
+	switch (bw) {
+	case WL_CHANSPEC_BW_20:
+	case WL_CHANSPEC_BW_40:
+	case WL_CHANSPEC_BW_80:
+	case WL_CHANSPEC_BW_160:
+#ifdef WL11BE
+	case WL_CHANSPEC_BW_240:
+	case WL_CHANSPEC_BW_320:
+#endif
+		iter->range.start = CH_MIN_6G_CHANNEL;
+		iter->range.end   = CH_MAX_6G_CHANNEL;
+		break;
+	default:
+		ASSERT(0);
+		break;
+	}
+}
+
+/**
+ * Verify that the channel is a valid 20MHz channel according to 802.11.
+ *
+ * @param  channel   20MHz channel number to validate
+ * @param  band      chanspec band
+ *
+ * @return Return TRUE if valid
+ */
+bool
+wf_valid_20MHz_chan(uint channel, chanspec_band_t band)
+{
+	if (band == WL_CHANSPEC_BAND_2G) {
+		/* simple range check for 2GHz */
+		return (channel >= CH_MIN_2G_CHANNEL &&
+		        channel <= CH_MAX_2G_CHANNEL);
+	} else if (band == WL_CHANSPEC_BAND_5G) {
+		const uint8 *center_ch = wf_5g_40m_chans;
+		uint num_ch = WF_NUM_5G_40M_CHANS;
+		uint i;
+
+		/* We don't have an array of legal 20MHz 5G channels, but they are
+		 * each side of the legal 40MHz channels.  Check the chanspec
+		 * channel against either side of the 40MHz channels.
+		 */
+		for (i = 0; i < num_ch; i ++) {
+			if (channel == (uint)LOWER_20_SB(center_ch[i]) ||
+			    channel == (uint)UPPER_20_SB(center_ch[i])) {
+				break; /* match found */
+			}
+		}
+
+		if (i == num_ch) {
+			/* check for channel 165 which is not the side band
+			 * of 40MHz 5G channel
+			 */
+			if (channel == 165) {
+				i = 0;
+			}
+
+			/* check for legacy JP channels on failure */
+			if (channel == 34 || channel == 38 ||
+			    channel == 42 || channel == 46) {
+				i = 0;
+			}
+		}
+
+		if (i < num_ch) {
+			/* match found */
+			return TRUE;
+		}
+	}
+
+	else if (band == WL_CHANSPEC_BAND_6G) {
+		/* Use the simple pattern of 6GHz 20MHz channels for validity check */
+		if ((channel >= CH_MIN_6G_CHANNEL &&
+		     channel <= CH_MAX_6G_CHANNEL) &&
+		    ((((channel - CH_MIN_6G_CHANNEL) % 4) == 0) || // even multiple of 4
+		     channel == 2)) { // Or the oddball channel 2
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/**
+ * Verify that the center channel is a valid 40MHz center channel according to 802.11.
+ *
+ * @param  center_channel   40MHz center channel to validate
+ * @param  band             chanspec band
+ *
+ * @return Return TRUE if valid
+ */
+bool
+wf_valid_40MHz_center_chan(uint center_channel, chanspec_band_t band)
+{
+	if (band == WL_CHANSPEC_BAND_2G) {
+		/* simple range check for 2GHz */
+		return (center_channel >= CH_MIN_2G_40M_CHANNEL &&
+		        center_channel <= CH_MAX_2G_40M_CHANNEL);
+	} else if (band == WL_CHANSPEC_BAND_5G) {
+		uint i;
+
+		/* use the 5GHz lookup of 40MHz channels */
+		for (i = 0; i < WF_NUM_5G_40M_CHANS; i++) {
+			if (center_channel == wf_5g_40m_chans[i]) {
+				return TRUE;
+			}
+		}
+	}
+	else if (band == WL_CHANSPEC_BAND_6G) {
+		/* Use the simple pattern of 6GHz center channels */
+		if ((center_channel >= CH_MIN_6G_40M_CHANNEL &&
+		     center_channel <= CH_MAX_6G_40M_CHANNEL) &&
+		    ((center_channel - CH_MIN_6G_40M_CHANNEL) % 8) == 0) {  // even multiple of 8
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/**
+ * Verify that the center channel is a valid 80MHz center channel according to 802.11.
+ *
+ * @param  center_channel   80MHz center channel to validate
+ * @param  band             chanspec band
+ *
+ * @return Return TRUE if valid
+ */
+bool
+wf_valid_80MHz_center_chan(uint center_channel, chanspec_band_t band)
+{
+	if (band == WL_CHANSPEC_BAND_5G) {
+		/* use the 80MHz ID lookup to validate the center channel */
+		if (channel_80mhz_to_id(center_channel) >= 0) {
+			return TRUE;
+		}
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		/* use the 80MHz ID lookup to validate the center channel */
+		if (channel_6g_80mhz_to_id(center_channel) >= 0) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/**
+ * Verify that the center channel is a valid 160MHz center channel according to 802.11.
+ *
+ * @param  center_channel   160MHz center channel to validate
+ * @param  band             chanspec band
+ *
+ * @return Return TRUE if valid
+ */
+bool
+wf_valid_160MHz_center_chan(uint center_channel, chanspec_band_t band)
+{
+	if (band == WL_CHANSPEC_BAND_5G) {
+		uint i;
+
+		/* use the 5GHz lookup of 40MHz channels */
+		for (i = 0; i < WF_NUM_5G_160M_CHANS; i++) {
+			if (center_channel == wf_5g_160m_chans[i]) {
+				return TRUE;
+			}
+		}
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		/* Use the simple pattern of 6GHz center channels */
+		if ((center_channel >= CH_MIN_6G_160M_CHANNEL &&
+		     center_channel <= CH_MAX_6G_160M_CHANNEL) &&
+		    ((center_channel - CH_MIN_6G_160M_CHANNEL) % 32) == 0) { // even multiple of 32
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/**
+ * Verify that the center channel is a valid 240MHz center channel according to 802.11.
+ *
+ * @param  center_channel   240MHz center channel to validate
+ * @param  band             chanspec band
+ *
+ * @return Return TRUE if valid
+ */
+bool
+wf_valid_240MHz_center_chan(uint center_channel, chanspec_band_t band)
+{
+	if (band == WL_CHANSPEC_BAND_6G) {
+		/* Use the simple pattern of 6GHz center channels */
+		if ((center_channel >= CH_MIN_6G_240M_CHANNEL &&
+		     center_channel <= CH_MAX_6G_240M_CHANNEL) &&
+		    ((center_channel - CH_MIN_6G_240M_CHANNEL) % 48) == 0) { // even multiple of 48
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/**
+ * Verify that the center channel is a valid 320MHz center channel according to 802.11.
+ *
+ * @param  center_channel   320MHz center channel to validate
+ * @param  band             chanspec band
+ *
+ * @return Return TRUE if valid
+ */
+bool
+wf_valid_320MHz_center_chan(uint center_channel, chanspec_band_t band)
+{
+	if (band == WL_CHANSPEC_BAND_6G) {
+		/* Use the simple pattern of 6GHz center channels */
+		if ((center_channel >= CH_MIN_6G_320M_CHANNEL &&
+		     center_channel <= CH_MAX_6G_320M_CHANNEL) &&
+		    ((center_channel - CH_MIN_6G_320M_CHANNEL) % 64) == 0) { // even multiple of 64
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+/*
+ * This function returns TRUE if both the chanspec can co-exist in PHY.
+ * Addition to primary20 channel, the function checks for side band for 2g 40 channels
+ */
+bool
+wf_chspec_coexist(chanspec_t chspec1, chanspec_t chspec2)
+{
+	bool same_primary;
+
+	same_primary = (wf_chspec_primary20_chan(chspec1) == wf_chspec_primary20_chan(chspec2));
+
+	if (same_primary && CHSPEC_IS2G(chspec1)) {
+	    if (CHSPEC_IS40(chspec1) && CHSPEC_IS40(chspec2)) {
+	        return (CHSPEC_CTL_SB(chspec1) == CHSPEC_CTL_SB(chspec2));
+	    }
+	}
+	return same_primary;
+}
+
+/**
+ * Create a 20MHz chanspec for the given band.
+ *
+ * This function returns a 20MHz chanspec in the given band.
+ *
+ * @param	channel   20MHz channel number
+ * @param	band      a chanspec band (e.g. WL_CHANSPEC_BAND_2G)
+ *
+ * @return Returns a 20MHz chanspec, or IVNCHANSPEC in case of error.
+ */
+chanspec_t
+wf_create_20MHz_chspec(uint channel, chanspec_band_t band)
+{
+	chanspec_t chspec;
+
+	if (channel <= WL_CHANSPEC_CHAN_MASK &&
+	    (band == WL_CHANSPEC_BAND_2G ||
+	     band == WL_CHANSPEC_BAND_5G ||
+	     band == WL_CHANSPEC_BAND_6G)) {
+		chspec = band | WL_CHANSPEC_BW_20 | WL_CHANSPEC_CTL_SB_NONE | channel;
+		if (!wf_chspec_valid(chspec)) {
+			chspec = INVCHANSPEC;
+		}
+	} else {
+		chspec = INVCHANSPEC;
+	}
+
+	return chspec;
+}
+
+/**
+ * Returns the chanspec for a 40MHz channel given the primary 20MHz channel number,
+ * the center channel number, and the band.
+ *
+ * @param  primary_channel  primary 20Mhz channel
+ * @param  center_channel   center channel of the 40MHz channel
+ * @param  band             band of the 40MHz channel (chanspec_band_t value)
+ *
+ * The center_channel can be one of the 802.11 spec valid 40MHz chenter channels
+ * in the given band.
+ *
+ * @return returns a 40MHz chanspec, or INVCHANSPEC in case of error
+ */
+chanspec_t
+wf_create_40MHz_chspec(uint primary_channel, uint center_channel,
+                       chanspec_band_t band)
+{
+	int sb;
+
+	/* Calculate the sideband value for the center and primary channel.
+	 * Will return -1 if not a valid pair for 40MHz
+	 */
+	sb = channel_to_sb(center_channel, primary_channel, WL_CHANSPEC_BW_40);
+
+	/* return err if the sideband was bad or the center channel is not
+	 * valid for the given band.
+	 */
+	if (sb < 0 || !wf_valid_40MHz_center_chan(center_channel, band)) {
+		return INVCHANSPEC;
+	}
+
+	/* othewise construct and return the valid 40MHz chanspec */
+	return (chanspec_t)(center_channel | WL_CHANSPEC_BW_40 | band |
+	                    ((uint)sb << WL_CHANSPEC_CTL_SB_SHIFT));
+}
+
+/**
+ * Returns the chanspec for a 40MHz channel given the primary 20MHz channel number,
+ * the sub-band for the primary 20MHz channel, and the band.
+ *
+ * @param  primary_channel  primary 20Mhz channel
+ * @param  primary_subband  sub-band of the 20MHz primary channel (chanspec_subband_t value)
+ * @param  band             band of the 40MHz channel (chanspec_band_t value)
+ *
+ * The primary channel and sub-band should describe one of the 802.11 spec valid
+ * 40MHz channels in the given band.
+ *
+ * @return returns a 40MHz chanspec, or INVCHANSPEC in case of error
+ */
+chanspec_t
+wf_create_40MHz_chspec_primary_sb(uint primary_channel, chanspec_subband_t primary_subband,
+                                  chanspec_band_t band)
+{
+	uint center_channel;
+
+	/* find the center channel */
+	if (primary_subband == WL_CHANSPEC_CTL_SB_L) {
+		center_channel = primary_channel + CH_10MHZ_APART;
+	} else if (primary_subband == WL_CHANSPEC_CTL_SB_U) {
+		center_channel = primary_channel - CH_10MHZ_APART;
+	} else {
+		return INVCHANSPEC;
+	}
+
+	return wf_create_40MHz_chspec(primary_channel, center_channel, band);
+}
+
+/**
+ * Returns the chanspec for an 80MHz channel given the primary 20MHz channel number,
+ * the center channel number, and the band.
+ *
+ * @param  primary_channel  primary 20Mhz channel
+ * @param  center_channel   center channel of the 80MHz channel
+ * @param  band             band of the 80MHz channel (chanspec_band_t value)
+ *
+ * The center_channel can be one of {42, 58, 106, 122, 138, 155} for 5G,
+ * or {7 + 16*X for 0 <= X <= 13} for 6G.
+ *
+ * @return returns an 80MHz chanspec, or INVCHANSPEC in case of error
+ */
+chanspec_t
+wf_create_80MHz_chspec(uint primary_channel, uint center_channel,
+                       chanspec_band_t band)
+{
+	int sb;
+
+	/* Calculate the sideband value for the center and primary channel.
+	 * Will return -1 if not a valid pair for 80MHz
+	 */
+	sb = channel_to_sb(center_channel, primary_channel, WL_CHANSPEC_BW_80);
+
+	/* return err if the sideband was bad or the center channel is not
+	 * valid for the given band.
+	 */
+	if (sb < 0 || !wf_valid_80MHz_center_chan(center_channel, band)) {
+		return INVCHANSPEC;
+	}
+
+	/* othewise construct and return the valid 80MHz chanspec */
+	return (chanspec_t)(center_channel | WL_CHANSPEC_BW_80 | band |
+	                    ((uint)sb << WL_CHANSPEC_CTL_SB_SHIFT));
+}
+
+/**
+ * Returns the chanspec for an 160MHz channel given the primary 20MHz channel number,
+ * the center channel number, and the band.
+ *
+ * @param  primary_channel  primary 20Mhz channel
+ * @param  center_channel   center channel of the 160MHz channel
+ * @param  band             band of the 160MHz channel (chanspec_band_t value)
+ *
+ * The center_channel can be one of {50, 114} for 5G,
+ * or {15 + 32*X for 0 <= X <= 7} for 6G.
+ *
+ * @return returns an 160MHz chanspec, or INVCHANSPEC in case of error
+ */
+chanspec_t
+wf_create_160MHz_chspec(uint primary_channel, uint center_channel, chanspec_band_t band)
+{
+	int sb;
+
+	/* Calculate the sideband value for the center and primary channel.
+	 * Will return -1 if not a valid pair for 160MHz
+	 */
+	sb = channel_to_sb(center_channel, primary_channel, WL_CHANSPEC_BW_160);
+
+	/* return err if the sideband was bad or the center channel is not
+	 * valid for the given band.
+	 */
+	if (sb < 0 || !wf_valid_160MHz_center_chan(center_channel, band)) {
+		return INVCHANSPEC;
+	}
+
+	/* othewise construct and return the valid 160MHz chanspec */
+	return (chanspec_t)(center_channel | WL_CHANSPEC_BW_160 | band |
+	                    ((uint)sb << WL_CHANSPEC_CTL_SB_SHIFT));
+}
+
+/**
+ * Returns the chanspec for an 80+80MHz channel given the primary 20MHz channel number,
+ * the center channel numbers for each frequency segment, and the band.
+ *
+ * @param  primary_channel  primary 20 Mhz channel
+ * @param  chan0            center channel number of one frequency segment
+ * @param  chan1            center channel number of the other frequency segment
+ * @param  band             band of the 80+80 MHz channel (chanspec_band_t value)
+ *
+ * Parameters chan0 and chan1 are valid 80 MHz center channel numbers for the given band.
+ * The primary channel must be contained in one of the 80 MHz channels. This routine
+ * will determine which frequency segment is the primary 80 MHz segment.
+ *
+ * @return returns an 80+80 MHz chanspec, or INVCHANSPEC in case of error
+ *
+ * Refer to 802.11-2016 section 21.3.14 "Channelization".
+ */
+chanspec_t
+wf_create_8080MHz_chspec(uint primary_channel, uint chan0, uint chan1,
+                       chanspec_band_t band)
+{
+	int sb = 0;
+	chanspec_t chanspec = 0;
+	int chan0_id = -1, chan1_id = -1;
+	int seg0, seg1;
+
+	/* frequency segments need to be non-contiguous, so the channel separation needs
+	 * to be greater than 80MHz
+	 */
+	if ((uint)ABS((int)(chan0 - chan1)) <= CH_80MHZ_APART) {
+		return INVCHANSPEC;
+	}
+
+	if (band == WL_CHANSPEC_BAND_5G) {
+		chan0_id = channel_80mhz_to_id(chan0);
+		chan1_id = channel_80mhz_to_id(chan1);
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		chan0_id = channel_6g_80mhz_to_id(chan0);
+		chan1_id = channel_6g_80mhz_to_id(chan1);
+	}
+
+	/* make sure the channel numbers were valid */
+	if (chan0_id == -1 || chan1_id == -1) {
+		return INVCHANSPEC;
+	}
+
+	/* does the primary channel fit with the 1st 80MHz channel ? */
+	sb = channel_to_sb(chan0, primary_channel, WL_CHANSPEC_BW_80);
+	if (sb >= 0) {
+		/* yes, so chan0 is frequency segment 0, and chan1 is seg 1 */
+		seg0 = chan0_id;
+		seg1 = chan1_id;
+	} else {
+		/* no, so does the primary channel fit with the 2nd 80MHz channel ? */
+		sb = channel_to_sb(chan1, primary_channel, WL_CHANSPEC_BW_80);
+		if (sb < 0) {
+			/* no match for pri_ch to either 80MHz center channel */
+			return INVCHANSPEC;
+		}
+		/* swapped, so chan1 is frequency segment 0, and chan0 is seg 1 */
+		seg0 = chan1_id;
+		seg1 = chan0_id;
+	}
+
+	chanspec = ((seg0 << WL_CHANSPEC_CHAN0_SHIFT) |
+	            (seg1 << WL_CHANSPEC_CHAN1_SHIFT) |
+	            (sb << WL_CHANSPEC_CTL_SB_SHIFT) |
+	            WL_CHANSPEC_BW_8080 |
+	            band);
+
+	return chanspec;
+}
+
+/**
+ * Returns the chanspec for an 160+160MHz channel given the primary 20MHz channel number,
+ * the center channel numbers for each frequency segment, and the band.
+ *
+ * @param  primary_channel  primary 20 Mhz channel
+ * @param  chan0            center channel number of one frequency segment
+ * @param  chan1            center channel number of the other frequency segment
+ * @param  band             band of the 160+160 MHz channel (chanspec_band_t value)
+ *
+ * Parameters chan0 and chan1 are valid 160 MHz center channel numbers for the given band.
+ * The primary channel must be contained in one of the 160 MHz channels. This routine
+ * will determine which frequency segment is the primary 160 MHz segment.
+ *
+ * @return returns an 160+160 MHz chanspec, or INVCHANSPEC in case of error
+ *
+ * Refer to <TBD> "Channelization".
+ */
+chanspec_t
+wf_create_160160MHz_chspec(uint primary_channel, uint chan0, uint chan1,
+                           chanspec_band_t band)
+{
+	int sb = 0;
+	chanspec_t chanspec = 0;
+	int chan0_id = -1, chan1_id = -1;
+	int seg0, seg1;
+
+	/* frequency segments need to be non-contiguous, so the channel separation needs
+	 * to be greater than 160MHz
+	 */
+	if ((uint)ABS((int)(chan0 - chan1)) <= CH_160MHZ_APART) {
+		return INVCHANSPEC;
+	}
+
+	if (band == WL_CHANSPEC_BAND_5G) {
+		chan0_id = channel_5g_160mhz_to_id(chan0);
+		chan1_id = channel_5g_160mhz_to_id(chan1);
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		chan0_id = channel_6g_160mhz_to_id(chan0);
+		chan1_id = channel_6g_160mhz_to_id(chan1);
+	}
+
+	/* make sure the channel numbers were valid */
+	if (chan0_id == -1 || chan1_id == -1) {
+		return INVCHANSPEC;
+	}
+
+	/* does the primary channel fit with the 1st 160MHz channel ? */
+	sb = channel_to_sb(chan0, primary_channel, WL_CHANSPEC_BW_160);
+	if (sb >= 0) {
+		/* yes, so chan0 is frequency segment 0, and chan1 is seg 1 */
+		seg0 = chan0_id;
+		seg1 = chan1_id;
+	} else {
+		/* no, so does the primary channel fit with the 2nd 160MHz channel ? */
+		sb = channel_to_sb(chan1, primary_channel, WL_CHANSPEC_BW_160);
+		if (sb < 0) {
+			/* no match for pri_ch to either 160MHz center channel */
+			return INVCHANSPEC;
+		}
+		/* swapped, so chan1 is frequency segment 0, and chan0 is seg 1 */
+		seg0 = chan1_id;
+		seg1 = chan0_id;
+	}
+
+	chanspec = ((seg0 << WL_CHANSPEC_CHAN0_SHIFT) |
+	            (seg1 << WL_CHANSPEC_CHAN1_SHIFT) |
+	            (sb << WL_CHANSPEC_CTL_SB_SHIFT) |
+	            WL_CHANSPEC_BW_160160 |
+	            band);
+
+	return chanspec;
+}
+
+/**
+ * Returns the chanspec for an 240MHz channel given the primary 20MHz channel number,
+ * the center channel number, and the band.
+ *
+ * @param  primary_channel  primary 20 Mhz channel
+ * @param  chan             center channel number
+ * @param  band             band of the 240 MHz channel (chanspec_band_t value)
+ *
+ * @return returns an 240 MHz chanspec, or INVCHANSPEC in case of error
+ *
+ * Refer to <TBD> "Channelization".
+ */
+chanspec_t
+wf_create_240MHz_chspec(uint primary_channel, uint center_channel, chanspec_band_t band)
+{
+	int sb = 0;
+	chanspec_t chanspec = 0;
+	int chan_id = -1;
+
+	if (band == WL_CHANSPEC_BAND_6G) {
+		chan_id = channel_6g_240mhz_to_id(center_channel);
+	}
+
+	/* make sure the channel number were valid */
+	if (chan_id == -1) {
+		return INVCHANSPEC;
+	}
+
+	/* Calculate the sideband value for the center and primary channel.
+	 * Will return -1 if not a valid pair for 240MHz
+	 */
+	sb = channel_to_sb(center_channel, primary_channel, WL_CHANSPEC_BW_240);
+
+	/* return err if the sideband was bad or the center channel is not
+	 * valid for the given band.
+	 */
+	if (sb < 0 || !wf_valid_240MHz_center_chan(center_channel, band)) {
+		return INVCHANSPEC;
+	}
+
+	chanspec = ((chan_id << WL_CHANSPEC_GE240_CHAN_SHIFT) |
+	            (sb << WL_CHANSPEC_GE240_SB_SHIFT) |
+	            WL_CHANSPEC_BW_240 |
+	            band);
+
+	return chanspec;
+}
+
+/**
+ * Returns the chanspec for an 320MHz channel given the primary 20MHz channel number,
+ * the center channel number, and the band.
+ *
+ * @param  primary_channel  primary 20 Mhz channel
+ * @param  chan             center channel number
+ * @param  band             band of the 320 MHz channel (chanspec_band_t value)
+ *
+ * Parameters chan is valid 320 MHz center channel numbers for the given band.
+ * The primary channel must be contained in one of the 320 MHz channels.
+ *
+ * @return returns an 320 MHz chanspec, or INVCHANSPEC in case of error
+ *
+ * Refer to <TBD> "Channelization".
+ */
+chanspec_t
+wf_create_320MHz_chspec(uint primary_channel, uint center_channel, chanspec_band_t band)
+{
+	int sb = 0;
+	chanspec_t chanspec = 0;
+	int chan_id = -1;
+
+	if (band == WL_CHANSPEC_BAND_6G) {
+		chan_id = channel_6g_320mhz_to_id(center_channel);
+	}
+
+	/* make sure the channel number were valid */
+	if (chan_id == -1) {
+		return INVCHANSPEC;
+	}
+
+	/* Calculate the sideband value for the center and primary channel.
+	 * Will return -1 if not a valid pair for 320MHz
+	 */
+	sb = channel_to_sb(center_channel, primary_channel, WL_CHANSPEC_BW_320);
+
+	/* return err if the sideband was bad or the center channel is not
+	 * valid for the given band.
+	 */
+	if (sb < 0 || !wf_valid_320MHz_center_chan(center_channel, band)) {
+		return INVCHANSPEC;
+	}
+
+	chanspec = ((chan_id << WL_CHANSPEC_GE240_CHAN_SHIFT) |
+	            (sb << WL_CHANSPEC_GE240_SB_SHIFT) |
+	            WL_CHANSPEC_BW_320 |
+	            band);
+
+	return chanspec;
+}
+
+/**
+ * Returns the chanspec given the primary 20MHz channel number,
+ * the center channel number, channel width, and the band. The channel width
+ * must be 20, 40, 80, 160, 240 or 320 MHz.
+ * 80+80 or 160+160 MHz chanspec creation is not handled by this function,
+ * use wf_create_8080MHz_chspec() or wf_create_160160MHz_chspec()instead.
+ *
+ * @param  primary_channel  primary 20Mhz channel
+ * @param  center_channel   center channel of the channel
+ * @param  bw               width of the channel (chanspec_bw_t)
+ * @param  band             chanspec band of channel  (chanspec_band_t)
+ *
+ * The center_channel can be one of the 802.11 spec valid center channels
+ * for the given bandwidth in the given band.
+ *
+ * @return returns a chanspec, or INVCHANSPEC in case of error
+ */
+chanspec_t
+wf_create_chspec(uint primary_channel, uint center_channel,
+                 chanspec_bw_t bw, chanspec_band_t band)
+{
+	chanspec_t chspec = INVCHANSPEC;
+	int sb = -1;
+	uint sb_shift;
+
+	/* 20MHz channels have matching center and primary channels */
+	if (bw == WL_CHANSPEC_BW_20 && primary_channel == center_channel) {
+
+		sb = 0;
+
+	} else if (bw == WL_CHANSPEC_BW_40 ||
+		bw == WL_CHANSPEC_BW_80 ||
+		bw == WL_CHANSPEC_BW_160 ||
+		WFC_BW_EQ(bw, WL_CHANSPEC_BW_240) ||
+		WFC_BW_EQ(bw, WL_CHANSPEC_BW_320)) {
+
+		/* calculate the sub-band index */
+		sb = channel_to_sb(center_channel, primary_channel, bw);
+	}
+
+	/* if we have a good sub-band, assemble the chanspec, and use wf_chspec_valid()
+	 * to check it for correctness
+	 */
+	if (sb >= 0) {
+		if (WFC_BW_EQ(bw, WL_CHANSPEC_BW_240)) {
+			if (band == WL_CHANSPEC_BAND_6G) {
+				center_channel = channel_6g_240mhz_to_id(center_channel);
+				sb_shift = WL_CHANSPEC_GE240_SB_SHIFT;
+			} else {
+				return INVCHANSPEC;
+			}
+		} else if (WFC_BW_EQ(bw, WL_CHANSPEC_BW_320)) {
+			if (band == WL_CHANSPEC_BAND_6G) {
+				center_channel = channel_6g_320mhz_to_id(center_channel);
+				sb_shift = WL_CHANSPEC_GE240_SB_SHIFT;
+			} else {
+				return INVCHANSPEC;
+			}
+		} else {
+			sb_shift = WL_CHANSPEC_CTL_SB_SHIFT;
+		}
+		chspec = center_channel | band | bw |
+		        ((uint)sb << sb_shift);
+		if (!wf_chspec_valid(chspec)) {
+			chspec = INVCHANSPEC;
+		}
+	}
+
+	return chspec;
+}
+
+/**
+ * Returns the chanspec given the primary 20MHz channel number,
+ * channel width, and the band.
+ *
+ * @param  primary_channel  primary 20Mhz channel
+ * @param  bw               width of the channel (chanspec_bw_t)
+ * @param  band             chanspec band of channel  (chanspec_band_t)
+ *
+ * @return returns a chanspec, or INVCHANSPEC in case of error
+ *
+ * This function is a similar to wf_create_chspec() but does not require the
+ * center_channel parameter. As a result, it can not create 40MHz channels on
+ * the 2G band.
+ *
+ * This function supports creating 20MHz bandwidth chanspecs on any band.
+ *
+ * For the 2GHz band, 40MHz channels overlap, so two 40MHz channels may
+ * have the same primary 20MHz channel. This function will return INVCHANSPEC
+ * whenever called with a bandwidth of 40MHz or wider for the 2GHz band.
+ *
+ * 5GHz and 6GHz bands have non-overlapping 40/80/160 MHz channels, so a
+ * 20MHz primary channel uniquely specifies a wider channel in a given band.
+ *
+ * 80+80MHz channels also cannot be uniquely defined. This function will return
+ * INVCHANSPEC whenever bandwidth of WL_CHANSPEC_BW_8080.
+ */
+chanspec_t
+wf_create_chspec_from_primary(uint primary_channel, chanspec_bw_t bw, chanspec_band_t band)
+{
+	chanspec_t chspec = INVCHANSPEC;
+
+	if (bw == WL_CHANSPEC_BW_20) {
+		chspec = wf_create_20MHz_chspec(primary_channel, band);
+	} else if (band == WL_CHANSPEC_BAND_2G || band == WL_CHANSPEC_BAND_5G) {
+		/* For 5GHz, use the lookup tables for valid 40/80/160 center channels
+		 * and search for a center channel compatible with the given primary channel.
+		 */
+		const uint8 *center_ch = NULL;
+		uint num_ch, i;
+
+		if (band == WL_CHANSPEC_BAND_2G && bw == WL_CHANSPEC_BW_40) {
+			center_ch = wf_2g_40m_chans;
+			num_ch = WF_NUM_2G_40M_CHANS;
+		} else
+		if (bw == WL_CHANSPEC_BW_40) {
+			center_ch = wf_5g_40m_chans;
+			num_ch = WF_NUM_5G_40M_CHANS;
+		} else if (bw == WL_CHANSPEC_BW_80) {
+			center_ch = wf_5g_80m_chans;
+			num_ch = WF_NUM_5G_80M_CHANS;
+		} else if (bw == WL_CHANSPEC_BW_160) {
+			center_ch = wf_5g_160m_chans;
+			num_ch = WF_NUM_5G_160M_CHANS;
+		} else {
+			num_ch = 0;
+		}
+
+		for (i = 0; i < num_ch; i ++) {
+			chspec = wf_create_chspec(primary_channel, center_ch[i], bw, band);
+			if (chspec != INVCHANSPEC) {
+				break;
+			}
+		}
+	}
+	else if (band == WL_CHANSPEC_BAND_6G) {
+		/* For 6GHz, use a formula to calculate the valid 40/80/160 center channel from
+		 * the primary channel.
+		 */
+		uint ch_per_block;
+		uint mask;
+		uint base, center;
+
+		if (bw == WL_CHANSPEC_BW_40) {
+			ch_per_block = 8;
+		} else if (bw == WL_CHANSPEC_BW_80) {
+			ch_per_block = 16;
+		} else if (bw == WL_CHANSPEC_BW_160) {
+			ch_per_block = 32;
+		} else if (WFC_BW_EQ(bw, WL_CHANSPEC_BW_240)) {
+			ch_per_block = 48;
+		} else if (WFC_BW_EQ(bw, WL_CHANSPEC_BW_320)) {
+			ch_per_block = 64;
+		} else {
+			ch_per_block = 0;
+		}
+
+		if (ch_per_block) {
+			/* calculate the base of the block of channel numbers
+			 * covered by the given bw
+			 */
+			mask = ~(ch_per_block - 1);
+			base = 1 + ((primary_channel - 1) & mask);
+
+			/* calculate the center channel from the base channel */
+			center = base + center_chan_to_edge(bw);
+
+			chspec = wf_create_chspec(primary_channel, center, bw, band);
+		}
+	}
+
+	return chspec;
+}
+
+/**
+ * Return the primary 20MHz channel.
+ *
+ * This function returns the channel number of the primary 20MHz channel. For
+ * 20MHz channels this is just the channel number. For 40MHz or wider channels
+ * it is the primary 20MHz channel specified by the chanspec.
+ *
+ * @param	chspec    input chanspec
+ *
+ * @return Returns the channel number of the primary 20MHz channel
+ */
+uint8
+wf_chspec_primary20_chan(chanspec_t chspec)
+{
+	uint center_chan = INVCHANNEL;
+	chanspec_bw_t bw;
+	uint sb;
+
+	ASSERT(!wf_chspec_malformed(chspec));
+
+	/* Is there a sideband ? */
+	if (CHSPEC_IS20(chspec)) {
+		return CHSPEC_CHANNEL(chspec);
+	} else {
+		if ((CHSPEC_IS240(chspec)) || (CHSPEC_IS320(chspec))) {
+			sb = CHSPEC_GE240_SB(chspec) >> WL_CHANSPEC_GE240_SB_SHIFT;
+		} else {
+			sb = CHSPEC_CTL_SB(chspec) >> WL_CHANSPEC_CTL_SB_SHIFT;
+		}
+
+		if (CHSPEC_IS8080(chspec)) {
+			/* For an 80+80 MHz channel, the sideband 'sb' field is an 80 MHz sideband
+			 * (LL, LU, UL, LU) for the 80 MHz frequency segment 0.
+			 */
+
+			/* use bw 80MHz for the primary channel lookup */
+			bw = WL_CHANSPEC_BW_80;
+
+			/* convert from channel index to channel number */
+			if (CHSPEC_IS5G(chspec)) {
+				center_chan = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN0(chspec));
+			} else if (CHSPEC_IS6G(chspec)) {
+				center_chan = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN0(chspec));
+			}
+		} else if (CHSPEC_IS160160(chspec)) {
+			/* For an 160+160 MHz channel, the sideband 'sb' field is an 160MHz sideband
+			 * for the 160 MHz frequency segment 0.
+			 */
+
+			/* use bw 160MHz for the primary channel lookup */
+			bw = WL_CHANSPEC_BW_160;
+
+			/* convert from channel index to channel number */
+			if (CHSPEC_IS5G(chspec)) {
+				center_chan = wf_chspec_5G_id160_to_ch(CHSPEC_CHAN0(chspec));
+			} else if (CHSPEC_IS6G(chspec)) {
+				center_chan = wf_chspec_6G_id160_to_ch(CHSPEC_CHAN0(chspec));
+			}
+		} else if (CHSPEC_IS240(chspec)) {
+			/* use bw 240MHz for the primary channel lookup */
+			bw = WL_CHANSPEC_BW_240;
+
+			/* convert from channel index to channel number */
+			if (CHSPEC_IS6G(chspec)) {
+				center_chan = wf_chspec_6G_id240_to_ch(CHSPEC_GE240_CHAN(chspec));
+			}
+		} else if (CHSPEC_IS320(chspec)) {
+			/* use bw 320MHz for the primary channel lookup */
+			bw = WL_CHANSPEC_BW_320;
+
+			/* convert from channel index to channel number */
+			if (CHSPEC_IS6G(chspec)) {
+				center_chan = wf_chspec_6G_id320_to_ch(CHSPEC_GE240_CHAN(chspec));
+			}
+			/* What to return otherwise? */
+		}
+		else {
+			bw = CHSPEC_BW(chspec);
+			center_chan = CHSPEC_CHANNEL(chspec) >> WL_CHANSPEC_CHAN_SHIFT;
+		}
+
+		return (uint8)(channel_to_primary20_chan((uint8)center_chan, bw, sb));
+	}
+}
+
+/**
+ * Return the bandwidth string for a given chanspec
+ *
+ * This function returns the bandwidth string for the passed chanspec.
+ *
+ * @param	chspec    input chanspec
+ *
+ * @return Returns the bandwidth string:
+ *         "320", "160+160", "20", "40", "80", "160", "80+80", "240"
+ */
+const char *
+BCMRAMFN(wf_chspec_to_bw_str)(chanspec_t chspec)
+{
+	return wf_chspec_bw_str[(CHSPEC_BW(chspec) >> WL_CHANSPEC_BW_SHIFT)];
+}
+
+/**
+ * Return the primary 20MHz chanspec of a given chanspec
+ *
+ * This function returns the chanspec of the primary 20MHz channel. For 20MHz
+ * channels this is just the chanspec. For 40MHz or wider channels it is the
+ * chanspec of the primary 20MHz channel specified by the chanspec.
+ *
+ * @param	chspec    input chanspec
+ *
+ * @return Returns the chanspec of the primary 20MHz channel
+ */
+chanspec_t
+wf_chspec_primary20_chspec(chanspec_t chspec)
+{
+	chanspec_t pri_chspec = chspec;
+	uint8 pri_chan;
+
+	ASSERT(!wf_chspec_malformed(chspec));
+
+	/* Is there a sideband ? */
+	if (!CHSPEC_IS20(chspec)) {
+		pri_chan = wf_chspec_primary20_chan(chspec);
+		pri_chspec = pri_chan | WL_CHANSPEC_BW_20;
+		pri_chspec |= CHSPEC_BAND(chspec);
+	}
+	return pri_chspec;
+}
+
+/* return chanspec given primary 20MHz channel and bandwidth
+ * return 0 on error
+ * does not support 6G
+ */
+uint16
+wf_channel2chspec(uint pri_ch, uint bw)
+{
+	uint16 chspec;
+	const uint8 *center_ch = NULL;
+	int num_ch = 0;
+	int sb = -1;
+	int i = 0;
+
+	chspec = ((pri_ch <= CH_MAX_2G_CHANNEL) ? WL_CHANSPEC_BAND_2G : WL_CHANSPEC_BAND_5G);
+
+	chspec |= bw;
+
+	if (bw == WL_CHANSPEC_BW_40) {
+		if (pri_ch <= CH_MAX_2G_CHANNEL) {
+			center_ch = wf_2g_40m_chans;
+			num_ch = WF_NUM_2G_40M_CHANS;
+		} else {
+			center_ch = wf_5g_40m_chans;
+			num_ch = WF_NUM_5G_40M_CHANS;
+		}
+	} else if (bw == WL_CHANSPEC_BW_80) {
+		center_ch = wf_5g_80m_chans;
+		num_ch = WF_NUM_5G_80M_CHANS;
+	} else if (bw == WL_CHANSPEC_BW_160) {
+		center_ch = wf_5g_160m_chans;
+		num_ch = WF_NUM_5G_160M_CHANS;
+	} else if (bw == WL_CHANSPEC_BW_20) {
+		chspec |= pri_ch;
+		return chspec;
+	} else {
+		return 0;
+	}
+
+	for (i = 0; i < num_ch; i ++) {
+		sb = channel_to_sb(center_ch[i], pri_ch, (chanspec_bw_t)bw);
+		if (sb >= 0) {
+			chspec |= center_ch[i];
+			chspec |= (sb << WL_CHANSPEC_CTL_SB_SHIFT);
+			break;
+		}
+	}
+
+	/* check for no matching sb/center */
+	if (sb < 0) {
+		return 0;
+	}
+
+	return chspec;
+}
+
+/**
+ * Return the primary 40MHz chanspec or a 40MHz or wider channel
+ *
+ * This function returns the chanspec for the primary 40MHz of an 80MHz or wider channel.
+ * The primary 40MHz channel is the 40MHz sub-band that contains the primary 20MHz channel.
+ * The primary 20MHz channel of the returned 40MHz chanspec is the same as the primary 20MHz
+ * channel of the input chanspec.
+ *
+ * @param	chspec    input chanspec
+ *
+ * @return Returns the chanspec of the primary 20MHz channel
+ */
+chanspec_t
+wf_chspec_primary40_chspec(chanspec_t chspec)
+{
+	chanspec_t chspec40 = chspec;
+	uint center_chan;
+	uint sb;
+
+	ASSERT(!wf_chspec_malformed(chspec));
+
+	/* if the chanspec is > 80MHz, use the helper routine to find the primary 80 MHz channel */
+	if (CHSPEC_IS8080(chspec) || CHSPEC_IS160(chspec)) {
+		chspec = wf_chspec_primary80_chspec(chspec);
+	}
+
+	/* determine primary 40 MHz sub-channel of an 80 MHz chanspec */
+	if (CHSPEC_IS80(chspec)) {
+		center_chan = CHSPEC_CHANNEL(chspec);
+		sb = CHSPEC_CTL_SB(chspec);
+
+		if (sb < WL_CHANSPEC_CTL_SB_UL) {
+			/* Primary 40MHz is on lower side */
+			center_chan -= CH_20MHZ_APART;
+			/* sideband bits are the same for LL/LU and L/U */
+		} else {
+			/* Primary 40MHz is on upper side */
+			center_chan += CH_20MHZ_APART;
+			/* sideband bits need to be adjusted by UL offset */
+			sb -= WL_CHANSPEC_CTL_SB_UL;
+		}
+
+		/* Create primary 40MHz chanspec */
+		chspec40 = (CHSPEC_BAND(chspec) | WL_CHANSPEC_BW_40 |
+		            sb | center_chan);
+	}
+
+	return chspec40;
+}
+
+/**
+ * Return the chanspec band for a given frequency.
+ *
+ * @param  freq		frequency in MHz of the channel center
+ *
+ * @return  Returns chanspec band of frequency (chanspec_band_t)
+ */
+chanspec_band_t
+wf_mhz2chanspec_band(uint freq)
+{
+	chanspec_band_t band = INVCHANSPEC;
+
+	if (freq >= 2400u && freq <= 2500u) {
+		band = WL_CHANSPEC_BAND_2G;
+	} else if (freq >= 5000u && freq < 5935u) {
+		band = WL_CHANSPEC_BAND_5G;
+	} else if (freq >= 5935u && freq <= 7205u) {
+		band = WL_CHANSPEC_BAND_6G;
+	}
+
+	return band;
+}
+
+/**
+ * Return the channel number for a given frequency and base frequency.
+ *
+ * @param   freq            frequency in MHz of the channel center
+ * @param   start_factor    starting base frequency in 500 KHz units
+ *
+ * @return  Returns a channel number > 0, or -1 on error
+ *
+ * The returned channel number is relative to the given base frequency.
+ *
+ * The base frequency is specified as (start_factor * 500 kHz).
+ * Constants WF_CHAN_FACTOR_2_4_G, WF_CHAN_FACTOR_5_G, and WF_CHAN_FACTOR_6_G are
+ * defined for 2.4 GHz, 5 GHz, and 6 GHz bands.
+ *
+ * If the given base frequency is zero these base frequencies are assumed:
+ *
+ *              freq (GHz)  -> assumed base freq (GHz)
+ *  2G band   2.4   - 2.5      2.407
+ *  5G band   5.0   - 5.940    5.000
+ *  6G band   5.940 - 7.205    5.940
+ *
+ * It is an error if the start_factor is zero and the freq is not in one of
+ * these ranges.
+ *
+ * The returned channel will be in the range [1, 14] in the 2.4 GHz band,
+ * [1, 253] for 6 GHz band, or [1, 200] otherwise.
+ *
+ * It is an error if the start_factor is WF_CHAN_FACTOR_2_4_G and the
+ * frequency is not a 2.4 GHz channel. For any other start factor the frequency
+ * must be an even 5 MHz multiple greater than the base frequency.
+ *
+ * For a start_factor WF_CHAN_FACTOR_6_G, the frequency may be up to 7.205 MHz
+ * (channel 253). For any other start_factor, the frequence can be up to
+ * 1 GHz from the base freqency (channel 200).
+ *
+ * Reference 802.11-2016, section 17.3.8.3 and section 16.3.6.3
+ */
+int
+wf_mhz2channel(uint freq, uint start_factor)
+{
+	int ch = -1;
+	uint base;
+	int offset;
+
+	/* take the default channel start frequency */
+	if (start_factor == 0) {
+		if (freq >= 2400 && freq <= 2500) {
+			start_factor = WF_CHAN_FACTOR_2_4_G;
+		} else if (freq >= 5000 && freq < 5935) {
+			start_factor = WF_CHAN_FACTOR_5_G;
+		} else if (freq >= 5935 && freq <= 7205) {
+			start_factor = WF_CHAN_FACTOR_6_G;
+		}
+	}
+
+	if (freq == 2484 && start_factor == WF_CHAN_FACTOR_2_4_G) {
+		return 14;
+	} else if (freq == 5935 && start_factor == WF_CHAN_FACTOR_6_G) {
+		/* channel #2 is an oddball, 10MHz below chan #1 */
+		return 2;
+	} else if (freq == 5960 && start_factor == WF_CHAN_FACTOR_6_G) {
+		/* do not return ch #2 for the convetional location that #2 would appear */
+		return -1;
+	}
+
+	base = start_factor / 2;
+
+	if (freq < base) {
+		return -1;
+	}
+
+	offset = freq - base;
+	ch = offset / 5;
+
+	/* check that frequency is a 5MHz multiple from the base */
+	if (offset != (ch * 5))
+		return -1;
+
+	/* channel range checks */
+	if (start_factor == WF_CHAN_FACTOR_2_4_G) {
+		/* 2G should only be up to 13 here as 14 is
+		 * handled above as it is a non-5MHz offset
+		 */
+		if (ch > 13) {
+			ch = -1;
+		}
+	}
+	else if (start_factor == WF_CHAN_FACTOR_6_G) {
+		/* 6G has a higher channel range than 5G channelization specifies [1,200] */
+		if ((uint)ch > CH_MAX_6G_CHANNEL) {
+			ch = -1;
+		}
+	} else if (ch > 200) {
+			ch = -1;
+	}
+
+	return ch;
+}
+
+/**
+ * Return the center frequency in MHz of the given channel and base frequency.
+ *
+ * The channel number is interpreted relative to the given base frequency.
+ *
+ * The valid channel range is [1, 14] in the 2.4 GHz band, [1,253] in the 6 GHz
+ * band, and [1, 200] otherwise.
+ * The base frequency is specified as (start_factor * 500 kHz).
+ * Constants WF_CHAN_FACTOR_2_4_G, WF_CHAN_FACTOR_5_G, and WF_CHAN_FACTOR_6_G are
+ * defined for 2.4 GHz, 5 GHz, and 6 GHz bands.
+ * The channel range of [1, 14] is only checked for a start_factor of
+ * WF_CHAN_FACTOR_2_4_G (4814).
+ * Odd start_factors produce channels on .5 MHz boundaries, in which case
+ * the answer is rounded down to an integral MHz.
+ * -1 is returned for an out of range channel.
+ *
+ * Reference 802.11-2016, section 17.3.8.3 and section 16.3.6.3
+ *
+ * @param	channel       input channel number
+ * @param	start_factor  base frequency in 500 kHz units, e.g. 10000 for 5 GHz
+ *
+ * @return Returns a frequency in MHz
+ *
+ * @see  WF_CHAN_FACTOR_2_4_G
+ * @see  WF_CHAN_FACTOR_5_G
+ * @see  WF_CHAN_FACTOR_6_G
+ */
+int
+wf_channel2mhz(uint ch, uint start_factor)
+{
+	int freq;
+
+	if ((start_factor == WF_CHAN_FACTOR_2_4_G && (ch < 1 || ch > 14)) ||
+	    (start_factor == WF_CHAN_FACTOR_6_G && (ch < 1 || ch > 253)) ||
+	    (start_factor != WF_CHAN_FACTOR_6_G && (ch < 1 || ch > 200))) {
+		freq = -1;
+	} else if ((start_factor == WF_CHAN_FACTOR_2_4_G) && (ch == 14)) {
+		freq = 2484;
+	} else if ((start_factor == WF_CHAN_FACTOR_6_G) && (ch == 2)) {
+		freq = 5935;
+	} else {
+		freq = ch * 5 + start_factor / 2;
+	}
+
+	return freq;
+}
+
+static const uint16 sidebands[] = {
+	WL_CHANSPEC_CTL_SB_LLL, WL_CHANSPEC_CTL_SB_LLU,
+	WL_CHANSPEC_CTL_SB_LUL, WL_CHANSPEC_CTL_SB_LUU,
+	WL_CHANSPEC_CTL_SB_ULL, WL_CHANSPEC_CTL_SB_ULU,
+	WL_CHANSPEC_CTL_SB_UUL, WL_CHANSPEC_CTL_SB_UUU
+};
+
+/*
+ * Returns the chanspec 80Mhz channel corresponding to the following input
+ * parameters
+ *
+ *	primary_channel - primary 20Mhz channel
+ *	center_channel   - center frequecny of the 80Mhz channel
+ *
+ * The center_channel can be one of {42, 58, 106, 122, 138, 155}
+ *
+ * returns INVCHANSPEC in case of error
+ *
+ * does not support 6G
+ */
+chanspec_t
+wf_chspec_80(uint8 center_channel, uint8 primary_channel)
+{
+
+	chanspec_t chanspec = INVCHANSPEC;
+	chanspec_t chanspec_cur;
+	uint i;
+
+	for (i = 0; i < WF_NUM_SIDEBANDS_80MHZ; i++) {
+		chanspec_cur = CH80MHZ_CHSPEC(center_channel, sidebands[i]);
+		if (primary_channel == wf_chspec_primary20_chan(chanspec_cur)) {
+			chanspec = chanspec_cur;
+			break;
+		}
+	}
+	/* If the loop ended early, we are good, otherwise we did not
+	* find a 80MHz chanspec with the given center_channel that had a primary channel
+	*matching the given primary_channel.
+	*/
+	return chanspec;
+}
+
+/*
+ * Returns the 80+80 chanspec corresponding to the following input parameters
+ *
+ *    primary_20mhz - Primary 20 MHz channel
+ *    chan0 - center channel number of one frequency segment
+ *    chan1 - center channel number of the other frequency segment
+ *
+ * Parameters chan0 and chan1 are channel numbers in {42, 58, 106, 122, 138, 155}.
+ * The primary channel must be contained in one of the 80MHz channels. This routine
+ * will determine which frequency segment is the primary 80 MHz segment.
+ *
+ * Returns INVCHANSPEC in case of error.
+ *
+ * Refer to 802.11-2016 section 22.3.14 "Channelization".
+ *
+ * does not support 6G
+ */
+chanspec_t
+wf_chspec_get8080_chspec(uint8 primary_20mhz, uint8 chan0, uint8 chan1)
+{
+	int sb = 0;
+	uint16 chanspec = 0;
+	int chan0_id = 0, chan1_id = 0;
+	int seg0, seg1;
+
+	chan0_id = channel_80mhz_to_id(chan0);
+	chan1_id = channel_80mhz_to_id(chan1);
+
+	/* make sure the channel numbers were valid */
+	if (chan0_id == -1 || chan1_id == -1)
+		return INVCHANSPEC;
+
+	/* does the primary channel fit with the 1st 80MHz channel ? */
+	sb = channel_to_sb(chan0, primary_20mhz, WL_CHANSPEC_BW_80);
+	if (sb >= 0) {
+		/* yes, so chan0 is frequency segment 0, and chan1 is seg 1 */
+		seg0 = chan0_id;
+		seg1 = chan1_id;
+	} else {
+		/* no, so does the primary channel fit with the 2nd 80MHz channel ? */
+		sb = channel_to_sb(chan1, primary_20mhz, WL_CHANSPEC_BW_80);
+		if (sb < 0) {
+			/* no match for pri_ch to either 80MHz center channel */
+			return INVCHANSPEC;
+		}
+		/* swapped, so chan1 is frequency segment 0, and chan0 is seg 1 */
+		seg0 = chan1_id;
+		seg1 = chan0_id;
+	}
+
+	chanspec = ((seg0 << WL_CHANSPEC_CHAN0_SHIFT) |
+	            (seg1 << WL_CHANSPEC_CHAN1_SHIFT) |
+	            (sb << WL_CHANSPEC_CTL_SB_SHIFT) |
+	            WL_CHANSPEC_BW_8080 |
+	            WL_CHANSPEC_BAND_5G);
+
+	return chanspec;
+}
+
+/*
+ * Returns the center channel of the primary 80 MHz sub-band of the provided chanspec
+ */
+uint8
+wf_chspec_primary80_channel(chanspec_t chanspec)
+{
+	chanspec_t primary80_chspec;
+	uint8 primary80_chan;
+
+	primary80_chspec = wf_chspec_primary80_chspec(chanspec);
+
+	if (primary80_chspec == INVCHANSPEC) {
+		primary80_chan = INVCHANNEL;
+	} else {
+		primary80_chan = CHSPEC_CHANNEL(primary80_chspec);
+	}
+
+	return primary80_chan;
+}
+
+/*
+ * Returns the center channel of the secondary 80 MHz sub-band of the provided chanspec
+ */
+uint8
+wf_chspec_secondary80_channel(chanspec_t chanspec)
+{
+	chanspec_t secondary80_chspec;
+	uint8 secondary80_chan;
+
+	secondary80_chspec = wf_chspec_secondary80_chspec(chanspec);
+
+	if (secondary80_chspec == INVCHANSPEC) {
+		secondary80_chan = INVCHANNEL;
+	} else {
+		secondary80_chan = CHSPEC_CHANNEL(secondary80_chspec);
+	}
+
+	return secondary80_chan;
+}
+
+/*
+ * Returns the chanspec for the primary 80MHz sub-band of an 160MHz or 80+80 channel
+ */
+chanspec_t
+wf_chspec_primary80_chspec(chanspec_t chspec)
+{
+	chanspec_t chspec80;
+	uint center_chan;
+	uint sb;
+
+	ASSERT(!wf_chspec_malformed(chspec));
+
+	if (CHSPEC_IS80(chspec)) {
+		chspec80 = chspec;
+	}
+	else if (CHSPEC_IS8080(chspec)) {
+		sb = CHSPEC_CTL_SB(chspec);
+
+		/* primary sub-band is stored in seg0 */
+		if (CHSPEC_IS5G(chspec)) {
+			center_chan = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN0(chspec));
+		} else if (CHSPEC_IS6G(chspec)) {
+			center_chan = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN0(chspec));
+		} else {
+			center_chan = INVCHANNEL;
+		}
+
+		/* Create primary 80MHz chanspec */
+		chspec80 = (CHSPEC_BAND(chspec) | WL_CHANSPEC_BW_80 | sb | center_chan);
+	}
+	else if (CHSPEC_IS160(chspec)) {
+		center_chan = CHSPEC_CHANNEL(chspec);
+		sb = CHSPEC_CTL_SB(chspec);
+
+		if (sb < WL_CHANSPEC_CTL_SB_ULL) {
+			/* Primary 80MHz is on lower side */
+			center_chan -= CH_40MHZ_APART;
+		}
+		else {
+			/* Primary 80MHz is on upper side */
+			center_chan += CH_40MHZ_APART;
+			sb -= WL_CHANSPEC_CTL_SB_ULL;
+		}
+
+		/* Create primary 80MHz chanspec */
+		chspec80 = (CHSPEC_BAND(chspec) | WL_CHANSPEC_BW_80 | sb | center_chan);
+	}
+	else {
+		chspec80 = INVCHANSPEC;
+	}
+
+	return chspec80;
+}
+
+/*
+ * Returns the chanspec for the secondary 80MHz sub-band of an 160MHz or 80+80 channel
+ */
+chanspec_t
+wf_chspec_secondary80_chspec(chanspec_t chspec)
+{
+	chanspec_t chspec80;
+	uint center_chan;
+
+	ASSERT(!wf_chspec_malformed(chspec));
+
+	if (CHSPEC_IS8080(chspec)) {
+		/* secondary sub-band is stored in seg1 */
+		if (CHSPEC_IS5G(chspec)) {
+			center_chan = wf_chspec_5G_id80_to_ch(CHSPEC_CHAN1(chspec));
+		} else if (CHSPEC_IS6G(chspec)) {
+			center_chan = wf_chspec_6G_id80_to_ch(CHSPEC_CHAN1(chspec));
+		} else {
+			center_chan = INVCHANNEL;
+		}
+
+		/* Create secondary 80MHz chanspec */
+		chspec80 = (CHSPEC_BAND(chspec) |
+		            WL_CHANSPEC_BW_80 |
+		            WL_CHANSPEC_CTL_SB_LL |
+		            center_chan);
+	}
+	else if (CHSPEC_IS160(chspec)) {
+		center_chan = CHSPEC_CHANNEL(chspec);
+
+		if (CHSPEC_CTL_SB(chspec) < WL_CHANSPEC_CTL_SB_ULL) {
+			/* Primary 80MHz is on lower side, so the secondary is on
+			 * the upper side
+			 */
+			center_chan += CH_40MHZ_APART;
+		} else {
+			/* Primary 80MHz is on upper side, so the secondary is on
+			 * the lower side
+			 */
+			center_chan -= CH_40MHZ_APART;
+		}
+
+		/* Create secondary 80MHz chanspec */
+		chspec80 = (CHSPEC_BAND(chspec) |
+		            WL_CHANSPEC_BW_80 |
+		            WL_CHANSPEC_CTL_SB_LL |
+		            center_chan);
+	}
+	else {
+		chspec80 = INVCHANSPEC;
+	}
+
+	return chspec80;
+}
+
+/*
+ * For 160MHz or 80P80 chanspec, set ch[0]/ch[1] to be the low/high 80 Mhz channels
+ *
+ * For 20/40/80MHz chanspec, set ch[0] to be the center freq, and chan[1]=-1
+ */
+void
+wf_chspec_get_80p80_channels(chanspec_t chspec, uint8 *ch)
+{
+
+	if (CHSPEC_IS160(chspec)) {
+		uint8 center_chan = CHSPEC_CHANNEL(chspec);
+		ch[0] = center_chan - CH_40MHZ_APART;
+		ch[1] = center_chan + CH_40MHZ_APART;
+	}
+	else {
+		/* for 20, 40, and 80 Mhz */
+		ch[0] = CHSPEC_CHANNEL(chspec);
+		ch[1] = -1;
+	}
+	return;
+
+}
+
+/*
+ * Returns the center channel of the primary 160MHz sub-band of the provided chanspec
+ */
+uint8
+wf_chspec_primary160_channel(chanspec_t chanspec)
+{
+	chanspec_t primary160_chspec;
+	uint8 primary160_chan;
+
+	primary160_chspec = wf_chspec_primary160_chspec(chanspec);
+
+	if (primary160_chspec == INVCHANSPEC) {
+		primary160_chan = INVCHANNEL;
+	} else {
+		primary160_chan = CHSPEC_CHANNEL(primary160_chspec);
+	}
+
+	return primary160_chan;
+}
+
+/*
+ * Returns the chanspec for the primary 160MHz sub-band of an 240/320MHz or 160+160 channel
+ */
+chanspec_t
+wf_chspec_primary160_chspec(chanspec_t chspec)
+{
+	chanspec_t chspec160;
+	uint center_chan;
+	uint sb;
+
+	ASSERT(!wf_chspec_malformed(chspec));
+
+	if (CHSPEC_IS160(chspec)) {
+		chspec160 = chspec;
+	}
+	else if (CHSPEC_IS240(chspec)) {
+		uint8 ch_id = CHSPEC_GE240_CHAN(chspec);
+		center_chan = wf_chspec_240_id2cch(chspec);
+		sb = CHSPEC_GE240_SB(chspec) >> WL_CHANSPEC_GE240_SB_SHIFT;
+		/*
+		 * Identify the chanspec is of the form 160+80 or 80+160 from the channel ID.
+		 * Channel ID : even for 160+80 and odd for 80+160
+		 */
+		if ((!(ch_id & 0x1u)) && (sb < 8u)) {
+			/* Primary 160MHz is on lower side */
+			center_chan -= CH_40MHZ_APART;
+		} else if ((ch_id & 0x1u) && (sb >= 4u)) {
+			/* Primary 160MHz is on upper side */
+			center_chan += CH_40MHZ_APART;
+			sb -= 4u;
+		} else {
+			chspec160 = INVCHANSPEC;
+			goto done;
+		}
+
+		/* Create primary 160MHz chanspec */
+		chspec160 = (CHSPEC_BAND(chspec) |
+		             WL_CHANSPEC_BW_160 |
+		             (sb << WL_CHANSPEC_CTL_SB_SHIFT) |
+		             center_chan);
+	} else if (CHSPEC_IS320(chspec)) {
+		center_chan = wf_chspec_320_id2cch(chspec);
+		sb = CHSPEC_GE240_SB(chspec) >> WL_CHANSPEC_GE240_SB_SHIFT;
+
+		if (sb < 8u) {
+			/* Primary 160MHz is on lower side */
+			center_chan -= CH_80MHZ_APART;
+		}
+		else {
+			/* Primary 160MHz is on upper side */
+			center_chan += CH_80MHZ_APART;
+			sb -= 8u;
+		}
+
+		/* Create primary 160MHz chanspec */
+		chspec160 = (CHSPEC_BAND(chspec) |
+		             WL_CHANSPEC_BW_160 |
+		             (sb << WL_CHANSPEC_CTL_SB_SHIFT) |
+		             center_chan);
+	}
+	else {
+		chspec160 = INVCHANSPEC;
+	}
+done:
+	return chspec160;
+}
+
+/* Populates array with all 20MHz side bands of a given chanspec_t in the following order:
+ *		primary20, secondary20, two secondary40s, four secondary80s.
+ *    'chspec' is the chanspec of interest
+ *    'pext' must point to an uint8 array of long enough to hold all side bands of the given chspec
+ *
+ * Works with 20, 40, 80, and 160MHz chspec
+ */
+void
+wf_get_all_ext(chanspec_t chspec, uint8 *pext)
+{
+	chanspec_t t = (CHSPEC_IS160(chspec)) ? /* if bw > 80MHz */
+		wf_chspec_primary80_chspec(chspec) : (chspec); /* extract primary 80 */
+	/* primary20 channel as first element */
+	uint8 pri_ch = (pext)[0] = wf_chspec_primary20_chan(t);
+
+	if (CHSPEC_IS20(chspec)) {
+		return; /* nothing more to do since 20MHz chspec */
+	}
+	/* 20MHz EXT */
+	(pext)[1] = (IS_CTL_IN_L20(t) ? pri_ch + CH_20MHZ_APART : pri_ch - CH_20MHZ_APART);
+
+	if (CHSPEC_IS40(chspec)) {
+		return; /* nothing more to do since 40MHz chspec */
+	}
+	/* center 40MHz EXT */
+	t = wf_channel2chspec((IS_CTL_IN_L40(chspec) ?
+		pri_ch + CH_40MHZ_APART : pri_ch - CH_40MHZ_APART), WL_CHANSPEC_BW_40);
+	GET_ALL_SB(t, &((pext)[2])); /* get the 20MHz side bands in 40MHz EXT */
+
+	if (CHSPEC_IS80(chspec)) {
+		return; /* nothing more to do since 80MHz chspec */
+	}
+	t = CH80MHZ_CHSPEC(wf_chspec_secondary80_channel(chspec), WL_CHANSPEC_CTL_SB_LLL);
+	/* get the 20MHz side bands in 80MHz EXT (secondary) */
+	GET_ALL_SB(t, &((pext)[4]));
+}
+
+/*
+ * Given two chanspecs, returns true if they overlap.
+ * (Overlap: At least one 20MHz subband is common between the two chanspecs provided)
+ */
+bool wf_chspec_overlap(chanspec_t chspec0, chanspec_t chspec1)
+{
+	uint8 ch0, ch1;
+
+	if (CHSPEC_BAND(chspec0) != CHSPEC_BAND(chspec1)) {
+		return FALSE;
+	}
+
+	FOREACH_20_SB(chspec0, ch0) {
+		FOREACH_20_SB(chspec1, ch1) {
+			if ((uint)ABS(ch0 - ch1) < CH_20MHZ_APART) {
+				return TRUE;
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+uint8
+channel_bw_to_width(chanspec_t chspec)
+{
+	uint8 channel_width;
+
+	if (CHSPEC_IS80(chspec))
+		channel_width = VHT_OP_CHAN_WIDTH_80;
+	else if (CHSPEC_IS160(chspec))
+		channel_width = VHT_OP_CHAN_WIDTH_160;
+	else if (CHSPEC_IS8080(chspec))
+		channel_width = VHT_OP_CHAN_WIDTH_80_80;
+	else
+		channel_width = VHT_OP_CHAN_WIDTH_20_40;
+
+	return channel_width;
+}
+
+uint wf_chspec_first_20_sb(chanspec_t chspec)
+{
+	if (CHSPEC_IS80(chspec)) {
+		return LL_20_SB(CHSPEC_CHANNEL(chspec));
+	} else if (CHSPEC_IS40(chspec)) {
+		return LOWER_20_SB(CHSPEC_CHANNEL(chspec));
+	} else {
+		return CHSPEC_CHANNEL(chspec);
+	}
+}
+
+chanspec_t
+wf_create_chspec_sb(uint sb, uint center_channel, chanspec_bw_t bw, chanspec_band_t band)
+{
+	chanspec_t chspec;
+	if (sb > (WL_CHANSPEC_CTL_SB_MASK >> WL_CHANSPEC_CTL_SB_SHIFT)) {
+		return INVCHANSPEC;
+	}
+	chspec = center_channel | band | bw | ((uint)sb << WL_CHANSPEC_CTL_SB_SHIFT);
+	return wf_chspec_valid(chspec) ? chspec : INVCHANSPEC;
+}
+
+chanspec_t
+wf_create_8080MHz_chspec_sb(uint sb, uint chan0, uint chan1, chanspec_band_t band)
+{
+	int chan0_id, chan1_id, seg0, seg1;
+	chanspec_t chspec;
+
+	if (sb > (WL_CHANSPEC_CTL_SB_UUU >> WL_CHANSPEC_CTL_SB_SHIFT)) {
+		return INVCHANSPEC;
+	}
+	/* From here on sb is not an index, but value for SB field */
+	sb <<= WL_CHANSPEC_CTL_SB_SHIFT;
+
+	/* frequency segments need to be non-contiguous, so the channel
+	 * separation needs to be greater than 80MHz
+	 */
+	if ((uint)ABS((int)(chan0 - chan1)) <= CH_80MHZ_APART) {
+		return INVCHANSPEC;
+	}
+
+	if (band == WL_CHANSPEC_BAND_5G) {
+		chan0_id = channel_80mhz_to_id(chan0);
+		chan1_id = channel_80mhz_to_id(chan1);
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		chan0_id = channel_6g_80mhz_to_id(chan0);
+		chan1_id = channel_6g_80mhz_to_id(chan1);
+	} else {
+		return INVCHANSPEC;
+	}
+
+	/* make sure the channel numbers were valid */
+	if ((chan0_id == -1) || (chan1_id == -1)) {
+		return INVCHANSPEC;
+	}
+	/* Optionally swapping channel IDs to make sure that control subchannel
+	 * is in chan0
+	 */
+	if (sb < WL_CHANSPEC_CTL_SB_ULL) {
+		seg0 = chan0_id;
+		seg1 = chan1_id;
+	} else {
+		seg0 = chan1_id;
+		seg1 = chan0_id;
+		sb -= WL_CHANSPEC_CTL_SB_ULL;
+	}
+	chspec = ((seg0 << WL_CHANSPEC_CHAN0_SHIFT) |
+	         (seg1 << WL_CHANSPEC_CHAN1_SHIFT) |
+	         sb | WL_CHANSPEC_BW_8080 | band);
+	return wf_chspec_valid(chspec) ? chspec : INVCHANSPEC;
+}
+
+chanspec_t
+wf_create_160160MHz_chspec_sb(uint sb, uint chan0, uint chan1, chanspec_band_t band)
+{
+	int chan0_id, chan1_id, seg0, seg1;
+	chanspec_t chspec;
+
+	if (sb > (WL_CHANSPEC_CTL_SB_UUU >> WL_CHANSPEC_CTL_SB_SHIFT)) {
+		return INVCHANSPEC;
+	}
+	/* From here on sb is not an index, but value for SB field */
+	sb <<= WL_CHANSPEC_CTL_SB_SHIFT;
+
+	/* frequency segments need to be non-contiguous, so the channel
+	 * separation needs to be greater than 160MHz
+	 */
+	if ((uint)ABS((int)(chan0 - chan1)) <= CH_160MHZ_APART) {
+		return INVCHANSPEC;
+	}
+
+	if (band == WL_CHANSPEC_BAND_5G) {
+		chan0_id = channel_5g_160mhz_to_id(chan0);
+		chan1_id = channel_5g_160mhz_to_id(chan1);
+	} else if (band == WL_CHANSPEC_BAND_6G) {
+		chan0_id = channel_6g_160mhz_to_id(chan0);
+		chan1_id = channel_6g_160mhz_to_id(chan1);
+	} else {
+		return INVCHANSPEC;
+	}
+
+	/* make sure the channel numbers were valid */
+	if ((chan0_id == -1) || (chan1_id == -1)) {
+		return INVCHANSPEC;
+	}
+	/* Optionally swapping channel IDs to make sure that control subchannel
+	 * is in chan0
+	 */
+	if (sb < WL_CHANSPEC_CTL_SB_ULL) {
+		seg0 = chan0_id;
+		seg1 = chan1_id;
+	} else {
+		seg0 = chan1_id;
+		seg1 = chan0_id;
+		sb -= WL_CHANSPEC_CTL_SB_ULL;
+	}
+	chspec = ((seg0 << WL_CHANSPEC_CHAN0_SHIFT) |
+	         (seg1 << WL_CHANSPEC_CHAN1_SHIFT) |
+	         sb | WL_CHANSPEC_BW_160160 | band);
+	return wf_chspec_valid(chspec) ? chspec : INVCHANSPEC;
+}
diff --git a/drivers/net/wireless/ap12275/bcmwifi_radiotap.c b/drivers/net/wireless/ap12275/bcmwifi_radiotap.c
new file mode 100644
index 000000000000..acdb551fc046
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmwifi_radiotap.c
@@ -0,0 +1,1035 @@
+/*
+ * RadioTap utility routines for WL
+ * This file housing the functions use by
+ * wl driver.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmwifi_channels.h>
+#include <hndd11.h>
+#include <bcmwifi_radiotap.h>
+
+const struct rtap_field rtap_parse_info[] = {
+	{8, 8}, /* 0:  IEEE80211_RADIOTAP_TSFT */
+	{1, 1}, /* 1:  IEEE80211_RADIOTAP_FLAGS */
+	{1, 1}, /* 2:  IEEE80211_RADIOTAP_RATE */
+	{4, 2}, /* 3:  IEEE80211_RADIOTAP_CHANNEL */
+	{2, 2}, /* 4:  IEEE80211_RADIOTAP_FHSS */
+	{1, 1}, /* 5:  IEEE80211_RADIOTAP_DBM_ANTSIGNAL */
+	{1, 1}, /* 6:  IEEE80211_RADIOTAP_DBM_ANTNOISE */
+	{2, 2}, /* 7:  IEEE80211_RADIOTAP_LOCK_QUALITY */
+	{2, 2}, /* 8:  IEEE80211_RADIOTAP_TX_ATTENUATION */
+	{2, 2}, /* 9:  IEEE80211_RADIOTAP_DB_TX_ATTENUATION */
+	{1, 1}, /* 10: IEEE80211_RADIOTAP_DBM_TX_POWER */
+	{1, 1}, /* 11: IEEE80211_RADIOTAP_ANTENNA */
+	{1, 1}, /* 12: IEEE80211_RADIOTAP_DB_ANTSIGNAL */
+	{1, 1}, /* 13: IEEE80211_RADIOTAP_DB_ANTNOISE */
+	{0, 0}, /* 14: netbsd */
+	{2, 2}, /* 15: IEEE80211_RADIOTAP_TXFLAGS */
+	{0, 0}, /* 16: missing */
+	{1, 1}, /* 17: IEEE80211_RADIOTAP_RETRIES */
+	{8, 4}, /* 18: IEEE80211_RADIOTAP_XCHANNEL */
+	{3, 1}, /* 19: IEEE80211_RADIOTAP_MCS */
+	{8, 4}, /* 20: IEEE80211_RADIOTAP_AMPDU_STATUS */
+	{12, 2}, /* 21: IEEE80211_RADIOTAP_VHT */
+	{0, 0}, /* 22: */
+	{0, 0}, /* 23: */
+	{0, 0}, /* 24: */
+	{0, 0}, /* 25: */
+	{0, 0}, /* 26: */
+	{0, 0}, /* 27: */
+	{0, 0}, /* 28: */
+	{0, 0}, /* 29: IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE */
+	{6, 2}, /* 30: IEEE80211_RADIOTAP_VENDOR_NAMESPACE */
+	{0, 0}  /* 31: IEEE80211_RADIOTAP_EXT */
+};
+
+static int bitmap = 0;
+
+void
+radiotap_add_vendor_ns(ieee80211_radiotap_header_t *hdr);
+
+void
+radiotap_encode_multi_rssi(monitor_pkt_rxsts_t* rxsts, ieee80211_radiotap_header_t *hdr);
+void
+radiotap_encode_bw_signaling(uint16 mask, struct wl_rxsts* rxsts, ieee80211_radiotap_header_t *hdr);
+#ifdef MONITOR_DNGL_CONV
+void radiotap_encode_alignpad(ieee80211_radiotap_header_t *hdr, uint16 pad_req);
+#endif
+
+static const uint8 brcm_oui[] =  {0x00, 0x10, 0x18};
+
+static void
+wl_rtapParseReset(radiotap_parse_t *rtap)
+{
+	rtap->idx = 0;		/* reset parse index */
+	rtap->offset = 0;	/* reset current field pointer */
+}
+
+static void*
+wl_rtapParseFindField(radiotap_parse_t *rtap, uint search_idx)
+{
+	uint idx;	/* first bit index to parse */
+	uint32 btmap;	/* presence bitmap */
+	uint offset, field_offset;
+	uint align, len;
+	void *ptr = NULL;
+
+	if (search_idx > IEEE80211_RADIOTAP_EXT)
+		return ptr;
+
+	if (search_idx < rtap->idx)
+		wl_rtapParseReset(rtap);
+
+	btmap = rtap->hdr->it_present;
+	idx = rtap->idx;
+	offset = rtap->offset;
+
+	/* loop through each field index until we get to the target idx */
+	while (idx <= search_idx) {
+		/* if field 'idx' is present, update the offset and check for a match */
+		if ((1 << idx) & btmap) {
+			/* if we hit a field for which we have no parse info
+			 * we need to just bail out
+			 */
+			if (rtap_parse_info[idx].align == 0)
+				break;
+
+			/* step past any alignment padding */
+			align = rtap_parse_info[idx].align;
+			len = rtap_parse_info[idx].len;
+
+			/* ROUNDUP */
+			field_offset = ((offset + (align - 1)) / align) * align;
+
+			/* if this field is not in the boulds of the header
+			 * just bail out
+			 */
+			if (field_offset + len > rtap->fields_len)
+				break;
+
+			/* did we find the field? */
+			if (idx == search_idx)
+				ptr = (uint8*)rtap->fields + field_offset;
+
+			/* step past this field */
+			offset = field_offset + len;
+		}
+
+		idx++;
+	}
+
+	rtap->idx = idx;
+	rtap->offset = offset;
+
+	return ptr;
+}
+
+ratespec_t
+wl_calcRspecFromRTap(uint8 *rtap_header)
+{
+	ratespec_t rspec = 0;
+	radiotap_parse_t rtap;
+	uint8 rate = 0;
+	uint8 flags = 0;
+	int flags_present = FALSE;
+	uint8 mcs = 0;
+	uint8 mcs_flags = 0;
+	uint8 mcs_known = 0;
+	int mcs_present = FALSE;
+	void *p;
+
+	wl_rtapParseInit(&rtap, rtap_header);
+
+	p = wl_rtapParseFindField(&rtap, IEEE80211_RADIOTAP_FLAGS);
+	if (p != NULL) {
+		flags_present = TRUE;
+		flags = ((uint8*)p)[0];
+	}
+
+	p = wl_rtapParseFindField(&rtap, IEEE80211_RADIOTAP_RATE);
+	if (p != NULL)
+		rate = ((uint8*)p)[0];
+
+	p = wl_rtapParseFindField(&rtap, IEEE80211_RADIOTAP_MCS);
+	if (p != NULL) {
+		mcs_present = TRUE;
+		mcs_known = ((uint8*)p)[0];
+		mcs_flags = ((uint8*)p)[1];
+		mcs = ((uint8*)p)[2];
+	}
+
+	if (rate != 0) {
+		/* validate the DSSS rates 1,2,5.5,11 */
+		if (rate == 2 || rate == 4 || rate == 11 || rate == 22) {
+			rspec = LEGACY_RSPEC(rate) | WL_RSPEC_OVERRIDE_RATE;
+			if (flags_present && (flags & IEEE80211_RADIOTAP_F_SHORTPRE)) {
+				rspec |= WL_RSPEC_OVERRIDE_MODE | WL_RSPEC_SHORT_PREAMBLE;
+			}
+		}
+	} else if (mcs_present) {
+		/* validate the MCS value */
+		if (mcs <= 23 || mcs == 32) {
+			uint32 override = 0;
+			if (mcs_known &
+			    (IEEE80211_RADIOTAP_MCS_HAVE_GI |
+			     IEEE80211_RADIOTAP_MCS_HAVE_FMT |
+			     IEEE80211_RADIOTAP_MCS_HAVE_FEC)) {
+				override = WL_RSPEC_OVERRIDE_MODE;
+			}
+
+			rspec = HT_RSPEC(mcs) | WL_RSPEC_OVERRIDE_RATE;
+
+			if ((mcs_known & IEEE80211_RADIOTAP_MCS_HAVE_GI) &&
+			    (mcs_flags & IEEE80211_RADIOTAP_MCS_SGI))
+				rspec |= WL_RSPEC_SGI;
+			if ((mcs_known & IEEE80211_RADIOTAP_MCS_HAVE_FMT) &&
+			    (mcs_flags & IEEE80211_RADIOTAP_MCS_FMT_GF))
+				rspec |= WL_RSPEC_SHORT_PREAMBLE;
+			if ((mcs_known & IEEE80211_RADIOTAP_MCS_HAVE_FEC) &&
+			    (mcs_flags & IEEE80211_RADIOTAP_MCS_FEC_LDPC))
+				rspec |= WL_RSPEC_LDPC;
+
+			rspec |= override;
+		}
+	}
+
+	return rspec;
+}
+
+bool
+wl_rtapFlags(uint8 *rtap_header, uint8* flags)
+{
+	radiotap_parse_t rtap;
+	void *p;
+
+	wl_rtapParseInit(&rtap, rtap_header);
+
+	p = wl_rtapParseFindField(&rtap, IEEE80211_RADIOTAP_FLAGS);
+	if (p != NULL) {
+		*flags = ((uint8*)p)[0];
+	}
+
+	return (p != NULL);
+}
+
+void
+wl_rtapParseInit(radiotap_parse_t *rtap, uint8 *rtap_header)
+{
+	uint rlen;
+	uint32 *present_word;
+	struct ieee80211_radiotap_header *hdr = (struct ieee80211_radiotap_header*)rtap_header;
+
+	bzero(rtap, sizeof(radiotap_parse_t));
+
+	rlen = hdr->it_len; /* total space in rtap_header */
+
+	/* If a precence word has the IEEE80211_RADIOTAP_EXT bit set it indicates
+	 * that there is another precence word.
+	 * Step over the presence words until we find the end of the list
+	 */
+	present_word = &hdr->it_present;
+	/* remaining length in header past it_present */
+	rlen -= sizeof(struct ieee80211_radiotap_header);
+
+	while ((*present_word & (1<<IEEE80211_RADIOTAP_EXT)) && rlen >= 4) {
+		present_word++;
+		rlen -= 4;	/* account for 4 bytes of present_word */
+	}
+
+	rtap->hdr = hdr;
+	rtap->fields = (uint8*)(present_word + 1);
+	rtap->fields_len = rlen;
+	wl_rtapParseReset(rtap);
+}
+
+uint
+wl_radiotap_rx(struct dot11_header *mac_header,	wl_rxsts_t *rxsts, bsd_header_rx_t *bsd_header)
+{
+	int channel_frequency;
+	uint32 channel_flags;
+	uint8 flags;
+	uint8 *cp;
+	uint pad_len;
+	uint32 field_map;
+	uint16 fc;
+	uint bsd_header_len;
+	uint16 ampdu_flags = 0;
+
+	fc = LTOH16(mac_header->fc);
+	pad_len = 3;
+	field_map = WL_RADIOTAP_PRESENT_BASIC;
+
+	if (CHSPEC_IS2G(rxsts->chanspec)) {
+		channel_flags = IEEE80211_CHAN_2GHZ | IEEE80211_CHAN_DYN;
+		channel_frequency = wf_channel2mhz(wf_chspec_ctlchan(rxsts->chanspec),
+			WF_CHAN_FACTOR_2_4_G);
+	} else if (CHSPEC_IS5G(rxsts->chanspec)) {
+		channel_flags = IEEE80211_CHAN_5GHZ | IEEE80211_CHAN_OFDM;
+		channel_frequency = wf_channel2mhz(wf_chspec_ctlchan(rxsts->chanspec),
+			WF_CHAN_FACTOR_5_G);
+	} else {
+		channel_flags = IEEE80211_CHAN_OFDM;
+		channel_frequency = wf_channel2mhz(wf_chspec_ctlchan(rxsts->chanspec),
+			WF_CHAN_FACTOR_6_G);
+	}
+
+	if ((rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_FIRST) ||
+		(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_SUB)) {
+
+		ampdu_flags = IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN;
+	}
+
+	flags = IEEE80211_RADIOTAP_F_FCS;
+
+	if (rxsts->preamble == WL_RXS_PREAMBLE_SHORT)
+		flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
+
+	if ((fc &  FC_WEP) == FC_WEP)
+		flags |= IEEE80211_RADIOTAP_F_WEP;
+
+	if ((fc & FC_MOREFRAG) == FC_MOREFRAG)
+		flags |= IEEE80211_RADIOTAP_F_FRAG;
+
+	if (rxsts->pkterror & WL_RXS_CRC_ERROR)
+		flags |= IEEE80211_RADIOTAP_F_BADFCS;
+
+	if (rxsts->encoding == WL_RXS_ENCODING_HT)
+		field_map = WL_RADIOTAP_PRESENT_HT;
+	else if (rxsts->encoding == WL_RXS_ENCODING_VHT)
+		field_map = WL_RADIOTAP_PRESENT_VHT;
+
+	bsd_header_len = sizeof(struct wl_radiotap_sna); /* start with sna size */
+	/* Test for signal/noise values and update length and field bitmap */
+	if (rxsts->signal == 0) {
+		field_map &= ~(1 << IEEE80211_RADIOTAP_DBM_ANTSIGNAL);
+		pad_len = (pad_len - 1);
+		bsd_header_len--;
+	}
+
+	if (rxsts->noise == 0) {
+		field_map &= ~(1 << IEEE80211_RADIOTAP_DBM_ANTNOISE);
+		pad_len = (pad_len - 1);
+		bsd_header_len--;
+	}
+
+	if (rxsts->encoding == WL_RXS_ENCODING_HT ||
+	    rxsts->encoding == WL_RXS_ENCODING_VHT) {
+		struct wl_radiotap_hdr *rtht = &bsd_header->hdr;
+		struct wl_radiotap_ht_tail *tail;
+
+		/*
+		 * Header length is complicated due to dynamic
+		 * presence of signal and noise fields
+		 * and padding for xchannel following
+		 * signal/noise/ant.
+		 * Start with length of wl_radiotap_ht plus
+		 * signal/noise/ant
+		 */
+		bsd_header_len += sizeof(struct wl_radiotap_hdr) + pad_len;
+		bsd_header_len += sizeof(struct wl_radiotap_xchan);
+		if (rxsts->nfrmtype == WL_RXS_NFRM_AMPDU_FIRST ||
+			rxsts->nfrmtype == WL_RXS_NFRM_AMPDU_SUB) {
+			bsd_header_len += sizeof(struct wl_radiotap_ampdu);
+		}
+		/* add the length of the tail end of the structure */
+		if (rxsts->encoding == WL_RXS_ENCODING_HT)
+			bsd_header_len += sizeof(struct wl_htmcs);
+		else if (rxsts->encoding == WL_RXS_ENCODING_VHT)
+			bsd_header_len += sizeof(struct wl_vhtmcs);
+		bzero((uint8 *)rtht, sizeof(*rtht));
+
+		rtht->ieee_radiotap.it_version = 0;
+		rtht->ieee_radiotap.it_pad = 0;
+		rtht->ieee_radiotap.it_len = (uint16)HTOL16(bsd_header_len);
+		rtht->ieee_radiotap.it_present = HTOL32(field_map);
+
+		rtht->tsft = HTOL64((uint64)rxsts->mactime);
+		rtht->flags = flags;
+		rtht->channel_freq = (uint16)HTOL16(channel_frequency);
+		rtht->channel_flags = (uint16)HTOL16(channel_flags);
+
+		cp = bsd_header->pad;
+		/* add in signal/noise/ant */
+		if (rxsts->signal != 0) {
+			*cp++ = (int8)rxsts->signal;
+			pad_len--;
+		}
+		if (rxsts->noise != 0) {
+			*cp++ = (int8)rxsts->noise;
+			pad_len--;
+		}
+		*cp++ = (int8)rxsts->antenna;
+		pad_len--;
+
+		tail = (struct wl_radiotap_ht_tail *)(bsd_header->ht);
+		/* Fill in XCHANNEL */
+		if (CHSPEC_IS40(rxsts->chanspec)) {
+			if (CHSPEC_SB_UPPER(rxsts->chanspec))
+				channel_flags |= IEEE80211_CHAN_HT40D;
+			else
+				channel_flags |= IEEE80211_CHAN_HT40U;
+		} else
+			channel_flags |= IEEE80211_CHAN_HT20;
+
+		tail->xc.xchannel_flags = HTOL32(channel_flags);
+		tail->xc.xchannel_freq = (uint16)HTOL16(channel_frequency);
+		tail->xc.xchannel_channel = wf_chspec_ctlchan(rxsts->chanspec);
+		tail->xc.xchannel_maxpower = (17*2);
+		/* fill in A-mpdu Status */
+		tail->ampdu.ref_num = mac_header->seq;
+		tail->ampdu.flags = ampdu_flags;
+		tail->ampdu.delimiter_crc = 0;
+		tail->ampdu.reserved = 0;
+
+		if (rxsts->encoding == WL_RXS_ENCODING_HT) {
+			tail->u.ht.mcs_index = rxsts->mcs;
+			tail->u.ht.mcs_known = (IEEE80211_RADIOTAP_MCS_HAVE_BW |
+			                        IEEE80211_RADIOTAP_MCS_HAVE_MCS |
+			                        IEEE80211_RADIOTAP_MCS_HAVE_GI |
+			                        IEEE80211_RADIOTAP_MCS_HAVE_FEC |
+			                        IEEE80211_RADIOTAP_MCS_HAVE_FMT);
+			tail->u.ht.mcs_flags = 0;
+
+			switch (rxsts->htflags & WL_RXS_HTF_BW_MASK) {
+			case WL_RXS_HTF_20L:
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_20L;
+				break;
+			case WL_RXS_HTF_20U:
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_20U;
+				break;
+			case WL_RXS_HTF_40:
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_40;
+				break;
+			default:
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_20;
+				break;
+			}
+
+			if (rxsts->htflags & WL_RXS_HTF_SGI)
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_SGI;
+			if (rxsts->preamble & WL_RXS_PREAMBLE_HT_GF)
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_FMT_GF;
+			if (rxsts->htflags & WL_RXS_HTF_LDPC)
+				tail->u.ht.mcs_flags |= IEEE80211_RADIOTAP_MCS_FEC_LDPC;
+		} else if (rxsts->encoding == WL_RXS_ENCODING_VHT) {
+			tail->u.vht.vht_known = (IEEE80211_RADIOTAP_VHT_HAVE_STBC |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_TXOP_PS |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_GI |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_SGI_NSYM_DA |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_LDPC_EXTRA |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_BF |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_BW |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_GID |
+			                         IEEE80211_RADIOTAP_VHT_HAVE_PAID);
+
+			tail->u.vht.vht_flags = (uint8)HTOL16(rxsts->vhtflags);
+
+			switch (rxsts->bw) {
+			case WL_RXS_VHT_BW_20:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20;
+				break;
+			case WL_RXS_VHT_BW_40:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_40;
+				break;
+			case WL_RXS_VHT_BW_20L:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20L;
+				break;
+			case WL_RXS_VHT_BW_20U:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20U;
+				break;
+			case WL_RXS_VHT_BW_80:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_80;
+				break;
+			case WL_RXS_VHT_BW_40L:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_40L;
+				break;
+			case WL_RXS_VHT_BW_40U:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_40U;
+				break;
+			case WL_RXS_VHT_BW_20LL:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20LL;
+				break;
+			case WL_RXS_VHT_BW_20LU:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20LU;
+				break;
+			case WL_RXS_VHT_BW_20UL:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20UL;
+				break;
+			case WL_RXS_VHT_BW_20UU:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20UU;
+				break;
+			default:
+				tail->u.vht.vht_bw = IEEE80211_RADIOTAP_VHT_BW_20;
+				break;
+			}
+
+			tail->u.vht.vht_mcs_nss[0] = (rxsts->mcs << 4) |
+				(rxsts->nss & IEEE80211_RADIOTAP_VHT_NSS);
+			tail->u.vht.vht_mcs_nss[1] = 0;
+			tail->u.vht.vht_mcs_nss[2] = 0;
+			tail->u.vht.vht_mcs_nss[3] = 0;
+
+			tail->u.vht.vht_coding = rxsts->coding;
+			tail->u.vht.vht_group_id = rxsts->gid;
+			tail->u.vht.vht_partial_aid = HTOL16(rxsts->aid);
+		}
+	} else {
+		struct wl_radiotap_hdr *rtl = &bsd_header->hdr;
+
+		/*
+		 * Header length is complicated due to dynamic presence of signal and noise fields
+		 * Start with length of wl_radiotap_legacy plus signal/noise/ant
+		 */
+		bsd_header_len = sizeof(struct wl_radiotap_hdr) + pad_len;
+		bzero((uint8 *)rtl, sizeof(*rtl));
+
+		rtl->ieee_radiotap.it_version = 0;
+		rtl->ieee_radiotap.it_pad = 0;
+		rtl->ieee_radiotap.it_len = (uint16)HTOL16(bsd_header_len);
+		rtl->ieee_radiotap.it_present = HTOL32(field_map);
+
+		rtl->tsft = HTOL64((uint64)rxsts->mactime);
+		rtl->flags = flags;
+		rtl->u.rate = (uint8)rxsts->datarate;
+		rtl->channel_freq = (uint16)HTOL16(channel_frequency);
+		rtl->channel_flags = (uint16)HTOL16(channel_flags);
+
+		/* add in signal/noise/ant */
+		cp = bsd_header->pad;
+		if (rxsts->signal != 0)
+			*cp++ = (int8)rxsts->signal;
+		if (rxsts->noise != 0)
+			*cp++ = (int8)rxsts->noise;
+		*cp++ = (int8)rxsts->antenna;
+	}
+	return bsd_header_len;
+}
+
+static int
+wl_radiotap_rx_channel_frequency(wl_rxsts_t *rxsts)
+{
+	if (CHSPEC_IS2G(rxsts->chanspec)) {
+		return wf_channel2mhz(wf_chspec_ctlchan(rxsts->chanspec),
+			WF_CHAN_FACTOR_2_4_G);
+	} else if (CHSPEC_IS5G(rxsts->chanspec)) {
+		return wf_channel2mhz(wf_chspec_ctlchan(rxsts->chanspec),
+			WF_CHAN_FACTOR_5_G);
+	} else {
+		return wf_channel2mhz(wf_chspec_ctlchan(rxsts->chanspec),
+			WF_CHAN_FACTOR_6_G);
+	}
+}
+
+static uint16
+wl_radiotap_rx_channel_flags(wl_rxsts_t *rxsts)
+{
+	if (CHSPEC_IS2G(rxsts->chanspec)) {
+		return (IEEE80211_CHAN_2GHZ | IEEE80211_CHAN_DYN);
+	} else if (CHSPEC_IS5G(rxsts->chanspec)) {
+		return (IEEE80211_CHAN_5GHZ | IEEE80211_CHAN_OFDM);
+	} else {
+		return (IEEE80211_CHAN_OFDM);
+	}
+}
+
+static uint8
+wl_radiotap_rx_flags(struct dot11_header *mac_header, wl_rxsts_t *rxsts)
+{
+	uint8 flags;
+	uint16 fc;
+
+	fc = ltoh16(mac_header->fc);
+
+	flags = IEEE80211_RADIOTAP_F_FCS;
+
+	if (rxsts->preamble == WL_RXS_PREAMBLE_SHORT)
+		flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
+
+	if (fc & FC_WEP)
+		flags |= IEEE80211_RADIOTAP_F_WEP;
+
+	if (fc & FC_MOREFRAG)
+		flags |= IEEE80211_RADIOTAP_F_FRAG;
+
+	return flags;
+}
+
+uint
+wl_radiotap_rx_legacy(struct dot11_header *mac_header,
+	wl_rxsts_t *rxsts, ieee80211_radiotap_header_t *rtap_hdr)
+{
+	int channel_frequency;
+	uint16 channel_flags;
+	uint8 flags;
+	uint16 rtap_len = LTOH16(rtap_hdr->it_len);
+	wl_radiotap_legacy_t *rtl = (wl_radiotap_legacy_t *)((uint8*)rtap_hdr + rtap_len);
+
+	rtap_len += sizeof(wl_radiotap_legacy_t);
+	rtap_hdr->it_len = HTOL16(rtap_len);
+	rtap_hdr->it_present |= HTOL32(WL_RADIOTAP_PRESENT_LEGACY);
+
+	channel_frequency = (uint16)wl_radiotap_rx_channel_frequency(rxsts);
+	channel_flags = wl_radiotap_rx_channel_flags(rxsts);
+	flags = wl_radiotap_rx_flags(mac_header, rxsts);
+
+	rtl->basic.tsft_l = HTOL32(rxsts->mactime);
+	rtl->basic.tsft_h = 0;
+	rtl->basic.flags = flags;
+	rtl->basic.rate = (uint8)rxsts->datarate;
+	rtl->basic.channel_freq = (uint16)HTOL16(channel_frequency);
+	rtl->basic.channel_flags = HTOL16(channel_flags);
+	rtl->basic.signal = (int8)rxsts->signal;
+	rtl->basic.noise = (int8)rxsts->noise;
+	rtl->basic.antenna = (int8)rxsts->antenna;
+
+	return 0;
+}
+
+uint
+wl_radiotap_rx_ht(struct dot11_header *mac_header,
+                  wl_rxsts_t *rxsts, ieee80211_radiotap_header_t *rtap_hdr)
+{
+	int channel_frequency;
+	uint16 channel_flags;
+	uint32 xchannel_flags;
+	uint8 flags;
+
+	uint16 rtap_len = LTOH16(rtap_hdr->it_len);
+	wl_radiotap_ht_t *rtht = (wl_radiotap_ht_t *)((uint8*)rtap_hdr + rtap_len);
+
+	rtap_len += sizeof(wl_radiotap_ht_t);
+	rtap_hdr->it_len = HTOL16(rtap_len);
+	rtap_hdr->it_present |= HTOL32(WL_RADIOTAP_PRESENT_HT);
+
+	channel_frequency = (uint16)wl_radiotap_rx_channel_frequency(rxsts);
+	channel_flags = wl_radiotap_rx_channel_flags(rxsts);
+	flags = wl_radiotap_rx_flags(mac_header, rxsts);
+
+	rtht->basic.tsft_l = HTOL32(rxsts->mactime);
+	rtht->basic.tsft_h = 0;
+	rtht->basic.flags = flags;
+	rtht->basic.channel_freq = (uint16)HTOL16(channel_frequency);
+	rtht->basic.channel_flags = HTOL16(channel_flags);
+	rtht->basic.signal = (int8)rxsts->signal;
+	rtht->basic.noise = (int8)rxsts->noise;
+	rtht->basic.antenna = (uint8)rxsts->antenna;
+
+	/* xchannel */
+	xchannel_flags = (uint32)channel_flags;
+	if (CHSPEC_IS40(rxsts->chanspec)) {
+		if (CHSPEC_SB_UPPER(rxsts->chanspec))
+			xchannel_flags |= IEEE80211_CHAN_HT40D;
+		else {
+			xchannel_flags |= IEEE80211_CHAN_HT40U;
+		}
+	} else {
+		xchannel_flags |= IEEE80211_CHAN_HT20;
+	}
+
+	rtht->xchannel_flags = HTOL32(xchannel_flags);
+	rtht->xchannel_freq = (uint16)HTOL16(channel_frequency);
+	rtht->xchannel_channel = wf_chspec_ctlchan(rxsts->chanspec);
+	rtht->xchannel_maxpower = (17*2);
+
+	/* add standard MCS */
+	rtht->mcs_known = (IEEE80211_RADIOTAP_MCS_HAVE_BW |
+		IEEE80211_RADIOTAP_MCS_HAVE_MCS |
+		IEEE80211_RADIOTAP_MCS_HAVE_GI |
+		IEEE80211_RADIOTAP_MCS_HAVE_FEC |
+		IEEE80211_RADIOTAP_MCS_HAVE_FMT);
+
+	rtht->mcs_flags = 0;
+	switch (rxsts->htflags & WL_RXS_HTF_BW_MASK) {
+		case WL_RXS_HTF_20L:
+			rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_20L;
+			break;
+		case WL_RXS_HTF_20U:
+			rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_20U;
+			break;
+		case WL_RXS_HTF_40:
+			rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_40;
+			break;
+		default:
+			rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_BW_20;
+	}
+
+	if (rxsts->htflags & WL_RXS_HTF_SGI) {
+		rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_SGI;
+	}
+	if (rxsts->preamble & WL_RXS_PREAMBLE_HT_GF) {
+		rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_FMT_GF;
+	}
+	if (rxsts->htflags & WL_RXS_HTF_LDPC) {
+		rtht->mcs_flags |= IEEE80211_RADIOTAP_MCS_FEC_LDPC;
+	}
+	rtht->mcs_index = rxsts->mcs;
+	rtht->ampdu_flags = 0;
+	rtht->ampdu_delim_crc = 0;
+
+	rtht->ampdu_ref_num = rxsts->ampdu_counter;
+
+	if (!(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_FIRST) &&
+		!(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_SUB)) {
+		rtht->ampdu_flags |= IEEE80211_RADIOTAP_AMPDU_IS_LAST;
+	} else {
+		rtht->ampdu_flags |= IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN;
+	}
+	return 0;
+}
+
+uint
+wl_radiotap_rx_vht(struct dot11_header *mac_header,
+                   wl_rxsts_t *rxsts, ieee80211_radiotap_header_t *rtap_hdr)
+{
+	int channel_frequency;
+	uint16 channel_flags;
+	uint8 flags;
+
+	uint16 rtap_len = LTOH16(rtap_hdr->it_len);
+	wl_radiotap_vht_t *rtvht = (wl_radiotap_vht_t *)((uint8*)rtap_hdr + rtap_len);
+
+	rtap_len += sizeof(wl_radiotap_vht_t);
+	rtap_hdr->it_len = HTOL16(rtap_len);
+	rtap_hdr->it_present |= HTOL32(WL_RADIOTAP_PRESENT_VHT);
+
+	channel_frequency = (uint16)wl_radiotap_rx_channel_frequency(rxsts);
+	channel_flags = wl_radiotap_rx_channel_flags(rxsts);
+	flags = wl_radiotap_rx_flags(mac_header, rxsts);
+
+	rtvht->basic.tsft_l = HTOL32(rxsts->mactime);
+	rtvht->basic.tsft_h = 0;
+	rtvht->basic.flags = flags;
+	rtvht->basic.channel_freq = (uint16)HTOL16(channel_frequency);
+	rtvht->basic.channel_flags = HTOL16(channel_flags);
+	rtvht->basic.signal = (int8)rxsts->signal;
+	rtvht->basic.noise = (int8)rxsts->noise;
+	rtvht->basic.antenna = (uint8)rxsts->antenna;
+
+	rtvht->vht_known = (IEEE80211_RADIOTAP_VHT_HAVE_STBC |
+		IEEE80211_RADIOTAP_VHT_HAVE_TXOP_PS |
+		IEEE80211_RADIOTAP_VHT_HAVE_GI |
+		IEEE80211_RADIOTAP_VHT_HAVE_SGI_NSYM_DA |
+		IEEE80211_RADIOTAP_VHT_HAVE_LDPC_EXTRA |
+		IEEE80211_RADIOTAP_VHT_HAVE_BF |
+		IEEE80211_RADIOTAP_VHT_HAVE_BW |
+		IEEE80211_RADIOTAP_VHT_HAVE_GID |
+		IEEE80211_RADIOTAP_VHT_HAVE_PAID);
+
+	STATIC_ASSERT(WL_RXS_VHTF_STBC ==
+		IEEE80211_RADIOTAP_VHT_STBC);
+	STATIC_ASSERT(WL_RXS_VHTF_TXOP_PS ==
+		IEEE80211_RADIOTAP_VHT_TXOP_PS);
+	STATIC_ASSERT(WL_RXS_VHTF_SGI ==
+		IEEE80211_RADIOTAP_VHT_SGI);
+	STATIC_ASSERT(WL_RXS_VHTF_SGI_NSYM_DA ==
+		IEEE80211_RADIOTAP_VHT_SGI_NSYM_DA);
+	STATIC_ASSERT(WL_RXS_VHTF_LDPC_EXTRA ==
+		IEEE80211_RADIOTAP_VHT_LDPC_EXTRA);
+	STATIC_ASSERT(WL_RXS_VHTF_BF ==
+		IEEE80211_RADIOTAP_VHT_BF);
+
+	rtvht->vht_flags = (uint8)HTOL16(rxsts->vhtflags);
+
+	STATIC_ASSERT(WL_RXS_VHT_BW_20 ==
+		IEEE80211_RADIOTAP_VHT_BW_20);
+	STATIC_ASSERT(WL_RXS_VHT_BW_40 ==
+		IEEE80211_RADIOTAP_VHT_BW_40);
+	STATIC_ASSERT(WL_RXS_VHT_BW_20L ==
+		IEEE80211_RADIOTAP_VHT_BW_20L);
+	STATIC_ASSERT(WL_RXS_VHT_BW_20U ==
+		IEEE80211_RADIOTAP_VHT_BW_20U);
+	STATIC_ASSERT(WL_RXS_VHT_BW_80 ==
+		IEEE80211_RADIOTAP_VHT_BW_80);
+	STATIC_ASSERT(WL_RXS_VHT_BW_40L ==
+		IEEE80211_RADIOTAP_VHT_BW_40L);
+	STATIC_ASSERT(WL_RXS_VHT_BW_40U ==
+		IEEE80211_RADIOTAP_VHT_BW_40U);
+	STATIC_ASSERT(WL_RXS_VHT_BW_20LL ==
+		IEEE80211_RADIOTAP_VHT_BW_20LL);
+	STATIC_ASSERT(WL_RXS_VHT_BW_20LU ==
+		IEEE80211_RADIOTAP_VHT_BW_20LU);
+	STATIC_ASSERT(WL_RXS_VHT_BW_20UL ==
+		IEEE80211_RADIOTAP_VHT_BW_20UL);
+	STATIC_ASSERT(WL_RXS_VHT_BW_20UU ==
+		IEEE80211_RADIOTAP_VHT_BW_20UU);
+
+	rtvht->vht_bw = rxsts->bw;
+
+	rtvht->vht_mcs_nss[0] = (rxsts->mcs << 4) |
+		(rxsts->nss & IEEE80211_RADIOTAP_VHT_NSS);
+	rtvht->vht_mcs_nss[1] = 0;
+	rtvht->vht_mcs_nss[2] = 0;
+	rtvht->vht_mcs_nss[3] = 0;
+
+	STATIC_ASSERT(WL_RXS_VHTF_CODING_LDCP ==
+		IEEE80211_RADIOTAP_VHT_CODING_LDPC);
+
+	rtvht->vht_coding = rxsts->coding;
+	rtvht->vht_group_id = rxsts->gid;
+	rtvht->vht_partial_aid = HTOL16(rxsts->aid);
+
+	rtvht->ampdu_flags = 0;
+	rtvht->ampdu_delim_crc = 0;
+	rtvht->ampdu_ref_num = HTOL32(rxsts->ampdu_counter);
+	if (!(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_FIRST) &&
+		!(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_SUB)) {
+		rtvht->ampdu_flags |= HTOL16(IEEE80211_RADIOTAP_AMPDU_IS_LAST);
+	} else {
+		rtvht->ampdu_flags |= HTOL16(IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN);
+	}
+
+	return 0;
+}
+
+/* Rx status to radiotap conversion of HE type */
+uint
+wl_radiotap_rx_he(struct dot11_header *mac_header, wl_rxsts_t *rxsts,
+		ieee80211_radiotap_header_t *rtap_hdr)
+{
+	int channel_frequency;
+	uint16 channel_flags;
+	uint8 flags;
+	uint16 rtap_len = LTOH16(rtap_hdr->it_len);
+	wl_radiotap_he_t *rthe = (wl_radiotap_he_t *)((uint8*)rtap_hdr + rtap_len);
+
+	rtap_len += sizeof(wl_radiotap_he_t);
+	rtap_hdr->it_len = HTOL16(rtap_len);
+	rtap_hdr->it_present |= HTOL32(WL_RADIOTAP_PRESENT_HE);
+
+	channel_frequency = (uint16)wl_radiotap_rx_channel_frequency(rxsts);
+	channel_flags = wl_radiotap_rx_channel_flags(rxsts);
+	flags = wl_radiotap_rx_flags(mac_header, rxsts);
+
+	rthe->basic.tsft_l = HTOL32(rxsts->mactime);
+	rthe->basic.tsft_h = 0;
+	rthe->basic.flags = flags;
+	rthe->basic.channel_freq = (uint16)HTOL16(channel_frequency);
+	rthe->basic.channel_flags = HTOL16(channel_flags);
+	rthe->basic.signal = (int8)rxsts->signal;
+	rthe->basic.noise = (int8)rxsts->noise;
+	rthe->basic.antenna = (uint8)rxsts->antenna;
+
+	rthe->ampdu_flags = 0;
+	rthe->ampdu_delim_crc = 0;
+	rthe->ampdu_ref_num = HTOL32(rxsts->ampdu_counter);
+	if (!(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_FIRST) &&
+		!(rxsts->nfrmtype & WL_RXS_NFRM_AMPDU_SUB)) {
+		rthe->ampdu_flags |= HTOL16(IEEE80211_RADIOTAP_AMPDU_IS_LAST);
+	} else {
+		rthe->ampdu_flags |= HTOL16(IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN);
+	}
+
+	rthe->data1 = HTOL16(rxsts->data1);
+	rthe->data2 = HTOL16(rxsts->data2);
+	rthe->data3 = HTOL16(rxsts->data3);
+	rthe->data4 = HTOL16(rxsts->data4);
+	rthe->data5 = HTOL16(rxsts->data5);
+	rthe->data6 = HTOL16(rxsts->data6);
+
+	return 0;
+}
+
+/* Rx status to radiotap conversion of EHT type */
+uint
+wl_radiotap_rx_eht(struct dot11_header *mac_header, wl_rxsts_t *rxsts,
+	ieee80211_radiotap_header_t *rtap_hdr)
+{
+	ASSERT(!"wl_radiotap_rx_eht: not implemented!");
+	return 0;
+}
+
+uint16
+wl_rxsts_to_rtap(monitor_pkt_rxsts_t *pkt_rxsts, void *payload,
+                 uint16 len, void* pout, uint16 pad_req)
+{
+	uint16 rtap_len = 0;
+	struct dot11_header* mac_header;
+	uint8* p = payload;
+	ieee80211_radiotap_header_t* rtap_hdr = (ieee80211_radiotap_header_t*)pout;
+	wl_rxsts_t* rxsts;
+
+	ASSERT(p && pkt_rxsts);
+	rxsts = pkt_rxsts->rxsts;
+	rtap_hdr->it_version = 0;
+	rtap_hdr->it_pad = 0;
+	rtap_hdr->it_len = HTOL16(sizeof(*rtap_hdr));
+	rtap_hdr->it_present = 0;
+	bitmap = 0;
+
+#ifdef MONITOR_DNGL_CONV
+	if (pad_req) {
+		radiotap_add_vendor_ns(rtap_hdr);
+	}
+#endif
+
+#ifdef BCM_MON_QDBM_RSSI
+	/* if per-core RSSI is present, add vendor element */
+	if (pkt_rxsts->corenum != 0) {
+		radiotap_add_vendor_ns(rtap_hdr);
+	}
+#endif
+	mac_header = (struct dot11_header *)(p);
+
+	if (rxsts->encoding == WL_RXS_ENCODING_EHT) {
+		wl_radiotap_rx_eht(mac_header, rxsts, rtap_hdr);
+	} else if (rxsts->encoding == WL_RXS_ENCODING_HE) {
+		wl_radiotap_rx_he(mac_header, rxsts, rtap_hdr);
+	} else if (rxsts->encoding == WL_RXS_ENCODING_VHT) {
+		wl_radiotap_rx_vht(mac_header, rxsts, rtap_hdr);
+	} else if (rxsts->encoding == WL_RXS_ENCODING_HT) {
+		wl_radiotap_rx_ht(mac_header, rxsts, rtap_hdr);
+	} else {
+		uint16 mask = ltoh16(mac_header->fc) & FC_KIND_MASK;
+		if (mask == FC_RTS || mask == FC_CTS) {
+			radiotap_add_vendor_ns(rtap_hdr);
+		}
+		wl_radiotap_rx_legacy(mac_header, rxsts, rtap_hdr);
+		if (mask == FC_RTS || mask == FC_CTS) {
+			radiotap_encode_bw_signaling(mask, rxsts, rtap_hdr);
+		}
+	}
+#ifdef BCM_MON_QDBM_RSSI
+	/* if per-core RSSI is present, add vendor element */
+	if (pkt_rxsts->corenum != 0) {
+		radiotap_encode_multi_rssi(pkt_rxsts, rtap_hdr);
+	}
+#endif
+#ifdef MONITOR_DNGL_CONV
+	if (pad_req) {
+		radiotap_encode_alignpad(rtap_hdr, pad_req);
+	}
+#endif
+	rtap_len = LTOH16(rtap_hdr->it_len);
+	len += rtap_len;
+
+#ifndef MONITOR_DNGL_CONV
+	if (len > MAX_MON_PKT_SIZE) {
+		return 0;
+	}
+	/* copy payload */
+	if (!(rxsts->nfrmtype & WL_RXS_NFRM_AMSDU_FIRST) &&
+			!(rxsts->nfrmtype & WL_RXS_NFRM_AMSDU_SUB)) {
+		memcpy((uint8*)pout + rtap_len, (uint8*)p, len - rtap_len);
+	}
+#endif
+#ifdef MONITOR_DNGL_CONV
+	return rtap_len;
+#else
+	return len;
+#endif
+}
+
+#ifdef BCM_MON_QDBM_RSSI
+void
+radiotap_encode_multi_rssi(monitor_pkt_rxsts_t* rxsts, ieee80211_radiotap_header_t *hdr)
+{
+	uint16 cur_len = LTOH16(hdr->it_len);
+	uint16 len = ROUNDUP(1 + rxsts->corenum * sizeof(monitor_pkt_rssi_t), 4);
+	int i = 0;
+	uint8 *vend_p = (uint8 *)hdr + cur_len;
+	radiotap_vendor_ns_t *vendor_ns = (radiotap_vendor_ns_t*)vend_p;
+	memcpy(vendor_ns->vend_oui, brcm_oui, sizeof(vendor_ns->vend_oui));
+	vendor_ns->sns = 1;
+	vendor_ns->skip_len = HTOL16(len);
+	vend_p += sizeof(*vendor_ns);
+	vend_p[0] = rxsts->corenum;
+	for (i = 0; i < rxsts->corenum; i++) {
+	  vend_p[2*i + 1] = rxsts->rxpwr[i].dBm;
+	  vend_p[2*i + 2] = rxsts->rxpwr[i].decidBm;
+	}
+	hdr->it_len = HTOL16(cur_len + sizeof(radiotap_vendor_ns_t) + len);
+}
+#endif /* BCM_CORE_RSSI */
+
+#ifdef MONITOR_DNGL_CONV
+#define AILIGN_4BYTES	(4u)
+void
+radiotap_encode_alignpad(ieee80211_radiotap_header_t *hdr, uint16 pad_req)
+{
+	uint16 cur_len = LTOH16(hdr->it_len);
+	uint8 *vend_p = (uint8 *)hdr + cur_len;
+	radiotap_vendor_ns_t *vendor_ns = (radiotap_vendor_ns_t*)vend_p;
+	uint16 len;
+	uint16 align_pad = 0;
+
+	memcpy(vendor_ns->vend_oui, brcm_oui, sizeof(vendor_ns->vend_oui));
+	vendor_ns->sns = WL_RADIOTAP_BRCM_PAD_SNS;
+	len = cur_len + sizeof(radiotap_vendor_ns_t);
+	if (len % AILIGN_4BYTES	!= 0) {
+		align_pad = (AILIGN_4BYTES - (len % AILIGN_4BYTES));
+	}
+	hdr->it_len = HTOL16(len + pad_req + align_pad);
+	vendor_ns->skip_len = HTOL16(pad_req + align_pad);
+}
+#endif /* MONITOR_DNGL_CONV */
+
+void
+radiotap_encode_bw_signaling(uint16 mask,
+                             struct wl_rxsts* rxsts, ieee80211_radiotap_header_t *hdr)
+{
+	uint16 cur_len = LTOH16(hdr->it_len);
+	uint8 *vend_p = (uint8 *)hdr + cur_len;
+	radiotap_vendor_ns_t *vendor_ns = (radiotap_vendor_ns_t  *)vend_p;
+	wl_radiotap_nonht_vht_t* nonht_vht;
+
+	memcpy(vendor_ns->vend_oui, brcm_oui, sizeof(vendor_ns->vend_oui));
+	vendor_ns->sns = 0;
+	vendor_ns->skip_len = sizeof(wl_radiotap_nonht_vht_t);
+	nonht_vht = (wl_radiotap_nonht_vht_t *)(vend_p + sizeof(*vendor_ns));
+
+	/* VHT b/w signalling */
+	bzero((uint8 *)nonht_vht, sizeof(wl_radiotap_nonht_vht_t));
+	nonht_vht->len = WL_RADIOTAP_NONHT_VHT_LEN;
+	nonht_vht->flags |= WL_RADIOTAP_F_NONHT_VHT_BW;
+	nonht_vht->bw = (uint8)rxsts->bw_nonht;
+
+	if (mask == FC_RTS) {
+		if (rxsts->vhtflags & WL_RXS_VHTF_DYN_BW_NONHT) {
+			nonht_vht->flags |= WL_RADIOTAP_F_NONHT_VHT_DYN_BW;
+		}
+	}
+	hdr->it_len = HTOL16(cur_len + sizeof(radiotap_vendor_ns_t) +
+	                     sizeof(wl_radiotap_nonht_vht_t));
+}
+
+void
+radiotap_add_vendor_ns(ieee80211_radiotap_header_t *hdr)
+{
+
+	uint32 * it_present = &hdr->it_present;
+	uint16 len = LTOH16(hdr->it_len);
+
+	/* if the last bitmap has a vendor ns, add a new one */
+	if (it_present[bitmap] & (1 << IEEE80211_RADIOTAP_VENDOR_NAMESPACE)) {
+		it_present[bitmap] |= 1 << IEEE80211_RADIOTAP_EXT;
+		bitmap++;
+		/* align to 8 bytes */
+		if (bitmap%2) {
+			hdr->it_len = HTOL16(len + 8);
+		}
+		it_present[bitmap] = 1 << IEEE80211_RADIOTAP_VENDOR_NAMESPACE;
+	} else {
+		it_present[bitmap] |= 1 << IEEE80211_RADIOTAP_VENDOR_NAMESPACE;
+	}
+}
diff --git a/drivers/net/wireless/ap12275/bcmxtlv.c b/drivers/net/wireless/ap12275/bcmxtlv.c
new file mode 100644
index 000000000000..33d02ae6a3c8
--- /dev/null
+++ b/drivers/net/wireless/ap12275/bcmxtlv.c
@@ -0,0 +1,644 @@
+/*
+ * Driver O/S-independent utility routines
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <typedefs.h>
+#include <bcmdefs.h>
+
+#ifdef BCMDRIVER
+#include <osl.h>
+#else /* !BCMDRIVER */
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#ifndef ASSERT
+#define ASSERT(exp)
+#endif
+#endif /* !BCMDRIVER */
+
+#include <bcmtlv.h>
+#include <bcmendian.h>
+#include <bcmutils.h>
+
+int
+BCMPOSTTRAPFN(bcm_xtlv_hdr_size)(bcm_xtlv_opts_t opts)
+{
+	int len = (int)OFFSETOF(bcm_xtlv_t, data); /* nominal */
+	if (opts & BCM_XTLV_OPTION_LENU8) --len;
+	if (opts & BCM_XTLV_OPTION_IDU8) --len;
+
+	return len;
+}
+
+bool
+bcm_valid_xtlv(const bcm_xtlv_t *elt, int buf_len, bcm_xtlv_opts_t opts)
+{
+	return elt != NULL &&
+	        buf_len >= bcm_xtlv_hdr_size(opts) &&
+		buf_len  >= bcm_xtlv_size(elt, opts);
+}
+
+int
+BCMPOSTTRAPFN(bcm_xtlv_size_for_data)(int dlen, bcm_xtlv_opts_t opts)
+{
+	int hsz;
+
+	hsz = bcm_xtlv_hdr_size(opts);
+	return ((opts & BCM_XTLV_OPTION_ALIGN32) ? ALIGN_SIZE(dlen + hsz, 4)
+		: (dlen + hsz));
+}
+
+int
+bcm_xtlv_size(const bcm_xtlv_t *elt, bcm_xtlv_opts_t opts)
+{
+	int size;	/* size including header, data, and any pad */
+	int len;	/* length wthout padding */
+
+	len = BCM_XTLV_LEN_EX(elt, opts);
+	size = bcm_xtlv_size_for_data(len, opts);
+	return size;
+}
+
+int
+bcm_xtlv_len(const bcm_xtlv_t *elt, bcm_xtlv_opts_t opts)
+{
+	const uint8 *lenp;
+	int len;
+
+	lenp = (const uint8 *)elt + OFFSETOF(bcm_xtlv_t, len); /* nominal */
+	if (opts & BCM_XTLV_OPTION_IDU8) {
+		--lenp;
+	}
+
+	if (opts & BCM_XTLV_OPTION_LENU8) {
+		len = *lenp;
+	} else if (opts & BCM_XTLV_OPTION_LENBE) {
+		len = (uint32)hton16(elt->len);
+	} else {
+		len = ltoh16_ua(lenp);
+	}
+
+	return len;
+}
+
+int
+bcm_xtlv_id(const bcm_xtlv_t *elt, bcm_xtlv_opts_t opts)
+{
+	int id = 0;
+	if (opts & BCM_XTLV_OPTION_IDU8) {
+		id =  *(const uint8 *)elt;
+	} else if (opts & BCM_XTLV_OPTION_IDBE) {
+		id = (uint32)hton16(elt->id);
+	} else {
+		id = ltoh16_ua((const uint8 *)elt);
+	}
+
+	return id;
+}
+
+bcm_xtlv_t *
+bcm_next_xtlv(const bcm_xtlv_t *elt, int *buflen, bcm_xtlv_opts_t opts)
+{
+	uint sz;
+
+	COV_TAINTED_DATA_SINK(buflen);
+	COV_NEG_SINK(buflen);
+
+	/* validate current elt */
+	if (!bcm_valid_xtlv(elt, *buflen, opts))
+		return NULL;
+
+	/* advance to next elt */
+	sz = BCM_XTLV_SIZE_EX(elt, opts);
+	elt = (const bcm_xtlv_t*)((const uint8 *)elt + sz);
+
+#if defined(__COVERITY__)
+	/* The 'sz' value is tainted in Coverity because it is read from the tainted data pointed
+	 * to by 'elt'.  However, bcm_valid_xtlv() verifies that the elt pointer is a valid element,
+	 * so its size, sz = BCM_XTLV_SIZE_EX(elt, opts), is in the bounds of the buffer.
+	 * Clearing the tainted attribute of 'sz' for Coverity.
+	 */
+	__coverity_tainted_data_sanitize__(sz);
+	if (sz > *buflen) {
+		return NULL;
+	}
+#endif /* __COVERITY__ */
+
+	*buflen -= sz;
+
+	/* validate next elt */
+	if (!bcm_valid_xtlv(elt, *buflen, opts))
+		return NULL;
+
+	COV_TAINTED_DATA_ARG(elt);
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST()
+	return (bcm_xtlv_t *)(elt);
+	GCC_DIAGNOSTIC_POP()
+}
+
+int
+bcm_xtlv_buf_init(bcm_xtlvbuf_t *tlv_buf, uint8 *buf, uint16 len, bcm_xtlv_opts_t opts)
+{
+	if (!tlv_buf || !buf || !len)
+		return BCME_BADARG;
+
+	tlv_buf->opts = opts;
+	tlv_buf->size = len;
+	tlv_buf->head = buf;
+	tlv_buf->buf  = buf;
+	return BCME_OK;
+}
+
+uint16
+bcm_xtlv_buf_len(bcm_xtlvbuf_t *tbuf)
+{
+	uint16 len;
+
+	if (tbuf)
+		len = (uint16)(tbuf->buf - tbuf->head);
+	else
+		len = 0;
+
+	return len;
+}
+
+uint16
+bcm_xtlv_buf_rlen(bcm_xtlvbuf_t *tbuf)
+{
+	uint16 rlen;
+	if (tbuf)
+		rlen = tbuf->size - bcm_xtlv_buf_len(tbuf);
+	else
+		rlen = 0;
+
+	return rlen;
+}
+
+uint8 *
+bcm_xtlv_buf(bcm_xtlvbuf_t *tbuf)
+{
+	return tbuf ? tbuf->buf : NULL;
+}
+
+uint8 *
+bcm_xtlv_head(bcm_xtlvbuf_t *tbuf)
+{
+	return tbuf ? tbuf->head : NULL;
+}
+
+void
+BCMPOSTTRAPFN(bcm_xtlv_pack_xtlv)(bcm_xtlv_t *xtlv, uint16 type, uint16 len, const uint8 *data,
+	bcm_xtlv_opts_t opts)
+{
+	uint8 *data_buf;
+	bcm_xtlv_opts_t mask = BCM_XTLV_OPTION_IDU8 | BCM_XTLV_OPTION_LENU8;
+
+	if (!(opts & mask)) {		/* default */
+		uint8 *idp = (uint8 *)xtlv;
+		uint8 *lenp = idp + sizeof(xtlv->id);
+		htol16_ua_store(type, idp);
+		htol16_ua_store(len, lenp);
+		data_buf = lenp + sizeof(uint16);
+	} else if ((opts & mask) == mask) { /* u8 id and u8 len */
+		uint8 *idp = (uint8 *)xtlv;
+		uint8 *lenp = idp + 1;
+		*idp = (uint8)type;
+		*lenp = (uint8)len;
+		data_buf = lenp + sizeof(uint8);
+	} else if (opts & BCM_XTLV_OPTION_IDU8) { /* u8 id, u16 len */
+		uint8 *idp = (uint8 *)xtlv;
+		uint8 *lenp = idp + 1;
+		*idp = (uint8)type;
+		htol16_ua_store(len, lenp);
+		data_buf = lenp + sizeof(uint16);
+	} else if (opts & BCM_XTLV_OPTION_LENU8) { /* u16 id, u8 len */
+		uint8 *idp = (uint8 *)xtlv;
+		uint8 *lenp = idp + sizeof(uint16);
+		htol16_ua_store(type, idp);
+		*lenp = (uint8)len;
+		data_buf = lenp + sizeof(uint8);
+	} else {
+		ASSERT(!"Unexpected xtlv option");
+		return;
+	}
+
+	if (opts & BCM_XTLV_OPTION_LENU8) {
+		ASSERT(len <= 0x00ff);
+		len &= 0xff;
+	}
+
+	if (data != NULL) {
+		memcpy(data_buf, data, len);
+	}
+}
+
+/* xtlv header is always packed in LE order */
+void
+bcm_xtlv_unpack_xtlv(const bcm_xtlv_t *xtlv, uint16 *type, uint16 *len,
+	const uint8 **data, bcm_xtlv_opts_t opts)
+{
+	if (type)
+		*type = (uint16)bcm_xtlv_id(xtlv, opts);
+	if (len)
+		*len = (uint16)bcm_xtlv_len(xtlv, opts);
+	if (data)
+		*data = (const uint8 *)xtlv + BCM_XTLV_HDR_SIZE_EX(opts);
+}
+
+int
+bcm_xtlv_put_data(bcm_xtlvbuf_t *tbuf, uint16 type, const uint8 *data, int n)
+{
+	bcm_xtlv_t *xtlv;
+	int size;
+
+	if (tbuf == NULL)
+		return BCME_BADARG;
+
+	size = bcm_xtlv_size_for_data(n, tbuf->opts);
+	if (bcm_xtlv_buf_rlen(tbuf) < size)
+		return BCME_NOMEM;
+
+	xtlv = (bcm_xtlv_t *)bcm_xtlv_buf(tbuf);
+	bcm_xtlv_pack_xtlv(xtlv, type, (uint16)n, data, tbuf->opts);
+	tbuf->buf += size; /* note: data may be NULL, reserves space */
+	return BCME_OK;
+}
+
+static int
+bcm_xtlv_put_int(bcm_xtlvbuf_t *tbuf, uint16 type, const uint8 *data, int n, int int_sz)
+{
+	bcm_xtlv_t *xtlv;
+	int xtlv_len;
+	uint8 *xtlv_data;
+	int err = BCME_OK;
+
+	if (tbuf == NULL) {
+		err = BCME_BADARG;
+		goto done;
+	}
+
+	xtlv = (bcm_xtlv_t *)bcm_xtlv_buf(tbuf);
+
+	/* put type and length in xtlv and reserve data space */
+	xtlv_len = n * int_sz;
+	err = bcm_xtlv_put_data(tbuf, type, NULL, xtlv_len);
+	if (err != BCME_OK)
+		goto done;
+
+	xtlv_data = (uint8 *)xtlv + bcm_xtlv_hdr_size(tbuf->opts);
+
+	/* write data w/ little-endianness into buffer - single loop, aligned access */
+	for (; n != 0; --n, xtlv_data += int_sz, data += int_sz) {
+		switch (int_sz) {
+		case sizeof(uint8):
+			break;
+		case sizeof(uint16):
+			{
+				uint16 v =  load16_ua(data);
+				htol16_ua_store(v, xtlv_data);
+				break;
+			}
+		case sizeof(uint32):
+			{
+				uint32 v = load32_ua(data);
+				htol32_ua_store(v, xtlv_data);
+				break;
+			}
+		case sizeof(uint64):
+			{
+				uint64 v = load64_ua(data);
+				htol64_ua_store(v, xtlv_data);
+				break;
+			}
+		default:
+			err = BCME_UNSUPPORTED;
+			goto done;
+		}
+	}
+
+done:
+	return err;
+}
+
+int
+bcm_xtlv_put16(bcm_xtlvbuf_t *tbuf, uint16 type, const uint16 *data, int n)
+{
+	return bcm_xtlv_put_int(tbuf, type, (const uint8 *)data, n, sizeof(uint16));
+}
+
+int
+bcm_xtlv_put32(bcm_xtlvbuf_t *tbuf, uint16 type, const uint32 *data, int n)
+{
+	return bcm_xtlv_put_int(tbuf, type, (const uint8 *)data, n, sizeof(uint32));
+}
+
+int
+bcm_xtlv_put64(bcm_xtlvbuf_t *tbuf, uint16 type, const uint64 *data, int n)
+{
+	return bcm_xtlv_put_int(tbuf, type, (const uint8 *)data, n, sizeof(uint64));
+}
+
+/*
+ *  upacks xtlv record from buf checks the type
+ *  copies data to callers buffer
+ *  advances tlv pointer to next record
+ *  caller's resposible for dst space check
+ */
+int
+bcm_unpack_xtlv_entry(const uint8 **tlv_buf, uint16 xpct_type, uint16 xpct_len,
+	uint8 *dst_data, bcm_xtlv_opts_t opts)
+{
+	const bcm_xtlv_t *ptlv = (const bcm_xtlv_t *)*tlv_buf;
+	uint16 len;
+	uint16 type;
+	const uint8 *data;
+
+	ASSERT(ptlv);
+
+	bcm_xtlv_unpack_xtlv(ptlv, &type, &len, &data, opts);
+	if (len) {
+		if ((type != xpct_type) || (len > xpct_len))
+			return BCME_BADARG;
+		if (dst_data && data)
+			memcpy(dst_data, data, len); /* copy data to dst */
+	}
+
+	*tlv_buf += BCM_XTLV_SIZE_EX(ptlv, opts);
+	return BCME_OK;
+}
+
+/*
+ *  packs user data into tlv record and advances tlv pointer to next xtlv slot
+ *  buflen is used for tlv_buf space check
+ */
+int
+bcm_pack_xtlv_entry(uint8 **tlv_buf, uint16 *buflen, uint16 type, uint16 len,
+	const uint8 *src_data, bcm_xtlv_opts_t opts)
+{
+	bcm_xtlv_t *ptlv = (bcm_xtlv_t *)*tlv_buf;
+	int size;
+
+	ASSERT(ptlv);
+
+	size = bcm_xtlv_size_for_data(len, opts);
+
+	/* copy data from tlv buffer to dst provided by user */
+	if (size > *buflen) {
+		return BCME_BADLEN;
+	}
+
+	bcm_xtlv_pack_xtlv(ptlv, type, len, src_data, opts);
+
+	/* advance callers pointer to tlv buff */
+	*tlv_buf = (uint8*)(*tlv_buf) + size;
+	/* decrement the len */
+	*buflen -= (uint16)size;
+	return BCME_OK;
+}
+
+/*
+ *  unpack all xtlv records from the issue a callback
+ *  to set function one call per found tlv record
+ */
+int
+bcm_unpack_xtlv_buf(void *ctx, const uint8 *tlv_buf, uint16 buflen, bcm_xtlv_opts_t opts,
+	bcm_xtlv_unpack_cbfn_t *cbfn)
+{
+	int res = BCME_OK;
+	const bcm_xtlv_t *ptlv;
+	uint16 len;
+	uint16 type;
+	const uint8 *data;
+	uint32 sbuflen = buflen;
+	uint32 hdr_size;	/* size of tlv header */
+	uint32 size;		/* size of tlv including header + data */
+
+	ASSERT(!buflen || tlv_buf);
+	ASSERT(!buflen || cbfn);
+
+	hdr_size = BCM_XTLV_HDR_SIZE_EX(opts);
+	while (sbuflen >= hdr_size) {
+		ptlv = (const bcm_xtlv_t *)tlv_buf;
+
+		bcm_xtlv_unpack_xtlv(ptlv, &type, &len, &data, opts);
+		size = bcm_xtlv_size_for_data(len, opts);
+
+		/* check for buffer overrun */
+		if (sbuflen < size) {
+			break;
+		}
+		sbuflen -= size;
+
+		if ((res = cbfn(ctx, data, type, len)) != BCME_OK) {
+			break;
+		}
+		tlv_buf += size;
+	}
+	return res;
+}
+
+int
+bcm_pack_xtlv_buf(void *ctx, uint8 *tlv_buf, uint16 buflen, bcm_xtlv_opts_t opts,
+	bcm_pack_xtlv_next_info_cbfn_t get_next, bcm_pack_xtlv_pack_next_cbfn_t pack_next,
+	int *outlen)
+{
+	int res = BCME_OK;
+	uint16 tlv_id;
+	uint16 tlv_len;
+	uint8 *startp;
+	uint8 *endp;
+	uint8 *buf;
+	bool more;
+	int size;
+	int hdr_size;
+
+	ASSERT(get_next && pack_next);
+
+	buf = tlv_buf;
+	startp = buf;
+	endp = (uint8 *)buf + buflen;
+	more = TRUE;
+	hdr_size = BCM_XTLV_HDR_SIZE_EX(opts);
+
+	while (more && (buf < endp)) {
+		more = get_next(ctx, &tlv_id, &tlv_len);
+		size = bcm_xtlv_size_for_data(tlv_len, opts);
+		if ((buf + size) > endp) {
+			res = BCME_BUFTOOSHORT;
+			goto done;
+		}
+
+		bcm_xtlv_pack_xtlv((bcm_xtlv_t *)buf, tlv_id, tlv_len, NULL, opts);
+		pack_next(ctx, tlv_id, tlv_len, buf + hdr_size);
+		buf += size;
+	}
+
+	if (more)
+		res = BCME_BUFTOOSHORT;
+
+done:
+	if (outlen) {
+		*outlen = (int)(buf - startp);
+	}
+	return res;
+}
+
+/*
+ *  pack xtlv buffer from memory according to xtlv_desc_t
+ */
+int
+bcm_pack_xtlv_buf_from_mem(uint8 **tlv_buf, uint16 *buflen, const xtlv_desc_t *items,
+	bcm_xtlv_opts_t opts)
+{
+	int res = BCME_OK;
+	uint8 *ptlv = *tlv_buf;
+
+	while (items->type != 0) {
+		if (items->len && items->ptr) {
+			res = bcm_pack_xtlv_entry(&ptlv, buflen, items->type,
+				items->len, items->ptr, opts);
+			if (res != BCME_OK)
+				break;
+		}
+		items++;
+	}
+
+	*tlv_buf = ptlv; /* update the external pointer */
+	return res;
+}
+
+/*
+ *  unpack xtlv buffer to memory according to xtlv_desc_t
+ *
+ */
+int
+bcm_unpack_xtlv_buf_to_mem(const uint8 *tlv_buf, int *buflen, xtlv_desc_t *items,
+	bcm_xtlv_opts_t opts)
+{
+	int res = BCME_OK;
+	const bcm_xtlv_t *elt;
+
+	elt =  bcm_valid_xtlv((const bcm_xtlv_t *)tlv_buf, *buflen, opts) ?
+		(const bcm_xtlv_t *)tlv_buf : NULL;
+	if (!elt || !items) {
+		res = BCME_BADARG;
+		return res;
+	}
+
+	for (; elt != NULL && res == BCME_OK; elt = bcm_next_xtlv(elt, buflen, opts)) {
+		/*  find matches in desc_t items  */
+		xtlv_desc_t *dst_desc = items;
+		uint16 len, type;
+		const uint8 *data;
+
+		bcm_xtlv_unpack_xtlv(elt, &type, &len, &data, opts);
+		while (dst_desc->type != 0) {
+			if (type == dst_desc->type) {
+				if (len != dst_desc->len) {
+					res = BCME_BADLEN;
+				} else {
+					memcpy(dst_desc->ptr, data, len);
+				}
+				break;
+			}
+			dst_desc++;
+		}
+	}
+
+	if (res == BCME_OK && *buflen != 0)		/* this does not look right */
+		res =  BCME_BUFTOOSHORT;
+
+	return res;
+}
+
+/*
+ * return data pointer of a given ID from xtlv buffer.
+ * If the specified xTLV ID is found, on return *datalen will contain
+ * the the data length of the xTLV ID.
+ */
+const uint8*
+bcm_get_data_from_xtlv_buf(const uint8 *tlv_buf, uint16 buflen, uint16 id,
+	uint16 *datalen, bcm_xtlv_opts_t opts)
+{
+	const uint8 *retptr = NULL;
+	uint16 type, len;
+	int size;
+	const bcm_xtlv_t *ptlv;
+	int sbuflen = buflen;
+	const uint8 *data;
+	int hdr_size;
+
+	COV_TAINTED_DATA_SINK(buflen);
+	COV_NEG_SINK(buflen);
+
+	hdr_size = BCM_XTLV_HDR_SIZE_EX(opts);
+
+	/* Init the datalength */
+	if (datalen) {
+		*datalen = 0;
+	}
+	while (sbuflen >= hdr_size) {
+		ptlv = (const bcm_xtlv_t *)tlv_buf;
+		bcm_xtlv_unpack_xtlv(ptlv, &type, &len, &data, opts);
+
+		size = bcm_xtlv_size_for_data(len, opts);
+		sbuflen -= size;
+		if (sbuflen < 0) /* buffer overrun? */
+			break;
+
+		if (id == type) {
+			retptr = data;
+			if (datalen)
+				*datalen = len;
+			break;
+		}
+
+		tlv_buf += size;
+	}
+
+	COV_TAINTED_DATA_ARG(retptr);
+
+	return retptr;
+}
+
+bcm_xtlv_t*
+bcm_xtlv_bcopy(const bcm_xtlv_t *src, bcm_xtlv_t *dst,
+	int src_buf_len, int dst_buf_len, bcm_xtlv_opts_t opts)
+{
+	bcm_xtlv_t *dst_next = NULL;
+	src =  (src && bcm_valid_xtlv(src, src_buf_len, opts)) ? src : NULL;
+	if (src && dst) {
+		uint16 type;
+		uint16 len;
+		const uint8 *data;
+		int size;
+		bcm_xtlv_unpack_xtlv(src, &type, &len, &data, opts);
+		size = bcm_xtlv_size_for_data(len, opts);
+		if (size <= dst_buf_len) {
+			bcm_xtlv_pack_xtlv(dst, type, len, data, opts);
+			dst_next = (bcm_xtlv_t *)((uint8 *)dst + size);
+		}
+	}
+
+	return dst_next;
+}
diff --git a/drivers/net/wireless/ap12275/dbus.c b/drivers/net/wireless/ap12275/dbus.c
new file mode 100644
index 000000000000..260b93788956
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dbus.c
@@ -0,0 +1,3516 @@
+/** @file dbus.c
+ *
+ * Hides details of USB / SDIO / SPI interfaces and OS details. It is intended to shield details and
+ * provide the caller with one common bus interface for all dongle devices. In practice, it is only
+ * used for USB interfaces. DBUS is not a protocol, but an abstraction layer.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#include <linux/usb.h>
+#include "osl.h"
+#include "dbus.h"
+#include <bcmutils.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_proto.h>
+#ifdef PROP_TXSTATUS /* a form of flow control between host and dongle */
+#include <dhd_wlfc.h>
+#endif
+#include <dhd_config.h>
+#ifdef WL_CFG80211
+#include <wl_cfg80211.h>
+#include <wl_cfgp2p.h>
+#endif
+
+#include <bcmdevs_legacy.h>
+#if defined(BCM_DNGL_EMBEDIMAGE)
+#include <bcmsrom_fmt.h>
+#include <trxhdr.h>
+#include <usbrdl.h>
+#include <bcmendian.h>
+#include <zutil.h>
+#include <sbpcmcia.h>
+#include <bcmnvram.h>
+#include <bcmdevs.h>
+#elif defined(BCM_REQUEST_FW)
+#include <bcmsrom_fmt.h>
+#include <trxhdr.h>
+#include <usbrdl.h>
+#include <bcmendian.h>
+#include <sbpcmcia.h>
+#include <bcmnvram.h>
+#include <bcmdevs.h>
+#endif /* #if defined(BCM_DNGL_EMBEDIMAGE) */
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX)
+#include <linux/usb.h>
+#endif /* EHCI_FASTPATH_TX || EHCI_FASTPATH_RX */
+
+#if defined(BCM_DNGL_EMBEDIMAGE)
+/* zlib file format field ids etc from gzio.c */
+#define Z_DEFLATED     8
+#define ASCII_FLAG     0x01 /**< bit 0 set: file probably ascii text */
+#define HEAD_CRC       0x02 /**< bit 1 set: header CRC present */
+#define EXTRA_FIELD    0x04 /**< bit 2 set: extra field present */
+#define ORIG_NAME      0x08 /**< bit 3 set: original file name present */
+#define COMMENT        0x10 /**< bit 4 set: file comment present */
+#define RESERVED       0xE0 /**< bits 5..7: reserved */
+
+/* Rename define */
+#ifdef WL_FW_DECOMP
+#define UNZIP_ENAB(info)  1
+#else
+#define UNZIP_ENAB(info)  0
+
+#ifdef inflateInit2
+#undef inflateInit2
+#define inflateInit2(a, b)  Z_ERRNO
+#endif
+#define inflate(a, b)       Z_STREAM_ERROR
+#define inflateEnd(a)       do {} while (0)
+#define crc32(a, b, c)      -1
+#define free(a)             do {} while (0)
+#endif /* WL_FW_DECOMP */
+
+#elif defined(BCM_REQUEST_FW)
+#ifndef VARS_MAX
+#define VARS_MAX            8192
+#endif
+#endif /* #if defined(BCM_DNGL_EMBEDIMAGE) */
+
+#ifdef DBUS_USB_LOOPBACK
+extern bool is_loopback_pkt(void *buf);
+extern int matches_loopback_pkt(void *buf);
+#endif
+
+/** General info for all BUS types */
+typedef struct dbus_irbq {
+	dbus_irb_t *head;
+	dbus_irb_t *tail;
+	int cnt;
+} dbus_irbq_t;
+
+/**
+ * This private structure dhd_bus_t is also declared in dbus_usb_linux.c.
+ * All the fields must be consistent in both declarations.
+ */
+typedef struct dhd_bus {
+	dbus_pub_t   pub; /* MUST BE FIRST */
+	dhd_pub_t *dhd;
+
+	void        *cbarg;
+	dbus_callbacks_t *cbs; /**< callbacks to higher level, eg dhd_linux.c */
+	void        *bus_info;
+	dbus_intf_t *drvintf;  /**< callbacks to lower level, eg dbus_usb.c or dbus_usb_linux.c */
+	uint8       *fw;
+	int         fwlen;
+	uint32      errmask;
+	int         rx_low_watermark;  /**< avoid rx overflow by filling rx with free IRBs */
+	int         tx_low_watermark;
+	bool        txoff;
+	bool        txoverride;   /**< flow control related */
+	bool        rxoff;
+	bool        tx_timer_ticking;
+	uint ctl_completed;
+
+	dbus_irbq_t *rx_q;
+	dbus_irbq_t *tx_q;
+
+#ifdef BCMDBG
+	int         *txpend_q_hist;
+	int         *rxpend_q_hist;
+#endif /* BCMDBG */
+#ifdef EHCI_FASTPATH_RX
+	atomic_t    rx_outstanding;
+#endif
+	uint8        *nvram;
+	int          nvram_len;
+	uint8        *image;  /**< buffer for combine fw and nvram */
+	int          image_len;
+	uint8        *orig_fw;
+	int          origfw_len;
+	int          decomp_memsize;
+	dbus_extdl_t extdl;
+	int          nvram_nontxt;
+#if defined(BCM_REQUEST_FW)
+	void         *firmware;
+	void         *nvfile;
+#endif
+	char		*fw_path;		/* module_param: path to firmware image */
+	char		*nv_path;		/* module_param: path to nvram vars file */
+	uint64 last_suspend_end_time;
+} dhd_bus_t;
+
+struct exec_parms {
+	union {
+		/* Can consolidate same params, if need be, but this shows
+		 * group of parameters per function
+		 */
+		struct {
+			dbus_irbq_t  *q;
+			dbus_irb_t   *b;
+		} qenq;
+
+		struct {
+			dbus_irbq_t  *q;
+		} qdeq;
+	};
+};
+
+#define EXEC_RXLOCK(info, fn, a) \
+	info->drvintf->exec_rxlock(dhd_bus->bus_info, ((exec_cb_t)fn), ((struct exec_parms *) a))
+
+#define EXEC_TXLOCK(info, fn, a) \
+	info->drvintf->exec_txlock(dhd_bus->bus_info, ((exec_cb_t)fn), ((struct exec_parms *) a))
+
+/*
+ * Callbacks common for all BUS
+ */
+static void dbus_if_send_irb_timeout(void *handle, dbus_irb_tx_t *txirb);
+static void dbus_if_send_irb_complete(void *handle, dbus_irb_tx_t *txirb, int status);
+static void dbus_if_recv_irb_complete(void *handle, dbus_irb_rx_t *rxirb, int status);
+static void dbus_if_errhandler(void *handle, int err);
+static void dbus_if_ctl_complete(void *handle, int type, int status);
+static void dbus_if_state_change(void *handle, int state);
+static void *dbus_if_pktget(void *handle, uint len, bool send);
+static void dbus_if_pktfree(void *handle, void *p, bool send);
+static struct dbus_irb *dbus_if_getirb(void *cbarg, bool send);
+static void dbus_if_rxerr_indicate(void *handle, bool on);
+
+static int dbus_suspend(void *context);
+static int dbus_resume(void *context);
+static void * dhd_dbus_probe_cb(uint16 bus_no, uint16 slot, uint32 hdrlen);
+static void dhd_dbus_disconnect_cb(void *arg);
+static void dbus_detach(dhd_bus_t *pub);
+
+/** functions in this file that are called by lower DBUS levels, eg dbus_usb.c */
+static dbus_intf_callbacks_t dbus_intf_cbs = {
+	dbus_if_send_irb_timeout,
+	dbus_if_send_irb_complete,
+	dbus_if_recv_irb_complete,
+	dbus_if_errhandler,
+	dbus_if_ctl_complete,
+	dbus_if_state_change,
+	NULL,			/**< isr */
+	NULL,			/**< dpc */
+	NULL,			/**< watchdog */
+	dbus_if_pktget,
+	dbus_if_pktfree,
+	dbus_if_getirb,
+	dbus_if_rxerr_indicate
+};
+
+/*
+ * Need global for probe() and disconnect() since
+ * attach() is not called at probe and detach()
+ * can be called inside disconnect()
+ */
+static dbus_intf_t     *g_busintf = NULL;
+
+#if defined(BCM_REQUEST_FW)
+int8 *nonfwnvram = NULL; /**< stand-alone multi-nvram given with driver load */
+int nonfwnvramlen = 0;
+#endif /* #if defined(BCM_REQUEST_FW) */
+
+static void* q_enq(dbus_irbq_t *q, dbus_irb_t *b);
+static void* q_enq_exec(struct exec_parms *args);
+static dbus_irb_t*q_deq(dbus_irbq_t *q);
+static void* q_deq_exec(struct exec_parms *args);
+static int   dbus_tx_timer_init(dhd_bus_t *dhd_bus);
+static int   dbus_tx_timer_start(dhd_bus_t *dhd_bus, uint timeout);
+static int   dbus_tx_timer_stop(dhd_bus_t *dhd_bus);
+static int   dbus_irbq_init(dhd_bus_t *dhd_bus, dbus_irbq_t *q, int nq, int size_irb);
+static int   dbus_irbq_deinit(dhd_bus_t *dhd_bus, dbus_irbq_t *q, int size_irb);
+static int   dbus_rxirbs_fill(dhd_bus_t *dhd_bus);
+static int   dbus_send_irb(dbus_pub_t *pub, uint8 *buf, int len, void *pkt, void *info);
+
+#if (defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW))
+#if defined(BCM_REQUEST_FW)
+extern char * dngl_firmware;
+extern unsigned int dngl_fwlen;
+#endif  /* #if defined(BCM_REQUEST_FW) */
+#ifndef EXTERNAL_FW_PATH
+static int dbus_get_nvram(dhd_bus_t *dhd_bus);
+static int dbus_jumbo_nvram(dhd_bus_t *dhd_bus);
+static int dbus_select_nvram(dhd_bus_t *dhd_bus, int8 *jumbonvram, int jumbolen,
+uint16 boardtype, uint16 boardrev, int8 **nvram, int *nvram_len);
+#endif /* !EXTERNAL_FW_PATH */
+#ifndef BCM_REQUEST_FW
+static int dbus_zlib_decomp(dhd_bus_t *dhd_bus);
+extern void *dbus_zlib_calloc(int num, int size);
+extern void dbus_zlib_free(void *ptr);
+#endif
+#endif /* defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW) */
+
+/* function */
+void
+dbus_flowctrl_tx(void *dbi, bool on)
+{
+	dhd_bus_t *dhd_bus = dbi;
+
+	if (dhd_bus == NULL)
+		return;
+
+	DBUSTRACE(("%s on %d\n", __FUNCTION__, on));
+
+	if (dhd_bus->txoff == on)
+		return;
+
+	dhd_bus->txoff = on;
+
+	if (dhd_bus->cbs && dhd_bus->cbs->txflowcontrol)
+		dhd_bus->cbs->txflowcontrol(dhd_bus->cbarg, on);
+}
+
+/**
+ * if lower level DBUS signaled a rx error, more free rx IRBs should be allocated or flow control
+ * should kick in to make more free rx IRBs available.
+ */
+static void
+dbus_if_rxerr_indicate(void *handle, bool on)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+
+	DBUSTRACE(("%s, on %d\n", __FUNCTION__, on));
+
+	if (dhd_bus == NULL)
+		return;
+
+	if (dhd_bus->txoverride == on)
+		return;
+
+	dhd_bus->txoverride = on;	/* flow control */
+
+	if (!on)
+		dbus_rxirbs_fill(dhd_bus);
+
+}
+
+/** q_enq()/q_deq() are executed with protection via exec_rxlock()/exec_txlock() */
+static void*
+q_enq(dbus_irbq_t *q, dbus_irb_t *b)
+{
+	ASSERT(q->tail != b);
+	ASSERT(b->next == NULL);
+	b->next = NULL;
+	if (q->tail) {
+		q->tail->next = b;
+		q->tail = b;
+	} else
+		q->head = q->tail = b;
+
+	q->cnt++;
+
+	return b;
+}
+
+static void*
+q_enq_exec(struct exec_parms *args)
+{
+	return q_enq(args->qenq.q, args->qenq.b);
+}
+
+static dbus_irb_t*
+q_deq(dbus_irbq_t *q)
+{
+	dbus_irb_t *b;
+
+	b = q->head;
+	if (b) {
+		q->head = q->head->next;
+		b->next = NULL;
+
+		if (q->head == NULL)
+			q->tail = q->head;
+
+		q->cnt--;
+	}
+	return b;
+}
+
+static void*
+q_deq_exec(struct exec_parms *args)
+{
+	return q_deq(args->qdeq.q);
+}
+
+/**
+ * called during attach phase. Status @ Dec 2012: this function does nothing since for all of the
+ * lower DBUS levels dhd_bus->drvintf->tx_timer_init is NULL.
+ */
+static int
+dbus_tx_timer_init(dhd_bus_t *dhd_bus)
+{
+	if (dhd_bus && dhd_bus->drvintf && dhd_bus->drvintf->tx_timer_init)
+		return dhd_bus->drvintf->tx_timer_init(dhd_bus->bus_info);
+	else
+		return DBUS_ERR;
+}
+
+static int
+dbus_tx_timer_start(dhd_bus_t *dhd_bus, uint timeout)
+{
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (dhd_bus->tx_timer_ticking)
+		return DBUS_OK;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->tx_timer_start) {
+		if (dhd_bus->drvintf->tx_timer_start(dhd_bus->bus_info, timeout) == DBUS_OK) {
+			dhd_bus->tx_timer_ticking = TRUE;
+			return DBUS_OK;
+		}
+	}
+
+	return DBUS_ERR;
+}
+
+static int
+dbus_tx_timer_stop(dhd_bus_t *dhd_bus)
+{
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (!dhd_bus->tx_timer_ticking)
+		return DBUS_OK;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->tx_timer_stop) {
+		if (dhd_bus->drvintf->tx_timer_stop(dhd_bus->bus_info) == DBUS_OK) {
+			dhd_bus->tx_timer_ticking = FALSE;
+			return DBUS_OK;
+		}
+	}
+
+	return DBUS_ERR;
+}
+
+/** called during attach phase. */
+static int
+dbus_irbq_init(dhd_bus_t *dhd_bus, dbus_irbq_t *q, int nq, int size_irb)
+{
+	int i;
+	dbus_irb_t *irb;
+
+	ASSERT(q);
+	ASSERT(dhd_bus);
+
+	for (i = 0; i < nq; i++) {
+		/* MALLOCZ dbus_irb_tx or dbus_irb_rx, but cast to simple dbus_irb_t linkedlist */
+		irb = (dbus_irb_t *) MALLOCZ(dhd_bus->pub.osh, size_irb);
+		if (irb == NULL) {
+			ASSERT(irb);
+			return DBUS_ERR;
+		}
+
+		/* q_enq() does not need to go through EXEC_xxLOCK() during init() */
+		q_enq(q, irb);
+	}
+
+	return DBUS_OK;
+}
+
+/** called during detach phase or when attach failed */
+static int
+dbus_irbq_deinit(dhd_bus_t *dhd_bus, dbus_irbq_t *q, int size_irb)
+{
+	dbus_irb_t *irb;
+
+	ASSERT(q);
+	ASSERT(dhd_bus);
+
+	/* q_deq() does not need to go through EXEC_xxLOCK()
+	 * during deinit(); all callbacks are stopped by this time
+	 */
+	while ((irb = q_deq(q)) != NULL) {
+		MFREE(dhd_bus->pub.osh, irb, size_irb);
+	}
+
+	if (q->cnt)
+		DBUSERR(("deinit: q->cnt=%d > 0\n", q->cnt));
+	return DBUS_OK;
+}
+
+/** multiple code paths require the rx queue to be filled with more free IRBs */
+static int
+dbus_rxirbs_fill(dhd_bus_t *dhd_bus)
+{
+	int err = DBUS_OK;
+
+#ifdef EHCI_FASTPATH_RX
+	while (atomic_read(&dhd_bus->rx_outstanding) < 100) /* TODO: Improve constant */
+	{
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+		/* NOCOPY force new packet allocation */
+		optimize_submit_rx_request(&dhd_bus->pub, 1, NULL, NULL);
+#else
+		/* Copy mode - allocate own buffer to be reused */
+		void *buf = MALLOCZ(dhd_bus->pub.osh, 4000); /* usbos_info->rxbuf_len */
+		optimize_submit_rx_request(&dhd_bus->pub, 1, NULL, buf);
+		/* ME: Need to check result and set err = DBUS_ERR_RXDROP */
+#endif /* BCM_RPC_NOCOPY || BCM_RPC_RXNOCOPY */
+		atomic_inc(&dhd_bus->rx_outstanding);
+	}
+#else /* EHCI_FASTPATH_RX */
+
+	dbus_irb_rx_t *rxirb;
+	struct exec_parms args;
+
+	ASSERT(dhd_bus);
+	if (dhd_bus->pub.busstate != DBUS_STATE_UP) {
+		DBUSERR(("dbus_rxirbs_fill: DBUS not up \n"));
+		return DBUS_ERR;
+	} else if (!dhd_bus->drvintf || (dhd_bus->drvintf->recv_irb == NULL)) {
+		/* Lower edge bus interface does not support recv_irb().
+		 * No need to pre-submit IRBs in this case.
+		 */
+		return DBUS_ERR;
+	}
+
+	/* The dongle recv callback is freerunning without lock. So multiple callbacks(and this
+	 *  refill) can run in parallel. While the rxoff condition is triggered outside,
+	 *  below while loop has to check and abort posting more to avoid RPC rxq overflow.
+	 */
+	args.qdeq.q = dhd_bus->rx_q;
+	while ((!dhd_bus->rxoff) &&
+	       (rxirb = (EXEC_RXLOCK(dhd_bus, q_deq_exec, &args))) != NULL) {
+		err = dhd_bus->drvintf->recv_irb(dhd_bus->bus_info, rxirb);
+		if (err == DBUS_ERR_RXDROP || err == DBUS_ERR_RXFAIL) {
+			/* Add the the free rxirb back to the queue
+			 * and wait till later
+			 */
+			bzero(rxirb, sizeof(dbus_irb_rx_t));
+			args.qenq.q = dhd_bus->rx_q;
+			args.qenq.b = (dbus_irb_t *) rxirb;
+			EXEC_RXLOCK(dhd_bus, q_enq_exec, &args);
+			break;
+		} else if (err != DBUS_OK) {
+			int i = 0;
+			while (i++ < 100) {
+				DBUSERR(("%s :: memory leak for rxirb note?\n", __FUNCTION__));
+			}
+		}
+	}
+#endif /* EHCI_FASTPATH_RX */
+	return err;
+} /* dbus_rxirbs_fill */
+
+/** called when the DBUS interface state changed. */
+void
+dbus_flowctrl_rx(dbus_pub_t *pub, bool on)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if (dhd_bus == NULL)
+		return;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus->rxoff == on)
+		return;
+
+	dhd_bus->rxoff = on;
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP) {
+		if (!on) {
+			/* post more irbs, resume rx if necessary */
+			dbus_rxirbs_fill(dhd_bus);
+			if (dhd_bus && dhd_bus->drvintf->recv_resume) {
+				dhd_bus->drvintf->recv_resume(dhd_bus->bus_info);
+			}
+		} else {
+			/* ??? cancell posted irbs first */
+
+			if (dhd_bus && dhd_bus->drvintf->recv_stop) {
+				dhd_bus->drvintf->recv_stop(dhd_bus->bus_info);
+			}
+		}
+	}
+}
+
+/**
+ * Several code paths in this file want to send a buffer to the dongle. This function handles both
+ * sending of a buffer or a pkt.
+ */
+static int
+dbus_send_irb(dbus_pub_t *pub, uint8 *buf, int len, void *pkt, void *info)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_OK;
+#ifndef EHCI_FASTPATH_TX
+	dbus_irb_tx_t *txirb = NULL;
+	int txirb_pending;
+	struct exec_parms args;
+#endif /* EHCI_FASTPATH_TX */
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP ||
+		dhd_bus->pub.busstate == DBUS_STATE_SLEEP) {
+#ifdef EHCI_FASTPATH_TX
+		struct ehci_qtd *qtd;
+		int token = EHCI_QTD_SET_CERR(3);
+		int len;
+
+		ASSERT(buf == NULL); /* Not handled */
+		ASSERT(pkt != NULL);
+
+		qtd = optimize_ehci_qtd_alloc(GFP_KERNEL);
+
+		if (qtd == NULL)
+			return DBUS_ERR;
+
+		len = PKTLEN(pub->osh, pkt);
+
+		len = ROUNDUP(len, sizeof(uint32));
+
+#ifdef BCMDBG
+		/* The packet length is already padded to not to be multiple of 512 bytes
+		 * in bcm_rpc_tp_buf_send_internal(), so it should not be 512*N bytes here.
+		 */
+		if (len % EHCI_BULK_PACKET_SIZE == 0) {
+			DBUSERR(("%s: len = %d (multiple of 512 bytes)\n", __FUNCTION__, len));
+			return DBUS_ERR_TXDROP;
+		}
+#endif /* BCMDBG */
+
+		optimize_qtd_fill_with_rpc(pub, 0, qtd, pkt, token, len);
+		err = optimize_submit_async(qtd, 0);
+
+		if (err) {
+			optimize_ehci_qtd_free(qtd);
+			err = DBUS_ERR_TXDROP;
+		}
+
+		/* ME: Add timeout? */
+#else
+		args.qdeq.q = dhd_bus->tx_q;
+		if (dhd_bus->drvintf)
+			txirb = EXEC_TXLOCK(dhd_bus, q_deq_exec, &args);
+
+		if (txirb == NULL) {
+			DBUSERR(("Out of tx dbus_bufs\n"));
+			return DBUS_ERR;
+		}
+
+		if (pkt != NULL) {
+			txirb->pkt = pkt;
+			txirb->buf = NULL;
+			txirb->len = 0;
+		} else if (buf != NULL) {
+			txirb->pkt = NULL;
+			txirb->buf = buf;
+			txirb->len = len;
+		} else {
+			ASSERT(0); /* Should not happen */
+		}
+		txirb->info = info;
+		txirb->arg = NULL;
+		txirb->retry_count = 0;
+
+		if (dhd_bus->drvintf && dhd_bus->drvintf->send_irb) {
+			/* call lower DBUS level send_irb function */
+			err = dhd_bus->drvintf->send_irb(dhd_bus->bus_info, txirb);
+			if (err == DBUS_ERR_TXDROP) {
+				/* tx fail and no completion routine to clean up, reclaim irb NOW */
+				DBUSERR(("%s: send_irb failed, status = %d\n", __FUNCTION__, err));
+				bzero(txirb, sizeof(dbus_irb_tx_t));
+				args.qenq.q = dhd_bus->tx_q;
+				args.qenq.b = (dbus_irb_t *) txirb;
+				EXEC_TXLOCK(dhd_bus, q_enq_exec, &args);
+			} else {
+				dbus_tx_timer_start(dhd_bus, DBUS_TX_TIMEOUT_INTERVAL);
+				txirb_pending = dhd_bus->pub.ntxq - dhd_bus->tx_q->cnt;
+#ifdef BCMDBG
+				dhd_bus->txpend_q_hist[txirb_pending]++;
+#endif /* BCMDBG */
+				if (txirb_pending > (dhd_bus->tx_low_watermark * 3)) {
+					dbus_flowctrl_tx(dhd_bus, TRUE);
+				}
+			}
+		}
+#endif /* EHCI_FASTPATH_TX */
+	} else {
+		err = DBUS_ERR_TXFAIL;
+		DBUSTRACE(("%s: bus down, send_irb failed\n", __FUNCTION__));
+	}
+
+	return err;
+} /* dbus_send_irb */
+
+#if (defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW))
+
+/**
+ * Before downloading a firmware image into the dongle, the validity of the image must be checked.
+ */
+static int
+check_file(osl_t *osh, unsigned char *headers)
+{
+	struct trx_header *trx;
+	int actual_len = -1;
+
+	/* Extract trx header */
+	trx = (struct trx_header *)headers;
+	if (ltoh32(trx->magic) != TRX_MAGIC) {
+		printf("Error: trx bad hdr %x\n", ltoh32(trx->magic));
+		return -1;
+	}
+
+	headers += SIZEOF_TRX(trx);
+
+	/* TRX V1: get firmware len */
+	/* TRX V2: get firmware len and DSG/CFG lengths */
+	if (ltoh32(trx->flag_version) & TRX_UNCOMP_IMAGE) {
+		actual_len = ltoh32(trx->offsets[TRX_OFFSETS_DLFWLEN_IDX]) +
+		                     SIZEOF_TRX(trx);
+#ifdef BCMTRXV2
+		if (ISTRX_V2(trx)) {
+			actual_len += ltoh32(trx->offsets[TRX_OFFSETS_DSG_LEN_IDX]) +
+				ltoh32(trx->offsets[TRX_OFFSETS_CFG_LEN_IDX]);
+		}
+#endif
+		return actual_len;
+	}  else {
+		printf("compressed image\n");
+	}
+
+	return -1;
+}
+
+#ifdef EXTERNAL_FW_PATH
+static int
+dbus_get_fw_nvram(dhd_bus_t *dhd_bus)
+{
+	int bcmerror = -1, i;
+	uint len, total_len;
+	void *nv_image = NULL, *fw_image = NULL;
+	char *nv_memblock = NULL, *fw_memblock = NULL;
+	char *bufp;
+	bool file_exists;
+	uint8 nvram_words_pad = 0;
+	uint memblock_size = 2048;
+	uint8 *memptr;
+	int	actual_fwlen;
+	struct trx_header *hdr;
+	uint32 img_offset = 0;
+	int offset = 0;
+	char *pfw_path = dhd_bus->fw_path;
+	char *pnv_path = dhd_bus->nv_path;
+
+	/* For Get nvram */
+	file_exists = ((pnv_path != NULL) && (pnv_path[0] != '\0'));
+	if (file_exists) {
+		nv_image = dhd_os_open_image1(dhd_bus->dhd, pnv_path);
+		if (nv_image == NULL) {
+			printf("%s: Open nvram file failed %s\n", __FUNCTION__, pnv_path);
+			goto err;
+		}
+	}
+	nv_memblock = MALLOC(dhd_bus->pub.osh, MAX_NVRAMBUF_SIZE);
+	if (nv_memblock == NULL) {
+		DBUSERR(("%s: Failed to allocate memory %d bytes\n",
+		           __FUNCTION__, MAX_NVRAMBUF_SIZE));
+		goto err;
+	}
+	len = dhd_os_get_image_block(nv_memblock, MAX_NVRAMBUF_SIZE, nv_image);
+	if (len > 0 && len < MAX_NVRAMBUF_SIZE) {
+		bufp = (char *)nv_memblock;
+		bufp[len] = 0;
+		dhd_bus->nvram_len = process_nvram_vars(bufp, len);
+		if (dhd_bus->nvram_len % 4)
+			nvram_words_pad = 4 - dhd_bus->nvram_len % 4;
+	} else {
+		DBUSERR(("%s: error reading nvram file: %d\n", __FUNCTION__, len));
+		bcmerror = DBUS_ERR_NVRAM;
+		goto err;
+	}
+	if (nv_image) {
+		dhd_os_close_image1(dhd_bus->dhd, nv_image);
+		nv_image = NULL;
+	}
+
+	/* For Get first block of fw to calculate total_len */
+	file_exists = ((pfw_path != NULL) && (pfw_path[0] != '\0'));
+	if (file_exists) {
+		fw_image = dhd_os_open_image1(dhd_bus->dhd, pfw_path);
+		if (fw_image == NULL) {
+			printf("%s: Open fw file failed %s\n", __FUNCTION__, pfw_path);
+			goto err;
+		}
+	}
+	memptr = fw_memblock = MALLOC(dhd_bus->pub.osh, memblock_size);
+	if (fw_memblock == NULL) {
+		DBUSERR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__,
+			memblock_size));
+		goto err;
+	}
+	len = dhd_os_get_image_block((char*)memptr, memblock_size, fw_image);
+	if ((actual_fwlen = check_file(dhd_bus->pub.osh, memptr)) <= 0) {
+		DBUSERR(("%s: bad firmware format!\n", __FUNCTION__));
+		goto err;
+	}
+
+	total_len = actual_fwlen + dhd_bus->nvram_len + nvram_words_pad;
+#if defined(CONFIG_DHD_USE_STATIC_BUF)
+	dhd_bus->image = (uint8*)DHD_OS_PREALLOC(dhd_bus->dhd,
+		DHD_PREALLOC_MEMDUMP_RAM, total_len);
+#else
+	dhd_bus->image = MALLOC(dhd_bus->pub.osh, total_len);
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+	dhd_bus->image_len = total_len;
+	if (dhd_bus->image == NULL) {
+		DBUSERR(("%s: malloc failed! size=%d\n", __FUNCTION__, total_len));
+		goto err;
+	}
+
+	/* Step1: Copy trx header + firmwre */
+	memptr = fw_memblock;
+	do {
+		if (len < 0) {
+			DBUSERR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+		bcopy(memptr, dhd_bus->image+offset, len);
+		offset += len;
+	} while ((len = dhd_os_get_image_block((char*)memptr, memblock_size, fw_image)));
+	/* Step2: Copy NVRAM + pad */
+	hdr = (struct trx_header *)dhd_bus->image;
+	img_offset = SIZEOF_TRX(hdr) + hdr->offsets[TRX_OFFSETS_DLFWLEN_IDX];
+	bcopy(nv_memblock, (uint8 *)(dhd_bus->image + img_offset),
+		dhd_bus->nvram_len);
+	img_offset += dhd_bus->nvram_len;
+	if (nvram_words_pad) {
+		bzero(&dhd_bus->image[img_offset], nvram_words_pad);
+		img_offset += nvram_words_pad;
+	}
+#ifdef BCMTRXV2
+	/* Step3: Copy DSG/CFG for V2 */
+	if (ISTRX_V2(hdr) &&
+		(hdr->offsets[TRX_OFFSETS_DSG_LEN_IDX] ||
+		hdr->offsets[TRX_OFFSETS_CFG_LEN_IDX])) {
+		DBUSERR(("%s: fix me\n", __FUNCTION__));
+	}
+#endif /* BCMTRXV2 */
+	/* Step4: update TRX header for nvram size */
+	hdr = (struct trx_header *)dhd_bus->image;
+	hdr->len = htol32(total_len);
+	/* Pass the actual fw len */
+	hdr->offsets[TRX_OFFSETS_NVM_LEN_IDX] =
+		htol32(dhd_bus->nvram_len + nvram_words_pad);
+	/* Calculate CRC over header */
+	hdr->crc32 = hndcrc32((uint8 *)&hdr->flag_version,
+		SIZEOF_TRX(hdr) - OFFSETOF(struct trx_header, flag_version),
+		CRC32_INIT_VALUE);
+
+	/* Calculate CRC over data */
+	for (i = SIZEOF_TRX(hdr); i < total_len; ++i)
+			hdr->crc32 = hndcrc32((uint8 *)&dhd_bus->image[i], 1, hdr->crc32);
+	hdr->crc32 = htol32(hdr->crc32);
+
+	bcmerror = DBUS_OK;
+
+err:
+	if (fw_memblock)
+		MFREE(dhd_bus->pub.osh, fw_memblock, MAX_NVRAMBUF_SIZE);
+	if (fw_image)
+		dhd_os_close_image1(dhd_bus->dhd, fw_image);
+	if (nv_memblock)
+		MFREE(dhd_bus->pub.osh, nv_memblock, MAX_NVRAMBUF_SIZE);
+	if (nv_image)
+		dhd_os_close_image1(dhd_bus->dhd, nv_image);
+
+	return bcmerror;
+}
+
+/**
+ * during driver initialization ('attach') or after PnP 'resume', firmware needs to be loaded into
+ * the dongle
+ */
+static int
+dbus_do_download(dhd_bus_t *dhd_bus)
+{
+	int err = DBUS_OK;
+
+	err = dbus_get_fw_nvram(dhd_bus);
+	if (err) {
+		DBUSERR(("dbus_do_download: fail to get nvram %d\n", err));
+		return err;
+	}
+
+	if (dhd_bus->drvintf->dlstart && dhd_bus->drvintf->dlrun) {
+		err = dhd_bus->drvintf->dlstart(dhd_bus->bus_info,
+			dhd_bus->image, dhd_bus->image_len);
+		if (err == DBUS_OK) {
+			err = dhd_bus->drvintf->dlrun(dhd_bus->bus_info);
+		}
+	} else
+		err = DBUS_ERR;
+
+	if (dhd_bus->image) {
+#if defined(CONFIG_DHD_USE_STATIC_BUF)
+		DHD_OS_PREFREE(dhd_bus->dhd, dhd_bus->image, dhd_bus->image_len);
+#else
+		MFREE(dhd_bus->pub.osh, dhd_bus->image, dhd_bus->image_len);
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+		dhd_bus->image = NULL;
+		dhd_bus->image_len = 0;
+	}
+
+	return err;
+} /* dbus_do_download */
+#else
+
+/**
+ * It is easy for the user to pass one jumbo nvram file to the driver than a set of smaller files.
+ * The 'jumbo nvram' file format is essentially a set of nvram files. Before commencing firmware
+ * download, the dongle needs to be probed so that the correct nvram contents within the jumbo nvram
+ * file is selected.
+ */
+static int
+dbus_jumbo_nvram(dhd_bus_t *dhd_bus)
+{
+	int8 *nvram = NULL;
+	int nvram_len = 0;
+	int ret = DBUS_OK;
+	uint16 boardrev = 0xFFFF;
+	uint16 boardtype = 0xFFFF;
+
+	ret = dbus_select_nvram(dhd_bus, dhd_bus->extdl.vars, dhd_bus->extdl.varslen,
+		boardtype, boardrev, &nvram, &nvram_len);
+
+	if (ret == DBUS_JUMBO_BAD_FORMAT)
+			return DBUS_ERR_NVRAM;
+	else if (ret == DBUS_JUMBO_NOMATCH &&
+		(boardtype != 0xFFFF || boardrev  != 0xFFFF)) {
+			DBUSERR(("No matching NVRAM for boardtype 0x%02x boardrev 0x%02x\n",
+				boardtype, boardrev));
+			return DBUS_ERR_NVRAM;
+	}
+	dhd_bus->nvram = nvram;
+	dhd_bus->nvram_len =  nvram_len;
+
+	return DBUS_OK;
+}
+
+/** before commencing fw download, the correct NVRAM image to download has to be picked */
+static int
+dbus_get_nvram(dhd_bus_t *dhd_bus)
+{
+	int len, i;
+	struct trx_header *hdr;
+	int	actual_fwlen;
+	uint32 img_offset = 0;
+
+	dhd_bus->nvram_len = 0;
+	if (dhd_bus->extdl.varslen) {
+		if (DBUS_OK != dbus_jumbo_nvram(dhd_bus))
+			return DBUS_ERR_NVRAM;
+		DBUSERR(("NVRAM %d bytes downloaded\n", dhd_bus->nvram_len));
+	}
+#if defined(BCM_REQUEST_FW)
+	else if (nonfwnvram) {
+		dhd_bus->nvram = nonfwnvram;
+		dhd_bus->nvram_len = nonfwnvramlen;
+		DBUSERR(("NVRAM %d bytes downloaded\n", dhd_bus->nvram_len));
+	}
+#endif
+	if (dhd_bus->nvram) {
+		uint8 nvram_words_pad = 0;
+		/* Validate the format/length etc of the file */
+		if ((actual_fwlen = check_file(dhd_bus->pub.osh, dhd_bus->fw)) <= 0) {
+			DBUSERR(("%s: bad firmware format!\n", __FUNCTION__));
+			return DBUS_ERR_NVRAM;
+		}
+
+		if (!dhd_bus->nvram_nontxt) {
+			/* host supplied nvram could be in .txt format
+			* with all the comments etc...
+			*/
+			dhd_bus->nvram_len = process_nvram_vars(dhd_bus->nvram,
+				dhd_bus->nvram_len);
+		}
+		if (dhd_bus->nvram_len % 4)
+			nvram_words_pad = 4 - dhd_bus->nvram_len % 4;
+
+		len = actual_fwlen + dhd_bus->nvram_len + nvram_words_pad;
+#ifdef USBAP
+		/* Allocate virtual memory otherwise it might fail on embedded systems */
+		dhd_bus->image = VMALLOC(dhd_bus->pub.osh, len);
+#else
+#if defined(CONFIG_DHD_USE_STATIC_BUF)
+		dhd_bus->image = (uint8*)DHD_OS_PREALLOC(dhd_bus->dhd,
+			DHD_PREALLOC_MEMDUMP_RAM, len);
+#else
+		dhd_bus->image = MALLOCZ(dhd_bus->pub.osh, len);
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+#endif /* USBAP */
+		dhd_bus->image_len = len;
+		if (dhd_bus->image == NULL) {
+			DBUSERR(("%s: malloc failed!\n", __FUNCTION__));
+			return DBUS_ERR_NVRAM;
+		}
+		hdr = (struct trx_header *)dhd_bus->fw;
+		/* Step1: Copy trx header + firmwre */
+		img_offset = SIZEOF_TRX(hdr) + hdr->offsets[TRX_OFFSETS_DLFWLEN_IDX];
+		bcopy(dhd_bus->fw, dhd_bus->image, img_offset);
+		/* Step2: Copy NVRAM + pad */
+		bcopy(dhd_bus->nvram, (uint8 *)(dhd_bus->image + img_offset),
+			dhd_bus->nvram_len);
+		img_offset += dhd_bus->nvram_len;
+		if (nvram_words_pad) {
+			bzero(&dhd_bus->image[img_offset],
+				nvram_words_pad);
+			img_offset += nvram_words_pad;
+		}
+#ifdef BCMTRXV2
+		/* Step3: Copy DSG/CFG for V2 */
+		if (ISTRX_V2(hdr) &&
+			(hdr->offsets[TRX_OFFSETS_DSG_LEN_IDX] ||
+			hdr->offsets[TRX_OFFSETS_CFG_LEN_IDX])) {
+
+			bcopy(dhd_bus->fw + SIZEOF_TRX(hdr) +
+				hdr->offsets[TRX_OFFSETS_DLFWLEN_IDX] +
+				hdr->offsets[TRX_OFFSETS_NVM_LEN_IDX],
+				dhd_bus->image + img_offset,
+				hdr->offsets[TRX_OFFSETS_DSG_LEN_IDX] +
+				hdr->offsets[TRX_OFFSETS_CFG_LEN_IDX]);
+
+			/* Not needed */
+			img_offset += hdr->offsets[TRX_OFFSETS_DSG_LEN_IDX] +
+				hdr->offsets[TRX_OFFSETS_CFG_LEN_IDX];
+		}
+#endif /* BCMTRXV2 */
+		/* Step4: update TRX header for nvram size */
+		hdr = (struct trx_header *)dhd_bus->image;
+		hdr->len = htol32(len);
+		/* Pass the actual fw len */
+		hdr->offsets[TRX_OFFSETS_NVM_LEN_IDX] =
+			htol32(dhd_bus->nvram_len + nvram_words_pad);
+		/* Calculate CRC over header */
+		hdr->crc32 = hndcrc32((uint8 *)&hdr->flag_version,
+			SIZEOF_TRX(hdr) - OFFSETOF(struct trx_header, flag_version),
+			CRC32_INIT_VALUE);
+
+		/* Calculate CRC over data */
+		for (i = SIZEOF_TRX(hdr); i < len; ++i)
+				hdr->crc32 = hndcrc32((uint8 *)&dhd_bus->image[i], 1, hdr->crc32);
+		hdr->crc32 = htol32(hdr->crc32);
+	} else {
+		dhd_bus->image = dhd_bus->fw;
+		dhd_bus->image_len = (uint32)dhd_bus->fwlen;
+	}
+
+	return DBUS_OK;
+} /* dbus_get_nvram */
+
+/**
+ * during driver initialization ('attach') or after PnP 'resume', firmware needs to be loaded into
+ * the dongle
+ */
+static int
+dbus_do_download(dhd_bus_t *dhd_bus)
+{
+	int err = DBUS_OK;
+#ifndef BCM_REQUEST_FW
+	int decomp_override = 0;
+#endif
+#ifdef BCM_REQUEST_FW
+	uint16 boardrev = 0xFFFF, boardtype = 0xFFFF;
+	int8 *temp_nvram;
+	int temp_len;
+#endif
+
+#if defined(BCM_DNGL_EMBEDIMAGE)
+	if (dhd_bus->extdl.fw && (dhd_bus->extdl.fwlen > 0)) {
+		dhd_bus->fw = (uint8 *)dhd_bus->extdl.fw;
+		dhd_bus->fwlen = dhd_bus->extdl.fwlen;
+		DBUSERR(("dbus_do_download: using override firmmware %d bytes\n",
+			dhd_bus->fwlen));
+	} else
+		dbus_bus_fw_get(dhd_bus->bus_info, &dhd_bus->fw, &dhd_bus->fwlen,
+			&decomp_override);
+
+	if (!dhd_bus->fw) {
+		DBUSERR(("dbus_do_download: devid 0x%x / %d not supported\n",
+			dhd_bus->pub.attrib.devid, dhd_bus->pub.attrib.devid));
+		return DBUS_ERR;
+	}
+#elif defined(BCM_REQUEST_FW)
+	dhd_bus->firmware = dbus_get_fw_nvfile(dhd_bus->pub.attrib.devid,
+		dhd_bus->pub.attrib.chiprev, &dhd_bus->fw, &dhd_bus->fwlen,
+		DBUS_FIRMWARE, 0, 0, dhd_bus->fw_path);
+	if (!dhd_bus->firmware)
+		return DBUS_ERR;
+#endif /* defined(BCM_DNGL_EMBEDIMAGE) */
+
+	dhd_bus->image = dhd_bus->fw;
+	dhd_bus->image_len = (uint32)dhd_bus->fwlen;
+
+#ifndef BCM_REQUEST_FW
+	if (UNZIP_ENAB(dhd_bus) && !decomp_override) {
+		err = dbus_zlib_decomp(dhd_bus);
+		if (err) {
+			DBUSERR(("dbus_attach: fw decompress fail %d\n", err));
+			return err;
+		}
+	}
+#endif
+
+#if defined(BCM_REQUEST_FW)
+	/* check if nvram is provided as separte file */
+	nonfwnvram = NULL;
+	nonfwnvramlen = 0;
+	dhd_bus->nvfile = dbus_get_fw_nvfile(dhd_bus->pub.attrib.devid,
+		dhd_bus->pub.attrib.chiprev, (void *)&temp_nvram, &temp_len,
+		DBUS_NVFILE, boardtype, boardrev, dhd_bus->nv_path);
+	if (dhd_bus->nvfile) {
+		int8 *tmp = MALLOCZ(dhd_bus->pub.osh, temp_len);
+		if (tmp) {
+			bcopy(temp_nvram, tmp, temp_len);
+			nonfwnvram = tmp;
+			nonfwnvramlen = temp_len;
+		} else {
+			err = DBUS_ERR;
+			goto fail;
+		}
+	}
+#endif /* defined(BCM_REQUEST_FW) */
+
+	err = dbus_get_nvram(dhd_bus);
+	if (err) {
+		DBUSERR(("dbus_do_download: fail to get nvram %d\n", err));
+		return err;
+	}
+
+	if (dhd_bus->drvintf->dlstart && dhd_bus->drvintf->dlrun) {
+		err = dhd_bus->drvintf->dlstart(dhd_bus->bus_info,
+			dhd_bus->image, dhd_bus->image_len);
+
+		if (err == DBUS_OK)
+			err = dhd_bus->drvintf->dlrun(dhd_bus->bus_info);
+	} else
+		err = DBUS_ERR;
+
+	if (dhd_bus->nvram) {
+#ifdef USBAP
+		VMFREE(dhd_bus->pub.osh, dhd_bus->image, dhd_bus->image_len);
+#else
+#if defined(CONFIG_DHD_USE_STATIC_BUF)
+		DHD_OS_PREFREE(dhd_bus->dhd, dhd_bus->image, dhd_bus->image_len);
+#else
+		MFREE(dhd_bus->pub.osh, dhd_bus->image, dhd_bus->image_len);
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+#endif /* USBAP */
+		dhd_bus->image = dhd_bus->fw;
+		dhd_bus->image_len = (uint32)dhd_bus->fwlen;
+	}
+
+#ifndef BCM_REQUEST_FW
+	if (UNZIP_ENAB(dhd_bus) && (!decomp_override) && dhd_bus->orig_fw) {
+		MFREE(dhd_bus->pub.osh, dhd_bus->fw, dhd_bus->decomp_memsize);
+		dhd_bus->image = dhd_bus->fw = dhd_bus->orig_fw;
+		dhd_bus->image_len = dhd_bus->fwlen = dhd_bus->origfw_len;
+	}
+#endif
+
+#if defined(BCM_REQUEST_FW)
+fail:
+	if (dhd_bus->firmware) {
+		dbus_release_fw_nvfile(dhd_bus->firmware);
+		dhd_bus->firmware = NULL;
+	}
+	if (dhd_bus->nvfile) {
+		dbus_release_fw_nvfile(dhd_bus->nvfile);
+		dhd_bus->nvfile = NULL;
+	}
+	if (nonfwnvram) {
+		MFREE(dhd_bus->pub.osh, nonfwnvram, nonfwnvramlen);
+		nonfwnvram = NULL;
+		nonfwnvramlen = 0;
+	}
+#endif
+	return err;
+} /* dbus_do_download */
+#endif /* EXTERNAL_FW_PATH */
+#endif /* defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW) */
+
+/**
+ * This function is called when the sent irb times out without a tx response status.
+ * DBUS adds reliability by resending timed out IRBs DBUS_TX_RETRY_LIMIT times.
+ */
+static void
+dbus_if_send_irb_timeout(void *handle, dbus_irb_tx_t *txirb)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+
+	if ((dhd_bus == NULL) || (dhd_bus->drvintf == NULL) || (txirb == NULL)) {
+		return;
+	}
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	/* Timeout in DBUS is fatal until we have a reproducible case. If the time out really
+	 * happen, then retry needs to be fixed to maintain the order of IRPs going out
+	 * (cancel in reverse order)
+	 */
+	return;
+
+} /* dbus_if_send_irb_timeout */
+
+/**
+ * When lower DBUS level signals that a send IRB completed, either successful or not, the higher
+ * level (eg dhd_linux.c) has to be notified, and transmit flow control has to be evaluated.
+ */
+static void
+dbus_if_send_irb_complete(void *handle, dbus_irb_tx_t *txirb, int status)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+	int txirb_pending;
+	struct exec_parms args;
+	void *pktinfo;
+
+	if ((dhd_bus == NULL) || (txirb == NULL)) {
+		return;
+	}
+
+	DBUSTRACE(("%s: status = %d\n", __FUNCTION__, status));
+
+	dbus_tx_timer_stop(dhd_bus);
+
+	/* re-queue BEFORE calling send_complete which will assume that this irb
+	   is now available.
+	 */
+	pktinfo = txirb->info;
+	bzero(txirb, sizeof(dbus_irb_tx_t));
+	args.qenq.q = dhd_bus->tx_q;
+	args.qenq.b = (dbus_irb_t *) txirb;
+	EXEC_TXLOCK(dhd_bus, q_enq_exec, &args);
+
+	if (dhd_bus->pub.busstate != DBUS_STATE_DOWN) {
+		if ((status == DBUS_OK) || (status == DBUS_ERR_NODEVICE)) {
+			if (dhd_bus->cbs && dhd_bus->cbs->send_complete)
+				dhd_bus->cbs->send_complete(dhd_bus->cbarg, pktinfo,
+					status);
+
+			if (status == DBUS_OK) {
+				txirb_pending = dhd_bus->pub.ntxq - dhd_bus->tx_q->cnt;
+				if (txirb_pending)
+					dbus_tx_timer_start(dhd_bus, DBUS_TX_TIMEOUT_INTERVAL);
+				if ((txirb_pending < dhd_bus->tx_low_watermark) &&
+					dhd_bus->txoff && !dhd_bus->txoverride) {
+					dbus_flowctrl_tx(dhd_bus, OFF);
+				}
+			}
+		} else {
+			DBUSERR(("%s: %d WARNING freeing orphan pkt %p\n", __FUNCTION__, __LINE__,
+				pktinfo));
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_TXNOCOPY) || defined(BCM_RPC_TOC)
+			if (pktinfo)
+				if (dhd_bus->cbs && dhd_bus->cbs->send_complete)
+					dhd_bus->cbs->send_complete(dhd_bus->cbarg, pktinfo,
+						status);
+#else
+			dbus_if_pktfree(dhd_bus, (void*)pktinfo, TRUE);
+#endif /* defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_TXNOCOPY) || defined(BCM_RPC_TOC) */
+		}
+	} else {
+		DBUSERR(("%s: %d WARNING freeing orphan pkt %p\n", __FUNCTION__, __LINE__,
+			pktinfo));
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_TXNOCOPY) || defined(BCM_RPC_TOC)
+		if (pktinfo)
+			if (dhd_bus->cbs && dhd_bus->cbs->send_complete)
+				dhd_bus->cbs->send_complete(dhd_bus->cbarg, pktinfo,
+					status);
+#else
+		dbus_if_pktfree(dhd_bus, (void*)pktinfo, TRUE);
+#endif /* defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_TXNOCOPY) defined(BCM_RPC_TOC) */
+	}
+} /* dbus_if_send_irb_complete */
+
+/**
+ * When lower DBUS level signals that a receive IRB completed, either successful or not, the higher
+ * level (e.g. dhd_linux.c) has to be notified, and fresh free receive IRBs may have to be given
+ * to lower levels.
+ */
+static void
+dbus_if_recv_irb_complete(void *handle, dbus_irb_rx_t *rxirb, int status)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+	int rxirb_pending;
+	struct exec_parms args;
+
+	if ((dhd_bus == NULL) || (rxirb == NULL)) {
+		return;
+	}
+	DBUSTRACE(("%s\n", __FUNCTION__));
+	if (dhd_bus->pub.busstate != DBUS_STATE_DOWN &&
+		dhd_bus->pub.busstate != DBUS_STATE_SLEEP) {
+		if (status == DBUS_OK) {
+			if ((rxirb->buf != NULL) && (rxirb->actual_len > 0)) {
+#ifdef DBUS_USB_LOOPBACK
+				if (is_loopback_pkt(rxirb->buf)) {
+					matches_loopback_pkt(rxirb->buf);
+				} else
+#endif
+				if (dhd_bus->cbs && dhd_bus->cbs->recv_buf) {
+					dhd_bus->cbs->recv_buf(dhd_bus->cbarg, rxirb->buf,
+					rxirb->actual_len);
+				}
+			} else if (rxirb->pkt != NULL) {
+				if (dhd_bus->cbs && dhd_bus->cbs->recv_pkt)
+					dhd_bus->cbs->recv_pkt(dhd_bus->cbarg, rxirb->pkt);
+			} else {
+				ASSERT(0); /* Should not happen */
+			}
+
+			rxirb_pending = dhd_bus->pub.nrxq - dhd_bus->rx_q->cnt - 1;
+#ifdef BCMDBG
+			dhd_bus->rxpend_q_hist[rxirb_pending]++;
+#endif /* BCMDBG */
+			if ((rxirb_pending <= dhd_bus->rx_low_watermark) &&
+				!dhd_bus->rxoff) {
+				DBUSTRACE(("Low watermark so submit more %d <= %d \n",
+					dhd_bus->rx_low_watermark, rxirb_pending));
+				dbus_rxirbs_fill(dhd_bus);
+			} else if (dhd_bus->rxoff)
+				DBUSTRACE(("rx flow controlled. not filling more. cut_rxq=%d\n",
+					dhd_bus->rx_q->cnt));
+		} else if (status == DBUS_ERR_NODEVICE) {
+			DBUSERR(("%s: %d status = %d, buf %p\n", __FUNCTION__, __LINE__, status,
+				rxirb->buf));
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+			if (rxirb->buf) {
+				PKTFRMNATIVE(dhd_bus->pub.osh, rxirb->buf);
+				PKTFREE(dhd_bus->pub.osh, rxirb->buf, FALSE);
+			}
+#endif /* BCM_RPC_NOCOPY || BCM_RPC_TXNOCOPY || BCM_RPC_TOC */
+		} else {
+			if (status != DBUS_ERR_RXZLP)
+				DBUSERR(("%s: %d status = %d, buf %p\n", __FUNCTION__, __LINE__,
+					status, rxirb->buf));
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+			if (rxirb->buf) {
+				PKTFRMNATIVE(dhd_bus->pub.osh, rxirb->buf);
+				PKTFREE(dhd_bus->pub.osh, rxirb->buf, FALSE);
+			}
+#endif /* BCM_RPC_NOCOPY || BCM_RPC_TXNOCOPY || BCM_RPC_TOC */
+		}
+	} else {
+		DBUSTRACE(("%s: DBUS down, ignoring recv callback. buf %p\n", __FUNCTION__,
+			rxirb->buf));
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+		if (rxirb->buf) {
+			PKTFRMNATIVE(dhd_bus->pub.osh, rxirb->buf);
+			PKTFREE(dhd_bus->pub.osh, rxirb->buf, FALSE);
+		}
+#endif /* BCM_RPC_NOCOPY || BCM_RPC_TXNOCOPY || BCM_RPC_TOC */
+	}
+	if (dhd_bus->rx_q != NULL) {
+		bzero(rxirb, sizeof(dbus_irb_rx_t));
+		args.qenq.q = dhd_bus->rx_q;
+		args.qenq.b = (dbus_irb_t *) rxirb;
+		EXEC_RXLOCK(dhd_bus, q_enq_exec, &args);
+	} else
+		MFREE(dhd_bus->pub.osh, rxirb, sizeof(dbus_irb_tx_t));
+} /* dbus_if_recv_irb_complete */
+
+/**
+ *  Accumulate errors signaled by lower DBUS levels and signal them to higher (eg dhd_linux.c)
+ *  level.
+ */
+static void
+dbus_if_errhandler(void *handle, int err)
+{
+	dhd_bus_t *dhd_bus = handle;
+	uint32 mask = 0;
+
+	if (dhd_bus == NULL)
+		return;
+
+	switch (err) {
+		case DBUS_ERR_TXFAIL:
+			dhd_bus->pub.stats.tx_errors++;
+			mask |= ERR_CBMASK_TXFAIL;
+			break;
+		case DBUS_ERR_TXDROP:
+			dhd_bus->pub.stats.tx_dropped++;
+			mask |= ERR_CBMASK_TXFAIL;
+			break;
+		case DBUS_ERR_RXFAIL:
+			dhd_bus->pub.stats.rx_errors++;
+			mask |= ERR_CBMASK_RXFAIL;
+			break;
+		case DBUS_ERR_RXDROP:
+			dhd_bus->pub.stats.rx_dropped++;
+			mask |= ERR_CBMASK_RXFAIL;
+			break;
+		default:
+			break;
+	}
+
+	if (dhd_bus->cbs && dhd_bus->cbs->errhandler && (dhd_bus->errmask & mask))
+		dhd_bus->cbs->errhandler(dhd_bus->cbarg, err);
+}
+
+/**
+ * When lower DBUS level signals control IRB completed, higher level (eg dhd_linux.c) has to be
+ * notified.
+ */
+static void
+dbus_if_ctl_complete(void *handle, int type, int status)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL) {
+		DBUSERR(("%s: dhd_bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (dhd_bus->pub.busstate != DBUS_STATE_DOWN) {
+		if (dhd_bus->cbs && dhd_bus->cbs->ctl_complete)
+			dhd_bus->cbs->ctl_complete(dhd_bus->cbarg, type, status);
+	}
+}
+
+/**
+ * Rx related functionality (flow control, posting of free IRBs to rx queue) is dependent upon the
+ * bus state. When lower DBUS level signals a change in the interface state, take appropriate action
+ * and forward the signaling to the higher (eg dhd_linux.c) level.
+ */
+static void
+dbus_if_state_change(void *handle, int state)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+	int old_state;
+
+	if (dhd_bus == NULL)
+		return;
+
+	if (dhd_bus->pub.busstate == state)
+		return;
+	old_state = dhd_bus->pub.busstate;
+	if (state == DBUS_STATE_DISCONNECT) {
+		DBUSERR(("DBUS disconnected\n"));
+	}
+
+	/* Ignore USB SUSPEND while not up yet */
+	if (state == DBUS_STATE_SLEEP && old_state != DBUS_STATE_UP)
+		return;
+
+	DBUSTRACE(("dbus state change from %d to to %d\n", old_state, state));
+
+	/* Don't update state if it's PnP firmware re-download */
+	if (state != DBUS_STATE_PNP_FWDL)
+		dhd_bus->pub.busstate = state;
+	else
+		dbus_flowctrl_rx(handle, FALSE);
+	if (state == DBUS_STATE_SLEEP)
+		dbus_flowctrl_rx(handle, TRUE);
+	if (state == DBUS_STATE_UP) {
+		dbus_rxirbs_fill(dhd_bus);
+		dbus_flowctrl_rx(handle, FALSE);
+	}
+
+	if (dhd_bus->cbs && dhd_bus->cbs->state_change)
+		dhd_bus->cbs->state_change(dhd_bus->cbarg, state);
+}
+
+/** Forward request for packet from lower DBUS layer to higher layer (e.g. dhd_linux.c) */
+static void *
+dbus_if_pktget(void *handle, uint len, bool send)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+	void *p = NULL;
+
+	if (dhd_bus == NULL)
+		return NULL;
+
+	if (dhd_bus->cbs && dhd_bus->cbs->pktget)
+		p = dhd_bus->cbs->pktget(dhd_bus->cbarg, len, send);
+	else
+		ASSERT(0);
+
+	return p;
+}
+
+/** Forward request to free packet from lower DBUS layer to higher layer (e.g. dhd_linux.c) */
+static void
+dbus_if_pktfree(void *handle, void *p, bool send)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) handle;
+
+	if (dhd_bus == NULL)
+		return;
+
+	if (dhd_bus->cbs && dhd_bus->cbs->pktfree)
+		dhd_bus->cbs->pktfree(dhd_bus->cbarg, p, send);
+	else
+		ASSERT(0);
+}
+
+/** Lower DBUS level requests either a send or receive IRB */
+static struct dbus_irb*
+dbus_if_getirb(void *cbarg, bool send)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) cbarg;
+	struct exec_parms args;
+	struct dbus_irb *irb;
+
+	if ((dhd_bus == NULL) || (dhd_bus->pub.busstate != DBUS_STATE_UP))
+		return NULL;
+
+	if (send == TRUE) {
+		args.qdeq.q = dhd_bus->tx_q;
+		irb = EXEC_TXLOCK(dhd_bus, q_deq_exec, &args);
+	} else {
+		args.qdeq.q = dhd_bus->rx_q;
+		irb = EXEC_RXLOCK(dhd_bus, q_deq_exec, &args);
+	}
+
+	return irb;
+}
+
+/* Register/Unregister functions are called by the main DHD entry
+ * point (e.g. module insertion) to link with the bus driver, in
+ * order to look for or await the device.
+ */
+
+static dbus_driver_t dhd_dbus = {
+	dhd_dbus_probe_cb,
+	dhd_dbus_disconnect_cb,
+	dbus_suspend,
+	dbus_resume
+};
+
+/**
+ * As part of initialization, higher level (eg dhd_linux.c) requests DBUS to prepare for
+ * action.
+ */
+int
+dhd_bus_register(void)
+{
+	int err;
+
+	DBUSTRACE(("%s: Enter\n", __FUNCTION__));
+
+	err = dbus_bus_register(&dhd_dbus, &g_busintf);
+
+	/* Device not detected */
+	if (err == DBUS_ERR_NODEVICE)
+		err = DBUS_OK;
+
+	return err;
+}
+
+dhd_pub_t *g_pub = NULL;
+bool net_attached = FALSE;
+void
+dhd_bus_unregister(void)
+{
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	DHD_MUTEX_LOCK();
+	if (g_pub) {
+		g_pub->dhd_remove = TRUE;
+		if (!g_pub->bus) {
+			dhd_dbus_disconnect_cb(g_pub->bus);
+		}
+	}
+	DHD_MUTEX_UNLOCK();
+	dbus_bus_deregister();
+}
+
+/** As part of initialization, data structures have to be allocated and initialized */
+dhd_bus_t *
+dbus_attach(osl_t *osh, int rxsize, int nrxq, int ntxq, dhd_pub_t *pub,
+	dbus_callbacks_t *cbs, dbus_extdl_t *extdl, struct shared_info *sh)
+{
+	dhd_bus_t *dhd_bus;
+	int err;
+
+	if ((g_busintf == NULL) || (g_busintf->attach == NULL) || (cbs == NULL))
+		return NULL;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if ((nrxq <= 0) || (ntxq <= 0))
+		return NULL;
+
+	dhd_bus = MALLOCZ(osh, sizeof(dhd_bus_t));
+	if (dhd_bus == NULL) {
+		DBUSERR(("%s: malloc failed %zu\n", __FUNCTION__, sizeof(dhd_bus_t)));
+		return NULL;
+	}
+
+	/* BUS-specific driver interface (at a lower DBUS level) */
+	dhd_bus->drvintf = g_busintf;
+	dhd_bus->cbarg = pub;
+	dhd_bus->cbs = cbs;
+
+	dhd_bus->pub.sh = sh;
+	dhd_bus->pub.osh = osh;
+	dhd_bus->pub.rxsize = rxsize;
+
+#ifdef EHCI_FASTPATH_RX
+	atomic_set(&dhd_bus->rx_outstanding, 0);
+#endif
+
+	dhd_bus->pub.nrxq = nrxq;
+	dhd_bus->rx_low_watermark = nrxq / 2;	/* keep enough posted rx urbs */
+	dhd_bus->pub.ntxq = ntxq;
+	dhd_bus->tx_low_watermark = ntxq / 4;	/* flow control when too many tx urbs posted */
+
+	dhd_bus->tx_q = MALLOCZ(osh, sizeof(dbus_irbq_t));
+	if (dhd_bus->tx_q == NULL)
+		goto error;
+	else {
+		err = dbus_irbq_init(dhd_bus, dhd_bus->tx_q, ntxq, sizeof(dbus_irb_tx_t));
+		if (err != DBUS_OK)
+			goto error;
+	}
+
+	dhd_bus->rx_q = MALLOCZ(osh, sizeof(dbus_irbq_t));
+	if (dhd_bus->rx_q == NULL)
+		goto error;
+	else {
+		err = dbus_irbq_init(dhd_bus, dhd_bus->rx_q, nrxq, sizeof(dbus_irb_rx_t));
+		if (err != DBUS_OK)
+			goto error;
+	}
+
+#ifdef BCMDBG
+	dhd_bus->txpend_q_hist = MALLOCZ(osh, dhd_bus->pub.ntxq * sizeof(int));
+	if (dhd_bus->txpend_q_hist == NULL)
+		goto error;
+
+	dhd_bus->rxpend_q_hist = MALLOCZ(osh, dhd_bus->pub.nrxq * sizeof(int));
+	if (dhd_bus->rxpend_q_hist == NULL)
+		goto error;
+#endif /* BCMDBG */
+
+	dhd_bus->bus_info = (void *)g_busintf->attach(&dhd_bus->pub,
+		dhd_bus, &dbus_intf_cbs);
+	if (dhd_bus->bus_info == NULL)
+		goto error;
+
+	dbus_tx_timer_init(dhd_bus);
+
+#if defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW)
+	/* Need to copy external image for re-download */
+	if (extdl && extdl->fw && (extdl->fwlen > 0)) {
+		dhd_bus->extdl.fw = MALLOCZ(osh, extdl->fwlen);
+		if (dhd_bus->extdl.fw) {
+			bcopy(extdl->fw, dhd_bus->extdl.fw, extdl->fwlen);
+			dhd_bus->extdl.fwlen = extdl->fwlen;
+		}
+	}
+
+	if (extdl && extdl->vars && (extdl->varslen > 0)) {
+		dhd_bus->extdl.vars = MALLOCZ(osh, extdl->varslen);
+		if (dhd_bus->extdl.vars) {
+			bcopy(extdl->vars, dhd_bus->extdl.vars, extdl->varslen);
+			dhd_bus->extdl.varslen = extdl->varslen;
+		}
+	}
+#endif /* defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW) */
+
+	return (dhd_bus_t *)dhd_bus;
+
+error:
+	DBUSERR(("%s: Failed\n", __FUNCTION__));
+	dbus_detach(dhd_bus);
+	return NULL;
+} /* dbus_attach */
+
+static void
+dbus_detach(dhd_bus_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	osl_t *osh;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return;
+
+	dbus_tx_timer_stop(dhd_bus);
+
+	osh = pub->pub.osh;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->detach)
+		 dhd_bus->drvintf->detach((dbus_pub_t *)dhd_bus, dhd_bus->bus_info);
+
+	if (dhd_bus->tx_q) {
+		dbus_irbq_deinit(dhd_bus, dhd_bus->tx_q, sizeof(dbus_irb_tx_t));
+		MFREE(osh, dhd_bus->tx_q, sizeof(dbus_irbq_t));
+		dhd_bus->tx_q = NULL;
+	}
+
+	if (dhd_bus->rx_q) {
+		dbus_irbq_deinit(dhd_bus, dhd_bus->rx_q, sizeof(dbus_irb_rx_t));
+		MFREE(osh, dhd_bus->rx_q, sizeof(dbus_irbq_t));
+		dhd_bus->rx_q = NULL;
+	}
+
+#ifdef BCMDBG
+	if (dhd_bus->txpend_q_hist)
+		MFREE(osh, dhd_bus->txpend_q_hist, dhd_bus->pub.ntxq * sizeof(int));
+	if (dhd_bus->rxpend_q_hist)
+		MFREE(osh, dhd_bus->rxpend_q_hist, dhd_bus->pub.nrxq * sizeof(int));
+#endif /* BCMDBG */
+
+	if (dhd_bus->extdl.fw && (dhd_bus->extdl.fwlen > 0)) {
+		MFREE(osh, dhd_bus->extdl.fw, dhd_bus->extdl.fwlen);
+		dhd_bus->extdl.fw = NULL;
+		dhd_bus->extdl.fwlen = 0;
+	}
+
+	if (dhd_bus->extdl.vars && (dhd_bus->extdl.varslen > 0)) {
+		MFREE(osh, dhd_bus->extdl.vars, dhd_bus->extdl.varslen);
+		dhd_bus->extdl.vars = NULL;
+		dhd_bus->extdl.varslen = 0;
+	}
+
+	MFREE(osh, dhd_bus, sizeof(dhd_bus_t));
+} /* dbus_detach */
+
+int dbus_dlneeded(dhd_bus_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int dlneeded = DBUS_ERR;
+
+	if (!dhd_bus) {
+		DBUSERR(("%s: dhd_bus is NULL\n", __FUNCTION__));
+		return DBUS_ERR;
+	}
+
+	DBUSTRACE(("%s: state %d\n", __FUNCTION__, dhd_bus->pub.busstate));
+
+	if (dhd_bus->drvintf->dlneeded) {
+		dlneeded = dhd_bus->drvintf->dlneeded(dhd_bus->bus_info);
+	}
+	printf("%s: dlneeded=%d\n", __FUNCTION__, dlneeded);
+
+	/* dlneeded > 0: need to download
+	  * dlneeded = 0: downloaded
+	  * dlneeded < 0: bus error*/
+	return dlneeded;
+}
+
+#if (defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW))
+int dbus_download_firmware(dhd_bus_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_OK;
+
+	if (!dhd_bus) {
+		DBUSERR(("%s: dhd_bus is NULL\n", __FUNCTION__));
+		return DBUS_ERR;
+	}
+
+	DBUSTRACE(("%s: state %d\n", __FUNCTION__, dhd_bus->pub.busstate));
+
+	dhd_bus->pub.busstate = DBUS_STATE_DL_PENDING;
+	err = dbus_do_download(dhd_bus);
+	if (err == DBUS_OK) {
+		dhd_bus->pub.busstate = DBUS_STATE_DL_DONE;
+	} else {
+		DBUSERR(("%s: download failed (%d)\n", __FUNCTION__, err));
+	}
+
+	return err;
+}
+#endif /* BCM_DNGL_EMBEDIMAGE || BCM_REQUEST_FW */
+
+/**
+ * higher layer requests us to 'up' the interface to the dongle. Prerequisite is that firmware (not
+ * bootloader) must be active in the dongle.
+ */
+int
+dbus_up(struct dhd_bus *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_OK;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL) {
+		DBUSERR(("%s: dhd_bus is NULL\n", __FUNCTION__));
+		return DBUS_ERR;
+	}
+
+	if ((dhd_bus->pub.busstate == DBUS_STATE_DL_DONE) ||
+		(dhd_bus->pub.busstate == DBUS_STATE_DOWN) ||
+		(dhd_bus->pub.busstate == DBUS_STATE_SLEEP)) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->up) {
+			err = dhd_bus->drvintf->up(dhd_bus->bus_info);
+
+			if (err == DBUS_OK) {
+				dbus_rxirbs_fill(dhd_bus);
+			}
+		}
+	} else
+		err = DBUS_ERR;
+
+	return err;
+}
+
+/** higher layer requests us to 'down' the interface to the dongle. */
+int
+dbus_down(dbus_pub_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	dbus_tx_timer_stop(dhd_bus);
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP ||
+		dhd_bus->pub.busstate == DBUS_STATE_SLEEP) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->down)
+			return dhd_bus->drvintf->down(dhd_bus->bus_info);
+	}
+
+	return DBUS_ERR;
+}
+
+int
+dbus_shutdown(dbus_pub_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->shutdown)
+		return dhd_bus->drvintf->shutdown(dhd_bus->bus_info);
+
+	return DBUS_OK;
+}
+
+int
+dbus_stop(struct dhd_bus *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP ||
+		dhd_bus->pub.busstate == DBUS_STATE_SLEEP) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->stop)
+			return dhd_bus->drvintf->stop(dhd_bus->bus_info);
+	}
+
+	return DBUS_ERR;
+}
+
+int
+dbus_send_buf(dbus_pub_t *pub, uint8 *buf, int len, void *info)
+{
+	return dbus_send_irb(pub, buf, len, NULL, info);
+}
+
+static int
+dbus_send_pkt(dbus_pub_t *pub, void *pkt, void *info)
+{
+	return dbus_send_irb(pub, NULL, 0, pkt, info);
+}
+
+static int
+dbus_send_ctl(struct dhd_bus *pub, uint8 *buf, int len)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if (dhd_bus == NULL) {
+		DBUSERR(("%s: dhd_bus is NULL\n", __FUNCTION__));
+		return DBUS_ERR;
+	}
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP ||
+		dhd_bus->pub.busstate == DBUS_STATE_SLEEP) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->send_ctl)
+			return dhd_bus->drvintf->send_ctl(dhd_bus->bus_info, buf, len);
+	} else {
+		DBUSERR(("%s: bustate=%d\n", __FUNCTION__, dhd_bus->pub.busstate));
+	}
+
+	return DBUS_ERR;
+}
+
+static int
+dbus_recv_ctl(struct dhd_bus *pub, uint8 *buf, int len)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if ((dhd_bus == NULL) || (buf == NULL))
+		return DBUS_ERR;
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP ||
+		dhd_bus->pub.busstate == DBUS_STATE_SLEEP) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->recv_ctl)
+			return dhd_bus->drvintf->recv_ctl(dhd_bus->bus_info, buf, len);
+	} else {
+		DBUSERR(("%s: bustate=%d\n", __FUNCTION__, dhd_bus->pub.busstate));
+	}
+
+	return DBUS_ERR;
+}
+
+/** Only called via RPC (Dec 2012) */
+int
+dbus_recv_bulk(dbus_pub_t *pub, uint32 ep_idx)
+{
+#ifdef EHCI_FASTPATH_RX
+	/* 2nd bulk in not supported for EHCI_FASTPATH_RX */
+	ASSERT(0);
+#else
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	dbus_irb_rx_t *rxirb;
+	struct exec_parms args;
+	int status;
+
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	args.qdeq.q = dhd_bus->rx_q;
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->recv_irb_from_ep) {
+			if ((rxirb = (EXEC_RXLOCK(dhd_bus, q_deq_exec, &args))) != NULL) {
+				status = dhd_bus->drvintf->recv_irb_from_ep(dhd_bus->bus_info,
+					rxirb, ep_idx);
+				if (status == DBUS_ERR_RXDROP) {
+					bzero(rxirb, sizeof(dbus_irb_rx_t));
+					args.qenq.q = dhd_bus->rx_q;
+					args.qenq.b = (dbus_irb_t *) rxirb;
+					EXEC_RXLOCK(dhd_bus, q_enq_exec, &args);
+				}
+			}
+		}
+	}
+#endif /* EHCI_FASTPATH_RX */
+
+	return DBUS_ERR; /* ME DVV: This does not seem right :-) */
+}
+
+#ifdef INTR_EP_ENABLE
+/** only called by dhd_cdc.c (Dec 2012) */
+static int
+dbus_poll_intr(dbus_pub_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	int status = DBUS_ERR;
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP) {
+		if (dhd_bus->drvintf && dhd_bus->drvintf->recv_irb_from_ep) {
+			status = dhd_bus->drvintf->recv_irb_from_ep(dhd_bus->bus_info,
+				NULL, 0xff);
+		}
+	}
+	return status;
+}
+#endif /* INTR_EP_ENABLE */
+
+/** called by nobody (Dec 2012) */
+void *
+dbus_pktget(dbus_pub_t *pub, int len)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if ((dhd_bus == NULL) || (len < 0))
+		return NULL;
+
+	return PKTGET(dhd_bus->pub.osh, len, TRUE);
+}
+
+/** called by nobody (Dec 2012) */
+void
+dbus_pktfree(dbus_pub_t *pub, void* pkt)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if ((dhd_bus == NULL) || (pkt == NULL))
+		return;
+
+	PKTFREE(dhd_bus->pub.osh, pkt, TRUE);
+}
+
+/** called by nobody (Dec 2012) */
+int
+dbus_get_stats(dbus_pub_t *pub, dbus_stats_t *stats)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if ((dhd_bus == NULL) || (stats == NULL))
+		return DBUS_ERR;
+
+	bcopy(&dhd_bus->pub.stats, stats, sizeof(dbus_stats_t));
+
+	return DBUS_OK;
+}
+
+int
+dbus_get_attrib(dhd_bus_t *pub, dbus_attrib_t *attrib)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_ERR;
+
+	if ((dhd_bus == NULL) || (attrib == NULL))
+		return DBUS_ERR;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->get_attrib) {
+		err = dhd_bus->drvintf->get_attrib(dhd_bus->bus_info,
+		&dhd_bus->pub.attrib);
+	}
+
+	bcopy(&dhd_bus->pub.attrib, attrib, sizeof(dbus_attrib_t));
+	return err;
+}
+
+int
+dbus_get_device_speed(dbus_pub_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+
+	if (dhd_bus == NULL)
+		return INVALID_SPEED;
+
+	return (dhd_bus->pub.device_speed);
+}
+
+int
+dbus_set_config(dbus_pub_t *pub, dbus_config_t *config)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_ERR;
+
+	if ((dhd_bus == NULL) || (config == NULL))
+		return DBUS_ERR;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->set_config) {
+		err = dhd_bus->drvintf->set_config(dhd_bus->bus_info,
+			config);
+
+		if ((config->config_id == DBUS_CONFIG_ID_AGGR_LIMIT) &&
+			(!err) &&
+			(dhd_bus->pub.busstate == DBUS_STATE_UP)) {
+			dbus_rxirbs_fill(dhd_bus);
+		}
+	}
+
+	return err;
+}
+
+int
+dbus_get_config(dbus_pub_t *pub, dbus_config_t *config)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_ERR;
+
+	if ((dhd_bus == NULL) || (config == NULL))
+		return DBUS_ERR;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->get_config) {
+		err = dhd_bus->drvintf->get_config(dhd_bus->bus_info,
+		config);
+	}
+
+	return err;
+}
+
+int
+dbus_set_errmask(dbus_pub_t *pub, uint32 mask)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_OK;
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	dhd_bus->errmask = mask;
+	return err;
+}
+
+int
+dbus_pnp_resume(dbus_pub_t *pub, int *fw_reload)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_ERR;
+	bool fwdl = FALSE;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (dhd_bus->pub.busstate == DBUS_STATE_UP) {
+		return DBUS_OK;
+	}
+
+#if defined(BCM_DNGL_EMBEDIMAGE)
+	if (dhd_bus->drvintf->device_exists &&
+		dhd_bus->drvintf->device_exists(dhd_bus->bus_info)) {
+		if (dhd_bus->drvintf->dlneeded) {
+			if (dhd_bus->drvintf->dlneeded(dhd_bus->bus_info)) {
+				err = dbus_do_download(dhd_bus);
+				if (err == DBUS_OK) {
+					fwdl = TRUE;
+				}
+				if (dhd_bus->pub.busstate == DBUS_STATE_DL_DONE)
+					dbus_up(&dhd_bus->pub);
+			}
+		}
+	} else {
+		return DBUS_ERR;
+	}
+#endif /* BCM_DNGL_EMBEDIMAGE */
+
+
+	if (dhd_bus->drvintf->pnp) {
+		err = dhd_bus->drvintf->pnp(dhd_bus->bus_info,
+			DBUS_PNP_RESUME);
+	}
+
+	if (dhd_bus->drvintf->recv_needed) {
+		if (dhd_bus->drvintf->recv_needed(dhd_bus->bus_info)) {
+			/* Refill after sleep/hibernate */
+			dbus_rxirbs_fill(dhd_bus);
+		}
+	}
+
+#if defined(BCM_DNGL_EMBEDIMAGE)
+	if (fwdl == TRUE) {
+		dbus_if_state_change(dhd_bus, DBUS_STATE_PNP_FWDL);
+	}
+#endif /* BCM_DNGL_EMBEDIMAGE */
+
+	if (fw_reload)
+		*fw_reload = fwdl;
+
+	return err;
+} /* dbus_pnp_resume */
+
+int
+dbus_pnp_sleep(dbus_pub_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_ERR;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	dbus_tx_timer_stop(dhd_bus);
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->pnp) {
+		err = dhd_bus->drvintf->pnp(dhd_bus->bus_info,
+			DBUS_PNP_SLEEP);
+	}
+
+	return err;
+}
+
+int
+dbus_pnp_disconnect(dbus_pub_t *pub)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) pub;
+	int err = DBUS_ERR;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	dbus_tx_timer_stop(dhd_bus);
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->pnp) {
+		err = dhd_bus->drvintf->pnp(dhd_bus->bus_info,
+			DBUS_PNP_DISCONNECT);
+	}
+
+	return err;
+}
+
+int
+dhd_bus_iovar_op(dhd_pub_t *dhdp, const char *name,
+	void *params, int plen, void *arg, int len, bool set)
+{
+	dhd_bus_t *dhd_bus = (dhd_bus_t *) dhdp->bus;
+	int err = DBUS_ERR;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (dhd_bus == NULL)
+		return DBUS_ERR;
+
+	if (dhd_bus->drvintf && dhd_bus->drvintf->iovar_op) {
+		err = dhd_bus->drvintf->iovar_op(dhd_bus->bus_info,
+			name, params, plen, arg, len, set);
+	}
+
+	return err;
+}
+
+#ifdef BCMDBG
+void
+dbus_hist_dump(dbus_pub_t *pub, struct bcmstrbuf *b)
+{
+	int i = 0, j = 0;
+	dbus_info_t *dbus_info = (dbus_info_t *) pub;
+
+	bcm_bprintf(b, "\nDBUS histogram\n");
+	bcm_bprintf(b, "txq\n");
+	for (i = 0; i < dbus_info->pub.ntxq; i++) {
+		if (dbus_info->txpend_q_hist[i]) {
+			bcm_bprintf(b, "%d: %d ", i, dbus_info->txpend_q_hist[i]);
+			j++;
+			if (j % 10 == 0) {
+				bcm_bprintf(b, "\n");
+			}
+		}
+	}
+
+	j = 0;
+	bcm_bprintf(b, "\nrxq\n");
+	for (i = 0; i < dbus_info->pub.nrxq; i++) {
+		if (dbus_info->rxpend_q_hist[i]) {
+			bcm_bprintf(b, "%d: %d ", i, dbus_info->rxpend_q_hist[i]);
+			j++;
+			if (j % 10 == 0) {
+				bcm_bprintf(b, "\n");
+			}
+		}
+	}
+	bcm_bprintf(b, "\n");
+
+	if (dbus_info->drvintf && dbus_info->drvintf->dump)
+		dbus_info->drvintf->dump(dbus_info->bus_info, b);
+}
+#endif /* BCMDBG */
+
+void *
+dhd_dbus_txq(const dbus_pub_t *pub)
+{
+	return NULL;
+}
+
+uint
+dhd_dbus_hdrlen(const dbus_pub_t *pub)
+{
+	return 0;
+}
+
+void *
+dbus_get_devinfo(dbus_pub_t *pub)
+{
+	return pub->dev_info;
+}
+
+#if (defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW)) && !defined(EXTERNAL_FW_PATH)
+static int
+dbus_select_nvram(dhd_bus_t *dhd_bus, int8 *jumbonvram, int jumbolen,
+uint16 boardtype, uint16 boardrev, int8 **nvram, int *nvram_len)
+{
+	/* Multi board nvram file format is contenation of nvram info with \r
+	*  The file format for two contatenated set is
+	*  \nBroadcom Jumbo Nvram file\nfirst_set\nsecond_set\nthird_set\n
+	*/
+	uint8 *nvram_start = NULL, *nvram_end = NULL;
+	uint8 *nvram_start_prev = NULL, *nvram_end_prev = NULL;
+	uint16 btype = 0, brev = 0;
+	int len  = 0;
+	char *field;
+
+	*nvram = NULL;
+	*nvram_len = 0;
+
+	if (strncmp(BCM_JUMBO_START, jumbonvram, strlen(BCM_JUMBO_START))) {
+		/* single nvram file in the native format */
+		DBUSTRACE(("%s: Non-Jumbo NVRAM File \n", __FUNCTION__));
+		*nvram = jumbonvram;
+		*nvram_len = jumbolen;
+		return DBUS_OK;
+	} else {
+		DBUSTRACE(("%s: Jumbo NVRAM File \n", __FUNCTION__));
+	}
+
+	/* sanity test the end of the config sets for proper ending */
+	if (jumbonvram[jumbolen - 1] != BCM_JUMBO_NVRAM_DELIMIT ||
+		jumbonvram[jumbolen - 2] != '\0') {
+		DBUSERR(("%s: Bad Jumbo NVRAM file format\n", __FUNCTION__));
+		return DBUS_JUMBO_BAD_FORMAT;
+	}
+
+	dhd_bus->nvram_nontxt = DBUS_NVRAM_NONTXT;
+
+	nvram_start = jumbonvram;
+
+	while (*nvram_start != BCM_JUMBO_NVRAM_DELIMIT && len < jumbolen) {
+
+		/* consume the  first file info line
+		* \nBroadcom Jumbo Nvram file\nfile1\n ...
+		*/
+		len ++;
+		nvram_start ++;
+	}
+
+	nvram_end = nvram_start;
+
+	/* search for "boardrev=0xabcd" and "boardtype=0x1234" information in
+	* the concatenated nvram config files /sets
+	*/
+
+	while (len < jumbolen) {
+
+		if (*nvram_end == '\0') {
+			/* end of a config set is marked by multiple null characters */
+			len ++;
+			nvram_end ++;
+			DBUSTRACE(("%s: NULL chr len = %d char = 0x%x\n", __FUNCTION__,
+				len, *nvram_end));
+			continue;
+
+		} else if (*nvram_end == BCM_JUMBO_NVRAM_DELIMIT) {
+
+			/* config set delimiter is reached */
+			/* check if next config set is present or not
+			*  return  if next config is not present
+			*/
+
+			/* start search the next config set */
+			nvram_start_prev = nvram_start;
+			nvram_end_prev = nvram_end;
+
+			nvram_end ++;
+			nvram_start = nvram_end;
+			btype = brev = 0;
+			DBUSTRACE(("%s: going to next record len = %d "
+					"char = 0x%x \n", __FUNCTION__, len, *nvram_end));
+			len ++;
+			if (len >= jumbolen) {
+
+				*nvram = nvram_start_prev;
+				*nvram_len = (int)(nvram_end_prev - nvram_start_prev);
+
+				DBUSTRACE(("%s: no more len = %d nvram_end = 0x%p",
+					__FUNCTION__, len, nvram_end));
+
+				return DBUS_JUMBO_NOMATCH;
+
+			} else {
+				continue;
+			}
+
+		} else {
+
+			DBUSTRACE(("%s: config str = %s\n", __FUNCTION__, nvram_end));
+
+			if (bcmp(nvram_end, "boardtype", strlen("boardtype")) == 0) {
+
+				field = strchr(nvram_end, '=');
+				field++;
+				btype = (uint16)bcm_strtoul(field, NULL, 0);
+
+				DBUSTRACE(("%s: btype = 0x%x boardtype = 0x%x \n", __FUNCTION__,
+					btype, boardtype));
+			}
+
+			if (bcmp(nvram_end, "boardrev", strlen("boardrev")) == 0) {
+
+				field = strchr(nvram_end, '=');
+				field++;
+				brev = (uint16)bcm_strtoul(field, NULL, 0);
+
+				DBUSTRACE(("%s: brev = 0x%x boardrev = 0x%x \n", __FUNCTION__,
+					brev, boardrev));
+			}
+			if (btype == boardtype && brev == boardrev) {
+				/* locate nvram config set end - ie.find '\r' char */
+				while (*nvram_end != BCM_JUMBO_NVRAM_DELIMIT)
+					nvram_end ++;
+				*nvram = nvram_start;
+				*nvram_len = (int) (nvram_end - nvram_start);
+				DBUSTRACE(("found len = %d nvram_start = 0x%p "
+					"nvram_end = 0x%p\n", *nvram_len, nvram_start, nvram_end));
+				return DBUS_OK;
+			}
+
+			len += (strlen(nvram_end) + 1);
+			nvram_end += (strlen(nvram_end) + 1);
+		}
+	}
+	return DBUS_JUMBO_NOMATCH;
+} /* dbus_select_nvram */
+
+#endif
+
+#if defined(BCM_DNGL_EMBEDIMAGE)
+
+/* store the global osh handle */
+static osl_t *osl_handle = NULL;
+
+/** this function is a combination of trx.c and bcmdl.c plus dbus adaptation */
+static int
+dbus_zlib_decomp(dhd_bus_t *dhd_bus)
+{
+
+	int method, flags, len, status;
+	unsigned int uncmp_len, uncmp_crc, dec_crc, crc_init;
+	struct trx_header *trx, *newtrx;
+	unsigned char *file = NULL;
+	unsigned char gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
+	z_stream d_stream;
+	unsigned char unused;
+	int actual_len = -1;
+	unsigned char *headers;
+	unsigned int trxhdrsize, nvramsize, decomp_memsize, i;
+
+	(void)actual_len;
+	(void)unused;
+	(void)crc_init;
+
+	osl_handle = dhd_bus->pub.osh;
+	dhd_bus->orig_fw = NULL;
+
+	headers = dhd_bus->fw;
+	/* Extract trx header */
+	trx = (struct trx_header *)headers;
+	trxhdrsize = sizeof(struct trx_header);
+
+	if (ltoh32(trx->magic) != TRX_MAGIC) {
+		DBUSERR(("%s: Error: trx bad hdr %x\n", __FUNCTION__,
+			ltoh32(trx->magic)));
+		return -1;
+	}
+
+	headers += sizeof(struct trx_header);
+
+	if (ltoh32(trx->flag_version) & TRX_UNCOMP_IMAGE) {
+		actual_len = ltoh32(trx->offsets[TRX_OFFSETS_DLFWLEN_IDX]) +
+		                     sizeof(struct trx_header);
+		DBUSERR(("%s: not a compressed image\n", __FUNCTION__));
+		return 0;
+	} else {
+		/* Extract the gzip header info */
+		if ((*headers++ != gz_magic[0]) || (*headers++ != gz_magic[1])) {
+			DBUSERR(("%s: Error: gzip bad hdr\n", __FUNCTION__));
+			return -1;
+		}
+
+		method = (int) *headers++;
+		flags = (int) *headers++;
+
+		if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
+			DBUSERR(("%s: Error: gzip bad hdr not a Z_DEFLATED file\n", __FUNCTION__));
+			return -1;
+		}
+	}
+
+	/* Discard time, xflags and OS code: */
+	for (len = 0; len < 6; len++)
+		unused = *headers++;
+
+	if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
+		len = (uint32) *headers++;
+		len += ((uint32)*headers++)<<8;
+		/* len is garbage if EOF but the loop below will quit anyway */
+		while (len-- != 0) unused = *headers++;
+	}
+
+	if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
+		while (*headers++ && (*headers != 0));
+	}
+
+	if ((flags & COMMENT) != 0) {   /* skip the .gz file comment */
+		while (*headers++ && (*headers != 0));
+	}
+
+	if ((flags & HEAD_CRC) != 0) {  /* skip the header crc */
+		for (len = 0; len < 2; len++) unused = *headers++;
+	}
+
+	headers++;	/* I need this, why ? */
+
+	/* create space for the uncompressed file */
+	/* the space is for trx header, uncompressed image  and nvram file */
+	/* with typical compression of 0.6, space double of firmware should be ok */
+
+	decomp_memsize = dhd_bus->fwlen * 2;
+	dhd_bus->decomp_memsize = decomp_memsize;
+	if (!(file = MALLOCZ(osl_handle, decomp_memsize))) {
+		DBUSERR(("%s: check_file : failed malloc\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* Initialise the decompression struct */
+	d_stream.next_in = NULL;
+	d_stream.avail_in = 0;
+	d_stream.next_out = NULL;
+	d_stream.avail_out = decomp_memsize - trxhdrsize;
+	d_stream.zalloc = (alloc_func)0;
+	d_stream.zfree = (free_func)0;
+	if (inflateInit2(&d_stream, -15) != Z_OK) {
+		DBUSERR(("%s: Err: inflateInit2\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* Inflate the code */
+	d_stream.next_in = headers;
+	d_stream.avail_in = ltoh32(trx->len);
+	d_stream.next_out = (unsigned char*)(file + trxhdrsize);
+
+	status = inflate(&d_stream, Z_SYNC_FLUSH);
+
+	if (status != Z_STREAM_END)	{
+		DBUSERR(("%s: Error: decompression failed\n", __FUNCTION__));
+		goto err;
+	}
+
+	uncmp_crc = *d_stream.next_in++;
+	uncmp_crc |= *d_stream.next_in++<<8;
+	uncmp_crc |= *d_stream.next_in++<<16;
+	uncmp_crc |= *d_stream.next_in++<<24;
+
+	uncmp_len = *d_stream.next_in++;
+	uncmp_len |= *d_stream.next_in++<<8;
+	uncmp_len |= *d_stream.next_in++<<16;
+	uncmp_len |= *d_stream.next_in++<<24;
+
+	actual_len = (int) (d_stream.next_in - (unsigned char *)trx);
+
+	inflateEnd(&d_stream);
+
+	/* Do a CRC32 on the uncompressed data */
+	crc_init = crc32(0L, Z_NULL, 0);
+	dec_crc = crc32(crc_init, file + trxhdrsize, uncmp_len);
+
+	if (dec_crc != uncmp_crc) {
+		DBUSERR(("%s: decompression: bad crc check \n", __FUNCTION__));
+		goto err;
+	}
+	else {
+		DBUSTRACE(("%s: decompression: good crc check \n", __FUNCTION__));
+	}
+
+	/* rebuild the new trx header and calculate crc */
+	newtrx = (struct trx_header *)file;
+	newtrx->magic = trx->magic;
+	/* add the uncompressed image flag */
+	newtrx->flag_version = trx->flag_version;
+	newtrx->flag_version  |= htol32(TRX_UNCOMP_IMAGE);
+	newtrx->offsets[TRX_OFFSETS_DLFWLEN_IDX] = htol32(uncmp_len);
+	newtrx->offsets[TRX_OFFSETS_JUMPTO_IDX] = trx->offsets[TRX_OFFSETS_JUMPTO_IDX];
+	newtrx->offsets[TRX_OFFSETS_NVM_LEN_IDX] = trx->offsets[TRX_OFFSETS_NVM_LEN_IDX];
+
+	nvramsize = ltoh32(trx->offsets[TRX_OFFSETS_NVM_LEN_IDX]);
+
+	/* the original firmware has nvram file appended */
+	/* copy the nvram file to uncompressed firmware */
+
+	if (nvramsize) {
+		if (nvramsize + uncmp_len > decomp_memsize) {
+			DBUSERR(("%s: nvram cannot be accomodated\n", __FUNCTION__));
+			goto err;
+		}
+		bcopy(d_stream.next_in, &file[uncmp_len], nvramsize);
+		uncmp_len += nvramsize;
+	}
+
+	/* add trx header size to uncmp_len */
+	uncmp_len += trxhdrsize;
+	/* PR14373 WAR: Pad to multiple of 4096 bytes */
+	uncmp_len = ROUNDUP(uncmp_len, 4096);
+	newtrx->len	= htol32(uncmp_len);
+
+	/* Calculate CRC over header */
+	newtrx->crc32 = hndcrc32((uint8 *)&newtrx->flag_version,
+	sizeof(struct trx_header) - OFFSETOF(struct trx_header, flag_version),
+	CRC32_INIT_VALUE);
+
+	/* Calculate CRC over data */
+	for (i = trxhdrsize; i < (uncmp_len); ++i)
+				newtrx->crc32 = hndcrc32((uint8 *)&file[i], 1, newtrx->crc32);
+	newtrx->crc32 = htol32(newtrx->crc32);
+
+	dhd_bus->orig_fw = dhd_bus->fw;
+	dhd_bus->origfw_len = dhd_bus->fwlen;
+	dhd_bus->image = dhd_bus->fw = file;
+	dhd_bus->image_len = dhd_bus->fwlen = uncmp_len;
+
+	return 0;
+
+err:
+	if (file)
+		free(file);
+	return -1;
+} /* dbus_zlib_decomp */
+
+void *
+dbus_zlib_calloc(int num, int size)
+{
+	uint *ptr;
+	uint totalsize;
+
+	if (osl_handle == NULL)
+		return NULL;
+
+	totalsize = (num * (size + 1));
+
+	ptr  = MALLOCZ(osl_handle, totalsize);
+
+	if (ptr == NULL)
+		return NULL;
+
+	/* store the size in the first integer space */
+
+	ptr[0] = totalsize;
+
+	return ((void *) &ptr[1]);
+}
+
+void
+dbus_zlib_free(void *ptr)
+{
+	uint totalsize;
+	uchar *memptr = (uchar *)ptr;
+
+	if (ptr && osl_handle) {
+		memptr -= sizeof(uint);
+		totalsize = *(uint *) memptr;
+		MFREE(osl_handle, memptr, totalsize);
+	}
+}
+
+#endif /* #if defined(BCM_DNGL_EMBEDIMAGE) */
+
+#define DBUS_NRXQ	50
+#define DBUS_NTXQ	100
+
+static void
+dhd_dbus_send_complete(void *handle, void *info, int status)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+	void *pkt = info;
+
+	if ((dhd == NULL) || (pkt == NULL)) {
+		DBUSERR(("dhd or pkt is NULL\n"));
+		return;
+	}
+
+	if (status == DBUS_OK) {
+		dhd->dstats.tx_packets++;
+	} else {
+		DBUSERR(("TX error=%d\n", status));
+		dhd->dstats.tx_errors++;
+	}
+#ifdef PROP_TXSTATUS
+	if (DHD_PKTTAG_WLFCPKT(PKTTAG(pkt)) &&
+		(dhd_wlfc_txcomplete(dhd, pkt, status == 0) != WLFC_UNSUPPORTED)) {
+		return;
+	} else
+#endif /* PROP_TXSTATUS */
+	dhd_txcomplete(dhd, pkt, status == 0);
+	PKTFREE(dhd->osh, pkt, TRUE);
+}
+
+static void
+dhd_dbus_recv_pkt(void *handle, void *pkt)
+{
+	uchar reorder_info_buf[WLHOST_REORDERDATA_TOTLEN];
+	uint reorder_info_len;
+	uint pkt_count;
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+	int ifidx = 0;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	/* If the protocol uses a data header, check and remove it */
+	if (dhd_prot_hdrpull(dhd, &ifidx, pkt, reorder_info_buf,
+		&reorder_info_len) != 0) {
+		DBUSERR(("rx protocol error\n"));
+		PKTFREE(dhd->osh, pkt, FALSE);
+		dhd->rx_errors++;
+		return;
+	}
+
+	if (reorder_info_len) {
+		/* Reordering info from the firmware */
+		dhd_process_pkt_reorder_info(dhd, reorder_info_buf, reorder_info_len,
+			&pkt, &pkt_count);
+		if (pkt_count == 0)
+			return;
+	}
+	else {
+		pkt_count = 1;
+	}
+	dhd_rx_frame(dhd, ifidx, pkt, pkt_count, 0);
+}
+
+static void
+dhd_dbus_recv_buf(void *handle, uint8 *buf, int len)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+	void *pkt;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if ((pkt = PKTGET(dhd->osh, len, FALSE)) == NULL) {
+		DBUSERR(("PKTGET (rx) failed=%d\n", len));
+		return;
+	}
+
+	bcopy(buf, PKTDATA(dhd->osh, pkt), len);
+	dhd_dbus_recv_pkt(dhd, pkt);
+}
+
+static void
+dhd_dbus_txflowcontrol(void *handle, bool onoff)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+	bool wlfc_enabled = FALSE;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+#ifdef PROP_TXSTATUS
+	wlfc_enabled = (dhd_wlfc_flowcontrol(dhd, onoff, !onoff) != WLFC_UNSUPPORTED);
+#endif
+
+	if (!wlfc_enabled) {
+		dhd_txflowcontrol(dhd, ALL_INTERFACES, onoff);
+	}
+}
+
+static void
+dhd_dbus_errhandler(void *handle, int err)
+{
+}
+
+static void
+dhd_dbus_ctl_complete(void *handle, int type, int status)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (type == DBUS_CBCTL_READ) {
+		if (status == DBUS_OK)
+			dhd->rx_ctlpkts++;
+		else
+			dhd->rx_ctlerrs++;
+	} else if (type == DBUS_CBCTL_WRITE) {
+		if (status == DBUS_OK)
+			dhd->tx_ctlpkts++;
+		else
+			dhd->tx_ctlerrs++;
+	}
+
+	dhd->bus->ctl_completed = TRUE;
+	dhd_os_ioctl_resp_wake(dhd);
+}
+
+static void
+dhd_dbus_state_change(void *handle, int state)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+	unsigned long flags;
+	wifi_adapter_info_t *adapter;
+	int wowl_dngldown = 0;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+	adapter = (wifi_adapter_info_t *)dhd->adapter;
+#ifdef WL_EXT_WOWL
+	wowl_dngldown = dhd_conf_wowl_dngldown(dhd);
+#endif
+
+	if ((dhd->busstate == DHD_BUS_SUSPEND && state == DBUS_STATE_DOWN) ||
+			(dhd->hostsleep && wowl_dngldown)) {
+		DBUSERR(("%s: switch state %d to %d\n", __FUNCTION__, state, DBUS_STATE_SLEEP));
+		state = DBUS_STATE_SLEEP;
+	}
+	switch (state) {
+		case DBUS_STATE_DL_NEEDED:
+			DBUSERR(("%s: firmware request cannot be handled\n", __FUNCTION__));
+			break;
+		case DBUS_STATE_DOWN:
+			DHD_LINUX_GENERAL_LOCK(dhd, flags);
+			dhd_txflowcontrol(dhd, ALL_INTERFACES, ON);
+			DBUSTRACE(("%s: DBUS is down\n", __FUNCTION__));
+			dhd->busstate = DHD_BUS_DOWN;
+			DHD_LINUX_GENERAL_UNLOCK(dhd, flags);
+			break;
+		case DBUS_STATE_UP:
+			DBUSTRACE(("%s: DBUS is up\n", __FUNCTION__));
+			DHD_LINUX_GENERAL_LOCK(dhd, flags);
+			dhd_txflowcontrol(dhd, ALL_INTERFACES, OFF);
+			dhd->busstate = DHD_BUS_DATA;
+			DHD_LINUX_GENERAL_UNLOCK(dhd, flags);
+			break;
+		default:
+			break;
+	}
+
+	DBUSERR(("%s: DBUS current state=%d\n", __FUNCTION__, state));
+}
+
+static void *
+dhd_dbus_pktget(void *handle, uint len, bool send)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+	void *p = NULL;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return NULL;
+	}
+
+	if (send == TRUE) {
+		dhd_os_sdlock_txq(dhd);
+		p = PKTGET(dhd->osh, len, TRUE);
+		dhd_os_sdunlock_txq(dhd);
+	} else {
+		dhd_os_sdlock_rxq(dhd);
+		p = PKTGET(dhd->osh, len, FALSE);
+		dhd_os_sdunlock_rxq(dhd);
+	}
+
+	return p;
+}
+
+static void
+dhd_dbus_pktfree(void *handle, void *p, bool send)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)handle;
+
+	if (dhd == NULL) {
+		DBUSERR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (send == TRUE) {
+#ifdef PROP_TXSTATUS
+		if (DHD_PKTTAG_WLFCPKT(PKTTAG(p)) &&
+			(dhd_wlfc_txcomplete(dhd, p, FALSE) != WLFC_UNSUPPORTED)) {
+			return;
+		}
+#endif /* PROP_TXSTATUS */
+
+		dhd_os_sdlock_txq(dhd);
+		PKTFREE(dhd->osh, p, TRUE);
+		dhd_os_sdunlock_txq(dhd);
+	} else {
+		dhd_os_sdlock_rxq(dhd);
+		PKTFREE(dhd->osh, p, FALSE);
+		dhd_os_sdunlock_rxq(dhd);
+	}
+}
+
+
+static dbus_callbacks_t dhd_dbus_cbs = {
+	dhd_dbus_send_complete,
+	dhd_dbus_recv_buf,
+	dhd_dbus_recv_pkt,
+	dhd_dbus_txflowcontrol,
+	dhd_dbus_errhandler,
+	dhd_dbus_ctl_complete,
+	dhd_dbus_state_change,
+	dhd_dbus_pktget,
+	dhd_dbus_pktfree
+};
+
+uint
+dhd_bus_chip(struct dhd_bus *bus)
+{
+	ASSERT(bus != NULL);
+	return bus->pub.attrib.devid;
+}
+
+uint
+dhd_bus_chiprev(struct dhd_bus *bus)
+{
+	ASSERT(bus);
+	ASSERT(bus != NULL);
+	return bus->pub.attrib.chiprev;
+}
+
+struct device *
+dhd_bus_to_dev(struct dhd_bus *bus)
+{
+	return dbus_get_dev();
+}
+
+void
+dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	bcm_bprintf(strbuf, "Bus USB\n");
+}
+
+void
+dhd_bus_clearcounts(dhd_pub_t *dhdp)
+{
+}
+
+int
+dhd_bus_txdata(struct dhd_bus *bus, void *pktbuf)
+{
+	DBUSTRACE(("%s\n", __FUNCTION__));
+	if (bus->txoff) {
+		DBUSTRACE(("txoff\n"));
+		return BCME_EPERM;
+	}
+	return dbus_send_pkt(&bus->pub, pktbuf, pktbuf);
+}
+
+int
+dhd_bus_txctl(struct dhd_bus *bus, uchar *msg, uint msglen)
+{
+	int timeleft = 0;
+	int ret = -1;
+
+	DBUSTRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd->dongle_reset)
+		return -EIO;
+
+	bus->ctl_completed = FALSE;
+	ret = dbus_send_ctl(bus, (void *)msg, msglen);
+	if (ret) {
+		DBUSERR(("%s: dbus_send_ctl error %d\n", __FUNCTION__, ret));
+		return ret;
+	}
+
+	timeleft = dhd_os_ioctl_resp_wait(bus->dhd, &bus->ctl_completed);
+	if ((!timeleft) || (!bus->ctl_completed)) {
+		DBUSERR(("%s: Txctl timeleft %d ctl_completed %d\n",
+			__FUNCTION__, timeleft, bus->ctl_completed));
+		ret = -1;
+#ifdef OEM_ANDROID
+		bus->dhd->hang_reason = HANG_REASON_IOCTL_RESP_TIMEOUT_SCHED_ERROR;
+		dhd_os_send_hang_message(bus->dhd);
+#endif /* OEM_ANDROID */
+	}
+
+#ifdef INTR_EP_ENABLE
+	/* If the ctl write is successfully completed, wait for an acknowledgement
+	* that indicates that it is now ok to do ctl read from the dongle
+	*/
+	if (ret != -1) {
+		bus->ctl_completed = FALSE;
+		if (dbus_poll_intr(bus->pub)) {
+			DBUSERR(("%s: dbus_poll_intr not submitted\n", __FUNCTION__));
+		} else {
+			/* interrupt polling is sucessfully submitted. Wait for dongle to send
+			* interrupt
+			*/
+			timeleft = dhd_os_ioctl_resp_wait(bus->dhd, &bus->ctl_completed);
+			if (!timeleft) {
+				DBUSERR(("%s: intr poll wait timed out\n", __FUNCTION__));
+			}
+		}
+	}
+#endif /* INTR_EP_ENABLE */
+
+	return ret;
+}
+
+int
+dhd_bus_rxctl(struct dhd_bus *bus, uchar *msg, uint msglen)
+{
+	int timeleft;
+	int ret = -1;
+
+	DBUSTRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd->dongle_reset)
+		return -EIO;
+
+	bus->ctl_completed = FALSE;
+	ret = dbus_recv_ctl(bus, (uchar*)msg, msglen);
+	if (ret) {
+		DBUSERR(("%s: dbus_recv_ctl error %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+	timeleft = dhd_os_ioctl_resp_wait(bus->dhd, &bus->ctl_completed);
+	if ((!timeleft) || (!bus->ctl_completed)) {
+		DBUSERR(("%s: Rxctl timeleft %d ctl_completed %d\n", __FUNCTION__,
+			timeleft, bus->ctl_completed));
+		ret = -ETIMEDOUT;
+		goto done;
+	}
+
+	/* XXX FIX: Must return cdc_len, not len, because after query_ioctl()
+	 * it subtracts sizeof(cdc_ioctl_t);  The other approach is
+	 * to have dbus_recv_ctl() return actual len.
+	 */
+	ret = msglen;
+
+done:
+	return ret;
+}
+
+static void
+dhd_dbus_advertise_bus_cleanup(dhd_pub_t *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+	dhdp->busstate = DHD_BUS_DOWN_IN_PROGRESS;
+	DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+	if ((timeleft == 0) || (timeleft == 1)) {
+		DBUSERR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+		ASSERT(0);
+	}
+
+	return;
+}
+
+static void
+dhd_dbus_advertise_bus_remove(dhd_pub_t *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+	if (dhdp->busstate != DHD_BUS_SUSPEND)
+		dhdp->busstate = DHD_BUS_REMOVE;
+	DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+	if ((timeleft == 0) || (timeleft == 1)) {
+		DBUSERR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+		ASSERT(0);
+	}
+
+	return;
+}
+
+int
+dhd_bus_devreset(dhd_pub_t *dhdp, uint8 flag)
+{
+	int bcmerror = 0;
+	unsigned long flags;
+	wifi_adapter_info_t *adapter = (wifi_adapter_info_t *)dhdp->adapter;
+
+	if (flag == TRUE) {
+		if (!dhdp->dongle_reset) {
+			DBUSINFO(("%s: == Power OFF ==\n", __FUNCTION__));
+			dhd_dbus_advertise_bus_cleanup(dhdp);
+			dhd_os_wd_timer(dhdp, 0);
+#if !defined(IGNORE_ETH0_DOWN)
+			/* Force flow control as protection when stop come before ifconfig_down */
+			dhd_txflowcontrol(dhdp, ALL_INTERFACES, ON);
+#endif /* !defined(IGNORE_ETH0_DOWN) */
+			dbus_stop(dhdp->bus);
+
+			dhdp->dongle_reset = TRUE;
+			dhdp->up = FALSE;
+
+			DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+			dhdp->busstate = DHD_BUS_DOWN;
+			DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+			wifi_clr_adapter_status(adapter, WIFI_STATUS_FW_READY);
+
+			DBUSINFO("%s:  WLAN OFF DONE\n", __FUNCTION__);
+			/* App can now remove power from device */
+		} else
+			bcmerror = BCME_ERROR;
+	} else {
+		/* App must have restored power to device before calling */
+		DBUSINFO("\n\n%s: == WLAN ON ==\n", __FUNCTION__);
+		if (dhdp->dongle_reset) {
+			/* Turn on WLAN */
+			DHD_MUTEX_UNLOCK();
+			wait_event_interruptible_timeout(adapter->status_event,
+				wifi_get_adapter_status(adapter, WIFI_STATUS_FW_READY),
+				msecs_to_jiffies(DHD_FW_READY_TIMEOUT));
+			DHD_MUTEX_LOCK();
+			bcmerror = dbus_up(dhdp->bus);
+			if (bcmerror == BCME_OK) {
+				dhdp->dongle_reset = FALSE;
+				dhdp->up = TRUE;
+#if !defined(IGNORE_ETH0_DOWN)
+				/* Restore flow control  */
+				dhd_txflowcontrol(dhdp, ALL_INTERFACES, OFF);
+#endif
+				dhd_os_wd_timer(dhdp, dhd_watchdog_ms);
+
+				DBUSINFO(("%s: WLAN ON DONE\n", __FUNCTION__));
+			} else {
+				DBUSERR(("%s: failed to dbus_up with code %d\n", __FUNCTION__, bcmerror));
+			}
+		}
+	}
+
+	return bcmerror;
+}
+
+void
+dhd_bus_update_fw_nv_path(struct dhd_bus *bus, char *pfw_path,
+	char *pnv_path, char *pclm_path, char *pconf_path)
+{
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (bus == NULL) {
+		DBUSERR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	bus->fw_path = pfw_path;
+	bus->nv_path = pnv_path;
+	bus->dhd->clm_path = pclm_path;
+	bus->dhd->conf_path = pconf_path;
+
+	dhd_conf_set_path_params(bus->dhd, bus->fw_path, bus->nv_path);
+
+}
+
+static int
+dhd_dbus_sync_dongle(dhd_pub_t *pub, int dlneeded)
+{
+	int ret = 0;
+
+	if (dlneeded == 0) {
+		ret = dbus_up(pub->bus);
+		if (ret) {
+			DBUSERR(("%s: dbus_up failed!!\n", __FUNCTION__));
+			goto exit;
+		}
+		ret = dhd_sync_with_dongle(pub);
+		if (ret < 0) {
+			DBUSERR(("%s: failed with code ret=%d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+	}
+
+exit:
+	return ret;
+}
+
+static int
+dbus_suspend(void *context)
+{
+	int ret = 0;
+
+#if defined(LINUX)
+	dhd_bus_t *bus = (dhd_bus_t*)context;
+	unsigned long flags;
+
+	DBUSERR(("%s Enter\n", __FUNCTION__));
+	if (bus->dhd == NULL) {
+		DBUSERR(("bus not inited\n"));
+		return BCME_ERROR;
+	}
+	if (bus->dhd->prot == NULL) {
+		DBUSERR(("prot is not inited\n"));
+		return BCME_ERROR;
+	}
+
+	if (bus->dhd->up == FALSE) {
+		return BCME_OK;
+	}
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	if (bus->dhd->busstate != DHD_BUS_DATA && bus->dhd->busstate != DHD_BUS_SUSPEND) {
+		DBUSERR(("not in a readystate to LPBK  is not inited\n"));
+		DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+		return BCME_ERROR;
+	}
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+	if (bus->dhd->dongle_reset) {
+		DBUSERR(("Dongle is in reset state.\n"));
+		return -EIO;
+	}
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	/* stop all interface network queue. */
+	dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, ON);
+	bus->dhd->busstate = DHD_BUS_SUSPEND;
+#if defined(LINUX) || defined(linux)
+	if (DHD_BUS_BUSY_CHECK_IN_TX(bus->dhd)) {
+		DBUSERR(("Tx Request is not ended\n"));
+		bus->dhd->busstate = DHD_BUS_DATA;
+		/* resume all interface network queue. */
+		dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+		DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+		return -EBUSY;
+	}
+#endif /* LINUX || linux */
+	DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+	ret = dhd_os_check_wakelock_all(bus->dhd);
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	if (ret) {
+		bus->dhd->busstate = DHD_BUS_DATA;
+		/* resume all interface network queue. */
+		dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+	} else {
+		bus->last_suspend_end_time = OSL_LOCALTIME_NS();
+	}
+	bus->dhd->hostsleep = 2;
+	DHD_BUS_BUSY_CLEAR_SUSPEND_IN_PROGRESS(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+#endif /* LINUX */
+	DBUSERR(("%s Exit ret=%d\n", __FUNCTION__, ret));
+	return ret;
+}
+
+static int
+dbus_resume(void *context)
+{
+	dhd_bus_t *bus = (dhd_bus_t*)context;
+	ulong flags;
+	int dlneeded = 0;
+	int ret = 0;
+
+	DBUSERR(("%s Enter\n", __FUNCTION__));
+
+	if (bus->dhd->up == FALSE) {
+		return BCME_OK;
+	}
+
+	dlneeded = dbus_dlneeded(bus);
+	if (dlneeded == 0) {
+		ret = dbus_up(bus);
+		if (ret) {
+			DBUSERR(("%s: dbus_up failed!!\n", __FUNCTION__));
+		}
+	}
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_SET_RESUME_IN_PROGRESS(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_RESUME_IN_PROGRESS(bus->dhd);
+	bus->dhd->hostsleep = 0;
+	bus->dhd->busstate = DHD_BUS_DATA;
+	dhd_os_busbusy_wake(bus->dhd);
+	/* resume all interface network queue. */
+	dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+//	dhd_conf_set_suspend_resume(bus->dhd, 0);
+
+	return 0;
+}
+
+/*
+ * hdrlen is space to reserve in pkt headroom for DBUS
+ */
+static void *
+dhd_dbus_probe_cb(uint16 bus_no, uint16 slot, uint32 hdrlen)
+{
+	osl_t *osh = NULL;
+	dhd_bus_t *bus = NULL;
+	dhd_pub_t *pub = NULL;
+	uint rxsz;
+	int dlneeded = 0, ret = DBUS_OK;
+	wifi_adapter_info_t *adapter = NULL;
+	bool net_attach_now = TRUE;
+
+	DBUSTRACE(("%s: Enter\n", __FUNCTION__));
+
+	adapter = dhd_wifi_platform_get_adapter(USB_BUS, bus_no, slot);
+
+	if (!g_pub) {
+		/* Ask the OS interface part for an OSL handle */
+		if (!(osh = osl_attach(NULL, USB_BUS, TRUE))) {
+			DBUSERR(("%s: OSL attach failed\n", __FUNCTION__));
+			goto fail;
+		}
+
+		/* Attach to the dhd/OS interface */
+		if (!(pub = dhd_attach(osh, bus, hdrlen, adapter))) {
+			DBUSERR(("%s: dhd_attach failed\n", __FUNCTION__));
+			goto fail;
+		}
+	} else {
+		pub = g_pub;
+		osh = pub->osh;
+	}
+
+	if (pub->bus) {
+		DBUSERR(("%s: wrong probe\n", __FUNCTION__));
+		goto fail;
+	}
+
+	rxsz = dhd_get_rxsz(pub);
+	bus = dbus_attach(osh, rxsz, DBUS_NRXQ, DBUS_NTXQ, pub, &dhd_dbus_cbs, NULL, NULL);
+	if (bus) {
+		pub->bus = bus;
+		bus->dhd = pub;
+
+		dlneeded = dbus_dlneeded(bus);
+		if (dlneeded >= 0 && !g_pub) {
+			dhd_conf_reset(pub);
+			dhd_conf_set_chiprev(pub, bus->pub.attrib.devid, bus->pub.attrib.chiprev);
+			dhd_conf_preinit(pub);
+		}
+
+#if defined(BCMDHD_MODULAR) && defined(INSMOD_FW_LOAD)
+		if (1)
+#else
+		if (g_pub || dhd_download_fw_on_driverload)
+#endif
+		{
+			if (dlneeded == 0)
+				wifi_set_adapter_status(adapter, WIFI_STATUS_FW_READY);
+#ifdef BCM_REQUEST_FW
+			else if (dlneeded > 0) {
+				struct dhd_conf *conf = pub->conf;
+				unsigned long flags;
+				bool suspended;
+				wifi_clr_adapter_status(adapter, WIFI_STATUS_FW_READY);
+				suspended = conf->suspended;
+				dhd_set_path(bus->dhd);
+				conf->suspended = suspended;
+				if (dbus_download_firmware(bus) != DBUS_OK)
+					goto fail;
+				DHD_LINUX_GENERAL_LOCK(pub, flags);
+				if (bus->dhd->busstate != DHD_BUS_SUSPEND)
+					bus->dhd->busstate = DHD_BUS_LOAD;
+				DHD_LINUX_GENERAL_UNLOCK(pub, flags);
+			}
+#endif
+			else {
+				goto fail;
+			}
+		}
+	}
+	else {
+		DBUSERR(("%s: dbus_attach failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+#if defined(BCMDHD_MODULAR) && defined(INSMOD_FW_LOAD)
+	if (dlneeded > 0)
+		net_attach_now = FALSE;
+#endif
+
+	if (!net_attached && (net_attach_now || (dlneeded == 0))) {
+		if (dhd_dbus_sync_dongle(pub, dlneeded)) {
+			goto fail;
+		}
+		if (dhd_attach_net(bus->dhd, TRUE) != 0) {
+			DBUSERR(("%s: Net attach failed!!\n", __FUNCTION__));
+			goto fail;
+		}
+		pub->hang_report  = TRUE;
+#if defined(MULTIPLE_SUPPLICANT)
+		wl_android_post_init(); // terence 20120530: fix critical section in dhd_open and dhdsdio_probe
+#endif
+		net_attached = TRUE;
+	}
+	else if (net_attached && (pub->up == 1) && (dlneeded == 0)) {
+		// kernel resume case
+		pub->hostsleep = 0;
+		ret = dhd_dbus_sync_dongle(pub, dlneeded);
+#ifdef WL_CFG80211
+		__wl_cfg80211_up_resume(pub);
+		wl_cfgp2p_start_p2p_device_resume(pub);
+#endif
+		dhd_conf_set_suspend_resume(pub, 0);
+		if (ret != DBUS_OK)
+			goto fail;
+	}
+
+	if (!g_pub) {
+		g_pub = pub;
+	}
+
+	DBUSTRACE(("%s: Exit\n", __FUNCTION__));
+	wifi_clr_adapter_status(adapter, WIFI_STATUS_BUS_DISCONNECTED);
+	if (net_attached) {
+		wifi_set_adapter_status(adapter, WIFI_STATUS_NET_ATTACHED);
+		wake_up_interruptible(&adapter->status_event);
+		/* This is passed to dhd_dbus_disconnect_cb */
+	}
+	return bus;
+
+fail:
+	if (pub && pub->bus) {
+		dbus_detach(pub->bus);
+		pub->bus = NULL;
+	}
+	/* Release resources in reverse order */
+	if (!g_pub) {
+		if (pub) {
+			dhd_detach(pub);
+			dhd_free(pub);
+		}
+		if (osh) {
+			osl_detach(osh);
+		}
+	}
+
+	printf("%s: Failed\n", __FUNCTION__);
+	return NULL;
+}
+
+static void
+dhd_dbus_disconnect_cb(void *arg)
+{
+	dhd_bus_t *bus = (dhd_bus_t *)arg;
+	dhd_pub_t *pub = g_pub;
+	osl_t *osh;
+	wifi_adapter_info_t *adapter = NULL;
+
+	adapter = (wifi_adapter_info_t *)pub->adapter;
+
+	if (pub && !pub->dhd_remove && bus == NULL) {
+		DBUSERR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+	if (!adapter) {
+		DBUSERR(("%s: adapter is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	printf("%s: Enter dhd_remove=%d on %s\n", __FUNCTION__,
+		pub->dhd_remove, adapter->name);
+	if (!pub->dhd_remove) {
+		/* Advertise bus remove during rmmod */
+		dhd_dbus_advertise_bus_remove(bus->dhd);
+		dbus_detach(pub->bus);
+		pub->bus = NULL;
+		wifi_set_adapter_status(adapter, WIFI_STATUS_BUS_DISCONNECTED);
+		wake_up_interruptible(&adapter->status_event);
+	} else {
+		osh = pub->osh;
+		dhd_detach(pub);
+		if (pub->bus) {
+			dbus_detach(pub->bus);
+			pub->bus = NULL;
+		}
+		dhd_free(pub);
+		g_pub = NULL;
+		net_attached = FALSE;
+		wifi_clr_adapter_status(adapter, WIFI_STATUS_NET_ATTACHED);
+		if (MALLOCED(osh)) {
+			DBUSERR(("%s: MEMORY LEAK %d bytes\n", __FUNCTION__, MALLOCED(osh)));
+		}
+		osl_detach(osh);
+	}
+
+	DBUSTRACE(("%s: Exit\n", __FUNCTION__));
+}
+
+int
+dhd_bus_sleep(dhd_pub_t *dhdp, bool sleep, uint32 *intstatus)
+{
+	wifi_adapter_info_t *adapter = (wifi_adapter_info_t *)dhdp->adapter;
+	s32 timeout = -1;
+	int err = 0;
+
+	timeout = wait_event_interruptible_timeout(adapter->status_event,
+		wifi_get_adapter_status(adapter, WIFI_STATUS_BUS_DISCONNECTED),
+		msecs_to_jiffies(12000));
+	if (timeout <= 0) {
+		err = -1;
+		DBUSERR(("%s: bus disconnected timeout\n", __FUNCTION__));
+	}
+
+	return err;
+}
+
+#ifdef LINUX_EXTERNAL_MODULE_DBUS
+
+static int __init
+bcm_dbus_module_init(void)
+{
+	printf("Inserting bcm_dbus module \n");
+	return 0;
+}
+
+static void __exit
+bcm_dbus_module_exit(void)
+{
+	printf("Removing bcm_dbus module \n");
+	return;
+}
+
+EXPORT_SYMBOL(dbus_pnp_sleep);
+EXPORT_SYMBOL(dhd_bus_register);
+EXPORT_SYMBOL(dbus_get_devinfo);
+#ifdef BCMDBG
+EXPORT_SYMBOL(dbus_hist_dump);
+#endif
+EXPORT_SYMBOL(dbus_detach);
+EXPORT_SYMBOL(dbus_get_attrib);
+EXPORT_SYMBOL(dbus_down);
+EXPORT_SYMBOL(dbus_pnp_resume);
+EXPORT_SYMBOL(dbus_set_config);
+EXPORT_SYMBOL(dbus_flowctrl_rx);
+EXPORT_SYMBOL(dbus_up);
+EXPORT_SYMBOL(dbus_get_device_speed);
+EXPORT_SYMBOL(dbus_send_pkt);
+EXPORT_SYMBOL(dbus_recv_ctl);
+EXPORT_SYMBOL(dbus_attach);
+EXPORT_SYMBOL(dhd_bus_unregister);
+
+MODULE_LICENSE("GPL");
+
+module_init(bcm_dbus_module_init);
+module_exit(bcm_dbus_module_exit);
+
+#endif  /* #ifdef LINUX_EXTERNAL_MODULE_DBUS */
diff --git a/drivers/net/wireless/ap12275/dbus_usb.c b/drivers/net/wireless/ap12275/dbus_usb.c
new file mode 100644
index 000000000000..28cf3f092f1c
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dbus_usb.c
@@ -0,0 +1,1088 @@
+/*
+ * Dongle BUS interface for USB, OS independent
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+/**
+ * @file @brief
+ * This file contains DBUS code that is USB, but not OS specific. DBUS is a Broadcom proprietary
+ * host specific abstraction layer.
+ */
+
+#include <osl.h>
+#include <bcmdefs.h>
+#include <bcmutils.h>
+#include <dbus.h>
+#include <usbrdl.h>
+#include <bcmdevs_legacy.h>
+#include <bcmdevs.h>
+#include <bcmendian.h>
+
+#if defined(BCM_DNGL_EMBEDIMAGE)
+#ifdef EMBED_IMAGE_43526a
+#include "rtecdc_43526a.h"
+#endif /* EMBED_IMAGE_43526a */
+#ifdef EMBED_IMAGE_43526b
+#include "rtecdc_43526b.h"
+#endif /* EMBED_IMAGE_43526b */
+#ifdef EMBED_IMAGE_GENERIC
+#include "rtecdc.h"
+#endif
+#endif /* BCM_DNGL_EMBEDIMAGE */
+
+uint dbus_msglevel = DBUS_ERROR_VAL;
+module_param(dbus_msglevel, int, 0);
+
+#define USB_DLIMAGE_RETRY_TIMEOUT    3000    /* retry Timeout */
+#define USB_SFLASH_DLIMAGE_SPINWAIT  150     /* in unit of ms */
+#define USB_SFLASH_DLIMAGE_LIMIT     2000    /* spinwait limit (ms) */
+#define POSTBOOT_ID                  0xA123  /* ID to detect if dongle has boot up */
+#define USB_RESETCFG_SPINWAIT        1       /* wait after resetcfg (ms) */
+#define USB_DEV_ISBAD(u)             (u->pub->attrib.devid == 0xDEAD)
+#define USB_DLGO_SPINWAIT            100     /* wait after DL_GO (ms) */
+#define TEST_CHIP                    0x4328
+
+/* driver info, initialized when bcmsdh_register is called */
+static dbus_driver_t drvinfo = {NULL, NULL, NULL, NULL};
+
+typedef struct {
+	dbus_pub_t  *pub;
+
+	void        *cbarg;
+	dbus_intf_callbacks_t *cbs;  /** callbacks into higher DBUS level (dbus.c) */
+	dbus_intf_t *drvintf;
+	void        *usbosl_info;
+	uint32      rdlram_base_addr;
+	uint32      rdlram_size;
+} usb_info_t;
+
+/*
+ * Callbacks common to all USB
+ */
+static void dbus_usb_disconnect(void *handle);
+static void dbus_usb_send_irb_timeout(void *handle, dbus_irb_tx_t *txirb);
+static void dbus_usb_send_irb_complete(void *handle, dbus_irb_tx_t *txirb, int status);
+static void dbus_usb_recv_irb_complete(void *handle, dbus_irb_rx_t *rxirb, int status);
+static void dbus_usb_errhandler(void *handle, int err);
+static void dbus_usb_ctl_complete(void *handle, int type, int status);
+static void dbus_usb_state_change(void *handle, int state);
+static struct dbus_irb* dbus_usb_getirb(void *handle, bool send);
+static void dbus_usb_rxerr_indicate(void *handle, bool on);
+#if !defined(BCM_REQUEST_FW)
+static int dbus_usb_resetcfg(usb_info_t *usbinfo);
+#endif
+static int dbus_usb_iovar_op(void *bus, const char *name,
+	void *params, uint plen, void *arg, uint len, bool set);
+static int dbus_iovar_process(usb_info_t* usbinfo, const char *name,
+                 void *params, int plen, void *arg, uint len, bool set);
+static int dbus_usb_doiovar(usb_info_t *bus, const bcm_iovar_t *vi, uint32 actionid,
+	const char *name, void *params, int plen, void *arg, uint len, int val_size);
+static int dhdusb_downloadvars(usb_info_t *bus, void *arg, int len);
+
+static int dbus_usb_dl_writeimage(usb_info_t *usbinfo, uint8 *fw, int fwlen);
+static int dbus_usb_dlstart(void *bus, uint8 *fw, int len);
+static int dbus_usb_dlneeded(void *bus);
+static int dbus_usb_dlrun(void *bus);
+static int dbus_usb_rdl_dwnld_state(usb_info_t *usbinfo);
+
+#ifdef BCM_DNGL_EMBEDIMAGE
+static bool dbus_usb_device_exists(void *bus);
+#endif
+
+/* OS specific */
+extern bool dbus_usbos_dl_cmd(void *info, uint8 cmd, void *buffer, int buflen);
+extern int dbus_usbos_wait(void *info, uint16 ms);
+extern int dbus_write_membytes(usb_info_t *usbinfo, bool set, uint32 address,
+	uint8 *data, uint size);
+extern bool dbus_usbos_dl_send_bulk(void *info, void *buffer, int len);
+extern int dbus_usbos_loopback_tx(void *usbos_info_ptr, int cnt, int size);
+
+/**
+ * These functions are called by the lower DBUS level (dbus_usb_os.c) to notify this DBUS level
+ * (dbus_usb.c) of an event.
+ */
+static dbus_intf_callbacks_t dbus_usb_intf_cbs = {
+	dbus_usb_send_irb_timeout,
+	dbus_usb_send_irb_complete,
+	dbus_usb_recv_irb_complete,
+	dbus_usb_errhandler,
+	dbus_usb_ctl_complete,
+	dbus_usb_state_change,
+	NULL,			/* isr */
+	NULL,			/* dpc */
+	NULL,			/* watchdog */
+	NULL,			/* dbus_if_pktget */
+	NULL, 			/* dbus_if_pktfree */
+	dbus_usb_getirb,
+	dbus_usb_rxerr_indicate
+};
+
+/* IOVar table */
+enum {
+	IOV_SET_DOWNLOAD_STATE = 1,
+	IOV_DBUS_MSGLEVEL,
+	IOV_MEMBYTES,
+	IOV_VARS,
+	IOV_LOOPBACK_TX
+};
+
+const bcm_iovar_t dhdusb_iovars[] = {
+	{"vars",	IOV_VARS,	0,	0, IOVT_BUFFER,	0 },
+	{"dbus_msglevel",	IOV_DBUS_MSGLEVEL,	0,	0, IOVT_UINT32,	0 },
+	{"dwnldstate",	IOV_SET_DOWNLOAD_STATE,	0,	0, IOVT_BOOL,	0 },
+	{"membytes",	IOV_MEMBYTES,	0,	0, IOVT_BUFFER,	2 * sizeof(int) },
+	{"usb_lb_txfer", IOV_LOOPBACK_TX, 0,    0, IOVT_BUFFER,    2 * sizeof(int) },
+	{NULL, 0, 0, 0, 0, 0 }
+};
+
+/*
+ * Need global for probe() and disconnect() since
+ * attach() is not called at probe and detach()
+ * can be called inside disconnect()
+ */
+static dbus_intf_t	*g_dbusintf = NULL;
+static dbus_intf_t	dbus_usb_intf; /** functions called by higher layer DBUS into lower layer */
+
+/*
+ * dbus_intf_t common to all USB
+ * These functions override dbus_usb_<os>.c.
+ */
+static void *dbus_usb_attach(dbus_pub_t *pub, void *cbarg, dbus_intf_callbacks_t *cbs);
+static void dbus_usb_detach(dbus_pub_t *pub, void *info);
+/* : g_usb_info needed for over-ridden functions
+ * since the bus argument is actually from dbus_usb_<os>.c.
+ */
+static void * dbus_usb_probe(uint16 bus_no, uint16 slot, uint32 hdrlen);
+
+/* functions */
+
+/**
+ * As part of DBUS initialization/registration, the higher level DBUS (dbus.c) needs to know what
+ * lower level DBUS functions to call (in both dbus_usb.c and dbus_usb_os.c).
+ */
+static void *
+dbus_usb_probe(uint16 bus_no, uint16 slot, uint32 hdrlen)
+{
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+	if (drvinfo.probe) {
+		if (g_dbusintf != NULL) {
+			/* First, initialize all lower-level functions as default
+			 * so that dbus.c simply calls directly to dbus_usb_os.c.
+			 */
+			bcopy(g_dbusintf, &dbus_usb_intf, sizeof(dbus_intf_t));
+
+			/* Second, selectively override functions we need, if any. */
+			dbus_usb_intf.attach = dbus_usb_attach;
+			dbus_usb_intf.detach = dbus_usb_detach;
+			dbus_usb_intf.iovar_op = dbus_usb_iovar_op;
+			dbus_usb_intf.dlstart = dbus_usb_dlstart;
+			dbus_usb_intf.dlneeded = dbus_usb_dlneeded;
+			dbus_usb_intf.dlrun = dbus_usb_dlrun;
+#ifdef BCM_DNGL_EMBEDIMAGE
+			dbus_usb_intf.device_exists = dbus_usb_device_exists;
+#endif
+		}
+
+		return drvinfo.probe(bus_no, slot, hdrlen);
+	}
+
+	return NULL;
+}
+
+static int
+dbus_usb_suspend(void *handle)
+{
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+
+	if (drvinfo.suspend)
+		return drvinfo.suspend(handle);
+
+	return BCME_OK;
+}
+
+static int
+dbus_usb_resume(void *handle)
+{
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+
+	if (drvinfo.resume)
+		drvinfo.resume(handle);
+
+	return 0;
+}
+
+static dbus_driver_t dhd_usb_dbus = {
+	dbus_usb_probe,
+	dbus_usb_disconnect,
+	dbus_usb_suspend,
+	dbus_usb_resume
+};
+
+/**
+ * On return, *intf contains this or lower-level DBUS functions to be called by higher
+ * level (dbus.c)
+ */
+int
+dbus_bus_register(dbus_driver_t *driver, dbus_intf_t **intf)
+{
+	int err;
+
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+	drvinfo = *driver;
+	*intf = &dbus_usb_intf;
+
+	err = dbus_bus_osl_register(&dhd_usb_dbus, &g_dbusintf);
+
+	ASSERT(g_dbusintf);
+	return err;
+}
+
+int
+dbus_bus_deregister()
+{
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+	return dbus_bus_osl_deregister();
+}
+
+/** initialization consists of registration followed by 'attach'. */
+void *
+dbus_usb_attach(dbus_pub_t *pub, void *cbarg, dbus_intf_callbacks_t *cbs)
+{
+	usb_info_t *usb_info;
+
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+
+	if ((g_dbusintf == NULL) || (g_dbusintf->attach == NULL))
+		return NULL;
+
+	/* Sanity check for BUS_INFO() */
+	ASSERT(OFFSETOF(usb_info_t, pub) == 0);
+
+	usb_info = MALLOC(pub->osh, sizeof(usb_info_t));
+	if (usb_info == NULL)
+		return NULL;
+
+	bzero(usb_info, sizeof(usb_info_t));
+
+	usb_info->pub = pub;
+	usb_info->cbarg = cbarg;
+	usb_info->cbs = cbs;
+
+	usb_info->usbosl_info = (dbus_pub_t *)g_dbusintf->attach(pub,
+		usb_info, &dbus_usb_intf_cbs);
+	if (usb_info->usbosl_info == NULL) {
+		MFREE(pub->osh, usb_info, sizeof(usb_info_t));
+		return NULL;
+	}
+
+	/* Save USB OS-specific driver entry points */
+	usb_info->drvintf = g_dbusintf;
+
+	pub->bus = usb_info;
+#if !defined(BCM_REQUEST_FW)
+	if (!dbus_usb_resetcfg(usb_info)) {
+		usb_info->pub->busstate = DBUS_STATE_DL_DONE;
+	}
+#endif
+	/* Return Lower layer info */
+	return (void *) usb_info->usbosl_info;
+}
+
+void
+dbus_usb_detach(dbus_pub_t *pub, void *info)
+{
+	usb_info_t *usb_info = (usb_info_t *) pub->bus;
+	osl_t *osh = pub->osh;
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->drvintf && usb_info->drvintf->detach)
+		usb_info->drvintf->detach(pub, usb_info->usbosl_info);
+
+	MFREE(osh, usb_info, sizeof(usb_info_t));
+}
+
+void
+dbus_usb_disconnect(void *handle)
+{
+	DBUSTRACE(("%s(): \n", __FUNCTION__));
+	if (drvinfo.remove)
+		drvinfo.remove(handle);
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_send_irb_timeout(void *handle, dbus_irb_tx_t *txirb)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->cbs && usb_info->cbs->send_irb_timeout)
+		usb_info->cbs->send_irb_timeout(usb_info->cbarg, txirb);
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_send_irb_complete(void *handle, dbus_irb_tx_t *txirb, int status)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->cbs && usb_info->cbs->send_irb_complete)
+		usb_info->cbs->send_irb_complete(usb_info->cbarg, txirb, status);
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_recv_irb_complete(void *handle, dbus_irb_rx_t *rxirb, int status)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->cbs && usb_info->cbs->recv_irb_complete)
+		usb_info->cbs->recv_irb_complete(usb_info->cbarg, rxirb, status);
+}
+
+/** Lower DBUS level (dbus_usb_os.c) requests a free IRB. Pass this on to the higher DBUS level. */
+static struct dbus_irb*
+dbus_usb_getirb(void *handle, bool send)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	if (usb_info == NULL)
+		return NULL;
+
+	if (usb_info->cbs && usb_info->cbs->getirb)
+		return usb_info->cbs->getirb(usb_info->cbarg, send);
+
+	return NULL;
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_rxerr_indicate(void *handle, bool on)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->cbs && usb_info->cbs->rxerr_indicate)
+		usb_info->cbs->rxerr_indicate(usb_info->cbarg, on);
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_errhandler(void *handle, int err)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->cbs && usb_info->cbs->errhandler)
+		usb_info->cbs->errhandler(usb_info->cbarg, err);
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_ctl_complete(void *handle, int type, int status)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usb_info == NULL) {
+		DBUSERR(("%s: usb_info is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (usb_info->cbs && usb_info->cbs->ctl_complete)
+		usb_info->cbs->ctl_complete(usb_info->cbarg, type, status);
+}
+
+/**
+ * When the lower DBUS level (dbus_usb_os.c) signals this event, the higher DBUS level has to be
+ * notified.
+ */
+static void
+dbus_usb_state_change(void *handle, int state)
+{
+	usb_info_t *usb_info = (usb_info_t *) handle;
+
+	if (usb_info == NULL)
+		return;
+
+	if (usb_info->cbs && usb_info->cbs->state_change)
+		usb_info->cbs->state_change(usb_info->cbarg, state);
+}
+
+/** called by higher DBUS level (dbus.c) */
+static int
+dbus_usb_iovar_op(void *bus, const char *name,
+	void *params, uint plen, void *arg, uint len, bool set)
+{
+	int err = DBUS_OK;
+
+	err = dbus_iovar_process((usb_info_t*)bus, name, params, plen, arg, len, set);
+	return err;
+}
+
+/** process iovar request from higher DBUS level */
+static int
+dbus_iovar_process(usb_info_t* usbinfo, const char *name,
+                 void *params, int plen, void *arg, uint len, bool set)
+{
+	const bcm_iovar_t *vi = NULL;
+	int bcmerror = 0;
+	int val_size;
+	uint32 actionid;
+
+	DBUSTRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(name);
+
+	/* Get MUST have return space */
+	ASSERT(set || (arg && len));
+
+	/* Set does NOT take qualifiers */
+	ASSERT(!set || (!params && !plen));
+
+	/* Look up var locally; if not found pass to host driver */
+	if ((vi = bcm_iovar_lookup(dhdusb_iovars, name)) == NULL) {
+		/* Not Supported */
+		bcmerror = BCME_UNSUPPORTED;
+		DBUSTRACE(("%s: IOVAR %s is not supported\n", name, __FUNCTION__));
+		goto exit;
+
+	}
+
+	DBUSTRACE(("%s: %s %s, len %d plen %d\n", __FUNCTION__,
+	         name, (set ? "set" : "get"), len, plen));
+
+	/* set up 'params' pointer in case this is a set command so that
+	 * the convenience int and bool code can be common to set and get
+	 */
+	if (params == NULL) {
+		params = arg;
+		plen = len;
+	}
+
+	if (vi->type == IOVT_VOID)
+		val_size = 0;
+	else if (vi->type == IOVT_BUFFER)
+		val_size = len;
+	else
+		/* all other types are integer sized */
+		val_size = sizeof(int);
+
+	actionid = set ? IOV_SVAL(vi->varid) : IOV_GVAL(vi->varid);
+	bcmerror = dbus_usb_doiovar(usbinfo, vi, actionid,
+		name, params, plen, arg, len, val_size);
+
+exit:
+	return bcmerror;
+} /* dbus_iovar_process */
+
+static int
+dbus_usb_doiovar(usb_info_t *bus, const bcm_iovar_t *vi, uint32 actionid, const char *name,
+                void *params, int plen, void *arg, uint len, int val_size)
+{
+	int bcmerror = 0;
+	int32 int_val = 0;
+	int32 int_val2 = 0;
+	bool bool_val = 0;
+
+	DBUSTRACE(("%s: Enter, action %d name %s params %p plen %d arg %p len %d val_size %d\n",
+	           __FUNCTION__, actionid, name, params, plen, arg, len, val_size));
+
+	if ((bcmerror = bcm_iovar_lencheck(vi, arg, len, IOV_ISSET(actionid))) != 0)
+		goto exit;
+
+	if (plen >= (int)sizeof(int_val))
+		bcopy(params, &int_val, sizeof(int_val));
+
+	if (plen >= (int)sizeof(int_val) * 2)
+		bcopy((void*)((uintptr)params + sizeof(int_val)), &int_val2, sizeof(int_val2));
+
+	bool_val = (int_val != 0) ? TRUE : FALSE;
+
+	switch (actionid) {
+
+	case IOV_SVAL(IOV_MEMBYTES):
+	case IOV_GVAL(IOV_MEMBYTES):
+	{
+		uint32 address;
+		uint size, dsize;
+		uint8 *data;
+
+		bool set = (actionid == IOV_SVAL(IOV_MEMBYTES));
+
+		ASSERT(plen >= 2*sizeof(int));
+
+		address = (uint32)int_val;
+		BCM_REFERENCE(address);
+		bcopy((char *)params + sizeof(int_val), &int_val, sizeof(int_val));
+		size = (uint)int_val;
+
+		/* Do some validation */
+		dsize = set ? plen - (2 * sizeof(int)) : len;
+		if (dsize < size) {
+			DBUSTRACE(("%s: error on %s membytes, addr 0x%08x size %d dsize %d\n",
+			           __FUNCTION__, (set ? "set" : "get"), address, size, dsize));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		DBUSTRACE(("%s: Request to %s %d bytes at address 0x%08x\n", __FUNCTION__,
+		          (set ? "write" : "read"), size, address));
+
+		/* Generate the actual data pointer */
+		data = set ? (uint8*)params + 2 * sizeof(int): (uint8*)arg;
+
+		/* Call to do the transfer */
+		bcmerror = dbus_usb_dl_writeimage(BUS_INFO(bus, usb_info_t), data, size);
+	}
+		break;
+
+	case IOV_SVAL(IOV_SET_DOWNLOAD_STATE):
+
+		if (bool_val == TRUE) {
+			bcmerror = dbus_usb_dlneeded(bus);
+			dbus_usb_rdl_dwnld_state(BUS_INFO(bus, usb_info_t));
+		} else {
+			usb_info_t *usbinfo = BUS_INFO(bus, usb_info_t);
+			bcmerror = dbus_usb_dlrun(bus);
+			usbinfo->pub->busstate = DBUS_STATE_DL_DONE;
+		}
+		break;
+
+	case IOV_SVAL(IOV_VARS):
+		bcmerror = dhdusb_downloadvars(BUS_INFO(bus, usb_info_t), arg, len);
+		break;
+
+	case IOV_GVAL(IOV_DBUS_MSGLEVEL):
+		int_val = (int32)dbus_msglevel;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DBUS_MSGLEVEL):
+		dbus_msglevel = int_val;
+		break;
+
+#ifdef DBUS_USB_LOOPBACK
+	case IOV_SVAL(IOV_LOOPBACK_TX):
+			bcmerror = dbus_usbos_loopback_tx(BUS_INFO(bus, usb_info_t), int_val,
+			  int_val2);
+			break;
+#endif
+	default:
+		bcmerror = BCME_UNSUPPORTED;
+		break;
+	}
+
+exit:
+	return bcmerror;
+} /* dbus_usb_doiovar */
+
+/** higher DBUS level (dbus.c) wants to set NVRAM variables in dongle */
+static int
+dhdusb_downloadvars(usb_info_t *bus, void *arg, int len)
+{
+	int bcmerror = 0;
+	uint32 varsize;
+	uint32 varaddr;
+	uint32 varsizew;
+
+	if (!len) {
+		bcmerror = BCME_BUFTOOSHORT;
+		goto err;
+	}
+
+	/* RAM size is not set. Set it at dbus_usb_dlneeded */
+	if (!bus->rdlram_size)
+		bcmerror = BCME_ERROR;
+
+	/* Even if there are no vars are to be written, we still need to set the ramsize. */
+	varsize = len ? ROUNDUP(len, 4) : 0;
+	varaddr = (bus->rdlram_size - 4) - varsize;
+
+	/* Write the vars list */
+	DBUSTRACE(("WriteVars: @%x varsize=%d\n", varaddr, varsize));
+	bcmerror = dbus_write_membytes(bus->usbosl_info, TRUE, (varaddr + bus->rdlram_base_addr),
+		arg, varsize);
+
+	/* adjust to the user specified RAM */
+	DBUSTRACE(("Usable memory size: %d\n", bus->rdlram_size));
+	DBUSTRACE(("Vars are at %d, orig varsize is %d\n", varaddr, varsize));
+
+	varsize = ((bus->rdlram_size - 4) - varaddr);
+
+	/*
+	 * Determine the length token:
+	 * Varsize, converted to words, in lower 16-bits, checksum in upper 16-bits.
+	 */
+	if (bcmerror) {
+		varsizew = 0;
+	} else {
+		varsizew = varsize / 4;
+		varsizew = (~varsizew << 16) | (varsizew & 0x0000FFFF);
+		varsizew = htol32(varsizew);
+	}
+
+	DBUSTRACE(("New varsize is %d, length token=0x%08x\n", varsize, varsizew));
+
+	/* Write the length token to the last word */
+	bcmerror = dbus_write_membytes(bus->usbosl_info, TRUE, ((bus->rdlram_size - 4) +
+		bus->rdlram_base_addr), (uint8*)&varsizew, 4);
+err:
+	return bcmerror;
+} /* dbus_usb_doiovar */
+
+#if !defined(BCM_REQUEST_FW)
+/**
+ * After downloading firmware into dongle and starting it, we need to know if the firmware is
+ * indeed up and running.
+ */
+static int
+dbus_usb_resetcfg(usb_info_t *usbinfo)
+{
+	void *osinfo;
+	bootrom_id_t id;
+	uint16 waittime = 0;
+
+	uint32 starttime = 0;
+	uint32 endtime = 0;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	osinfo = usbinfo->usbosl_info;
+	ASSERT(osinfo);
+
+	/* Give dongle chance to boot */
+	dbus_usbos_wait(osinfo, USB_SFLASH_DLIMAGE_SPINWAIT);
+	waittime = USB_SFLASH_DLIMAGE_SPINWAIT;
+	while (waittime < USB_DLIMAGE_RETRY_TIMEOUT) {
+
+		starttime = OSL_SYSUPTIME();
+
+		id.chip = 0xDEAD;       /* Get the ID */
+		dbus_usbos_dl_cmd(osinfo, DL_GETVER, &id, sizeof(bootrom_id_t));
+		id.chip = ltoh32(id.chip);
+
+		endtime = OSL_SYSUPTIME();
+		waittime += (endtime - starttime);
+
+		if (id.chip == POSTBOOT_ID)
+			break;
+	}
+
+	if (id.chip == POSTBOOT_ID) {
+		DBUSERR(("%s: download done. Bootup time = %d ms postboot chip 0x%x/rev 0x%x\n",
+			__FUNCTION__, waittime, id.chip, id.chiprev));
+
+		dbus_usbos_dl_cmd(osinfo, DL_RESETCFG, &id, sizeof(bootrom_id_t));
+
+		/* this wait may not be necessary */
+		dbus_usbos_wait(osinfo, USB_RESETCFG_SPINWAIT);
+		return DBUS_OK;
+	} else {
+		DBUSERR(("%s: Cannot talk to Dongle. Wait time = %d ms. Firmware is not UP \n",
+			__FUNCTION__, waittime));
+		return DBUS_ERR;
+	}
+
+	return DBUS_OK;
+}
+#endif
+
+/** before firmware download, the dongle has to be prepared to receive the fw image */
+static int
+dbus_usb_rdl_dwnld_state(usb_info_t *usbinfo)
+{
+	void *osinfo = usbinfo->usbosl_info;
+	rdl_state_t state;
+	int err = DBUS_OK;
+
+	/* 1) Prepare USB boot loader for runtime image */
+	dbus_usbos_dl_cmd(osinfo, DL_START, &state, sizeof(rdl_state_t));
+
+	state.state = ltoh32(state.state);
+	state.bytes = ltoh32(state.bytes);
+
+	/* 2) Check we are in the Waiting state */
+	if (state.state != DL_WAITING) {
+		DBUSERR(("%s: Failed to DL_START\n", __FUNCTION__));
+		err = DBUS_ERR;
+		goto fail;
+	}
+
+fail:
+	return err;
+}
+
+/**
+ * Dongle contains bootcode in ROM but firmware is (partially) contained in dongle RAM. Therefore,
+ * firmware has to be downloaded into dongle RAM.
+ */
+static int
+dbus_usb_dl_writeimage(usb_info_t *usbinfo, uint8 *fw, int fwlen)
+{
+	osl_t *osh = usbinfo->pub->osh;
+	void *osinfo = usbinfo->usbosl_info;
+	unsigned int sendlen, sent, dllen;
+	char *bulkchunk = NULL, *dlpos;
+	rdl_state_t state;
+	int err = DBUS_OK;
+	uint32 dl_trunk_size = RDL_CHUNK;
+
+	while (!bulkchunk) {
+		bulkchunk = MALLOC(osh, dl_trunk_size);
+		if (dl_trunk_size == RDL_CHUNK)
+			break;
+		if (!bulkchunk) {
+			dl_trunk_size /= 2;
+			if (dl_trunk_size < RDL_CHUNK)
+				dl_trunk_size = RDL_CHUNK;
+		}
+	}
+
+	if (bulkchunk == NULL) {
+		err = DBUS_ERR;
+		goto fail;
+	}
+
+	sent = 0;
+	dlpos = fw;
+	dllen = fwlen;
+
+	DBUSTRACE(("enter %s: fwlen=%d\n", __FUNCTION__, fwlen));
+
+	dbus_usbos_dl_cmd(osinfo, DL_GETSTATE, &state, sizeof(rdl_state_t));
+
+	/* 3) Load the image */
+	while ((sent < dllen)) {
+		/* Wait until the usb device reports it received all the bytes we sent */
+
+		/* add a timeout so we dont wait indefinetly */
+		if (sent < dllen) {
+			if ((dllen-sent) < dl_trunk_size)
+				sendlen = dllen-sent;
+			else
+				sendlen = dl_trunk_size;
+
+			/* simply avoid having to send a ZLP by ensuring we never have an even
+			 * multiple of 64
+			 */
+			if (!(sendlen % 64))
+				sendlen -= 4;
+
+			/* send data */
+			memcpy(bulkchunk, dlpos, sendlen);
+			if (!dbus_usbos_dl_send_bulk(osinfo, bulkchunk, sendlen)) {
+				err = DBUS_ERR;
+				goto fail;
+			}
+
+			dlpos += sendlen;
+			sent += sendlen;
+			DBUSTRACE(("%s: sendlen %d\n", __FUNCTION__, sendlen));
+		}
+
+		/* Wait for downloaded image crc check to complete in the dongle */
+		while (!dbus_usbos_dl_cmd(osinfo, DL_GETSTATE, &state, sizeof(rdl_state_t))) {
+			DBUSERR(("%s: DL_GETSTATE Failed xxxx\n", __FUNCTION__));
+			err = DBUS_ERR;
+			goto fail;
+		}
+
+		state.state = ltoh32(state.state);
+		state.bytes = ltoh32(state.bytes);
+
+		/* restart if an error is reported */
+		if ((state.state == DL_BAD_HDR) || (state.state == DL_BAD_CRC)) {
+			DBUSERR(("%s: Bad Hdr or Bad CRC\n", __FUNCTION__));
+			err = DBUS_ERR;
+			goto fail;
+		}
+
+	}
+fail:
+	if (bulkchunk)
+		MFREE(osh, bulkchunk, dl_trunk_size);
+
+	return err;
+} /* dbus_usb_dl_writeimage */
+
+/** Higher level DBUS layer (dbus.c) requests this layer to download image into dongle */
+static int
+dbus_usb_dlstart(void *bus, uint8 *fw, int len)
+{
+	usb_info_t *usbinfo = BUS_INFO(bus, usb_info_t);
+	int err;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	if (USB_DEV_ISBAD(usbinfo))
+		return DBUS_ERR;
+
+	err = dbus_usb_rdl_dwnld_state(usbinfo);
+
+	if (DBUS_OK == err) {
+		err = dbus_usb_dl_writeimage(usbinfo, fw, len);
+		if (err == DBUS_OK)
+			usbinfo->pub->busstate = DBUS_STATE_DL_DONE;
+		else
+			usbinfo->pub->busstate = DBUS_STATE_DL_PENDING;
+	} else
+		usbinfo->pub->busstate = DBUS_STATE_DL_PENDING;
+
+	return err;
+}
+
+static bool
+dbus_usb_update_chipinfo(usb_info_t *usbinfo, uint32 chip)
+{
+	bool retval = TRUE;
+	/* based on the CHIP Id, store the ram size which is needed for NVRAM download. */
+	switch (chip) {
+		case 0x4360:
+		case 0xAA06:
+			usbinfo->rdlram_size = RDL_RAM_SIZE_4360;
+			usbinfo->rdlram_base_addr = RDL_RAM_BASE_4360;
+			break;
+		case 43242:
+			usbinfo->rdlram_size = RDL_RAM_SIZE_43242;
+			usbinfo->rdlram_base_addr = RDL_RAM_BASE_43242;
+			break;
+		case 43143:
+			usbinfo->rdlram_size = RDL_RAM_SIZE_43143;
+			usbinfo->rdlram_base_addr = RDL_RAM_BASE_43143;
+			break;
+		case 43569:
+			usbinfo->rdlram_size = RDL_RAM_SIZE_4350;
+			usbinfo->rdlram_base_addr = RDL_RAM_BASE_4350;
+			break;
+		case BCM4381_CHIP_ID:
+			usbinfo->rdlram_size = RDL_RAM_SIZE_4381;
+			usbinfo->rdlram_base_addr = RDL_RAM_BASE_4381;
+			break;
+
+		case BCM4382_CHIP_ID:
+			usbinfo->rdlram_size = RDL_RAM_SIZE_4382;
+			usbinfo->rdlram_base_addr = RDL_RAM_BASE_4382;
+			break;
+
+		case POSTBOOT_ID:
+			break;
+
+		default:
+			DBUSERR(("%s: Chip 0x%x Ram size is not known\n", __FUNCTION__, chip));
+			retval = FALSE;
+			break;
+
+	}
+
+	return retval;
+} /* dbus_usb_update_chipinfo */
+
+/** higher DBUS level (dbus.c) wants to know if firmware download is required. */
+static int
+dbus_usb_dlneeded(void *bus)
+{
+	usb_info_t *usbinfo = BUS_INFO(bus, usb_info_t);
+	void *osinfo;
+	bootrom_id_t id;
+	int dl_needed = 1;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	osinfo = usbinfo->usbosl_info;
+	ASSERT(osinfo);
+
+	/* Check if firmware downloaded already by querying runtime ID */
+	id.chip = 0xDEAD;
+	dbus_usbos_dl_cmd(osinfo, DL_GETVER, &id, sizeof(bootrom_id_t));
+
+	id.chip = ltoh32(id.chip);
+	id.chiprev = ltoh32(id.chiprev);
+
+	if (FALSE == dbus_usb_update_chipinfo(usbinfo, id.chip)) {
+		dl_needed = DBUS_ERR;
+		goto exit;
+	}
+
+	DBUSERR(("%s: chip 0x%x rev 0x%x\n", __FUNCTION__, id.chip, id.chiprev));
+	if (id.chip == POSTBOOT_ID) {
+		/* This code is  needed to support two enumerations on USB1.1 scenario */
+		DBUSERR(("%s: Firmware already downloaded\n", __FUNCTION__));
+
+		dbus_usbos_dl_cmd(osinfo, DL_RESETCFG, &id, sizeof(bootrom_id_t));
+		dl_needed = DBUS_OK;
+		if (usbinfo->pub->busstate == DBUS_STATE_DL_PENDING)
+			usbinfo->pub->busstate = DBUS_STATE_DL_DONE;
+	} else {
+		usbinfo->pub->attrib.devid = id.chip;
+		usbinfo->pub->attrib.chiprev = id.chiprev;
+	}
+
+exit:
+	return dl_needed;
+}
+
+/** After issuing firmware download, higher DBUS level (dbus.c) wants to start the firmware. */
+static int
+dbus_usb_dlrun(void *bus)
+{
+	usb_info_t *usbinfo = BUS_INFO(bus, usb_info_t);
+	void *osinfo;
+	rdl_state_t state;
+	int err = DBUS_OK;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	if (USB_DEV_ISBAD(usbinfo))
+		return DBUS_ERR;
+
+	osinfo = usbinfo->usbosl_info;
+	ASSERT(osinfo);
+
+	/* Check we are runnable */
+	dbus_usbos_dl_cmd(osinfo, DL_GETSTATE, &state, sizeof(rdl_state_t));
+
+	state.state = ltoh32(state.state);
+	state.bytes = ltoh32(state.bytes);
+
+	/* Start the image */
+	if (state.state == DL_RUNNABLE) {
+		DBUSTRACE(("%s: Issue DL_GO\n", __FUNCTION__));
+		dbus_usbos_dl_cmd(osinfo, DL_GO, &state, sizeof(rdl_state_t));
+
+		/* : Need this for 4326 for some reason
+		 * Same issue under both Linux/Windows
+		 */
+		if (usbinfo->pub->attrib.devid == TEST_CHIP)
+			dbus_usbos_wait(osinfo, USB_DLGO_SPINWAIT);
+
+//		dbus_usb_resetcfg(usbinfo);
+		/* The Donlge may go for re-enumeration. */
+	} else {
+		DBUSERR(("%s: Dongle not runnable\n", __FUNCTION__));
+		err = DBUS_ERR;
+	}
+
+	return err;
+}
+
+/**
+ * As preparation for firmware download, higher DBUS level (dbus.c) requests the firmware image
+ * to be used for the type of dongle detected. Directly called by dbus.c (so not via a callback
+ * construction)
+ */
+void
+dbus_bus_fw_get(void *bus, uint8 **fw, int *fwlen, int *decomp)
+{
+	usb_info_t *usbinfo = BUS_INFO(bus, usb_info_t);
+	unsigned int devid;
+	unsigned int crev;
+
+	devid = usbinfo->pub->attrib.devid;
+	crev = usbinfo->pub->attrib.chiprev;
+	BCM_REFERENCE(crev);
+
+	*fw = NULL;
+	*fwlen = 0;
+
+	switch (devid) {
+	default:
+#ifdef EMBED_IMAGE_GENERIC
+		*fw = (uint8 *)dlarray;
+		*fwlen = sizeof(dlarray);
+#endif
+		break;
+	}
+} /* dbus_bus_fw_get */
+
+#ifdef BCM_DNGL_EMBEDIMAGE
+static bool
+dbus_usb_device_exists(void *bus)
+{
+	usb_info_t *usbinfo = BUS_INFO(bus, usb_info_t);
+	void *osinfo;
+	bootrom_id_t id;
+
+	DBUSTRACE(("%s\n", __FUNCTION__));
+
+	if (usbinfo == NULL)
+		return FALSE;
+
+	osinfo = usbinfo->usbosl_info;
+	ASSERT(osinfo);
+
+	id.chip = 0xDEAD;
+	/* Query device to see if we get a response */
+	dbus_usbos_dl_cmd(osinfo, DL_GETVER, &id, sizeof(bootrom_id_t));
+
+	usbinfo->pub->attrib.devid = id.chip;
+	if (id.chip == 0xDEAD)
+		return FALSE;
+	else
+		return TRUE;
+}
+#endif /* BCM_DNGL_EMBEDIMAGE */
diff --git a/drivers/net/wireless/ap12275/dbus_usb_linux.c b/drivers/net/wireless/ap12275/dbus_usb_linux.c
new file mode 100644
index 000000000000..f113d5b8d6c5
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dbus_usb_linux.c
@@ -0,0 +1,4807 @@
+/*
+ * Dongle BUS interface
+ * USB Linux Implementation
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+/**
+ * @file @brief
+ * This file contains DBUS code that is USB *and* OS (Linux) specific. DBUS is a Broadcom
+ * proprietary host specific abstraction layer.
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+
+/**
+ * DBUS_LINUX_RXDPC is created for router platform performance tuning. A separate thread is created
+ * to handle USB RX and avoid the call chain getting too long and enhance cache hit rate.
+ *
+ * DBUS_LINUX_RXDPC setting is in wlconfig file.
+ */
+
+/*
+ * If DBUS_LINUX_RXDPC is off, spin_lock_bh() for CTFPOOL in
+ * linux_osl.c has to be changed to spin_lock_irqsave() because
+ * PKTGET/PKTFREE are no longer in bottom half.
+ *
+ * Right now we have another queue rpcq in wl_linux.c. Maybe we
+ * can eliminate that one to reduce the overhead.
+ *
+ * Enabling 2nd EP and DBUS_LINUX_RXDPC causing traffic from
+ * both EP's to be queued in the same rx queue. If we want
+ * RXDPC to work with 2nd EP. The EP for RPC call return
+ * should bypass the dpc and go directly up.
+ */
+
+/* #define DBUS_LINUX_RXDPC */
+
+/* Dbus histogram for ntxq, nrxq, dpc parameter tuning */
+/* #define DBUS_LINUX_HIST */
+
+#include <usbrdl.h>
+#include <bcmendian.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/usb.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/random.h>
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <asm/uaccess.h>
+#include <asm/unaligned.h>
+#include <dbus.h>
+#include <bcmutils.h>
+#include <bcmdevs_legacy.h>
+#include <bcmdevs.h>
+#include <linux/usb.h>
+#include <usbrdl.h>
+#include <linux/firmware.h>
+#ifdef DBUS_LINUX_RXDPC
+#include <linux/sched.h>
+#endif
+#include <dngl_stats.h>
+#include <dhd.h>
+
+#if defined(USBOS_THREAD) || defined(USBOS_TX_THREAD)
+
+/**
+ * The usb-thread is designed to provide currency on multiprocessors and SMP linux kernels. On the
+ * dual cores platform, the WLAN driver, without threads, executed only on CPU0. The driver consumed
+ * almost of 100% on CPU0, while CPU1 remained idle. The behavior was observed on Broadcom's STB.
+ *
+ * The WLAN driver consumed most of CPU0 and not CPU1 because tasklets/queues, software irq, and
+ * hardware irq are executing from CPU0, only. CPU0 became the system's bottle-neck. TPUT is lower
+ * and system's responsiveness is slower.
+ *
+ * To improve system responsiveness and TPUT usb-thread was implemented. The system's threads could
+ * be scheduled to run on any core. One core could be processing data in the usb-layer and the other
+ * core could be processing data in the wl-layer.
+ */
+
+#include <linux/kthread.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <asm/hardirq.h>
+#include <linux/list.h>
+#include <linux_osl.h>
+#endif /* USBOS_THREAD || USBOS_TX_THREAD */
+
+#ifdef DBUS_LINUX_RXDPC
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25))
+#define RESCHED()   _cond_resched()
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0))
+#define RESCHED()   cond_resched()
+#else
+#define RESCHED()   __cond_resched()
+#endif /* LINUX_VERSION_CODE  */
+#endif	/* DBUS_LINUX_RXDPC */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0))
+#define KERNEL26
+#endif
+
+/**
+ * Starting with the 3.10 kernel release, dynamic PM support for USB is present whenever
+ * the kernel was built with CONFIG_PM_RUNTIME enabled. The CONFIG_USB_SUSPEND option has
+ * been eliminated.
+ */
+#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 21)) && defined(CONFIG_USB_SUSPEND)) ||\
+	((LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)) && defined(CONFIG_PM_RUNTIME)) ||\
+	(LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+/* For USB power management support, see Linux kernel: Documentation/usb/power-management.txt */
+#define USB_SUSPEND_AVAILABLE
+#endif
+
+static inline int usb_submit_urb_linux(struct urb *urb)
+{
+
+#ifdef BCM_MAX_URB_LEN
+	if (urb && (urb->transfer_buffer_length > BCM_MAX_URB_LEN)) {
+		DBUSERR(("URB transfer length=%d exceeded %d ra=%p\n", urb->transfer_buffer_length,
+		BCM_MAX_URB_LEN, CALL_SITE));
+		return DBUS_ERR;
+	}
+#endif
+
+#ifdef KERNEL26
+	return usb_submit_urb(urb, GFP_ATOMIC);
+#else
+	return usb_submit_urb(urb);
+#endif
+
+}
+
+#define USB_SUBMIT_URB(urb) usb_submit_urb_linux(urb)
+
+#ifdef KERNEL26
+
+#define USB_ALLOC_URB()				usb_alloc_urb(0, GFP_ATOMIC)
+#define USB_UNLINK_URB(urb)			(usb_kill_urb(urb))
+#define USB_FREE_URB(urb)			(usb_free_urb(urb))
+#define USB_REGISTER()				usb_register(&dbus_usbdev)
+#define USB_DEREGISTER()			usb_deregister(&dbus_usbdev)
+
+#ifdef USB_SUSPEND_AVAILABLE
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33))
+#define USB_AUTOPM_SET_INTERFACE(intf)		usb_autopm_set_interface(intf)
+#else
+#define USB_ENABLE_AUTOSUSPEND(udev)		usb_enable_autosuspend(udev)
+#define USB_DISABLE_AUTOSUSPEND(udev)       usb_disable_autosuspend(udev)
+#endif  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33))  */
+
+#define USB_AUTOPM_GET_INTERFACE(intf)		usb_autopm_get_interface(intf)
+#define USB_AUTOPM_PUT_INTERFACE(intf)		usb_autopm_put_interface(intf)
+#define USB_AUTOPM_GET_INTERFACE_ASYNC(intf)	usb_autopm_get_interface_async(intf)
+#define USB_AUTOPM_PUT_INTERFACE_ASYNC(intf)	usb_autopm_put_interface_async(intf)
+#define USB_MARK_LAST_BUSY(dev)			usb_mark_last_busy(dev)
+
+#else /* USB_SUSPEND_AVAILABLE */
+
+#define USB_AUTOPM_GET_INTERFACE(intf)		do {} while (0)
+#define USB_AUTOPM_PUT_INTERFACE(intf)		do {} while (0)
+#define USB_AUTOPM_GET_INTERFACE_ASYNC(intf)	do {} while (0)
+#define USB_AUTOPM_PUT_INTERFACE_ASYNC(intf)	do {} while (0)
+#define USB_MARK_LAST_BUSY(dev)			do {} while (0)
+#endif /* USB_SUSPEND_AVAILABLE */
+
+#define USB_CONTROL_MSG(dev, pipe, request, requesttype, value, index, data, size, timeout) \
+	usb_control_msg((dev), (pipe), (request), (requesttype), (value), (index), \
+	(data), (size), (timeout))
+#define USB_BULK_MSG(dev, pipe, data, len, actual_length, timeout) \
+	usb_bulk_msg((dev), (pipe), (data), (len), (actual_length), (timeout))
+#define USB_BUFFER_ALLOC(dev, size, mem, dma)	usb_buffer_alloc(dev, size, mem, dma)
+#define USB_BUFFER_FREE(dev, size, data, dma)	usb_buffer_free(dev, size, data, dma)
+
+#ifdef WL_URB_ZPKT
+#define URB_QUEUE_BULK   URB_ZERO_PACKET
+#else
+#define URB_QUEUE_BULK   0
+#endif /* WL_URB_ZPKT */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0))
+#define CALLBACK_ARGS		struct urb *urb
+#define CALLBACK_ARGS_DATA	urb
+#else
+#define CALLBACK_ARGS		struct urb *urb, struct pt_regs *regs
+#define CALLBACK_ARGS_DATA	urb, regs
+#endif /* 5.18 */
+
+#define CONFIGDESC(usb)		(&((usb)->actconfig)->desc)
+#define IFPTR(usb, idx)		((usb)->actconfig->interface[idx])
+#define IFALTS(usb, idx)	(IFPTR((usb), (idx))->altsetting[0])
+#define IFDESC(usb, idx)	IFALTS((usb), (idx)).desc
+#define IFEPDESC(usb, idx, ep)	(IFALTS((usb), (idx)).endpoint[ep]).desc
+#ifdef DBUS_LINUX_RXDPC
+#define DAEMONIZE(a)		daemonize(a); allow_signal(SIGKILL); allow_signal(SIGTERM);
+#define SET_NICE(n)		set_user_nice(current, n)
+#endif
+
+#else /* KERNEL26 */
+
+#define USB_ALLOC_URB()				usb_alloc_urb(0)
+#define USB_UNLINK_URB(urb)			usb_unlink_urb(urb)
+#define USB_FREE_URB(urb)			(usb_free_urb(urb))
+#define USB_REGISTER()				usb_register(&dbus_usbdev)
+#define USB_DEREGISTER()			usb_deregister(&dbus_usbdev)
+#define USB_AUTOPM_GET_INTERFACE(intf)		do {} while (0)
+#define USB_AUTOPM_GET_INTERFACE_ASYNC(intf)	do {} while (0)
+#define USB_AUTOPM_PUT_INTERFACE_ASYNC(intf)	do {} while (0)
+#define USB_MARK_LAST_BUSY(dev)			do {} while (0)
+
+#define USB_CONTROL_MSG(dev, pipe, request, requesttype, value, index, data, size, timeout) \
+	usb_control_msg((dev), (pipe), (request), (requesttype), (value), (index), \
+	(data), (size), (timeout))
+#define USB_BUFFER_ALLOC(dev, size, mem, dma)  kmalloc(size, mem)
+#define USB_BUFFER_FREE(dev, size, data, dma)  kfree(data)
+
+#ifdef WL_URB_ZPKT
+#define URB_QUEUE_BULK   USB_QUEUE_BULK|URB_ZERO_PACKET
+#else
+#define URB_QUEUE_BULK   0
+#endif /*  WL_URB_ZPKT */
+
+#define CALLBACK_ARGS		struct urb *urb
+#define CALLBACK_ARGS_DATA	urb
+#define CONFIGDESC(usb)		((usb)->actconfig)
+#define IFPTR(usb, idx)		(&(usb)->actconfig->interface[idx])
+#define IFALTS(usb, idx)	((usb)->actconfig->interface[idx].altsetting[0])
+#define IFDESC(usb, idx)	IFALTS((usb), (idx))
+#define IFEPDESC(usb, idx, ep)	(IFALTS((usb), (idx)).endpoint[ep])
+
+#ifdef DBUS_LINUX_RXDPC
+#define DAEMONIZE(a)    daemonize();
+#define SET_NICE(n)     do {current->nice = (n);} while (0)
+#endif /* DBUS_LINUX_RXDPC */
+
+#endif /* KERNEL26 */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31))
+#define USB_SPEED_SUPER		5
+#endif  /* #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)) */
+
+#define CONTROL_IF   0
+#define BULK_IF      0
+
+#ifdef BCMUSBDEV_COMPOSITE
+#define USB_COMPIF_MAX       4
+
+/* temp defines, should come from the standard usb.h file */
+#define USB_CLASS_WIRELESS	0xe0
+#define USB_CLASS_MISC		0xef
+#define USB_SUBCLASS_COMMON	0x02
+#define USB_PROTO_IAD		0x01
+#define USB_PROTO_VENDOR	0xff
+
+#define USB_QUIRK_NO_SET_INTF   0x04 /**< device does not support set_interface */
+#endif /* BCMUSBDEV_COMPOSITE */
+
+#define USB_SYNC_WAIT_TIMEOUT  300  /* ms */
+
+/* Private data kept in skb */
+#define SKB_PRIV(skb, idx)  (&((void **)skb->cb)[idx])
+#define SKB_PRIV_URB(skb)   (*(struct urb **)SKB_PRIV(skb, 0))
+
+#ifndef DBUS_USB_RXQUEUE_BATCH_ADD
+/** items to add each time within limit */
+#define DBUS_USB_RXQUEUE_BATCH_ADD            8
+#endif
+
+#ifndef DBUS_USB_RXQUEUE_LOWER_WATERMARK
+/** add a new batch req to rx queue when waiting item count reduce to this number */
+#define DBUS_USB_RXQUEUE_LOWER_WATERMARK      4
+#endif
+
+enum usbos_suspend_state {
+	USBOS_SUSPEND_STATE_DEVICE_ACTIVE = 0, /**< Device is busy, won't allow suspend */
+	USBOS_SUSPEND_STATE_SUSPEND_PENDING,   /**< Device is idle, can be suspended */
+	                                       /* Wating PM to suspend */
+	USBOS_SUSPEND_STATE_SUSPENDED          /**< Device suspended */
+};
+
+enum usbos_request_state {
+	USBOS_REQUEST_STATE_UNSCHEDULED = 0,	/**< USB TX request not scheduled */
+	USBOS_REQUEST_STATE_SCHEDULED,		/**< USB TX request given to TX thread */
+	USBOS_REQUEST_STATE_SUBMITTED		/**< USB TX request submitted */
+};
+
+typedef struct {
+	uint32 notification;
+	uint32 reserved;
+} intr_t;
+
+typedef struct {
+	dbus_pub_t *pub;
+
+	void *cbarg;
+	dbus_intf_callbacks_t *cbs;
+
+	/* Imported */
+	struct usb_device *usb;	/**< USB device pointer from OS */
+	struct urb *intr_urb; /**< URB for interrupt endpoint */
+	struct list_head req_rxfreeq;
+	struct list_head req_txfreeq;
+	struct list_head req_rxpostedq;	/**< Posted down to USB driver for RX */
+	struct list_head req_txpostedq;	/**< Posted down to USB driver for TX */
+	spinlock_t rxfree_lock; /**< Lock for rx free list */
+	spinlock_t txfree_lock; /**< Lock for tx free list */
+	spinlock_t rxposted_lock; /**< Lock for rx posted list */
+	spinlock_t txposted_lock; /**< Lock for tx posted list */
+	uint rx_pipe, tx_pipe, intr_pipe, rx_pipe2; /**< Pipe numbers for USB I/O */
+	uint rxbuf_len;
+
+	struct list_head req_rxpendingq; /**< RXDPC: Pending for dpc to send up */
+	spinlock_t rxpending_lock;	/**< RXDPC: Lock for rx pending list */
+	long dpc_pid;
+	struct semaphore dpc_sem;
+	struct completion dpc_exited;
+	int rxpending;
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	int	dpc_cnt, dpc_pktcnt, dpc_maxpktcnt;
+#endif
+
+	struct urb               *ctl_urb;
+	int                      ctl_in_pipe, ctl_out_pipe;
+	struct usb_ctrlrequest   ctl_write;
+	struct usb_ctrlrequest   ctl_read;
+	struct semaphore         ctl_lock;     /* Lock for CTRL transfers via tx_thread */
+#ifdef USBOS_TX_THREAD
+	enum usbos_request_state ctl_state;
+#endif /* USBOS_TX_THREAD */
+
+	spinlock_t rxlock;      /**< Lock for rxq management */
+	spinlock_t txlock;      /**< Lock for txq management */
+
+	int intr_size;          /**< Size of interrupt message */
+	int interval;           /**< Interrupt polling interval */
+	intr_t intr;            /**< Data buffer for interrupt endpoint */
+
+	int maxps;
+	atomic_t txposted;
+	atomic_t rxposted;
+	atomic_t txallocated;
+	atomic_t rxallocated;
+	bool rxctl_deferrespok;	/**< Get a response for setup from dongle */
+
+	wait_queue_head_t wait;
+	bool waitdone;
+	int sync_urb_status;
+
+	struct urb *blk_urb; /**< Used for downloading embedded image */
+
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	int *txposted_hist;
+	int *rxposted_hist;
+#endif
+#ifdef USBOS_THREAD
+	spinlock_t              usbos_list_lock;
+	struct list_head        usbos_list;
+	struct list_head        usbos_free_list;
+	atomic_t                usbos_list_cnt;
+	wait_queue_head_t       usbos_queue_head;
+	struct task_struct      *usbos_kt;
+#endif /* USBOS_THREAD */
+
+#ifdef USBOS_TX_THREAD
+	spinlock_t              usbos_tx_list_lock;
+	struct list_head	usbos_tx_list;
+	wait_queue_head_t	usbos_tx_queue_head;
+	struct task_struct      *usbos_tx_kt;
+#endif /* USBOS_TX_THREAD */
+
+	struct dma_pool *qtd_pool; /**< QTD pool for USB optimization only */
+	int tx_ep, rx_ep, rx2_ep;  /**< EPs for USB optimization */
+	struct usb_device *usb_device; /**< USB device for optimization */
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX) /** Linux USB AP related */
+	spinlock_t fastpath_lock;
+#endif
+} usbos_info_t;
+
+typedef struct urb_req {
+	void         *pkt;
+	int          buf_len;
+	struct urb   *urb;
+	void         *arg;
+	usbos_info_t *usbinfo;
+	struct list_head urb_list;
+} urb_req_t;
+
+#ifdef USBOS_THREAD
+typedef struct usbos_list_entry {
+	struct list_head    list;   /* must be first */
+	void               *urb_context;
+	int                 urb_length;
+	int                 urb_status;
+} usbos_list_entry_t;
+
+static void* dbus_usbos_thread_init(usbos_info_t *usbos_info);
+static void  dbus_usbos_thread_deinit(usbos_info_t *usbos_info);
+static void  dbus_usbos_dispatch_schedule(CALLBACK_ARGS);
+static int   dbus_usbos_thread_func(void *data);
+#endif /* USBOS_THREAD */
+
+#ifdef USBOS_TX_THREAD
+void* dbus_usbos_tx_thread_init(usbos_info_t *usbos_info);
+void  dbus_usbos_tx_thread_deinit(usbos_info_t *usbos_info);
+int   dbus_usbos_tx_thread_func(void *data);
+#endif /* USBOS_TX_THREAD */
+
+/* Shared Function prototypes */
+bool dbus_usbos_dl_cmd(usbos_info_t *usbinfo, uint8 cmd, void *buffer, int buflen);
+int dbus_usbos_wait(usbos_info_t *usbinfo, uint16 ms);
+bool dbus_usbos_dl_send_bulk(usbos_info_t *usbinfo, void *buffer, int len);
+int dbus_write_membytes(usbos_info_t *usbinfo, bool set, uint32 address, uint8 *data, uint size);
+
+/* Local function prototypes */
+static void dbus_usbos_send_complete(CALLBACK_ARGS);
+#ifdef DBUS_LINUX_RXDPC
+static void dbus_usbos_recv_dpc(usbos_info_t *usbos_info);
+static int dbus_usbos_dpc_thread(void *data);
+#endif /* DBUS_LINUX_RXDPC */
+static void dbus_usbos_recv_complete(CALLBACK_ARGS);
+static int  dbus_usbos_errhandler(void *bus, int err);
+static int  dbus_usbos_state_change(void *bus, int state);
+static void dbusos_stop(usbos_info_t *usbos_info);
+
+#ifdef KERNEL26
+static int dbus_usbos_probe(struct usb_interface *intf, const struct usb_device_id *id);
+static void dbus_usbos_disconnect(struct usb_interface *intf);
+#if defined(USB_SUSPEND_AVAILABLE)
+static int dbus_usbos_resume(struct usb_interface *intf);
+static int dbus_usbos_suspend(struct usb_interface *intf, pm_message_t message);
+/** at the moment, used for full dongle host driver only */
+static int dbus_usbos_reset_resume(struct usb_interface *intf);
+#endif /* USB_SUSPEND_AVAILABLE */
+#else /* KERNEL26 */
+static void *dbus_usbos_probe(struct usb_device *usb, unsigned int ifnum,
+	const struct usb_device_id *id);
+static void dbus_usbos_disconnect(struct usb_device *usb, void *ptr);
+#endif /* KERNEL26 */
+
+#ifdef USB_TRIGGER_DEBUG
+static bool dbus_usbos_ctl_send_debugtrig(usbos_info_t *usbinfo);
+#endif /* USB_TRIGGER_DEBUG */
+
+#ifdef KEEPIF_ON_DEVICE_RESET
+enum usb_dev_status {
+	USB_DEVICE_INIT = 0,
+	USB_DEVICE_RESETTED = 1,
+	USB_DEVICE_DISCONNECTED = 2,
+	USB_DEVICE_PROBED
+};
+static int dbus_usbos_usbreset_func(void *data);
+#endif /* KEEPIF_ON_DEVICE_RESET */
+
+/**
+ * have to disable missing-field-initializers warning as last element {} triggers it
+ * and different versions of kernel have different number of members so it is impossible
+ * to specify the initializer. BTW issuing the warning here is bug og GCC as  universal
+ * zero {0} specified in C99 standard as correct way of initialization of struct to all zeros
+ */
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == \
+	4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#endif
+
+static struct usb_device_id devid_table[] = {
+	{ USB_DEVICE(BCM_DNGL_VID, 0x0000) }, /* Configurable via register() */
+#if defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW)
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BL_PID_4328) },
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BL_PID_43143) },
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BL_PID_43242) },
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BL_PID_4360) },
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BL_PID_43569) },
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BL_PID_4381) },
+#endif
+#ifdef EXTENDED_VID_PID
+	EXTENDED_VID_PID,
+#endif /* EXTENDED_VID_PID */
+	{ USB_DEVICE(BCM_DNGL_VID, BCM_DNGL_BDC_PID) }, /* Default BDC */
+	{ }
+};
+
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == \
+	4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic pop
+#endif
+
+MODULE_DEVICE_TABLE(usb, devid_table);
+
+/** functions called by the Linux kernel USB subsystem */
+static struct usb_driver dbus_usbdev = {
+	name:           "dbus_usbdev"BUS_TYPE,
+	probe:          dbus_usbos_probe,
+	disconnect:     dbus_usbos_disconnect,
+	id_table:       devid_table,
+#if defined(USB_SUSPEND_AVAILABLE)
+	suspend:        dbus_usbos_suspend,
+	resume:         dbus_usbos_resume,
+	reset_resume:	dbus_usbos_reset_resume,
+	/** Linux USB core will allow autosuspend for devices bound to this driver */
+	supports_autosuspend: 1
+#endif /* USB_SUSPEND_AVAILABLE */
+};
+
+/**
+ * This stores USB info during Linux probe callback since attach() is not called yet at this point
+ */
+typedef struct {
+	void    *usbos_info;
+	struct usb_device *usb; /**< USB device pointer from OS */
+	uint    rx_pipe;   /**< Pipe numbers for USB I/O */
+	uint    tx_pipe;   /**< Pipe numbers for USB I/O */
+	uint    intr_pipe; /**< Pipe numbers for USB I/O */
+	uint    rx_pipe2;  /**< Pipe numbers for USB I/O */
+	int     intr_size; /**< Size of interrupt message */
+	int     interval;  /**< Interrupt polling interval */
+	bool    dldone;
+	int     vid;
+	int     pid;
+	bool    dereged;
+	bool    disc_cb_done;
+	DEVICE_SPEED    device_speed;
+	enum usbos_suspend_state suspend_state;
+	struct usb_interface     *intf;
+#ifdef KEEPIF_ON_DEVICE_RESET
+	bool               keepif_on_devreset; /* if need keep network interface on dev reset */
+	bool               dev_resetted; /* if dev reset event occured */
+	enum dbus_state    busstate_bf_devreset; /* busstate before devreset */
+	atomic_t           usbdev_stat; /* usb device status: resetted? disconnected? re-probed? */
+	wait_queue_head_t  usbreset_queue_head; /* kernel thread waiting queue */
+	struct task_struct *usbreset_kt; /* kernel thread handle dev reset/probe event */
+#endif /* KEEPIF_ON_DEVICE_RESET */
+} probe_info_t;
+
+/* driver info, initialized when bcmsdh_register is called */
+static dbus_driver_t drvinfo = {NULL, NULL, NULL, NULL};
+
+/*
+ * USB Linux dbus_intf_t
+ */
+static void dbus_usbos_init_info(void);
+static void *dbus_usbos_intf_attach(dbus_pub_t *pub, void *cbarg, dbus_intf_callbacks_t *cbs);
+static void dbus_usbos_intf_detach(dbus_pub_t *pub, void *info);
+static int  dbus_usbos_intf_send_irb(void *bus, dbus_irb_tx_t *txirb);
+static int  dbus_usbos_intf_recv_irb(void *bus, dbus_irb_rx_t *rxirb);
+static int  dbus_usbos_intf_recv_irb_from_ep(void *bus, dbus_irb_rx_t *rxirb, uint32 ep_idx);
+static int  dbus_usbos_intf_cancel_irb(void *bus, dbus_irb_tx_t *txirb);
+static int  dbus_usbos_intf_send_ctl(void *bus, uint8 *buf, int len);
+static int  dbus_usbos_intf_recv_ctl(void *bus, uint8 *buf, int len);
+static int  dbus_usbos_intf_get_attrib(void *bus, dbus_attrib_t *attrib);
+static int  dbus_usbos_intf_up(void *bus);
+static int  dbus_usbos_intf_down(void *bus);
+static int  dbus_usbos_intf_stop(void *bus);
+static int  dbus_usbos_readreg(void *bus, uint32 regaddr, int datalen, uint32 *value);
+extern int dbus_usbos_loopback_tx(void *usbos_info_ptr, int cnt, int size);
+int dbus_usbos_writereg(void *bus, uint32 regaddr, int datalen, uint32 data);
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+static void dbus_usbos_intf_dump(void *bus, struct bcmstrbuf *b);
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+static int  dbus_usbos_intf_set_config(void *bus, dbus_config_t *config);
+static bool dbus_usbos_intf_recv_needed(void *bus);
+static void *dbus_usbos_intf_exec_rxlock(void *bus, exec_cb_t cb, struct exec_parms *args);
+static void *dbus_usbos_intf_exec_txlock(void *bus, exec_cb_t cb, struct exec_parms *args);
+#ifdef BCMUSBDEV_COMPOSITE
+static int dbus_usbos_intf_wlan(struct usb_device *usb);
+#endif /* BCMUSBDEV_COMPOSITE */
+
+/** functions called by dbus_usb.c */
+static dbus_intf_t dbus_usbos_intf = {
+	.attach = dbus_usbos_intf_attach,
+	.detach = dbus_usbos_intf_detach,
+	.up = dbus_usbos_intf_up,
+	.down = dbus_usbos_intf_down,
+	.send_irb = dbus_usbos_intf_send_irb,
+	.recv_irb = dbus_usbos_intf_recv_irb,
+	.cancel_irb = dbus_usbos_intf_cancel_irb,
+	.send_ctl = dbus_usbos_intf_send_ctl,
+	.recv_ctl = dbus_usbos_intf_recv_ctl,
+	.get_stats = NULL,
+	.get_attrib = dbus_usbos_intf_get_attrib,
+	.remove = NULL,
+	.resume = NULL,
+	.suspend = NULL,
+	.stop = dbus_usbos_intf_stop,
+	.reset = NULL,
+	.pktget = NULL,
+	.pktfree = NULL,
+	.iovar_op = NULL,
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	.dump = dbus_usbos_intf_dump,
+#else
+	.dump = NULL,
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+	.set_config = dbus_usbos_intf_set_config,
+	.get_config = NULL,
+	.device_exists = NULL,
+	.dlneeded = NULL,
+	.dlstart = NULL,
+	.dlrun = NULL,
+	.recv_needed = dbus_usbos_intf_recv_needed,
+	.exec_rxlock = dbus_usbos_intf_exec_rxlock,
+	.exec_txlock = dbus_usbos_intf_exec_txlock,
+
+	.tx_timer_init = NULL,
+	.tx_timer_start = NULL,
+	.tx_timer_stop = NULL,
+
+	.sched_dpc = NULL,
+	.lock = NULL,
+	.unlock = NULL,
+	.sched_probe_cb = NULL,
+
+	.shutdown = NULL,
+
+	.recv_stop = NULL,
+	.recv_resume = NULL,
+
+	.recv_irb_from_ep = dbus_usbos_intf_recv_irb_from_ep,
+	.readreg = dbus_usbos_readreg
+};
+
+static probe_info_t    g_probe_info;
+static void            *disc_arg = NULL;
+
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX)
+
+#define EHCI_PAGE_SIZE    4096
+
+/* Copies of structures located elsewhere. */
+/* ME: Unify (move to include file) */
+
+typedef struct {
+	dbus_pub_t *pub;
+	void *cbarg;
+	dbus_intf_callbacks_t *cbs;
+	dbus_intf_t *drvintf;
+	void *usbosl_info;
+} usb_info_t;
+
+/** General info for all BUS */
+typedef struct dbus_irbq {
+	dbus_irb_t *head;
+	dbus_irb_t *tail;
+	int cnt;
+} dbus_irbq_t;
+
+/**
+ * This private structure dbus_info_t is also declared in dbus.c.
+ * All the fields must be consistent in both declarations.
+ */
+typedef struct dbus_info {
+	dbus_pub_t pub; /* MUST BE FIRST */
+
+	void *cbarg;
+	dbus_callbacks_t *cbs;
+	void *bus_info;
+	dbus_intf_t *drvintf;
+	uint8 *fw;
+	int fwlen;
+	uint32 errmask;
+	int rx_low_watermark;
+	int tx_low_watermark;
+	bool txoff;
+	bool txoverride;
+	bool rxoff;
+	bool tx_timer_ticking;
+	dbus_irbq_t *rx_q;
+	dbus_irbq_t *tx_q;
+
+#ifdef BCMDBG
+	int *txpend_q_hist;
+	int *rxpend_q_hist;
+#endif /* BCMDBG */
+#ifdef EHCI_FASTPATH_RX
+	atomic_t rx_outstanding;
+#endif
+	uint8 *nvram;
+	int	nvram_len;
+	uint8 *image;	/**< buffer for combine fw and nvram */
+	int image_len;
+	uint8 *orig_fw;
+	int origfw_len;
+	int decomp_memsize;
+	dbus_extdl_t extdl;
+	int nvram_nontxt;
+} dbus_info_t;
+
+/* ME: Move to device extension */
+static atomic_t s_tx_pending;
+
+static int optimize_init(usbos_info_t *usbos_info, struct usb_device *usb, int out,
+	int in, int in2);
+static int optimize_deinit(usbos_info_t *usbos_info, struct usb_device *usb);
+#endif  /* #if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX) */
+
+static volatile int loopback_rx_cnt, loopback_tx_cnt;
+int loopback_size;
+bool is_loopback_pkt(void *buf);
+int matches_loopback_pkt(void *buf);
+
+/**
+ * multiple code paths in this file dequeue a URB request, this function makes sure that it happens
+ * in a concurrency save manner. Don't call this from a sleepable process context.
+ */
+static urb_req_t *
+dbus_usbos_qdeq(struct list_head *urbreq_q, spinlock_t *lock)
+{
+	unsigned long flags;
+	urb_req_t *req;
+
+	ASSERT(urbreq_q != NULL);
+
+	spin_lock_irqsave(lock, flags);
+
+	if (list_empty(urbreq_q)) {
+		req = NULL;
+	} else {
+		ASSERT(urbreq_q->next != NULL);
+		ASSERT(urbreq_q->next != urbreq_q);
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		req = list_entry(urbreq_q->next, urb_req_t, urb_list);
+		GCC_DIAGNOSTIC_POP();
+		list_del_init(&req->urb_list);
+	}
+
+	spin_unlock_irqrestore(lock, flags);
+
+	return req;
+}
+
+static void
+dbus_usbos_qenq(struct list_head *urbreq_q, urb_req_t *req, spinlock_t *lock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(lock, flags);
+
+	list_add_tail(&req->urb_list, urbreq_q);
+
+	spin_unlock_irqrestore(lock, flags);
+}
+
+/**
+ * multiple code paths in this file remove a URB request from a list, this function makes sure that
+ * it happens in a concurrency save manner. Don't call this from a sleepable process context.
+ * Is quite similar to dbus_usbos_qdeq(), I wonder why this function is needed.
+ */
+static void
+dbus_usbos_req_del(urb_req_t *req, spinlock_t *lock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(lock, flags);
+
+	list_del_init(&req->urb_list);
+
+	spin_unlock_irqrestore(lock, flags);
+}
+
+/**
+ * Driver requires a pool of URBs to operate. This function is called during
+ * initialization (attach phase), allocates a number of URBs, and puts them
+ * on the free (req_rxfreeq and req_txfreeq) queue
+ */
+static int
+dbus_usbos_urbreqs_alloc(usbos_info_t *usbos_info, uint32 count, bool is_rx)
+{
+	int i;
+	int allocated = 0;
+	int err = DBUS_OK;
+
+	for (i = 0; i < count; i++) {
+		urb_req_t *req;
+
+		req = MALLOC(usbos_info->pub->osh, sizeof(urb_req_t));
+		if (req == NULL) {
+			DBUSERR(("%s: MALLOC req failed\n", __FUNCTION__));
+			err = DBUS_ERR_NOMEM;
+			goto fail;
+		}
+		bzero(req, sizeof(urb_req_t));
+
+		req->urb = USB_ALLOC_URB();
+		if (req->urb == NULL) {
+			DBUSERR(("%s: USB_ALLOC_URB req->urb failed\n", __FUNCTION__));
+			err = DBUS_ERR_NOMEM;
+			goto fail;
+		}
+
+		INIT_LIST_HEAD(&req->urb_list);
+
+		if (is_rx) {
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+			/* don't allocate now. Do it on demand */
+			req->pkt = NULL;
+#else
+			/* pre-allocate  buffers never to be released */
+			req->pkt = MALLOC(usbos_info->pub->osh, usbos_info->rxbuf_len);
+			if (req->pkt == NULL) {
+				DBUSERR(("%s: MALLOC req->pkt failed\n", __FUNCTION__));
+				err = DBUS_ERR_NOMEM;
+				goto fail;
+			}
+#endif
+			req->buf_len = usbos_info->rxbuf_len;
+			dbus_usbos_qenq(&usbos_info->req_rxfreeq, req, &usbos_info->rxfree_lock);
+		} else {
+			req->buf_len = 0;
+			dbus_usbos_qenq(&usbos_info->req_txfreeq, req, &usbos_info->txfree_lock);
+		}
+		allocated++;
+		continue;
+
+fail:
+		if (req) {
+			if (is_rx && req->pkt) {
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+				/* req->pkt is NULL in "NOCOPY" mode */
+#else
+				MFREE(usbos_info->pub->osh, req->pkt, req->buf_len);
+#endif
+			}
+			if (req->urb) {
+				USB_FREE_URB(req->urb);
+			}
+			MFREE(usbos_info->pub->osh, req, sizeof(urb_req_t));
+		}
+		break;
+	}
+
+	atomic_add(allocated, is_rx ? &usbos_info->rxallocated : &usbos_info->txallocated);
+
+	if (is_rx) {
+		DBUSTRACE(("%s: add %d (total %d) rx buf, each has %d bytes\n", __FUNCTION__,
+			allocated, atomic_read(&usbos_info->rxallocated), usbos_info->rxbuf_len));
+	} else {
+		DBUSTRACE(("%s: add %d (total %d) tx req\n", __FUNCTION__,
+			allocated, atomic_read(&usbos_info->txallocated)));
+	}
+
+	return err;
+} /* dbus_usbos_urbreqs_alloc */
+
+/** Typically called during detach or when attach failed. Don't call until all URBs unlinked */
+static int
+dbus_usbos_urbreqs_free(usbos_info_t *usbos_info, bool is_rx)
+{
+	int rtn = 0;
+	urb_req_t *req;
+	struct list_head *req_q;
+	spinlock_t *lock;
+
+	if (is_rx) {
+		req_q = &usbos_info->req_rxfreeq;
+		lock = &usbos_info->rxfree_lock;
+	} else {
+		req_q = &usbos_info->req_txfreeq;
+		lock = &usbos_info->txfree_lock;
+	}
+
+	while ((req = dbus_usbos_qdeq(req_q, lock)) != NULL) {
+		if (is_rx) {
+			if (req->pkt) {
+				/* We do MFREE instead of PKTFREE because the pkt has been
+				 * converted to native already
+				 */
+				MFREE(usbos_info->pub->osh, req->pkt, req->buf_len);
+				req->buf_len = 0;
+			}
+		} else {
+			/* sending req should not be assigned pkt buffer */
+			ASSERT(req->pkt == NULL);
+		}
+
+		if (req->urb) {
+			USB_FREE_URB(req->urb);
+			req->urb = NULL;
+		}
+		MFREE(usbos_info->pub->osh, req, sizeof(urb_req_t));
+
+		rtn++;
+	}
+
+	return rtn;
+} /* dbus_usbos_urbreqs_free */
+
+/**
+ * called by Linux kernel on URB completion. Upper DBUS layer (dbus_usb.c) has to be notified of
+ * send completion.
+ */
+void
+dbus_usbos_send_complete(CALLBACK_ARGS)
+{
+	urb_req_t *req = urb->context;
+	dbus_irb_tx_t *txirb = req->arg;
+	usbos_info_t *usbos_info = req->usbinfo;
+	unsigned long flags;
+	int status = DBUS_OK;
+	int txposted;
+
+	USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+
+	spin_lock_irqsave(&usbos_info->txlock, flags);
+
+	dbus_usbos_req_del(req, &usbos_info->txposted_lock);
+	txposted = atomic_dec_return(&usbos_info->txposted);
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	if (usbos_info->txposted_hist) {
+		usbos_info->txposted_hist[txposted]++;
+	}
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+	if (unlikely (txposted < 0)) {
+		DBUSERR(("%s ERROR: txposted is negative (%d)!!\n", __FUNCTION__, txposted));
+	}
+	spin_unlock_irqrestore(&usbos_info->txlock, flags);
+
+	if (unlikely (urb->status)) {
+		status = DBUS_ERR_TXFAIL;
+		DBUSTRACE(("txfail status %d\n", urb->status));
+	}
+
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+	/* sending req should not be assigned pkt buffer */
+	ASSERT(req->pkt == NULL);
+#endif
+	/*  txirb should always be set, except for ZLP. ZLP is reusing this callback function. */
+	if (txirb != NULL) {
+		if (txirb->send_buf != NULL) {
+			MFREE(usbos_info->pub->osh, txirb->send_buf, req->buf_len);
+			req->buf_len = 0;
+		}
+		if (likely (usbos_info->cbarg && usbos_info->cbs)) {
+			if (likely (usbos_info->cbs->send_irb_complete != NULL))
+			    usbos_info->cbs->send_irb_complete(usbos_info->cbarg, txirb, status);
+		}
+	}
+
+	dbus_usbos_qenq(&usbos_info->req_txfreeq, req, &usbos_info->txfree_lock);
+} /* dbus_usbos_send_complete */
+
+/**
+ * In order to receive USB traffic from the dongle, we need to supply the Linux kernel with a free
+ * URB that is going to contain received data.
+ */
+static int
+dbus_usbos_recv_urb_submit(usbos_info_t *usbos_info, dbus_irb_rx_t *rxirb,
+	uint32 ep_idx)
+{
+	urb_req_t *req;
+	int ret = DBUS_OK;
+	unsigned long flags;
+	void *p;
+	uint rx_pipe;
+	int rxposted;
+
+	BCM_REFERENCE(rxposted);
+
+	if (!(req = dbus_usbos_qdeq(&usbos_info->req_rxfreeq, &usbos_info->rxfree_lock))) {
+		DBUSTRACE(("%s No free URB!\n", __FUNCTION__));
+		return DBUS_ERR_RXDROP;
+	}
+
+	spin_lock_irqsave(&usbos_info->rxlock, flags);
+
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+	req->pkt = rxirb->pkt = PKTGET(usbos_info->pub->osh, req->buf_len, FALSE);
+	if (!rxirb->pkt) {
+		DBUSERR(("%s: PKTGET failed\n", __FUNCTION__));
+		dbus_usbos_qenq(&usbos_info->req_rxfreeq, req, &usbos_info->rxfree_lock);
+		ret = DBUS_ERR_RXDROP;
+		goto fail;
+	}
+	/* consider the packet "native" so we don't count it as MALLOCED in the osl */
+	PKTTONATIVE(usbos_info->pub->osh, req->pkt);
+	rxirb->buf = NULL;
+	p = PKTDATA(usbos_info->pub->osh, req->pkt);
+#else
+	if (req->buf_len != usbos_info->rxbuf_len) {
+		ASSERT(req->pkt);
+		MFREE(usbos_info->pub->osh, req->pkt, req->buf_len);
+		DBUSTRACE(("%s: replace rx buff: old len %d, new len %d\n", __FUNCTION__,
+			req->buf_len, usbos_info->rxbuf_len));
+		req->buf_len = 0;
+		req->pkt = MALLOC(usbos_info->pub->osh, usbos_info->rxbuf_len);
+		if (req->pkt == NULL) {
+			DBUSERR(("%s: MALLOC req->pkt failed\n", __FUNCTION__));
+			ret = DBUS_ERR_NOMEM;
+			goto fail;
+		}
+		req->buf_len = usbos_info->rxbuf_len;
+	}
+	rxirb->buf = req->pkt;
+	p = rxirb->buf;
+#endif /* defined(BCM_RPC_NOCOPY) */
+	rxirb->buf_len = req->buf_len;
+	req->usbinfo = usbos_info;
+	req->arg = rxirb;
+	if (ep_idx == 0) {
+		rx_pipe = usbos_info->rx_pipe;
+	} else {
+		rx_pipe = usbos_info->rx_pipe2;
+		ASSERT(usbos_info->rx_pipe2);
+	}
+	/* Prepare the URB */
+	usb_fill_bulk_urb(req->urb, usbos_info->usb, rx_pipe,
+		p,
+		rxirb->buf_len,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+		(void *)dbus_usbos_recv_complete, req);
+#else
+		(usb_complete_t)dbus_usbos_recv_complete, req);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0) */
+		req->urb->transfer_flags |= URB_QUEUE_BULK;
+
+	if ((ret = USB_SUBMIT_URB(req->urb))) {
+		DBUSERR(("%s USB_SUBMIT_URB failed. status %d\n", __FUNCTION__, ret));
+		dbus_usbos_qenq(&usbos_info->req_rxfreeq, req, &usbos_info->rxfree_lock);
+		ret = DBUS_ERR_RXFAIL;
+		goto fail;
+	}
+	rxposted = atomic_inc_return(&usbos_info->rxposted);
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	if (usbos_info->rxposted_hist) {
+		usbos_info->rxposted_hist[rxposted]++;
+	}
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+
+	dbus_usbos_qenq(&usbos_info->req_rxpostedq, req, &usbos_info->rxposted_lock);
+fail:
+	spin_unlock_irqrestore(&usbos_info->rxlock, flags);
+	return ret;
+} /* dbus_usbos_recv_urb_submit */
+
+#ifdef DBUS_LINUX_RXDPC
+
+static void
+dbus_usbos_recv_dpc(usbos_info_t *usbos_info)
+{
+	urb_req_t *req = NULL;
+	dbus_irb_rx_t *rxirb = NULL;
+	int dbus_status = DBUS_OK;
+	bool killed = (g_probe_info.suspend_state == USBOS_SUSPEND_STATE_SUSPEND_PENDING) ? 1 : 0;
+
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	int cnt = 0;
+
+	usbos_info->dpc_cnt++;
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+
+	/* make it bounded if we think it can take too long */
+	while ((req = dbus_usbos_qdeq(&usbos_info->req_rxpendingq,
+		&usbos_info->rxpending_lock)) != NULL) {
+		struct urb *urb = req->urb;
+		rxirb = req->arg;
+
+		/* Handle errors */
+		if (urb->status) {
+			/*
+			 * Linux 2.4 disconnect: -ENOENT or -EILSEQ for CRC error; rmmod: -ENOENT
+			 * Linux 2.6 disconnect: -EPROTO, rmmod: -ESHUTDOWN
+			 */
+			if ((urb->status == -ENOENT && (!killed)) || urb->status == -ESHUTDOWN) {
+				/* NOTE: unlink() can not be called from URB callback().
+				 * Do not call dbusos_stop() here.
+				 */
+				dbus_usbos_state_change(usbos_info, DBUS_STATE_DOWN);
+			} else if (urb->status == -EPROTO) {
+			} else {
+				DBUSERR(("%s rx error %d\n", __FUNCTION__, urb->status));
+				dbus_usbos_errhandler(usbos_info, DBUS_ERR_RXFAIL);
+			}
+
+			/* On error, don't submit more URBs yet */
+			DBUSERR(("%s %d rx error %d\n", __FUNCTION__, __LINE__, urb->status));
+			rxirb->buf = NULL;
+			rxirb->actual_len = 0;
+			dbus_status = DBUS_ERR_RXFAIL;
+			goto fail;
+		}
+
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+		/* detach the packet from the req */
+		req->pkt = NULL;
+#endif
+		/* Make the skb represent the received urb */
+		rxirb->actual_len = urb->actual_length;
+
+fail:
+		/* this is just a stats, we don't protect it with locks for now */
+		usbos_info->rxpending--;
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+		cnt++;
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+		if (usbos_info->cbarg && usbos_info->cbs &&
+			usbos_info->cbs->recv_irb_complete) {
+			usbos_info->cbs->recv_irb_complete(usbos_info->cbarg, rxirb, dbus_status);
+		}
+		dbus_usbos_qenq(&usbos_info->req_rxfreeq, req, &usbos_info->rxfree_lock);
+	}
+
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	usbos_info->dpc_pktcnt += cnt;
+	usbos_info->dpc_maxpktcnt = MAX(cnt, usbos_info->dpc_maxpktcnt);
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+#ifdef DBUS_LINUX_HIST
+	{
+		static unsigned long last_dump = 0;
+
+		/* dump every 20 sec */
+		if (jiffies > (last_dump + 20*HZ)) {
+			dbus_usbos_intf_dump(usbos_info, NULL);
+			last_dump = jiffies;
+		}
+	}
+#endif /* DBUS_LINUX_HIST */
+} /* dbus_usbos_recv_dpc */
+
+static int
+dbus_usbos_dpc_thread(void *data)
+{
+	usbos_info_t *usbos_info = (usbos_info_t*)data;
+
+	DAEMONIZE("dbus_rx_dpc");
+	/* High priority for short response time. We will yield by ourselves. */
+	/* SET_NICE(-10); */
+
+	/* Run until signal received */
+	while (1) {
+		if (down_interruptible(&usbos_info->dpc_sem) == 0) {
+			dbus_usbos_recv_dpc(usbos_info);
+			RESCHED();
+		} else
+			break;
+	}
+
+	complete_and_exit(&usbos_info->dpc_exited, 0);
+	return 0;
+}
+
+#endif /* DBUS_LINUX_RXDPC */
+
+/**
+ * Called by worked thread when a 'receive URB' completed or Linux kernel when it returns a URB to
+ * this driver.
+ */
+static void
+dbus_usbos_recv_complete_handle(urb_req_t *req, int len, int status)
+{
+#ifdef DBUS_LINUX_RXDPC
+	usbos_info_t *usbos_info = req->usbinfo;
+	unsigned long flags;
+	int rxallocated, rxposted;
+
+	spin_lock_irqsave(&usbos_info->rxlock, flags);
+	/* detach the packet from the queue */
+	dbus_usbos_req_del(req, &usbos_info->rxposted_lock);
+	rxposted = atomic_dec_return(&usbos_info->rxposted);
+	rxallocated = atomic_read(&usbos_info->rxallocated);
+
+	/* Enqueue to rxpending queue */
+	usbos_info->rxpending++;
+	dbus_usbos_qenq(&usbos_info->req_rxpendingq, req, &usbos_info->rxpending_lock);
+	spin_unlock_irqrestore(&usbos_info->rxlock, flags);
+
+#error "RX req/buf appending-mode not verified for DBUS_LINUX_RXDPC because it was disabled"
+	if ((rxallocated < usbos_info->pub->nrxq) && (!status) &&
+		(rxposted == DBUS_USB_RXQUEUE_LOWER_WATERMARK)) {
+			DBUSTRACE(("%s: need more rx buf: rxallocated %d rxposted %d!\n",
+				__FUNCTION__, rxallocated, rxposted));
+			dbus_usbos_urbreqs_alloc(usbos_info,
+				MIN(DBUS_USB_RXQUEUE_BATCH_ADD,
+				usbos_info->pub->nrxq - rxallocated), TRUE);
+	}
+#error "Please verify above code works if you happened to enable DBUS_LINUX_RXDPC!!"
+
+	/* Wake up dpc for further processing */
+	ASSERT(usbos_info->dpc_pid >= 0);
+	up(&usbos_info->dpc_sem);
+#else
+	dbus_irb_rx_t *rxirb = req->arg;
+	usbos_info_t *usbos_info = req->usbinfo;
+	unsigned long flags;
+	int rxallocated, rxposted;
+	int dbus_status = DBUS_OK;
+	bool killed = (g_probe_info.suspend_state == USBOS_SUSPEND_STATE_SUSPEND_PENDING) ? 1 : 0;
+
+	spin_lock_irqsave(&usbos_info->rxlock, flags);
+	dbus_usbos_req_del(req, &usbos_info->rxposted_lock);
+	rxposted = atomic_dec_return(&usbos_info->rxposted);
+	rxallocated = atomic_read(&usbos_info->rxallocated);
+	spin_unlock_irqrestore(&usbos_info->rxlock, flags);
+
+	if ((rxallocated < usbos_info->pub->nrxq) && (!status) &&
+		(rxposted == DBUS_USB_RXQUEUE_LOWER_WATERMARK)) {
+			DBUSTRACE(("%s: need more rx buf: rxallocated %d rxposted %d!\n",
+				__FUNCTION__, rxallocated, rxposted));
+			dbus_usbos_urbreqs_alloc(usbos_info,
+				MIN(DBUS_USB_RXQUEUE_BATCH_ADD,
+				usbos_info->pub->nrxq - rxallocated), TRUE);
+	}
+
+	/* Handle errors */
+	if (status) {
+		/*
+		 * Linux 2.4 disconnect: -ENOENT or -EILSEQ for CRC error; rmmod: -ENOENT
+		 * Linux 2.6 disconnect: -EPROTO, rmmod: -ESHUTDOWN
+		 */
+		if ((status == -ENOENT && (!killed))|| status == -ESHUTDOWN) {
+			/* NOTE: unlink() can not be called from URB callback().
+			 * Do not call dbusos_stop() here.
+			 */
+			DBUSTRACE(("%s rx error %d\n", __FUNCTION__, status));
+			dbus_usbos_state_change(usbos_info, DBUS_STATE_DOWN);
+		} else if (status == -EPROTO) {
+			DBUSTRACE(("%s rx error %d\n", __FUNCTION__, status));
+		} else if (killed && (status == -EHOSTUNREACH || status == -ENOENT)) {
+			/* Device is suspended */
+		} else {
+			DBUSTRACE(("%s rx error %d\n", __FUNCTION__, status));
+			dbus_usbos_errhandler(usbos_info, DBUS_ERR_RXFAIL);
+		}
+
+		/* On error, don't submit more URBs yet */
+		rxirb->buf = NULL;
+		rxirb->actual_len = 0;
+		dbus_status = DBUS_ERR_RXFAIL;
+		goto fail;
+	}
+
+	/* Make the skb represent the received urb */
+	rxirb->actual_len = len;
+
+	if (rxirb->actual_len < sizeof(uint32)) {
+		DBUSTRACE(("small pkt len %d, process as ZLP\n", rxirb->actual_len));
+		dbus_status = DBUS_ERR_RXZLP;
+	}
+
+fail:
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+	/* detach the packet from the queue */
+	req->pkt = NULL;
+#endif /* BCM_RPC_NOCOPY || BCM_RPC_RXNOCOPY */
+
+	if (usbos_info->cbarg && usbos_info->cbs) {
+		if (usbos_info->cbs->recv_irb_complete) {
+			usbos_info->cbs->recv_irb_complete(usbos_info->cbarg, rxirb, dbus_status);
+		}
+	}
+
+	dbus_usbos_qenq(&usbos_info->req_rxfreeq, req, &usbos_info->rxfree_lock);
+#endif /* DBUS_LINUX_RXDPC */
+
+	/* Mark the interface as busy to reset USB autosuspend timer */
+	USB_MARK_LAST_BUSY(usbos_info->usb);
+} /* dbus_usbos_recv_complete_handle */
+
+/** called by Linux kernel when it returns a URB to this driver */
+static void
+dbus_usbos_recv_complete(CALLBACK_ARGS)
+{
+#ifdef USBOS_THREAD
+	dbus_usbos_dispatch_schedule(CALLBACK_ARGS_DATA);
+#else /*  !USBOS_THREAD */
+	dbus_usbos_recv_complete_handle(urb->context, urb->actual_length, urb->status);
+#endif /*  USBOS_THREAD */
+}
+
+/**
+ * If Linux notifies our driver that a control read or write URB has completed, we should notify
+ * the DBUS layer above us (dbus_usb.c in this case).
+ */
+static void
+dbus_usbos_ctl_complete(usbos_info_t *usbos_info, int type, int urbstatus)
+{
+	int status = DBUS_ERR;
+
+	if (usbos_info == NULL)
+		return;
+
+	switch (urbstatus) {
+		case 0:
+			status = DBUS_OK;
+		break;
+		case -EINPROGRESS:
+		case -ENOENT:
+		default:
+#ifdef INTR_EP_ENABLE
+			DBUSERR(("%s:%d fail status %d bus:%d susp:%d intr:%d ctli:%d ctlo:%d\n",
+				__FUNCTION__, type, urbstatus,
+				usbos_info->pub->busstate, g_probe_info.suspend_state,
+				usbos_info->intr_urb_submitted, usbos_info->ctlin_urb_submitted,
+				usbos_info->ctlout_urb_submitted));
+#else
+			DBUSERR(("%s: failed with status %d\n", __FUNCTION__, urbstatus));
+			status = DBUS_ERR;
+		break;
+#endif /* INTR_EP_ENABLE */
+	}
+
+	if (usbos_info->cbarg && usbos_info->cbs) {
+		if (usbos_info->cbs->ctl_complete)
+			usbos_info->cbs->ctl_complete(usbos_info->cbarg, type, status);
+	}
+}
+
+/** called by Linux */
+static void
+dbus_usbos_ctlread_complete(CALLBACK_ARGS)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *)urb->context;
+
+	ASSERT(urb);
+	usbos_info = (usbos_info_t *)urb->context;
+
+	dbus_usbos_ctl_complete(usbos_info, DBUS_CBCTL_READ, urb->status);
+
+#ifdef USBOS_THREAD
+	if (usbos_info->rxctl_deferrespok) {
+		usbos_info->ctl_read.bRequestType = USB_DIR_IN | USB_TYPE_CLASS |
+		USB_RECIP_INTERFACE;
+		usbos_info->ctl_read.bRequest = 1;
+	}
+#endif
+
+	up(&usbos_info->ctl_lock);
+
+	USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+}
+
+/** called by Linux */
+static void
+dbus_usbos_ctlwrite_complete(CALLBACK_ARGS)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *)urb->context;
+
+	ASSERT(urb);
+	usbos_info = (usbos_info_t *)urb->context;
+
+	dbus_usbos_ctl_complete(usbos_info, DBUS_CBCTL_WRITE, urb->status);
+
+#ifdef USBOS_TX_THREAD
+	usbos_info->ctl_state = USBOS_REQUEST_STATE_UNSCHEDULED;
+#endif /* USBOS_TX_THREAD */
+
+	up(&usbos_info->ctl_lock);
+
+	USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+}
+
+#ifdef INTR_EP_ENABLE
+/** called by Linux */
+static void
+dbus_usbos_intr_complete(CALLBACK_ARGS)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *)urb->context;
+	bool killed = (g_probe_info.suspend_state == USBOS_SUSPEND_STATE_SUSPEND_PENDING) ? 1 : 0;
+
+	if (usbos_info == NULL || usbos_info->pub == NULL)
+		return;
+	if ((urb->status == -ENOENT && (!killed)) || urb->status == -ESHUTDOWN ||
+		urb->status == -ENODEV) {
+		dbus_usbos_state_change(usbos_info, DBUS_STATE_DOWN);
+	}
+
+	if (usbos_info->pub->busstate == DBUS_STATE_DOWN) {
+		DBUSERR(("%s: intr cb when DBUS down, ignoring\n", __FUNCTION__));
+		return;
+	}
+	dbus_usbos_ctl_complete(usbos_info, DBUS_CBINTR_POLL, urb->status);
+}
+#endif	/* INTR_EP_ENABLE */
+
+/**
+ * when the bus is going to sleep or halt, the Linux kernel requires us to take ownership of our
+ * URBs again. Multiple code paths in this file require a list of URBs to be cancelled in a
+ * concurrency save manner.
+ */
+static void
+dbus_usbos_unlink(struct list_head *urbreq_q, spinlock_t *lock)
+{
+	urb_req_t *req;
+
+	/* dbus_usbos_recv_complete() adds req back to req_freeq */
+	while ((req = dbus_usbos_qdeq(urbreq_q, lock)) != NULL) {
+		ASSERT(req->urb != NULL);
+		USB_UNLINK_URB(req->urb);
+	}
+}
+
+/** multiple code paths in this file require the bus to stop */
+static void
+dbus_usbos_cancel_all_urbs(usbos_info_t *usbos_info)
+{
+	int rxposted, txposted;
+
+	DBUSTRACE(("%s: unlink all URBs\n", __FUNCTION__));
+
+#ifdef USBOS_TX_THREAD
+	usbos_info->ctl_state = USBOS_REQUEST_STATE_UNSCHEDULED;
+
+	/* Yield the CPU to TX thread so all pending requests are submitted */
+	while (!list_empty(&usbos_info->usbos_tx_list)) {
+		wake_up_interruptible(&usbos_info->usbos_tx_queue_head);
+		OSL_SLEEP(10);
+	}
+#endif /* USBOS_TX_THREAD */
+
+	/* tell Linux kernel to cancel a single intr, ctl and blk URB */
+	if (usbos_info->intr_urb)
+		USB_UNLINK_URB(usbos_info->intr_urb);
+	if (usbos_info->ctl_urb)
+		USB_UNLINK_URB(usbos_info->ctl_urb);
+	if (usbos_info->blk_urb)
+		USB_UNLINK_URB(usbos_info->blk_urb);
+
+	dbus_usbos_unlink(&usbos_info->req_txpostedq, &usbos_info->txposted_lock);
+	dbus_usbos_unlink(&usbos_info->req_rxpostedq, &usbos_info->rxposted_lock);
+
+	/* Wait until the callbacks for all submitted URBs have been called, because the
+	 * handler needs to know is an USB suspend is in progress.
+	 */
+	SPINWAIT((atomic_read(&usbos_info->txposted) != 0 ||
+		atomic_read(&usbos_info->rxposted) != 0), 10000);
+
+	txposted = atomic_read(&usbos_info->txposted);
+	rxposted = atomic_read(&usbos_info->rxposted);
+	if (txposted != 0 || rxposted != 0) {
+		DBUSERR(("%s ERROR: REQs posted, rx=%d tx=%d!\n",
+			__FUNCTION__, rxposted, txposted));
+	}
+} /* dbus_usbos_cancel_all_urbs */
+
+/** multiple code paths require the bus to stop */
+static void
+dbusos_stop(usbos_info_t *usbos_info)
+{
+	urb_req_t *req;
+	int rxposted;
+	req = NULL;
+	BCM_REFERENCE(req);
+
+	ASSERT(usbos_info);
+
+#ifdef USB_TRIGGER_DEBUG
+	dbus_usbos_ctl_send_debugtrig(usbos_info);
+#endif /* USB_TRIGGER_DEBUG */
+	dbus_usbos_state_change(usbos_info, DBUS_STATE_DOWN);
+
+	dbus_usbos_cancel_all_urbs(usbos_info);
+
+#ifdef USBOS_THREAD
+	/* yield the CPU to rx packet thread */
+	while (1) {
+		if (atomic_read(&usbos_info->usbos_list_cnt) <= 0)	break;
+		wake_up_interruptible(&usbos_info->usbos_queue_head);
+		OSL_SLEEP(3);
+	}
+#endif /* USBOS_THREAD */
+
+	rxposted = atomic_read(&usbos_info->rxposted);
+	if (rxposted > 0) {
+		DBUSERR(("%s ERROR: rx REQs posted=%d in stop!\n", __FUNCTION__,
+			rxposted));
+	}
+
+	ASSERT(atomic_read(&usbos_info->txposted) == 0 && rxposted == 0);
+
+#ifdef DBUS_LINUX_RXDPC
+	/* Stop the dpc thread */
+	if (usbos_info->dpc_pid >= 0) {
+		KILL_PROC(usbos_info->dpc_pid, SIGTERM);
+		wait_for_completion(&usbos_info->dpc_exited);
+	}
+
+	/* Move pending reqs to free queue so they can be freed */
+	while ((req = dbus_usbos_qdeq(&usbos_info->req_rxpendingq,
+		&usbos_info->rxpending_lock)) != NULL) {
+		dbus_usbos_qenq(&usbos_info->req_rxfreeq, req,
+			&usbos_info->rxfree_lock);
+	}
+#endif /* DBUS_LINUX_RXDPC */
+} /* dbusos_stop */
+
+#if defined(USB_SUSPEND_AVAILABLE)
+
+/**
+ * Linux kernel sports a 'USB auto suspend' feature. See: http://lwn.net/Articles/373550/
+ * The suspend method is called by the Linux kernel to warn the driver that the device is going to
+ * be suspended.  If the driver returns a negative error code, the suspend will be aborted. If the
+ * driver returns 0, it must cancel all outstanding URBs (usb_kill_urb()) and not submit any more.
+ */
+static int
+dbus_usbos_suspend(struct usb_interface *intf,
+            pm_message_t message)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) g_probe_info.usbos_info;
+	int err = 0;
+
+	printf("%s Enter\n", __FUNCTION__);
+
+	/* DHD for full dongle model */
+	g_probe_info.suspend_state = USBOS_SUSPEND_STATE_SUSPEND_PENDING;
+	if (drvinfo.suspend && disc_arg)
+		err = drvinfo.suspend(disc_arg);
+	if (err) {
+		printf("%s: err=%d\n", __FUNCTION__, err);
+//		g_probe_info.suspend_state = USBOS_SUSPEND_STATE_DEVICE_ACTIVE;
+//		return err;
+	}
+	usbos_info->pub->busstate = DBUS_STATE_SLEEP;
+
+	dbus_usbos_cancel_all_urbs((usbos_info_t*)g_probe_info.usbos_info);
+	g_probe_info.suspend_state = USBOS_SUSPEND_STATE_SUSPENDED;
+
+	printf("%s Exit err=%d\n", __FUNCTION__, err);
+	return err;
+}
+
+/**
+ * The resume method is called to tell the driver that the device has been resumed and the driver
+ * can return to normal operation.  URBs may once more be submitted.
+ */
+static int
+dbus_usbos_resume(struct usb_interface *intf)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) g_probe_info.usbos_info;
+
+	printf("%s Enter\n", __FUNCTION__);
+
+	g_probe_info.suspend_state = USBOS_SUSPEND_STATE_DEVICE_ACTIVE;
+	if (drvinfo.resume && disc_arg)
+		drvinfo.resume(disc_arg);
+	usbos_info->pub->busstate = DBUS_STATE_UP;
+
+	printf("%s Exit\n", __FUNCTION__);
+	return 0;
+}
+
+/**
+* This function is directly called by the Linux kernel, when the suspended device has been reset
+* instead of being resumed
+*/
+static int
+dbus_usbos_reset_resume(struct usb_interface *intf)
+{
+	printf("%s Enter\n", __FUNCTION__);
+
+#ifdef KEEPIF_ON_DEVICE_RESET
+	if (g_probe_info.keepif_on_devreset) {
+		atomic_set(&g_probe_info.usbdev_stat, USB_DEVICE_RESETTED);
+		wake_up_interruptible(&g_probe_info.usbreset_queue_head);
+	} else
+#endif /* KEEPIF_ON_DEVICE_RESET */
+	{
+		g_probe_info.suspend_state = USBOS_SUSPEND_STATE_DEVICE_ACTIVE;
+		if (drvinfo.resume && disc_arg)
+			drvinfo.resume(disc_arg);
+	}
+
+	printf("%s Exit\n", __FUNCTION__);
+	return 0;
+}
+
+#endif /* USB_SUSPEND_AVAILABLE */
+
+/**
+ * Called by Linux kernel at initialization time, kernel wants to know if our driver will accept the
+ * caller supplied USB interface. Note that USB drivers are bound to interfaces, and not to USB
+ * devices.
+ */
+#ifdef KERNEL26
+#define DBUS_USBOS_PROBE() static int dbus_usbos_probe(struct usb_interface *intf, const struct usb_device_id *id)
+#define DBUS_USBOS_DISCONNECT() static void dbus_usbos_disconnect(struct usb_interface *intf)
+#else
+#define DBUS_USBOS_PROBE() static void * dbus_usbos_probe(struct usb_device *usb, unsigned int ifnum, const struct usb_device_id *id)
+#define DBUS_USBOS_DISCONNECT() static void dbus_usbos_disconnect(struct usb_device *usb, void *ptr)
+#endif /* KERNEL26 */
+
+DBUS_USBOS_PROBE()
+{
+	int ep;
+	struct usb_endpoint_descriptor *endpoint;
+	int ret = 0;
+#ifdef KERNEL26
+	struct usb_device *usb = interface_to_usbdev(intf);
+#else
+	int claimed = 0;
+#endif
+	int num_of_eps;
+#ifdef BCMUSBDEV_COMPOSITE
+	int wlan_if = -1;
+	bool intr_ep = FALSE;
+#endif /* BCMUSBDEV_COMPOSITE */
+	wifi_adapter_info_t *adapter;
+
+	DHD_MUTEX_LOCK();
+
+	DBUSERR(("%s: bus num(busnum)=%d, slot num (portnum)=%d\n", __FUNCTION__,
+		usb->bus->busnum, usb->portnum));
+	adapter = dhd_wifi_platform_attach_adapter(USB_BUS, usb->bus->busnum,
+		usb->portnum, WIFI_STATUS_POWER_ON);
+	if (adapter == NULL) {
+		DBUSERR(("%s: can't find adapter info for this chip\n", __FUNCTION__));
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+#ifdef BCMUSBDEV_COMPOSITE
+	wlan_if = dbus_usbos_intf_wlan(usb);
+#ifdef KERNEL26
+	if ((wlan_if >= 0) && (IFPTR(usb, wlan_if) == intf))
+#else
+	if (wlan_if == ifnum)
+#endif /* KERNEL26 */
+	{
+#endif /* BCMUSBDEV_COMPOSITE */
+		g_probe_info.usb = usb;
+		g_probe_info.dldone = TRUE;
+#ifdef BCMUSBDEV_COMPOSITE
+	} else {
+		DBUSTRACE(("dbus_usbos_probe: skip probe for non WLAN interface\n"));
+		ret = BCME_UNSUPPORTED;
+		goto fail;
+	}
+#endif /* BCMUSBDEV_COMPOSITE */
+
+#ifdef KERNEL26
+	g_probe_info.intf = intf;
+#endif /* KERNEL26 */
+
+#ifdef BCMUSBDEV_COMPOSITE
+	if (IFDESC(usb, wlan_if).bInterfaceNumber > USB_COMPIF_MAX)
+#else
+	if (IFDESC(usb, CONTROL_IF).bInterfaceNumber)
+#endif /* BCMUSBDEV_COMPOSITE */
+	{
+		ret = -1;
+		goto fail;
+	}
+	if (id != NULL) {
+		g_probe_info.vid = id->idVendor;
+		g_probe_info.pid = id->idProduct;
+	}
+
+#ifdef KERNEL26
+	usb_set_intfdata(intf, &g_probe_info);
+#endif
+
+	/* Check that the device supports only one configuration */
+	if (usb->descriptor.bNumConfigurations != 1) {
+		ret = -1;
+		goto fail;
+	}
+
+	if (usb->descriptor.bDeviceClass != USB_CLASS_VENDOR_SPEC) {
+#ifdef BCMUSBDEV_COMPOSITE
+		if ((usb->descriptor.bDeviceClass != USB_CLASS_MISC) &&
+			(usb->descriptor.bDeviceClass != USB_CLASS_WIRELESS)) {
+#endif /* BCMUSBDEV_COMPOSITE */
+			ret = -1;
+			goto fail;
+#ifdef BCMUSBDEV_COMPOSITE
+		}
+#endif /* BCMUSBDEV_COMPOSITE */
+	}
+
+	/*
+	 * Only the BDC interface configuration is supported:
+	 *	Device class: USB_CLASS_VENDOR_SPEC
+	 *	if0 class: USB_CLASS_VENDOR_SPEC
+	 *	if0/ep0: control
+	 *	if0/ep1: bulk in
+	 *	if0/ep2: bulk out (ok if swapped with bulk in)
+	 */
+	if (CONFIGDESC(usb)->bNumInterfaces != 1) {
+#ifdef BCMUSBDEV_COMPOSITE
+		if (CONFIGDESC(usb)->bNumInterfaces > USB_COMPIF_MAX) {
+#endif /* BCMUSBDEV_COMPOSITE */
+			ret = -1;
+			goto fail;
+#ifdef BCMUSBDEV_COMPOSITE
+		}
+#endif /* BCMUSBDEV_COMPOSITE */
+	}
+
+	/* Check interface */
+#ifndef KERNEL26
+#ifdef BCMUSBDEV_COMPOSITE
+	if (usb_interface_claimed(IFPTR(usb, wlan_if)))
+#else
+	if (usb_interface_claimed(IFPTR(usb, CONTROL_IF)))
+#endif /* BCMUSBDEV_COMPOSITE */
+	{
+		ret = -1;
+		goto fail;
+	}
+#endif /* !KERNEL26 */
+
+#ifdef BCMUSBDEV_COMPOSITE
+	if ((IFDESC(usb, wlan_if).bInterfaceClass != USB_CLASS_VENDOR_SPEC ||
+		IFDESC(usb, wlan_if).bInterfaceSubClass != 2 ||
+		IFDESC(usb, wlan_if).bInterfaceProtocol != 0xff) &&
+		(IFDESC(usb, wlan_if).bInterfaceClass != USB_CLASS_MISC ||
+		IFDESC(usb, wlan_if).bInterfaceSubClass != USB_SUBCLASS_COMMON ||
+		IFDESC(usb, wlan_if).bInterfaceProtocol != USB_PROTO_IAD))
+#else
+	if (IFDESC(usb, CONTROL_IF).bInterfaceClass != USB_CLASS_VENDOR_SPEC ||
+		IFDESC(usb, CONTROL_IF).bInterfaceSubClass != 2 ||
+		IFDESC(usb, CONTROL_IF).bInterfaceProtocol != 0xff)
+#endif /* BCMUSBDEV_COMPOSITE */
+	{
+#ifdef BCMUSBDEV_COMPOSITE
+			DBUSERR(("%s: invalid control interface: class %d, subclass %d, proto %d\n",
+				__FUNCTION__,
+				IFDESC(usb, wlan_if).bInterfaceClass,
+				IFDESC(usb, wlan_if).bInterfaceSubClass,
+				IFDESC(usb, wlan_if).bInterfaceProtocol));
+#else
+			DBUSERR(("%s: invalid control interface: class %d, subclass %d, proto %d\n",
+				__FUNCTION__,
+				IFDESC(usb, CONTROL_IF).bInterfaceClass,
+				IFDESC(usb, CONTROL_IF).bInterfaceSubClass,
+				IFDESC(usb, CONTROL_IF).bInterfaceProtocol));
+#endif /* BCMUSBDEV_COMPOSITE */
+			ret = -1;
+			goto fail;
+	}
+
+	/* Check control endpoint */
+#ifdef BCMUSBDEV_COMPOSITE
+	endpoint = &IFEPDESC(usb, wlan_if, 0);
+#else
+	endpoint = &IFEPDESC(usb, CONTROL_IF, 0);
+#endif /* BCMUSBDEV_COMPOSITE */
+	if ((endpoint->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) != USB_ENDPOINT_XFER_INT) {
+#ifdef BCMUSBDEV_COMPOSITE
+		if ((endpoint->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) !=
+			USB_ENDPOINT_XFER_BULK) {
+#endif /* BCMUSBDEV_COMPOSITE */
+			DBUSERR(("%s: invalid control endpoint %d\n",
+				__FUNCTION__, endpoint->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK));
+			ret = -1;
+			goto fail;
+#ifdef BCMUSBDEV_COMPOSITE
+		}
+#endif /* BCMUSBDEV_COMPOSITE */
+	}
+
+#ifdef BCMUSBDEV_COMPOSITE
+	if ((endpoint->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) == USB_ENDPOINT_XFER_INT) {
+#endif /* BCMUSBDEV_COMPOSITE */
+		g_probe_info.intr_pipe =
+			usb_rcvintpipe(usb, endpoint->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK);
+#ifdef BCMUSBDEV_COMPOSITE
+		intr_ep = TRUE;
+	}
+#endif /* BCMUSBDEV_COMPOSITE */
+
+#ifndef KERNEL26
+	/* Claim interface */
+#ifdef BCMUSBDEV_COMPOSITE
+	usb_driver_claim_interface(&dbus_usbdev, IFPTR(usb, wlan_if), &g_probe_info);
+#else
+	usb_driver_claim_interface(&dbus_usbdev, IFPTR(usb, CONTROL_IF), &g_probe_info);
+#endif /* BCMUSBDEV_COMPOSITE */
+	claimed = 1;
+#endif /* !KERNEL26 */
+	g_probe_info.rx_pipe = 0;
+	g_probe_info.rx_pipe2 = 0;
+	g_probe_info.tx_pipe = 0;
+#ifdef BCMUSBDEV_COMPOSITE
+	if (intr_ep)
+		ep = 1;
+	else
+		ep = 0;
+	num_of_eps = IFDESC(usb, wlan_if).bNumEndpoints - 1;
+#else
+	num_of_eps = IFDESC(usb, BULK_IF).bNumEndpoints - 1;
+#endif /* BCMUSBDEV_COMPOSITE */
+
+	if ((num_of_eps != 2) && (num_of_eps != 3)) {
+#ifdef BCMUSBDEV_COMPOSITE
+		if (num_of_eps > 7)
+#endif /* BCMUSBDEV_COMPOSITE */
+			ASSERT(0);
+	}
+	/* Check data endpoints and get pipes */
+#ifdef BCMUSBDEV_COMPOSITE
+	for (; ep <= num_of_eps; ep++)
+#else
+	for (ep = 1; ep <= num_of_eps; ep++)
+#endif /* BCMUSBDEV_COMPOSITE */
+	{
+#ifdef BCMUSBDEV_COMPOSITE
+		endpoint = &IFEPDESC(usb, wlan_if, ep);
+#else
+		endpoint = &IFEPDESC(usb, BULK_IF, ep);
+#endif /* BCMUSBDEV_COMPOSITE */
+		if ((endpoint->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) !=
+		    USB_ENDPOINT_XFER_BULK) {
+			DBUSERR(("%s: invalid data endpoint %d\n",
+			           __FUNCTION__, ep));
+			ret = -1;
+			goto fail;
+		}
+
+		if ((endpoint->bEndpointAddress & USB_ENDPOINT_DIR_MASK) == USB_DIR_IN) {
+			/* direction: dongle->host */
+			if (!g_probe_info.rx_pipe) {
+				g_probe_info.rx_pipe = usb_rcvbulkpipe(usb,
+					(endpoint->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK));
+			} else {
+				g_probe_info.rx_pipe2 = usb_rcvbulkpipe(usb,
+					(endpoint->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK));
+			}
+
+		} else
+			g_probe_info.tx_pipe = usb_sndbulkpipe(usb, (endpoint->bEndpointAddress &
+			     USB_ENDPOINT_NUMBER_MASK));
+	}
+
+	/* Allocate interrupt URB and data buffer */
+	/* RNDIS says 8-byte intr, our old drivers used 4-byte */
+#ifdef BCMUSBDEV_COMPOSITE
+	g_probe_info.intr_size = (IFEPDESC(usb, wlan_if, 0).wMaxPacketSize == 16) ? 8 : 4;
+	g_probe_info.interval = IFEPDESC(usb, wlan_if, 0).bInterval;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 21))
+	usb->quirks |= USB_QUIRK_NO_SET_INTF;
+#endif
+#else
+	g_probe_info.intr_size = (IFEPDESC(usb, CONTROL_IF, 0).wMaxPacketSize == 16) ? 8 : 4;
+	g_probe_info.interval = IFEPDESC(usb, CONTROL_IF, 0).bInterval;
+#endif /* BCMUSBDEV_COMPOSITE */
+
+#ifndef KERNEL26
+	/* usb_fill_int_urb does the interval decoding in 2.6 */
+	if (usb->speed == USB_SPEED_HIGH)
+		g_probe_info.interval = 1 << (g_probe_info.interval - 1);
+#endif
+	if (usb->speed == USB_SPEED_SUPER) {
+		g_probe_info.device_speed = SUPER_SPEED;
+		DBUSERR(("super speed device detected\n"));
+	} else if (usb->speed == USB_SPEED_HIGH) {
+		g_probe_info.device_speed = HIGH_SPEED;
+		DBUSERR(("high speed device detected\n"));
+	} else {
+		g_probe_info.device_speed = FULL_SPEED;
+		DBUSERR(("full speed device detected\n"));
+	}
+#ifdef KEEPIF_ON_DEVICE_RESET
+	if (g_probe_info.keepif_on_devreset && g_probe_info.dev_resetted) {
+		atomic_set(&g_probe_info.usbdev_stat, USB_DEVICE_PROBED);
+		wake_up_interruptible(&g_probe_info.usbreset_queue_head);
+	} else
+#endif /* KEEPIF_ON_DEVICE_RESET */
+	if (g_probe_info.dereged == FALSE && drvinfo.probe) {
+		disc_arg = drvinfo.probe(usb->bus->busnum, usb->portnum, 0);
+	}
+
+	g_probe_info.disc_cb_done = FALSE;
+
+#ifdef KERNEL26
+	intf->needs_remote_wakeup = 1;
+#endif /* KERNEL26 */
+	DHD_MUTEX_UNLOCK();
+
+	/* Success */
+#ifdef KERNEL26
+	return DBUS_OK;
+#else
+	usb_inc_dev_use(usb);
+	return &g_probe_info;
+#endif
+
+fail:
+	printf("%s: Exit ret=%d\n", __FUNCTION__, ret);
+#ifdef BCMUSBDEV_COMPOSITE
+	if (ret != BCME_UNSUPPORTED)
+#endif /* BCMUSBDEV_COMPOSITE */
+		DBUSERR(("%s: failed with errno %d\n", __FUNCTION__, ret));
+#ifndef KERNEL26
+	if (claimed)
+#ifdef BCMUSBDEV_COMPOSITE
+		usb_driver_release_interface(&dbus_usbdev, IFPTR(usb, wlan_if));
+#else
+		usb_driver_release_interface(&dbus_usbdev, IFPTR(usb, CONTROL_IF));
+#endif /* BCMUSBDEV_COMPOSITE */
+#endif /* !KERNEL26 */
+
+	DHD_MUTEX_UNLOCK();
+#ifdef KERNEL26
+	usb_set_intfdata(intf, NULL);
+	return ret;
+#else
+	return NULL;
+#endif
+} /* dbus_usbos_probe */
+
+/** Called by Linux kernel, is the counter part of dbus_usbos_probe() */
+DBUS_USBOS_DISCONNECT()
+{
+#ifdef KERNEL26
+	struct usb_device *usb = interface_to_usbdev(intf);
+	probe_info_t *probe_usb_init_data = usb_get_intfdata(intf);
+#else
+	probe_info_t *probe_usb_init_data = (probe_info_t *) ptr;
+#endif
+	usbos_info_t *usbos_info;
+
+	DHD_MUTEX_LOCK();
+
+	DBUSERR(("%s: bus num(busnum)=%d, slot num (portnum)=%d\n", __FUNCTION__,
+		usb->bus->busnum, usb->portnum));
+
+	if (probe_usb_init_data) {
+		usbos_info = (usbos_info_t *) probe_usb_init_data->usbos_info;
+		if (usbos_info) {
+			if ((probe_usb_init_data->dereged == FALSE) && drvinfo.remove && disc_arg) {
+				bool remove_if = TRUE;
+#ifdef KEEPIF_ON_DEVICE_RESET
+				if (g_probe_info.keepif_on_devreset)
+					remove_if = FALSE;
+#endif /* KEEPIF_ON_DEVICE_RESET */
+				if (remove_if) {
+					drvinfo.remove(disc_arg);
+					disc_arg = NULL;
+					probe_usb_init_data->disc_cb_done = TRUE;
+				}
+			}
+		}
+	}
+
+	if (usb) {
+#ifndef KERNEL26
+#ifdef BCMUSBDEV_COMPOSITE
+		usb_driver_release_interface(&dbus_usbdev, IFPTR(usb, wlan_if));
+#else
+		usb_driver_release_interface(&dbus_usbdev, IFPTR(usb, CONTROL_IF));
+#endif /* BCMUSBDEV_COMPOSITE */
+		usb_dec_dev_use(usb);
+#endif /* !KERNEL26 */
+	}
+
+#ifdef KEEPIF_ON_DEVICE_RESET
+	if (g_probe_info.keepif_on_devreset && (!g_probe_info.dev_resetted)) {
+		atomic_set(&g_probe_info.usbdev_stat, USB_DEVICE_DISCONNECTED);
+		wake_up_interruptible(&g_probe_info.usbreset_queue_head);
+	}
+#endif /* KEEPIF_ON_DEVICE_RESET */
+
+	DHD_MUTEX_UNLOCK();
+} /* dbus_usbos_disconnect */
+
+#define LOOPBACK_PKT_START 0xBABE1234
+
+bool is_loopback_pkt(void *buf)
+{
+
+	uint32 *buf_ptr = (uint32 *) buf;
+
+	if (*buf_ptr == LOOPBACK_PKT_START)
+		return TRUE;
+	return FALSE;
+
+}
+
+int matches_loopback_pkt(void *buf)
+{
+	int i, j;
+	unsigned char *cbuf = (unsigned char *) buf;
+
+	for (i = 4; i < loopback_size; i++) {
+		if (cbuf[i] != (i % 256)) {
+			printf("%s: mismatch at i=%d %d : ", __FUNCTION__, i, cbuf[i]);
+			for (j = i; ((j < i+ 16) && (j < loopback_size)); j++) {
+				printf("%d ", cbuf[j]);
+			}
+			printf("\n");
+			return 0;
+		}
+	}
+	loopback_rx_cnt++;
+	return 1;
+}
+
+int dbus_usbos_loopback_tx(void *usbos_info_ptr, int cnt, int size)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) usbos_info_ptr;
+	unsigned char *buf;
+	int j;
+	void* p = NULL;
+	int rc, last_rx_cnt;
+	int tx_failed_cnt;
+	int max_size = 1650;
+	int usb_packet_size = 512;
+	int min_packet_size = 10;
+
+	if (size % usb_packet_size == 0) {
+		size = size - 1;
+		DBUSERR(("%s: overriding size=%d \n", __FUNCTION__, size));
+	}
+
+	if (size < min_packet_size) {
+		size = min_packet_size;
+		DBUSERR(("%s: overriding size=%d\n", __FUNCTION__, min_packet_size));
+	}
+	if (size > max_size) {
+		size = max_size;
+		DBUSERR(("%s: overriding size=%d\n", __FUNCTION__, max_size));
+	}
+
+	loopback_tx_cnt = 0;
+	loopback_rx_cnt = 0;
+	tx_failed_cnt = 0;
+	loopback_size   = size;
+
+	while (loopback_tx_cnt < cnt) {
+		uint32 *x;
+		int pkt_size = loopback_size;
+
+		p = PKTGET(usbos_info->pub->osh, pkt_size, TRUE);
+		if (p == NULL) {
+			DBUSERR(("%s:%d Failed to allocate packet sz=%d\n",
+			       __FUNCTION__, __LINE__, pkt_size));
+			return BCME_ERROR;
+		}
+		x = (uint32*) PKTDATA(usbos_info->pub->osh, p);
+		*x = LOOPBACK_PKT_START;
+		buf = (unsigned char*) x;
+		for (j = 4; j < pkt_size; j++) {
+			buf[j] = j % 256;
+		}
+		rc = dbus_send_buf(usbos_info->pub, buf, pkt_size, p);
+		if (rc != BCME_OK) {
+			DBUSERR(("%s:%d Freeing packet \n", __FUNCTION__, __LINE__));
+			PKTFREE(usbos_info->pub->osh, p, TRUE);
+			dbus_usbos_wait(usbos_info, 1);
+			tx_failed_cnt++;
+		} else {
+			loopback_tx_cnt++;
+			tx_failed_cnt = 0;
+		}
+		if (tx_failed_cnt == 5) {
+			DBUSERR(("%s : Failed to send loopback packets cnt=%d loopback_tx_cnt=%d\n",
+			 __FUNCTION__, cnt, loopback_tx_cnt));
+			break;
+		}
+	}
+	printf("Transmitted %d loopback packets of size %d\n", loopback_tx_cnt, loopback_size);
+
+	last_rx_cnt = loopback_rx_cnt;
+	while (loopback_rx_cnt < loopback_tx_cnt) {
+		dbus_usbos_wait(usbos_info, 1);
+		if (loopback_rx_cnt <= last_rx_cnt) {
+			DBUSERR(("%s: Matched rx cnt stuck at %d \n", __FUNCTION__, last_rx_cnt));
+			return BCME_ERROR;
+		}
+		last_rx_cnt = loopback_rx_cnt;
+	}
+	printf("Received %d loopback packets of size %d\n", loopback_tx_cnt, loopback_size);
+
+	return BCME_OK;
+} /* dbus_usbos_loopback_tx */
+
+/**
+ * Higher layer (dbus_usb.c) wants to transmit an I/O Request Block
+ *     @param[in] txirb txirb->pkt, if non-zero, contains a single or a chain of packets
+ */
+static int
+dbus_usbos_intf_send_irb(void *bus, dbus_irb_tx_t *txirb)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	urb_req_t *req, *req_zlp = NULL;
+	int ret = DBUS_OK;
+	unsigned long flags;
+	void *pkt;
+	uint32 buffer_length;
+	uint8 *buf;
+
+	if ((usbos_info == NULL) || !usbos_info->tx_pipe) {
+		return DBUS_ERR;
+	}
+
+	if (txirb->pkt != NULL) {
+		buffer_length = pkttotlen(usbos_info->pub->osh, txirb->pkt);
+		/* In case of multiple packets the values below may be overwritten */
+		txirb->send_buf = NULL;
+		buf = PKTDATA(usbos_info->pub->osh, txirb->pkt);
+	} else { /* txirb->buf != NULL */
+		ASSERT(txirb->buf != NULL);
+		ASSERT(txirb->send_buf == NULL);
+		buffer_length = txirb->len;
+		buf = txirb->buf;
+	}
+
+	if (!(req = dbus_usbos_qdeq(&usbos_info->req_txfreeq, &usbos_info->txfree_lock))) {
+		DBUSERR(("%s No free URB!\n", __FUNCTION__));
+		return DBUS_ERR_TXDROP;
+	}
+
+	/* If not using standard Linux kernel functionality for handling Zero Length Packet(ZLP),
+	 * the dbus needs to generate ZLP when length is multiple of MaxPacketSize.
+	 */
+#ifndef WL_URB_ZPKT
+	if (!(buffer_length % usbos_info->maxps)) {
+		if (!(req_zlp =
+			dbus_usbos_qdeq(&usbos_info->req_txfreeq, &usbos_info->txfree_lock))) {
+			DBUSERR(("%s No free URB for ZLP!\n", __FUNCTION__));
+			dbus_usbos_qenq(&usbos_info->req_txfreeq, req, &usbos_info->txfree_lock);
+			return DBUS_ERR_TXDROP;
+		}
+
+		/* No txirb, so that dbus_usbos_send_complete can differentiate between
+		 * DATA and ZLP.
+		 */
+		req_zlp->arg = NULL;
+		req_zlp->usbinfo = usbos_info;
+		req_zlp->buf_len = 0;
+
+		usb_fill_bulk_urb(req_zlp->urb, usbos_info->usb, usbos_info->tx_pipe, NULL,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+			0, (void *)dbus_usbos_send_complete, req_zlp);
+#else
+			0, (usb_complete_t)dbus_usbos_send_complete, req_zlp);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0) */
+
+		req_zlp->urb->transfer_flags |= URB_QUEUE_BULK;
+	}
+#endif /* !WL_URB_ZPKT */
+
+#ifndef USBOS_TX_THREAD
+	/* Disable USB autosuspend until this request completes, request USB resume if needed.
+	 * Because this call runs asynchronously, there is no guarantee the bus is resumed before
+	 * the URB is submitted, and the URB might be dropped. Use USBOS_TX_THREAD to avoid
+	 * this.
+	 */
+	USB_AUTOPM_GET_INTERFACE_ASYNC(g_probe_info.intf);
+#endif /* !USBOS_TX_THREAD */
+
+	spin_lock_irqsave(&usbos_info->txlock, flags);
+
+	req->arg = txirb;
+	req->usbinfo = usbos_info;
+	req->buf_len = 0;
+
+	/* Prepare the URB */
+	if (txirb->pkt != NULL) {
+		uint32 pktlen;
+		uint8 *transfer_buf;
+
+		/* For multiple packets, allocate contiguous buffer and copy packet data to it */
+		if (PKTNEXT(usbos_info->pub->osh, txirb->pkt)) {
+			transfer_buf = MALLOC(usbos_info->pub->osh, buffer_length);
+			if (!transfer_buf) {
+				ret = DBUS_ERR_TXDROP;
+				DBUSERR(("fail to alloc to usb buffer\n"));
+				goto fail;
+			}
+
+			pkt = txirb->pkt;
+			txirb->send_buf = transfer_buf;
+			req->buf_len = buffer_length;
+
+			while (pkt) {
+				pktlen = PKTLEN(usbos_info->pub->osh, pkt);
+				bcopy(PKTDATA(usbos_info->pub->osh, pkt), transfer_buf, pktlen);
+				transfer_buf += pktlen;
+				pkt = PKTNEXT(usbos_info->pub->osh, pkt);
+			}
+
+			ASSERT(((uint8 *) txirb->send_buf + buffer_length) == transfer_buf);
+
+			/* Overwrite buf pointer with pointer to allocated contiguous transfer_buf
+			 */
+			buf = txirb->send_buf;
+		}
+	}
+
+	usb_fill_bulk_urb(req->urb, usbos_info->usb, usbos_info->tx_pipe, buf,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+		buffer_length, (void *)dbus_usbos_send_complete, req);
+#else
+		buffer_length, (usb_complete_t)dbus_usbos_send_complete, req);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0) */
+
+	req->urb->transfer_flags |= URB_QUEUE_BULK;
+
+#ifdef USBOS_TX_THREAD
+	/* Enqueue TX request, the TX thread will resume the bus if needed and submit
+	 * it asynchronously
+	 */
+	dbus_usbos_qenq(&usbos_info->usbos_tx_list, req, &usbos_info->usbos_tx_list_lock);
+	if (req_zlp != NULL) {
+		dbus_usbos_qenq(&usbos_info->usbos_tx_list, req_zlp,
+			&usbos_info->usbos_tx_list_lock);
+	}
+	spin_unlock_irqrestore(&usbos_info->txlock, flags);
+
+	wake_up_interruptible(&usbos_info->usbos_tx_queue_head);
+	return DBUS_OK;
+#else
+	if ((ret = USB_SUBMIT_URB(req->urb))) {
+		ret = DBUS_ERR_TXDROP;
+		goto fail;
+	}
+
+	dbus_usbos_qenq(&usbos_info->req_txpostedq, req, &usbos_info->txposted_lock);
+	atomic_inc(&usbos_info->txposted);
+
+	if (req_zlp != NULL) {
+		if ((ret = USB_SUBMIT_URB(req_zlp->urb))) {
+			DBUSERR(("failed to submit ZLP URB!\n"));
+			ASSERT(0);
+			ret = DBUS_ERR_TXDROP;
+			goto fail2;
+		}
+
+		dbus_usbos_qenq(&usbos_info->req_txpostedq, req_zlp, &usbos_info->txposted_lock);
+		/* Also increment txposted for zlp packet, as it will be decremented in
+		 * dbus_usbos_send_complete()
+		 */
+		atomic_inc(&usbos_info->txposted);
+	}
+
+	spin_unlock_irqrestore(&usbos_info->txlock, flags);
+	return DBUS_OK;
+#endif /* USBOS_TX_THREAD */
+
+fail:
+	if (txirb->send_buf != NULL) {
+		MFREE(usbos_info->pub->osh, txirb->send_buf, req->buf_len);
+		req->buf_len = 0;
+	}
+	dbus_usbos_qenq(&usbos_info->req_txfreeq, req, &usbos_info->txfree_lock);
+#ifndef USBOS_TX_THREAD
+fail2:
+#endif
+	if (req_zlp != NULL) {
+		dbus_usbos_qenq(&usbos_info->req_txfreeq, req_zlp, &usbos_info->txfree_lock);
+	}
+
+	spin_unlock_irqrestore(&usbos_info->txlock, flags);
+
+#ifndef USBOS_TX_THREAD
+	USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+#endif /* !USBOS_TX_THREAD */
+
+	return ret;
+} /* dbus_usbos_intf_send_irb */
+
+/** Higher layer (dbus_usb.c) recycles a received (and used) packet. */
+static int
+dbus_usbos_intf_recv_irb(void *bus, dbus_irb_rx_t *rxirb)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	int ret = DBUS_OK;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	ret = dbus_usbos_recv_urb_submit(usbos_info, rxirb, 0);
+	return ret;
+}
+
+static int
+dbus_usbos_intf_recv_irb_from_ep(void *bus, dbus_irb_rx_t *rxirb, uint32 ep_idx)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	int ret = DBUS_OK;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+#ifdef INTR_EP_ENABLE
+		/* By specifying the ep_idx value of 0xff, the cdc layer is asking to
+		* submit an interrupt URB
+		*/
+		if (rxirb == NULL && ep_idx == 0xff) {
+			/* submit intr URB */
+			if ((ret = USB_SUBMIT_URB(usbos_info->intr_urb)) < 0) {
+				DBUSERR(("%s intr USB_SUBMIT_URB failed, status %d\n",
+					__FUNCTION__, ret));
+			}
+			return ret;
+		}
+#else
+		if (rxirb == NULL) {
+			return DBUS_ERR;
+		}
+#endif /* INTR_EP_ENABLE */
+
+	ret = dbus_usbos_recv_urb_submit(usbos_info, rxirb, ep_idx);
+	return ret;
+}
+
+/** Higher layer (dbus_usb.c) want to cancel an IRB */
+static int
+dbus_usbos_intf_cancel_irb(void *bus, dbus_irb_tx_t *txirb)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	/* : Need to implement */
+	return DBUS_ERR;
+}
+
+/** Only one CTL transfer can be pending at any time. This function may block. */
+static int
+dbus_usbos_intf_send_ctl(void *bus, uint8 *buf, int len)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	uint16 size;
+#ifndef USBOS_TX_THREAD
+	int status;
+#endif /* USBOS_TX_THREAD */
+
+	if ((usbos_info == NULL) || (buf == NULL) || (len == 0))
+		return DBUS_ERR;
+
+	if (usbos_info->ctl_urb == NULL)
+		return DBUS_ERR;
+
+	/* Block until a pending CTL transfer has completed */
+	if (down_interruptible(&usbos_info->ctl_lock) != 0) {
+		return DBUS_ERR_TXCTLFAIL;
+	}
+
+#ifdef USBOS_TX_THREAD
+	ASSERT(usbos_info->ctl_state == USBOS_REQUEST_STATE_UNSCHEDULED);
+#else
+	/* Disable USB autosuspend until this request completes, request USB resume if needed.
+	 * Because this call runs asynchronously, there is no guarantee the bus is resumed before
+	 * the URB is submitted, and the URB might be dropped. Use USBOS_TX_THREAD to avoid
+	 * this.
+	 */
+	USB_AUTOPM_GET_INTERFACE_ASYNC(g_probe_info.intf);
+#endif /* USBOS_TX_THREAD */
+
+	size = len;
+	usbos_info->ctl_write.wLength = cpu_to_le16p(&size);
+	usbos_info->ctl_urb->transfer_buffer_length = size;
+
+	usb_fill_control_urb(usbos_info->ctl_urb,
+		usbos_info->usb,
+		usb_sndctrlpipe(usbos_info->usb, 0),
+		(unsigned char *) &usbos_info->ctl_write,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+		buf, size, (void *)dbus_usbos_ctlwrite_complete, usbos_info);
+#else
+		buf, size, (usb_complete_t)dbus_usbos_ctlwrite_complete, usbos_info);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0) */
+
+#ifdef USBOS_TX_THREAD
+	/* Enqueue CTRL request for transmission by the TX thread. The
+	 * USB bus will first be resumed if needed.
+	 */
+	usbos_info->ctl_state = USBOS_REQUEST_STATE_SCHEDULED;
+	wake_up_interruptible(&usbos_info->usbos_tx_queue_head);
+#else
+	status = USB_SUBMIT_URB(usbos_info->ctl_urb);
+	if (status < 0) {
+		DBUSERR(("%s: usb_submit_urb failed %d\n", __FUNCTION__, status));
+		up(&usbos_info->ctl_lock);
+
+		USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+
+		return DBUS_ERR_TXCTLFAIL;
+	}
+#endif /* USBOS_TX_THREAD */
+
+	return DBUS_OK;
+} /* dbus_usbos_intf_send_ctl */
+
+/** This function does not seem to be called by anyone, including dbus_usb.c */
+static int
+dbus_usbos_intf_recv_ctl(void *bus, uint8 *buf, int len)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	int status;
+	uint16 size;
+
+	if ((usbos_info == NULL) || (buf == NULL) || (len == 0))
+		return DBUS_ERR;
+
+	if (usbos_info->ctl_urb == NULL)
+		return DBUS_ERR;
+
+	/* Block until a pending CTRL transfer has completed */
+	if (down_interruptible(&usbos_info->ctl_lock) != 0) {
+		return DBUS_ERR_TXCTLFAIL;
+	}
+
+	/* Disable USB autosuspend until this request completes, request USB resume if needed. */
+	USB_AUTOPM_GET_INTERFACE_ASYNC(g_probe_info.intf);
+
+	size = len;
+	usbos_info->ctl_read.wLength = cpu_to_le16p(&size);
+	usbos_info->ctl_urb->transfer_buffer_length = size;
+
+	if (usbos_info->rxctl_deferrespok) {
+		/* BMAC model */
+		usbos_info->ctl_read.bRequestType = USB_DIR_IN | USB_TYPE_VENDOR |
+			USB_RECIP_INTERFACE;
+		usbos_info->ctl_read.bRequest = DL_DEFER_RESP_OK;
+	} else {
+		/* full dongle model */
+		usbos_info->ctl_read.bRequestType = USB_DIR_IN | USB_TYPE_CLASS |
+			USB_RECIP_INTERFACE;
+		usbos_info->ctl_read.bRequest = 1;
+	}
+
+	usb_fill_control_urb(usbos_info->ctl_urb,
+		usbos_info->usb,
+		usb_rcvctrlpipe(usbos_info->usb, 0),
+		(unsigned char *) &usbos_info->ctl_read,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+		buf, size, (void *)dbus_usbos_ctlread_complete, usbos_info);
+#else
+		buf, size, (usb_complete_t)dbus_usbos_ctlread_complete, usbos_info);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0) */
+
+	status = USB_SUBMIT_URB(usbos_info->ctl_urb);
+	if (status < 0) {
+		DBUSERR(("%s: usb_submit_urb failed %d\n", __FUNCTION__, status));
+		up(&usbos_info->ctl_lock);
+
+		USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+
+		return DBUS_ERR_RXCTLFAIL;
+	}
+
+	return DBUS_OK;
+}
+
+static int
+dbus_usbos_intf_get_attrib(void *bus, dbus_attrib_t *attrib)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+
+	if ((usbos_info == NULL) || (attrib == NULL))
+		return DBUS_ERR;
+
+	attrib->bustype = DBUS_USB;
+	attrib->vid = g_probe_info.vid;
+	attrib->pid = g_probe_info.pid;
+	attrib->devid = 0x4322;
+
+	/* : Need nchan for both TX and RX?;
+	 * BDC uses one RX pipe and one TX pipe
+	 * RPC may use two RX pipes and one TX pipe?
+	 */
+	attrib->nchan = 1;
+
+	/* MaxPacketSize for USB hi-speed bulk out is 512 bytes
+	 * and 64-bytes for full-speed.
+	 * When sending pkt > MaxPacketSize, Host SW breaks it
+	 * up into multiple packets.
+	 */
+	attrib->mtu = usbos_info->maxps;
+
+	return DBUS_OK;
+}
+
+/** Called by higher layer (dbus_usb.c) when it wants to 'up' the USB interface to the dongle */
+static int
+dbus_usbos_intf_up(void *bus)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	uint16 ifnum;
+#ifdef BCMUSBDEV_COMPOSITE
+	int wlan_if = 0;
+#endif
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	if (usbos_info->usb == NULL)
+		return DBUS_ERR;
+
+#if defined(INTR_EP_ENABLE)
+	/* full dongle use intr EP, bmac doesn't use it */
+	if (usbos_info->intr_urb) {
+		int ret;
+
+		usb_fill_int_urb(usbos_info->intr_urb, usbos_info->usb,
+			usbos_info->intr_pipe, &usbos_info->intr,
+			usbos_info->intr_size, (usb_complete_t)dbus_usbos_intr_complete,
+			usbos_info, usbos_info->interval);
+
+		if ((ret = USB_SUBMIT_URB(usbos_info->intr_urb))) {
+			DBUSERR(("%s USB_SUBMIT_URB failed with status %d\n", __FUNCTION__, ret));
+			return DBUS_ERR;
+		}
+	}
+#endif	/* INTR_EP_ENABLE */
+
+	if (usbos_info->ctl_urb) {
+		usbos_info->ctl_in_pipe = usb_rcvctrlpipe(usbos_info->usb, 0);
+		usbos_info->ctl_out_pipe = usb_sndctrlpipe(usbos_info->usb, 0);
+
+#ifdef BCMUSBDEV_COMPOSITE
+		wlan_if = dbus_usbos_intf_wlan(usbos_info->usb);
+		ifnum = cpu_to_le16(IFDESC(usbos_info->usb, wlan_if).bInterfaceNumber);
+#else
+		ifnum = cpu_to_le16(IFDESC(usbos_info->usb, CONTROL_IF).bInterfaceNumber);
+#endif /* BCMUSBDEV_COMPOSITE */
+		/* CTL Write */
+		usbos_info->ctl_write.bRequestType =
+			USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
+		usbos_info->ctl_write.bRequest = 0;
+		usbos_info->ctl_write.wValue = cpu_to_le16(0);
+		usbos_info->ctl_write.wIndex = cpu_to_le16p(&ifnum);
+
+		/* CTL Read */
+		usbos_info->ctl_read.bRequestType =
+			USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
+		usbos_info->ctl_read.bRequest = 1;
+		usbos_info->ctl_read.wValue = cpu_to_le16(0);
+		usbos_info->ctl_read.wIndex = cpu_to_le16p(&ifnum);
+	}
+
+	/* Success, indicate usbos_info is fully up */
+	dbus_usbos_state_change(usbos_info, DBUS_STATE_UP);
+
+	return DBUS_OK;
+} /* dbus_usbos_intf_up */
+
+static int
+dbus_usbos_intf_down(void *bus)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	dbusos_stop(usbos_info);
+	return DBUS_OK;
+}
+
+static int
+dbus_usbos_intf_stop(void *bus)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	dbusos_stop(usbos_info);
+	return DBUS_OK;
+}
+
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+static void
+dbus_usbos_intf_dump(void *bus, struct bcmstrbuf *b)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	int i = 0, j = 0, rxposted, txposted;
+
+	rxposted = atomic_read(&usbos_info->rxposted);
+	txposted = atomic_read(&usbos_info->txposted);
+	if (b) {
+		bcm_bprintf(b, "\ndbus linux dump\n");
+		bcm_bprintf(b, "txposted %d rxposted %d\n",
+			txposted, rxposted);
+
+		bcm_bprintf(b, "RXDPC: dpc_cnt %d dpc_pktcnt %d dpc_maxpktcnt %d avg_dpc_pktcnt\n",
+			usbos_info->dpc_cnt, usbos_info->dpc_pktcnt,
+			usbos_info->dpc_maxpktcnt, usbos_info->dpc_cnt ?
+			(usbos_info->dpc_pktcnt/usbos_info->dpc_cnt):1);
+
+		/* Histogram */
+		bcm_bprintf(b, "txposted\n");
+	} else {
+		printf("\ndbus linux dump\n");
+		printf("txposted %d rxposted %d\n",
+			txposted, rxposted);
+		printf("RXDPC: dpc_cnt %d dpc_pktcnt %d dpc_maxpktcnt %d avg_dpc_pktcnt %d\n",
+			usbos_info->dpc_cnt, usbos_info->dpc_pktcnt,
+			usbos_info->dpc_maxpktcnt, usbos_info->dpc_cnt ?
+			(usbos_info->dpc_pktcnt/usbos_info->dpc_cnt):1);
+
+		/* Histogram */
+		printf("txposted\n");
+	}
+
+	for (i = 0; i < usbos_info->pub->ntxq; i++) {
+		if (usbos_info->txposted_hist == NULL) {
+			break;
+		}
+		if (usbos_info->txposted_hist[i]) {
+			if (b)
+				bcm_bprintf(b, "%d: %d ", i, usbos_info->txposted_hist[i]);
+			else
+				printf("%d: %d ", i, usbos_info->txposted_hist[i]);
+			j++;
+			if (j % 10 == 0) {
+				if (b)
+					bcm_bprintf(b, "\n");
+				else
+					printf("\n");
+			}
+		}
+	}
+
+	j = 0;
+	if (b)
+		bcm_bprintf(b, "\nrxposted\n");
+	else
+		printf("\nrxposted\n");
+	for (i = 0; i < usbos_info->pub->nrxq; i++) {
+		if (usbos_info->rxposted_hist == NULL) {
+			break;
+		}
+		if (usbos_info->rxposted_hist[i]) {
+			if (b)
+				bcm_bprintf(b, "%d: %d ", i, usbos_info->rxposted_hist[i]);
+			else
+				printf("%d: %d ", i, usbos_info->rxposted_hist[i]);
+			j++;
+			if (j % 10 == 0) {
+				if (b)
+					bcm_bprintf(b, "\n");
+				else
+					printf("\n");
+			}
+		}
+	}
+	if (b)
+		bcm_bprintf(b, "\n");
+	else
+		printf("\n");
+
+	return;
+}
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+
+/** Called by higher layer (dbus_usb.c) */
+static int
+dbus_usbos_intf_set_config(void *bus, dbus_config_t *config)
+{
+	int err = DBUS_ERR;
+	usbos_info_t* usbos_info = bus;
+
+	if (config->config_id == DBUS_CONFIG_ID_RXCTL_DEFERRES) {
+		usbos_info->rxctl_deferrespok = config->rxctl_deferrespok;
+		err = DBUS_OK;
+	}
+#ifdef KEEPIF_ON_DEVICE_RESET
+	else if (config->config_id == DBUS_CONFIG_ID_KEEPIF_ON_DEVRESET) {
+		g_probe_info.keepif_on_devreset = config->general_param ? TRUE : FALSE;
+		err = DBUS_OK;
+	}
+#endif /* KEEPIF_ON_DEVICE_RESET */
+	else if (config->config_id == DBUS_CONFIG_ID_AGGR_LIMIT) {
+#ifndef BCM_FD_AGGR
+		/* DBUS_CONFIG_ID_AGGR_LIMIT shouldn't be called after probe stage */
+		ASSERT(disc_arg == NULL);
+#endif /* BCM_FD_AGGR */
+		ASSERT(config->aggr_param.maxrxsf > 0);
+		ASSERT(config->aggr_param.maxrxsize > 0);
+		if (config->aggr_param.maxrxsize > usbos_info->rxbuf_len) {
+			int state = usbos_info->pub->busstate;
+			dbus_usbos_unlink(&usbos_info->req_rxpostedq, &usbos_info->rxposted_lock);
+			while (atomic_read(&usbos_info->rxposted)) {
+				DBUSTRACE(("%s rxposted is %d, delay 1 ms\n", __FUNCTION__,
+					atomic_read(&usbos_info->rxposted)));
+				dbus_usbos_wait(usbos_info, 1);
+			}
+			usbos_info->rxbuf_len = config->aggr_param.maxrxsize;
+			dbus_usbos_state_change(usbos_info, state);
+		}
+		err = DBUS_OK;
+	}
+
+	return err;
+}
+
+#ifdef EHCI_FASTPATH_TX
+
+/**
+ * In some cases, the code must submit an URB and wait for its completion.
+ * Related: dbus_usbos_sync_complete()
+ */
+static int
+dbus_usbos_sync_wait(usbos_info_t *usbinfo, uint16 time)
+{
+	int ret;
+	int err = DBUS_OK;
+	int ms = time;
+
+	ret = wait_event_interruptible_timeout(usbinfo->wait,
+		usbinfo->waitdone == TRUE, (ms * HZ / 1000));
+
+	if ((usbinfo->waitdone == FALSE) || (usbinfo->sync_urb_status)) {
+		DBUSERR(("%s: timeout(%d) or urb err=0x%x\n",
+			__FUNCTION__, ret, usbinfo->sync_urb_status));
+		err = DBUS_ERR;
+		BCM_REFERENCE(ret);
+	}
+	usbinfo->waitdone = FALSE;
+	return err;
+}
+
+#endif /* EHCI_FASTPATH_TX */
+
+/** Called by dbus_usb.c when it wants to download firmware into the dongle */
+bool
+dbus_usbos_dl_cmd(usbos_info_t *usbinfo, uint8 cmd, void *buffer, int buflen)
+{
+	int transferred;
+	int index = 0;
+	char *tmpbuf;
+
+	if ((usbinfo == NULL) || (buffer == NULL) || (buflen == 0))
+		return FALSE;
+
+	/* PR82642: Input buffer on stack is causing failures
+	 * on some platforms (STB boxes)
+	 */
+	tmpbuf = (char *) MALLOC(usbinfo->pub->osh, buflen);
+	if (!tmpbuf) {
+		DBUSERR(("%s: Unable to allocate memory \n", __FUNCTION__));
+		return FALSE;
+	}
+
+#ifdef BCM_REQUEST_FW
+	/* Adopted from Windows layer dbus_usb_ndis.c to support USB3.0 core */
+	if (cmd == DL_GO) {
+		index = 1;
+	}
+#endif
+
+	/* Disable USB autosuspend until this request completes, request USB resume if needed. */
+	USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+	transferred = USB_CONTROL_MSG(usbinfo->usb, usb_rcvctrlpipe(usbinfo->usb, 0),
+		cmd, (USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE),
+		0, index,
+		(void*) tmpbuf, buflen, USB_CTRL_EP_TIMEOUT);
+	if (transferred == buflen) {
+		memcpy(buffer, tmpbuf, buflen);
+	} else {
+		DBUSERR(("%s: usb_control_msg failed %d\n", __FUNCTION__, transferred));
+	}
+
+	USB_AUTOPM_PUT_INTERFACE(g_probe_info.intf);
+
+	MFREE(usbinfo->pub->osh, tmpbuf, buflen);
+	return (transferred == buflen);
+}
+
+/**
+ * Called by dbus_usb.c when it wants to download a buffer into the dongle (eg as part of the
+ * download process, when writing nvram variables).
+ */
+int
+dbus_write_membytes(usbos_info_t* usbinfo, bool set, uint32 address, uint8 *data, uint size)
+{
+	hwacc_t hwacc;
+	int write_bytes = 4;
+	int status;
+	int retval = 0;
+
+	DBUSTRACE(("Enter:%s\n", __FUNCTION__));
+
+	/* Read is not supported */
+	if (set == 0) {
+		DBUSERR(("Currently read is not supported!!\n"));
+		return -1;
+	}
+
+	USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+	hwacc.cmd = DL_CMD_WRHW;
+	hwacc.addr = address;
+
+	DBUSTRACE(("Address:%x size:%d", hwacc.addr, size));
+	do {
+		if (size >= 4) {
+			write_bytes = 4;
+		} else if (size >= 2) {
+			write_bytes = 2;
+		} else {
+			write_bytes = 1;
+		}
+
+		hwacc.len = write_bytes;
+
+		while (size >= write_bytes) {
+			hwacc.data = *((unsigned int*)data);
+
+			status = USB_CONTROL_MSG(usbinfo->usb, usb_sndctrlpipe(usbinfo->usb, 0),
+				DL_WRHW, (USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE),
+				1, 0, (char *)&hwacc, sizeof(hwacc_t), USB_CTRL_EP_TIMEOUT);
+
+			if (status < 0) {
+				retval = -1;
+				DBUSERR((" Ctrl write hwacc failed w/status %d @ address:%x \n",
+					status, hwacc.addr));
+				goto err;
+			}
+
+			hwacc.addr += write_bytes;
+			data += write_bytes;
+			size -= write_bytes;
+		}
+	} while (size > 0);
+
+err:
+	USB_AUTOPM_PUT_INTERFACE(g_probe_info.intf);
+
+	return retval;
+} /* dbus_write_membytes */
+
+int
+dbus_usbos_readreg(void *bus, uint32 regaddr, int datalen, uint32 *value)
+{
+	usbos_info_t *usbinfo = (usbos_info_t *) bus;
+	int ret = DBUS_OK;
+	int transferred;
+	uint32 cmd;
+	hwacc_t	hwacc;
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	if (datalen == 1)
+		cmd = DL_RDHW8;
+	else if (datalen == 2)
+		cmd = DL_RDHW16;
+	else
+		cmd = DL_RDHW32;
+
+	USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+	transferred = USB_CONTROL_MSG(usbinfo->usb, usb_rcvctrlpipe(usbinfo->usb, 0),
+		cmd, (USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE),
+		(uint16)(regaddr), (uint16)(regaddr >> 16),
+		(void *) &hwacc, sizeof(hwacc_t), USB_CTRL_EP_TIMEOUT);
+
+	if (transferred >= sizeof(hwacc_t)) {
+		*value = hwacc.data;
+	} else {
+		DBUSERR(("%s: usb_control_msg failed %d\n", __FUNCTION__, transferred));
+		ret = DBUS_ERR;
+	}
+
+	USB_AUTOPM_PUT_INTERFACE(g_probe_info.intf);
+
+	return ret;
+} /* dbus_usbos_readreg */
+
+int
+dbus_usbos_writereg(void *bus, uint32 regaddr, int datalen, uint32 data)
+{
+	usbos_info_t *usbinfo = (usbos_info_t *) bus;
+	int ret = DBUS_OK;
+	int transferred;
+	uint32 cmd = DL_WRHW;
+	hwacc_t	hwacc;
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+	hwacc.cmd = DL_WRHW;
+	hwacc.addr = regaddr;
+	hwacc.data = data;
+	hwacc.len = datalen;
+
+	transferred = USB_CONTROL_MSG(usbinfo->usb, usb_sndctrlpipe(usbinfo->usb, 0),
+		cmd, (USB_DIR_OUT| USB_TYPE_VENDOR | USB_RECIP_INTERFACE),
+		1, 0,
+		(void *) &hwacc, sizeof(hwacc_t), USB_CTRL_EP_TIMEOUT);
+
+	if (transferred != sizeof(hwacc_t)) {
+		DBUSERR(("%s: usb_control_msg failed %d\n", __FUNCTION__, transferred));
+		ret = DBUS_ERR;
+	}
+
+	USB_AUTOPM_PUT_INTERFACE(g_probe_info.intf);
+
+	return ret;
+}
+
+int
+dbus_usbos_wait(usbos_info_t *usbinfo, uint16 ms)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0))
+	/* : Need to test under 2.6 kernel */
+	if (in_interrupt())
+		mdelay(ms);
+	else
+		msleep_interruptible(ms);
+#else
+	wait_ms(ms);
+#endif
+	return DBUS_OK;
+}
+
+/** Called by dbus_usb.c as part of the firmware download process */
+bool
+dbus_usbos_dl_send_bulk(usbos_info_t *usbinfo, void *buffer, int len)
+{
+#ifdef EHCI_FASTPATH_TX
+	int ret = DBUS_ERR;
+
+	struct ehci_qtd *qtd = optimize_ehci_qtd_alloc(GFP_KERNEL);
+	int token = EHCI_QTD_SET_CERR(3);
+
+	if (qtd == NULL)
+		goto fail;
+
+	optimize_qtd_fill_with_data(usbinfo->pub, 0, qtd, buffer, token, len);
+	optimize_submit_async(qtd, 0);
+
+	ret = dbus_usbos_sync_wait(usbinfo, USB_SYNC_WAIT_TIMEOUT);
+
+	return (ret == DBUS_OK);
+fail:
+	return FALSE;
+#else
+	bool ret = TRUE;
+	int status;
+	int transferred = 0;
+
+	if (usbinfo == NULL)
+		return DBUS_ERR;
+
+	USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+	status = USB_BULK_MSG(usbinfo->usb, usbinfo->tx_pipe,
+		buffer, len,
+		&transferred, USB_BULK_EP_TIMEOUT);
+
+	if (status < 0) {
+		DBUSERR(("%s: usb_bulk_msg failed %d\n", __FUNCTION__, status));
+		ret = FALSE;
+	}
+
+	USB_AUTOPM_PUT_INTERFACE(g_probe_info.intf);
+
+	return ret;
+#endif /* EHCI_FASTPATH_TX */
+}
+
+static bool
+dbus_usbos_intf_recv_needed(void *bus)
+{
+	return FALSE;
+}
+
+/**
+ * Higher layer (dbus_usb.c) wants to execute a function on the condition that the rx spin lock has
+ * been acquired.
+ */
+static void*
+dbus_usbos_intf_exec_rxlock(void *bus, exec_cb_t cb, struct exec_parms *args)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	void *ret;
+	unsigned long flags;
+
+	if (usbos_info == NULL)
+		return NULL;
+
+	spin_lock_irqsave(&usbos_info->rxlock, flags);
+	ret = cb(args);
+	spin_unlock_irqrestore(&usbos_info->rxlock, flags);
+
+	return ret;
+}
+
+static void*
+dbus_usbos_intf_exec_txlock(void *bus, exec_cb_t cb, struct exec_parms *args)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+	void *ret;
+	unsigned long flags;
+
+	if (usbos_info == NULL)
+		return NULL;
+
+	spin_lock_irqsave(&usbos_info->txlock, flags);
+	ret = cb(args);
+	spin_unlock_irqrestore(&usbos_info->txlock, flags);
+
+	return ret;
+}
+
+/**
+ * if an error condition was detected in this module, the higher DBUS layer (dbus_usb.c) has to
+ * be notified.
+ */
+int
+dbus_usbos_errhandler(void *bus, int err)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	if (usbos_info->cbarg && usbos_info->cbs) {
+		if (usbos_info->cbs->errhandler)
+			usbos_info->cbs->errhandler(usbos_info->cbarg, err);
+	}
+
+	return DBUS_OK;
+}
+
+/**
+ * if a change in bus state was detected in this module, the higher DBUS layer (dbus_usb.c) has to
+ * be notified.
+ */
+int
+dbus_usbos_state_change(void *bus, int state)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) bus;
+
+	if (usbos_info == NULL)
+		return DBUS_ERR;
+
+	if (usbos_info->cbarg && usbos_info->cbs) {
+		if (usbos_info->cbs->state_change)
+			usbos_info->cbs->state_change(usbos_info->cbarg, state);
+	}
+
+	usbos_info->pub->busstate = state;
+	return DBUS_OK;
+}
+
+int
+dbus_bus_osl_register(dbus_driver_t *driver, dbus_intf_t **intf)
+{
+	bzero(&g_probe_info, sizeof(probe_info_t));
+
+	drvinfo = *driver;
+	*intf = &dbus_usbos_intf;
+
+	USB_REGISTER();
+
+#ifdef KEEPIF_ON_DEVICE_RESET
+	init_waitqueue_head(&g_probe_info.usbreset_queue_head);
+	atomic_set(&g_probe_info.usbdev_stat, USB_DEVICE_INIT);
+	g_probe_info.usbreset_kt = kthread_create(dbus_usbos_usbreset_func,
+		&g_probe_info, "usbreset-thread");
+	ASSERT(!IS_ERR(g_probe_info.usbreset_kt));
+	wake_up_process(g_probe_info.usbreset_kt);
+#endif /* KEEPIF_ON_DEVICE_RESET */
+
+	return DBUS_ERR_NODEVICE;
+}
+
+int
+dbus_bus_osl_deregister()
+{
+	g_probe_info.dereged = TRUE;
+
+	DHD_MUTEX_LOCK();
+#ifdef KEEPIF_ON_DEVICE_RESET
+	wake_up_interruptible(&g_probe_info.usbreset_queue_head);
+	kthread_stop(g_probe_info.usbreset_kt);
+#endif /* KEEPIF_ON_DEVICE_RESET */
+
+	if (drvinfo.remove && disc_arg && (g_probe_info.disc_cb_done == FALSE)) {
+		drvinfo.remove(disc_arg);
+		disc_arg = NULL;
+	}
+	DHD_MUTEX_UNLOCK();
+
+	USB_DEREGISTER();
+
+	return DBUS_OK;
+}
+
+static void
+dbus_usbos_init_info(void)
+{
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+
+	if (!usbos_info) {
+		return;
+	}
+
+	/* Update USB Info */
+	usbos_info->usb = g_probe_info.usb;
+	usbos_info->rx_pipe = g_probe_info.rx_pipe;
+	usbos_info->rx_pipe2 = g_probe_info.rx_pipe2;
+	usbos_info->tx_pipe = g_probe_info.tx_pipe;
+	usbos_info->intr_pipe = g_probe_info.intr_pipe;
+	usbos_info->intr_size = g_probe_info.intr_size;
+	usbos_info->interval = g_probe_info.interval;
+	usbos_info->pub->device_speed = g_probe_info.device_speed;
+	usbos_info->pub->dev_info = g_probe_info.usb;
+	if (usbos_info->rx_pipe2) {
+		usbos_info->pub->attrib.has_2nd_bulk_in_ep = 1;
+	} else {
+		usbos_info->pub->attrib.has_2nd_bulk_in_ep = 0;
+	}
+
+	if (usbos_info->tx_pipe)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+		usbos_info->maxps = usb_maxpacket(usbos_info->usb,
+			usbos_info->tx_pipe);
+#else
+		usbos_info->maxps = usb_maxpacket(usbos_info->usb,
+			usbos_info->tx_pipe, usb_pipeout(usbos_info->tx_pipe));
+#endif  /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0) */
+}
+
+void *
+dbus_usbos_intf_attach(dbus_pub_t *pub, void *cbarg, dbus_intf_callbacks_t *cbs)
+{
+	usbos_info_t *usbos_info;
+
+	if (g_probe_info.dldone == FALSE) {
+		DBUSERR(("%s: err device not downloaded!\n", __FUNCTION__));
+		return NULL;
+	}
+
+	/* Sanity check for BUS_INFO() */
+	ASSERT(OFFSETOF(usbos_info_t, pub) == 0);
+
+	usbos_info = MALLOC(pub->osh, sizeof(usbos_info_t));
+	if (usbos_info == NULL)
+		return NULL;
+
+	bzero(usbos_info, sizeof(usbos_info_t));
+
+	usbos_info->pub = pub;
+	usbos_info->cbarg = cbarg;
+	usbos_info->cbs = cbs;
+
+	/* Needed for disconnect() */
+	g_probe_info.usbos_info = usbos_info;
+	dbus_usbos_init_info();
+
+	INIT_LIST_HEAD(&usbos_info->req_rxfreeq);
+	INIT_LIST_HEAD(&usbos_info->req_txfreeq);
+	INIT_LIST_HEAD(&usbos_info->req_rxpostedq);
+	INIT_LIST_HEAD(&usbos_info->req_txpostedq);
+	spin_lock_init(&usbos_info->rxfree_lock);
+	spin_lock_init(&usbos_info->txfree_lock);
+	spin_lock_init(&usbos_info->rxposted_lock);
+	spin_lock_init(&usbos_info->txposted_lock);
+	spin_lock_init(&usbos_info->rxlock);
+	spin_lock_init(&usbos_info->txlock);
+
+	atomic_set(&usbos_info->rxposted, 0);
+	atomic_set(&usbos_info->txposted, 0);
+
+#ifdef DBUS_LINUX_RXDPC
+	INIT_LIST_HEAD(&usbos_info->req_rxpendingq);
+	spin_lock_init(&usbos_info->rxpending_lock);
+#endif /* DBUS_LINUX_RXDPC */
+
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	usbos_info->txposted_hist = MALLOC(pub->osh, (usbos_info->pub->ntxq+1) * sizeof(int));
+	if (usbos_info->txposted_hist) {
+		bzero(usbos_info->txposted_hist, (usbos_info->pub->ntxq+1) * sizeof(int));
+	}
+	usbos_info->rxposted_hist = MALLOC(pub->osh, (usbos_info->pub->nrxq+1) * sizeof(int));
+	if (usbos_info->rxposted_hist) {
+		bzero(usbos_info->rxposted_hist, (usbos_info->pub->nrxq+1) * sizeof(int));
+	}
+#endif
+#ifdef USB_DISABLE_INT_EP
+	/* PR 82639  Don't use interrupt EP */
+	usbos_info->intr_urb = NULL;
+#else
+	if (!(usbos_info->intr_urb = USB_ALLOC_URB())) {
+		DBUSERR(("%s: usb_alloc_urb (tx) failed\n", __FUNCTION__));
+		goto fail;
+	}
+#endif
+
+	if (!(usbos_info->ctl_urb = USB_ALLOC_URB())) {
+		DBUSERR(("%s: usb_alloc_urb (tx) failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	init_waitqueue_head(&usbos_info->wait);
+
+	if (!(usbos_info->blk_urb = USB_ALLOC_URB())) {	/* for embedded image downloading */
+		DBUSERR(("%s: usb_alloc_urb (tx) failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	usbos_info->rxbuf_len = (uint)usbos_info->pub->rxsize;
+
+#ifdef DBUS_LINUX_RXDPC		/* Initialize DPC thread */
+	sema_init(&usbos_info->dpc_sem, 0);
+	init_completion(&usbos_info->dpc_exited);
+	usbos_info->dpc_pid = kernel_thread(dbus_usbos_dpc_thread, usbos_info, 0);
+	if (usbos_info->dpc_pid < 0) {
+		DBUSERR(("%s: failed to create dpc thread\n", __FUNCTION__));
+		goto fail;
+	}
+#endif /* DBUS_LINUX_RXDPC */
+
+	atomic_set(&usbos_info->txallocated, 0);
+	if (DBUS_OK != dbus_usbos_urbreqs_alloc(usbos_info,
+		usbos_info->pub->ntxq, FALSE)) {
+		goto fail;
+	}
+
+	atomic_set(&usbos_info->rxallocated, 0);
+	if (DBUS_OK != dbus_usbos_urbreqs_alloc(usbos_info,
+#ifdef CTFPOOL
+		usbos_info->pub->nrxq,
+#else
+		MIN(DBUS_USB_RXQUEUE_BATCH_ADD, usbos_info->pub->nrxq),
+#endif
+		TRUE)) {
+		goto fail;
+	}
+
+	sema_init(&usbos_info->ctl_lock, 1);
+
+#ifdef USBOS_THREAD
+	if (dbus_usbos_thread_init(usbos_info) == NULL)
+		goto fail;
+#endif /* USBOS_THREAD */
+
+#ifdef USBOS_TX_THREAD
+	if (dbus_usbos_tx_thread_init(usbos_info) == NULL)
+		goto fail;
+#endif /* USBOS_TX_THREAD */
+
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX)
+	spin_lock_init(&usbos_info->fastpath_lock);
+	if (optimize_init(usbos_info, usbos_info->usb, usbos_info->tx_pipe,
+		usbos_info->rx_pipe, usbos_info->rx_pipe2) != 0) {
+		DBUSERR(("%s: optimize_init failed!\n", __FUNCTION__));
+		goto fail;
+	}
+
+#endif /* EHCI_FASTPATH_TX || EHCI_FASTPATH_RX */
+
+	return (void *) usbos_info;
+fail:
+#ifdef DBUS_LINUX_RXDPC
+	if (usbos_info->dpc_pid >= 0) {
+		KILL_PROC(usbos_info->dpc_pid, SIGTERM);
+		wait_for_completion(&usbos_info->dpc_exited);
+	}
+#endif /* DBUS_LINUX_RXDPC */
+	if (usbos_info->intr_urb) {
+		USB_FREE_URB(usbos_info->intr_urb);
+		usbos_info->intr_urb = NULL;
+	}
+
+	if (usbos_info->ctl_urb) {
+		USB_FREE_URB(usbos_info->ctl_urb);
+		usbos_info->ctl_urb = NULL;
+	}
+
+#if defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW)
+	if (usbos_info->blk_urb) {
+		USB_FREE_URB(usbos_info->blk_urb);
+		usbos_info->blk_urb = NULL;
+	}
+#endif
+
+	dbus_usbos_urbreqs_free(usbos_info, TRUE);
+	atomic_set(&usbos_info->rxallocated, 0);
+	dbus_usbos_urbreqs_free(usbos_info, FALSE);
+	atomic_set(&usbos_info->txallocated, 0);
+
+	g_probe_info.usbos_info = NULL;
+
+	MFREE(pub->osh, usbos_info, sizeof(usbos_info_t));
+	return NULL;
+} /* dbus_usbos_intf_attach */
+
+void
+dbus_usbos_intf_detach(dbus_pub_t *pub, void *info)
+{
+	usbos_info_t *usbos_info = (usbos_info_t *) info;
+	osl_t *osh = pub->osh;
+
+	if (usbos_info == NULL) {
+		return;
+	}
+
+#ifdef USBOS_TX_THREAD
+	dbus_usbos_tx_thread_deinit(usbos_info);
+#endif /* USBOS_TX_THREAD */
+
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX)
+	optimize_deinit(usbos_info, usbos_info->usb);
+#endif
+	/* Must unlink all URBs prior to driver unload;
+	 * otherwise an URB callback can occur after driver
+	 * has been de-allocated and rmmod'd
+	 */
+	dbusos_stop(usbos_info);
+
+	if (usbos_info->intr_urb) {
+		USB_FREE_URB(usbos_info->intr_urb);
+		usbos_info->intr_urb = NULL;
+	}
+
+	if (usbos_info->ctl_urb) {
+		USB_FREE_URB(usbos_info->ctl_urb);
+		usbos_info->ctl_urb = NULL;
+	}
+
+	if (usbos_info->blk_urb) {
+		USB_FREE_URB(usbos_info->blk_urb);
+		usbos_info->blk_urb = NULL;
+	}
+
+	dbus_usbos_urbreqs_free(usbos_info, TRUE);
+	atomic_set(&usbos_info->rxallocated, 0);
+	dbus_usbos_urbreqs_free(usbos_info, FALSE);
+	atomic_set(&usbos_info->txallocated, 0);
+
+#if defined(BCMDBG) || defined(DBUS_LINUX_HIST)
+	if (usbos_info->txposted_hist) {
+		MFREE(osh, usbos_info->txposted_hist, (usbos_info->pub->ntxq+1) * sizeof(int));
+	}
+	if (usbos_info->rxposted_hist) {
+		MFREE(osh, usbos_info->rxposted_hist, (usbos_info->pub->nrxq+1) * sizeof(int));
+	}
+#endif /* BCMDBG || DBUS_LINUX_HIST */
+#ifdef USBOS_THREAD
+	dbus_usbos_thread_deinit(usbos_info);
+#endif /* USBOS_THREAD */
+
+	g_probe_info.usbos_info = NULL;
+	MFREE(osh, usbos_info, sizeof(usbos_info_t));
+} /* dbus_usbos_intf_detach */
+
+#ifdef USBOS_TX_THREAD
+
+void*
+dbus_usbos_tx_thread_init(usbos_info_t *usbos_info)
+{
+	spin_lock_init(&usbos_info->usbos_tx_list_lock);
+	INIT_LIST_HEAD(&usbos_info->usbos_tx_list);
+	init_waitqueue_head(&usbos_info->usbos_tx_queue_head);
+
+	usbos_info->usbos_tx_kt = kthread_create(dbus_usbos_tx_thread_func,
+		usbos_info, "usb-tx-thread");
+
+	if (IS_ERR(usbos_info->usbos_tx_kt)) {
+		DBUSERR(("Thread Creation failed\n"));
+		return (NULL);
+	}
+
+	usbos_info->ctl_state = USBOS_REQUEST_STATE_UNSCHEDULED;
+	wake_up_process(usbos_info->usbos_tx_kt);
+
+	return (usbos_info->usbos_tx_kt);
+}
+
+void
+dbus_usbos_tx_thread_deinit(usbos_info_t *usbos_info)
+{
+	urb_req_t *req;
+
+	if (usbos_info->usbos_tx_kt) {
+		wake_up_interruptible(&usbos_info->usbos_tx_queue_head);
+		kthread_stop(usbos_info->usbos_tx_kt);
+	}
+
+	/* Move pending requests to free queue so they can be freed */
+	while ((req = dbus_usbos_qdeq(
+		&usbos_info->usbos_tx_list, &usbos_info->usbos_tx_list_lock)) != NULL) {
+		dbus_usbos_qenq(&usbos_info->req_txfreeq, req, &usbos_info->txfree_lock);
+	}
+}
+
+/**
+ * Allow USB in-band resume to block by submitting CTRL and DATA URBs on a separate thread.
+ */
+int
+dbus_usbos_tx_thread_func(void *data)
+{
+	usbos_info_t  *usbos_info = (usbos_info_t *)data;
+	urb_req_t     *req;
+	dbus_irb_tx_t *txirb;
+	int           ret;
+	unsigned long flags;
+
+#ifdef WL_THREADNICE
+	set_user_nice(current, WL_THREADNICE);
+#endif
+
+	while (1) {
+		/* Wait until there are URBs to submit */
+		wait_event_interruptible_timeout(
+			usbos_info->usbos_tx_queue_head,
+			!list_empty(&usbos_info->usbos_tx_list) ||
+			usbos_info->ctl_state == USBOS_REQUEST_STATE_SCHEDULED,
+			100);
+
+		if (kthread_should_stop())
+			break;
+
+		/* Submit CTRL URB if needed */
+		if (usbos_info->ctl_state == USBOS_REQUEST_STATE_SCHEDULED) {
+
+			/* Disable USB autosuspend until this request completes. If the
+			 * interface was suspended, this call blocks until it has been resumed.
+			 */
+			USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+			usbos_info->ctl_state = USBOS_REQUEST_STATE_SUBMITTED;
+
+			ret = USB_SUBMIT_URB(usbos_info->ctl_urb);
+			if (ret != 0) {
+				DBUSERR(("%s CTRL USB_SUBMIT_URB failed, status %d\n",
+					__FUNCTION__, ret));
+
+				usbos_info->ctl_state = USBOS_REQUEST_STATE_UNSCHEDULED;
+				up(&usbos_info->ctl_lock);
+
+				USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+			}
+		}
+
+		/* Submit all available TX URBs */
+		while ((req = dbus_usbos_qdeq(&usbos_info->usbos_tx_list,
+			&usbos_info->usbos_tx_list_lock)) != NULL) {
+
+			/* Disable USB autosuspend until this request completes. If the
+			 * interface was suspended, this call blocks until it has been resumed.
+			 */
+			USB_AUTOPM_GET_INTERFACE(g_probe_info.intf);
+
+			spin_lock_irqsave(&usbos_info->txlock, flags);
+
+			ret = USB_SUBMIT_URB(req->urb);
+			if (ret == 0) {
+				/* URB submitted successfully */
+				dbus_usbos_qenq(&usbos_info->req_txpostedq, req,
+					&usbos_info->txposted_lock);
+				atomic_inc(&usbos_info->txposted);
+			} else {
+				/* Submitting the URB failed. */
+				DBUSERR(("%s TX USB_SUBMIT_URB failed, status %d\n",
+					__FUNCTION__, ret));
+
+				USB_AUTOPM_PUT_INTERFACE_ASYNC(g_probe_info.intf);
+			}
+
+			spin_unlock_irqrestore(&usbos_info->txlock, flags);
+
+			if (ret != 0) {
+				/* Cleanup and notify higher layers */
+				dbus_usbos_qenq(&usbos_info->req_txfreeq, req,
+					&usbos_info->txfree_lock);
+
+				txirb = req->arg;
+				if (txirb->send_buf) {
+					MFREE(usbos_info->pub->osh, txirb->send_buf, req->buf_len);
+					req->buf_len = 0;
+				}
+
+				if (likely (usbos_info->cbarg && usbos_info->cbs)) {
+					if (likely (usbos_info->cbs->send_irb_complete != NULL))
+						usbos_info->cbs->send_irb_complete(
+							usbos_info->cbarg, txirb, DBUS_ERR_TXDROP);
+				}
+			}
+		}
+	}
+
+	return 0;
+} /* dbus_usbos_tx_thread_func */
+
+#endif /* USBOS_TX_THREAD */
+
+#ifdef USBOS_THREAD
+
+/**
+ * Increase system performance by creating a USB thread that runs parallel to other system
+ * activity.
+ */
+static void*
+dbus_usbos_thread_init(usbos_info_t *usbos_info)
+{
+	usbos_list_entry_t  *entry;
+	unsigned long       flags, ii;
+
+	spin_lock_init(&usbos_info->usbos_list_lock);
+	INIT_LIST_HEAD(&usbos_info->usbos_list);
+	INIT_LIST_HEAD(&usbos_info->usbos_free_list);
+	init_waitqueue_head(&usbos_info->usbos_queue_head);
+	atomic_set(&usbos_info->usbos_list_cnt, 0);
+
+	for (ii = 0; ii < (usbos_info->pub->nrxq + usbos_info->pub->ntxq); ii++) {
+		entry = MALLOC(usbos_info->pub->osh, sizeof(usbos_list_entry_t));
+		if (entry) {
+			spin_lock_irqsave(&usbos_info->usbos_list_lock, flags);
+			list_add_tail((struct list_head*) entry, &usbos_info->usbos_free_list);
+			spin_unlock_irqrestore(&usbos_info->usbos_list_lock, flags);
+		} else {
+			DBUSERR(("Failed to create list\n"));
+		}
+	}
+
+	usbos_info->usbos_kt = kthread_create(dbus_usbos_thread_func,
+		usbos_info, "usb-thread");
+
+	if (IS_ERR(usbos_info->usbos_kt)) {
+		DBUSERR(("Thread Creation failed\n"));
+		return (NULL);
+	}
+
+	wake_up_process(usbos_info->usbos_kt);
+
+	return (usbos_info->usbos_kt);
+}
+
+static void
+dbus_usbos_thread_deinit(usbos_info_t *usbos_info)
+{
+	struct list_head    *cur, *next;
+	usbos_list_entry_t  *entry;
+	unsigned long       flags;
+
+	if (usbos_info->usbos_kt) {
+		wake_up_interruptible(&usbos_info->usbos_queue_head);
+		kthread_stop(usbos_info->usbos_kt);
+	}
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_safe(cur, next, &usbos_info->usbos_list)
+	{
+		entry = list_entry(cur, struct usbos_list_entry, list);
+		/* detach this entry from the list and then free the entry */
+		spin_lock_irqsave(&usbos_info->usbos_list_lock, flags);
+		list_del(cur);
+		MFREE(usbos_info->pub->osh, entry, sizeof(usbos_list_entry_t));
+		spin_unlock_irqrestore(&usbos_info->usbos_list_lock, flags);
+	}
+
+	list_for_each_safe(cur, next, &usbos_info->usbos_free_list)
+	{
+		entry = list_entry(cur, struct usbos_list_entry, list);
+		/* detach this entry from the list and then free the entry */
+		spin_lock_irqsave(&usbos_info->usbos_list_lock, flags);
+		list_del(cur);
+		MFREE(usbos_info->pub->osh, entry, sizeof(usbos_list_entry_t));
+		spin_unlock_irqrestore(&usbos_info->usbos_list_lock, flags);
+	}
+	GCC_DIAGNOSTIC_POP();
+}
+
+/** Process completed URBs in a worker thread */
+static int
+dbus_usbos_thread_func(void *data)
+{
+	usbos_info_t        *usbos_info = (usbos_info_t *)data;
+	usbos_list_entry_t  *entry;
+	struct list_head    *cur, *next;
+	unsigned long       flags;
+
+#ifdef WL_THREADNICE
+	set_user_nice(current, WL_THREADNICE);
+#endif
+
+	while (1) {
+		/* If the list is empty, then go to sleep */
+		wait_event_interruptible_timeout
+		(usbos_info->usbos_queue_head,
+			atomic_read(&usbos_info->usbos_list_cnt) > 0,
+			100);
+
+		if (kthread_should_stop())
+			break;
+
+		spin_lock_irqsave(&usbos_info->usbos_list_lock, flags);
+
+		/* For each entry on the list, process it.  Remove the entry from
+		* the list when done.
+		*/
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		list_for_each_safe(cur, next, &usbos_info->usbos_list)
+		{
+			urb_req_t           *req;
+			int                 len;
+			int                 stat;
+			usbos_info_t        *usbos_info_local;
+
+			entry = list_entry(cur, struct usbos_list_entry, list);
+			if (entry == NULL)
+				break;
+
+			req = entry->urb_context;
+			len = entry->urb_length;
+			stat = entry->urb_status;
+			usbos_info_local = req->usbinfo;
+
+			/* detach this entry from the list and attach it to the free list */
+			list_del_init(cur);
+			spin_unlock_irqrestore(&usbos_info_local->usbos_list_lock, flags);
+
+			dbus_usbos_recv_complete_handle(req, len, stat);
+
+			spin_lock_irqsave(&usbos_info_local->usbos_list_lock, flags);
+
+			list_add_tail(cur, &usbos_info_local->usbos_free_list);
+
+			atomic_dec(&usbos_info_local->usbos_list_cnt);
+		}
+
+		spin_unlock_irqrestore(&usbos_info->usbos_list_lock, flags);
+
+	}
+	GCC_DIAGNOSTIC_POP();
+
+	return 0;
+} /* dbus_usbos_thread_func */
+
+/** Called on Linux calling URB callback, see dbus_usbos_recv_complete() */
+static void
+dbus_usbos_dispatch_schedule(CALLBACK_ARGS)
+{
+	urb_req_t           *req = urb->context;
+	usbos_info_t        *usbos_info = req->usbinfo;
+	usbos_list_entry_t  *entry;
+	unsigned long       flags;
+	struct list_head    *cur;
+
+	spin_lock_irqsave(&usbos_info->usbos_list_lock, flags);
+
+	cur   = usbos_info->usbos_free_list.next;
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	entry = list_entry(cur, struct usbos_list_entry, list);
+	GCC_DIAGNOSTIC_POP();
+
+	/* detach this entry from the free list and prepare it insert it to use list */
+	list_del_init(cur);
+
+	if (entry) {
+		entry->urb_context = urb->context;
+		entry->urb_length  = urb->actual_length;
+		entry->urb_status  = urb->status;
+
+		atomic_inc(&usbos_info->usbos_list_cnt);
+		list_add_tail(cur, &usbos_info->usbos_list);
+	} else {
+		DBUSERR(("!!!!!!OUT OF MEMORY!!!!!!!\n"));
+	}
+
+	spin_unlock_irqrestore(&usbos_info->usbos_list_lock, flags);
+
+	/* thread */
+	wake_up_interruptible(&usbos_info->usbos_queue_head);
+} /* dbus_usbos_dispatch_schedule */
+
+#endif /* USBOS_THREAD */
+
+#ifdef USB_TRIGGER_DEBUG
+static bool
+dbus_usbos_ctl_send_debugtrig(usbos_info_t* usbinfo)
+{
+	bootrom_id_t id;
+
+	if (usbinfo == NULL)
+		return FALSE;
+
+	id.chip = 0xDEAD;
+
+	dbus_usbos_dl_cmd(usbinfo, DL_DBGTRIG, &id, sizeof(bootrom_id_t));
+
+	/* ignore the result for now */
+	return TRUE;
+}
+#endif /* USB_TRIGGER_DEBUG */
+
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX)
+
+/** New optimized code for USB AP. EHCI fastpath specific function. */
+void inline optimize_ehci_qtd_init(struct ehci_qtd *qtd, dma_addr_t dma)
+{
+	bzero(qtd, sizeof(*qtd));
+	wmb();
+	qtd->qtd_self = dma;
+	qtd->qtd_status = cpu_to_le32(EHCI_QTD_HALTED);
+	qtd->qtd_next = EHCI_NULL;
+	qtd->qtd_altnext = EHCI_NULL;
+	qtd->obj_next = NULL;
+	qtd->rpc = NULL;
+	/* qtd->buff = NULL; */
+	qtd->xacterrs = EHCI_QTD_XACTERR_MAX;
+	wmb();
+}
+
+/** EHCI fastpath specific function */
+struct ehci_qtd *optimize_ehci_qtd_alloc(gfp_t flags)
+{
+	struct ehci_qtd		*qtd;
+	dma_addr_t		dma;
+
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+
+	struct dma_pool *pool = usbos_info->qtd_pool;
+
+	qtd = dma_pool_alloc(pool, flags, &dma);
+	if (qtd != NULL) {
+		/* ME: Only clear the necessary fields:
+		 * - hw
+		 * - chain of QTDs in mapped space
+		 */
+		optimize_ehci_qtd_init(qtd, dma);
+	}
+
+	return qtd;
+}
+
+/** EHCI fastpath specific function */
+void optimize_ehci_qtd_free(struct ehci_qtd *qtd)
+{
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+	struct dma_pool *pool = usbos_info->qtd_pool;
+	dma_pool_free(pool, qtd, qtd->qtd_self);
+}
+
+/**
+ * EHCI fastpath specific function. Loosely follows qtd_copy_status
+ * Greatly simplified as there are only three options: normal, short read, and disaster
+ */
+static int
+get_qtd_status(struct ehci_qtd *qtd, int token, int *actual_length)
+{
+	int	status = -EINPROGRESS;
+
+	*actual_length += qtd->length - EHCI_QTD_GET_BYTES(token);
+
+	/* Short read is not an error */
+	if (unlikely (SHORT_READ_Q (token)))
+		status = -EREMOTEIO;
+
+	/* Check for serious problems */
+	if (token & EHCI_QTD_HALTED) {
+		status = -EPROTO;
+		if (token & (EHCI_QTD_BABBLE | EHCI_QTD_MISSEDMICRO | EHCI_QTD_BUFERR |
+			EHCI_QTD_XACTERR))
+			printk("EHCI Fastpath: Serious USB issue qtd %p token %08x --> status %d\n",
+				qtd, token, status);
+	}
+
+	return status;
+}
+
+/** This only works in the BCM embedded world (code uses primitive address mapping) */
+static void dump_qtd(struct ehci_qtd *qtd)
+{
+	printk("qtd_next %08x qtd_altnext %08x qtd_status %08x\n", qtd->qtd_next,
+		qtd->qtd_altnext, qtd->qtd_status);
+}
+
+static void dump_qh(struct ehci_qh *qh)
+{
+	struct ehci_qtd *qtd = (struct ehci_qtd *)(qh->qh_curqtd | 0xa0000000);
+	printk("EHCI Fastpath: QH %p Dump\n", qh);
+	printk("qtd_next %08x info1 %08x info2 %08x current %08x\n", qh->qh_link, qh->qh_endp,
+		qh->qh_endphub, qh->qh_curqtd);
+	printk("overlay\n");
+	dump_qtd((struct ehci_qtd *)&qh->ow_next);
+	while ((((int)qtd)&EHCI_NULL) == 0)
+	{
+		printk("QTD %p\n", qtd);
+		dump_qtd((struct ehci_qtd *)qtd);
+		qtd = (struct ehci_qtd *)(qtd->qtd_next | 0xa0000000);
+	}
+}
+
+/**
+ * EHCI fastpath specific function.
+ * This code assumes the caller holding a lock
+ * It is currently called from scan_async that should have the lock
+ * Lock shall be dropped around the actual completion, then reacquired
+ * This is a clean implementation of the qh_completions()
+ */
+static void
+ehci_bypass_callback(int pipeindex, struct ehci_qh *qh, spinlock_t *lock)
+{
+							/* Loop variables */
+	struct ehci_qtd		*qtd, 			/* current QTD */
+				*end = qh->dummy, 	/* "afterend" */
+				*next;
+	int			stopped;
+
+	/* ME: Obtain the same info in cleaner way */
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+
+	/* printk("EHCI Fastpath: callback pipe %d QH %p lock %p\n", pipeindex, qh, lock); */
+
+	/*
+	 * This code should not require any interlocking with QTD additions
+	 * The additions never touch QH, we should never touch 'end'
+	 * Note that QTD additions will keep 'end' in place
+	 */
+	for (qtd = qh->first_qtd; qtd != end; qtd = next)
+	{
+		u32		status;		/* Status bits from QTD */
+
+		/* Get the status bits from the QTD */
+		rmb();
+		status = hc32_to_cpu(qtd->qtd_status);
+
+		if ((status & EHCI_QTD_ACTIVE) == 0) {
+			if (unlikely((status & EHCI_QTD_HALTED) != 0)) {
+				/* Retry transaction errors until we
+				 * reach the software xacterr limit
+				 */
+				if ((status & EHCI_QTD_XACTERR) &&
+					EHCI_QTD_GET_CERR(status) == 0 &&
+					--qtd->xacterrs > 0) {
+					/* Reset the token in the qtd and the
+					 * qh overlay (which still contains
+					 * the qtd) so that we pick up from
+					 * where we left off
+					 */
+					printk("EHCI Fastpath: detected XactErr "
+						"qtd %p len %d/%d retry %d\n",
+						qtd, qtd->length - EHCI_QTD_GET_BYTES(status),
+						qtd->length,
+						EHCI_QTD_XACTERR_MAX - qtd->xacterrs);
+
+					status &= ~EHCI_QTD_HALTED;
+					status |= EHCI_QTD_ACTIVE | EHCI_QTD_SET_CERR(3);
+					qtd->qtd_status = cpu_to_le32(status);
+					wmb();
+					qh->ow_status = cpu_to_le32(status);
+
+					break;
+				}
+
+				/* QTD processing was aborted - highly unlikely (never seen, so not
+				 * tested). In very new 2.6, we can retry. In 2.4 and older 2.6,
+				 * life sucks (the USB stack does the same)
+				 */
+				printk("EHCI Fastpath: QTD halted\n");
+				dump_qh(qh);
+				stopped = 1;
+			}
+		} else {
+			/* Inactive QTD is an afterend, finished the list */
+			break;
+		}
+
+		/* Remove the QTD from software QH. This should be done before dropping the lock
+		 * in for upper layer
+		 */
+		next = qtd->obj_next;
+		qh->first_qtd = next;
+
+		/* Upper layer processing. */
+		if (EHCI_QTD_GET_PID(status) == 0)  /* OUT (host->dongle) pipe */
+		{
+			if (qtd->rpc == NULL)
+			{
+				/* Download, just signal completion
+				 * ME: Use non-global OS handle
+				 * Code from dbus_usbos_sync_complete
+				 */
+				usbos_info->waitdone = TRUE;
+				wake_up_interruptible(&usbos_info->wait);
+				/* ME: Set to meaningful status */
+				usbos_info->sync_urb_status = 0;
+			} else {
+				/* RPC packet handling
+				 * First, relevant processing from dbus_if_send_irb_complete:
+				 * ME: Check for bus state here
+				 */
+
+				/* Now propagate to the RPC layer from dbus_usbos_send_complete:
+				 *  usbos_info->cbs->send_irb_complete(usbos_info->cbarg,
+				 *  txirb, status);
+				 */
+
+				/* From dbus_usb_send_irb_complete: */
+
+				/* usb_info_t *usb_info = (usb_info_t *) handle; */
+				usb_info_t *usb_info = (usb_info_t *) usbos_info->cbarg;
+				/* if(usb_info && usb_info->cbs && usb_info->cbs->send_irb_complete)
+				 * usb_info->cbs->send_irb_complete(usb_info->cbarg, txirb, status);
+				 */
+
+				/* From dbus_if_send_irb_complete
+				 * 	dbus_info_t *dbus_info = (dbus_info_t *) handle;
+				 */
+				dbus_info_t *dbus_info = (dbus_info_t *)usb_info->cbarg;
+				/* printk("RPC send completed %p %p %p\n", usbos_info,
+				 * 	usb_info, dbus_info);
+				 * dbus_tx_timer_stop(dbus_info);
+				 * if (txirb_pending)
+				 * 	dbus_tx_timer_start(dbus_info, DBUS_TX_TIMEOUT_INTERVAL);
+				 * pktinfo = txirb->info; // This is the same as the pkt :-)
+				 * if (dbus_info->cbs && dbus_info->cbs->send_complete)
+				 * dbus_info->cbs->send_complete(dbus_info->cbarg, pktinfo,
+				 *	status);
+				 */
+
+				/* Free the coalesce buffer, if multi-buffer packet only. Do not
+				 * rely on buff, as it might not even exist
+				 */
+				if (PKTNEXT(usbos_info->pub->osh, qtd->rpc)) {
+					/* ME: dma_unmap_single */
+					/* printk("k-Freeing %p\n", qtd->buff); */
+					kfree(qtd->buff);
+				}
+
+				if (dbus_info->cbs && dbus_info->cbs->send_complete)
+				{
+					atomic_dec(&s_tx_pending);
+					spin_unlock(lock);
+					/* printk("Sending to RPC qtd %p\n", qtd); */
+#if !(defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_TXNOCOPY) || defined(BCM_RPC_TOC))
+	#error Configuration not supported; read dbus_if_send_irb_complete for guidelines
+#endif
+					/* ME: Set to meaningful status */
+					dbus_info->cbs->send_complete(dbus_info->cbarg, qtd->rpc,
+						0);
+					if ((atomic_read(&s_tx_pending) < 16) &&
+						/* ME: 16 or dbus_info->tx_low_watermark? */
+						dbus_info->txoff && !dbus_info->txoverride) {
+						dbus_flowctrl_tx(dbus_info, OFF);
+					}
+					spin_lock(lock);
+
+					/* Things could have happened while the lock was gone,
+					 * resync to the hardware
+					 */
+					next = qh->first_qtd;
+					end = qh->dummy;
+				}
+			}
+
+			optimize_ehci_qtd_free(qtd);
+		} else {   /* IN (dongle->host) pipe */
+			/* Simulates the upstream travel */
+			usb_info_t *usb_info = (usb_info_t *) usbos_info->cbarg;
+			dbus_info_t *dbus_info = (dbus_info_t *)usb_info->cbarg;
+			/* unsigned long       flags; */
+			int actual_length = 0;
+
+			/* All our reads must be short */
+			if (!SHORT_READ_Q (status)) ASSERT(0);
+
+			/* Done with hardware, convert status to error codes */
+			status = get_qtd_status(qtd, status, &actual_length);
+
+			switch (status) {
+			/* success */
+			case 0:
+			case -EINPROGRESS:
+			case -EREMOTEIO:
+				status = 0;
+				break;
+
+			case -ECONNRESET:		/* canceled */
+			case -ENOENT:
+			case -EPROTO:
+				DBUSERR(("%s: ehci unlink. status %x\n", __FUNCTION__, status));
+				break;
+			}
+
+			if (g_probe_info.dereged) {
+				printk("%s: DBUS deregistering, ignoring recv callback\n",
+					__FUNCTION__);
+				return;
+			}
+
+			dma_unmap_single(
+				usbos_info->usb->bus->controller,
+				(dma_addr_t)qtd->qtd_buffer_hi[0],
+				actual_length,
+				DMA_FROM_DEVICE);
+
+			/* From dbus_usb_recv_irb_complete
+			 *
+			 * if (usb_info->cbs && usb_info->cbs->recv_irb_complete)
+			 * 	usb_info->cbs->recv_irb_complete(usb_info->cbarg, rxirb, status);
+			 */
+
+			/* From dbus_if_recv_irb_complete */
+			if (dbus_info->pub.busstate != DBUS_STATE_DOWN) {
+				if (status == 0) {
+					void *buf = qtd->rpc;
+
+					ASSERT(buf != NULL);
+					spin_unlock(lock);
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+					/* Note that these ifdefs are indirectly coming from
+					 * dbus_usbos_recv_urb_submit The code itself is from
+					 * dbus_if_recv_irb_complete that makes the decision
+					 * at runtime, yet it is only pkt or buf depending on
+					 * the NOCOPY setup, never both :-)
+					 */
+					if (dbus_info->cbs && dbus_info->cbs->recv_pkt)
+						dbus_info->cbs->recv_pkt(dbus_info->cbarg, buf);
+#else
+					if (actual_length > 0) {
+						if (dbus_info->cbs && dbus_info->cbs->recv_buf)
+							dbus_info->cbs->recv_buf(dbus_info->cbarg,
+							buf, actual_length);
+					}
+#endif
+					spin_lock(lock);
+
+					/* Things could have happened while the lock was gone,
+					 * resync to the hardware
+					 */
+					next = qh->first_qtd;
+					end = qh->dummy;
+
+					/* Reinitialize this qtd since it will be reused. */
+					optimize_ehci_qtd_init(qtd, qtd->qtd_self);
+
+#if defined(BCM_RPC_NOCOPY) || defined(BCM_RPC_RXNOCOPY)
+					/* Note that these ifdefs are coming from
+					 * dbus_usbos_recv_urb_submit. In the NOCOPY configuration,
+					 * force an allocation of a new packet
+					 */
+					optimize_submit_rx_request(&dbus_info->pub, 1, qtd, NULL);
+
+#else
+					/* In the copy mode, simply reuse the buffer; upper level
+					 * had already consumed the data
+					 */
+					optimize_submit_rx_request(&dbus_info->pub, 1, qtd, buf);
+#endif
+					/* Not to free this qtd because it will be reused. */
+					continue;
+				}
+			} else {
+				printk("%s: DBUS down, ignoring recv callback\n", __FUNCTION__);
+			}
+			optimize_ehci_qtd_free(qtd);
+		}
+	}
+} /* ehci_bypass_callback */
+
+/** EHCI fastpath specific function */
+static void optimize_urb_callback(struct urb *urb)
+{
+	struct usb_ctrlrequest *req = urb->context;
+
+	kfree(req);
+	USB_FREE_URB(urb);
+}
+
+/**
+ * EHCI fastpath specific function. Shall be called under an external lock (currently RPC_TP_LOCK)
+ */
+static int optimize_submit_urb(struct usb_device *usb, void *ptr, int request)
+{
+	struct usb_ctrlrequest *req;
+	struct urb *urb;
+
+	if ((urb = USB_ALLOC_URB()) == NULL) {
+		printk("EHCI Fastpath: Error allocating URB in optimize_EP!");
+		return -ENOMEM;
+	}
+
+	if ((req = kmalloc(sizeof(struct usb_ctrlrequest), GFP_ATOMIC)) == NULL) {
+		printk("EHCI Fastpath: Failed to allocate memory for control request in"
+			" optimize_EP!");
+		USB_FREE_URB(urb);
+		return -ENOMEM;
+	}
+
+	req->bRequestType = (USB_TYPE_VENDOR | USB_RECIP_OTHER);
+	req->bRequest = request;
+
+	/* Use this instead of a buffer */
+	req->wValue = ((int)ptr & 0xffff);
+	req->wIndex = ((((int)ptr)>>16) & 0xffff);
+	req->wLength = 0;
+
+	printk("EHCI Fastpath: usb_dev %p\n", usb);
+	printk("EHCI Fastpath: bus %p\n", usb->bus);
+	printk("EHCI Fastpath: Hub %p\n", usb->bus->root_hub);
+
+	usb_fill_control_urb(
+		urb,
+		usb->bus->root_hub,
+		usb_sndctrlpipe(usb->bus->root_hub, 0),
+		(void *)req,
+		NULL,
+		0,
+		optimize_urb_callback,
+		req);
+
+	USB_SUBMIT_URB(urb);
+
+	if (urb->status != 0) {
+		printk("EHCI Fastpath: Cannot submit URB in optimize_EP: %d\n", urb->status);
+	}
+
+	return urb->status;
+} /* optimize_submit_urb */
+
+/** EHCI fastpath specific function */
+static int epnum(int pipe)
+{
+	int epn = usb_pipeendpoint(pipe);
+	if (usb_pipein (pipe))
+		epn |= 0x10;
+	return epn;
+}
+
+/** EHCI fastpath specific function */
+static int optimize_init(usbos_info_t *usbos_info, struct usb_device *usb, int out, int in, int in2)
+{
+	int retval = -EPIPE;
+
+	atomic_set(&s_tx_pending, 0);
+	/* atomic_set(&s_rx_pending, 0); */
+
+	usbos_info->tx_ep = epnum(out);
+	usbos_info->rx_ep = epnum(in);
+	usbos_info->rx2_ep = epnum(in2);
+	usbos_info->usb_device = usb;
+
+	/* printk("EHCI Fastpath: Create pool %p %p %p\n", usb, usb->bus, usb->bus->controller); */
+
+	/* QTDs for bulk transfers - separate pool */
+	/* ME: Destroy pool */
+	usbos_info->qtd_pool = dma_pool_create("usbnet_qtd",
+		usb->bus->controller,
+		sizeof(struct ehci_qtd),
+		EHCI_QTD_ALIGN /* byte alignment (for hw parts) */,
+		4096 /* can't cross 4K */);
+	if (!usbos_info->qtd_pool) {
+		printk("EHCI Fastpath: Cannot create the QTD pool\n");
+		goto fail;
+	}
+
+	/* detaching the EP */
+	if (optimize_submit_urb(usb, usb, EHCI_SET_BYPASS_DEV) != 0)
+		goto fail;
+	optimize_submit_urb(usb, ehci_bypass_callback, EHCI_SET_BYPASS_CB);
+	optimize_submit_urb(usb, usbos_info->qtd_pool, EHCI_SET_BYPASS_POOL);
+#ifdef EHCI_FASTPATH_TX
+	optimize_submit_urb(usb, (void*)((0<<16)|usbos_info->tx_ep), EHCI_FASTPATH);
+#endif
+#ifdef EHCI_FASTPATH_RX
+	optimize_submit_urb(usb, (void*)((1<<16)|usbos_info->rx_ep),   EHCI_FASTPATH);
+#endif
+	/* ME: Later optimize_submit_urb(usb, ((2<<16)|s_rx2_ep), EHCI_FASTPATH); */
+
+	/* getting the QH */
+	printk("EHCI Fastpath: EP in %d EP in2 %d EP out %d\n", usbos_info->rx_ep,
+		usbos_info->rx2_ep, usbos_info->tx_ep);
+
+	return 0;
+
+fail:
+	return retval;
+} /* optimize_submit_urb */
+
+/** EHCI fastpath specific function */
+static int optimize_deinit_qtds(struct ehci_qh *qh, int coalesce_buf)
+{
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+	struct ehci_qtd *qtd, *end, *next;
+	unsigned long	flags;
+
+	if (qh == NULL)
+		return 0;
+
+	end = qh->dummy;
+
+	printk("%s %d. qh = %p\n", __func__, __LINE__, qh);
+
+	spin_lock_irqsave(&usbos_info->fastpath_lock, flags);
+	for (qtd = qh->first_qtd; qtd != end; qtd = next) {
+		next = qtd->obj_next;
+		qh->first_qtd = next;
+
+		/* Free the coalesce buffer, if multi-buffer packet only. Do not
+		 * rely on buff, as it might not even exist
+		 */
+		if (coalesce_buf && PKTNEXT(usbos_info->pub->osh, qtd->rpc)) {
+			/* ME: dma_unmap_single */
+			printk("k-Freeing %p, ", qtd->buff);
+			kfree(qtd->buff);
+		}
+		printk("freeing qtd %p\n", qtd);
+
+		optimize_ehci_qtd_free(qtd);
+	}
+	spin_unlock_irqrestore(&usbos_info->fastpath_lock, flags);
+
+	return 0;
+}
+
+/** EHCI fastpath specific function. This code might deserve a separate file */
+static struct ehci_qh *
+get_ep(usbos_info_t *usbos_info, int ep)
+{
+#ifdef KERNEL26
+	struct usb_host_endpoint *epp = NULL;
+	switch (ep)
+	{
+	case 0: epp = usbos_info->usb_device->ep_out[usbos_info->tx_ep&0xf]; break;
+	case 1: epp = usbos_info->usb_device->ep_in[usbos_info->rx_ep&0xf]; break;
+	case 2: epp = usbos_info->usb_device->ep_in[usbos_info->rx2_ep&0xf]; break;
+	default: ASSERT(0);
+	}
+	if (epp != NULL)
+		return (struct ehci_qh *)epp->hcpriv;
+	else return NULL;
+#else
+	switch (ep)
+	{
+	case 0: return (struct ehci_qh *)(((struct hcd_dev*)(usbos_info->
+		usb_device->hcpriv))->ep[usbos_info->tx_ep]);
+	case 1: return (struct ehci_qh *)(((struct hcd_dev*)(usbos_info->
+		usb_device->hcpriv))->ep[usbos_info->rx_ep]);
+	case 2: return (struct ehci_qh *)(((struct hcd_dev*)(usbos_info->
+		usb_device->hcpriv))->ep[usbos_info->rx2_ep]);
+	default: ASSERT(0);
+	}
+	return NULL;
+#endif /* KERNEL26 */
+}
+
+/** EHCI fastpath specific function */
+int optimize_deinit(usbos_info_t *usbos_info, struct usb_device *usb)
+{
+	optimize_deinit_qtds(get_ep(usbos_info, 0), 1);
+	optimize_deinit_qtds(get_ep(usbos_info, 1), 0);
+#if defined(EHCI_FASTPATH_TX) || defined(EHCI_FASTPATH_RX)
+	optimize_submit_urb(usb, (void *)0, EHCI_CLR_EP_BYPASS);
+#endif
+	dma_pool_destroy(usbos_info->qtd_pool);
+	return 0;
+}
+
+/** EHCI fastpath specific function. Reassemble the segmented packet */
+static int
+optimize_gather(const dbus_pub_t *pub, void *pkt, void **buf)
+{
+	int len = 0;
+
+	void *transfer_buf = kmalloc(pkttotlen(pub->osh, pkt),
+		GFP_ATOMIC);
+	*buf = transfer_buf;
+
+	if (!transfer_buf) {
+		printk("fail to alloc to usb buffer\n");
+		return 0;
+	}
+
+	while (pkt) {
+		int pktlen = PKTLEN(pub->osh, pkt);
+		bcopy(PKTDATA(pub->osh, pkt), transfer_buf, pktlen);
+		transfer_buf += pktlen;
+		len += pktlen;
+		pkt = PKTNEXT(pub->osh, pkt);
+	}
+
+	/* printk("Coalesced a %d-byte buffer\n", len); */
+
+	return len;
+}
+
+/** EHCI fastpath specific function */
+int
+optimize_qtd_fill_with_rpc(const dbus_pub_t *pub, int epn,
+	struct ehci_qtd *qtd, void *rpc, int token, int len)
+{
+	void *data = NULL;
+
+	if (len == 0)
+		return optimize_qtd_fill_with_data(pub, epn, qtd, data, token, len);
+
+	ASSERT(rpc != NULL);
+	data = PKTDATA(pub->osh, rpc);
+	qtd->rpc = rpc;
+
+	if (PKTNEXT(pub->osh, rpc)) {
+		len = optimize_gather(pub, rpc, &data);
+		qtd->buff = data;
+	}
+
+	return optimize_qtd_fill_with_data(pub, epn, qtd, data, token, len);
+}
+
+/** EHCI fastpath specific function. Fill the QTD from the data buffer */
+int
+optimize_qtd_fill_with_data(const dbus_pub_t *pub, int epn,
+	struct ehci_qtd *qtd, void *data, int token, int len)
+{
+	int		i, bytes_fit, page_offset;
+	dma_addr_t	addr = 0;
+
+	/* struct usb_host_endpoint *ep = get_ep(epn); */
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+
+	token |= (EHCI_QTD_ACTIVE | EHCI_QTD_IOC); /* Allow execution, force interrupt */
+
+	if (len > 0) {
+		addr = dma_map_single(
+			usbos_info->usb->bus->controller,
+			data,
+			len,
+			EHCI_QTD_GET_PID(token) ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
+	}
+
+	qtd->qtd_buffer[0] = cpu_to_hc32((u32)addr);
+	/* Here qtd->qtd_buffer_hi[0] is leveraged to store addr value, which
+	 * is needed when invoking dma_unmap_single() in ehci_bypass_callback().
+	 * This is valid for EHCI 32bit only.
+	 */
+	qtd->qtd_buffer_hi[0] = cpu_to_hc32((u32)addr);
+	page_offset = (addr & (EHCI_PAGE_SIZE-1));
+	bytes_fit = EHCI_PAGE_SIZE - page_offset;
+	addr -= page_offset;
+	if (len < bytes_fit)
+		bytes_fit = len;
+	else {
+		addr +=  EHCI_PAGE_SIZE;
+
+		for (i = 1; bytes_fit < len && i < EHCI_QTD_NBUFFERS; i++) {
+			qtd->qtd_buffer[i] = cpu_to_hc32((u32)addr);
+			qtd->qtd_buffer_hi[i] = 0;
+			addr += EHCI_PAGE_SIZE;
+			if ((bytes_fit + EHCI_PAGE_SIZE) < len)
+				bytes_fit += EHCI_PAGE_SIZE;
+			else
+				bytes_fit = len;
+		}
+
+		if (bytes_fit != len)
+		{
+			/* ME: Handle long packets? Does not happen */
+			ASSERT(0);
+		}
+	}
+	qtd->qtd_status = cpu_to_hc32((bytes_fit << 16) | token);
+	qtd->length = bytes_fit;
+
+	return bytes_fit;
+} /* optimize_qtd_fill_with_data */
+
+/**
+ * EHCI fastpath specific function. Reimplementation of qh_append_tds(). Returns nonzero if too many
+ * requests pending.
+ */
+int
+optimize_submit_async(struct ehci_qtd *qtd, int epn)
+{
+	/* Clean implementation along the lines of qh_append_tds() */
+
+	struct ehci_qtd		*afterend; /* Element at the end of the QTD chain (after the
+					    * last useful one, "after-end")
+					    */
+	dma_addr_t		hw_addr;
+	__hc32			status;
+	unsigned long	flags;
+
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+	struct ehci_qh *qh = get_ep(usbos_info, epn);
+	usb_info_t *usb_info = (usb_info_t *) usbos_info->cbarg;
+	dbus_info_t *dbus_info = (dbus_info_t *)usb_info->cbarg;
+
+	/* printk("Submit qtd %p to pipe %d (%p)\n", qtd, epn, qh); */
+	if (qh == NULL)
+	{
+		printk("EHCI Fastpath: Attempt of optimized submit to a non-optimized pipe\n");
+		return -1;
+	}
+
+	spin_lock_irqsave(&usbos_info->fastpath_lock, flags);
+
+	/* Limit outstanding - for rpc behavior only */
+	/* printk("QH qtd_status %08x\n", qh->hw->qtd_status); */
+
+	if ((qtd->qtd_status & (1<<8)) == 0)
+	{
+		atomic_inc(&s_tx_pending);
+		if (atomic_read(&s_tx_pending) > 16*2) /* (dbus_info->tx_low_watermark * 3)) */
+			dbus_flowctrl_tx(dbus_info, TRUE);
+	}
+
+	ASSERT(qh != NULL);
+
+	/* ME: Check for hardware availability here */
+
+	/*
+	 * Standard list processing trick:
+	 *   * old "afterend" is filled with the incoming data while still HALTed
+	 *   * new element is appended and prepared to serve as new afterend
+	 *   * now old afterend is activated
+	 * This way, HW never races the SW - no semaphores are necessary, as long as this function
+	 * is not reentered for the same QH
+	 */
+
+	/* Make new QTD to be HALTed, wait for it to actually happen */
+	status = qtd->qtd_status;
+	qtd->qtd_status = cpu_to_le32(EHCI_QTD_HALTED);
+	wmb();
+
+	/* Now copy all information from the new QTD to the old afterend,
+	 * except the own HW address
+	 */
+	afterend = qh->dummy;
+	hw_addr = afterend->qtd_self;
+	*afterend = *qtd;
+	afterend->qtd_self = hw_addr;
+
+	/* The new QTD is ready to serve as a new afterend, append it */
+	qh->dummy = qtd;
+	afterend->qtd_next = qtd->qtd_self;
+	afterend->qtd_altnext = qtd->qtd_self;  /* Always assume short read. Harmless in our case */
+	afterend->obj_next = qtd;
+
+	/* Wait for writes to happen and enable the old afterend (now containing the QTD data) */
+	wmb();
+	afterend->qtd_status = status;
+	wmb();
+
+	spin_unlock_irqrestore(&usbos_info->fastpath_lock, flags);
+
+	return 0;
+} /* optimize_submit_async */
+
+/** EHCI fastpath specific function. ME: Error return and handling */
+void
+optimize_submit_rx_request(const dbus_pub_t *pub, int epn, struct ehci_qtd *qtd_in,
+                                            void *buf)
+{
+	/* ME: Replace with getting usbos_info from pub */
+	usbos_info_t *usbos_info = g_probe_info.usbos_info;
+	int len = usbos_info->rxbuf_len;
+	void *pkt;
+	struct ehci_qtd *qtd;
+	int token = EHCI_QTD_SET_CERR(3) | EHCI_QTD_SET_PID(1);
+
+	if (qtd_in == NULL) {
+		qtd = optimize_ehci_qtd_alloc(GFP_KERNEL);
+		if (!qtd) {
+			printk("EHCI Fastpath: Out of QTDs\n");
+			return;
+		}
+	} else {
+		qtd = qtd_in;
+	}
+
+	if (buf == NULL)
+	{
+		/* NOCOPY, allocate own packet */
+		/* Follow dbus_usbos_recv_urb_submit */
+		pkt = PKTGET(usbos_info->pub->osh, len, FALSE);
+		if (pkt == NULL) {
+			printk("%s: PKTGET failed\n", __FUNCTION__);
+			optimize_ehci_qtd_free(qtd);
+			return;
+		}
+		/* consider the packet "native" so we don't count it as MALLOCED in the osl */
+		PKTTONATIVE(usbos_info->pub->osh, pkt);
+		qtd->rpc = pkt;
+		buf = PKTDATA(usbos_info->pub->osh, pkt);
+
+	} else {
+		qtd->rpc = buf;
+	}
+
+	optimize_qtd_fill_with_data(pub, epn, qtd, buf, token, len);
+	optimize_submit_async(qtd, epn);
+} /* optimize_submit_rx_request */
+
+#endif /* EHCI_FASTPATH_TX || EHCI_FASTPATH_RX */
+
+#ifdef BCM_REQUEST_FW
+
+struct request_fw_context {
+	const struct firmware *firmware;
+	struct semaphore lock;
+};
+
+/*
+ * Callback for dbus_request_firmware().
+ */
+static void
+dbus_request_firmware_done(const struct firmware *firmware, void *ctx)
+{
+	struct request_fw_context *context = (struct request_fw_context*)ctx;
+
+	/* Store the received firmware handle in the context and wake requester */
+	context->firmware = firmware;
+	up(&context->lock);
+}
+
+/*
+ * Send a firmware request and wait for completion.
+ *
+ * The use of the asynchronous version of request_firmware() is needed to avoid
+ * kernel oopses when we just come out of system hibernate.
+ */
+static int
+dbus_request_firmware(const char *name, const struct firmware **firmware)
+{
+	struct request_fw_context *context;
+	int ret;
+
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return -ENOMEM;
+
+	sema_init(&context->lock, 0);
+
+	ret = request_firmware_nowait(THIS_MODULE, true, name, &g_probe_info.usb->dev,
+	                              GFP_KERNEL, context, dbus_request_firmware_done);
+	if (ret) {
+		kfree(context);
+		return ret;
+	}
+
+	/* Wait for completion */
+	if (down_interruptible(&context->lock) != 0) {
+		kfree(context);
+		return -ERESTARTSYS;
+	}
+
+	*firmware = context->firmware;
+	kfree(context);
+
+	return *firmware != NULL ? 0 : -ENOENT;
+}
+
+#ifndef DHD_LINUX_STD_FW_API
+static void *
+dbus_get_fwfile(int devid, int chiprev, uint8 **fw, int *fwlen,
+	uint16 boardtype, uint16 boardrev, char *path)
+{
+	const struct firmware *firmware = NULL;
+	s8 *device_id = NULL;
+	s8 *chip_rev = "";
+	s8 file_name[64] = {0, };
+	int ret;
+
+	switch (devid) {
+		case BCM4381_CHIP_ID:
+			device_id = "4381";
+			switch (chiprev) {
+				case 1:
+					chip_rev = "a0";
+					break;
+				default:
+					break;
+			}
+			break;
+		case BCM4382_CHIP_ID:
+			device_id = "4382";
+			switch (chiprev) {
+				case 1:
+					chip_rev = "a0";
+					break;
+				default:
+					break;
+			}
+			break;
+
+		default:
+			DBUSERR(("unsupported device %x\n", devid));
+			return NULL;
+	}
+
+	/* Load firmware */
+	snprintf(file_name, sizeof(file_name), "brcm/bcm%s%s-usb.bin", device_id, chip_rev);
+
+	ret = dbus_request_firmware(path, &firmware);
+	if (ret) {
+		DBUSERR(("fail to request firmware %s\n", path));
+		return NULL;
+	} else
+		DBUSERR(("%s: %s (%zu bytes) open success\n", __FUNCTION__, path, firmware->size));
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	*fwlen = firmware->size;
+	*fw = (uint8 *)firmware->data;
+	return (void *)firmware;
+	GCC_DIAGNOSTIC_POP();
+}
+
+static void *
+dbus_get_nvfile(int devid, int chiprev, uint8 **fw, int *fwlen,
+	uint16 boardtype, uint16 boardrev, char *path)
+{
+	const struct firmware *firmware = NULL;
+	s8 *device_id = NULL;
+	s8 *chip_rev = "";
+	s8 file_name[64];
+	int ret;
+
+	switch (devid) {
+		case BCM4381_CHIP_ID:
+			device_id = "4381";
+			switch (chiprev) {
+				case 1:
+					chip_rev = "a0";
+					break;
+				default:
+					break;
+			}
+			break;
+		case BCM4382_CHIP_ID:
+			device_id = "4382";
+			switch (chiprev) {
+				case 1:
+					chip_rev = "a0";
+					break;
+				default:
+					break;
+			}
+			break;
+
+		default:
+			DBUSERR(("unsupported device %x\n", devid));
+			return NULL;
+	}
+
+	/* Load board specific nvram file */
+	snprintf(file_name, sizeof(file_name), "brcm/bcm%s%s-usb.nvm",
+	         device_id, chip_rev);
+
+	ret = dbus_request_firmware(path, &firmware);
+	if (ret) {
+		DBUSERR(("fail to request nvram %s\n", path));
+
+		/* Load generic nvram file */
+		snprintf(file_name, sizeof(file_name), "brcm/bcm%s%s.nvm",
+		         device_id, chip_rev);
+
+		ret = dbus_request_firmware(file_name, &firmware);
+		if (ret) {
+			DBUSERR(("fail to request nvram %s\n", path));
+			return NULL;
+		}
+	} else
+		DBUSERR(("%s: %s (%zu bytes) open success\n", __FUNCTION__, path, firmware->size));
+
+	*fwlen = firmware->size;
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	*fw = (uint8 *)firmware->data;
+	return (void *)firmware;
+	GCC_DIAGNOSTIC_POP();
+} /* dbus_get_fw_nvfile */
+
+void *
+dbus_get_fw_nvfile(int devid, int chiprev, uint8 **fw, int *fwlen, int type, uint16 boardtype,
+	uint16 boardrev, char *path)
+{
+	switch (type) {
+		case DBUS_FIRMWARE:
+			return dbus_get_fwfile(devid, chiprev, fw, fwlen, boardtype, boardrev, path);
+		case DBUS_NVFILE:
+			return dbus_get_nvfile(devid, chiprev, fw, fwlen, boardtype, boardrev, path);
+		default:
+			return NULL;
+	}
+}
+#else
+void *
+dbus_get_fw_nvfile(int devid, int chiprev, uint8 **fw, int *fwlen, int type, uint16 boardtype,
+	uint16 boardrev, char *path)
+{
+	const struct firmware *firmware = NULL;
+	int err = DBUS_OK;
+
+	err = dbus_request_firmware(path, &firmware);
+	if (err) {
+		DBUSERR(("fail to request firmware %s\n", path));
+		return NULL;
+	} else {
+		DBUSERR(("%s: %s (%zu bytes) open success\n",
+			__FUNCTION__, path, firmware->size));
+	}
+
+	*fwlen = firmware->size;
+	*fw = (uint8 *)firmware->data;
+	return (void *)firmware;
+}
+#endif
+
+void
+dbus_release_fw_nvfile(void *firmware)
+{
+	release_firmware((struct firmware *)firmware);
+}
+#endif /* BCM_REQUEST_FW */
+
+#ifdef BCMUSBDEV_COMPOSITE
+/**
+ * For a composite device the interface order is not guaranteed, scan the device struct for the WLAN
+ * interface.
+ */
+static int
+dbus_usbos_intf_wlan(struct usb_device *usb)
+{
+	int i, num_of_eps, ep, intf_wlan = -1;
+	int num_intf = CONFIGDESC(usb)->bNumInterfaces;
+	struct usb_endpoint_descriptor *endpoint;
+
+	for (i = 0; i < num_intf; i++) {
+		if (IFDESC(usb, i).bInterfaceClass != USB_CLASS_VENDOR_SPEC)
+			continue;
+		num_of_eps = IFDESC(usb, i).bNumEndpoints;
+
+		for (ep = 0; ep < num_of_eps; ep++) {
+			endpoint = &IFEPDESC(usb, i, ep);
+			if ((endpoint->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
+				USB_ENDPOINT_XFER_BULK) {
+				intf_wlan = i;
+				break;
+			}
+		}
+		if (ep < num_of_eps)
+			break;
+	}
+
+	return intf_wlan;
+}
+#endif /* BCMUSBDEV_COMPOSITE */
+
+#ifdef KEEPIF_ON_DEVICE_RESET
+/* if keepif_on_devreset is set, and reset_resume or disconnect got called, we need de-register or
+ * register the usb driver to OS again; but this operation can't be called inside the reset_resume
+ * or disconnect callback, so introduce a kernel thread to do these work
+ */
+static int
+dbus_usbos_usbreset_func(void *data)
+{
+	enum usb_dev_status flag = USB_DEVICE_INIT;
+	usbos_info_t *usbos_info;
+
+	while (1) {
+		wait_event_interruptible_timeout(g_probe_info.usbreset_queue_head,
+			atomic_read(&g_probe_info.usbdev_stat), 100); /* 100 ms */
+
+		if (kthread_should_stop())
+			break;
+		if (!(usbos_info = g_probe_info.usbos_info))
+			continue;
+		if (!(flag = atomic_read(&g_probe_info.usbdev_stat)))
+			continue;
+
+		usbos_info = g_probe_info.usbos_info;
+		flag = atomic_read(&g_probe_info.usbdev_stat);
+		atomic_set(&g_probe_info.usbdev_stat, USB_DEVICE_INIT);
+
+		if ((flag == USB_DEVICE_RESETTED) || (flag == USB_DEVICE_DISCONNECTED)) {
+			DBUSERR(("Device reset, re-register USB driver\n"));
+			g_probe_info.dev_resetted = TRUE;
+			g_probe_info.busstate_bf_devreset = usbos_info->pub->busstate;
+			dbus_usbos_state_change(usbos_info, DBUS_STATE_DOWN);
+
+			g_probe_info.dereged = TRUE;
+			USB_DEREGISTER();
+			USB_REGISTER();
+			g_probe_info.dereged = FALSE;
+		} else if (flag == USB_DEVICE_PROBED) {
+			DBUSERR(("Device probed, but no need to initialize higher layer\n"));
+			g_probe_info.dev_resetted = FALSE;
+			dbus_usbos_init_info();
+			/* The device may have lost power, so a firmware download may be required */
+			dbus_usbos_state_change(usbos_info,
+				DBUS_STATE_DL_NEEDED);
+			g_probe_info.suspend_state = USBOS_SUSPEND_STATE_DEVICE_ACTIVE;
+			dbus_usbos_state_change(usbos_info,
+				g_probe_info.busstate_bf_devreset);
+		}
+	}
+	return 0;
+}
+#endif /* KEEPIF_ON_DEVICE_RESET */
+
+#ifdef LINUX
+struct device * dbus_get_dev(void)
+{
+	return &g_probe_info.usb->dev;
+}
+#endif /* LINUX */
diff --git a/drivers/net/wireless/ap12275/dhd.h b/drivers/net/wireless/ap12275/dhd.h
new file mode 100644
index 000000000000..84dc41ac1357
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd.h
@@ -0,0 +1,4798 @@
+/*
+ * Header file describing the internal (inter-module) DHD interfaces.
+ *
+ * Provides type definitions and function prototypes used to link the
+ * DHD OS, bus, and protocol modules.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/****************
+ * Common types *
+ */
+
+#ifndef _dhd_h_
+#define _dhd_h_
+
+#if defined(LINUX)
+#include <linux/firmware.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/random.h>
+#include <linux/spinlock.h>
+#include <linux/ethtool.h>
+#include <linux/proc_fs.h>
+#include <asm/uaccess.h>
+#include <asm/unaligned.h>
+#include <linux/fs.h>
+#include <linux/namei.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#include <linux/sched/types.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */
+#ifdef DHD_BUZZZ_LOG_ENABLED
+#include <dhd_buzzz.h>
+#endif /* DHD_BUZZZ_LOG_ENABLED */
+/* The kernel threading is sdio-specific */
+struct task_struct;
+struct sched_param;
+#if defined(BT_OVER_SDIO)
+#include <dhd_bt_interface.h>
+#endif /* defined (BT_OVER_SDIO) */
+int setScheduler(struct task_struct *p, int policy, struct sched_param *param);
+int get_scheduler_policy(struct task_struct *p);
+#else /* LINUX */
+#define ENOMEM		1
+#define EFAULT      2
+#define EINVAL		3
+#define EIO			4
+#define ETIMEDOUT	5
+#define ENODATA 6
+#define EREMOTEIO   7
+#define ENODEV      8
+#define ENOCSI		50	/* No CSI structure available */
+#define ERESTARTSYS 512
+#endif /* LINUX */
+#define MAX_EVENT	16
+
+#define ALL_INTERFACES	0xff
+
+/* H2D and D2H ring dump is enabled by default */
+#ifdef PCIE_FULL_DONGLE
+#define DHD_DUMP_PCIE_RINGS
+#endif /* PCIE_FULL_DONGLE */
+
+#include <osl.h>
+
+#include <wlioctl.h>
+#include <dhdioctl.h>
+#include <wlfc_proto.h>
+#include <hnd_armtrap.h>
+#if defined(DUMP_IOCTL_IOV_LIST) || defined(DHD_DEBUG)
+#include <bcmutils.h>
+#endif /* DUMP_IOCTL_IOV_LIST || DHD_DEBUG */
+
+#if defined(BCMWDF)
+#include <wdf.h>
+#include <WdfMiniport.h>
+#endif /* (BCMWDF)  */
+
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+#include <dnglioctl.h>
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+#include <dngl_stats.h>
+#include <hnd_pktq.h>
+
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+#define MAX_RESCHED_CNT 600
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+
+#if defined(LINUX) || defined(linux)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) && LINUX_VERSION_CODE < \
+	KERNEL_VERSION(3, 18, 0) || defined(CONFIG_BCMDHD_VENDOR_EXT))
+#ifndef WL_VENDOR_EXT_SUPPORT
+#define WL_VENDOR_EXT_SUPPORT
+#endif /* WL_VENDOR_EXT_SUPPORT */
+#endif /* 3.18 > KERNEL_VER >= 3.14 || defined(CONFIG_BCMDHD_VENDOR_EXT) */
+#endif /* defined (LINUX) || defined(linux) */
+
+#if defined(KEEP_ALIVE)
+/* Default KEEP_ALIVE Period is 55 sec to prevent AP from sending Keep Alive probe frame */
+#define KEEP_ALIVE_PERIOD 55000
+#define NULL_PKT_STR	"null_pkt"
+#endif /* KEEP_ALIVE */
+
+/* By default enabled from here, later the WQ code will be removed */
+#define DHD_USE_KTHREAD_FOR_LOGTRACE
+
+/* Forward decls */
+struct dhd_bus;
+struct dhd_prot;
+struct dhd_info;
+struct dhd_ioctl;
+struct dhd_dbg;
+struct dhd_ts;
+#ifdef DNGL_AXI_ERROR_LOGGING
+struct dhd_axi_error_dump;
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+/* The level of bus communication with the dongle */
+enum dhd_bus_state {
+	DHD_BUS_DOWN,		/* Not ready for frame transfers */
+	DHD_BUS_LOAD,		/* Download access only (CPU reset) */
+	DHD_BUS_DATA,		/* Ready for frame transfers */
+	DHD_BUS_SUSPEND,	/* Bus has been suspended */
+	DHD_BUS_DOWN_IN_PROGRESS,	/* Bus going Down */
+	DHD_BUS_REMOVE,	/* Bus has been removed */
+};
+
+/* The level of bus communication with the dongle */
+enum dhd_bus_devreset_type {
+	DHD_BUS_DEVRESET_ON = 0,	/* ON */
+	DHD_BUS_DEVRESET_OFF = 1,		/* OFF */
+	DHD_BUS_DEVRESET_FLR = 2,		/* FLR */
+	DHD_BUS_DEVRESET_FLR_FORCE_FAIL = 3,	/* FLR FORCE FAIL */
+	DHD_BUS_DEVRESET_QUIESCE = 4,		/* FLR */
+};
+
+/*
+ * Bit fields to Indicate clean up process that wait till they are finished.
+ * Future synchronizable processes can add their bit filed below and update
+ * their functionalities accordingly
+ */
+#define DHD_BUS_BUSY_IN_TX                   0x01
+#define DHD_BUS_BUSY_IN_SEND_PKT             0x02
+#define DHD_BUS_BUSY_IN_DPC                  0x04
+#define DHD_BUS_BUSY_IN_WD                   0x08
+#define DHD_BUS_BUSY_IN_IOVAR                0x10
+#define DHD_BUS_BUSY_IN_DHD_IOVAR            0x20
+#define DHD_BUS_BUSY_SUSPEND_IN_PROGRESS     0x40
+#define DHD_BUS_BUSY_RESUME_IN_PROGRESS      0x80
+#define DHD_BUS_BUSY_RPM_SUSPEND_IN_PROGRESS 0x100
+#define DHD_BUS_BUSY_RPM_SUSPEND_DONE        0x200
+#define DHD_BUS_BUSY_RPM_RESUME_IN_PROGRESS  0x400
+#define DHD_BUS_BUSY_RPM_ALL                 (DHD_BUS_BUSY_RPM_SUSPEND_DONE | \
+		DHD_BUS_BUSY_RPM_SUSPEND_IN_PROGRESS | \
+		DHD_BUS_BUSY_RPM_RESUME_IN_PROGRESS)
+#define DHD_BUS_BUSY_IN_CHECKDIED		0x800
+#define DHD_BUS_BUSY_IN_MEMDUMP			0x1000
+#define DHD_BUS_BUSY_IN_SSSRDUMP		0x2000
+#define DHD_BUS_BUSY_IN_LOGDUMP			0x4000
+#define DHD_BUS_BUSY_IN_HALDUMP			0x8000
+#define DHD_BUS_BUSY_IN_NAPI			0x10000
+#define DHD_BUS_BUSY_IN_DS_DEASSERT		0x20000
+#define DHD_BUS_BUSY_IN_DUMP_DONGLE_MEM		0x40000
+#define DHD_BUS_BUSY_IN_SYSFS_DUMP		0x80000
+#define DHD_BUS_BUSY_IN_PM_CALLBACK		0x100000
+
+#define DHD_BUS_BUSY_SET_IN_TX(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_TX
+#define DHD_BUS_BUSY_SET_IN_SEND_PKT(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_SEND_PKT
+#define DHD_BUS_BUSY_SET_IN_DPC(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_DPC
+#define DHD_BUS_BUSY_SET_IN_WD(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_WD
+#define DHD_BUS_BUSY_SET_IN_IOVAR(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_IOVAR
+#define DHD_BUS_BUSY_SET_IN_DHD_IOVAR(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_DHD_IOVAR
+#define DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_SUSPEND_IN_PROGRESS
+#define DHD_BUS_BUSY_SET_RESUME_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_RESUME_IN_PROGRESS
+#define DHD_BUS_BUSY_SET_RPM_SUSPEND_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_RPM_SUSPEND_IN_PROGRESS
+#define DHD_BUS_BUSY_SET_RPM_SUSPEND_DONE(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_RPM_SUSPEND_DONE
+#define DHD_BUS_BUSY_SET_RPM_RESUME_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_RPM_RESUME_IN_PROGRESS
+#define DHD_BUS_BUSY_SET_IN_CHECKDIED(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_CHECKDIED
+#define DHD_BUS_BUSY_SET_IN_MEMDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_MEMDUMP
+#define DHD_BUS_BUSY_SET_IN_SSSRDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_SSSRDUMP
+#define DHD_BUS_BUSY_SET_IN_LOGDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_LOGDUMP
+#define DHD_BUS_BUSY_SET_IN_HALDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_HALDUMP
+#define DHD_BUS_BUSY_SET_IN_NAPI(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_NAPI
+#define DHD_BUS_BUSY_SET_IN_DS_DEASSERT(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_DS_DEASSERT
+#define DHD_BUS_BUSY_SET_IN_DUMP_DONGLE_MEM(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_DUMP_DONGLE_MEM
+#define DHD_BUS_BUSY_SET_IN_SYSFS_DUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_SYSFS_DUMP
+#define DHD_BUS_BUSY_SET_IN_PM_CALLBACK(dhdp) \
+	(dhdp)->dhd_bus_busy_state |= DHD_BUS_BUSY_IN_PM_CALLBACK
+
+#define DHD_BUS_BUSY_CLEAR_IN_TX(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_TX
+#define DHD_BUS_BUSY_CLEAR_IN_SEND_PKT(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_SEND_PKT
+#define DHD_BUS_BUSY_CLEAR_IN_DPC(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_DPC
+#define DHD_BUS_BUSY_CLEAR_IN_WD(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_WD
+#define DHD_BUS_BUSY_CLEAR_IN_IOVAR(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_IOVAR
+#define DHD_BUS_BUSY_CLEAR_IN_DHD_IOVAR(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_DHD_IOVAR
+#define DHD_BUS_BUSY_CLEAR_SUSPEND_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_SUSPEND_IN_PROGRESS
+#define DHD_BUS_BUSY_CLEAR_RESUME_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_RESUME_IN_PROGRESS
+#define DHD_BUS_BUSY_CLEAR_RPM_SUSPEND_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_RPM_SUSPEND_IN_PROGRESS
+#define DHD_BUS_BUSY_CLEAR_RPM_SUSPEND_DONE(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_RPM_SUSPEND_DONE
+#define DHD_BUS_BUSY_CLEAR_RPM_RESUME_IN_PROGRESS(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_RPM_RESUME_IN_PROGRESS
+#define DHD_BUS_BUSY_CLEAR_IN_CHECKDIED(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_CHECKDIED
+#define DHD_BUS_BUSY_CLEAR_IN_MEMDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_MEMDUMP
+#define DHD_BUS_BUSY_CLEAR_IN_SSSRDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_SSSRDUMP
+#define DHD_BUS_BUSY_CLEAR_IN_LOGDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_LOGDUMP
+#define DHD_BUS_BUSY_CLEAR_IN_HALDUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_HALDUMP
+#define DHD_BUS_BUSY_CLEAR_IN_NAPI(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_NAPI
+#define DHD_BUS_BUSY_CLEAR_IN_DS_DEASSERT(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_DS_DEASSERT
+#define DHD_BUS_BUSY_CLEAR_IN_DUMP_DONGLE_MEM(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_DUMP_DONGLE_MEM
+#define DHD_BUS_BUSY_CLEAR_IN_SYSFS_DUMP(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_SYSFS_DUMP
+#define DHD_BUS_BUSY_CLEAR_IN_PM_CALLBACK(dhdp) \
+	(dhdp)->dhd_bus_busy_state &= ~DHD_BUS_BUSY_IN_PM_CALLBACK
+
+#define DHD_BUS_BUSY_CHECK_IN_TX(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_TX)
+#define DHD_BUS_BUSY_CHECK_IN_SEND_PKT(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_SEND_PKT)
+#define DHD_BUS_BUSY_CHECK_IN_DPC(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_DPC)
+#define DHD_BUS_BUSY_CHECK_IN_WD(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_WD)
+#define DHD_BUS_BUSY_CHECK_IN_IOVAR(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_IOVAR)
+#define DHD_BUS_BUSY_CHECK_IN_DHD_IOVAR(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_DHD_IOVAR)
+#define DHD_BUS_BUSY_CHECK_SUSPEND_IN_PROGRESS(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_SUSPEND_IN_PROGRESS)
+#define DHD_BUS_BUSY_CHECK_RESUME_IN_PROGRESS(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_RESUME_IN_PROGRESS)
+#define DHD_BUS_BUSY_CHECK_RPM_SUSPEND_IN_PROGRESS(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_RPM_SUSPEND_IN_PROGRESS)
+#define DHD_BUS_BUSY_CHECK_RPM_SUSPEND_DONE(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_RPM_SUSPEND_DONE)
+#define DHD_BUS_BUSY_CHECK_RPM_RESUME_IN_PROGRESS(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_RPM_RESUME_IN_PROGRESS)
+#define DHD_BUS_BUSY_CHECK_RPM_ALL(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_RPM_ALL)
+#define DHD_BUS_BUSY_CHECK_IN_CHECKDIED(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_CHECKDIED)
+#define DHD_BUS_BUSY_CHECK_IN_MEMDUMP(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_MEMDUMP)
+#define DHD_BUS_BUSY_CHECK_IN_SSSRDUMP(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_SSSRDUMP)
+#define DHD_BUS_BUSY_CHECK_IN_LOGDUMP(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_LOGDUMP)
+#define DHD_BUS_BUSY_CHECK_IN_HALDUMP(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_HALDUMP)
+#define DHD_BUS_BUSY_CHECK_IN_DS_DEASSERT(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_DS_DEASSERT)
+#define DHD_BUS_BUSY_CHECK_IN_DUMP_DONGLE_MEM(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_DUMP_DONGLE_MEM)
+#define DHD_BUS_BUSY_CHECK_IN_SYSFS_DUMP(dhdp) \
+	((dhdp)->dhd_bus_busy_state & DHD_BUS_BUSY_IN_SYSFS_DUMP)
+#define DHD_BUS_BUSY_CHECK_IDLE(dhdp) \
+	((dhdp)->dhd_bus_busy_state == 0)
+
+#define DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(dhdp) \
+	((dhdp)->busstate == DHD_BUS_SUSPEND || DHD_BUS_BUSY_CHECK_SUSPEND_IN_PROGRESS(dhdp))
+
+#define DHD_BUS_CHECK_ANY_SUSPEND_IN_PROGRESS(dhdp) \
+		(DHD_BUS_BUSY_CHECK_SUSPEND_IN_PROGRESS(dhdp) || \
+		 DHD_BUS_BUSY_CHECK_RPM_SUSPEND_IN_PROGRESS(dhdp))
+
+#define DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhdp) \
+	((dhdp)->busstate == DHD_BUS_SUSPEND || DHD_BUS_CHECK_ANY_SUSPEND_IN_PROGRESS(dhdp))
+
+#define DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp) \
+		((dhdp)->busstate == DHD_BUS_DOWN || (dhdp)->busstate == DHD_BUS_DOWN_IN_PROGRESS || \
+		(dhdp)->busstate == DHD_BUS_REMOVE)
+
+#define DHD_BUS_CHECK_REMOVE(dhdp) \
+		((dhdp)->busstate == DHD_BUS_REMOVE)
+
+/* IOVar flags for common error checks */
+#define DHD_IOVF_PWRREQ_BYPASS	(1<<0) /* flags to prevent bp access during host sleep state */
+
+#define MAX_MTU_SZ (1600u)
+
+#ifdef PCIE_INB_DW
+#define DHD_CHECK_CFG_IN_PROGRESS(dhdp) \
+	((INBAND_DW_ENAB((dhdp)->bus)) ? dhd_check_cfg_in_progress(dhdp) : FALSE)
+#else
+#define DHD_CHECK_CFG_IN_PROGRESS(dhdp) FALSE
+#endif /* PCIE_INB_DW */
+
+#ifndef USEC_PER_SEC
+#define USEC_PER_SEC (1000 * 1000)
+#endif
+#ifndef MSEC_PER_SEC
+#define MSEC_PER_SEC 1000u
+#endif
+
+#define TIMESPEC64_TO_US(ts) \
+	(((ts).tv_sec * USEC_PER_SEC) + ((ts).tv_nsec / NSEC_PER_USEC))
+
+#if (defined(LINUX) || defined(linux))
+/* (u64)result = (u64)dividend / (u64)divisor */
+#define DIV_U64_BY_U64(dividend, divisor)	div64_u64(dividend, divisor)
+
+/* (u64)result = (u64)dividend / (u32)divisor */
+#define DIV_U64_BY_U32(dividend, divisor)	div_u64(dividend, divisor)
+
+/* Be careful while using this, as it divides dividend also
+ * (u32)remainder = (u64)dividend % (u32)divisor
+ * (u64)dividend = (u64)dividend / (u32)divisor
+ */
+#define DIV_AND_MOD_U64_BY_U32(dividend, divisor)	do_div(dividend, divisor)
+
+/* (u32)remainder = (u64)dividend % (u32)divisor */
+#define MOD_U64_BY_U32(dividend, divisor) ({				\
+	uint64 temp_dividend = (dividend);				\
+	uint32 rem = DIV_AND_MOD_U64_BY_U32(temp_dividend, (divisor));	\
+	rem;								\
+})
+
+#define SEC_USEC_FMT \
+	"%5llu.%06u"
+#else
+/* (u64)result = (u64)dividend / (u64)divisor */
+#define DIV_U64_BY_U64(dividend, divisor)	(uint64)(dividend) / (uint64)(divisor)
+
+/* (u64)result = (u64)dividend / (u32)divisor */
+#define DIV_U64_BY_U32(dividend, divisor)	(uint64)(dividend) / (uint32)(divisor)
+
+/* Be careful while using this, as it divides dividend also
+ * (u32)remainder = (u64)dividend % (u32)divisor
+ * (u64)dividend = (u64)dividend / (u32)divisor
+ */
+#define DIV_AND_MOD_U64_BY_U32(dividend, divisor) ({		\
+	uint32 rem = (uint64)(dividend) % (uint32)(divisor);	\
+	(dividend) = (uint64)(dividend) / (uint32)(divisor);	\
+	rem;							\
+})
+
+/* (u32)remainder = (u64)dividend % (u32)divisor */
+#define MOD_U64_BY_U32(dividend, divisor)	(uint32)((uint64)(dividend) % (uint32)(divisor))
+
+#define SEC_USEC_FMT \
+	"%015llu.%06u"
+#endif /* LINUX || linux */
+
+/* t: time in nano second */
+#define GET_SEC_USEC(t) \
+	DIV_U64_BY_U32(t, NSEC_PER_SEC), \
+	((uint32)(MOD_U64_BY_U32(t, NSEC_PER_SEC) / (uint32)NSEC_PER_USEC))
+
+/* Download Types */
+typedef enum download_type {
+	FW,
+	NVRAM,
+	CLM_BLOB,
+	TXCAP_BLOB
+} download_type_t;
+
+#if defined(NDIS)
+/* Firmware requested operation mode */
+#define STA_MASK			0x0001
+#define HOSTAPD_MASK		0x0002
+#define WFD_MASK			0x0004
+#define SOFTAP_FW_MASK	0x0008
+#define P2P_GO_ENABLED		0x0010
+#define P2P_GC_ENABLED		0x0020
+#define CONCURENT_MASK		0x00F0
+#endif /* #if defined(NDIS) */
+
+/* For supporting multiple interfaces */
+#define DHD_MAX_IFS			16
+#ifndef DHD_MAX_STATIC_IFS
+#define DHD_MAX_STATIC_IFS	1
+#endif
+#define DHD_DEL_IF			-0xE
+#define DHD_BAD_IF			-0xF
+#define DHD_DUMMY_INFO_IF	0xDEAF	/* Hack i/f to handle events from INFO Ring */
+/* XXX to avoid build error for NDIS for timebeing */
+#define DHD_EVENT_IF DHD_DUMMY_INFO_IF
+
+#if defined(LINUX) || defined(linux)
+enum dhd_op_flags {
+/* Firmware requested operation mode */
+	DHD_FLAG_STA_MODE				= (1 << (0)), /* STA only */
+	DHD_FLAG_HOSTAP_MODE				= (1 << (1)), /* SOFTAP only */
+	DHD_FLAG_P2P_MODE				= (1 << (2)), /* P2P Only */
+	/* STA + P2P */
+	DHD_FLAG_CONCURR_SINGLE_CHAN_MODE = (DHD_FLAG_STA_MODE | DHD_FLAG_P2P_MODE),
+	/* STA + SoftAP */
+	DHD_FLAG_CONCURR_STA_HOSTAP_MODE = (DHD_FLAG_STA_MODE | DHD_FLAG_HOSTAP_MODE),
+	/* XXX MULTI_CHAN mode is meaningful only if it is conccurncy mode */
+	DHD_FLAG_CONCURR_MULTI_CHAN_MODE		= (1 << (4)), /* STA + P2P */
+	/* Current P2P mode for P2P connection */
+	DHD_FLAG_P2P_GC_MODE				= (1 << (5)),
+	DHD_FLAG_P2P_GO_MODE				= (1 << (6)),
+	DHD_FLAG_MBSS_MODE				= (1 << (7)), /* MBSS in future */
+	DHD_FLAG_IBSS_MODE				= (1 << (8)),
+	DHD_FLAG_MFG_MODE				= (1 << (9)),
+	DHD_FLAG_RSDB_MODE				= (1 << (10)),
+	DHD_FLAG_MP2P_MODE				= (1 << (11))
+};
+#endif /* defined (LINUX) || defined(linux) */
+
+#if defined(BCMDONGLEHOST)
+#define DHD_OPMODE_SUPPORTED(dhd, opmode_flag) \
+	(dhd ? ((((dhd_pub_t *)dhd)->op_mode)  &  opmode_flag) : -1)
+#define DHD_OPMODE_STA_SOFTAP_CONCURR(dhd) \
+	(dhd ? (((((dhd_pub_t *)dhd)->op_mode) & DHD_FLAG_CONCURR_STA_HOSTAP_MODE) == \
+	DHD_FLAG_CONCURR_STA_HOSTAP_MODE) : 0)
+#else
+#define DHD_OPMODE_SUPPORTED(dhd, opmode_flag)  -1
+#define DHD_OPMODE_STA_SOFTAP_CONCURR(dhd)	0
+#endif /* defined (BCMDONGLEHOST) */
+
+/* Max sequential TX/RX Control timeouts to set HANG event */
+#ifndef MAX_CNTL_TX_TIMEOUT
+#define MAX_CNTL_TX_TIMEOUT 2
+#endif /* MAX_CNTL_TX_TIMEOUT */
+#ifndef MAX_CNTL_RX_TIMEOUT
+#define MAX_CNTL_RX_TIMEOUT 1
+#endif /* MAX_CNTL_RX_TIMEOUT */
+
+#define DHD_SCAN_ASSOC_ACTIVE_TIME	40 /* ms: Embedded default Active setting from DHD */
+#ifndef CUSTOM_SCAN_UNASSOC_ACTIVE_TIME
+#define DHD_SCAN_UNASSOC_ACTIVE_TIME	80 /* ms: Embedded def. Unassoc Active setting from DHD */
+#else
+#define DHD_SCAN_UNASSOC_ACTIVE_TIME	CUSTOM_SCAN_UNASSOC_ACTIVE_TIME
+#endif /* CUSTOM_SCAN_UNASSOC_ACTIVE_TIME */
+#define DHD_SCAN_HOME_TIME		45 /* ms: Embedded default Home time setting from DHD */
+#ifndef CUSTOM_SCAN_HOME_AWAY_TIME
+#define DHD_SCAN_HOME_AWAY_TIME	100 /* ms: Embedded default Home Away time setting from DHD */
+#else
+#define DHD_SCAN_HOME_AWAY_TIME CUSTOM_SCAN_HOME_AWAY_TIME /* ms: Custom setting from DHD */
+#endif /* CUSTOM_SCAN_HOME_AWAY_TIME */
+#ifndef CUSTOM_SCAN_PASSIVE_TIME
+#define DHD_SCAN_PASSIVE_TIME		130 /* ms: Embedded default Passive setting from DHD */
+#else
+#define DHD_SCAN_PASSIVE_TIME	CUSTOM_SCAN_PASSIVE_TIME /* ms: Custom Passive setting from DHD */
+#endif /* CUSTOM_SCAN_PASSIVE_TIME */
+
+#ifndef POWERUP_MAX_RETRY
+#define POWERUP_MAX_RETRY	3 /* how many times we retry to power up the chip */
+#endif
+#ifndef POWERUP_WAIT_MS
+#define POWERUP_WAIT_MS		2000 /* ms: time out in waiting wifi to come up */
+#endif
+/*
+ * MAX_NVRAMBUF_SIZE determines the size of the Buffer in the DHD that holds
+ * the NVRAM data. That is the size of the buffer pointed by bus->vars
+ * This also needs to be increased if NVRAM files size increases
+ */
+#define MAX_NVRAMBUF_SIZE	(32 * 1024) /* max nvram buf size */
+#define MAX_CLM_BUF_SIZE	(64 * 1024) /* max clm blob size */
+#define MAX_TXCAP_BUF_SIZE	(16 * 1024) /* max txcap blob size */
+#ifdef DHD_DEBUG
+#define DHD_JOIN_MAX_TIME_DEFAULT 10000 /* ms: Max time out for joining AP */
+#define DHD_SCAN_DEF_TIMEOUT 10000 /* ms: Max time out for scan in progress */
+#endif /* DHD_DEBUG */
+
+/* Enable Flow ring pre-memory allocation by default for Android */
+#ifdef OEM_ANDROID
+#define FLOW_RING_PREALLOC
+#endif /* OEM_ANDROID */
+
+#ifndef CONFIG_BCMDHD_FW_PATH
+#ifdef OEM_ANDROID
+#define CONFIG_BCMDHD_FW_PATH  "/system/vendor/etc/wifi/fw_bcmdhd.bin"
+#else
+#define CONFIG_BCMDHD_FW_PATH  "/var/run/rtecdc.bin"
+#endif /* OEM_ANDROID */
+#endif /* CONFIG_BCMDHD_FW_PATH */
+
+#ifndef CONFIG_BCMDHD_NVRAM_PATH
+#ifdef OEM_ANDROID
+#define CONFIG_BCMDHD_NVRAM_PATH  "/system/vendor/etc/wifi/bcmdhd.cal"
+#else
+#define CONFIG_BCMDHD_NVRAM_PATH  "/var/run/nvram.txt"
+#endif /* OEM_ANDROID */
+#endif /* CONFIG_BCMDHD_NVRAM_PATH */
+
+#ifndef CONFIG_BCMDHD_CLM_PATH
+#ifdef OEM_ANDROID
+#define CONFIG_BCMDHD_CLM_PATH "/etc/wifi/bcmdhd_clm.blob"
+#elif defined(LINUX) || defined(linux)
+#define CONFIG_BCMDHD_CLM_PATH "/var/run/bcmdhd_clm.blob"
+#else
+/* clm download will fail on empty path */
+#define CONFIG_BCMDHD_CLM_PATH ""
+#endif /* OEM_ANDROID */
+#endif /* CONFIG_BCMDHD_CLM_PATH */
+
+#ifdef DHD_LINUX_STD_FW_API
+#ifndef CONFIG_BCMDHD_CONFIG_SAR_PATH
+#define CONFIG_BCMDHD_CONFIG_SAR_PATH  "sar_config.ini"
+#endif /* CONFIG_BCMDHD_CONFIG_SAR_PATH */
+#ifndef CONFIG_BCMDHD_CONFIG_PATH
+#define CONFIG_BCMDHD_CONFIG_PATH	"wlan_config.ini"
+#endif /* CONFIG_BCMDHD_CONFIG_PATH */
+#else
+#ifndef CONFIG_BCMDHD_CONFIG_SAR_PATH
+#ifdef OEM_ANDROID
+#define CONFIG_BCMDHD_CONFIG_SAR_PATH  "/system/vendor/etc/wifi/sar_config.ini"
+#else /* OEM_ANDROID */
+#define CONFIG_BCMDHD_CONFIG_SAR_PATH  "/var/run/sar_config.ini"
+#endif /* OEM_ANDROID */
+#endif /* CONFIG_BCMDHD_CONFIG_SAR_PATH */
+
+#ifndef CONFIG_BCMDHD_CONFIG_PATH
+#ifdef OEM_ANDROID
+#define CONFIG_BCMDHD_CONFIG_PATH  "/system/vendor/etc/wifi/wlan_config.ini"
+#else /* OEM_ANDROID */
+#define CONFIG_BCMDHD_CONFIG_PATH  "/var/run/wlan_config.ini"
+#endif /* OEM_ANDROID */
+#endif /* CONFIG_BCMDHD_CONFIG_PATH */
+#endif /* DHD_LINUX_STD_FW_API */
+
+#define WL_CCODE_NULL_COUNTRY  "#n"
+
+#define FW_VER_STR_LEN	128
+#define FWID_STR_LEN 256
+#define CLM_VER_STR_LEN 128
+#define BUS_API_REV_STR_LEN	128
+#define FW_VER_STR "Version"
+#define FWID_STR_1 "FWID: 01-"
+#define FWID_STR_2 "FWID=01-"
+extern char bus_api_revision[];
+
+enum dhd_bus_wake_state {
+	WAKE_LOCK_OFF			= 0,
+	WAKE_LOCK_PRIV			= 1,
+	WAKE_LOCK_DPC			= 2,
+	WAKE_LOCK_IOCTL			= 3,
+	WAKE_LOCK_DOWNLOAD		= 4,
+	WAKE_LOCK_TMOUT			= 5,
+	WAKE_LOCK_WATCHDOG		= 6,
+	WAKE_LOCK_LINK_DOWN_TMOUT	= 7,
+	WAKE_LOCK_PNO_FIND_TMOUT	= 8,
+	WAKE_LOCK_SOFTAP_SET		= 9,
+	WAKE_LOCK_SOFTAP_STOP		= 10,
+	WAKE_LOCK_SOFTAP_START		= 11,
+	WAKE_LOCK_SOFTAP_THREAD		= 12
+};
+
+enum {
+	EVENT_BUF_POOL_LOW = 32,
+	EVENT_BUF_POOL_MEDIUM = 64,
+	EVENT_BUF_POOL_HIGH = 128,
+	EVENT_BUF_POOL_HIGHEST = 256
+};
+
+#ifdef PCIE_INB_DW
+enum dhd_bus_ds_state {
+	DW_DEVICE_DS_INVALID		= -1,
+	DW_DEVICE_DS_DEV_SLEEP		= 0,
+	DW_DEVICE_DS_DEV_SLEEP_PEND	= 1,
+	DW_DEVICE_DS_DISABLED_WAIT	= 2,
+	DW_DEVICE_DS_DEV_WAKE		= 3,
+	DW_DEVICE_DS_ACTIVE		= 4,
+	DW_DEVICE_HOST_SLEEP_WAIT	= 5,
+	DW_DEVICE_HOST_SLEEP		= 6,
+	DW_DEVICE_HOST_WAKE_WAIT	= 7,
+	DW_DEVICE_DS_D3_INFORM_WAIT	= 8
+};
+#ifdef PCIE_INB_DSACK_EXT_WAIT
+/* DS ack timeout is 3sec, FW retry count is 5.
+ * 5times * 3sec = 15sec.
+ */
+#define DW_DS_ACK_RETRY_THRESHOLD		15u
+#endif /* PCIE_INB_DSACK_EXT_WAIT */
+#endif /* PCIE_INB_DW */
+
+enum dhd_prealloc_index {
+	DHD_PREALLOC_PROT			= 0,
+	DHD_PREALLOC_RXBUF			= 1,
+	DHD_PREALLOC_DATABUF			= 2,
+	DHD_PREALLOC_OSL_BUF			= 3,
+	DHD_PREALLOC_SKB_BUF = 4,
+	DHD_PREALLOC_WIPHY_ESCAN0		= 5,
+	DHD_PREALLOC_WIPHY_ESCAN1		= 6,
+	DHD_PREALLOC_DHD_INFO			= 7,
+	DHD_PREALLOC_DHD_WLFC_INFO		= 8,
+	DHD_PREALLOC_IF_FLOW_LKUP		= 9,
+	/* 10 */
+	DHD_PREALLOC_MEMDUMP_RAM		= 11,
+	DHD_PREALLOC_DHD_WLFC_HANGER		= 12,
+	DHD_PREALLOC_PKTID_MAP			= 13,	/* Deprecated */
+	DHD_PREALLOC_PKTID_MAP_IOCTL		= 14,	/* Deprecated */
+	DHD_PREALLOC_DHD_LOG_DUMP_BUF		= 15,
+	DHD_PREALLOC_DHD_LOG_DUMP_BUF_EX	= 16,
+	DHD_PREALLOC_DHD_PKTLOG_DUMP_BUF	= 17,	/* Deprecated */
+	DHD_PREALLOC_STAT_REPORT_BUF = 18,
+	DHD_PREALLOC_WL_ESCAN = 19,
+	DHD_PREALLOC_FW_VERBOSE_RING = 20,
+	DHD_PREALLOC_FW_EVENT_RING = 21,
+	DHD_PREALLOC_DHD_EVENT_RING = 22,
+	DHD_PREALLOC_NAN_EVENT_RING = 23
+};
+
+enum dhd_dongledump_mode {
+	DUMP_DISABLED		= 0,
+	DUMP_MEMONLY		= 1,
+	DUMP_MEMFILE		= 2,
+	DUMP_MEMFILE_BUGON	= 3,
+	DUMP_MEMFILE_MAX	= 4
+};
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+typedef enum dhd_dongledump_status {
+	DUMP_READY		= 0,
+	DUMP_IN_PROGRESS	= 1,
+	DUMP_FAILURE		= 2,
+	DUMP_NOT_READY		= 3
+} dhd_dongledump_status_t;
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+enum dhd_dongledump_type {
+	DUMP_TYPE_RESUMED_ON_TIMEOUT		= 1,
+	DUMP_TYPE_D3_ACK_TIMEOUT		= 2,
+	DUMP_TYPE_DONGLE_TRAP			= 3,
+	DUMP_TYPE_MEMORY_CORRUPTION		= 4,
+	DUMP_TYPE_PKTID_AUDIT_FAILURE		= 5,
+	DUMP_TYPE_PKTID_INVALID			= 6,
+	DUMP_TYPE_SCAN_TIMEOUT			= 7,
+	DUMP_TYPE_SCAN_BUSY			= 8,
+	DUMP_TYPE_BY_SYSDUMP			= 9,
+	DUMP_TYPE_BY_LIVELOCK			= 10,
+	DUMP_TYPE_AP_LINKUP_FAILURE		= 11,
+	DUMP_TYPE_AP_ABNORMAL_ACCESS		= 12,
+	DUMP_TYPE_CFG_VENDOR_TRIGGERED		= 13,
+	DUMP_TYPE_RESUMED_ON_TIMEOUT_TX		= 14,
+	DUMP_TYPE_RESUMED_ON_TIMEOUT_RX		= 15,
+	DUMP_TYPE_RESUMED_ON_INVALID_RING_RDWR	= 16,
+	DUMP_TYPE_TRANS_ID_MISMATCH		= 17,
+	DUMP_TYPE_IFACE_OP_FAILURE		= 18,
+	DUMP_TYPE_DONGLE_INIT_FAILURE		= 19,
+	DUMP_TYPE_READ_SHM_FAIL			= 20,
+	DUMP_TYPE_DONGLE_HOST_EVENT		= 21,
+	DUMP_TYPE_SMMU_FAULT			= 22,
+	DUMP_TYPE_RESUMED_UNKNOWN		= 23,
+	DUMP_TYPE_DUE_TO_BT			= 24,
+	DUMP_TYPE_LOGSET_BEYOND_RANGE		= 25,
+	DUMP_TYPE_BY_USER			= 26,
+	DUMP_TYPE_CTO_RECOVERY			= 27,
+	DUMP_TYPE_SEQUENTIAL_PRIVCMD_ERROR	= 28,
+	DUMP_TYPE_PROXD_TIMEOUT			= 29,
+	DUMP_TYPE_INBAND_DEVICE_WAKE_FAILURE	= 30,
+	DUMP_TYPE_PKTID_POOL_DEPLETED		= 31,
+	DUMP_TYPE_ESCAN_SYNCID_MISMATCH		= 32,
+	DUMP_TYPE_INVALID_SHINFO_NRFRAGS	= 33,
+	DUMP_TYPE_P2P_DISC_BUSY			= 34,
+	DUMP_TYPE_CONT_EXCESS_PM_AWAKE		= 35,
+	DUMP_TYPE_DONGLE_TRAP_DURING_WIFI_ONOFF	= 36,
+	DUMP_TYPE_BY_DSACK_HC_DUE_TO_ISR_DELAY	= 37,
+	DUMP_TYPE_BY_DSACK_HC_DUE_TO_DPC_DELAY	= 38
+};
+
+enum dhd_hang_reason {
+	HANG_REASON_MASK				= 0x8000,
+	HANG_REASON_IOCTL_RESP_TIMEOUT			= 0x8001,
+	HANG_REASON_DONGLE_TRAP				= 0x8002,
+	HANG_REASON_D3_ACK_TIMEOUT			= 0x8003,
+	HANG_REASON_BUS_DOWN				= 0x8004,
+	HANG_REASON_MSGBUF_LIVELOCK			= 0x8006,
+	HANG_REASON_IFACE_DEL_FAILURE			= 0x8007,
+	HANG_REASON_HT_AVAIL_ERROR			= 0x8008,
+	HANG_REASON_PCIE_RC_LINK_UP_FAIL		= 0x8009,
+	HANG_REASON_PCIE_PKTID_ERROR			= 0x800A,
+	HANG_REASON_IFACE_ADD_FAILURE			= 0x800B,
+	HANG_REASON_IOCTL_RESP_TIMEOUT_SCHED_ERROR	= 0x800C,
+	HANG_REASON_D3_ACK_TIMEOUT_SCHED_ERROR		= 0x800D,
+	HANG_REASON_SEQUENTIAL_PRIVCMD_ERROR		= 0x800E,
+	HANG_REASON_SCAN_BUSY				= 0x800F,
+	HANG_REASON_BSS_UP_FAILURE			= 0x8010,
+	HANG_REASON_BSS_DOWN_FAILURE			= 0x8011,
+	HANG_REASON_IOCTL_SUSPEND_ERROR			= 0x8012,
+	HANG_REASON_ESCAN_SYNCID_MISMATCH		= 0x8013,
+	HANG_REASON_SCAN_TIMEOUT			= 0x8014,
+	HANG_REASON_SCAN_TIMEOUT_SCHED_ERROR		= 0x8015,
+	HANG_REASON_P2P_DISC_BUSY			= 0x8016,
+	HANG_REASON_PCIE_LINK_DOWN_RC_DETECT		= 0x8805,
+	HANG_REASON_INVALID_EVENT_OR_DATA		= 0x8806,
+	HANG_REASON_UNKNOWN				= 0x8807,
+	HANG_REASON_PCIE_LINK_DOWN_EP_DETECT		= 0x8808,
+	HANG_REASON_PCIE_CTO_DETECT			= 0x8809,
+	HANG_REASON_SLEEP_FAILURE			= 0x880A,
+	HANG_REASON_DS_SKIP_TIMEOUT			= 0x880B,
+	HANG_REASON_BT2WL_REG_RESET			= 0x880C,
+	HANG_REASON_MAX					= 0x880D
+};
+
+#define WLC_E_DEAUTH_MAX_REASON 0x0FFF
+
+enum dhd_rsdb_scan_features {
+	/* Downgraded scan feature for AP active */
+	RSDB_SCAN_DOWNGRADED_AP_SCAN = 0x01,
+	/* Downgraded scan feature for P2P Discovery */
+	RSDB_SCAN_DOWNGRADED_P2P_DISC_SCAN = 0x02,
+	/* Enable channel pruning for ROAM SCAN */
+	RSDB_SCAN_DOWNGRADED_CH_PRUNE_ROAM = 0x10,
+	/* Enable channel pruning for any SCAN */
+	RSDB_SCAN_DOWNGRADED_CH_PRUNE_ALL  = 0x20
+};
+
+#define VENDOR_SEND_HANG_EXT_INFO_LEN (800 + 1)
+#ifdef DHD_EWPR_VER2
+#define VENDOR_SEND_HANG_EXT_INFO_VER 20181111
+#else
+#define VENDOR_SEND_HANG_EXT_INFO_VER 20170905
+#endif /* DHD_EWPR_VER2 */
+
+#define HANG_INFO_TRAP_T_NAME_MAX 6
+#define HANG_INFO_TRAP_T_REASON_IDX 0
+#define HANG_INFO_TRAP_T_SUBTYPE_IDX 2
+#define HANG_INFO_TRAP_T_OFFSET_IDX 3
+#define HANG_INFO_TRAP_T_EPC_IDX 4
+#define HANG_FIELD_STR_MAX_LEN 9
+#define HANG_FIELD_CNT_MAX 69
+#define HANG_FIELD_IF_FAILURE_CNT 10
+#define HANG_FIELD_IOCTL_RESP_TIMEOUT_CNT 8
+#define HANG_FIELD_TRAP_T_STACK_CNT_MAX 16
+#define HANG_FIELD_MISMATCH_CNT 10
+#define HANG_INFO_BIGDATA_KEY_STACK_CNT 4
+
+#define DEBUG_DUMP_TIME_BUF_LEN (22 + 1)
+/* delimiter between values */
+#define HANG_KEY_DEL    ' '
+#define HANG_RAW_DEL    '_'
+
+#ifdef DHD_EWPR_VER2
+#define HANG_INFO_BIGDATA_EXTRA_KEY 4
+#define HANG_INFO_TRAP_T_EXTRA_KEY_IDX 5
+#endif
+
+/* Packet alignment for most efficient SDIO (can change based on platform) */
+#ifndef DHD_SDALIGN
+#define DHD_SDALIGN	32
+#endif
+
+#define DHD_TX_CONTEXT_MASK 0xff
+#define DHD_TX_START_XMIT   0x01
+#define DHD_TX_SEND_PKT     0x02
+#define DHD_IF_SET_TX_ACTIVE(ifp, context)	\
+    ifp->tx_paths_active |= context;
+#define DHD_IF_CLR_TX_ACTIVE(ifp, context)	\
+    ifp->tx_paths_active &= ~context;
+#define DHD_IF_IS_TX_ACTIVE(ifp)	\
+	(ifp->tx_paths_active)
+/**
+ * DMA-able buffer parameters
+ * - dmaaddr_t is 32bits on a 32bit host.
+ *   dhd_dma_buf::pa may not be used as a sh_addr_t, bcm_addr64_t or uintptr
+ * - dhd_dma_buf::_alloced is ONLY for freeing a DMA-able buffer.
+ */
+typedef struct dhd_dma_buf {
+	void      *va;      /* virtual address of buffer */
+	uint32    len;      /* user requested buffer length */
+	dmaaddr_t pa;       /* physical address of buffer */
+	void      *dmah;    /* dma mapper handle */
+	void      *secdma;  /* secure dma sec_cma_info handle */
+	uint32    _alloced; /* actual size of buffer allocated with align and pad */
+} dhd_dma_buf_t;
+
+/* host reordering packts logic */
+/* followed the structure to hold the reorder buffers (void **p) */
+typedef struct reorder_info {
+	void **p;
+	uint8 flow_id;
+	uint8 cur_idx;
+	uint8 exp_idx;
+	uint8 max_idx;
+	uint8 pend_pkts;
+} reorder_info_t;
+
+/* throughput test packet format */
+typedef struct tput_pkt {
+	/* header */
+	uint8 mac_sta[ETHER_ADDR_LEN];
+	uint8 mac_ap[ETHER_ADDR_LEN];
+	uint16 pkt_type;
+	uint8 PAD[2];
+	/* data */
+	uint32 crc32;
+	uint32 pkt_id;
+	uint32 num_pkts;
+} tput_pkt_t;
+
+typedef enum {
+	TPUT_PKT_TYPE_NORMAL,
+	TPUT_PKT_TYPE_STOP
+} tput_pkt_type_t;
+
+#define TPUT_TEST_MAX_PAYLOAD 1500
+#define TPUT_TEST_WAIT_TIMEOUT_DEFAULT 5000
+
+#ifdef DHDTCPACK_SUPPRESS
+
+enum {
+	/* TCPACK suppress off */
+	TCPACK_SUP_OFF,
+	/* Replace TCPACK in txq when new coming one has higher ACK number. */
+	TCPACK_SUP_REPLACE,
+	/* TCPACK_SUP_REPLACE + delayed TCPACK TX unless ACK to PSH DATA.
+	 * This will give benefits to Half-Duplex bus interface(e.g. SDIO) that
+	 * 1. we are able to read TCP DATA packets first from the bus
+	 * 2. TCPACKs that don't need to hurry delivered remains longer in TXQ so can be suppressed.
+	 */
+	TCPACK_SUP_DELAYTX,
+	TCPACK_SUP_HOLD,
+	TCPACK_SUP_LAST_MODE
+};
+#endif /* DHDTCPACK_SUPPRESS */
+
+#if defined(BCM_ROUTER_DHD)
+#define DHD_DWM_TBL_SIZE           57
+/* DSCP WMM AC Mapping macros and structures */
+#define DHD_TRF_MGMT_DWM_FILTER_BIT                 0x8
+#define DHD_TRF_MGMT_DWM_PRIO_BITS                  0x7
+#define DHD_TRF_MGMT_DWM_FAVORED_BIT                0x10
+#define DHD_TRF_MGMT_DWM_PRIO(dwm_tbl_entry) ((dwm_tbl_entry) & DHD_TRF_MGMT_DWM_PRIO_BITS)
+#define DHD_TRF_MGMT_DWM_IS_FAVORED_SET(dwm_tbl_entry) \
+	((dwm_tbl_entry) & DHD_TRF_MGMT_DWM_FAVORED_BIT)
+#define DHD_TRF_MGMT_DWM_SET_FAVORED(dwm_tbl_entry) \
+	((dwm_tbl_entry) |= DHD_TRF_MGMT_DWM_FAVORED_BIT)
+#define DHD_TRF_MGMT_DWM_IS_FILTER_SET(dwm_tbl_entry) \
+	((dwm_tbl_entry) & DHD_TRF_MGMT_DWM_FILTER_BIT)
+#define DHD_TRF_MGMT_DWM_SET_FILTER(dwm_tbl_entry) \
+	((dwm_tbl_entry) |= DHD_TRF_MGMT_DWM_FILTER_BIT)
+
+typedef struct {
+	uint8 dhd_dwm_enabled;
+	uint8 dhd_dwm_tbl[DHD_DWM_TBL_SIZE];
+} dhd_trf_mgmt_dwm_tbl_t;
+#endif /* for BCM_ROUTER_DHD */
+
+#define DHD_NULL_CHK_AND_RET(cond) \
+	if (!cond) { \
+		DHD_ERROR(("%s " #cond " is NULL\n", __FUNCTION__)); \
+		return; \
+	}
+
+#define DHD_NULL_CHK_AND_RET_VAL(cond, value) \
+	if (!cond) { \
+		DHD_ERROR(("%s " #cond " is NULL\n", __FUNCTION__)); \
+		return value; \
+	}
+
+#define DHD_NULL_CHK_AND_GOTO(cond, label) \
+	if (!cond) { \
+		DHD_ERROR(("%s " #cond " is NULL\n", __FUNCTION__)); \
+		goto label; \
+	}
+
+/*
+ * Accumulating the queue lengths of all flowring queues in a parent object,
+ * to assert flow control, when the cummulative queue length crosses an upper
+ * threshold defined on a parent object. Upper threshold may be maintained
+ * at a station level, at an interface level, or at a dhd instance.
+ *
+ * cumm_ctr_t abstraction:
+ * cumm_ctr_t abstraction may be enhanced to use an object with a hysterisis
+ * pause on/off threshold callback.
+ * All macros use the address of the cummulative length in the parent objects.
+ *
+ * Cummulative counters in parent objects may be updated without spinlocks.
+ *
+ * If a cummulative queue length is desired across all flows
+ * belonging to either of (a station, or an interface or a dhd instance), then
+ * an atomic operation is required using an atomic_t cummulative counters or
+ * using a spinlock. BCM_ROUTER_DHD uses the Linux atomic_t construct.
+ */
+#if defined(BCM_ROUTER_DHD)
+
+typedef atomic_t cumm_ctr_t;       /* BCM_ROUTER_DHD Linux: atomic operations */
+#define DHD_CUMM_CTR_PTR(clen)     ((cumm_ctr_t*)(clen))
+#define DHD_CUMM_CTR(clen)         DHD_CUMM_CTR_PTR(clen) /* atomic accessor */
+#define DHD_CUMM_CTR_READ(clen)    atomic_read(DHD_CUMM_CTR(clen)) /* read */
+#define DHD_CUMM_CTR_INIT(clen)                                                \
+	ASSERT(DHD_CUMM_CTR_PTR(clen) != DHD_CUMM_CTR_PTR(NULL));                  \
+	atomic_set(DHD_CUMM_CTR(clen), 0);
+#define DHD_CUMM_CTR_INCR(clen)                                                \
+	ASSERT(DHD_CUMM_CTR_PTR(clen) != DHD_CUMM_CTR_PTR(NULL));                  \
+	atomic_add(1, DHD_CUMM_CTR(clen));                                         \
+	ASSERT(DHD_CUMM_CTR_READ(clen) != 0); /* ensure it does not wrap */
+#define DHD_CUMM_CTR_DECR(clen)                                                \
+	ASSERT(DHD_CUMM_CTR_PTR(clen) != DHD_CUMM_CTR_PTR(NULL));                  \
+	ASSERT(DHD_CUMM_CTR_READ(clen) > 0);                                       \
+	atomic_sub(1, DHD_CUMM_CTR(clen));
+
+#else  /* ! BCM_ROUTER_DHD */
+
+/* Cummulative length not supported. */
+typedef uint32 cumm_ctr_t;
+#define DHD_CUMM_CTR_PTR(clen)     ((cumm_ctr_t*)(clen))
+#define DHD_CUMM_CTR(clen)         *(DHD_CUMM_CTR_PTR(clen)) /* accessor */
+#define DHD_CUMM_CTR_READ(clen)    DHD_CUMM_CTR(clen) /* read access */
+#define DHD_CUMM_CTR_INIT(clen)                                                \
+	ASSERT(DHD_CUMM_CTR_PTR(clen) != DHD_CUMM_CTR_PTR(NULL));
+#define DHD_CUMM_CTR_INCR(clen)                                                \
+	ASSERT(DHD_CUMM_CTR_PTR(clen) != DHD_CUMM_CTR_PTR(NULL));
+#define DHD_CUMM_CTR_DECR(clen)                                                \
+	ASSERT(DHD_CUMM_CTR_PTR(clen) != DHD_CUMM_CTR_PTR(NULL));
+
+#endif /* ! BCM_ROUTER_DHD */
+
+#if defined(WLTDLS) && defined(PCIE_FULL_DONGLE)
+struct tdls_peer_node {
+	uint8 addr[ETHER_ADDR_LEN];
+	struct tdls_peer_node *next;
+};
+typedef struct tdls_peer_node tdls_peer_node_t;
+typedef struct {
+	tdls_peer_node_t *node;
+	uint8 tdls_peer_count;
+} tdls_peer_tbl_t;
+#endif /* defined(WLTDLS) && defined(PCIE_FULL_DONGLE) */
+
+typedef enum dhd_ring_id {
+	DEBUG_RING_ID_INVALID = 0x1,
+	FW_VERBOSE_RING_ID = 0x2,
+	DHD_EVENT_RING_ID = 0x3,
+	DRIVER_LOG_RING_ID = 0x4,
+	ROAM_STATS_RING_ID = 0x5,
+	BT_LOG_RING_ID = 0x6,
+	PACKET_LOG_RING_ID = 0x7,
+	DEBUG_DUMP_RING1_ID = 0x8,
+	DEBUG_DUMP_RING2_ID = 0x9,
+	MEM_DUMP_RING_ID = 0xa,
+	DEBUG_RING_ID_MAX = 0xb
+} dhd_ring_id_t;
+
+#define HEALTH_CHK_BUF_SIZE 256
+
+/* DEBUG_DUMP SUB COMMAND */
+enum {
+	CMD_DEFAULT,
+	CMD_UNWANTED,
+	CMD_DISCONNECTED,
+	CMD_MAX
+};
+
+#define DHD_DUMP_SUBSTR_UNWANTED	"_unwanted"
+#define DHD_DUMP_SUBSTR_DISCONNECTED	"_disconnected"
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+#define DHD_DUMP_AXI_ERROR_FILENAME	"axi_error"
+#define DHD_DUMP_HAL_FILENAME_SUFFIX	"_hal"
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#define FW_LOGSET_MASK_ALL 0xFFFFu
+
+#if defined(CUSTOMER_HW7_DEBUG)
+#define DHD_COMMON_DUMP_PATH    PLATFORM_PATH
+#elif defined(CUSTOM_DHD_COMMON_DUMP_PATH)
+#define DHD_COMMON_DUMP_PATH	CUSTOM_DHD_COMMON_DUMP_PATH
+#elif defined(BOARD_HIKEY)
+#ifndef DHD_COMMON_DUMP_PATH
+#define DHD_COMMON_DUMP_PATH	"/data/misc/wifi/"
+#endif /* !DHD_COMMON_DUMP_PATH */
+#elif defined(OEM_ANDROID) && defined(__ARM_ARCH_7A__)
+#define DHD_COMMON_DUMP_PATH	"/data/vendor/wifi/"
+#elif defined(OEM_ANDROID) /* For Brix Live Image */
+#define DHD_COMMON_DUMP_PATH	"/installmedia/"
+#else /* Default */
+#define DHD_COMMON_DUMP_PATH	"/root/"
+#endif /* CUSTOMER_HW7_DEBUG */
+
+#define DHD_MEMDUMP_LONGSTR_LEN 180
+
+struct cntry_locales_custom {
+	char iso_abbrev[WLC_CNTRY_BUF_SZ];      /* ISO 3166-1 country abbreviation */
+	char custom_locale[WLC_CNTRY_BUF_SZ];   /* Custom firmware locale */
+	int32 custom_locale_rev;                /* Custom local revisin default -1 */
+};
+
+#ifdef __linux__
+extern uint16 dhd_plat_align_rxbuf_size(uint16 rxbufpost_sz);
+#else
+static INLINE uint16 dhd_plat_align_rxbuf_size(uint16 rxbufpost_sz) { return rxbufpost_sz; }
+#endif /* !__linux__ */
+
+#if defined(LINUX) || defined(linux)
+int dhd_send_msg_to_daemon(struct sk_buff *skb, void *data, int size);
+#endif /* LINUX || linux */
+#ifdef REPORT_FATAL_TIMEOUTS
+typedef struct timeout_info {
+	void	*scan_timer_lock;
+	void	*join_timer_lock;
+	void	*cmd_timer_lock;
+	void	*bus_timer_lock;
+	uint32 scan_timeout_val;
+	uint32 join_timeout_val;
+	uint32 cmd_timeout_val;
+	uint32 bus_timeout_val;
+	bool scan_timer_active;
+	bool join_timer_active;
+	bool cmd_timer_active;
+	bool bus_timer_active;
+	osl_timer_t *scan_timer;
+	osl_timer_t *join_timer;
+	osl_timer_t *cmd_timer;
+	osl_timer_t *bus_timer;
+	uint32 cmd_request_id;
+	uint32 cmd;
+	uint32 cmd_join_error;
+	uint16 escan_syncid;
+	bool escan_aborted;
+	uint16 abort_syncid;
+} timeout_info_t;
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+#ifdef DMAMAP_STATS
+typedef struct dmamap_stats {
+	uint64 txdata;
+	uint64 txdata_sz;
+	uint64 rxdata;
+	uint64 rxdata_sz;
+	uint64 ioctl_rx;
+	uint64 ioctl_rx_sz;
+	uint64 event_rx;
+	uint64 event_rx_sz;
+	uint64 info_rx;
+	uint64 info_rx_sz;
+	uint64 tsbuf_rx;
+	uint64 tsbuf_rx_sz;
+} dma_stats_t;
+#endif /* DMAMAP_STATS */
+
+/*  see wlfc_proto.h for tx status details */
+#define DHD_MAX_TX_STATUS_MSGS     9u
+
+#ifdef TX_STATUS_LATENCY_STATS
+typedef struct dhd_if_tx_status_latency {
+	/* total number of tx_status received on this interface */
+	uint64 num_tx_status;
+	/* cumulative tx_status latency for this interface */
+	uint64 cum_tx_status_latency;
+} dhd_if_tx_status_latency_t;
+#endif /* TX_STATUS_LATENCY_STATS */
+
+/* Bit in dhd_pub_t::gdb_proxy_stop_count set when firmware is stopped by GDB */
+#define GDB_PROXY_STOP_MASK 1
+
+/* Enable Reserve STA flowrings only for Android */
+#if defined(OEM_ANDROID)
+#define DHD_LIMIT_MULTI_CLIENT_FLOWRINGS
+#endif /* OEM_ANDROID */
+
+typedef enum {
+	FW_UNLOADED = 0,
+	FW_DOWNLOAD_IN_PROGRESS = 1,
+	FW_DOWNLOAD_DONE = 2
+} fw_download_status_t;
+
+#define PCIE_DB7_MAGIC_NUMBER_ISR_TRAP	0xdead0001
+#define PCIE_DB7_MAGIC_NUMBER_DPC_TRAP	0xdead0002
+#define PCIE_DB7_MAGIC_NUMBER_DPC_LPM	0xdead0003
+
+typedef struct dhd_db7_info {
+	bool	fw_db7w_trap;
+	bool	fw_db7w_trap_inprogress;
+	bool	fw_db7w_trap_recieved;
+	uint32	db7_magic_number;
+
+	uint32	debug_db7_send_cnt;
+	uint32	debug_db7_trap_cnt;
+	uint32	debug_db7_timing_error_cnt;
+	uint64	debug_db7_send_time;
+	uint64	debug_db7_trap_time;
+	uint64	debug_max_db7_dur;
+	uint64	debug_max_db7_send_time;
+	uint64	debug_max_db7_trap_time;
+} dhd_db7_info_t;
+
+typedef enum dhd_induce_error_states
+{
+	DHD_INDUCE_ERROR_CLEAR		= 0x0,
+	DHD_INDUCE_IOCTL_TIMEOUT	= 0x1,
+	DHD_INDUCE_D3_ACK_TIMEOUT	= 0x2,
+	DHD_INDUCE_LIVELOCK		= 0x3,
+	DHD_INDUCE_DROP_OOB_IRQ		= 0x4,
+	DHD_INDUCE_DROP_AXI_SIG		= 0x5,
+	DHD_INDUCE_TX_BIG_PKT		= 0x6,
+	DHD_INDUCE_IOCTL_SUSPEND_ERROR	= 0x7,
+	DHD_INDUCE_SCAN_TIMEOUT         = 0x8,
+	DHD_INDUCE_SCAN_TIMEOUT_SCHED_ERROR     = 0x9,
+	DHD_INDUCE_PKTID_INVALID_SAVE   = 0xA,
+	DHD_INDUCE_PKTID_INVALID_FREE   = 0xB,
+	DHD_INDUCE_PCIE_LINK_DOWN	= 0xC,
+	/* Big hammer induction */
+	DHD_INDUCE_BH_ON_FAIL_ONCE	= 0x10,
+	DHD_INDUCE_BH_ON_FAIL_ALWAYS	= 0x11,
+	DHD_INDUCE_BH_CBP_HANG		= 0x12,
+	DHD_INDUCE_P2P_DISC_BUSY	= 0x13,
+	DHD_INDUCE_PCIE_LINK_DOWN_IN_ISR	= 0x14,
+	DHD_INDUCE_ERROR_MAX
+} dhd_induce_error_states_t;
+
+#define MAX_MDRING_ITEM_DUMP	20
+
+#define MAX_RX_LAT_PRIO		8
+#define MAX_RX_LAT_HIST_BIN	50
+#define MAX_RXCPL_HISTORY	50
+#define	RX_LAT_TIME_SCALE	32
+#define	RX_LAT_BIN_SCALE	500
+
+typedef struct rx_cpl_history {
+	uint32 host_time;
+	uint32 ptm_high;
+	uint32 ptm_low;
+	uint32 rx_t0;
+	uint32 latency;
+	uint8 slice;
+	uint8 priority;
+	int8 rssi;
+} rx_cpl_history_t;
+
+typedef struct rx_cpl_lat_info {
+	rx_cpl_history_t rx_history[MAX_RXCPL_HISTORY];
+	uint32 rxcpl_hist_count;
+	uint32	rx_dur_2g[MAX_RX_LAT_PRIO][MAX_RX_LAT_HIST_BIN];
+	uint32	rx_dur_5g[MAX_RX_LAT_PRIO][MAX_RX_LAT_HIST_BIN];
+} rx_cpl_lat_info_t;
+
+#define MAX_TXCPL_HISTORY	50u
+typedef struct tx_cpl_history {
+	uint32 host_time;
+	uint32 ptm_high;
+	uint32 ptm_low;
+	uint16 latency;
+	uint16 flowid;
+	uint8 tid;
+} tx_cpl_history_t;
+
+typedef struct tx_cpl_info {
+	tx_cpl_history_t tx_history[MAX_TXCPL_HISTORY];
+	uint32 txcpl_hist_count;
+} tx_cpl_info_t;
+
+#define DHD_PTM_CLK_ID		0xEu		/* PTM clock ID; IDs 0-3 are used in tsync module */
+#define DHD_PTM_CLK_ID_MASK	0xF0000000u	/* Bitmask for clkid */
+#define DHD_PTM_CLK_ID_SHIFT	28u		/* Bitshift for clkid */
+
+#define DHD_PTM_GET_CLKID(ts) (((ts) & DHD_PTM_CLK_ID_MASK) >> DHD_PTM_CLK_ID_SHIFT)
+#define DHD_PTM_CLKID(ts) (DHD_PTM_GET_CLKID(ts) == DHD_PTM_CLK_ID)
+
+#ifdef DHD_USE_STATIC_CTRLBUF
+#define PKTFREE_CTRLBUF(osh, p, flag) PKTFREE_STATIC(osh, p, flag);
+#else
+#define PKTFREE_CTRLBUF(osh, p, flag) PKTFREE(osh, p, flag);
+#endif /* DHD_USE_STATIC_CTRLBUF */
+
+#define RX_PKTFREE(osh, pkt_type, p, flag) \
+do { \
+	if (ntoh16(pkt_type) == ETHER_TYPE_BRCM) { \
+		PKTFREE_CTRLBUF(osh, p, flag); \
+	} else { \
+		PKTCFREE(osh, p, flag); \
+	} \
+} while (0);
+
+#if defined(SHOW_LOGTRACE) && defined(DHD_USE_KTHREAD_FOR_LOGTRACE)
+/* Timestamps to trace dhd_logtrace_thread() */
+struct dhd_logtrace_thr_ts {
+	uint64 entry_time;
+	uint64 sem_down_time;
+	uint64 flush_time;
+	uint64 unexpected_break_time;
+	uint64 complete_time;
+};
+#endif /* SHOW_LOGTRACE && DHD_USE_KTHREAD_FOR_LOGTRACE */
+#ifdef SUPPORT_OTA_UPDATE
+#define MAX_EXT_INFO_LEN	10u
+typedef struct ota_update_info {
+	uint32 clm_len;
+	uint8 *clm_buf;
+	uint32 nvram_len;
+	uint8 *nvram_buf;
+	uint8 nvram_ext[MAX_EXT_INFO_LEN];
+} ota_update_info_t;
+#endif /* SUPPORT_OTA_UPDATE */
+/**
+ * Common structure for module and instance linkage.
+ * Instantiated once per hardware (dongle) instance that this DHD manages.
+ */
+typedef struct dhd_pub {
+	/* Linkage ponters */
+	osl_t *osh;		/* OSL handle */
+	struct dhd_bus *bus;	/* Bus module handle */
+	struct dhd_prot *prot;	/* Protocol module handle */
+	struct dhd_info  *info; /* Info module handle */
+	struct dhd_dbg *dbg;	/* Debugability module handle */
+#if defined(SHOW_LOGTRACE) && defined(DHD_USE_KTHREAD_FOR_LOGTRACE)
+	struct dhd_logtrace_thr_ts logtrace_thr_ts;
+#endif /* SHOW_LOGTRACE && DHD_USE_KTHREAD_FOR_LOGTRACE */
+
+	/* to NDIS developer, the structure dhd_common is redundant,
+	 * please do NOT merge it back from other branches !!!
+	 */
+
+#ifdef BCMDBUS
+	struct dbus_pub *dbus;
+#endif
+
+	/* Internal dhd items */
+	bool up;		/* Driver up/down (to OS) */
+#ifdef WL_CFG80211
+	spinlock_t up_lock;	/* Synchronization with CFG80211 down */
+#endif /* WL_CFG80211 */
+	bool txoff;		/* Transmit flow-controlled */
+	bool dongle_reset;  /* TRUE = DEVRESET put dongle into reset */
+	enum dhd_bus_state busstate;
+	uint dhd_bus_busy_state;	/* Bus busy state */
+	uint hdrlen;		/* Total DHD header length (proto + bus) */
+	uint maxctl;		/* Max size rxctl request from proto to bus */
+	uint rxsz;		/* Rx buffer size bus module should use */
+	uint8 wme_dp;	/* wme discard priority */
+#ifdef DNGL_AXI_ERROR_LOGGING
+	uint32 axierror_logbuf_addr;
+	bool axi_error;
+	struct dhd_axi_error_dump *axi_err_dump;
+#endif /* DNGL_AXI_ERROR_LOGGING */
+	/* Dongle media info */
+	bool iswl;		/* Dongle-resident driver is wl */
+	ulong drv_version;	/* Version of dongle-resident driver */
+	struct ether_addr mac;	/* MAC address obtained from dongle */
+	dngl_stats_t dstats;	/* Stats for dongle-based data */
+
+	/* Additional stats for the bus level */
+	ulong tx_packets;	/* Data packets sent to dongle */
+	ulong actual_tx_pkts;	/* Actual data packets sent to dongle */
+	ulong tot_txcpl;	/* Total Tx completion received */
+	ulong tx_dropped;	/* Data packets dropped in dhd */
+	ulong tx_multicast;	/* Multicast data packets sent to dongle */
+	ulong tx_errors;	/* Errors in sending data to dongle */
+	ulong tx_ctlpkts;	/* Control packets sent to dongle */
+	ulong tx_ctlerrs;	/* Errors sending control frames to dongle */
+	ulong rx_packets;	/* Packets sent up the network interface */
+	ulong rx_forward;	/* Packets forwarded at DHD driver level */
+	ulong rx_multicast;	/* Multicast packets sent up the network interface */
+	ulong rx_errors;	/* Errors processing rx data packets */
+	ulong rx_ctlpkts;	/* Control frames processed from dongle */
+	ulong rx_ctlerrs;	/* Errors in processing rx control frames */
+	ulong rx_dropped;	/* Packets dropped locally (no memory) */
+	ulong rx_flushed;  /* Packets flushed due to unscheduled sendup thread */
+	ulong wd_dpc_sched;   /* Number of times dhd dpc scheduled by watchdog timer */
+	ulong rx_pktgetfail; /* Number of PKTGET failures in DHD on RX */
+	ulong rx_pktgetpool_fail;	/* Number of PKTGET_POOL failures in DHD on RX */
+	ulong tx_pktgetfail; /* Number of PKTGET failures in DHD on TX */
+	ulong rx_readahead_cnt;	/* Number of packets where header read-ahead was used. */
+	ulong tx_realloc;	/* Number of tx packets we had to realloc for headroom */
+	ulong fc_packets;       /* Number of flow control pkts recvd */
+	ulong tx_big_packets;	/* Dropped data packets that are larger than MAX_MTU_SZ */
+	ulong tx_bytes;		/* total no. of bytes transmitted successfully */
+	ulong rx_bytes;		/* total no. of bytes received */
+#ifdef DMAMAP_STATS
+	/* DMA Mapping statistics */
+	dma_stats_t dma_stats;
+#endif /* DMAMAP_STATS */
+#ifdef WL_MONITOR
+	bool monitor_enable;
+#endif /* WL_MONITOR */
+	/* Last error return */
+	int bcmerror;
+	uint tickcnt;
+
+	/* Last error from dongle */
+	int dongle_error;
+
+	uint8 country_code[WLC_CNTRY_BUF_SZ];
+
+	/* Suspend disable flag and "in suspend" flag */
+	int suspend_disable_flag; /* "1" to disable all extra powersaving during suspend */
+	int in_suspend;			/* flag set to 1 when early suspend called */
+#ifdef PNO_SUPPORT
+	int pno_enable;			/* pno status : "1" is pno enable */
+	int pno_suspend;		/* pno suspend status : "1" is pno suspended */
+#endif /* PNO_SUPPORT */
+	/* DTIM skip value, default 0(or 1) means wake each DTIM
+	 * 3 means skip 2 DTIMs and wake up 3rd DTIM(9th beacon when AP DTIM is 3)
+	 */
+	int early_suspended;		/* Early suspend status */
+#if defined(PKT_FILTER_SUPPORT)
+	int dhcp_in_progress;		/* DHCP period */
+#endif /* PKT_FILTER_SUPPORT */
+	/* Pkt filter defination */
+	char * pktfilter[100];
+	int pktfilter_count;
+
+	wl_country_t dhd_cspec;		/* Current Locale info */
+#ifdef CUSTOM_COUNTRY_CODE
+	uint dhd_cflags;
+#endif /* CUSTOM_COUNTRY_CODE */
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+	bool is_blob;			/* Checking for existance of Blob file */
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+	bool force_country_change;
+	int	op_mode;				/* STA, HostAPD, WFD, SoftAP */
+
+#if defined(LINUX) || defined(linux)
+#if defined(OEM_ANDROID)
+	struct mutex wl_start_stop_lock; /* lock/unlock for Android start/stop */
+	struct mutex wl_softap_lock;		 /* lock/unlock for any SoftAP/STA settings */
+#endif /* defined(OEM_ANDROID) */
+#endif /* defined (LINUX) || defined(linux) */
+
+#ifdef NDIS
+	PDEVICE_OBJECT pdo;
+	PDEVICE_OBJECT fdo;
+	PDEVICE_OBJECT nextDeviceObj;
+#if defined(BCMWDF)
+	WDFDEVICE wdfDevice;
+#endif /* (BCMWDF)  */
+#endif /* NDIS */
+#ifdef PROP_TXSTATUS
+	bool	wlfc_enabled;
+	int	wlfc_mode;
+	void*	wlfc_state;
+	/*
+	Mode in which the dhd flow control shall operate. Must be set before
+	traffic starts to the device.
+	0 - Do not do any proptxtstatus flow control
+	1 - Use implied credit from a packet status
+	2 - Use explicit credit
+	3 - Only AMPDU hostreorder used. no wlfc.
+	*/
+	uint8	proptxstatus_mode;
+	bool	proptxstatus_txoff;
+	bool	proptxstatus_module_ignore;
+	bool	proptxstatus_credit_ignore;
+	bool	proptxstatus_txstatus_ignore;
+
+	bool	wlfc_rxpkt_chk;
+#ifdef LIMIT_BORROW
+	bool wlfc_borrow_allowed;
+#endif /* LIMIT_BORROW */
+	/*
+	 * implement below functions in each platform if needed.
+	 */
+	/* platform specific function whether to skip flow control */
+	bool (*skip_fc)(void * dhdp, uint8 ifx);
+	/* platform specific function for wlfc_enable and wlfc_deinit */
+	void (*plat_init)(void *dhd);
+	void (*plat_deinit)(void *dhd);
+#ifdef DHD_WLFC_THREAD
+	bool                wlfc_thread_go;
+#if defined(LINUX)
+	struct task_struct* wlfc_thread;
+	wait_queue_head_t   wlfc_wqhead;
+#else
+	#error "wlfc thread not enabled"
+#endif /* LINUX */
+#endif /* DHD_WLFC_THREAD */
+#endif /* PROP_TXSTATUS */
+#ifdef PNO_SUPPORT
+	void *pno_state;
+#endif
+#ifdef RTT_SUPPORT
+	void *rtt_state;
+	bool rtt_supported;
+#endif
+#ifdef ROAM_AP_ENV_DETECTION
+	bool	roam_env_detection;
+#endif
+	bool	dongle_isolation;
+	bool	is_pcie_watchdog_reset;
+
+/* Begin - Variables to track Bus Errors */
+	bool	dongle_trap_occured;	/* flag for sending HANG event to upper layer */
+	bool	iovar_timeout_occured;	/* flag to indicate iovar resumed on timeout */
+	bool	invalid_shinfo_nrfrags;	/* flag to indicate invlaid shinfo nrfrags */
+	bool	is_sched_error;		/* flag to indicate timeout due to scheduling issue */
+#ifdef PCIE_FULL_DONGLE
+	bool	d3ack_timeout_occured;	/* flag to indicate d3ack resumed on timeout */
+	bool	livelock_occured;	/* flag to indicate livelock occured */
+	bool	pktid_audit_failed;	/* flag to indicate pktid audit failure */
+	bool	pktid_invalid_occured;	/* flag to indicate invalid pktid */
+#endif /* PCIE_FULL_DONGLE */
+	bool	iface_op_failed;	/* flag to indicate interface operation failed */
+	bool	scan_timeout_occurred;	/* flag to indicate scan has timedout */
+	bool	scan_busy_occurred;	/* flag to indicate scan busy occurred */
+#ifdef BT_OVER_SDIO
+	bool	is_bt_recovery_required;
+#endif
+	bool	smmu_fault_occurred;	/* flag to indicate SMMU Fault */
+	bool	p2p_disc_busy_occurred;
+	bool	dongle_trap_during_wifi_onoff;	/* flag to indicate trap during wifi on/off */
+	bool	dsack_hc_due_to_isr_delay;	/* flag to indicate DSACK HC by ISR delay */
+	bool	dsack_hc_due_to_dpc_delay;	/* flag to indicate DSACK HC by DPC delay */
+/*
+ * Add any new variables to track Bus errors above
+ * this line. Also ensure that the variable is
+ * cleared from dhd_clear_bus_errors
+ */
+/* End - Variables to track Bus Errors */
+
+	int   hang_was_sent;
+	int   hang_was_pending;
+	int   rxcnt_timeout;		/* counter rxcnt timeout to send HANG */
+	int   txcnt_timeout;		/* counter txcnt timeout to send HANG */
+#ifdef BCMPCIE
+	int   d3ackcnt_timeout;		/* counter d3ack timeout to send HANG */
+#endif /* BCMPCIE */
+	bool hang_report;		/* enable hang report by default */
+	uint16 hang_reason;		/* reason codes for HANG event */
+#if defined(DHD_HANG_SEND_UP_TEST)
+	uint req_hang_type;
+#endif /* DHD_HANG_SEND_UP_TEST */
+#ifdef DHD_DETECT_CONSECUTIVE_MFG_HANG
+	uint hang_count;
+#endif /* DHD_DETECT_CONSECUTIVE_MFG_HANG */
+#ifdef WLTDLS
+	bool tdls_enable;
+#endif
+	struct reorder_info *reorder_bufs[WLHOST_REORDERDATA_MAXFLOWS];
+	#define WLC_IOCTL_MAXBUF_FWCAP	1024
+	char  fw_capabilities[WLC_IOCTL_MAXBUF_FWCAP];
+	#define DHD_IOCTL_MAXBUF_DHDCAP	1024
+	char  dhd_capabilities[DHD_IOCTL_MAXBUF_DHDCAP];
+	#define MAXSKBPEND 1024
+	void *skbbuf[MAXSKBPEND];
+	uint32 store_idx;
+	uint32 sent_idx;
+#ifdef DHDTCPACK_SUPPRESS
+	uint8 tcpack_sup_mode;		/* TCPACK suppress mode */
+	void *tcpack_sup_module;	/* TCPACK suppress module */
+	uint32 tcpack_sup_ratio;
+	uint32 tcpack_sup_delay;
+#endif /* DHDTCPACK_SUPPRESS */
+#if defined(ARP_OFFLOAD_SUPPORT)
+	uint32 arp_version;
+	bool hmac_updated;
+#endif
+#if defined(BCMSUP_4WAY_HANDSHAKE)
+	bool fw_4way_handshake;		/* Whether firmware will to do the 4way handshake. */
+#endif
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+	bool dhd_bug_on;
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+#ifdef CUSTOM_SET_CPUCORE
+	struct task_struct * current_dpc;
+	struct task_struct * current_rxf;
+	int chan_isvht80;
+#endif /* CUSTOM_SET_CPUCORE */
+	void    *sta_pool;          /* pre-allocated pool of sta objects */
+	void    *staid_allocator;   /* allocator of sta indexes */
+#ifdef PCIE_FULL_DONGLE
+	bool	flow_rings_inited;	/* set this flag after initializing flow rings */
+#endif /* PCIE_FULL_DONGLE */
+	void    *flowid_allocator;  /* unique flowid allocator */
+	void    *htput_flowid_allocator;  /* unique htput flowid allocator */
+	uint16	htput_client_flow_rings;  /* current number of htput client flowrings */
+	uint16	htput_flow_ring_start;	  /* start index of htput flow rings */
+	void	*flow_ring_table;   /* flow ring table, include prot and bus info */
+	void	*if_flow_lkup;      /* per interface flowid lkup hash table */
+	void    *flowid_lock;       /* per os lock for flowid info protection */
+	void    *flowring_list_lock;       /* per os lock for flowring list protection */
+	uint8	max_multi_client_flow_rings;
+	osl_atomic_t multi_client_flow_rings;
+	uint32  num_h2d_rings;		/* Max h2d rings including static and dynamic rings */
+	uint32  max_tx_flowid;		/* used to validate flowid */
+	cumm_ctr_t cumm_ctr;        /* cumm queue length placeholder  */
+	cumm_ctr_t l2cumm_ctr;      /* level 2 cumm queue length placeholder */
+	uint32 d2h_sync_mode;       /* D2H DMA completion sync mode */
+	uint8  flow_prio_map[NUMPRIO];
+	uint8	flow_prio_map_type;
+	char enable_log[MAX_EVENT];
+	bool dma_d2h_ring_upd_support;
+	bool dma_h2d_ring_upd_support;
+	bool dma_ring_upd_overwrite;	/* host overwrites support setting */
+
+	bool idma_enable;
+	uint idma_inited;
+
+	bool ifrm_enable;			/* implicit frm enable */
+	uint ifrm_inited;			/* implicit frm init */
+
+	bool dar_enable;		/* use DAR registers */
+	uint dar_inited;
+
+	bool fast_delete_ring_support;		/* fast delete ring supported */
+
+#ifdef DHD_WMF
+	bool wmf_ucast_igmp;
+#ifdef DHD_IGMP_UCQUERY
+	bool wmf_ucast_igmp_query;
+#endif
+#ifdef DHD_UCAST_UPNP
+	bool wmf_ucast_upnp;
+#endif
+#endif /* DHD_WMF */
+#if defined(BCM_ROUTER_DHD)
+	dhd_trf_mgmt_dwm_tbl_t dhd_tm_dwm_tbl;
+#endif /* BCM_ROUTER_DHD */
+#ifdef DHD_L2_FILTER
+	unsigned long l2_filter_cnt;	/* for L2_FILTER ARP table timeout */
+#endif /* DHD_L2_FILTER */
+#ifdef DHD_SSSR_DUMP
+	bool sssr_inited;
+	bool force_sssr_init;
+	bool sssr_dump_collected;	/* Flag to indicate sssr dump is collected */
+	sssr_reg_info_cmn_t *sssr_reg_info;
+	uint8 *sssr_mempool;
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	uint *sssr_d11_before[MAX_NUM_D11_CORES_WITH_SCAN];
+	uint *sssr_dig_buf_before;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	uint *sssr_d11_after[MAX_NUM_D11_CORES_WITH_SCAN];
+	bool sssr_d11_outofreset[MAX_NUM_D11_CORES_WITH_SCAN];
+	uint *sssr_dig_buf_after;
+	uint32 sssr_dump_mode;
+	bool collect_sssr;		/* Flag to indicate SSSR dump is required */
+	bool fis_triggered;
+#endif /* DHD_SSSR_DUMP */
+#ifdef DHD_SDTC_ETB_DUMP
+	etb_addr_info_t etb_addr_info;
+	uint8 *sdtc_etb_mempool;
+	uint sdtc_etb_dump_len;
+	bool sdtc_etb_inited;
+	bool collect_sdtc;		/* Flag to indicate SDTC dump is required */
+#endif /* DHD_SDTC_ETB_DUMP */
+	uint8 *soc_ram;
+	uint32 soc_ram_length;
+	uint32 memdump_type;
+#ifdef DHD_COREDUMP
+	uint8 *coredump_mem;
+	uint32 coredump_len;
+	char memdump_str[DHD_MEMDUMP_LONGSTR_LEN];
+#endif /* DHD_COREDUMP */
+#ifdef DHD_FW_COREDUMP
+	uint32 memdump_enabled;
+#ifdef DHD_DEBUG_UART
+	bool memdump_success;
+#endif	/* DHD_DEBUG_UART */
+#endif /* DHD_FW_COREDUMP */
+#ifdef PCIE_FULL_DONGLE
+#ifdef WLTDLS
+	tdls_peer_tbl_t peer_tbl;
+#endif /* WLTDLS */
+#if defined(LINUX) || defined(linux)
+	uint8 tx_in_progress;
+#endif /* LINUX || linux */
+#endif /* PCIE_FULL_DONGLE */
+#ifdef CACHE_FW_IMAGES
+	char	*cached_fw;
+	int	cached_fw_length;
+	char	*cached_nvram;
+	int	cached_nvram_length;
+	char	*cached_clm;
+	int	cached_clm_length;
+	char	*cached_txcap;
+	int	cached_txcap_length;
+#endif
+#ifdef KEEP_JP_REGREV
+/* XXX Needed by customer's request */
+	char vars_ccode[WLC_CNTRY_BUF_SZ];
+	uint vars_regrev;
+#endif /* KEEP_JP_REGREV */
+#ifdef WLTDLS
+	uint32 tdls_mode;
+#endif
+#ifdef GSCAN_SUPPORT
+	bool lazy_roam_enable;
+#endif
+#if defined(APF)
+	bool apf_set;
+#endif /* APF */
+	void *macdbg_info;
+#ifdef DHD_WET
+	void *wet_info;
+#endif
+	bool	h2d_phase_supported;
+	bool	force_dongletrap_on_bad_h2d_phase;
+	uint32	dongle_trap_data;
+	fw_download_status_t	fw_download_status;
+	trap_t	last_trap_info; /* trap info from the last trap */
+	uint8 rand_mac_oui[DOT11_OUI_LEN];
+#ifdef DHD_LOSSLESS_ROAMING
+	uint8 dequeue_prec_map;
+	uint8 prio_8021x;
+#endif
+	/* timesync link */
+	struct dhd_ts *ts;
+	bool	d2h_hostrdy_supported;
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	atomic_t block_bus;
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+#if defined(DBG_PKT_MON) || defined(DHD_PKT_LOGGING)
+	bool d11_tx_status;
+#endif /* DBG_PKT_MON || DHD_PKT_LOGGING */
+	uint16 ndo_version;	/* ND offload version supported */
+#ifdef NDO_CONFIG_SUPPORT
+	bool ndo_enable;		/* ND offload feature enable */
+	bool ndo_host_ip_overflow;	/* # of host ip addr exceed FW capacity */
+	uint32 ndo_max_host_ip;		/* # of host ip addr supported by FW */
+#endif /* NDO_CONFIG_SUPPORT */
+#if defined(DHD_LOG_DUMP)
+	/* buffer to hold 'dhd dump' data before dumping to file */
+	uint8 *concise_dbg_buf;
+	uint64 last_file_posn;
+	int logdump_periodic_flush;
+#ifdef EWP_ECNTRS_LOGGING
+	void *ecntr_dbg_ring;
+#endif
+#ifdef EWP_RTT_LOGGING
+	void *rtt_dbg_ring;
+#endif
+#ifdef EWP_BCM_TRACE
+	void *bcm_trace_dbg_ring;
+#endif
+#ifdef DNGL_EVENT_SUPPORT
+	uint8 health_chk_event_data[HEALTH_CHK_BUF_SIZE];
+#endif
+	void *logdump_cookie;
+#endif /* DHD_LOG_DUMP */
+	uint32 dhd_console_ms; /** interval for polling the dongle for console (log) messages */
+	bool ext_trap_data_supported;
+	uint32 *extended_trap_data;
+#ifdef DUMP_IOCTL_IOV_LIST
+	/* dump iovar list */
+	dll_t dump_iovlist_head;
+	uint8 dump_iovlist_len;
+#endif /* DUMP_IOCTL_IOV_LIST */
+#ifdef REPORT_FATAL_TIMEOUTS
+	timeout_info_t *timeout_info;
+	uint16 esync_id; /* used to track escans */
+	osl_atomic_t set_ssid_rcvd; /* to track if WLC_E_SET_SSID is received during join IOVAR */
+	bool secure_join; /* field to note that the join is secure or not */
+#endif /* REPORT_FATAL_TIMEOUTS */
+#ifdef CUSTOM_SET_ANTNPM
+	uint32 mimo_ant_set;
+#endif /* CUSTOM_SET_ANTNPM */
+#ifdef CUSTOM_SET_OCLOFF
+	bool ocl_off;
+#endif /* CUSTOM_SET_OCLOFF */
+#ifdef DHD_DEBUG
+	/* memwaste feature */
+	dll_t mw_list_head; /* memwaste list head */
+	uint32 mw_id; /* memwaste list unique id */
+#endif /* DHD_DEBUG */
+#ifdef WLTDLS
+	spinlock_t tdls_lock;
+#endif /* WLTDLS */
+	uint pcie_txs_metadata_enable;
+	uint wbtext_policy;	/* wbtext policy of dongle */
+	bool wbtext_support;	/* for product policy only */
+#ifdef PCIE_OOB
+	bool	d2h_no_oob_dw;
+#endif /* PCIE_OOB */
+#ifdef PCIE_INB_DW
+	bool	d2h_inband_dw;
+	enum dhd_bus_ds_state	ds_state;
+#endif /* PCIE_INB_DW */
+	tput_test_t tput_data;
+	uint64 tput_start_ts;
+	uint64 tput_stop_ts;
+	uint dhd_watchdog_ms_backup;
+	bool wl_event_enabled;
+	bool logtrace_pkt_sendup;
+#ifdef GDB_PROXY
+	/* True if firmware runs under gdb control (this may cause timeouts at any point) */
+	bool gdb_proxy_active;
+	/* True if deadman_to shall be forced to 0 */
+	bool gdb_proxy_nodeadman;
+	/* Counter incremented at each firmware stop/go transition. LSB (GDB_PROXY_STOP_MASK)
+	 * is set when firmware is stopped, clear when running
+	 */
+	uint32 gdb_proxy_stop_count;
+#endif /* GDB_PROXY */
+	int debug_dump_subcmd;
+	uint64 debug_dump_time_sec;
+	bool hscb_enable;
+	uint64 logset_prsrv_mask;
+#ifdef DHD_PKT_LOGGING
+	struct dhd_pktlog *pktlog;
+	char debug_dump_time_pktlog_str[DEBUG_DUMP_TIME_BUF_LEN];
+	bool pktlog_debug;
+#endif /* DHD_PKT_LOGGING */
+#ifdef EWP_EDL
+	bool dongle_edl_support;
+	dhd_dma_buf_t edl_ring_mem;
+#endif /* EWP_EDL */
+#if defined(LINUX) || defined(linux)
+	struct mutex ndev_op_sync;
+#endif /* defined (LINUX) || defined(linux) */
+	bool debug_buf_dest_support;
+	uint32 debug_buf_dest_stat[DEBUG_BUF_DEST_MAX];
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+	char *hang_info;
+	int hang_info_cnt;
+	char debug_dump_time_hang_str[DEBUG_DUMP_TIME_BUF_LEN];
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+	char debug_dump_time_str[DEBUG_DUMP_TIME_BUF_LEN];
+	void *event_log_filter;
+	uint tput_test_done;
+#if defined(LINUX) || defined(linux)
+	wait_queue_head_t tx_tput_test_wait;
+	wait_queue_head_t tx_completion_wait;
+#endif /* defined(LINUX) || defined(linux) */
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+#define DHD_H2D_LOG_TIME_STAMP_MATCH	(10000) /* 10 Seconds */
+	/*
+	 * Interval for updating the dongle console message time stamp with the Host (DHD)
+	 * time stamp
+	 */
+	uint32 dhd_rte_time_sync_ms;
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+	uint32 batch_tx_pkts_cmpl;
+	uint32 batch_tx_num_pkts;
+#ifdef DHD_DUMP_MNGR
+	struct _dhd_dump_file_manage *dump_file_manage;
+#endif /* DHD_DUMP_MNGR */
+	bool event_log_max_sets_queried;
+	uint32 event_log_max_sets;
+#ifdef DHD_STATUS_LOGGING
+	void *statlog;
+#endif /* DHD_STATUS_LOGGING */
+#ifdef DHD_MAP_PKTID_LOGGING
+	bool enable_pktid_log_dump;
+#endif /* DHD_MAP_PKTID_LOGGING */
+	bool extdtxs_in_txcpl;
+	bool hostrdy_after_init;
+	uint16 dhd_induce_error;
+	uint16 dhd_induce_bh_error;
+	int wlc_ver_major;
+	int wlc_ver_minor;
+#ifdef SUPPORT_SET_TID
+	uint8 tid_mode;
+	uint32 target_uid;
+	uint8 target_tid;
+#endif /* SUPPORT_SET_TID */
+#ifdef DHD_PKTDUMP_ROAM
+	void *pktcnts;
+#endif /* DHD_PKTDUMP_ROAM */
+	dhd_db7_info_t db7_trap;
+	bool fw_preinit;
+	bool ring_attached;
+	union {
+		wl_roam_stats_v1_t v1;
+	} roam_evt;
+	bool arpoe_enable;
+	bool arpol_configured;
+#ifdef DHD_TX_PROFILE
+	bool tx_profile_enab;
+	uint8 num_profiles;
+	dhd_tx_profile_protocol_t *protocol_filters;
+#endif /* defined(DHD_TX_PROFILE) */
+#ifdef DHD_MEM_STATS
+	void *mem_stats_lock;
+	uint64 txpath_mem;
+	uint64 rxpath_mem;
+#endif /* DHD_MEM_STATS */
+#ifdef DHD_LB_RXP
+	atomic_t lb_rxp_flow_ctrl;
+	uint32 lb_rxp_stop_thr;
+	uint32 lb_rxp_strt_thr;
+#endif /* DHD_LB_RXP */
+#ifdef DHD_LB_STATS
+	uint64 lb_rxp_stop_thr_hitcnt;
+	uint64 lb_rxp_strt_thr_hitcnt;
+	uint64 lb_rxp_napi_sched_cnt;
+	uint64 lb_rxp_napi_complete_cnt;
+	uint64 rx_dma_stall_hc_ignore_cnt;
+#endif /* DHD_LB_STATS */
+	bool check_trap_rot;
+	/* if FW supports host insertion of SFH LLC */
+	bool host_sfhllc_supported;
+#ifdef DHD_GRO_ENABLE_HOST_CTRL
+	bool permitted_gro;
+#endif /* DHD_GRO_ENABLE_HOST_CTRL */
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+	uint32 alert_reason;		/* reason codes for alert event */
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+#ifdef DBG_PKT_MON
+	bool aml_enable;	/* aml(assoc mgmt frame logger) enable flags */
+#endif /* DBG_PKT_MON */
+	bool fw_mode_changed;
+	bool do_chip_bighammer;
+	uint chip_bighammer_count;
+	bool rx_cpl_lat_capable; /* FW supports latency posting in Rx cpl ring */
+	rx_cpl_lat_info_t rxcpl_lat_info; /* Rx Cpl latency information */
+	tx_cpl_info_t txcpl_info; /* TX completion timestamp info */
+#ifdef SUPPORT_OTA_UPDATE
+	ota_update_info_t ota_update_info;
+#endif /* SUPPORT_OTA_UPDATE */
+#ifdef CSI_SUPPORT
+	struct mutex	 csi_lock;
+	struct list_head csi_list;
+	uint csi_count;
+	uint  csi_data_send_manner;
+#endif /* CSI_SUPPORT */
+	bool stop_in_progress;
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+	uint32 dhd_sar_mode;
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+
+	/* Pointer to Platform Layer */
+	void *plat_info;
+	uint32 plat_info_size;
+	bool htput_support;
+#ifdef FLOW_RING_PREALLOC
+	uint16 non_htput_total_flow_rings;
+#endif /* FLOW_RING_PREALLOC */
+	bool fw_lpm_support; /* fw supports LPM mode */
+	bool igmpo_enable; /* fw supports igmp offload */
+#ifdef DHD_SUPPORT_SPMI_MODE
+	uint dhd_spmi_mode;
+#endif /* DHD_SUPPORT_SPMI_MODE */
+	/* Flag to indicate H2D flowring phase nibble value support */
+	uint8 fr_phase_nibble;
+	bool mdring_capable; /* FW supports metadata ring cpl ring */
+	uint8 *mdring_info;  /* Saved metadta ring work items */
+	uint32 md_item_count; /* # of saved metadata work items */
+#if defined(DEVICE_TX_STUCK_DETECT) && defined(ASSOC_CHECK_SR)
+	/* is infra associated or not before suspend ? */
+	bool assoc_at_suspend;
+#endif /* DEVICE_TX_STUCK_DETECT && ASSOC_CHECK_SR */
+	uint32 p2p_disc_busy_cnt;
+	bool skip_memdump_map_read;
+#if defined(DHD_SI_WD_RESET)
+	bool si_wd;
+#endif
+#ifdef CSI_SUPPORT
+	struct list_head csi_list;
+	int csi_count;
+#endif /* CSI_SUPPORT */
+	char *clm_path;		/* module_param: path to clm vars file */
+	char *conf_path;		/* module_param: path to config vars file */
+	struct dhd_conf *conf;	/* Bus module handle */
+	void *adapter;			/* adapter information, interrupt, fw path etc. */
+	void *event_params;
+#ifdef WL_TIMER
+	void *timer_params;
+#endif /* WL_TIMER */
+#ifdef BCMDBUS
+	bool dhd_remove;
+#endif /* BCMDBUS */
+#ifdef WL_ESCAN
+	struct wl_escan_info *escan;
+#endif
+#if defined(WL_WIRELESS_EXT)
+	void *wext_info;
+#endif
+#ifdef WL_EXT_IAPSTA
+	void *iapsta_params;
+#endif
+	int hostsleep;
+#ifdef SENDPROB
+	bool recv_probereq;
+#endif
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	bool skip_dhd_stop;
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+#ifdef WL_EXT_GENL
+	void *zconf;
+#endif
+} dhd_pub_t;
+
+#if defined(__linux__)
+int dhd_wifi_platform_set_power(dhd_pub_t *pub, bool on);
+#else
+static INLINE int dhd_wifi_platform_set_power(dhd_pub_t *pub, bool on)  { return 0; }
+#endif /* __linux__ */
+
+#define MAX_WAKE_REASON_STATS	32u
+typedef struct {
+	uint rxwake;
+	uint rcwake;
+#ifdef DHD_WAKE_RX_STATUS
+	uint rx_bcast;
+	uint rx_arp;
+	uint rx_mcast;
+	uint rx_multi_ipv6;
+	uint rx_icmp;
+	uint rx_icmpv6;
+	uint rx_icmpv6_ra;
+	uint rx_icmpv6_na;
+	uint rx_icmpv6_ns;
+	uint rx_multi_ipv4;
+	uint rx_multi_other;
+	uint rx_ucast;
+#endif /* DHD_WAKE_RX_STATUS */
+#ifdef DHD_WAKE_EVENT_STATUS
+#ifdef CUSTOM_WAKE_REASON_STATS
+	int rc_event[MAX_WAKE_REASON_STATS];
+	int rc_event_idx;
+#else
+	uint rc_event[WLC_E_LAST];
+#endif /* CUSTOM_WAKE_REASON_STATS */
+#endif /* DHD_WAKE_EVENT_STATUS */
+} wake_counts_t;
+
+#if defined(PCIE_FULL_DONGLE)
+/*
+ * XXX: WARNING: dhd_wlfc.h also defines a dhd_pkttag_t
+ * making wlfc incompatible with PCIE_FULL DONGLE
+ */
+
+/* Packet Tag for PCIE Full Dongle DHD */
+typedef struct dhd_pkttag_fd {
+	uint16    flowid;   /* Flowring Id */
+	uint16    ifid;
+#ifdef DHD_SBN
+	uint8	  pkt_udr;
+	uint8	  pad;
+#endif /* DHD_SBN */
+#if defined(BCM_ROUTER_DHD) && defined(BCM_GMAC3)
+	uint16    dataoff;  /* start of packet */
+#endif /* BCM_ROUTER_DHD && BCM_GMAC3 */
+#ifndef DHD_PCIE_PKTID
+	uint16    dma_len;  /* pkt len for DMA_MAP/UNMAP */
+	dmaaddr_t pa;       /* physical address */
+	void      *dmah;    /* dma mapper handle */
+	void      *secdma; /* secure dma sec_cma_info handle */
+#endif /* !DHD_PCIE_PKTID */
+#if defined(TX_STATUS_LATENCY_STATS)
+	uint64	   q_time_us; /* time when tx pkt queued to flowring */
+#endif
+} dhd_pkttag_fd_t;
+
+/* Packet Tag for DHD PCIE Full Dongle */
+#define DHD_PKTTAG_FD(pkt)          ((dhd_pkttag_fd_t *)(PKTTAG(pkt)))
+
+#define DHD_PKT_GET_FLOWID(pkt)     ((DHD_PKTTAG_FD(pkt))->flowid)
+#define DHD_PKT_SET_FLOWID(pkt, pkt_flowid) \
+	DHD_PKTTAG_FD(pkt)->flowid = (uint16)(pkt_flowid)
+
+#define DHD_PKT_GET_DATAOFF(pkt)    ((DHD_PKTTAG_FD(pkt))->dataoff)
+#define DHD_PKT_SET_DATAOFF(pkt, pkt_dataoff) \
+	DHD_PKTTAG_FD(pkt)->dataoff = (uint16)(pkt_dataoff)
+
+#define DHD_PKT_GET_DMA_LEN(pkt)    ((DHD_PKTTAG_FD(pkt))->dma_len)
+#define DHD_PKT_SET_DMA_LEN(pkt, pkt_dma_len) \
+	DHD_PKTTAG_FD(pkt)->dma_len = (uint16)(pkt_dma_len)
+
+#define DHD_PKT_GET_PA(pkt)         ((DHD_PKTTAG_FD(pkt))->pa)
+#define DHD_PKT_SET_PA(pkt, pkt_pa) \
+	DHD_PKTTAG_FD(pkt)->pa = (dmaaddr_t)(pkt_pa)
+
+#define DHD_PKT_GET_DMAH(pkt)       ((DHD_PKTTAG_FD(pkt))->dmah)
+#define DHD_PKT_SET_DMAH(pkt, pkt_dmah) \
+	DHD_PKTTAG_FD(pkt)->dmah = (void *)(pkt_dmah)
+
+#define DHD_PKT_GET_SECDMA(pkt)    ((DHD_PKTTAG_FD(pkt))->secdma)
+#define DHD_PKT_SET_SECDMA(pkt, pkt_secdma) \
+	DHD_PKTTAG_FD(pkt)->secdma = (void *)(pkt_secdma)
+
+#if defined(TX_STATUS_LATENCY_STATS)
+#define DHD_PKT_GET_QTIME(pkt)    ((DHD_PKTTAG_FD(pkt))->q_time_us)
+#define DHD_PKT_SET_QTIME(pkt, pkt_q_time_us) \
+	DHD_PKTTAG_FD(pkt)->q_time_us = (uint64)(pkt_q_time_us)
+#endif
+#endif /* PCIE_FULL_DONGLE */
+
+#if defined(BCMWDF)
+typedef struct {
+	dhd_pub_t *dhd_pub;
+} dhd_workitem_context_t;
+
+WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(dhd_workitem_context_t, dhd_get_dhd_workitem_context)
+#endif /* (BCMWDF)  */
+
+#if defined(LINUX) || defined(linux)
+#if defined(CONFIG_PM_SLEEP)
+
+	#define DHD_PM_RESUME_WAIT_INIT(a) DECLARE_WAIT_QUEUE_HEAD(a);
+	#define _DHD_PM_RESUME_WAIT(a, b) do {\
+			int retry = 0; \
+			SMP_RD_BARRIER_DEPENDS(); \
+			while (dhd_mmc_suspend && retry++ != b) { \
+				SMP_RD_BARRIER_DEPENDS(); \
+				wait_event_interruptible_timeout(a, !dhd_mmc_suspend, 1); \
+			} \
+		} 	while (0)
+	#define DHD_PM_RESUME_WAIT(a) 		_DHD_PM_RESUME_WAIT(a, 200)
+	#define DHD_PM_RESUME_WAIT_FOREVER(a) 	_DHD_PM_RESUME_WAIT(a, ~0)
+	#define DHD_PM_RESUME_RETURN_ERROR(a)   do { \
+			if (dhd_mmc_suspend) { \
+				printf("%s[%d]: mmc is still in suspend state!!!\n", \
+					__FUNCTION__, __LINE__); \
+				return a; \
+			} \
+		} while (0)
+	#define DHD_PM_RESUME_RETURN		do { if (dhd_mmc_suspend) return; } while (0)
+
+	#define DHD_SPINWAIT_SLEEP_INIT(a) DECLARE_WAIT_QUEUE_HEAD(a);
+	#define SPINWAIT_SLEEP(a, exp, us) do { \
+		uint countdown = (us) + 9999; \
+		while ((exp) && (countdown >= 10000)) { \
+			wait_event_interruptible_timeout(a, FALSE, 1); \
+			countdown -= 10000; \
+		} \
+	} while (0)
+
+#else
+
+	#define DHD_PM_RESUME_WAIT_INIT(a)
+	#define DHD_PM_RESUME_WAIT(a)
+	#define DHD_PM_RESUME_WAIT_FOREVER(a)
+	#define DHD_PM_RESUME_RETURN_ERROR(a)
+	#define DHD_PM_RESUME_RETURN
+
+	#define DHD_SPINWAIT_SLEEP_INIT(a)
+	#define SPINWAIT_SLEEP(a, exp, us)  do { \
+		uint countdown = (us) + 9; \
+		while ((exp) && (countdown >= 10)) { \
+			OSL_DELAY(10);  \
+			countdown -= 10;  \
+		} \
+	} while (0)
+
+#endif /* CONFIG_PM_SLEEP */
+#else
+	#define DHD_SPINWAIT_SLEEP_INIT(a)
+	#define SPINWAIT_SLEEP(a, exp, us)  do { \
+		uint countdown = (us) + 9; \
+		while ((exp) && (countdown >= 10)) { \
+			OSL_DELAY(10);  \
+			countdown -= 10;  \
+		} \
+	} while (0)
+#endif /* defined (LINUX) || defined(linux) */
+
+#define DHD_IF_VIF	0x01	/* Virtual IF (Hidden from user) */
+
+#ifdef PNO_SUPPORT
+int dhd_pno_clean(dhd_pub_t *dhd);
+#endif /* PNO_SUPPORT */
+
+/*
+ *  Wake locks are an Android power management concept. They are used by applications and services
+ *  to request CPU resources.
+ */
+#if defined(linux) && defined(OEM_ANDROID)
+extern int dhd_os_wake_lock(dhd_pub_t *pub);
+extern int dhd_os_wake_unlock(dhd_pub_t *pub);
+extern int dhd_os_wake_lock_waive(dhd_pub_t *pub);
+extern int dhd_os_wake_lock_restore(dhd_pub_t *pub);
+extern void dhd_event_wake_lock(dhd_pub_t *pub);
+extern void dhd_event_wake_unlock(dhd_pub_t *pub);
+extern void dhd_pm_wake_lock_timeout(dhd_pub_t *pub, int val);
+extern void dhd_pm_wake_unlock(dhd_pub_t *pub);
+extern void dhd_txfl_wake_lock_timeout(dhd_pub_t *pub, int val);
+extern void dhd_txfl_wake_unlock(dhd_pub_t *pub);
+extern void dhd_nan_wake_lock_timeout(dhd_pub_t *pub, int val);
+extern void dhd_nan_wake_unlock(dhd_pub_t *pub);
+extern int dhd_os_wake_lock_timeout(dhd_pub_t *pub);
+extern int dhd_os_wake_lock_rx_timeout_enable(dhd_pub_t *pub, int val);
+extern int dhd_os_wake_lock_ctrl_timeout_enable(dhd_pub_t *pub, int val);
+extern int dhd_os_wake_lock_ctrl_timeout_cancel(dhd_pub_t *pub);
+extern int dhd_os_wd_wake_lock(dhd_pub_t *pub);
+extern int dhd_os_wd_wake_unlock(dhd_pub_t *pub);
+extern void dhd_os_wake_lock_init(struct dhd_info *dhd);
+extern void dhd_os_wake_lock_destroy(struct dhd_info *dhd);
+#ifdef DHD_USE_SCAN_WAKELOCK
+extern void dhd_os_scan_wake_lock_timeout(dhd_pub_t *pub, int val);
+extern void dhd_os_scan_wake_unlock(dhd_pub_t *pub);
+#endif /* BCMPCIE_SCAN_WAKELOCK */
+
+#ifdef WLEASYMESH
+extern int dhd_get_1905_almac(dhd_pub_t *dhdp, uint8 ifidx, uint8* ea, bool mcast);
+extern int dhd_set_1905_almac(dhd_pub_t *dhdp, uint8 ifidx, uint8* ea, bool mcast);
+#endif /* WLEASYMESH */
+
+inline static void MUTEX_LOCK_SOFTAP_SET_INIT(dhd_pub_t * dhdp)
+{
+#if defined(OEM_ANDROID)
+	mutex_init(&dhdp->wl_softap_lock);
+#endif /* OEM_ANDROID */
+}
+
+inline static void MUTEX_LOCK_SOFTAP_SET(dhd_pub_t * dhdp)
+{
+#if defined(OEM_ANDROID)
+	mutex_lock(&dhdp->wl_softap_lock);
+#endif /* OEM_ANDROID */
+}
+
+inline static void MUTEX_UNLOCK_SOFTAP_SET(dhd_pub_t * dhdp)
+{
+#if defined(OEM_ANDROID)
+	mutex_unlock(&dhdp->wl_softap_lock);
+#endif /* OEM_ANDROID */
+}
+
+#ifdef DHD_DEBUG_WAKE_LOCK
+extern int dhd_wakelock_counter_get(dhd_pub_t *pub);
+extern int dhd_wakelock_wd_counter_get(dhd_pub_t *pub);
+
+#define DHD_OS_WAKE_LOCK(pub) \
+	do { \
+		printf("call wake_lock: %s %d (%d)\n", \
+			__FUNCTION__, __LINE__, dhd_wakelock_counter_get(pub)); \
+		dhd_os_wake_lock(pub); \
+	} while (0)
+#define DHD_OS_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call wake_unlock: %s %d (%d)\n", \
+			__FUNCTION__, __LINE__, dhd_wakelock_counter_get(pub)); \
+		dhd_os_wake_unlock(pub); \
+	} while (0)
+#define DHD_EVENT_WAKE_LOCK(pub) \
+	do { \
+		printf("call event wake_lock: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_event_wake_lock(pub); \
+	} while (0)
+#define DHD_EVENT_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call event wake_unlock: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_event_wake_unlock(pub); \
+	} while (0)
+#define DHD_PM_WAKE_LOCK_TIMEOUT(pub, val) \
+	do { \
+		printf("call pm_wake_timeout enable: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_pm_wake_lock_timeout(pub, val); \
+	} while (0)
+#define DHD_PM_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call pm_wake unlock: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_pm_wake_unlock(pub); \
+	} while (0)
+#define DHD_TXFL_WAKE_LOCK_TIMEOUT(pub, val) \
+	do { \
+		printf("call txfl_wake_timeout enable: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_txfl_wake_lock_timeout(pub, val); \
+	} while (0)
+#define DHD_TXFL_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call txfl_wake unlock: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_txfl_wake_unlock(pub); \
+	} while (0)
+#define DHD_NAN_WAKE_LOCK_TIMEOUT(pub, val) \
+	do { \
+		printf("call nan_wake_timeout enable: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_nan_wake_lock_timeout(pub, val); \
+	} while (0)
+#define DHD_NAN_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call nan_wake unlock: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_nan_wake_unlock(pub); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_TIMEOUT(pub) \
+	do { \
+		printf("call wake_lock_timeout: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_timeout(pub); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_RX_TIMEOUT_ENABLE(pub, val) \
+	do { \
+		printf("call dhd_wake_lock_rx_timeout_enable[%d]: %s %d\n", \
+			val, __FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_rx_timeout_enable(pub, val); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_ENABLE(pub, val) \
+	do { \
+		printf("call dhd_wake_lock_ctrl_timeout_enable[%d]: %s %d\n", \
+			val, __FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_ctrl_timeout_enable(pub, val); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_CANCEL(pub) \
+	do { \
+		printf("call dhd_wake_lock_ctrl_timeout_cancel: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_ctrl_timeout_cancel(pub); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_WAIVE(pub) \
+	do { \
+		printf("call dhd_wake_lock_waive: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_waive(pub); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_RESTORE(pub) \
+	do { \
+		printf("call dhd_wake_lock_restore: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_restore(pub); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_INIT(dhd) \
+	do { \
+		printf("call dhd_wake_lock_init: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_init(dhd); \
+	} while (0)
+#define DHD_OS_WAKE_LOCK_DESTROY(dhd) \
+	do { \
+		printf("call dhd_wake_dhd_lock_destroy: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_wake_lock_destroy(dhd); \
+	} while (0)
+#define DHD_OS_WD_WAKE_LOCK(pub) \
+	do { \
+		printf("call dhd_os_WD_wake_lock: %s %d (%d)\n", \
+			__FUNCTION__, __LINE__, dhd_wakelock_wd_counter_get(pub)); \
+		dhd_os_wd_wake_lock(pub); \
+	} while (0)
+#define DHD_OS_WD_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call dhd_os_WD_wake_unlock: %s %d (%d)\n", \
+			__FUNCTION__, __LINE__, dhd_wakelock_wd_counter_get(pub)); \
+		dhd_os_wd_wake_unlock(pub); \
+	} while (0)
+#else
+#define DHD_OS_WAKE_LOCK(pub)			dhd_os_wake_lock(pub)
+#define DHD_OS_WAKE_UNLOCK(pub)		dhd_os_wake_unlock(pub)
+#define DHD_EVENT_WAKE_LOCK(pub)			dhd_event_wake_lock(pub)
+#define DHD_EVENT_WAKE_UNLOCK(pub)		dhd_event_wake_unlock(pub)
+#define DHD_PM_WAKE_LOCK_TIMEOUT(pub, val)  dhd_pm_wake_lock_timeout(pub, val)
+#define DHD_PM_WAKE_UNLOCK(pub) 			dhd_pm_wake_unlock(pub)
+#define DHD_TXFL_WAKE_LOCK_TIMEOUT(pub, val)	dhd_txfl_wake_lock_timeout(pub, val)
+#define DHD_TXFL_WAKE_UNLOCK(pub) 			dhd_txfl_wake_unlock(pub)
+#define DHD_NAN_WAKE_LOCK_TIMEOUT(pub, val)	dhd_nan_wake_lock_timeout(pub, val)
+#define DHD_NAN_WAKE_UNLOCK(pub)		dhd_nan_wake_unlock(pub)
+#define DHD_OS_WAKE_LOCK_TIMEOUT(pub)		dhd_os_wake_lock_timeout(pub)
+#define DHD_OS_WAKE_LOCK_RX_TIMEOUT_ENABLE(pub, val) \
+	dhd_os_wake_lock_rx_timeout_enable(pub, val)
+#define DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_ENABLE(pub, val) \
+	dhd_os_wake_lock_ctrl_timeout_enable(pub, val)
+#define DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_CANCEL(pub) \
+	dhd_os_wake_lock_ctrl_timeout_cancel(pub)
+#define DHD_OS_WAKE_LOCK_WAIVE(pub)			dhd_os_wake_lock_waive(pub)
+#define DHD_OS_WAKE_LOCK_RESTORE(pub)		dhd_os_wake_lock_restore(pub)
+#define DHD_OS_WAKE_LOCK_INIT(dhd)		dhd_os_wake_lock_init(dhd);
+#define DHD_OS_WAKE_LOCK_DESTROY(dhd)		dhd_os_wake_lock_destroy(dhd);
+#define DHD_OS_WD_WAKE_LOCK(pub)		dhd_os_wd_wake_lock(pub)
+#define DHD_OS_WD_WAKE_UNLOCK(pub)		dhd_os_wd_wake_unlock(pub)
+#endif /* DHD_DEBUG_WAKE_LOCK */
+
+#ifdef DHD_USE_SCAN_WAKELOCK
+#ifdef DHD_DEBUG_SCAN_WAKELOCK
+#define DHD_OS_SCAN_WAKE_LOCK_TIMEOUT(pub, val) \
+	do { \
+		printf("call wake_lock_scan: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_scan_wake_lock_timeout(pub, val); \
+	} while (0)
+#define DHD_OS_SCAN_WAKE_UNLOCK(pub) \
+	do { \
+		printf("call wake_unlock_scan: %s %d\n", \
+			__FUNCTION__, __LINE__); \
+		dhd_os_scan_wake_unlock(pub); \
+	} while (0)
+#else
+#define DHD_OS_SCAN_WAKE_LOCK_TIMEOUT(pub, val)		dhd_os_scan_wake_lock_timeout(pub, val)
+#define DHD_OS_SCAN_WAKE_UNLOCK(pub)			dhd_os_scan_wake_unlock(pub)
+#endif /* DHD_DEBUG_SCAN_WAKELOCK */
+#else
+#define DHD_OS_SCAN_WAKE_LOCK_TIMEOUT(pub, val)
+#define DHD_OS_SCAN_WAKE_UNLOCK(pub)
+#endif /* DHD_USE_SCAN_WAKELOCK */
+
+#else
+
+/* Wake lock are used in Android only (until the Linux community accepts it) */
+#define DHD_OS_WAKE_LOCK(pub)
+#define DHD_OS_WAKE_UNLOCK(pub)
+#define DHD_EVENT_WAKE_LOCK(pub)
+#define DHD_EVENT_WAKE_UNLOCK(pub)
+#define DHD_PM_WAKE_LOCK_TIMEOUT(pub, val)
+#define DHD_PM_WAKE_UNLOCK(pub)
+#define DHD_TXFL_WAKE_LOCK_TIMEOUT(pub, val)
+#define DHD_TXFL_WAKE_UNLOCK(pub)
+#define DHD_NAN_WAKE_LOCK_TIMEOUT(pub, val)
+#define DHD_NAN_WAKE_UNLOCK(pub)
+#define DHD_OS_WD_WAKE_LOCK(pub)
+#define DHD_OS_WD_WAKE_UNLOCK(pub)
+#define DHD_OS_WAKE_LOCK_TIMEOUT(pub)
+#define DHD_OS_WAKE_LOCK_RX_TIMEOUT_ENABLE(pub, val)	UNUSED_PARAMETER(val)
+#define DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_ENABLE(pub, val)	UNUSED_PARAMETER(val)
+#define DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_CANCEL(pub, val)
+#define DHD_OS_WAKE_LOCK_WAIVE(pub)
+#define DHD_OS_WAKE_LOCK_RESTORE(pub)
+#define DHD_OS_SCAN_WAKE_LOCK_TIMEOUT(pub, val)
+#define DHD_OS_SCAN_WAKE_UNLOCK(pub)
+#define DHD_OS_WAKE_LOCK_INIT(dhd)
+#define DHD_OS_WAKE_LOCK_DESTROY(dhd)
+
+#endif /* #defined(linux) && defined(OEM_ANDROID) */
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+#define OOB_WAKE_LOCK_TIMEOUT 500
+extern void dhd_os_oob_irq_wake_lock_timeout(dhd_pub_t *pub, int val);
+extern void dhd_os_oob_irq_wake_unlock(dhd_pub_t *pub);
+
+#define DHD_OS_OOB_IRQ_WAKE_LOCK_TIMEOUT(pub, val)	dhd_os_oob_irq_wake_lock_timeout(pub, val)
+#define DHD_OS_OOB_IRQ_WAKE_UNLOCK(pub)			dhd_os_oob_irq_wake_unlock(pub)
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+/* The DHD_PACKET_TIMEOUT_MS & MAX_TX_TIMEOUT are used for wake_lock timeout
+ * to prevent the system from entering suspend during TX/RX frame processing.
+ * It can be adjusted depending on the host platform.
+ */
+#define DHD_MONITOR_TIMEOUT_MS	1000
+#ifndef DHD_PACKET_TIMEOUT_MS
+#define DHD_PACKET_TIMEOUT_MS	100
+#endif
+#define DHD_HANDSHAKE_TIMEOUT_MS	1000
+#define DHD_EVENT_TIMEOUT_MS	1500
+#define SCAN_WAKE_LOCK_TIMEOUT	10000
+#ifndef MAX_TX_TIMEOUT
+#define MAX_TX_TIMEOUT			100
+#endif
+
+/* Enum for IOCTL recieved status */
+typedef enum dhd_ioctl_recieved_status
+{
+	IOCTL_WAIT = 0,
+	IOCTL_RETURN_ON_SUCCESS,
+	IOCTL_RETURN_ON_TRAP,
+	IOCTL_RETURN_ON_BUS_STOP,
+	IOCTL_RETURN_ON_ERROR
+} dhd_ioctl_recieved_status_t;
+
+/* interface operations (register, remove) should be atomic, use this lock to prevent race
+ * condition among wifi on/off and interface operation functions
+ */
+#if defined(LINUX)
+void dhd_net_if_lock(struct net_device *dev);
+void dhd_net_if_unlock(struct net_device *dev);
+bool dhd_net_if_locked(struct net_device *dev);
+#endif /* LINUX */
+
+#if defined(LINUX) || defined(linux)
+#if defined(MULTIPLE_SUPPLICANT)
+extern void wl_android_post_init(void); // terence 20120530: fix critical section in dhd_open and dhdsdio_probe
+#endif /* MULTIPLE_SUPPLICANT */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)) && defined(MULTIPLE_SUPPLICANT)
+extern struct mutex _dhd_mutex_lock_;
+#define DHD_MUTEX_IS_LOCK_RETURN() \
+	if (mutex_is_locked(&_dhd_mutex_lock_) != 0) { \
+		printf("%s : probe is already running! return.\n", __FUNCTION__); \
+		return -EBUSY;; \
+	}
+#define DHD_MUTEX_LOCK() \
+	do { \
+		if (mutex_is_locked(&_dhd_mutex_lock_) == 0) { \
+			DHD_INFO(("%s : no mutex held\n", __FUNCTION__)); \
+		} else { \
+			DHD_INFO(("%s : mutex is locked!. wait for unlocking\n", __FUNCTION__)); \
+		} \
+		mutex_lock(&_dhd_mutex_lock_); \
+		DHD_INFO(("%s : set mutex lock\n", __FUNCTION__)); \
+	} while (0)
+#define DHD_MUTEX_UNLOCK() \
+	do { \
+		DHD_INFO(("%s : mutex is released.\n", __FUNCTION__)); \
+		mutex_unlock(&_dhd_mutex_lock_); \
+	} while (0)
+#else
+#define DHD_MUTEX_IS_LOCK_RETURN(a)	do {} while (0)
+#define DHD_MUTEX_LOCK(a)	do {} while (0)
+#define DHD_MUTEX_UNLOCK(a)	do {} while (0)
+#endif
+#endif /* defined (LINUX) || defined(linux) */
+
+typedef enum dhd_attach_states
+{
+	DHD_ATTACH_STATE_INIT = 0x0,
+	DHD_ATTACH_STATE_NET_ALLOC = 0x1,
+	DHD_ATTACH_STATE_DHD_ALLOC = 0x2,
+	DHD_ATTACH_STATE_ADD_IF = 0x4,
+	DHD_ATTACH_STATE_PROT_ATTACH = 0x8,
+	DHD_ATTACH_STATE_WL_ATTACH = 0x10,
+	DHD_ATTACH_STATE_THREADS_CREATED = 0x20,
+	DHD_ATTACH_STATE_WAKELOCKS_INIT = 0x40,
+	DHD_ATTACH_STATE_CFG80211 = 0x80,
+	DHD_ATTACH_STATE_EARLYSUSPEND_DONE = 0x100,
+	DHD_ATTACH_TIMESYNC_ATTACH_DONE = 0x200,
+	DHD_ATTACH_LOGTRACE_INIT = 0x400,
+	DHD_ATTACH_STATE_LB_ATTACH_DONE = 0x800,
+	DHD_ATTACH_STATE_DONE = 0x1000
+} dhd_attach_states_t;
+
+/* Value -1 means we are unsuccessful in creating the kthread. */
+#define DHD_PID_KT_INVALID		-1
+/* Value -2 means we are unsuccessful in both creating the kthread and tasklet */
+#define DHD_PID_KT_TL_INVALID		-2
+
+/* default reporting period */
+#define ECOUNTERS_DEFAULT_PERIOD	0
+
+/* default number of reports. '0' indicates forever */
+#define ECOUNTERS_NUM_REPORTS		0
+
+typedef struct ecounters_cfg {
+	uint16 type;
+	uint16 if_slice_idx;
+	uint16 stats_rep;
+} ecounters_cfg_t;
+
+typedef struct event_ecounters_cfg {
+	uint16 event_id;
+	uint16 type;
+	uint16 if_slice_idx;
+	uint16 stats_rep;
+} event_ecounters_cfg_t;
+
+typedef struct ecountersv2_xtlv_list_elt {
+	/* Not quite the exact bcm_xtlv_t type as data could be pointing to other pieces in
+	 * memory at the time of parsing arguments.
+	 */
+	uint16 id;
+	uint16 len;
+	uint8 *data;
+	struct ecountersv2_xtlv_list_elt *next;
+} ecountersv2_xtlv_list_elt_t;
+
+typedef struct ecountersv2_processed_xtlv_list_elt {
+	uint8 *data;
+	struct ecountersv2_processed_xtlv_list_elt *next;
+} ecountersv2_processed_xtlv_list_elt;
+
+/*
+ * Exported from dhd OS modules (dhd_linux/dhd_ndis)
+ */
+
+/* Indication from bus module regarding presence/insertion of dongle.
+ * Return dhd_pub_t pointer, used as handle to OS module in later calls.
+ * Returned structure should have bus and prot pointers filled in.
+ * bus_hdrlen specifies required headroom for bus module header.
+ */
+extern dhd_pub_t *dhd_attach(osl_t *osh, struct dhd_bus *bus, uint bus_hdrlen
+#ifdef BCMDBUS
+	, void *adapter
+#endif
+);
+extern int dhd_attach_net(dhd_pub_t *dhdp, bool need_rtnl_lock);
+#if defined(WLP2P) && defined(WL_CFG80211)
+/* To allow attach/detach calls corresponding to p2p0 interface  */
+extern int dhd_attach_p2p(dhd_pub_t *);
+extern int dhd_detach_p2p(dhd_pub_t *);
+#endif /* WLP2P && WL_CFG80211 */
+extern int dhd_register_if(dhd_pub_t *dhdp, int idx, bool need_rtnl_lock);
+
+/* Indication from bus module regarding removal/absence of dongle */
+extern void dhd_detach(dhd_pub_t *dhdp);
+extern void dhd_free(dhd_pub_t *dhdp);
+extern void dhd_clear(dhd_pub_t *dhdp);
+
+/* Indication from bus module to change flow-control state */
+extern void dhd_txflowcontrol(dhd_pub_t *dhdp, int ifidx, bool on);
+
+#ifdef BCMDONGLEHOST
+/* Store the status of a connection attempt for later retrieval by an iovar */
+extern void dhd_store_conn_status(uint32 event, uint32 status, uint32 reason);
+#endif /* BCMDONGLEHOST */
+
+extern bool dhd_prec_enq(dhd_pub_t *dhdp, struct pktq *q, void *pkt, int prec);
+
+extern void dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *rxp, int numpkt, uint8 chan);
+
+/* Return pointer to interface name */
+extern char *dhd_ifname(dhd_pub_t *dhdp, int idx);
+
+#ifdef DHD_UCODE_DOWNLOAD
+/* Returns the ucode path */
+extern char *dhd_get_ucode_path(dhd_pub_t *dhdp);
+#endif /* DHD_UCODE_DOWNLOAD */
+
+/* Request scheduling of the bus dpc */
+extern void dhd_sched_dpc(dhd_pub_t *dhdp);
+
+/* Notify tx completion */
+extern void dhd_txcomplete(dhd_pub_t *dhdp, void *txp, bool success);
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+extern void dhd_eap_txcomplete(dhd_pub_t *dhdp, void *txp, bool success, int ifidx);
+extern void dhd_cleanup_m4_state_work(dhd_pub_t *dhdp, int ifidx);
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+extern void dhd_bus_wakeup_work(dhd_pub_t *dhdp);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#define WIFI_FEATURE_INFRA              0x0001      /* Basic infrastructure mode        */
+#define WIFI_FEATURE_INFRA_5G           0x0002      /* Support for 5 GHz Band           */
+#define WIFI_FEATURE_HOTSPOT            0x0004      /* Support for GAS/ANQP             */
+#define WIFI_FEATURE_P2P                0x0008      /* Wifi-Direct                      */
+#define WIFI_FEATURE_SOFT_AP            0x0010      /* Soft AP                          */
+#define WIFI_FEATURE_GSCAN              0x0020      /* Google-Scan APIs                 */
+#define WIFI_FEATURE_NAN                0x0040      /* Neighbor Awareness Networking    */
+#define WIFI_FEATURE_D2D_RTT            0x0080      /* Device-to-device RTT             */
+#define WIFI_FEATURE_D2AP_RTT           0x0100      /* Device-to-AP RTT                 */
+#define WIFI_FEATURE_BATCH_SCAN         0x0200      /* Batched Scan (legacy)            */
+#define WIFI_FEATURE_PNO                0x0400      /* Preferred network offload        */
+#define WIFI_FEATURE_ADDITIONAL_STA     0x0800      /* Support for two STAs             */
+#define WIFI_FEATURE_TDLS               0x1000      /* Tunnel directed link setup       */
+#define WIFI_FEATURE_TDLS_OFFCHANNEL    0x2000      /* Support for TDLS off channel     */
+#define WIFI_FEATURE_EPR                0x4000      /* Enhanced power reporting         */
+#define WIFI_FEATURE_AP_STA             0x8000      /* Support for AP STA Concurrency   */
+#define WIFI_FEATURE_LINKSTAT           0x10000     /* Support for Linkstats            */
+#define WIFI_FEATURE_LOGGER             0x20000     /* WiFi Logger			*/
+#define WIFI_FEATURE_HAL_EPNO           0x40000	    /* WiFi PNO enhanced                */
+#define WIFI_FEATURE_RSSI_MONITOR       0x80000     /* RSSI Monitor                     */
+#define WIFI_FEATURE_MKEEP_ALIVE        0x100000    /* WiFi mkeep_alive			*/
+#define WIFI_FEATURE_CONFIG_NDO         0x200000    /* ND offload configure             */
+#define WIFI_FEATURE_TX_TRANSMIT_POWER  0x400000    /* Capture Tx transmit power levels */
+#define WIFI_FEATURE_CONTROL_ROAMING    0x800000    /* Enable/Disable firmware roaming  */
+#define WIFI_FEATURE_FILTER_IE          0x1000000   /* Probe req ie filter              */
+#define WIFI_FEATURE_SCAN_RAND          0x2000000   /* MAC & Prb SN randomization       */
+#define WIFI_FEATURE_SET_TX_POWER_LIMIT 0x4000000   /* Support Tx Power Limit setting   */
+#define WIFI_FEATURE_USE_BODY_HEAD_SAR  0x8000000   /* Support Body/Head Proximity SAR  */
+#define WIFI_FEATURE_SET_LATENCY_MODE   0x40000000  /* Support Latency mode setting     */
+#define WIFI_FEATURE_P2P_RAND_MAC       0x80000000  /* Support P2P MAC randomization    */
+#define WIFI_FEATURE_INVALID            0xFFFFFFFF  /* Invalid Feature                  */
+
+#define MAX_FEATURE_SET_CONCURRRENT_GROUPS  3
+
+#if defined(linux) || defined(LINUX) || defined(OEM_ANDROID)
+extern int dhd_dev_indoor_cfg(struct net_device *dev, u8 enable);
+extern int dhd_dev_get_feature_set(struct net_device *dev);
+extern int dhd_dev_get_feature_set_matrix(struct net_device *dev, int num);
+extern int dhd_dev_cfg_rand_mac_oui(struct net_device *dev, uint8 *oui);
+extern int dhd_update_rand_mac_addr(dhd_pub_t *dhd);
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+extern int dhd_dev_set_nodfs(struct net_device *dev, uint nodfs);
+#endif /* CUSTOM_FORCE_NODFS_FLAG */
+#ifdef NDO_CONFIG_SUPPORT
+#ifndef NDO_MAX_HOST_IP_ENTRIES
+#define NDO_MAX_HOST_IP_ENTRIES	10
+#endif /* NDO_MAX_HOST_IP_ENTRIES */
+
+extern int dhd_dev_ndo_cfg(struct net_device *dev, u8 enable);
+extern int dhd_dev_ndo_update_inet6addr(struct net_device * dev);
+#endif /* NDO_CONFIG_SUPPORT */
+#endif /* (linux) || (LINUX) || (OEM_ANDROID) */
+extern int dhd_set_rand_mac_oui(dhd_pub_t *dhd);
+#ifdef GSCAN_SUPPORT
+extern int dhd_dev_set_lazy_roam_cfg(struct net_device *dev,
+             wlc_roam_exp_params_t *roam_param);
+extern int dhd_dev_lazy_roam_enable(struct net_device *dev, uint32 enable);
+extern int dhd_dev_set_lazy_roam_bssid_pref(struct net_device *dev,
+       wl_bssid_pref_cfg_t *bssid_pref, uint32 flush);
+#endif /* GSCAN_SUPPORT */
+#if defined(GSCAN_SUPPORT) || defined(ROAMEXP_SUPPORT)
+extern int dhd_dev_set_blacklist_bssid(struct net_device *dev, maclist_t *blacklist,
+    uint32 len, uint32 flush);
+extern int dhd_dev_set_whitelist_ssid(struct net_device *dev, wl_ssid_whitelist_t *whitelist,
+    uint32 len, uint32 flush);
+#endif /* GSCAN_SUPPORT || ROAMEXP_SUPPORT */
+
+/* OS independent layer functions */
+extern void dhd_os_dhdiovar_lock(dhd_pub_t *pub);
+extern void dhd_os_dhdiovar_unlock(dhd_pub_t *pub);
+void dhd_os_logdump_lock(dhd_pub_t *pub);
+void dhd_os_logdump_unlock(dhd_pub_t *pub);
+extern int dhd_os_proto_block(dhd_pub_t * pub);
+extern int dhd_os_proto_unblock(dhd_pub_t * pub);
+extern int dhd_os_ioctl_resp_wait(dhd_pub_t * pub, uint * condition);
+extern int dhd_os_ioctl_resp_wake(dhd_pub_t * pub);
+extern unsigned int dhd_os_get_ioctl_resp_timeout(void);
+extern void dhd_os_set_ioctl_resp_timeout(unsigned int timeout_msec);
+extern void dhd_os_ioctl_resp_lock(dhd_pub_t * pub);
+extern void dhd_os_ioctl_resp_unlock(dhd_pub_t * pub);
+extern int dhd_os_read_file(void *file, char *buf, uint32 size);
+extern int dhd_os_seek_file(void *file, int64 offset);
+#ifdef PCIE_FULL_DONGLE
+extern void dhd_wakeup_ioctl_event(dhd_pub_t *pub, dhd_ioctl_recieved_status_t reason);
+#else
+static INLINE void dhd_wakeup_ioctl_event(dhd_pub_t *pub, dhd_ioctl_recieved_status_t reason)
+{ printf("%s is NOT implemented for SDIO", __FUNCTION__); return; }
+#endif
+#ifdef SHOW_LOGTRACE
+/* Bound and delay are fine tuned after several experiments and these
+ * are the best case values to handle bombarding of console logs.
+ */
+#define DHD_EVENT_LOGTRACE_BOUND 10u
+#define DHD_EVENT_LOGTRACE_RESCHEDULE_DELAY_MS 10u
+#endif /* SHOW_LOGTRACE */
+int dhd_os_write_file_posn(void *fp, unsigned long *posn,
+		void *buf, unsigned long buflen);
+int dhd_msix_message_set(dhd_pub_t *dhdp, uint table_entry,
+    uint message_number, bool unmask);
+
+#ifdef DHD_DEBUG
+void dhd_convert_memdump_type_to_str(uint32 type, char *buf, size_t buf_len, int substr_type);
+#endif /* DHD_DEBUG */
+
+#ifdef DHD_COREDUMP
+void dhd_convert_hang_reason_to_str(uint32 reason, char *buf, size_t buf_len);
+#endif /* DHD_COREDUMP */
+
+extern void
+dhd_pcie_dump_core_regs(dhd_pub_t * pub, uint32 index, uint32 first_addr, uint32 last_addr);
+extern void wl_dhdpcie_dump_regs(void * context);
+
+#define DHD_OS_IOCTL_RESP_LOCK(x)
+#define DHD_OS_IOCTL_RESP_UNLOCK(x)
+
+#if defined(NDIS)
+#define dhd_os_open_image(a) wl_os_open_image(a)
+#define dhd_os_close_image(a) wl_os_close_image(a)
+#define dhd_os_get_image_block(a, b, c) wl_os_get_image_block(a, b, c)
+#define dhd_os_get_image_size(a) wl_os_get_image_size(a)
+extern void dhd_os_wakeind(dhd_pub_t * pub, uint32 *val);
+extern void dhd_bus_check_died(void *bus);
+extern void pci_save_state(osl_t *osh, uint32 *buffer);
+extern void pci_restore_state(osl_t *osh, uint32 *buffer);
+#endif /* #if defined(NDIS) */
+
+extern int dhd_os_get_image_block(char * buf, int len, void * image);
+extern int dhd_os_get_image_size(void * image);
+#if defined(BT_OVER_SDIO)
+extern int dhd_os_gets_image(dhd_pub_t *pub, char *str, int len, void *image);
+extern void dhdsdio_bus_usr_cnt_inc(dhd_pub_t *pub);
+extern void dhdsdio_bus_usr_cnt_dec(dhd_pub_t *pub);
+#endif /* (BT_OVER_SDIO) */
+extern void *dhd_os_open_image1(dhd_pub_t *pub, char *filename); /* rev1 function signature */
+extern void dhd_os_close_image1(dhd_pub_t *pub, void *image);
+extern void dhd_os_wd_timer(void *bus, uint wdtick);
+extern void dhd_os_sdlock(dhd_pub_t * pub);
+extern void dhd_os_sdunlock(dhd_pub_t * pub);
+extern void dhd_os_sdlock_txq(dhd_pub_t * pub);
+extern void dhd_os_sdunlock_txq(dhd_pub_t * pub);
+extern unsigned long dhd_os_sdlock_txoff(dhd_pub_t * pub);
+extern void dhd_os_sdunlock_txoff(dhd_pub_t * pub, unsigned long flags);
+extern void dhd_os_sdlock_rxq(dhd_pub_t * pub);
+extern void dhd_os_sdunlock_rxq(dhd_pub_t * pub);
+extern void dhd_os_sdlock_sndup_rxq(dhd_pub_t * pub);
+extern void dhd_os_tracelog(const char *format, ...);
+#ifdef DHDTCPACK_SUPPRESS
+extern unsigned long dhd_os_tcpacklock(dhd_pub_t *pub);
+extern void dhd_os_tcpackunlock(dhd_pub_t *pub, unsigned long flags);
+#endif /* DHDTCPACK_SUPPRESS */
+
+extern int dhd_customer_oob_irq_map(void *adapter, unsigned long *irq_flags_ptr);
+extern int dhd_customer_gpio_wlan_ctrl(void *adapter, int onoff);
+extern int dhd_custom_get_mac_address(void *adapter, unsigned char *buf);
+#if defined(CUSTOM_COUNTRY_CODE)
+extern void get_customized_country_code(void *adapter, char *country_iso_code,
+	wl_country_t *cspec, u32 flags);
+#else
+extern void get_customized_country_code(void *adapter, char *country_iso_code, wl_country_t *cspec);
+#endif /* CUSTOM_COUNTRY_CODE */
+
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+
+enum sar_para_mode {
+	SAR_OFF          = 0,
+	HEAD_NORMAL      = 1,
+	HEAD_AIRPLANE    = 2,
+	GRIP_NORMAL      = 3,
+	GRIP_AIRPLANE    = 4,
+	BT_NORMAL        = 5,
+	BT_AIRPLANE      = 6,
+	HOTSPOT_NORMAL   = 7,
+	HOTSPOT_AIRPLANE = 8
+};
+
+typedef enum _eCountry_type {
+	SYNA_COUNTRY_TYPE_DEFAULT,
+	SYNA_COUNTRY_TYPE_FCC,
+	SYNA_COUNTRY_TYPE_EU,
+	SYNA_COUNTRY_TYPE_LATAM,
+	SYNA_COUNTRY_TYPE_CANADA,
+	SYNA_COUNTRY_TYPE_JAPAN,
+	SYNA_COUNTRY_TYPE_QTY,
+	SYNA_COUNTRY_TYPE_INVALID = SYNA_COUNTRY_TYPE_QTY
+} eCountry_flag_type;
+
+/* SAR parameters from dhd */
+typedef struct _dhd_sar_parameter {
+	sarctrl_set  dynamic;
+	sarctrl_set  grip;
+	sarctrl_set  bt;
+	sarctrl_set  hotspot;
+} dhd_sar_parameter;
+
+extern eCountry_flag_type syna_country_check_type(char *cntry);
+extern int syna_country_update_type_list(eCountry_flag_type type, char *list_str);
+
+extern int dhd_sar_init_parameter(dhd_pub_t *dhd, eCountry_flag_type type, int advance_mode,
+	char *list_str);
+extern int dhd_sar_reset_parameter(void);
+extern int dhd_sar_set_parameter(dhd_pub_t *dhd_pub, int advance_mode);
+
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+
+extern void dhd_os_sdunlock_sndup_rxq(dhd_pub_t * pub);
+extern void dhd_os_sdlock_eventq(dhd_pub_t * pub);
+extern void dhd_os_sdunlock_eventq(dhd_pub_t * pub);
+extern bool dhd_os_check_hang(dhd_pub_t *dhdp, int ifidx, int ret);
+extern int dhd_os_send_hang_message(dhd_pub_t *dhdp);
+extern void dhd_set_version_info(dhd_pub_t *pub, char *fw);
+extern bool dhd_os_check_if_up(dhd_pub_t *pub);
+extern int dhd_os_check_wakelock(dhd_pub_t *pub);
+extern int dhd_os_check_wakelock_all(dhd_pub_t *pub);
+extern int dhd_get_instance(dhd_pub_t *pub);
+#ifdef CUSTOM_SET_CPUCORE
+extern void dhd_set_cpucore(dhd_pub_t *dhd, int set);
+#endif /* CUSTOM_SET_CPUCORE */
+
+#ifdef DHD_DETECT_CONSECUTIVE_MFG_HANG
+#define MAX_CONSECUTIVE_MFG_HANG_COUNT 2
+#endif /* DHD_DETECT_CONSECUTIVE_MFG_HANG */
+
+#if defined(KEEP_ALIVE)
+extern int dhd_keep_alive_onoff(dhd_pub_t *dhd);
+#endif /* KEEP_ALIVE */
+
+#if defined(DHD_FW_COREDUMP)
+#ifdef LINUX
+extern bool dhd_memdump_is_scheduled(dhd_pub_t *dhdp);
+#else
+static INLINE bool dhd_memdump_is_scheduled(dhd_pub_t *dhdp) { return FALSE; }
+#endif /* LINUX */
+void dhd_schedule_memdump(dhd_pub_t *dhdp, uint8 *buf, uint32 size);
+#endif /* DHD_FW_COREDUMP */
+
+#if defined(linux) || defined(LINUX)
+#ifdef DHD_LINUX_STD_FW_API
+int dhd_os_get_img_fwreq(const struct firmware **fw, char *file_path);
+void dhd_os_close_img_fwreq(const struct firmware *fw);
+#endif /* DHD_LINUX_STD_FW_API */
+#if defined(DHD_SSSR_DUMP)
+void dhd_write_sssr_dump(dhd_pub_t *dhdp, uint32 dump_mode);
+#endif /* DHD_SSSR_DUMP */
+#ifdef DNGL_AXI_ERROR_LOGGING
+void dhd_schedule_axi_error_dump(dhd_pub_t *dhdp, void *type);
+#endif /* DNGL_AXI_ERROR_LOGGING */
+#ifdef BCMPCIE
+void dhd_schedule_cto_recovery(dhd_pub_t *dhdp);
+#endif /* BCMPCIE */
+#else
+#if defined(DHD_SSSR_DUMP)
+static INLINE void dhd_write_sssr_dump(dhd_pub_t *dhd, uint32 dump_mode) { return; }
+#endif /* DHD_SSSR_DUMP */
+#ifdef DNGL_AXI_ERROR_LOGGING
+static INLINE void dhd_schedule_axi_error_dump(dhd_pub_t *dhdp, void *type) { return; }
+#endif /* DNGL_AXI_ERROR_LOGGING */
+/* For non-linux map dhd_schedule_cto_recovery to dhdpcie_cto_recovery_handler */
+#ifdef BCMPCIE
+#define dhd_schedule_cto_recovery(dhdp) dhdpcie_cto_recovery_handler(dhdp)
+#endif /* BCMPCIE */
+#endif /* linux || LINUX */
+
+#ifdef EWP_EDL
+#define EDL_SCHEDULE_DELAY 500 /* 500ms */
+#if defined(linux) || defined(LINUX)
+void dhd_schedule_edl_work(dhd_pub_t *dhdp, uint delay_ms);
+#else
+static INLINE void dhd_schedule_edl_work(dhd_pub_t *dhd, uint delay_ms) { return; }
+#endif /* linux || LINUX */
+#endif /* EWP_EDL */
+
+#ifdef SUPPORT_AP_POWERSAVE
+extern int dhd_set_ap_powersave(dhd_pub_t *dhdp, int ifidx, int enable);
+#endif /* SUPPORT_AP_POWERSAVE */
+
+#ifdef PKT_FILTER_SUPPORT
+#define DHD_UNICAST_FILTER_NUM		0
+#define DHD_BROADCAST_FILTER_NUM	1
+#define DHD_MULTICAST4_FILTER_NUM	2
+#define DHD_MULTICAST6_FILTER_NUM	3
+#define DHD_MDNS_FILTER_NUM		4
+#define DHD_ARP_FILTER_NUM		5
+#define DHD_BROADCAST_ARP_FILTER_NUM	6
+#define DHD_IP4BCAST_DROP_FILTER_NUM	7
+#define DHD_LLC_STP_DROP_FILTER_NUM	8
+#define DHD_LLC_XID_DROP_FILTER_NUM	9
+#define DHD_UDPNETBIOS_DROP_FILTER_NUM	10
+#define DISCARD_IPV4_MCAST	"102 1 6 IP4_H:16 0xf0 0xe0"
+#define DISCARD_IPV6_MCAST	"103 1 6 IP6_H:24 0xff 0xff"
+#define DISCARD_IPV4_BCAST	"107 1 6 IP4_H:16 0xffffffff 0xffffffff"
+#define DISCARD_LLC_STP		"108 1 6 ETH_H:14 0xFFFFFFFFFFFF 0xAAAA0300000C"
+#define DISCARD_LLC_XID		"109 1 6 ETH_H:14 0xFFFFFF 0x0001AF"
+#define DISCARD_UDPNETBIOS	"110 1 6 UDP_H:2 0xffff 0x0089"
+extern int dhd_os_enable_packet_filter(dhd_pub_t *dhdp, int val);
+extern void dhd_enable_packet_filter(int value, dhd_pub_t *dhd);
+extern int dhd_packet_filter_add_remove(dhd_pub_t *dhdp, int add_remove, int num);
+extern int net_os_enable_packet_filter(struct net_device *dev, int val);
+extern int net_os_rxfilter_add_remove(struct net_device *dev, int val, int num);
+
+#define MAX_PKTFLT_BUF_SIZE		2048
+#define MAX_PKTFLT_FIXED_PATTERN_SIZE	32
+#define MAX_PKTFLT_FIXED_BUF_SIZE	\
+	(WL_PKT_FILTER_FIXED_LEN + MAX_PKTFLT_FIXED_PATTERN_SIZE * 2)
+#define MAXPKT_ARG	16
+#endif /* PKT_FILTER_SUPPORT */
+
+extern bool dhd_support_sta_mode(dhd_pub_t *dhd);
+extern int write_to_file(dhd_pub_t *dhd, uint8 *buf, int size);
+
+#ifdef RSSI_MONITOR_SUPPORT
+extern int dhd_dev_set_rssi_monitor_cfg(struct net_device *dev, int start,
+             int8 max_rssi, int8 min_rssi);
+#endif /* RSSI_MONITOR_SUPPORT */
+
+#ifdef DHDTCPACK_SUPPRESS
+int dhd_dev_set_tcpack_sup_mode_cfg(struct net_device *dev, uint8 enable);
+#endif /* DHDTCPACK_SUPPRESS */
+
+#define DHD_RSSI_MONITOR_EVT_VERSION   1
+typedef struct {
+	uint8 version;
+	int8 cur_rssi;
+	struct ether_addr BSSID;
+} dhd_rssi_monitor_evt_t;
+
+typedef struct {
+	uint32 limit;		/* Expiration time (usec) */
+	uint32 increment;	/* Current expiration increment (usec) */
+	uint32 elapsed;		/* Current elapsed time (usec) */
+	uint32 tick;		/* O/S tick time (usec) */
+} dhd_timeout_t;
+
+#ifdef SHOW_LOGTRACE
+typedef struct {
+	uint  num_fmts;
+	char **fmts;
+	char *raw_fmts;
+	char *raw_sstr;
+	uint32 fmts_size;
+	uint32 raw_fmts_size;
+	uint32 raw_sstr_size;
+	uint32 ramstart;
+	uint32 rodata_start;
+	uint32 rodata_end;
+	char *rom_raw_sstr;
+	uint32 rom_raw_sstr_size;
+	uint32 rom_ramstart;
+	uint32 rom_rodata_start;
+	uint32 rom_rodata_end;
+} dhd_event_log_t;
+#endif /* SHOW_LOGTRACE */
+
+#if defined(APF)
+/*
+ * As per Google's current implementation, there will be only one APF filter.
+ * Therefore, userspace doesn't bother about filter id and because of that
+ * DHD has to manage the filter id.
+ */
+#define PKT_FILTER_APF_ID		200
+#define DHD_APF_LOCK(ndev)		dhd_apf_lock(ndev)
+#define DHD_APF_UNLOCK(ndev)		dhd_apf_unlock(ndev)
+
+extern void dhd_apf_lock(struct net_device *dev);
+extern void dhd_apf_unlock(struct net_device *dev);
+extern int dhd_dev_apf_get_version(struct net_device *ndev, uint32 *version);
+extern int dhd_dev_apf_get_max_len(struct net_device *ndev, uint32 *max_len);
+extern int dhd_dev_apf_add_filter(struct net_device *ndev, u8* program, uint32 program_len);
+extern int dhd_dev_apf_enable_filter(struct net_device *ndev);
+extern int dhd_dev_apf_disable_filter(struct net_device *ndev);
+extern int dhd_dev_apf_delete_filter(struct net_device *ndev);
+extern int dhd_dev_apf_read_filter_data(struct net_device *ndev, u8* buf, uint32 buf_len);
+#endif /* APF */
+
+extern void dhd_timeout_start(dhd_timeout_t *tmo, uint usec);
+extern int dhd_timeout_expired(dhd_timeout_t *tmo);
+
+extern int dhd_ifname2idx(struct dhd_info *dhd, char *name);
+#ifdef LINUX
+extern int dhd_net2idx(struct dhd_info *dhd, struct net_device *net);
+extern struct net_device * dhd_idx2net(void *pub, int ifidx);
+extern int net_os_send_hang_message(struct net_device *dev);
+extern int net_os_send_hang_message_reason(struct net_device *dev, const char *string_num);
+#endif
+extern bool dhd_wowl_cap(void *bus);
+extern int wl_host_event(dhd_pub_t *dhd_pub, int *idx, void *pktdata, uint pktlen,
+	wl_event_msg_t *, void **data_ptr,  void *);
+extern int wl_process_host_event(dhd_pub_t *dhd_pub, int *idx, void *pktdata, uint pktlen,
+	wl_event_msg_t *, void **data_ptr,  void *);
+extern void wl_event_to_host_order(wl_event_msg_t * evt);
+extern int wl_host_event_get_data(void *pktdata, uint pktlen, bcm_event_msg_u_t *evu);
+extern int dhd_wl_ioctl(dhd_pub_t *dhd_pub, int ifindex, wl_ioctl_t *ioc, void *buf, int len);
+extern int dhd_wl_ioctl_cmd(dhd_pub_t *dhd_pub, int cmd, void *arg, int len, uint8 set,
+                            int ifindex);
+extern int dhd_wl_ioctl_get_intiovar(dhd_pub_t *dhd_pub, char *name, uint *pval,
+	int cmd, uint8 set, int ifidx);
+extern int dhd_wl_ioctl_set_intiovar(dhd_pub_t *dhd_pub, char *name, uint val,
+	int cmd, uint8 set, int ifidx);
+extern void dhd_common_init(osl_t *osh);
+
+#if defined(linux) || defined(LINUX) || defined(OEM_ANDROID)
+extern int dhd_do_driver_init(struct net_device *net);
+#endif
+extern int dhd_event_ifadd(struct dhd_info *dhd, struct wl_event_data_if *ifevent,
+	char *name, uint8 *mac);
+extern int dhd_event_ifdel(struct dhd_info *dhd, struct wl_event_data_if *ifevent,
+	char *name, uint8 *mac);
+extern int dhd_event_ifchange(struct dhd_info *dhd, struct wl_event_data_if *ifevent,
+       char *name, uint8 *mac);
+#ifdef DHD_UPDATE_INTF_MAC
+extern int dhd_op_if_update(dhd_pub_t *dhdpub, int ifidx);
+#endif /* DHD_UPDATE_INTF_MAC */
+extern struct net_device* dhd_allocate_if(dhd_pub_t *dhdpub, int ifidx, const char *name,
+	uint8 *mac, uint8 bssidx, bool need_rtnl_lock, const char *dngl_name);
+extern int dhd_remove_if(dhd_pub_t *dhdpub, int ifidx, bool need_rtnl_lock);
+#ifdef WL_STATIC_IF
+extern s32 dhd_update_iflist_info(dhd_pub_t *dhdp, struct net_device *ndev, int ifidx,
+	uint8 *mac, uint8 bssidx, const char *dngl_name, int if_state);
+#endif /* WL_STATIC_IF */
+extern void dhd_vif_add(struct dhd_info *dhd, int ifidx, char * name);
+extern void dhd_vif_del(struct dhd_info *dhd, int ifidx);
+extern void dhd_event(struct dhd_info *dhd, char *evpkt, int evlen, int ifidx);
+extern void dhd_vif_sendup(struct dhd_info *dhd, int ifidx, uchar *cp, int len);
+
+/* Send packet to dongle via data channel */
+extern int dhd_sendpkt(dhd_pub_t *dhdp, int ifidx, void *pkt);
+
+/* send up locally generated event */
+extern void dhd_sendup_event_common(dhd_pub_t *dhdp, wl_event_msg_t *event, void *data);
+/* Send event to host */
+extern void dhd_sendup_event(dhd_pub_t *dhdp, wl_event_msg_t *event, void *data);
+#ifdef LOG_INTO_TCPDUMP
+extern void dhd_sendup_log(dhd_pub_t *dhdp, void *data, int len);
+#endif /* LOG_INTO_TCPDUMP */
+#if defined(SHOW_LOGTRACE) && defined(EWP_EDL)
+void dhd_sendup_info_buf(dhd_pub_t *dhdp, uint8 *msg);
+#endif
+#if defined(WIFI_TURNON_USE_HALINIT)
+extern int dhd_open(struct net_device *net);
+#endif /* WIFI_TURNON_USE_HALINIT */
+#if defined(LINUX) || defined(linux)
+extern int dhd_stop(struct net_device *net);
+#endif /* LINUX */
+extern int dhd_bus_devreset(dhd_pub_t *dhdp, uint8 flag);
+extern uint dhd_bus_status(dhd_pub_t *dhdp);
+extern int  dhd_bus_start(dhd_pub_t *dhdp);
+extern int dhd_bus_suspend(dhd_pub_t *dhdpub);
+extern int dhd_bus_resume(dhd_pub_t *dhdpub, int stage);
+extern int dhd_bus_membytes(dhd_pub_t *dhdp, bool set, uint32 address, uint8 *data, uint size);
+extern void dhd_print_buf(void *pbuf, int len, int bytes_per_line);
+extern bool dhd_is_associated(dhd_pub_t *dhd, uint8 ifidx, int *retval);
+#if defined(LINUX) || defined(linux)
+extern bool dhd_scan_associated(dhd_pub_t *dhd, struct net_device *dev);
+#endif /* LINUX */
+#if defined(BCMSDIO) || defined(BCMPCIE)
+extern uint dhd_bus_chip_id(dhd_pub_t *dhdp);
+extern uint dhd_bus_chiprev_id(dhd_pub_t *dhdp);
+extern uint dhd_bus_chippkg_id(dhd_pub_t *dhdp);
+#endif /* defined(BCMSDIO) || defined(BCMPCIE) */
+#if defined(LINUX) || defined(linux)
+int dhd_bus_get_fw_mode(dhd_pub_t *dhdp);
+#else
+static INLINE int dhd_bus_get_fw_mode(dhd_pub_t *dhdp) { return 0; }
+#endif /* LINUX || linux */
+
+#if defined(KEEP_ALIVE)
+extern int dhd_keep_alive_onoff(dhd_pub_t *dhd);
+#endif /* KEEP_ALIVE */
+
+/* linux is defined for DHD EFI builds also,
+* since its cross-compiled for EFI from linux.
+* dbgring_lock apis are meant only for linux
+* to use mutexes, other OSes will continue to
+* use osl_spin_lock
+*/
+#if (defined(LINUX) || defined(linux))
+void *dhd_os_dbgring_lock_init(osl_t *osh);
+void dhd_os_dbgring_lock_deinit(osl_t *osh, void *mtx);
+unsigned long dhd_os_dbgring_lock(void *lock);
+void dhd_os_dbgring_unlock(void *lock, unsigned long flags);
+#endif
+
+#ifdef PCIE_INB_DW
+static INLINE int dhd_os_ds_enter_wait(dhd_pub_t * pub, uint * condition)
+{ return 1; }
+static INLINE int dhd_os_ds_enter_wake(dhd_pub_t * pub)
+{ return 0; }
+#endif /* PCIE_INB_DW */
+
+#define FWBOOT_INTR_TIMEOUT_MS 5000u
+
+#if defined(LINUX) || defined(linux)
+extern int dhd_os_busbusy_wait_negation(dhd_pub_t * pub, uint * condition);
+extern int dhd_os_busbusy_wake(dhd_pub_t * pub);
+extern void dhd_os_tx_completion_wake(dhd_pub_t *dhd);
+extern int dhd_os_busbusy_wait_condition(dhd_pub_t *pub, uint *var, uint condition);
+extern int dhd_os_d3ack_wait(dhd_pub_t * pub, uint * condition);
+extern int dhd_os_d3ack_wake(dhd_pub_t * pub);
+extern int dhd_os_dmaxfer_wait(dhd_pub_t *pub, uint *condition);
+extern int dhd_os_dmaxfer_wake(dhd_pub_t *pub);
+int dhd_os_busbusy_wait_bitmask(dhd_pub_t *pub, uint *var,
+		uint bitmask, uint condition);
+#ifdef PCIE_INB_DW
+extern int dhd_os_ds_exit_wait(dhd_pub_t * pub, uint * condition);
+extern int dhd_os_ds_exit_wake(dhd_pub_t * pub);
+#endif /* PCIE_INB_DW */
+int dhd_os_tput_test_wait(dhd_pub_t *pub, uint *condition, uint timeout_ms);
+int dhd_os_tput_test_wake(dhd_pub_t * pub);
+#else
+static INLINE int dhd_os_tput_test_wait(dhd_pub_t *pub, uint *condition,
+		uint timeout_ms)
+{ return 0; }
+static INLINE int dhd_os_tput_test_wake(dhd_pub_t * pub)
+{ return 0; }
+static INLINE int dhd_os_d3ack_wait(dhd_pub_t * pub, uint * condition)
+{ return dhd_os_ioctl_resp_wait(pub, condition); }
+static INLINE int dhd_os_d3ack_wake(dhd_pub_t * pub)
+{ return dhd_os_ioctl_resp_wake(pub); }
+#ifdef PCIE_INB_DW
+static INLINE int dhd_os_ds_exit_wait(dhd_pub_t * pub, uint * condition)
+{ DHD_ERROR(("%s is Not supported for this platform", __FUNCTION__)); return 0; }
+static INLINE int dhd_os_ds_exit_wake(dhd_pub_t * pub)
+{ DHD_ERROR(("%s is Not supported for this platform", __FUNCTION__)); return 0; }
+#endif /* PCIE_INB_DW */
+static INLINE int dhd_os_busbusy_wait_negation(dhd_pub_t * pub, uint * condition)
+{ return 1; }
+static INLINE int dhd_os_busbusy_wake(dhd_pub_t * pub)
+{ return 0; }
+static INLINE int dhd_os_busbusy_wait_condition(dhd_pub_t *pub, uint *var, uint condition)
+{ return 0; }
+static INLINE int dhd_os_dmaxfer_wait(dhd_pub_t *pub, uint *condition)
+{ return 0; }
+static INLINE int dhd_os_dmaxfer_wake(dhd_pub_t *pub)
+{ return 0; }
+static INLINE int dhd_os_busbusy_wait_bitmask(dhd_pub_t *pub, uint *var,
+		uint bitmask, uint condition)
+{ return 0; }
+#endif /* LINUX || linux */
+
+#if defined(LINUX) || defined(linux)
+/*
+ * Manage sta objects in an interface. Interface is identified by an ifindex and
+ * sta(s) within an interfaces are managed using a MacAddress of the sta.
+ */
+struct dhd_sta;
+extern bool dhd_sta_associated(dhd_pub_t *dhdp, uint32 bssidx, uint8 *mac);
+extern struct dhd_sta *dhd_find_sta(void *pub, int ifidx, void *ea);
+extern struct dhd_sta *dhd_findadd_sta(void *pub, int ifidx, void *ea);
+extern void dhd_del_all_sta(void *pub, int ifidx);
+extern void dhd_del_sta(void *pub, int ifidx, void *ea);
+extern void dhd_update_sta_chanspec_info(void *pub, int ifidx, const uint8 *ea,
+	chanspec_t chanspec);
+extern bool dhd_is_sta_htput(void *pub, int ifidx, void *ea);
+extern int dhd_get_ap_isolate(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_ap_isolate(dhd_pub_t *dhdp, uint32 idx, int val);
+extern int dhd_bssidx2idx(dhd_pub_t *dhdp, uint32 bssidx);
+extern struct net_device *dhd_linux_get_primary_netdev(dhd_pub_t *dhdp);
+#else /* LINUX */
+static INLINE bool dhd_sta_associated(dhd_pub_t *dhdp, uint32 bssidx, uint8 *mac)
+{ return FALSE;}
+static INLINE void* dhd_find_sta(void *pub, int ifidx, void *ea) { return NULL;}
+static INLINE void *dhd_findadd_sta(void *pub, int ifidx, void *ea) { return NULL; }
+static INLINE void dhd_del_all_sta(void *pub, int ifidx) { }
+static INLINE void dhd_del_sta(void *pub, int ifidx, void *ea) { }
+static INLINE void dhd_update_sta_chanspec_info(void *pub, int ifidx, const uint8 *ea,
+	chanspec_t chanspec) {}
+static INLINE bool dhd_is_sta_htput(void *pub, int ifidx, void *ea) { return FALSE; }
+static INLINE int dhd_get_ap_isolate(dhd_pub_t *dhdp, uint32 idx) { return 0; }
+static INLINE int dhd_set_ap_isolate(dhd_pub_t *dhdp, uint32 idx, int val) { return 0; }
+static INLINE int dhd_bssidx2idx(dhd_pub_t *dhdp, uint32 bssidx) { return 0; }
+#endif /* LINUX */
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+dhd_dongledump_status_t dhd_get_dump_status(dhd_pub_t *pub);
+void dhd_set_dump_status(dhd_pub_t *pub, dhd_dongledump_status_t status);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+extern bool dhd_is_concurrent_mode(dhd_pub_t *dhd);
+int dhd_iovar(dhd_pub_t *pub, int ifidx, char *name, char *param_buf, uint param_len,
+		char *res_buf, uint res_len, bool set);
+extern int dhd_getiovar(dhd_pub_t *pub, int ifidx, char *name, char *cmd_buf,
+		uint cmd_len, char **resptr, uint resp_len);
+
+#ifdef DHD_MCAST_REGEN
+extern int dhd_get_mcast_regen_bss_enable(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_mcast_regen_bss_enable(dhd_pub_t *dhdp, uint32 idx, int val);
+#endif
+typedef enum cust_gpio_modes {
+	WLAN_RESET_ON,
+	WLAN_RESET_OFF,
+	WLAN_POWER_ON,
+	WLAN_POWER_OFF
+} cust_gpio_modes_t;
+
+typedef struct dmaxref_mem_map {
+	dhd_dma_buf_t *srcmem;
+	dhd_dma_buf_t *dstmem;
+} dmaxref_mem_map_t;
+
+#if defined(OEM_ANDROID)
+extern int wl_iw_iscan_set_scan_broadcast_prep(struct net_device *dev, uint flag);
+extern int wl_iw_send_priv_event(struct net_device *dev, char *flag);
+#endif /* defined(OEM_ANDROID) */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+extern void dhd_flush_rx_tx_wq(dhd_pub_t *dhdp);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+/*
+ * Insmod parameters for debug/test
+ */
+
+/* Watchdog timer interval */
+extern uint dhd_watchdog_ms;
+extern bool dhd_os_wd_timer_enabled(void *bus);
+
+/** Default console output poll interval */
+extern uint dhd_console_ms;
+
+extern uint android_msg_level;
+extern uint config_msg_level;
+extern uint sd_msglevel;
+extern uint dump_msg_level;
+#ifdef BCMDBUS
+extern uint dbus_msglevel;
+#endif /* BCMDBUS */
+#ifdef WL_WIRELESS_EXT
+extern uint iw_msg_level;
+#endif
+#ifdef WL_CFG80211
+extern uint wl_dbg_level;
+#endif
+
+extern uint dhd_slpauto;
+
+/* Use interrupts */
+extern uint dhd_intr;
+
+/* Use polling */
+extern uint dhd_poll;
+
+/* ARP offload agent mode */
+extern uint dhd_arp_mode;
+
+/* Pkt filte enable control */
+extern uint dhd_pkt_filter_enable;
+
+/*  Pkt filter init setup */
+extern uint dhd_pkt_filter_init;
+
+/* Pkt filter mode control */
+extern uint dhd_master_mode;
+
+/* Roaming mode control */
+extern uint dhd_roam_disable;
+
+/* Roaming mode control */
+extern uint dhd_radio_up;
+
+/* TCM verification control */
+extern uint dhd_tcm_test_enable;
+
+/* TCM test status */
+typedef enum tcm_test_status {
+	TCM_TEST_NOT_RUN = 0,
+	TCM_TEST_FAILED = 1,
+	TCM_TEST_PASSED = 2
+} tcm_test_status_t;
+extern tcm_test_status_t dhd_tcm_test_status;
+
+/* TCM test mode */
+typedef enum tcm_test_mode {
+	TCM_TEST_MODE_DISABLE = 0,
+	TCM_TEST_MODE_ONCE = 1,
+	TCM_TEST_MODE_ALWAYS = 2
+} tcm_test_mode_t;
+extern tcm_test_mode_t dhd_tcm_test_mode;
+
+/* Initial idletime ticks (may be -1 for immediate idle, 0 for no idle) */
+extern int dhd_idletime;
+#ifdef DHD_USE_IDLECOUNT
+#define DHD_IDLETIME_TICKS 5
+#else
+#define DHD_IDLETIME_TICKS 1
+#endif /* DHD_USE_IDLECOUNT */
+
+/* SDIO Drive Strength */
+extern uint dhd_sdiod_drive_strength;
+
+/* triggers bcm_bprintf to print to kernel log */
+extern bool bcm_bprintf_bypass;
+
+/* Override to force tx queueing all the time */
+extern uint dhd_force_tx_queueing;
+
+/* Default bcn_timeout value is 4 */
+#define DEFAULT_BCN_TIMEOUT_VALUE	4
+#ifndef CUSTOM_BCN_TIMEOUT_SETTING
+#define CUSTOM_BCN_TIMEOUT_SETTING	DEFAULT_BCN_TIMEOUT_VALUE
+#endif
+
+/* Default KEEP_ALIVE Period is 55 sec to prevent AP from sending Keep Alive probe frame */
+#define DEFAULT_KEEP_ALIVE_VALUE 	55000 /* msec */
+#ifndef CUSTOM_KEEP_ALIVE_SETTING
+#define CUSTOM_KEEP_ALIVE_SETTING 	DEFAULT_KEEP_ALIVE_VALUE
+#endif /* DEFAULT_KEEP_ALIVE_VALUE */
+
+#define NULL_PKT_STR	"null_pkt"
+
+/* hooks for custom glom setting option via Makefile */
+#define DEFAULT_GLOM_VALUE 	-1
+#ifndef CUSTOM_GLOM_SETTING
+#define CUSTOM_GLOM_SETTING 	DEFAULT_GLOM_VALUE
+#endif
+#define WL_AUTO_ROAM_TRIGGER -75
+/* hooks for custom Roaming Trigger  setting via Makefile */
+#define DEFAULT_ROAM_TRIGGER_VALUE -75 /* dBm default roam trigger all band */
+#define DEFAULT_ROAM_TRIGGER_SETTING 	-1
+#ifndef CUSTOM_ROAM_TRIGGER_SETTING
+#define CUSTOM_ROAM_TRIGGER_SETTING 	DEFAULT_ROAM_TRIGGER_VALUE
+#endif
+
+/* hooks for custom Roaming Romaing  setting via Makefile */
+#define DEFAULT_ROAM_DELTA_VALUE  10 /* dBm default roam delta all band */
+#define DEFAULT_ROAM_DELTA_SETTING 	-1
+#ifndef CUSTOM_ROAM_DELTA_SETTING
+#define CUSTOM_ROAM_DELTA_SETTING 	DEFAULT_ROAM_DELTA_VALUE
+#endif
+
+/* hooks for custom PNO Event wake lock to guarantee enough time
+	for the Platform to detect Event before system suspended
+*/
+#define DEFAULT_PNO_EVENT_LOCK_xTIME 	2 	/* multiplay of DHD_PACKET_TIMEOUT_MS */
+#ifndef CUSTOM_PNO_EVENT_LOCK_xTIME
+#define CUSTOM_PNO_EVENT_LOCK_xTIME	 DEFAULT_PNO_EVENT_LOCK_xTIME
+#endif
+/* hooks for custom dhd_dpc_prio setting option via Makefile */
+#define DEFAULT_DHP_DPC_PRIO  1
+#ifndef CUSTOM_DPC_PRIO_SETTING
+#define CUSTOM_DPC_PRIO_SETTING 	DEFAULT_DHP_DPC_PRIO
+#endif
+
+#ifndef CUSTOM_LISTEN_INTERVAL
+#define CUSTOM_LISTEN_INTERVAL 		LISTEN_INTERVAL
+#endif /* CUSTOM_LISTEN_INTERVAL */
+
+#define DEFAULT_SUSPEND_BCN_LI_DTIM		3
+#ifndef CUSTOM_SUSPEND_BCN_LI_DTIM
+#define CUSTOM_SUSPEND_BCN_LI_DTIM		DEFAULT_SUSPEND_BCN_LI_DTIM
+#endif
+
+#ifdef OEM_ANDROID
+#ifndef BCN_TIMEOUT_IN_SUSPEND
+#define BCN_TIMEOUT_IN_SUSPEND			6 /* bcn timeout value in suspend mode */
+#endif
+#endif /* OEM_ANDROID */
+
+#ifndef CUSTOM_RXF_PRIO_SETTING
+#define CUSTOM_RXF_PRIO_SETTING		MAX((CUSTOM_DPC_PRIO_SETTING - 1), 1)
+#endif
+
+#define DEFAULT_WIFI_TURNOFF_DELAY		10
+#ifndef WIFI_TURNOFF_DELAY
+#define WIFI_TURNOFF_DELAY		DEFAULT_WIFI_TURNOFF_DELAY
+#endif /* WIFI_TURNOFF_DELAY */
+
+#define DEFAULT_WIFI_TURNON_DELAY		200
+#ifndef WIFI_TURNON_DELAY
+#define WIFI_TURNON_DELAY		DEFAULT_WIFI_TURNON_DELAY
+#endif /* WIFI_TURNON_DELAY */
+
+#ifdef BCMSDIO
+#define DEFAULT_DHD_WATCHDOG_INTERVAL_MS	10 /* msec */
+#else
+#define DEFAULT_DHD_WATCHDOG_INTERVAL_MS	0 /* msec */
+#endif
+#ifndef CUSTOM_DHD_WATCHDOG_MS
+#define CUSTOM_DHD_WATCHDOG_MS			DEFAULT_DHD_WATCHDOG_INTERVAL_MS
+#endif /* DEFAULT_DHD_WATCHDOG_INTERVAL_MS */
+
+#if defined(DHD_DEBUG)
+#ifdef OEM_ANDROID
+#define DEFAULT_DHD_CONSOLE_INTERVAL_MS		0 /* android by default disable console msgs */
+#else
+#define DEFAULT_DHD_CONSOLE_INTERVAL_MS		250 /* msec */
+#endif /* OEM_ANDROID */
+
+#ifndef CUSTOM_DHD_CONSOLE_MS
+#define CUSTOM_DHD_CONSOLE_MS			DEFAULT_DHD_CONSOLE_INTERVAL_MS
+#endif /* DEFAULT_DHD_WATCHDOG_INTERVAL_MS */
+#endif /* DHD_DEBUG */
+
+#define	DHD_INB_DW_DEASSERT_MS			250
+
+#define DEFAULT_ASSOC_RETRY_MAX			3
+#ifndef CUSTOM_ASSOC_RETRY_MAX
+#define CUSTOM_ASSOC_RETRY_MAX			DEFAULT_ASSOC_RETRY_MAX
+#endif /* DEFAULT_ASSOC_RETRY_MAX */
+
+#if defined(DISABLE_FRAMEBURST)
+#define DEFAULT_FRAMEBURST_SET			0
+#else
+#define DEFAULT_FRAMEBURST_SET			1
+#endif /* BCMSDIO */
+
+#ifndef CUSTOM_FRAMEBURST_SET
+#define CUSTOM_FRAMEBURST_SET			DEFAULT_FRAMEBURST_SET
+#endif /* CUSTOM_FRAMEBURST_SET */
+
+#ifdef WLTDLS
+#ifndef CUSTOM_TDLS_IDLE_MODE_SETTING
+#define CUSTOM_TDLS_IDLE_MODE_SETTING  60000 /* 60sec to tear down TDLS of not active */
+#endif
+#ifndef CUSTOM_TDLS_RSSI_THRESHOLD_HIGH
+#define CUSTOM_TDLS_RSSI_THRESHOLD_HIGH -70 /* rssi threshold for establishing TDLS link */
+#endif
+#ifndef CUSTOM_TDLS_RSSI_THRESHOLD_LOW
+#define CUSTOM_TDLS_RSSI_THRESHOLD_LOW -80 /* rssi threshold for tearing down TDLS link */
+#endif
+#ifndef CUSTOM_TDLS_PCKTCNT_THRESHOLD_HIGH
+#define CUSTOM_TDLS_PCKTCNT_THRESHOLD_HIGH 100 /* pkt/sec threshold for establishing TDLS link */
+#endif
+#ifndef CUSTOM_TDLS_PCKTCNT_THRESHOLD_LOW
+#define CUSTOM_TDLS_PCKTCNT_THRESHOLD_LOW 10 /* pkt/sec threshold for tearing down TDLS link */
+#endif
+#endif /* WLTDLS */
+
+#if defined(VSDB) || defined(ROAM_ENABLE)
+#define DEFAULT_BCN_TIMEOUT            6
+#else
+#define DEFAULT_BCN_TIMEOUT            4
+#endif
+
+#ifndef CUSTOM_BCN_TIMEOUT
+#define CUSTOM_BCN_TIMEOUT             DEFAULT_BCN_TIMEOUT
+#endif
+
+#define DEFAULT_BCN_TIMEOUT_IN_SUSPEND	10
+#ifndef CUSTOM_BCN_TIMEOUT_IN_SUSPEND
+#define CUSTOM_BCN_TIMEOUT_IN_SUSPEND	DEFAULT_BCN_TIMEOUT_IN_SUSPEND
+#endif /* CUSTOM_BCN_TIMEOUT_IN_SUSPEND */
+
+#define MAX_DTIM_SKIP_BEACON_INTERVAL	100 /* max allowed associated AP beacon for DTIM skip */
+#ifndef MAX_DTIM_ALLOWED_INTERVAL
+#define MAX_DTIM_ALLOWED_INTERVAL 600 /* max allowed total beacon interval for DTIM skip */
+#endif
+
+#ifdef OEM_ANDROID
+#ifndef MIN_DTIM_FOR_ROAM_THRES_EXTEND
+#define MIN_DTIM_FOR_ROAM_THRES_EXTEND	600 /* minimum dtim interval to extend roam threshold */
+#endif
+#endif /* OEM_ANDROID */
+
+#ifdef CONFIG_ROAM_RSSI_LIMIT
+extern int dhd_roam_rssi_limit_get(dhd_pub_t *dhd, int *lmt2g, int *lmt5g);
+extern int dhd_roam_rssi_limit_set(dhd_pub_t *dhd, int lmt2g, int lmt5g);
+#ifndef CUSTOM_ROAMRSSI_2G
+#define CUSTOM_ROAMRSSI_2G		ROAMRSSI_2G_DEFAULT
+#endif /* CUSTOM_ROAMRSSI_2G */
+#ifndef CUSTOM_ROAMRSSI_5G
+#define CUSTOM_ROAMRSSI_5G		ROAMRSSI_5G_DEFAULT
+#endif /* CUSTOM_ROAMRSSI_5G */
+#endif /* CONFIG_ROAM_RSSI_LIMIT */
+#ifdef CONFIG_ROAM_MIN_DELTA
+extern int dhd_roam_min_delta_get(dhd_pub_t *dhd, uint32 *dt2g, uint32 *dt5g);
+extern int dhd_roam_min_delta_set(dhd_pub_t *dhd, uint32 dt2g, uint32 dt5g);
+#ifndef CUSTOM_ROAM_MIN_DELTA
+#define CUSTOM_ROAM_MIN_DELTA		ROAM_MIN_DELTA_DEFAULT
+#endif /* CUSTOM_ROAM_MIN_DELTA */
+#endif /* CONFIG_ROAM_MIN_DELTA */
+
+#define NO_DTIM_SKIP 1
+#ifdef SDTEST
+/* Echo packet generator (SDIO), pkts/s */
+extern uint dhd_pktgen;
+
+/* Echo packet len (0 => sawtooth, max 1800) */
+extern uint dhd_pktgen_len;
+#define MAX_PKTGEN_LEN 1800
+#endif
+
+#ifdef BCMSLTGT
+/* Account for slow hardware (QT) */
+extern uint htclkratio;
+extern int dngl_xtalfreq;
+#endif
+
+/* optionally set by a module_param_string() */
+#define MOD_PARAM_PATHLEN	2048
+#define MOD_PARAM_INFOLEN	512
+#define MOD_PARAM_SRLEN		64
+
+#ifdef SOFTAP
+extern char fw_path2[MOD_PARAM_PATHLEN];
+#endif
+
+#if defined(SUPPORT_MULTIPLE_NVRAM) || defined(SUPPORT_MULTIPLE_CLMBLOB)
+#define MAX_HW_INFO_LEN   10u
+#define MAX_HW_EXT_LEN    (MAX_HW_INFO_LEN * 2)
+#define MAX_FILE_COUNT    8u
+#define MAX_FILE_LEN      90u
+#endif /* SUPPORT_MULTIPLE_NVRAM || SUPPORT_MULTIPLE_CLMBLOB */
+
+#define VENDOR_PATH ""
+
+/* Platform path Name -
+ * Used to find out where to find the FW debug support files.
+ * 1) If the Platform Makefile mentions from where it should be
+ *    picked from use it.
+ * 2) If Platform Makefile does not mention anything,use the
+ *    scheme as mapped below
+ */
+#if !defined(PLATFORM_PATH)
+/* First Overrides */
+#if defined(DHD_LEGACY_FILE_PATH)
+/* If Legacy file path is to be used */
+#define PLATFORM_PATH	"/data/"
+#elif defined(PLATFORM_SLP)
+/* Path Name for SLP */
+#define PLATFORM_PATH	"/opt/etc/"
+#else
+/* End of Overrides, rely on what is dictated by Android */
+#define PLATFORM_PATH	"/data/misc/conn/"
+#define DHD_MAC_ADDR_EXPORT
+#define DHD_EXPORT_CNTL_FILE
+#define DHD_SOFTAP_DUAL_IF_INFO
+#define DHD_SEND_HANG_PRIVCMD_ERRORS
+#endif /* DHD_LEGACY_FILE_PATH */
+#endif /* !PLATFORM_PATH */
+
+#ifdef DHD_MAC_ADDR_EXPORT
+extern struct ether_addr sysfs_mac_addr;
+#endif /* DHD_MAC_ADDR_EXPORT */
+
+#if defined(LINUX) || defined(linux)
+/* Flag to indicate if we should download firmware on driver load */
+extern uint dhd_download_fw_on_driverload;
+#ifndef BCMDBUS
+extern int allow_delay_fwdl;
+#endif /* !BCMDBUS */
+
+extern int dhd_process_cid_mac(dhd_pub_t *dhdp, bool prepost);
+extern int dhd_write_file(const char *filepath, char *buf, int buf_len);
+extern int dhd_read_file(const char *filepath, char *buf, int buf_len);
+extern int dhd_write_file_and_check(const char *filepath, char *buf, int buf_len);
+extern int dhd_file_delete(char *path);
+
+#ifdef READ_MACADDR
+extern int dhd_set_macaddr_from_file(dhd_pub_t *dhdp);
+#else
+static INLINE int dhd_set_macaddr_from_file(dhd_pub_t *dhdp) { return 0; }
+#endif /* READ_MACADDR */
+#ifdef WRITE_MACADDR
+extern int dhd_write_macaddr(struct ether_addr *mac);
+#else
+static INLINE int dhd_write_macaddr(struct ether_addr *mac) { return 0; }
+#endif /* WRITE_MACADDR */
+
+#if defined(USE_CID_CHECK) || defined(USE_DIRECT_VID_TAG)
+#if defined(BCM4361_CHIP) || defined(BCM4375_CHIP) || defined(BCM4389_CHIP_DEF)
+#define DHD_USE_CISINFO_FROM_OTP
+/* For COB, can't check CID/MAC in OTP, so, define it here */
+#define DHD_READ_CIS_FROM_BP
+#endif /* CONFIG_BCM4361 || CONFIG_BCM4375 || CONFIG_BCM4389_DEF */
+#define MAX_VNAME_LEN		64
+#define MAX_VID_LEN		8
+#define MODULE_NAME_INDEX_MAX	3
+#define MAX_EXTENSION 20
+typedef struct {
+	char cid_ext[MAX_EXTENSION];
+	char nvram_ext[MAX_EXTENSION];
+	char fw_ext[MAX_EXTENSION];
+} naming_info_t;
+extern uint32 cur_vid_info;
+#ifdef DHD_EXPORT_CNTL_FILE
+extern char cidinfostr[MAX_VNAME_LEN];
+#endif /* DHD_EXPORT_CNTL_FILE */
+extern int dhd_check_module_cid(dhd_pub_t *dhdp);
+#else
+static INLINE int dhd_check_module_cid(dhd_pub_t *dhdp) { return 0; }
+#endif /* USE_CID_CHECK || USE_DIRECT_VID_TAG */
+#ifdef USE_CID_CHECK
+extern char *dhd_get_cid_info(unsigned char *vid, int vid_length);
+#endif /* USE_CID_CHECK */
+#ifdef GET_MAC_FROM_OTP
+extern int dhd_check_module_mac(dhd_pub_t *dhdp);
+#else
+static INLINE int dhd_check_module_mac(dhd_pub_t *dhdp) { return 0; }
+#endif /* GET_MAC_FROM_OTP */
+
+#if defined(READ_MACADDR) || defined(WRITE_MACADDR) || defined(USE_CID_CHECK) || \
+	defined(GET_MAC_FROM_OTP)
+#define DHD_USE_CISINFO
+#endif /* READ_MACADDR || WRITE_MACADDR || USE_CID_CHECK || GET_MAC_FROM_OTP */
+
+#ifdef DHD_USE_CISINFO
+int dhd_read_cis(dhd_pub_t *dhdp);
+int dhd_read_otp_sw_rgn(dhd_pub_t *dhdp);
+void dhd_clear_cis(dhd_pub_t *dhdp);
+int dhd_alloc_cis(dhd_pub_t *dhdp);
+#if defined(SUPPORT_MULTIPLE_MODULE_CIS) && defined(USE_CID_CHECK)
+extern int dhd_check_module_b85a(void);
+extern int dhd_check_module_b90(void);
+#define BCM4359_MODULE_TYPE_B90B 1
+#define BCM4359_MODULE_TYPE_B90S 2
+#endif /* defined(SUPPORT_MULTIPLE_MODULE_CIS) && defined(USE_CID_CHECK) */
+#if defined(USE_CID_CHECK)
+extern int dhd_check_module_bcm(char *module_type, int index, bool *is_murata_fem);
+extern naming_info_t *
+dhd_find_naming_info(dhd_pub_t *dhdp, char *module_type);
+extern naming_info_t * dhd_find_naming_info_by_chip_rev(dhd_pub_t *dhdp, bool *is_murata_fem);
+#endif /* defined(USE_CID_CHECK) */
+#ifdef USE_DIRECT_VID_TAG
+#define VENDOR_OFF 1
+#define MD_REV_OFF 0
+#define A0_REV "_a0"
+#define B0_REV "_b0"
+#define C0_REV "_c0"
+#define C1_REV "_c1"
+extern int dhd_check_stored_module_info(char *vid);
+extern int concate_nvram_by_vid(dhd_pub_t *dhdp, char *nv_path, char *chipstr);
+#endif /* USE_DIRECT_VID_TAG */
+#if defined(USE_CID_CHECK) && defined(USE_DIRECT_VID_TAG)
+#error Please use USE_CID_CHECK/USE_DIRECT_VID_TAG exclusively
+#endif /* USE_CID_CHECK && USE_DIRECT_VID_TAG */
+#else
+static INLINE int dhd_read_cis(dhd_pub_t *dhdp) { return 0; }
+static INLINE int dhd_read_otp_sw_rgn(dhd_pub_t *dhdp) { return 0; }
+static INLINE void dhd_clear_cis(dhd_pub_t *dhdp) { }
+static INLINE int dhd_alloc_cis(dhd_pub_t *dhdp) { return 0; }
+#endif /* DHD_USE_CISINFO */
+
+#else /* LINUX || linux */
+static INLINE int dhd_process_cid_mac(dhd_pub_t *dhdp, bool prepost) { return 0; }
+#endif /* LINUX || linux */
+
+#if defined(WL_CFG80211) && defined(SUPPORT_DEEP_SLEEP)
+/* Flags to indicate if we distingish power off policy when
+ * user set the memu "Keep Wi-Fi on during sleep" to "Never"
+ */
+extern int trigger_deep_sleep;
+int dhd_deepsleep(struct net_device *dev, int flag);
+#endif /* WL_CFG80211 && SUPPORT_DEEP_SLEEP */
+
+extern void dhd_wait_for_event(dhd_pub_t *dhd, bool *lockvar);
+extern void dhd_wait_event_wakeup(dhd_pub_t*dhd);
+
+#define IFLOCK_INIT(lock)       *lock = 0
+#define IFLOCK(lock)    while (InterlockedCompareExchange((lock), 1, 0))	\
+	NdisStallExecution(1);
+#define IFUNLOCK(lock)  InterlockedExchange((lock), 0)
+#define IFLOCK_FREE(lock)
+#define FW_SUPPORTED(dhd, capa) ((strstr(dhd->fw_capabilities, " " #capa " ") != NULL))
+#ifdef ARP_OFFLOAD_SUPPORT
+#define MAX_IPV4_ENTRIES	8
+void dhd_arp_offload_set(dhd_pub_t * dhd, int arp_mode);
+void dhd_arp_offload_enable(dhd_pub_t * dhd, int arp_enable);
+
+/* dhd_commn arp offload wrapers */
+void dhd_aoe_hostip_clr(dhd_pub_t *dhd, int idx);
+void dhd_aoe_arp_clr(dhd_pub_t *dhd, int idx);
+int dhd_arp_get_arp_hostip_table(dhd_pub_t *dhd, void *buf, int buflen, int idx);
+void dhd_arp_offload_add_ip(dhd_pub_t *dhd, uint32 ipaddr, int idx);
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#ifdef WLTDLS
+int dhd_tdls_enable(struct net_device *dev, bool tdls_on, bool auto_on, struct ether_addr *mac);
+int dhd_tdls_set_mode(dhd_pub_t *dhd, bool wfd_mode);
+#ifdef PCIE_FULL_DONGLE
+int dhd_tdls_update_peer_info(dhd_pub_t *dhdp, wl_event_msg_t *event);
+int dhd_tdls_event_handler(dhd_pub_t *dhd_pub, wl_event_msg_t *event);
+int dhd_free_tdls_peer_list(dhd_pub_t *dhd_pub);
+#endif /* PCIE_FULL_DONGLE */
+#endif /* WLTDLS */
+
+/* Neighbor Discovery Offload Support */
+extern int dhd_ndo_enable(dhd_pub_t * dhd, int ndo_enable);
+int dhd_ndo_add_ip(dhd_pub_t *dhd, char* ipaddr, int idx);
+int dhd_ndo_remove_ip(dhd_pub_t *dhd, int idx);
+
+/* Enhanced ND offload support */
+uint16 dhd_ndo_get_version(dhd_pub_t *dhdp);
+int dhd_ndo_add_ip_with_type(dhd_pub_t *dhdp, char *ipv6addr, uint8 type, int idx);
+int dhd_ndo_remove_ip_by_addr(dhd_pub_t *dhdp, char *ipv6addr, int idx);
+int dhd_ndo_remove_ip_by_type(dhd_pub_t *dhdp, uint8 type, int idx);
+int dhd_ndo_unsolicited_na_filter_enable(dhd_pub_t *dhdp, int enable);
+
+/* ioctl processing for nl80211 */
+int dhd_ioctl_process(dhd_pub_t *pub, int ifidx, struct dhd_ioctl *ioc, void *data_buf);
+
+#if defined(SUPPORT_MULTIPLE_REVISION)
+extern int
+concate_revision(struct dhd_bus *bus, char *fwpath, char *nvpath);
+#endif /* SUPPORT_MULTIPLE_REVISION */
+void dhd_bus_update_fw_nv_path(struct dhd_bus *bus, char *pfw_path, char *pnv_path,
+											char *pclm_path, char *pconf_path);
+void dhd_set_bus_state(void *bus, uint32 state);
+
+/* Remove proper pkts(either one no-frag pkt or whole fragmented pkts) */
+typedef int (*f_droppkt_t)(dhd_pub_t *dhdp, int prec, void* p, bool bPktInQ);
+extern bool dhd_prec_drop_pkts(dhd_pub_t *dhdp, struct pktq *pq, int prec, f_droppkt_t fn);
+
+extern const uint8 wme_fifo2ac[];
+extern const uint8 prio2fifo[];
+#ifdef PROP_TXSTATUS
+int dhd_os_wlfc_block(dhd_pub_t *pub);
+int dhd_os_wlfc_unblock(dhd_pub_t *pub);
+#endif /* PROP_TXSTATUS */
+
+int dhd_os_socram_dump(struct net_device *dev, uint32 *dump_size);
+int dhd_os_get_socram_dump(struct net_device *dev, char **buf, uint32 *size);
+int dhd_common_socram_dump(dhd_pub_t *dhdp);
+
+int dhd_dump(dhd_pub_t *dhdp, char *buf, int buflen);
+int dhd_counters(dhd_pub_t *dhdp, char *buf, int buflen);
+
+int dhd_os_get_version(struct net_device *dev, bool dhd_ver, char **buf, uint32 size);
+void dhd_get_memdump_filename(struct net_device *ndev, char *memdump_path, int len, char *fname);
+uint8* dhd_os_prealloc(dhd_pub_t *dhdpub, int section, uint size, bool kmalloc_if_fail);
+void dhd_os_prefree(dhd_pub_t *dhdpub, void *addr, uint size);
+
+#if defined(CONFIG_DHD_USE_STATIC_BUF)
+#define DHD_OS_PREALLOC(dhdpub, section, size) dhd_os_prealloc(dhdpub, section, size, FALSE)
+#define DHD_OS_PREFREE(dhdpub, addr, size) dhd_os_prefree(dhdpub, addr, size)
+#else
+#define DHD_OS_PREALLOC(dhdpub, section, size) MALLOC(dhdpub->osh, size)
+#define DHD_OS_PREFREE(dhdpub, addr, size) MFREE(dhdpub->osh, addr, size)
+#endif /* defined(CONFIG_DHD_USE_STATIC_BUF) */
+
+#ifdef USE_WFA_CERT_CONF
+enum {
+	SET_PARAM_BUS_TXGLOM_MODE,
+	SET_PARAM_ROAMOFF,
+#ifdef USE_WL_FRAMEBURST
+	SET_PARAM_FRAMEBURST,
+#endif /* USE_WL_FRAMEBURST */
+#ifdef USE_WL_TXBF
+	SET_PARAM_TXBF,
+#endif /* USE_WL_TXBF */
+#ifdef PROP_TXSTATUS
+	SET_PARAM_PROPTX,
+	SET_PARAM_PROPTXMODE,
+#endif /* PROP_TXSTATUS */
+	PARAM_LAST_VALUE
+};
+extern int sec_get_param_wfa_cert(dhd_pub_t *dhd, int mode, uint* read_val);
+#ifdef DHD_EXPORT_CNTL_FILE
+#define VALUENOTSET 0xFFFFFFFFu
+extern uint32 bus_txglom;
+extern uint32 roam_off;
+#ifdef USE_WL_FRAMEBURST
+extern uint32 frameburst;
+#endif /* USE_WL_FRAMEBURST */
+#ifdef USE_WL_TXBF
+extern uint32 txbf;
+#endif /* USE_WL_TXBF */
+#ifdef PROP_TXSTATUS
+extern uint32 proptx;
+#endif /* PROP_TXSTATUS */
+#endif /* DHD_EXPORT_CNTL_FILE */
+#endif /* USE_WFA_CERT_CONF */
+
+#if defined(BCM_ROUTER_DHD)
+#if defined(HNDCTF)
+bool dhd_ctf_hotbrc_check(dhd_pub_t *dhdp, uint8 *eh, int ifidx);
+void dhd_ctf_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+bool dhd_l2_filter_chainable(dhd_pub_t *dhdp, uint8 *eh, int ifidx);
+bool dhd_wet_chainable(dhd_pub_t *dhdap);
+bool dhd_rx_pkt_chainable(dhd_pub_t *dhdp, int ifidx);
+#endif /* HNDCTF */
+extern void dhd_schedule_trap_log_dump(dhd_pub_t *dhdp,
+	uint8 *buf, uint32 size);
+/* When a new flowid is allocated/deallocated, inform dhd. */
+extern void dhd_add_flowid(dhd_pub_t * dhdp, int ifidx,
+                          uint8 ac_prio, void * ea, uint16 flowid);
+extern void dhd_del_flowid(dhd_pub_t * dhdp, int ifidx, uint16 flowid);
+#else  /* ! BCM_ROUTER_DHD */
+#define dhd_add_flowid(pub, ifidx, ac_prio, ea, flowid)  do {} while (0)
+#define dhd_del_flowid(pub, ifidx, flowid)               do {} while (0)
+bool dhd_wet_chainable(dhd_pub_t *dhdp);
+#endif /* ! BCM_ROUTER_DHD */
+
+extern unsigned long dhd_os_general_spin_lock(dhd_pub_t *pub);
+extern void dhd_os_general_spin_unlock(dhd_pub_t *pub, unsigned long flags);
+
+/** Miscellaenous DHD Spin Locks */
+
+/* Enable DHD general spin lock/unlock */
+#define DHD_GENERAL_LOCK(dhdp, flags) \
+	(flags) = dhd_os_general_spin_lock(dhdp)
+#define DHD_GENERAL_UNLOCK(dhdp, flags) \
+	dhd_os_general_spin_unlock((dhdp), (flags))
+
+/* Enable DHD timer spin lock/unlock */
+#define DHD_TIMER_LOCK(lock, flags)     (flags) = osl_spin_lock(lock)
+#define DHD_TIMER_UNLOCK(lock, flags)   osl_spin_unlock(lock, (flags))
+
+/* Enable DHD flowring spin lock/unlock */
+#define DHD_FLOWRING_LOCK(lock, flags)     (flags) = osl_spin_lock(lock)
+#define DHD_FLOWRING_UNLOCK(lock, flags)   osl_spin_unlock((lock), (flags))
+
+/* Enable DHD common flowring info spin lock/unlock */
+#define DHD_FLOWID_LOCK(lock, flags)       (flags) = osl_spin_lock(lock)
+#define DHD_FLOWID_UNLOCK(lock, flags)     osl_spin_unlock((lock), (flags))
+
+/* Enable DHD common flowring list spin lock/unlock */
+#define DHD_FLOWRING_LIST_LOCK(lock, flags)       (flags) = osl_spin_lock(lock)
+#define DHD_FLOWRING_LIST_UNLOCK(lock, flags)     osl_spin_unlock((lock), (flags))
+
+#define DHD_RING_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_RING_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_BUS_LP_STATE_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BUS_LP_STATE_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_BAR1_SWITCH_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BAR1_SWITCH_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_BUS_PWR_REQ_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BUS_PWR_REQ_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#ifdef PCIE_INB_DW
+#define DHD_BUS_DONGLE_DS_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BUS_DONGLE_DS_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+#endif /* PCIE_INB_DW */
+
+/* Enable DHD backplane spin lock/unlock */
+#define DHD_BACKPLANE_ACCESS_LOCK(lock, flags)     (flags) = osl_spin_lock(lock)
+#define DHD_BACKPLANE_ACCESS_UNLOCK(lock, flags)   osl_spin_unlock((lock), (flags))
+
+#define DHD_BUS_INB_DW_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BUS_INB_DW_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+/* Enable DHD TDLS peer list spin lock/unlock */
+#ifdef WLTDLS
+#define DHD_TDLS_LOCK(lock, flags)       (flags) = osl_spin_lock(lock)
+#define DHD_TDLS_UNLOCK(lock, flags)     osl_spin_unlock((lock), (flags))
+#endif /* WLTDLS */
+
+#define DHD_BUS_INB_DW_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BUS_INB_DW_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#ifdef DBG_PKT_MON
+/* Enable DHD PKT MON spin lock/unlock */
+#define DHD_PKT_MON_LOCK(lock, flags)     (flags) = osl_spin_lock(lock)
+#define DHD_PKT_MON_UNLOCK(lock, flags)   osl_spin_unlock(lock, (flags))
+#endif /* DBG_PKT_MON */
+
+#ifdef DHD_PKT_LOGGING
+/* Enable DHD PKT LOG spin lock/unlock */
+#define DHD_PKT_LOG_LOCK(lock, flags)     (flags) = osl_spin_lock(lock)
+#define DHD_PKT_LOG_UNLOCK(lock, flags)   osl_spin_unlock(lock, (flags))
+#endif /* DHD_PKT_LOGGING */
+
+#if defined(linux) || defined(LINUX)
+#define DHD_LINUX_GENERAL_LOCK(dhdp, flags)	DHD_GENERAL_LOCK(dhdp, flags)
+#define DHD_LINUX_GENERAL_UNLOCK(dhdp, flags)	DHD_GENERAL_UNLOCK(dhdp, flags)
+#else
+#define DHD_LINUX_GENERAL_LOCK(dhdp, flags)	do {BCM_REFERENCE(flags);} while (0)
+#define DHD_LINUX_GENERAL_UNLOCK(dhdp, flags)	do {BCM_REFERENCE(flags);} while (0)
+#endif
+
+#define DHD_BUS_INB_DW_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_BUS_INB_DW_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_RX_NAPI_QUEUE_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_RX_NAPI_QUEUE_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_UP_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_UP_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_WAKE_SPIN_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_WAKE_SPIN_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+/*
+ * Temporarily change log dump lock to spin_lock_irqsave as DHD_ERROR/DHD_LOG_MEM
+ * are being called from dhdpcie_bus_isr.
+ * This will be reverted after proper solution is implemented to handle isr prints
+ */
+#define DHD_LOG_DUMP_BUF_LOCK(lock, flags)	(flags) = osl_spin_lock_irq(lock)
+#define DHD_LOG_DUMP_BUF_UNLOCK(lock, flags)	osl_spin_unlock_irq((lock), (flags))
+
+#define DHD_PKT_WAKE_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_PKT_WAKE_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+/* wlan_oob_irq is called in hard irq context */
+#define DHD_OOB_IRQ_LOCK(lock, flags)	(flags) = osl_spin_lock_irq(lock)
+#define DHD_OOB_IRQ_UNLOCK(lock, flags)	osl_spin_unlock_irq((lock), (flags))
+
+#define DHD_IF_STA_LIST_LOCK(lock, flags)	(flags) = osl_spin_lock(lock)
+#define DHD_IF_STA_LIST_UNLOCK(lock, flags)	osl_spin_unlock((lock), (flags))
+
+#define DHD_DBG_RING_LOCK_INIT(osh)		osl_spin_lock_init(osh)
+#define DHD_DBG_RING_LOCK_DEINIT(osh, lock)	osl_spin_lock_deinit(osh, (lock))
+#define DHD_DBG_RING_LOCK(lock, flags)		(flags) = osl_spin_lock(lock)
+#define DHD_DBG_RING_UNLOCK(lock, flags)	osl_spin_unlock((lock), flags)
+
+#ifdef DHD_MEM_STATS
+/* memory stats lock/unlock */
+#define DHD_MEM_STATS_LOCK(lock, flags)     (flags) = osl_spin_lock(lock)
+#define DHD_MEM_STATS_UNLOCK(lock, flags)   osl_spin_unlock((lock), (flags))
+#endif /* DHD_MEM_STATS */
+
+extern void dhd_dump_to_kernelog(dhd_pub_t *dhdp);
+
+#if defined(LINUX) || defined(linux)
+extern void dhd_print_tasklet_status(dhd_pub_t *dhd);
+#ifdef PCIE_INB_DW
+extern bool dhd_check_cfg_in_progress(dhd_pub_t *dhdp);
+#endif
+#else
+static INLINE void dhd_print_tasklet_status(dhd_pub_t *dhd) { }
+static INLINE bool dhd_check_cfg_in_progress(dhd_pub_t *dhdp)
+{ return FALSE; }
+#endif /* LINUX | linux */
+
+#ifdef BCMDBUS
+extern uint dhd_get_rxsz(dhd_pub_t *pub);
+extern void dhd_set_path(dhd_pub_t *pub);
+extern void dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+extern void dhd_bus_clearcounts(dhd_pub_t *dhdp);
+#endif /* BCMDBUS */
+
+#ifdef DHD_L2_FILTER
+extern int dhd_get_parp_status(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_parp_status(dhd_pub_t *dhdp, uint32 idx, int val);
+extern int dhd_get_dhcp_unicast_status(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_dhcp_unicast_status(dhd_pub_t *dhdp, uint32 idx, int val);
+extern int dhd_get_block_ping_status(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_block_ping_status(dhd_pub_t *dhdp, uint32 idx, int val);
+extern int dhd_get_grat_arp_status(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_grat_arp_status(dhd_pub_t *dhdp, uint32 idx, int val);
+extern int dhd_get_block_tdls_status(dhd_pub_t *dhdp, uint32 idx);
+extern int dhd_set_block_tdls_status(dhd_pub_t *dhdp, uint32 idx, int val);
+#endif /* DHD_L2_FILTER */
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+extern int dhd_set_qosmap_up_table(dhd_pub_t *dhdp, uint32 idx, bcm_tlv_t *qos_map_ie);
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+
+typedef struct wl_io_pport {
+	dhd_pub_t *dhd_pub;
+	uint ifidx;
+} wl_io_pport_t;
+
+typedef struct wl_evt_pport {
+	dhd_pub_t *dhd_pub;
+	int *ifidx;
+	void *pktdata;
+	uint data_len;
+	void **data_ptr;
+	void *raw_event;
+} wl_evt_pport_t;
+
+extern void *dhd_pub_shim(dhd_pub_t *dhd_pub);
+#ifdef DHD_FW_COREDUMP
+void* dhd_get_fwdump_buf(dhd_pub_t *dhd_pub, uint32 length);
+#endif /* DHD_FW_COREDUMP */
+
+#if defined(SET_XPS_CPUS)
+int dhd_xps_cpus_enable(struct net_device *net, int enable);
+int custom_xps_map_set(struct net_device *net, char *buf, size_t len);
+void custom_xps_map_clear(struct net_device *net);
+#endif
+
+#if defined(SET_RPS_CPUS)
+int dhd_rps_cpus_enable(struct net_device *net, int enable);
+int custom_rps_map_set(struct netdev_rx_queue *queue, char *buf, size_t len);
+void custom_rps_map_clear(struct netdev_rx_queue *queue);
+#define PRIMARY_INF 0
+#define VIRTUAL_INF 1
+#if defined(CONFIG_MACH_UNIVERSAL7420) || defined(CONFIG_SOC_EXYNOS8890)
+#define RPS_CPUS_MASK "10"
+#define RPS_CPUS_MASK_P2P "10"
+#define RPS_CPUS_MASK_IBSS "10"
+#define RPS_CPUS_WLAN_CORE_ID 4
+#else
+#if defined(DHD_TPUT_PATCH)
+#define RPS_CPUS_MASK "f"
+#define RPS_CPUS_MASK_P2P "f"
+#define RPS_CPUS_MASK_IBSS "f"
+#else
+#define RPS_CPUS_MASK "6"
+#define RPS_CPUS_MASK_P2P "6"
+#define RPS_CPUS_MASK_IBSS "6"
+#endif
+#endif /* CONFIG_MACH_UNIVERSAL7420 || CONFIG_SOC_EXYNOS8890 */
+#endif // endif
+
+int dhd_get_download_buffer(dhd_pub_t	*dhd, char *file_path, download_type_t component,
+	char ** buffer, int *length);
+
+void dhd_free_download_buffer(dhd_pub_t	*dhd, void *buffer, int length);
+
+int dhd_download_blob(dhd_pub_t *dhd, unsigned char *buf,
+		uint32 len, char *iovar);
+
+int dhd_download_blob_cached(dhd_pub_t *dhd, char *file_path,
+	uint32 len, char *iovar);
+
+int dhd_apply_default_txcap(dhd_pub_t *dhd, char *txcap_path);
+int dhd_apply_default_clm(dhd_pub_t *dhd, char *clm_path);
+
+#ifdef SHOW_LOGTRACE
+int dhd_parse_logstrs_file(osl_t *osh, char *raw_fmts, int logstrs_size,
+		dhd_event_log_t *event_log);
+#ifdef DHD_LINUX_STD_FW_API
+int dhd_parse_map_file(osl_t *osh, const void *ptr, uint32 *ramstart,
+		uint32 *rodata_start, uint32 *rodata_end);
+#else
+int dhd_parse_map_file(osl_t *osh, void *file, uint32 *ramstart,
+		uint32 *rodata_start, uint32 *rodata_end);
+#endif /* DHD_LINUX_STD_FW_API */
+#ifdef PCIE_FULL_DONGLE
+int dhd_event_logtrace_infobuf_pkt_process(dhd_pub_t *dhdp, void *pktbuf,
+		dhd_event_log_t *event_data);
+#endif /* PCIE_FULL_DONGLE */
+#endif /* SHOW_LOGTRACE */
+
+/*
+ * control_logtrace:
+ * "0" -> do not print event log messages in any form
+ * "1" -> print event log messages as EL
+ * "2" -> print event log messages as formatted CONSOLE_E if logstrs.bin etc. files are available
+ */
+typedef enum logtrace_ctrl {
+	LOGTRACE_DISABLE = 0,
+	LOGTRACE_RAW_FMT = 1,
+	LOGTRACE_PARSED_FMT = 2
+} logtrace_ctrl_t;
+
+#define DEFAULT_CONTROL_LOGTRACE	LOGTRACE_PARSED_FMT
+#ifndef CUSTOM_CONTROL_LOGTRACE
+#define CUSTOM_CONTROL_LOGTRACE		DEFAULT_CONTROL_LOGTRACE
+#endif
+
+extern uint8 control_logtrace;
+
+#if defined(NDIS)
+bool dhd_is_device_removed(dhd_pub_t *dhd);
+#else
+#define dhd_is_device_removed(x) FALSE
+#define dhd_os_ind_firmware_stall(x)
+#endif /* defined(NDIS) */
+
+#if defined(DHD_FW_COREDUMP)
+#if defined(linux) || defined(LINUX)
+extern void dhd_get_memdump_info(dhd_pub_t *dhd);
+#else
+static INLINE void dhd_get_memdump_info(dhd_pub_t *dhd)
+{ return; }
+#endif /* linux || LINUX */
+#endif /* defined(DHD_FW_COREDUMP) */
+#ifdef BCMASSERT_LOG
+extern void dhd_get_assert_info(dhd_pub_t *dhd);
+#else
+static INLINE void dhd_get_assert_info(dhd_pub_t *dhd) { }
+#endif /* BCMASSERT_LOG */
+
+#if defined(LINUX) || defined(linux)
+#define DMAXFER_FREE(dhdp, dmap) dhd_schedule_dmaxfer_free(dhdp, dmap);
+#else  /* !(LINUX || linux) */
+#define DMAXFER_FREE(dhdp, dmmap) dmaxfer_free_prev_dmaaddr(dhdp, dmmap);
+#endif  /* linux || LINUX */
+
+#if defined(PCIE_FULL_DONGLE)
+extern void dmaxfer_free_prev_dmaaddr(dhd_pub_t *dhdp, dmaxref_mem_map_t *dmmap);
+void dhd_schedule_dmaxfer_free(dhd_pub_t *dhdp, dmaxref_mem_map_t *dmmap);
+
+extern int ring_size_version;
+
+extern uint h2d_max_txpost;
+extern uint h2d_htput_max_txpost;
+extern uint d2h_max_txcpl;
+
+extern uint h2d_max_rxpost;
+extern uint d2h_max_rxcpl;
+
+extern uint h2d_max_ctrlpost;
+extern uint d2h_max_ctrlcpl;
+
+extern uint rx_buf_burst;
+extern uint rx_bufpost_threshold;
+
+#endif  /* PCIE_FULL_DONGLE */
+
+#define DHD_LB_STATS_NOOP	do { /* noop */ } while (0)
+#if defined(DHD_LB_STATS)
+#include <bcmutils.h>
+extern void dhd_lb_stats_init(dhd_pub_t *dhd);
+extern void dhd_lb_stats_deinit(dhd_pub_t *dhd);
+extern void dhd_lb_stats_reset(dhd_pub_t *dhd);
+#ifdef DHD_MEM_STATS
+extern uint64 dhd_lb_mem_usage(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+#endif /* DHD_MEM_STATS */
+extern void dhd_lb_stats_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+extern void dhd_lb_stats_update_napi_histo(dhd_pub_t *dhdp, uint32 count);
+extern void dhd_lb_stats_update_txc_histo(dhd_pub_t *dhdp, uint32 count);
+extern void dhd_lb_stats_update_rxc_histo(dhd_pub_t *dhdp, uint32 count);
+extern void dhd_lb_stats_txc_percpu_cnt_incr(dhd_pub_t *dhdp);
+extern void dhd_lb_stats_rxc_percpu_cnt_incr(dhd_pub_t *dhdp);
+#define DHD_LB_STATS_INIT(dhdp)	dhd_lb_stats_init(dhdp)
+#define DHD_LB_STATS_DEINIT(dhdp) dhd_lb_stats_deinit(dhdp)
+/* Reset is called from common layer so it takes dhd_pub_t as argument */
+#define DHD_LB_STATS_RESET(dhdp) dhd_lb_stats_reset(dhdp)
+#define DHD_LB_STATS_CLR(x)	(x) = 0U
+#define DHD_LB_STATS_INCR(x)	(x) = (x) + 1
+#define DHD_LB_STATS_ADD(x, c)	(x) = (x) + (c)
+#define DHD_LB_STATS_PERCPU_ARR_INCR(x) \
+	{ \
+		int cpu = get_cpu(); put_cpu(); \
+		DHD_LB_STATS_INCR(x[cpu]); \
+	}
+#define DHD_LB_STATS_UPDATE_NAPI_HISTO(dhdp, x)	dhd_lb_stats_update_napi_histo(dhdp, x)
+#else /* !DHD_LB_STATS */
+#define DHD_LB_STATS_INIT(dhdp)	 DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_DEINIT(dhdp) DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_RESET(dhdp) DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_CLR(x)	 DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_INCR(x)	 DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_ADD(x, c)	 DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_PERCPU_ARR_INCR(x)	 DHD_LB_STATS_NOOP
+#define DHD_LB_STATS_UPDATE_NAPI_HISTO(dhd, x) DHD_LB_STATS_NOOP
+#endif /* !DHD_LB_STATS */
+
+#if defined(BCMDBG) && defined(DHD_MACDBG)
+extern void dhd_schedule_macdbg_dump(dhd_pub_t *dhdp);
+#endif /* BCMDBG && DHD_MACDBG */
+
+#ifdef DHD_SSSR_DUMP
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+#define DHD_SSSR_MEMPOOL_SIZE	(2 * 1024 * 1024) /* 2MB size */
+#else
+#define DHD_SSSR_MEMPOOL_SIZE	(1 * 1024 * 1024) /* 1MB size */
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+/* used in sssr_dump_mode */
+#define SSSR_DUMP_MODE_SSSR	0	/* dump both *before* and *after* files */
+#define SSSR_DUMP_MODE_FIS	1	/* dump *after* files only */
+
+extern int dhd_sssr_mempool_init(dhd_pub_t *dhd);
+extern void dhd_sssr_mempool_deinit(dhd_pub_t *dhd);
+extern int dhd_sssr_dump_init(dhd_pub_t *dhd);
+extern void dhd_sssr_dump_deinit(dhd_pub_t *dhd);
+extern int dhdpcie_sssr_dump(dhd_pub_t *dhd);
+extern void dhd_sssr_print_filepath(dhd_pub_t *dhd, char *path);
+extern int dhd_sssr_reg_info_init(dhd_pub_t *dhd);
+extern void dhd_sssr_reg_info_deinit(dhd_pub_t *dhd);
+extern uint dhd_sssr_dig_buf_size(dhd_pub_t *dhdp);
+extern uint dhd_sssr_dig_buf_addr(dhd_pub_t *dhdp);
+extern uint dhd_sssr_mac_buf_size(dhd_pub_t *dhdp, uint8 core_idx);
+extern uint dhd_sssr_mac_xmtaddress(dhd_pub_t *dhdp, uint8 core_idx);
+extern uint dhd_sssr_mac_xmtdata(dhd_pub_t *dhdp, uint8 core_idx);
+
+#define DHD_SSSR_MEMPOOL_INIT(dhdp)	dhd_sssr_mempool_init(dhdp)
+#define DHD_SSSR_MEMPOOL_DEINIT(dhdp) dhd_sssr_mempool_deinit(dhdp)
+#define DHD_SSSR_DUMP_INIT(dhdp)	dhd_sssr_dump_init(dhdp)
+#define DHD_SSSR_DUMP_DEINIT(dhdp) dhd_sssr_dump_deinit(dhdp)
+#define DHD_SSSR_PRINT_FILEPATH(dhdp, path) dhd_sssr_print_filepath(dhdp, path)
+#define DHD_SSSR_REG_INFO_INIT(dhdp)	dhd_sssr_reg_info_init(dhdp)
+#define DHD_SSSR_REG_INFO_DEINIT(dhdp) dhd_sssr_reg_info_deinit(dhdp)
+#else
+#define DHD_SSSR_MEMPOOL_INIT(dhdp)		do { /* noop */ } while (0)
+#define DHD_SSSR_MEMPOOL_DEINIT(dhdp)		do { /* noop */ } while (0)
+#define DHD_SSSR_DUMP_INIT(dhdp)		do { /* noop */ } while (0)
+#define DHD_SSSR_DUMP_DEINIT(dhdp)		do { /* noop */ } while (0)
+#define DHD_SSSR_PRINT_FILEPATH(dhdp, path)	do { /* noop */ } while (0)
+#define DHD_SSSR_REG_INFO_INIT(dhdp)		do { /* noop */ } while (0)
+#define DHD_SSSR_REG_INFO_DEINIT(dhdp)		do { /* noop */ } while (0)
+#endif /* DHD_SSSR_DUMP */
+
+#ifdef DHD_COREDUMP
+#define WLAN_DHD_MEMDUMP_SIZE	(3u * 1024u * 1024u)
+#ifdef DHD_SSSR_DUMP
+#define DHD_MEMDUMP_BUFFER_SIZE	(WLAN_DHD_MEMDUMP_SIZE + DHD_SSSR_MEMPOOL_SIZE)
+#else
+#define DHD_MEMDUMP_BUFFER_SIZE	(WLAN_DHD_MEMDUMP_SIZE)
+#endif
+extern int dhd_coredump_mempool_init(dhd_pub_t *dhd);
+extern void dhd_coredump_mempool_deinit(dhd_pub_t *dhd);
+#define DHD_COREDUMP_MEMPOOL_INIT(dhdp)		dhd_coredump_mempool_init(dhdp)
+#define DHD_COREDUMP_MEMPOOL_DEINIT(dhdp)	dhd_coredump_mempool_deinit(dhdp)
+#else
+#define DHD_COREDUMP_MEMPOOL_INIT(dhdp)		do { /* noop */ } while (0)
+#define DHD_COREDUMP_MEMPOOL_DEINIT(dhdp)	do { /* noop */ } while (0)
+#endif /* DHD_COREDUMP */
+
+#ifdef BCMPCIE
+extern int dhd_prot_debug_info_print(dhd_pub_t *dhd);
+extern bool dhd_bus_skip_clm(dhd_pub_t *dhdp);
+extern void dhd_pcie_dump_rc_conf_space_cap(dhd_pub_t *dhd);
+extern bool dhd_pcie_dump_int_regs(dhd_pub_t *dhd);
+extern bool dhd_pcie_check_lps_d3_acked(dhd_pub_t *dhd);
+void dhd_prot_ctrl_info_print(dhd_pub_t *dhd);
+#else
+#define dhd_prot_debug_info_print(x)
+static INLINE bool dhd_bus_skip_clm(dhd_pub_t *dhd_pub)
+{ return 0; }
+#endif /* BCMPCIE */
+
+#if defined(LINUX) || defined(linux)
+void dhd_show_kirqstats(dhd_pub_t *dhd);
+#else
+static INLINE void dhd_show_kirqstats(dhd_pub_t *dhd)
+{ return; }
+#endif /* defined(LINUX) || defined(linux) */
+
+/* Bitmask used for Join Timeout */
+#define WLC_SSID_MASK          0x01
+#define WLC_WPA_MASK           0x02
+
+#if defined(LINUX) || defined(linux)
+fw_download_status_t dhd_fw_download_status(dhd_pub_t *dhd_pub);
+extern int dhd_start_join_timer(dhd_pub_t *pub);
+extern int dhd_stop_join_timer(dhd_pub_t *pub);
+extern int dhd_start_scan_timer(dhd_pub_t *pub, bool is_escan);
+extern int dhd_stop_scan_timer(dhd_pub_t *pub, bool is_escan, uint16 sync_id);
+extern int dhd_start_cmd_timer(dhd_pub_t *pub);
+extern int dhd_stop_cmd_timer(dhd_pub_t *pub);
+extern int dhd_start_bus_timer(dhd_pub_t *pub);
+extern int dhd_stop_bus_timer(dhd_pub_t *pub);
+extern uint16 dhd_get_request_id(dhd_pub_t *pub);
+extern int dhd_set_request_id(dhd_pub_t *pub, uint16 id, uint32 cmd);
+extern void dhd_clear_join_error(dhd_pub_t *pub, uint32 mask);
+extern void dhd_get_scan_to_val(dhd_pub_t *pub, uint32 *to_val);
+extern void dhd_set_scan_to_val(dhd_pub_t *pub, uint32 to_val);
+extern void dhd_get_join_to_val(dhd_pub_t *pub, uint32 *to_val);
+extern void dhd_set_join_to_val(dhd_pub_t *pub, uint32 to_val);
+extern void dhd_get_cmd_to_val(dhd_pub_t *pub, uint32 *to_val);
+extern void dhd_set_cmd_to_val(dhd_pub_t *pub, uint32 to_val);
+extern void dhd_get_bus_to_val(dhd_pub_t *pub, uint32 *to_val);
+extern void dhd_set_bus_to_val(dhd_pub_t *pub, uint32 to_val);
+extern int dhd_start_timesync_timer(dhd_pub_t *pub);
+extern int dhd_stop_timesync_timer(dhd_pub_t *pub);
+#else
+static INLINE fw_download_status_t dhd_fw_download_status(dhd_pub_t *dhd_pub)
+{ return FW_UNLOADED; }
+static INLINE int dhd_start_join_timer(dhd_pub_t *pub) { return 0; }
+static INLINE int dhd_stop_join_timer(dhd_pub_t *pub) { return 0; }
+static INLINE int dhd_start_scan_timer(dhd_pub_t *pub, bool is_escan) { return 0; }
+static INLINE int dhd_stop_scan_timer(dhd_pub_t *pub, bool is_escan, uint16 sync_id) { return 0; }
+static INLINE int dhd_start_cmd_timer(dhd_pub_t *pub) { return 0; }
+static INLINE int dhd_stop_cmd_timer(dhd_pub_t *pub) { return 0; }
+static INLINE int dhd_start_bus_timer(dhd_pub_t *pub) { return 0; }
+static INLINE int dhd_stop_bus_timer(dhd_pub_t *pub) { return 0; }
+static INLINE uint16 dhd_get_request_id(dhd_pub_t *pub) {  return 0; }
+static INLINE int dhd_set_request_id(dhd_pub_t *pub, uint16 id) {  return 0; }
+static INLINE void dhd_clear_join_error(dhd_pub_t *pub, uint32 mask) {  return; }
+static INLINE void dhd_get_scan_to_val(dhd_pub_t *pub, uint32 *to_val) {  return; }
+static INLINE void dhd_set_scan_to_val(dhd_pub_t *pub, uint32 to_val) {  return; }
+static INLINE void dhd_get_join_to_val(dhd_pub_t *pub, uint32 *to_val) {  return; }
+static INLINE void dhd_set_join_to_val(dhd_pub_t *pub, uint32 to_val) {  return; }
+static INLINE void dhd_get_cmd_to_val(dhd_pub_t *pub, uint32 *to_val) {  return; }
+static INLINE void dhd_set_cmd_to_val(dhd_pub_t *pub, uint32 to_val) {  return; }
+static INLINE void dhd_get_bus_to_val(dhd_pub_t *pub, uint32 *to_val) {  return; }
+static INLINE void dhd_set_bus_to_val(dhd_pub_t *pub, uint32 to_val) {  return; }
+static INLINE int dhd_start_timesync_timer(dhd_pub_t *pub) { return 0; }
+static INLINE int dhd_stop_timesync_timer(dhd_pub_t *pub) { return 0; }
+#endif /* defined(LINUX) || defined(linux) */
+
+#ifdef DHD_PKTID_AUDIT_ENABLED
+#if defined(LINUX) || defined(linux)
+void dhd_pktid_error_handler(dhd_pub_t *dhdp);
+#else /* !(LINUX || linux) */
+static INLINE void dhd_pktid_error_handler(dhd_pub_t *dhdp) { ASSERT(0); }
+#endif /* LINUX || linux */
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+#ifdef DHD_MAP_PKTID_LOGGING
+#if defined(LINUX) || defined(linux)
+extern void dhd_pktid_logging_dump(dhd_pub_t *dhdp);
+#else /* !(LINUX || linux) */
+static INLINE void dhd_pktid_logging_dump(dhd_pub_t *dhdp) { }
+#endif /* LINUX || linux */
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+#define DHD_DISABLE_RUNTIME_PM(dhdp)
+#define DHD_ENABLE_RUNTIME_PM(dhdp)
+#define DHD_STOP_RPM_TIMER(dhdp)
+#define DHD_START_RPM_TIMER(dhdp)
+
+extern bool dhd_prot_is_h2d_ring_empty(dhd_pub_t *dhd, void *prot_info);
+extern void dhd_prot_dump_ring_ptrs(void *prot_info);
+extern bool dhd_prot_check_pending_ctrl_cmpls(dhd_pub_t *dhd);
+
+#if defined(LINUX) || defined(linux)
+#if defined(DHD_TRACE_WAKE_LOCK)
+void dhd_wk_lock_stats_dump(dhd_pub_t *dhdp);
+#endif
+#endif /* LINUX || linux */
+
+extern bool dhd_query_bus_erros(dhd_pub_t *dhdp);
+void dhd_clear_bus_errors(dhd_pub_t *dhdp);
+
+#if (defined(linux) || defined(LINUX)) && defined(CONFIG_64BIT)
+#define DHD_SUPPORT_64BIT
+#endif /* (linux || LINUX) && CONFIG_64BIT */
+
+static INLINE void dhd_schedule_reset(dhd_pub_t *dhdp) {;}
+
+extern void init_dhd_timeouts(dhd_pub_t *pub);
+extern void deinit_dhd_timeouts(dhd_pub_t *pub);
+
+typedef enum timeout_resons {
+	DHD_REASON_COMMAND_TO,
+	DHD_REASON_JOIN_TO,
+	DHD_REASON_SCAN_TO,
+	DHD_REASON_OQS_TO
+} timeout_reasons_t;
+
+#ifdef REPORT_FATAL_TIMEOUTS
+void dhd_send_trap_to_fw_for_timeout(dhd_pub_t * pub, timeout_reasons_t reason);
+#endif
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+extern int dhd_bus_set_device_wake(struct dhd_bus *bus, bool val);
+extern void dhd_bus_dw_deassert(dhd_pub_t *dhd);
+#endif /* defined(PCIE_OOB) || defined(PCIE_INB_DW) */
+extern void dhd_prhex(const char *msg, volatile uchar *buf, uint nbytes, uint32 dbg_level);
+int dhd_tput_test(dhd_pub_t *dhd, tput_test_t *tput_data);
+void dhd_tput_test_rx(dhd_pub_t *dhd, void *pkt);
+static INLINE int dhd_get_max_txbufs(dhd_pub_t *dhdp)
+{ return -1; }
+
+#ifdef FILTER_IE
+int dhd_read_from_file(dhd_pub_t *dhd);
+int dhd_parse_filter_ie(dhd_pub_t *dhd, uint8 *buf);
+int dhd_get_filter_ie_count(dhd_pub_t *dhd, uint8 *buf);
+int dhd_parse_oui(dhd_pub_t *dhd, uint8 *inbuf, uint8 *oui, int len);
+int dhd_check_valid_ie(dhd_pub_t *dhdp, uint8 *buf, int len);
+#endif /* FILTER_IE */
+
+uint16 dhd_prot_get_ioctl_trans_id(dhd_pub_t *dhdp);
+
+#ifdef SET_PCIE_IRQ_CPU_CORE
+enum {
+	DHD_AFFINITY_OFF = 0,
+	DHD_AFFINITY_TPUT_150MBPS,
+	DHD_AFFINITY_TPUT_300MBPS,
+	DHD_AFFINITY_LAST
+};
+
+extern void dhd_set_irq_cpucore(dhd_pub_t *dhdp, int affinity_cmd);
+#endif /* SET_PCIE_IRQ_CPU_CORE */
+#if defined(DHD_HANG_SEND_UP_TEST)
+extern void dhd_make_hang_with_reason(struct net_device *dev, const char *string_num);
+#endif /* DHD_HANG_SEND_UP_TEST */
+
+#ifdef DHD_WAKE_STATUS
+wake_counts_t* dhd_get_wakecount(dhd_pub_t *dhdp);
+#endif /* DHD_WAKE_STATUS */
+extern int dhd_get_random_bytes(uint8 *buf, uint len);
+
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+extern void dhd_set_blob_support(dhd_pub_t *dhdp, char *fw_path);
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+
+int dhd_get_preserve_log_numbers(dhd_pub_t *dhd, uint64 *logset_mask);
+
+#define DHD_WORD_TO_LEN_SHIFT		(2u)       /* WORD to BYTES SHIFT */
+
+#if defined(linux) || defined(LINUX)
+int dhd_export_debug_data(void *mem_buf, void *fp, const void *user_buf, uint32 buf_len, void *pos);
+#else
+static int dhd_export_debug_data(void *mem_buf, void *fp, const void *user_buf,
+	uint32 buf_len, void *pos)
+{ return 0; }
+#endif /* linux || LINUX */
+#if defined(linux) || defined(LINUX)
+#define DHD_PCIE_CONFIG_SAVE(bus)	pci_save_state(bus->dev)
+#define DHD_PCIE_CONFIG_RESTORE(bus)	pci_restore_state(bus->dev)
+#elif defined(NDIS)
+/* For EFI the pcie config space which is saved during init
+* is the one that should always be restored, so NOP for save
+*/
+#define DHD_PCIE_CONFIG_SAVE(bus)
+#define DHD_PCIE_CONFIG_RESTORE(bus)	dhdpcie_config_restore(bus, TRUE)
+#else
+#define DHD_PCIE_CONFIG_SAVE(bus)	do { /* noop */ } while (0)
+#define DHD_PCIE_CONFIG_RESTORE(bus)	do { /* noop */ } while (0)
+#endif /* linux || LINUX */
+
+typedef struct dhd_pkt_parse {
+	uint32 proto;	/* Network layer protocol */
+	uint32 t1;	/* n-tuple */
+	uint32 t2;
+} dhd_pkt_parse_t;
+
+/* ========= RING API functions : exposed to others ============= */
+#define DHD_RING_TYPE_FIXED		1
+#define DHD_RING_TYPE_SINGLE_IDX	2
+uint32 dhd_ring_get_hdr_size(void);
+void *dhd_ring_init(dhd_pub_t *dhdp, uint8 *buf, uint32 buf_size, uint32 elem_size,
+	uint32 elem_cnt, uint32 type);
+void dhd_ring_deinit(dhd_pub_t *dhdp, void *_ring);
+void *dhd_ring_get_first(void *_ring);
+void dhd_ring_free_first(void *_ring);
+void dhd_ring_set_read_idx(void *_ring, uint32 read_idx);
+void dhd_ring_set_write_idx(void *_ring, uint32 write_idx);
+uint32 dhd_ring_get_read_idx(void *_ring);
+uint32 dhd_ring_get_write_idx(void *_ring);
+void *dhd_ring_get_last(void *_ring);
+void *dhd_ring_get_next(void *_ring, void *cur);
+void *dhd_ring_get_prev(void *_ring, void *cur);
+void *dhd_ring_get_empty(void *_ring);
+int dhd_ring_get_cur_size(void *_ring);
+void dhd_ring_lock(void *ring, void *fist_ptr, void *last_ptr);
+void dhd_ring_lock_free(void *ring);
+void *dhd_ring_lock_get_first(void *_ring);
+void *dhd_ring_lock_get_last(void *_ring);
+int dhd_ring_lock_get_count(void *_ring);
+void dhd_ring_lock_free_first(void *ring);
+void dhd_ring_whole_lock(void *ring);
+void dhd_ring_whole_unlock(void *ring);
+
+#ifdef GDB_PROXY
+/** Firmware loaded and GDB proxy may access memory and registers */
+#define DHD_GDB_PROXY_PROBE_ACCESS_ENABLED		0x00000001
+/** Firmware loaded, access to it is enabled but it is not running yet */
+#define DHD_GDB_PROXY_PROBE_FIRMWARE_NOT_RUNNING	0x00000002
+/** Firmware is running */
+#define DHD_GDB_PROXY_PROBE_FIRMWARE_RUNNING		0x00000004
+/** Firmware was started in bootloader mode */
+#define DHD_GDB_PROXY_PROBE_BOOTLOADER_MODE		0x00000008
+/** Host memory code offload present */
+#define DHD_GDB_PROXY_PROBE_HOSTMEM_CODE		0x00000010
+/** Memory dump count field present */
+#define DHD_GDB_PROXY_PROBE_MEMDUMP_COUNT		0x00000020
+
+/* Data structure, returned by "gdb_proxy_probe" iovar */
+typedef struct dhd_gdb_proxy_probe_data {
+	uint32 data_len;		/* Length of data in structure */
+	uint32 magic;			/* Must contain DHD_IOCTL_MAGIC */
+	uint32 flags;			/* Set of DHD_GDB_PROXY_PROBE_... bits */
+	uint32 last_id;			/* 0 or proxy ID last set */
+	uint32 hostmem_code_win_base;	/* Hostmem code window start in ARM physical address space
+					 */
+	uint32 hostmem_code_win_length;	/* Hostmem code window length */
+	uint32 mem_dump_count;		/* Number of memory dumps made so far */
+} dhd_gdb_proxy_probe_data_t;
+
+/* Timeout loop brackets */
+/* Intended usage is to keep repeating timeoutable operation (operation that expected to
+ * succeed but may fail because of timeout) while timeout may be explained by firmware
+ * stop in gdb
+ * Example:
+ *	dhd_pub_t *dhd;
+ *	...
+ *	#ifdef GDB_PROXY
+ *	GDB_PROXY_TIMEOUT_DO(dhd) {
+ *		operation_that_may_succeed_or_timeout();
+ *	} GDB_PROXY_TIMEOUT_WHILE(operation_timed_out());
+ *	#else
+ *	operation_that_may_succeed_or_timeout();
+ *	#endif
+ *	...
+ *	if (operation_timed_out()) {
+ *		bang_memdump_crash_horror();
+ *	}
+ */
+/* Argument is pointer to dhd_pub_t structure */
+#define GDB_PROXY_TIMEOUT_DO(dhd)				\
+{								\
+	uint32 gdb_prev_stop_count;				\
+	dhd_pub_t *gdb_dhd = dhd;				\
+	do {							\
+		gdb_prev_stop_count = gdb_dhd->gdb_proxy_stop_count;
+
+/* Argument should evaluate to true if operation failed because of timeout */
+#define GDB_PROXY_TIMEOUT_WHILE(timeout_cond)							\
+	} while ((timeout_cond) && ((gdb_dhd->gdb_proxy_stop_count != gdb_prev_stop_count) ||	\
+				(gdb_dhd->gdb_proxy_stop_count & GDB_PROXY_STOP_MASK)));	\
+}
+#endif /* GDB_PROXY */
+
+#ifdef PKT_FILTER_SUPPORT
+extern void dhd_pktfilter_offload_set(dhd_pub_t * dhd, char *arg);
+extern void dhd_pktfilter_offload_enable(dhd_pub_t * dhd, char *arg, int enable, int master_mode);
+extern void dhd_pktfilter_offload_delete(dhd_pub_t *dhd, int id);
+#endif
+
+#ifdef DHD_DUMP_PCIE_RINGS
+extern int dhd_d2h_h2d_ring_dump(dhd_pub_t *dhd, void *file, const void *user_buf,
+	unsigned long *file_posn, bool file_write);
+#endif /* DHD_DUMP_PCIE_RINGS */
+
+#ifdef EWP_EDL
+#define DHD_EDL_RING_SIZE (D2HRING_EDL_MAX_ITEM * D2HRING_EDL_ITEMSIZE)
+int dhd_event_logtrace_process_edl(dhd_pub_t *dhdp, uint8 *data,
+		void *evt_decode_data);
+int dhd_edl_mem_init(dhd_pub_t *dhd);
+void dhd_edl_mem_deinit(dhd_pub_t *dhd);
+void dhd_prot_edl_ring_tcm_rd_update(dhd_pub_t *dhd);
+#define DHD_EDL_MEM_INIT(dhdp) dhd_edl_mem_init(dhdp)
+#define DHD_EDL_MEM_DEINIT(dhdp) dhd_edl_mem_deinit(dhdp)
+#define DHD_EDL_RING_TCM_RD_UPDATE(dhdp) \
+	dhd_prot_edl_ring_tcm_rd_update(dhdp)
+#else
+#define DHD_EDL_MEM_INIT(dhdp) do { /* noop */ } while (0)
+#define DHD_EDL_MEM_DEINIT(dhdp) do { /* noop */ } while (0)
+#define DHD_EDL_RING_TCM_RD_UPDATE(dhdp) do { /* noop */ } while (0)
+#endif /* EWP_EDL */
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+void dhd_h2d_log_time_sync_deferred_wq_schedule(dhd_pub_t *dhdp);
+void dhd_h2d_log_time_sync(dhd_pub_t *dhdp);
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+extern void dhd_cleanup_if(struct net_device *net);
+
+void dhd_schedule_logtrace(void *dhd_info);
+int dhd_print_fw_ver_from_file(dhd_pub_t *dhdp, char *fwpath);
+
+#if defined(LINUX) || defined(linux)
+/* configuration of ecounters. API's tp start/stop. currently supported only for linux */
+extern int dhd_ecounter_configure(dhd_pub_t *dhd, bool enable);
+extern int dhd_start_ecounters(dhd_pub_t *dhd);
+extern int dhd_stop_ecounters(dhd_pub_t *dhd);
+extern int dhd_start_event_ecounters(dhd_pub_t *dhd);
+extern int dhd_stop_event_ecounters(dhd_pub_t *dhd);
+#endif /* LINUX || linux */
+
+#define DHD_DUMP_TYPE_NAME_SIZE		32
+#define DHD_DUMP_FILE_PATH_SIZE		256
+#define DHD_DUMP_FILE_COUNT_MAX		5
+#define DHD_DUMP_TYPE_COUNT_MAX		10
+
+#ifdef DHD_DUMP_MNGR
+typedef struct _DFM_elem {
+	char type_name[DHD_DUMP_TYPE_NAME_SIZE];
+	char file_path[DHD_DUMP_FILE_COUNT_MAX][DHD_DUMP_FILE_PATH_SIZE];
+	int file_idx;
+} DFM_elem_t;
+
+typedef struct _dhd_dump_file_manage {
+	DFM_elem_t elems[DHD_DUMP_TYPE_COUNT_MAX];
+} dhd_dump_file_manage_t;
+
+extern void dhd_dump_file_manage_enqueue(dhd_pub_t *dhd, char *dump_path, char *fname);
+#endif /* DHD_DUMP_MNGR */
+
+#define HD_PREFIX_SIZE  2   /* hexadecimal prefix size */
+#define HD_BYTE_SIZE    2   /* hexadecimal byte size */
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+extern void dhd_axi_error(dhd_pub_t *dhd);
+#ifdef DHD_USE_WQ_FOR_DNGL_AXI_ERROR
+extern void dhd_axi_error_dispatch(dhd_pub_t *dhdp);
+#endif /* DHD_USE_WQ_FOR_DNGL_AXI_ERROR */
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#ifdef DHD_STATUS_LOGGING
+#include <dhd_statlog.h>
+#else
+#define ST(x)		0
+#define STDIR(x)	0
+#define DHD_STATLOG_CTRL(dhdp, stat, ifidx, reason) \
+	do { /* noop */ } while (0)
+#define DHD_STATLOG_DATA(dhdp, stat, ifidx, dir, cond) \
+	do { BCM_REFERENCE(cond); } while (0)
+#define DHD_STATLOG_DATA_RSN(dhdp, stat, ifidx, dir, reason) \
+	do { /* noop */ } while (0)
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef SUPPORT_SET_TID
+enum dhd_set_tid_mode {
+	/* Disalbe changing TID */
+	SET_TID_OFF = 0,
+	/* Change TID for all UDP frames */
+	SET_TID_ALL_UDP,
+	/* Change TID for UDP frames based on UID */
+	SET_TID_BASED_ON_UID
+};
+
+#define DEFAULT_SET_TID_TARGET_UID	0
+#define DEFAULT_SET_TID_TARGET_PRIO	0
+#if defined(linux) || defined(LINUX)
+extern void dhd_set_tid_based_on_uid(dhd_pub_t *dhdp, void *pkt);
+#else
+static INLINE void dhd_set_tid_based_on_uid(dhd_pub_t *dhdp, void *pkt) { return; }
+#endif /* linux || LINUX */
+#endif /* SUPPORT_SET_TID */
+
+#ifdef CONFIG_SILENT_ROAM
+extern int dhd_sroam_set_mon(dhd_pub_t *dhd, bool set);
+typedef wlc_sroam_info_v1_t wlc_sroam_info_t;
+#endif /* CONFIG_SILENT_ROAM */
+
+#ifdef DHD_DUMP_FILE_WRITE_FROM_KERNEL
+#define FILE_NAME_HAL_TAG	""
+#else
+#define FILE_NAME_HAL_TAG	"_hal" /* The tag name concatenated by HAL */
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+
+void dhd_bus_counters(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+
+#define DHD_HISTOGRAM_ENTRIES	(14u)
+#define DHD_HISTOGRAM_SIZE	(sizeof(uint64) * DHD_HISTOGRAM_ENTRIES)
+
+void dhd_histo_update(dhd_pub_t *dhd, uint64 *histo, uint32 value);
+void dhd_histo_clear(dhd_pub_t *dhd, uint64 *histo);
+void dhd_histo_tag_dump(dhd_pub_t *dhd, struct bcmstrbuf *strbuf, char *tagname);
+void dhd_histo_dump(dhd_pub_t *dhd, struct bcmstrbuf *strbuf, uint64 *histo, char *histoname);
+uint64 *dhd_histo_init(dhd_pub_t *dhd);
+void dhd_histo_deinit(dhd_pub_t *dhd, uint64 *histo);
+
+/* Given a number 'n' returns 'm' that is next larger power of 2 after n */
+static inline uint32 next_larger_power2(uint32 num)
+{
+	if (num) {
+		num--;
+		num |= (num >> 1);
+		num |= (num >> 2);
+		num |= (num >> 4);
+		num |= (num >> 8);
+		num |= (num >> 16);
+	}
+	return (num + 1);
+}
+
+extern struct dhd_if * dhd_get_ifp(dhd_pub_t *dhdp, uint32 ifidx);
+uint8 dhd_d11_slices_num_get(dhd_pub_t *dhdp);
+#if defined(WL_AUTO_QOS) && defined(DHD_QOS_ON_SOCK_FLOW)
+extern void dhd_wl_sock_qos_set_status(dhd_pub_t *dhdp, unsigned long on_off);
+#endif /* WL_AUTO_QOS && DHD_QOS_ON_SOCK_FLOW */
+
+extern void *dhd_get_roam_evt(dhd_pub_t *dhdp);
+#if defined(DISABLE_HE_ENAB) || defined(CUSTOM_CONTROL_HE_ENAB)
+extern int dhd_control_he_enab(dhd_pub_t * dhd, uint8 he_enab);
+extern uint8 control_he_enab;
+#endif /* DISABLE_HE_ENAB  || CUSTOM_CONTROL_HE_ENAB */
+
+#ifdef DHD_SDTC_ETB_DUMP
+
+#define DHD_SDTC_ETB_MEMPOOL_SIZE (33 * 1024)
+extern int dhd_sdtc_etb_mempool_init(dhd_pub_t *dhd);
+extern void dhd_sdtc_etb_mempool_deinit(dhd_pub_t *dhd);
+extern void dhd_sdtc_etb_init(dhd_pub_t *dhd);
+extern void dhd_sdtc_etb_deinit(dhd_pub_t *dhd);
+extern void dhd_sdtc_etb_dump(dhd_pub_t *dhd);
+extern int dhd_sdtc_etb_hal_file_dump(void *dev, const void *user_buf, uint32 len);
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#ifdef DHD_TX_PROFILE
+/* whether or not a dhd_tx_profile_protocol_t matches with data in a packet */
+bool dhd_protocol_matches_profile(uint8 *p, int plen,
+	const dhd_tx_profile_protocol_t *proto, bool is_host_sfhllc);
+int dhd_tx_profile_attach(dhd_pub_t *dhdp);
+int dhd_tx_profile_detach(dhd_pub_t *dhdp);
+#endif /* defined (DHD_TX_PROFILE) */
+#if defined(DHD_LB_RXP)
+uint32 dhd_lb_rxp_process_qlen(dhd_pub_t *dhdp);
+/*
+ * To avoid OOM, Flow control will be kicked in when packet size in process_queue
+ * crosses LB_RXP_STOP_THR * rcpl ring size * 1500(pkt size) and will stop
+ * when it goes below LB_RXP_STRT_THR * rcpl ring size * 1500(pkt size)
+ * By default flow control is diabled. i.e tunables set to 0/0.
+ * to enable flow control, these tunables need to be passed from platform Makefiles.
+ */
+#ifndef LB_RXP_STOP_THR
+#define LB_RXP_STOP_THR 0
+#endif /* LB_RXP_STOP_THR */
+#ifndef LB_RXP_STRT_THR
+#define LB_RXP_STRT_THR 0
+#endif /* LB_RXP_STRT_THR */
+
+#endif /* DHD_LB_RXP */
+#ifdef DHD_SUPPORT_HDM
+extern bool hdm_trigger_init;
+extern int dhd_module_init_hdm(void);
+extern void dhd_hdm_wlan_sysfs_init(void);
+extern void dhd_hdm_wlan_sysfs_deinit(struct work_struct *);
+extern int hdm_wifi_support;
+#define SYSFS_DEINIT_MS 10
+#endif /* DHD_SUPPORT_HDM */
+
+#if defined(linux) || defined(LINUX)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_LIMIT_OUTPUT)
+void dhd_ctrl_tcp_limit_output_bytes(int level);
+#endif /* LINUX_VERSION_CODE > 4.19.0 && DHD_TCP_LIMIT_OUTPUT */
+#endif /* linux || LINUX */
+
+#if defined(__linux__)
+extern void dhd_schedule_delayed_dpc_on_dpc_cpu(dhd_pub_t *dhdp, ulong delay);
+extern void dhd_handle_pktdata(dhd_pub_t *dhdp, int ifidx, void *pkt, uint8 *pktdata,
+	uint32 pktid, uint32 pktlen, uint16 *pktfate, uint8 *dhd_udr, uint8 *dhd_igmp,
+	bool tx, int pkt_wake, bool pkt_log);
+#else
+static INLINE void dhd_schedule_delayed_dpc_on_dpc_cpu(dhd_pub_t *dhdp, ulong delay)
+	{ return; }
+static INLINE void dhd_handle_pktdata(dhd_pub_t *dhdp, int ifidx, void *pkt, uint8 *pktdata,
+	uint32 pktid, uint32 pktlen, uint16 *pktfate, uint8 *dhd_udr, uint8 *dhd_igmp,
+	bool tx, int pkt_wake, bool pkt_log) { return; }
+#endif /* __linux */
+
+#if defined(BCMPCIE) && defined(__linux__)
+extern int dhd_check_shinfo_nrfrags(dhd_pub_t *dhdp, void *pktbuf, dmaaddr_t *pa, uint32 pktid);
+#else
+static INLINE int dhd_check_shinfo_nrfrags(dhd_pub_t *dhdp, void *pktbuf, dmaaddr_t *pa,
+	uint32 pktid) { return BCME_OK; }
+#endif /* BCMPCIE && __linux__ */
+
+int dhd_schedule_socram_dump(dhd_pub_t *dhdp);
+
+extern int dhd_dev_set_accel_force_reg_on(struct net_device *dev);
+
+#if defined(DHD_DEBUGABILITY_LOG_DUMP_RING) || defined(DHD_DEBUGABILITY_EVENT_RING)
+#ifndef DEBUGABILITY
+#error "DHD_DEBUGABILITY_LOG_DUMP_RING or DHD_DEBUGABILITY_EVENT_RING without DEBUGABILITY"
+#endif /* DEBUGABILITY */
+#endif /* defined(DHD_DEBUGABILITY_LOG_DUMP_RING) || defined(DHD_DEBUGABILITY_EVENT_RING) */
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+typedef enum dhd_alert_reason_codes {
+	ALERT_BCN_LOST = 0,
+	ALERT_TX_STALL = 1,
+	ALERT_RX_STALL = 2,
+	ALERT_IOCTL_TIMEOUT = 3,
+	ALERT_DONGLE_TRAP = 4,
+	ALERT_SCAN_STALL = 5,
+	ALERT_SCAN_ERR = 6,
+	ALERT_SCAN_BUSY = 7,
+	ALERT_FW_QUEUE_STALL = 8
+} dhd_alert_reason_codes_t;
+int dhd_os_send_alert_message(dhd_pub_t *dhdp);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+
+#ifdef RX_PKT_POOL
+#define MAX_RX_PKT_POOL	(1024)
+void dhd_rx_pktpool_create(struct dhd_info *dhd, uint16 len);
+void * BCMFASTPATH(dhd_rxpool_pktget)(osl_t *osh, struct dhd_info *dhd, uint16 len);
+#endif /* RX_PKT_POOL */
+
+int dhd_rxf_thread(void *data);
+void dhd_sched_rxf(dhd_pub_t *dhdp, void *skb);
+int dhd_os_wake_lock_rx_timeout_enable(dhd_pub_t *pub, int val);
+
+#if defined(__linux__)
+#ifdef DHD_SUPPORT_VFS_CALL
+static INLINE struct file *dhd_filp_open(const char *filename, int flags, int mode)
+{
+	return filp_open(filename, flags, mode);
+}
+
+static INLINE int dhd_filp_close(void *image, void *id)
+{
+	return filp_close((struct file *)image, id);
+}
+
+static INLINE int dhd_i_size_read(const struct inode *inode)
+{
+	return i_size_read(inode);
+}
+
+static INLINE int dhd_kernel_read_compat(struct file *fp, loff_t pos, void *buf, size_t count)
+{
+	return kernel_read_compat(fp, pos, buf, count);
+}
+
+static INLINE int dhd_vfs_read(struct file *filep, char *buf, size_t size, loff_t *pos)
+{
+	return vfs_read(filep, buf, size, pos);
+}
+
+static INLINE int dhd_vfs_write(struct file *filep, char *buf, size_t size, loff_t *pos)
+{
+	return vfs_write(filep, buf, size, pos);
+}
+
+static INLINE int dhd_vfs_fsync(struct file *filep, int datasync)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	return vfs_fsync(filep, datasync);
+#else
+	return vfs_fsync(filep, filep->f_path.dentry, 0);
+#endif
+}
+
+static INLINE long dhd_vfs_size_read(struct file *fp)
+{
+	return dhd_i_size_read(file_inode(fp));
+}
+
+static INLINE int dhd_kern_path(char *name, int flags, struct path *file_path)
+{
+	return kern_path(name, flags, file_path);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))
+#define DHD_VFS_INODE(dir) (dir->d_inode)
+#else
+#define DHD_VFS_INODE(dir) d_inode(dir)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0)) && (RHEL_RELEASE_CODE < \
+	RHEL_RELEASE_VERSION(7, 6))
+#define DHD_VFS_UNLINK(dir, file_path, c) vfs_unlink(DHD_VFS_INODE(dir), file_path.dentry)
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+#define DHD_VFS_UNLINK(dir, file_path, c)  \
+	vfs_unlink(mnt_idmap(file_path.mnt), DHD_VFS_INODE(dir), file_path.dentry, c)
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
+#define DHD_VFS_UNLINK(dir, file_path, c)  \
+	vfs_unlink(mnt_user_ns(file_path.mnt), DHD_VFS_INODE(dir), file_path.dentry, c)
+#else
+#define DHD_VFS_UNLINK(dir, file_path, c) vfs_unlink(DHD_VFS_INODE(dir), file_path.dentry, c)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) */
+#else
+#define DHD_VFS_UNLINK(dir, file_path, c) 0
+
+static INLINE struct file *dhd_filp_open(const char *filename, int flags, int mode)
+	{ printf("%s: DHD_SUPPORT_VFS_CALL not defined\n", __FUNCTION__); return NULL; }
+static INLINE int dhd_filp_close(void *image, void *id)
+	{ return 0; }
+static INLINE int dhd_i_size_read(const struct inode *inode)
+	{ return 0; }
+static INLINE int dhd_kernel_read_compat(struct file *fp, loff_t pos, void *buf, size_t count)
+	{ return 0; }
+static INLINE int dhd_vfs_read(struct file *filep, char *buf, size_t size, loff_t *pos)
+	{ return 0; }
+static INLINE int dhd_vfs_write(struct file *filep, char *buf, size_t size, loff_t *pos)
+	{ return 0; }
+static INLINE int dhd_vfs_fsync(struct file *filep, int datasync)
+	{ return 0; }
+static INLINE long dhd_vfs_size_read(struct file *fp)
+	{ return 0; }
+static INLINE int dhd_kern_path(char *name, int flags, struct path *file_path)
+	{ return 0; }
+#endif /* DHD_SUPPORT_VFS_CALL */
+#endif /* __linux__ */
+#ifdef WL_TWT
+extern int dhd_config_twt_event_mask_in_suspend(dhd_pub_t *dhdp, bool suspend);
+extern int dhd_send_twt_info_suspend(dhd_pub_t *dhdp, bool suspend);
+#endif /* WL_TWT */
+#if defined(DHD_WAKE_STATUS_PRINT) && defined(DHD_WAKE_RX_STATUS) && \
+	defined(DHD_WAKE_STATUS)
+void dhd_dump_wake_status(dhd_pub_t *dhdp, wake_counts_t *wcp, struct ether_header *eh);
+#endif /* DHD_WAKE_STATUS_PRINT && DHD_WAKE_RX_STATUS && DHD_WAKE_STATUS */
+#ifdef SUPPORT_OTA_UPDATE
+void dhd_ota_buf_clean(dhd_pub_t *dhdp);
+#endif /* SUPPORT_OTA_UPDATE */
+#if defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P)
+extern uint32 dhd_get_concurrent_capabilites(dhd_pub_t *dhd);
+#endif
+#ifdef DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND
+#ifndef CUSTOM_RETRUN_TO_SLEEP_TIME_SUSPEND
+#define CUSTOM_RETRUN_TO_SLEEP_TIME_SUSPEND 20
+#endif
+#ifndef CUSTOM_RETRUN_TO_SLEEP_TIME_DEFAULT
+#define CUSTOM_RETRUN_TO_SLEEP_TIME_DEFAULT 200
+#endif
+int dhd_config_rts_in_suspend(dhd_pub_t *dhdp, bool suspend);
+#endif /* DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND */
+
+extern void dhd_unregister_net(struct net_device *net, bool need_rtnl_lock);
+extern int dhd_register_net(struct net_device *net, bool need_rtnl_lock);
+
+#ifdef WL_MONITOR
+void dhd_set_monitor(dhd_pub_t *pub, int ifidx, int val);
+#ifdef BCMSDIO
+extern void dhd_rx_mon_pkt_sdio(dhd_pub_t *dhdp, void *pkt, int ifidx);
+bool dhd_monitor_enabled(dhd_pub_t *dhd, int ifidx);
+#endif /* BCMSDIO */
+#endif /* WL_MONITOR */
+#endif /* _dhd_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_bitpack.h b/drivers/net/wireless/ap12275/dhd_bitpack.h
new file mode 100644
index 000000000000..8332fb790a78
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_bitpack.h
@@ -0,0 +1,33 @@
+/*
+ * Bit packing and Base64 utils for EWP
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __BITPACK_H_
+#define __BITPACK_H_
+
+#define BYTE_SIZE(a) ((a + 7)/8)
+
+extern int32 dhd_bit_pack(char *buf, int32 buf_len, int bit_offset, uint32 data, int32 bit_length);
+extern int32 dhd_base64_encode(char* in_buf, int32 in_buf_len, char* out_buf, int32 out_buf_len);
+#endif /* __BITPACK_H */
diff --git a/drivers/net/wireless/ap12275/dhd_bus.h b/drivers/net/wireless/ap12275/dhd_bus.h
new file mode 100644
index 000000000000..e5067ea47a13
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_bus.h
@@ -0,0 +1,432 @@
+/*
+ * Header file describing the internal (inter-module) DHD interfaces.
+ *
+ * Provides type definitions and function prototypes used to link the
+ * DHD OS, bus, and protocol modules.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef _dhd_bus_h_
+#define _dhd_bus_h_
+
+extern int dbus_up(struct dhd_bus *pub);
+extern int dbus_stop(struct dhd_bus *pub);
+/*
+ * Exported from dhd bus module (dhd_usb, dhd_sdio)
+ */
+
+/* global variable for the bus */
+extern struct dhd_bus *g_dhd_bus;
+
+/* Indicate (dis)interest in finding dongles. */
+extern int dhd_bus_register(void);
+extern void dhd_bus_unregister(void);
+
+/* Download firmware image and nvram image */
+extern int dhd_bus_download_firmware(struct dhd_bus *bus, osl_t *osh,
+	char *fw_path, char *nv_path, char *clm_path, char *conf_path);
+#if defined(BT_OVER_SDIO)
+extern int dhd_bus_download_btfw(struct dhd_bus *bus, osl_t *osh, char *btfw_path);
+#endif /* defined (BT_OVER_SDIO) */
+
+/* Stop bus module: clear pending frames, disable data flow */
+extern void dhd_bus_stop(struct dhd_bus *bus, bool enforce_mutex);
+
+/* Initialize bus module: prepare for communication w/dongle */
+extern int dhd_bus_init(dhd_pub_t *dhdp, bool enforce_mutex);
+
+/* Get the Bus Idle Time */
+extern void dhd_bus_getidletime(dhd_pub_t *dhdp, int *idletime);
+
+/* Set the Bus Idle Time */
+extern void dhd_bus_setidletime(dhd_pub_t *dhdp, int idle_time);
+
+/* Send a data frame to the dongle.  Callee disposes of txp. */
+#ifdef BCMPCIE
+extern int dhd_bus_txdata(struct dhd_bus *bus, void *txp, uint8 ifidx);
+#else
+extern int dhd_bus_txdata(struct dhd_bus *bus, void *txp);
+#endif
+
+#ifdef BCMPCIE
+extern uint16 dhd_prot_get_rxbufpost_sz(dhd_pub_t *dhd);
+extern uint16 dhd_prot_get_rxbufpost_alloc_sz(dhd_pub_t *dhd);
+extern uint16 dhd_prot_get_h2d_rx_post_active(dhd_pub_t *dhd);
+extern uint16 dhd_prot_get_d2h_rx_cpln_active(dhd_pub_t *dhd);
+extern void dhdpcie_cto_recovery_handler(dhd_pub_t *dhd);
+#endif /* BCMPCIE */
+
+/* Send/receive a control message to/from the dongle.
+ * Expects caller to enforce a single outstanding transaction.
+ */
+extern int dhd_bus_txctl(struct dhd_bus *bus, uchar *msg, uint msglen);
+extern int dhd_bus_rxctl(struct dhd_bus *bus, uchar *msg, uint msglen);
+
+/* Watchdog timer function */
+extern bool dhd_bus_watchdog(dhd_pub_t *dhd);
+
+extern int dhd_bus_oob_intr_register(dhd_pub_t *dhdp);
+extern void dhd_bus_oob_intr_unregister(dhd_pub_t *dhdp);
+extern void dhd_bus_oob_intr_set(dhd_pub_t *dhdp, bool enable);
+extern int dhd_bus_get_oob_irq_num(dhd_pub_t *dhdp);
+extern struct device * dhd_bus_to_dev(struct dhd_bus *bus);
+extern void dhd_bus_dev_pm_stay_awake(dhd_pub_t *dhdpub);
+extern void dhd_bus_dev_pm_relax(dhd_pub_t *dhdpub);
+extern bool dhd_bus_dev_pm_enabled(dhd_pub_t *dhdpub);
+
+/* Device console input function */
+extern int dhd_bus_console_in(dhd_pub_t *dhd, uchar *msg, uint msglen);
+#ifdef CONSOLE_DPC
+extern int dhd_bus_txcons(dhd_pub_t *dhd, uchar *msg, uint msglen);
+#endif
+
+/* Deferred processing for the bus, return TRUE requests reschedule */
+extern bool dhd_bus_dpc(struct dhd_bus *bus);
+extern void dhd_bus_isr(bool * InterruptRecognized, bool * QueueMiniportHandleInterrupt, void *arg);
+
+/* Check for and handle local prot-specific iovar commands */
+extern int dhd_bus_iovar_op(dhd_pub_t *dhdp, const char *name,
+                            void *params, uint plen, void *arg, uint len, bool set);
+
+/* Add bus dump output to a buffer */
+extern void dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+extern void dhd_bus_dump_flowring(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+
+/* Clear any bus counters */
+extern void dhd_bus_clearcounts(dhd_pub_t *dhdp);
+
+/* return the dongle chipid */
+extern uint dhd_bus_chip(struct dhd_bus *bus);
+
+/* return the dongle chiprev */
+extern uint dhd_bus_chiprev(struct dhd_bus *bus);
+
+/* Set user-specified nvram parameters. */
+extern void dhd_bus_set_nvram_params(struct dhd_bus * bus, const char *nvram_params);
+
+extern void *dhd_bus_pub(struct dhd_bus *bus);
+extern void *dhd_bus_txq(struct dhd_bus *bus);
+extern void *dhd_bus_sih(struct dhd_bus *bus);
+extern uint dhd_bus_hdrlen(struct dhd_bus *bus);
+#ifdef BCMSDIO
+extern void dhd_bus_set_dotxinrx(struct dhd_bus *bus, bool val);
+/* return sdio io status */
+extern uint8 dhd_bus_is_ioready(struct dhd_bus *bus);
+#else
+#define dhd_bus_set_dotxinrx(a, b) do {} while (0)
+#endif
+
+#define DHD_SET_BUS_STATE_DOWN(_bus)  do { \
+	(_bus)->dhd->busstate = DHD_BUS_DOWN; \
+} while (0)
+
+/* Register a dummy SDIO client driver in order to be notified of new SDIO device */
+extern int dhd_bus_reg_sdio_notify(void* semaphore);
+extern void dhd_bus_unreg_sdio_notify(void);
+extern void dhd_txglom_enable(dhd_pub_t *dhdp, bool enable);
+extern int dhd_bus_get_ids(struct dhd_bus *bus, uint32 *bus_type, uint32 *bus_num,
+	uint32 *slot_num);
+
+#if defined(DHD_FW_COREDUMP) && (defined(BCMPCIE) || defined(BCMSDIO))
+extern int dhd_bus_mem_dump(dhd_pub_t *dhd);
+extern int dhd_bus_get_mem_dump(dhd_pub_t *dhdp);
+#else
+#define dhd_bus_mem_dump(x)
+#define dhd_bus_get_mem_dump(x)
+#endif /* DHD_FW_COREDUMP && (BCMPCIE || BCMSDIO) */
+
+#ifdef BCMPCIE
+enum {
+	/* Scratch buffer confiuguration update */
+	D2H_DMA_SCRATCH_BUF,
+	D2H_DMA_SCRATCH_BUF_LEN,
+
+	/* DMA Indices array buffers for: H2D WR and RD, and D2H WR and RD */
+	H2D_DMA_INDX_WR_BUF, /* update H2D WR dma indices buf base addr to dongle */
+	H2D_DMA_INDX_RD_BUF, /* update H2D RD dma indices buf base addr to dongle */
+	D2H_DMA_INDX_WR_BUF, /* update D2H WR dma indices buf base addr to dongle */
+	D2H_DMA_INDX_RD_BUF, /* update D2H RD dma indices buf base addr to dongle */
+
+	/* DHD sets/gets WR or RD index, in host's H2D and D2H DMA indices buffer */
+	H2D_DMA_INDX_WR_UPD, /* update H2D WR index in H2D WR dma indices buf */
+	H2D_DMA_INDX_RD_UPD, /* update H2D RD index in H2D RD dma indices buf */
+	D2H_DMA_INDX_WR_UPD, /* update D2H WR index in D2H WR dma indices buf */
+	D2H_DMA_INDX_RD_UPD, /* update D2H RD index in D2H RD dma indices buf */
+
+	/* DHD Indices array buffers and update for: H2D flow ring WR */
+	H2D_IFRM_INDX_WR_BUF, /* update H2D WR dma indices buf base addr to dongle */
+	H2D_IFRM_INDX_WR_UPD, /* update H2D WR dma indices buf base addr to dongle */
+
+	/* H2D and D2H Mailbox data update */
+	H2D_MB_DATA,
+	D2H_MB_DATA,
+
+	/* (Common) MsgBuf Ring configuration update */
+	RING_BUF_ADDR,       /* update ring base address to dongle */
+	RING_ITEM_LEN,       /* update ring item size to dongle */
+	RING_MAX_ITEMS,      /* update ring max items to dongle */
+
+	/* Update of WR or RD index, for a MsgBuf Ring */
+	RING_RD_UPD,         /* update ring read index from/to dongle */
+	RING_WR_UPD,         /* update ring write index from/to dongle */
+
+	TOTAL_LFRAG_PACKET_CNT,
+	MAX_HOST_RXBUFS,
+	HOST_API_VERSION,
+#ifdef D2H_MINIDUMP
+	DNGL_TO_HOST_TRAP_ADDR_LEN,
+#endif /* D2H_MINIDUMP */
+	DNGL_TO_HOST_TRAP_ADDR,
+	HOST_SCB_ADDR,		/* update host scb base address to dongle */
+	HOST_CAP2,		/* Host capability 2 flags */
+};
+
+typedef void (*dhd_mb_ring_t) (struct dhd_bus *, uint32);
+typedef void (*dhd_mb_ring_2_t) (struct dhd_bus *, uint32, bool);
+extern void dhd_bus_cmn_writeshared(struct dhd_bus *bus, void * data, uint32 len, uint8 type,
+	uint16 ringid);
+extern void dhd_bus_ringbell(struct dhd_bus *bus, uint32 value);
+extern void dhd_bus_ringbell_2(struct dhd_bus *bus, uint32 value, bool devwake);
+extern void dhd_bus_cmn_readshared(struct dhd_bus *bus, void* data, uint8 type, uint16 ringid);
+extern uint32 dhd_bus_get_sharedflags(struct dhd_bus *bus);
+extern void dhd_bus_rx_frame(struct dhd_bus *bus, void* pkt, int ifidx, uint pkt_count);
+extern void dhd_bus_start_queue(struct dhd_bus *bus);
+extern void dhd_bus_stop_queue(struct dhd_bus *bus);
+extern dhd_mb_ring_t dhd_bus_get_mbintr_fn(struct dhd_bus *bus);
+extern dhd_mb_ring_2_t dhd_bus_get_mbintr_2_fn(struct dhd_bus *bus);
+extern void dhd_bus_write_flow_ring_states(struct dhd_bus *bus,
+	void * data, uint16 flowid);
+extern void dhd_bus_read_flow_ring_states(struct dhd_bus *bus,
+	void * data, uint8 flowid);
+extern int dhd_bus_flow_ring_create_request(struct dhd_bus *bus, void *flow_ring_node);
+extern void dhd_bus_clean_flow_ring(struct dhd_bus *bus, void *flow_ring_node);
+extern void dhd_bus_flow_ring_create_response(struct dhd_bus *bus, uint16 flow_id, int32 status);
+extern int dhd_bus_flow_ring_delete_request(struct dhd_bus *bus, void *flow_ring_node);
+extern void dhd_bus_flow_ring_delete_response(struct dhd_bus *bus, uint16 flowid, uint32 status);
+extern int dhd_bus_flow_ring_flush_request(struct dhd_bus *bus, void *flow_ring_node);
+extern void dhd_bus_flow_ring_flush_response(struct dhd_bus *bus, uint16 flowid, uint32 status);
+extern uint32 dhd_bus_max_h2d_queues(struct dhd_bus *bus);
+extern int dhd_bus_schedule_queue(struct dhd_bus *bus, uint16 flow_id, bool txs,
+		uint32 bound, bool *is_qempty);
+
+#ifdef IDLE_TX_FLOW_MGMT
+extern void dhd_bus_flow_ring_resume_response(struct dhd_bus *bus, uint16 flowid, int32 status);
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#ifdef BCMDBG
+extern void
+dhd_bus_flow_ring_cnt_update(struct dhd_bus *bus, uint16 flowid, uint32 txstatus);
+#endif
+
+#if defined(LINUX) || defined(linux)
+extern int dhdpcie_bus_start_host_dev(struct dhd_bus *bus);
+extern int dhdpcie_bus_stop_host_dev(struct dhd_bus *bus);
+extern int dhdpcie_bus_enable_device(struct dhd_bus *bus);
+extern int dhdpcie_bus_disable_device(struct dhd_bus *bus);
+extern int dhdpcie_bus_alloc_resource(struct dhd_bus *bus);
+extern void dhdpcie_bus_free_resource(struct dhd_bus *bus);
+extern bool dhdpcie_bus_dongle_attach(struct dhd_bus *bus);
+extern int dhd_bus_release_dongle(struct dhd_bus *bus);
+extern int dhd_bus_request_irq(struct dhd_bus *bus);
+extern int dhdpcie_get_pcieirq(struct dhd_bus *bus, unsigned int *irq);
+extern void dhd_bus_aer_config(struct dhd_bus *bus);
+#else
+static INLINE void dhd_bus_aer_config(struct dhd_bus *bus) { }
+#endif /* LINUX || linux */
+
+extern int dhdpcie_cto_init(struct dhd_bus *bus, bool enable);
+extern int dhdpcie_cto_cfg_init(struct dhd_bus *bus, bool enable);
+
+extern void dhdpcie_ssreset_dis_enum_rst(struct dhd_bus *bus);
+
+#ifdef DHD_FW_COREDUMP
+extern int dhd_dongle_mem_dump(void);
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef IDLE_TX_FLOW_MGMT
+extern void dhd_bus_idle_tx_ring_suspend(dhd_pub_t *dhd, uint16 flow_ring_id);
+#endif /* IDLE_TX_FLOW_MGMT */
+extern void dhd_bus_handle_mb_data(struct dhd_bus *bus, uint32 d2h_mb_data);
+#endif /* BCMPCIE */
+
+/* dump the device trap informtation  */
+extern void dhd_bus_dump_trap_info(struct dhd_bus *bus, struct bcmstrbuf *b);
+extern void dhd_bus_copy_trap_sig(struct dhd_bus *bus,  trap_t *tr);
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+void copy_hang_info_linkdown(dhd_pub_t *dhd);
+void copy_ext_trap_sig(dhd_pub_t *dhd, trap_t *tr);
+void copy_hang_info_trap(dhd_pub_t *dhd);
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+
+/* Function to set default min res mask */
+extern bool dhd_bus_set_default_min_res_mask(struct dhd_bus *bus);
+
+/* Function to reset PMU registers */
+extern void dhd_bus_pmu_reg_reset(dhd_pub_t *dhdp);
+
+extern void dhd_bus_ucode_download(struct dhd_bus *bus);
+
+extern int dhd_bus_readwrite_bp_addr(dhd_pub_t *dhdp, uint addr, uint size, uint* data, bool read);
+extern int dhd_get_idletime(dhd_pub_t *dhd);
+extern bool dhd_get_rpm_state(dhd_pub_t *dhd);
+extern void dhd_set_rpm_state(dhd_pub_t *dhd, bool state);
+#ifdef BCMPCIE
+extern void dhd_bus_dump_console_buffer(struct dhd_bus *bus);
+extern void dhd_bus_intr_count_dump(dhd_pub_t *dhdp);
+extern bool dhd_bus_query_dpc_sched_errors(dhd_pub_t *dhdp);
+extern int dhd_bus_dmaxfer_lpbk(dhd_pub_t *dhdp, uint32 type);
+extern bool dhd_bus_check_driver_up(void);
+extern int dhd_bus_get_cto(dhd_pub_t *dhdp);
+extern bool dhd_bus_get_read_shm(dhd_pub_t *dhdp);
+extern void dhd_bus_set_linkdown(dhd_pub_t *dhdp, bool val);
+extern int dhd_bus_get_linkdown(dhd_pub_t *dhdp);
+#if defined(CONFIG_ARCH_MSM) && defined(CONFIG_SEC_PCIE_L1SS)
+extern void dhd_bus_inform_ep_loaded_to_rc(dhd_pub_t *dhdp, bool up);
+#endif /* CONFIG_ARCH_MSM  && CONFIG_SEC_PCIE_L1SS */
+extern int dhd_bus_checkdied(struct dhd_bus *bus, char *data, uint size);
+#else
+#define dhd_bus_dump_console_buffer(x)
+static INLINE void dhd_bus_intr_count_dump(dhd_pub_t *dhdp) { UNUSED_PARAMETER(dhdp); }
+static INLINE bool dhd_bus_query_dpc_sched_errors(dhd_pub_t *dhdp) { return 0; }
+static INLINE int dhd_bus_dmaxfer_lpbk(dhd_pub_t *dhdp, uint32 type) { return 0; }
+static INLINE bool dhd_bus_check_driver_up(void) { return FALSE; }
+static INLINE void dhd_bus_set_linkdown(dhd_pub_t *dhdp, bool val) { }
+static INLINE int dhd_bus_get_linkdown(dhd_pub_t *dhdp) { return 0; }
+static INLINE int dhd_bus_get_cto(dhd_pub_t *dhdp) { return 0; }
+static INLINE bool dhd_bus_get_read_shm(dhd_pub_t *dhdp) { return 0; }
+static INLINE int dhd_bus_checkdied(struct dhd_bus *bus, char *data, uint size) { return 0; }
+#endif /* BCMPCIE */
+
+#if defined(BCMPCIE) && defined(EWP_ETD_PRSRV_LOGS)
+void dhdpcie_get_etd_preserve_logs(dhd_pub_t *dhd, uint8 *ext_trap_data,
+		void *event_decode_data);
+#endif
+
+extern uint16 dhd_get_chipid(struct dhd_bus *bus);
+extern uint16 dhd_get_chiprev(struct dhd_bus *bus);
+
+#ifdef DHD_WAKE_STATUS
+extern wake_counts_t* dhd_bus_get_wakecount(dhd_pub_t *dhd);
+extern int dhd_bus_get_bus_wake(dhd_pub_t * dhd);
+extern int dhd_bus_set_get_bus_wake(dhd_pub_t * dhd, int set);
+#if defined(BCMPCIE)
+extern int dhd_bus_set_get_bus_wake_pkt_dump(dhd_pub_t *dhd, int wake_pkt_dump);
+#endif /* BCMPCIE */
+#endif /* DHD_WAKE_STATUS */
+
+#ifdef BT_OVER_SDIO
+/*
+ * SDIO layer clock control functions exposed to be called from other layers.
+ * This is required especially in the case where the BUS is shared between
+ * BT and SDIO and we have to control the clock. The callers of this function
+ * are expected to hold the sdlock
+ */
+int __dhdsdio_clk_enable(struct dhd_bus *bus, bus_owner_t owner, int can_wait);
+int __dhdsdio_clk_disable(struct dhd_bus *bus, bus_owner_t owner, int can_wait);
+void dhdsdio_reset_bt_use_count(struct dhd_bus *bus);
+#endif /* BT_OVER_SDIO */
+
+int dhd_bus_perform_flr(struct dhd_bus *bus, bool force_fail);
+extern bool dhd_bus_get_flr_force_fail(struct dhd_bus *bus);
+
+extern bool dhd_bus_aspm_enable_rc_ep(struct dhd_bus *bus, bool enable);
+extern bool dhd_bus_is_aspm_enab_rc_ep(struct dhd_bus *bus);
+extern void dhd_bus_l1ss_enable_rc_ep(struct dhd_bus *bus, bool enable);
+extern bool dhd_bus_is_l1ss_enab_rc_ep(struct dhd_bus *bus);
+
+bool dhd_bus_is_multibp_capable(struct dhd_bus *bus);
+
+#ifdef BCMPCIE
+extern void dhdpcie_advertise_bus_cleanup(dhd_pub_t  *dhdp);
+extern void dhd_msgbuf_iovar_timeout_dump(dhd_pub_t *dhd);
+extern void dhdpcie_induce_cbp_hang(dhd_pub_t *dhd);
+extern void dhdpcie_busbusy_wait(dhd_pub_t *dhdp);
+extern int dhd_dump_flowrings(dhd_pub_t *dhdp, char *buf, int buflen);
+#endif /* BCMPCIE */
+
+extern bool dhd_bus_force_bt_quiesce_enabled(struct dhd_bus *bus);
+extern void dhd_bwm_bt_quiesce(struct dhd_bus *bus);
+extern void dhd_bwm_bt_resume(struct dhd_bus *bus);
+
+#ifdef DHD_SSSR_DUMP
+extern int dhd_bus_fis_trigger(dhd_pub_t *dhd);
+extern int dhd_bus_fis_dump(dhd_pub_t *dhd);
+#endif /* DHD_SSSR_DUMP */
+
+#ifdef PCIE_FULL_DONGLE
+extern int dhdpcie_set_dma_ring_indices(dhd_pub_t *dhd, int32 int_val);
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef D2H_MINIDUMP
+#ifndef DHD_FW_COREDUMP
+/* Minidump depends on DHD_FW_COREDUMP to dump minidup
+ * This dependency is intentional to avoid multiple work queue
+ * to dump the SOCRAM, minidum ..etc.
+ */
+#error "Minidump doesnot work as DHD_FW_COREDUMP is not defined"
+#endif /* DHD_FW_COREDUMP */
+#ifdef BCM_BUZZZ
+/*
+ * In pciedev_shared_t buzz_dbg_ptr and device_trap_debug_buffer_len
+ * are overloaded. So when BCM_BUZZZ is defined MINIDUMP should not be defined or
+ * vice versa.
+ */
+#error "Minidump doesnot work as BCM_BUZZZ is defined"
+#endif /* BCM_BUZZZ */
+extern bool dhd_bus_is_minidump_enabled(dhd_pub_t  *dhdp);
+dhd_dma_buf_t* dhd_prot_get_minidump_buf(dhd_pub_t *dhd);
+#endif /* D2H_MINIDUMP */
+
+#ifdef DHD_CFG80211_SUSPEND_RESUME
+extern void dhd_cfg80211_suspend(dhd_pub_t *dhdp);
+extern void dhd_cfg80211_resume(dhd_pub_t *dhdp);
+#endif /* DHD_CFG80211_SUSPEND_RESUME */
+
+#ifdef DHD_SDTC_ETB_DUMP
+extern int dhd_bus_get_etb_info(dhd_pub_t *dhd, uint32 etb_info_addr, etb_info_t *etb_info);
+extern int dhd_bus_get_sdtc_etb(dhd_pub_t *dhd, uint8 *sdtc_etb_mempool,
+	uint addr, uint read_bytes);
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#ifndef BCMDBUS
+extern int dhd_socram_dump(struct dhd_bus *bus);
+#endif /* BCMDBUS */
+
+extern int dhdpcie_get_max_eventbufpost(struct dhd_bus *bus);
+
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+extern void dhd_bus_flow_ring_status_isr_trace(dhd_pub_t *dhd);
+extern void dhd_bus_flow_ring_status_dpc_trace(dhd_pub_t *dhd);
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+
+extern bool dhd_bus_init_done(struct dhd_bus *bus);
+
+extern void dhdpcie_db7_trap(struct dhd_bus *bus);
+
+#if defined(NDIS)
+void * dhd_bus_get_socram_buf(struct dhd_bus *bus, struct dhd_pub *dhdp);
+#endif
+
+void dhd_bus_set_signature_path(struct dhd_bus *bus, char *sig_path);
+
+#endif /* _dhd_bus_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_ccode.c b/drivers/net/wireless/ap12275/dhd_ccode.c
new file mode 100644
index 000000000000..af4a74161b4b
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_ccode.c
@@ -0,0 +1,376 @@
+
+#ifdef CCODE_LIST
+#ifdef CONFIG_COMPAT
+#include <linux/compat.h>
+#endif /* COMFIG_COMPAT */
+#include <typedefs.h>
+#include <dhd_config.h>
+
+#ifdef BCMSDIO
+#define CCODE_43438
+#define CCODE_43436
+#define CCODE_43455C0
+#endif
+#if defined(BCMSDIO) || defined(BCMPCIE)
+#define CCODE_4356A2
+#define CCODE_4359C0
+#endif
+#if defined(BCMPCIE)
+//#define CCODE_4375B4
+#endif
+#ifdef BCMDBUS
+#define CCODE_4358U
+#endif
+
+#ifdef BCMSDIO
+#ifdef CCODE_43438
+const char ccode_43438[] = "RU/13";
+#else
+const char ccode_43438[] = "";
+#endif
+
+#ifdef CCODE_43436
+const char ccode_43436[] = \
+"AE/1 AR/1 AT/1 AU/2 "\
+"BE/1 BG/1 BN/1 "\
+"CA/2 CH/1 CN/38 CY/1 CZ/1 "\
+"DE/3 DK/1 "\
+"EE/1 ES/1 "\
+"FI/1 FR/1 "\
+"GB/1 GR/1 "\
+"HR/1 HU/1 "\
+"ID/5 IE/1 IS/1 IT/1 "\
+"JP/3 "\
+"KR/4 KW/1 "\
+"LI/1 LT/1 LU/1 LV/1 "\
+"MA/1 MT/1 MX/1 "\
+"NL/1 NO/1 "\
+"PL/1 PT/1 PY/1 "\
+"RO/1 RU/5 "\
+"SE/1 SI/1 SK/1 "\
+"TR/7 TW/2 "\
+"US/26 "\
+"XZ/11";
+#else
+const char ccode_43436[] = "";
+#endif
+
+#ifdef CCODE_43455C0
+const char ccode_43455c0[] = \
+"AE/6 AG/2 AI/1 AL/2 AS/12 AT/4 AU/6 AW/2 AZ/2 "\
+"BA/2 BD/1 BE/4 BG/4 BH/4 BM/12 BN/4 BR/2 BS/2 BY/3 "\
+"CA/2 CH/4 CN/38 CO/17 CR/17 CY/4 CZ/4 "\
+"DE/7 DK/4 "\
+"EC/21 EE/4 EG/13 ES/4 ET/2 "\
+"FI/4 FR/5 "\
+"GB/6 GD/2 GF/2 GP/2 GR/4 GT/1 GU/30 "\
+"HK/2 HR/4 HU/4 "\
+"ID/1 IE/5 IL/14 IN/3 IS/4 IT/4 "\
+"JO/3 JP/45 "\
+"KH/2 KR/96 KW/5 KY/3 "\
+"LA/2 LB/5 LI/4 LK/1 LS/2 LT/4 LU/3 LV/4 "\
+"MA/2 MC/1 MD/2 ME/2 MK/2 MN/1 MQ/2 MR/2 MT/4 MU/2 MV/3 MW/1 MX/44 MY/3 "\
+"NI/2 NL/4 NO/4 NZ/4 "\
+"OM/4 "\
+"PA/17 PE/20 PH/5 PL/4 PR/38 PT/4 PY/2 "\
+"Q2/993 "\
+"RE/2 RO/4 RS/2 RU/13 "\
+"SE/4 SI/4 SK/4 SV/25 "\
+"TH/5 TN/1 TR/7 TT/3 TW/65 "\
+"UA/8 US/988 "\
+"VA/2 VE/3 VG/2 VN/4 "\
+"XZ/11 "\
+"YT/2 "\
+"ZA/6";
+#else
+const char ccode_43455c0[] = "";
+#endif
+#endif
+
+#ifdef CCODE_4356A2
+const char ccode_4356a2[] = \
+"AE/6 AG/2 AI/1 AL/2 AN/2 AR/21 AS/12 AT/4 AU/6 AW/2 AZ/2 "\
+"BA/2 BD/2 BE/4 BG/4 BH/4 BM/12 BN/4 BR/4 BS/2 BY/3 "\
+"CA/31 CH/4 CN/38 CO/17 CR/17 CY/4 CZ/4 "\
+"DE/7 DK/4 DZ/1 "\
+"EC/21 EE/4 ES/4 ET/2 "\
+"FI/4 FR/5 "\
+"GB/6 GD/2 GF/2 GP/2 GR/4 GT/1 GU/12 "\
+"HK/2 HR/4 HU/4 "\
+"ID/13 IE/5 IL/7 IN/28 IS/4 IT/4 "\
+"JO/3 JP/45 "\
+"KH/2 KR/57 KW/5 KY/3 "\
+"LA/2 LB/5 LI/4 LK/1 LS/2 LT/4 LU/3 LV/4 "\
+"MA/2 MC/1 MD/2 ME/2 MK/2 MN/1 MO/2 MR/2 MT/4 MQ/2 MU/2 MV/3 MW/1 MX/20 MY/16 "\
+"NI/2 NL/4 NO/4 NP/3 NZ/4 "\
+"OM/4 "\
+"PA/17 PE/20 PG/2 PH/5 PL/4 PR/20 PT/4 PY/2 "\
+"RE/2 RO/4 RS/2 RU/986 "\
+"SE/4 SG/19 SI/4 SK/4 SN/2 SV/19 "\
+"TH/9 TN/1 TR/7 TT/3 TW/1 "\
+"UA/8 UG/2 US/1 UY/1 "\
+"VA/2 VE/3 VG/2 VI/13 VN/4 "\
+"XZ/11 "\
+"YT/2 "\
+"ZM/2 "\
+"E0/32";
+#else
+const char ccode_4356a2[] = "";
+#endif
+
+#ifdef CCODE_4359C0
+const char ccode_4359c0[] = \
+"AD/1 AE/6 AG/2 AI/1 AL/3 AS/12 AT/21 AU/6 AW/2 AZ/8 "\
+"BA/4 BD/1 BE/19 BG/18 BH/4 BM/12 BN/4 BR/2 BS/2 BY/3 "\
+"CA/2 CN/38 CO/17 CR/17 CY/18 CZ/18 "\
+"DE/30 DK/19 "\
+"E0/32 EC/21 EE/18 EG/13 ES/21 ET/2 "\
+"FI/19 FR/21 "\
+"GB/996 GD/2 GE/1 GF/2 GP/2 GR/18 GT/1 GU/30 "\
+"HK/2 HR/18 HU/18 "\
+"ID/1 IE/21 IL/276 IN/3 IS/17 IT/20 "\
+"JO/3 JP/967 "\
+"KH/2 KR/70 KW/5 KY/3 "\
+"LA/2 LB/5 LI/17 LK/1 LS/2 LT/18 LU/18 LV/18 "\
+"MA/2 MC/2 MD/3 ME/5 MK/4 MN/1 MQ/2 MR/2 MT/18 MU/2 MV/3 MW/1 MX/44 MY/3 "\
+"NI/2 NL/19 NO/18 NZ/4 "\
+"OM/4 "\
+"PA/17 PE/20 PH/5 PL/18 PR/38 PT/20 PY/2 "\
+"Q1/947 Q2/993 "\
+"RE/2 RO/18 RS/4 RU/986 "\
+"SE/19 SI/18 SK/18 SM/1 SV/25 "\
+"TH/5 TN/1 TR/18 TT/3 TW/980 "\
+"UA/16 US/988 "\
+"VA/3 VE/3 VG/2 VN/4 "\
+"XZ/11 "\
+"YT/2 "\
+"ZA/6";
+#else
+const char ccode_4359c0[] = "";
+#endif
+
+#ifdef CCODE_4375B4
+const char ccode_4375b4[] = \
+"AE/6 AL/2 AM/1 AN/5 AR/21 AT/4 AU/6 AZ/2 "\
+"BA/2 BE/4 BG/4 BH/4 BN/4 BO/5 BR/17 BY/3 "\
+"CA/2 CH/4 CL/7 CN/38 CO/17 CR/17 CY/4 CZ/4 "\
+"DE/7 DK/4 DZ/2 EC/18 EE/4 EG/13 ES/4 "\
+"FI/4 FR/5 "\
+"GB/6 GR/4 "\
+"HK/999 HN/8 HR/4 HU/4 "\
+"ID/5 IE/5 IL/7 IN/3 IS/4 IT/4 "\
+"JO/3 JP/72 "\
+"KE/1 KR/96 KW/5 KZ/5 "\
+"LA/2 LB/5 LI/4 LK/2 LT/4 LU/4 LV/4 "\
+"MA/7 MC/1 ME/2 MK/2 MO/4 MT/4 MX/20 MY/19 "\
+"NL/4 NO/4 NZ/4 "\
+"OM/4 "\
+"PA/17 PE/20 PH/5 PK/2 PL/4 PR/20 PT/4 "\
+"RO/4 RU/62 "\
+"SA/5 SE/4 SG/12 SI/4 SK/4 SV/17 "\
+"TH/5 TN/1 TR/7 TT/3 TW/65 "\
+"UA/16 US/140 UY/10 "\
+"VE/3 VN/4 "\
+"XZ/11 "\
+"ZA/19";
+#else
+const char ccode_4375b4[] = "";
+#endif
+
+#ifdef CCODE_4358U
+const char ccode_4358u[] = \
+"BE/4 BR/4 "\
+"CA/2 CH/4 CN/38 CY/4 "\
+"DE/7 DK/4 "\
+"ES/4 "\
+"FI/4 FR/5 "\
+"GB/6 GR/4 "\
+"HK/2 HU/4 "\
+"IE/5 IL/7 IS/4 IT/4 "\
+"JP/72 "\
+"KE/0 KR/4 "\
+"MY/3 "\
+"NL/4 "\
+"PT/4 "\
+"SA/5 SE/4 SG/0 SZ/0 "\
+"TH/5 TR/7 TW/230 "\
+"US/0 "\
+"VN/4";
+#else
+const char ccode_4358u[] = "";
+#endif
+
+typedef struct ccode_list_map_t {
+	uint chip;
+	uint chiprev;
+	const char *ccode_list;
+	const char *ccode_ww;
+} ccode_list_map_t;
+
+const ccode_list_map_t ccode_list_map[] = {
+	/* ChipID		Chiprev		ccode  */
+#if defined(BCMSDIO) || defined(BCMPCIE)
+	{BCM4354_CHIP_ID,	2,	ccode_4356a2,	"XZ/11"},
+	{BCM4356_CHIP_ID,	2,	ccode_4356a2,	"XZ/11"},
+	{BCM4359_CHIP_ID,	9,	ccode_4359c0,	"XZ/11"},
+#endif
+#ifdef BCMSDIO
+	{BCM43430_CHIP_ID,	0,	ccode_43438,	""},
+	{BCM43430_CHIP_ID,	1,	ccode_43438,	""},
+	{BCM43430_CHIP_ID,	2,	ccode_43436,	""},
+	{BCM4345_CHIP_ID,	6,	ccode_43455c0,	"XZ/11"},
+	{BCM43454_CHIP_ID,	6,	ccode_43455c0,	"XZ/11"},
+	{BCM4345_CHIP_ID,	9,	ccode_43455c0,	"XZ/11"},
+	{BCM43454_CHIP_ID,	9,	ccode_43455c0,	"XZ/11"},
+	{BCM4371_CHIP_ID,	2,	ccode_4356a2,	"XZ/11"},
+#endif
+#ifdef BCMPCIE
+	{BCM4375_CHIP_ID,	5,	ccode_4375b4,	""},
+#endif
+#ifdef BCMDBUS
+	{BCM43569_CHIP_ID,	2,	ccode_4358u, "XW/0"},
+#endif
+};
+
+/*
+ * [ccode]:[disable_6g_band][disable_5g_band][ww_2g_chan_only]
+ * [ccode]:[x][xxxx][x]
+ */
+const char ccode_all_map[] = \
+"AA:000002 "\
+"AU:E08203 "\
+"BH:F0E042 BR:00E002 "\
+"CA:00E203 CN:F0E042 "\
+"EU:E08002 "\
+"GB:E08002 "\
+"ID:F0F042 IL:F08002 IN:F08002 "\
+"JP:E00082 "\
+"KR:00E002 "\
+"NZ:E0E002 "\
+"PH:F0C002 "\
+"RU:E0C302 "\
+"SG:E0E002 "\
+"TR:F00882 TW:F0E043 "\
+"US:000003 "\
+"VN:F0E002 "\
+"ZA:F00882 ";
+
+typedef struct ccode_all_chip_t {
+	uint chip;
+	uint chiprev;
+} ccode_all_chip_t;
+
+const ccode_all_chip_t ccode_all_chip[] = {
+	/* ChipID		Chiprev  */
+//	{0,	0},
+#if defined(BCMSDIO) || defined(BCMPCIE)
+	{BCM43752_CHIP_ID,	4},
+	{BCM43756_CHIP_ID,	4},
+	{BCM43756_CHIP_ID,	6},
+#endif
+};
+
+int
+dhd_ccode_map_country_list(dhd_pub_t *dhd, wl_country_t *cspec)
+{
+	int ret = -1, i;
+	uint chip = dhd->conf->chip, chiprev = dhd->conf->chiprev;
+	const char *ccode_list = NULL, *ccode_ww = NULL;
+	char *pch;
+
+	for (i=0; i<sizeof(ccode_list_map)/sizeof(ccode_list_map[0]); i++) {
+		const ccode_list_map_t* row = &ccode_list_map[i];
+		if (row->chip == chip && row->chiprev == chiprev) {
+			ccode_list = row->ccode_list;
+			ccode_ww = row->ccode_ww;
+			break;
+		}
+	}
+
+	if (ccode_list) {
+		pch = strstr(ccode_list, cspec->ccode);
+		if (pch) {
+			cspec->rev = (int)simple_strtol(pch+strlen(cspec->ccode)+1, NULL, 0);
+			ret = 0;
+		}
+	}
+
+	if (dhd->op_mode != DHD_FLAG_MFG_MODE) {
+		if (ret && ccode_ww && strlen(ccode_ww)>=4) {
+			memcpy(cspec->ccode, ccode_ww, 2);
+			cspec->rev = (int)simple_strtol(ccode_ww+3, NULL, 0);
+		}
+	}
+
+	return ret;
+}
+
+#define WW_2G_CHAN_ONLY_MASK	0x00000F
+#define DISABLE_5G_BAND_MASK	0x0FFFF0
+#define DISABLE_6G_BAND_MASK	0xF00000
+
+#define DISABLE_5G_BAND_OFFSET	4
+#define DISABLE_6G_BAND_OFFSET	20
+
+int
+dhd_ccode_map_country_all(dhd_pub_t *dhd, wl_country_t *cspec)
+{
+	int ret = -1, i;
+	struct dhd_conf *conf = dhd->conf;
+	wl_ccode_all_t *ccode_all = &dhd->conf->ccode_all;
+	uint chip = conf->chip, chiprev = conf->chiprev, val;
+	char *pch, *pick_tmp;
+	bool picked = FALSE;
+
+	if (conf->ccode_all_list) {
+		pch = strstr(conf->ccode_all_list, cspec->ccode);
+		if (!pch)
+			goto exit;
+		pick_tmp = bcmstrtok(&pch, ", ", 0);
+		if (!pick_tmp)
+			goto exit;
+		ccode_all->ww_2g_chan_only = 0x2;
+		ccode_all->disable_5g_band = 0;
+		ccode_all->disable_6g_band = 0;
+		pch = strstr(pick_tmp, "2g");
+		if (pch) {
+			ccode_all->ww_2g_chan_only = (int)simple_strtol(pch+strlen("2g"), NULL, 16);
+			picked = TRUE;
+		}
+		pch = strstr(pick_tmp, "5g");
+		if (pch) {
+			ccode_all->disable_5g_band = (int)simple_strtol(pch+strlen("5g"), NULL, 16);
+			picked = TRUE;
+		}
+		pch = strstr(pick_tmp, "6g");
+		if (pch) {
+			ccode_all->disable_6g_band = (int)simple_strtol(pch+strlen("6g"), NULL, 16);
+			picked = TRUE;
+		}
+		if (picked) {
+			memcpy(cspec->ccode, &ccode_all->cspec.ccode, WLC_CNTRY_BUF_SZ);
+			return 0;
+		}
+	}
+
+exit:
+	for (i=0; i<sizeof(ccode_all_chip)/sizeof(ccode_all_chip[0]); i++) {
+		const ccode_all_chip_t* row = &ccode_all_chip[i];
+		if ((row->chip == chip && row->chiprev == chiprev) ||
+				(row->chip == 0 && row->chiprev == 0)) {
+			pch = strstr(ccode_all_map, cspec->ccode);
+			if (pch) {
+				val = simple_strtol(pch+strlen(cspec->ccode)+1, NULL, 16);
+				ccode_all->ww_2g_chan_only = val & WW_2G_CHAN_ONLY_MASK;
+				ccode_all->disable_5g_band = (val & DISABLE_5G_BAND_MASK) >> DISABLE_5G_BAND_OFFSET;
+				ccode_all->disable_6g_band = (val & DISABLE_6G_BAND_MASK) >> DISABLE_6G_BAND_OFFSET;
+				memcpy(cspec->ccode, &ccode_all->cspec.ccode, WLC_CNTRY_BUF_SZ);
+				ret = 0;
+				break;
+			}
+		}
+	}
+
+	return ret;
+}
+#endif
diff --git a/drivers/net/wireless/ap12275/dhd_cdc.c b/drivers/net/wireless/ap12275/dhd_cdc.c
new file mode 100644
index 000000000000..7ab331c53449
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_cdc.c
@@ -0,0 +1,994 @@
+/*
+ * DHD Protocol Module for CDC and BDC.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ *
+ * BDC is like CDC, except it includes a header for data packets to convey
+ * packet priority over the bus, and flags (e.g. to indicate checksum status
+ * for dongle offload.)
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+
+#include <bcmutils.h>
+#include <bcmcdc.h>
+#include <bcmendian.h>
+
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_proto.h>
+#include <dhd_bus.h>
+#include <dhd_dbg.h>
+
+#ifdef EXT_STA
+#include <siutils.h>
+#include <wlc_cfg.h>
+#include <wlc_pub.h>
+#endif /* EXT_STA */
+
+#ifdef PROP_TXSTATUS
+#include <wlfc_proto.h>
+#include <dhd_wlfc.h>
+#endif
+#ifdef BCMDBUS
+#include <dhd_config.h>
+#endif /* BCMDBUS */
+
+#define RETRIES 2		/* # of retries to retrieve matching ioctl response */
+#define BUS_HEADER_LEN	(24+DHD_SDALIGN)	/* Must be at least SDPCM_RESERVE
+				 * defined in dhd_sdio.c (amount of header tha might be added)
+				 * plus any space that might be needed for alignment padding.
+				 */
+#define ROUND_UP_MARGIN	2048	/* Biggest SDIO block size possible for
+				 * round off at the end of buffer
+				 */
+
+/* This value is from Legacy chipsets */
+#define DEFAULT_WLC_API_VERSION_MAJOR	3
+#define DEFAULT_WLC_API_VERSION_MINOR	0
+
+typedef struct dhd_prot {
+	uint16 reqid;
+	uint8 pending;
+	uint32 lastcmd;
+	uint8 bus_header[BUS_HEADER_LEN];
+	cdc_ioctl_t msg;
+	unsigned char buf[WLC_IOCTL_MAXLEN + ROUND_UP_MARGIN];
+} dhd_prot_t;
+
+uint16
+dhd_prot_get_ioctl_trans_id(dhd_pub_t *dhdp)
+{
+	/* SDIO does not have ioctl_trans_id yet, so return -1 */
+	return -1;
+}
+
+static int
+dhdcdc_msg(dhd_pub_t *dhd)
+{
+	int err = 0;
+	dhd_prot_t *prot = dhd->prot;
+	int len = ltoh32(prot->msg.len) + sizeof(cdc_ioctl_t);
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	DHD_OS_WAKE_LOCK(dhd);
+
+	/* NOTE : cdc->msg.len holds the desired length of the buffer to be
+	 *        returned. Only up to CDC_MAX_MSG_SIZE of this buffer area
+	 *	  is actually sent to the dongle
+	 */
+	if (len > CDC_MAX_MSG_SIZE)
+		len = CDC_MAX_MSG_SIZE;
+
+	/* Send request */
+	err = dhd_bus_txctl(dhd->bus, (uchar*)&prot->msg, len);
+
+	DHD_OS_WAKE_UNLOCK(dhd);
+	return err;
+}
+
+static int
+dhdcdc_cmplt(dhd_pub_t *dhd, uint32 id, uint32 len)
+{
+	int ret;
+	int cdc_len = len + sizeof(cdc_ioctl_t);
+	dhd_prot_t *prot = dhd->prot;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	/*
+	 * prot->msg is the buffer used to send the ioctl msg in dhdcdc_msg and the same is
+	 * used for receiving the ioctl response.
+	 * As this buffer is not cleared after sending message and before re-using for receive
+	 * operation, problems are seen when bus errors occur.
+	 * Example:- An error is seen on the bus where a 0 byte pkt is received.
+	 * the bus-controller "layer below cdc" does not update the buffer in this 0 byte pkt case.
+	 * bus-controller layer calls the completion callback without any error and with the
+	 * same buffer.(no change)
+	 * This issue is seen on DBUS/USB + FPGA platforms
+	 * In this function if there is no update to the buffer, the stale id field of sent msg
+	 * matches to expected ID and further processing is done thinking that
+	 * proper response is received.
+	 * This is a generic problem and its a good idea to clear the buffer or atleast
+	 * the buffer's key value (ioctl ID within flags field in this case) before re-using it.
+	 *
+	 * To ensure that a new content is indeed received from the bus making flags = 0.
+	 * Note that ID=0 is invalid value.
+	 */
+	prot->msg.flags = 0;
+
+	do {
+		ret = dhd_bus_rxctl(dhd->bus, (uchar*)&prot->msg, cdc_len);
+		if (ret < 0)
+			break;
+	} while (CDC_IOC_ID(ltoh32(prot->msg.flags)) != id);
+
+	/* update ret to len on success */
+	if (ret == cdc_len) {
+		ret = len;
+	}
+
+	return ret;
+}
+
+/* XXX: due to overlays this should not be called directly; call dhd_wl_ioctl_cmd() instead */
+static int
+dhdcdc_query_ioctl(dhd_pub_t *dhd, int ifidx, uint cmd, void *buf, uint len, uint8 action)
+{
+	dhd_prot_t *prot = dhd->prot;
+	cdc_ioctl_t *msg = &prot->msg;
+	int ret = 0, retries = 0;
+	uint32 id, flags = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	DHD_CTL(("%s: cmd %d len %d\n", __FUNCTION__, cmd, len));
+
+	/* Respond "bcmerror" and "bcmerrorstr" with local cache */
+	if (cmd == WLC_GET_VAR && buf)
+	{
+		if (!strcmp((char *)buf, "bcmerrorstr"))
+		{
+			strlcpy((char *)buf, bcmerrorstr(dhd->dongle_error), len);
+			goto done;
+		}
+		else if (!strcmp((char *)buf, "bcmerror"))
+		{
+			*(int *)buf = dhd->dongle_error;
+			goto done;
+		}
+	}
+
+	if (ifidx >= DHD_MAX_IFS) {
+		DHD_ERROR(("%s: IF index %d Invalid for the dongle FW\n",
+			__FUNCTION__, ifidx));
+		return -EIO;
+	}
+
+	memset(msg, 0, sizeof(cdc_ioctl_t));
+
+#ifdef BCMSPI
+	/* 11bit gSPI bus allows 2048bytes of max-data.  We restrict 'len'
+	 * value which is 8Kbytes for various 'get' commands to 2000.  48 bytes are
+	 * left for sw headers and misc.
+	 */
+	if (len > 2000) {
+		DHD_ERROR(("dhdcdc_query_ioctl: len is truncated to 2000 bytes\n"));
+		len = 2000;
+	}
+#endif /* BCMSPI */
+	msg->cmd = htol32(cmd);
+	msg->len = htol32(len);
+	msg->flags = (++prot->reqid << CDCF_IOC_ID_SHIFT);
+	CDC_SET_IF_IDX(msg, ifidx);
+	/* add additional action bits */
+	action &= WL_IOCTL_ACTION_MASK;
+	msg->flags |= (action << CDCF_IOC_ACTION_SHIFT);
+	msg->flags = htol32(msg->flags);
+
+	if (buf)
+		memcpy(prot->buf, buf, len);
+
+	if ((ret = dhdcdc_msg(dhd)) < 0) {
+		if (!dhd->hang_was_sent)
+		DHD_ERROR(("dhdcdc_query_ioctl: dhdcdc_msg failed w/status %d\n", ret));
+		goto done;
+	}
+
+retry:
+	/* wait for interrupt and get first fragment */
+	if ((ret = dhdcdc_cmplt(dhd, prot->reqid, len)) < 0)
+		goto done;
+
+	flags = ltoh32(msg->flags);
+	id = (flags & CDCF_IOC_ID_MASK) >> CDCF_IOC_ID_SHIFT;
+
+	if ((id < prot->reqid) && (++retries < RETRIES))
+		goto retry;
+	if (id != prot->reqid) {
+		DHD_ERROR(("%s: %s: unexpected request id %d (expected %d)\n",
+		           dhd_ifname(dhd, ifidx), __FUNCTION__, id, prot->reqid));
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/* Copy info buffer */
+	if (buf)
+	{
+		if (ret < (int)len)
+			len = ret;
+		memcpy(buf, (void*) prot->buf, len);
+	}
+
+	/* Check the ERROR flag */
+	if (flags & CDCF_IOC_ERROR)
+	{
+		ret = ltoh32(msg->status);
+		/* Cache error from dongle */
+		dhd->dongle_error = ret;
+	}
+
+done:
+	return ret;
+}
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+extern bool g_pm_control;
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+
+/* XXX: due to overlays this should not be called directly; call dhd_wl_ioctl_cmd() instead */
+static int
+dhdcdc_set_ioctl(dhd_pub_t *dhd, int ifidx, uint cmd, void *buf, uint len, uint8 action)
+{
+	dhd_prot_t *prot = dhd->prot;
+	cdc_ioctl_t *msg = &prot->msg;
+	int ret = 0;
+	uint32 flags, id;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	DHD_CTL(("%s: cmd %d len %d\n", __FUNCTION__, cmd, len));
+
+	if (dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
+		return -EIO;
+	}
+
+	/* don't talk to the dongle if fw is about to be reloaded */
+	if (dhd->hang_was_sent) {
+		DHD_ERROR(("%s: HANG was sent up earlier. Not talking to the chip\n",
+			__FUNCTION__));
+		return -EIO;
+	}
+
+	if (cmd == WLC_SET_PM) {
+#ifdef DHD_PM_CONTROL_FROM_FILE
+		if (g_pm_control == TRUE) {
+			DHD_ERROR(("%s: SET PM ignored!(Requested:%d)\n",
+				__FUNCTION__, buf ? *(char *)buf : 0));
+			goto done;
+		}
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+#ifdef DHD_PM_OVERRIDE
+		{
+			extern bool g_pm_override;
+			if (g_pm_override == TRUE) {
+				DHD_ERROR(("%s: PM override SET PM ignored!(Requested:%d)\n",
+					__FUNCTION__, buf ? *(char *)buf : 0));
+				goto done;
+			}
+		}
+#endif /* DHD_PM_OVERRIDE */
+		DHD_TRACE_HW4(("%s: SET PM to %d\n", __FUNCTION__, buf ? *(char *)buf : 0));
+	}
+
+	if (ifidx >= DHD_MAX_IFS) {
+		DHD_ERROR(("%s: IF index %d Invalid for the dongle FW\n",
+			__FUNCTION__, ifidx));
+		return -EIO;
+	}
+
+	memset(msg, 0, sizeof(cdc_ioctl_t));
+
+	msg->cmd = htol32(cmd);
+	msg->len = htol32(len);
+	msg->flags = (++prot->reqid << CDCF_IOC_ID_SHIFT);
+	CDC_SET_IF_IDX(msg, ifidx);
+	/* add additional action bits */
+	action &= WL_IOCTL_ACTION_MASK;
+	msg->flags |= (action << CDCF_IOC_ACTION_SHIFT) | CDCF_IOC_SET;
+	msg->flags = htol32(msg->flags);
+
+	if (buf)
+		memcpy(prot->buf, buf, len);
+
+	if ((ret = dhdcdc_msg(dhd)) < 0) {
+		DHD_ERROR(("%s: dhdcdc_msg failed w/status %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+	if ((ret = dhdcdc_cmplt(dhd, prot->reqid, len)) < 0)
+		goto done;
+
+	flags = ltoh32(msg->flags);
+	id = (flags & CDCF_IOC_ID_MASK) >> CDCF_IOC_ID_SHIFT;
+
+	if (id != prot->reqid) {
+		DHD_ERROR(("%s: %s: unexpected request id %d (expected %d)\n",
+		           dhd_ifname(dhd, ifidx), __FUNCTION__, id, prot->reqid));
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/* Copy fw response to buf */
+	if (buf) {
+		ASSERT(ret == len);
+		memcpy(buf, (void*) prot->buf, len);
+	}
+
+	/* Check the ERROR flag */
+	if (flags & CDCF_IOC_ERROR)
+	{
+		ret = ltoh32(msg->status);
+		/* Cache error from dongle */
+		dhd->dongle_error = ret;
+	}
+
+done:
+	return ret;
+}
+
+/* XXX: due to overlays this should not be called directly; call dhd_wl_ioctl() instead */
+int
+dhd_prot_ioctl(dhd_pub_t *dhd, int ifidx, wl_ioctl_t * ioc, void * buf, int len)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = -1;
+	uint8 action;
+	static int error_cnt = 0;
+
+	if ((dhd->busstate == DHD_BUS_DOWN) || dhd->hang_was_sent) {
+		DHD_ERROR(("%s : bus is down. we have nothing to do - bs: %d, has: %d\n",
+				__FUNCTION__, dhd->busstate, dhd->hang_was_sent));
+		goto done;
+	}
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(len <= WLC_IOCTL_MAXLEN);
+
+	if (len > WLC_IOCTL_MAXLEN)
+		goto done;
+
+	if (prot->pending == TRUE) {
+		DHD_ERROR(("CDC packet is pending!!!! cmd=0x%x (%lu) lastcmd=0x%x (%lu)\n",
+			ioc->cmd, (unsigned long)ioc->cmd, prot->lastcmd,
+			(unsigned long)prot->lastcmd));
+		if ((ioc->cmd == WLC_SET_VAR) || (ioc->cmd == WLC_GET_VAR)) {
+			DHD_TRACE(("iovar cmd=%s\n", buf ? (char*)buf : "\0"));
+		}
+		goto done;
+	}
+
+	prot->pending = TRUE;
+	prot->lastcmd = ioc->cmd;
+	action = ioc->set;
+	if (action & WL_IOCTL_ACTION_SET)
+		ret = dhdcdc_set_ioctl(dhd, ifidx, ioc->cmd, buf, len, action);
+	else {
+		ret = dhdcdc_query_ioctl(dhd, ifidx, ioc->cmd, buf, len, action);
+		if (ret > 0)
+			ioc->used = ret - sizeof(cdc_ioctl_t);
+	}
+	// terence 20130805: send hang event to wpa_supplicant
+	if (ret == -EIO) {
+		error_cnt++;
+		if (error_cnt > 2)
+			ret = -ETIMEDOUT;
+	} else
+		error_cnt = 0;
+
+	/* Too many programs assume ioctl() returns 0 on success */
+	if (ret >= 0)
+		ret = 0;
+	else {
+		cdc_ioctl_t *msg = &prot->msg;
+		ioc->needed = ltoh32(msg->len); /* len == needed when set/query fails from dongle */
+	}
+
+	/* Intercept the wme_dp ioctl here */
+	if ((!ret) && (ioc->cmd == WLC_SET_VAR) && (!strcmp(buf, "wme_dp"))) {
+		int slen, val = 0;
+
+		slen = strlen("wme_dp") + 1;
+		if (len >= (int)(slen + sizeof(int)))
+			bcopy(((char *)buf + slen), &val, sizeof(int));
+		dhd->wme_dp = (uint8) ltoh32(val);
+	}
+
+	prot->pending = FALSE;
+
+done:
+
+	return ret;
+}
+
+int
+dhd_prot_iovar_op(dhd_pub_t *dhdp, const char *name,
+                  void *params, int plen, void *arg, int len, bool set)
+{
+	return BCME_UNSUPPORTED;
+}
+
+void
+dhd_prot_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	if (!dhdp || !dhdp->prot) {
+		return;
+	}
+
+	bcm_bprintf(strbuf, "Protocol CDC: reqid %d\n", dhdp->prot->reqid);
+#ifdef PROP_TXSTATUS
+	dhd_wlfc_dump(dhdp, strbuf);
+#endif
+}
+
+void
+dhd_prot_counters(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf,
+	bool print_ringinfo, bool print_pktidinfo)
+{
+	return;
+}
+
+void
+dhd_bus_counters(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	return;
+}
+
+/*	The FreeBSD PKTPUSH could change the packet buf pinter
+	so we need to make it changable
+*/
+#define PKTBUF pktbuf
+void
+dhd_prot_hdrpush(dhd_pub_t *dhd, int ifidx, void *PKTBUF)
+{
+#ifdef BDC
+	struct bdc_header *h;
+#endif /* BDC */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#ifdef BDC
+	/* Push BDC header used to convey priority for buses that don't */
+
+	PKTPUSH(dhd->osh, PKTBUF, BDC_HEADER_LEN);
+
+	h = (struct bdc_header *)PKTDATA(dhd->osh, PKTBUF);
+
+	h->flags = (BDC_PROTO_VER << BDC_FLAG_VER_SHIFT);
+	if (PKTSUMNEEDED(PKTBUF))
+		h->flags |= BDC_FLAG_SUM_NEEDED;
+
+#ifdef EXT_STA
+	/* save pkt encryption exemption info for dongle */
+	h->flags &= ~BDC_FLAG_EXEMPT;
+	h->flags |= (WLPKTFLAG_EXEMPT_GET(WLPKTTAG(pktbuf)) & BDC_FLAG_EXEMPT);
+#endif /* EXT_STA */
+
+	h->priority = (PKTPRIO(PKTBUF) & BDC_PRIORITY_MASK);
+	h->flags2 = 0;
+	h->dataOffset = 0;
+#endif /* BDC */
+	BDC_SET_IF_IDX(h, ifidx);
+}
+#undef PKTBUF	/* Only defined in the above routine */
+
+uint
+dhd_prot_hdrlen(dhd_pub_t *dhd, void *PKTBUF)
+{
+	uint hdrlen = 0;
+#ifdef BDC
+	/* Length of BDC(+WLFC) headers pushed */
+	hdrlen = BDC_HEADER_LEN + (((struct bdc_header *)PKTBUF)->dataOffset * 4);
+#endif
+	return hdrlen;
+}
+
+int
+dhd_prot_hdrpull(dhd_pub_t *dhd, int *ifidx, void *pktbuf, uchar *reorder_buf_info,
+	uint *reorder_info_len)
+{
+#ifdef BDC
+	struct bdc_header *h;
+#endif
+	uint8 data_offset = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#ifdef BDC
+	if (reorder_info_len)
+		*reorder_info_len = 0;
+	/* Pop BDC header used to convey priority for buses that don't */
+
+	if (PKTLEN(dhd->osh, pktbuf) < BDC_HEADER_LEN) {
+		DHD_ERROR(("%s: rx data too short (%d < %d)\n", __FUNCTION__,
+		           PKTLEN(dhd->osh, pktbuf), BDC_HEADER_LEN));
+		return BCME_ERROR;
+	}
+
+	h = (struct bdc_header *)PKTDATA(dhd->osh, pktbuf);
+
+	if (!ifidx) {
+		/* for tx packet, skip the analysis */
+		data_offset = h->dataOffset;
+		PKTPULL(dhd->osh, pktbuf, BDC_HEADER_LEN);
+		goto exit;
+	}
+
+	*ifidx = BDC_GET_IF_IDX(h);
+
+	if (((h->flags & BDC_FLAG_VER_MASK) >> BDC_FLAG_VER_SHIFT) != BDC_PROTO_VER) {
+		DHD_ERROR(("%s: non-BDC packet received, flags = 0x%x\n",
+		           dhd_ifname(dhd, *ifidx), h->flags));
+		if (((h->flags & BDC_FLAG_VER_MASK) >> BDC_FLAG_VER_SHIFT) == BDC_PROTO_VER_1)
+			h->dataOffset = 0;
+		else
+		return BCME_ERROR;
+	}
+
+	if (h->flags & BDC_FLAG_SUM_GOOD) {
+		DHD_INFO(("%s: BDC packet received with good rx-csum, flags 0x%x\n",
+		          dhd_ifname(dhd, *ifidx), h->flags));
+		PKTSETSUMGOOD(pktbuf, TRUE);
+	}
+
+	PKTSETPRIO(pktbuf, (h->priority & BDC_PRIORITY_MASK));
+	data_offset = h->dataOffset;
+	PKTPULL(dhd->osh, pktbuf, BDC_HEADER_LEN);
+#endif /* BDC */
+
+#ifdef PROP_TXSTATUS
+	if (!DHD_PKTTAG_PKTDIR(PKTTAG(pktbuf))) {
+		/*
+		- parse txstatus only for packets that came from the firmware
+		*/
+		dhd_wlfc_parse_header_info(dhd, pktbuf, (data_offset << 2),
+			reorder_buf_info, reorder_info_len);
+
+#ifdef BCMDBUS
+#ifndef DHD_WLFC_THREAD
+		dhd_wlfc_commit_packets(dhd,
+			(f_commitpkt_t)dhd_bus_txdata, dhd->bus, NULL, FALSE);
+#endif /* DHD_WLFC_THREAD */
+#endif /* BCMDBUS */
+	}
+#endif /* PROP_TXSTATUS */
+
+exit:
+	PKTPULL(dhd->osh, pktbuf, (data_offset << 2));
+	return 0;
+}
+
+#ifdef DHD_LOSSLESS_ROAMING
+int dhd_update_sdio_data_prio_map(dhd_pub_t *dhdp)
+{
+	const uint8 prio2tid[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+
+	bcopy(prio2tid, dhdp->flow_prio_map, sizeof(uint8) * NUMPRIO);
+
+	return BCME_OK;
+}
+#endif // DHD_LOSSLESS_ROAMING
+
+int
+dhd_prot_attach(dhd_pub_t *dhd)
+{
+	dhd_prot_t *cdc;
+
+	if (!(cdc = (dhd_prot_t *)DHD_OS_PREALLOC(dhd, DHD_PREALLOC_PROT, sizeof(dhd_prot_t)))) {
+		DHD_ERROR(("%s: kmalloc failed\n", __FUNCTION__));
+		goto fail;
+	}
+	memset(cdc, 0, sizeof(dhd_prot_t));
+
+	/* ensure that the msg buf directly follows the cdc msg struct */
+	if ((uintptr)(&cdc->msg + 1) != (uintptr)cdc->buf) {
+		DHD_ERROR(("dhd_prot_t is not correctly defined\n"));
+		goto fail;
+	}
+
+	dhd->prot = cdc;
+#ifdef BDC
+	dhd->hdrlen += BDC_HEADER_LEN;
+#endif
+	dhd->maxctl = WLC_IOCTL_MAXLEN + sizeof(cdc_ioctl_t) + ROUND_UP_MARGIN;
+	return 0;
+
+fail:
+	if (cdc != NULL)
+		DHD_OS_PREFREE(dhd, cdc, sizeof(dhd_prot_t));
+	return BCME_NOMEM;
+}
+
+/* ~NOTE~ What if another thread is waiting on the semaphore?  Holding it? */
+void
+dhd_prot_detach(dhd_pub_t *dhd)
+{
+#ifdef PROP_TXSTATUS
+	dhd_wlfc_deinit(dhd);
+#endif
+	DHD_OS_PREFREE(dhd, dhd->prot, sizeof(dhd_prot_t));
+	dhd->prot = NULL;
+}
+
+void
+dhd_prot_dstats(dhd_pub_t *dhd)
+{
+	/*  copy bus stats */
+
+	dhd->dstats.tx_packets = dhd->tx_packets;
+	dhd->dstats.tx_errors = dhd->tx_errors;
+	dhd->dstats.rx_packets = dhd->rx_packets;
+	dhd->dstats.rx_errors = dhd->rx_errors;
+	dhd->dstats.rx_dropped = dhd->rx_dropped;
+	dhd->dstats.multicast = dhd->rx_multicast;
+	return;
+}
+
+int
+dhd_sync_with_dongle(dhd_pub_t *dhd)
+{
+	int ret = 0;
+	wlc_rev_info_t revinfo;
+	char buf[128];
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#ifndef OEM_ANDROID
+	/* Get the device MAC address */
+	strcpy(buf, "cur_etheraddr");
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
+	if (ret < 0)
+		goto done;
+	memcpy(dhd->mac.octet, buf, ETHER_ADDR_LEN);
+#endif /* OEM_ANDROID */
+#ifdef DHD_FW_COREDUMP
+	/* Check the memdump capability */
+	dhd_get_memdump_info(dhd);
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef BCMASSERT_LOG
+	dhd_get_assert_info(dhd);
+#endif /* BCMASSERT_LOG */
+
+	/* Get the device rev info */
+	memset(&revinfo, 0, sizeof(revinfo));
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_REVINFO, &revinfo, sizeof(revinfo), FALSE, 0);
+	if (ret < 0)
+		goto done;
+#if defined(BCMDBUS)
+	if (dhd_download_fw_on_driverload) {
+		dhd_conf_reset(dhd);
+		dhd_conf_set_chiprev(dhd, revinfo.chipnum, revinfo.chiprev);
+		dhd_conf_preinit(dhd);
+		dhd_conf_read_config(dhd, dhd->conf_path);
+	}
+#endif /* BCMDBUS */
+
+	/* query for 'wlc_ver' to get version info from firmware */
+	/* memsetting to zero */
+	bzero(buf, sizeof(buf));
+	ret = bcm_mkiovar("wlc_ver", NULL, 0, buf, sizeof(buf));
+	if (ret == 0) {
+		ret = BCME_BUFTOOSHORT;
+		goto done;
+	}
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
+	if (ret == BCME_UNSUPPORTED) {
+		dhd->wlc_ver_major = DEFAULT_WLC_API_VERSION_MAJOR;
+		dhd->wlc_ver_minor = DEFAULT_WLC_API_VERSION_MINOR;
+	} else if (ret < 0) {
+		DHD_ERROR(("%s failed %d\n", __FUNCTION__, ret));
+		goto done;
+	} else {
+		dhd->wlc_ver_major = ((wl_wlc_version_t*)buf)->wlc_ver_major;
+		dhd->wlc_ver_minor = ((wl_wlc_version_t*)buf)->wlc_ver_minor;
+	}
+	DHD_INFO(("wlc_ver_major %d, wlc_ver_minor %d\n", dhd->wlc_ver_major, dhd->wlc_ver_minor));
+
+#if defined(BCMDBUS) && defined(BCMDHDUSB)
+	/* dbus_set_revinfo(dhd->dbus, revinfo.chipnum, revinfo.chiprev); */
+#endif /* BCMDBUS && BCMDHDUSB */
+
+	DHD_SSSR_DUMP_INIT(dhd);
+
+	dhd_process_cid_mac(dhd, TRUE);
+	ret = dhd_preinit_ioctls(dhd);
+	dhd_process_cid_mac(dhd, FALSE);
+
+	/* Always assumes wl for now */
+	dhd->iswl = TRUE;
+
+	/* XXX Could use WLC_GET_REVINFO to get driver version? */
+done:
+	return ret;
+}
+
+int dhd_prot_init(dhd_pub_t *dhd)
+{
+	return BCME_OK;
+}
+
+void
+dhd_prot_stop(dhd_pub_t *dhd)
+{
+/* Nothing to do for CDC */
+}
+
+static void
+dhd_get_hostreorder_pkts(void *osh, struct reorder_info *ptr, void **pkt,
+	uint32 *pkt_count, void **pplast, uint8 start, uint8 end)
+{
+	void *plast = NULL, *p;
+	uint32 pkt_cnt = 0;
+
+	if (ptr->pend_pkts == 0) {
+		DHD_REORDER(("%s: no packets in reorder queue \n", __FUNCTION__));
+		*pplast = NULL;
+		*pkt_count = 0;
+		*pkt = NULL;
+		return;
+	}
+	do {
+		p = (void *)(ptr->p[start]);
+		ptr->p[start] = NULL;
+
+		if (p != NULL) {
+			if (plast == NULL)
+				*pkt = p;
+			else
+				PKTSETNEXT(osh, plast, p);
+
+			plast = p;
+			pkt_cnt++;
+		}
+		start++;
+		if (start > ptr->max_idx)
+			start = 0;
+	} while (start != end);
+	*pplast = plast;
+	*pkt_count = pkt_cnt;
+	ptr->pend_pkts -= (uint8)pkt_cnt;
+}
+
+int
+dhd_process_pkt_reorder_info(dhd_pub_t *dhd, uchar *reorder_info_buf, uint reorder_info_len,
+	void **pkt, uint32 *pkt_count)
+{
+	uint8 flow_id, max_idx, cur_idx, exp_idx;
+	struct reorder_info *ptr;
+	uint8 flags;
+	void *cur_pkt, *plast = NULL;
+	uint32 cnt = 0;
+
+	if (pkt == NULL) {
+		if (pkt_count != NULL)
+			*pkt_count = 0;
+		return 0;
+	}
+
+	flow_id = reorder_info_buf[WLHOST_REORDERDATA_FLOWID_OFFSET];
+	flags = reorder_info_buf[WLHOST_REORDERDATA_FLAGS_OFFSET];
+
+	DHD_REORDER(("flow_id %d, flags 0x%02x, idx(%d, %d, %d)\n", flow_id, flags,
+		reorder_info_buf[WLHOST_REORDERDATA_CURIDX_OFFSET],
+		reorder_info_buf[WLHOST_REORDERDATA_EXPIDX_OFFSET],
+		reorder_info_buf[WLHOST_REORDERDATA_MAXIDX_OFFSET]));
+
+	/* validate flags and flow id */
+	if (flags == 0xFF) {
+		DHD_ERROR(("%s: invalid flags...so ignore this packet\n", __FUNCTION__));
+		*pkt_count = 1;
+		return 0;
+	}
+
+	cur_pkt = *pkt;
+	*pkt = NULL;
+
+	ptr = dhd->reorder_bufs[flow_id];
+	if (flags & WLHOST_REORDERDATA_DEL_FLOW) {
+		uint32 buf_size = sizeof(struct reorder_info);
+
+		DHD_REORDER(("%s: Flags indicating to delete a flow id %d\n",
+			__FUNCTION__, flow_id));
+
+		if (ptr == NULL) {
+			DHD_REORDER(("%s: received flags to cleanup, but no flow (%d) yet\n",
+				__FUNCTION__, flow_id));
+			*pkt_count = 1;
+			*pkt = cur_pkt;
+			return 0;
+		}
+
+		dhd_get_hostreorder_pkts(dhd->osh, ptr, pkt, &cnt, &plast,
+			ptr->exp_idx, ptr->exp_idx);
+		/* set it to the last packet */
+		if (plast) {
+			PKTSETNEXT(dhd->osh, plast, cur_pkt);
+			cnt++;
+		}
+		else {
+			if (cnt != 0) {
+				DHD_ERROR(("%s: del flow: something fishy, pending packets %d\n",
+					__FUNCTION__, cnt));
+			}
+			*pkt = cur_pkt;
+			cnt = 1;
+		}
+		buf_size += ((ptr->max_idx + 1) * sizeof(void *));
+		MFREE(dhd->osh, ptr, buf_size);
+		dhd->reorder_bufs[flow_id] = NULL;
+		*pkt_count = cnt;
+		return 0;
+	}
+	/* all the other cases depend on the existance of the reorder struct for that flow id */
+	if (ptr == NULL) {
+		uint32 buf_size_alloc = sizeof(reorder_info_t);
+		max_idx = reorder_info_buf[WLHOST_REORDERDATA_MAXIDX_OFFSET];
+
+		buf_size_alloc += ((max_idx + 1) * sizeof(void*));
+		/* allocate space to hold the buffers, index etc */
+
+		DHD_REORDER(("%s: alloc buffer of size %d size, reorder info id %d, maxidx %d\n",
+			__FUNCTION__, buf_size_alloc, flow_id, max_idx));
+		ptr = (struct reorder_info *)MALLOC(dhd->osh, buf_size_alloc);
+		if (ptr == NULL) {
+			DHD_ERROR(("%s: Malloc failed to alloc buffer\n", __FUNCTION__));
+			*pkt_count = 1;
+			return 0;
+		}
+		bzero(ptr, buf_size_alloc);
+		dhd->reorder_bufs[flow_id] = ptr;
+		ptr->p = (void *)(ptr+1);
+		ptr->max_idx = max_idx;
+	}
+	/* XXX: validate cur, exp indices */
+	if (flags & WLHOST_REORDERDATA_NEW_HOLE)  {
+		DHD_REORDER(("%s: new hole, so cleanup pending buffers\n", __FUNCTION__));
+		if (ptr->pend_pkts) {
+			dhd_get_hostreorder_pkts(dhd->osh, ptr, pkt, &cnt, &plast,
+				ptr->exp_idx, ptr->exp_idx);
+			ptr->pend_pkts = 0;
+		}
+		ptr->cur_idx = reorder_info_buf[WLHOST_REORDERDATA_CURIDX_OFFSET];
+		ptr->exp_idx = reorder_info_buf[WLHOST_REORDERDATA_EXPIDX_OFFSET];
+		ptr->max_idx = reorder_info_buf[WLHOST_REORDERDATA_MAXIDX_OFFSET];
+		ptr->p[ptr->cur_idx] = cur_pkt;
+		ptr->pend_pkts++;
+		*pkt_count = cnt;
+	}
+	else if (flags & WLHOST_REORDERDATA_CURIDX_VALID) {
+		cur_idx = reorder_info_buf[WLHOST_REORDERDATA_CURIDX_OFFSET];
+		exp_idx = reorder_info_buf[WLHOST_REORDERDATA_EXPIDX_OFFSET];
+
+		if ((exp_idx == ptr->exp_idx) && (cur_idx != ptr->exp_idx)) {
+			/* still in the current hole */
+			/* enqueue the current on the buffer chain */
+			if (ptr->p[cur_idx] != NULL) {
+				DHD_REORDER(("%s: HOLE: ERROR buffer pending..free it\n",
+					__FUNCTION__));
+				PKTFREE(dhd->osh, ptr->p[cur_idx], TRUE);
+				ptr->p[cur_idx] = NULL;
+			}
+			ptr->p[cur_idx] = cur_pkt;
+			ptr->pend_pkts++;
+			ptr->cur_idx = cur_idx;
+			DHD_REORDER(("%s: fill up a hole..pending packets is %d\n",
+				__FUNCTION__, ptr->pend_pkts));
+			*pkt_count = 0;
+			*pkt = NULL;
+		}
+		else if (ptr->exp_idx == cur_idx) {
+			/* got the right one ..flush from cur to exp and update exp */
+			DHD_REORDER(("%s: got the right one now, cur_idx is %d\n",
+				__FUNCTION__, cur_idx));
+			if (ptr->p[cur_idx] != NULL) {
+				DHD_REORDER(("%s: Error buffer pending..free it\n",
+					__FUNCTION__));
+				PKTFREE(dhd->osh, ptr->p[cur_idx], TRUE);
+				ptr->p[cur_idx] = NULL;
+			}
+			ptr->p[cur_idx] = cur_pkt;
+			ptr->pend_pkts++;
+
+			ptr->cur_idx = cur_idx;
+			ptr->exp_idx = exp_idx;
+
+			dhd_get_hostreorder_pkts(dhd->osh, ptr, pkt, &cnt, &plast,
+				cur_idx, exp_idx);
+			*pkt_count = cnt;
+			DHD_REORDER(("%s: freeing up buffers %d, still pending %d\n",
+				__FUNCTION__, cnt, ptr->pend_pkts));
+		}
+		else {
+			uint8 end_idx;
+			bool flush_current = FALSE;
+			/* both cur and exp are moved now .. */
+			DHD_REORDER(("%s:, flow %d, both moved, cur %d(%d), exp %d(%d)\n",
+				__FUNCTION__, flow_id, ptr->cur_idx, cur_idx,
+				ptr->exp_idx, exp_idx));
+			if (flags & WLHOST_REORDERDATA_FLUSH_ALL)
+				end_idx = ptr->exp_idx;
+			else
+				end_idx = exp_idx;
+
+			/* flush pkts first */
+			dhd_get_hostreorder_pkts(dhd->osh, ptr, pkt, &cnt, &plast,
+				ptr->exp_idx, end_idx);
+
+			if (cur_idx == ptr->max_idx) {
+				if (exp_idx == 0)
+					flush_current = TRUE;
+			} else {
+				if (exp_idx == cur_idx + 1)
+					flush_current = TRUE;
+			}
+			if (flush_current) {
+				if (plast)
+					PKTSETNEXT(dhd->osh, plast, cur_pkt);
+				else
+					*pkt = cur_pkt;
+				cnt++;
+			}
+			else {
+				ptr->p[cur_idx] = cur_pkt;
+				ptr->pend_pkts++;
+			}
+			ptr->exp_idx = exp_idx;
+			ptr->cur_idx = cur_idx;
+			*pkt_count = cnt;
+		}
+	}
+	else {
+		uint8 end_idx;
+		/* no real packet but update to exp_seq...that means explicit window move */
+		exp_idx = reorder_info_buf[WLHOST_REORDERDATA_EXPIDX_OFFSET];
+
+		DHD_REORDER(("%s: move the window, cur_idx is %d, exp is %d, new exp is %d\n",
+			__FUNCTION__, ptr->cur_idx, ptr->exp_idx, exp_idx));
+		if (flags & WLHOST_REORDERDATA_FLUSH_ALL)
+			end_idx =  ptr->exp_idx;
+		else
+			end_idx =  exp_idx;
+
+		dhd_get_hostreorder_pkts(dhd->osh, ptr, pkt, &cnt, &plast, ptr->exp_idx, end_idx);
+		if (plast)
+			PKTSETNEXT(dhd->osh, plast, cur_pkt);
+		else
+			*pkt = cur_pkt;
+		cnt++;
+		*pkt_count = cnt;
+		/* set the new expected idx */
+		ptr->exp_idx = exp_idx;
+	}
+	return 0;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_cfg80211.c b/drivers/net/wireless/ap12275/dhd_cfg80211.c
new file mode 100644
index 000000000000..370510189ffe
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_cfg80211.c
@@ -0,0 +1,615 @@
+/*
+ * Linux cfg80211 driver - Dongle Host Driver (DHD) related
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <linux/vmalloc.h>
+#include <net/rtnetlink.h>
+
+#include <bcmutils.h>
+#include <wldev_common.h>
+#include <wl_cfg80211.h>
+#include <dhd_cfg80211.h>
+
+#ifdef PKT_FILTER_SUPPORT
+#include <dngl_stats.h>
+#include <dhd.h>
+#endif
+
+#ifdef PKT_FILTER_SUPPORT
+extern uint dhd_pkt_filter_enable;
+extern uint dhd_master_mode;
+extern void dhd_pktfilter_offload_enable(dhd_pub_t * dhd, char *arg, int enable, int master_mode);
+#endif
+
+static int dhd_dongle_up = FALSE;
+#define PKT_FILTER_BUF_SIZE 64
+
+#if defined(BCMDONGLEHOST)
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhdioctl.h>
+#include <wlioctl.h>
+#include <brcm_nl80211.h>
+#include <dhd_cfg80211.h>
+#endif /* defined(BCMDONGLEHOST) */
+
+static s32 wl_dongle_up(struct net_device *ndev);
+static s32 wl_dongle_down(struct net_device *ndev);
+#ifndef OEM_ANDROID
+static s32 wl_dongle_power(struct net_device *ndev, u32 power_mode);
+#ifdef BCMSDIO /* glomming is a sdio specific feature */
+static s32 wl_dongle_glom(struct net_device *ndev, s32 glom, u32 dongle_align);
+#endif
+static s32 wl_dongle_scantime(struct net_device *ndev, s32 scan_assoc_time, s32 scan_unassoc_time);
+static s32 wl_dongle_offload(struct net_device *ndev, s32 arpoe, s32 arp_ol);
+static s32 wl_pattern_atoh(s8 *src, s8 *dst);
+static s32 wl_dongle_filter(struct net_device *ndev, u32 filter_mode);
+#endif /* !OEM_ANDROID */
+
+/**
+ * Function implementations
+ */
+
+s32 dhd_cfg80211_init(struct bcm_cfg80211 *cfg)
+{
+	dhd_dongle_up = FALSE;
+	return 0;
+}
+
+s32 dhd_cfg80211_deinit(struct bcm_cfg80211 *cfg)
+{
+	dhd_dongle_up = FALSE;
+	return 0;
+}
+
+s32 dhd_cfg80211_down(struct bcm_cfg80211 *cfg)
+{
+	struct net_device *ndev;
+	s32 err = 0;
+
+	WL_TRACE(("In\n"));
+	if (!dhd_dongle_up) {
+		WL_INFORM_MEM(("Dongle is already down\n"));
+		err = 0;
+		goto done;
+	}
+	ndev = bcmcfg_to_prmry_ndev(cfg);
+	wl_dongle_down(ndev);
+done:
+	return err;
+}
+
+s32 dhd_cfg80211_set_p2p_info(struct bcm_cfg80211 *cfg, int val)
+{
+	dhd_pub_t *dhd =  (dhd_pub_t *)(cfg->pub);
+	dhd->op_mode |= val;
+	WL_ERR(("Set : op_mode=0x%04x\n", dhd->op_mode));
+
+	return 0;
+}
+
+s32 dhd_cfg80211_clean_p2p_info(struct bcm_cfg80211 *cfg)
+{
+	dhd_pub_t *dhd =  (dhd_pub_t *)(cfg->pub);
+	dhd->op_mode &= ~(DHD_FLAG_P2P_GC_MODE | DHD_FLAG_P2P_GO_MODE);
+	WL_ERR(("Clean : op_mode=0x%04x\n", dhd->op_mode));
+
+	return 0;
+}
+#ifdef WL_STATIC_IF
+int32
+wl_cfg80211_update_iflist_info(struct bcm_cfg80211 *cfg, struct net_device *ndev,
+	int ifidx, uint8 *addr, int bssidx, char *name, int if_state)
+{
+		return dhd_update_iflist_info(cfg->pub, ndev, ifidx, addr, bssidx, name, if_state);
+}
+#endif /* WL_STATIC_IF */
+struct net_device* wl_cfg80211_allocate_if(struct bcm_cfg80211 *cfg, int ifidx, const char *name,
+	uint8 *mac, uint8 bssidx, const char *dngl_name)
+{
+	return dhd_allocate_if(cfg->pub, ifidx, name, mac, bssidx, FALSE, dngl_name);
+}
+
+int wl_cfg80211_register_if(struct bcm_cfg80211 *cfg,
+	int ifidx, struct net_device* ndev, bool rtnl_lock_reqd)
+{
+	return dhd_register_if(cfg->pub, ifidx, rtnl_lock_reqd);
+}
+
+int wl_cfg80211_remove_if(struct bcm_cfg80211 *cfg,
+	int ifidx, struct net_device* ndev, bool rtnl_lock_reqd)
+{
+	return dhd_remove_if(cfg->pub, ifidx, rtnl_lock_reqd);
+}
+
+void wl_cfg80211_cleanup_if(struct net_device *net)
+{
+	struct bcm_cfg80211 *cfg = wl_get_cfg(net);
+	BCM_REFERENCE(cfg);
+	dhd_cleanup_if(net);
+}
+
+struct net_device * dhd_cfg80211_netdev_free(struct net_device *ndev)
+{
+	struct bcm_cfg80211 *cfg;
+
+	if (ndev) {
+		cfg = wl_get_cfg(ndev);
+		if (ndev->ieee80211_ptr) {
+			MFREE(cfg->osh, ndev->ieee80211_ptr, sizeof(struct wireless_dev));
+			ndev->ieee80211_ptr = NULL;
+		}
+		free_netdev(ndev);
+		return NULL;
+	}
+
+	return ndev;
+}
+
+void dhd_netdev_free(struct net_device *ndev)
+{
+#ifdef WL_CFG80211
+	ndev = dhd_cfg80211_netdev_free(ndev);
+#endif
+	if (ndev)
+		free_netdev(ndev);
+}
+
+static s32
+wl_dongle_up(struct net_device *ndev)
+{
+	s32 err = 0;
+	u32 local_up = 0;
+#ifdef WLAN_ACCEL_BOOT
+	u32 bus_host_access = 1;
+	err = wldev_iovar_setint(ndev, "bus:host_access", bus_host_access);
+	if (unlikely(err)) {
+		WL_ERR(("bus:host_access(%d) error (%d)\n", bus_host_access, err));
+	}
+#endif /* WLAN_ACCEL_BOOT */
+	err = wldev_ioctl_set(ndev, WLC_UP, &local_up, sizeof(local_up));
+	if (unlikely(err)) {
+		WL_ERR(("WLC_UP error (%d)\n", err));
+	} else {
+		WL_INFORM_MEM(("wl up\n"));
+		dhd_dongle_up = TRUE;
+	}
+	return err;
+}
+
+static s32
+wl_dongle_down(struct net_device *ndev)
+{
+	s32 err = 0;
+	u32 local_down = 0;
+#ifdef WLAN_ACCEL_BOOT
+	u32 bus_host_access = 0;
+#endif /* WLAN_ACCEL_BOOT */
+
+	err = wldev_ioctl_set(ndev, WLC_DOWN, &local_down, sizeof(local_down));
+	if (unlikely(err)) {
+		WL_ERR(("WLC_DOWN error (%d)\n", err));
+	}
+#ifdef WLAN_ACCEL_BOOT
+	err = wldev_iovar_setint(ndev, "bus:host_access", bus_host_access);
+	if (unlikely(err)) {
+		WL_ERR(("bus:host_access(%d) error (%d)\n", bus_host_access, err));
+	}
+#endif /* WLAN_ACCEL_BOOT */
+	WL_INFORM_MEM(("wl down\n"));
+	dhd_dongle_up = FALSE;
+
+	return err;
+}
+
+#ifndef OEM_ANDROID
+static s32 wl_dongle_power(struct net_device *ndev, u32 power_mode)
+{
+	s32 err = 0;
+
+	WL_TRACE(("In\n"));
+	err = wldev_ioctl_set(ndev, WLC_SET_PM, &power_mode, sizeof(power_mode));
+	if (unlikely(err)) {
+		WL_ERR(("WLC_SET_PM error (%d)\n", err));
+	}
+	return err;
+}
+
+#ifdef BCMSDIO
+static s32
+wl_dongle_glom(struct net_device *ndev, s32 glom, u32 dongle_align)
+{
+	s32 err = 0;
+
+	/* Match Host and Dongle rx alignment */
+	err = wldev_iovar_setint(ndev, "bus:txglomalign", dongle_align);
+	if (unlikely(err)) {
+		WL_ERR(("txglomalign error (%d)\n", err));
+		goto dongle_glom_out;
+	}
+	/* disable glom option per default */
+	if (glom != DEFAULT_GLOM_VALUE) {
+		err = wldev_iovar_setint(ndev, "bus:txglom", glom);
+		if (unlikely(err)) {
+			WL_ERR(("txglom error (%d)\n", err));
+			goto dongle_glom_out;
+		}
+	}
+dongle_glom_out:
+	return err;
+}
+
+#endif /* BCMSDIO */
+#endif /* !OEM_ANDROID */
+
+s32
+wl_dongle_roam(struct net_device *ndev, u32 roamvar, u32 bcn_timeout)
+{
+	s32 err = 0;
+
+	/* Setup timeout if Beacons are lost and roam is off to report link down */
+	if (roamvar) {
+		err = wldev_iovar_setint(ndev, "bcn_timeout", bcn_timeout);
+		if (unlikely(err)) {
+			WL_ERR(("bcn_timeout error (%d)\n", err));
+			goto dongle_rom_out;
+		}
+	}
+	/* Enable/Disable built-in roaming to allow supplicant to take care of roaming */
+	err = wldev_iovar_setint(ndev, "roam_off", roamvar);
+	if (unlikely(err)) {
+		WL_ERR(("roam_off error (%d)\n", err));
+		goto dongle_rom_out;
+	}
+	ROAMOFF_DBG_SAVE(ndev, SET_ROAM_FILS_TOGGLE, roamvar);
+dongle_rom_out:
+	return err;
+}
+
+#ifndef OEM_ANDROID
+static s32
+wl_dongle_scantime(struct net_device *ndev, s32 scan_assoc_time,
+	s32 scan_unassoc_time)
+{
+	s32 err = 0;
+
+	err = wldev_ioctl_set(ndev, WLC_SET_SCAN_CHANNEL_TIME, &scan_assoc_time,
+		sizeof(scan_assoc_time));
+	if (err) {
+		if (err == -EOPNOTSUPP) {
+			WL_INFORM(("Scan assoc time is not supported\n"));
+		} else {
+			WL_ERR(("Scan assoc time error (%d)\n", err));
+		}
+		goto dongle_scantime_out;
+	}
+	err = wldev_ioctl_set(ndev, WLC_SET_SCAN_UNASSOC_TIME, &scan_unassoc_time,
+		sizeof(scan_unassoc_time));
+	if (err) {
+		if (err == -EOPNOTSUPP) {
+			WL_INFORM(("Scan unassoc time is not supported\n"));
+		} else {
+			WL_ERR(("Scan unassoc time error (%d)\n", err));
+		}
+		goto dongle_scantime_out;
+	}
+
+dongle_scantime_out:
+	return err;
+}
+
+static s32
+wl_dongle_offload(struct net_device *ndev, s32 arpoe, s32 arp_ol)
+{
+	s8 iovbuf[WLC_IOCTL_SMLEN];
+	s32 err = 0;
+	s32 len;
+	struct bcm_cfg80211 *cfg = wl_get_cfg(ndev);
+	dhd_pub_t *dhd = (dhd_pub_t *)(cfg->pub);
+
+	/* Set ARP offload */
+	len = bcm_mkiovar("arpoe", (char *)&arpoe, sizeof(arpoe), iovbuf, sizeof(iovbuf));
+	if (!len) {
+		WL_ERR(("%s: bcm_mkiovar failed:%d\n", __FUNCTION__, len));
+		return BCME_BADARG;
+	}
+	err = wldev_ioctl_set(ndev, WLC_SET_VAR, iovbuf, len);
+	if (err) {
+		if (err == -EOPNOTSUPP)
+			WL_INFORM(("arpoe is not supported\n"));
+		else
+			WL_ERR(("arpoe error (%d)\n", err));
+
+		goto dongle_offload_out;
+	}
+	len = bcm_mkiovar("arp_ol", (char *)&arp_ol, sizeof(arp_ol), iovbuf, sizeof(iovbuf));
+	if (!len) {
+		WL_ERR(("%s: bcm_mkiovar failed:%d\n", __FUNCTION__, len));
+		return BCME_BADARG;
+	}
+	err = wldev_ioctl_set(ndev, WLC_SET_VAR, iovbuf, len);
+	if (err) {
+		if (err == -EOPNOTSUPP)
+			WL_INFORM(("arp_ol is not supported\n"));
+		else
+			WL_ERR(("arp_ol error (%d)\n", err));
+
+		goto dongle_offload_out;
+	}
+
+	dhd->arpoe_enable = TRUE;
+	dhd->arpol_configured = TRUE;
+	WL_ERR(("arpoe:%d arpol:%d\n",
+		dhd->arpoe_enable, dhd->arpol_configured));
+
+dongle_offload_out:
+	return err;
+}
+
+static s32 wl_pattern_atoh(s8 *src, s8 *dst)
+{
+	int i;
+	if (strncmp(src, "0x", 2) != 0 && strncmp(src, "0X", 2) != 0) {
+		WL_ERR(("Mask invalid format. Needs to start with 0x\n"));
+		return -1;
+	}
+	src = src + 2;		/* Skip past 0x */
+	if (strlen(src) % 2 != 0) {
+		WL_ERR(("Mask invalid format. Needs to be of even length\n"));
+		return -1;
+	}
+
+	for (i = 0; *src != '\0'; i++) {
+		char num[3];
+		if ((num[0] = src[0]) != '\0') {
+			num[1] = src[1];
+		}
+		num[2] = '\0';
+		dst[i] = (u8) simple_strtoul(num, NULL, 16);
+		src += 2;
+	}
+
+	return i;
+}
+
+static s32 wl_dongle_filter(struct net_device *ndev, u32 filter_mode)
+{
+	const s8 *str;
+	struct wl_pkt_filter pkt_filter;
+	struct wl_pkt_filter *pkt_filterp;
+	s32 buf_len;
+	s32 str_len;
+	u32 mask_size;
+	u32 pattern_size;
+	s8 buf[PKT_FILTER_BUF_SIZE] = {0};
+	s32 err = 0;
+
+	/* add a default packet filter pattern */
+	str = "pkt_filter_add";
+	str_len = strlen(str);
+	strlcpy(buf, str, sizeof(buf));
+	buf_len = str_len + 1;
+
+	pkt_filterp = (struct wl_pkt_filter *)(buf + str_len + 1);
+
+	/* Parse packet filter id. */
+	pkt_filter.id = htod32(100);
+
+	/* Parse filter polarity. */
+	pkt_filter.negate_match = htod32(0);
+
+	/* Parse filter type. */
+	pkt_filter.type = htod32(0);
+
+	/* Parse pattern filter offset. */
+	pkt_filter.u.pattern.offset = htod32(0);
+
+	/* Parse pattern filter mask. */
+	mask_size = htod32(wl_pattern_atoh("0xff",
+		(char *)pkt_filterp->u.pattern.
+		    mask_and_pattern));
+
+	if (mask_size == (typeof(mask_size))-1 ||
+		(mask_size > (PKT_FILTER_BUF_SIZE - (buf_len) +
+		WL_PKT_FILTER_FIXED_LEN + WL_PKT_FILTER_PATTERN_FIXED_LEN))) {
+		/* mask_size has to be equal to pattern_size */
+		err = -EINVAL;
+		goto dongle_filter_out;
+	}
+	/* Parse pattern filter pattern. */
+	pattern_size = htod32(wl_pattern_atoh("0x00",
+		(char *)&pkt_filterp->u.pattern.mask_and_pattern[mask_size]));
+
+	if (mask_size != pattern_size) {
+		WL_ERR(("Mask and pattern not the same size\n"));
+		err = -EINVAL;
+		goto dongle_filter_out;
+	}
+
+	pkt_filter.u.pattern.size_bytes = mask_size;
+	buf_len += WL_PKT_FILTER_FIXED_LEN;
+	buf_len += (WL_PKT_FILTER_PATTERN_FIXED_LEN + 2 * mask_size);
+
+	/* Keep-alive attributes are set in local
+	 * variable (keep_alive_pkt), and
+	 * then memcpy'ed into buffer (keep_alive_pktp) since there is no
+	 * guarantee that the buffer is properly aligned.
+	 */
+	memcpy((char *)pkt_filterp, &pkt_filter,
+		WL_PKT_FILTER_FIXED_LEN + WL_PKT_FILTER_PATTERN_FIXED_LEN);
+
+	err = wldev_ioctl_set(ndev, WLC_SET_VAR, buf, buf_len);
+	if (err) {
+		if (err == -EOPNOTSUPP) {
+			WL_INFORM(("filter not supported\n"));
+		} else {
+			WL_ERR(("filter (%d)\n", err));
+		}
+		goto dongle_filter_out;
+	}
+
+	/* set mode to allow pattern */
+	err = wldev_iovar_setint(ndev, "pkt_filter_mode", filter_mode);
+	if (err) {
+		if (err == -EOPNOTSUPP) {
+			WL_INFORM(("filter_mode not supported\n"));
+		} else {
+			WL_ERR(("filter_mode (%d)\n", err));
+		}
+		goto dongle_filter_out;
+	}
+
+dongle_filter_out:
+	return err;
+}
+#endif /* !OEM_ANDROID */
+
+s32 dhd_config_dongle(struct bcm_cfg80211 *cfg)
+{
+#ifndef DHD_SDALIGN
+#define DHD_SDALIGN	32
+#endif
+	struct net_device *ndev;
+	s32 err = 0;
+	dhd_pub_t *dhd = NULL;
+#if !defined(OEM_ANDROID) && defined(BCMSDIO)
+	s32 glom = CUSTOM_GLOM_SETTING;
+	BCM_REFERENCE(glom);
+#endif
+
+	WL_TRACE(("In\n"));
+
+	ndev = bcmcfg_to_prmry_ndev(cfg);
+	dhd = (dhd_pub_t *)(cfg->pub);
+
+	err = wl_dongle_up(ndev);
+	if (unlikely(err)) {
+		WL_ERR(("wl_dongle_up failed\n"));
+		goto default_conf_out;
+	}
+
+	if (dhd && dhd->fw_preinit) {
+		/* Init config will be done by fw preinit context */
+		return BCME_OK;
+	}
+
+#ifndef OEM_ANDROID
+	err = wl_dongle_power(ndev, PM_FAST);
+	if (unlikely(err)) {
+		WL_ERR(("wl_dongle_power failed\n"));
+		goto default_conf_out;
+	}
+#ifdef BCMSDIO
+	err = wl_dongle_glom(ndev, glom, DHD_SDALIGN);
+	if (unlikely(err)) {
+		WL_ERR(("wl_dongle_glom failed\n"));
+		goto default_conf_out;
+	}
+#endif /* BCMSDIO */
+	err = wl_dongle_roam(ndev, (cfg->roam_on ? 0 : 1), 3);
+	if (unlikely(err)) {
+		WL_ERR(("wl_dongle_roam failed\n"));
+		goto default_conf_out;
+	}
+	wl_dongle_scantime(ndev, 40, 80);
+	wl_dongle_offload(ndev, 1, 0xf);
+	wl_dongle_filter(ndev, 1);
+#endif /* OEM_ANDROID */
+
+default_conf_out:
+
+	return err;
+
+}
+
+int dhd_cfgvendor_priv_string_handler(struct bcm_cfg80211 *cfg, struct wireless_dev *wdev,
+	const struct bcm_nlmsg_hdr *nlioc, void *buf)
+{
+	struct net_device *ndev = NULL;
+	dhd_pub_t *dhd;
+	dhd_ioctl_t ioc = { 0, NULL, 0, 0, 0, 0, 0};
+	int ret = 0;
+	int8 index;
+
+	WL_TRACE(("entry: cmd = %d\n", nlioc->cmd));
+
+	dhd = cfg->pub;
+	DHD_OS_WAKE_LOCK(dhd);
+
+	ndev = wdev_to_wlc_ndev(wdev, cfg);
+	index = dhd_net2idx(dhd->info, ndev);
+	if (index == DHD_BAD_IF) {
+		WL_ERR(("Bad ifidx from wdev:%p\n", wdev));
+		ret = BCME_ERROR;
+		goto done;
+	}
+
+	ioc.cmd = nlioc->cmd;
+	ioc.len = nlioc->len;
+	ioc.set = nlioc->set;
+	ioc.driver = nlioc->magic;
+	ioc.buf = buf;
+	ret = dhd_ioctl_process(dhd, index, &ioc, buf);
+	if (ret) {
+		WL_TRACE(("dhd_ioctl_process return err %d\n", ret));
+		ret = OSL_ERROR(ret);
+		goto done;
+	}
+
+done:
+	DHD_OS_WAKE_UNLOCK(dhd);
+	return ret;
+}
+
+int
+dhd_set_wsec_info(dhd_pub_t *dhd, uint32 data, int tag)
+{
+	struct net_device *ndev;
+	uint32  wsec_info = data;
+	int ret = BCME_OK;
+
+	ndev = dhd_linux_get_primary_netdev(dhd);
+	if (!ndev) {
+		WL_ERR(("Cannot find primary netdev\n"));
+		return -ENODEV;
+	}
+	BCM_REFERENCE(wsec_info);
+#if defined(WL_CFG80211)
+	ret = wl_cfg80211_set_wsec_info(ndev, &wsec_info, sizeof(wsec_info), tag);
+	if (unlikely(ret)) {
+		WL_ERR(("Set wsec_info tag 0x%04x failed \n", tag));
+	}
+#endif /* WL_CFG80211 */
+
+	return ret;
+}
+
+#ifdef RPM_FAST_TRIGGER
+void
+dhd_trigger_rpm_fast(struct bcm_cfg80211 *cfg)
+{
+	dhd_pub_t *dhd = (dhd_pub_t *)cfg->pub;
+	dhdpcie_trigger_rpm_fast(dhd);
+}
+#endif /* RPM_FAST_TRIGGER */
diff --git a/drivers/net/wireless/ap12275/dhd_cfg80211.h b/drivers/net/wireless/ap12275/dhd_cfg80211.h
new file mode 100644
index 000000000000..ae24bce5503c
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_cfg80211.h
@@ -0,0 +1,53 @@
+/*
+ * Linux cfg80211 driver - Dongle Host Driver (DHD) related
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_CFG80211__
+#define __DHD_CFG80211__
+
+#include <wl_cfg80211.h>
+#include <wl_cfgp2p.h>
+#include <brcm_nl80211.h>
+
+#ifndef WL_ERR
+#define WL_ERR CFG80211_ERR
+#endif
+#ifndef WL_TRACE
+#define WL_TRACE CFG80211_TRACE
+#endif
+
+s32 dhd_cfg80211_init(struct bcm_cfg80211 *cfg);
+s32 dhd_cfg80211_deinit(struct bcm_cfg80211 *cfg);
+s32 dhd_cfg80211_down(struct bcm_cfg80211 *cfg);
+s32 dhd_cfg80211_set_p2p_info(struct bcm_cfg80211 *cfg, int val);
+s32 dhd_cfg80211_clean_p2p_info(struct bcm_cfg80211 *cfg);
+s32 dhd_config_dongle(struct bcm_cfg80211 *cfg);
+int dhd_cfgvendor_priv_string_handler(struct bcm_cfg80211 *cfg,
+	struct wireless_dev *wdev, const struct bcm_nlmsg_hdr *nlioc, void  *data);
+s32 wl_dongle_roam(struct net_device *ndev, u32 roamvar, u32 bcn_timeout);
+int dhd_set_wsec_info(dhd_pub_t *dhd, uint32 data, int tag);
+#ifdef RPM_FAST_TRIGGER
+void dhd_trigger_rpm_fast(struct bcm_cfg80211 *cfg);
+#endif /* RPM_FAST_TRIGGER */
+#endif /* __DHD_CFG80211__ */
diff --git a/drivers/net/wireless/ap12275/dhd_common.c b/drivers/net/wireless/ap12275/dhd_common.c
new file mode 100644
index 000000000000..6a1cfe9c3175
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_common.c
@@ -0,0 +1,12043 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), common DHD core.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <typedefs.h>
+#include <osl.h>
+
+#include <epivers.h>
+#include <bcmutils.h>
+#include <bcmstdlib_s.h>
+
+#include <bcmdevs.h>
+#include <bcmendian.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_ip.h>
+#include <bcmevent.h>
+#include <dhdioctl.h>
+#ifdef DHD_SDTC_ETB_DUMP
+#include <bcmiov.h>
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#if defined(DHD_MACDBG) && defined(BCMDBG)
+#include <dhd_macdbg.h>
+#endif /* defined(DHD_MACDBG) && defined(BCMDBG) */
+
+#ifdef PCIE_FULL_DONGLE
+#include <bcmmsgbuf.h>
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef SHOW_LOGTRACE
+#include <event_log.h>
+#endif /* SHOW_LOGTRACE */
+
+#ifdef BCMPCIE
+#include <dhd_flowring.h>
+#endif
+
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <bcmsdbus.h>
+#include <dhd_dbg.h>
+#include <802.1d.h>
+#include <dhd_debug.h>
+#ifdef DHD_LOG_DUMP
+#include <dhd_log_dump.h>
+#endif
+#include <dhd_dbg_ring.h>
+#include <dhd_mschdbg.h>
+#include <msgtrace.h>
+#include <dhd_config.h>
+#include <wl_android.h>
+
+#ifdef WL_CFG80211
+#include <wl_cfg80211.h>
+#include <wl_cfgvif.h>
+#endif
+#if defined(OEM_ANDROID) && defined(PNO_SUPPORT)
+#include <dhd_pno.h>
+#endif /* (OEM_ANDROID) && (PNO_SUPPORT) */
+#ifdef RTT_SUPPORT
+#include <dhd_rtt.h>
+#endif
+
+#ifdef DNGL_EVENT_SUPPORT
+#include <dnglevent.h>
+#endif
+
+#ifdef IL_BIGENDIAN
+#include <bcmendian.h>
+#define htod32(i) (bcmswap32(i))
+#define htod16(i) (bcmswap16(i))
+#define dtoh32(i) (bcmswap32(i))
+#define dtoh16(i) (bcmswap16(i))
+#define htodchanspec(i) htod16(i)
+#define dtohchanspec(i) dtoh16(i)
+#else
+#define htod32(i) (i)
+#define htod16(i) (i)
+#define dtoh32(i) (i)
+#define dtoh16(i) (i)
+#define htodchanspec(i) (i)
+#define dtohchanspec(i) (i)
+#endif /* IL_BIGENDINA */
+
+#ifdef PROP_TXSTATUS
+#include <wlfc_proto.h>
+#include <dhd_wlfc.h>
+#endif
+
+#if defined(__linux__)
+#include <dhd_linux.h>
+#endif /* __linux__ */
+
+#ifdef DHD_WMF
+#include <dhd_wmf_linux.h>
+#endif /* DHD_WMF */
+
+#ifdef DHD_L2_FILTER
+#include <dhd_l2_filter.h>
+#endif /* DHD_L2_FILTER */
+
+#ifdef DHD_PSTA
+#include <dhd_psta.h>
+#endif /* DHD_PSTA */
+
+#ifdef DHD_WET
+#include <dhd_wet.h>
+#endif /* DHD_WET */
+#if defined(NDIS)
+#include <siutils.h>
+#endif
+
+#ifdef DHD_LOG_DUMP
+#include <dhd_dbg.h>
+#ifdef DHD_PKT_LOGGING
+#include <dhd_pktlog.h>
+#endif
+#endif /* DHD_LOG_DUMP */
+
+#ifdef CSI_SUPPORT
+#include <dhd_csi.h>
+#endif /* CSI_SUPPORT */
+
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+int log_print_threshold = 0;
+#endif /* DHD_LOG_PRINT_RATE_LIMIT */
+
+/* dhd_msg_level : a default level to print to dmesg buffer
+ * dhd_log_level : a default level to log to DLD or Ring
+ * To keep one level operation(dhd_msg_level) in HW4,
+ * dhd_msg_level and dhd_log_level have the same level
+ */
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+int dhd_msg_level = DHD_ERROR_VAL | DHD_EVENT_VAL
+#ifdef BOARD_HIKEY
+		| DHD_FWLOG_VAL
+#endif /* BOARD_HIKEY */
+#ifndef DHD_REDUCE_PM_LOG
+		| DHD_RPM_VAL
+#endif /* REDUCE_PM_LOG */
+	        | DHD_PKT_MON_VAL;
+int dhd_log_level = DHD_ERROR_VAL | DHD_EVENT_VAL
+		| DHD_RPM_VAL
+		| DHD_PKT_MON_VAL | DHD_FWLOG_VAL | DHD_IOVAR_MEM_VAL;
+#else
+/* For CUSTOMER_HW4/Hikey do not enable DHD_ERROR_MEM_VAL by default */
+int dhd_msg_level = DHD_ERROR_VAL | DHD_FWLOG_VAL;
+
+int dhd_log_level = DHD_ERROR_VAL | DHD_FWLOG_VAL | DHD_EVENT_VAL
+	/* For CUSTOMER_HW4 do not enable DHD_IOVAR_MEM_VAL by default */
+#if !defined(BOARD_HIKEY)
+	| DHD_IOVAR_MEM_VAL
+#endif
+#ifndef OEM_ANDROID
+	| DHD_MSGTRACE_VAL
+#endif /* OEM_ANDROID */
+	| DHD_RPM_VAL
+	| DHD_PKT_MON_VAL;
+
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+
+#ifdef NDIS
+extern uint wl_msg_level;
+#endif
+
+#if defined(WL_WLC_SHIM)
+#include <wl_shim.h>
+#else
+#if defined(NDIS)
+#include <wl_port_if.h>
+#endif
+#endif /* WL_WLC_SHIM */
+
+#ifdef DHD_DEBUG
+#include <sdiovar.h>
+#endif /* DHD_DEBUG */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#include <linux/pm_runtime.h>
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if defined(DHD_LOGLEVEL) && !defined(DHD_DEBUG)
+#error "Log level control is supported only with DHD_DEBUG"
+#endif  /* defined(DHD_LOGLEVEL) && !defined(DHD_DEBUG) */
+
+#ifdef SOFTAP
+char fw_path2[MOD_PARAM_PATHLEN];
+extern bool softap_enabled;
+#endif
+#ifdef PROP_TXSTATUS
+extern int disable_proptx;
+#endif /* PROP_TXSTATUS */
+
+#ifdef REPORT_FATAL_TIMEOUTS
+/* Default timeout value in ms */
+#define BUS_TIMEOUT_DEFAULT     800
+#define CMD_TIMEOUT_DEFAULT     1200
+#define SCAN_TIMEOUT_DEFAULT    17000
+#define JOIN_TIMEOUT_DEFAULT    7500
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+#ifdef SHOW_LOGTRACE
+#define BYTES_AHEAD_NUM		10	/* address in map file is before these many bytes */
+#define READ_NUM_BYTES		1000 /* read map file each time this No. of bytes */
+#define GO_BACK_FILE_POS_NUM_BYTES	100 /* set file pos back to cur pos */
+static char *ramstart_str = " text_start"; /* string in mapfile has addr ramstart */
+static char *rodata_start_str = " rodata_start"; /* string in mapfile has addr rodata start */
+static char *rodata_end_str = " rodata_end"; /* string in mapfile has addr rodata end */
+#define RAMSTART_BIT	0x01
+#define RDSTART_BIT		0x02
+#define RDEND_BIT		0x04
+#define ALL_MAP_VAL		(RAMSTART_BIT | RDSTART_BIT | RDEND_BIT)
+#endif /* SHOW_LOGTRACE */
+
+#ifdef SHOW_LOGTRACE
+#if defined(LINUX) || defined(linux)
+/* the fw file path is taken from either the module parameter at
+ * insmod time or is defined as a constant of different values
+ * for different platforms
+ */
+extern char *st_str_file_path;
+#else
+static char *st_str_file_path = "rtecdc.bin";
+#endif /* LINUX */
+#endif /* SHOW_LOGTRACE */
+
+#ifdef EWP_EDL
+typedef struct msg_hdr_edl {
+	uint32 infobuf_ver;
+	info_buf_payload_hdr_t pyld_hdr;
+	msgtrace_hdr_t trace_hdr;
+} msg_hdr_edl_t;
+#endif /* EWP_EDL */
+
+#define DHD_TPUT_MAX_TX_PKTS_BATCH	1000
+
+/* Last connection success/failure status */
+uint32 dhd_conn_event;
+uint32 dhd_conn_status;
+uint32 dhd_conn_reason;
+
+extern int dhd_iscan_request(void * dhdp, uint16 action);
+extern void dhd_ind_scan_confirm(void *h, bool status);
+extern int dhd_iscan_in_progress(void *h);
+void dhd_iscan_lock(void);
+void dhd_iscan_unlock(void);
+extern int dhd_change_mtu(dhd_pub_t *dhd, int new_mtu, int ifidx);
+
+#ifndef BCMDBUS
+extern int dhd_socram_dump(struct dhd_bus *bus);
+#endif /* BCMDBUS */
+
+extern void dhd_set_packet_filter(dhd_pub_t *dhd);
+
+#ifdef DNGL_EVENT_SUPPORT
+static void dngl_host_event_process(dhd_pub_t *dhdp, bcm_dngl_event_t *event,
+	bcm_dngl_event_msg_t *dngl_event, size_t pktlen);
+static int dngl_host_event(dhd_pub_t *dhdp, void *pktdata, bcm_dngl_event_msg_t *dngl_event,
+	size_t pktlen);
+#endif /* DNGL_EVENT_SUPPORT */
+
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+static void copy_hang_info_ioctl_timeout(dhd_pub_t *dhd, int ifidx, wl_ioctl_t *ioc);
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+
+#ifdef REPORT_FATAL_TIMEOUTS
+static void dhd_set_join_error(dhd_pub_t *pub, uint32 mask);
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+#ifdef DHD_SEND_HANG_IOCTL_SUSPEND_ERROR
+#define MAX_IOCTL_SUSPEND_ERROR	10
+static int ioctl_suspend_error = 0;
+#endif /* DHD_SEND_HANG_IOCTL_SUSPEND_ERROR */
+
+/* Should ideally read this from target(taken from wlu) */
+#define MAX_CHUNK_LEN 1408 /* 8 * 8 * 22 */
+
+#if defined(OEM_ANDROID)
+/* note these variables will be used with wext */
+bool ap_cfg_running = FALSE;
+bool ap_fw_loaded = FALSE;
+#endif /* defined(OEM_ANDROID) && defined(SOFTAP) */
+
+#ifdef WLEASYMESH
+extern int dhd_set_1905_almac(dhd_pub_t *dhdp, uint8 ifidx, uint8* ea, bool mcast);
+extern int dhd_get_1905_almac(dhd_pub_t *dhdp, uint8 ifidx, uint8* ea, bool mcast);
+#endif /* WLEASYMESH */
+
+#define CHIPID_MISMATCH	8
+
+#define DHD_VERSION "Dongle Host Driver, version " EPI_VERSION_STR EPI_COMMIT_ID "\n"
+
+#if defined(DHD_DEBUG) && defined(DHD_COMPILED)
+const char dhd_version[] = DHD_VERSION DHD_COMPILED " compiled on "
+			__DATE__ " at " __TIME__ "\n\0<TIMESTAMP>";
+#else
+const char dhd_version[] = DHD_VERSION;
+#endif /* DHD_DEBUG && DHD_COMPILED */
+
+char fw_version[FW_VER_STR_LEN] = "\0";
+char clm_version[CLM_VER_STR_LEN] = "\0";
+
+char bus_api_revision[BUS_API_REV_STR_LEN] = "\0";
+
+void dhd_set_timer(void *bus, uint wdtick);
+
+#if defined(BCM_ROUTER_DHD)
+static int traffic_mgmt_add_dwm_filter(dhd_pub_t *dhd,
+	trf_mgmt_filter_list_t * trf_mgmt_filter_list, int len);
+#endif
+
+static char* ioctl2str(uint32 ioctl);
+
+#ifdef DHD_LOGLEVEL
+void dhd_loglevel_set(dhd_loglevel_data_t *);
+void dhd_loglevel_get(dhd_loglevel_data_t *);
+#endif /* DHD_LOGLEVEL */
+
+/* IOVar table */
+enum {
+	IOV_VERSION = 1,
+	IOV_WLMSGLEVEL,
+	IOV_MSGLEVEL,
+	IOV_BCMERRORSTR,
+	IOV_BCMERROR,
+	IOV_WDTICK,
+	IOV_DUMP,
+	IOV_RESET_COUNTS,
+	IOV_LOGDUMP,
+	IOV_LOGCAL,
+	IOV_LOGSTAMP,
+	IOV_GPIOOB,
+	IOV_IOCTLTIMEOUT,
+	IOV_CONS,
+	IOV_DCONSOLE_POLL,
+#if defined(DHD_DEBUG)
+	IOV_DHD_JOIN_TIMEOUT_DBG,
+	IOV_SCAN_TIMEOUT,
+	IOV_MEM_DEBUG,
+#ifdef BCMPCIE
+	IOV_FLOW_RING_DEBUG,
+#endif /* BCMPCIE */
+#endif /* defined(DHD_DEBUG) */
+#ifdef PROP_TXSTATUS
+	IOV_PROPTXSTATUS_ENABLE,
+	IOV_PROPTXSTATUS_MODE,
+	IOV_PROPTXSTATUS_OPT,
+#ifdef QMONITOR
+	IOV_QMON_TIME_THRES,
+	IOV_QMON_TIME_PERCENT,
+#endif /* QMONITOR */
+	IOV_PROPTXSTATUS_MODULE_IGNORE,
+	IOV_PROPTXSTATUS_CREDIT_IGNORE,
+	IOV_PROPTXSTATUS_TXSTATUS_IGNORE,
+	IOV_PROPTXSTATUS_RXPKT_CHK,
+#endif /* PROP_TXSTATUS */
+	IOV_BUS_TYPE,
+	IOV_CHANGEMTU,
+	IOV_HOSTREORDER_FLOWS,
+#ifdef DHDTCPACK_SUPPRESS
+	IOV_TCPACK_SUPPRESS,
+#endif /* DHDTCPACK_SUPPRESS */
+#ifdef DHD_WMF
+	IOV_WMF_BSS_ENAB,
+	IOV_WMF_UCAST_IGMP,
+	IOV_WMF_MCAST_DATA_SENDUP,
+#ifdef WL_IGMP_UCQUERY
+	IOV_WMF_UCAST_IGMP_QUERY,
+#endif /* WL_IGMP_UCQUERY */
+#ifdef DHD_UCAST_UPNP
+	IOV_WMF_UCAST_UPNP,
+#endif /* DHD_UCAST_UPNP */
+	IOV_WMF_PSTA_DISABLE,
+#endif /* DHD_WMF */
+#if defined(BCM_ROUTER_DHD)
+	IOV_TRAFFIC_MGMT_DWM,
+#endif /* BCM_ROUTER_DHD */
+	IOV_AP_ISOLATE,
+#ifdef DHD_L2_FILTER
+	IOV_DHCP_UNICAST,
+	IOV_BLOCK_PING,
+	IOV_PROXY_ARP,
+	IOV_GRAT_ARP,
+	IOV_BLOCK_TDLS,
+#endif /* DHD_L2_FILTER */
+	IOV_DHD_IE,
+#ifdef DHD_PSTA
+	IOV_PSTA,
+#endif /* DHD_PSTA */
+#ifdef DHD_WET
+	IOV_WET,
+	IOV_WET_HOST_IPV4,
+	IOV_WET_HOST_MAC,
+#endif /* DHD_WET */
+	IOV_CFG80211_OPMODE,
+	IOV_ASSERT_TYPE,
+#if defined(NDIS)
+	IOV_WAKEIND,
+#endif /* NDIS */
+#if !defined(NDIS) && !defined(BCM_ROUTER_DHD)
+	IOV_LMTEST,
+#endif
+#ifdef DHD_MCAST_REGEN
+	IOV_MCAST_REGEN_BSS_ENABLE,
+#endif
+#if defined(DHD_MACDBG) && defined(BCMDBG)
+	IOV_MACDBG_PD11REGS,
+	IOV_MACDBG_REGLIST,
+	IOV_MACDBG_PSVMPMEMS,
+#endif /* defined(DHD_MACDBG) && defined(BCMDBG) */
+#ifdef SHOW_LOGTRACE
+	IOV_DUMP_TRACE_LOG,
+#endif /* SHOW_LOGTRACE */
+#ifdef REPORT_FATAL_TIMEOUTS
+	IOV_SCAN_TO,
+	IOV_JOIN_TO,
+	IOV_CMD_TO,
+	IOV_OQS_TO,
+#endif /* REPORT_FATAL_TIMEOUTS */
+	IOV_DONGLE_TRAP_TYPE,
+	IOV_DONGLE_TRAP_INFO,
+	IOV_BPADDR,
+	IOV_DUMP_DONGLE, /**< dumps core registers and d11 memories */
+#if defined(DHD_LOG_DUMP)
+	IOV_LOG_DUMP,
+#endif /* DHD_LOG_DUMP */
+	IOV_TPUT_TEST,
+	IOV_DEBUG_BUF_DEST_STAT,
+#ifdef DHD_DEBUG
+	IOV_INDUCE_ERROR,
+#endif /* DHD_DEBUG */
+	IOV_FIS_TRIGGER,
+#ifdef WL_IFACE_MGMT_CONF
+#ifdef WL_CFG80211
+#ifdef WL_NANP2P
+	IOV_CONC_DISC,
+#endif /* WL_NANP2P */
+#ifdef WL_IFACE_MGMT
+	IOV_IFACE_POLICY,
+#endif /* WL_IFACE_MGMT */
+#endif /* WL_CFG80211 */
+#endif /* WL_IFACE_MGMT_CONF */
+#ifdef RTT_GEOFENCE_CONT
+#if defined(RTT_SUPPORT) && defined(WL_NAN)
+	IOV_RTT_GEOFENCE_TYPE_OVRD,
+#endif /* RTT_SUPPORT && WL_NAN */
+#endif /* RTT_GEOFENCE_CONT */
+	IOV_FW_VBS,
+#ifdef DHD_TX_PROFILE
+	IOV_TX_PROFILE_TAG,
+	IOV_TX_PROFILE_ENABLE,
+	IOV_TX_PROFILE_DUMP,
+#endif /* defined(DHD_TX_PROFILE) */
+	IOV_CHECK_TRAP_ROT,
+#ifdef DHD_LOGLEVEL
+	IOV_LOGLEVEL,
+#endif /* DHD_LOGLEVEL */
+	IOV_COUNTERS,
+#ifdef BCMPCIE
+	IOV_DUMP_FLOWRINGS,
+#endif
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+	IOV_SAR_MODE,
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+#ifdef CSI_SUPPORT
+	IOV_CSI_CONFIG,
+#endif /* CSI_SUPPORT */
+#ifdef WLEASYMESH
+	IOV_1905_AL_UCAST,
+	IOV_1905_AL_MCAST,
+#endif /* WLEASYMESH */
+	IOV_LAST
+};
+
+const bcm_iovar_t dhd_iovars[] = {
+	/* name         varid                   flags   flags2 type     minlen */
+	{"version",	IOV_VERSION,		0,	0, IOVT_BUFFER,	0},
+	{"wlmsglevel",	IOV_WLMSGLEVEL,	0,	0,	IOVT_UINT32,	0 },
+#if defined(DHD_DEBUG) || defined(DHD_MSG_LEVEL_SUPPORT)
+	{"msglevel",	IOV_MSGLEVEL,		0,	0, IOVT_UINT32,	0},
+#endif /* DHD_DEBUG || DHD_MSG_LEVEL_SUPPORT */
+#ifdef DHD_DEBUG
+	{"mem_debug",   IOV_MEM_DEBUG,  0,      0,      IOVT_BUFFER,    0 },
+#ifdef BCMPCIE
+	{"flow_ring_debug", IOV_FLOW_RING_DEBUG, 0, 0, IOVT_BUFFER, 0 },
+#endif /* BCMPCIE */
+#ifdef NDIS
+	{"wlmsglevel",	IOV_WLMSGLEVEL,		0,	0, IOVT_UINT32,	0},
+#endif /* NDIS */
+#endif /* DHD_DEBUG */
+	{"bcmerrorstr", IOV_BCMERRORSTR,	0,	0, IOVT_BUFFER,	BCME_STRLEN},
+	{"bcmerror",	IOV_BCMERROR,		0,	0, IOVT_INT8,	0},
+	{"wdtick",	IOV_WDTICK,		0,	0, IOVT_UINT32,	0},
+	{"dump",	IOV_DUMP,		0,	0, IOVT_BUFFER,	DHD_IOCTL_MAXLEN_32K},
+	{"cons",	IOV_CONS,		0,	0, IOVT_BUFFER,	0},
+	{"dconpoll",	IOV_DCONSOLE_POLL,	0,	0, IOVT_UINT32,	0},
+	{"reset_cnts", 	IOV_RESET_COUNTS,	0,	0, IOVT_VOID,	0},
+#ifdef BCMPERFSTATS
+	{"logdump", IOV_LOGDUMP,		0,	0, IOVT_BUFFER,	DHD_IOCTL_MAXLEN},
+	{"logcal",	IOV_LOGCAL,		0,	0, IOVT_UINT32,	0},
+	{"logstamp",	IOV_LOGSTAMP,		0,	0, IOVT_BUFFER,	0},
+#endif
+	{"gpioob",	IOV_GPIOOB,		0,	0, IOVT_UINT32,	0},
+	{"ioctl_timeout", IOV_IOCTLTIMEOUT,	0,	0, IOVT_UINT32,	0},
+#ifdef PROP_TXSTATUS
+	{"proptx",	IOV_PROPTXSTATUS_ENABLE,	0,	0, IOVT_BOOL,	0 },
+	/*
+	set the proptxtstatus operation mode:
+	0 - Do not do any proptxtstatus flow control
+	1 - Use implied credit from a packet status
+	2 - Use explicit credit
+	*/
+	{"ptxmode",	IOV_PROPTXSTATUS_MODE,	0,	0, IOVT_UINT32,	0 },
+	{"proptx_opt", IOV_PROPTXSTATUS_OPT,	0,	0, IOVT_UINT32,	0 },
+#ifdef QMONITOR
+	{"qtime_thres",	IOV_QMON_TIME_THRES,	0,	0, IOVT_UINT32,	0 },
+	{"qtime_percent", IOV_QMON_TIME_PERCENT, 0,	0, IOVT_UINT32,	0 },
+#endif /* QMONITOR */
+	{"pmodule_ignore", IOV_PROPTXSTATUS_MODULE_IGNORE, 0, 0, IOVT_BOOL, 0 },
+	{"pcredit_ignore", IOV_PROPTXSTATUS_CREDIT_IGNORE, 0, 0, IOVT_BOOL, 0 },
+	{"ptxstatus_ignore", IOV_PROPTXSTATUS_TXSTATUS_IGNORE, 0, 0,  IOVT_BOOL, 0 },
+	{"rxpkt_chk", IOV_PROPTXSTATUS_RXPKT_CHK, 0, 0, IOVT_BOOL, 0 },
+#endif /* PROP_TXSTATUS */
+	{"bustype", IOV_BUS_TYPE, 0, 0, IOVT_UINT32, 0},
+	{"changemtu", IOV_CHANGEMTU, 0, 0, IOVT_UINT32, 0 },
+	{"host_reorder_flows", IOV_HOSTREORDER_FLOWS, 0, 0, IOVT_BUFFER,
+	(WLHOST_REORDERDATA_MAXFLOWS + 1) },
+#ifdef DHDTCPACK_SUPPRESS
+	{"tcpack_suppress",	IOV_TCPACK_SUPPRESS,	0,	0, IOVT_UINT8,	0 },
+#endif /* DHDTCPACK_SUPPRESS */
+#ifdef DHD_WMF
+	{"wmf_bss_enable", IOV_WMF_BSS_ENAB,	0,	0, IOVT_BOOL,	0 },
+	{"wmf_ucast_igmp", IOV_WMF_UCAST_IGMP,	0,	0, IOVT_BOOL,	0 },
+	{"wmf_mcast_data_sendup", IOV_WMF_MCAST_DATA_SENDUP,	0,	0, IOVT_BOOL,	0 },
+#ifdef WL_IGMP_UCQUERY
+	{"wmf_ucast_igmp_query", IOV_WMF_UCAST_IGMP_QUERY, (0), 0, IOVT_BOOL, 0 },
+#endif /* WL_IGMP_UCQUERY */
+#ifdef DHD_UCAST_UPNP
+	{"wmf_ucast_upnp", IOV_WMF_UCAST_UPNP, (0), 0, IOVT_BOOL, 0 },
+#endif /* DHD_UCAST_UPNP */
+	{"wmf_psta_disable", IOV_WMF_PSTA_DISABLE, (0), 0, IOVT_BOOL, 0 },
+#endif /* DHD_WMF */
+#if defined(BCM_ROUTER_DHD)
+	{"trf_mgmt_filters_add", IOV_TRAFFIC_MGMT_DWM, (0), 0, IOVT_BUFFER, 0},
+#endif /* BCM_ROUTER_DHD */
+#ifdef DHD_L2_FILTER
+	{"dhcp_unicast", IOV_DHCP_UNICAST, (0), 0, IOVT_BOOL, 0 },
+#endif /* DHD_L2_FILTER */
+	{"ap_isolate", IOV_AP_ISOLATE, (0), 0, IOVT_BOOL, 0},
+#ifdef DHD_L2_FILTER
+	{"block_ping", IOV_BLOCK_PING, (0), 0, IOVT_BOOL, 0},
+	{"proxy_arp", IOV_PROXY_ARP, (0), 0, IOVT_BOOL, 0},
+	{"grat_arp", IOV_GRAT_ARP, (0), 0, IOVT_BOOL, 0},
+	{"block_tdls", IOV_BLOCK_TDLS, (0), IOVT_BOOL, 0},
+#endif /* DHD_L2_FILTER */
+	{"dhd_ie", IOV_DHD_IE, (0), 0, IOVT_BUFFER, 0},
+#ifdef DHD_PSTA
+	/* PSTA/PSR Mode configuration. 0: DIABLED 1: PSTA 2: PSR */
+	{"psta", IOV_PSTA, 0, 0, IOVT_UINT32, 0},
+#endif /* DHD PSTA */
+#ifdef DHD_WET
+	/* WET Mode configuration. 0: DIABLED 1: WET */
+	{"wet", IOV_WET, 0, 0, IOVT_UINT32, 0},
+	{"wet_host_ipv4", IOV_WET_HOST_IPV4, 0, 0, IOVT_UINT32, 0},
+	{"wet_host_mac", IOV_WET_HOST_MAC, 0, 0, IOVT_BUFFER, 0},
+#endif /* DHD WET */
+	{"op_mode",	IOV_CFG80211_OPMODE,	0,	0, IOVT_UINT32,	0 },
+	{"assert_type", IOV_ASSERT_TYPE, (0), 0, IOVT_UINT32, 0},
+#if defined(NDIS)
+	{ "wowl_wakeind", IOV_WAKEIND, 0, 0, IOVT_UINT32, 0 },
+#endif /* NDIS */
+#if !defined(NDIS) && !defined(BCM_ROUTER_DHD)
+	{"lmtest", IOV_LMTEST,	0,	0, IOVT_UINT32,	0 },
+#endif
+#ifdef DHD_MCAST_REGEN
+	{"mcast_regen_bss_enable", IOV_MCAST_REGEN_BSS_ENABLE, 0, 0, IOVT_BOOL, 0},
+#endif
+#if defined(DHD_MACDBG) && defined(BCMDBG)
+	{"pd11regs", IOV_MACDBG_PD11REGS, 0, 0, IOVT_BUFFER, 0},
+	{"mreglist", IOV_MACDBG_REGLIST, 0, 0, IOVT_BUFFER, 0},
+	{"psvmpmems", IOV_MACDBG_PSVMPMEMS, 0, 0, IOVT_BUFFER, 0},
+#endif /* defined(DHD_MACDBG) && defined(BCMDBG) */
+#ifdef SHOW_LOGTRACE
+	{"dump_trace_buf", IOV_DUMP_TRACE_LOG,	0, 0, IOVT_BUFFER,	sizeof(trace_buf_info_t) },
+#endif /* SHOW_LOGTRACE */
+#ifdef REPORT_FATAL_TIMEOUTS
+	{"scan_timeout", IOV_SCAN_TO, 0, 0, IOVT_UINT32, 0 },
+	{"join_timeout", IOV_JOIN_TO, 0, 0, IOVT_UINT32, 0 },
+	{"cmd_timeout", IOV_CMD_TO, 0, 0, IOVT_UINT32, 0 },
+	{"oqs_timeout", IOV_OQS_TO, 0, 0, IOVT_UINT32, 0 },
+#endif /* REPORT_FATAL_TIMEOUTS */
+	{"trap_type", IOV_DONGLE_TRAP_TYPE, 0, 0, IOVT_UINT32, 0 },
+	{"trap_info", IOV_DONGLE_TRAP_INFO, 0, 0, IOVT_BUFFER, sizeof(trap_t) },
+#ifdef DHD_DEBUG
+	{"bpaddr", IOV_BPADDR,	0, 0, IOVT_BUFFER,	sizeof(sdreg_t) },
+#endif /* DHD_DEBUG */
+	{"dump_dongle", IOV_DUMP_DONGLE, 0, 0, IOVT_BUFFER,
+	MAX(sizeof(dump_dongle_in_t), sizeof(dump_dongle_out_t)) },
+#if defined(DHD_LOG_DUMP)
+	{"log_dump", IOV_LOG_DUMP,	0, 0, IOVT_UINT8, 0},
+#endif /* DHD_LOG_DUMP */
+	{"tput_test", IOV_TPUT_TEST, 0, 0, IOVT_BUFFER, sizeof(tput_test_t)},
+	{"debug_buf_dest_stat", IOV_DEBUG_BUF_DEST_STAT, 0, 0, IOVT_UINT32, 0 },
+#if defined(DHD_SSSR_DUMP)
+	{"fis_trigger", IOV_FIS_TRIGGER, 0, 0, IOVT_UINT32, 0},
+#endif
+#ifdef DHD_DEBUG
+	{"induce_error", IOV_INDUCE_ERROR, (0), 0, IOVT_UINT16, 0 },
+#endif /* DHD_DEBUG */
+#ifdef WL_IFACE_MGMT_CONF
+#ifdef WL_CFG80211
+#ifdef WL_NANP2P
+	{"conc_disc", IOV_CONC_DISC, (0), 0, IOVT_UINT16, 0 },
+#endif /* WL_NANP2P */
+#ifdef WL_IFACE_MGMT
+	{"if_policy", IOV_IFACE_POLICY, (0), 0, IOVT_BUFFER, sizeof(iface_mgmt_data_t)},
+#endif /* WL_IFACE_MGMT */
+#endif /* WL_CFG80211 */
+#endif /* WL_IFACE_MGMT_CONF */
+#ifdef RTT_GEOFENCE_CONT
+#if defined(RTT_SUPPORT) && defined(WL_NAN)
+	{"rtt_geofence_type_ovrd", IOV_RTT_GEOFENCE_TYPE_OVRD, (0), 0, IOVT_BOOL, 0},
+#endif /* RTT_SUPPORT && WL_NAN */
+#endif /* RTT_GEOFENCE_CONT */
+	{"fw_verbose", IOV_FW_VBS, 0, 0, IOVT_UINT32, 0},
+#ifdef DHD_TX_PROFILE
+	{"tx_profile_tag", IOV_TX_PROFILE_TAG, 0, 0, IOVT_BUFFER,
+	sizeof(dhd_tx_profile_protocol_t)},
+	{"tx_profile_enable",	IOV_TX_PROFILE_ENABLE,	0,	0,	IOVT_BOOL,	0},
+	{"tx_profile_dump",	IOV_TX_PROFILE_DUMP,	0,	0,	IOVT_UINT32,	0},
+#endif /* defined(DHD_TX_PROFILE) */
+	{"check_trap_rot", IOV_CHECK_TRAP_ROT, (0), 0, IOVT_BOOL, 0},
+#ifdef DHD_LOGLEVEL
+	{"loglevel", IOV_LOGLEVEL, (0), 0, IOVT_BUFFER, sizeof(dhd_loglevel_data_t)},
+#endif /* DHD_LOGLEVEL */
+	{"counters",	IOV_COUNTERS, 0, 0, IOVT_BUFFER, DHD_IOCTL_MAXLEN_32K},
+#ifdef BCMPCIE
+	{"dump_flowrings", IOV_DUMP_FLOWRINGS, 0, 0, IOVT_BUFFER, DHD_IOCTL_MAXLEN_32K},
+#endif
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+	{"sarmode", IOV_SAR_MODE, (0), 0, IOVT_UINT32, 0},
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+#ifdef CSI_SUPPORT
+	{"csi_config", IOV_CSI_CONFIG, (0), 0, IOVT_BUFFER, sizeof(uint32)},
+#endif /* CSI_SUPPORT */
+	/* --- add new iovars *ABOVE* this line --- */
+#ifdef WLEASYMESH
+	{"1905_al_ucast", IOV_1905_AL_UCAST, 0, 0, IOVT_BUFFER, ETHER_ADDR_LEN},
+	{"1905_al_mcast", IOV_1905_AL_MCAST, 0, 0, IOVT_BUFFER, ETHER_ADDR_LEN},
+#endif /* WLEASYMESH */
+	{NULL, 0, 0, 0, 0, 0 }
+};
+
+#define DHD_IOVAR_BUF_SIZE	128
+
+#if defined(LINUX) || defined(linux)
+fw_download_status_t
+dhd_fw_download_status(dhd_pub_t * dhd_pub)
+{
+	return dhd_pub->fw_download_status;
+}
+#endif
+
+bool
+dhd_query_bus_erros(dhd_pub_t *dhdp)
+{
+	bool ret = FALSE;
+
+	if (dhdp->dongle_trap_occured) {
+		DHD_ERROR_RLMT(("%s: FW TRAP has occurred, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+#ifdef OEM_ANDROID
+		dhdp->hang_reason = HANG_REASON_DONGLE_TRAP;
+		dhd_os_send_hang_message(dhdp);
+#endif /* OEM_ANDROID */
+	}
+
+	if (dhdp->iovar_timeout_occured) {
+		DHD_ERROR_RLMT(("%s: Resumed on timeout for previous IOVAR, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+#ifdef PCIE_FULL_DONGLE
+	if (dhdp->d3ack_timeout_occured) {
+		DHD_ERROR_RLMT(("%s: Resumed on timeout for previous D3ACK, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+	if (dhdp->livelock_occured) {
+		DHD_ERROR_RLMT(("%s: LIVELOCK occurred for previous msg, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+	if (dhdp->pktid_audit_failed) {
+		DHD_ERROR_RLMT(("%s: pktid_audit_failed, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+	if (dhdp->pktid_invalid_occured) {
+		DHD_ERROR_RLMT(("%s: invalid pktid, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+#endif /* PCIE_FULL_DONGLE */
+
+	if (dhdp->iface_op_failed) {
+		DHD_ERROR_RLMT(("%s: iface_op_failed, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+	if (dhdp->scan_timeout_occurred) {
+		DHD_ERROR_RLMT(("%s: scan_timeout_occurred, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+	if (dhdp->scan_busy_occurred) {
+		DHD_ERROR_RLMT(("%s: scan_busy_occurred, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+	if (dhdp->p2p_disc_busy_occurred) {
+		DHD_ERROR_RLMT(("%s: p2p_disc_busy_occurred, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+	if (dhdp->axi_error) {
+		DHD_ERROR_RLMT(("%s: AXI error occurred, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#ifdef BCMPCIE
+	if (dhd_bus_get_linkdown(dhdp)) {
+		/* pcie link down, so do chip big hammer */
+		DHD_ERROR_RLMT(("%s: set do_chip_bighammer\n", __FUNCTION__));
+		dhdp->do_chip_bighammer = TRUE;
+		DHD_ERROR_RLMT(("%s : PCIE Link down occurred, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+	if (dhd_bus_get_cto(dhdp)) {
+		DHD_ERROR_RLMT(("%s : CTO Recovery reported, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+
+	if (dhd_bus_get_read_shm(dhdp)) {
+		DHD_ERROR_RLMT(("%s : read shm check failed, cannot proceed\n",
+			__FUNCTION__));
+		ret = TRUE;
+	}
+#endif /* BCMPCIE */
+	return ret;
+}
+
+void
+dhd_clear_bus_errors(dhd_pub_t *dhdp)
+{
+	if (!dhdp)
+		return;
+
+	dhdp->dongle_reset = FALSE;
+	dhdp->dongle_trap_occured = FALSE;
+	dhdp->dsack_hc_due_to_isr_delay = FALSE;
+	dhdp->dsack_hc_due_to_dpc_delay = FALSE;
+	dhdp->iovar_timeout_occured = FALSE;
+#ifdef PCIE_FULL_DONGLE
+	dhdp->d3ack_timeout_occured = FALSE;
+	dhdp->livelock_occured = FALSE;
+	dhdp->pktid_audit_failed = FALSE;
+	dhdp->pktid_invalid_occured = FALSE;
+#endif
+	dhdp->iface_op_failed = FALSE;
+	dhdp->scan_timeout_occurred = FALSE;
+	dhdp->scan_busy_occurred = FALSE;
+	dhdp->p2p_disc_busy_occurred = FALSE;
+}
+
+#ifdef DHD_SSSR_DUMP
+
+/* This can be overwritten by module parameter defined in dhd_linux.c */
+#ifdef GDB_PROXY
+/* GDB Proxy can't connect to crashed firmware after SSSR dump is generated.
+ * SSSR dump generation disabled for GDB Proxy enabled firmware by default.
+ * Still it can be explicitly enabled by echo 1 > /sys/wifi/sssr_enab or by
+ * sssr_enab=1 in insmod command line
+ */
+uint sssr_enab = FALSE;
+#else /* GDB_PROXY */
+uint sssr_enab = TRUE;
+#endif /* else GDB_PROXY */
+
+#ifdef DHD_FIS_DUMP
+uint fis_enab = TRUE;
+#else
+uint fis_enab = FALSE;
+#endif /* DHD_FIS_DUMP */
+
+int
+dhd_sssr_mempool_init(dhd_pub_t *dhd)
+{
+#ifdef CONFIG_BCMDHD_PCIE
+	dhd->sssr_mempool = (uint8 *) VMALLOCZ(dhd->osh, DHD_SSSR_MEMPOOL_SIZE);
+#else
+	dhd->sssr_mempool = (uint8 *) MALLOCZ(dhd->osh, DHD_SSSR_MEMPOOL_SIZE);
+#endif /* CONFIG_BCMDHD_PCIE */
+	if (dhd->sssr_mempool == NULL) {
+		DHD_ERROR(("%s: MALLOC of sssr_mempool failed\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+
+void
+dhd_sssr_mempool_deinit(dhd_pub_t *dhd)
+{
+	if (dhd->sssr_mempool) {
+#ifdef CONFIG_BCMDHD_PCIE
+		VMFREE(dhd->osh, dhd->sssr_mempool, DHD_SSSR_MEMPOOL_SIZE);
+#else
+		MFREE(dhd->osh, dhd->sssr_mempool, DHD_SSSR_MEMPOOL_SIZE);
+#endif /* CONFIG_BCMDHD_PCIE */
+		dhd->sssr_mempool = NULL;
+	}
+}
+
+int
+dhd_sssr_reg_info_init(dhd_pub_t *dhd)
+{
+	dhd->sssr_reg_info = (sssr_reg_info_cmn_t *) MALLOCZ(dhd->osh, sizeof(sssr_reg_info_cmn_t));
+	if (dhd->sssr_reg_info == NULL) {
+		DHD_ERROR(("%s: MALLOC of sssr_reg_info failed\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+
+void
+dhd_sssr_reg_info_deinit(dhd_pub_t *dhd)
+{
+	if (dhd->sssr_reg_info) {
+		MFREE(dhd->osh, dhd->sssr_reg_info, sizeof(sssr_reg_info_cmn_t));
+		dhd->sssr_reg_info = NULL;
+	}
+}
+
+#ifdef DHD_PCIE_REG_ACCESS
+static void
+dhd_dump_sssr_reg_info_v2(dhd_pub_t *dhd)
+{
+	sssr_reg_info_cmn_t *sssr_reg_info_cmn = dhd->sssr_reg_info;
+	sssr_reg_info_v2_t *sssr_reg_info = (sssr_reg_info_v2_t *)&sssr_reg_info_cmn->rev2;
+	int i, j;
+	uint8 num_d11cores = dhd_d11_slices_num_get(dhd);
+	DHD_ERROR(("pmu_regs\n"));
+	DHD_ERROR(("pmuintmask0=0x%x pmuintmask1=0x%x resreqtimer=0x%x "
+		"macresreqtimer=0x%x macresreqtimer1=0x%x\n",
+		sssr_reg_info->pmu_regs.base_regs.pmuintmask0,
+		sssr_reg_info->pmu_regs.base_regs.pmuintmask1,
+		sssr_reg_info->pmu_regs.base_regs.resreqtimer,
+		sssr_reg_info->pmu_regs.base_regs.macresreqtimer,
+		sssr_reg_info->pmu_regs.base_regs.macresreqtimer1));
+	DHD_ERROR(("chipcommon_regs\n"));
+	DHD_ERROR(("intmask=0x%x powerctrl=0x%x clockcontrolstatus=0x%x powerctrl_mask=0x%x\n",
+		sssr_reg_info->chipcommon_regs.base_regs.intmask,
+		sssr_reg_info->chipcommon_regs.base_regs.powerctrl,
+		sssr_reg_info->chipcommon_regs.base_regs.clockcontrolstatus,
+		sssr_reg_info->chipcommon_regs.base_regs.powerctrl_mask));
+	DHD_ERROR(("arm_regs\n"));
+	DHD_ERROR(("clockcontrolstatus=0x%x clockcontrolstatus_val=0x%x"
+		" resetctrl=0x%x extrsrcreq=0x%x\n",
+		sssr_reg_info->arm_regs.base_regs.clockcontrolstatus,
+		sssr_reg_info->arm_regs.base_regs.clockcontrolstatus_val,
+		sssr_reg_info->arm_regs.wrapper_regs.resetctrl,
+		sssr_reg_info->arm_regs.wrapper_regs.extrsrcreq));
+	DHD_ERROR(("pcie_regs\n"));
+	DHD_ERROR(("ltrstate=0x%x clockcontrolstatus=0x%x "
+		"clockcontrolstatus_val=0x%x extrsrcreq=0x%x\n",
+		sssr_reg_info->pcie_regs.base_regs.ltrstate,
+		sssr_reg_info->pcie_regs.base_regs.clockcontrolstatus,
+		sssr_reg_info->pcie_regs.base_regs.clockcontrolstatus_val,
+		sssr_reg_info->pcie_regs.wrapper_regs.extrsrcreq));
+
+	for (i = 0; i < num_d11cores; i++) {
+		DHD_ERROR(("mac_regs core[%d]\n", i));
+		DHD_ERROR(("xmtaddress=0x%x xmtdata=0x%x clockcontrolstatus=0x%x "
+			"clockcontrolstatus_val=0x%x\n",
+			sssr_reg_info->mac_regs[i].base_regs.xmtaddress,
+			sssr_reg_info->mac_regs[i].base_regs.xmtdata,
+			sssr_reg_info->mac_regs[i].base_regs.clockcontrolstatus,
+			sssr_reg_info->mac_regs[i].base_regs.clockcontrolstatus_val));
+		DHD_ERROR(("resetctrl=0x%x extrsrcreq=0x%x ioctrl=0x%x\n",
+			sssr_reg_info->mac_regs[i].wrapper_regs.resetctrl,
+			sssr_reg_info->mac_regs[i].wrapper_regs.extrsrcreq,
+			sssr_reg_info->mac_regs[i].wrapper_regs.ioctrl));
+		for (j = 0; j < SSSR_D11_RESET_SEQ_STEPS; j++) {
+			DHD_ERROR(("ioctrl_resetseq_val[%d] 0x%x\n", j,
+				sssr_reg_info->mac_regs[i].wrapper_regs.ioctrl_resetseq_val[j]));
+		}
+		DHD_ERROR(("sr_size=0x%x\n", sssr_reg_info->mac_regs[i].sr_size));
+	}
+	DHD_ERROR(("dig_regs\n"));
+	DHD_ERROR(("dig_sr_addr=0x%x dig_sr_size=0x%x\n",
+		sssr_reg_info->dig_mem_info.dig_sr_addr,
+		sssr_reg_info->dig_mem_info.dig_sr_size));
+}
+
+static void
+dhd_dump_sssr_reg_info_v3(dhd_pub_t *dhd)
+{
+	sssr_reg_info_cmn_t *sssr_reg_info_cmn = dhd->sssr_reg_info;
+	sssr_reg_info_v3_t *sssr_reg_info = (sssr_reg_info_v3_t *)&sssr_reg_info_cmn->rev3;
+	int i;
+
+	dhd_dump_sssr_reg_info_v2(dhd);
+
+	DHD_ERROR(("FIS Enab in fw : %d\n", sssr_reg_info->fis_enab));
+
+	DHD_ERROR(("HWA regs for reset \n"));
+	DHD_ERROR(("clkenable 0x%x, clkgatingenable 0x%x, clkext 0x%x, "
+		"clkctlstatus 0x%x, ioctrl 0x%x, resetctrl 0x%x\n",
+		sssr_reg_info->hwa_regs.base_regs.clkenable,
+		sssr_reg_info->hwa_regs.base_regs.clkgatingenable,
+		sssr_reg_info->hwa_regs.base_regs.clkext,
+		sssr_reg_info->hwa_regs.base_regs.clkctlstatus,
+		sssr_reg_info->hwa_regs.wrapper_regs.ioctrl,
+		sssr_reg_info->hwa_regs.wrapper_regs.resetctrl));
+	DHD_ERROR(("HWA regs value seq for reset \n"));
+	for (i = 0; i < SSSR_HWA_RESET_SEQ_STEPS; i++) {
+		DHD_ERROR(("hwa_resetseq_val[%d] 0x%x\n", i,
+			sssr_reg_info->hwa_regs.hwa_resetseq_val[i]));
+	}
+}
+
+static void
+dhd_dump_sssr_reg_info_v1(dhd_pub_t *dhd)
+{
+	sssr_reg_info_cmn_t *sssr_reg_info_cmn = dhd->sssr_reg_info;
+	sssr_reg_info_v1_t *sssr_reg_info = (sssr_reg_info_v1_t *)&sssr_reg_info_cmn->rev1;
+	int i, j;
+	uint8 num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	DHD_ERROR(("pmu_regs\n"));
+	DHD_ERROR(("pmuintmask0=0x%x pmuintmask1=0x%x resreqtimer=0x%x "
+		"macresreqtimer=0x%x macresreqtimer1=0x%x\n",
+		sssr_reg_info->pmu_regs.base_regs.pmuintmask0,
+		sssr_reg_info->pmu_regs.base_regs.pmuintmask1,
+		sssr_reg_info->pmu_regs.base_regs.resreqtimer,
+		sssr_reg_info->pmu_regs.base_regs.macresreqtimer,
+		sssr_reg_info->pmu_regs.base_regs.macresreqtimer1));
+	DHD_ERROR(("chipcommon_regs\n"));
+	DHD_ERROR(("intmask=0x%x powerctrl=0x%x clockcontrolstatus=0x%x powerctrl_mask=0x%x\n",
+		sssr_reg_info->chipcommon_regs.base_regs.intmask,
+		sssr_reg_info->chipcommon_regs.base_regs.powerctrl,
+		sssr_reg_info->chipcommon_regs.base_regs.clockcontrolstatus,
+		sssr_reg_info->chipcommon_regs.base_regs.powerctrl_mask));
+	DHD_ERROR(("arm_regs\n"));
+	DHD_ERROR(("clockcontrolstatus=0x%x clockcontrolstatus_val=0x%x"
+		" resetctrl=0x%x itopoobb=0x%x\n",
+		sssr_reg_info->arm_regs.base_regs.clockcontrolstatus,
+		sssr_reg_info->arm_regs.base_regs.clockcontrolstatus_val,
+		sssr_reg_info->arm_regs.wrapper_regs.resetctrl,
+		sssr_reg_info->arm_regs.wrapper_regs.itopoobb));
+	DHD_ERROR(("pcie_regs\n"));
+	DHD_ERROR(("ltrstate=0x%x clockcontrolstatus=0x%x "
+		"clockcontrolstatus_val=0x%x itopoobb=0x%x\n",
+		sssr_reg_info->pcie_regs.base_regs.ltrstate,
+		sssr_reg_info->pcie_regs.base_regs.clockcontrolstatus,
+		sssr_reg_info->pcie_regs.base_regs.clockcontrolstatus_val,
+		sssr_reg_info->pcie_regs.wrapper_regs.itopoobb));
+	DHD_ERROR(("vasip_regs\n"));
+	DHD_ERROR(("ioctrl=0x%x vasip_sr_addr=0x%x vasip_sr_size=0x%x\n",
+		sssr_reg_info->vasip_regs.wrapper_regs.ioctrl,
+		sssr_reg_info->vasip_regs.vasip_sr_addr,
+		sssr_reg_info->vasip_regs.vasip_sr_size));
+
+	for (i = 0; i < num_d11cores; i++) {
+		DHD_ERROR(("mac_regs core[%d]\n", i));
+		DHD_ERROR(("xmtaddress=0x%x xmtdata=0x%x clockcontrolstatus=0x%x "
+			"clockcontrolstatus_val=0x%x\n",
+			sssr_reg_info->mac_regs[i].base_regs.xmtaddress,
+			sssr_reg_info->mac_regs[i].base_regs.xmtdata,
+			sssr_reg_info->mac_regs[i].base_regs.clockcontrolstatus,
+			sssr_reg_info->mac_regs[i].base_regs.clockcontrolstatus_val));
+		DHD_ERROR(("resetctrl=0x%x itopoobb=0x%x ioctrl=0x%x\n",
+			sssr_reg_info->mac_regs[i].wrapper_regs.resetctrl,
+			sssr_reg_info->mac_regs[i].wrapper_regs.itopoobb,
+			sssr_reg_info->mac_regs[i].wrapper_regs.ioctrl));
+		for (j = 0; j < SSSR_D11_RESET_SEQ_STEPS; j++) {
+			DHD_ERROR(("ioctrl_resetseq_val[%d] 0x%x\n", j,
+				sssr_reg_info->mac_regs[i].wrapper_regs.ioctrl_resetseq_val[j]));
+		}
+		DHD_ERROR(("sr_size=0x%x\n", sssr_reg_info->mac_regs[i].sr_size));
+	}
+}
+
+#endif /* DHD_PCIE_REG_ACCESS */
+
+void
+dhd_dump_sssr_reg_info(dhd_pub_t *dhd)
+{
+#ifdef DHD_PCIE_REG_ACCESS
+	sssr_reg_info_cmn_t *sssr_reg_info_cmn = dhd->sssr_reg_info;
+	sssr_reg_info_v1_t *sssr_reg_info = (sssr_reg_info_v1_t *)&sssr_reg_info_cmn->rev1;
+
+	DHD_ERROR(("************** SSSR REG INFO start version:%d ****************\n",
+		sssr_reg_info->version));
+	switch (sssr_reg_info->version) {
+		case SSSR_REG_INFO_VER_3 :
+			dhd_dump_sssr_reg_info_v3(dhd);
+			break;
+		case SSSR_REG_INFO_VER_2 :
+			dhd_dump_sssr_reg_info_v2(dhd);
+			break;
+		default:
+			dhd_dump_sssr_reg_info_v1(dhd);
+			break;
+	}
+	DHD_ERROR(("************** SSSR REG INFO end ****************\n"));
+#endif /* DHD_PCIE_REG_ACCESS */
+}
+
+void dhdpcie_fill_sssr_reg_info(dhd_pub_t *dhd);
+
+int
+dhd_get_sssr_reg_info(dhd_pub_t *dhd)
+{
+	int ret;
+
+	if (dhd->force_sssr_init) {
+		dhdpcie_fill_sssr_reg_info(dhd);
+		dhd->force_sssr_init = FALSE;
+		goto done;
+	}
+
+	/* get sssr_reg_info from firmware */
+	ret = dhd_iovar(dhd, 0, "sssr_reg_info", NULL, 0,  (char *)dhd->sssr_reg_info,
+		sizeof(sssr_reg_info_cmn_t), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: sssr_reg_info failed (error=%d)\n",
+			__FUNCTION__, ret));
+		return BCME_ERROR;
+	}
+
+done:
+	dhd_dump_sssr_reg_info(dhd);
+	return BCME_OK;
+}
+
+uint32
+dhd_get_sssr_bufsize(dhd_pub_t *dhd)
+{
+	int i;
+	uint32 sssr_bufsize = 0;
+	uint8 num_d11cores;
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			for (i = 0; i < num_d11cores; i++) {
+				sssr_bufsize += dhd->sssr_reg_info->rev2.mac_regs[i].sr_size;
+			}
+			if ((dhd->sssr_reg_info->rev2.length >
+			 OFFSETOF(sssr_reg_info_v2_t, dig_mem_info)) &&
+			 dhd->sssr_reg_info->rev2.dig_mem_info.dig_sr_addr) {
+				sssr_bufsize += 0; /* TBD */
+			}
+			break;
+		case SSSR_REG_INFO_VER_1 :
+			for (i = 0; i < num_d11cores; i++) {
+				sssr_bufsize += dhd->sssr_reg_info->rev1.mac_regs[i].sr_size;
+			}
+			if (dhd->sssr_reg_info->rev1.vasip_regs.vasip_sr_size) {
+				sssr_bufsize += dhd->sssr_reg_info->rev1.vasip_regs.vasip_sr_size;
+			} else if ((dhd->sssr_reg_info->rev1.length > OFFSETOF(sssr_reg_info_v1_t,
+				dig_mem_info)) && dhd->sssr_reg_info->rev1.
+				dig_mem_info.dig_sr_addr) {
+				sssr_bufsize += dhd->sssr_reg_info->rev1.dig_mem_info.dig_sr_size;
+			}
+			break;
+		case SSSR_REG_INFO_VER_0 :
+			for (i = 0; i < num_d11cores; i++) {
+				sssr_bufsize += dhd->sssr_reg_info->rev0.mac_regs[i].sr_size;
+			}
+			if (dhd->sssr_reg_info->rev0.vasip_regs.vasip_sr_size) {
+				sssr_bufsize += dhd->sssr_reg_info->rev0.vasip_regs.vasip_sr_size;
+			}
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	/* Double the size as different dumps will be saved before and after SR */
+	sssr_bufsize = 2 * sssr_bufsize;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+	return sssr_bufsize;
+}
+
+int
+dhd_sssr_dump_init(dhd_pub_t *dhd)
+{
+	int i;
+	uint32 sssr_bufsize;
+	uint32 mempool_used = 0;
+	uint8 num_d11cores = 0;
+	bool alloc_sssr = FALSE;
+	uint32 sr_size = 0;
+
+	dhd->sssr_inited = FALSE;
+	if (!sssr_enab) {
+		DHD_ERROR(("%s: sssr dump not inited as instructed by mod param\n", __FUNCTION__));
+		return BCME_OK;
+	}
+
+	/* check if sssr mempool is allocated */
+	if (dhd->sssr_mempool == NULL) {
+		DHD_ERROR(("%s: sssr_mempool is not allocated\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* check if sssr mempool is allocated */
+	if (dhd->sssr_reg_info == NULL) {
+		DHD_ERROR(("%s: sssr_reg_info is not allocated\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Get SSSR reg info */
+	if (dhd_get_sssr_reg_info(dhd) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_get_sssr_reg_info failed\n", __FUNCTION__));
+		DHD_CONS_ONLY(("DEBUG_SSSr: %s: dhd_get_sssr_reg_info failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+	/* Validate structure version and length */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			if (dhd->sssr_reg_info->rev3.length != sizeof(sssr_reg_info_v3_t)) {
+				DHD_ERROR(("%s: dhd->sssr_reg_info->rev2.length (%d : %d)"
+					 "mismatch on rev2\n", __FUNCTION__,
+					 (int)dhd->sssr_reg_info->rev3.length,
+					 (int)sizeof(sssr_reg_info_v3_t)));
+				return BCME_ERROR;
+			}
+			break;
+		case SSSR_REG_INFO_VER_2 :
+			if (dhd->sssr_reg_info->rev2.length != sizeof(sssr_reg_info_v2_t)) {
+				DHD_ERROR(("%s: dhd->sssr_reg_info->rev2.length (%d : %d)"
+					 "mismatch on rev2\n", __FUNCTION__,
+					 (int)dhd->sssr_reg_info->rev2.length,
+					 (int)sizeof(sssr_reg_info_v2_t)));
+				return BCME_ERROR;
+			}
+			break;
+		case SSSR_REG_INFO_VER_1 :
+			if (dhd->sssr_reg_info->rev1.length != sizeof(sssr_reg_info_v1_t)) {
+				DHD_ERROR(("%s: dhd->sssr_reg_info->rev1.length (%d : %d)"
+					 "mismatch on rev1\n", __FUNCTION__,
+					 (int)dhd->sssr_reg_info->rev1.length,
+					 (int)sizeof(sssr_reg_info_v1_t)));
+				return BCME_ERROR;
+			}
+			break;
+		case SSSR_REG_INFO_VER_0 :
+			if (dhd->sssr_reg_info->rev0.length != sizeof(sssr_reg_info_v0_t)) {
+				DHD_ERROR(("%s: dhd->sssr_reg_info->rev0.length (%d : %d)"
+					 "mismatch on rev0\n", __FUNCTION__,
+					 (int)dhd->sssr_reg_info->rev0.length,
+					 (int)sizeof(sssr_reg_info_v0_t)));
+				return BCME_ERROR;
+			}
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* validate fifo size */
+	sssr_bufsize = dhd_get_sssr_bufsize(dhd);
+	if (sssr_bufsize > DHD_SSSR_MEMPOOL_SIZE) {
+		DHD_ERROR(("%s: sssr_bufsize(%d) is greater than sssr_mempool(%d)\n",
+			__FUNCTION__, (int)sssr_bufsize, DHD_SSSR_MEMPOOL_SIZE));
+		return BCME_ERROR;
+	}
+
+	/* init all pointers to NULL */
+	for (i = 0; i < num_d11cores; i++) {
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		dhd->sssr_d11_before[i] = NULL;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+		dhd->sssr_d11_after[i] = NULL;
+	}
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	dhd->sssr_dig_buf_before = NULL;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	dhd->sssr_dig_buf_after = NULL;
+
+	/* Allocate memory */
+	for (i = 0; i < num_d11cores; i++) {
+		alloc_sssr = FALSE;
+		sr_size = 0;
+
+		switch (dhd->sssr_reg_info->rev2.version) {
+			case SSSR_REG_INFO_VER_3 :
+				/* intentional fall through */
+			case SSSR_REG_INFO_VER_2 :
+				if (dhd->sssr_reg_info->rev2.mac_regs[i].sr_size) {
+					alloc_sssr = TRUE;
+					sr_size = dhd->sssr_reg_info->rev2.mac_regs[i].sr_size;
+				}
+				break;
+			case SSSR_REG_INFO_VER_1 :
+				if (dhd->sssr_reg_info->rev1.mac_regs[i].sr_size) {
+					alloc_sssr = TRUE;
+					sr_size = dhd->sssr_reg_info->rev1.mac_regs[i].sr_size;
+				}
+				break;
+			case SSSR_REG_INFO_VER_0 :
+				if (dhd->sssr_reg_info->rev0.mac_regs[i].sr_size) {
+					alloc_sssr = TRUE;
+					sr_size = dhd->sssr_reg_info->rev0.mac_regs[i].sr_size;
+				}
+				break;
+			default :
+				DHD_ERROR(("invalid sssr_reg_ver"));
+				return BCME_UNSUPPORTED;
+		}
+
+		if (alloc_sssr) {
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+			dhd->sssr_d11_before[i] = (uint32 *)(dhd->sssr_mempool + mempool_used);
+			mempool_used += sr_size;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+			dhd->sssr_d11_after[i] = (uint32 *)(dhd->sssr_mempool + mempool_used);
+			mempool_used += sr_size;
+		}
+	}
+
+	/* Allocate dump memory for VASIP (version 0 or 1) or digital core (version 0, 1, or 2) */
+	alloc_sssr = FALSE;
+	sr_size = 0;
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			if ((dhd->sssr_reg_info->rev2.length >
+			 OFFSETOF(sssr_reg_info_v2_t, dig_mem_info)) &&
+			 dhd->sssr_reg_info->rev2.dig_mem_info.dig_sr_addr) {
+				alloc_sssr = TRUE;
+				sr_size = dhd->sssr_reg_info->rev2.dig_mem_info.dig_sr_size;
+			}
+			break;
+		case SSSR_REG_INFO_VER_1 :
+			if (dhd->sssr_reg_info->rev1.vasip_regs.vasip_sr_size) {
+				alloc_sssr = TRUE;
+				sr_size = dhd->sssr_reg_info->rev1.vasip_regs.vasip_sr_size;
+			} else if ((dhd->sssr_reg_info->rev1.length > OFFSETOF(sssr_reg_info_v1_t,
+				dig_mem_info)) && dhd->sssr_reg_info->rev1.
+				dig_mem_info.dig_sr_addr) {
+				alloc_sssr = TRUE;
+				sr_size = dhd->sssr_reg_info->rev1.dig_mem_info.dig_sr_size;
+			}
+			break;
+		case SSSR_REG_INFO_VER_0 :
+			if (dhd->sssr_reg_info->rev0.vasip_regs.vasip_sr_size) {
+				alloc_sssr = TRUE;
+				sr_size = dhd->sssr_reg_info->rev0.vasip_regs.vasip_sr_size;
+			}
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	if (alloc_sssr) {
+		dhd->sssr_dig_buf_after = (uint32 *)(dhd->sssr_mempool + mempool_used);
+		mempool_used += sr_size;
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		/* DIG dump before suspend is not applicable. */
+		dhd->sssr_dig_buf_before = NULL;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	}
+
+	dhd->sssr_inited = TRUE;
+
+	return BCME_OK;
+
+}
+
+void
+dhd_sssr_dump_deinit(dhd_pub_t *dhd)
+{
+	int i;
+
+	dhd->sssr_inited = FALSE;
+	/* init all pointers to NULL */
+	for (i = 0; i < MAX_NUM_D11_CORES_WITH_SCAN; i++) {
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		dhd->sssr_d11_before[i] = NULL;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+		dhd->sssr_d11_after[i] = NULL;
+	}
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	dhd->sssr_dig_buf_before = NULL;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	dhd->sssr_dig_buf_after = NULL;
+
+	return;
+}
+
+void
+dhd_sssr_print_filepath(dhd_pub_t *dhd, char *path)
+{
+	bool print_info = FALSE;
+	int dump_mode;
+
+	if (!dhd || !path) {
+		DHD_ERROR(("%s: dhd or memdump_path is NULL\n",
+			__FUNCTION__));
+		return;
+	}
+
+	if (!dhd->sssr_dump_collected) {
+		/* SSSR dump is not collected */
+		return;
+	}
+
+	dump_mode = dhd->sssr_dump_mode;
+
+	if (bcmstrstr(path, "core_0_before")) {
+		if (dhd->sssr_d11_outofreset[0] &&
+			dump_mode == SSSR_DUMP_MODE_SSSR) {
+			print_info = TRUE;
+		}
+	} else if (bcmstrstr(path, "core_0_after")) {
+		if (dhd->sssr_d11_outofreset[0]) {
+			print_info = TRUE;
+		}
+	} else if (bcmstrstr(path, "core_1_before")) {
+		if (dhd->sssr_d11_outofreset[1] &&
+			dump_mode == SSSR_DUMP_MODE_SSSR) {
+			print_info = TRUE;
+		}
+	} else if (bcmstrstr(path, "core_1_after")) {
+		if (dhd->sssr_d11_outofreset[1]) {
+			print_info = TRUE;
+		}
+	} else if (bcmstrstr(path, "core_2_before")) {
+		if (dhd->sssr_d11_outofreset[2] &&
+			dump_mode == SSSR_DUMP_MODE_SSSR) {
+			print_info = TRUE;
+		}
+	} else if (bcmstrstr(path, "core_2_after")) {
+		if (dhd->sssr_d11_outofreset[2]) {
+			print_info = TRUE;
+		}
+	} else {
+		print_info = TRUE;
+	}
+
+	if (print_info) {
+		DHD_ERROR(("%s: file_path = %s%s\n", __FUNCTION__,
+			path, FILE_NAME_HAL_TAG));
+	}
+}
+#endif /* DHD_SSSR_DUMP */
+
+#ifdef DHD_COREDUMP
+int
+dhd_coredump_mempool_init(dhd_pub_t *dhd)
+{
+	dhd->coredump_mem = (uint8*) VMALLOCZ(dhd->osh, DHD_MEMDUMP_BUFFER_SIZE);
+	if (dhd->coredump_mem == NULL) {
+		DHD_ERROR(("%s: VMALLOCZ of coredump_mem failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dhd->coredump_len = DHD_MEMDUMP_BUFFER_SIZE;
+	return BCME_OK;
+}
+
+void
+dhd_coredump_mempool_deinit(dhd_pub_t *dhd)
+{
+	if (dhd->coredump_mem) {
+		VMFREE(dhd->osh, dhd->coredump_mem, DHD_MEMDUMP_BUFFER_SIZE);
+		dhd->coredump_mem = NULL;
+		dhd->coredump_len = 0;
+	}
+}
+
+#ifdef DHD_SSSR_DUMP
+dhd_coredump_t dhd_coredump_types[] = {
+	{DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_BEFORE, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_AFTER, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_BEFORE, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_AFTER, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_BEFORE, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_AFTER, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_DIG_BEFORE, 0, NULL},
+	{DHD_COREDUMP_TYPE_SSSRDUMP_DIG_AFTER, 0, NULL},
+	{DHD_COREDUMP_TYPE_SOCRAMDUMP, 0, NULL}
+};
+
+static int dhd_append_sssr_tlv(uint8 *buf_dst, int type_idx, int buf_remain)
+{
+	uint32 type_val, length_val;
+	uint32 *type, *length;
+	void *buf_src;
+	int total_size = 0, ret = 0;
+
+	/* DHD_COREDUMP_TYPE_SSSRDUMP_[CORE[0|1|2]|DIG]_[BEFORE|AFTER] */
+	type_val = dhd_coredump_types[type_idx].type;
+	length_val = dhd_coredump_types[type_idx].length;
+
+	if (length_val == 0) {
+		return 0;
+	}
+
+	type = (uint32*)buf_dst;
+	*type = type_val;
+	length = (uint32*)(buf_dst + sizeof(*type));
+	*length = length_val;
+
+	buf_dst += TLV_TYPE_LENGTH_SIZE;
+	total_size += TLV_TYPE_LENGTH_SIZE;
+
+	buf_src = dhd_coredump_types[type_idx].bufptr;
+	ret = memcpy_s(buf_dst, buf_remain, buf_src, *length);
+
+	if (ret) {
+		DHD_ERROR(("Failed to memcpy_s() for coredump.\n"));
+		return BCME_ERROR;
+	}
+
+	DHD_INFO(("%s: type: %u, length: %u\n",	__FUNCTION__, *type, *length));
+
+	total_size += *length;
+	return total_size;
+}
+
+/* reconstruct dump memory with socram and sssr dump as TLV format */
+int dhd_collect_coredump(dhd_pub_t *dhdp, dhd_dump_t *dump)
+{
+	uint8 *buf_ptr = dhdp->coredump_mem;
+	int buf_len = dhdp->coredump_len;
+	uint8 *socram_mem = dump->buf;
+	int socram_len = dump->bufsize;
+	int ret = 0, i = 0;
+	int total_size = 0;
+	uint32 *magic, *type, *length;
+	uint8 num_d11cores = 0, offset = 0;
+
+	/* SSSR dump */
+	if (!sssr_enab || !dhdp->collect_sssr) {
+		DHD_ERROR(("SSSR is not enabled or not collected yet.\n"));
+		return BCME_UNSUPPORTED;
+	}
+
+	/* SOCRAM dump start with magic code */
+	magic = (uint32*)buf_ptr;
+	*magic = DHD_COREDUMP_MAGIC;
+
+	/* DHD_COREDUMP_TYPE_SOCRAMDUMP */
+	type = (uint32*)(buf_ptr + sizeof(*magic));
+	*type = dhd_coredump_types[DHD_COREDUMP_TYPE_SOCRAMDUMP].type;
+	length = (uint32*)(buf_ptr + sizeof(*magic) + sizeof(*type));
+	*length = socram_len;
+
+	offset = sizeof(*magic) + sizeof(*type) + sizeof(*length);
+	ret = memcpy_s(buf_ptr + offset, buf_len - offset,
+		socram_mem, socram_len);
+
+	if (ret) {
+		DHD_ERROR(("Failed to memmove_s() for coredump.\n"));
+		return BCME_ERROR;
+	}
+	total_size = offset + socram_len;
+	buf_ptr += total_size;
+
+	DHD_ERROR(("%s: type: %u, length: %u\n", __FUNCTION__, *type, *length));
+
+	num_d11cores = dhd_d11_slices_num_get(dhdp);
+	for (i = 0; i < num_d11cores + 1; i++) {
+		int written_bytes = 0;
+		int type_idx = i * 2;
+		if ((i < num_d11cores) && (!dhdp->sssr_d11_outofreset[i])) {
+			continue;
+		}
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		/* DHD_COREDUMP_TYPE_SSSRDUMP_[CORE[0|1|2]|DIG]_BEFORE */
+		written_bytes = dhd_append_sssr_tlv(buf_ptr, type_idx,
+			buf_len - total_size);
+		if (written_bytes == BCME_ERROR) {
+			return BCME_ERROR;
+		}
+		buf_ptr += written_bytes;
+		total_size += written_bytes;
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+		/* DHD_COREDUMP_TYPE_SSSRDUMP_[CORE[0|1|2]|DIG]_AFTER */
+		written_bytes = dhd_append_sssr_tlv(buf_ptr, type_idx + 1,
+			buf_len - total_size);
+		if (written_bytes == BCME_ERROR) {
+			return BCME_ERROR;
+		}
+		buf_ptr += written_bytes;
+		total_size += written_bytes;
+	}
+
+	dump->buf = dhdp->coredump_mem;
+	dump->bufsize = total_size;
+
+	return BCME_OK;
+}
+#endif /* DHD_SSSR_DUMP */
+#endif /* DHD_COREDUMP */
+
+#ifdef DHD_SDTC_ETB_DUMP
+/*
+ * sdtc: system debug trace controller
+ * etb: embedded trace buf
+ */
+void
+dhd_sdtc_etb_init(dhd_pub_t *dhd)
+{
+	bcm_iov_buf_t *iov_req = NULL;
+	etb_addr_info_t *p_etb_addr_info = NULL;
+	bcm_iov_buf_t *iov_resp = NULL;
+	uint8 *buf = NULL;
+	int ret = 0;
+	uint16 iovlen = 0;
+	uint16 version = 0;
+
+	BCM_REFERENCE(p_etb_addr_info);
+	dhd->sdtc_etb_inited = FALSE;
+
+	iov_req = MALLOCZ(dhd->osh, WLC_IOCTL_SMLEN);
+	if (iov_req == NULL) {
+		DHD_ERROR(("%s: Failed to alloc buffer for iovar request\n", __FUNCTION__));
+		goto exit;
+	}
+
+	buf = MALLOCZ(dhd->osh, WLC_IOCTL_MAXLEN);
+	if (buf == NULL) {
+		DHD_ERROR(("%s: Failed to alloc buffer for iovar response\n", __FUNCTION__));
+		goto exit;
+	}
+
+	/* fill header */
+	iov_req->version = WL_SDTC_IOV_VERSION;
+	iov_req->id = WL_SDTC_CMD_ETB_INFO;
+	iov_req->len = sizeof(etb_addr_info_t);
+	iovlen = OFFSETOF(bcm_iov_buf_t, data) + iov_req->len;
+
+	ret = dhd_iovar(dhd, 0, "sdtc", (char *)iov_req, iovlen,
+		(char *)buf, WLC_IOCTL_MAXLEN, FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s failed to get sdtc etb_info %d\n", __FUNCTION__, ret));
+		goto exit;
+	}
+
+	version = dtoh16(*(uint16 *)buf);
+	/* Check for version */
+	if (version != WL_SDTC_IOV_VERSION) {
+		DHD_ERROR(("%s WL_SDTC_IOV_VERSION mis match\n", __FUNCTION__));
+		goto exit;
+	}
+	iov_resp = (bcm_iov_buf_t *)buf;
+	if (iov_resp->id == iov_req->id) {
+		p_etb_addr_info = (etb_addr_info_t*)iov_resp->data;
+		dhd->etb_addr_info.version = p_etb_addr_info->version;
+		dhd->etb_addr_info.len = p_etb_addr_info->len;
+		dhd->etb_addr_info.etbinfo_addr = p_etb_addr_info->etbinfo_addr;
+
+		DHD_ERROR(("%s etb_addr_info: ver:%d, len:%d, addr:0x%x\n", __FUNCTION__,
+			dhd->etb_addr_info.version, dhd->etb_addr_info.len,
+			dhd->etb_addr_info.etbinfo_addr));
+	} else {
+		DHD_ERROR(("%s Unknown CMD-ID (%d) as  response for request ID %d\n",
+			__FUNCTION__, iov_resp->id, iov_req->id));
+		goto exit;
+	}
+
+	/* since all the requirements for SDTC and ETB are met mark the capability as TRUE */
+	dhd->sdtc_etb_inited = TRUE;
+	DHD_ERROR(("%s sdtc_etb_inited: %d\n", __FUNCTION__, dhd->sdtc_etb_inited));
+exit:
+	if (iov_req) {
+		MFREE(dhd->osh, iov_req, WLC_IOCTL_SMLEN);
+	}
+	if (buf) {
+		MFREE(dhd->osh, buf, WLC_IOCTL_MAXLEN);
+	}
+	return;
+}
+
+void
+dhd_sdtc_etb_deinit(dhd_pub_t *dhd)
+{
+	dhd->sdtc_etb_inited = FALSE;
+	dhd->sdtc_etb_dump_len = 0;
+}
+
+int
+dhd_sdtc_etb_mempool_init(dhd_pub_t *dhd)
+{
+	dhd->sdtc_etb_dump_len = 0;
+	dhd->sdtc_etb_mempool = (uint8 *) MALLOCZ(dhd->osh, DHD_SDTC_ETB_MEMPOOL_SIZE);
+	if (dhd->sdtc_etb_mempool == NULL) {
+		DHD_ERROR(("%s: MALLOC of sdtc_etb_mempool failed\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+
+void
+dhd_sdtc_etb_mempool_deinit(dhd_pub_t *dhd)
+{
+	if (dhd->sdtc_etb_mempool) {
+		MFREE(dhd->osh, dhd->sdtc_etb_mempool, DHD_SDTC_ETB_MEMPOOL_SIZE);
+		dhd->sdtc_etb_mempool = NULL;
+	}
+}
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#ifdef DHD_FW_COREDUMP
+void* dhd_get_fwdump_buf(dhd_pub_t *dhd_pub, uint32 length)
+{
+	if (!dhd_pub->soc_ram) {
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+		dhd_pub->soc_ram = (uint8*)DHD_OS_PREALLOC(dhd_pub,
+			DHD_PREALLOC_MEMDUMP_RAM, length);
+#else
+		dhd_pub->soc_ram = (uint8*) MALLOC(dhd_pub->osh, length);
+
+		if ((dhd_pub->soc_ram == NULL) && CAN_SLEEP()) {
+			DHD_ERROR(("%s: Try to allocate virtual memory for fw crash snap shot.\n",
+				__FUNCTION__));
+			dhd_pub->soc_ram = (uint8*) VMALLOC(dhd_pub->osh, length);
+		}
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_USE_STATIC_MEMDUMP */
+	}
+
+	if (dhd_pub->soc_ram == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory for fw crash snap shot.\n",
+			__FUNCTION__));
+		dhd_pub->soc_ram_length = 0;
+	} else {
+		memset(dhd_pub->soc_ram, 0, length);
+		dhd_pub->soc_ram_length = length;
+	}
+
+	/* soc_ram free handled in dhd_{free,clear} */
+	return dhd_pub->soc_ram;
+}
+#endif /* DHD_FW_COREDUMP */
+
+/* to NDIS developer, the structure dhd_common is redundant,
+ * please do NOT merge it back from other branches !!!
+ */
+
+int
+dhd_common_socram_dump(dhd_pub_t *dhdp)
+{
+#ifndef BCMDBUS
+	return dhd_socram_dump(dhdp->bus);
+#else
+	return -1;
+#endif /* BCMDBUS */
+}
+
+static void
+dhd_dump_txrx_stats(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	/* TX Stats -- add any Tx counters in this section only */
+	bcm_bprintf(strbuf, "\nTX stats:\n=========\n");
+	bcm_bprintf(strbuf, "tx_packets %lu tx_bytes %lu tx_dropped %lu"
+		" tx_multicast %lu tx_errors %lu\n",
+		dhdp->tx_packets, dhdp->tx_bytes, dhdp->tx_dropped,
+		dhdp->tx_multicast, dhdp->tx_errors);
+	bcm_bprintf(strbuf, "tx_ctlpkts %lu tx_ctlerrs %lu\n",
+	            dhdp->tx_ctlpkts, dhdp->tx_ctlerrs);
+	bcm_bprintf(strbuf, "tx_realloc %lu tx_pktgetfail %lu tx_big_packets %lu\n",
+	            dhdp->tx_realloc, dhdp->tx_pktgetfail, dhdp->tx_big_packets);
+	/* ----------------------------------------------------- */
+
+	/* RX Stats -- add any Rx counters in this section only */
+	bcm_bprintf(strbuf, "\nRX stats:\n=========\n");
+	bcm_bprintf(strbuf, "rx_packets %lu rx_bytes %lu rx_forward"
+		" %lu rx_multicast %lu rx_errors %lu \n",
+		dhdp->rx_packets, dhdp->rx_bytes, dhdp->rx_forward,
+		dhdp->rx_multicast, dhdp->rx_errors);
+	bcm_bprintf(strbuf, "rx_ctlpkts %lu rx_ctlerrs %lu rx_dropped %lu\n",
+	            dhdp->rx_ctlpkts, dhdp->rx_ctlerrs, dhdp->rx_dropped);
+	bcm_bprintf(strbuf, "rx_readahead_cnt %lu rx_pktgetfail %lu rx_pktgetpool_fail %lu\n",
+	            dhdp->rx_readahead_cnt, dhdp->rx_pktgetfail, dhdp->rx_pktgetpool_fail);
+	bcm_bprintf(strbuf, "\n");
+	/* ----------------------------------------------------- */
+}
+
+#ifdef DHD_MEM_STATS
+static void
+dhd_dump_memstats(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	uint64 malloc_mem = 0;
+	uint64 total_txpath_mem = 0;
+	uint64 txpath_bkpq_len = 0;
+	uint64 txpath_bkpq_mem = 0;
+	uint64 total_dhd_mem = 0;
+
+	malloc_mem = MALLOCED(dhdp->osh);
+
+	UNUSED_PARAMETER(malloc_mem);
+	UNUSED_PARAMETER(total_txpath_mem);
+	UNUSED_PARAMETER(txpath_bkpq_len);
+	UNUSED_PARAMETER(txpath_bkpq_mem);
+
+#ifdef BCMPCIE
+	txpath_bkpq_len = dhd_active_tx_flowring_bkpq_len(dhdp);
+	/*
+	 * Instead of traversing the entire queue to find the skbs length,
+	 * considering MAX_MTU_SZ as lenth of each skb.
+	 */
+	txpath_bkpq_mem = (txpath_bkpq_len* MAX_MTU_SZ);
+	total_txpath_mem = dhdp->txpath_mem + txpath_bkpq_mem;
+
+	bcm_bprintf(strbuf, "\nDHD malloc memory_usage: %llubytes %lluKB\n",
+		malloc_mem, (malloc_mem / 1024));
+
+	bcm_bprintf(strbuf, "\nDHD tx-bkpq len: %llu memory_usage: %llubytes %lluKB\n",
+		txpath_bkpq_len, txpath_bkpq_mem, (txpath_bkpq_mem / 1024));
+	bcm_bprintf(strbuf, "DHD tx-path memory_usage: %llubytes %lluKB\n",
+		total_txpath_mem, (total_txpath_mem / 1024));
+
+	total_dhd_mem = malloc_mem + total_txpath_mem;
+#endif /* BCMPCIE */
+
+#if defined(DHD_LB_STATS)
+	total_dhd_mem += dhd_lb_mem_usage(dhdp, strbuf);
+#endif /* DHD_LB_STATS */
+	bcm_bprintf(strbuf, "\nDHD Total memory_usage: %llubytes %lluKB \n",
+		total_dhd_mem, (total_dhd_mem / 1024));
+}
+#endif /* DHD_MEM_STATS */
+
+int
+dhd_dump(dhd_pub_t *dhdp, char *buf, int buflen)
+{
+	struct bcmstrbuf b;
+	struct bcmstrbuf *strbuf = &b;
+
+	if (!dhdp || !dhdp->prot || !buf) {
+		return BCME_ERROR;
+	}
+
+	bcm_binit(strbuf, buf, buflen);
+
+	/* Base DHD info */
+	bcm_bprintf(strbuf, "%s\n", dhd_version);
+	bcm_bprintf(strbuf, "\n");
+	bcm_bprintf(strbuf, "pub.up %d pub.txoff %d pub.busstate %d\n",
+	            dhdp->up, dhdp->txoff, dhdp->busstate);
+#ifdef WBRC
+	bcm_bprintf(strbuf, "chip_bighammer_count:%d\n", dhdp->chip_bighammer_count);
+#endif /* WBRC */
+	bcm_bprintf(strbuf, "pub.hdrlen %u pub.maxctl %u pub.rxsz %u\n",
+	            dhdp->hdrlen, dhdp->maxctl, dhdp->rxsz);
+	bcm_bprintf(strbuf, "pub.iswl %d pub.drv_version %ld pub.mac "MACDBG"\n",
+	            dhdp->iswl, dhdp->drv_version, MAC2STRDBG(&dhdp->mac));
+	bcm_bprintf(strbuf, "pub.bcmerror %d tickcnt %u\n", dhdp->bcmerror, dhdp->tickcnt);
+
+	dhd_dump_txrx_stats(dhdp, strbuf);
+
+#ifdef DMAMAP_STATS
+	/* Add DMA MAP info */
+	bcm_bprintf(strbuf, "DMA MAP stats: \n");
+	bcm_bprintf(strbuf, "txdata: %lu size: %luK, rxdata: %lu size: %luK\n",
+			dhdp->dma_stats.txdata, KB(dhdp->dma_stats.txdata_sz),
+			dhdp->dma_stats.rxdata, KB(dhdp->dma_stats.rxdata_sz));
+#ifndef IOCTLRESP_USE_CONSTMEM
+	bcm_bprintf(strbuf, "IOCTL RX: %lu size: %luK ,",
+			dhdp->dma_stats.ioctl_rx, KB(dhdp->dma_stats.ioctl_rx_sz));
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+	bcm_bprintf(strbuf, "EVENT RX: %lu size: %luK, INFO RX: %lu size: %luK, "
+			"TSBUF RX: %lu size %luK\n",
+			dhdp->dma_stats.event_rx, KB(dhdp->dma_stats.event_rx_sz),
+			dhdp->dma_stats.info_rx, KB(dhdp->dma_stats.info_rx_sz),
+			dhdp->dma_stats.tsbuf_rx, KB(dhdp->dma_stats.tsbuf_rx_sz));
+	bcm_bprintf(strbuf, "Total : %luK \n",
+			KB(dhdp->dma_stats.txdata_sz + dhdp->dma_stats.rxdata_sz +
+			dhdp->dma_stats.ioctl_rx_sz + dhdp->dma_stats.event_rx_sz +
+			dhdp->dma_stats.tsbuf_rx_sz));
+#endif /* DMAMAP_STATS */
+	bcm_bprintf(strbuf, "dhd_induce_error : %u\n", dhdp->dhd_induce_error);
+	/* Add any prot info */
+	dhd_prot_dump(dhdp, strbuf);
+	bcm_bprintf(strbuf, "\n");
+
+	/* Add any bus info */
+	dhd_bus_dump(dhdp, strbuf);
+#if defined(BCM_ROUTER_DHD) && defined(HNDCTF)
+	/* Add ctf info */
+	dhd_ctf_dump(dhdp, strbuf);
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#if defined(DHD_LB_STATS)
+	dhd_lb_stats_dump(dhdp, strbuf);
+#endif /* DHD_LB_STATS */
+
+#ifdef DHD_MEM_STATS
+	dhd_dump_memstats(dhdp, strbuf);
+#endif /* DHD_MEM_STATS */
+
+#if defined(DHD_MQ) && defined(DHD_MQ_STATS)
+	dhd_mqstats_dump(dhdp, strbuf);
+#endif
+
+#ifdef DHD_WET
+	if (dhd_get_wet_mode(dhdp)) {
+		bcm_bprintf(strbuf, "Wet Dump:\n");
+		dhd_wet_dump(dhdp, strbuf);
+		}
+#endif /* DHD_WET */
+
+	DHD_ERROR(("%s bufsize: %d free: %d\n", __FUNCTION__, buflen, strbuf->size));
+	/* return remaining buffer length */
+	return (!strbuf->size ? BCME_BUFTOOSHORT : strbuf->size);
+}
+
+int
+dhd_counters(dhd_pub_t *dhdp, char *buf, int buflen)
+{
+	struct bcmstrbuf bcmbuf;
+	struct bcmstrbuf *strbuf = &bcmbuf;
+
+	if (!dhdp || !dhdp->prot || !buf) {
+		return BCME_ERROR;
+	}
+
+	bcm_binit(strbuf, buf, buflen);
+
+	dhd_dump_txrx_stats(dhdp, strbuf);
+
+	dhd_bus_counters(dhdp, strbuf);
+
+#if defined(DHD_LB_STATS)
+	dhd_lb_stats_dump(dhdp, strbuf);
+#endif /* DHD_LB_STATS */
+
+#ifdef DHD_MEM_STATS
+	dhd_dump_memstats(dhdp, strbuf);
+#endif /* DHD_MEM_STATS */
+
+	return (!strbuf->size ? BCME_BUFTOOSHORT : strbuf->size);
+}
+
+void
+dhd_dump_to_kernelog(dhd_pub_t *dhdp)
+{
+	char buf[512];
+
+	DHD_ERROR(("F/W version: %s\n", fw_version));
+	bcm_bprintf_bypass = TRUE;
+	dhd_dump(dhdp, buf, sizeof(buf));
+	bcm_bprintf_bypass = FALSE;
+}
+
+int
+dhd_wl_ioctl_cmd(dhd_pub_t *dhd_pub, int cmd, void *arg, int len, uint8 set, int ifidx)
+{
+	wl_ioctl_t ioc;
+
+	ioc.cmd = cmd;
+	ioc.buf = arg;
+	ioc.len = len;
+	ioc.set = set;
+
+	return dhd_wl_ioctl(dhd_pub, ifidx, &ioc, arg, len);
+}
+
+int
+dhd_wl_ioctl_get_intiovar(dhd_pub_t *dhd_pub, char *name, uint *pval,
+	int cmd, uint8 set, int ifidx)
+{
+	char iovbuf[WLC_IOCTL_SMLEN];
+	int ret = -1;
+
+	memset(iovbuf, 0, sizeof(iovbuf));
+	if (bcm_mkiovar(name, NULL, 0, iovbuf, sizeof(iovbuf))) {
+		ret = dhd_wl_ioctl_cmd(dhd_pub, cmd, iovbuf, sizeof(iovbuf), set, ifidx);
+		if (!ret) {
+			*pval = ltoh32(*((uint*)iovbuf));
+		} else {
+			DHD_ERROR(("%s: get int iovar %s failed, ERR %d\n",
+				__FUNCTION__, name, ret));
+		}
+	} else {
+		DHD_ERROR(("%s: mkiovar %s failed\n",
+			__FUNCTION__, name));
+	}
+
+	return ret;
+}
+
+int
+dhd_wl_ioctl_set_intiovar(dhd_pub_t *dhd_pub, char *name, uint val,
+	int cmd, uint8 set, int ifidx)
+{
+	char iovbuf[WLC_IOCTL_SMLEN];
+	int ret = -1;
+	int lval = htol32(val);
+	uint len;
+
+	len = bcm_mkiovar(name, (char*)&lval, sizeof(lval), iovbuf, sizeof(iovbuf));
+
+	if (len) {
+		ret = dhd_wl_ioctl_cmd(dhd_pub, cmd, iovbuf, len, set, ifidx);
+		if (ret) {
+			DHD_ERROR(("%s: set int iovar %s failed, ERR %d\n",
+				__FUNCTION__, name, ret));
+		}
+	} else {
+		DHD_ERROR(("%s: mkiovar %s failed\n",
+			__FUNCTION__, name));
+	}
+
+	return ret;
+}
+
+static struct ioctl2str_s {
+	uint32 ioctl;
+	char *name;
+} ioctl2str_array[] = {
+	{WLC_UP, "UP"},
+	{WLC_DOWN, "DOWN"},
+	{WLC_SET_PROMISC, "SET_PROMISC"},
+	{WLC_SET_INFRA, "SET_INFRA"},
+	{WLC_SET_AUTH, "SET_AUTH"},
+	{WLC_SET_SSID, "SET_SSID"},
+	{WLC_RESTART, "RESTART"},
+	{WLC_SET_CHANNEL, "SET_CHANNEL"},
+	{WLC_SET_RATE_PARAMS, "SET_RATE_PARAMS"},
+	{WLC_SET_KEY, "SET_KEY"},
+	{WLC_SCAN, "SCAN"},
+	{WLC_DISASSOC, "DISASSOC"},
+	{WLC_REASSOC, "REASSOC"},
+	{WLC_SET_COUNTRY, "SET_COUNTRY"},
+	{WLC_SET_WAKE, "SET_WAKE"},
+	{WLC_SET_SCANSUPPRESS, "SET_SCANSUPPRESS"},
+	{WLC_SCB_DEAUTHORIZE, "SCB_DEAUTHORIZE"},
+	{WLC_SET_WSEC, "SET_WSEC"},
+	{WLC_SET_INTERFERENCE_MODE, "SET_INTERFERENCE_MODE"},
+	{WLC_SET_RADAR, "SET_RADAR"},
+	{0, NULL}
+};
+
+static char *
+ioctl2str(uint32 ioctl)
+{
+	struct ioctl2str_s *p = ioctl2str_array;
+
+	while (p->name != NULL) {
+		if (p->ioctl == ioctl) {
+			return p->name;
+		}
+		p++;
+	}
+
+	return "";
+}
+
+/**
+ * @param ioc          IO control struct, members are partially used by this function.
+ * @param buf [inout]  Contains parameters to send to dongle, contains dongle response on return.
+ * @param len          Maximum number of bytes that dongle is allowed to write into 'buf'.
+ */
+int
+dhd_wl_ioctl(dhd_pub_t *dhd_pub, int ifidx, wl_ioctl_t *ioc, void *buf, int len)
+{
+	int ret = BCME_ERROR;
+	unsigned long flags;
+#ifdef DUMP_IOCTL_IOV_LIST
+	dhd_iov_li_t *iov_li;
+#endif /* DUMP_IOCTL_IOV_LIST */
+#ifdef REPORT_FATAL_TIMEOUTS
+	wl_escan_params_v1_t *eparams;
+	uint8 *buf_ptr = (uint8 *)buf;
+	uint16 action = 0;
+#endif /* REPORT_FATAL_TIMEOUTS */
+	int hostsleep_set = 0;
+	int hostsleep_val = 0;
+
+	if (dhd_query_bus_erros(dhd_pub)) {
+		return -ENODEV;
+	}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	DHD_OS_WAKE_LOCK(dhd_pub);
+	if (pm_runtime_get_sync(dhd_bus_to_dev(dhd_pub->bus)) < 0) {
+		DHD_RPM(("%s: pm_runtime_get_sync error. \n", __FUNCTION__));
+		DHD_OS_WAKE_UNLOCK(dhd_pub);
+		return BCME_ERROR;
+	}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef KEEPIF_ON_DEVICE_RESET
+	if (ioc->cmd == WLC_GET_VAR) {
+		dbus_config_t config;
+		config.general_param = 0;
+		if (buf) {
+			if (!strcmp(buf, "wowl_activate")) {
+				 /* 1 (TRUE) after decreased by 1 */
+				config.general_param = 2;
+			} else if (!strcmp(buf, "wowl_clear")) {
+				 /* 0 (FALSE) after decreased by 1 */
+				config.general_param = 1;
+			}
+		}
+		if (config.general_param) {
+			config.config_id = DBUS_CONFIG_ID_KEEPIF_ON_DEVRESET;
+			config.general_param--;
+			dbus_set_config(dhd_pub->dbus, &config);
+		}
+	}
+#endif /* KEEPIF_ON_DEVICE_RESET */
+
+	if (dhd_os_proto_block(dhd_pub))
+	{
+#ifdef DHD_LOG_DUMP
+		int slen, val, lval, min_len;
+		char *msg, tmp[64];
+
+		/* WLC_GET_VAR */
+		if (ioc->cmd == WLC_GET_VAR && buf) {
+			min_len = MIN(sizeof(tmp) - 1, strlen(buf));
+			memset(tmp, 0, sizeof(tmp));
+			bcopy(buf, tmp, min_len);
+			tmp[min_len] = '\0';
+		}
+#endif /* DHD_LOG_DUMP */
+
+#ifdef DHD_DISCONNECT_TRACE
+		if (WLC_DISASSOC == ioc->cmd || WLC_DOWN == ioc->cmd ||
+			WLC_DISASSOC_MYAP == ioc->cmd) {
+			DHD_ERROR(("IOCTL Disconnect WiFi: %d\n", ioc->cmd));
+		}
+#endif /* HW_DISCONNECT_TRACE */
+		/* logging of iovars that are send to the dongle, ./dhd msglevel +iovar */
+		if (ioc->set == TRUE) {
+			char *pars = (char *)buf; // points at user buffer
+			if (ioc->cmd == WLC_SET_VAR && buf) {
+				DHD_DNGL_IOVAR_SET(("iovar:%d: set %s", ifidx, pars));
+				if (ioc->len > 1 + sizeof(uint32)) {
+					// skip iovar name:
+					pars += strnlen(pars, ioc->len - 1 - sizeof(uint32));
+					pars++;               // skip NULL character
+				}
+			} else {
+				DHD_DNGL_IOVAR_SET(("ioctl:%d: set %d %s",
+					ifidx, ioc->cmd, ioctl2str(ioc->cmd)));
+			}
+			if (pars != NULL) {
+				DHD_DNGL_IOVAR_SET((" 0x%x\n", *(uint32*)pars));
+			} else {
+				DHD_DNGL_IOVAR_SET((" NULL\n"));
+			}
+		}
+
+		DHD_LINUX_GENERAL_LOCK(dhd_pub, flags);
+		if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhd_pub)) {
+			DHD_INFO(("%s: returning as busstate=%d\n",
+				__FUNCTION__, dhd_pub->busstate));
+			DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+			dhd_os_proto_unblock(dhd_pub);
+			return -ENODEV;
+		}
+		DHD_BUS_BUSY_SET_IN_IOVAR(dhd_pub);
+		DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+
+#ifdef OEM_ANDROID
+		/*
+		 * If system suspend started before ioctl, after getting d3 ack,
+		 * it will check for wakelock and as ioctl will hold wakelock,
+		 * it will put bus back to d0 and bail out.
+		 * So the current ioctl should wait till it is bailed out.
+		 */
+		if (DHD_BUS_BUSY_CHECK_SUSPEND_IN_PROGRESS(dhd_pub)) {
+			int timeleft = 0;
+			DHD_PRINT(("%s: wait till system suspend bails out\n", __FUNCTION__));
+			/* wait till SUSPEND_IN_PROGRESS bit is cleared */
+			timeleft = dhd_os_busbusy_wait_bitmask(dhd_pub,
+					&dhd_pub->dhd_bus_busy_state,
+					DHD_BUS_BUSY_SUSPEND_IN_PROGRESS, 0);
+			if ((dhd_pub->dhd_bus_busy_state & DHD_BUS_BUSY_SUSPEND_IN_PROGRESS) != 0) {
+				DHD_ERROR(("%s: system suspend wait timed out(%d)"
+					" dhd_bus_busy_state=0x%x\n", __FUNCTION__, timeleft,
+					dhd_pub->dhd_bus_busy_state));
+			}
+		}
+#endif /* OEM_ANDROID */
+
+		DHD_LINUX_GENERAL_LOCK(dhd_pub, flags);
+		if (DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhd_pub) ||
+			dhd_pub->dhd_induce_error == DHD_INDUCE_IOCTL_SUSPEND_ERROR) {
+			DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state!!\n",
+				__FUNCTION__, dhd_pub->busstate, dhd_pub->dhd_bus_busy_state));
+#ifdef DHD_SEND_HANG_IOCTL_SUSPEND_ERROR
+			ioctl_suspend_error++;
+			if (ioctl_suspend_error > MAX_IOCTL_SUSPEND_ERROR) {
+				dhd_pub->hang_reason = HANG_REASON_IOCTL_SUSPEND_ERROR;
+				dhd_os_send_hang_message(dhd_pub);
+				ioctl_suspend_error = 0;
+			}
+#endif /* DHD_SEND_HANG_IOCTL_SUSPEND_ERROR */
+			DHD_BUS_BUSY_CLEAR_IN_IOVAR(dhd_pub);
+			dhd_os_busbusy_wake(dhd_pub);
+			DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+			dhd_os_proto_unblock(dhd_pub);
+			return -ENODEV;
+		}
+#ifdef DHD_SEND_HANG_IOCTL_SUSPEND_ERROR
+		ioctl_suspend_error = 0;
+#endif /* DHD_SEND_HANG_IOCTL_SUSPEND_ERROR */
+		DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+
+#if defined(WL_WLC_SHIM)
+		{
+			struct wl_shim_node *shim = dhd_pub_shim(dhd_pub);
+
+			wl_io_pport_t io_pport;
+			io_pport.dhd_pub = dhd_pub;
+			io_pport.ifidx = ifidx;
+
+			ret = wl_shim_ioctl(shim, ioc, len, &io_pport);
+			if (ret != BCME_OK) {
+				DHD_TRACE(("%s: wl_shim_ioctl(%d) ERR %d\n",
+					__FUNCTION__, ioc->cmd, ret));
+			}
+		}
+#else
+#ifdef DUMP_IOCTL_IOV_LIST
+		if (ioc->cmd != WLC_GET_MAGIC && ioc->cmd != WLC_GET_VERSION && buf) {
+			if (!(iov_li = MALLOC(dhd_pub->osh, sizeof(*iov_li)))) {
+				DHD_ERROR(("iovar dump list item allocation Failed\n"));
+			} else {
+				iov_li->cmd = ioc->cmd;
+				if (buf)
+					bcopy((char *)buf, iov_li->buff, strlen((char *)buf)+1);
+				dhd_iov_li_append(dhd_pub, &dhd_pub->dump_iovlist_head,
+						&iov_li->list);
+			}
+		}
+#endif /* DUMP_IOCTL_IOV_LIST */
+
+#ifdef REPORT_FATAL_TIMEOUTS
+		/* fill in the sync_id to ensure that the scan timeout is always for the
+		* current running escan in the FW - the wl app does not fill in an
+		* incrementing number for sync_id, it only fills in a random number which
+		* increases the chance of 2 consecutive escans having the same sync id
+		* This should happen here after dhd_proto_block()
+		* is called, so that sync_id does not
+		* get incremented if 2 consecutive escans are fired in quick succession
+		*/
+		if ((ioc->cmd == WLC_SET_VAR &&
+				buf != NULL &&
+				strcmp("escan", buf) == 0)) {
+			eparams = (wl_escan_params_v1_t *) (buf_ptr + strlen("escan") + 1);
+			action = dtoh16(eparams->action);
+			if (action == WL_SCAN_ACTION_START) {
+				++dhd_pub->esync_id;
+				/* sync id of 0 is not used for escan,
+				* it is used to indicate
+				* a normal scan timer is running, so as
+				* to ensure that escan abort event
+				* does not cancel a normal scan timeout
+				*/
+				if (dhd_pub->esync_id == 0)
+					++dhd_pub->esync_id;
+				DHD_INFO(("%s:escan sync id set to = %u \n",
+					__FUNCTION__, dhd_pub->esync_id));
+				eparams->sync_id = htod16(dhd_pub->esync_id);
+			}
+		}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+		if (dhd_conf_check_hostsleep(dhd_pub, ioc->cmd, ioc->buf, len,
+				&hostsleep_set, &hostsleep_val, &ret))
+			goto exit;
+		ret = dhd_prot_ioctl(dhd_pub, ifidx, ioc, buf, len);
+		dhd_conf_get_hostsleep(dhd_pub, hostsleep_set, hostsleep_val, ret);
+
+#ifdef DUMP_IOCTL_IOV_LIST
+		if (ret == -ETIMEDOUT) {
+			DHD_ERROR(("Last %d issued commands: Latest one is at bottom.\n",
+				IOV_LIST_MAX_LEN));
+			dhd_iov_li_print(&dhd_pub->dump_iovlist_head);
+		}
+#endif /* DUMP_IOCTL_IOV_LIST */
+#endif /* defined(WL_WLC_SHIM) */
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+		if (ret == -ETIMEDOUT) {
+			copy_hang_info_ioctl_timeout(dhd_pub, ifidx, ioc);
+		}
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+#ifdef DHD_LOG_DUMP
+		if ((ioc->cmd == WLC_GET_VAR || ioc->cmd == WLC_SET_VAR) &&
+				buf != NULL) {
+			if (buf) {
+				lval = 0;
+				slen = strlen(buf) + 1;
+				msg = (char*)buf;
+				if (len >= slen + sizeof(lval)) {
+					if (ioc->cmd == WLC_GET_VAR) {
+						msg = tmp;
+						lval = *(int*)buf;
+					} else {
+						min_len = MIN(ioc->len - slen, sizeof(int));
+						bcopy((msg + slen), &lval, min_len);
+					}
+					if (!strncmp(msg, "cur_etheraddr",
+						strlen("cur_etheraddr"))) {
+						lval = 0;
+					}
+				}
+				DHD_IOVAR_LOG(dhd_pub, ioc->cmd, msg,
+					("%s: cmd: %d, msg: %s val: 0x%x,"
+					" len: %d, set: %d, txn-id: %d\n",
+					ioc->cmd == WLC_GET_VAR ?
+					"WLC_GET_VAR" : "WLC_SET_VAR",
+					ioc->cmd, msg, lval, ioc->len, ioc->set,
+					dhd_prot_get_ioctl_trans_id(dhd_pub)));
+			} else {
+				DHD_IOVAR_MEM(("%s: cmd: %d, len: %d, set: %d, txn-id: %d\n",
+					ioc->cmd == WLC_GET_VAR ? "WLC_GET_VAR" : "WLC_SET_VAR",
+					ioc->cmd, ioc->len, ioc->set,
+					dhd_prot_get_ioctl_trans_id(dhd_pub)));
+			}
+		} else {
+			slen = ioc->len;
+			if (buf != NULL && slen != 0) {
+				if (slen >= 4) {
+					val = *(int*)buf;
+				} else if (slen >= 2) {
+					val = *(short*)buf;
+				} else {
+					val = *(char*)buf;
+				}
+				/* Do not dump for WLC_GET_MAGIC and WLC_GET_VERSION */
+				if (ioc->cmd != WLC_GET_MAGIC && ioc->cmd != WLC_GET_VERSION) {
+					DHD_IOVAR_MEM(("WLC_IOCTL: cmd: %d, val: %d, len: %d, "
+						"set: %d\n", ioc->cmd, val, ioc->len, ioc->set));
+				}
+			} else {
+				DHD_IOVAR_MEM(("WLC_IOCTL: cmd: %d, buf is NULL\n", ioc->cmd));
+			}
+		}
+#endif /* DHD_LOG_DUMP */
+#if defined(OEM_ANDROID)
+		if (ret && dhd_pub->up) {
+			/* Send hang event only if dhd_open() was success */
+			dhd_os_check_hang(dhd_pub, ifidx, ret);
+		}
+
+		if (ret == -ETIMEDOUT && !dhd_pub->up) {
+			DHD_ERROR(("%s: 'resumed on timeout' error is "
+				"occurred before the interface does not"
+				" bring up\n", __FUNCTION__));
+		}
+#endif /* defined(OEM_ANDROID) */
+
+exit:
+		DHD_LINUX_GENERAL_LOCK(dhd_pub, flags);
+		DHD_BUS_BUSY_CLEAR_IN_IOVAR(dhd_pub);
+		dhd_os_busbusy_wake(dhd_pub);
+		DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+
+#ifdef REPORT_FATAL_TIMEOUTS
+		if ((ret == BCME_OK && ioc->cmd == WLC_SET_VAR &&
+				buf != NULL &&
+				strcmp("escan", buf) == 0)) {
+			if (action == WL_SCAN_ACTION_START)
+				dhd_start_scan_timer(dhd_pub, TRUE);
+		}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+		dhd_os_proto_unblock(dhd_pub);
+
+#ifdef DETAIL_DEBUG_LOG_FOR_IOCTL
+		if (ret < 0) {
+			if ((ioc->cmd == WLC_GET_VAR || ioc->cmd == WLC_SET_VAR) &&
+					buf != NULL) {
+				if (ret == BCME_UNSUPPORTED || ret == BCME_NOTASSOCIATED) {
+					DHD_ERROR_MEM(("%s: %s: %s, %s\n",
+						__FUNCTION__, ioc->cmd == WLC_GET_VAR ?
+						"WLC_GET_VAR" : "WLC_SET_VAR",
+						buf? (char *)buf:"NO MESSAGE",
+						ret == BCME_UNSUPPORTED ? "UNSUPPORTED"
+						: "NOT ASSOCIATED"));
+				} else {
+					DHD_ERROR_MEM(("%s: %s: %s, ret = %d\n",
+						__FUNCTION__, ioc->cmd == WLC_GET_VAR ?
+						"WLC_GET_VAR" : "WLC_SET_VAR",
+						(char *)buf, ret));
+				}
+			} else {
+				if (ret == BCME_UNSUPPORTED || ret == BCME_NOTASSOCIATED) {
+					DHD_ERROR_MEM(("%s: WLC_IOCTL: cmd: %d, %s\n",
+						__FUNCTION__, ioc->cmd,
+						ret == BCME_UNSUPPORTED ? "UNSUPPORTED" :
+						"NOT ASSOCIATED"));
+				} else {
+					DHD_ERROR_MEM(("%s: WLC_IOCTL: cmd: %d, ret = %d\n",
+						__FUNCTION__, ioc->cmd, ret));
+				}
+			}
+		}
+#endif /* DETAIL_DEBUG_LOG_FOR_IOCTL */
+	}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	pm_runtime_mark_last_busy(dhd_bus_to_dev(dhd_pub->bus));
+	pm_runtime_put_autosuspend(dhd_bus_to_dev(dhd_pub->bus));
+
+	DHD_OS_WAKE_UNLOCK(dhd_pub);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef WL_MONITOR
+	/* Intercept monitor ioctl here, add/del monitor if */
+	if (ret == BCME_OK && ioc->cmd == WLC_SET_MONITOR) {
+		int val = 0;
+		if (buf != NULL && len != 0) {
+			if (len >= 4) {
+				val = *(int*)buf;
+			} else if (len >= 2) {
+				val = *(short*)buf;
+			} else {
+				val = *(char*)buf;
+			}
+		}
+		dhd_set_monitor(dhd_pub, ifidx, val);
+	}
+#endif /* WL_MONITOR */
+
+	return ret;
+}
+
+uint wl_get_port_num(wl_io_pport_t *io_pport)
+{
+	return 0;
+}
+
+/* Get bssidx from iovar params
+ * Input:   dhd_pub - pointer to dhd_pub_t
+ *	    params  - IOVAR params
+ * Output:  idx	    - BSS index
+ *	    val	    - ponter to the IOVAR arguments
+ */
+static int
+dhd_iovar_parse_bssidx(dhd_pub_t *dhd_pub, const char *params, uint32 *idx, const char **val)
+{
+	char *prefix = "bsscfg:";
+	uint32	bssidx;
+
+	if (!(strncmp(params, prefix, strlen(prefix)))) {
+		/* per bss setting should be prefixed with 'bsscfg:' */
+		const char *p = params + strlen(prefix);
+
+		/* Skip Name */
+		while (*p != '\0')
+			p++;
+		/* consider null */
+		p = p + 1;
+		bcopy(p, &bssidx, sizeof(uint32));
+		/* Get corresponding dhd index */
+		bssidx = dhd_bssidx2idx(dhd_pub, htod32(bssidx));
+
+		if (bssidx >= DHD_MAX_IFS) {
+			DHD_ERROR(("%s Wrong bssidx provided\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+
+		/* skip bss idx */
+		p += sizeof(uint32);
+		*val = p;
+		*idx = bssidx;
+	} else {
+		DHD_ERROR(("%s: bad parameter for per bss iovar\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	return BCME_OK;
+}
+
+#if defined(DHD_DEBUG) && defined(BCMDBUS)
+/* USB Device console input function */
+int dhd_bus_console_in(dhd_pub_t *dhd, uchar *msg, uint msglen)
+{
+	DHD_TRACE(("%s \n", __FUNCTION__));
+
+	return dhd_iovar(dhd, 0, "cons", msg, msglen, NULL, 0, TRUE);
+
+}
+#endif /* DHD_DEBUG && BCMDBUS  */
+
+#ifdef DHD_DEBUG
+int
+dhd_mem_debug(dhd_pub_t *dhd, uchar *msg, uint msglen)
+{
+	unsigned long int_arg = 0;
+	char *p;
+	char *end_ptr = NULL;
+	dhd_dbg_mwli_t *mw_li;
+	dll_t *item, *next;
+	/* check if mwalloc, mwquery or mwfree was supplied arguement with space */
+	p = bcmstrstr((char *)msg, " ");
+	if (p != NULL) {
+		/* space should be converted to null as separation flag for firmware */
+		*p = '\0';
+		/* store the argument in int_arg */
+		int_arg = bcm_strtoul(p+1, &end_ptr, 10);
+	}
+
+	if (!p && !strcmp(msg, "query")) {
+		/* lets query the list inetrnally */
+		if (dll_empty(dll_head_p(&dhd->mw_list_head))) {
+			DHD_ERROR(("memwaste list is empty, call mwalloc < size > to allocate\n"));
+		} else {
+			for (item = dll_head_p(&dhd->mw_list_head);
+					!dll_end(&dhd->mw_list_head, item); item = next) {
+				next = dll_next_p(item);
+				mw_li = (dhd_dbg_mwli_t *)CONTAINEROF(item, dhd_dbg_mwli_t, list);
+				DHD_ERROR(("item: <id=%d, size=%d>\n", mw_li->id, mw_li->size));
+			}
+		}
+	} else if (p && end_ptr && (*end_ptr == '\0') && !strcmp(msg, "alloc")) {
+		int32 alloc_handle;
+		/* convert size into KB and append as integer */
+		*((int32 *)(p+1)) = int_arg*1024;
+		*(p+1+sizeof(int32)) = '\0';
+
+		/* recalculated length -> 5 bytes for "alloc" + 4 bytes for size +
+		 * 1 bytes for null caracter
+		 */
+		msglen = strlen(msg) + sizeof(int32) + 1;
+		if (dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, msg, msglen+1, FALSE, 0) < 0) {
+			DHD_ERROR(("IOCTL failed for memdebug alloc\n"));
+		}
+
+		/* returned allocated handle from dongle, basically address of the allocated unit */
+		alloc_handle = *((int32 *)msg);
+
+		/* add a node in the list with tuple <id, handle, size> */
+		if (alloc_handle == 0) {
+			DHD_ERROR(("Reuqested size could not be allocated\n"));
+		} else if (!(mw_li = MALLOC(dhd->osh, sizeof(*mw_li)))) {
+			DHD_ERROR(("mw list item allocation Failed\n"));
+		} else {
+			mw_li->id = dhd->mw_id++;
+			mw_li->handle = alloc_handle;
+			mw_li->size = int_arg;
+			/* append the node in the list */
+			dll_append(&dhd->mw_list_head, &mw_li->list);
+		}
+	} else if (p && end_ptr && (*end_ptr == '\0') && !strcmp(msg, "free")) {
+		/* inform dongle to free wasted chunk */
+		int handle = 0;
+		int size = 0;
+		for (item = dll_head_p(&dhd->mw_list_head);
+				!dll_end(&dhd->mw_list_head, item); item = next) {
+			next = dll_next_p(item);
+			mw_li = (dhd_dbg_mwli_t *)CONTAINEROF(item, dhd_dbg_mwli_t, list);
+
+			if (mw_li->id == (int)int_arg) {
+				handle = mw_li->handle;
+				size = mw_li->size;
+				dll_delete(item);
+				MFREE(dhd->osh, mw_li, sizeof(*mw_li));
+				if (dll_empty(dll_head_p(&dhd->mw_list_head))) {
+					/* reset the id */
+					dhd->mw_id = 0;
+				}
+			}
+		}
+		if (handle) {
+			int len;
+			/* append the free handle and the chunk size in first 8 bytes
+			 * after the command and null character
+			 */
+			*((int32 *)(p+1)) = handle;
+			*((int32 *)((p+1)+sizeof(int32))) = size;
+			/* append null as terminator */
+			*(p+1+2*sizeof(int32)) = '\0';
+			/* recalculated length -> 4 bytes for "free" + 8 bytes for hadnle and size
+			 * + 1 bytes for null caracter
+			 */
+			len = strlen(msg) + 2*sizeof(int32) + 1;
+			/* send iovar to free the chunk */
+			if (dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, msg, len, FALSE, 0) < 0) {
+				DHD_ERROR(("IOCTL failed for memdebug free\n"));
+			}
+		} else {
+			DHD_ERROR(("specified id does not exist\n"));
+		}
+	} else {
+		/* for all the wrong argument formats */
+		return BCME_BADARG;
+	}
+	return 0;
+}
+extern void
+dhd_mw_list_delete(dhd_pub_t *dhd, dll_t *list_head)
+{
+	dll_t *item;
+	dhd_dbg_mwli_t *mw_li;
+	while (!(dll_empty(list_head))) {
+		item = dll_head_p(list_head);
+		mw_li = (dhd_dbg_mwli_t *)CONTAINEROF(item, dhd_dbg_mwli_t, list);
+		dll_delete(item);
+		MFREE(dhd->osh, mw_li, sizeof(*mw_li));
+	}
+}
+#ifdef BCMPCIE
+int
+dhd_flow_ring_debug(dhd_pub_t *dhd, char *msg, uint msglen)
+{
+	flow_ring_table_t *flow_ring_table;
+	char *cmd;
+	char *end_ptr = NULL;
+	uint8 prio;
+	uint16 flowid;
+	int i;
+	int ret = 0;
+	cmd = bcmstrstr(msg, " ");
+	BCM_REFERENCE(prio);
+	if (cmd != NULL) {
+		/* in order to use string operations append null */
+		*cmd = '\0';
+	} else {
+		DHD_ERROR(("missing: create/delete args\n"));
+		return BCME_ERROR;
+	}
+	if (cmd && !strcmp(msg, "create")) {
+		/* extract <"source address", "destination address", "priority"> */
+		uint8 sa[ETHER_ADDR_LEN], da[ETHER_ADDR_LEN];
+		BCM_REFERENCE(sa);
+		BCM_REFERENCE(da);
+		msg = msg + strlen("create") + 1;
+		/* fill ethernet source address */
+		for (i = 0; i < ETHER_ADDR_LEN; i++) {
+			sa[i] = (uint8)bcm_strtoul(msg, &end_ptr, 16);
+			if (*end_ptr == ':') {
+				msg = (end_ptr + 1);
+			} else if (i != 5) {
+				DHD_ERROR(("not a valid source mac addr\n"));
+				return BCME_ERROR;
+			}
+		}
+		if (*end_ptr != ' ') {
+			DHD_ERROR(("missing: destiantion mac id\n"));
+			return BCME_ERROR;
+		} else {
+			/* skip space */
+			msg = end_ptr + 1;
+		}
+		/* fill ethernet destination address */
+		for (i = 0; i < ETHER_ADDR_LEN; i++) {
+			da[i] = (uint8)bcm_strtoul(msg, &end_ptr, 16);
+			if (*end_ptr == ':') {
+				msg = (end_ptr + 1);
+			} else if (i != 5) {
+				DHD_ERROR(("not a valid destination  mac addr\n"));
+				return BCME_ERROR;
+			}
+		}
+		if (*end_ptr != ' ') {
+			DHD_ERROR(("missing: priority\n"));
+			return BCME_ERROR;
+		} else {
+			msg = end_ptr + 1;
+		}
+		/* parse priority */
+		prio = (uint8)bcm_strtoul(msg, &end_ptr, 10);
+		if (prio > MAXPRIO) {
+			DHD_ERROR(("%s: invalid priority. Must be between 0-7 inclusive\n",
+				__FUNCTION__));
+			return BCME_ERROR;
+		}
+
+		if (*end_ptr != '\0') {
+			DHD_ERROR(("msg not truncated with NULL character\n"));
+			return BCME_ERROR;
+		}
+		ret = dhd_flowid_debug_create(dhd, 0, prio, (char *)sa, (char *)da, &flowid);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: flowring creation failed ret: %d\n", __FUNCTION__, ret));
+			return BCME_ERROR;
+		}
+		return BCME_OK;
+
+	} else if (cmd && !strcmp(msg, "delete")) {
+		msg = msg + strlen("delete") + 1;
+		/* parse flowid */
+		flowid = (uint16)bcm_strtoul(msg, &end_ptr, 10);
+		if (*end_ptr != '\0') {
+			DHD_ERROR(("msg not truncated with NULL character\n"));
+			return BCME_ERROR;
+		}
+
+		/* Find flowid from ifidx 0 since this IOVAR creating flowring with ifidx 0 */
+		if (dhd_flowid_find_by_ifidx(dhd, 0, flowid) != BCME_OK)
+		{
+			DHD_ERROR(("%s : Deleting not created flowid: %u\n", __FUNCTION__, flowid));
+			return BCME_ERROR;
+		}
+
+		flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
+		ret = dhd_bus_flow_ring_delete_request(dhd->bus, (void *)&flow_ring_table[flowid]);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: flowring deletion failed ret: %d\n", __FUNCTION__, ret));
+			return BCME_ERROR;
+		}
+		return BCME_OK;
+	}
+	DHD_ERROR(("%s: neither create nor delete\n", __FUNCTION__));
+	return BCME_ERROR;
+}
+#endif /* BCMPCIE */
+#endif /* DHD_DEBUG */
+
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+
+typedef struct _dhd_sar_parameters {
+	dhd_sar_parameter  other;
+	dhd_sar_parameter  fcc;
+	dhd_sar_parameter  eu;
+	dhd_sar_parameter  latam;
+	dhd_sar_parameter  canada;
+	dhd_sar_parameter  japan;
+} dhd_sar_parameters;
+
+static dhd_sar_parameters  gSAR_params;
+
+/* pick up usable parameters */
+static sarctrl_set *dhd_sar_ctrlset_get(
+		dhd_sar_parameter *dhd_sar_param, int advance_mode)
+{
+	sarctrl_set *sarctrl = NULL;
+
+	if (dhd_sar_param == NULL) {
+		DHD_ERROR(("%s: *invalid argument, sar_param=0x%p\n",
+		           __FUNCTION__, dhd_sar_param));
+	} else switch (advance_mode) {
+		case SAR_HEAD:
+			sarctrl = &dhd_sar_param->dynamic;
+			DHD_TRACE(("%s: dynamic=0x%p, sarctrl=0x%p, ver=%d,"
+			           " sarctrl_2g=0x%X, sarctrl_2g_2=0x%X\n",
+			           __FUNCTION__, dhd_sar_param, sarctrl, sarctrl->ver,
+			           sarctrl->basic.sarctrl_2g, sarctrl->basic.sarctrl_2g_2));
+			break;
+		case SAR_GRIP:
+			sarctrl = &dhd_sar_param->grip;
+			DHD_TRACE(("%s: grip=0x%p, sarctrl=0x%p, ver=%d,"
+			           " sarctrl_2g=0x%X, sarctrl_2g_2=0x%X\n",
+			           __FUNCTION__, dhd_sar_param, sarctrl, sarctrl->ver,
+			           sarctrl->basic.sarctrl_2g, sarctrl->basic.sarctrl_2g_2));
+			break;
+		case SAR_BT:
+			sarctrl = &dhd_sar_param->bt;
+			DHD_TRACE(("%s: bt=0x%p, sarctrl=0x%p, ver=%d,"
+			           " sarctrl_2g=0x%X, sarctrl_2g_2=0x%X\n",
+			           __FUNCTION__, dhd_sar_param, sarctrl, sarctrl->ver,
+			           sarctrl->basic.sarctrl_2g, sarctrl->basic.sarctrl_2g_2));
+			break;
+		case SAR_HOTSPOT:
+			sarctrl = &dhd_sar_param->hotspot;
+			DHD_TRACE(("%s: hotspot=0x%p, sarctrl=0x%p, ver=%d,"
+			           " sarctrl_2g=0x%X, sarctrl_2g_2=0x%X\n",
+			           __FUNCTION__, dhd_sar_param, sarctrl, sarctrl->ver,
+			           sarctrl->basic.sarctrl_2g, sarctrl->basic.sarctrl_2g_2));
+			break;
+		default:
+			DHD_ERROR(("%s: *Error, invalid advance_mode=%d\n",
+			           __FUNCTION__, advance_mode));
+	}
+
+	return sarctrl;
+}
+
+int dhd_sar_init_parameter(dhd_pub_t *dhd, eCountry_flag_type type, int advance_mode,
+	char *list_str)
+{
+	char          temp_str[16];
+	sarctrl_set  *sarctrl = NULL;
+	uint32       *txpwr_list = NULL;
+	char         *next = NULL;
+	int          len = 0, idx = 0;
+
+	if (list_str == NULL) {
+		DHD_ERROR(("%s: *Error, invalid parameber\n", __FUNCTION__));
+		return BCME_BADARG;
+	} else {
+		next = list_str;
+	}
+
+	switch (type) {
+		case SYNA_COUNTRY_TYPE_FCC:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.fcc, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_EU:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.eu, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_LATAM:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.latam, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_CANADA:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.canada, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_JAPAN:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.japan, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_DEFAULT:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.other, advance_mode);
+			break;
+		default:
+			DHD_ERROR(("%s: *Error, invalid type=%d\n", __FUNCTION__, type));
+			return BCME_UNSUPPORTED;
+			break;
+	}
+	if (!sarctrl) {
+		DHD_ERROR(("%s: *Error, invalid advance_mode=%d\n",
+		           __FUNCTION__, advance_mode));
+		return BCME_UNSUPPORTED;
+	}
+
+	/* change dhd_sar_parameter structure to uint32 array.
+	 * use txpwr_list pointer to fill the data
+	 * one by one in following loop code
+	 */
+	txpwr_list = (uint32 *)(&sarctrl->basic);
+	idx = 0;
+	while ((len = strcspn(next, " ,")) > 0) {
+		if (len >= sizeof(temp_str)) {
+			DHD_ERROR(("%s: init '%s' before ','/' ' is too long\n",
+			           __FUNCTION__, next));
+			return BCME_ERROR;
+		}
+		strlcpy(temp_str, next, sizeof(temp_str));
+		temp_str[len] = 0;
+		txpwr_list[idx] = (uint32)simple_strtoul(temp_str, NULL, 0);
+		DHD_TRACE(("%s: get txpwr[%d]=0x%x\n",
+		           __FUNCTION__, idx, txpwr_list[idx]));
+
+		if (idx == CONST_SARCTRL_SET_QTY) {
+			DHD_ERROR(("%s: too many parameters(more than %d): '%s'\n",
+			           __FUNCTION__, idx, list_str));
+			break;
+		}
+
+		idx++;
+		next += len;
+		next += strspn(next, " ,");
+	}
+
+#ifdef VSDB
+	if (FW_SUPPORTED(dhd, 6g)) {
+		sarctrl->ver = SAR_PARAM_6G_V3_VSDB | (idx << SAR_PARAM_NUM_OFFSET);
+	} else {
+		sarctrl->ver = SAR_PARAM_V1_VSDB | (idx << SAR_PARAM_NUM_OFFSET);
+	}
+	if (CONST_SARCTRL_SINGLE_SET_QTY != idx) {
+		DHD_ERROR(("%s: *Waring, idx=%d mismatch VSDB expect=%d\n",
+		           __FUNCTION__, idx, (int)CONST_SARCTRL_SINGLE_SET_QTY));
+	}
+#else /* VSDB */
+	if (FW_SUPPORTED(dhd, 6g)) {
+		sarctrl->ver = SAR_PARAM_6G_V4_RSDB | (idx << SAR_PARAM_NUM_OFFSET);
+	} else {
+		sarctrl->ver = SAR_PARAM_V2_RSDB | (idx << SAR_PARAM_NUM_OFFSET);
+	}
+	if (CONST_SARCTRL_SET_QTY != idx) {
+		DHD_ERROR(("%s: *Waring, idx=%d mismatch RSDB expect=%d\n",
+		           __FUNCTION__, idx, (int)CONST_SARCTRL_SET_QTY));
+	}
+#endif /* VSDB */
+
+	return 0;
+}
+
+int dhd_sar_reset_parameter(void)
+{
+	bzero(&gSAR_params, sizeof(gSAR_params));
+
+	return BCME_OK;
+}
+
+int dhd_sar_set_parameter(dhd_pub_t *dhd_pub, int advance_mode)
+{
+	int                 err = BCME_OK;
+	char                iovbuf[WLC_IOCTL_SMLEN];
+	wl_country_t       *cspec = NULL;
+	sarctrl_set         sarctrl_iov = { 0 }, *sarctrl = NULL;
+	eCountry_flag_type  type = SYNA_COUNTRY_TYPE_INVALID;
+	int sar_ver = 0, sar_param_num = 0;
+
+	if (dhd_pub == NULL) {
+		DHD_ERROR(("%s: bcm_mkiovar failed.", __FUNCTION__));
+		return BCME_BADARG;
+	} else if (SAR_DISABLE == advance_mode) {
+		DHD_ERROR(("%s: SKIP since disabled.", __FUNCTION__));
+		return BCME_OK;
+	}
+
+	memset(iovbuf, 0, sizeof(iovbuf));
+	if (0 >= bcm_mkiovar("country", NULL, 0, iovbuf, sizeof(iovbuf))) {
+		err = BCME_BUFTOOSHORT;
+		DHD_ERROR(("%s: sar country mkiovar failed.", __FUNCTION__));
+		return err;
+	}
+	err = dhd_wl_ioctl_cmd(dhd_pub, WLC_GET_VAR, iovbuf, sizeof(iovbuf), FALSE, 0);
+	if (err) {
+		DHD_ERROR(("%s: country code get failed\n", __FUNCTION__));
+		return err;
+	}
+	cspec = (wl_country_t *)iovbuf;
+	type = syna_country_check_type(cspec->ccode);
+	switch (type) {
+		case SYNA_COUNTRY_TYPE_FCC:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.fcc, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_EU:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.eu, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_LATAM:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.latam, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_CANADA:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.canada, advance_mode);
+			break;
+		case SYNA_COUNTRY_TYPE_JAPAN:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.japan, advance_mode);
+			break;
+		default:
+			sarctrl = dhd_sar_ctrlset_get(&gSAR_params.other, advance_mode);
+			break;
+	}
+	/* check if parameters valid */
+	if ((!sarctrl) || (!sarctrl->ver)) {
+		DHD_ERROR(("%s: *warning, try default for country='%s'(type=%d)\n",
+		           __FUNCTION__, cspec->ccode, type));
+		sarctrl = dhd_sar_ctrlset_get(&gSAR_params.other, advance_mode);
+		if ((!sarctrl) || (!sarctrl->ver)) {
+			/* stop/block setting if customer not set parameters */
+			DHD_ERROR(("%s: *skip since parameter not set\n", __FUNCTION__));
+			return BCME_UNSUPPORTED;
+		}
+	}
+	DHD_TRACE(("%s: country='%s', type=%d, advance_mode=%d\n",
+	           __FUNCTION__, cspec->ccode, type, advance_mode));
+	/* Try to get sar version from FW */
+	err = dhd_iovar(dhd_pub, 0, "sar_params", NULL, 0,
+		(char *)&sarctrl_iov, sizeof(sarctrl_iov), FALSE);
+	if (err) {
+		/* Not support sar_params read */
+	} else {
+		sar_ver = sarctrl_iov.ver & SAR_PARAM_VER_MASK;
+		sar_param_num = (sarctrl_iov.ver & SAR_PARAM_NUM_MASK) >> SAR_PARAM_NUM_OFFSET;
+	}
+
+	memset(&sarctrl_iov, 0, sizeof(sarctrl_iov));
+	if (sar_ver == SAR_PARAM_DYN) {
+		int file_sar_num = (sarctrl->ver & SAR_PARAM_NUM_MASK) >> SAR_PARAM_NUM_OFFSET;
+		if (file_sar_num  != sar_param_num) {
+			DHD_ERROR(("%s: Need confirm SAR PARAMETERS configuration based on FW\n",
+				__FUNCTION__));
+			return BCME_BADARG;
+		}
+		sarctrl_iov.ver = sarctrl->ver;
+		err = memcpy_s(sarctrl_iov.sarctrl, MAX_SAR_PARAMS_NUM * sizeof(u32),
+			sarctrl->sarctrl, file_sar_num * sizeof(u32));
+		if (err) {
+			DHD_ERROR(("%s: memcpy error num = %d\n",
+				__FUNCTION__, file_sar_num));
+			return BCME_BADARG;
+		}
+	} else {
+		/* prepare IOVAR for old version*/
+		sarctrl_iov.ver   = sarctrl->ver & SAR_PARAM_VER_MASK;
+		sarctrl_iov.basic = sarctrl->basic;
+		sarctrl_iov.rsdb  = sarctrl->rsdb;
+	}
+	DHD_TRACE(("%s: sarctrl_iov=0x%p, ver=%x, sarctrl_2g=0x%X, sarctrl_2g_2=0x%X\n",
+	           __FUNCTION__, &sarctrl_iov, sarctrl_iov.ver,
+	           sarctrl_iov.basic.sarctrl_2g,
+	           sarctrl_iov.basic.sarctrl_2g_2));
+	err = dhd_iovar(dhd_pub, 0, "sar_params", (char *)&sarctrl_iov,
+		sizeof(sarctrl_iov), NULL, 0, TRUE);
+	if (unlikely(err)) {
+		DHD_ERROR(("%s: Failed to set sar_params - error (%d)\n",
+		           __FUNCTION__, err));
+	}
+
+	return err;
+}
+
+int dhd_sar_set(dhd_pub_t *dhd_pub, int sar_val)
+{
+	sar_advance_modes sar_tx_power_val = SAR_DISABLE;
+	int airplane_mode = 0;
+	int err = 0;
+
+	/* Map Android TX power modes to power mode */
+	switch (sar_val) {
+		case SAR_OFF:
+			/* SAR disabled */
+			sar_tx_power_val = SAR_DISABLE;
+			airplane_mode = 0;
+			break;
+		case HEAD_NORMAL:
+			/* HEAD mode, Normal */
+			sar_tx_power_val = SAR_HEAD;
+			airplane_mode = 0;
+			break;
+		case HEAD_AIRPLANE:
+			/* HEAD mode, Airplane */
+			sar_tx_power_val = SAR_HEAD;
+			airplane_mode = 1;
+			break;
+		case GRIP_NORMAL:
+			/* GRIP mode, Normal */
+			sar_tx_power_val = SAR_GRIP;
+			airplane_mode = 0;
+			break;
+		case GRIP_AIRPLANE:
+			/* GRIP mode, Airplane */
+			sar_tx_power_val = SAR_GRIP;
+			airplane_mode = 1;
+			break;
+		case BT_NORMAL:
+			/* BT mode, Normal */
+			sar_tx_power_val = SAR_BT;
+			airplane_mode = 0;
+			break;
+		case BT_AIRPLANE:
+			/* BT mode, Airplane */
+			sar_tx_power_val = SAR_BT;
+			airplane_mode = 1;
+			break;
+		case HOTSPOT_NORMAL:
+			/* HOTSPOT mode, Normal */
+			sar_tx_power_val = SAR_HOTSPOT;
+			airplane_mode = 0;
+			break;
+		case HOTSPOT_AIRPLANE:
+			/* HOTSPOT mode, AIRPLANE */
+			sar_tx_power_val = SAR_HOTSPOT;
+			airplane_mode = 1;
+			break;
+		default:
+			DHD_ERROR(("%s: invalid wifi tx power scenario = %d\n",
+			        __FUNCTION__, sar_tx_power_val));
+			err = -EINVAL;
+			goto exit;
+	}
+
+	if (sar_tx_power_val) {
+		err = dhd_sar_set_parameter(dhd_pub, sar_tx_power_val);
+		if (!err) {
+			/* If dhd and FW all support this feature, it will return 0
+			 * DHD already set sar parameter to FW.
+			 * so only need to set sar_enable 1 to enable SAR
+			 */
+			sar_tx_power_val = 1;
+		} else if (err == BCME_UNSUPPORTED) {
+			/* If dhd or FW not support/set these parameters
+			 * use traditional way
+			 */
+			DHD_TRACE(("DHD/FW not support sar_param,"
+					"try to use SAR parameters in nvram"));
+		} else {
+			DHD_ERROR(("%s: Failed to set sar_params - error (%d)\n",
+			           __FUNCTION__, err));
+			goto exit;
+		}
+	}
+
+	DHD_TRACE(("%s: sar_mode %d airplane_mode %d\n",
+	           __FUNCTION__, sar_tx_power_val, airplane_mode));
+	err = dhd_iovar(dhd_pub, 0, "fccpwrlimit2g", (char *)&airplane_mode,
+	                sizeof(airplane_mode), NULL, 0, TRUE);
+	if (unlikely(err)) {
+		DHD_ERROR(("%s: Failed to set airplane_mode - error (%d)\n",
+		           __FUNCTION__, err));
+		goto exit;
+	}
+	err = dhd_iovar(dhd_pub, 0, "sar_enable", (char *)&sar_tx_power_val,
+	                sizeof(sar_tx_power_val), NULL, 0, TRUE);
+	if (unlikely(err)) {
+		DHD_ERROR(("%s: Failed to set sar_enable - error (%d)\n",
+		           __FUNCTION__, err));
+		goto exit;
+	}
+
+	/* Cache the tx power mode sent by the hal */
+	dhd_pub->dhd_sar_mode = sar_val;
+	DHD_TRACE(("%s: tx_power_mode %d SUCCESS\n", __FUNCTION__, sar_val));
+
+exit:
+	return err;
+
+}
+
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+
+static int
+dhd_doiovar(dhd_pub_t *dhd_pub, const bcm_iovar_t *vi, uint32 actionid, const char *name,
+            void *params, int plen, void *arg, uint len, int val_size)
+{
+	int bcmerror = 0;
+	int32 int_val = 0;
+	uint32 dhd_ver_len, bus_api_rev_len;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	DHD_TRACE(("%s: actionid = %d; name %s\n", __FUNCTION__, actionid, name));
+
+	if ((bcmerror = bcm_iovar_lencheck(vi, arg, len, IOV_ISSET(actionid))) != 0)
+		goto exit;
+
+	if (plen >= (int)sizeof(int_val))
+		bcopy(params, &int_val, sizeof(int_val));
+
+	switch (actionid) {
+	case IOV_GVAL(IOV_VERSION):
+		/* Need to have checked buffer length */
+		dhd_ver_len = sizeof(dhd_version) - 1;
+		bus_api_rev_len = strlen(bus_api_revision);
+		if (len > dhd_ver_len + bus_api_rev_len) {
+			bcmerror = memcpy_s((char *)arg, len, dhd_version, dhd_ver_len);
+			if (bcmerror != BCME_OK) {
+				break;
+			}
+			bcmerror = memcpy_s((char *)arg + dhd_ver_len, len - dhd_ver_len,
+				bus_api_revision, bus_api_rev_len);
+			if (bcmerror != BCME_OK) {
+				break;
+			}
+			*((char *)arg + dhd_ver_len + bus_api_rev_len) = '\0';
+		}
+		break;
+
+	case IOV_GVAL(IOV_WLMSGLEVEL):
+		printf("android_msg_level=0x%x\n", android_msg_level);
+		printf("config_msg_level=0x%x\n", config_msg_level);
+#if defined(WL_WIRELESS_EXT)
+		int_val = (int32)iw_msg_level;
+		bcopy(&int_val, arg, val_size);
+		printf("iw_msg_level=0x%x\n", iw_msg_level);
+#endif
+#ifdef WL_CFG80211
+		int_val = (int32)wl_dbg_level;
+		bcopy(&int_val, arg, val_size);
+		printf("cfg_msg_level=0x%x\n", wl_dbg_level);
+#endif
+		break;
+
+	case IOV_SVAL(IOV_WLMSGLEVEL):
+		if (int_val & DHD_ANDROID_VAL) {
+			android_msg_level = (uint)(int_val & 0xFFFF);
+			printf("android_msg_level=0x%x\n", android_msg_level);
+		}
+		if (int_val & DHD_CONFIG_VAL) {
+			config_msg_level = (uint)(int_val & 0xFFFF);
+			printf("config_msg_level=0x%x\n", config_msg_level);
+		}
+#if defined(WL_WIRELESS_EXT)
+		if (int_val & DHD_IW_VAL) {
+			iw_msg_level = (uint)(int_val & 0xFFFF);
+			printf("iw_msg_level=0x%x\n", iw_msg_level);
+		}
+#endif
+#ifdef WL_CFG80211
+		if (int_val & DHD_CFG_VAL) {
+			wl_cfg80211_enable_trace((u32)(int_val & 0xFFFF));
+		}
+#endif
+		break;
+
+	case IOV_GVAL(IOV_MSGLEVEL):
+		int_val = (int32)dhd_msg_level;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_MSGLEVEL):
+		dhd_msg_level = int_val;
+		break;
+
+#ifdef DHD_LOGLEVEL
+	case IOV_GVAL(IOV_LOGLEVEL):
+		dhd_loglevel_get((dhd_loglevel_data_t *)arg);
+		break;
+
+	case IOV_SVAL(IOV_LOGLEVEL):
+		dhd_loglevel_set((dhd_loglevel_data_t *)arg);
+		break;
+#endif /* DHD_LOGLEVEL */
+	case IOV_GVAL(IOV_BCMERRORSTR):
+		bcm_strncpy_s((char *)arg, len, bcmerrorstr(dhd_pub->bcmerror), BCME_STRLEN);
+		((char *)arg)[BCME_STRLEN - 1] = 0x00;
+		break;
+
+	case IOV_GVAL(IOV_BCMERROR):
+		int_val = (int32)dhd_pub->bcmerror;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+#ifndef BCMDBUS
+	case IOV_GVAL(IOV_WDTICK):
+		int_val = (int32)dhd_watchdog_ms;
+		bcopy(&int_val, arg, val_size);
+		break;
+#endif /* !BCMDBUS */
+
+	case IOV_SVAL(IOV_WDTICK):
+		if (!dhd_pub->up) {
+			bcmerror = BCME_NOTUP;
+			break;
+		}
+
+		dhd_watchdog_ms = (uint)int_val;
+
+		dhd_os_wd_timer(dhd_pub, (uint)int_val);
+		break;
+
+	case IOV_GVAL(IOV_DUMP):
+		if (dhd_dump(dhd_pub, arg, len) <= 0)
+			bcmerror = BCME_ERROR;
+		else
+			bcmerror = BCME_OK;
+		break;
+
+#ifndef BCMDBUS
+	case IOV_GVAL(IOV_DCONSOLE_POLL):
+		int_val = (int32)dhd_pub->dhd_console_ms;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DCONSOLE_POLL):
+		dhd_pub->dhd_console_ms = (uint)int_val;
+		break;
+
+#if defined(DHD_DEBUG)
+	case IOV_SVAL(IOV_CONS):
+		if (len > 0) {
+#ifdef CONSOLE_DPC
+			bcmerror = dhd_bus_txcons(dhd_pub, arg, len - 1);
+#else
+			bcmerror = dhd_bus_console_in(dhd_pub, arg, len - 1);
+#endif
+		}
+		break;
+#endif /* DHD_DEBUG */
+#endif /* !BCMDBUS */
+
+	case IOV_SVAL(IOV_RESET_COUNTS):
+		dhd_pub->tx_packets = dhd_pub->rx_packets = 0;
+		dhd_pub->tx_multicast = dhd_pub->rx_multicast = 0;
+		dhd_pub->tx_bytes = dhd_pub->rx_bytes = 0;
+		dhd_pub->tx_errors = dhd_pub->rx_errors = 0;
+		dhd_pub->tx_ctlpkts = dhd_pub->rx_ctlpkts = 0;
+		dhd_pub->tx_ctlerrs = dhd_pub->rx_ctlerrs = 0;
+		dhd_pub->tx_dropped = 0;
+		dhd_pub->rx_dropped = 0;
+		dhd_pub->tx_pktgetfail = 0;
+		dhd_pub->rx_pktgetfail = 0;
+		dhd_pub->rx_readahead_cnt = 0;
+		dhd_pub->tx_realloc = 0;
+		dhd_pub->wd_dpc_sched = 0;
+		dhd_pub->tx_big_packets = 0;
+		memset(&dhd_pub->dstats, 0, sizeof(dhd_pub->dstats));
+		dhd_bus_clearcounts(dhd_pub);
+#ifdef PROP_TXSTATUS
+		/* clear proptxstatus related counters */
+		dhd_wlfc_clear_counts(dhd_pub);
+#endif /* PROP_TXSTATUS */
+#if defined(DHD_LB_STATS)
+		DHD_LB_STATS_RESET(dhd_pub);
+#endif /* DHD_LB_STATS */
+		break;
+
+#ifdef BCMPERFSTATS
+	case IOV_GVAL(IOV_LOGDUMP): {
+		bcmdumplog((char*)arg, len);
+		break;
+	}
+
+	case IOV_SVAL(IOV_LOGCAL): {
+		bcmlog("Starting OSL_DELAY (%d usecs)", (uint)int_val, 0);
+		OSL_DELAY((uint)int_val);
+		bcmlog("Finished OSL_DELAY (%d usecs)", (uint)int_val, 0);
+		break;
+	}
+
+	case IOV_SVAL(IOV_LOGSTAMP): {
+		int int_val2;
+
+		if (plen >= 2 * sizeof(int)) {
+			bcopy((char *)params + sizeof(int_val), &int_val2, sizeof(int_val2));
+			bcmlog("User message %d %d", (uint)int_val, (uint)int_val2);
+		} else if (plen >= sizeof(int)) {
+			bcmlog("User message %d", (uint)int_val, 0);
+		} else {
+			bcmlog("User message", 0, 0);
+		}
+		break;
+	}
+#endif /* BCMPERFSTATS */
+
+	case IOV_GVAL(IOV_IOCTLTIMEOUT): {
+		int_val = (int32)dhd_os_get_ioctl_resp_timeout();
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+
+	case IOV_SVAL(IOV_IOCTLTIMEOUT): {
+		if (int_val <= 0)
+			bcmerror = BCME_BADARG;
+		else
+			dhd_os_set_ioctl_resp_timeout((unsigned int)int_val);
+		break;
+	}
+
+#ifdef PROP_TXSTATUS
+	case IOV_GVAL(IOV_PROPTXSTATUS_ENABLE): {
+		bool wlfc_enab = FALSE;
+		bcmerror = dhd_wlfc_get_enable(dhd_pub, &wlfc_enab);
+		if (bcmerror != BCME_OK)
+			goto exit;
+		int_val = wlfc_enab ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_PROPTXSTATUS_ENABLE): {
+		bool wlfc_enab = FALSE;
+		bcmerror = dhd_wlfc_get_enable(dhd_pub, &wlfc_enab);
+		if (bcmerror != BCME_OK)
+			goto exit;
+
+		/* wlfc is already set as desired */
+		if (wlfc_enab == (int_val == 0 ? FALSE : TRUE))
+			goto exit;
+
+		if (int_val == TRUE && disable_proptx) {
+			disable_proptx = 0;
+		}
+
+		if (int_val == TRUE) {
+			uint32 up;
+			bcmerror = dhd_wlfc_init(dhd_pub);
+			/* set WLC_UP to update wlfc credit */
+			if (dhd_wl_ioctl_cmd(dhd_pub, WLC_UP, &up, sizeof(uint32), TRUE, 0) < 0) {
+				DHD_ERROR(("%s: WLC_UP return err\n", __FUNCTION__));
+			}
+		} else
+			bcmerror = dhd_wlfc_deinit(dhd_pub);
+
+		break;
+	}
+	case IOV_GVAL(IOV_PROPTXSTATUS_MODE):
+		bcmerror = dhd_wlfc_get_mode(dhd_pub, &int_val);
+		if (bcmerror != BCME_OK)
+			goto exit;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PROPTXSTATUS_MODE):
+		dhd_wlfc_set_mode(dhd_pub, int_val);
+		break;
+#ifdef QMONITOR
+	case IOV_GVAL(IOV_QMON_TIME_THRES): {
+		int_val = dhd_qmon_thres(dhd_pub, FALSE, 0);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+
+	case IOV_SVAL(IOV_QMON_TIME_THRES): {
+		dhd_qmon_thres(dhd_pub, TRUE, int_val);
+		break;
+	}
+
+	case IOV_GVAL(IOV_QMON_TIME_PERCENT): {
+		int_val = dhd_qmon_getpercent(dhd_pub);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+#endif /* QMONITOR */
+
+	case IOV_GVAL(IOV_PROPTXSTATUS_MODULE_IGNORE):
+		bcmerror = dhd_wlfc_get_module_ignore(dhd_pub, &int_val);
+		if (bcmerror != BCME_OK)
+			goto exit;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PROPTXSTATUS_MODULE_IGNORE):
+		dhd_wlfc_set_module_ignore(dhd_pub, int_val);
+		break;
+
+	case IOV_GVAL(IOV_PROPTXSTATUS_CREDIT_IGNORE):
+		bcmerror = dhd_wlfc_get_credit_ignore(dhd_pub, &int_val);
+		if (bcmerror != BCME_OK)
+			goto exit;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PROPTXSTATUS_CREDIT_IGNORE):
+		dhd_wlfc_set_credit_ignore(dhd_pub, int_val);
+		break;
+
+	case IOV_GVAL(IOV_PROPTXSTATUS_TXSTATUS_IGNORE):
+		bcmerror = dhd_wlfc_get_txstatus_ignore(dhd_pub, &int_val);
+		if (bcmerror != BCME_OK)
+			goto exit;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PROPTXSTATUS_TXSTATUS_IGNORE):
+		dhd_wlfc_set_txstatus_ignore(dhd_pub, int_val);
+		break;
+
+	case IOV_GVAL(IOV_PROPTXSTATUS_RXPKT_CHK):
+		bcmerror = dhd_wlfc_get_rxpkt_chk(dhd_pub, &int_val);
+		if (bcmerror != BCME_OK)
+			goto exit;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PROPTXSTATUS_RXPKT_CHK):
+		dhd_wlfc_set_rxpkt_chk(dhd_pub, int_val);
+		break;
+
+#endif /* PROP_TXSTATUS */
+
+	case IOV_GVAL(IOV_BUS_TYPE):
+		/* The dhd application queries the driver to check if its usb or sdio.  */
+#ifdef BCMDBUS
+		int_val = BUS_TYPE_USB;
+#endif
+#ifdef BCMSDIO
+		int_val = BUS_TYPE_SDIO;
+#endif
+#ifdef PCIE_FULL_DONGLE
+		int_val = BUS_TYPE_PCIE;
+#endif
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_CHANGEMTU):
+		int_val &= 0xffff;
+		bcmerror = dhd_change_mtu(dhd_pub, int_val, 0);
+		break;
+
+	case IOV_GVAL(IOV_HOSTREORDER_FLOWS):
+	{
+		uint i = 0;
+		uint8 *ptr = (uint8 *)arg;
+		uint8 count = 0;
+
+		ptr++;
+		for (i = 0; i < WLHOST_REORDERDATA_MAXFLOWS; i++) {
+			if (dhd_pub->reorder_bufs[i] != NULL) {
+				*ptr = dhd_pub->reorder_bufs[i]->flow_id;
+				ptr++;
+				count++;
+			}
+		}
+		ptr = (uint8 *)arg;
+		*ptr = count;
+		break;
+	}
+#ifdef DHDTCPACK_SUPPRESS
+	case IOV_GVAL(IOV_TCPACK_SUPPRESS): {
+		int_val = (uint32)dhd_pub->tcpack_sup_mode;
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_TCPACK_SUPPRESS): {
+		bcmerror = dhd_tcpack_suppress_set(dhd_pub, (uint8)int_val);
+		break;
+	}
+#endif /* DHDTCPACK_SUPPRESS */
+#ifdef DHD_WMF
+	case IOV_GVAL(IOV_WMF_BSS_ENAB): {
+		uint32	bssidx;
+		dhd_wmf_t *wmf;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: wmf_bss_enable: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		wmf = dhd_wmf_conf(dhd_pub, bssidx);
+		int_val = wmf->wmf_enable ? 1 :0;
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_WMF_BSS_ENAB): {
+		/* Enable/Disable WMF */
+		uint32	bssidx;
+		dhd_wmf_t *wmf;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: wmf_bss_enable: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		ASSERT(val);
+		bcopy(val, &int_val, sizeof(uint32));
+		wmf = dhd_wmf_conf(dhd_pub, bssidx);
+		if (wmf->wmf_enable == int_val)
+			break;
+		if (int_val) {
+			/* Enable WMF */
+			if (dhd_wmf_instance_add(dhd_pub, bssidx) != BCME_OK) {
+				DHD_ERROR(("%s: Error in creating WMF instance\n",
+				__FUNCTION__));
+				break;
+			}
+			if (dhd_wmf_start(dhd_pub, bssidx) != BCME_OK) {
+				DHD_ERROR(("%s: Failed to start WMF\n", __FUNCTION__));
+				break;
+			}
+			wmf->wmf_enable = TRUE;
+		} else {
+			/* Disable WMF */
+			wmf->wmf_enable = FALSE;
+			dhd_wmf_stop(dhd_pub, bssidx);
+			dhd_wmf_instance_del(dhd_pub, bssidx);
+		}
+		break;
+	}
+	case IOV_GVAL(IOV_WMF_UCAST_IGMP):
+		int_val = dhd_pub->wmf_ucast_igmp ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_WMF_UCAST_IGMP):
+		if (dhd_pub->wmf_ucast_igmp == int_val)
+			break;
+
+		if (int_val >= OFF && int_val <= ON)
+			dhd_pub->wmf_ucast_igmp = int_val;
+		else
+			bcmerror = BCME_RANGE;
+		break;
+	case IOV_GVAL(IOV_WMF_MCAST_DATA_SENDUP):
+		int_val = dhd_wmf_mcast_data_sendup(dhd_pub, 0, FALSE, FALSE);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_WMF_MCAST_DATA_SENDUP):
+		dhd_wmf_mcast_data_sendup(dhd_pub, 0, TRUE, int_val);
+		break;
+
+#ifdef WL_IGMP_UCQUERY
+	case IOV_GVAL(IOV_WMF_UCAST_IGMP_QUERY):
+		int_val = dhd_pub->wmf_ucast_igmp_query ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_WMF_UCAST_IGMP_QUERY):
+		if (dhd_pub->wmf_ucast_igmp_query == int_val)
+			break;
+
+		if (int_val >= OFF && int_val <= ON)
+			dhd_pub->wmf_ucast_igmp_query = int_val;
+		else
+			bcmerror = BCME_RANGE;
+		break;
+#endif /* WL_IGMP_UCQUERY */
+#ifdef DHD_UCAST_UPNP
+	case IOV_GVAL(IOV_WMF_UCAST_UPNP):
+		int_val = dhd_pub->wmf_ucast_upnp ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_WMF_UCAST_UPNP):
+		if (dhd_pub->wmf_ucast_upnp == int_val)
+			break;
+
+		if (int_val >= OFF && int_val <= ON)
+			dhd_pub->wmf_ucast_upnp = int_val;
+		else
+			bcmerror = BCME_RANGE;
+		break;
+#endif /* DHD_UCAST_UPNP */
+
+	case IOV_GVAL(IOV_WMF_PSTA_DISABLE): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: ap isoalate: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		int_val = dhd_get_wmf_psta_disable(dhd_pub, bssidx);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+
+	case IOV_SVAL(IOV_WMF_PSTA_DISABLE): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: ap isolate: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		ASSERT(val);
+		bcopy(val, &int_val, sizeof(uint32));
+		dhd_set_wmf_psta_disable(dhd_pub, bssidx, int_val);
+		break;
+	}
+#endif /* DHD_WMF */
+
+#if defined(BCM_ROUTER_DHD)
+	case IOV_SVAL(IOV_TRAFFIC_MGMT_DWM): {
+			trf_mgmt_filter_list_t   *trf_mgmt_filter_list =
+				(trf_mgmt_filter_list_t *)(arg);
+			bcmerror = traffic_mgmt_add_dwm_filter(dhd_pub, trf_mgmt_filter_list, len);
+		}
+		break;
+#endif /* BCM_ROUTER_DHD */
+
+#ifdef DHD_L2_FILTER
+	case IOV_GVAL(IOV_DHCP_UNICAST): {
+		uint32 bssidx;
+		const char *val;
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_DHCP_UNICAST: bad parameterand name = %s\n",
+				__FUNCTION__, name));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = dhd_get_dhcp_unicast_status(dhd_pub, bssidx);
+		memcpy(arg, &int_val, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_DHCP_UNICAST): {
+		uint32	bssidx;
+		const char *val;
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_DHCP_UNICAST: bad parameterand name = %s\n",
+				__FUNCTION__, name));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		memcpy(&int_val, val, sizeof(int_val));
+		bcmerror = dhd_set_dhcp_unicast_status(dhd_pub, bssidx, int_val ? 1 : 0);
+		break;
+	}
+	case IOV_GVAL(IOV_BLOCK_PING): {
+		uint32 bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_BLOCK_PING: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = dhd_get_block_ping_status(dhd_pub, bssidx);
+		memcpy(arg, &int_val, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_BLOCK_PING): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_BLOCK_PING: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		memcpy(&int_val, val, sizeof(int_val));
+		bcmerror = dhd_set_block_ping_status(dhd_pub, bssidx, int_val ? 1 : 0);
+		break;
+	}
+	case IOV_GVAL(IOV_PROXY_ARP): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_PROXY_ARP: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = dhd_get_parp_status(dhd_pub, bssidx);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_PROXY_ARP): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_PROXY_ARP: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		bcopy(val, &int_val, sizeof(int_val));
+
+		/* Issue a iovar request to WL to update the proxy arp capability bit
+		 * in the Extended Capability IE of beacons/probe responses.
+		 */
+		bcmerror = dhd_iovar(dhd_pub, bssidx, "proxy_arp_advertise", val, sizeof(int_val),
+				NULL, 0, TRUE);
+		if (bcmerror == BCME_OK) {
+			dhd_set_parp_status(dhd_pub, bssidx, int_val ? 1 : 0);
+		}
+		break;
+	}
+	case IOV_GVAL(IOV_GRAT_ARP): {
+		uint32 bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_GRAT_ARP: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = dhd_get_grat_arp_status(dhd_pub, bssidx);
+		memcpy(arg, &int_val, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_GRAT_ARP): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_GRAT_ARP: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		memcpy(&int_val, val, sizeof(int_val));
+		bcmerror = dhd_set_grat_arp_status(dhd_pub, bssidx, int_val ? 1 : 0);
+		break;
+	}
+	case IOV_GVAL(IOV_BLOCK_TDLS): {
+		uint32 bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_BLOCK_TDLS: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		int_val = dhd_get_block_tdls_status(dhd_pub, bssidx);
+		memcpy(arg, &int_val, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_BLOCK_TDLS): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: IOV_BLOCK_TDLS: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		memcpy(&int_val, val, sizeof(int_val));
+		bcmerror = dhd_set_block_tdls_status(dhd_pub, bssidx, int_val ? 1 : 0);
+		break;
+	}
+#endif /* DHD_L2_FILTER */
+	case IOV_SVAL(IOV_DHD_IE): {
+		uint32	bssidx;
+		const char *val;
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+		uint8 ie_type;
+		bcm_tlv_t *qos_map_ie = NULL;
+		ie_setbuf_t *ie_getbufp = (ie_setbuf_t *)(arg+4);
+		ie_type = ie_getbufp->ie_buffer.ie_list[0].ie_data.id;
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: dhd ie: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+		qos_map_ie = (bcm_tlv_t *)(&(ie_getbufp->ie_buffer.ie_list[0].ie_data));
+		if (qos_map_ie != NULL && (ie_type == DOT11_MNG_QOS_MAP_ID)) {
+				bcmerror = dhd_set_qosmap_up_table(dhd_pub, bssidx, qos_map_ie);
+		}
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+		break;
+	}
+	case IOV_GVAL(IOV_AP_ISOLATE): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: ap isoalate: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		int_val = dhd_get_ap_isolate(dhd_pub, bssidx);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_AP_ISOLATE): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: ap isolate: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		ASSERT(val);
+		bcopy(val, &int_val, sizeof(uint32));
+		dhd_set_ap_isolate(dhd_pub, bssidx, int_val);
+		break;
+	}
+#ifdef DHD_PSTA
+	case IOV_GVAL(IOV_PSTA): {
+		int_val = dhd_get_psta_mode(dhd_pub);
+		bcopy(&int_val, arg, val_size);
+		break;
+		}
+	case IOV_SVAL(IOV_PSTA): {
+		if (int_val >= DHD_MODE_PSTA_DISABLED && int_val <= DHD_MODE_PSR) {
+			dhd_set_psta_mode(dhd_pub, int_val);
+		} else {
+			bcmerror = BCME_RANGE;
+		}
+		break;
+		}
+#endif /* DHD_PSTA */
+#ifdef DHD_WET
+	case IOV_GVAL(IOV_WET):
+		 int_val = dhd_get_wet_mode(dhd_pub);
+		 bcopy(&int_val, arg, val_size);
+		 break;
+
+	case IOV_SVAL(IOV_WET):
+		 if (int_val == 0 || int_val == 1) {
+			 dhd_set_wet_mode(dhd_pub, int_val);
+			 /* Delete the WET DB when disabled */
+			 if (!int_val) {
+				 dhd_wet_sta_delete_list(dhd_pub);
+			 }
+		 } else {
+			 bcmerror = BCME_RANGE;
+		 }
+				 break;
+	case IOV_SVAL(IOV_WET_HOST_IPV4):
+			dhd_set_wet_host_ipv4(dhd_pub, params, plen);
+			break;
+	case IOV_SVAL(IOV_WET_HOST_MAC):
+			dhd_set_wet_host_mac(dhd_pub, params, plen);
+		break;
+#endif /* DHD_WET */
+#ifdef DHD_MCAST_REGEN
+	case IOV_GVAL(IOV_MCAST_REGEN_BSS_ENABLE): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: mcast_regen_bss_enable: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		int_val = dhd_get_mcast_regen_bss_enable(dhd_pub, bssidx);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+
+	case IOV_SVAL(IOV_MCAST_REGEN_BSS_ENABLE): {
+		uint32	bssidx;
+		const char *val;
+
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: mcast_regen_bss_enable: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		ASSERT(val);
+		bcopy(val, &int_val, sizeof(uint32));
+		dhd_set_mcast_regen_bss_enable(dhd_pub, bssidx, int_val);
+		break;
+	}
+#endif /* DHD_MCAST_REGEN */
+
+	case IOV_GVAL(IOV_CFG80211_OPMODE): {
+		int_val = (int32)dhd_pub->op_mode;
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+		}
+	case IOV_SVAL(IOV_CFG80211_OPMODE): {
+		if (int_val <= 0)
+			bcmerror = BCME_BADARG;
+		else
+			dhd_pub->op_mode = int_val;
+		break;
+	}
+
+	case IOV_GVAL(IOV_ASSERT_TYPE):
+		int_val = g_assert_type;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_ASSERT_TYPE):
+		g_assert_type = (uint32)int_val;
+		break;
+
+#if defined(NDIS)
+	case IOV_GVAL(IOV_WAKEIND):
+		dhd_os_wakeind(dhd_pub, &int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+#endif /* NDIS */
+
+#if !defined(NDIS) && !defined(BCM_ROUTER_DHD)
+	case IOV_GVAL(IOV_LMTEST): {
+		*(uint32 *)arg = (uint32)lmtest;
+		break;
+	}
+
+	case IOV_SVAL(IOV_LMTEST): {
+		uint32 val = *(uint32 *)arg;
+		if (val > 50)
+			bcmerror = BCME_BADARG;
+		else {
+			lmtest = (uint)val;
+			DHD_ERROR(("%s: lmtest %s\n",
+				__FUNCTION__, (lmtest == FALSE)? "OFF" : "ON"));
+		}
+		break;
+	}
+#endif /* !NDIS && !BCM_ROUTER_DHD */
+#if defined(DHD_MACDBG) && defined(BCMDBG)
+	case IOV_GVAL(IOV_MACDBG_PD11REGS):
+		bcmerror = dhd_macdbg_pd11regs(dhd_pub, params, plen, arg, len);
+		break;
+	case IOV_GVAL(IOV_MACDBG_REGLIST):
+		bcmerror = dhd_macdbg_reglist(dhd_pub, arg, len);
+		break;
+	case IOV_GVAL(IOV_MACDBG_PSVMPMEMS):
+		bcmerror = dhd_macdbg_psvmpmems(dhd_pub, params, plen, arg, len);
+		break;
+#endif /* defined(DHD_MACDBG) && defined(BCMDBG) */
+
+#ifdef SHOW_LOGTRACE
+	case IOV_GVAL(IOV_DUMP_TRACE_LOG): {
+		trace_buf_info_t *trace_buf_info = (trace_buf_info_t *)arg;
+		dhd_dbg_ring_t *dbg_verbose_ring = NULL;
+
+		dbg_verbose_ring = dhd_dbg_get_ring_from_ring_id(dhd_pub, FW_VERBOSE_RING_ID);
+		if (dbg_verbose_ring == NULL) {
+			DHD_ERROR(("dbg_verbose_ring is NULL\n"));
+			bcmerror = BCME_UNSUPPORTED;
+			break;
+		}
+
+		if (trace_buf_info != NULL) {
+			bzero(trace_buf_info, sizeof(trace_buf_info_t));
+			dhd_dbg_read_ring_into_trace_buf(dbg_verbose_ring, trace_buf_info);
+		} else {
+			DHD_ERROR(("%s: arg is NULL\n", __FUNCTION__));
+			bcmerror = BCME_NOMEM;
+		}
+		break;
+	}
+#endif /* SHOW_LOGTRACE */
+#ifdef REPORT_FATAL_TIMEOUTS
+	case IOV_GVAL(IOV_SCAN_TO): {
+		dhd_get_scan_to_val(dhd_pub, (uint32 *)&int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_SCAN_TO): {
+		dhd_set_scan_to_val(dhd_pub, (uint32)int_val);
+		break;
+	}
+	case IOV_GVAL(IOV_JOIN_TO): {
+		dhd_get_join_to_val(dhd_pub, (uint32 *)&int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_JOIN_TO): {
+		dhd_set_join_to_val(dhd_pub, (uint32)int_val);
+		break;
+	}
+	case IOV_GVAL(IOV_CMD_TO): {
+		dhd_get_cmd_to_val(dhd_pub, (uint32 *)&int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_CMD_TO): {
+		dhd_set_cmd_to_val(dhd_pub, (uint32)int_val);
+		break;
+	}
+	case IOV_GVAL(IOV_OQS_TO): {
+		dhd_get_bus_to_val(dhd_pub, (uint32 *)&int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_OQS_TO): {
+		dhd_set_bus_to_val(dhd_pub, (uint32)int_val);
+		break;
+	}
+#endif /* REPORT_FATAL_TIMEOUTS */
+	case IOV_GVAL(IOV_DONGLE_TRAP_TYPE):
+		if (dhd_pub->dongle_trap_occured)
+			int_val = ltoh32(dhd_pub->last_trap_info.type);
+		else
+			int_val = 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_DONGLE_TRAP_INFO):
+	{
+		struct bcmstrbuf strbuf;
+		bcm_binit(&strbuf, arg, len);
+		if (dhd_pub->dongle_trap_occured == FALSE) {
+			bcm_bprintf(&strbuf, "no trap recorded\n");
+			break;
+		}
+#ifndef BCMDBUS
+		dhd_bus_dump_trap_info(dhd_pub->bus, &strbuf);
+#endif /* BCMDBUS */
+		break;
+	}
+#ifdef DHD_DEBUG
+#if defined(BCMSDIO) || defined(BCMPCIE)
+
+	case IOV_GVAL(IOV_BPADDR):
+		{
+			sdreg_t sdreg;
+			uint32 addr, size;
+
+			memcpy(&sdreg, params, sizeof(sdreg));
+
+			addr = sdreg.offset;
+			size = sdreg.func;
+
+			bcmerror = dhd_bus_readwrite_bp_addr(dhd_pub, addr, size,
+				(uint *)&int_val, TRUE);
+
+			memcpy(arg, &int_val, sizeof(int32));
+
+			break;
+		}
+
+	case IOV_SVAL(IOV_BPADDR):
+		{
+			sdreg_t sdreg;
+			uint32 addr, size;
+
+			memcpy(&sdreg, params, sizeof(sdreg));
+
+			addr = sdreg.offset;
+			size = sdreg.func;
+
+			bcmerror = dhd_bus_readwrite_bp_addr(dhd_pub, addr, size,
+				(uint *)&sdreg.value,
+				FALSE);
+
+			break;
+		}
+#endif /* BCMSDIO || BCMPCIE */
+#ifdef BCMPCIE
+	case IOV_SVAL(IOV_FLOW_RING_DEBUG):
+		{
+			bcmerror = dhd_flow_ring_debug(dhd_pub, arg, len);
+			break;
+		}
+#endif /* BCMPCIE */
+	case IOV_SVAL(IOV_MEM_DEBUG):
+		if (len > 0) {
+			bcmerror = dhd_mem_debug(dhd_pub, arg, len - 1);
+		}
+		break;
+#endif /* DHD_DEBUG */
+#if defined(DHD_LOG_DUMP)
+	case IOV_GVAL(IOV_LOG_DUMP):
+		{
+			dhd_prot_debug_info_print(dhd_pub);
+			dhd_log_dump_trigger(dhd_pub, CMD_DEFAULT);
+			break;
+		}
+#endif /* DHD_LOG_DUMP */
+
+	case IOV_GVAL(IOV_TPUT_TEST):
+		{
+			tput_test_t *tput_data = NULL;
+			if (params && plen >= sizeof(tput_test_t)) {
+				tput_data = (tput_test_t *)params;
+				bcmerror = dhd_tput_test(dhd_pub, tput_data);
+			} else {
+				DHD_ERROR(("%s: tput test - no input params ! \n", __FUNCTION__));
+				bcmerror = BCME_BADARG;
+			}
+			break;
+		}
+	case IOV_GVAL(IOV_DEBUG_BUF_DEST_STAT):
+		{
+			if (dhd_pub->debug_buf_dest_support) {
+				debug_buf_dest_stat_t *debug_buf_dest_stat =
+					(debug_buf_dest_stat_t *)arg;
+				memcpy(debug_buf_dest_stat, dhd_pub->debug_buf_dest_stat,
+					sizeof(dhd_pub->debug_buf_dest_stat));
+			} else {
+				bcmerror = BCME_DISABLED;
+			}
+			break;
+		}
+
+#if defined(DHD_SSSR_DUMP)
+	case IOV_GVAL(IOV_FIS_TRIGGER):
+		bcmerror = dhd_bus_fis_trigger(dhd_pub);
+
+		if (bcmerror == BCME_OK) {
+			bcmerror = dhd_bus_fis_dump(dhd_pub);
+		}
+
+		int_val = bcmerror;
+		bcopy(&int_val, arg, val_size);
+		break;
+#endif /* defined(DHD_SSSR_DUMP) */
+
+#ifdef DHD_DEBUG
+	case IOV_SVAL(IOV_INDUCE_ERROR): {
+		if (int_val >= DHD_INDUCE_ERROR_MAX) {
+			DHD_ERROR(("%s: Invalid command : %u\n", __FUNCTION__, (uint16)int_val));
+		} else {
+			dhd_pub->dhd_induce_error = (uint16)int_val;
+#ifdef BCMPCIE
+			if (dhd_pub->dhd_induce_error == DHD_INDUCE_BH_CBP_HANG) {
+				dhdpcie_induce_cbp_hang(dhd_pub);
+			}
+			if (dhd_pub->dhd_induce_error == DHD_INDUCE_PCIE_LINK_DOWN) {
+				dhd_bus_set_linkdown(dhd_pub, TRUE);
+#ifdef OEM_ANDROID
+				dhd_pub->hang_reason = HANG_REASON_PCIE_LINK_DOWN_EP_DETECT;
+				dhd_os_send_hang_message(dhd_pub);
+#endif /* OEM_ANDROID */
+			}
+#endif /* BCMPCIE */
+		}
+		break;
+	}
+#endif /* DHD_DEBUG */
+#ifdef WL_IFACE_MGMT_CONF
+#ifdef WL_CFG80211
+#ifdef WL_NANP2P
+	case IOV_GVAL(IOV_CONC_DISC): {
+		int_val = wl_cfg80211_get_iface_conc_disc(
+			dhd_linux_get_primary_netdev(dhd_pub));
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+	case IOV_SVAL(IOV_CONC_DISC): {
+		bcmerror = wl_cfg80211_set_iface_conc_disc(
+			dhd_linux_get_primary_netdev(dhd_pub), (uint8)int_val);
+		break;
+	}
+#endif /* WL_NANP2P */
+#ifdef WL_IFACE_MGMT
+	case IOV_GVAL(IOV_IFACE_POLICY): {
+		int_val = wl_cfg80211_get_iface_policy(
+		dhd_linux_get_primary_netdev(dhd_pub));
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+	case IOV_SVAL(IOV_IFACE_POLICY): {
+		bcmerror = wl_cfg80211_set_iface_policy(
+			dhd_linux_get_primary_netdev(dhd_pub),
+			arg, len);
+		break;
+	}
+#endif /* WL_IFACE_MGMT */
+#endif /* WL_CFG80211 */
+#endif /* WL_IFACE_MGMT_CONF */
+#ifdef RTT_GEOFENCE_CONT
+#if defined(RTT_SUPPORT) && defined(WL_NAN)
+	case IOV_GVAL(IOV_RTT_GEOFENCE_TYPE_OVRD): {
+		bool enable = 0;
+		dhd_rtt_get_geofence_cont_ind(dhd_pub, &enable);
+		int_val = enable ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_RTT_GEOFENCE_TYPE_OVRD): {
+		bool enable = *(bool *)arg;
+		dhd_rtt_set_geofence_cont_ind(dhd_pub, enable);
+		break;
+	}
+#endif /* RTT_SUPPORT && WL_NAN */
+#endif /* RTT_GEOFENCE_CONT */
+	case IOV_GVAL(IOV_FW_VBS): {
+		*(uint32 *)arg = (uint32)dhd_dbg_get_fwverbose(dhd_pub);
+		break;
+	}
+
+	case IOV_SVAL(IOV_FW_VBS): {
+		if (int_val < 0) {
+			int_val = 0;
+		}
+		dhd_dbg_set_fwverbose(dhd_pub, (uint32)int_val);
+		break;
+	}
+
+#ifdef DHD_TX_PROFILE
+	case IOV_SVAL(IOV_TX_PROFILE_TAG):
+	{
+		/* note: under the current implementation only one type of packet may be
+		 * tagged per profile
+		 */
+		const dhd_tx_profile_protocol_t *protocol = NULL;
+		/* for example, we might have a profile of profile_index 6, but at
+		 * offset 2 from dhd_pub->protocol_filters.
+		 */
+		uint8 offset;
+
+		if (params == NULL) {
+			bcmerror = BCME_ERROR;
+			break;
+		}
+
+		protocol = (dhd_tx_profile_protocol_t *)params;
+
+		/* validate */
+		if (protocol->version != DHD_TX_PROFILE_VERSION) {
+			bcmerror = BCME_VERSION;
+			break;
+		}
+		if (protocol->profile_index > DHD_MAX_PROFILE_INDEX) {
+			DHD_ERROR(("%s:\tprofile index must be between 0 and %d\n",
+					__FUNCTION__, DHD_MAX_PROFILE_INDEX));
+			bcmerror = BCME_RANGE;
+			break;
+		}
+		if (protocol->layer != DHD_TX_PROFILE_DATA_LINK_LAYER && protocol->layer
+				!= DHD_TX_PROFILE_NETWORK_LAYER) {
+			DHD_ERROR(("%s:\tlayer must be %d or %d\n", __FUNCTION__,
+					DHD_TX_PROFILE_DATA_LINK_LAYER,
+					DHD_TX_PROFILE_NETWORK_LAYER));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		if (protocol->protocol_number > __UINT16_MAX__) {
+			DHD_ERROR(("%s:\tprotocol number must be <= %d\n", __FUNCTION__,
+					__UINT16_MAX__));
+			bcmerror = BCME_BADLEN;
+			break;
+		}
+
+		/* find the dhd_tx_profile_protocol_t */
+		for (offset = 0; offset < dhd_pub->num_profiles; offset++) {
+			if (dhd_pub->protocol_filters[offset].profile_index ==
+					protocol->profile_index) {
+				break;
+			}
+		}
+
+		if (offset >= DHD_MAX_PROFILES) {
+#if DHD_MAX_PROFILES > 1
+			DHD_ERROR(("%s:\tonly %d profiles supported at present\n",
+					__FUNCTION__, DHD_MAX_PROFILES));
+#else /* DHD_MAX_PROFILES > 1 */
+			DHD_ERROR(("%s:\tonly %d profile supported at present\n",
+					__FUNCTION__, DHD_MAX_PROFILES));
+			DHD_ERROR(("%s:\tthere is a profile of index %d\n", __FUNCTION__,
+					dhd_pub->protocol_filters->profile_index));
+#endif /* DHD_MAX_PROFILES > 1 */
+			bcmerror = BCME_NOMEM;
+			break;
+		}
+
+		/* memory already allocated in dhd_attach; just assign the value */
+		dhd_pub->protocol_filters[offset] = *protocol;
+
+		if (offset >= dhd_pub->num_profiles) {
+			dhd_pub->num_profiles = offset + 1;
+		}
+
+		break;
+	}
+
+	case IOV_SVAL(IOV_TX_PROFILE_ENABLE):
+		dhd_pub->tx_profile_enab = int_val ? TRUE : FALSE;
+		break;
+
+	case IOV_GVAL(IOV_TX_PROFILE_ENABLE):
+		int_val = dhd_pub->tx_profile_enab;
+		bcmerror = memcpy_s(arg, val_size, &int_val, sizeof(int_val));
+		break;
+
+	case IOV_SVAL(IOV_TX_PROFILE_DUMP):
+	{
+		const dhd_tx_profile_protocol_t *protocol = NULL;
+		uint8 offset;
+
+		for (offset = 0; offset < dhd_pub->num_profiles; offset++) {
+			if (dhd_pub->protocol_filters[offset].profile_index == int_val) {
+				protocol = &(dhd_pub->protocol_filters[offset]);
+				break;
+			}
+		}
+
+		if (protocol == NULL) {
+			DHD_ERROR(("%s:\tno profile with index %d\n", __FUNCTION__,
+					int_val));
+			bcmerror = BCME_ERROR;
+			break;
+		}
+
+		DHD_CONS_ONLY(("%s:\ttx_profile %s: %d\n", __FUNCTION__,
+			"profile_index", protocol->profile_index));
+		DHD_CONS_ONLY(("%s:\ttx_profile %s: %d\n", __FUNCTION__,
+			"layer", protocol->layer));
+		DHD_CONS_ONLY(("%s:\ttx_profile %s: %d\n", __FUNCTION__,
+			"protocol_number", protocol->protocol_number));
+		DHD_CONS_ONLY(("%s:\ttx_profile %s: %d\n", __FUNCTION__,
+			"src_port", protocol->src_port));
+		DHD_CONS_ONLY(("%s:\ttx_profile %s: %d\n", __FUNCTION__,
+			"dest_port", protocol->dest_port));
+
+		break;
+	}
+#endif /* defined(DHD_TX_PROFILE) */
+
+	case IOV_GVAL(IOV_CHECK_TRAP_ROT): {
+		int_val = dhd_pub->check_trap_rot? 1 : 0;
+		(void)memcpy_s(arg, val_size, &int_val, sizeof(int_val));
+		break;
+	}
+	case IOV_SVAL(IOV_CHECK_TRAP_ROT): {
+		dhd_pub->check_trap_rot = *(bool *)arg;
+		break;
+	}
+
+	case IOV_GVAL(IOV_COUNTERS):
+		if (dhd_counters(dhd_pub, arg, len) <= 0)
+			bcmerror = BCME_ERROR;
+		else
+			bcmerror = BCME_OK;
+		break;
+#ifdef BCMPCIE
+	case IOV_GVAL(IOV_DUMP_FLOWRINGS):
+		if (dhd_dump_flowrings(dhd_pub, arg, len) <= 0)
+			bcmerror = BCME_ERROR;
+		else
+			bcmerror = BCME_OK;
+		break;
+#endif /* BCMPCIE */
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+	case IOV_GVAL(IOV_SAR_MODE):
+		int_val = dhd_pub->dhd_sar_mode;
+		(void)memcpy_s(arg, val_size, &int_val, sizeof(int_val));
+		break;
+	case IOV_SVAL(IOV_SAR_MODE):
+		dhd_sar_set(dhd_pub, int_val);
+		break;
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+
+#ifdef CSI_SUPPORT
+	case IOV_GVAL(IOV_CSI_CONFIG):
+		bcmerror = dhd_csi_config(dhd_pub, arg, val_size, FALSE);
+		break;
+	case IOV_SVAL(IOV_CSI_CONFIG):
+		bcmerror = dhd_csi_config(dhd_pub, arg, val_size, TRUE);
+		break;
+#endif /* CSI_SUPPORT */
+#ifdef WLEASYMESH
+	case IOV_SVAL(IOV_1905_AL_UCAST): {
+		uint32  bssidx;
+		const char *val;
+		uint8 ea[6] = {0};
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: 1905_al_ucast: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		bcopy(val, ea, ETHER_ADDR_LEN);
+		printf("IOV_1905_AL_UCAST:" MACDBG "\n", MAC2STRDBG(ea));
+		bcmerror = dhd_set_1905_almac(dhd_pub, bssidx, ea, FALSE);
+		break;
+	}
+	case IOV_GVAL(IOV_1905_AL_UCAST): {
+		uint32  bssidx;
+		const char *val;
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: 1905_al_ucast: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		bcmerror = dhd_get_1905_almac(dhd_pub, bssidx, arg, FALSE);
+		break;
+	}
+	case IOV_SVAL(IOV_1905_AL_MCAST): {
+		uint32  bssidx;
+		const char *val;
+		uint8 ea[6] = {0};
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: 1905_al_mcast: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		bcopy(val, ea, ETHER_ADDR_LEN);
+		printf("IOV_1905_AL_MCAST:" MACDBG "\n", MAC2STRDBG(ea));
+		bcmerror = dhd_set_1905_almac(dhd_pub, bssidx, ea, TRUE);
+		break;
+	}
+	case IOV_GVAL(IOV_1905_AL_MCAST): {
+		uint32  bssidx;
+		const char *val;
+		if (dhd_iovar_parse_bssidx(dhd_pub, (char *)name, &bssidx, &val) != BCME_OK) {
+			DHD_ERROR(("%s: 1905_al_mcast: bad parameter\n", __FUNCTION__));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		bcmerror = dhd_get_1905_almac(dhd_pub, bssidx, arg, TRUE);
+		break;
+	}
+#endif /* WLEASYMESH */
+
+	default:
+		bcmerror = BCME_UNSUPPORTED;
+		break;
+	}
+
+exit:
+	DHD_TRACE(("%s: actionid %d, bcmerror %d\n", __FUNCTION__, actionid, bcmerror));
+	return bcmerror;
+}
+
+#ifdef BCMDONGLEHOST
+/* Store the status of a connection attempt for later retrieval by an iovar */
+void
+dhd_store_conn_status(uint32 event, uint32 status, uint32 reason)
+{
+	/* Do not overwrite a WLC_E_PRUNE with a WLC_E_SET_SSID
+	 * because an encryption/rsn mismatch results in both events, and
+	 * the important information is in the WLC_E_PRUNE.
+	 */
+	if (!(event == WLC_E_SET_SSID && status == WLC_E_STATUS_FAIL &&
+	      dhd_conn_event == WLC_E_PRUNE)) {
+		dhd_conn_event = event;
+		dhd_conn_status = status;
+		dhd_conn_reason = reason;
+	}
+}
+#else
+#error "BCMDONGLEHOST not defined"
+#endif /* BCMDONGLEHOST */
+
+bool
+dhd_prec_enq(dhd_pub_t *dhdp, struct pktq *q, void *pkt, int prec)
+{
+	void *p;
+	int eprec = -1;		/* precedence to evict from */
+	bool discard_oldest;
+
+	/* Fast case, precedence queue is not full and we are also not
+	 * exceeding total queue length
+	 */
+	if (!pktqprec_full(q, prec) && !pktq_full(q)) {
+		pktq_penq(q, prec, pkt);
+		return TRUE;
+	}
+
+	/* Determine precedence from which to evict packet, if any */
+	if (pktqprec_full(q, prec))
+		eprec = prec;
+	else if (pktq_full(q)) {
+		p = pktq_peek_tail(q, &eprec);
+		ASSERT(p);
+		if (eprec > prec || eprec < 0)
+			return FALSE;
+	}
+
+	/* Evict if needed */
+	if (eprec >= 0) {
+		/* Detect queueing to unconfigured precedence */
+		ASSERT(!pktqprec_empty(q, eprec));
+		discard_oldest = AC_BITMAP_TST(dhdp->wme_dp, eprec);
+		if (eprec == prec && !discard_oldest)
+			return FALSE;		/* refuse newer (incoming) packet */
+		/* Evict packet according to discard policy */
+		p = discard_oldest ? pktq_pdeq(q, eprec) : pktq_pdeq_tail(q, eprec);
+		ASSERT(p);
+#ifdef DHDTCPACK_SUPPRESS
+		if (dhd_tcpack_check_xmit(dhdp, p) == BCME_ERROR) {
+			DHD_ERROR(("%s %d: tcpack_suppress ERROR!!! Stop using it\n",
+				__FUNCTION__, __LINE__));
+			dhd_tcpack_suppress_set(dhdp, TCPACK_SUP_OFF);
+		}
+#endif /* DHDTCPACK_SUPPRESS */
+		PKTFREE(dhdp->osh, p, TRUE);
+	}
+
+	/* Enqueue */
+	p = pktq_penq(q, prec, pkt);
+	ASSERT(p);
+
+	return TRUE;
+}
+
+/*
+ * Functions to drop proper pkts from queue:
+ *	If one pkt in queue is non-fragmented, drop first non-fragmented pkt only
+ *	If all pkts in queue are all fragmented, find and drop one whole set fragmented pkts
+ *	If can't find pkts matching upper 2 cases, drop first pkt anyway
+ */
+bool
+dhd_prec_drop_pkts(dhd_pub_t *dhdp, struct pktq *pq, int prec, f_droppkt_t fn)
+{
+	struct pktq_prec *q = NULL;
+	void *p, *prev = NULL, *next = NULL, *first = NULL, *last = NULL, *prev_first = NULL;
+	pkt_frag_t frag_info;
+
+	ASSERT(dhdp && pq);
+	ASSERT(prec >= 0 && prec < pq->num_prec);
+
+	q = &pq->q[prec];
+	p = q->head;
+
+	if (p == NULL)
+		return FALSE;
+
+	while (p) {
+		frag_info = pkt_frag_info(dhdp->osh, p);
+		if (frag_info == DHD_PKT_FRAG_NONE) {
+			break;
+		} else if (frag_info == DHD_PKT_FRAG_FIRST) {
+			if (first) {
+				/* No last frag pkt, use prev as last */
+				last = prev;
+				break;
+			} else {
+				first = p;
+				prev_first = prev;
+			}
+		} else if (frag_info == DHD_PKT_FRAG_LAST) {
+			if (first) {
+				last = p;
+				break;
+			}
+		}
+
+		prev = p;
+		p = PKTLINK(p);
+	}
+
+	if ((p == NULL) || ((frag_info != DHD_PKT_FRAG_NONE) && !(first && last))) {
+		/* Not found matching pkts, use oldest */
+		prev = NULL;
+		p = q->head;
+		frag_info = 0;
+	}
+
+	if (frag_info == DHD_PKT_FRAG_NONE) {
+		first = last = p;
+		prev_first = prev;
+	}
+
+	p = first;
+	while (p) {
+		next = PKTLINK(p);
+		q->n_pkts--;
+		pq->n_pkts_tot--;
+
+#ifdef WL_TXQ_STALL
+		q->dequeue_count++;
+#endif
+
+		PKTSETLINK(p, NULL);
+
+		if (fn)
+			fn(dhdp, prec, p, TRUE);
+
+		if (p == last)
+			break;
+
+		p = next;
+	}
+
+	if (prev_first == NULL) {
+		if ((q->head = next) == NULL)
+			q->tail = NULL;
+	} else {
+		PKTSETLINK(prev_first, next);
+		if (!next)
+			q->tail = prev_first;
+	}
+
+	return TRUE;
+}
+
+static int
+dhd_iovar_op(dhd_pub_t *dhd_pub, const char *name,
+	void *params, int plen, void *arg, uint len, bool set)
+{
+	int bcmerror = 0;
+	uint val_size;
+	const bcm_iovar_t *vi = NULL;
+	uint32 actionid;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(name);
+
+	/* Get MUST have return space */
+	ASSERT(set || (arg && len));
+
+	/* Set does NOT take qualifiers */
+	ASSERT(!set || (!params && !plen));
+
+	if ((vi = bcm_iovar_lookup(dhd_iovars, name)) == NULL) {
+		bcmerror = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	DHD_CTL(("%s: %s %s, len %d plen %d\n", __FUNCTION__,
+		name, (set ? "set" : "get"), len, plen));
+
+	/* set up 'params' pointer in case this is a set command so that
+	 * the convenience int and bool code can be common to set and get
+	 */
+	if (params == NULL) {
+		params = arg;
+		plen = len;
+	}
+
+	if (vi->type == IOVT_VOID)
+		val_size = 0;
+	else if (vi->type == IOVT_BUFFER)
+		val_size = len;
+	else
+		/* all other types are integer sized */
+		val_size = sizeof(int);
+
+	actionid = set ? IOV_SVAL(vi->varid) : IOV_GVAL(vi->varid);
+
+	bcmerror = dhd_doiovar(dhd_pub, vi, actionid, name, params, plen, arg, len, val_size);
+
+exit:
+	return bcmerror;
+}
+
+int
+dhd_ioctl(dhd_pub_t * dhd_pub, dhd_ioctl_t *ioc, void *buf, uint buflen)
+{
+	int bcmerror = 0;
+	unsigned long flags;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (!buf) {
+		return BCME_BADARG;
+	}
+
+	dhd_os_dhdiovar_lock(dhd_pub);
+	switch (ioc->cmd) {
+		case DHD_GET_MAGIC:
+			if (buflen < sizeof(int))
+				bcmerror = BCME_BUFTOOSHORT;
+			else
+				*(int*)buf = DHD_IOCTL_MAGIC;
+			break;
+
+		case DHD_GET_VERSION:
+			if (buflen < sizeof(int))
+				bcmerror = BCME_BUFTOOSHORT;
+			else
+				*(int*)buf = DHD_IOCTL_VERSION;
+			break;
+
+		case DHD_GET_VAR:
+		case DHD_SET_VAR:
+			{
+				char *arg;
+				uint arglen;
+
+				DHD_LINUX_GENERAL_LOCK(dhd_pub, flags);
+				if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhd_pub) &&
+					bcmstricmp((char *)buf, "devreset")) {
+					/* In platforms like FC19, the FW download is done via IOCTL
+					 * and should not return error for IOCTLs fired before FW
+					 * Download is done
+					 */
+					if (dhd_fw_download_status(dhd_pub) == FW_DOWNLOAD_DONE) {
+						DHD_ERROR(("%s: return as fw_download_status=%d\n",
+							__FUNCTION__,
+							dhd_fw_download_status(dhd_pub)));
+						DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+						dhd_os_dhdiovar_unlock(dhd_pub);
+						return -ENODEV;
+					}
+				}
+				DHD_BUS_BUSY_SET_IN_DHD_IOVAR(dhd_pub);
+				DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+
+				DHD_LINUX_GENERAL_LOCK(dhd_pub, flags);
+				if (DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(dhd_pub)) {
+					/* If Suspend/Resume is tested via pcie_suspend IOVAR
+					 * then continue to execute the IOVAR, return from here for
+					 * other IOVARs, also include pciecfgreg and devreset to go
+					 * through.
+					 */
+					if (bcmstricmp((char *)buf, "pcie_suspend") &&
+					    bcmstricmp((char *)buf, "pciecfgreg") &&
+					    bcmstricmp((char *)buf, "devreset") &&
+					    bcmstricmp((char *)buf, "sdio_suspend")) {
+						DHD_ERROR(("%s: bus is in suspend(%d)"
+							"or suspending(0x%x) state\n",
+							__FUNCTION__, dhd_pub->busstate,
+							dhd_pub->dhd_bus_busy_state));
+						DHD_BUS_BUSY_CLEAR_IN_DHD_IOVAR(dhd_pub);
+						dhd_os_busbusy_wake(dhd_pub);
+						DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+						dhd_os_dhdiovar_unlock(dhd_pub);
+						return -ENODEV;
+					}
+				}
+				/* During devreset ioctl, we call dhdpcie_advertise_bus_cleanup,
+				 * which will wait for all the busy contexts to get over for
+				 * particular time and call ASSERT if timeout happens. As during
+				 * devreset ioctal, we made DHD_BUS_BUSY_SET_IN_DHD_IOVAR,
+				 * to avoid ASSERT, clear the IOCTL busy state. "devreset" ioctl is
+				 * not used in Production platforms but only used in FC19 setups.
+				 */
+				if (!bcmstricmp((char *)buf, "devreset") ||
+#ifdef BCMPCIE
+				    (dhd_bus_is_multibp_capable(dhd_pub->bus) &&
+				    !bcmstricmp((char *)buf, "dwnldstate")) ||
+#endif /* BCMPCIE */
+				    FALSE)
+				{
+					DHD_BUS_BUSY_CLEAR_IN_DHD_IOVAR(dhd_pub);
+				}
+				DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+
+				/* scan past the name to any arguments */
+				for (arg = buf, arglen = buflen; *arg && arglen; arg++, arglen--)
+					;
+
+				if (arglen == 0 || *arg) {
+					bcmerror = BCME_BUFTOOSHORT;
+					goto unlock_exit;
+				}
+
+				/* account for the NUL terminator */
+				arg++, arglen--;
+				/* call with the appropriate arguments */
+				if (ioc->cmd == DHD_GET_VAR) {
+					bcmerror = dhd_iovar_op(dhd_pub, buf, arg, arglen,
+							buf, buflen, IOV_GET);
+				} else {
+					bcmerror = dhd_iovar_op(dhd_pub, buf, NULL, 0,
+							arg, arglen, IOV_SET);
+				}
+				if (bcmerror != BCME_UNSUPPORTED) {
+					goto unlock_exit;
+				}
+
+				/* not in generic table, try protocol module */
+				if (ioc->cmd == DHD_GET_VAR) {
+					bcmerror = dhd_prot_iovar_op(dhd_pub, buf, arg,
+							arglen, buf, buflen, IOV_GET);
+				} else {
+					bcmerror = dhd_prot_iovar_op(dhd_pub, buf,
+							NULL, 0, arg, arglen, IOV_SET);
+				}
+				if (bcmerror != BCME_UNSUPPORTED) {
+					goto unlock_exit;
+				}
+
+				/* if still not found, try bus module */
+				if (ioc->cmd == DHD_GET_VAR) {
+					bcmerror = dhd_bus_iovar_op(dhd_pub, buf,
+							arg, arglen, buf, buflen, IOV_GET);
+				} else {
+					bcmerror = dhd_bus_iovar_op(dhd_pub, buf,
+							NULL, 0, arg, arglen, IOV_SET);
+				}
+				if (bcmerror != BCME_UNSUPPORTED) {
+					goto unlock_exit;
+				}
+
+			}
+			goto unlock_exit;
+
+		default:
+			bcmerror = BCME_UNSUPPORTED;
+	}
+	dhd_os_dhdiovar_unlock(dhd_pub);
+	return bcmerror;
+
+unlock_exit:
+	DHD_LINUX_GENERAL_LOCK(dhd_pub, flags);
+	DHD_BUS_BUSY_CLEAR_IN_DHD_IOVAR(dhd_pub);
+	dhd_os_busbusy_wake(dhd_pub);
+	DHD_LINUX_GENERAL_UNLOCK(dhd_pub, flags);
+	dhd_os_dhdiovar_unlock(dhd_pub);
+	return bcmerror;
+}
+
+#ifdef SHOW_EVENTS
+
+static void
+wl_show_roam_event(dhd_pub_t *dhd_pub, uint status, uint datalen,
+	const char *event_name, char *eabuf, void *event_data)
+{
+#ifdef REPORT_FATAL_TIMEOUTS
+	OSL_ATOMIC_SET(dhd_pub->osh, &dhd_pub->set_ssid_rcvd, TRUE);
+	dhd_clear_join_error(dhd_pub, WLC_SSID_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+	if (status == WLC_E_STATUS_SUCCESS) {
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+	} else {
+#ifdef REPORT_FATAL_TIMEOUTS
+		/*
+		 * For secure join if WLC_E_SET_SSID returns with any failure case,
+		 * donot expect WLC_E_PSK_SUP. So clear the mask.
+		 */
+		dhd_clear_join_error(dhd_pub, WLC_WPA_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		if (status == WLC_E_STATUS_FAIL) {
+			DHD_EVENT(("MACEVENT: %s, failed status %d\n", event_name, status));
+		} else if (status == WLC_E_STATUS_NO_NETWORKS) {
+			if (datalen) {
+				uint8 id = *((uint8 *)event_data);
+				if (id != DOT11_MNG_PROPR_ID) {
+					wl_roam_event_t *roam_data =
+						(wl_roam_event_t *)event_data;
+					bcm_xtlv_t *tlv = (bcm_xtlv_t *)roam_data->xtlvs;
+					if (tlv->id == WLC_ROAM_NO_NETWORKS_TLV_ID) {
+						uint32 *fail_reason = (uint32 *)tlv->data;
+						switch (*fail_reason) {
+							case WLC_E_REASON_NO_NETWORKS:
+								DHD_EVENT(("MACEVENT: %s,"
+									" no networks found\n",
+									event_name));
+								break;
+							case WLC_E_REASON_NO_NETWORKS_BY_SCORE:
+								DHD_EVENT(("MACEVENT: %s,"
+								" no networks found by score\n",
+									event_name));
+								break;
+							default:
+								DHD_ERROR(("MACEVENT: %s,"
+								" unknown fail reason 0x%x\n",
+									event_name,
+									*fail_reason));
+								ASSERT(0);
+						}
+					} else {
+						DHD_EVENT(("MACEVENT: %s,"
+							" no networks found\n",
+							event_name));
+					}
+				} else {
+					DHD_EVENT(("MACEVENT: %s,"
+						" no networks found\n",
+						event_name));
+				}
+			} else {
+				DHD_EVENT(("MACEVENT: %s, no networks found\n",
+					event_name));
+			}
+		} else {
+			DHD_EVENT(("MACEVENT: %s, unexpected status %d\n",
+				event_name, (int)status));
+		}
+	}
+}
+
+static void
+wl_show_roam_cache_update_event(const char *name, uint status,
+	uint reason, uint datalen, void *event_data)
+{
+	wlc_roam_cache_update_event_t *cache_update;
+	uint16 len_of_tlvs;
+	void *val_tlv_ptr;
+	bcm_xtlv_t *val_xtlv;
+	char ntoa_buf[ETHER_ADDR_STR_LEN];
+	uint idx;
+	const char* reason_name = NULL;
+	const char* status_name = NULL;
+	static struct {
+		uint event;
+		const char *event_name;
+	} reason_names[] = {
+		{WLC_E_REASON_INITIAL_ASSOC, "INITIAL ASSOCIATION"},
+		{WLC_E_REASON_LOW_RSSI, "LOW_RSSI"},
+		{WLC_E_REASON_DEAUTH, "RECEIVED DEAUTHENTICATION"},
+		{WLC_E_REASON_DISASSOC, "RECEIVED DISASSOCATION"},
+		{WLC_E_REASON_BCNS_LOST, "BEACONS LOST"},
+		{WLC_E_REASON_BETTER_AP, "BETTER AP FOUND"},
+		{WLC_E_REASON_MINTXRATE, "STUCK AT MIN TX RATE"},
+		{WLC_E_REASON_BSSTRANS_REQ, "REQUESTED ROAM"},
+		{WLC_E_REASON_TXFAIL, "TOO MANY TXFAILURES"}
+	};
+
+	static struct {
+		uint event;
+		const char *event_name;
+	} status_names[] = {
+		{WLC_E_STATUS_SUCCESS, "operation was successful"},
+		{WLC_E_STATUS_FAIL, "operation failed"},
+		{WLC_E_STATUS_TIMEOUT, "operation timed out"},
+		{WLC_E_STATUS_NO_NETWORKS, "failed due to no matching network found"},
+		{WLC_E_STATUS_ABORT, "operation was aborted"},
+		{WLC_E_STATUS_NO_ACK, "protocol failure: packet not ack'd"},
+		{WLC_E_STATUS_UNSOLICITED, "AUTH or ASSOC packet was unsolicited"},
+		{WLC_E_STATUS_ATTEMPT, "attempt to assoc to an auto auth configuration"},
+		{WLC_E_STATUS_PARTIAL, "scan results are incomplete"},
+		{WLC_E_STATUS_NEWSCAN, "scan aborted by another scan"},
+		{WLC_E_STATUS_NEWASSOC, "scan aborted due to assoc in progress"},
+		{WLC_E_STATUS_11HQUIET, "802.11h quiet period started"},
+		{WLC_E_STATUS_SUPPRESS, "user disabled scanning"},
+		{WLC_E_STATUS_NOCHANS, "no allowable channels to scan"},
+		{WLC_E_STATUS_CS_ABORT, "abort channel select"},
+		{WLC_E_STATUS_ERROR, "request failed due to error"},
+		{WLC_E_STATUS_INVALID, "Invalid status code"}
+	};
+
+	UNUSED_PARAMETER(ntoa_buf);
+	UNUSED_PARAMETER(reason_name);
+	UNUSED_PARAMETER(status_name);
+
+	switch (reason) {
+	case WLC_ROAM_CACHE_UPDATE_NEW_ROAM_CACHE:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is new roam cache\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_JOIN:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is start of join\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_RSSI_DELTA:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is delta in rssi\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_MOTION_RSSI_DELTA:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is motion delta in rssi\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_CHANNEL_MISS:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is missed channel\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_START_SPLIT_SCAN:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is start of split scan\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_START_FULL_SCAN:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is start of full scan\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_INIT_ASSOC:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is init association\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_FULL_SCAN_FAILED:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is failure in full scan\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_NO_AP_FOUND:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is empty scan result\n", status));
+		break;
+	case WLC_ROAM_CACHE_UPDATE_MISSING_AP:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is missed ap\n", status));
+		break;
+	default:
+		DHD_EVENT(("Current roam cache status %d, "
+			"reason for cache update is unknown %d\n", status, reason));
+		break;
+	}
+
+	if (datalen < sizeof(wlc_roam_cache_update_event_t)) {
+		DHD_ERROR(("MACEVENT: %s, missing event data\n", name));
+		return;
+	}
+
+	cache_update = (wlc_roam_cache_update_event_t *)event_data;
+	val_tlv_ptr = (void *)cache_update->xtlvs;
+	len_of_tlvs = datalen - sizeof(wlc_roam_cache_update_event_t);
+	val_xtlv = (bcm_xtlv_t *)val_tlv_ptr;
+	if (val_xtlv->id != WL_RMC_RPT_CMD_DATA) {
+		DHD_ERROR(("MACEVENT: %s, unexpected xtlv id %d\n",
+			name, val_xtlv->id));
+		return;
+	}
+	val_tlv_ptr = (uint8 *)val_tlv_ptr + BCM_XTLV_HDR_SIZE;
+	len_of_tlvs = val_xtlv->len;
+
+	while (len_of_tlvs && len_of_tlvs > BCM_XTLV_HDR_SIZE) {
+		val_xtlv = (bcm_xtlv_t *)val_tlv_ptr;
+		switch (val_xtlv->id) {
+			case WL_RMC_RPT_XTLV_BSS_INFO:
+			{
+				rmc_bss_info_v1_t *bss_info = (rmc_bss_info_v1_t *)(val_xtlv->data);
+				DHD_EVENT(("\t Current BSS INFO:\n"));
+				DHD_EVENT(("\t\tRSSI: %d\n", bss_info->rssi));
+				DHD_EVENT(("\t\tNumber of full scans performed "
+					"on current BSS: %d\n", bss_info->fullscan_count));
+				for (idx = 0; idx < ARRAYSIZE(reason_names); idx++) {
+					if (reason_names[idx].event == bss_info->reason) {
+						reason_name = reason_names[idx].event_name;
+					}
+				}
+				DHD_EVENT(("\t\tReason code for last full scan: %s(%d)\n",
+					reason_name, bss_info->reason));
+				DHD_EVENT(("\t\tDelta between current time and "
+					"last full scan: %d\n", bss_info->time_full_scan));
+				for (idx = 0; idx < ARRAYSIZE(status_names); idx++) {
+					if (status_names[idx].event == bss_info->status)
+						status_name = status_names[idx].event_name;
+				}
+				DHD_EVENT(("\t\tLast status code for not roaming: %s(%d)\n",
+					status_name, bss_info->status));
+
+			}
+				break;
+			case WL_RMC_RPT_XTLV_CANDIDATE_INFO:
+			case WL_RMC_RPT_XTLV_USER_CACHE_INFO:
+			{
+				rmc_candidate_info_v1_t *candidate_info =
+					(rmc_candidate_info_v1_t *)(val_xtlv->data);
+				UNUSED_PARAMETER(candidate_info);
+				if (val_xtlv->id == WL_RMC_RPT_XTLV_CANDIDATE_INFO) {
+					DHD_EVENT(("\t Candidate INFO:\n"));
+				} else {
+					DHD_EVENT(("\t User Candidate INFO:\n"));
+				}
+				DHD_EVENT(("\t\tBSSID: %s\n",
+					bcm_ether_ntoa((const struct ether_addr *)
+					&candidate_info->bssid, ntoa_buf)));
+				DHD_EVENT(("\t\tRSSI: %d\n", candidate_info->rssi));
+				DHD_EVENT(("\t\tChannel: %d\n", candidate_info->ctl_channel));
+				DHD_EVENT(("\t\tDelta between current time and last "
+					"seen time: %d\n", candidate_info->time_last_seen));
+				DHD_EVENT(("\t\tBSS load: %d\n", candidate_info->bss_load));
+			}
+				break;
+			default:
+				DHD_ERROR(("MACEVENT: %s, unexpected xtlv id %d\n",
+					name, val_xtlv->id));
+				return;
+		}
+		val_tlv_ptr = (uint8 *)val_tlv_ptr + bcm_xtlv_size(val_xtlv,
+			BCM_XTLV_OPTION_NONE);
+		len_of_tlvs -= (uint16)bcm_xtlv_size(val_xtlv, BCM_XTLV_OPTION_NONE);
+	}
+}
+
+static void
+wl_show_host_event(dhd_pub_t *dhd_pub, wl_event_msg_t *event, void *event_data,
+	void *raw_event_ptr, char *eventmask)
+{
+	uint i, status, reason;
+	bool group = FALSE, flush_txq = FALSE, link = FALSE;
+	bool host_data = FALSE; /* prints  event data after the case  when set */
+	const char *auth_str;
+	const char *event_name;
+	const uchar *buf;
+	char err_msg[256], eabuf[ETHER_ADDR_STR_LEN];
+	uint event_type, flags, auth_type, datalen;
+
+	event_type = ntoh32(event->event_type);
+	flags = ntoh16(event->flags);
+	status = ntoh32(event->status);
+	reason = ntoh32(event->reason);
+	BCM_REFERENCE(reason);
+	auth_type = ntoh32(event->auth_type);
+	datalen = (event_data != NULL) ? ntoh32(event->datalen) : 0;
+
+	/* debug dump of event messages */
+	snprintf(eabuf, sizeof(eabuf), MACDBG, MAC2STRDBG(event->addr.octet));
+
+	event_name = bcmevent_get_name(event_type);
+	BCM_REFERENCE(event_name);
+
+	if (flags & WLC_EVENT_MSG_LINK)
+		link = TRUE;
+	if (flags & WLC_EVENT_MSG_GROUP)
+		group = TRUE;
+	if (flags & WLC_EVENT_MSG_FLUSHTXQ)
+		flush_txq = TRUE;
+
+	switch (event_type) {
+	case WLC_E_START:
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+		break;
+	case WLC_E_DEAUTH:
+	case WLC_E_DISASSOC:
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+#ifdef REPORT_FATAL_TIMEOUTS
+		dhd_clear_join_error(dhd_pub, WLC_SSID_MASK | WLC_WPA_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		break;
+
+	case WLC_E_ASSOC_IND:
+	case WLC_E_REASSOC_IND:
+
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+#ifdef REPORT_FATAL_TIMEOUTS
+		if (status != WLC_E_STATUS_SUCCESS) {
+			dhd_clear_join_error(dhd_pub, WLC_SSID_MASK | WLC_WPA_MASK);
+		}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+		break;
+
+	case WLC_E_BSSID:
+		DHD_EVENT(("MACEVENT: %s, MAC %s, reason %d datalen %d\n", event_name,
+			eabuf, (int)reason, datalen));
+		break;
+
+	case WLC_E_ASSOC:
+	case WLC_E_REASSOC:
+		if (status == WLC_E_STATUS_SUCCESS) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, SUCCESS\n", event_name, eabuf));
+		} else if (status == WLC_E_STATUS_TIMEOUT) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, TIMEOUT\n", event_name, eabuf));
+		} else if (status == WLC_E_STATUS_FAIL) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, FAILURE, status %d reason %d\n",
+			       event_name, eabuf, (int)status, (int)reason));
+		} else if (status == WLC_E_STATUS_SUPPRESS) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, SUPPRESS\n", event_name, eabuf));
+		} else if (status == WLC_E_STATUS_NO_ACK) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, NOACK\n", event_name, eabuf));
+		} else {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, unexpected status %d\n",
+			       event_name, eabuf, (int)status));
+		}
+#ifdef REPORT_FATAL_TIMEOUTS
+		if (status != WLC_E_STATUS_SUCCESS) {
+			dhd_clear_join_error(dhd_pub, WLC_SSID_MASK | WLC_WPA_MASK);
+		}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+		break;
+
+	case WLC_E_DEAUTH_IND:
+	case WLC_E_DISASSOC_IND:
+#ifdef REPORT_FATAL_TIMEOUTS
+		dhd_clear_join_error(dhd_pub, WLC_SSID_MASK | WLC_WPA_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		DHD_EVENT(("MACEVENT: %s, MAC %s, reason %d\n", event_name, eabuf, (int)reason));
+		break;
+
+	case WLC_E_AUTH:
+	case WLC_E_AUTH_IND:
+		if (auth_type == DOT11_OPEN_SYSTEM)
+			auth_str = "Open System";
+		else if (auth_type == DOT11_SHARED_KEY)
+			auth_str = "Shared Key";
+		else if (auth_type == DOT11_SAE)
+			auth_str = "SAE";
+		else {
+			snprintf(err_msg, sizeof(err_msg), "AUTH unknown: %d", (int)auth_type);
+			auth_str = err_msg;
+		}
+
+		if (event_type == WLC_E_AUTH_IND) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s\n", event_name, eabuf, auth_str));
+		} else if (status == WLC_E_STATUS_SUCCESS) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s, SUCCESS\n",
+				event_name, eabuf, auth_str));
+		} else if (status == WLC_E_STATUS_TIMEOUT) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s, TIMEOUT\n",
+				event_name, eabuf, auth_str));
+		} else if (status == WLC_E_STATUS_FAIL) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s, FAILURE, status %d reason %d\n",
+			       event_name, eabuf, auth_str, (int)status, (int)reason));
+		} else if (status == WLC_E_STATUS_SUPPRESS) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s, SUPPRESS\n",
+			       event_name, eabuf, auth_str));
+		} else if (status == WLC_E_STATUS_NO_ACK) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s, NOACK\n",
+			       event_name, eabuf, auth_str));
+		} else {
+			DHD_EVENT(("MACEVENT: %s, MAC %s, %s, status %d reason %d\n",
+				event_name, eabuf, auth_str, (int)status, (int)reason));
+		}
+		BCM_REFERENCE(auth_str);
+#ifdef REPORT_FATAL_TIMEOUTS
+		if (status != WLC_E_STATUS_SUCCESS) {
+			dhd_clear_join_error(dhd_pub, WLC_SSID_MASK | WLC_WPA_MASK);
+		}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+		break;
+
+	case WLC_E_ROAM:
+		wl_show_roam_event(dhd_pub, status, datalen,
+			event_name, eabuf, event_data);
+		break;
+	case WLC_E_ROAM_START:
+		if (datalen >= sizeof(wlc_roam_start_event_t)) {
+			const wlc_roam_start_event_t *roam_start =
+				(wlc_roam_start_event_t *)event_data;
+			UNUSED_PARAMETER(roam_start);
+			DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d,"
+				" reason %d, auth %d, current bss rssi %d\n",
+				event_name, event_type, eabuf, (int)status, (int)reason,
+				(int)auth_type, (int)roam_start->rssi));
+		} else {
+			DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d\n",
+				event_name, event_type, eabuf, (int)status, (int)reason,
+				(int)auth_type));
+		}
+		break;
+	case WLC_E_ROAM_PREP:
+		if (datalen >= sizeof(wlc_roam_prep_event_t)) {
+			const wlc_roam_prep_event_t *roam_prep =
+				(wlc_roam_prep_event_t *)event_data;
+			UNUSED_PARAMETER(roam_prep);
+			DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d,"
+				" reason %d, auth %d, target bss rssi %d\n",
+				event_name, event_type, eabuf, (int)status, (int)reason,
+				(int)auth_type, (int)roam_prep->rssi));
+		} else {
+			DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d\n",
+				event_name, event_type, eabuf, (int)status, (int)reason,
+				(int)auth_type));
+		}
+		break;
+	case WLC_E_ROAM_CACHE_UPDATE:
+		DHD_EVENT(("MACEVENT: %s\n", event_name));
+		wl_show_roam_cache_update_event(event_name, status,
+			reason, datalen, event_data);
+		break;
+	case WLC_E_JOIN:
+	case WLC_E_SET_SSID:
+#ifdef REPORT_FATAL_TIMEOUTS
+		OSL_ATOMIC_SET(dhd_pub->osh, &dhd_pub->set_ssid_rcvd, TRUE);
+		dhd_clear_join_error(dhd_pub, WLC_SSID_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		if (status == WLC_E_STATUS_SUCCESS) {
+			DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+		} else {
+#ifdef REPORT_FATAL_TIMEOUTS
+			/*
+			 * For secure join if WLC_E_SET_SSID returns with any failure case,
+			 * donot expect WLC_E_PSK_SUP. So clear the mask.
+			 */
+			dhd_clear_join_error(dhd_pub, WLC_WPA_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+			if (status == WLC_E_STATUS_FAIL) {
+				DHD_EVENT(("MACEVENT: %s, failed status %d\n", event_name, status));
+			} else if (status == WLC_E_STATUS_NO_NETWORKS) {
+				DHD_EVENT(("MACEVENT: %s, no networks found\n", event_name));
+			} else {
+				DHD_EVENT(("MACEVENT: %s, unexpected status %d\n",
+					event_name, (int)status));
+			}
+		}
+		break;
+
+	case WLC_E_JOIN_START:
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+		break;
+
+	case WLC_E_BEACON_RX:
+		if (status == WLC_E_STATUS_SUCCESS) {
+			DHD_EVENT(("MACEVENT: %s, SUCCESS\n", event_name));
+		} else if (status == WLC_E_STATUS_FAIL) {
+			DHD_EVENT(("MACEVENT: %s, FAIL\n", event_name));
+		} else {
+			DHD_EVENT(("MACEVENT: %s, status %d\n", event_name, status));
+		}
+		break;
+
+	case WLC_E_LINK:
+		DHD_EVENT(("MACEVENT: %s %s flags:0x%x status:%d reason:%d\n",
+			event_name, link?"UP":"DOWN", flags, status, reason));
+#ifdef PCIE_FULL_DONGLE
+#ifdef REPORT_FATAL_TIMEOUTS
+		{
+			uint8 ifindex = (uint8)dhd_ifname2idx(dhd_pub->info, event->ifname);
+			uint8 role = dhd_flow_rings_ifindex2role(dhd_pub, ifindex);
+			if ((role == WLC_E_IF_ROLE_STA) && (!link)) {
+				dhd_clear_join_error(dhd_pub, WLC_SSID_MASK | WLC_WPA_MASK);
+			}
+		}
+#endif /* PCIE_FULL_DONGLE */
+#endif /* REPORT_FATAL_TIMEOUTS */
+		BCM_REFERENCE(link);
+		break;
+
+	case WLC_E_MIC_ERROR:
+		DHD_EVENT(("MACEVENT: %s, MAC %s, Group %d, Flush %d\n",
+		       event_name, eabuf, group, flush_txq));
+		BCM_REFERENCE(group);
+		BCM_REFERENCE(flush_txq);
+		break;
+
+	case WLC_E_ICV_ERROR:
+	case WLC_E_UNICAST_DECODE_ERROR:
+	case WLC_E_MULTICAST_DECODE_ERROR:
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n",
+		       event_name, eabuf));
+		break;
+
+	case WLC_E_TXFAIL:
+		DHD_EVENT(("MACEVENT: %s, RA %s status %d\n", event_name, eabuf, status));
+		break;
+
+	case WLC_E_ASSOC_REQ_IE:
+	case WLC_E_ASSOC_RESP_IE:
+	case WLC_E_PMKID_CACHE:
+		DHD_EVENT(("MACEVENT: %s\n", event_name));
+		break;
+
+	case WLC_E_SCAN_COMPLETE:
+		DHD_EVENT(("MACEVENT: %s\n", event_name));
+#ifdef REPORT_FATAL_TIMEOUTS
+		dhd_stop_scan_timer(dhd_pub, FALSE, 0);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		break;
+	case WLC_E_RSSI_LQM:
+	case WLC_E_PFN_NET_FOUND:
+	case WLC_E_PFN_NET_LOST:
+	case WLC_E_PFN_SCAN_COMPLETE:
+	case WLC_E_PFN_SCAN_NONE:
+	case WLC_E_PFN_SCAN_ALLGONE:
+	case WLC_E_PFN_GSCAN_FULL_RESULT:
+#ifdef WL_SCHED_SCAN
+	case WLC_E_PFN_PARTIAL_RESULT:
+#endif /* WL_SCHED_SCAN */
+	case WLC_E_PFN_SSID_EXT:
+		DHD_EVENT(("PNOEVENT: %s\n", event_name));
+		break;
+
+	case WLC_E_PFN_SCAN_BACKOFF:
+	case WLC_E_PFN_BSSID_SCAN_BACKOFF:
+		DHD_EVENT(("PNOEVENT: %s, status %d, reason %d\n",
+		           event_name, (int)status, (int)reason));
+		break;
+
+	case WLC_E_PSK_SUP:
+	case WLC_E_PRUNE:
+		DHD_EVENT(("MACEVENT: %s, status %d, reason %d\n",
+		           event_name, (int)status, (int)reason));
+#ifdef REPORT_FATAL_TIMEOUTS
+		dhd_clear_join_error(dhd_pub, WLC_WPA_MASK);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		break;
+
+#ifdef WIFI_ACT_FRAME
+	case WLC_E_ACTION_FRAME:
+		DHD_TRACE(("MACEVENT: %s Bssid %s\n", event_name, eabuf));
+		break;
+	case WLC_E_ACTION_FRAME_COMPLETE:
+		if (datalen >= sizeof(uint32)) {
+			const uint32 *pktid = event_data;
+			BCM_REFERENCE(pktid);
+			DHD_EVENT(("MACEVENT: %s status %d, reason %d, pktid 0x%x\n",
+				event_name, (int)status, (int)reason, *pktid));
+		}
+		break;
+#endif /* WIFI_ACT_FRAME */
+
+#ifdef SHOW_LOGTRACE
+	case WLC_E_TRACE:
+	{
+		dhd_dbg_trace_evnt_handler(dhd_pub, event_data, raw_event_ptr, datalen);
+		break;
+	}
+#endif /* SHOW_LOGTRACE */
+
+	case WLC_E_RSSI: {
+		wl_event_data_rssi_t *e_data = (wl_event_data_rssi_t *)event_data;
+		UNUSED_PARAMETER(e_data);
+		DHD_EVENT(("MACEVENT: %s (RSSI=%d SNR=%d NF=%d)\n", event_name,
+			ntoh32(e_data->rssi), ntoh32(e_data->snr), ntoh32(e_data->noise)));
+		} break;
+
+	case WLC_E_SERVICE_FOUND:
+	case WLC_E_P2PO_ADD_DEVICE:
+	case WLC_E_P2PO_DEL_DEVICE:
+		DHD_EVENT(("MACEVENT: %s, MAC %s\n", event_name, eabuf));
+		break;
+
+	case WLC_E_CCA_CHAN_QUAL:
+		/* I would like to check here that datalen >= sizeof(cca_chan_qual_event_t)
+		 * but since definition of cca_chan_qual_event_t is different
+		 * between blazar and legacy firmware, I will
+		 * check only that datalen is bigger than 0.
+		 */
+		if (datalen >= sizeof(cca_chan_qual_event_t)) {
+			cca_chan_qual_event_t *cca_event =
+				(cca_chan_qual_event_t *)event_data;
+			if ((cca_event->id == WL_CHAN_QUAL_FULLPM_CCA) ||
+			    (cca_event->id == WL_CHAN_QUAL_FULLPM_CCA_OFDM_DESENSE) ||
+			    (cca_event->id == WL_CHAN_QUAL_FULLPM_CCA_ALL_DESENSE)) {
+				cca_only_chan_qual_event_t *cca_only_event =
+					(cca_only_chan_qual_event_t *)cca_event;
+				BCM_REFERENCE(cca_only_event);
+				DHD_EVENT((
+					"MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d,"
+					" channel 0x%02x\n",
+					event_name, event_type, eabuf, (int)status,
+					(int)reason, (int)auth_type, cca_event->chanspec));
+				DHD_EVENT((
+					"\tTOTAL (dur %dms me %dms notme %dms interf %dms"
+					" ts 0x%08x)\n",
+					cca_only_event->cca_busy_ext.duration,
+					cca_only_event->cca_busy_ext.congest_ibss,
+					cca_only_event->cca_busy_ext.congest_obss,
+					cca_only_event->cca_busy_ext.interference,
+					cca_only_event->cca_busy_ext.timestamp));
+				DHD_EVENT((
+					"\t  !PM (dur %dms me %dms notme %dms interf %dms)\n",
+					cca_only_event->cca_busy_nopm.duration,
+					cca_only_event->cca_busy_nopm.congest_ibss,
+					cca_only_event->cca_busy_nopm.congest_obss,
+					cca_only_event->cca_busy_nopm.interference));
+				DHD_EVENT((
+					"\t   PM (dur %dms me %dms notme %dms interf %dms)\n",
+					cca_only_event->cca_busy_pm.duration,
+					cca_only_event->cca_busy_pm.congest_ibss,
+					cca_only_event->cca_busy_pm.congest_obss,
+					cca_only_event->cca_busy_pm.interference));
+				if (cca_event->id == WL_CHAN_QUAL_FULLPM_CCA_OFDM_DESENSE) {
+					cca_only_chan_qual_event_v2_t *cca_event_v2 =
+						(cca_only_chan_qual_event_v2_t *)
+						cca_only_event;
+					if (cca_event->len >=
+						(STRUCT_SIZE_THROUGH(cca_event_v2, ofdm_desense) -
+						STRUCT_SIZE_THROUGH(cca_event_v2, len))) {
+						DHD_EVENT(("\t OFDM desense %d\n",
+							cca_event_v2->ofdm_desense));
+					} else {
+						DHD_EVENT(("\t OFDM desense: missing data\n"));
+					}
+				} else if (cca_event->id == WL_CHAN_QUAL_FULLPM_CCA_ALL_DESENSE) {
+					cca_only_chan_qual_event_v3_t *cca_event_v3 =
+						(cca_only_chan_qual_event_v3_t *)
+						cca_only_event;
+					if (cca_event->len >=
+						(STRUCT_SIZE_THROUGH(cca_event_v3, desense) -
+						STRUCT_SIZE_THROUGH(cca_event_v3, len))) {
+						DHD_EVENT(("\t OFDM desense %d BPHY desense %d"
+							" reason 0x%x)\n",
+							cca_event_v3->desense.ofdm_desense,
+							cca_event_v3->desense.bphy_desense,
+							cca_event_v3->desense.reason));
+					} else {
+						DHD_EVENT(("\t  PHY desense: missing data\n"));
+					}
+				}
+			} else if (cca_event->id == WL_CHAN_QUAL_FULL_CCA) {
+				DHD_EVENT((
+					"MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d,"
+					" channel 0x%02x (dur %dms ibss %dms obss %dms interf %dms"
+					" ts 0x%08x)\n",
+					event_name, event_type, eabuf, (int)status,
+					(int)reason, (int)auth_type, cca_event->chanspec,
+					cca_event->cca_busy_ext.duration,
+					cca_event->cca_busy_ext.congest_ibss,
+					cca_event->cca_busy_ext.congest_obss,
+					cca_event->cca_busy_ext.interference,
+					cca_event->cca_busy_ext.timestamp));
+			} else if (cca_event->id == WL_CHAN_QUAL_CCA) {
+				DHD_EVENT((
+					"MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d,"
+					" channel 0x%02x (dur %dms busy %dms ts 0x%08x)\n",
+					event_name, event_type, eabuf, (int)status,
+					(int)reason, (int)auth_type, cca_event->chanspec,
+					cca_event->cca_busy.duration,
+					cca_event->cca_busy.congest,
+					cca_event->cca_busy.timestamp));
+			} else if ((cca_event->id == WL_CHAN_QUAL_NF) ||
+			           (cca_event->id == WL_CHAN_QUAL_NF_LTE)) {
+				DHD_EVENT((
+					"MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d,"
+					" channel 0x%02x (NF[%d] %ddB)\n",
+					event_name, event_type, eabuf, (int)status,
+					(int)reason, (int)auth_type, cca_event->chanspec,
+					cca_event->id, cca_event->noise));
+			} else {
+				DHD_EVENT((
+					"MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d,"
+					" channel 0x%02x (unknown ID %d)\n",
+					event_name, event_type, eabuf, (int)status,
+					(int)reason, (int)auth_type, cca_event->chanspec,
+					cca_event->id));
+			}
+		}
+		break;
+	case WLC_E_ESCAN_RESULT:
+#if defined(NDIS)
+		break;
+#endif
+		if (datalen >= sizeof(wl_escan_result_v2_t)) {
+			const wl_escan_result_v2_t *escan_result =
+				(wl_escan_result_v2_t *)event_data;
+			BCM_REFERENCE(escan_result);
+#ifdef OEM_ANDROID
+			/* Because WLC_E_ESCAN_RESULT event log are being print too many.
+			* So, DHD_EVENT() changes to be used DHD_TRACE() in HW4 platform.
+			*/
+			DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d \n",
+				event_name, event_type, eabuf, (int)status));
+#else
+			DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d sync-id %u\n",
+				event_name, event_type, eabuf,
+				(int)status, dtoh16(escan_result->sync_id)));
+#endif
+#ifdef REPORT_FATAL_TIMEOUTS
+			/* a 'partial' status means the escan is still in progress
+			* any other status implies the escan has either finished or aborted
+			*/
+			if (status != WLC_E_STATUS_PARTIAL) {
+				unsigned long timeout_flags = 0;
+				uint16 syncid = dtoh16(escan_result->sync_id);
+				/* this is to take care of the specific case where
+				* escan event returns abort and is processed immediately
+				* by dhd before the escan iovar has returned. In that case
+				* if the iovar returns success, then we will be starting a
+				* timeout even though the escan has already been aborted !
+				* So the flag below is checked before starting the escan timeout
+				*/
+				if (dhd_pub->timeout_info) {
+					DHD_TIMER_LOCK(dhd_pub->timeout_info->scan_timer_lock,
+						timeout_flags);
+					if (!dhd_pub->timeout_info->scan_timer_active &&
+						syncid == dhd_pub->esync_id) {
+						dhd_pub->timeout_info->escan_aborted = TRUE;
+						dhd_pub->timeout_info->abort_syncid = syncid;
+						DHD_TIMER_UNLOCK(
+							dhd_pub->timeout_info->scan_timer_lock,
+							timeout_flags);
+						break;
+					} else {
+						dhd_pub->timeout_info->escan_aborted = FALSE;
+					}
+					DHD_TIMER_UNLOCK(dhd_pub->timeout_info->scan_timer_lock,
+						timeout_flags);
+				}
+				dhd_stop_scan_timer(dhd_pub, TRUE, dtoh16(escan_result->sync_id));
+			}
+#endif /* REPORT_FATAL_TIMEOUTS */
+		}
+		break;
+	case WLC_E_IF:
+		if (datalen >= sizeof(struct wl_event_data_if)) {
+			const struct wl_event_data_if *ifevent =
+				(struct wl_event_data_if *)event_data;
+			BCM_REFERENCE(ifevent);
+
+			DHD_EVENT(("MACEVENT: %s, opcode:0x%d  ifidx:%d role:%d\n",
+				event_name, ifevent->opcode, ifevent->ifidx, ifevent->role));
+		}
+		break;
+#ifdef SHOW_LOGTRACE
+	case WLC_E_MSCH:
+	{
+		wl_mschdbg_event_handler(dhd_pub, raw_event_ptr, reason, event_data, datalen);
+		break;
+	}
+#endif /* SHOW_LOGTRACE */
+
+	case WLC_E_PSK_AUTH:
+		DHD_EVENT(("MACEVENT: %s, RA %s status %d Reason:%d\n",
+			event_name, eabuf, status, reason));
+		break;
+	case WLC_E_AGGR_EVENT:
+		if (datalen >= sizeof(event_aggr_data_t)) {
+			const event_aggr_data_t *aggrbuf = event_data;
+			int j = 0, len = 0;
+			const uint8 *data = aggrbuf->data;
+			DHD_EVENT(("MACEVENT: %s, num of events %d total len %d sub events: ",
+				event_name, aggrbuf->num_events, aggrbuf->len));
+			for (j = 0; j < aggrbuf->num_events; j++)
+			{
+				const wl_event_msg_t * sub_event = (const wl_event_msg_t *)data;
+				if (len > aggrbuf->len) {
+					DHD_ERROR(("%s: Aggr events corrupted!",
+						__FUNCTION__));
+					break;
+				}
+				DHD_EVENT(("\n Event type: %d ", ntoh32(sub_event->event_type)));
+				len += ALIGN_SIZE((ntoh32(sub_event->datalen) +
+					sizeof(wl_event_msg_t)), sizeof(uint64));
+				buf = (const uchar *)(data + sizeof(wl_event_msg_t));
+				BCM_REFERENCE(buf);
+				DHD_EVENT((" data (%d) : ", ntoh32(sub_event->datalen)));
+				for (i = 0; i < ntoh32(sub_event->datalen); i++) {
+					DHD_EVENT((" 0x%02x ", buf[i]));
+				}
+				data = aggrbuf->data + len;
+			}
+			DHD_EVENT(("\n"));
+		}
+		break;
+	case WLC_E_PHY_CAL:
+		{
+			DHD_EVENT(("MACEVENT: %s, reason:%d\n", event_name, reason));
+			break;
+		}
+	case WLC_E_NAN_CRITICAL:
+		{
+			DHD_EVENT(("MACEVENT: %s, type:%d\n", event_name, reason));
+			break;
+		}
+	case WLC_E_NAN_NON_CRITICAL:
+		{
+			DHD_TRACE(("MACEVENT: %s, type:%d\n", event_name, reason));
+			break;
+		}
+	case WLC_E_PROXD:
+		if (datalen >= sizeof(wl_proxd_event_t)) {
+			const wl_proxd_event_t *proxd =
+				(wl_proxd_event_t*)event_data;
+			DHD_LOG_MEM(("MACEVENT: %s, event:%d, status:%d\n",
+				event_name, proxd->type, reason));
+		}
+		break;
+	case WLC_E_RPSNOA:
+		if (datalen >= sizeof(rpsnoa_stats_t)) {
+			const rpsnoa_stats_t *stat = event_data;
+			if (datalen == sizeof(*stat)) {
+				DHD_EVENT(("MACEVENT: %s, band %s, status %d, pps %d\n", event_name,
+					(stat->band == WLC_BAND_2G) ? "2G":"5G",
+					stat->state, stat->last_pps));
+			}
+		}
+		break;
+	case WLC_E_WA_LQM:
+		if (datalen >= sizeof(wl_event_wa_lqm_t)) {
+			const wl_event_wa_lqm_t *event_wa_lqm =
+				(wl_event_wa_lqm_t *)event_data;
+			const bcm_xtlv_t *subevent;
+			const wl_event_wa_lqm_basic_t *elqm_basic;
+
+			if ((event_wa_lqm->ver != WL_EVENT_WA_LQM_VER) ||
+			    (event_wa_lqm->len < sizeof(wl_event_wa_lqm_t) + BCM_XTLV_HDR_SIZE)) {
+				DHD_ERROR(("MACEVENT: %s invalid (ver=%d len=%d)\n",
+					event_name, event_wa_lqm->ver, event_wa_lqm->len));
+				break;
+			}
+
+			subevent = (const bcm_xtlv_t *)event_wa_lqm->subevent;
+			 if ((subevent->id != WL_EVENT_WA_LQM_BASIC) ||
+			     (subevent->len < sizeof(wl_event_wa_lqm_basic_t))) {
+				DHD_ERROR(("MACEVENT: %s invalid sub-type (id=%d len=%d)\n",
+					event_name, subevent->id, subevent->len));
+				break;
+			}
+
+			elqm_basic = (const wl_event_wa_lqm_basic_t *)subevent->data;
+			BCM_REFERENCE(elqm_basic);
+			DHD_EVENT(("MACEVENT: %s (RSSI=%d SNR=%d TxRate=%d RxRate=%d)\n",
+				event_name, elqm_basic->rssi, elqm_basic->snr,
+				elqm_basic->tx_rate, elqm_basic->rx_rate));
+		}
+		break;
+
+	case WLC_E_OBSS_DETECTION:
+		{
+			DHD_EVENT(("MACEVENT: %s, type:%d\n", event_name, reason));
+			break;
+		}
+
+	case WLC_E_AP_BCN_MUTE:
+		if (datalen >= sizeof(wlc_bcn_mute_miti_event_data_v1_t)) {
+			const wlc_bcn_mute_miti_event_data_v1_t
+				*bcn_mute_miti_evnt_data = event_data;
+			UNUSED_PARAMETER(bcn_mute_miti_evnt_data);
+			DHD_EVENT(("MACEVENT: %s, reason :%d uatbtt_count: %d\n",
+				event_name, reason, bcn_mute_miti_evnt_data->uatbtt_count));
+		}
+		break;
+#ifdef WL_TWT
+	case WLC_E_TWT:
+		DHD_EVENT(("MACEVENT: %s, type:%d\n", event_name, reason));
+		break;
+#endif /* WL_TWT */
+	case WLC_E_COUNTRY_CODE_CHANGED:
+		DHD_EVENT(("MACEVENT: %s: Country code changed to %s\n", event_name,
+			(char*)event_data));
+		break;
+#if defined(CSI_SUPPORT)
+	case WLC_E_CSI:
+		/* defferred the process due to extra IOVAR may execute */
+		break;
+#endif /* CSI_SUPPORT */
+	case WLC_E_SCAN:
+		{
+			const char *scan_state;
+			UNUSED_PARAMETER(scan_state);
+			if (reason == WL_SCAN_START) {
+				scan_state = "WL_SCAN_START";
+			} else if (reason == WL_SCAN_END) {
+				scan_state = "WL_SCAN_END";
+			} else {
+				scan_state = "Unknown";
+			}
+			DHD_EVENT(("MACEVENT: %s: scan state %s\n", event_name, scan_state));
+			break;
+		}
+	case WLC_E_OWE_INFO:
+		DHD_EVENT(("MACEVENT: %s, MAC %s type:%d\n", event_name, eabuf, reason));
+		break;
+	default:
+		DHD_EVENT(("MACEVENT: %s %d, MAC %s, status %d, reason %d, auth %d\n",
+		       event_name, event_type, eabuf, (int)status, (int)reason,
+		       (int)auth_type));
+		break;
+	}
+
+	/* show any appended data if message level is set to bytes or host_data is set */
+	if ((DHD_BYTES_ON() || (host_data == TRUE)) && DHD_EVENT_ON() && datalen) {
+		buf = (uchar *) event_data;
+		BCM_REFERENCE(buf);
+		DHD_EVENT((" data (%d) : ", datalen));
+		for (i = 0; i < datalen; i++) {
+			DHD_EVENT((" 0x%02x ", buf[i]));
+		}
+		DHD_EVENT(("\n"));
+	}
+} /* wl_show_host_event */
+#endif /* SHOW_EVENTS */
+
+#ifdef DNGL_EVENT_SUPPORT
+/* Check whether packet is a BRCM dngl event pkt. If it is, process event data. */
+int
+dngl_host_event(dhd_pub_t *dhdp, void *pktdata, bcm_dngl_event_msg_t *dngl_event, size_t pktlen)
+{
+	bcm_dngl_event_t *pvt_data = (bcm_dngl_event_t *)pktdata;
+
+	dngl_host_event_process(dhdp, pvt_data, dngl_event, pktlen);
+	return BCME_OK;
+}
+
+#ifdef PARSE_DONGLE_HOST_EVENT
+typedef struct hck_id_to_str_s {
+	uint32 id;
+	char *name;
+} hck_id_to_str_t;
+
+hck_id_to_str_t hck_sw_id_to_str[] = {
+	{WL_HC_DD_PCIE, "WL_HC_DD_PCIE"},
+	{WL_HC_DD_RX_DMA_STALL, "WL_HC_DD_RX_DMA_STALL"},
+	{WL_HC_DD_RX_STALL, "WL_HC_DD_RX_STALL"},
+	{WL_HC_DD_TX_STALL, "WL_HC_DD_TX_STALL"},
+	{WL_HC_DD_SCAN_STALL, "WL_HC_DD_SCAN_STALL"},
+	{WL_HC_DD_PHY, "WL_HC_DD_PHY"},
+	{WL_HC_DD_REINIT, "WL_HC_DD_REINIT"},
+	{WL_HC_DD_TXQ_STALL, "WL_HC_DD_TXQ_STALL"},
+	{0, NULL}
+};
+
+hck_id_to_str_t hck_pcie_module_to_str[] = {
+	{HEALTH_CHECK_PCIEDEV_INDUCED_IND, "PCIEDEV_INDUCED_IND"},
+	{HEALTH_CHECK_PCIEDEV_H2D_DMA_IND, "PCIEDEV_H2D_DMA_IND"},
+	{HEALTH_CHECK_PCIEDEV_D2H_DMA_IND, "PCIEDEV_D2H_DMA_IND"},
+	{HEALTH_CHECK_PCIEDEV_IOCTL_STALL_IND, "PCIEDEV_IOCTL_STALL_IND"},
+	{HEALTH_CHECK_PCIEDEV_D3ACK_STALL_IND, "PCIEDEV_D3ACK_STALL_IND"},
+	{HEALTH_CHECK_PCIEDEV_NODS_IND, "PCIEDEV_NODS_IND"},
+	{HEALTH_CHECK_PCIEDEV_LINKSPEED_FALLBACK_IND, "PCIEDEV_LINKSPEED_FALLBACK_IND"},
+	{HEALTH_CHECK_PCIEDEV_DSACK_STALL_IND, "PCIEDEV_DSACK_STALL_IND"},
+	{0, NULL}
+};
+
+hck_id_to_str_t hck_rx_stall_v2_to_str[] = {
+	{BCM_RX_HC_RESERVED, "BCM_RX_HC_RESERVED"},
+	{BCM_RX_HC_UNSPECIFIED, "BCM_RX_HC_UNSPECIFIED"},
+	{BCM_RX_HC_UNICAST_DECRYPT_FAIL, "BCM_RX_HC_UNICAST_DECRYPT_FAIL"},
+	{BCM_RX_HC_BCMC_DECRYPT_FAIL, "BCM_RX_HC_BCMC_DECRYPT_FAIL"},
+	{BCM_RX_HC_UNICAST_REPLAY, "BCM_RX_HC_UNICAST_REPLAY"},
+	{BCM_RX_HC_BCMC_REPLAY, "BCM_RX_HC_BCMC_REPLAY"},
+	{BCM_RX_HC_AMPDU_DUP, "BCM_RX_HC_AMPDU_DUP"},
+	{0, NULL}
+};
+
+static void
+dhd_print_dongle_hck_id(uint32 id, hck_id_to_str_t *hck)
+{
+	while (hck->name != NULL) {
+		if (hck->id == id) {
+			DHD_ERROR(("DONGLE_HCK_EVENT: %s\n", hck->name));
+			return;
+		}
+		hck++;
+	}
+}
+
+void
+dhd_parse_hck_common_sw_event(bcm_xtlv_t *wl_hc)
+{
+
+	wl_rx_hc_info_v2_t *hck_rx_stall_v2;
+	uint16 id;
+
+	id = ltoh16(wl_hc->id);
+
+	if (id == WL_HC_DD_RX_STALL_V2) {
+		/*  map the hck_rx_stall_v2 structure to the value of the XTLV */
+		hck_rx_stall_v2 =
+			(wl_rx_hc_info_v2_t*)wl_hc;
+		DHD_ERROR(("type:%d len:%d if_idx:%d ac:%d pkts:%d"
+			" drop:%d alert_th:%d reason:%d peer_ea:"MACF"\n",
+			hck_rx_stall_v2->type,
+			hck_rx_stall_v2->length,
+			hck_rx_stall_v2->if_idx,
+			hck_rx_stall_v2->ac,
+			hck_rx_stall_v2->rx_hc_pkts,
+			hck_rx_stall_v2->rx_hc_dropped_all,
+			hck_rx_stall_v2->rx_hc_alert_th,
+			hck_rx_stall_v2->reason,
+			ETHER_TO_MACF(hck_rx_stall_v2->peer_ea)));
+		dhd_print_dongle_hck_id(
+				ltoh32(hck_rx_stall_v2->reason),
+				hck_rx_stall_v2_to_str);
+	} else {
+		dhd_print_dongle_hck_id(ltoh16(wl_hc->id),
+				hck_sw_id_to_str);
+	}
+
+}
+
+#endif /* PARSE_DONGLE_HOST_EVENT */
+#ifdef HCHK_COMMON_SW_EVENT
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+static void
+dhd_send_error_alert_event(dhd_pub_t *dhdp, bcm_xtlv_t *wl_hc)
+{
+	uint16 id;
+
+	id = ltoh16(wl_hc->id);
+
+	switch (id) {
+		case WL_HC_DD_RX_STALL:
+			dhdp->alert_reason = ALERT_RX_STALL;
+			break;
+		case WL_HC_DD_TX_STALL:
+			dhdp->alert_reason = ALERT_TX_STALL;
+			break;
+		case WL_HC_DD_SCAN_STALL:
+			dhdp->alert_reason = ALERT_SCAN_STALL;
+			break;
+		case WL_HC_DD_TXQ_STALL:
+			dhdp->alert_reason = ALERT_FW_QUEUE_STALL;
+			break;
+		default:
+			return;
+	}
+	dhd_os_send_alert_message(dhdp);
+}
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+#endif /* HCHK_COMMON_SW_EVENT */
+
+void
+dngl_host_event_process(dhd_pub_t *dhdp, bcm_dngl_event_t *event,
+	bcm_dngl_event_msg_t *dngl_event, size_t pktlen)
+{
+	uint8 *p = (uint8 *)(event + 1);
+	uint16 type = ntoh16_ua((void *)&dngl_event->event_type);
+	uint16 datalen = ntoh16_ua((void *)&dngl_event->datalen);
+	uint16 version = ntoh16_ua((void *)&dngl_event->version);
+	bool ignore_hc = FALSE;
+
+	DHD_EVENT(("VERSION:%d, EVENT TYPE:%d, DATALEN:%d\n", version, type, datalen));
+	if (datalen > (pktlen - sizeof(bcm_dngl_event_t) + ETHER_TYPE_LEN)) {
+		return;
+	}
+	if (version != BCM_DNGL_EVENT_MSG_VERSION) {
+		DHD_ERROR(("%s:version mismatch:%d:%d\n", __FUNCTION__,
+			version, BCM_DNGL_EVENT_MSG_VERSION));
+		return;
+	}
+	switch (type) {
+	   case DNGL_E_SOCRAM_IND:
+		{
+		   bcm_dngl_socramind_t *socramind_ptr = (bcm_dngl_socramind_t *)p;
+		   uint16 tag = ltoh32(socramind_ptr->tag);
+		   uint16 taglen = ltoh32(socramind_ptr->length);
+		   p = (uint8 *)socramind_ptr->value;
+		   DHD_EVENT(("Tag:%d Len:%d Datalen:%d\n", tag, taglen, datalen));
+		   switch (tag) {
+			case SOCRAM_IND_ASSERT_TAG:
+			    {
+				/*
+				* The payload consists of -
+				* null terminated function name padded till 32 bit boundary +
+				* Line number - (32 bits)
+				* Caller address (32 bits)
+				*/
+				char *fnname = (char *)p;
+				if (datalen < (ROUNDUP(strlen(fnname) + 1, sizeof(uint32)) +
+					sizeof(uint32) * 2)) {
+					DHD_ERROR(("Wrong length:%d\n", datalen));
+					return;
+				}
+				DHD_EVENT(("ASSRT Function:%s ", p));
+				p += ROUNDUP(strlen(p) + 1, sizeof(uint32));
+				DHD_EVENT(("Line:%d ", *(uint32 *)p));
+				p += sizeof(uint32);
+				DHD_EVENT(("Caller Addr:0x%x\n", *(uint32 *)p));
+#ifdef PARSE_DONGLE_HOST_EVENT
+				DHD_ERROR(("DONGLE_HCK_EVENT: SOCRAM_IND_ASSERT_TAG\n"));
+#endif /* PARSE_DONGLE_HOST_EVENT */
+				break;
+			    }
+			case SOCRAM_IND_TAG_HEALTH_CHECK:
+			   {
+				bcm_dngl_healthcheck_t *dngl_hc = (bcm_dngl_healthcheck_t *)p;
+				DHD_EVENT(("SOCRAM_IND_HEALTHCHECK_TAG:%d Len:%d datalen:%d\n",
+					ltoh32(dngl_hc->top_module_tag),
+					ltoh32(dngl_hc->top_module_len),
+					datalen));
+				if (DHD_EVENT_ON()) {
+					prhex("HEALTHCHECK", p, MIN(ltoh32(dngl_hc->top_module_len)
+						+ BCM_XTLV_HDR_SIZE, datalen));
+				}
+#ifdef DHD_LOG_DUMP
+				memset(dhdp->health_chk_event_data, 0, HEALTH_CHK_BUF_SIZE);
+				memcpy(dhdp->health_chk_event_data, p,
+						MIN(ltoh32(dngl_hc->top_module_len),
+						HEALTH_CHK_BUF_SIZE));
+#endif /* DHD_LOG_DUMP */
+				p = (uint8 *)dngl_hc->value;
+
+				switch (ltoh32(dngl_hc->top_module_tag)) {
+					case HEALTH_CHECK_TOP_LEVEL_MODULE_PCIEDEV_RTE:
+					   {
+						bcm_dngl_pcie_hc_t *pcie_hc;
+						pcie_hc = (bcm_dngl_pcie_hc_t *)p;
+						BCM_REFERENCE(pcie_hc);
+						if (ltoh32(dngl_hc->top_module_len) <
+								sizeof(bcm_dngl_pcie_hc_t)) {
+							DHD_ERROR(("Wrong length:%d\n",
+								ltoh32(dngl_hc->top_module_len)));
+							return;
+						}
+						DHD_EVENT(("%d:PCIE HC error:%d flag:0x%x,"
+							" control:0x%x\n",
+							ltoh32(pcie_hc->version),
+							ltoh32(pcie_hc->pcie_err_ind_type),
+							ltoh32(pcie_hc->pcie_flag),
+							ltoh32(pcie_hc->pcie_control_reg)));
+#ifdef PARSE_DONGLE_HOST_EVENT
+						dhd_print_dongle_hck_id(
+							ltoh32(pcie_hc->pcie_err_ind_type),
+								hck_pcie_module_to_str);
+#endif /* PARSE_DONGLE_HOST_EVENT */
+						break;
+					   }
+#ifdef HCHK_COMMON_SW_EVENT
+					case HCHK_SW_ENTITY_WL_PRIMARY:
+					case HCHK_SW_ENTITY_WL_SECONDARY:
+					{
+						bcm_xtlv_t *wl_hc = (bcm_xtlv_t*)p;
+
+						if (ltoh32(dngl_hc->top_module_len) <
+								sizeof(bcm_xtlv_t)) {
+							DHD_ERROR(("WL SW HC Wrong length:%d\n",
+								ltoh32(dngl_hc->top_module_len)));
+							return;
+						}
+						BCM_REFERENCE(wl_hc);
+						DHD_EVENT(("WL SW HC type %d len %d\n",
+						ltoh16(wl_hc->id), ltoh16(wl_hc->len)));
+#ifdef DHD_LB_STATS
+						if ((ltoh16(wl_hc->id) == WL_HC_DD_RX_DMA_STALL) &&
+							(dhdp->lb_rxp_stop_thr_hitcnt >
+							dhdp->lb_rxp_strt_thr_hitcnt)) {
+							ignore_hc = TRUE;
+							dhdp->rx_dma_stall_hc_ignore_cnt++;
+							DHD_ERROR(("%s:lb_rxp_stop_thr_hitcnt(%llu)"
+								" > lb_rxp_strt_thr_hitcnt(%llu)"
+								" rx_dma_stall_hc_ignore_cnt(%llu)"
+								"\n",
+								__FUNCTION__,
+								dhdp->lb_rxp_stop_thr_hitcnt,
+								dhdp->lb_rxp_strt_thr_hitcnt,
+								dhdp->rx_dma_stall_hc_ignore_cnt));
+						}
+#endif /* DHD_LB_STATS */
+
+#ifdef PARSE_DONGLE_HOST_EVENT
+						dhd_parse_hck_common_sw_event(wl_hc);
+#endif /* PARSE_DONGLE_HOST_EVENT */
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+						dhd_send_error_alert_event(dhdp, wl_hc);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+						break;
+
+					}
+#endif /* HCHK_COMMON_SW_EVENT */
+					default:
+					{
+						DHD_ERROR(("%s:Unknown module TAG:%d\n",
+						  __FUNCTION__,
+						  ltoh32(dngl_hc->top_module_tag)));
+						break;
+					}
+				}
+				break;
+			   }
+			default:
+			   DHD_ERROR(("%s:Unknown TAG\n", __FUNCTION__));
+			   if (p && DHD_EVENT_ON()) {
+				   prhex("SOCRAMIND", p, taglen);
+			   }
+			   break;
+		   }
+		   break;
+		}
+	case DNGL_E_SPMI_RESET_IND:
+		{
+			bcm_dngl_spmi_reset_ind_v1_t *spmi_reset_ind_v1_ptr =
+				(bcm_dngl_spmi_reset_ind_v1_t *)p;
+			uint16 ver = ltoh32(spmi_reset_ind_v1_ptr->version);
+			switch (ver) {
+			case DNGL_E_SPMI_RESET_IND_VERSION_1:
+			{
+				uint16 num_resets = ltoh32(spmi_reset_ind_v1_ptr->num_resets);
+				uint16 slave_idx = ltoh32(spmi_reset_ind_v1_ptr->slave_idx);
+				UNUSED_PARAMETER(num_resets);
+				UNUSED_PARAMETER(slave_idx);
+				DHD_EVENT(("DNGL_E_SPMI_RESET_IND resets=%u SPMI core=%u\n",
+					num_resets, slave_idx));
+				break;
+			}
+			default:
+				DHD_ERROR(("DNGL_E_SPMI_RESET_IND: unknown version\n"));
+				break;
+			}
+			break;
+		}
+	   default:
+		DHD_ERROR(("%s:Unknown DNGL Event Type:%d\n", __FUNCTION__, type));
+		if (p && DHD_EVENT_ON()) {
+			prhex("SOCRAMIND", p, datalen);
+		}
+		break;
+	}
+
+	/* Skip socram collection if ignore_hc is set */
+	if (!ignore_hc) {
+#ifndef BCMDBUS
+#ifdef DHD_FW_COREDUMP
+		if (dhdp->memdump_enabled) {
+			dhdp->memdump_type = DUMP_TYPE_DONGLE_HOST_EVENT;
+			if (
+#ifdef GDB_PROXY
+				!dhdp->gdb_proxy_active &&
+#endif /* GDB_PROXY */
+				dhd_schedule_socram_dump(dhdp)) {
+					DHD_ERROR(("%s: socram dump failed\n", __FUNCTION__));
+			}
+		}
+#else
+		dhd_dbg_send_urgent_evt(dhdp, p, datalen);
+#endif /* DHD_FW_COREDUMP */
+#endif /* !BCMDBUS */
+	}
+
+}
+
+#endif /* DNGL_EVENT_SUPPORT */
+
+/* Stub for now. Will become real function as soon as shim
+ * is being integrated to Android, Linux etc.
+ */
+#if !defined(NDIS)
+int
+wl_event_process_default(wl_event_msg_t *event, struct wl_evt_pport *evt_pport)
+{
+	return BCME_OK;
+}
+#endif
+
+int
+wl_event_process(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata,
+	uint pktlen, void **data_ptr, void *raw_event)
+{
+	wl_evt_pport_t evt_pport;
+	wl_event_msg_t event;
+	bcm_event_msg_u_t evu;
+	int ret;
+
+	/* make sure it is a BRCM event pkt and record event data */
+	ret = wl_host_event_get_data(pktdata, pktlen, &evu);
+	if (ret != BCME_OK) {
+		return ret;
+	}
+
+	memcpy(&event, &evu.event, sizeof(wl_event_msg_t));
+
+	/* convert event from network order to host order */
+	wl_event_to_host_order(&event);
+
+	/* record event params to evt_pport */
+	evt_pport.dhd_pub = dhd_pub;
+	evt_pport.ifidx = ifidx;
+	evt_pport.pktdata = pktdata;
+	evt_pport.data_ptr = data_ptr;
+	evt_pport.raw_event = raw_event;
+	evt_pport.data_len = pktlen;
+
+#if defined(WL_WLC_SHIM) && defined(WL_WLC_SHIM_EVENTS)
+	{
+		struct wl_shim_node *shim = dhd_pub_shim(dhd_pub);
+		if (shim) {
+			ret = wl_shim_event_process(shim, &event, &evt_pport);
+		} else {
+			/* events can come even before shim is initialized
+			 (when waiting for "wlc_ver" response)
+			 * handle them in a non-shim way.
+			 */
+			DHD_ERROR(("%s: Events coming before shim initialization!\n",
+				__FUNCTION__));
+			ret = wl_event_process_default(&event, &evt_pport);
+		}
+	}
+#else
+	ret = wl_event_process_default(&event, &evt_pport);
+#endif /* WL_WLC_SHIM && WL_WLC_SHIM_EVENTS */
+
+	return ret;
+} /* wl_event_process */
+
+/* Check whether packet is a BRCM event pkt. If it is, record event data. */
+int
+wl_host_event_get_data(void *pktdata, uint pktlen, bcm_event_msg_u_t *evu)
+{
+	int ret;
+
+	ret = is_wlc_event_frame(pktdata, pktlen, 0, evu);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: Invalid event frame, err = %d\n",
+			__FUNCTION__, ret));
+	}
+
+	return ret;
+}
+
+int
+wl_process_host_event(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata, uint pktlen,
+	wl_event_msg_t *event, void **data_ptr, void *raw_event)
+{
+	bcm_event_t *pvt_data = (bcm_event_t *)pktdata;
+	bcm_event_msg_u_t evu;
+	uint8 *event_data;
+	uint32 type, status, datalen, reason;
+	uint16 flags;
+	uint evlen;
+	int ret;
+	uint16 usr_subtype;
+#if defined(__linux__)
+	dhd_if_t *ifp = NULL;
+	BCM_REFERENCE(ifp);
+#endif /* DHD_POST_EAPOL_M1_AFTER_ROAM_EVT */
+
+	ret = wl_host_event_get_data(pktdata, pktlen, &evu);
+	if (ret != BCME_OK) {
+		return ret;
+	}
+
+	usr_subtype = ntoh16_ua((void *)&pvt_data->bcm_hdr.usr_subtype);
+	switch (usr_subtype) {
+	case BCMILCP_BCM_SUBTYPE_EVENT:
+		memcpy(event, &evu.event, sizeof(wl_event_msg_t));
+		*data_ptr = &pvt_data[1];
+		break;
+	case BCMILCP_BCM_SUBTYPE_DNGLEVENT:
+#ifdef DNGL_EVENT_SUPPORT
+		/* If it is a DNGL event process it first */
+		if (dngl_host_event(dhd_pub, pktdata, &evu.dngl_event, pktlen) == BCME_OK) {
+			/*
+			 * Return error purposely to prevent DNGL event being processed
+			 * as BRCM event
+			 */
+			return BCME_ERROR;
+		}
+#endif /* DNGL_EVENT_SUPPORT */
+		return BCME_NOTFOUND;
+	default:
+		return BCME_NOTFOUND;
+	}
+
+	/* start wl_event_msg process */
+	event_data = *data_ptr;
+	type = ntoh32_ua((void *)&event->event_type);
+	flags = ntoh16_ua((void *)&event->flags);
+	status = ntoh32_ua((void *)&event->status);
+	reason = ntoh32_ua((void *)&event->reason);
+	datalen = ntoh32_ua((void *)&event->datalen);
+	evlen = datalen + sizeof(bcm_event_t);
+
+	switch (type) {
+#ifdef PROP_TXSTATUS
+	case WLC_E_FIFO_CREDIT_MAP:
+		dhd_wlfc_enable(dhd_pub);
+		dhd_wlfc_FIFOcreditmap_event(dhd_pub, event_data);
+		WLFC_DBGMESG(("WLC_E_FIFO_CREDIT_MAP:(AC0,AC1,AC2,AC3),(BC_MC),(OTHER): "
+			"(%d,%d,%d,%d),(%d),(%d)\n", event_data[0], event_data[1],
+			event_data[2],
+			event_data[3], event_data[4], event_data[5]));
+		break;
+
+	case WLC_E_BCMC_CREDIT_SUPPORT:
+		dhd_wlfc_BCMCCredit_support_event(dhd_pub);
+		break;
+#ifdef LIMIT_BORROW
+	case WLC_E_ALLOW_CREDIT_BORROW:
+		dhd_wlfc_disable_credit_borrow_event(dhd_pub, event_data);
+		break;
+#endif /* LIMIT_BORROW */
+#endif /* PROP_TXSTATUS */
+
+	case WLC_E_ULP:
+		break;
+	case WLC_E_TDLS_PEER_EVENT:
+#if defined(WLTDLS) && defined(PCIE_FULL_DONGLE)
+		{
+			dhd_tdls_event_handler(dhd_pub, event);
+		}
+#endif
+		break;
+
+	case WLC_E_IF:
+		{
+		struct wl_event_data_if *ifevent = (struct wl_event_data_if *)event_data;
+#if defined(__linux__)
+		struct net_device *ndev = NULL;
+#endif /* __linux__ */
+
+		/* Ignore the event if NOIF is set */
+		if (ifevent->reserved & WLC_E_IF_FLAGS_BSSCFG_NOIF) {
+			DHD_INFO(("WLC_E_IF: NO_IF set, event Ignored\r\n"));
+			return (BCME_UNSUPPORTED);
+		}
+#ifdef PCIE_FULL_DONGLE
+		dhd_update_interface_flow_info(dhd_pub, ifevent->ifidx,
+			ifevent->opcode, ifevent->role);
+#endif
+#ifdef PROP_TXSTATUS
+		{
+			uint8* ea = pvt_data->eth.ether_dhost;
+			WLFC_DBGMESG(("WLC_E_IF: idx:%d, action:%s, iftype:%s, ["MACDBG"]\n"
+						  ,ifevent->ifidx,
+						  ((ifevent->opcode == WLC_E_IF_ADD) ? "ADD":"DEL"),
+						  ((ifevent->role == 0) ? "STA":"AP "),
+						  MAC2STRDBG(ea)));
+			(void)ea;
+
+			if (ifevent->opcode == WLC_E_IF_CHANGE)
+				dhd_wlfc_interface_event(dhd_pub,
+					eWLFC_MAC_ENTRY_ACTION_UPDATE,
+					ifevent->ifidx, ifevent->role, ea);
+			else
+				dhd_wlfc_interface_event(dhd_pub,
+					((ifevent->opcode == WLC_E_IF_ADD) ?
+					eWLFC_MAC_ENTRY_ACTION_ADD : eWLFC_MAC_ENTRY_ACTION_DEL),
+					ifevent->ifidx, ifevent->role, ea);
+
+			/* dhd already has created an interface by default, for 0 */
+			if (ifevent->ifidx == 0)
+				break;
+		}
+#endif /* PROP_TXSTATUS */
+
+		if (ifevent->ifidx > 0 && ifevent->ifidx < DHD_MAX_IFS) {
+			if (ifevent->opcode == WLC_E_IF_ADD) {
+				if (dhd_event_ifadd(dhd_pub->info, ifevent, event->ifname,
+					event->addr.octet)) {
+
+					DHD_ERROR(("%s: dhd_event_ifadd failed ifidx: %d  %s\n",
+						__FUNCTION__, ifevent->ifidx, event->ifname));
+					return (BCME_ERROR);
+				}
+#if defined(__linux__)
+				ndev = dhd_idx2net(dhd_pub, ifevent->ifidx);
+				if (ndev) {
+					dhd_clear_del_in_progress(dhd_pub, ndev);
+				}
+#endif /* __linux__ */
+			} else if (ifevent->opcode == WLC_E_IF_DEL) {
+#if defined(__linux__)
+				ndev = dhd_idx2net(dhd_pub, ifevent->ifidx);
+				if (ndev) {
+					dhd_set_del_in_progress(dhd_pub, ndev);
+					netif_tx_disable(ndev);
+				}
+#endif /* __linux__ */
+#ifdef PCIE_FULL_DONGLE
+				dhd_flow_rings_delete(dhd_pub,
+					(uint8)dhd_ifname2idx(dhd_pub->info, event->ifname));
+#endif /* PCIE_FULL_DONGLE */
+				dhd_event_ifdel(dhd_pub->info, ifevent, event->ifname,
+					event->addr.octet);
+			} else if (ifevent->opcode == WLC_E_IF_CHANGE) {
+#ifdef WL_CFG80211
+				dhd_event_ifchange(dhd_pub->info, ifevent, event->ifname,
+					event->addr.octet);
+#endif /* WL_CFG80211 */
+			}
+		} else {
+#if !defined(PROP_TXSTATUS) && !defined(PCIE_FULL_DONGLE) && defined(WL_CFG80211)
+			DHD_INFO(("%s: Invalid ifidx %d for %s\n",
+			   __FUNCTION__, ifevent->ifidx, event->ifname));
+#endif /* !PROP_TXSTATUS && !PCIE_FULL_DONGLE && WL_CFG80211 */
+		}
+			/* send up the if event: btamp user needs it */
+			*ifidx = dhd_ifname2idx(dhd_pub->info, event->ifname);
+			/* push up to external supp/auth */
+			dhd_event(dhd_pub->info, (char *)pvt_data, evlen, *ifidx);
+		break;
+	}
+
+	case WLC_E_NDIS_LINK:
+		break;
+	case WLC_E_PFN_NET_FOUND:
+	case WLC_E_PFN_SCAN_ALLGONE: /* share with WLC_E_PFN_BSSID_NET_LOST */
+	case WLC_E_PFN_NET_LOST:
+		break;
+#if defined(OEM_ANDROID) && defined(PNO_SUPPORT)
+	case WLC_E_PFN_BSSID_NET_FOUND:
+	case WLC_E_PFN_BEST_BATCHING:
+		dhd_pno_event_handler(dhd_pub, event, (void *)event_data);
+		break;
+#endif /* #if defined(OEM_ANDROID) && defined(PNO_SUPPORT) */
+#if defined(RTT_SUPPORT)
+	case WLC_E_PROXD:
+#ifndef WL_CFG80211
+#endif /* WL_CFG80211 */
+		break;
+#endif /* RTT_SUPPORT */
+		/* These are what external supplicant/authenticator wants */
+	case WLC_E_ASSOC_IND:
+	case WLC_E_AUTH_IND:
+	case WLC_E_REASSOC_IND:
+		dhd_findadd_sta(dhd_pub,
+			dhd_ifname2idx(dhd_pub->info, event->ifname),
+			&event->addr.octet);
+		break;
+#if !defined(BCMDBUS) && defined(DHD_FW_COREDUMP)
+	case WLC_E_PSM_WATCHDOG:
+		DHD_ERROR(("%s: WLC_E_PSM_WATCHDOG event received : \n", __FUNCTION__));
+		if (dhd_socram_dump(dhd_pub->bus) != BCME_OK) {
+			DHD_ERROR(("%s: socram dump ERROR : \n", __FUNCTION__));
+		}
+	break;
+#endif
+#ifdef DHD_WMF
+	case WLC_E_PSTA_PRIMARY_INTF_IND:
+		dhd_update_psta_interface_for_sta(dhd_pub, event->ifname,
+			(void *)(event->addr.octet), (void*) event_data);
+		break;
+#endif
+#ifdef BCM_ROUTER_DHD
+	case WLC_E_DPSTA_INTF_IND:
+		dhd_update_dpsta_interface_for_sta(dhd_pub, (uint8)dhd_ifname2idx(dhd_pub->info,
+			event->ifname), (void*) event_data);
+		break;
+#endif /* BCM_ROUTER_DHD */
+#if defined(DHD_MACDBG) && defined(BCMDBG)
+	case WLC_E_MACDBG:
+		dhd_macdbg_event_handler(dhd_pub, reason, event_data, datalen);
+		break;
+#endif /* defined(DHD_MACDBG) && defined(BCMDBG) */
+	case WLC_E_NATOE_NFCT:
+	break;
+	case WLC_E_SLOTTED_BSS_PEER_OP:
+		DHD_EVENT(("%s: WLC_E_SLOTTED_BSS_PEER_OP event received for peer: "
+			"" MACDBG ", status = %d\n",
+			__FUNCTION__, MAC2STRDBG(event->addr.octet), status));
+		if (status == WLC_E_STATUS_SLOTTED_PEER_ADD) {
+			dhd_findadd_sta(dhd_pub, dhd_ifname2idx(dhd_pub->info,
+				event->ifname), &event->addr.octet);
+		} else if (status == WLC_E_STATUS_SLOTTED_PEER_DEL) {
+			uint8 ifindex = (uint8)dhd_ifname2idx(dhd_pub->info, event->ifname);
+			BCM_REFERENCE(ifindex);
+			dhd_del_sta(dhd_pub, dhd_ifname2idx(dhd_pub->info,
+				event->ifname), &event->addr.octet);
+#ifdef PCIE_FULL_DONGLE
+			dhd_flow_rings_delete_for_peer(dhd_pub, ifindex,
+				(char *)&event->addr.octet[0]);
+#endif
+		} else {
+			DHD_ERROR(("%s: WLC_E_SLOTTED_BSS_PEER_OP: Status is not expected = %d\n",
+				__FUNCTION__, status));
+		}
+		break;
+#ifdef DHD_POST_EAPOL_M1_AFTER_ROAM_EVT
+	case WLC_E_REASSOC:
+		ifp = dhd_get_ifp(dhd_pub, event->ifidx);
+
+		if (!ifp)
+			break;
+
+		/* Consider both STA and GC due to CSA.
+		 * Drop EAPOL M1 frame only if roam is done to the same BSSID.
+		 * wpa_supplicant only handles different BSSID case.
+		 */
+		if ((status == WLC_E_STATUS_SUCCESS) &&
+			(IS_STA_IFACE(ndev_to_wdev(ifp->net)) || IS_P2P_GC(ndev_to_wdev(ifp->net))) &&
+			wl_cfg80211_is_event_from_connected_bssid(ifp->net, event, event->ifidx)) {
+			ifp->recv_reassoc_evt = TRUE;
+		}
+		break;
+#endif /* DHD_POST_EAPOL_M1_AFTER_ROAM_EVT */
+#if defined(CSI_SUPPORT)
+	case WLC_E_CSI:
+		dhd_csi_event_handler(dhd_pub, event, (void *)event_data);
+		break;
+#endif /* CSI_SUPPORT */
+	case WLC_E_LINK:
+#ifdef PCIE_FULL_DONGLE
+		if (dhd_update_interface_link_status(dhd_pub, (uint8)dhd_ifname2idx(dhd_pub->info,
+			event->ifname), (uint8)flags) != BCME_OK) {
+			DHD_ERROR(("%s: dhd_update_interface_link_status Failed.\n",
+				__FUNCTION__));
+			break;
+		}
+		if (!flags) {
+			DHD_ERROR(("%s: Deleting all STA from assoc list and flowrings.\n",
+				__FUNCTION__));
+			/* Delete all sta and flowrings */
+			dhd_del_all_sta(dhd_pub, dhd_ifname2idx(dhd_pub->info, event->ifname));
+			dhd_flow_rings_delete(dhd_pub, (uint8)dhd_ifname2idx(dhd_pub->info,
+				event->ifname));
+		}
+#endif /* PCIE_FULL_DONGLE */
+		/* fall through */
+		fallthrough;
+	case WLC_E_DEAUTH:
+	case WLC_E_DEAUTH_IND:
+	case WLC_E_DISASSOC:
+	case WLC_E_DISASSOC_IND:
+#ifdef PCIE_FULL_DONGLE
+		if (type != WLC_E_LINK) {
+			uint8 ifindex = (uint8)dhd_ifname2idx(dhd_pub->info, event->ifname);
+			uint8 role = dhd_flow_rings_ifindex2role(dhd_pub, ifindex);
+			uint8 del_sta = TRUE;
+#ifdef WL_CFG80211
+			if (role == WLC_E_IF_ROLE_STA &&
+				!wl_cfg80211_is_roam_offload(dhd_idx2net(dhd_pub, ifindex)) &&
+					!wl_cfg80211_is_event_from_connected_bssid(
+						dhd_idx2net(dhd_pub, ifindex), event, *ifidx)) {
+				del_sta = FALSE;
+			}
+#endif /* WL_CFG80211 */
+			DHD_EVENT(("%s: Link event %d, flags %x, status %x, "
+				"reason=%d, role %d, del_sta %d\n",
+				__FUNCTION__, type, flags, status,
+				reason, role, del_sta));
+
+			if (del_sta) {
+				DHD_EVENT(("%s: Deleting STA " MACDBG "\n",
+					__FUNCTION__, MAC2STRDBG(event->addr.octet)));
+
+				dhd_del_sta(dhd_pub, dhd_ifname2idx(dhd_pub->info,
+					event->ifname), &event->addr.octet);
+				/* Delete all flowrings for STA and P2P Client */
+				if (role == WLC_E_IF_ROLE_STA || role == WLC_E_IF_ROLE_P2P_CLIENT) {
+					dhd_flow_rings_delete(dhd_pub, ifindex);
+				} else {
+					dhd_flow_rings_delete_for_peer(dhd_pub, ifindex,
+						(char *)&event->addr.octet[0]);
+				}
+			}
+		}
+#endif /* PCIE_FULL_DONGLE */
+#ifdef DHD_POST_EAPOL_M1_AFTER_ROAM_EVT
+		ifp = dhd_get_ifp(dhd_pub, event->ifidx);
+		if (ifp) {
+			ifp->recv_reassoc_evt = FALSE;
+			ifp->post_roam_evt = FALSE;
+		}
+#endif /* DHD_POST_EAPOL_M1_AFTER_ROAM_EVT */
+		/* fall through */
+		fallthrough;
+	default:
+		*ifidx = dhd_ifname2idx(dhd_pub->info, event->ifname);
+#ifdef DHD_UPDATE_INTF_MAC
+		if ((WLC_E_LINK==type)&&(WLC_EVENT_MSG_LINK&flags)) {
+			dhd_event_ifchange(dhd_pub->info,
+			(struct wl_event_data_if *)event,
+			event->ifname,
+			event->addr.octet);
+		}
+#endif /* DHD_UPDATE_INTF_MAC */
+		/* push up to external supp/auth */
+		dhd_event(dhd_pub->info, (char *)pvt_data, evlen, *ifidx);
+		DHD_TRACE(("%s: MAC event %d, flags %x, status %x\n",
+			__FUNCTION__, type, flags, status));
+		BCM_REFERENCE(flags);
+		BCM_REFERENCE(status);
+		BCM_REFERENCE(reason);
+
+		break;
+	}
+#if defined(BCM_ROUTER_DHD) || defined(STBAP)
+	/* For routers, EAPD will be working on these events.
+	 * Overwrite interface name to that event is pushed
+	 * to host with its registered interface name
+	 */
+	memcpy(pvt_data->event.ifname, dhd_ifname(dhd_pub, *ifidx), IFNAMSIZ);
+#endif
+
+#ifdef DHD_STATUS_LOGGING
+	if (dhd_pub->statlog) {
+		dhd_statlog_process_event(dhd_pub, type, *ifidx,
+			status, reason, flags);
+	}
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef SHOW_EVENTS
+	if (DHD_FWLOG_ON() || DHD_EVENT_ON()) {
+		wl_show_host_event(dhd_pub, event,
+			(void *)event_data, raw_event, dhd_pub->enable_log);
+	}
+#endif /* SHOW_EVENTS */
+
+	return (BCME_OK);
+} /* wl_process_host_event */
+
+int
+wl_host_event(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata, uint pktlen,
+	wl_event_msg_t *event, void **data_ptr, void *raw_event)
+{
+	return wl_process_host_event(dhd_pub, ifidx, pktdata, pktlen, event, data_ptr,
+			raw_event);
+}
+
+void
+dhd_print_buf(void *pbuf, int len, int bytes_per_line)
+{
+#ifdef DHD_DEBUG
+	int i, j = 0;
+	unsigned char *buf = pbuf;
+
+	if (bytes_per_line == 0) {
+		bytes_per_line = len;
+	}
+
+	for (i = 0; i < len; i++) {
+		printf("%2.2x", *buf++);
+		j++;
+		if (j == bytes_per_line) {
+			printf("\n");
+			j = 0;
+		} else {
+			printf(":");
+		}
+	}
+	printf("\n");
+#endif /* DHD_DEBUG */
+}
+#ifndef strtoul
+#define strtoul(nptr, endptr, base) bcm_strtoul((nptr), (endptr), (base))
+#endif
+
+/* Convert user's input in hex pattern to byte-size mask */
+int
+wl_pattern_atoh(char *src, char *dst)
+{
+	int i;
+	if (strncmp(src, "0x", 2) != 0 &&
+	    strncmp(src, "0X", 2) != 0) {
+		DHD_ERROR(("Mask invalid format. Needs to start with 0x\n"));
+		return -1;
+	}
+	src = src + 2; /* Skip past 0x */
+	if (strlen(src) % 2 != 0) {
+		DHD_ERROR(("Mask invalid format. Needs to be of even length\n"));
+		return -1;
+	}
+	for (i = 0; *src != '\0'; i++) {
+		char num[3];
+		bcm_strncpy_s(num, sizeof(num), src, 2);
+		num[2] = '\0';
+		dst[i] = (uint8)strtoul(num, NULL, 16);
+		src += 2;
+	}
+	return i;
+}
+
+#if defined(PKT_FILTER_SUPPORT) || defined(DHD_PKT_LOGGING)
+int
+pattern_atoh_len(char *src, char *dst, int len)
+{
+	int i;
+	if (strncmp(src, "0x", HD_PREFIX_SIZE) != 0 &&
+			strncmp(src, "0X", HD_PREFIX_SIZE) != 0) {
+		DHD_ERROR(("Mask invalid format. Needs to start with 0x\n"));
+		return -1;
+	}
+	src = src + HD_PREFIX_SIZE; /* Skip past 0x */
+	if (strlen(src) % HD_BYTE_SIZE != 0) {
+		DHD_ERROR(("Mask invalid format. Needs to be of even length\n"));
+		return -1;
+	}
+	for (i = 0; *src != '\0'; i++) {
+		char num[HD_BYTE_SIZE + 1];
+
+		if (i > len - 1) {
+			DHD_ERROR(("pattern not in range, idx: %d len: %d\n", i, len));
+			return -1;
+		}
+		bcm_strncpy_s(num, sizeof(num), src, HD_BYTE_SIZE);
+		num[HD_BYTE_SIZE] = '\0';
+		dst[i] = (uint8)strtoul(num, NULL, 16);
+		src += HD_BYTE_SIZE;
+	}
+	return i;
+}
+#endif /* PKT_FILTER_SUPPORT || DHD_PKT_LOGGING */
+
+#ifdef PKT_FILTER_SUPPORT
+void
+dhd_pktfilter_offload_enable(dhd_pub_t * dhd, char *arg, int enable, int master_mode)
+{
+	char				*argv[8];
+	int					i = 0;
+	const char			*str;
+	int					buf_len;
+	int					str_len;
+	char				*arg_save = 0, *arg_org = 0;
+	int					rc;
+	char				buf[32] = {0};
+	wl_pkt_filter_enable_t	enable_parm;
+	wl_pkt_filter_enable_t	* pkt_filterp;
+
+	if (!arg)
+		return;
+
+	if (!(arg_save = MALLOC(dhd->osh, strlen(arg) + 1))) {
+		DHD_ERROR(("%s: malloc failed\n", __FUNCTION__));
+		goto fail;
+	}
+	arg_org = arg_save;
+	memcpy(arg_save, arg, strlen(arg) + 1);
+
+	argv[i] = bcmstrtok(&arg_save, " ", 0);
+
+	i = 0;
+	if (argv[i] == NULL) {
+		DHD_ERROR(("No args provided\n"));
+		goto fail;
+	}
+
+	str = "pkt_filter_enable";
+	str_len = strlen(str);
+	bcm_strncpy_s(buf, sizeof(buf) - 1, str, sizeof(buf) - 1);
+	buf[ sizeof(buf) - 1 ] = '\0';
+	buf_len = str_len + 1;
+
+	pkt_filterp = (wl_pkt_filter_enable_t *)(buf + str_len + 1);
+
+	/* Parse packet filter id. */
+	enable_parm.id = htod32(strtoul(argv[i], NULL, 0));
+	if (dhd_conf_del_pkt_filter(dhd, enable_parm.id))
+		goto fail;
+
+	/* Parse enable/disable value. */
+	enable_parm.enable = htod32(enable);
+
+	buf_len += sizeof(enable_parm);
+	memcpy((char *)pkt_filterp,
+	       &enable_parm,
+	       sizeof(enable_parm));
+
+	/* Enable/disable the specified filter. */
+	rc = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, buf_len, TRUE, 0);
+	rc = rc >= 0 ? 0 : rc;
+	if (rc) {
+		DHD_ERROR(("%s: failed to %s pktfilter %s, retcode = %d\n",
+		__FUNCTION__, enable?"enable":"disable", arg, rc));
+		dhd_set_packet_filter(dhd);
+		rc = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, buf_len, TRUE, 0);
+		rc = rc >= 0 ? 0 : rc;
+		if (rc) {
+			DHD_TRACE_HW4(("%s: 2nd retry failed to add pktfilter %s, retcode = %d\n",
+			__FUNCTION__, arg, rc));
+		} else {
+			DHD_TRACE_HW4(("%s: 2nd retry successfully added pktfilter %s\n",
+			__FUNCTION__, arg));
+		}
+	}
+	else
+		DHD_TRACE(("%s: successfully %s pktfilter %s\n",
+		__FUNCTION__, enable?"enable":"disable", arg));
+
+	/* Contorl the master mode */
+	rc = dhd_wl_ioctl_set_intiovar(dhd, "pkt_filter_mode",
+		master_mode, WLC_SET_VAR, TRUE, 0);
+	rc = rc >= 0 ? 0 : rc;
+	if (rc)
+		DHD_TRACE(("%s: failed to set pkt_filter_mode %d, retcode = %d\n",
+			__FUNCTION__, master_mode, rc));
+
+fail:
+	if (arg_org)
+		MFREE(dhd->osh, arg_org, strlen(arg) + 1);
+}
+
+/* Packet filter section: extended filters have named offsets, add table here */
+typedef struct {
+	char *name;
+	uint16 base;
+} wl_pfbase_t;
+
+static wl_pfbase_t basenames[] = { WL_PKT_FILTER_BASE_NAMES };
+
+static int
+wl_pkt_filter_base_parse(char *name)
+{
+	uint i;
+	char *bname, *uname;
+
+	for (i = 0; i < ARRAYSIZE(basenames); i++) {
+		bname = basenames[i].name;
+		for (uname = name; *uname; bname++, uname++) {
+			if (*bname != bcm_toupper(*uname)) {
+				break;
+			}
+		}
+		if (!*uname && !*bname) {
+			break;
+		}
+	}
+
+	if (i < ARRAYSIZE(basenames)) {
+		return basenames[i].base;
+	} else {
+		return -1;
+	}
+}
+
+void
+dhd_pktfilter_offload_set(dhd_pub_t * dhd, char *arg)
+{
+	const char			*str;
+	wl_pkt_filter_t			pkt_filter;
+	wl_pkt_filter_t			*pkt_filterp;
+	int				buf_len;
+	int				str_len;
+	int				rc = -1;
+	uint32				mask_size;
+	uint32				pattern_size;
+	char				*argv[MAXPKT_ARG] = {0}, * buf = 0;
+	int				i = 0;
+	char				*arg_save = 0, *arg_org = 0;
+
+	if (!arg)
+		return;
+
+	if (!(arg_save = MALLOC(dhd->osh, strlen(arg) + 1))) {
+		DHD_ERROR(("%s: malloc failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	arg_org = arg_save;
+
+	if (!(buf = MALLOC(dhd->osh, MAX_PKTFLT_BUF_SIZE))) {
+		DHD_ERROR(("%s: malloc failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	memset(buf, 0, MAX_PKTFLT_BUF_SIZE);
+	memcpy(arg_save, arg, strlen(arg) + 1);
+
+	if (strlen(arg) > MAX_PKTFLT_BUF_SIZE) {
+		DHD_ERROR(("Not enough buffer %d < %d\n", (int)strlen(arg), (int)sizeof(buf)));
+		goto fail;
+	}
+
+	argv[i] = bcmstrtok(&arg_save, " ", 0);
+	while (argv[i++]) {
+		if (i >= MAXPKT_ARG) {
+			DHD_ERROR(("Invalid args provided\n"));
+			goto fail;
+		}
+		argv[i] = bcmstrtok(&arg_save, " ", 0);
+	}
+
+	i = 0;
+	if (argv[i] == NULL) {
+		DHD_ERROR(("No args provided\n"));
+		goto fail;
+	}
+
+	str = "pkt_filter_add";
+	str_len = strlen(str);
+	bcm_strncpy_s(buf, MAX_PKTFLT_BUF_SIZE, str, str_len);
+	buf[ str_len ] = '\0';
+	buf_len = str_len + 1;
+
+	pkt_filterp = (wl_pkt_filter_t *) (buf + str_len + 1);
+
+	/* Parse packet filter id. */
+	pkt_filter.id = htod32(strtoul(argv[i], NULL, 0));
+
+	if (argv[++i] == NULL) {
+		DHD_ERROR(("Polarity not provided\n"));
+		goto fail;
+	}
+
+	/* Parse filter polarity. */
+	pkt_filter.negate_match = htod32(strtoul(argv[i], NULL, 0));
+
+	if (argv[++i] == NULL) {
+		DHD_ERROR(("Filter type not provided\n"));
+		goto fail;
+	}
+
+	/* Parse filter type. */
+	pkt_filter.type = htod32(strtoul(argv[i], NULL, 0));
+
+	if ((pkt_filter.type == 0) || (pkt_filter.type == 1)) {
+		if (argv[++i] == NULL) {
+			DHD_ERROR(("Offset not provided\n"));
+			goto fail;
+		}
+
+		/* Parse pattern filter offset. */
+		pkt_filter.u.pattern.offset = htod32(strtoul(argv[i], NULL, 0));
+
+		if (argv[++i] == NULL) {
+			DHD_ERROR(("Bitmask not provided\n"));
+			goto fail;
+		}
+
+		/* Parse pattern filter mask. */
+		rc  = wl_pattern_atoh(argv[i],
+			(char *) pkt_filterp->u.pattern.mask_and_pattern);
+
+		if (rc == -1) {
+			DHD_ERROR(("Rejecting: %s\n", argv[i]));
+			goto fail;
+		}
+		mask_size = htod32(rc);
+		if (argv[++i] == NULL) {
+			DHD_ERROR(("Pattern not provided\n"));
+			goto fail;
+		}
+
+		/* Parse pattern filter pattern. */
+		rc = wl_pattern_atoh(argv[i],
+			(char *) &pkt_filterp->u.pattern.mask_and_pattern[rc]);
+
+		if (rc == -1) {
+			DHD_ERROR(("Rejecting: %s\n", argv[i]));
+			goto fail;
+		}
+		pattern_size = htod32(rc);
+		if (mask_size != pattern_size) {
+			DHD_ERROR(("Mask and pattern not the same size\n"));
+			goto fail;
+		}
+
+		pkt_filter.u.pattern.size_bytes = mask_size;
+		buf_len += WL_PKT_FILTER_FIXED_LEN;
+		buf_len += (WL_PKT_FILTER_PATTERN_FIXED_LEN + 2 * rc);
+
+		/* Keep-alive attributes are set in local	variable (keep_alive_pkt), and
+		 * then memcpy'ed into buffer (keep_alive_pktp) since there is no
+		 * guarantee that the buffer is properly aligned.
+		 */
+		memcpy((char *)pkt_filterp,
+			&pkt_filter,
+			WL_PKT_FILTER_FIXED_LEN + WL_PKT_FILTER_PATTERN_FIXED_LEN);
+	} else if ((pkt_filter.type == 2) || (pkt_filter.type == 6)) {
+		int list_cnt = 0;
+		char *endptr = NULL;
+		wl_pkt_filter_pattern_listel_t *pf_el =
+			(wl_pkt_filter_pattern_listel_t *)&pkt_filterp->u.patlist.patterns[0];
+
+		while (argv[++i] != NULL) {
+			/* Check valid buffer size. */
+			if ((buf_len + MAX_PKTFLT_FIXED_BUF_SIZE) > MAX_PKTFLT_BUF_SIZE) {
+				DHD_ERROR(("buffer over length MAX_PKTFLT_FIXED_BUF_SIZE\n"));
+				goto fail;
+			}
+
+			/* Parse pattern filter base and offset. */
+			if (bcm_isdigit(*argv[i])) {
+				/* Numeric base */
+				rc = strtoul(argv[i], &endptr, 0);
+			} else {
+				endptr = strchr(argv[i], ':');
+				if (endptr) {
+					*endptr = '\0';
+					rc = wl_pkt_filter_base_parse(argv[i]);
+					if (rc == -1) {
+						DHD_ERROR(("Invalid base %s\n", argv[i]));
+						goto fail;
+					}
+					*endptr = ':';
+				}
+			}
+
+			if (endptr == NULL) {
+				DHD_ERROR(("Invalid [base:]offset format: %s\n", argv[i]));
+				goto fail;
+			}
+
+			if (*endptr == ':') {
+				pf_el->base_offs = htod16(rc);
+				rc = strtoul(endptr + 1, &endptr, 0);
+			} else {
+				/* Must have had a numeric offset only */
+				pf_el->base_offs = htod16(0);
+			}
+
+			if (*endptr) {
+				DHD_ERROR(("Invalid [base:]offset format: %s\n", argv[i]));
+				goto fail;
+			}
+			if (rc > 0x0000FFFF) {
+				DHD_ERROR(("Offset too large\n"));
+				goto fail;
+			}
+			pf_el->rel_offs = htod16(rc);
+
+			/* Clear match_flag (may be set in parsing which follows) */
+			pf_el->match_flags = htod16(0);
+
+			/* Parse pattern filter mask and pattern directly into ioctl buffer */
+			if (argv[++i] == NULL) {
+				DHD_ERROR(("Bitmask not provided\n"));
+				goto fail;
+			}
+			rc = wl_pattern_atoh(argv[i], (char*)pf_el->mask_and_data);
+			if ((rc == -1) || (rc > MAX_PKTFLT_FIXED_PATTERN_SIZE)) {
+				DHD_ERROR(("Rejecting: %s\n", argv[i]));
+				goto fail;
+			}
+			mask_size = htod16(rc);
+
+			if (argv[++i] == NULL) {
+				DHD_ERROR(("Pattern not provided\n"));
+				goto fail;
+			}
+
+			endptr = argv[i];
+			if (*endptr == '!') {
+				pf_el->match_flags =
+					htod16(WL_PKT_FILTER_MFLAG_NEG);
+				if (*(++endptr) == '\0') {
+					DHD_ERROR(("Pattern not provided\n"));
+					goto fail;
+				}
+			}
+			rc = wl_pattern_atoh(endptr, (char*)&pf_el->mask_and_data[rc]);
+			if ((rc == -1) || (rc > MAX_PKTFLT_FIXED_PATTERN_SIZE)) {
+				DHD_ERROR(("Rejecting: %s\n", argv[i]));
+				goto fail;
+			}
+			pattern_size = htod16(rc);
+
+			if (mask_size != pattern_size) {
+				DHD_ERROR(("Mask and pattern not the same size\n"));
+				goto fail;
+			}
+
+			pf_el->size_bytes = mask_size;
+
+			/* Account for the size of this pattern element */
+			buf_len += WL_PKT_FILTER_PATTERN_LISTEL_FIXED_LEN + 2 * rc;
+
+			/* Move to next element location in ioctl buffer */
+			pf_el = (wl_pkt_filter_pattern_listel_t*)
+				((uint8*)pf_el + WL_PKT_FILTER_PATTERN_LISTEL_FIXED_LEN + 2 * rc);
+
+			/* Count list element */
+			list_cnt++;
+		}
+
+		/* Account for initial fixed size, and copy initial fixed fields */
+		buf_len += WL_PKT_FILTER_FIXED_LEN + WL_PKT_FILTER_PATTERN_LIST_FIXED_LEN;
+
+		if (buf_len > MAX_PKTFLT_BUF_SIZE) {
+			DHD_ERROR(("buffer over length MAX_PKTFLT_BUF_SIZE\n"));
+			goto fail;
+		}
+
+		/* Update list count and total size */
+		pkt_filter.u.patlist.list_cnt = list_cnt;
+		pkt_filter.u.patlist.PAD1[0] = 0;
+		pkt_filter.u.patlist.totsize = buf + buf_len - (char*)pkt_filterp;
+		pkt_filter.u.patlist.totsize -= WL_PKT_FILTER_FIXED_LEN;
+
+		memcpy((char *)pkt_filterp, &pkt_filter,
+			WL_PKT_FILTER_FIXED_LEN + WL_PKT_FILTER_PATTERN_LIST_FIXED_LEN);
+	} else {
+		DHD_ERROR(("Invalid filter type %d\n", pkt_filter.type));
+		goto fail;
+	}
+
+	rc = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, buf_len, TRUE, 0);
+	rc = rc >= 0 ? 0 : rc;
+
+	if (rc)
+		DHD_ERROR(("%s: failed to add pktfilter %s, retcode = %d\n",
+		__FUNCTION__, arg, rc));
+	else
+		DHD_TRACE(("%s: successfully added pktfilter %s\n",
+		__FUNCTION__, arg));
+
+fail:
+	if (arg_org)
+		MFREE(dhd->osh, arg_org, strlen(arg) + 1);
+
+	if (buf)
+		MFREE(dhd->osh, buf, MAX_PKTFLT_BUF_SIZE);
+}
+
+void
+dhd_pktfilter_offload_delete(dhd_pub_t *dhd, int id)
+{
+	int ret;
+
+	ret = dhd_wl_ioctl_set_intiovar(dhd, "pkt_filter_delete",
+		id, WLC_SET_VAR, TRUE, 0);
+	if (ret < 0) {
+		DHD_ERROR(("%s: Failed to delete filter ID:%d, ret=%d\n",
+			__FUNCTION__, id, ret));
+	}
+	else
+		DHD_TRACE(("%s: successfully deleted pktfilter %d\n",
+		__FUNCTION__, id));
+}
+#endif /* PKT_FILTER_SUPPORT */
+
+/* ========================== */
+/* ==== ARP OFFLOAD SUPPORT = */
+/* ========================== */
+#ifdef ARP_OFFLOAD_SUPPORT
+void
+dhd_arp_offload_set(dhd_pub_t * dhd, int arp_mode)
+{
+	int retcode;
+
+	retcode = dhd_wl_ioctl_set_intiovar(dhd, "arp_ol",
+		arp_mode, WLC_SET_VAR, TRUE, 0);
+
+	retcode = retcode >= 0 ? 0 : retcode;
+	if (retcode) {
+		DHD_ERROR(("%s: failed to set ARP offload mode to 0x%x, retcode = %d\n",
+			__FUNCTION__, arp_mode, retcode));
+	} else {
+		DHD_ARPOE(("%s: successfully set ARP offload mode to 0x%x\n",
+			__FUNCTION__, arp_mode));
+		dhd->arpol_configured = TRUE;
+	}
+}
+
+void
+dhd_arp_offload_enable(dhd_pub_t * dhd, int arp_enable)
+{
+	int retcode;
+
+	if (!dhd->arpol_configured) {
+		/* If arpol is not applied, apply it */
+		dhd_arp_offload_set(dhd, dhd_arp_mode);
+	}
+
+	retcode = dhd_wl_ioctl_set_intiovar(dhd, "arpoe",
+		arp_enable, WLC_SET_VAR, TRUE, 0);
+	retcode = retcode >= 0 ? 0 : retcode;
+	if (retcode)
+		DHD_ERROR(("%s: failed to enabe ARP offload to %d, retcode = %d\n",
+			__FUNCTION__, arp_enable, retcode));
+	else
+#ifdef DHD_LOG_DUMP
+		DHD_LOG_MEM(("%s: successfully enabed ARP offload to %d\n",
+			__FUNCTION__, arp_enable));
+#else
+		DHD_ARPOE(("%s: successfully enabed ARP offload to %d\n",
+			__FUNCTION__, arp_enable));
+#endif /* DHD_LOG_DUMP */
+	if (arp_enable) {
+		uint32 version;
+		retcode = dhd_wl_ioctl_get_intiovar(dhd, "arp_version",
+			&version, WLC_GET_VAR, FALSE, 0);
+		if (retcode) {
+			DHD_INFO(("%s: fail to get version (maybe version 1:retcode = %d\n",
+				__FUNCTION__, retcode));
+			dhd->arp_version = 1;
+		}
+		else {
+			DHD_INFO(("%s: ARP Version= %x\n", __FUNCTION__, version));
+			dhd->arp_version = version;
+		}
+	}
+}
+
+/* XXX ANDREY: clear AOE arp_table  */
+void
+dhd_aoe_arp_clr(dhd_pub_t *dhd, int idx)
+{
+	int ret = 0;
+
+	if (dhd == NULL) return;
+	if (dhd->arp_version == 1)
+		idx = 0;
+
+	ret = dhd_iovar(dhd, idx, "arp_table_clear", NULL, 0, NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s failed code %d\n", __FUNCTION__, ret));
+	else {
+#ifdef DHD_LOG_DUMP
+		DHD_LOG_MEM(("%s: ARP table clear\n", __FUNCTION__));
+#else
+		DHD_TRACE(("%s: ARP table clear\n", __FUNCTION__));
+#endif /* DHD_LOG_DUMP */
+	}
+	/* mac address isn't cleared here but it will be cleared after dongle off */
+	dhd->hmac_updated = 0;
+}
+
+/* XXX ANDREY: clear hostip table  */
+void
+dhd_aoe_hostip_clr(dhd_pub_t *dhd, int idx)
+{
+	int ret = 0;
+
+	if (dhd == NULL) return;
+	if (dhd->arp_version == 1)
+		idx = 0;
+
+	ret = dhd_iovar(dhd, idx, "arp_hostip_clear", NULL, 0, NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s failed code %d\n", __FUNCTION__, ret));
+	else {
+#ifdef DHD_LOG_DUMP
+		DHD_LOG_MEM(("%s: ARP host ip clear\n", __FUNCTION__));
+#else
+		DHD_TRACE(("%s: ARP host ip clear\n", __FUNCTION__));
+#endif /* DHD_LOG_DUMP */
+	}
+}
+
+void
+dhd_arp_offload_add_ip(dhd_pub_t *dhd, uint32 ipaddr, int idx)
+{
+	int ret;
+
+	if (dhd == NULL) return;
+	if (dhd->arp_version == 1)
+		idx = 0;
+
+	ret = dhd_iovar(dhd, idx, "arp_hostip", (char *)&ipaddr, sizeof(ipaddr),
+			NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s: ARP ip addr add failed, ret = %d\n", __FUNCTION__, ret));
+	else {
+		/* mac address is updated in the dongle */
+		dhd->hmac_updated = 1;
+#ifdef DHD_LOG_DUMP
+		DHD_LOG_MEM(("%s: ARP ip addr entry added \n", __FUNCTION__));
+#else
+		DHD_ARPOE(("%s: ARP ip addr entry added \n", __FUNCTION__));
+#endif /* DHD_LOG_DUMP */
+	}
+}
+
+int
+dhd_arp_get_arp_hostip_table(dhd_pub_t *dhd, void *buf, int buflen, int idx)
+{
+	int ret, i;
+	uint32 *ptr32 = buf;
+	bool clr_bottom = FALSE;
+
+	if (!buf)
+		return -1;
+	if (dhd == NULL) return -1;
+	if (dhd->arp_version == 1)
+		idx = 0;
+
+	ret = dhd_iovar(dhd, idx, "arp_hostip", NULL, 0, (char *)buf, buflen,
+			FALSE);
+	if (ret) {
+		DHD_ERROR(("%s: ioctl WLC_GET_VAR error %d\n",
+		__FUNCTION__, ret));
+
+		return -1;
+	}
+
+	/* clean up the buf, ascii reminder */
+	for (i = 0; i < MAX_IPV4_ENTRIES; i++) {
+		if (!clr_bottom) {
+			if (*ptr32 == 0)
+				clr_bottom = TRUE;
+		} else {
+			*ptr32 = 0;
+		}
+		ptr32++;
+	}
+
+	return 0;
+}
+#endif /* ARP_OFFLOAD_SUPPORT  */
+
+/*
+ * Neighbor Discovery Offload: enable NDO feature
+ * Called  by ipv6 event handler when interface comes up/goes down
+ */
+int
+dhd_ndo_enable(dhd_pub_t * dhd, int ndo_enable)
+{
+	int retcode;
+
+	if (dhd == NULL)
+		return -1;
+
+#if defined(WL_CFG80211) && defined(WL_NAN)
+	if (wl_cfgnan_is_dp_active(dhd_linux_get_primary_netdev(dhd))) {
+		/* If nan dp is active, skip NDO */
+		DHD_INFO(("Active NAN DP, skip NDO\n"));
+		return 0;
+	}
+#endif /* WL_CFG80211 && WL_NAN */
+#ifdef WL_CFG80211
+	if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+		/* NDO disable on STA+SOFTAP mode */
+		ndo_enable = FALSE;
+	}
+#endif /* WL_CFG80211 */
+	retcode = dhd_wl_ioctl_set_intiovar(dhd, "ndoe",
+		ndo_enable, WLC_SET_VAR, TRUE, 0);
+	if (retcode)
+		DHD_ERROR(("%s: failed to enabe ndo to %d, retcode = %d\n",
+			__FUNCTION__, ndo_enable, retcode));
+	else
+		DHD_TRACE(("%s: successfully enabed ndo offload to %d\n",
+			__FUNCTION__, ndo_enable));
+
+	return retcode;
+}
+
+/*
+ * Neighbor Discover Offload: enable NDO feature
+ * Called  by ipv6 event handler when interface comes up
+ */
+int
+dhd_ndo_add_ip(dhd_pub_t *dhd, char* ipv6addr, int idx)
+{
+	int iov_len = 0;
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int retcode;
+
+	if (dhd == NULL)
+		return -1;
+
+	iov_len = bcm_mkiovar("nd_hostip", (char *)ipv6addr,
+		IPV6_ADDR_LEN, iovbuf, sizeof(iovbuf));
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return -1;
+	}
+	retcode = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, iov_len, TRUE, idx);
+
+	if (retcode)
+		DHD_ERROR(("%s: ndo ip addr add failed, retcode = %d\n",
+		__FUNCTION__, retcode));
+	else
+		DHD_TRACE(("%s: ndo ipaddr entry added \n",
+		__FUNCTION__));
+
+	return retcode;
+}
+
+/*
+ * Neighbor Discover Offload: enable NDO feature
+ * Called  by ipv6 event handler when interface goes down
+ */
+int
+dhd_ndo_remove_ip(dhd_pub_t *dhd, int idx)
+{
+	int iov_len = 0;
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int retcode;
+
+	if (dhd == NULL)
+		return -1;
+
+	iov_len = bcm_mkiovar("nd_hostip_clear", NULL,
+		0, iovbuf, sizeof(iovbuf));
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return -1;
+	}
+	retcode = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, iov_len, TRUE, idx);
+
+	if (retcode)
+		DHD_ERROR(("%s: ndo ip addr remove failed, retcode = %d\n",
+		__FUNCTION__, retcode));
+	else
+		DHD_TRACE(("%s: ndo ipaddr entry removed \n",
+		__FUNCTION__));
+
+	return retcode;
+}
+/* Enhanced ND offload */
+uint16
+dhd_ndo_get_version(dhd_pub_t *dhdp)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	wl_nd_hostip_t ndo_get_ver;
+	int iov_len;
+	int retcode;
+	uint16 ver = 0;
+
+	if (dhdp == NULL) {
+		return BCME_ERROR;
+	}
+
+	memset(&iovbuf, 0, sizeof(iovbuf));
+	ndo_get_ver.version = htod16(WL_ND_HOSTIP_IOV_VER);
+	ndo_get_ver.op_type = htod16(WL_ND_HOSTIP_OP_VER);
+	ndo_get_ver.length = htod32(WL_ND_HOSTIP_FIXED_LEN + sizeof(uint16));
+	ndo_get_ver.u.version = 0;
+	iov_len = bcm_mkiovar("nd_hostip", (char *)&ndo_get_ver,
+		WL_ND_HOSTIP_FIXED_LEN + sizeof(uint16), iovbuf, sizeof(iovbuf));
+
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return BCME_ERROR;
+	}
+
+	retcode = dhd_wl_ioctl_cmd(dhdp, WLC_GET_VAR, iovbuf, iov_len, FALSE, 0);
+
+	if (retcode) {
+		DHD_ERROR(("%s: failed, retcode = %d\n", __FUNCTION__, retcode));
+		/* ver iovar not supported. NDO version is 0 */
+		ver = 0;
+	} else {
+		wl_nd_hostip_t *ndo_ver_ret = (wl_nd_hostip_t *)iovbuf;
+
+		if ((dtoh16(ndo_ver_ret->version) == WL_ND_HOSTIP_IOV_VER) &&
+				(dtoh16(ndo_ver_ret->op_type) == WL_ND_HOSTIP_OP_VER) &&
+				(dtoh32(ndo_ver_ret->length) == WL_ND_HOSTIP_FIXED_LEN
+					+ sizeof(uint16))) {
+			/* nd_hostip iovar version */
+			ver = dtoh16(ndo_ver_ret->u.version);
+		}
+
+		DHD_TRACE(("%s: successfully get version: %d\n", __FUNCTION__, ver));
+	}
+
+	return ver;
+}
+
+int
+dhd_ndo_add_ip_with_type(dhd_pub_t *dhdp, char *ipv6addr, uint8 type, int idx)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	wl_nd_hostip_t ndo_add_addr;
+	int iov_len;
+	int retcode;
+
+	if (dhdp == NULL || ipv6addr == 0) {
+		return BCME_ERROR;
+	}
+
+	/* wl_nd_hostip_t fixed param */
+	ndo_add_addr.version = htod16(WL_ND_HOSTIP_IOV_VER);
+	ndo_add_addr.op_type = htod16(WL_ND_HOSTIP_OP_ADD);
+	ndo_add_addr.length = htod32(WL_ND_HOSTIP_WITH_ADDR_LEN);
+	/* wl_nd_host_ip_addr_t param for add */
+	memcpy(&ndo_add_addr.u.host_ip.ip_addr, ipv6addr, IPV6_ADDR_LEN);
+	ndo_add_addr.u.host_ip.type = type;
+
+	iov_len = bcm_mkiovar("nd_hostip", (char *)&ndo_add_addr,
+		WL_ND_HOSTIP_WITH_ADDR_LEN, iovbuf, sizeof(iovbuf));
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return BCME_ERROR;
+	}
+
+	retcode = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, iovbuf, iov_len, TRUE, idx);
+	if (retcode) {
+		DHD_ERROR(("%s: failed, retcode = %d\n", __FUNCTION__, retcode));
+#ifdef NDO_CONFIG_SUPPORT
+		if (retcode == BCME_NORESOURCE) {
+			/* number of host ip addr exceeds FW capacity, Deactivate ND offload */
+			DHD_INFO(("%s: Host IP count exceed device capacity,"
+				"ND offload deactivated\n", __FUNCTION__));
+			dhdp->ndo_host_ip_overflow = TRUE;
+			dhd_ndo_enable(dhdp, FALSE);
+		}
+#endif /* NDO_CONFIG_SUPPORT */
+	} else {
+		DHD_TRACE(("%s: successfully added: %d\n", __FUNCTION__, retcode));
+	}
+
+	return retcode;
+}
+
+int
+dhd_ndo_remove_ip_by_addr(dhd_pub_t *dhdp, char *ipv6addr, int idx)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	wl_nd_hostip_t ndo_del_addr;
+	int iov_len;
+	int retcode;
+
+	if (dhdp == NULL || ipv6addr == 0) {
+		return BCME_ERROR;
+	}
+
+	/* wl_nd_hostip_t fixed param */
+	ndo_del_addr.version = htod16(WL_ND_HOSTIP_IOV_VER);
+	ndo_del_addr.op_type = htod16(WL_ND_HOSTIP_OP_DEL);
+	ndo_del_addr.length = htod32(WL_ND_HOSTIP_WITH_ADDR_LEN);
+	/* wl_nd_host_ip_addr_t param for del */
+	memcpy(&ndo_del_addr.u.host_ip.ip_addr, ipv6addr, IPV6_ADDR_LEN);
+	ndo_del_addr.u.host_ip.type = 0;	/* don't care */
+
+	iov_len = bcm_mkiovar("nd_hostip", (char *)&ndo_del_addr,
+		WL_ND_HOSTIP_WITH_ADDR_LEN, iovbuf, sizeof(iovbuf));
+
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return BCME_ERROR;
+	}
+
+	retcode = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, iovbuf, iov_len, TRUE, idx);
+	if (retcode) {
+		DHD_ERROR(("%s: failed, retcode = %d\n", __FUNCTION__, retcode));
+	} else {
+		DHD_TRACE(("%s: successfully removed: %d\n", __FUNCTION__, retcode));
+	}
+
+	return retcode;
+}
+
+int
+dhd_ndo_remove_ip_by_type(dhd_pub_t *dhdp, uint8 type, int idx)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	wl_nd_hostip_t ndo_del_addr;
+	int iov_len;
+	int retcode;
+
+	if (dhdp == NULL) {
+		return BCME_ERROR;
+	}
+
+	/* wl_nd_hostip_t fixed param */
+	ndo_del_addr.version = htod16(WL_ND_HOSTIP_IOV_VER);
+	if (type == WL_ND_IPV6_ADDR_TYPE_UNICAST) {
+		ndo_del_addr.op_type = htod16(WL_ND_HOSTIP_OP_DEL_UC);
+	} else if (type == WL_ND_IPV6_ADDR_TYPE_ANYCAST) {
+		ndo_del_addr.op_type = htod16(WL_ND_HOSTIP_OP_DEL_AC);
+	} else {
+		return BCME_BADARG;
+	}
+	ndo_del_addr.length = htod32(WL_ND_HOSTIP_FIXED_LEN);
+
+	iov_len = bcm_mkiovar("nd_hostip", (char *)&ndo_del_addr, WL_ND_HOSTIP_FIXED_LEN,
+			iovbuf, sizeof(iovbuf));
+
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return BCME_ERROR;
+	}
+
+	retcode = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, iovbuf, iov_len, TRUE, idx);
+	if (retcode) {
+		DHD_ERROR(("%s: failed, retcode = %d\n", __FUNCTION__, retcode));
+	} else {
+		DHD_TRACE(("%s: successfully removed: %d\n", __FUNCTION__, retcode));
+	}
+
+	return retcode;
+}
+
+int
+dhd_ndo_unsolicited_na_filter_enable(dhd_pub_t *dhdp, int enable)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int iov_len;
+	int retcode;
+
+	if (dhdp == NULL) {
+		return BCME_ERROR;
+	}
+
+	iov_len = bcm_mkiovar("nd_unsolicited_na_filter", (char *)&enable, sizeof(int),
+			iovbuf, sizeof(iovbuf));
+
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %zu \n",
+			__FUNCTION__, sizeof(iovbuf)));
+		return BCME_ERROR;
+	}
+
+	retcode = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, iovbuf, iov_len, TRUE, 0);
+	if (retcode)
+		DHD_ERROR(("%s: failed to enable Unsolicited NA filter to %d, retcode = %d\n",
+			__FUNCTION__, enable, retcode));
+	else {
+		DHD_TRACE(("%s: successfully enabled Unsolicited NA filter to %d\n",
+			__FUNCTION__, enable));
+	}
+
+	return retcode;
+}
+#ifdef SIMPLE_ISCAN
+
+uint iscan_thread_id = 0;
+iscan_buf_t * iscan_chain = 0;
+
+iscan_buf_t *
+dhd_iscan_allocate_buf(dhd_pub_t *dhd, iscan_buf_t **iscanbuf)
+{
+	iscan_buf_t *iscanbuf_alloc = 0;
+	iscan_buf_t *iscanbuf_head;
+
+	DHD_ISCAN(("%s: Entered\n", __FUNCTION__));
+	dhd_iscan_lock();
+
+	iscanbuf_alloc = (iscan_buf_t*)MALLOC(dhd->osh, sizeof(iscan_buf_t));
+	if (iscanbuf_alloc == NULL)
+		goto fail;
+
+	iscanbuf_alloc->next = NULL;
+	iscanbuf_head = *iscanbuf;
+
+	DHD_ISCAN(("%s: addr of allocated node = 0x%X"
+		   "addr of iscanbuf_head = 0x%X dhd = 0x%X\n",
+		   __FUNCTION__, iscanbuf_alloc, iscanbuf_head, dhd));
+
+	if (iscanbuf_head == NULL) {
+		*iscanbuf = iscanbuf_alloc;
+		DHD_ISCAN(("%s: Head is allocated\n", __FUNCTION__));
+		goto fail;
+	}
+
+	while (iscanbuf_head->next)
+		iscanbuf_head = iscanbuf_head->next;
+
+	iscanbuf_head->next = iscanbuf_alloc;
+
+fail:
+	dhd_iscan_unlock();
+	return iscanbuf_alloc;
+}
+
+void
+dhd_iscan_free_buf(void *dhdp, iscan_buf_t *iscan_delete)
+{
+	iscan_buf_t *iscanbuf_free = 0;
+	iscan_buf_t *iscanbuf_prv = 0;
+	iscan_buf_t *iscanbuf_cur;
+	dhd_pub_t *dhd = dhd_bus_pub(dhdp);
+	DHD_ISCAN(("%s: Entered\n", __FUNCTION__));
+
+	dhd_iscan_lock();
+
+	iscanbuf_cur = iscan_chain;
+
+	/* If iscan_delete is null then delete the entire
+	 * chain or else delete specific one provided
+	 */
+	if (!iscan_delete) {
+		while (iscanbuf_cur) {
+			iscanbuf_free = iscanbuf_cur;
+			iscanbuf_cur = iscanbuf_cur->next;
+			iscanbuf_free->next = 0;
+			MFREE(dhd->osh, iscanbuf_free, sizeof(iscan_buf_t));
+		}
+		iscan_chain = 0;
+	} else {
+		while (iscanbuf_cur) {
+			if (iscanbuf_cur == iscan_delete)
+				break;
+			iscanbuf_prv = iscanbuf_cur;
+			iscanbuf_cur = iscanbuf_cur->next;
+		}
+		if (iscanbuf_prv)
+			iscanbuf_prv->next = iscan_delete->next;
+
+		iscan_delete->next = 0;
+		MFREE(dhd->osh, iscan_delete, sizeof(iscan_buf_t));
+
+		if (!iscanbuf_prv)
+			iscan_chain = 0;
+	}
+	dhd_iscan_unlock();
+}
+
+iscan_buf_t *
+dhd_iscan_result_buf(void)
+{
+	return iscan_chain;
+}
+
+int
+dhd_iscan_issue_request(void * dhdp, wl_iscan_params_t *pParams, uint32 size)
+{
+	int rc = -1;
+	dhd_pub_t *dhd = dhd_bus_pub(dhdp);
+	char *buf;
+	char iovar[] = "iscan";
+	uint32 allocSize = 0;
+	wl_ioctl_t ioctl;
+	int len;
+
+	if (pParams) {
+		allocSize = (size + strlen(iovar) + 1);
+		if ((allocSize < size) || (allocSize < strlen(iovar)))
+		{
+			DHD_ERROR(("%s: overflow - allocation size too large %d < %d + %d!\n",
+				__FUNCTION__, allocSize, size, strlen(iovar)));
+			goto cleanUp;
+		}
+		buf = MALLOC(dhd->osh, allocSize);
+
+		if (buf == NULL)
+			{
+			DHD_ERROR(("%s: malloc of size %d failed!\n", __FUNCTION__, allocSize));
+			goto cleanUp;
+			}
+		ioctl.cmd = WLC_SET_VAR;
+		len = bcm_mkiovar(iovar, (char *)pParams, size, buf, allocSize);
+		if (len == 0) {
+			rc = BCME_BUFTOOSHORT;
+			goto cleanUp;
+		}
+		rc = dhd_wl_ioctl(dhd, 0, &ioctl, buf, len);
+	}
+
+cleanUp:
+	if (buf) {
+		MFREE(dhd->osh, buf, allocSize);
+	}
+
+	return rc;
+}
+
+static int
+dhd_iscan_get_partial_result(void *dhdp, uint *scan_count)
+{
+	wl_iscan_results_t *list_buf;
+	wl_iscan_results_t list;
+	wl_scan_results_v109_t *results;
+	iscan_buf_t *iscan_cur;
+	int status = -1;
+	dhd_pub_t *dhd = dhd_bus_pub(dhdp);
+	int rc;
+	wl_ioctl_t ioctl;
+	int len;
+
+	DHD_ISCAN(("%s: Enter\n", __FUNCTION__));
+
+	iscan_cur = dhd_iscan_allocate_buf(dhd, &iscan_chain);
+	if (!iscan_cur) {
+		DHD_ERROR(("%s: Failed to allocate node\n", __FUNCTION__));
+		dhd_iscan_free_buf(dhdp, 0);
+		dhd_iscan_request(dhdp, WL_SCAN_ACTION_ABORT);
+		dhd_ind_scan_confirm(dhdp, FALSE);
+		goto fail;
+	}
+
+	dhd_iscan_lock();
+
+	memset(iscan_cur->iscan_buf, 0, WLC_IW_ISCAN_MAXLEN);
+	list_buf = (wl_iscan_results_t*)iscan_cur->iscan_buf;
+	results = &list_buf->results;
+	results->buflen = WL_ISCAN_RESULTS_FIXED_SIZE;
+	results->version = 0;
+	results->count = 0;
+
+	memset(&list, 0, sizeof(list));
+	list.results.buflen = htod32(WLC_IW_ISCAN_MAXLEN);
+	len = bcm_mkiovar("iscanresults", (char *)&list, WL_ISCAN_RESULTS_FIXED_SIZE,
+		iscan_cur->iscan_buf, WLC_IW_ISCAN_MAXLEN);
+	if (len == 0) {
+		dhd_iscan_free_buf(dhdp, 0);
+		dhd_iscan_request(dhdp, WL_SCAN_ACTION_ABORT);
+		dhd_ind_scan_confirm(dhdp, FALSE);
+		status = BCME_BUFTOOSHORT;
+		goto fail;
+	}
+	ioctl.cmd = WLC_GET_VAR;
+	ioctl.set = FALSE;
+	rc = dhd_wl_ioctl(dhd, 0, &ioctl, iscan_cur->iscan_buf, WLC_IW_ISCAN_MAXLEN);
+
+	results->buflen = dtoh32(results->buflen);
+	results->version = dtoh32(results->version);
+	*scan_count = results->count = dtoh32(results->count);
+	status = dtoh32(list_buf->status);
+	DHD_ISCAN(("%s: Got %d resuls status = (%x)\n", __FUNCTION__, results->count, status));
+
+	dhd_iscan_unlock();
+
+	if (!(*scan_count)) {
+		 /* TODO: race condition when FLUSH already called */
+		dhd_iscan_free_buf(dhdp, 0);
+	}
+fail:
+	return status;
+}
+
+#ifdef NDIS
+/* XXXX Following code had bit of OS dependency.
+ * Cleanup to move the OS dependency to other
+ * per port code so that iscan logic here can be
+ * leveraged across all OS's
+ */
+NDIS_EVENT iscan_event;
+HANDLE tHandle;
+NDIS_SPIN_LOCK	dhd_iscan_queue_lock;
+
+void
+dhd_iscan_lock(void)
+{
+	NdisAcquireSpinLock(&dhd_iscan_queue_lock);
+}
+
+void
+dhd_iscan_unlock(void)
+{
+	NdisReleaseSpinLock(&dhd_iscan_queue_lock);
+}
+
+void
+dhd_iscan_notify(void)
+{
+	DHD_ISCAN(("%s: Entered\n", __FUNCTION__));
+	NdisSetEvent(&iscan_event);
+}
+
+static void
+dhd_iscan_func(void *h)
+{
+	int status;
+	uint scan_count;
+	dhd_pub_t *dhd = dhd_bus_pub(h);
+
+	/* Read the priority from registry */
+	CeSetThreadPriority(GetCurrentThread(), 128);
+	DHD_ISCAN(("%s: thread created\n", __FUNCTION__));
+
+	while (TRUE) {
+		NdisWaitEvent(&iscan_event, 0);		/* wait forever */
+		NdisResetEvent(&iscan_event);		/* reset the event */
+		DHD_ISCAN(("%s: thread scheduled\n", __FUNCTION__));
+
+		status = dhd_iscan_get_partial_result(h, &scan_count);
+
+		if (status == WL_SCAN_RESULTS_PARTIAL) {
+			dhd_iscan_request(h, WL_SCAN_ACTION_CONTINUE);
+		} else if (status == WL_SCAN_RESULTS_SUCCESS) {
+			if (dhd_iscan_in_progress(h)) {
+				dhd_ind_scan_confirm(h, TRUE);
+			}
+		} else if (status == WL_SCAN_RESULTS_ABORTED ||
+			status == WL_SCAN_RESULTS_NO_MEM) {
+			dhd_iscan_request(h, WL_SCAN_ACTION_ABORT);
+			dhd_ind_scan_confirm(h, FALSE);
+		} else {
+			dhd_iscan_request(h, WL_SCAN_ACTION_ABORT);
+			dhd_ind_scan_confirm(h, FALSE);
+		}
+	}
+}
+
+int
+dhd_iscan_attach(void *dhdp)
+{
+	DHD_ISCAN(("%s: dhdp = 0x%x\n", __FUNCTION__, dhdp));
+
+	NdisInitializeEvent(&iscan_event);
+	NdisResetEvent(&iscan_event);
+	NdisAllocateSpinLock(&dhd_iscan_queue_lock);
+
+	/* XXX - should move to ndishared sublayer */
+	tHandle = CreateThread(NULL,
+		0,
+		(LPTHREAD_START_ROUTINE)dhd_iscan_func,
+		(void *)dhdp,
+		0,
+		&iscan_thread_id);
+
+	if (!iscan_thread_id)
+		return NDIS_STATUS_FAILURE;
+
+	return NDIS_STATUS_SUCCESS;
+}
+
+void
+dhd_iscan_deattach(void *dhdp)
+{
+	if (iscan_thread_id)
+	{
+	NdisFreeEvent(&iscan_event);
+	NdisFreeSpinLock(&dhd_iscan_queue_lock);
+	CloseHandle(tHandle);
+		iscan_thread_id = 0;
+	}
+}
+#endif /* NDIS */
+#endif /* SIMPLE_ISCAN */
+
+/*
+ * returns = TRUE if associated, FALSE if not associated
+ */
+bool dhd_is_associated(dhd_pub_t *dhd, uint8 ifidx, int *retval)
+{
+	char bssid[6], zbuf[6];
+	int ret = -1;
+
+	bzero(bssid, 6);
+	bzero(zbuf, 6);
+
+	ret  = dhd_wl_ioctl_cmd(dhd, WLC_GET_BSSID, (char *)&bssid,
+		ETHER_ADDR_LEN, FALSE, ifidx);
+	/* XXX:AS!!! res can be: -17(BCME_NOTASSOCIATED),-22(BCME_NORESOURCE), and 0(OK)
+	  OK - doesn't mean associated yet, the returned bssid
+	  still needs to be checked for non zero array
+	*/
+	DHD_TRACE((" %s WLC_GET_BSSID ioctl res = %d\n", __FUNCTION__, ret));
+
+	if (ret == BCME_NOTASSOCIATED) {
+		DHD_ERROR(("%s: WLC_GET_BSSID, NOT ASSOCIATED\n", __FUNCTION__));
+	}
+
+	if (retval)
+		*retval = ret;
+
+	if (ret < 0)
+		return FALSE;
+
+	if ((memcmp(bssid, zbuf, ETHER_ADDR_LEN) == 0)) {
+		DHD_TRACE(("%s: WLC_GET_BSSID ioctl returned zero bssid\n", __FUNCTION__));
+		return FALSE;
+	}
+	return TRUE;
+}
+
+#if defined(__linux__)
+/*
+ * returns = TRUE if anyone is associated, FALSE if all are disassociated
+ */
+bool dhd_scan_associated(dhd_pub_t *dhd, struct net_device *ndev)
+{
+	dhd_if_t *ifp = NULL;
+	u32 idx = 0;
+
+	for (idx = 0; idx < DHD_MAX_IFS; idx++) {
+		ifp = dhd_get_ifp(dhd, idx);
+		if (!ifp)
+			continue;
+		if (ifp->net == ndev)
+			continue;
+		if (!dhd_is_associated(dhd, idx, NULL))
+			continue;
+		return TRUE;
+	}
+
+	return FALSE;
+}
+#endif /* __linux__ */
+
+/* Check if the mode supports STA MODE */
+bool dhd_support_sta_mode(dhd_pub_t *dhd)
+{
+
+#ifdef  WL_CFG80211
+	if (!(dhd->op_mode & DHD_FLAG_STA_MODE))
+		return FALSE;
+	else
+#endif /* WL_CFG80211 */
+		return TRUE;
+}
+
+#if defined(KEEP_ALIVE)
+int dhd_keep_alive_onoff(dhd_pub_t *dhd)
+{
+	char				buf[32] = {0};
+	const char			*str;
+	wl_mkeep_alive_pkt_v1_t mkeep_alive_pkt;
+	wl_mkeep_alive_pkt_v1_t	*mkeep_alive_pktp;
+	int					buf_len;
+	int					str_len;
+	int res					= -1;
+
+	if (!dhd_support_sta_mode(dhd))
+		return res;
+
+	DHD_TRACE(("%s execution\n", __FUNCTION__));
+
+	str = "mkeep_alive";
+	str_len = strlen(str);
+	strlcpy(buf, str, sizeof(buf));
+	mkeep_alive_pktp = (wl_mkeep_alive_pkt_v1_t *) (buf + str_len + 1);
+	bzero(&mkeep_alive_pkt, sizeof(mkeep_alive_pkt));
+	mkeep_alive_pkt.period_msec = dhd->conf->keep_alive_period;
+	buf_len = str_len + 1;
+	mkeep_alive_pkt.version = htod16(WL_MKEEP_ALIVE_VERSION_1);
+	mkeep_alive_pkt.length = htod16(WL_MKEEP_ALIVE_FIXED_LEN);
+	/* Setup keep alive zero for null packet generation */
+	mkeep_alive_pkt.keep_alive_id = 0;
+	mkeep_alive_pkt.len_bytes = 0;
+	buf_len += WL_MKEEP_ALIVE_FIXED_LEN;
+	/* Keep-alive attributes are set in local	variable (mkeep_alive_pkt), and
+	 * then memcpy'ed into buffer (mkeep_alive_pktp) since there is no
+	 * guarantee that the buffer is properly aligned.
+	 */
+	memcpy((char *)mkeep_alive_pktp, &mkeep_alive_pkt, WL_MKEEP_ALIVE_FIXED_LEN);
+
+	res = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, buf_len, TRUE, 0);
+
+	return res;
+}
+#endif /* defined(KEEP_ALIVE) */
+#if defined(OEM_ANDROID)
+#define	CSCAN_TLV_TYPE_SSID_IE	'S'
+/*
+ *  SSIDs list parsing from cscan tlv list
+ */
+int
+wl_parse_ssid_list_tlv(char** list_str, wlc_ssid_ext_t* ssid, int max, int *bytes_left)
+{
+	char* str;
+	int idx = 0;
+	uint8 len;
+
+	if ((list_str == NULL) || (*list_str == NULL) || (*bytes_left < 0)) {
+		DHD_ERROR(("%s error paramters\n", __FUNCTION__));
+		return BCME_BADARG;
+	}
+	str = *list_str;
+	while (*bytes_left > 0) {
+		if (str[0] != CSCAN_TLV_TYPE_SSID_IE) {
+			*list_str = str;
+			DHD_TRACE(("nssid=%d left_parse=%d %d\n", idx, *bytes_left, str[0]));
+			return idx;
+		}
+
+		if (idx >= max) {
+			DHD_ERROR(("%s number of SSIDs more than %d\n", __FUNCTION__, idx));
+			return BCME_BADARG;
+		}
+
+		/* Get proper CSCAN_TLV_TYPE_SSID_IE */
+		*bytes_left -= 1;
+		if (*bytes_left == 0) {
+			DHD_ERROR(("%s no length field.\n", __FUNCTION__));
+			return BCME_BADARG;
+		}
+		str += 1;
+		ssid[idx].rssi_thresh = 0;
+		ssid[idx].flags = 0;
+		len = str[0];
+		if (len == 0) {
+			/* Broadcast SSID */
+			ssid[idx].SSID_len = 0;
+			memset((char*)ssid[idx].SSID, 0x0, DOT11_MAX_SSID_LEN);
+			*bytes_left -= 1;
+			str += 1;
+
+			DHD_TRACE(("BROADCAST SCAN  left=%d\n", *bytes_left));
+		} else if (len <= DOT11_MAX_SSID_LEN) {
+			/* Get proper SSID size */
+			ssid[idx].SSID_len = len;
+			*bytes_left -= 1;
+			/* Get SSID */
+			if (ssid[idx].SSID_len > *bytes_left) {
+				DHD_ERROR(("%s out of memory range len=%d but left=%d\n",
+				__FUNCTION__, ssid[idx].SSID_len, *bytes_left));
+				return BCME_BADARG;
+			}
+			str += 1;
+			memcpy((char*)ssid[idx].SSID, str, ssid[idx].SSID_len);
+
+			*bytes_left -= ssid[idx].SSID_len;
+			str += ssid[idx].SSID_len;
+			ssid[idx].hidden = TRUE;
+
+			DHD_TRACE(("%s :size=%d left=%d\n",
+				(char*)ssid[idx].SSID, ssid[idx].SSID_len, *bytes_left));
+		} else {
+			DHD_ERROR(("### SSID size more than %d\n", str[0]));
+			return BCME_BADARG;
+		}
+		idx++;
+	}
+
+	*list_str = str;
+	return idx;
+}
+
+#if defined(WL_WIRELESS_EXT)
+/* Android ComboSCAN support */
+
+/*
+ *  data parsing from ComboScan tlv list
+*/
+int
+wl_iw_parse_data_tlv(char** list_str, void *dst, int dst_size, const char token,
+                     int input_size, int *bytes_left)
+{
+	char* str;
+	uint16 short_temp;
+	uint32 int_temp;
+
+	if ((list_str == NULL) || (*list_str == NULL) ||(bytes_left == NULL) || (*bytes_left < 0)) {
+		DHD_ERROR(("%s error paramters\n", __FUNCTION__));
+		return -1;
+	}
+	str = *list_str;
+
+	/* Clean all dest bytes */
+	memset(dst, 0, dst_size);
+	if (*bytes_left > 0) {
+
+		if (str[0] != token) {
+			DHD_TRACE(("%s NOT Type=%d get=%d left_parse=%d \n",
+				__FUNCTION__, token, str[0], *bytes_left));
+			return -1;
+		}
+
+		*bytes_left -= 1;
+		str += 1;
+
+		if (input_size == 1) {
+			memcpy(dst, str, input_size);
+		}
+		else if (input_size == 2) {
+			memcpy(dst, (char *)htod16(memcpy(&short_temp, str, input_size)),
+				input_size);
+		}
+		else if (input_size == 4) {
+			memcpy(dst, (char *)htod32(memcpy(&int_temp, str, input_size)),
+				input_size);
+		}
+
+		*bytes_left -= input_size;
+		str += input_size;
+		*list_str = str;
+		return 1;
+	}
+	return 1;
+}
+
+/*
+ *  channel list parsing from cscan tlv list
+*/
+int
+wl_iw_parse_channel_list_tlv(char** list_str, uint16* channel_list,
+                             int channel_num, int *bytes_left)
+{
+	char* str;
+	int idx = 0;
+
+	if ((list_str == NULL) || (*list_str == NULL) ||(bytes_left == NULL) || (*bytes_left < 0)) {
+		DHD_ERROR(("%s error paramters\n", __FUNCTION__));
+		return -1;
+	}
+	str = *list_str;
+
+	while (*bytes_left > 0) {
+
+		if (str[0] != CSCAN_TLV_TYPE_CHANNEL_IE) {
+			*list_str = str;
+			DHD_TRACE(("End channel=%d left_parse=%d %d\n", idx, *bytes_left, str[0]));
+			return idx;
+		}
+		/* Get proper CSCAN_TLV_TYPE_CHANNEL_IE */
+		*bytes_left -= 1;
+		str += 1;
+
+		if (str[0] == 0) {
+			/* All channels */
+			channel_list[idx] = 0x0;
+		}
+		else {
+			channel_list[idx] = (uint16)str[0];
+			DHD_TRACE(("%s channel=%d \n", __FUNCTION__,  channel_list[idx]));
+		}
+		*bytes_left -= 1;
+		str += 1;
+
+		if (idx++ > 255) {
+			DHD_ERROR(("%s Too many channels \n", __FUNCTION__));
+			return -1;
+		}
+	}
+
+	*list_str = str;
+	return idx;
+}
+
+/* Parse a comma-separated list from list_str into ssid array, starting
+ * at index idx.  Max specifies size of the ssid array.  Parses ssids
+ * and returns updated idx; if idx >= max not all fit, the excess have
+ * not been copied.  Returns -1 on empty string, or on ssid too long.
+ */
+int
+wl_iw_parse_ssid_list(char** list_str, wlc_ssid_t* ssid, int idx, int max)
+{
+	char* str, *ptr;
+
+	if ((list_str == NULL) || (*list_str == NULL))
+		return -1;
+
+	for (str = *list_str; str != NULL; str = ptr) {
+
+		/* check for next TAG */
+		if (!strncmp(str, GET_CHANNEL, strlen(GET_CHANNEL))) {
+			*list_str	 = str + strlen(GET_CHANNEL);
+			return idx;
+		}
+
+		if ((ptr = strchr(str, ',')) != NULL) {
+			*ptr++ = '\0';
+		}
+
+		if (strlen(str) > DOT11_MAX_SSID_LEN) {
+			DHD_ERROR(("ssid <%s> exceeds %d\n", str, DOT11_MAX_SSID_LEN));
+			return -1;
+		}
+
+		if (strlen(str) == 0)
+			ssid[idx].SSID_len = 0;
+
+		if (idx < max) {
+			bzero(ssid[idx].SSID, sizeof(ssid[idx].SSID));
+			strlcpy((char*)ssid[idx].SSID, str, sizeof(ssid[idx].SSID));
+			ssid[idx].SSID_len = sizeof(ssid[idx].SSID);
+		}
+		idx++;
+	}
+	return idx;
+}
+
+/*
+ * Parse channel list from iwpriv CSCAN
+ */
+int
+wl_iw_parse_channel_list(char** list_str, uint16* channel_list, int channel_num)
+{
+	int num;
+	int val;
+	char* str;
+	char* endptr = NULL;
+
+	if ((list_str == NULL)||(*list_str == NULL))
+		return -1;
+
+	str = *list_str;
+	num = 0;
+	while (strncmp(str, GET_NPROBE, strlen(GET_NPROBE))) {
+		val = (int)strtoul(str, &endptr, 0);
+		if (endptr == str) {
+			printf("could not parse channel number starting at"
+				" substring \"%s\" in list:\n%s\n",
+				str, *list_str);
+			return -1;
+		}
+		str = endptr + strspn(endptr, " ,");
+
+		if (num == channel_num) {
+			DHD_ERROR(("too many channels (more than %d) in channel list:\n%s\n",
+				channel_num, *list_str));
+			return -1;
+		}
+
+		channel_list[num++] = (uint16)val;
+	}
+	*list_str = str;
+	return num;
+}
+#endif
+#endif /* defined(OEM_ANDROID) */
+
+#if defined(BCM_ROUTER_DHD)
+static int traffic_mgmt_add_dwm_filter(dhd_pub_t *dhd,
+	trf_mgmt_filter_list_t * trf_mgmt_filter_list, int len)
+{
+	int ret = 0;
+	uint32              i;
+	trf_mgmt_filter_t   *trf_mgmt_filter;
+	uint8               dwm_tbl_entry;
+	uint32              dscp = 0;
+	uint16              dwm_filter_enabled = 0;
+
+	/* Check parameter length is adequate */
+	if (len < (OFFSETOF(trf_mgmt_filter_list_t, filter) +
+		trf_mgmt_filter_list->num_filters * sizeof(trf_mgmt_filter_t))) {
+		ret = BCME_BUFTOOSHORT;
+		return ret;
+	}
+
+	bzero(&dhd->dhd_tm_dwm_tbl, sizeof(dhd_trf_mgmt_dwm_tbl_t));
+
+	for (i = 0; i < trf_mgmt_filter_list->num_filters; i++) {
+		trf_mgmt_filter = &trf_mgmt_filter_list->filter[i];
+
+		dwm_filter_enabled = (trf_mgmt_filter->flags & TRF_FILTER_DWM);
+
+		if (dwm_filter_enabled) {
+			dscp = trf_mgmt_filter->dscp;
+			if (dscp >= DHD_DWM_TBL_SIZE) {
+				ret = BCME_BADARG;
+			return ret;
+			}
+		}
+
+		dhd->dhd_tm_dwm_tbl.dhd_dwm_enabled = 1;
+		/* set WMM AC bits */
+		dwm_tbl_entry = (uint8) trf_mgmt_filter->priority;
+		DHD_TRF_MGMT_DWM_SET_FILTER(dwm_tbl_entry);
+
+		/* set favored bits */
+		if (trf_mgmt_filter->flags & TRF_FILTER_FAVORED)
+			DHD_TRF_MGMT_DWM_SET_FAVORED(dwm_tbl_entry);
+
+		dhd->dhd_tm_dwm_tbl.dhd_dwm_tbl[dscp] =  dwm_tbl_entry;
+	}
+	return ret;
+}
+#endif /* BCM_ROUTER_DHD */
+
+#ifdef DHD_LINUX_STD_FW_API
+int dhd_get_download_buffer(dhd_pub_t	*dhd, char *file_path, download_type_t component,
+	char ** buffer, int *length)
+{
+	int ret = BCME_ERROR;
+	const struct firmware *fw = NULL;
+#ifdef SUPPORT_OTA_UPDATE
+	uint8 *buf = NULL;
+	int len = 0;
+	ota_update_info_t *ota_info = &dhd->ota_update_info;
+#endif /* SUPPORT_OTA_UPDATE */
+
+#ifdef SUPPORT_OTA_UPDATE
+	if (component == CLM_BLOB) {
+		if (ota_info->clm_len) {
+			DHD_ERROR(("Using OTA CLM_BLOB\n"));
+			buf = ota_info->clm_buf;
+			len = ota_info->clm_len;
+		}
+	}
+	else if (component == NVRAM) {
+		if (ota_info->nvram_len) {
+			DHD_ERROR(("Using OTA NVRAM.\n"));
+			buf = ota_info->nvram_buf;
+			len = ota_info->nvram_len;
+		}
+	}
+#endif /* SUPPORT_OTA_UPDATE */
+
+#ifdef SUPPORT_OTA_UPDATE
+	if (len) {
+		*buffer = (char *)buf;
+		*length = len;
+	}
+	else
+#endif /* SUPPORT_OTA_UPDATE */
+	{
+		if (file_path) {
+			ret = dhd_os_get_img_fwreq(&fw, file_path);
+			if (ret < 0) {
+				DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+					ret));
+				goto err;
+			} else {
+				if ((fw->size <= 0 || fw->size > *length)) {
+					DHD_ERROR(("fw->size = %zu, *length = %d\n", fw->size, *length));
+					*length = fw->size;
+					goto err;
+				}
+				*buffer = VMALLOCZ(dhd->osh, *length);
+				if (*buffer == NULL) {
+					DHD_ERROR(("%s: Failed to allocate memory %d bytes\n",
+						__FUNCTION__, (int)fw->size));
+					ret = BCME_NOMEM;
+					goto err;
+				}
+				*length = fw->size;
+				ret = memcpy_s(*buffer, fw->size, fw->data, fw->size);
+				if (ret != BCME_OK) {
+					DHD_ERROR(("%s: memcpy_s failed, err : %d\n",
+							__FUNCTION__, ret));
+					goto err;
+				}
+				ret = BCME_OK;
+			}
+		}
+	}
+err:
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+	return ret;
+}
+
+#else
+
+/* Given filename and download type,  returns a buffer pointer and length
+* for download to f/w. Type can be FW or NVRAM.
+*
+*/
+int dhd_get_download_buffer(dhd_pub_t	*dhd, char *file_path, download_type_t component,
+	char ** buffer, int *length)
+{
+	int ret = BCME_ERROR;
+	int len = 0;
+	int file_len;
+	void *image = NULL;
+	uint8 *buf = NULL;
+
+	/* Point to cache if available. */
+#ifdef CACHE_FW_IMAGES
+	if (component == FW) {
+		if (dhd->cached_fw_length) {
+			len = dhd->cached_fw_length;
+			buf = dhd->cached_fw;
+		}
+	} else if (component == NVRAM) {
+		if (dhd->cached_nvram_length) {
+			len = dhd->cached_nvram_length;
+			buf = dhd->cached_nvram;
+		}
+	} else if (component == CLM_BLOB) {
+		if (dhd->cached_clm_length) {
+			len = dhd->cached_clm_length;
+			buf = dhd->cached_clm;
+		}
+	} else if (component == TXCAP_BLOB) {
+		if (dhd->cached_txcap_length) {
+			len = dhd->cached_txcap_length;
+			buf = dhd->cached_txcap;
+		}
+	} else {
+		DHD_ERROR(("%s: Invalid component arg %d\n",
+			__FUNCTION__, component));
+		ret = BCME_BADARG;
+		return ret;
+	}
+#endif /* CACHE_FW_IMAGES */
+	/* No Valid cache found on this call */
+	if (!len) {
+		file_len = *length;
+		*length = 0;
+
+		if (file_path) {
+			image = dhd_os_open_image1(dhd, file_path);
+			if (image == NULL) {
+				DHD_TRACE(("%s: Open image file failed %s\n", __FUNCTION__, file_path));
+				goto err;
+			}
+		}
+		buf = MALLOCZ(dhd->osh, file_len);
+		if (buf == NULL) {
+			DHD_TRACE(("%s: Failed to allocate memory %d bytes\n",
+				__FUNCTION__, file_len));
+			ret = BCME_NOMEM;
+			goto err;
+		}
+
+		/* Download image */
+		len = dhd_os_get_image_block((char *)buf, file_len, image);
+		if ((len <= 0 || len > file_len)) {
+			MFREE(dhd->osh, buf, file_len);
+			goto err;
+		}
+	}
+
+	ret = BCME_OK;
+	*length = len;
+	*buffer = (char *)buf;
+
+	/* Cache if first call. */
+#ifdef CACHE_FW_IMAGES
+	if (component == FW) {
+		if (!dhd->cached_fw_length) {
+			dhd->cached_fw = buf;
+			dhd->cached_fw_length = len;
+		}
+	} else if (component == NVRAM) {
+		if (!dhd->cached_nvram_length) {
+			dhd->cached_nvram = buf;
+			dhd->cached_nvram_length = len;
+		}
+	} else if (component == CLM_BLOB) {
+		if (!dhd->cached_clm_length) {
+			dhd->cached_clm = buf;
+			dhd->cached_clm_length = len;
+		}
+	} else if (component == TXCAP_BLOB) {
+		if (!dhd->cached_txcap_length) {
+			dhd->cached_txcap = buf;
+			dhd->cached_txcap_length = len;
+		}
+	}
+#endif /* CACHE_FW_IMAGES */
+
+err:
+	if (image)
+		dhd_os_close_image1(dhd, image);
+	return ret;
+}
+#endif /* DHD_LINUX_STD_FW_API */
+
+int
+dhd_download_2_dongle(dhd_pub_t	*dhd, char *iovar, uint16 flag, uint16 dload_type,
+	unsigned char *dload_buf, int len)
+{
+	struct wl_dload_data *dload_ptr = (struct wl_dload_data *)dload_buf;
+	int err = 0;
+	int dload_data_offset;
+	static char iovar_buf[WLC_IOCTL_MEDLEN];
+	int iovar_len;
+
+	memset(iovar_buf, 0, sizeof(iovar_buf));
+
+	dload_data_offset = OFFSETOF(wl_dload_data_t, data);
+	dload_ptr->flag = (DLOAD_HANDLER_VER << DLOAD_FLAG_VER_SHIFT) | flag;
+	dload_ptr->dload_type = dload_type;
+	dload_ptr->len = htod32(len - dload_data_offset);
+	dload_ptr->crc = 0;
+	len = ROUNDUP(len, 8);
+
+	iovar_len = bcm_mkiovar(iovar, (char *)dload_buf,
+		(uint)len, iovar_buf, sizeof(iovar_buf));
+	if (iovar_len == 0) {
+		DHD_ERROR(("%s: insufficient buffer space passed to bcm_mkiovar for '%s' \n",
+		           __FUNCTION__, iovar));
+		return BCME_BUFTOOSHORT;
+	}
+
+	err = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovar_buf,
+			iovar_len, IOV_SET, 0);
+
+	return err;
+}
+
+int
+dhd_download_blob(dhd_pub_t *dhd, unsigned char *buf,
+		uint32 len, char *iovar)
+
+{
+	int chunk_len;
+#if (!defined(LINUX) && !defined(linux)) || defined(DHD_LINUX_STD_FW_API)
+	int cumulative_len = 0;
+#endif /* !LINUX && !linux || DHD_LINUX_STD_FW_API */
+	int size2alloc;
+	unsigned char *new_buf;
+	int err = 0, data_offset;
+	uint16 dl_flag = DL_BEGIN;
+
+	data_offset = OFFSETOF(wl_dload_data_t, data);
+	size2alloc = data_offset + MAX_CHUNK_LEN;
+	size2alloc = ROUNDUP(size2alloc, 8);
+
+	if ((new_buf = (unsigned char *)MALLOCZ(dhd->osh, size2alloc)) != NULL) {
+		do {
+#if (!defined(LINUX) && !defined(linux)) || defined(DHD_LINUX_STD_FW_API)
+			if (len >= MAX_CHUNK_LEN)
+				chunk_len = MAX_CHUNK_LEN;
+			else
+				chunk_len = len;
+
+			memcpy(new_buf + data_offset, buf + cumulative_len, chunk_len);
+			cumulative_len += chunk_len;
+#else
+			chunk_len = dhd_os_get_image_block((char *)(new_buf + data_offset),
+				MAX_CHUNK_LEN, buf);
+			if (chunk_len < 0) {
+				DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n",
+					__FUNCTION__, chunk_len));
+				err = BCME_ERROR;
+				goto exit;
+			}
+#endif /* !LINUX && !linux || DHD_LINUX_STD_FW_API */
+			if (len - chunk_len == 0)
+				dl_flag |= DL_END;
+
+			err = dhd_download_2_dongle(dhd, iovar, dl_flag, DL_TYPE_CLM,
+				new_buf, data_offset + chunk_len);
+
+			dl_flag &= ~DL_BEGIN;
+
+			len = len - chunk_len;
+		} while ((len > 0) && (err == 0));
+#if (!defined(LINUX) && !defined(linux)) || defined(DHD_LINUX_STD_FW_API)
+		MFREE(dhd->osh, new_buf, size2alloc);
+#endif /* !LINUX && !linux */
+	} else {
+		err = BCME_NOMEM;
+	}
+#if (defined(LINUX) || defined(linux)) && !defined(DHD_LINUX_STD_FW_API)
+exit:
+	if (new_buf) {
+		MFREE(dhd->osh, new_buf, size2alloc);
+	}
+#endif /* LINUX || linux */
+	return err;
+}
+
+#if defined(CACHE_FW_IMAGES)
+int
+dhd_download_blob_cached(dhd_pub_t *dhd, char *file_path,
+	uint32 len, char *iovar)
+{
+	int ret = BCME_ERROR;
+	uint chunk_len, size2alloc, data_offset, file_offset;
+	unsigned char *pay_load, *dnld_buf;
+	char *memblock = NULL;
+	uint16 dl_flag = DL_BEGIN;
+	download_type_t dl_type;
+	uint32 memblock_len = len;
+
+	data_offset = OFFSETOF(wl_dload_data_t, data);
+	size2alloc = data_offset + MAX_CHUNK_LEN;
+	size2alloc = ROUNDUP(size2alloc, 8);
+	file_offset = 0;
+
+	if ((dnld_buf = MALLOCZ(dhd->osh, size2alloc)) == NULL) {
+		ret = BCME_NOMEM;
+		goto exit;
+	}
+	pay_load = (dnld_buf + data_offset);
+
+	if (!memcmp("clmload", iovar, strlen("clmload"))) {
+		dl_type = CLM_BLOB;
+	} else if (!memcmp("txcapload", iovar, strlen("txcapload"))) {
+		dl_type = TXCAP_BLOB;
+	} else {
+		DHD_ERROR(("%s Invalid iovar :%s \n", __FUNCTION__, iovar));
+		ret = BCME_BADARG;
+		goto exit;
+	}
+
+	ret = dhd_get_download_buffer(dhd, file_path, dl_type, &memblock, (int *)&len);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: error getting buffer for %s, %s \n", __FUNCTION__,
+			file_path, bcmerrorstr(ret)));
+		goto exit;
+	}
+
+	do {
+		chunk_len = MIN(len, MAX_CHUNK_LEN);
+		memcpy(pay_load, memblock + file_offset, chunk_len);
+		if (len - chunk_len == 0) {
+			dl_flag |= DL_END;
+		}
+
+		ret = dhd_download_2_dongle(dhd, iovar, dl_flag, DL_TYPE_CLM,
+			dnld_buf, data_offset + chunk_len);
+
+		dl_flag &= ~DL_BEGIN;
+		len = len - chunk_len;
+		file_offset += chunk_len;
+	} while ((len > 0) && (ret == 0));
+
+exit:
+	if (dnld_buf) {
+		MFREE(dhd->osh, dnld_buf, size2alloc);
+	}
+	if (memblock) {
+		dhd_free_download_buffer(dhd, memblock, memblock_len);
+	}
+
+	return ret;
+}
+
+int
+dhd_apply_default_txcap(dhd_pub_t  *dhd, char *path)
+{
+	int ret = BCME_ERROR;
+	ret = dhd_download_blob_cached(dhd, path, MAX_TXCAP_BUF_SIZE, "txcapload");
+	if (ret) {
+		DHD_ERROR(("%s: error downloading blob: %s \n", __FUNCTION__, bcmerrorstr(ret)));
+	}
+	return ret;
+}
+
+int
+dhd_apply_default_clm(dhd_pub_t *dhd, char *clm_path)
+{
+	char *clm_blob_path;
+	int len;
+	unsigned char *imgbuf = NULL;
+	int err = BCME_OK;
+	char iovbuf[WLC_IOCTL_SMLEN];
+	wl_country_t *cspec;
+
+	if (clm_path[0] != '\0') {
+		if (strlen(clm_path) > MOD_PARAM_PATHLEN) {
+			DHD_ERROR(("clm path exceeds max len\n"));
+			return BCME_ERROR;
+		}
+		clm_blob_path = clm_path;
+		DHD_TRACE(("clm path from module param:%s\n", clm_path));
+	} else {
+		clm_blob_path = VENDOR_PATH CONFIG_BCMDHD_CLM_PATH;
+	}
+
+	/* If CLM blob file is found on the filesystem, download the file.
+	* After CLM file download or If the blob file is not present,
+	* validate the country code before proceeding with the initialization.
+	* If country code is not valid, fail the initialization.
+	*/
+
+	imgbuf = dhd_os_open_image((char *)clm_blob_path);
+	if (imgbuf == NULL) {
+		DHD_ERROR(("%s: clm from path %s open fail\n", __func__, clm_path));
+		goto exit;
+	}
+
+	len = dhd_os_get_image_size(imgbuf);
+
+	if ((len > 0) && (len < MAX_CLM_BUF_SIZE) && imgbuf) {
+		len = bcm_mkiovar("country", NULL, 0, iovbuf, sizeof(iovbuf));
+		if (len == 0) {
+			err = BCME_BUFTOOSHORT;
+			goto exit;
+		}
+		err = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf, sizeof(iovbuf), FALSE, 0);
+		if (err) {
+			DHD_ERROR(("%s: country code get failed\n", __FUNCTION__));
+			goto exit;
+		}
+
+		cspec = (wl_country_t *)iovbuf;
+		if ((strncmp(cspec->ccode, WL_CCODE_NULL_COUNTRY, WLC_CNTRY_BUF_SZ)) != 0) {
+			DHD_ERROR(("%s: CLM already exist in F/W, "
+				"new CLM data will be added to the end of existing CLM data!\n",
+				__FUNCTION__));
+		}
+
+		/* Found blob file. Download the file */
+		DHD_ERROR(("clm file download from %s \n", clm_blob_path));
+		if (imgbuf) {
+			dhd_os_close_image(imgbuf);
+			imgbuf = NULL;
+		}
+		err = dhd_download_blob_cached(dhd, clm_blob_path, MAX_CLM_BUF_SIZE, "clmload");
+		if (err) {
+			DHD_ERROR(("%s: CLM download failed err=%d\n", __FUNCTION__, err));
+			if (!dhd_bus_skip_clm(dhd)) {
+				/* Retrieve clmload_status and print */
+				len = bcm_mkiovar("clmload_status", NULL, 0, iovbuf,
+						sizeof(iovbuf));
+				if (len == 0) {
+					err = BCME_BUFTOOSHORT;
+					goto exit;
+				}
+				err = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf,
+					sizeof(iovbuf), FALSE, 0);
+				if (err) {
+					DHD_ERROR(("%s: clmload_status get failed err=%d \n",
+						__FUNCTION__, err));
+				} else {
+					DHD_ERROR(("%s: clmload_status: %d \n",
+						__FUNCTION__, *((int *)iovbuf)));
+					if (*((int *)iovbuf) == CHIPID_MISMATCH) {
+						DHD_ERROR(("Chip ID mismatch error \n"));
+					}
+				}
+				err = BCME_ERROR;
+				goto exit;
+			}
+		} else {
+			DHD_INFO(("%s: CLM download succeeded \n", __FUNCTION__));
+		}
+	} else {
+		DHD_INFO(("Skipping the clm download. len:%d memblk:%p \n", len, imgbuf));
+#ifdef DHD_USE_CLMINFO_PARSER
+		err = BCME_ERROR;
+		goto exit;
+#endif /* DHD_USE_CLMINFO_PARSER */
+	}
+
+	/* Verify country code */
+	len = bcm_mkiovar("country", NULL, 0, iovbuf, sizeof(iovbuf));
+	if (len == 0) {
+		err = BCME_BUFTOOSHORT;
+		goto exit;
+	}
+	err = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf, sizeof(iovbuf), FALSE, 0);
+	if (err) {
+		DHD_ERROR(("%s: country code get failed\n", __FUNCTION__));
+		goto exit;
+	}
+
+	cspec = (wl_country_t *)iovbuf;
+	if ((strncmp(cspec->ccode, WL_CCODE_NULL_COUNTRY, WLC_CNTRY_BUF_SZ)) == 0) {
+		/* Country code not initialized or CLM download not proper */
+		DHD_ERROR(("country code not initialized\n"));
+		err = BCME_ERROR;
+	}
+exit:
+
+	if (imgbuf) {
+		dhd_os_close_image(imgbuf);
+	}
+
+	return err;
+}
+#else
+
+int
+dhd_apply_default_txcap(dhd_pub_t  *dhd, char *path)
+{
+	return 0;
+}
+
+int
+dhd_check_current_clm_data(dhd_pub_t *dhd)
+{
+	char iovbuf[WLC_IOCTL_SMLEN];
+	wl_country_t *cspec;
+	int err = BCME_OK;
+
+	memset(iovbuf, 0, sizeof(iovbuf));
+	err = bcm_mkiovar("country", NULL, 0, iovbuf, sizeof(iovbuf));
+	if (err == 0) {
+		err = BCME_BUFTOOSHORT;
+		DHD_ERROR(("%s: bcm_mkiovar failed.", __FUNCTION__));
+		return err;
+	}
+	err = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf, sizeof(iovbuf), FALSE, 0);
+	if (err) {
+		DHD_ERROR(("%s: country code get failed\n", __FUNCTION__));
+		return err;
+	}
+	cspec = (wl_country_t *)iovbuf;
+	if ((strncmp(cspec->ccode, WL_CCODE_NULL_COUNTRY, WLC_CNTRY_BUF_SZ)) == 0) {
+		DHD_ERROR(("%s: ----- This FW is not included CLM data -----\n",
+			__FUNCTION__));
+		return FALSE;
+	}
+	DHD_ERROR(("%s: ----- This FW is included CLM data -----\n",
+		__FUNCTION__));
+	return TRUE;
+}
+
+int
+dhd_apply_default_clm(dhd_pub_t *dhd, char *clm_path)
+{
+	char *clm_blob_path;
+	int len = 0, memblock_len = 0;
+	char *memblock = NULL;
+	int err = BCME_OK;
+	char iovbuf[WLC_IOCTL_SMLEN];
+	int status = FALSE;
+#ifdef SUPPORT_OTA_UPDATE
+	ota_update_info_t *ota_info = &dhd->ota_update_info;
+#endif /* SUPPORT_OTA_UPDATE */
+
+	if (clm_path && clm_path[0] != '\0') {
+		if (strlen(clm_path) > MOD_PARAM_PATHLEN) {
+			DHD_ERROR(("clm path exceeds max len\n"));
+			return BCME_ERROR;
+		}
+		clm_blob_path = clm_path;
+		DHD_TRACE(("clm path from module param:%s\n", clm_path));
+	} else {
+#ifdef DHD_LINUX_STD_FW_API
+		clm_blob_path = DHD_CLM_NAME;
+#else
+		clm_blob_path = VENDOR_PATH CONFIG_BCMDHD_CLM_PATH;
+#endif /* DHD_LINUX_STD_FW_API */
+	}
+
+	/* If CLM blob file is found on the filesystem, download the file.
+	 * After CLM file download or If the blob file is not present,
+	 * validate the country code before proceeding with the initialization.
+	 * If country code is not valid, fail the initialization.
+	 */
+#if (!defined(LINUX) && !defined(linux)) || defined(DHD_LINUX_STD_FW_API)
+	len = MAX_CLM_BUF_SIZE;
+	dhd_get_download_buffer(dhd, clm_blob_path, CLM_BLOB, &memblock, &len);
+#ifdef DHD_LINUX_STD_FW_API
+	memblock_len = len;
+#else
+	memblock_len = MAX_CLM_BUF_SIZE;
+#endif /* DHD_LINUX_STD_FW_API */
+#else
+	memblock = dhd_os_open_image1(dhd, (char *)clm_blob_path);
+	len = dhd_os_get_image_size(memblock);
+	BCM_REFERENCE(memblock_len);
+#endif /* !LINUX && !linux || DHD_LINUX_STD_FW_API */
+
+#if defined(LINUX) || defined(linux)
+	if (memblock == NULL) {
+		DHD_TRACE(("%s: Ignore clm file %s\n", __FUNCTION__, clm_path));
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+		if (!dhd->is_blob) {
+			status = dhd_check_current_clm_data(dhd);
+			if (status == TRUE) {
+				err = BCME_OK;
+			} else {
+				err = status;
+			}
+		}
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+		goto exit;
+	}
+#endif /* defined(LINUX) || defined(linux) */
+
+	if ((len > 0) && (len < MAX_CLM_BUF_SIZE) && memblock) {
+		status = dhd_check_current_clm_data(dhd);
+		if (status == TRUE) {
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+			if (dhd->op_mode != DHD_FLAG_MFG_MODE) {
+				if (dhd->is_blob) {
+					err = BCME_ERROR;
+				}
+				goto exit;
+			}
+#else
+			DHD_ERROR(("%s: CLM already exist in F/W, "
+				"new CLM data will be added to the end of existing CLM data!\n",
+				__FUNCTION__));
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+		} else if (status != FALSE) {
+			err = status;
+			goto exit;
+		}
+
+		/* Found blob file. Download the file */
+		DHD_TRACE(("clm file download from %s \n", clm_blob_path));
+		err = dhd_download_blob(dhd, (unsigned char*)memblock, len, "clmload");
+		if (err) {
+			DHD_ERROR(("%s: CLM download failed err=%d\n", __FUNCTION__, err));
+			/* Retrieve clmload_status and print */
+			memset(iovbuf, 0, sizeof(iovbuf));
+			len = bcm_mkiovar("clmload_status", NULL, 0, iovbuf, sizeof(iovbuf));
+			if (len == 0) {
+				err = BCME_BUFTOOSHORT;
+				goto exit;
+			}
+			err = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf, sizeof(iovbuf), FALSE, 0);
+			if (err) {
+				DHD_ERROR(("%s: clmload_status get failed err=%d \n",
+					__FUNCTION__, err));
+			} else {
+				DHD_ERROR(("%s: clmload_status: %d \n",
+					__FUNCTION__, *((int *)iovbuf)));
+				if (*((int *)iovbuf) == CHIPID_MISMATCH) {
+					DHD_ERROR(("Chip ID mismatch error \n"));
+				}
+			}
+			err = BCME_ERROR;
+			goto exit;
+		} else {
+			DHD_ERROR(("%s: CLM download succeeded \n", __FUNCTION__));
+		}
+	} else {
+		DHD_INFO(("Skipping the clm download. len:%d memblk:%p \n", len, memblock));
+	}
+
+	/* Verify country code */
+	status = dhd_check_current_clm_data(dhd);
+
+	if (status != TRUE) {
+		/* Country code not initialized or CLM download not proper */
+		DHD_ERROR(("country code not initialized\n"));
+		err = status;
+	}
+exit:
+
+	if (memblock) {
+#ifdef SUPPORT_OTA_UPDATE
+		if (ota_info->clm_len) {
+			MFREE(dhd->osh, ota_info->clm_buf, ota_info->clm_len);
+			ota_info->clm_len = 0;
+		}
+		else
+#endif /* SUPPORT_OTA_UPDATE */
+		{
+#if (defined(LINUX) || defined(linux)) && !defined(DHD_LINUX_STD_FW_API)
+			dhd_os_close_image1(dhd, memblock);
+#else
+			dhd_free_download_buffer(dhd, memblock, memblock_len);
+#endif /* LINUX || linux */
+		}
+	}
+
+	return err;
+}
+#endif /* defined(CACHE_FW_IMAGES) */
+
+void dhd_free_download_buffer(dhd_pub_t	*dhd, void *buffer, int length)
+{
+#ifdef CACHE_FW_IMAGES
+	return;
+#endif
+#if defined(DHD_LINUX_STD_FW_API)
+	VMFREE(dhd->osh, buffer, length);
+#else
+	MFREE(dhd->osh, buffer, length);
+#endif /* DHD_LINUX_STD_FW_API */
+}
+
+#ifdef REPORT_FATAL_TIMEOUTS
+void
+init_dhd_timeouts(dhd_pub_t *pub)
+{
+	pub->timeout_info = MALLOC(pub->osh, sizeof(timeout_info_t));
+	if (pub->timeout_info == NULL) {
+		DHD_ERROR(("%s: Failed to alloc timeout_info\n", __FUNCTION__));
+	} else {
+		DHD_INFO(("Initializing dhd_timeouts\n"));
+		pub->timeout_info->scan_timer_lock = osl_spin_lock_init(pub->osh);
+		pub->timeout_info->join_timer_lock = osl_spin_lock_init(pub->osh);
+		pub->timeout_info->bus_timer_lock = osl_spin_lock_init(pub->osh);
+		pub->timeout_info->cmd_timer_lock = osl_spin_lock_init(pub->osh);
+		pub->timeout_info->scan_timeout_val = SCAN_TIMEOUT_DEFAULT;
+		pub->timeout_info->join_timeout_val = JOIN_TIMEOUT_DEFAULT;
+		pub->timeout_info->cmd_timeout_val = CMD_TIMEOUT_DEFAULT;
+		pub->timeout_info->bus_timeout_val = BUS_TIMEOUT_DEFAULT;
+		pub->timeout_info->scan_timer_active = FALSE;
+		pub->timeout_info->join_timer_active = FALSE;
+		pub->timeout_info->cmd_timer_active = FALSE;
+		pub->timeout_info->bus_timer_active = FALSE;
+		pub->timeout_info->cmd_join_error = FALSE;
+		pub->timeout_info->cmd_request_id = 0;
+		OSL_ATOMIC_SET(pub->osh, &pub->set_ssid_rcvd, FALSE);
+	}
+}
+
+void
+deinit_dhd_timeouts(dhd_pub_t *pub)
+{
+	/* stop the join, scan bus, cmd timers
+	 * as failing to do so may cause a kernel panic if
+	 * an rmmod is done
+	 */
+	if (!pub->timeout_info) {
+		DHD_ERROR(("%s timeout_info pointer is NULL\n", __FUNCTION__));
+		ASSERT(0);
+		return;
+	}
+	if (dhd_stop_scan_timer(pub, FALSE, 0)) {
+		DHD_ERROR(("%s dhd_stop_scan_timer failed\n", __FUNCTION__));
+		ASSERT(0);
+	}
+	if (dhd_stop_bus_timer(pub)) {
+		DHD_ERROR(("%s dhd_stop_bus_timer failed\n", __FUNCTION__));
+		ASSERT(0);
+	}
+	if (dhd_stop_cmd_timer(pub)) {
+		DHD_ERROR(("%s dhd_stop_cmd_timer failed\n", __FUNCTION__));
+		ASSERT(0);
+	}
+	if (dhd_stop_join_timer(pub)) {
+		DHD_ERROR(("%s dhd_stop_join_timer failed\n", __FUNCTION__));
+		ASSERT(0);
+	}
+
+	osl_spin_lock_deinit(pub->osh, pub->timeout_info->scan_timer_lock);
+	osl_spin_lock_deinit(pub->osh, pub->timeout_info->join_timer_lock);
+	osl_spin_lock_deinit(pub->osh, pub->timeout_info->bus_timer_lock);
+	osl_spin_lock_deinit(pub->osh, pub->timeout_info->cmd_timer_lock);
+	MFREE(pub->osh, pub->timeout_info, sizeof(timeout_info_t));
+}
+
+static void
+dhd_cmd_timeout(void *ctx)
+{
+	dhd_pub_t *pub = (dhd_pub_t *)ctx;
+	unsigned long flags;
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ASSERT(0);
+		return;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->cmd_timer_lock, flags);
+	if (pub->timeout_info && pub->timeout_info->cmd_timer_active) {
+		DHD_ERROR(("\nERROR COMMAND TIMEOUT TO:%d\n", pub->timeout_info->cmd_timeout_val));
+		DHD_TIMER_UNLOCK(pub->timeout_info->cmd_timer_lock, flags);
+#ifdef PCIE_OOB
+		/* Assert device_wake so that UART_Rx is available */
+		if (dhd_bus_set_device_wake(pub->bus, TRUE)) {
+			DHD_ERROR(("%s: dhd_bus_set_device_wake() failed\n", __FUNCTION__));
+			ASSERT(0);
+		}
+#endif /* PCIE_OOB */
+		if (dhd_stop_cmd_timer(pub)) {
+			DHD_ERROR(("%s: dhd_stop_cmd_timer() failed\n", __FUNCTION__));
+			ASSERT(0);
+		}
+		if (!dhd_query_bus_erros(pub))
+			dhd_send_trap_to_fw_for_timeout(pub, DHD_REASON_COMMAND_TO);
+	} else {
+		DHD_TIMER_UNLOCK(pub->timeout_info->cmd_timer_lock, flags);
+	}
+}
+
+int
+dhd_start_cmd_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	unsigned long flags = 0;
+	uint32 cmd_to_ms;
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit_null;
+	}
+	DHD_TIMER_LOCK(pub->timeout_info->cmd_timer_lock, flags);
+	cmd_to_ms = pub->timeout_info->cmd_timeout_val;
+
+	if (pub->timeout_info->cmd_timeout_val == 0) {
+		/* Disable Command timer timeout */
+		DHD_INFO(("DHD: Command Timeout Disabled\n"));
+		goto exit;
+	}
+	if (pub->timeout_info->cmd_timer_active) {
+		DHD_ERROR(("%s:Timer already active\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		ASSERT(0);
+	} else {
+		pub->timeout_info->cmd_timer = osl_timer_init(pub->osh,
+			"cmd_timer", dhd_cmd_timeout, pub);
+		osl_timer_update(pub->osh, pub->timeout_info->cmd_timer,
+			cmd_to_ms, 0);
+		pub->timeout_info->cmd_timer_active = TRUE;
+	}
+	if (ret == BCME_OK) {
+		DHD_INFO(("%s Cmd Timer started\n", __FUNCTION__));
+	}
+exit:
+	DHD_TIMER_UNLOCK(pub->timeout_info->cmd_timer_lock, flags);
+exit_null:
+	return ret;
+}
+
+int
+dhd_stop_cmd_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	unsigned long flags = 0;
+
+	if (!pub) {
+		DHD_ERROR(("DHD: pub NULL\n"));
+		ASSERT(0);
+		return BCME_ERROR;
+	}
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit;
+	}
+	DHD_TIMER_LOCK(pub->timeout_info->cmd_timer_lock, flags);
+
+	if (pub->timeout_info->cmd_timer_active) {
+		osl_timer_del(pub->osh, pub->timeout_info->cmd_timer);
+		pub->timeout_info->cmd_timer_active = FALSE;
+	}
+	else {
+		DHD_INFO(("DHD: CMD timer is not active\n"));
+	}
+	if (ret == BCME_OK) {
+		DHD_INFO(("%s Cmd Timer Stopped\n", __FUNCTION__));
+	}
+	DHD_TIMER_UNLOCK(pub->timeout_info->cmd_timer_lock, flags);
+exit:
+	return ret;
+}
+
+static int
+__dhd_stop_join_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	if (!pub) {
+		DHD_ERROR(("DHD: pub NULL\n"));
+		ASSERT(0);
+		return BCME_ERROR;
+	}
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ASSERT(0);
+		return BCME_ERROR;
+	}
+
+	if (pub->timeout_info->join_timer_active) {
+		osl_timer_del(pub->osh, pub->timeout_info->join_timer);
+		pub->timeout_info->join_timer_active = FALSE;
+		DHD_INFO(("%s join timer stopped\n", __FUNCTION__));
+	} else {
+		DHD_INFO(("%s join timer is not active\n", __FUNCTION__));
+	}
+
+	return ret;
+}
+
+static void
+dhd_join_timeout(void *ctx)
+{
+	dhd_pub_t *pub = (dhd_pub_t *)ctx;
+	unsigned long flags;
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("%s DHD: timeout_info NULL\n", __FUNCTION__));
+		ASSERT(0);
+		return;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->join_timer_lock, flags);
+	if (pub->timeout_info->join_timer_active) {
+		if (__dhd_stop_join_timer(pub)) {
+			DHD_ERROR(("%s: __dhd_stop_join_timer() failed\n", __FUNCTION__));
+			ASSERT(0);
+		}
+		if (pub->timeout_info->cmd_join_error) {
+			DHD_ERROR(("\n%s ERROR JOIN TIMEOUT TO:%d:0x%x\n", __FUNCTION__,
+				pub->timeout_info->join_timeout_val,
+				pub->timeout_info->cmd_join_error));
+			if (!dhd_query_bus_erros(pub)) {
+				dhd_send_trap_to_fw_for_timeout(pub, DHD_REASON_JOIN_TO);
+			}
+			pub->timeout_info->cmd_join_error = 0;
+		}
+	}
+	DHD_TIMER_UNLOCK(pub->timeout_info->join_timer_lock, flags);
+}
+
+int
+dhd_start_join_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	unsigned long flags = 0;
+	uint32 join_to_ms;
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("%s DHD: timeout_info NULL\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit;
+	}
+
+	join_to_ms = pub->timeout_info->join_timeout_val;
+	DHD_TIMER_LOCK(pub->timeout_info->join_timer_lock, flags);
+	if (pub->timeout_info->join_timer_active) {
+		DHD_ERROR(("%s: stopping active timer\n", __FUNCTION__));
+		__dhd_stop_join_timer(pub);
+	}
+	if (pub->timeout_info->join_timeout_val == 0) {
+		/* Disable Join timer timeout */
+		DHD_INFO(("%s DHD: join timeout disabled\n", __FUNCTION__));
+	} else {
+		pub->timeout_info->join_timer = osl_timer_init(pub->osh,
+			"join_timer", dhd_join_timeout, pub);
+		osl_timer_update(pub->osh, pub->timeout_info->join_timer, join_to_ms, 0);
+		pub->timeout_info->join_timer_active = TRUE;
+		pub->timeout_info->cmd_join_error = 0;
+		dhd_set_join_error(pub, WLC_SSID_MASK);
+		if (pub->secure_join) {
+			dhd_set_join_error(pub, WLC_WPA_MASK);
+		}
+		DHD_ERROR(("%s: join timer started 0x%x\n", __FUNCTION__,
+			pub->timeout_info->cmd_join_error));
+	}
+	DHD_TIMER_UNLOCK(pub->timeout_info->join_timer_lock, flags);
+exit:
+	return ret;
+}
+
+int
+dhd_stop_join_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	unsigned long flags;
+
+	if (!pub) {
+		DHD_ERROR(("%s DHD: pub NULL\n", __FUNCTION__));
+		ASSERT(0);
+		return BCME_ERROR;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->join_timer_lock, flags);
+	ret = __dhd_stop_join_timer(pub);
+	DHD_TIMER_UNLOCK(pub->timeout_info->join_timer_lock, flags);
+	return ret;
+}
+
+static void
+dhd_set_join_error(dhd_pub_t *pub, uint32 mask)
+{
+	DHD_INFO(("Setting join Error %d\n", mask));
+	if (pub->timeout_info) {
+		pub->timeout_info->cmd_join_error |= mask;
+	}
+}
+
+void
+dhd_clear_join_error(dhd_pub_t *pub, uint32 mask)
+{
+	unsigned long flags;
+
+	DHD_INFO(("%s clear join error %d\n", __FUNCTION__, mask));
+	if (!(pub->timeout_info)) {
+		return;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->join_timer_lock, flags);
+	pub->timeout_info->cmd_join_error &= ~mask;
+	/* If both WLC_SSID_MASK, WLC_WPA_MASK are received cancel the timer */
+	if (!(pub->timeout_info->cmd_join_error)) {
+		if (__dhd_stop_join_timer(pub)) {
+			DHD_ERROR(("%s: dhd_stop_join_timer failed\n", __FUNCTION__));
+			ASSERT(0);
+		}
+	}
+	DHD_TIMER_UNLOCK(pub->timeout_info->join_timer_lock, flags);
+}
+
+static void
+dhd_scan_timeout(void *ctx)
+{
+	dhd_pub_t *pub = (dhd_pub_t *)ctx;
+	unsigned long flags;
+
+	if (!pub) {
+		DHD_ERROR(("DHD: pub NULL\n"));
+		ASSERT(0);
+		return;
+	}
+
+	if (pub->timeout_info == NULL) {
+		DHD_ERROR(("timeout_info pointer is NULL\n"));
+		ASSERT(0);
+		return;
+	}
+	DHD_TIMER_LOCK(pub->timeout_info->scan_timer_lock, flags);
+	if (pub->timeout_info->scan_timer_active) {
+		DHD_ERROR(("\nERROR SCAN TIMEOUT TO:%d\n", pub->timeout_info->scan_timeout_val));
+		DHD_TIMER_UNLOCK(pub->timeout_info->scan_timer_lock, flags);
+		dhd_stop_scan_timer(pub, FALSE, 0);
+		if (!dhd_query_bus_erros(pub))
+			dhd_send_trap_to_fw_for_timeout(pub, DHD_REASON_SCAN_TO);
+	} else {
+		DHD_TIMER_UNLOCK(pub->timeout_info->scan_timer_lock, flags);
+	}
+}
+
+int
+dhd_start_scan_timer(dhd_pub_t *pub, bool is_escan)
+{
+	int ret = BCME_OK;
+	unsigned long flags = 0;
+	uint32 scan_to_ms;
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit_null;
+	}
+	DHD_TIMER_LOCK(pub->timeout_info->scan_timer_lock, flags);
+	scan_to_ms = pub->timeout_info->scan_timeout_val;
+
+	if (is_escan) {
+		if (pub->timeout_info->escan_aborted &&
+				pub->esync_id == pub->timeout_info->abort_syncid) {
+			pub->timeout_info->escan_aborted = FALSE;
+			DHD_INFO(("%s: escan already aborted, do not start timer \n",
+				__FUNCTION__));
+			goto exit;
+		}
+		pub->timeout_info->escan_syncid = pub->esync_id;
+	} else {
+		pub->timeout_info->escan_syncid = 0;
+	}
+
+	if (pub->timeout_info->scan_timer_active) {
+		/* cancel any earlier running timer */
+		DHD_INFO(("%s:Timer already active, stopping it.\n", __FUNCTION__));
+		osl_timer_del(pub->osh, pub->timeout_info->scan_timer);
+		pub->timeout_info->scan_timer_active = FALSE;
+	}
+
+	if (pub->timeout_info->scan_timeout_val == 0) {
+		/* Disable Scan timer timeout */
+		DHD_INFO(("DHD: Scan Timeout Disabled\n"));
+	} else {
+		pub->timeout_info->scan_timer = osl_timer_init(pub->osh, "scan_timer",
+			dhd_scan_timeout, pub);
+		pub->timeout_info->scan_timer_active = TRUE;
+		osl_timer_update(pub->osh, pub->timeout_info->scan_timer, scan_to_ms, 0);
+		DHD_INFO(("%s Scan Timer started\n", __FUNCTION__));
+	}
+
+exit:
+	DHD_TIMER_UNLOCK(pub->timeout_info->scan_timer_lock, flags);
+exit_null:
+	return ret;
+}
+
+int
+dhd_stop_scan_timer(dhd_pub_t *pub, bool is_escan, uint16 sync_id)
+{
+	int ret = BCME_OK;
+	unsigned long flags = 0;
+
+	if (!pub) {
+		DHD_ERROR(("DHD: pub NULL\n"));
+		ASSERT(0);
+		return BCME_ERROR;
+	}
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit_null;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->scan_timer_lock, flags);
+
+	if (pub->timeout_info->scan_timer_active) {
+		if (is_escan) {
+			if (sync_id == pub->timeout_info->escan_syncid) {
+				osl_timer_del(pub->osh, pub->timeout_info->scan_timer);
+				pub->timeout_info->scan_timer_active = FALSE;
+				DHD_INFO(("%s Scan Timer Stopped\n", __FUNCTION__));
+			}
+		} else {
+			osl_timer_del(pub->osh, pub->timeout_info->scan_timer);
+			pub->timeout_info->scan_timer_active = FALSE;
+			DHD_INFO(("%s Scan Timer Stopped\n", __FUNCTION__));
+		}
+
+	} else {
+		DHD_INFO(("DHD: SCAN timer is not active\n"));
+	}
+
+	DHD_TIMER_UNLOCK(pub->timeout_info->scan_timer_lock, flags);
+
+exit_null:
+	return ret;
+}
+
+static void
+dhd_bus_timeout(void *ctx)
+{
+	dhd_pub_t *pub = (dhd_pub_t *)ctx;
+	unsigned long flags;
+
+	if (pub->timeout_info == NULL) {
+		DHD_ERROR(("timeout_info pointer is NULL\n"));
+		ASSERT(0);
+		return;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->bus_timer_lock, flags);
+	if (pub->timeout_info && pub->timeout_info->bus_timer_active) {
+		DHD_ERROR(("\nERROR BUS TIMEOUT TO:%d\n", pub->timeout_info->bus_timeout_val));
+		DHD_TIMER_UNLOCK(pub->timeout_info->bus_timer_lock, flags);
+#ifdef PCIE_OOB
+		/* Assert device_wake so that UART_Rx is available */
+		if (dhd_bus_set_device_wake(pub->bus, TRUE)) {
+			DHD_ERROR(("%s: dhd_bus_set_device_wake() failed\n", __FUNCTION__));
+			ASSERT(0);
+		}
+#endif /* PCIE_OOB */
+		if (dhd_stop_bus_timer(pub)) {
+			DHD_ERROR(("%s: dhd_stop_bus_timer() failed\n", __FUNCTION__));
+			ASSERT(0);
+		}
+		if (!dhd_query_bus_erros(pub)) {
+			dhd_send_trap_to_fw_for_timeout(pub, DHD_REASON_OQS_TO);
+		}
+#ifdef BCMPCIE
+		dhd_msgbuf_iovar_timeout_dump(pub);
+#endif /* BCMPCIE */
+	} else {
+		DHD_TIMER_UNLOCK(pub->timeout_info->bus_timer_lock, flags);
+	}
+}
+
+int
+dhd_start_bus_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	unsigned long flags = 0;
+	uint32 bus_to_ms;
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit_null;
+	}
+	DHD_TIMER_LOCK(pub->timeout_info->bus_timer_lock, flags);
+	bus_to_ms = pub->timeout_info->bus_timeout_val;
+
+	if (pub->timeout_info->bus_timeout_val == 0) {
+		/* Disable Bus timer timeout */
+		DHD_INFO(("DHD: Bus Timeout Disabled\n"));
+		goto exit;
+	}
+	if (pub->timeout_info->bus_timer_active) {
+		DHD_ERROR(("%s:Timer already active\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		ASSERT(0);
+	} else {
+		pub->timeout_info->bus_timer = osl_timer_init(pub->osh,
+			"bus_timer", dhd_bus_timeout, pub);
+		pub->timeout_info->bus_timer_active = TRUE;
+		osl_timer_update(pub->osh, pub->timeout_info->bus_timer, bus_to_ms, 0);
+	}
+	if (ret == BCME_OK) {
+		DHD_INFO(("%s: BUS Timer started\n", __FUNCTION__));
+	}
+exit:
+	DHD_TIMER_UNLOCK(pub->timeout_info->bus_timer_lock, flags);
+exit_null:
+	return ret;
+}
+
+int
+dhd_stop_bus_timer(dhd_pub_t *pub)
+{
+	int ret = BCME_OK;
+	unsigned long flags;
+
+	if (!pub) {
+		DHD_ERROR(("DHD: pub NULL\n"));
+		ASSERT(0);
+		return BCME_ERROR;
+	}
+
+	if (!pub->timeout_info) {
+		DHD_ERROR(("DHD: timeout_info NULL\n"));
+		ret = BCME_ERROR;
+		ASSERT(0);
+		goto exit;
+	}
+
+	DHD_TIMER_LOCK(pub->timeout_info->bus_timer_lock, flags);
+
+	if (pub->timeout_info->bus_timer_active) {
+		osl_timer_del(pub->osh, pub->timeout_info->bus_timer);
+		pub->timeout_info->bus_timer_active = FALSE;
+	}
+	else {
+		DHD_INFO(("DHD: BUS timer is not active\n"));
+	}
+	if (ret == BCME_OK) {
+		DHD_INFO(("%s: Bus Timer Stopped\n", __FUNCTION__));
+	}
+	DHD_TIMER_UNLOCK(pub->timeout_info->bus_timer_lock, flags);
+exit:
+	return ret;
+}
+
+int
+dhd_set_request_id(dhd_pub_t *pub, uint16 id, uint32 cmd)
+{
+	DHD_INFO(("%s: id:%d\n", __FUNCTION__, id));
+	if (pub->timeout_info) {
+		pub->timeout_info->cmd_request_id = id;
+		pub->timeout_info->cmd = cmd;
+		return BCME_OK;
+	} else {
+		return BCME_ERROR;
+	}
+}
+
+uint16
+dhd_get_request_id(dhd_pub_t *pub)
+{
+	if (pub->timeout_info) {
+		return (pub->timeout_info->cmd_request_id);
+	} else {
+		return 0;
+	}
+}
+
+void
+dhd_get_scan_to_val(dhd_pub_t *pub, uint32 *to_val)
+{
+	if (pub->timeout_info) {
+		*to_val = pub->timeout_info->scan_timeout_val;
+	} else {
+		*to_val = 0;
+	}
+}
+
+void
+dhd_set_scan_to_val(dhd_pub_t *pub, uint32 to_val)
+{
+	if (pub->timeout_info) {
+		DHD_INFO(("Setting scan TO val:%d\n", to_val));
+		pub->timeout_info->scan_timeout_val = to_val;
+	}
+}
+
+void
+dhd_get_join_to_val(dhd_pub_t *pub, uint32 *to_val)
+{
+	if (pub->timeout_info) {
+		*to_val = pub->timeout_info->join_timeout_val;
+	} else {
+		*to_val = 0;
+	}
+}
+
+void
+dhd_set_join_to_val(dhd_pub_t *pub, uint32 to_val)
+{
+	if (pub->timeout_info) {
+		DHD_INFO(("Setting join TO val:%d\n", to_val));
+		pub->timeout_info->join_timeout_val = to_val;
+	}
+}
+
+void
+dhd_get_cmd_to_val(dhd_pub_t *pub, uint32 *to_val)
+{
+	if (pub->timeout_info) {
+		*to_val = pub->timeout_info->cmd_timeout_val;
+	} else {
+		*to_val = 0;
+	}
+}
+
+void
+dhd_set_cmd_to_val(dhd_pub_t *pub, uint32 to_val)
+{
+	if (pub->timeout_info) {
+		DHD_INFO(("Setting cmd TO val:%d\n", to_val));
+		pub->timeout_info->cmd_timeout_val = to_val;
+	}
+}
+
+void
+dhd_get_bus_to_val(dhd_pub_t *pub, uint32 *to_val)
+{
+	if (pub->timeout_info) {
+		*to_val = pub->timeout_info->bus_timeout_val;
+	} else {
+		*to_val = 0;
+	}
+}
+
+void
+dhd_set_bus_to_val(dhd_pub_t *pub, uint32 to_val)
+{
+	if (pub->timeout_info) {
+		DHD_INFO(("Setting bus TO val:%d\n", to_val));
+		pub->timeout_info->bus_timeout_val = to_val;
+	}
+}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+#ifdef SHOW_LOGTRACE
+int
+dhd_parse_logstrs_file(osl_t *osh, char *raw_fmts, int logstrs_size,
+		dhd_event_log_t *event_log)
+{
+	uint32 *lognums = NULL;
+	char *logstrs = NULL;
+	logstr_trailer_t *trailer = NULL;
+	int ram_index = 0;
+	char **fmts = NULL;
+	int num_fmts = 0;
+	bool match_fail = TRUE;
+	int32 i = 0;
+	uint8 *pfw_id = NULL;
+	uint32 fwid = 0;
+	char fwid_str[FWID_STR_LEN];
+	uint32 hdr_logstrs_size = 0;
+#ifdef DHD_LINUX_STD_FW_API
+	int err = 0;
+	const struct firmware *fw = NULL;
+#else
+	void *file = NULL;
+	int file_len = 0;
+#endif /* DHD_LINUX_STD_FW_API */
+
+	/* Read last three words in the logstrs.bin file */
+	trailer = (logstr_trailer_t *) (raw_fmts + logstrs_size -
+		sizeof(logstr_trailer_t));
+	if (trailer->log_magic == LOGSTRS_MAGIC) {
+		/*
+		* logstrs.bin has a header.
+		*/
+		if (trailer->version == 1) {
+			logstr_header_v1_t *hdr_v1 = (logstr_header_v1_t *) (raw_fmts +
+					logstrs_size - sizeof(logstr_header_v1_t));
+			DHD_INFO(("%s: logstr header version = %u\n",
+					__FUNCTION__, hdr_v1->version));
+			num_fmts =	hdr_v1->rom_logstrs_offset / sizeof(uint32);
+			ram_index = (hdr_v1->ram_lognums_offset -
+				hdr_v1->rom_lognums_offset) / sizeof(uint32);
+			lognums = (uint32 *) &raw_fmts[hdr_v1->rom_lognums_offset];
+			logstrs = (char *)	 &raw_fmts[hdr_v1->rom_logstrs_offset];
+			hdr_logstrs_size = hdr_v1->logstrs_size;
+		} else if (trailer->version == 2) {
+			logstr_header_t *hdr = (logstr_header_t *) (raw_fmts + logstrs_size -
+					sizeof(logstr_header_t));
+			DHD_INFO(("%s: logstr header version = %u; flags = %x\n",
+					__FUNCTION__, hdr->version, hdr->flags));
+
+			/* For ver. 2 of the header, need to match fwid of
+			 *  both logstrs.bin and fw bin
+			 */
+#ifdef DHD_LINUX_STD_FW_API
+			err = dhd_os_get_img_fwreq(&fw, st_str_file_path);
+			if (err < 0) {
+				DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+					err));
+				goto error;
+			}
+			memset(fwid_str, 0, sizeof(fwid_str));
+			err = memcpy_s(fwid_str, (sizeof(fwid_str) - 1),
+				&(fw->data[fw->size - (sizeof(fwid_str) - 1)]),
+				(sizeof(fwid_str) - 1));
+			if (err) {
+				DHD_ERROR(("%s: failed to copy raw_fmts, err=%d\n",
+					__FUNCTION__, err));
+				goto error;
+			}
+#else
+			/* read the FWID from fw bin */
+			file = dhd_os_open_image1(NULL, st_str_file_path);
+			if (!file) {
+				DHD_ERROR(("%s: cannot open fw file !\n", __FUNCTION__));
+				goto error;
+			}
+			file_len = dhd_os_get_image_size(file);
+			if (file_len <= 0) {
+				DHD_ERROR(("%s: bad fw file length !\n", __FUNCTION__));
+				goto error;
+			}
+			/* fwid is at the end of fw bin in string format */
+			if (dhd_os_seek_file(file, file_len - (sizeof(fwid_str) - 1)) < 0) {
+				DHD_ERROR(("%s: can't seek file \n", __FUNCTION__));
+				goto error;
+			}
+
+			memset(fwid_str, 0, sizeof(fwid_str));
+			if (dhd_os_get_image_block(fwid_str, sizeof(fwid_str) - 1, file) <= 0) {
+				DHD_ERROR(("%s: read fw file failed !\n", __FUNCTION__));
+				goto error;
+			}
+#endif /* DHD_LINUX_STD_FW_API */
+			pfw_id = (uint8 *)bcmstrnstr(fwid_str, sizeof(fwid_str) - 1,
+					FWID_STR_1, strlen(FWID_STR_1));
+			if (!pfw_id) {
+				pfw_id = (uint8 *)bcmstrnstr(fwid_str, sizeof(fwid_str) - 1,
+					FWID_STR_2, strlen(FWID_STR_2));
+				if (!pfw_id) {
+					DHD_ERROR(("%s: could not find id in FW bin!\n",
+							__FUNCTION__));
+					goto error;
+				}
+			}
+			/* search for the '-' in the fw id str, after which the
+			 * actual 4 byte fw id is present
+			 */
+			while (pfw_id && *pfw_id != '-') {
+				++pfw_id;
+			}
+			++pfw_id;
+			fwid = bcm_strtoul((char *)pfw_id, NULL, 16);
+
+			/* check if fw id in logstrs.bin matches the fw one */
+			if (hdr->fw_id != fwid) {
+				DHD_ERROR(("%s: logstr id does not match FW!"
+					"logstrs_fwid:0x%x, rtecdc_fwid:0x%x\n",
+					__FUNCTION__, hdr->fw_id, fwid));
+				goto error;
+			}
+			match_fail = FALSE;
+			num_fmts = hdr->rom_logstrs_offset / sizeof(uint32);
+			ram_index = (hdr->ram_lognums_offset -
+				hdr->rom_lognums_offset) / sizeof(uint32);
+			lognums = (uint32 *) &raw_fmts[hdr->rom_lognums_offset];
+			logstrs = (char *)	 &raw_fmts[hdr->rom_logstrs_offset];
+			hdr_logstrs_size = hdr->logstrs_size;
+
+error:
+#ifdef DHD_LINUX_STD_FW_API
+			if (fw) {
+				dhd_os_close_img_fwreq(fw);
+			}
+#else
+			if (file) {
+				dhd_os_close_image1(NULL, file);
+			}
+#endif /* DHD_LINUX_STD_FW_API */
+			if (match_fail) {
+				return BCME_DECERR;
+			}
+		} else {
+			DHD_ERROR(("%s: Invalid logstr version %u\n", __FUNCTION__,
+					trailer->version));
+			return BCME_ERROR;
+		}
+		if (logstrs_size != hdr_logstrs_size) {
+			DHD_ERROR(("%s: bad logstrs_size %d\n", __FUNCTION__, hdr_logstrs_size));
+			return BCME_ERROR;
+		}
+	} else {
+		/*
+		 * Legacy logstrs.bin format without header.
+		 */
+		num_fmts = *((uint32 *) (raw_fmts)) / sizeof(uint32);
+
+		/* Legacy RAM-only logstrs.bin format:
+		 *	  - RAM 'lognums' section
+		 *	  - RAM 'logstrs' section.
+		 *
+		 * 'lognums' is an array of indexes for the strings in the
+		 * 'logstrs' section. The first uint32 is an index to the
+		 * start of 'logstrs'. Therefore, if this index is divided
+		 * by 'sizeof(uint32)' it provides the number of logstr
+		 *	entries.
+		 */
+		ram_index = 0;
+		lognums = (uint32 *) raw_fmts;
+		logstrs = (char *) &raw_fmts[num_fmts << 2];
+	}
+	if (num_fmts) {
+		if (event_log->fmts != NULL) {
+			fmts = event_log->fmts;	/* reuse existing malloced fmts */
+		} else {
+			fmts = MALLOC(osh, num_fmts  * sizeof(char *));
+		}
+	}
+	if (fmts == NULL) {
+		DHD_ERROR(("%s: Failed to allocate fmts memory\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	event_log->fmts_size = num_fmts  * sizeof(char *);
+	for (i = 0; i < num_fmts; i++) {
+		/* ROM lognums index into logstrs using 'rom_logstrs_offset' as a base
+		* (they are 0-indexed relative to 'rom_logstrs_offset').
+		*
+		* RAM lognums are already indexed to point to the correct RAM logstrs (they
+		* are 0-indexed relative to the start of the logstrs.bin file).
+		*/
+		if (i == ram_index) {
+			logstrs = raw_fmts;
+		}
+		fmts[i] = &logstrs[lognums[i]];
+	}
+	event_log->fmts = fmts;
+	event_log->raw_fmts_size = logstrs_size;
+	event_log->raw_fmts = raw_fmts;
+	event_log->num_fmts = num_fmts;
+	return BCME_OK;
+} /* dhd_parse_logstrs_file */
+
+#ifdef DHD_LINUX_STD_FW_API
+int dhd_parse_map_file(osl_t *osh, const void *ptr, uint32 *ramstart, uint32 *rodata_start,
+		uint32 *rodata_end)
+{
+	char *raw_fmts =  NULL, *raw_fmts_loc = NULL;
+	uint32 read_size = READ_NUM_BYTES, offset = 0;
+	int error = 0;
+	char * cptr = NULL;
+	char c;
+	uint8 count = 0;
+	uint32 size = 0;
+
+	*ramstart = 0;
+	*rodata_start = 0;
+	*rodata_end = 0;
+	size = (uint32)(((const struct firmware *)ptr)->size);
+
+	/* Allocate 1 byte more than read_size to terminate it with NULL */
+	raw_fmts = MALLOCZ(osh, read_size + 1);
+	if (raw_fmts == NULL) {
+		DHD_ERROR(("%s: Failed to allocate raw_fmts memory \n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* read ram start, rodata_start and rodata_end values from map  file */
+	while (count != ALL_MAP_VAL)
+	{
+		/* Bound check for size before doing memcpy() */
+		if ((offset + read_size) > size) {
+			read_size = size - offset;
+		}
+
+		error = memcpy_s(raw_fmts, read_size,
+			(((const char *)((const struct firmware *)ptr)->data) + offset), read_size);
+		if (error) {
+			DHD_ERROR(("%s: failed to copy raw_fmts, err=%d\n",
+				__FUNCTION__, error));
+			goto fail;
+		}
+		/* End raw_fmts with NULL as strstr expects NULL terminated strings */
+		raw_fmts[read_size] = '\0';
+
+		/* Get ramstart address */
+		raw_fmts_loc = raw_fmts;
+		if (!(count & RAMSTART_BIT) &&
+			(cptr = bcmstrnstr(raw_fmts_loc, read_size, ramstart_str,
+			strlen(ramstart_str)))) {
+			cptr = cptr - BYTES_AHEAD_NUM;
+			sscanf(cptr, "%x %c text_start", ramstart, &c);
+			count |= RAMSTART_BIT;
+		}
+
+		/* Get ram rodata start address */
+		raw_fmts_loc = raw_fmts;
+		if (!(count & RDSTART_BIT) &&
+			(cptr = bcmstrnstr(raw_fmts_loc, read_size, rodata_start_str,
+			strlen(rodata_start_str)))) {
+			cptr = cptr - BYTES_AHEAD_NUM;
+			sscanf(cptr, "%x %c rodata_start", rodata_start, &c);
+			count |= RDSTART_BIT;
+		}
+
+		/* Get ram rodata end address */
+		raw_fmts_loc = raw_fmts;
+		if (!(count & RDEND_BIT) &&
+			(cptr = bcmstrnstr(raw_fmts_loc, read_size, rodata_end_str,
+			strlen(rodata_end_str)))) {
+			cptr = cptr - BYTES_AHEAD_NUM;
+			sscanf(cptr, "%x %c rodata_end", rodata_end, &c);
+			count |= RDEND_BIT;
+		}
+
+		if ((offset + read_size) >= size) {
+			break;
+		}
+
+		memset(raw_fmts, 0, read_size);
+		offset += (read_size - GO_BACK_FILE_POS_NUM_BYTES);
+	}
+
+fail:
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, read_size + 1);
+		raw_fmts = NULL;
+	}
+	if (count == ALL_MAP_VAL) {
+		return BCME_OK;
+	}
+	else {
+		DHD_ERROR(("%s: readmap error 0X%x \n", __FUNCTION__,
+				count));
+		return BCME_ERROR;
+	}
+} /* dhd_parse_map_file */
+#else
+int dhd_parse_map_file(osl_t *osh, void *file, uint32 *ramstart, uint32 *rodata_start,
+		uint32 *rodata_end)
+{
+	char *raw_fmts =  NULL, *raw_fmts_loc = NULL;
+	uint32 read_size = READ_NUM_BYTES;
+	int error = 0;
+	char * cptr = NULL;
+	char c;
+	uint8 count = 0;
+
+	*ramstart = 0;
+	*rodata_start = 0;
+	*rodata_end = 0;
+
+	/* Allocate 1 byte more than read_size to terminate it with NULL */
+	raw_fmts = MALLOCZ(osh, read_size + 1);
+	if (raw_fmts == NULL) {
+		DHD_ERROR(("%s: Failed to allocate raw_fmts memory \n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* read ram start, rodata_start and rodata_end values from map  file */
+	while (count != ALL_MAP_VAL)
+	{
+		error = dhd_os_read_file(file, raw_fmts, read_size);
+		if (error < 0) {
+			DHD_ERROR(("%s: map file read failed err:%d \n", __FUNCTION__,
+					error));
+			goto fail;
+		}
+
+		/* End raw_fmts with NULL as strstr expects NULL terminated strings */
+		raw_fmts[read_size] = '\0';
+		/* Get ramstart address */
+		raw_fmts_loc = raw_fmts;
+		if (!(count & RAMSTART_BIT) &&
+			(cptr = bcmstrnstr(raw_fmts_loc, read_size, ramstart_str,
+			strlen(ramstart_str)))) {
+			cptr = cptr - BYTES_AHEAD_NUM;
+			sscanf(cptr, "%x %c text_start", ramstart, &c);
+			count |= RAMSTART_BIT;
+		}
+
+		/* Get ram rodata start address */
+		raw_fmts_loc = raw_fmts;
+		if (!(count & RDSTART_BIT) &&
+			(cptr = bcmstrnstr(raw_fmts_loc, read_size, rodata_start_str,
+			strlen(rodata_start_str)))) {
+			cptr = cptr - BYTES_AHEAD_NUM;
+			sscanf(cptr, "%x %c rodata_start", rodata_start, &c);
+			count |= RDSTART_BIT;
+		}
+
+		/* Get ram rodata end address */
+		raw_fmts_loc = raw_fmts;
+		if (!(count & RDEND_BIT) &&
+			(cptr = bcmstrnstr(raw_fmts_loc, read_size, rodata_end_str,
+			strlen(rodata_end_str)))) {
+			cptr = cptr - BYTES_AHEAD_NUM;
+			sscanf(cptr, "%x %c rodata_end", rodata_end, &c);
+			count |= RDEND_BIT;
+		}
+
+		if (error < (int)read_size) {
+			/*
+			* since we reset file pos back to earlier pos by
+			* GO_BACK_FILE_POS_NUM_BYTES bytes we won't reach EOF.
+			* The reason for this is if string is spreaded across
+			* bytes, the read function should not miss it.
+			* So if ret value is less than read_size, reached EOF don't read further
+			*/
+			break;
+		}
+		memset(raw_fmts, 0, read_size);
+		/*
+		* go back to predefined NUM of bytes so that we won't miss
+		* the string and  addr even if it comes as splited in next read.
+		*/
+		error = dhd_os_seek_file(file, -GO_BACK_FILE_POS_NUM_BYTES);
+		if (error < 0) {
+			DHD_ERROR(("%s: map file seek failed err:%d \n", __FUNCTION__,
+					error));
+			goto fail;
+		}
+	}
+
+fail:
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, read_size + 1);
+		raw_fmts = NULL;
+	}
+	if (count == ALL_MAP_VAL) {
+		return BCME_OK;
+	}
+	else {
+		DHD_ERROR(("%s: readmap error 0X%x \n", __FUNCTION__,
+				count));
+		return BCME_ERROR;
+	}
+
+} /* dhd_parse_map_file */
+#endif /* DHD_LINUX_STD_FW_API */
+
+#ifdef PCIE_FULL_DONGLE
+int
+dhd_event_logtrace_infobuf_pkt_process(dhd_pub_t *dhdp, void *pktbuf,
+		dhd_event_log_t *event_data)
+{
+	uint32 infobuf_version;
+	info_buf_payload_hdr_t *payload_hdr_ptr;
+	uint16 payload_hdr_type;
+	uint16 payload_hdr_length;
+
+	DHD_TRACE(("%s:Enter\n", __FUNCTION__));
+
+	if (PKTLEN(dhdp->osh, pktbuf) < sizeof(uint32)) {
+		DHD_ERROR(("%s: infobuf too small for version field\n",
+			__FUNCTION__));
+		goto exit;
+	}
+	infobuf_version = *((uint32 *)PKTDATA(dhdp->osh, pktbuf));
+	PKTPULL(dhdp->osh, pktbuf, sizeof(uint32));
+	if (infobuf_version != PCIE_INFOBUF_V1) {
+		DHD_ERROR(("%s: infobuf version %d is not PCIE_INFOBUF_V1\n",
+			__FUNCTION__, infobuf_version));
+		goto exit;
+	}
+
+	/* Version 1 infobuf has a single type/length (and then value) field */
+	if (PKTLEN(dhdp->osh, pktbuf) < sizeof(info_buf_payload_hdr_t)) {
+		DHD_ERROR(("%s: infobuf too small for v1 type/length  fields\n",
+			__FUNCTION__));
+		goto exit;
+	}
+	/* Process/parse the common info payload header (type/length) */
+	payload_hdr_ptr = (info_buf_payload_hdr_t *)PKTDATA(dhdp->osh, pktbuf);
+	payload_hdr_type = ltoh16(payload_hdr_ptr->type);
+	payload_hdr_length = ltoh16(payload_hdr_ptr->length);
+	if (payload_hdr_type != PCIE_INFOBUF_V1_TYPE_LOGTRACE) {
+		DHD_ERROR(("%s: payload_hdr_type %d is not V1_TYPE_LOGTRACE\n",
+			__FUNCTION__, payload_hdr_type));
+		goto exit;
+	}
+	PKTPULL(dhdp->osh, pktbuf, sizeof(info_buf_payload_hdr_t));
+
+	/* Validate that the specified length isn't bigger than the
+	 * provided data.
+	 */
+	if (payload_hdr_length > PKTLEN(dhdp->osh, pktbuf)) {
+		DHD_ERROR(("%s: infobuf logtrace length is bigger"
+			" than actual buffer data\n", __FUNCTION__));
+		goto exit;
+	}
+	dhd_dbg_trace_evnt_handler(dhdp, PKTDATA(dhdp->osh, pktbuf),
+		event_data, payload_hdr_length);
+
+	return BCME_OK;
+
+exit:
+	return BCME_ERROR;
+} /* dhd_event_logtrace_infobuf_pkt_process */
+#endif /* PCIE_FULL_DONGLE */
+#endif /* SHOW_LOGTRACE */
+
+#if defined(WLTDLS) && defined(PCIE_FULL_DONGLE)
+
+/* To handle the TDLS event in the dhd_common.c
+ */
+int dhd_tdls_event_handler(dhd_pub_t *dhd_pub, wl_event_msg_t *event)
+{
+	int ret = BCME_OK;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST()
+	ret = dhd_tdls_update_peer_info(dhd_pub, event);
+	GCC_DIAGNOSTIC_POP()
+
+	return ret;
+}
+
+int dhd_free_tdls_peer_list(dhd_pub_t *dhd_pub)
+{
+	tdls_peer_node_t *cur = NULL, *prev = NULL;
+	if (!dhd_pub)
+		return BCME_ERROR;
+	cur = dhd_pub->peer_tbl.node;
+
+	if ((dhd_pub->peer_tbl.node == NULL) && !dhd_pub->peer_tbl.tdls_peer_count)
+		return BCME_ERROR;
+
+	while (cur != NULL) {
+		prev = cur;
+		cur = cur->next;
+		MFREE(dhd_pub->osh, prev, sizeof(tdls_peer_node_t));
+	}
+	dhd_pub->peer_tbl.tdls_peer_count = 0;
+	dhd_pub->peer_tbl.node = NULL;
+	return BCME_OK;
+}
+#endif	/* #if defined(WLTDLS) && defined(PCIE_FULL_DONGLE) */
+
+/* pretty hex print a contiguous buffer
+* based on the debug level specified
+*/
+void
+dhd_prhex(const char *msg, volatile uchar *buf, uint nbytes, uint32 dbg_level)
+{
+	char line[128], *p;
+	int len = sizeof(line);
+	int nchar;
+	uint i;
+
+	if (msg && (msg[0] != '\0')) {
+		if (dbg_level == DHD_ERROR_VAL) {
+			DHD_ERROR(("%s:\n", msg));
+		} else if (dbg_level == DHD_INFO_VAL) {
+			DHD_INFO(("%s:\n", msg));
+		} else if (dbg_level == DHD_TRACE_VAL) {
+			DHD_TRACE(("%s:\n", msg));
+		} else if (dbg_level == DHD_RPM_VAL) {
+			DHD_RPM(("%s:\n", msg));
+		}
+	}
+
+	p = line;
+	for (i = 0; i < nbytes; i++) {
+		if (i % 16 == 0) {
+			nchar = snprintf(p, len, "  %04x: ", i);	/* line prefix */
+			p += nchar;
+			len -= nchar;
+		}
+		if (len > 0) {
+			nchar = snprintf(p, len, "%02x ", buf[i]);
+			p += nchar;
+			len -= nchar;
+		}
+
+		if (i % 16 == 15) {
+			/* flush line */
+			if (dbg_level == DHD_ERROR_VAL) {
+				DHD_ERROR(("%s:\n", line));
+			} else if (dbg_level == DHD_INFO_VAL) {
+				DHD_INFO(("%s:\n", line));
+			} else if (dbg_level == DHD_TRACE_VAL) {
+				DHD_TRACE(("%s:\n", line));
+			} else if (dbg_level == DHD_RPM_VAL) {
+				DHD_RPM(("%s:\n", line));
+			}
+
+			p = line;
+			len = sizeof(line);
+		}
+	}
+
+	/* flush last partial line */
+	if (p != line) {
+		if (dbg_level == DHD_ERROR_VAL) {
+			DHD_ERROR(("%s:\n", line));
+		} else if (dbg_level == DHD_INFO_VAL) {
+			DHD_INFO(("%s:\n", line));
+		} else if (dbg_level == DHD_TRACE_VAL) {
+			DHD_TRACE(("%s:\n", line));
+		} else if (dbg_level == DHD_RPM_VAL) {
+			DHD_RPM(("%s:\n", line));
+		}
+	}
+}
+
+int
+dhd_tput_test(dhd_pub_t *dhd, tput_test_t *tput_data)
+{
+	struct ether_header ether_hdr;
+	tput_pkt_t tput_pkt;
+	void *pkt = NULL;
+	uint8 *pktdata = NULL;
+	uint32 pktsize = 0;
+	uint64 total_size = 0;
+	uint32 *crc = 0;
+	uint32 pktid = 0;
+	uint32 total_num_tx_pkts = 0;
+	int err = 0, err_exit = 0;
+	uint32 i = 0;
+	uint64 time_taken = 0;
+	int max_txbufs = 0;
+	uint32 n_batches = 0;
+	uint32 n_remain = 0;
+	uint8 tput_pkt_hdr_size = 0;
+	bool batch_cnt = FALSE;
+	bool tx_stop_pkt = FALSE;
+
+	if (tput_data->version != TPUT_TEST_T_VER ||
+		tput_data->length != TPUT_TEST_T_LEN) {
+		DHD_ERROR(("%s: wrong structure ver/len! \n", __FUNCTION__));
+		err_exit = BCME_BADARG;
+		goto exit_error;
+	}
+
+	if (dhd->tput_data.tput_test_running) {
+		DHD_ERROR(("%s: tput test already running ! \n", __FUNCTION__));
+		err_exit = BCME_BUSY;
+		goto exit_error;
+	}
+#ifdef PCIE_FULL_DONGLE
+	/*
+	 * 100 bytes to accommodate ether header and tput header. As of today
+	 * both occupy 30 bytes. Rest is reserved.
+	 */
+	if ((tput_data->payload_size > TPUT_TEST_MAX_PAYLOAD) ||
+		(tput_data->payload_size > (DHD_FLOWRING_RX_BUFPOST_PKTSZ - 100))) {
+		DHD_ERROR(("%s: payload size is too large! max_payload=%u rx_bufpost_size=%u\n",
+			__FUNCTION__, TPUT_TEST_MAX_PAYLOAD,
+			(DHD_FLOWRING_RX_BUFPOST_PKTSZ - 100)));
+		err_exit = BCME_BUFTOOLONG;
+		goto exit_error;
+	}
+#endif
+	max_txbufs = dhd_get_max_txbufs(dhd);
+	max_txbufs = MIN(max_txbufs, DHD_TPUT_MAX_TX_PKTS_BATCH);
+
+	if (!(tput_data->num_pkts > 0)) {
+		DHD_ERROR(("%s: invalid num_pkts: %d to tx\n",
+			__FUNCTION__, tput_data->num_pkts));
+		err_exit = BCME_ERROR;
+		goto exit_error;
+	}
+
+	memset(&dhd->tput_data, 0, sizeof(dhd->tput_data));
+	memcpy(&dhd->tput_data, tput_data, sizeof(*tput_data));
+	dhd->tput_data.pkts_bad = dhd->tput_data.pkts_good = 0;
+	dhd->tput_data.pkts_cmpl = 0;
+	dhd->tput_start_ts = dhd->tput_stop_ts = 0;
+
+	if (tput_data->flags & TPUT_TEST_USE_ETHERNET_HDR) {
+		pktsize = sizeof(ether_hdr) + sizeof(tput_pkt_t) +
+				(tput_data->payload_size - 12);
+	} else {
+		pktsize = sizeof(tput_pkt_t) +
+				(tput_data->payload_size - 12);
+	}
+
+	tput_pkt_hdr_size = (uint8)((uint8 *)&tput_pkt.crc32 -
+			(uint8 *)&tput_pkt.mac_sta);
+
+	/* mark the tput test as started */
+	dhd->tput_data.tput_test_running = TRUE;
+
+	if (tput_data->direction == TPUT_DIR_TX) {
+		/* for ethernet header */
+		memcpy(ether_hdr.ether_shost, tput_data->mac_sta, ETHER_ADDR_LEN);
+		memcpy(ether_hdr.ether_dhost, tput_data->mac_ap, ETHER_ADDR_LEN);
+		ether_hdr.ether_type = hton16(ETHER_TYPE_IP);
+
+		/* fill in the tput pkt */
+		memset(&tput_pkt, 0, sizeof(tput_pkt));
+		memcpy(tput_pkt.mac_ap, tput_data->mac_ap, ETHER_ADDR_LEN);
+		memcpy(tput_pkt.mac_sta, tput_data->mac_sta, ETHER_ADDR_LEN);
+		tput_pkt.pkt_type = hton16(TPUT_PKT_TYPE_NORMAL);
+		tput_pkt.num_pkts = hton32(tput_data->num_pkts);
+
+		if (tput_data->num_pkts > (uint32)max_txbufs) {
+			n_batches = tput_data->num_pkts / max_txbufs;
+			n_remain = tput_data->num_pkts % max_txbufs;
+		} else {
+			n_batches = 0;
+			n_remain = tput_data->num_pkts;
+		}
+		DHD_ERROR(("%s: num_pkts: %u n_batches: %u n_remain: %u\n",
+			__FUNCTION__, tput_data->num_pkts, n_batches, n_remain));
+
+		do {
+			/* reset before every batch */
+			dhd->batch_tx_pkts_cmpl = 0;
+			if (n_batches) {
+				dhd->batch_tx_num_pkts = max_txbufs;
+				--n_batches;
+			} else if (n_remain) {
+				dhd->batch_tx_num_pkts = n_remain;
+				n_remain = 0;
+			} else {
+				DHD_ERROR(("Invalid. This should not hit\n"));
+			}
+
+			dhd->tput_start_ts = OSL_SYSUPTIME_US();
+			for (i = 0; (i < dhd->batch_tx_num_pkts) || (tx_stop_pkt); ++i) {
+				pkt = PKTGET(dhd->osh, pktsize, TRUE);
+				if (!pkt) {
+					dhd->tput_data.tput_test_running = FALSE;
+					DHD_ERROR(("%s: PKTGET fails ! Not enough Tx buffers\n",
+						__FUNCTION__));
+					DHD_ERROR(("%s: pkts_good:%u; pkts_bad:%u; pkts_cmpl:%u\n",
+						__FUNCTION__, dhd->tput_data.pkts_good,
+						dhd->tput_data.pkts_bad, dhd->tput_data.pkts_cmpl));
+					err_exit = BCME_NOMEM;
+					goto exit_error;
+				}
+				pktdata = PKTDATA(dhd->osh, pkt);
+				PKTSETLEN(dhd->osh, pkt, pktsize);
+				memset(pktdata, 0, pktsize);
+				if (tput_data->flags & TPUT_TEST_USE_ETHERNET_HDR) {
+					memcpy(pktdata, &ether_hdr, sizeof(ether_hdr));
+					pktdata += sizeof(ether_hdr);
+				}
+				/* send stop pkt as last pkt */
+				if (tx_stop_pkt) {
+					tput_pkt.pkt_type = hton16(TPUT_PKT_TYPE_STOP);
+					tx_stop_pkt = FALSE;
+				} else
+					tput_pkt.pkt_type = hton16(TPUT_PKT_TYPE_NORMAL);
+				tput_pkt.pkt_id = hton32(pktid++);
+				tput_pkt.crc32 = 0;
+				memcpy(pktdata, &tput_pkt, sizeof(tput_pkt));
+				/* compute crc32 over the pkt-id, num-pkts and data fields */
+				crc = (uint32 *)(pktdata + tput_pkt_hdr_size);
+				*crc = hton32(hndcrc32(pktdata + tput_pkt_hdr_size + 4,
+						8 + (tput_data->payload_size - 12),
+						CRC32_INIT_VALUE));
+
+				err = dhd_sendpkt(dhd, 0, pkt);
+				if (err != BCME_OK) {
+					DHD_INFO(("%s: send pkt (id = %u) fails (err = %d) ! \n",
+						__FUNCTION__, pktid, err));
+					dhd->tput_data.pkts_bad++;
+				}
+				total_num_tx_pkts++;
+				if ((total_num_tx_pkts == tput_data->num_pkts) && (!tx_stop_pkt)) {
+					tx_stop_pkt = TRUE;
+				}
+			}
+			DHD_INFO(("%s: TX done, wait for completion...\n", __FUNCTION__));
+			if (!dhd_os_tput_test_wait(dhd, NULL,
+					TPUT_TEST_WAIT_TIMEOUT_DEFAULT)) {
+				dhd->tput_stop_ts = OSL_SYSUPTIME_US();
+				dhd->tput_data.tput_test_running = FALSE;
+				DHD_ERROR(("%s: TX completion timeout !"
+					" Total Tx pkts (including STOP) = %u; pkts cmpl = %u; \n",
+					__FUNCTION__, total_num_tx_pkts, dhd->batch_tx_pkts_cmpl));
+				err_exit = BCME_ERROR;
+				goto exit_error;
+			}
+			if ((dhd->tput_start_ts && dhd->tput_stop_ts &&
+				(dhd->tput_stop_ts > dhd->tput_start_ts)) || (time_taken)) {
+				if (!time_taken) {
+					time_taken = dhd->tput_stop_ts - dhd->tput_start_ts;
+				}
+			} else {
+				dhd->tput_data.tput_test_running = FALSE;
+				DHD_ERROR(("%s: bad timestamp while cal tx batch time\n",
+					__FUNCTION__));
+				err_exit = BCME_ERROR;
+				goto exit_error;
+			}
+			if (n_batches || n_remain) {
+				batch_cnt = TRUE;
+			} else {
+				batch_cnt = FALSE;
+			}
+		} while (batch_cnt);
+	} else {
+		/* TPUT_DIR_RX */
+		DHD_INFO(("%s: waiting for RX completion... \n", __FUNCTION__));
+		if (!dhd_os_tput_test_wait(dhd, NULL, tput_data->timeout_ms)) {
+			DHD_ERROR(("%s: RX completion timeout ! \n", __FUNCTION__));
+			dhd->tput_stop_ts = OSL_SYSUPTIME_US();
+		}
+	}
+
+	/* calculate the throughput in bits per sec */
+	if (dhd->tput_start_ts && dhd->tput_stop_ts &&
+		(dhd->tput_stop_ts > dhd->tput_start_ts)) {
+		time_taken = dhd->tput_stop_ts - dhd->tput_start_ts;
+		time_taken = DIV_U64_BY_U32(time_taken, MSEC_PER_SEC); /* convert to ms */
+		dhd->tput_data.time_ms = time_taken;
+		if (time_taken) {
+			total_size = pktsize * dhd->tput_data.pkts_cmpl * 8;
+			dhd->tput_data.tput_bps = DIV_U64_BY_U64(total_size, time_taken);
+			/* convert from ms to seconds */
+			dhd->tput_data.tput_bps = dhd->tput_data.tput_bps * 1000;
+		}
+	} else {
+		DHD_ERROR(("%s: bad timestamp !\n", __FUNCTION__));
+	}
+	DHD_INFO(("%s: DONE. tput = %llu bps, time = %llu ms\n", __FUNCTION__,
+		dhd->tput_data.tput_bps, dhd->tput_data.time_ms));
+
+	memcpy(tput_data, &dhd->tput_data, sizeof(dhd->tput_data));
+
+	dhd->tput_data.tput_test_running = FALSE;
+
+	err_exit = BCME_OK;
+
+exit_error:
+	DHD_ERROR(("%s: pkts_good = %u; pkts_bad = %u; pkts_cmpl = %u\n",
+		__FUNCTION__, dhd->tput_data.pkts_good,
+		dhd->tput_data.pkts_bad, dhd->tput_data.pkts_cmpl));
+
+	return err_exit;
+}
+
+void
+dhd_tput_test_rx(dhd_pub_t *dhd, void *pkt)
+{
+	uint8 *pktdata = NULL;
+	tput_pkt_t *tput_pkt = NULL;
+	uint32 crc = 0;
+	uint8 tput_pkt_hdr_size = 0;
+
+	pktdata = PKTDATA(dhd->osh, pkt);
+	if (dhd->tput_data.flags & TPUT_TEST_USE_ETHERNET_HDR)
+		pktdata += sizeof(struct ether_header);
+	tput_pkt = (tput_pkt_t *)pktdata;
+
+	/* record the timestamp of the first packet received */
+	if (dhd->tput_data.pkts_cmpl == 0) {
+		dhd->tput_start_ts = OSL_SYSUPTIME_US();
+	}
+
+	if (ntoh16(tput_pkt->pkt_type) != TPUT_PKT_TYPE_STOP &&
+			dhd->tput_data.pkts_cmpl <= dhd->tput_data.num_pkts) {
+		dhd->tput_data.pkts_cmpl++;
+	}
+	/* drop rx packets received beyond the specified # */
+	if (dhd->tput_data.pkts_cmpl > dhd->tput_data.num_pkts)
+		return;
+
+	DHD_TRACE(("%s: Rx tput test pkt, id = %u ; type = %u\n", __FUNCTION__,
+		ntoh32(tput_pkt->pkt_id), ntoh16(tput_pkt->pkt_type)));
+
+	/* discard if mac addr of AP/STA does not match the specified ones */
+	if ((memcmp(tput_pkt->mac_ap, dhd->tput_data.mac_ap,
+			ETHER_ADDR_LEN) != 0) ||
+		(memcmp(tput_pkt->mac_sta, dhd->tput_data.mac_sta,
+			ETHER_ADDR_LEN) != 0)) {
+		dhd->tput_data.pkts_bad++;
+		DHD_INFO(("%s: dropping tput pkt with id %u due to bad AP/STA mac !\n",
+			__FUNCTION__, ntoh32(tput_pkt->pkt_id)));
+		return;
+	}
+
+	tput_pkt_hdr_size = (uint8)((uint8 *)&tput_pkt->crc32 -
+			(uint8 *)&tput_pkt->mac_sta);
+	pktdata += tput_pkt_hdr_size + 4;
+	crc = hndcrc32(pktdata, 8 + (dhd->tput_data.payload_size - 12),
+			CRC32_INIT_VALUE);
+	if (crc != ntoh32(tput_pkt->crc32)) {
+		DHD_INFO(("%s: dropping tput pkt with id %u due to bad CRC !\n",
+			__FUNCTION__, ntoh32(tput_pkt->pkt_id)));
+		dhd->tput_data.pkts_bad++;
+		return;
+	}
+
+	if (ntoh16(tput_pkt->pkt_type) != TPUT_PKT_TYPE_STOP)
+		dhd->tput_data.pkts_good++;
+
+	/* if we have received the stop packet or all the # of pkts, we're done */
+	if (ntoh16(tput_pkt->pkt_type) == TPUT_PKT_TYPE_STOP ||
+			dhd->tput_data.pkts_cmpl == dhd->tput_data.num_pkts) {
+		dhd->tput_stop_ts = OSL_SYSUPTIME_US();
+		dhd_os_tput_test_wake(dhd);
+	}
+}
+
+#ifdef DUMP_IOCTL_IOV_LIST
+void
+dhd_iov_li_append(dhd_pub_t *dhd, dll_t *list_head, dll_t *node)
+{
+	dll_t *item;
+	dhd_iov_li_t *iov_li;
+	dhd->dump_iovlist_len++;
+
+	if (dhd->dump_iovlist_len == IOV_LIST_MAX_LEN+1) {
+		item = dll_head_p(list_head);
+		iov_li = (dhd_iov_li_t *)CONTAINEROF(item, dhd_iov_li_t, list);
+		dll_delete(item);
+		MFREE(dhd->osh, iov_li, sizeof(*iov_li));
+		dhd->dump_iovlist_len--;
+	}
+	dll_append(list_head, node);
+}
+
+void
+dhd_iov_li_print(dll_t *list_head)
+{
+	dhd_iov_li_t *iov_li;
+	dll_t *item, *next;
+	uint8 index = 0;
+	for (item = dll_head_p(list_head); !dll_end(list_head, item); item = next) {
+		next = dll_next_p(item);
+		iov_li = (dhd_iov_li_t *)CONTAINEROF(item, dhd_iov_li_t, list);
+		DHD_ERROR(("%d:cmd_name = %s, cmd = %d.\n", ++index, iov_li->buff, iov_li->cmd));
+	}
+}
+
+void
+dhd_iov_li_delete(dhd_pub_t *dhd, dll_t *list_head)
+{
+	dll_t *item;
+	dhd_iov_li_t *iov_li;
+	while (!(dll_empty(list_head))) {
+		item = dll_head_p(list_head);
+		iov_li = (dhd_iov_li_t *)CONTAINEROF(item, dhd_iov_li_t, list);
+		dll_delete(item);
+		MFREE(dhd->osh, iov_li, sizeof(*iov_li));
+	}
+}
+#endif /* DUMP_IOCTL_IOV_LIST */
+
+#ifdef EWP_EDL
+/* For now we are allocating memory for EDL ring using DMA_ALLOC_CONSISTENT
+* The reason being that, in hikey, if we try to DMA_MAP prealloced memory
+* it is failing with an 'out of space in SWIOTLB' error
+*/
+int
+dhd_edl_mem_init(dhd_pub_t *dhd)
+{
+	int ret = 0;
+
+	memset(&dhd->edl_ring_mem, 0, sizeof(dhd->edl_ring_mem));
+	ret = dhd_dma_buf_alloc(dhd, &dhd->edl_ring_mem, DHD_EDL_RING_SIZE);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: alloc of edl_ring_mem failed\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+
+/*
+ * NOTE:- that dhd_edl_mem_deinit need NOT be called explicitly, because the dma_buf
+ * for EDL is freed during 'dhd_prot_detach_edl_rings' which is called during de-init.
+ */
+void
+dhd_edl_mem_deinit(dhd_pub_t *dhd)
+{
+	if (dhd->edl_ring_mem.va != NULL)
+		dhd_dma_buf_free(dhd, &dhd->edl_ring_mem);
+}
+
+int
+dhd_event_logtrace_process_edl(dhd_pub_t *dhdp, uint8 *data,
+		void *evt_decode_data)
+{
+	msg_hdr_edl_t *msg = NULL;
+	cmn_msg_hdr_t *cmn_msg_hdr = NULL;
+	uint8 *buf = NULL;
+
+	if (!data || !dhdp || !evt_decode_data) {
+		DHD_ERROR(("%s: invalid args ! \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* format of data in each work item in the EDL ring:
+	* |cmn_msg_hdr_t |payload (var len)|cmn_msg_hdr_t|
+	* payload = |infobuf_ver(u32)|info_buf_payload_hdr_t|msgtrace_hdr_t|<var len data>|
+	*/
+	cmn_msg_hdr = (cmn_msg_hdr_t *)data;
+	msg = (msg_hdr_edl_t *)(data + sizeof(cmn_msg_hdr_t));
+	buf = (uint8 *)msg;
+	/* validate the fields */
+	if (ltoh32(msg->infobuf_ver) != PCIE_INFOBUF_V1) {
+		DHD_ERROR(("%s: Skipping msg with invalid infobuf ver (0x%x)"
+			" expected (0x%x)\n", __FUNCTION__,
+			msg->infobuf_ver, PCIE_INFOBUF_V1));
+		return BCME_VERSION;
+	}
+
+	/* in EDL, the request_id field of cmn_msg_hdr is overloaded to carry payload length */
+	if (sizeof(info_buf_payload_hdr_t) > cmn_msg_hdr->request_id) {
+		DHD_ERROR(("%s: infobuf too small for v1 type/length fields\n",
+			__FUNCTION__));
+		return BCME_BUFTOOLONG;
+	}
+
+	if (ltoh16(msg->pyld_hdr.type) != PCIE_INFOBUF_V1_TYPE_LOGTRACE) {
+		DHD_ERROR(("%s: payload_hdr_type %d is not V1_TYPE_LOGTRACE\n",
+			__FUNCTION__, ltoh16(msg->pyld_hdr.type)));
+		return BCME_BADOPTION;
+	}
+
+	if (ltoh16(msg->pyld_hdr.length) > cmn_msg_hdr->request_id) {
+		DHD_ERROR(("%s: infobuf logtrace length %u is bigger"
+			" than available buffer size %u\n", __FUNCTION__,
+			ltoh16(msg->pyld_hdr.length), cmn_msg_hdr->request_id));
+		return BCME_BADLEN;
+	}
+
+	/* dhd_dbg_trace_evnt_handler expects the data to start from msgtrace_hdr_t */
+	buf += sizeof(msg->infobuf_ver) + sizeof(msg->pyld_hdr);
+	dhd_dbg_trace_evnt_handler(dhdp, buf, evt_decode_data,
+		ltoh16(msg->pyld_hdr.length));
+
+	/*
+	 * check 'dhdp->logtrace_pkt_sendup' and if true alloc an skb
+	 * copy the event data to the skb and send it up the stack
+	 */
+	if (dhdp->logtrace_pkt_sendup) {
+		DHD_INFO(("%s: send up event log, len %u bytes\n", __FUNCTION__,
+				(uint32)(ltoh16(msg->pyld_hdr.length) +
+				sizeof(info_buf_payload_hdr_t) + 4)));
+		dhd_sendup_info_buf(dhdp, (uint8 *)msg);
+	}
+
+	return BCME_OK;
+}
+#endif /* EWP_EDL */
+
+#if defined(LINUX) && defined(SHOW_LOGTRACE)
+int
+dhd_print_fw_ver_from_file(dhd_pub_t *dhdp, char *fwpath)
+{
+	void *file = NULL;
+	int size = 0;
+	char buf[FW_VER_STR_LEN];
+	char *str = NULL;
+	int ret = BCME_OK;
+
+	if (!fwpath)
+		return BCME_BADARG;
+	file = dhd_os_open_image1(dhdp, fwpath);
+	if (!file) {
+		ret = BCME_ERROR;
+		goto exit;
+	}
+	size = dhd_os_get_image_size(file);
+	if (!size) {
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	/* seek to the last 'X' bytes in the file */
+	if (dhd_os_seek_file(file, size - FW_VER_STR_LEN) != BCME_OK) {
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	/* read the last 'X' bytes of the file to a buffer */
+	memset(buf, 0, FW_VER_STR_LEN);
+	if (dhd_os_get_image_block(buf, FW_VER_STR_LEN - 1, file) < 0) {
+		ret = BCME_ERROR;
+		goto exit;
+	}
+	/* search for 'Version' in the buffer */
+	str = bcmstrnstr(buf, FW_VER_STR_LEN, FW_VER_STR, strlen(FW_VER_STR));
+	if (!str) {
+		ret = BCME_ERROR;
+		goto exit;
+	}
+	/* go back in the buffer to the last ascii character */
+	while (str != buf &&
+		(*str >= ' ' && *str <= '~')) {
+		--str;
+	}
+	/* reverse the final decrement, so that str is pointing
+	* to the first ascii character in the buffer
+	*/
+	++str;
+
+	if (strlen(str) > (FW_VER_STR_LEN - 1)) {
+		ret = BCME_BADLEN;
+		goto exit;
+	}
+
+	DHD_INFO(("FW version in file '%s': %s\n", fwpath, str));
+	/* copy to global variable, so that in case FW load fails, the
+	* core capture logs will contain FW version read from the file
+	*/
+	memset(fw_version, 0, FW_VER_STR_LEN);
+	strlcpy(fw_version, str, FW_VER_STR_LEN);
+
+exit:
+	if (file)
+		dhd_os_close_image1(dhdp, file);
+
+	return ret;
+}
+#endif /* LINUX && SHOW_LOGTRACE */
+
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+static void
+copy_hang_info_ioctl_timeout(dhd_pub_t *dhd, int ifidx, wl_ioctl_t *ioc)
+{
+	int remain_len;
+	int i;
+	int *cnt;
+	char *dest;
+	int bytes_written;
+	uint32 ioc_dwlen = 0;
+
+	if (!dhd || !dhd->hang_info) {
+		DHD_ERROR(("%s dhd=%p hang_info=%p\n",
+			__FUNCTION__, dhd, (dhd ? dhd->hang_info : NULL)));
+		return;
+	}
+
+	cnt = &dhd->hang_info_cnt;
+	dest = dhd->hang_info;
+
+	memset(dest, 0, VENDOR_SEND_HANG_EXT_INFO_LEN);
+	(*cnt) = 0;
+
+	bytes_written = 0;
+	remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - bytes_written;
+
+	get_debug_dump_time(dhd->debug_dump_time_hang_str);
+	copy_debug_dump_time(dhd->debug_dump_time_str, dhd->debug_dump_time_hang_str);
+
+	bytes_written += scnprintf(&dest[bytes_written], remain_len, "%d %d %s %d %d %d %d %d %d ",
+			HANG_REASON_IOCTL_RESP_TIMEOUT, VENDOR_SEND_HANG_EXT_INFO_VER,
+			dhd->debug_dump_time_hang_str,
+			ifidx, ioc->cmd, ioc->len, ioc->set, ioc->used, ioc->needed);
+	(*cnt) = HANG_FIELD_IOCTL_RESP_TIMEOUT_CNT;
+
+	clear_debug_dump_time(dhd->debug_dump_time_hang_str);
+
+	/* Access ioc->buf only if the ioc->len is more than 4 bytes */
+	ioc_dwlen = (uint32)(ioc->len / sizeof(uint32));
+	if (ioc_dwlen > 0) {
+		const uint32 *ioc_buf = (const uint32 *)ioc->buf;
+
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - bytes_written;
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		bytes_written += scnprintf(&dest[bytes_written], remain_len,
+			"%08x", *(uint32 *)(ioc_buf++));
+		GCC_DIAGNOSTIC_POP();
+		(*cnt)++;
+		if ((*cnt) >= HANG_FIELD_CNT_MAX) {
+			return;
+		}
+
+		for (i = 1; i < ioc_dwlen && *cnt <= HANG_FIELD_CNT_MAX;
+			i++, (*cnt)++) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - bytes_written;
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			bytes_written += scnprintf(&dest[bytes_written], remain_len, "%c%08x",
+				HANG_RAW_DEL, *(uint32 *)(ioc_buf++));
+			GCC_DIAGNOSTIC_POP();
+		}
+	}
+
+	DHD_INFO(("%s hang info len: %d data: %s\n",
+		__FUNCTION__, (int)strlen(dhd->hang_info), dhd->hang_info));
+}
+
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+/*
+ * Helper function:
+ * Used for Dongle console message time syncing with Host printk
+ */
+void dhd_h2d_log_time_sync(dhd_pub_t *dhd)
+{
+	uint64 ts;
+
+	/*
+	 * local_clock() returns time in nano seconds.
+	 * Dongle understand only milli seconds time.
+	 */
+	ts = local_clock();
+	/* Nano seconds to milli seconds */
+	do_div(ts, 1000000);
+	if (dhd_wl_ioctl_set_intiovar(dhd,  "rte_timesync", ts, WLC_SET_VAR, TRUE, 0)) {
+		DHD_ERROR(("%s rte_timesync **** FAILED ****\n", __FUNCTION__));
+		/* Stopping HOST Dongle console time syncing */
+		dhd->dhd_rte_time_sync_ms = 0;
+	}
+}
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+#if defined(LINUX) || defined(linux)
+/* configuations of ecounters to be enabled by default in FW */
+static ecounters_cfg_t ecounters_cfg_tbl[] = {
+	/* Global ecounters */
+	{ECOUNTERS_STATS_TYPES_FLAG_GLOBAL, 0x0, WL_IFSTATS_XTLV_BUS_PCIE},
+	// {ECOUNTERS_STATS_TYPES_FLAG_GLOBAL, 0x0, WL_IFSTATS_XTLV_TX_AMPDU_STATS},
+	// {ECOUNTERS_STATS_TYPES_FLAG_GLOBAL, 0x0, WL_IFSTATS_XTLV_RX_AMPDU_STATS},
+
+	/* Slice specific ecounters */
+	{ECOUNTERS_STATS_TYPES_FLAG_SLICE, 0x0, WL_SLICESTATS_XTLV_PERIODIC_STATE},
+	{ECOUNTERS_STATS_TYPES_FLAG_SLICE, 0x1, WL_SLICESTATS_XTLV_PERIODIC_STATE},
+	{ECOUNTERS_STATS_TYPES_FLAG_SLICE, 0x1, WL_IFSTATS_XTLV_WL_SLICE_BTCOEX},
+
+	/* Interface specific ecounters */
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x0, WL_IFSTATS_XTLV_IF_PERIODIC_STATE},
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x0, WL_IFSTATS_XTLV_GENERIC},
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x0, WL_IFSTATS_XTLV_INFRA_SPECIFIC},
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x0, WL_IFSTATS_XTLV_MGT_CNT},
+
+	/* secondary interface */
+	/* XXX REMOVE for temporal, will be enabled after decision
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x1, WL_IFSTATS_XTLV_IF_PERIODIC_STATE},
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x1, WL_IFSTATS_XTLV_GENERIC},
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x1, WL_IFSTATS_XTLV_INFRA_SPECIFIC},
+	{ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x1, WL_IFSTATS_XTLV_MGT_CNT},
+	*/
+};
+
+/* XXX: Same event id shall be defined in consecutive order in the below table */
+static event_ecounters_cfg_t event_ecounters_cfg_tbl[] = {
+	/* Interface specific event ecounters */
+	{WLC_E_DEAUTH_IND, ECOUNTERS_STATS_TYPES_FLAG_IFACE, 0x0, WL_IFSTATS_XTLV_IF_EVENT_STATS},
+};
+
+/* Accepts an argument to -s, -g or -f and creates an XTLV */
+int
+dhd_create_ecounters_params(dhd_pub_t *dhd, uint16 type, uint16 if_slice_idx,
+	uint16 stats_rep, uint8 **xtlv)
+{
+	uint8 *req_xtlv = NULL;
+	ecounters_stats_types_report_req_t *req;
+	bcm_xtlvbuf_t xtlvbuf, container_xtlvbuf;
+	ecountersv2_xtlv_list_elt_t temp;
+	uint16 xtlv_len = 0, total_len = 0;
+	int rc = BCME_OK;
+
+	/* fill in the stat type XTLV. For now there is no explicit TLV for the stat type. */
+	temp.id = stats_rep;
+	temp.len = 0;
+
+	/* Hence len/data = 0/NULL */
+	xtlv_len += temp.len + BCM_XTLV_HDR_SIZE;
+
+	/* Total length of the container */
+	total_len = BCM_XTLV_HDR_SIZE +
+		OFFSETOF(ecounters_stats_types_report_req_t, stats_types_req) + xtlv_len;
+
+	/* Now allocate a structure for the entire request */
+	if ((req_xtlv = (uint8 *)MALLOCZ(dhd->osh, total_len)) == NULL) {
+		rc = BCME_NOMEM;
+		goto fail;
+	}
+
+	/* container XTLV context */
+	bcm_xtlv_buf_init(&container_xtlvbuf, (uint8 *)req_xtlv, total_len,
+		BCM_XTLV_OPTION_ALIGN32);
+
+	/* Fill other XTLVs in the container. Leave space for XTLV headers */
+	req = (ecounters_stats_types_report_req_t *)(req_xtlv + BCM_XTLV_HDR_SIZE);
+	req->flags = type;
+	if (type == ECOUNTERS_STATS_TYPES_FLAG_SLICE) {
+		req->slice_mask = 0x1 << if_slice_idx;
+	} else if (type == ECOUNTERS_STATS_TYPES_FLAG_IFACE) {
+		req->if_index = if_slice_idx;
+	}
+
+	/* Fill remaining XTLVs */
+	bcm_xtlv_buf_init(&xtlvbuf, (uint8*) req->stats_types_req, xtlv_len,
+		BCM_XTLV_OPTION_ALIGN32);
+	if (bcm_xtlv_put_data(&xtlvbuf, temp.id, NULL, temp.len)) {
+		DHD_ERROR(("Error creating XTLV for requested stats type = %d\n", temp.id));
+		rc = BCME_ERROR;
+		goto fail;
+	}
+
+	/* fill the top level container and get done with the XTLV container */
+	rc = bcm_xtlv_put_data(&container_xtlvbuf, WL_ECOUNTERS_XTLV_REPORT_REQ, NULL,
+		bcm_xtlv_buf_len(&xtlvbuf) + OFFSETOF(ecounters_stats_types_report_req_t,
+		stats_types_req));
+
+	if (rc) {
+		DHD_ERROR(("Error creating parent XTLV for type = %d\n", req->flags));
+		goto fail;
+	}
+
+fail:
+	if (rc && req_xtlv) {
+		MFREE(dhd->osh, req_xtlv, total_len);
+		req_xtlv = NULL;
+	}
+
+	/* update the xtlv pointer */
+	*xtlv = req_xtlv;
+	return rc;
+}
+
+static int
+dhd_ecounter_autoconfig(dhd_pub_t *dhd)
+{
+	int rc = BCME_OK;
+	uint32 buf;
+	rc = dhd_iovar(dhd, 0, "ecounters_autoconfig", NULL, 0, (char *)&buf, sizeof(buf), FALSE);
+
+	if (rc != BCME_OK) {
+
+		if (rc != BCME_UNSUPPORTED) {
+			rc = BCME_OK;
+			DHD_ERROR(("%s Ecounter autoconfig in fw failed : %d\n", __FUNCTION__, rc));
+		} else {
+			DHD_ERROR(("%s Ecounter autoconfig in FW not supported\n", __FUNCTION__));
+		}
+	}
+
+	return rc;
+}
+
+int
+dhd_ecounter_configure(dhd_pub_t *dhd, bool enable)
+{
+	int rc = BCME_OK;
+	if (enable) {
+		if (dhd_ecounter_autoconfig(dhd) != BCME_OK) {
+			if ((rc = dhd_start_ecounters(dhd)) != BCME_OK) {
+				DHD_ERROR(("%s Ecounters start failed\n", __FUNCTION__));
+			} else if ((rc = dhd_start_event_ecounters(dhd)) != BCME_OK) {
+				DHD_ERROR(("%s Event_Ecounters start failed\n", __FUNCTION__));
+			}
+		}
+	} else {
+		if ((rc = dhd_stop_ecounters(dhd)) != BCME_OK) {
+			DHD_ERROR(("%s Ecounters stop failed\n", __FUNCTION__));
+		} else if ((rc = dhd_stop_event_ecounters(dhd)) != BCME_OK) {
+			DHD_ERROR(("%s Event_Ecounters stop failed\n", __FUNCTION__));
+		}
+	}
+	return rc;
+}
+
+int
+dhd_start_ecounters(dhd_pub_t *dhd)
+{
+	uint8 i = 0;
+	uint8 *start_ptr;
+	int rc = BCME_OK;
+	bcm_xtlv_t *elt;
+	ecounters_config_request_v2_t *req = NULL;
+	ecountersv2_processed_xtlv_list_elt *list_elt, *tail = NULL;
+	ecountersv2_processed_xtlv_list_elt *processed_containers_list = NULL;
+	uint16 total_processed_containers_len = 0;
+
+	for (i = 0; i < ARRAYSIZE(ecounters_cfg_tbl); i++) {
+		ecounters_cfg_t *ecounter_stat = &ecounters_cfg_tbl[i];
+
+		if ((list_elt = (ecountersv2_processed_xtlv_list_elt *)
+			MALLOCZ(dhd->osh, sizeof(*list_elt))) == NULL) {
+			DHD_ERROR(("Ecounters v2: No memory to process\n"));
+			goto fail;
+		}
+
+		rc = dhd_create_ecounters_params(dhd, ecounter_stat->type,
+			ecounter_stat->if_slice_idx, ecounter_stat->stats_rep, &list_elt->data);
+
+		if (rc) {
+			DHD_ERROR(("Ecounters v2: Could not process: stat: %d return code: %d\n",
+				ecounter_stat->stats_rep, rc));
+
+			/* Free allocated memory and go to fail to release any memories allocated
+			 * in previous iterations. Note that list_elt->data gets populated in
+			 * dhd_create_ecounters_params() and gets freed there itself.
+			 */
+			MFREE(dhd->osh, list_elt, sizeof(*list_elt));
+			list_elt = NULL;
+			goto fail;
+		}
+		elt = (bcm_xtlv_t *) list_elt->data;
+
+		/* Put the elements in the order they are processed */
+		if (processed_containers_list == NULL) {
+			processed_containers_list = list_elt;
+		} else {
+			tail->next = list_elt;
+		}
+		tail = list_elt;
+		/* Size of the XTLV returned */
+		total_processed_containers_len += BCM_XTLV_LEN(elt) + BCM_XTLV_HDR_SIZE;
+	}
+
+	/* Now create ecounters config request with totallength */
+	req = (ecounters_config_request_v2_t *)MALLOCZ(dhd->osh, sizeof(*req) +
+		total_processed_containers_len);
+
+	if (req == NULL) {
+		rc = BCME_NOMEM;
+		goto fail;
+	}
+
+	req->version = ECOUNTERS_VERSION_2;
+	req->logset = EVENT_LOG_SET_ECOUNTERS;
+	req->reporting_period = ECOUNTERS_DEFAULT_PERIOD;
+	req->num_reports = ECOUNTERS_NUM_REPORTS;
+	req->len = total_processed_containers_len +
+		OFFSETOF(ecounters_config_request_v2_t, ecounters_xtlvs);
+
+	/* Copy config */
+	start_ptr = req->ecounters_xtlvs;
+
+	/* Now go element by element in the list */
+	while (processed_containers_list) {
+		list_elt = processed_containers_list;
+
+		elt = (bcm_xtlv_t *)list_elt->data;
+
+		memcpy(start_ptr, list_elt->data, BCM_XTLV_LEN(elt) + BCM_XTLV_HDR_SIZE);
+		start_ptr += (size_t)(BCM_XTLV_LEN(elt) + BCM_XTLV_HDR_SIZE);
+		processed_containers_list = processed_containers_list->next;
+
+		/* Free allocated memories */
+		MFREE(dhd->osh, elt, elt->len + BCM_XTLV_HDR_SIZE);
+		MFREE(dhd->osh, list_elt, sizeof(*list_elt));
+	}
+
+	if ((rc = dhd_iovar(dhd, 0, "ecounters", (char *)req, req->len, NULL, 0, TRUE)) < 0) {
+		DHD_ERROR(("failed to start ecounters\n"));
+	}
+
+fail:
+	if (req) {
+		MFREE(dhd->osh, req, sizeof(*req) + total_processed_containers_len);
+	}
+
+	/* Now go element by element in the list */
+	while (processed_containers_list) {
+		list_elt = processed_containers_list;
+		elt = (bcm_xtlv_t *)list_elt->data;
+		processed_containers_list = processed_containers_list->next;
+
+		/* Free allocated memories */
+		MFREE(dhd->osh, elt, elt->len + BCM_XTLV_HDR_SIZE);
+		MFREE(dhd->osh, list_elt, sizeof(*list_elt));
+	}
+	return rc;
+}
+
+int
+dhd_stop_ecounters(dhd_pub_t *dhd)
+{
+	int rc = BCME_OK;
+	ecounters_config_request_v2_t *req;
+
+	/* Now create ecounters config request with totallength */
+	req = (ecounters_config_request_v2_t *)MALLOCZ(dhd->osh, sizeof(*req));
+
+	if (req == NULL) {
+		rc = BCME_NOMEM;
+		goto fail;
+	}
+
+	req->version = ECOUNTERS_VERSION_2;
+	req->len = OFFSETOF(ecounters_config_request_v2_t, ecounters_xtlvs);
+
+	if ((rc = dhd_iovar(dhd, 0, "ecounters", (char *)req, req->len, NULL, 0, TRUE)) < 0) {
+		DHD_ERROR(("failed to stop ecounters\n"));
+	}
+
+fail:
+	if (req) {
+		MFREE(dhd->osh, req, sizeof(*req));
+	}
+	return rc;
+}
+
+/* configured event_id_array for event ecounters */
+typedef struct event_id_array {
+	uint8	event_id;
+	uint8	str_idx;
+} event_id_array_t;
+
+/* get event id array only from event_ecounters_cfg_tbl[] */
+static inline int __dhd_event_ecounters_get_event_id_array(event_id_array_t *event_array)
+{
+	uint8 i;
+	uint8 idx = 0;
+	int32 prev_evt_id = -1;
+
+	for (i = 0; i < (uint8)ARRAYSIZE(event_ecounters_cfg_tbl); i++) {
+		if (prev_evt_id != event_ecounters_cfg_tbl[i].event_id) {
+			if (prev_evt_id >= 0)
+				idx++;
+			event_array[idx].event_id = event_ecounters_cfg_tbl[i].event_id;
+			event_array[idx].str_idx = i;
+		}
+		prev_evt_id = event_ecounters_cfg_tbl[i].event_id;
+	}
+	return idx;
+}
+
+/* One event id has limit xtlv num to request based on wl_ifstats_xtlv_id * 2 interface */
+#define ECNTRS_MAX_XTLV_NUM (31 * 2)
+
+int
+dhd_start_event_ecounters(dhd_pub_t *dhd)
+{
+	uint8 i, j = 0;
+	uint8 event_id_cnt = 0;
+	uint16 processed_containers_len = 0;
+	uint16 max_xtlv_len = 0;
+	int rc = BCME_OK;
+	uint8 *ptr;
+	uint8 *data;
+	event_id_array_t *id_array;
+	bcm_xtlv_t *elt = NULL;
+	event_ecounters_config_request_v2_t *req = NULL;
+
+	/* XXX: the size of id_array is limited by the size of event_ecounters_cfg_tbl */
+	id_array = (event_id_array_t *)MALLOCZ(dhd->osh, sizeof(event_id_array_t) *
+		ARRAYSIZE(event_ecounters_cfg_tbl));
+
+	if (id_array == NULL) {
+		rc = BCME_NOMEM;
+		goto fail;
+	}
+	event_id_cnt = __dhd_event_ecounters_get_event_id_array(id_array);
+
+	max_xtlv_len = ((BCM_XTLV_HDR_SIZE +
+		OFFSETOF(event_ecounters_config_request_v2_t, ecounters_xtlvs)) *
+		ECNTRS_MAX_XTLV_NUM);
+
+	/* Now create ecounters config request with max allowed length */
+	req = (event_ecounters_config_request_v2_t *)MALLOCZ(dhd->osh,
+		sizeof(event_ecounters_config_request_v2_t *) + max_xtlv_len);
+
+	if (req == NULL) {
+		rc = BCME_NOMEM;
+		goto fail;
+	}
+
+	for (i = 0; i <= event_id_cnt; i++) {
+		/* req initialization by event id */
+		req->version = ECOUNTERS_VERSION_2;
+		req->logset = EVENT_LOG_SET_ECOUNTERS;
+		req->event_id = id_array[i].event_id;
+		req->flags = EVENT_ECOUNTERS_FLAGS_ADD;
+		req->len = 0;
+		processed_containers_len = 0;
+
+		/* Copy config */
+		ptr = req->ecounters_xtlvs;
+
+		for (j = id_array[i].str_idx; j < (uint8)ARRAYSIZE(event_ecounters_cfg_tbl); j++) {
+			event_ecounters_cfg_t *event_ecounter_stat = &event_ecounters_cfg_tbl[j];
+			if (id_array[i].event_id != event_ecounter_stat->event_id)
+				break;
+
+			rc = dhd_create_ecounters_params(dhd, event_ecounter_stat->type,
+				event_ecounter_stat->if_slice_idx, event_ecounter_stat->stats_rep,
+				&data);
+
+			if (rc) {
+				DHD_ERROR(("%s: Could not process: stat: %d return code: %d\n",
+					__FUNCTION__, event_ecounter_stat->stats_rep, rc));
+				goto fail;
+			}
+
+			elt = (bcm_xtlv_t *)data;
+
+			memcpy(ptr, elt, BCM_XTLV_LEN(elt) + BCM_XTLV_HDR_SIZE);
+			ptr += (size_t)(BCM_XTLV_LEN(elt) + BCM_XTLV_HDR_SIZE);
+			processed_containers_len += BCM_XTLV_LEN(elt) + BCM_XTLV_HDR_SIZE;
+
+			/* Free allocated memories alloced by dhd_create_ecounters_params */
+			MFREE(dhd->osh, elt, elt->len + BCM_XTLV_HDR_SIZE);
+
+			if (processed_containers_len > max_xtlv_len) {
+				DHD_ERROR(("%s XTLV NUM IS OVERFLOWED THAN ALLOWED!!\n",
+					__FUNCTION__));
+				rc = BCME_BADLEN;
+				goto fail;
+			}
+		}
+
+		req->len = processed_containers_len +
+			OFFSETOF(event_ecounters_config_request_v2_t, ecounters_xtlvs);
+
+		DHD_INFO(("%s req version %d logset %d event_id %d flags %d len %d\n",
+			__FUNCTION__, req->version, req->logset, req->event_id,
+			req->flags, req->len));
+
+		rc = dhd_iovar(dhd, 0, "event_ecounters", (char *)req, req->len, NULL, 0, TRUE);
+
+		if (rc < 0) {
+			DHD_ERROR(("failed to start event_ecounters(event id %d) with rc %d\n",
+				req->event_id, rc));
+			goto fail;
+		}
+	}
+
+fail:
+	/* Free allocated memories */
+	if (req) {
+		MFREE(dhd->osh, req, sizeof(event_ecounters_config_request_v2_t *) + max_xtlv_len);
+	}
+	if (id_array) {
+		MFREE(dhd->osh, id_array, sizeof(event_id_array_t) *
+			ARRAYSIZE(event_ecounters_cfg_tbl));
+	}
+
+	return rc;
+}
+
+int
+dhd_stop_event_ecounters(dhd_pub_t *dhd)
+{
+	int rc = BCME_OK;
+	event_ecounters_config_request_v2_t *req;
+
+	/* Now create ecounters config request with totallength */
+	req = (event_ecounters_config_request_v2_t *)MALLOCZ(dhd->osh, sizeof(*req));
+
+	if (req == NULL) {
+		rc = BCME_NOMEM;
+		goto fail;
+	}
+
+	req->version = ECOUNTERS_VERSION_2;
+	req->flags = EVENT_ECOUNTERS_FLAGS_DEL_ALL;
+	req->len = OFFSETOF(event_ecounters_config_request_v2_t, ecounters_xtlvs);
+
+	if ((rc = dhd_iovar(dhd, 0, "event_ecounters", (char *)req, req->len, NULL, 0, TRUE)) < 0) {
+		DHD_ERROR(("failed to stop event_ecounters\n"));
+	}
+
+fail:
+	if (req) {
+		MFREE(dhd->osh, req, sizeof(*req));
+	}
+	return rc;
+}
+
+#ifdef DHD_TX_PROFILE
+int
+dhd_tx_profile_detach(dhd_pub_t *dhdp)
+{
+	int result = BCME_ERROR;
+
+	if (dhdp != NULL && dhdp->protocol_filters != NULL) {
+		MFREE(dhdp->osh, dhdp->protocol_filters, DHD_MAX_PROFILES *
+				sizeof(*(dhdp->protocol_filters)));
+		dhdp->protocol_filters = NULL;
+
+		result = BCME_OK;
+	}
+
+	return result;
+}
+
+int
+dhd_tx_profile_attach(dhd_pub_t *dhdp)
+{
+	int result = BCME_ERROR;
+
+	if (dhdp != NULL) {
+		dhdp->protocol_filters = (dhd_tx_profile_protocol_t*)MALLOCZ(dhdp->osh,
+				DHD_MAX_PROFILES * sizeof(*(dhdp->protocol_filters)));
+
+		if (dhdp->protocol_filters != NULL) {
+			result = BCME_OK;
+		}
+	}
+
+	if (result != BCME_OK) {
+		DHD_ERROR(("%s:\tMALLOC of tx profile protocol filters failed\n",
+			__FUNCTION__));
+	}
+
+	return result;
+}
+#endif /* defined(DHD_TX_PROFILE) */
+
+#endif /* LINUX || linux */
+
+#if defined(DISABLE_HE_ENAB) || defined(CUSTOM_CONTROL_HE_ENAB)
+int
+dhd_control_he_enab(dhd_pub_t * dhd, uint8 he_enab)
+{
+	int ret = BCME_OK;
+	bcm_xtlv_t *pxtlv = NULL;
+	uint8 mybuf[DHD_IOVAR_BUF_SIZE];
+	uint16 mybuf_len = sizeof(mybuf);
+	pxtlv = (bcm_xtlv_t *)mybuf;
+
+	ret = bcm_pack_xtlv_entry((uint8**)&pxtlv, &mybuf_len, WL_HE_CMD_ENAB, sizeof(he_enab),
+			&he_enab, BCM_XTLV_OPTION_ALIGN32);
+
+	if (ret != BCME_OK) {
+		ret = -EINVAL;
+		DHD_ERROR(("%s failed to pack he enab, err: %s\n", __FUNCTION__, bcmerrorstr(ret)));
+		return ret;
+	}
+
+	ret = dhd_iovar(dhd, 0, "he", (char *)&mybuf, sizeof(mybuf), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s he_enab (%d) set failed, err: %s\n",
+				__FUNCTION__, he_enab, bcmerrorstr(ret)));
+	} else {
+		DHD_ERROR(("%s he_enab (%d) set successed\n", __FUNCTION__, he_enab));
+	}
+
+	return ret;
+}
+#endif /* DISABLE_HE_ENAB || CUSTOM_CONTROL_HE_ENAB */
+
+#ifdef CONFIG_ROAM_RSSI_LIMIT
+int
+dhd_roam_rssi_limit_get(dhd_pub_t *dhd, int *lmt2g, int *lmt5g)
+{
+	wlc_roam_rssi_limit_t *plmt;
+	wlc_roam_rssi_lmt_info_v1_t *pinfo;
+	int ret = BCME_OK;
+	int plmt_len = sizeof(*pinfo) + ROAMRSSI_HDRLEN;
+
+	plmt = (wlc_roam_rssi_limit_t *)MALLOCZ(dhd->osh, plmt_len);
+	if (!plmt) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+
+	/* Get roam rssi limit */
+	ret = dhd_iovar(dhd, 0, "roam_rssi_limit", NULL, 0, (char *)plmt, plmt_len, FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Failed to Get roam_rssi_limit %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+	if (plmt->ver != WLC_ROAM_RSSI_LMT_VER_1) {
+	    ret = BCME_VERSION;
+	    goto done;
+	}
+
+	pinfo = (wlc_roam_rssi_lmt_info_v1_t *)plmt->data;
+	*lmt2g = (int)pinfo->rssi_limit_2g;
+	*lmt5g = (int)pinfo->rssi_limit_5g;
+
+done:
+	if (plmt) {
+		MFREE(dhd->osh, plmt, plmt_len);
+	}
+	return ret;
+}
+
+int
+dhd_roam_rssi_limit_set(dhd_pub_t *dhd, int lmt2g, int lmt5g)
+{
+	wlc_roam_rssi_limit_t *plmt;
+	wlc_roam_rssi_lmt_info_v1_t *pinfo;
+	int ret = BCME_OK;
+	int plmt_len = sizeof(*pinfo) + ROAMRSSI_HDRLEN;
+
+	/* Sanity check RSSI limit Value */
+	if ((lmt2g < ROAMRSSI_2G_MIN) || (lmt2g > ROAMRSSI_2G_MAX)) {
+		DHD_ERROR(("%s Not In Range 2G ROAM RSSI Limit\n", __FUNCTION__));
+		return BCME_RANGE;
+	}
+	if ((lmt2g < ROAMRSSI_5G_MIN) || (lmt2g > ROAMRSSI_5G_MAX)) {
+		DHD_ERROR(("%s Not In Range 5G ROAM RSSI Limit\n", __FUNCTION__));
+		return BCME_RANGE;
+	}
+
+	plmt = (wlc_roam_rssi_limit_t *)MALLOCZ(dhd->osh, plmt_len);
+	if (!plmt) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+	plmt->ver = WLC_ROAM_RSSI_LMT_VER_1;
+	plmt->len = sizeof(*pinfo);
+	pinfo = (wlc_roam_rssi_lmt_info_v1_t *)plmt->data;
+	pinfo->rssi_limit_2g = (int16)lmt2g;
+	pinfo->rssi_limit_5g = (int16)lmt5g;
+
+	/* Set roam rssi limit */
+	ret = dhd_iovar(dhd, 0, "roam_rssi_limit", (char *)plmt, plmt_len, NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Failed to Get roam_rssi_limit %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+done:
+	if (plmt) {
+		MFREE(dhd->osh, plmt, plmt_len);
+	}
+	return ret;
+}
+#endif /* CONFIG_ROAM_RSSI_LIMIT */
+
+#ifdef CONFIG_ROAM_MIN_DELTA
+int
+dhd_roam_min_delta_get(dhd_pub_t *dhd, uint32 *dt2g, uint32 *dt5g)
+{
+	wlc_roam_min_delta_t *pmin_delta;
+	wlc_roam_min_delta_info_v1_t *pmin_delta_info;
+	int ret = BCME_OK;
+	int plen = sizeof(*pmin_delta_info) + ROAM_MIN_DELTA_HDRLEN;
+
+	pmin_delta = (wlc_roam_min_delta_t *)MALLOCZ(dhd->osh, plen);
+	if (!pmin_delta) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+
+	/* Get Minimum ROAM score delta */
+	ret = dhd_iovar(dhd, 0, "roam_min_delta", NULL, 0, (char *)pmin_delta, plen, FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Failed to Get roam_min_delta %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+	if (pmin_delta->ver != WLC_ROAM_MIN_DELTA_VER_1) {
+		ret = BCME_VERSION;
+		goto done;
+	}
+
+	pmin_delta_info = (wlc_roam_min_delta_info_v1_t *)pmin_delta->data;
+	*dt2g = (uint32)pmin_delta_info->roam_min_delta_2g;
+	*dt5g = (uint32)pmin_delta_info->roam_min_delta_5g;
+
+done:
+	if (pmin_delta) {
+		MFREE(dhd->osh, pmin_delta, plen);
+	}
+	return ret;
+}
+
+int
+dhd_roam_min_delta_set(dhd_pub_t *dhd, uint32 dt2g, uint32 dt5g)
+{
+	wlc_roam_min_delta_t *pmin_delta;
+	wlc_roam_min_delta_info_v1_t *pmin_delta_info;
+	int ret = BCME_OK;
+	int plen = sizeof(*pmin_delta_info) + ROAM_MIN_DELTA_HDRLEN;
+
+	/* Sanity check Minimum ROAM score delta */
+	if ((dt2g > ROAM_MIN_DELTA_MAX) || (dt5g > ROAM_MIN_DELTA_MAX)) {
+		DHD_ERROR(("%s Not In Range Minimum ROAM score delta, 2G: %d, 5G: %d\n",
+			__FUNCTION__, dt2g, dt5g));
+		return BCME_RANGE;
+	}
+
+	pmin_delta = (wlc_roam_min_delta_t *)MALLOCZ(dhd->osh, plen);
+	if (!pmin_delta) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+	pmin_delta->ver = WLC_ROAM_MIN_DELTA_VER_1;
+	pmin_delta->len = sizeof(*pmin_delta_info);
+	pmin_delta_info = (wlc_roam_min_delta_info_v1_t *)pmin_delta->data;
+	pmin_delta_info->roam_min_delta_2g = (uint32)dt2g;
+	pmin_delta_info->roam_min_delta_5g = (uint32)dt5g;
+
+	/* Set Minimum ROAM score delta */
+	ret = dhd_iovar(dhd, 0, "roam_min_delta", (char *)pmin_delta, plen, NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Failed to Set roam_min_delta %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+done:
+	if (pmin_delta) {
+		MFREE(dhd->osh, pmin_delta, plen);
+	}
+	return ret;
+}
+#endif /* CONFIG_ROAM_MIN_DELTA */
+
+int
+dhd_iovar(dhd_pub_t *pub, int ifidx, char *name, char *param_buf, uint param_len, char *res_buf,
+		uint res_len, bool set)
+{
+	char *buf = NULL;
+	uint input_len;
+	wl_ioctl_t ioc;
+	int ret;
+
+	if (res_len > WLC_IOCTL_MAXLEN || param_len > WLC_IOCTL_MAXLEN)
+		return BCME_BADARG;
+
+	input_len = strlen(name) + 1 + param_len;
+
+	/* WAR to fix GET iovar returning buf too short error
+	 * If param len is 0 for get iovar, increment input_len by sizeof(int)
+	 * to avoid the length check error in fw
+	 */
+	if (!set && !param_len) {
+		input_len += (uint) sizeof(int);
+	}
+	if (input_len > WLC_IOCTL_MAXLEN)
+		return BCME_BADARG;
+
+	buf = NULL;
+	if (set) {
+		if (res_buf || res_len != 0) {
+			DHD_ERROR(("%s: SET wrong arguemnet\n", __FUNCTION__));
+			ret = BCME_BADARG;
+			goto exit;
+		}
+		buf = MALLOCZ(pub->osh, input_len);
+		if (!buf) {
+			DHD_ERROR(("%s: mem alloc failed\n", __FUNCTION__));
+			ret = BCME_NOMEM;
+			goto exit;
+		}
+		ret = bcm_mkiovar(name, param_buf, param_len, buf, input_len);
+		if (!ret) {
+			ret = BCME_NOMEM;
+			goto exit;
+		}
+
+		ioc.cmd = WLC_SET_VAR;
+		ioc.buf = buf;
+		ioc.len = input_len;
+		ioc.set = set;
+
+		ret = dhd_wl_ioctl(pub, ifidx, &ioc, ioc.buf, ioc.len);
+	} else {
+		if (!res_buf || !res_len) {
+			DHD_ERROR(("%s: GET failed. resp_buf NULL or length 0.\n", __FUNCTION__));
+			ret = BCME_BADARG;
+			goto exit;
+		}
+
+		if (res_len < input_len) {
+			DHD_INFO(("%s: res_len(%d) < input_len(%d)\n", __FUNCTION__,
+					res_len, input_len));
+			buf = MALLOCZ(pub->osh, input_len);
+			if (!buf) {
+				DHD_ERROR(("%s: mem alloc failed\n", __FUNCTION__));
+				ret = BCME_NOMEM;
+				goto exit;
+			}
+			ret = bcm_mkiovar(name, param_buf, param_len, buf, input_len);
+			if (!ret) {
+				ret = BCME_NOMEM;
+				goto exit;
+			}
+
+			ioc.cmd = WLC_GET_VAR;
+			ioc.buf = buf;
+			ioc.len = input_len;
+			ioc.set = set;
+
+			ret = dhd_wl_ioctl(pub, ifidx, &ioc, ioc.buf, ioc.len);
+
+			if (ret == BCME_OK) {
+				memcpy(res_buf, buf, res_len);
+			}
+		} else {
+			memset(res_buf, 0, res_len);
+			ret = bcm_mkiovar(name, param_buf, param_len, res_buf, res_len);
+			if (!ret) {
+				ret = BCME_NOMEM;
+				goto exit;
+			}
+
+			ioc.cmd = WLC_GET_VAR;
+			ioc.buf = res_buf;
+			ioc.len = res_len;
+			ioc.set = set;
+
+			ret = dhd_wl_ioctl(pub, ifidx, &ioc, ioc.buf, ioc.len);
+		}
+	}
+exit:
+	if (buf) {
+		MFREE(pub->osh, buf, input_len);
+	}
+	return ret;
+}
+
+#ifdef DHD_DEBUG
+void
+dhd_convert_memdump_type_to_str(uint32 type, char *buf, size_t buf_len, int substr_type)
+{
+	char *type_str = NULL;
+
+	switch (type) {
+		case DUMP_TYPE_RESUMED_ON_TIMEOUT:
+			type_str = "resumed_on_timeout";
+			break;
+		case DUMP_TYPE_D3_ACK_TIMEOUT:
+			type_str = "D3_ACK_timeout";
+			break;
+		case DUMP_TYPE_DONGLE_TRAP:
+			type_str = "Dongle_Trap";
+			break;
+		case DUMP_TYPE_DONGLE_TRAP_DURING_WIFI_ONOFF:
+			type_str = "Dongle_Trap_During_Wifi_OnOff";
+			break;
+		case DUMP_TYPE_BY_DSACK_HC_DUE_TO_ISR_DELAY:
+			type_str = "Dongle_Trap_DSACK_HC_due_to_ISR_delay";
+			break;
+		case DUMP_TYPE_BY_DSACK_HC_DUE_TO_DPC_DELAY:
+			type_str = "Dongle_Trap_DSACK_HC_due_to_DPC_delay";
+			break;
+		case DUMP_TYPE_MEMORY_CORRUPTION:
+			type_str = "Memory_Corruption";
+			break;
+		case DUMP_TYPE_PKTID_AUDIT_FAILURE:
+			type_str = "PKTID_AUDIT_Fail";
+			break;
+		case DUMP_TYPE_PKTID_INVALID:
+			type_str = "PKTID_INVALID";
+			break;
+		case DUMP_TYPE_SCAN_TIMEOUT:
+			type_str = "SCAN_timeout";
+			break;
+		case DUMP_TYPE_SCAN_BUSY:
+			type_str = "SCAN_Busy";
+			break;
+		case DUMP_TYPE_BY_SYSDUMP:
+			if (substr_type == CMD_UNWANTED) {
+				type_str = "BY_SYSDUMP_FORUSER_unwanted";
+			} else if (substr_type == CMD_DISCONNECTED) {
+				type_str = "BY_SYSDUMP_FORUSER_disconnected";
+			} else {
+				type_str = "BY_SYSDUMP_FORUSER";
+			}
+			break;
+		case DUMP_TYPE_BY_LIVELOCK:
+			type_str = "BY_LIVELOCK";
+			break;
+		case DUMP_TYPE_AP_LINKUP_FAILURE:
+			type_str = "BY_AP_LINK_FAILURE";
+			break;
+		case DUMP_TYPE_AP_ABNORMAL_ACCESS:
+			type_str = "INVALID_ACCESS";
+			break;
+		case DUMP_TYPE_RESUMED_ON_TIMEOUT_RX:
+			type_str = "ERROR_RX_TIMED_OUT";
+			break;
+		case DUMP_TYPE_RESUMED_ON_TIMEOUT_TX:
+			type_str = "ERROR_TX_TIMED_OUT";
+			break;
+		case DUMP_TYPE_CFG_VENDOR_TRIGGERED:
+			type_str = "CFG_VENDOR_TRIGGERED";
+			break;
+		case DUMP_TYPE_RESUMED_ON_INVALID_RING_RDWR:
+			type_str = "BY_INVALID_RING_RDWR";
+			break;
+		case DUMP_TYPE_IFACE_OP_FAILURE:
+			type_str = "BY_IFACE_OP_FAILURE";
+			break;
+		case DUMP_TYPE_TRANS_ID_MISMATCH:
+			type_str = "BY_TRANS_ID_MISMATCH";
+			break;
+#ifdef DEBUG_DNGL_INIT_FAIL
+		case DUMP_TYPE_DONGLE_INIT_FAILURE:
+			type_str = "DONGLE_INIT_FAIL";
+			break;
+#endif /* DEBUG_DNGL_INIT_FAIL */
+		case DUMP_TYPE_DONGLE_HOST_EVENT:
+			type_str = "BY_DONGLE_HOST_EVENT";
+			break;
+		case DUMP_TYPE_SMMU_FAULT:
+			type_str = "SMMU_FAULT";
+			break;
+		case DUMP_TYPE_BY_USER:
+			type_str = "BY_USER";
+			break;
+		case DUMP_TYPE_LOGSET_BEYOND_RANGE:
+			type_str = "LOGSET_BEYOND_RANGE";
+			break;
+		case DUMP_TYPE_CTO_RECOVERY:
+			type_str = "CTO_RECOVERY";
+			break;
+		case DUMP_TYPE_SEQUENTIAL_PRIVCMD_ERROR:
+			type_str = "SEQUENTIAL_PRIVCMD_ERROR";
+			break;
+		case DUMP_TYPE_PROXD_TIMEOUT:
+			type_str = "PROXD_TIMEOUT";
+			break;
+		case DUMP_TYPE_INBAND_DEVICE_WAKE_FAILURE:
+			type_str = "INBAND_DEVICE_WAKE_FAILURE";
+			break;
+		case DUMP_TYPE_PKTID_POOL_DEPLETED:
+			type_str = "PKTID_POOL_DEPLETED";
+			break;
+		case DUMP_TYPE_ESCAN_SYNCID_MISMATCH:
+			type_str = "ESCAN_SYNCID_MISMATCH";
+			break;
+		case DUMP_TYPE_INVALID_SHINFO_NRFRAGS:
+			type_str = "INVALID_SHINFO_NRFRAGS";
+			break;
+		case DUMP_TYPE_P2P_DISC_BUSY:
+			type_str = "P2P_DISC_BUSY";
+			break;
+		case DUMP_TYPE_CONT_EXCESS_PM_AWAKE:
+			type_str = "CONT_EXCESS_PM_AWAKE";
+			break;
+		default:
+			type_str = "Unknown_type";
+			break;
+	}
+
+	strlcpy(buf, type_str, buf_len);
+}
+#endif /* DHD_DEBUG */
+
+#ifdef DHD_LOGLEVEL
+void dhd_loglevel_get(dhd_loglevel_data_t *level_data)
+{
+	level_data->dhd_print_lv = dhd_msg_level;
+	level_data->dhd_log_lv = dhd_log_level;
+	level_data->wl_print_lv = wl_cfg80211_get_print_level();
+	level_data->wl_log_lv = wl_cfg80211_get_log_level();
+}
+
+void dhd_loglevel_set(dhd_loglevel_data_t *level_data)
+{
+	if (level_data->type == DHD_LOGLEVEL_TYPE_ALL) {
+#ifdef WL_CFG80211
+		if (level_data->component == DHD_LOGLEVEL_COMP_WL) {
+			wl_cfg80211_enable_trace(level_data->wl_print_lv);
+			wl_cfg80211_enable_log_trace(TRUE, level_data->wl_log_lv);
+		} else
+#endif /* WL_CFG80211 */
+		{
+			dhd_msg_level = level_data->dhd_print_lv;
+			dhd_log_level = level_data->dhd_log_lv;
+		}
+	} else if (level_data->type == DHD_LOGLEVEL_TYPE_PRINT) {
+#ifdef WL_CFG80211
+		if (level_data->component == DHD_LOGLEVEL_COMP_WL) {
+			wl_cfg80211_enable_trace(level_data->wl_print_lv);
+		} else
+#endif /* WL_CFG80211 */
+		{
+			dhd_msg_level = level_data->dhd_print_lv;
+		}
+	} else if (level_data->type == DHD_LOGLEVEL_TYPE_LOG) {
+#ifdef WL_CFG80211
+		if (level_data->component == DHD_LOGLEVEL_COMP_WL) {
+			wl_cfg80211_enable_log_trace(TRUE, level_data->wl_log_lv);
+		} else
+#endif /* WL_CFG80211 */
+		{
+			dhd_log_level = level_data->dhd_log_lv;
+		}
+	}
+	DHD_ERROR(("%s: type:%u dhd_msg_level:0x%x dhd_log_level:0x%x\n",
+		__func__, level_data->type, dhd_msg_level, dhd_log_level));
+	DHD_ERROR(("%s: type:%u wl_dbg_level:0x%x wl_log_level:0x%x\n",
+		__func__, level_data->type, wl_dbg_level, wl_log_level));
+}
+#endif /* DHD_LOGLEVEL */
+
+#if defined(DHD_WAKE_STATUS_PRINT) && defined(DHD_WAKE_RX_STATUS) && \
+	defined(DHD_WAKE_STATUS)
+#ifdef BCMPCIE
+extern int bcmpcie_get_total_wake(struct dhd_bus *bus);
+#endif /* BCMPCIE */
+void
+dhd_dump_wake_status(dhd_pub_t *dhdp, wake_counts_t *wcp, struct ether_header *eh)
+{
+	uint wake = 0;
+
+	BCM_REFERENCE(wake);
+#ifdef BCMPCIE
+	wake = bcmpcie_get_total_wake(dhdp->bus);
+#endif /* BCMPCIE */
+
+	DHD_ERROR(("wake %u rxwake %u readctrlwake %u\n",
+		wake, wcp->rxwake, wcp->rcwake));
+	DHD_ERROR(("unicast %u muticast %u broadcast %u icmp %u\n",
+		wcp->rx_ucast, wcp->rx_mcast, wcp->rx_bcast, wcp->rx_icmp));
+	DHD_ERROR(("multi4 %u multi6 %u icmp6 %u multiother %u\n",
+		wcp->rx_multi_ipv4, wcp->rx_multi_ipv6, wcp->rx_icmpv6, wcp->rx_multi_other));
+	DHD_ERROR(("icmp6_ra %u, icmp6_na %u, icmp6_ns %u\n",
+		wcp->rx_icmpv6_ra, wcp->rx_icmpv6_na, wcp->rx_icmpv6_ns));
+	DHD_ERROR(("Src_mac: "MACDBG", Dst_mac: "MACDBG"\n",
+		MAC2STRDBG(eh->ether_shost), MAC2STRDBG(eh->ether_dhost)));
+	return;
+}
+#endif /* DHD_WAKE_STATUS_PRINT && DHD_WAKE_RX_STATUS && DHD_WAKE_STATUS */
+#ifdef SUPPORT_OTA_UPDATE
+void
+dhd_ota_buf_clean(dhd_pub_t *dhdp)
+{
+	ota_update_info_t *ota_info = &dhdp->ota_update_info;
+
+	if (ota_info->clm_buf) {
+		MFREE(dhdp->osh, ota_info->clm_buf, ota_info->clm_len);
+	}
+	if (ota_info->nvram_buf) {
+		MFREE(dhdp->osh, ota_info->nvram_buf, ota_info->nvram_len);
+	}
+	ota_info->nvram_len = 0;
+	ota_info->clm_len = 0;
+	return;
+}
+#endif /* SUPPORT_OTA_UPDATE */
+
+#if defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P)
+/* From Android JerryBean release, the concurrent mode is enabled by default and the firmware
+ * name would be fw_bcmdhd.bin. So we need to determine whether P2P is enabled in the STA
+ * firmware and accordingly enable concurrent mode (Apply P2P settings). SoftAP firmware
+ * would still be named as fw_bcmdhd_apsta.
+ */
+uint32
+dhd_get_concurrent_capabilites(dhd_pub_t *dhd)
+{
+	int32 ret = 0;
+	char buf[WLC_IOCTL_SMLEN];
+	bool mchan_supported = FALSE;
+	/* if dhd->op_mode is already set for HOSTAP and Manufacturing
+	 * test mode, that means we only will use the mode as it is
+	 */
+	if (dhd->op_mode & (DHD_FLAG_HOSTAP_MODE | DHD_FLAG_MFG_MODE))
+		return 0;
+	if (FW_SUPPORTED(dhd, vsdb)) {
+		mchan_supported = TRUE;
+	}
+	if (!FW_SUPPORTED(dhd, p2p)) {
+		DHD_TRACE(("Chip does not support p2p\n"));
+		return 0;
+	} else {
+		/* Chip supports p2p but ensure that p2p is really implemented in firmware or not */
+		memset(buf, 0, sizeof(buf));
+		ret = dhd_iovar(dhd, 0, "p2p", NULL, 0, (char *)&buf,
+				sizeof(buf), FALSE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: Get P2P failed (error=%d)\n", __FUNCTION__, ret));
+			return 0;
+		} else {
+			if (buf[0] == 1) {
+				/* By default, chip supports single chan concurrency,
+				* now lets check for mchan
+				*/
+				ret = DHD_FLAG_CONCURR_SINGLE_CHAN_MODE;
+				if (mchan_supported)
+					ret |= DHD_FLAG_CONCURR_MULTI_CHAN_MODE;
+				if (FW_SUPPORTED(dhd, rsdb)) {
+					ret |= DHD_FLAG_RSDB_MODE;
+				}
+#ifdef WL_SUPPORT_MULTIP2P
+				if (FW_SUPPORTED(dhd, mp2p)) {
+					ret |= DHD_FLAG_MP2P_MODE;
+				}
+#endif /* WL_SUPPORT_MULTIP2P */
+#if defined(WL_CFG80211_P2P_DEV_IF)
+				return ret;
+#else
+				return 0;
+#endif
+			}
+		}
+	}
+	return 0;
+}
+#endif /* defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P) */
+
+#ifdef SUPPORT_AP_POWERSAVE
+int dhd_set_ap_powersave(dhd_pub_t *dhdp, int ifidx, int enable)
+{
+	int32 pps = RXCHAIN_PWRSAVE_PPS;
+	int32 quiet_time = RXCHAIN_PWRSAVE_QUIET_TIME;
+	int32 stas_assoc_check = RXCHAIN_PWRSAVE_STAS_ASSOC_CHECK;
+	int ret;
+
+	if (enable) {
+		ret = dhd_iovar(dhdp, 0, "rxchain_pwrsave_enable", (char *)&enable, sizeof(enable),
+				NULL, 0, TRUE);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("Failed to enable AP power save"));
+		}
+		ret = dhd_iovar(dhdp, 0, "rxchain_pwrsave_pps", (char *)&pps, sizeof(pps), NULL, 0,
+				TRUE);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("Failed to set pps"));
+		}
+		ret = dhd_iovar(dhdp, 0, "rxchain_pwrsave_quiet_time", (char *)&quiet_time,
+				sizeof(quiet_time), NULL, 0, TRUE);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("Failed to set quiet time"));
+		}
+		ret = dhd_iovar(dhdp, 0, "rxchain_pwrsave_stas_assoc_check",
+				(char *)&stas_assoc_check, sizeof(stas_assoc_check), NULL, 0, TRUE);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("Failed to set stas assoc check"));
+		}
+	} else {
+		ret = dhd_iovar(dhdp, 0, "rxchain_pwrsave_enable", (char *)&enable, sizeof(enable),
+				NULL, 0, TRUE);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("Failed to disable AP power save"));
+		}
+	}
+
+	return 0;
+}
+#endif /* SUPPORT_AP_POWERSAVE */
+#ifdef DHD_COREDUMP
+void
+dhd_convert_hang_reason_to_str(uint32 reason, char *buf, size_t buf_len)
+{
+	char *type_str = NULL;
+
+	switch (reason) {
+		case HANG_REASON_BUS_DOWN:
+			type_str = "BUS_DOWN";
+			break;
+		case HANG_REASON_PCIE_RC_LINK_UP_FAIL:
+			type_str = "PCIE_RC_LINK_UP_FAIL";
+			break;
+		case HANG_REASON_IOCTL_SUSPEND_ERROR:
+			type_str = "IOCTL_SUSPEND_ERROR";
+			break;
+		case HANG_REASON_PCIE_LINK_DOWN_RC_DETECT:
+			type_str = "PCIE_LINK_DOWN_RC_DETECT";
+			break;
+		case HANG_REASON_PCIE_LINK_DOWN_EP_DETECT:
+			type_str = "PCIE_LINK_DOWN_EP_DETECT";
+			break;
+		case HANG_REASON_MSGBUF_LIVELOCK:
+			type_str = "BY_LIVELOCK";
+			break;
+		case HANG_REASON_IOCTL_RESP_TIMEOUT:
+		case HANG_REASON_IOCTL_RESP_TIMEOUT_SCHED_ERROR:
+			type_str = "resumed_on_timeout";
+			break;
+		case HANG_REASON_D3_ACK_TIMEOUT:
+		case HANG_REASON_D3_ACK_TIMEOUT_SCHED_ERROR:
+			type_str = "D3_ACK_timeout";
+			break;
+		case HANG_REASON_DONGLE_TRAP:
+			type_str = "Dongle_Trap";
+			break;
+		case HANG_REASON_PCIE_PKTID_ERROR:
+			type_str = "PKTID_INVALID";
+			break;
+		case HANG_REASON_SCAN_TIMEOUT:
+		case HANG_REASON_SCAN_TIMEOUT_SCHED_ERROR:
+			type_str = "SCAN_timeout";
+			break;
+		case HANG_REASON_SCAN_BUSY:
+			type_str = "SCAN_Busy";
+			break;
+		case HANG_REASON_IFACE_DEL_FAILURE:
+		case HANG_REASON_IFACE_ADD_FAILURE:
+		case HANG_REASON_BSS_UP_FAILURE:
+		case HANG_REASON_BSS_DOWN_FAILURE:
+			type_str = "BY_IFACE_OP_FAILURE";
+			break;
+		case HANG_REASON_PCIE_CTO_DETECT:
+			type_str = "CTO_RECOVERY";
+			break;
+		case HANG_REASON_SEQUENTIAL_PRIVCMD_ERROR:
+			type_str = "SEQUENTIAL_PRIVCMD_ERROR";
+			break;
+		case HANG_REASON_ESCAN_SYNCID_MISMATCH:
+			type_str = "ESCAN_SYNCID_MISMATCH";
+			break;
+		case HANG_REASON_P2P_DISC_BUSY:
+			type_str = "P2P_DISC_BUSY";
+			break;
+		case HANG_REASON_DS_SKIP_TIMEOUT:
+			type_str = "DS_SKIP_TIMEOUT";
+			break;
+		case HANG_REASON_BT2WL_REG_RESET:
+			type_str = "BT2WL_REG_RESET";
+			break;
+		default:
+			type_str = "Unknown_type";
+			break;
+	}
+
+	strlcpy(buf, type_str, buf_len);
+}
+#endif /* DHD_COREDUMP */
+#ifdef DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND
+int dhd_config_rts_in_suspend(dhd_pub_t *dhdp, bool suspend)
+{
+	int ret = BCME_OK;
+	uint return_to_sleep_time = 0;
+
+	int buf_len;
+	bcm_iov_buf_t *iov_buf = NULL;
+	wl_adps_suspend_v1_t *data = NULL;
+
+	if (!dhdp || dhdp->up == 0) {
+		return ret;
+	}
+
+	/* if it's not associated, skip to set */
+	if (!dhd_is_associated(dhdp, 0, NULL)) {
+		return ret;
+	}
+
+	if (suspend) {
+		return_to_sleep_time = CUSTOM_RETRUN_TO_SLEEP_TIME_SUSPEND;
+	}
+	else {
+		return_to_sleep_time = CUSTOM_RETRUN_TO_SLEEP_TIME_DEFAULT;
+	}
+
+	ret = dhd_iovar(dhdp, 0, "pm2_sleep_ret", (char *)&return_to_sleep_time,
+			sizeof(return_to_sleep_time), NULL, 0, TRUE);
+
+	if (ret < 0) {
+		DHD_ERROR(("%s set pm2_sleep_ret failed %d\n", __FUNCTION__, ret));
+		goto exit;
+	}
+
+	buf_len = OFFSETOF(bcm_iov_buf_t, data) + sizeof(*data);
+	iov_buf = MALLOCZ(dhdp->osh, buf_len);
+	if (iov_buf == NULL) {
+		DHD_ERROR(("%s - failed to alloc %d bytes for iov_buf\n",
+			__FUNCTION__, buf_len));
+		ret = BCME_NOMEM;
+		goto exit;
+	}
+
+	iov_buf->version = WL_ADPS_IOV_VER;
+	iov_buf->len = sizeof(*data);
+	iov_buf->id = WL_ADPS_IOV_SUSPEND;
+
+	data = (wl_adps_suspend_v1_t *)iov_buf->data;
+	data->version = ADPS_SUB_IOV_VERSION_1;
+	data->length = sizeof(*data);
+	data->suspend = suspend;
+
+	ret = dhd_iovar(dhdp, 0, "adps", (char *)iov_buf, buf_len,
+		NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s - fail to set adps suspend %d (%d)\n",
+				__FUNCTION__, suspend, ret));
+		goto exit;
+	}
+
+exit:
+	if (ret == BCME_OK) {
+		DHD_ERROR(("%s - Success to set RTS [%d] and ADPS suspend [%d].\n",
+				__FUNCTION__, return_to_sleep_time, suspend));
+	}
+	if (iov_buf) {
+		MFREE(dhdp->osh, iov_buf, buf_len);
+	}
+	return ret;
+}
+#endif /* DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND */
+
+void
+dhd_histo_update(dhd_pub_t *dhd, uint64 *histo, uint32 value)
+{
+	uint64 *p;
+	uint64 *bin = histo;
+	uint32 bin_power = next_larger_power2(value);
+	if (histo == NULL) {
+		DHD_ERROR(("%s: histo is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	switch (bin_power) {
+		case   1: p = bin + 0; break;
+		case   2: p = bin + 1; break;
+		case   4: p = bin + 2; break;
+		case   8: p = bin + 3; break;
+		case  16: p = bin + 4; break;
+		case  32: p = bin + 5; break;
+		case  64: p = bin + 6; break;
+		case 128: p = bin + 7; break;
+		case 256: p = bin + 8; break;
+		case 512: p = bin + 9; break;
+		case 1024: p = bin + 10; break;
+		case 2048: p = bin + 11; break;
+		case 4096: p = bin + 12; break;
+		case 8192: p = bin + 13; break;
+		default : p = bin + 13; break;
+	}
+	ASSERT((p - bin) < DHD_HISTOGRAM_ENTRIES);
+	*p = *p + 1;
+	return;
+}
+
+void
+dhd_histo_clear(dhd_pub_t *dhd, uint64 *histo)
+{
+
+	if (histo == NULL) {
+		return;
+	}
+
+	bzero(histo, DHD_HISTOGRAM_SIZE);
+}
+
+void
+dhd_histo_tag_dump(dhd_pub_t *dhd, struct bcmstrbuf *strbuf, char *tagname)
+{
+	uint32 i;
+
+	bcm_bprintf(strbuf, "%20s: ", tagname);
+	for (i = 0; i < DHD_HISTOGRAM_ENTRIES; i++) {
+		bcm_bprintf(strbuf, "%10llu ", 1ULL<<i);
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+
+void
+dhd_histo_dump(dhd_pub_t *dhd, struct bcmstrbuf *strbuf, uint64 *histo, char *histoname)
+{
+	uint32 i;
+
+	if (histo == NULL) {
+		return;
+	}
+
+	bcm_bprintf(strbuf, "%20s: ", histoname);
+	for (i = 0; i < DHD_HISTOGRAM_ENTRIES; i++) {
+		bcm_bprintf(strbuf, "%10llu ", histo[i]);
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+
+uint64 *
+dhd_histo_init(dhd_pub_t *dhd)
+{
+	uint64 *histo;
+	histo = (uint64 *)MALLOCZ(dhd->osh, DHD_HISTOGRAM_SIZE);
+	if (histo == NULL) {
+		DHD_ERROR(("%s: unable to alloc histo\n", __FUNCTION__));
+	}
+	return histo;
+}
+
+void
+dhd_histo_deinit(dhd_pub_t *dhd, uint64 *histo)
+{
+	if (histo) {
+		MFREE(dhd->osh, histo, DHD_HISTOGRAM_SIZE);
+	}
+}
diff --git a/drivers/net/wireless/ap12275/dhd_config.c b/drivers/net/wireless/ap12275/dhd_config.c
new file mode 100644
index 000000000000..480f11632739
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_config.c
@@ -0,0 +1,5832 @@
+
+#include <typedefs.h>
+#include <osl.h>
+
+#include <bcmendian.h>
+#include <bcmutils.h>
+#include <hndsoc.h>
+#include <bcmsdbus.h>
+#if defined(HW_OOB) || defined(FORCE_WOWLAN)
+#include <bcmdefs.h>
+#include <bcmsdh.h>
+#include <sdio.h>
+#include <sbchipc.h>
+#endif
+#ifdef DHDTCPACK_SUPPRESS
+#include <dhd_ip.h>
+#endif /* DHDTCPACK_SUPPRESS */
+#ifdef WL_CFG80211
+#include <wl_cfg80211.h>
+#endif
+
+#include <dhd_config.h>
+#include <dhd_dbg.h>
+#include <wl_android.h>
+#ifdef BCMPCIE
+#include <dhd_flowring.h>
+#endif
+
+#if defined(BCMSDIO) || defined(BCMPCIE)
+#include <dhd_linux.h>
+#include <dhd_bus.h>
+#ifdef BCMSDIO
+#include <linux/mmc/core.h>
+#include <linux/mmc/card.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/sdio_func.h>
+#endif /* defined(BCMSDIO) */
+#endif
+
+/* message levels */
+#define CONFIG_ERROR_LEVEL	(1 << 0)
+#define CONFIG_TRACE_LEVEL	(1 << 1)
+#ifndef ENODEBUG  
+#define CONFIG_MSG_LEVEL	(1 << 2)
+uint config_msg_level = CONFIG_ERROR_LEVEL | CONFIG_MSG_LEVEL;
+#else
+#define CONFIG_MSG_LEVEL	(1 << 2)
+uint config_msg_level = CONFIG_ERROR_LEVEL;
+#endif
+uint dump_msg_level = 0;
+
+#define CONFIG_MSG(x, args...) \
+	do { \
+		if (config_msg_level & CONFIG_MSG_LEVEL) { \
+			printf("%s : " x, __func__, ## args); \
+		} \
+	} while (0)
+#define CONFIG_ERROR(x, args...) \
+	do { \
+		if (config_msg_level & CONFIG_ERROR_LEVEL) { \
+			printf("CONFIG-ERROR) %s : " x, __func__, ## args); \
+		} \
+	} while (0)
+#define CONFIG_TRACE(x, args...) \
+	do { \
+		if (config_msg_level & CONFIG_TRACE_LEVEL) { \
+			printf("CONFIG-TRACE) %s : " x, __func__, ## args); \
+		} \
+	} while (0)
+
+#define MAXSZ_BUF		4096
+#define MAXSZ_CONFIG	8192
+
+extern uint wl_reassoc_support;
+#ifdef BTC_WAR
+extern int btc_war;
+#endif /* BTC_WAR */
+#if defined(BCMSDIO) && defined(DYNAMIC_MAX_HDR_READ)
+extern uint firstread;
+#endif
+
+#if defined(PROP_TXSTATUS)
+#include <dhd_wlfc.h>
+#endif /* PROP_TXSTATUS */
+
+#define MAX_EVENT_BUF_NUM 16
+typedef struct eventmsg_buf {
+	u16 num;
+	struct {
+		u16 type;
+		bool set;
+	} event [MAX_EVENT_BUF_NUM];
+} eventmsg_buf_t;
+
+typedef struct chip_name_map_t {
+	uint chip;
+	uint chiprev;
+	uint ag_type;
+	char *chip_name;
+	char *module_name;
+} chip_name_map_t;
+
+/* Map of WLC_E events to connection failure strings */
+#define DONT_CARE	9999
+const chip_name_map_t chip_name_map[] = {
+	/* ChipID			Chiprev	AG	 	ChipName	ModuleName  */
+#ifdef BCMSDIO
+	{BCM43362_CHIP_ID,	0,	DONT_CARE,	"bcm40181a0",		""},
+	{BCM43362_CHIP_ID,	1,	DONT_CARE,	"bcm40181a2",		""},
+	{BCM4330_CHIP_ID,	4,	FW_TYPE_G,	"bcm40183b2",		""},
+	{BCM4330_CHIP_ID,	4,	FW_TYPE_AG,	"bcm40183b2_ag",	""},
+	{BCM43430_CHIP_ID,	0,	DONT_CARE,	"bcm43438a0",		""},
+	{BCM43430_CHIP_ID,	1,	DONT_CARE,	"bcm43438a1",		""},
+	{BCM43430_CHIP_ID,	2,	DONT_CARE,	"bcm43436b0",		""},
+	{BCM43012_CHIP_ID,	1,	FW_TYPE_G,	"bcm43013b0",		""},
+	{BCM43012_CHIP_ID,	1,	FW_TYPE_AG,	"bcm43013c0_ag",	""},
+	{BCM43012_CHIP_ID,	2,	DONT_CARE,	"bcm43013c1_ag",	""},
+	{BCM4334_CHIP_ID,	3,	DONT_CARE,	"bcm4334b1_ag",		""},
+	{BCM43340_CHIP_ID,	2,	DONT_CARE,	"bcm43341b0_ag",	""},
+	{BCM43341_CHIP_ID,	2,	DONT_CARE,	"bcm43341b0_ag",	""},
+	{BCM4324_CHIP_ID,	5,	DONT_CARE,	"bcm43241b4_ag",	""},
+	{BCM4335_CHIP_ID,	2,	DONT_CARE,	"bcm4339a0_ag",		""},
+	{BCM4339_CHIP_ID,	1,	DONT_CARE,	"bcm4339a0_ag",		""},
+	{BCM4345_CHIP_ID,	6,	DONT_CARE,	"bcm43455c0_ag",	""},
+	{BCM43454_CHIP_ID,	6,	DONT_CARE,	"bcm43455c0_ag",	""},
+	{BCM4345_CHIP_ID,	9,	DONT_CARE,	"bcm43456c5_ag",	""},
+	{BCM43454_CHIP_ID,	9,	DONT_CARE,	"bcm43456c5_ag",	""},
+	{BCM4354_CHIP_ID,	1,	DONT_CARE,	"bcm4354a1_ag",		""},
+	{BCM4354_CHIP_ID,	2,	DONT_CARE,	"bcm4356a2_ag",		""},
+	{BCM4356_CHIP_ID,	2,	DONT_CARE,	"bcm4356a2_ag",		""},
+	{BCM4371_CHIP_ID,	2,	DONT_CARE,	"bcm4356a2_ag",		""},
+	{BCM43569_CHIP_ID,	3,	DONT_CARE,	"bcm4358a3_ag",		""},
+	{BCM4359_CHIP_ID,	5,	DONT_CARE,	"bcm4359b1_ag",		""},
+	{BCM4359_CHIP_ID,	9,	DONT_CARE,	"bcm4359c0_ag",		""},
+	{BCM43751_CHIP_ID,	1,	DONT_CARE,	"bcm43751a1_ag",	""},
+	{BCM43751_CHIP_ID,	2,	DONT_CARE,	"bcm43751a2_ag",	""},
+	{BCM43752_CHIP_ID,	1,	DONT_CARE,	"bcm43752a1_ag",	""},
+	{BCM43752_CHIP_ID,	2,	DONT_CARE,	"bcm43752a2_ag",	""},
+#endif
+#ifdef BCMPCIE
+	{BCM4354_CHIP_ID,	2,	DONT_CARE,	"bcm4356a2_pcie_ag",	""},
+	{BCM4356_CHIP_ID,	2,	DONT_CARE,	"bcm4356a2_pcie_ag",	""},
+	{BCM4359_CHIP_ID,	9,	DONT_CARE,	"bcm4359c0_pcie_ag",	""},
+	{BCM43751_CHIP_ID,	1,	DONT_CARE,	"bcm43751a1_pcie_ag",	""},
+	{BCM43751_CHIP_ID,	2,	DONT_CARE,	"bcm43751a2_pcie_ag",	""},
+	{BCM43752_CHIP_ID,	1,	DONT_CARE,	"bcm43752a1_pcie_ag",	""},
+	{BCM43752_CHIP_ID,	2,	DONT_CARE,	"bcm43752a2_pcie_ag",	""},
+	{BCM4375_CHIP_ID,	5,	DONT_CARE,	"bcm4375b4_pcie_ag",	""},
+#endif
+#ifdef BCMDBUS
+	{BCM43143_CHIP_ID,	2,	DONT_CARE,	"bcm43143b0",			""},
+	{BCM43242_CHIP_ID,	1,	DONT_CARE,	"bcm43242a1_ag",		""},
+	{BCM43569_CHIP_ID,	2,	DONT_CARE,	"bcm4358u_ag",			""},
+#endif
+};
+
+typedef struct module_name_map_v2_t {
+	uint chip;
+	uint chiprev;
+	char *module_name;
+} module_name_map_v2_t;
+
+const module_name_map_v2_t module_name_map_v2[] = {
+#ifdef BCMSDIO
+	{BCM4381_CHIP_ID,	0,	""},
+	{BCM4381_CHIP_ID,	1,	""},
+	{BCM43752_CHIP_ID,	4,	""},
+	{BCM43756_CHIP_ID,	4,	""},
+	{BCM43756_CHIP_ID,	6,	""},
+	{BCM43711_CHIP_ID,	0,	""},
+	{BCM4382_CHIP_ID,	3,	""},
+#endif
+#ifdef BCMPCIE
+	{BCM4381_CHIP_ID,	0,	""},
+	{BCM4381_CHIP_ID,	1,	""},
+	{BCM43752_CHIP_ID,	4,	""},
+	{BCM43756_CHIP_ID,	4,	""},
+	{BCM43756_CHIP_ID,	6,	""},
+	{BCM4382_CHIP_ID,	3,	""},
+#endif
+#ifdef BCMDBUS
+	{BCM4381_CHIP_ID,	0,	""},
+	{BCM4381_CHIP_ID,	1,	""},
+	{BCM4382_CHIP_ID,	3,	""},
+#endif
+};
+
+typedef struct chip_name_map_v2_t {
+	uint chip;
+	uint chiprev;
+	char *chip_name;
+} chip_name_map_v2_t;
+
+const chip_name_map_v2_t chip_name_map_v2[] = {
+	{BCM4381_CHIP_ID,	0,	"syn4381a0"},
+	{BCM4381_CHIP_ID,	1,	"syn4381a0"},
+	{BCM43752_CHIP_ID,	4,	"syn43756b0"},
+	{BCM43756_CHIP_ID,	4,	"syn43756b0"},
+	{BCM43756_CHIP_ID,	6,	"syn43756c0"},
+	{BCM43711_CHIP_ID,	0,	"syn43711a0"},
+	{BCM4382_CHIP_ID,	3,	"syn4382a0"},
+};
+
+#ifdef UPDATE_MODULE_NAME
+typedef void (compat_func_t)(dhd_pub_t *dhd);
+typedef struct module_name_map_t {
+	uint devid;
+	uint chip;
+	uint chiprev;
+	uint svid;
+	uint ssid;
+	char *module_name;
+	char *chip_name;
+	compat_func_t *compat_func;
+} module_name_map_t;
+
+#if defined(BCMSDIO) || defined(BCMPCIE)
+static void dhd_conf_compat_vht(dhd_pub_t *dhd);
+#endif
+
+const module_name_map_t module_name_map[] = {
+	/* Devce ID			Chip ID			Chiprev	SVID	SSID
+	 *  ModuleName		ChipName			Compat function
+	 */
+#ifdef BCMSDIO
+	{BCM43751_CHIP_ID,	BCM43752_CHIP_ID,	2,	0, 0,
+		"ap6398s2",		"bcm4359c51a2_ag",	dhd_conf_compat_vht},
+	{BCM43751_CHIP_ID,	BCM43752_CHIP_ID,	2,	0, 0,
+		"ap6398sr32",	"bcm4359c51a2_ag",	dhd_conf_compat_vht},
+	{BCM43751_CHIP_ID,	BCM43752_CHIP_ID,	2,	0, 0,
+		"ap6398sv",		"bcm4359c51a2_ag",	dhd_conf_compat_vht},
+	{BCM43751_CHIP_ID,	BCM43752_CHIP_ID,	2,	0, 0,
+		"ap6398sv3",	"bcm4359c51a2_ag",	dhd_conf_compat_vht},
+#endif
+#ifdef BCMPCIE
+	{BCM43751_D11AX_ID,	BCM43752_CHIP_ID,	2,	0x179F, 0x003C,
+		"ap6398p2",		"bcm4359c51a2_pcie_ag",	dhd_conf_compat_vht},
+	{BCM43751_D11AX_ID,	BCM43752_CHIP_ID,	2,	0x17F9, 0x003C,
+		"ap6398p2",		"bcm4359c51a2_pcie_ag",	dhd_conf_compat_vht},
+	{BCM43751_D11AX_ID,	BCM43752_CHIP_ID,	2,	0x17F9, 0x003D,
+		"ap6398pr32",	"bcm4359c51a2_pcie_ag",	dhd_conf_compat_vht},
+	{BCM43751_D11AX_ID,	BCM43752_CHIP_ID,	2,	0x17F9, 0x003E,
+		"ap6398pv",		"bcm4359c51a2_pcie_ag",	dhd_conf_compat_vht},
+	{BCM43751_D11AX_ID,	BCM43752_CHIP_ID,	2,	0x17F9, 0x003F,
+		"ap6398pv3",	"bcm4359c51a2_pcie_ag",	dhd_conf_compat_vht},
+#endif
+};
+#endif
+
+#ifdef BCMPCIE
+typedef struct chip_cisaddr_map_t {
+	uint chip;
+	uint chiprev;
+	uint start_addr;
+	uint end_addr;
+} chip_cisaddr_map_t;
+const chip_cisaddr_map_t chip_cisaddr_map[] = {
+	/* ChipID			Chiprev	Start	 	End  */
+	{BCM4354_CHIP_ID,	2,		0x0,		0x0},
+	{BCM4356_CHIP_ID,	2,		0x0,		0x0},
+	{BCM4359_CHIP_ID,	9,		0x0,		0x0},
+//	{BCM43752_CHIP_ID,	2,		0x18011120,	0x18011177},
+//	{BCM4375_CHIP_ID,	5,		0x18011120,	0x18011177},
+};
+#endif
+
+#ifdef DHD_TPUT_PATCH
+extern int dhd_change_mtu(dhd_pub_t *dhd, int new_mtu, int ifidx);
+#endif
+
+#ifdef WL_CFG80211
+bool
+dhd_conf_legacy_chip_check(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+
+	if (chip == BCM43362_CHIP_ID || chip == BCM4330_CHIP_ID ||
+			chip == BCM4334_CHIP_ID || chip == BCM43340_CHIP_ID ||
+			chip == BCM43341_CHIP_ID || chip == BCM4324_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4371_CHIP_ID ||
+			chip == BCM43430_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM43454_CHIP_ID ||
+			chip == BCM4359_CHIP_ID ||
+			chip == BCM43143_CHIP_ID || chip == BCM43242_CHIP_ID ||
+			chip == BCM43569_CHIP_ID) {
+		return true;
+	}
+
+	return false;
+}
+
+bool
+dhd_conf_new_chip_check(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+
+	if (chip == BCM43362_CHIP_ID || chip == BCM4330_CHIP_ID ||
+			chip == BCM4334_CHIP_ID || chip == BCM43340_CHIP_ID ||
+			chip == BCM43341_CHIP_ID || chip == BCM4324_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4371_CHIP_ID ||
+			chip == BCM43430_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM43454_CHIP_ID ||
+			chip == BCM43143_CHIP_ID || chip == BCM43242_CHIP_ID ||
+			chip == BCM43569_CHIP_ID) {
+		return false;
+	}
+
+	return true;
+}
+
+bool
+dhd_conf_extsae_chip(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+
+	if (chip == BCM43362_CHIP_ID || chip == BCM4330_CHIP_ID ||
+			chip == BCM4334_CHIP_ID || chip == BCM43340_CHIP_ID ||
+			chip == BCM43341_CHIP_ID || chip == BCM4324_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM43143_CHIP_ID || chip == BCM43242_CHIP_ID ||
+			chip == BCM43569_CHIP_ID) {
+		return false;
+	}
+
+	return true;
+}
+#endif
+
+#ifdef BCMSDIO
+static void
+dhd_conf_disable_slpauto(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+
+	if (chip == BCM43362_CHIP_ID || chip == BCM4330_CHIP_ID ||
+			chip == BCM4334_CHIP_ID || chip == BCM43340_CHIP_ID ||
+			chip == BCM43341_CHIP_ID || chip == BCM4324_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4371_CHIP_ID ||
+			chip == BCM43430_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM43454_CHIP_ID ||
+			chip == BCM4359_CHIP_ID) {
+		dhd_slpauto = FALSE;
+	}
+	CONFIG_MSG("dhd_slpauto = %d\n", dhd_slpauto);
+}
+#endif
+
+void
+dhd_conf_free_chip_nv_path_list(wl_chip_nv_path_list_ctrl_t *chip_nv_list)
+{
+	CONFIG_TRACE("called\n");
+
+	if (chip_nv_list->m_chip_nv_path_head) {
+		CONFIG_TRACE("Free %p\n", chip_nv_list->m_chip_nv_path_head);
+		kfree(chip_nv_list->m_chip_nv_path_head);
+		chip_nv_list->m_chip_nv_path_head = NULL;
+	}
+	chip_nv_list->count = 0;
+}
+
+#if defined(BCMSDIO) || defined(BCMPCIE)
+typedef struct cis_tuple_format {
+	uint8	id;
+	uint8	len;	/* total length of tag and data */
+	uint8	tag;
+	uint8	data[1];
+} cis_tuple_format_t;
+#define SBSDIO_CIS_SIZE_LIMIT		0x200
+#define SBSDIO_TUPLE_SIZE_LIMIT		0xff
+#define CIS_TUPLE_ID_BRCM			0x80
+#define CIS_TUPLE_TAG_MACADDR		0x19
+#define CIS_TUPLE_ID_AMPAK			0x8e
+#define CIS_TUPLE_TAG_MODULE		0x41
+#define CIS_TUPLE_LENGTH		1
+#define CIS_TUPLE_HDR_LEN		2
+#endif
+
+#ifdef BCMSDIO
+#if defined(HW_OOB) || defined(FORCE_WOWLAN)
+void
+dhd_conf_set_hw_oob_intr(bcmsdh_info_t *sdh, struct si_pub *sih)
+{
+	uint32 gpiocontrol, addr;
+
+	if (CHIPID(sih->chip) == BCM43362_CHIP_ID) {
+		CONFIG_MSG("Enable HW OOB for 43362\n");
+		addr = SI_ENUM_BASE(sih) + OFFSETOF(chipcregs_t, gpiocontrol);
+		gpiocontrol = bcmsdh_reg_read(sdh, addr, 4);
+		gpiocontrol |= 0x2;
+		bcmsdh_reg_write(sdh, addr, 4, gpiocontrol);
+		bcmsdh_cfg_write(sdh, SDIO_FUNC_1, 0x10005, 0xf, NULL);
+		bcmsdh_cfg_write(sdh, SDIO_FUNC_1, 0x10006, 0x0, NULL);
+		bcmsdh_cfg_write(sdh, SDIO_FUNC_1, 0x10007, 0x2, NULL);
+	}
+}
+#endif
+
+void
+dhd_conf_get_otp(dhd_pub_t *dhd, bcmsdh_info_t *sdh, si_t *sih)
+{
+	int i, err = -1;
+	uint8 *cis, *ptr = 0;
+	uint8 mac_header[3] = {0x80, 0x07, 0x19};
+	cis_tuple_format_t *tuple;
+	int totlen, len;
+
+	if (!(cis = MALLOC(dhd->osh, SBSDIO_CIS_SIZE_LIMIT))) {
+		CONFIG_ERROR("cis malloc failed\n");
+	}
+	bzero(cis, SBSDIO_CIS_SIZE_LIMIT);
+
+	if ((err = bcmsdh_cis_read(sdh, 0, cis, SBSDIO_CIS_SIZE_LIMIT))) {
+		CONFIG_ERROR("cis read err %d\n", err);
+		MFREE(dhd->osh, cis, SBSDIO_CIS_SIZE_LIMIT);
+		return;
+	}
+	tuple = (cis_tuple_format_t *)cis;
+	totlen = SBSDIO_CIS_SIZE_LIMIT;
+	if (config_msg_level & CONFIG_TRACE_LEVEL) {
+		prhex("CIS", &tuple->id, totlen);
+	}
+	while (totlen >= (tuple->len + CIS_TUPLE_HDR_LEN)) {
+		len = tuple->len;
+		if ((config_msg_level & CONFIG_TRACE_LEVEL) && tuple->id) {
+			prhex("TPL", &tuple->id, tuple->len + CIS_TUPLE_HDR_LEN);
+		}
+		if (tuple->id == 0xff || tuple->len == 0xff)
+			break;
+		if ((tuple->id == CIS_TUPLE_ID_BRCM) &&
+				(tuple->tag == CIS_TUPLE_TAG_MACADDR) &&
+				(totlen >= (int)(len + CIS_TUPLE_HDR_LEN))) {
+			memcpy(&dhd->conf->otp_mac, tuple->data, ETHER_ADDR_LEN);
+		}
+#ifdef GET_OTP_MODULE_NAME
+		else if (tuple->id == CIS_TUPLE_ID_AMPAK && (tuple->len) &&
+				tuple->tag == CIS_TUPLE_TAG_MODULE) {
+			int len = tuple->len - 1;
+			if (len <= sizeof(dhd->conf->module_name) - 1) {
+				strncpy(dhd->conf->module_name, tuple->data, len);
+				CONFIG_MSG("module_name=%s\n", dhd->conf->module_name);
+			} else {
+				CONFIG_ERROR("len is too long %d >= %d\n",
+					len, (int)sizeof(dhd->conf->module_name) - 1);
+			}
+		}
+#endif
+		tuple = (cis_tuple_format_t*)((uint8*)tuple + (len + CIS_TUPLE_HDR_LEN));
+		totlen -= (len + CIS_TUPLE_HDR_LEN);
+	}
+
+	if (!memcmp(&ether_null, &dhd->conf->otp_mac, ETHER_ADDR_LEN)) {
+		ptr = cis;
+		/* Special OTP */
+		if (bcmsdh_reg_read(sdh, SI_ENUM_BASE(sih), 4) == 0x16044330) {
+			for (i=0; i<SBSDIO_CIS_SIZE_LIMIT; i++) {
+				if (!memcmp(mac_header, ptr, 3)) {
+					memcpy(&dhd->conf->otp_mac, ptr+3, ETHER_ADDR_LEN);
+					break;
+				}
+				ptr++;
+			}
+		}
+	}
+
+	ASSERT(cis);
+	MFREE(dhd->osh, cis, SBSDIO_CIS_SIZE_LIMIT);
+}
+
+#ifdef SET_FWNV_BY_MAC
+void
+dhd_conf_free_mac_list(wl_mac_list_ctrl_t *mac_list)
+{
+	int i;
+
+	CONFIG_TRACE("called\n");
+	if (mac_list->m_mac_list_head) {
+		for (i=0; i<mac_list->count; i++) {
+			if (mac_list->m_mac_list_head[i].mac) {
+				CONFIG_TRACE("Free mac %p\n", mac_list->m_mac_list_head[i].mac);
+				kfree(mac_list->m_mac_list_head[i].mac);
+			}
+		}
+		CONFIG_TRACE("Free m_mac_list_head %p\n", mac_list->m_mac_list_head);
+		kfree(mac_list->m_mac_list_head);
+	}
+	mac_list->count = 0;
+}
+
+void
+dhd_conf_set_fw_name_by_mac(dhd_pub_t *dhd, char *fw_path)
+{
+	int i, j;
+	uint8 *mac = (uint8 *)&dhd->conf->otp_mac;
+	int fw_num=0, mac_num=0;
+	uint32 oui, nic;
+	wl_mac_list_t *mac_list;
+	wl_mac_range_t *mac_range;
+	int fw_type, fw_type_new;
+	char *name_ptr;
+
+	mac_list = dhd->conf->fw_by_mac.m_mac_list_head;
+	fw_num = dhd->conf->fw_by_mac.count;
+	if (!mac_list || !fw_num)
+		return;
+
+	oui = (mac[0] << 16) | (mac[1] << 8) | (mac[2]);
+	nic = (mac[3] << 16) | (mac[4] << 8) | (mac[5]);
+
+	/* find out the last '/' */
+	i = strlen(fw_path);
+	while (i > 0) {
+		if (fw_path[i] == '/') {
+			i++;
+			break;
+		}
+		i--;
+	}
+	name_ptr = &fw_path[i];
+
+	if (strstr(name_ptr, "_apsta"))
+		fw_type = FW_TYPE_APSTA;
+	else if (strstr(name_ptr, "_p2p"))
+		fw_type = FW_TYPE_P2P;
+	else if (strstr(name_ptr, "_mesh"))
+		fw_type = FW_TYPE_MESH;
+	else if (strstr(name_ptr, "_ezmesh"))
+		fw_type = FW_TYPE_EZMESH;
+	else if (strstr(name_ptr, "_es"))
+		fw_type = FW_TYPE_ES;
+	else if (strstr(name_ptr, "_mfg"))
+		fw_type = FW_TYPE_MFG;
+	else
+		fw_type = FW_TYPE_STA;
+
+	for (i=0; i<fw_num; i++) {
+		mac_num = mac_list[i].count;
+		mac_range = mac_list[i].mac;
+		if (strstr(mac_list[i].name, "_apsta"))
+			fw_type_new = FW_TYPE_APSTA;
+		else if (strstr(mac_list[i].name, "_p2p"))
+			fw_type_new = FW_TYPE_P2P;
+		else if (strstr(mac_list[i].name, "_mesh"))
+			fw_type_new = FW_TYPE_MESH;
+		else if (strstr(mac_list[i].name, "_ezmesh"))
+			fw_type_new = FW_TYPE_EZMESH;
+		else if (strstr(mac_list[i].name, "_es"))
+			fw_type_new = FW_TYPE_ES;
+		else if (strstr(mac_list[i].name, "_mfg"))
+			fw_type_new = FW_TYPE_MFG;
+		else
+			fw_type_new = FW_TYPE_STA;
+		if (fw_type != fw_type_new) {
+			CONFIG_MSG("fw_typ=%d != fw_type_new=%d\n", fw_type, fw_type_new);
+			continue;
+		}
+		for (j=0; j<mac_num; j++) {
+			if (oui == mac_range[j].oui) {
+				if (nic >= mac_range[j].nic_start && nic <= mac_range[j].nic_end) {
+					strcpy(name_ptr, mac_list[i].name);
+					CONFIG_MSG("matched oui=0x%06X, nic=0x%06X\n", oui, nic);
+					CONFIG_MSG("fw_path=%s\n", fw_path);
+					return;
+				}
+			}
+		}
+	}
+}
+
+void
+dhd_conf_set_nv_name_by_mac(dhd_pub_t *dhd, char *nv_path)
+{
+	int i, j;
+	uint8 *mac = (uint8 *)&dhd->conf->otp_mac;
+	int nv_num=0, mac_num=0;
+	uint32 oui, nic;
+	wl_mac_list_t *mac_list;
+	wl_mac_range_t *mac_range;
+	char *pnv_name;
+
+	mac_list = dhd->conf->nv_by_mac.m_mac_list_head;
+	nv_num = dhd->conf->nv_by_mac.count;
+	if (!mac_list || !nv_num)
+		return;
+
+	oui = (mac[0] << 16) | (mac[1] << 8) | (mac[2]);
+	nic = (mac[3] << 16) | (mac[4] << 8) | (mac[5]);
+
+	/* find out the last '/' */
+	i = strlen(nv_path);
+	while (i > 0) {
+		if (nv_path[i] == '/') break;
+		i--;
+	}
+	pnv_name = &nv_path[i+1];
+
+	for (i=0; i<nv_num; i++) {
+		mac_num = mac_list[i].count;
+		mac_range = mac_list[i].mac;
+		for (j=0; j<mac_num; j++) {
+			if (oui == mac_range[j].oui) {
+				if (nic >= mac_range[j].nic_start && nic <= mac_range[j].nic_end) {
+					strcpy(pnv_name, mac_list[i].name);
+					CONFIG_MSG("matched oui=0x%06X, nic=0x%06X\n", oui, nic);
+					CONFIG_MSG("nv_path=%s\n", nv_path);
+					return;
+				}
+			}
+		}
+	}
+}
+#endif
+
+bool
+dhd_conf_legacy_otp_chip(dhd_pub_t *dhd)
+{
+	uint chip;
+
+	chip = dhd->conf->chip;
+
+	if (chip == BCM43362_CHIP_ID || chip == BCM4330_CHIP_ID ||
+			chip == BCM4334_CHIP_ID || chip == BCM43340_CHIP_ID ||
+			chip == BCM43341_CHIP_ID || chip == BCM4324_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4371_CHIP_ID ||
+			chip == BCM43430_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM43454_CHIP_ID ||
+			chip == BCM4359_CHIP_ID || chip == BCM43012_CHIP_ID ||
+			chip == BCM43751_CHIP_ID || chip == BCM43752_CHIP_ID ||
+			chip == BCM43756_CHIP_ID || chip == BCM43711_CHIP_ID) {
+		return true;
+	}
+
+	return false;
+}
+#endif
+
+#ifdef BCMPCIE
+static int
+dhd_conf_read_otp_from_bp(si_t *sih, uint32 *data_buf,
+	uint32 cis_start_addr, uint32 cis_max_cnt)
+{
+	int int_val = 0, i = 0, bp_idx = 0;
+	int boardtype_backplane_addr[] = {
+		0x18010324, /* OTP Control 1 */
+		0x18012618, /* PMU min resource mask */
+	};
+	int boardtype_backplane_data[] = {
+		0x00fa0000,
+		0x0e4fffff /* Keep on ARMHTAVAIL */
+	};
+	uint32 org_boardtype_backplane_data[] = {
+		0,
+		0
+	};
+
+	for (bp_idx=0; bp_idx<ARRAYSIZE(boardtype_backplane_addr); bp_idx++) {
+		/* Read OTP Control 1 and PMU min_rsrc_mask before writing */
+		if (si_backplane_access(sih, boardtype_backplane_addr[bp_idx], sizeof(int),
+				&org_boardtype_backplane_data[bp_idx], TRUE) != BCME_OK) {
+			CONFIG_ERROR("invalid size/addr combination\n");
+			return BCME_ERROR;
+		}
+
+		/* Write new OTP and PMU configuration */
+		if (si_backplane_access(sih, boardtype_backplane_addr[bp_idx], sizeof(int),
+				&boardtype_backplane_data[bp_idx], FALSE) != BCME_OK) {
+			CONFIG_ERROR("invalid size/addr combination\n");
+			return BCME_ERROR;
+		}
+
+		if (si_backplane_access(sih, boardtype_backplane_addr[bp_idx], sizeof(int),
+				&int_val, TRUE) != BCME_OK) {
+			CONFIG_ERROR("invalid size/addr combination\n");
+			return BCME_ERROR;
+		}
+
+		CONFIG_TRACE("boardtype_backplane_addr 0x%08x rdata 0x%04x\n",
+			boardtype_backplane_addr[bp_idx], int_val);
+	}
+
+	/* read tuple raw data */
+	for (i=0; i<cis_max_cnt; i++) {
+		if (si_backplane_access(sih, cis_start_addr + i * sizeof(uint32),
+				sizeof(uint32),	&data_buf[i], TRUE) != BCME_OK) {
+			break;
+		}
+		CONFIG_TRACE("tuple index %d, raw data 0x%08x\n", i,  data_buf[i]);
+	}
+
+	for (bp_idx=0; bp_idx<ARRAYSIZE(boardtype_backplane_addr); bp_idx++) {
+		/* Write original OTP and PMU configuration */
+		if (si_backplane_access(sih, boardtype_backplane_addr[bp_idx], sizeof(int),
+				&org_boardtype_backplane_data[bp_idx], FALSE) != BCME_OK) {
+			CONFIG_ERROR("invalid size/addr combination\n");
+			return BCME_ERROR;
+		}
+
+		if (si_backplane_access(sih, boardtype_backplane_addr[bp_idx], sizeof(int),
+				&int_val, TRUE) != BCME_OK) {
+			CONFIG_ERROR("invalid size/addr combination\n");
+			return BCME_ERROR;
+		}
+
+		CONFIG_TRACE("boardtype_backplane_addr 0x%08x rdata 0x%04x\n",
+			boardtype_backplane_addr[bp_idx], int_val);
+	}
+
+	return i * sizeof(uint32);
+}
+
+int
+dhd_conf_get_otp(dhd_pub_t *dhd, si_t *sih)
+{
+	int totlen, len;
+	uint32 *raw_data = NULL;
+	cis_tuple_format_t *tuple;
+	uint32 cis_start_addr = 0, cis_end_addr = 0, cis_max_cnt;
+	uint chip, chiprev;
+	int i, ret = BCME_OK;
+
+	chip = dhd->conf->chip;
+	chiprev = dhd->conf->chiprev;
+
+	for (i=0; i<sizeof(chip_cisaddr_map)/sizeof(chip_cisaddr_map[0]); i++) {
+		const chip_cisaddr_map_t* row = &chip_cisaddr_map[i];
+		if (row->chip == chip && row->chiprev == chiprev) {
+			cis_start_addr = row->start_addr;
+			cis_end_addr = row->end_addr;
+		}
+	}
+
+	if (!cis_start_addr || !cis_end_addr) {
+		CONFIG_TRACE("no matched chip\n");
+		goto exit;
+	}
+	cis_max_cnt = (cis_end_addr - cis_start_addr + 1) / sizeof(uint32);
+
+	raw_data = kmalloc(cis_max_cnt, GFP_KERNEL);
+	if (raw_data == NULL) {
+		CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", cis_max_cnt);
+		goto exit;
+	}
+
+	totlen = dhd_conf_read_otp_from_bp(sih, raw_data, cis_start_addr, cis_max_cnt);
+	if (totlen == BCME_ERROR || totlen == 0) {
+		CONFIG_ERROR("Can't read the OTP\n");
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	tuple = (cis_tuple_format_t *)raw_data;
+
+	if (config_msg_level & CONFIG_TRACE_LEVEL) {
+		CONFIG_TRACE("start: 0x%x, end: 0x%x, totlen: %d\n",
+			cis_start_addr, cis_end_addr, totlen);
+		prhex("CIS", &tuple->id, totlen);
+	}
+
+	/* check the first tuple has tag 'start' */
+	if (tuple->id != CIS_TUPLE_ID_BRCM) {
+		CONFIG_ERROR("Can not find the TAG\n");
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	/* find tagged parameter */
+	while (totlen >= (tuple->len + CIS_TUPLE_HDR_LEN)) {
+		len = tuple->len;
+		if ((config_msg_level & CONFIG_TRACE_LEVEL) && tuple->id) {
+			prhex("TPL", &tuple->id, tuple->len+CIS_TUPLE_HDR_LEN);
+		}
+		if ((tuple->id == CIS_TUPLE_ID_BRCM) &&
+				(tuple->tag == CIS_TUPLE_TAG_MACADDR) &&
+				(totlen >= (int)(len + CIS_TUPLE_HDR_LEN))) {
+			memcpy(&dhd->conf->otp_mac, tuple->data, ETHER_ADDR_LEN);
+		}
+		tuple = (cis_tuple_format_t*)((uint8*)tuple + (len + CIS_TUPLE_HDR_LEN));
+		totlen -= (len + CIS_TUPLE_HDR_LEN);
+	}
+
+exit:
+	if(raw_data)
+		kfree(raw_data);
+	return ret;
+}
+
+bool
+dhd_conf_legacy_msi_chip(dhd_pub_t *dhd)
+{
+	uint chip;
+
+	chip = dhd->conf->chip;
+
+	if (chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+		chip == BCM4371_CHIP_ID ||
+		chip == BCM4359_CHIP_ID) {
+		return true;
+	}
+
+	return false;
+}
+#endif
+
+void
+dhd_conf_free_country_list(struct dhd_conf *conf)
+{
+	country_list_t *country = conf->country_head;
+	int count = 0;
+
+	CONFIG_TRACE("called\n");
+	while (country) {
+		CONFIG_TRACE("Free cspec %s\n", country->cspec.country_abbrev);
+		conf->country_head = country->next;
+		kfree(country);
+		country = conf->country_head;
+		count++;
+	}
+	CONFIG_TRACE("%d country released\n", count);
+}
+
+void
+dhd_conf_free_mchan_list(struct dhd_conf *conf)
+{
+	mchan_params_t *mchan = conf->mchan;
+	int count = 0;
+
+	CONFIG_TRACE("called\n");
+	while (mchan) {
+		CONFIG_TRACE("Free cspec %p\n", mchan);
+		conf->mchan = mchan->next;
+		kfree(mchan);
+		mchan = conf->mchan;
+		count++;
+	}
+	CONFIG_TRACE("%d mchan released\n", count);
+}
+
+const chip_name_map_t*
+dhd_conf_match_chip(dhd_pub_t *dhd, uint ag_type)
+{
+	uint chip, chiprev;
+	int i;
+
+	chip = dhd->conf->chip;
+	chiprev = dhd->conf->chiprev;
+
+	for (i=0; i<sizeof(chip_name_map)/sizeof(chip_name_map[0]); i++) {
+		const chip_name_map_t *row = &chip_name_map[i];
+		if (row->chip == chip && row->chiprev == chiprev &&
+				(row->ag_type == ag_type ||
+					ag_type == DONT_CARE || row->ag_type == DONT_CARE)) {
+			return row;
+		}
+	}
+
+	return NULL;
+}
+
+const chip_name_map_v2_t*
+dhd_conf_match_chip_v2(dhd_pub_t *dhd)
+{
+	uint chip, chiprev;
+	int i;
+
+	chip = dhd->conf->chip;
+	chiprev = dhd->conf->chiprev;
+
+	for (i=0; i<sizeof(chip_name_map_v2)/sizeof(chip_name_map_v2[0]); i++) {
+		const chip_name_map_v2_t *row = &chip_name_map_v2[i];
+		if (row->chip == chip && row->chiprev == chiprev) {
+			return row;
+		}
+	}
+
+	return NULL;
+}
+
+const module_name_map_v2_t*
+dhd_conf_match_module_v2(dhd_pub_t *dhd)
+{
+	uint chip, chiprev;
+	int i;
+
+	chip = dhd->conf->chip;
+	chiprev = dhd->conf->chiprev;
+
+	for (i=0; i<sizeof(module_name_map_v2)/sizeof(module_name_map_v2[0]); i++) {
+		const module_name_map_v2_t *row = &module_name_map_v2[i];
+		if (row->chip == chip && row->chiprev == chiprev) {
+			return row;
+		}
+	}
+
+	return NULL;
+}
+
+#ifdef UPDATE_MODULE_NAME
+const module_name_map_t*
+dhd_conf_match_module(dhd_pub_t *dhd)
+{
+	uint devid, chip, chiprev;
+#ifdef BCMPCIE
+	uint svid, ssid;
+#endif
+#if defined(BCMSDIO) || defined(BCMPCIE)
+	int i;
+#endif
+
+	devid = dhd->conf->devid;
+	chip = dhd->conf->chip;
+	chiprev = dhd->conf->chiprev;
+#ifdef BCMPCIE
+	svid = dhd->conf->svid;
+	ssid = dhd->conf->ssid;
+#endif
+
+#ifdef BCMSDIO
+	for (i=0; i<sizeof(module_name_map)/sizeof(module_name_map[0]); i++) {
+		const module_name_map_t *row = &module_name_map[i];
+		if (row->devid == devid && row->chip == chip && row->chiprev == chiprev &&
+				!strcmp(row->module_name, dhd->conf->module_name)) {
+			return row;
+		}
+	}
+#endif
+
+#ifdef BCMPCIE
+	for (i=0; i<sizeof(module_name_map)/sizeof(module_name_map[0]); i++) {
+		const module_name_map_t *row = &module_name_map[i];
+		if (row->devid == devid && row->chip == chip && row->chiprev == chiprev &&
+				row->svid == svid && row->ssid == ssid) {
+			return row;
+		}
+	}
+#endif
+
+	return NULL;
+}
+#endif
+
+char *
+dhd_conf_get_chip_name(dhd_pub_t *dhd, int ag_type, bool *chip_map_v2)
+{
+#ifdef UPDATE_MODULE_NAME
+	const module_name_map_t *row_module = NULL;
+#endif
+	const chip_name_map_t *row_chip = NULL;
+	const chip_name_map_v2_t *row_chip_v2 = NULL;
+	char *name = NULL;
+
+	*chip_map_v2 = FALSE;
+#ifdef UPDATE_MODULE_NAME
+	row_module = dhd_conf_match_module(dhd);
+	if (row_module && strlen(row_module->chip_name)) {
+		name = row_module->chip_name;
+	} else
+#endif
+	{
+		row_chip = dhd_conf_match_chip(dhd, ag_type);
+		if (row_chip && strlen(row_chip->chip_name)) {
+			name = row_chip->chip_name;
+		}
+		else {
+			row_chip_v2 = dhd_conf_match_chip_v2(dhd);
+			if (row_chip_v2 && strlen(row_chip_v2->chip_name)) {
+				name = row_chip_v2->chip_name;
+				*chip_map_v2 = TRUE;
+			}
+		}
+	}
+
+	return name;
+}
+
+char *
+dhd_conf_get_module_name(dhd_pub_t *dhd, int ag_type)
+{
+#if defined(BCMPCIE) && defined(UPDATE_MODULE_NAME)
+	const module_name_map_t *row_module = NULL;
+#endif
+	const chip_name_map_t *row_chip = NULL;
+	const module_name_map_v2_t *row_module_v2 = NULL;
+	char *name = NULL;
+
+#if defined(BCMPCIE) && defined(UPDATE_MODULE_NAME)
+	row_module = dhd_conf_match_module(dhd);
+	if (row_module && strlen(row_module->module_name)) {
+		name = row_module->module_name;
+	} else
+#endif
+	{
+		row_chip = dhd_conf_match_chip(dhd, ag_type);
+		if (row_chip && strlen(row_chip->module_name)) {
+			name = row_chip->module_name;
+		}
+		else {
+			row_module_v2 = dhd_conf_match_module_v2(dhd);
+			if (row_module_v2 && strlen(row_module_v2->module_name)) {
+				name = row_module_v2->module_name;
+			}
+		}
+	}
+
+	return name;
+}
+
+int
+dhd_conf_set_fw_name_by_chip(dhd_pub_t *dhd, char *fw_path)
+{
+	int fw_type, ag_type;
+	char *name_ptr, *chip_name = NULL;
+	bool chip_map_v2;
+	int i;
+
+	if (fw_path[0] == '\0') {
+#ifdef CONFIG_BCMDHD_FW_PATH
+		bcm_strncpy_s(fw_path, MOD_PARAM_PATHLEN-1, CONFIG_BCMDHD_FW_PATH, MOD_PARAM_PATHLEN-1);
+		if (fw_path[0] == '\0')
+#endif
+		{
+			CONFIG_MSG("firmware path is null\n");
+			return 0;
+		}
+	}
+#ifndef FW_PATH_AUTO_SELECT
+	return DONT_CARE;
+#endif
+
+	/* find out the last '/' */
+	i = strlen(fw_path);
+	while (i > 0) {
+		if (fw_path[i] == '/') {
+			i++;
+			break;
+		}
+		i--;
+	}
+	name_ptr = &fw_path[i];
+#ifdef BAND_AG
+	ag_type = FW_TYPE_AG;
+#else
+	ag_type = strstr(name_ptr, "_ag") ? FW_TYPE_AG : FW_TYPE_G;
+#endif
+	if (strstr(name_ptr, "_apsta"))
+		fw_type = FW_TYPE_APSTA;
+	else if (strstr(name_ptr, "_p2p"))
+		fw_type = FW_TYPE_P2P;
+	else if (strstr(name_ptr, "_mesh"))
+		fw_type = FW_TYPE_MESH;
+	else if (strstr(name_ptr, "_ezmesh"))
+		fw_type = FW_TYPE_EZMESH;
+	else if (strstr(name_ptr, "_es"))
+		fw_type = FW_TYPE_ES;
+	else if (strstr(name_ptr, "_mfg"))
+		fw_type = FW_TYPE_MFG;
+	else if (strstr(name_ptr, "_minime"))
+		fw_type = FW_TYPE_MINIME;
+	else
+		fw_type = FW_TYPE_STA;
+#ifdef WLEASYMESH
+	if (dhd->conf->fw_type == FW_TYPE_EZMESH)
+		fw_type = FW_TYPE_EZMESH;
+#endif /* WLEASYMESH */
+
+	chip_name = dhd_conf_get_chip_name(dhd, ag_type, &chip_map_v2);
+	if (chip_name) {
+		strcpy(name_ptr, "fw_");
+		strcat(name_ptr, chip_name);
+		if (chip_map_v2) {
+#ifdef BCMSDIO
+			strcat(name_ptr, "_sdio");
+#endif
+#ifdef BCMPCIE
+			strcat(name_ptr, "_pcie");
+#endif
+#if defined(BCMDBUS) && !defined(BCMUSBDEV_COMPOSITE)
+			strcat(name_ptr, "_usb");
+#endif
+		}
+#ifdef BCMUSBDEV_COMPOSITE
+		strcat(name_ptr, "_cusb");
+#endif
+		if (fw_type == FW_TYPE_APSTA)
+			strcat(name_ptr, "_apsta.bin");
+		else if (fw_type == FW_TYPE_P2P)
+			strcat(name_ptr, "_p2p.bin");
+		else if (fw_type == FW_TYPE_MESH)
+			strcat(name_ptr, "_mesh.bin");
+		else if (fw_type == FW_TYPE_EZMESH)
+			strcat(name_ptr, "_ezmesh.bin");
+		else if (fw_type == FW_TYPE_ES)
+			strcat(name_ptr, "_es.bin");
+		else if (fw_type == FW_TYPE_MFG)
+			strcat(name_ptr, "_mfg.bin");
+		else if (fw_type == FW_TYPE_MINIME)
+			strcat(name_ptr, "_minime.bin");
+		else
+			strcat(name_ptr, ".bin");
+	}
+
+	dhd->conf->fw_type = fw_type;
+
+#ifndef MINIME
+	if (fw_type == FW_TYPE_MINIME)
+		CONFIG_ERROR("***** Please enable MINIME in Makefile *****\n");
+#endif
+
+	CONFIG_TRACE("firmware_path=%s\n", fw_path);
+	return ag_type;
+}
+
+void
+dhd_conf_set_clm_name_by_chip(dhd_pub_t *dhd, char *clm_path, int ag_type)
+{
+	char *name_ptr, *chip_name = NULL;
+	bool chip_map_v2;
+	int i;
+
+	if (clm_path[0] == '\0') {
+		CONFIG_MSG("clm path is null\n");
+		return;
+	}
+
+	/* find out the last '/' */
+	i = strlen(clm_path);
+	while (i > 0) {
+		if (clm_path[i] == '/') {
+			i++;
+			break;
+		}
+		i--;
+	}
+	name_ptr = &clm_path[i];
+
+	chip_name = dhd_conf_get_chip_name(dhd, ag_type, &chip_map_v2);
+	if (chip_name) {
+		strcpy(name_ptr, "clm_");
+		strcat(name_ptr, chip_name);
+		strcat(name_ptr, ".blob");
+	}
+
+	CONFIG_TRACE("clm_path=%s\n", clm_path);
+}
+
+void
+dhd_conf_set_nv_name_by_chip(dhd_pub_t *dhd, char *nv_path, int ag_type)
+{
+	uint chip, chiprev;
+	char *name_ptr, *module_name = NULL, nv_name[32];
+	int i;
+
+	chip = dhd->conf->chip;
+	chiprev = dhd->conf->chiprev;
+
+	if (nv_path[0] == '\0') {
+#ifdef CONFIG_BCMDHD_NVRAM_PATH
+		bcm_strncpy_s(nv_path, MOD_PARAM_PATHLEN-1, CONFIG_BCMDHD_NVRAM_PATH, MOD_PARAM_PATHLEN-1);
+		if (nv_path[0] == '\0')
+#endif
+		{
+			CONFIG_MSG("nvram path is null\n");
+			return;
+		}
+	}
+
+	/* find out the last '/' */
+	i = strlen(nv_path);
+	while (i > 0) {
+		if (nv_path[i] == '/') {
+			i++;
+			break;
+		}
+		i--;
+	}
+	name_ptr = &nv_path[i];
+
+	module_name = dhd_conf_get_module_name(dhd, ag_type);
+	if (module_name) {
+		strcpy(name_ptr, "nvram_");
+		strcat(name_ptr, module_name);
+#ifdef BCMUSBDEV_COMPOSITE
+		strcat(name_ptr, "_cusb");
+#endif
+		strcat(name_ptr, ".txt");
+	}
+	strcpy(nv_name, name_ptr);
+
+#if defined(BCMSDIO) && defined(GET_OTP_MODULE_NAME)
+	if (strlen(dhd->conf->module_name)) {
+		strcpy(name_ptr, "nvram_");
+		strcat(name_ptr, dhd->conf->module_name);
+		strcat(name_ptr, ".txt");
+#ifdef COMPAT_OLD_MODULE
+		if (dhd->conf->chip == BCM4359_CHIP_ID) {
+			// compatible for AP6398S and AP6398SA
+#ifdef DHD_LINUX_STD_FW_API
+			const struct firmware *nv = NULL;
+			int ret = BCME_ERROR;
+			ret = dhd_os_get_img_fwreq(&nv, nv_path);
+			if (ret < 0) {
+				strcpy(name_ptr, nv_name);
+			}
+			if (nv) {
+				dhd_os_close_img_fwreq(nv);
+			}
+#else
+			struct file *fp;
+			fp = dhd_filp_open(nv_path, O_RDONLY, 0);
+			if (IS_ERR(fp) || (fp == NULL)) {
+				strcpy(name_ptr, nv_name);
+			} else {
+				dhd_filp_close((struct file *)fp, NULL);
+			}
+#endif
+		}
+#endif
+	}
+#endif
+
+	for (i=0; i<dhd->conf->nv_by_chip.count; i++) {
+		if (chip==dhd->conf->nv_by_chip.m_chip_nv_path_head[i].chip &&
+				chiprev==dhd->conf->nv_by_chip.m_chip_nv_path_head[i].chiprev) {
+			strcpy(name_ptr, dhd->conf->nv_by_chip.m_chip_nv_path_head[i].name);
+			break;
+		}
+	}
+
+	CONFIG_TRACE("nvram_path=%s\n", nv_path);
+}
+
+void
+dhd_conf_copy_path(dhd_pub_t *dhd, char *dst_name, char *dst_path, char *src_path)
+{
+	int i;
+
+	if (src_path[0] == '\0') {
+		CONFIG_MSG("src_path is null\n");
+		return;
+	} else
+		strcpy(dst_path, src_path);
+
+	/* find out the last '/' */
+	i = strlen(dst_path);
+	while (i > 0) {
+		if (dst_path[i] == '/') {
+			i++;
+			break;
+		}
+		i--;
+	}
+	strcpy(&dst_path[i], dst_name);
+
+	CONFIG_TRACE("dst_path=%s\n", dst_path);
+}
+
+#ifdef CONFIG_PATH_AUTO_SELECT
+void
+dhd_conf_set_conf_name_by_chip(dhd_pub_t *dhd, char *conf_path)
+{
+	char *name_ptr, *chip_name = NULL;
+	bool chip_map_v2;
+	int i;
+
+	if (conf_path[0] == '\0') {
+		CONFIG_MSG("config path is null\n");
+		return;
+	}
+
+	/* find out the last '/' */
+	i = strlen(conf_path);
+	while (i > 0) {
+		if (conf_path[i] == '/') {
+			i++;
+			break;
+		}
+		i--;
+	}
+	name_ptr = &conf_path[i];
+
+	chip_name = dhd_conf_get_chip_name(dhd, DONT_CARE, &chip_map_v2);
+	if (chip_name) {
+		strcpy(name_ptr, "config_");
+		strcat(name_ptr, chip_name);
+		strcat(name_ptr, ".txt");
+	}
+
+	CONFIG_TRACE("config_path=%s\n", conf_path);
+}
+#endif
+
+#ifdef TPUT_MONITOR
+void
+dhd_conf_tput_monitor(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	if (conf->tput_monitor_ms && conf->data_drop_mode >= FW_DROP) {
+		if (conf->tput_ts.tv_sec == 0 && conf->tput_ts.tv_nsec == 0) {
+			osl_do_gettimeofday(&conf->tput_ts);
+		} else {
+			struct osl_timespec cur_ts;
+			int32 tput_tx = 0, tput_rx = 0, tput_tx_kb = 0,
+				tput_rx_kb = 0, tput_net = 0, tput_net_kb = 0;
+			uint32 diff_ms;
+			unsigned long diff_bytes;
+			osl_do_gettimeofday(&cur_ts);
+			diff_ms = osl_do_gettimediff(&cur_ts, &conf->tput_ts)/1000;
+			if (diff_ms >= conf->tput_monitor_ms) {
+				diff_bytes = dhd->dstats.tx_bytes - conf->last_tx;
+				tput_tx = (int32)((diff_bytes/1024/1024)*8)*1000/diff_ms;
+				if (tput_tx == 0) {
+					tput_tx = (int32)(diff_bytes*8/1024/1024)*1000/diff_ms;
+					tput_tx_kb = (int32)(diff_bytes*8*1000/1024)/diff_ms;
+					tput_tx_kb = tput_tx_kb % 1000;
+				}
+				diff_bytes = dhd->dstats.rx_bytes - conf->last_rx;
+				tput_rx = (int32)((diff_bytes/1024/1024)*8)*1000/diff_ms;
+				if (tput_rx == 0) {
+					tput_rx = (int32)(diff_bytes*8/1024/1024)*1000/diff_ms;
+					tput_rx_kb = (int32)(diff_bytes*8*1000/1024)/diff_ms;
+					tput_rx_kb = tput_rx_kb % 1000;
+				}
+				diff_bytes = conf->net_len - conf->last_net_tx;
+				tput_net = (int32)((diff_bytes/1024/1024)*8)*1000/diff_ms;
+				if (tput_net == 0) {
+					tput_net = (int32)(diff_bytes*8/1024/1024)*1000/diff_ms;
+					tput_net_kb = (int32)(diff_bytes*8*1000/1024)/diff_ms;
+					tput_net_kb = tput_net_kb % 1000;
+				}
+				conf->last_tx = dhd->dstats.tx_bytes;
+				conf->last_rx = dhd->dstats.rx_bytes;
+				conf->last_net_tx = conf->net_len;
+				memcpy(&conf->tput_ts, &cur_ts, sizeof(struct osl_timespec));
+				CONFIG_TRACE("xmit=%3d.%d%d%d Mbps, tx=%3d.%d%d%d Mbps, rx=%3d.%d%d%d Mbps\n",
+					tput_net, (tput_net_kb/100)%10, (tput_net_kb/10)%10, (tput_net_kb)%10,
+					tput_tx, (tput_tx_kb/100)%10, (tput_tx_kb/10)%10, (tput_tx_kb)%10,
+					tput_rx, (tput_rx_kb/100)%10, (tput_rx_kb/10)%10, (tput_rx_kb)%10);
+			}
+		}
+	}
+}
+#endif
+
+#ifdef DHD_TPUT_PATCH
+void
+dhd_conf_set_tput_patch(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	if (conf->tput_patch) {
+		conf->mtu = 1500;
+/* set pktsetsum false by default since this will cause to
+  * the checksum is wrong of downloaded file
+*/
+		conf->pktsetsum = FALSE;
+#ifdef BCMSDIO
+		conf->dhd_dpc_prio = 98;
+/* need to check if CPU can support multi-core first,
+ * so don't enable it by default.
+ */
+//		conf->dpc_cpucore = 2;
+//		conf->rxf_cpucore = 3;
+//		conf->disable_proptx = 1;
+		conf->frameburst = 1;
+#ifdef DYNAMIC_MAX_HDR_READ
+		conf->max_hdr_read = 256;
+		firstread = 256;
+#endif /* DYNAMIC_MAX_HDR_READ */
+		dhd_rxbound = 512;
+#endif /* BCMSDIO */
+#ifdef BCMPCIE
+#if defined(SET_XPS_CPUS)
+		conf->xps_cpus = TRUE;
+#endif /* SET_XPS_CPUS */
+#if defined(SET_RPS_CPUS)
+		conf->rps_cpus = TRUE;
+#endif /* SET_RPS_CPUS */
+		conf->orphan_move = 3;
+		conf->flow_ring_queue_threshold = 2048;
+#endif /* BCMPCIE */
+#ifdef DHDTCPACK_SUPPRESS
+		conf->tcpack_sup_ratio = 15;
+		conf->tcpack_sup_delay = 10;
+#endif /* DHDTCPACK_SUPPRESS */
+	}
+	else {
+		conf->mtu = 0;
+		conf->pktsetsum = FALSE;
+#ifdef BCMSDIO
+		conf->dhd_dpc_prio = -1;
+		conf->disable_proptx = -1;
+		conf->frameburst = 1;
+#ifdef DYNAMIC_MAX_HDR_READ
+		conf->max_hdr_read = 0;
+		firstread = 32;
+#endif /* DYNAMIC_MAX_HDR_READ */
+		dhd_rxbound = 128;
+#endif /* BCMSDIO */
+#ifdef BCMPCIE
+#if defined(SET_XPS_CPUS)
+		conf->xps_cpus = FALSE;
+#endif /* SET_XPS_CPUS */
+#if defined(SET_RPS_CPUS)
+		conf->rps_cpus = FALSE;
+#endif /* SET_RPS_CPUS */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+		conf->orphan_move = 1;
+#else
+		conf->orphan_move = 0;
+#endif
+		conf->flow_ring_queue_threshold = 2048;
+#endif /* BCMPCIE */
+#ifdef DHDTCPACK_SUPPRESS
+		conf->tcpack_sup_ratio = CUSTOM_TCPACK_SUPP_RATIO;
+		conf->tcpack_sup_delay = CUSTOM_TCPACK_DELAY_TIME;
+#endif /* DHDTCPACK_SUPPRESS */
+	}
+}
+
+void
+dhd_conf_dump_tput_patch(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	CONFIG_TRACE("tput_patch=%d\n", conf->tput_patch);
+	CONFIG_TRACE("mtu=%d\n", conf->mtu);
+	CONFIG_TRACE("pktsetsum=%d\n", conf->pktsetsum);
+	CONFIG_TRACE("orphan_move=%d\n", conf->orphan_move);
+#ifdef DHDTCPACK_SUPPRESS
+	CONFIG_TRACE("tcpack_sup_ratio=%d\n", conf->tcpack_sup_ratio);
+	CONFIG_TRACE("tcpack_sup_delay=%d\n", conf->tcpack_sup_delay);
+#endif
+
+#ifdef BCMSDIO
+	CONFIG_TRACE("dhd_dpc_prio=%d\n", conf->dhd_dpc_prio);
+	CONFIG_TRACE("dhd_poll=%d\n", conf->dhd_poll);
+	CONFIG_TRACE("disable_proptx=%d\n", conf->disable_proptx);
+	CONFIG_TRACE("frameburst=%d\n", conf->frameburst);
+#ifdef DYNAMIC_MAX_HDR_READ
+	CONFIG_TRACE("max_hdr_read=%d\n", conf->max_hdr_read);
+	CONFIG_TRACE("firstread=%d\n", firstread);
+#endif
+	CONFIG_TRACE("dhd_rxbound=%d\n", dhd_rxbound);
+#endif
+
+#ifdef BCMPCIE
+	CONFIG_TRACE("flow_ring_queue_threshold=%d\n", conf->flow_ring_queue_threshold);
+#endif
+
+#if defined(SET_XPS_CPUS)
+	CONFIG_TRACE("xps_cpus=%d\n", conf->xps_cpus);
+#endif
+#if defined(SET_RPS_CPUS)
+	CONFIG_TRACE("rps_cpus=%d\n", conf->rps_cpus);
+#endif
+
+}
+#endif /* DHD_TPUT_PATCH */
+
+#ifdef DHD_REQUEST_FW_PATH
+#define FIRMWARE_CLASS_PATH "/sys/module/firmware_class/parameters/path"
+static void
+dhd_conf_get_filename(char *pFilename)
+{
+	const char *pName = NULL;
+
+	if ((pFilename) && (*pFilename)) {
+		// back/reverse search the '/'
+		pName = strrchr(pFilename, '/');
+		if (NULL == pName) {
+			pName = pFilename;
+		} else {
+			if (pName[1]) {
+				pName++;
+			} else {
+				pName = NULL;
+			}
+		}
+	}
+
+	if (pName)
+		strcpy(pFilename, pName);
+
+	return;
+}
+
+static void
+dhd_conf_add_filepath(dhd_pub_t *dhd, char *pFilename)
+{
+	char path[WLC_IOCTL_SMLEN];
+	char *name_ptr, *module_name = NULL;
+
+	if (strlen(pFilename)) {
+		name_ptr = path;
+		strcpy(name_ptr, "");
+#ifdef FW_AMPAK_PATH
+		strcat(name_ptr, "/");
+		strcat(name_ptr, FW_AMPAK_PATH);
+#endif
+#ifdef MODULE_PATH
+		module_name = dhd_conf_get_module_name(dhd, DONT_CARE);
+#endif
+		if (module_name) {
+			strcat(name_ptr, "/");
+			strcat(name_ptr, module_name);
+		}
+		strcat(name_ptr, "/");
+		strcat(name_ptr, pFilename);
+		strcpy(pFilename, path);
+	}
+
+	return;
+}
+#endif /* DHD_REQUEST_FW_PATH */
+
+void
+dhd_conf_set_path_params(dhd_pub_t *dhd, char *fw_path, char *nv_path)
+{
+	int ag_type;
+
+	/* External conf takes precedence if specified */
+	dhd_conf_preinit(dhd);
+
+#ifdef DHD_REQUEST_FW_PATH
+	// preprocess the filename to only left 'name'
+	dhd_conf_get_filename(fw_path);
+	dhd_conf_get_filename(nv_path);
+	dhd_conf_get_filename(dhd->clm_path);
+	dhd_conf_get_filename(dhd->conf_path);
+#endif
+
+	if (dhd->conf_path[0] == '\0') {
+		dhd_conf_copy_path(dhd, "config.txt", dhd->conf_path, nv_path);
+	}
+	if (dhd->clm_path[0] == '\0') {
+		dhd_conf_copy_path(dhd, "clm.blob", dhd->clm_path, fw_path);
+	}
+#ifdef CONFIG_PATH_AUTO_SELECT
+	dhd_conf_set_conf_name_by_chip(dhd, dhd->conf_path);
+#endif
+
+	ag_type = dhd_conf_set_fw_name_by_chip(dhd, fw_path);
+	dhd_conf_set_nv_name_by_chip(dhd, nv_path, ag_type);
+	dhd_conf_set_clm_name_by_chip(dhd, dhd->clm_path, ag_type);
+#ifdef SET_FWNV_BY_MAC
+	dhd_conf_set_fw_name_by_mac(dhd, fw_path);
+	dhd_conf_set_nv_name_by_mac(dhd, nv_path);
+#endif
+
+#ifdef DHD_REQUEST_FW_PATH
+	dhd_conf_add_filepath(dhd, fw_path);
+	dhd_conf_add_filepath(dhd, nv_path);
+	dhd_conf_add_filepath(dhd, dhd->clm_path);
+	dhd_conf_add_filepath(dhd, dhd->conf_path);
+#endif
+
+	CONFIG_MSG("Final fw_path=%s\n", fw_path);
+	CONFIG_MSG("Final nv_path=%s\n", nv_path);
+	CONFIG_MSG("Final clm_path=%s\n", dhd->clm_path);
+	CONFIG_MSG("Final conf_path=%s\n", dhd->conf_path);
+
+	dhd_conf_read_config(dhd, dhd->conf_path);
+#ifdef DHD_TPUT_PATCH
+	dhd_conf_dump_tput_patch(dhd);
+#endif
+}
+
+int
+dhd_conf_set_intiovar(dhd_pub_t *dhd, int ifidx, uint cmd, char *name, int val,
+	int def, bool down)
+{
+	int ret = -1;
+	char iovbuf[WL_EVENTING_MASK_LEN + 12];	/*  Room for "event_msgs" + '\0' + bitvec  */
+
+	if (val >= def) {
+		if (down) {
+			if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_DOWN, NULL, 0, TRUE, ifidx)) < 0)
+				CONFIG_ERROR("WLC_DOWN setting failed %d\n", ret);
+		}
+		if (cmd == WLC_SET_VAR) {
+			CONFIG_TRACE("set %s %d\n", name, val);
+			bcm_mkiovar(name, (char *)&val, sizeof(val), iovbuf, sizeof(iovbuf));
+			if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, sizeof(iovbuf), TRUE, ifidx)) < 0)
+				CONFIG_ERROR("%s setting failed %d\n", name, ret);
+		} else {
+			CONFIG_TRACE("set %s %d %d\n", name, cmd, val);
+			if ((ret = dhd_wl_ioctl_cmd(dhd, cmd, &val, sizeof(val), TRUE, ifidx)) < 0)
+				CONFIG_ERROR("%s setting failed %d\n", name, ret);
+		}
+	}
+
+	return ret;
+}
+
+static int
+dhd_conf_set_bufiovar(dhd_pub_t *dhd, int ifidx, uint cmd, char *name,
+	char *buf, int len, bool down)
+{
+	char iovbuf[WLC_IOCTL_SMLEN];
+	s32 iovar_len;
+	int ret = -1;
+
+	if (down) {
+		if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_DOWN, NULL, 0, TRUE, ifidx)) < 0)
+			CONFIG_ERROR("WLC_DOWN setting failed %d\n", ret);
+	}
+
+	if (cmd == WLC_SET_VAR) {
+		iovar_len = bcm_mkiovar(name, buf, len, iovbuf, sizeof(iovbuf));
+		if (iovar_len > 0)
+			ret = dhd_wl_ioctl_cmd(dhd, cmd, iovbuf, iovar_len, TRUE, ifidx);
+		else
+			ret = BCME_BUFTOOSHORT;
+		if (ret < 0)
+			CONFIG_ERROR("%s setting failed %d, len=%d\n", name, ret, len);
+	} else {
+		if ((ret = dhd_wl_ioctl_cmd(dhd, cmd, buf, len, TRUE, ifidx)) < 0)
+			CONFIG_ERROR("%s setting failed %d\n", name, ret);
+	}
+
+	return ret;
+}
+
+static int
+dhd_conf_iovar_buf(dhd_pub_t *dhd, int ifidx, int cmd, char *name,
+	char *buf, int len)
+{
+	char *iovbuf = NULL;
+	int ret = -1, iovbuf_len = WLC_IOCTL_MEDLEN;
+	s32 iovar_len;
+
+	iovbuf = kmalloc(iovbuf_len, GFP_KERNEL);
+	if (iovbuf == NULL) {
+		CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", iovbuf_len);
+		goto exit;
+	}
+
+	if (cmd == WLC_GET_VAR) {
+		if (bcm_mkiovar(name, buf, len, iovbuf, iovbuf_len)) {
+			ret = dhd_wl_ioctl_cmd(dhd, cmd, iovbuf, iovbuf_len, FALSE, ifidx);
+			if (!ret) {
+				memcpy(buf, iovbuf, len);
+			} else {
+				CONFIG_ERROR("get iovar %s failed %d\n", name, ret);
+			}
+		} else {
+			CONFIG_ERROR("mkiovar %s failed\n", name);
+		}
+	} else if (cmd == WLC_SET_VAR) {
+		iovar_len = bcm_mkiovar(name, buf, len, iovbuf, iovbuf_len);
+		if (iovar_len > 0)
+			ret = dhd_wl_ioctl_cmd(dhd, cmd, iovbuf, iovar_len, TRUE, ifidx);
+		else
+			ret = BCME_BUFTOOSHORT;
+		if (ret < 0)
+			CONFIG_ERROR("%s setting failed %d, len=%d\n", name, ret, len);
+	}
+
+exit:
+	if (iovbuf)
+		kfree(iovbuf);
+	return ret;
+}
+
+static int
+dhd_conf_get_iovar(dhd_pub_t *dhd, int ifidx, int cmd, char *name,
+	char *buf, int len)
+{
+	char iovbuf[WLC_IOCTL_SMLEN];
+	int ret = -1;
+
+	if (cmd == WLC_GET_VAR) {
+		if (bcm_mkiovar(name, NULL, 0, iovbuf, sizeof(iovbuf))) {
+			ret = dhd_wl_ioctl_cmd(dhd, cmd, iovbuf, sizeof(iovbuf), FALSE, ifidx);
+			if (!ret) {
+				memcpy(buf, iovbuf, len);
+			} else {
+				CONFIG_ERROR("get iovar %s failed %d\n", name, ret);
+			}
+		} else {
+			CONFIG_ERROR("mkiovar %s failed\n", name);
+		}
+	} else {
+		ret = dhd_wl_ioctl_cmd(dhd, cmd, buf, len, FALSE, 0);
+		if (ret < 0)
+			CONFIG_ERROR("get iovar %s failed %d\n", name, ret);
+	}
+
+	return ret;
+}
+
+static int
+dhd_conf_intiovar(dhd_pub_t *dhd, int ifidx, uint cmd, char *name,
+	char *buf, int len)
+{
+	char iovbuf[WL_EVENTING_MASK_LEN + 12];	/*  Room for "event_msgs" + '\0' + bitvec  */
+	int ret = -1, val;
+
+	if (cmd == WLC_SET_VAR) {
+		memcpy(&val, buf, len);
+		CONFIG_TRACE("set %s %d\n", name, val);
+		bcm_mkiovar(name, (char *)&val, sizeof(val), iovbuf, sizeof(iovbuf));
+		ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, sizeof(iovbuf), TRUE, ifidx);
+	}
+	else if (cmd == WLC_GET_VAR) {
+		if (bcm_mkiovar(name, NULL, 0, iovbuf, sizeof(iovbuf))) {
+			ret = dhd_wl_ioctl_cmd(dhd, cmd, iovbuf, sizeof(iovbuf), FALSE, ifidx);
+			if (!ret) {
+				memcpy(buf, iovbuf, len);
+				CONFIG_TRACE("get %s %d\n", name, *(int *)buf);
+			}
+		} else {
+			CONFIG_ERROR("mkiovar %s failed\n", name);
+		}
+	}
+
+	return ret;
+}
+
+static int
+dhd_conf_get_ioctl_ver(dhd_pub_t *dhd)
+{
+	int ret = 0;
+	s32 val = 0;
+
+	dhd->conf->ioctl_ver = WLC_IOCTL_VERSION;
+	ret = dhd_conf_get_iovar(dhd, 0, WLC_GET_VERSION, "WLC_GET_VERSION",
+		(char *)&val, sizeof(val));
+	if (ret) {
+		return ret;
+	}
+	val = dtoh32(val);
+	if (val != WLC_IOCTL_VERSION && val != 1) {
+		CONFIG_ERROR("Version mismatch, please upgrade. Got %d, expected %d or 1\n",
+			val, WLC_IOCTL_VERSION);
+		return BCME_VERSION;
+	}
+	dhd->conf->ioctl_ver = val;
+	CONFIG_TRACE("ioctl_ver=%d\n", dhd->conf->ioctl_ver);
+
+	return ret;
+}
+
+int
+dhd_conf_get_country(dhd_pub_t *dhd, wl_country_t *cspec)
+{
+	int bcmerror = -1;
+
+	memset(cspec, 0, sizeof(wl_country_t));
+	bcm_mkiovar("country", NULL, 0, (char*)cspec, sizeof(wl_country_t));
+	if ((bcmerror = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, cspec, sizeof(wl_country_t),
+			FALSE, 0)) < 0)
+		CONFIG_ERROR("country code getting failed %d\n", bcmerror);
+
+	return bcmerror;
+}
+
+static int
+dhd_conf_check_ccode_all(dhd_pub_t *dhd)
+{
+	int ret = -1;
+	wl_channels_in_country_t *cic;
+
+	cic = kmalloc(WLC_IOCTL_SMLEN, GFP_KERNEL);
+	if (!cic) {
+		CONFIG_ERROR("kmalloc failed for cic\n");
+		goto exit;
+	}
+	memset(cic, 0, WLC_IOCTL_SMLEN);
+	cic->buflen = WLC_IOCTL_SMLEN;
+	cic->count = 0;
+	strcpy(cic->country_abbrev, dhd->conf->ccode_all.cspec.ccode);
+	cic->band = WLC_BAND_2G;
+	cic->buflen = htod32(cic->buflen);
+	cic->band = htod32(cic->band);
+	cic->count = htod32(cic->count);
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_CHANNELS_IN_COUNTRY, (char *)cic,
+		WLC_IOCTL_SMLEN, FALSE, 0);
+exit:
+	if (ret)
+		memset(&dhd->conf->ccode_all, 0, sizeof(wl_ccode_all_t));
+	if (cic)
+		kfree(cic);
+
+	return ret;
+}
+
+int
+dhd_conf_map_country_list(dhd_pub_t *dhd, wl_country_t *cspec)
+{
+	int ret = -1;
+	struct dhd_conf *conf = dhd->conf;
+	country_list_t *country = conf->country_head;
+
+#ifdef CCODE_LIST
+	if (strlen(dhd->conf->ccode_all.cspec.ccode)) {
+		ret = dhd_ccode_map_country_all(dhd, cspec);
+	}
+	if (ret)
+		ret = dhd_ccode_map_country_list(dhd, cspec);
+#endif
+	while (country != NULL) {
+		if (!strncmp("**", country->cspec.country_abbrev, 2)) {
+			if (!strncmp("**", country->cspec.ccode, 2)) {
+				// **:**/0 => return user specified ccode if not found, but set regrev 0
+				cspec->rev = 0;
+				ret = 0;
+				break;
+			}
+			// **:XZ/11 => return XZ/11 if not found
+			memcpy(cspec->ccode, country->cspec.ccode, WLC_CNTRY_BUF_SZ);
+			cspec->rev = country->cspec.rev;
+			ret = 0;
+			break;
+		} else if (!strncmp(cspec->country_abbrev,
+				country->cspec.country_abbrev, 2)) {
+			memcpy(cspec->ccode, country->cspec.ccode, WLC_CNTRY_BUF_SZ);
+			cspec->rev = country->cspec.rev;
+			ret = 0;
+			break;
+		}
+		country = country->next;
+	}
+
+	return ret;
+}
+
+static int
+dhd_conf_set_country(dhd_pub_t *dhd, wl_country_t *cspec)
+{
+	int bcmerror = -1;
+	struct net_device *net;
+	int bytes_written = 0;
+	char event_msg[32];
+
+	memset(&dhd->dhd_cspec, 0, sizeof(wl_country_t));
+
+	net = dhd_idx2net(dhd, 0);
+	snprintf(event_msg, sizeof(event_msg), "wl event_msg %d 0", WLC_E_COUNTRY_CODE_CHANGED);
+	wl_android_ext_priv_cmd(net, event_msg, 0, &bytes_written);
+
+	CONFIG_MSG("set country %s, revision %d\n", cspec->ccode, cspec->rev);
+	bcmerror = dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "country", (char *)cspec,
+		sizeof(wl_country_t), FALSE);
+	dhd_conf_get_country(dhd, cspec);
+	CONFIG_MSG("Country code: %s (%s/%d)\n",
+		cspec->country_abbrev, cspec->ccode, cspec->rev);
+
+	snprintf(event_msg, sizeof(event_msg), "wl event_msg %d 1", WLC_E_COUNTRY_CODE_CHANGED);
+	wl_android_ext_priv_cmd(net, event_msg, 0, &bytes_written);
+
+	return bcmerror;
+}
+
+int
+dhd_conf_fix_country(dhd_pub_t *dhd)
+{
+	int bcmerror = -1;
+	int band;
+	wl_uint32_list_t *list;
+	u8 valid_chan_list[sizeof(u32)*(WL_NUMCHANNELS + 1)];
+	wl_country_t cspec;
+
+	if (!(dhd && dhd->conf)) {
+		return bcmerror;
+	}
+
+	memset(valid_chan_list, 0, sizeof(valid_chan_list));
+	list = (wl_uint32_list_t *)(void *) valid_chan_list;
+	list->count = htod32(WL_NUMCHANNELS);
+	if ((bcmerror = dhd_wl_ioctl_cmd(dhd, WLC_GET_VALID_CHANNELS, valid_chan_list,
+			sizeof(valid_chan_list), FALSE, 0)) < 0) {
+		CONFIG_ERROR("get channels failed with %d\n", bcmerror);
+	}
+
+	band = dhd_conf_get_band(dhd);
+
+	if (bcmerror || ((band==WLC_BAND_AUTO || band==WLC_BAND_2G || band==-1) &&
+			dtoh32(list->count)<11)) {
+		CONFIG_ERROR("bcmerror=%d, # of channels %d\n",
+			bcmerror, dtoh32(list->count));
+		dhd_conf_map_country_list(dhd, &dhd->conf->cspec);
+		if ((bcmerror = dhd_conf_set_country(dhd, &dhd->conf->cspec)) < 0) {
+			strcpy(cspec.country_abbrev, "US");
+			cspec.rev = 0;
+			strcpy(cspec.ccode, "US");
+			dhd_conf_map_country_list(dhd, &cspec);
+			dhd_conf_set_country(dhd, &cspec);
+		}
+	}
+
+	return bcmerror;
+}
+
+static int
+dhd_conf_rsdb_mode(dhd_pub_t *dhd, char *cmd, char *buf)
+{
+	wl_config_t rsdb_mode_cfg = {1, 0};
+
+	if (buf) {
+		rsdb_mode_cfg.config = (int)simple_strtol(buf, NULL, 0);
+		CONFIG_MSG("rsdb_mode %d\n", rsdb_mode_cfg.config);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, cmd, (char *)&rsdb_mode_cfg,
+			sizeof(rsdb_mode_cfg), TRUE);
+	}
+
+	return 0;
+}
+
+int
+dhd_conf_reg2args(dhd_pub_t *dhd, char *cmd, bool set, uint32 index, uint32 *val)
+{
+	char var[WLC_IOCTL_SMLEN];
+	uint32 int_val, len;
+	void *ptr = NULL;
+	int ret = 0;
+
+	len = sizeof(int_val);
+	int_val = htod32(index);
+	memset(var, 0, sizeof(var));
+	memcpy(var, (char *)&int_val, sizeof(int_val));
+
+	if (set) {
+		int_val = htod32(*val);
+		memcpy(&var[len], (char *)&int_val, sizeof(int_val));
+		len += sizeof(int_val);
+		dhd_conf_iovar_buf(dhd, 0, WLC_SET_VAR, cmd, var, sizeof(var));
+	} else {
+		ret = dhd_conf_iovar_buf(dhd, 0, WLC_GET_VAR, cmd, var, sizeof(var));
+		if (ret < 0)
+			return ret;
+		ptr = var;
+		*val = dtoh32(*(int *)ptr);
+	}
+
+	return ret;
+}
+
+static int
+dhd_conf_btc_params(dhd_pub_t *dhd, char *cmd, char *buf)
+{
+	int ret = BCME_OK;
+	uint32 cur_val;
+	int index = 0, mask = 0, value = 0;
+	// btc_params=[index] [mask] [value]
+	// Ex: btc_params=82 0x0021 0x0001
+
+	if (buf) {
+		sscanf(buf, "%d %x %x", &index, &mask, &value);
+	}
+
+	CONFIG_TRACE("%s%d mask=0x%04x value=0x%04x\n", cmd, index, mask, value);
+
+	ret = dhd_conf_reg2args(dhd, cmd, FALSE, index, &cur_val);
+	CONFIG_TRACE("%s%d = 0x%04x\n", cmd, index, cur_val);
+	cur_val &= (~mask);
+	cur_val |= value;
+
+	// need to WLC_UP before btc_params
+	dhd_conf_set_intiovar(dhd, 0, WLC_UP, "WLC_UP", 0, 0, FALSE);
+
+	CONFIG_TRACE("wl %s%d 0x%04x\n", cmd, index, cur_val);
+	ret = dhd_conf_reg2args(dhd, cmd, TRUE, index, &cur_val);
+
+	ret = dhd_conf_reg2args(dhd, cmd, FALSE, index, &cur_val);
+	CONFIG_MSG("%s%d = 0x%04x\n", cmd, index, cur_val);
+
+	return ret;
+}
+
+typedef struct sub_cmd_t {
+	char *name;
+	uint16 id;		/* id for the dongle f/w switch/case  */
+	uint16 type;		/* base type of argument IOVT_XXXX */
+} sub_cmd_t;
+
+/* wl he sub cmd list */
+static const sub_cmd_t he_cmd_list[] = {
+	{"enab", WL_HE_CMD_ENAB, IOVT_UINT8},
+	{"features", WL_HE_CMD_FEATURES, IOVT_UINT32},
+	{"bsscolor", WL_HE_CMD_BSSCOLOR, IOVT_UINT8},
+	{"partialbsscolor", WL_HE_CMD_PARTIAL_BSSCOLOR, IOVT_UINT8},
+	{"cap", WL_HE_CMD_CAP, IOVT_UINT8},
+	{"staid", WL_HE_CMD_STAID, IOVT_UINT16},
+	{"rtsdurthresh", WL_HE_CMD_RTSDURTHRESH, IOVT_UINT16},
+	{"peduration", WL_HE_CMD_PEDURATION, IOVT_UINT8},
+	{"testbed_mode", WL_HE_CMD_TESTBED_MODE, IOVT_UINT32},
+	{"omi_ulmu_throttle", WL_HE_CMD_OMI_ULMU_THROTTLE, IOVT_UINT16},
+	{"omi_dlmu_rr_mpf_map", WL_HE_CMD_OMI_DLMU_RSD_RCM_MPF_MAP, IOVT_UINT32},
+	{"ulmu_disable_policy", WL_HE_CMD_ULMU_DISABLE_POLICY, IOVT_UINT8},
+	{"sr_prohibit", WL_HE_CMD_SR_PROHIBIT, IOVT_UINT8},
+};
+
+static uint
+wl_he_iovt2len(uint iovt)
+{
+	switch (iovt) {
+	case IOVT_BOOL:
+	case IOVT_INT8:
+	case IOVT_UINT8:
+		return sizeof(uint8);
+	case IOVT_INT16:
+	case IOVT_UINT16:
+		return sizeof(uint16);
+	case IOVT_INT32:
+	case IOVT_UINT32:
+		return sizeof(uint32);
+	default:
+		/* ASSERT(0); */
+		return 0;
+	}
+}
+
+static int
+dhd_conf_he_cmd(dhd_pub_t * dhd, char *cmd, char *buf)
+{
+	int ret = BCME_OK, i;
+	bcm_xtlv_t *pxtlv = NULL;
+	uint8 mybuf[128];
+	uint16 he_id = -1, he_len = 0, mybuf_len = sizeof(mybuf);
+	uint32 he_val;
+	const sub_cmd_t *tpl = he_cmd_list;
+	char sub_cmd[32] = "", he_val_str[10] = "";
+
+	if (buf) {
+		sscanf(buf, "%s %s", sub_cmd, he_val_str);
+	}
+
+	for (i=0; i<ARRAY_SIZE(he_cmd_list); i++, tpl++) {
+		if (!strcmp(tpl->name, sub_cmd)) {
+			he_id = tpl->id;
+			he_len = wl_he_iovt2len(tpl->type);
+			break;
+		}
+	}
+	if (he_id < 0) {
+		CONFIG_ERROR("No he id found for %s\n", sub_cmd);
+		return 0;
+	}
+
+	pxtlv = (bcm_xtlv_t *)mybuf;
+
+	if (strlen(he_val_str)) {
+		he_val = simple_strtol(he_val_str, NULL, 0);
+		ret = bcm_pack_xtlv_entry((uint8**)&pxtlv, &mybuf_len, he_id,
+			he_len, (uint8 *)&he_val, BCM_XTLV_OPTION_ALIGN32);
+		if (ret != BCME_OK) {
+			CONFIG_ERROR("failed to pack he enab, err: %s\n", bcmerrorstr(ret));
+			return 0;
+		}
+		CONFIG_TRACE("he %s 0x%x\n", sub_cmd, he_val);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, cmd, (char *)&mybuf,
+			sizeof(mybuf), TRUE);
+	}
+
+	return 0;
+}
+
+#ifndef SUPPORT_RANDOM_MAC_SCAN
+int
+dhd_conf_scan_mac(dhd_pub_t * dhd, char *cmd, char *buf)
+{
+	uint8 buffer[WLC_IOCTL_SMLEN] = {0, };
+	wl_scanmac_t *sm = NULL;
+	wl_scanmac_enable_t *sm_enable = NULL;
+	int enable = 0, len = 0, ret = -1;
+	char sub_cmd[32], iovbuf[WLC_IOCTL_SMLEN];
+	s32 iovar_len;
+
+	memset(sub_cmd, 0, sizeof(sub_cmd));
+	if (buf) {
+		sscanf(buf, "%s %d", sub_cmd, &enable);
+	}
+
+	if (!strcmp(sub_cmd, "enable")) {
+		sm = (wl_scanmac_t *)buffer;
+		sm_enable = (wl_scanmac_enable_t *)sm->data;
+		sm->len = sizeof(*sm_enable);
+		sm_enable->enable = enable;
+		len = OFFSETOF(wl_scanmac_t, data) + sm->len;
+		sm->subcmd_id = WL_SCANMAC_SUBCMD_ENABLE;
+		CONFIG_TRACE("scanmac enable %d\n", sm_enable->enable);
+
+		iovar_len = bcm_mkiovar("scanmac", buffer, len, iovbuf, sizeof(iovbuf));
+		if (iovar_len > 0)
+			ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, iovar_len, TRUE, 0);
+		else
+			ret = BCME_BUFTOOSHORT;
+		if (ret == BCME_UNSUPPORTED)
+			CONFIG_TRACE("scanmac, UNSUPPORTED\n");
+		else if (ret != BCME_OK)
+			CONFIG_ERROR("%s setting failed %d, len=%d\n", "scanmac", ret, len);
+	}
+	else {
+		CONFIG_ERROR("wrong cmd \"%s %d\"\n", sub_cmd, enable);
+	}
+
+	return 0;
+}
+#endif
+
+static void
+dhd_conf_reset_country_all(dhd_pub_t *dhd)
+{
+	int val;
+	val = 2;
+	dhd_conf_intiovar(dhd, 0, WLC_SET_VAR, "ww_2g_chan_only", (char *)&val, sizeof(val));
+	val = 0;
+	dhd_conf_intiovar(dhd, 0, WLC_SET_VAR, "disable_5g_band", (char *)&val, sizeof(val));
+	val = 0;
+	dhd_conf_intiovar(dhd, 0, WLC_SET_VAR, "disable_6g_band", (char *)&val, sizeof(val));
+}
+
+static int
+dhd_conf_set_country_all(dhd_pub_t *dhd, wl_country_t *cspec)
+{
+	struct dhd_conf *conf = dhd->conf;
+	wl_ccode_all_t *ccode_all = &conf->ccode_all;
+	wl_country_t cspec_all;
+	int ret = -1;
+
+	CONFIG_MSG("%s:%s/%d(2g%x-5g%x-6g%x)\n",
+		cspec->country_abbrev, ccode_all->cspec.ccode, ccode_all->cspec.rev,
+		ccode_all->ww_2g_chan_only,
+		ccode_all->disable_5g_band, ccode_all->disable_6g_band);
+
+	memcpy(&cspec_all, &ccode_all->cspec, sizeof(wl_country_t));
+	ret = dhd_conf_set_country(dhd, &cspec_all);
+	if (!ret) {
+		dhd_conf_intiovar(dhd, 0, WLC_SET_VAR, "ww_2g_chan_only",
+			(char *)&ccode_all->ww_2g_chan_only, sizeof(ccode_all->ww_2g_chan_only));
+		dhd_conf_intiovar(dhd, 0, WLC_SET_VAR, "disable_5g_band",
+			(char *)&ccode_all->disable_5g_band, sizeof(ccode_all->disable_5g_band));
+		dhd_conf_intiovar(dhd, 0, WLC_SET_VAR, "disable_6g_band",
+			(char *)&ccode_all->disable_6g_band, sizeof(ccode_all->disable_6g_band));
+	}
+
+	return ret;
+}
+
+bool
+dhd_conf_same_country(dhd_pub_t *dhd, char *buf)
+{
+	wl_ccode_all_t *ccode_all = &dhd->conf->ccode_all;
+	wl_country_t cspec = {{0}, 0, {0}};
+	wl_country_t cur_cspec = {{0}, 0, {0}};
+	uint32 ww_2g_chan_only = 0, disable_5g_band = 0, disable_6g_band = 0;
+	int ret;
+	bool match = TRUE;
+
+	strlcpy(cspec.country_abbrev, buf, WL_CCODE_LEN + 1);
+	strlcpy(cspec.ccode, buf, WL_CCODE_LEN + 1);
+	dhd_conf_map_country_list(dhd, &cspec);
+	dhd_conf_get_country(dhd, &cur_cspec);
+	if (!strncmp(cspec.ccode, ccode_all->cspec.ccode, 2) &&
+			!strncmp(cspec.ccode, cur_cspec.ccode, 2)) {
+		ret = dhd_conf_intiovar(dhd, 0, WLC_GET_VAR, "ww_2g_chan_only",
+			(char *)&ww_2g_chan_only, sizeof(ww_2g_chan_only));
+		if (!ret && ww_2g_chan_only != ccode_all->ww_2g_chan_only)
+			match = FALSE;	
+		ret = dhd_conf_intiovar(dhd, 0, WLC_GET_VAR, "disable_5g_band",
+			(char *)&disable_5g_band, sizeof(disable_5g_band));
+		if (!ret && disable_5g_band != ccode_all->disable_5g_band)
+			match = FALSE;
+		ret = dhd_conf_intiovar(dhd, 0, WLC_GET_VAR, "disable_6g_band",
+			(char *)&disable_6g_band, sizeof(disable_6g_band));
+		if (!ret && disable_6g_band != ccode_all->disable_6g_band)
+			match = FALSE;
+		if (match) {
+			CONFIG_MSG("%s/%d(2g%x-5g%x-6g%x) is already configured\n",
+				cur_cspec.ccode, cur_cspec.rev,
+				ww_2g_chan_only, disable_5g_band, disable_6g_band);
+			return TRUE;
+		}
+	}
+	if (!memcmp(&cspec.ccode, &cur_cspec.ccode, WL_CCODE_LEN) &&
+			(cspec.rev == cur_cspec.rev) &&
+			strncmp(cspec.ccode, ccode_all->cspec.ccode, 2)) {
+		CONFIG_MSG("country code = %s/%d is already configured\n",
+			cspec.ccode, cspec.rev);
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+int
+dhd_conf_country(dhd_pub_t *dhd, char *cmd, char *buf)
+{
+	wl_ccode_all_t *ccode_all = &dhd->conf->ccode_all;
+	wl_country_t cspec = {{0}, 0, {0}};
+	int err = 0;
+
+	if (buf) {
+		if (dhd_conf_same_country(dhd, buf)) {
+			return 0;
+		}
+		strlcpy(cspec.country_abbrev, buf, WL_CCODE_LEN + 1);
+		strlcpy(cspec.ccode, buf, WL_CCODE_LEN + 1);
+		dhd_conf_map_country_list(dhd, &cspec);
+		if (!strncmp(cspec.ccode, ccode_all->cspec.ccode, 2))
+			err = dhd_conf_set_country_all(dhd, &cspec);
+		else {
+			if (strlen(dhd->conf->ccode_all.cspec.ccode)) {
+				dhd_conf_reset_country_all(dhd);
+			}
+			err = dhd_conf_set_country(dhd, &cspec);
+		}
+		if (!err) {
+			dhd_conf_fix_country(dhd);
+		}
+		dhd_conf_get_country(dhd, &dhd->dhd_cspec);
+	}
+
+	return err;
+}
+
+int
+dhd_conf_autocountry(dhd_pub_t *dhd, char *cmd, char *buf)
+{
+	struct net_device *net;
+	int bytes_written = 0;
+	char event_msg[32];
+	int enable = 0;
+
+	if (buf) {
+		sscanf(buf, "%d", &enable);
+	}
+
+	net = dhd_idx2net(dhd, 0);
+	snprintf(event_msg, sizeof(event_msg), "wl event_msg %d 0", WLC_E_COUNTRY_CODE_CHANGED);
+	wl_android_ext_priv_cmd(net, event_msg, 0, &bytes_written);
+
+	CONFIG_MSG("autocountry %d\n", enable);
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "autocountry", enable, 0, FALSE);
+
+	snprintf(event_msg, sizeof(event_msg), "wl event_msg %d 1", WLC_E_COUNTRY_CODE_CHANGED);
+	wl_android_ext_priv_cmd(net, event_msg, 0, &bytes_written);
+
+	dhd_conf_country(dhd, "country", dhd->conf->cspec.country_abbrev);
+
+	return 0;
+}
+
+typedef int (tpl_parse_t)(dhd_pub_t *dhd, char *name, char *buf);
+
+typedef struct iovar_tpl_t {
+	int cmd;
+	char *name;
+	tpl_parse_t *parse;
+} iovar_tpl_t;
+
+const iovar_tpl_t iovar_tpl_list[] = {
+	{WLC_SET_VAR,	"rsdb_mode",	dhd_conf_rsdb_mode},
+	{WLC_SET_VAR,	"he",		dhd_conf_he_cmd},
+	{WLC_SET_VAR,	"btc_params",	dhd_conf_btc_params},
+#ifndef SUPPORT_RANDOM_MAC_SCAN
+	{WLC_SET_VAR,	"scanmac",		dhd_conf_scan_mac},
+#endif
+	{WLC_SET_VAR,	"country",		dhd_conf_country},
+	{WLC_SET_VAR,	"autocountry",	dhd_conf_autocountry},
+};
+
+static int iovar_tpl_parse(const iovar_tpl_t *tpl, int tpl_count,
+	dhd_pub_t *dhd, int cmd, char *name, char *buf)
+{
+	int i, ret = 0;
+
+	/* look for a matching code in the table */
+	for (i = 0; i < tpl_count; i++, tpl++) {
+		if (tpl->cmd == cmd && !strcmp(tpl->name, name))
+			break;
+	}
+	if (i < tpl_count && tpl->parse) {
+		ret = tpl->parse(dhd, name, buf);
+	} else {
+		ret = -1;
+	}
+
+	return ret;
+}
+
+static bool
+dhd_conf_set_wl_cmd(dhd_pub_t *dhd, char *data, bool down)
+{
+	int cmd, val, ret = 0, len;
+	char name[32], *pch, *pick_tmp, *pick_tmp2, *pdata = NULL;
+
+	/* Process wl_preinit:
+	 * wl_preinit=[cmd]=[val], [cmd]=[val]
+	 * Ex: wl_preinit=86=0, mpc=0
+	 */
+
+	if (data == NULL)
+		return FALSE;
+
+	len = strlen(data);
+	pdata = kmalloc(len+1, GFP_KERNEL);
+	if (pdata == NULL) {
+		CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", len+1);
+		goto exit;
+	}
+	memset(pdata, 0, len+1);
+	strcpy(pdata, data);
+
+	pick_tmp = pdata;
+	while (pick_tmp && (pick_tmp2 = bcmstrtok(&pick_tmp, ",", 0)) != NULL) {
+		char *pEnd;
+		pch = bcmstrtok(&pick_tmp2, "=", 0);
+		if (!pch)
+			break;
+		if (*pch == ' ') {
+			pch++;
+		}
+		memset(name, 0 , sizeof (name));
+		cmd = bcm_strtoul(pch, &pEnd, 0);
+		if (cmd == 0 || strlen(pEnd)) {
+			cmd = WLC_SET_VAR;
+			strcpy(name, pch);
+		}
+		pch = bcmstrtok(&pick_tmp2, ",", 0);
+		if (!pch) {
+			break;
+		}
+		ret = iovar_tpl_parse(iovar_tpl_list, ARRAY_SIZE(iovar_tpl_list),
+			dhd, cmd, name, pch);
+		if (ret) {
+			val = (int)simple_strtol(pch, NULL, 0);
+			dhd_conf_set_intiovar(dhd, 0, cmd, name, val, -1, down);
+		}
+	}
+
+exit:
+	if (pdata)
+		kfree(pdata);
+	return true;
+}
+
+int
+dhd_conf_get_band(dhd_pub_t *dhd)
+{
+	int band = -1;
+
+	if (dhd && dhd->conf)
+		band = dhd->conf->band;
+	else
+		CONFIG_ERROR("dhd or conf is NULL\n");
+
+	return band;
+}
+
+int
+dhd_conf_set_roam(dhd_pub_t *dhd, int ifidx)
+{
+	int bcmerror = -1;
+	struct dhd_conf *conf = dhd->conf;
+	uint wnm_bsstrans_resp = 0;
+
+	if (dhd->conf->chip == BCM4359_CHIP_ID) {
+		dhd_conf_get_iovar(dhd, ifidx, WLC_GET_VAR, "wnm_bsstrans_resp",
+			(char *)&wnm_bsstrans_resp, sizeof(wnm_bsstrans_resp));
+		if (wnm_bsstrans_resp == WL_BSSTRANS_POLICY_PRODUCT) {
+			dhd->wbtext_policy = WL_BSSTRANS_POLICY_ROAM_ALWAYS;
+			dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "wnm_bsstrans_resp",
+				WL_BSSTRANS_POLICY_ROAM_ALWAYS, 0, FALSE);
+		}
+	}
+
+	dhd_roam_disable = conf->roam_off;
+	dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "roam_off", dhd->conf->roam_off, 0, FALSE);
+
+	if (!conf->roam_off || !conf->roam_off_suspend) {
+		CONFIG_MSG("set roam_trigger %d\n", conf->roam_trigger[0]);
+		dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_ROAM_TRIGGER, "WLC_SET_ROAM_TRIGGER",
+			(char *)conf->roam_trigger, sizeof(conf->roam_trigger), FALSE);
+
+		CONFIG_MSG("set roam_scan_period %d\n", conf->roam_scan_period[0]);
+		dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_ROAM_SCAN_PERIOD, "WLC_SET_ROAM_SCAN_PERIOD",
+			(char *)conf->roam_scan_period, sizeof(conf->roam_scan_period), FALSE);
+
+		CONFIG_MSG("set roam_delta %d\n", conf->roam_delta[0]);
+		dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_ROAM_DELTA, "WLC_SET_ROAM_DELTA",
+			(char *)conf->roam_delta, sizeof(conf->roam_delta), FALSE);
+
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "fullroamperiod",
+			dhd->conf->fullroamperiod, 1, FALSE);
+	}
+
+	return bcmerror;
+}
+
+void
+dhd_conf_add_to_eventbuffer(struct eventmsg_buf *ev, u16 event, bool set)
+{
+	if (!ev || (event > WLC_E_LAST))
+		return;
+
+	if (ev->num < MAX_EVENT_BUF_NUM) {
+		ev->event[ev->num].type = event;
+		ev->event[ev->num].set = set;
+		ev->num++;
+	} else {
+		CONFIG_ERROR("evenbuffer doesn't support > %u events. Update"
+			" the define MAX_EVENT_BUF_NUM \n", MAX_EVENT_BUF_NUM);
+		ASSERT(0);
+	}
+}
+
+s32
+dhd_conf_apply_eventbuffer(dhd_pub_t *dhd, eventmsg_buf_t *ev)
+{
+	char eventmask[WL_EVENTING_MASK_LEN];
+	int i, ret = 0;
+
+	if (!ev || (!ev->num))
+		return -EINVAL;
+
+	/* Read event_msgs mask */
+	ret = dhd_conf_get_iovar(dhd, 0, WLC_GET_VAR, "event_msgs", eventmask,
+		sizeof(eventmask));
+	if (unlikely(ret)) {
+		CONFIG_ERROR("Get event_msgs error (%d)\n", ret);
+		goto exit;
+	}
+
+	/* apply the set bits */
+	for (i = 0; i < ev->num; i++) {
+		if (ev->event[i].set)
+			setbit(eventmask, ev->event[i].type);
+		else
+			clrbit(eventmask, ev->event[i].type);
+	}
+
+	/* Write updated Event mask */
+	ret = dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "event_msgs", eventmask,
+		sizeof(eventmask), FALSE);
+	if (unlikely(ret)) {
+		CONFIG_ERROR("Set event_msgs error (%d)\n", ret);
+	}
+
+exit:
+	return ret;
+}
+
+static int
+dhd_conf_enable_roam_offload(dhd_pub_t *dhd, int enable)
+{
+	int err;
+	eventmsg_buf_t ev_buf;
+
+	if (dhd->conf->roam_off_suspend)
+		return 0;
+
+	err = dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "roam_offload", enable, 0, FALSE);
+	if (err)
+		return err;
+
+	bzero(&ev_buf, sizeof(eventmsg_buf_t));
+	dhd_conf_add_to_eventbuffer(&ev_buf, WLC_E_PSK_SUP, !enable);
+	dhd_conf_add_to_eventbuffer(&ev_buf, WLC_E_ASSOC_REQ_IE, !enable);
+	dhd_conf_add_to_eventbuffer(&ev_buf, WLC_E_ASSOC_RESP_IE, !enable);
+	dhd_conf_add_to_eventbuffer(&ev_buf, WLC_E_REASSOC, !enable);
+	dhd_conf_add_to_eventbuffer(&ev_buf, WLC_E_JOIN, !enable);
+	dhd_conf_add_to_eventbuffer(&ev_buf, WLC_E_ROAM, !enable);
+	err = dhd_conf_apply_eventbuffer(dhd, &ev_buf);
+
+	CONFIG_TRACE("roam_offload %d\n", enable);
+
+	return err;
+}
+
+void
+dhd_conf_set_bw_cap(dhd_pub_t *dhd)
+{
+	struct {
+		u32 band;
+		u32 bw_cap;
+	} param = {0, 0};
+
+	if (dhd->conf->bw_cap[0] >= 0) {
+		memset(&param, 0, sizeof(param));
+		param.band = WLC_BAND_2G;
+		param.bw_cap = (uint)dhd->conf->bw_cap[0];
+		CONFIG_MSG("set bw_cap 2g 0x%x\n", param.bw_cap);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "bw_cap", (char *)&param,
+			sizeof(param), TRUE);
+	}
+
+	if (dhd->conf->bw_cap[1] >= 0) {
+		memset(&param, 0, sizeof(param));
+		param.band = WLC_BAND_5G;
+		param.bw_cap = (uint)dhd->conf->bw_cap[1];
+		CONFIG_MSG("set bw_cap 5g 0x%x\n", param.bw_cap);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "bw_cap", (char *)&param,
+			sizeof(param), TRUE);
+	}
+}
+
+void
+dhd_conf_get_wme(dhd_pub_t *dhd, int ifidx, int mode, edcf_acparam_t *acp)
+{
+	int bcmerror = -1;
+	char iovbuf[WLC_IOCTL_SMLEN];
+	edcf_acparam_t *acparam;
+
+	bzero(iovbuf, sizeof(iovbuf));
+
+	/*
+	 * Get current acparams, using buf as an input buffer.
+	 * Return data is array of 4 ACs of wme params.
+	 */
+	if (mode == 0)
+		bcm_mkiovar("wme_ac_sta", NULL, 0, iovbuf, sizeof(iovbuf));
+	else
+		bcm_mkiovar("wme_ac_ap", NULL, 0, iovbuf, sizeof(iovbuf));
+	if ((bcmerror = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf, sizeof(iovbuf),
+			FALSE, ifidx)) < 0) {
+		CONFIG_ERROR("wme_ac_sta getting failed %d\n", bcmerror);
+		return;
+	}
+	memcpy((char*)acp, iovbuf, sizeof(edcf_acparam_t)*AC_COUNT);
+
+	acparam = &acp[AC_BK];
+	CONFIG_TRACE("BK: aci %d aifsn %d ecwmin %d ecwmax %d txop 0x%x\n",
+		acparam->ACI, acparam->ACI&EDCF_AIFSN_MASK,
+		acparam->ECW&EDCF_ECWMIN_MASK, (acparam->ECW&EDCF_ECWMAX_MASK)>>EDCF_ECWMAX_SHIFT,
+		acparam->TXOP);
+	acparam = &acp[AC_BE];
+	CONFIG_TRACE("BE: aci %d aifsn %d ecwmin %d ecwmax %d txop 0x%x\n",
+		acparam->ACI, acparam->ACI&EDCF_AIFSN_MASK,
+		acparam->ECW&EDCF_ECWMIN_MASK, (acparam->ECW&EDCF_ECWMAX_MASK)>>EDCF_ECWMAX_SHIFT,
+		acparam->TXOP);
+	acparam = &acp[AC_VI];
+	CONFIG_TRACE("VI: aci %d aifsn %d ecwmin %d ecwmax %d txop 0x%x\n",
+		acparam->ACI, acparam->ACI&EDCF_AIFSN_MASK,
+		acparam->ECW&EDCF_ECWMIN_MASK, (acparam->ECW&EDCF_ECWMAX_MASK)>>EDCF_ECWMAX_SHIFT,
+		acparam->TXOP);
+	acparam = &acp[AC_VO];
+	CONFIG_TRACE("VO: aci %d aifsn %d ecwmin %d ecwmax %d txop 0x%x\n",
+		acparam->ACI, acparam->ACI&EDCF_AIFSN_MASK,
+		acparam->ECW&EDCF_ECWMIN_MASK, (acparam->ECW&EDCF_ECWMAX_MASK)>>EDCF_ECWMAX_SHIFT,
+		acparam->TXOP);
+
+	return;
+}
+
+void
+dhd_conf_update_wme(dhd_pub_t *dhd, int ifidx, int mode,
+	edcf_acparam_t *acparam_cur, int aci)
+{
+	int aifsn, ecwmin, ecwmax, txop;
+	edcf_acparam_t *acp;
+	struct dhd_conf *conf = dhd->conf;
+	wme_param_t *wme;
+
+	if (mode == 0)
+		wme = &conf->wme_sta;
+	else
+		wme = &conf->wme_ap;
+
+	/* Default value */
+	aifsn = acparam_cur->ACI&EDCF_AIFSN_MASK;
+	ecwmin = acparam_cur->ECW&EDCF_ECWMIN_MASK;
+	ecwmax = (acparam_cur->ECW&EDCF_ECWMAX_MASK)>>EDCF_ECWMAX_SHIFT;
+	txop = acparam_cur->TXOP;
+
+	/* Modified value */
+	if (wme->aifsn[aci] > 0)
+		aifsn = wme->aifsn[aci];
+	if (wme->ecwmin[aci] > 0)
+		ecwmin = wme->ecwmin[aci];
+	if (wme->ecwmax[aci] > 0)
+		ecwmax = wme->ecwmax[aci];
+	if (wme->txop[aci] > 0)
+		txop = wme->txop[aci];
+
+	if (!(wme->aifsn[aci] || wme->ecwmin[aci] ||
+			wme->ecwmax[aci] || wme->txop[aci]))
+		return;
+
+	/* Update */
+	acp = acparam_cur;
+	acp->ACI = (acp->ACI & ~EDCF_AIFSN_MASK) | (aifsn & EDCF_AIFSN_MASK);
+	acp->ECW = ((ecwmax << EDCF_ECWMAX_SHIFT) & EDCF_ECWMAX_MASK) | (acp->ECW & EDCF_ECWMIN_MASK);
+	acp->ECW = ((acp->ECW & EDCF_ECWMAX_MASK) | (ecwmin & EDCF_ECWMIN_MASK));
+	acp->TXOP = txop;
+
+	CONFIG_MSG("wme_ac %s aci %d aifsn %d ecwmin %d ecwmax %d txop 0x%x\n",
+		mode?"ap":"sta", acp->ACI, acp->ACI&EDCF_AIFSN_MASK,
+		acp->ECW&EDCF_ECWMIN_MASK, (acp->ECW&EDCF_ECWMAX_MASK)>>EDCF_ECWMAX_SHIFT,
+		acp->TXOP);
+
+	/*
+	* Now use buf as an output buffer.
+	* Put WME acparams after "wme_ac\0" in buf.
+	* NOTE: only one of the four ACs can be set at a time.
+	*/
+	if (mode == 0)
+		dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "wme_ac_sta", (char *)acp,
+			sizeof(edcf_acparam_t), FALSE);
+	else
+		dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "wme_ac_ap", (char *)acp,
+			sizeof(edcf_acparam_t), FALSE);
+
+}
+
+void
+dhd_conf_set_wme(dhd_pub_t *dhd, int ifidx, int mode)
+{
+	edcf_acparam_t acparam_cur[AC_COUNT];
+
+	if (dhd && dhd->conf) {
+		if (!dhd->conf->force_wme_ac) {
+			CONFIG_TRACE("force_wme_ac is not enabled %d\n",
+				dhd->conf->force_wme_ac);
+			return;
+		}
+
+		CONFIG_TRACE("Before change:\n");
+		dhd_conf_get_wme(dhd, ifidx, mode, acparam_cur);
+
+		dhd_conf_update_wme(dhd, ifidx, mode, &acparam_cur[AC_BK], AC_BK);
+		dhd_conf_update_wme(dhd, ifidx, mode, &acparam_cur[AC_BE], AC_BE);
+		dhd_conf_update_wme(dhd, ifidx, mode, &acparam_cur[AC_VI], AC_VI);
+		dhd_conf_update_wme(dhd, ifidx, mode, &acparam_cur[AC_VO], AC_VO);
+
+		CONFIG_TRACE("After change:\n");
+		dhd_conf_get_wme(dhd, ifidx, mode, acparam_cur);
+	} else {
+		CONFIG_ERROR("dhd or conf is NULL\n");
+	}
+
+	return;
+}
+
+void
+dhd_conf_set_mchan_bw(dhd_pub_t *dhd, int p2p_mode, int miracast_mode)
+{
+	struct dhd_conf *conf = dhd->conf;
+	mchan_params_t *mchan = conf->mchan;
+	bool set = true;
+
+	while (mchan != NULL) {
+		set = true;
+		set &= (mchan->bw >= 0);
+		set &= ((mchan->p2p_mode == -1) | (mchan->p2p_mode == p2p_mode));
+		set &= ((mchan->miracast_mode == -1) | (mchan->miracast_mode == miracast_mode));
+		if (set) {
+			dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "mchan_bw", mchan->bw, 0, FALSE);
+		}
+		mchan = mchan->next;
+	}
+
+	return;
+}
+
+#ifdef PKT_FILTER_SUPPORT
+void
+dhd_conf_add_pkt_filter(dhd_pub_t *dhd)
+{
+	int i, j, magic_id;
+	char str[16], magic[20];
+#define MACS "%02x%02x%02x%02x%02x%02x"
+
+	/*  0) suspend_mode=1
+	 * Case 0: default is unicast pkt and event wake up
+	 * Case 1: no connection in suspend
+	 *   1) wl_suspend=52=0, 3=0
+	 *   2) wl_resume=2=0
+	 *   3) insuspend=0x7
+	 * Case 2: keep connection in suspend, but no pkt and event wake up
+	 *   1) dhd_master_mode=1
+	 *   2) pkt_filter_delete=100, 102, 103, 104, 105, 106, 107
+	 *   3) pkt_filter_add=141 0 0 0 0xFFFFFFFFFFFF 0x000000000000
+	 *   4) insuspend=0x7
+	 *   5) rekey_offload=1
+	 * Case 3: magic pkt and event wake up
+	 *   1) dhd_master_mode=1
+	 *   2) pkt_filter_delete=100, 102, 103, 104, 105, 106, 107
+	 *   3) magic_pkt_filter_add=141 0 1 12
+	 *   4) rekey_offload=1
+	 */
+	if (!dhd->conf->pkt_filter_cnt_default) {
+		dhd->conf->pkt_filter_cnt_default = dhd->pktfilter_count;
+		if (dhd->conf->magic_pkt_filter_add)
+			dhd->conf->magic_pkt_hdr_len = strlen(dhd->conf->magic_pkt_filter_add);
+	}
+	dhd->pktfilter_count = dhd->conf->pkt_filter_cnt_default;
+
+	for(i=0; i<dhd->conf->pkt_filter_add.count; i++) {
+		dhd->pktfilter[i+dhd->pktfilter_count] = dhd->conf->pkt_filter_add.filter[i];
+		CONFIG_MSG("%s\n", dhd->pktfilter[i+dhd->pktfilter_count]);
+	}
+	dhd->pktfilter_count += i;
+
+	if (dhd->conf->magic_pkt_filter_add) {
+		strncpy(magic, dhd->conf->magic_pkt_filter_add, dhd->conf->magic_pkt_hdr_len);
+		memset(dhd->conf->magic_pkt_filter_add, 0, MAGIC_PKT_FILTER_LEN);
+		strcpy(dhd->conf->magic_pkt_filter_add, magic);
+		magic_id = (int32)simple_strtol(magic, NULL, 10);
+		strcat(dhd->conf->magic_pkt_filter_add, " 0x");
+		strcat(dhd->conf->magic_pkt_filter_add, "FFFFFFFFFFFF");
+		for (j=0; j<16; j++)
+			strcat(dhd->conf->magic_pkt_filter_add, "FFFFFFFFFFFF");
+		strcat(dhd->conf->magic_pkt_filter_add, " 0x");
+		strcat(dhd->conf->magic_pkt_filter_add, "FFFFFFFFFFFF");
+		sprintf(str, MACS, MAC2STRDBG(dhd->mac.octet));
+		for (j=0; j<16; j++)
+			strncat(dhd->conf->magic_pkt_filter_add, str, 12);
+		dhd->pktfilter[dhd->pktfilter_count] = dhd->conf->magic_pkt_filter_add;
+		dhd_pktfilter_offload_delete(dhd, magic_id);
+		dhd_pktfilter_offload_set(dhd, dhd->pktfilter[dhd->pktfilter_count]);
+		dhd->pktfilter_count += 1;
+		CONFIG_MSG("%s\n", dhd->conf->magic_pkt_filter_add);
+	}
+}
+
+bool
+dhd_conf_del_pkt_filter(dhd_pub_t *dhd, uint32 id)
+{
+	int i;
+
+	if (dhd && dhd->conf) {
+		for (i=0; i<dhd->conf->pkt_filter_del.count; i++) {
+			if (id == dhd->conf->pkt_filter_del.id[i]) {
+				CONFIG_MSG("%d\n", dhd->conf->pkt_filter_del.id[i]);
+				return true;
+			}
+		}
+		return false;
+	}
+	return false;
+}
+
+void
+dhd_conf_discard_pkt_filter(dhd_pub_t *dhd)
+{
+	dhd->pktfilter_count = 6;
+	dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = NULL;
+	dhd->pktfilter[DHD_BROADCAST_FILTER_NUM] = "101 0 0 0 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF";
+	dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = "102 0 0 0 0xFFFFFF 0x01005E";
+	dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = "103 0 0 0 0xFFFF 0x3333";
+	dhd->pktfilter[DHD_MDNS_FILTER_NUM] = NULL;
+	/* Do not enable ARP to pkt filter if dhd_master_mode is false.*/
+	dhd->pktfilter[DHD_ARP_FILTER_NUM] = NULL;
+
+	/* IPv4 broadcast address XXX.XXX.XXX.255 */
+	dhd->pktfilter[dhd->pktfilter_count] = "110 0 0 12 0xFFFF00000000000000000000000000000000000000FF 0x080000000000000000000000000000000000000000FF";
+	dhd->pktfilter_count++;
+	/* discard IPv4 multicast address 224.0.0.0/4 */
+	dhd->pktfilter[dhd->pktfilter_count] = "111 0 0 12 0xFFFF00000000000000000000000000000000F0 0x080000000000000000000000000000000000E0";
+	dhd->pktfilter_count++;
+	/* discard IPv6 multicast address FF00::/8 */
+	dhd->pktfilter[dhd->pktfilter_count] = "112 0 0 12 0xFFFF000000000000000000000000000000000000000000000000FF 0x86DD000000000000000000000000000000000000000000000000FF";
+	dhd->pktfilter_count++;
+	/* discard Netbios pkt */
+	dhd->pktfilter[dhd->pktfilter_count] = "121 0 0 12 0xFFFF000000000000000000FF000000000000000000000000FFFF 0x0800000000000000000000110000000000000000000000000089";
+	dhd->pktfilter_count++;
+
+}
+#endif /* PKT_FILTER_SUPPORT */
+
+int
+dhd_conf_get_pm(dhd_pub_t *dhd)
+{
+	if (dhd && dhd->conf) {
+		return dhd->conf->pm;
+	}
+	return -1;
+}
+
+int
+dhd_conf_check_hostsleep(dhd_pub_t *dhd, int cmd, void *buf, int len,
+	int *hostsleep_set, int *hostsleep_val, int *ret)
+{
+	if (dhd->conf->insuspend & (NO_TXCTL_IN_SUSPEND | WOWL_IN_SUSPEND)) {
+		int wowl_dngldown = 0;
+#ifdef WL_EXT_WOWL
+		wowl_dngldown = dhd_conf_wowl_dngldown(dhd);
+#endif
+		if (cmd == WLC_SET_VAR) {
+			char *psleep = NULL;
+			if (wowl_dngldown) {
+				psleep = strstr(buf, "wowl_activate");
+				if (psleep) {
+					*hostsleep_set = 1;
+					memcpy(hostsleep_val, psleep+strlen("wowl_activate")+1, sizeof(int));
+				}
+			} else {
+				psleep = strstr(buf, "hostsleep");
+				if (psleep) {
+					*hostsleep_set = 1;
+					memcpy(hostsleep_val, psleep+strlen("hostsleep")+1, sizeof(int));
+				}
+			}
+		}
+		if (dhd->hostsleep && (!*hostsleep_set || *hostsleep_val)) {
+			CONFIG_TRACE("block all none hostsleep clr cmd\n");
+			*ret = BCME_EPERM;
+			goto exit;
+		} else if (*hostsleep_set && *hostsleep_val) {
+			CONFIG_TRACE("hostsleep %d => %d\n", dhd->hostsleep, *hostsleep_val);
+			dhd->hostsleep = *hostsleep_val;
+			if (dhd->conf->insuspend & NO_TXDATA_IN_SUSPEND) {
+				dhd_txflowcontrol(dhd, ALL_INTERFACES, ON);
+			}
+			if (dhd->hostsleep == 2) {
+				*ret = 0;
+				goto exit;
+			}
+		} else if (dhd->hostsleep == 2 && !*hostsleep_val) {
+			CONFIG_TRACE("hostsleep %d => %d\n", dhd->hostsleep, *hostsleep_val);
+			dhd->hostsleep = *hostsleep_val;
+			if (dhd->conf->insuspend & NO_TXDATA_IN_SUSPEND) {
+				dhd_txflowcontrol(dhd, ALL_INTERFACES, OFF);
+			}
+			*ret = 0;
+			goto exit;
+		}
+	}
+#ifdef NO_POWER_SAVE
+	if (cmd == WLC_SET_PM) {
+		if (*(const u32*)buf != 0) {
+			CONFIG_TRACE("skip PM\n");
+			*ret = BCME_OK;
+			goto exit;
+		}
+	} else if (cmd == WLC_SET_VAR) {
+		int cmd_len = strlen("mpc");
+		if (!strncmp(buf, "mpc", cmd_len)) {
+			if (*((u32 *)((u8*)buf+cmd_len+1)) != 0) {
+				CONFIG_TRACE("skip mpc\n");
+				*ret = BCME_OK;
+				goto exit;
+			}
+		}
+	}
+#endif
+
+	return 0;
+exit:
+	return -1;
+}
+
+void
+dhd_conf_get_hostsleep(dhd_pub_t *dhd,
+	int hostsleep_set, int hostsleep_val, int ret)
+{
+	if (dhd->conf->insuspend & (NO_TXCTL_IN_SUSPEND | WOWL_IN_SUSPEND)) {
+		if (hostsleep_set) {
+			if (hostsleep_val && ret) {
+				CONFIG_TRACE("reset hostsleep %d => 0\n", dhd->hostsleep);
+				dhd->hostsleep = 0;
+				if (dhd->conf->insuspend & NO_TXDATA_IN_SUSPEND) {
+					dhd_txflowcontrol(dhd, ALL_INTERFACES, OFF);
+				}
+			} else if (!hostsleep_val && !ret) {
+				CONFIG_TRACE("set hostsleep %d => 0\n", dhd->hostsleep);
+				dhd->hostsleep = 0;
+				if (dhd->conf->insuspend & NO_TXDATA_IN_SUSPEND) {
+					dhd_txflowcontrol(dhd, ALL_INTERFACES, OFF);
+				}
+			}
+		}
+	}
+}
+
+#ifdef WL_EXT_WOWL
+#define WL_WOWL_TCPFIN	(1 << 26)
+typedef struct wl_wowl_pattern2 {
+	char cmd[4];
+	wl_wowl_pattern_t wowl_pattern;
+} wl_wowl_pattern2_t;
+static int
+dhd_conf_wowl_pattern(dhd_pub_t *dhd, int ifidx, bool add, char *data)
+{
+	uint buf_len = 0;
+	int	id, type, polarity, offset;
+	char cmd[4]="\0", mask[128]="\0", pattern[128]="\0", mask_tmp[128]="\0", *pmask_tmp;
+	uint32 masksize, patternsize, pad_len = 0;
+	wl_wowl_pattern2_t *wowl_pattern2 = NULL;
+	char *mask_and_pattern;
+	int ret = 0, i, j, v;
+
+	if (data) {
+		if (add)
+			strcpy(cmd, "add");
+		else
+			strcpy(cmd, "clr");
+		if (!strcmp(cmd, "clr")) {
+			CONFIG_TRACE("wowl_pattern clr\n");
+			ret = dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "wowl_pattern", cmd,
+				sizeof(cmd), FALSE);
+			goto exit;
+		}
+		sscanf(data, "%d %d %d %d %s %s", &id, &type, &polarity, &offset,
+			mask_tmp, pattern);
+		masksize = strlen(mask_tmp) -2;
+		CONFIG_TRACE("0 mask_tmp=%s, masksize=%d\n", mask_tmp, masksize);
+
+		// add pading
+		if (masksize % 16)
+			pad_len = (16 - masksize % 16);
+		for (i=0; i<pad_len; i++)
+			strcat(mask_tmp, "0");
+		masksize += pad_len;
+		CONFIG_TRACE("1 mask_tmp=%s, masksize=%d\n", mask_tmp, masksize);
+
+		// translate 0x00 to 0, others to 1
+		j = 0;
+		pmask_tmp = &mask_tmp[2];
+		for (i=0; i<masksize/2; i++) {
+			if(strncmp(&pmask_tmp[i*2], "00", 2))
+				pmask_tmp[j] = '1';
+			else
+				pmask_tmp[j] = '0';
+			j++;
+		}
+		pmask_tmp[j] = '\0';
+		masksize = masksize / 2;
+		CONFIG_TRACE("2 mask_tmp=%s, masksize=%d\n", mask_tmp, masksize);
+
+		// reorder per 8bits
+		pmask_tmp = &mask_tmp[2];
+		for (i=0; i<masksize/8; i++) {
+			char c;
+			for (j=0; j<4; j++) {
+				c = pmask_tmp[i*8+j];
+				pmask_tmp[i*8+j] = pmask_tmp[(i+1)*8-j-1];
+				pmask_tmp[(i+1)*8-j-1] = c;
+			}
+		}
+		CONFIG_TRACE("3 mask_tmp=%s, masksize=%d\n", mask_tmp, masksize);
+
+		// translate 8bits to 1byte
+		j = 0; v = 0;
+		pmask_tmp = &mask_tmp[2];
+		strcpy(mask, "0x");
+		for (i=0; i<masksize; i++) {
+			v = (v<<1) | (pmask_tmp[i]=='1');
+			if (((i+1)%4) == 0) {
+				if (v < 10)
+					mask[j+2] = v + '0';
+				else
+					mask[j+2] = (v-10) + 'a';
+				j++;
+				v = 0;
+			}
+		}
+		mask[j+2] = '\0';
+		masksize = j/2;
+		CONFIG_TRACE("4 mask=%s, masksize=%d\n", mask, masksize);
+
+		patternsize = (strlen(pattern)-2)/2;
+		buf_len = sizeof(wl_wowl_pattern2_t) + patternsize + masksize;
+		wowl_pattern2 = kmalloc(buf_len, GFP_KERNEL);
+		if (wowl_pattern2 == NULL) {
+			CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", buf_len);
+			goto exit;
+		}
+		memset(wowl_pattern2, 0, sizeof(wl_wowl_pattern2_t));
+
+		strncpy(wowl_pattern2->cmd, cmd, sizeof(cmd));
+		wowl_pattern2->wowl_pattern.id = id;
+		wowl_pattern2->wowl_pattern.type = 0;
+		wowl_pattern2->wowl_pattern.offset = offset;
+		mask_and_pattern = (char*)wowl_pattern2 + sizeof(wl_wowl_pattern2_t);
+
+		wowl_pattern2->wowl_pattern.masksize = masksize;
+		ret = wl_pattern_atoh(mask, mask_and_pattern);
+		if (ret == -1) {
+			CONFIG_ERROR("rejecting mask=%s\n", mask);
+			goto exit;
+		}
+
+		mask_and_pattern += wowl_pattern2->wowl_pattern.masksize;
+		wowl_pattern2->wowl_pattern.patternoffset = sizeof(wl_wowl_pattern_t) +
+			wowl_pattern2->wowl_pattern.masksize;
+
+		wowl_pattern2->wowl_pattern.patternsize = patternsize;
+		ret = wl_pattern_atoh(pattern, mask_and_pattern);
+		if (ret == -1) {
+			CONFIG_ERROR("rejecting pattern=%s\n", pattern);
+			goto exit;
+		}
+
+		CONFIG_TRACE("%s %d %s %s\n", cmd, offset, mask, pattern);
+
+		ret = dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "wowl_pattern",
+			(char *)wowl_pattern2, buf_len, FALSE);
+	}
+
+exit:
+	if (wowl_pattern2)
+		kfree(wowl_pattern2);
+	return ret;
+}
+
+static int
+dhd_conf_wowl_wakeind(dhd_pub_t *dhd, int ifidx, bool clear)
+{
+	s8 iovar_buf[WLC_IOCTL_SMLEN];
+	wl_wowl_wakeind_t *wake = NULL;
+	int ret = -1;
+	char clr[6]="clear", wakeind_str[32]="\0";
+
+	if (clear) {
+		CONFIG_TRACE("wowl_wakeind clear\n");
+		ret = dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "wowl_wakeind",
+			clr, sizeof(clr), 0);
+	} else {
+		ret = dhd_conf_get_iovar(dhd, ifidx, WLC_GET_VAR, "wowl_wakeind",
+			iovar_buf, sizeof(iovar_buf));
+		if (!ret) {
+			wake = (wl_wowl_wakeind_t *) iovar_buf;
+			if (wake->ucode_wakeind & WL_WOWL_MAGIC)
+				strcpy(wakeind_str, "(MAGIC packet)");
+			if (wake->ucode_wakeind & WL_WOWL_NET)
+				strcpy(wakeind_str, "(Netpattern)");
+			if (wake->ucode_wakeind & WL_WOWL_DIS)
+				strcpy(wakeind_str, "(Disassoc/Deauth)");
+			if (wake->ucode_wakeind & WL_WOWL_BCN)
+				strcpy(wakeind_str, "(Loss of beacon)");
+			if (wake->ucode_wakeind & WL_WOWL_TCPKEEP_TIME)
+				strcpy(wakeind_str, "(TCPKA timeout)");
+			if (wake->ucode_wakeind & WL_WOWL_TCPKEEP_DATA)
+				strcpy(wakeind_str, "(TCPKA data)");
+			if (wake->ucode_wakeind & WL_WOWL_TCPFIN)
+				strcpy(wakeind_str, "(TCP FIN)");
+			CONFIG_MSG("wakeind=0x%x %s\n", wake->ucode_wakeind, wakeind_str);
+		}
+	}
+
+	return ret;
+}
+
+int
+dhd_conf_wowl_dngldown(dhd_pub_t *dhd)
+{
+	int wowl_dngldown = 0;
+#ifdef BCMDBUS
+	uint insuspend = 0;
+	insuspend = dhd_conf_get_insuspend(dhd, ALL_IN_SUSPEND);
+	if ((insuspend & WOWL_IN_SUSPEND) && dhd_master_mode) {
+		wowl_dngldown = dhd->conf->wowl_dngldown;
+	}
+#endif
+
+	return wowl_dngldown;
+}
+#endif
+
+int
+dhd_conf_mkeep_alive(dhd_pub_t *dhd, int ifidx, int id, int period,
+	char *packet, bool bcast)
+{
+	wl_mkeep_alive_pkt_v1_t *mkeep_alive_pktp;
+	int ret = 0, len_bytes=0, buf_len=0;
+	char *buf = NULL, *iovar_buf = NULL;
+	uint8 *pdata;
+
+	CONFIG_TRACE("id=%d, period=%d, packet=%s\n", id, period, packet);
+	if (period >= 0) {
+		buf = kmalloc(WLC_IOCTL_SMLEN, GFP_KERNEL);
+		if (buf == NULL) {
+			CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", WLC_IOCTL_SMLEN);
+			goto exit;
+		}
+		iovar_buf = kmalloc(WLC_IOCTL_SMLEN, GFP_KERNEL);
+		if (iovar_buf == NULL) {
+			CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", WLC_IOCTL_SMLEN);
+			goto exit;
+		}
+		mkeep_alive_pktp = (wl_mkeep_alive_pkt_v1_t *)buf;
+		mkeep_alive_pktp->version = htod16(WL_MKEEP_ALIVE_VERSION_1);
+		mkeep_alive_pktp->length = htod16(WL_MKEEP_ALIVE_FIXED_LEN);
+		mkeep_alive_pktp->keep_alive_id = id;
+		buf_len += WL_MKEEP_ALIVE_FIXED_LEN;
+		mkeep_alive_pktp->period_msec = period;
+		if (packet && strlen(packet)) {
+			len_bytes = wl_pattern_atoh(packet, (char *)mkeep_alive_pktp->data);
+			buf_len += len_bytes;
+			if (bcast) {
+				memcpy(mkeep_alive_pktp->data, &ether_bcast, ETHER_ADDR_LEN);
+			}
+			ret = dhd_conf_get_iovar(dhd, ifidx, WLC_GET_VAR, "cur_etheraddr",
+				iovar_buf, WLC_IOCTL_SMLEN);
+			if (!ret) {
+				pdata = mkeep_alive_pktp->data;
+				memcpy(pdata+6, iovar_buf, ETHER_ADDR_LEN);
+			}
+		}
+		mkeep_alive_pktp->len_bytes = htod16(len_bytes);
+		ret = dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "mkeep_alive",
+			buf, buf_len, FALSE);
+	}
+
+exit:
+	if (buf)
+		kfree(buf);
+	if (iovar_buf)
+		kfree(iovar_buf);
+	return ret;
+}
+
+#ifdef ARP_OFFLOAD_SUPPORT
+void
+dhd_conf_set_garp(dhd_pub_t *dhd, int ifidx, uint32 ipa, bool enable)
+{
+	int i, len = 0, total_len = WLC_IOCTL_SMLEN;
+	char *iovar_buf = NULL, *packet = NULL;
+
+	if (!dhd->conf->garp || ifidx != 0 || !(dhd->op_mode & DHD_FLAG_STA_MODE))
+		return;
+
+	CONFIG_TRACE("enable=%d\n", enable);
+
+	if (enable) {
+		iovar_buf = kmalloc(total_len, GFP_KERNEL);
+		if (iovar_buf == NULL) {
+			CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", total_len);
+			goto exit;
+		}
+		packet = kmalloc(total_len, GFP_KERNEL);
+		if (packet == NULL) {
+			CONFIG_ERROR("Failed to allocate buffer of %d bytes\n", total_len);
+			goto exit;
+		}
+		dhd_conf_get_iovar(dhd, ifidx, WLC_GET_VAR, "cur_etheraddr", iovar_buf, total_len);
+
+		len += snprintf(packet+len, total_len, "0xffffffffffff");
+		for (i=0; i<ETHER_ADDR_LEN; i++)
+			len += snprintf(packet+len, total_len, "%02x", iovar_buf[i]);
+		len += snprintf(packet+len, total_len, "08060001080006040001");
+		 // Sender Hardware Addr.
+		for (i=0; i<ETHER_ADDR_LEN; i++)
+			len += snprintf(packet+len, total_len, "%02x", iovar_buf[i]);
+		 // Sender IP Addr.
+		len += snprintf(packet+len, total_len, "%02x%02x%02x%02x",
+			ipa&0xff, (ipa>>8)&0xff, (ipa>>16)&0xff, (ipa>>24)&0xff);
+		 // Target Hardware Addr.
+		len += snprintf(packet+len, total_len, "ffffffffffff");
+		 // Target IP Addr.
+		len += snprintf(packet+len, total_len, "%02x%02x%02x%02x",
+			ipa&0xff, (ipa>>8)&0xff, (ipa>>16)&0xff, (ipa>>24)&0xff);
+		len += snprintf(packet+len, total_len, "000000000000000000000000000000000000");
+	}
+
+	dhd_conf_mkeep_alive(dhd, ifidx, 0, dhd->conf->keep_alive_period, packet, TRUE);
+
+exit:
+	if (iovar_buf)
+		kfree(iovar_buf);
+	if (packet)
+		kfree(packet);
+	return;
+}
+#endif
+
+uint
+dhd_conf_get_insuspend(dhd_pub_t *dhd, uint mask)
+{
+	uint insuspend = 0;
+
+	if (dhd->op_mode & DHD_FLAG_STA_MODE) {
+		insuspend = dhd->conf->insuspend &
+			(NO_EVENT_IN_SUSPEND | NO_TXDATA_IN_SUSPEND | NO_TXCTL_IN_SUSPEND |
+			ROAM_OFFLOAD_IN_SUSPEND | WOWL_IN_SUSPEND);
+	} else if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+		insuspend = dhd->conf->insuspend &
+			(NO_EVENT_IN_SUSPEND | NO_TXDATA_IN_SUSPEND | NO_TXCTL_IN_SUSPEND |
+			AP_DOWN_IN_SUSPEND | AP_FILTER_IN_SUSPEND);
+	}
+
+	return (insuspend & mask);
+}
+
+static void
+dhd_conf_check_connection(dhd_pub_t *dhd, int ifidx, int suspend)
+{
+	struct dhd_conf *conf = dhd->conf;
+	struct ether_addr bssid;
+	wl_event_msg_t msg;
+	int pm;
+#ifdef WL_CFG80211
+	struct net_device *net;
+	unsigned long flags = 0;
+#endif /* defined(WL_CFG80211) */
+
+	if (suspend) {
+		memset(&bssid, 0, ETHER_ADDR_LEN);
+		dhd_wl_ioctl_cmd(dhd, WLC_GET_BSSID, &bssid, ETHER_ADDR_LEN, FALSE, ifidx);
+		if (memcmp(&ether_null, &bssid, ETHER_ADDR_LEN))
+			memcpy(&conf->bssid_insuspend, &bssid, ETHER_ADDR_LEN);
+		else
+			memset(&conf->bssid_insuspend, 0, ETHER_ADDR_LEN);
+	}
+	else {
+		if (memcmp(&ether_null, &conf->bssid_insuspend, ETHER_ADDR_LEN)) {
+			memset(&bssid, 0, ETHER_ADDR_LEN);
+			dhd_wl_ioctl_cmd(dhd, WLC_GET_BSSID, &bssid, ETHER_ADDR_LEN, FALSE, ifidx);
+			if (memcmp(&ether_null, &bssid, ETHER_ADDR_LEN)) {
+				dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_PM, "WLC_SET_PM", 0, 0, FALSE);
+				dhd_conf_set_bufiovar(dhd, ifidx, WLC_SET_VAR, "send_nulldata",
+					(char *)&bssid, ETHER_ADDR_LEN, FALSE);
+				OSL_SLEEP(100);
+				if (conf->pm >= 0)
+					pm = conf->pm;
+				else
+					pm = PM_FAST;
+				dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_PM, "WLC_SET_PM", pm, 0, FALSE);
+			} else {
+				CONFIG_TRACE("send WLC_E_DEAUTH_IND event\n");
+				bzero(&msg, sizeof(wl_event_msg_t));
+				msg.ifidx = ifidx;
+				memcpy(&msg.addr, &conf->bssid_insuspend, ETHER_ADDR_LEN);
+				msg.event_type = hton32(WLC_E_DEAUTH_IND);
+				msg.status = 0;
+				msg.reason = hton32(DOT11_RC_DEAUTH_LEAVING);
+#ifdef WL_EVENT
+				wl_ext_event_send(dhd->event_params, &msg, NULL);
+#endif
+#ifdef WL_CFG80211
+				spin_lock_irqsave(&dhd->up_lock, flags);
+				net = dhd_idx2net(dhd, ifidx);
+				if (net && dhd->up) {
+					wl_cfg80211_event(net, &msg, NULL);
+				}
+				spin_unlock_irqrestore(&dhd->up_lock, flags);
+#endif /* defined(WL_CFG80211) */
+			}
+		}
+	}
+}
+
+#ifdef SUSPEND_EVENT
+static void
+dhd_conf_set_suspend_event(dhd_pub_t *dhd, int suspend)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char suspend_eventmask[WL_EVENTING_MASK_LEN];
+
+	CONFIG_TRACE("Enter\n");
+	if (suspend) {
+#ifdef PROP_TXSTATUS
+#if defined(BCMSDIO) || defined(BCMDBUS)
+		if (dhd->wlfc_enabled) {
+			dhd_wlfc_deinit(dhd);
+			conf->wlfc = TRUE;
+		} else {
+			conf->wlfc = FALSE;
+		}
+#endif /* BCMSDIO || BCMDBUS */
+#endif /* PROP_TXSTATUS */
+		dhd_conf_get_iovar(dhd, 0, WLC_GET_VAR, "event_msgs",
+			conf->resume_eventmask, sizeof(conf->resume_eventmask));
+		memset(suspend_eventmask, 0, sizeof(suspend_eventmask));
+		setbit(suspend_eventmask, WLC_E_ESCAN_RESULT);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "event_msgs",
+			suspend_eventmask, sizeof(suspend_eventmask), FALSE);
+	}
+	else {
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "event_msgs",
+			conf->resume_eventmask, sizeof(conf->resume_eventmask), FALSE);
+#ifdef PROP_TXSTATUS
+#if defined(BCMSDIO) || defined(BCMDBUS)
+		if (conf->wlfc) {
+			dhd_wlfc_init(dhd);
+			dhd_conf_set_intiovar(dhd, 0, WLC_UP, "WLC_UP", 0, 0, FALSE);
+		}
+#endif
+#endif /* PROP_TXSTATUS */
+	}
+
+}
+#endif
+
+int
+dhd_conf_suspend_resume_sta(dhd_pub_t *dhd, int ifidx, int suspend)
+{
+	struct dhd_conf *conf = dhd->conf;
+	uint insuspend = 0;
+	int pm;
+#ifdef WL_EXT_WOWL
+	int i;
+#endif
+
+	insuspend = dhd_conf_get_insuspend(dhd, ALL_IN_SUSPEND);
+	if (insuspend)
+		WL_MSG(dhd_ifname(dhd, ifidx), "suspend %d\n", suspend);
+
+	if (suspend) {
+		dhd_conf_check_connection(dhd, ifidx, suspend);
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "roam_off",
+			conf->roam_off_suspend, 0, FALSE);
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "bcn_li_dtim",
+			conf->suspend_bcn_li_dtim, 0, FALSE);
+		if (conf->pm_in_suspend >= 0)
+			pm = conf->pm_in_suspend;
+		else if (conf->pm >= 0)
+			pm = conf->pm;
+		else
+			pm = PM_FAST;
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_PM, "WLC_SET_PM", pm, 0, FALSE);
+#ifdef WL_EXT_WOWL
+		if ((insuspend & WOWL_IN_SUSPEND) && dhd_master_mode) {
+			dhd_conf_wowl_pattern(dhd, ifidx, FALSE, "clr");
+			for(i=0; i<conf->pkt_filter_add.count; i++) {
+				dhd_conf_wowl_pattern(dhd, ifidx, TRUE, conf->pkt_filter_add.filter[i]);
+			}
+			CONFIG_MSG("wowl = 0x%x\n", conf->wowl);
+			dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "wowl", conf->wowl, 0, FALSE);
+#ifdef BCMDBUS
+			CONFIG_MSG("wowl_dngldown = %d\n", conf->wowl_dngldown);
+			dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "wowl_dngldown", conf->wowl_dngldown, 1, FALSE);
+#endif
+			dhd_conf_wowl_wakeind(dhd, ifidx, TRUE);
+		}
+#endif
+	}
+	else {
+		dhd_conf_get_iovar(dhd, ifidx, WLC_GET_PM, "WLC_GET_PM", (char *)&pm, sizeof(pm));
+		CONFIG_TRACE("PM in suspend = %d\n", pm);
+		if (conf->pm >= 0)
+			pm = conf->pm;
+		else
+			pm = PM_FAST;
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_PM, "WLC_SET_PM", pm, 0, FALSE);
+#ifdef WL_EXT_WOWL
+		if (insuspend & WOWL_IN_SUSPEND) {
+			dhd_conf_wowl_wakeind(dhd, ifidx, FALSE);
+//			dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "wowl_activate", 0, 0, FALSE);
+			dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "wowl", 0, 0, FALSE);
+			dhd_conf_wowl_pattern(dhd, ifidx, FALSE, "clr");
+		}
+#endif
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "bcn_li_dtim", 0, 0, FALSE);
+		dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "roam_off",
+			conf->roam_off, 0, FALSE);
+		dhd_conf_check_connection(dhd, ifidx, suspend);
+	}
+
+	return 0;
+}
+
+#ifndef WL_EXT_IAPSTA
+static int
+dhd_conf_suspend_resume_ap(dhd_pub_t *dhd, int ifidx, int suspend)
+{
+	uint insuspend = 0;
+
+	insuspend = dhd_conf_get_insuspend(dhd, ALL_IN_SUSPEND);
+	if (insuspend)
+		WL_MSG(dhd_ifname(dhd, ifidx), "suspend %d\n", suspend);
+
+	if (suspend) {
+		if (insuspend & AP_DOWN_IN_SUSPEND) {
+			dhd_conf_set_intiovar(dhd, ifidx, WLC_DOWN, "WLC_DOWN", 1, 0, FALSE);
+		}
+	} else {
+		if (insuspend & AP_DOWN_IN_SUSPEND) {
+			dhd_conf_set_intiovar(dhd, ifidx, WLC_UP, "WLC_UP", 0, 0, FALSE);
+		}
+	}
+
+	return 0;
+}
+#endif /* !WL_EXT_IAPSTA */
+
+static int
+dhd_conf_suspend_resume_bus(dhd_pub_t *dhd, int suspend)
+{
+	uint insuspend = 0;
+
+	insuspend = dhd_conf_get_insuspend(dhd, ALL_IN_SUSPEND);
+	if (insuspend)
+		CONFIG_MSG("suspend %d\n", suspend);
+
+	if (suspend) {
+		if (insuspend & (WOWL_IN_SUSPEND | NO_TXCTL_IN_SUSPEND)) {
+			uint32 intstatus = 0;
+			int ret = 0, hostsleep = 2, wowl_dngldown = 0;
+#ifdef WL_EXT_WOWL
+			hostsleep = 1;
+			if ((insuspend & WOWL_IN_SUSPEND) && dhd_master_mode) {
+				dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "wowl_activate", 1, 0, FALSE);
+#ifdef BCMDBUS
+				wowl_dngldown = dhd->conf->wowl_dngldown;
+#endif
+			}
+#endif
+			if (!wowl_dngldown) {
+				dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "hostsleep", hostsleep, 0, FALSE);
+			}
+#ifdef BCMSDIO
+			ret = dhd_bus_sleep(dhd, TRUE, &intstatus);
+#endif
+#ifdef BCMPCIE
+			ret = dhd_bus_sleep(dhd, TRUE, NULL);
+#endif
+#ifdef BCMDBUS
+			if (wowl_dngldown) {
+				ret = dhd_bus_sleep(dhd, TRUE, NULL);
+			}
+#endif
+			CONFIG_MSG("ret = %d, intstatus = 0x%x\n", ret, intstatus);
+		}
+	} else {
+		if (insuspend & (WOWL_IN_SUSPEND | NO_TXCTL_IN_SUSPEND)) {
+			dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "hostsleep", 0, 0, FALSE);
+		}
+	}
+
+	return 0;
+}
+
+int
+dhd_conf_set_suspend_resume(dhd_pub_t *dhd, int suspend)
+{
+	struct dhd_conf *conf = dhd->conf;
+	uint insuspend = 0;
+
+	insuspend = dhd_conf_get_insuspend(dhd, ALL_IN_SUSPEND);
+	if (insuspend)
+		CONFIG_MSG("op_mode %d, suspend %d, suspended %d, insuspend 0x%x, suspend_mode=%d\n",
+			dhd->op_mode, suspend, conf->suspended, insuspend, conf->suspend_mode);
+
+	if (conf->suspended == suspend || !dhd->up) {
+		return 0;
+	}
+
+	if (suspend) {
+		if (insuspend & (NO_EVENT_IN_SUSPEND|NO_TXCTL_IN_SUSPEND|WOWL_IN_SUSPEND)) {
+			if (conf->suspend_mode == PM_NOTIFIER)
+#ifdef WL_EXT_IAPSTA
+				wl_iapsta_wait_event_complete(dhd);
+#else
+				wl_ext_wait_event_complete(dhd, 0);
+#endif /* WL_EXT_IAPSTA */
+		}
+		if (insuspend & NO_TXDATA_IN_SUSPEND) {
+			dhd_txflowcontrol(dhd, ALL_INTERFACES, ON);
+		}
+#if defined(WL_CFG80211) || defined(WL_ESCAN)
+		if (insuspend & (NO_EVENT_IN_SUSPEND|NO_TXCTL_IN_SUSPEND|WOWL_IN_SUSPEND)) {
+			if (conf->suspend_mode == PM_NOTIFIER)
+				wl_ext_user_sync(dhd, 0, TRUE);
+		}
+#endif
+		if (insuspend & ROAM_OFFLOAD_IN_SUSPEND)
+			dhd_conf_enable_roam_offload(dhd, 2);
+#ifdef SUSPEND_EVENT
+		if (insuspend & NO_EVENT_IN_SUSPEND) {
+			dhd_conf_set_suspend_event(dhd, suspend);
+		}
+#endif
+#ifdef WL_EXT_IAPSTA
+		wl_iapsta_suspend_resume(dhd, suspend);
+#else
+		if (dhd->op_mode & DHD_FLAG_STA_MODE) {
+			dhd_conf_suspend_resume_sta(dhd, 0, suspend);
+		} else if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+			dhd_conf_suspend_resume_ap(dhd, 0, suspend);
+		}
+#endif /* WL_EXT_IAPSTA */
+		dhd_conf_set_wl_cmd(dhd, conf->wl_suspend, FALSE);
+		dhd_conf_suspend_resume_bus(dhd, suspend);
+		conf->suspended = TRUE;
+	}
+	else {
+		dhd_conf_suspend_resume_bus(dhd, suspend);
+#ifdef SUSPEND_EVENT
+		if (insuspend & NO_EVENT_IN_SUSPEND) {
+			dhd_conf_set_suspend_event(dhd, suspend);
+		}
+#endif
+		if (insuspend & ROAM_OFFLOAD_IN_SUSPEND)
+			dhd_conf_enable_roam_offload(dhd, 0);
+		dhd_conf_set_wl_cmd(dhd, conf->wl_resume, FALSE);
+#ifdef WL_EXT_IAPSTA
+		wl_iapsta_suspend_resume(dhd, suspend);
+#else
+		if (dhd->op_mode & DHD_FLAG_STA_MODE) {
+			dhd_conf_suspend_resume_sta(dhd, 0, suspend);
+		} else if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+			dhd_conf_suspend_resume_ap(dhd, 0, suspend);
+		}
+#endif /* WL_EXT_IAPSTA */
+#if defined(WL_CFG80211) || defined(WL_ESCAN)
+		if (insuspend & (NO_EVENT_IN_SUSPEND|NO_TXCTL_IN_SUSPEND|WOWL_IN_SUSPEND)) {
+			if (conf->suspend_mode == PM_NOTIFIER)
+				wl_ext_user_sync(dhd, 0, FALSE);
+		}
+#endif
+		if (insuspend & NO_TXDATA_IN_SUSPEND) {
+			dhd_txflowcontrol(dhd, ALL_INTERFACES, OFF);
+		}
+		conf->suspended = FALSE;
+	}
+
+	return 0;
+}
+
+#ifdef PROP_TXSTATUS
+int
+dhd_conf_get_disable_proptx(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+	int disable_proptx = -1;
+	int fw_proptx = 0;
+
+	/* check fw proptx priority:
+	  * 1st: check fw support by wl cap
+	  * 2nd: 4334/43340/43341/43241 support proptx but not show in wl cap, so enable it by default
+	  * 	   if you would like to disable it, please set disable_proptx=1 in config.txt
+	  * 3th: disable when proptxstatus not support in wl cap
+	  */
+	if (FW_SUPPORTED(dhd, proptxstatus)) {
+		fw_proptx = 1;
+	} else if (conf->chip == BCM4334_CHIP_ID || conf->chip == BCM43340_CHIP_ID ||
+			dhd->conf->chip == BCM43340_CHIP_ID || conf->chip == BCM4324_CHIP_ID) {
+		fw_proptx = 1;
+	} else {
+		fw_proptx = 0;
+	}
+
+	/* returned disable_proptx value:
+	  * -1: disable in STA and enable in P2P(follow original dhd settings when PROP_TXSTATUS_VSDB enabled)
+	  * 0: depend on fw support
+	  * 1: always disable proptx
+	  */
+	if (conf->disable_proptx == 0) {
+		// check fw support as well
+		if (fw_proptx)
+			disable_proptx = 0;
+		else
+			disable_proptx = 1;
+	} else if (conf->disable_proptx >= 1) {
+		disable_proptx = 1;
+	} else {
+		// check fw support as well
+		if (fw_proptx)
+			disable_proptx = -1;
+		else
+			disable_proptx = 1;
+	}
+
+	CONFIG_MSG("fw_proptx=%d, disable_proptx=%d\n", fw_proptx, disable_proptx);
+
+	return disable_proptx;
+}
+#endif
+
+uint
+pick_config_vars(char *varbuf, uint len, uint start_pos, char *pickbuf, int picklen)
+{
+	bool findNewline, changenewline=FALSE, pick=FALSE;
+	int column;
+	uint n, pick_column=0;
+
+	findNewline = FALSE;
+	column = 0;
+
+	if (start_pos >= len) {
+		CONFIG_ERROR("wrong start pos\n");
+		return 0;
+	}
+
+	for (n = start_pos; n < len; n++) {
+		if (varbuf[n] == '\r')
+			continue;
+		if ((findNewline || changenewline) && varbuf[n] != '\n')
+			continue;
+		findNewline = FALSE;
+		if (varbuf[n] == '#') {
+			findNewline = TRUE;
+			continue;
+		}
+		if (varbuf[n] == '\\') {
+			changenewline = TRUE;
+			continue;
+		}
+		if (!changenewline && varbuf[n] == '\n') {
+			if (column == 0)
+				continue;
+			column = 0;
+			continue;
+		}
+		if (changenewline && varbuf[n] == '\n') {
+			changenewline = FALSE;
+			continue;
+		}
+
+		if (column==0 && !pick) { // start to pick
+			pick = TRUE;
+			column++;
+			pick_column = 0;
+		} else {
+			if (pick && column==0) { // stop to pick
+				pick = FALSE;
+				break;
+			} else
+				column++;
+		}
+		if (pick) {
+			if (varbuf[n] == 0x9)
+				continue;
+			if (pick_column >= picklen)
+				break;
+			pickbuf[pick_column] = varbuf[n];
+			pick_column++;
+		}
+	}
+
+	return n; // return current position
+}
+
+bool
+dhd_conf_read_chiprev(dhd_pub_t *dhd, int *chip_match,
+	char *full_param, uint len_param)
+{
+	char *data = full_param+len_param, *pick_tmp, *pch;
+	uint chip = 0, rev = 0;
+
+	/* Process chip, regrev:
+	 * chip=[chipid], rev==[rev]
+	 * Ex: chip=0x4359, rev=9
+	 */
+	if (!strncmp("chip=", full_param, len_param)) {
+		chip = (int)simple_strtol(data, NULL, 0);
+		pick_tmp = data;
+		pch = bcmstrstr(pick_tmp, "rev=");
+		if (pch) {
+			rev = (int)simple_strtol(pch+strlen("rev="), NULL, 0);
+		}
+		if (chip == dhd->conf->chip && rev == dhd->conf->chiprev)
+			*chip_match = 1;
+		else
+			*chip_match = 0;
+		CONFIG_MSG("chip=0x%x, rev=%d, chip_match=%d\n", chip, rev, *chip_match);
+	}
+
+	return TRUE;
+}
+
+bool
+dhd_conf_read_log_level(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	char *data = full_param+len_param;
+
+	if (!strncmp("dhd_msg_level=", full_param, len_param)) {
+		dhd_msg_level = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("dhd_msg_level = 0x%X\n", dhd_msg_level);
+	}
+	else if (!strncmp("dump_msg_level=", full_param, len_param)) {
+		dump_msg_level = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("dump_msg_level = 0x%X\n", dump_msg_level);
+	}
+#ifdef BCMSDIO
+	else if (!strncmp("sd_msglevel=", full_param, len_param)) {
+		sd_msglevel = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("sd_msglevel = 0x%X\n", sd_msglevel);
+	}
+#endif
+#ifdef BCMDBUS
+	else if (!strncmp("dbus_msglevel=", full_param, len_param)) {
+		dbus_msglevel = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("dbus_msglevel = 0x%X\n", dbus_msglevel);
+	}
+#endif
+	else if (!strncmp("android_msg_level=", full_param, len_param)) {
+		android_msg_level = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("android_msg_level = 0x%X\n", android_msg_level);
+	}
+	else if (!strncmp("config_msg_level=", full_param, len_param)) {
+		config_msg_level = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("config_msg_level = 0x%X\n", config_msg_level);
+	}
+#ifdef WL_CFG80211
+	else if (!strncmp("wl_dbg_level=", full_param, len_param)) {
+		wl_dbg_level = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("wl_dbg_level = 0x%X\n", wl_dbg_level);
+	}
+#endif
+#if defined(WL_WIRELESS_EXT)
+	else if (!strncmp("iw_msg_level=", full_param, len_param)) {
+		iw_msg_level = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("iw_msg_level = 0x%X\n", iw_msg_level);
+	}
+#endif
+#if defined(DHD_DEBUG)
+	else if (!strncmp("dhd_console_ms=", full_param, len_param)) {
+		dhd->dhd_console_ms = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("dhd_console_ms = %d\n", dhd->dhd_console_ms);
+	}
+#endif
+	else
+		return false;
+
+	return true;
+}
+
+void
+dhd_conf_read_wme_ac_value(wme_param_t *wme, char *pick, int ac_val)
+{
+	char *pick_tmp, *pch;
+
+	pick_tmp = pick;
+	pch = bcmstrstr(pick_tmp, "aifsn ");
+	if (pch) {
+		wme->aifsn[ac_val] = (int)simple_strtol(pch+strlen("aifsn "), NULL, 0);
+		CONFIG_MSG("ac_val=%d, aifsn=%d\n", ac_val, wme->aifsn[ac_val]);
+	}
+	pick_tmp = pick;
+	pch = bcmstrstr(pick_tmp, "ecwmin ");
+	if (pch) {
+		wme->ecwmin[ac_val] = (int)simple_strtol(pch+strlen("ecwmin "), NULL, 0);
+		CONFIG_MSG("ac_val=%d, ecwmin=%d\n", ac_val, wme->ecwmin[ac_val]);
+	}
+	pick_tmp = pick;
+	pch = bcmstrstr(pick_tmp, "ecwmax ");
+	if (pch) {
+		wme->ecwmax[ac_val] = (int)simple_strtol(pch+strlen("ecwmax "), NULL, 0);
+		CONFIG_MSG("ac_val=%d, ecwmax=%d\n", ac_val, wme->ecwmax[ac_val]);
+	}
+	pick_tmp = pick;
+	pch = bcmstrstr(pick_tmp, "txop ");
+	if (pch) {
+		wme->txop[ac_val] = (int)simple_strtol(pch+strlen("txop "), NULL, 0);
+		CONFIG_MSG("ac_val=%d, txop=0x%x\n", ac_val, wme->txop[ac_val]);
+	}
+
+}
+
+bool
+dhd_conf_read_wme_ac_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	// wme_ac_sta_be=aifsn 1 ecwmin 2 ecwmax 3 txop 0x5e
+	// wme_ac_sta_vo=aifsn 1 ecwmin 1 ecwmax 1 txop 0x5e
+
+	if (!strncmp("force_wme_ac=", full_param, len_param)) {
+		conf->force_wme_ac = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("force_wme_ac = %d\n", conf->force_wme_ac);
+	}
+	else if (!strncmp("wme_ac_sta_be=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_sta, data, AC_BE);
+	}
+	else if (!strncmp("wme_ac_sta_bk=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_sta, data, AC_BK);
+	}
+	else if (!strncmp("wme_ac_sta_vi=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_sta, data, AC_VI);
+	}
+	else if (!strncmp("wme_ac_sta_vo=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_sta, data, AC_VO);
+	}
+	else if (!strncmp("wme_ac_ap_be=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_ap, data, AC_BE);
+	}
+	else if (!strncmp("wme_ac_ap_bk=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_ap, data, AC_BK);
+	}
+	else if (!strncmp("wme_ac_ap_vi=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_ap, data, AC_VI);
+	}
+	else if (!strncmp("wme_ac_ap_vo=", full_param, len_param)) {
+		dhd_conf_read_wme_ac_value(&conf->wme_ap, data, AC_VO);
+	}
+	else
+		return false;
+
+	return true;
+}
+
+#ifdef SET_FWNV_BY_MAC
+bool
+dhd_conf_read_fw_by_mac(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	int i, j;
+	char *pch, *pick_tmp;
+	wl_mac_list_t *mac_list;
+	wl_mac_range_t *mac_range;
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	/* Process fw_by_mac:
+	 * fw_by_mac=[fw_mac_num] \
+	 *  [fw_name1] [mac_num1] [oui1-1] [nic_start1-1] [nic_end1-1] \
+	 *                                    [oui1-1] [nic_start1-1] [nic_end1-1]... \
+	 *                                    [oui1-n] [nic_start1-n] [nic_end1-n] \
+	 *  [fw_name2] [mac_num2] [oui2-1] [nic_start2-1] [nic_end2-1] \
+	 *                                    [oui2-1] [nic_start2-1] [nic_end2-1]... \
+	 *                                    [oui2-n] [nic_start2-n] [nic_end2-n] \
+	 * Ex: fw_by_mac=2 \
+	 *  fw_bcmdhd1.bin 2 0x0022F4 0xE85408 0xE8549D 0x983B16 0x3557A9 0x35582A \
+	 *  fw_bcmdhd2.bin 3 0x0022F4 0xE85408 0xE8549D 0x983B16 0x3557A9 0x35582A \
+	 *                           0x983B16 0x916157 0x916487
+	 */
+
+	if (!strncmp("fw_by_mac=", full_param, len_param)) {
+		dhd_conf_free_mac_list(&conf->fw_by_mac);
+		pick_tmp = data;
+		pch = bcmstrtok(&pick_tmp, " ", 0);
+		conf->fw_by_mac.count = (uint32)simple_strtol(pch, NULL, 0);
+		if (!(mac_list = kmalloc(sizeof(wl_mac_list_t)*conf->fw_by_mac.count,
+				GFP_KERNEL))) {
+			conf->fw_by_mac.count = 0;
+			CONFIG_ERROR("kmalloc failed\n");
+		}
+		CONFIG_MSG("fw_count=%d\n", conf->fw_by_mac.count);
+		conf->fw_by_mac.m_mac_list_head = mac_list;
+		for (i=0; i<conf->fw_by_mac.count; i++) {
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			strcpy(mac_list[i].name, pch);
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			mac_list[i].count = (uint32)simple_strtol(pch, NULL, 0);
+			CONFIG_MSG("name=%s, mac_count=%d\n",
+				mac_list[i].name, mac_list[i].count);
+			if (!(mac_range = kmalloc(sizeof(wl_mac_range_t)*mac_list[i].count,
+					GFP_KERNEL))) {
+				mac_list[i].count = 0;
+				CONFIG_ERROR("kmalloc failed\n");
+				break;
+			}
+			mac_list[i].mac = mac_range;
+			for (j=0; j<mac_list[i].count; j++) {
+				pch = bcmstrtok(&pick_tmp, " ", 0);
+				mac_range[j].oui = (uint32)simple_strtol(pch, NULL, 0);
+				pch = bcmstrtok(&pick_tmp, " ", 0);
+				mac_range[j].nic_start = (uint32)simple_strtol(pch, NULL, 0);
+				pch = bcmstrtok(&pick_tmp, " ", 0);
+				mac_range[j].nic_end = (uint32)simple_strtol(pch, NULL, 0);
+				CONFIG_MSG("oui=0x%06X, nic_start=0x%06X, nic_end=0x%06X\n",
+					mac_range[j].oui, mac_range[j].nic_start, mac_range[j].nic_end);
+			}
+		}
+	}
+	else
+		return false;
+
+	return true;
+}
+
+bool
+dhd_conf_read_nv_by_mac(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	int i, j;
+	char *pch, *pick_tmp;
+	wl_mac_list_t *mac_list;
+	wl_mac_range_t *mac_range;
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	/* Process nv_by_mac:
+	 * [nv_by_mac]: The same format as fw_by_mac
+	 */
+	if (!strncmp("nv_by_mac=", full_param, len_param)) {
+		dhd_conf_free_mac_list(&conf->nv_by_mac);
+		pick_tmp = data;
+		pch = bcmstrtok(&pick_tmp, " ", 0);
+		conf->nv_by_mac.count = (uint32)simple_strtol(pch, NULL, 0);
+		if (!(mac_list = kmalloc(sizeof(wl_mac_list_t)*conf->nv_by_mac.count,
+				GFP_KERNEL))) {
+			conf->nv_by_mac.count = 0;
+			CONFIG_ERROR("kmalloc failed\n");
+		}
+		CONFIG_MSG("nv_count=%d\n", conf->nv_by_mac.count);
+		conf->nv_by_mac.m_mac_list_head = mac_list;
+		for (i=0; i<conf->nv_by_mac.count; i++) {
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			strcpy(mac_list[i].name, pch);
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			mac_list[i].count = (uint32)simple_strtol(pch, NULL, 0);
+			CONFIG_MSG("name=%s, mac_count=%d\n",
+				mac_list[i].name, mac_list[i].count);
+			if (!(mac_range = kmalloc(sizeof(wl_mac_range_t)*mac_list[i].count,
+					GFP_KERNEL))) {
+				mac_list[i].count = 0;
+				CONFIG_ERROR("kmalloc failed\n");
+				break;
+			}
+			mac_list[i].mac = mac_range;
+			for (j=0; j<mac_list[i].count; j++) {
+				pch = bcmstrtok(&pick_tmp, " ", 0);
+				mac_range[j].oui = (uint32)simple_strtol(pch, NULL, 0);
+				pch = bcmstrtok(&pick_tmp, " ", 0);
+				mac_range[j].nic_start = (uint32)simple_strtol(pch, NULL, 0);
+				pch = bcmstrtok(&pick_tmp, " ", 0);
+				mac_range[j].nic_end = (uint32)simple_strtol(pch, NULL, 0);
+				CONFIG_MSG("oui=0x%06X, nic_start=0x%06X, nic_end=0x%06X\n",
+					mac_range[j].oui, mac_range[j].nic_start, mac_range[j].nic_end);
+			}
+		}
+	}
+	else
+		return false;
+
+	return true;
+}
+#endif
+
+bool
+dhd_conf_read_nv_by_chip(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	int i;
+	char *pch, *pick_tmp;
+	wl_chip_nv_path_t *chip_nv_path;
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	/* Process nv_by_chip:
+	 * nv_by_chip=[nv_chip_num] \
+	 *  [chip1] [chiprev1] [nv_name1] [chip2] [chiprev2] [nv_name2] \
+	 * Ex: nv_by_chip=2 \
+	 *  43430 0 nvram_ap6212.txt 43430 1 nvram_ap6212a.txt \
+	 */
+	if (!strncmp("nv_by_chip=", full_param, len_param)) {
+		dhd_conf_free_chip_nv_path_list(&conf->nv_by_chip);
+		pick_tmp = data;
+		pch = bcmstrtok(&pick_tmp, " ", 0);
+		conf->nv_by_chip.count = (uint32)simple_strtol(pch, NULL, 0);
+		if (!(chip_nv_path = kmalloc(sizeof(wl_chip_nv_path_t)*conf->nv_by_chip.count,
+				GFP_KERNEL))) {
+			conf->nv_by_chip.count = 0;
+			CONFIG_ERROR("kmalloc failed\n");
+		}
+		CONFIG_MSG("nv_by_chip_count=%d\n", conf->nv_by_chip.count);
+		conf->nv_by_chip.m_chip_nv_path_head = chip_nv_path;
+		for (i=0; i<conf->nv_by_chip.count; i++) {
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			chip_nv_path[i].chip = (uint32)simple_strtol(pch, NULL, 0);
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			chip_nv_path[i].chiprev = (uint32)simple_strtol(pch, NULL, 0);
+			pch = bcmstrtok(&pick_tmp, " ", 0);
+			strcpy(chip_nv_path[i].name, pch);
+			CONFIG_MSG("chip=0x%x, chiprev=%d, name=%s\n",
+				chip_nv_path[i].chip, chip_nv_path[i].chiprev, chip_nv_path[i].name);
+		}
+	}
+	else
+		return false;
+
+	return true;
+}
+
+bool
+dhd_conf_read_roam_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	if (!strncmp("roam_off=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->roam_off = 0;
+		else
+			conf->roam_off = 1;
+		CONFIG_MSG("roam_off = %d\n", conf->roam_off);
+	}
+	else if (!strncmp("roam_off_suspend=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->roam_off_suspend = 0;
+		else
+			conf->roam_off_suspend = 1;
+		CONFIG_MSG("roam_off_suspend = %d\n", conf->roam_off_suspend);
+	}
+	else if (!strncmp("roam_trigger=", full_param, len_param)) {
+		conf->roam_trigger[0] = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("roam_trigger = %d\n", conf->roam_trigger[0]);
+	}
+	else if (!strncmp("roam_scan_period=", full_param, len_param)) {
+		conf->roam_scan_period[0] = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("roam_scan_period = %d\n", conf->roam_scan_period[0]);
+	}
+	else if (!strncmp("roam_delta=", full_param, len_param)) {
+		conf->roam_delta[0] = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("roam_delta = %d\n", conf->roam_delta[0]);
+	}
+	else if (!strncmp("fullroamperiod=", full_param, len_param)) {
+		conf->fullroamperiod = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("fullroamperiod = %d\n", conf->fullroamperiod);
+	}
+	else if (!strncmp("wl_reassoc_support=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			wl_reassoc_support = FALSE;
+		else
+			wl_reassoc_support = TRUE;
+		CONFIG_MSG("wl_reassoc_support = %d\n", wl_reassoc_support);
+	}
+	else
+		return false;
+
+	return true;
+}
+
+bool
+dhd_conf_read_country(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	country_list_t *country_next = NULL, *country;
+	int i, count = 0;
+	char *pch, *pick_tmp, *pick_tmp2;
+	char *data = full_param+len_param;
+	uint len_data = strlen(data);
+
+	/* Process country_list:
+	 * country_list=[country1]:[ccode1]/[regrev1],
+	 * [country2]:[ccode2]/[regrev2] \
+	 * Ex: country_list=US:US/0, TW:TW/1
+	 */
+	/* Process ccode_all_list:
+	 * ccode_all_list=[ccode1]:[2g1]-[5g1]-[6g1], [ccode2]:[2g2]-[5g2]-[6g2], \
+	 * Ex: ccode_all_list=AU:2g3-5g0820-6gE, TW:2g3-5g0E04-6gF, US:2g3
+	 */
+	if (!strncmp("ccode=", full_param, len_param)) {
+		len_data = min((uint)WLC_CNTRY_BUF_SZ, len_data);
+		memset(&conf->cspec, 0, sizeof(wl_country_t));
+		memcpy(conf->cspec.country_abbrev, data, len_data);
+		memcpy(conf->cspec.ccode, data, len_data);
+		CONFIG_MSG("ccode = %s\n", conf->cspec.ccode);
+	}
+	else if (!strncmp("regrev=", full_param, len_param)) {
+		conf->cspec.rev = (int32)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("regrev = %d\n", conf->cspec.rev);
+	}
+	else if (!strncmp("ccode_all=", full_param, len_param)) {
+		len_data = min((uint)WLC_CNTRY_BUF_SZ, len_data);
+		memset(&conf->ccode_all.cspec, 0, sizeof(wl_country_t));
+		memcpy(conf->ccode_all.cspec.country_abbrev, data, len_data);
+		memcpy(conf->ccode_all.cspec.ccode, data, len_data);
+		CONFIG_MSG("ccode_all = %s\n", conf->ccode_all.cspec.ccode);
+	}
+	else if (!strncmp("ccode_all_list=", full_param, len_param)) {
+		if (conf->ccode_all_list) {
+			kfree(conf->ccode_all_list);
+			conf->ccode_all_list = NULL;
+		}
+		if (!(conf->ccode_all_list = kzalloc(len_data+1, GFP_KERNEL))) {
+			CONFIG_ERROR("kmalloc failed\n");
+			return TRUE;
+		}
+		memcpy(conf->ccode_all_list, data, len_data);
+		CONFIG_MSG("ccode_all_list size = %d\n", len_data);
+	}
+	else if (!strncmp("country_list=", full_param, len_param)) {
+		dhd_conf_free_country_list(conf);
+		pick_tmp = data;
+		for (i=0; i<CONFIG_COUNTRY_LIST_SIZE; i++) {
+			pick_tmp2 = bcmstrtok(&pick_tmp, ", ", 0);
+			if (!pick_tmp2)
+				break;
+			pch = bcmstrtok(&pick_tmp2, ":", 0);
+			if (!pch)
+				break;
+			country = NULL;
+			if (!(country = kmalloc(sizeof(country_list_t), GFP_KERNEL))) {
+				CONFIG_ERROR("kmalloc failed\n");
+				break;
+			}
+			memset(country, 0, sizeof(country_list_t));
+
+			memcpy(country->cspec.country_abbrev, pch, 2);
+			pch = bcmstrtok(&pick_tmp2, "/", 0);
+			if (!pch) {
+				kfree(country);
+				break;
+			}
+			memcpy(country->cspec.ccode, pch, 2);
+			pch = bcmstrtok(&pick_tmp2, "/", 0);
+			if (!pch) {
+				kfree(country);
+				break;
+			}
+			country->cspec.rev = (int32)simple_strtol(pch, NULL, 10);
+			count++;
+			if (!conf->country_head) {
+				conf->country_head = country;
+				country_next = country;
+			} else {
+				country_next->next = country;
+				country_next = country;
+			}
+			CONFIG_TRACE("abbrev=%s, ccode=%s, regrev=%d\n",
+				country->cspec.country_abbrev, country->cspec.ccode, country->cspec.rev);
+		}
+		CONFIG_MSG("%d country in list\n", count);
+	}
+	else
+		return false;
+
+	return true;
+}
+
+bool
+dhd_conf_read_mchan_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	int i;
+	char *pch, *pick_tmp, *pick_tmp2;
+	struct dhd_conf *conf = dhd->conf;
+	mchan_params_t *mchan_next = NULL, *mchan;
+	char *data = full_param+len_param;
+
+	/* Process mchan_bw:
+	 * mchan_bw=[val]/[any/go/gc]/[any/source/sink]
+	 * Ex: mchan_bw=80/go/source, 30/gc/sink
+	 */
+	if (!strncmp("mchan_bw=", full_param, len_param)) {
+		dhd_conf_free_mchan_list(conf);
+		pick_tmp = data;
+		for (i=0; i<MCHAN_MAX_NUM; i++) {
+			pick_tmp2 = bcmstrtok(&pick_tmp, ", ", 0);
+			if (!pick_tmp2)
+				break;
+			pch = bcmstrtok(&pick_tmp2, "/", 0);
+			if (!pch)
+				break;
+
+			mchan = NULL;
+			if (!(mchan = kmalloc(sizeof(mchan_params_t), GFP_KERNEL))) {
+				CONFIG_ERROR("kmalloc failed\n");
+				break;
+			}
+			memset(mchan, 0, sizeof(mchan_params_t));
+
+			mchan->bw = (int)simple_strtol(pch, NULL, 0);
+			if (mchan->bw < 0 || mchan->bw > 100) {
+				CONFIG_ERROR("wrong bw %d\n", mchan->bw);
+				kfree(mchan);
+				break;
+			}
+
+			pch = bcmstrtok(&pick_tmp2, "/", 0);
+			if (!pch) {
+				kfree(mchan);
+				break;
+			} else {
+				if (bcmstrstr(pch, "any")) {
+					mchan->p2p_mode = -1;
+				} else if (bcmstrstr(pch, "go")) {
+					mchan->p2p_mode = WL_P2P_IF_GO;
+				} else if (bcmstrstr(pch, "gc")) {
+					mchan->p2p_mode = WL_P2P_IF_CLIENT;
+				}
+			}
+			pch = bcmstrtok(&pick_tmp2, "/", 0);
+			if (!pch) {
+				kfree(mchan);
+				break;
+			} else {
+				if (bcmstrstr(pch, "any")) {
+					mchan->miracast_mode = -1;
+				} else if (bcmstrstr(pch, "source")) {
+					mchan->miracast_mode = MIRACAST_SOURCE;
+				} else if (bcmstrstr(pch, "sink")) {
+					mchan->miracast_mode = MIRACAST_SINK;
+				}
+			}
+			if (!conf->mchan) {
+				conf->mchan = mchan;
+				mchan_next = mchan;
+			} else {
+				mchan_next->next = mchan;
+				mchan_next = mchan;
+			}
+			CONFIG_TRACE("mchan_bw=%d/%d/%d\n", mchan->bw,mchan->p2p_mode,
+				mchan->miracast_mode);
+		}
+	}
+	else
+		return false;
+
+	return true;
+}
+
+#ifdef PKT_FILTER_SUPPORT
+bool
+dhd_conf_read_pkt_filter(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+	char *pch, *pick_tmp;
+	int i;
+
+	/* Process pkt filter:
+	 * 1) pkt_filter_add=99 0 0 0 0x000000000000 0x000000000000
+	 * 2) pkt_filter_delete=100, 102, 103, 104, 105
+	 * 3) magic_pkt_filter_add=141 0 1 12
+	 */
+	if (!strncmp("dhd_master_mode=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			dhd_master_mode = FALSE;
+		else
+			dhd_master_mode = TRUE;
+		CONFIG_MSG("dhd_master_mode = %d\n", dhd_master_mode);
+	}
+	else if (!strncmp("pkt_filter_add=", full_param, len_param)) {
+		pick_tmp = data;
+		pch = bcmstrtok(&pick_tmp, ",.-", 0);
+		i=0;
+		while (pch != NULL && i<DHD_CONF_FILTER_MAX) {
+			strcpy(&conf->pkt_filter_add.filter[i][0], pch);
+			CONFIG_MSG("pkt_filter_add[%d][] = %s\n",
+				i, &conf->pkt_filter_add.filter[i][0]);
+			pch = bcmstrtok(&pick_tmp, ",.-", 0);
+			i++;
+		}
+		conf->pkt_filter_add.count = i;
+	}
+	else if (!strncmp("pkt_filter_delete=", full_param, len_param) ||
+			!strncmp("pkt_filter_del=", full_param, len_param)) {
+		pick_tmp = data;
+		pch = bcmstrtok(&pick_tmp, " ,.-", 0);
+		i=0;
+		while (pch != NULL && i<DHD_CONF_FILTER_MAX) {
+			conf->pkt_filter_del.id[i] = (uint32)simple_strtol(pch, NULL, 10);
+			pch = bcmstrtok(&pick_tmp, " ,.-", 0);
+			i++;
+		}
+		conf->pkt_filter_del.count = i;
+		CONFIG_MSG("pkt_filter_del id = ");
+		for (i=0; i<conf->pkt_filter_del.count; i++)
+			printk(KERN_CONT "%d ", conf->pkt_filter_del.id[i]);
+		printk(KERN_CONT "\n");
+	}
+	else if (!strncmp("magic_pkt_filter_add=", full_param, len_param)) {
+		if (conf->magic_pkt_filter_add) {
+			kfree(conf->magic_pkt_filter_add);
+			conf->magic_pkt_filter_add = NULL;
+		}
+		if (!(conf->magic_pkt_filter_add = kmalloc(MAGIC_PKT_FILTER_LEN, GFP_KERNEL))) {
+			CONFIG_ERROR("kmalloc failed\n");
+		} else {
+			memset(conf->magic_pkt_filter_add, 0, MAGIC_PKT_FILTER_LEN);
+			strcpy(conf->magic_pkt_filter_add, data);
+			CONFIG_MSG("magic_pkt_filter_add = %s\n", conf->magic_pkt_filter_add);
+		}
+	}
+	else
+		return false;
+
+	return true;
+}
+#endif
+
+#ifdef ISAM_PREINIT
+#if !defined(WL_EXT_IAPSTA)
+#error "WL_EXT_IAPSTA should be defined to enable ISAM_PREINIT"
+#endif /* !WL_EXT_IAPSTA */
+/*
+ * isam_init=mode [sta|ap|apsta|dualap] vifname [wlan1]
+ * isam_config=ifname [wlan0|wlan1] ssid [xxx] chan [x]
+		 hidden [y|n] maxassoc [x]
+		 amode [open|shared|wpapsk|wpa2psk|wpawpa2psk]
+		 emode [none|wep|tkip|aes|tkipaes]
+		 key [xxxxx]
+ * isam_enable=ifname [wlan0|wlan1]
+*/
+bool
+dhd_conf_read_isam(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	if (!strncmp("isam_init=", full_param, len_param)) {
+		sprintf(conf->isam_init, "isam_init %s", data);
+		CONFIG_MSG("isam_init=%s\n", conf->isam_init);
+	}
+	else if (!strncmp("isam_config=", full_param, len_param)) {
+		sprintf(conf->isam_config, "isam_config %s", data);
+		CONFIG_MSG("isam_config=%s\n", conf->isam_config);
+	}
+	else if (!strncmp("isam_enable=", full_param, len_param)) {
+		sprintf(conf->isam_enable, "isam_enable %s", data);
+		CONFIG_MSG("isam_enable=%s\n", conf->isam_enable);
+	}
+	else
+		return false;
+
+	return true;
+}
+#endif
+
+#ifdef IDHCP
+bool
+dhd_conf_read_dhcp_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+	struct ipv4_addr ipa_set;
+
+	if (!strncmp("dhcpc_enable=", full_param, len_param)) {
+		conf->dhcpc_enable = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhcpc_enable = %d\n", conf->dhcpc_enable);
+	}
+	else if (!strncmp("dhcpd_enable=", full_param, len_param)) {
+		conf->dhcpd_enable = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhcpd_enable = %d\n", conf->dhcpd_enable);
+	}
+	else if (!strncmp("dhcpd_ip_addr=", full_param, len_param)) {
+		if (!bcm_atoipv4(data, &ipa_set)) {
+			CONFIG_ERROR("dhcpd_ip_addr adress setting failed.n");
+			return false;
+		}
+		memcpy(&conf->dhcpd_ip_addr, &ipa_set, sizeof(struct ipv4_addr));
+		CONFIG_MSG("dhcpd_ip_addr = %s\n", data);
+	}
+	else if (!strncmp("dhcpd_ip_mask=", full_param, len_param)) {
+		if (!bcm_atoipv4(data, &ipa_set)) {
+			CONFIG_ERROR("dhcpd_ip_mask adress setting failed\n");
+			return false;
+		}
+		memcpy(&conf->dhcpd_ip_mask, &ipa_set, sizeof(struct ipv4_addr));
+		CONFIG_MSG("dhcpd_ip_mask = %s\n", data);
+	}
+	else if (!strncmp("dhcpd_ip_start=", full_param, len_param)) {
+		if (!bcm_atoipv4(data, &ipa_set)) {
+			CONFIG_ERROR("dhcpd_ip_start adress setting failed\n");
+			return false;
+		}
+		memcpy(&conf->dhcpd_ip_start, &ipa_set, sizeof(struct ipv4_addr));
+		CONFIG_MSG("dhcpd_ip_start = %s\n", data);
+	}
+	else if (!strncmp("dhcpd_ip_end=", full_param, len_param)) {
+		if (!bcm_atoipv4(data, &ipa_set)) {
+			CONFIG_ERROR("dhcpd_ip_end adress setting failed\n");
+			return false;
+		}
+		memcpy(&conf->dhcpd_ip_end, &ipa_set, sizeof(struct ipv4_addr));
+		CONFIG_MSG("dhcpd_ip_end = %s\n", data);
+	}
+	else
+		return false;
+
+	return true;
+}
+#endif
+
+#ifdef BCMSDIO
+bool
+dhd_conf_read_sdio_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	if (!strncmp("dhd_doflow=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			dhd_doflow = FALSE;
+		else
+			dhd_doflow = TRUE;
+		CONFIG_MSG("dhd_doflow = %d\n", dhd_doflow);
+	}
+	else if (!strncmp("dhd_slpauto=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1)) {
+			dhd_conf_disable_slpauto(dhd);
+		} else
+			dhd_slpauto = TRUE;
+		CONFIG_MSG("dhd_slpauto = %d\n", dhd_slpauto);
+	}
+	else if (!strncmp("use_rxchain=", full_param, len_param)) {
+		conf->use_rxchain = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("use_rxchain = %d\n", conf->use_rxchain);
+	}
+	else if (!strncmp("dhd_txminmax=", full_param, len_param)) {
+		conf->dhd_txminmax = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhd_txminmax = %d\n", conf->dhd_txminmax);
+	}
+	else if (!strncmp("txinrx_thres=", full_param, len_param)) {
+		conf->txinrx_thres = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("txinrx_thres = %d\n", conf->txinrx_thres);
+	}
+#ifdef DYNAMIC_MAX_HDR_READ
+	else if (!strncmp("max_hdr_read=", full_param, len_param)) {
+		conf->max_hdr_read = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("max_hdr_read = %d\n", conf->max_hdr_read);
+	}
+	else if (!strncmp("dhd_firstread=", full_param, len_param)) {
+		firstread = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhd_firstread = %d\n", firstread);
+	}
+#endif
+#if defined(HW_OOB)
+	else if (!strncmp("oob_enabled_later=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->oob_enabled_later = FALSE;
+		else
+			conf->oob_enabled_later = TRUE;
+		CONFIG_MSG("oob_enabled_later = %d\n", conf->oob_enabled_later);
+	}
+#endif
+	else if (!strncmp("dpc_cpucore=", full_param, len_param)) {
+		conf->dpc_cpucore = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dpc_cpucore = %d\n", conf->dpc_cpucore);
+	}
+	else if (!strncmp("rxf_cpucore=", full_param, len_param)) {
+		conf->rxf_cpucore = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("rxf_cpucore = %d\n", conf->rxf_cpucore);
+	}
+	else if (!strncmp("dhd_dpc_prio=", full_param, len_param)) {
+		conf->dhd_dpc_prio = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhd_dpc_prio = %d\n", conf->dhd_dpc_prio);
+	}
+#if defined(BCMSDIOH_TXGLOM)
+	else if (!strncmp("txglomsize=", full_param, len_param)) {
+		conf->txglomsize = (uint)simple_strtol(data, NULL, 10);
+		if (conf->txglomsize > SDPCM_MAXGLOM_SIZE)
+			conf->txglomsize = SDPCM_MAXGLOM_SIZE;
+		CONFIG_MSG("txglomsize = %d\n", conf->txglomsize);
+	}
+	else if (!strncmp("txglom_ext=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->txglom_ext = FALSE;
+		else
+			conf->txglom_ext = TRUE;
+		CONFIG_MSG("txglom_ext = %d\n", conf->txglom_ext);
+		if (conf->txglom_ext) {
+			if ((conf->chip == BCM43362_CHIP_ID) || (conf->chip == BCM4330_CHIP_ID))
+				conf->txglom_bucket_size = 1680;
+			else if (conf->chip == BCM43340_CHIP_ID || conf->chip == BCM43341_CHIP_ID ||
+					conf->chip == BCM4334_CHIP_ID || conf->chip == BCM4324_CHIP_ID)
+				conf->txglom_bucket_size = 1684;
+		}
+		CONFIG_MSG("txglom_bucket_size = %d\n", conf->txglom_bucket_size);
+	}
+	else if (!strncmp("bus:rxglom=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->bus_rxglom = FALSE;
+		else
+			conf->bus_rxglom = TRUE;
+		CONFIG_MSG("bus:rxglom = %d\n", conf->bus_rxglom);
+	}
+	else if (!strncmp("deferred_tx_len=", full_param, len_param)) {
+		conf->deferred_tx_len = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("deferred_tx_len = %d\n", conf->deferred_tx_len);
+	}
+	else if (!strncmp("txctl_tmo_fix=", full_param, len_param)) {
+		conf->txctl_tmo_fix = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("txctl_tmo_fix = %d\n", conf->txctl_tmo_fix);
+	}
+	else if (!strncmp("tx_max_offset=", full_param, len_param)) {
+		conf->tx_max_offset = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("tx_max_offset = %d\n", conf->tx_max_offset);
+	}
+	else if (!strncmp("txglom_mode=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->txglom_mode = FALSE;
+		else
+			conf->txglom_mode = TRUE;
+		CONFIG_MSG("txglom_mode = %d\n", conf->txglom_mode);
+	}
+#if defined(SDIO_ISR_THREAD)
+	else if (!strncmp("intr_extn=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->intr_extn = FALSE;
+		else
+			conf->intr_extn = TRUE;
+		CONFIG_MSG("intr_extn = %d\n", conf->intr_extn);
+	}
+#endif
+#ifdef BCMSDIO_RXLIM_POST
+	else if (!strncmp("rxlim_en=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->rxlim_en = FALSE;
+		else
+			conf->rxlim_en = TRUE;
+		CONFIG_MSG("rxlim_en = %d\n", conf->rxlim_en);
+	}
+#endif
+#ifdef BCMSDIO_TXSEQ_SYNC
+	else if (!strncmp("txseq_sync=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->txseq_sync = FALSE;
+		else
+			conf->txseq_sync = TRUE;
+		CONFIG_MSG("txseq_sync = %d\n", conf->txseq_sync);
+	}
+#endif
+#endif
+#ifdef MINIME
+	else if (!strncmp("ramsize=", full_param, len_param)) {
+		conf->ramsize = (uint32)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("ramsize = %d\n", conf->ramsize);
+	}
+#endif
+#ifdef BCMSDIO_INTSTATUS_WAR
+	else if (!strncmp("read_intr_mode=", full_param, len_param)) {
+		conf->read_intr_mode = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("read_intr_mode = %d\n", conf->read_intr_mode);
+	}
+#endif
+	else if (!strncmp("kso_try_max=", full_param, len_param)) {
+		conf->kso_try_max = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("kso_try_max = %d\n", conf->kso_try_max);
+	}
+	else
+		return false;
+
+	return true;
+}
+#endif
+
+#ifdef BCMPCIE
+bool
+dhd_conf_read_pcie_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	if (!strncmp("bus:deepsleep_disable=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->bus_deepsleep_disable = 0;
+		else
+			conf->bus_deepsleep_disable = 1;
+		CONFIG_MSG("bus:deepsleep_disable = %d\n", conf->bus_deepsleep_disable);
+	}
+	else if (!strncmp("flow_ring_queue_threshold=", full_param, len_param)) {
+		conf->flow_ring_queue_threshold = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("flow_ring_queue_threshold = %d\n", conf->flow_ring_queue_threshold);
+	}
+	else if (!strncmp("d2h_intr_control=", full_param, len_param)) {
+		conf->d2h_intr_control = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("d2h_intr_control = %d\n", conf->d2h_intr_control);
+	}
+	else if (!strncmp("enq_hdr_pkt=", full_param, len_param)) {
+		conf->enq_hdr_pkt = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("enq_hdr_pkt = 0x%x\n", conf->enq_hdr_pkt);
+	}
+	else if (!strncmp("aspm=", full_param, len_param)) {
+		conf->aspm = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("aspm = %d\n", conf->aspm);
+	}
+	else if (!strncmp("l1ss=", full_param, len_param)) {
+		conf->l1ss = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("l1ss = %d\n", conf->l1ss);
+	}
+	else
+		return false;
+
+	return true;
+}
+#endif
+
+bool
+dhd_conf_read_pm_params(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+
+	if (!strncmp("deepsleep=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->deepsleep = TRUE;
+		else
+			conf->deepsleep = FALSE;
+		CONFIG_MSG("deepsleep = %d\n", conf->deepsleep);
+	}
+	else if (!strncmp("PM=", full_param, len_param)) {
+		conf->pm = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("PM = %d\n", conf->pm);
+	}
+	else if (!strncmp("pm_in_suspend=", full_param, len_param)) {
+		conf->pm_in_suspend = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("pm_in_suspend = %d\n", conf->pm_in_suspend);
+	}
+	else if (!strncmp("suspend_mode=", full_param, len_param)) {
+		conf->suspend_mode = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("suspend_mode = %d\n", conf->suspend_mode);
+		if (conf->suspend_mode == EARLY_SUSPEND)
+			conf->insuspend &= ~(NO_TXDATA_IN_SUSPEND | NO_TXCTL_IN_SUSPEND);
+		else if (conf->suspend_mode == PM_NOTIFIER ||
+				conf->suspend_mode == SUSPEND_MODE_2)
+			conf->insuspend |= (NO_TXDATA_IN_SUSPEND | NO_TXCTL_IN_SUSPEND);
+		CONFIG_MSG("insuspend = 0x%x\n", conf->insuspend);
+	}
+	else if (!strncmp("suspend_bcn_li_dtim=", full_param, len_param)) {
+		conf->suspend_bcn_li_dtim = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("suspend_bcn_li_dtim = %d\n", conf->suspend_bcn_li_dtim);
+	}
+	else if (!strncmp("xmit_in_suspend=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->insuspend &= ~NO_TXDATA_IN_SUSPEND;
+		else
+			conf->insuspend |= NO_TXDATA_IN_SUSPEND;
+		CONFIG_MSG("insuspend = 0x%x\n", conf->insuspend);
+	}
+	else if (!strncmp("insuspend=", full_param, len_param)) {
+		conf->insuspend = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("insuspend = 0x%x\n", conf->insuspend);
+	}
+#ifdef WL_EXT_WOWL
+	else if (!strncmp("wowl=", full_param, len_param)) {
+		conf->wowl = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("wowl = 0x%x\n", conf->wowl);
+	}
+#ifdef BCMDBUS
+	else if (!strncmp("wowl_dngldown=", full_param, len_param)) {
+		conf->wowl_dngldown = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("wowl_dngldown = 0x%x\n", conf->wowl_dngldown);
+	}
+#endif
+#endif
+	else if (!strncmp("rekey_offload=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->rekey_offload = TRUE;
+		else
+			conf->rekey_offload = FALSE;
+		CONFIG_MSG("rekey_offload = %d\n", conf->rekey_offload);
+	}
+	else
+		return false;
+
+	return true;
+}
+
+#ifdef GET_CUSTOM_MAC_FROM_CONFIG
+int
+bcm_str2hex(const char *p, char *ea, int size)
+{
+	int i = 0;
+	char *ep;
+
+	for (;;) {
+		ea[i++] = (char) bcm_strtoul(p, &ep, 16);
+		p = ep;
+		if (!*p++ || i == size)
+			break;
+	}
+
+	return (i == size);
+}
+#endif
+
+bool
+dhd_conf_read_others(dhd_pub_t *dhd, char *full_param, uint len_param)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char *data = full_param+len_param;
+	char *pch, *pick_tmp;
+#ifdef GET_CUSTOM_MAC_FROM_CONFIG
+	struct ether_addr ea_addr;
+	char macpad[56];
+#endif
+
+	if (!strncmp("dhd_poll=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->dhd_poll = 0;
+		else
+			conf->dhd_poll = 1;
+		CONFIG_MSG("dhd_poll = %d\n", conf->dhd_poll);
+	}
+	else if (!strncmp("dhd_watchdog_ms=", full_param, len_param)) {
+		dhd_watchdog_ms = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhd_watchdog_ms = %d\n", dhd_watchdog_ms);
+	}
+	else if (!strncmp("band=", full_param, len_param)) {
+		/* Process band:
+		 * band=a for 5GHz only and band=b for 2.4GHz only
+		 */
+		if (!strcmp(data, "b"))
+			conf->band = WLC_BAND_2G;
+		else if (!strcmp(data, "a"))
+			conf->band = WLC_BAND_5G;
+		else
+			conf->band = WLC_BAND_AUTO;
+		CONFIG_MSG("band = %d\n", conf->band);
+	}
+	else if (!strncmp("bw_cap_2g=", full_param, len_param)) {
+		conf->bw_cap[0] = (uint)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("bw_cap_2g = %d\n", conf->bw_cap[0]);
+	}
+	else if (!strncmp("bw_cap_5g=", full_param, len_param)) {
+		conf->bw_cap[1] = (uint)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("bw_cap_5g = %d\n", conf->bw_cap[1]);
+	}
+	else if (!strncmp("bw_cap=", full_param, len_param)) {
+		pick_tmp = data;
+		pch = bcmstrtok(&pick_tmp, " ,.-", 0);
+		if (pch != NULL) {
+			conf->bw_cap[0] = (uint32)simple_strtol(pch, NULL, 0);
+			CONFIG_MSG("bw_cap 2g = %d\n", conf->bw_cap[0]);
+		}
+		pch = bcmstrtok(&pick_tmp, " ,.-", 0);
+		if (pch != NULL) {
+			conf->bw_cap[1] = (uint32)simple_strtol(pch, NULL, 0);
+			CONFIG_MSG("bw_cap 5g = %d\n", conf->bw_cap[1]);
+		}
+	}
+	else if (!strncmp("keep_alive_period=", full_param, len_param)) {
+		conf->keep_alive_period = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("keep_alive_period = %d\n", conf->keep_alive_period);
+	}
+#ifdef ARP_OFFLOAD_SUPPORT
+	else if (!strncmp("garp=", full_param, len_param)) {
+		if (!strncmp(data, "0", 1))
+			conf->garp = FALSE;
+		else
+			conf->garp = TRUE;
+		CONFIG_MSG("garp = %d\n", conf->garp);
+	}
+#endif
+	else if (!strncmp("srl=", full_param, len_param)) {
+		conf->srl = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("srl = %d\n", conf->srl);
+	}
+	else if (!strncmp("lrl=", full_param, len_param)) {
+		conf->lrl = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("lrl = %d\n", conf->lrl);
+	}
+	else if (!strncmp("bcn_timeout=", full_param, len_param)) {
+		conf->bcn_timeout= (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("bcn_timeout = %d\n", conf->bcn_timeout);
+	}
+	else if (!strncmp("frameburst=", full_param, len_param)) {
+		conf->frameburst = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("frameburst = %d\n", conf->frameburst);
+	}
+	else if (!strncmp("disable_proptx=", full_param, len_param)) {
+		conf->disable_proptx = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("disable_proptx = %d\n", conf->disable_proptx);
+	}
+#ifdef DHDTCPACK_SUPPRESS
+	else if (!strncmp("tcpack_sup_mode=", full_param, len_param)) {
+		conf->tcpack_sup_mode = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("tcpack_sup_mode = %d\n", conf->tcpack_sup_mode);
+	}
+	else if (!strncmp("tcpack_sup_ratio=", full_param, len_param)) {
+		conf->tcpack_sup_ratio = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("tcpack_sup_ratio = %d\n", conf->tcpack_sup_ratio);
+	}
+	else if (!strncmp("tcpack_sup_delay=", full_param, len_param)) {
+		conf->tcpack_sup_delay = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("tcpack_sup_delay = %d\n", conf->tcpack_sup_delay);
+	}
+#endif
+	else if (!strncmp("pktprio8021x=", full_param, len_param)) {
+		conf->pktprio8021x = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("pktprio8021x = %d\n", conf->pktprio8021x);
+	}
+#if defined(BCMSDIO)
+	else if (!strncmp("dhd_txbound=", full_param, len_param)) {
+		dhd_txbound = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhd_txbound = %d\n", dhd_txbound);
+	}
+	else if (!strncmp("dhd_rxbound=", full_param, len_param)) {
+		dhd_rxbound = (uint)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("dhd_rxbound = %d\n", dhd_rxbound);
+	}
+#endif
+	else if (!strncmp("orphan_move=", full_param, len_param)) {
+		conf->orphan_move = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("orphan_move = %d\n", conf->orphan_move);
+	}
+	else if (!strncmp("tsq=", full_param, len_param)) {
+		conf->tsq = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("tsq = %d\n", conf->tsq);
+	}
+	else if (!strncmp("ctrl_resched=", full_param, len_param)) {
+		conf->ctrl_resched = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("ctrl_resched = %d\n", conf->ctrl_resched);
+	}
+	else if (!strncmp("rxcnt_timeout=", full_param, len_param)) {
+		conf->rxcnt_timeout = (int)simple_strtol(data, NULL, 10);
+		CONFIG_MSG("rxcnt_timeout = %d\n", conf->rxcnt_timeout);
+	}
+	else if (!strncmp("in4way=", full_param, len_param)) {
+		conf->in4way = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("in4way = 0x%x\n", conf->in4way);
+	}
+#ifdef BTC_WAR
+	else if (!strncmp("btc_war=", full_param, len_param)) {
+		btc_war = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("btc_war = 0x%x\n", btc_war);
+	}
+#endif /* BTC_WAR */
+	else if (!strncmp("war=", full_param, len_param)) {
+		conf->war = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("war = 0x%x\n", conf->war);
+	}
+	else if (!strncmp("wl_preinit=", full_param, len_param)) {
+		if (conf->wl_preinit) {
+			kfree(conf->wl_preinit);
+			conf->wl_preinit = NULL;
+		}
+		if (!(conf->wl_preinit = kmalloc(strlen(data)+1, GFP_KERNEL))) {
+			CONFIG_ERROR("kmalloc failed\n");
+		} else {
+			memset(conf->wl_preinit, 0, strlen(data)+1);
+			strcpy(conf->wl_preinit, data);
+			CONFIG_MSG("wl_preinit = %s\n", conf->wl_preinit);
+		}
+	}
+	else if (!strncmp("wl_suspend=", full_param, len_param)) {
+		if (conf->wl_suspend) {
+			kfree(conf->wl_suspend);
+			conf->wl_suspend = NULL;
+		}
+		if (!(conf->wl_suspend = kmalloc(strlen(data)+1, GFP_KERNEL))) {
+			CONFIG_ERROR("kmalloc failed\n");
+		} else {
+			memset(conf->wl_suspend, 0, strlen(data)+1);
+			strcpy(conf->wl_suspend, data);
+			CONFIG_MSG("wl_suspend = %s\n", conf->wl_suspend);
+		}
+	}
+	else if (!strncmp("wl_resume=", full_param, len_param)) {
+		if (conf->wl_resume) {
+			kfree(conf->wl_resume);
+			conf->wl_resume = NULL;
+		}
+		if (!(conf->wl_resume = kmalloc(strlen(data)+1, GFP_KERNEL))) {
+			CONFIG_ERROR("kmalloc failed\n");
+		} else {
+			memset(conf->wl_resume, 0, strlen(data)+1);
+			strcpy(conf->wl_resume, data);
+			CONFIG_MSG("wl_resume = %s\n", conf->wl_resume);
+		}
+	}
+#ifdef GET_CUSTOM_MAC_FROM_CONFIG
+	else if (!strncmp("mac=", full_param, len_param)) {
+		if (!bcm_ether_atoe(data, &ea_addr)) {
+			CONFIG_ERROR("mac adress read error");
+			return false;
+		}
+		memcpy(&conf->hw_ether, &ea_addr, ETHER_ADDR_LEN);
+		CONFIG_MSG("mac = %s\n", data);
+	}
+	else if (!strncmp("macpad=", full_param, len_param)) {
+		if (!bcm_str2hex(data, macpad, sizeof(macpad))) {
+			CONFIG_ERROR("macpad adress read error");
+			return false;
+		}
+		memcpy(&conf->hw_ether[ETHER_ADDR_LEN], macpad, sizeof(macpad));
+		if (config_msg_level & CONFIG_TRACE_LEVEL) {
+			CONFIG_MSG("macpad =\n");
+			for (i=0; i<sizeof(macpad); i++) {
+				printk(KERN_CONT "0x%02x, ", conf->hw_ether[ETHER_ADDR_LEN+i]);
+				if ((i+1)%8 == 0)
+					printk(KERN_CONT "\n");
+			}
+		}
+	}
+#endif
+#ifdef PROPTX_MAXCOUNT
+	else if (!strncmp("proptx_maxcnt_2g=", full_param, len_param)) {
+		conf->proptx_maxcnt_2g = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("proptx_maxcnt_2g = %d\n", conf->proptx_maxcnt_2g);
+	}
+	else if (!strncmp("proptx_maxcnt_5g=", full_param, len_param)) {
+		conf->proptx_maxcnt_5g = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("proptx_maxcnt_5g = %d\n", conf->proptx_maxcnt_5g);
+	}
+#endif
+#ifdef TPUT_MONITOR
+	else if (!strncmp("data_drop_mode=", full_param, len_param)) {
+		conf->data_drop_mode = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("data_drop_mode = %d\n", conf->data_drop_mode);
+	}
+	else if (!strncmp("tput_monitor_ms=", full_param, len_param)) {
+		conf->tput_monitor_ms = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("tput_monitor_ms = %d\n", conf->tput_monitor_ms);
+	}
+#ifdef BCMSDIO
+	else if (!strncmp("doflow_tput_thresh=", full_param, len_param)) {
+		conf->doflow_tput_thresh = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("doflow_tput_thresh = %d\n", conf->doflow_tput_thresh);
+		if (conf->doflow_tput_thresh > 0)
+			conf->tput_monitor_ms = 1000;
+	}
+#endif
+#endif
+#ifdef SCAN_SUPPRESS
+	else if (!strncmp("scan_intput=", full_param, len_param)) {
+		conf->scan_intput = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("scan_intput = 0x%x\n", conf->scan_intput);
+	}
+	else if (!strncmp("scan_tput_thresh=", full_param, len_param)) {
+		conf->scan_tput_thresh = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("scan_tput_thresh = %d\n", conf->scan_tput_thresh);
+		if (conf->scan_tput_thresh > 0)
+			conf->tput_monitor_ms = 1000;
+	}
+	else if (!strncmp("scan_busy_tmo=", full_param, len_param)) {
+		conf->scan_busy_tmo = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("scan_busy_tmo = %d\n", conf->scan_busy_tmo);
+	}
+	else if (!strncmp("scan_busy_thresh=", full_param, len_param)) {
+		conf->scan_busy_thresh = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("scan_busy_thresh = %d\n", conf->scan_busy_thresh);
+	}
+#endif
+#ifdef DHD_TPUT_PATCH
+	else if (!strncmp("tput_patch=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->tput_patch = TRUE;
+		else
+			conf->tput_patch = FALSE;
+		CONFIG_MSG("tput_patch = %d\n", conf->tput_patch);
+		dhd_conf_set_tput_patch(dhd);
+	}
+	else if (!strncmp("mtu=", full_param, len_param)) {
+		conf->mtu = (int)simple_strtol(data, NULL, 0);
+		CONFIG_MSG("mtu = %d\n", conf->mtu);
+	}
+	else if (!strncmp("pktsetsum=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->pktsetsum = TRUE;
+		else
+			conf->pktsetsum = FALSE;
+		CONFIG_MSG("pktsetsum = %d\n", conf->pktsetsum);
+	}
+#endif
+#ifdef SET_XPS_CPUS
+	else if (!strncmp("xps_cpus=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->xps_cpus = TRUE;
+		else
+			conf->xps_cpus = FALSE;
+		CONFIG_MSG("xps_cpus = %d\n", conf->xps_cpus);
+	}
+#endif
+#ifdef SET_RPS_CPUS
+	else if (!strncmp("rps_cpus=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->rps_cpus = TRUE;
+		else
+			conf->rps_cpus = FALSE;
+		CONFIG_MSG("rps_cpus = %d\n", conf->rps_cpus);
+	}
+#endif
+#ifdef CHECK_DOWNLOAD_FW
+	else if (!strncmp("fwchk=", full_param, len_param)) {
+		if (!strncmp(data, "1", 1))
+			conf->fwchk = TRUE;
+		else
+			conf->fwchk = FALSE;
+		CONFIG_MSG("fwchk = %d\n", conf->fwchk);
+	}
+#endif
+	else if (!strncmp("vndr_ie_assocreq=", full_param, len_param)) {
+		if (conf->vndr_ie_assocreq) {
+			kfree(conf->vndr_ie_assocreq);
+			conf->vndr_ie_assocreq = NULL;
+		}
+		if (!(conf->vndr_ie_assocreq = kmalloc(strlen(data)+1, GFP_KERNEL))) {
+			CONFIG_ERROR("kmalloc failed\n");
+		} else {
+			memset(conf->vndr_ie_assocreq, 0, strlen(data)+1);
+			strcpy(conf->vndr_ie_assocreq, data);
+			CONFIG_MSG("vndr_ie_assocreq = %s\n", conf->vndr_ie_assocreq);
+		}
+	}
+	else
+		return false;
+
+	return true;
+}
+
+int
+dhd_conf_read_config(dhd_pub_t *dhd, char *conf_path)
+{
+	int bcmerror = -1, chip_match = -1;
+	uint len = 0, memblock_len = 0, start_pos=0, end_pos=0;
+	char *memblock = NULL;
+	char *bufp, *pick = NULL, *pch;
+	bool conf_file_exists;
+	uint len_param;
+
+	len = MAXSZ_CONFIG;
+
+	conf_file_exists = ((conf_path != NULL) && (conf_path[0] != '\0'));
+	if (!conf_file_exists) {
+		CONFIG_MSG("config path %s\n", conf_path);
+		return (0);
+	}
+
+	if (conf_file_exists)
+		bcmerror = dhd_get_download_buffer(dhd, conf_path, NVRAM, &memblock,
+			(int *)&len);
+	else
+		bcmerror = dhd_get_download_buffer(dhd, NULL, NVRAM, &memblock, (int *)&len);
+
+	if (bcmerror != BCME_OK) {
+		CONFIG_MSG("Ignore config file %s\n", conf_path);
+		goto err;
+	}
+
+	memblock_len = MAXSZ_CONFIG;
+
+	pick = MALLOC(dhd->osh, MAXSZ_BUF);
+	if (!pick) {
+		CONFIG_ERROR("Failed to allocate memory %d bytes\n", MAXSZ_BUF);
+		goto err;
+	}
+
+	/* Read variables */
+	if (len > 0 && len < MAXSZ_CONFIG) {
+		bufp = (char *)memblock;
+		bufp[len] = 0;
+
+		while (start_pos < len) {
+			memset(pick, 0, MAXSZ_BUF);
+			end_pos = pick_config_vars(bufp, len, start_pos, pick, MAXSZ_BUF);
+			if (end_pos - start_pos >= MAXSZ_BUF)
+				CONFIG_ERROR("out of buf to read MAXSIZ_BUF=%d\n", MAXSZ_BUF);
+			start_pos = end_pos;
+			pch = strchr(pick, '=');
+			if (pch != NULL) {
+				len_param = pch-pick+1;
+				if (len_param == strlen(pick)) {
+					CONFIG_ERROR("not a right parameter %s\n", pick);
+					continue;
+				}
+			} else {
+				CONFIG_ERROR("not a right parameter %s\n", pick);
+				continue;
+			}
+
+			dhd_conf_read_chiprev(dhd, &chip_match, pick, len_param);
+			if (!chip_match)
+				continue;
+
+			if (dhd_conf_read_log_level(dhd, pick, len_param))
+				continue;
+			else if (dhd_conf_read_roam_params(dhd, pick, len_param))
+				continue;
+			else if (dhd_conf_read_wme_ac_params(dhd, pick, len_param))
+				continue;
+#ifdef SET_FWNV_BY_MAC
+			else if (dhd_conf_read_fw_by_mac(dhd, pick, len_param))
+				continue;
+			else if (dhd_conf_read_nv_by_mac(dhd, pick, len_param))
+				continue;
+#endif
+			else if (dhd_conf_read_nv_by_chip(dhd, pick, len_param))
+				continue;
+			else if (dhd_conf_read_country(dhd, pick, len_param))
+				continue;
+			else if (dhd_conf_read_mchan_params(dhd, pick, len_param))
+				continue;
+#ifdef PKT_FILTER_SUPPORT
+			else if (dhd_conf_read_pkt_filter(dhd, pick, len_param))
+				continue;
+#endif /* PKT_FILTER_SUPPORT */
+#ifdef ISAM_PREINIT
+			else if (dhd_conf_read_isam(dhd, pick, len_param))
+				continue;
+#endif /* ISAM_PREINIT */
+#ifdef IDHCP
+			else if (dhd_conf_read_dhcp_params(dhd, pick, len_param))
+				continue;
+#endif /* IDHCP */
+#ifdef BCMSDIO
+			else if (dhd_conf_read_sdio_params(dhd, pick, len_param))
+				continue;
+#endif /* BCMSDIO */
+#ifdef BCMPCIE
+			else if (dhd_conf_read_pcie_params(dhd, pick, len_param))
+				continue;
+#endif /* BCMPCIE */
+			else if (dhd_conf_read_pm_params(dhd, pick, len_param))
+				continue;
+			else if (dhd_conf_read_others(dhd, pick, len_param))
+				continue;
+			else
+				continue;
+		}
+
+		bcmerror = 0;
+	} else {
+		CONFIG_ERROR("error reading config file: %d\n", len);
+		bcmerror = BCME_SDIO_ERROR;
+	}
+
+err:
+	if (pick)
+		MFREE(dhd->osh, pick, MAXSZ_BUF);
+
+	if (memblock)
+		dhd_free_download_buffer(dhd, memblock, memblock_len);
+
+	return bcmerror;
+}
+
+#if defined(BCMSDIO) || defined(BCMPCIE)
+void
+dhd_conf_set_devid(dhd_pub_t *dhd)
+{
+	wifi_adapter_info_t *adapter = NULL;
+	uint32 bus_type = -1;
+	uint32 bus_num = -1;
+	uint32 slot_num = -1;
+
+	dhd_bus_get_ids(dhd->bus, &bus_type, &bus_num, &slot_num);
+	adapter = dhd_wifi_platform_get_adapter(bus_type, bus_num, slot_num);
+	if (adapter) {
+#if defined(BCMSDIO)
+		dhd->conf->devid = adapter->sdio_func->device;
+#endif
+#if defined(BCMPCIE)
+		dhd->conf->devid = adapter->pci_dev->device;
+		dhd->conf->svid = adapter->pci_dev->subsystem_vendor;
+		dhd->conf->ssid = adapter->pci_dev->subsystem_device;
+#endif
+	} else {
+		CONFIG_ERROR("can't find adapter\n");
+	}
+
+	return;
+}
+#endif
+
+int
+dhd_conf_set_chiprev(dhd_pub_t *dhd, uint chip, uint chiprev)
+{
+#if defined(BCMSDIO) || defined(BCMPCIE)
+	dhd_conf_set_devid(dhd);
+#endif
+	dhd->conf->chip = chip;
+	dhd->conf->chiprev = chiprev;
+
+#if defined(BCMSDIO)
+	CONFIG_MSG("devid=0x%x, chip=0x%x, chiprev=%d\n",
+		dhd->conf->devid, dhd->conf->chip, dhd->conf->chiprev);
+#endif
+#if defined(BCMPCIE)
+	CONFIG_MSG("devid=0x%x, chip=0x%x, chiprev=%d, svid=0x%04x, ssid=0x%04x\n",
+		dhd->conf->devid, dhd->conf->chip, dhd->conf->chiprev,
+		dhd->conf->svid, dhd->conf->ssid);
+#endif
+#if defined(BCMDBUS)
+	CONFIG_MSG("chip=0x%x, chiprev=%d\n", dhd->conf->chip, dhd->conf->chiprev);
+#endif
+
+	return 0;
+}
+
+uint
+dhd_conf_get_chip(void *context)
+{
+	dhd_pub_t *dhd = context;
+
+	if (dhd && dhd->conf)
+		return dhd->conf->chip;
+	return 0;
+}
+
+uint
+dhd_conf_get_chiprev(void *context)
+{
+	dhd_pub_t *dhd = context;
+
+	if (dhd && dhd->conf)
+		return dhd->conf->chiprev;
+	return 0;
+}
+
+#ifdef BCMSDIO
+void
+dhd_conf_set_txglom_params(dhd_pub_t *dhd, bool enable)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	if (enable) {
+#if defined(BCMSDIOH_TXGLOM_EXT)
+		if (conf->chip == BCM43362_CHIP_ID || conf->chip == BCM4330_CHIP_ID ||
+				conf->chip == BCM43340_CHIP_ID || conf->chip == BCM43341_CHIP_ID ||
+				conf->chip == BCM4334_CHIP_ID || conf->chip == BCM4324_CHIP_ID) {
+			conf->txglom_mode = SDPCM_TXGLOM_CPY;
+		}
+#endif
+		// other parameters set in preinit or config.txt
+		if (conf->txglom_ext)
+			CONFIG_MSG("txglom_ext=%d, txglom_bucket_size=%d\n",
+				conf->txglom_ext, conf->txglom_bucket_size);
+		CONFIG_MSG("txglom_mode=%s\n",
+			conf->txglom_mode==SDPCM_TXGLOM_MDESC?"multi-desc":"copy");
+		CONFIG_MSG("txglomsize=%d, deferred_tx_len=%d\n",
+			conf->txglomsize, conf->deferred_tx_len);
+		CONFIG_MSG("txinrx_thres=%d, dhd_txminmax=%d\n",
+			conf->txinrx_thres, conf->dhd_txminmax);
+		CONFIG_MSG("tx_max_offset=%d, txctl_tmo_fix=%d\n",
+			conf->tx_max_offset, conf->txctl_tmo_fix);
+	} else {
+		// clear txglom parameters
+		conf->txglom_ext = FALSE;
+		conf->txglom_bucket_size = 0;
+		conf->txglomsize = 0;
+		conf->deferred_tx_len = 0;
+	}
+
+}
+
+static void
+dhd_conf_set_ampdu_mpdu(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+	char ampdu_mpdu[32] = "ampdu_mpdu=";
+	int val = -1;
+
+	if (chip == BCM43362_CHIP_ID || chip == BCM4330_CHIP_ID ||
+			chip == BCM4334_CHIP_ID || chip == BCM43340_CHIP_ID ||
+			chip == BCM43341_CHIP_ID || chip == BCM4324_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4371_CHIP_ID ||
+			chip == BCM43430_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM43454_CHIP_ID ||
+			chip == BCM4359_CHIP_ID || chip == BCM43012_CHIP_ID) {
+		val = 16;
+	} else if (chip == BCM43751_CHIP_ID || chip == BCM43752_CHIP_ID ||
+			chip == BCM43756_CHIP_ID) {
+		val = 32;
+	}
+
+	if (val > 0) {
+		snprintf(ampdu_mpdu+strlen(ampdu_mpdu), sizeof(ampdu_mpdu), "%d", val);
+		dhd_conf_set_wl_cmd(dhd, ampdu_mpdu, TRUE);
+	}
+}
+
+#if defined(SDIO_ISR_THREAD)
+static void
+dhd_conf_set_intr_extn(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+
+	if (chip == BCM43012_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM43454_CHIP_ID || chip == BCM4345_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM4371_CHIP_ID ||
+			chip == BCM4359_CHIP_ID ||
+			chip == BCM43751_CHIP_ID || chip == BCM43752_CHIP_ID ||
+			chip == BCM4375_CHIP_ID || chip == BCM43756_CHIP_ID ||
+			chip == BCM43711_CHIP_ID ||
+			chip == BCM4381_CHIP_ID || chip == BCM4382_CHIP_ID) {
+		CONFIG_TRACE("enable intr_extn\n");
+		dhd->conf->intr_extn = TRUE;
+	}
+}
+#endif /* SDIO_ISR_THREAD */
+#endif
+
+static void
+dhd_conf_set_txbf(dhd_pub_t *dhd)
+{
+	uint chip = dhd->conf->chip;
+
+	if (chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4371_CHIP_ID || chip == BCM4359_CHIP_ID ||
+			chip == BCM43569_CHIP_ID ||
+			chip == BCM43751_CHIP_ID || chip == BCM43752_CHIP_ID ||
+			chip == BCM4375_CHIP_ID || chip == BCM43756_CHIP_ID ||
+			chip == BCM4382_CHIP_ID) {
+		CONFIG_TRACE("enable txbf\n");
+		dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "txbf", 1, 0, FALSE);
+	}
+}
+
+static void
+dhd_conf_tput_improve(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+	uint chip = conf->chip;
+	uint chiprev = conf->chiprev;
+
+	if ((chip == BCM43430_CHIP_ID && chiprev == 2) ||
+			chip == BCM43012_CHIP_ID ||
+			chip == BCM4335_CHIP_ID || chip == BCM4339_CHIP_ID ||
+			chip == BCM43454_CHIP_ID || chip == BCM4345_CHIP_ID ||
+			chip == BCM4354_CHIP_ID || chip == BCM4356_CHIP_ID ||
+			chip == BCM4345_CHIP_ID || chip == BCM4371_CHIP_ID ||
+			chip == BCM43569_CHIP_ID || chip == BCM4359_CHIP_ID ||
+			chip == BCM43751_CHIP_ID || chip == BCM43752_CHIP_ID ||
+			chip == BCM4375_CHIP_ID || chip == BCM43756_CHIP_ID ||
+			chip == BCM43711_CHIP_ID ||
+			chip == BCM4381_CHIP_ID || chip == BCM4382_CHIP_ID) {
+		CONFIG_TRACE("enable tput parameters\n");
+#ifdef DHDTCPACK_SUPPRESS
+#ifdef BCMSDIO
+		conf->tcpack_sup_mode = TCPACK_SUP_REPLACE;
+#endif
+#endif
+#if defined(BCMSDIO)
+		dhd_rxbound = 128;
+		dhd_txbound = 64;
+#endif
+		conf->frameburst = 1;
+#ifdef BCMSDIO
+		conf->dhd_txminmax = -1;
+		conf->txinrx_thres = 128;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+		conf->orphan_move = 1;
+#else
+		conf->orphan_move = 0;
+#endif
+	}
+}
+
+#ifdef UPDATE_MODULE_NAME
+#if defined(BCMSDIO) || defined(BCMPCIE)
+static void
+dhd_conf_compat_vht(dhd_pub_t *dhd)
+{
+	char vht_features[] = "vht_features=0";
+
+	CONFIG_TRACE("Enter\n");
+
+	dhd_conf_set_wl_cmd(dhd, vht_features, TRUE);
+}
+#endif
+
+int
+dhd_conf_compat_func(dhd_pub_t *dhd)
+{
+	const module_name_map_t *row = NULL;
+
+	row = dhd_conf_match_module(dhd);
+	if (row && row->compat_func) {
+		row->compat_func(dhd);
+	}
+
+	return 0;
+}
+#endif
+
+void
+dhd_conf_preinit_ioctls_sta(dhd_pub_t *dhd, int ifidx)
+{
+	struct dhd_conf *conf = dhd->conf;
+	int pm;
+
+	dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "bcn_timeout", conf->bcn_timeout, 0, FALSE);
+#ifdef NO_POWER_SAVE
+	pm = PM_OFF;
+#else
+	if (conf->pm >= 0)
+		pm = conf->pm;
+	else
+		pm = PM_FAST;
+#endif
+	dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_PM, "WLC_SET_PM", pm, 0, FALSE);
+	dhd_conf_set_intiovar(dhd, ifidx, WLC_SET_VAR, "assoc_retry_max", 10, 0, FALSE);
+	dhd_conf_set_roam(dhd, ifidx);
+}
+
+void
+dhd_conf_postinit_ioctls(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+	char wl_preinit[] = "";
+#ifdef NO_POWER_SAVE
+	char wl_no_power_save[] = "mpc=0, 86=0";
+	dhd_conf_set_wl_cmd(dhd, wl_no_power_save, FALSE);
+#endif
+
+	dhd_conf_get_ioctl_ver(dhd);
+	dhd_conf_set_intiovar(dhd, 0, WLC_UP, "WLC_UP", 0, 0, FALSE);
+	dhd_conf_check_ccode_all(dhd);
+	dhd_conf_country(dhd, "country", conf->cspec.country_abbrev);
+
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_BAND, "WLC_SET_BAND", conf->band, 0, FALSE);
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_SRL, "WLC_SET_SRL", conf->srl, 0, FALSE);
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_LRL, "WLC_SET_LRL", conf->lrl, 0, FALSE);
+	dhd_conf_set_bw_cap(dhd);
+
+#if defined(BCMPCIE)
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "bus:deepsleep_disable",
+		conf->bus_deepsleep_disable, 0, FALSE);
+#endif /* defined(BCMPCIE) */
+
+#ifdef IDHCP
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "dhcpc_enable", conf->dhcpc_enable,
+		0, FALSE);
+	if (conf->dhcpd_enable >= 0) {
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "dhcpd_ip_addr",
+			(char *)&conf->dhcpd_ip_addr, sizeof(conf->dhcpd_ip_addr), FALSE);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "dhcpd_ip_mask",
+			(char *)&conf->dhcpd_ip_mask, sizeof(conf->dhcpd_ip_mask), FALSE);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "dhcpd_ip_start",
+			(char *)&conf->dhcpd_ip_start, sizeof(conf->dhcpd_ip_start), FALSE);
+		dhd_conf_set_bufiovar(dhd, 0, WLC_SET_VAR, "dhcpd_ip_end",
+			(char *)&conf->dhcpd_ip_end, sizeof(conf->dhcpd_ip_end), FALSE);
+		dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "dhcpd_enable",
+			conf->dhcpd_enable, 0, FALSE);
+	}
+#endif
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_FAKEFRAG, "WLC_SET_FAKEFRAG",
+		conf->frameburst, 0, FALSE);
+
+	dhd_conf_preinit_ioctls_sta(dhd, 0);
+	dhd_conf_set_wl_cmd(dhd, wl_preinit, TRUE);
+#if defined(BCMSDIO)
+	dhd_conf_set_ampdu_mpdu(dhd);
+#endif
+
+#ifdef DHD_TPUT_PATCH
+	if (dhd->conf->mtu)
+		dhd_change_mtu(dhd, dhd->conf->mtu, 0);
+#endif
+	dhd_conf_set_txbf(dhd);
+	if (conf->chip == BCM4375_CHIP_ID) {
+		char he_cmd[] = "110=1, nmode=1, vhtmode=1, he=enab 1";
+		dhd_conf_set_wl_cmd(dhd, he_cmd, TRUE);
+	}
+	if (conf->chip == BCM43752_CHIP_ID || conf->chip == BCM4359_CHIP_ID) {
+		char txack_alive[] = "txack_alive=0";
+		dhd_conf_set_wl_cmd(dhd, txack_alive, TRUE);
+	}
+#ifdef WLDWDS
+	{
+		char dwds[] = "dwds=1";
+		dhd_conf_set_wl_cmd(dhd, dwds, TRUE);
+	}
+#endif /* WLDWDS */
+#if defined(WLEASYMESH)
+	if (conf->fw_type == FW_TYPE_EZMESH) {
+		if (conf->chip == BCM4359_CHIP_ID) {
+			char ezmesh[] = "mbss=1, rsdb_mode=0";
+			dhd_conf_set_wl_cmd(dhd, ezmesh, TRUE);
+		} else {
+			char ezmesh[] = "mbss=1";
+			dhd_conf_set_wl_cmd(dhd, ezmesh, TRUE);
+		}
+	}
+#endif /* WLEASYMESH */
+#if defined(BCMSDIO) || defined(BCMPCIE)
+#if defined(BCMSDIO)
+	if (conf->devid == BCM43751_CHIP_ID)
+#elif defined(BCMPCIE)
+	if (conf->devid == BCM43751_D11AX_ID)
+#endif
+	{
+		if (FW_SUPPORTED(dhd, mbo)) {
+			char he_features[] = "he=enab 0,he=features 0";
+			dhd_conf_set_wl_cmd(dhd, he_features, TRUE);
+		}
+	}
+#endif
+#ifdef UPDATE_MODULE_NAME
+	dhd_conf_compat_func(dhd);
+#endif
+#ifndef SUPPORT_RANDOM_MAC_SCAN
+	{
+		char scanmac[] = "scanmac=enable 0";
+		dhd_conf_set_wl_cmd(dhd, scanmac, TRUE);
+	}
+#endif
+	if (conf->chip == BCM4381_CHIP_ID) {
+		char ampdu_hostreorder[] = "ampdu_hostreorder=1";
+		dhd_conf_set_wl_cmd(dhd, ampdu_hostreorder, TRUE);
+	}
+	dhd_conf_set_wl_cmd(dhd, conf->wl_preinit, TRUE);
+
+#ifndef WL_CFG80211
+	dhd_conf_set_intiovar(dhd, 0, WLC_UP, "WLC_UP", 0, 0, FALSE);
+#endif
+
+}
+
+void
+dhd_conf_free_preinit(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+#ifdef SET_FWNV_BY_MAC
+	dhd_conf_free_mac_list(&conf->fw_by_mac);
+	dhd_conf_free_mac_list(&conf->nv_by_mac);
+#endif
+	dhd_conf_free_chip_nv_path_list(&conf->nv_by_chip);
+	dhd_conf_free_country_list(conf);
+	if (conf->ccode_all_list) {
+		kfree(conf->ccode_all_list);
+		conf->ccode_all_list = NULL;
+	}
+	dhd_conf_free_mchan_list(conf);
+#ifdef PKT_FILTER_SUPPORT
+	if (conf->magic_pkt_filter_add) {
+		kfree(conf->magic_pkt_filter_add);
+		conf->magic_pkt_filter_add = NULL;
+	}
+	conf->magic_pkt_hdr_len = 0;
+	conf->pkt_filter_cnt_default = 0;
+#endif
+	if (conf->wl_preinit) {
+		kfree(conf->wl_preinit);
+		conf->wl_preinit = NULL;
+	}
+	if (conf->wl_suspend) {
+		kfree(conf->wl_suspend);
+		conf->wl_suspend = NULL;
+	}
+	if (conf->wl_resume) {
+		kfree(conf->wl_resume);
+		conf->wl_resume = NULL;
+	}
+	if (conf->vndr_ie_assocreq) {
+		kfree(conf->vndr_ie_assocreq);
+		conf->vndr_ie_assocreq = NULL;
+	}
+}
+
+int
+dhd_conf_preinit(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	CONFIG_TRACE("Enter\n");
+
+	dhd_conf_free_preinit(dhd);
+	conf->band = -1;
+	memset(&conf->bw_cap, -1, sizeof(conf->bw_cap));
+	if (conf->chip == BCM43362_CHIP_ID || conf->chip == BCM4330_CHIP_ID) {
+		strcpy(conf->cspec.country_abbrev, "ALL");
+		strcpy(conf->cspec.ccode, "ALL");
+		conf->cspec.rev = 0;
+	}
+	else if (conf->chip == BCM4335_CHIP_ID || conf->chip == BCM4339_CHIP_ID ||
+			conf->chip == BCM4354_CHIP_ID || conf->chip == BCM4356_CHIP_ID ||
+			conf->chip == BCM4345_CHIP_ID || conf->chip == BCM4371_CHIP_ID ||
+			conf->chip == BCM43569_CHIP_ID || conf->chip == BCM4359_CHIP_ID ||
+			conf->chip == BCM4375_CHIP_ID) {
+		strcpy(conf->cspec.country_abbrev, "CN");
+		strcpy(conf->cspec.ccode, "CN");
+		conf->cspec.rev = 38;
+	}
+	else if ((conf->chip == BCM43752_CHIP_ID && conf->chiprev == 4) ||
+			conf->chip == BCM43756_CHIP_ID || conf->chip == BCM43711_CHIP_ID ||
+			conf->chip == BCM4381_CHIP_ID || conf->chip == BCM4382_CHIP_ID) {
+		strcpy(conf->cspec.country_abbrev, "US");
+		strcpy(conf->cspec.ccode, "US");
+		conf->cspec.rev = 0;
+	}
+	else {
+		strcpy(conf->cspec.country_abbrev, "CN");
+		strcpy(conf->cspec.ccode, "CN");
+		conf->cspec.rev = 0;
+	}
+	memset(&conf->ccode_all.cspec, 0, sizeof(wl_country_t));
+//	strcpy(conf->ccode_all.cspec.ccode, "AA");
+	conf->ccode_all.ww_2g_chan_only = 0x2;
+
+	conf->ioctl_ver = WLC_IOCTL_VERSION;
+	conf->roam_off = 1;
+	conf->roam_off_suspend = 1;
+	conf->roam_trigger[0] = -65;
+	conf->roam_trigger[1] = WLC_BAND_ALL;
+	conf->roam_scan_period[0] = 10;
+	conf->roam_scan_period[1] = WLC_BAND_ALL;
+	conf->roam_delta[0] = 10;
+	conf->roam_delta[1] = WLC_BAND_ALL;
+	conf->fullroamperiod = 20;
+	conf->keep_alive_period = 30000;
+#ifdef ARP_OFFLOAD_SUPPORT
+	conf->garp = FALSE;
+#endif
+	conf->force_wme_ac = 0;
+	memset(&conf->wme_sta, 0, sizeof(wme_param_t));
+	memset(&conf->wme_ap, 0, sizeof(wme_param_t));
+#ifdef PKT_FILTER_SUPPORT
+	memset(&conf->pkt_filter_add, 0, sizeof(conf_pkt_filter_add_t));
+	memset(&conf->pkt_filter_del, 0, sizeof(conf_pkt_filter_del_t));
+#endif
+	conf->srl = -1;
+	conf->lrl = -1;
+	conf->bcn_timeout = 16;
+	conf->disable_proptx = -1;
+	conf->dhd_poll = -1;
+#ifdef BCMSDIO
+	conf->use_rxchain = 0;
+	conf->bus_rxglom = TRUE;
+	conf->txglom_ext = FALSE;
+	conf->tx_max_offset = 0;
+	conf->txglomsize = SDPCM_DEFGLOM_SIZE;
+	conf->txctl_tmo_fix = 300;
+	conf->txglom_mode = SDPCM_TXGLOM_CPY;
+	conf->deferred_tx_len = 0;
+	conf->dhd_txminmax = 1;
+	conf->txinrx_thres = -1;
+#ifdef MINIME
+	conf->ramsize = 0x80000;
+#endif
+#if defined(SDIO_ISR_THREAD)
+	conf->intr_extn = FALSE;
+#endif
+#ifdef BCMSDIO_RXLIM_POST
+	conf->rxlim_en = FALSE;
+#endif
+#ifdef BCMSDIO_TXSEQ_SYNC
+	conf->txseq_sync = FALSE;
+#endif
+#if defined(HW_OOB)
+	conf->oob_enabled_later = FALSE;
+#endif
+#ifdef BCMSDIO_INTSTATUS_WAR
+	conf->read_intr_mode = 0;
+#endif
+	conf->kso_try_max = 0;
+#ifdef KSO_DEBUG
+	memset(&conf->kso_try_array, 0, sizeof(conf->kso_try_array));
+#endif
+#endif
+#ifdef BCMPCIE
+	conf->bus_deepsleep_disable = -1;
+	conf->flow_ring_queue_threshold = FLOW_RING_QUEUE_THRESHOLD;
+	conf->d2h_intr_method = -1;
+	conf->d2h_intr_control = -1;
+	conf->aspm = -1;
+	conf->l1ss = -1;
+	conf->enq_hdr_pkt = 0;
+#endif
+	conf->dpc_cpucore = -1;
+	conf->rxf_cpucore = -1;
+	conf->dhd_dpc_prio = -1;
+	conf->frameburst = -1;
+	conf->deepsleep = FALSE;
+	conf->pm = -1;
+	conf->pm_in_suspend = -1;
+	conf->insuspend = 0;
+	conf->suspend_mode = PM_NOTIFIER;
+#ifndef WL_CFG80211
+	conf->suspend_bcn_li_dtim = 3;
+#else
+	conf->suspend_bcn_li_dtim = -1;
+#endif
+	conf->rekey_offload = FALSE;
+#ifdef WL_EXT_WOWL
+	dhd_master_mode = TRUE;
+	conf->wowl = WL_WOWL_NET|WL_WOWL_DIS|WL_WOWL_BCN;
+#ifdef BCMDBUS
+	conf->wowl_dngldown = 0;
+#endif
+	conf->insuspend |= (WOWL_IN_SUSPEND | NO_TXDATA_IN_SUSPEND);
+#endif
+	if (conf->suspend_mode == PM_NOTIFIER || conf->suspend_mode == SUSPEND_MODE_2)
+		conf->insuspend |= (NO_TXDATA_IN_SUSPEND | NO_TXCTL_IN_SUSPEND);
+	conf->suspended = FALSE;
+	memset(&conf->bssid_insuspend, 0, ETHER_ADDR_LEN);
+#ifdef SUSPEND_EVENT
+	memset(&conf->resume_eventmask, 0, sizeof(conf->resume_eventmask));
+	conf->wlfc = FALSE;
+#endif
+#ifdef GET_CUSTOM_MAC_FROM_CONFIG
+	memset(&conf->hw_ether, 0, sizeof(conf->hw_ether));
+#endif
+#ifdef IDHCP
+	conf->dhcpc_enable = -1;
+	conf->dhcpd_enable = -1;
+#endif
+	conf->orphan_move = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+	conf->tsq = 10;
+#else
+	conf->tsq = 0;
+#endif
+#ifdef DHDTCPACK_SUPPRESS
+#ifdef BCMPCIE
+	conf->tcpack_sup_mode = TCPACK_SUP_HOLD;
+#else
+	conf->tcpack_sup_mode = TCPACK_SUP_OFF;
+#endif
+	conf->tcpack_sup_ratio = CUSTOM_TCPACK_SUPP_RATIO;
+	conf->tcpack_sup_delay = CUSTOM_TCPACK_DELAY_TIME;
+#endif
+	conf->pktprio8021x = -1;
+	conf->ctrl_resched = 2;
+	conf->rxcnt_timeout = 3;
+	conf->in4way = STA_NO_SCAN_IN4WAY | STA_WAIT_DISCONNECTED |
+		AP_WAIT_STA_RECONNECT;
+#ifdef WL_EXT_RECONNECT
+	conf->in4way |= STA_REASSOC_RETRY;
+#endif
+	if (conf->chip == BCM43752_CHIP_ID)
+		conf->war = SET_CHAN_INCONN | FW_REINIT_INCSA | FW_REINIT_EMPTY_SCAN |
+			FW_REINIT_RXF0OVFL;
+	else
+		conf->war = 0;
+#ifdef EAPOL_RESEND
+	conf->war |= RESEND_EAPOL_PKT;
+#endif
+#ifdef P2P_AP_CONCURRENT
+	conf->war |= P2P_AP_MAC_CONFLICT;
+#endif
+#ifdef PROPTX_MAXCOUNT
+	conf->proptx_maxcnt_2g = 46;
+	conf->proptx_maxcnt_5g = WL_TXSTATUS_FREERUNCTR_MASK;
+#endif /* DYNAMIC_PROPTX_MAXCOUNT */
+#ifdef TPUT_MONITOR
+	conf->data_drop_mode = NO_DATA_DROP;
+	conf->tput_monitor_ms = 0;
+#ifdef BCMSDIO
+	if (conf->chip == BCM43752_CHIP_ID || conf->chip == BCM43756_CHIP_ID ||
+			conf->chip == BCM43711_CHIP_ID ||
+			conf->chip == BCM4381_CHIP_ID || conf->chip == BCM4382_CHIP_ID)
+		conf->doflow_tput_thresh = 200;
+	else
+		conf->doflow_tput_thresh = 9999;
+	if (conf->doflow_tput_thresh > 0 && conf->doflow_tput_thresh < 9999)
+		conf->tput_monitor_ms = 1000;
+#endif
+#endif
+#ifdef SCAN_SUPPRESS
+	conf->scan_intput = SCAN_CURCHAN_INTPUT;
+	conf->scan_busy_thresh = 10;
+	conf->scan_busy_tmo = 120;
+	if (conf->chip == BCM43752_CHIP_ID || conf->chip == BCM4375_CHIP_ID ||
+			conf->chip == BCM43756_CHIP_ID || conf->chip == BCM43711_CHIP_ID ||
+			conf->chip == BCM4381_CHIP_ID || conf->chip == BCM4382_CHIP_ID)
+		conf->scan_tput_thresh = 100;
+	else
+		conf->scan_tput_thresh = 9999;
+	if (conf->scan_tput_thresh > 0 && conf->scan_tput_thresh < 9999)
+		conf->tput_monitor_ms = 1000;
+#endif
+#ifdef DHD_TPUT_PATCH
+	conf->tput_patch = FALSE;
+	conf->mtu = 0;
+	conf->pktsetsum = FALSE;
+#endif
+#ifdef SET_XPS_CPUS
+	conf->xps_cpus = FALSE;
+#endif
+#ifdef SET_RPS_CPUS
+	conf->rps_cpus = FALSE;
+#endif
+#ifdef CHECK_DOWNLOAD_FW
+	conf->fwchk = FALSE;
+#endif
+#ifdef ISAM_PREINIT
+	memset(conf->isam_init, 0, sizeof(conf->isam_init));
+	memset(conf->isam_config, 0, sizeof(conf->isam_config));
+	memset(conf->isam_enable, 0, sizeof(conf->isam_enable));
+#endif
+#if defined(SDIO_ISR_THREAD)
+	dhd_conf_set_intr_extn(dhd);
+#endif
+	dhd_conf_tput_improve(dhd);
+#ifdef DHD_TPUT_PATCH
+	if (conf->chip == BCM43751_CHIP_ID || conf->chip == BCM43752_CHIP_ID ||
+			conf->chip == BCM4375_CHIP_ID || conf->chip == BCM43756_CHIP_ID ||
+			conf->chip == BCM43711_CHIP_ID ||
+			conf->chip == BCM4381_CHIP_ID || conf->chip == BCM4382_CHIP_ID) {
+		conf->tput_patch = TRUE;
+		dhd_conf_set_tput_patch(dhd);
+	}
+#endif
+
+#ifdef BCMSDIO
+#if defined(BCMSDIOH_TXGLOM_EXT)
+	if (conf->chip == BCM43362_CHIP_ID || conf->chip == BCM4330_CHIP_ID ||
+			conf->chip == BCM43340_CHIP_ID || conf->chip == BCM43341_CHIP_ID ||
+			conf->chip == BCM4334_CHIP_ID || conf->chip == BCM4324_CHIP_ID) {
+		conf->txglom_ext = TRUE;
+	} else {
+		conf->txglom_ext = FALSE;
+	}
+	if (conf->chip == BCM43362_CHIP_ID || conf->chip == BCM4330_CHIP_ID) {
+		conf->txglom_bucket_size = 1680; // fixed value, don't change
+		conf->txglomsize = 6;
+	}
+	if (conf->chip == BCM4334_CHIP_ID || conf->chip == BCM43340_CHIP_ID ||
+			conf->chip == BCM43341_CHIP_ID || conf->chip == BCM4324_CHIP_ID) {
+		conf->txglom_bucket_size = 1684; // fixed value, don't change
+		conf->txglomsize = 16;
+	}
+#endif
+	if (conf->txglomsize > SDPCM_MAXGLOM_SIZE)
+		conf->txglomsize = SDPCM_MAXGLOM_SIZE;
+#endif
+	init_waitqueue_head(&conf->event_complete);
+
+	return 0;
+}
+
+int
+dhd_conf_reset(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	dhd_conf_free_preinit(dhd);
+	memset(conf, 0, sizeof(dhd_conf_t));
+	return 0;
+}
+
+int
+dhd_conf_attach(dhd_pub_t *dhd)
+{
+	dhd_conf_t *conf;
+
+	CONFIG_TRACE("Enter\n");
+
+	if (dhd->conf != NULL) {
+		CONFIG_MSG("config is attached before!\n");
+		return 0;
+	}
+	/* Allocate private bus interface state */
+	if (!(conf = MALLOC(dhd->osh, sizeof(dhd_conf_t)))) {
+		CONFIG_ERROR("MALLOC failed\n");
+		goto fail;
+	}
+	memset(conf, 0, sizeof(dhd_conf_t));
+
+	dhd->conf = conf;
+
+	return 0;
+
+fail:
+	if (conf != NULL)
+		MFREE(dhd->osh, conf, sizeof(dhd_conf_t));
+	return BCME_NOMEM;
+}
+
+void
+dhd_conf_detach(dhd_pub_t *dhd)
+{
+	struct dhd_conf *conf = dhd->conf;
+
+	CONFIG_TRACE("Enter\n");
+	if (dhd->conf) {
+		dhd_conf_free_preinit(dhd);
+		MFREE(dhd->osh, conf, sizeof(dhd_conf_t));
+	}
+	dhd->conf = NULL;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_config.h b/drivers/net/wireless/ap12275/dhd_config.h
new file mode 100644
index 000000000000..34fb51fffc37
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_config.h
@@ -0,0 +1,477 @@
+
+#ifndef _dhd_config_
+#define _dhd_config_
+
+#include <bcmdevs.h>
+#include <siutils.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <wlioctl.h>
+#include <802.11.h>
+
+#define FW_TYPE_STA     0
+#define FW_TYPE_APSTA   1
+#define FW_TYPE_P2P     2
+#define FW_TYPE_MESH    3
+#define FW_TYPE_EZMESH  4
+#define FW_TYPE_ES      5
+#define FW_TYPE_MFG     6
+#define FW_TYPE_MINIME  7
+#define FW_TYPE_G       0
+#define FW_TYPE_AG      1
+
+#define FW_PATH_AUTO_SELECT 1
+#ifdef BCMDHD_MDRIVER
+#define CONFIG_PATH_AUTO_SELECT
+#else
+//#define CONFIG_PATH_AUTO_SELECT
+#endif
+extern char firmware_path[MOD_PARAM_PATHLEN];
+#if defined(BCMSDIO)
+extern uint dhd_rxbound;
+extern uint dhd_txbound;
+#endif
+#ifdef BCMSDIO
+#define TXGLOM_RECV_OFFSET 8
+extern uint dhd_doflow;
+extern uint dhd_slpauto;
+#endif
+
+#ifdef SET_FWNV_BY_MAC
+typedef struct wl_mac_range {
+	uint32 oui;
+	uint32 nic_start;
+	uint32 nic_end;
+} wl_mac_range_t;
+
+typedef struct wl_mac_list {
+	int count;
+	wl_mac_range_t *mac;
+	char name[MOD_PARAM_PATHLEN];
+} wl_mac_list_t;
+
+typedef struct wl_mac_list_ctrl {
+	int count;
+	struct wl_mac_list *m_mac_list_head;
+} wl_mac_list_ctrl_t;
+#endif
+
+typedef struct wl_chip_nv_path {
+	uint chip;
+	uint chiprev;
+	char name[MOD_PARAM_PATHLEN];
+} wl_chip_nv_path_t;
+
+typedef struct wl_chip_nv_path_list_ctrl {
+	int count;
+	struct wl_chip_nv_path *m_chip_nv_path_head;
+} wl_chip_nv_path_list_ctrl_t;
+
+typedef struct wmes_param {
+	int aifsn[AC_COUNT];
+	int ecwmin[AC_COUNT];
+	int ecwmax[AC_COUNT];
+	int txop[AC_COUNT];
+} wme_param_t;
+
+#ifdef PKT_FILTER_SUPPORT
+#define DHD_CONF_FILTER_MAX	8
+#define PKT_FILTER_LEN 300
+#define MAGIC_PKT_FILTER_LEN 450
+typedef struct conf_pkt_filter_add {
+	uint32 count;
+	char filter[DHD_CONF_FILTER_MAX][PKT_FILTER_LEN];
+} conf_pkt_filter_add_t;
+
+typedef struct conf_pkt_filter_del {
+	uint32 count;
+	uint32 id[DHD_CONF_FILTER_MAX];
+} conf_pkt_filter_del_t;
+#endif
+
+#define CONFIG_COUNTRY_LIST_SIZE 500
+typedef struct country_list {
+	struct country_list *next;
+	wl_country_t cspec;
+} country_list_t;
+
+typedef struct wl_ccode_all {
+	wl_country_t cspec;
+	int32 ww_2g_chan_only;
+	int32 disable_5g_band;
+	int32 disable_6g_band;
+} wl_ccode_all_t;
+
+/* mchan_params */
+#define MCHAN_MAX_NUM 4
+#define MIRACAST_SOURCE	1
+#define MIRACAST_SINK	2
+typedef struct mchan_params {
+	struct mchan_params *next;
+	int bw;
+	int p2p_mode;
+	int miracast_mode;
+} mchan_params_t;
+
+#ifdef SCAN_SUPPRESS
+enum scan_intput_flags {
+	NO_SCAN_INTPUT	= (1 << (0)),
+	SCAN_CURCHAN_INTPUT	= (1 << (1)),
+	SCAN_LIGHT_INTPUT	= (1 << (2)),
+};
+#endif
+
+enum war_flags {
+	SET_CHAN_INCONN	= (1 << (0)),
+	FW_REINIT_INCSA	= (1 << (1)),
+	FW_REINIT_EMPTY_SCAN	= (1 << (2)),
+	P2P_AP_MAC_CONFLICT	= (1 << (3)),
+	RESEND_EAPOL_PKT	= (1 << (4)),
+	FW_REINIT_RXF0OVFL	= (1 << (5))
+};
+
+enum in4way_flags {
+	STA_NO_SCAN_IN4WAY	= (1 << (0)),
+	STA_NO_BTC_IN4WAY	= (1 << (1)),
+	STA_WAIT_DISCONNECTED	= (1 << (2)),
+	AP_WAIT_STA_RECONNECT	= (1 << (3)),
+	STA_FAKE_SCAN_IN_CONNECT	= (1 << (4)),
+	STA_REASSOC_RETRY	= (1 << (5)),
+};
+
+enum in_suspend_flags {
+	NO_EVENT_IN_SUSPEND		= (1 << (0)),
+	NO_TXDATA_IN_SUSPEND	= (1 << (1)),
+	NO_TXCTL_IN_SUSPEND		= (1 << (2)),
+	AP_DOWN_IN_SUSPEND		= (1 << (3)),
+	ROAM_OFFLOAD_IN_SUSPEND	= (1 << (4)),
+	AP_FILTER_IN_SUSPEND	= (1 << (5)),
+	WOWL_IN_SUSPEND			= (1 << (6)),
+	ALL_IN_SUSPEND 			= 0xFFFFFFFF,
+};
+
+enum in_suspend_mode {
+	EARLY_SUSPEND = 0,
+	PM_NOTIFIER = 1,
+	SUSPEND_MODE_2 = 2
+};
+
+#ifdef TPUT_MONITOR
+enum data_drop_mode {
+	NO_DATA_DROP = -1,
+	FW_DROP = 0,
+	TXPKT_DROP = 1,
+	XMIT_DROP = 2
+};
+#endif
+
+enum conn_state {
+	CONN_STATE_IDLE = 0,
+	CONN_STATE_CONNECTING = 1,
+	CONN_STATE_AUTH_SAE_M1 = 2,
+	CONN_STATE_AUTH_SAE_M2 = 3,
+	CONN_STATE_AUTH_SAE_M3 = 4,
+	CONN_STATE_AUTH_SAE_M4 = 5,
+	CONN_STATE_ASSOCIATED = 6,
+	CONN_STATE_REQID = 7,
+	CONN_STATE_RSPID = 8,
+	CONN_STATE_WSC_START = 9,
+	CONN_STATE_WPS_M1 = 10,
+	CONN_STATE_WPS_M2 = 11,
+	CONN_STATE_WPS_M3 = 12,
+	CONN_STATE_WPS_M4 = 13,
+	CONN_STATE_WPS_M5 = 14,
+	CONN_STATE_WPS_M6 = 15,
+	CONN_STATE_WPS_M7 = 16,
+	CONN_STATE_WPS_M8 = 17,
+	CONN_STATE_WSC_DONE = 18,
+	CONN_STATE_4WAY_M1 = 19,
+	CONN_STATE_4WAY_M2 = 20,
+	CONN_STATE_4WAY_M3 = 21,
+	CONN_STATE_4WAY_M4 = 22,
+	CONN_STATE_ADD_KEY = 23,
+	CONN_STATE_CONNECTED = 24,
+	CONN_STATE_GROUPKEY_M1 = 25,
+	CONN_STATE_GROUPKEY_M2 = 26,
+};
+
+enum enq_pkt_type {
+	ENQ_PKT_TYPE_EAPOL	= (1 << (0)),
+	ENQ_PKT_TYPE_ARP	= (1 << (1)),
+	ENQ_PKT_TYPE_DHCP	= (1 << (2)),
+	ENQ_PKT_TYPE_ICMP	= (1 << (3)),
+};
+
+typedef struct dhd_conf {
+	uint devid;
+	uint chip;
+	uint chiprev;
+#if defined(BCMPCIE)
+	uint svid;
+	uint ssid;
+#endif
+#ifdef GET_OTP_MODULE_NAME
+	char module_name[16];
+#endif
+	struct ether_addr otp_mac;
+	int fw_type;
+#ifdef SET_FWNV_BY_MAC
+	wl_mac_list_ctrl_t fw_by_mac;
+	wl_mac_list_ctrl_t nv_by_mac;
+#endif
+	wl_chip_nv_path_list_ctrl_t nv_by_chip;
+	country_list_t *country_head;
+	char *ccode_all_list;
+	wl_ccode_all_t ccode_all;
+	int ioctl_ver;
+	int band;
+	int bw_cap[2];
+	wl_country_t cspec;
+	uint roam_off;
+	uint roam_off_suspend;
+	int roam_trigger[2];
+	int roam_scan_period[2];
+	int roam_delta[2];
+	int fullroamperiod;
+	uint keep_alive_period;
+	bool rekey_offload;
+#ifdef ARP_OFFLOAD_SUPPORT
+	bool garp;
+#endif
+	int force_wme_ac;
+	wme_param_t wme_sta;
+	wme_param_t wme_ap;
+#ifdef PKT_FILTER_SUPPORT
+	conf_pkt_filter_add_t pkt_filter_add;
+	conf_pkt_filter_del_t pkt_filter_del;
+	char *magic_pkt_filter_add;
+	int magic_pkt_hdr_len;
+	int pkt_filter_cnt_default;
+#endif
+	int srl;
+	int lrl;
+	uint bcn_timeout;
+	int disable_proptx;
+	int dhd_poll;
+#ifdef BCMSDIO
+	int use_rxchain;
+	bool bus_rxglom;
+	bool txglom_ext; /* Only for 43362/4330/43340/43341/43241 */
+	/* terence 20161011:
+	    1) conf->tx_max_offset = 1 to fix credict issue in adaptivity testing
+	    2) conf->tx_max_offset = 1 will cause to UDP Tx not work in rxglom supported,
+	        but not happened in sw txglom
+	*/
+	int tx_max_offset;
+	uint txglomsize;
+	int txctl_tmo_fix;
+	bool txglom_mode;
+	uint deferred_tx_len;
+	/*txglom_bucket_size:
+	 * 43362/4330: 1680
+	 * 43340/43341/43241: 1684
+	 */
+	int txglom_bucket_size;
+	int txinrx_thres;
+	int dhd_txminmax; // -1=DATABUFCNT(bus)
+#ifdef DYNAMIC_MAX_HDR_READ
+	int max_hdr_read;
+#endif
+	bool oob_enabled_later;
+#ifdef MINIME
+	uint32 ramsize;
+#endif
+#if defined(SDIO_ISR_THREAD)
+	bool intr_extn;
+#endif
+#ifdef BCMSDIO_RXLIM_POST
+	bool rxlim_en;
+#endif
+#ifdef BCMSDIO_TXSEQ_SYNC
+	bool txseq_sync;
+#endif
+#ifdef BCMSDIO_INTSTATUS_WAR
+	uint read_intr_mode;
+#endif
+	int kso_try_max;
+#ifdef KSO_DEBUG
+	uint kso_try_array[10];
+#endif
+#endif
+#ifdef BCMPCIE
+	int bus_deepsleep_disable;
+	int flow_ring_queue_threshold;
+	int d2h_intr_method;
+	int d2h_intr_control;
+	int enq_hdr_pkt;
+	int aspm;
+	int l1ss;
+#endif
+	int dpc_cpucore;
+	int rxf_cpucore;
+	int dhd_dpc_prio;
+	int frameburst;
+	bool deepsleep;
+	int pm;
+	int pm_in_suspend;
+	int suspend_mode;
+	int suspend_bcn_li_dtim;
+#ifdef DHDTCPACK_SUPPRESS
+	uint8 tcpack_sup_mode;
+	uint32 tcpack_sup_ratio;
+	uint32 tcpack_sup_delay;
+#endif
+	int pktprio8021x;
+	uint insuspend;
+	bool suspended;
+	struct ether_addr bssid_insuspend;
+#ifdef SUSPEND_EVENT
+	char resume_eventmask[WL_EVENTING_MASK_LEN];
+	bool wlfc;
+#endif
+#ifdef IDHCP
+	int dhcpc_enable;
+	int dhcpd_enable;
+	struct ipv4_addr dhcpd_ip_addr;
+	struct ipv4_addr dhcpd_ip_mask;
+	struct ipv4_addr dhcpd_ip_start;
+	struct ipv4_addr dhcpd_ip_end;
+#endif
+#ifdef ISAM_PREINIT
+	char isam_init[50];
+	char isam_config[300];
+	char isam_enable[50];
+#endif
+	int ctrl_resched;
+	uint rxcnt_timeout;
+	mchan_params_t *mchan;
+	char *wl_preinit;
+	char *wl_suspend;
+	char *wl_resume;
+	int tsq;
+	int orphan_move;
+	uint in4way;
+	uint war;
+#ifdef WL_EXT_WOWL
+	uint wowl;
+#ifdef BCMDBUS
+	uint wowl_dngldown;
+#endif
+#endif
+#ifdef GET_CUSTOM_MAC_FROM_CONFIG
+	char hw_ether[62];
+#endif
+	wait_queue_head_t event_complete;
+#ifdef PROPTX_MAXCOUNT
+	int proptx_maxcnt_2g;
+	int proptx_maxcnt_5g;
+#endif /* DYNAMIC_PROPTX_MAXCOUNT */
+#ifdef TPUT_MONITOR
+	int data_drop_mode;
+	unsigned long net_len;
+	uint tput_monitor_ms;
+	struct osl_timespec tput_ts;
+	unsigned long last_tx;
+	unsigned long last_rx;
+	unsigned long last_net_tx;
+#ifdef BCMSDIO
+	int32 doflow_tput_thresh;
+#endif
+#endif
+#ifdef SCAN_SUPPRESS
+	uint scan_intput;
+	int scan_busy_thresh;
+	int scan_busy_tmo;
+	int32 scan_tput_thresh;
+#endif
+#ifdef DHD_TPUT_PATCH
+	bool tput_patch;
+	int mtu;
+	bool pktsetsum;
+#endif
+#ifdef SET_XPS_CPUS
+	bool xps_cpus;
+#endif
+#ifdef SET_RPS_CPUS
+	bool rps_cpus;
+#endif
+#ifdef CHECK_DOWNLOAD_FW
+	bool fwchk;
+#endif
+	char *vndr_ie_assocreq;
+} dhd_conf_t;
+
+#ifdef BCMSDIO
+void dhd_conf_get_otp(dhd_pub_t *dhd, bcmsdh_info_t *sdh, si_t *sih);
+#if defined(HW_OOB) || defined(FORCE_WOWLAN)
+void dhd_conf_set_hw_oob_intr(bcmsdh_info_t *sdh, struct si_pub *sih);
+#endif
+void dhd_conf_set_txglom_params(dhd_pub_t *dhd, bool enable);
+bool dhd_conf_legacy_otp_chip(dhd_pub_t *dhd);
+#endif
+#ifdef BCMPCIE
+int dhd_conf_get_otp(dhd_pub_t *dhd, si_t *sih);
+bool dhd_conf_legacy_msi_chip(dhd_pub_t *dhd);
+#endif
+#ifdef WL_CFG80211
+bool dhd_conf_legacy_chip_check(dhd_pub_t *dhd);
+bool dhd_conf_new_chip_check(dhd_pub_t *dhd);
+bool dhd_conf_extsae_chip(dhd_pub_t *dhd);
+#endif
+void dhd_conf_set_path_params(dhd_pub_t *dhd, char *fw_path, char *nv_path);
+int dhd_conf_set_intiovar(dhd_pub_t *dhd, int ifidx, uint cmd, char *name,
+	int val, int def, bool down);
+int dhd_conf_get_band(dhd_pub_t *dhd);
+bool dhd_conf_same_country(dhd_pub_t *dhd, char *buf);
+int dhd_conf_country(dhd_pub_t *dhd, char *cmd, char *buf);
+int dhd_conf_get_country(dhd_pub_t *dhd, wl_country_t *cspec);
+#ifdef CCODE_LIST
+int dhd_ccode_map_country_all(dhd_pub_t *dhd, wl_country_t *cspec);
+int dhd_ccode_map_country_list(dhd_pub_t *dhd, wl_country_t *cspec);
+#endif
+void dhd_conf_set_wme(dhd_pub_t *dhd, int ifidx, int mode);
+void dhd_conf_set_mchan_bw(dhd_pub_t *dhd, int go, int source);
+void dhd_conf_add_pkt_filter(dhd_pub_t *dhd);
+bool dhd_conf_del_pkt_filter(dhd_pub_t *dhd, uint32 id);
+void dhd_conf_discard_pkt_filter(dhd_pub_t *dhd);
+int dhd_conf_read_config(dhd_pub_t *dhd, char *conf_path);
+int dhd_conf_set_chiprev(dhd_pub_t *dhd, uint chip, uint chiprev);
+uint dhd_conf_get_chip(void *context);
+uint dhd_conf_get_chiprev(void *context);
+int dhd_conf_get_pm(dhd_pub_t *dhd);
+int dhd_conf_reg2args(dhd_pub_t *dhd, char *cmd, bool set, uint32 index, uint32 *val);
+int dhd_conf_check_hostsleep(dhd_pub_t *dhd, int cmd, void *buf, int len,
+	int *hostsleep_set, int *hostsleep_val, int *ret);
+void dhd_conf_get_hostsleep(dhd_pub_t *dhd,
+	int hostsleep_set, int hostsleep_val, int ret);
+int dhd_conf_mkeep_alive(dhd_pub_t *dhd, int ifidx, int id, int period,
+	char *packet, bool bcast);
+#ifdef ARP_OFFLOAD_SUPPORT
+void dhd_conf_set_garp(dhd_pub_t *dhd, int ifidx, uint32 ipa, bool enable);
+#endif
+#ifdef PROP_TXSTATUS
+int dhd_conf_get_disable_proptx(dhd_pub_t *dhd);
+#endif
+#ifdef TPUT_MONITOR
+void dhd_conf_tput_monitor(dhd_pub_t *dhd);
+#endif
+uint dhd_conf_get_insuspend(dhd_pub_t *dhd, uint mask);
+int dhd_conf_set_suspend_resume(dhd_pub_t *dhd, int suspend);
+void dhd_conf_postinit_ioctls(dhd_pub_t *dhd);
+void dhd_conf_preinit_ioctls_sta(dhd_pub_t *dhd, int ifidx);
+int dhd_conf_preinit(dhd_pub_t *dhd);
+int dhd_conf_reset(dhd_pub_t *dhd);
+int dhd_conf_attach(dhd_pub_t *dhd);
+void dhd_conf_detach(dhd_pub_t *dhd);
+void *dhd_get_pub(struct net_device *dev);
+int wl_pattern_atoh(char *src, char *dst);
+int dhd_conf_suspend_resume_sta(dhd_pub_t *dhd, int ifidx, int suspend);
+/* Add to adjust 802.1x priority */
+extern void pktset8021xprio(void *pkt, int prio);
+#if defined(BCMSDIO) || defined(BCMPCIE) || defined(BCMDBUS)
+extern int dhd_bus_sleep(dhd_pub_t *dhdp, bool sleep, uint32 *intstatus);
+#endif
+#ifdef WL_EXT_WOWL
+int dhd_conf_wowl_dngldown(dhd_pub_t *dhd);
+#endif
+#endif /* _dhd_config_ */
diff --git a/drivers/net/wireless/ap12275/dhd_csi.c b/drivers/net/wireless/ap12275/dhd_csi.c
new file mode 100644
index 000000000000..aa243a3c2aa5
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_csi.c
@@ -0,0 +1,827 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Channel State information Module
+ *
+ * Copyright (C) 2023, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifdef CSI_SUPPORT
+#include <osl.h>
+
+#include <bcmutils.h>
+
+#include <bcmendian.h>
+#include <linuxver.h>
+#include <linux/list.h>
+#include <linux/sort.h>
+#include <dngl_stats.h>
+#include <wlioctl.h>
+
+#include <bcmevent.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_csi.h>
+#include <wldev_common.h>
+#include <dhd_linux.h>
+
+#define SYNA_ALIGN_BYTES      (sizeof(uint32))
+#define SYNA_ALIGN_PADDING    (SYNA_ALIGN_BYTES -1LL)
+#define SYNA_ALIGN_MASK       (~SYNA_ALIGN_PADDING)
+#define SYNA_ALIGN(value) \
+(\
+	(SYNA_ALIGN_PADDING + ((long long int)(value))) \
+	& SYNA_ALIGN_MASK \
+)
+
+static const uint8  _gpMAC_zero[6] = {0, 0, 0, 0, 0, 0};
+
+static int dhd_csi_data_append(uint32 remain_length, uint32 data_length,
+	const uint8 *pData, struct csi_cfr_node *ptr)
+{
+	struct syna_csi_header  *pEntry = NULL;
+	int  error_status = 0;
+	int  ret = BCME_OK;
+	int  append_len = 0;
+
+	if ((!pData) || (!ptr) || (ptr->pNode != ptr)) {
+		DHD_ERROR(("%s-%d: *Error, invalid parameter, "
+		           "pData=0x%px, ptr=0x%px, pNode=0x%px\n",
+		           __func__, __LINE__,
+		           pData, ptr, ptr?(ptr->pNode):(NULL)));
+		ret = BCME_BADARG;
+		goto done;
+	} else {
+		pEntry = (struct syna_csi_header *)&(ptr->entry);
+		error_status = SYNA_CSI_FLAG_ERROR_MASK & pEntry->flags;
+	}
+	/* find the append length */
+	if (remain_length) {
+		append_len =   data_length
+		             - pEntry->data_length
+		             - remain_length;
+	} else if (pEntry->remain_length) {
+		append_len = pEntry->remain_length;
+	} else if (SYNA_CSI_CFR_NODE_FREE_LEN(ptr) >= data_length) {
+		append_len = data_length;
+	}
+
+	DHD_TRACE(("%s-%d: data_length=%d, remain=%d, append=%d    "
+	           "pNode: pEntry=0x%px, global_id=%d, flags=0x%x, "
+	           "data_len=%d, remain_len=%d, copied_len=%d\n",
+	           __func__, __LINE__,
+	           data_length, remain_length, append_len,
+	           pEntry, pEntry->global_id, pEntry->flags,
+	           pEntry->data_length,
+	           pEntry->remain_length, pEntry->copied_length));
+
+	/* copy and update */
+	if (append_len) {
+		memcpy((pEntry->data_length + ((uint8 *)pEntry->data)),
+		       pData, append_len);
+		pEntry->data_length += append_len;
+
+		if ((!error_status)
+		    && (!remain_length)
+		    && (pEntry->checksum)) {
+			uint16  checksum = 0;
+			uint    i;
+			pData = (uint8 *)pEntry->data;
+			for (i = 0; i < data_length; i++) {
+				checksum += pData[i];
+			}
+			if (checksum != pEntry->checksum) {
+				DHD_ERROR(("%s-%d: *Error, global_id=%d, "
+				           "checksum=0x%X mismatch to FW=0x%X, "
+				           "data_length=%d, remain_length=%d, "
+				           "pData=0x%px\n",
+				           __func__, __LINE__,
+				           pEntry->global_id,
+				           checksum, pEntry->checksum,
+				           data_length, remain_length, pData));
+				pEntry->flags |= SYNA_CSI_FLAG_ERROR_CHECKSUM;
+			}
+		}
+	} else if (!error_status) {
+		DHD_ERROR(("%s-%d: *Error, no data append, "
+		           "total_size=%d, free_len=%d, "
+		           "in_data_length=%d, in_remain_length=%d, "
+		           "    Node: global_id=%d, flags=0x%X, "
+		           "data_length=%d, remain_length=%d\n",
+		           __func__, __LINE__,
+		           ptr->total_size,
+		           (int)SYNA_CSI_CFR_NODE_FREE_LEN(ptr),
+		           data_length, remain_length,
+		           pEntry->global_id, pEntry->flags,
+		           pEntry->data_length, pEntry->remain_length));
+	} else {
+		DHD_ERROR(("%s-%d: *Warning, error report frame, "
+		           "size=%d, free=%d, "
+		           "in_data_length=%d, in_remain_length=%d, "
+		           " Node: global_id=%d, flags=0x%X%s, "
+		           "data_length=%d, remain_length=%d\n",
+		           __func__, __LINE__,
+		           ptr->total_size,
+		           (int)SYNA_CSI_CFR_NODE_FREE_LEN(ptr),
+		           data_length, remain_length,
+		           pEntry->global_id, pEntry->flags,
+		             (SYNA_CSI_FLAG_ERROR_CHECKSUM & pEntry->flags)
+		           ? "(error_checksum)"
+		           : (SYNA_CSI_FLAG_ERROR_NO_ACK & pEntry->flags)
+		           ? "(error_no_ack)"
+		           : (SYNA_CSI_FLAG_ERROR_READ & pEntry->flags)
+		           ? "(error_read)"
+		           : (SYNA_CSI_FLAG_ERROR_PS & pEntry->flags)
+		           ? "(error_ps)"
+		           : (SYNA_CSI_FLAG_ERROR_GENERIC & pEntry->flags)
+		           ? "(error_generic)"
+		           : "",
+		           pEntry->data_length, pEntry->remain_length));
+	}
+
+	/* make this as final step to avoid packet early retrieved */
+	pEntry->copied_length = 0;
+	pEntry->remain_length = remain_length;
+
+	ret = append_len;
+
+done:
+	return ret;
+}
+
+static int dhd_csi_data_new_v0(dhd_pub_t *dhdp, const wl_event_msg_t *event,
+	struct dhd_cfr_header_v0 *pEvent)
+{
+	struct csi_cfr_node    *ptr = NULL;
+	struct syna_csi_header *pEntry = NULL;
+	struct timespec64 ts;
+	dhd_if_t *ifp = NULL;
+	int       ret = BCME_ERROR;
+	uint      total_size = 0;
+	u32       chanspec = 0;
+	uint32    remain_length = 0;
+	uint32    data_length = 0;
+	uint8    *pData = NULL;
+
+	ktime_get_real_ts64(&ts);
+	total_size = SYNA_CSI_CFR_NODE_LEN + ltoh32_ua(&(pEvent->cfr_dump_length));
+
+	ptr = (struct csi_cfr_node *)MALLOCZ(dhdp->osh, total_size);
+	if (!ptr) {
+		DHD_ERROR(("%s-%d: *Error, malloc %d for cfr dump list error\n",
+		           __func__, __LINE__, total_size));
+		return BCME_NOMEM;
+	}
+	ptr->pNode = ptr; /* Not useful */
+	ptr->total_size = total_size;
+	pEntry = &(ptr->entry);
+
+	DHD_TRACE(("%s-%d: ptr=0x%p, ptr=0x%px, pEntry=0x%px, "
+	           "delta=%d, header_size=%d\n",
+	           __func__, __LINE__, ptr, ptr, pEntry,
+	           (int)((uintptr)pEntry - (uintptr)ptr),
+	           (int)SYNA_CSI_CFR_NODE_LEN));
+
+	/* process one by one for alignment consideration */
+	pEntry->magic_flag = CONST_SYNA_CSI_MAGIC_FLAG;
+	pEntry->version = CONST_SYNA_CSI_COMMON_HEADER_VERSION;
+	pEntry->format_type = SYNA_CSI_FORMAT_Q8;
+	pEntry->fc = 0;
+	if (pEvent->status) {
+		pEntry->flags |= SYNA_CSI_FLAG_ERROR_GENERIC;
+	}
+	memcpy(pEntry->client_ea, pEvent->peer_macaddr, ETHER_ADDR_LEN);
+
+	ifp = dhd_get_ifp(dhdp, event->ifidx);
+	if (!ifp) {
+		DHD_ERROR(("%s-%d: *Error, get ifp error\n",
+		           __func__, __LINE__));
+	} else {
+		memcpy(pEntry->bsscfg_ea, ifp->mac_addr, ETHER_ADDR_LEN);
+	}
+
+	if ((ret = dhd_iovar(dhdp, event->ifidx, "chanspec",
+	                     NULL, 0, (char*)&chanspec,
+	                     sizeof(chanspec), FALSE) != BCME_OK)) {
+		pEntry->band = SYNA_CSI_BAND_UNKNOWN;
+		pEntry->bandwidth = SYNA_CSI_BW_UNKNOWN;
+		pEntry->channel = 0;
+	} else {
+		switch (CHSPEC_BAND(chanspec)) {
+		case WL_CHANSPEC_BAND_2G:
+			pEntry->band = SYNA_CSI_BAND_2G;
+			break;
+		case WL_CHANSPEC_BAND_5G:
+			pEntry->band = SYNA_CSI_BAND_5G;
+			break;
+		case WL_CHANSPEC_BAND_6G:
+			pEntry->band = SYNA_CSI_BAND_6G;
+			break;
+		default:
+			pEntry->band = SYNA_CSI_BAND_UNKNOWN;
+			break;
+		}
+
+		switch (CHSPEC_BW(chanspec)) {
+		case WL_CHANSPEC_BW_20:
+			pEntry->bandwidth = SYNA_CSI_BW_20MHz;
+			break;
+		case WL_CHANSPEC_BW_40:
+			pEntry->bandwidth = SYNA_CSI_BW_40MHz;
+			break;
+		case WL_CHANSPEC_BW_80:
+			pEntry->bandwidth = SYNA_CSI_BW_80MHz;
+			break;
+		case WL_CHANSPEC_BW_160:
+			pEntry->bandwidth = SYNA_CSI_BW_160MHz;
+			break;
+		case WL_CHANSPEC_BW_320:
+			pEntry->bandwidth = SYNA_CSI_BW_320MHz;
+			break;
+		default:
+			pEntry->bandwidth = SYNA_CSI_BAND_UNKNOWN;
+			break;
+		}
+
+		pEntry->channel = CHSPEC_CHANNEL(chanspec);
+	}
+
+	pEntry->num_txstream = pEvent->sts;
+	pEntry->num_rxchain = pEvent->num_rx;
+	pEntry->num_subcarrier = ltoh16_ua(&(pEvent->num_carrier));
+	pEntry->report_tsf = (uint64)TIMESPEC64_TO_US(ts);
+	pEntry->rssi = pEvent->rssi;
+	pEntry->data_length = 0;
+	pEntry->remain_length = 0;
+	pEntry->copied_length = 0;
+
+	DHD_TRACE(("%s-%d: Entry data_size=%d, remain_size=%d, @%llu\n",
+	           __func__, __LINE__,
+	           ltoh32_ua(&(pEvent->cfr_dump_length)),
+	           ltoh32_ua(&(pEvent->remain_length)),
+	           ltoh64_ua(&(pEntry->report_tsf))));
+
+	remain_length = ltoh32_ua(&(pEvent->remain_length));
+	data_length = ltoh32_ua(&(pEvent->cfr_dump_length));
+	pData = (uint8 *)pEvent->data;
+	ret = dhd_csi_data_append(remain_length, data_length, pData, ptr);
+	if (0 <= ret) {
+		INIT_LIST_HEAD(&(ptr->list));
+		mutex_lock(&dhdp->csi_lock);
+		list_add_tail(&(ptr->list), &dhdp->csi_list);
+		dhdp->csi_count++;
+		mutex_unlock(&dhdp->csi_lock);
+	} else {
+		MFREE(dhdp->osh, ptr, total_size);
+	}
+
+	return ret;
+}
+
+static int dhd_csi_data_new_v1(dhd_pub_t *dhdp, const wl_event_msg_t *event,
+	struct dhd_cfr_header_v1 *pEvent)
+{
+	struct csi_cfr_node    *ptr = NULL;
+	struct syna_csi_header *pEntry = NULL;
+	dhd_if_t *ifp = NULL;
+	int       ret = BCME_ERROR;
+	uint      total_size = 0;
+	u32       chanspec = 0;
+	uint32    remain_length = 0;
+	uint32    data_length = 0;
+	uint8    *pData = NULL;
+
+	total_size = SYNA_CSI_CFR_NODE_LEN + ltoh32_ua(&(pEvent->cfr_dump_length));
+
+	ptr = (struct csi_cfr_node *)MALLOCZ(dhdp->osh, total_size);
+	if (!ptr) {
+		DHD_ERROR(("%s-%d: *Error, malloc %d for cfr dump list error\n",
+		           __func__, __LINE__, total_size));
+		return BCME_NOMEM;
+	}
+	ptr->pNode = ptr; /* Not useful */
+	ptr->total_size = total_size;
+	pEntry = &(ptr->entry);
+
+	DHD_TRACE(("%s-%d: ptr=0x%p, ptr=0x%px, pEntry=0x%px, "
+	           "delta=%d, header_size=%d\n",
+	           __func__, __LINE__, ptr, ptr, pEntry,
+	           (int)((uintptr)pEntry - (uintptr)ptr),
+	           (int)SYNA_CSI_CFR_NODE_LEN));
+
+	/* process one by one for alignment consideration */
+	pEntry->magic_flag = CONST_SYNA_CSI_MAGIC_FLAG;
+	pEntry->version = CONST_SYNA_CSI_COMMON_HEADER_VERSION;
+	pEntry->format_type = SYNA_CSI_FORMAT_Q8;
+	pEntry->fc = pEvent->fc;
+	if (pEvent->status) {
+		pEntry->flags |= SYNA_CSI_FLAG_ERROR_GENERIC;
+	}
+
+	memcpy(pEntry->client_ea, pEvent->peer_macaddr, ETHER_ADDR_LEN);
+
+	ifp = dhd_get_ifp(dhdp, event->ifidx);
+
+	if (!ifp) {
+		DHD_ERROR(("%s-%d: *Error, get ifp error\n",
+		           __func__, __LINE__));
+	} else {
+		memcpy(pEntry->bsscfg_ea, ifp->mac_addr, ETHER_ADDR_LEN);
+	}
+
+	if ((ret = dhd_iovar(dhdp, event->ifidx, "chanspec",
+		NULL, 0, (char*)&chanspec, sizeof(chanspec), FALSE) != BCME_OK)) {
+		pEntry->band = SYNA_CSI_BAND_UNKNOWN;
+		pEntry->bandwidth = SYNA_CSI_BW_UNKNOWN;
+		pEntry->channel = 0;
+	} else {
+		switch (CHSPEC_BAND(chanspec)) {
+		case WL_CHANSPEC_BAND_2G:
+			pEntry->band = SYNA_CSI_BAND_2G;
+			break;
+		case WL_CHANSPEC_BAND_5G:
+			pEntry->band = SYNA_CSI_BAND_5G;
+			break;
+		case WL_CHANSPEC_BAND_6G:
+			pEntry->band = SYNA_CSI_BAND_6G;
+			break;
+		default:
+			pEntry->band = SYNA_CSI_BAND_UNKNOWN;
+			break;
+		}
+
+		switch (CHSPEC_BW(chanspec)) {
+		case WL_CHANSPEC_BW_20:
+			pEntry->bandwidth = SYNA_CSI_BW_20MHz;
+			break;
+		case WL_CHANSPEC_BW_40:
+			pEntry->bandwidth = SYNA_CSI_BW_40MHz;
+			break;
+		case WL_CHANSPEC_BW_80:
+			pEntry->bandwidth = SYNA_CSI_BW_80MHz;
+			break;
+		case WL_CHANSPEC_BW_160:
+			pEntry->bandwidth = SYNA_CSI_BW_160MHz;
+			break;
+		case WL_CHANSPEC_BW_320:
+			pEntry->bandwidth = SYNA_CSI_BW_320MHz;
+			break;
+		default:
+			pEntry->bandwidth = SYNA_CSI_BAND_UNKNOWN;
+			break;
+		}
+
+		pEntry->channel = CHSPEC_CHANNEL(chanspec);
+	}
+
+	pEntry->num_txstream = pEvent->sts;
+	pEntry->num_rxchain = pEvent->num_rx;
+	pEntry->num_subcarrier = ltoh16_ua(&(pEvent->num_carrier));
+	pEntry->report_tsf = ltoh64_ua(&(pEvent->cfr_timestamp));
+	pEntry->rssi = pEvent->rssi;
+	pEntry->data_length = 0;
+	pEntry->remain_length = 0;
+	pEntry->copied_length = 0;
+
+	DHD_TRACE(("%s-%d: Entry data_size=%d, remain_size=%d, @%llu\n",
+	           __func__, __LINE__,
+	           ltoh32_ua(&(pEvent->cfr_dump_length)),
+	           ltoh32_ua(&(pEvent->remain_length)),
+	           ltoh64_ua(&(pEntry->report_tsf))));
+
+	remain_length = ltoh32_ua(&(pEvent->remain_length));
+	data_length = ltoh32_ua(&(pEvent->cfr_dump_length));
+	pData = (uint8 *)pEvent->data;
+	ret = dhd_csi_data_append(remain_length, data_length, pData, ptr);
+	if (0 <= ret) {
+		INIT_LIST_HEAD(&(ptr->list));
+		mutex_lock(&dhdp->csi_lock);
+		list_add_tail(&(ptr->list), &dhdp->csi_list);
+		dhdp->csi_count++;
+		mutex_unlock(&dhdp->csi_lock);
+	} else {
+		MFREE(dhdp->osh, ptr, total_size);
+	}
+
+	return ret;
+}
+
+static int dhd_csi_data_new_v2(dhd_pub_t *dhdp, const wl_event_msg_t *event,
+	struct dhd_cfr_header_v2 *pEvent)
+{
+	struct csi_cfr_node    *ptr = NULL;
+	struct syna_csi_header *pEntry = NULL;
+	int     ret = BCME_ERROR;
+	uint    total_size = 0;
+	uint32  remain_length = 0;
+	uint32  data_length = 0;
+	uint8  *pData = NULL;
+
+	if (SYNA_ALIGN_PADDING & (uintptr)pEvent) {
+		DHD_ERROR(("%s-%d: *Error, pEvent=0x%px is not aligned!",
+		           __func__, __LINE__, pEvent));
+	}
+
+	total_size = SYNA_CSI_CFR_NODE_LEN + pEvent->data_length;
+
+	ptr = (struct csi_cfr_node *)MALLOCZ(dhdp->osh, total_size);
+	if (!ptr) {
+		DHD_ERROR(("%s-%d: *Error, malloc %d for cfr dump list error\n",
+		           __func__, __LINE__, total_size));
+		return BCME_NOMEM;
+	}
+	ptr->pNode = ptr; /* Not useful */
+	ptr->total_size = total_size;
+	pEntry = &(ptr->entry);
+
+	DHD_TRACE(("%s-%d: ptr=0x%p, ptr=0x%px, pEntry=0x%px, "
+	           "delta=%d, header_size=%d\n",
+	           __func__, __LINE__, ptr, ptr, pEntry,
+	           (int)((uintptr)pEntry - (uintptr)ptr),
+	           (int)SYNA_CSI_CFR_NODE_LEN));
+
+	/* process one by one for alignment consideration */
+	memcpy(pEntry, pEvent, sizeof(struct syna_csi_header));
+	pEntry->magic_flag = CONST_SYNA_CSI_MAGIC_FLAG;
+	pEntry->version = CONST_SYNA_CSI_COMMON_HEADER_VERSION;
+
+	if ((pEvent->status_compat)
+	    && (!(SYNA_CSI_FLAG_ERROR_MASK & pEntry->flags))) {
+		pEntry->flags |= SYNA_CSI_FLAG_ERROR_GENERIC;
+		DHD_ERROR(("%s-%d: *Warning, status=0x%X, "
+		           " global_id=%d!\n",
+		           __func__, __LINE__,
+		           pEvent->status_compat,
+		           pEvent->global_id));
+	}
+	if (!pEntry->format_type) {
+		pEntry->format_type = SYNA_CSI_FORMAT_Q8;
+	}
+	if (!memcmp(_gpMAC_zero, pEntry->bsscfg_ea, ETHER_ADDR_LEN)) {
+		dhd_if_t *ifp = dhd_get_ifp(dhdp, event->ifidx);
+		if (!ifp) {
+			DHD_ERROR(("%s-%d: *Error, get ifp error\n",
+			           __func__, __LINE__));
+		} else {
+			memcpy(pEntry->bsscfg_ea, ifp->mac_addr, ETHER_ADDR_LEN);
+		}
+	}
+
+	if (!pEntry->report_tsf) {
+		struct timespec64  ts;
+		ktime_get_real_ts64(&ts);
+		pEntry->report_tsf = (uint64)TIMESPEC64_TO_US(ts);
+	}
+
+	pEntry->data_length = 0;
+	pEntry->remain_length = 0;
+	pEntry->copied_length = 0;
+	pEntry->checksum = pEvent->checksum;
+
+	DHD_TRACE(("%s-%d: Entry data_size=%d, remain_size=%d, @%llu\n",
+	           __func__, __LINE__,
+	           pEvent->data_length,
+	           pEvent->remain_length,
+	           pEntry->report_tsf));
+
+	remain_length = pEvent->remain_length;
+	data_length = pEvent->data_length;
+	pData = (uint8 *)pEvent->data;
+	ret = dhd_csi_data_append(remain_length, data_length, pData, ptr);
+	if (0 <= ret) {
+		INIT_LIST_HEAD(&(ptr->list));
+		mutex_lock(&dhdp->csi_lock);
+		list_add_tail(&(ptr->list), &dhdp->csi_list);
+		dhdp->csi_count++;
+		mutex_unlock(&dhdp->csi_lock);
+	} else {
+		MFREE(dhdp->osh, ptr, total_size);
+	}
+
+	return ret;
+}
+
+int dhd_csi_event_handler(dhd_pub_t *dhdp, const wl_event_msg_t *event, void *event_data)
+{
+	union dhd_cfr_header  *pEvent = NULL;
+	struct csi_cfr_node   *ptr = NULL, *next = NULL, *save_ptr = NULL;
+	const uint8  *pMAC = NULL, *pData = NULL;
+	uint8  version = 0xff;
+	uint32 data_length = 0, remain_length = 0, header_length = 0;
+	int    ret = BCME_OK;
+
+	NULL_CHECK(dhdp, "dhdp is NULL", ret);
+
+	DHD_TRACE(("%s-%d: Enter\n", __func__, __LINE__));
+
+	if (!event_data) {
+		DHD_ERROR(("%s-%d: event_data is NULL\n",
+		           __func__, __LINE__));
+		ret = BCME_BADARG;
+		goto done;
+	}
+
+	pEvent = (union dhd_cfr_header *)event_data;
+	if (pEvent->header_v0.status) {
+		DHD_ERROR(("%s-%d: *Warning, status=0x%X "
+		           " may indicate error!\n",
+		           __func__, __LINE__,
+		           pEvent->header_v0.status));
+		/* TODO: do need to abandon this scenario? */
+	}
+
+	version = pEvent->header_v0.version;
+	switch (version) {
+	case CSI_VERSION_V0:
+		pMAC = pEvent->header_v0.peer_macaddr;
+		header_length = sizeof(struct dhd_cfr_header_v0);
+		remain_length = ltoh32_ua(&(pEvent->header_v0.remain_length));
+		data_length = ltoh32_ua(&(pEvent->header_v0.cfr_dump_length));
+		pData = (uint8 *)pEvent->header_v0.data;
+		break;
+	case CSI_VERSION_V1:
+		pMAC = pEvent->header_v1.peer_macaddr;
+		header_length = sizeof(struct dhd_cfr_header_v1);
+		remain_length = ltoh32_ua(&(pEvent->header_v1.remain_length));
+		data_length = ltoh32_ua(&(pEvent->header_v1.cfr_dump_length));
+		pData = (uint8 *)pEvent->header_v1.data;
+		break;
+	case CSI_VERSION_V2:
+		pMAC = pEvent->header_v2.client_ea;
+		header_length = sizeof(struct dhd_cfr_header_v2);
+		remain_length = pEvent->header_v2.remain_length;
+		data_length = pEvent->header_v2.data_length;
+		pData = (uint8 *)pEvent->header_v2.data;
+		break;
+	default:
+		DHD_ERROR(("%s-%d: CSI version=%d error\n",
+		           __func__, __LINE__, version));
+		ret = BCME_BADOPTION;
+		goto done;
+		break;
+	}
+
+	if (header_length <= remain_length) {
+		remain_length -= header_length;
+	}
+	if (CONST_CSI_MAXIMUM_DATA_BYTES < remain_length) {
+		DHD_ERROR(("%s-%d: *Error, drop too big length, "
+		           "global_id=%d, version=%d, status=%d, "
+		           "data_length=%d(0x%X), remain_length=%d(0x%X) > %d\n",
+		           __func__, __LINE__,
+		           pEvent->header_v2.global_id,
+		           version, pEvent->header_v2.status_compat,
+		           data_length, data_length,
+		           remain_length, remain_length,
+		           (int)CONST_CSI_MAXIMUM_DATA_BYTES));
+		prhex("TOO_BIG_LENGTH", (uchar *)pEvent, 128);
+		ret = BCME_BADLEN;
+		goto done;
+	}
+
+	mutex_lock(&dhdp->csi_lock);
+	/* check if this addr exist */
+	if (!list_empty(&dhdp->csi_list)) {
+		list_for_each_entry_safe(ptr, next, &dhdp->csi_list, list) {
+			struct syna_csi_header *pEntry = &(ptr->entry);
+			/* check if is new */
+			if (bcmp(&pEntry->client_ea, pMAC, ETHER_ADDR_LEN) == 0) {
+				if (pEntry->remain_length > 0) {
+					save_ptr = ptr;
+					break;
+				}
+			}
+		}
+	}
+	/* check if need to drop head packet */
+	if ((!save_ptr) && (dhdp->csi_count >= MAX_CSI_BUF_NUM)) {
+		DHD_ERROR(("%s-%d: CSI data is full, drop pkt \n",
+		           __func__, __LINE__));
+		ptr = list_first_entry(&dhdp->csi_list, struct csi_cfr_node, list);
+		list_del_init(&ptr->list);
+		MFREE(dhdp->osh, ptr, ptr->total_size);
+		dhdp->csi_count--;
+	}
+	mutex_unlock(&dhdp->csi_lock);
+
+	/* try to append new node if needs */
+	DHD_TRACE(("%s-%d: save_ptr=0x%px, version=%d\n",
+	           __func__, __LINE__, save_ptr, version));
+	if (!save_ptr) {
+		switch (version) {
+		case CSI_VERSION_V0:
+			ret = dhd_csi_data_new_v0(dhdp, event, &(pEvent->header_v0));
+			break;
+		case CSI_VERSION_V1:
+			ret = dhd_csi_data_new_v1(dhdp, event, &(pEvent->header_v1));
+			break;
+		case CSI_VERSION_V2:
+			ret = dhd_csi_data_new_v2(dhdp, event, &(pEvent->header_v2));
+			break;
+		default:
+			DHD_ERROR(("%s-%d: CSI version=%d error\n",
+			           __func__, __LINE__, version));
+			ret = BCME_BADOPTION;
+			goto done;
+			break;
+		}
+	} else {
+		ret = dhd_csi_data_append(remain_length, data_length, pData, save_ptr);
+	}
+
+done:
+	return ret;
+}
+
+static void
+dhd_csi_clean_list(dhd_pub_t *dhdp)
+{
+	struct csi_cfr_node *ptr, *next;
+	int num = 0;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s-%d: *Error, NULL POINTER\n",
+		           __func__, __LINE__));
+		return;
+	}
+
+	mutex_lock(&dhdp->csi_lock);
+	if (!list_empty(&dhdp->csi_list)) {
+		list_for_each_entry_safe(ptr, next, &dhdp->csi_list, list) {
+			list_del(&ptr->list);
+			num++;
+			MFREE(dhdp->osh, ptr, ptr->total_size);
+		}
+	}
+	dhdp->csi_count = 0;
+	mutex_unlock(&dhdp->csi_lock);
+
+	DHD_TRACE(("%s-%d: Clean up %d record\n",
+	           __func__, __LINE__, num));
+}
+
+int dhd_csi_config(dhd_pub_t *dhdp, char *pBuf, uint length, bool is_set)
+{
+	int  ret = BCME_OK;
+
+	if ((!dhdp) || (!pBuf) || (sizeof(uint32) > length)) {
+		DHD_ERROR(("%s-%d: *Error, Bad argument\n",
+		           __func__, __LINE__));
+		ret = BCME_BADARG;
+		goto done;
+	}
+
+	if (is_set) {
+		uint  value = *((uint *)pBuf);
+		if (SYNA_CSI_DATA_MODE_LAST <= value) {
+			ret = BCME_BADOPTION;
+			goto done;
+		} else if (SYNA_CSI_DATA_MODE_NONE != value) {
+			dhdp->csi_data_send_manner = value;
+			dhd_csi_clean_list(dhdp);
+		}
+	} else {
+		memcpy(pBuf, &dhdp->csi_data_send_manner, sizeof(uint32));
+	}
+
+done:
+	return ret;
+}
+
+int dhd_csi_init(dhd_pub_t *dhdp)
+{
+	int err = BCME_OK;
+
+	mutex_init(&dhdp->csi_lock);
+	mutex_lock(&dhdp->csi_lock);
+	NULL_CHECK(dhdp, "dhdp is NULL", err);
+	INIT_LIST_HEAD(&dhdp->csi_list);
+	dhdp->csi_count = 0;
+	mutex_unlock(&dhdp->csi_lock);
+
+	return err;
+}
+
+int
+dhd_csi_deinit(dhd_pub_t *dhdp)
+{
+	int err = BCME_OK;
+
+	NULL_CHECK(dhdp, "dhdp is NULL", err);
+
+	dhd_csi_clean_list(dhdp);
+
+	mutex_destroy(&dhdp->csi_lock);
+
+	return err;
+}
+
+int dhd_csi_retrieve_queue_data(dhd_pub_t *dhdp, char *buf, uint count)
+{
+	struct csi_cfr_node *ptr = NULL, *next = NULL;
+	int   ret = BCME_OK;
+	char *pbuf = buf;
+	int   num = 0;
+	int   copy_len = 0;
+	int   left_data = 0;
+	int   total_copy_len = 0;
+
+	NULL_CHECK(dhdp, "dhdp is NULL", ret);
+
+	mutex_lock(&dhdp->csi_lock);
+	if (!list_empty(&dhdp->csi_list)) {
+		list_for_each_entry_safe(ptr, next, &dhdp->csi_list, list) {
+			struct syna_csi_header  *pEntry = &(ptr->entry);
+			if (pEntry->remain_length) {
+				DHD_ERROR(("%s-%d: *Warning, data not ready "
+				           "for %02X:%02X:%02X:%02X:%02X:%02X,"
+				           "global_id=%d, data_length=%d, "
+				           "remain_length=%d\n",
+				           __func__, __LINE__,
+				           pEntry->client_ea[0],
+				           pEntry->client_ea[1],
+				           pEntry->client_ea[2],
+				           pEntry->client_ea[3],
+				           pEntry->client_ea[4],
+				           pEntry->client_ea[5],
+				           pEntry->global_id,
+				           pEntry->data_length,
+				           pEntry->remain_length));
+				continue;
+			}
+
+			left_data =   SYNA_CSI_PKT_TOTAL_LEN(pEntry)
+			            - pEntry->copied_length;
+			copy_len = count;
+
+			if (0 > left_data) {
+				DHD_ERROR(("%s-%d: *Error, invalid case, "
+				           "total_len=%d, data_len=%d, "
+				           "remain_len=%d, copied_len=%d\n",
+				           __func__, __LINE__,
+				           (int)SYNA_CSI_PKT_TOTAL_LEN(pEntry),
+				           pEntry->data_length,
+				           pEntry->remain_length,
+				           pEntry->copied_length));
+				copy_len = 0;
+			} else if (copy_len > left_data) {
+				copy_len = left_data;
+			}
+
+			DHD_TRACE(("%s-%d: Packet[%d]: "
+			           "left_room=%d, copy_len=%d, "
+			           "pNode_data_len=%d, pNode_reamin_len=%d, "
+			           "pNode_copied_len=%d, csi_count=%d\n",
+			           __func__, __LINE__,
+			           num, count, copy_len,
+			           pEntry->data_length,
+			           pEntry->remain_length,
+			           pEntry->copied_length,
+			           dhdp->csi_count));
+
+			if (0 < copy_len) {
+				memcpy(pbuf,
+				       pEntry->copied_length + (uint8 *)pEntry,
+				       copy_len);
+				pEntry->copied_length += copy_len;
+				count -= copy_len;
+				pbuf  += copy_len;
+				num++;
+			}
+
+			if (pEntry->copied_length >= SYNA_CSI_PKT_TOTAL_LEN(pEntry)) {
+				list_del(&ptr->list);
+				MFREE(dhdp->osh, ptr, ptr->total_size);
+				dhdp->csi_count--;
+			}
+
+			if (0 >= copy_len) {
+				break;
+			}
+		}
+	}
+	mutex_unlock(&dhdp->csi_lock);
+	total_copy_len = pbuf - buf;
+	DHD_TRACE(("%s-%d: dump %d record %d bytes\n",
+	           __func__, __LINE__, num, total_copy_len));
+
+	return total_copy_len;
+}
+#endif /* CSI_SUPPORT */
diff --git a/drivers/net/wireless/ap12275/dhd_csi.h b/drivers/net/wireless/ap12275/dhd_csi.h
new file mode 100644
index 000000000000..02ee564f6572
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_csi.h
@@ -0,0 +1,466 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Linux-specific network interface.
+ * Basically selected code segments from usb-cdc.c and usb-rndis.c
+ *
+ * Copyright (C) 2023, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_CSI_H__
+#define __DHD_CSI_H__
+
+/* definition for structure and API between APPP and driver */
+
+/* **** SYNA CSI header interface with driver **** */
+
+enum syna_csi_data_mode_type {
+	SYNA_CSI_DATA_MODE_NONE = 0,
+	/* upper layer will fetch data by sysfs */
+	SYNA_CSI_DATA_MODE_SYSFS,
+	/* upper layer will fetch data by UDP socket */
+	SYNA_CSI_DATA_MODE_UDP_SOCKET,
+	SYNA_CSI_DATA_MODE_LAST
+};
+
+enum syna_csi_format_type {
+	SYNA_CSI_FORMAT_UNKNOWN = 0,
+	SYNA_CSI_FORMAT_Q8,  /* (9,9,5) Q8 floating point
+	                      * BCM4358/4359
+	                      */
+	SYNA_CSI_FORMAT_Q13, /* (14,14) Q13 fixed point
+	                      * BCM4339/43455/43456/43458
+	                      */
+	SYNA_CSI_FORMAT_Q11, /* (12,12,6) Q11 floating point
+	                      * BCM4360/4366
+	                      */
+	SYNA_CSI_FORMAT_LAST
+};
+/* (9,9,5) Q8 floating point data format */
+#define _Q8_REAL_MA(t)         (0x000ff &((t)>>14))
+#define _Q8_REAL_SI(t)         (0x00001 &((t)>>22))
+#define _Q8_IMAGE_MA(t)        (0x000ff &((t)>> 5))
+#define _Q8_IMAGE_SI(t)        (0x00001 &((t)>>13))
+#define SYNA_CSI_DATA_Q8_REAL(t) \
+(((0 < _Q8_REAL_SI(t)) \
+	? (-1) \
+	: (1)) \
+	* ((int32)_Q8_REAL_MA(t)) \
+)
+#define SYNA_CSI_DATA_Q8_IMAGE(t) \
+(((0 < _Q8_IMAGE_SI(t)) \
+	? (-1) \
+	: (1)) \
+	* ((int32)_Q8_IMAGE_MA(t)) \
+)
+#define SYNA_CSI_DATA_Q8_EXP(t) \
+(((int8)(0xf8 \
+	& ((t) << 3))) \
+	>> 3 \
+)
+/* (14,14) Q13 fixed point data format */
+#define SYNA_CSI_DATA_Q13_REAL(t) \
+(((int32)(0xfff30000 \
+	& ((t) << 4))) \
+	>> (14 + 4) \
+)
+#define SYNA_CSI_DATA_Q13_IMAGE(t) \
+(((int32)(0xfff30000 \
+	& ((t) << 18))) \
+	>> (0 + 18) \
+)
+/* (12,12,6) Q11 floating point */
+#define _Q11_REAL_MA(t)        (0x007ff &((t)>>18))
+#define _Q11_REAL_SI(t)        (0x00001 &((t)>>29))
+#define _Q11_IMAGE_MA(t)       (0x007ff &((t)>> 6))
+#define _Q11_IMAGE_SI(t)       (0x00001 &((t)>>17))
+#define SYNA_CSI_DATA_Q11_REAL(t) \
+(((0 < _Q11_REAL_SI(t)) \
+	? (-1) \
+	: (1)) \
+	* ((int32)_Q11_REAL_MA(t)) \
+)
+#define SYNA_CSI_DATA_Q11_IMAGE(t) \
+(((0 < _Q11_IMAGE_SI(t)) \
+	? (-1) \
+	: (1)) \
+	* ((int32)_Q11_IMAGE_MA(t)) \
+)
+#define SYNA_CSI_DATA_Q11_EXP(t) \
+(((int8)(0xfc \
+	& ((t) << 2))) \
+	>> 2 \
+)
+
+enum syna_csi_error_status {
+	SYNA_CSI_ERROR_NONE,
+	SYNA_CSI_ERROR_GENERIC,  /* generic error */
+	SYNA_CSI_ERROR_NO_ACK,   /* peer doesn't ACK */
+	SYNA_CSI_ERROR_READ,     /* read data error */
+	SYNA_CSI_ERROR_PS,       /* peer device under powersaving */
+	SYNA_CSI_ERROR_LAST
+};
+
+enum syna_csi_flag {
+	/* bit 0 ~ bit 15 are for error information */
+	SYNA_CSI_FLAG_ERROR_MASK = 0x0FFFF,
+	SYNA_CSI_FLAG_ERROR_CHECKSUM = (1 << 0),
+	SYNA_CSI_FLAG_ERROR_GENERIC = (1 << SYNA_CSI_ERROR_GENERIC),
+	SYNA_CSI_FLAG_ERROR_NO_ACK = (1 << SYNA_CSI_ERROR_NO_ACK),
+	SYNA_CSI_FLAG_ERROR_READ = (1 << SYNA_CSI_ERROR_READ),
+	SYNA_CSI_FLAG_ERROR_PS = (1 << SYNA_CSI_ERROR_PS),
+
+	/* subcarrier real position along FFT index: (total N=2*M subcarries)
+	 * -M, -(M-1), -(M-2), -(M-3),...,-1, 0, 1, 2,...M-1, M
+	 *
+	 *  Data Output order with positive part first(*default*):
+	 * 0, 1, 2,...M-1, M,   -M, -(M-1), -(M-2), -(M-3),...,-1,
+	 *
+	 *  Data Output order with negative part first:
+	 * -M, -(M-1), -(M-2), -(M-3),...,-1, 0, 1, 2,...M-1, M
+	 */
+	SYNA_CSI_FLAG_FFT_NEGATIVE_FIRST = (1 << 16),
+	SYNA_CSI_FLAG_LAST
+};
+
+enum syna_csi_band_type {
+	SYNA_CSI_BAND_UNKNOWN = 0,
+	SYNA_CSI_BAND_2G,
+	SYNA_CSI_BAND_5G,
+	SYNA_CSI_BAND_6G,
+	SYNA_CSI_BAND_LAST
+};
+
+enum syna_csi_bandwidth_type {
+	SYNA_CSI_BW_UNKNOWN = 0x000,
+	SYNA_CSI_BW_5MHz    = 0x001,
+	SYNA_CSI_BW_10MHz   = 0x002,
+	SYNA_CSI_BW_20MHz   = 0x004,
+	SYNA_CSI_BW_40MHz   = 0x008,
+	SYNA_CSI_BW_80MHz   = 0x010,
+	SYNA_CSI_BW_160MHz  = 0x020,
+	SYNA_CSI_BW_320MHz  = 0x040,
+	SYNA_CSI_BW_LAST
+};
+
+/* 'SYNA' -> 'A'53, 'N'59, 'Y'4e, 'S'41 */
+#define CONST_SYNA_CSI_MAGIC_FLAG    0x414E5953
+
+/* CSI API common header structure (8 bytes alignment for 64bit system) */
+/* below is version 1 'syna_csi_header' */
+#define CONST_SYNA_CSI_COMMON_HEADER_VERSION    0x01
+struct syna_csi_header {
+	/* byte 0 ~ 3 */
+	uint32  magic_flag;       /* flag for protecting and detecting header */
+
+	/* byte 4 ~ 7 */
+	uint8  version;           /* header version of this structure */
+	uint8  format_type;       /* data format type of current CSI packet,
+	                           * enumerated in 'syna_csi_format_type'
+	                           */
+	uint8  fc;                /* packet frame control type of current
+	                           * CSI data for further checking if it
+	                           * is correct expected result
+	                           */
+	uint8  flags;             /* flag may use for extra emendation
+	                           * which enumerated as 'syna_csi_flag'
+	                           */
+
+	/* byte 8 ~ 19 */
+	uint8  client_ea[6];      /* client MAC address */
+	uint8  bsscfg_ea[6];      /* BSSCFG address of current interface */
+
+	/* byte 20 ~ 23 */
+	uint8  band;              /* enumerated in 'syna_csi_band_type' */
+	uint8  bandwidth;         /* enumerated in 'syna_csi_bandwidth_type' */
+	uint16 channel;           /* channel number of corresponding band */
+
+	/* byte 24 ~ 31 */
+	uint64 report_tsf;        /* current CSI DATA RX timestamp */
+
+	/* byte 32 ~ 35 */
+	uint8  num_txstream;      /* peer side TX spatial steams */
+	uint8  num_rxchain;       /* number of RX side chain/antenna */
+	uint16 num_subcarrier;    /* Number of subcarrier */
+
+	/* byte 36 ~ 39 */
+	int8   rssi;              /* average RSSI, and goto check the
+	                           * 'rssi_ant' if this 'rssi' is zero
+	                           */
+	int8   noise;             /* average noise */
+	int16  global_id;         /* CSI frame global ID */
+
+	/* byte 40 ~ 47 */
+	int8   rssi_ant[8];       /* RSSI of each RX chain/antenna
+	                           * (Depends on solution, and some
+	                           * chipsets may only provide 'rssi'
+	                           * rather than 'rssi_ant')
+	                           */
+
+	/* byte 48 ~ 55 */
+	uint64 padding_future;    /* reserved */
+
+	/* byte 56 ~ 63 */
+	uint16 data_length;       /* the bytes of 'data[]' */
+	uint16 remain_length;     /* remain length of not received data */
+	uint16 copied_length;     /* the length has been copied to user */
+	uint16 checksum;          /* 'data[]' checksum, 0 - not used */
+
+	/* byte 64 ~ End */
+	uint32 data[0];           /* variable length according to the
+	                           * 'data_length', but the minimal
+	                           * valid data length will be
+	                           * decided by the 'num_txchains'
+	                           * 'num_rxchain' 'num_subcarrier'.
+	                           * For example, the index quantity
+	                           * of 'uint32' in the 11N 80MHz can
+	                           * be got via:
+	                           * 256 SubCarriers * 2 TXStream * 2 RXChain
+	                           * Note: 'data_length' is bytes quantity,
+	                           * and the 'data[]' array index range should
+	                           * be 0 ~ (data_length/4 - 1)
+	                           */
+};
+
+#define SYNA_CSI_PKT_TOTAL_LEN(ptr) \
+(sizeof(struct syna_csi_header) \
+	+ ((struct syna_csi_header *)(ptr))->data_length \
+)
+
+/* definition for structure and functions between driver and FW */
+
+/* Maxinum csi file dump size */
+#define MAX_CSI_FILESZ		(32 * 1024)
+/* Maxinum subcarrier number */
+#define MAXIMUM_CFR_DATA	2048
+#define CSI_DUMP_PATH		"/sys/bcm-dhd/csi"
+#define MAX_EVENT_SIZE		1400
+/* maximun csi number stored at dhd */
+#define MAX_CSI_NUM		16
+#define MAX_CSI_BUF_NUM		1024
+
+/* CSI VERSION */
+#define CSI_VERSION_V0		0 /* V0 */
+#define CSI_VERSION_V1		1 /* V1 */
+#define CSI_VERSION_V2		2 /* V2 */
+
+struct dhd_cfr_header_v0 {
+	uint8 status:4;     /* bit3-0 */
+	uint8 version:4;    /* bit7-4 */
+	/* Peer MAC address */
+	uint8 peer_macaddr[6];
+	/* Number of Space Time Streams */
+	uint8 sts;
+	/* Number of RX chain */
+	uint8 num_rx;
+	/* Number of subcarrier */
+	uint16 num_carrier;
+	/* Length of the CSI dump */
+	uint32 cfr_dump_length;
+	/* remain unsend CSI data length */
+	uint32 remain_length;
+	/* RSSI */
+	int8 rssi;
+	uint32 data[0];
+} __attribute__ ((packed));
+
+struct dhd_cfr_header_v1 {
+	uint8 status:4;     /* bit3-0 */
+	uint8 version:4;    /* bit7-4 */
+	/* Peer MAC address */
+	uint8 peer_macaddr[6];
+	/* Number of Space Time Streams */
+	uint8 sts;
+	/* Number of RX chain */
+	uint8 num_rx;
+	/* Number of subcarrier */
+	uint16 num_carrier;
+	/* Length of the CSI dump */
+	uint32 cfr_dump_length;
+	/* remain unsend CSI data length */
+	uint32 remain_length;
+	/* RSSI */
+	int8 rssi;
+
+	/* Chip id. 1 for BCM43456/8 */
+	uint8 chip_id;
+	/* Frame control field */
+	uint8 fc;
+	/* Time stamp when CFR capture is taken,
+	 * in microseconds since the epoch
+	 */
+	uint64 cfr_timestamp;
+
+	uint32 data[0];
+} __attribute__ ((packed));
+
+struct dhd_cfr_header_v2 { /* 8 bytes aligment for x64 */
+	/* byte 0 ~ 7 */
+	uint8  status_compat:4;   /* bit3-0, current CSI data status: 0->good */
+	uint8  version_compat:4;  /* bit7-4, duplicate but compatible part */
+	uint8  padding_magic;
+	uint16 magic_flag;        /* flag for protecting and detecting header */
+
+	uint8  version;           /* header version of this structure */
+	uint8  format_type;       /* data format type of current CSI packet,
+	                           * enumerated in 'syna_csi_format_type'
+	                           */
+	uint8  fc;                /* packet frame control type of current
+	                           * CSI data for further checking if it
+	                           * is correct expected result
+	                           */
+	uint8  flags;             /* flag may use for extra emendation
+	                           * which enumerated as 'syna_csi_flag'
+	                           */
+
+	/* byte 8 ~ 19 */
+	uint8  client_ea[6];      /* client MAC address */
+	uint8  bsscfg_ea[6];      /* BSSCFG address of current interface */
+
+	/* byte 20 ~ 23 */
+	uint8  band;              /* enumerated in 'syna_csi_band_type' */
+	uint8  bandwidth;         /* enumerated in 'syna_csi_bandwidth_type' */
+	uint16 channel;           /* channel number of corresponding band */
+
+	/* byte 24 ~ 31 */
+	uint64 report_tsf;        /* current CSI DATA RX timestamp */
+
+	/* byte 32 ~ 35 */
+	uint8  num_txstream;      /* peer side TX spatial steams */
+	uint8  num_rxchain;       /* number of RX side chain/antenna */
+	uint16 num_subcarrier;    /* Number of subcarrier */
+
+	/* byte 36 ~ 39 */
+	int8   rssi;              /* average RSSI, and goto check the
+	                           * 'rssi_ant' if this 'rssi' is zero
+	                           */
+	int8   noise;             /* average noise */
+	int16  global_id;         /* CSI frame global ID */
+
+	/* byte 40 ~ 47 */
+	int8   rssi_ant[8];       /* RSSI of each RX chain/antenna
+	                           * (Depends on solution, and some
+	                           * chipsets may only provide 'rssi'
+	                           * rather than 'rssi_ant')
+	                           */
+
+	/* byte 48 ~ 55 */
+	uint64 padding_future;    /* reserved */
+
+	/* byte 56 ~ 63 */
+	uint16 data_length;       /* the data length of 'data[]' */
+	uint16 remain_length;     /* the remain unsent length of 'data[]' */
+	uint16 padding_length;    /* reserved */
+	uint16 checksum;          /* 'data[]' checksum, 0 - not used */
+
+	/* byte 64 ~ End */
+	uint32 data[0];           /* variable according to 'data_length', but
+	                           * the minial valid data length will be
+	                           * decided by 'num_txchains', 'num_rxchain'
+	                           * 'num_subcarrier'. For example, the
+	                           * quantity of uint32(4 bytes) in the
+	                           * 11N 80MHz can be got via:
+	                           * 256 SubCarriers * 2 TXStream * 2 RXChain
+	                           */
+};
+
+union dhd_cfr_header {
+	struct dhd_cfr_header_v0  header_v0;
+	struct dhd_cfr_header_v1  header_v1;
+	struct dhd_cfr_header_v2  header_v2;
+};
+#define CSI_CFR_TOTAL_LENGTH(data_len) \
+	(sizeof(union dhd_cfr_header) + data_len)
+
+/* BW80 2x2 => 4bytes*256subcarries*2txstream*2rxchain */
+#define CONST_CSI_MAXIMUM_DATA_BYTES	(CSI_CFR_TOTAL_LENGTH(256 *2 *2 *4))
+
+struct csi_cfr_node {
+	struct list_head  list;
+	void  *pNode; /* recording current node pointer
+	               * especially for UDP data manner
+	               */
+	uint32 total_size;
+
+	struct syna_csi_header entry;
+};
+
+#define SYNA_CSI_CFR_NODE_LEN		sizeof(struct csi_cfr_node)
+
+#define SYNA_CSI_CFR_NODE_TOTAL_LEN(ptr, data_length) \
+(\
+	SYNA_CSI_CFR_NODE_LEN + data_length \
+)
+
+#define SYNA_CSI_CFR_NODE_FREE_LEN(ptr) \
+(\
+	((struct csi_cfr_node *)(ptr))->total_size \
+		- SYNA_CSI_CFR_NODE_LEN \
+)
+
+extern int dhd_csi_config(dhd_pub_t *dhdp, char *buf, uint length, bool is_set);
+
+/* Function: change the upper layer fetching data manner
+ * Input:  uint  is_set        indicate it's set or get action
+ *         uint  type          the expected fetch type(ignore when get)
+ *         uint  param         extra param when set(UDP socket port)
+ * Output: <  0        error encounter
+ *         == 0        successful
+ *         >  0        corresponding 'fetch type' when get
+ */
+extern int dhd_csi_data_fetch_type_access(uint is_set, uint type, uint param);
+
+/* Function: parse the CSI event and convert to 'dhd_csi_header' format,
+ *           and queue the item into the common 'csi_cfr_queue'
+ * Input:  dhd_pub_t    *dhdp     context for operation
+ *         char         *buf      buffer for storing csi data
+ *         uint          count    total bytes available in 'buf'
+ * Output: <  0        error encounter
+ *         == 0        successful but no csi data in queue
+ *         >  0        bytes write into 'buf'
+ */
+extern int dhd_csi_retrieve_queue_data(dhd_pub_t *dhdp, char *buf, uint count);
+
+/* Function: parse the CSI event and convert to 'dhd_csi_header' format,
+ *           and queue the item into the common 'csi_cfr_queue'
+ * Input:  dhd_pub_t      *dhdp          context for operation
+ *         wl_event_msg_t *event         wl event message header
+ *         void           *event_data    extra event data
+ * Output: <  0        error encounter
+ *         == 0        successful
+ */
+extern int dhd_csi_event_handler(dhd_pub_t *dhdp,
+	const wl_event_msg_t *event, void *event_data);
+
+/* Function: Initialize the CSI module
+ * Input:  dhd_pub_t      *dhdp          context for operation
+ * Output: <  0        error encounter
+ *         == 0        successful
+ */
+extern int dhd_csi_init(dhd_pub_t *dhdp);
+
+/* Function: Deinitialize the CSI module
+ * Input:  dhd_pub_t      *dhdp          context for operation
+ * Output: <  0        error encounter
+ *         == 0        successful
+ */
+extern int dhd_csi_deinit(dhd_pub_t *dhdp);
+
+#endif /* __DHD_CSI_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_custom_gpio.c b/drivers/net/wireless/ap12275/dhd_custom_gpio.c
new file mode 100644
index 000000000000..21edfc12a9cb
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_custom_gpio.c
@@ -0,0 +1,490 @@
+/*
+ * Customer code to add GPIO control during WLAN start/stop
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <linuxver.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_linux.h>
+
+#include <wlioctl.h>
+
+#include <dhd_dbg.h>
+
+#ifndef BCMDONGLEHOST
+#include <wlc_pub.h>
+#include <wl_dbg.h>
+#else
+#define WL_ERROR(x) printf x
+#define WL_TRACE(x)
+#endif
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+
+#if defined(BCMLXSDMMC)
+extern int sdioh_mmc_irq(int irq);
+#endif /* (BCMLXSDMMC)  */
+
+/* Customer specific Host GPIO defintion  */
+static int dhd_oob_gpio_num = -1;
+
+module_param(dhd_oob_gpio_num, int, 0644);
+MODULE_PARM_DESC(dhd_oob_gpio_num, "DHD oob gpio number");
+
+/* This function will return:
+ *  1) return :  Host gpio interrupt number per customer platform
+ *  2) irq_flags_ptr : Type of Host interrupt as Level or Edge
+ *
+ *  NOTE :
+ *  Customer should check his platform definitions
+ *  and his Host Interrupt spec
+ *  to figure out the proper setting for his platform.
+ *  Broadcom provides just reference settings as example.
+ *
+ */
+int dhd_customer_oob_irq_map(void *adapter, unsigned long *irq_flags_ptr)
+{
+	int  host_oob_irq = 0;
+
+#if defined(BOARD_HIKEY)
+	host_oob_irq = wifi_platform_get_irq_number(adapter, irq_flags_ptr);
+
+#else
+#if defined(CUSTOM_OOB_GPIO_NUM)
+	if (dhd_oob_gpio_num < 0) {
+		dhd_oob_gpio_num = CUSTOM_OOB_GPIO_NUM;
+	}
+#endif /* CUSTOMER_OOB_GPIO_NUM */
+
+	if (dhd_oob_gpio_num < 0) {
+		WL_ERROR(("%s: ERROR customer specific Host GPIO is NOT defined \n",
+		__FUNCTION__));
+		return (dhd_oob_gpio_num);
+	}
+
+	WL_ERROR(("%s: customer specific Host GPIO number is (%d)\n",
+	         __FUNCTION__, dhd_oob_gpio_num));
+
+#endif /* BOARD_HIKER */
+
+	return (host_oob_irq);
+}
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+
+/* Customer function to control hw specific wlan gpios */
+int
+dhd_customer_gpio_wlan_ctrl(void *adapter, int onoff)
+{
+	int err = 0;
+
+	return err;
+}
+
+#if 0
+/* Function to get custom MAC address */
+int
+dhd_custom_get_mac_address(void *adapter, unsigned char *buf)
+{
+	int ret = 0;
+
+	WL_TRACE(("%s Enter\n", __FUNCTION__));
+	if (!buf)
+		return -EINVAL;
+
+	/* Customer access to MAC address stored outside of DHD driver */
+#if defined(BOARD_HIKEY) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	ret = wifi_platform_get_mac_addr(adapter, buf);
+#endif
+
+#ifdef EXAMPLE_GET_MAC
+	/* EXAMPLE code */
+	{
+		struct ether_addr ea_example = {{0x00, 0x11, 0x22, 0x33, 0x44, 0xFF}};
+		bcopy((char *)&ea_example, buf, sizeof(struct ether_addr));
+	}
+#endif /* EXAMPLE_GET_MAC */
+
+	return ret;
+}
+#endif /* GET_CUSTOM_MAC_ENABLE */
+
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+struct customer_cntry_list {
+	eCountry_flag_type  type;
+	char                ccode[WLC_CNTRY_BUF_SZ];   /* Country code */
+};
+
+/* maximum: (A~Z+0~9) * (A~Z+0~9) = (26+10)*(26+10) = 1296 */
+#define MAX_COUNTRY_LIST        500
+struct customer_cntry_list syna_cntry_flag[MAX_COUNTRY_LIST] = {
+	/* FCC TYPE */
+	{SYNA_COUNTRY_TYPE_FCC, "AG"},
+	{SYNA_COUNTRY_TYPE_FCC, "AI"},
+	{SYNA_COUNTRY_TYPE_FCC, "AR"},
+	{SYNA_COUNTRY_TYPE_FCC, "AW"},
+	{SYNA_COUNTRY_TYPE_FCC, "BB"},
+	{SYNA_COUNTRY_TYPE_FCC, "BL"},
+	{SYNA_COUNTRY_TYPE_FCC, "BM"},
+	{SYNA_COUNTRY_TYPE_FCC, "BO"},
+	{SYNA_COUNTRY_TYPE_FCC, "BS"},
+	{SYNA_COUNTRY_TYPE_FCC, "BZ"},
+	{SYNA_COUNTRY_TYPE_FCC, "CA"},
+	{SYNA_COUNTRY_TYPE_FCC, "CL"},
+	{SYNA_COUNTRY_TYPE_FCC, "CO"},
+	{SYNA_COUNTRY_TYPE_FCC, "CR"},
+	{SYNA_COUNTRY_TYPE_FCC, "CW"},
+	{SYNA_COUNTRY_TYPE_FCC, "DO"},
+	{SYNA_COUNTRY_TYPE_FCC, "EC"},
+	{SYNA_COUNTRY_TYPE_FCC, "FM"},
+	{SYNA_COUNTRY_TYPE_FCC, "GT"},
+	{SYNA_COUNTRY_TYPE_FCC, "GU"},
+	{SYNA_COUNTRY_TYPE_FCC, "GY"},
+	{SYNA_COUNTRY_TYPE_FCC, "HN"},
+	{SYNA_COUNTRY_TYPE_FCC, "HT"},
+	{SYNA_COUNTRY_TYPE_FCC, "IN"},
+	{SYNA_COUNTRY_TYPE_FCC, "IR"},
+	{SYNA_COUNTRY_TYPE_FCC, "JM"},
+	{SYNA_COUNTRY_TYPE_FCC, "KH"},
+	{SYNA_COUNTRY_TYPE_FCC, "KN"},
+	{SYNA_COUNTRY_TYPE_FCC, "KR"},
+	{SYNA_COUNTRY_TYPE_FCC, "KY"},
+	{SYNA_COUNTRY_TYPE_FCC, "LC"},
+	{SYNA_COUNTRY_TYPE_FCC, "MF"},
+	{SYNA_COUNTRY_TYPE_FCC, "MP"},
+	{SYNA_COUNTRY_TYPE_FCC, "NI"},
+	{SYNA_COUNTRY_TYPE_FCC, "PA"},
+	{SYNA_COUNTRY_TYPE_FCC, "PE"},
+	{SYNA_COUNTRY_TYPE_FCC, "PG"},
+	{SYNA_COUNTRY_TYPE_FCC, "PM"},
+	{SYNA_COUNTRY_TYPE_FCC, "PW"},
+	{SYNA_COUNTRY_TYPE_FCC, "PR"},
+	{SYNA_COUNTRY_TYPE_FCC, "PY"},
+	{SYNA_COUNTRY_TYPE_FCC, "SR"},
+	{SYNA_COUNTRY_TYPE_FCC, "SV"},
+	{SYNA_COUNTRY_TYPE_FCC, "SX"},
+	{SYNA_COUNTRY_TYPE_FCC, "TC"},
+	{SYNA_COUNTRY_TYPE_FCC, "TT"},
+	{SYNA_COUNTRY_TYPE_FCC, "US"},
+	{SYNA_COUNTRY_TYPE_FCC, "UY"},
+	{SYNA_COUNTRY_TYPE_FCC, "VC"},
+	{SYNA_COUNTRY_TYPE_FCC, "VE"},
+	{SYNA_COUNTRY_TYPE_FCC, "VG"},
+	{SYNA_COUNTRY_TYPE_FCC, "VI"},
+	{SYNA_COUNTRY_TYPE_FCC, "VU"},
+	{SYNA_COUNTRY_TYPE_FCC, "WF"},
+	{SYNA_COUNTRY_TYPE_FCC, "WS"},
+	{SYNA_COUNTRY_TYPE_INVALID, ""}
+};
+
+static int gSyna_country_flag_qty = 55;
+
+static int syna_country_flag_find(char *cntry)
+{
+	char country[WLC_CNTRY_BUF_SZ];
+	int  ret = BCME_NOTFOUND;
+	int  n;
+
+	if (!cntry) {
+		DHD_ERROR(("%s: invalid cntry!\n", __FUNCTION__));
+		return BCME_BADARG;
+	} else {
+		n = strcspn(cntry, " ,");
+		if ((n != 2) && (n != 3)) {
+			DHD_ERROR(("%s: invalid n=%d!\n", __FUNCTION__, n));
+			return BCME_BUFTOOLONG;
+		} else {
+			strlcpy(country, cntry, WLC_CNTRY_BUF_SZ);
+			country[n] = '\0';
+			DHD_TRACE(("%s: here n=%d, country='%s'!\n",
+			           __FUNCTION__, n, country));
+		}
+	}
+
+	n = 0;
+	while ((MAX_COUNTRY_LIST > n) &&
+	       (SYNA_COUNTRY_TYPE_INVALID != syna_cntry_flag[n].type) &&
+	       (0 < strlen(syna_cntry_flag[n].ccode))) {
+		if (!strncmp(country, syna_cntry_flag[n].ccode, WLC_CNTRY_BUF_SZ)) {
+			ret = n;
+			break;
+		} else {
+			n++;
+		}
+	}
+
+	/* sync up the quantity for appending new item */
+	if ((MAX_COUNTRY_LIST > n) &&
+	    (SYNA_COUNTRY_TYPE_INVALID == syna_cntry_flag[n].type)) {
+		gSyna_country_flag_qty = n;
+		DHD_TRACE(("%s: update qty=%d!\n", __FUNCTION__, n));
+	}
+
+	return ret;
+}
+
+eCountry_flag_type syna_country_check_type(char *cntry)
+{
+	int ret = SYNA_COUNTRY_TYPE_INVALID;
+	int i;
+
+	i = syna_country_flag_find(cntry);
+	if (0 <= i) {
+		ret = syna_cntry_flag[i].type;
+	}
+
+	return ret;
+}
+
+int syna_country_update_type_list(eCountry_flag_type type, char *list_str)
+{
+	char   *next = NULL;
+	int     n = 0;
+	size_t  len;
+
+	if (SYNA_COUNTRY_TYPE_QTY <= type) {
+		return BCME_BADARG;
+	} else if ((next = list_str) == NULL) {
+		return BCME_BADARG;
+	}
+
+	while ((len = strcspn(next, " ,")) > 0) {
+		if (len >= WLC_CNTRY_BUF_SZ) {
+			DHD_ERROR(("%s: string '%s' before ',' or ' ' is too long\n",
+			           __FUNCTION__, next));
+			return BCME_ERROR;
+		}
+
+		n = syna_country_flag_find(next);
+		if (0 > n) {
+			if (BCME_NOTFOUND == n) {
+				if (MAX_COUNTRY_LIST <= (1 + gSyna_country_flag_qty)) {
+					DHD_ERROR(("%s: too many ccode(max=%d) list: '%s'\n",
+					           __FUNCTION__, MAX_COUNTRY_LIST, list_str));
+					break;
+				} else {
+					/* adopt this new item */
+					n = gSyna_country_flag_qty;
+					gSyna_country_flag_qty++;
+				}
+			} else {
+				DHD_ERROR(("%s: skip this invalid item '%s', err=%d\n",
+				           __FUNCTION__, next, n));
+			}
+		}
+
+		if (0 <= n) {
+			/* update or add a new item */
+			syna_cntry_flag[n].type = type;
+			strlcpy(syna_cntry_flag[n].ccode, next, WLC_CNTRY_BUF_SZ);
+			syna_cntry_flag[n].ccode[len] = '\0';
+			DHD_TRACE(("%s: cntry list [%d] type=%d ccode=%s\n",
+			           __FUNCTION__, n,
+			           syna_cntry_flag[n].type, syna_cntry_flag[n].ccode));
+
+			/* fill with a new blank item as terminate */
+			if ((gSyna_country_flag_qty == (n + 1)) &&
+				(MAX_COUNTRY_LIST > gSyna_country_flag_qty)) {
+				syna_cntry_flag[n+1].type = SYNA_COUNTRY_TYPE_INVALID;
+				syna_cntry_flag[n+1].ccode[0] = '\0';
+			}
+		}
+
+		next += len;
+		next += strspn(next, " ,");
+	}
+
+	return BCME_OK;
+}
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+
+/* Customized Locale table : OPTIONAL feature */
+const struct cntry_locales_custom translate_custom_table[] = {
+/* Table should be filled out based on custom platform regulatory requirement */
+#ifdef EXAMPLE_TABLE
+	{"",   "XY", 4},  /* Universal if Country code is unknown or empty */
+	{"US", "US", 69}, /* input ISO "US" to : US regrev 69 */
+	{"CA", "US", 69}, /* input ISO "CA" to : US regrev 69 */
+	{"EU", "EU", 5},  /* European union countries to : EU regrev 05 */
+	{"AT", "EU", 5},
+	{"BE", "EU", 5},
+	{"BG", "EU", 5},
+	{"CY", "EU", 5},
+	{"CZ", "EU", 5},
+	{"DK", "EU", 5},
+	{"EE", "EU", 5},
+	{"FI", "EU", 5},
+	{"FR", "EU", 5},
+	{"DE", "EU", 5},
+	{"GR", "EU", 5},
+	{"HU", "EU", 5},
+	{"IE", "EU", 5},
+	{"IT", "EU", 5},
+	{"LV", "EU", 5},
+	{"LI", "EU", 5},
+	{"LT", "EU", 5},
+	{"LU", "EU", 5},
+	{"MT", "EU", 5},
+	{"NL", "EU", 5},
+	{"PL", "EU", 5},
+	{"PT", "EU", 5},
+	{"RO", "EU", 5},
+	{"SK", "EU", 5},
+	{"SI", "EU", 5},
+	{"ES", "EU", 5},
+	{"SE", "EU", 5},
+	{"GB", "EU", 5},
+	{"KR", "XY", 3},
+	{"AU", "XY", 3},
+	{"CN", "XY", 3}, /* input ISO "CN" to : XY regrev 03 */
+	{"TW", "XY", 3},
+	{"AR", "XY", 3},
+	{"MX", "XY", 3},
+	{"IL", "IL", 0},
+	{"CH", "CH", 0},
+	{"TR", "TR", 0},
+	{"NO", "NO", 0},
+#endif /* EXMAPLE_TABLE */
+#if defined(BOARD_HIKEY)
+#if defined(BCM4335_CHIP)
+	{"",   "XZ", 11},  /* Universal if Country code is unknown or empty */
+#endif
+	{"AE", "AE", 1},
+	{"AR", "AR", 1},
+	{"AT", "AT", 1},
+	{"AU", "AU", 2},
+	{"BE", "BE", 1},
+	{"BG", "BG", 1},
+	{"BN", "BN", 1},
+	{"CA", "CA", 2},
+	{"CH", "CH", 1},
+	{"CY", "CY", 1},
+	{"CZ", "CZ", 1},
+	{"DE", "DE", 3},
+	{"DK", "DK", 1},
+	{"EE", "EE", 1},
+	{"ES", "ES", 1},
+	{"FI", "FI", 1},
+	{"FR", "FR", 1},
+	{"GB", "GB", 1},
+	{"GR", "GR", 1},
+	{"HR", "HR", 1},
+	{"HU", "HU", 1},
+	{"IE", "IE", 1},
+	{"IS", "IS", 1},
+	{"IT", "IT", 1},
+	{"ID", "ID", 1},
+	{"JP", "JP", 8},
+	{"KR", "KR", 24},
+	{"KW", "KW", 1},
+	{"LI", "LI", 1},
+	{"LT", "LT", 1},
+	{"LU", "LU", 1},
+	{"LV", "LV", 1},
+	{"MA", "MA", 1},
+	{"MT", "MT", 1},
+	{"MX", "MX", 1},
+	{"NL", "NL", 1},
+	{"NO", "NO", 1},
+	{"PL", "PL", 1},
+	{"PT", "PT", 1},
+	{"PY", "PY", 1},
+	{"RO", "RO", 1},
+	{"SE", "SE", 1},
+	{"SI", "SI", 1},
+	{"SK", "SK", 1},
+	{"TR", "TR", 7},
+	{"TW", "TW", 1},
+	{"IR", "XZ", 11},	/* Universal if Country code is IRAN, (ISLAMIC REPUBLIC OF) */
+	{"SD", "XZ", 11},	/* Universal if Country code is SUDAN */
+	{"SY", "XZ", 11},	/* Universal if Country code is SYRIAN ARAB REPUBLIC */
+	{"GL", "XZ", 11},	/* Universal if Country code is GREENLAND */
+	{"PS", "XZ", 11},	/* Universal if Country code is PALESTINIAN TERRITORY, OCCUPIED */
+	{"TL", "XZ", 11},	/* Universal if Country code is TIMOR-LESTE (EAST TIMOR) */
+	{"MH", "XZ", 11},	/* Universal if Country code is MARSHALL ISLANDS */
+#endif /* BOARD_HIKEY */
+};
+
+/* Customized Locale convertor
+*  input : ISO 3166-1 country abbreviation
+*  output: customized cspec
+*/
+void
+#ifdef CUSTOM_COUNTRY_CODE
+get_customized_country_code(void *adapter, char *country_iso_code,
+	wl_country_t *cspec, u32 flags)
+#else
+get_customized_country_code(void *adapter, char *country_iso_code, wl_country_t *cspec)
+#endif /* CUSTOM_COUNTRY_CODE */
+{
+#if defined(OEM_ANDROID)
+#if (defined(CUSTOMER_HW) || defined(CUSTOMER_HW2)) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+
+	struct cntry_locales_custom *cloc_ptr;
+
+	if (!cspec)
+		return;
+#ifdef CUSTOM_COUNTRY_CODE
+	cloc_ptr = wifi_platform_get_country_code(adapter, country_iso_code, flags);
+#else
+	cloc_ptr = wifi_platform_get_country_code(adapter, country_iso_code);
+#endif /* CUSTOM_COUNTRY_CODE */
+
+	if (cloc_ptr) {
+		strlcpy(cspec->ccode, cloc_ptr->custom_locale, WLC_CNTRY_BUF_SZ);
+		cspec->rev = cloc_ptr->custom_locale_rev;
+	}
+	return;
+#else
+	int size, i;
+
+	size = ARRAYSIZE(translate_custom_table);
+
+	if (cspec == 0)
+		 return;
+
+	if (size == 0)
+		 return;
+
+	for (i = 0; i < size; i++) {
+		if (strcmp(country_iso_code, translate_custom_table[i].iso_abbrev) == 0) {
+			memcpy(cspec->ccode,
+				translate_custom_table[i].custom_locale, WLC_CNTRY_BUF_SZ);
+			cspec->rev = translate_custom_table[i].custom_locale_rev;
+			return;
+		}
+	}
+#ifdef EXAMPLE_TABLE
+	/* if no country code matched return first universal code from translate_custom_table */
+	memcpy(cspec->ccode, translate_custom_table[0].custom_locale, WLC_CNTRY_BUF_SZ);
+	cspec->rev = translate_custom_table[0].custom_locale_rev;
+#endif /* EXMAPLE_TABLE */
+	return;
+#endif /* (defined(CUSTOMER_HW2) || defined(BOARD_HIKEY)) &&
+	* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36))
+	*/
+#endif /* OEM_ANDROID */
+}
diff --git a/drivers/net/wireless/ap12275/dhd_custom_hikey.c b/drivers/net/wireless/ap12275/dhd_custom_hikey.c
new file mode 100644
index 000000000000..a148c51891c3
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_custom_hikey.c
@@ -0,0 +1,342 @@
+/*
+ * Platform Dependent file for Hikey
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/skbuff.h>
+#include <linux/fcntl.h>
+#include <linux/fs.h>
+#include <linux/of_gpio.h>
+#ifdef CONFIG_WIFI_CONTROL_FUNC
+#include <linux/wlan_plat.h>
+#else
+#include <dhd_plat.h>
+#endif /* CONFIG_WIFI_CONTROL_FUNC */
+#include <dhd_dbg.h>
+#include <dhd.h>
+
+#ifdef CONFIG_BROADCOM_WIFI_RESERVED_MEM
+extern int dhd_init_wlan_mem(void);
+extern void *dhd_wlan_mem_prealloc(int section, unsigned long size);
+#endif /* CONFIG_BROADCOM_WIFI_RESERVED_MEM */
+
+#define HIKEY_PCIE_VENDOR_ID 0x19e5
+#define HIKEY_PCIE_DEVICE_ID 0x3660
+
+#define WLAN_REG_ON_GPIO		491
+#define WLAN_HOST_WAKE_GPIO		493
+
+static int wlan_reg_on = -1;
+#define DHD_DT_COMPAT_ENTRY		"android,bcmdhd_wlan"
+#define WIFI_WL_REG_ON_PROPNAME		"wl_reg_on"
+
+static int wlan_host_wake_up = -1;
+static int wlan_host_wake_irq = 0;
+#define WIFI_WLAN_HOST_WAKE_PROPNAME    "wl_host_wake"
+
+extern void kirin_pcie_power_on_atu_fixup(void) __attribute__ ((weak));
+extern int kirin_pcie_lp_ctrl(u32 enable) __attribute__ ((weak));
+
+void
+dhd_wifi_deinit_gpio(void)
+{
+	/* Disable ASPM before powering off */
+	if (kirin_pcie_lp_ctrl) {
+		kirin_pcie_lp_ctrl(0);
+	} else {
+		DHD_TRACE(("[%s] kirin_pcie_lp_ctrl is NULL. "
+			"ASPM may not work\n", __func__));
+	}
+	if (gpio_direction_output(wlan_reg_on, 0)) {
+		DHD_TRACE(("%s: WL_REG_ON is failed to pull up\n", __FUNCTION__));
+	}
+	if (gpio_get_value(wlan_reg_on)) {
+		DHD_INFO(("WL_REG_ON on-step-2 : [%d]\n",
+			gpio_get_value(wlan_reg_on)));
+	}
+	gpio_free(wlan_host_wake_up);
+	gpio_free(wlan_reg_on);
+}
+
+int
+dhd_wifi_init_gpio(void)
+{
+	int gpio_reg_on_val;
+	/* ========== WLAN_PWR_EN ============ */
+	char *wlan_node = DHD_DT_COMPAT_ENTRY;
+	struct device_node *root_node = NULL;
+
+	root_node = of_find_compatible_node(NULL, NULL, wlan_node);
+	if (root_node) {
+		wlan_reg_on = of_get_named_gpio(root_node, WIFI_WL_REG_ON_PROPNAME, 0);
+		wlan_host_wake_up = of_get_named_gpio(root_node, WIFI_WLAN_HOST_WAKE_PROPNAME, 0);
+	} else {
+		DHD_ERROR(("failed to get device node of BRCM WLAN, use default GPIOs\n"));
+		wlan_reg_on = WLAN_REG_ON_GPIO;
+		wlan_host_wake_up = WLAN_HOST_WAKE_GPIO;
+	}
+
+	/* ========== WLAN_PWR_EN ============ */
+	DHD_INFO(("%s: gpio_wlan_power : %d\n", __FUNCTION__, wlan_reg_on));
+
+	/*
+	 * For reg_on, gpio_request will fail if the gpio is configured to output-high
+	 * in the dts using gpio-hog, so do not return error for failure.
+	 */
+	if (gpio_request_one(wlan_reg_on, GPIOF_DIR_OUT, "WL_REG_ON")) {
+		DHD_INFO(("%s: Failed to request gpio %d for WL_REG_ON, "
+			"might have configured in the dts\n",
+			__FUNCTION__, wlan_reg_on));
+	} else {
+		DHD_INFO(("%s: gpio_request WL_REG_ON done - WLAN_EN: GPIO %d\n",
+			__FUNCTION__, wlan_reg_on));
+	}
+
+	gpio_reg_on_val = gpio_get_value(wlan_reg_on);
+	DHD_INFO(("%s: Initial WL_REG_ON: [%d]\n",
+		__FUNCTION__, gpio_get_value(wlan_reg_on)));
+
+	if (gpio_reg_on_val == 0) {
+		DHD_INFO(("%s: WL_REG_ON is LOW, drive it HIGH\n", __FUNCTION__));
+		if (gpio_direction_output(wlan_reg_on, 1)) {
+			DHD_ERROR(("%s: WL_REG_ON is failed to pull up\n", __FUNCTION__));
+			return -EIO;
+		}
+		/* Wait for WIFI_TURNON_DELAY due to power stability */
+		msleep(WIFI_TURNON_DELAY);
+
+		/*
+		 * Call Kiric RC ATU fixup else si_attach will fail due to
+		 * improper BAR0/1 address translations
+		 */
+		if (kirin_pcie_power_on_atu_fixup) {
+			kirin_pcie_power_on_atu_fixup();
+		} else {
+			DHD_INFO(("[%s] kirin_pcie_power_on_atu_fixup is NULL. "
+				"REG_ON may not work\n", __func__));
+		}
+		/* Enable ASPM after powering ON */
+		if (kirin_pcie_lp_ctrl) {
+			kirin_pcie_lp_ctrl(1);
+		} else {
+			DHD_INFO(("[%s] kirin_pcie_lp_ctrl is NULL. "
+				"ASPM may not work\n", __func__));
+		}
+	}
+
+	/* ========== WLAN_HOST_WAKE ============ */
+	DHD_INFO(("%s: gpio_wlan_host_wake : %d\n", __FUNCTION__, wlan_host_wake_up));
+
+	if (gpio_request_one(wlan_host_wake_up, GPIOF_IN, "WLAN_HOST_WAKE")) {
+		DHD_ERROR(("%s: Failed to request gpio %d for WLAN_HOST_WAKE\n",
+			__FUNCTION__, wlan_host_wake_up));
+			return -ENODEV;
+	} else {
+		DHD_INFO(("%s: gpio_request WLAN_HOST_WAKE done"
+			" - WLAN_HOST_WAKE: GPIO %d\n",
+			__FUNCTION__, wlan_host_wake_up));
+	}
+
+	if (gpio_direction_input(wlan_host_wake_up)) {
+		DHD_ERROR(("%s: Failed to set WL_HOST_WAKE gpio direction\n", __FUNCTION__));
+		return -EIO;
+	}
+
+	wlan_host_wake_irq = gpio_to_irq(wlan_host_wake_up);
+
+	return 0;
+}
+
+int
+dhd_wlan_power(int onoff)
+{
+	DHD_INFO(("------------------------------------------------"));
+	DHD_INFO(("------------------------------------------------\n"));
+	DHD_INFO(("%s Enter: power %s\n", __func__, onoff ? "on" : "off"));
+
+	if (onoff) {
+		if (gpio_direction_output(wlan_reg_on, 1)) {
+			DHD_ERROR(("%s: WL_REG_ON is failed to pull up\n", __FUNCTION__));
+			return -EIO;
+		}
+		if (gpio_get_value(wlan_reg_on)) {
+			DHD_INFO(("WL_REG_ON on-step-2 : [%d]\n",
+				gpio_get_value(wlan_reg_on)));
+		} else {
+			DHD_ERROR(("[%s] gpio value is 0. We need reinit.\n", __func__));
+			if (gpio_direction_output(wlan_reg_on, 1)) {
+				DHD_ERROR(("%s: WL_REG_ON is "
+					"failed to pull up\n", __func__));
+			}
+		}
+
+		/* Wait for WIFI_TURNON_DELAY due to power stability */
+		msleep(WIFI_TURNON_DELAY);
+
+		/*
+		 * Call Kiric RC ATU fixup else si_attach will fail due to
+		 * improper BAR0/1 address translations
+		 */
+		if (kirin_pcie_power_on_atu_fixup) {
+			kirin_pcie_power_on_atu_fixup();
+		} else {
+			DHD_ERROR(("[%s] kirin_pcie_power_on_atu_fixup is NULL. "
+				"REG_ON may not work\n", __func__));
+		}
+		/* Enable ASPM after powering ON */
+		if (kirin_pcie_lp_ctrl) {
+			kirin_pcie_lp_ctrl(onoff);
+		} else {
+			DHD_ERROR(("[%s] kirin_pcie_lp_ctrl is NULL. "
+				"ASPM may not work\n", __func__));
+		}
+	} else {
+		/* Disable ASPM before powering off */
+		if (kirin_pcie_lp_ctrl) {
+			kirin_pcie_lp_ctrl(onoff);
+		} else {
+			DHD_ERROR(("[%s] kirin_pcie_lp_ctrl is NULL. "
+				"ASPM may not work\n", __func__));
+		}
+		if (gpio_direction_output(wlan_reg_on, 0)) {
+			DHD_ERROR(("%s: WL_REG_ON is failed to pull up\n", __FUNCTION__));
+			return -EIO;
+		}
+		if (gpio_get_value(wlan_reg_on)) {
+			DHD_INFO(("WL_REG_ON on-step-2 : [%d]\n",
+				gpio_get_value(wlan_reg_on)));
+		}
+	}
+	return 0;
+}
+EXPORT_SYMBOL(dhd_wlan_power);
+
+static int
+dhd_wlan_reset(int onoff)
+{
+	return 0;
+}
+
+static int
+dhd_wlan_set_carddetect(int val)
+{
+	return 0;
+}
+
+#ifdef BCMSDIO
+static int dhd_wlan_get_wake_irq(void)
+{
+	return gpio_to_irq(wlan_host_wake_up);
+}
+#endif /* BCMSDIO */
+
+#if defined(CONFIG_BCMDHD_OOB_HOST_WAKE) && defined(CONFIG_BCMDHD_GET_OOB_STATE)
+int
+dhd_get_wlan_oob_gpio(void)
+{
+	return gpio_is_valid(wlan_host_wake_up) ?
+		gpio_get_value(wlan_host_wake_up) : -1;
+}
+EXPORT_SYMBOL(dhd_get_wlan_oob_gpio);
+#endif /* CONFIG_BCMDHD_OOB_HOST_WAKE && CONFIG_BCMDHD_GET_OOB_STATE */
+
+struct resource dhd_wlan_resources = {
+	.name	= "bcmdhd_wlan_irq",
+	.start	= 0, /* Dummy */
+	.end	= 0, /* Dummy */
+	.flags	= IORESOURCE_IRQ | IORESOURCE_IRQ_SHAREABLE |
+	IORESOURCE_IRQ_HIGHEDGE,
+};
+EXPORT_SYMBOL(dhd_wlan_resources);
+
+struct wifi_platform_data dhd_wlan_control = {
+	.set_power	= dhd_wlan_power,
+	.set_reset	= dhd_wlan_reset,
+	.set_carddetect	= dhd_wlan_set_carddetect,
+#ifdef CONFIG_BROADCOM_WIFI_RESERVED_MEM
+	.mem_prealloc	= dhd_wlan_mem_prealloc,
+#endif /* CONFIG_BROADCOM_WIFI_RESERVED_MEM */
+#ifdef BCMSDIO
+	.get_wake_irq   = dhd_wlan_get_wake_irq,
+#endif
+};
+EXPORT_SYMBOL(dhd_wlan_control);
+
+int
+dhd_wlan_init(void)
+{
+	int ret;
+
+	DHD_INFO(("%s: START.......\n", __FUNCTION__));
+#ifdef CONFIG_BROADCOM_WIFI_RESERVED_MEM
+	ret = dhd_init_wlan_mem();
+	if (ret < 0) {
+		DHD_ERROR(("%s: failed to alloc reserved memory,"
+				" ret=%d\n", __FUNCTION__, ret));
+		goto fail;
+	}
+#endif /* CONFIG_BROADCOM_WIFI_RESERVED_MEM */
+
+	ret = dhd_wifi_init_gpio();
+	if (ret < 0) {
+		DHD_ERROR(("%s: failed to initiate GPIO, ret=%d\n",
+			__FUNCTION__, ret));
+		goto fail;
+	}
+
+	dhd_wlan_resources.start = wlan_host_wake_irq;
+	dhd_wlan_resources.end = wlan_host_wake_irq;
+
+fail:
+	DHD_INFO(("%s: FINISH.......\n", __FUNCTION__));
+	return ret;
+}
+
+int
+dhd_wlan_deinit(void)
+{
+	dhd_wifi_deinit_gpio();
+	return 0;
+}
+
+uint32 dhd_plat_get_rc_vendor_id(void)
+{
+	return HIKEY_PCIE_VENDOR_ID;
+}
+
+uint32 dhd_plat_get_rc_device_id(void)
+{
+	return HIKEY_PCIE_DEVICE_ID;
+}
+
+#ifndef BCMDHD_MODULAR
+/* Required only for Built-in DHD */
+device_initcall(dhd_wlan_init);
+#endif /* BOARD_HIKEY_MODULAR */
diff --git a/drivers/net/wireless/ap12275/dhd_custom_memprealloc.c b/drivers/net/wireless/ap12275/dhd_custom_memprealloc.c
new file mode 100644
index 000000000000..fac72fe11be3
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_custom_memprealloc.c
@@ -0,0 +1,476 @@
+/*
+ * Platform Dependent file for usage of Preallocted Memory
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/miscdevice.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/list.h>
+#include <linux/io.h>
+#include <linux/workqueue.h>
+#include <linux/unistd.h>
+#include <linux/bug.h>
+#include <linux/skbuff.h>
+#include <linux/init.h>
+
+#ifdef CONFIG_BROADCOM_WIFI_RESERVED_MEM
+/* Please refer to 'enum dhd_prealloc_index' in dhd.h */
+#define WLAN_STATIC_SCAN_BUF0		5
+#define WLAN_STATIC_SCAN_BUF1		6
+#define WLAN_STATIC_DHD_INFO_BUF	7
+#define WLAN_STATIC_DHD_WLFC_BUF	8
+#define WLAN_STATIC_DHD_IF_FLOW_LKUP	9
+#define WLAN_STATIC_DHD_MEMDUMP_RAM	11
+#define WLAN_STATIC_DHD_WLFC_HANGER	12
+#define WLAN_STATIC_DHD_PKTID_MAP	13	/* Deprecated */
+#define WLAN_STATIC_DHD_PKTID_IOCTL_MAP	14	/* Deprecated */
+#define WLAN_STATIC_DHD_LOG_DUMP_BUF	15
+#define WLAN_STATIC_DHD_LOG_DUMP_BUF_EX	16
+#define WLAN_STATIC_DHD_PKTLOG_DUMP_BUF	17	/* Deprecated */
+
+#define WLAN_SCAN_BUF_SIZE		(64 * 1024)
+
+#define WLAN_DHD_INFO_BUF_SIZE		(64 * 1024)
+#define WLAN_DHD_WLFC_BUF_SIZE		(64 * 1024)
+#define WLAN_DHD_IF_FLOW_LKUP_SIZE	(64 * 1024)
+/* Have 2MB ramsize to accomodate future chips */
+#define WLAN_DHD_MEMDUMP_SIZE		(3 * 1024 * 1024)
+
+#define PREALLOC_WLAN_SEC_NUM		4
+#define PREALLOC_WLAN_BUF_NUM		160
+#define PREALLOC_WLAN_SECTION_HEADER	24
+
+#ifdef CONFIG_BCMDHD_PCIE
+#define DHD_SKB_1PAGE_BUFSIZE	(PAGE_SIZE*1)
+#define DHD_SKB_2PAGE_BUFSIZE	(PAGE_SIZE*2)
+#define DHD_SKB_4PAGE_BUFSIZE	(PAGE_SIZE*4)
+
+#define WLAN_SECTION_SIZE_0	(PREALLOC_WLAN_BUF_NUM * 128)
+#define WLAN_SECTION_SIZE_1	0
+#define WLAN_SECTION_SIZE_2	0
+#define WLAN_SECTION_SIZE_3	(PREALLOC_WLAN_BUF_NUM * 1024)
+
+#define DHD_SKB_1PAGE_BUF_NUM	0
+#define DHD_SKB_2PAGE_BUF_NUM	192
+#define DHD_SKB_4PAGE_BUF_NUM	0
+
+#else
+#define DHD_SKB_HDRSIZE		336
+#define DHD_SKB_1PAGE_BUFSIZE	((PAGE_SIZE*1)-DHD_SKB_HDRSIZE)
+#define DHD_SKB_2PAGE_BUFSIZE	((PAGE_SIZE*2)-DHD_SKB_HDRSIZE)
+#define DHD_SKB_4PAGE_BUFSIZE	((PAGE_SIZE*4)-DHD_SKB_HDRSIZE)
+
+#define WLAN_SECTION_SIZE_0	(PREALLOC_WLAN_BUF_NUM * 128)
+#define WLAN_SECTION_SIZE_1	(PREALLOC_WLAN_BUF_NUM * 128)
+#define WLAN_SECTION_SIZE_2	(PREALLOC_WLAN_BUF_NUM * 512)
+#define WLAN_SECTION_SIZE_3	(PREALLOC_WLAN_BUF_NUM * 1024)
+
+#define DHD_SKB_1PAGE_BUF_NUM	8
+#define DHD_SKB_2PAGE_BUF_NUM	8
+#define DHD_SKB_4PAGE_BUF_NUM	1
+#endif /* CONFIG_BCMDHD_PCIE */
+
+#define WLAN_SKB_1_2PAGE_BUF_NUM	((DHD_SKB_1PAGE_BUF_NUM) + \
+		(DHD_SKB_2PAGE_BUF_NUM))
+#define WLAN_SKB_BUF_NUM	((WLAN_SKB_1_2PAGE_BUF_NUM) + \
+		(DHD_SKB_4PAGE_BUF_NUM))
+
+#define WLAN_MAX_PKTID_ITEMS		(8192)
+#define WLAN_DHD_PKTID_MAP_HDR_SIZE	(20 + 4*(WLAN_MAX_PKTID_ITEMS + 1))
+#define WLAN_DHD_PKTID_MAP_ITEM_SIZE	(32)
+#define WLAN_DHD_PKTID_MAP_SIZE		((WLAN_DHD_PKTID_MAP_HDR_SIZE) + \
+		((WLAN_MAX_PKTID_ITEMS+1) * WLAN_DHD_PKTID_MAP_ITEM_SIZE))
+
+#define WLAN_MAX_PKTID_IOCTL_ITEMS	(32)
+#define WLAN_DHD_PKTID_IOCTL_MAP_HDR_SIZE	(20 + 4*(WLAN_MAX_PKTID_IOCTL_ITEMS + 1))
+#define WLAN_DHD_PKTID_IOCTL_MAP_ITEM_SIZE	(32)
+#define WLAN_DHD_PKTID_IOCTL_MAP_SIZE		((WLAN_DHD_PKTID_IOCTL_MAP_HDR_SIZE) + \
+		((WLAN_MAX_PKTID_IOCTL_ITEMS+1) * WLAN_DHD_PKTID_IOCTL_MAP_ITEM_SIZE))
+
+#define DHD_LOG_DUMP_BUF_SIZE	(1024 * 1024 * 4)
+#define DHD_LOG_DUMP_BUF_EX_SIZE	(1024 * 8)
+
+#define DHD_PKTLOG_DUMP_BUF_SIZE	(64 * 1024)
+
+#define WLAN_DHD_WLFC_HANGER_MAXITEMS		3072
+#define WLAN_DHD_WLFC_HANGER_ITEM_SIZE		32
+#define WLAN_DHD_WLFC_HANGER_SIZE	((WLAN_DHD_WLFC_HANGER_ITEM_SIZE) + \
+	((WLAN_DHD_WLFC_HANGER_MAXITEMS) * (WLAN_DHD_WLFC_HANGER_ITEM_SIZE)))
+
+static struct sk_buff *wlan_static_skb[WLAN_SKB_BUF_NUM];
+
+struct wlan_mem_prealloc {
+	void *mem_ptr;
+	unsigned long size;
+};
+
+static struct wlan_mem_prealloc wlan_mem_array[PREALLOC_WLAN_SEC_NUM] = {
+	{NULL, (WLAN_SECTION_SIZE_0 + PREALLOC_WLAN_SECTION_HEADER)},
+	{NULL, (WLAN_SECTION_SIZE_1 + PREALLOC_WLAN_SECTION_HEADER)},
+	{NULL, (WLAN_SECTION_SIZE_2 + PREALLOC_WLAN_SECTION_HEADER)},
+	{NULL, (WLAN_SECTION_SIZE_3 + PREALLOC_WLAN_SECTION_HEADER)}
+};
+
+static void *wlan_static_scan_buf0 = NULL;
+static void *wlan_static_scan_buf1 = NULL;
+static void *wlan_static_dhd_info_buf = NULL;
+#ifdef CONFIG_BCMDHD_PCIE
+static void *wlan_static_if_flow_lkup = NULL;
+#else
+static void *wlan_static_dhd_wlfc_buf = NULL;
+static void *wlan_static_dhd_wlfc_hanger = NULL;
+#endif /* CONFIG_BCMDHD_PCIE */
+#ifdef CONFIG_BCMDHD_PREALLOC_MEMDUMP
+static void *wlan_static_dhd_memdump_ram = NULL;
+#endif /* #ifdef CONFIG_BCMDHD_PREALLOC_MEMDUMP */
+static void *wlan_static_dhd_log_dump_buf = NULL;
+static void *wlan_static_dhd_log_dump_buf_ex = NULL;
+
+void dhd_exit_wlan_mem(void);
+
+void
+*dhd_wlan_mem_prealloc(int section, unsigned long size)
+{
+	if (section == PREALLOC_WLAN_SEC_NUM) {
+		return wlan_static_skb;
+	}
+
+	if (section == WLAN_STATIC_SCAN_BUF0) {
+		return wlan_static_scan_buf0;
+	}
+
+	if (section == WLAN_STATIC_SCAN_BUF1) {
+		return wlan_static_scan_buf1;
+	}
+
+	if (section == WLAN_STATIC_DHD_INFO_BUF) {
+		if (size > WLAN_DHD_INFO_BUF_SIZE) {
+			pr_err("request DHD_INFO size(%lu) is bigger than"
+				" static size(%d).\n", size,
+				WLAN_DHD_INFO_BUF_SIZE);
+			return NULL;
+		}
+		return wlan_static_dhd_info_buf;
+	}
+#ifdef CONFIG_BCMDHD_PCIE
+	if (section == WLAN_STATIC_DHD_IF_FLOW_LKUP)  {
+		if (size > WLAN_DHD_IF_FLOW_LKUP_SIZE) {
+			pr_err("request DHD_WLFC size(%lu) is bigger than"
+				" static size(%d).\n",
+				size, WLAN_DHD_WLFC_BUF_SIZE);
+			return NULL;
+		}
+		return wlan_static_if_flow_lkup;
+	}
+#else
+	if (section == WLAN_STATIC_DHD_WLFC_BUF)  {
+		if (size > WLAN_DHD_WLFC_BUF_SIZE) {
+			pr_err("request DHD_WLFC size(%lu) is bigger than"
+				" static size(%d).\n",
+				size, WLAN_DHD_WLFC_BUF_SIZE);
+			return NULL;
+		}
+		return wlan_static_dhd_wlfc_buf;
+	}
+
+	if (section == WLAN_STATIC_DHD_WLFC_HANGER) {
+		if (size > WLAN_DHD_WLFC_HANGER_SIZE) {
+			pr_err("request DHD_WLFC_HANGER size(%lu) is bigger than"
+				" static size(%d).\n",
+				size, WLAN_DHD_WLFC_HANGER_SIZE);
+			return NULL;
+		}
+		return wlan_static_dhd_wlfc_hanger;
+	}
+#endif /* CONFIG_BCMDHD_PCIE */
+#ifdef CONFIG_BCMDHD_PREALLOC_MEMDUMP
+	if (section == WLAN_STATIC_DHD_MEMDUMP_RAM) {
+		if (size > WLAN_DHD_MEMDUMP_SIZE) {
+			pr_err("request DHD_MEMDUMP_RAM size(%lu) is bigger"
+				" than static size(%d).\n",
+				size, WLAN_DHD_MEMDUMP_SIZE);
+			return NULL;
+		}
+		return wlan_static_dhd_memdump_ram;
+	}
+#endif /* CONFIG_BCMDHD_PREALLOC_MEMDUMP */
+	if (section == WLAN_STATIC_DHD_LOG_DUMP_BUF) {
+		if (size > DHD_LOG_DUMP_BUF_SIZE) {
+			pr_err("request DHD_LOG_DUMP_BUF size(%lu) is bigger then"
+				" static size(%d).\n",
+				size, DHD_LOG_DUMP_BUF_SIZE);
+			return NULL;
+		}
+		return wlan_static_dhd_log_dump_buf;
+	}
+
+	if (section == WLAN_STATIC_DHD_LOG_DUMP_BUF_EX) {
+		if (size > DHD_LOG_DUMP_BUF_EX_SIZE) {
+			pr_err("request DHD_LOG_DUMP_BUF_EX size(%lu) is bigger then"
+				" static size(%d).\n",
+				size, DHD_LOG_DUMP_BUF_EX_SIZE);
+			return NULL;
+		}
+		return wlan_static_dhd_log_dump_buf_ex;
+	}
+
+	if ((section < 0) || (section >= PREALLOC_WLAN_SEC_NUM)) {
+		return NULL;
+	}
+
+	if (wlan_mem_array[section].size < size) {
+		return NULL;
+	}
+
+	return wlan_mem_array[section].mem_ptr;
+}
+EXPORT_SYMBOL(dhd_wlan_mem_prealloc);
+
+#ifdef DHD_DUMP_BUF_KVMALLOC
+#define DUMP_BUF_MALLOC(size)   kvmalloc(size, GFP_KERNEL)
+#define DUMP_BUF_MFREE(addr)    kvfree(addr)
+#else
+#define DUMP_BUF_MALLOC(size)   kmalloc(size, GFP_KERNEL)
+#define DUMP_BUF_MFREE(addr)    kfree(addr)
+#endif /* DHD_DUMP_BUF_KVMALLOC */
+int
+dhd_init_wlan_mem(void)
+{
+	int i;
+	int j;
+
+#if !defined(CONFIG_BCMDHD_PCIE)
+	for (i = 0; i < DHD_SKB_1PAGE_BUF_NUM; i++) {
+		wlan_static_skb[i] = __dev_alloc_skb(DHD_SKB_1PAGE_BUFSIZE, GFP_KERNEL);
+		if (!wlan_static_skb[i]) {
+			pr_err("Failed to alloc 1PAGE SKB BUF\n");
+			goto err_skb_alloc;
+		}
+	}
+#endif /* !CONFIG_BCMDHD_PCIE */
+
+	for (i = DHD_SKB_1PAGE_BUF_NUM; i < WLAN_SKB_1_2PAGE_BUF_NUM; i++) {
+		wlan_static_skb[i] = __dev_alloc_skb(DHD_SKB_2PAGE_BUFSIZE, GFP_KERNEL);
+		if (!wlan_static_skb[i]) {
+			pr_err("Failed to alloc 2PAGE SKB BUF\n");
+			goto err_skb_alloc;
+		}
+	}
+
+#if !defined(CONFIG_BCMDHD_PCIE)
+	wlan_static_skb[i] = __dev_alloc_skb(DHD_SKB_4PAGE_BUFSIZE, GFP_KERNEL);
+	if (!wlan_static_skb[i]) {
+		pr_err("Failed to alloc 4PAGE SKB BUF\n");
+		goto err_skb_alloc;
+	}
+#endif /* !CONFIG_BCMDHD_PCIE */
+
+	for (i = 0; i < PREALLOC_WLAN_SEC_NUM; i++) {
+		if (wlan_mem_array[i].size > 0) {
+#ifdef CONFIG_BCMDHD_PCIE
+			wlan_mem_array[i].mem_ptr =
+				kvmalloc(wlan_mem_array[i].size, GFP_KERNEL);
+#else
+			wlan_mem_array[i].mem_ptr =
+				kmalloc(wlan_mem_array[i].size, GFP_KERNEL);
+#endif /* CONFIG_BCMDHD_PCIE */
+
+			if (!wlan_mem_array[i].mem_ptr) {
+				pr_err("Failed to mem_alloc for WLAN\n");
+				goto err_mem_alloc;
+			}
+		}
+	}
+
+#ifdef CONFIG_BCMDHD_PCIE
+	wlan_static_scan_buf0 = kvmalloc(WLAN_SCAN_BUF_SIZE, GFP_KERNEL);
+#else
+	wlan_static_scan_buf0 = kmalloc(WLAN_SCAN_BUF_SIZE, GFP_KERNEL);
+#endif /* CONFIG_BCMDHD_PCIE */
+	if (!wlan_static_scan_buf0) {
+		pr_err("Failed to alloc wlan_static_scan_buf0\n");
+		goto err_mem_alloc;
+	}
+
+#ifdef CONFIG_BCMDHD_PCIE
+	wlan_static_scan_buf1 = kvmalloc(WLAN_SCAN_BUF_SIZE, GFP_KERNEL);
+#else
+	wlan_static_scan_buf1 = kmalloc(WLAN_SCAN_BUF_SIZE, GFP_KERNEL);
+#endif /* CONFIG_BCMDHD_PCIE */
+	if (!wlan_static_scan_buf1) {
+		pr_err("Failed to alloc wlan_static_scan_buf1\n");
+		goto err_mem_alloc;
+	}
+
+	wlan_static_dhd_log_dump_buf = DUMP_BUF_MALLOC(DHD_LOG_DUMP_BUF_SIZE);
+	if (!wlan_static_dhd_log_dump_buf) {
+		pr_err("Failed to alloc wlan_static_dhd_log_dump_buf\n");
+		goto err_mem_alloc;
+	}
+
+	wlan_static_dhd_log_dump_buf_ex = DUMP_BUF_MALLOC(DHD_LOG_DUMP_BUF_EX_SIZE);
+	if (!wlan_static_dhd_log_dump_buf_ex) {
+		pr_err("Failed to alloc wlan_static_dhd_log_dump_buf_ex\n");
+		goto err_mem_alloc;
+	}
+
+#ifdef CONFIG_BCMDHD_PCIE
+	wlan_static_dhd_info_buf = kvmalloc(WLAN_DHD_INFO_BUF_SIZE, GFP_KERNEL);
+#else
+	wlan_static_dhd_info_buf = kmalloc(WLAN_DHD_INFO_BUF_SIZE, GFP_KERNEL);
+#endif /* CONFIG_BCMDHD_PCIE */
+	if (!wlan_static_dhd_info_buf) {
+		pr_err("Failed to alloc wlan_static_dhd_info_buf\n");
+		goto err_mem_alloc;
+	}
+
+#ifdef CONFIG_BCMDHD_PCIE
+	wlan_static_if_flow_lkup = kvmalloc(WLAN_DHD_IF_FLOW_LKUP_SIZE,
+		GFP_KERNEL);
+	if (!wlan_static_if_flow_lkup) {
+		pr_err("Failed to alloc wlan_static_if_flow_lkup\n");
+		goto err_mem_alloc;
+	}
+#else
+	wlan_static_dhd_wlfc_buf = kmalloc(WLAN_DHD_WLFC_BUF_SIZE,
+		GFP_KERNEL);
+	if (!wlan_static_dhd_wlfc_buf) {
+		pr_err("Failed to alloc wlan_static_dhd_wlfc_buf\n");
+		goto err_mem_alloc;
+	}
+
+	wlan_static_dhd_wlfc_hanger = kmalloc(WLAN_DHD_WLFC_HANGER_SIZE,
+		GFP_KERNEL);
+	if (!wlan_static_dhd_wlfc_hanger) {
+		pr_err("Failed to alloc wlan_static_dhd_wlfc_hanger\n");
+		goto err_mem_alloc;
+	}
+#endif /* CONFIG_BCMDHD_PCIE */
+
+#ifdef CONFIG_BCMDHD_PREALLOC_MEMDUMP
+	wlan_static_dhd_memdump_ram = DUMP_BUF_MALLOC(WLAN_DHD_MEMDUMP_SIZE);
+	if (!wlan_static_dhd_memdump_ram) {
+		pr_err("Failed to alloc wlan_static_dhd_memdump_ram\n");
+		goto err_mem_alloc;
+	}
+#endif /* CONFIG_BCMDHD_PREALLOC_MEMDUMP */
+
+	pr_err("%s: WIFI MEM Allocated\n", __FUNCTION__);
+	return 0;
+
+err_mem_alloc:
+	dhd_exit_wlan_mem();
+	return -ENOMEM;
+
+err_skb_alloc:
+	/*
+	 * When all the skb alloc buf couldn't alloced, free these buf with alloced size
+	 * dhd_exit_wlan_mem will free with total size (don't know alloced size)
+	 */
+	pr_err("Failed to skb_alloc for WLAN\n");
+	for (j = 0; j < i; j++) {
+		dev_kfree_skb(wlan_static_skb[j]);
+	}
+	return -ENOMEM;
+}
+
+EXPORT_SYMBOL(dhd_init_wlan_mem);
+
+void
+dhd_exit_wlan_mem(void)
+{
+	int i = 0;
+
+#ifdef CONFIG_BCMDHD_PREALLOC_MEMDUMP
+	if (wlan_static_dhd_memdump_ram) {
+		DUMP_BUF_MFREE(wlan_static_dhd_memdump_ram);
+	}
+#endif /* CONFIG_BCMDHD_PREALLOC_MEMDUMP */
+
+#ifdef CONFIG_BCMDHD_PCIE
+	if (wlan_static_if_flow_lkup) {
+		kvfree(wlan_static_if_flow_lkup);
+	}
+#else
+	if (wlan_static_dhd_wlfc_buf) {
+		kfree(wlan_static_dhd_wlfc_buf);
+	}
+
+	if (wlan_static_dhd_wlfc_hanger) {
+		kfree(wlan_static_dhd_wlfc_hanger);
+	}
+#endif /* CONFIG_BCMDHD_PCIE */
+	if (wlan_static_dhd_info_buf) {
+#ifdef CONFIG_BCMDHD_PCIE
+		kvfree(wlan_static_dhd_info_buf);
+#else
+		kfree(wlan_static_dhd_info_buf);
+#endif /* CONFIG_BCMDHD_PCIE */
+	}
+
+	if (wlan_static_dhd_log_dump_buf) {
+		DUMP_BUF_MFREE(wlan_static_dhd_log_dump_buf);
+	}
+
+	if (wlan_static_dhd_log_dump_buf_ex) {
+		DUMP_BUF_MFREE(wlan_static_dhd_log_dump_buf_ex);
+	}
+
+	if (wlan_static_scan_buf1) {
+#ifdef CONFIG_BCMDHD_PCIE
+		kvfree(wlan_static_scan_buf1);
+#else
+		kfree(wlan_static_scan_buf1);
+#endif /* CONFIG_BCMDHD_PCIE */
+	}
+
+	if (wlan_static_scan_buf0) {
+#ifdef CONFIG_BCMDHD_PCIE
+		kvfree(wlan_static_scan_buf0);
+#else
+		kfree(wlan_static_scan_buf0);
+#endif /* CONFIG_BCMDHD_PCIE */
+	}
+
+	for (i = 0; i < PREALLOC_WLAN_SEC_NUM; i++) {
+		if (wlan_mem_array[i].mem_ptr) {
+#ifdef CONFIG_BCMDHD_PCIE
+			kvfree(wlan_mem_array[i].mem_ptr);
+#else
+			kfree(wlan_mem_array[i].mem_ptr);
+#endif /* CONFIG_BCMDHD_PCIE */
+		}
+	}
+
+	for (i = 0; i < WLAN_SKB_BUF_NUM; i++) {
+		dev_kfree_skb(wlan_static_skb[i]);
+	}
+
+	return;
+}
+EXPORT_SYMBOL(dhd_exit_wlan_mem);
+#endif /* CONFIG_BROADCOM_WIFI_RESERVED_MEM */
diff --git a/drivers/net/wireless/ap12275/dhd_dbg.h b/drivers/net/wireless/ap12275/dhd_dbg.h
new file mode 100644
index 000000000000..cac6787a0066
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_dbg.h
@@ -0,0 +1,573 @@
+/*
+ * Debug/trace/assert driver definitions for Dongle Host Driver.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef _dhd_dbg_
+#define _dhd_dbg_
+
+#if defined(NDIS)
+#include "wl_nddbg.h"
+#endif /* defined(NDIS) */
+#ifdef DHD_LOG_DUMP
+#include <dhd_log_dump.h>
+#endif
+
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+/* Only for writing to ring */
+#define DHD_INFO_RING(args)	DHD_ERROR(args)
+/* FW_VERBOSE RING */
+#define DHD_LOG_DUMP_FWLOG	DHD_LOG_DUMP_WRITE_EX
+#define DHD_LOG_DUMP_FWLOG_TS	DHD_LOG_DUMP_WRITE_EX_TS
+#else
+#define DHD_INFO_RING(args)
+/* DLD_BUF_TYPE_GENERAL */
+#define DHD_LOG_DUMP_FWLOG	DHD_LOG_DUMP_WRITE
+#define DHD_LOG_DUMP_FWLOG_TS	DHD_LOG_DUMP_WRITE_TS
+#endif
+
+#ifdef CUSTOM_PREFIX
+#define DBG_PRINT_PREFIX "[%s]"CUSTOM_PREFIX, OSL_GET_RTCTIME()
+#elif defined(CUSTOM_PREFIX_NORTCTIME)
+#define DBG_PRINT_PREFIX CUSTOM_PREFIX_NORTCTIME
+
+#define DBG_PRINT_SYSTEM_TIME pr_cont(DBG_PRINT_PREFIX)
+#define DHD_CONS_ONLY(args)	\
+do {	\
+	DBG_PRINT_SYSTEM_TIME;	\
+	pr_cont args;		\
+} while (0)
+#else
+#define DBG_PRINT_SYSTEM_TIME
+#define DHD_CONS_ONLY(args) do { printf args;} while (0)
+#endif /* CUSTOM_PREFIX */
+
+#if defined(BCMDBG) || defined(DHD_DEBUG)
+
+#if defined(NDIS)
+#define DHD_ERROR(args)		do {if (dhd_msg_level & DHD_ERROR_VAL) \
+					{printf args;  DHD_NDDBG_OUTPUT args;}} while (0)
+#define DHD_TRACE(args)		do {if (dhd_msg_level & DHD_TRACE_VAL) \
+					{printf args; DHD_NDDBG_OUTPUT args;}} while (0)
+#define DHD_INFO(args)		do {if (dhd_msg_level & DHD_INFO_VAL) \
+					{printf args; DHD_NDDBG_OUTPUT args;}} while (0)
+#define DHD_ERROR_ROAM(args)	DHD_ERROR(args)
+#else
+/* NON-NDIS cases */
+#ifdef DHD_LOG_DUMP
+/* !defined(DHD_EFI) and defined(DHD_LOG_DUMP) */
+#define DHD_ERROR(args)	\
+do {	\
+	if (dhd_msg_level & DHD_ERROR_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_ERROR_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+#define DHD_INFO(args)	\
+do {	\
+	if (dhd_msg_level & DHD_INFO_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_INFO_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+#else
+#define DHD_INFO(args)		do {if (dhd_msg_level & DHD_INFO_VAL) printf args;} while (0)
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+#else /* DHD_LOG_DUMP */
+/* !defined(DHD_LOG_DUMP cases) */
+#define DHD_ERROR(args)		do {if (dhd_msg_level & DHD_ERROR_VAL) printf args;} while (0)
+#define DHD_INFO(args)		do {if (dhd_msg_level & DHD_INFO_VAL) printf args;} while (0)
+#define DHD_ERROR_ROAM(args)	DHD_ERROR(args)
+#endif /* DHD_LOG_DUMP */
+
+#define DHD_TRACE(args)		do {if (dhd_msg_level & DHD_TRACE_VAL) printf args;} while (0)
+#endif /* defined(NDIS) */
+
+#ifdef DHD_LOG_DUMP
+/* LOG_DUMP defines common to EFI and NON-EFI */
+/* NON-EFI builds with LOG DUMP enabled */
+#define DHD_ERROR_MEM(args) \
+do {	\
+	if (dhd_msg_level & DHD_ERROR_VAL) {	\
+		if (dhd_msg_level & DHD_ERROR_MEM_VAL) {	\
+			printf args;		\
+		}	\
+	}	\
+	if (dhd_log_level & DHD_ERROR_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;		\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+#define DHD_IOVAR_MEM(args) \
+do {	\
+	if (dhd_msg_level & DHD_ERROR_VAL) {	\
+		if (dhd_msg_level & DHD_IOVAR_MEM_VAL) {	\
+			printf args;		\
+		}	\
+	}	\
+	if (dhd_log_level & DHD_ERROR_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;		\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+#define DHD_LOG_MEM(args) \
+do {	\
+	if (dhd_log_level & DHD_ERROR_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;		\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_EVENT(args) \
+do {	\
+	if (dhd_msg_level & DHD_EVENT_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_EVENT_VAL) {	\
+		DHD_LOG_DUMP_WRITE_PRSRV_TS;	\
+		DHD_LOG_DUMP_WRITE_PRSRV args;	\
+	}	\
+} while (0)
+#define DHD_PRSRV_MEM(args) \
+do {	\
+	if (dhd_msg_level & DHD_EVENT_VAL) {	\
+		if (dhd_msg_level & DHD_PRSRV_MEM_VAL) {	\
+			printf args;		\
+		}	\
+	}	\
+	if (dhd_log_level & DHD_EVENT_VAL) {	\
+		DHD_LOG_DUMP_WRITE_PRSRV_TS;		\
+		DHD_LOG_DUMP_WRITE_PRSRV args;	\
+	}	\
+} while (0)
+/* Re-using 'DHD_MSGTRACE_VAL' for controlling printing of ecounter binary event
+* logs to console and debug dump -- need to cleanup in the future to use separate
+* 'DHD_ECNTR_VAL' bitmap flag. 'DHD_MSGTRACE_VAL' will be defined only
+* for non-android builds.
+*/
+#define DHD_ECNTR_LOG(args) \
+do {	\
+	if (dhd_msg_level & DHD_EVENT_VAL) {	\
+		if (dhd_msg_level & DHD_MSGTRACE_VAL) { \
+			printf args;		\
+		}	\
+	}	\
+	if (dhd_log_level & DHD_EVENT_VAL) {	\
+		if (dhd_log_level & DHD_MSGTRACE_VAL) { \
+			DHD_LOG_DUMP_WRITE_TS;		\
+			DHD_LOG_DUMP_WRITE args;	\
+		}	\
+	}	\
+} while (0)
+#define DHD_ERROR_EX(args)					\
+do {										\
+	if (dhd_msg_level & DHD_ERROR_VAL) {    \
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_ERROR_VAL) {    \
+		DHD_LOG_DUMP_WRITE_EX_TS;	\
+		DHD_LOG_DUMP_WRITE_EX args;	\
+	}	\
+} while (0)
+#define DHD_MSGTRACE_LOG(args)	\
+do {	\
+	if (dhd_msg_level & DHD_MSGTRACE_VAL) {	\
+		printf args;		\
+	}	\
+	DHD_LOG_DUMP_WRITE_TS;		\
+	DHD_LOG_DUMP_WRITE args;	\
+} while (0)
+
+#define DHD_ERROR_ROAM(args)	\
+do {	\
+	if (dhd_msg_level & DHD_ERROR_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_ERROR_VAL) {	\
+		DHD_LOG_DUMP_WRITE_ROAM_TS;	\
+		DHD_LOG_DUMP_WRITE_ROAM args;	\
+	}	\
+} while (0)
+
+#define DHD_PKT_MON(args)	\
+do {	\
+	if (dhd_msg_level & DHD_PKT_MON_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_PKT_MON_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_CTL(args)	\
+do {	\
+	if (dhd_msg_level & DHD_CTL_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_CTL_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_TIMER(args)	\
+do {	\
+	if (dhd_msg_level & DHD_TIMER_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_TIMER_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_INTR(args)	\
+do {	\
+	if (dhd_msg_level & DHD_INTR_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_INTR_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_ISCAN(args)	\
+do {	\
+	if (dhd_msg_level & DHD_ISCAN_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_ISCAN_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_ARPOE(args)	\
+do {	\
+	if (dhd_msg_level & DHD_ARPOE_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_ARPOE_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_REORDER(args)	\
+do {	\
+	if (dhd_msg_level & DHD_REORDER_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_REORDER_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_PNO(args)	\
+do {	\
+	if (dhd_msg_level & DHD_PNO_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_PNO_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_RTT(args)	\
+do {	\
+	if (dhd_msg_level & DHD_RTT_VAL) {	\
+		printf args;		\
+	}	\
+	if (dhd_log_level & DHD_RTT_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+
+#define DHD_RPM(args)	\
+do { \
+	if (dhd_msg_level & DHD_RPM_VAL) {	\
+		printf args;	\
+	}	\
+	if (dhd_log_level & DHD_RPM_VAL) {	\
+		DHD_LOG_DUMP_WRITE_TS;	\
+		DHD_LOG_DUMP_WRITE args;	\
+	}	\
+} while (0)
+#else /* DHD_LOG_DUMP */
+/* !DHD_LOG_DUMP */
+#define DHD_MSGTRACE_LOG(args)  do {if (dhd_msg_level & DHD_MSGTRACE_VAL) printf args;} while (0)
+#define DHD_ERROR_MEM(args)	DHD_ERROR(args)
+#define DHD_IOVAR_MEM(args)	DHD_ERROR(args)
+#define DHD_LOG_MEM(args)	DHD_ERROR(args)
+#define DHD_EVENT(args)		do {if (dhd_msg_level & DHD_EVENT_VAL) printf args;} while (0)
+#define DHD_ECNTR_LOG(args)	DHD_EVENT(args)
+#define DHD_PRSRV_MEM(args)	DHD_EVENT(args)
+#define DHD_ERROR_EX(args)	DHD_ERROR(args)
+#define DHD_ERROR_ROAM(args)	DHD_ERROR(args)
+#define DHD_PKT_MON(args)	DHD_ERROR(args)
+#endif /* DHD_LOG_DUMP */
+
+#if !defined(DHD_LOG_DUMP)
+#define DHD_CTL(args)		do {if (dhd_msg_level & DHD_CTL_VAL) printf args;} while (0)
+#define DHD_TIMER(args)		do {if (dhd_msg_level & DHD_TIMER_VAL) printf args;} while (0)
+#define DHD_INTR(args)		do {if (dhd_msg_level & DHD_INTR_VAL) printf args;} while (0)
+#define DHD_ISCAN(args)		do {if (dhd_msg_level & DHD_ISCAN_VAL) printf args;} while (0)
+#define DHD_ARPOE(args)		do {if (dhd_msg_level & DHD_ARPOE_VAL) printf args;} while (0)
+#define DHD_REORDER(args)	do {if (dhd_msg_level & DHD_REORDER_VAL) printf args;} while (0)
+#define DHD_PNO(args)		do {if (dhd_msg_level & DHD_PNO_VAL) printf args;} while (0)
+#define DHD_RTT(args)		do {if (dhd_msg_level & DHD_RTT_VAL) printf args;} while (0)
+#define DHD_RPM(args)		do {if (dhd_msg_level & DHD_RPM_VAL) printf args;} while (0)
+#endif
+
+#define DHD_DATA(args)		do {if (dhd_msg_level & DHD_DATA_VAL) printf args;} while (0)
+#define DHD_HDRS(args)		do {if (dhd_msg_level & DHD_HDRS_VAL) printf args;} while (0)
+#define DHD_BYTES(args)		do {if (dhd_msg_level & DHD_BYTES_VAL) printf args;} while (0)
+#define DHD_GLOM(args)		do {if (dhd_msg_level & DHD_GLOM_VAL) printf args;} while (0)
+#define DHD_BTA(args)		do {if (dhd_msg_level & DHD_BTA_VAL) printf args;} while (0)
+
+#if defined(DHD_LOG_DUMP)
+#if defined(DHD_LOG_PRINT_RATE_LIMIT)
+
+#define DHD_FWLOG(args)	\
+do { \
+	if (dhd_msg_level & DHD_FWLOG_VAL) { \
+		if (control_logtrace && !log_print_threshold) \
+			printf args; \
+	} \
+	if (dhd_log_level & DHD_FWLOG_VAL) { \
+		DHD_LOG_DUMP_FWLOG_TS;	\
+		DHD_LOG_DUMP_FWLOG args; \
+	} \
+} while (0)
+
+#else
+#define DHD_FWLOG(args)	\
+	do { \
+		if (dhd_msg_level & DHD_FWLOG_VAL) { \
+			if (control_logtrace) \
+				printf args; \
+			DHD_LOG_DUMP_WRITE args; \
+		} \
+	} while (0)
+#endif
+#else /* DHD_LOG_DUMP */
+#if defined(NDIS) && (NDISVER >= 0x0630)
+#define DHD_FWLOG(args)		do {if (dhd_msg_level & DHD_FWLOG_VAL) \
+					{printf args;  DHD_NDDBG_OUTPUT args;}} while (0)
+#else
+#define DHD_FWLOG(args)		do {if (dhd_msg_level & DHD_FWLOG_VAL) printf args;} while (0)
+#endif /* defined(NDIS) && (NDISVER >= 0x0630) */
+#endif /* DHD_LOG_DUMP */
+
+#define DHD_DBGIF(args)		do {if (dhd_msg_level & DHD_DBGIF_VAL) printf args;} while (0)
+#define DHD_TXFLOWCTL(args)     DHD_RPM(args)
+
+#define DHD_TRACE_HW4	DHD_TRACE
+#define DHD_INFO_HW4	DHD_INFO
+#define DHD_ERROR_NO_HW4	DHD_ERROR
+
+#define DHD_ERROR_ON()		(dhd_msg_level & DHD_ERROR_VAL)
+#define DHD_TRACE_ON()		(dhd_msg_level & DHD_TRACE_VAL)
+#define DHD_INFO_ON()		(dhd_msg_level & DHD_INFO_VAL)
+#define DHD_DATA_ON()		(dhd_msg_level & DHD_DATA_VAL)
+#define DHD_CTL_ON()		(dhd_msg_level & DHD_CTL_VAL)
+#define DHD_TIMER_ON()		(dhd_msg_level & DHD_TIMER_VAL)
+#define DHD_HDRS_ON()		(dhd_msg_level & DHD_HDRS_VAL)
+#define DHD_BYTES_ON()		(dhd_msg_level & DHD_BYTES_VAL)
+#define DHD_INTR_ON()		(dhd_msg_level & DHD_INTR_VAL)
+#define DHD_GLOM_ON()		(dhd_msg_level & DHD_GLOM_VAL)
+#define DHD_EVENT_ON()		(dhd_msg_level & DHD_EVENT_VAL)
+#define DHD_BTA_ON()		(dhd_msg_level & DHD_BTA_VAL)
+#define DHD_ISCAN_ON()		(dhd_msg_level & DHD_ISCAN_VAL)
+#define DHD_ARPOE_ON()		(dhd_msg_level & DHD_ARPOE_VAL)
+#define DHD_REORDER_ON()	(dhd_msg_level & DHD_REORDER_VAL)
+#define DHD_NOCHECKDIED_ON()	(dhd_msg_level & DHD_NOCHECKDIED_VAL)
+#define DHD_PNO_ON()		(dhd_msg_level & DHD_PNO_VAL)
+#define DHD_RTT_ON()		(dhd_msg_level & DHD_RTT_VAL)
+#define DHD_MSGTRACE_ON()	(dhd_msg_level & DHD_MSGTRACE_VAL)
+#define DHD_FWLOG_ON()		(dhd_msg_level & DHD_FWLOG_VAL)
+#define DHD_DBGIF_ON()		(dhd_msg_level & DHD_DBGIF_VAL)
+#define DHD_PKT_MON_ON()	(dhd_msg_level & DHD_PKT_MON_VAL)
+#define DHD_PKT_MON_DUMP_ON()	(dhd_msg_level & DHD_PKT_MON_DUMP_VAL)
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#define DHD_RPM_ON()		(dhd_msg_level & DHD_RPM_VAL)
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#else /* defined(BCMDBG) || defined(DHD_DEBUG) */
+
+#if defined(NDIS)
+#define DHD_ERROR(args)		do {if (dhd_msg_level & DHD_ERROR_VAL) \
+					{printf args;  DHD_NDDBG_OUTPUT args;}} while (0)
+#define DHD_TRACE(args)		do {if (dhd_msg_level & DHD_TRACE_VAL) \
+					{DHD_NDDBG_OUTPUT args;}} while (0)
+#define DHD_INFO(args)		do {if (dhd_msg_level & DHD_INFO_VAL) \
+					{DHD_NDDBG_OUTPUT args;}} while (0)
+#else /* DHD_EFI && DHD_LOG_DUMP */
+
+#define DHD_ERROR(args)		do {if (dhd_msg_level & DHD_ERROR_VAL) \
+								printf args;} while (0)
+#define DHD_TRACE(args)
+#define DHD_INFO(args)
+#define DHD_ERROR_ROAM(args)	DHD_ERROR(args)
+#endif /* defined(NDIS) */
+
+#define DHD_DATA(args)
+#define DHD_CTL(args)
+#define DHD_TIMER(args)
+#define DHD_HDRS(args)
+#define DHD_BYTES(args)
+#define DHD_INTR(args)
+#define DHD_GLOM(args)
+
+#define DHD_EVENT(args)
+#define DHD_ECNTR_LOG(args)	DHD_EVENT(args)
+
+#define DHD_PRSRV_MEM(args)	DHD_EVENT(args)
+
+#define DHD_BTA(args)
+#define DHD_ISCAN(args)
+#define DHD_ARPOE(args)
+#define DHD_REORDER(args)
+#define DHD_PNO(args)
+#define DHD_RTT(args)
+#define DHD_PKT_MON(args)
+
+#define DHD_MSGTRACE_LOG(args)
+#define DHD_FWLOG(args)
+
+#define DHD_DBGIF(args)
+
+#define DHD_ERROR_MEM(args)	DHD_ERROR(args)
+#define DHD_IOVAR_MEM(args)	DHD_ERROR(args)
+#define DHD_LOG_MEM(args)	DHD_ERROR(args)
+#define DHD_ERROR_EX(args)	DHD_ERROR(args)
+#define DHD_ERROR_ROAM(args)    DHD_ERROR(args)
+#define DHD_RPM(args)		DHD_ERROR(args)
+#define DHD_TXFLOWCTL(args)     DHD_ERROR(args)
+#define DHD_TRACE_HW4	DHD_TRACE
+#define DHD_INFO_HW4	DHD_INFO
+#define DHD_ERROR_NO_HW4	DHD_ERROR
+
+#define DHD_ERROR_ON()		0
+#define DHD_TRACE_ON()		0
+#define DHD_INFO_ON()		0
+#define DHD_DATA_ON()		0
+#define DHD_CTL_ON()		0
+#define DHD_TIMER_ON()		0
+#define DHD_HDRS_ON()		0
+#define DHD_BYTES_ON()		0
+#define DHD_INTR_ON()		0
+#define DHD_GLOM_ON()		0
+#define DHD_EVENT_ON()		0
+#define DHD_BTA_ON()		0
+#define DHD_ISCAN_ON()		0
+#define DHD_ARPOE_ON()		0
+#define DHD_REORDER_ON()	0
+#define DHD_NOCHECKDIED_ON()	0
+#define DHD_PNO_ON()		0
+#define DHD_RTT_ON()		0
+#define DHD_PKT_MON_ON()	0
+#define DHD_PKT_MON_DUMP_ON()	0
+#define DHD_MSGTRACE_ON()	0
+#define DHD_FWLOG_ON()		0
+#define DHD_DBGIF_ON()		0
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#define DHD_RPM_ON()		0
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+#endif /* defined(BCMDBG) || defined(DHD_DEBUG) */
+
+#define PRINT_RATE_LIMIT_PERIOD 5000000u /* 5s in units of us */
+#define DHD_ERROR_RLMT(args) \
+do {	\
+	if (dhd_msg_level & DHD_ERROR_VAL) {	\
+		static uint64 __err_ts = 0; \
+		static uint32 __err_cnt = 0; \
+		uint64 __cur_ts = 0; \
+		__cur_ts = OSL_SYSUPTIME_US(); \
+		if (__err_ts == 0 || (__cur_ts > __err_ts && \
+		(__cur_ts - __err_ts > PRINT_RATE_LIMIT_PERIOD))) { \
+			__err_ts = __cur_ts; \
+			DHD_ERROR(args);	\
+			DHD_ERROR(("[Repeats %u times]\n", __err_cnt)); \
+			__err_cnt = 0; \
+		} else { \
+			++__err_cnt; \
+		} \
+	}	\
+} while (0)
+
+/* even in non-BCMDBG builds, logging of dongle iovars should be available */
+#define DHD_DNGL_IOVAR_SET(args) \
+	do {if (dhd_msg_level & DHD_DNGL_IOVAR_SET_VAL) printf args;} while (0)
+
+#ifdef BCMPERFSTATS
+#define DHD_LOG(args)		do {if (dhd_msg_level & DHD_LOG_VAL) bcmlog args;} while (0)
+#else
+#define DHD_LOG(args)
+#endif
+
+#define DHD_BLOG(cp, size)
+
+#define DHD_NONE(args)
+extern int dhd_msg_level;
+extern int dhd_log_level;
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+extern int log_print_threshold;
+#endif /* DHD_LOG_PRINT_RATE_LIMIT */
+
+#ifdef DHD_IOVAR_LOG_FILTER_DUMP
+#define DHD_IOVAR_LOG_CHECK(dhd_pub, ioc_cmd, ioc_msg) \
+	dhd_iovar_log_dump_check(dhd_pub, ioc_cmd, ioc_msg)
+#else
+#define DHD_IOVAR_LOG_CHECK(dhd_pub, ioc_cmd, ioc_msg) TRUE
+#endif /* DHD_IOVAR_LOG_FILTER_DUMP */
+
+#define DHD_IOVAR_LOG(dhd_pub, ioc_cmd, ioc_msg, fmt) \
+	do { \
+		if (DHD_IOVAR_LOG_CHECK(dhd_pub, ioc_cmd, ioc_msg)) { \
+			DHD_IOVAR_MEM(fmt); \
+		} else { \
+			DHD_TRACE(fmt); \
+		} \
+	} while (0)
+
+/* DHD_PRINT - informational non-error messages which need to be always printed */
+#define DHD_PRINT	DHD_ERROR
+
+/* Defines msg bits */
+#include <dhdioctl.h>
+
+#endif /* _dhd_dbg_ */
diff --git a/drivers/net/wireless/ap12275/dhd_dbg_ring.c b/drivers/net/wireless/ap12275/dhd_dbg_ring.c
new file mode 100644
index 000000000000..b1c3d85c89fe
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_dbg_ring.c
@@ -0,0 +1,561 @@
+/*
+ * DHD debug ring API and structures - implementation
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <typedefs.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_dbg_ring.h>
+#include <dhd_debug.h>
+
+dhd_dbg_ring_t *
+dhd_dbg_ring_alloc_init(dhd_pub_t *dhd, uint16 ring_id,
+	char *ring_name, uint32 ring_sz, void *allocd_buf,
+	bool pull_inactive)
+{
+	dhd_dbg_ring_t *ring = NULL;
+	int ret = 0;
+	unsigned long flags = 0;
+
+	ring = MALLOCZ(dhd->osh, sizeof(dhd_dbg_ring_t));
+	if (!ring)
+		goto fail;
+
+	ret = dhd_dbg_ring_init(dhd, ring, ring_id,
+			(uint8 *)ring_name, ring_sz,
+			allocd_buf, pull_inactive);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: unable to init ring %s!\n",
+				__FUNCTION__, ring_name));
+		goto fail;
+	}
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->state = RING_ACTIVE;
+	ring->threshold = 0;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return ring;
+
+fail:
+	if (ring) {
+		dhd_dbg_ring_deinit(dhd, ring);
+		ring->ring_buf = NULL;
+		ring->ring_size = 0;
+		MFREE(dhd->osh, ring, sizeof(dhd_dbg_ring_t));
+	}
+	return NULL;
+}
+
+void
+dhd_dbg_ring_dealloc_deinit(void **ring_ptr, dhd_pub_t *dhd)
+{
+	dhd_dbg_ring_t *ring = NULL;
+	dhd_dbg_ring_t **dbgring = (dhd_dbg_ring_t **)ring_ptr;
+
+	if (!dbgring)
+		return;
+
+	ring = *dbgring;
+
+	if (ring) {
+		dhd_dbg_ring_deinit(dhd, ring);
+		ring->ring_buf = NULL;
+		ring->ring_size = 0;
+		MFREE(dhd->osh, ring, sizeof(dhd_dbg_ring_t));
+		*dbgring = NULL;
+	}
+}
+
+int
+dhd_dbg_ring_init(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring, uint16 id, uint8 *name,
+		uint32 ring_sz, void *allocd_buf, bool pull_inactive)
+{
+	void *buf;
+	unsigned long flags = 0;
+
+	if (allocd_buf == NULL) {
+		/* for DEBUG_DUMP and MEM_DUMP, buffer can be NULL
+		 * since act as delayed allocation or fake rings
+		 */
+		if (id != DEBUG_DUMP_RING1_ID && id != DEBUG_DUMP_RING2_ID &&
+				id != MEM_DUMP_RING_ID) {
+			return BCME_NOMEM;
+		}
+		buf = NULL;
+	} else {
+		buf = allocd_buf;
+	}
+
+	ring->lock = DHD_DBG_RING_LOCK_INIT(dhdp->osh);
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->id = id;
+	strlcpy((char *)ring->name, (char *)name, sizeof(ring->name));
+	ring->ring_size = ring_sz;
+	ring->wp = ring->rp = 0;
+	ring->ring_buf = buf;
+	ring->threshold = DBGRING_FLUSH_THRESHOLD(ring);
+	ring->state = RING_SUSPEND;
+	ring->rem_len = 0;
+	ring->sched_pull = TRUE;
+	ring->pull_inactive = pull_inactive;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return BCME_OK;
+}
+
+int
+dhd_dbg_ring_set_buf(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring, void *buf)
+{
+	unsigned long flags = 0;
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->ring_buf = buf;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return BCME_OK;
+}
+void
+dhd_dbg_ring_deinit(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring)
+{
+	unsigned long flags = 0;
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->id = 0;
+	ring->name[0] = 0;
+	ring->wp = ring->rp = 0;
+	memset(&ring->stat, 0, sizeof(ring->stat));
+	ring->threshold = 0;
+	ring->state = RING_STOP;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	DHD_DBG_RING_LOCK_DEINIT(dhdp->osh, ring->lock);
+}
+
+void
+dhd_dbg_ring_sched_pull(dhd_dbg_ring_t *ring, uint32 pending_len,
+		os_pullreq_t pull_fn, void *os_pvt, const int id)
+{
+	unsigned long flags = 0;
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	/* if the current pending size is bigger than threshold and
+	 * threshold is set
+	 */
+	if (ring->threshold > 0 &&
+	   (pending_len >= ring->threshold) && ring->sched_pull) {
+		/*
+		 * Update the state and release the lock before calling
+		 * the pull_fn. Do not transfer control to other layers
+		 * with locks held. If the call back again calls into
+		 * the same layer fro this context, can lead to deadlock.
+		 */
+		ring->sched_pull = FALSE;
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+		pull_fn(os_pvt, id);
+	} else {
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+	}
+}
+
+uint32
+dhd_dbg_ring_get_pending_len(dhd_dbg_ring_t *ring)
+{
+	uint32 pending_len = 0;
+	unsigned long flags = 0;
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	if (ring->stat.written_bytes > ring->stat.read_bytes) {
+		pending_len = ring->stat.written_bytes - ring->stat.read_bytes;
+	} else if (ring->stat.written_bytes < ring->stat.read_bytes) {
+		pending_len = PENDING_LEN_MAX - ring->stat.read_bytes + ring->stat.written_bytes;
+	} else {
+		pending_len = 0;
+	}
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return pending_len;
+}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+int
+dhd_dbg_ring_update(void *dbg_ring, uint32 w_len)
+{
+	unsigned long flags;
+	dhd_dbg_ring_t *ring = (dhd_dbg_ring_t *)dbg_ring;
+
+	if (ring->id != PACKET_LOG_RING_ID) {
+		return BCME_UNSUPPORTED;
+	}
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+
+	if (ring->state != RING_ACTIVE) {
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+		return BCME_OK;
+	}
+
+	/* update statistics */
+	ring->stat.written_records++;
+	ring->stat.written_bytes += w_len;
+	DHD_DBGIF(("%s : RING%d[%s] written_records %d, written_bytes %d, read_bytes=%d,"
+		" ring->threshold=%d, wp=%d, rp=%d\n", __FUNCTION__, ring->id, ring->name,
+		ring->stat.written_records, ring->stat.written_bytes, ring->stat.read_bytes,
+		ring->threshold, ring->wp, ring->rp));
+
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return BCME_OK;
+}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+int
+dhd_dbg_ring_push(dhd_dbg_ring_t *ring, dhd_dbg_ring_entry_t *hdr, void *data)
+{
+	unsigned long flags;
+	uint32 w_len;
+	uint32 avail_size;
+	dhd_dbg_ring_entry_t *w_entry, *r_entry;
+	int ret;
+
+	if (!ring || !hdr || !data) {
+		return BCME_BADARG;
+	}
+
+#if defined(__linux__)
+	/* Prevents the case of accessing the ring buffer in the HardIRQ context.
+	 * If an interrupt arise after holding ring lock, It could try the same lock.
+	 * This is to use the ring lock as spin_lock_bh instead of spin_lock_irqsave.
+	 */
+	if (in_irq()) {
+		return BCME_BUSY;
+	}
+#endif /* defined(__linux__) */
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+
+	if (ring->state != RING_ACTIVE) {
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+		return BCME_OK;
+	}
+
+	w_len = ENTRY_LENGTH(hdr);
+
+	DHD_DBGIF(("%s: RING%d[%s] hdr->len=%u, w_len=%u, wp=%d, rp=%d, ring_start=0x%p;"
+		" ring_size=%u\n",
+		__FUNCTION__, ring->id, ring->name, hdr->len, w_len, ring->wp, ring->rp,
+		ring->ring_buf, ring->ring_size));
+
+	if (w_len > ring->ring_size) {
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+		DHD_ERROR(("%s: RING%d[%s] w_len=%u, ring_size=%u,"
+			" write size exceeds ring size !\n",
+			__FUNCTION__, ring->id, ring->name, w_len, ring->ring_size));
+		return BCME_ERROR;
+	}
+	/* Claim the space */
+	do {
+		avail_size = DBG_RING_CHECK_WRITE_SPACE(ring->rp, ring->wp, ring->ring_size);
+		if (avail_size <= w_len) {
+			/* Prepare the space */
+			if (ring->rp <= ring->wp) {
+				ring->tail_padded = TRUE;
+				ring->rem_len = ring->ring_size - ring->wp;
+				DHD_DBGIF(("%s: RING%d[%s] Insuffient tail space,"
+					" rp=%d, wp=%d, rem_len=%d, ring_size=%d,"
+					" avail_size=%d, w_len=%d\n", __FUNCTION__,
+					ring->id, ring->name, ring->rp, ring->wp,
+					ring->rem_len, ring->ring_size, avail_size,
+					w_len));
+
+				/* 0 pad insufficient tail space */
+				memset((uint8 *)ring->ring_buf + ring->wp, 0, ring->rem_len);
+				/* If read pointer is still at the beginning, make some room */
+				if (ring->rp == 0) {
+					r_entry = (dhd_dbg_ring_entry_t *)((uint8 *)ring->ring_buf +
+						ring->rp);
+					ring->rp += ENTRY_LENGTH(r_entry);
+					ring->stat.read_bytes += ENTRY_LENGTH(r_entry);
+					DHD_DBGIF(("%s: rp at 0, move by one entry length"
+						" (%u bytes)\n",
+						__FUNCTION__, (uint32)ENTRY_LENGTH(r_entry)));
+				}
+				if (ring->rp == ring->wp) {
+					ring->rp = 0;
+				}
+				ring->wp = 0;
+				DHD_DBGIF(("%s: new rp=%u, wp=%u\n",
+					__FUNCTION__, ring->rp, ring->wp));
+			} else {
+				/* Not enough space for new entry, free some up */
+				r_entry = (dhd_dbg_ring_entry_t *)((uint8 *)ring->ring_buf +
+					ring->rp);
+				/* check bounds before incrementing read ptr */
+				if (ring->rp + ENTRY_LENGTH(r_entry) >= ring->ring_size) {
+					DHD_DBGIF(("%s: RING%d[%s] rp points out of boundary,"
+						"ring->wp=%u, ring->rp=%u, ring->ring_size=%d\n",
+						__FUNCTION__, ring->id, ring->name, ring->wp,
+						ring->rp, ring->ring_size));
+					ASSERT(0);
+					DHD_DBG_RING_UNLOCK(ring->lock, flags);
+					return BCME_BUFTOOSHORT;
+				}
+				ring->rp += ENTRY_LENGTH(r_entry);
+				/* skip padding if there is one */
+				if (ring->tail_padded &&
+					((ring->rp + ring->rem_len) == ring->ring_size)) {
+					DHD_DBGIF(("%s: RING%d[%s] Found padding,"
+						" avail_size=%d, w_len=%d, set rp = 0\n",
+						__FUNCTION__,
+						ring->id, ring->name, avail_size, w_len));
+					ring->rp = 0;
+					ring->tail_padded = FALSE;
+					ring->rem_len = 0;
+				}
+				ring->stat.read_bytes += ENTRY_LENGTH(r_entry);
+				DHD_DBGIF(("%s: RING%d[%s] read_bytes=%d, wp=%d, rp=%d\n",
+					__FUNCTION__, ring->id, ring->name, ring->stat.read_bytes,
+					ring->wp, ring->rp));
+			}
+		} else {
+			break;
+		}
+	} while (TRUE);
+
+	/* check before writing to the ring */
+	if (ring->wp + w_len >= ring->ring_size) {
+		DHD_DBGIF(("%s: RING%d[%s] wp pointed out of ring boundary, "
+			"wp=%d, ring_size=%d, w_len=%u\n", __FUNCTION__, ring->id,
+			ring->name, ring->wp, ring->ring_size, w_len));
+		ASSERT(0);
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+		return BCME_BUFTOOLONG;
+	}
+
+	w_entry = (dhd_dbg_ring_entry_t *)((uint8 *)ring->ring_buf + ring->wp);
+	/* header */
+	ret = memcpy_s(w_entry, avail_size, hdr, DBG_RING_ENTRY_SIZE);
+	if (ret) {
+		DHD_ERROR((" memcpy_s() error : %d, destsz: %d, n: %d\n",
+			ret, avail_size, (int)DBG_RING_ENTRY_SIZE));
+		return BCME_ERROR;
+	}
+	w_entry->len = hdr->len;
+	/* payload */
+	avail_size -= DBG_RING_ENTRY_SIZE;
+	ret = memcpy_s((char *)w_entry + DBG_RING_ENTRY_SIZE,
+		avail_size, data, w_entry->len);
+	if (ret) {
+		DHD_ERROR((" memcpy_s() error : %d, destsz: %d, n: %d\n",
+			ret, avail_size, w_entry->len));
+		return BCME_ERROR;
+	}
+	/* update write pointer */
+	ring->wp += w_len;
+
+	/* update statistics */
+	ring->stat.written_records++;
+	ring->stat.written_bytes += w_len;
+	DHD_DBGIF(("%s : RING%d[%s] written_records %d, written_bytes %d, read_bytes=%d,"
+		" ring->threshold=%d, wp=%d, rp=%d\n", __FUNCTION__, ring->id, ring->name,
+		ring->stat.written_records, ring->stat.written_bytes, ring->stat.read_bytes,
+		ring->threshold, ring->wp, ring->rp));
+
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+	return BCME_OK;
+}
+
+/*
+ * This function folds ring->lock, so callers of this function
+ * should not hold ring->lock.
+ */
+int
+dhd_dbg_ring_pull_single(dhd_dbg_ring_t *ring, void *data, uint32 buf_len, bool strip_header)
+{
+	dhd_dbg_ring_entry_t *r_entry = NULL;
+	uint32 rlen = 0;
+	char *buf = NULL;
+	unsigned long flags;
+
+	if (!ring || !data || buf_len <= 0) {
+		return 0;
+	}
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	/* pull from ring is allowed for inactive (suspended) ring
+	 * in case of ecounters only, this is because, for ecounters
+	 * when a trap occurs the ring is suspended and data is then
+	 * pulled to dump it to a file. For other rings if ring is
+	 * not in active state return without processing (as before)
+	 */
+	if (!ring->pull_inactive && (ring->state != RING_ACTIVE)) {
+		goto exit;
+	}
+
+	if (ring->rp == ring->wp) {
+		goto exit;
+	}
+
+	DHD_DBGIF(("%s: RING%d[%s] buf_len=%u, wp=%d, rp=%d, ring_start=0x%p; ring_size=%u\n",
+		__FUNCTION__, ring->id, ring->name, buf_len, ring->wp, ring->rp,
+		ring->ring_buf, ring->ring_size));
+
+	r_entry = (dhd_dbg_ring_entry_t *)((uint8 *)ring->ring_buf + ring->rp);
+
+	/* Boundary Check */
+	rlen = ENTRY_LENGTH(r_entry);
+	if ((ring->rp + rlen) > ring->ring_size) {
+		DHD_DBGIF(("%s: entry len %d is out of boundary of ring size %d,"
+			" current ring %d[%s] - rp=%d\n", __FUNCTION__, rlen,
+			ring->ring_size, ring->id, ring->name, ring->rp));
+		rlen = 0;
+		goto exit;
+	}
+
+	if (strip_header) {
+		rlen = r_entry->len;
+		buf = (char *)r_entry + DBG_RING_ENTRY_SIZE;
+	} else {
+		rlen = ENTRY_LENGTH(r_entry);
+		buf = (char *)r_entry;
+	}
+	if (rlen > buf_len) {
+		DHD_DBGIF(("%s: buf len %d is too small for entry len %d\n",
+			__FUNCTION__, buf_len, rlen));
+		DHD_DBGIF(("%s: ring %d[%s] - ring size=%d, wp=%d, rp=%d\n",
+			__FUNCTION__, ring->id, ring->name, ring->ring_size,
+			ring->wp, ring->rp));
+		/* The state of ringbuffer is different between calculating buf_len
+		 * and current. ring->rp have chance to be update by pushing data
+		 * to ring buffer when unlocking after calculating buf_len.
+		 * But, It doesn't need to ASSERT because we only send up the
+		 * entries stored so far.
+		 */
+		rlen = 0;
+		goto exit;
+	}
+
+	memcpy(data, buf, rlen);
+	/* update ring context */
+	ring->rp += ENTRY_LENGTH(r_entry);
+	/* don't pass wp but skip padding if there is one */
+	if (ring->rp != ring->wp &&
+	    ring->tail_padded && ((ring->rp + ring->rem_len) >= ring->ring_size)) {
+		DHD_DBGIF(("%s: RING%d[%s] Found padding, rp=%d, wp=%d\n",
+			__FUNCTION__, ring->id, ring->name, ring->rp, ring->wp));
+		ring->rp = 0;
+		ring->tail_padded = FALSE;
+		ring->rem_len = 0;
+	}
+	if (ring->rp >= ring->ring_size) {
+		DHD_DBGIF(("%s: RING%d[%s] rp pointed out of ring boundary,"
+			" rp=%d, ring_size=%d\n", __FUNCTION__, ring->id,
+			ring->name, ring->rp, ring->ring_size));
+		ASSERT(0);
+		rlen = 0;
+		goto exit;
+	}
+	ring->stat.read_bytes += ENTRY_LENGTH(r_entry);
+	DHD_DBGIF(("%s RING%d[%s]read_bytes %d, wp=%d, rp=%d\n", __FUNCTION__,
+		ring->id, ring->name, ring->stat.read_bytes, ring->wp, ring->rp));
+
+exit:
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return rlen;
+}
+
+int
+dhd_dbg_ring_pull(dhd_dbg_ring_t *ring, void *data, uint32 buf_len,
+		bool strip_hdr, int *num_entries)
+{
+	int32 r_len, total_r_len = 0;
+	unsigned long flags;
+
+	if (!ring || !data)
+		return 0;
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	if (!ring->pull_inactive && (ring->state != RING_ACTIVE)) {
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+		return 0;
+	}
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	while (buf_len > 0) {
+		r_len = dhd_dbg_ring_pull_single(ring, data, buf_len, strip_hdr);
+		if (r_len == 0)
+			break;
+		data = (uint8 *)data + r_len;
+		buf_len -= r_len;
+		total_r_len += r_len;
+		(*num_entries)++;
+	}
+
+	return total_r_len;
+}
+
+int
+dhd_dbg_ring_config(dhd_dbg_ring_t *ring, int log_level, uint32 threshold)
+{
+	unsigned long flags = 0;
+
+	if (!ring)
+		return BCME_BADADDR;
+
+	if (ring->state == RING_STOP)
+		return BCME_UNSUPPORTED;
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+
+	if (log_level == 0)
+		ring->state = RING_SUSPEND;
+	else
+		ring->state = RING_ACTIVE;
+
+	ring->log_level = log_level;
+	ring->threshold = MIN(threshold, DBGRING_FLUSH_THRESHOLD(ring));
+
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return BCME_OK;
+}
+
+void
+dhd_dbg_ring_start(dhd_dbg_ring_t *ring)
+{
+	if (!ring)
+		return;
+
+	/* Initialize the information for the ring */
+	ring->state = RING_SUSPEND;
+	ring->log_level = 0;
+	ring->rp = ring->wp = 0;
+	ring->threshold = 0;
+	memset(&ring->stat, 0, sizeof(struct ring_statistics));
+	memset(ring->ring_buf, 0, ring->ring_size);
+}
diff --git a/drivers/net/wireless/ap12275/dhd_dbg_ring.h b/drivers/net/wireless/ap12275/dhd_dbg_ring.h
new file mode 100644
index 000000000000..cce8f15c64fc
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_dbg_ring.h
@@ -0,0 +1,158 @@
+/*
+ * DHD debug ring header file - interface
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_DBG_RING_H__
+#define __DHD_DBG_RING_H__
+
+#include <bcmutils.h>
+
+#if defined(LINUX)
+#define PACKED_STRUCT __attribute__ ((packed))
+#else
+#define PACKED_STRUCT
+#endif
+
+#define DBGRING_NAME_MAX 32
+
+enum dbg_ring_state {
+	RING_STOP = 0,  /* ring is not initialized */
+	RING_ACTIVE,    /* ring is live and logging */
+	RING_SUSPEND    /* ring is initialized but not logging */
+};
+
+/* each entry in dbg ring has below header, to handle
+ * variable length records in ring
+ */
+typedef struct dhd_dbg_ring_entry {
+	uint16 len; /* payload length excluding the header */
+	uint8 flags;
+	uint8 type; /* Per ring specific */
+	uint64 timestamp; /* present if has_timestamp bit is set. */
+} PACKED_STRUCT dhd_dbg_ring_entry_t;
+
+typedef struct dhd_dbg_ring_entry_pack {
+	uint32 magic;
+	int num_entries;
+} PACKED_STRUCT dhd_dbg_ring_entry_pack_t;
+
+struct ring_statistics {
+	/* number of bytes that was written to the buffer by driver */
+	uint32 written_bytes;
+	/* number of bytes that was read from the buffer by user land */
+	uint32 read_bytes;
+	/* number of records that was written to the buffer by driver */
+	uint32 written_records;
+};
+
+typedef struct dhd_dbg_ring_status {
+	uint8 name[DBGRING_NAME_MAX];
+	uint32 flags;
+	int ring_id; /* unique integer representing the ring */
+	/* total memory size allocated for the buffer */
+	uint32 ring_buffer_byte_size;
+	uint32 verbose_level;
+	/* number of bytes that was written to the buffer by driver */
+	uint32 written_bytes;
+	/* number of bytes that was read from the buffer by user land */
+	uint32 read_bytes;
+	/* number of records that was read from the buffer by user land */
+	uint32 written_records;
+} dhd_dbg_ring_status_t;
+
+typedef struct dhd_dbg_ring {
+	int     id;		/* ring id */
+	uint8   name[DBGRING_NAME_MAX]; /* name string */
+	uint32  ring_size;	/* numbers of item in ring */
+	uint32  wp;		/* write pointer */
+	uint32  rp;		/* read pointer */
+	uint32  rp_tmp;		/* tmp read pointer */
+	uint32  log_level;	/* log_level */
+	uint32  threshold;	/* threshold bytes */
+	void *  ring_buf;	/* pointer of actually ring buffer */
+	void *  lock;		/* lock for ring access */
+	struct ring_statistics stat;	/* statistics */
+	enum dbg_ring_state state;	/* ring state enum */
+	bool tail_padded;	/* writer does not have enough space */
+	uint32 rem_len;		/* number of bytes from wp_pad to end */
+	bool sched_pull;	/* schedule reader immediately */
+	bool pull_inactive;	/* pull contents from ring even if it is inactive */
+} dhd_dbg_ring_t;
+
+#define DBGRING_FLUSH_THRESHOLD(ring)		\
+	(ring->id != PACKET_LOG_RING_ID) ?		\
+	(ring->ring_size / 3u) : (ring->ring_size / 4u)
+#define RING_STAT_TO_STATUS(ring, status) \
+	do { \
+		/* status.name/ring->name are the same length so no need to check return value */ \
+		(void)memcpy_s(status.name, sizeof(status.name), ring->name, sizeof(ring->name)); \
+		status.ring_id = ring->id;     \
+		status.ring_buffer_byte_size = ring->ring_size;  \
+		status.written_bytes = ring->stat.written_bytes; \
+		status.written_records = ring->stat.written_records; \
+		status.read_bytes = ring->stat.read_bytes; \
+		status.verbose_level = ring->log_level; \
+	} while (0)
+
+#define DBG_RING_PACK_MAGIC 0xDBAADBAA
+#define DBG_RING_ENTRY_PACK_SIZE (sizeof(dhd_dbg_ring_entry_pack_t))
+#define DBG_RING_ENTRY_SIZE (sizeof(dhd_dbg_ring_entry_t))
+#define ENTRY_LENGTH(hdr) ((hdr)->len + DBG_RING_ENTRY_SIZE)
+#define PAYLOAD_MAX_LEN 65535
+#define PAYLOAD_RTT_MAX_LEN 1648u
+#define PAYLOAD_BCM_TRACE_MAX_LEN 1648u
+#define PENDING_LEN_MAX 0xFFFFFFFF
+#define DBG_RING_STATUS_SIZE (sizeof(dhd_dbg_ring_status_t))
+
+#define TXACTIVESZ(r, w, d)			(((r) <= (w)) ? ((w) - (r)) : ((d) - (r) + (w)))
+#define DBG_RING_READ_AVAIL_SPACE(w, r, d)		(((w) >= (r)) ? ((w) - (r)) : ((d) - (r)))
+#define DBG_RING_WRITE_SPACE_AVAIL_CONT(r, w, d)	(((w) >= (r)) ? ((d) - (w)) : ((r) - (w)))
+#define DBG_RING_WRITE_SPACE_AVAIL(r, w, d)		(d - (TXACTIVESZ(r, w, d)))
+#define DBG_RING_CHECK_WRITE_SPACE(r, w, d)		\
+		MIN(DBG_RING_WRITE_SPACE_AVAIL(r, w, d), DBG_RING_WRITE_SPACE_AVAIL_CONT(r, w, d))
+
+typedef void (*os_pullreq_t)(void *os_priv, const int ring_id);
+
+dhd_dbg_ring_t *dhd_dbg_ring_alloc_init(dhd_pub_t *dhd, uint16 ring_id,
+	char *ring_name, uint32 ring_sz, void *allocd_buf,
+	bool pull_inactive);
+void dhd_dbg_ring_dealloc_deinit(void **dbgring, dhd_pub_t *dhd);
+int dhd_dbg_ring_init(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring, uint16 id, uint8 *name,
+		uint32 ring_sz, void *allocd_buf, bool pull_inactive);
+void dhd_dbg_ring_deinit(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring);
+int dhd_dbg_ring_set_buf(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring, void *buf);
+#ifdef DHD_PKT_LOGGING_DBGRING
+int dhd_dbg_ring_update(void *dbg_ring, uint32 w_len);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+int dhd_dbg_ring_push(dhd_dbg_ring_t *ring, dhd_dbg_ring_entry_t *hdr, void *data);
+int dhd_dbg_ring_pull(dhd_dbg_ring_t *ring, void *data, uint32 buf_len,
+		bool strip_hdr, int* num_entries);
+int dhd_dbg_ring_pull_single(dhd_dbg_ring_t *ring, void *data, uint32 buf_len,
+	bool strip_header);
+uint32 dhd_dbg_ring_get_pending_len(dhd_dbg_ring_t *ring);
+void dhd_dbg_ring_sched_pull(dhd_dbg_ring_t *ring, uint32 pending_len,
+		os_pullreq_t pull_fn, void *os_pvt, const int id);
+int dhd_dbg_ring_config(dhd_dbg_ring_t *ring, int log_level, uint32 threshold);
+void dhd_dbg_ring_start(dhd_dbg_ring_t *ring);
+#endif /* __DHD_DBG_RING_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_debug.c b/drivers/net/wireless/ap12275/dhd_debug.c
new file mode 100644
index 000000000000..e065e2804167
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_debug.c
@@ -0,0 +1,3673 @@
+/*
+ * DHD debugability support
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_dbg_ring.h>
+#include <dhd_debug.h>
+#include <dhd_mschdbg.h>
+#include <dhd_bus.h>
+
+#include <event_log.h>
+#include <event_trace.h>
+#include <msgtrace.h>
+#if defined(DBG_PKT_MON)
+#include <dhd_linux_priv.h>
+#include <dhd_linux_wq.h>
+#endif /* DBG_PKT_MON */
+#if defined(DHD_PKT_LOGGING) && defined(DHD_PKT_LOGGING_DBGRING)
+#include <dhd_pktlog.h>
+#endif /* DHD_PKT_LOGGING && DHD_PKT_LOGGING_DBGRING */
+
+#if defined(DHD_EVENT_LOG_FILTER)
+#include <dhd_event_log_filter.h>
+#endif /* DHD_EVENT_LOG_FILTER */
+
+#ifdef WL_CFGVENDOR_CUST_ADVLOG
+void wl_cfgvendor_custom_advlog_roam_log(void *plog, uint32 armcycle);
+#endif /* WL_CFGVENDOR_CUST_ADVLOG */
+
+#if defined(NDIS)
+#if !defined(offsetof)
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif	/* !defined(offsetof) */
+
+#define container_of(ptr, type, member) \
+		(type *)((char *)(ptr) - offsetof(type, member))
+#endif /* NDIS */
+
+uint8 control_logtrace = CUSTOM_CONTROL_LOGTRACE;
+
+struct map_table {
+	uint16 fw_id;
+	uint16 host_id;
+	char *desc;
+};
+
+struct map_table event_map[] = {
+	{WLC_E_AUTH, WIFI_EVENT_AUTH_COMPLETE, "AUTH_COMPLETE"},
+	{WLC_E_ASSOC, WIFI_EVENT_ASSOC_COMPLETE, "ASSOC_COMPLETE"},
+	{TRACE_FW_AUTH_STARTED, WIFI_EVENT_FW_AUTH_STARTED, "AUTH STARTED"},
+	{TRACE_FW_ASSOC_STARTED, WIFI_EVENT_FW_ASSOC_STARTED, "ASSOC STARTED"},
+	{TRACE_FW_RE_ASSOC_STARTED, WIFI_EVENT_FW_RE_ASSOC_STARTED, "REASSOC STARTED"},
+	{TRACE_G_SCAN_STARTED, WIFI_EVENT_G_SCAN_STARTED, "GSCAN STARTED"},
+	{WLC_E_PFN_SCAN_COMPLETE, WIFI_EVENT_G_SCAN_COMPLETE, "GSCAN COMPLETE"},
+	{WLC_E_DISASSOC, WIFI_EVENT_DISASSOCIATION_REQUESTED, "DIASSOC REQUESTED"},
+	{WLC_E_REASSOC, WIFI_EVENT_RE_ASSOCIATION_REQUESTED, "REASSOC REQUESTED"},
+	{TRACE_ROAM_SCAN_STARTED, WIFI_EVENT_ROAM_REQUESTED, "ROAM REQUESTED"},
+	{WLC_E_BEACON_FRAME_RX, WIFI_EVENT_BEACON_RECEIVED, "BEACON Received"},
+	{TRACE_ROAM_SCAN_STARTED, WIFI_EVENT_ROAM_SCAN_STARTED, "ROAM SCAN STARTED"},
+	{TRACE_ROAM_SCAN_COMPLETE, WIFI_EVENT_ROAM_SCAN_COMPLETE, "ROAM SCAN COMPLETED"},
+	{TRACE_ROAM_AUTH_STARTED, WIFI_EVENT_ROAM_AUTH_STARTED, "ROAM AUTH STARTED"},
+	{WLC_E_AUTH, WIFI_EVENT_ROAM_AUTH_COMPLETE, "ROAM AUTH COMPLETED"},
+	{TRACE_FW_RE_ASSOC_STARTED, WIFI_EVENT_ROAM_ASSOC_STARTED, "ROAM ASSOC STARTED"},
+	{WLC_E_ASSOC, WIFI_EVENT_ROAM_ASSOC_COMPLETE, "ROAM ASSOC COMPLETED"},
+	{TRACE_ROAM_SCAN_COMPLETE, WIFI_EVENT_ROAM_SCAN_COMPLETE, "ROAM SCAN COMPLETED"},
+	{TRACE_BT_COEX_BT_SCO_START, WIFI_EVENT_BT_COEX_BT_SCO_START, "BT SCO START"},
+	{TRACE_BT_COEX_BT_SCO_STOP, WIFI_EVENT_BT_COEX_BT_SCO_STOP, "BT SCO STOP"},
+	{TRACE_BT_COEX_BT_SCAN_START, WIFI_EVENT_BT_COEX_BT_SCAN_START, "BT COEX SCAN START"},
+	{TRACE_BT_COEX_BT_SCAN_STOP, WIFI_EVENT_BT_COEX_BT_SCAN_STOP, "BT COEX SCAN STOP"},
+	{TRACE_BT_COEX_BT_HID_START, WIFI_EVENT_BT_COEX_BT_HID_START, "BT HID START"},
+	{TRACE_BT_COEX_BT_HID_STOP, WIFI_EVENT_BT_COEX_BT_HID_STOP, "BT HID STOP"},
+	{WLC_E_EAPOL_MSG, WIFI_EVENT_FW_EAPOL_FRAME_RECEIVED, "FW EAPOL PKT RECEIVED"},
+	{TRACE_FW_EAPOL_FRAME_TRANSMIT_START, WIFI_EVENT_FW_EAPOL_FRAME_TRANSMIT_START,
+	"FW EAPOL PKT TRANSMITED"},
+	{TRACE_FW_EAPOL_FRAME_TRANSMIT_STOP, WIFI_EVENT_FW_EAPOL_FRAME_TRANSMIT_STOP,
+	"FW EAPOL PKT TX STOPPED"},
+	{TRACE_BLOCK_ACK_NEGOTIATION_COMPLETE, WIFI_EVENT_BLOCK_ACK_NEGOTIATION_COMPLETE,
+	"BLOCK ACK NEGO COMPLETED"},
+};
+
+struct map_table event_tag_map[] = {
+	{TRACE_TAG_VENDOR_SPECIFIC, WIFI_TAG_VENDOR_SPECIFIC, "VENDOR SPECIFIC DATA"},
+	{TRACE_TAG_BSSID, WIFI_TAG_BSSID, "BSSID"},
+	{TRACE_TAG_ADDR, WIFI_TAG_ADDR, "ADDR_0"},
+	{TRACE_TAG_SSID, WIFI_TAG_SSID, "SSID"},
+	{TRACE_TAG_STATUS, WIFI_TAG_STATUS, "STATUS"},
+	{TRACE_TAG_CHANNEL_SPEC, WIFI_TAG_CHANNEL_SPEC, "CHANSPEC"},
+	{TRACE_TAG_WAKE_LOCK_EVENT, WIFI_TAG_WAKE_LOCK_EVENT, "WAKELOCK EVENT"},
+	{TRACE_TAG_ADDR1, WIFI_TAG_ADDR1, "ADDR_1"},
+	{TRACE_TAG_ADDR2, WIFI_TAG_ADDR2, "ADDR_2"},
+	{TRACE_TAG_ADDR3, WIFI_TAG_ADDR3, "ADDR_3"},
+	{TRACE_TAG_ADDR4, WIFI_TAG_ADDR4, "ADDR_4"},
+	{TRACE_TAG_TSF, WIFI_TAG_TSF, "TSF"},
+	{TRACE_TAG_IE, WIFI_TAG_IE, "802.11 IE"},
+	{TRACE_TAG_INTERFACE, WIFI_TAG_INTERFACE, "INTERFACE"},
+	{TRACE_TAG_REASON_CODE, WIFI_TAG_REASON_CODE, "REASON CODE"},
+	{TRACE_TAG_RATE_MBPS, WIFI_TAG_RATE_MBPS, "RATE"},
+};
+
+struct event_log_table {
+	int log_level;
+	uint16 tag;
+	uint8 set;
+	char *desc;
+};
+
+/* define log level per ring type */
+struct event_log_table fw_verbose_level_map[] = {
+	{1, EVENT_LOG_TAG_PCI_ERROR, EVENT_LOG_SET_BUS, "PCI_ERROR"},
+	{1, EVENT_LOG_TAG_PCI_WARN, EVENT_LOG_SET_PRSRV_BUS, "PCI_WARN"},
+	{1, EVENT_LOG_TAG_PCI_INFO, EVENT_LOG_SET_BUS, "PCI_INFO"},
+	{3, EVENT_LOG_TAG_PCI_DBG, EVENT_LOG_SET_BUS, "PCI_DEBUG"},
+	{2, EVENT_LOG_TAG_BEACON_LOG, EVENT_LOG_SET_PRSRV_CHATTY, "BEACON_LOG"},
+	{1, EVENT_LOG_TAG_WL_ASSOC_LOG, EVENT_LOG_SET_PRSRV, "ASSOC_LOG"},
+	{1, EVENT_LOG_TAG_WL_ROAM_LOG, EVENT_LOG_SET_PRSRV, "ROAM_LOG"},
+	{1, EVENT_LOG_TAG_TRACE_WL_INFO, EVENT_LOG_SET_WL, "WL INFO"},
+	{1, EVENT_LOG_TAG_TRACE_BTCOEX_INFO, EVENT_LOG_SET_WL, "BTCOEX INFO"},
+#ifdef DHD_RANDMAC_LOGGING
+	{1, EVENT_LOG_TAG_RANDMAC_ERR, EVENT_LOG_SET_PRSRV, "RANDMAC_ERR"},
+#endif /* DHD_RANDMAC_LOGGING */
+	{1, EVENT_LOG_TAG_SCAN_WARN, EVENT_LOG_SET_PRSRV_CHATTY, "SCAN_WARN"},
+	{2, EVENT_LOG_TAG_SCAN_ERROR, EVENT_LOG_SET_ERROR, "SCAN_ERROR"},
+	{1, EVENT_LOG_TAG_SCAN_TRACE_LOW, EVENT_LOG_SET_PRSRV, "SCAN_TRACE_LOW"},
+	{1, EVENT_LOG_TAG_SCAN_TRACE_HIGH, EVENT_LOG_SET_WL, "SCAN_TRACE_HIGH"},
+};
+
+/* reference tab table */
+uint ref_tag_tbl[EVENT_LOG_TAG_MAX + 1] = {0};
+
+typedef struct dhddbg_loglist_item {
+	dll_t list;
+	prcd_event_log_hdr_t prcd_log_hdr;
+} loglist_item_t;
+
+typedef struct dhbdbg_pending_item {
+	dll_t list;
+	dhd_dbg_ring_status_t ring_status;
+	dhd_dbg_ring_entry_t *ring_entry;
+} pending_item_t;
+
+/* trace log entry header user space processing */
+struct tracelog_header {
+	int magic_num;
+	int buf_size;
+	int seq_num;
+};
+#define TRACE_LOG_MAGIC_NUMBER 0xEAE47C06
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+int
+dhd_dbg_update_to_ring(dhd_pub_t *dhdp, void *ring, uint32 w_len)
+{
+	return dhd_dbg_push_to_ring(dhdp, ((dhd_dbg_ring_t *)ring)->id, NULL, (void*)&w_len);
+}
+
+static uint32
+dhd_dbg_urgent_pull(dhd_pub_t *dhdp, dhd_dbg_ring_t *ring)
+{
+	uint32 pending_len = 0;
+	struct dhd_pktlog_ring *pktlog_ring;
+
+	if (ring->id != PACKET_LOG_RING_ID) {
+		return pending_len;
+	}
+
+	if (ring->stat.written_bytes > ring->stat.read_bytes) {
+		pending_len = ring->stat.written_bytes - ring->stat.read_bytes;
+	} else {
+		pending_len = 0;
+	}
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	if (pktlog_ring->pktcount >= DHD_PACKET_LOG_RING_SUSPEND_THRESHOLD) {
+		dhd_pktlog_suspend(dhdp);
+	}
+
+	if (pending_len > ring->threshold) {
+		DHD_INFO(("%s: pending_len(%d) is exceeded threshold(%d), pktcount(%d)\n",
+			__FUNCTION__, pending_len, ring->threshold, pktlog_ring->pktcount));
+	}
+
+	return pending_len;
+}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+int
+dhd_debug_dump_ring_push(dhd_pub_t *dhdp, int ring_id, uint32 len, void *data)
+{
+	dhd_dbg_ring_t *ring;
+	int ret = 0;
+	uint32 remain_len = 0;
+	char *cur_buf;
+	dhd_dbg_ring_entry_t msg_hdr;
+
+	if (!dhdp || !dhdp->dbg) {
+		return BCME_BADADDR;
+	}
+
+	if (!VALID_RING(ring_id)) {
+		DHD_ERROR(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return BCME_RANGE;
+	}
+
+	ring = &dhdp->dbg->dbg_rings[ring_id];
+
+	remain_len = len;
+	cur_buf = data;
+
+	memset(&msg_hdr, 0, sizeof(msg_hdr));
+	while (remain_len > 0) {
+		msg_hdr.type = DBG_RING_ENTRY_DATA_TYPE;
+		msg_hdr.len = MIN(remain_len, DHD_DEBUG_DUMP_NETLINK_MAX);
+		ret = dhd_dbg_ring_push(ring, &msg_hdr, cur_buf);
+		if (ret == BCME_OK) {
+			cur_buf += msg_hdr.len;
+			remain_len -= msg_hdr.len;
+		}
+	}
+	return ret;
+}
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+
+int
+dhd_dbg_push_to_ring(dhd_pub_t *dhdp, int ring_id, dhd_dbg_ring_entry_t *hdr, void *data)
+{
+	dhd_dbg_ring_t *ring;
+	int ret = 0;
+	uint32 pending_len = 0;
+
+	if (!dhdp || !dhdp->dbg) {
+		return BCME_BADADDR;
+	}
+
+	if (!VALID_RING(ring_id)) {
+		DHD_ERROR(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return BCME_RANGE;
+	}
+
+	ring = &dhdp->dbg->dbg_rings[ring_id];
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (ring_id == PACKET_LOG_RING_ID) {
+		ret = dhd_dbg_ring_update(ring, *(uint32*)data);
+	} else
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	{
+		ret = dhd_dbg_ring_push(ring, hdr, data);
+	}
+	if (ret != BCME_OK)
+		return ret;
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	pending_len = dhd_dbg_urgent_pull(dhdp, ring);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	if (pending_len == 0) {
+		pending_len = dhd_dbg_ring_get_pending_len(ring);
+	}
+	dhd_dbg_ring_sched_pull(ring, pending_len, dhdp->dbg->pullreq,
+		dhdp->dbg->private, ring->id);
+
+	return ret;
+}
+
+dhd_dbg_ring_t *
+dhd_dbg_get_ring_from_ring_id(dhd_pub_t *dhdp, int ring_id)
+{
+	if (!dhdp || !dhdp->dbg) {
+		return NULL;
+	}
+
+	if (!VALID_RING(ring_id)) {
+		DHD_ERROR(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return NULL;
+	}
+
+	return &dhdp->dbg->dbg_rings[ring_id];
+}
+
+int
+dhd_dbg_pull_single_from_ring(dhd_pub_t *dhdp, int ring_id, void *data, uint32 buf_len,
+	bool strip_header)
+{
+	dhd_dbg_ring_t *ring;
+
+	if (!dhdp || !dhdp->dbg) {
+		return 0;
+	}
+
+	if (!VALID_RING(ring_id)) {
+		DHD_ERROR(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return BCME_RANGE;
+	}
+
+	ring = &dhdp->dbg->dbg_rings[ring_id];
+
+	return dhd_dbg_ring_pull_single(ring, data, buf_len, strip_header);
+}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+extern int dhd_pktlog_dump_write(dhd_pub_t *dhdp, void *file,
+	const void *user_buf, uint32 *written_bytes);
+
+int
+dhd_dbg_pull_from_pktlog(dhd_pub_t *dhdp, int ring_id, void *data, uint32 buf_len)
+{
+	dhd_dbg_ring_entry_t *r_entry = NULL;
+	char *buf = NULL;
+	int ret = BCME_OK;
+	uint32 written_bytes = 0;
+
+	if (!dhdp || !dhdp->dbg) {
+		return 0;
+	}
+	if (!VALID_RING(ring_id)) {
+		DHD_DBGIF(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return BCME_RANGE;
+	}
+	if (ring_id != PACKET_LOG_RING_ID) {
+		return BCME_UNSUPPORTED;
+	}
+
+	r_entry = (dhd_dbg_ring_entry_t *)data;
+	r_entry->len = buf_len - DBG_RING_ENTRY_SIZE;
+	written_bytes = (uint32)r_entry->len;
+	if (!written_bytes) {
+		return 0;
+	}
+
+	r_entry->type = DBG_RING_ENTRY_DATA_TYPE;
+	r_entry->flags = (DBG_RING_ENTRY_FLAGS_HAS_TIMESTAMP |
+		DBG_RING_ENTRY_FLAGS_HAS_BINARY);
+	r_entry->timestamp = local_clock();
+	r_entry->timestamp = DIV_U64_BY_U32(r_entry->timestamp, NSEC_PER_MSEC);
+	buf = (char*)data;
+
+	ret = dhd_pktlog_dump_write(dhdp, NULL, buf + DBG_RING_ENTRY_SIZE, &written_bytes);
+
+	if (ret != BCME_OK) {
+		DHD_DBGIF(("%s : pktlog_dump error\n", __FUNCTION__));
+		buf_len = 0;
+	} else {
+		r_entry->len = (uint16)written_bytes;
+		buf_len = r_entry->len + DBG_RING_ENTRY_SIZE;
+	}
+
+	return buf_len;
+}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+int
+dhd_dbg_pull_from_ring(dhd_pub_t *dhdp, int ring_id, void *data, uint32 buf_len,
+		int *num_entries)
+{
+	dhd_dbg_ring_t *ring;
+
+	if (!dhdp || !dhdp->dbg)
+		return 0;
+	if (!VALID_RING(ring_id)) {
+		DHD_DBGIF(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return BCME_RANGE;
+	}
+	ring = &dhdp->dbg->dbg_rings[ring_id];
+	return dhd_dbg_ring_pull(ring, data, buf_len, FALSE, num_entries);
+}
+
+static int
+dhd_dbg_msgtrace_seqchk(uint32 *prev, uint32 cur)
+{
+	/* normal case including wrap around */
+	if ((cur == 0 && *prev == 0xFFFFFFFF) || ((cur - *prev) == 1)) {
+		goto done;
+	} else if (cur == *prev) {
+		DHD_EVENT(("%s duplicate trace\n", __FUNCTION__));
+		return -1;
+	} else if (cur > *prev) {
+		DHD_EVENT(("%s lost %d packets\n", __FUNCTION__, cur - *prev));
+	} else {
+		DHD_EVENT(("%s seq out of order, dhd %d, dongle %d\n",
+			__FUNCTION__, *prev, cur));
+	}
+done:
+	*prev = cur;
+	return 0;
+}
+
+static void
+dhd_dbg_msgtrace_msg_parser(void *event_data)
+{
+	msgtrace_hdr_t *hdr;
+	char *data, *s;
+	static uint32 seqnum_prev = 0;
+
+	if (!event_data) {
+		DHD_ERROR(("%s: event_data is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	hdr = (msgtrace_hdr_t *)event_data;
+	data = (char *)event_data + MSGTRACE_HDRLEN;
+
+	/* There are 2 bytes available at the end of data */
+	data[ntoh16(hdr->len)] = '\0';
+
+	if (ntoh32(hdr->discarded_bytes) || ntoh32(hdr->discarded_printf)) {
+		DHD_DBGIF(("WLC_E_TRACE: [Discarded traces in dongle -->"
+			"discarded_bytes %d discarded_printf %d]\n",
+			ntoh32(hdr->discarded_bytes),
+			ntoh32(hdr->discarded_printf)));
+	}
+
+	if (dhd_dbg_msgtrace_seqchk(&seqnum_prev, ntoh32(hdr->seqnum)))
+		return;
+
+	/* Display the trace buffer. Advance from
+	 * \n to \n to avoid display big
+	 * printf (issue with Linux printk )
+	 */
+	while (*data != '\0' && (s = strstr(data, "\n")) != NULL) {
+		*s = '\0';
+		printf("[FWLOG] %s\n", data);
+		data = s+1;
+	}
+	if (*data)
+		printf("[FWLOG] %s", data);
+}
+#ifdef SHOW_LOGTRACE
+#define DATA_UNIT_FOR_LOG_CNT 4
+
+int
+replace_percent_p_to_x(char *fmt)
+{
+	int p_to_x_done = FALSE;
+
+	while (*fmt != '\0')
+	{
+		/* Skip characters will we see a % */
+		if (*fmt++ != '%')
+		{
+			continue;
+		}
+
+		/*
+		 * Skip any flags, field width and precision:
+		 *Flags: Followed by %
+		 * #, 0, -, ' ', +
+		 */
+		if (*fmt == '#')
+			fmt++;
+
+		if (*fmt == '0' || *fmt == '-' || *fmt == '+')
+			fmt++;
+
+		/*
+		 * Field width:
+		 * An optional decimal digit string (with non-zero first digit)
+		 * specifying a minimum field width
+		 */
+		while (*fmt && bcm_isdigit(*fmt))
+			fmt++;
+
+		/*
+		 * Precision:
+		 * An optional precision, in the form of a period ('.')  followed by an
+		 * optional decimal digit string.
+		 */
+		if (*fmt == '.')
+		{
+			fmt++;
+			while (*fmt && bcm_isdigit(*fmt)) fmt++;
+		}
+
+		/* If %p is seen, change it to %x */
+		if (*fmt == 'p')
+		{
+			*fmt = 'x';
+			p_to_x_done = TRUE;
+		}
+		if (*fmt)
+			fmt++;
+	}
+
+	return p_to_x_done;
+}
+
+/* To identify format of types %Ns where N >= 0 is a number */
+bool
+check_valid_string_format(char *curr_ptr)
+{
+	char *next_ptr;
+	if ((next_ptr = bcmstrstr(curr_ptr, "s")) != NULL) {
+		/* Default %s format */
+		if (curr_ptr == next_ptr) {
+			return TRUE;
+		}
+
+		/* Verify each charater between '%' and 's' is a valid number */
+		while (curr_ptr < next_ptr) {
+			if (bcm_isdigit(*curr_ptr) == FALSE) {
+				return FALSE;
+			}
+			curr_ptr++;
+		}
+
+		return TRUE;
+	} else {
+		return FALSE;
+	}
+}
+
+/* To identify format of non string format types */
+bool
+check_valid_non_string_format(char *curr_ptr)
+{
+	char *next_ptr;
+	char *next_fmt_stptr;
+	char valid_fmt_types[17] = {'d', 'i', 'x', 'X', 'c', 'p', 'u',
+			'f', 'F', 'e', 'E', 'g', 'G', 'o',
+			'a', 'A', 'n'};
+	int i;
+	bool valid = FALSE;
+
+	/* Check for next % in the fmt str */
+	next_fmt_stptr = bcmstrstr(curr_ptr, "%");
+
+	for (next_ptr = curr_ptr; *next_ptr != '\0'; next_ptr++) {
+		for (i = 0; i < (int)((sizeof(valid_fmt_types))/sizeof(valid_fmt_types[0])); i++) {
+			if (*next_ptr == valid_fmt_types[i]) {
+				/* Check whether format type found corresponds to current %
+				 * and not the next one, if exists.
+				 */
+				if ((next_fmt_stptr == NULL) ||
+						(next_fmt_stptr && (next_ptr < next_fmt_stptr))) {
+					/* Not validating for length/width fields in
+					 * format specifier.
+					 */
+					valid = TRUE;
+				}
+				goto done;
+			}
+		}
+	}
+
+done:
+	return valid;
+}
+
+#define MAX_NO_OF_ARG	16
+#define FMTSTR_SIZE	200
+#define ROMSTR_SIZE	268
+#define SIZE_LOC_STR	50
+#define LOG_PRINT_CNT_MAX	16u
+#define EL_MSEC_PER_SEC	1000
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+#ifdef DHD_DEBUG
+#undef MAX_LOG_PRINT_COUNT
+#define MAX_LOG_PRINT_COUNT   50000u
+#undef LOG_PRINT_THRESH
+#define LOG_PRINT_THRESH      (1u * USEC_PER_SEC)
+#else /* DHD_DEBUG */
+#ifndef MAX_LOG_PRINT_COUNT
+#define MAX_LOG_PRINT_COUNT   5000u
+#endif /* MAX_LOG_PRINT_COUNT */
+#ifndef LOG_PRINT_THRESH
+#define LOG_PRINT_THRESH      (1u * USEC_PER_SEC)
+#endif /* LOG_PRINT_THRESH */
+#endif /* DHD_DEBUG */
+#endif
+#define EL_PARSE_VER	"V02"
+static uint64 verboselog_ts_saved = 0;
+
+bool
+dhd_dbg_process_event_log_hdr(event_log_hdr_t *log_hdr, prcd_event_log_hdr_t *prcd_log_hdr)
+{
+	event_log_extended_hdr_t *ext_log_hdr;
+	uint16 event_log_fmt_num;
+	uint8 event_log_hdr_type;
+	uint8 extended_fmtnum = 0;
+
+	/* Identify the type of event tag, payload type etc..  */
+	event_log_hdr_type = log_hdr->fmt_num & DHD_EVENT_LOG_HDR_MASK;
+	event_log_fmt_num = (log_hdr->fmt_num >> DHD_EVENT_LOG_FMT_NUM_OFFSET) &
+		DHD_EVENT_LOG_FMT_NUM_MASK;
+
+	switch (event_log_hdr_type) {
+		case DHD_OW_NB_EVENT_LOG_HDR:
+			prcd_log_hdr->ext_event_log_hdr = FALSE;
+			prcd_log_hdr->binary_payload = FALSE;
+			break;
+		case DHD_TW_NB_EVENT_LOG_HDR:
+			prcd_log_hdr->ext_event_log_hdr = TRUE;
+			prcd_log_hdr->binary_payload = FALSE;
+			break;
+		case DHD_BI_EVENT_LOG_HDR:
+			if (event_log_fmt_num == DHD_OW_BI_EVENT_FMT_NUM) {
+				prcd_log_hdr->ext_event_log_hdr = FALSE;
+				prcd_log_hdr->binary_payload = TRUE;
+			} else if (event_log_fmt_num == DHD_TW_BI_EVENT_FMT_NUM) {
+				prcd_log_hdr->ext_event_log_hdr = TRUE;
+				prcd_log_hdr->binary_payload = TRUE;
+			} else {
+				DHD_ERROR(("%s: invalid format number 0x%X\n",
+					__FUNCTION__, event_log_fmt_num));
+				return FALSE;
+			}
+			break;
+		case DHD_INVALID_EVENT_LOG_HDR:
+		default:
+			DHD_ERROR(("%s: invalid event log header type 0x%X\n",
+				__FUNCTION__, event_log_hdr_type));
+			return FALSE;
+	}
+
+	/* Parse extended and legacy event log headers and populate prcd_event_log_hdr_t */
+	if (prcd_log_hdr->ext_event_log_hdr) {
+		ext_log_hdr = (event_log_extended_hdr_t *)
+			((uint8 *)log_hdr - sizeof(event_log_hdr_t));
+		extended_fmtnum = (ext_log_hdr->extended_fmtnum & DHD_EXT_FMTNUM_MASK);
+	}
+	/* Parse extended event log tag */
+	if (prcd_log_hdr->ext_event_log_hdr) {
+		prcd_log_hdr->tag = ((ext_log_hdr->extended_tag &
+			DHD_TW_VALID_TAG_BITS_MASK) << DHD_TW_EVENT_LOG_TAG_OFFSET) | log_hdr->tag;
+	} else {
+		prcd_log_hdr->tag = log_hdr->tag;
+	}
+	/* Parse extended format number */
+	if ((event_log_hdr_type == DHD_TW_NB_EVENT_LOG_HDR) && (extended_fmtnum != 0)) {
+		prcd_log_hdr->fmt_num_raw = (extended_fmtnum << 16) | log_hdr->fmt_num;
+		prcd_log_hdr->fmt_num = (extended_fmtnum << 14) | event_log_fmt_num;
+
+	} else {
+		prcd_log_hdr->fmt_num_raw = log_hdr->fmt_num;
+		prcd_log_hdr->fmt_num = event_log_fmt_num;
+	}
+	prcd_log_hdr->count = log_hdr->count;
+
+	/* update arm cycle */
+	/*
+	 * For loegacy event tag :-
+	 * |payload........|Timestamp| Tag
+	 *
+	 * For extended event tag:-
+	 * |payload........|Timestamp|extended Tag| Tag.
+	 *
+	 */
+	prcd_log_hdr->armcycle = prcd_log_hdr->ext_event_log_hdr ?
+		*(uint32 *)(log_hdr - EVENT_TAG_TIMESTAMP_EXT_OFFSET) :
+		*(uint32 *)(log_hdr - EVENT_TAG_TIMESTAMP_OFFSET);
+
+	/* update event log data pointer address */
+	prcd_log_hdr->log_ptr =
+		(uint32 *)log_hdr - log_hdr->count - prcd_log_hdr->ext_event_log_hdr;
+
+	/* handle error cases above this */
+	return TRUE;
+}
+
+static void
+dhd_dbg_verboselog_handler(dhd_pub_t *dhdp, prcd_event_log_hdr_t *plog_hdr,
+		void *raw_event_ptr, uint32 logset, uint16 block, uint32* data)
+{
+	event_log_hdr_t *ts_hdr;
+	uint32 *log_ptr = plog_hdr->log_ptr;
+	char fmtstr_loc_buf[ROMSTR_SIZE] = { 0 };
+	uint32 rom_str_len = 0;
+	uint32 *ts_data;
+
+	if (!raw_event_ptr) {
+		return;
+	}
+
+	if (log_ptr < data) {
+		DHD_ERROR(("Invalid log pointer, logptr : %p data : %p \n", log_ptr, data));
+		return;
+	}
+
+	if (log_ptr > data) {
+		/* Get time stamp if it's updated */
+		ts_hdr = (event_log_hdr_t *)((char *)log_ptr - sizeof(event_log_hdr_t));
+		if (ts_hdr->tag == EVENT_LOG_TAG_TS) {
+			ts_data = (uint32 *)ts_hdr - ts_hdr->count;
+			if (ts_data >= data) {
+				verboselog_ts_saved = (uint64)ts_data[0];
+				DHD_MSGTRACE_LOG(("EVENT_LOG_TS[0x%08x]: SYS:%08x CPU:%08x\n",
+					ts_data[ts_hdr->count - 1], ts_data[0], ts_data[1]));
+			}
+		} else if (ts_hdr->tag == EVENT_LOG_TAG_ENHANCED_TS) {
+			ets_msg_v1_t *ets;
+			ets = (ets_msg_v1_t *)ts_hdr - ts_hdr->count;
+			if ((uint32*)ets >= data &&
+				ts_hdr->count >= (sizeof(ets_msg_v1_t) / sizeof(uint32)) &&
+				ets->version == ENHANCED_TS_MSG_VERSION_1) {
+				DHD_MSGTRACE_LOG(("EVENT_LOG_ENHANCED_TS_V1: "
+					"SYS:%08x CPU:%08x CPUFREQ:%u\n",
+					ets->timestamp, ets->cyclecount, ets->cpu_freq));
+			}
+		}
+	}
+
+	if (plog_hdr->tag == EVENT_LOG_TAG_ROM_PRINTF) {
+		rom_str_len = (plog_hdr->count - 1) * sizeof(uint32);
+		if (rom_str_len >= (ROMSTR_SIZE -1))
+			rom_str_len = ROMSTR_SIZE - 1;
+
+		/* copy all ascii data for ROM printf to local string */
+		memcpy(fmtstr_loc_buf, log_ptr, rom_str_len);
+		/* add end of line at last */
+		fmtstr_loc_buf[rom_str_len] = '\0';
+
+		DHD_MSGTRACE_LOG(("EVENT_LOG_ROM[0x%08x]: %s",
+				log_ptr[plog_hdr->count - 1], fmtstr_loc_buf));
+
+		if (!strlen(fmtstr_loc_buf)) {
+			DHD_ERROR(("%s fmt_str_loc_buf len can not be zero\n", __FUNCTION__));
+			return;
+		}
+
+		/* Add newline if missing */
+		if (fmtstr_loc_buf[strlen(fmtstr_loc_buf) - 1] != '\n')
+			DHD_MSGTRACE_LOG(("\n"));
+
+		return;
+	}
+
+	if (plog_hdr->tag == EVENT_LOG_TAG_MSCHPROFILE ||
+		plog_hdr->tag == EVENT_LOG_TAG_MSCHPROFILE_TLV) {
+		wl_mschdbg_verboselog_handler(dhdp, raw_event_ptr, plog_hdr, log_ptr);
+		return;
+	}
+
+	/* print the message out in a logprint  */
+	dhd_dbg_verboselog_printf(dhdp, plog_hdr, raw_event_ptr, log_ptr, logset, block);
+}
+
+void
+dhd_dbg_verboselog_printf(dhd_pub_t *dhdp, prcd_event_log_hdr_t *plog_hdr,
+	void *raw_event_ptr, uint32 *log_ptr, uint32 logset, uint16 block)
+{
+	dhd_event_log_t *raw_event = (dhd_event_log_t *)raw_event_ptr;
+	uint16 count;
+	int log_level, id;
+	char fmtstr_loc_buf[ROMSTR_SIZE] = { 0 };
+	char (*str_buf)[SIZE_LOC_STR] = NULL;
+	char *str_tmpptr = NULL;
+	uint32 addr = 0;
+	typedef union {
+		uint32 val;
+		char * addr;
+	} u_arg;
+	u_arg arg[MAX_NO_OF_ARG] = {{0}};
+	char *c_ptr = NULL;
+	struct bcmstrbuf b;
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+	static int log_print_count = 0;
+	static uint64 ts0 = 0;
+	uint64 ts1 = 0;
+#endif /* DHD_LOG_PRINT_RATE_LIMIT */
+
+	BCM_REFERENCE(arg);
+
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+	if (!ts0)
+		ts0 = OSL_SYSUPTIME_US();
+
+	ts1 = OSL_SYSUPTIME_US();
+
+	if (((ts1 - ts0) <= LOG_PRINT_THRESH) && (log_print_count >= MAX_LOG_PRINT_COUNT)) {
+		log_print_threshold = 1;
+		ts0 = 0;
+		log_print_count = 0;
+		DHD_ERROR(("%s: Log print water mark is reached,"
+			" console logs are dumped only to debug_dump file\n", __FUNCTION__));
+	} else if ((ts1 - ts0) > LOG_PRINT_THRESH) {
+		log_print_threshold = 0;
+		ts0 = 0;
+		log_print_count = 0;
+	}
+
+#endif /* DHD_LOG_PRINT_RATE_LIMIT */
+	/* print the message out in a logprint  */
+	if ((control_logtrace == LOGTRACE_RAW_FMT) || !(raw_event->fmts)) {
+		if (dhdp->dbg && dhdp->dbg->dbg_rings[FW_VERBOSE_RING_ID].state != RING_STOP) {
+			log_level = dhdp->dbg->dbg_rings[FW_VERBOSE_RING_ID].log_level;
+			for (id = 0; id < ARRAYSIZE(fw_verbose_level_map); id++) {
+				if ((fw_verbose_level_map[id].tag == plog_hdr->tag) &&
+					(fw_verbose_level_map[id].log_level > log_level))
+					return;
+			}
+		}
+		if (plog_hdr->binary_payload) {
+			DHD_ECNTR_LOG(("%d.%d EL:tag=%d len=%d fmt=0x%x",
+				(uint32)(log_ptr[plog_hdr->count - 1] / EL_MSEC_PER_SEC),
+				(uint32)(log_ptr[plog_hdr->count - 1] % EL_MSEC_PER_SEC),
+				plog_hdr->tag,
+				plog_hdr->count,
+				plog_hdr->fmt_num_raw));
+
+			for (count = 0; count < (plog_hdr->count - 1); count++) {
+				/* XXX: skip first line feed in case count 0 */
+				if (count && (count % LOG_PRINT_CNT_MAX == 0)) {
+					DHD_ECNTR_LOG(("\n\t%08x", log_ptr[count]));
+				} else {
+					DHD_ECNTR_LOG((" %08x", log_ptr[count]));
+				}
+			}
+			DHD_ECNTR_LOG(("\n"));
+		}
+		else {
+			bcm_binit(&b, fmtstr_loc_buf, FMTSTR_SIZE);
+			/* XXX: The 'hdr->count - 1' is dongle time */
+#ifndef OEM_ANDROID
+			bcm_bprintf(&b, "%06d.%03d EL: %d 0x%x",
+				(uint32)(log_ptr[plog_hdr->count - 1] / EL_MSEC_PER_SEC),
+				(uint32)(log_ptr[plog_hdr->count - 1] % EL_MSEC_PER_SEC),
+				plog_hdr->tag,
+				plog_hdr->fmt_num_raw);
+#else
+			bcm_bprintf(&b, "%06d.%03d EL:%s:%u:%u %d %d 0x%x",
+				(uint32)(log_ptr[plog_hdr->count - 1] / EL_MSEC_PER_SEC),
+				(uint32)(log_ptr[plog_hdr->count - 1] % EL_MSEC_PER_SEC),
+				EL_PARSE_VER, logset, block,
+				plog_hdr->tag,
+				plog_hdr->count,
+				plog_hdr->fmt_num_raw);
+#endif /* !OEM_ANDROID */
+			for (count = 0; count < (plog_hdr->count - 1); count++) {
+				bcm_bprintf(&b, " %x", log_ptr[count]);
+			}
+
+			/* ensure preserve fw logs go to debug_dump only in case of customer4 */
+			if (logset < dhdp->event_log_max_sets &&
+				((0x01ULL << logset) & dhdp->logset_prsrv_mask)) {
+				DHD_PRSRV_MEM(("%s\n", b.origbuf));
+			} else {
+				DHD_FWLOG(("%s\n", b.origbuf));
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+				log_print_count++;
+#endif /* DHD_LOG_PRINT_RATE_LIMIT */
+			}
+		}
+		return;
+	}
+
+	str_buf = MALLOCZ(dhdp->osh, (MAX_NO_OF_ARG * SIZE_LOC_STR));
+	if (!str_buf) {
+		DHD_ERROR(("%s: malloc failed str_buf\n", __FUNCTION__));
+		return;
+	}
+
+	if ((plog_hdr->fmt_num < raw_event->num_fmts) && (plog_hdr->binary_payload == FALSE)) {
+		if (plog_hdr->tag == EVENT_LOG_TAG_MSCHPROFILE) {
+			snprintf(fmtstr_loc_buf, FMTSTR_SIZE, "%s",
+				raw_event->fmts[plog_hdr->fmt_num]);
+			plog_hdr->count++;
+		} else {
+			snprintf(fmtstr_loc_buf, FMTSTR_SIZE, "CONSOLE_E:%u:%u %06d.%03d %s",
+				logset, block,
+				(uint32)(log_ptr[plog_hdr->count - 1] / EL_MSEC_PER_SEC),
+				(uint32)(log_ptr[plog_hdr->count - 1] % EL_MSEC_PER_SEC),
+				raw_event->fmts[plog_hdr->fmt_num]);
+		}
+		c_ptr = fmtstr_loc_buf;
+	} else {
+		/* for ecounters, don't print the error as it will flood */
+		if ((plog_hdr->fmt_num != DHD_OW_BI_EVENT_FMT_NUM) &&
+			(plog_hdr->fmt_num != DHD_TW_BI_EVENT_FMT_NUM)) {
+			DHD_ERROR(("%s: fmt number: 0x%x out of range\n",
+				__FUNCTION__, plog_hdr->fmt_num));
+		} else {
+			DHD_INFO(("%s: fmt number: 0x%x out of range\n",
+				__FUNCTION__, plog_hdr->fmt_num));
+		}
+
+		goto exit;
+	}
+
+	if (plog_hdr->binary_payload == FALSE && plog_hdr->count > MAX_NO_OF_ARG) {
+		DHD_INFO(("%s: plog_hdr->count(%d) out of range\n",
+			__FUNCTION__, plog_hdr->count));
+		goto exit;
+	}
+
+	/* print the format string which will be needed for debugging incorrect formats */
+	DHD_INFO(("%s: fmtstr_loc_buf = %s\n", __FUNCTION__, fmtstr_loc_buf));
+
+	/* Replace all %p to %x to handle 32 bit %p */
+	replace_percent_p_to_x(fmtstr_loc_buf);
+
+	for (count = 0; count < (plog_hdr->count - 1); count++) {
+		if (c_ptr != NULL)
+			if ((c_ptr = bcmstrstr(c_ptr, "%")) != NULL)
+				c_ptr++;
+
+		if (c_ptr != NULL) {
+			if (check_valid_string_format(c_ptr)) {
+				if ((raw_event->raw_sstr) &&
+					((log_ptr[count] > raw_event->rodata_start) &&
+					(log_ptr[count] < raw_event->rodata_end))) {
+					/* ram static string */
+					addr = log_ptr[count] - raw_event->rodata_start;
+					str_tmpptr = raw_event->raw_sstr + addr;
+					memcpy(str_buf[count], str_tmpptr,
+						SIZE_LOC_STR);
+					str_buf[count][SIZE_LOC_STR-1] = '\0';
+					arg[count].addr = str_buf[count];
+				} else if ((raw_event->rom_raw_sstr) &&
+						((log_ptr[count] >
+						raw_event->rom_rodata_start) &&
+						(log_ptr[count] <
+						raw_event->rom_rodata_end))) {
+					/* rom static string */
+					addr = log_ptr[count] - raw_event->rom_rodata_start;
+					str_tmpptr = raw_event->rom_raw_sstr + addr;
+					memcpy(str_buf[count], str_tmpptr,
+						SIZE_LOC_STR);
+					str_buf[count][SIZE_LOC_STR-1] = '\0';
+					arg[count].addr = str_buf[count];
+				} else {
+					/*
+					*  Dynamic string OR
+					* No data for static string.
+					* So store all string's address as string.
+					*/
+					snprintf(str_buf[count], SIZE_LOC_STR,
+						"(s)0x%x", log_ptr[count]);
+					arg[count].addr = str_buf[count];
+				}
+			} else if (check_valid_non_string_format(c_ptr)) {
+				/* Other than string format */
+				arg[count].val = log_ptr[count];
+			} else {
+				/* There is nothing copied after % or improper format specifier
+				 * after current %, because of not enough buffer size for complete
+				 * copy of original fmt string.
+				 * This is causing error mentioned below.
+				 * Error: "Please remove unsupported %\x00 in format string"
+				 * error(lib/vsprintf.c:1900 format_decode+0x3bc/0x470).
+				 * Refer to JIRA: SWWLAN-200629 for detailed info.
+				 *
+				 * Terminate the string at current .
+				 */
+				*(c_ptr - 1) = '\0';
+				break;
+			}
+		}
+	}
+
+	/* ensure preserve fw logs go to debug_dump only in case of customer4 */
+	if (logset < dhdp->event_log_max_sets &&
+			((0x01ULL << logset) & dhdp->logset_prsrv_mask)) {
+		if (dhd_msg_level & DHD_EVENT_VAL) {
+			if (dhd_msg_level & DHD_PRSRV_MEM_VAL)
+				printk(fmtstr_loc_buf, arg[0], arg[1], arg[2], arg[3],
+					arg[4], arg[5], arg[6], arg[7], arg[8], arg[9], arg[10],
+					arg[11], arg[12], arg[13], arg[14], arg[15]);
+		}
+	} else {
+		if (dhd_msg_level & DHD_FWLOG_VAL) {
+			printk(fmtstr_loc_buf, arg[0], arg[1], arg[2], arg[3],
+				arg[4], arg[5], arg[6], arg[7], arg[8], arg[9], arg[10],
+				arg[11], arg[12], arg[13], arg[14], arg[15]);
+		}
+#ifdef DHD_LOG_PRINT_RATE_LIMIT
+		log_print_count++;
+#endif /* DHD_LOG_PRINT_RATE_LIMIT */
+	}
+
+exit:
+	MFREE(dhdp->osh, str_buf, (MAX_NO_OF_ARG * SIZE_LOC_STR));
+}
+
+#if defined(EWP_BCM_TRACE) || defined(EWP_RTT_LOGGING) || defined(EWP_ECNTRS_LOGGING)
+static int
+dhd_dbg_send_evtlog_to_ring(prcd_event_log_hdr_t *plog_hdr,
+	dhd_dbg_ring_entry_t *msg_hdr, dhd_dbg_ring_t *ring,
+	uint16 max_payload_len, uint8 *logbuf)
+{
+	event_log_hdr_t *log_hdr;
+	struct tracelog_header *logentry_header;
+	uint16 len_chk = 0;
+
+	BCM_REFERENCE(log_hdr);
+	BCM_REFERENCE(logentry_header);
+	/*
+	 * check msg hdr len before pushing.
+	 * FW msg_hdr.len includes length of event log hdr,
+	 * logentry header and payload.
+	 */
+	len_chk = (sizeof(*logentry_header) + sizeof(*log_hdr) +
+			max_payload_len);
+	/* account extended event log header(extended_event_log_hdr) */
+	if (plog_hdr->ext_event_log_hdr) {
+		len_chk += sizeof(*log_hdr);
+	}
+	if (msg_hdr->len > len_chk) {
+		DHD_ERROR(("%s: EVENT_LOG_VALIDATION_FAILS: "
+			"msg_hdr->len=%u, max allowed for %s=%u\n",
+			__FUNCTION__, msg_hdr->len, ring->name, len_chk));
+		return BCME_ERROR;
+	}
+	dhd_dbg_ring_push(ring, msg_hdr, logbuf);
+	return BCME_OK;
+}
+#endif /* EWP_BCM_TRACE || EWP_RTT_LOGGING || EWP_ECNTRS_LOGGING */
+
+void
+dhd_dbg_msgtrace_log_parser(dhd_pub_t *dhdp, void *event_data,
+	void *raw_event_ptr, uint datalen, bool msgtrace_hdr_present,
+	uint32 msgtrace_seqnum)
+{
+	msgtrace_hdr_t *hdr;
+	char *data, *tmpdata;
+	const uint32 log_hdr_len = sizeof(event_log_hdr_t);
+	uint32 log_pyld_len;
+	static uint32 seqnum_prev = 0;
+	event_log_hdr_t *log_hdr;
+	bool msg_processed = FALSE;
+	prcd_event_log_hdr_t prcd_log_hdr;
+	prcd_event_log_hdr_t *plog_hdr;
+	dll_t list_head, *cur;
+	loglist_item_t *log_item;
+	dhd_dbg_ring_entry_t msg_hdr;
+	char *logbuf;
+	struct tracelog_header *logentry_header;
+	uint ring_data_len = 0;
+	bool ecntr_pushed = FALSE;
+	bool rtt_pushed = FALSE;
+	bool bcm_trace_pushed = FALSE;
+	bool dll_inited = FALSE;
+	uint32 logset = 0;
+	uint16 block = 0;
+	bool event_log_max_sets_queried;
+	uint32 event_log_max_sets;
+	uint min_expected_len = 0;
+	uint16 len_chk = 0;
+	uint32 datalen_bak = datalen;
+	uint16 block_hdr_len;
+
+	BCM_REFERENCE(ecntr_pushed);
+	BCM_REFERENCE(rtt_pushed);
+	BCM_REFERENCE(bcm_trace_pushed);
+	BCM_REFERENCE(len_chk);
+
+	/* store event_logset_queried and event_log_max_sets in local variables
+	 * to avoid race conditions as they were set from different contexts(preinit)
+	 */
+	event_log_max_sets_queried = dhdp->event_log_max_sets_queried;
+	/* Make sure queried is read first with wmb and then max_sets,
+	 * as it is done in reverse order during preinit ioctls.
+	 */
+	OSL_SMP_WMB();
+	event_log_max_sets = dhdp->event_log_max_sets;
+
+	if (msgtrace_hdr_present)
+		min_expected_len = (MSGTRACE_HDRLEN + EVENT_LOG_BLOCK_LEN);
+	else
+		min_expected_len = EVENT_LOG_BLOCK_LEN;
+
+	/* log trace event consists of:
+	 * msgtrace header
+	 * event log block header
+	 * event log payload
+	 */
+	if (!event_data || (datalen <= min_expected_len)) {
+		DHD_ERROR(("%s: Not processing due to invalid event_data : %p or length : %d\n",
+			__FUNCTION__, event_data, datalen));
+		if (event_data && msgtrace_hdr_present) {
+			prhex("event_data dump", event_data, datalen);
+			tmpdata = (char *)event_data + MSGTRACE_HDRLEN;
+			if (tmpdata) {
+				DHD_ERROR(("EVENT_LOG_HDR[0x%x]: Set: 0x%08x length = %d\n",
+					ltoh16(*((uint16 *)(tmpdata+2))),
+					ltoh32(*((uint32 *)(tmpdata + 4))),
+					ltoh16(*((uint16 *)(tmpdata)))));
+			}
+		} else if (!event_data) {
+			DHD_ERROR(("%s: event_data is NULL, cannot dump prhex\n", __FUNCTION__));
+		}
+
+		return;
+	}
+
+	if (msgtrace_hdr_present) {
+		hdr = (msgtrace_hdr_t *)event_data;
+		data = (char *)event_data + MSGTRACE_HDRLEN;
+		datalen -= MSGTRACE_HDRLEN;
+		msgtrace_seqnum = ntoh32(hdr->seqnum);
+	} else {
+		data = (char *)event_data;
+	}
+
+	if (dhd_dbg_msgtrace_seqchk(&seqnum_prev, msgtrace_seqnum))
+		return;
+
+	/* Save the whole message to event log ring */
+	memset(&msg_hdr, 0, sizeof(dhd_dbg_ring_entry_t));
+	logbuf = VMALLOC(dhdp->osh, sizeof(*logentry_header) + datalen);
+	if (logbuf == NULL)
+		return;
+	logentry_header = (struct tracelog_header *)logbuf;
+	logentry_header->magic_num = TRACE_LOG_MAGIC_NUMBER;
+	logentry_header->buf_size = datalen;
+	logentry_header->seq_num = msgtrace_seqnum;
+	msg_hdr.type = DBG_RING_ENTRY_DATA_TYPE;
+
+	ring_data_len = datalen + sizeof(*logentry_header);
+
+	if ((sizeof(*logentry_header) + datalen) > PAYLOAD_MAX_LEN) {
+		DHD_ERROR(("%s:Payload len=%u exceeds max len\n", __FUNCTION__,
+			((uint)sizeof(*logentry_header) + datalen)));
+		goto exit;
+	}
+
+	msg_hdr.len = sizeof(*logentry_header) + datalen;
+	memcpy(logbuf + sizeof(*logentry_header), data, datalen);
+	DHD_DBGIF(("%s: datalen %d %d\n", __FUNCTION__, msg_hdr.len, datalen));
+
+	/* Print sequence number, originating set and length of received
+	 * event log buffer. Refer to event log buffer structure in
+	 * event_log.h
+	 */
+	DHD_MSGTRACE_LOG(("EVENT_LOG_HDR[0x%x]: Set: 0x%08x length = %d\n",
+		ltoh16(*((uint16 *)(data+2))), ltoh32(*((uint32 *)(data + 4))),
+		ltoh16(*((uint16 *)(data)))));
+
+	logset = ltoh32(*((uint32 *)(data + 4)));
+	block_hdr_len = ltoh16(*((uint16 *)(data)));
+
+	if (logset >= event_log_max_sets) {
+		DHD_ERROR(("%s logset: %d max: %d out of range queried: %d\n",
+			__FUNCTION__, logset, event_log_max_sets, event_log_max_sets_queried));
+#ifdef DHD_FW_COREDUMP
+		if (event_log_max_sets_queried && !dhd_memdump_is_scheduled(dhdp)) {
+			DHD_ERROR(("%s: collect socram for DUMP_TYPE_LOGSET_BEYOND_RANGE\n",
+				__FUNCTION__));
+			dhdp->memdump_type = DUMP_TYPE_LOGSET_BEYOND_RANGE;
+			dhd_bus_mem_dump(dhdp);
+		}
+#endif /* DHD_FW_COREDUMP */
+	}
+
+	block = ltoh16(*((uint16 *)(data + 2)));
+
+	data += EVENT_LOG_BLOCK_HDRLEN;
+	datalen -= EVENT_LOG_BLOCK_HDRLEN;
+
+	/* start parsing from the tail of packet
+	 * Sameple format of a meessage
+	 * 001d3c54 00000064 00000064 001d3c54 001dba08 035d6ce1 0c540639
+	 * 001d3c54 00000064 00000064 035d6d89 0c580439
+	 * 0x0c580439 -- 39 is tag, 04 is count, 580c is format number
+	 * all these uint32 values comes in reverse order as group as EL data
+	 * while decoding we can only parse from last to first
+	 * |<-                     datalen                     ->|
+	 * |----(payload and maybe more logs)----|event_log_hdr_t|
+	 * data                                  log_hdr
+	 */
+	dll_init(&list_head);
+	dll_inited = TRUE;
+
+	while (datalen > log_hdr_len) {
+		log_hdr = (event_log_hdr_t *)(data + datalen - log_hdr_len);
+		memset(&prcd_log_hdr, 0, sizeof(prcd_log_hdr));
+		if (!dhd_dbg_process_event_log_hdr(log_hdr, &prcd_log_hdr)) {
+			DHD_ERROR(("%s: Error while parsing event log header "
+				"set:%u block:%u block_hdr_len:%d "
+				"datalen:%u cur_datalen:%u msgtrace_hdr_present:%d\n",
+				__FUNCTION__, logset, block, block_hdr_len,
+				datalen_bak, datalen, msgtrace_hdr_present));
+			dhd_prhex("[event_data]", (char*)event_data, datalen_bak,
+				DHD_ERROR_VAL);
+			break;
+		}
+
+		/* skip zero padding at end of frame */
+		if (prcd_log_hdr.tag == EVENT_LOG_TAG_NULL) {
+			datalen -= log_hdr_len;
+			continue;
+		}
+		/* Check argument count (for non-ecounter events only),
+		 * any event log should contain at least
+		 * one argument (4 bytes) for arm cycle count and up to 16
+		 * arguments except EVENT_LOG_TAG_STATS which could use the
+		 * whole payload of 256 words
+		 */
+		if (prcd_log_hdr.count == 0) {
+			break;
+		}
+		/* Both tag_stats and proxd are binary payloads so skip
+		 * argument count check for these.
+		 */
+		if ((prcd_log_hdr.tag != EVENT_LOG_TAG_STATS) &&
+			(prcd_log_hdr.tag != EVENT_LOG_TAG_PROXD_SAMPLE_COLLECT) &&
+			(prcd_log_hdr.tag != EVENT_LOG_TAG_ROAM_ENHANCED_LOG) &&
+			(prcd_log_hdr.tag != EVENT_LOG_TAG_BCM_TRACE) &&
+			(prcd_log_hdr.count > MAX_NO_OF_ARG)) {
+			break;
+		}
+
+		log_pyld_len = (prcd_log_hdr.count + prcd_log_hdr.ext_event_log_hdr) *
+			DATA_UNIT_FOR_LOG_CNT;
+		/* log data should not cross the event data boundary */
+		if ((uint32)((char *)log_hdr - data) < log_pyld_len) {
+			break;
+		}
+		/* skip 4 bytes time stamp packet */
+		if (prcd_log_hdr.tag == EVENT_LOG_TAG_TS ||
+			prcd_log_hdr.tag == EVENT_LOG_TAG_ENHANCED_TS) {
+			if (datalen >= (log_pyld_len + log_hdr_len)) {
+				datalen -= (log_pyld_len + log_hdr_len);
+			} else {
+				DHD_ERROR(("%s: invalid length : %d < %d + %d\n",
+					__FUNCTION__, datalen, log_pyld_len, log_hdr_len));
+				datalen = 0;
+			}
+			continue;
+		}
+		if (!(log_item = VMALLOC(dhdp->osh, sizeof(*log_item)))) {
+			DHD_ERROR(("%s allocating log list item failed\n",
+				__FUNCTION__));
+			break;
+		}
+
+		log_item->prcd_log_hdr.tag = prcd_log_hdr.tag;
+		log_item->prcd_log_hdr.count = prcd_log_hdr.count;
+		log_item->prcd_log_hdr.fmt_num = prcd_log_hdr.fmt_num;
+		log_item->prcd_log_hdr.fmt_num_raw = prcd_log_hdr.fmt_num_raw;
+		log_item->prcd_log_hdr.armcycle = prcd_log_hdr.armcycle;
+		log_item->prcd_log_hdr.log_ptr = prcd_log_hdr.log_ptr;
+		log_item->prcd_log_hdr.payload_len = prcd_log_hdr.payload_len;
+		log_item->prcd_log_hdr.ext_event_log_hdr = prcd_log_hdr.ext_event_log_hdr;
+		log_item->prcd_log_hdr.binary_payload = prcd_log_hdr.binary_payload;
+
+		dll_insert(&log_item->list, &list_head);
+		if (datalen >= (log_pyld_len + log_hdr_len)) {
+			datalen -= (log_pyld_len + log_hdr_len);
+		} else {
+			DHD_ERROR(("%s: invalid length : %d < %d + %d\n",
+				__FUNCTION__, datalen, log_pyld_len, log_hdr_len));
+			datalen = 0;
+		}
+	}
+
+	while (!dll_empty(&list_head)) {
+		msg_processed = FALSE;
+		cur = dll_head_p(&list_head);
+
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		log_item = (loglist_item_t *)container_of(cur, loglist_item_t, list);
+		GCC_DIAGNOSTIC_POP();
+
+		plog_hdr = &log_item->prcd_log_hdr;
+#if defined(EWP_ECNTRS_LOGGING) && defined(DHD_LOG_DUMP)
+		/* Ecounter tag can be time_data or log_stats+binary paloaod */
+		if ((plog_hdr->tag == EVENT_LOG_TAG_ECOUNTERS_TIME_DATA) ||
+			((plog_hdr->tag == EVENT_LOG_TAG_STATS) &&
+			(plog_hdr->binary_payload))) {
+			if (!ecntr_pushed && dhd_log_dump_ecntr_enabled()) {
+				if (dhd_dbg_send_evtlog_to_ring(plog_hdr, &msg_hdr,
+					dhdp->ecntr_dbg_ring,
+					EVENT_LOG_MAX_BLOCK_SIZE, logbuf) != BCME_OK) {
+					goto exit;
+				}
+				ecntr_pushed = TRUE;
+			}
+		}
+#endif /* EWP_ECNTRS_LOGGING && DHD_LOG_DUMP */
+
+		if (plog_hdr->tag == EVENT_LOG_TAG_ROAM_ENHANCED_LOG) {
+			print_roam_enhanced_log(plog_hdr);
+			msg_processed = TRUE;
+		}
+#if defined(EWP_RTT_LOGGING) && defined(DHD_LOG_DUMP)
+		if ((plog_hdr->tag == EVENT_LOG_TAG_PROXD_SAMPLE_COLLECT) &&
+				plog_hdr->binary_payload) {
+			if (!rtt_pushed && dhd_log_dump_rtt_enabled()) {
+				if (dhd_dbg_send_evtlog_to_ring(plog_hdr, &msg_hdr,
+					dhdp->rtt_dbg_ring,
+					PAYLOAD_RTT_MAX_LEN, logbuf) != BCME_OK) {
+					goto exit;
+				}
+				rtt_pushed = TRUE;
+			}
+		}
+#endif /* EWP_RTT_LOGGING && DHD_LOG_DUMP */
+
+#ifdef EWP_BCM_TRACE
+		if ((logset == EVENT_LOG_SET_BCM_TRACE) && !bcm_trace_pushed &&
+			plog_hdr->binary_payload) {
+			if (dhd_dbg_send_evtlog_to_ring(plog_hdr, &msg_hdr,
+				dhdp->bcm_trace_dbg_ring,
+				PAYLOAD_BCM_TRACE_MAX_LEN, logbuf) != BCME_OK) {
+				goto exit;
+			}
+			bcm_trace_pushed = TRUE;
+		}
+#endif /* EWP_BCM_TRACE */
+
+#if defined(DHD_EVENT_LOG_FILTER)
+		if (plog_hdr->tag == EVENT_LOG_TAG_STATS) {
+			dhd_event_log_filter_event_handler(dhdp, plog_hdr, plog_hdr->log_ptr);
+		}
+#endif /* DHD_EVENT_LOG_FILTER */
+		if (!msg_processed) {
+			dhd_dbg_verboselog_handler(dhdp, plog_hdr, raw_event_ptr,
+				logset, block, (uint32 *)data);
+		}
+		dll_delete(cur);
+		VMFREE(dhdp->osh, log_item, sizeof(*log_item));
+
+	}
+	BCM_REFERENCE(log_hdr);
+exit:
+	while (dll_inited && (!dll_empty(&list_head))) {
+		cur = dll_head_p(&list_head);
+
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		log_item = (loglist_item_t *)container_of(cur, loglist_item_t, list);
+		GCC_DIAGNOSTIC_POP();
+
+		dll_delete(cur);
+		VMFREE(dhdp->osh, log_item, sizeof(*log_item));
+	}
+
+	VMFREE(dhdp->osh, logbuf, ring_data_len);
+}
+#else /* !SHOW_LOGTRACE */
+static INLINE void dhd_dbg_verboselog_handler(dhd_pub_t *dhdp,
+	prcd_event_log_hdr_t *plog_hdr, void *raw_event_ptr, uint32 logset, uint16 block,
+	uint32 *data) {};
+INLINE void dhd_dbg_msgtrace_log_parser(dhd_pub_t *dhdp,
+	void *event_data, void *raw_event_ptr, uint datalen,
+	bool msgtrace_hdr_present, uint32 msgtrace_seqnum) {};
+#endif /* SHOW_LOGTRACE */
+void
+dhd_dbg_trace_evnt_handler(dhd_pub_t *dhdp, void *event_data,
+		void *raw_event_ptr, uint datalen)
+{
+	msgtrace_hdr_t *hdr;
+
+	hdr = (msgtrace_hdr_t *)event_data;
+
+	if (hdr->version != MSGTRACE_VERSION) {
+		DHD_DBGIF(("%s unsupported MSGTRACE version, dhd %d, dongle %d\n",
+			__FUNCTION__, MSGTRACE_VERSION, hdr->version));
+		return;
+	}
+
+	if (hdr->trace_type == MSGTRACE_HDR_TYPE_MSG)
+		dhd_dbg_msgtrace_msg_parser(event_data);
+	else if (hdr->trace_type == MSGTRACE_HDR_TYPE_LOG)
+		dhd_dbg_msgtrace_log_parser(dhdp, event_data, raw_event_ptr, datalen, TRUE, 0);
+}
+
+/*
+ * dhd_dbg_set_event_log_tag : modify the state of an event log tag
+ */
+void
+dhd_dbg_set_event_log_tag(dhd_pub_t *dhdp, uint16 tag, uint8 set_num, uint8 is_set)
+{
+	wl_el_tag_params_t pars;
+	char *cmd = "event_log_tag_control";
+	char iovbuf[WLC_IOCTL_SMLEN] = { 0 };
+	int ret;
+
+	memset(&pars, 0, sizeof(pars));
+	pars.tag = tag;
+	pars.set = set_num;
+	pars.flags = (is_set) ? EVENT_LOG_TAG_FLAG_LOG : EVENT_LOG_TAG_FLAG_NONE;
+
+	if (!bcm_mkiovar(cmd, (char *)&pars, sizeof(pars), iovbuf, sizeof(iovbuf))) {
+		DHD_ERROR(("%s mkiovar failed\n", __FUNCTION__));
+		return;
+	}
+
+	ret = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, iovbuf, sizeof(iovbuf), TRUE, 0);
+	if (ret) {
+		DHD_ERROR(("%s set log tag iovar failed %d\n", __FUNCTION__, ret));
+	}
+}
+
+int
+dhd_dbg_set_configuration(dhd_pub_t *dhdp, int ring_id, int log_level, int flags, uint32 threshold)
+{
+	dhd_dbg_ring_t *ring;
+	uint8 set = 1;
+	int i, array_len = 0;
+	struct event_log_table *log_level_tbl = NULL;
+	if (!dhdp || !dhdp->dbg)
+		return BCME_BADADDR;
+
+	if (!VALID_RING(ring_id)) {
+		DHD_ERROR(("%s : invalid ring_id : %d\n", __FUNCTION__, ring_id));
+		return BCME_RANGE;
+	}
+
+	ring = &dhdp->dbg->dbg_rings[ring_id];
+	dhd_dbg_ring_config(ring, log_level, threshold);
+
+	if (log_level > 0)
+		set = TRUE;
+
+	if (ring->id == FW_VERBOSE_RING_ID) {
+		log_level_tbl = fw_verbose_level_map;
+		array_len = ARRAYSIZE(fw_verbose_level_map);
+	}
+
+	for (i = 0; i < array_len; i++) {
+		if (log_level == 0 || (log_level_tbl[i].log_level > log_level)) {
+			/* clear the reference per ring */
+			ref_tag_tbl[log_level_tbl[i].tag] &= ~(1 << ring_id);
+		} else {
+			/* set the reference per ring */
+			ref_tag_tbl[log_level_tbl[i].tag] |= (1 << ring_id);
+		}
+		set = (ref_tag_tbl[log_level_tbl[i].tag])? 1 : 0;
+		DHD_DBGIF(("%s TAG(%s) is %s for the ring(%s)\n", __FUNCTION__,
+			log_level_tbl[i].desc, (set)? "SET" : "CLEAR", ring->name));
+		dhd_dbg_set_event_log_tag(dhdp, log_level_tbl[i].tag,
+			log_level_tbl[i].set, set);
+	}
+	return BCME_OK;
+}
+
+int
+__dhd_dbg_get_ring_status(dhd_dbg_ring_t *ring, dhd_dbg_ring_status_t *get_ring_status)
+{
+	dhd_dbg_ring_status_t ring_status;
+	int ret = BCME_OK;
+
+	if (ring == NULL) {
+		return BCME_BADADDR;
+	}
+
+	bzero(&ring_status, sizeof(dhd_dbg_ring_status_t));
+	RING_STAT_TO_STATUS(ring, ring_status);
+	*get_ring_status = ring_status;
+
+	return ret;
+}
+
+/*
+* dhd_dbg_get_ring_status : get the ring status from the coresponding ring buffer
+* Return: An error code or 0 on success.
+*/
+
+int
+dhd_dbg_get_ring_status(dhd_pub_t *dhdp, int ring_id, dhd_dbg_ring_status_t *dbg_ring_status)
+{
+	int ret = BCME_OK;
+	int id = 0;
+	dhd_dbg_t *dbg;
+	dhd_dbg_ring_t *dbg_ring;
+	if (!dhdp || !dhdp->dbg)
+		return BCME_BADADDR;
+	dbg = dhdp->dbg;
+
+	for (id = DEBUG_RING_ID_INVALID + 1; id < DEBUG_RING_ID_MAX; id++) {
+		dbg_ring = &dbg->dbg_rings[id];
+		if (VALID_RING(dbg_ring->id) && (dbg_ring->id == ring_id)) {
+			__dhd_dbg_get_ring_status(dbg_ring, dbg_ring_status);
+			break;
+		}
+	}
+	if (!VALID_RING(id)) {
+		DHD_DBGIF(("%s : cannot find the ring_id : %d\n", __FUNCTION__, ring_id));
+		ret = BCME_NOTFOUND;
+	}
+	return ret;
+}
+
+#ifdef SHOW_LOGTRACE
+void
+dhd_dbg_read_ring_into_trace_buf(dhd_dbg_ring_t *ring, trace_buf_info_t *trace_buf_info)
+{
+	dhd_dbg_ring_status_t ring_status;
+	uint32 rlen = 0;
+
+	rlen = dhd_dbg_ring_pull_single(ring, trace_buf_info->buf, TRACE_LOG_BUF_MAX_SIZE, TRUE);
+
+	trace_buf_info->size = rlen;
+	trace_buf_info->availability = NEXT_BUF_NOT_AVAIL;
+	if (rlen == 0) {
+		trace_buf_info->availability = BUF_NOT_AVAILABLE;
+		return;
+	}
+
+	__dhd_dbg_get_ring_status(ring, &ring_status);
+
+	if (ring_status.written_bytes != ring_status.read_bytes) {
+		trace_buf_info->availability = NEXT_BUF_AVAIL;
+	}
+}
+#endif /* SHOW_LOGTRACE */
+
+/*
+* dhd_dbg_find_ring_id : return ring_id based on ring_name
+* Return: An invalid ring id for failure or valid ring id on success.
+*/
+
+int
+dhd_dbg_find_ring_id(dhd_pub_t *dhdp, char *ring_name)
+{
+	int id;
+	dhd_dbg_t *dbg;
+	dhd_dbg_ring_t *ring;
+
+	if (!dhdp || !dhdp->dbg)
+		return BCME_BADADDR;
+
+	dbg = dhdp->dbg;
+	for (id = DEBUG_RING_ID_INVALID + 1; id < DEBUG_RING_ID_MAX; id++) {
+		ring = &dbg->dbg_rings[id];
+		if (!strncmp((char *)ring->name, ring_name, sizeof(ring->name) - 1))
+			break;
+	}
+	return id;
+}
+
+/*
+* dhd_dbg_get_priv : get the private data of dhd dbugability module
+* Return : An NULL on failure or valid data address
+*/
+void *
+dhd_dbg_get_priv(dhd_pub_t *dhdp)
+{
+	if (!dhdp || !dhdp->dbg)
+		return NULL;
+	return dhdp->dbg->private;
+}
+
+/*
+* dhd_dbg_start : start and stop All of Ring buffers
+* Return: An error code or 0 on success.
+*/
+int
+dhd_dbg_start(dhd_pub_t *dhdp, bool start)
+{
+	int ret = BCME_OK;
+	int ring_id;
+	dhd_dbg_t *dbg;
+	dhd_dbg_ring_t *dbg_ring;
+	if (!dhdp)
+		return BCME_BADARG;
+	dbg = dhdp->dbg;
+
+	for (ring_id = DEBUG_RING_ID_INVALID + 1; ring_id < DEBUG_RING_ID_MAX; ring_id++) {
+		dbg_ring = &dbg->dbg_rings[ring_id];
+		if (!start) {
+			if (VALID_RING(dbg_ring->id)) {
+				dhd_dbg_ring_start(dbg_ring);
+			}
+		}
+	}
+	return ret;
+}
+
+/*
+ * dhd_dbg_send_urgent_evt: send the health check evt to Upper layer
+ *
+ * Return: An error code or 0 on success.
+ */
+
+int
+dhd_dbg_send_urgent_evt(dhd_pub_t *dhdp, const void *data, const uint32 len)
+{
+	dhd_dbg_t *dbg;
+	int ret = BCME_OK;
+	if (!dhdp || !dhdp->dbg)
+		return BCME_BADADDR;
+
+	dbg = dhdp->dbg;
+	if (dbg->urgent_notifier) {
+		dbg->urgent_notifier(dhdp, data, len);
+	}
+	return ret;
+}
+
+#if defined(DBG_PKT_MON) || defined(DHD_PKT_LOGGING)
+uint32
+__dhd_dbg_pkt_hash(uintptr_t pkt, uint32 pktid)
+{
+	uint32 __pkt;
+	uint32 __pktid;
+
+	__pkt = ((int)pkt) >= 0 ? (2 * pkt) : (-2 * pkt - 1);
+	__pktid = ((int)pktid) >= 0 ? (2 * pktid) : (-2 * pktid - 1);
+
+	return (__pkt >= __pktid ? (__pkt * __pkt + __pkt + __pktid) :
+			(__pkt + __pktid * __pktid));
+}
+
+#define __TIMESPEC64_TO_US(ts) \
+	(((ts).tv_sec * USEC_PER_SEC) + ((ts).tv_nsec / NSEC_PER_USEC))
+
+uint32
+__dhd_dbg_driver_ts_usec(void)
+{
+	struct osl_timespec ts;
+
+	osl_get_monotonic_boottime(&ts);
+	return ((uint32)(__TIMESPEC64_TO_US(ts)));
+}
+
+wifi_tx_packet_fate
+__dhd_dbg_map_tx_status_to_pkt_fate(uint16 status)
+{
+	wifi_tx_packet_fate pkt_fate;
+
+	switch (status) {
+		case WLFC_CTL_PKTFLAG_DISCARD:
+			pkt_fate = TX_PKT_FATE_ACKED;
+			break;
+		case WLFC_CTL_PKTFLAG_D11SUPPRESS:
+			/* intensional fall through */
+		case WLFC_CTL_PKTFLAG_WLSUPPRESS:
+			pkt_fate = TX_PKT_FATE_FW_QUEUED;
+			break;
+		case WLFC_CTL_PKTFLAG_TOSSED_BYWLC:
+			pkt_fate = TX_PKT_FATE_FW_DROP_INVALID;
+			break;
+		case WLFC_CTL_PKTFLAG_DISCARD_NOACK:
+			pkt_fate = TX_PKT_FATE_SENT;
+			break;
+		case WLFC_CTL_PKTFLAG_EXPIRED:
+			pkt_fate = TX_PKT_FATE_FW_DROP_EXPTIME;
+			break;
+		case WLFC_CTL_PKTFLAG_DROPPED:
+			pkt_fate = TX_PKT_FATE_DRV_DROP_OTHER;
+			break;
+		case WLFC_CTL_PKTFLAG_MKTFREE:
+			pkt_fate = TX_PKT_FATE_FW_PKT_FREE;
+			break;
+		case WLFC_CTL_PKTFLAG_MAX_SUP_RETR:
+			pkt_fate = TX_PKT_FATE_FW_MAX_SUP_RETR;
+			break;
+		case WLFC_CTL_PKTFLAG_FORCED_EXPIRED:
+			pkt_fate = TX_PKT_FATE_FW_FORCED_EXPIRED;
+			break;
+		default:
+			pkt_fate = TX_PKT_FATE_FW_DROP_OTHER;
+			break;
+	}
+
+	return pkt_fate;
+}
+#endif /* DBG_PKT_MON || DHD_PKT_LOGGING */
+
+#ifdef DBG_PKT_MON
+static int do_iovar_aml_enable(dhd_pub_t *dhdp, uint val);
+static void dhd_do_aml_disable(void *handle, void *event_info, u8 event);
+void dhd_schedule_aml_disable(dhd_pub_t *dhdp);
+
+static int
+do_iovar_aml_enable(dhd_pub_t *dhdp, uint val)
+{
+	wl_aml_iovar_t *iov_in;
+	wl_aml_iov_uint_data_t *subcmd;
+	int buf_size, alloc_len, ret = BCME_OK;
+
+	buf_size = OFFSETOF(wl_aml_iovar_t, data);
+	alloc_len = buf_size + sizeof(wl_aml_iov_uint_data_t);
+
+	iov_in = MALLOCZ(dhdp->osh, alloc_len);
+	if (!iov_in) {
+		DHD_ERROR(("%s: Error allocating %u bytes for aml enable\n",
+			__FUNCTION__, alloc_len));
+		return BCME_NOMEM;
+	}
+
+	iov_in->hdr.ver = htod16(WL_AML_IOV_VERSION_1_0);
+	iov_in->hdr.len = htod16(alloc_len);
+	iov_in->hdr.subcmd = htod16(WL_AML_SUBCMD_ENABLE);
+
+	if (val & ~(1u << WL_AML_ASSOC_ENABLE | 1u << WL_AML_ROAM_ENABLE)) {
+		ret = BCME_BADARG;
+		goto fail;
+	}
+
+	subcmd = (wl_aml_iov_uint_data_t *)iov_in->data;
+	subcmd->val = htod32(val);
+
+	ret = dhd_iovar(dhdp, 0, "aml", (char *)iov_in, alloc_len, NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s aml failed %d\n", __FUNCTION__, ret));
+		ret = BCME_ERROR;
+		goto fail;
+	}
+
+fail:
+	if (iov_in) {
+		MFREE(dhdp->osh, iov_in, alloc_len);
+	}
+
+	return ret;
+}
+
+static void dhd_do_aml_disable(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = (dhd_info_t *)handle;
+	dhd_pub_t *dhdp = NULL;
+	uint val = 0; /* Disabled */
+
+	dhdp = &dhd->pub;
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (do_iovar_aml_enable(dhdp, val) == BCME_OK) {
+		dhdp->aml_enable = FALSE;
+	}
+
+	return;
+}
+
+void dhd_schedule_aml_disable(dhd_pub_t *dhdp)
+{
+	if (dhdp->dbg->pkt_mon.tx_pkt_state == PKT_MON_STOPPED &&
+			dhdp->dbg->pkt_mon.rx_pkt_state == PKT_MON_STOPPED) {
+		DHD_ERROR(("%s: scheduling aml iovar..\n", __FUNCTION__));
+		dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq, NULL,
+			DHD_WQ_WORK_AML_IOVAR, dhd_do_aml_disable, DHD_WQ_WORK_PRIORITY_HIGH);
+	}
+}
+
+static int
+__dhd_dbg_free_tx_pkts(dhd_pub_t *dhdp, dhd_dbg_tx_info_t *tx_pkts,
+	uint16 pkt_count)
+{
+	uint16 count;
+
+	count = 0;
+	while ((count < pkt_count) && tx_pkts) {
+		if (tx_pkts->info.pkt) {
+			PKTFREE(dhdp->osh, tx_pkts->info.pkt, TRUE);
+			/* Set NULL pointer after freeing for preventing dangling pointer problem */
+			tx_pkts->info.pkt = NULL;
+		}
+		tx_pkts++;
+		count++;
+	}
+
+	return BCME_OK;
+}
+
+static int
+__dhd_dbg_free_rx_pkts(dhd_pub_t *dhdp, dhd_dbg_rx_info_t *rx_pkts,
+	uint16 pkt_count)
+{
+	uint16 count;
+
+	count = 0;
+	while ((count < pkt_count) && rx_pkts) {
+		if (rx_pkts->info.pkt) {
+			PKTFREE(dhdp->osh, rx_pkts->info.pkt, TRUE);
+			/* Set NULL pointer after freeing for preventing dangling pointer problem */
+			rx_pkts->info.pkt = NULL;
+		}
+		rx_pkts++;
+		count++;
+	}
+
+	return BCME_OK;
+}
+
+void
+__dhd_dbg_dump_pkt_info(dhd_pub_t *dhdp, dhd_dbg_pkt_info_t *info)
+{
+	if (DHD_PKT_MON_DUMP_ON()) {
+		DHD_PKT_MON(("payload type   = %d\n", info->payload_type));
+		DHD_PKT_MON(("driver ts      = %u\n", info->driver_ts));
+		DHD_PKT_MON(("firmware ts    = %u\n", info->firmware_ts));
+		DHD_PKT_MON(("packet hash    = %u\n", info->pkt_hash));
+		DHD_PKT_MON(("packet length  = %zu\n", info->pkt_len));
+		DHD_PKT_MON(("packet address = %p\n", info->pkt));
+		DHD_PKT_MON(("packet data    = \n"));
+		if (DHD_PKT_MON_ON()) {
+			prhex(NULL, PKTDATA(dhdp->osh, info->pkt), info->pkt_len);
+		}
+	}
+}
+
+void
+__dhd_dbg_dump_tx_pkt_info(dhd_pub_t *dhdp, dhd_dbg_tx_info_t *tx_pkt,
+	uint16 count)
+{
+	if (DHD_PKT_MON_DUMP_ON()) {
+		DHD_PKT_MON(("\nTX (count: %d)\n", ++count));
+		DHD_PKT_MON(("packet fate    = %d\n", tx_pkt->fate));
+		__dhd_dbg_dump_pkt_info(dhdp, &tx_pkt->info);
+	}
+}
+
+void
+__dhd_dbg_dump_rx_pkt_info(dhd_pub_t *dhdp, dhd_dbg_rx_info_t *rx_pkt,
+	uint16 count)
+{
+	if (DHD_PKT_MON_DUMP_ON()) {
+		DHD_PKT_MON(("\nRX (count: %d)\n", ++count));
+		DHD_PKT_MON(("packet fate    = %d\n", rx_pkt->fate));
+		__dhd_dbg_dump_pkt_info(dhdp, &rx_pkt->info);
+	}
+}
+
+int
+dhd_dbg_attach_pkt_monitor(dhd_pub_t *dhdp,
+	dbg_mon_tx_pkts_t tx_pkt_mon,
+	dbg_mon_tx_status_t tx_status_mon,
+	dbg_mon_rx_pkts_t rx_pkt_mon)
+{
+
+	dhd_dbg_tx_report_t *tx_report = NULL;
+	dhd_dbg_rx_report_t *rx_report = NULL;
+	dhd_dbg_tx_info_t *tx_pkts = NULL;
+	dhd_dbg_rx_info_t *rx_pkts = NULL;
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+	uint32 alloc_len;
+	int ret = BCME_OK;
+	unsigned long flags;
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	tx_status_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+
+	if (PKT_MON_ATTACHED(tx_pkt_state) || PKT_MON_ATTACHED(tx_status_state) ||
+			PKT_MON_ATTACHED(rx_pkt_state)) {
+		DHD_PKT_MON(("%s(): packet monitor is already attached, "
+			"tx_pkt_state=%d, tx_status_state=%d, rx_pkt_state=%d\n",
+			__FUNCTION__, tx_pkt_state, tx_status_state, rx_pkt_state));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		/* return success as the intention was to initialize packet monitor */
+		return BCME_OK;
+	}
+
+	/* allocate and initialize tx packet monitoring */
+	alloc_len = sizeof(*tx_report);
+	tx_report = (dhd_dbg_tx_report_t *)MALLOCZ(dhdp->osh, alloc_len);
+	if (unlikely(!tx_report)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+			"dhd_dbg_tx_report_t\n", __FUNCTION__));
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	alloc_len = (sizeof(*tx_pkts) * MAX_FATE_LOG_LEN);
+	tx_pkts = (dhd_dbg_tx_info_t *)MALLOCZ(dhdp->osh, alloc_len);
+	if (unlikely(!tx_pkts)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+			"dhd_dbg_tx_info_t\n", __FUNCTION__));
+		ret = -ENOMEM;
+		goto fail;
+	}
+	dhdp->dbg->pkt_mon.tx_report = tx_report;
+	dhdp->dbg->pkt_mon.tx_report->tx_pkts = tx_pkts;
+	dhdp->dbg->pkt_mon.tx_pkt_mon = tx_pkt_mon;
+	dhdp->dbg->pkt_mon.tx_status_mon = tx_status_mon;
+	dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_ATTACHED;
+	dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_ATTACHED;
+
+	/* allocate and initialze rx packet monitoring */
+	alloc_len = sizeof(*rx_report);
+	rx_report = (dhd_dbg_rx_report_t *)MALLOCZ(dhdp->osh, alloc_len);
+	if (unlikely(!rx_report)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+			"dhd_dbg_rx_report_t\n", __FUNCTION__));
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	alloc_len = (sizeof(*rx_pkts) * MAX_FATE_LOG_LEN);
+	rx_pkts = (dhd_dbg_rx_info_t *)MALLOCZ(dhdp->osh, alloc_len);
+	if (unlikely(!rx_pkts)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+			"dhd_dbg_rx_info_t\n", __FUNCTION__));
+		ret = -ENOMEM;
+		goto fail;
+	}
+	dhdp->dbg->pkt_mon.rx_report = rx_report;
+	dhdp->dbg->pkt_mon.rx_report->rx_pkts = rx_pkts;
+	dhdp->dbg->pkt_mon.rx_pkt_mon = rx_pkt_mon;
+	dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_ATTACHED;
+
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+	DHD_PKT_MON(("%s(): packet monitor attach succeeded\n", __FUNCTION__));
+	return ret;
+
+fail:
+	/* tx packet monitoring */
+	if (tx_pkts) {
+		alloc_len = (sizeof(*tx_pkts) * MAX_FATE_LOG_LEN);
+		MFREE(dhdp->osh, tx_pkts, alloc_len);
+	}
+	if (tx_report) {
+		alloc_len = sizeof(*tx_report);
+		MFREE(dhdp->osh, tx_report, alloc_len);
+	}
+	dhdp->dbg->pkt_mon.tx_report = NULL;
+	dhdp->dbg->pkt_mon.tx_report->tx_pkts = NULL;
+	dhdp->dbg->pkt_mon.tx_pkt_mon = NULL;
+	dhdp->dbg->pkt_mon.tx_status_mon = NULL;
+	dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_DETACHED;
+	dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_DETACHED;
+
+	/* rx packet monitoring */
+	if (rx_pkts) {
+		alloc_len = (sizeof(*rx_pkts) * MAX_FATE_LOG_LEN);
+		MFREE(dhdp->osh, rx_pkts, alloc_len);
+	}
+	if (rx_report) {
+		alloc_len = sizeof(*rx_report);
+		MFREE(dhdp->osh, rx_report, alloc_len);
+	}
+	dhdp->dbg->pkt_mon.rx_report = NULL;
+	dhdp->dbg->pkt_mon.rx_report->rx_pkts = NULL;
+	dhdp->dbg->pkt_mon.rx_pkt_mon = NULL;
+	dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_DETACHED;
+
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+	DHD_ERROR(("%s(): packet monitor attach failed\n", __FUNCTION__));
+	return ret;
+}
+
+int
+dhd_dbg_start_pkt_monitor(dhd_pub_t *dhdp)
+{
+	dhd_dbg_tx_report_t *tx_report;
+	dhd_dbg_rx_report_t *rx_report;
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+	unsigned long flags;
+#ifdef DBG_PKT_MON_ROAM
+	/* assoc mgmt logging for assoc/roam is allowed by default */
+	uint32 enable = (1u << WL_AML_ROAM_ENABLE | 1u << WL_AML_ASSOC_ENABLE);
+#else
+	/* assoc mgmt logging for assoc is allowed by default */
+	uint32 enable = (1u << WL_AML_ASSOC_ENABLE);
+#endif
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	if (do_iovar_aml_enable(dhdp, enable) == BCME_OK) {
+		dhdp->aml_enable = TRUE;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	tx_status_state = dhdp->dbg->pkt_mon.tx_status_state;
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+
+	if (PKT_MON_DETACHED(tx_pkt_state) || PKT_MON_DETACHED(tx_status_state) ||
+			PKT_MON_DETACHED(rx_pkt_state)) {
+		DHD_PKT_MON(("%s(): packet monitor is not yet enabled, "
+			"tx_pkt_state=%d, tx_status_state=%d, rx_pkt_state=%d\n",
+			__FUNCTION__, tx_pkt_state, tx_status_state, rx_pkt_state));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		return -EINVAL;
+	}
+
+	dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_STARTING;
+	dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_STARTING;
+	dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_STARTING;
+
+	tx_report = dhdp->dbg->pkt_mon.tx_report;
+	rx_report = dhdp->dbg->pkt_mon.rx_report;
+	if (!tx_report || !rx_report) {
+		DHD_PKT_MON(("%s(): tx_report=%p, rx_report=%p\n",
+			__FUNCTION__, tx_report, rx_report));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		return -EINVAL;
+	}
+
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	tx_status_state = dhdp->dbg->pkt_mon.tx_status_state;
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+
+	/* Safe to free packets as state pkt_state is STARTING */
+	__dhd_dbg_free_tx_pkts(dhdp, tx_report->tx_pkts, tx_report->pkt_pos);
+
+	__dhd_dbg_free_rx_pkts(dhdp, rx_report->rx_pkts, rx_report->pkt_pos);
+
+	/* reset array postion */
+	tx_report->pkt_pos = 0;
+	tx_report->status_pos = 0;
+	dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_STARTED;
+	dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_STARTED;
+
+	rx_report->pkt_pos = 0;
+	dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_STARTED;
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+
+	DHD_PKT_MON(("%s(): packet monitor started\n", __FUNCTION__));
+	return BCME_OK;
+}
+
+int
+dhd_dbg_monitor_tx_pkts(dhd_pub_t *dhdp, void *pkt, uint32 pktid, frame_type type, uint8 mgmt_acked)
+{
+	dhd_dbg_tx_report_t *tx_report;
+	dhd_dbg_tx_info_t *tx_pkts;
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	uint32 pkt_hash, driver_ts;
+	uint16 pkt_pos;
+	unsigned long flags;
+#if !defined(PCIE_FULL_DONGLE)
+	void *clone_pkt = NULL;
+#endif
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	if (PKT_MON_STARTED(tx_pkt_state)) {
+		tx_report = dhdp->dbg->pkt_mon.tx_report;
+		pkt_pos = tx_report->pkt_pos;
+
+		if (!PKT_MON_PKT_FULL(pkt_pos)) {
+			tx_pkts = tx_report->tx_pkts;
+			pkt_hash = __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
+			driver_ts = __dhd_dbg_driver_ts_usec();
+
+			if (type == FRAME_TYPE_80211_MGMT) {
+				tx_pkts[pkt_pos].info.pkt = pkt;
+				if (mgmt_acked) {
+					tx_pkts[pkt_pos].fate = TX_PKT_FATE_ACKED;
+				} else {
+					tx_pkts[pkt_pos].fate = TX_PKT_FATE_SENT;
+				}
+				tx_pkts[pkt_pos].info.pkt_len = PKTLEN(dhdp->osh, pkt);
+			} else {
+#if !defined(PCIE_FULL_DONGLE)
+				clone_pkt = PKTDUP(dhdp->osh, pkt);
+				skb_pull((struct sk_buff*)clone_pkt, SDIO_HLEN);
+				tx_pkts[pkt_pos].info.pkt = clone_pkt;
+				tx_pkts[pkt_pos].info.pkt_len = PKTLEN(dhdp->osh, clone_pkt) - SDIO_HLEN;
+#else
+				tx_pkts[pkt_pos].info.pkt = PKTDUP(dhdp->osh, pkt);
+				tx_pkts[pkt_pos].info.pkt_len = PKTLEN(dhdp->osh, pkt);
+#endif
+				tx_pkts[pkt_pos].fate = TX_PKT_FATE_DRV_QUEUED;
+			}
+
+			tx_pkts[pkt_pos].info.pkt_hash = pkt_hash;
+			tx_pkts[pkt_pos].info.driver_ts = driver_ts;
+			tx_pkts[pkt_pos].info.firmware_ts = 0U;
+			tx_pkts[pkt_pos].info.payload_type = type;
+
+			tx_report->pkt_pos++;
+			/* TX mgmt packet is updated with the final fate reason,
+			 * so status_pos should be also increased.
+			 */
+			if (type == FRAME_TYPE_80211_MGMT) {
+				tx_report->status_pos++;
+			}
+		} else {
+			if (type == FRAME_TYPE_80211_MGMT) {
+				PKTFREE(dhdp->osh, pkt, TRUE);
+			}
+			dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_STOPPED;
+			DHD_PKT_MON(("%s(): tx pkt logging stopped, reached "
+				"max limit\n", __FUNCTION__));
+			if (dhdp->aml_enable) {
+				dhd_schedule_aml_disable(dhdp);
+			}
+		}
+	}
+
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+	return BCME_OK;
+}
+
+int
+dhd_dbg_monitor_tx_status(dhd_pub_t *dhdp, void *pkt, uint32 pktid,
+		uint16 status)
+{
+	dhd_dbg_tx_report_t *tx_report;
+	dhd_dbg_tx_info_t *tx_pkt;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	wifi_tx_packet_fate pkt_fate;
+	uint32 pkt_hash, temp_hash;
+	uint16 pkt_pos, status_pos;
+	int16 count;
+	bool found = FALSE;
+	unsigned long flags;
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_status_state = dhdp->dbg->pkt_mon.tx_status_state;
+	if (PKT_MON_STARTED(tx_status_state)) {
+		tx_report = dhdp->dbg->pkt_mon.tx_report;
+		pkt_pos = tx_report->pkt_pos;
+		status_pos = tx_report->status_pos;
+
+		if (!PKT_MON_STATUS_FULL(pkt_pos, status_pos)) {
+			pkt_hash = __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
+			pkt_fate = __dhd_dbg_map_tx_status_to_pkt_fate(status);
+
+			/* best bet (in-order tx completion) */
+			count = status_pos;
+			tx_pkt = (((dhd_dbg_tx_info_t *)tx_report->tx_pkts) + status_pos);
+			while ((count < pkt_pos) && tx_pkt) {
+				temp_hash = tx_pkt->info.pkt_hash;
+				if (temp_hash == pkt_hash) {
+					tx_pkt->fate = pkt_fate;
+					tx_report->status_pos++;
+					found = TRUE;
+					break;
+				}
+				tx_pkt++;
+				count++;
+			}
+
+			/* search until beginning (handles out-of-order completion) */
+			if (!found) {
+				count = status_pos - 1;
+				tx_pkt = (((dhd_dbg_tx_info_t *)tx_report->tx_pkts) + count);
+				while ((count >= 0) && tx_pkt) {
+					temp_hash = tx_pkt->info.pkt_hash;
+					if (temp_hash == pkt_hash) {
+						tx_pkt->fate = pkt_fate;
+						tx_report->status_pos++;
+						found = TRUE;
+						break;
+					}
+					tx_pkt--;
+					count--;
+				}
+
+				if (!found) {
+					/* still couldn't match tx_status */
+					DHD_INFO(("%s(): couldn't match tx_status, pkt_pos=%u, "
+						"status_pos=%u, pkt_fate=%u\n", __FUNCTION__,
+						pkt_pos, status_pos, pkt_fate));
+				}
+			}
+		} else {
+			dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_STOPPED;
+			DHD_PKT_MON(("%s(): tx_status logging stopped, reached "
+				"max limit\n", __FUNCTION__));
+		}
+	}
+
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+	return BCME_OK;
+}
+
+int
+dhd_dbg_monitor_rx_pkts(dhd_pub_t *dhdp, void *pkt, frame_type type)
+{
+	dhd_dbg_rx_report_t *rx_report;
+	dhd_dbg_rx_info_t *rx_pkts;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+	uint32 driver_ts;
+	uint16 pkt_pos;
+	unsigned long flags;
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+	if (PKT_MON_STARTED(rx_pkt_state)) {
+		rx_report = dhdp->dbg->pkt_mon.rx_report;
+		pkt_pos = rx_report->pkt_pos;
+
+		if (!PKT_MON_PKT_FULL(pkt_pos)) {
+			rx_pkts = rx_report->rx_pkts;
+			driver_ts = __dhd_dbg_driver_ts_usec();
+
+			if (type == FRAME_TYPE_80211_MGMT) {
+				rx_pkts[pkt_pos].info.pkt = pkt;
+			} else {
+				rx_pkts[pkt_pos].info.pkt = PKTDUP(dhdp->osh, pkt);
+			}
+			rx_pkts[pkt_pos].info.pkt_len = PKTLEN(dhdp->osh, pkt);
+			rx_pkts[pkt_pos].info.pkt_hash = 0U;
+			rx_pkts[pkt_pos].info.driver_ts = driver_ts;
+			rx_pkts[pkt_pos].info.firmware_ts = 0U;
+			rx_pkts[pkt_pos].info.payload_type = type;
+			rx_pkts[pkt_pos].fate = RX_PKT_FATE_SUCCESS;
+
+			rx_report->pkt_pos++;
+		} else {
+			if (type == FRAME_TYPE_80211_MGMT) {
+				PKTFREE(dhdp->osh, pkt, TRUE);
+			}
+			dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_STOPPED;
+			DHD_PKT_MON(("%s(): rx pkt logging stopped, reached "
+					"max limit\n", __FUNCTION__));
+			if (dhdp->aml_enable) {
+				dhd_schedule_aml_disable(dhdp);
+			}
+		}
+	}
+
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+	return BCME_OK;
+}
+
+int
+dhd_dbg_stop_pkt_monitor(dhd_pub_t *dhdp)
+{
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+	unsigned long flags;
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	tx_status_state = dhdp->dbg->pkt_mon.tx_status_state;
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+
+	if (PKT_MON_DETACHED(tx_pkt_state) || PKT_MON_DETACHED(tx_status_state) ||
+			PKT_MON_DETACHED(rx_pkt_state)) {
+		DHD_PKT_MON(("%s(): packet monitor is not yet enabled, "
+			"tx_pkt_state=%d, tx_status_state=%d, rx_pkt_state=%d\n",
+			__FUNCTION__, tx_pkt_state, tx_status_state, rx_pkt_state));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		return -EINVAL;
+	}
+	dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_STOPPED;
+	dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_STOPPED;
+	dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_STOPPED;
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+
+	DHD_PKT_MON(("%s(): packet monitor stopped\n", __FUNCTION__));
+	return BCME_OK;
+}
+
+#define __COPY_TO_USER(to, from, n) \
+	do { \
+		int __ret; \
+		__ret = copy_to_user((void __user *)(to), (void *)(from), \
+				(unsigned long)(n)); \
+		if (unlikely(__ret)) { \
+			DHD_ERROR(("%s():%d: copy_to_user failed, ret=%d\n", \
+				__FUNCTION__, __LINE__, __ret)); \
+			return __ret; \
+		} \
+	} while (0);
+
+static wifi_tx_packet_fate
+__dhd_dbg_convert_fate(wifi_tx_packet_fate fate)
+{
+	wifi_tx_packet_fate new_fate = fate;
+
+	 /* To prevent SIG-ABORT, packet_fate > TX_PKT_FATE_DRV_DROP_OTHER */
+	if (fate > TX_PKT_FATE_DRV_DROP_OTHER) {
+		new_fate = TX_PKT_FATE_FW_DROP_OTHER;
+	}
+
+	return new_fate;
+}
+
+int
+dhd_dbg_monitor_get_tx_pkts(dhd_pub_t *dhdp, void __user *user_buf,
+		uint16 req_count, uint16 *resp_count)
+{
+	dhd_dbg_tx_report_t *tx_report;
+	dhd_dbg_tx_info_t *tx_pkt, *ori_tx_pkt;
+	wifi_tx_report_t *ptr;
+	compat_wifi_tx_report_t *cptr;
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	uint16 pkt_count, count;
+	unsigned long flags;
+	dhd_dbg_tx_info_t *tmp_tx_pkt = NULL;
+	uint32 alloc_len, i, ret;
+
+	BCM_REFERENCE(ptr);
+	BCM_REFERENCE(cptr);
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	tx_status_state = dhdp->dbg->pkt_mon.tx_status_state;
+	if (!PKT_MON_ATTACHED(tx_pkt_state) ||
+			!PKT_MON_ATTACHED(tx_status_state)) {
+		DHD_PKT_MON(("%s(): packet monitor is not yet enabled, "
+			"tx_pkt_state=%d, tx_status_state=%d\n", __FUNCTION__,
+			tx_pkt_state, tx_status_state));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		return -EINVAL;
+	}
+
+	count = 0;
+	tx_report = dhdp->dbg->pkt_mon.tx_report;
+	ori_tx_pkt = tx_report->tx_pkts;
+	pkt_count = MIN(req_count, tx_report->status_pos);
+
+	alloc_len = (sizeof(*tmp_tx_pkt) * pkt_count);
+	tmp_tx_pkt = (dhd_dbg_tx_info_t *)MALLOCZ(dhdp->osh, alloc_len);
+	if (unlikely(!tmp_tx_pkt)) {
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		DHD_ERROR(("%s: failed to allocate tmp_tx_pkt", __FUNCTION__));
+		return -ENOMEM;
+	}
+	if ((ret = memcpy_s(tmp_tx_pkt, alloc_len, ori_tx_pkt, alloc_len))) {
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		DHD_ERROR(("%s: failed to copy tmp_tx_pkt ret:%d", __FUNCTION__, ret));
+		return -EINVAL;
+	}
+	for (i = 0; i < pkt_count; i++) {
+		tmp_tx_pkt[i].info.pkt = skb_copy((struct sk_buff*)ori_tx_pkt[i].info.pkt,
+			GFP_ATOMIC);
+		if (!tmp_tx_pkt[i].info.pkt) {
+			DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+			DHD_ERROR(("%s: failed to copy skb", __FUNCTION__));
+			return -ENOMEM;
+		}
+	}
+	tx_pkt = tmp_tx_pkt;
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+
+#ifdef CONFIG_COMPAT
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0))
+	if (in_compat_syscall())
+#else
+	if (is_compat_task())
+#endif /* LINUX_VER >= 4.6 */
+	{
+		cptr = (compat_wifi_tx_report_t *)user_buf;
+		while ((count < pkt_count) && tx_pkt && cptr) {
+			compat_wifi_tx_report_t *comp_ptr = compat_ptr((uintptr_t) cptr);
+			compat_dhd_dbg_pkt_info_t compat_tx_pkt;
+			__dhd_dbg_dump_tx_pkt_info(dhdp, tx_pkt, count);
+			/* fate convert asscording to wifi_logger.h */
+			{
+				wifi_tx_packet_fate new_fate = tx_pkt->fate;
+				new_fate = __dhd_dbg_convert_fate(new_fate);
+				__COPY_TO_USER(&comp_ptr->fate, &new_fate, sizeof(new_fate));
+			}
+
+			compat_tx_pkt.payload_type = tx_pkt->info.payload_type;
+			compat_tx_pkt.pkt_len = tx_pkt->info.pkt_len;
+			compat_tx_pkt.driver_ts = tx_pkt->info.driver_ts;
+			compat_tx_pkt.firmware_ts = tx_pkt->info.firmware_ts;
+			compat_tx_pkt.pkt_hash = tx_pkt->info.pkt_hash;
+			__COPY_TO_USER(&comp_ptr->frame_inf.payload_type,
+				&compat_tx_pkt.payload_type,
+				OFFSETOF(compat_dhd_dbg_pkt_info_t, pkt_hash));
+			__COPY_TO_USER(comp_ptr->frame_inf.frame_content.ethernet_ii,
+				PKTDATA(dhdp->osh, tx_pkt->info.pkt), tx_pkt->info.pkt_len);
+
+			cptr++;
+			tx_pkt++;
+			count++;
+		}
+	} else
+#endif /* CONFIG_COMPAT */
+	{
+		ptr = (wifi_tx_report_t *)user_buf;
+		while ((count < pkt_count) && tx_pkt && ptr) {
+			__dhd_dbg_dump_tx_pkt_info(dhdp, tx_pkt, count);
+			/* fate convert asscording to wifi_logger.h */
+			{
+				wifi_tx_packet_fate new_fate = tx_pkt->fate;
+				new_fate = __dhd_dbg_convert_fate(new_fate);
+				__COPY_TO_USER(&ptr->fate, &new_fate, sizeof(new_fate));
+			}
+			__COPY_TO_USER(&ptr->frame_inf.payload_type,
+				&tx_pkt->info.payload_type,
+				OFFSETOF(dhd_dbg_pkt_info_t, pkt_hash));
+			__COPY_TO_USER(ptr->frame_inf.frame_content.ethernet_ii,
+				PKTDATA(dhdp->osh, tx_pkt->info.pkt), tx_pkt->info.pkt_len);
+
+			ptr++;
+			tx_pkt++;
+			count++;
+		}
+	}
+	*resp_count = pkt_count;
+
+	if (!pkt_count) {
+		DHD_ERROR(("%s(): no tx_status in tx completion messages, "
+			"make sure that 'd11status' is enabled in firmware, "
+			"status_pos=%u\n", __FUNCTION__, pkt_count));
+	}
+
+	for (i = 0; i < pkt_count; i++) {
+		PKTFREE(dhdp->osh, tmp_tx_pkt[i].info.pkt, TRUE);
+	}
+	MFREE(dhdp->osh, tmp_tx_pkt, alloc_len);
+
+	return BCME_OK;
+}
+
+int
+dhd_dbg_monitor_get_rx_pkts(dhd_pub_t *dhdp, void __user *user_buf,
+		uint16 req_count, uint16 *resp_count)
+{
+	dhd_dbg_rx_report_t *rx_report;
+	dhd_dbg_rx_info_t *rx_pkt, *ori_rx_pkt;
+	wifi_rx_report_t *ptr;
+	compat_wifi_rx_report_t *cptr;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+	uint16 pkt_count, count;
+	unsigned long flags;
+	dhd_dbg_rx_info_t *tmp_rx_pkt = NULL;
+	uint32 alloc_len, i, ret = BCME_OK;
+
+	BCM_REFERENCE(ptr);
+	BCM_REFERENCE(cptr);
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+	if (!PKT_MON_ATTACHED(rx_pkt_state)) {
+		DHD_PKT_MON(("%s(): packet fetch is not allowed , "
+			"rx_pkt_state=%d\n", __FUNCTION__, rx_pkt_state));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		return -EINVAL;
+	}
+
+	count = 0;
+	rx_report = dhdp->dbg->pkt_mon.rx_report;
+	ori_rx_pkt = rx_report->rx_pkts;
+	pkt_count = MIN(req_count, rx_report->pkt_pos);
+
+	alloc_len = (sizeof(*tmp_rx_pkt) * pkt_count);
+	tmp_rx_pkt = (dhd_dbg_rx_info_t *)MALLOCZ(dhdp->osh, alloc_len);
+	if (unlikely(!tmp_rx_pkt)) {
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		DHD_ERROR(("%s: failed to allocate tmp_rx_pkt", __FUNCTION__));
+		return -ENOMEM;
+	}
+	if ((ret = memcpy_s(tmp_rx_pkt, alloc_len, ori_rx_pkt, alloc_len))) {
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		DHD_ERROR(("%s: failed to copy tmp_rx_pkt ret:%d", __FUNCTION__, ret));
+		goto exit;
+	}
+	for (i = 0; i < pkt_count; i++) {
+		tmp_rx_pkt[i].info.pkt = skb_copy((struct sk_buff*)ori_rx_pkt[i].info.pkt,
+			GFP_ATOMIC);
+		if (!tmp_rx_pkt[i].info.pkt) {
+			DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+			DHD_ERROR(("%s: failed to copy skb", __FUNCTION__));
+			goto exit;
+		}
+	}
+	rx_pkt = tmp_rx_pkt;
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+
+#ifdef CONFIG_COMPAT
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0))
+	if (in_compat_syscall())
+#else
+	if (is_compat_task())
+#endif /* LINUX_VER >= 4.6 */
+	{
+		cptr = (compat_wifi_rx_report_t *)user_buf;
+		while ((count < pkt_count) && rx_pkt && cptr) {
+			compat_wifi_rx_report_t *comp_ptr = compat_ptr((uintptr_t) cptr);
+			compat_dhd_dbg_pkt_info_t compat_rx_pkt;
+			__dhd_dbg_dump_rx_pkt_info(dhdp, rx_pkt, count);
+			__COPY_TO_USER(&comp_ptr->fate, &rx_pkt->fate, sizeof(rx_pkt->fate));
+
+			compat_rx_pkt.payload_type = rx_pkt->info.payload_type;
+			compat_rx_pkt.pkt_len = rx_pkt->info.pkt_len;
+			compat_rx_pkt.driver_ts = rx_pkt->info.driver_ts;
+			compat_rx_pkt.firmware_ts = rx_pkt->info.firmware_ts;
+			compat_rx_pkt.pkt_hash = rx_pkt->info.pkt_hash;
+			__COPY_TO_USER(&comp_ptr->frame_inf.payload_type,
+				&compat_rx_pkt.payload_type,
+				OFFSETOF(compat_dhd_dbg_pkt_info_t, pkt_hash));
+			__COPY_TO_USER(comp_ptr->frame_inf.frame_content.ethernet_ii,
+				PKTDATA(dhdp->osh, rx_pkt->info.pkt), rx_pkt->info.pkt_len);
+
+			cptr++;
+			rx_pkt++;
+			count++;
+		}
+	} else
+#endif /* CONFIG_COMPAT */
+	{
+		ptr = (wifi_rx_report_t *)user_buf;
+		while ((count < pkt_count) && rx_pkt && ptr) {
+			__dhd_dbg_dump_rx_pkt_info(dhdp, rx_pkt, count);
+			__COPY_TO_USER(&ptr->fate, &rx_pkt->fate, sizeof(rx_pkt->fate));
+			__COPY_TO_USER(&ptr->frame_inf.payload_type,
+				&rx_pkt->info.payload_type,
+				OFFSETOF(dhd_dbg_pkt_info_t, pkt_hash));
+			__COPY_TO_USER(ptr->frame_inf.frame_content.ethernet_ii,
+				PKTDATA(dhdp->osh, rx_pkt->info.pkt), rx_pkt->info.pkt_len);
+
+			ptr++;
+			rx_pkt++;
+			count++;
+		}
+	}
+
+	*resp_count = pkt_count;
+
+exit:
+	for (i = 0; i < pkt_count; i++) {
+		PKTFREE(dhdp->osh, tmp_rx_pkt[i].info.pkt, TRUE);
+	}
+	MFREE(dhdp->osh, tmp_rx_pkt, alloc_len);
+
+	return ret;
+}
+
+int
+dhd_dbg_detach_pkt_monitor(dhd_pub_t *dhdp)
+{
+	dhd_dbg_tx_report_t *tx_report;
+	dhd_dbg_rx_report_t *rx_report;
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+	unsigned long flags;
+
+	if (!dhdp || !dhdp->dbg) {
+		DHD_PKT_MON(("%s(): dhdp=%p, dhdp->dbg=%p\n", __FUNCTION__,
+			dhdp, (dhdp ? dhdp->dbg : NULL)));
+		return -EINVAL;
+	}
+
+	DHD_PKT_MON_LOCK(dhdp->dbg->pkt_mon_lock, flags);
+	tx_pkt_state = dhdp->dbg->pkt_mon.tx_pkt_state;
+	tx_status_state = dhdp->dbg->pkt_mon.tx_status_state;
+	rx_pkt_state = dhdp->dbg->pkt_mon.rx_pkt_state;
+
+	if (PKT_MON_DETACHED(tx_pkt_state) || PKT_MON_DETACHED(tx_status_state) ||
+			PKT_MON_DETACHED(rx_pkt_state)) {
+		DHD_PKT_MON(("%s(): packet monitor is already detached, "
+			"tx_pkt_state=%d, tx_status_state=%d, rx_pkt_state=%d\n",
+			__FUNCTION__, tx_pkt_state, tx_status_state, rx_pkt_state));
+		DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+		return -EINVAL;
+	}
+
+	tx_report = dhdp->dbg->pkt_mon.tx_report;
+	rx_report = dhdp->dbg->pkt_mon.rx_report;
+
+	/* free and de-initalize tx packet monitoring */
+	dhdp->dbg->pkt_mon.tx_pkt_state = PKT_MON_DETACHED;
+	dhdp->dbg->pkt_mon.tx_status_state = PKT_MON_DETACHED;
+	if (tx_report) {
+		if (tx_report->tx_pkts) {
+			__dhd_dbg_free_tx_pkts(dhdp, tx_report->tx_pkts,
+				tx_report->pkt_pos);
+			MFREE(dhdp->osh, tx_report->tx_pkts,
+				(sizeof(*tx_report->tx_pkts) * MAX_FATE_LOG_LEN));
+			dhdp->dbg->pkt_mon.tx_report->tx_pkts = NULL;
+		}
+		MFREE(dhdp->osh, tx_report, sizeof(*tx_report));
+		dhdp->dbg->pkt_mon.tx_report = NULL;
+	}
+	dhdp->dbg->pkt_mon.tx_pkt_mon = NULL;
+	dhdp->dbg->pkt_mon.tx_status_mon = NULL;
+
+	/* free and de-initalize rx packet monitoring */
+	dhdp->dbg->pkt_mon.rx_pkt_state = PKT_MON_DETACHED;
+	if (rx_report) {
+		if (rx_report->rx_pkts) {
+			__dhd_dbg_free_rx_pkts(dhdp, rx_report->rx_pkts,
+				rx_report->pkt_pos);
+			MFREE(dhdp->osh, rx_report->rx_pkts,
+				(sizeof(*rx_report->rx_pkts) * MAX_FATE_LOG_LEN));
+			dhdp->dbg->pkt_mon.rx_report->rx_pkts = NULL;
+		}
+		MFREE(dhdp->osh, rx_report, sizeof(*rx_report));
+		dhdp->dbg->pkt_mon.rx_report = NULL;
+	}
+	dhdp->dbg->pkt_mon.rx_pkt_mon = NULL;
+
+	DHD_PKT_MON_UNLOCK(dhdp->dbg->pkt_mon_lock, flags);
+	DHD_PKT_MON(("%s(): packet monitor detach succeeded\n", __FUNCTION__));
+	return BCME_OK;
+}
+#endif /* DBG_PKT_MON */
+
+#if defined(DBG_PKT_MON) || defined(DHD_PKT_LOGGING)
+/*
+ * XXX: WAR: Because of the overloading by DMA marker field,
+ * tx_status in TX completion message cannot be used. As a WAR,
+ * send d11 tx_status through unused status field of PCIe
+ * completion header.
+ */
+bool
+dhd_dbg_process_tx_status(dhd_pub_t *dhdp, void *pkt, uint32 pktid,
+		uint16 status)
+{
+	bool pkt_fate = TRUE;
+	if (dhdp->d11_tx_status) {
+		pkt_fate = (status == WLFC_CTL_PKTFLAG_DISCARD) ? TRUE : FALSE;
+		DHD_DBG_PKT_MON_TX_STATUS(dhdp, pkt, pktid, status);
+	}
+	return pkt_fate;
+}
+#else /* DBG_PKT_MON || DHD_PKT_LOGGING */
+bool
+dhd_dbg_process_tx_status(dhd_pub_t *dhdp, void *pkt,
+		uint32 pktid, uint16 status)
+{
+	return TRUE;
+}
+#endif /* DBG_PKT_MON || DHD_PKT_LOGGING */
+
+#define	EL_LOG_STR_LEN	512
+
+#define PRINT_CHN_PER_LINE 8
+#define PRINT_CHAN_LINE(cnt) \
+{\
+	cnt ++; \
+	if (cnt >= PRINT_CHN_PER_LINE) { \
+		DHD_ERROR(("%s\n", b.origbuf)); \
+		bcm_binit(&b, pr_buf, EL_LOG_STR_LEN); \
+		bcm_bprintf(&b, "%s: ", prefix); \
+		cnt = 0; \
+	} \
+}
+
+void print_roam_chan_list(char *prefix, uint chan_num, uint16 band_2g,
+	uint16 uni2a, uint8 uni3, uint8 *uni2c)
+{
+	struct bcmstrbuf b;
+	char pr_buf[EL_LOG_STR_LEN] = { 0 };
+	int cnt = 0;
+	int idx, idx2;
+
+	bcm_binit(&b, pr_buf, EL_LOG_STR_LEN);
+	bcm_bprintf(&b, "%s: count(%d)", prefix, chan_num);
+	/* 2G channnels */
+	for (idx = 0; idx < NBITS(uint16); idx++) {
+		if (BCM_BIT(idx) & band_2g) {
+			bcm_bprintf(&b, " %d", idx);
+			PRINT_CHAN_LINE(cnt);
+
+		}
+	}
+
+	/* 5G UNII BAND 1, UNII BAND 2A */
+	for (idx = 0; idx < NBITS(uint16); idx++) {
+		if (BCM_BIT(idx) & uni2a) {
+			bcm_bprintf(&b, " %u", ROAM_CHN_UNI_2A + idx * ROAM_CHN_SPACE);
+			PRINT_CHAN_LINE(cnt);
+		}
+	}
+
+	/* 5G UNII BAND 2C */
+	for (idx2 = 0; idx2 < 3; idx2++) {
+		for (idx = 0; idx < NBITS(uint8); idx++) {
+			if (BCM_BIT(idx) & uni2c[idx2]) {
+				bcm_bprintf(&b, " %u", ROAM_CHN_UNI_2C +
+					idx2 * ROAM_CHN_SPACE * NBITS(uint8) +
+					idx * ROAM_CHN_SPACE);
+				PRINT_CHAN_LINE(cnt);
+			}
+		}
+	}
+
+	/* 5G UNII BAND 3 */
+	for (idx = 0; idx < NBITS(uint8); idx++) {
+		if (BCM_BIT(idx) & uni3) {
+			bcm_bprintf(&b, " %u", ROAM_CHN_UNI_3 + idx * ROAM_CHN_SPACE);
+			PRINT_CHAN_LINE(cnt);
+		}
+	}
+
+	if (cnt != 0) {
+		DHD_ERROR(("%s\n", b.origbuf));
+	}
+}
+
+void pr_roam_scan_start_v1(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_scan_cmpl_v1(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_cmpl_v1(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_nbr_req_v1(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_nbr_rep_v1(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_bcn_req_v1(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_bcn_rep_v1(prcd_event_log_hdr_t *plog_hdr);
+
+void pr_roam_scan_start_v2(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_scan_cmpl_v2(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_nbr_rep_v2(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_bcn_rep_v2(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_btm_rep_v2(prcd_event_log_hdr_t *plog_hdr);
+
+void pr_roam_bcn_req_v3(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_bcn_rep_v3(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_btm_rep_v3(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_6g_novlp_rep_v3(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_wtc_btm_rep_v3(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_btm_query_v3(prcd_event_log_hdr_t *plog_hdr);
+
+void pr_roam_btm_resp_v4(prcd_event_log_hdr_t *plog_hdr);
+void pr_roam_btm_req_v4(prcd_event_log_hdr_t *plog_hdr);
+
+/* Will find an entity which has highest version of that ROAM_LOG id.
+ * ALL structure should support backward compatibility.
+ * { ROAM_LOG_VER, ROAM_LOG_ID, print ROAM_LOG function() },
+ */
+
+static const pr_roam_tbl_t roam_log_print_tbl[] =
+{
+	/* ROAM Scan Start */
+	{ROAM_LOG_VER_2, ROAM_LOG_SCANSTART, pr_roam_scan_start_v2},
+	{ROAM_LOG_VER_1, ROAM_LOG_SCANSTART, pr_roam_scan_start_v1},
+	/* ROAM Scan Completed */
+	{ROAM_LOG_VER_2, ROAM_LOG_SCAN_CMPLT, pr_roam_scan_cmpl_v2},
+	{ROAM_LOG_VER_1, ROAM_LOG_SCAN_CMPLT, pr_roam_scan_cmpl_v1},
+	/* ROAM Completed */
+	{ROAM_LOG_VER_1, ROAM_LOG_ROAM_CMPLT, pr_roam_cmpl_v1},
+	/* Neighbor Request */
+	{ROAM_LOG_VER_1, ROAM_LOG_NBR_REQ, pr_roam_nbr_req_v1},
+	/* Neighbor Report */
+	{ROAM_LOG_VER_2, ROAM_LOG_NBR_REP, pr_roam_nbr_rep_v2},
+	{ROAM_LOG_VER_1, ROAM_LOG_NBR_REP, pr_roam_nbr_rep_v1},
+	/* Beacon Request */
+	{ROAM_LOG_VER_3, ROAM_LOG_BCN_REQ, pr_roam_bcn_req_v3},
+	{ROAM_LOG_VER_1, ROAM_LOG_BCN_REQ, pr_roam_bcn_req_v1},
+	/* Beacon Report */
+	{ROAM_LOG_VER_3, ROAM_LOG_BCN_REP, pr_roam_bcn_rep_v3},
+	{ROAM_LOG_VER_2, ROAM_LOG_BCN_REP, pr_roam_bcn_rep_v2},
+	{ROAM_LOG_VER_1, ROAM_LOG_BCN_REP, pr_roam_bcn_rep_v1},
+	/* BTM Response */
+	{ROAM_LOG_VER_4, ROAM_LOG_BTM_REP, pr_roam_btm_resp_v4},
+	{ROAM_LOG_VER_3, ROAM_LOG_BTM_REP, pr_roam_btm_rep_v3},
+	{ROAM_LOG_VER_2, ROAM_LOG_BTM_REP, pr_roam_btm_rep_v2},
+	/* SCAN 6G no VLP channels */
+	{ROAM_LOG_VER_3, ROAM_LOG_6G_NOVLP_REP, pr_roam_6g_novlp_rep_v3},
+	/* BTM WTC Request/Response */
+	{ROAM_LOG_VER_3, ROAM_LOG_WTC_BTM_REP, pr_roam_wtc_btm_rep_v3},
+	/* BTM Query */
+	{ROAM_LOG_VER_3, ROAM_LOG_BTM_QUERY, pr_roam_btm_query_v3},
+	/* BTM Request */
+	{ROAM_LOG_VER_4, ROAM_LOG_BTM_REQ, pr_roam_btm_req_v4},
+
+	{0, PRSV_PERIODIC_ID_MAX, NULL}
+};
+
+void pr_roam_scan_start_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_trig_v1_t *log = (roam_log_trig_v1_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_SCANSTART time: %d,"
+		" version:%d reason: %d rssi:%d cu:%d result:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->reason,
+		log->rssi, log->current_cu, log->result));
+	if (log->reason == WLC_E_REASON_DEAUTH ||
+		log->reason == WLC_E_REASON_DISASSOC) {
+		DHD_ERROR_ROAM(("  ROAM_LOG_PRT_ROAM: RCVD reason:%d\n",
+			log->prt_roam.rcvd_reason));
+	} else if (log->reason == WLC_E_REASON_BSSTRANS_REQ) {
+		DHD_ERROR_ROAM(("  ROAM_LOG_BSS_REQ: mode:%d candidate:%d token:%d "
+			"duration disassoc:%d valid:%d term:%d\n",
+			log->bss_trans.req_mode, log->bss_trans.nbrlist_size,
+			log->bss_trans.token, log->bss_trans.disassoc_dur,
+			log->bss_trans.validity_dur, log->bss_trans.bss_term_dur));
+	}
+}
+
+void pr_roam_scan_cmpl_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_scan_cmplt_v1_t *log = (roam_log_scan_cmplt_v1_t *)plog_hdr->log_ptr;
+	char chanspec_buf[CHANSPEC_STR_LEN];
+	int i;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_SCAN_CMPL: time:%d version:%d"
+		"is_full:%d scan_count:%d score_delta:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->full_scan,
+		log->scan_count, log->score_delta));
+	DHD_ERROR_ROAM(("  ROAM_LOG_CUR_AP: " MACDBG "rssi:%d score:%d channel:%s\n",
+			MAC2STRDBG((uint8 *)&log->cur_info.addr),
+			log->cur_info.rssi,
+			log->cur_info.score,
+			wf_chspec_ntoa_ex(log->cur_info.chanspec, chanspec_buf)));
+	for (i = 0; i < log->scan_list_size; i++) {
+		DHD_ERROR_ROAM(("  ROAM_LOG_CANDIDATE %d: " MACDBG
+			"rssi:%d score:%d channel:%s TPUT:%dkbps\n",
+			i, MAC2STRDBG((uint8 *)&log->scan_list[i].addr),
+			log->scan_list[i].rssi, log->scan_list[i].score,
+			wf_chspec_ntoa_ex(log->scan_list[i].chanspec,
+			chanspec_buf),
+			log->scan_list[i].estm_tput != ROAM_LOG_INVALID_TPUT?
+			log->scan_list[i].estm_tput:0));
+	}
+}
+
+void pr_roam_cmpl_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_cmplt_v1_t *log = (roam_log_cmplt_v1_t *)plog_hdr->log_ptr;
+	char chanspec_buf[CHANSPEC_STR_LEN];
+
+	DHD_ERROR_ROAM(("ROAM_LOG_ROAM_CMPL: time: %d, version:%d"
+		"status: %d reason: %d channel:%s retry:%d " MACDBG "\n",
+		plog_hdr->armcycle, log->hdr.version, log->status, log->reason,
+		wf_chspec_ntoa_ex(log->chanspec, chanspec_buf),
+		log->retry, MAC2STRDBG((uint8 *)&log->addr)));
+}
+
+void pr_roam_nbr_req_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_nbrreq_v1_t *log = (roam_log_nbrreq_v1_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_NBR_REQ: time: %d, version:%d token:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->token));
+}
+
+void pr_roam_nbr_rep_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_nbrrep_v1_t *log = (roam_log_nbrrep_v1_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_NBR_REP: time:%d, veresion:%d chan_num:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->channel_num));
+}
+
+void pr_roam_bcn_req_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_bcnrpt_req_v1_t *log = (roam_log_bcnrpt_req_v1_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REQ: time:%d, version:%d ret:%d"
+		"class:%d num_chan:%d ",
+		plog_hdr->armcycle, log->hdr.version,
+		log->result, log->reg, log->channel));
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REQ: mode:%d is_wild:%d duration:%d"
+		"ssid_len:%d\n", log->mode, log->bssid_wild,
+		log->duration, log->ssid_len));
+}
+
+void pr_roam_bcn_rep_v1(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_bcnrpt_rep_v1_t *log = (roam_log_bcnrpt_rep_v1_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REP: time:%d, verseion:%d count:%d\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->count));
+}
+
+void pr_roam_scan_start_v2(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_trig_v2_t *log = (roam_log_trig_v2_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_SCANSTART time: %d,"
+		" version:%d reason: %d rssi:%d cu:%d result:%d full_scan:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->reason,
+		log->rssi, log->current_cu, log->result,
+		log->result?(-1):log->full_scan));
+	if (log->reason == WLC_E_REASON_DEAUTH ||
+		log->reason == WLC_E_REASON_DISASSOC) {
+		DHD_ERROR_ROAM(("  ROAM_LOG_PRT_ROAM: RCVD reason:%d\n",
+			log->prt_roam.rcvd_reason));
+	} else if (log->reason == WLC_E_REASON_LOW_RSSI) {
+		DHD_ERROR_ROAM((" ROAM_LOG_LOW_RSSI: threshold:%d\n",
+			log->low_rssi.rssi_threshold));
+	}
+}
+
+void pr_roam_scan_cmpl_v2(prcd_event_log_hdr_t *plog_hdr)
+{
+	int i;
+	roam_log_scan_cmplt_v2_t *log = (roam_log_scan_cmplt_v2_t *)plog_hdr->log_ptr;
+	char chanspec_buf[CHANSPEC_STR_LEN];
+
+	DHD_ERROR_ROAM(("ROAM_LOG_SCAN_CMPL: time:%d version:%d"
+		"scan_count:%d score_delta:%d\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->scan_count, log->score_delta));
+	DHD_ERROR_ROAM(("  ROAM_LOG_CUR_AP: " MACDBG "rssi:%d score:%d channel:%s\n",
+			MAC2STRDBG((uint8 *)&log->cur_info.addr),
+			log->cur_info.rssi,
+			log->cur_info.score,
+			wf_chspec_ntoa_ex(log->cur_info.chanspec, chanspec_buf)));
+	for (i = 0; i < log->scan_list_size; i++) {
+		DHD_ERROR_ROAM(("  ROAM_LOG_CANDIDATE %d: " MACDBG
+			"rssi:%d score:%d cu :%d channel:%s TPUT:%dkbps\n",
+			i, MAC2STRDBG((uint8 *)&log->scan_list[i].addr),
+			log->scan_list[i].rssi, log->scan_list[i].score,
+			log->scan_list[i].cu_avail ?
+			(log->scan_list[i].cu * 100 / WL_MAX_CHANNEL_USAGE) : WL_CU_NOT_AVAIL,
+			wf_chspec_ntoa_ex(log->scan_list[i].chanspec,
+			chanspec_buf),
+			log->scan_list[i].estm_tput != ROAM_LOG_INVALID_TPUT?
+			log->scan_list[i].estm_tput:0));
+	}
+	if (log->chan_num != 0) {
+		print_roam_chan_list("ROAM_LOG_SCAN_CHANLIST", log->chan_num,
+			log->band2g_chan_list, log->uni2a_chan_list,
+			log->uni3_chan_list, log->uni2c_chan_list);
+	}
+
+}
+
+void pr_roam_nbr_rep_v2(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_nbrrep_v2_t *log = (roam_log_nbrrep_v2_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_NBR_REP: time:%d, veresion:%d chan_num:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->channel_num));
+	if (log->channel_num != 0) {
+		print_roam_chan_list("ROAM_LOG_NBR_REP_CHANLIST", log->channel_num,
+			log->band2g_chan_list, log->uni2a_chan_list,
+			log->uni3_chan_list, log->uni2c_chan_list);
+	}
+}
+
+void pr_roam_bcn_rep_v2(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_bcnrpt_rep_v2_t *log = (roam_log_bcnrpt_rep_v2_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REP: time:%d, verseion:%d count:%d mode:%d\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->count, log->reason));
+}
+
+void pr_roam_btm_rep_v2(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_btm_rep_v2_t *log = (roam_log_btm_rep_v2_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_BTM_REP: time:%d version:%d req_mode:%d "
+		"status:%d ret:%d\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->req_mode, log->status, log->result));
+}
+
+void pr_roam_bcn_req_v3(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_bcnrpt_req_v3_t *log = (roam_log_bcnrpt_req_v3_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REQ: time:%d, version:%d ret:%d"
+		"class:%d %s ",
+		plog_hdr->armcycle, log->hdr.version,
+		log->result, log->reg, log->channel?"":"all_chan"));
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REQ: mode:%d is_wild:%d duration:%d"
+		"ssid_len:%d\n", log->mode, log->bssid_wild,
+		log->duration, log->ssid_len));
+	if (log->channel_num != 0) {
+		print_roam_chan_list("ROAM_LOG_BCNREQ_SCAN_CHANLIST", log->channel_num,
+			log->band2g_chan_list, log->uni2a_chan_list,
+			log->uni3_chan_list, log->uni2c_chan_list);
+	}
+}
+
+static const char*
+pr_roam_bcn_rep_reason(uint16 reason_detail)
+{
+	static const char* reason_tbl[] = {
+		"BCNRPT_RSN_SUCCESS",
+		"BCNRPT_RSN_BADARG",
+		"BCNRPT_RSN_SCAN_ING",
+		"BCNRPT_RSN_SCAN_FAIL",
+		"UNKNOWN"
+	};
+
+	if (reason_detail >= ARRAYSIZE(reason_tbl)) {
+		DHD_ERROR_ROAM(("UNKNOWN Reason:%u\n", reason_detail));
+		ASSERT(0);
+		reason_detail = ARRAYSIZE(reason_tbl) - 1;
+
+	}
+	return reason_tbl[reason_detail];
+}
+
+void pr_roam_bcn_rep_v3(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_bcnrpt_rep_v3_t *log = (roam_log_bcnrpt_rep_v3_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REP: time:%d, verseion:%d count:%d mode:%d\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->count, log->reason));
+	DHD_ERROR_ROAM(("ROAM_LOG_BCN_REP: mode reason(%d):%s scan_stus:%u duration:%u\n",
+		log->reason_detail, pr_roam_bcn_rep_reason(log->reason_detail),
+		(log->reason_detail == BCNRPT_RSN_SCAN_FAIL)? log->scan_status:0,
+		log->duration));
+}
+
+void pr_roam_btm_rep_v3(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_btm_rep_v3_t *log = (roam_log_btm_rep_v3_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_BTM_REP: time:%d version:%d req_mode:%d "
+		"status:%d ret:%d target:" MACDBG "\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->req_mode, log->status, log->result,
+		MAC2STRDBG((uint8 *)&log->target_addr)));
+}
+
+void
+pr_roam_btm_query_v3(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_btm_query_v3_t *log = (roam_log_btm_query_v3_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_BTM_QUERY: time:%d version:%d token:%d reason:%d\n",
+		plog_hdr->armcycle, log->hdr.version, log->token, log->reason));
+}
+
+void
+pr_roam_6g_novlp_rep_v3(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_6g_novlp_v3_t *log = (roam_log_6g_novlp_v3_t *)plog_hdr->log_ptr;
+	char chanspec_buf[CHANSPEC_STR_LEN];
+
+	DHD_ERROR_ROAM(("ROAM_LOG_6G_NoVPL_Filtered: time:%d version:%d "
+		"CH:%s(0x%04x) BSSID:" MACDBG "\n",
+		plog_hdr->armcycle, log->hdr.version,
+		wf_chspec_ntoa_ex(log->chanspec, chanspec_buf),
+		log->chanspec, MAC2STRDBG((uint8 *)&log->bssid)));
+}
+
+void
+pr_roam_wtc_btm_rep_v3(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_wtc_btmrep_v3_t *log = (roam_log_wtc_btmrep_v3_t *)plog_hdr->log_ptr;
+
+	DHD_ERROR_ROAM(("ROAM_LOG_WTC: time:%d version:%d WTC BTM %s\n",
+		plog_hdr->armcycle, log->hdr.version,
+		(log->wtc_type == WTC_BTMREQ) ? "Request" : "Response"));
+
+	if (log->wtc_type == WTC_BTMREQ) {
+		DHD_ERROR_ROAM(("ROAM_LOG_WTC_CFG: mode:%d Scan mode:%d RSSI TH:%d "
+			"Candidate RSSI TH:%d %d %d\n",
+			log->wtcreq.mode, log->wtcreq.scantype,
+			log->wtcreq.rssithresh[WTC_BAND_2G],
+			log->wtcreq.ap_rssithresh[WTC_BAND_2G],
+			log->wtcreq.ap_rssithresh[WTC_BAND_5G],
+			log->wtcreq.ap_rssithresh[WTC_BAND_6G]));
+		if (log->wtcreq.status) {
+			DHD_ERROR_ROAM(("  Recvd invalid WTC Req len:%d ver:%d\n",
+				log->ie_length, log->wtc_ver));
+		} else {
+			DHD_ERROR_ROAM(("ROAM_LOG_WTC_REQ len:%d WTC ver:%d "
+				"Reason code:%d Subcode:%d duration:%d\n",
+				log->ie_length, log->wtc_ver,
+				log->wtcreq.rsn_code, log->wtcreq.subcode,
+				log->wtcreq.duration));
+		}
+	} else if (log->wtc_type == WTC_BTMRESP) {
+		DHD_ERROR_ROAM(("ROAM_LOG_WTC_RESP len:%d WTC ver:%d "
+			"Reason code:%d Resp status:%d\n",
+			log->ie_length, log->wtc_ver,
+			log->wtcresp.rsn_code, log->wtcresp.status));
+	}
+}
+/* ROAM logging BTM Request */
+void
+pr_roam_btm_req_v4(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_btm_req_v4_t *log = (roam_log_btm_req_v4_t *)plog_hdr->log_ptr;
+	int i;
+	DHD_ERROR_ROAM(("ROAM_LOG_BTM_REQ: time:%d version:%d req_mode:%d "
+		"token:%d candi:%d duration disassoc:%d valid:%d term:%d\n",
+		plog_hdr->armcycle, log->hdr.version,
+		log->req_mode, log->token, log->nbrlist_size,
+		log->disassoc_dur, log->validity_dur, log->bss_term_dur));
+	if (log->nbrlist_size) {
+		int max_idx = MIN(log->nbrlist_size, ROAM_NBR_RPT_LIST_SIZE);
+		for (i = 0; i < max_idx; i++) {
+			DHD_ERROR_ROAM(("   ROAM_LOG_NBR: [%d]" MACDBG " pref:0x%x\n",
+				i, MAC2STRDBG((uint8 *)&log->nbr_list[i].bssid),
+				log->nbr_list[i].preference));
+		}
+	}
+}
+/* ROAM logging BTM Response */
+void
+pr_roam_btm_resp_v4(prcd_event_log_hdr_t *plog_hdr)
+{
+	roam_log_btm_resp_v4_t *log = (roam_log_btm_resp_v4_t *)plog_hdr->log_ptr;
+	DHD_ERROR_ROAM(("ROAM_LOG_BTM_REP: time:%d version:%d req_mode:%d "
+		    "status:%d token:%d term_delay:%d ret:%d target:" MACDBG "\n",
+		    plog_hdr->armcycle, log->hdr.version,
+		    log->req_mode, log->status, log->token, log->term_delay, log->result,
+		    MAC2STRDBG((uint8 *)&log->target_addr)));
+}
+
+void
+print_roam_enhanced_log(prcd_event_log_hdr_t *plog_hdr)
+{
+	prsv_periodic_log_hdr_t *hdr = (prsv_periodic_log_hdr_t *)plog_hdr->log_ptr;
+	uint32 *ptr = (uint32 *)plog_hdr->log_ptr;
+	int i;
+	int loop_cnt = hdr->length / sizeof(uint32);
+	struct bcmstrbuf b;
+	char pr_buf[EL_LOG_STR_LEN] = { 0 };
+	const pr_roam_tbl_t *cur_elem = &roam_log_print_tbl[0];
+
+#ifdef WL_CFGVENDOR_CUST_ADVLOG
+	wl_cfgvendor_custom_advlog_roam_log(plog_hdr->log_ptr, plog_hdr->armcycle);
+#endif /* WL_CFGVENDOR_CUST_ADVLOG */
+
+	while (cur_elem && cur_elem->pr_func) {
+		if (hdr->version >= cur_elem->version &&
+			hdr->id == cur_elem->id) {
+			cur_elem->pr_func(plog_hdr);
+			return;
+		}
+		cur_elem++;
+	}
+
+	bcm_binit(&b, pr_buf, EL_LOG_STR_LEN);
+	bcm_bprintf(&b, "ROAM_LOG_UNKNOWN ID:%d ver:%d armcycle:%d",
+		hdr->id, hdr->version, plog_hdr->armcycle);
+	for (i = 0; i < loop_cnt && b.size > 0; i++) {
+		bcm_bprintf(&b, " %x", *ptr);
+		ptr++;
+	}
+	DHD_ERROR_ROAM(("%s\n", b.origbuf));
+}
+
+/*
+ * dhd_dbg_attach: initialziation of dhd dbugability module
+ *
+ * Return: An error code or 0 on success.
+ */
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+struct dhd_dbg_ring_buf g_ring_buf;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+int
+dhd_dbg_attach(dhd_pub_t *dhdp, dbg_pullreq_t os_pullreq,
+	dbg_urgent_noti_t os_urgent_notifier, void *os_priv)
+{
+	dhd_dbg_t *dbg = NULL;
+#if defined(DHD_DEBUGABILITY_LOG_DUMP_RING) || defined(DHD_DEBUGABILITY_EVENT_RING) || \
+	defined(DHD_PKT_LOGGING_DBGRING)
+	dhd_dbg_ring_t *ring = NULL;
+	int ring_id = 0;
+	void *buf = NULL;
+	int ret = BCME_ERROR;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING || BTLOG ||
+	* DHD_DEBUGABILITY_EVENT_RING || DHD_PKT_LOGGING_DBGRING ||
+	* (DEBUGABILITY && CUSTOMER_HW6)
+	*/
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	struct dhd_dbg_ring_buf *ring_buf;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	dhd_pktlog_ring_t *pktlog_ring = NULL;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	dbg = VMALLOCZ(dhdp->osh, sizeof(dhd_dbg_t));
+	if (!dbg) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for dbg, size %d\n",
+			__FUNCTION__, __LINE__, (uint32)sizeof(dhd_dbg_t)));
+		return BCME_NOMEM;
+	}
+
+#ifdef DHD_DEBUGABILITY_EVENT_RING
+#ifdef CONFIG_DHD_USE_STATIC_BUF
+	buf = DHD_OS_PREALLOC(dhdp, DHD_PREALLOC_DHD_EVENT_RING, DHD_EVENT_RING_SIZE);
+#else
+	buf = VMALLOCZ(dhdp->osh, DHD_EVENT_RING_SIZE);
+#endif
+	if (!buf) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for event_ring, size %d\n",
+			__FUNCTION__, __LINE__, DHD_EVENT_RING_SIZE));
+		ret = BCME_NOMEM;
+		goto error;
+	}
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[DHD_EVENT_RING_ID],
+		DHD_EVENT_RING_ID, (uint8 *)DHD_EVENT_RING_NAME,
+		DHD_EVENT_RING_SIZE, buf, FALSE);
+	if (ret) {
+		goto error;
+	}
+#endif /* DHD_DEBUGABILITY_EVENT_RING */
+
+#if defined(DHD_DEBUGABILITY_LOG_DUMP_RING)
+#ifdef CONFIG_DHD_USE_STATIC_BUF
+	buf = DHD_OS_PREALLOC(dhdp, DHD_PREALLOC_FW_VERBOSE_RING, FW_VERBOSE_RING_SIZE);
+#else
+	buf = VMALLOCZ(dhdp->osh, FW_VERBOSE_RING_SIZE);
+#endif
+	if (!buf) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for fw_verbose_ring, size %d\n",
+			__FUNCTION__, __LINE__, FW_VERBOSE_RING_SIZE));
+		ret = BCME_NOMEM;
+		goto error;
+	}
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[FW_VERBOSE_RING_ID],
+		FW_VERBOSE_RING_ID, (uint8 *)FW_VERBOSE_RING_NAME,
+		FW_VERBOSE_RING_SIZE, buf, FALSE);
+	if (ret) {
+		goto error;
+	}
+#endif
+
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	buf = VMALLOCZ(dhdp->osh, DRIVER_LOG_RING_SIZE);
+	if (!buf) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for driver_log_ring, size %d\n",
+			__FUNCTION__, __LINE__, DRIVER_LOG_RING_SIZE));
+		ret = BCME_NOMEM;
+		goto error;
+	}
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[DRIVER_LOG_RING_ID],
+		DRIVER_LOG_RING_ID, (uint8 *)DRIVER_LOG_RING_NAME,
+		DRIVER_LOG_RING_SIZE, buf, FALSE);
+	if (ret) {
+		goto error;
+	}
+
+	buf = VMALLOCZ(dhdp->osh, ROAM_STATS_RING_SIZE);
+	if (!buf) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for roam_stats_ring, size %d\n",
+			__FUNCTION__, __LINE__, ROAM_STATS_RING_SIZE));
+		ret = BCME_NOMEM;
+		goto error;
+	}
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[ROAM_STATS_RING_ID],
+		ROAM_STATS_RING_ID, (uint8 *)ROAM_STATS_RING_NAME,
+		ROAM_STATS_RING_SIZE, buf, FALSE);
+	if (ret) {
+		goto error;
+	}
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+
+#if defined(DHD_DEBUGABILITY_DEBUG_DUMP) || defined(DHD_HAL_RING_DUMP)
+	/*
+	 * delayed memory allocation. memory will be allocated when debug_dump is invoked
+	 * To prepare the ringbuffer in legacy HAL, we should initialize ring at this time
+	 */
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[DEBUG_DUMP_RING1_ID],
+		DEBUG_DUMP_RING1_ID, (uint8 *)DEBUG_DUMP_RING1_NAME,
+		DEBUG_DUMP_RING1_SIZE, NULL, FALSE);
+	if (ret) {
+		DHD_ERROR(("%s: Failed to init debug ring1\n", __func__));
+		goto error;
+	}
+
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[DEBUG_DUMP_RING2_ID],
+		DEBUG_DUMP_RING2_ID, (uint8 *)DEBUG_DUMP_RING2_NAME,
+		DEBUG_DUMP_RING2_SIZE, NULL, FALSE);
+	if (ret) {
+		DHD_ERROR(("%s: Failed to init debug ring2\n", __func__));
+		goto error;
+	}
+#endif /* defined(DHD_DEBUGABILITY_DEBUG_DUMP) || defined(DHD_HAL_RING_DUMP) */
+
+#if defined(DHD_HAL_RING_DUMP_MEMDUMP)
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[MEM_DUMP_RING_ID], MEM_DUMP_RING_ID,
+			(uint8 *)MEM_DUMP_RING_NAME, MEM_DUMP_RING_SIZE, NULL, FALSE);
+	if (ret) {
+		DHD_ERROR(("%s: Failed to init mem dump ring\n", __func__));
+		goto error;
+	}
+#endif /* DHD_HAL_RING_DUMP_MEMDUMP */
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (dhdp && dhdp->pktlog && dhdp->pktlog->pktlog_ring) {
+		pktlog_ring = dhdp->pktlog->pktlog_ring;
+	}
+
+	if (!pktlog_ring) {
+		DHD_ERROR(("%s: pktlog_ring is NULL. return.\n", __FUNCTION__));
+		ret = BCME_NOMEM;
+		goto error;
+	}
+
+	buf = pktlog_ring->ring_info_mem;
+	if (!buf) {
+		DHD_ERROR(("%s: ring_info_mem is NULL. return.\n", __FUNCTION__));
+		ret = BCME_NOMEM;
+		goto error;
+	}
+
+	ret = dhd_dbg_ring_init(dhdp, &dbg->dbg_rings[PACKET_LOG_RING_ID],
+			PACKET_LOG_RING_ID, (uint8 *)DHD_PACKET_LOG_RING_NAME,
+			DHD_PACKET_LOG_RING_SIZE, buf, FALSE);
+	if (ret) {
+		goto error;
+	}
+
+	pktlog_ring->dbg_ring = &dbg->dbg_rings[PACKET_LOG_RING_ID];
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	dbg->private = os_priv;
+	dbg->pullreq = os_pullreq;
+	dbg->urgent_notifier = os_urgent_notifier;
+	dhdp->dbg = dbg;
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	ring_buf = &g_ring_buf;
+	ring_buf->dhd_pub = dhdp;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+
+	return BCME_OK;
+
+#if defined(DHD_DEBUGABILITY_LOG_DUMP_RING) || defined(DHD_DEBUGABILITY_EVENT_RING) || \
+	defined(DHD_PKT_LOGGING_DBGRING)
+error:
+	for (ring_id = DEBUG_RING_ID_INVALID + 1; ring_id < DEBUG_RING_ID_MAX; ring_id++) {
+		if (VALID_RING(dbg->dbg_rings[ring_id].id)) {
+			ring = &dbg->dbg_rings[ring_id];
+			dhd_dbg_ring_deinit(dhdp, ring);
+			if (ring->ring_buf) {
+#ifdef DHD_PKT_LOGGING_DBGRING
+				if (ring_id != PACKET_LOG_RING_ID)
+#endif /* DHD_PKT_LOGGING_DBGRING */
+				{
+#ifndef CONFIG_DHD_USE_STATIC_BUF
+					VMFREE(dhdp->osh, ring->ring_buf, ring->ring_size);
+#endif
+				}
+				ring->ring_buf = NULL;
+			}
+			ring->ring_size = 0;
+		}
+	}
+
+	VMFREE(dhdp->osh, dbg, sizeof(dhd_dbg_t));
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	ring_buf = &g_ring_buf;
+	ring_buf->dhd_pub = NULL;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+
+	return ret;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING || BTLOG ||
+	* DHD_DEBUGABILITY_EVENT_RING || DHD_PKT_LOGGING_DBGRING ||
+	* (DEBUGABILITY && CUSTOMER_HW6)
+	*/
+}
+
+/*
+ * dhd_dbg_detach: clean up dhd dbugability module
+ */
+void
+dhd_dbg_detach(dhd_pub_t *dhdp)
+{
+	dhd_dbg_t *dbg;
+#if defined(DHD_DEBUGABILITY_LOG_DUMP_RING) || defined(DHD_DEBUGABILITY_EVENT_RING) || \
+	defined(DHD_PKT_LOGGING_DBGRING)
+	int ring_id;
+	dhd_dbg_ring_t *ring = NULL;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING || BTLOG ||
+	* DHD_DEBUGABILITY_EVENT_RING || DHD_PKT_LOGGING_DBGRING ||
+	* (DEBUGABILITY && CUSTOMER_HW6)
+	*/
+
+	dbg = dhdp->dbg;
+	if (!dbg) {
+		return;
+	}
+
+#if defined(DHD_DEBUGABILITY_LOG_DUMP_RING) || defined(DHD_DEBUGABILITY_EVENT_RING) || \
+	defined(DHD_PKT_LOGGING_DBGRING)
+	for (ring_id = DEBUG_RING_ID_INVALID + 1; ring_id < DEBUG_RING_ID_MAX; ring_id++) {
+		if (VALID_RING(dbg->dbg_rings[ring_id].id)) {
+			ring = &dbg->dbg_rings[ring_id];
+			dhd_dbg_ring_deinit(dhdp, ring);
+			if (ring->ring_buf) {
+#ifdef DHD_PKT_LOGGING_DBGRING
+				if (ring_id != PACKET_LOG_RING_ID)
+#endif /* DHD_PKT_LOGGING_DBGRING */
+				{
+#ifndef CONFIG_DHD_USE_STATIC_BUF
+					VMFREE(dhdp->osh, ring->ring_buf, ring->ring_size);
+#endif
+				}
+				ring->ring_buf = NULL;
+			}
+			ring->ring_size = 0;
+		}
+	}
+
+	VMFREE(dhdp->osh, dbg, sizeof(dhd_dbg_t));
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING || BTLOG ||
+	* DHD_DEBUGABILITY_EVENT_RING || DHD_PKT_LOGGING_DBGRING ||
+	* (DEBUGABILITY && CUSTOMER_HW6)
+	*/
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	g_ring_buf.dhd_pub = NULL;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+}
+
+uint32
+dhd_dbg_get_fwverbose(dhd_pub_t *dhdp)
+{
+	if (dhdp && dhdp->dbg) {
+		return dhdp->dbg->dbg_rings[FW_VERBOSE_RING_ID].log_level;
+	}
+	return 0;
+}
+
+void
+dhd_dbg_set_fwverbose(dhd_pub_t *dhdp, uint32 new_val)
+{
+
+	if (dhdp && dhdp->dbg) {
+		dhdp->dbg->dbg_rings[FW_VERBOSE_RING_ID].log_level = new_val;
+	}
+}
+
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+void
+dhd_dbg_ring_write(int type, char *binary_data,
+		int binary_len, const char *fmt, ...)
+{
+	int len = 0;
+	va_list args;
+	struct dhd_dbg_ring_buf *ring_buf = NULL;
+	char tmp_buf[DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE] = {0, };
+
+	ring_buf = &g_ring_buf;
+
+	va_start(args, fmt);
+	len = vsnprintf(tmp_buf, DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE, fmt, args);
+	/* Non ANSI C99 compliant returns -1,
+	 * ANSI compliant return len >= DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE
+	 */
+	va_end(args);
+	if (len < 0) {
+		return;
+	}
+
+	if (len >= DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE) {
+		len = DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE - 1;
+		tmp_buf[len] = '\0';
+	}
+
+	if (ring_buf->dhd_pub) {
+		dhd_pub_t *dhdp = (dhd_pub_t *)ring_buf->dhd_pub;
+		if (type == DRIVER_LOG_RING_ID || type == FW_VERBOSE_RING_ID ||
+				type == ROAM_STATS_RING_ID) {
+			if (DBG_RING_ACTIVE(dhdp, type)) {
+				dhd_os_push_push_ring_data(dhdp, type,
+						tmp_buf, strlen(tmp_buf));
+				return;
+			}
+		}
+	}
+	return;
+}
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+void dhd_debug_dump_get_section_len(dhd_pub_t *dhdp, uint32 sec_len[])
+{
+	sec_len[LOG_DUMP_SECTION_TIMESTAMP] = dhd_get_time_str_len();
+#ifdef EWP_ECNTRS_LOGGING
+	sec_len[LOG_DUMP_SECTION_ECNTRS] = dhd_get_ecntrs_len(NULL, dhdp);
+#endif /* EWP_ECNTRS_LOGGING */
+	sec_len[LOG_DUMP_SECTION_DHD_DUMP] = dhd_get_dhd_dump_len(NULL, dhdp);
+#if defined(BCMPCIE)
+	sec_len[LOG_DUMP_SECTION_EXT_TRAP] = dhd_get_ext_trap_len(NULL, dhdp);
+#endif /* BCMPCIE */
+
+#if defined(DHD_FW_COREDUMP) && defined(DNGL_EVENT_SUPPORT)
+	sec_len[LOG_DUMP_SECTION_HEALTH_CHK] = dhd_get_health_chk_len(NULL, dhdp);
+#endif
+	sec_len[LOG_DUMP_SECTION_COOKIE] = dhd_get_cookie_log_len(NULL, dhdp);
+#ifdef DHD_DUMP_PCIE_RINGS
+	sec_len[LOG_DUMP_SECTION_RING] = dhd_get_flowring_len(NULL, dhdp);
+#endif
+#ifdef DHD_STATUS_LOGGING
+	sec_len[LOG_DUMP_SECTION_STATUS] = dhd_get_status_log_len(NULL, dhdp);
+#endif /* DHD_STATUS_LOGGING */
+#ifdef EWP_RTT_LOGGING
+	sec_len[LOG_DUMP_SECTION_RTT] = dhd_get_rtt_len(NULL, dhdp);
+#endif /* EWP_RTT_LOGGING */
+#ifdef DHD_MAP_PKTID_LOGGING
+	sec_len[LOG_DUMP_SECTION_PKTID_MAP_LOG] =
+		dhd_get_pktid_map_logging_len(NULL, dhdp, TRUE);
+	sec_len[LOG_DUMP_SECTION_PKTID_UNMAP_LOG] =
+		dhd_get_pktid_map_logging_len(NULL, dhdp, FALSE);
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+	DHD_ERROR(("%s: TS:%d ECNTRS:%d DHD_DUMP:%d ETRAP:%d"
+		" HCK:%d CKI:%d FLOW:%d STATUS:%d RTT:%d PKTID_MAP:%d PKTID_UNMAP:%d\n",
+		__func__, sec_len[LOG_DUMP_SECTION_TIMESTAMP], sec_len[LOG_DUMP_SECTION_ECNTRS],
+		sec_len[LOG_DUMP_SECTION_DHD_DUMP], sec_len[LOG_DUMP_SECTION_EXT_TRAP],
+		sec_len[LOG_DUMP_SECTION_HEALTH_CHK], sec_len[LOG_DUMP_SECTION_COOKIE],
+		sec_len[LOG_DUMP_SECTION_RING], sec_len[LOG_DUMP_SECTION_STATUS],
+		sec_len[LOG_DUMP_SECTION_RTT], sec_len[LOG_DUMP_SECTION_PKTID_MAP_LOG],
+		sec_len[LOG_DUMP_SECTION_PKTID_UNMAP_LOG]));
+	return;
+}
+
+int dhd_debug_dump_buf_alloc(dhd_pub_t *dhdp, int id, char* name, int size)
+{
+	dhd_dbg_t *dbg = dhdp->dbg;
+	dhd_dbg_ring_t *ring = &dbg->dbg_rings[id];
+	char *buf = NULL;
+
+	if (!ring->ring_buf) {
+		buf = VMALLOCZ(dhdp->osh, size);
+		if (!buf) {
+			DHD_ERROR(("%s: Failed to alloc id:%d, size:%d\n", __func__, id, size));
+			return BCME_NOMEM;
+		}
+		dhd_dbg_ring_set_buf(dhdp, &dbg->dbg_rings[id], buf);
+		DHD_ERROR(("%s: success to allocate ring id:%d name:%s\n", __func__, id, name));
+	} else {
+		DHD_ERROR(("%s: already allocated id:%d name:%s\n", __func__, id, name));
+	}
+
+	return BCME_OK;
+}
+
+int dhd_debug_dump_to_ring(dhd_pub_t *dhdp)
+{
+	dhd_dbg_t *dbg = dhdp->dbg;
+	dhd_dbg_ring_t *dbg_ring;
+	int ret = BCME_OK;
+	int ring_num, id;
+	uint32 sec_len[LOG_DUMP_SECTION_MAX] = {0};
+	dhd_dbg_ring_status_t ring_status;
+	uint32 sync_retry;
+	unsigned long flags = 0;
+
+	ret = dhd_debug_dump_buf_alloc(dhdp, DEBUG_DUMP_RING1_ID,
+		DEBUG_DUMP_RING1_NAME, DEBUG_DUMP_RING1_SIZE);
+	if (ret) {
+		return ret;
+	}
+
+	ret = dhd_debug_dump_buf_alloc(dhdp, DEBUG_DUMP_RING2_ID,
+		DEBUG_DUMP_RING2_NAME, DEBUG_DUMP_RING2_SIZE);
+	if (ret) {
+		return ret;
+	}
+
+#ifdef DHD_MAP_PKTID_LOGGING
+	dhd_pktid_logging_dump(dhdp);
+#endif /* DHD_MAP_PKTID_LOGGING */
+	dhd_debug_dump_get_section_len(dhdp, sec_len);
+
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_TIMESTAMP);
+	if (dhd_print_time_str(NULL, NULL, sec_len[LOG_DUMP_SECTION_TIMESTAMP], &ring_num)) {
+		DHD_ERROR(("Error section: dhd_print_time_str\n"));
+	}
+#ifdef EWP_ECNTRS_LOGGING
+	if (dhd_print_ecntrs_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_ECNTRS], NULL)) {
+		DHD_ERROR(("Error section: ECNTRS\n"));
+	}
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef DHD_STATUS_LOGGING
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_STATUS);
+	if (dhd_print_status_log_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_STATUS], &ring_num)) {
+		DHD_ERROR(("Error section: STATUS_LOG\n"));
+	}
+#endif /* DHD_STATUS_LOGGING */
+#ifdef EWP_RTT_LOGGING
+	if (dhd_print_rtt_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_RTT], NULL)) {
+		DHD_ERROR(("Error section: RTT_LOG\n"));
+	}
+#endif /* EWP_RTT_LOGGING */
+#ifdef DHD_MAP_PKTID_LOGGING
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_PKTID_MAP_LOG);
+	if (dhd_print_pktid_map_log_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_PKTID_MAP_LOG], &ring_num, TRUE)) {
+		DHD_ERROR(("Error section: PKTID_MAP_LOG\n"));
+	}
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_PKTID_UNMAP_LOG);
+	if (dhd_print_pktid_map_log_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_PKTID_UNMAP_LOG], &ring_num, FALSE)) {
+		DHD_ERROR(("Error section: PKTID_UNMAP_LOG\n"));
+	}
+#endif /* DHD_MAP_PKTID_LOGGING */
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_DHD_DUMP);
+	if (dhd_print_dump_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_DHD_DUMP], &ring_num)) {
+		DHD_ERROR(("Error section: DHD_DUMP\n"));
+	}
+#if defined(BCMPCIE)
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_EXT_TRAP);
+	if (dhd_print_ext_trap_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_EXT_TRAP], &ring_num)) {
+		DHD_ERROR(("Error section: EXT TRAP\n"));
+	}
+#endif /* BCMPCIE */
+#if defined(DHD_FW_COREDUMP) && defined(DNGL_EVENT_SUPPORT)
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_HEALTH_CHK);
+	if (dhd_print_health_chk_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_HEALTH_CHK], &ring_num)) {
+		DHD_ERROR(("Error section: HEALTH CHECK\n"));
+	}
+#endif
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_COOKIE);
+	if (dhd_print_cookie_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_COOKIE], &ring_num)) {
+		DHD_ERROR(("Error section: COOKIE DATA\n"));
+	}
+#ifdef DHD_DUMP_PCIE_RINGS
+	ring_num = dhd_debug_dump_get_ring_num(LOG_DUMP_SECTION_RING);
+	if (dhd_print_flowring_data(NULL, dhdp, NULL, NULL,
+		sec_len[LOG_DUMP_SECTION_RING], &ring_num)) {
+		DHD_ERROR(("Error section: FLOWRING_DUMP\n"));
+	}
+#endif
+	for (id = DEBUG_RING_ID_INVALID + 1; id < DEBUG_RING_ID_MAX; id++) {
+		dbg_ring = &dbg->dbg_rings[id];
+		if (VALID_RING(dbg_ring->id)) {
+			/* To sync up with the ringbuffer in legacy HAL */
+			dhd_os_trigger_get_ring_data(dhdp, dbg_ring->name);
+
+			sync_retry = 0;
+			while (sync_retry < DHD_DEBUG_DUMP_MAX_SYNC_CNT) {
+				DHD_DBG_RING_LOCK(dbg_ring->lock, flags);
+				__dhd_dbg_get_ring_status(dbg_ring, &ring_status);
+				DHD_DBG_RING_UNLOCK(dbg_ring->lock, flags);
+				if (ring_status.written_bytes == ring_status.read_bytes) {
+					break;
+				} else {
+					OSL_SLEEP(50);
+				}
+				sync_retry++;
+			}
+			DHD_ERROR(("%s: ring sync up name:%s w/r bytes(%u/%u) retry:%d\n",
+				__func__, dbg_ring->name, ring_status.written_bytes,
+				ring_status.read_bytes, sync_retry));
+		}
+	}
+	return ret;
+}
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
diff --git a/drivers/net/wireless/ap12275/dhd_debug.h b/drivers/net/wireless/ap12275/dhd_debug.h
new file mode 100644
index 000000000000..6631c9830950
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_debug.h
@@ -0,0 +1,943 @@
+/*
+ * DHD debugability header file
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef _dhd_debug_h_
+#define _dhd_debug_h_
+#include <event_log.h>
+#include <bcmutils.h>
+#include <dhd_dbg_ring.h>
+
+enum {
+	/* Feature set */
+	DBG_MEMORY_DUMP_SUPPORTED = (1 << (0)), /* Memory dump of FW */
+	DBG_PER_PACKET_TX_RX_STATUS_SUPPORTED = (1 << (1)), /* PKT Status */
+	DBG_CONNECT_EVENT_SUPPORTED = (1 << (2)), /* Connectivity Event */
+	DBG_POWER_EVENT_SUPOORTED = (1 << (3)), /* POWER of Driver */
+	DBG_WAKE_LOCK_SUPPORTED = (1 << (4)), /* WAKE LOCK of Driver */
+	DBG_VERBOSE_LOG_SUPPORTED = (1 << (5)), /* verbose log of FW */
+	DBG_HEALTH_CHECK_SUPPORTED = (1 << (6)), /* monitor the health of FW */
+	DBG_DRIVER_DUMP_SUPPORTED = (1 << (7)), /* dumps driver state */
+	DBG_PACKET_FATE_SUPPORTED = (1 << (8)), /* tracks connection packets' fate */
+	DBG_NAN_EVENT_SUPPORTED = (1 << (9)), /* NAN Events */
+	DBG_PACKET_LOG_SUPPORTED = (1 << (10)), /* Packet log with dbgring */
+};
+
+enum {
+	/* set for binary entries */
+	DBG_RING_ENTRY_FLAGS_HAS_BINARY = (1 << (0)),
+	/* set if 64 bits timestamp is present */
+	DBG_RING_ENTRY_FLAGS_HAS_TIMESTAMP = (1 << (1))
+};
+
+enum {
+	/* Verbose logging level to set for each debug ring buffer */
+	RING_LOG_LEVEL_NONE = 0,
+	RING_LOG_LEVEL_DEFAULT = 1,
+	RING_LOG_LEVEL_VERBOSE = 2,
+	RING_LOG_LEVEL_EXCESSIVE = 3
+};
+
+/* firmware verbose ring, ring id 1 */
+#define FW_VERBOSE_RING_NAME		"fw_verbose"
+#define FW_VERBOSE_RING_SIZE		(256 * 1024)
+/* firmware event ring, ring id 2 */
+#define FW_EVENT_RING_NAME		"fw_event"
+#define FW_EVENT_RING_SIZE		(64 * 1024)
+/* DHD connection event ring, ring id 3 */
+#define DHD_EVENT_RING_NAME		"dhd_event"
+#define DHD_EVENT_RING_SIZE		(64 * 1024)
+/* NAN event ring, ring id 4 */
+#define NAN_EVENT_RING_NAME		"nan_event"
+#define NAN_EVENT_RING_SIZE		(64 * 1024)
+
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+/* DHD driver log ring */
+#define DRIVER_LOG_RING_NAME		"driver_log"
+#define DRIVER_LOG_RING_SIZE		(256 * 1024)
+/* ROAM stats log ring */
+#define ROAM_STATS_RING_NAME		"roam_stats"
+#define ROAM_STATS_RING_SIZE		(64 * 1024)
+
+#define DEBUG_DUMP_RING1_NAME		"debug_dump1_"
+#define DEBUG_DUMP_RING1_SIZE		(2 * 1024 * 1024)
+
+#define DEBUG_DUMP_RING2_NAME		"debug_dump2_"
+#define DEBUG_DUMP_RING2_SIZE		(2 * 1024 * 1024)
+
+#define MEM_DUMP_RING_NAME		"mem_dump"
+#define MEM_DUMP_RING_SIZE		(3 * 1024 * 1024)
+
+#define DHD_DEBUG_DUMP_NETLINK_MAX	(1024 * 8)
+#define DHD_DEBUG_DUMP_MAX_SYNC_CNT	5u
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+
+/* Packet log ring, ring id 7 */
+#ifdef DHD_PKT_LOGGING_DBGRING
+#define DHD_PACKET_LOG_RING_NAME	"packet_log"
+#define DHD_PACKET_LOG_RING_PKTS	MIN_PKTLOG_LEN
+#define DHD_PACKET_LOG_RING_SUSPEND_THRESHOLD	\
+	(DHD_PACKET_LOG_RING_PKTS * 8u / 10u)
+#define DHD_PACKET_LOG_RING_RESUME_THRESHOLD	\
+	(DHD_PACKET_LOG_RING_PKTS * 2u / 10u)
+#define DHD_PACKET_LOG_RING_SIZE	\
+	(DHD_PACKET_LOG_RING_PKTS * sizeof(dhd_pktlog_ring_info_t))
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+#define TLV_LOG_SIZE(tlv) ((tlv) ? (sizeof(tlv_log) + (tlv)->len) : 0)
+
+#define TLV_LOG_NEXT(tlv) \
+	((tlv) ? ((tlv_log *)((uint8 *)tlv + TLV_LOG_SIZE(tlv))) : 0)
+
+#define DBG_RING_STATUS_SIZE (sizeof(dhd_dbg_ring_status_t))
+
+#define VALID_RING(id)	\
+	((id > DEBUG_RING_ID_INVALID) && (id < DEBUG_RING_ID_MAX))
+
+#ifdef DEBUGABILITY
+#define DBG_RING_ACTIVE(dhdp, ring_id) \
+	((dhdp)->dbg && (dhdp)->dbg->dbg_rings[(ring_id)].state == RING_ACTIVE)
+#else
+#define DBG_RING_ACTIVE(dhdp, ring_id) 0
+#endif /* DEBUGABILITY */
+
+enum {
+	/* driver receive association command from kernel */
+	WIFI_EVENT_ASSOCIATION_REQUESTED	= 0,
+	WIFI_EVENT_AUTH_COMPLETE,
+	WIFI_EVENT_ASSOC_COMPLETE,
+	/* received firmware event indicating auth frames are sent */
+	WIFI_EVENT_FW_AUTH_STARTED,
+	/* received firmware event indicating assoc frames are sent */
+	WIFI_EVENT_FW_ASSOC_STARTED,
+	/* received firmware event indicating reassoc frames are sent */
+	WIFI_EVENT_FW_RE_ASSOC_STARTED,
+	WIFI_EVENT_DRIVER_SCAN_REQUESTED,
+	WIFI_EVENT_DRIVER_SCAN_RESULT_FOUND,
+	WIFI_EVENT_DRIVER_SCAN_COMPLETE,
+	WIFI_EVENT_G_SCAN_STARTED,
+	WIFI_EVENT_G_SCAN_COMPLETE,
+	WIFI_EVENT_DISASSOCIATION_REQUESTED,
+	WIFI_EVENT_RE_ASSOCIATION_REQUESTED,
+	WIFI_EVENT_ROAM_REQUESTED,
+	/* received beacon from AP (event enabled only in verbose mode) */
+	WIFI_EVENT_BEACON_RECEIVED,
+	/* firmware has triggered a roam scan (not g-scan) */
+	WIFI_EVENT_ROAM_SCAN_STARTED,
+	/* firmware has completed a roam scan (not g-scan) */
+	WIFI_EVENT_ROAM_SCAN_COMPLETE,
+	/* firmware has started searching for roam candidates (with reason =xx) */
+	WIFI_EVENT_ROAM_SEARCH_STARTED,
+	/* firmware has stopped searching for roam candidates (with reason =xx) */
+	WIFI_EVENT_ROAM_SEARCH_STOPPED,
+	WIFI_EVENT_UNUSED_0,
+	/* received channel switch anouncement from AP */
+	WIFI_EVENT_CHANNEL_SWITCH_ANOUNCEMENT,
+	/* fw start transmit eapol frame, with EAPOL index 1-4 */
+	WIFI_EVENT_FW_EAPOL_FRAME_TRANSMIT_START,
+	/* fw gives up eapol frame, with rate, success/failure and number retries */
+	WIFI_EVENT_FW_EAPOL_FRAME_TRANSMIT_STOP,
+	/* kernel queue EAPOL for transmission in driver with EAPOL index 1-4 */
+	WIFI_EVENT_DRIVER_EAPOL_FRAME_TRANSMIT_REQUESTED,
+	/* with rate, regardless of the fact that EAPOL frame is accepted or
+	 * rejected by firmware
+	 */
+	WIFI_EVENT_FW_EAPOL_FRAME_RECEIVED,
+	WIFI_EVENT_UNUSED_1,
+	/* with rate, and eapol index, driver has received */
+	/* EAPOL frame and will queue it up to wpa_supplicant */
+	WIFI_EVENT_DRIVER_EAPOL_FRAME_RECEIVED,
+	/* with success/failure, parameters */
+	WIFI_EVENT_BLOCK_ACK_NEGOTIATION_COMPLETE,
+	WIFI_EVENT_BT_COEX_BT_SCO_START,
+	WIFI_EVENT_BT_COEX_BT_SCO_STOP,
+	/* for paging/scan etc..., when BT starts transmiting twice per BT slot */
+	WIFI_EVENT_BT_COEX_BT_SCAN_START,
+	WIFI_EVENT_BT_COEX_BT_SCAN_STOP,
+	WIFI_EVENT_BT_COEX_BT_HID_START,
+	WIFI_EVENT_BT_COEX_BT_HID_STOP,
+	/* firmware sends auth frame in roaming to next candidate */
+	WIFI_EVENT_ROAM_AUTH_STARTED,
+	/* firmware receive auth confirm from ap */
+	WIFI_EVENT_ROAM_AUTH_COMPLETE,
+	/* firmware sends assoc/reassoc frame in */
+	WIFI_EVENT_ROAM_ASSOC_STARTED,
+	/* firmware receive assoc/reassoc confirm from ap */
+	WIFI_EVENT_ROAM_ASSOC_COMPLETE,
+	/* firmware sends stop G_SCAN */
+	WIFI_EVENT_G_SCAN_STOP,
+	/* firmware indicates G_SCAN scan cycle started */
+	WIFI_EVENT_G_SCAN_CYCLE_STARTED,
+	/* firmware indicates G_SCAN scan cycle completed */
+	WIFI_EVENT_G_SCAN_CYCLE_COMPLETED,
+	/* firmware indicates G_SCAN scan start for a particular bucket */
+	WIFI_EVENT_G_SCAN_BUCKET_STARTED,
+	/* firmware indicates G_SCAN scan completed for particular bucket */
+	WIFI_EVENT_G_SCAN_BUCKET_COMPLETED,
+	/* Event received from firmware about G_SCAN scan results being available */
+	WIFI_EVENT_G_SCAN_RESULTS_AVAILABLE,
+	/* Event received from firmware with G_SCAN capabilities */
+	WIFI_EVENT_G_SCAN_CAPABILITIES,
+	/* Event received from firmware when eligible candidate is found */
+	WIFI_EVENT_ROAM_CANDIDATE_FOUND,
+	/* Event received from firmware when roam scan configuration gets enabled or disabled */
+	WIFI_EVENT_ROAM_SCAN_CONFIG,
+	/* firmware/driver timed out authentication */
+	WIFI_EVENT_AUTH_TIMEOUT,
+	/* firmware/driver timed out association */
+	WIFI_EVENT_ASSOC_TIMEOUT,
+	/* firmware/driver encountered allocation failure */
+	WIFI_EVENT_MEM_ALLOC_FAILURE,
+	/* driver added a PNO network in firmware */
+	WIFI_EVENT_DRIVER_PNO_ADD,
+	/* driver removed a PNO network in firmware */
+	WIFI_EVENT_DRIVER_PNO_REMOVE,
+	/* driver received PNO networks found indication from firmware */
+	WIFI_EVENT_DRIVER_PNO_NETWORK_FOUND,
+	/* driver triggered a scan for PNO networks */
+	WIFI_EVENT_DRIVER_PNO_SCAN_REQUESTED,
+	/* driver received scan results of PNO networks */
+	WIFI_EVENT_DRIVER_PNO_SCAN_RESULT_FOUND,
+	/* driver updated scan results from PNO candidates to cfg */
+	WIFI_EVENT_DRIVER_PNO_SCAN_COMPLETE
+};
+
+enum {
+	WIFI_TAG_VENDOR_SPECIFIC = 0, /* take a byte stream as parameter */
+	WIFI_TAG_BSSID, /* takes a 6 bytes MAC address as parameter */
+	WIFI_TAG_ADDR, /* takes a 6 bytes MAC address as parameter */
+	WIFI_TAG_SSID, /* takes a 32 bytes SSID address as parameter */
+	WIFI_TAG_STATUS, /* takes an integer as parameter */
+	WIFI_TAG_CHANNEL_SPEC, /* takes one or more wifi_channel_spec as parameter */
+	WIFI_TAG_WAKE_LOCK_EVENT, /* takes a wake_lock_event struct as parameter */
+	WIFI_TAG_ADDR1, /* takes a 6 bytes MAC address as parameter */
+	WIFI_TAG_ADDR2, /* takes a 6 bytes MAC address as parameter */
+	WIFI_TAG_ADDR3, /* takes a 6 bytes MAC address as parameter */
+	WIFI_TAG_ADDR4, /* takes a 6 bytes MAC address as parameter */
+	WIFI_TAG_TSF, /* take a 64 bits TSF value as parameter */
+	WIFI_TAG_IE,
+	/* take one or more specific 802.11 IEs parameter, IEs are in turn
+	 * indicated in TLV format as per 802.11 spec
+	 */
+	WIFI_TAG_INTERFACE,	/* take interface name as parameter */
+	WIFI_TAG_REASON_CODE,	/* take a reason code as per 802.11 as parameter */
+	WIFI_TAG_RATE_MBPS,	/* take a wifi rate in 0.5 mbps */
+	WIFI_TAG_REQUEST_ID,	/* take an integer as parameter */
+	WIFI_TAG_BUCKET_ID,	/* take an integer as parameter */
+	WIFI_TAG_GSCAN_PARAMS,	/* takes a wifi_scan_cmd_params struct as parameter */
+	WIFI_TAG_GSCAN_CAPABILITIES, /* takes a wifi_gscan_capabilities struct as parameter */
+	WIFI_TAG_SCAN_ID,	/* take an integer as parameter */
+	WIFI_TAG_RSSI,		/* takes s16 as parameter */
+	WIFI_TAG_CHANNEL,	/* takes u16 as parameter */
+	WIFI_TAG_LINK_ID,	/* take an integer as parameter */
+	WIFI_TAG_LINK_ROLE,	/* take an integer as parameter */
+	WIFI_TAG_LINK_STATE,	/* take an integer as parameter */
+	WIFI_TAG_LINK_TYPE,	/* take an integer as parameter */
+	WIFI_TAG_TSCO,		/* take an integer as parameter */
+	WIFI_TAG_RSCO,		/* take an integer as parameter */
+	WIFI_TAG_EAPOL_MESSAGE_TYPE /* take an integer as parameter */
+};
+
+/* NAN  events */
+typedef enum {
+	NAN_EVENT_INVALID = 0,
+	NAN_EVENT_CLUSTER_STARTED = 1,
+	NAN_EVENT_CLUSTER_JOINED = 2,
+	NAN_EVENT_CLUSTER_MERGED = 3,
+	NAN_EVENT_ROLE_CHANGED = 4,
+	NAN_EVENT_SCAN_COMPLETE = 5,
+	NAN_EVENT_STATUS_CHNG  = 6,
+	/* ADD new events before this line */
+	NAN_EVENT_MAX
+} nan_event_id_t;
+
+typedef struct {
+    uint16 tag;
+    uint16 len; /* length of value */
+    uint8 value[0];
+} tlv_log;
+
+typedef struct per_packet_status_entry {
+    uint8 flags;
+    uint8 tid; /* transmit or received tid */
+    uint16 MCS; /* modulation and bandwidth */
+	/*
+	* TX: RSSI of ACK for that packet
+	* RX: RSSI of packet
+	*/
+    uint8 rssi;
+    uint8 num_retries; /* number of attempted retries */
+    uint16 last_transmit_rate; /* last transmit rate in .5 mbps */
+	 /* transmit/reeive sequence for that MPDU packet */
+    uint16 link_layer_transmit_sequence;
+	/*
+	* TX: firmware timestamp (us) when packet is queued within firmware buffer
+	* for SDIO/HSIC or into PCIe buffer
+	* RX : firmware receive timestamp
+	*/
+    uint64 firmware_entry_timestamp;
+	/*
+	* firmware timestamp (us) when packet start contending for the
+	* medium for the first time, at head of its AC queue,
+	* or as part of an MPDU or A-MPDU. This timestamp is not updated
+	* for each retry, only the first transmit attempt.
+	*/
+    uint64 start_contention_timestamp;
+	/*
+	* fimrware timestamp (us) when packet is successfully transmitted
+	* or aborted because it has exhausted its maximum number of retries
+	*/
+	uint64 transmit_success_timestamp;
+	/*
+	* packet data. The length of packet data is determined by the entry_size field of
+	* the wifi_ring_buffer_entry structure. It is expected that first bytes of the
+	* packet, or packet headers only (up to TCP or RTP/UDP headers) will be copied into the ring
+	*/
+    uint8 *data;
+} per_packet_status_entry_t;
+
+#if defined(LINUX)
+#define PACKED_STRUCT __attribute__ ((packed))
+#else
+#define PACKED_STRUCT
+#endif
+
+#if defined(LINUX)
+typedef struct log_conn_event {
+    uint16 event;
+    tlv_log tlvs[0];
+	/*
+	* separate parameter structure per event to be provided and optional data
+	* the event_data is expected to include an official android part, with some
+	* parameter as transmit rate, num retries, num scan result found etc...
+	* as well, event_data can include a vendor proprietary part which is
+	* understood by the developer only.
+	*/
+} PACKED_STRUCT log_conn_event_t;
+#endif /* defined(LINUX) */
+
+/*
+ * Ring buffer name for power events ring. note that power event are extremely frequents
+ * and thus should be stored in their own ring/file so as not to clobber connectivity events
+ */
+
+typedef struct wake_lock_event {
+    uint32 status; /* 0 taken, 1 released */
+    uint32 reason; /* reason why this wake lock is taken */
+    char *name; /* null terminated */
+} wake_lock_event_t;
+
+typedef struct wifi_power_event {
+    uint16 event;
+    tlv_log *tlvs;
+} wifi_power_event_t;
+
+#define NAN_EVENT_VERSION 1
+typedef struct log_nan_event {
+    uint8 version;
+    uint8 pad;
+    uint16 event;
+    tlv_log *tlvs;
+} log_nan_event_t;
+
+/* entry type */
+enum {
+	DBG_RING_ENTRY_EVENT_TYPE = 1,
+	DBG_RING_ENTRY_PKT_TYPE,
+	DBG_RING_ENTRY_WAKE_LOCK_EVENT_TYPE,
+	DBG_RING_ENTRY_POWER_EVENT_TYPE,
+	DBG_RING_ENTRY_DATA_TYPE,
+	DBG_RING_ENTRY_NAN_EVENT_TYPE
+};
+
+struct log_level_table {
+	int log_level;
+	uint16 tag;
+	char *desc;
+};
+
+#ifdef OEM_ANDROID
+/*
+ * Assuming that the Ring lock is mutex, bailing out if the
+ * callers are from atomic context. On a long term, one has to
+ * schedule a job to execute in sleepable context so that
+ * contents are pushed to the ring.
+ */
+#define DBG_EVENT_LOG(dhdp, connect_state)					\
+{										\
+	do {									\
+		uint16 state = connect_state;					\
+		if (CAN_SLEEP() && DBG_RING_ACTIVE(dhdp, DHD_EVENT_RING_ID))			\
+			dhd_os_push_push_ring_data(dhdp, DHD_EVENT_RING_ID,	\
+				&state, sizeof(state));				\
+	} while (0);								\
+}
+#else
+#define DBG_EVENT_LOG(dhd, connect_state)
+#endif /* !OEM_ANDROID */
+
+/*
+ * Packet logging - HAL specific data
+ * XXX: These should be moved to wl_cfgvendor.h
+ */
+
+#define MD5_PREFIX_LEN				4
+#define MAX_FATE_LOG_LEN			32
+#define MAX_FRAME_LEN_ETHERNET		1518
+#define MAX_FRAME_LEN_80211_MGMT	2352 /* 802.11-2012 Fig. 8-34 */
+
+typedef enum {
+	/* Sent over air and ACKed. */
+	TX_PKT_FATE_ACKED,
+
+	/* Sent over air but not ACKed. (Normal for broadcast/multicast.) */
+	TX_PKT_FATE_SENT,
+
+	/* Queued within firmware, but not yet sent over air. */
+	TX_PKT_FATE_FW_QUEUED,
+
+	/*
+	 * Dropped by firmware as invalid. E.g. bad source address,
+	 * bad checksum, or invalid for current state.
+	 */
+	TX_PKT_FATE_FW_DROP_INVALID,
+
+	/* Dropped by firmware due to lifetime expiration. */
+	TX_PKT_FATE_FW_DROP_EXPTIME,
+
+	/*
+	 * Dropped by firmware for any other reason. Includes
+	 * frames that were sent by driver to firmware, but
+	 * unaccounted for by firmware.
+	 */
+	TX_PKT_FATE_FW_DROP_OTHER,
+
+	/* Queued within driver, not yet sent to firmware. */
+	TX_PKT_FATE_DRV_QUEUED,
+
+	/*
+	 * Dropped by driver as invalid. E.g. bad source address,
+	 * or invalid for current state.
+	 */
+	TX_PKT_FATE_DRV_DROP_INVALID,
+
+	/* Dropped by driver due to lack of buffer space. */
+	TX_PKT_FATE_DRV_DROP_NOBUFS,
+
+	/*  Dropped by driver for any other reason. */
+	TX_PKT_FATE_DRV_DROP_OTHER,
+
+	/* Packet free by firmware. */
+	TX_PKT_FATE_FW_PKT_FREE,
+
+	/* Firmware dropped the frame after suppress retries reached max */
+	TX_PKT_FATE_FW_MAX_SUP_RETR,
+
+	/* Firmware forced packet lifetime expiry */
+	TX_PKT_FATE_FW_FORCED_EXPIRED,
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	/* Indicate to wait for updating txfate. */
+	TX_PKT_FATE_DRV_WAIT_UPDATE = 0x80000000,
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	} wifi_tx_packet_fate;
+
+typedef enum {
+	/* Valid and delivered to network stack (e.g., netif_rx()). */
+	RX_PKT_FATE_SUCCESS,
+
+	/* Queued within firmware, but not yet sent to driver. */
+	RX_PKT_FATE_FW_QUEUED,
+
+	/* Dropped by firmware due to host-programmable filters. */
+	RX_PKT_FATE_FW_DROP_FILTER,
+
+	/*
+	 * Dropped by firmware as invalid. E.g. bad checksum,
+	 * decrypt failed, or invalid for current state.
+	 */
+	RX_PKT_FATE_FW_DROP_INVALID,
+
+	/* Dropped by firmware due to lack of buffer space. */
+	RX_PKT_FATE_FW_DROP_NOBUFS,
+
+	/* Dropped by firmware for any other reason. */
+	RX_PKT_FATE_FW_DROP_OTHER,
+
+	/* Queued within driver, not yet delivered to network stack. */
+	RX_PKT_FATE_DRV_QUEUED,
+
+	/* Dropped by driver due to filter rules. */
+	RX_PKT_FATE_DRV_DROP_FILTER,
+
+	/* Dropped by driver as invalid. E.g. not permitted in current state. */
+	RX_PKT_FATE_DRV_DROP_INVALID,
+
+	/* Dropped by driver due to lack of buffer space. */
+	RX_PKT_FATE_DRV_DROP_NOBUFS,
+
+	/* Dropped by driver for any other reason. */
+	RX_PKT_FATE_DRV_DROP_OTHER,
+
+	/* Indicate RX Host Wake up packet. */
+	RX_PKT_FATE_WAKE_PKT,
+
+	} wifi_rx_packet_fate;
+
+typedef enum {
+	FRAME_TYPE_UNKNOWN,
+	FRAME_TYPE_ETHERNET_II,
+	FRAME_TYPE_80211_MGMT,
+	} frame_type;
+
+typedef struct wifi_frame_info {
+	/*
+	 * The type of MAC-layer frame that this frame_info holds.
+	 * - For data frames, use FRAME_TYPE_ETHERNET_II.
+	 * - For management frames, use FRAME_TYPE_80211_MGMT.
+	 * - If the type of the frame is unknown, use FRAME_TYPE_UNKNOWN.
+	 */
+	frame_type payload_type;
+
+	/*
+	 * The number of bytes included in |frame_content|. If the frame
+	 * contents are missing (e.g. RX frame dropped in firmware),
+	 * |frame_len| should be set to 0.
+	 */
+	size_t frame_len;
+
+	/*
+	 * Host clock when this frame was received by the driver (either
+	 *	outbound from the host network stack, or inbound from the
+	 *	firmware).
+	 *	- The timestamp should be taken from a clock which includes time
+	 *	  the host spent suspended (e.g. ktime_get_boottime()).
+	 *	- If no host timestamp is available (e.g. RX frame was dropped in
+	 *	  firmware), this field should be set to 0.
+	 */
+	uint32 driver_timestamp_usec;
+
+	/*
+	 * Firmware clock when this frame was received by the firmware
+	 *	(either outbound from the host, or inbound from a remote
+	 *	station).
+	 *	- The timestamp should be taken from a clock which includes time
+	 *	  firmware spent suspended (if applicable).
+	 *	- If no firmware timestamp is available (e.g. TX frame was
+	 *	  dropped by driver), this field should be set to 0.
+	 *	- Consumers of |frame_info| should _not_ assume any
+	 *	  synchronization between driver and firmware clocks.
+	 */
+	uint32 firmware_timestamp_usec;
+
+	/*
+	 * Actual frame content.
+	 * - Should be provided for TX frames originated by the host.
+	 * - Should be provided for RX frames received by the driver.
+	 * - Optionally provided for TX frames originated by firmware. (At
+	 *   discretion of HAL implementation.)
+	 * - Optionally provided for RX frames dropped in firmware. (At
+	 *   discretion of HAL implementation.)
+	 * - If frame content is not provided, |frame_len| should be set
+	 *   to 0.
+	 */
+	union {
+		char ethernet_ii[MAX_FRAME_LEN_ETHERNET];
+		char ieee_80211_mgmt[MAX_FRAME_LEN_80211_MGMT];
+	} frame_content;
+} wifi_frame_info_t;
+
+typedef struct wifi_tx_report {
+	/*
+	 * Prefix of MD5 hash of |frame_inf.frame_content|. If frame
+	 * content is not provided, prefix of MD5 hash over the same data
+	 * that would be in frame_content, if frame content were provided.
+	 */
+	char md5_prefix[MD5_PREFIX_LEN];
+	wifi_tx_packet_fate fate;
+	wifi_frame_info_t frame_inf;
+} wifi_tx_report_t;
+
+typedef struct wifi_rx_report {
+	/*
+	 * Prefix of MD5 hash of |frame_inf.frame_content|. If frame
+	 * content is not provided, prefix of MD5 hash over the same data
+	 * that would be in frame_content, if frame content were provided.
+	 */
+	char md5_prefix[MD5_PREFIX_LEN];
+	wifi_rx_packet_fate fate;
+	wifi_frame_info_t frame_inf;
+} wifi_rx_report_t;
+
+typedef struct compat_wifi_frame_info {
+	frame_type payload_type;
+
+	uint32 frame_len;
+
+	uint32 driver_timestamp_usec;
+
+	uint32 firmware_timestamp_usec;
+
+	union {
+		char ethernet_ii[MAX_FRAME_LEN_ETHERNET];
+		char ieee_80211_mgmt[MAX_FRAME_LEN_80211_MGMT];
+	} frame_content;
+} compat_wifi_frame_info_t;
+
+typedef struct compat_wifi_tx_report {
+	char md5_prefix[MD5_PREFIX_LEN];
+	wifi_tx_packet_fate fate;
+	compat_wifi_frame_info_t frame_inf;
+} compat_wifi_tx_report_t;
+
+typedef struct compat_wifi_rx_report {
+	char md5_prefix[MD5_PREFIX_LEN];
+	wifi_rx_packet_fate fate;
+	compat_wifi_frame_info_t frame_inf;
+} compat_wifi_rx_report_t;
+
+/*
+ * Packet logging - internal data
+ */
+
+typedef enum dhd_dbg_pkt_mon_state {
+	PKT_MON_INVALID = 0,
+	PKT_MON_ATTACHED,
+	PKT_MON_STARTING,
+	PKT_MON_STARTED,
+	PKT_MON_STOPPING,
+	PKT_MON_STOPPED,
+	PKT_MON_DETACHED,
+	} dhd_dbg_pkt_mon_state_t;
+
+typedef struct dhd_dbg_pkt_info {
+	frame_type payload_type;
+	size_t pkt_len;
+	uint32 driver_ts;
+	uint32 firmware_ts;
+	uint32 pkt_hash;
+	void *pkt;
+} dhd_dbg_pkt_info_t;
+
+typedef struct compat_dhd_dbg_pkt_info {
+	frame_type payload_type;
+	uint32 pkt_len;
+	uint32 driver_ts;
+	uint32 firmware_ts;
+	uint32 pkt_hash;
+	void *pkt;
+} compat_dhd_dbg_pkt_info_t;
+
+typedef struct dhd_dbg_tx_info
+{
+	wifi_tx_packet_fate fate;
+	dhd_dbg_pkt_info_t info;
+} dhd_dbg_tx_info_t;
+
+typedef struct dhd_dbg_rx_info
+{
+	wifi_rx_packet_fate fate;
+	dhd_dbg_pkt_info_t info;
+} dhd_dbg_rx_info_t;
+
+typedef struct dhd_dbg_tx_report
+{
+	dhd_dbg_tx_info_t *tx_pkts;
+	/* Indicates how many packets queued to send over the air */
+	uint16 pkt_pos;
+	/* Indicates how many packets sent over the air and received txstatus */
+	uint16 status_pos;
+} dhd_dbg_tx_report_t;
+
+typedef struct dhd_dbg_rx_report
+{
+	dhd_dbg_rx_info_t *rx_pkts;
+	/* Indicates how many packets sent over the air and received txstatus */
+	uint16 pkt_pos;
+} dhd_dbg_rx_report_t;
+
+typedef void (*dbg_pullreq_t)(void *os_priv, const int ring_id);
+typedef void (*dbg_urgent_noti_t) (dhd_pub_t *dhdp, const void *data, const uint32 len);
+typedef int (*dbg_mon_tx_pkts_t) (dhd_pub_t *dhdp, void *pkt, uint32 pktid,
+	frame_type type, uint8 mgmt_acked);
+typedef int (*dbg_mon_tx_status_t) (dhd_pub_t *dhdp, void *pkt,
+	uint32 pktid, uint16 status);
+typedef int (*dbg_mon_rx_pkts_t) (dhd_pub_t *dhdp, void *pkt, frame_type type);
+
+typedef struct dhd_dbg_pkt_mon
+{
+	dhd_dbg_tx_report_t *tx_report;
+	dhd_dbg_rx_report_t *rx_report;
+	dhd_dbg_pkt_mon_state_t tx_pkt_state;
+	dhd_dbg_pkt_mon_state_t tx_status_state;
+	dhd_dbg_pkt_mon_state_t rx_pkt_state;
+
+	/* call backs */
+	dbg_mon_tx_pkts_t tx_pkt_mon;
+	dbg_mon_tx_status_t tx_status_mon;
+	dbg_mon_rx_pkts_t rx_pkt_mon;
+} dhd_dbg_pkt_mon_t;
+
+typedef struct dhd_dbg {
+	dhd_dbg_ring_t dbg_rings[DEBUG_RING_ID_MAX];
+	void *private;          /* os private_data */
+	dhd_dbg_pkt_mon_t pkt_mon;
+	void *pkt_mon_lock; /* spin lock for packet monitoring */
+	dbg_pullreq_t pullreq;
+	dbg_urgent_noti_t urgent_notifier;
+} dhd_dbg_t;
+
+#define PKT_MON_ATTACHED(state) \
+		(((state) > PKT_MON_INVALID) && ((state) < PKT_MON_DETACHED))
+#define PKT_MON_DETACHED(state) \
+		(((state) == PKT_MON_INVALID) || ((state) == PKT_MON_DETACHED))
+#define PKT_MON_STARTED(state) ((state) == PKT_MON_STARTED)
+#define PKT_MON_STOPPED(state) ((state) == PKT_MON_STOPPED)
+#define PKT_MON_NOT_OPERATIONAL(state) \
+	(((state) != PKT_MON_STARTED) && ((state) != PKT_MON_STOPPED))
+#define PKT_MON_SAFE_TO_FREE(state) \
+	(((state) == PKT_MON_STARTING) || ((state) == PKT_MON_STOPPED))
+#define PKT_MON_PKT_FULL(pkt_count) ((pkt_count) >= MAX_FATE_LOG_LEN)
+#define PKT_MON_STATUS_FULL(pkt_count, status_count) \
+		(((status_count) >= (pkt_count)) || ((status_count) >= MAX_FATE_LOG_LEN))
+
+#ifdef DBG_PKT_MON
+#define DHD_DBG_PKT_MON_TX(dhdp, pkt, pktid, type, mgmt_acked) \
+	do { \
+		if ((dhdp) && (dhdp)->dbg && (dhdp)->dbg->pkt_mon.tx_pkt_mon && (pkt)) { \
+			(dhdp)->dbg->pkt_mon.tx_pkt_mon((dhdp), (pkt), \
+			(pktid), (type), (mgmt_acked)); \
+		} \
+	} while (0);
+#define DHD_DBG_PKT_MON_TX_STATUS(dhdp, pkt, pktid, status) \
+	do { \
+		if ((dhdp) && (dhdp)->dbg && (dhdp)->dbg->pkt_mon.tx_status_mon && (pkt)) { \
+			(dhdp)->dbg->pkt_mon.tx_status_mon((dhdp), (pkt), (pktid), (status)); \
+		} \
+	} while (0);
+#define DHD_DBG_PKT_MON_RX(dhdp, pkt, type) \
+	do { \
+		if ((dhdp) && (dhdp)->dbg && (dhdp)->dbg->pkt_mon.rx_pkt_mon && (pkt)) { \
+			if (ntoh16((pkt)->protocol) != ETHER_TYPE_BRCM) { \
+				(dhdp)->dbg->pkt_mon.rx_pkt_mon((dhdp), (pkt), (type)); \
+			} \
+		} \
+	} while (0);
+
+#define DHD_DBG_PKT_MON_START(dhdp) \
+		dhd_os_dbg_start_pkt_monitor((dhdp));
+#define DHD_DBG_PKT_MON_STOP(dhdp) \
+		dhd_os_dbg_stop_pkt_monitor((dhdp));
+#else
+#define DHD_DBG_PKT_MON_TX(dhdp, pkt, pktid, type, mgmt_acked)
+#define DHD_DBG_PKT_MON_TX_STATUS(dhdp, pkt, pktid, status)
+#define DHD_DBG_PKT_MON_RX(dhdp, pkt, type)
+#define DHD_DBG_PKT_MON_START(dhdp)
+#define DHD_DBG_PKT_MON_STOP(dhdp)
+#endif /* DBG_PKT_MON */
+
+#ifdef DUMP_IOCTL_IOV_LIST
+typedef struct dhd_iov_li {
+	dll_t list;
+	uint32 cmd; /* command number */
+	char buff[100]; /* command name */
+} dhd_iov_li_t;
+#endif /* DUMP_IOCTL_IOV_LIST */
+
+#define IOV_LIST_MAX_LEN 5
+
+#ifdef DHD_DEBUG
+typedef struct {
+	dll_t list;
+	uint32 id; /* wasted chunk id */
+	uint32 handle; /* wasted chunk handle */
+	uint32 size; /* wasted chunk size */
+} dhd_dbg_mwli_t;
+#endif /* DHD_DEBUG */
+
+#define DHD_OW_BI_RAW_EVENT_LOG_FMT 0xFFFF
+
+/* LSB 2 bits of format number to identify the type of event log */
+#define DHD_EVENT_LOG_HDR_MASK 0x3
+
+#define DHD_EVENT_LOG_FMT_NUM_OFFSET 2
+#define DHD_EVENT_LOG_FMT_NUM_MASK 0x3FFF
+
+#define DHD_EXT_FMTNUM_MASK 0xF /* bits[3..0] used */
+/**
+ * OW:- one word
+ * TW:- two word
+ * NB:- non binary
+ * BI:- binary
+ */
+#define	DHD_OW_NB_EVENT_LOG_HDR 0
+#define DHD_TW_NB_EVENT_LOG_HDR 1
+#define DHD_BI_EVENT_LOG_HDR 3
+#define DHD_INVALID_EVENT_LOG_HDR 2
+
+#define DHD_TW_VALID_TAG_BITS_MASK 0xF
+#define DHD_OW_BI_EVENT_FMT_NUM 0x3FFF
+#define DHD_TW_BI_EVENT_FMT_NUM 0x3FFE
+
+#define DHD_TW_EVENT_LOG_TAG_OFFSET 8
+
+#define EVENT_TAG_TIMESTAMP_OFFSET 1
+#define EVENT_TAG_TIMESTAMP_EXT_OFFSET 2
+
+typedef struct prcd_event_log_hdr {
+	uint32 tag;		/* Event_log entry tag */
+	uint32 count;		/* Count of 4-byte entries */
+	uint32 fmt_num_raw;	/* Format number */
+	uint32 fmt_num;		/* Format number >> 2 */
+	uint32 armcycle;	/* global ARM CYCLE for TAG */
+	uint32 *log_ptr;	/* start of payload */
+	uint32	payload_len;
+	/* Extended event log header info
+	 * 0 - legacy, 1 - extended event log header present
+	 */
+	bool ext_event_log_hdr;
+	bool binary_payload;	/* 0 - non binary payload, 1 - binary payload */
+} prcd_event_log_hdr_t;		/* Processed event log header */
+
+/* dhd_dbg functions */
+extern void dhd_dbg_trace_evnt_handler(dhd_pub_t *dhdp, void *event_data,
+		void *raw_event_ptr, uint datalen);
+void dhd_dbg_msgtrace_log_parser(dhd_pub_t *dhdp, void *event_data,
+	void *raw_event_ptr, uint datalen, bool msgtrace_hdr_present,
+	uint32 msgtrace_seqnum);
+
+extern int dhd_dbg_attach(dhd_pub_t *dhdp, dbg_pullreq_t os_pullreq,
+	dbg_urgent_noti_t os_urgent_notifier, void *os_priv);
+extern void dhd_dbg_detach(dhd_pub_t *dhdp);
+extern int dhd_dbg_start(dhd_pub_t *dhdp, bool start);
+extern int dhd_dbg_set_configuration(dhd_pub_t *dhdp, int ring_id,
+		int log_level, int flags, uint32 threshold);
+extern int dhd_dbg_find_ring_id(dhd_pub_t *dhdp, char *ring_name);
+extern dhd_dbg_ring_t *dhd_dbg_get_ring_from_ring_id(dhd_pub_t *dhdp, int ring_id);
+extern void *dhd_dbg_get_priv(dhd_pub_t *dhdp);
+extern int dhd_dbg_send_urgent_evt(dhd_pub_t *dhdp, const void *data, const uint32 len);
+extern void dhd_dbg_verboselog_printf(dhd_pub_t *dhdp, prcd_event_log_hdr_t *plog_hdr,
+	void *raw_event_ptr, uint32 *log_ptr, uint32 logset, uint16 block);
+int dhd_dbg_pull_from_ring(dhd_pub_t *dhdp, int ring_id, void *data, uint32 buf_len,
+	int *num_entries);
+int dhd_dbg_pull_single_from_ring(dhd_pub_t *dhdp, int ring_id, void *data, uint32 buf_len,
+	bool strip_header);
+#ifdef DHD_PKT_LOGGING_DBGRING
+int dhd_dbg_update_to_ring(dhd_pub_t *dhdp, void *ring, uint32 w_len);
+int dhd_dbg_pull_from_pktlog(dhd_pub_t *dhdp, int ring_id, void *data, uint32 buf_len);
+void dhd_os_dbg_urgent_pullreq(void *os_priv, int ring_id);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+int dhd_debug_dump_ring_push(dhd_pub_t *dhdp, int ring_id, uint32 len, void *data);
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+int dhd_dbg_push_to_ring(dhd_pub_t *dhdp, int ring_id, dhd_dbg_ring_entry_t *hdr,
+		void *data);
+int __dhd_dbg_get_ring_status(dhd_dbg_ring_t *ring, dhd_dbg_ring_status_t *ring_status);
+int dhd_dbg_get_ring_status(dhd_pub_t *dhdp, int ring_id,
+		dhd_dbg_ring_status_t *dbg_ring_status);
+#ifdef SHOW_LOGTRACE
+void dhd_dbg_read_ring_into_trace_buf(dhd_dbg_ring_t *ring, trace_buf_info_t *trace_buf_info);
+#endif /* SHOW_LOGTRACE */
+
+#ifdef DBG_PKT_MON
+extern int dhd_dbg_attach_pkt_monitor(dhd_pub_t *dhdp,
+		dbg_mon_tx_pkts_t tx_pkt_mon,
+		dbg_mon_tx_status_t tx_status_mon,
+		dbg_mon_rx_pkts_t rx_pkt_mon);
+extern int dhd_dbg_start_pkt_monitor(dhd_pub_t *dhdp);
+extern int dhd_dbg_monitor_tx_pkts(dhd_pub_t *dhdp, void *pkt,
+		uint32 pktid, frame_type type, uint8 mgmt_acked);
+extern int dhd_dbg_monitor_tx_status(dhd_pub_t *dhdp, void *pkt,
+		uint32 pktid, uint16 status);
+extern int dhd_dbg_monitor_rx_pkts(dhd_pub_t *dhdp, void *pkt, frame_type type);
+extern int dhd_dbg_stop_pkt_monitor(dhd_pub_t *dhdp);
+extern int dhd_dbg_monitor_get_tx_pkts(dhd_pub_t *dhdp, void __user *user_buf,
+		uint16 req_count, uint16 *resp_count);
+extern int dhd_dbg_monitor_get_rx_pkts(dhd_pub_t *dhdp, void __user *user_buf,
+		uint16 req_count, uint16 *resp_count);
+extern int dhd_dbg_detach_pkt_monitor(dhd_pub_t *dhdp);
+#endif /* DBG_PKT_MON */
+
+extern bool dhd_dbg_process_tx_status(dhd_pub_t *dhdp, void *pkt,
+		uint32 pktid, uint16 status);
+
+/* os wrapper function */
+extern int dhd_os_dbg_attach(dhd_pub_t *dhdp);
+extern void dhd_os_dbg_detach(dhd_pub_t *dhdp);
+extern int dhd_os_dbg_register_callback(int ring_id,
+	void (*dbg_ring_sub_cb)(void *ctx, const int ring_id, const void *data,
+		const uint32 len, const dhd_dbg_ring_status_t dbg_ring_status));
+extern int dhd_os_dbg_register_urgent_notifier(dhd_pub_t *dhdp,
+	void (*urgent_noti)(void *ctx, const void *data, const uint32 len, const uint32 fw_len));
+
+extern int dhd_os_start_logging(dhd_pub_t *dhdp, char *ring_name, int log_level,
+		int flags, int time_intval, int threshold);
+extern int dhd_os_reset_logging(dhd_pub_t *dhdp);
+extern int dhd_os_suppress_logging(dhd_pub_t *dhdp, bool suppress);
+
+extern int dhd_os_get_ring_status(dhd_pub_t *dhdp, int ring_id,
+		dhd_dbg_ring_status_t *dbg_ring_status);
+extern int dhd_os_trigger_get_ring_data(dhd_pub_t *dhdp, char *ring_name);
+extern int dhd_os_push_push_ring_data(dhd_pub_t *dhdp, int ring_id, void *data, int32 data_len);
+extern int dhd_os_dbg_get_feature(dhd_pub_t *dhdp, int32 *features);
+
+#ifdef DBG_PKT_MON
+extern int dhd_os_dbg_attach_pkt_monitor(dhd_pub_t *dhdp);
+extern int dhd_os_dbg_start_pkt_monitor(dhd_pub_t *dhdp);
+extern int dhd_os_dbg_monitor_tx_pkts(dhd_pub_t *dhdp, void *pkt,
+	uint32 pktid, frame_type type, uint8 mgmt_acked);
+extern int dhd_os_dbg_monitor_tx_status(dhd_pub_t *dhdp, void *pkt,
+	uint32 pktid, uint16 status);
+extern int dhd_os_dbg_monitor_rx_pkts(dhd_pub_t *dhdp, void *pkt,
+	frame_type type);
+extern int dhd_os_dbg_stop_pkt_monitor(dhd_pub_t *dhdp);
+extern int dhd_os_dbg_monitor_get_tx_pkts(dhd_pub_t *dhdp,
+	void __user *user_buf, uint16 req_count, uint16 *resp_count);
+extern int dhd_os_dbg_monitor_get_rx_pkts(dhd_pub_t *dhdp,
+	void __user *user_buf, uint16 req_count, uint16 *resp_count);
+extern int dhd_os_dbg_detach_pkt_monitor(dhd_pub_t *dhdp);
+#endif /* DBG_PKT_MON */
+
+#ifdef DUMP_IOCTL_IOV_LIST
+extern void dhd_iov_li_append(dhd_pub_t *dhd, dll_t *list_head, dll_t *node);
+extern void dhd_iov_li_print(dll_t *list_head);
+extern void dhd_iov_li_delete(dhd_pub_t *dhd, dll_t *list_head);
+#endif /* DUMP_IOCTL_IOV_LIST */
+
+#ifdef DHD_DEBUG
+extern void dhd_mw_list_delete(dhd_pub_t *dhd, dll_t *list_head);
+#endif /* DHD_DEBUG */
+
+void print_roam_enhanced_log(prcd_event_log_hdr_t *plog_hdr);
+
+typedef void (*print_roam_enhance_log_func)(prcd_event_log_hdr_t *plog_hdr);
+typedef struct _pr_roam_tbl {
+	uint8 version;
+	uint8 id;
+	print_roam_enhance_log_func pr_func;
+} pr_roam_tbl_t;
+
+extern uint32 dhd_dbg_get_fwverbose(dhd_pub_t *dhdp);
+extern void dhd_dbg_set_fwverbose(dhd_pub_t *dhdp, uint32 new_val);
+#endif /* _dhd_debug_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_debug_linux.c b/drivers/net/wireless/ap12275/dhd_debug_linux.c
new file mode 100644
index 000000000000..8f2f21a26815
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_debug_linux.c
@@ -0,0 +1,630 @@
+/*
+ * DHD debugability Linux os layer
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_debug.h>
+
+#include <net/cfg80211.h>
+#include <wl_cfgvendor.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)
+#include <linux/sched/clock.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) */
+#include <dhd_config.h>
+
+typedef void (*dbg_ring_send_sub_t)(void *ctx, const int ring_id, const void *data,
+	const uint32 len, const dhd_dbg_ring_status_t ring_status);
+typedef void (*dbg_urgent_noti_sub_t)(void *ctx, const void *data,
+	const uint32 len, const uint32 fw_len);
+
+static dbg_ring_send_sub_t ring_send_sub_cb[DEBUG_RING_ID_MAX];
+static dbg_urgent_noti_sub_t urgent_noti_sub_cb;
+typedef struct dhd_dbg_os_ring_info {
+	dhd_pub_t *dhdp;
+	int ring_id;
+	int log_level;
+	unsigned long interval;
+	struct delayed_work work;
+	uint64 tsoffset;
+} linux_dbgring_info_t;
+
+struct log_level_table dhd_event_map[] = {
+	{1, WIFI_EVENT_DRIVER_EAPOL_FRAME_TRANSMIT_REQUESTED, "DRIVER EAPOL TX REQ"},
+	{1, WIFI_EVENT_DRIVER_EAPOL_FRAME_RECEIVED, "DRIVER EAPOL RX"},
+	{2, WIFI_EVENT_DRIVER_SCAN_REQUESTED, "SCAN_REQUESTED"},
+	{2, WIFI_EVENT_DRIVER_SCAN_COMPLETE, "SCAN COMPELETE"},
+	{3, WIFI_EVENT_DRIVER_SCAN_RESULT_FOUND, "SCAN RESULT FOUND"},
+	{2, WIFI_EVENT_DRIVER_PNO_ADD, "PNO ADD"},
+	{2, WIFI_EVENT_DRIVER_PNO_REMOVE, "PNO REMOVE"},
+	{2, WIFI_EVENT_DRIVER_PNO_NETWORK_FOUND, "PNO NETWORK FOUND"},
+	{2, WIFI_EVENT_DRIVER_PNO_SCAN_REQUESTED, "PNO SCAN_REQUESTED"},
+	{1, WIFI_EVENT_DRIVER_PNO_SCAN_RESULT_FOUND, "PNO SCAN RESULT FOUND"},
+	{1, WIFI_EVENT_DRIVER_PNO_SCAN_COMPLETE, "PNO SCAN COMPELETE"}
+};
+
+static void
+debug_data_send(dhd_pub_t *dhdp, int ring_id, const void *data, const uint32 len,
+	const dhd_dbg_ring_status_t ring_status)
+{
+	struct net_device *ndev;
+	dbg_ring_send_sub_t ring_sub_send;
+	ndev = dhd_linux_get_primary_netdev(dhdp);
+
+	if (!ndev)
+		return;
+	if (!VALID_RING(ring_id))
+		return;
+	if (ring_send_sub_cb[ring_id]) {
+		ring_sub_send = ring_send_sub_cb[ring_id];
+		ring_sub_send(ndev, ring_id, data, len, ring_status);
+	}
+}
+
+static void
+dhd_os_dbg_urgent_notifier(dhd_pub_t *dhdp, const void *data, const uint32 len)
+{
+	struct net_device *ndev;
+	ndev = dhd_linux_get_primary_netdev(dhdp);
+	if (!ndev)
+		return;
+	if (urgent_noti_sub_cb) {
+		urgent_noti_sub_cb(ndev, data, len, dhdp->soc_ram_length);
+	}
+}
+
+static void
+dbg_ring_poll_worker(struct work_struct *work)
+{
+	struct delayed_work *d_work = to_delayed_work(work);
+	bool sched = TRUE;
+	dhd_dbg_ring_t *ring;
+	linux_dbgring_info_t *ring_info;
+	dhd_pub_t *dhdp;
+	int ringid;
+	dhd_dbg_ring_status_t ring_status;
+	void *buf, *buf_entries;
+	dhd_dbg_ring_entry_pack_t *pack_hdr;
+	dhd_dbg_ring_entry_t *hdr;
+	int32 buflen, rlen, remain_buflen;
+	int32 alloc_len;
+	unsigned long flags;
+
+	BCM_REFERENCE(hdr);
+
+	if (!CAN_SLEEP()) {
+		DHD_CONS_ONLY(("this context should be sleepable\n"));
+		sched = FALSE;
+		goto exit;
+	}
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	ring_info = container_of(d_work, linux_dbgring_info_t, work);
+	GCC_DIAGNOSTIC_POP();
+
+	dhdp = ring_info->dhdp;
+	ringid = ring_info->ring_id;
+
+	ring = &dhdp->dbg->dbg_rings[ringid];
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	dhd_dbg_get_ring_status(dhdp, ringid, &ring_status);
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (ringid == PACKET_LOG_RING_ID) {
+		struct net_device *ndev;
+		ndev = dhd_linux_get_primary_netdev(dhdp);
+		buflen = DBG_RING_ENTRY_SIZE;
+		buflen += dhd_os_get_pktlog_dump_size(ndev);
+		DHD_DBGIF(("%s: buflen: %d\n", __FUNCTION__, buflen));
+		alloc_len = buflen + DBG_RING_ENTRY_PACK_SIZE;
+	} else
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	{
+		DHD_DBG_RING_LOCK(ring->lock, flags);
+		if (ring->wp > ring->rp) {
+			buflen = ring->wp - ring->rp;
+		} else if (ring->wp < ring->rp) {
+			buflen = ring->ring_size - ring->rp + ring->wp;
+		} else {
+			DHD_DBG_RING_UNLOCK(ring->lock, flags);
+			goto exit;
+		}
+
+		if (buflen > ring->ring_size) {
+			DHD_DBG_RING_UNLOCK(ring->lock, flags);
+			goto exit;
+		}
+		DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+		alloc_len = NLMSG_DEFAULT_SIZE;
+	}
+	buf = VMALLOCZ(dhdp->osh, alloc_len);
+
+	if (!buf) {
+		DHD_CONS_ONLY(("%s failed to allocate read buf\n", __FUNCTION__));
+		sched = FALSE;
+		goto exit;
+	}
+
+	pack_hdr = (dhd_dbg_ring_entry_pack_t *)buf;
+	pack_hdr->magic = DBG_RING_PACK_MAGIC;
+
+	buf_entries = (char *)buf + sizeof(dhd_dbg_ring_entry_pack_t);
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (ringid == PACKET_LOG_RING_ID) {
+		pack_hdr->num_entries = 1u;
+		rlen = dhd_dbg_pull_from_pktlog(dhdp, ringid, buf_entries, buflen);
+		DHD_DBGIF(("%s: rlen: %d\n", __FUNCTION__, rlen));
+		hdr = (dhd_dbg_ring_entry_t *)buf_entries;
+		while (rlen > 0) {
+			DHD_DBG_RING_LOCK(ring->lock, flags);
+			ring_status.read_bytes += (rlen - DBG_RING_ENTRY_SIZE);
+			ring->stat.read_bytes += (rlen - DBG_RING_ENTRY_SIZE);
+			if (ring->stat.read_bytes > ring->stat.written_bytes) {
+				DHD_DBGIF(("%s READ/WRITE counter mismatched!\n", __FUNCTION__));
+				ring->stat.read_bytes = ring->stat.written_bytes;
+			}
+			DHD_DBGIF(("%s RING%d[%s]read_bytes %d, written_bytes %d, "
+				"writen_records %d\n", __FUNCTION__, ring->id, ring->name,
+				ring->stat.read_bytes, ring->stat.written_bytes,
+				ring->stat.written_records));
+			DHD_DBG_RING_UNLOCK(ring->lock, flags);
+			/* offset fw ts to host ts */
+			hdr->timestamp += ring_info->tsoffset;
+			debug_data_send(dhdp, ringid, pack_hdr,
+					ENTRY_LENGTH(hdr) + DBG_RING_ENTRY_PACK_SIZE, ring_status);
+			rlen -= ENTRY_LENGTH(hdr);
+			hdr = (dhd_dbg_ring_entry_t *)((char *)hdr + ENTRY_LENGTH(hdr));
+		}
+	} else
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	{
+		remain_buflen = buflen;
+		while (remain_buflen > 0) {
+			pack_hdr->num_entries = 0;
+			memset_s(buf_entries, NLMSG_DEFAULT_SIZE - DBG_RING_ENTRY_PACK_SIZE,
+				0, NLMSG_DEFAULT_SIZE - DBG_RING_ENTRY_PACK_SIZE);
+			/* Returns as much as possible with the size of the passed buffer
+			 * rlen means the total length of multiple entries including entry hdr
+			 */
+			rlen = dhd_dbg_pull_from_ring(dhdp, ringid, buf_entries,
+				NLMSG_DEFAULT_SIZE - DBG_RING_ENTRY_PACK_SIZE,
+				&pack_hdr->num_entries);
+			if (rlen <= 0) {
+				break;
+			}
+
+			DHD_DBG_RING_LOCK(ring->lock, flags);
+			ring_status.read_bytes += rlen;
+			DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+			/* payload length includes pack_hdr size */
+			debug_data_send(dhdp, ringid, pack_hdr,
+					rlen + DBG_RING_ENTRY_PACK_SIZE, ring_status);
+			remain_buflen -= rlen;
+		}
+	}
+
+	VMFREE(dhdp->osh, buf, alloc_len);
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	if (!ring->sched_pull) {
+		ring->sched_pull = TRUE;
+	}
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+exit:
+	if (sched) {
+		/* retrigger the work at same interval */
+		if ((ring_info->interval)) {
+			if ((ring_status.written_bytes == ring_status.read_bytes) ||
+#ifdef DHD_PKT_LOGGING_DBGRING
+				(ringid == PACKET_LOG_RING_ID)) {
+#else
+				FALSE) {
+#endif /* DHD_PKT_LOGGING_DBGRING */
+				schedule_delayed_work(d_work, ring_info->interval);
+			}
+		}
+	}
+
+	return;
+}
+
+int
+dhd_os_dbg_register_callback(int ring_id, dbg_ring_send_sub_t callback)
+{
+	if (!VALID_RING(ring_id))
+		return BCME_RANGE;
+
+	ring_send_sub_cb[ring_id] = callback;
+	return BCME_OK;
+}
+
+int
+dhd_os_dbg_register_urgent_notifier(dhd_pub_t *dhdp, dbg_urgent_noti_sub_t urgent_noti_sub)
+{
+	if (!dhdp || !urgent_noti_sub)
+		return BCME_BADARG;
+	urgent_noti_sub_cb = urgent_noti_sub;
+
+	return BCME_OK;
+}
+
+int
+dhd_os_start_logging(dhd_pub_t *dhdp, char *ring_name, int log_level,
+	int flags, int time_intval, int threshold)
+{
+	int ret = BCME_OK;
+	int ring_id;
+	linux_dbgring_info_t *os_priv, *ring_info;
+
+	ring_id = dhd_dbg_find_ring_id(dhdp, ring_name);
+	if (!VALID_RING(ring_id))
+		return BCME_UNSUPPORTED;
+
+	DHD_ERROR(("%s , ring_id : %d log_level : %d, "
+			"time_intval : %d, threshod %d Bytes\n",
+			__FUNCTION__, ring_id, log_level, time_intval, threshold));
+
+	/* change the configuration */
+	ret = dhd_dbg_set_configuration(dhdp, ring_id, log_level, flags, threshold);
+	if (ret) {
+		DHD_ERROR(("dhd_set_configuration is failed : %d\n", ret));
+		return ret;
+	}
+
+	os_priv = dhd_dbg_get_priv(dhdp);
+	if (!os_priv)
+		return BCME_ERROR;
+	ring_info = &os_priv[ring_id];
+	ring_info->log_level = log_level;
+	if (time_intval == 0 || log_level == 0) {
+		ring_info->interval = 0;
+		cancel_delayed_work_sync(&ring_info->work);
+	} else {
+		ring_info->interval = msecs_to_jiffies(time_intval * MSEC_PER_SEC);
+		cancel_delayed_work_sync(&ring_info->work);
+		schedule_delayed_work(&ring_info->work, ring_info->interval);
+	}
+
+	return ret;
+}
+
+int
+dhd_os_reset_logging(dhd_pub_t *dhdp)
+{
+	int ret = BCME_OK;
+	int ring_id;
+	linux_dbgring_info_t *os_priv, *ring_info;
+
+	os_priv = dhd_dbg_get_priv(dhdp);
+	if (!os_priv)
+		return BCME_ERROR;
+
+	/* Stop all rings */
+	for (ring_id = DEBUG_RING_ID_INVALID + 1; ring_id < DEBUG_RING_ID_MAX; ring_id++) {
+		DHD_INFO(("%s: Stop ring buffer %d\n", __FUNCTION__, ring_id));
+
+		ring_info = &os_priv[ring_id];
+		/* log level zero makes stop logging on that ring */
+		ring_info->log_level = 0;
+		ring_info->interval = 0;
+		/* change the configuration */
+		ret = dhd_dbg_set_configuration(dhdp, ring_id, 0, 0, 0);
+		if (ret) {
+			DHD_ERROR(("dhd_set_configuration is failed : %d\n", ret));
+			return ret;
+		}
+		/* cancel any pending work */
+		cancel_delayed_work_sync(&ring_info->work);
+	}
+	return ret;
+}
+
+#define SUPPRESS_LOG_LEVEL 1
+int
+dhd_os_suppress_logging(dhd_pub_t *dhdp, bool suppress)
+{
+	int ret = BCME_OK;
+	int max_log_level;
+	int enable = (suppress) ? 0 : 1;
+	linux_dbgring_info_t *os_priv;
+
+	os_priv = dhd_dbg_get_priv(dhdp);
+	if (!os_priv)
+		return BCME_ERROR;
+
+	max_log_level = os_priv[FW_VERBOSE_RING_ID].log_level;
+
+	if (max_log_level == SUPPRESS_LOG_LEVEL) {
+		/* suppress the logging in FW not to wake up host while device in suspend mode */
+		ret = dhd_iovar(dhdp, 0, "logtrace", (char *)&enable, sizeof(enable), NULL, 0,
+				TRUE);
+		if (ret < 0 && (ret != BCME_UNSUPPORTED)) {
+			DHD_ERROR(("logtrace is failed : %d\n", ret));
+		}
+	}
+
+	return ret;
+}
+
+int
+dhd_os_get_ring_status(dhd_pub_t *dhdp, int ring_id, dhd_dbg_ring_status_t *dbg_ring_status)
+{
+	return dhd_dbg_get_ring_status(dhdp, ring_id, dbg_ring_status);
+}
+
+int
+dhd_os_trigger_get_ring_data(dhd_pub_t *dhdp, char *ring_name)
+{
+	int ret = BCME_OK;
+	int ring_id;
+	linux_dbgring_info_t *os_priv, *ring_info;
+	ring_id = dhd_dbg_find_ring_id(dhdp, ring_name);
+	if (!VALID_RING(ring_id))
+		return BCME_UNSUPPORTED;
+	os_priv = dhd_dbg_get_priv(dhdp);
+	if (os_priv) {
+		ring_info = &os_priv[ring_id];
+		if (ring_info->interval) {
+			cancel_delayed_work_sync(&ring_info->work);
+		}
+		schedule_delayed_work(&ring_info->work, 0);
+	} else {
+		DHD_ERROR(("%s : os_priv is NULL\n", __FUNCTION__));
+		ret = BCME_ERROR;
+	}
+	return ret;
+}
+
+int
+dhd_os_push_push_ring_data(dhd_pub_t *dhdp, int ring_id, void *data, int32 data_len)
+{
+	int ret = BCME_OK, i;
+	dhd_dbg_ring_entry_t msg_hdr;
+	log_conn_event_t *event_data = (log_conn_event_t *)data;
+	linux_dbgring_info_t *os_priv, *ring_info = NULL;
+
+	if (!VALID_RING(ring_id))
+		return BCME_UNSUPPORTED;
+	os_priv = dhd_dbg_get_priv(dhdp);
+
+	if (os_priv) {
+		ring_info = &os_priv[ring_id];
+	} else
+		return BCME_NORESOURCE;
+
+	memset(&msg_hdr, 0, sizeof(dhd_dbg_ring_entry_t));
+
+	if (ring_id == DHD_EVENT_RING_ID) {
+		msg_hdr.type = DBG_RING_ENTRY_EVENT_TYPE;
+		msg_hdr.flags |= DBG_RING_ENTRY_FLAGS_HAS_TIMESTAMP;
+		msg_hdr.flags |= DBG_RING_ENTRY_FLAGS_HAS_BINARY;
+		msg_hdr.timestamp = osl_localtime_ns();
+		/* convert to ms */
+		msg_hdr.timestamp = DIV_U64_BY_U32(msg_hdr.timestamp, NSEC_PER_MSEC);
+		msg_hdr.len = data_len;
+		/* filter the event for higher log level with current log level */
+		for (i = 0; i < ARRAYSIZE(dhd_event_map); i++) {
+			if ((dhd_event_map[i].tag == event_data->event) &&
+				dhd_event_map[i].log_level > ring_info->log_level) {
+				return ret;
+			}
+		}
+	}
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	else if (ring_id == FW_VERBOSE_RING_ID || ring_id == DRIVER_LOG_RING_ID ||
+			ring_id == ROAM_STATS_RING_ID) {
+		msg_hdr.type = DBG_RING_ENTRY_DATA_TYPE;
+		msg_hdr.flags |= DBG_RING_ENTRY_FLAGS_HAS_TIMESTAMP;
+		msg_hdr.timestamp = osl_localtime_ns();
+		msg_hdr.timestamp = DIV_U64_BY_U32(msg_hdr.timestamp, NSEC_PER_MSEC);
+		msg_hdr.len = strlen(data);
+	}
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+	ret = dhd_dbg_push_to_ring(dhdp, ring_id, &msg_hdr, event_data);
+	if (ret && ret != BCME_BUSY) {
+		DHD_ERROR(("%s : failed to push data into the ring (%d) with ret(%d)\n",
+			__FUNCTION__, ring_id, ret));
+	}
+
+	return ret;
+}
+
+#ifdef DBG_PKT_MON
+int
+dhd_os_dbg_attach_pkt_monitor(dhd_pub_t *dhdp)
+{
+	return dhd_dbg_attach_pkt_monitor(dhdp, dhd_os_dbg_monitor_tx_pkts,
+		dhd_os_dbg_monitor_tx_status, dhd_os_dbg_monitor_rx_pkts);
+}
+
+int
+dhd_os_dbg_start_pkt_monitor(dhd_pub_t *dhdp)
+{
+	return dhd_dbg_start_pkt_monitor(dhdp);
+}
+
+int
+dhd_os_dbg_monitor_tx_pkts(dhd_pub_t *dhdp, void *pkt, uint32 pktid,
+	frame_type type, uint8 mgmt_acked)
+{
+	return dhd_dbg_monitor_tx_pkts(dhdp, pkt, pktid, type, mgmt_acked);
+}
+
+int
+dhd_os_dbg_monitor_tx_status(dhd_pub_t *dhdp, void *pkt, uint32 pktid,
+	uint16 status)
+{
+	return dhd_dbg_monitor_tx_status(dhdp, pkt, pktid, status);
+}
+
+int
+dhd_os_dbg_monitor_rx_pkts(dhd_pub_t *dhdp, void *pkt, frame_type type)
+{
+	return dhd_dbg_monitor_rx_pkts(dhdp, pkt, type);
+}
+
+int
+dhd_os_dbg_stop_pkt_monitor(dhd_pub_t *dhdp)
+{
+	return dhd_dbg_stop_pkt_monitor(dhdp);
+}
+
+int
+dhd_os_dbg_monitor_get_tx_pkts(dhd_pub_t *dhdp, void __user *user_buf,
+	uint16 req_count, uint16 *resp_count)
+{
+	return dhd_dbg_monitor_get_tx_pkts(dhdp, user_buf, req_count, resp_count);
+}
+
+int
+dhd_os_dbg_monitor_get_rx_pkts(dhd_pub_t *dhdp, void __user *user_buf,
+	uint16 req_count, uint16 *resp_count)
+{
+	return dhd_dbg_monitor_get_rx_pkts(dhdp, user_buf, req_count, resp_count);
+}
+
+int
+dhd_os_dbg_detach_pkt_monitor(dhd_pub_t *dhdp)
+{
+	return dhd_dbg_detach_pkt_monitor(dhdp);
+}
+#endif /* DBG_PKT_MON */
+
+int
+dhd_os_dbg_get_feature(dhd_pub_t *dhdp, int32 *features)
+{
+	int ret = BCME_OK;
+#ifdef DEBUGABILITY
+#ifndef DEBUGABILITY_DISABLE_MEMDUMP
+	struct dhd_conf *conf = dhdp->conf;
+#endif /* !DEBUGABILITY_DISABLE_MEMDUMP */
+#endif
+
+	/* XXX : we need to find a way to get the features for dbg */
+	*features = 0;
+#ifdef DEBUGABILITY
+#ifndef DEBUGABILITY_DISABLE_MEMDUMP
+	// fix for RequestFirmwareDebugDump issue of VTS
+	if ((conf->chip != BCM4359_CHIP_ID) && (conf->chip != BCM43751_CHIP_ID) &&
+			(conf->chip != BCM43752_CHIP_ID) && (conf->chip != BCM4375_CHIP_ID))
+		*features |= DBG_MEMORY_DUMP_SUPPORTED;
+#endif /* !DEBUGABILITY_DISABLE_MEMDUMP */
+	if (FW_SUPPORTED(dhdp, logtrace)) {
+		*features |= DBG_CONNECT_EVENT_SUPPORTED;
+		*features |= DBG_VERBOSE_LOG_SUPPORTED;
+	}
+	if (FW_SUPPORTED(dhdp, hchk)) {
+		*features |= DBG_HEALTH_CHECK_SUPPORTED;
+	}
+#ifdef DBG_PKT_MON
+	if (FW_SUPPORTED(dhdp, d11status)) {
+		*features |= DBG_PACKET_FATE_SUPPORTED;
+	}
+#endif /* DBG_PKT_MON */
+#endif /* DEBUGABILITY */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	*features |= DBG_PACKET_LOG_SUPPORTED;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	return ret;
+}
+
+static void
+dhd_os_dbg_pullreq(void *os_priv, int ring_id)
+{
+	linux_dbgring_info_t *ring_info;
+
+	ring_info = &((linux_dbgring_info_t *)os_priv)[ring_id];
+	cancel_delayed_work(&ring_info->work);
+	schedule_delayed_work(&ring_info->work, 0);
+}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+void
+dhd_os_dbg_urgent_pullreq(void *os_priv, int ring_id)
+{
+	dhd_os_dbg_pullreq(os_priv, ring_id);
+}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+int
+dhd_os_dbg_attach(dhd_pub_t *dhdp)
+{
+	int ret = BCME_OK;
+	linux_dbgring_info_t *os_priv, *ring_info;
+	int ring_id;
+
+	/* os_dbg data */
+	os_priv = VMALLOCZ(dhdp->osh, sizeof(*os_priv) * DEBUG_RING_ID_MAX);
+	if (!os_priv) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for os_priv, size %d\n", __FUNCTION__,
+			__LINE__, (uint32)sizeof(*os_priv) * DEBUG_RING_ID_MAX));
+		return BCME_NOMEM;
+	}
+
+	for (ring_id = DEBUG_RING_ID_INVALID + 1; ring_id < DEBUG_RING_ID_MAX;
+	     ring_id++) {
+		ring_info = &os_priv[ring_id];
+		INIT_DELAYED_WORK(&ring_info->work, dbg_ring_poll_worker);
+		ring_info->dhdp = dhdp;
+		ring_info->ring_id = ring_id;
+	}
+
+	ret = dhd_dbg_attach(dhdp, dhd_os_dbg_pullreq, dhd_os_dbg_urgent_notifier, os_priv);
+	if (ret) {
+		VMFREE(dhdp->osh, os_priv, sizeof(*os_priv) * DEBUG_RING_ID_MAX);
+	}
+
+	return ret;
+}
+
+void
+dhd_os_dbg_detach(dhd_pub_t *dhdp)
+{
+	linux_dbgring_info_t *os_priv, *ring_info;
+	int ring_id;
+	/* free os_dbg data */
+	os_priv = dhd_dbg_get_priv(dhdp);
+	if (!os_priv)
+		return;
+	/* abort pending any job */
+	for (ring_id = DEBUG_RING_ID_INVALID + 1; ring_id < DEBUG_RING_ID_MAX; ring_id++) {
+		ring_info = &os_priv[ring_id];
+		if (ring_info->interval) {
+			ring_info->interval = 0;
+			cancel_delayed_work_sync(&ring_info->work);
+		}
+	}
+	VMFREE(dhdp->osh, os_priv, sizeof(*os_priv) * DEBUG_RING_ID_MAX);
+
+	return dhd_dbg_detach(dhdp);
+}
diff --git a/drivers/net/wireless/ap12275/dhd_event_log_filter.c b/drivers/net/wireless/ap12275/dhd_event_log_filter.c
new file mode 100644
index 000000000000..b5417aece43d
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_event_log_filter.c
@@ -0,0 +1,3258 @@
+/*
+ * Wifi dongle status Filter and Report
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/*
+ * Filter MODULE and Report MODULE
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_debug.h>
+#include <event_log.h>
+#include <event_trace.h>
+#include <bcmtlv.h>
+#include <bcmwifi_channels.h>
+#include <dhd_event_log_filter.h>
+#include <wl_cfg80211.h>
+#include <dhd_bitpack.h>
+#include <dhd_pktlog.h>
+#ifdef DHD_STATUS_LOGGING
+#include <dhd_statlog.h>
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef IL_BIGENDIAN
+#include <bcmendian.h>
+#define htod32(i) (bcmswap32(i))
+#define htod16(i) (bcmswap16(i))
+#define dtoh32(i) (bcmswap32(i))
+#define dtoh16(i) (bcmswap16(i))
+#define htodchanspec(i) htod16(i)
+#define dtohchanspec(i) dtoh16(i)
+#else
+#define htod32(i) (i)
+#define htod16(i) (i)
+#define dtoh32(i) (i)
+#define dtoh16(i) (i)
+#define htodchanspec(i) (i)
+#define dtohchanspec(i) (i)
+#endif /* IL_BIGENDINAN */
+
+#define DHD_FILTER_ERR_INTERNAL(fmt, ...) DHD_ERROR(("EWPF-" fmt, ##__VA_ARGS__))
+#define DHD_FILTER_TRACE_INTERNAL(fmt, ...) DHD_INFO(("EWPF-" fmt, ##__VA_ARGS__))
+
+#define DHD_FILTER_ERR(x) DHD_FILTER_ERR_INTERNAL x
+#define DHD_FILTER_TRACE(x) DHD_FILTER_TRACE_INTERNAL x
+
+/* ========= EWP Filter functions ============= */
+//#define EWPF_DEBUG
+#define EWPF_DEBUG_BUF_LEN	512
+#define EWPF_VAL_CNT_PLINE	16
+
+#define EWPF_REPORT_MAX_DATA	32	/* MAX record per slice */
+
+#define EWPF_INVALID	(-1)
+#define EWPF_XTLV_INVALID		0
+
+#define EWPF_MAX_IDX_TYPE		4
+#define EWPF_IDX_TYPE_SLICE		1
+#define EWPF_IDX_TYPE_IFACE		2
+#define EWPF_IDX_TYPE_EVENT		3
+#define EWPF_IDX_TYPE_KEY_INFO		4
+
+#define EWPF_MAX_SLICE			2	/* MAX slice in dongle */
+#define EWPF_SLICE_MAIN			0	/* SLICE ID for 5GHZ */
+#define EWPF_SLICE_AUX			1	/* SLICE ID for 2GHZ */
+
+#define EWPF_MAX_IFACE			3	/* MAX IFACE supported */
+#define EWPF_MAX_EVENT			1	/* MAX EVENT counter supported */
+#define EWPF_MAX_KEY_INFO			1	/* MAX KEY INFO counter supported */
+
+#define EWPF_ARM_TO_MSEC		1
+#define EWPF_NO_UNIT_CONV		1
+#define EWPF_MSEC_TO_SEC		1000
+#define EWPF_USEC_TO_MSEC		1000
+#define EWPF_NSEC_TO_MSEC		1000000
+#define EWPF_USEC_TO_SEC		1000000
+#define EWPF_EPOCH				1000
+#define EWPF_NONSEC_TO_SEC		1000000000
+#define EWPF_REPORT_YEAR_MUL	10000
+#define EWPF_REPORT_MON_MUL		100
+#define EWPF_REPORT_HOUR_MUL	10000
+#define EWPF_REPORT_MIN_MUL		100
+#define EWPF_REPORT_MINUTES		60
+#define EWPF_REPORT_YEAR_BASE	1900
+
+#define EWPF_NO_ABS		FALSE
+#define EWPF_NEED_ABS		TRUE
+
+#define EWPF_MAX_INFO_TYPE	5
+#define EWPF_INFO_VER		0
+#define EWPF_INFO_TYPE		1
+#define EWPF_INFO_ECNT		2
+#define EWPF_INFO_IOVAR		3
+#define EWPF_INFO_CPLOG		4
+#define EWPF_INFO_DHDSTAT	5
+
+#define EWPF_UPDATE_ARM_CYCLE_OFFSET	1
+
+/* EWPF element of slice type */
+typedef struct {
+	uint32 armcycle; /* dongle arm cycle for this record */
+	union {
+		wl_periodic_compact_cntrs_v1_t compact_cntr_v1;
+		wl_periodic_compact_cntrs_v2_t compact_cntr_v2;
+		wl_periodic_compact_cntrs_v3_t compact_cntr_v3;
+		wl_periodic_compact_cntrs_v4_t compact_cntr_v4;
+	};
+	evt_hist_compact_toss_stats_v1_t hist_tx_toss_stat;
+	evt_hist_compact_toss_stats_v1_t hist_rx_toss_stat;
+	wlc_btc_stats_v4_t btc_stat;
+	wl_compact_he_cnt_wlc_v2_t compact_he_cnt;
+} EWPF_slc_elem_t;
+
+/* EWPF element for interface type */
+typedef struct {
+	uint32 armcycle; /* dongle arm cycle for this record */
+	wl_if_stats_t if_stat;
+	wl_lqm_t lqm;
+	wl_if_infra_stats_t infra;
+	wl_if_mgt_stats_t mgmt_stat;
+	wl_if_state_compact_t if_comp_stat;
+	wl_adps_dump_summary_v2_t adps_dump_summary;
+	wl_adps_energy_gain_v1_t adps_energy_gain;
+	wl_roam_stats_v1_t roam_stat;
+} EWPF_ifc_elem_t;
+
+typedef struct {
+	uint32 first_armcycle; /* first dongle arm cycle for this record */
+	uint32 updated_armcycle; /* last updated dongle arm cycle for this record */
+	wl_event_based_statistics_v4_t event_stat;
+} EWPF_event_elem_t;
+
+typedef struct {
+	uint32 first_armcycle; /* first dongle arm cycle for this record */
+	uint32 updated_armcycle; /* last updaated dongle arm cycle for this record */
+	key_update_info_v1_t key_update_info;
+} EWPF_key_info_elem_t;
+
+typedef struct {
+	uint32 first_armcycle; /* first dongle arm cycle for this record */
+	uint32 updated_armcycle; /* last updated dongle arm cycle for this record */
+	wl_roam_stats_v1_t roam_stat;
+} EWPF_roam_stats_event_elem_t;
+
+typedef struct {
+	int enabled;			/* enabled/disabled */
+	dhd_pub_t *dhdp;
+	uint32 tmp_armcycle;	/* global ARM CYCLE for TAG */
+	int idx_type;			/* 0 : SLICE, 1: IFACE */
+	int xtlv_idx;			/* Slice/Interface index : global for TAG */
+	void *s_ring[EWPF_MAX_SLICE];
+	void *i_ring[EWPF_MAX_IFACE];
+	void *e_ring[EWPF_MAX_EVENT];
+	void *k_ring[EWPF_MAX_KEY_INFO];
+
+	/* used by Report module */
+	uint8 last_bssid[ETHER_ADDR_LEN];	/* BSSID of last conencted/request */
+	int last_channel;
+	uint32 last_armcycle;	/* ARM CYCLE prior last connection */
+} EWP_filter_t;
+
+/* status gathering functions : XTLV callback functions */
+typedef int (*EWPF_filter_cb)(void *ctx, const uint8 *data, uint16 type, uint16 len);
+static int evt_xtlv_print_cb(void *ctx, const uint8 *data, uint16 type, uint16 len);
+static int evt_xtlv_copy_cb(void *ctx, const uint8 *data, uint16 type, uint16 len);
+static int evt_xtlv_idx_cb(void *ctx, const uint8 *data, uint16 type, uint16 len);
+static int evt_xtlv_type_cb(void *ctx, const uint8 *data, uint16 type, uint16 len);
+static int filter_main_cb(void *ctx, const uint8 *data, uint16 type, uint16 len);
+static int evt_xtlv_roam_cb(void *ctx, const uint8 *data, uint16 type, uint16 len);
+
+/* ========= Event Handler functions and its callbacks: ============= */
+typedef struct _EWPF_tbl {
+	uint16 xtlv_id; /* XTLV ID, to handle */
+	EWPF_filter_cb cb_func; /* specific call back function, usually for structre */
+	int idx_type;			/* structure specific info: belonged type */
+	int	max_idx;			/* structure specific info: ALLOWED MAX IDX */
+	uint32 offset; /* offset of structure in EWPF_elem-t, valid if cb is not null */
+	uint32 member_length;	/* MAX length of reserved for this structure */
+	struct _EWPF_tbl *tbl; /* sub table if XTLV map to XLTV */
+} EWPF_tbl_t;
+
+/* Context structre for XTLV callback */
+typedef struct {
+	dhd_pub_t *dhdp;
+	EWPF_tbl_t *tbl;
+} EWPF_ctx_t;
+
+#define SLICE_INFO(a) EWPF_IDX_TYPE_SLICE, EWPF_MAX_SLICE, OFFSETOF(EWPF_slc_elem_t, a), \
+	sizeof(((EWPF_slc_elem_t *)NULL)->a)
+#define IFACE_INFO(a) EWPF_IDX_TYPE_IFACE, EWPF_MAX_IFACE, OFFSETOF(EWPF_ifc_elem_t, a), \
+	sizeof(((EWPF_ifc_elem_t *)NULL)->a)
+#define EVENT_INFO(a) EWPF_IDX_TYPE_EVENT, EWPF_MAX_EVENT, OFFSETOF(EWPF_event_elem_t, a), \
+		sizeof(((EWPF_event_elem_t *)NULL)->a)
+#define KEY_INFO(a) EWPF_IDX_TYPE_KEY_INFO, EWPF_MAX_KEY_INFO, OFFSETOF(EWPF_key_info_elem_t, a), \
+		sizeof(((EWPF_key_info_elem_t *)NULL)->a)
+
+#define SLICE_U_SIZE(a) sizeof(((EWPF_slc_elem_t *)NULL)->a)
+#define SLICE_INFO_UNION(a) EWPF_IDX_TYPE_SLICE, EWPF_MAX_SLICE, OFFSETOF(EWPF_slc_elem_t, a)
+#define NONE_INFO(a) 0, 0, a, 0
+/* XTLV TBL for WL_SLICESTATS_XTLV_PERIODIC_STATE */
+static EWPF_tbl_t EWPF_periodic[] =
+{
+	{
+		WL_STATE_COMPACT_COUNTERS,
+		evt_xtlv_copy_cb,
+		SLICE_INFO(compact_cntr_v4),
+		NULL
+	},
+	{
+		WL_STATE_COMPACT_HE_COUNTERS,
+		evt_xtlv_copy_cb,
+		SLICE_INFO(compact_he_cnt),
+		NULL
+	},
+	{EWPF_XTLV_INVALID, NULL, NONE_INFO(0), NULL}
+};
+
+static EWPF_tbl_t EWPF_if_periodic[] =
+{
+	{
+		WL_STATE_IF_COMPACT_STATE,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(if_comp_stat),
+		NULL
+	},
+	{
+		WL_STATE_IF_ADPS_STATE,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(adps_dump_summary),
+		NULL
+	},
+	{
+		WL_STATE_IF_ADPS_ENERGY_GAIN,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(adps_energy_gain),
+		NULL
+	},
+	{EWPF_XTLV_INVALID, NULL, NONE_INFO(0), NULL}
+};
+
+static EWPF_tbl_t EWPF_roam[] =
+{
+	{
+		WL_IFSTATS_XTLV_ROAM_STATS_EVENT,
+		evt_xtlv_print_cb,
+		NONE_INFO(0),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_ROAM_STATS_PERIODIC,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(roam_stat),
+		NULL
+	},
+	{EWPF_XTLV_INVALID, NULL, NONE_INFO(0), NULL}
+};
+
+/* XTLV TBL for EVENT_LOG_TAG_STATS */
+static EWPF_tbl_t EWPF_main[] =
+{
+	/* MAIN XTLV */
+	{
+		WL_IFSTATS_XTLV_WL_SLICE,
+		evt_xtlv_type_cb,
+		NONE_INFO(0),
+		EWPF_main
+	},
+	{
+		WL_IFSTATS_XTLV_IF,
+		evt_xtlv_type_cb,
+		NONE_INFO(0),
+		EWPF_main
+	},
+	/* ID XTLVs */
+	{
+		WL_IFSTATS_XTLV_SLICE_INDEX,
+		evt_xtlv_idx_cb,
+		NONE_INFO(0),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_IF_INDEX,
+		evt_xtlv_idx_cb,
+		NONE_INFO(0),
+		NULL
+	},
+	/* NORMAL XTLVS */
+	{
+		WL_SLICESTATS_XTLV_PERIODIC_STATE,
+		NULL,
+		NONE_INFO(0),
+		EWPF_periodic
+	},
+	{
+		WL_IFSTATS_XTLV_IF_LQM,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(lqm),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_GENERIC,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(if_stat),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_MGT_CNT,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(mgmt_stat),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_IF_PERIODIC_STATE,
+		NULL,
+		NONE_INFO(0),
+		EWPF_if_periodic
+	},
+	{
+		WL_IFSTATS_XTLV_INFRA_SPECIFIC,
+		evt_xtlv_copy_cb,
+		IFACE_INFO(infra),
+		NULL
+	},
+	{
+		WL_SLICESTATS_XTLV_HIST_TX_STATS,
+		evt_xtlv_copy_cb,
+		SLICE_INFO(hist_tx_toss_stat),
+		NULL
+	},
+	{
+		WL_SLICESTATS_XTLV_HIST_RX_STATS,
+		evt_xtlv_copy_cb,
+		SLICE_INFO(hist_rx_toss_stat),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_WL_SLICE_BTCOEX,
+		evt_xtlv_copy_cb,
+		SLICE_INFO(btc_stat),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_IF_EVENT_STATS,
+		evt_xtlv_copy_cb,
+		EVENT_INFO(event_stat),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_IF_EVENT_STATS,
+		evt_xtlv_print_cb,
+		NONE_INFO(0),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_KEY_PLUMB_INFO,
+		evt_xtlv_copy_cb,
+		KEY_INFO(key_update_info),
+		NULL
+	},
+	{
+		WL_IFSTATS_XTLV_ROAM_STATS_EVENT,
+		evt_xtlv_roam_cb,
+		NONE_INFO(0),
+		EWPF_roam
+	},
+	{
+		WL_IFSTATS_XTLV_ROAM_STATS_PERIODIC,
+		evt_xtlv_roam_cb,
+		IFACE_INFO(roam_stat),
+		EWPF_roam
+	},
+
+	{EWPF_XTLV_INVALID, NULL, NONE_INFO(0), NULL}
+};
+
+#if defined(DHD_EWPR_VER2) && defined(DHD_STATUS_LOGGING)
+
+#define EWP_DHD_STAT_SIZE 2
+
+uint8
+dhd_statlog_filter[] =
+{
+	ST(WLAN_POWER_ON),	/* Wi-Fi Power on */
+	ST(WLAN_POWER_OFF),	/* Wi-Fi Power off */
+	ST(ASSOC_START),	/* connect to the AP triggered by upper layer */
+	ST(AUTH_DONE),		/* complete to authenticate with the AP */
+	ST(ASSOC_REQ),		/* send or receive Assoc Req */
+	ST(ASSOC_RESP),		/* send or receive Assoc Resp */
+	ST(ASSOC_DONE),		/* complete to disconnect to the associated AP */
+	ST(DISASSOC_START),	/* disconnect to the associated AP by upper layer */
+	ST(DISASSOC_INT_START),	/* initiate the disassoc by DHD */
+	ST(DISASSOC_DONE),	/* complete to disconnect to the associated AP */
+	ST(DISASSOC),		/* send or receive Disassoc */
+	ST(DEAUTH),		/* send or receive Deauth */
+	ST(LINKDOWN),		/* receive the link down event */
+	ST(REASSOC_START),	/* reassoc the candidate AP */
+	ST(REASSOC_INFORM),	/* inform reassoc completion to upper layer */
+	ST(REASSOC_DONE),	/* complete to reassoc */
+	ST(EAPOL_M1),		/* send or receive the EAPOL M1 */
+	ST(EAPOL_M2),		/* send or receive the EAPOL M2 */
+	ST(EAPOL_M3),		/* send or receive the EAPOL M3 */
+	ST(EAPOL_M4),		/* send or receive the EAPOL M4 */
+	ST(EAPOL_GROUPKEY_M1),	/* send or receive the EAPOL Group key handshake M1 */
+	ST(EAPOL_GROUPKEY_M2),	/* send or receive the EAPOL Group key handshake M2 */
+	ST(EAP_REQ_IDENTITY),	/* send or receive the EAP REQ IDENTITY */
+	ST(EAP_RESP_IDENTITY),	/* send or receive the EAP RESP IDENTITY */
+	ST(EAP_REQ_TLS),	/* send or receive the EAP REQ TLS */
+	ST(EAP_RESP_TLS),	/* send or receive the EAP RESP TLS */
+	ST(EAP_REQ_LEAP),	/* send or receive the EAP REQ LEAP */
+	ST(EAP_RESP_LEAP),	/* send or receive the EAP RESP LEAP */
+	ST(EAP_REQ_TTLS),	/* send or receive the EAP REQ TTLS */
+	ST(EAP_RESP_TTLS),	/* send or receive the EAP RESP TTLS */
+	ST(EAP_REQ_AKA),	/* send or receive the EAP REQ AKA */
+	ST(EAP_RESP_AKA),	/* send or receive the EAP RESP AKA */
+	ST(EAP_REQ_PEAP),	/* send or receive the EAP REQ PEAP */
+	ST(EAP_RESP_PEAP),	/* send or receive the EAP RESP PEAP */
+	ST(EAP_REQ_FAST),	/* send or receive the EAP REQ FAST */
+	ST(EAP_RESP_FAST),	/* send or receive the EAP RESP FAST */
+	ST(EAP_REQ_PSK),	/* send or receive the EAP REQ PSK */
+	ST(EAP_RESP_PSK),	/* send or receive the EAP RESP PSK */
+	ST(EAP_REQ_AKAP),	/* send or receive the EAP REQ AKAP */
+	ST(EAP_RESP_AKAP),	/* send or receive the EAP RESP AKAP */
+	ST(EAP_SUCCESS),	/* send or receive the EAP SUCCESS */
+	ST(EAP_FAILURE),	/* send or receive the EAP FAILURE */
+	ST(EAPOL_START),	/* send or receive the EAPOL-START */
+	ST(WSC_START),		/* send or receive the WSC START */
+	ST(WSC_DONE),		/* send or receive the WSC DONE */
+	ST(WPS_M1),		/* send or receive the WPS M1 */
+	ST(WPS_M2),		/* send or receive the WPS M2 */
+	ST(WPS_M3),		/* send or receive the WPS M3 */
+	ST(WPS_M4),		/* send or receive the WPS M4 */
+	ST(WPS_M5),		/* send or receive the WPS M5 */
+	ST(WPS_M6),		/* send or receive the WPS M6 */
+	ST(WPS_M7),		/* send or receive the WPS M7 */
+	ST(WPS_M8),		/* send or receive the WPS M8 */
+	ST(8021X_OTHER),	/* send or receive the other 8021X frames */
+	ST(INSTALL_KEY),	/* install the key */
+	ST(DELETE_KEY),		/* remove the key */
+	ST(INSTALL_PMKSA),	/* install PMKID information */
+	ST(INSTALL_OKC_PMK),	/* install PMKID information for OKC */
+	ST(DHCP_DISCOVER),	/* send or recv DHCP Discover */
+	ST(DHCP_OFFER),		/* send or recv DHCP Offer */
+	ST(DHCP_REQUEST),	/* send or recv DHCP Request */
+	ST(DHCP_DECLINE),	/* send or recv DHCP Decline */
+	ST(DHCP_ACK),		/* send or recv DHCP ACK */
+	ST(DHCP_NAK),		/* send or recv DHCP NACK */
+	ST(DHCP_RELEASE),	/* send or recv DHCP Release */
+	ST(DHCP_INFORM),	/* send or recv DHCP Inform */
+	ST(REASSOC_SUCCESS),	/* reassociation success */
+	ST(REASSOC_FAILURE),	/* reassociation failure */
+	ST(AUTH_TIMEOUT),	/* authentication timeout */
+	ST(AUTH_FAIL),		/* authentication failure */
+	ST(AUTH_NO_ACK),	/* authentication failure due to no ACK */
+	ST(AUTH_OTHERS),	/* authentication failure with other status */
+	ST(ASSOC_TIMEOUT),	/* association timeout */
+	ST(ASSOC_FAIL),		/* association failure */
+	ST(ASSOC_NO_ACK),	/* association failure due to no ACK */
+	ST(ASSOC_ABORT),	/* association abort */
+	ST(ASSOC_UNSOLICITED),	/* association unsolicited */
+	ST(ASSOC_NO_NETWORKS),	/* association failure due to no networks */
+	ST(ASSOC_OTHERS),	/* association failure due to no networks */
+	ST(REASSOC_DONE_OTHERS)	/* complete to reassoc with other reason */
+};
+#endif /* DHD_EWPR_VER2 && DHD_STATUS_LOGGING  */
+
+/* ========= Module functions : exposed to others ============= */
+int
+dhd_event_log_filter_init(dhd_pub_t *dhdp, uint8 *buf, uint32 buf_size)
+{
+
+	EWP_filter_t *filter;
+	int idx;
+	uint32 req_size;
+	uint32 s_ring_size; /* slice ring */
+	uint32 i_ring_size; /* interface ring */
+	uint32 e_ring_size; /* event counter ring */
+	uint32 k_ring_size; /* key info ring */
+	uint8 *buf_ptr = buf;
+	EWPF_ctx_t ctx;
+	wl_event_based_statistics_v4_t dummy_event_stat;
+	key_update_info_v1_t dummy_key_update_info;
+#if defined(DHD_EWPR_VER2) && defined(DHD_STATUS_LOGGING)
+	stat_bdmask_req_t req;
+#endif /* DHD_EWPR_VER2 && DHD_STATUS_LOGGING */
+
+	DHD_FILTER_ERR(("STARTED\n"));
+
+	if (!dhdp || !buf) {
+		DHD_FILTER_ERR(("INVALID PTR: dhdp:%p buf:%p\n", dhdp, buf));
+		return BCME_ERROR;
+	}
+
+	i_ring_size = s_ring_size = e_ring_size = k_ring_size = dhd_ring_get_hdr_size();
+	s_ring_size += ((uint32)sizeof(EWPF_slc_elem_t)) * EWPF_REPORT_MAX_DATA;
+	i_ring_size += ((uint32)sizeof(EWPF_ifc_elem_t)) * EWPF_REPORT_MAX_DATA;
+	e_ring_size += ((uint32)sizeof(EWPF_event_elem_t)) * EWPF_REPORT_MAX_DATA;
+	k_ring_size += ((uint32)sizeof(EWPF_key_info_elem_t)) * EWPF_REPORT_MAX_DATA;
+
+	req_size = s_ring_size * EWPF_MAX_SLICE + i_ring_size * EWPF_MAX_IFACE +
+		e_ring_size * EWPF_MAX_EVENT + k_ring_size * EWPF_MAX_KEY_INFO;
+	req_size += (uint32)sizeof(EWP_filter_t);
+
+	if (buf_size < req_size) {
+		DHD_FILTER_ERR(("BUF SIZE IS TO SHORT: req:%d buf_size:%d\n",
+			req_size, buf_size));
+		return BCME_ERROR;
+	}
+
+	BCM_REFERENCE(dhdp);
+	filter = (EWP_filter_t *)buf;
+	buf_ptr += sizeof(EWP_filter_t);
+
+	/* initialize control block */
+	memset(filter, 0, sizeof(EWP_filter_t));
+
+	filter->idx_type = EWPF_INVALID;
+	filter->xtlv_idx = EWPF_INVALID;
+	filter->tmp_armcycle = 0;
+
+	for (idx = 0; idx < EWPF_MAX_SLICE; idx++) {
+		filter->s_ring[idx] = dhd_ring_init(dhdp, buf_ptr, s_ring_size,
+			sizeof(EWPF_slc_elem_t), EWPF_REPORT_MAX_DATA,
+			DHD_RING_TYPE_FIXED);
+		if (!filter->s_ring[idx]) {
+			DHD_FILTER_ERR(("FAIL TO INIT SLICE RING: %d\n", idx));
+			return BCME_ERROR;
+		}
+		buf_ptr += s_ring_size;
+	}
+
+	for (idx = 0; idx < EWPF_MAX_IFACE; idx++) {
+		filter->i_ring[idx] = dhd_ring_init(dhdp, buf_ptr, i_ring_size,
+			sizeof(EWPF_ifc_elem_t), EWPF_REPORT_MAX_DATA,
+			DHD_RING_TYPE_FIXED);
+		if (!filter->i_ring[idx]) {
+			DHD_FILTER_ERR(("FAIL TO INIT INTERFACE RING: %d\n", idx));
+			return BCME_ERROR;
+		}
+		buf_ptr += i_ring_size;
+	}
+
+	for (idx = 0; idx < EWPF_MAX_EVENT; idx++) {
+		filter->e_ring[idx] = dhd_ring_init(dhdp, buf_ptr, e_ring_size,
+			sizeof(EWPF_event_elem_t), EWPF_REPORT_MAX_DATA,
+			DHD_RING_TYPE_FIXED);
+		if (!filter->e_ring[idx]) {
+			DHD_FILTER_ERR(("FAIL TO INIT INTERFACE RING: %d\n", idx));
+			return BCME_ERROR;
+		}
+		buf_ptr += e_ring_size;
+	}
+
+	for (idx = 0; idx < EWPF_MAX_KEY_INFO; idx++) {
+		filter->k_ring[idx] = dhd_ring_init(dhdp, buf_ptr, k_ring_size,
+			sizeof(EWPF_key_info_elem_t), EWPF_REPORT_MAX_DATA,
+			DHD_RING_TYPE_FIXED);
+		if (!filter->k_ring[idx]) {
+			DHD_FILTER_ERR(("FAIL TO INIT INTERFACE RING: %d\n", idx));
+			return BCME_ERROR;
+		}
+		buf_ptr += k_ring_size;
+	}
+
+	dhdp->event_log_filter = filter;
+	filter->dhdp = dhdp;
+	filter->enabled = TRUE;
+
+	/*
+	 * put dummy element of event based encounters to prevent error
+	 * in case of no event happened when data collection is triggered
+	 */
+	ctx.dhdp = dhdp;
+	ctx.tbl = EWPF_main;
+	memset(&dummy_event_stat, 0x00, sizeof(dummy_event_stat));
+	evt_xtlv_copy_cb(&ctx, (uint8 *)&dummy_event_stat, WL_IFSTATS_XTLV_IF_EVENT_STATS,
+		sizeof(wl_event_based_statistics_v4_t));
+
+	memset(&dummy_key_update_info, 0x00, sizeof(dummy_key_update_info));
+	evt_xtlv_copy_cb(&ctx, (uint8 *)&dummy_key_update_info, WL_IFSTATS_XTLV_KEY_PLUMB_INFO,
+		sizeof(key_update_info_v1_t));
+
+#if defined(DHD_EWPR_VER2) && defined(DHD_STATUS_LOGGING)
+	/* create status filter for bigdata logging */
+	req.req_buf = dhd_statlog_filter;
+	req.req_buf_len = sizeof(dhd_statlog_filter);
+	dhd_statlog_generate_bdmask(dhdp, &req);
+#endif /* DHD_EWPR_VER2 && DHD_STATUS_LOGGING */
+
+	return BCME_OK;
+}
+
+void
+dhd_event_log_filter_deinit(dhd_pub_t *dhdp)
+{
+	EWP_filter_t *filter;
+	int idx;
+
+	if (!dhdp) {
+		return;
+	}
+
+	if (dhdp->event_log_filter) {
+		filter = (EWP_filter_t *)dhdp->event_log_filter;
+		for (idx = 0; idx < EWPF_MAX_SLICE; idx ++) {
+			dhd_ring_deinit(dhdp, filter->s_ring[idx]);
+		}
+		for (idx = 0; idx < EWPF_MAX_IFACE; idx ++) {
+			dhd_ring_deinit(dhdp, filter->i_ring[idx]);
+		}
+		for (idx = 0; idx < EWPF_MAX_EVENT; idx ++) {
+			dhd_ring_deinit(dhdp, filter->e_ring[idx]);
+		}
+		for (idx = 0; idx < EWPF_MAX_KEY_INFO; idx ++) {
+			dhd_ring_deinit(dhdp, filter->k_ring[idx]);
+		}
+		dhdp->event_log_filter = NULL;
+	}
+}
+
+void
+dhd_event_log_filter_notify_connect_request(dhd_pub_t *dhdp, uint8 *bssid, int channel)
+{
+	EWP_filter_t *filter;
+	void *last_elem;
+
+	if (!dhdp || !dhdp->event_log_filter) {
+		return;
+	}
+
+	filter = (EWP_filter_t *)dhdp->event_log_filter;
+	if (filter->enabled != TRUE) {
+		DHD_FILTER_ERR(("EWP Filter is not enabled\n"));
+		return;
+	}
+
+	memcpy(filter->last_bssid, bssid, ETHER_ADDR_LEN);
+	filter->last_channel = channel;
+
+	/* Refer STA interface */
+	last_elem = dhd_ring_get_last(filter->i_ring[0]);
+	if (last_elem == NULL) {
+		filter->last_armcycle = 0;
+	} else {
+		/* EXCLUDE before connect start */
+		filter->last_armcycle = *(uint32 *)last_elem + EWPF_EPOCH + 1;
+	}
+}
+
+void
+dhd_event_log_filter_notify_connect_done(dhd_pub_t *dhdp, uint8 *bssid, int roam)
+{
+	EWP_filter_t *filter;
+	void *last_elem;
+	int channel;
+	char buf[EWPF_DEBUG_BUF_LEN];
+	int ret;
+	uint32 armcycle;
+	struct channel_info *ci;
+
+	if (!dhdp || !dhdp->event_log_filter) {
+		return;
+	}
+
+	filter = (EWP_filter_t *)dhdp->event_log_filter;
+	if (filter->enabled != TRUE) {
+		DHD_FILTER_ERR(("EWP Filter is not enabled\n"));
+		return;
+	}
+
+	/* GET CHANNEL */
+	*(uint32 *)buf = htod32(EWPF_DEBUG_BUF_LEN);
+	ret = dhd_wl_ioctl_cmd(dhdp, WLC_GET_CHANNEL, buf, EWPF_DEBUG_BUF_LEN, FALSE, 0);
+	if (ret != BCME_OK) {
+		DHD_FILTER_ERR(("FAIL TO GET BSS INFO: %d\n", ret));
+		return;
+	}
+
+	ci = (struct channel_info *)(buf + sizeof(uint32));
+	channel = dtoh32(ci->hw_channel);
+	DHD_FILTER_TRACE(("CHANNEL:prev %d new:%d\n", filter->last_channel, channel));
+
+	memcpy(filter->last_bssid, bssid, ETHER_ADDR_LEN);
+	filter->last_channel = channel;
+	if (roam == FALSE) {
+		return;
+	}
+
+	/* update connect time for roam */
+	/* Refer STA interface */
+	last_elem = dhd_ring_get_last(filter->i_ring[0]);
+	if (last_elem == NULL) {
+		armcycle = 0;
+	} else {
+		/* EXCLUDE before roam done */
+		armcycle = *(uint32 *)last_elem + EWPF_EPOCH + 1;
+	}
+
+	filter->last_armcycle = armcycle;
+}
+
+static int
+evt_xtlv_print_cb(void *ctx, const uint8 *data, uint16 type, uint16 len)
+{
+	EWPF_ctx_t *cur_ctx = (EWPF_ctx_t *)ctx;
+	EWP_filter_t *filter = (EWP_filter_t *)cur_ctx->dhdp->event_log_filter;
+	uint32 armcycle = 0;
+	uint8 bssid[ETHER_ADDR_LEN];
+	uint32 initial_assoc_time = 0;
+	uint32 prev_roam_time = 0;
+	uint32 last_roam_event_type = 0;
+	uint32 last_roam_event_status = 0;
+	uint32 last_roam_event_reason = 0;
+	wl_wips_event_info_t wips_event;
+	bzero(&wips_event, sizeof(wips_event));
+
+	DHD_FILTER_TRACE(("%s type:%d %x len:%d %x\n", __FUNCTION__, type, type, len, len));
+
+	/* get current armcycle */
+	if (filter) {
+		armcycle = filter->tmp_armcycle;
+	}
+	if (type == WL_IFSTATS_XTLV_IF_EVENT_STATS) {
+		wl_event_based_statistics_v1_t *elem;
+
+		elem = (wl_event_based_statistics_v1_t *)(uintptr_t)data;
+		if (elem->txdeauthivalclass > 0) {
+			memcpy(bssid, &elem->BSSID, ETHER_ADDR_LEN);
+			DHD_ERROR(("DHD STA sent DEAUTH frame with invalid class : %d times"
+				", BSSID("MACDBG")\n", elem->txdeauthivalclass, MAC2STRDBG(bssid)));
+		}
+		if (elem->version == WL_EVENT_STATISTICS_VER_2) {
+			wl_event_based_statistics_v2_t *elem_v2;
+
+			elem_v2 = (wl_event_based_statistics_v2_t *)(uintptr_t)data;
+			memcpy(&wips_event.bssid, &elem_v2->last_deauth, ETHER_ADDR_LEN);
+			wips_event.misdeauth = elem_v2->misdeauth;
+			wips_event.current_RSSI = elem_v2->cur_rssi;
+			wips_event.deauth_RSSI = elem_v2->deauth_rssi;
+			wips_event.timestamp = elem_v2->timestamp;
+		} else if (elem->version == WL_EVENT_STATISTICS_VER_3) {
+			wl_event_based_statistics_v3_t *elem_v3;
+
+			elem_v3 = (wl_event_based_statistics_v3_t *)(uintptr_t)data;
+			memcpy(&wips_event.bssid, &elem_v3->last_deauth, ETHER_ADDR_LEN);
+			wips_event.misdeauth = elem_v3->misdeauth;
+			wips_event.current_RSSI = elem_v3->cur_rssi;
+			wips_event.deauth_RSSI = elem_v3->deauth_rssi;
+			wips_event.timestamp = elem_v3->timestamp;
+			/* roam statistics */
+			initial_assoc_time = elem_v3->initial_assoc_time;
+			prev_roam_time = elem_v3->prev_roam_time;
+			last_roam_event_type = elem_v3->last_roam_event_type;
+			last_roam_event_status = elem_v3->last_roam_event_status;
+			last_roam_event_reason = elem_v3->last_roam_event_reason;
+		} else if (elem->version == WL_EVENT_STATISTICS_VER_4) {
+			wl_event_based_statistics_v4_t *elem_v4;
+
+			elem_v4 = (wl_event_based_statistics_v4_t *)(uintptr_t)data;
+			memcpy(&wips_event.bssid, &elem_v4->last_deauth, ETHER_ADDR_LEN);
+			wips_event.misdeauth = elem_v4->misdeauth;
+			wips_event.current_RSSI = elem_v4->cur_rssi;
+			wips_event.deauth_RSSI = elem_v4->deauth_rssi;
+			wips_event.timestamp = elem_v4->timestamp;
+		}
+		if (wips_event.misdeauth > 1) {
+			DHD_ERROR(("WIPS attack!! cnt=%d, curRSSI=%d, deauthRSSI=%d "
+				", time=%d, MAC="MACDBG"\n",
+				wips_event.misdeauth, wips_event.current_RSSI,
+				wips_event.deauth_RSSI,	wips_event.timestamp,
+				MAC2STRDBG(&wips_event.bssid)));
+#if defined(WL_CFG80211) && defined(WL_WIPSEVT)
+			wl_cfg80211_wips_event_ext(&wips_event);
+#endif /* WL_CFG80211 && WL_WIPSEVT */
+		}
+	} else if (type == WL_IFSTATS_XTLV_ROAM_STATS_EVENT) {
+		wl_roam_stats_v1_t *roam_elem;
+		roam_elem = (wl_roam_stats_v1_t *)(uintptr_t)data;
+		if (roam_elem->version == WL_ROAM_STATS_VER_1) {
+			wl_roam_stats_v1_t *roam_elem_v1;
+
+			roam_elem_v1 = (wl_roam_stats_v1_t *)(uintptr_t)data;
+			/* roam statistics */
+			initial_assoc_time = roam_elem_v1->initial_assoc_time;
+			prev_roam_time = roam_elem_v1->prev_roam_time;
+			last_roam_event_type = roam_elem_v1->last_roam_event_type;
+			last_roam_event_status = roam_elem_v1->last_roam_event_status;
+			last_roam_event_reason = roam_elem_v1->last_roam_event_reason;
+		}
+	} else {
+		DHD_FILTER_ERR(("%s TYPE(%d) IS NOT SUPPORTED TO PRINT\n",
+			__FUNCTION__, type));
+		return BCME_ERROR;
+	}
+	if (initial_assoc_time > 0 && prev_roam_time > 0) {
+		DHD_ERROR(("Last roam event before disconnection : "
+			"current armcycle %d, initial assoc time %d, "
+			"last event time %d, type %d, status %d, reason %d\n",
+			armcycle, initial_assoc_time, prev_roam_time,
+			last_roam_event_type, last_roam_event_status,
+			last_roam_event_reason));
+	}
+
+	return BCME_OK;
+}
+
+#ifdef BCM_SDC
+static int
+evt_get_last_toss_hist(uint8 *ptr, const uint8 *data, uint16 len)
+{
+	bcm_xtlv_t *bcm_xtlv_desc = (bcm_xtlv_t *)data;
+	wl_hist_compact_toss_stats_v3_t *ewp_stats;
+	evt_hist_compact_toss_stats_v1_t bidata_stats;
+	int16 max_rcidx = EWPF_INVALID, secnd_rcidx = EWPF_INVALID;
+	uint16 cur_rnidx = 0, prev_rnidx = 0;
+	uint16 max_rccnt = 0, cur_rccnt = 0;
+	uint16 idx;
+
+	if (!ptr || !data) {
+		return BCME_ERROR;
+	}
+
+	if (bcm_xtlv_desc->len != sizeof(wl_hist_compact_toss_stats_v3_t)) {
+		DHD_FILTER_ERR(("%s : size is not matched  %d\n", __FUNCTION__,
+			bcm_xtlv_desc->len));
+		return BCME_ERROR;
+	}
+
+	ewp_stats = (wl_hist_compact_toss_stats_v3_t *)(&bcm_xtlv_desc->data[0]);
+	if (ewp_stats->htr_type == WL_STATE_HIST_TX_TOSS_REASONS) {
+		if (ewp_stats->version != WL_HIST_COMPACT_TOSS_STATS_TX_VER_3) {
+			DHD_FILTER_ERR(("%s : unsupported version %d (type: %d)\n",
+				__FUNCTION__, ewp_stats->version, ewp_stats->htr_type));
+			return BCME_ERROR;
+		}
+	} else if (ewp_stats->htr_type == WL_STATE_HIST_RX_TOSS_REASONS) {
+		if (ewp_stats->version != WL_HIST_COMPACT_TOSS_STATS_RX_VER_3) {
+			DHD_FILTER_ERR(("%s : unsupported version %d (type: %d)\n",
+				__FUNCTION__, ewp_stats->version, ewp_stats->htr_type));
+			return BCME_ERROR;
+		}
+	} else {
+		DHD_FILTER_ERR(("%s : unsupported type %d\n", __FUNCTION__,
+			ewp_stats->htr_type));
+		return BCME_ERROR;
+	}
+	/*
+	 * htr_rnidx is pointing the next empty slot to be used
+	 * Need to get previous index which is valid
+	 */
+	if (ewp_stats->htr_rnidx > 0) {
+		cur_rnidx = ewp_stats->htr_rnidx - 1;
+	} else {
+		cur_rnidx = WLC_HIST_TOSS_LEN - 1;
+	}
+	if (cur_rnidx > 0) {
+		prev_rnidx = cur_rnidx - 1;
+	} else {
+		prev_rnidx = WLC_HIST_TOSS_LEN - 1;
+	}
+	/*
+	 * Need to get largest count of toss reasons
+	 */
+	for (idx = 0; idx < WLC_HIST_TOSS_LEN; idx ++) {
+		cur_rccnt = (uint16)(ewp_stats->htr_rc[idx].toss_cnt);
+		DHD_FILTER_TRACE(("%s: idx %d htr_rc %04x cur_rccnt %d\n",
+			__FUNCTION__, idx, ewp_stats->htr_rc[idx].toss_reason, cur_rccnt));
+		if (ewp_stats->htr_rc_ts[idx] && max_rccnt < cur_rccnt) {
+			max_rccnt = cur_rccnt;
+			secnd_rcidx = max_rcidx;
+			max_rcidx = idx;
+			DHD_FILTER_TRACE(("%s: max_rcidx updated -"
+				"max_rcidx %d secnd_rcidx %d\n",
+				__FUNCTION__, max_rcidx, secnd_rcidx));
+		}
+	}
+
+	memset(&bidata_stats, 0x00, sizeof(bidata_stats));
+	bidata_stats.version = ewp_stats->version;
+	bidata_stats.htr_type = ewp_stats->htr_type;
+	bidata_stats.htr_num = ewp_stats->htr_num;
+	bidata_stats.htr_rn_last = *(uint32*)(&ewp_stats->htr_running[cur_rnidx]);
+	bidata_stats.htr_rn_ts_last = ewp_stats->htr_rn_ts[cur_rnidx];
+	bidata_stats.htr_rn_prev = *(uint32*)(&ewp_stats->htr_running[prev_rnidx]);
+	bidata_stats.htr_rn_ts_prev = ewp_stats->htr_rn_ts[prev_rnidx];
+	if (max_rcidx != EWPF_INVALID) {
+		bidata_stats.htr_rc_max = *(uint32*)(&ewp_stats->htr_rc[max_rcidx]);
+		bidata_stats.htr_rc_ts_max = ewp_stats->htr_rc_ts[max_rcidx];
+	}
+	if (secnd_rcidx != EWPF_INVALID) {
+		bidata_stats.htr_rc_secnd = *(uint32*)(&ewp_stats->htr_rc[secnd_rcidx]);
+		bidata_stats.htr_rc_ts_secnd = ewp_stats->htr_rc_ts[secnd_rcidx];
+	}
+	DHD_FILTER_TRACE(("%s: ver %d type %d num %d "
+		"htr_rn_last %d htr_rn_ts_last %d htr_rn_prev %d htr_rn_ts_prev %d "
+		"htr_rc_max %d htr_rc_ts_max %d htr_rc_secnd %d htr_rc_ts_secnd %d\n",
+		__FUNCTION__, bidata_stats.version,
+		bidata_stats.htr_type, bidata_stats.htr_num,
+		bidata_stats.htr_rn_last, bidata_stats.htr_rn_ts_last,
+		bidata_stats.htr_rn_prev, bidata_stats.htr_rn_ts_prev,
+		bidata_stats.htr_rc_max, bidata_stats.htr_rc_ts_max,
+		bidata_stats.htr_rc_secnd, bidata_stats.htr_rc_ts_secnd));
+
+	memcpy(ptr, &bidata_stats, sizeof(bidata_stats));
+
+	return BCME_OK;
+}
+#endif /* BCM_SDC */
+
+static int
+evt_xtlv_copy_cb(void *ctx, const uint8 *data, uint16 type, uint16 len)
+{
+	EWPF_ctx_t *cur_ctx = (EWPF_ctx_t *)ctx;
+	EWP_filter_t *filter = (EWP_filter_t *)cur_ctx->dhdp->event_log_filter;
+	uint32 *armcycle;
+	EWPF_tbl_t *tbl;
+	void *ring;
+	void *target;
+	uint8 *ptr;
+	int tbl_idx;
+	uint32 elem_size;
+
+	DHD_FILTER_TRACE(("%s type:%d %x len:%d %x\n", __FUNCTION__, type, type, len, len));
+
+	for (tbl_idx = 0; ; tbl_idx++) {
+		if (cur_ctx->tbl[tbl_idx].xtlv_id == EWPF_XTLV_INVALID) {
+			DHD_FILTER_ERR(("%s NOT SUPPORTED TYPE(%d)\n", __FUNCTION__, type));
+			return BCME_OK;
+		}
+		if (cur_ctx->tbl[tbl_idx].xtlv_id == type) {
+			tbl = &cur_ctx->tbl[tbl_idx];
+			break;
+		}
+	}
+
+	/* Set index type and xtlv_idx for event stats and key plumb info */
+	if (type == WL_IFSTATS_XTLV_IF_EVENT_STATS) {
+		filter->idx_type = EWPF_IDX_TYPE_EVENT;
+		filter->xtlv_idx = 0;
+		DHD_FILTER_TRACE(("EVENT XTLV\n"));
+	} else if (type == WL_IFSTATS_XTLV_KEY_PLUMB_INFO) {
+		filter->idx_type = EWPF_IDX_TYPE_KEY_INFO;
+		filter->xtlv_idx = 0;
+		DHD_FILTER_TRACE(("KEY INFO XTLV\n"));
+	}
+
+	/* Check Validation */
+	if (filter->idx_type == EWPF_INVALID ||
+		filter->xtlv_idx == EWPF_INVALID ||
+		filter->idx_type != tbl->idx_type ||
+		filter->xtlv_idx >= tbl->max_idx) {
+		DHD_FILTER_ERR(("XTLV VALIDATION FAILED: type:%x xtlv:%x idx:%d\n",
+			filter->idx_type, tbl->xtlv_id, filter->xtlv_idx));
+		return BCME_OK;
+	}
+
+	/* SET RING INFO */
+	if (filter->idx_type == EWPF_IDX_TYPE_SLICE) {
+		ring = filter->s_ring[filter->xtlv_idx];
+		elem_size = sizeof(EWPF_slc_elem_t);
+	} else if (filter->idx_type == EWPF_IDX_TYPE_IFACE) {
+		ring = filter->i_ring[filter->xtlv_idx];
+		elem_size = sizeof(EWPF_ifc_elem_t);
+	} else if (filter->idx_type == EWPF_IDX_TYPE_EVENT) {
+		DHD_FILTER_TRACE(("%s: EWPF_IDX_TYPE_EVENT FOUND\n",
+		__FUNCTION__));
+		ring = filter->e_ring[filter->xtlv_idx];
+		elem_size = sizeof(EWPF_event_elem_t);
+	} else if (filter->idx_type == EWPF_IDX_TYPE_KEY_INFO) {
+		DHD_FILTER_TRACE(("%s: EWPF_IDX_TYPE_KEY_INFO FOUND\n",
+		__FUNCTION__));
+		ring = filter->k_ring[filter->xtlv_idx];
+		elem_size = sizeof(EWPF_key_info_elem_t);
+	} else {
+		DHD_FILTER_TRACE(("%s unsupported idx_type:%d\n",
+			__FUNCTION__, filter->idx_type));
+		return BCME_OK;
+	}
+
+	/* Check armcycle epoch is changed */
+	target = dhd_ring_get_last(ring);
+	if (target != NULL) {
+		armcycle = (uint32 *)target;
+		if (*armcycle + EWPF_EPOCH <= filter->tmp_armcycle) {
+			/* EPOCH is changed (longer than 1sec) */
+			target = NULL;
+		} else if (*armcycle - EWPF_EPOCH >= filter->tmp_armcycle) {
+			/* dongle is rebooted */
+			target = NULL;
+		}
+	}
+
+	if (target == NULL) {
+		/* Get new idx */
+		target = dhd_ring_get_empty(ring);
+		if (target == NULL) {
+			/* no available slot due to oldest slot is locked */
+			DHD_FILTER_ERR(("SKIP to logging xltv(%x) due to locking\n", type));
+			return BCME_OK;
+		}
+
+		/* clean up target */
+		armcycle = (uint32 *)target;
+		memset(target, 0, elem_size);
+		memcpy(armcycle, &filter->tmp_armcycle, sizeof(*armcycle));
+	}
+
+#ifdef EWPF_DEBUG
+	DHD_FILTER_ERR(("idx:%d write_:%p %u %u\n",
+		filter->xtlv_idx, target, *armcycle, filter->tmp_armcycle));
+#endif
+
+	/* Additionally put updated armcycle for event based EWP */
+	if (filter->idx_type == EWPF_IDX_TYPE_EVENT ||
+		filter->idx_type == EWPF_IDX_TYPE_KEY_INFO) {
+		DHD_FILTER_TRACE(("%s: updated armcycle for event based EWP\n",
+		__FUNCTION__));
+		memcpy((uint32 *)(armcycle + EWPF_UPDATE_ARM_CYCLE_OFFSET),
+			&filter->tmp_armcycle, sizeof(*armcycle));
+	}
+
+	ptr = (uint8 *)target;
+
+#ifdef DHD_EWPR_VER2
+	if (tbl->xtlv_id == WL_SLICESTATS_XTLV_HIST_TX_STATS ||
+			tbl->xtlv_id == WL_SLICESTATS_XTLV_HIST_RX_STATS) {
+#ifdef BCM_SDC
+		int err;
+
+		DHD_FILTER_TRACE(("TOSS_REASONS received (%d)\n", tbl->xtlv_id));
+
+		err = evt_get_last_toss_hist(ptr + cur_ctx->tbl[tbl_idx].offset, data, len);
+		if (err) {
+			DHD_FILTER_ERR(("%s: get toss hist failed\n",
+				__FUNCTION__));
+			return BCME_ERROR;
+		}
+#else
+		DHD_FILTER_ERR(("%s: Unabled to copy hist TX/RX stats, BCM_SDC must be included\n",
+			__FUNCTION__));
+#endif /* BCM_SDC */
+	} else
+#endif /* DHD_EWPR_VER2 */
+	{
+		/* XXX multiversion shall be use same structure of old version */
+		if (len > cur_ctx->tbl[tbl_idx].member_length) {
+			DHD_FILTER_TRACE(("data Length is too big to save: (alloc = %d), "
+				"(data = %d)\n", cur_ctx->tbl[tbl_idx].member_length, len));
+			len = cur_ctx->tbl[tbl_idx].member_length;
+		}
+
+		memcpy(ptr + cur_ctx->tbl[tbl_idx].offset, data, len);
+	}
+	return BCME_OK;
+}
+
+static int
+evt_xtlv_idx_cb(void *ctx, const uint8 *data, uint16 type, uint16 len)
+{
+	EWPF_ctx_t *cur_ctx = (EWPF_ctx_t *)ctx;
+	EWP_filter_t *filter = (EWP_filter_t *)cur_ctx->dhdp->event_log_filter;
+
+	filter->xtlv_idx = data[0];
+
+	if (filter->idx_type == EWPF_IDX_TYPE_SLICE) {
+		if (type != WL_IFSTATS_XTLV_SLICE_INDEX ||
+			filter->xtlv_idx >= EWPF_MAX_SLICE) {
+			goto idx_fail;
+		}
+	} else if (filter->idx_type == EWPF_IDX_TYPE_IFACE) {
+		if (type != WL_IFSTATS_XTLV_IF_INDEX ||
+			filter->xtlv_idx >= EWPF_MAX_IFACE) {
+			DHD_FILTER_ERR(("CHANGE IFACE TO 0 in FORCE\n"));
+			return BCME_OK;
+		}
+	} else {
+		goto idx_fail;
+	}
+	return BCME_OK;
+
+idx_fail:
+	DHD_FILTER_ERR(("UNEXPECTED IDX XTLV: filter_type:%d input_type%x idx:%d\n",
+		filter->idx_type, type, filter->xtlv_idx));
+	filter->idx_type = EWPF_INVALID;
+	filter->xtlv_idx = EWPF_INVALID;
+	return BCME_OK;
+}
+
+static int
+evt_xtlv_type_cb(void *ctx, const uint8 *data, uint16 type, uint16 len)
+{
+	EWPF_ctx_t *cur_ctx = (EWPF_ctx_t *)ctx;
+	EWP_filter_t *filter = (EWP_filter_t *)cur_ctx->dhdp->event_log_filter;
+
+	if (type == WL_IFSTATS_XTLV_WL_SLICE) {
+		filter->idx_type = EWPF_IDX_TYPE_SLICE;
+		DHD_FILTER_TRACE(("SLICE XTLV\n"));
+	} else if (type == WL_IFSTATS_XTLV_IF) {
+		filter->idx_type = EWPF_IDX_TYPE_IFACE;
+		DHD_FILTER_TRACE(("IFACE XTLV\n"));
+	}
+
+	bcm_unpack_xtlv_buf(ctx, data, len,
+		BCM_XTLV_OPTION_ALIGN32, filter_main_cb);
+	return BCME_OK;
+}
+
+static int
+evt_xtlv_roam_cb(void *ctx, const uint8 *data, uint16 type, uint16 len)
+{
+
+	EWPF_ctx_t *cur_ctx = (EWPF_ctx_t *)ctx;
+	EWPF_tbl_t *new_tbl = EWPF_roam;
+	EWPF_ctx_t sub_ctx;
+	int idx;
+
+	for (idx = 0; ; idx++) {
+		if (new_tbl[idx].xtlv_id == EWPF_XTLV_INVALID) {
+			DHD_FILTER_TRACE(("%s NOT SUPPORTED TYPE(%d)\n", __FUNCTION__, type));
+			return BCME_OK;
+		}
+		if (new_tbl[idx].xtlv_id == type) {
+			break;
+		}
+	}
+
+	/* MULTI version may not applied */
+	if (len > sizeof(cur_ctx->dhdp->roam_evt)) {
+		DHD_FILTER_ERR(("data length is too big :max= %d, cur=%d\n",
+				(int)sizeof(cur_ctx->dhdp->roam_evt), len));
+		len = sizeof(cur_ctx->dhdp->roam_evt);
+	}
+
+	/* save latest roam event to report via get_bss_info */
+	(void)memcpy_s((char *)&cur_ctx->dhdp->roam_evt, sizeof(cur_ctx->dhdp->roam_evt),
+			data, len);
+
+	sub_ctx.dhdp = cur_ctx->dhdp;
+	sub_ctx.tbl = new_tbl;
+	new_tbl[idx].cb_func(&sub_ctx, data, type, len);
+	return BCME_OK;
+}
+
+static int
+filter_main_cb(void *ctx, const uint8 *data, uint16 type, uint16 len)
+{
+	EWPF_ctx_t *cur_ctx = (EWPF_ctx_t *)ctx;
+	EWPF_ctx_t sub_ctx;
+	int idx;
+	int err = BCME_OK;
+
+	DHD_FILTER_TRACE(("%s type:%x len:%d\n", __FUNCTION__, type, len));
+
+	sub_ctx.dhdp = cur_ctx->dhdp;
+	for (idx = 0; ; idx++) {
+		if (cur_ctx->tbl[idx].xtlv_id == EWPF_XTLV_INVALID) {
+			DHD_FILTER_TRACE(("%s NOT SUPPORTED TYPE(%d)\n", __FUNCTION__, type));
+			return BCME_OK;
+		}
+		if (cur_ctx->tbl[idx].xtlv_id == type) {
+			/* parse sub xtlv */
+			if (cur_ctx->tbl[idx].cb_func == NULL) {
+				sub_ctx.tbl = cur_ctx->tbl[idx].tbl;
+				err = bcm_unpack_xtlv_buf(&sub_ctx, data, len,
+						BCM_XTLV_OPTION_ALIGN32, filter_main_cb);
+				return err;
+			}
+
+			/* handle for structure/variable */
+			err = cur_ctx->tbl[idx].cb_func(ctx, data, type, len);
+			if (err != BCME_OK) {
+				return err;
+			}
+		}
+	}
+
+	return err;
+}
+
+void
+dhd_event_log_filter_event_handler(dhd_pub_t *dhdp, prcd_event_log_hdr_t *plog_hdr, uint32 *data)
+{
+	int err;
+	EWP_filter_t *filter;
+	EWPF_ctx_t ctx;
+
+	if (!dhdp->event_log_filter) {
+		DHD_FILTER_ERR(("NO FILTER MODULE\n"));
+		return;
+	}
+
+	if (!plog_hdr || !data) {
+		/* XXX Validation check is done by caller */
+		DHD_FILTER_ERR(("INVALID PARAMETER\n"));
+		return;
+	}
+
+	filter = (EWP_filter_t *)dhdp->event_log_filter;
+	if (filter->enabled != TRUE) {
+		DHD_FILTER_ERR(("FITLER IS NOT STARTED\n"));
+		return;
+	}
+
+	/* get ARMCYCLE */
+	filter->tmp_armcycle = plog_hdr->armcycle;
+	filter->idx_type = EWPF_INVALID;
+	filter->xtlv_idx = EWPF_INVALID;
+
+#ifdef EWPF_DEBUG
+	{
+		char buf[EWPF_DEBUG_BUF_LEN];
+		int idx;
+
+		memset(buf, 0, sizeof(buf));
+		DHD_FILTER_ERR(("tag %d(%x) count %d(%x)\n",
+			plog_hdr->tag, plog_hdr->tag, plog_hdr->count, plog_hdr->count));
+		for (idx = 0; idx < plog_hdr->count; idx++) {
+			sprintf(&buf[strlen(buf)], "%08x ", data[idx]);
+			if ((idx + 1) % EWPF_VAL_CNT_PLINE == 0) {
+				DHD_FILTER_ERR(("%s\n", buf));
+				memset(buf, 0, sizeof(buf));
+			}
+		}
+		if (strlen(buf) > 0) {
+			DHD_FILTER_ERR(("%s\n", buf));
+		}
+	}
+#endif /* EWPF_DEBUG */
+
+	ctx.dhdp = dhdp;
+	ctx.tbl = EWPF_main;
+	if ((err = bcm_unpack_xtlv_buf(
+		&ctx,
+		(const uint8 *)data,
+		(plog_hdr->count - 1) * sizeof(uint32),
+		BCM_XTLV_OPTION_ALIGN32,
+		filter_main_cb))) {
+		DHD_FILTER_ERR(("FAIL TO UNPACK XTLV: err(%d)\n", err));
+	}
+}
+/* ========= Private Command(Serialize) ============= */
+/* XXX REPORT MODULE will be done after discuss with customer */
+/* XXX Current implementation is temporal to verify FILTER MODULE works */
+//#define EWPR_DEBUG
+#ifdef EWPR_DEBUG
+#undef DHD_FILTER_TRACE
+#define DHD_FILTER_TRACE DHD_FILTER_ERR
+#endif /* EWPR_DEBUG */
+#define EWPR_DEBUG_BUF_LEN	512
+
+#define EWP_REPORT_ELEM_PRINT_BUF	256
+#define EWP_REPORT_NAME_MAX		64
+
+#ifdef DHD_EWPR_VER2
+#define EWP_REPORT_VERSION	0x20200722
+#define EWP_REPORT_SET_DEFAULT	0x01
+#define EWPR_CSDCLIENT_DIFF	10
+#define EWPR_INTERVAL	3
+#define EWPR_DELTA_CNT	1	/* 3 seconds before */
+#define EWPR_ARRAY_CNT	10	/* INTERVAL * ARRAY total 30 seconds to lock */
+#define EWPR_DELTA_LAST_POS		6
+
+#define INDEX_STR_SIZE		6
+#define	DHD_STAT_STR_SIZE	2
+#define REPORT_VERSION_STR_SIZE	8
+#define DELIMITER_LEN		1
+#else
+#define EWP_REPORT_VERSION	0x20170905
+#define EWPR_CSDCLIENT_DIFF	4
+#define EWPR_INTERVAL	3
+#define EWPR_ARRAY_CNT	10	/* INTERVAL * ARRAY total 30 seconds to lock */
+#endif /* DHD_EWPR_VER2 */
+
+#define EWPR_DELTA3_POS		3
+#define EWPR_DELTA2_POS		2
+#define EWPR_DELTA1_POS		1
+#define EWPR_NOW_POS		0
+
+#define EWPR_DELTA1_CNT	2	/* 6 seconds before */
+#define EWPR_DELTA2_CNT	5	/* 15 seconds before */
+#define EWPR_DELTA3_CNT	9	/* 27 seconds before */
+
+#define EWPR_CNT_PER_LINE	5
+
+/* EWP Reporter display format */
+#define EWP_DEC	1
+#define EWP_HEX	2
+#define EWP_BIN	3
+
+/* EWP Filter Data type */
+/* BASIC : signed + length */
+#define EWP_UINT8	2
+#define EWP_UINT16	4
+#define EWP_UINT32	8
+#define EWP_UINT64	16
+#define EWP_INT8	102
+#define EWP_INT16	104
+#define EWP_INT32	108
+#define EWP_BIT		201
+
+/* NON BAISC : need special handling */
+#define EWP_NON_BASIC	200
+#define EWP_DATE		201
+#define EWP_TIME		202
+#define EWP_BSSID		203
+#define EWP_OUI			204
+
+/* Delimiter between values */
+#define KEY_DEL	' '
+#define RAW_DEL '_'
+
+/* IOVAR BUF SIZE */
+#define EWPR_IOV_BUF_LEN	64
+
+typedef struct {
+	void *ring;				/* INPUT ring to lock */
+	void **elem_list;		/* OUTPUT elem ptr list for each delta */
+	uint32 max_armcycle;	/* IN/OUT arm cycle should be less than this */
+	uint32 min_armcycle;	/* IN/OUT arm cycle should be bigger than this */
+	uint32 max_period;		/* IN allowed time diff between first and last */
+	uint32 delta_cnt;		/* IN finding delta count */
+	uint32 *delta_list;		/* IN delta values to find */
+} ewpr_lock_param_t;
+
+#define MAX_MULTI_VER	4
+typedef struct {
+	uint32	version;		/* VERSION for multiple version struct */
+	uint32	offset;			/* offset of the member at the version */
+} ewpr_MVT_offset_elem_t;	/* elem for multi version type */
+
+typedef struct {
+	uint32	version_offset;		/* offset of version */
+	ewpr_MVT_offset_elem_t opv[MAX_MULTI_VER];	/* offset per version */
+} ewpr_MVT_offset_t;			/* multi_version type */
+
+typedef struct {
+	char name[EWP_REPORT_NAME_MAX];
+	int ring_type;		/* Ring Type : EWPF_IDX_TYPE_SLICE, EWPF_IDX_TYPE_IFACE */
+	int	is_multi_version;		/* is multi version */
+	union {
+		uint32 offset;			/* Offset from start of element structure */
+		ewpr_MVT_offset_t v_info;
+	};
+	int data_type;				/* Data type : one of EWP Filter Data Type */
+	int display_format;			/* Display format : one of EWP Reporter display */
+	int display_type;			/* MAX display BYTE : valid for HEX and BIN FORM */
+#ifdef DHD_EWPR_VER2
+	int info_type;			/* info type : EWPF_INFO_ECNT, EWPF_INFO_IOVAR, ... */
+	int display_bit_length;		/* packing bit : valid for BIN FORM */
+	int display_array_size;		/* array size */
+	int display_method;		/* serial or diff */
+	int unit_convert;		/* unit conversion
+					 * 0 or 1 : no conversion, put data as is
+					 * greater than 1, divide value by unit_convert
+					 */
+	bool need_abs;			/* need absolute function for negative value */
+#endif /* DHD_EWPR_VER2 */
+} ewpr_serial_info_t;
+
+/* offset defines */
+#define EWPR_CNT_VERSION_OFFSET \
+	OFFSETOF(EWPF_slc_elem_t, compact_cntr_v4)
+
+#define EWPR_CNT_V1_OFFSET(a) \
+	WL_PERIODIC_COMPACT_CNTRS_VER_1, \
+	(OFFSETOF(EWPF_slc_elem_t, compact_cntr_v1) + OFFSETOF(wl_periodic_compact_cntrs_v1_t, a))
+#define EWPR_CNT_V2_OFFSET(a) \
+	WL_PERIODIC_COMPACT_CNTRS_VER_2, \
+	(OFFSETOF(EWPF_slc_elem_t, compact_cntr_v2) + OFFSETOF(wl_periodic_compact_cntrs_v2_t, a))
+#define EWPR_CNT_V3_OFFSET(a) \
+	WL_PERIODIC_COMPACT_CNTRS_VER_3, \
+	(OFFSETOF(EWPF_slc_elem_t, compact_cntr_v3) + OFFSETOF(wl_periodic_compact_cntrs_v3_t, a))
+#define EWPR_CNT_V4_OFFSET(a) \
+	WL_PERIODIC_COMPACT_CNTRS_VER_4, \
+	(OFFSETOF(EWPF_slc_elem_t, compact_cntr_v4) + OFFSETOF(wl_periodic_compact_cntrs_v4_t, a))
+#define EWPR_STAT_OFFSET(a) \
+	(OFFSETOF(EWPF_ifc_elem_t, if_stat) + OFFSETOF(wl_if_stats_t, a))
+#define EWPR_INFRA_OFFSET(a) \
+	(OFFSETOF(EWPF_ifc_elem_t, infra) + OFFSETOF(wl_if_infra_stats_t, a))
+#define EWPR_MGMT_OFFSET(a) \
+	(OFFSETOF(EWPF_ifc_elem_t, mgmt_stat) + OFFSETOF(wl_if_mgt_stats_t, a))
+#define EWPR_LQM_OFFSET(a) \
+	(OFFSETOF(EWPF_ifc_elem_t, lqm) + OFFSETOF(wl_lqm_t, a))
+#define EWPR_SIGNAL_OFFSET(a) \
+	(EWPR_LQM_OFFSET(current_bss) + OFFSETOF(wl_rx_signal_metric_t, a))
+#define EWPR_IF_COMP_OFFSET(a) \
+	(OFFSETOF(EWPF_ifc_elem_t, if_comp_stat) + OFFSETOF(wl_if_state_compact_t, a))
+#define EWPR_EVENT_COUNTER_OFFSET(a) \
+	(OFFSETOF(EWPF_event_elem_t, event_stat) + OFFSETOF(wl_event_based_statistics_v4_t, a))
+#define EWPR_KEY_INFO_OFFSET(a) \
+	(OFFSETOF(EWPF_key_info_elem_t, key_update_info) + OFFSETOF(key_update_info_v1_t, a))
+#define EWPR_TX_TOSS_HIST_OFFSET(a) \
+	(OFFSETOF(EWPF_slc_elem_t, hist_tx_toss_stat) + \
+		OFFSETOF(evt_hist_compact_toss_stats_v1_t, a))
+#define EWPR_RX_TOSS_HIST_OFFSET(a) \
+	(OFFSETOF(EWPF_slc_elem_t, hist_rx_toss_stat) + \
+		OFFSETOF(evt_hist_compact_toss_stats_v1_t, a))
+#define EWPR_BTC_STAT_OFFSET(a) \
+	(OFFSETOF(EWPF_slc_elem_t, btc_stat) + \
+		OFFSETOF(wlc_btc_stats_v4_t, a))
+#define EWPR_COMPACT_HE_CNT_OFFSET(a) \
+	(OFFSETOF(EWPF_slc_elem_t, compact_he_cnt) + \
+		OFFSETOF(wl_compact_he_cnt_wlc_v2_t, a))
+#define EWPR_ROAM_STATS_PERIODIC_OFFSET(a) \
+	(OFFSETOF(EWPF_ifc_elem_t, roam_stat) + OFFSETOF(wl_roam_stats_v1_t, a))
+
+/* serail info type define */
+#define EWPR_SERIAL_CNT(a) {\
+	#a, EWPF_IDX_TYPE_SLICE, TRUE, \
+	.v_info = { EWPR_CNT_VERSION_OFFSET, \
+		{{EWPR_CNT_V1_OFFSET(a)}, \
+		{EWPR_CNT_V2_OFFSET(a)}, \
+		{EWPR_CNT_V3_OFFSET(a)}, \
+		{EWPR_CNT_V4_OFFSET(a)}}}, \
+	EWP_UINT32, EWP_HEX, EWP_UINT32}
+#define EWPR_SERIAL_CNT_16(a) {\
+	#a, EWPF_IDX_TYPE_SLICE, TRUE, \
+	.v_info = { EWPR_CNT_VERSION_OFFSET, \
+		{{EWPR_CNT_V1_OFFSET(a)}, \
+		{EWPR_CNT_V2_OFFSET(a)}, \
+		{EWPR_CNT_V3_OFFSET(a)}, \
+		{EWPR_CNT_V4_OFFSET(a)}}}, \
+	EWP_UINT32, EWP_HEX, EWP_UINT16}
+#define EWPR_SERIAL_STAT(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_STAT_OFFSET(a), \
+	EWP_UINT64, EWP_HEX, EWP_UINT32}
+#define EWPR_SERIAL_INFRA(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_INFRA_OFFSET(a), \
+	EWP_UINT32, EWP_HEX, EWP_UINT16}
+#define EWPR_SERIAL_MGMT(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_MGMT_OFFSET(a), \
+	EWP_UINT32, EWP_HEX, EWP_UINT16}
+#define EWPR_SERIAL_LQM(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_LQM_OFFSET(a), \
+	EWP_INT32, EWP_DEC, EWP_INT8}
+#define EWPR_SERIAL_SIGNAL(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_SIGNAL_OFFSET(a), \
+	EWP_INT32, EWP_DEC, EWP_INT8}
+#define EWPR_SERIAL_IFCOMP_8(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_IF_COMP_OFFSET(a), \
+	EWP_INT8, EWP_DEC, EWP_INT8}
+#define EWPR_SERIAL_IFCOMP_16(a) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_IF_COMP_OFFSET(a), \
+	EWP_UINT16, EWP_DEC, EWP_UINT16}
+#define EWPR_SERIAL_ARM(a) {\
+	"armcycle:" #a, EWPF_IDX_TYPE_##a, FALSE, {0, }, \
+	EWP_UINT32, EWP_DEC, EWP_UINT32}
+#define EWPR_SERIAL_NONE {"", EWPF_INVALID, FALSE, {0, }, 0, 0, 0}
+
+#ifdef DHD_EWPR_VER2
+
+#define RAW_BUFFER_SIZE		720u
+#define BASE64_BUFFER_SIZE	960u /* 33 percent larger than original binary data */
+#define EWPR_HEADER_SIZE	39u
+#define	EWPR_MAX_STR_SIZE	EWPR_HEADER_SIZE + EWPR_HEADER_SIZE
+
+#define EWPR_DISPLAY_METHOD_SINGLE	0
+#define EWPR_DISPLAY_METHOD_DIFF	1
+
+#define MAX_BIT_SIZE	8
+#define MAX_BIT_SHIFT	7
+
+#define INDEX_UNSPECIFIED 0u
+
+enum ewpr_context_type {
+	EWP_CONTEXT_TYPE_UNWANTED_NETWORK = 0,
+	EWP_CONTEXT_TYPE_ASSOC_FAIL = 1,
+	EWP_CONTEXT_TYPE_ABNORMAL_DISCONNECT = 2,
+	EWP_CONTEXT_TYPE_MAX = 3
+};
+
+enum ewpr_unwanted_net_sub_type {
+	EWP_UNWANT_NET_SUB_TYPE_UNSPECIFIED = 0,
+	EWP_UNWANT_NET_SUB_TYPE_ARP_FAIL = 1,
+	EWP_UNWANT_NET_SUB_TYPE_TXBAD = 2,
+	EWP_UNWANT_NET_SUB_TYPE_MAX = 3
+};
+
+enum ewpr_assoc_fail_sub_type {
+	EWP_ASSOC_FAIL_SUB_TYPE_UNSPECIFIED = 0,
+	EWP_ASSOC_FAIL_SUB_TYPE_DHCP_FAIL = 1,
+	EWP_ASSOC_FAIL_SUB_TYPE_EAP_FAIL = 2,
+	EWP_ASSOC_FAIL_SUB_TYPE_EAP_TIMEOUT = 3,
+	EWP_ASSOC_FAIL_SUB_TYPE_4WAY_FAIL = 4,
+	EWP_ASSOC_FAIL_SUB_TYPE_MAX = 5
+};
+
+enum ewpr_abnormal_disconnect_sub_type {
+	EWP_ABNRML_DISCONNCET_SUB_TYPE_UNSPECIFIED = 0,
+	EWP_ABNRML_DISCONNCET_SUB_TYPE_DISCONNECT_BY_HOST = 1,
+	EWP_ABNRML_DISCONNCET_SUB_TYPE_MAX = 2
+};
+
+typedef struct {
+	uint32 index1;
+	uint32 index2;
+	uint32 index3;
+	ewpr_serial_info_t *table;
+} ewpr_serial_context_info_t;
+
+#define EWPR_SINGLE_DEFAULT EWPR_DISPLAY_METHOD_SINGLE, EWPF_NO_UNIT_CONV
+#define EWPR_DIFF_DEFAULT EWPR_DISPLAY_METHOD_DIFF, EWPF_NO_UNIT_CONV
+
+#define EWPR_SINGLE_NSEC_TO_MSEC EWPR_DISPLAY_METHOD_SINGLE, EWPF_NSEC_TO_MSEC
+#define EWPR_SINGLE_USEC_TO_MSEC EWPR_DISPLAY_METHOD_SINGLE, EWPF_USEC_TO_MSEC
+#define EWPR_SINGLE_USEC_TO_SEC EWPR_DISPLAY_METHOD_SINGLE, EWPF_USEC_TO_SEC
+
+/* serail info type define */
+#define EWPR_SERIAL_CNT_V4_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_SLICE, TRUE, \
+	.v_info = { EWPR_CNT_VERSION_OFFSET, \
+		{{EWPR_CNT_V4_OFFSET(a)}}}, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_STAT_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_STAT_OFFSET(a), \
+	EWP_UINT64, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_INFRA_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_INFRA_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_MGMT_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_MGMT_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_LQM_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_LQM_OFFSET(a), \
+	EWP_INT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_SIGNAL_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_SIGNAL_OFFSET(a), \
+	EWP_INT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_IFCOMP_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_IF_COMP_OFFSET(a), \
+	EWP_INT8, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_EVENT_COUNTER_16_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_EVENT, FALSE, .offset = EWPR_EVENT_COUNTER_OFFSET(a), \
+	EWP_UINT16, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_EVENT_COUNTER_32_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_EVENT, FALSE, .offset = EWPR_EVENT_COUNTER_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_KEY_INFO_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_KEY_INFO, FALSE, .offset = EWPR_KEY_INFO_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_ROAM_STATS_PERIODIC_OFFSET(a), \
+	EWP_UINT16, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_IFACE, FALSE, .offset = EWPR_ROAM_STATS_PERIODIC_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_ARM_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_##a, FALSE, {0, }, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_IOVAR_BIT(a, b) {\
+	#a, 0, 0, .offset = 0, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_IOVAR, b, 1, EWPR_SINGLE_DEFAULT}
+#define EWPR_SERIAL_VERSION_BIT(a, b) {\
+	#a, 0, 0, .offset = 0, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_VER, b, 1, EWPR_SINGLE_DEFAULT}
+#define EWPR_SERIAL_TYPE_BIT(a, b) {\
+	#a, 0, 0, .offset = 0, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_TYPE, b, 1, EWPR_SINGLE_DEFAULT}
+#define EWPR_SERIAL_CPLOG_BIT(a, b, c) {\
+	#a, 0, 0, .offset = 0, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_CPLOG, b, c, EWPR_SINGLE_DEFAULT}
+#define EWPR_SERIAL_DHDSTAT_BIT(a, b, c, d) {\
+	#a, 0, 0, .offset = 0, \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_DHDSTAT, b, c, d}
+#define EWPR_SERIAL_TX_TOSS_HIST_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_SLICE, FALSE, .offset = EWPR_TX_TOSS_HIST_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_RX_TOSS_HIST_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_SLICE, FALSE, .offset = EWPR_RX_TOSS_HIST_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_BTC_STAT_16_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_SLICE, FALSE, .offset = EWPR_BTC_STAT_OFFSET(a), \
+	EWP_UINT16, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_BTC_STAT_32_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_SLICE, FALSE, .offset = EWPR_BTC_STAT_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+#define EWPR_SERIAL_COMPACT_HE_CNT_BIT(a, b, c, d) {\
+	#a, EWPF_IDX_TYPE_SLICE, FALSE, .offset = EWPR_COMPACT_HE_CNT_OFFSET(a), \
+	EWP_UINT32, EWP_BIN, EWP_BIT, EWPF_INFO_ECNT, b, c, d}
+
+#define EWPR_SERIAL_NONE_BIT {"", EWPF_INVALID, FALSE, {0, }, 0, 0, 0, 0, 0, 0, 0}
+
+#ifdef EWPR_DEBUG
+static void ewpr_print_byte_as_bits(char val);
+#endif /* EWPR_DEBUG */
+
+static int32
+ewpr_diff_bit_pack(ewpr_serial_info_t *info, char *buf, int buf_len,
+	void *_f_op, void *_s_op, int32 bit_offset);
+static int32
+ewpr_single_bit_pack(ewpr_serial_info_t *info, char *buf, int buf_len,
+	void *_ptr, int32 bit_offset);
+static int32
+ewpr_bit_pack_basic(char *buf, int buf_len, uint32 data, int32 format,
+	int32 display_type, int32 display_bit_length, int32 bit_offset);
+
+static char*
+ewpr_base64_encode(dhd_pub_t *dhdp, char* input, int32 length);
+#endif /* DHD_EWPR_VER2 */
+
+ewpr_serial_info_t
+ewpr_serial_CSDCLIENT_key_tbl[] = {
+	EWPR_SERIAL_STAT(txframe),
+	EWPR_SERIAL_STAT(txerror),
+	EWPR_SERIAL_STAT(rxframe),
+	EWPR_SERIAL_STAT(rxerror),
+	EWPR_SERIAL_STAT(txretrans),
+	EWPR_SERIAL_INFRA(rxbeaconmbss),
+	EWPR_SERIAL_CNT(txallfrm),
+	EWPR_SERIAL_CNT(rxrsptmout),
+	EWPR_SERIAL_CNT(rxbadplcp),
+	EWPR_SERIAL_CNT(rxcrsglitch),
+	EWPR_SERIAL_CNT(rxbadfcs),
+	EWPR_SERIAL_CNT_16(rxbeaconmbss),
+	EWPR_SERIAL_CNT_16(rxbeaconobss),
+	EWPR_SERIAL_NONE
+};
+
+ewpr_serial_info_t
+ewpr_serial_CSDCLIENT_diff_tbl[] = {
+	EWPR_SERIAL_STAT(txframe),
+	EWPR_SERIAL_STAT(txerror),
+	EWPR_SERIAL_STAT(rxframe),
+	EWPR_SERIAL_STAT(rxerror),
+	EWPR_SERIAL_STAT(txretrans),
+	EWPR_SERIAL_INFRA(rxbeaconmbss),
+	EWPR_SERIAL_MGMT(txassocreq),
+	EWPR_SERIAL_MGMT(txreassocreq),
+	EWPR_SERIAL_MGMT(txdisassoc),
+	EWPR_SERIAL_MGMT(rxdisassoc),
+	EWPR_SERIAL_MGMT(rxassocrsp),
+	EWPR_SERIAL_MGMT(rxreassocrsp),
+	EWPR_SERIAL_MGMT(txauth),
+	EWPR_SERIAL_MGMT(rxauth),
+	EWPR_SERIAL_MGMT(txdeauth),
+	EWPR_SERIAL_MGMT(rxdeauth),
+	EWPR_SERIAL_MGMT(txaction),
+	EWPR_SERIAL_MGMT(rxaction),
+	EWPR_SERIAL_CNT(txallfrm),
+	EWPR_SERIAL_CNT(rxrsptmout),
+	EWPR_SERIAL_CNT(rxbadplcp),
+	EWPR_SERIAL_CNT(rxcrsglitch),
+	EWPR_SERIAL_CNT(rxbadfcs),
+	EWPR_SERIAL_CNT_16(rxbeaconmbss),
+	EWPR_SERIAL_CNT_16(rxbeaconobss),
+	EWPR_SERIAL_NONE
+};
+
+ewpr_serial_info_t
+ewpr_serial_CSDCLIENT_array_tbl[] = {
+	EWPR_SERIAL_IFCOMP_8(rssi_sum),
+	EWPR_SERIAL_IFCOMP_8(snr),
+	EWPR_SERIAL_IFCOMP_8(noise_level),
+	EWPR_SERIAL_NONE
+};
+
+#ifdef EWPR_DEBUG
+ewpr_serial_info_t
+ewpr_serial_dbg_tbl[] = {
+	EWPR_SERIAL_ARM(IFACE),
+	EWPR_SERIAL_ARM(SLICE),
+	EWPR_SERIAL_NONE
+};
+#endif /* EWPR_DEBUG */
+
+#ifdef DHD_EWPR_VER2
+
+ewpr_serial_info_t
+ewpr_serial_bit_unwanted_network_default_tbl[] = {
+	EWPR_SERIAL_VERSION_BIT(version, 32),
+	EWPR_SERIAL_TYPE_BIT(type, 5),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_last_ts, 32, 1, EWPR_SINGLE_USEC_TO_SEC),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_last, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_prev_ts, 32, 1, EWPR_SINGLE_USEC_TO_SEC),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_prev, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IOVAR_BIT(auth, 8),
+	EWPR_SERIAL_IOVAR_BIT(wsec, 8),
+	EWPR_SERIAL_IOVAR_BIT(mfp, 1),
+	EWPR_SERIAL_IOVAR_BIT(bip, 8),
+	EWPR_SERIAL_ARM_BIT(IFACE, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txframe, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txerror, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(rxframe, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(rxerror, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txretrans, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txassocreq, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txreassocreq, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txdisassoc, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxdisassoc, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxassocrsp, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxreassocrsp, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txdeauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxdeauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txaction, 7, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxaction, 7, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(txallfrm, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxrsptmout, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbadplcp, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxcrsglitch, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbadfcs, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbeaconmbss, 5, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbeaconobss, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(lqcm_report, 19, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(tx_toss_cnt, 18, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rx_toss_cnt, 18, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxretry, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxdup, 15, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(chswitch_cnt, 8, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pm_dur, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxholes, 15, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxundec, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxundec_mcst, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(replay, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(replay_mcst, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pktfilter_discard, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pktfilter_forward, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(mac_rxfilter, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_dcsn_map, 16, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_dcsn_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_a2dp_hiwat_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_datadelay_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_crtpri_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_pri_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf5cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf6cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf7cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_32_BIT(bt_gnt_dur, 12, 3, EWPR_SINGLE_USEC_TO_MSEC),
+	EWPR_SERIAL_IFCOMP_BIT(rssi_sum, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IFCOMP_BIT(snr, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IFCOMP_BIT(noise_level, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(initial_assoc_time, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(prev_roam_time, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(last_roam_event_type, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(last_roam_event_status, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(last_roam_event_reason, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(roam_success_cnt, 10, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(roam_fail_cnt, 10, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(roam_attempt_cnt, 11, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(max_roam_target_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(min_roam_target_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(max_cached_ch_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(min_cached_ch_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(partial_roam_scan_cnt, 11, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(full_roam_scan_cnt, 11, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_rxtrig_myaid, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_colormiss_cnt, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_rxmsta_back, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_txtbppdu, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_null_tbppdu, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(timestamp, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(algo, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(key_flags, 16, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_ts_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_ts_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_ts_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_ts_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_ts_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_ts_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_ts_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_ts_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_CPLOG_BIT(packtlog, 22, 50),
+	EWPR_SERIAL_NONE
+};
+
+ewpr_serial_info_t
+ewpr_serial_bit_assoc_fail_default_tbl[] = {
+	EWPR_SERIAL_VERSION_BIT(version, 32),
+	EWPR_SERIAL_TYPE_BIT(type, 5),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_last_ts, 32, 1, EWPR_SINGLE_USEC_TO_SEC),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_last, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_prev_ts, 32, 1, EWPR_SINGLE_USEC_TO_SEC),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_prev, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IOVAR_BIT(auth, 8),
+	EWPR_SERIAL_IOVAR_BIT(wsec, 8),
+	EWPR_SERIAL_IOVAR_BIT(mfp, 1),
+	EWPR_SERIAL_IOVAR_BIT(bip, 8),
+	EWPR_SERIAL_ARM_BIT(IFACE, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txframe, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txerror, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(rxframe, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(rxerror, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txretrans, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txassocreq, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txreassocreq, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txdisassoc, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxdisassoc, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxassocrsp, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxreassocrsp, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txdeauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxdeauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txaction, 7, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxaction, 7, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(txallfrm, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxrsptmout, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbadplcp, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxcrsglitch, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbadfcs, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbeaconmbss, 5, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbeaconobss, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(lqcm_report, 19, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(tx_toss_cnt, 18, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rx_toss_cnt, 18, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxretry, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxdup, 15, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(chswitch_cnt, 8, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pm_dur, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxholes, 15, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxundec, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxundec_mcst, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(replay, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(replay_mcst, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pktfilter_discard, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pktfilter_forward, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(mac_rxfilter, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_dcsn_map, 16, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_dcsn_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_a2dp_hiwat_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_datadelay_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_crtpri_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_pri_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf5cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf6cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf7cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_32_BIT(bt_gnt_dur, 12, 3, EWPR_SINGLE_USEC_TO_MSEC),
+	EWPR_SERIAL_IFCOMP_BIT(rssi_sum, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IFCOMP_BIT(snr, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IFCOMP_BIT(noise_level, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_rxtrig_myaid, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_colormiss_cnt, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_rxmsta_back, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_txtbppdu, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_null_tbppdu, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(timestamp, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(algo, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(key_flags, 16, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_ts_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_ts_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_ts_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_ts_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_ts_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_ts_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_ts_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_ts_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_CPLOG_BIT(packtlog, 22, 50),
+	EWPR_SERIAL_NONE
+};
+
+ewpr_serial_info_t
+ewpr_serial_bit_abnormal_disconnect_default_tbl[] = {
+	EWPR_SERIAL_VERSION_BIT(version, 32),
+	EWPR_SERIAL_TYPE_BIT(type, 5),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_last_ts, 32, 1, EWPR_SINGLE_USEC_TO_SEC),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_last, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_prev_ts, 32, 1, EWPR_SINGLE_USEC_TO_SEC),
+	EWPR_SERIAL_DHDSTAT_BIT(dhdstat_prev, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IOVAR_BIT(auth, 8),
+	EWPR_SERIAL_IOVAR_BIT(wsec, 8),
+	EWPR_SERIAL_IOVAR_BIT(mfp, 1),
+	EWPR_SERIAL_IOVAR_BIT(bip, 8),
+	EWPR_SERIAL_ARM_BIT(IFACE, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txframe, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txerror, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(rxframe, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(rxerror, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_STAT_BIT(txretrans, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txassocreq, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txreassocreq, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txdisassoc, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxdisassoc, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxassocrsp, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxreassocrsp, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txdeauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxdeauth, 4, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(txaction, 7, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_MGMT_BIT(rxaction, 7, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(txallfrm, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxrsptmout, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbadplcp, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxcrsglitch, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbadfcs, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbeaconmbss, 5, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxbeaconobss, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(lqcm_report, 19, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(tx_toss_cnt, 18, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rx_toss_cnt, 18, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxretry, 17, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxdup, 15, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(chswitch_cnt, 8, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pm_dur, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxholes, 15, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxundec, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(rxundec_mcst, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(replay, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(replay_mcst, 12, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pktfilter_discard, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(pktfilter_forward, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_CNT_V4_BIT(mac_rxfilter, 16, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_dcsn_map, 16, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_dcsn_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_a2dp_hiwat_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_datadelay_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_crtpri_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(bt_pri_cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf5cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf6cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_16_BIT(a2dpbuf7cnt, 12, 3, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_BTC_STAT_32_BIT(bt_gnt_dur, 12, 3, EWPR_SINGLE_USEC_TO_MSEC),
+	EWPR_SERIAL_IFCOMP_BIT(rssi_sum, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IFCOMP_BIT(snr, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_IFCOMP_BIT(noise_level, 7, 6, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(initial_assoc_time, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(prev_roam_time, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(last_roam_event_type, 8, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(last_roam_event_status, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_32_BIT(last_roam_event_reason, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(roam_success_cnt, 10, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(roam_fail_cnt, 10, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(roam_attempt_cnt, 11, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(max_roam_target_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(min_roam_target_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(max_cached_ch_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(min_cached_ch_cnt, 5, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(partial_roam_scan_cnt, 11, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_ROAM_STAT_PERIODIC_16_BIT(full_roam_scan_cnt, 11, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_rxtrig_myaid, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_colormiss_cnt, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_rxmsta_back, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_txtbppdu, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_COMPACT_HE_CNT_BIT(he_null_tbppdu, 10, 6, EWPR_DIFF_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(timestamp, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(algo, 6, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_KEY_INFO_BIT(key_flags, 16, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_ts_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_ts_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rn_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_ts_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_ts_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_TX_TOSS_HIST_BIT(htr_rc_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_ts_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_last, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_ts_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rn_prev, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_ts_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_max, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_ts_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_RX_TOSS_HIST_BIT(htr_rc_secnd, 32, 1, EWPR_SINGLE_DEFAULT),
+	EWPR_SERIAL_CPLOG_BIT(packtlog, 22, 50),
+	EWPR_SERIAL_NONE
+};
+
+ewpr_serial_context_info_t ewpr_serial_context_info[] = {
+	{EWP_CONTEXT_TYPE_UNWANTED_NETWORK, EWP_UNWANT_NET_SUB_TYPE_UNSPECIFIED,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_unwanted_network_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_UNWANTED_NETWORK, EWP_UNWANT_NET_SUB_TYPE_ARP_FAIL,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_unwanted_network_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_UNWANTED_NETWORK, EWP_UNWANT_NET_SUB_TYPE_TXBAD,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_unwanted_network_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ASSOC_FAIL, EWP_ASSOC_FAIL_SUB_TYPE_UNSPECIFIED,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_assoc_fail_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ASSOC_FAIL, EWP_ASSOC_FAIL_SUB_TYPE_DHCP_FAIL,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_assoc_fail_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ASSOC_FAIL, EWP_ASSOC_FAIL_SUB_TYPE_EAP_FAIL,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_assoc_fail_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ASSOC_FAIL, EWP_ASSOC_FAIL_SUB_TYPE_EAP_TIMEOUT,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_assoc_fail_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ASSOC_FAIL, EWP_ASSOC_FAIL_SUB_TYPE_4WAY_FAIL,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_assoc_fail_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ABNORMAL_DISCONNECT, EWP_ABNRML_DISCONNCET_SUB_TYPE_UNSPECIFIED,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_abnormal_disconnect_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_ABNORMAL_DISCONNECT, EWP_ABNRML_DISCONNCET_SUB_TYPE_DISCONNECT_BY_HOST,
+	INDEX_UNSPECIFIED, &ewpr_serial_bit_abnormal_disconnect_default_tbl[0]},
+	{EWP_CONTEXT_TYPE_MAX, INDEX_UNSPECIFIED, INDEX_UNSPECIFIED, NULL}
+};
+#endif /* DHD_EWPR_VER2 */
+
+int ewpr_set_period_lock(ewpr_lock_param_t *param);
+int ewpr_diff_serial(ewpr_serial_info_t *info, char *buf,
+	int buf_len, void *_f_op, void *_s_op, char del);
+int ewpr_single_serial(ewpr_serial_info_t *info, char *buf, int buf_len, void *ptr, char del);
+
+int
+ewpr_serial_basic(char *buf, int buf_len, uint32 data, int format, int display_type, char del)
+{
+	if (format == EWP_HEX) {
+		switch (display_type) {
+			case EWP_INT8:
+			case EWP_UINT8:
+				return scnprintf(buf, buf_len, "%c%02x", del, data & 0xff);
+			case EWP_INT16:
+			case EWP_UINT16:
+				return scnprintf(buf, buf_len, "%c%04x", del, data & 0xffff);
+			case EWP_INT32:
+			case EWP_UINT32:
+				return scnprintf(buf, buf_len, "%c%08x", del, data & 0xffffffff);
+			default:
+				DHD_FILTER_ERR(("INVALID TYPE for Serial:%d", display_type));
+				return 0;
+		}
+	}
+
+	if (format == EWP_DEC) {
+		int32 sdata = (int32) data;
+		switch (display_type) {
+			case EWP_INT8:
+			case EWP_UINT8:
+				return scnprintf(buf, buf_len, "%c%04d", del, sdata);
+			case EWP_INT16:
+			case EWP_UINT16:
+				return scnprintf(buf, buf_len, "%c%06d", del, sdata);
+			case EWP_INT32:
+			case EWP_UINT32:
+				return scnprintf(buf, buf_len, "%c%011d", del, sdata);
+			default:
+				DHD_FILTER_ERR(("INVALID TYPE for Serial:%d", display_type));
+				return 0;
+		}
+	}
+
+	if (format == EWP_BIN) {
+		int32 sdata = (int32) data;
+		switch (display_type) {
+			case EWP_BIT:
+				return scnprintf(buf, buf_len, "%c%011d", del, sdata);
+			default:
+				DHD_FILTER_ERR(("INVALID TYPE for Serial:%d", display_type));
+				return 0;
+		}
+	}
+
+	DHD_FILTER_ERR(("INVALID FORMAT for Serial:%d", format));
+	return 0;
+}
+
+static int
+ewpr_get_multi_offset(uint16 looking_version, ewpr_serial_info_t *info)
+{
+	int idx;
+	ewpr_MVT_offset_elem_t *opv;
+
+	DHD_FILTER_TRACE(("FINDING MULTI OFFSET: type = %s version = %d\n",
+		info->name, looking_version));
+	for (idx = 0; idx < MAX_MULTI_VER; idx ++) {
+		opv = &(info->v_info.opv[idx]);
+
+		/* END OF MULTI VERSION */
+		if (opv->version == 0) {
+			break;
+		}
+		if (looking_version == opv->version) {
+			return opv->offset;
+		}
+	}
+	/* return first version if no version is found */
+	return info->v_info.opv[0].offset;
+}
+int
+ewpr_single_serial(ewpr_serial_info_t *info, char *buf, int buf_len, void *_ptr, char del)
+{
+	uint32 sval = 0;
+	char *ptr = (char *)_ptr;
+	uint32 offset = EWPF_INVALID;
+	uint16	version;
+
+	if (info->is_multi_version == TRUE) {
+		version = *(uint16 *)((char *)_ptr + info->v_info.version_offset);
+		offset = ewpr_get_multi_offset(version, info);
+	} else {
+		offset = info->offset;
+	}
+
+	if (offset == EWPF_INVALID) {
+		DHD_FILTER_ERR(("INVALID TYPE to OFFSET:%s\n", info->name));
+		return 0;
+	}
+
+	ptr += offset;
+
+	switch (info->data_type) {
+		case EWP_INT8:
+			sval = *(int8 *)ptr;
+			break;
+		case EWP_UINT8:
+			sval = *(uint8 *)ptr;
+			break;
+		case EWP_INT16:
+			sval = *(int16 *)ptr;
+			break;
+		case EWP_UINT16:
+			sval = *(uint16 *)ptr;
+			break;
+		case EWP_INT32:
+			sval = *(int32 *)ptr;
+			break;
+		case EWP_UINT32:
+			sval = *(uint32 *)ptr;
+			break;
+		/* XXX UINT64 is used only for debug */
+#ifdef EWPR_DEBUG
+		case EWP_UINT64:
+			sval = (uint32)(*(uint64 *)ptr);
+			break;
+#endif /* EWPR_DEBUG */
+		case EWP_BIT:
+		default:
+			DHD_FILTER_ERR(("INVALID TYPE for Single Serial:%d", info->data_type));
+			return 0;
+	}
+
+	return ewpr_serial_basic(buf, buf_len, sval, info->display_format, info->display_type, del);
+}
+
+int
+ewpr_diff_serial(ewpr_serial_info_t *info,
+	char *buf, int buf_len, void *_f_op, void *_s_op, char del)
+{
+	char *f_op = (char *)_f_op;
+	char *s_op = (char *)_s_op;
+	uint32 diff;
+	uint32 offset = EWPF_INVALID;
+	uint16	version;
+
+	if (info->is_multi_version == TRUE) {
+		version = *(uint16 *)(f_op + info->v_info.version_offset);
+		offset = ewpr_get_multi_offset(version, info);
+	} else {
+		offset = info->offset;
+	}
+
+	if (offset == EWPF_INVALID) {
+		DHD_FILTER_ERR(("INVALID TYPE to OFFSET:%s\n", info->name));
+		return 0;
+	}
+
+	f_op = f_op + offset;
+	s_op = s_op + offset;
+
+	switch (info->data_type) {
+		case EWP_INT8:
+		case EWP_UINT8:
+			diff = *(uint8 *)f_op - *(uint8 *)s_op;
+			break;
+		case EWP_INT16:
+		case EWP_UINT16:
+			diff = *(uint16 *)f_op - *(uint16 *)s_op;
+			break;
+		case EWP_INT32:
+		case EWP_UINT32:
+			diff = *(uint32 *)f_op - *(uint32 *)s_op;
+			break;
+		case EWP_UINT64:
+			diff = (uint32)(*(uint64 *)f_op - *(uint64 *)s_op);
+			break;
+		case EWP_BIT:
+		default:
+			DHD_FILTER_ERR(("INVALID TYPE to DIFF:%d", info->data_type));
+			return 0;
+	}
+
+	return ewpr_serial_basic(buf, buf_len, diff, info->display_format, info->display_type, del);
+}
+
+#ifdef EWPR_DEBUG
+void
+ewpr_debug_dump(ewpr_serial_info_t *tbl, void **ring)
+{
+	void *elem;
+	int idx, idx2;
+	ewpr_serial_info_t *info;
+	char buf[EWPR_DEBUG_BUF_LEN];
+	uint32 bytes_written;
+	int lock_cnt;
+
+	for (idx = 0; strlen(tbl[idx].name) != 0; idx++) {
+		info = &tbl[idx];
+#ifdef DHD_EWPR_VER2
+		if (info->info_type != EWPF_INFO_ECNT) {
+			DHD_FILTER_ERR(("%s: unable to dump value\n", info->name));
+			break;
+		}
+#endif /* DHD_EWPR_VER2 */
+		memset(buf, 0, sizeof(buf));
+		lock_cnt = dhd_ring_lock_get_count(ring[info->ring_type - 1]);
+		elem = dhd_ring_lock_get_first(ring[info->ring_type - 1]);
+		bytes_written = scnprintf(buf, EWPR_DEBUG_BUF_LEN, "%s:", info->name);
+		for (idx2 = 0; elem && (idx2 < lock_cnt); idx2++) {
+			bytes_written += ewpr_single_serial(info, &buf[bytes_written],
+				EWPR_DEBUG_BUF_LEN - bytes_written, elem, KEY_DEL);
+			elem = dhd_ring_get_next(ring[info->ring_type - 1], elem);
+		}
+		DHD_FILTER_ERR(("%s\n", buf));
+	}
+}
+#endif /* EWPR_DEBUG */
+
+uint32
+dhd_event_log_filter_serialize(dhd_pub_t *dhdp, char *in_buf, uint32 tot_len, int type)
+{
+	EWP_filter_t *filter = (EWP_filter_t *)dhdp->event_log_filter;
+	void *ring[EWPF_MAX_IDX_TYPE];
+	char *ret_buf = in_buf;
+	int slice_id;
+	int iface_id;
+	int idx, idx2;
+	uint32 bytes_written = 0;
+	void *elem[EWPF_MAX_IDX_TYPE][EWPR_CSDCLIENT_DIFF];
+	void **elem_list;
+	int lock_cnt, lock_cnt2;
+	char *last_print;
+	void *arr_elem;
+	uint32 delta_list[EWPR_CSDCLIENT_DIFF];
+	ewpr_lock_param_t lock_param;
+	int print_name = FALSE;
+	char cookie_str[DEBUG_DUMP_TIME_BUF_LEN];
+	char iov_buf[EWPR_IOV_BUF_LEN];
+
+	if (type != 0) {
+		DHD_FILTER_ERR(("NOT SUPPORTED TYPE: %d\n", type));
+		return 0;
+	}
+
+	iface_id = 0; /* STA INTERFACE ONLY */
+	if (filter->last_channel <= CH_MAX_2G_CHANNEL) {
+		slice_id = EWPF_SLICE_AUX;
+	} else {
+		slice_id = EWPF_SLICE_MAIN;
+	}
+	ring[EWPF_IDX_TYPE_SLICE - 1] = filter->s_ring[slice_id];
+	ring[EWPF_IDX_TYPE_IFACE - 1] = filter->i_ring[iface_id];
+
+	/* Configure common LOCK parameter */
+	lock_param.max_armcycle = (uint32)EWPF_INVALID;
+	lock_param.min_armcycle = filter->last_armcycle;
+	lock_param.max_period = (EWPR_ARRAY_CNT - 1)* EWPR_INTERVAL;
+	lock_param.max_period *= EWPF_MSEC_TO_SEC * EWPF_ARM_TO_MSEC;
+	lock_param.delta_cnt = ARRAYSIZE(delta_list);
+	lock_param.delta_list = delta_list;
+
+	delta_list[EWPR_DELTA3_POS] = EWPR_DELTA3_CNT;
+	delta_list[EWPR_DELTA2_POS] = EWPR_DELTA2_CNT;
+	delta_list[EWPR_DELTA1_POS] = EWPR_DELTA1_CNT;
+	delta_list[EWPR_NOW_POS] = 0;
+	lock_param.ring = ring[EWPF_IDX_TYPE_IFACE -1];
+	lock_param.elem_list = elem[EWPF_IDX_TYPE_IFACE -1];
+	lock_cnt = ewpr_set_period_lock(&lock_param);
+	if (lock_cnt <= 0) {
+		DHD_FILTER_ERR(("FAIL TO GET IFACE LOCK: %d\n", iface_id));
+		bytes_written = 0;
+		goto finished;
+	}
+
+	delta_list[EWPR_DELTA3_POS] = EWPR_DELTA3_CNT;
+	delta_list[EWPR_DELTA2_POS] = EWPR_DELTA2_CNT;
+	delta_list[EWPR_DELTA1_POS] = EWPR_DELTA1_CNT;
+	delta_list[EWPR_NOW_POS] = 0;
+	lock_param.ring = ring[EWPF_IDX_TYPE_SLICE -1];
+	lock_param.elem_list = elem[EWPF_IDX_TYPE_SLICE -1];
+	lock_cnt2 = ewpr_set_period_lock(&lock_param);
+	if (lock_cnt2 <= 0) {
+		DHD_FILTER_ERR(("FAIL TO GET SLICE LOCK: %d\n", slice_id));
+		goto finished;
+	}
+
+	if (lock_cnt != lock_cnt2) {
+		DHD_FILTER_ERR(("Lock Count is Diff: iface:%d slice:%d\n", lock_cnt, lock_cnt2));
+		lock_cnt = MIN(lock_cnt, lock_cnt2);
+	}
+
+#ifdef EWPR_DEBUG
+	print_name = TRUE;
+	ewpr_debug_dump(ewpr_serial_dbg_tbl, ring);
+	ewpr_debug_dump(ewpr_serial_CSDCLIENT_diff_tbl, ring);
+	ewpr_debug_dump(ewpr_serial_CSDCLIENT_array_tbl, ring);
+#endif /* EWPR_DEBUG */
+
+	memset(ret_buf, 0, tot_len);
+	memset(cookie_str, 0, DEBUG_DUMP_TIME_BUF_LEN);
+	bytes_written = 0;
+	last_print = ret_buf;
+
+	/* XXX Counters BIG DATA not matched to file yet */
+	get_debug_dump_time(cookie_str);
+#ifdef DHD_LOG_DUMP
+	dhd_logdump_cookie_save(dhdp, cookie_str, "ECNT");
+#endif
+
+	/* KEY DATA */
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, "%08x", EWP_REPORT_VERSION);
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, "%c%s", KEY_DEL, cookie_str);
+	DHD_FILTER_ERR(("%d: %s\n", bytes_written, last_print));
+	last_print = &ret_buf[bytes_written];
+
+	for (idx = 0; strlen(ewpr_serial_CSDCLIENT_key_tbl[idx].name) != 0; idx++) {
+		ewpr_serial_info_t *info = &ewpr_serial_CSDCLIENT_key_tbl[idx];
+		elem_list = elem[info->ring_type - 1];
+		if (print_name) {
+			bytes_written += scnprintf(&ret_buf[bytes_written],
+				tot_len - bytes_written, " %s:", info->name);
+		}
+		bytes_written += ewpr_diff_serial(info, &ret_buf[bytes_written],
+			tot_len - bytes_written,
+			elem_list[EWPR_NOW_POS],
+			elem_list[EWPR_DELTA1_POS],
+			KEY_DEL);
+		if ((idx + 1) % EWPR_CNT_PER_LINE == 0) {
+			DHD_FILTER_ERR(("%d:%s\n", bytes_written, last_print));
+			last_print = &ret_buf[bytes_written];
+		}
+	}
+
+	/* RAW DATA */
+	/* XXX FIRST data shall use space:KEY delimiter */
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, "%c%08x", KEY_DEL, EWP_REPORT_VERSION);
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, "%c%s", RAW_DEL, cookie_str);
+
+	for (idx = 0; strlen(ewpr_serial_CSDCLIENT_diff_tbl[idx].name) != 0; idx++) {
+		ewpr_serial_info_t *info = &ewpr_serial_CSDCLIENT_diff_tbl[idx];
+		elem_list = elem[info->ring_type - 1];
+		if (print_name) {
+			bytes_written += scnprintf(&ret_buf[bytes_written],
+				tot_len - bytes_written, " %s:", info->name);
+		}
+		bytes_written += ewpr_diff_serial(info, &ret_buf[bytes_written],
+			tot_len - bytes_written,
+			elem_list[EWPR_NOW_POS],
+			elem_list[EWPR_DELTA1_POS],
+			RAW_DEL);
+		bytes_written += ewpr_diff_serial(info, &ret_buf[bytes_written],
+			tot_len - bytes_written,
+			elem_list[EWPR_DELTA1_POS],
+			elem_list[EWPR_DELTA2_POS],
+			RAW_DEL);
+		if ((idx + 1) % EWPR_CNT_PER_LINE == 0) {
+			DHD_FILTER_ERR(("%d:%s\n", bytes_written, last_print));
+			last_print = &ret_buf[bytes_written];
+		}
+	}
+
+	/* FILL BSS SPECIFIC DATA LATER */
+	if (dhd_iovar(dhdp, 0, "auth", NULL, 0, iov_buf, ARRAYSIZE(iov_buf), FALSE) < 0) {
+		DHD_FILTER_ERR(("fail to get auth\n"));
+		*(uint32 *)iov_buf = EWPF_INVALID;
+
+	}
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+			tot_len - bytes_written, "%c%08x", RAW_DEL, *(uint32 *)iov_buf);
+
+	if (dhd_iovar(dhdp, 0, "wsec", NULL, 0, iov_buf, ARRAYSIZE(iov_buf), FALSE) < 0) {
+		DHD_FILTER_ERR(("fail to get wsec\n"));
+		*(uint32 *)iov_buf = EWPF_INVALID;
+
+	}
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+			tot_len - bytes_written, "%c%08x", RAW_DEL, *(uint32 *)iov_buf);
+
+	if (dhd_iovar(dhdp, 0, "mfp", NULL, 0, iov_buf, ARRAYSIZE(iov_buf), FALSE) < 0) {
+		DHD_FILTER_ERR(("fail to get mfp\n"));
+		*(uint8 *)iov_buf = EWPF_INVALID;
+
+	}
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+			tot_len - bytes_written, "%c%02x", RAW_DEL, *(uint8 *)iov_buf);
+
+	if (dhd_iovar(dhdp, 0, "bip", NULL, 0, iov_buf, ARRAYSIZE(iov_buf), FALSE) < 0) {
+		DHD_FILTER_ERR(("fail to get bip\n"));
+		*(uint8 *)iov_buf = EWPF_INVALID;
+	}
+	bytes_written += scnprintf(&ret_buf[bytes_written],
+			tot_len - bytes_written, "%c%02x", RAW_DEL, *(uint8 *)iov_buf);
+
+	for (idx = 0; strlen(ewpr_serial_CSDCLIENT_array_tbl[idx].name) != 0; idx++) {
+		ewpr_serial_info_t *info = &ewpr_serial_CSDCLIENT_array_tbl[idx];
+		if (print_name) {
+			bytes_written += scnprintf(&ret_buf[bytes_written],
+				tot_len - bytes_written, " %s:", info->name);
+		}
+		for (idx2 = 0; idx2 < EWPR_ARRAY_CNT - lock_cnt; idx2++) {
+			bytes_written += ewpr_serial_basic(&ret_buf[bytes_written],
+				tot_len - bytes_written, 0,
+				info->display_format, info->display_type, RAW_DEL);
+		}
+		arr_elem = elem[info->ring_type - 1][EWPR_DELTA3_POS];
+		for (; idx2 < EWPR_ARRAY_CNT; idx2++) {
+			if (arr_elem == NULL) {
+				DHD_FILTER_ERR(("ARR IS NULL : %d %p \n",
+					idx2, elem[info->ring_type - 1][EWPR_DELTA3_POS]));
+				break;
+			}
+			bytes_written += ewpr_single_serial(info, &ret_buf[bytes_written],
+				tot_len - bytes_written, arr_elem, RAW_DEL);
+			arr_elem = dhd_ring_get_next(ring[info->ring_type - 1], arr_elem);
+		}
+		DHD_FILTER_ERR(("%d:%s\n", bytes_written, last_print));
+		last_print = &ret_buf[bytes_written];
+	}
+
+finished:
+	DHD_FILTER_ERR(("RET LEN:%d\n", (int)strlen(ret_buf)));
+	dhd_ring_lock_free(ring[EWPF_IDX_TYPE_SLICE - 1]);
+	dhd_ring_lock_free(ring[EWPF_IDX_TYPE_IFACE - 1]);
+	return bytes_written;
+}
+
+int
+ewpr_set_period_lock(ewpr_lock_param_t *param)
+{
+	void *last;
+	void *first;
+	void *cur;
+	int lock_cnt;
+	int idx2;
+	int delta_idx;
+	uint32 last_armcycle;
+	uint32 first_armcycle;
+	uint32 cur_armcycle = 0;
+	void *ring = param->ring;
+
+	/* GET LATEST PTR */
+	last = dhd_ring_get_last(ring);
+	while (TRUE) {
+		if (last == NULL) {
+			DHD_FILTER_ERR(("NO LAST\n"));
+			return -1;
+		}
+		last_armcycle = *(uint32 *)last;
+		if (last_armcycle <= param->max_armcycle ||
+			last_armcycle + EWPF_EPOCH >= param->max_armcycle) {
+			break;
+		}
+		last = dhd_ring_get_prev(ring, last);
+	}
+
+	if (last_armcycle != param->max_armcycle) {
+		DHD_FILTER_TRACE(("MAX ARMCYCLE IS CHANGEd new:%u prev:%u\n",
+			last_armcycle, param->max_armcycle));
+		param->max_armcycle = last_armcycle;
+	}
+
+	if (last_armcycle < param->min_armcycle) {
+		param->min_armcycle = 0;
+	}
+
+	/* GET FIRST PTR */
+	first_armcycle = last_armcycle;
+	first = last;
+	while (TRUE) {
+		cur = dhd_ring_get_prev(ring, first);
+		if (cur == NULL) {
+			break;
+		}
+		cur_armcycle = *(uint32 *)cur;
+		if (cur_armcycle >= first_armcycle) {
+			DHD_FILTER_TRACE(("case 1: %u %u\n", first_armcycle, cur_armcycle));
+			/* dongle is rebooted */
+			break;
+		}
+		if (cur_armcycle + EWPF_EPOCH < param->min_armcycle) {
+			DHD_FILTER_TRACE(("case 2: %u %u\n", param->min_armcycle, cur_armcycle));
+			/* Reach Limitation */
+			break;
+		}
+		if (cur_armcycle + param->max_period + EWPF_EPOCH < last_armcycle) {
+			DHD_FILTER_TRACE(("case 3: %u %u\n", param->max_period, cur_armcycle));
+			/* exceed max period */
+			break;
+		}
+		first = cur;
+		first_armcycle = cur_armcycle;
+	}
+
+	if (first_armcycle != param->min_armcycle) {
+		DHD_FILTER_TRACE(("MIN ARMCYCLE IS CHANGEd new:%u prev:%u %u\n",
+			first_armcycle, param->min_armcycle, cur_armcycle));
+		param->min_armcycle = first_armcycle;
+	}
+
+	DHD_FILTER_TRACE(("ARM CYCLE of first(%u), last(%u)\n", first_armcycle, last_armcycle));
+
+	dhd_ring_lock(ring, first, last);
+
+	lock_cnt = dhd_ring_lock_get_count(ring);
+	if (lock_cnt <= 0) {
+		DHD_FILTER_ERR((" NO VALID RECORD : %d\n", lock_cnt));
+		return -1;
+	}
+	DHD_FILTER_TRACE(("Lock Count:%d\n", lock_cnt));
+
+	/* Validate delta position */
+	for (idx2 = 0; idx2 < param->delta_cnt - 1; idx2++) {
+		if (param->delta_list[idx2] >= param->delta_list[idx2 + 1]) {
+			DHD_FILTER_ERR(("INVALID DELTA at %d\n", idx2 + 1));
+			param->delta_list[idx2 + 1] = param->delta_list[idx2];
+		}
+	}
+
+	delta_idx = 0;
+	for (idx2 = 0; idx2 < lock_cnt && delta_idx < param->delta_cnt; idx2++) {
+		if (idx2 == 0) {
+			cur = dhd_ring_lock_get_last(ring);
+		} else {
+			cur = dhd_ring_get_prev(ring, cur);
+		}
+
+		if (idx2 >= param->delta_list[delta_idx]) {
+			param->elem_list[delta_idx] = cur;
+			delta_idx ++;
+		}
+	}
+
+	/* COPY last elem to rest of the list */
+	delta_idx--;
+	for (idx2 = delta_idx + 1; idx2 < param->delta_cnt; idx2++) {
+		param->elem_list[idx2] = cur;
+	}
+	return lock_cnt;
+}
+
+#ifdef DHD_EWPR_VER2
+static int
+ewpr_single_bit_pack(ewpr_serial_info_t * info, char * buf, int buf_len,
+	void * _ptr, int bit_offset)
+{
+	int32 sval = 0;
+	char *ptr = (char *)_ptr;
+	uint32 offset = EWPF_INVALID;
+	uint16	version;
+	bool is_signed = FALSE;
+
+	if (info->is_multi_version == TRUE) {
+		version = *(uint16 *)((char *)_ptr + info->v_info.version_offset);
+		offset = ewpr_get_multi_offset(version, info);
+	} else {
+		offset = info->offset;
+	}
+
+	if (offset == EWPF_INVALID) {
+		DHD_FILTER_ERR(("INVALID TYPE to OFFSET:%s\n", info->name));
+		return 0;
+	}
+
+	ptr += offset;
+
+	switch (info->data_type) {
+		case EWP_INT8:
+			sval = *(int8 *)ptr;
+			is_signed = TRUE;
+			break;
+		case EWP_UINT8:
+			sval = *(uint8 *)ptr;
+			break;
+		case EWP_INT16:
+			sval = *(int16 *)ptr;
+			is_signed = TRUE;
+			break;
+		case EWP_UINT16:
+			sval = *(uint16 *)ptr;
+			break;
+		case EWP_INT32:
+			sval = *(int32 *)ptr;
+			is_signed = TRUE;
+			break;
+		case EWP_UINT32:
+			sval = *(uint32 *)ptr;
+			break;
+#ifdef EWPR_DEBUG
+		case EWP_UINT64:
+			sval = (int32)(*(uint64 *)ptr);
+			break;
+#endif /* EWPR_DEBUG */
+		default:
+			DHD_FILTER_ERR(("INVALID TYPE for Single Serial:%d", info->data_type));
+			return 0;
+	}
+
+	/* convert negative value to positive before bit packing */
+	if (is_signed) {
+		if (sval < 0) {
+			DHD_FILTER_TRACE(("convert to positive value %d\n", sval));
+			sval = ABS(sval);
+		}
+	}
+
+	if (info->unit_convert > 1) {
+		DHD_FILTER_TRACE(("convert unit %d / %d\n", sval, info->unit_convert));
+		sval = sval / info->unit_convert;
+	}
+
+	if (is_signed) {
+		DHD_FILTER_TRACE(("%s : signed value : %d, bit length: %d",
+			info->name, sval, info->display_bit_length));
+	} else {
+		DHD_FILTER_TRACE(("%s : unsigned value : %u, bit length: %d",
+			info->name, sval, info->display_bit_length));
+	}
+
+	return ewpr_bit_pack_basic(buf, buf_len, sval, info->display_format,
+			info->display_type, info->display_bit_length, bit_offset);
+}
+
+static int
+ewpr_diff_bit_pack(ewpr_serial_info_t *info, char *buf, int buf_len,
+	void *_f_op, void *_s_op, int bit_offset)
+{
+	char *f_op = (char *)_f_op;
+	char *s_op = (char *)_s_op;
+	int32 diff;
+	uint32 offset = EWPF_INVALID;
+	uint16	version;
+
+	if (info->is_multi_version == TRUE) {
+		version = *(uint16 *)(f_op + info->v_info.version_offset);
+		offset = ewpr_get_multi_offset(version, info);
+	} else {
+		offset = info->offset;
+	}
+
+	if (offset == EWPF_INVALID) {
+		DHD_FILTER_ERR(("INVALID TYPE to OFFSET:%s\n", info->name));
+		return 0;
+	}
+
+	f_op = f_op + offset;
+	s_op = s_op + offset;
+
+	switch (info->data_type) {
+		case EWP_INT8:
+		case EWP_UINT8:
+			diff = *(uint8 *)f_op - *(uint8 *)s_op;
+			break;
+		case EWP_INT16:
+		case EWP_UINT16:
+			diff = *(uint16 *)f_op - *(uint16 *)s_op;
+			break;
+		case EWP_INT32:
+		case EWP_UINT32:
+			diff = *(uint32 *)f_op - *(uint32 *)s_op;
+			break;
+		case EWP_UINT64:
+			diff = (uint32)(*(uint64 *)f_op - *(uint64 *)s_op);
+			break;
+		default:
+			DHD_FILTER_ERR(("INVALID TYPE to DIFF:%d", info->data_type));
+			return 0;
+	}
+
+	if (diff < 0) {
+		DHD_FILTER_TRACE(("convert to positive value %d\n", diff));
+		diff = ABS(diff);
+	}
+
+	if (info->unit_convert > 1) {
+		DHD_FILTER_TRACE(("convert unit %d / %d\n", diff, info->unit_convert));
+		diff = diff / info->unit_convert;
+	}
+
+	DHD_FILTER_TRACE(("%s : value : %d, bit length: %d",
+		info->name, diff, info->display_bit_length));
+	return ewpr_bit_pack_basic(buf, buf_len, diff, info->display_format,
+		info->display_type, info->display_bit_length, bit_offset);
+}
+
+static int
+ewpr_bit_pack_basic(char *buf, int buf_len, uint32 data, int format, int display_type,
+	int display_bit_length, int bit_offset)
+{
+	if (format == EWP_BIN) {
+		uint32 sdata = (uint32) data;
+		switch (display_type) {
+			case EWP_BIT:
+				/* call bit packing */
+				return dhd_bit_pack(buf, buf_len, bit_offset,
+					sdata, display_bit_length);
+			default:
+				DHD_FILTER_ERR(("INVALID TYPE for Serial:%d", display_type));
+				return 0;
+		}
+	}
+
+	DHD_FILTER_ERR(("INVALID FORMAT for Serial:%d", format));
+	return 0;
+}
+
+static ewpr_serial_info_t*
+ewpr_find_context_info(int index1, int index2, int index3)
+{
+	int idx = 0;
+	ewpr_serial_info_t *context_info = NULL;
+
+	for (idx = 0; ewpr_serial_context_info[idx].table != NULL; idx++) {
+		if (index1 == ewpr_serial_context_info[idx].index1 &&
+				index2 == ewpr_serial_context_info[idx].index2 &&
+				index3 == ewpr_serial_context_info[idx].index3) {
+			context_info = ewpr_serial_context_info[idx].table;
+			break;
+		}
+	}
+
+	if (context_info == NULL) {
+		DHD_FILTER_ERR(("unable to find context info for index number: %02x:%02x:%02x\n",
+			index1, index2, index3));
+		return NULL;
+	}
+
+	return context_info;
+}
+
+static int
+ewpr_find_context_type(ewpr_serial_info_t* context_info)
+{
+	int idx = 0;
+	int context_type = BCME_ERROR;
+
+	/* index2, index3 are reserved */
+
+	for (idx = 0; ewpr_serial_context_info[idx].table != NULL; idx++) {
+		if (context_info == ewpr_serial_context_info[idx].table) {
+			context_type = idx;
+			break;
+		}
+	}
+
+	return context_type;
+}
+
+static uint32
+ewpr_scnprintf(char *buf, uint32 buf_len, uint32 input_len, char *data_type, char *fmt, ...)
+{
+	va_list args;
+
+	if (buf_len < input_len) {
+		DHD_FILTER_ERR(("%s: input length(%d) is larger than "
+			"remain buffer length(%d)\n", data_type,
+			input_len, buf_len));
+	}
+	va_start(args, fmt);
+	buf_len = vscnprintf(buf, buf_len, fmt, args);
+	va_end(args);
+
+	return buf_len;
+}
+
+uint32
+dhd_event_log_filter_serialize_bit(dhd_pub_t *dhdp, char *in_buf, uint32 tot_len,
+	int index1, int index2, int index3)
+{
+	EWP_filter_t *filter = (EWP_filter_t *)dhdp->event_log_filter;
+	void *ring[EWPF_MAX_IDX_TYPE];
+	char *ret_buf = in_buf;
+	int slice_id;
+	int iface_id;
+	int event_id;
+	int key_info_id;
+	int idx;
+	int idx2;
+	uint32 bytes_written = 0;
+	int bits_written = 0;
+	void *elem[EWPF_MAX_IDX_TYPE][EWPR_CSDCLIENT_DIFF];
+	void **elem_list;
+	int lock_cnt, lock_cnt2;
+	uint32 delta_list[EWPR_CSDCLIENT_DIFF];
+	ewpr_lock_param_t lock_param;
+	char cookie_str[DEBUG_DUMP_TIME_BUF_LEN];
+	char iov_buf[EWPR_IOV_BUF_LEN];
+	char *raw_buf = NULL;
+	char *raw_encode_buf = NULL;
+	int raw_buf_size;
+	int ret = 0;
+	ewpr_serial_info_t *context_info = NULL;
+	int context_type;
+#ifdef DHD_STATUS_LOGGING
+	uint32 conv_cnt = 0;
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef DHD_STATUS_LOGGING
+	stat_elem_t dhd_stat[EWP_DHD_STAT_SIZE];
+	stat_query_t query;
+
+	memset(&dhd_stat[0], 0x00, sizeof(stat_elem_t) * EWP_DHD_STAT_SIZE);
+#endif /* DHD_STATUS_LOGGING */
+
+	context_info = ewpr_find_context_info(index1, index2, index3);
+	if (!context_info) {
+		return bytes_written;
+	}
+
+	if (tot_len < EWPR_MAX_STR_SIZE) {
+		DHD_FILTER_ERR(("%s: insufficient buffer size %d\n",
+			__FUNCTION__, tot_len));
+		return bytes_written;
+	}
+
+	iface_id = 0; /* STA INTERFACE ONLY */
+	event_id = 0; /* COMMON ID */
+	key_info_id = 0; /* COMMON ID */
+	if (filter->last_channel <= CH_MAX_2G_CHANNEL) {
+		slice_id = EWPF_SLICE_AUX;
+	} else {
+		slice_id = EWPF_SLICE_MAIN;
+	}
+	ring[EWPF_IDX_TYPE_SLICE - 1] = filter->s_ring[slice_id];
+	ring[EWPF_IDX_TYPE_IFACE - 1] = filter->i_ring[iface_id];
+	ring[EWPF_IDX_TYPE_EVENT - 1] = filter->e_ring[event_id];
+	ring[EWPF_IDX_TYPE_KEY_INFO - 1] = filter->k_ring[key_info_id];
+
+	/* Configure common LOCK parameter */
+	lock_param.max_armcycle = (uint32)EWPF_INVALID;
+	lock_param.min_armcycle = filter->last_armcycle;
+	lock_param.max_period = (EWPR_ARRAY_CNT - 1)* EWPR_INTERVAL;
+	lock_param.max_period *= EWPF_MSEC_TO_SEC * EWPF_ARM_TO_MSEC;
+	lock_param.delta_cnt = ARRAYSIZE(delta_list);
+	lock_param.delta_list = delta_list;
+
+	for (idx = 0; idx < EWPR_CSDCLIENT_DIFF; idx++) {
+		delta_list[idx] = idx * EWPR_DELTA_CNT;
+	}
+
+	lock_param.ring = ring[EWPF_IDX_TYPE_IFACE -1];
+	lock_param.elem_list = elem[EWPF_IDX_TYPE_IFACE -1];
+	lock_cnt = ewpr_set_period_lock(&lock_param);
+	if (lock_cnt <= 0) {
+		DHD_FILTER_ERR(("FAIL TO GET IFACE LOCK: %d\n", iface_id));
+		bytes_written = 0;
+		goto finished;
+	}
+
+	for (idx = 0; idx < EWPR_CSDCLIENT_DIFF; idx++) {
+		delta_list[idx] = idx * EWPR_DELTA_CNT;
+	}
+
+	lock_param.ring = ring[EWPF_IDX_TYPE_SLICE -1];
+	lock_param.elem_list = elem[EWPF_IDX_TYPE_SLICE -1];
+	lock_cnt2 = ewpr_set_period_lock(&lock_param);
+	if (lock_cnt2 <= 0) {
+		DHD_FILTER_ERR(("FAIL TO GET SLICE LOCK: %d\n", slice_id));
+		goto finished;
+	}
+
+	if (lock_cnt != lock_cnt2) {
+		DHD_FILTER_ERR(("Lock Count is Diff: iface:%d slice:%d\n", lock_cnt, lock_cnt2));
+		lock_cnt = MIN(lock_cnt, lock_cnt2);
+	}
+
+	for (idx = 0; idx < EWPR_CSDCLIENT_DIFF; idx++) {
+		delta_list[idx] = idx * EWPR_DELTA_CNT;
+	}
+
+	lock_param.ring = ring[EWPF_IDX_TYPE_EVENT -1];
+	lock_param.elem_list = elem[EWPF_IDX_TYPE_EVENT -1];
+	lock_cnt = ewpr_set_period_lock(&lock_param);
+	if (lock_cnt <= 0) {
+		DHD_FILTER_ERR(("FAIL TO GET EVENT ECNT LOCK: %d\n", iface_id));
+		bytes_written = 0;
+		goto finished;
+	}
+
+	for (idx = 0; idx < EWPR_CSDCLIENT_DIFF; idx++) {
+		delta_list[idx] = idx * EWPR_DELTA_CNT;
+	}
+
+	lock_param.ring = ring[EWPF_IDX_TYPE_KEY_INFO -1];
+	lock_param.elem_list = elem[EWPF_IDX_TYPE_KEY_INFO -1];
+	lock_cnt = ewpr_set_period_lock(&lock_param);
+	if (lock_cnt <= 0) {
+		DHD_FILTER_ERR(("FAIL TO GET KEY INFO LOCK: %d\n", iface_id));
+		bytes_written = 0;
+		goto finished;
+	}
+
+#ifdef EWPR_DEBUG
+	ewpr_debug_dump(context_info, ring);
+#endif /* EWPR_DEBUG */
+
+	memset(ret_buf, 0, tot_len);
+	memset(cookie_str, 0, DEBUG_DUMP_TIME_BUF_LEN);
+	bytes_written = 0;
+	bits_written = 0;
+
+	/* XXX Counters BIG DATA not matched to file yet */
+	get_debug_dump_time(cookie_str);
+#ifdef DHD_LOG_DUMP
+	dhd_logdump_cookie_save(dhdp, cookie_str, "ECNT");
+#endif /* DHD_LOG_DUMP */
+
+	/* KEY DATA */
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, REPORT_VERSION_STR_SIZE,
+		"report version", "%08x", EWP_REPORT_VERSION);
+
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + strlen(cookie_str),
+		"cookie string", "%c%s", KEY_DEL, cookie_str);
+
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + INDEX_STR_SIZE,
+		"host trigger index", "%c%02x%02x%02x", KEY_DEL, index1, index2, index3);
+
+#ifdef DHD_STATUS_LOGGING
+	DHD_FILTER_TRACE(("print dhd_stat size:%d * %d, size of filter list: %d\n",
+		(uint32)sizeof(dhd_stat[0]), EWP_DHD_STAT_SIZE,
+		(uint32)sizeof(dhd_statlog_filter)));
+	query.req_buf = NULL;
+	query.req_buf_len = 0;
+	query.resp_buf = (char *)dhd_stat;
+	query.resp_buf_len = DHD_STATLOG_RING_SIZE(EWP_DHD_STAT_SIZE);
+	query.req_num = EWP_DHD_STAT_SIZE;
+	ret = dhd_statlog_get_latest_info(dhdp, (void *)&query);
+	if (ret < 0) {
+		DHD_FILTER_ERR(("fail to get dhd statlog - %d\n", ret));
+	}
+#ifdef EWPR_DEBUG
+	for (idx = 0; idx < EWP_DHD_STAT_SIZE; idx++) {
+		DHD_FILTER_TRACE(("DHD status index: %d, timestamp: %llu, stat: %d\n",
+			idx, dhd_stat[idx].ts, dhd_stat[idx].stat));
+	}
+#endif
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + DHD_STAT_STR_SIZE,
+		"current dhd status", "%c%02x", KEY_DEL, dhd_stat[0].stat);
+
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + DHD_STAT_STR_SIZE,
+		"previous dhd status", "%c%02x", KEY_DEL, dhd_stat[1].stat);
+#else
+	/* reserved for dhd status information */
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + DHD_STAT_STR_SIZE,
+		"current dhd status", "%c%02x", KEY_DEL, 0x00);
+
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + DHD_STAT_STR_SIZE,
+		"previous dhd status", "%c%02x", KEY_DEL, 0x00);
+#endif /* DHD_STATUS_LOGGING */
+
+	/* RAW DATA */
+	raw_buf = MALLOCZ(dhdp->osh, RAW_BUFFER_SIZE);
+
+	for (idx = 0; strlen(context_info[idx].name) != 0; idx++) {
+		ewpr_serial_info_t *info = &context_info[idx];
+		elem_list = elem[info->ring_type - 1];
+		DHD_FILTER_TRACE(("%s : array_size: %d\n", info->name, info->display_array_size));
+		switch (info->info_type) {
+			case EWPF_INFO_VER:
+				DHD_FILTER_TRACE(("write %s - value: %d\n", info->name,
+					EWP_REPORT_VERSION));
+				bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE, bits_written,
+					EWP_REPORT_VERSION, info->display_bit_length);
+				break;
+			case EWPF_INFO_TYPE:
+				context_type = ewpr_find_context_type(context_info);
+				if (context_type < 0) {
+					DHD_FILTER_ERR(("fail to get context_type - %d\n",
+						context_type));
+					break;
+				}
+				DHD_FILTER_TRACE(("write %s - value: %d\n", info->name,
+					(uint32)context_type));
+				bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE, bits_written,
+					(uint32)context_type, info->display_bit_length);
+				break;
+			case EWPF_INFO_DHDSTAT:
+				if (strcmp("dhdstat_last_ts", info->name) == 0) {
+#ifdef DHD_STATUS_LOGGING
+					if (info->unit_convert > 1) {
+						conv_cnt = dhd_stat[0].ts_tz / info->unit_convert;
+					} else {
+						conv_cnt = dhd_stat[0].ts_tz;
+					}
+					DHD_FILTER_TRACE(("DHD status last timestamp:"
+						" %llu, %u", dhd_stat[0].ts_tz,
+						conv_cnt));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, conv_cnt,
+						info->display_bit_length);
+#else
+					DHD_FILTER_TRACE(("No DHD status log timestamp\n"));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, 0x00, info->display_bit_length);
+#endif /* DHD_STATUS_LOGGING */
+				} else if (strcmp("dhdstat_last", info->name) == 0) {
+#ifdef DHD_STATUS_LOGGING
+					DHD_FILTER_TRACE(("DHD status last stat: %d(0x%02x)",
+						dhd_stat[0].stat, dhd_stat[0].stat));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, (uint32)dhd_stat[0].stat,
+						info->display_bit_length);
+#else
+					DHD_FILTER_TRACE(("No DHD status log value\n"));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, 0x00, info->display_bit_length);
+#endif /* DHD_STATUS_LOGGING */
+				} else if (strcmp("dhdstat_prev_ts", info->name) == 0) {
+#ifdef DHD_STATUS_LOGGING
+					if (info->unit_convert > 1) {
+						conv_cnt = dhd_stat[1].ts_tz / info->unit_convert;
+					} else {
+						conv_cnt = dhd_stat[1].ts_tz;
+					}
+					DHD_FILTER_TRACE(("DHD status prev timestamp:"
+						" %llu, %u", dhd_stat[1].ts_tz,
+						conv_cnt));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, conv_cnt,
+						info->display_bit_length);
+#else
+					DHD_FILTER_TRACE(("No DHD status log timestamp\n"));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, 0x00, info->display_bit_length);
+#endif /* DHD_STATUS_LOGGING */
+				} else if (strcmp("dhdstat_prev", info->name) == 0) {
+#ifdef DHD_STATUS_LOGGING
+					DHD_FILTER_TRACE(("DHD status prev stat: %d(0x%02x)",
+						dhd_stat[1].stat, dhd_stat[1].stat));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, (uint32)dhd_stat[1].stat,
+						info->display_bit_length);
+#else
+					DHD_FILTER_TRACE(("No DHD status log value\n"));
+					bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+						bits_written, 0x00, info->display_bit_length);
+#endif /* DHD_STATUS_LOGGING */
+				} else {
+					DHD_FILTER_ERR(("unknown dhdstat name - %s\n",
+						info->name));
+				}
+				break;
+			case EWPF_INFO_ECNT:
+				for (idx2 = 0; idx2 < info->display_array_size; idx2++) {
+					if (info->display_method == EWPR_DISPLAY_METHOD_SINGLE) {
+						bits_written = ewpr_single_bit_pack(info, raw_buf,
+							RAW_BUFFER_SIZE, elem_list[idx2],
+							bits_written);
+					} else {
+						bits_written = ewpr_diff_bit_pack(info, raw_buf,
+							RAW_BUFFER_SIZE, elem_list[idx2],
+							elem_list[idx2+1], bits_written);
+					}
+				}
+				break;
+			case EWPF_INFO_IOVAR:
+				if (dhd_iovar(dhdp, 0, info->name, NULL, 0, iov_buf,
+						ARRAYSIZE(iov_buf), FALSE) < 0) {
+					DHD_FILTER_ERR(("fail to get auth\n"));
+					*(uint32 *)iov_buf = EWPF_INVALID;
+				}
+				DHD_FILTER_TRACE(("write %s - value: %d\n", info->name,
+					*(uint8 *)iov_buf));
+				bits_written = dhd_bit_pack(raw_buf, RAW_BUFFER_SIZE,
+					bits_written, *(uint8 *)iov_buf,
+					info->display_bit_length);
+				break;
+			case EWPF_INFO_CPLOG:
+				DHD_FILTER_TRACE(("write compact packt log\n"));
+				ret = 0;
+#if defined(DHD_PKT_LOGGING) && defined(DHD_COMPACT_PKT_LOG)
+				ret = dhd_cpkt_log_proc(dhdp, raw_buf, RAW_BUFFER_SIZE,
+					bits_written, info->display_array_size);
+#endif /* DHD_PKT_LOGGING && DHD_COMPACT_PKT_LOG */
+				if (ret < 0) {
+					DHD_FILTER_ERR(("fail to get compact packet log - %d\n",
+						ret));
+					break;
+				}
+				/* update bit offset */
+				DHD_FILTER_TRACE(("%d bits written\n", ret));
+				if (ret > 0) {
+					bits_written = ret;
+				}
+				break;
+			default:
+				DHD_FILTER_ERR(("unsupported info type\n"));
+				break;
+		}
+		DHD_FILTER_TRACE(("%d bits written\n", bits_written));
+	}
+
+	/* encode data */
+	raw_buf_size = BYTE_SIZE(bits_written);
+	raw_encode_buf = ewpr_base64_encode(dhdp, raw_buf, raw_buf_size);
+
+#ifdef EWPR_DEBUG
+	DHD_FILTER_ERR(("raw_buf:\n"));
+	for (idx = 0; idx < raw_buf_size; idx++) {
+		ewpr_print_byte_as_bits(raw_buf[idx]);
+	}
+#endif /* EWPR_DEBUG */
+	DHD_FILTER_TRACE(("base64 encoding result:\n"));
+	DHD_FILTER_TRACE(("%s", raw_encode_buf));
+
+	bytes_written += ewpr_scnprintf(&ret_buf[bytes_written],
+		tot_len - bytes_written, DELIMITER_LEN + strlen(raw_encode_buf),
+		"base64 encoded raw data", "%c%s", KEY_DEL, raw_encode_buf);
+
+finished:
+	DHD_FILTER_ERR(("RET LEN:%d\n", (int)strlen(ret_buf)));
+	DHD_FILTER_TRACE(("ret_buf: %s", ret_buf));
+
+	dhd_ring_lock_free(ring[EWPF_IDX_TYPE_SLICE - 1]);
+	dhd_ring_lock_free(ring[EWPF_IDX_TYPE_IFACE - 1]);
+	dhd_ring_lock_free(ring[EWPF_IDX_TYPE_EVENT - 1]);
+	dhd_ring_lock_free(ring[EWPF_IDX_TYPE_KEY_INFO - 1]);
+
+	MFREE(dhdp->osh, raw_buf, RAW_BUFFER_SIZE);
+	MFREE(dhdp->osh, raw_encode_buf, BASE64_BUFFER_SIZE);
+	return bytes_written;
+}
+
+#ifdef EWPR_DEBUG
+static void
+ewpr_print_byte_as_bits(char val)
+{
+	int32 idx;
+	char buf[EWPR_DEBUG_BUF_LEN];
+	for (idx = 0; idx < MAX_BIT_SIZE; idx++) {
+		scnprintf(&buf[idx], EWPR_DEBUG_BUF_LEN-idx, "%c",
+			(val & (1 << (MAX_BIT_SHIFT-idx))) ? '1' : '0');
+	}
+	buf[MAX_BIT_SIZE] = 0x0;
+	DHD_FILTER_ERR(("%s\n", buf));
+}
+#endif /* EWPR_DEBUG */
+
+static char*
+ewpr_base64_encode(dhd_pub_t *dhdp, char* input, int32 length)
+{
+	/* set up a destination buffer large enough to hold the encoded data */
+	char *output = MALLOCZ(dhdp->osh, BASE64_BUFFER_SIZE);
+	int32 cnt = 0;
+
+	if (length > RAW_BUFFER_SIZE) {
+		DHD_FILTER_ERR(("%s: input data size is too big, size is limited to %d\n",
+			__FUNCTION__, RAW_BUFFER_SIZE));
+		length = RAW_BUFFER_SIZE;
+	}
+
+	cnt = dhd_base64_encode(input, length, output, BASE64_BUFFER_SIZE);
+	if (cnt == 0) {
+		DHD_FILTER_ERR(("%s: base64 encoding error\n", __FUNCTION__));
+	}
+	return output;
+}
+#endif /* DHD_EWPR_VER2 */
+
+#ifdef WLADPS_ENERGY_GAIN
+#define ADPS_GAIN_ENERGY_CONV_UNIT	100000	/* energy unit(10^-2) * dur unit(10^-3) */
+static int
+dhd_calculate_adps_energy_gain(wl_adps_energy_gain_v1_t *data)
+{
+	int i;
+	int energy_gain = 0;
+
+	/* energy unit: (uAh * 10^-2)/sec */
+	int pm0_idle_energy[MAX_BANDS] =
+		{ADPS_GAIN_2G_PM0_IDLE, ADPS_GAIN_5G_PM0_IDLE};
+	int txpspoll_energy[MAX_BANDS] =
+		{ADPS_GAIN_2G_TX_PSPOLL, ADPS_GAIN_5G_TX_PSPOLL};
+
+	if (data->version == 0 || data->length != sizeof(*data)) {
+		DHD_FILTER_ERR(("%s - invalid adps_energy_gain data\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* dur unit: mSec */
+	for (i = 0; i < MAX_BANDS; i++) {
+		energy_gain += (data->gain_data[i].pm_dur_gain * pm0_idle_energy[i]);
+		energy_gain -= (data->gain_data[i].step0_dur * txpspoll_energy[i]);
+	}
+	energy_gain /= ADPS_GAIN_ENERGY_CONV_UNIT;
+
+	if (energy_gain < 0) {
+		energy_gain = 0;
+	}
+
+	return energy_gain;
+}
+
+int dhd_event_log_filter_adps_energy_gain(dhd_pub_t *dhdp)
+{
+	int ret;
+
+	void *last_elem;
+	EWP_filter_t *filter;
+	EWPF_ifc_elem_t *ifc_elem;
+
+	if (!dhdp || !dhdp->event_log_filter) {
+		DHD_FILTER_ERR(("%s - dhdp or event_log_filter is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	filter = (EWP_filter_t *)dhdp->event_log_filter;
+
+	if (filter->enabled != TRUE) {
+		DHD_FILTER_ERR(("%s - EWP Filter is not enabled\n", __FUNCTION__));
+		return BCME_UNSUPPORTED;
+	}
+
+	/* Refer to STA interface */
+	last_elem = dhd_ring_get_last(filter->i_ring[0]);
+	if (last_elem == NULL) {
+		DHD_FILTER_ERR(("%s - last_elem is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	ifc_elem = (EWPF_ifc_elem_t *)last_elem;
+	ret = dhd_calculate_adps_energy_gain(&ifc_elem->adps_energy_gain);
+
+	return ret;
+}
+#endif /* WLADPS_ENERGY_GAIN */
diff --git a/drivers/net/wireless/ap12275/dhd_event_log_filter.h b/drivers/net/wireless/ap12275/dhd_event_log_filter.h
new file mode 100644
index 000000000000..37612db18fe2
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_event_log_filter.h
@@ -0,0 +1,56 @@
+/*
+ * Wifi dongle status Filter and Report
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef dhd_event_log_filter_h
+#define dhd_event_log_filter_h
+#include <dhd.h>
+#include <event_log_tag.h>
+#include <dhd_debug.h>
+
+typedef struct {
+	uint16	version;
+	uint8	htr_type;	/* from wl_slice_hist_XX_stats_xtlv_id */
+	uint8	htr_num;	/* number of elements in htr_running or htr_rc */
+	uint32	htr_rn_last;	/* last reasons along with seq, etc */
+	uint32	htr_rn_ts_last;	/* last time stamps corr to htr_rn_last */
+	uint32	htr_rn_prev;	/* last reasons along with seq, etc */
+	uint32	htr_rn_ts_prev;	/* last time stamps corr to htr_rn_prev */
+	uint32	htr_rc_max;	/* largest toss reasons and counts */
+	uint32	htr_rc_ts_max;	/* latest time stamp corr to htr_rc_max */
+	uint32	htr_rc_secnd;	/* second largest toss reasons and counts */
+	uint32	htr_rc_ts_secnd;	/* latest time stamps corr to htr_rc_second */
+} evt_hist_compact_toss_stats_v1_t;
+
+int dhd_event_log_filter_init(dhd_pub_t *dhdp, uint8 *buf, uint32 buf_size);
+void dhd_event_log_filter_deinit(dhd_pub_t *dhdp);
+void dhd_event_log_filter_event_handler(
+	dhd_pub_t *dhdp, prcd_event_log_hdr_t *plog_hdr, uint32 *data);
+
+void dhd_event_log_filter_notify_connect_request(dhd_pub_t *dhdp, uint8 *bssid, int channel);
+void dhd_event_log_filter_notify_connect_done(dhd_pub_t *dhdp, uint8 *bssid, int roam);
+#ifdef WLADPS_ENERGY_GAIN
+int dhd_event_log_filter_adps_energy_gain(dhd_pub_t *dhdp);
+#endif /* WLADPS_ENERGY_GAIN */
+#endif /* !dhd_event_log_filter_h */
diff --git a/drivers/net/wireless/ap12275/dhd_flowring.c b/drivers/net/wireless/ap12275/dhd_flowring.c
new file mode 100644
index 000000000000..1b3b20c34f57
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_flowring.c
@@ -0,0 +1,1511 @@
+/*
+ * @file Broadcom Dongle Host Driver (DHD), Flow ring specific code at top level
+ *
+ * Flow rings are transmit traffic (=propagating towards antenna) related entities
+ *
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/** XXX Twiki: [PCIeFullDongleArchitecture] */
+
+#include <typedefs.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+
+#include <ethernet.h>
+#include <bcmevent.h>
+#include <dngl_stats.h>
+
+#include <dhd.h>
+
+#include <dhd_flowring.h>
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <dhd_dbg.h>
+#include <802.1d.h>
+#include <pcie_core.h>
+#include <bcmmsgbuf.h>
+#include <dhd_pcie.h>
+#include <dhd_config.h>
+
+static INLINE int dhd_flow_queue_throttle(flow_queue_t *queue);
+
+static INLINE uint16 dhd_flowid_find(dhd_pub_t *dhdp, uint8 ifindex,
+                                     uint8 prio, char *sa, char *da);
+
+static INLINE uint16 dhd_flowid_alloc(dhd_pub_t *dhdp, uint8 ifindex,
+                                      uint8 prio, char *sa, char *da);
+
+static INLINE int dhd_flowid_lookup(dhd_pub_t *dhdp, uint8 ifindex,
+                                uint8 prio, char *sa, char *da, uint16 *flowid);
+int dhd_flow_queue_overflow(flow_queue_t *queue, void *pkt);
+
+#define FLOW_QUEUE_PKT_NEXT(p)          PKTLINK(p)
+#define FLOW_QUEUE_PKT_SETNEXT(p, x)    PKTSETLINK((p), (x))
+
+const uint8 prio2ac[8] = { 0, 1, 1, 0, 2, 2, 3, 3 };
+const uint8 prio2tid[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+
+/** Queue overflow throttle. Return value: TRUE if throttle needs to be applied */
+static INLINE int
+dhd_flow_queue_throttle(flow_queue_t *queue)
+{
+#if defined(BCM_ROUTER_DHD)
+	/* Two tests
+	 * 1) Test whether overall level 2 (grandparent) cummulative threshold crossed.
+	 * 2) Or test whether queue's budget and overall cummulative threshold crossed.
+	 */
+	void *gp_clen_ptr = DHD_FLOW_QUEUE_L2CLEN_PTR(queue);
+	void *parent_clen_ptr = DHD_FLOW_QUEUE_CLEN_PTR(queue);
+	int gp_cumm_threshold = DHD_FLOW_QUEUE_L2THRESHOLD(queue);
+	int cumm_threshold = DHD_FLOW_QUEUE_THRESHOLD(queue);
+
+	int ret = ((DHD_CUMM_CTR_READ(gp_clen_ptr) > gp_cumm_threshold) ||
+		((DHD_FLOW_QUEUE_OVFL(queue, DHD_FLOW_QUEUE_MAX(queue))) &&
+		(DHD_CUMM_CTR_READ(parent_clen_ptr) > cumm_threshold)));
+	return ret;
+#else
+	return DHD_FLOW_QUEUE_FULL(queue);
+#endif /* ! BCM_ROUTER_DHD */
+}
+
+int
+BCMFASTPATH(dhd_flow_queue_overflow)(flow_queue_t *queue, void *pkt)
+{
+	return BCME_NORESOURCE;
+}
+
+/** Returns flow ring given a flowid */
+flow_ring_node_t *
+dhd_flow_ring_node(dhd_pub_t *dhdp, uint16 flowid)
+{
+	flow_ring_node_t * flow_ring_node;
+
+	ASSERT(dhdp != (dhd_pub_t*)NULL);
+	ASSERT(flowid <= dhdp->max_tx_flowid);
+	if (flowid > dhdp->max_tx_flowid) {
+		return NULL;
+	}
+
+	flow_ring_node = &(((flow_ring_node_t*)(dhdp->flow_ring_table))[flowid]);
+
+	ASSERT(flow_ring_node->flowid == flowid);
+	return flow_ring_node;
+}
+
+/** Returns 'backup' queue given a flowid */
+flow_queue_t *
+dhd_flow_queue(dhd_pub_t *dhdp, uint16 flowid)
+{
+	flow_ring_node_t * flow_ring_node = NULL;
+
+	flow_ring_node = dhd_flow_ring_node(dhdp, flowid);
+	if (flow_ring_node)
+		return &flow_ring_node->queue;
+	else
+		return NULL;
+}
+
+/* Flow ring's queue management functions */
+
+/** Reinitialize a flow ring's queue. */
+void
+dhd_flow_queue_reinit(dhd_pub_t *dhdp, flow_queue_t *queue, int max)
+{
+	ASSERT((queue != NULL) && (max > 0));
+
+	queue->head = queue->tail = NULL;
+	queue->len = 0;
+
+	/* Set queue's threshold and queue's parent cummulative length counter */
+	ASSERT(max > 1);
+	DHD_FLOW_QUEUE_SET_MAX(queue, max);
+	DHD_FLOW_QUEUE_SET_THRESHOLD(queue, max);
+	DHD_FLOW_QUEUE_SET_CLEN(queue, &dhdp->cumm_ctr);
+	DHD_FLOW_QUEUE_SET_L2CLEN(queue, &dhdp->l2cumm_ctr);
+
+	queue->failures = 0U;
+	queue->cb = &dhd_flow_queue_overflow;
+}
+
+/** Initialize a flow ring's queue, called on driver initialization. */
+void
+dhd_flow_queue_init(dhd_pub_t *dhdp, flow_queue_t *queue, int max)
+{
+	ASSERT((queue != NULL) && (max > 0));
+
+	dll_init(&queue->list);
+	dhd_flow_queue_reinit(dhdp, queue, max);
+}
+
+/** Register an enqueue overflow callback handler */
+void
+dhd_flow_queue_register(flow_queue_t *queue, flow_queue_cb_t cb)
+{
+	ASSERT(queue != NULL);
+	queue->cb = cb;
+}
+
+/**
+ * Enqueue an 802.3 packet at the back of a flow ring's queue. From there, it will travel later on
+ * to the flow ring itself.
+ */
+int
+BCMFASTPATH(dhd_flow_queue_enqueue)(dhd_pub_t *dhdp, flow_queue_t *queue, void *pkt)
+{
+	int ret = BCME_OK;
+
+	ASSERT(queue != NULL);
+
+	if (dhd_flow_queue_throttle(queue)) {
+		queue->failures++;
+		ret = (*queue->cb)(queue, pkt);
+		goto done;
+	}
+
+	if (queue->head) {
+		FLOW_QUEUE_PKT_SETNEXT(queue->tail, pkt);
+	} else {
+		queue->head = pkt;
+	}
+
+	FLOW_QUEUE_PKT_SETNEXT(pkt, NULL);
+
+	queue->tail = pkt; /* at tail */
+
+	queue->len++;
+	/* increment parent's cummulative length */
+	DHD_CUMM_CTR_INCR(DHD_FLOW_QUEUE_CLEN_PTR(queue));
+	/* increment grandparent's cummulative length */
+	DHD_CUMM_CTR_INCR(DHD_FLOW_QUEUE_L2CLEN_PTR(queue));
+
+done:
+	return ret;
+}
+
+int BCMFASTPATH
+(dhd_flow_queue_enqueue_head)(dhd_pub_t *dhdp, flow_queue_t *queue, void *pkt)
+{
+	int ret = BCME_OK;
+
+	ASSERT(queue != NULL);
+
+	if (dhd_flow_queue_throttle(queue)) {
+		queue->failures++;
+		ret = (*queue->cb)(queue, pkt);
+		goto done;
+	}
+
+	if (queue->head) {
+		FLOW_QUEUE_PKT_SETNEXT(pkt, queue->head);
+		queue->head = pkt;
+
+	} else {
+		queue->head = pkt;
+		FLOW_QUEUE_PKT_SETNEXT(pkt, NULL);
+		queue->tail = pkt; /* at tail */
+	}
+
+	queue->len++;
+	/* increment parent's cummulative length */
+	DHD_CUMM_CTR_INCR(DHD_FLOW_QUEUE_CLEN_PTR(queue));
+	/* increment grandparent's cummulative length */
+	DHD_CUMM_CTR_INCR(DHD_FLOW_QUEUE_L2CLEN_PTR(queue));
+
+done:
+	return ret;
+}
+
+/** Dequeue an 802.3 packet from a flow ring's queue, from head (FIFO) */
+void *
+BCMFASTPATH(dhd_flow_queue_dequeue)(dhd_pub_t *dhdp, flow_queue_t *queue)
+{
+	void * pkt;
+
+	ASSERT(queue != NULL);
+
+	pkt = queue->head; /* from head */
+
+	if (pkt == NULL) {
+		goto done;
+	}
+
+	queue->head = FLOW_QUEUE_PKT_NEXT(pkt);
+	if (queue->head == NULL)
+		queue->tail = NULL;
+
+	queue->len--;
+	/* decrement parent's cummulative length */
+	DHD_CUMM_CTR_DECR(DHD_FLOW_QUEUE_CLEN_PTR(queue));
+	/* decrement grandparent's cummulative length */
+	DHD_CUMM_CTR_DECR(DHD_FLOW_QUEUE_L2CLEN_PTR(queue));
+
+	FLOW_QUEUE_PKT_SETNEXT(pkt, NULL); /* dettach packet from queue */
+
+done:
+	return pkt;
+}
+
+/** Reinsert a dequeued 802.3 packet back at the head */
+void
+BCMFASTPATH(dhd_flow_queue_reinsert)(dhd_pub_t *dhdp, flow_queue_t *queue, void *pkt)
+{
+	if (queue->head == NULL) {
+		queue->tail = pkt;
+	}
+
+	FLOW_QUEUE_PKT_SETNEXT(pkt, queue->head);
+	queue->head = pkt;
+	queue->len++;
+	/* increment parent's cummulative length */
+	DHD_CUMM_CTR_INCR(DHD_FLOW_QUEUE_CLEN_PTR(queue));
+	/* increment grandparent's cummulative length */
+	DHD_CUMM_CTR_INCR(DHD_FLOW_QUEUE_L2CLEN_PTR(queue));
+}
+
+/** Fetch the backup queue for a flowring, and assign flow control thresholds */
+void
+dhd_flow_ring_config_thresholds(dhd_pub_t *dhdp, uint16 flowid,
+                     int queue_budget, int cumm_threshold, void *cumm_ctr,
+                     int l2cumm_threshold, void *l2cumm_ctr)
+{
+	flow_queue_t * queue = NULL;
+
+	ASSERT(dhdp != (dhd_pub_t*)NULL);
+	ASSERT(queue_budget > 1);
+	ASSERT(cumm_threshold > 1);
+	ASSERT(cumm_ctr != (void*)NULL);
+	ASSERT(l2cumm_threshold > 1);
+	ASSERT(l2cumm_ctr != (void*)NULL);
+
+	queue = dhd_flow_queue(dhdp, flowid);
+	if (queue) {
+		DHD_FLOW_QUEUE_SET_MAX(queue, queue_budget); /* Max queue length */
+
+		/* Set the queue's parent threshold and cummulative counter */
+		DHD_FLOW_QUEUE_SET_THRESHOLD(queue, cumm_threshold);
+		DHD_FLOW_QUEUE_SET_CLEN(queue, cumm_ctr);
+
+		/* Set the queue's grandparent threshold and cummulative counter */
+		DHD_FLOW_QUEUE_SET_L2THRESHOLD(queue, l2cumm_threshold);
+		DHD_FLOW_QUEUE_SET_L2CLEN(queue, l2cumm_ctr);
+	}
+}
+
+/*
+ * This function returns total number of flowrings that can be created for a INFRA STA.
+ * For prio2ac mapping, it will return 4, prio2ac[8] = { 0, 1, 1, 0, 2, 2, 3, 3 }
+ * For prio2tid mapping, it will return 8, prio2tid[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }
+ */
+uint8
+dhd_num_prio_supported_per_flow_ring(dhd_pub_t *dhdp)
+{
+	uint8 prio_count = 0;
+	int i;
+	/* Pick all elements one by one */
+	for (i = 0; i < NUMPRIO; i++)
+	{
+		/* Check if the picked element is already counted */
+		int j;
+		for (j = 0; j < i; j++) {
+			if (dhdp->flow_prio_map[i] == dhdp->flow_prio_map[j]) {
+				break;
+			}
+		}
+		/* If not counted earlier, then count it */
+		if (i == j) {
+			prio_count++;
+		}
+	}
+
+	return prio_count;
+}
+
+uint8
+dhd_get_max_multi_client_flow_rings(dhd_pub_t *dhdp)
+{
+	uint8 reserved_infra_sta_flow_rings = dhd_num_prio_supported_per_flow_ring(dhdp);
+	uint8 total_tx_flow_rings = (uint8)dhd_get_max_flow_rings(dhdp);
+	uint8 max_multi_client_flow_rings = total_tx_flow_rings - reserved_infra_sta_flow_rings;
+	return max_multi_client_flow_rings;
+}
+
+int
+dhd_flowid_map_init(dhd_pub_t *dhdp, uint16 max_tx_flow_rings)
+{
+	uint16 max_normal_tx_flow_rings = max_tx_flow_rings;
+
+	if (dhdp->htput_support) {
+		max_normal_tx_flow_rings = max_tx_flow_rings - HTPUT_TOTAL_FLOW_RINGS;
+	}
+
+	/* Construct a normal flowid allocator from FLOWID_RESERVED to
+	 * (max_normal_tx_flow_rings - 1)
+	 */
+	dhdp->flowid_allocator = id16_map_init(dhdp->osh, max_normal_tx_flow_rings,
+		FLOWID_RESERVED);
+	if (dhdp->flowid_allocator == NULL) {
+		DHD_ERROR(("%s: flowid allocator init failure\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+
+	dhdp->htput_flowid_allocator = NULL;
+
+	if (dhdp->htput_support) {
+		if (HTPUT_TOTAL_FLOW_RINGS > 0) {
+			dhdp->htput_flow_ring_start = max_normal_tx_flow_rings + FLOWID_RESERVED;
+			/* Construct a htput flowid allocator from htput_flow_ring_start to
+			 * (htput_flow_ring_start + HTPUT_TOTAL_FLOW_RINGS - 1)
+			 */
+			dhdp->htput_flowid_allocator = id16_map_init(dhdp->osh,
+				HTPUT_TOTAL_FLOW_RINGS,	dhdp->htput_flow_ring_start);
+			if (dhdp->htput_flowid_allocator == NULL) {
+				DHD_ERROR(("%s: htput flowid allocator init failure\n",
+					__FUNCTION__));
+				return BCME_NOMEM;
+			}
+			dhdp->htput_client_flow_rings = 0u;
+		}
+	}
+
+	return BCME_OK;
+}
+
+void
+dhd_flowid_map_deinit(dhd_pub_t *dhdp)
+{
+	if (dhdp->flowid_allocator) {
+		dhdp->flowid_allocator = id16_map_fini(dhdp->osh, dhdp->flowid_allocator);
+	}
+	ASSERT(dhdp->flowid_allocator == NULL);
+
+	if (dhdp->htput_flowid_allocator) {
+		dhdp->htput_flowid_allocator = id16_map_fini(dhdp->osh,
+			dhdp->htput_flowid_allocator);
+		ASSERT(dhdp->htput_flowid_allocator == NULL);
+	}
+	dhdp->htput_client_flow_rings = 0u;
+
+	return;
+}
+
+/** Initializes data structures of multiple flow rings
+ * num_h2d_rings - max_h2d_rings including static and dynamic rings
+ */
+int
+dhd_flow_rings_init(dhd_pub_t *dhdp, uint32 num_h2d_rings)
+{
+	uint32 idx;
+	uint32 flow_ring_table_sz = 0;
+	uint32 if_flow_lkup_sz = 0;
+	flow_ring_table_t *flow_ring_table = NULL;
+	if_flow_lkup_t *if_flow_lkup = NULL;
+	void *lock = NULL;
+	void *list_lock = NULL;
+	unsigned long flags;
+	uint16 max_tx_flow_rings;
+
+	DHD_INFO(("%s\n", __FUNCTION__));
+
+	/*
+	 * Only 16-bit flowid map will be allocated for actual number of Tx flowrings
+	 * excluding common rings.
+	 * Rest all flowring data structure will be allocated for all num_h2d_rings.
+	 */
+	max_tx_flow_rings = dhd_get_max_flow_rings(dhdp);
+	if (dhd_flowid_map_init(dhdp, max_tx_flow_rings) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_flowid_map_init failure\n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* Any Tx flow id should not be > max_tx_flowid */
+	dhdp->max_tx_flowid = max_tx_flow_rings + FLOWID_RESERVED - 1;
+
+	/* Allocate a flow ring table, comprising of requested number of rings */
+	flow_ring_table_sz = (num_h2d_rings * sizeof(flow_ring_node_t));
+	flow_ring_table = (flow_ring_table_t *)MALLOCZ(dhdp->osh, flow_ring_table_sz);
+	if (flow_ring_table == NULL) {
+		DHD_ERROR(("%s: flow ring table alloc failure\n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* Initialize flow ring table state */
+	DHD_CUMM_CTR_INIT(&dhdp->cumm_ctr);
+	DHD_CUMM_CTR_INIT(&dhdp->l2cumm_ctr);
+	bzero((uchar *)flow_ring_table, flow_ring_table_sz);
+	for (idx = 0; idx < num_h2d_rings; idx++) {
+		flow_ring_table[idx].status = FLOW_RING_STATUS_CLOSED;
+		flow_ring_table[idx].flowid = (uint16)idx;
+		flow_ring_table[idx].lock = osl_spin_lock_init(dhdp->osh);
+#ifdef IDLE_TX_FLOW_MGMT
+		flow_ring_table[idx].last_active_ts = OSL_SYSUPTIME();
+#endif /* IDLE_TX_FLOW_MGMT */
+		if (flow_ring_table[idx].lock == NULL) {
+			DHD_ERROR(("%s: Failed to init spinlock for queue!\n", __FUNCTION__));
+			goto fail;
+		}
+
+		dll_init(&flow_ring_table[idx].list);
+
+		/* Initialize the per flow ring backup queue */
+		dhd_flow_queue_init(dhdp, &flow_ring_table[idx].queue,
+		                    dhdp->conf->flow_ring_queue_threshold);
+	}
+
+	/* Allocate per interface hash table (for fast lookup from interface to flow ring) */
+	if_flow_lkup_sz = sizeof(if_flow_lkup_t) * DHD_MAX_IFS;
+	if_flow_lkup = (if_flow_lkup_t *)DHD_OS_PREALLOC(dhdp,
+		DHD_PREALLOC_IF_FLOW_LKUP, if_flow_lkup_sz);
+	if (if_flow_lkup == NULL) {
+		DHD_ERROR(("%s: if flow lkup alloc failure\n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* Initialize per interface hash table */
+	for (idx = 0; idx < DHD_MAX_IFS; idx++) {
+		int hash_ix;
+		if_flow_lkup[idx].status = 0;
+		if_flow_lkup[idx].role = 0;
+		for (hash_ix = 0; hash_ix < DHD_FLOWRING_HASH_SIZE; hash_ix++)
+			if_flow_lkup[idx].fl_hash[hash_ix] = NULL;
+	}
+
+	lock = osl_spin_lock_init(dhdp->osh);
+	if (lock == NULL)
+		goto fail;
+
+	list_lock = osl_spin_lock_init(dhdp->osh);
+	if (list_lock == NULL)
+		goto lock_fail;
+
+	dhdp->flow_prio_map_type = DHD_FLOW_PRIO_AC_MAP;
+	bcopy(prio2ac, dhdp->flow_prio_map, sizeof(uint8) * NUMPRIO);
+
+	dhdp->max_multi_client_flow_rings = dhd_get_max_multi_client_flow_rings(dhdp);
+
+	OSL_ATOMIC_INIT(dhdp->osh, &dhdp->multi_client_flow_rings);
+
+#ifdef DHD_LOSSLESS_ROAMING
+	dhdp->dequeue_prec_map = ALLPRIO;
+#endif
+	/* Now populate into dhd pub */
+	DHD_FLOWID_LOCK(lock, flags);
+	dhdp->num_h2d_rings = num_h2d_rings;
+	dhdp->flow_ring_table = (void *)flow_ring_table;
+	dhdp->if_flow_lkup = (void *)if_flow_lkup;
+	dhdp->flowid_lock = lock;
+	dhdp->flow_rings_inited = TRUE;
+	dhdp->flowring_list_lock = list_lock;
+	DHD_FLOWID_UNLOCK(lock, flags);
+
+	DHD_INFO(("%s done\n", __FUNCTION__));
+	return BCME_OK;
+
+lock_fail:
+	/* deinit the spinlock */
+	osl_spin_lock_deinit(dhdp->osh, lock);
+
+fail:
+	/* Destruct the per interface flow lkup table */
+	if (if_flow_lkup != NULL) {
+		DHD_OS_PREFREE(dhdp, if_flow_lkup, if_flow_lkup_sz);
+	}
+	if (flow_ring_table != NULL) {
+		for (idx = 0; idx < num_h2d_rings; idx++) {
+			if (flow_ring_table[idx].lock != NULL)
+				osl_spin_lock_deinit(dhdp->osh, flow_ring_table[idx].lock);
+		}
+		MFREE(dhdp->osh, flow_ring_table, flow_ring_table_sz);
+	}
+	dhd_flowid_map_deinit(dhdp);
+
+	return BCME_NOMEM;
+}
+
+/** Deinit Flow Ring specific data structures */
+void dhd_flow_rings_deinit(dhd_pub_t *dhdp)
+{
+	uint16 idx;
+	uint32 flow_ring_table_sz;
+	uint32 if_flow_lkup_sz;
+	flow_ring_table_t *flow_ring_table;
+	unsigned long flags;
+	void *lock;
+
+	DHD_INFO(("dhd_flow_rings_deinit\n"));
+
+	if (!(dhdp->flow_rings_inited)) {
+		DHD_ERROR(("dhd_flow_rings not initialized!\n"));
+		return;
+	}
+
+	if (dhdp->flow_ring_table != NULL) {
+
+		ASSERT(dhdp->num_h2d_rings > 0);
+
+		DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+		flow_ring_table = (flow_ring_table_t *)dhdp->flow_ring_table;
+		dhdp->flow_ring_table = NULL;
+		DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+		for (idx = 0; idx < dhdp->num_h2d_rings; idx++) {
+			if (flow_ring_table[idx].active) {
+				dhd_bus_clean_flow_ring(dhdp->bus, &flow_ring_table[idx]);
+			}
+			ASSERT(DHD_FLOW_QUEUE_EMPTY(&flow_ring_table[idx].queue));
+
+			/* Deinit flow ring queue locks before destroying flow ring table */
+			if (flow_ring_table[idx].lock != NULL) {
+				osl_spin_lock_deinit(dhdp->osh, flow_ring_table[idx].lock);
+			}
+			flow_ring_table[idx].lock = NULL;
+
+		}
+
+		/* Destruct the flow ring table */
+		flow_ring_table_sz = dhdp->num_h2d_rings * sizeof(flow_ring_table_t);
+		MFREE(dhdp->osh, flow_ring_table, flow_ring_table_sz);
+	}
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+
+	/* Destruct the per interface flow lkup table */
+	if (dhdp->if_flow_lkup != NULL) {
+		if_flow_lkup_sz = sizeof(if_flow_lkup_t) * DHD_MAX_IFS;
+		bzero((uchar *)dhdp->if_flow_lkup, if_flow_lkup_sz);
+		DHD_OS_PREFREE(dhdp, dhdp->if_flow_lkup, if_flow_lkup_sz);
+		dhdp->if_flow_lkup = NULL;
+	}
+
+	/* Destruct the flowid allocator */
+	dhd_flowid_map_deinit(dhdp);
+
+	dhdp->num_h2d_rings = 0U;
+	bzero(dhdp->flow_prio_map, sizeof(uint8) * NUMPRIO);
+
+	dhdp->max_multi_client_flow_rings = 0U;
+
+	OSL_ATOMIC_INIT(dhdp->osh, &dhdp->multi_client_flow_rings);
+
+	lock = dhdp->flowid_lock;
+	dhdp->flowid_lock = NULL;
+
+	if (lock) {
+		DHD_FLOWID_UNLOCK(lock, flags);
+		osl_spin_lock_deinit(dhdp->osh, lock);
+	}
+
+	osl_spin_lock_deinit(dhdp->osh, dhdp->flowring_list_lock);
+	dhdp->flowring_list_lock = NULL;
+
+	ASSERT(dhdp->if_flow_lkup == NULL);
+	ASSERT(dhdp->flow_ring_table == NULL);
+	dhdp->flow_rings_inited = FALSE;
+}
+
+/** Uses hash table to quickly map from ifindex to a flow ring 'role' (STA/AP) */
+uint8
+dhd_flow_rings_ifindex2role(dhd_pub_t *dhdp, uint8 ifindex)
+{
+	if_flow_lkup_t *if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+	ASSERT(if_flow_lkup);
+	return if_flow_lkup[ifindex].role;
+}
+
+#ifdef WLTDLS
+bool is_tdls_destination(dhd_pub_t *dhdp, uint8 *da)
+{
+	unsigned long flags;
+	tdls_peer_node_t *cur = NULL;
+
+	DHD_TDLS_LOCK(&dhdp->tdls_lock, flags);
+	/* Check only if tdls peer is added */
+	if (dhdp->peer_tbl.tdls_peer_count && !(ETHER_ISMULTI(da))) {
+		cur = dhdp->peer_tbl.node;
+
+		while (cur != NULL) {
+			if (!memcmp(da, cur->addr, ETHER_ADDR_LEN)) {
+				DHD_TDLS_UNLOCK(&dhdp->tdls_lock, flags);
+				return TRUE;
+			}
+			cur = cur->next;
+		}
+	}
+	DHD_TDLS_UNLOCK(&dhdp->tdls_lock, flags);
+	return FALSE;
+}
+#endif /* WLTDLS */
+
+/** Uses hash table to quickly map from ifindex+prio+da to a flow ring id */
+static INLINE uint16
+dhd_flowid_find(dhd_pub_t *dhdp, uint8 ifindex, uint8 prio, char *sa, char *da)
+{
+	int hash;
+	bool ismcast = FALSE;
+	flow_hash_info_t *cur;
+	if_flow_lkup_t *if_flow_lkup;
+	unsigned long flags;
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return FLOWID_INVALID;
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+
+	ASSERT(if_flow_lkup);
+
+	if (DHD_IF_ROLE_GENERIC_STA(dhdp, ifindex)) {
+#ifdef WLTDLS
+		if (is_tdls_destination(dhdp, da)) {
+			hash = DHD_FLOWRING_HASHINDEX(da, prio);
+			cur = if_flow_lkup[ifindex].fl_hash[hash];
+			while (cur != NULL) {
+				if (!memcmp(cur->flow_info.da, da, ETHER_ADDR_LEN)) {
+					DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+					return cur->flowid;
+				}
+				cur = cur->next;
+			}
+			DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+			return FLOWID_INVALID;
+		}
+#endif /* WLTDLS */
+		/* For STA non TDLS dest and WDS dest flow ring id is mapped based on prio only */
+		cur = if_flow_lkup[ifindex].fl_hash[prio];
+		if (cur) {
+			DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+			return cur->flowid;
+		}
+	} else {
+
+		if (ETHER_ISMULTI(da) &&
+			TRUE) {
+			ismcast = TRUE;
+			hash = 0;
+		} else {
+			hash = DHD_FLOWRING_HASHINDEX(da, prio);
+		}
+
+		cur = if_flow_lkup[ifindex].fl_hash[hash];
+
+		while (cur) {
+			if ((ismcast && ETHER_ISMULTI(cur->flow_info.da)) ||
+				(!memcmp(cur->flow_info.da, da, ETHER_ADDR_LEN) &&
+				(cur->flow_info.tid == prio))) {
+				DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+				return cur->flowid;
+			}
+			cur = cur->next;
+		}
+	}
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+	DHD_INFO(("%s: cannot find flowid\n", __FUNCTION__));
+	return FLOWID_INVALID;
+} /* dhd_flowid_find */
+
+static uint16
+dhd_flowid_map_alloc(dhd_pub_t *dhdp, uint8 ifindex, uint8 prio, char *da)
+{
+	uint16 flowid = FLOWID_INVALID;
+	ASSERT(dhdp->flowid_allocator != NULL);
+
+	/* P2P Connections are always 80Mhz */
+	if (DHD_IF_ROLE_P2PGC(dhdp, ifindex) ||
+	    DHD_IF_ROLE_P2PGO(dhdp, ifindex)) {
+		flowid = id16_map_alloc(dhdp->flowid_allocator);
+		return flowid;
+	}
+
+	if (dhdp->htput_flowid_allocator) {
+		if (prio == HTPUT_FLOW_RING_PRIO) {
+			if (DHD_IF_ROLE_GENERIC_STA(dhdp, ifindex)) {
+				/* For STA case, only one flowring per PRIO is created,
+				 * so no need to have a HTPUT counter variable for STA case.
+				 * If already HTPUT flowring is allocated for given HTPUT_PRIO,
+				 * then this function will not even get called as dhd_flowid_find
+				 * will take care assigning same for those HTPUT_PRIO packets.
+				 */
+				flowid = id16_map_alloc(dhdp->htput_flowid_allocator);
+			} else if (DHD_IF_ROLE_MULTI_CLIENT(dhdp, ifindex) && !ETHER_ISMULTI(da) &&
+				dhd_is_sta_htput(dhdp, ifindex, (uint8 *)da)) {
+				/* Use HTPUT flowrings for only HTPUT_NUM_CLIENT_FLOW_RINGS */
+				if (dhdp->htput_client_flow_rings < HTPUT_NUM_CLIENT_FLOW_RINGS) {
+					flowid = id16_map_alloc(dhdp->htput_flowid_allocator);
+					/* increment htput client counter */
+					if (flowid != FLOWID_INVALID) {
+						dhdp->htput_client_flow_rings++;
+					}
+				}
+			}
+		}
+	}
+
+	BCM_REFERENCE(flowid);
+
+	/*
+	 * For HTPUT case, if the high throughput flowrings are already allocated
+	 * for the given role, the control comes here.
+	 */
+	if (flowid == FLOWID_INVALID) {
+		flowid = id16_map_alloc(dhdp->flowid_allocator);
+	}
+
+	return flowid;
+}
+
+/** Create unique Flow ID, called when a flow ring is created. */
+static INLINE uint16
+dhd_flowid_alloc(dhd_pub_t *dhdp, uint8 ifindex, uint8 prio, char *sa, char *da)
+{
+	flow_hash_info_t *fl_hash_node, *cur;
+	if_flow_lkup_t *if_flow_lkup;
+	int hash;
+	uint16 flowid;
+	unsigned long flags;
+
+	fl_hash_node = (flow_hash_info_t *) MALLOCZ(dhdp->osh, sizeof(flow_hash_info_t));
+	if (fl_hash_node == NULL) {
+		DHD_ERROR(("%s: flow_hash_info_t memory allocation failed \n", __FUNCTION__));
+		return FLOWID_INVALID;
+	}
+	memcpy(fl_hash_node->flow_info.da, da, sizeof(fl_hash_node->flow_info.da));
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	flowid = dhd_flowid_map_alloc(dhdp, ifindex, prio, da);
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+	if (flowid == FLOWID_INVALID) {
+		MFREE(dhdp->osh, fl_hash_node,  sizeof(flow_hash_info_t));
+		DHD_ERROR_RLMT(("%s: cannot get free flowid \n", __FUNCTION__));
+		return FLOWID_INVALID;
+	}
+
+	fl_hash_node->flowid = flowid;
+	fl_hash_node->flow_info.tid = prio;
+	fl_hash_node->flow_info.ifindex = ifindex;
+	fl_hash_node->next = NULL;
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+
+	if (DHD_IF_ROLE_GENERIC_STA(dhdp, ifindex)) {
+		/* For STA/GC non TDLS dest and WDS dest we allocate entry based on prio only */
+#ifdef WLTDLS
+		if (is_tdls_destination(dhdp, da)) {
+			hash = DHD_FLOWRING_HASHINDEX(da, prio);
+			cur = if_flow_lkup[ifindex].fl_hash[hash];
+			if (cur) {
+				while (cur->next) {
+					cur = cur->next;
+				}
+				cur->next = fl_hash_node;
+			} else {
+				if_flow_lkup[ifindex].fl_hash[hash] = fl_hash_node;
+			}
+		} else
+#endif /* WLTDLS */
+			if_flow_lkup[ifindex].fl_hash[prio] = fl_hash_node;
+	} else {
+
+		/* For bcast/mcast assign first slot in in interface */
+		hash = (ETHER_ISMULTI(da) &&
+			TRUE) ?  0 : DHD_FLOWRING_HASHINDEX(da, prio);
+
+		cur = if_flow_lkup[ifindex].fl_hash[hash];
+		if (cur) {
+			while (cur->next) {
+				cur = cur->next;
+			}
+			cur->next = fl_hash_node;
+		} else
+			if_flow_lkup[ifindex].fl_hash[hash] = fl_hash_node;
+	}
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+	DHD_INFO(("%s: allocated flowid %d\n", __FUNCTION__, fl_hash_node->flowid));
+
+	if (fl_hash_node->flowid > dhdp->max_tx_flowid) {
+		DHD_ERROR(("%s: flowid=%d max_tx_flowid=%d ifindex=%d prio=%d role=%d\n",
+			__FUNCTION__, fl_hash_node->flowid, dhdp->max_tx_flowid,
+			ifindex, prio, if_flow_lkup[ifindex].role));
+		dhd_prhex("da", (uchar *)da, ETHER_ADDR_LEN, DHD_ERROR_VAL);
+		dhd_prhex("sa", (uchar *)sa, ETHER_ADDR_LEN, DHD_ERROR_VAL);
+		return FLOWID_INVALID;
+	}
+
+	return fl_hash_node->flowid;
+} /* dhd_flowid_alloc */
+
+/** Get flow ring ID, if not present try to create one */
+static INLINE int
+dhd_flowid_lookup(dhd_pub_t *dhdp, uint8 ifindex,
+                  uint8 prio, char *sa, char *da, uint16 *flowid)
+{
+	uint16 id;
+	flow_ring_node_t *flow_ring_node;
+	flow_ring_table_t *flow_ring_table;
+	unsigned long flags;
+	int ret;
+
+	DHD_TRACE(("%s\n", __FUNCTION__));
+
+	if (!dhdp->flow_ring_table) {
+		return BCME_ERROR;
+	}
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return BCME_BADARG;
+
+	flow_ring_table = (flow_ring_table_t *)dhdp->flow_ring_table;
+
+	id = dhd_flowid_find(dhdp, ifindex, prio, sa, da);
+
+	if (id == FLOWID_INVALID || (id > dhdp->max_tx_flowid)) {
+		bool if_role_multi_client;
+		if_flow_lkup_t *if_flow_lkup;
+		if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+
+		if (!if_flow_lkup[ifindex].status)
+			return BCME_ERROR;
+
+		/* check role for multi client case */
+		if_role_multi_client = DHD_IF_ROLE_MULTI_CLIENT(dhdp, ifindex);
+
+		/* Abort Flowring creation if multi client flowrings crossed the threshold */
+#ifdef DHD_LIMIT_MULTI_CLIENT_FLOWRINGS
+		if (if_role_multi_client &&
+			(OSL_ATOMIC_READ(dhdp->osh, &dhdp->multi_client_flow_rings) >=
+				dhdp->max_multi_client_flow_rings)) {
+			DHD_ERROR_RLMT(("%s: Max multi client flow rings reached: %d:%d\n",
+				__FUNCTION__,
+				OSL_ATOMIC_READ(dhdp->osh, &dhdp->multi_client_flow_rings),
+				dhdp->max_multi_client_flow_rings));
+			return BCME_ERROR;
+		}
+#endif /* DHD_LIMIT_MULTI_CLIENT_FLOWRINGS */
+
+		/* Do not create Flowring if peer is not associated */
+#if (defined(linux) || defined(LINUX)) && defined(PCIE_FULL_DONGLE)
+		if (if_role_multi_client && !ETHER_ISMULTI(da) &&
+			!dhd_sta_associated(dhdp, ifindex, (uint8 *)da)) {
+			DHD_ERROR_RLMT(("%s: Skip send pkt without peer addition\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+#endif /* (linux || LINUX) && PCIE_FULL_DONGLE */
+
+		id = dhd_flowid_alloc(dhdp, ifindex, prio, sa, da);
+		if (id == FLOWID_INVALID) {
+			DHD_ERROR_RLMT(("%s: alloc flowid ifindex %u status %u\n",
+				__FUNCTION__, ifindex, if_flow_lkup[ifindex].status));
+			return BCME_ERROR;
+		}
+
+		ASSERT(id <= dhdp->max_tx_flowid);
+
+		/* Only after flowid alloc, increment multi_client_flow_rings */
+		if (if_role_multi_client) {
+			OSL_ATOMIC_INC(dhdp->osh, &dhdp->multi_client_flow_rings);
+		}
+
+		/* register this flowid in dhd_pub */
+		dhd_add_flowid(dhdp, ifindex, prio, da, id);
+
+		flow_ring_node = (flow_ring_node_t *) &flow_ring_table[id];
+
+		DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+
+		/* Init Flow info */
+		memcpy(flow_ring_node->flow_info.sa, sa, sizeof(flow_ring_node->flow_info.sa));
+		memcpy(flow_ring_node->flow_info.da, da, sizeof(flow_ring_node->flow_info.da));
+		flow_ring_node->flow_info.tid = prio;
+		flow_ring_node->flow_info.ifindex = ifindex;
+		flow_ring_node->active = TRUE;
+		flow_ring_node->status = FLOW_RING_STATUS_CREATE_PENDING;
+
+#ifdef DEVICE_TX_STUCK_DETECT
+		flow_ring_node->tx_cmpl = flow_ring_node->tx_cmpl_prev = OSL_SYSUPTIME();
+		flow_ring_node->stuck_count = 0;
+#endif /* DEVICE_TX_STUCK_DETECT */
+#ifdef TX_STATUS_LATENCY_STATS
+		flow_ring_node->flow_info.num_tx_status = 0;
+		flow_ring_node->flow_info.cum_tx_status_latency = 0;
+		flow_ring_node->flow_info.num_tx_pkts = 0;
+#endif /* TX_STATUS_LATENCY_STATS */
+#ifdef BCMDBG
+		bzero(&flow_ring_node->flow_info.tx_status[0],
+			sizeof(uint32) * DHD_MAX_TX_STATUS_MSGS);
+#endif
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+		/* Create and inform device about the new flow */
+		if (dhd_bus_flow_ring_create_request(dhdp->bus, (void *)flow_ring_node)
+				!= BCME_OK) {
+			DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+			flow_ring_node->status = FLOW_RING_STATUS_CLOSED;
+			flow_ring_node->active = FALSE;
+			DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+			DHD_ERROR(("%s: create error %d\n", __FUNCTION__, id));
+			return BCME_ERROR;
+		}
+
+		*flowid = id;
+		return BCME_OK;
+	} else {
+		/* if the Flow id was found in the hash */
+
+		if (id > dhdp->max_tx_flowid) {
+			DHD_ERROR(("%s: Invalid flow id : %u, max_tx_flowid : %u\n",
+				__FUNCTION__, id, dhdp->max_tx_flowid));
+			*flowid = FLOWID_INVALID;
+			ASSERT(0);
+			return BCME_ERROR;
+		}
+
+		flow_ring_node = (flow_ring_node_t *) &flow_ring_table[id];
+		DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+
+		/*
+		 * If the flow_ring_node is in Open State or Status pending state then
+		 * we can return the Flow id to the caller.If the flow_ring_node is in
+		 * FLOW_RING_STATUS_PENDING this means the creation is in progress and
+		 * hence the packets should be queued.
+		 *
+		 * If the flow_ring_node is in FLOW_RING_STATUS_DELETE_PENDING Or
+		 * FLOW_RING_STATUS_CLOSED, then we should return Error.
+		 * Note that if the flowing is being deleted we would mark it as
+		 * FLOW_RING_STATUS_DELETE_PENDING.  Now before Dongle could respond and
+		 * before we mark it as FLOW_RING_STATUS_CLOSED we could get tx packets.
+		 * We should drop the packets in that case.
+		 * The decission to return OK should NOT be based on 'active' variable, beause
+		 * active is made TRUE when a flow_ring_node gets allocated and is made
+		 * FALSE when the flow ring gets removed and does not reflect the True state
+		 * of the Flow ring.
+		 * In case if IDLE_TX_FLOW_MGMT is defined, we have to handle two more flowring
+		 * states. If the flow_ring_node's status is FLOW_RING_STATUS_SUSPENDED, the flowid
+		 * is to be returned and from dhd_bus_txdata, the flowring would be resumed again.
+		 * The status FLOW_RING_STATUS_RESUME_PENDING, is equivalent to
+		 * FLOW_RING_STATUS_CREATE_PENDING.
+		 */
+		if (flow_ring_node->status == FLOW_RING_STATUS_DELETE_PENDING ||
+			flow_ring_node->status == FLOW_RING_STATUS_CLOSED) {
+			*flowid = FLOWID_INVALID;
+			ret = BCME_ERROR;
+		} else {
+			*flowid = id;
+			ret = BCME_OK;
+		}
+
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+		return ret;
+	} /* Flow Id found in the hash */
+} /* dhd_flowid_lookup */
+
+int
+dhd_flowid_find_by_ifidx(dhd_pub_t *dhdp, uint8 ifindex, uint16 flowid)
+{
+	int hashidx = 0;
+	bool found = FALSE;
+	flow_hash_info_t *cur;
+	if_flow_lkup_t *if_flow_lkup;
+	unsigned long flags;
+
+	if (!dhdp->flow_ring_table) {
+		DHD_ERROR(("%s : dhd->flow_ring_table is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+	for (hashidx = 0; hashidx < DHD_FLOWRING_HASH_SIZE; hashidx++) {
+		cur = if_flow_lkup[ifindex].fl_hash[hashidx];
+		if (cur) {
+			if (cur->flowid == flowid) {
+				found = TRUE;
+			}
+
+			while (!found && cur) {
+				if (cur->flowid == flowid) {
+					found = TRUE;
+					break;
+				}
+				cur = cur->next;
+			}
+
+			if (found) {
+				DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+				return BCME_OK;
+			}
+		}
+	}
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+	return BCME_ERROR;
+}
+
+int
+dhd_flowid_debug_create(dhd_pub_t *dhdp, uint8 ifindex,
+	uint8 prio, char *sa, char *da, uint16 *flowid)
+{
+	return dhd_flowid_lookup(dhdp, ifindex, prio, sa, da, flowid);
+}
+
+/**
+ * Assign existing or newly created flowid to an 802.3 packet. This flowid is later on used to
+ * select the flowring to send the packet to the dongle.
+ */
+int
+BCMFASTPATH(dhd_flowid_update)(dhd_pub_t *dhdp, uint8 ifindex, uint8 prio, void *pktbuf)
+{
+	uint8 *pktdata = (uint8 *)PKTDATA(dhdp->osh, pktbuf);
+	struct ether_header *eh = (struct ether_header *)pktdata;
+	uint16 flowid = 0;
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+
+	if (ifindex >= DHD_MAX_IFS) {
+		return BCME_BADARG;
+	}
+
+	if (!dhdp->flowid_allocator) {
+		DHD_ERROR(("%s: Flow ring not intited yet  \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhd_flowid_lookup(dhdp, ifindex, prio, (char *)eh->ether_shost, (char *)eh->ether_dhost,
+		&flowid) != BCME_OK) {
+		return BCME_ERROR;
+	}
+
+	DHD_INFO(("%s: prio %d flowid %d\n", __FUNCTION__, prio, flowid));
+
+	/* Tag the packet with flowid */
+	DHD_PKT_SET_FLOWID(pktbuf, flowid);
+	return BCME_OK;
+}
+
+static void
+dhd_flowid_map_free(dhd_pub_t *dhdp, uint8 ifindex, uint16 flowid)
+{
+	if (dhdp->htput_flowid_allocator) {
+		if (DHD_IS_FLOWID_HTPUT(dhdp, flowid)) {
+			id16_map_free(dhdp->htput_flowid_allocator, flowid);
+			/* decrement htput client counter */
+			if (DHD_IF_ROLE_MULTI_CLIENT(dhdp, ifindex)) {
+				dhdp->htput_client_flow_rings--;
+			}
+			return;
+		}
+	}
+
+	id16_map_free(dhdp->flowid_allocator, flowid);
+
+	return;
+}
+
+void
+dhd_flowid_free(dhd_pub_t *dhdp, uint8 ifindex, uint16 flowid)
+{
+	int hashix;
+	bool found = FALSE;
+	flow_hash_info_t *cur, *prev;
+	if_flow_lkup_t *if_flow_lkup;
+	unsigned long flags;
+	bool if_role_multi_client;
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return;
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+
+	if_role_multi_client = DHD_IF_ROLE_MULTI_CLIENT(dhdp, ifindex);
+
+	for (hashix = 0; hashix < DHD_FLOWRING_HASH_SIZE; hashix++) {
+
+		cur = if_flow_lkup[ifindex].fl_hash[hashix];
+
+		if (cur) {
+			if (cur->flowid == flowid) {
+				found = TRUE;
+			}
+
+			prev = NULL;
+			while (!found && cur) {
+				if (cur->flowid == flowid) {
+					found = TRUE;
+					break;
+				}
+				prev = cur;
+				cur = cur->next;
+			}
+			if (found) {
+				if (!prev) {
+					if_flow_lkup[ifindex].fl_hash[hashix] = cur->next;
+				} else {
+					prev->next = cur->next;
+				}
+
+				/* Decrement multi_client_flow_rings */
+				if (if_role_multi_client) {
+					if (OSL_ATOMIC_READ(dhdp->osh,
+						&dhdp->multi_client_flow_rings)) {
+						OSL_ATOMIC_DEC(dhdp->osh,
+							&dhdp->multi_client_flow_rings);
+					}
+				}
+
+				/* deregister flowid from dhd_pub. */
+				dhd_del_flowid(dhdp, ifindex, flowid);
+
+				dhd_flowid_map_free(dhdp, ifindex, flowid);
+				MFREE(dhdp->osh, cur, sizeof(flow_hash_info_t));
+				DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+				return;
+			}
+		}
+	}
+
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+	DHD_ERROR(("%s: could not free flow ring hash entry flowid %d\n",
+	           __FUNCTION__, flowid));
+} /* dhd_flowid_free */
+
+/**
+ * Delete all Flow rings associated with the given interface. Is called when eg the dongle
+ * indicates that a wireless link has gone down.
+ */
+void
+dhd_flow_rings_delete(dhd_pub_t *dhdp, uint8 ifindex)
+{
+	uint32 id;
+	flow_ring_table_t *flow_ring_table;
+
+	DHD_ERROR(("%s: ifindex %u\n", __FUNCTION__, ifindex));
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return;
+
+	if (!dhdp->flow_ring_table)
+		return;
+
+	flow_ring_table = (flow_ring_table_t *)dhdp->flow_ring_table;
+	for (id = 0; id < dhdp->num_h2d_rings; id++) {
+		if (flow_ring_table[id].active &&
+			(flow_ring_table[id].flow_info.ifindex == ifindex) &&
+			(flow_ring_table[id].status == FLOW_RING_STATUS_OPEN)) {
+			dhd_bus_flow_ring_delete_request(dhdp->bus,
+			                                 (void *) &flow_ring_table[id]);
+		}
+	}
+}
+
+void
+dhd_update_multicilent_flow_rings(dhd_pub_t *dhdp, uint8 ifindex, bool increment)
+{
+	uint32 id;
+	flow_ring_table_t *flow_ring_table;
+
+	DHD_ERROR(("%s: ifindex %u\n", __FUNCTION__, ifindex));
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return;
+
+	if (!dhdp->flow_ring_table)
+		return;
+
+	flow_ring_table = (flow_ring_table_t *)dhdp->flow_ring_table;
+	for (id = 0; id < dhdp->num_h2d_rings; id++) {
+		if (flow_ring_table[id].active &&
+			(flow_ring_table[id].flow_info.ifindex == ifindex) &&
+			(flow_ring_table[id].status == FLOW_RING_STATUS_OPEN)) {
+			if (increment) {
+				if (OSL_ATOMIC_READ(dhdp->osh, &dhdp->multi_client_flow_rings) <
+					dhdp->max_multi_client_flow_rings) {
+					OSL_ATOMIC_INC(dhdp->osh, &dhdp->multi_client_flow_rings);
+				} else {
+					DHD_ERROR(("%s: multi_client_flow_rings:%u"
+						" reached max:%d\n", __FUNCTION__,
+						OSL_ATOMIC_READ(dhdp->osh,
+						&dhdp->multi_client_flow_rings),
+						dhdp->max_multi_client_flow_rings));
+				}
+			} else {
+				if (OSL_ATOMIC_READ(dhdp->osh, &dhdp->multi_client_flow_rings)) {
+					OSL_ATOMIC_DEC(dhdp->osh, &dhdp->multi_client_flow_rings);
+				} else {
+					DHD_ERROR(("%s: multi_client_flow_rings:%u"
+						" reached ZERO\n", __FUNCTION__,
+						OSL_ATOMIC_READ(dhdp->osh,
+						&dhdp->multi_client_flow_rings)));
+				}
+			}
+		}
+	}
+}
+void
+dhd_flow_rings_flush(dhd_pub_t *dhdp, uint8 ifindex)
+{
+	uint32 id;
+	flow_ring_table_t *flow_ring_table;
+
+	DHD_INFO(("%s: ifindex %u\n", __FUNCTION__, ifindex));
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return;
+
+	if (!dhdp->flow_ring_table)
+		return;
+	flow_ring_table = (flow_ring_table_t *)dhdp->flow_ring_table;
+
+	for (id = 0; id < dhdp->num_h2d_rings; id++) {
+		if (flow_ring_table[id].active &&
+			(flow_ring_table[id].flow_info.ifindex == ifindex) &&
+			(flow_ring_table[id].status == FLOW_RING_STATUS_OPEN)) {
+			dhd_bus_flow_ring_flush_request(dhdp->bus,
+			                                 (void *) &flow_ring_table[id]);
+		}
+	}
+}
+
+/** Delete flow ring(s) for given peer address. */
+void
+dhd_flow_rings_delete_for_peer(dhd_pub_t *dhdp, uint8 ifindex, char *addr)
+{
+	uint32 id;
+	flow_ring_table_t *flow_ring_table;
+
+	DHD_INFO(("%s: ifindex %u\n", __FUNCTION__, ifindex));
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return;
+
+	if (!dhdp->flow_ring_table)
+		return;
+
+	flow_ring_table = (flow_ring_table_t *)dhdp->flow_ring_table;
+	for (id = 0; id < dhdp->num_h2d_rings; id++) {
+		/*
+		 * Send flowring delete request even if flowring status is
+		 * FLOW_RING_STATUS_CREATE_PENDING, to handle cases where DISASSOC_IND
+		 * event comes ahead of flowring create response.
+		 * Otherwise the flowring will not be deleted later as there will not be any
+		 * DISASSOC_IND event. With this change, when create response event comes to DHD,
+		 * it will change the status to FLOW_RING_STATUS_OPEN and soon delete response
+		 * event will come, upon which DHD will delete the flowring.
+		 */
+		if (flow_ring_table[id].active &&
+			(flow_ring_table[id].flow_info.ifindex == ifindex) &&
+			(!memcmp(flow_ring_table[id].flow_info.da, addr, ETHER_ADDR_LEN)) &&
+			((flow_ring_table[id].status == FLOW_RING_STATUS_OPEN) ||
+			(flow_ring_table[id].status == FLOW_RING_STATUS_CREATE_PENDING))) {
+			DHD_INFO(("%s: deleting flowid %d\n",
+				__FUNCTION__, flow_ring_table[id].flowid));
+			dhd_bus_flow_ring_delete_request(dhdp->bus,
+				(void *) &flow_ring_table[id]);
+		}
+	}
+}
+
+/** Handles interface ADD, CHANGE, DEL indications from the dongle */
+void
+dhd_update_interface_flow_info(dhd_pub_t *dhdp, uint8 ifindex,
+                               uint8 op, uint8 role)
+{
+	if_flow_lkup_t *if_flow_lkup;
+	unsigned long flags;
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return;
+
+	DHD_INFO(("%s: ifindex %u op %u role is %u \n",
+	          __FUNCTION__, ifindex, op, role));
+	if (!dhdp->flowid_allocator) {
+		DHD_ERROR(("%s: Flow ring not intited yet  \n", __FUNCTION__));
+		return;
+	}
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+
+	if (op == WLC_E_IF_ADD || op == WLC_E_IF_CHANGE) {
+		DHD_ERROR(("%s: ifindex:%d previous role:%d new role:%d\n",
+			__FUNCTION__, ifindex, if_flow_lkup[ifindex].role, role));
+		if_flow_lkup[ifindex].role = role;
+#ifdef PCIE_FULL_DONGLE
+		if (op == WLC_E_IF_CHANGE) {
+			bool increment = DHD_IF_ROLE_MULTI_CLIENT(dhdp, ifindex);
+			dhd_update_multicilent_flow_rings(dhdp, ifindex, increment);
+		}
+#endif /* PCIE_FULL_DONGLE */
+
+		if (role == WLC_E_IF_ROLE_WDS) {
+			/**
+			 * WDS role does not send WLC_E_LINK event after interface is up.
+			 * So to create flowrings for WDS, make status as TRUE in WLC_E_IF itself.
+			 * same is true while making the status as FALSE.
+			 * TODO: Fix FW to send WLC_E_LINK for WDS role aswell. So that all the
+			 * interfaces are handled uniformly.
+			 */
+			if_flow_lkup[ifindex].status = TRUE;
+			DHD_INFO(("%s: Mcast Flow ring for ifindex %d role is %d \n",
+			          __FUNCTION__, ifindex, role));
+		}
+	} else	if ((op == WLC_E_IF_DEL) && (role == WLC_E_IF_ROLE_WDS)) {
+		if_flow_lkup[ifindex].status = FALSE;
+		DHD_INFO(("%s: cleanup all Flow rings for ifindex %d role is %d \n",
+		          __FUNCTION__, ifindex, role));
+	}
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+}
+
+/** Handles a STA 'link' indication from the dongle */
+int
+dhd_update_interface_link_status(dhd_pub_t *dhdp, uint8 ifindex, uint8 status)
+{
+	if_flow_lkup_t *if_flow_lkup;
+	unsigned long flags;
+
+	ASSERT(ifindex < DHD_MAX_IFS);
+	if (ifindex >= DHD_MAX_IFS)
+		return BCME_BADARG;
+
+	DHD_INFO(("%s: ifindex %d status %d\n", __FUNCTION__, ifindex, status));
+
+	DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+
+	if (status) {
+		if_flow_lkup[ifindex].status = TRUE;
+	} else {
+		if_flow_lkup[ifindex].status = FALSE;
+	}
+
+	DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+	return BCME_OK;
+}
+
+/** Update flow priority mapping, called on IOVAR */
+int dhd_update_flow_prio_map(dhd_pub_t *dhdp, uint8 map)
+{
+	uint16 flowid;
+	flow_ring_node_t *flow_ring_node;
+
+	if (map > DHD_FLOW_PRIO_LLR_MAP)
+		return BCME_BADOPTION;
+
+	/* Check if we need to change prio map */
+	if (map == dhdp->flow_prio_map_type)
+		return BCME_OK;
+
+	/* If any ring is active we cannot change priority mapping for flow rings */
+	for (flowid = 0; flowid < dhdp->num_h2d_rings; flowid++) {
+		flow_ring_node = DHD_FLOW_RING(dhdp, flowid);
+		if (flow_ring_node->active)
+			return BCME_EPERM;
+	}
+
+	/* Inform firmware about new mapping type */
+	if (BCME_OK != dhd_flow_prio_map(dhdp, &map, TRUE))
+		return BCME_ERROR;
+
+	/* update internal structures */
+	dhdp->flow_prio_map_type = map;
+	if (dhdp->flow_prio_map_type == DHD_FLOW_PRIO_TID_MAP)
+		bcopy(prio2tid, dhdp->flow_prio_map, sizeof(uint8) * NUMPRIO);
+	else
+		bcopy(prio2ac, dhdp->flow_prio_map, sizeof(uint8) * NUMPRIO);
+
+	dhdp->max_multi_client_flow_rings = dhd_get_max_multi_client_flow_rings(dhdp);
+
+	return BCME_OK;
+}
+
+/** Inform firmware on updated flow priority mapping, called on IOVAR */
+int dhd_flow_prio_map(dhd_pub_t *dhd, uint8 *map, bool set)
+{
+	uint8 iovbuf[WLC_IOCTL_SMLEN];
+	int len;
+	uint32 val;
+	if (!set) {
+		bzero(&iovbuf, sizeof(iovbuf));
+		len = bcm_mkiovar("bus:fl_prio_map", NULL, 0, (char*)iovbuf, sizeof(iovbuf));
+		if (len == 0) {
+			return BCME_BUFTOOSHORT;
+		}
+		if (dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, iovbuf, sizeof(iovbuf), FALSE, 0) < 0) {
+			DHD_ERROR(("%s: failed to get fl_prio_map\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+		*map = iovbuf[0];
+		return BCME_OK;
+	}
+	val = (uint32)map[0];
+	len = bcm_mkiovar("bus:fl_prio_map", (char *)&val, sizeof(val),
+		(char*)iovbuf, sizeof(iovbuf));
+	if (len == 0) {
+		return BCME_BUFTOOSHORT;
+	}
+	if (dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, len, TRUE, 0) < 0) {
+		DHD_ERROR(("%s: failed to set fl_prio_map \n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+
+uint32
+dhd_active_tx_flowring_bkpq_len(dhd_pub_t *dhd)
+{
+	unsigned long list_lock_flags;
+	dll_t *item, *prev;
+	flow_ring_node_t *flow_ring_node;
+	dhd_bus_t *bus = dhd->bus;
+	uint32 active_tx_flowring_qlen = 0;
+
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, list_lock_flags);
+
+	for (item = dll_tail_p(&bus->flowring_active_list);
+			!dll_end(&bus->flowring_active_list, item); item = prev) {
+
+		prev = dll_prev_p(item);
+
+		flow_ring_node = dhd_constlist_to_flowring(item);
+		if (flow_ring_node->active) {
+			DHD_INFO(("%s :%d\n", __FUNCTION__, flow_ring_node->queue.len));
+			active_tx_flowring_qlen += flow_ring_node->queue.len;
+		}
+	}
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, list_lock_flags);
+	return active_tx_flowring_qlen;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_flowring.h b/drivers/net/wireless/ap12275/dhd_flowring.h
new file mode 100644
index 000000000000..1b1d1d0d20b9
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_flowring.h
@@ -0,0 +1,310 @@
+/*
+ * @file Header file describing the flow rings DHD interfaces.
+ *
+ * Flow rings are transmit traffic (=propagating towards antenna) related entities.
+ *
+ * Provides type definitions and function prototypes used to create, delete and manage flow rings at
+ * high level.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/** XXX Twiki: [PCIeFullDongleArchitecture] */
+
+/****************
+ * Common types *
+ */
+
+#ifndef _dhd_flowrings_h_
+#define _dhd_flowrings_h_
+
+/* Max pkts held in a flow ring's backup queue */
+#define FLOW_RING_QUEUE_THRESHOLD       (2048)
+
+/* Number of H2D common rings */
+#define FLOW_RING_COMMON                BCMPCIE_H2D_COMMON_MSGRINGS
+
+#define FLOWID_INVALID                  (ID16_INVALID)
+#define FLOWID_RESERVED                 (FLOW_RING_COMMON)
+
+#define FLOW_RING_STATUS_OPEN           0
+#define FLOW_RING_STATUS_CREATE_PENDING	1
+#define FLOW_RING_STATUS_CLOSED         2
+#define FLOW_RING_STATUS_DELETE_PENDING 3
+#define FLOW_RING_STATUS_FLUSH_PENDING  4
+
+#ifdef IDLE_TX_FLOW_MGMT
+#define FLOW_RING_STATUS_SUSPENDED	5
+#define FLOW_RING_STATUS_RESUME_PENDING	6
+#endif /* IDLE_TX_FLOW_MGMT */
+#define FLOW_RING_STATUS_STA_FREEING    7
+
+#define HTPUT_FLOW_RING_PRIO		PRIO_8021D_BE
+#define HTPUT_NUM_STA_FLOW_RINGS	1u
+#define HTPUT_NUM_CLIENT_FLOW_RINGS	3u
+#define HTPUT_TOTAL_FLOW_RINGS		(HTPUT_NUM_STA_FLOW_RINGS + HTPUT_NUM_CLIENT_FLOW_RINGS)
+#define DHD_IS_FLOWID_HTPUT(pub, flowid) \
+	((pub)->htput_support && (flowid >= (pub)->htput_flow_ring_start) && \
+	((uint16)flowid < ((pub)->htput_flow_ring_start + HTPUT_TOTAL_FLOW_RINGS)))
+
+#if defined(FLOW_RING_PREALLOC)
+#define MAX_FLOW_RINGS 64
+#endif /* FLOW_RING_PREALLOC */
+
+#define DHD_FLOWRING_RX_BUFPOST_PKTSZ	2048
+
+#define DHD_FLOWRING_RX_BUFPOST_PKTSZ_MAX 4096
+
+#define DHD_FLOWRING_TX_BIG_PKT_SIZE	(3700u)
+
+#define DHD_FLOW_PRIO_AC_MAP		0
+#define DHD_FLOW_PRIO_TID_MAP		1
+/* Flow ring prority map for lossless roaming */
+#define DHD_FLOW_PRIO_LLR_MAP		2
+
+/* Hashing a MacAddress for lkup into a per interface flow hash table */
+#define DHD_FLOWRING_HASH_SIZE    256
+#define	DHD_FLOWRING_HASHINDEX(ea, prio) \
+	       ((((uint8 *)(ea))[3] ^ ((uint8 *)(ea))[4] ^ ((uint8 *)(ea))[5] ^ ((uint8)(prio))) \
+		% DHD_FLOWRING_HASH_SIZE)
+
+#define DHD_IF_ROLE(pub, idx)		(((if_flow_lkup_t *)(pub)->if_flow_lkup)[idx].role)
+#define DHD_IF_ROLE_AP(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_AP)
+#define DHD_IF_ROLE_STA(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_STA)
+#define DHD_IF_ROLE_P2PGC(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_P2P_CLIENT)
+#define DHD_IF_ROLE_P2PGO(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_P2P_GO)
+#define DHD_IF_ROLE_WDS(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_WDS)
+#define DHD_IF_ROLE_IBSS(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_IBSS)
+#define DHD_IF_ROLE_NAN(pub, idx)	(DHD_IF_ROLE(pub, idx) == WLC_E_IF_ROLE_NAN)
+
+#define DHD_IF_ROLE_GENERIC_STA(pub, idx) \
+	(DHD_IF_ROLE_STA(pub, idx) || DHD_IF_ROLE_P2PGC(pub, idx) || DHD_IF_ROLE_WDS(pub, idx))
+
+#define DHD_IF_ROLE_MULTI_CLIENT(pub, idx) \
+	(DHD_IF_ROLE_AP(pub, idx) || DHD_IF_ROLE_P2PGO(pub, idx) ||\
+		DHD_IF_ROLE_NAN(pub, idx))
+
+#define DHD_FLOW_RING(dhdp, flowid) \
+	(flow_ring_node_t *)&(((flow_ring_node_t *)((dhdp)->flow_ring_table))[flowid])
+
+struct flow_queue;
+
+/* Flow Ring Queue Enqueue overflow callback */
+typedef int (*flow_queue_cb_t)(struct flow_queue * queue, void * pkt);
+
+/**
+ * Each flow ring has an associated (tx flow controlled) queue. 802.3 packets are transferred
+ * between queue and ring. A packet from the host stack is first added to the queue, and in a later
+ * stage transferred to the flow ring. Packets in the queue are dhd owned, whereas packets in the
+ * flow ring are device owned.
+ */
+typedef struct flow_queue {
+	dll_t  list;                /* manage a flowring queue in a double linked list */
+	void * head;                /* first packet in the queue */
+	void * tail;                /* last packet in the queue */
+	uint16 len;                 /* number of packets in the queue */
+	uint16 max;                 /* maximum or min budget (used in cumm) */
+	uint32 threshold;           /* parent's cummulative length threshold */
+	void * clen_ptr;            /* parent's cummulative length counter */
+	uint32 failures;            /* enqueue failures due to queue overflow */
+	flow_queue_cb_t cb;         /* callback invoked on threshold crossing */
+	uint32 l2threshold;         /* grandparent's (level 2) cummulative length threshold */
+	void * l2clen_ptr;          /* grandparent's (level 2) cummulative length counter */
+} flow_queue_t;
+
+#define DHD_FLOW_QUEUE_LEN(queue)       ((int)(queue)->len)
+#define DHD_FLOW_QUEUE_MAX(queue)       ((int)(queue)->max)
+#define DHD_FLOW_QUEUE_THRESHOLD(queue) ((int)(queue)->threshold)
+#define DHD_FLOW_QUEUE_L2THRESHOLD(queue) ((int)(queue)->l2threshold)
+#define DHD_FLOW_QUEUE_EMPTY(queue)     ((queue)->len == 0)
+#define DHD_FLOW_QUEUE_FAILURES(queue)  ((queue)->failures)
+
+#define DHD_FLOW_QUEUE_AVAIL(queue)     ((int)((queue)->max - (queue)->len))
+#define DHD_FLOW_QUEUE_FULL(queue)      ((queue)->len >= (queue)->max)
+
+#define DHD_FLOW_QUEUE_OVFL(queue, budget)  \
+	(((queue)->len) > budget)
+
+#define DHD_FLOW_QUEUE_SET_MAX(queue, budget) \
+	((queue)->max) = ((budget) - 1)
+
+/* Queue's cummulative threshold. */
+#define DHD_FLOW_QUEUE_SET_THRESHOLD(queue, cumm_threshold) \
+	((queue)->threshold) = ((cumm_threshold) - 1)
+
+/* Queue's cummulative length object accessor. */
+#define DHD_FLOW_QUEUE_CLEN_PTR(queue)  ((queue)->clen_ptr)
+
+/* Set a queue's cumm_len point to a parent's cumm_ctr_t cummulative length */
+#define DHD_FLOW_QUEUE_SET_CLEN(queue, parent_clen_ptr)  \
+	((queue)->clen_ptr) = (void *)(parent_clen_ptr)
+
+/* Queue's level 2 cummulative threshold. */
+#define DHD_FLOW_QUEUE_SET_L2THRESHOLD(queue, l2cumm_threshold) \
+	((queue)->l2threshold) = ((l2cumm_threshold) - 1)
+
+/* Queue's level 2 cummulative length object accessor. */
+#define DHD_FLOW_QUEUE_L2CLEN_PTR(queue)  ((queue)->l2clen_ptr)
+
+/* Set a queue's level 2 cumm_len point to a grandparent's cumm_ctr_t cummulative length */
+#define DHD_FLOW_QUEUE_SET_L2CLEN(queue, grandparent_clen_ptr)  \
+	((queue)->l2clen_ptr) = (void *)(grandparent_clen_ptr)
+
+#if defined(BCMDBG)
+#define DHD_FLOWRING_TXSTATUS_CNT_UPDATE(bus, flowid, txstatus) \
+		dhd_bus_flow_ring_cnt_update(bus, flowid, txstatus)
+#else
+#define DHD_FLOWRING_TXSTATUS_CNT_UPDATE(bus, flowid, txstatus)
+#endif /* BCMDBG */
+
+/* Pkttag not compatible with PROP_TXSTATUS or WLFC */
+typedef struct dhd_pkttag_fr {
+	uint16  flowid;
+	uint16  ifid;
+} dhd_pkttag_fr_t;
+
+#define DHD_PKTTAG_SET_IFID(tag, idx)       ((tag)->ifid = (uint16)(idx))
+#define DHD_PKTTAG_SET_PA(tag, pa)          ((tag)->physaddr = (pa))
+#define DHD_PKTTAG_SET_PA_LEN(tag, palen)   ((tag)->pa_len = (palen))
+#define DHD_PKTTAG_IFID(tag)                ((tag)->ifid)
+#define DHD_PKTTAG_PA(tag)                  ((tag)->physaddr)
+#define DHD_PKTTAG_PA_LEN(tag)              ((tag)->pa_len)
+
+/** each flow ring is dedicated to a tid/sa/da combination */
+typedef struct flow_info {
+	uint8		tid;
+	uint8		ifindex;
+	uchar		sa[ETHER_ADDR_LEN];
+	uchar		da[ETHER_ADDR_LEN];
+#if defined(BCMDBG)
+	uint32		tx_status[DHD_MAX_TX_STATUS_MSGS];
+#endif
+#ifdef TX_STATUS_LATENCY_STATS
+	/* total number of tx_status received on this flowid */
+	uint64           num_tx_status;
+	/* cumulative tx_status latency for this flowid */
+	uint64          cum_tx_status_latency;
+	/* num tx packets sent on this flowring */
+	uint64		num_tx_pkts;
+#endif /* TX_STATUS_LATENCY_STATS */
+} flow_info_t;
+
+/** a flow ring is used for outbound (towards antenna) 802.3 packets */
+typedef struct flow_ring_node {
+	dll_t		list;  /* manage a constructed flowring in a dll, must be at first place */
+	flow_queue_t	queue; /* queues packets before they enter the flow ring, flow control */
+	bool		active;
+	uint8		status;
+	/*
+	 * flowid: unique ID of a flow ring, which can either be unicast or broadcast/multicast. For
+	 * unicast flow rings, the flow id accelerates ARM 802.3->802.11 header translation.
+	 */
+	uint16		flowid;
+	flow_info_t	flow_info;
+	void		*prot_info;
+	void		*lock; /* lock for flowring access protection */
+
+#ifdef IDLE_TX_FLOW_MGMT
+	uint64		last_active_ts; /* contains last active timestamp */
+#endif /* IDLE_TX_FLOW_MGMT */
+#ifdef DEVICE_TX_STUCK_DETECT
+	/* Time stamp(msec) when last time a Tx packet completion is received on this flow ring */
+	uint32		tx_cmpl;
+	/*
+	 * Holds the tx_cmpl which was read during the previous
+	 * iteration of the stuck detection algo
+	 */
+	uint32		tx_cmpl_prev;
+	/* counter to decide if this particlur flow is stuck or not */
+	uint32		stuck_count;
+#endif /* DEVICE_TX_STUCK_DETECT */
+} flow_ring_node_t;
+
+typedef flow_ring_node_t flow_ring_table_t;
+
+typedef struct flow_hash_info {
+	uint16			flowid;
+	flow_info_t		flow_info;
+	struct flow_hash_info	*next;
+} flow_hash_info_t;
+
+typedef struct if_flow_lkup {
+	bool		status;
+	uint8		role; /* Interface role: STA/AP */
+	flow_hash_info_t *fl_hash[DHD_FLOWRING_HASH_SIZE]; /* Lkup Hash table */
+} if_flow_lkup_t;
+
+static INLINE flow_ring_node_t *
+dhd_constlist_to_flowring(dll_t *item)
+{
+	return ((flow_ring_node_t *)item);
+}
+
+/* Exported API */
+
+/* Flow ring's queue management functions */
+extern flow_ring_node_t * dhd_flow_ring_node(dhd_pub_t *dhdp, uint16 flowid);
+extern flow_queue_t * dhd_flow_queue(dhd_pub_t *dhdp, uint16 flowid);
+
+extern void dhd_flow_queue_init(dhd_pub_t *dhdp, flow_queue_t *queue, int max);
+extern void dhd_flow_queue_reinit(dhd_pub_t *dhdp, flow_queue_t *queue, int max);
+extern void dhd_flow_queue_register(flow_queue_t *queue, flow_queue_cb_t cb);
+extern int  dhd_flow_queue_enqueue(dhd_pub_t *dhdp, flow_queue_t *queue, void *pkt);
+extern int  dhd_flow_queue_enqueue_head(dhd_pub_t *dhdp, flow_queue_t *queue, void *pkt);
+extern void * dhd_flow_queue_dequeue(dhd_pub_t *dhdp, flow_queue_t *queue);
+extern void dhd_flow_queue_reinsert(dhd_pub_t *dhdp, flow_queue_t *queue, void *pkt);
+
+extern void dhd_flow_ring_config_thresholds(dhd_pub_t *dhdp, uint16 flowid,
+                          int queue_budget, int cumm_threshold, void *cumm_ctr,
+                          int l2cumm_threshold, void *l2cumm_ctr);
+extern int  dhd_flow_rings_init(dhd_pub_t *dhdp, uint32 num_h2d_rings);
+
+extern void dhd_flow_rings_deinit(dhd_pub_t *dhdp);
+
+extern int dhd_flowid_update(dhd_pub_t *dhdp, uint8 ifindex, uint8 prio,
+                void *pktbuf);
+extern int dhd_flowid_debug_create(dhd_pub_t *dhdp, uint8 ifindex,
+	uint8 prio, char *sa, char *da, uint16 *flowid);
+extern int dhd_flowid_find_by_ifidx(dhd_pub_t *dhdp, uint8 ifidex, uint16 flowid);
+
+extern void dhd_flowid_free(dhd_pub_t *dhdp, uint8 ifindex, uint16 flowid);
+
+extern void dhd_flow_rings_delete(dhd_pub_t *dhdp, uint8 ifindex);
+extern void dhd_update_multicilent_flow_rings(dhd_pub_t *dhdp, uint8 ifindex,
+	bool increment);
+extern void dhd_flow_rings_flush(dhd_pub_t *dhdp, uint8 ifindex);
+
+extern void dhd_flow_rings_delete_for_peer(dhd_pub_t *dhdp, uint8 ifindex,
+                char *addr);
+
+/* Handle Interface ADD, DEL operations */
+extern void dhd_update_interface_flow_info(dhd_pub_t *dhdp, uint8 ifindex,
+                uint8 op, uint8 role);
+
+/* Handle a STA interface link status update */
+extern int dhd_update_interface_link_status(dhd_pub_t *dhdp, uint8 ifindex,
+                uint8 status);
+extern int dhd_flow_prio_map(dhd_pub_t *dhd, uint8 *map, bool set);
+extern int dhd_update_flow_prio_map(dhd_pub_t *dhdp, uint8 map);
+extern uint32 dhd_active_tx_flowring_bkpq_len(dhd_pub_t *dhdp);
+extern uint8 dhd_flow_rings_ifindex2role(dhd_pub_t *dhdp, uint8 ifindex);
+#endif /* _dhd_flowrings_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_gpio.c b/drivers/net/wireless/ap12275/dhd_gpio.c
new file mode 100644
index 000000000000..8449b9bf626f
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_gpio.c
@@ -0,0 +1,406 @@
+
+#include <osl.h>
+#include <dhd_linux.h>
+#include <linux/gpio.h>
+#ifdef BCMDHD_DTS
+#include <linux/of_gpio.h>
+#endif
+#ifdef BCMDHD_PLATDEV
+#include <linux/platform_device.h>
+#endif
+#include <dhd_dbg.h>
+
+#ifdef CONFIG_DHD_USE_STATIC_BUF
+#if defined(BCMDHD_MDRIVER) && !defined(DHD_STATIC_IN_DRIVER)
+extern void *dhd_wlan_mem_prealloc(uint bus_type, int index,
+	int section, unsigned long size);
+#else
+extern void *dhd_wlan_mem_prealloc(int section, unsigned long size);
+#endif
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+
+#ifdef BCMDHD_DTS
+/* This is sample code in dts file.
+bcmdhd_wlan {
+	compatible = "android,bcmdhd_wlan";
+	gpio_wl_reg_on = <&gpio GPIOH_4 GPIO_ACTIVE_HIGH>;
+	gpio_wl_host_wake = <&gpio GPIOZ_15 GPIO_ACTIVE_HIGH>;
+};
+*/
+#define DHD_DT_COMPAT_ENTRY		"android,bcmdhd_wlan"
+#define GPIO_WL_REG_ON_PROPNAME		"gpio_wl_reg_on"
+#define GPIO_WL_HOST_WAKE_PROPNAME	"gpio_wl_host_wake"
+#endif
+
+static int
+dhd_wlan_set_power(int on, wifi_adapter_info_t *adapter)
+{
+	int gpio_wl_reg_on = adapter->gpio_wl_reg_on;
+	int err = 0;
+
+	if (on) {
+		DHD_INFO(("======== PULL WL_REG_ON(%d) HIGH! ========\n", gpio_wl_reg_on));
+		if (gpio_wl_reg_on >= 0) {
+			err = gpio_direction_output(gpio_wl_reg_on, 1);
+			if (err) {
+				printf("%s: WL_REG_ON didn't output high\n", __FUNCTION__);
+				return -EIO;
+			}
+		}
+#ifdef BUS_POWER_RESTORE
+#ifdef BCMPCIE
+		if (adapter->pci_dev) {
+			mdelay(100);
+			DHD_INFO(("======== pci_set_power_state PCI_D0! ========\n"));
+			pci_set_power_state(adapter->pci_dev, PCI_D0);
+			if (adapter->pci_saved_state)
+				pci_load_and_free_saved_state(adapter->pci_dev, &adapter->pci_saved_state);
+			pci_restore_state(adapter->pci_dev);
+			err = pci_enable_device(adapter->pci_dev);
+			if (err < 0)
+				printf("%s: PCI enable device failed", __FUNCTION__);
+			pci_set_master(adapter->pci_dev);
+		}
+#endif /* BCMPCIE */
+#endif /* BUS_POWER_RESTORE */
+		/* Lets customer power to get stable */
+	} else {
+#ifdef BUS_POWER_RESTORE
+#ifdef BCMPCIE
+		if (adapter->pci_dev) {
+			DHD_INFO(("======== pci_set_power_state PCI_D3hot! ========\n"));
+			pci_save_state(adapter->pci_dev);
+			adapter->pci_saved_state = pci_store_saved_state(adapter->pci_dev);
+			if (pci_is_enabled(adapter->pci_dev))
+				pci_disable_device(adapter->pci_dev);
+			pci_set_power_state(adapter->pci_dev, PCI_D3hot);
+		}
+#endif /* BCMPCIE */
+#endif /* BUS_POWER_RESTORE */
+		DHD_INFO(("======== PULL WL_REG_ON(%d) LOW! ========\n", gpio_wl_reg_on));
+		if (gpio_wl_reg_on >= 0) {
+			err = gpio_direction_output(gpio_wl_reg_on, 0);
+			if (err) {
+				printf("%s: WL_REG_ON didn't output low\n", __FUNCTION__);
+				return -EIO;
+			}
+		}
+	}
+
+	return err;
+}
+
+static int
+dhd_wlan_set_reset(int onoff)
+{
+	return 0;
+}
+
+static int
+dhd_wlan_set_carddetect(int present)
+{
+	int err = 0;
+
+	if (present) {
+#if defined(BCMSDIO)
+		DHD_INFO(("======== Card detection to detect SDIO card! ========\n"));
+#ifdef CUSTOMER_HW_PLATFORM
+		err = sdhci_force_presence_change(&sdmmc_channel, 1);
+#endif /* CUSTOMER_HW_PLATFORM */
+#elif defined(BCMPCIE)
+		DHD_INFO(("======== Card detection to detect PCIE card! ========\n"));
+#endif
+	} else {
+#if defined(BCMSDIO)
+		DHD_INFO(("======== Card detection to remove SDIO card! ========\n"));
+#ifdef CUSTOMER_HW_PLATFORM
+		err = sdhci_force_presence_change(&sdmmc_channel, 0);
+#endif /* CUSTOMER_HW_PLATFORM */
+#elif defined(BCMPCIE)
+		DHD_INFO(("======== Card detection to remove PCIE card! ========\n"));
+#endif
+	}
+
+	return err;
+}
+
+static int
+dhd_wlan_get_mac_addr(unsigned char *buf, int ifidx)
+{
+	int err = -1;
+
+	if (ifidx == 1) {
+#ifdef EXAMPLE_GET_MAC
+		struct ether_addr ea_example = {{0x00, 0x11, 0x22, 0x33, 0x44, 0xFF}};
+		bcopy((char *)&ea_example, buf, sizeof(struct ether_addr));
+#endif /* EXAMPLE_GET_MAC */
+	} else {
+#ifdef EXAMPLE_GET_MAC
+		struct ether_addr ea_example = {{0x02, 0x11, 0x22, 0x33, 0x44, 0x55}};
+		bcopy((char *)&ea_example, buf, sizeof(struct ether_addr));
+#endif /* EXAMPLE_GET_MAC */
+	}
+
+#ifdef EXAMPLE_GET_MAC_VER2
+	/* EXAMPLE code */
+	{
+		char macpad[56]= {
+		0x00,0xaa,0x9c,0x84,0xc7,0xbc,0x9b,0xf6,
+		0x02,0x33,0xa9,0x4d,0x5c,0xb4,0x0a,0x5d,
+		0xa8,0xef,0xb0,0xcf,0x8e,0xbf,0x24,0x8a,
+		0x87,0x0f,0x6f,0x0d,0xeb,0x83,0x6a,0x70,
+		0x4a,0xeb,0xf6,0xe6,0x3c,0xe7,0x5f,0xfc,
+		0x0e,0xa7,0xb3,0x0f,0x00,0xe4,0x4a,0xaf,
+		0x87,0x08,0x16,0x6d,0x3a,0xe3,0xc7,0x80};
+		bcopy(macpad, buf+6, sizeof(macpad));
+	}
+#endif /* EXAMPLE_GET_MAC_VER2 */
+
+	DHD_INFO(("======== %s err=%d ========\n", __FUNCTION__, err));
+
+	return err;
+}
+
+static struct cntry_locales_custom brcm_wlan_translate_custom_table[] = {
+	/* Table should be filled out based on custom platform regulatory requirement */
+#ifdef EXAMPLE_TABLE
+	{"",   "XT", 49},  /* Universal if Country code is unknown or empty */
+	{"US", "US", 0},
+#endif /* EXMAPLE_TABLE */
+};
+
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+struct cntry_locales_custom brcm_wlan_translate_nodfs_table[] = {
+#ifdef EXAMPLE_TABLE
+	{"",   "XT", 50},  /* Universal if Country code is unknown or empty */
+	{"US", "US", 0},
+#endif /* EXMAPLE_TABLE */
+};
+#endif
+
+static void *dhd_wlan_get_country_code(char *ccode
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+	, u32 flags
+#endif
+)
+{
+	struct cntry_locales_custom *locales;
+	int size;
+	int i;
+
+	if (!ccode)
+		return NULL;
+
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+	if (flags & WLAN_PLAT_NODFS_FLAG) {
+		locales = brcm_wlan_translate_nodfs_table;
+		size = ARRAY_SIZE(brcm_wlan_translate_nodfs_table);
+	} else {
+#endif
+		locales = brcm_wlan_translate_custom_table;
+		size = ARRAY_SIZE(brcm_wlan_translate_custom_table);
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+	}
+#endif
+
+	for (i = 0; i < size; i++)
+		if (strcmp(ccode, locales[i].iso_abbrev) == 0)
+			return &locales[i];
+	return NULL;
+}
+
+struct wifi_platform_data dhd_wlan_control = {
+	.set_power	= dhd_wlan_set_power,
+	.set_reset	= dhd_wlan_set_reset,
+	.set_carddetect	= dhd_wlan_set_carddetect,
+	.get_mac_addr	= dhd_wlan_get_mac_addr,
+#ifdef CONFIG_DHD_USE_STATIC_BUF
+	.mem_prealloc	= dhd_wlan_mem_prealloc,
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+	.get_country_code = dhd_wlan_get_country_code,
+};
+
+static int
+dhd_wlan_init_gpio(wifi_adapter_info_t *adapter)
+{
+#ifdef BCMDHD_DTS
+	char wlan_node[32];
+	struct device_node *root_node = NULL;
+#endif
+	int err = 0;
+	int gpio_wl_reg_on = -1;
+#ifdef CUSTOMER_OOB
+	int gpio_wl_host_wake = -1;
+	int host_oob_irq = -1;
+	uint host_oob_irq_flags = 0;
+#endif
+
+	/* Please check your schematic and fill right GPIO number which connected to
+	* WL_REG_ON and WL_HOST_WAKE.
+	*/
+#ifdef BCMDHD_DTS
+#ifdef BCMDHD_PLATDEV
+	if (adapter->pdev) {
+		root_node = adapter->pdev->dev.of_node;
+		strcpy(wlan_node, root_node->name);
+	} else {
+		printf("%s: adapter->pdev is NULL\n", __FUNCTION__);
+		return -1;
+	}
+#else
+	strcpy(wlan_node, DHD_DT_COMPAT_ENTRY);
+	root_node = of_find_compatible_node(NULL, NULL, wlan_node);
+#endif
+	DHD_INFO(("======== Get GPIO from DTS(%s) ========\n", wlan_node));
+	if (root_node) {
+		gpio_wl_reg_on = of_get_named_gpio(root_node, GPIO_WL_REG_ON_PROPNAME, 0);
+#ifdef CUSTOMER_OOB
+		gpio_wl_host_wake = of_get_named_gpio(root_node, GPIO_WL_HOST_WAKE_PROPNAME, 0);
+#endif
+	} else
+#endif
+	{
+		gpio_wl_reg_on = -1;
+#ifdef CUSTOMER_OOB
+		gpio_wl_host_wake = -1;
+#endif
+	}
+
+	if (gpio_wl_reg_on >= 0) {
+		err = gpio_request(gpio_wl_reg_on, "WL_REG_ON");
+		if (err < 0) {
+			printf("%s: gpio_request(%d) for WL_REG_ON failed %d\n",
+				__FUNCTION__, gpio_wl_reg_on, err);
+			gpio_wl_reg_on = -1;
+		}
+	}
+	adapter->gpio_wl_reg_on = gpio_wl_reg_on;
+
+#ifdef CUSTOMER_OOB
+	if (gpio_wl_host_wake >= 0) {
+		err = gpio_request(gpio_wl_host_wake, "bcmdhd");
+		if (err < 0) {
+			printf("%s: gpio_request(%d) for WL_HOST_WAKE failed %d\n",
+				__FUNCTION__, gpio_wl_host_wake, err);
+			return -1;
+		}
+		adapter->gpio_wl_host_wake = gpio_wl_host_wake;
+		err = gpio_direction_input(gpio_wl_host_wake);
+		if (err < 0) {
+			printf("%s: gpio_direction_input(%d) for WL_HOST_WAKE failed %d\n",
+				__FUNCTION__, gpio_wl_host_wake, err);
+			gpio_free(gpio_wl_host_wake);
+			return -1;
+		}
+		host_oob_irq = gpio_to_irq(gpio_wl_host_wake);
+		if (host_oob_irq < 0) {
+			printf("%s: gpio_to_irq(%d) for WL_HOST_WAKE failed %d\n",
+				__FUNCTION__, gpio_wl_host_wake, host_oob_irq);
+			gpio_free(gpio_wl_host_wake);
+			return -1;
+		}
+	}
+
+#ifdef HW_OOB
+#ifdef HW_OOB_LOW_LEVEL
+	host_oob_irq_flags = IORESOURCE_IRQ | IORESOURCE_IRQ_LOWLEVEL | IORESOURCE_IRQ_SHAREABLE;
+#else
+	host_oob_irq_flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL | IORESOURCE_IRQ_SHAREABLE;
+#endif
+#else
+	host_oob_irq_flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHEDGE | IORESOURCE_IRQ_SHAREABLE;
+#endif
+	host_oob_irq_flags &= IRQF_TRIGGER_MASK;
+
+	adapter->irq_num = host_oob_irq;
+	adapter->intr_flags = host_oob_irq_flags;
+	DHD_INFO(("%s: WL_HOST_WAKE=%d, oob_irq=%d, oob_irq_flags=0x%x\n", __FUNCTION__,
+		gpio_wl_host_wake, host_oob_irq, host_oob_irq_flags));
+#endif /* CUSTOMER_OOB */
+	DHD_INFO(("%s: WL_REG_ON=%d\n", __FUNCTION__, gpio_wl_reg_on));
+
+	return 0;
+}
+
+static void
+dhd_wlan_deinit_gpio(wifi_adapter_info_t *adapter)
+{
+	int gpio_wl_reg_on = adapter->gpio_wl_reg_on;
+#ifdef CUSTOMER_OOB
+	int gpio_wl_host_wake = adapter->gpio_wl_host_wake;
+#endif
+
+	if (gpio_wl_reg_on >= 0) {
+		DHD_INFO(("%s: gpio_free(WL_REG_ON %d)\n", __FUNCTION__, gpio_wl_reg_on));
+		gpio_free(gpio_wl_reg_on);
+		adapter->gpio_wl_reg_on = -1;
+	}
+#ifdef CUSTOMER_OOB
+	if (gpio_wl_host_wake >= 0) {
+		DHD_INFO(("%s: gpio_free(WL_HOST_WAKE %d)\n", __FUNCTION__, gpio_wl_host_wake));
+		gpio_free(gpio_wl_host_wake);
+		adapter->gpio_wl_host_wake = -1;
+	}
+#endif /* CUSTOMER_OOB */
+}
+
+#if defined(BCMDHD_MDRIVER)
+static void
+dhd_wlan_init_adapter(wifi_adapter_info_t *adapter)
+{
+#ifdef ADAPTER_IDX
+	if (ADAPTER_IDX == 0) {
+		adapter->bus_num = 1;
+		adapter->slot_num = 1;
+	} else if (ADAPTER_IDX == 1) {
+		adapter->bus_num = 2;
+		adapter->slot_num = 1;
+	}
+	adapter->index = ADAPTER_IDX;
+#ifdef BCMSDIO
+	adapter->bus_type = SDIO_BUS;
+#elif defined(BCMPCIE)
+	adapter->bus_type = PCI_BUS;
+#elif defined(BCMDBUS)
+	adapter->bus_type = USB_BUS;
+#endif
+	DHD_INFO(("bus_type=%d, bus_num=%d, slot_num=%d\n",
+		adapter->bus_type, adapter->bus_num, adapter->slot_num));
+#endif /* ADAPTER_IDX */
+
+#ifdef DHD_STATIC_IN_DRIVER
+	adapter->index = 0;
+#elif !defined(ADAPTER_IDX)
+#ifdef BCMSDIO
+	adapter->index = 0;
+#elif defined(BCMPCIE)
+	adapter->index = 1;
+#elif defined(BCMDBUS)
+	adapter->index = 2;
+#endif
+#endif /* DHD_STATIC_IN_DRIVER */
+}
+#endif /* BCMDHD_MDRIVER */
+
+int
+dhd_wlan_init_plat_data(wifi_adapter_info_t *adapter)
+{
+	int err = 0;
+
+#ifdef BCMDHD_MDRIVER
+	dhd_wlan_init_adapter(adapter);
+#endif /* BCMDHD_MDRIVER */
+
+	err = dhd_wlan_init_gpio(adapter);
+	if (err)
+		goto exit;
+
+exit:
+	return err;
+}
+
+void
+dhd_wlan_deinit_plat_data(wifi_adapter_info_t *adapter)
+{
+	dhd_wlan_deinit_gpio(adapter);
+}
diff --git a/drivers/net/wireless/ap12275/dhd_ip.c b/drivers/net/wireless/ap12275/dhd_ip.c
new file mode 100644
index 000000000000..32788cccf560
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_ip.c
@@ -0,0 +1,1435 @@
+/*
+ * IP Packet Parser Module.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <typedefs.h>
+#include <osl.h>
+
+#include <ethernet.h>
+#include <vlan.h>
+#include <802.3.h>
+#include <bcmip.h>
+#include <bcmendian.h>
+
+#include <dhd.h>
+#include <dhd_dbg.h>
+
+#include <dhd_ip.h>
+#include <dhd_config.h>
+
+#if defined(DHDTCPACK_SUPPRESS) || defined(DHDTCPSYNC_FLOOD_BLK)
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <bcmtcp.h>
+#endif /* DHDTCPACK_SUPPRESS || DHDTCPSYNC_FLOOD_BLK */
+
+/* special values */
+/* 802.3 llc/snap header */
+static const uint8 llc_snap_hdr[SNAP_HDR_LEN] = {0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00};
+
+pkt_frag_t pkt_frag_info(osl_t *osh, void *p)
+{
+	uint8 *frame;
+	int length;
+	uint8 *pt;			/* Pointer to type field */
+	uint16 ethertype;
+	struct ipv4_hdr *iph;		/* IP frame pointer */
+	int ipl;			/* IP frame length */
+	uint16 iph_frag;
+
+	ASSERT(osh && p);
+
+	frame = PKTDATA(osh, p);
+	length = PKTLEN(osh, p);
+
+	/* Process Ethernet II or SNAP-encapsulated 802.3 frames */
+	if (length < ETHER_HDR_LEN) {
+		DHD_INFO(("%s: short eth frame (%d)\n", __FUNCTION__, length));
+		return DHD_PKT_FRAG_NONE;
+	} else if (ntoh16(*(uint16 *)(frame + ETHER_TYPE_OFFSET)) >= ETHER_TYPE_MIN) {
+		/* Frame is Ethernet II */
+		pt = frame + ETHER_TYPE_OFFSET;
+	} else if (length >= ETHER_HDR_LEN + SNAP_HDR_LEN + ETHER_TYPE_LEN &&
+	           !bcmp(llc_snap_hdr, frame + ETHER_HDR_LEN, SNAP_HDR_LEN)) {
+		pt = frame + ETHER_HDR_LEN + SNAP_HDR_LEN;
+	} else {
+		DHD_INFO(("%s: non-SNAP 802.3 frame\n", __FUNCTION__));
+		return DHD_PKT_FRAG_NONE;
+	}
+
+	ethertype = ntoh16(*(uint16 *)pt);
+
+	/* Skip VLAN tag, if any */
+	if (ethertype == ETHER_TYPE_8021Q) {
+		pt += VLAN_TAG_LEN;
+
+		if (pt + ETHER_TYPE_LEN > frame + length) {
+			DHD_INFO(("%s: short VLAN frame (%d)\n", __FUNCTION__, length));
+			return DHD_PKT_FRAG_NONE;
+		}
+
+		ethertype = ntoh16(*(uint16 *)pt);
+	}
+
+	if (ethertype != ETHER_TYPE_IP) {
+		DHD_INFO(("%s: non-IP frame (ethertype 0x%x, length %d)\n",
+			__FUNCTION__, ethertype, length));
+		return DHD_PKT_FRAG_NONE;
+	}
+
+	iph = (struct ipv4_hdr *)(pt + ETHER_TYPE_LEN);
+	ipl = (uint)(length - (pt + ETHER_TYPE_LEN - frame));
+
+	/* We support IPv4 only */
+	if ((ipl < IPV4_OPTIONS_OFFSET) || (IP_VER(iph) != IP_VER_4)) {
+		DHD_INFO(("%s: short frame (%d) or non-IPv4\n", __FUNCTION__, ipl));
+		return DHD_PKT_FRAG_NONE;
+	}
+
+	iph_frag = ntoh16(iph->frag);
+
+	if (iph_frag & IPV4_FRAG_DONT) {
+		return DHD_PKT_FRAG_NONE;
+	} else if ((iph_frag & IPV4_FRAG_MORE) == 0) {
+		return DHD_PKT_FRAG_LAST;
+	} else {
+		return (iph_frag & IPV4_FRAG_OFFSET_MASK)? DHD_PKT_FRAG_CONT : DHD_PKT_FRAG_FIRST;
+	}
+}
+
+#ifdef DHDTCPACK_SUPPRESS
+
+typedef struct {
+	void *pkt_in_q;		/* TCP ACK packet that is already in txq or DelayQ */
+	void *pkt_ether_hdr;	/* Ethernet header pointer of pkt_in_q */
+	int ifidx;
+	uint8 supp_cnt;
+	dhd_pub_t *dhdp;
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+	timer_list_compat_t timer;
+#else
+	struct tasklet_hrtimer timer;
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+} tcpack_info_t;
+
+typedef struct _tdata_psh_info_t {
+	uint32 end_seq;			/* end seq# of a received TCP PSH DATA pkt */
+	struct _tdata_psh_info_t *next;	/* next pointer of the link chain */
+} tdata_psh_info_t;
+
+typedef struct {
+	struct {
+		uint8 src[IPV4_ADDR_LEN];	/* SRC ip addrs of this TCP stream */
+		uint8 dst[IPV4_ADDR_LEN];	/* DST ip addrs of this TCP stream */
+	} ip_addr;
+	struct {
+		uint8 src[TCP_PORT_LEN];	/* SRC tcp ports of this TCP stream */
+		uint8 dst[TCP_PORT_LEN];	/* DST tcp ports of this TCP stream */
+	} tcp_port;
+	tdata_psh_info_t *tdata_psh_info_head;	/* Head of received TCP PSH DATA chain */
+	tdata_psh_info_t *tdata_psh_info_tail;	/* Tail of received TCP PSH DATA chain */
+	uint32 last_used_time;	/* The last time this tcpdata_info was used(in ms) */
+} tcpdata_info_t;
+
+/* TCPACK SUPPRESS module */
+typedef struct {
+	int tcpack_info_cnt;
+	tcpack_info_t tcpack_info_tbl[TCPACK_INFO_MAXNUM];	/* Info of TCP ACK to send */
+	int tcpdata_info_cnt;
+	tcpdata_info_t tcpdata_info_tbl[TCPDATA_INFO_MAXNUM];	/* Info of received TCP DATA */
+	tdata_psh_info_t *tdata_psh_info_pool;	/* Pointer to tdata_psh_info elements pool */
+	tdata_psh_info_t *tdata_psh_info_free;	/* free tdata_psh_info elements chain in pool */
+#ifdef DHDTCPACK_SUP_DBG
+	int psh_info_enq_num;	/* Number of free TCP PSH DATA info elements in pool */
+#endif /* DHDTCPACK_SUP_DBG */
+} tcpack_sup_module_t;
+
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+counter_tbl_t tack_tbl = {"tcpACK", 0, 1000, 10, {0, }, 1};
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+
+static void
+_tdata_psh_info_pool_enq(tcpack_sup_module_t *tcpack_sup_mod,
+	tdata_psh_info_t *tdata_psh_info)
+{
+	if ((tcpack_sup_mod == NULL) || (tdata_psh_info == NULL)) {
+		DHD_ERROR(("%s %d: ERROR %p %p\n", __FUNCTION__, __LINE__,
+			tcpack_sup_mod, tdata_psh_info));
+		return;
+	}
+
+	ASSERT(tdata_psh_info->next == NULL);
+	tdata_psh_info->next = tcpack_sup_mod->tdata_psh_info_free;
+	tcpack_sup_mod->tdata_psh_info_free = tdata_psh_info;
+#ifdef DHDTCPACK_SUP_DBG
+	tcpack_sup_mod->psh_info_enq_num++;
+#endif
+}
+
+static tdata_psh_info_t*
+_tdata_psh_info_pool_deq(tcpack_sup_module_t *tcpack_sup_mod)
+{
+	tdata_psh_info_t *tdata_psh_info = NULL;
+
+	if (tcpack_sup_mod == NULL) {
+		DHD_ERROR(("%s %d: ERROR %p\n", __FUNCTION__, __LINE__,
+			tcpack_sup_mod));
+		return NULL;
+	}
+
+	tdata_psh_info = tcpack_sup_mod->tdata_psh_info_free;
+	if (tdata_psh_info == NULL)
+		DHD_ERROR(("%s %d: Out of tdata_disc_grp\n", __FUNCTION__, __LINE__));
+	else {
+		tcpack_sup_mod->tdata_psh_info_free = tdata_psh_info->next;
+		tdata_psh_info->next = NULL;
+#ifdef DHDTCPACK_SUP_DBG
+		tcpack_sup_mod->psh_info_enq_num--;
+#endif /* DHDTCPACK_SUP_DBG */
+	}
+
+	return tdata_psh_info;
+}
+
+#ifdef BCMSDIO
+static int _tdata_psh_info_pool_init(dhd_pub_t *dhdp,
+	tcpack_sup_module_t *tcpack_sup_mod)
+{
+	tdata_psh_info_t *tdata_psh_info_pool = NULL;
+	uint i;
+
+	DHD_TRACE(("%s %d: Enter\n", __FUNCTION__, __LINE__));
+
+	if (tcpack_sup_mod == NULL)
+		return BCME_ERROR;
+
+	ASSERT(tcpack_sup_mod->tdata_psh_info_pool == NULL);
+	ASSERT(tcpack_sup_mod->tdata_psh_info_free == NULL);
+
+	tdata_psh_info_pool =
+		MALLOC(dhdp->osh, sizeof(tdata_psh_info_t) * TCPDATA_PSH_INFO_MAXNUM);
+
+	if (tdata_psh_info_pool == NULL)
+		return BCME_NOMEM;
+	bzero(tdata_psh_info_pool, sizeof(tdata_psh_info_t) * TCPDATA_PSH_INFO_MAXNUM);
+#ifdef DHDTCPACK_SUP_DBG
+	tcpack_sup_mod->psh_info_enq_num = 0;
+#endif /* DHDTCPACK_SUP_DBG */
+
+	/* Enqueue newly allocated tcpdata psh info elements to the pool */
+	for (i = 0; i < TCPDATA_PSH_INFO_MAXNUM; i++)
+		_tdata_psh_info_pool_enq(tcpack_sup_mod, &tdata_psh_info_pool[i]);
+
+	ASSERT(tcpack_sup_mod->tdata_psh_info_free != NULL);
+	tcpack_sup_mod->tdata_psh_info_pool = tdata_psh_info_pool;
+
+	return BCME_OK;
+}
+
+static void _tdata_psh_info_pool_deinit(dhd_pub_t *dhdp,
+	tcpack_sup_module_t *tcpack_sup_mod)
+{
+	uint i;
+	tdata_psh_info_t *tdata_psh_info;
+
+	DHD_TRACE(("%s %d: Enter\n", __FUNCTION__, __LINE__));
+
+	if (tcpack_sup_mod == NULL) {
+		DHD_ERROR(("%s %d: ERROR tcpack_sup_mod NULL!\n",
+			__FUNCTION__, __LINE__));
+		return;
+	}
+
+	for (i = 0; i < tcpack_sup_mod->tcpdata_info_cnt; i++) {
+		tcpdata_info_t *tcpdata_info = &tcpack_sup_mod->tcpdata_info_tbl[i];
+		/* Return tdata_psh_info elements allocated to each tcpdata_info to the pool */
+		while ((tdata_psh_info = tcpdata_info->tdata_psh_info_head)) {
+			tcpdata_info->tdata_psh_info_head = tdata_psh_info->next;
+			tdata_psh_info->next = NULL;
+			_tdata_psh_info_pool_enq(tcpack_sup_mod, tdata_psh_info);
+		}
+		tcpdata_info->tdata_psh_info_tail = NULL;
+	}
+#ifdef DHDTCPACK_SUP_DBG
+	DHD_ERROR(("%s %d: PSH INFO ENQ %d\n",
+		__FUNCTION__, __LINE__, tcpack_sup_mod->psh_info_enq_num));
+#endif /* DHDTCPACK_SUP_DBG */
+
+	i = 0;
+	/* Be sure we recollected all tdata_psh_info elements */
+	while ((tdata_psh_info = tcpack_sup_mod->tdata_psh_info_free)) {
+		tcpack_sup_mod->tdata_psh_info_free = tdata_psh_info->next;
+		tdata_psh_info->next = NULL;
+		i++;
+	}
+	ASSERT(i == TCPDATA_PSH_INFO_MAXNUM);
+	MFREE(dhdp->osh, tcpack_sup_mod->tdata_psh_info_pool,
+		sizeof(tdata_psh_info_t) * TCPDATA_PSH_INFO_MAXNUM);
+
+	return;
+}
+#endif /* BCMSDIO */
+
+#ifdef BCMPCIE
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+static void dhd_tcpack_send(ulong data)
+#else
+static enum hrtimer_restart dhd_tcpack_send(struct hrtimer *timer)
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+{
+	tcpack_sup_module_t *tcpack_sup_mod;
+	tcpack_info_t *cur_tbl;
+	dhd_pub_t *dhdp;
+	int ifidx;
+	void* pkt;
+	unsigned long flags;
+
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+	cur_tbl = (tcpack_info_t *)data;
+#else
+	cur_tbl = container_of(timer, tcpack_info_t, timer.timer);
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+
+	if (!cur_tbl) {
+		goto done;
+	}
+
+	dhdp = cur_tbl->dhdp;
+	if (!dhdp) {
+		goto done;
+	}
+
+	flags = dhd_os_tcpacklock(dhdp);
+
+	if (unlikely(dhdp->tcpack_sup_mode != TCPACK_SUP_HOLD)) {
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto done;
+	}
+
+	tcpack_sup_mod = dhdp->tcpack_sup_module;
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n",
+			__FUNCTION__, __LINE__));
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto done;
+	}
+	pkt = cur_tbl->pkt_in_q;
+	ifidx = cur_tbl->ifidx;
+	if (!pkt) {
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto done;
+	}
+	cur_tbl->pkt_in_q = NULL;
+	cur_tbl->pkt_ether_hdr = NULL;
+	cur_tbl->ifidx = 0;
+	cur_tbl->supp_cnt = 0;
+	if (--tcpack_sup_mod->tcpack_info_cnt < 0) {
+		DHD_ERROR(("%s %d: ERROR!!! tcp_ack_info_cnt %d\n",
+			__FUNCTION__, __LINE__, tcpack_sup_mod->tcpack_info_cnt));
+	}
+
+	dhd_os_tcpackunlock(dhdp, flags);
+
+	dhd_sendpkt(dhdp, ifidx, pkt);
+
+done:
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+	return;
+#else
+	return HRTIMER_NORESTART;
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+}
+#endif /* BCMPCIE */
+
+int dhd_tcpack_suppress_set(dhd_pub_t *dhdp, uint8 mode)
+{
+	int ret = BCME_OK;
+	unsigned long flags;
+	tcpack_sup_module_t *tcpack_sup_module;
+	uint8 invalid_mode = FALSE;
+	int prev_mode;
+	int i = 0;
+
+	flags = dhd_os_tcpacklock(dhdp);
+	tcpack_sup_module = dhdp->tcpack_sup_module;
+	prev_mode = dhdp->tcpack_sup_mode;
+
+	if (prev_mode == mode) {
+		DHD_ERROR(("%s %d: already set to %d\n", __FUNCTION__, __LINE__, mode));
+		goto exit;
+	}
+
+	invalid_mode |= (mode >= TCPACK_SUP_LAST_MODE);
+#ifdef BCMSDIO
+	invalid_mode |= (mode == TCPACK_SUP_HOLD);
+#endif /* BCMSDIO */
+#ifdef BCMPCIE
+	invalid_mode |= ((mode == TCPACK_SUP_REPLACE) || (mode == TCPACK_SUP_DELAYTX));
+#endif /* BCMPCIE */
+
+	if (invalid_mode) {
+		DHD_ERROR(("%s %d: Invalid TCP ACK Suppress mode %d\n",
+			__FUNCTION__, __LINE__, mode));
+		ret = BCME_BADARG;
+		goto exit;
+	}
+
+	DHD_INFO(("%s: TCP ACK Suppress mode %d -> mode %d\n",
+		__FUNCTION__, dhdp->tcpack_sup_mode, mode));
+	DHD_INFO(("%s: TCPACK_INFO_MAXNUM=%d, TCPDATA_INFO_MAXNUM=%d\n",
+		__FUNCTION__, TCPACK_INFO_MAXNUM, TCPDATA_INFO_MAXNUM));
+
+	/* Pre-process routines to change a new mode as per previous mode */
+	switch (prev_mode) {
+		case TCPACK_SUP_OFF:
+			if (tcpack_sup_module == NULL) {
+				tcpack_sup_module = MALLOC(dhdp->osh, sizeof(tcpack_sup_module_t));
+				if (tcpack_sup_module == NULL) {
+					DHD_ERROR(("%s[%d]: Failed to allocate the new memory for "
+						"tcpack_sup_module\n", __FUNCTION__, __LINE__));
+					dhdp->tcpack_sup_mode = TCPACK_SUP_OFF;
+					ret = BCME_NOMEM;
+					goto exit;
+				}
+				dhdp->tcpack_sup_module = tcpack_sup_module;
+			}
+			bzero(tcpack_sup_module, sizeof(tcpack_sup_module_t));
+			break;
+#ifdef BCMSDIO
+		case TCPACK_SUP_DELAYTX:
+			if (tcpack_sup_module) {
+				/* We won't need tdata_psh_info pool and
+				 * tcpddata_info_tbl anymore
+				 */
+				_tdata_psh_info_pool_deinit(dhdp, tcpack_sup_module);
+				tcpack_sup_module->tcpdata_info_cnt = 0;
+				bzero(tcpack_sup_module->tcpdata_info_tbl,
+					sizeof(tcpdata_info_t) * TCPDATA_INFO_MAXNUM);
+			}
+
+			/* For half duplex bus interface, tx precedes rx by default */
+			if (dhdp->bus) {
+				dhd_bus_set_dotxinrx(dhdp->bus, TRUE);
+			}
+
+			if (tcpack_sup_module == NULL) {
+				DHD_ERROR(("%s[%d]: tcpack_sup_module should not be NULL\n",
+					__FUNCTION__, __LINE__));
+				dhdp->tcpack_sup_mode = TCPACK_SUP_OFF;
+				goto exit;
+			}
+			break;
+#endif /* BCMSDIO */
+	}
+
+	/* Update a new mode */
+	dhdp->tcpack_sup_mode = mode;
+
+	/* Process for a new mode */
+	switch (mode) {
+		case TCPACK_SUP_OFF:
+			ASSERT(tcpack_sup_module != NULL);
+			/* Clean up timer/data structure for
+			 * any remaining/pending packet or timer.
+			 */
+			if (tcpack_sup_module) {
+				/* Check if previous mode is TCAPACK_SUP_HOLD */
+				if (prev_mode == TCPACK_SUP_HOLD) {
+					for (i = 0; i < TCPACK_INFO_MAXNUM; i++) {
+						tcpack_info_t *tcpack_info_tbl =
+							&tcpack_sup_module->tcpack_info_tbl[i];
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+						del_timer(&tcpack_info_tbl->timer);
+#else
+						hrtimer_cancel(&tcpack_info_tbl->timer.timer);
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+						if (tcpack_info_tbl->pkt_in_q) {
+							PKTFREE(dhdp->osh,
+								tcpack_info_tbl->pkt_in_q, TRUE);
+							tcpack_info_tbl->pkt_in_q = NULL;
+						}
+					}
+				}
+				MFREE(dhdp->osh, tcpack_sup_module, sizeof(tcpack_sup_module_t));
+				dhdp->tcpack_sup_module = NULL;
+			} else {
+				DHD_ERROR(("%s[%d]: tcpack_sup_module should not be NULL\n",
+					__FUNCTION__, __LINE__));
+			}
+			break;
+#ifdef BCMSDIO
+		case TCPACK_SUP_REPLACE:
+			/* There is nothing to configure for this mode */
+			break;
+		case TCPACK_SUP_DELAYTX:
+			ret = _tdata_psh_info_pool_init(dhdp, tcpack_sup_module);
+			if (ret != BCME_OK) {
+				DHD_ERROR(("%s %d: pool init fail with %d\n",
+					__FUNCTION__, __LINE__, ret));
+				break;
+			}
+			if (dhdp->bus) {
+				dhd_bus_set_dotxinrx(dhdp->bus, FALSE);
+			}
+			break;
+#endif /* BCMSDIO */
+#ifdef BCMPCIE
+		case TCPACK_SUP_HOLD:
+			dhdp->tcpack_sup_ratio = dhdp->conf->tcpack_sup_ratio;
+			dhdp->tcpack_sup_delay = dhdp->conf->tcpack_sup_delay;
+			for (i = 0; i < TCPACK_INFO_MAXNUM; i++) {
+				tcpack_info_t *tcpack_info_tbl =
+					&tcpack_sup_module->tcpack_info_tbl[i];
+				tcpack_info_tbl->dhdp = dhdp;
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+				init_timer_compat(&tcpack_info_tbl->timer, dhd_tcpack_send,
+					tcpack_info_tbl);
+#else
+				tasklet_hrtimer_init(&tcpack_info_tbl->timer,
+					dhd_tcpack_send, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+			}
+			break;
+#endif /* BCMPCIE */
+	}
+
+exit:
+	dhd_os_tcpackunlock(dhdp, flags);
+	return ret;
+}
+
+void
+dhd_tcpack_info_tbl_clean(dhd_pub_t *dhdp)
+{
+	tcpack_sup_module_t *tcpack_sup_mod = dhdp->tcpack_sup_module;
+	int i;
+	unsigned long flags;
+
+	if (dhdp->tcpack_sup_mode == TCPACK_SUP_OFF)
+		goto exit;
+
+	flags = dhd_os_tcpacklock(dhdp);
+
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n",
+			__FUNCTION__, __LINE__));
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+
+	if (dhdp->tcpack_sup_mode == TCPACK_SUP_HOLD) {
+		for (i = 0; i < TCPACK_INFO_MAXNUM; i++) {
+			if (tcpack_sup_mod->tcpack_info_tbl[i].pkt_in_q) {
+				PKTFREE(dhdp->osh, tcpack_sup_mod->tcpack_info_tbl[i].pkt_in_q,
+					TRUE);
+				tcpack_sup_mod->tcpack_info_tbl[i].pkt_in_q = NULL;
+				tcpack_sup_mod->tcpack_info_tbl[i].pkt_ether_hdr = NULL;
+				tcpack_sup_mod->tcpack_info_tbl[i].ifidx = 0;
+				tcpack_sup_mod->tcpack_info_tbl[i].supp_cnt = 0;
+			}
+		}
+	} else {
+		tcpack_sup_mod->tcpack_info_cnt = 0;
+		bzero(tcpack_sup_mod->tcpack_info_tbl, sizeof(tcpack_info_t) * TCPACK_INFO_MAXNUM);
+	}
+
+	dhd_os_tcpackunlock(dhdp, flags);
+
+	if (dhdp->tcpack_sup_mode == TCPACK_SUP_HOLD) {
+		for (i = 0; i < TCPACK_INFO_MAXNUM; i++) {
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+			del_timer_sync(&tcpack_sup_mod->tcpack_info_tbl[i].timer);
+#else
+			hrtimer_cancel(&tcpack_sup_mod->tcpack_info_tbl[i].timer.timer);
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+		}
+	}
+
+exit:
+	return;
+}
+
+inline int dhd_tcpack_check_xmit(dhd_pub_t *dhdp, void *pkt)
+{
+	uint8 i;
+	tcpack_sup_module_t *tcpack_sup_mod;
+	tcpack_info_t *tcpack_info_tbl;
+	int tbl_cnt;
+	int ret = BCME_OK;
+	void *pdata;
+	uint32 pktlen;
+	unsigned long flags;
+
+	if (dhdp->tcpack_sup_mode == TCPACK_SUP_OFF)
+		goto exit;
+
+	pdata = PKTDATA(dhdp->osh, pkt);
+	pktlen = PKTLEN(dhdp->osh, pkt) - dhd_prot_hdrlen(dhdp, pdata);
+
+	if (pktlen < TCPACKSZMIN || pktlen > TCPACKSZMAX) {
+		DHD_TRACE(("%s %d: Too short or long length %d to be TCP ACK\n",
+			__FUNCTION__, __LINE__, pktlen));
+		goto exit;
+	}
+
+	flags = dhd_os_tcpacklock(dhdp);
+	tcpack_sup_mod = dhdp->tcpack_sup_module;
+
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n", __FUNCTION__, __LINE__));
+		ret = BCME_ERROR;
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+	tbl_cnt = tcpack_sup_mod->tcpack_info_cnt;
+	tcpack_info_tbl = tcpack_sup_mod->tcpack_info_tbl;
+
+	ASSERT(tbl_cnt <= TCPACK_INFO_MAXNUM);
+
+	for (i = 0; i < tbl_cnt; i++) {
+		if (tcpack_info_tbl[i].pkt_in_q == pkt) {
+			DHD_TRACE(("%s %d: pkt %p sent out. idx %d, tbl_cnt %d\n",
+				__FUNCTION__, __LINE__, pkt, i, tbl_cnt));
+			/* This pkt is being transmitted so remove the tcp_ack_info of it. */
+			if (i < tbl_cnt - 1) {
+				bcopy(&tcpack_info_tbl[tbl_cnt - 1],
+					&tcpack_info_tbl[i], sizeof(tcpack_info_t));
+			}
+			bzero(&tcpack_info_tbl[tbl_cnt - 1], sizeof(tcpack_info_t));
+			if (--tcpack_sup_mod->tcpack_info_cnt < 0) {
+				DHD_ERROR(("%s %d: ERROR!!! tcp_ack_info_cnt %d\n",
+					__FUNCTION__, __LINE__, tcpack_sup_mod->tcpack_info_cnt));
+				ret = BCME_ERROR;
+			}
+			break;
+		}
+	}
+	dhd_os_tcpackunlock(dhdp, flags);
+
+exit:
+	return ret;
+}
+
+static INLINE bool dhd_tcpdata_psh_acked(dhd_pub_t *dhdp, uint8 *ip_hdr,
+	uint8 *tcp_hdr, uint32 tcp_ack_num)
+{
+	tcpack_sup_module_t *tcpack_sup_mod;
+	int i;
+	tcpdata_info_t *tcpdata_info = NULL;
+	tdata_psh_info_t *tdata_psh_info = NULL;
+	bool ret = FALSE;
+
+	if (dhdp->tcpack_sup_mode != TCPACK_SUP_DELAYTX)
+		goto exit;
+
+	tcpack_sup_mod = dhdp->tcpack_sup_module;
+
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n", __FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	DHD_TRACE(("%s %d: IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR
+		" TCP port %d %d, ack %u\n", __FUNCTION__, __LINE__,
+		IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_SRC_IP_OFFSET])),
+		IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_DEST_IP_OFFSET])),
+		ntoh16_ua(&tcp_hdr[TCP_SRC_PORT_OFFSET]),
+		ntoh16_ua(&tcp_hdr[TCP_DEST_PORT_OFFSET]),
+		tcp_ack_num));
+
+	for (i = 0; i < tcpack_sup_mod->tcpdata_info_cnt; i++) {
+		tcpdata_info_t *tcpdata_info_tmp = &tcpack_sup_mod->tcpdata_info_tbl[i];
+		DHD_TRACE(("%s %d: data info[%d], IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR
+			" TCP port %d %d\n", __FUNCTION__, __LINE__, i,
+			IPV4_ADDR_TO_STR(ntoh32_ua(tcpdata_info_tmp->ip_addr.src)),
+			IPV4_ADDR_TO_STR(ntoh32_ua(tcpdata_info_tmp->ip_addr.dst)),
+			ntoh16_ua(tcpdata_info_tmp->tcp_port.src),
+			ntoh16_ua(tcpdata_info_tmp->tcp_port.dst)));
+
+		/* If either IP address or TCP port number does not match, skip. */
+		if (memcmp(&ip_hdr[IPV4_SRC_IP_OFFSET],
+			tcpdata_info_tmp->ip_addr.dst, IPV4_ADDR_LEN) == 0 &&
+			memcmp(&ip_hdr[IPV4_DEST_IP_OFFSET],
+			tcpdata_info_tmp->ip_addr.src, IPV4_ADDR_LEN) == 0 &&
+			memcmp(&tcp_hdr[TCP_SRC_PORT_OFFSET],
+			tcpdata_info_tmp->tcp_port.dst, TCP_PORT_LEN) == 0 &&
+			memcmp(&tcp_hdr[TCP_DEST_PORT_OFFSET],
+			tcpdata_info_tmp->tcp_port.src, TCP_PORT_LEN) == 0) {
+			tcpdata_info = tcpdata_info_tmp;
+			break;
+		}
+	}
+
+	if (tcpdata_info == NULL) {
+		DHD_TRACE(("%s %d: no tcpdata_info!\n", __FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	if (tcpdata_info->tdata_psh_info_head == NULL) {
+		DHD_TRACE(("%s %d: No PSH DATA to be acked!\n", __FUNCTION__, __LINE__));
+	}
+
+	while ((tdata_psh_info = tcpdata_info->tdata_psh_info_head)) {
+		if (IS_TCPSEQ_GE(tcp_ack_num, tdata_psh_info->end_seq)) {
+			DHD_TRACE(("%s %d: PSH ACKED! %u >= %u\n",
+				__FUNCTION__, __LINE__, tcp_ack_num, tdata_psh_info->end_seq));
+			tcpdata_info->tdata_psh_info_head = tdata_psh_info->next;
+			tdata_psh_info->next = NULL;
+			_tdata_psh_info_pool_enq(tcpack_sup_mod, tdata_psh_info);
+			ret = TRUE;
+		} else
+			break;
+	}
+	if (tdata_psh_info == NULL)
+		tcpdata_info->tdata_psh_info_tail = NULL;
+
+#ifdef DHDTCPACK_SUP_DBG
+	DHD_TRACE(("%s %d: PSH INFO ENQ %d\n",
+		__FUNCTION__, __LINE__, tcpack_sup_mod->psh_info_enq_num));
+#endif /* DHDTCPACK_SUP_DBG */
+
+exit:
+	return ret;
+}
+
+bool
+dhd_tcpack_suppress(dhd_pub_t *dhdp, void *pkt)
+{
+	uint8 *new_ether_hdr;	/* Ethernet header of the new packet */
+	uint16 new_ether_type;	/* Ethernet type of the new packet */
+	uint8 *new_ip_hdr;		/* IP header of the new packet */
+	uint8 *new_tcp_hdr;		/* TCP header of the new packet */
+	uint32 new_ip_hdr_len;	/* IP header length of the new packet */
+	uint32 cur_framelen;
+	uint32 new_tcp_ack_num;		/* TCP acknowledge number of the new packet */
+	uint16 new_ip_total_len;	/* Total length of IP packet for the new packet */
+	uint32 new_tcp_hdr_len;		/* TCP header length of the new packet */
+	tcpack_sup_module_t *tcpack_sup_mod;
+	tcpack_info_t *tcpack_info_tbl;
+	int i;
+	bool ret = FALSE;
+	bool set_dotxinrx = TRUE;
+	unsigned long flags;
+
+	if (dhdp->tcpack_sup_mode == TCPACK_SUP_OFF)
+		goto exit;
+
+	new_ether_hdr = PKTDATA(dhdp->osh, pkt);
+	cur_framelen = PKTLEN(dhdp->osh, pkt);
+
+	if (cur_framelen < TCPACKSZMIN || cur_framelen > TCPACKSZMAX) {
+		DHD_TRACE(("%s %d: Too short or long length %d to be TCP ACK\n",
+			__FUNCTION__, __LINE__, cur_framelen));
+		goto exit;
+	}
+
+	new_ether_type = new_ether_hdr[12] << 8 | new_ether_hdr[13];
+
+	if (new_ether_type != ETHER_TYPE_IP) {
+		DHD_TRACE(("%s %d: Not a IP packet 0x%x\n",
+			__FUNCTION__, __LINE__, new_ether_type));
+		goto exit;
+	}
+
+	DHD_TRACE(("%s %d: IP pkt! 0x%x\n", __FUNCTION__, __LINE__, new_ether_type));
+
+	new_ip_hdr = new_ether_hdr + ETHER_HDR_LEN;
+	cur_framelen -= ETHER_HDR_LEN;
+
+	ASSERT(cur_framelen >= IPV4_MIN_HEADER_LEN);
+
+	new_ip_hdr_len = IPV4_HLEN(new_ip_hdr);
+	if (IP_VER(new_ip_hdr) != IP_VER_4 || IPV4_PROT(new_ip_hdr) != IP_PROT_TCP) {
+		DHD_TRACE(("%s %d: Not IPv4 nor TCP! ip ver %d, prot %d\n",
+			__FUNCTION__, __LINE__, IP_VER(new_ip_hdr), IPV4_PROT(new_ip_hdr)));
+		goto exit;
+	}
+
+	new_tcp_hdr = new_ip_hdr + new_ip_hdr_len;
+	cur_framelen -= new_ip_hdr_len;
+
+	ASSERT(cur_framelen >= TCP_MIN_HEADER_LEN);
+
+	DHD_TRACE(("%s %d: TCP pkt!\n", __FUNCTION__, __LINE__));
+
+	/* is it an ack ? Allow only ACK flag, not to suppress others. */
+	if (new_tcp_hdr[TCP_FLAGS_OFFSET] != TCP_FLAG_ACK) {
+		DHD_TRACE(("%s %d: Do not touch TCP flag 0x%x\n",
+			__FUNCTION__, __LINE__, new_tcp_hdr[TCP_FLAGS_OFFSET]));
+		goto exit;
+	}
+
+	new_ip_total_len = ntoh16_ua(&new_ip_hdr[IPV4_PKTLEN_OFFSET]);
+	new_tcp_hdr_len = 4 * TCP_HDRLEN(new_tcp_hdr[TCP_HLEN_OFFSET]);
+
+	/* This packet has TCP data, so just send */
+	if (new_ip_total_len > new_ip_hdr_len + new_tcp_hdr_len) {
+		DHD_TRACE(("%s %d: Do nothing for TCP DATA\n", __FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	ASSERT(new_ip_total_len == new_ip_hdr_len + new_tcp_hdr_len);
+
+	new_tcp_ack_num = ntoh32_ua(&new_tcp_hdr[TCP_ACK_NUM_OFFSET]);
+
+	DHD_TRACE(("%s %d: TCP ACK with zero DATA length"
+		" IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR" TCP port %d %d\n",
+		__FUNCTION__, __LINE__,
+		IPV4_ADDR_TO_STR(ntoh32_ua(&new_ip_hdr[IPV4_SRC_IP_OFFSET])),
+		IPV4_ADDR_TO_STR(ntoh32_ua(&new_ip_hdr[IPV4_DEST_IP_OFFSET])),
+		ntoh16_ua(&new_tcp_hdr[TCP_SRC_PORT_OFFSET]),
+		ntoh16_ua(&new_tcp_hdr[TCP_DEST_PORT_OFFSET])));
+
+	/* Look for tcp_ack_info that has the same ip src/dst addrs and tcp src/dst ports */
+	flags = dhd_os_tcpacklock(dhdp);
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+	counter_printlog(&tack_tbl);
+	tack_tbl.cnt[0]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+
+	tcpack_sup_mod = dhdp->tcpack_sup_module;
+	tcpack_info_tbl = tcpack_sup_mod->tcpack_info_tbl;
+
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n", __FUNCTION__, __LINE__));
+		ret = BCME_ERROR;
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+
+	if (dhd_tcpdata_psh_acked(dhdp, new_ip_hdr, new_tcp_hdr, new_tcp_ack_num)) {
+		/* This TCPACK is ACK to TCPDATA PSH pkt, so keep set_dotxinrx TRUE */
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+		tack_tbl.cnt[5]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+	} else
+		set_dotxinrx = FALSE;
+
+	for (i = 0; i < tcpack_sup_mod->tcpack_info_cnt; i++) {
+		void *oldpkt;	/* TCPACK packet that is already in txq or DelayQ */
+		uint8 *old_ether_hdr, *old_ip_hdr, *old_tcp_hdr;
+		uint32 old_ip_hdr_len, old_tcp_hdr_len;
+		uint32 old_tcpack_num;	/* TCP ACK number of old TCPACK packet in Q */
+
+		if ((oldpkt = tcpack_info_tbl[i].pkt_in_q) == NULL) {
+			DHD_ERROR(("%s %d: Unexpected error!! cur idx %d, ttl cnt %d\n",
+				__FUNCTION__, __LINE__, i, tcpack_sup_mod->tcpack_info_cnt));
+			break;
+		}
+
+		if (PKTDATA(dhdp->osh, oldpkt) == NULL) {
+			DHD_ERROR(("%s %d: oldpkt data NULL!! cur idx %d, ttl cnt %d\n",
+				__FUNCTION__, __LINE__, i, tcpack_sup_mod->tcpack_info_cnt));
+			break;
+		}
+
+		old_ether_hdr = tcpack_info_tbl[i].pkt_ether_hdr;
+		old_ip_hdr = old_ether_hdr + ETHER_HDR_LEN;
+		old_ip_hdr_len = IPV4_HLEN(old_ip_hdr);
+		old_tcp_hdr = old_ip_hdr + old_ip_hdr_len;
+		old_tcp_hdr_len = 4 * TCP_HDRLEN(old_tcp_hdr[TCP_HLEN_OFFSET]);
+
+		DHD_TRACE(("%s %d: oldpkt %p[%d], IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR
+			" TCP port %d %d\n", __FUNCTION__, __LINE__, oldpkt, i,
+			IPV4_ADDR_TO_STR(ntoh32_ua(&old_ip_hdr[IPV4_SRC_IP_OFFSET])),
+			IPV4_ADDR_TO_STR(ntoh32_ua(&old_ip_hdr[IPV4_DEST_IP_OFFSET])),
+			ntoh16_ua(&old_tcp_hdr[TCP_SRC_PORT_OFFSET]),
+			ntoh16_ua(&old_tcp_hdr[TCP_DEST_PORT_OFFSET])));
+
+		/* If either of IP address or TCP port number does not match, skip.
+		 * Note that src/dst addr fields in ip header are contiguous being 8 bytes in total.
+		 * Also, src/dst port fields in TCP header are contiguous being 4 bytes in total.
+		 */
+		if (memcmp(&new_ip_hdr[IPV4_SRC_IP_OFFSET],
+			&old_ip_hdr[IPV4_SRC_IP_OFFSET], IPV4_ADDR_LEN * 2) ||
+			memcmp(&new_tcp_hdr[TCP_SRC_PORT_OFFSET],
+			&old_tcp_hdr[TCP_SRC_PORT_OFFSET], TCP_PORT_LEN * 2))
+			continue;
+
+		old_tcpack_num = ntoh32_ua(&old_tcp_hdr[TCP_ACK_NUM_OFFSET]);
+
+		if (IS_TCPSEQ_GT(new_tcp_ack_num, old_tcpack_num)) {
+			/* New packet has higher TCP ACK number, so it replaces the old packet */
+			if (new_ip_hdr_len == old_ip_hdr_len &&
+				new_tcp_hdr_len == old_tcp_hdr_len) {
+				ASSERT(memcmp(new_ether_hdr, old_ether_hdr, ETHER_HDR_LEN) == 0);
+				bcopy(new_ip_hdr, old_ip_hdr, new_ip_total_len);
+				PKTFREE(dhdp->osh, pkt, FALSE);
+				DHD_TRACE(("%s %d: TCP ACK replace %u -> %u\n",
+					__FUNCTION__, __LINE__, old_tcpack_num, new_tcp_ack_num));
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+				tack_tbl.cnt[2]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+				ret = TRUE;
+			} else {
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+				tack_tbl.cnt[6]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+				DHD_TRACE(("%s %d: lenth mismatch %d != %d || %d != %d"
+					" ACK %u -> %u\n", __FUNCTION__, __LINE__,
+					new_ip_hdr_len, old_ip_hdr_len,
+					new_tcp_hdr_len, old_tcp_hdr_len,
+					old_tcpack_num, new_tcp_ack_num));
+			}
+		} else if (new_tcp_ack_num == old_tcpack_num) {
+			set_dotxinrx = TRUE;
+			/* TCPACK retransmission */
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+			tack_tbl.cnt[3]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+		} else {
+			DHD_TRACE(("%s %d: ACK number reverse old %u(0x%p) new %u(0x%p)\n",
+				__FUNCTION__, __LINE__, old_tcpack_num, oldpkt,
+				new_tcp_ack_num, pkt));
+		}
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+
+	if (i == tcpack_sup_mod->tcpack_info_cnt && i < TCPACK_INFO_MAXNUM) {
+		/* No TCPACK packet with the same IP addr and TCP port is found
+		 * in tcp_ack_info_tbl. So add this packet to the table.
+		 */
+		DHD_TRACE(("%s %d: Add pkt 0x%p(ether_hdr 0x%p) to tbl[%d]\n",
+			__FUNCTION__, __LINE__, pkt, new_ether_hdr,
+			tcpack_sup_mod->tcpack_info_cnt));
+
+		tcpack_info_tbl[tcpack_sup_mod->tcpack_info_cnt].pkt_in_q = pkt;
+		tcpack_info_tbl[tcpack_sup_mod->tcpack_info_cnt].pkt_ether_hdr = new_ether_hdr;
+		tcpack_sup_mod->tcpack_info_cnt++;
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+		tack_tbl.cnt[1]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+	} else {
+		ASSERT(i == tcpack_sup_mod->tcpack_info_cnt);
+		DHD_TRACE(("%s %d: No empty tcp ack info tbl\n",
+			__FUNCTION__, __LINE__));
+	}
+	dhd_os_tcpackunlock(dhdp, flags);
+
+exit:
+	/* Unless TCPACK_SUP_DELAYTX, dotxinrx is alwasy TRUE, so no need to set here */
+	if (dhdp->tcpack_sup_mode == TCPACK_SUP_DELAYTX && set_dotxinrx)
+		dhd_bus_set_dotxinrx(dhdp->bus, TRUE);
+
+	return ret;
+}
+
+bool
+dhd_tcpdata_info_get(dhd_pub_t *dhdp, void *pkt)
+{
+	uint8 *ether_hdr;	/* Ethernet header of the new packet */
+	uint16 ether_type;	/* Ethernet type of the new packet */
+	uint8 *ip_hdr;		/* IP header of the new packet */
+	uint8 *tcp_hdr;		/* TCP header of the new packet */
+	uint32 ip_hdr_len;	/* IP header length of the new packet */
+	uint32 cur_framelen;
+	uint16 ip_total_len;	/* Total length of IP packet for the new packet */
+	uint32 tcp_hdr_len;		/* TCP header length of the new packet */
+	uint32 tcp_seq_num;		/* TCP sequence number of the new packet */
+	uint16 tcp_data_len;	/* TCP DATA length that excludes IP and TCP headers */
+	uint32 end_tcp_seq_num;	/* TCP seq number of the last byte in the new packet */
+	tcpack_sup_module_t *tcpack_sup_mod;
+	tcpdata_info_t *tcpdata_info = NULL;
+	tdata_psh_info_t *tdata_psh_info;
+
+	int i;
+	bool ret = FALSE;
+	unsigned long flags;
+
+	if (dhdp->tcpack_sup_mode != TCPACK_SUP_DELAYTX)
+		goto exit;
+
+	ether_hdr = PKTDATA(dhdp->osh, pkt);
+	cur_framelen = PKTLEN(dhdp->osh, pkt);
+
+	ether_type = ether_hdr[12] << 8 | ether_hdr[13];
+
+	if (ether_type != ETHER_TYPE_IP) {
+		DHD_TRACE(("%s %d: Not a IP packet 0x%x\n",
+			__FUNCTION__, __LINE__, ether_type));
+		goto exit;
+	}
+
+	DHD_TRACE(("%s %d: IP pkt! 0x%x\n", __FUNCTION__, __LINE__, ether_type));
+
+	ip_hdr = ether_hdr + ETHER_HDR_LEN;
+	cur_framelen -= ETHER_HDR_LEN;
+
+	if (cur_framelen < IPV4_MIN_HEADER_LEN) {
+		DHD_ERROR(("%s %d: cur_framelen(%d) < IPV4_MIN_HEADER_LEN\n", __FUNCTION__,
+			__LINE__, cur_framelen));
+		goto exit;
+	}
+
+	ip_hdr_len = IPV4_HLEN(ip_hdr);
+	if (IP_VER(ip_hdr) != IP_VER_4 || IPV4_PROT(ip_hdr) != IP_PROT_TCP) {
+		DHD_TRACE(("%s %d: Not IPv4 nor TCP! ip ver %d, prot %d\n",
+			__FUNCTION__, __LINE__, IP_VER(ip_hdr), IPV4_PROT(ip_hdr)));
+		goto exit;
+	}
+
+	tcp_hdr = ip_hdr + ip_hdr_len;
+	cur_framelen -= ip_hdr_len;
+
+	ASSERT(cur_framelen >= TCP_MIN_HEADER_LEN);
+	if (cur_framelen < TCP_MIN_HEADER_LEN) {
+		DHD_TRACE(("%s %d: cur_framelen is less than TCP_MIN_HEADER_LEN\n",
+			__FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	DHD_TRACE(("%s %d: TCP pkt!\n", __FUNCTION__, __LINE__));
+
+	ip_total_len = ntoh16_ua(&ip_hdr[IPV4_PKTLEN_OFFSET]);
+	tcp_hdr_len = 4 * TCP_HDRLEN(tcp_hdr[TCP_HLEN_OFFSET]);
+
+	/* This packet is mere TCP ACK, so do nothing */
+	if (ip_total_len == ip_hdr_len + tcp_hdr_len) {
+		DHD_TRACE(("%s %d: Do nothing for no data TCP ACK\n", __FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	ASSERT(ip_total_len > ip_hdr_len + tcp_hdr_len);
+
+	if ((tcp_hdr[TCP_FLAGS_OFFSET] & TCP_FLAG_PSH) == 0) {
+		DHD_TRACE(("%s %d: Not interested TCP DATA packet\n", __FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	DHD_TRACE(("%s %d: TCP DATA with nonzero DATA length"
+		" IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR" TCP port %d %d, flag 0x%x\n",
+		__FUNCTION__, __LINE__,
+		IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_SRC_IP_OFFSET])),
+		IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_DEST_IP_OFFSET])),
+		ntoh16_ua(&tcp_hdr[TCP_SRC_PORT_OFFSET]),
+		ntoh16_ua(&tcp_hdr[TCP_DEST_PORT_OFFSET]),
+		tcp_hdr[TCP_FLAGS_OFFSET]));
+
+	flags = dhd_os_tcpacklock(dhdp);
+	tcpack_sup_mod = dhdp->tcpack_sup_module;
+
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n", __FUNCTION__, __LINE__));
+		ret = BCME_ERROR;
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+
+	/* Look for tcpdata_info that has the same ip src/dst addrs and tcp src/dst ports */
+	i = 0;
+	while (i < tcpack_sup_mod->tcpdata_info_cnt) {
+		tcpdata_info_t *tdata_info_tmp = &tcpack_sup_mod->tcpdata_info_tbl[i];
+		uint32 now_in_ms = OSL_SYSUPTIME();
+		DHD_TRACE(("%s %d: data info[%d], IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR
+			" TCP port %d %d\n", __FUNCTION__, __LINE__, i,
+			IPV4_ADDR_TO_STR(ntoh32_ua(tdata_info_tmp->ip_addr.src)),
+			IPV4_ADDR_TO_STR(ntoh32_ua(tdata_info_tmp->ip_addr.dst)),
+			ntoh16_ua(tdata_info_tmp->tcp_port.src),
+			ntoh16_ua(tdata_info_tmp->tcp_port.dst)));
+
+		/* If both IP address and TCP port number match, we found it so break.
+		 * Note that src/dst addr fields in ip header are contiguous being 8 bytes in total.
+		 * Also, src/dst port fields in TCP header are contiguous being 4 bytes in total.
+		 */
+		if (memcmp(&ip_hdr[IPV4_SRC_IP_OFFSET],
+			(void *)&tdata_info_tmp->ip_addr, IPV4_ADDR_LEN * 2) == 0 &&
+			memcmp(&tcp_hdr[TCP_SRC_PORT_OFFSET],
+			(void *)&tdata_info_tmp->tcp_port, TCP_PORT_LEN * 2) == 0) {
+			tcpdata_info = tdata_info_tmp;
+			tcpdata_info->last_used_time = now_in_ms;
+			break;
+		}
+
+		if (now_in_ms - tdata_info_tmp->last_used_time > TCPDATA_INFO_TIMEOUT) {
+			tdata_psh_info_t *tdata_psh_info_tmp;
+			tcpdata_info_t *last_tdata_info;
+
+			while ((tdata_psh_info_tmp = tdata_info_tmp->tdata_psh_info_head)) {
+				tdata_info_tmp->tdata_psh_info_head = tdata_psh_info_tmp->next;
+				tdata_psh_info_tmp->next = NULL;
+				DHD_TRACE(("%s %d: Clean tdata_psh_info(end_seq %u)!\n",
+					__FUNCTION__, __LINE__, tdata_psh_info_tmp->end_seq));
+				_tdata_psh_info_pool_enq(tcpack_sup_mod, tdata_psh_info_tmp);
+			}
+#ifdef DHDTCPACK_SUP_DBG
+			DHD_ERROR(("%s %d: PSH INFO ENQ %d\n",
+				__FUNCTION__, __LINE__, tcpack_sup_mod->psh_info_enq_num));
+#endif /* DHDTCPACK_SUP_DBG */
+			tcpack_sup_mod->tcpdata_info_cnt--;
+			ASSERT(tcpack_sup_mod->tcpdata_info_cnt >= 0);
+
+			last_tdata_info =
+				&tcpack_sup_mod->tcpdata_info_tbl[tcpack_sup_mod->tcpdata_info_cnt];
+			if (i < tcpack_sup_mod->tcpdata_info_cnt) {
+				ASSERT(last_tdata_info != tdata_info_tmp);
+				bcopy(last_tdata_info, tdata_info_tmp, sizeof(tcpdata_info_t));
+			}
+			bzero(last_tdata_info, sizeof(tcpdata_info_t));
+			DHD_INFO(("%s %d: tcpdata_info(idx %d) is aged out. ttl cnt is now %d\n",
+				__FUNCTION__, __LINE__, i, tcpack_sup_mod->tcpdata_info_cnt));
+			/* Don't increase "i" here, so that the prev last tcpdata_info is checked */
+		} else
+			 i++;
+	}
+
+	tcp_seq_num = ntoh32_ua(&tcp_hdr[TCP_SEQ_NUM_OFFSET]);
+	tcp_data_len = ip_total_len - ip_hdr_len - tcp_hdr_len;
+	end_tcp_seq_num = tcp_seq_num + tcp_data_len;
+
+	if (tcpdata_info == NULL) {
+		ASSERT(i == tcpack_sup_mod->tcpdata_info_cnt);
+		if (i >= TCPDATA_INFO_MAXNUM) {
+			DHD_TRACE(("%s %d: tcp_data_info_tbl FULL! %d %d"
+				" IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR" TCP port %d %d\n",
+				__FUNCTION__, __LINE__, i, tcpack_sup_mod->tcpdata_info_cnt,
+				IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_SRC_IP_OFFSET])),
+				IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_DEST_IP_OFFSET])),
+				ntoh16_ua(&tcp_hdr[TCP_SRC_PORT_OFFSET]),
+				ntoh16_ua(&tcp_hdr[TCP_DEST_PORT_OFFSET])));
+			dhd_os_tcpackunlock(dhdp, flags);
+			goto exit;
+		}
+		tcpdata_info = &tcpack_sup_mod->tcpdata_info_tbl[i];
+
+		/* No TCP flow with the same IP addr and TCP port is found
+		 * in tcp_data_info_tbl. So add this flow to the table.
+		 */
+		DHD_INFO(("%s %d: Add data info to tbl[%d]: IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR
+			" TCP port %d %d\n",
+			__FUNCTION__, __LINE__, tcpack_sup_mod->tcpdata_info_cnt,
+			IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_SRC_IP_OFFSET])),
+			IPV4_ADDR_TO_STR(ntoh32_ua(&ip_hdr[IPV4_DEST_IP_OFFSET])),
+			ntoh16_ua(&tcp_hdr[TCP_SRC_PORT_OFFSET]),
+			ntoh16_ua(&tcp_hdr[TCP_DEST_PORT_OFFSET])));
+		/* Note that src/dst addr fields in ip header are contiguous being 8 bytes in total.
+		 * Also, src/dst port fields in TCP header are contiguous being 4 bytes in total.
+		 */
+		bcopy(&ip_hdr[IPV4_SRC_IP_OFFSET], (void *)&tcpdata_info->ip_addr,
+			IPV4_ADDR_LEN * 2);
+		bcopy(&tcp_hdr[TCP_SRC_PORT_OFFSET], (void *)&tcpdata_info->tcp_port,
+			TCP_PORT_LEN * 2);
+
+		tcpdata_info->last_used_time = OSL_SYSUPTIME();
+		tcpack_sup_mod->tcpdata_info_cnt++;
+	}
+
+	ASSERT(tcpdata_info != NULL);
+
+	tdata_psh_info = _tdata_psh_info_pool_deq(tcpack_sup_mod);
+#ifdef DHDTCPACK_SUP_DBG
+	DHD_TRACE(("%s %d: PSH INFO ENQ %d\n",
+		__FUNCTION__, __LINE__, tcpack_sup_mod->psh_info_enq_num));
+#endif /* DHDTCPACK_SUP_DBG */
+
+	if (tdata_psh_info == NULL) {
+		DHD_ERROR(("%s %d: No more free tdata_psh_info!!\n", __FUNCTION__, __LINE__));
+		ret = BCME_ERROR;
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+	tdata_psh_info->end_seq = end_tcp_seq_num;
+
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+	tack_tbl.cnt[4]++;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+
+	DHD_TRACE(("%s %d: TCP PSH DATA recvd! end seq %u\n",
+		__FUNCTION__, __LINE__, tdata_psh_info->end_seq));
+
+	ASSERT(tdata_psh_info->next == NULL);
+
+	if (tcpdata_info->tdata_psh_info_head == NULL)
+		tcpdata_info->tdata_psh_info_head = tdata_psh_info;
+	else {
+		ASSERT(tcpdata_info->tdata_psh_info_tail);
+		tcpdata_info->tdata_psh_info_tail->next = tdata_psh_info;
+	}
+	tcpdata_info->tdata_psh_info_tail = tdata_psh_info;
+
+	dhd_os_tcpackunlock(dhdp, flags);
+
+exit:
+	return ret;
+}
+
+bool
+dhd_tcpack_hold(dhd_pub_t *dhdp, void *pkt, int ifidx)
+{
+	uint8 *new_ether_hdr;	/* Ethernet header of the new packet */
+	uint16 new_ether_type;	/* Ethernet type of the new packet */
+	uint8 *new_ip_hdr;		/* IP header of the new packet */
+	uint8 *new_tcp_hdr;		/* TCP header of the new packet */
+	uint32 new_ip_hdr_len;	/* IP header length of the new packet */
+	uint32 cur_framelen;
+	uint32 new_tcp_ack_num;		/* TCP acknowledge number of the new packet */
+	uint16 new_ip_total_len;	/* Total length of IP packet for the new packet */
+	uint32 new_tcp_hdr_len;		/* TCP header length of the new packet */
+	tcpack_sup_module_t *tcpack_sup_mod;
+	tcpack_info_t *tcpack_info_tbl;
+	int i, free_slot = TCPACK_INFO_MAXNUM;
+	bool hold = FALSE;
+	unsigned long flags;
+
+	if (dhdp->tcpack_sup_mode != TCPACK_SUP_HOLD) {
+		goto exit;
+	}
+
+	if (dhdp->tcpack_sup_ratio == 1) {
+		goto exit;
+	}
+
+	new_ether_hdr = PKTDATA(dhdp->osh, pkt);
+	cur_framelen = PKTLEN(dhdp->osh, pkt);
+
+	if (cur_framelen < TCPACKSZMIN || cur_framelen > TCPACKSZMAX) {
+		DHD_TRACE(("%s %d: Too short or long length %d to be TCP ACK\n",
+			__FUNCTION__, __LINE__, cur_framelen));
+		goto exit;
+	}
+
+	new_ether_type = new_ether_hdr[12] << 8 | new_ether_hdr[13];
+
+	if (new_ether_type != ETHER_TYPE_IP) {
+		DHD_TRACE(("%s %d: Not a IP packet 0x%x\n",
+			__FUNCTION__, __LINE__, new_ether_type));
+		goto exit;
+	}
+
+	DHD_TRACE(("%s %d: IP pkt! 0x%x\n", __FUNCTION__, __LINE__, new_ether_type));
+
+	new_ip_hdr = new_ether_hdr + ETHER_HDR_LEN;
+	cur_framelen -= ETHER_HDR_LEN;
+
+	ASSERT(cur_framelen >= IPV4_MIN_HEADER_LEN);
+
+	new_ip_hdr_len = IPV4_HLEN(new_ip_hdr);
+	if (IP_VER(new_ip_hdr) != IP_VER_4 || IPV4_PROT(new_ip_hdr) != IP_PROT_TCP) {
+		DHD_TRACE(("%s %d: Not IPv4 nor TCP! ip ver %d, prot %d\n",
+			__FUNCTION__, __LINE__, IP_VER(new_ip_hdr), IPV4_PROT(new_ip_hdr)));
+		goto exit;
+	}
+
+	new_tcp_hdr = new_ip_hdr + new_ip_hdr_len;
+	cur_framelen -= new_ip_hdr_len;
+
+	ASSERT(cur_framelen >= TCP_MIN_HEADER_LEN);
+
+	DHD_TRACE(("%s %d: TCP pkt!\n", __FUNCTION__, __LINE__));
+
+	/* is it an ack ? Allow only ACK flag, not to suppress others. */
+	if (new_tcp_hdr[TCP_FLAGS_OFFSET] != TCP_FLAG_ACK) {
+		DHD_TRACE(("%s %d: Do not touch TCP flag 0x%x\n",
+			__FUNCTION__, __LINE__, new_tcp_hdr[TCP_FLAGS_OFFSET]));
+		goto exit;
+	}
+
+	new_ip_total_len = ntoh16_ua(&new_ip_hdr[IPV4_PKTLEN_OFFSET]);
+	new_tcp_hdr_len = 4 * TCP_HDRLEN(new_tcp_hdr[TCP_HLEN_OFFSET]);
+
+	/* This packet has TCP data, so just send */
+	if (new_ip_total_len > new_ip_hdr_len + new_tcp_hdr_len) {
+		DHD_TRACE(("%s %d: Do nothing for TCP DATA\n", __FUNCTION__, __LINE__));
+		goto exit;
+	}
+
+	ASSERT(new_ip_total_len == new_ip_hdr_len + new_tcp_hdr_len);
+
+	new_tcp_ack_num = ntoh32_ua(&new_tcp_hdr[TCP_ACK_NUM_OFFSET]);
+
+	DHD_TRACE(("%s %d: TCP ACK with zero DATA length"
+		" IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR" TCP port %d %d\n",
+		__FUNCTION__, __LINE__,
+		IPV4_ADDR_TO_STR(ntoh32_ua(&new_ip_hdr[IPV4_SRC_IP_OFFSET])),
+		IPV4_ADDR_TO_STR(ntoh32_ua(&new_ip_hdr[IPV4_DEST_IP_OFFSET])),
+		ntoh16_ua(&new_tcp_hdr[TCP_SRC_PORT_OFFSET]),
+		ntoh16_ua(&new_tcp_hdr[TCP_DEST_PORT_OFFSET])));
+
+	/* Look for tcp_ack_info that has the same ip src/dst addrs and tcp src/dst ports */
+	flags = dhd_os_tcpacklock(dhdp);
+
+	tcpack_sup_mod = dhdp->tcpack_sup_module;
+	tcpack_info_tbl = tcpack_sup_mod->tcpack_info_tbl;
+
+	if (!tcpack_sup_mod) {
+		DHD_ERROR(("%s %d: tcpack suppress module NULL!!\n", __FUNCTION__, __LINE__));
+		dhd_os_tcpackunlock(dhdp, flags);
+		goto exit;
+	}
+
+	hold = TRUE;
+
+	for (i = 0; i < TCPACK_INFO_MAXNUM; i++) {
+		void *oldpkt;	/* TCPACK packet that is already in txq or DelayQ */
+		uint8 *old_ether_hdr, *old_ip_hdr, *old_tcp_hdr;
+		uint32 old_ip_hdr_len;
+		uint32 old_tcpack_num;	/* TCP ACK number of old TCPACK packet in Q */
+
+		if ((oldpkt = tcpack_info_tbl[i].pkt_in_q) == NULL) {
+			if (free_slot == TCPACK_INFO_MAXNUM) {
+				free_slot = i;
+			}
+			continue;
+		}
+
+		if (PKTDATA(dhdp->osh, oldpkt) == NULL) {
+			DHD_ERROR(("%s %d: oldpkt data NULL!! cur idx %d\n",
+				__FUNCTION__, __LINE__, i));
+			hold = FALSE;
+			dhd_os_tcpackunlock(dhdp, flags);
+			goto exit;
+		}
+
+		old_ether_hdr = tcpack_info_tbl[i].pkt_ether_hdr;
+		old_ip_hdr = old_ether_hdr + ETHER_HDR_LEN;
+		old_ip_hdr_len = IPV4_HLEN(old_ip_hdr);
+		old_tcp_hdr = old_ip_hdr + old_ip_hdr_len;
+
+		DHD_TRACE(("%s %d: oldpkt %p[%d], IP addr "IPV4_ADDR_STR" "IPV4_ADDR_STR
+			" TCP port %d %d\n", __FUNCTION__, __LINE__, oldpkt, i,
+			IPV4_ADDR_TO_STR(ntoh32_ua(&old_ip_hdr[IPV4_SRC_IP_OFFSET])),
+			IPV4_ADDR_TO_STR(ntoh32_ua(&old_ip_hdr[IPV4_DEST_IP_OFFSET])),
+			ntoh16_ua(&old_tcp_hdr[TCP_SRC_PORT_OFFSET]),
+			ntoh16_ua(&old_tcp_hdr[TCP_DEST_PORT_OFFSET])));
+
+		/* If either of IP address or TCP port number does not match, skip. */
+		if (memcmp(&new_ip_hdr[IPV4_SRC_IP_OFFSET],
+			&old_ip_hdr[IPV4_SRC_IP_OFFSET], IPV4_ADDR_LEN * 2) ||
+			memcmp(&new_tcp_hdr[TCP_SRC_PORT_OFFSET],
+			&old_tcp_hdr[TCP_SRC_PORT_OFFSET], TCP_PORT_LEN * 2)) {
+			continue;
+		}
+
+		old_tcpack_num = ntoh32_ua(&old_tcp_hdr[TCP_ACK_NUM_OFFSET]);
+
+		if (IS_TCPSEQ_GE(new_tcp_ack_num, old_tcpack_num)) {
+			tcpack_info_tbl[i].supp_cnt++;
+			if (tcpack_info_tbl[i].supp_cnt >= dhdp->tcpack_sup_ratio) {
+				tcpack_info_tbl[i].pkt_in_q = NULL;
+				tcpack_info_tbl[i].pkt_ether_hdr = NULL;
+				tcpack_info_tbl[i].ifidx = 0;
+				tcpack_info_tbl[i].supp_cnt = 0;
+				hold = FALSE;
+			} else {
+				tcpack_info_tbl[i].pkt_in_q = pkt;
+				tcpack_info_tbl[i].pkt_ether_hdr = new_ether_hdr;
+				tcpack_info_tbl[i].ifidx = ifidx;
+			}
+			PKTFREE(dhdp->osh, oldpkt, TRUE);
+		} else {
+			PKTFREE(dhdp->osh, pkt, TRUE);
+		}
+		dhd_os_tcpackunlock(dhdp, flags);
+
+		if (!hold) {
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+			del_timer_sync(&tcpack_info_tbl[i].timer);
+#else
+			hrtimer_cancel(&tcpack_sup_mod->tcpack_info_tbl[i].timer.timer);
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+		}
+		goto exit;
+	}
+
+	if (free_slot < TCPACK_INFO_MAXNUM) {
+		/* No TCPACK packet with the same IP addr and TCP port is found
+		 * in tcp_ack_info_tbl. So add this packet to the table.
+		 */
+		DHD_TRACE(("%s %d: Add pkt 0x%p(ether_hdr 0x%p) to tbl[%d]\n",
+			__FUNCTION__, __LINE__, pkt, new_ether_hdr,
+			free_slot));
+
+		tcpack_info_tbl[free_slot].pkt_in_q = pkt;
+		tcpack_info_tbl[free_slot].pkt_ether_hdr = new_ether_hdr;
+		tcpack_info_tbl[free_slot].ifidx = ifidx;
+		tcpack_info_tbl[free_slot].supp_cnt = 1;
+#ifndef TCPACK_SUPPRESS_HOLD_HRT
+		mod_timer(&tcpack_sup_mod->tcpack_info_tbl[free_slot].timer,
+			jiffies + msecs_to_jiffies(dhdp->tcpack_sup_delay));
+#else
+		tasklet_hrtimer_start(&tcpack_sup_mod->tcpack_info_tbl[free_slot].timer,
+			ktime_set(0, dhdp->tcpack_sup_delay*1000000),
+			HRTIMER_MODE_REL);
+#endif /* TCPACK_SUPPRESS_HOLD_HRT */
+		tcpack_sup_mod->tcpack_info_cnt++;
+	} else {
+		DHD_TRACE(("%s %d: No empty tcp ack info tbl\n",
+			__FUNCTION__, __LINE__));
+	}
+	dhd_os_tcpackunlock(dhdp, flags);
+
+exit:
+	return hold;
+}
+#endif /* DHDTCPACK_SUPPRESS */
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+tcp_hdr_flag_t
+dhd_tcpdata_get_flag(dhd_pub_t *dhdp, void *pkt)
+{
+	uint8 *ether_hdr;	/* Ethernet header of the new packet */
+	uint16 ether_type;	/* Ethernet type of the new packet */
+	uint8 *ip_hdr;		/* IP header of the new packet */
+	uint8 *tcp_hdr;		/* TCP header of the new packet */
+	uint32 ip_hdr_len;	/* IP header length of the new packet */
+	uint32 cur_framelen;
+	uint8 flags;
+
+	ether_hdr = PKTDATA(dhdp->osh, pkt);
+	cur_framelen = PKTLEN(dhdp->osh, pkt);
+
+	ether_type = ether_hdr[12] << 8 | ether_hdr[13];
+
+	if (ether_type != ETHER_TYPE_IP) {
+		DHD_TRACE(("%s %d: Not a IP packet 0x%x\n",
+			__FUNCTION__, __LINE__, ether_type));
+		return FLAG_OTHERS;
+	}
+
+	ip_hdr = ether_hdr + ETHER_HDR_LEN;
+	cur_framelen -= ETHER_HDR_LEN;
+
+	if (cur_framelen < IPV4_MIN_HEADER_LEN) {
+		return FLAG_OTHERS;
+	}
+
+	ip_hdr_len = IPV4_HLEN(ip_hdr);
+	if (IP_VER(ip_hdr) != IP_VER_4 || IPV4_PROT(ip_hdr) != IP_PROT_TCP) {
+		DHD_TRACE(("%s %d: Not IPv4 nor TCP! ip ver %d, prot %d\n",
+			__FUNCTION__, __LINE__, IP_VER(ip_hdr), IPV4_PROT(ip_hdr)));
+		return FLAG_OTHERS;
+	}
+
+	tcp_hdr = ip_hdr + ip_hdr_len;
+
+	flags = (uint8)tcp_hdr[TCP_FLAGS_OFFSET];
+
+	if (flags & TCP_FLAG_SYN) {
+		if (flags & TCP_FLAG_ACK) {
+			return FLAG_SYNCACK;
+		}
+		return FLAG_SYNC;
+	}
+	return FLAG_OTHERS;
+}
+#endif /* DHDTCPSYNC_FLOOD_BLK */
diff --git a/drivers/net/wireless/ap12275/dhd_ip.h b/drivers/net/wireless/ap12275/dhd_ip.h
new file mode 100644
index 000000000000..efbf2d3f62a6
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_ip.h
@@ -0,0 +1,96 @@
+/*
+ * Header file describing the common ip parser function.
+ *
+ * Provides type definitions and function prototypes used to parse ip packet.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef _dhd_ip_h_
+#define _dhd_ip_h_
+
+#if defined(DHDTCPACK_SUPPRESS) || defined(DHDTCPSYNC_FLOOD_BLK)
+#include <dngl_stats.h>
+#include <bcmutils.h>
+#include <dhd.h>
+#endif /* DHDTCPACK_SUPPRESS || DHDTCPSYNC_FLOOD_BLK */
+
+typedef enum pkt_frag
+{
+	DHD_PKT_FRAG_NONE = 0,
+	DHD_PKT_FRAG_FIRST,
+	DHD_PKT_FRAG_CONT,
+	DHD_PKT_FRAG_LAST
+} pkt_frag_t;
+
+extern pkt_frag_t pkt_frag_info(osl_t *osh, void *p);
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+typedef enum tcp_hdr_flags {
+	FLAG_SYNC,
+	FLAG_SYNCACK,
+	FLAG_RST,
+	FLAG_OTHERS
+} tcp_hdr_flag_t;
+
+extern tcp_hdr_flag_t dhd_tcpdata_get_flag(dhd_pub_t *dhdp, void *pkt);
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+#ifdef DHDTCPACK_SUPPRESS
+#define	TCPACKSZMIN	(ETHER_HDR_LEN + IPV4_MIN_HEADER_LEN + TCP_MIN_HEADER_LEN)
+/* Size of MAX possible TCP ACK packet. Extra bytes for IP/TCP option fields */
+#define	TCPACKSZMAX	(TCPACKSZMIN + 100)
+
+/* Max number of TCP streams that have own src/dst IP addrs and TCP ports */
+#ifndef TCPACK_INFO_MAXNUM
+#define TCPACK_INFO_MAXNUM 40
+#endif
+#ifndef TCPDATA_INFO_MAXNUM
+#define TCPDATA_INFO_MAXNUM 40
+#endif
+#define TCPDATA_PSH_INFO_MAXNUM (8 * TCPDATA_INFO_MAXNUM)
+
+#define TCPDATA_INFO_TIMEOUT 5000	/* Remove tcpdata_info if inactive for this time (in ms) */
+
+#define DEFAULT_TCPACK_SUPP_RATIO 3
+#ifndef CUSTOM_TCPACK_SUPP_RATIO
+#define CUSTOM_TCPACK_SUPP_RATIO DEFAULT_TCPACK_SUPP_RATIO
+#endif /* CUSTOM_TCPACK_SUPP_RATIO */
+
+#define DEFAULT_TCPACK_DELAY_TIME 10 /* ms */
+#ifndef CUSTOM_TCPACK_DELAY_TIME
+#define CUSTOM_TCPACK_DELAY_TIME DEFAULT_TCPACK_DELAY_TIME
+#endif /* CUSTOM_TCPACK_DELAY_TIME */
+
+extern int dhd_tcpack_suppress_set(dhd_pub_t *dhdp, uint8 on);
+extern void dhd_tcpack_info_tbl_clean(dhd_pub_t *dhdp);
+extern int dhd_tcpack_check_xmit(dhd_pub_t *dhdp, void *pkt);
+extern bool dhd_tcpack_suppress(dhd_pub_t *dhdp, void *pkt);
+extern bool dhd_tcpdata_info_get(dhd_pub_t *dhdp, void *pkt);
+extern bool dhd_tcpack_hold(dhd_pub_t *dhdp, void *pkt, int ifidx);
+/* #define DHDTCPACK_SUP_DBG */
+#if defined(DEBUG_COUNTER) && defined(DHDTCPACK_SUP_DBG)
+extern counter_tbl_t tack_tbl;
+#endif /* DEBUG_COUNTER && DHDTCPACK_SUP_DBG */
+#endif /* DHDTCPACK_SUPPRESS */
+
+#endif /* _dhd_ip_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_linux.c b/drivers/net/wireless/ap12275/dhd_linux.c
new file mode 100644
index 000000000000..37c3bd9c6b2b
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux.c
@@ -0,0 +1,24268 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Linux-specific network interface.
+ * Basically selected code segments from usb-cdc.c and usb-rndis.c
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <linux/ctype.h>
+#include <typedefs.h>
+#include <linuxver.h>
+#include <osl.h>
+#ifdef SHOW_LOGTRACE
+#include <linux/syscalls.h>
+#include <event_log.h>
+#endif /* SHOW_LOGTRACE */
+
+#if defined(PCIE_FULL_DONGLE) || defined(SHOW_LOGTRACE)
+#include <bcmmsgbuf.h>
+#endif /* PCIE_FULL_DONGLE */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/rtnetlink.h>
+#include <linux/etherdevice.h>
+#include <linux/random.h>
+#include <linux/spinlock.h>
+#include <linux/ethtool.h>
+#include <linux/fcntl.h>
+#include <linux/ip.h>
+#include <linux/reboot.h>
+#include <linux/notifier.h>
+#include <linux/irq.h>
+#if defined(CONFIG_TIZEN)
+#include <linux/net_stat_tizen.h>
+#endif /* CONFIG_TIZEN */
+#include <net/addrconf.h>
+#ifdef ENABLE_ADAPTIVE_SCHED
+#include <linux/cpufreq.h>
+#endif /* ENABLE_ADAPTIVE_SCHED */
+#include <linux/rtc.h>
+#include <asm/uaccess.h>
+#include <asm/unaligned.h>
+#include <dhd_linux_priv.h>
+
+#include <epivers.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+#include <bcmdevs_legacy.h>    /* need to still support chips no longer in trunk firmware */
+#include <bcmiov.h>
+#include <bcmstdlib_s.h>
+
+#include <ethernet.h>
+#include <bcmevent.h>
+#include <vlan.h>
+#include <802.3.h>
+
+#include <dhd_linux_wq.h>
+#include <dhd.h>
+#include <dhd_linux_tx.h>
+#include <dhd_linux.h>
+#include <dhd_linux_pktdump.h>
+#ifdef DHD_WET
+#include <dhd_wet.h>
+#endif /* DHD_WET */
+#ifdef PCIE_FULL_DONGLE
+#include <dhd_flowring.h>
+#endif
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <dhd_config.h>
+#ifdef WL_ESCAN
+#include <wl_escan.h>
+#endif
+#include <dhd_dbg.h>
+#include <dhd_dbg_ring.h>
+#include <dhd_debug.h>
+#ifdef DHD_LOG_DUMP
+#include <dhd_log_dump.h>
+#endif
+#if defined(WL_CFG80211)
+#include <wl_cfg80211.h>
+#include <wl_cfgvif.h>
+#ifdef WL_BAM
+#include <wl_bam.h>
+#endif	/* WL_BAM */
+#endif	/* WL_CFG80211 */
+#ifdef PNO_SUPPORT
+#include <dhd_pno.h>
+#endif
+#ifdef RTT_SUPPORT
+#include <dhd_rtt.h>
+#endif
+
+#include <dhd_linux_sock_qos.h>
+
+#ifdef CONFIG_COMPAT
+#include <linux/compat.h>
+#endif
+
+#ifdef DHD_WMF
+#include <dhd_wmf_linux.h>
+#endif /* DHD_WMF */
+
+#ifdef DHD_L2_FILTER
+#include <bcmicmp.h>
+#include <bcm_l2_filter.h>
+#include <dhd_l2_filter.h>
+#endif /* DHD_L2_FILTER */
+
+#ifdef DHD_PSTA
+#include <dhd_psta.h>
+#endif /* DHD_PSTA */
+
+#ifdef AMPDU_VO_ENABLE
+/* XXX: Enabling VO AMPDU to reduce FER */
+#include <802.1d.h>
+#endif /* AMPDU_VO_ENABLE */
+
+#if defined(DHDTCPACK_SUPPRESS) || defined(DHDTCPSYNC_FLOOD_BLK)
+#include <dhd_ip.h>
+#endif /* DHDTCPACK_SUPPRESS || DHDTCPSYNC_FLOOD_BLK */
+#include <dhd_daemon.h>
+#ifdef DHD_PKT_LOGGING
+#include <dhd_pktlog.h>
+#endif /* DHD_PKT_LOGGING */
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+#include <eapol.h>
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+#ifdef DHD_DEBUG_PAGEALLOC
+typedef void (*page_corrupt_cb_t)(void *handle, void *addr_corrupt, size_t len);
+void dhd_page_corrupt_cb(void *handle, void *addr_corrupt, size_t len);
+extern void register_page_corrupt_cb(page_corrupt_cb_t cb, void* handle);
+#endif /* DHD_DEBUG_PAGEALLOC */
+
+#ifdef ENABLE_DHD_GRO
+#include <net/sch_generic.h>
+#endif /* ENABLE_DHD_GRO */
+
+#ifndef EVENT_LOG_RATE_HC_THRESHOLD
+/* FW checking period is 1 second, and once log quantity exceeds this
+ * threshold, FW will directly trap to indicate, so enlarge this
+ */
+#ifdef DHD_DEBUG
+#define EVENT_LOG_RATE_HC_THRESHOLD    99999999
+#else /* DHD_DEBUG */
+#define EVENT_LOG_RATE_HC_THRESHOLD    50000
+#endif /* DHD_DEBUG */
+#endif /* EVENT_LOG_RATE_HC_THRESHOLD */
+
+#define IP_PROT_RESERVED	0xFF
+
+#ifdef DHD_MQ
+#define MQ_MAX_QUEUES AC_COUNT
+#define MQ_MAX_CPUS 16
+int enable_mq = TRUE;
+module_param(enable_mq, int, 0644);
+int mq_select_disable = FALSE;
+#endif
+
+#if defined(DHD_LB)
+#if !defined(PCIE_FULL_DONGLE)
+#error "DHD Loadbalancing only supported on PCIE_FULL_DONGLE"
+#endif /* !PCIE_FULL_DONGLE */
+#endif /* DHD_LB */
+
+#if defined(DHD_LB_RXP) || defined(DHD_LB_TXP) || defined(DHD_LB_STATS)
+#if !defined(DHD_LB)
+#error "DHD loadbalance derivatives are supported only if DHD_LB is defined"
+#endif /* !DHD_LB */
+#endif /* DHD_LB_RXP || DHD_LB_TXP || DHD_LB_STATS */
+
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+static void dhd_m4_state_handler(struct work_struct * work);
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+static int dhd_wait_for_file_dump(dhd_pub_t *dhdp);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+#ifdef FIX_CPU_MIN_CLOCK
+#include <linux/pm_qos.h>
+#endif /* FIX_CPU_MIN_CLOCK */
+
+#ifdef PROP_TXSTATUS
+#include <wlfc_proto.h>
+#include <dhd_wlfc.h>
+#endif
+
+#if defined(OEM_ANDROID)
+#include <wl_android.h>
+#endif
+
+/* Maximum STA per radio */
+#if defined(BCM_ROUTER_DHD)
+#define DHD_MAX_STA     128
+#else
+#define DHD_MAX_STA     32
+#endif /* BCM_ROUTER_DHD */
+
+#if defined(IFACE_HANG_FORCE_DEV_CLOSE) && defined(HANG_DELAY_BEFORE_DEV_CLOSE)
+#ifndef WAIT_FOR_DEV_CLOSE_MAX
+#define WAIT_FOR_DEV_CLOSE_MAX 50
+#endif /* WAIT_FOR_DEV_CLOSE_MAX */
+#endif /* IFACE_HANG_FORCE_DEV_CLOSE && HANG_DELAY_BEFORE_DEV_CLOSE */
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+#include <ctf/hndctf.h>
+
+#ifdef CTFPOOL
+#define RXBUFPOOLSZ		2048
+#define	RXBUFSZ			DHD_FLOWRING_RX_BUFPOST_PKTSZ /* packet data buffer size */
+#endif /* CTFPOOL */
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#if defined(BCMDBG) && defined(DHD_MACDBG)
+#include <dhd_macdbg.h>
+#endif /* BCMDBG && DHD_MACDBG */
+
+#ifdef DHD_EVENT_LOG_FILTER
+#include <dhd_event_log_filter.h>
+#endif /* DHD_EVENT_LOG_FILTER */
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+static void dhd_blk_tsfl_handler(struct work_struct * work);
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+#ifdef DHD_TX_PROFILE
+#include <bcmarp.h>
+#include <bcmicmp.h>
+#include <bcmudp.h>
+#include <bcmproto.h>
+#endif /* defined(DHD_TX_PROFILE) */
+
+#ifdef CSI_SUPPORT
+#include <dhd_csi.h>
+#endif /* CSI_SUPPORT */
+
+#include <dhd_plat.h>
+
+#ifdef SET_RANDOM_MAC_SOFTAP
+#ifndef CONFIG_DHD_SET_RANDOM_MAC_VAL
+#define CONFIG_DHD_SET_RANDOM_MAC_VAL	0x001A11
+#endif
+static u32 vendor_oui = CONFIG_DHD_SET_RANDOM_MAC_VAL;
+#endif /* SET_RANDOM_MAC_SOFTAP */
+
+#if defined(BCM_ROUTER_DHD)
+/*
+ * Queue budget: Minimum number of packets that a queue must be allowed to hold
+ * to prevent starvation.
+ */
+#define DHD_QUEUE_BUDGET_DEFAULT    (256)
+int dhd_queue_budget = DHD_QUEUE_BUDGET_DEFAULT;
+
+module_param(dhd_queue_budget, int, 0);
+
+/*
+ * Per station pkt threshold: Sum total of all packets in the backup queues of
+ * flowrings belonging to the station, not including packets already admitted
+ * to flowrings.
+ */
+#define DHD_STA_THRESHOLD_DEFAULT   (2048)
+int dhd_sta_threshold = DHD_STA_THRESHOLD_DEFAULT;
+module_param(dhd_sta_threshold, int, 0);
+
+/*
+ * Per interface pkt threshold: Sum total of all packets in the backup queues of
+ * flowrings belonging to the interface, not including packets already admitted
+ * to flowrings.
+ */
+#define DHD_IF_THRESHOLD_DEFAULT   (2048 * 32)
+int dhd_if_threshold = DHD_IF_THRESHOLD_DEFAULT;
+module_param(dhd_if_threshold, int, 0);
+#endif /* BCM_ROUTER_DHD */
+
+/* XXX: where does this belong? */
+/* XXX: this needs to reviewed for host OS. */
+const uint8 wme_fifo2ac[] = { 0, 1, 2, 3, 1, 1 };
+const uint8 prio2fifo[8] = { 1, 0, 0, 1, 2, 2, 3, 3 };
+
+#ifdef ARP_OFFLOAD_SUPPORT
+void aoe_update_host_ipv4_table(dhd_pub_t *dhd_pub, u32 ipa, bool add, int idx);
+static int dhd_inetaddr_notifier_call(struct notifier_block *this,
+	unsigned long event, void *ptr);
+static struct notifier_block dhd_inetaddr_notifier = {
+	.notifier_call = dhd_inetaddr_notifier_call
+};
+/* to make sure we won't register the same notifier twice, otherwise a loop is likely to be
+ * created in the kernel notifier link list (with 'next' pointing to itself)
+ */
+static bool dhd_inetaddr_notifier_registered = FALSE;
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+int dhd_inet6addr_notifier_call(struct notifier_block *this,
+	unsigned long event, void *ptr);
+static struct notifier_block dhd_inet6addr_notifier = {
+	.notifier_call = dhd_inet6addr_notifier_call
+};
+/* to make sure we won't register the same notifier twice, otherwise a loop is likely to be
+ * created in kernel notifier link list (with 'next' pointing to itself)
+ */
+static bool dhd_inet6addr_notifier_registered = FALSE;
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+
+#if defined(CONFIG_PM_SLEEP)
+#include <linux/suspend.h>
+volatile bool dhd_mmc_suspend = FALSE;
+DECLARE_WAIT_QUEUE_HEAD(dhd_dpc_wait);
+#ifdef ENABLE_WAKEUP_PKT_DUMP
+volatile bool dhd_mmc_wake = FALSE;
+long long temp_raw;
+#endif /* ENABLE_WAKEUP_PKT_DUMP */
+#endif /* defined(CONFIG_PM_SLEEP) */
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) || defined(FORCE_WOWLAN)
+extern void dhd_enable_oob_intr(struct dhd_bus *bus, bool enable);
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+#if defined(OEM_ANDROID)
+static void dhd_hang_process(struct work_struct *work_data);
+#endif /* OEM_ANDROID */
+MODULE_LICENSE("GPL and additional rights");
+
+#if defined(MULTIPLE_SUPPLICANT)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25))
+DEFINE_MUTEX(_dhd_mutex_lock_);
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)) */
+#endif
+static int dhd_suspend_resume_helper(struct dhd_info *dhd, int val, int force);
+
+#include <dhd_bus.h>
+
+/* XXX Set up an MTU change notifier per linux/notifier.h? */
+#ifndef PROP_TXSTATUS
+#define DBUS_RX_BUFFER_SIZE_DHD(net)	(net->mtu + net->hard_header_len + dhd->pub.hdrlen)
+#else
+#define DBUS_RX_BUFFER_SIZE_DHD(net)	(net->mtu + net->hard_header_len + dhd->pub.hdrlen + 128)
+#endif
+
+#ifdef PROP_TXSTATUS
+extern bool dhd_wlfc_skip_fc(void * dhdp, uint8 idx);
+extern void dhd_wlfc_plat_init(void *dhd);
+extern void dhd_wlfc_plat_deinit(void *dhd);
+#endif /* PROP_TXSTATUS */
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+extern uint sd_f2_blocksize;
+extern int dhdsdio_func_blocksize(dhd_pub_t *dhd, int function_num, int block_size);
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+
+/* Linux wireless extension support */
+#if defined(WL_WIRELESS_EXT)
+#include <wl_iw.h>
+#endif /* defined(WL_WIRELESS_EXT) */
+
+#if defined(CONFIG_PARTIALSUSPEND_SLP) && !defined(DHD_USE_PM_SLEEP)
+/* XXX SLP use defferent earlysuspend header file and some functions
+ * But most of meaning is same as Android
+ */
+#include <linux/partialsuspend_slp.h>
+#define CONFIG_HAS_EARLYSUSPEND
+#define DHD_USE_EARLYSUSPEND
+#define register_early_suspend		register_pre_suspend
+#define unregister_early_suspend	unregister_pre_suspend
+#define early_suspend				pre_suspend
+#define EARLY_SUSPEND_LEVEL_BLANK_SCREEN		50
+#else
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+#include <linux/earlysuspend.h>
+#endif /* defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND) */
+#endif /* CONFIG_PARTIALSUSPEND_SLP && !DHD_USE_PM_SLEEP */
+
+#if defined(APF)
+void dhd_dev_pkt_filter_stats(struct net_device *ndev, int pkt_filter_id);
+static int _dhd_apf_add_filter(struct net_device *ndev, uint32 filter_id, u8* program,
+	uint32 program_len);
+static int _dhd_apf_config_filter(struct net_device *ndev, uint32 filter_id, uint32 mode,
+	uint32 enable);
+static int _dhd_apf_delete_filter(struct net_device *ndev, uint32 filter_id);
+static int _dhd_apf_read_filter_data(struct net_device *ndev, uint32 filter_id, u8* buf,
+	uint32 buf_len);
+#endif /* APF */
+
+#ifdef DHD_FW_COREDUMP
+static void dhd_mem_dump(void *dhd_info, void *event_info, u8 event);
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#include <linux/workqueue.h>
+#include <linux/pm_runtime.h>
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef DHD_DEBUG_UART
+#include <linux/kmod.h>
+#define DHD_DEBUG_UART_EXEC_PATH	"/system/bin/wldu"
+static void dhd_debug_uart_exec_rd(void *handle, void *event_info, u8 event);
+static void dhd_debug_uart_exec(dhd_pub_t *dhdp, char *cmd);
+#endif	/* DHD_DEBUG_UART */
+
+static int dhd_reboot_callback(struct notifier_block *this, unsigned long code, void *unused);
+static struct notifier_block dhd_reboot_notifier = {
+	.notifier_call = dhd_reboot_callback,
+	.priority = 1,
+};
+
+dhd_pub_t	*g_dhd_pub = NULL;
+
+#if defined(BT_OVER_SDIO)
+#include "dhd_bt_interface.h"
+#endif /* defined (BT_OVER_SDIO) */
+
+#if defined(DHD_ENABLE_S2MPU)
+#include <soc/samsung/exynos-s2mpu.h>
+#endif /* DHD_ENABLE_S2MPU */
+
+#ifdef WL_STATIC_IF
+bool dhd_is_static_ndev(dhd_pub_t *dhdp, struct net_device *ndev);
+#endif /* WL_STATIC_IF */
+
+atomic_t exit_in_progress = ATOMIC_INIT(0);
+atomic_t reboot_in_progress = ATOMIC_INIT(-1);
+
+static void dhd_process_daemon_msg(struct sk_buff *skb);
+static void dhd_destroy_to_notifier_skt(void);
+static int dhd_create_to_notifier_skt(void);
+static struct sock *nl_to_event_sk = NULL;
+int sender_pid = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+struct netlink_kernel_cfg dhd_netlink_cfg = {
+	.groups = 1,
+	.input = dhd_process_daemon_msg,
+};
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0)) */
+
+#if defined(BT_OVER_SDIO)
+/* Flag to indicate if driver is initialized */
+uint dhd_driver_init_done = TRUE;
+#else
+/* Flag to indicate if driver is initialized */
+uint dhd_driver_init_done = FALSE;
+#endif
+/* Flag to indicate if we should download firmware on driver load */
+uint dhd_download_fw_on_driverload = TRUE;
+
+/* Definitions to provide path to the firmware and nvram
+ * example nvram_path[MOD_PARAM_PATHLEN]="/projects/wlan/nvram.txt"
+ */
+#ifdef DHD_LINUX_STD_FW_API
+char firmware_path[MOD_PARAM_PATHLEN] = DHD_FW_NAME;
+char nvram_path[MOD_PARAM_PATHLEN] = DHD_NVRAM_NAME;
+char clm_path[MOD_PARAM_PATHLEN];
+char config_path[MOD_PARAM_PATHLEN];
+#else
+char firmware_path[MOD_PARAM_PATHLEN] = CONFIG_BCMDHD_FW_PATH;
+char nvram_path[MOD_PARAM_PATHLEN] = CONFIG_BCMDHD_NVRAM_PATH;
+char clm_path[MOD_PARAM_PATHLEN];
+char config_path[MOD_PARAM_PATHLEN];
+#endif /* DHD_LINUX_STD_FW_API */
+char signature_path[MOD_PARAM_PATHLEN];
+#ifdef DHD_UCODE_DOWNLOAD
+char ucode_path[MOD_PARAM_PATHLEN];
+#endif /* DHD_UCODE_DOWNLOAD */
+
+module_param_string(clm_path, clm_path, MOD_PARAM_PATHLEN, 0660);
+
+/* backup buffer for firmware and nvram path */
+char fw_bak_path[MOD_PARAM_PATHLEN];
+char nv_bak_path[MOD_PARAM_PATHLEN];
+
+/* information string to keep firmware, chio, cheip version info visiable from log */
+char info_string[MOD_PARAM_INFOLEN];
+module_param_string(info_string, info_string, MOD_PARAM_INFOLEN, 0444);
+
+#ifdef SYNA_SAR_CUSTOMER_PARAMETER
+char config_sar_path[MOD_PARAM_PATHLEN] = CONFIG_BCMDHD_CONFIG_SAR_PATH;
+module_param_string(config_sar_path, config_sar_path, MOD_PARAM_PATHLEN, 0660);
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER */
+int op_mode = 0;
+int disable_proptx = 0;
+module_param(op_mode, int, 0644);
+#if defined(OEM_ANDROID)
+extern int wl_control_wl_start(struct net_device *dev);
+#if defined(BCMLXSDMMC) || defined(BCMDBUS)
+struct semaphore dhd_registration_sem;
+#endif /* BCMXSDMMC */
+#endif /* defined(OEM_ANDROID) */
+void dhd_generate_rand_mac_addr(struct ether_addr *ea_addr);
+
+#ifdef EWP_EDL
+int host_edl_support = TRUE;
+module_param(host_edl_support, int, 0644);
+#endif
+
+/* deferred handlers */
+static void dhd_ifadd_event_handler(void *handle, void *event_info, u8 event);
+static void dhd_ifdel_event_handler(void *handle, void *event_info, u8 event);
+static void dhd_set_mac_addr_handler(void *handle, void *event_info, u8 event);
+static void dhd_set_mcast_list_handler(void *handle, void *event_info, u8 event);
+static void dhd_ndev_upd_features_handler(void *handle, void *event_info, u8 event);
+#ifdef BCM_ROUTER_DHD
+static void dhd_inform_dhd_monitor_handler(void *handle, void *event_info, u8 event);
+#endif
+
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_STATIC_IN_DRIVER)
+extern int dhd_static_buf_init(void);
+extern void dhd_static_buf_exit(void);
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_STATIC_IN_DRIVER */
+#ifdef DHD_UPDATE_INTF_MAC
+static void dhd_ifupdate_event_handler(void *handle, void *event_info, u8 event);
+#endif /* DHD_UPDATE_INTF_MAC */
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+static void dhd_inet6_work_handler(void *dhd_info, void *event_data, u8 event);
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+#ifdef WL_CFG80211
+extern void dhd_netdev_free(struct net_device *ndev);
+#endif /* WL_CFG80211 */
+static dhd_if_t * dhd_get_ifp_by_ndev(dhd_pub_t *dhdp, struct net_device *ndev);
+
+#if defined(WLDWDS) && defined(FOURADDR_AUTO_BRG)
+static void dhd_bridge_dev_set(dhd_info_t *dhd, int ifidx, struct net_device *dev);
+#endif /* defiend(WLDWDS) && defined(FOURADDR_AUTO_BRG) */
+
+#if (defined(DHD_WET) || defined(DHD_MCAST_REGEN) || defined(DHD_L2_FILTER))
+/* update rx_pkt_chainable state of dhd interface */
+static void dhd_update_rx_pkt_chainable_state(dhd_pub_t* dhdp, uint32 idx);
+#endif /* DHD_WET || DHD_MCAST_REGEN || DHD_L2_FILTER */
+
+/* Error bits */
+module_param(dhd_msg_level, int, 0644);
+#if defined(WL_WIRELESS_EXT)
+module_param(iw_msg_level, int, 0);
+#endif
+#ifdef WL_CFG80211
+module_param(wl_dbg_level, int, 0);
+#endif
+module_param(android_msg_level, int, 0);
+module_param(config_msg_level, int, 0);
+
+#ifdef ARP_OFFLOAD_SUPPORT
+/* ARP offload agent mode : Enable ARP Host Auto-Reply and ARP Peer Auto-Reply */
+/* XXX ARP HOST Auto Reply can cause dongle trap at VSDB situation */
+/* XXX ARP OL SNOOP can be used to more good quility */
+
+#ifdef ENABLE_ARP_SNOOP_MODE
+uint dhd_arp_mode = (ARP_OL_AGENT | ARP_OL_PEER_AUTO_REPLY | ARP_OL_SNOOP | ARP_OL_HOST_AUTO_REPLY |
+		ARP_OL_UPDATE_HOST_CACHE);
+#else
+uint dhd_arp_mode = ARP_OL_AGENT | ARP_OL_PEER_AUTO_REPLY | ARP_OL_UPDATE_HOST_CACHE;
+#endif /* ENABLE_ARP_SNOOP_MODE */
+
+module_param(dhd_arp_mode, uint, 0);
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+/* Disable Prop tx */
+module_param(disable_proptx, int, 0644);
+/* load firmware and/or nvram values from the filesystem */
+module_param_string(firmware_path, firmware_path, MOD_PARAM_PATHLEN, 0660);
+module_param_string(nvram_path, nvram_path, MOD_PARAM_PATHLEN, 0660);
+module_param_string(config_path, config_path, MOD_PARAM_PATHLEN, 0);
+module_param_string(signature_path, signature_path, MOD_PARAM_PATHLEN, 0660);
+#ifdef DHD_UCODE_DOWNLOAD
+module_param_string(ucode_path, ucode_path, MOD_PARAM_PATHLEN, 0660);
+#endif /* DHD_UCODE_DOWNLOAD */
+
+/* wl event forwarding */
+#ifdef WL_EVENT_ENAB
+uint wl_event_enable = true;
+#else
+uint wl_event_enable = false;
+#endif /* WL_EVENT_ENAB */
+module_param(wl_event_enable, uint, 0660);
+
+/* wl event forwarding */
+#ifdef LOGTRACE_PKT_SENDUP
+uint logtrace_pkt_sendup = true;
+#else
+uint logtrace_pkt_sendup = false;
+#endif /* LOGTRACE_PKT_SENDUP */
+module_param(logtrace_pkt_sendup, uint, 0660);
+
+/* Watchdog interval */
+/* extend watchdog expiration to 2 seconds when DPC is running */
+#define WATCHDOG_EXTEND_INTERVAL (2000)
+
+uint dhd_watchdog_ms = CUSTOM_DHD_WATCHDOG_MS;
+module_param(dhd_watchdog_ms, uint, 0);
+
+#if defined(DHD_DEBUG)
+/* Console poll interval */
+uint dhd_console_ms = CUSTOM_DHD_CONSOLE_MS;
+module_param(dhd_console_ms, uint, 0644);
+#else
+uint dhd_console_ms = 0;
+#endif /* DHD_DEBUG */
+
+uint dhd_slpauto = TRUE;
+module_param(dhd_slpauto, uint, 0);
+
+#ifdef PKT_FILTER_SUPPORT
+/* Global Pkt filter enable control */
+uint dhd_pkt_filter_enable = TRUE;
+module_param(dhd_pkt_filter_enable, uint, 0);
+#endif
+
+/* Pkt filter init setup */
+uint dhd_pkt_filter_init = 0;
+module_param(dhd_pkt_filter_init, uint, 0);
+
+/* Pkt filter mode control */
+#ifdef GAN_LITE_NAT_KEEPALIVE_FILTER
+uint dhd_master_mode = FALSE;
+#else
+uint dhd_master_mode = FALSE;
+#endif /* GAN_LITE_NAT_KEEPALIVE_FILTER */
+module_param(dhd_master_mode, uint, 0);
+
+int dhd_watchdog_prio = 0;
+module_param(dhd_watchdog_prio, int, 0);
+
+/* DPC thread priority */
+int dhd_dpc_prio = CUSTOM_DPC_PRIO_SETTING;
+module_param(dhd_dpc_prio, int, 0);
+
+#if !defined(BCMDBUS)
+extern int dhd_dongle_ramsize;
+module_param(dhd_dongle_ramsize, int, 0);
+#endif /* !BCMDBUS */
+
+#ifdef WL_CFG80211
+int passive_channel_skip = 0;
+module_param(passive_channel_skip, int, (S_IRUSR|S_IWUSR));
+#endif /* WL_CFG80211 */
+static dhd_if_t * dhd_get_ifp_by_ndev(dhd_pub_t *dhdp, struct net_device *ndev);
+
+#ifdef DHD_MSI_SUPPORT
+uint enable_msi = TRUE;
+module_param(enable_msi, uint, 0);
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef DHD_SSSR_DUMP
+int dhdpcie_sssr_dump_get_before_after_len(dhd_pub_t *dhd, uint32 *arr_len);
+module_param(sssr_enab, uint, 0);
+module_param(fis_enab, uint, 0);
+#endif /* DHD_SSSR_DUMP */
+
+/* Keep track of number of instances */
+static int dhd_found = 0;
+static int instance_base = 0; /* Starting instance number */
+module_param(instance_base, int, 0644);
+
+#if defined(DHD_LB_RXP) && defined(PCIE_FULL_DONGLE)
+/*
+ * Rx path process budget(dhd_napi_weight) number of packets in one go and hands over
+ * the packets to network stack.
+ *
+ * dhd_dpc tasklet is the producer(packets received from dongle) and dhd_napi_poll()
+ * is the consumer. The maximum number of packets that can be received from the dongle
+ * at any given point of time are D2HRING_RXCMPLT_MAX_ITEM.
+ * Also DHD will always post fresh rx buffers to dongle while processing rx completions.
+ *
+ * The consumer must consume the packets at equal are better rate than the producer.
+ * i.e if dhd_napi_poll() does not process at the same rate as the producer(dhd_dpc),
+ * rx_process_queue depth increases, which can even consume the entire system memory.
+ * Such situation will be tacken care by rx flow control.
+ *
+ * Device drivers are strongly advised to not use bigger value than NAPI_POLL_WEIGHT
+ */
+static int dhd_napi_weight = NAPI_POLL_WEIGHT;
+module_param(dhd_napi_weight, int, 0644);
+#endif /* DHD_LB_RXP && PCIE_FULL_DONGLE */
+
+#ifdef PCIE_FULL_DONGLE
+extern uint h2d_max_txpost;
+module_param(h2d_max_txpost, uint, 0644);
+
+extern uint h2d_htput_max_txpost;
+module_param(h2d_htput_max_txpost, uint, 0644);
+
+extern uint d2h_max_txcpl;
+module_param(d2h_max_txcpl, uint, 0644);
+
+extern uint h2d_max_ctrlpost;
+module_param(h2d_max_ctrlpost, uint, 0644);
+extern uint d2h_max_ctrlcpl;
+module_param(d2h_max_ctrlcpl, uint, 0644);
+
+extern uint h2d_max_rxpost;
+module_param(h2d_max_rxpost, uint, 0644);
+extern uint d2h_max_rxcpl;
+module_param(d2h_max_rxcpl, uint, 0644);
+
+extern uint rx_buf_burst;
+module_param(rx_buf_burst, uint, 0644);
+
+extern uint rx_bufpost_threshold;
+module_param(rx_bufpost_threshold, uint, 0644);
+
+#ifdef AGG_H2D_DB
+extern bool agg_h2d_db_enab;
+module_param(agg_h2d_db_enab, bool, 0644);
+extern uint agg_h2d_db_timeout;
+module_param(agg_h2d_db_timeout, uint, 0644);
+extern uint agg_h2d_db_inflight_thresh;
+module_param(agg_h2d_db_inflight_thresh, uint, 0644);
+#endif /* AGG_H2D_DB */
+
+extern uint dma_ring_indices;
+module_param(dma_ring_indices, uint, 0644);
+
+extern bool h2d_phase;
+module_param(h2d_phase, bool, 0644);
+extern bool force_trap_bad_h2d_phase;
+module_param(force_trap_bad_h2d_phase, bool, 0644);
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef FORCE_TPOWERON
+/*
+ * On Fire's reference platform, coming out of L1.2,
+ * there is a constant delay of 45us between CLKREQ# and stable REFCLK
+ * Due to this delay, with tPowerOn < 50
+ * there is a chance of the refclk sense to trigger on noise.
+ *
+ * 0x29 when written to L1SSControl2 translates to 50us.
+ */
+#define FORCE_TPOWERON_50US 0x29
+uint32 tpoweron_scale = FORCE_TPOWERON_50US; /* default 50us */
+module_param(tpoweron_scale, uint, 0644);
+#endif /* FORCE_TPOWERON */
+
+#ifdef SHOW_LOGTRACE
+#ifdef DHD_LINUX_STD_FW_API
+static char *logstrs_path = "logstrs.bin";
+char *st_str_file_path = "rtecdc.bin";
+static char *map_file_path = "rtecdc.map";
+static char *rom_st_str_file_path = "roml.bin";
+static char *rom_map_file_path = "roml.map";
+#else
+static char *logstrs_path = PLATFORM_PATH"logstrs.bin";
+char *st_str_file_path = PLATFORM_PATH"rtecdc.bin";
+static char *map_file_path = PLATFORM_PATH"rtecdc.map";
+static char *rom_st_str_file_path = PLATFORM_PATH"roml.bin";
+static char *rom_map_file_path = PLATFORM_PATH"roml.map";
+#endif /* DHD_LINUX_STD_FW_API */
+
+static char *ram_file_str = "rtecdc";
+static char *rom_file_str = "roml";
+
+module_param(logstrs_path, charp, S_IRUGO);
+module_param(st_str_file_path, charp, S_IRUGO);
+module_param(map_file_path, charp, S_IRUGO);
+module_param(rom_st_str_file_path, charp, S_IRUGO);
+module_param(rom_map_file_path, charp, S_IRUGO);
+
+static int dhd_init_logstrs_array(osl_t *osh, dhd_event_log_t *temp);
+static int dhd_read_map(osl_t *osh, char *fname, uint32 *ramstart, uint32 *rodata_start,
+	uint32 *rodata_end);
+static int dhd_init_static_strs_array(osl_t *osh, dhd_event_log_t *temp, char *str_file,
+	char *map_file);
+#endif /* SHOW_LOGTRACE */
+
+#ifdef D2H_MINIDUMP
+void dhd_d2h_minidump(dhd_pub_t *dhdp);
+#endif /* D2H_MINIDUMP */
+
+#ifdef DHD_TX_PROFILE
+/* process layer 3 headers, to ultimately determine if a
+ * dhd_tx_profile_protocol_t matches
+ */
+static int process_layer3_headers(uint8 **p, int plen, uint16 *type);
+
+/* process layer 2 headers, to ultimately determine if a
+ * dhd_tx_profile_protocol_t matches
+ */
+static int process_layer2_headers(uint8 **p, int *plen, uint16 *type, bool is_host_sfhllc);
+
+/* whether or not a dhd_tx_profile_protocol_t matches with data in a packet */
+bool dhd_protocol_matches_profile(uint8 *p, int plen, const
+		dhd_tx_profile_protocol_t *proto, bool is_host_sfhllc);
+#endif /* defined(DHD_TX_PROFILE) */
+
+#define PATH_BANDLOCK_INFO PLATFORM_PATH".bandlock.info"
+
+static void dhd_set_bandlock(dhd_pub_t * dhd);
+
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+void dhd_alert_process(struct work_struct *work_data);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+
+#ifdef USE_WFA_CERT_CONF
+int g_frameburst = 1;
+#endif /* USE_WFA_CERT_CONF */
+
+static int dhd_get_pend_8021x_cnt(dhd_info_t *dhd);
+
+#ifdef PCIE_FULL_DONGLE
+#define DHD_IF_STA_LIST_LOCK_INIT(lock) spin_lock_init(lock)
+
+#if defined(DHD_IGMP_UCQUERY) || defined(DHD_UCAST_UPNP)
+static struct list_head * dhd_sta_list_snapshot(dhd_info_t *dhd, dhd_if_t *ifp,
+	struct list_head *snapshot_list);
+static void dhd_sta_list_snapshot_free(dhd_info_t *dhd, struct list_head *snapshot_list);
+#define DHD_IF_WMF_UCFORWARD_LOCK(dhd, ifp, slist) ({ dhd_sta_list_snapshot(dhd, ifp, slist); })
+#define DHD_IF_WMF_UCFORWARD_UNLOCK(dhd, slist) ({ dhd_sta_list_snapshot_free(dhd, slist); })
+#endif /* DHD_IGMP_UCQUERY || DHD_UCAST_UPNP */
+#endif /* PCIE_FULL_DONGLE */
+
+/* hostwake delay after d3 enter in ms */
+#define D3_HOSTWAKE_DELAY	200
+
+/* Control fw roaming */
+#ifdef BCMCCX
+uint dhd_roam_disable = 0;
+#else
+#ifdef OEM_ANDROID
+uint dhd_roam_disable = 0;
+#else
+uint dhd_roam_disable = 1;
+#endif
+#endif /* BCMCCX */
+
+#ifdef BCMDBGFS
+extern void dhd_dbgfs_init(dhd_pub_t *dhdp);
+extern void dhd_dbgfs_remove(void);
+#endif
+
+#ifdef BCMPCIE
+/* Tx/Rx/Ctrl cpl/post bounds */
+extern uint dhd_tx_cpl_bound;
+extern uint dhd_rx_cpl_post_bound;
+extern uint dhd_tx_post_bound;
+extern uint dhd_ctrl_cpl_post_bound;
+
+module_param(dhd_tx_cpl_bound, uint, 0);
+module_param(dhd_rx_cpl_post_bound, uint, 0);
+module_param(dhd_tx_post_bound, uint, 0);
+module_param(dhd_ctrl_cpl_post_bound, uint, 0);
+#endif /* BCMPCIE */
+
+/* Enable TX status metadta report: 0=disable 1=enable 2=debug */
+static uint pcie_txs_metadata_enable = 0;
+module_param(pcie_txs_metadata_enable, int, 0);
+
+/* Control radio state */
+uint dhd_radio_up = 1;
+
+/* Network inteface name */
+char iface_name[IFNAMSIZ] = {'\0'};
+module_param_string(iface_name, iface_name, IFNAMSIZ, 0);
+
+/* The following are specific to the SDIO dongle */
+
+/* IOCTL response timeout */
+int dhd_ioctl_timeout_msec = IOCTL_RESP_TIMEOUT;
+
+/* DS Exit response timeout */
+int ds_exit_timeout_msec = DS_EXIT_TIMEOUT;
+
+/* Idle timeout for backplane clock */
+int dhd_idletime = DHD_IDLETIME_TICKS;
+module_param(dhd_idletime, int, 0);
+
+/* Use polling */
+uint dhd_poll = FALSE;
+module_param(dhd_poll, uint, 0);
+
+/* Use interrupts */
+uint dhd_intr = TRUE;
+module_param(dhd_intr, uint, 0);
+
+/* SDIO Drive Strength (in milliamps) */
+uint dhd_sdiod_drive_strength = 6;
+module_param(dhd_sdiod_drive_strength, uint, 0);
+
+#ifdef BCMSDIO
+/* Tx/Rx bounds */
+extern uint dhd_txbound;
+extern uint dhd_rxbound;
+module_param(dhd_txbound, uint, 0);
+module_param(dhd_rxbound, uint, 0);
+
+/* Deferred transmits */
+extern uint dhd_deferred_tx;
+module_param(dhd_deferred_tx, uint, 0);
+
+#endif /* BCMSDIO */
+
+#ifdef BCMSLTGT
+#ifdef BCMFPGA_HW
+/* For FPGA use fixed htclkration as 30 */
+uint htclkratio = 30;
+#else
+uint htclkratio = 1;
+#endif /* BCMFPGA_HW */
+module_param(htclkratio, uint, 0);
+
+int dngl_xtalfreq = 0;
+module_param(dngl_xtalfreq, int, 0);
+#endif /* BCMSLTGT */
+
+#ifdef SDTEST
+/* Echo packet generator (pkts/s) */
+uint dhd_pktgen = 0;
+module_param(dhd_pktgen, uint, 0);
+
+/* Echo packet len (0 => sawtooth, max 2040) */
+uint dhd_pktgen_len = 0;
+module_param(dhd_pktgen_len, uint, 0);
+#endif /* SDTEST */
+
+#if defined(BCMSUP_4WAY_HANDSHAKE)
+/* Use in dongle supplicant for 4-way handshake */
+#if defined(WLFBT) || defined(WL_ENABLE_IDSUP)
+/* Enable idsup by default (if supported in fw) */
+uint dhd_use_idsup = 1;
+#else
+uint dhd_use_idsup = 0;
+#endif /* WLFBT || WL_ENABLE_IDSUP */
+module_param(dhd_use_idsup, uint, 0);
+#endif /* BCMSUP_4WAY_HANDSHAKE */
+
+#ifndef BCMDBUS
+#if defined(OEM_ANDROID)
+/* Allow delayed firmware download for debug purpose */
+int allow_delay_fwdl = FALSE;
+#elif defined(BCM_ROUTER_DHD)
+/* Allow delayed firmware download for debug purpose */
+int allow_delay_fwdl = FALSE;
+#else
+int allow_delay_fwdl = TRUE;
+#endif /* OEM_ANDROID */
+module_param(allow_delay_fwdl, int, 0);
+#endif /* !BCMDBUS */
+
+#ifdef GDB_PROXY
+/* Adds/replaces deadman_to= in NVRAM file with deadman_to=0 */
+static uint nodeadman = 0;
+module_param(nodeadman, uint, 0);
+#endif /* GDB_PROXY */
+
+#ifdef ECOUNTER_PERIODIC_DISABLE
+uint enable_ecounter = FALSE;
+#else
+uint enable_ecounter = TRUE;
+#endif
+module_param(enable_ecounter, uint, 0);
+
+#ifdef BCMQT_HW
+int qt_flr_reset = FALSE;
+module_param(qt_flr_reset, int, 0);
+
+int qt_dngl_timeout = 0; // dongle attach timeout in ms
+module_param(qt_dngl_timeout, int, 0);
+#endif /* BCMQT_HW */
+
+#ifdef DHD_SUPPORT_SPMI_MODE
+uint spmi_mode = 0;
+module_param(spmi_mode, uint, 0644);
+#endif /* DHD_SUPPORT_SPMI_MODE */
+
+/* TCM verification flag */
+uint dhd_tcm_test_enable = FALSE;
+module_param(dhd_tcm_test_enable, uint, 0644);
+
+tcm_test_status_t dhd_tcm_test_status = TCM_TEST_NOT_RUN;
+tcm_test_mode_t dhd_tcm_test_mode = TCM_TEST_MODE_ALWAYS;
+
+extern char dhd_version[];
+extern char fw_version[];
+extern char clm_version[];
+
+int dhd_net_bus_devreset(struct net_device *dev, uint8 flag);
+static void dhd_net_if_lock_local(dhd_info_t *dhd);
+static void dhd_net_if_unlock_local(dhd_info_t *dhd);
+static void dhd_suspend_lock(dhd_pub_t *dhdp);
+static void dhd_suspend_unlock(dhd_pub_t *dhdp);
+
+/* Monitor interface */
+int dhd_monitor_init(void *dhd_pub);
+int dhd_monitor_uninit(void);
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+bool g_pm_control;
+#ifdef DHD_EXPORT_CNTL_FILE
+uint32 pmmode_val = 0xFF;
+#endif /* DHD_EXPORT_CNTL_FILE */
+void sec_control_pm(dhd_pub_t *dhd, uint *);
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+
+#if defined(WL_WIRELESS_EXT)
+struct iw_statistics *dhd_get_wireless_stats(struct net_device *dev);
+#endif /* defined(WL_WIRELESS_EXT) */
+
+#ifdef DHD_PM_OVERRIDE
+bool g_pm_override;
+#endif /* DHD_PM_OVERRIDE */
+
+#ifndef BCMDBUS
+static void dhd_dpc(ulong data);
+#endif /* !BCMDBUS */
+/* forward decl */
+extern int dhd_wait_pend8021x(struct net_device *dev);
+void dhd_os_wd_timer_extend(void *bus, bool extend);
+
+#ifdef TOE
+#ifndef BDC
+#error TOE requires BDC
+#endif /* !BDC */
+static int dhd_toe_get(dhd_info_t *dhd, int idx, uint32 *toe_ol);
+static int dhd_toe_set(dhd_info_t *dhd, int idx, uint32 toe_ol);
+#endif /* TOE */
+
+#ifdef DHD_MAP_LOGGING
+void dhd_smmu_fault_handler(uint32 axid, ulong fault_addr);
+#endif /* DHD_MAP_LOGGING */
+
+#if defined(DHD_ENABLE_S2MPU)
+int s2mpufd_notifier_callback(struct s2mpufd_notifier_block *block,
+		struct s2mpufd_notifier_info *info);
+static void dhd_module_s2mpu_register(struct device *dev);
+#endif /* DHD_ENABLE_S2MPU */
+#ifdef DHD_REGISTER_SMMU_FAULT_HANDLER
+extern int pcie_sysmmu_add_fault_handler(struct notifier_block *pcie_sysmmu_nb);
+static int sysmmu_notifier_callback(struct notifier_block *nb, unsigned long addr,
+	void *flag);
+static struct notifier_block dhd_pcie_sysmmu_nb = {
+	.notifier_call = sysmmu_notifier_callback
+};
+#endif /* DHD_REGISTER_SMMU_FAULT_HANDLER */
+
+static int dhd_suspend_resume_helper(struct dhd_info *dhd, int val, int force);
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+static void dhd_dump_proc(struct work_struct *work_data);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+#if defined(BCMSDIO)
+extern void dhd_bus_check_srmemsize(dhd_pub_t *dhdp);
+#endif /* BCMSDIO */
+
+#ifdef CUSTOM_EVENT_PM_WAKE
+void
+dhd_set_excess_pm_awake(dhd_pub_t *dhd, bool suspend)
+{
+	int ret = 0;
+	uint32 iovar_val = 0;   /* Disable the excess PM notify */
+	char *iovar_name;
+
+#ifdef CUSTOM_EVENT_PM_PERCENT
+	iovar_name = "excess_pm_period";
+#else
+	iovar_name = "const_awake_thresh";
+	iovar_val = CUSTOM_EVENT_PM_WAKE;
+#endif /* CUSTOM_EVENT_PM_PERCENT */
+
+	if (suspend) {
+		iovar_val = CUSTOM_EVENT_PM_WAKE * 4;
+	}
+
+	ret = dhd_iovar(dhd, 0, iovar_name, (char *)&iovar_val,
+			sizeof(iovar_val), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set %s failed %d\n", __FUNCTION__, iovar_name, ret));
+	}
+	return;
+}
+void
+dhd_init_excess_pm_awake(dhd_pub_t *dhd)
+{
+	int ret = 0;
+	uint32 pm_awake_thresh = CUSTOM_EVENT_PM_WAKE;
+#ifdef CUSTOM_EVENT_PM_PERCENT
+	uint32 pm_awake_percent = CUSTOM_EVENT_PM_PERCENT;
+
+	ret = dhd_iovar(dhd, 0, "excess_pm_percent", (char *)&pm_awake_percent,
+			sizeof(pm_awake_percent), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set excess_pm_percent failed %d\n", __FUNCTION__, ret));
+	}
+	pm_awake_thresh = 0;    /* Disable the excess PM notify */
+#endif /* CUSTOM_EVENT_PM_PERCENT */
+	ret = dhd_iovar(dhd, 0, "const_awake_thresh", (char *)&pm_awake_thresh,
+			sizeof(pm_awake_thresh), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set const_awake_thresh failed %d\n", __FUNCTION__, ret));
+	}
+
+	return;
+}
+#endif /* CUSTOM_EVENT_PM_WAKE */
+
+#if defined(CONFIG_PM_SLEEP)
+#ifdef WL_TWT
+#define TWT_NOMINAL_RESUME	(5U * 1024U) /* 5ms */
+
+int dhd_send_twt_info_suspend(dhd_pub_t *dhdp, bool suspend)
+{
+	int ret = BCME_OK;
+	wl_twt_info_t ti;
+	u8 buf[WLC_IOCTL_SMLEN] = {0};
+
+	uint8 *pbuf = buf;
+	uint16 param_len  = sizeof(buf);
+
+	bzero(&ti, sizeof(ti));
+	ti.version = WL_TWT_INFO_VER;
+	ti.length = sizeof(ti.version) + sizeof(ti.length);
+
+	if (!dhdp || dhdp->up == 0) {
+		return ret;
+	}
+
+	/* if it's not associated, skip send info */
+	if (!dhd_is_associated(dhdp, 0, NULL)) {
+		return ret;
+	}
+
+	/* Default values, Overide Below */
+	ti.infodesc.flow_id = 0xFF;
+	ti.desc.next_twt_h = 0xFFFFFFFF;
+	ti.desc.next_twt_l = 0xFFFFFFFF;
+
+	/* Set allTWT info suspend, ConfigID = 0 */
+	ti.configID = 0;
+
+	/* resume TWT session in resume */
+	if (suspend == FALSE) {
+		/* Resume TWT session */
+		ti.infodesc.next_twt_h = htod32((u32)((u64)TWT_NOMINAL_RESUME >> 32));
+		ti.infodesc.next_twt_l = htod32((u32)TWT_NOMINAL_RESUME);
+		ti.infodesc.flow_flags |= WL_TWT_INFO_FLAG_RESUME;
+	}
+
+	/* Packing parameters */
+	ret = bcm_pack_xtlv_entry(&pbuf, &param_len, WL_TWT_CMD_INFO,
+			sizeof(ti), (uint8 *)&ti, BCM_XTLV_OPTION_ALIGN32);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s : parameter packing error \n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = dhd_iovar(dhdp, 0, "twt", buf, sizeof(buf) - param_len, NULL, 0, TRUE);
+	if (ret) {
+		DHD_ERROR(("%s : TWT info failed ret : %d\n", __FUNCTION__, ret));
+	}
+	return ret;
+
+}
+int dhd_config_twt_event_mask_in_suspend(dhd_pub_t *dhdp, bool suspend)
+{
+	int ret = BCME_OK;
+	u8 buf[WLC_IOCTL_SMLEN] = {0};
+	eventmsgs_ext_t *eventmask_msg = NULL;
+
+	int msglen = WL_EVENTING_MASK_EXT_LEN + EVENTMSGS_EXT_STRUCT_SIZE;
+
+	if (!dhdp || dhdp->up == 0) {
+		return ret;
+	}
+
+	/* if it's not associated in suspend, skip teardown */
+	if (suspend && !dhd_is_associated(dhdp, 0, NULL)) {
+		return ret;
+	}
+
+	/* TWT_E_TWT event mask configuration */
+	eventmask_msg = (eventmsgs_ext_t *)MALLOC(dhdp->osh, msglen);
+	if (eventmask_msg == NULL) {
+		DHD_ERROR(("%s : failed to allocate for event_msg_ext\n", __FUNCTION__));
+		return ret;
+	}
+
+	bzero(eventmask_msg, msglen);
+	eventmask_msg->ver = EVENTMSGS_VER;
+	eventmask_msg->len = ROUNDUP(WLC_E_LAST, NBBY)/NBBY;
+
+	/* Read event_msgs_ext mask */
+	ret = dhd_iovar(dhdp, 0, "event_msgs_ext", (char *)eventmask_msg, msglen, buf,
+			WLC_IOCTL_SMLEN, FALSE);
+	/* event_msgs_ext must be supported */
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s read event mask ext failed %d\n", __FUNCTION__, ret));
+		goto fail;
+	}
+
+	bcopy(buf, eventmask_msg, msglen);
+
+	/* suspend */
+	if (suspend) {
+		/* Clear TWT EVENT bit mask */
+		clrbit(eventmask_msg->mask, WLC_E_TWT);
+	} else {
+	/* resume */
+		/* Set TWT EVENT bit mask */
+		setbit(eventmask_msg->mask, WLC_E_TWT);
+	}
+
+	/* Write updated Event mask */
+	eventmask_msg->ver = EVENTMSGS_VER;
+	eventmask_msg->command = EVENTMSGS_SET_MASK;
+	eventmask_msg->len = WL_EVENTING_MASK_EXT_LEN;
+
+	ret = dhd_iovar(dhdp, 0, "event_msgs_ext", (char *)eventmask_msg, msglen,
+			NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s write event mask ext failed %d\n", __FUNCTION__, ret));
+		goto fail;
+	}
+
+fail:
+	if (eventmask_msg) {
+		MFREE(dhdp->osh, eventmask_msg, msglen);
+	}
+
+	return ret;
+
+}
+#endif /* WL_TWT */
+
+static int dhd_pm_callback(struct notifier_block *nfb, unsigned long action, void *ignored)
+{
+	int ret = NOTIFY_DONE;
+	bool suspend = FALSE;
+	dhd_info_t *dhdinfo;
+	unsigned long flags = 0;
+	dhd_pub_t *dhd;
+	struct dhd_conf *conf;
+	int suspend_mode;
+#if defined(BCMDBUS) && defined(WL_EXT_WOWL)
+	int wowl_dngldown = 0;
+#endif
+
+	BCM_REFERENCE(dhdinfo);
+	BCM_REFERENCE(suspend);
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhdinfo = container_of(nfb, struct dhd_info, pm_notifier);
+	GCC_DIAGNOSTIC_POP();
+
+	if (!dhdinfo->pub.up) {
+		return ret;
+	}
+	dhd = &dhdinfo->pub;
+	conf = dhd->conf;
+	suspend_mode = conf->suspend_mode;
+
+	DHD_GENERAL_LOCK(&dhdinfo->pub, flags);
+#if defined(BCMDBUS) && defined(WL_EXT_WOWL)
+	wowl_dngldown = dhd_conf_wowl_dngldown(dhd);
+	if (!wowl_dngldown)
+#endif
+	DHD_BUS_BUSY_SET_IN_PM_CALLBACK(&dhdinfo->pub);
+	DHD_GENERAL_UNLOCK(&dhdinfo->pub, flags);
+
+	switch (action) {
+	case PM_HIBERNATION_PREPARE:
+	case PM_SUSPEND_PREPARE:
+		suspend = TRUE;
+		break;
+
+	case PM_POST_HIBERNATION:
+	case PM_POST_SUSPEND:
+		suspend = FALSE;
+		break;
+	}
+
+	if(!dhd->early_suspended && suspend_mode != PM_NOTIFIER) {
+		suspend_mode = PM_NOTIFIER;
+		conf->suspend_mode = PM_NOTIFIER;
+		conf->insuspend |= (NO_TXDATA_IN_SUSPEND | NO_TXCTL_IN_SUSPEND);
+		printf("%s: switch suspend_mode to %d\n", __FUNCTION__, suspend_mode);
+	}
+	printf("%s: action=%ld, suspend=%d, suspend_mode=%d\n",
+		__FUNCTION__, action, suspend, suspend_mode);
+	if (suspend) {
+#ifdef WL_TWT
+		dhd_config_twt_event_mask_in_suspend(&dhdinfo->pub, TRUE);
+		dhd_send_twt_info_suspend(&dhdinfo->pub, TRUE);
+#endif /* WL_TWT */
+		DHD_OS_WAKE_LOCK_WAIVE(dhd);
+		if (suspend_mode == PM_NOTIFIER)
+			dhd_suspend_resume_helper(dhdinfo, suspend, 0);
+#ifdef DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND
+		dhd_config_rts_in_suspend(&dhdinfo->pub, suspend);
+#endif /* DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND */
+#if defined(SUPPORT_P2P_GO_PS) && defined(PROP_TXSTATUS)
+		dhd_wlfc_suspend(dhd);
+#endif /* defined(SUPPORT_P2P_GO_PS) && defined(PROP_TXSTATUS) */
+		if (suspend_mode == PM_NOTIFIER || suspend_mode == SUSPEND_MODE_2)
+			dhd_conf_set_suspend_resume(dhd, suspend);
+		DHD_OS_WAKE_LOCK_RESTORE(dhd);
+	} else {
+#ifdef BCMDBUS
+		if (dhd->busstate == DHD_BUS_DATA) {
+#endif
+		if (suspend_mode == PM_NOTIFIER || suspend_mode == SUSPEND_MODE_2)
+			dhd_conf_set_suspend_resume(dhd, suspend);
+#if defined(SUPPORT_P2P_GO_PS) && defined(PROP_TXSTATUS)
+		dhd_wlfc_resume(dhd);
+#endif /* defined(SUPPORT_P2P_GO_PS) && defined(PROP_TXSTATUS) */
+#ifdef CUSTOM_EVENT_PM_WAKE
+		dhd_set_excess_pm_awake(&dhdinfo->pub, suspend);
+#endif /* CUSTOM_EVENT_PM_WAKE */
+#ifdef DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND
+		dhd_config_rts_in_suspend(&dhdinfo->pub, suspend);
+#endif /* DHD_CUSTOM_CONFIG_RTS_IN_SUSPEND */
+		if (suspend_mode == PM_NOTIFIER)
+			dhd_suspend_resume_helper(dhdinfo, suspend, 0);
+#ifdef BCMDBUS
+		} else {
+#if defined(BCMDBUS) && defined(WL_EXT_WOWL)
+			if (wowl_dngldown) {
+				printf("%s: reset power\n", __FUNCTION__);
+				dhd_wifi_platform_set_power(dhd, FALSE);
+				dhd_wifi_platform_set_power(dhd, TRUE);
+			} else
+#endif
+			{
+				printf("%s: skip resume since bus suspeneded\n", __FUNCTION__);
+			}
+		}
+#endif
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && (LINUX_VERSION_CODE <= \
+	KERNEL_VERSION(2, 6, 39))
+	dhd_mmc_suspend = suspend;
+	smp_mb();
+#endif
+
+	DHD_GENERAL_LOCK(&dhdinfo->pub, flags);
+	DHD_BUS_BUSY_CLEAR_IN_PM_CALLBACK(&dhdinfo->pub);
+	DHD_GENERAL_UNLOCK(&dhdinfo->pub, flags);
+	return ret;
+}
+
+/* to make sure we won't register the same notifier twice, otherwise a loop is likely to be
+ * created in kernel notifier link list (with 'next' pointing to itself)
+ */
+static bool dhd_pm_notifier_registered = FALSE;
+
+extern int register_pm_notifier(struct notifier_block *nb);
+extern int unregister_pm_notifier(struct notifier_block *nb);
+#endif /* CONFIG_PM_SLEEP */
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+static void
+dhd_deferred_work_rte_log_time_sync(void *handle, void *event_info, u8 event);
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+/** Clear the dhd net_device's private structure. */
+static inline void
+dhd_dev_priv_clear(struct net_device * dev)
+{
+	dhd_dev_priv_t * dev_priv;
+	ASSERT(dev != (struct net_device *)NULL);
+	dev_priv = DHD_DEV_PRIV(dev);
+	dev_priv->dhd = (dhd_info_t *)NULL;
+	dev_priv->ifp = (dhd_if_t *)NULL;
+	dev_priv->ifidx = DHD_BAD_IF;
+	dev_priv->lkup = (void *)NULL;
+}
+
+/** Setup the dhd net_device's private structure. */
+static inline void
+dhd_dev_priv_save(struct net_device * dev, dhd_info_t * dhd, dhd_if_t * ifp,
+                  int ifidx)
+{
+	dhd_dev_priv_t * dev_priv;
+	ASSERT(dev != (struct net_device *)NULL);
+	dev_priv = DHD_DEV_PRIV(dev);
+	dev_priv->dhd = dhd;
+	dev_priv->ifp = ifp;
+	dev_priv->ifidx = ifidx;
+}
+
+/* Return interface pointer */
+struct dhd_if * dhd_get_ifp(dhd_pub_t *dhdp, uint32 ifidx)
+{
+	ASSERT(ifidx < DHD_MAX_IFS);
+
+	if (!dhdp || !dhdp->info || ifidx >= DHD_MAX_IFS)
+		return NULL;
+
+	return dhdp->info->iflist[ifidx];
+}
+
+#ifdef WLEASYMESH
+int
+dhd_set_1905_almac(dhd_pub_t *dhdp, uint8 ifidx, uint8* ea, bool mcast)
+{
+	dhd_if_t *ifp;
+
+	ASSERT(ea != NULL);
+	ifp = dhd_get_ifp(dhdp, ifidx);
+	if (ifp == NULL) {
+		return BCME_ERROR;
+	}
+	if (mcast) {
+		memcpy(ifp->_1905_al_mcast, ea, ETHER_ADDR_LEN);
+	} else {
+		memcpy(ifp->_1905_al_ucast, ea, ETHER_ADDR_LEN);
+	}
+	return BCME_OK;
+}
+int
+dhd_get_1905_almac(dhd_pub_t *dhdp, uint8 ifidx, uint8* ea, bool mcast)
+{
+	dhd_if_t *ifp;
+
+	ASSERT(ea != NULL);
+	ifp = dhd_get_ifp(dhdp, ifidx);
+	if (ifp == NULL) {
+		return BCME_ERROR;
+	}
+	if (mcast) {
+		memcpy(ea, ifp->_1905_al_mcast, ETHER_ADDR_LEN);
+	} else {
+		memcpy(ea, ifp->_1905_al_ucast, ETHER_ADDR_LEN);
+	}
+	return BCME_OK;
+}
+#endif /* WLEASYMESH */
+
+#ifdef PCIE_FULL_DONGLE
+
+/** Dummy objects are defined with state representing bad|down.
+ * Performance gains from reducing branch conditionals, instruction parallelism,
+ * dual issue, reducing load shadows, avail of larger pipelines.
+ * Use DHD_XXX_NULL instead of (dhd_xxx_t *)NULL, whenever an object pointer
+ * is accessed via the dhd_sta_t.
+ */
+
+/* Dummy dhd_info object */
+dhd_info_t dhd_info_null = {
+	.pub = {
+	         .info = &dhd_info_null,
+#ifdef DHDTCPACK_SUPPRESS
+	         .tcpack_sup_mode = TCPACK_SUP_REPLACE,
+#endif /* DHDTCPACK_SUPPRESS */
+#if defined(BCM_ROUTER_DHD)
+	         .dhd_tm_dwm_tbl = { .dhd_dwm_enabled = TRUE },
+#endif
+	         .up = FALSE,
+	         .busstate = DHD_BUS_DOWN
+	}
+};
+#define DHD_INFO_NULL (&dhd_info_null)
+#define DHD_PUB_NULL  (&dhd_info_null.pub)
+
+/* Dummy netdevice object */
+struct net_device dhd_net_dev_null = {
+	.reg_state = NETREG_UNREGISTERED
+};
+#define DHD_NET_DEV_NULL (&dhd_net_dev_null)
+
+/* Dummy dhd_if object */
+dhd_if_t dhd_if_null = {
+#ifdef WMF
+	.wmf = { .wmf_enable = TRUE },
+#endif
+	.info = DHD_INFO_NULL,
+	.net = DHD_NET_DEV_NULL,
+	.idx = DHD_BAD_IF
+};
+#define DHD_IF_NULL  (&dhd_if_null)
+
+/* XXX should we use the sta_pool[0] object as DHD_STA_NULL? */
+#define DHD_STA_NULL ((dhd_sta_t *)NULL)
+
+/** Interface STA list management. */
+
+/** Alloc/Free a dhd_sta object from the dhd instances' sta_pool. */
+static void dhd_sta_free(dhd_pub_t *pub, dhd_sta_t *sta);
+static dhd_sta_t * dhd_sta_alloc(dhd_pub_t * dhdp);
+
+/* Delete a dhd_sta or flush all dhd_sta in an interface's sta_list. */
+static void dhd_if_del_sta_list(dhd_if_t * ifp);
+
+/* Construct/Destruct a sta pool. */
+static int dhd_sta_pool_init(dhd_pub_t *dhdp, int max_sta);
+static void dhd_sta_pool_fini(dhd_pub_t *dhdp, int max_sta);
+/* Clear the pool of dhd_sta_t objects for built-in type driver */
+static void dhd_sta_pool_clear(dhd_pub_t *dhdp, int max_sta);
+
+/** Reset a dhd_sta object and free into the dhd pool. */
+static void
+dhd_sta_free(dhd_pub_t * dhdp, dhd_sta_t * sta)
+{
+	int prio;
+
+	ASSERT((sta != DHD_STA_NULL) && (sta->idx != ID16_INVALID));
+
+	ASSERT((dhdp->staid_allocator != NULL) && (dhdp->sta_pool != NULL));
+
+	/*
+	 * Flush and free all packets in all flowring's queues belonging to sta.
+	 * Packets in flow ring will be flushed later.
+	 */
+	for (prio = 0; prio < (int)NUMPRIO; prio++) {
+		uint16 flowid = sta->flowid[prio];
+
+		if (flowid != FLOWID_INVALID) {
+			unsigned long flags;
+			flow_ring_node_t * flow_ring_node;
+
+#ifdef DHDTCPACK_SUPPRESS
+			/* Clean tcp_ack_info_tbl in order to prevent access to flushed pkt,
+			 * when there is a newly coming packet from network stack.
+			 */
+			dhd_tcpack_info_tbl_clean(dhdp);
+#endif /* DHDTCPACK_SUPPRESS */
+
+			flow_ring_node = dhd_flow_ring_node(dhdp, flowid);
+			if (flow_ring_node) {
+				flow_queue_t *queue = &flow_ring_node->queue;
+
+				DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+				flow_ring_node->status = FLOW_RING_STATUS_STA_FREEING;
+
+				if (!DHD_FLOW_QUEUE_EMPTY(queue)) {
+					void * pkt;
+					while ((pkt = dhd_flow_queue_dequeue(dhdp, queue)) !=
+						NULL) {
+						PKTFREE(dhdp->osh, pkt, TRUE);
+					}
+				}
+
+				DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+				ASSERT(DHD_FLOW_QUEUE_EMPTY(queue));
+			}
+		}
+
+		sta->flowid[prio] = FLOWID_INVALID;
+	}
+
+	id16_map_free(dhdp->staid_allocator, sta->idx);
+	DHD_CUMM_CTR_INIT(&sta->cumm_ctr);
+	sta->ifp = DHD_IF_NULL; /* dummy dhd_if object */
+	sta->ifidx = DHD_BAD_IF;
+	sta->chanspec = 0;
+	bzero(sta->ea.octet, ETHER_ADDR_LEN);
+	INIT_LIST_HEAD(&sta->list);
+	sta->idx = ID16_INVALID; /* implying free */
+}
+
+/** Allocate a dhd_sta object from the dhd pool. */
+static dhd_sta_t *
+dhd_sta_alloc(dhd_pub_t * dhdp)
+{
+	uint16 idx;
+	dhd_sta_t * sta;
+	dhd_sta_pool_t * sta_pool;
+
+	ASSERT((dhdp->staid_allocator != NULL) && (dhdp->sta_pool != NULL));
+
+	idx = id16_map_alloc(dhdp->staid_allocator);
+	if (idx == ID16_INVALID) {
+		DHD_ERROR(("%s: cannot get free staid\n", __FUNCTION__));
+		return DHD_STA_NULL;
+	}
+
+	sta_pool = (dhd_sta_pool_t *)(dhdp->sta_pool);
+	sta = &sta_pool[idx];
+
+	ASSERT((sta->idx == ID16_INVALID) &&
+	       (sta->ifp == DHD_IF_NULL) && (sta->ifidx == DHD_BAD_IF));
+
+	DHD_CUMM_CTR_INIT(&sta->cumm_ctr);
+
+	sta->idx = idx; /* implying allocated */
+
+	return sta;
+}
+
+/** Delete all STAs in an interface's STA list. */
+static void
+dhd_if_del_sta_list(dhd_if_t *ifp)
+{
+	dhd_sta_t *sta, *next;
+	unsigned long flags;
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry_safe(sta, next, &ifp->sta_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		list_del(&sta->list);
+		dhd_sta_free(&ifp->info->pub, sta);
+	}
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return;
+}
+
+/** Construct a pool of dhd_sta_t objects to be used by interfaces. */
+static int
+dhd_sta_pool_init(dhd_pub_t *dhdp, int max_sta)
+{
+	int idx, prio, sta_pool_memsz;
+	dhd_sta_t * sta;
+	dhd_sta_pool_t * sta_pool;
+	void * staid_allocator;
+
+	ASSERT(dhdp != (dhd_pub_t *)NULL);
+	ASSERT((dhdp->staid_allocator == NULL) && (dhdp->sta_pool == NULL));
+
+	/* dhd_sta objects per radio are managed in a table. id#0 reserved. */
+	staid_allocator = id16_map_init(dhdp->osh, max_sta, 1);
+	if (staid_allocator == NULL) {
+		DHD_ERROR(("%s: sta id allocator init failure\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Pre allocate a pool of dhd_sta objects (one extra). */
+	sta_pool_memsz = ((max_sta + 1) * sizeof(dhd_sta_t)); /* skip idx 0 */
+	sta_pool = (dhd_sta_pool_t *)MALLOC(dhdp->osh, sta_pool_memsz);
+	if (sta_pool == NULL) {
+		DHD_ERROR(("%s: sta table alloc failure\n", __FUNCTION__));
+		id16_map_fini(dhdp->osh, staid_allocator);
+		return BCME_ERROR;
+	}
+
+	dhdp->sta_pool = sta_pool;
+	dhdp->staid_allocator = staid_allocator;
+
+	/* Initialize all sta(s) for the pre-allocated free pool. */
+	bzero((uchar *)sta_pool, sta_pool_memsz);
+	for (idx = max_sta; idx >= 1; idx--) { /* skip sta_pool[0] */
+		sta = &sta_pool[idx];
+		sta->idx = id16_map_alloc(staid_allocator);
+		ASSERT(sta->idx <= max_sta);
+	}
+
+	/* Now place them into the pre-allocated free pool. */
+	for (idx = 1; idx <= max_sta; idx++) {
+		sta = &sta_pool[idx];
+		for (prio = 0; prio < (int)NUMPRIO; prio++) {
+			sta->flowid[prio] = FLOWID_INVALID; /* Flow rings do not exist */
+		}
+		dhd_sta_free(dhdp, sta);
+	}
+
+	return BCME_OK;
+}
+
+/** Destruct the pool of dhd_sta_t objects.
+ * Caller must ensure that no STA objects are currently associated with an if.
+ */
+static void
+dhd_sta_pool_fini(dhd_pub_t *dhdp, int max_sta)
+{
+	dhd_sta_pool_t * sta_pool = (dhd_sta_pool_t *)dhdp->sta_pool;
+
+	if (sta_pool) {
+		int idx;
+		int sta_pool_memsz = ((max_sta + 1) * sizeof(dhd_sta_t));
+		for (idx = 1; idx <= max_sta; idx++) {
+			ASSERT(sta_pool[idx].ifp == DHD_IF_NULL);
+			ASSERT(sta_pool[idx].idx == ID16_INVALID);
+		}
+		MFREE(dhdp->osh, dhdp->sta_pool, sta_pool_memsz);
+	}
+
+	id16_map_fini(dhdp->osh, dhdp->staid_allocator);
+	dhdp->staid_allocator = NULL;
+}
+
+/* Clear the pool of dhd_sta_t objects for built-in type driver */
+static void
+dhd_sta_pool_clear(dhd_pub_t *dhdp, int max_sta)
+{
+	int idx, prio, sta_pool_memsz;
+	dhd_sta_t * sta;
+	dhd_sta_pool_t * sta_pool;
+	void *staid_allocator;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	sta_pool = (dhd_sta_pool_t *)dhdp->sta_pool;
+	staid_allocator = dhdp->staid_allocator;
+
+	if (!sta_pool) {
+		DHD_ERROR(("%s: sta_pool is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (!staid_allocator) {
+		DHD_ERROR(("%s: staid_allocator is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	/* clear free pool */
+	sta_pool_memsz = ((max_sta + 1) * sizeof(dhd_sta_t));
+	bzero((uchar *)sta_pool, sta_pool_memsz);
+
+	/* dhd_sta objects per radio are managed in a table. id#0 reserved. */
+	id16_map_clear(staid_allocator, max_sta, 1);
+
+	/* Initialize all sta(s) for the pre-allocated free pool. */
+	for (idx = max_sta; idx >= 1; idx--) { /* skip sta_pool[0] */
+		sta = &sta_pool[idx];
+		sta->idx = id16_map_alloc(staid_allocator);
+		ASSERT(sta->idx <= max_sta);
+	}
+	/* Now place them into the pre-allocated free pool. */
+	for (idx = 1; idx <= max_sta; idx++) {
+		sta = &sta_pool[idx];
+		for (prio = 0; prio < (int)NUMPRIO; prio++) {
+			sta->flowid[prio] = FLOWID_INVALID; /* Flow rings do not exist */
+		}
+		dhd_sta_free(dhdp, sta);
+	}
+}
+
+/*
+ * Lockless variant of dhd_find_sta()
+ * Find STA with MAC address ea in an interface's STA list.
+ */
+dhd_sta_t *
+__dhd_find_sta(dhd_if_t *ifp, void *pub, int ifidx, void *ea)
+{
+	dhd_sta_t *sta;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry(sta, &ifp->sta_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		if (!memcmp(sta->ea.octet, ea, ETHER_ADDR_LEN)) {
+			DHD_INFO(("%s: Found STA " MACDBG "\n",
+				__FUNCTION__, MAC2STRDBG((char *)ea)));
+			return sta;
+		}
+	}
+
+	return DHD_STA_NULL;
+}
+
+/** Find STA with MAC address ea in an interface's STA list. */
+dhd_sta_t *
+dhd_find_sta(void *pub, int ifidx, void *ea)
+{
+	dhd_sta_t *sta;
+	dhd_if_t *ifp;
+	unsigned long flags;
+
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL)
+		return DHD_STA_NULL;
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+
+	sta = __dhd_find_sta(ifp, pub, ifidx, ea);
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return sta;
+}
+
+/*
+ * Lockless variant of dhd_add_sta()
+ * Add STA into the interface's STA list.
+ */
+dhd_sta_t *
+__dhd_add_sta(dhd_if_t *ifp, void *pub, int ifidx, void *ea)
+{
+	dhd_sta_t *sta;
+
+	if (!memcmp(ifp->net->dev_addr, ea, ETHER_ADDR_LEN)) {
+		DHD_ERROR(("%s: Serious FAILURE, receive own MAC %pM !!\n", __FUNCTION__, ea));
+		return DHD_STA_NULL;
+	}
+
+	sta = dhd_sta_alloc((dhd_pub_t *)pub);
+	if (sta == DHD_STA_NULL) {
+		DHD_ERROR(("%s: Alloc failed\n", __FUNCTION__));
+		return DHD_STA_NULL;
+	}
+
+	memcpy(sta->ea.octet, ea, ETHER_ADDR_LEN);
+
+	/* link the sta and the dhd interface */
+	sta->ifp = ifp;
+	sta->ifidx = ifidx;
+#ifdef DHD_WMF
+	sta->psta_prim = NULL;
+#endif
+	INIT_LIST_HEAD(&sta->list);
+
+	list_add_tail(&sta->list, &ifp->sta_list);
+
+	DHD_ERROR(("%s: Adding  STA " MACDBG "\n",
+		__FUNCTION__, MAC2STRDBG((char *)ea)));
+
+	return sta;
+}
+
+/** Add STA into the interface's STA list. */
+dhd_sta_t *
+dhd_add_sta(void *pub, int ifidx, void *ea)
+{
+	dhd_sta_t *sta;
+	dhd_if_t *ifp;
+	unsigned long flags;
+
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL)
+		return DHD_STA_NULL;
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+
+	sta = __dhd_add_sta(ifp, pub, ifidx, ea);
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return sta;
+}
+
+/** Delete all STAs from the interface's STA list. */
+void
+dhd_del_all_sta(void *pub, int ifidx)
+{
+	dhd_sta_t *sta, *next;
+	dhd_if_t *ifp;
+	unsigned long flags;
+
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL)
+		return;
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry_safe(sta, next, &ifp->sta_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		list_del(&sta->list);
+		dhd_sta_free(&ifp->info->pub, sta);
+#ifdef DHD_L2_FILTER
+		if (ifp->parp_enable) {
+			/* clear Proxy ARP cache of specific Ethernet Address */
+			bcm_l2_filter_arp_table_update(((dhd_pub_t*)pub)->osh,
+					ifp->phnd_arp_table, FALSE,
+					sta->ea.octet, FALSE, ((dhd_pub_t*)pub)->tickcnt);
+		}
+#endif /* DHD_L2_FILTER */
+	}
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return;
+}
+
+/** Delete STA from the interface's STA list. */
+void
+dhd_del_sta(void *pub, int ifidx, void *ea)
+{
+	dhd_sta_t *sta, *next;
+	dhd_if_t *ifp;
+	unsigned long flags;
+
+	ASSERT(ea != NULL);
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL)
+		return;
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry_safe(sta, next, &ifp->sta_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		if (!memcmp(sta->ea.octet, ea, ETHER_ADDR_LEN)) {
+			DHD_ERROR(("%s: Deleting STA " MACDBG "\n",
+				__FUNCTION__, MAC2STRDBG(sta->ea.octet)));
+			list_del(&sta->list);
+			dhd_sta_free(&ifp->info->pub, sta);
+		}
+	}
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+#ifdef DHD_L2_FILTER
+	if (ifp->parp_enable) {
+		/* clear Proxy ARP cache of specific Ethernet Address */
+		bcm_l2_filter_arp_table_update(((dhd_pub_t*)pub)->osh, ifp->phnd_arp_table, FALSE,
+			ea, FALSE, ((dhd_pub_t*)pub)->tickcnt);
+	}
+#endif /* DHD_L2_FILTER */
+	return;
+}
+
+/** Add STA if it doesn't exist. Not reentrant. */
+dhd_sta_t*
+dhd_findadd_sta(void *pub, int ifidx, void *ea)
+{
+	dhd_sta_t *sta;
+	dhd_if_t *ifp;
+	unsigned long flags;
+
+	ASSERT(ea != NULL);
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL)
+		return DHD_STA_NULL;
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+	sta = __dhd_find_sta(ifp, pub, ifidx, ea);
+
+	if (!sta) {
+		/* Add entry */
+		sta = __dhd_add_sta(ifp, pub, ifidx, ea);
+	}
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return sta;
+}
+
+void
+dhd_update_sta_chanspec_info(void *pub, int ifidx, const uint8 *ea, chanspec_t chanspec)
+{
+	dhd_sta_t *sta;
+	dhd_if_t *ifp;
+	unsigned long flags;
+
+	ASSERT(ea != NULL);
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL) {
+		DHD_ERROR(("%s: NULL ifp for ifidx:%d\n", __FUNCTION__, ifidx));
+		return;
+	}
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	sta = __dhd_find_sta(ifp, pub, ifidx, (void *)ea);
+	GCC_DIAGNOSTIC_POP();
+
+	if (sta) {
+		sta->chanspec = chanspec;
+		DHD_ERROR(("%s: updated chanspec:0x%x for STA:" MACDBG "\n",
+			__FUNCTION__, sta->chanspec, MAC2STRDBG(sta->ea.octet)));
+	} else {
+		DHD_ERROR(("%s: found no STA:" MACDBG "\n",
+			__FUNCTION__, MAC2STRDBG(ea)));
+	}
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+}
+
+bool
+dhd_is_sta_htput(void *pub, int ifidx, void *ea)
+{
+#ifdef WL_CFG80211
+	dhd_sta_t *sta;
+	dhd_if_t *ifp;
+	unsigned long flags;
+	bool htput = FALSE;
+
+	/* For non-AP role, chanspec will not be updated and always use htput */
+	if (!DHD_IF_ROLE_AP((dhd_pub_t *)pub, ifidx)) {
+		DHD_ERROR(("%s: non-AP role, set htput\n", __FUNCTION__));
+		htput = TRUE;
+		return htput;
+	}
+
+	ASSERT(ea != NULL);
+	ifp = dhd_get_ifp((dhd_pub_t *)pub, ifidx);
+	if (ifp == NULL) {
+		DHD_ERROR(("%s: NULL ifp for ifidx:%d\n", __FUNCTION__, ifidx));
+		return htput;
+	}
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+	sta = __dhd_find_sta(ifp, pub, ifidx, ea);
+	/* Set htput for 160 Mhz */
+	if (sta) {
+		if (CHSPEC_IS160(sta->chanspec)) {
+			htput = TRUE;
+		}
+		DHD_ERROR(("%s: ifidx:%d htput:%d chanspec:0x%x of STA:" MACDBG "\n",
+			__FUNCTION__, ifidx, htput, sta->chanspec, MAC2STRDBG(sta->ea.octet)));
+	} else {
+		DHD_ERROR(("%s: found no STA " MACDBG "\n",
+			__FUNCTION__, MAC2STRDBG(ea)));
+	}
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return htput;
+#else
+	/* For Non-CFG builds, always create HTPUT flowrings ,
+	 * as dhd_update_sta_chanspec_info will not be called.
+	*/
+	return TRUE;
+#endif /* WL_CFG80211 */
+}
+
+#if defined(DHD_IGMP_UCQUERY) || defined(DHD_UCAST_UPNP)
+static struct list_head *
+dhd_sta_list_snapshot(dhd_info_t *dhd, dhd_if_t *ifp, struct list_head *snapshot_list)
+{
+	unsigned long flags;
+	dhd_sta_t *sta, *snapshot;
+
+	INIT_LIST_HEAD(snapshot_list);
+
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+
+	list_for_each_entry(sta, &ifp->sta_list, list) {
+		/* allocate one and add to snapshot */
+		snapshot = (dhd_sta_t *)MALLOC(dhd->pub.osh, sizeof(dhd_sta_t));
+		if (snapshot == NULL) {
+			DHD_ERROR(("%s: Cannot allocate memory\n", __FUNCTION__));
+			continue;
+		}
+
+		memcpy(snapshot->ea.octet, sta->ea.octet, ETHER_ADDR_LEN);
+
+		INIT_LIST_HEAD(&snapshot->list);
+		list_add_tail(&snapshot->list, snapshot_list);
+	}
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+
+	return snapshot_list;
+}
+
+static void
+dhd_sta_list_snapshot_free(dhd_info_t *dhd, struct list_head *snapshot_list)
+{
+	dhd_sta_t *sta, *next;
+
+	list_for_each_entry_safe(sta, next, snapshot_list, list) {
+		list_del(&sta->list);
+		MFREE(dhd->pub.osh, sta, sizeof(dhd_sta_t));
+	}
+}
+#endif /* DHD_IGMP_UCQUERY || DHD_UCAST_UPNP */
+
+#else
+static inline void dhd_if_del_sta_list(dhd_if_t *ifp) {}
+static inline int dhd_sta_pool_init(dhd_pub_t *dhdp, int max_sta) { return BCME_OK; }
+static inline void dhd_sta_pool_fini(dhd_pub_t *dhdp, int max_sta) {}
+static inline void dhd_sta_pool_clear(dhd_pub_t *dhdp, int max_sta) {}
+dhd_sta_t *dhd_findadd_sta(void *pub, int ifidx, void *ea) { return NULL; }
+dhd_sta_t *dhd_find_sta(void *pub, int ifidx, void *ea) { return NULL; }
+void dhd_del_sta(void *pub, int ifidx, void *ea) {}
+void dhd_update_sta_chanspec_info(void *pub, int ifidx, const uint8 *ea,
+	chanspec_t chanspec) {}
+bool dhd_is_sta_htput(void *pub, int ifidx, void *ea) { return FALSE; }
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef BCM_ROUTER_DHD
+/** Bind a flowid to the dhd_sta's flowid table. */
+void
+dhd_add_flowid(dhd_pub_t * dhdp, int ifidx, uint8 ac_prio, void * ea,
+                uint16 flowid)
+{
+	int prio;
+	dhd_if_t * ifp;
+	dhd_sta_t * sta;
+	flow_queue_t * queue;
+
+	ASSERT((dhdp != (dhd_pub_t *)NULL) && (ea != NULL));
+
+	/* Fetch the dhd_if object given the if index */
+	ifp = dhd_get_ifp(dhdp, ifidx);
+	if (ifp == (dhd_if_t *)NULL) /* ifp fetched from dhdp iflist[] */
+		return;
+
+	/* Intializing the backup queue parameters */
+	if (DHD_IF_ROLE_WDS(dhdp, ifidx) ||
+#ifdef DHD_WET
+		WET_ENABLED(dhdp) ||
+#endif /* DHD_WET */
+		0) {
+		queue = dhd_flow_queue(dhdp, flowid);
+		dhd_flow_ring_config_thresholds(dhdp, flowid,
+			dhd_queue_budget, queue->max, DHD_FLOW_QUEUE_CLEN_PTR(queue),
+			dhd_if_threshold, (void *)&ifp->cumm_ctr);
+		return;
+	} else if ((sta = dhd_find_sta(dhdp, ifidx, ea)) == DHD_STA_NULL) {
+		/* Fetch the station with a matching Mac address. */
+		/* Update queue's grandparent cummulative length threshold */
+		if (ETHER_ISMULTI((char *)ea)) {
+			queue = dhd_flow_queue(dhdp, flowid);
+			if (ifidx != 0 && DHD_IF_ROLE_STA(dhdp, ifidx)) {
+				/* Use default dhdp->cumm_ctr and dhdp->l2cumm_ctr,
+				 * in PSTA mode the ifp will be deleted but we don't delete
+				 * the PSTA flowring.
+				 */
+				dhd_flow_ring_config_thresholds(dhdp, flowid,
+					queue->max, queue->max, DHD_FLOW_QUEUE_CLEN_PTR(queue),
+					dhd_if_threshold, DHD_FLOW_QUEUE_L2CLEN_PTR(queue));
+			}
+			else if (DHD_FLOW_QUEUE_L2CLEN_PTR(queue) != (void *)&ifp->cumm_ctr) {
+				dhd_flow_ring_config_thresholds(dhdp, flowid,
+					queue->max, queue->max, DHD_FLOW_QUEUE_CLEN_PTR(queue),
+					dhd_if_threshold, (void *)&ifp->cumm_ctr);
+			}
+		}
+		return;
+	}
+
+	/* Set queue's min budget and queue's parent cummulative length threshold */
+	dhd_flow_ring_config_thresholds(dhdp, flowid, dhd_queue_budget,
+	                                dhd_sta_threshold, (void *)&sta->cumm_ctr,
+	                                dhd_if_threshold, (void *)&ifp->cumm_ctr);
+
+	/* Populate the flowid into the stations flowid table, for all packet
+	 * priorities that would match the given flow's ac priority.
+	 */
+	for (prio = 0; prio < (int)NUMPRIO; prio++) {
+		if (dhdp->flow_prio_map[prio] == ac_prio) {
+			/* flowring shared for all these pkt prio */
+			sta->flowid[prio] = flowid;
+		}
+	}
+}
+
+/** Unbind a flowid to the sta's flowid table. */
+void
+dhd_del_flowid(dhd_pub_t * dhdp, int ifidx, uint16 flowid)
+{
+	int prio;
+	dhd_if_t * ifp;
+	dhd_sta_t * sta;
+	unsigned long flags;
+
+	/* Fetch the dhd_if object given the if index */
+	ifp = dhd_get_ifp(dhdp, ifidx);
+	if (ifp == (dhd_if_t *)NULL) /* ifp fetched from dhdp iflist[] */
+		return;
+
+	/* Walk all stations and delete clear any station's reference to flowid */
+	DHD_IF_STA_LIST_LOCK(&ifp->sta_list_lock, flags);
+
+	list_for_each_entry(sta, &ifp->sta_list, list) {
+		for (prio = 0; prio < (int)NUMPRIO; prio++) {
+			if (sta->flowid[prio] == flowid) {
+				sta->flowid[prio] = FLOWID_INVALID;
+			}
+		}
+	}
+
+	DHD_IF_STA_LIST_UNLOCK(&ifp->sta_list_lock, flags);
+}
+#endif /* BCM_ROUTER_DHD */
+
+#if defined(DNGL_AXI_ERROR_LOGGING) && defined(DHD_USE_WQ_FOR_DNGL_AXI_ERROR)
+void
+dhd_axi_error_dispatch(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	schedule_work(&dhd->axi_error_dispatcher_work);
+}
+
+static void dhd_axi_error_dispatcher_fn(struct work_struct * work)
+{
+	struct dhd_info *dhd =
+		container_of(work, struct dhd_info, axi_error_dispatcher_work);
+	dhd_axi_error(&dhd->pub);
+}
+#endif /* DNGL_AXI_ERROR_LOGGING && DHD_USE_WQ_FOR_DNGL_AXI_ERROR */
+
+/** Returns dhd iflist index corresponding the the bssidx provided by apps */
+int dhd_bssidx2idx(dhd_pub_t *dhdp, uint32 bssidx)
+{
+	dhd_if_t *ifp;
+	dhd_info_t *dhd = dhdp->info;
+	int i;
+
+	ASSERT(bssidx < DHD_MAX_IFS);
+	ASSERT(dhdp);
+
+	for (i = 0; i < DHD_MAX_IFS; i++) {
+		ifp = dhd->iflist[i];
+		if (ifp && (ifp->bssidx == bssidx)) {
+			DHD_TRACE(("Index manipulated for %s from %d to %d\n",
+				ifp->name, bssidx, i));
+			break;
+		}
+	}
+	return i;
+}
+
+int dhd_process_cid_mac(dhd_pub_t *dhdp, bool prepost)
+{
+	uint chipid = 0;
+	int ret = BCME_OK;
+#ifndef BCMDBUS
+	chipid = dhd_bus_chip_id(dhdp);
+#endif /* BCMDBUS */
+	if (prepost) { /* pre process */
+		ret = dhd_alloc_cis(dhdp);
+		if (ret != BCME_OK) {
+			return ret;
+		}
+		switch (chipid) {
+#ifndef DHD_READ_CIS_FROM_BP
+			case BCM4389_CHIP_GRPID:
+				/* BCM4389B0 or higher rev is used new otp iovar */
+				dhd_read_otp_sw_rgn(dhdp);
+				break;
+#endif /* !DHD_READ_CIS_FROM_BP */
+			default:
+				dhd_read_cis(dhdp);
+				break;
+		}
+		dhd_check_module_cid(dhdp);
+		dhd_check_module_mac(dhdp);
+		dhd_set_macaddr_from_file(dhdp);
+	} else { /* post process */
+		dhd_write_macaddr(&dhdp->mac);
+		dhd_clear_cis(dhdp);
+	}
+
+	return BCME_OK;
+}
+
+// terence 20160615: fix building error if ARP_OFFLOAD_SUPPORT removed
+#if defined(PKT_FILTER_SUPPORT)
+#if defined(ARP_OFFLOAD_SUPPORT) && !defined(GAN_LITE_NAT_KEEPALIVE_FILTER)
+static bool
+_turn_on_arp_filter(dhd_pub_t *dhd, int op_mode_param)
+{
+	bool _apply = FALSE;
+	/* In case of IBSS mode, apply arp pkt filter */
+	if (op_mode_param & DHD_FLAG_IBSS_MODE) {
+		_apply = TRUE;
+		goto exit;
+	}
+	/* In case of P2P GO or GC, apply pkt filter to pass arp pkt to host */
+	if (op_mode_param & (DHD_FLAG_P2P_GC_MODE | DHD_FLAG_P2P_GO_MODE)) {
+		_apply = TRUE;
+		goto exit;
+	}
+
+exit:
+	return _apply;
+}
+#endif /* !GAN_LITE_NAT_KEEPALIVE_FILTER */
+
+void
+dhd_set_packet_filter(dhd_pub_t *dhd)
+{
+	int i;
+
+	DHD_TRACE(("%s: enter\n", __FUNCTION__));
+	if (dhd_pkt_filter_enable) {
+		for (i = 0; i < dhd->pktfilter_count; i++) {
+			dhd_pktfilter_offload_set(dhd, dhd->pktfilter[i]);
+		}
+	}
+}
+
+void
+dhd_enable_packet_filter(int value, dhd_pub_t *dhd)
+{
+	int i;
+
+	DHD_ERROR(("%s: enter, value = %d\n", __FUNCTION__, value));
+	if ((dhd->op_mode & (DHD_FLAG_HOSTAP_MODE | DHD_FLAG_P2P_GO_MODE)) && value &&
+			!dhd_conf_get_insuspend(dhd, AP_FILTER_IN_SUSPEND)) {
+		DHD_ERROR(("%s: DHD_FLAG_HOSTAP_MODE or DHD_FLAG_P2P_GO_MODE\n", __FUNCTION__));
+		return;
+	}
+	if (value)
+		dhd_conf_add_pkt_filter(dhd);
+	/* 1 - Enable packet filter, only allow unicast packet to send up */
+	/* 0 - Disable packet filter */
+	if (dhd_pkt_filter_enable && (!value ||
+	    (dhd_support_sta_mode(dhd) && !dhd->dhcp_in_progress) ||
+	    dhd_conf_get_insuspend(dhd, AP_FILTER_IN_SUSPEND)))
+	{
+		for (i = 0; i < dhd->pktfilter_count; i++) {
+// terence 20160615: fix building error if ARP_OFFLOAD_SUPPORT removed
+#if defined(ARP_OFFLOAD_SUPPORT) && !defined(GAN_LITE_NAT_KEEPALIVE_FILTER)
+			if (value && (i == DHD_ARP_FILTER_NUM) &&
+				!_turn_on_arp_filter(dhd, dhd->op_mode)) {
+				DHD_TRACE(("Do not turn on ARP white list pkt filter:"
+					"val %d, cnt %d, op_mode 0x%x\n",
+					value, i, dhd->op_mode));
+				continue;
+			}
+#endif /* !GAN_LITE_NAT_KEEPALIVE_FILTER */
+#ifdef APSTA_BLOCK_ARP_DURING_DHCP
+			if (value && (i == DHD_BROADCAST_ARP_FILTER_NUM) &&
+				dhd->pktfilter[DHD_BROADCAST_ARP_FILTER_NUM]) {
+				/* XXX: BROADCAST_ARP_FILTER is only for the
+				 * STA/SoftAP concurrent mode (Please refer to RB:90348)
+				 * Remove the filter for other cases explicitly
+				 */
+				DHD_ERROR(("%s: Remove the DHD_BROADCAST_ARP_FILTER\n",
+					__FUNCTION__));
+				dhd_packet_filter_add_remove(dhd, FALSE,
+					DHD_BROADCAST_ARP_FILTER_NUM);
+			}
+#endif /* APSTA_BLOCK_ARP_DURING_DHCP */
+			dhd_pktfilter_offload_enable(dhd, dhd->pktfilter[i],
+				value, dhd_master_mode);
+		}
+	}
+}
+
+int
+dhd_packet_filter_add_remove(dhd_pub_t *dhdp, int add_remove, int num)
+{
+	char *filterp = NULL;
+	int filter_id = 0;
+
+	switch (num) {
+		case DHD_BROADCAST_FILTER_NUM:
+			filterp = "101 0 0 0 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF";
+			filter_id = 101;
+			break;
+		case DHD_MULTICAST4_FILTER_NUM:
+			filter_id = 102;
+			if (FW_SUPPORTED((dhdp), pf6)) {
+				if (dhdp->pktfilter[num] != NULL) {
+					dhd_pktfilter_offload_delete(dhdp, filter_id);
+					dhdp->pktfilter[num] = NULL;
+				}
+				if (!add_remove) {
+					filterp = DISCARD_IPV4_MCAST;
+					add_remove = 1;
+					break;
+				}
+			} /* XXX: intend omitting else case */
+			filterp = "102 0 0 0 0xFFFFFF 0x01005E";
+			break;
+		case DHD_MULTICAST6_FILTER_NUM:
+			filter_id = 103;
+			if (FW_SUPPORTED((dhdp), pf6)) {
+				if (dhdp->pktfilter[num] != NULL) {
+					dhd_pktfilter_offload_delete(dhdp, filter_id);
+					dhdp->pktfilter[num] = NULL;
+				}
+				if (!add_remove) {
+					filterp = DISCARD_IPV6_MCAST;
+					add_remove = 1;
+					break;
+				}
+			} /* XXX: intend omitting else case */
+			filterp = "103 0 0 0 0xFFFF 0x3333";
+			break;
+		case DHD_MDNS_FILTER_NUM:
+			filterp = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB";
+			filter_id = 104;
+			break;
+		case DHD_ARP_FILTER_NUM:
+			filterp = "105 0 0 12 0xFFFF 0x0806";
+			filter_id = 105;
+			break;
+		case DHD_BROADCAST_ARP_FILTER_NUM:
+			filterp = "106 0 0 0 0xFFFFFFFFFFFF0000000000000806"
+				" 0xFFFFFFFFFFFF0000000000000806";
+			filter_id = 106;
+			break;
+		default:
+			return -EINVAL;
+	}
+
+	/* Add filter */
+	if (add_remove) {
+		dhdp->pktfilter[num] = filterp;
+		dhd_pktfilter_offload_set(dhdp, dhdp->pktfilter[num]);
+	} else { /* Delete filter */
+		if (dhdp->pktfilter[num] != NULL) {
+			dhd_pktfilter_offload_delete(dhdp, filter_id);
+			dhdp->pktfilter[num] = NULL;
+		}
+	}
+
+	return 0;
+}
+#endif /* PKT_FILTER_SUPPORT */
+
+static int dhd_set_suspend(int value, dhd_pub_t *dhd)
+{
+	/* wl_pkt_filter_enable_t	enable_parm; */
+	int ret = 0;
+#if defined(DHD_USE_EARLYSUSPEND) || defined(DHD_USE_PM_SLEEP)
+	uint nd_ra_filter = 0;
+#endif /* DHD_USE_EARLYSUSPEND || DHD_USE_PM_SLEEP */
+#ifdef PASS_ALL_MCAST_PKTS
+	struct dhd_info *dhdinfo;
+	uint32 allmulti;
+	uint i;
+#endif /* PASS_ALL_MCAST_PKTS */
+#ifdef DYNAMIC_SWOOB_DURATION
+#ifndef CUSTOM_INTR_WIDTH
+#define CUSTOM_INTR_WIDTH 100
+	int intr_width = 0;
+#endif /* CUSTOM_INTR_WIDTH */
+#endif /* DYNAMIC_SWOOB_DURATION */
+#ifdef WL_CFG80211
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+#endif /* WL_CFG80211 */
+#if defined(DHD_BCN_TIMEOUT_IN_SUSPEND) && (defined(DHD_USE_EARLYSUSPEND) || \
+	defined(DHD_USE_PM_SLEEP))
+	/* CUSTOM_BCN_TIMEOUT_IN_SUSPEND in suspend, otherwise CUSTOM_BCN_TIMEOUT */
+	int bcn_timeout = CUSTOM_BCN_TIMEOUT;
+#endif /* DHD_BCN_TIMEOUT_IN_SUSPEND && (DHD_USE_EARLYSUSPEND || DHD_USE_PM_SLEEP) */
+
+	BCM_REFERENCE(ret);
+	if (!dhd)
+		return -ENODEV;
+
+#ifdef PASS_ALL_MCAST_PKTS
+	dhdinfo = dhd->info;
+#endif /* PASS_ALL_MCAST_PKTS */
+
+	DHD_TRACE(("%s: enter, value = %d in_suspend=%d\n",
+		__FUNCTION__, value, dhd->in_suspend));
+
+	dhd_suspend_lock(dhd);
+
+#ifdef CUSTOM_SET_CPUCORE
+	DHD_TRACE(("%s set cpucore(suspend%d)\n", __FUNCTION__, value));
+	/* set specific cpucore */
+	dhd_set_cpucore(dhd, TRUE);
+#endif /* CUSTOM_SET_CPUCORE */
+	if (dhd->up) {
+		if (value && dhd->in_suspend) {
+				dhd->early_suspended = 1;
+				/* Kernel suspended */
+				DHD_ERROR(("%s: force extra Suspend setting \n", __FUNCTION__));
+
+#ifdef PKT_FILTER_SUPPORT
+				/* Enable packet filter,
+				 * only allow unicast packet to send up
+				 */
+				dhd_enable_packet_filter(1, dhd);
+#endif /* PKT_FILTER_SUPPORT */
+#ifdef APF
+				dhd_dev_apf_enable_filter(dhd_linux_get_primary_netdev(dhd));
+#endif /* APF */
+#ifdef ARP_OFFLOAD_SUPPORT
+				if (dhd->arpoe_enable) {
+					dhd_arp_offload_enable(dhd, TRUE);
+				}
+#endif /* ARP_OFFLOAD_SUPPORT */
+#ifdef PASS_ALL_MCAST_PKTS
+				for (i = 0; i < DHD_MAX_IFS; i++) {
+					struct net_device *ndev = NULL;
+					if (!dhdinfo->iflist[i] || !dhdinfo->iflist[i]->net) {
+						continue;
+					}
+
+					ndev = dhdinfo->iflist[i]->net;
+#ifdef WL_CFG80211
+					if (ndev->ieee80211_ptr->iftype == NL80211_IFTYPE_P2P_GO ||
+						ndev->ieee80211_ptr->iftype == NL80211_IFTYPE_AP) {
+						allmulti = 1;
+						DHD_LOG_MEM(("%s: IF[%s] is AP/GO, set allmulti.\n",
+							__FUNCTION__, ndev->name));
+					} else
+#endif /* WL_CFG80211 */
+					{
+						allmulti = 0;
+					}
+
+					ret = dhd_iovar(dhd, i, "allmulti",
+						(char *)&allmulti,
+						sizeof(allmulti),
+						NULL, 0, TRUE);
+					if (ret < 0) {
+						DHD_ERROR(("%s allmulti failed %d\n",
+							__FUNCTION__, ret));
+					}
+				}
+#endif /* PASS_ALL_MCAST_PKTS */
+
+#if defined(DHD_USE_EARLYSUSPEND) || defined(DHD_USE_PM_SLEEP)
+#ifdef NDO_CONFIG_SUPPORT
+				if (dhd->ndo_enable) {
+					if (!dhd->ndo_host_ip_overflow) {
+						/* enable ND offload on suspend */
+						ret = dhd_ndo_enable(dhd, TRUE);
+						if (ret < 0) {
+							DHD_ERROR(("%s: failed to enable NDO\n",
+								__FUNCTION__));
+						}
+					} else {
+						DHD_INFO(("%s: NDO disabled on suspend due to"
+								"HW capacity\n", __FUNCTION__));
+					}
+				}
+#endif /* NDO_CONFIG_SUPPORT */
+#ifndef APF
+				if (FW_SUPPORTED(dhd, ndoe))
+#else
+				if (FW_SUPPORTED(dhd, ndoe) && !FW_SUPPORTED(dhd, apf))
+#endif /* APF */
+				{
+					/* enable IPv6 RA filter in  firmware during suspend */
+					nd_ra_filter = 1;
+					ret = dhd_iovar(dhd, 0, "nd_ra_filter_enable",
+							(char *)&nd_ra_filter, sizeof(nd_ra_filter),
+							NULL, 0, TRUE);
+					if (ret < 0)
+						DHD_ERROR(("failed to set nd_ra_filter (%d)\n",
+							ret));
+				}
+				dhd_os_suppress_logging(dhd, TRUE);
+#ifdef DYNAMIC_SWOOB_DURATION
+				intr_width = CUSTOM_INTR_WIDTH;
+				ret = dhd_iovar(dhd, 0, "bus:intr_width", (char *)&intr_width,
+						sizeof(intr_width), NULL, 0, TRUE);
+				if (ret < 0) {
+					DHD_ERROR(("failed to set intr_width (%d)\n", ret));
+				}
+#endif /* DYNAMIC_SWOOB_DURATION */
+#ifdef CUSTOM_EVENT_PM_WAKE
+				dhd_set_excess_pm_awake(dhd, TRUE);
+#endif /* CUSTOM_EVENT_PM_WAKE */
+#endif /* DHD_USE_EARLYSUSPEND || DHD_USE_PM_SLEEP */
+#ifdef WL_CFG80211
+				wl_cfg80211_soft_suspend(dev, TRUE);
+#endif /* WL_CFG80211 */
+			} else {
+				dhd->early_suspended = 0;
+				/* Kernel resumed  */
+				DHD_ERROR(("%s: Remove extra suspend setting \n", __FUNCTION__));
+#ifdef DYNAMIC_SWOOB_DURATION
+				intr_width = 0;
+				ret = dhd_iovar(dhd, 0, "bus:intr_width", (char *)&intr_width,
+						sizeof(intr_width), NULL, 0, TRUE);
+				if (ret < 0) {
+					DHD_ERROR(("failed to set intr_width (%d)\n", ret));
+				}
+#endif /* DYNAMIC_SWOOB_DURATION */
+#ifdef ARP_OFFLOAD_SUPPORT
+				if (dhd->arpoe_enable) {
+					dhd_arp_offload_enable(dhd, FALSE);
+				}
+#endif /* ARP_OFFLOAD_SUPPORT */
+#ifdef PKT_FILTER_SUPPORT
+				/* disable pkt filter */
+				dhd_enable_packet_filter(0, dhd);
+#endif /* PKT_FILTER_SUPPORT */
+#ifdef APF
+				dhd_dev_apf_disable_filter(dhd_linux_get_primary_netdev(dhd));
+#endif /* APF */
+#ifdef PASS_ALL_MCAST_PKTS
+				allmulti = 1;
+				for (i = 0; i < DHD_MAX_IFS; i++) {
+					if (dhdinfo->iflist[i] && dhdinfo->iflist[i]->net)
+						ret = dhd_iovar(dhd, i, "allmulti",
+								(char *)&allmulti,
+								sizeof(allmulti), NULL,
+								0, TRUE);
+					if (ret < 0) {
+						DHD_ERROR(("%s: allmulti failed:%d\n",
+								__FUNCTION__, ret));
+					}
+				}
+#endif /* PASS_ALL_MCAST_PKTS */
+#if defined(DHD_USE_EARLYSUSPEND) || defined(DHD_USE_PM_SLEEP)
+#ifdef NDO_CONFIG_SUPPORT
+				if (dhd->ndo_enable) {
+					/* Disable ND offload on resume */
+					ret = dhd_ndo_enable(dhd, FALSE);
+					if (ret < 0) {
+						DHD_ERROR(("%s: failed to disable NDO\n",
+							__FUNCTION__));
+					}
+				}
+#endif /* NDO_CONFIG_SUPPORT */
+#ifndef APF
+				if (FW_SUPPORTED(dhd, ndoe))
+#else
+				if (FW_SUPPORTED(dhd, ndoe) && !FW_SUPPORTED(dhd, apf))
+#endif /* APF */
+				{
+					/* disable IPv6 RA filter in  firmware during suspend */
+					nd_ra_filter = 0;
+					ret = dhd_iovar(dhd, 0, "nd_ra_filter_enable",
+							(char *)&nd_ra_filter, sizeof(nd_ra_filter),
+							NULL, 0, TRUE);
+					if (ret < 0) {
+						DHD_ERROR(("failed to set nd_ra_filter (%d)\n",
+							ret));
+					}
+				}
+				dhd_os_suppress_logging(dhd, FALSE);
+#ifdef CUSTOM_EVENT_PM_WAKE
+				dhd_set_excess_pm_awake(dhd, FALSE);
+#endif /* CUSTOM_EVENT_PM_WAKE */
+#endif /* DHD_USE_EARLYSUSPEND || DHD_USE_PM_SLEEP */
+#ifdef WL_CFG80211
+				wl_cfg80211_soft_suspend(dev, FALSE);
+#endif /* WL_CFG80211 */
+			}
+	}
+	dhd_suspend_unlock(dhd);
+
+	return 0;
+}
+
+static int dhd_suspend_resume_helper(struct dhd_info *dhd, int val, int force)
+{
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0;
+
+#if !defined(DHD_USE_PM_SLEEP)
+	DHD_OS_WAKE_LOCK(dhdp);
+#endif /* !defined(DHD_USE_PM_SLEEP) */
+
+	/* Set flag when early suspend was called */
+	dhdp->in_suspend = val;
+	if ((force || !dhdp->suspend_disable_flag) &&
+		(dhd_support_sta_mode(dhdp) || dhd_conf_get_insuspend(dhdp, ALL_IN_SUSPEND)))
+	{
+		ret = dhd_set_suspend(val, dhdp);
+	}
+
+#if !defined(DHD_USE_PM_SLEEP)
+	DHD_OS_WAKE_UNLOCK(dhdp);
+#endif /* !defined(DHD_USE_PM_SLEEP) */
+
+	return ret;
+}
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+static void dhd_early_suspend(struct early_suspend *h)
+{
+	struct dhd_info *dhd = container_of(h, struct dhd_info, early_suspend);
+	DHD_TRACE_HW4(("%s: enter\n", __FUNCTION__));
+
+	if (dhd && (dhd->pub.conf->suspend_mode == EARLY_SUSPEND ||
+			dhd->pub.conf->suspend_mode == SUSPEND_MODE_2)) {
+		dhd_suspend_resume_helper(dhd, 1, 0);
+		if (dhd->pub.conf->suspend_mode == EARLY_SUSPEND)
+			dhd_conf_set_suspend_resume(&dhd->pub, 1);
+	}
+}
+
+static void dhd_late_resume(struct early_suspend *h)
+{
+	struct dhd_info *dhd = container_of(h, struct dhd_info, early_suspend);
+	DHD_TRACE_HW4(("%s: enter\n", __FUNCTION__));
+
+	if (dhd && (dhd->pub.conf->suspend_mode == EARLY_SUSPEND ||
+			dhd->pub.conf->suspend_mode == SUSPEND_MODE_2)) {
+		dhd_conf_set_suspend_resume(&dhd->pub, 0);
+		if (dhd->pub.conf->suspend_mode == EARLY_SUSPEND)
+			dhd_suspend_resume_helper(dhd, 0, 0);
+	}
+}
+#endif /* CONFIG_HAS_EARLYSUSPEND && DHD_USE_EARLYSUSPEND */
+
+/*
+ * Generalized timeout mechanism.  Uses spin sleep with exponential back-off until
+ * the sleep time reaches one jiffy, then switches over to task delay.  Usage:
+ *
+ *      dhd_timeout_start(&tmo, usec);
+ *      while (!dhd_timeout_expired(&tmo))
+ *              if (poll_something())
+ *                      break;
+ *      if (dhd_timeout_expired(&tmo))
+ *              fatal();
+ */
+
+void
+dhd_timeout_start(dhd_timeout_t *tmo, uint usec)
+{
+#ifdef BCMQT
+	tmo->limit = usec * htclkratio;
+#else
+	tmo->limit = usec;
+#endif
+	tmo->increment = 0;
+	tmo->elapsed = 0;
+	tmo->tick = 10 * USEC_PER_MSEC;	/* 10 msec */
+}
+
+int
+dhd_timeout_expired(dhd_timeout_t *tmo)
+{
+	/* Does nothing the first call */
+	if (tmo->increment == 0) {
+		tmo->increment = USEC_PER_MSEC;	/* Start with 1 msec */
+		return 0;
+	}
+
+	if (tmo->elapsed >= tmo->limit)
+		return 1;
+
+	DHD_INFO(("%s: CAN_SLEEP():%d tmo->increment=%ld msec\n",
+		__FUNCTION__, CAN_SLEEP(), tmo->increment / USEC_PER_MSEC));
+
+	CAN_SLEEP() ? OSL_SLEEP(tmo->increment / USEC_PER_MSEC) : OSL_DELAY(tmo->increment);
+
+	/* Till tmo->tick, the delay will be in 2x, after that delay will be constant
+	 * tmo->tick (10 msec), till timer elapses.
+	 */
+	tmo->increment = (tmo->increment >= tmo->tick) ? tmo->tick : (tmo->increment * 2);
+
+	/* Add the delay that's about to take place */
+#ifdef BCMQT
+	tmo->elapsed += tmo->increment * htclkratio;
+#else
+	tmo->elapsed += tmo->increment;
+#endif
+
+	return 0;
+}
+
+int
+dhd_net2idx(dhd_info_t *dhd, struct net_device *net)
+{
+	int i = 0;
+
+	if (!dhd) {
+		DHD_ERROR(("%s : DHD_BAD_IF return\n", __FUNCTION__));
+		return DHD_BAD_IF;
+	}
+
+	while (i < DHD_MAX_IFS) {
+		if (dhd->iflist[i] && dhd->iflist[i]->net && (dhd->iflist[i]->net == net))
+			return i;
+		i++;
+	}
+
+	return DHD_BAD_IF;
+}
+
+struct net_device * dhd_idx2net(void *pub, int ifidx)
+{
+	struct dhd_pub *dhd_pub = (struct dhd_pub *)pub;
+	struct dhd_info *dhd_info;
+
+	if (!dhd_pub || ifidx < 0 || ifidx >= DHD_MAX_IFS)
+		return NULL;
+	dhd_info = dhd_pub->info;
+	if (dhd_info && dhd_info->iflist[ifidx])
+		return dhd_info->iflist[ifidx]->net;
+	return NULL;
+}
+
+int
+dhd_ifname2idx(dhd_info_t *dhd, char *name)
+{
+	int i = DHD_MAX_IFS;
+
+	ASSERT(dhd);
+
+	if (name == NULL || *name == '\0')
+		return 0;
+
+	while (--i > 0)
+		if (dhd->iflist[i] && !strncmp(dhd->iflist[i]->dngl_name, name, IFNAMSIZ))
+				break;
+
+	DHD_TRACE(("%s: return idx %d for \"%s\"\n", __FUNCTION__, i, name));
+
+	return i;	/* default - the primary interface */
+}
+
+char *
+dhd_ifname(dhd_pub_t *dhdp, int ifidx)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+
+	ASSERT(dhd);
+
+	if (ifidx < 0 || ifidx >= DHD_MAX_IFS) {
+		DHD_ERROR(("%s: ifidx %d out of range\n", __FUNCTION__, ifidx));
+		return "<if_bad>";
+	}
+
+	if (dhd->iflist[ifidx] == NULL) {
+		DHD_ERROR(("%s: null i/f %d\n", __FUNCTION__, ifidx));
+		return "<if_null>";
+	}
+
+	if (dhd->iflist[ifidx]->net)
+		return dhd->iflist[ifidx]->net->name;
+
+	return "<if_none>";
+}
+
+uint8 *
+dhd_bssidx2bssid(dhd_pub_t *dhdp, int idx)
+{
+	int i;
+	dhd_info_t *dhd = (dhd_info_t *)dhdp;
+
+	ASSERT(dhd);
+	for (i = 0; i < DHD_MAX_IFS; i++)
+	if (dhd->iflist[i] && dhd->iflist[i]->bssidx == idx)
+		return dhd->iflist[i]->mac_addr;
+
+	return NULL;
+}
+
+static void
+_dhd_set_multicast_list(dhd_info_t *dhd, int ifidx)
+{
+	struct net_device *dev;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35)
+	struct netdev_hw_addr *ha;
+#else
+	struct dev_mc_list *mclist;
+#endif
+	uint32 allmulti, cnt;
+
+	wl_ioctl_t ioc;
+	char *buf, *bufp;
+	uint buflen;
+	int ret;
+
+#ifdef MCAST_LIST_ACCUMULATION
+	int i;
+	uint32 cnt_iface[DHD_MAX_IFS];
+	cnt = 0;
+	allmulti = 0;
+
+	for (i = 0; i < DHD_MAX_IFS; i++) {
+		if (dhd->iflist[i]) {
+			dev = dhd->iflist[i]->net;
+			if (!dev)
+				continue;
+			netif_addr_lock_bh(dev);
+			cnt_iface[i] = netdev_mc_count(dev);
+			cnt += cnt_iface[i];
+			netif_addr_unlock_bh(dev);
+
+			/* Determine initial value of allmulti flag */
+			allmulti |= (dev->flags & IFF_ALLMULTI) ? TRUE : FALSE;
+		}
+	}
+#else /* !MCAST_LIST_ACCUMULATION */
+	if (!dhd->iflist[ifidx]) {
+		DHD_ERROR(("%s : dhd->iflist[%d] was NULL\n", __FUNCTION__, ifidx));
+		return;
+	}
+	dev = dhd->iflist[ifidx]->net;
+	if (!dev)
+		return;
+	netif_addr_lock_bh(dev);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35)
+	cnt = netdev_mc_count(dev);
+#else
+	cnt = dev->mc_count;
+#endif /* LINUX_VERSION_CODE */
+
+	netif_addr_unlock_bh(dev);
+
+	/* Determine initial value of allmulti flag */
+	allmulti = (dev->flags & IFF_ALLMULTI) ? TRUE : FALSE;
+#endif /* MCAST_LIST_ACCUMULATION */
+
+#ifdef PASS_ALL_MCAST_PKTS
+	if (!dhd->pub.early_suspended) {
+		allmulti = TRUE;
+	}
+#endif /* PASS_ALL_MCAST_PKTS */
+
+	/* Send down the multicast list first. */
+
+	/* XXX Not using MAXMULTILIST to avoid including wlc_pub.h; but
+	 * maybe we should?  (Or should that be in wlioctl.h instead?)
+	 */
+
+	buflen = sizeof("mcast_list") + sizeof(cnt) + (cnt * ETHER_ADDR_LEN);
+	if (!(bufp = buf = MALLOC(dhd->pub.osh, buflen))) {
+		DHD_ERROR(("%s: out of memory for mcast_list, cnt %d\n",
+		           dhd_ifname(&dhd->pub, ifidx), cnt));
+		return;
+	}
+
+	strlcpy(bufp, "mcast_list", buflen);
+	bufp += strlen("mcast_list") + 1;
+
+	cnt = htol32(cnt);
+	memcpy(bufp, &cnt, sizeof(cnt));
+	bufp += sizeof(cnt);
+
+#ifdef MCAST_LIST_ACCUMULATION
+	for (i = 0; i < DHD_MAX_IFS; i++) {
+		if (dhd->iflist[i]) {
+			DHD_TRACE(("_dhd_set_multicast_list: ifidx %d\n", i));
+			dev = dhd->iflist[i]->net;
+
+			netif_addr_lock_bh(dev);
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			netdev_for_each_mc_addr(ha, dev) {
+				GCC_DIAGNOSTIC_POP();
+				if (!cnt_iface[i])
+					break;
+				memcpy(bufp, ha->addr, ETHER_ADDR_LEN);
+				bufp += ETHER_ADDR_LEN;
+				DHD_TRACE(("_dhd_set_multicast_list: cnt "
+					"%d " MACDBG "\n",
+					cnt_iface[i], MAC2STRDBG(ha->addr)));
+				cnt_iface[i]--;
+			}
+			netif_addr_unlock_bh(dev);
+		}
+	}
+#else /* !MCAST_LIST_ACCUMULATION */
+	netif_addr_lock_bh(dev);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35)
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	netdev_for_each_mc_addr(ha, dev) {
+		GCC_DIAGNOSTIC_POP();
+		if (!cnt)
+			break;
+		memcpy(bufp, ha->addr, ETHER_ADDR_LEN);
+		bufp += ETHER_ADDR_LEN;
+		cnt--;
+	}
+#else
+	for (mclist = dev->mc_list; (mclist && (cnt > 0));
+			cnt--, mclist = mclist->next) {
+		memcpy(bufp, (void *)mclist->dmi_addr, ETHER_ADDR_LEN);
+		bufp += ETHER_ADDR_LEN;
+	}
+#endif /* LINUX_VERSION_CODE */
+	netif_addr_unlock_bh(dev);
+#endif /* MCAST_LIST_ACCUMULATION */
+
+	memset(&ioc, 0, sizeof(ioc));
+	ioc.cmd = WLC_SET_VAR;
+	ioc.buf = buf;
+	ioc.len = buflen;
+	ioc.set = TRUE;
+
+	ret = dhd_wl_ioctl(&dhd->pub, ifidx, &ioc, ioc.buf, ioc.len);
+	if (ret < 0) {
+		DHD_ERROR(("%s: set mcast_list failed, cnt %d\n",
+			dhd_ifname(&dhd->pub, ifidx), cnt));
+		allmulti = cnt ? TRUE : allmulti;
+	}
+
+	MFREE(dhd->pub.osh, buf, buflen);
+
+	/* Now send the allmulti setting.  This is based on the setting in the
+	 * net_device flags, but might be modified above to be turned on if we
+	 * were trying to set some addresses and dongle rejected it...
+	 */
+
+	allmulti = htol32(allmulti);
+	ret = dhd_iovar(&dhd->pub, ifidx, "allmulti", (char *)&allmulti,
+			sizeof(allmulti), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: set allmulti %d failed\n",
+		           dhd_ifname(&dhd->pub, ifidx), ltoh32(allmulti)));
+	}
+
+	/* Finally, pick up the PROMISC flag as well, like the NIC driver does */
+
+#ifdef MCAST_LIST_ACCUMULATION
+	allmulti = 0;
+	for (i = 0; i < DHD_MAX_IFS; i++) {
+		if (dhd->iflist[i]) {
+			dev = dhd->iflist[i]->net;
+			allmulti |= (dev->flags & IFF_PROMISC) ? TRUE : FALSE;
+		}
+	}
+#else
+	allmulti = (dev->flags & IFF_PROMISC) ? TRUE : FALSE;
+#endif /* MCAST_LIST_ACCUMULATION */
+
+	allmulti = htol32(allmulti);
+
+	memset(&ioc, 0, sizeof(ioc));
+	ioc.cmd = WLC_SET_PROMISC;
+	ioc.buf = &allmulti;
+	ioc.len = sizeof(allmulti);
+	ioc.set = TRUE;
+
+	ret = dhd_wl_ioctl(&dhd->pub, ifidx, &ioc, ioc.buf, ioc.len);
+	if (ret < 0) {
+		DHD_ERROR(("%s: set promisc %d failed\n",
+		           dhd_ifname(&dhd->pub, ifidx), ltoh32(allmulti)));
+	}
+}
+
+int
+_dhd_set_mac_address(dhd_info_t *dhd, int ifidx, uint8 *addr, bool skip_stop)
+{
+	int ret;
+
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	if (skip_stop) {
+		WL_MSG(dhd_ifname(&dhd->pub, ifidx), "close dev for mac changing\n");
+		dhd->pub.skip_dhd_stop = TRUE;
+		dev_close(dhd->iflist[ifidx]->net);
+	}
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+
+	ret = dhd_iovar(&dhd->pub, ifidx, "cur_etheraddr", (char *)addr,
+			ETHER_ADDR_LEN, NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: set cur_etheraddr %pM failed ret=%d\n",
+			dhd_ifname(&dhd->pub, ifidx), addr, ret));
+		goto exit;
+	} else {
+		DEV_ADDR_SET((dhd->iflist[ifidx]->net), addr);
+		if (ifidx == 0)
+			memcpy(dhd->pub.mac.octet, addr, ETHER_ADDR_LEN);
+		WL_MSG(dhd_ifname(&dhd->pub, ifidx), "MACID %pM is overwritten\n", addr);
+	}
+
+exit:
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	if (skip_stop) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+		dev_open(dhd->iflist[ifidx]->net, NULL);
+#else
+		dev_open(dhd->iflist[ifidx]->net);
+#endif
+		dhd->pub.skip_dhd_stop = FALSE;
+		WL_MSG(dhd_ifname(&dhd->pub, ifidx), "notify mac changed done\n");
+	}
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+
+	return ret;
+}
+
+int dhd_update_rand_mac_addr(dhd_pub_t *dhd)
+{
+	struct ether_addr mac_addr;
+	dhd_generate_rand_mac_addr(&mac_addr);
+	if (_dhd_set_mac_address(dhd->info, 0, mac_addr.octet, TRUE) != 0) {
+		DHD_ERROR(("randmac setting failed\n"));
+#ifdef STA_RANDMAC_ENFORCED
+		return  BCME_BADADDR;
+#endif /* STA_RANDMAC_ENFORCED */
+	}
+	return BCME_OK;
+}
+
+#ifdef BCM_ROUTER_DHD
+void dhd_update_dpsta_interface_for_sta(dhd_pub_t* dhdp, int ifidx, void* event_data)
+{
+	struct wl_dpsta_intf_event *dpsta_prim_event = (struct wl_dpsta_intf_event *)event_data;
+	dhd_if_t *ifp = dhdp->info->iflist[ifidx];
+
+	if (dpsta_prim_event->intf_type == WL_INTF_DWDS) {
+		ifp->primsta_dwds = TRUE;
+	} else {
+		ifp->primsta_dwds = FALSE;
+	}
+}
+#endif /* BCM_ROUTER_DHD */
+
+#ifdef DHD_WMF
+void dhd_update_psta_interface_for_sta(dhd_pub_t* dhdp, char* ifname, void* ea,
+		void* event_data)
+{
+	struct wl_psta_primary_intf_event *psta_prim_event =
+			(struct wl_psta_primary_intf_event*)event_data;
+	dhd_sta_t *psta_interface =  NULL;
+	dhd_sta_t *sta = NULL;
+	uint8 ifindex;
+	ASSERT(ifname);
+	ASSERT(psta_prim_event);
+	ASSERT(ea);
+
+	ifindex = (uint8)dhd_ifname2idx(dhdp->info, ifname);
+	sta = dhd_find_sta(dhdp, ifindex, ea);
+	if (sta != NULL) {
+		psta_interface = dhd_find_sta(dhdp, ifindex,
+				(void *)(psta_prim_event->prim_ea.octet));
+		if (psta_interface != NULL) {
+			sta->psta_prim = psta_interface;
+		}
+	}
+}
+
+/* Get wmf_psta_disable configuration configuration */
+int dhd_get_wmf_psta_disable(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+	return ifp->wmf_psta_disable;
+}
+
+/* Set wmf_psta_disable configuration configuration */
+int dhd_set_wmf_psta_disable(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+	ifp->wmf_psta_disable = val;
+	return 0;
+}
+#endif /* DHD_WMF */
+
+#ifdef DHD_PSTA
+/* Get psta/psr configuration configuration */
+int dhd_get_psta_mode(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	return (int)dhd->psta_mode;
+}
+/* Set psta/psr configuration configuration */
+int dhd_set_psta_mode(dhd_pub_t *dhdp, uint32 val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd->psta_mode = val;
+	return 0;
+}
+#endif /* DHD_PSTA */
+
+#if (defined(DHD_WET) || defined(DHD_MCAST_REGEN) || defined(DHD_L2_FILTER))
+static void
+dhd_update_rx_pkt_chainable_state(dhd_pub_t* dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	if (
+#ifdef DHD_L2_FILTER
+		(ifp->block_ping) ||
+#endif
+#ifdef DHD_WET
+		(dhd->wet_mode) ||
+#endif
+#ifdef DHD_MCAST_REGEN
+		(ifp->mcast_regen_bss_enable) ||
+#endif
+		FALSE) {
+		ifp->rx_pkt_chainable = FALSE;
+	}
+}
+#endif /* DHD_WET || DHD_MCAST_REGEN || DHD_L2_FILTER */
+
+#ifdef DHD_WET
+/* Get wet configuration configuration */
+int dhd_get_wet_mode(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	return (int)dhd->wet_mode;
+}
+
+/* Set wet configuration configuration */
+int dhd_set_wet_mode(dhd_pub_t *dhdp, uint32 val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd->wet_mode = val;
+	dhd_update_rx_pkt_chainable_state(dhdp, 0);
+	return 0;
+}
+#endif /* DHD_WET */
+
+#if defined(WL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+int32 dhd_role_to_nl80211_iftype(int32 role)
+{
+	switch (role) {
+	case WLC_E_IF_ROLE_STA:
+		return NL80211_IFTYPE_STATION;
+	case WLC_E_IF_ROLE_AP:
+		return NL80211_IFTYPE_AP;
+	case WLC_E_IF_ROLE_WDS:
+		return NL80211_IFTYPE_WDS;
+	case WLC_E_IF_ROLE_P2P_GO:
+		return NL80211_IFTYPE_P2P_GO;
+	case WLC_E_IF_ROLE_P2P_CLIENT:
+		return NL80211_IFTYPE_P2P_CLIENT;
+	case WLC_E_IF_ROLE_IBSS:
+	case WLC_E_IF_ROLE_NAN:
+		return NL80211_IFTYPE_ADHOC;
+	default:
+		return NL80211_IFTYPE_UNSPECIFIED;
+	}
+}
+#endif /* WL_CFG80211 && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+static void
+dhd_ifadd_event_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	dhd_if_event_t *if_event = event_info;
+	int ifidx, bssidx;
+	int ret = 0;
+#if defined(WL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	struct wl_if_event_info info;
+#endif /* WL_CFG80211 && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+	struct net_device *ndev = NULL;
+
+	BCM_REFERENCE(ret);
+	if (event != DHD_WQ_WORK_IF_ADD) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	if (!if_event) {
+		DHD_ERROR(("%s: event data is null \n", __FUNCTION__));
+		return;
+	}
+
+	dhd_net_if_lock_local(dhd);
+
+	ifidx = if_event->event.ifidx;
+	bssidx = if_event->event.bssidx;
+	DHD_TRACE(("%s: registering if with ifidx %d\n", __FUNCTION__, ifidx));
+
+#if defined(WL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	if (if_event->event.ifidx > 0) {
+		u8 *mac_addr;
+		bzero(&info, sizeof(info));
+		info.ifidx = ifidx;
+		info.bssidx = bssidx;
+		info.role = if_event->event.role;
+		strlcpy(info.name, if_event->name, sizeof(info.name));
+		if (is_valid_ether_addr(if_event->mac)) {
+			mac_addr = if_event->mac;
+		} else {
+			mac_addr = NULL;
+		}
+
+		if ((ndev = wl_cfg80211_post_ifcreate(dhd->pub.info->iflist[0]->net,
+			&info, mac_addr, NULL, true)) == NULL) {
+			/* Do the post interface create ops */
+			DHD_ERROR(("Post ifcreate ops failed. Returning \n"));
+			ret = BCME_ERROR;
+			goto done;
+		}
+	}
+#else
+	/* This path is for non-android case */
+	/* The interface name in host and in event msg are same */
+	/* if name in event msg is used to create dongle if list on host */
+	ndev = dhd_allocate_if(&dhd->pub, ifidx, if_event->name,
+		if_event->mac, bssidx, TRUE, if_event->name);
+	if (!ndev) {
+		DHD_ERROR(("%s: net device alloc failed  \n", __FUNCTION__));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+
+	ret = dhd_register_if(&dhd->pub, ifidx, TRUE);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: dhd_register_if failed\n", __FUNCTION__));
+		dhd_remove_if(&dhd->pub, ifidx, TRUE);
+		goto done;
+	}
+
+#endif /* WL_CFG80211 && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+#ifndef PCIE_FULL_DONGLE
+	/* Turn on AP isolation in the firmware for interfaces operating in AP mode */
+	if (FW_SUPPORTED((&dhd->pub), ap) && (if_event->event.role != WLC_E_IF_ROLE_STA)) {
+		uint32 var_int =  1;
+		ret = dhd_iovar(&dhd->pub, ifidx, "ap_isolate", (char *)&var_int, sizeof(var_int),
+				NULL, 0, TRUE);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: Failed to set ap_isolate to dongle\n", __FUNCTION__));
+			dhd_remove_if(&dhd->pub, ifidx, TRUE);
+		}
+	}
+#endif /* PCIE_FULL_DONGLE */
+
+done:
+
+	MFREE(dhd->pub.osh, if_event, sizeof(dhd_if_event_t));
+#if defined(WLDWDS) && defined(FOURADDR_AUTO_BRG)
+	if (ndev) {
+		dhd_bridge_dev_set(dhd, ifidx, ndev);
+	} else {
+		DHD_ERROR(("Netdev not found! Do nothing.\n"));
+	}
+#endif /* defiend(WLDWDS) && defined(FOURADDR_AUTO_BRG) */
+
+	dhd_net_if_unlock_local(dhd);
+}
+
+static void
+dhd_ifdel_event_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	int ifidx;
+	dhd_if_event_t *if_event = event_info;
+
+	if (event != DHD_WQ_WORK_IF_DEL) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	if (!if_event) {
+		DHD_ERROR(("%s: event data is null \n", __FUNCTION__));
+		return;
+	}
+
+	dhd_net_if_lock_local(dhd);
+
+	ifidx = if_event->event.ifidx;
+	DHD_TRACE(("Removing interface with idx %d\n", ifidx));
+
+	if (!dhd->pub.info->iflist[ifidx]) {
+		/* No matching netdev found */
+		DHD_ERROR(("Netdev not found! Do nothing.\n"));
+		goto done;
+	}
+#if defined(WLDWDS) && defined(FOURADDR_AUTO_BRG)
+	dhd_bridge_dev_set(dhd, ifidx, NULL);
+#endif /* WLDWDS && FOURADDR_AUTO_BRG */
+#if defined(WL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	if (if_event->event.ifidx > 0) {
+		/* Do the post interface del ops */
+		if (wl_cfg80211_post_ifdel(dhd->pub.info->iflist[ifidx]->net,
+				true, if_event->event.ifidx) != 0) {
+			DHD_TRACE(("Post ifdel ops failed. Returning \n"));
+			goto done;
+		}
+	}
+#else
+	/* For non-cfg80211 drivers */
+	dhd_remove_if(&dhd->pub, ifidx, TRUE);
+
+#endif /* WL_CFG80211 && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+done:
+	MFREE(dhd->pub.osh, if_event, sizeof(dhd_if_event_t));
+	dhd_net_if_unlock_local(dhd);
+}
+
+#ifdef DHD_UPDATE_INTF_MAC
+static void
+dhd_ifupdate_event_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	int ifidx;
+	dhd_if_event_t *if_event = event_info;
+
+	if (event != DHD_WQ_WORK_IF_UPDATE) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	if (!if_event) {
+		DHD_ERROR(("%s: event data is null \n", __FUNCTION__));
+		return;
+	}
+
+	dhd_net_if_lock_local(dhd);
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+
+	ifidx = if_event->event.ifidx;
+	DHD_TRACE(("%s: Update interface with idx %d\n", __FUNCTION__, ifidx));
+
+	dhd_op_if_update(&dhd->pub, ifidx);
+
+	MFREE(dhd->pub.osh, if_event, sizeof(dhd_if_event_t));
+
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+	dhd_net_if_unlock_local(dhd);
+}
+
+int dhd_op_if_update(dhd_pub_t *dhdpub, int ifidx)
+{
+	dhd_info_t *    dhdinfo = NULL;
+	dhd_if_t   *    ifp = NULL;
+	int             ret = 0;
+	char            buf[128];
+
+	if ((NULL==dhdpub)||(NULL==dhdpub->info)) {
+		DHD_ERROR(("%s: *** DHD handler is NULL!\n", __FUNCTION__));
+		return -1;
+	} else {
+		dhdinfo = (dhd_info_t *)dhdpub->info;
+		ifp = dhdinfo->iflist[ifidx];
+		if (NULL==ifp) {
+		    DHD_ERROR(("%s: *** ifp handler is NULL!\n", __FUNCTION__));
+		    return -2;
+		}
+	}
+
+	DHD_TRACE(("%s: idx %d\n", __FUNCTION__, ifidx));
+	// Get MAC address
+	strcpy(buf, "cur_etheraddr");
+	ret = dhd_wl_ioctl_cmd(&dhdinfo->pub, WLC_GET_VAR, buf, sizeof(buf), FALSE, ifp->idx);
+	if (0>ret) {
+		DHD_ERROR(("Failed to upudate the MAC address for itf=%s, ret=%d\n", ifp->name, ret));
+		// avoid collision
+		dhdinfo->iflist[ifp->idx]->mac_addr[5] += 1;
+		// force locally administrate address
+		ETHER_SET_LOCALADDR(&dhdinfo->iflist[ifp->idx]->mac_addr);
+	} else {
+		DHD_EVENT(("Got mac for itf %s, idx %d, MAC=%02X:%02X:%02X:%02X:%02X:%02X\n",
+		           ifp->name, ifp->idx,
+		           (unsigned char)buf[0], (unsigned char)buf[1], (unsigned char)buf[2],
+		           (unsigned char)buf[3], (unsigned char)buf[4], (unsigned char)buf[5]));
+		memcpy(dhdinfo->iflist[ifp->idx]->mac_addr, buf, ETHER_ADDR_LEN);
+		if (dhdinfo->iflist[ifp->idx]->net) {
+			dev_addr_set(dhdinfo->iflist[ifp->idx]->net, buf);
+		}
+	}
+
+	return ret;
+}
+#endif /* DHD_UPDATE_INTF_MAC */
+
+static void
+dhd_set_mac_addr_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	int ifidx = (int)((long int)event_info);
+	dhd_if_t *ifp = NULL;
+
+	if (event != DHD_WQ_WORK_SET_MAC) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+	}
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	rtnl_lock();
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+	dhd_net_if_lock_local(dhd);
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+
+	ifp = dhd->iflist[ifidx];
+
+	// terence 20160907: fix for not able to set mac when wlan0 is down
+	if (ifp == NULL || !ifp->set_macaddress) {
+		goto done;
+	}
+	if (ifp == NULL || !dhd->pub.up) {
+		DHD_ERROR(("%s: interface info not available/down \n", __FUNCTION__));
+		goto done;
+	}
+
+	ifp->set_macaddress = FALSE;
+
+	if (_dhd_set_mac_address(dhd, ifp->idx, ifp->mac_addr, TRUE) == 0)
+		DHD_INFO(("%s: MACID is overwritten\n",	__FUNCTION__));
+	else
+		DHD_ERROR(("%s: _dhd_set_mac_address() failed\n", __FUNCTION__));
+
+done:
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+	dhd_net_if_unlock_local(dhd);
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	rtnl_unlock();
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+}
+
+static void
+dhd_ndev_upd_features_handler(void *handle, void *event_info, u8 event)
+{
+	struct net_device *net = event_info;
+
+	if (event != DHD_WQ_WORK_NDEV_UPD_FEATURES) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+	if (!net) {
+		DHD_ERROR(("%s: event data is null \n", __FUNCTION__));
+		return;
+	}
+	rtnl_lock();
+	netdev_update_features(net);
+	rtnl_unlock();
+}
+
+static void
+dhd_ndev_upd_features(dhd_info_t *dhd, struct net_device *net)
+{
+	dhd_deferred_schedule_work(dhd->dhd_deferred_wq, (void *)net,
+		DHD_WQ_WORK_NDEV_UPD_FEATURES, dhd_ndev_upd_features_handler,
+		DHD_WQ_WORK_PRIORITY_HIGH);
+}
+
+static void
+dhd_set_mcast_list_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	int ifidx = (int)((long int)event_info);
+	dhd_if_t *ifp = NULL;
+
+	if (event != DHD_WQ_WORK_SET_MCAST_LIST) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	dhd_net_if_lock_local(dhd);
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+
+	ifp = dhd->iflist[ifidx];
+
+	if (ifp == NULL || !dhd->pub.up) {
+		DHD_ERROR(("%s: interface info not available/down \n", __FUNCTION__));
+		goto done;
+	}
+
+	ifidx = ifp->idx;
+
+#ifdef MCAST_LIST_ACCUMULATION
+	ifidx = 0;
+#endif /* MCAST_LIST_ACCUMULATION */
+
+	_dhd_set_multicast_list(dhd, ifidx);
+	DHD_INFO(("%s: set multicast list for if %d\n", __FUNCTION__, ifidx));
+
+done:
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+	dhd_net_if_unlock_local(dhd);
+}
+
+static int
+dhd_set_mac_address(struct net_device *dev, void *addr)
+{
+	int ret = 0;
+
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	struct sockaddr *sa = (struct sockaddr *)addr;
+	int ifidx;
+	dhd_if_t *dhdif;
+#ifdef WL_STATIC_IF
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+#endif /* WL_STATIC_IF */
+	dhd_pub_t *dhdp = &dhd->pub;
+	bool addr_chngd = TRUE;
+
+	BCM_REFERENCE(ifidx);
+	BCM_REFERENCE(addr_chngd);
+
+	DHD_TRACE(("%s \n", __func__));
+
+	dhdif = dhd_get_ifp_by_ndev(dhdp, dev);
+	if (!dhdif) {
+		return -ENODEV;
+	}
+	ifidx = dhdif->idx;
+	dhd_net_if_lock_local(dhd);
+	if (memcmp(dhdif->mac_addr, sa->sa_data, ETHER_ADDR_LEN) == 0) {
+		addr_chngd = FALSE;
+	}
+	memcpy(dhdif->mac_addr, sa->sa_data, ETHER_ADDR_LEN);
+	dhdif->set_macaddress = TRUE;
+	dhd_net_if_unlock_local(dhd);
+
+	WL_MSG(dev->name, "macaddr = %pM\n", dhdif->mac_addr);
+#ifdef WL_CFG80211
+	/* Check wdev->iftype for the role */
+	if (wl_cfg80211_macaddr_sync_reqd(dev)) {
+		/* Supplicant and certain user layer applications expect macaddress to be
+		 * set once the context returns. so set it from the same context
+		 */
+#ifdef WL_STATIC_IF
+		if (wl_cfg80211_static_if(cfg, dev) && !(dev->flags & IFF_UP)) {
+			/* In softap case, the macaddress will be applied before interface up
+			 * and hence curether_addr can't be done at this stage (no fw iface
+			 * available). Store the address and return. macaddr will be applied
+			 * from interface create context.
+			 */
+			DEV_ADDR_SET(dev, dhdif->mac_addr);
+#if 0 //def DHD_NOTIFY_MAC_CHANGED
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+			dev_open(dev, NULL);
+#else
+			dev_open(dev);
+#endif
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+			return ret;
+		}
+#endif /* WL_STATIC_IF */
+		wl_cfg80211_handle_macaddr_change(dev, dhdif->mac_addr);
+		ret = _dhd_set_mac_address(dhd, ifidx, dhdif->mac_addr, TRUE);
+		if ((ret == BCME_OK) && (addr_chngd == TRUE)) {
+			/* Notify Dev/Address change to upperlayer */
+			netdev_state_change(dev);
+		}
+		return ret;
+	}
+#endif /* WL_CFG80211 */
+
+	dhd_deferred_schedule_work(dhd->dhd_deferred_wq, (void *)((long int)ifidx),
+		DHD_WQ_WORK_SET_MAC, dhd_set_mac_addr_handler, DHD_WQ_WORK_PRIORITY_LOW);
+	return ret;
+}
+
+static void
+dhd_set_multicast_list(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ifidx;
+
+	ifidx = dhd_net2idx(dhd, dev);
+	if (ifidx == DHD_BAD_IF)
+		return;
+
+	dhd->iflist[ifidx]->set_multicast = TRUE;
+	dhd_deferred_schedule_work(dhd->dhd_deferred_wq, (void *)((long int)ifidx),
+		DHD_WQ_WORK_SET_MCAST_LIST, dhd_set_mcast_list_handler, DHD_WQ_WORK_PRIORITY_LOW);
+
+	// terence 20160907: fix for not able to set mac when wlan0 is down
+	dhd_deferred_schedule_work(dhd->dhd_deferred_wq, (void *)((long int)ifidx),
+		DHD_WQ_WORK_SET_MAC, dhd_set_mac_addr_handler, DHD_WQ_WORK_PRIORITY_LOW);
+}
+
+#ifdef DHD_UCODE_DOWNLOAD
+/* Get ucode path */
+char *
+dhd_get_ucode_path(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	return dhd->uc_path;
+}
+#endif /* DHD_UCODE_DOWNLOAD */
+
+#ifdef PROP_TXSTATUS
+int
+dhd_os_wlfc_block(dhd_pub_t *pub)
+{
+	dhd_info_t *di = (dhd_info_t *)(pub->info);
+	ASSERT(di != NULL);
+	/* terence 20161229: don't do spin lock if proptx not enabled */
+	if (disable_proptx)
+		return 1;
+#ifdef BCMDBUS
+	spin_lock_irqsave(&di->wlfc_spinlock, di->wlfc_lock_flags);
+#else
+	spin_lock_bh(&di->wlfc_spinlock);
+#endif /* BCMDBUS */
+	return 1;
+}
+
+int
+dhd_os_wlfc_unblock(dhd_pub_t *pub)
+{
+	dhd_info_t *di = (dhd_info_t *)(pub->info);
+
+	ASSERT(di != NULL);
+	/* terence 20161229: don't do spin lock if proptx not enabled */
+	if (disable_proptx)
+		return 1;
+#ifdef BCMDBUS
+	spin_unlock_irqrestore(&di->wlfc_spinlock, di->wlfc_lock_flags);
+#else
+	spin_unlock_bh(&di->wlfc_spinlock);
+#endif /* BCMDBUS */
+	return 1;
+}
+
+#endif /* PROP_TXSTATUS */
+
+void
+dhd_netif_rx_ni(struct sk_buff * skb)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)
+	/* Do not call netif_recieve_skb as this workqueue scheduler is
+	 * not from NAPI Also as we are not in INTR context, do not call
+	 * netif_rx, instead call netif_rx_ni (for kerenl >= 2.6) which
+	 * does netif_rx, disables irq, raise NET_IF_RX softirq and
+	 * enables interrupts back
+	 */
+	netif_rx_ni(skb);
+#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) */
+	netif_rx(skb);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) */
+}
+
+/*  This routine do not support Packet chain feature, Currently tested for
+ *  proxy arp feature
+ */
+int dhd_sendup(dhd_pub_t *dhdp, int ifidx, void *p)
+{
+	struct sk_buff *skb;
+	void *skbhead = NULL;
+	void *skbprev = NULL;
+	dhd_if_t *ifp;
+	ASSERT(!PKTISCHAINED(p));
+	skb = PKTTONATIVE(dhdp->osh, p);
+
+	ifp = dhdp->info->iflist[ifidx];
+	skb->dev = ifp->net;
+	skb->protocol = eth_type_trans(skb, skb->dev);
+
+	if (in_interrupt()) {
+		bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+			__FUNCTION__, __LINE__);
+		netif_rx(skb);
+	} else {
+		if (dhdp->info->rxthread_enabled) {
+			if (!skbhead) {
+				skbhead = skb;
+			} else {
+				PKTSETNEXT(dhdp->osh, skbprev, skb);
+			}
+			skbprev = skb;
+		} else {
+			/* If the receive is not processed inside an ISR,
+			 * the softirqd must be woken explicitly to service
+			 * the NET_RX_SOFTIRQ.	In 2.6 kernels, this is handled
+			 * by netif_rx_ni(), but in earlier kernels, we need
+			 * to do it manually.
+			 */
+			bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+				__FUNCTION__, __LINE__);
+#if defined(WL_MONITOR) && defined(BCMSDIO)
+			if (dhd_monitor_enabled(dhdp, ifidx))
+				dhd_rx_mon_pkt_sdio(dhdp, skb, ifidx);
+			else
+#endif /* WL_MONITOR && BCMSDIO */
+			dhd_netif_rx_ni(skb);
+		}
+	}
+
+	if (dhdp->info->rxthread_enabled && skbhead)
+		dhd_sched_rxf(dhdp, skbhead);
+
+	return BCME_OK;
+}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+void dhd_start_xmit_wq_adapter(struct work_struct *ptr)
+{
+	struct dhd_rx_tx_work *work;
+	netdev_tx_t ret;
+	dhd_info_t *dhd;
+	struct dhd_bus * bus;
+
+	work = container_of(ptr, struct dhd_rx_tx_work, work);
+
+	dhd = DHD_DEV_INFO(work->net);
+
+	bus = dhd->pub.bus;
+
+	if (atomic_read(&dhd->pub.block_bus)) {
+		kfree_skb(work->skb);
+		kfree(work);
+		dhd_netif_start_queue(bus);
+		return;
+	}
+
+	if (pm_runtime_get_sync(dhd_bus_to_dev(bus)) >= 0) {
+		ret = dhd_start_xmit(work->skb, work->net);
+		pm_runtime_mark_last_busy(dhd_bus_to_dev(bus));
+		pm_runtime_put_autosuspend(dhd_bus_to_dev(bus));
+	}
+	kfree(work);
+	dhd_netif_start_queue(bus);
+
+	if (ret)
+		netdev_err(work->net,
+			   "error: dhd_start_xmit():%d\n", ret);
+}
+
+netdev_tx_t
+BCMFASTPATH(dhd_start_xmit_wrapper)(struct sk_buff *skb, struct net_device *net)
+{
+	struct dhd_rx_tx_work *start_xmit_work;
+	netdev_tx_t ret;
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+
+	if (dhd->pub.busstate == DHD_BUS_SUSPEND) {
+		DHD_RPM(("%s: wakeup the bus using workqueue.\n", __FUNCTION__));
+
+		dhd_netif_stop_queue(dhd->pub.bus);
+
+		start_xmit_work = (struct dhd_rx_tx_work*)
+			kmalloc(sizeof(*start_xmit_work), GFP_ATOMIC);
+
+		if (!start_xmit_work) {
+			netdev_err(net,
+				   "error: failed to alloc start_xmit_work\n");
+			ret = -ENOMEM;
+			goto exit;
+		}
+
+		INIT_WORK(&start_xmit_work->work, dhd_start_xmit_wq_adapter);
+		start_xmit_work->skb = skb;
+		start_xmit_work->net = net;
+		queue_work(dhd->tx_wq, &start_xmit_work->work);
+		ret = NET_XMIT_SUCCESS;
+
+	} else if (dhd->pub.busstate == DHD_BUS_DATA) {
+		ret = dhd_start_xmit(skb, net);
+	} else {
+		/* when bus is down */
+		ret = -ENODEV;
+	}
+
+exit:
+	return ret;
+}
+void
+dhd_bus_wakeup_work(dhd_pub_t *dhdp)
+{
+	struct dhd_rx_tx_work *rx_work;
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+
+	rx_work = kmalloc(sizeof(*rx_work), GFP_ATOMIC);
+	if (!rx_work) {
+		DHD_ERROR(("%s: start_rx_work alloc error. \n", __FUNCTION__));
+		return;
+	}
+
+	INIT_WORK(&rx_work->work, dhd_rx_wq_wakeup);
+	rx_work->pub = dhdp;
+	queue_work(dhd->rx_wq, &rx_work->work);
+
+}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+
+/* Dump CTF stats */
+void
+dhd_ctf_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	dhd_info_t *dhd = dhdp->info;
+
+	bcm_bprintf(strbuf, "CTF stats:\n");
+	ctf_dump(dhd->cih, strbuf);
+}
+
+bool
+BCMFASTPATH(dhd_rx_pkt_chainable)(dhd_pub_t *dhdp, int ifidx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp = dhd->iflist[ifidx];
+
+	return ifp->rx_pkt_chainable;
+}
+
+/* Returns FALSE if block ping is enabled */
+bool
+BCMFASTPATH(dhd_l2_filter_chainable)(dhd_pub_t *dhdp, uint8 *eh, int ifidx)
+{
+#ifdef DHD_L2_FILTER
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp = dhd->iflist[ifidx];
+	ASSERT(ifp != NULL);
+	return ifp->block_ping ? FALSE : TRUE;
+#else
+	return TRUE;
+#endif /* DHD_L2_FILTER */
+}
+/* Returns FALSE if WET is enabled */
+bool
+BCMFASTPATH(dhd_wet_chainable)(dhd_pub_t *dhdp)
+{
+#ifdef DHD_WET
+	return (!WET_ENABLED(dhdp));
+#else
+	return TRUE;
+#endif
+}
+
+/* Returns TRUE if hot bridge entry for this da is present */
+bool
+BCMFASTPATH(dhd_ctf_hotbrc_check)(dhd_pub_t *dhdp, uint8 *eh, int ifidx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp = dhd->iflist[ifidx];
+
+	ASSERT(ifp != NULL);
+
+	if (!dhd->brc_hot)
+		return FALSE;
+
+	return CTF_HOTBRC_CMP(dhd->brc_hot, (eh), (void *)(ifp->net));
+}
+
+/*
+ * Try to forward the complete packet chain through CTF.
+ * If unsuccessful,
+ *   - link the chain by skb->next
+ *   - change the pnext to the 2nd packet of the chain
+ *   - the chained packets will be sent up to the n/w stack
+ */
+static inline int32
+BCMFASTPATH(dhd_ctf_forward)(dhd_info_t *dhd, struct sk_buff *skb, void **pnext)
+{
+	dhd_pub_t *dhdp = &dhd->pub;
+	void *p, *n;
+	void *old_pnext;
+
+	/* try cut thru first */
+	if (!CTF_ENAB(dhd->cih) || (ctf_forward(dhd->cih, skb, skb->dev) == BCME_ERROR)) {
+		/* Fall back to slow path if ctf is disabled or if ctf_forward fails */
+
+		/* clear skipct flag before sending up */
+		PKTCLRSKIPCT(dhdp->osh, skb);
+
+#ifdef CTFPOOL
+		/* allocate and add a new skb to the pkt pool */
+		if (PKTISFAST(dhdp->osh, skb))
+			osl_ctfpool_add(dhdp->osh);
+
+		/* clear fast buf flag before sending up */
+		PKTCLRFAST(dhdp->osh, skb);
+
+		/* re-init the hijacked field */
+		CTFPOOLPTR(dhdp->osh, skb) = NULL;
+#endif /* CTFPOOL */
+
+		/* link the chained packets by skb->next */
+		if (PKTISCHAINED(skb)) {
+			old_pnext = *pnext;
+			PKTFRMNATIVE(dhdp->osh, skb);
+			p = (void *)skb;
+			FOREACH_CHAINED_PKT(p, n) {
+				PKTCLRCHAINED(dhdp->osh, p);
+				PKTCCLRFLAGS(p);
+				if (p == (void *)skb)
+					PKTTONATIVE(dhdp->osh, p);
+				if (n)
+					PKTSETNEXT(dhdp->osh, p, n);
+				else
+					PKTSETNEXT(dhdp->osh, p, old_pnext);
+			}
+			*pnext = PKTNEXT(dhdp->osh, skb);
+			PKTSETNEXT(dhdp->osh, skb, NULL);
+		}
+		return (BCME_ERROR);
+	}
+
+	return (BCME_OK);
+}
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#ifdef DHD_MCAST_REGEN
+/*
+ * Description: This function is called to do the reverse translation
+ *
+ * Input    eh - pointer to the ethernet header
+ */
+int32
+dhd_mcast_reverse_translation(struct ether_header *eh)
+{
+	uint8 *iph;
+	uint32 dest_ip;
+
+	iph = (uint8 *)eh + ETHER_HDR_LEN;
+	dest_ip = ntoh32(*((uint32 *)(iph + IPV4_DEST_IP_OFFSET)));
+
+	/* Only IP packets are handled */
+	if (eh->ether_type != hton16(ETHER_TYPE_IP))
+		return BCME_ERROR;
+
+	/* Non-IPv4 multicast packets are not handled */
+	if (IP_VER(iph) != IP_VER_4)
+		return BCME_ERROR;
+
+	/*
+	 * The packet has a multicast IP and unicast MAC. That means
+	 * we have to do the reverse translation
+	 */
+	if (IPV4_ISMULTI(dest_ip) && !ETHER_ISMULTI(&eh->ether_dhost)) {
+		ETHER_FILL_MCAST_ADDR_FROM_IP(eh->ether_dhost, dest_ip);
+		return BCME_OK;
+	}
+
+	return BCME_ERROR;
+}
+#endif /* MCAST_REGEN */
+
+void
+dhd_dpc_tasklet_dispatcher_work(struct work_struct * work)
+{
+	struct delayed_work *dw = to_delayed_work(work);
+	struct dhd_info *dhd;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(dw, struct dhd_info, dhd_dpc_dispatcher_work);
+	GCC_DIAGNOSTIC_POP();
+
+	DHD_INFO(("%s:\n", __FUNCTION__));
+
+	tasklet_schedule(&dhd->tasklet);
+}
+
+void
+dhd_schedule_delayed_dpc_on_dpc_cpu(dhd_pub_t *dhdp, ulong delay)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	int dpc_cpu = atomic_read(&dhd->dpc_cpu);
+	DHD_INFO(("%s:\n", __FUNCTION__));
+
+	/* scheduler will take care of scheduling to appropriate cpu if dpc_cpu is not online */
+	schedule_delayed_work_on(dpc_cpu, &dhd->dhd_dpc_dispatcher_work, delay);
+
+	return;
+}
+
+#ifdef SHOW_LOGTRACE
+static int
+dhd_event_logtrace_pkt_process(dhd_pub_t *dhdp, struct sk_buff * skb)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	int ret = BCME_OK;
+	uint datalen;
+	bcm_event_msg_u_t evu;
+	void *data = NULL;
+	void *pktdata = NULL;
+	bcm_event_t *pvt_data;
+	uint pktlen;
+
+	DHD_TRACE(("%s:Enter\n", __FUNCTION__));
+
+	/* In dhd_rx_frame, header is stripped using skb_pull
+	 * of size ETH_HLEN, so adjust pktlen accordingly
+	 */
+	pktlen = skb->len + ETH_HLEN;
+
+	pktdata = (void *)skb_mac_header(skb);
+	ret = wl_host_event_get_data(pktdata, pktlen, &evu);
+
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: wl_host_event_get_data err = %d\n",
+			__FUNCTION__, ret));
+		goto exit;
+	}
+
+	datalen = ntoh32(evu.event.datalen);
+
+	pvt_data = (bcm_event_t *)pktdata;
+	data = &pvt_data[1];
+
+	dhd_dbg_trace_evnt_handler(dhdp, data, &dhd->event_data, datalen);
+
+exit:
+	return ret;
+}
+
+/*
+ * dhd_event_logtrace_process_items processes
+ * each skb from evt_trace_queue.
+ * Returns TRUE if more packets to be processed
+ * else returns FALSE
+ */
+
+static int
+dhd_event_logtrace_process_items(dhd_info_t *dhd)
+{
+	dhd_pub_t *dhdp;
+	struct sk_buff *skb;
+	uint32 qlen;
+	uint32 process_len;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return 0;
+	}
+
+	dhdp = &dhd->pub;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhd pub is null \n", __FUNCTION__));
+		return 0;
+	}
+
+	qlen = skb_queue_len(&dhd->evt_trace_queue);
+	process_len = MIN(qlen, DHD_EVENT_LOGTRACE_BOUND);
+
+	/* Run while loop till bound is reached or skb queue is empty */
+	while (process_len--) {
+		int ifid = 0;
+		skb = skb_dequeue(&dhd->evt_trace_queue);
+		if (skb == NULL) {
+			DHD_ERROR(("%s: skb is NULL, which is not valid case\n",
+				__FUNCTION__));
+			break;
+		}
+		BCM_REFERENCE(ifid);
+#ifdef PCIE_FULL_DONGLE
+		/* Check if pkt is from INFO ring or WLC_E_TRACE */
+		ifid = DHD_PKTTAG_IFID((dhd_pkttag_fr_t *)PKTTAG(skb));
+		if (ifid == DHD_DUMMY_INFO_IF) {
+			/* Process logtrace from info rings */
+			dhd_event_logtrace_infobuf_pkt_process(dhdp, skb, &dhd->event_data);
+		} else
+#endif /* PCIE_FULL_DONGLE */
+		{
+			/* Processing WLC_E_TRACE case OR non PCIE PCIE_FULL_DONGLE case */
+			dhd_event_logtrace_pkt_process(dhdp, skb);
+		}
+
+		/* Dummy sleep so that scheduler kicks in after processing any logprints */
+		OSL_SLEEP(0);
+
+		/* Send packet up if logtrace_pkt_sendup is TRUE */
+		if (dhdp->logtrace_pkt_sendup) {
+#ifdef DHD_USE_STATIC_CTRLBUF
+			/* If bufs are allocated via static buf pool
+			 * and logtrace_pkt_sendup enabled, make a copy,
+			 * free the local one and send the copy up.
+			 */
+			void *npkt = PKTDUP(dhdp->osh, skb);
+			/* Clone event and send it up */
+			PKTFREE_STATIC(dhdp->osh, skb, FALSE);
+			if (npkt) {
+				skb = npkt;
+			} else {
+				DHD_ERROR(("skb clone failed. dropping logtrace pkt.\n"));
+				/* Packet is already freed, go to next packet */
+				continue;
+			}
+#endif /* DHD_USE_STATIC_CTRLBUF */
+#ifdef PCIE_FULL_DONGLE
+			/* For infobuf packets as if is DHD_DUMMY_INFO_IF,
+			 * to send skb to network layer, assign skb->dev with
+			 * Primary interface n/w device
+			 */
+			if (ifid == DHD_DUMMY_INFO_IF) {
+				skb = PKTTONATIVE(dhdp->osh, skb);
+				skb->dev = dhd->iflist[0]->net;
+			}
+#endif /* PCIE_FULL_DONGLE */
+			/* Send pkt UP */
+			dhd_netif_rx_ni(skb);
+		} else	{
+			/* Don't send up. Free up the packet. */
+			PKTFREE_CTRLBUF(dhdp->osh, skb, FALSE);
+		}
+	}
+
+	/* Reschedule if more packets to be processed */
+	return (qlen >= DHD_EVENT_LOGTRACE_BOUND);
+}
+
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+static int
+dhd_logtrace_thread(void *data)
+{
+	tsk_ctl_t *tsk = (tsk_ctl_t *)data;
+	dhd_info_t *dhd = (dhd_info_t *)tsk->parent;
+	dhd_pub_t *dhdp = (dhd_pub_t *)&dhd->pub;
+	int ret;
+
+	while (1) {
+		dhdp->logtrace_thr_ts.entry_time = OSL_LOCALTIME_NS();
+		if (!binary_sema_down(tsk)) {
+			dhdp->logtrace_thr_ts.sem_down_time = OSL_LOCALTIME_NS();
+			SMP_RD_BARRIER_DEPENDS();
+
+			/* Check terminated before processing the items */
+			if (tsk->terminated) {
+				DHD_ERROR(("%s: task terminated\n", __FUNCTION__));
+				goto exit;
+			}
+
+			if (dhd->pub.dongle_reset == FALSE) {
+				do {
+#ifdef EWP_EDL
+					/* check if EDL is being used */
+					if (dhd->pub.dongle_edl_support) {
+						ret = dhd_prot_process_edl_complete(&dhd->pub,
+								&dhd->event_data);
+					} else {
+						ret = dhd_event_logtrace_process_items(dhd);
+					}
+#else
+					ret = dhd_event_logtrace_process_items(dhd);
+#endif /* EWP_EDL */
+					/* if ret > 0, bound has reached so to be fair to other
+					 * processes need to yield the scheduler.
+					 * The comment above yield()'s definition says:
+					 * If you want to use yield() to wait for something,
+					 * use wait_event().
+					 * If you want to use yield() to be 'nice' for others,
+					 * use cond_resched().
+					 * If you still want to use yield(), do not!
+					 */
+					if (ret > 0) {
+						cond_resched();
+						OSL_SLEEP(DHD_EVENT_LOGTRACE_RESCHEDULE_DELAY_MS);
+					} else if (ret < 0) {
+						DHD_ERROR(("%s: ERROR should not reach here\n",
+							__FUNCTION__));
+					}
+				} while (ret > 0);
+			}
+			/* Check terminated in case of dongle_reset */
+			if (tsk->terminated) {
+				DHD_ERROR(("%s: task terminated\n", __FUNCTION__));
+				goto exit;
+			}
+			if (tsk->flush_ind) {
+				DHD_ERROR(("%s: flushed\n", __FUNCTION__));
+				dhdp->logtrace_thr_ts.flush_time = OSL_LOCALTIME_NS();
+				tsk->flush_ind = 0;
+				complete(&tsk->flushed);
+			}
+		} else {
+			DHD_ERROR(("%s: unexpted break\n", __FUNCTION__));
+			dhdp->logtrace_thr_ts.unexpected_break_time = OSL_LOCALTIME_NS();
+			break;
+		}
+	}
+exit:
+	complete_and_exit(&tsk->completed, 0);
+	dhdp->logtrace_thr_ts.complete_time = OSL_LOCALTIME_NS();
+}
+#else
+static void
+dhd_event_logtrace_process(struct work_struct * work)
+{
+/* Ignore compiler warnings due to -Werror=cast-qual */
+	struct delayed_work *dw = to_delayed_work(work);
+	struct dhd_info *dhd;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(dw, struct dhd_info, event_log_dispatcher_work);
+	GCC_DIAGNOSTIC_POP();
+
+#ifdef EWP_EDL
+	if (dhd->pub.dongle_edl_support) {
+		ret = dhd_prot_process_edl_complete(&dhd->pub, &dhd->event_data);
+	} else {
+		ret = dhd_event_logtrace_process_items(dhd);
+	}
+#else
+	ret = dhd_event_logtrace_process_items(dhd);
+#endif /* EWP_EDL */
+
+	if (ret > 0) {
+		schedule_delayed_work(&(dhd)->event_log_dispatcher_work,
+			msecs_to_jiffies(DHD_EVENT_LOGTRACE_RESCHEDULE_DELAY_MS));
+	}
+	return;
+}
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+
+void
+dhd_schedule_logtrace(void *dhd_info)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhd_info;
+
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+	if (dhd->thr_logtrace_ctl.thr_pid >= 0) {
+		binary_sema_up(&dhd->thr_logtrace_ctl);
+	} else {
+		DHD_ERROR(("%s: thr_logtrace_ctl(%ld) not inited\n", __FUNCTION__,
+			dhd->thr_logtrace_ctl.thr_pid));
+	}
+#else
+	schedule_delayed_work(&dhd->event_log_dispatcher_work, 0);
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+	return;
+}
+
+void
+dhd_cancel_logtrace_process_sync(dhd_info_t *dhd)
+{
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+	tsk_ctl_t *tsk = &dhd->thr_logtrace_ctl;
+
+	if (tsk->parent && tsk->thr_pid >= 0) {
+		PROC_STOP_USING_BINARY_SEMA(tsk);
+	} else {
+		DHD_ERROR(("%s: thr_logtrace_ctl(%ld) not inited\n",
+			__FUNCTION__, tsk->thr_pid));
+	}
+#else
+	cancel_delayed_work_sync(&dhd->event_log_dispatcher_work);
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+}
+
+void
+dhd_flush_logtrace_process(dhd_info_t *dhd)
+{
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+	if (dhd->thr_logtrace_ctl.thr_pid >= 0) {
+		PROC_FLUSH_USING_BINARY_SEMA(&dhd->thr_logtrace_ctl);
+	} else {
+		DHD_ERROR(("%s: thr_logtrace_ctl(%ld) not inited\n", __FUNCTION__,
+			dhd->thr_logtrace_ctl.thr_pid));
+	}
+#else
+	flush_delayed_work(&dhd->event_log_dispatcher_work);
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+}
+
+int
+dhd_init_logtrace_process(dhd_info_t *dhd)
+{
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+	dhd->thr_logtrace_ctl.thr_pid = DHD_PID_KT_INVALID;
+	PROC_START(dhd_logtrace_thread, dhd, &dhd->thr_logtrace_ctl, 0, "dhd_logtrace_thread");
+	if (dhd->thr_logtrace_ctl.thr_pid < 0) {
+		DHD_ERROR(("%s: init logtrace process failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	} else {
+		DHD_ERROR(("%s: thr_logtrace_ctl(%ld) not inited\n", __FUNCTION__,
+			dhd->thr_logtrace_ctl.thr_pid));
+	}
+#else
+	INIT_DELAYED_WORK(&dhd->event_log_dispatcher_work, dhd_event_logtrace_process);
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+	return BCME_OK;
+}
+
+int
+dhd_reinit_logtrace_process(dhd_info_t *dhd)
+{
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+	/* Re-init only if PROC_STOP from dhd_stop was called
+	 * which can be checked via thr_pid
+	 */
+	if (dhd->thr_logtrace_ctl.thr_pid < 0) {
+		PROC_START(dhd_logtrace_thread, dhd, &dhd->thr_logtrace_ctl,
+			0, "dhd_logtrace_thread");
+		if (dhd->thr_logtrace_ctl.thr_pid < 0) {
+			DHD_ERROR(("%s: reinit logtrace process failed\n", __FUNCTION__));
+			return BCME_ERROR;
+		} else {
+		DHD_ERROR(("%s: thr_logtrace_ctl(%ld) not inited\n", __FUNCTION__,
+			dhd->thr_logtrace_ctl.thr_pid));
+	}
+	}
+#else
+	/* No need to re-init for WQ as calcel_delayed_work_sync will
+	 * will not delete the WQ
+	 */
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+	return BCME_OK;
+}
+
+void
+dhd_event_logtrace_enqueue(dhd_pub_t *dhdp, int ifidx, void *pktbuf)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	BCM_REFERENCE(dhd);
+#if 1
+#ifdef PCIE_FULL_DONGLE
+	/* Add ifidx in the PKTTAG */
+	DHD_PKTTAG_SET_IFID((dhd_pkttag_fr_t *)PKTTAG(pktbuf), ifidx);
+#endif /* PCIE_FULL_DONGLE */
+	skb_queue_tail(&dhd->evt_trace_queue, pktbuf);
+
+	dhd_schedule_logtrace(dhd);
+#else /* !BCMDBUS */
+#ifdef DHD_USE_STATIC_CTRLBUF
+	PKTFREE_STATIC(dhdp->osh, pktbuf, FALSE);
+#else
+	PKTFREE(dhdp->osh, pktbuf, FALSE);
+#endif /* DHD_USE_STATIC_CTRLBUF */
+#endif /* BCMDBUS */
+}
+
+void
+dhd_event_logtrace_flush_queue(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(&dhd->evt_trace_queue)) != NULL) {
+		PKTFREE_CTRLBUF(dhdp->osh, skb, FALSE);
+	}
+}
+
+#ifdef EWP_EDL
+void
+dhd_sendup_info_buf(dhd_pub_t *dhdp, uint8 *msg)
+{
+	struct sk_buff *skb = NULL;
+	uint32 pktsize = 0;
+	void *pkt = NULL;
+	info_buf_payload_hdr_t *infobuf = NULL;
+	dhd_info_t *dhd = dhdp->info;
+	uint8 *pktdata = NULL;
+
+	if (!msg)
+		return;
+
+	/* msg = |infobuf_ver(u32)|info_buf_payload_hdr_t|msgtrace_hdr_t|<var len data>| */
+	infobuf = (info_buf_payload_hdr_t *)(msg + sizeof(uint32));
+	pktsize = (uint32)(ltoh16(infobuf->length) + sizeof(info_buf_payload_hdr_t) +
+			sizeof(uint32));
+	pkt = PKTGET(dhdp->osh, pktsize, FALSE);
+	if (!pkt) {
+		DHD_ERROR(("%s: skb alloc failed ! not sending event log up.\n", __FUNCTION__));
+	} else {
+		PKTSETLEN(dhdp->osh, pkt, pktsize);
+		pktdata = PKTDATA(dhdp->osh, pkt);
+		memcpy(pktdata, msg, pktsize);
+		/* For infobuf packets assign skb->dev with
+		 * Primary interface n/w device
+		 */
+		skb = PKTTONATIVE(dhdp->osh, pkt);
+		skb->dev = dhd->iflist[0]->net;
+		/* Send pkt UP */
+		dhd_netif_rx_ni(skb);
+	}
+}
+#endif /* EWP_EDL */
+#endif /* SHOW_LOGTRACE */
+
+#ifdef EWP_EDL
+static void
+dhd_edl_process_work(struct work_struct *work)
+{
+	struct delayed_work *dw = to_delayed_work(work);
+	struct dhd_info *dhd_info;
+	/* Ignore compiler warnings due to -Werror=cast-qual */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd_info = container_of(dw, struct dhd_info, edl_dispatcher_work);
+	GCC_DIAGNOSTIC_POP();
+
+	if (dhd_info)
+		dhd_prot_process_edl_complete(&dhd_info->pub, &dhd_info->event_data);
+}
+
+void
+dhd_schedule_edl_work(dhd_pub_t *dhdp, uint delay_ms)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	schedule_delayed_work(&dhd->edl_dispatcher_work, msecs_to_jiffies(delay_ms));
+}
+#endif /* EWP_EDL */
+
+#ifdef ENABLE_WAKEUP_PKT_DUMP
+static void
+update_wake_pkt_info(struct sk_buff *skb)
+{
+	struct iphdr *ip_header;
+	struct ipv6hdr *ipv6_header;
+	struct udphdr *udp_header;
+	struct tcphdr *tcp_header;
+	uint16 dport = 0;
+
+	ip_header = (struct iphdr *)(skb->data);
+
+	temp_raw |= ((long long)ntoh16(skb->protocol)) << 48;
+
+	DHD_INFO(("eth_hdr(skb)->h_dest : %pM\n", eth_hdr(skb)->h_dest));
+	if (eth_hdr(skb)->h_dest[0] & 0x01) {
+		temp_raw |= (long long)1 << 39;
+	}
+
+	if (ntoh16(skb->protocol) == ETHER_TYPE_BRCM) {
+		wl_event_msg_t event;
+		bcm_event_msg_u_t evu;
+		int ret;
+		uint event_type;
+
+		ret = wl_host_event_get_data(
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
+			skb_mac_header(skb),
+#else
+			skb->mac.raw,
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22) */
+			skb->len, &evu);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: wl_host_event_get_data err = %d\n",
+				__FUNCTION__, ret));
+		}
+
+		memcpy(&event, &evu.event, sizeof(wl_event_msg_t));
+		event_type = ntoh32_ua((void *)&event.event_type);
+
+		temp_raw |= (long long)event_type << 40;
+	} else if (ntoh16(skb->protocol) == ETHER_TYPE_IP ||
+			ntoh16(skb->protocol) == ETHER_TYPE_IPV6) {
+		if (ip_header->version == 6) {
+			ipv6_header = (struct ipv6hdr *)ip_header;
+			temp_raw |= ((long long)ipv6_header->nexthdr) << 40;
+			dport = 0;
+
+			if (ipv6_header->daddr.s6_addr[0] & 0xff) {
+				temp_raw |= (long long)1 << 38;
+			}
+
+			DHD_INFO(("IPv6 [%x]%pI6c > %pI6c:%d\n",
+				ip_header->protocol, &(ipv6_header->saddr.s6_addr),
+				&(ipv6_header->daddr.s6_addr), dport));
+		} else if (ip_header->version == 4) {
+			temp_raw |= ((long long)ip_header->protocol) << 40;
+
+#define IP_HDR_OFFSET	((char *)ip_header + IPV4_HLEN(ip_header))
+			if (ip_header->protocol == IPPROTO_TCP) {
+				tcp_header = (struct tcphdr *)IP_HDR_OFFSET;
+				dport = ntohs(tcp_header->dest);
+			}
+			else if (ip_header->protocol == IPPROTO_UDP) {
+				udp_header = (struct udphdr *)IP_HDR_OFFSET;
+				dport = ntohs(udp_header->dest);
+			}
+
+			if (ipv4_is_multicast(ip_header->daddr)) {
+				temp_raw |= (long long)1 << 38;
+			}
+
+			DHD_INFO(("IP [%x] %pI4 > %pI4:%d\n",
+				ip_header->protocol, &(ip_header->saddr),
+				&(ip_header->daddr), dport));
+		}
+
+		temp_raw |= (long long)dport << 16;
+	}
+}
+#endif /* ENABLE_WAKEUP_PKT_DUMP */
+
+#if defined(BCMPCIE)
+int
+dhd_check_shinfo_nrfrags(dhd_pub_t *dhdp, void *pktbuf,
+	dmaaddr_t *pa, uint32 pktid)
+{
+	struct sk_buff *skb;
+	struct skb_shared_info *shinfo;
+
+	if (!pktbuf)
+		return BCME_ERROR;
+
+	skb = PKTTONATIVE(dhdp->osh, pktbuf);
+	shinfo = skb_shinfo(skb);
+
+	if (shinfo->nr_frags) {
+#ifdef BCMDMA64OSL
+		DHD_ERROR(("!!Invalid nr_frags: %u pa.loaddr: 0x%llx pa.hiaddr: 0x%llx "
+			"skb: 0x%llx skb_data: 0x%llx skb_head: 0x%llx skb_tail: 0x%llx "
+			"skb_end: 0x%llx skb_len: %u shinfo: 0x%llx pktid: %u\n",
+			shinfo->nr_frags, (uint64)(pa->loaddr), (uint64)(pa->hiaddr),
+			(uint64)skb, (uint64)(skb->data), (uint64)(skb->head), (uint64)(skb->tail),
+			(uint64)(skb->end), skb->len, (uint64)shinfo, pktid));
+#else
+		DHD_ERROR(("!!Invalid nr_frags: %u "
+			"skb: 0x%x skb_data: 0x%x skb_head: 0x%x skb_tail: 0x%x "
+			"skb_end: 0x%x skb_len: %u shinfo: 0x%x pktid: %u\n",
+			shinfo->nr_frags,
+			(uint)skb, (uint)(skb->data), (uint)(skb->head), (uint)(skb->tail),
+			(uint)(skb->end), skb->len, (uint)shinfo, pktid));
+#endif /* BCMDMA64OSL */
+		prhex("shinfo", (char*)shinfo, sizeof(struct skb_shared_info));
+		if (!dhd_query_bus_erros(dhdp)) {
+#ifdef DHD_FW_COREDUMP
+			/* Collect socram dump */
+			if (dhdp->memdump_enabled) {
+				/* collect core dump */
+				dhdp->memdump_type = DUMP_TYPE_INVALID_SHINFO_NRFRAGS;
+				dhd_bus_mem_dump(dhdp);
+			} else
+#endif /* DHD_FW_COREDUMP */
+			{
+				shinfo->nr_frags = 0;
+				/* In production case, free the packet and continue
+				 * if nfrags is corrupted. Whereas in non-production
+				 * case collect memdump and call BUG_ON().
+				 */
+				PKTCFREE(dhdp->osh, pktbuf, FALSE);
+			}
+		}
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+#endif /* BCMPCIE */
+
+void
+dhd_event(struct dhd_info *dhd, char *evpkt, int evlen, int ifidx)
+{
+	/* Linux version has nothing to do */
+	return;
+}
+
+int dhd_os_tput_test_wait(dhd_pub_t *pub, uint *condition,
+		uint timeout_ms)
+{
+	int timeout;
+
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(timeout_ms);
+	pub->tput_test_done = FALSE;
+	condition = (uint *)&pub->tput_test_done;
+	timeout = wait_event_timeout(pub->tx_tput_test_wait,
+		(*condition), timeout);
+
+	return timeout;
+}
+
+int dhd_os_tput_test_wake(dhd_pub_t * pub)
+{
+	OSL_SMP_WMB();
+	pub->tput_test_done = TRUE;
+	OSL_SMP_WMB();
+	wake_up(&(pub->tx_tput_test_wait));
+	return 0;
+}
+
+static struct net_device_stats *
+dhd_get_stats(struct net_device *net)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+	dhd_if_t *ifp;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (!dhd) {
+		DHD_ERROR(("%s : dhd is NULL\n", __FUNCTION__));
+		goto error;
+	}
+
+	ifp = dhd_get_ifp_by_ndev(&dhd->pub, net);
+	if (!ifp) {
+		/* return empty stats */
+		DHD_ERROR(("%s: BAD_IF\n", __FUNCTION__));
+		goto error;
+	}
+
+	if (dhd->pub.up) {
+		/* Use the protocol to get dongle stats */
+		dhd_prot_dstats(&dhd->pub);
+	}
+	return &ifp->stats;
+
+error:
+	memset(&net->stats, 0, sizeof(net->stats));
+	return &net->stats;
+}
+
+#ifndef BCMDBUS
+static int
+dhd_watchdog_thread(void *data)
+{
+	tsk_ctl_t *tsk = (tsk_ctl_t *)data;
+	dhd_info_t *dhd = (dhd_info_t *)tsk->parent;
+	/* This thread doesn't need any user-level access,
+	 * so get rid of all our resources
+	 */
+	if (dhd_watchdog_prio > 0) {
+		struct sched_param param;
+		param.sched_priority = (dhd_watchdog_prio < MAX_RT_PRIO)?
+			dhd_watchdog_prio:(MAX_RT_PRIO-1);
+		setScheduler(current, SCHED_FIFO, &param);
+	}
+
+	while (1) {
+		if (down_interruptible (&tsk->sema) == 0) {
+			unsigned long flags;
+			unsigned long jiffies_at_start = jiffies;
+			unsigned long time_lapse;
+#ifdef BCMPCIE
+			DHD_OS_WD_WAKE_LOCK(&dhd->pub);
+#endif /* BCMPCIE */
+
+			SMP_RD_BARRIER_DEPENDS();
+			if (tsk->terminated) {
+#ifdef BCMPCIE
+				DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+#endif /* BCMPCIE */
+				break;
+			}
+
+			if (dhd->pub.dongle_reset == FALSE) {
+				DHD_TIMER(("%s:\n", __FUNCTION__));
+				dhd_analyze_sock_flows(dhd, dhd_watchdog_ms);
+				dhd_bus_watchdog(&dhd->pub);
+
+#if defined(BCM_ROUTER_DHD) && defined(CTFPOOL)
+				/* allocate and add a new skb to the pkt pool */
+				if (CTF_ENAB(dhd->cih))
+					osl_ctfpool_replenish(dhd->pub.osh, CTFPOOL_REFILL_THRESH);
+#endif /* BCM_ROUTER_DHD && CTFPOOL */
+
+				DHD_GENERAL_LOCK(&dhd->pub, flags);
+				/* Count the tick for reference */
+				dhd->pub.tickcnt++;
+#ifdef DHD_L2_FILTER
+				dhd_l2_filter_watchdog(&dhd->pub);
+#endif /* DHD_L2_FILTER */
+				time_lapse = jiffies - jiffies_at_start;
+
+				/* Reschedule the watchdog */
+				if (dhd->wd_timer_valid) {
+					mod_timer(&dhd->timer,
+					    jiffies +
+					    msecs_to_jiffies(dhd_watchdog_ms) -
+					    min(msecs_to_jiffies(dhd_watchdog_ms), time_lapse));
+				}
+				DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+			}
+#ifdef BCMPCIE
+			DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+#endif /* BCMPCIE */
+		} else {
+			break;
+		}
+	}
+
+	complete_and_exit(&tsk->completed, 0);
+}
+
+static void dhd_watchdog(ulong data)
+{
+	dhd_info_t *dhd = (dhd_info_t *)data;
+	unsigned long flags;
+
+	if (dhd->pub.dongle_reset) {
+		return;
+	}
+
+	if (dhd->thr_wdt_ctl.thr_pid >= 0) {
+		up(&dhd->thr_wdt_ctl.sema);
+		return;
+	}
+
+#ifdef BCMPCIE
+	DHD_OS_WD_WAKE_LOCK(&dhd->pub);
+#endif /* BCMPCIE */
+	/* Call the bus module watchdog */
+	dhd_bus_watchdog(&dhd->pub);
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	/* Count the tick for reference */
+	dhd->pub.tickcnt++;
+
+#ifdef DHD_L2_FILTER
+	dhd_l2_filter_watchdog(&dhd->pub);
+#endif /* DHD_L2_FILTER */
+	/* Reschedule the watchdog */
+	if (dhd->wd_timer_valid)
+		mod_timer(&dhd->timer, jiffies + msecs_to_jiffies(dhd_watchdog_ms));
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+#ifdef BCMPCIE
+	DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+#endif /* BCMPCIE */
+#if defined(BCM_ROUTER_DHD) && defined(CTFPOOL)
+	    /* allocate and add a new skb to the pkt pool */
+	    if (CTF_ENAB(dhd->cih))
+			osl_ctfpool_replenish(dhd->pub.osh, CTFPOOL_REFILL_THRESH);
+#endif /* BCM_ROUTER_DHD && CTFPOOL */
+}
+
+#ifdef ENABLE_ADAPTIVE_SCHED
+static void
+dhd_sched_policy(int prio)
+{
+	struct sched_param param;
+	if (cpufreq_quick_get(0) <= CUSTOM_CPUFREQ_THRESH) {
+		param.sched_priority = 0;
+		setScheduler(current, SCHED_NORMAL, &param);
+	} else {
+		if (get_scheduler_policy(current) != SCHED_FIFO) {
+			param.sched_priority = (prio < MAX_RT_PRIO)? prio : (MAX_RT_PRIO-1);
+			setScheduler(current, SCHED_FIFO, &param);
+		}
+	}
+}
+#endif /* ENABLE_ADAPTIVE_SCHED */
+#ifdef DEBUG_CPU_FREQ
+static int dhd_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
+{
+	dhd_info_t *dhd = container_of(nb, struct dhd_info, freq_trans);
+	struct cpufreq_freqs *freq = data;
+	if (dhd) {
+		if (!dhd->new_freq)
+			goto exit;
+		if (val == CPUFREQ_POSTCHANGE) {
+			DHD_ERROR(("cpu freq is changed to %u kHZ on CPU %d\n",
+				freq->new, freq->cpu));
+			*per_cpu_ptr(dhd->new_freq, freq->cpu) = freq->new;
+		}
+	}
+exit:
+	return 0;
+}
+#endif /* DEBUG_CPU_FREQ */
+
+static int
+dhd_dpc_thread(void *data)
+{
+	tsk_ctl_t *tsk = (tsk_ctl_t *)data;
+	dhd_info_t *dhd = (dhd_info_t *)tsk->parent;
+
+	/* This thread doesn't need any user-level access,
+	 * so get rid of all our resources
+	 */
+	if (dhd_dpc_prio > 0)
+	{
+		struct sched_param param;
+		param.sched_priority = (dhd_dpc_prio < MAX_RT_PRIO)?dhd_dpc_prio:(MAX_RT_PRIO-1);
+		setScheduler(current, SCHED_FIFO, &param);
+	}
+
+#ifdef CUSTOM_DPC_CPUCORE
+	set_cpus_allowed_ptr(current, cpumask_of(CUSTOM_DPC_CPUCORE));
+#endif
+#ifdef CUSTOM_SET_CPUCORE
+	dhd->pub.current_dpc = current;
+#endif /* CUSTOM_SET_CPUCORE */
+	/* Run until signal received */
+	while (1) {
+		if (dhd->pub.conf->dpc_cpucore >= 0) {
+			printf("%s: set dpc_cpucore %d\n", __FUNCTION__, dhd->pub.conf->dpc_cpucore);
+			set_cpus_allowed_ptr(current, cpumask_of(dhd->pub.conf->dpc_cpucore));
+			dhd->pub.conf->dpc_cpucore = -1;
+		}
+		if (dhd->pub.conf->dhd_dpc_prio >= 0) {
+			struct sched_param param;
+			printf("%s: set dhd_dpc_prio %d\n", __FUNCTION__, dhd->pub.conf->dhd_dpc_prio);
+			param.sched_priority = (dhd->pub.conf->dhd_dpc_prio < MAX_RT_PRIO)?
+				dhd->pub.conf->dhd_dpc_prio:(MAX_RT_PRIO-1);
+			setScheduler(current, SCHED_FIFO, &param);
+			dhd->pub.conf->dhd_dpc_prio = -1;
+		}
+		if (!binary_sema_down(tsk)) {
+#ifdef ENABLE_ADAPTIVE_SCHED
+			dhd_sched_policy(dhd_dpc_prio);
+#endif /* ENABLE_ADAPTIVE_SCHED */
+			SMP_RD_BARRIER_DEPENDS();
+			if (tsk->terminated) {
+				DHD_OS_WAKE_UNLOCK(&dhd->pub);
+				break;
+			}
+
+			/* Call bus dpc unless it indicated down (then clean stop) */
+			if (dhd->pub.busstate != DHD_BUS_DOWN) {
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+				int resched_cnt = 0;
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+				dhd_os_wd_timer_extend(&dhd->pub, TRUE);
+				while (dhd_bus_dpc(dhd->pub.bus)) {
+					/* process all data */
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+					resched_cnt++;
+					if (resched_cnt > MAX_RESCHED_CNT) {
+						DHD_INFO(("%s Calling msleep to"
+							"let other processes run. \n",
+							__FUNCTION__));
+						dhd->pub.dhd_bug_on = true;
+						resched_cnt = 0;
+						OSL_SLEEP(1);
+					}
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+				}
+				dhd_os_wd_timer_extend(&dhd->pub, FALSE);
+				DHD_OS_WAKE_UNLOCK(&dhd->pub);
+			} else {
+				if (dhd->pub.up)
+					dhd_bus_stop(dhd->pub.bus, TRUE);
+				DHD_OS_WAKE_UNLOCK(&dhd->pub);
+			}
+		} else {
+			break;
+		}
+	}
+	complete_and_exit(&tsk->completed, 0);
+}
+
+#ifdef BCMPCIE
+void dhd_dpc_enable(dhd_pub_t *dhdp)
+{
+#if defined(DHD_LB_RXP) || defined(DHD_LB_TXP)
+	dhd_info_t *dhd;
+
+	if (!dhdp || !dhdp->info)
+		return;
+	dhd = dhdp->info;
+#endif /* DHD_LB_RXP || DHD_LB_TXP */
+
+#ifdef DHD_LB_RXP
+	__skb_queue_head_init(&dhd->rx_pend_queue);
+	skb_queue_head_init(&dhd->rx_emerge_queue);
+#endif /* DHD_LB_RXP */
+
+#ifdef DHD_LB_TXP
+	skb_queue_head_init(&dhd->tx_pend_queue);
+#endif /* DHD_LB_TXP */
+}
+#endif /* BCMPCIE */
+
+#ifdef BCMPCIE
+void
+dhd_dpc_kill(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+
+	if (!dhdp) {
+		return;
+	}
+
+	dhd = dhdp->info;
+
+	if (!dhd) {
+		return;
+	}
+
+	if (dhd->thr_dpc_ctl.thr_pid < 0) {
+		tasklet_kill(&dhd->tasklet);
+		DHD_ERROR(("%s: tasklet disabled\n", __FUNCTION__));
+	}
+
+	cancel_delayed_work_sync(&dhd->dhd_dpc_dispatcher_work);
+#ifdef DHD_LB
+#ifdef DHD_LB_RXP
+	cancel_work_sync(&dhd->rx_napi_dispatcher_work);
+	__skb_queue_purge(&dhd->rx_pend_queue);
+	skb_queue_purge(&dhd->rx_emerge_queue);
+#endif /* DHD_LB_RXP */
+#ifdef DHD_LB_TXP
+	cancel_work_sync(&dhd->tx_dispatcher_work);
+	skb_queue_purge(&dhd->tx_pend_queue);
+	tasklet_kill(&dhd->tx_tasklet);
+#endif /* DHD_LB_TXP */
+#endif /* DHD_LB */
+}
+
+void
+dhd_dpc_tasklet_kill(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+
+	if (!dhdp) {
+		return;
+	}
+
+	dhd = dhdp->info;
+
+	if (!dhd) {
+		return;
+	}
+
+	if (dhd->thr_dpc_ctl.thr_pid < 0) {
+		tasklet_kill(&dhd->tasklet);
+	}
+}
+#endif /* BCMPCIE */
+
+static void
+dhd_dpc(ulong data)
+{
+	dhd_info_t *dhd = (dhd_info_t *)data;
+	int curr_cpu = get_cpu();
+
+	put_cpu();
+
+	/* Store current cpu as dpc_cpu */
+	atomic_set(&dhd->dpc_cpu, curr_cpu);
+
+	/* this (tasklet) can be scheduled in dhd_sched_dpc[dhd_linux.c]
+	 * down below , wake lock is set,
+	 * the tasklet is initialized in dhd_attach()
+	 */
+	/* Call bus dpc unless it indicated down (then clean stop) */
+	if (dhd->pub.busstate != DHD_BUS_DOWN) {
+#if defined(DHD_LB_STATS) && defined(PCIE_FULL_DONGLE)
+		DHD_LB_STATS_INCR(dhd->dhd_dpc_cnt);
+#endif /* DHD_LB_STATS && PCIE_FULL_DONGLE */
+		if (dhd_bus_dpc(dhd->pub.bus)) {
+			tasklet_schedule(&dhd->tasklet);
+			dhd_plat_report_bh_sched(dhd->pub.plat_info, 1);
+		} else {
+			dhd_plat_report_bh_sched(dhd->pub.plat_info, 0);
+		}
+	} else {
+		dhd_bus_stop(dhd->pub.bus, TRUE);
+	}
+
+	/* Store as prev_dpc_cpu, which will be used in Rx load balancing for deciding candidacy */
+	atomic_set(&dhd->prev_dpc_cpu, curr_cpu);
+
+}
+
+void
+dhd_sched_dpc(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+
+	if (dhd->thr_dpc_ctl.thr_pid >= 0) {
+		DHD_OS_WAKE_LOCK(dhdp);
+		/* If the semaphore does not get up,
+		* wake unlock should be done here
+		*/
+		if (!binary_sema_up(&dhd->thr_dpc_ctl)) {
+			DHD_OS_WAKE_UNLOCK(dhdp);
+		}
+		return;
+	} else {
+		tasklet_schedule(&dhd->tasklet);
+	}
+}
+#endif /* BCMDBUS */
+
+#ifdef TOE
+/* Retrieve current toe component enables, which are kept as a bitmap in toe_ol iovar */
+static int
+dhd_toe_get(dhd_info_t *dhd, int ifidx, uint32 *toe_ol)
+{
+	char buf[32];
+	int ret;
+
+	ret = dhd_iovar(&dhd->pub, ifidx, "toe_ol", NULL, 0, (char *)&buf, sizeof(buf), FALSE);
+
+	if (ret < 0) {
+		if (ret == -EIO) {
+			DHD_ERROR(("%s: toe not supported by device\n", dhd_ifname(&dhd->pub,
+				ifidx)));
+			return -EOPNOTSUPP;
+		}
+
+		DHD_INFO(("%s: could not get toe_ol: ret=%d\n", dhd_ifname(&dhd->pub, ifidx), ret));
+		return ret;
+	}
+
+	memcpy(toe_ol, buf, sizeof(uint32));
+	return 0;
+}
+
+/* Set current toe component enables in toe_ol iovar, and set toe global enable iovar */
+static int
+dhd_toe_set(dhd_info_t *dhd, int ifidx, uint32 toe_ol)
+{
+	int toe, ret;
+
+	/* Set toe_ol as requested */
+	ret = dhd_iovar(&dhd->pub, ifidx, "toe_ol", (char *)&toe_ol, sizeof(toe_ol), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: could not set toe_ol: ret=%d\n",
+			dhd_ifname(&dhd->pub, ifidx), ret));
+		return ret;
+	}
+
+	/* Enable toe globally only if any components are enabled. */
+	toe = (toe_ol != 0);
+	ret = dhd_iovar(&dhd->pub, ifidx, "toe", (char *)&toe, sizeof(toe), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: could not set toe: ret=%d\n", dhd_ifname(&dhd->pub, ifidx), ret));
+		return ret;
+	}
+
+	return 0;
+}
+#endif /* TOE */
+
+#if defined(WL_CFG80211) && defined(NUM_SCB_MAX_PROBE)
+void dhd_set_scb_probe(dhd_pub_t *dhd)
+{
+	wl_scb_probe_t scb_probe;
+	char iovbuf[WL_EVENTING_MASK_LEN + sizeof(wl_scb_probe_t)];
+	int ret;
+
+	if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+		return;
+	}
+
+	ret = dhd_iovar(dhd, 0, "scb_probe", NULL, 0, iovbuf, sizeof(iovbuf), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: GET max_scb_probe failed\n", __FUNCTION__));
+	}
+
+	memcpy(&scb_probe, iovbuf, sizeof(wl_scb_probe_t));
+
+	scb_probe.scb_max_probe = NUM_SCB_MAX_PROBE;
+
+	ret = dhd_iovar(dhd, 0, "scb_probe", (char *)&scb_probe, sizeof(wl_scb_probe_t), NULL, 0,
+			TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: max_scb_probe setting failed\n", __FUNCTION__));
+		return;
+	}
+}
+#endif /* WL_CFG80211 && NUM_SCB_MAX_PROBE */
+
+static void
+dhd_ethtool_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *info)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+
+	snprintf(info->driver, sizeof(info->driver), "wl");
+	snprintf(info->version, sizeof(info->version), "%lu", dhd->pub.drv_version);
+}
+
+struct ethtool_ops dhd_ethtool_ops = {
+	.get_drvinfo = dhd_ethtool_get_drvinfo
+};
+
+static int
+dhd_ethtool(dhd_info_t *dhd, void *uaddr)
+{
+	struct ethtool_drvinfo info;
+	char drvname[sizeof(info.driver)];
+	uint32 cmd;
+#ifdef TOE
+	struct ethtool_value edata;
+	uint32 toe_cmpnt, csum_dir;
+	int ret;
+#endif
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	/* all ethtool calls start with a cmd word */
+	if (copy_from_user(&cmd, uaddr, sizeof (uint32)))
+		return -EFAULT;
+
+	switch (cmd) {
+	case ETHTOOL_GDRVINFO:
+		/* Copy out any request driver name */
+		bzero(&info.driver, sizeof(info.driver));
+		if (copy_from_user(&info, uaddr, sizeof(info)))
+			return -EFAULT;
+		if (info.driver[sizeof(info.driver) - 1] != '\0') {
+			DHD_ERROR(("%s: Exceeds the size of info.driver"
+				"truncating last byte with null\n", __FUNCTION__));
+			info.driver[sizeof(info.driver) - 1] = '\0';
+		}
+		strlcpy(drvname, info.driver, sizeof(drvname));
+
+		/* clear struct for return */
+		memset(&info, 0, sizeof(info));
+		info.cmd = cmd;
+
+		/* if dhd requested, identify ourselves */
+		if (strcmp(drvname, "?dhd") == 0) {
+			snprintf(info.driver, sizeof(info.driver), "dhd");
+			strlcpy(info.version, EPI_VERSION_STR, sizeof(info.version));
+		}
+
+		/* otherwise, require dongle to be up */
+		else if (!dhd->pub.up) {
+			DHD_ERROR(("%s: dongle is not up\n", __FUNCTION__));
+			return -ENODEV;
+		}
+
+		/* finally, report dongle driver type */
+		else if (dhd->pub.iswl)
+			snprintf(info.driver, sizeof(info.driver), "wl");
+		else
+			snprintf(info.driver, sizeof(info.driver), "xx");
+
+		snprintf(info.version, sizeof(info.version), "%lu", dhd->pub.drv_version);
+		if (copy_to_user(uaddr, &info, sizeof(info)))
+			return -EFAULT;
+		DHD_CTL(("%s: given %*s, returning %s\n", __FUNCTION__,
+		         (int)sizeof(drvname), drvname, info.driver));
+		break;
+
+#ifdef TOE
+	/* Get toe offload components from dongle */
+	case ETHTOOL_GRXCSUM:
+	case ETHTOOL_GTXCSUM:
+		if ((ret = dhd_toe_get(dhd, 0, &toe_cmpnt)) < 0)
+			return ret;
+
+		csum_dir = (cmd == ETHTOOL_GTXCSUM) ? TOE_TX_CSUM_OL : TOE_RX_CSUM_OL;
+
+		edata.cmd = cmd;
+		edata.data = (toe_cmpnt & csum_dir) ? 1 : 0;
+
+		if (copy_to_user(uaddr, &edata, sizeof(edata)))
+			return -EFAULT;
+		break;
+
+	/* Set toe offload components in dongle */
+	case ETHTOOL_SRXCSUM:
+	case ETHTOOL_STXCSUM:
+		if (copy_from_user(&edata, uaddr, sizeof(edata)))
+			return -EFAULT;
+
+		/* Read the current settings, update and write back */
+		if ((ret = dhd_toe_get(dhd, 0, &toe_cmpnt)) < 0)
+			return ret;
+
+		csum_dir = (cmd == ETHTOOL_STXCSUM) ? TOE_TX_CSUM_OL : TOE_RX_CSUM_OL;
+
+		if (edata.data != 0)
+			toe_cmpnt |= csum_dir;
+		else
+			toe_cmpnt &= ~csum_dir;
+
+		if ((ret = dhd_toe_set(dhd, 0, toe_cmpnt)) < 0)
+			return ret;
+
+		/* If setting TX checksum mode, tell Linux the new mode */
+		if (cmd == ETHTOOL_STXCSUM) {
+			if (edata.data)
+				dhd->iflist[0]->net->features |= NETIF_F_IP_CSUM;
+			else
+				dhd->iflist[0]->net->features &= ~NETIF_F_IP_CSUM;
+		}
+
+		break;
+#endif /* TOE */
+
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+/* XXX function to detect that FW is dead and send Event up */
+static bool dhd_check_hang(struct net_device *net, dhd_pub_t *dhdp, int error)
+{
+#if defined(OEM_ANDROID)
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	if (!dhdp->up)
+		return FALSE;
+
+#if (!defined(BCMDBUS) && !defined(BCMPCIE))
+	if (dhdp->info->thr_dpc_ctl.thr_pid < 0) {
+		DHD_ERROR(("%s : skipped due to negative pid - unloading?\n", __FUNCTION__));
+		return FALSE;
+	}
+#endif /* BCMDBUS */
+
+	if ((error == -ETIMEDOUT) || (error == -EREMOTEIO) ||
+		((dhdp->busstate == DHD_BUS_DOWN) && (!dhdp->dongle_reset))) {
+#ifdef BCMPCIE
+		DHD_ERROR(("%s: Event HANG send up due to  re=%d te=%d d3acke=%d e=%d s=%d\n",
+			__FUNCTION__, dhdp->rxcnt_timeout, dhdp->txcnt_timeout,
+			dhdp->d3ackcnt_timeout, error, dhdp->busstate));
+#else
+		DHD_ERROR(("%s: Event HANG send up due to  re=%d te=%d e=%d s=%d\n", __FUNCTION__,
+			dhdp->rxcnt_timeout, dhdp->txcnt_timeout, error, dhdp->busstate));
+#endif /* BCMPCIE */
+		if (dhdp->hang_reason == 0) {
+			if (dhdp->dongle_trap_occured) {
+				dhdp->hang_reason = HANG_REASON_DONGLE_TRAP;
+#ifdef BCMPCIE
+			} else if (dhdp->d3ackcnt_timeout) {
+				dhdp->hang_reason = dhdp->is_sched_error ?
+					HANG_REASON_D3_ACK_TIMEOUT_SCHED_ERROR :
+					HANG_REASON_D3_ACK_TIMEOUT;
+#endif /* BCMPCIE */
+			} else {
+				dhdp->hang_reason = dhdp->is_sched_error ?
+					HANG_REASON_IOCTL_RESP_TIMEOUT_SCHED_ERROR :
+					HANG_REASON_IOCTL_RESP_TIMEOUT;
+			}
+		}
+		printf("%s\n", info_string);
+		printf("MAC %pM\n", &dhdp->mac);
+		net_os_send_hang_message(net);
+		return TRUE;
+	}
+#endif /* OEM_ANDROID */
+	return FALSE;
+}
+
+#if defined(DBG_PKT_MON)
+#ifdef PCIE_FULL_DONGLE
+void
+dhd_80211_mon_pkt(dhd_pub_t *dhdp, host_rxbuf_cmpl_t* msg, void *pkt, int ifidx)
+{
+	/* Distinguish rx/tx frame */
+	wl_aml_header_v1_t hdr;
+
+	hdr = *(wl_aml_header_v1_t *)PKTDATA(dhdp->osh, pkt);
+	PKTPULL(dhdp->osh, pkt, sizeof(hdr));
+	if (hdr.flags & WL_AML_F_DIRECTION) {
+		bool ack = !!(hdr.flags & WL_AML_F_ACKED);
+		DHD_DBG_PKT_MON_TX(dhdp, pkt, 0, FRAME_TYPE_80211_MGMT, (uint8)ack);
+	} else {
+		DHD_DBG_PKT_MON_RX(dhdp, (struct sk_buff *)pkt, FRAME_TYPE_80211_MGMT);
+	}
+}
+#else
+bool
+dhd_80211_mon_pkt(dhd_pub_t *dhdp, void *pkt, int ifidx)
+{
+	/* Distinguish rx/tx frame */
+	wl_aml_header_v1_t hdr;
+
+	hdr = *(wl_aml_header_v1_t *)PKTDATA(dhdp->osh, pkt);
+	if (hdr.version == WL_AML_HEADER_VERSION) {
+		if (hdr.flags & WL_AML_F_MGMT) {
+			PKTPULL(dhdp->osh, pkt, sizeof(hdr));
+			if (hdr.flags & WL_AML_F_DIRECTION) {
+				bool ack = !!(hdr.flags & WL_AML_F_ACKED);
+				DHD_DBG_PKT_MON_TX(dhdp, pkt, 0, FRAME_TYPE_80211_MGMT, (uint8)ack);
+			} else {
+				DHD_DBG_PKT_MON_RX(dhdp, (struct sk_buff *)pkt,
+					FRAME_TYPE_80211_MGMT);
+			}
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+#endif /* PCIE_FULL_DONGLE */
+#endif /* DBG_PKT_MON */
+
+#if defined(BT_OVER_SDIO)
+
+void
+dhdsdio_bus_usr_cnt_inc(dhd_pub_t *dhdp)
+{
+	dhdp->info->bus_user_count++;
+}
+
+void
+dhdsdio_bus_usr_cnt_dec(dhd_pub_t *dhdp)
+{
+	dhdp->info->bus_user_count--;
+}
+
+/* Return values:
+ * Success: Returns 0
+ * Failure: Returns -1 or errono code
+ */
+int
+dhd_bus_get(wlan_bt_handle_t handle, bus_owner_t owner)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	int ret = 0;
+
+	mutex_lock(&dhd->bus_user_lock);
+	++dhd->bus_user_count;
+	if (dhd->bus_user_count < 0) {
+		DHD_ERROR(("%s(): bus_user_count is negative, which is invalid\n", __FUNCTION__));
+		ret = -1;
+		goto exit;
+	}
+
+	if (dhd->bus_user_count == 1) {
+
+		dhd->pub.hang_was_sent = 0;
+
+		/* First user, turn on WL_REG, start the bus */
+		DHD_ERROR(("%s(): First user Turn On WL_REG & start the bus", __FUNCTION__));
+
+		if (!wifi_platform_set_power(dhd->adapter, TRUE, WIFI_TURNON_DELAY)) {
+			/* Enable F1 */
+			ret = dhd_bus_resume(dhdp, 0);
+			if (ret) {
+				DHD_ERROR(("%s(): Failed to enable F1, err=%d\n",
+					__FUNCTION__, ret));
+				goto exit;
+			}
+		}
+
+		/* XXX Some DHD modules (e.g. cfg80211) configures operation mode based on firmware
+		 * name. This is indeed a hack but we have to make it work properly before we have
+		 * a better solution
+		 */
+		dhd_update_fw_nv_path(dhd);
+		/* update firmware and nvram path to sdio bus */
+		dhd_bus_update_fw_nv_path(dhd->pub.bus,
+			dhd->fw_path, dhd->nv_path);
+		/* download the firmware, Enable F2 */
+		/* TODO: Should be done only in case of FW switch */
+		ret = dhd_bus_devreset(dhdp, FALSE);
+		dhd_bus_resume(dhdp, 1);
+		if (!ret) {
+			if (dhd_sync_with_dongle(&dhd->pub) < 0) {
+				DHD_ERROR(("%s(): Sync with dongle failed!!\n", __FUNCTION__));
+				ret = -EFAULT;
+			}
+		} else {
+			DHD_ERROR(("%s(): Failed to download, err=%d\n", __FUNCTION__, ret));
+		}
+	} else {
+		DHD_ERROR(("%s(): BUS is already acquired, just increase the count %d \r\n",
+			__FUNCTION__, dhd->bus_user_count));
+	}
+exit:
+	mutex_unlock(&dhd->bus_user_lock);
+	return ret;
+}
+EXPORT_SYMBOL(dhd_bus_get);
+
+/* Return values:
+ * Success: Returns 0
+ * Failure: Returns -1 or errono code
+ */
+int
+dhd_bus_put(wlan_bt_handle_t handle, bus_owner_t owner)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	int ret = 0;
+	BCM_REFERENCE(owner);
+
+	mutex_lock(&dhd->bus_user_lock);
+	--dhd->bus_user_count;
+	if (dhd->bus_user_count < 0) {
+		DHD_ERROR(("%s(): bus_user_count is negative, which is invalid\n", __FUNCTION__));
+		dhd->bus_user_count = 0;
+		ret = -1;
+		goto exit;
+	}
+
+	if (dhd->bus_user_count == 0) {
+		/* Last user, stop the bus and turn Off WL_REG */
+		DHD_ERROR(("%s(): There are no owners left Trunf Off WL_REG & stop the bus \r\n",
+			__FUNCTION__));
+#ifdef PROP_TXSTATUS
+		if (dhd->pub.wlfc_enabled) {
+			dhd_wlfc_deinit(&dhd->pub);
+		}
+#endif /* PROP_TXSTATUS */
+#ifdef PNO_SUPPORT
+		if (dhd->pub.pno_state) {
+			dhd_pno_deinit(&dhd->pub);
+		}
+#endif /* PNO_SUPPORT */
+#ifdef RTT_SUPPORT
+		if (dhd->pub.rtt_state) {
+			dhd_rtt_deinit(&dhd->pub);
+		}
+#endif /* RTT_SUPPORT */
+		ret = dhd_bus_devreset(dhdp, TRUE);
+		if (!ret) {
+			dhd_bus_suspend(dhdp);
+			wifi_platform_set_power(dhd->adapter, FALSE, WIFI_TURNOFF_DELAY);
+		}
+	} else {
+		DHD_ERROR(("%s(): Other owners using bus, decrease the count %d \r\n",
+			__FUNCTION__, dhd->bus_user_count));
+	}
+exit:
+	mutex_unlock(&dhd->bus_user_lock);
+	return ret;
+}
+EXPORT_SYMBOL(dhd_bus_put);
+
+int
+dhd_net_bus_get(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return dhd_bus_get(&dhd->pub, WLAN_MODULE);
+}
+
+int
+dhd_net_bus_put(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return dhd_bus_put(&dhd->pub, WLAN_MODULE);
+}
+
+/*
+ * Function to enable the Bus Clock
+ * Returns BCME_OK on success and BCME_xxx on failure
+ *
+ * This function is not callable from non-sleepable context
+ */
+int dhd_bus_clk_enable(wlan_bt_handle_t handle, bus_owner_t owner)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+
+	int ret;
+
+	dhd_os_sdlock(dhdp);
+	/*
+	 * The second argument is TRUE, that means, we expect
+	 * the function to "wait" until the clocks are really
+	 * available
+	 */
+	ret = __dhdsdio_clk_enable(dhdp->bus, owner, TRUE);
+	dhd_os_sdunlock(dhdp);
+
+	return ret;
+}
+EXPORT_SYMBOL(dhd_bus_clk_enable);
+
+/*
+ * Function to disable the Bus Clock
+ * Returns BCME_OK on success and BCME_xxx on failure
+ *
+ * This function is not callable from non-sleepable context
+ */
+int dhd_bus_clk_disable(wlan_bt_handle_t handle, bus_owner_t owner)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+
+	int ret;
+
+	dhd_os_sdlock(dhdp);
+	/*
+	 * The second argument is TRUE, that means, we expect
+	 * the function to "wait" until the clocks are really
+	 * disabled
+	 */
+	ret = __dhdsdio_clk_disable(dhdp->bus, owner, TRUE);
+	dhd_os_sdunlock(dhdp);
+
+	return ret;
+}
+EXPORT_SYMBOL(dhd_bus_clk_disable);
+
+/*
+ * Function to reset bt_use_count counter to zero.
+ *
+ * This function is not callable from non-sleepable context
+ */
+void dhd_bus_reset_bt_use_count(wlan_bt_handle_t handle)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+
+	/* take the lock and reset bt use count */
+	dhd_os_sdlock(dhdp);
+	dhdsdio_reset_bt_use_count(dhdp->bus);
+	dhd_os_sdunlock(dhdp);
+}
+EXPORT_SYMBOL(dhd_bus_reset_bt_use_count);
+
+void dhd_bus_retry_hang_recovery(wlan_bt_handle_t handle)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+	dhd_info_t *dhd = (dhd_info_t*)dhdp->info;
+
+	dhdp->hang_was_sent = 0;
+
+	dhd_os_send_hang_message(&dhd->pub);
+}
+EXPORT_SYMBOL(dhd_bus_retry_hang_recovery);
+#endif /* BT_OVER_SDIO */
+
+void dhd_unregister_net(struct net_device *net, bool need_rtnl_lock)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	if (need_rtnl_lock) {
+		rtnl_lock();
+		cfg80211_unregister_netdevice(net);
+		rtnl_unlock();
+	} else {
+		cfg80211_unregister_netdevice(net);
+	}
+#else
+	if (need_rtnl_lock) {
+		unregister_netdev(net);
+	} else {
+		unregister_netdevice(net);
+	}
+#endif /* KERNEL_VER >= KERNEL_VERSION(5, 15, 0) */
+	return;
+}
+
+int dhd_register_net(struct net_device *net, bool need_rtnl_lock)
+{
+	int err = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	if (need_rtnl_lock) {
+		rtnl_lock();
+		err = cfg80211_register_netdevice(net);
+		rtnl_unlock();
+	} else {
+		err = cfg80211_register_netdevice(net);
+	}
+#else
+	if (need_rtnl_lock) {
+		err = register_netdev(net);
+	} else {
+		err = register_netdevice(net);
+	}
+#endif /* KERNEL_VER >= KERNEL_VERSION(5, 15, 0) */
+	return err;
+}
+
+#ifdef WL_MONITOR
+bool
+dhd_monitor_enabled(dhd_pub_t *dhd, int ifidx)
+{
+	return (dhd->info->monitor_type != 0);
+}
+
+typedef struct dhd_mon_dev_priv {
+	dhd_info_t *dhd;
+	struct net_device_stats stats;
+} dhd_mon_dev_priv_t;
+
+#define DHD_MON_DEV_PRIV_SIZE		(sizeof(dhd_mon_dev_priv_t))
+#define DHD_MON_DEV_PRIV(dev)		((dhd_mon_dev_priv_t *)DEV_PRIV(dev))
+#define DHD_MON_DEV_INFO(dev)		(((dhd_mon_dev_priv_t *)DEV_PRIV(dev))->dhd)
+#define DHD_MON_DEV_STATS(dev)		(((dhd_mon_dev_priv_t *)DEV_PRIV(dev))->stats)
+
+static netdev_tx_t
+dhd_monitor_start(struct sk_buff *skb, struct net_device *dev)
+{
+	PKTFREE(NULL, skb, FALSE);
+	return NETDEV_TX_OK;
+}
+
+#ifdef WL_CFG80211_MONITOR
+static int
+dhd_set_monitor_ioctl(dhd_pub_t *dhdp, bool val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	int ret = 0;
+	uint monitor = (uint)val;
+
+	DHD_TRACE(("%s: val %d\n", __FUNCTION__, val));
+
+	if ((ret = dhd_wl_ioctl_cmd(dhdp, WLC_SET_MONITOR, &monitor,
+			sizeof(monitor), TRUE, 0)) != 0) {
+		DHD_ERROR(("%s Failed to set monitor mode, err %d\n",
+			__FUNCTION__, ret));
+	} else {
+		dhd_net_if_lock_local(dhd);
+		dhd->monitor_type = monitor;
+		dhd_net_if_unlock_local(dhd);
+	}
+
+	return ret;
+}
+
+static int
+dhd_monitor_open(struct net_device *net)
+{
+	int ret = 0;
+	dhd_info_t *dhd = DHD_MON_DEV_INFO(net);
+	uint32 scan_suppress = FALSE;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (!dhd->pub.monitor_enable) {
+		DHD_ERROR(("%s: Monitor mode is not enabled in FW cap\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	ret = dhd_set_monitor_ioctl(&dhd->pub, TRUE);
+	if (ret) {
+		DHD_ERROR(("%s: Set monitor mode IOCTL failed.\n", __FUNCTION__));
+		return ret;
+	}
+
+	if (FW_SUPPORTED((&dhd->pub), monitor)) {
+		scan_suppress = TRUE;
+		/* Set the SCAN SUPPRESS Flag in the firmware to disable scan in Monitor mode */
+		ret = dhd_iovar(&dhd->pub, 0, "scansuppress", (char *)&scan_suppress,
+			sizeof(scan_suppress), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: scansuppress set failed, ret=%d\n", __FUNCTION__, ret));
+		}
+	}
+
+	return ret;
+}
+
+static int
+dhd_monitor_stop(struct net_device *net)
+{
+	int ret = 0;
+	dhd_info_t *dhd = DHD_MON_DEV_INFO(net);
+	uint32 scan_suppress = FALSE;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (!dhd->pub.monitor_enable) {
+		DHD_ERROR(("%s: Monitor mode is not enabled in FW cap\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (FW_SUPPORTED((&dhd->pub), monitor)) {
+		scan_suppress = FALSE;
+		/* Unset the SCAN SUPPRESS Flag in the firmware to enable scan */
+		ret = dhd_iovar(&dhd->pub, 0, "scansuppress", (char *)&scan_suppress,
+				sizeof(scan_suppress), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: scansuppress set failed, ret=%d\n", __FUNCTION__, ret));
+		}
+	}
+
+	ret = dhd_set_monitor_ioctl(&dhd->pub, FALSE);
+	if (ret) {
+		DHD_ERROR(("%s: Set monitor mode IOCTL failed.\n", __FUNCTION__));
+		return ret;
+	}
+
+	return ret;
+}
+#endif /* WL_CFG80211_MONITOR */
+
+static int
+dhd_monitor_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+{
+	return 0;
+}
+
+static struct net_device_stats*
+dhd_monitor_get_stats(struct net_device *dev)
+{
+	return &DHD_MON_DEV_STATS(dev);
+}
+
+static const struct net_device_ops netdev_monitor_ops =
+{
+	.ndo_start_xmit = dhd_monitor_start,
+	.ndo_get_stats = dhd_monitor_get_stats,
+#ifdef WL_CFG80211_MONITOR
+	.ndo_open = dhd_monitor_open,
+	.ndo_stop = dhd_monitor_stop,
+#endif /* WL_CFG80211_MONITOR */
+	.ndo_do_ioctl = dhd_monitor_ioctl
+};
+
+static void
+dhd_add_monitor_if(dhd_info_t *dhd)
+{
+	struct net_device *dev;
+	char *devname;
+#ifdef HOST_RADIOTAP_CONV
+	dhd_pub_t *dhdp = (dhd_pub_t *)&dhd->pub;
+#endif /* HOST_RADIOTAP_CONV */
+	dhd_mon_dev_priv_t *dev_priv;
+
+	BCM_REFERENCE(dev_priv);
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	if (dhd->monitor_dev) {
+		DHD_ERROR(("%s: monitor i/f already exists", __FUNCTION__));
+		return;
+	}
+
+	dev = alloc_etherdev(DHD_MON_DEV_PRIV_SIZE);
+	if (!dev) {
+		DHD_ERROR(("%s: alloc wlif failed\n", __FUNCTION__));
+		return;
+	}
+
+	devname = "radiotap";
+
+	snprintf(dev->name, sizeof(dev->name), "%s%u", devname, dhd->unit);
+
+#ifndef ARPHRD_IEEE80211_PRISM  /* From Linux 2.4.18 */
+#define ARPHRD_IEEE80211_PRISM 802
+#endif
+
+#ifndef ARPHRD_IEEE80211_RADIOTAP
+#define ARPHRD_IEEE80211_RADIOTAP	803 /* IEEE 802.11 + radiotap header */
+#endif /* ARPHRD_IEEE80211_RADIOTAP */
+
+	dev->type = ARPHRD_IEEE80211_RADIOTAP;
+
+	dev->netdev_ops = &netdev_monitor_ops;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 9))
+	/* as priv_destructor calls free_netdev, no need to set need_free_netdev */
+	dev->needs_free_netdev = 0;
+	dev->priv_destructor = free_netdev;
+#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 9) */
+	dev->destructor = free_netdev;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 9) */
+
+	/* XXX: This is called from IOCTL path, in this case, rtnl_lock is already taken.
+	 * So, register_netdev() shouldn't be called. It leads to deadlock.
+	 * To avoid deadlock due to rtnl_lock(), register_netdevice() should be used.
+	 */
+	if (register_netdevice(dev)) {
+		DHD_ERROR(("%s, register_netdev failed for %s\n",
+			__FUNCTION__, dev->name));
+		free_netdev(dev);
+		return;
+	}
+
+#ifdef HOST_RADIOTAP_CONV
+	bcmwifi_monitor_create(&dhd->monitor_info);
+	bcmwifi_set_corerev_major(dhd->monitor_info, dhdpcie_get_corerev_major(dhdp));
+	bcmwifi_set_corerev_minor(dhd->monitor_info, dhdpcie_get_corerev_minor(dhdp));
+#endif /* HOST_RADIOTAP_CONV */
+	dhd->monitor_dev = dev;
+#ifdef WL_CFG80211_MONITOR
+	dev_priv = DHD_MON_DEV_PRIV(dev);
+	dev_priv->dhd = dhd;
+	bzero(&dev_priv->stats, sizeof(dev_priv->stats));
+#endif /* WL_CFG80211_MONITOR */
+}
+
+static void
+dhd_del_monitor_if(dhd_info_t *dhd)
+{
+	dhd_mon_dev_priv_t *dev_priv;
+
+	BCM_REFERENCE(dev_priv);
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhd->monitor_dev) {
+		DHD_ERROR(("%s: monitor i/f doesn't exist\n", __FUNCTION__));
+		return;
+	}
+#ifdef WL_CFG80211_MONITOR
+	dhd->monitor_type = FALSE;
+	dev_priv = DHD_MON_DEV_PRIV(dhd->monitor_dev);
+	dev_priv->dhd = (dhd_info_t *)NULL;
+	bzero(&dev_priv->stats, sizeof(dev_priv->stats));
+#endif /* WL_CFG80211_MONITOR */
+
+	if (dhd->monitor_dev) {
+		if (dhd->monitor_dev->reg_state == NETREG_UNINITIALIZED) {
+			free_netdev(dhd->monitor_dev);
+		} else {
+			if (rtnl_is_locked()) {
+				unregister_netdevice(dhd->monitor_dev);
+			} else {
+				unregister_netdev(dhd->monitor_dev);
+			}
+		}
+		dhd->monitor_dev = NULL;
+	}
+#ifdef HOST_RADIOTAP_CONV
+	if (dhd->monitor_info) {
+		bcmwifi_monitor_delete(dhd->monitor_info);
+		dhd->monitor_info = NULL;
+	}
+#endif /* HOST_RADIOTAP_CONV */
+}
+
+void
+dhd_set_monitor(dhd_pub_t *pub, int ifidx, int val)
+{
+	dhd_info_t *dhd = pub->info;
+
+	DHD_TRACE(("%s: val %d\n", __FUNCTION__, val));
+
+	dhd_net_if_lock_local(dhd);
+	if (!val) {
+			/* Delete monitor */
+			dhd_del_monitor_if(dhd);
+	} else {
+			/* Add monitor */
+			dhd_add_monitor_if(dhd);
+	}
+	dhd->monitor_type = val;
+	dhd_net_if_unlock_local(dhd);
+}
+#endif /* WL_MONITOR */
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+/*
+ * Helper function:
+ * Used for RTE console message time syncing with Host printk
+ */
+void dhd_h2d_log_time_sync_deferred_wq_schedule(dhd_pub_t *dhdp)
+{
+	dhd_info_t *info = dhdp->info;
+
+	/* Ideally the "state" should be always TRUE */
+	dhd_deferred_schedule_work(info->dhd_deferred_wq, NULL,
+			DHD_WQ_WORK_H2D_CONSOLE_TIME_STAMP_MATCH,
+			dhd_deferred_work_rte_log_time_sync,
+			DHD_WQ_WORK_PRIORITY_LOW);
+}
+
+void
+dhd_deferred_work_rte_log_time_sync(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd_info = handle;
+	dhd_pub_t *dhd;
+
+	if (event != DHD_WQ_WORK_H2D_CONSOLE_TIME_STAMP_MATCH) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhd_info) {
+		DHD_ERROR(("%s: dhd info not available \n", __FUNCTION__));
+		return;
+	}
+
+	dhd = &dhd_info->pub;
+
+	/*
+	 * Function to send IOVAR for console timesyncing
+	 * between Host and Dongle.
+	 * If the IOVAR fails,
+	 * 1. dhd_rte_time_sync_ms is set to 0 and
+	 * 2. HOST Dongle console time sync will *not* happen.
+	 */
+	dhd_h2d_log_time_sync(dhd);
+}
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+int dhd_ioctl_process(dhd_pub_t *pub, int ifidx, dhd_ioctl_t *ioc, void *data_buf)
+{
+	int bcmerror = BCME_OK;
+	int buflen = 0;
+	struct net_device *net;
+
+	net = dhd_idx2net(pub, ifidx);
+	if (!net) {
+		bcmerror = BCME_BADARG;
+		/*
+		 * The netdev pointer is bad means the DHD can't communicate
+		 * to higher layers, so just return from here
+		 */
+		return bcmerror;
+	}
+
+	/* check for local dhd ioctl and handle it */
+	if (ioc->driver == DHD_IOCTL_MAGIC) {
+		if (data_buf) {
+			/* Return error if nvram size is too big */
+			if (!bcmstricmp((char *)data_buf, "vars")) {
+				DHD_ERROR(("%s: nvram len(%d) MAX_NVRAMBUF_SIZE(%d)\n",
+					__FUNCTION__, ioc->len, MAX_NVRAMBUF_SIZE));
+				if (ioc->len > MAX_NVRAMBUF_SIZE) {
+					DHD_ERROR(("%s: nvram len(%d) > MAX_NVRAMBUF_SIZE(%d)\n",
+						__FUNCTION__, ioc->len, MAX_NVRAMBUF_SIZE));
+					bcmerror = BCME_BUFTOOLONG;
+					goto done;
+				}
+				buflen = ioc->len;
+			} else if (!bcmstricmp((char *)data_buf, "dump") ||
+				!bcmstricmp((char *)data_buf, "counters") ||
+				!bcmstricmp((char *)data_buf, "dump_flowrings")) {
+				buflen = MIN(ioc->len, DHD_IOCTL_MAXLEN_32K);
+			} else {
+				/* This is a DHD IOVAR, truncate buflen to DHD_IOCTL_MAXLEN */
+				buflen = MIN(ioc->len, DHD_IOCTL_MAXLEN);
+			}
+		}
+		bcmerror = dhd_ioctl((void *)pub, ioc, data_buf, buflen);
+		if (bcmerror)
+			pub->bcmerror = bcmerror;
+		goto done;
+	}
+
+	/* This is a WL IOVAR, truncate buflen to WLC_IOCTL_MAXLEN */
+	if (data_buf)
+		buflen = MIN(ioc->len, WLC_IOCTL_MAXLEN);
+
+#ifndef BCMDBUS
+	/* send to dongle (must be up, and wl). */
+	if (pub->busstate == DHD_BUS_DOWN || pub->busstate == DHD_BUS_LOAD) {
+		if ((!pub->dongle_trap_occured) && allow_delay_fwdl) {
+			int ret;
+			if (atomic_read(&exit_in_progress)) {
+				DHD_ERROR(("%s module exit in progress\n", __func__));
+				bcmerror = BCME_DONGLE_DOWN;
+				goto done;
+			}
+			ret = dhd_bus_start(pub);
+			if (ret != 0) {
+				DHD_ERROR(("%s: failed with code %d\n", __FUNCTION__, ret));
+				bcmerror = BCME_DONGLE_DOWN;
+				goto done;
+			}
+		} else {
+			bcmerror = BCME_DONGLE_DOWN;
+			goto done;
+		}
+	}
+
+	if (!pub->iswl) {
+		bcmerror = BCME_DONGLE_DOWN;
+		goto done;
+	}
+#endif /* !BCMDBUS */
+
+	/*
+	 * Flush the TX queue if required for proper message serialization:
+	 * Intercept WLC_SET_KEY IOCTL - serialize M4 send and set key IOCTL to
+	 * prevent M4 encryption and
+	 * intercept WLC_DISASSOC IOCTL - serialize WPS-DONE and WLC_DISASSOC IOCTL to
+	 * prevent disassoc frame being sent before WPS-DONE frame.
+	 */
+	if (ioc->cmd == WLC_SET_KEY ||
+	    (ioc->cmd == WLC_SET_VAR && data_buf != NULL &&
+	     strncmp("wsec_key", data_buf, 9) == 0) ||
+	    (ioc->cmd == WLC_SET_VAR && data_buf != NULL &&
+	     strncmp("bsscfg:wsec_key", data_buf, 15) == 0) ||
+	    ioc->cmd == WLC_DISASSOC)
+		dhd_wait_pend8021x(net);
+
+	if ((ioc->cmd == WLC_SET_VAR || ioc->cmd == WLC_GET_VAR) &&
+		data_buf != NULL && strncmp("rpc_", data_buf, 4) == 0) {
+		bcmerror = BCME_UNSUPPORTED;
+		goto done;
+	}
+
+	/* XXX this typecast is BAD !!! */
+	bcmerror = dhd_wl_ioctl(pub, ifidx, (wl_ioctl_t *)ioc, data_buf, buflen);
+
+#ifdef REPORT_FATAL_TIMEOUTS
+	/* ensure that the timeouts/flags are started/set after the ioctl returns success */
+	if (bcmerror == BCME_OK) {
+		if (ioc->cmd == WLC_SET_WPA_AUTH) {
+			int wpa_auth;
+
+			wpa_auth = *((int *)ioc->buf);
+			DHD_INFO(("wpa_auth:%d\n", wpa_auth));
+			if (wpa_auth != WPA_AUTH_DISABLED) {
+				/* If AP is with security then enable
+				* WLC_E_PSK_SUP event checking
+				*/
+				pub->secure_join = TRUE;
+			} else {
+				/* If AP is with open then disable
+				* WLC_E_PSK_SUP event checking
+				*/
+				pub->secure_join = FALSE;
+			}
+		}
+
+		if (ioc->cmd == WLC_SET_AUTH) {
+			int auth;
+			auth = *((int *)ioc->buf);
+			DHD_INFO(("Auth:%d\n", auth));
+
+			if (auth != WL_AUTH_OPEN_SYSTEM) {
+				/* If AP is with security then enable
+				* WLC_E_PSK_SUP event checking
+				*/
+				pub->secure_join = TRUE;
+			} else {
+				/* If AP is with open then disable WLC_E_PSK_SUP event checking */
+				pub->secure_join = FALSE;
+			}
+		}
+
+		if (ioc->cmd == WLC_SET_SSID) {
+			char iov_buf[WLC_IOCTL_SMLEN];
+			wlc_bsscfg_info_t *info = (wlc_bsscfg_info_t *)iov_buf;
+			int  ret = BCME_OK;
+			/* we may apply join timer for STA only */
+			ret = dhd_iovar(pub, ifidx, "bsscfg_info", NULL, 0,
+			                (char *)iov_buf, sizeof(iov_buf), FALSE);
+			if (BCME_OK <= ret) {
+				if (BSSCFG_GENERIC_AP == info->subtype) {
+					ret = BCME_NOTSTA;
+				}
+			}
+			/* not apply for AP */
+			if (BCME_OK <= ret) {
+				bool set_ssid_rcvd = OSL_ATOMIC_READ(pub->osh, &pub->set_ssid_rcvd);
+				if ((!set_ssid_rcvd) && (!pub->secure_join)) {
+					dhd_start_join_timer(pub);
+				} else {
+					DHD_ERROR(("%s: didnot start join timer."
+						"open join, set_ssid_rcvd: %d secure_join: %d\n",
+						__FUNCTION__, set_ssid_rcvd, pub->secure_join));
+					OSL_ATOMIC_SET(pub->osh, &pub->set_ssid_rcvd, FALSE);
+				}
+			}
+		}
+
+		if (ioc->cmd == WLC_SCAN) {
+			dhd_start_scan_timer(pub, 0);
+		}
+	}
+#endif  /* REPORT_FATAL_TIMEOUTS */
+
+done:
+#if defined(OEM_ANDROID)
+	dhd_check_hang(net, pub, bcmerror);
+#endif /* OEM_ANDROID */
+
+	return bcmerror;
+}
+
+/* XXX For the moment, local ioctls will return BCM errors */
+/* XXX Others return linux codes, need to be changed... */
+/**
+ * Called by the OS (optionally via a wrapper function).
+ * @param net  Linux per dongle instance
+ * @param ifr  Linux request structure
+ * @param cmd  e.g. SIOCETHTOOL
+ */
+static int
+dhd_ioctl_entry(struct net_device *net, struct ifreq *ifr,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	void __user *data,
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+	int cmd)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+	dhd_ioctl_t ioc;
+	int bcmerror = 0;
+	int ifidx;
+	int ret;
+	void *local_buf = NULL;           /**< buffer in kernel space */
+	void __user *ioc_buf_user = NULL; /**< buffer in user space */
+	u16 buflen = 0;
+
+	if (atomic_read(&exit_in_progress)) {
+		DHD_ERROR(("%s module exit in progress\n", __func__));
+		bcmerror = BCME_DONGLE_DOWN;
+		return OSL_ERROR(bcmerror);
+	}
+
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+
+#if defined(OEM_ANDROID)
+	/* Interface up check for built-in type */
+	if (!dhd_download_fw_on_driverload && dhd->pub.up == FALSE) {
+		DHD_ERROR(("%s: Interface is down \n", __FUNCTION__));
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return OSL_ERROR(BCME_NOTUP);
+	}
+#endif /* (OEM_ANDROID) */
+
+	ifidx = dhd_net2idx(dhd, net);
+	DHD_TRACE(("%s: ifidx %d, cmd 0x%04x\n", __FUNCTION__, ifidx, cmd));
+
+#if defined(WL_STATIC_IF)
+	/* skip for static ndev when it is down */
+	if (dhd_is_static_ndev(&dhd->pub, net) && !(net->flags & IFF_UP)) {
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return -1;
+	}
+#endif /* WL_STATIC_iF */
+
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: BAD IF\n", __FUNCTION__));
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return -1;
+	}
+
+#if defined(WL_WIRELESS_EXT)
+	/* linux wireless extensions */
+	if ((cmd >= SIOCIWFIRST) && (cmd <= SIOCIWLAST)) {
+		/* may recurse, do NOT lock */
+		ret = wl_iw_ioctl(net, ifr, cmd);
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return ret;
+	}
+#endif /* defined(WL_WIRELESS_EXT) */
+
+	if (cmd == SIOCETHTOOL) {
+		ret = dhd_ethtool(dhd, (void*)ifr->ifr_data);
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return ret;
+	}
+
+#if defined(OEM_ANDROID)
+	if (cmd == SIOCDEVPRIVATE+1) {
+		ret = wl_android_priv_cmd(net, ifr);
+		dhd_check_hang(net, &dhd->pub, ret);
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return ret;
+	}
+
+#endif /* OEM_ANDROID */
+
+	if (cmd != SIOCDEVPRIVATE) {
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return -EOPNOTSUPP;
+	}
+
+	memset(&ioc, 0, sizeof(ioc));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	/* Copy the ioc control structure part of ioctl request */
+	if (copy_from_user(&ioc, data, sizeof(wl_ioctl_t))) {
+		bcmerror = BCME_BADADDR;
+		goto done;
+	}
+	/* To differentiate between wl and dhd read 4 more byes */
+	if ((copy_from_user(&ioc.driver, (char *)data + sizeof(wl_ioctl_t),
+			sizeof(uint)) != 0)) {
+		bcmerror = BCME_BADADDR;
+		goto done;
+	}
+#else
+#ifdef CONFIG_COMPAT
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0))
+	if (in_compat_syscall())
+#else
+	if (is_compat_task())
+#endif /* LINUX_VER >= 4.6 */
+	{
+		compat_wl_ioctl_t compat_ioc;
+		if (copy_from_user(&compat_ioc, ifr->ifr_data, sizeof(compat_wl_ioctl_t))) {
+			bcmerror = BCME_BADADDR;
+			goto done;
+		}
+		ioc.cmd = compat_ioc.cmd;
+		ioc.buf = compat_ptr(compat_ioc.buf);
+		ioc.len = compat_ioc.len;
+		ioc.set = compat_ioc.set;
+		ioc.used = compat_ioc.used;
+		ioc.needed = compat_ioc.needed;
+		/* To differentiate between wl and dhd read 4 more byes */
+		if ((copy_from_user(&ioc.driver, (char *)ifr->ifr_data + sizeof(compat_wl_ioctl_t),
+			sizeof(uint)) != 0)) {
+			bcmerror = BCME_BADADDR;
+			goto done;
+		}
+	} else
+#endif /* CONFIG_COMPAT */
+	{
+		/* Copy the ioc control structure part of ioctl request */
+		if (copy_from_user(&ioc, ifr->ifr_data, sizeof(wl_ioctl_t))) {
+			bcmerror = BCME_BADADDR;
+			goto done;
+		}
+
+		/* To differentiate between wl and dhd read 4 more byes */
+		if ((copy_from_user(&ioc.driver, (char *)ifr->ifr_data + sizeof(wl_ioctl_t),
+			sizeof(uint)) != 0)) {
+			bcmerror = BCME_BADADDR;
+			goto done;
+		}
+	}
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+
+	if (!capable(CAP_NET_ADMIN)) {
+		bcmerror = BCME_EPERM;
+		goto done;
+	}
+
+	/* Take backup of ioc.buf and restore later */
+	ioc_buf_user = ioc.buf;
+
+	if (ioc.len > 0) {
+		/*
+		* some IOVARs in DHD require 32K user memory. So allocate the
+		* maximum local buffer.
+		*
+		* For IOVARS which donot require 32K user memory, dhd_ioctl_process()
+		* takes care of trimming the length to DHD_IOCTL_MAXLEN(16K). So that DHD
+		* will not overflow the buffer size while updating the buffer.
+		*/
+		buflen = MIN(ioc.len, DHD_IOCTL_MAXLEN_32K);
+		if (!(local_buf = MALLOC(dhd->pub.osh, buflen+1))) {
+			bcmerror = BCME_NOMEM;
+			goto done;
+		}
+
+		if (copy_from_user(local_buf, ioc.buf, buflen)) {
+			bcmerror = BCME_BADADDR;
+			goto done;
+		}
+
+		*((char *)local_buf + buflen) = '\0';
+
+		/* For some platforms accessing userspace memory
+		 * of ioc.buf is causing kernel panic, so to avoid that
+		 * make ioc.buf pointing to kernel space memory local_buf
+		 */
+		ioc.buf = local_buf;
+	}
+
+#if defined(OEM_ANDROID)
+	/* Skip all the non DHD iovars (wl iovars) after f/w hang */
+	if (ioc.driver != DHD_IOCTL_MAGIC && dhd->pub.hang_was_sent) {
+		DHD_TRACE(("%s: HANG was sent up earlier\n", __FUNCTION__));
+		DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_ENABLE(&dhd->pub, DHD_EVENT_TIMEOUT_MS);
+		bcmerror = BCME_DONGLE_DOWN;
+		goto done;
+	}
+#endif /* OEM_ANDROID */
+
+	bcmerror = dhd_ioctl_process(&dhd->pub, ifidx, &ioc, local_buf);
+
+	/* Restore back userspace pointer to ioc.buf */
+	ioc.buf = ioc_buf_user;
+	if (!bcmerror && buflen && local_buf && ioc.buf) {
+		if (copy_to_user(ioc.buf, local_buf, buflen))
+			bcmerror = -EFAULT;
+	}
+
+done:
+	if (local_buf)
+		MFREE(dhd->pub.osh, local_buf, buflen+1);
+
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+
+	return OSL_ERROR(bcmerror);
+}
+
+#if defined(WL_CFG80211) && defined(SUPPORT_DEEP_SLEEP)
+/* Flags to indicate if we distingish power off policy when
+ * user set the memu "Keep Wi-Fi on during sleep" to "Never"
+ */
+int trigger_deep_sleep = 0;
+#endif /* WL_CFG80211 && SUPPORT_DEEP_SLEEP */
+
+#ifdef FIX_CPU_MIN_CLOCK
+static int dhd_init_cpufreq_fix(dhd_info_t *dhd)
+{
+	if (dhd) {
+		mutex_init(&dhd->cpufreq_fix);
+		dhd->cpufreq_fix_status = FALSE;
+	}
+	return 0;
+}
+
+static void dhd_fix_cpu_freq(dhd_info_t *dhd)
+{
+	mutex_lock(&dhd->cpufreq_fix);
+	if (dhd && !dhd->cpufreq_fix_status) {
+		pm_qos_add_request(&dhd->dhd_cpu_qos, PM_QOS_CPU_FREQ_MIN, 300000);
+#ifdef FIX_BUS_MIN_CLOCK
+		pm_qos_add_request(&dhd->dhd_bus_qos, PM_QOS_BUS_THROUGHPUT, 400000);
+#endif /* FIX_BUS_MIN_CLOCK */
+		DHD_ERROR(("pm_qos_add_requests called\n"));
+
+		dhd->cpufreq_fix_status = TRUE;
+	}
+	mutex_unlock(&dhd->cpufreq_fix);
+}
+
+static void dhd_rollback_cpu_freq(dhd_info_t *dhd)
+{
+	mutex_lock(&dhd ->cpufreq_fix);
+	if (dhd && dhd->cpufreq_fix_status != TRUE) {
+		mutex_unlock(&dhd->cpufreq_fix);
+		return;
+	}
+
+	pm_qos_remove_request(&dhd->dhd_cpu_qos);
+#ifdef FIX_BUS_MIN_CLOCK
+	pm_qos_remove_request(&dhd->dhd_bus_qos);
+#endif /* FIX_BUS_MIN_CLOCK */
+	DHD_ERROR(("pm_qos_add_requests called\n"));
+
+	dhd->cpufreq_fix_status = FALSE;
+	mutex_unlock(&dhd->cpufreq_fix);
+}
+#endif /* FIX_CPU_MIN_CLOCK */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+static int
+dhd_ioctl_entry_wrapper(struct net_device *net, struct ifreq *ifr,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	void __user *data,
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+	int cmd)
+{
+	int error;
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+
+	if (atomic_read(&dhd->pub.block_bus))
+		return -EHOSTDOWN;
+
+	if (pm_runtime_get_sync(dhd_bus_to_dev(dhd->pub.bus)) < 0)
+		return BCME_ERROR;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	error = dhd_ioctl_entry(net, ifr, data, cmd);
+#else
+	error = dhd_ioctl_entry(net, ifr, cmd);
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+
+	pm_runtime_mark_last_busy(dhd_bus_to_dev(dhd->pub.bus));
+	pm_runtime_put_autosuspend(dhd_bus_to_dev(dhd->pub.bus));
+
+	return error;
+}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_LIMIT_OUTPUT)
+void
+dhd_ctrl_tcp_limit_output_bytes(int level)
+{
+	if (level == 0) {
+		init_net.ipv4.sysctl_tcp_limit_output_bytes = TCP_DEFAULT_LIMIT_OUTPUT;
+	} else if (level == 1) {
+		init_net.ipv4.sysctl_tcp_limit_output_bytes = DHD_TCP_LIMIT_OUTPUT_BYTES;
+	}
+}
+#endif /* LINUX_VERSION_CODE > 4.19.0 && DHD_TCP_LIMIT_OUTPUT */
+
+void
+dhd_force_collect_socram_during_wifi_onoff(dhd_pub_t *dhdp)
+{
+#ifdef OEM_ANDROID
+#ifdef DHD_FW_COREDUMP
+	if (dhdp->memdump_enabled && (dhdp->busstate != DHD_BUS_DOWN)) {
+#ifdef DHD_SSSR_DUMP
+		dhdp->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+		dhdp->memdump_type = DUMP_TYPE_DONGLE_TRAP_DURING_WIFI_ONOFF;
+		dhd_bus_mem_dump(dhdp);
+	}
+#endif /* DHD_FW_COREDUMP */
+#endif /* OEM_ANDROID */
+}
+
+int
+dhd_stop(struct net_device *net)
+{
+	int ifidx = 0;
+	bool skip_reset = false;
+#ifdef WL_CFG80211
+	unsigned long flags = 0;
+#ifdef WL_STATIC_IF
+	struct bcm_cfg80211 *cfg = wl_get_cfg(net);
+#endif /* WL_STATIC_IF */
+#endif /* WL_CFG80211 */
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+	int timeleft = 0;
+	uint32 bitmask = (uint32)-1;
+
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+	WL_MSG(net->name, "Enter\n");
+	dhd->pub.rxcnt_timeout = 0;
+	dhd->pub.txcnt_timeout = 0;
+
+#ifdef BCMPCIE
+	dhd->pub.d3ackcnt_timeout = 0;
+#endif /* BCMPCIE */
+
+	/* Synchronize between the stop and rx path */
+	dhd->pub.stop_in_progress = true;
+	OSL_SMP_WMB();
+
+#ifdef DHD_COREDUMP
+	if (DHD_BUS_BUSY_CHECK_IN_HALDUMP(&dhd->pub)) {
+		DHD_ERROR(("%s: Cancel the triggerd HAL dump.\n", __FUNCTION__));
+		DHD_BUS_BUSY_CLEAR_IN_HALDUMP(&dhd->pub);
+	}
+	bitmask = ~(DHD_BUS_BUSY_IN_HALDUMP);
+#endif /* DHD_COREDUMP */
+	timeleft = dhd_os_busbusy_wait_bitmask(&dhd->pub,
+			&dhd->pub.dhd_bus_busy_state,
+			bitmask, 0);
+	if (dhd->pub.dhd_bus_busy_state & bitmask) {
+		DHD_ERROR(("%s: Timed out(%d) dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, timeleft, dhd->pub.dhd_bus_busy_state));
+	}
+
+	mutex_lock(&dhd->pub.ndev_op_sync);
+	if (dhd->pub.up == 0) {
+		goto exit;
+	}
+#if defined(DHD_HANG_SEND_UP_TEST)
+	if (dhd->pub.req_hang_type) {
+		DHD_ERROR(("%s, Clear HANG test request 0x%x\n",
+			__FUNCTION__, dhd->pub.req_hang_type));
+		dhd->pub.req_hang_type = 0;
+	}
+#endif /* DHD_HANG_SEND_UP_TEST */
+
+#if defined(WLAN_ACCEL_BOOT)
+	if (!dhd->wl_accel_force_reg_on && dhd_query_bus_erros(&dhd->pub)) {
+		DHD_ERROR(("%s: set force reg on\n", __FUNCTION__));
+		dhd->wl_accel_force_reg_on = TRUE;
+	}
+#endif /* WLAN_ACCEL_BOOT */
+
+#ifdef FIX_CPU_MIN_CLOCK
+	if (dhd_get_fw_mode(dhd) == DHD_FLAG_HOSTAP_MODE)
+		dhd_rollback_cpu_freq(dhd);
+#endif /* FIX_CPU_MIN_CLOCK */
+
+	ifidx = dhd_net2idx(dhd, net);
+	BCM_REFERENCE(ifidx);
+
+	DHD_INFO(("%s: ######### called for ifidx=%d #########\n", __FUNCTION__, ifidx));
+
+#if defined(WL_STATIC_IF) && defined(WL_CFG80211)
+	/* If static if is operational, don't reset the chip */
+	if ((!dhd->pub.hang_was_sent) && (wl_cfg80211_static_if_active(cfg) ||
+		(wl_cfgvif_get_iftype_count(cfg, WL_IF_TYPE_AP)) > 0)) {
+		WL_MSG(net->name, "static if operational. skip chip reset.\n");
+		skip_reset = true;
+		wl_cfg80211_sta_ifdown(net);
+		goto exit;
+	}
+#endif /* WL_STATIC_IF && WL_CFG80211 */
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	if (!dhd->pub.hang_was_sent && dhd->pub.skip_dhd_stop) {
+		WL_MSG(net->name, "skip chip reset.\n");
+		skip_reset = true;
+#if defined(WL_CFG80211)
+		wl_cfg80211_sta_ifdown(net);
+#endif /* WL_CFG80211 */
+		goto exit;
+	}
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+
+#ifdef WL_CFG80211
+	if (ifidx == 0) {
+		dhd_if_t *ifp;
+		wl_cfg80211_down(net);
+
+		DHD_INFO(("%s: making dhdpub up FALSE\n", __FUNCTION__));
+#ifdef WL_CFG80211
+		/* Disable Runtime PM before interface down */
+		DHD_STOP_RPM_TIMER(&dhd->pub);
+
+		DHD_UP_LOCK(&dhd->pub.up_lock, flags);
+		dhd->pub.up = 0;
+		DHD_UP_UNLOCK(&dhd->pub.up_lock, flags);
+#else
+		dhd->pub.up = 0;
+#endif /* WL_CFG80211 */
+#if defined(BCMPCIE) && defined(CONFIG_ARCH_MSM) && defined(CONFIG_SEC_PCIE_L1SS)
+		dhd_bus_inform_ep_loaded_to_rc(&dhd->pub, dhd->pub.up);
+#endif /* BCMPCIE && CONFIG_ARCH_MSM && CONFIG_SEC_PCIE_L1SS */
+
+		ifp = dhd->iflist[0];
+		/*
+		 * For CFG80211: Clean up all the left over virtual interfaces
+		 * when the primary Interface is brought down. [ifconfig wlan0 down]
+		 */
+		if (!dhd_download_fw_on_driverload) {
+			DHD_STATLOG_CTRL(&dhd->pub, ST(WLAN_POWER_OFF), ifidx, 0);
+			if ((dhd->dhd_state & DHD_ATTACH_STATE_ADD_IF) &&
+				(dhd->dhd_state & DHD_ATTACH_STATE_CFG80211)) {
+				int i;
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+				dhd_cleanup_m4_state_work(&dhd->pub, ifidx);
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+#ifdef DHD_PKTDUMP_ROAM
+				dhd_dump_pkt_clear(&dhd->pub);
+#endif /* DHD_PKTDUMP_ROAM */
+
+				dhd_net_if_lock_local(dhd);
+				for (i = 1; i < DHD_MAX_IFS; i++)
+					dhd_remove_if(&dhd->pub, i, FALSE);
+
+				if (ifp && ifp->net) {
+					dhd_if_del_sta_list(ifp);
+				}
+#ifdef ARP_OFFLOAD_SUPPORT
+				if (dhd_inetaddr_notifier_registered) {
+					dhd_inetaddr_notifier_registered = FALSE;
+					unregister_inetaddr_notifier(&dhd_inetaddr_notifier);
+				}
+#endif /* ARP_OFFLOAD_SUPPORT */
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+				if (dhd_inet6addr_notifier_registered) {
+					dhd_inet6addr_notifier_registered = FALSE;
+					unregister_inet6addr_notifier(&dhd_inet6addr_notifier);
+				}
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+				dhd_net_if_unlock_local(dhd);
+			}
+#if 0
+			// terence 20161024: remove this to prevent dev_close() get stuck in dhd_hang_process
+			cancel_work_sync(dhd->dhd_deferred_wq);
+#endif
+
+#ifdef SHOW_LOGTRACE
+			/* Wait till event logs work/kthread finishes */
+			dhd_cancel_logtrace_process_sync(dhd);
+#endif /* SHOW_LOGTRACE */
+
+#ifdef EWP_EDL
+			cancel_delayed_work_sync(&dhd->edl_dispatcher_work);
+#endif
+
+#if defined(DHD_LB_RXP)
+			__skb_queue_purge(&dhd->rx_pend_queue);
+			skb_queue_purge(&dhd->rx_emerge_queue);
+#endif /* DHD_LB_RXP */
+
+#if defined(DHD_LB_TXP)
+			skb_queue_purge(&dhd->tx_pend_queue);
+#endif /* DHD_LB_TXP */
+		}
+#ifdef DHDTCPACK_SUPPRESS
+		dhd_tcpack_suppress_set(&dhd->pub, TCPACK_SUP_OFF);
+#endif /* DHDTCPACK_SUPPRESS */
+#if defined(DHD_LB_RXP)
+		if (ifp && ifp->net == dhd->rx_napi_netdev) {
+			DHD_INFO(("%s napi<%p> disabled ifp->net<%p,%s>\n",
+				__FUNCTION__, &dhd->rx_napi_struct, net, net->name));
+			skb_queue_purge(&dhd->rx_napi_queue);
+			napi_disable(&dhd->rx_napi_struct);
+			netif_napi_del(&dhd->rx_napi_struct);
+			dhd->rx_napi_netdev = NULL;
+		}
+#endif /* DHD_LB_RXP */
+	}
+#endif /* WL_CFG80211 */
+
+#ifdef PROP_TXSTATUS
+	dhd_wlfc_cleanup(&dhd->pub, NULL, 0);
+#endif
+#ifdef SHOW_LOGTRACE
+	if (!dhd_download_fw_on_driverload) {
+		/* Release the skbs from queue for WLC_E_TRACE event */
+		dhd_event_logtrace_flush_queue(&dhd->pub);
+		if (dhd->dhd_state & DHD_ATTACH_LOGTRACE_INIT) {
+			if (dhd->event_data.fmts) {
+				MFREE(dhd->pub.osh, dhd->event_data.fmts,
+					dhd->event_data.fmts_size);
+			}
+			if (dhd->event_data.raw_fmts) {
+				MFREE(dhd->pub.osh, dhd->event_data.raw_fmts,
+					dhd->event_data.raw_fmts_size);
+			}
+			if (dhd->event_data.raw_sstr) {
+				MFREE(dhd->pub.osh, dhd->event_data.raw_sstr,
+					dhd->event_data.raw_sstr_size);
+			}
+			if (dhd->event_data.rom_raw_sstr) {
+				MFREE(dhd->pub.osh, dhd->event_data.rom_raw_sstr,
+					dhd->event_data.rom_raw_sstr_size);
+			}
+			dhd->dhd_state &= ~DHD_ATTACH_LOGTRACE_INIT;
+		}
+	}
+#endif /* SHOW_LOGTRACE */
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+	/* Stop all ring buffer */
+	dhd_os_reset_logging(&dhd->pub);
+#endif
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (dhd_pktlog_ring_reinit(&dhd->pub) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_pktlog_ring_reinit() error.\n", __FUNCTION__));
+	}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+#ifdef APF
+	dhd_dev_apf_delete_filter(net);
+#endif /* APF */
+#ifdef CUSTOM_EVENT_PM_WAKE
+	/* Clear EXCESS_PM_PERIOD explicitly when Wi-Fi turn off */
+	dhd_set_excess_pm_awake(&dhd->pub, FALSE);
+#endif /* CUSTOM_EVENT_PM_WAKE */
+
+	/* Stop the protocol module */
+	dhd_prot_stop(&dhd->pub);
+
+	OLD_MOD_DEC_USE_COUNT;
+exit:
+	if (skip_reset == false) {
+#ifdef WL_ESCAN
+		if (ifidx == 0) {
+			wl_escan_down(net);
+		}
+#endif /* WL_ESCAN */
+		if (ifidx == 0 && !dhd_download_fw_on_driverload) {
+#if defined(WLAN_ACCEL_BOOT)
+			if (dhd->pub.dongle_trap_during_wifi_onoff) {
+				DHD_ERROR(("%s: force collect socram due to trap "
+					"during wifi on/off.\n", __FUNCTION__));
+				dhd_force_collect_socram_during_wifi_onoff(&dhd->pub);
+				dhd->pub.dongle_trap_during_wifi_onoff = 0;
+			}
+
+			wl_android_wifi_accel_off(net, dhd->wl_accel_force_reg_on);
+#else
+#if defined(BT_OVER_SDIO)
+			dhd_bus_put(&dhd->pub, WLAN_MODULE);
+			wl_android_set_wifi_on_flag(FALSE);
+#else
+			wl_android_wifi_off(net, TRUE);
+#ifdef WL_EXT_IAPSTA
+			wl_ext_iapsta_dettach_netdev(net, ifidx);
+#endif /* WL_EXT_IAPSTA */
+#ifdef WL_ESCAN
+			wl_escan_event_dettach(net, ifidx);
+#endif /* WL_ESCAN */
+#ifdef WL_EVENT
+			wl_ext_event_dettach_netdev(net, ifidx);
+#endif /* WL_EVENT */
+#endif /* BT_OVER_SDIO */
+#endif /* WLAN_ACCEL_BOOT */
+		}
+#ifdef SUPPORT_DEEP_SLEEP
+		else {
+			/* CSP#505233: Flags to indicate if we distingish
+			 * power off policy when user set the memu
+			 * "Keep Wi-Fi on during sleep" to "Never"
+			 */
+			if (trigger_deep_sleep) {
+				dhd_deepsleep(net, 1);
+				trigger_deep_sleep = 0;
+			}
+		}
+#endif /* SUPPORT_DEEP_SLEEP */
+		dhd->pub.hang_was_sent = 0;
+		dhd->pub.hang_was_pending = 0;
+
+		/* Clear country spec for for built-in type driver */
+		if (!dhd_download_fw_on_driverload) {
+			dhd->pub.dhd_cspec.country_abbrev[0] = 0x00;
+			dhd->pub.dhd_cspec.rev = 0;
+			dhd->pub.dhd_cspec.ccode[0] = 0x00;
+		}
+
+#ifdef BCMDBGFS
+		dhd_dbgfs_remove();
+#endif
+	}
+
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+
+	/* Destroy wakelock */
+	if (!dhd_download_fw_on_driverload &&
+		(dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT) &&
+		(skip_reset == false)) {
+		DHD_OS_WAKE_LOCK_DESTROY(dhd);
+		dhd->dhd_state &= ~DHD_ATTACH_STATE_WAKELOCKS_INIT;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_LIMIT_OUTPUT)
+	dhd_ctrl_tcp_limit_output_bytes(0);
+#endif /* LINUX_VERSION_CODE > 4.19.0 && DHD_TCP_LIMIT_OUTPUT */
+	WL_MSG(net->name, "Exit\n");
+
+	mutex_unlock(&dhd->pub.ndev_op_sync);
+	/* Clear stop in progress flag */
+	dhd->pub.stop_in_progress = false;
+	return 0;
+}
+
+#if defined(OEM_ANDROID) && defined(WL_CFG80211) && (defined(USE_INITIAL_2G_SCAN) || \
+	defined(USE_INITIAL_SHORT_DWELL_TIME))
+extern bool g_first_broadcast_scan;
+#endif /* OEM_ANDROID && WL_CFG80211 && (USE_INITIAL_2G_SCAN || USE_INITIAL_SHORT_DWELL_TIME) */
+
+#ifdef WL11U
+static int dhd_interworking_enable(dhd_pub_t *dhd)
+{
+	uint32 enable = true;
+	int ret = BCME_OK;
+
+	ret = dhd_iovar(dhd, 0, "interworking", (char *)&enable, sizeof(enable), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: enableing interworking failed, ret=%d\n", __FUNCTION__, ret));
+	}
+
+	return ret;
+}
+#endif /* WL11u */
+
+#if defined(WLAN_ACCEL_BOOT)
+void
+dhd_verify_firmware_mode_change(dhd_info_t *dhd)
+{
+	int current_mode = 0;
+
+	dhd->pub.fw_mode_changed = FALSE;
+
+	DHD_ERROR(("%s: do_chip_bighammer:%d\n", __FUNCTION__, dhd->pub.do_chip_bighammer));
+	/*
+	 * check for the FW change
+	 * previous FW mode - dhd->pub.op_mode remember the previous mode
+	 * current mode - update fw/nv path, get current FW  mode from dhd->fw_path
+	 */
+	dhd_update_fw_nv_path(dhd);
+#ifdef WL_MONITOR
+	DHD_INFO(("%s : check monitor mode with fw_path : %s\n", __FUNCTION__, dhd->fw_path));
+
+	if (strstr(dhd->fw_path, "_mon") != NULL) {
+		DHD_ERROR(("%s : monitor mode is enabled, set force reg on\n", __FUNCTION__));
+		dhd->wl_accel_force_reg_on = TRUE;
+		dhd->pub.fw_mode_changed = TRUE;
+		return;
+	} else if (dhd->pub.monitor_enable == TRUE) {
+		DHD_ERROR(("%s : monitor was enabled, changed to other fw_mode\n", __FUNCTION__));
+		dhd->wl_accel_force_reg_on = TRUE;
+		dhd->pub.fw_mode_changed = TRUE;
+		return;
+	}
+#endif /* WL_MONITOR */
+	current_mode = dhd_get_fw_mode(dhd);
+
+	DHD_ERROR(("%s: current_mode 0x%x, prev_opmode 0x%x\n", __FUNCTION__,
+		current_mode, dhd->pub.op_mode));
+
+	if (!(dhd->pub.op_mode & current_mode)) {
+		DHD_ERROR(("%s: firmware path has changed, set force reg on\n", __FUNCTION__));
+		dhd->wl_accel_force_reg_on = TRUE;
+		dhd->pub.fw_mode_changed = TRUE;
+	}
+}
+#endif /* WLAN_ACCEL_BOOT */
+
+void
+dhd_force_collect_init_fail_dumps(dhd_pub_t *dhdp)
+{
+#ifndef SKIP_FORCE_INIT_DUMP
+#ifdef OEM_ANDROID
+
+#if defined(CUSTOMER_HW7_DEBUG)
+#ifdef DEBUG_DNGL_INIT_FAIL
+	/* As HAL is not inited, do force crash and collect from host dram */
+	dhdp->memdump_enabled = DUMP_MEMONLY;
+#else
+	DHD_ERROR(("%s: As DEBUG_DNGL_INIT_FAIL is not enabled, and HAL not inited,"
+		" cannot collect dumps\n", __FUNCTION__));
+	return;
+#endif /* DEBUG_DNGL_INIT_FAIL */
+#endif /* CUSTOMER_HW7_DEBUG */
+
+#ifdef DHD_FW_COREDUMP
+	/* save core dump or write to a file */
+	if (dhdp->memdump_enabled && (dhdp->busstate != DHD_BUS_DOWN)) {
+#ifdef DHD_SSSR_DUMP
+		if (!dhdp->sssr_inited) {
+			DHD_ERROR(("%s: #### force init SSSR ###\n", __FUNCTION__));
+			dhdp->force_sssr_init = TRUE;
+			/* Hard code sssr_reg_info for 4389 */
+			DHD_SSSR_DUMP_INIT(dhdp);
+			/* force collect FIS */
+			fis_enab = TRUE;
+		}
+		dhdp->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+		dhdp->memdump_type = DUMP_TYPE_DONGLE_INIT_FAILURE;
+		dhd_bus_mem_dump(dhdp);
+	}
+#endif /* DHD_FW_COREDUMP */
+#endif /* OEM_ANDROID */
+#endif /* SKIP_FORCE_INIT_DUMP */
+}
+
+int
+dhd_open(struct net_device *net)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+#ifdef TOE
+	uint32 toe_ol;
+#endif
+	int ifidx;
+	int32 ret = 0;
+#if defined(OOB_INTR_ONLY)
+	uint32 bus_type = -1;
+	uint32 bus_num = -1;
+	uint32 slot_num = -1;
+	wifi_adapter_info_t *adapter = NULL;
+#endif
+#if defined(WL_EXT_IAPSTA) && defined(ISAM_PREINIT)
+	int bytes_written = 0;
+#endif
+
+#if defined(PREVENT_REOPEN_DURING_HANG)
+	/* WAR : to prevent calling dhd_open abnormally in quick succession after hang event */
+	if (dhd->pub.hang_was_sent == 1) {
+		DHD_ERROR(("%s: HANG was sent up earlier\n", __FUNCTION__));
+		/* Force to bring down WLAN interface in case dhd_stop() is not called
+		 * from the upper layer when HANG event is triggered.
+		 */
+		if (!dhd_download_fw_on_driverload && dhd->pub.up == 1) {
+			DHD_ERROR(("%s: WLAN interface is not brought down\n", __FUNCTION__));
+			dhd_stop(net);
+		} else {
+			return -1;
+		}
+	}
+#endif /* PREVENT_REOPEN_DURING_HANG */
+
+	/* clear to run TCM test once per dhd_open() */
+	if (dhd_tcm_test_mode != TCM_TEST_MODE_ONCE) {
+		dhd_tcm_test_status = TCM_TEST_NOT_RUN;
+	}
+
+	mutex_lock(&dhd->pub.ndev_op_sync);
+
+#ifdef SCAN_SUPPRESS
+	wl_ext_reset_scan_busy(&dhd->pub);
+#endif
+
+	if (dhd->pub.up == 1) {
+		/* already up */
+		WL_MSG(net->name, "Primary net_device is already up\n");
+		mutex_unlock(&dhd->pub.ndev_op_sync);
+		return BCME_OK;
+	}
+
+	if (!dhd_download_fw_on_driverload) {
+#if defined(WLAN_ACCEL_BOOT)
+		if (!dhd->wl_accel_force_reg_on && dhd_query_bus_erros(&dhd->pub)) {
+			DHD_ERROR(("%s: set force reg on\n", __FUNCTION__));
+			dhd->wl_accel_force_reg_on = TRUE;
+		}
+		if (!dhd->wl_accel_force_reg_on && !DHD_BUS_BUSY_CHECK_IDLE(&dhd->pub)) {
+			DHD_ERROR(("%s: clear dhd_bus_busy_state: 0x%x\n",
+					__FUNCTION__, dhd->pub.dhd_bus_busy_state));
+			dhd->pub.dhd_bus_busy_state = 0;
+			dhd->wl_accel_force_reg_on = TRUE;
+		}
+#endif /* WLAN_ACCEL_BOOT */
+		if (!dhd_driver_init_done) {
+			DHD_ERROR(("%s: WLAN driver is not initialized\n", __FUNCTION__));
+			mutex_unlock(&dhd->pub.ndev_op_sync);
+			return -1;
+		}
+	}
+
+	WL_MSG(net->name, "Enter\n");
+	DHD_INFO(("%s\n", dhd_version));
+	if (ANDROID_VERSION > 0)
+		printf("ANDROID_VERSION = %d\n", ANDROID_VERSION);
+	/* Init wakelock */
+	if (!dhd_download_fw_on_driverload) {
+		if (!(dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+			DHD_OS_WAKE_LOCK_INIT(dhd);
+			dhd->dhd_state |= DHD_ATTACH_STATE_WAKELOCKS_INIT;
+		}
+
+#ifdef SHOW_LOGTRACE
+		skb_queue_head_init(&dhd->evt_trace_queue);
+
+		if (!(dhd->dhd_state & DHD_ATTACH_LOGTRACE_INIT)) {
+			ret = dhd_init_logstrs_array(dhd->pub.osh, &dhd->event_data);
+			if (ret == BCME_OK) {
+				dhd_init_static_strs_array(dhd->pub.osh, &dhd->event_data,
+					st_str_file_path, map_file_path);
+#ifdef DHD_LOAD_ROML_FILES
+				dhd_init_static_strs_array(dhd->pub.osh, &dhd->event_data,
+					rom_st_str_file_path, rom_map_file_path);
+#endif /* DHD_LOAD_ROML_FILES */
+				dhd->dhd_state |= DHD_ATTACH_LOGTRACE_INIT;
+			}
+		}
+#endif /* SHOW_LOGTRACE */
+	}
+
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+	dhd->pub.dongle_trap_occured = 0;
+	dhd->pub.dsack_hc_due_to_isr_delay = 0;
+	dhd->pub.dsack_hc_due_to_dpc_delay = 0;
+	dhd->pub.dongle_trap_during_wifi_onoff = 0;
+	dhd->pub.hang_was_sent = 0;
+	dhd->pub.hang_was_pending = 0;
+	dhd->pub.hang_reason = 0;
+	dhd->pub.iovar_timeout_occured = 0;
+#ifdef PCIE_FULL_DONGLE
+	dhd->pub.d3ack_timeout_occured = 0;
+	dhd->pub.livelock_occured = 0;
+	dhd->pub.pktid_audit_failed = 0;
+	dhd->pub.pktid_invalid_occured = 0;
+#endif /* PCIE_FULL_DONGLE */
+	dhd->pub.iface_op_failed = 0;
+	dhd->pub.scan_timeout_occurred = 0;
+	dhd->pub.scan_busy_occurred = 0;
+	dhd->pub.p2p_disc_busy_occurred = 0;
+	dhd->pub.smmu_fault_occurred = 0;
+#ifdef DHD_LOSSLESS_ROAMING
+	dhd->pub.dequeue_prec_map = ALLPRIO;
+#endif
+#ifdef DHD_SSSR_DUMP
+	dhd->pub.collect_sssr = FALSE;
+#endif /* DHD_SSSR_DUMP */
+#ifdef DHD_SDTC_ETB_DUMP
+	dhd->pub.collect_sdtc = FALSE;
+#endif /* DHD_SDTC_ETB_DUMP */
+#ifdef DHD_GRO_ENABLE_HOST_CTRL
+	dhd->pub.permitted_gro = TRUE;
+#endif /* DHD_GRO_ENABLE_HOST_CTRL */
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	dhd_set_dump_status(&dhd->pub, DUMP_NOT_READY);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+#if 0
+	/*
+	 * Force start if ifconfig_up gets called before START command
+	 *  We keep WEXT's wl_control_wl_start to provide backward compatibility
+	 *  This should be removed in the future
+	 */
+	ret = wl_control_wl_start(net);
+	if (ret != 0) {
+		DHD_ERROR(("%s: failed with code %d\n", __FUNCTION__, ret));
+		ret = -1;
+		goto exit;
+	}
+
+#endif /* defined(OEM_ANDROID) && !defined(WL_CFG80211) */
+
+	ifidx = dhd_net2idx(dhd, net);
+	DHD_TRACE(("%s: ifidx %d\n", __FUNCTION__, ifidx));
+
+	if (ifidx < 0) {
+		DHD_ERROR(("%s: Error: called with invalid IF\n", __FUNCTION__));
+		ret = -1;
+		goto exit;
+	}
+
+	if (!dhd->iflist[ifidx]) {
+		DHD_ERROR(("%s: Error: called when IF already deleted\n", __FUNCTION__));
+		ret = -1;
+		goto exit;
+	}
+
+	DHD_INFO(("%s: ######### called for ifidx=%d #########\n", __FUNCTION__, ifidx));
+
+	dhd->pub.p2p_disc_busy_cnt = 0;
+#if defined(WLAN_ACCEL_BOOT)
+	dhd_verify_firmware_mode_change(dhd);
+#endif /* WLAN_ACCEL_BOOT */
+
+	if (ifidx == 0) {
+		atomic_set(&dhd->pend_8021x_cnt, 0);
+		if (!dhd_download_fw_on_driverload) {
+			DHD_STATLOG_CTRL(&dhd->pub, ST(WLAN_POWER_ON), ifidx, 0);
+#ifdef WL_EVENT
+			wl_ext_event_attach_netdev(net, ifidx, dhd->iflist[ifidx]->bssidx);
+#endif /* WL_EVENT */
+#ifdef WL_ESCAN
+			wl_escan_event_attach(net, ifidx);
+#endif /* WL_ESCAN */
+#ifdef WL_EXT_IAPSTA
+			wl_ext_iapsta_attach_netdev(net, ifidx, dhd->iflist[ifidx]->bssidx);
+#endif /* WL_EXT_IAPSTA */
+#if defined(USE_INITIAL_2G_SCAN) || defined(USE_INITIAL_SHORT_DWELL_TIME)
+			g_first_broadcast_scan = TRUE;
+#endif /* USE_INITIAL_2G_SCAN || USE_INITIAL_SHORT_DWELL_TIME */
+#ifdef SHOW_LOGTRACE
+			/* dhd_cancel_logtrace_process_sync is called in dhd_stop
+			 * for built-in models. Need to start logtrace kthread before
+			 * calling wifi on, because once wifi is on, EDL will be in action
+			 * any moment, and if kthread is not active, FW event logs will
+			 * not be available
+			 */
+			if (dhd_reinit_logtrace_process(dhd) != BCME_OK) {
+				goto exit;
+			}
+#endif /* SHOW_LOGTRACE */
+#if defined(WLAN_ACCEL_BOOT)
+			ret = wl_android_wifi_accel_on(net, dhd->wl_accel_force_reg_on);
+			/* Enable wl_accel_force_reg_on if ON fails, else disable it */
+			if (ret) {
+				dhd->wl_accel_force_reg_on = TRUE;
+			} else {
+				dhd->wl_accel_force_reg_on = FALSE;
+			}
+#else
+#if defined(BT_OVER_SDIO)
+			ret = dhd_bus_get(&dhd->pub, WLAN_MODULE);
+			wl_android_set_wifi_on_flag(TRUE);
+#else
+			ret = wl_android_wifi_on(net);
+#endif /* BT_OVER_SDIO */
+#endif /* WLAN_ACCEL_BOOT */
+			if (ret != 0) {
+				DHD_ERROR(("%s : wl_android_wifi_on failed (%d)\n",
+					__FUNCTION__, ret));
+				/* Set chip big hammer */
+				dhd->pub.do_chip_bighammer = TRUE;
+				ret = -1;
+				goto exit;
+			}
+		}
+#ifdef SUPPORT_DEEP_SLEEP
+		else {
+			/* Flags to indicate if we distingish
+			 * power off policy when user set the memu
+			 * "Keep Wi-Fi on during sleep" to "Never"
+			 */
+			if (trigger_deep_sleep) {
+#if defined(USE_INITIAL_2G_SCAN) || defined(USE_INITIAL_SHORT_DWELL_TIME)
+				g_first_broadcast_scan = TRUE;
+#endif /* USE_INITIAL_2G_SCAN || USE_INITIAL_SHORT_DWELL_TIME */
+				dhd_deepsleep(net, 0);
+				trigger_deep_sleep = 0;
+			}
+		}
+#endif /* SUPPORT_DEEP_SLEEP */
+#ifdef FIX_CPU_MIN_CLOCK
+		if (dhd_get_fw_mode(dhd) == DHD_FLAG_HOSTAP_MODE) {
+			dhd_init_cpufreq_fix(dhd);
+			dhd_fix_cpu_freq(dhd);
+		}
+#endif /* FIX_CPU_MIN_CLOCK */
+#if defined(OOB_INTR_ONLY)
+		if (dhd->pub.conf->dpc_cpucore >= 0) {
+			dhd_bus_get_ids(dhd->pub.bus, &bus_type, &bus_num, &slot_num);
+			adapter = dhd_wifi_platform_get_adapter(bus_type, bus_num, slot_num);
+			if (adapter) {
+				printf("%s: set irq affinity hit %d\n", __FUNCTION__, dhd->pub.conf->dpc_cpucore);
+				irq_set_affinity_hint(adapter->irq_num, cpumask_of(dhd->pub.conf->dpc_cpucore));
+			}
+		}
+#endif
+
+		if (dhd->pub.busstate != DHD_BUS_DATA) {
+#ifdef BCMDBUS
+			dhd_set_path(&dhd->pub);
+			DHD_MUTEX_UNLOCK();
+			wait_event_interruptible_timeout(dhd->adapter->status_event,
+				wifi_get_adapter_status(dhd->adapter, WIFI_STATUS_FW_READY),
+				msecs_to_jiffies(DHD_FW_READY_TIMEOUT));
+			DHD_MUTEX_LOCK();
+			if ((ret = dbus_up(dhd->pub.bus)) != 0) {
+				DHD_ERROR(("%s: failed to dbus_up with code %d\n", __FUNCTION__, ret));
+				goto exit;
+			} else {
+				dhd->pub.busstate = DHD_BUS_DATA;
+			}
+			if ((ret = dhd_sync_with_dongle(&dhd->pub)) < 0) {
+				DHD_ERROR(("%s: failed with code %d\n", __FUNCTION__, ret));
+				goto exit;
+			}
+#else
+			/* try to bring up bus */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+			if (pm_runtime_get_sync(dhd_bus_to_dev(dhd->pub.bus)) >= 0) {
+				ret = dhd_bus_start(&dhd->pub);
+				pm_runtime_mark_last_busy(dhd_bus_to_dev(dhd->pub.bus));
+				pm_runtime_put_autosuspend(dhd_bus_to_dev(dhd->pub.bus));
+			}
+#else
+			ret = dhd_bus_start(&dhd->pub);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+			if (ret) {
+				DHD_ERROR(("%s: failed with code %d\n", __FUNCTION__, ret));
+				ret = -1;
+				goto exit;
+			}
+#endif /* !BCMDBUS */
+
+		}
+#ifdef WL_EXT_IAPSTA
+		wl_ext_iapsta_attach_name(net, ifidx);
+#endif
+
+#ifdef BT_OVER_SDIO
+		if (dhd->pub.is_bt_recovery_required) {
+			DHD_ERROR(("%s: Send Hang Notification 2 to BT\n", __FUNCTION__));
+			bcmsdh_btsdio_process_dhd_hang_notification(TRUE);
+		}
+		dhd->pub.is_bt_recovery_required = FALSE;
+#endif
+
+		/* dhd_sync_with_dongle has been called in dhd_bus_start or wl_android_wifi_on */
+		DEV_ADDR_SET(net, dhd->pub.mac.octet);
+		(void)memcpy_s(dhd_linux_get_primary_netdev(&dhd->pub)->perm_addr, ETHER_ADDR_LEN,
+			dhd->pub.mac.octet, ETHER_ADDR_LEN);
+
+#ifdef TOE
+		/* Get current TOE mode from dongle */
+		if (dhd_toe_get(dhd, ifidx, &toe_ol) >= 0 && (toe_ol & TOE_TX_CSUM_OL) != 0) {
+			dhd->iflist[ifidx]->net->features |= NETIF_F_IP_CSUM;
+		} else {
+			dhd->iflist[ifidx]->net->features &= ~NETIF_F_IP_CSUM;
+		}
+#endif /* TOE */
+
+#ifdef DHD_LB
+#ifdef ENABLE_DHD_GRO
+		dhd->iflist[ifidx]->net->features |= NETIF_F_GRO;
+#endif /* ENABLE_DHD_GRO */
+
+#if defined(DHD_LB_RXP)
+		__skb_queue_head_init(&dhd->rx_pend_queue);
+		skb_queue_head_init(&dhd->rx_emerge_queue);
+		if (dhd->rx_napi_netdev == NULL) {
+			dhd->rx_napi_netdev = dhd->iflist[ifidx]->net;
+			memset(&dhd->rx_napi_struct, 0, sizeof(struct napi_struct));
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
+			netif_napi_add_weight(dhd->rx_napi_netdev, &dhd->rx_napi_struct,
+				dhd_napi_poll, dhd_napi_weight);
+#else
+			netif_napi_add(dhd->rx_napi_netdev, &dhd->rx_napi_struct,
+				dhd_napi_poll, dhd_napi_weight);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) */
+			DHD_INFO(("%s napi<%p> enabled ifp->net<%p,%s> dhd_napi_weight: %d\n",
+				__FUNCTION__, &dhd->rx_napi_struct, net,
+				net->name, dhd_napi_weight));
+			napi_enable(&dhd->rx_napi_struct);
+			DHD_INFO(("%s load balance init rx_napi_struct\n", __FUNCTION__));
+			skb_queue_head_init(&dhd->rx_napi_queue);
+			__skb_queue_head_init(&dhd->rx_process_queue);
+		} /* rx_napi_netdev == NULL */
+#endif /* DHD_LB_RXP */
+
+#if defined(DHD_LB_TXP)
+		/* Use the variant that uses locks */
+		skb_queue_head_init(&dhd->tx_pend_queue);
+#endif /* DHD_LB_TXP */
+		dhd->dhd_lb_candidacy_override = FALSE;
+#endif /* DHD_LB */
+		dhd_ndev_upd_features(dhd, net);
+#ifdef DHD_PM_OVERRIDE
+		g_pm_override = FALSE;
+#endif /* DHD_PM_OVERRIDE */
+#if defined(WL_CFG80211)
+		if (unlikely(wl_cfg80211_up(net))) {
+			DHD_ERROR(("%s: failed to bring up cfg80211\n", __FUNCTION__));
+			ret = -1;
+			goto exit;
+		}
+		if (!dhd_download_fw_on_driverload) {
+#ifdef ARP_OFFLOAD_SUPPORT
+			dhd->pend_ipaddr = 0;
+			if (!dhd_inetaddr_notifier_registered) {
+				dhd_inetaddr_notifier_registered = TRUE;
+				register_inetaddr_notifier(&dhd_inetaddr_notifier);
+			}
+#endif /* ARP_OFFLOAD_SUPPORT */
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+			if (!dhd_inet6addr_notifier_registered) {
+				dhd_inet6addr_notifier_registered = TRUE;
+				register_inet6addr_notifier(&dhd_inet6addr_notifier);
+			}
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+		}
+
+#if defined(DHD_CONTROL_PCIE_ASPM_WIFI_TURNON)
+		dhd_bus_aspm_enable_rc_ep(dhd->pub.bus, TRUE);
+#endif /* DHD_CONTROL_PCIE_ASPM_WIFI_TURNON */
+#if defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON)
+		dhd_irq_set_affinity(&dhd->pub, cpumask_of(0));
+#endif /* DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON */
+#if defined(NUM_SCB_MAX_PROBE)
+		dhd_set_scb_probe(&dhd->pub);
+#endif /* NUM_SCB_MAX_PROBE */
+#endif /* WL_CFG80211 */
+#ifdef WL_ESCAN
+		if (unlikely(wl_escan_up(net))) {
+			DHD_ERROR(("%s: failed to bring up escan\n", __FUNCTION__));
+			ret = -1;
+			goto exit;
+		}
+#endif /* WL_ESCAN */
+#if defined(ISAM_PREINIT)
+		if (!dhd_download_fw_on_driverload) {
+			if (dhd->pub.conf) {
+				wl_android_ext_priv_cmd(net, dhd->pub.conf->isam_init, 0, &bytes_written);
+				wl_android_ext_priv_cmd(net, dhd->pub.conf->isam_config, 0, &bytes_written);
+				wl_android_ext_priv_cmd(net, dhd->pub.conf->isam_enable, 0, &bytes_written);
+			}
+		}
+#endif
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_LIMIT_OUTPUT)
+	dhd_ctrl_tcp_limit_output_bytes(1);
+#endif /* LINUX_VERSION_CODE > 4.19.0 && DHD_TCP_LIMIT_OUTPUT */
+
+	dhd->pub.up = 1;
+#if defined(BCMPCIE) && defined(CONFIG_ARCH_MSM) && defined(CONFIG_SEC_PCIE_L1SS)
+	dhd_bus_inform_ep_loaded_to_rc(&dhd->pub, dhd->pub.up);
+#endif /* BCMPCIE && CONFIG_ARCH_MSM  && CONFIG_SEC_PCIE_L1SS */
+	DHD_START_RPM_TIMER(&dhd->pub);
+
+	if (wl_event_enable) {
+		/* For wl utility to receive events */
+		dhd->pub.wl_event_enabled = true;
+	} else {
+		dhd->pub.wl_event_enabled = false;
+	}
+
+	if (logtrace_pkt_sendup) {
+		/* For any deamon to recieve logtrace */
+		dhd->pub.logtrace_pkt_sendup = true;
+	} else {
+		dhd->pub.logtrace_pkt_sendup = false;
+	}
+
+	OLD_MOD_INC_USE_COUNT;
+
+#ifdef BCMDBGFS
+	dhd_dbgfs_init(&dhd->pub);
+#endif
+
+exit:
+	mutex_unlock(&dhd->pub.ndev_op_sync);
+
+	if (dhd_query_bus_erros(&dhd->pub)) {
+		ret = BCME_ERROR;
+	}
+
+	if (ret) {
+		if (ret != BCME_NOMEM) {
+			dhd_force_collect_init_fail_dumps(&dhd->pub);
+		} else {
+			DHD_ERROR(("%s: skip collect dump in case of BCME_NOMEM\n",
+					__FUNCTION__));
+			ret = BCME_ERROR;
+		}
+		dhd_stop(net);
+	} else {
+#if defined(ENABLE_INSMOD_NO_FW_LOAD) && defined(NO_POWER_OFF_AFTER_OPEN)
+		dhd_download_fw_on_driverload = TRUE;
+		dhd_driver_init_done = TRUE;
+#elif defined(ENABLE_INSMOD_NO_FW_LOAD) && defined(ENABLE_INSMOD_NO_POWER_OFF)
+		dhd_download_fw_on_driverload = FALSE;
+		dhd_driver_init_done = TRUE;
+#endif
+	}
+
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+
+#if defined(SUPPORT_OTA_UPDATE) && defined(WLAN_ACCEL_BOOT)
+	if (ret == BCME_OK) {
+		(void)dhd_ota_buf_clean(&dhd->pub);
+	}
+#endif /* SUPPORT_OTA_UPDATE  && WLAN_ACCEL_BOOT */
+	WL_MSG(net->name, "Exit ret=%d\n", ret);
+	return ret;
+}
+
+/*
+ * ndo_start handler for primary ndev
+ */
+static int
+dhd_pri_open(struct net_device *net)
+{
+	s32 ret;
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+
+	DHD_MUTEX_IS_LOCK_RETURN();
+	DHD_MUTEX_LOCK();
+	BCM_REFERENCE(dhd);
+	ret = dhd_open(net);
+	if (unlikely(ret)) {
+		DHD_ERROR(("Failed to open primary dev ret %d\n", ret));
+		DHD_MUTEX_UNLOCK();
+		return ret;
+	}
+
+#ifdef WL_CFG80211_MONITOR
+	dhd_net_if_lock_local(dhd);
+	/* Add monitor */
+	dhd_add_monitor_if(dhd);
+	dhd_net_if_unlock_local(dhd);
+#endif /* WL_CFG80211_MONITOR */
+
+	/* Allow transmit calls */
+	dhd_tx_start_queues(net);
+	WL_MSG(net->name, "tx queue started\n");
+
+#if defined(SET_RPS_CPUS)
+	dhd_rps_cpus_enable(net, TRUE);
+#endif
+
+#if defined(SET_XPS_CPUS)
+	dhd_xps_cpus_enable(net, TRUE);
+#endif
+	DHD_MUTEX_UNLOCK();
+
+	return ret;
+}
+
+/*
+ * ndo_stop handler for primary ndev
+ */
+static int
+dhd_pri_stop(struct net_device *net)
+{
+	s32 ret;
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+
+	BCM_REFERENCE(dhd);
+	/* Set state and stop OS transmissions */
+	dhd_tx_stop_queues(net);
+	WL_MSG(net->name, "tx queue stopped\n");
+
+#ifdef WL_CFG80211_MONITOR
+	dhd_net_if_lock_local(dhd);
+	/* Del monitor */
+	dhd_del_monitor_if(dhd);
+	dhd_net_if_unlock_local(dhd);
+#endif /* WL_CFG80211_MONITOR */
+
+	ret = dhd_stop(net);
+	if (unlikely(ret)) {
+		DHD_ERROR(("dhd_stop failed: %d\n", ret));
+		return ret;
+	}
+
+	return ret;
+}
+
+#ifdef PCIE_INB_DW
+bool
+dhd_check_cfg_in_progress(dhd_pub_t *dhdp)
+{
+#if defined(WL_CFG80211)
+	return wl_cfg80211_check_in_progress(dhd_linux_get_primary_netdev(dhdp));
+#endif /* WL_CFG80211 */
+	return FALSE;
+}
+#endif
+
+#if defined(WL_STATIC_IF) && defined(WL_CFG80211)
+/*
+ * For static I/Fs, the firmware interface init
+ * is done from the IFF_UP context.
+ */
+static int
+dhd_static_if_open(struct net_device *net)
+{
+	s32 ret = 0;
+	struct bcm_cfg80211 *cfg;
+	struct net_device *primary_netdev = NULL;
+#ifdef WLEASYMESH
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+#endif /* WLEASYMESH */
+
+	DHD_MUTEX_LOCK();
+	cfg = wl_get_cfg(net);
+	primary_netdev = bcmcfg_to_prmry_ndev(cfg);
+
+	if (!wl_cfg80211_static_if(cfg, net)) {
+		WL_MSG(net->name, "non-static interface ..do nothing\n");
+		ret = BCME_OK;
+		goto done;
+	}
+
+	WL_MSG(net->name, "Enter\n");
+	/* Ensure fw is initialized. If it is already initialized,
+	 * dhd_open will return success.
+	 */
+#ifdef WLEASYMESH
+	WL_MSG(net->name, "switch to EasyMesh fw\n");
+	dhd->pub.conf->fw_type = FW_TYPE_EZMESH;
+	ret = dhd_stop(primary_netdev);
+	if (unlikely(ret)) {
+		printf("===>%s, Failed to close primary dev ret %d\n", __FUNCTION__, ret);
+		goto done;
+	}
+	OSL_SLEEP(1);
+#endif /* WLEASYMESH */
+	ret = dhd_open(primary_netdev);
+	if (unlikely(ret)) {
+		DHD_ERROR(("Failed to open primary dev ret %d\n", ret));
+		goto done;
+	}
+
+	ret = wl_cfg80211_static_if_open(net);
+	if (ret == BCME_OK) {
+		/* Allow transmit calls */
+		netif_start_queue(net);
+	}
+	dhd_clear_del_in_progress(cfg->pub, net);
+done:
+	WL_MSG(net->name, "Exit ret=%d\n", ret);
+	DHD_MUTEX_UNLOCK();
+	return ret;
+}
+
+static int
+dhd_static_if_stop(struct net_device *net)
+{
+	struct bcm_cfg80211 *cfg;
+	struct net_device *primary_netdev = NULL;
+	int ret = BCME_OK;
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+
+	WL_MSG(net->name, "Enter\n");
+
+	cfg = wl_get_cfg(net);
+	if (!wl_cfg80211_static_if(cfg, net)) {
+		DHD_TRACE(("non-static interface (%s)..do nothing \n", net->name));
+		return BCME_OK;
+	}
+#ifdef DHD_NOTIFY_MAC_CHANGED
+	if (dhd->pub.skip_dhd_stop) {
+		WL_MSG(net->name, "Exit skip stop\n");
+		return BCME_OK;
+	}
+#endif /* DHD_NOTIFY_MAC_CHANGED */
+
+	dhd_set_del_in_progress(cfg->pub, net);
+	/* Ensure queue is disabled */
+	netif_tx_disable(net);
+
+	dhd_net_if_lock_local(dhd);
+	ret = wl_cfg80211_static_if_close(net);
+	dhd_net_if_unlock_local(dhd);
+
+	if (dhd->pub.up == 0) {
+		/* If fw is down, return */
+		DHD_ERROR(("fw down\n"));
+		return BCME_OK;
+	}
+	/* If STA iface is not in operational, invoke dhd_close from this
+	* context.
+	*/
+	primary_netdev = bcmcfg_to_prmry_ndev(cfg);
+#ifdef WLEASYMESH
+	if (dhd->pub.conf->fw_type == FW_TYPE_EZMESH) {
+		WL_MSG(net->name, "switch to STA fw\n");
+		dhd->pub.conf->fw_type = FW_TYPE_STA;
+	} else
+#endif /* WLEASYMESH */
+	if (!(primary_netdev->flags & IFF_UP)) {
+		ret = dhd_stop(primary_netdev);
+	} else {
+		DHD_ERROR(("Skipped dhd_stop, as sta is operational\n"));
+	}
+	WL_MSG(net->name, "Exit ret=%d\n", ret);
+
+	return ret;
+}
+#endif /* WL_STATIC_IF && WL_CF80211 */
+
+int dhd_do_driver_init(struct net_device *net)
+{
+	dhd_info_t *dhd = NULL;
+	int ret = 0;
+
+	if (!net) {
+		DHD_ERROR(("Primary Interface not initialized \n"));
+		return -EINVAL;
+	}
+
+	DHD_MUTEX_IS_LOCK_RETURN();
+	DHD_MUTEX_LOCK();
+
+	/*  && defined(OEM_ANDROID) && defined(BCMSDIO) */
+	dhd = DHD_DEV_INFO(net);
+
+	/* If driver is already initialized, do nothing
+	 */
+	if (dhd->pub.busstate == DHD_BUS_DATA) {
+		DHD_TRACE(("Driver already Inititalized. Nothing to do"));
+		goto exit;
+	}
+
+	if (dhd_open(net) < 0) {
+		DHD_ERROR(("Driver Init Failed \n"));
+		ret = -1;
+		goto exit;
+	}
+
+exit:
+	DHD_MUTEX_UNLOCK();
+	return ret;
+}
+
+int
+dhd_event_ifadd(dhd_info_t *dhdinfo, wl_event_data_if_t *ifevent, char *name, uint8 *mac)
+{
+
+#ifdef WL_CFG80211
+		if (wl_cfg80211_notify_ifadd(dhd_linux_get_primary_netdev(&dhdinfo->pub),
+			ifevent->ifidx, name, mac, ifevent->bssidx, ifevent->role) == BCME_OK)
+		return BCME_OK;
+#endif
+
+	/* handle IF event caused by wl commands, SoftAP, WEXT and
+	 * anything else. This has to be done asynchronously otherwise
+	 * DPC will be blocked (and iovars will timeout as DPC has no chance
+	 * to read the response back)
+	 */
+	if (ifevent->ifidx > 0) {
+		dhd_if_event_t *if_event = MALLOC(dhdinfo->pub.osh, sizeof(dhd_if_event_t));
+		if (if_event == NULL) {
+			DHD_ERROR(("dhd_event_ifadd: Failed MALLOC, malloced %d bytes",
+				MALLOCED(dhdinfo->pub.osh)));
+			return BCME_NOMEM;
+		}
+
+		memcpy(&if_event->event, ifevent, sizeof(if_event->event));
+		memcpy(if_event->mac, mac, ETHER_ADDR_LEN);
+		strlcpy(if_event->name, name, sizeof(if_event->name));
+		dhd_deferred_schedule_work(dhdinfo->dhd_deferred_wq, (void *)if_event,
+			DHD_WQ_WORK_IF_ADD, dhd_ifadd_event_handler, DHD_WQ_WORK_PRIORITY_LOW);
+	}
+
+	return BCME_OK;
+}
+
+int
+dhd_event_ifdel(dhd_info_t *dhdinfo, wl_event_data_if_t *ifevent, char *name, uint8 *mac)
+{
+	dhd_if_event_t *if_event;
+
+#ifdef WL_CFG80211
+		if (wl_cfg80211_notify_ifdel(dhd_linux_get_primary_netdev(&dhdinfo->pub),
+			ifevent->ifidx, name, mac, ifevent->bssidx) == BCME_OK)
+		return BCME_OK;
+#endif /* WL_CFG80211 */
+
+	/* handle IF event caused by wl commands, SoftAP, WEXT and
+	 * anything else
+	 */
+	if_event = MALLOC(dhdinfo->pub.osh, sizeof(dhd_if_event_t));
+	if (if_event == NULL) {
+		DHD_ERROR(("dhd_event_ifdel: malloc failed for if_event, malloced %d bytes",
+			MALLOCED(dhdinfo->pub.osh)));
+		return BCME_NOMEM;
+	}
+	memcpy(&if_event->event, ifevent, sizeof(if_event->event));
+	memcpy(if_event->mac, mac, ETHER_ADDR_LEN);
+	strlcpy(if_event->name, name, sizeof(if_event->name));
+	dhd_deferred_schedule_work(dhdinfo->dhd_deferred_wq, (void *)if_event, DHD_WQ_WORK_IF_DEL,
+		dhd_ifdel_event_handler, DHD_WQ_WORK_PRIORITY_LOW);
+
+	return BCME_OK;
+}
+
+int
+dhd_event_ifchange(dhd_info_t *dhdinfo, wl_event_data_if_t *ifevent, char *name, uint8 *mac)
+{
+#ifdef DHD_UPDATE_INTF_MAC
+	dhd_if_event_t *if_event;
+#endif /* DHD_UPDATE_INTF_MAC */
+
+#ifdef WL_CFG80211
+	wl_cfg80211_notify_ifchange(dhd_linux_get_primary_netdev(&dhdinfo->pub),
+		ifevent->ifidx, name, mac, ifevent->bssidx);
+#endif /* WL_CFG80211 */
+
+#ifdef DHD_UPDATE_INTF_MAC
+	/* handle IF event caused by wl commands, SoftAP, WEXT, MBSS and
+	 * anything else
+	 */
+	if_event = MALLOC(dhdinfo->pub.osh, sizeof(dhd_if_event_t));
+	if (if_event == NULL) {
+		DHD_ERROR(("dhd_event_ifdel: malloc failed for if_event, malloced %d bytes",
+			MALLOCED(dhdinfo->pub.osh)));
+		return BCME_NOMEM;
+	}
+	memcpy(&if_event->event, ifevent, sizeof(if_event->event));
+	// construct a change event
+	if_event->event.ifidx = dhd_ifname2idx(dhdinfo, name);
+	if_event->event.opcode = WLC_E_IF_CHANGE;
+	memcpy(if_event->mac, mac, ETHER_ADDR_LEN);
+	strncpy(if_event->name, name, IFNAMSIZ);
+	if_event->name[IFNAMSIZ - 1] = '\0';
+	dhd_deferred_schedule_work(dhdinfo->dhd_deferred_wq, (void *)if_event, DHD_WQ_WORK_IF_UPDATE,
+		dhd_ifupdate_event_handler, DHD_WQ_WORK_PRIORITY_LOW);
+#endif /* DHD_UPDATE_INTF_MAC */
+
+	return BCME_OK;
+}
+
+/* This API maps ndev to ifp inclusive of static IFs */
+static dhd_if_t *
+dhd_get_ifp_by_ndev(dhd_pub_t *dhdp, struct net_device *ndev)
+{
+	dhd_if_t *ifp = NULL;
+#ifdef WL_STATIC_IF
+	u32 ifidx = (DHD_MAX_IFS + DHD_MAX_STATIC_IFS - 1);
+#else
+	u32 ifidx = (DHD_MAX_IFS - 1);
+#endif /* WL_STATIC_IF */
+
+	dhd_info_t *dhdinfo = (dhd_info_t *)dhdp->info;
+	do {
+		ifp = dhdinfo->iflist[ifidx];
+		if (ifp && (ifp->net == ndev)) {
+			DHD_TRACE(("match found for %s. ifidx:%d\n",
+				ndev->name, ifidx));
+			return ifp;
+		}
+	} while (ifidx--);
+
+	DHD_ERROR(("no entry found for %s\n", ndev->name));
+	return NULL;
+}
+
+bool
+dhd_is_static_ndev(dhd_pub_t *dhdp, struct net_device *ndev)
+{
+	dhd_if_t *ifp = NULL;
+
+	if (!dhdp || !ndev) {
+		DHD_ERROR(("wrong input\n"));
+		ASSERT(0);
+		return false;
+	}
+
+	ifp = dhd_get_ifp_by_ndev(dhdp, ndev);
+	return (ifp && (ifp->static_if == true));
+}
+
+#ifdef WL_STATIC_IF
+/* In some cases, while registering I/F, the actual ifidx, bssidx and dngl_name
+ * are not known. For e.g: static i/f case. This function lets to update it once
+ * it is known.
+ */
+s32
+dhd_update_iflist_info(dhd_pub_t *dhdp, struct net_device *ndev, int ifidx,
+	uint8 *mac, uint8 bssidx, const char *dngl_name, int if_state)
+{
+	dhd_info_t *dhdinfo = (dhd_info_t *)dhdp->info;
+	dhd_if_t *ifp, *ifp_new;
+	s32 cur_idx;
+	dhd_dev_priv_t * dev_priv;
+
+	DHD_TRACE(("[STATIC_IF] update ifinfo for state:%d ifidx:%d\n",
+			if_state, ifidx));
+
+	ASSERT(dhdinfo && (ifidx < (DHD_MAX_IFS + DHD_MAX_STATIC_IFS)));
+	if (ifidx < 0 || ifidx >= (DHD_MAX_IFS + DHD_MAX_STATIC_IFS)) {
+		return -EINVAL;
+	}
+
+	if ((ifp = dhd_get_ifp_by_ndev(dhdp, ndev)) == NULL) {
+		return -ENODEV;
+	}
+	cur_idx = ifp->idx;
+
+	if (if_state == NDEV_STATE_OS_IF_CREATED) {
+		/* mark static if */
+		ifp->static_if = TRUE;
+		return BCME_OK;
+	}
+
+	ifp_new = dhdinfo->iflist[ifidx];
+	if (ifp_new && (ifp_new != ifp)) {
+		/* There should be only one entry for a given ifidx. */
+		DHD_ERROR(("ifp ptr already present for ifidx:%d\n", ifidx));
+		ASSERT(0);
+		dhdp->hang_reason = HANG_REASON_IFACE_ADD_FAILURE;
+#ifdef OEM_ANDROID
+		net_os_send_hang_message(ifp->net);
+#endif /* OEM_ANDROID */
+		return -EINVAL;
+	}
+
+	/* For static if delete case, cleanup the if before ifidx update */
+	if ((if_state == NDEV_STATE_FW_IF_DELETED) ||
+		(if_state == NDEV_STATE_FW_IF_FAILED)) {
+		dhd_cleanup_if(ifp->net);
+		dev_priv = DHD_DEV_PRIV(ndev);
+		dev_priv->ifidx = ifidx;
+	}
+
+	/* update the iflist ifidx slot with cached info */
+	dhdinfo->iflist[ifidx] = ifp;
+	dhdinfo->iflist[cur_idx] = NULL;
+
+	/* update the values */
+	ifp->idx = ifidx;
+	ifp->bssidx = bssidx;
+
+	if (if_state == NDEV_STATE_FW_IF_CREATED) {
+		dhd_dev_priv_save(ndev, dhdinfo, ifp, ifidx);
+		/* initialize the dongle provided if name */
+		if (dngl_name) {
+			strncpy(ifp->dngl_name, dngl_name, IFNAMSIZ);
+		} else if (ndev->name[0] != '\0') {
+			strncpy(ifp->dngl_name, ndev->name, IFNAMSIZ);
+		}
+		if (mac != NULL && ifp->set_macaddress == FALSE) {
+			/* To and fro locations have same size - ETHER_ADDR_LEN */
+			(void)memcpy_s(&ifp->mac_addr, ETHER_ADDR_LEN, mac, ETHER_ADDR_LEN);
+		}
+#ifdef WL_EVENT
+		wl_ext_event_attach_netdev(ndev, ifidx, bssidx);
+#endif /* WL_EVENT */
+#ifdef WL_ESCAN
+		wl_escan_event_attach(ndev, ifidx);
+#endif /* WL_ESCAN */
+#ifdef WL_EXT_IAPSTA
+		wl_ext_iapsta_ifadding(ndev, ifidx);
+		wl_ext_iapsta_attach_netdev(ndev, ifidx, bssidx);
+		wl_ext_iapsta_attach_name(ndev, ifidx);
+#endif /* WL_EXT_IAPSTA */
+	}
+	else if (if_state == NDEV_STATE_FW_IF_DELETED) {
+#ifdef WL_EXT_IAPSTA
+		wl_ext_iapsta_dettach_netdev(ndev, cur_idx);
+#endif /* WL_EXT_IAPSTA */
+#ifdef WL_ESCAN
+		wl_escan_event_dettach(ndev, cur_idx);
+#endif /* WL_ESCAN */
+#ifdef WL_EVENT
+		wl_ext_event_dettach_netdev(ndev, cur_idx);
+#endif /* WL_EVENT */
+	}
+	DHD_INFO(("[STATIC_IF] ifp ptr updated for ifidx:%d curidx:%d if_state:%d\n",
+		ifidx, cur_idx, if_state));
+	return BCME_OK;
+}
+#endif /* WL_STATIC_IF */
+
+/* unregister and free the existing net_device interface (if any) in iflist and
+ * allocate a new one. the slot is reused. this function does NOT register the
+ * new interface to linux kernel. dhd_register_if does the job
+ */
+struct net_device*
+dhd_allocate_if(dhd_pub_t *dhdpub, int ifidx, const char *name,
+	uint8 *mac, uint8 bssidx, bool need_rtnl_lock, const char *dngl_name)
+{
+	dhd_info_t *dhdinfo = (dhd_info_t *)dhdpub->info;
+	dhd_if_t *ifp;
+
+	ASSERT(dhdinfo && (ifidx < (DHD_MAX_IFS + DHD_MAX_STATIC_IFS)));
+	if (!dhdinfo || ifidx < 0 || ifidx >= (DHD_MAX_IFS + DHD_MAX_STATIC_IFS)) {
+		return NULL;
+	}
+
+	ifp = dhdinfo->iflist[ifidx];
+
+	if (ifp != NULL) {
+		if (ifp->net != NULL) {
+			DHD_ERROR(("%s: free existing IF %s ifidx:%d \n",
+				__FUNCTION__, ifp->net->name, ifidx));
+
+			if (ifidx == 0) {
+				/* For primary ifidx (0), there shouldn't be
+				 * any netdev present already.
+				 */
+				DHD_ERROR(("Primary ifidx populated already\n"));
+				ASSERT(0);
+				return NULL;
+			}
+
+			dhd_dev_priv_clear(ifp->net); /* clear net_device private */
+
+			/* in unregister_netdev case, the interface gets freed by net->destructor
+			 * (which is set to free_netdev)
+			 */
+#if defined(CONFIG_TIZEN)
+			net_stat_tizen_unregister(ifp->net);
+#endif /* CONFIG_TIZEN */
+			if (ifp->net->reg_state == NETREG_UNINITIALIZED) {
+				free_netdev(ifp->net);
+			} else {
+				dhd_tx_stop_queues(ifp->net);
+				dhd_unregister_net(ifp->net, need_rtnl_lock);
+			}
+			ifp->net = NULL;
+		}
+	} else {
+		ifp = MALLOC(dhdinfo->pub.osh, sizeof(dhd_if_t));
+		if (ifp == NULL) {
+			DHD_ERROR(("%s: OOM - dhd_if_t(%zu)\n", __FUNCTION__, sizeof(dhd_if_t)));
+			return NULL;
+		}
+	}
+
+	memset(ifp, 0, sizeof(dhd_if_t));
+	ifp->info = dhdinfo;
+	ifp->idx = ifidx;
+	ifp->bssidx = bssidx;
+	ifp->del_in_progress = FALSE;
+#ifdef DHD_MCAST_REGEN
+	ifp->mcast_regen_bss_enable = FALSE;
+#endif
+	/* set to TRUE rx_pkt_chainable at alloc time */
+	ifp->rx_pkt_chainable = TRUE;
+
+	if (mac != NULL)
+		memcpy(&ifp->mac_addr, mac, ETHER_ADDR_LEN);
+
+	/* Allocate etherdev, including space for private structure */
+#ifdef DHD_MQ
+	if (enable_mq) {
+		ifp->net = alloc_etherdev_mq(DHD_DEV_PRIV_SIZE, MQ_MAX_QUEUES);
+	} else {
+		ifp->net = alloc_etherdev(DHD_DEV_PRIV_SIZE);
+	}
+#else
+	ifp->net = alloc_etherdev(DHD_DEV_PRIV_SIZE);
+#endif /* DHD_MQ */
+
+	if (ifp->net == NULL) {
+		DHD_ERROR(("%s: OOM - alloc_etherdev(%zu)\n", __FUNCTION__, sizeof(dhdinfo)));
+		goto fail;
+	}
+
+	/* Setup the dhd interface's netdevice private structure. */
+	dhd_dev_priv_save(ifp->net, dhdinfo, ifp, ifidx);
+
+	if (name && name[0]) {
+		strlcpy(ifp->net->name, name, IFNAMSIZ);
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 9))
+	/* as priv_destructor calls free_netdev, no need to set need_free_netdev */
+	ifp->net->needs_free_netdev = 0;
+#ifdef WL_CFG80211
+	if (ifidx == 0)
+		ifp->net->priv_destructor = free_netdev;
+	else
+		ifp->net->priv_destructor = dhd_netdev_free;
+#else
+	ifp->net->priv_destructor = free_netdev;
+#endif /* WL_CFG80211 */
+#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 9) */
+#ifdef WL_CFG80211
+	if (ifidx == 0)
+		ifp->net->destructor = free_netdev;
+	else
+		ifp->net->destructor = dhd_netdev_free;
+#else
+	ifp->net->destructor = free_netdev;
+#endif /* WL_CFG80211 */
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 9) */
+	strlcpy(ifp->name, ifp->net->name, sizeof(ifp->name));
+	dhdinfo->iflist[ifidx] = ifp;
+
+/* initialize the dongle provided if name */
+	if (dngl_name) {
+		strlcpy(ifp->dngl_name, dngl_name, sizeof(ifp->dngl_name));
+	} else if (name) {
+		strlcpy(ifp->dngl_name, name, sizeof(ifp->dngl_name));
+	}
+
+#ifdef PCIE_FULL_DONGLE
+	/* Initialize STA info list */
+	INIT_LIST_HEAD(&ifp->sta_list);
+	DHD_IF_STA_LIST_LOCK_INIT(&ifp->sta_list_lock);
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef DHD_L2_FILTER
+	ifp->phnd_arp_table = init_l2_filter_arp_table(dhdpub->osh);
+	ifp->parp_allnode = TRUE;
+#endif /* DHD_L2_FILTER */
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+	ifp->qosmap_up_table = ((uint8*)MALLOCZ(dhdpub->osh, UP_TABLE_MAX));
+	ifp->qosmap_up_table_enable = FALSE;
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+
+	DHD_CUMM_CTR_INIT(&ifp->cumm_ctr);
+
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+	INIT_DELAYED_WORK(&ifp->m4state_work, dhd_m4_state_handler);
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#ifdef DHD_POST_EAPOL_M1_AFTER_ROAM_EVT
+	ifp->recv_reassoc_evt = FALSE;
+	ifp->post_roam_evt = FALSE;
+#endif /* DHD_POST_EAPOL_M1_AFTER_ROAM_EVT */
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+	INIT_WORK(&ifp->blk_tsfl_work, dhd_blk_tsfl_handler);
+	dhd_reset_tcpsync_info_by_ifp(ifp);
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+	return ifp->net;
+
+fail:
+	if (ifp != NULL) {
+		if (ifp->net != NULL) {
+#if defined(DHD_LB_RXP) && defined(PCIE_FULL_DONGLE)
+			if (ifp->net == dhdinfo->rx_napi_netdev) {
+				napi_disable(&dhdinfo->rx_napi_struct);
+				netif_napi_del(&dhdinfo->rx_napi_struct);
+				skb_queue_purge(&dhdinfo->rx_napi_queue);
+				dhdinfo->rx_napi_netdev = NULL;
+			}
+#endif /* DHD_LB_RXP && PCIE_FULL_DONGLE */
+			dhd_dev_priv_clear(ifp->net);
+			free_netdev(ifp->net);
+			ifp->net = NULL;
+		}
+		MFREE(dhdinfo->pub.osh, ifp, sizeof(*ifp));
+	}
+	dhdinfo->iflist[ifidx] = NULL;
+	return NULL;
+}
+
+static void
+dhd_cleanup_ifp(dhd_pub_t *dhdp, dhd_if_t *ifp)
+{
+#ifdef PCIE_FULL_DONGLE
+	s32 ifidx = 0;
+	if_flow_lkup_t *if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+#endif /* PCIE_FULL_DONGLE */
+
+	if (ifp != NULL) {
+		if ((ifp->idx < 0) || (ifp->idx >= DHD_MAX_IFS)) {
+			DHD_ERROR(("Wrong idx:%d \n", ifp->idx));
+			ASSERT(0);
+			return;
+		}
+#ifdef DHD_L2_FILTER
+		bcm_l2_filter_arp_table_update(dhdpub->osh, ifp->phnd_arp_table, TRUE,
+			NULL, FALSE, dhdpub->tickcnt);
+		deinit_l2_filter_arp_table(dhdpub->osh, ifp->phnd_arp_table);
+		ifp->phnd_arp_table = NULL;
+#endif /* DHD_L2_FILTER */
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+		MFREE(dhdpub->osh, ifp->qosmap_up_table, UP_TABLE_MAX);
+		ifp->qosmap_up_table = NULL;
+		ifp->qosmap_up_table_enable = FALSE;
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+
+		dhd_if_del_sta_list(ifp);
+#ifdef PCIE_FULL_DONGLE
+		/* Delete flowrings of virtual interface */
+		ifidx = ifp->idx;
+		if ((ifidx != 0) &&
+		    ((if_flow_lkup != NULL) && (if_flow_lkup[ifidx].role != WLC_E_IF_ROLE_AP))) {
+			dhd_flow_rings_delete(dhdp, ifidx);
+		}
+#endif /* PCIE_FULL_DONGLE */
+	}
+}
+
+void
+dhd_cleanup_if(struct net_device *net)
+{
+	dhd_info_t *dhdinfo = DHD_DEV_INFO(net);
+	dhd_pub_t *dhdp = &dhdinfo->pub;
+	dhd_if_t *ifp;
+
+	ifp = dhd_get_ifp_by_ndev(dhdp, net);
+	if (ifp) {
+		if (ifp->idx >= DHD_MAX_IFS) {
+			DHD_ERROR(("Wrong ifidx: %p, %d\n", ifp, ifp->idx));
+			ASSERT(0);
+			return;
+		}
+		dhd_cleanup_ifp(dhdp, ifp);
+	}
+}
+
+/* unregister and free the the net_device interface associated with the indexed
+ * slot, also free the slot memory and set the slot pointer to NULL
+ */
+#define DHD_TX_COMPLETION_TIMEOUT 5000
+int
+dhd_remove_if(dhd_pub_t *dhdpub, int ifidx, bool need_rtnl_lock)
+{
+	dhd_info_t *dhdinfo = (dhd_info_t *)dhdpub->info;
+	dhd_if_t *ifp;
+	unsigned long flags;
+	long timeout;
+
+	ifp = dhdinfo->iflist[ifidx];
+
+	if (ifp != NULL) {
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+		cancel_delayed_work_sync(&ifp->m4state_work);
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+		cancel_work_sync(&ifp->blk_tsfl_work);
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+		dhd_cleanup_ifp(dhdpub, ifp);
+#ifdef WL_STATIC_IF
+		if (ifp->static_if) {
+			/* static IF will be handled in detach */
+			DHD_TRACE(("Skip del iface for static interface\n"));
+			return BCME_OK;
+		}
+#endif /* WL_STATIC_IF */
+		if (ifp->net != NULL) {
+			DHD_ERROR(("deleting interface '%s' idx %d\n", ifp->net->name, ifp->idx));
+
+			DHD_GENERAL_LOCK(dhdpub, flags);
+			ifp->del_in_progress = TRUE;
+			DHD_GENERAL_UNLOCK(dhdpub, flags);
+
+			/* If TX is in progress, hold the if del */
+			if (DHD_IF_IS_TX_ACTIVE(ifp)) {
+				DHD_INFO(("TX in progress. Wait for it to be complete."));
+				timeout = wait_event_timeout(dhdpub->tx_completion_wait,
+					((ifp->tx_paths_active & DHD_TX_CONTEXT_MASK) == 0),
+					msecs_to_jiffies(DHD_TX_COMPLETION_TIMEOUT));
+				if (!timeout) {
+					/* Tx completion timeout. Attempt proceeding ahead */
+					DHD_ERROR(("Tx completion timed out!\n"));
+					ASSERT(0);
+				}
+			} else {
+				DHD_TRACE(("No outstanding TX!\n"));
+			}
+			dhdinfo->iflist[ifidx] = NULL;
+			/* in unregister_netdev case, the interface gets freed by net->destructor
+			 * (which is set to free_netdev)
+			 */
+			if (ifp->net->reg_state == NETREG_UNINITIALIZED) {
+				free_netdev(ifp->net);
+			} else {
+				netif_tx_disable(ifp->net);
+
+#if defined(SET_RPS_CPUS)
+				custom_rps_map_clear(ifp->net->_rx);
+#endif /* SET_RPS_CPUS */
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+				if (dhdinfo->cih)
+					ctf_dev_unregister(dhdinfo->cih, ifp->net);
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#if (defined(DHDTCPACK_SUPPRESS) && defined(BCMPCIE))
+				dhd_tcpack_suppress_set(dhdpub, TCPACK_SUP_OFF);
+#endif /* DHDTCPACK_SUPPRESS && BCMPCIE */
+				dhd_unregister_net(ifp->net, need_rtnl_lock);
+#if defined(WLDWDS) && defined(WL_EXT_IAPSTA)
+				if (ifp->dwds) {
+					wl_ext_iapsta_dettach_dwds_netdev(ifp->net, ifidx, ifp->bssidx);
+				} else
+#endif /* WLDWDS && WL_EXT_IAPSTA */
+				{
+#ifdef WL_EXT_IAPSTA
+					wl_ext_iapsta_dettach_netdev(ifp->net, ifidx);
+#endif /* WL_EXT_IAPSTA */
+#ifdef WL_ESCAN
+					wl_escan_event_dettach(ifp->net, ifidx);
+#endif /* WL_ESCAN */
+#ifdef WL_EVENT
+					wl_ext_event_dettach_netdev(ifp->net, ifidx);
+#endif /* WL_EVENT */
+				}
+			}
+			ifp->net = NULL;
+		}
+#ifdef DHD_WMF
+		dhd_wmf_cleanup(dhdpub, ifidx);
+#endif /* DHD_WMF */
+		DHD_CUMM_CTR_INIT(&ifp->cumm_ctr);
+
+		MFREE(dhdinfo->pub.osh, ifp, sizeof(*ifp));
+		ifp = NULL;
+	}
+
+	return BCME_OK;
+}
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+int
+dhd_set_qosmap_up_table(dhd_pub_t *dhdp, uint32 idx, bcm_tlv_t *qos_map_ie)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+
+	if (!ifp)
+	    return BCME_ERROR;
+
+	wl_set_up_table(ifp->qosmap_up_table, qos_map_ie);
+	ifp->qosmap_up_table_enable = TRUE;
+
+	return BCME_OK;
+}
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+
+static struct net_device_ops dhd_ops_pri = {
+	.ndo_open = dhd_pri_open,
+	.ndo_stop = dhd_pri_stop,
+	.ndo_get_stats = dhd_get_stats,
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	.ndo_siocdevprivate = dhd_ioctl_entry_wrapper,
+#else
+	.ndo_do_ioctl = dhd_ioctl_entry_wrapper,
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+	.ndo_start_xmit = dhd_start_xmit_wrapper,
+#else
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	.ndo_siocdevprivate = dhd_ioctl_entry,
+#else
+	.ndo_do_ioctl = dhd_ioctl_entry,
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+	.ndo_start_xmit = dhd_start_xmit,
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+	.ndo_set_mac_address = dhd_set_mac_address,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	.ndo_set_rx_mode = dhd_set_multicast_list,
+#else
+	.ndo_set_multicast_list = dhd_set_multicast_list,
+#endif
+#ifdef DHD_MQ
+	.ndo_select_queue = dhd_select_queue
+#endif
+};
+
+static struct net_device_ops dhd_ops_virt = {
+#if defined(WL_CFG80211) && defined(WL_STATIC_IF)
+	.ndo_open = dhd_static_if_open,
+	.ndo_stop = dhd_static_if_stop,
+#endif
+	.ndo_get_stats = dhd_get_stats,
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	.ndo_siocdevprivate = dhd_ioctl_entry_wrapper,
+#else
+	.ndo_do_ioctl = dhd_ioctl_entry_wrapper,
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+	.ndo_start_xmit = dhd_start_xmit_wrapper,
+#else
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	.ndo_siocdevprivate = dhd_ioctl_entry,
+#else
+	.ndo_do_ioctl = dhd_ioctl_entry,
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(5, 15, 0) */
+	.ndo_start_xmit = dhd_start_xmit,
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+	.ndo_set_mac_address = dhd_set_mac_address,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	.ndo_set_rx_mode = dhd_set_multicast_list,
+#else
+	.ndo_set_multicast_list = dhd_set_multicast_list,
+#endif
+};
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+static void
+dhd_ctf_detach(ctf_t *ci, void *arg)
+{
+	dhd_info_t *dhd = (dhd_info_t *)arg;
+	dhd->cih = NULL;
+
+#ifdef CTFPOOL
+	/* free the buffers in fast pool */
+	osl_ctfpool_cleanup(dhd->pub.osh);
+#endif /* CTFPOOL */
+
+	return;
+}
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+int
+dhd_os_write_file_posn(void *fp, unsigned long *posn, void *buf,
+		unsigned long buflen)
+{
+	loff_t wr_posn = *posn;
+
+	if (!fp || !buf || buflen == 0)
+		return -1;
+
+	if (dhd_vfs_write((struct file *)fp, buf, buflen, &wr_posn) < 0)
+		return -1;
+
+	*posn = wr_posn;
+	return 0;
+}
+
+int
+dhd_os_read_file(void *file, char *buf, uint32 size)
+{
+	struct file *filep = (struct file *)file;
+
+	if (!file || !buf)
+		return -1;
+
+	return dhd_vfs_read(filep, buf, size, &filep->f_pos);
+}
+
+int
+dhd_os_seek_file(void *file, int64 offset)
+{
+	struct file *filep = (struct file *)file;
+	if (!file)
+		return -1;
+
+	/* offset can be -ve */
+	filep->f_pos = filep->f_pos + offset;
+
+	return 0;
+}
+
+#ifdef SHOW_LOGTRACE
+#ifdef DHD_COREDUMP
+static int
+dhd_lookup_map(osl_t *osh, char *fname, uint32 pc, char *pc_fn,
+		uint32 lr, char *lr_fn)
+{
+#ifdef DHD_LINUX_STD_FW_API
+	const struct firmware *fw = NULL;
+	uint32 size = 0, mem_offset = 0;
+#else
+	struct file *filep = NULL;
+#ifdef get_fs
+	mm_segment_t fs;
+#endif /* get_fs */
+#endif /* DHD_LINUX_STD_FW_API */
+	char *raw_fmts = NULL, *raw_fmts_loc = NULL, *cptr = NULL;
+	uint32 read_size = READ_NUM_BYTES;
+	int err = BCME_ERROR;
+	uint32 addr = 0, addr1 = 0, addr2 = 0;
+	char type = '?', type1 = '?', type2 = '?';
+	char func[DHD_FUNC_STR_LEN] = "\0";
+	char func1[DHD_FUNC_STR_LEN] = "\0";
+	char func2[DHD_FUNC_STR_LEN] = "\0";
+	uint8 count = 0;
+	int num, len = 0, offset;
+
+	BCM_REFERENCE(type1);
+	DHD_TRACE(("%s: fname %s pc 0x%x lr 0x%x \n",
+		__FUNCTION__, fname, pc, lr));
+	if (fname == NULL) {
+		DHD_ERROR(("%s: ERROR fname is NULL \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Allocate 1 byte more than read_size to terminate it with NULL */
+	raw_fmts = MALLOCZ(osh, read_size + 1);
+	if (raw_fmts == NULL) {
+		DHD_ERROR(("%s: Failed to allocate raw_fmts memory \n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+#ifdef DHD_LINUX_STD_FW_API
+	err = dhd_os_get_img_fwreq(&fw, fname);
+	if (err < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			err));
+		goto fail;
+	}
+	size = fw->size;
+#else
+#ifdef get_fs
+	fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+	filep = dhd_filp_open(fname, O_RDONLY, 0);
+	if (IS_ERR(filep) || (filep == NULL)) {
+		DHD_ERROR(("%s: Failed to open %s \n",  __FUNCTION__, fname));
+		goto fail;
+	}
+#endif /* DHD_LINUX_STD_FW_API */
+
+	if (pc_fn == NULL) {
+		count |= PC_FOUND_BIT;
+	}
+	if (lr_fn == NULL) {
+		count |= LR_FOUND_BIT;
+	}
+	while (count != ALL_ADDR_VAL)
+	{
+#ifdef DHD_LINUX_STD_FW_API
+		/* Bound check for size before doing memcpy() */
+		if ((mem_offset + read_size) > size) {
+			read_size = size - mem_offset;
+		}
+
+		err = memcpy_s(raw_fmts, read_size,
+			((char *)(fw->data) + mem_offset), read_size);
+		if (err) {
+			DHD_ERROR(("%s: failed to copy raw_fmts, err=%d\n",
+				__FUNCTION__, err));
+			goto fail;
+		}
+#else
+		err = dhd_os_read_file(filep, raw_fmts, read_size);
+		if (err < 0) {
+			DHD_ERROR(("%s: map file read failed err:%d \n",
+				__FUNCTION__, err));
+			goto fail;
+		}
+
+#endif /* DHD_LINUX_STD_FW_API */
+		/* End raw_fmts with NULL as strstr expects NULL terminated
+		* strings
+		*/
+		raw_fmts[read_size] = '\0';
+		raw_fmts_loc = raw_fmts;
+		offset = 0;
+
+		while ((count != ALL_ADDR_VAL) && (offset < read_size))
+		{
+			cptr = bcmstrtok(&raw_fmts_loc, "\n", 0);
+			if (cptr == NULL) {
+				DHD_TRACE(("%s: cptr is NULL, offset %d"
+					" raw_fmts_loc %s \n",
+					__FUNCTION__, offset, raw_fmts_loc));
+				break;
+			}
+			DHD_TRACE(("%s: %s \n", __FUNCTION__, cptr));
+			if ((type2 == 'A') ||
+				(type2 == 'T') ||
+				(type2 == 'W')) {
+				addr1 = addr2;
+				type1 = type2;
+				(void)memcpy_s(func1, DHD_FUNC_STR_LEN,
+					func2, DHD_FUNC_STR_LEN);
+				DHD_TRACE(("%s: %x %c %s \n",
+					__FUNCTION__, addr1, type1, func1));
+			}
+			len = strlen(cptr);
+			num = sscanf(cptr, "%x %c %79s", &addr, &type, func);
+			DHD_TRACE(("%s: num %d addr %x type %c func %s \n",
+				__FUNCTION__, num, addr, type, func));
+			if (num == 3) {
+				addr2 = addr;
+				type2 = type;
+				(void)memcpy_s(func2, DHD_FUNC_STR_LEN,
+					func, DHD_FUNC_STR_LEN);
+			}
+
+			if (!(count & PC_FOUND_BIT) &&
+				(pc >= addr1 && pc < addr2)) {
+				if ((cptr = strchr(func1, '$')) != NULL) {
+					(void)strncpy(func, cptr + 1,
+						DHD_FUNC_STR_LEN - 1);
+				} else {
+					(void)memcpy_s(func, DHD_FUNC_STR_LEN,
+						func1, DHD_FUNC_STR_LEN);
+				}
+				if ((cptr = strstr(func, "__bcmromfn"))
+					!= NULL) {
+					*cptr = 0;
+				}
+				if (pc > addr1) {
+					sprintf(pc_fn, "%.68s+0x%x",
+						func, pc - addr1);
+				} else {
+					(void)memcpy_s(pc_fn, DHD_FUNC_STR_LEN,
+						func, DHD_FUNC_STR_LEN);
+				}
+				count |= PC_FOUND_BIT;
+				DHD_INFO(("%s: found addr1 %x pc %x"
+					" addr2 %x \n",
+					__FUNCTION__, addr1, pc, addr2));
+			}
+			if (!(count & LR_FOUND_BIT) &&
+				(lr >= addr1 && lr < addr2)) {
+				if ((cptr = strchr(func1, '$')) != NULL) {
+					(void)strncpy(func, cptr + 1,
+						DHD_FUNC_STR_LEN - 1);
+				} else {
+					(void)memcpy_s(func, DHD_FUNC_STR_LEN,
+						func1, DHD_FUNC_STR_LEN);
+				}
+				if ((cptr = strstr(func, "__bcmromfn"))
+					!= NULL) {
+					*cptr = 0;
+				}
+				if (lr > addr1) {
+					sprintf(lr_fn, "%.68s+0x%x",
+						func, lr - addr1);
+				} else {
+					(void)memcpy_s(lr_fn, DHD_FUNC_STR_LEN,
+						func, DHD_FUNC_STR_LEN);
+				}
+				count |= LR_FOUND_BIT;
+				DHD_INFO(("%s: found addr1 %x lr %x"
+					" addr2 %x \n",
+					__FUNCTION__, addr1, lr, addr2));
+			}
+			offset += (len + 1);
+		}
+#ifdef DHD_LINUX_STD_FW_API
+		if ((mem_offset + read_size) >= size) {
+			break;
+		}
+
+		memset(raw_fmts, 0, read_size);
+		mem_offset += (read_size -(len + 1));
+#else
+		if (err < (int)read_size) {
+			/*
+			* since we reset file pos back to earlier pos by
+			* bytes of one line we won't reach EOF.
+			* The reason for this is if string is spreaded across
+			* bytes, the read function should not miss it.
+			* So if ret value is less than read_size, reached EOF
+			* don't read further
+			*/
+			break;
+		}
+		memset(raw_fmts, 0, read_size);
+		/*
+		* go back to bytes of one line so that we won't miss
+		* the string and addr even if it comes as splited in next read.
+		*/
+		dhd_os_seek_file(filep, -(len + 1));
+#endif /* DHD_LINUX_STD_FW_API */
+		DHD_TRACE(("%s: seek %d \n", __FUNCTION__, -(len + 1)));
+	}
+
+fail:
+#ifdef DHD_LINUX_STD_FW_API
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+#else
+	if (!IS_ERR(filep))
+		dhd_filp_close(filep, NULL);
+
+#ifdef get_fs
+	set_fs(fs);
+#endif /* get_fs */
+#endif /* DHD_LINUX_STD_FW_API */
+	if (!(count & PC_FOUND_BIT)) {
+		sprintf(pc_fn, "0x%08x", pc);
+	}
+	if (!(count & LR_FOUND_BIT)) {
+		sprintf(lr_fn, "0x%08x", lr);
+	}
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, read_size + 1);
+	}
+	return err;
+}
+#endif /* DHD_COREDUMP */
+
+#ifdef DHD_LINUX_STD_FW_API
+static int
+dhd_init_logstrs_array(osl_t *osh, dhd_event_log_t *temp)
+{
+	char *raw_fmts =  NULL;
+	int logstrs_size = 0;
+	int error = 0;
+	const struct firmware *fw = NULL;
+
+	if (control_logtrace != LOGTRACE_PARSED_FMT) {
+		DHD_ERROR_NO_HW4(("%s : turned off logstr parsing\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	error = dhd_os_get_img_fwreq(&fw, logstrs_path);
+	if (error < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			error));
+		goto fail;
+	}
+
+	logstrs_size = (int)fw->size;
+	if (logstrs_size == 0) {
+		DHD_ERROR(("%s: return as logstrs_size is 0\n", __FUNCTION__));
+		goto fail;
+	}
+
+	if (temp->raw_fmts != NULL) {
+		raw_fmts = temp->raw_fmts;	/* reuse already malloced raw_fmts */
+	} else {
+		raw_fmts = MALLOC(osh, logstrs_size);
+		if (raw_fmts == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory \n", __FUNCTION__));
+			goto fail;
+		}
+		temp->raw_fmts = raw_fmts;
+		temp->raw_fmts_size = logstrs_size;
+	}
+	error = memcpy_s(raw_fmts, logstrs_size, (const char *)(fw->data), logstrs_size);
+	if (error) {
+		DHD_ERROR(("%s: failed to copy raw_fmts, err=%d\n",
+			__FUNCTION__, error));
+		goto fail;
+	}
+	if (dhd_parse_logstrs_file(osh, raw_fmts, logstrs_size, temp) == BCME_OK) {
+		dhd_os_close_img_fwreq(fw);
+		DHD_ERROR(("%s: return ok\n", __FUNCTION__));
+		return BCME_OK;
+	}
+
+fail:
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, logstrs_size);
+	}
+	if (temp->fmts != NULL) {
+		MFREE(osh, temp->fmts, temp->num_fmts * sizeof(char *));
+	}
+
+	temp->fmts = NULL;
+	temp->raw_fmts = NULL;
+
+	return BCME_ERROR;
+}
+
+static int
+dhd_read_map(osl_t *osh, char *fname, uint32 *ramstart, uint32 *rodata_start,
+		uint32 *rodata_end)
+{
+	int err = BCME_ERROR;
+	const struct firmware *fw = NULL;
+
+	if (fname == NULL) {
+		DHD_ERROR(("%s: ERROR fname is NULL \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	err = dhd_os_get_img_fwreq(&fw, fname);
+	if (err < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			err));
+		goto fail;
+	}
+
+	if ((err = dhd_parse_map_file(osh, fw, ramstart,
+			rodata_start, rodata_end)) < 0) {
+		goto fail;
+	}
+
+fail:
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+
+	return err;
+}
+
+static int
+dhd_init_static_strs_array(osl_t *osh, dhd_event_log_t *temp, char *str_file, char *map_file)
+{
+	char *raw_fmts =  NULL;
+	uint32 logstrs_size = 0;
+	int error = 0;
+	uint32 ramstart = 0;
+	uint32 rodata_start = 0;
+	uint32 rodata_end = 0;
+	uint32 logfilebase = 0;
+	const struct firmware *fw = NULL;
+
+	error = dhd_read_map(osh, map_file, &ramstart, &rodata_start, &rodata_end);
+	if (error != BCME_OK) {
+		DHD_ERROR(("readmap Error!! \n"));
+		/* don't do event log parsing in actual case */
+		if (strstr(str_file, ram_file_str) != NULL) {
+			temp->raw_sstr = NULL;
+		} else if (strstr(str_file, rom_file_str) != NULL) {
+			temp->rom_raw_sstr = NULL;
+		}
+		return error;
+	}
+	DHD_ERROR(("ramstart: 0x%x, rodata_start: 0x%x, rodata_end:0x%x\n",
+		ramstart, rodata_start, rodata_end));
+
+	/* Full file size is huge. Just read required part */
+	logstrs_size = rodata_end - rodata_start;
+	logfilebase = rodata_start - ramstart;
+
+	if (logstrs_size == 0) {
+		DHD_ERROR(("%s: return as logstrs_size is 0\n", __FUNCTION__));
+		goto fail1;
+	}
+
+	if (strstr(str_file, ram_file_str) != NULL && temp->raw_sstr != NULL) {
+		raw_fmts = temp->raw_sstr;	/* reuse already malloced raw_fmts */
+	} else if (strstr(str_file, rom_file_str) != NULL && temp->rom_raw_sstr != NULL) {
+		raw_fmts = temp->rom_raw_sstr;	/* reuse already malloced raw_fmts */
+	} else {
+		raw_fmts = MALLOC(osh, logstrs_size);
+
+		if (raw_fmts == NULL) {
+			DHD_ERROR(("%s: Failed to allocate raw_fmts memory \n", __FUNCTION__));
+			goto fail;
+		}
+	}
+
+	error = dhd_os_get_img_fwreq(&fw, str_file);
+	if (error < 0 || (fw == NULL) || (fw->size < logfilebase)) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			error));
+		goto fail;
+	}
+
+	error = memcpy_s(raw_fmts, logstrs_size, (const char *)((fw->data) + logfilebase),
+		logstrs_size);
+	if (error) {
+		DHD_ERROR(("%s: failed to copy raw_fmts, err=%d\n",
+			__FUNCTION__, error));
+		goto fail;
+	}
+
+	if (strstr(str_file, ram_file_str) != NULL) {
+		temp->raw_sstr = raw_fmts;
+		temp->raw_sstr_size = logstrs_size;
+		temp->rodata_start = rodata_start;
+		temp->rodata_end = rodata_end;
+	} else if (strstr(str_file, rom_file_str) != NULL) {
+		temp->rom_raw_sstr = raw_fmts;
+		temp->rom_raw_sstr_size = logstrs_size;
+		temp->rom_rodata_start = rodata_start;
+		temp->rom_rodata_end = rodata_end;
+	}
+
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+
+	return BCME_OK;
+
+fail:
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, logstrs_size);
+	}
+
+fail1:
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+
+	if (strstr(str_file, ram_file_str) != NULL) {
+		temp->raw_sstr = NULL;
+	} else if (strstr(str_file, rom_file_str) != NULL) {
+		temp->rom_raw_sstr = NULL;
+	}
+
+	return error;
+} /* dhd_init_static_strs_array */
+#else
+static int
+dhd_init_logstrs_array(osl_t *osh, dhd_event_log_t *temp)
+{
+	struct file *filep = NULL;
+#ifdef get_fs
+	mm_segment_t fs;
+#endif /* get_fs */
+	char *raw_fmts =  NULL;
+	int logstrs_size = 0;
+
+	if (control_logtrace != LOGTRACE_PARSED_FMT) {
+		DHD_ERROR_NO_HW4(("%s : turned off logstr parsing\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+#ifdef get_fs
+	fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	filep = dhd_filp_open(logstrs_path, O_RDONLY, 0);
+
+	if (IS_ERR(filep) || (filep == NULL)) {
+		DHD_ERROR_NO_HW4(("%s: Failed to open the file %s \n",
+			__FUNCTION__, logstrs_path));
+		goto fail;
+	}
+
+	logstrs_size = dhd_vfs_size_read(filep);
+
+	if (logstrs_size == 0) {
+		DHD_ERROR(("%s: return as logstrs_size is 0\n", __FUNCTION__));
+		goto fail1;
+	}
+
+	if (temp->raw_fmts != NULL) {
+		raw_fmts = temp->raw_fmts;	   /* reuse already malloced raw_fmts */
+	} else {
+		raw_fmts = MALLOC(osh, logstrs_size);
+		if (raw_fmts == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory \n", __FUNCTION__));
+			goto fail;
+		}
+		temp->raw_fmts = raw_fmts;
+		temp->raw_fmts_size = logstrs_size;
+	}
+
+	if (dhd_vfs_read(filep, raw_fmts, logstrs_size, &filep->f_pos) != logstrs_size) {
+		DHD_ERROR_NO_HW4(("%s: Failed to read file %s\n", __FUNCTION__, logstrs_path));
+		goto fail;
+	}
+
+	if (dhd_parse_logstrs_file(osh, raw_fmts, logstrs_size, temp)
+			== BCME_OK) {
+		dhd_filp_close(filep, NULL);
+#ifdef get_fs
+		set_fs(fs);
+#endif /* get_fs */
+		return BCME_OK;
+	}
+
+	fail:
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, logstrs_size);
+	}
+	if (temp->fmts != NULL) {
+		MFREE(osh, temp->fmts, temp->num_fmts * sizeof(char *));
+	}
+
+	fail1:
+	if (!IS_ERR(filep))
+		dhd_filp_close(filep, NULL);
+
+#ifdef get_fs
+	set_fs(fs);
+#endif /* get_fs */
+	temp->fmts = NULL;
+	temp->raw_fmts = NULL;
+
+	return BCME_ERROR;
+}
+
+static int
+dhd_read_map(osl_t *osh, char *fname, uint32 *ramstart, uint32 *rodata_start,
+		uint32 *rodata_end)
+{
+	struct file *filep = NULL;
+#ifdef get_fs
+	mm_segment_t fs;
+#endif /* get_fs */
+	int err = BCME_ERROR;
+
+	if (fname == NULL) {
+		DHD_ERROR(("%s: ERROR fname is NULL \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+#ifdef get_fs
+	fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	filep = dhd_filp_open(fname, O_RDONLY, 0);
+	if (IS_ERR(filep) || (filep == NULL)) {
+		DHD_ERROR_NO_HW4(("%s: Failed to open %s \n",  __FUNCTION__, fname));
+		goto fail;
+	}
+
+	if ((err = dhd_parse_map_file(osh, filep, ramstart,
+			rodata_start, rodata_end)) < 0)
+		goto fail;
+
+fail:
+	if (!IS_ERR(filep))
+		dhd_filp_close(filep, NULL);
+
+#ifdef get_fs
+	set_fs(fs);
+#endif /* get_fs */
+
+	return err;
+}
+
+static int
+dhd_init_static_strs_array(osl_t *osh, dhd_event_log_t *temp, char *str_file, char *map_file)
+{
+	struct file *filep = NULL;
+#ifdef get_fs
+	mm_segment_t fs;
+#endif /* get_fs */
+	char *raw_fmts =  NULL;
+	uint32 logstrs_size = 0;
+	int error = 0;
+	uint32 ramstart = 0;
+	uint32 rodata_start = 0;
+	uint32 rodata_end = 0;
+	uint32 logfilebase = 0;
+
+	error = dhd_read_map(osh, map_file, &ramstart, &rodata_start, &rodata_end);
+	if (error != BCME_OK) {
+		DHD_ERROR(("readmap Error!! \n"));
+		/* don't do event log parsing in actual case */
+		if (strstr(str_file, ram_file_str) != NULL) {
+			temp->raw_sstr = NULL;
+		} else if (strstr(str_file, rom_file_str) != NULL) {
+			temp->rom_raw_sstr = NULL;
+		}
+		return error;
+	}
+	DHD_ERROR(("ramstart: 0x%x, rodata_start: 0x%x, rodata_end:0x%x\n",
+		ramstart, rodata_start, rodata_end));
+
+#ifdef get_fs
+	fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	filep = dhd_filp_open(str_file, O_RDONLY, 0);
+	if (IS_ERR(filep) || (filep == NULL)) {
+		DHD_ERROR(("%s: Failed to open the file %s \n",  __FUNCTION__, str_file));
+		goto fail;
+	}
+
+	if (TRUE) {
+		/* Full file size is huge. Just read required part */
+		logstrs_size = rodata_end - rodata_start;
+		logfilebase = rodata_start - ramstart;
+	}
+
+	if (logstrs_size == 0) {
+		DHD_ERROR(("%s: return as logstrs_size is 0\n", __FUNCTION__));
+		goto fail1;
+	}
+
+	if (strstr(str_file, ram_file_str) != NULL && temp->raw_sstr != NULL) {
+		raw_fmts = temp->raw_sstr;	/* reuse already malloced raw_fmts */
+	} else if (strstr(str_file, rom_file_str) != NULL && temp->rom_raw_sstr != NULL) {
+		raw_fmts = temp->rom_raw_sstr;	/* reuse already malloced raw_fmts */
+	} else {
+		raw_fmts = MALLOC(osh, logstrs_size);
+
+		if (raw_fmts == NULL) {
+			DHD_ERROR(("%s: Failed to allocate raw_fmts memory \n", __FUNCTION__));
+			goto fail;
+		}
+	}
+
+	if (TRUE) {
+		error = generic_file_llseek(filep, logfilebase, SEEK_SET);
+		if (error < 0) {
+			DHD_ERROR(("%s: %s llseek failed %d \n", __FUNCTION__, str_file, error));
+			goto fail;
+		}
+	}
+
+	error = dhd_vfs_read(filep, raw_fmts, logstrs_size, (&filep->f_pos));
+	if (error != logstrs_size) {
+		DHD_ERROR(("%s: %s read failed %d \n", __FUNCTION__, str_file, error));
+		goto fail;
+	}
+
+	if (strstr(str_file, ram_file_str) != NULL) {
+		temp->raw_sstr = raw_fmts;
+		temp->raw_sstr_size = logstrs_size;
+		temp->rodata_start = rodata_start;
+		temp->rodata_end = rodata_end;
+	} else if (strstr(str_file, rom_file_str) != NULL) {
+		temp->rom_raw_sstr = raw_fmts;
+		temp->rom_raw_sstr_size = logstrs_size;
+		temp->rom_rodata_start = rodata_start;
+		temp->rom_rodata_end = rodata_end;
+	} else {
+		if (raw_fmts) {
+			MFREE(osh, raw_fmts, logstrs_size);
+		}
+	}
+
+	dhd_filp_close(filep, NULL);
+#ifdef get_fs
+	set_fs(fs);
+#endif /* get_fs */
+
+	return BCME_OK;
+
+fail:
+	if (raw_fmts) {
+		MFREE(osh, raw_fmts, logstrs_size);
+	}
+
+fail1:
+	if (!IS_ERR(filep))
+		dhd_filp_close(filep, NULL);
+
+#ifdef get_fs
+	set_fs(fs);
+#endif /* get_fs */
+
+	if (strstr(str_file, ram_file_str) != NULL) {
+		temp->raw_sstr = NULL;
+	} else if (strstr(str_file, rom_file_str) != NULL) {
+		temp->rom_raw_sstr = NULL;
+	}
+
+	return error;
+} /* dhd_init_static_strs_array */
+#endif /* DHD_LINUX_STD_FW_API */
+#endif /* SHOW_LOGTRACE */
+
+#ifdef BCMDBUS
+uint
+dhd_get_rxsz(dhd_pub_t *pub)
+{
+	struct net_device *net = NULL;
+	dhd_info_t *dhd = NULL;
+	uint rxsz;
+
+	/* Assign rxsz for dbus_attach */
+	dhd = pub->info;
+	net = dhd->iflist[0]->net;
+	net->hard_header_len = ETH_HLEN + pub->hdrlen;
+	rxsz = DBUS_RX_BUFFER_SIZE_DHD(net);
+
+	return rxsz;
+}
+
+void
+dhd_set_path(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = NULL;
+
+	dhd = pub->info;
+
+	/* try to download image and nvram to the dongle */
+	if	(dhd_update_fw_nv_path(dhd) && dhd->pub.bus) {
+		DHD_INFO(("%s: fw %s, nv %s, conf %s\n",
+			__FUNCTION__, dhd->fw_path, dhd->nv_path, dhd->conf_path));
+		dhd_bus_update_fw_nv_path(dhd->pub.bus,
+				dhd->fw_path, dhd->nv_path, dhd->clm_path, dhd->conf_path);
+	}
+}
+#endif
+
+/** Called once for each hardware (dongle) instance that this DHD manages */
+dhd_pub_t *
+dhd_attach(osl_t *osh, struct dhd_bus *bus, uint bus_hdrlen
+#ifdef BCMDBUS
+	, void *data
+#endif
+)
+{
+	dhd_info_t *dhd = NULL;
+	struct net_device *net = NULL;
+	char if_name[IFNAMSIZ] = {'\0'};
+#if defined(BCMSDIO) || defined(BCMPCIE)
+	uint32 bus_type = -1;
+	uint32 bus_num = -1;
+	uint32 slot_num = -1;
+#if defined(SHOW_LOGTRACE) && !defined(OEM_ANDROID)
+	int ret;
+#endif /* SHOW_LOGTRACE && !OEM_ANDROID */
+	wifi_adapter_info_t *adapter = NULL;
+#elif defined(BCMDBUS)
+	wifi_adapter_info_t *adapter = data;
+#endif
+
+	dhd_attach_states_t dhd_state = DHD_ATTACH_STATE_INIT;
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#ifdef PCIE_FULL_DONGLE
+	ASSERT(sizeof(dhd_pkttag_fd_t) <= OSL_PKTTAG_SZ);
+	ASSERT(sizeof(dhd_pkttag_fr_t) <= OSL_PKTTAG_SZ);
+#endif /* PCIE_FULL_DONGLE */
+
+	/* will implement get_ids for DBUS later */
+#if defined(BCMSDIO) || defined(BCMPCIE)
+	dhd_bus_get_ids(bus, &bus_type, &bus_num, &slot_num);
+	adapter = dhd_wifi_platform_get_adapter(bus_type, bus_num, slot_num);
+#endif
+
+	/* Allocate primary dhd_info */
+	dhd = wifi_platform_prealloc(adapter, DHD_PREALLOC_DHD_INFO, sizeof(dhd_info_t));
+	if (dhd == NULL) {
+		dhd = MALLOC(osh, sizeof(dhd_info_t));
+		if (dhd == NULL) {
+			DHD_ERROR(("%s: OOM - alloc dhd_info\n", __FUNCTION__));
+			goto dhd_null_flag;
+		}
+	}
+	memset(dhd, 0, sizeof(dhd_info_t));
+	dhd_state |= DHD_ATTACH_STATE_DHD_ALLOC;
+
+	dhd->unit = dhd_found + instance_base; /* do not increment dhd_found, yet */
+
+	dhd->pub.osh = osh;
+#ifdef DUMP_IOCTL_IOV_LIST
+	dll_init(&(dhd->pub.dump_iovlist_head));
+#endif /* DUMP_IOCTL_IOV_LIST */
+
+	dhd->pub.dhd_console_ms = dhd_console_ms; /* assigns default value */
+
+	dhd->adapter = adapter;
+	dhd->pub.adapter = (void *)adapter;
+#ifdef BT_OVER_SDIO
+	dhd->pub.is_bt_recovery_required = FALSE;
+	mutex_init(&dhd->bus_user_lock);
+#endif /* BT_OVER_SDIO */
+
+	g_dhd_pub = &dhd->pub;
+
+#ifdef DHD_DEBUG
+	dll_init(&(dhd->pub.mw_list_head));
+#endif /* DHD_DEBUG */
+
+	/*
+	 * Attach DHD Core Layer to Platform Layer. For non
+	 * Embedded environment, where there is no dependency on platform
+	 * layer, dhd_get_plat_info_size can return 0, if the platform
+	 * layer does not exist or chooses not to implement it.
+	 */
+	dhd->pub.plat_info_size = dhd_plat_get_info_size();
+	if (dhd->pub.plat_info_size) {
+		dhd->pub.plat_info = MALLOCZ(osh, dhd->pub.plat_info_size);
+	}
+
+#ifdef CUSTOM_COUNTRY_CODE
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+	dhd->pub.dhd_cflags |= WLAN_PLAT_NODFS_FLAG;
+	dhd->pub.force_country_change = TRUE;
+#endif /* CUSTOM_FORCE_NODFS_FLAG */
+	get_customized_country_code(dhd->adapter,
+		dhd->pub.dhd_cspec.country_abbrev, &dhd->pub.dhd_cspec,
+		dhd->pub.dhd_cflags);
+#endif /* CUSTOM_COUNTRY_CODE */
+#ifndef BCMDBUS
+	dhd->thr_dpc_ctl.thr_pid = DHD_PID_KT_TL_INVALID;
+	dhd->thr_wdt_ctl.thr_pid = DHD_PID_KT_INVALID;
+#ifdef DHD_WET
+	dhd->pub.wet_info = dhd_get_wet_info(&dhd->pub);
+#endif /* DHD_WET */
+	/* Initialize thread based operation and lock */
+	sema_init(&dhd->sdsem, 1);
+#endif /* BCMDBUS */
+#if defined(WL_MONITOR) && defined(HOST_RADIOTAP_CONV)
+	dhd->host_radiotap_conv = FALSE;
+#endif /* WL_MONITOR */
+	dhd->pub.pcie_txs_metadata_enable = pcie_txs_metadata_enable;
+
+	/* Link to info module */
+	dhd->pub.info = dhd;
+
+	/* Link to bus module */
+	dhd->pub.bus = bus;
+	dhd->pub.hdrlen = bus_hdrlen;
+	dhd->pub.txoff = FALSE;
+#ifdef CHECK_TRAP_ROT
+	dhd->pub.check_trap_rot = TRUE;
+#else
+	dhd->pub.check_trap_rot = FALSE;
+#endif /* CHECK_TRAP_ROT */
+
+#ifdef WBRC
+	dhd->pub.chip_bighammer_count = 0;
+#endif /* WBRC */
+
+	/* dhd_conf must be attached after linking dhd to dhd->pub.info,
+	 * because dhd_detech will check .info is NULL or not.
+	*/
+	if (dhd_conf_attach(&dhd->pub) != 0) {
+		DHD_ERROR(("dhd_conf_attach failed\n"));
+		goto fail;
+	}
+#ifndef BCMDBUS
+	dhd_conf_reset(&dhd->pub);
+	dhd_conf_set_chiprev(&dhd->pub, dhd_bus_chip(bus), dhd_bus_chiprev(bus));
+	dhd_conf_preinit(&dhd->pub);
+#endif /* !BCMDBUS */
+
+	/* Some DHD modules (e.g. cfg80211) configures operation mode based on firmware name.
+	 * This is indeed a hack but we have to make it work properly before we have a better
+	 * solution
+	 */
+	dhd_update_fw_nv_path(dhd);
+
+	/* Set network interface name if it was provided as module parameter */
+	if (iface_name[0]) {
+		int len;
+		char ch;
+		strlcpy(if_name, iface_name, sizeof(if_name));
+		len = strlen(if_name);
+		ch = if_name[len - 1];
+		if ((ch > '9' || ch < '0') && (len < IFNAMSIZ - 2)) {
+			strncat(if_name, "%d", sizeof(if_name) - len - 1);
+		}
+	}
+
+	/* Passing NULL to dngl_name to ensure host gets if_name in dngl_name member */
+	net = dhd_allocate_if(&dhd->pub, 0, if_name, NULL, 0, TRUE, NULL);
+	if (net == NULL) {
+		goto fail;
+	}
+	mutex_init(&dhd->pub.ndev_op_sync);
+
+	dhd_state |= DHD_ATTACH_STATE_ADD_IF;
+#ifdef DHD_L2_FILTER
+	/* initialize the l2_filter_cnt */
+	dhd->pub.l2_filter_cnt = 0;
+#endif
+	net->netdev_ops = NULL;
+
+	mutex_init(&dhd->dhd_iovar_mutex);
+	sema_init(&dhd->proto_sem, 1);
+
+#if defined(DHD_HANG_SEND_UP_TEST)
+	dhd->pub.req_hang_type = 0;
+#endif /* DHD_HANG_SEND_UP_TEST */
+
+#ifdef PROP_TXSTATUS
+	spin_lock_init(&dhd->wlfc_spinlock);
+
+	dhd->pub.skip_fc = dhd_wlfc_skip_fc;
+	dhd->pub.plat_init = dhd_wlfc_plat_init;
+	dhd->pub.plat_deinit = dhd_wlfc_plat_deinit;
+
+#ifdef DHD_WLFC_THREAD
+	init_waitqueue_head(&dhd->pub.wlfc_wqhead);
+	dhd->pub.wlfc_thread = kthread_create(dhd_wlfc_transfer_packets, &dhd->pub, "wlfc-thread");
+	if (IS_ERR(dhd->pub.wlfc_thread)) {
+		DHD_ERROR(("create wlfc thread failed\n"));
+		goto fail;
+	} else {
+		wake_up_process(dhd->pub.wlfc_thread);
+	}
+#endif /* DHD_WLFC_THREAD */
+#endif /* PROP_TXSTATUS */
+
+	/* Initialize other structure content */
+	/* XXX Some of this goes away, leftover from USB */
+	/* XXX Some could also move to bus_init()? */
+	init_waitqueue_head(&dhd->ioctl_resp_wait);
+	init_waitqueue_head(&dhd->pub.tx_tput_test_wait);
+	init_waitqueue_head(&dhd->d3ack_wait);
+#ifdef PCIE_INB_DW
+	init_waitqueue_head(&dhd->ds_exit_wait);
+#endif /* PCIE_INB_DW */
+	init_waitqueue_head(&dhd->ctrl_wait);
+	init_waitqueue_head(&dhd->dhd_bus_busy_state_wait);
+	init_waitqueue_head(&dhd->dmaxfer_wait);
+	init_waitqueue_head(&dhd->pub.tx_completion_wait);
+	dhd->pub.dhd_bus_busy_state = 0;
+	/* Initialize the spinlocks */
+	spin_lock_init(&dhd->sdlock);
+	spin_lock_init(&dhd->txqlock);
+	spin_lock_init(&dhd->dhd_lock);
+	spin_lock_init(&dhd->txoff_lock);
+	spin_lock_init(&dhd->rxf_lock);
+#ifdef WLTDLS
+	spin_lock_init(&dhd->pub.tdls_lock);
+#endif /* WLTDLS */
+#if defined(RXFRAME_THREAD)
+	dhd->rxthread_enabled = TRUE;
+#endif /* defined(RXFRAME_THREAD) */
+
+#ifdef DHDTCPACK_SUPPRESS
+	spin_lock_init(&dhd->tcpack_lock);
+#endif /* DHDTCPACK_SUPPRESS */
+
+	/* Initialize Wakelock stuff */
+	spin_lock_init(&dhd->wakelock_spinlock);
+	spin_lock_init(&dhd->wakelock_evt_spinlock);
+	DHD_OS_WAKE_LOCK_INIT(dhd);
+	dhd->wakelock_counter = 0;
+	/* wakelocks prevent a system from going into a low power state */
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_wake_lock_init(dhd->wl_wdwake, dhd_bus_to_dev(bus), "wlan_wd_wake");
+#endif /* CONFIG_HAS_WAKELOCK */
+
+#if defined(OEM_ANDROID)
+	mutex_init(&dhd->dhd_net_if_mutex);
+	mutex_init(&dhd->dhd_suspend_mutex);
+#if defined(APF)
+	mutex_init(&dhd->dhd_apf_mutex);
+#endif /* APF */
+#endif /* defined(OEM_ANDROID) */
+	dhd_state |= DHD_ATTACH_STATE_WAKELOCKS_INIT;
+
+	/* Attach and link in the protocol */
+	if (dhd_prot_attach(&dhd->pub) != 0) {
+		DHD_ERROR(("dhd_prot_attach failed\n"));
+		goto fail;
+	}
+	dhd_state |= DHD_ATTACH_STATE_PROT_ATTACH;
+
+#ifdef WL_CFG80211
+	spin_lock_init(&dhd->pub.up_lock);
+	/* Attach and link in the cfg80211 */
+	if (unlikely(wl_cfg80211_attach(net, &dhd->pub))) {
+		DHD_ERROR(("wl_cfg80211_attach failed\n"));
+		goto fail;
+	}
+
+	dhd_monitor_init(&dhd->pub);
+	dhd_state |= DHD_ATTACH_STATE_CFG80211;
+#endif
+
+#if defined(SHOW_LOGTRACE) && !defined(OEM_ANDROID)
+	ret = dhd_init_logstrs_array(osh, &dhd->event_data);
+	if (ret == BCME_OK) {
+		dhd_init_static_strs_array(osh, &dhd->event_data, st_str_file_path, map_file_path);
+#ifdef DHD_LOAD_ROML_FILES
+		dhd_init_static_strs_array(osh, &dhd->event_data, rom_st_str_file_path,
+			rom_map_file_path);
+#endif /* DHD_LOAD_ROML_FILES */
+		dhd_state |= DHD_ATTACH_LOGTRACE_INIT;
+	}
+#endif /* SHOW_LOGTRACE && !OEM_ANDROID */
+
+#ifdef WL_EVENT
+	if (wl_ext_event_attach(net) != 0) {
+		DHD_ERROR(("wl_ext_event_attach failed\n"));
+		goto fail;
+	}
+#endif /* WL_EVENT */
+#ifdef WL_TIMER
+	if (wl_timer_attach(net) != 0) {
+		DHD_ERROR(("wl_ext_timer_attach failed\n"));
+		goto fail;
+	}
+#endif /* WL_TIMER */
+#ifdef WL_ESCAN
+	/* Attach and link in the escan */
+	if (wl_escan_attach(net) != 0) {
+		DHD_ERROR(("wl_escan_attach failed\n"));
+		goto fail;
+	}
+#endif /* WL_ESCAN */
+#ifdef WL_EXT_IAPSTA
+	if (wl_ext_iapsta_attach(net) != 0) {
+		DHD_ERROR(("wl_ext_iapsta_attach failed\n"));
+		goto fail;
+	}
+#endif /* WL_EXT_IAPSTA */
+#ifdef WL_EXT_GENL
+	if (wl_ext_genl_init(net)) {
+		DHD_ERROR(("wl_ext_genl_init failed\n"));
+		goto fail;
+	}
+#endif
+#if defined(WL_WIRELESS_EXT)
+	/* Attach and link in the iw */
+	if (wl_iw_attach(net) != 0) {
+		DHD_ERROR(("wl_iw_attach failed\n"));
+		goto fail;
+	}
+	dhd_state |= DHD_ATTACH_STATE_WL_ATTACH;
+#endif /* defined(WL_WIRELESS_EXT) */
+
+#ifdef DHD_LOG_DUMP
+	dhd_log_dump_init(&dhd->pub);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_PKTDUMP_ROAM
+	dhd_dump_pkt_init(&dhd->pub);
+#endif /* DHD_PKTDUMP_ROAM */
+#ifdef DHD_PKT_LOGGING
+	dhd_os_attach_pktlog(&dhd->pub);
+#endif /* DHD_PKT_LOGGING */
+
+	/* attach debug if support */
+	if (dhd_os_dbg_attach(&dhd->pub)) {
+		DHD_ERROR(("%s debug module attach failed\n", __FUNCTION__));
+		goto fail;
+	}
+#ifdef DEBUGABILITY
+#if !defined(OEM_ANDROID) && defined(SHOW_LOGTRACE)
+	/* enable verbose ring to support dump_trace_buf */
+	dhd_os_start_logging(&dhd->pub, FW_VERBOSE_RING_NAME, 3, 0, 0, 0);
+#endif /* !OEM_ANDROID && SHOW_LOGTRACE */
+
+#ifdef DBG_PKT_MON
+	dhd->pub.dbg->pkt_mon_lock = osl_spin_lock_init(dhd->pub.osh);
+#ifdef DBG_PKT_MON_INIT_DEFAULT
+	dhd_os_dbg_attach_pkt_monitor(&dhd->pub);
+#endif /* DBG_PKT_MON_INIT_DEFAULT */
+#endif /* DBG_PKT_MON */
+
+#endif /* DEBUGABILITY */
+
+#ifdef DHD_MEM_STATS
+	dhd->pub.mem_stats_lock = osl_spin_lock_init(dhd->pub.osh);
+	dhd->pub.txpath_mem = 0;
+	dhd->pub.rxpath_mem = 0;
+#endif /* DHD_MEM_STATS */
+
+#ifdef DHD_STATUS_LOGGING
+	dhd->pub.statlog = dhd_attach_statlog(&dhd->pub, MAX_STATLOG_ITEM,
+		MAX_STATLOG_REQ_ITEM, STATLOG_LOGBUF_LEN);
+	if (dhd->pub.statlog == NULL) {
+		DHD_ERROR(("%s: alloc statlog failed\n", __FUNCTION__));
+	}
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef RX_PKT_POOL
+	dhd_rx_pktpool_init(dhd);
+#endif /* RX_PKT_POOL */
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+	dhd->pub.hang_info = MALLOCZ(osh, VENDOR_SEND_HANG_EXT_INFO_LEN);
+	if (dhd->pub.hang_info == NULL) {
+		DHD_ERROR(("%s: alloc hang_info failed\n", __FUNCTION__));
+		goto fail;
+	}
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+
+	if (dhd_sta_pool_init(&dhd->pub, DHD_MAX_STA) != BCME_OK) {
+		DHD_ERROR(("%s: Initializing %u sta\n", __FUNCTION__, DHD_MAX_STA));
+		goto fail;
+	}
+
+#ifdef BCM_ROUTER_DHD
+#if defined(HNDCTF)
+	dhd->cih = ctf_attach(dhd->pub.osh, "dhd", &dhd_msg_level, dhd_ctf_detach, dhd);
+	if (!dhd->cih) {
+		DHD_ERROR(("%s: ctf_attach() failed\n", __FUNCTION__));
+	}
+#ifdef CTFPOOL
+	{
+		int poolsz = RXBUFPOOLSZ;
+		if (CTF_ENAB(dhd->cih) && (osl_ctfpool_init(dhd->pub.osh,
+			poolsz, RXBUFSZ + BCMEXTRAHDROOM) < 0)) {
+			DHD_ERROR(("%s: osl_ctfpool_init() failed\n", __FUNCTION__));
+		}
+	}
+#endif /* CTFPOOL */
+#endif /* HNDCTF */
+#endif /* BCM_ROUTER_DHD */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	dhd->tx_wq = alloc_workqueue("bcmdhd-tx-wq", WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1);
+	if (!dhd->tx_wq) {
+		DHD_ERROR(("%s: alloc_workqueue(bcmdhd-tx-wq) failed\n", __FUNCTION__));
+		goto fail;
+	}
+	dhd->rx_wq = alloc_workqueue("bcmdhd-rx-wq", WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1);
+	if (!dhd->rx_wq) {
+		DHD_ERROR(("%s: alloc_workqueue(bcmdhd-rx-wq) failed\n", __FUNCTION__));
+		destroy_workqueue(dhd->tx_wq);
+		dhd->tx_wq = NULL;
+		goto fail;
+	}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifndef BCMDBUS
+	/* Set up the watchdog timer */
+	init_timer_compat(&dhd->timer, dhd_watchdog, dhd);
+	dhd->default_wd_interval = dhd_watchdog_ms;
+
+	if (dhd_watchdog_prio >= 0) {
+		/* Initialize watchdog thread */
+		PROC_START(dhd_watchdog_thread, dhd, &dhd->thr_wdt_ctl, 0, "dhd_watchdog_thread");
+		if (dhd->thr_wdt_ctl.thr_pid < 0) {
+			goto fail;
+		}
+
+	} else {
+		dhd->thr_wdt_ctl.thr_pid = -1;
+	}
+
+	/* Set up the bottom half handler */
+	if (dhd_dpc_prio >= 0) {
+		/* Initialize DPC thread */
+		PROC_START(dhd_dpc_thread, dhd, &dhd->thr_dpc_ctl, 0, "dhd_dpc");
+		if (dhd->thr_dpc_ctl.thr_pid < 0) {
+			goto fail;
+		}
+	} else {
+		/*  use tasklet for dpc */
+		tasklet_init(&dhd->tasklet, dhd_dpc, (ulong)dhd);
+		dhd->thr_dpc_ctl.thr_pid = -1;
+	}
+
+	if (dhd->rxthread_enabled) {
+		bzero(&dhd->pub.skbbuf[0], sizeof(void *) * MAXSKBPEND);
+		/* Initialize RXF thread */
+		PROC_START(dhd_rxf_thread, dhd, &dhd->thr_rxf_ctl, 0, "dhd_rxf");
+		if (dhd->thr_rxf_ctl.thr_pid < 0) {
+			goto fail;
+		}
+	}
+#endif /* !BCMDBUS */
+
+#ifdef SHOW_LOGTRACE
+	skb_queue_head_init(&dhd->evt_trace_queue);
+
+	/* Create ring proc entries */
+	dhd_dbg_ring_proc_create(&dhd->pub);
+#endif /* SHOW_LOGTRACE */
+
+	dhd_state |= DHD_ATTACH_STATE_THREADS_CREATED;
+
+#if defined(CONFIG_PM_SLEEP)
+	if (!dhd_pm_notifier_registered) {
+		dhd_pm_notifier_registered = TRUE;
+		dhd->pm_notifier.notifier_call = dhd_pm_callback;
+		dhd->pm_notifier.priority = 10;
+		register_pm_notifier(&dhd->pm_notifier);
+	}
+
+#endif /* CONFIG_PM_SLEEP */
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+	dhd->early_suspend.level = EARLY_SUSPEND_LEVEL_BLANK_SCREEN + 20;
+	dhd->early_suspend.suspend = dhd_early_suspend;
+	dhd->early_suspend.resume = dhd_late_resume;
+	register_early_suspend(&dhd->early_suspend);
+	dhd_state |= DHD_ATTACH_STATE_EARLYSUSPEND_DONE;
+#endif /* CONFIG_HAS_EARLYSUSPEND && DHD_USE_EARLYSUSPEND */
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	dhd->pend_ipaddr = 0;
+	if (!dhd_inetaddr_notifier_registered) {
+		dhd_inetaddr_notifier_registered = TRUE;
+		register_inetaddr_notifier(&dhd_inetaddr_notifier);
+	}
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+	if (!dhd_inet6addr_notifier_registered) {
+		dhd_inet6addr_notifier_registered = TRUE;
+		register_inet6addr_notifier(&dhd_inet6addr_notifier);
+	}
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+	dhd->dhd_deferred_wq = dhd_deferred_work_init((void *)dhd);
+#if defined(OEM_ANDROID)
+	INIT_WORK(&dhd->dhd_hang_process_work, dhd_hang_process);
+#endif /* OEM_ANDROID */
+#ifdef DEBUG_CPU_FREQ
+	dhd->new_freq = alloc_percpu(int);
+	dhd->freq_trans.notifier_call = dhd_cpufreq_notifier;
+	cpufreq_register_notifier(&dhd->freq_trans, CPUFREQ_TRANSITION_NOTIFIER);
+#endif
+#ifdef DHDTCPACK_SUPPRESS
+#ifdef BCMSDIO
+	dhd_tcpack_suppress_set(&dhd->pub, TCPACK_SUP_DELAYTX);
+#elif defined(BCMPCIE)
+	/* xxx : In case of PCIe based Samsung Android project, enable TCP ACK Suppress
+	 *       when throughput is higher than threshold, following rps_cpus setting.
+	 */
+	dhd_tcpack_suppress_set(&dhd->pub, TCPACK_SUP_HOLD);
+#else
+	dhd_tcpack_suppress_set(&dhd->pub, TCPACK_SUP_OFF);
+#endif /* BCMSDIO */
+#endif /* DHDTCPACK_SUPPRESS */
+
+#if defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW)
+#endif /* defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW) */
+
+#ifdef DHD_DEBUG_PAGEALLOC
+	register_page_corrupt_cb(dhd_page_corrupt_cb, &dhd->pub);
+#endif /* DHD_DEBUG_PAGEALLOC */
+
+	INIT_DELAYED_WORK(&dhd->dhd_dpc_dispatcher_work, dhd_dpc_tasklet_dispatcher_work);
+
+#if defined(DHD_LB)
+#if defined(DHD_LB_HOST_CTRL)
+	dhd->permitted_primary_cpu = FALSE;
+#endif /* DHD_LB_HOST_CTRL */
+	dhd_lb_set_default_cpus(dhd);
+	DHD_LB_STATS_INIT(&dhd->pub);
+
+	/* Initialize the CPU Masks */
+	if (dhd_cpumasks_init(dhd) == 0) {
+		/* Now we have the current CPU maps, run through candidacy */
+		dhd_select_cpu_candidacy(dhd);
+
+		/* Register the call backs to CPU Hotplug sub-system */
+		dhd_register_cpuhp_callback(dhd);
+
+	} else {
+		/*
+		* We are unable to initialize CPU masks, so candidacy algorithm
+		* won't run, but still Load Balancing will be honoured based
+		* on the CPUs allocated for a given job statically during init
+		*/
+		dhd->cpu_notifier.notifier_call = NULL;
+		DHD_ERROR(("%s():dhd_cpumasks_init failed CPUs for JOB would be static\n",
+			__FUNCTION__));
+	}
+
+#ifdef DHD_LB_TXP
+#ifdef DHD_LB_TXP_DEFAULT_ENAB
+	/* Trun ON the feature by default */
+	atomic_set(&dhd->lb_txp_active, 1);
+#else
+	/* Trun OFF the feature by default */
+	atomic_set(&dhd->lb_txp_active, 0);
+#endif /* DHD_LB_TXP_DEFAULT_ENAB */
+#endif /* DHD_LB_TXP */
+
+#ifdef DHD_LB_RXP
+	/* Trun ON the feature by default */
+	atomic_set(&dhd->lb_rxp_active, 1);
+#endif /* DHD_LB_RXP */
+
+	/* Initialize the Load Balancing Tasklets and Napi object */
+#if defined(DHD_LB_RXP)
+	__skb_queue_head_init(&dhd->rx_pend_queue);
+	skb_queue_head_init(&dhd->rx_emerge_queue);
+	skb_queue_head_init(&dhd->rx_napi_queue);
+	__skb_queue_head_init(&dhd->rx_process_queue);
+	/* Initialize the work that dispatches NAPI job to a given core */
+	INIT_WORK(&dhd->rx_napi_dispatcher_work, dhd_rx_napi_dispatcher_work);
+	DHD_INFO(("%s load balance init rx_napi_queue\n", __FUNCTION__));
+	/* Initialize the work that dispatches DPC tasklet to a given core */
+#endif /* DHD_LB_RXP */
+
+#if defined(DHD_LB_TXP)
+	INIT_WORK(&dhd->tx_dispatcher_work, dhd_tx_dispatcher_work);
+	skb_queue_head_init(&dhd->tx_pend_queue);
+	/* Initialize the work that dispatches TX job to a given core */
+	tasklet_init(&dhd->tx_tasklet,
+		dhd_lb_tx_handler, (ulong)(dhd));
+	DHD_INFO(("%s load balance init tx_pend_queue\n", __FUNCTION__));
+#endif /* DHD_LB_TXP */
+
+	dhd_state |= DHD_ATTACH_STATE_LB_ATTACH_DONE;
+#endif /* DHD_LB */
+
+#ifdef CSI_SUPPORT
+	dhd_csi_init(&dhd->pub);
+#endif /* CSI_SUPPORT */
+
+#if defined(DNGL_AXI_ERROR_LOGGING) && defined(DHD_USE_WQ_FOR_DNGL_AXI_ERROR)
+	INIT_WORK(&dhd->axi_error_dispatcher_work, dhd_axi_error_dispatcher_fn);
+#endif /* DNGL_AXI_ERROR_LOGGING && DHD_USE_WQ_FOR_DNGL_AXI_ERROR */
+
+#if defined(BCMDBG) && defined(DHD_MACDBG)
+	if (dhd_macdbg_attach(&dhd->pub) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_macdbg_attach fail\n", __FUNCTION__));
+		goto fail;
+	}
+#endif /* BCMDBG && DHD_MACDBG */
+
+#ifdef REPORT_FATAL_TIMEOUTS
+	init_dhd_timeouts(&dhd->pub);
+#endif /* REPORT_FATAL_TIMEOUTS */
+#if defined(BCMPCIE)
+	dhd->pub.extended_trap_data = MALLOCZ(osh, BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+	if (dhd->pub.extended_trap_data == NULL) {
+		DHD_ERROR(("%s: Failed to alloc extended_trap_data\n", __FUNCTION__));
+	}
+#ifdef DNGL_AXI_ERROR_LOGGING
+	dhd->pub.axi_err_dump = MALLOCZ(osh, sizeof(dhd_axi_error_dump_t));
+	if (dhd->pub.axi_err_dump == NULL) {
+		DHD_ERROR(("%s: Failed to alloc axi_err_dump\n", __FUNCTION__));
+	}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+#endif /* BCMPCIE */
+
+#ifdef SHOW_LOGTRACE
+	if (dhd_init_logtrace_process(dhd) != BCME_OK) {
+		goto fail;
+	}
+#endif /* SHOW_LOGTRACE */
+
+#ifdef EWP_EDL
+	INIT_DELAYED_WORK(&dhd->edl_dispatcher_work, dhd_edl_process_work);
+#endif
+
+	DHD_COREDUMP_MEMPOOL_INIT(&dhd->pub);
+	DHD_SSSR_MEMPOOL_INIT(&dhd->pub);
+	DHD_SSSR_REG_INFO_INIT(&dhd->pub);
+
+#ifdef DHD_SDTC_ETB_DUMP
+	dhd_sdtc_etb_mempool_init(&dhd->pub);
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#ifdef EWP_EDL
+	if (host_edl_support) {
+		if (DHD_EDL_MEM_INIT(&dhd->pub) != BCME_OK) {
+			host_edl_support = FALSE;
+		}
+	}
+#endif /* EWP_EDL */
+
+	dhd_init_sock_flows_buf(dhd, dhd_watchdog_ms);
+
+	(void)dhd_sysfs_init(dhd);
+
+#ifdef GDB_PROXY
+	dhd->pub.gdb_proxy_nodeadman = nodeadman != 0;
+#endif /* GDB_PROXY */
+	dhd_state |= DHD_ATTACH_STATE_DONE;
+	dhd->dhd_state = dhd_state;
+
+	dhd_found++;
+
+#ifdef DHD_FW_COREDUMP
+	/* Set memdump default values */
+#if defined(OEM_ANDROID)
+#ifdef DHD_COREDUMP
+	dhd->pub.memdump_enabled = DUMP_MEMFILE;
+#else
+	dhd->pub.memdump_enabled = DUMP_MEMFILE_BUGON;
+#endif /* DHD_COREDUMP */
+#else
+	dhd->pub.memdump_enabled = DUMP_MEMFILE;
+#endif /* OEM_ANDROID */
+	/* Check the memdump capability */
+	dhd_get_memdump_info(&dhd->pub);
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef DHD_DUMP_MNGR
+	dhd->pub.dump_file_manage =
+		(dhd_dump_file_manage_t *)MALLOCZ(dhd->pub.osh, sizeof(dhd_dump_file_manage_t));
+	if (unlikely(!dhd->pub.dump_file_manage)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+					"dhd_dump_file_manage_t\n", __FUNCTION__));
+	}
+#endif /* DHD_DUMP_MNGR */
+
+#ifdef RTT_SUPPORT
+	if (dhd_rtt_attach(&dhd->pub)) {
+		DHD_ERROR(("dhd_rtt_attach has failed\n"));
+		goto fail;
+	}
+#endif /* RTT_SUPPORT */
+
+#ifdef DHD_TX_PROFILE
+	if (dhd_tx_profile_attach(&dhd->pub) != BCME_OK) {
+		DHD_ERROR(("%s:\tdhd_tx_profile_attach has failed\n", __FUNCTION__));
+		goto fail;
+	}
+#endif /* defined(DHD_TX_PROFILE) */
+
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+	INIT_WORK(&dhd->dhd_alert_process_work, dhd_alert_process);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+#if defined(DHD_ENABLE_S2MPU)
+	dhd_module_s2mpu_register(dhd_bus_to_dev(bus));
+#endif /* DHD_ENABLE_S2MPU */
+#ifdef DHD_REGISTER_SMMU_FAULT_HANDLER
+	pcie_sysmmu_add_fault_handler(&dhd_pcie_sysmmu_nb);
+#endif /* DHD_REGISTER_SMMU_FAULT_HANDLER */
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	OSL_ATOMIC_SET(dhd->pub.osh, &dhd->dump_status, DUMP_NOT_READY);
+	INIT_WORK(&dhd->dhd_dump_proc_work, dhd_dump_proc);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	return &dhd->pub;
+
+fail:
+	if (dhd_state >= DHD_ATTACH_STATE_DHD_ALLOC) {
+		DHD_TRACE(("%s: Calling dhd_detach dhd_state 0x%x &dhd->pub %p\n",
+			__FUNCTION__, dhd_state, &dhd->pub));
+		dhd->dhd_state = dhd_state;
+		dhd_detach(&dhd->pub);
+		dhd_free(&dhd->pub);
+	}
+
+dhd_null_flag:
+	return NULL;
+}
+
+int dhd_get_fw_mode(dhd_info_t *dhdinfo)
+{
+	if (strstr(dhdinfo->fw_path, "_apsta") != NULL)
+		return DHD_FLAG_HOSTAP_MODE;
+	if (strstr(dhdinfo->fw_path, "_p2p") != NULL)
+		return DHD_FLAG_P2P_MODE;
+	if (strstr(dhdinfo->fw_path, "_ibss") != NULL)
+		return DHD_FLAG_IBSS_MODE;
+	if (strstr(dhdinfo->fw_path, "_mfg") != NULL)
+		return DHD_FLAG_MFG_MODE;
+
+	return DHD_FLAG_STA_MODE;
+}
+
+int dhd_bus_get_fw_mode(dhd_pub_t *dhdp)
+{
+	return dhd_get_fw_mode(dhdp->info);
+}
+
+extern char * nvram_get(const char *name);
+bool dhd_update_fw_nv_path(dhd_info_t *dhdinfo)
+{
+	int fw_len;
+	int nv_len;
+	int clm_len;
+	int conf_len;
+	int sig_len;
+	const char *fw = NULL;
+	const char *nv = NULL;
+	const char *clm = NULL;
+	const char *conf = NULL;
+#ifdef DHD_UCODE_DOWNLOAD
+	int uc_len;
+	const char *uc = NULL;
+#endif /* DHD_UCODE_DOWNLOAD */
+	wifi_adapter_info_t *adapter = dhdinfo->adapter;
+	int fw_path_len = sizeof(dhdinfo->fw_path);
+	int nv_path_len = sizeof(dhdinfo->nv_path);
+	int clm_path_len = sizeof(dhdinfo->clm_path);
+	int sig_path_len = sizeof(dhdinfo->sig_path);
+
+	/* Update firmware and nvram path. The path may be from adapter info or module parameter
+	 * The path from adapter info is used for initialization only (as it won't change).
+	 *
+	 * The firmware_path/nvram_path module parameter may be changed by the system at run
+	 * time. When it changes we need to copy it to dhdinfo->fw_path. Also Android private
+	 * command may change dhdinfo->fw_path. As such we need to clear the path info in
+	 * module parameter after it is copied. We won't update the path until the module parameter
+	 * is changed again (first character is not '\0')
+	 */
+
+	/* set default firmware and nvram path for built-in type driver */
+//	if (!dhd_download_fw_on_driverload) {
+#ifdef DHD_LINUX_STD_FW_API
+		fw = DHD_FW_NAME;
+		nv = DHD_NVRAM_NAME;
+#else
+#ifdef CONFIG_BCMDHD_FW_PATH
+		fw = VENDOR_PATH CONFIG_BCMDHD_FW_PATH;
+#endif /* CONFIG_BCMDHD_FW_PATH */
+#ifdef CONFIG_BCMDHD_NVRAM_PATH
+		nv = VENDOR_PATH CONFIG_BCMDHD_NVRAM_PATH;
+#endif /* CONFIG_BCMDHD_NVRAM_PATH */
+#endif /* DHD_LINUX_STD_FW_API */
+//	}
+
+	/* check if we need to initialize the path */
+	if (dhdinfo->fw_path[0] == '\0') {
+		if (adapter && adapter->fw_path && adapter->fw_path[0] != '\0') {
+			fw = adapter->fw_path;
+		}
+	}
+
+	if (dhdinfo->nv_path[0] == '\0') {
+		if (adapter && adapter->nv_path && adapter->nv_path[0] != '\0')
+			nv = adapter->nv_path;
+	}
+	if (dhdinfo->clm_path[0] == '\0') {
+		if (adapter && adapter->clm_path && adapter->clm_path[0] != '\0')
+			clm = adapter->clm_path;
+	}
+	if (dhdinfo->conf_path[0] == '\0') {
+		if (adapter && adapter->conf_path && adapter->conf_path[0] != '\0')
+			conf = adapter->conf_path;
+	}
+
+	/* Use module parameter if it is valid, EVEN IF the path has not been initialized
+	 *
+	 * TODO: need a solution for multi-chip, can't use the same firmware for all chips
+	 */
+	if (firmware_path[0] != '\0') {
+		/* Newline is getting added sometimes when firmware_path module parameter
+		 * is updated via command line using echo command, due to which
+		 * file not found error is seen, hence replacing '\n' with '\0'.
+		 */
+		if (firmware_path[strlen(firmware_path)-1] == '\n') {
+			firmware_path[strlen(firmware_path)-1] = '\0';
+		}
+		fw = firmware_path;
+		DHD_INFO(("firmware_path:%s\n", firmware_path));
+	}
+
+	if (nvram_path[0] != '\0') {
+		if (nvram_path[strlen(nvram_path)-1] == '\n') {
+			nvram_path[strlen(nvram_path)-1] = '\0';
+		}
+		nv = nvram_path;
+		DHD_INFO(("nvram_path:%s\n", nvram_path));
+	}
+
+	if (clm_path[0] != '\0') {
+		if (clm_path[strlen(clm_path)-1] == '\n') {
+			clm_path[strlen(clm_path)-1] = '\0';
+		}
+		clm = clm_path;
+		DHD_INFO(("clm_path:%s\n", clm_path));
+	}
+
+	if (config_path[0] != '\0') {
+		if (config_path[strlen(config_path)-1] == '\n') {
+			config_path[strlen(config_path)-1] = '\0';
+		}
+		conf = config_path;
+		DHD_INFO(("config_path:%s\n", config_path));
+	}
+
+	if (signature_path[0] != '\0') {
+		if (signature_path[strlen(signature_path) - 1] == '\n') {
+			signature_path[strlen(signature_path) - 1] = '\0';
+		}
+	}
+
+#ifdef DHD_UCODE_DOWNLOAD
+	if (ucode_path[0] != '\0')
+		uc = ucode_path;
+#endif /* DHD_UCODE_DOWNLOAD */
+
+#ifdef BCM_ROUTER_DHD
+	if (!fw) {
+		char var[32];
+
+		snprintf(var, sizeof(var), "firmware_path%d", dhdinfo->unit);
+		fw = nvram_get(var);
+	}
+	if (!nv) {
+		char var[32];
+
+		snprintf(var, sizeof(var), "nvram_path%d", dhdinfo->unit);
+		nv = nvram_get(var);
+	}
+	DHD_INFO(("dhd:%d: fw path:%s nv path:%s\n", dhdinfo->unit, fw, nv));
+#endif
+
+	if (fw && fw[0] != '\0') {
+		fw_len = strlen(fw);
+		if (fw_len >= fw_path_len) {
+			DHD_ERROR(("fw path len exceeds max len of dhdinfo->fw_path\n"));
+			return FALSE;
+		}
+		strlcpy(dhdinfo->fw_path, fw, fw_path_len);
+	}
+	if (nv && nv[0] != '\0') {
+		nv_len = strlen(nv);
+		if (nv_len >= nv_path_len) {
+			DHD_ERROR(("nvram path len exceeds max len of dhdinfo->nv_path\n"));
+			return FALSE;
+		}
+		memset(dhdinfo->nv_path, 0, nv_path_len);
+		strlcpy(dhdinfo->nv_path, nv, nv_path_len);
+#ifdef DHD_USE_SINGLE_NVRAM_FILE
+		/* Remove "_net" or "_mfg" tag from current nvram path */
+		{
+			char *nvram_tag = "nvram_";
+			char *ext_tag = ".txt";
+			char *sp_nvram = strnstr(dhdinfo->nv_path, nvram_tag, nv_path_len);
+			bool valid_buf = sp_nvram && ((uint32)(sp_nvram + strlen(nvram_tag) +
+				strlen(ext_tag) - dhdinfo->nv_path) <= nv_path_len);
+			if (valid_buf) {
+				char *sp = sp_nvram + strlen(nvram_tag) - 1;
+				uint32 padding_size = (uint32)(dhdinfo->nv_path +
+					nv_path_len - sp);
+				memset(sp, 0, padding_size);
+				strncat(dhdinfo->nv_path, ext_tag, strlen(ext_tag));
+				nv_len = strlen(dhdinfo->nv_path);
+				DHD_INFO(("%s: new nvram path = %s\n",
+					__FUNCTION__, dhdinfo->nv_path));
+			} else if (sp_nvram) {
+				DHD_ERROR(("%s: buffer space for nvram path is not enough\n",
+					__FUNCTION__));
+				return FALSE;
+			} else {
+				DHD_ERROR(("%s: Couldn't find the nvram tag. current"
+					" nvram path = %s\n", __FUNCTION__, dhdinfo->nv_path));
+			}
+		}
+#endif /* DHD_USE_SINGLE_NVRAM_FILE */
+	}
+	/* CLM blob */
+	if (clm && clm[0] != '\0') {
+		clm_len = strlen(clm);
+		if (clm_len >= clm_path_len) {
+			DHD_ERROR(("clm path len exceeds max len of dhdinfo->clm_path\n"));
+			return FALSE;
+		}
+		strlcpy(dhdinfo->clm_path, clm, clm_path_len);
+	}
+
+	if (conf && conf[0] != '\0') {
+		conf_len = strlen(conf);
+		if (conf_len >= sizeof(dhdinfo->conf_path)) {
+			DHD_ERROR(("config path len exceeds max len of dhdinfo->conf_path\n"));
+			return FALSE;
+		}
+		strncpy(dhdinfo->conf_path, conf, sizeof(dhdinfo->conf_path));
+		if (dhdinfo->conf_path[conf_len-1] == '\n')
+		       dhdinfo->conf_path[conf_len-1] = '\0';
+	}
+	if (signature_path[0] != '\0') {
+		sig_len = strlen(signature_path);
+		if (sig_len >= sig_path_len) {
+			DHD_ERROR(("signature path len exceeds max len of dhdinfo->sig_path\n"));
+			return FALSE;
+		}
+		strlcpy(dhdinfo->sig_path, signature_path, sig_path_len);
+	}
+
+#ifdef DHD_UCODE_DOWNLOAD
+	if (uc && uc[0] != '\0') {
+		uc_len = strlen(uc);
+		if (uc_len >= sizeof(dhdinfo->uc_path)) {
+			DHD_ERROR(("uc path len exceeds max len of dhdinfo->uc_path\n"));
+			return FALSE;
+		}
+		strlcpy(dhdinfo->uc_path, uc, sizeof(dhdinfo->uc_path));
+	}
+#endif /* DHD_UCODE_DOWNLOAD */
+
+#if 0
+	/* clear the path in module parameter */
+	if (dhd_download_fw_on_driverload) {
+		firmware_path[0] = '\0';
+		nvram_path[0] = '\0';
+		signature_path[0] = '\0';
+	}
+#endif
+#ifdef DHD_UCODE_DOWNLOAD
+	ucode_path[0] = '\0';
+	DHD_INFO(("ucode path: %s\n", dhdinfo->uc_path));
+#endif /* DHD_UCODE_DOWNLOAD */
+
+#ifndef BCMEMBEDIMAGE
+	/* fw_path and nv_path are not mandatory for BCMEMBEDIMAGE */
+	if (dhdinfo->fw_path[0] == '\0') {
+		DHD_ERROR(("firmware path not found\n"));
+		return FALSE;
+	}
+	if (dhdinfo->nv_path[0] == '\0') {
+		DHD_ERROR(("nvram path not found\n"));
+		return FALSE;
+	}
+#endif /* BCMEMBEDIMAGE */
+
+	return TRUE;
+}
+
+#if defined(BT_OVER_SDIO)
+extern bool dhd_update_btfw_path(dhd_info_t *dhdinfo, char* btfw_path)
+{
+	int fw_len;
+	const char *fw = NULL;
+	wifi_adapter_info_t *adapter = dhdinfo->adapter;
+
+	/* Update bt firmware path. The path may be from adapter info or module parameter
+	 * The path from adapter info is used for initialization only (as it won't change).
+	 *
+	 * The btfw_path module parameter may be changed by the system at run
+	 * time. When it changes we need to copy it to dhdinfo->btfw_path. Also Android private
+	 * command may change dhdinfo->btfw_path. As such we need to clear the path info in
+	 * module parameter after it is copied. We won't update the path until the module parameter
+	 * is changed again (first character is not '\0')
+	 */
+
+	/* set default firmware and nvram path for built-in type driver */
+	if (!dhd_download_fw_on_driverload) {
+#ifdef CONFIG_BCMDHD_BTFW_PATH
+		fw = CONFIG_BCMDHD_BTFW_PATH;
+#endif /* CONFIG_BCMDHD_FW_PATH */
+	}
+
+	/* check if we need to initialize the path */
+	if (dhdinfo->btfw_path[0] == '\0') {
+		if (adapter && adapter->btfw_path && adapter->btfw_path[0] != '\0')
+			fw = adapter->btfw_path;
+	}
+
+	/* Use module parameter if it is valid, EVEN IF the path has not been initialized
+	 */
+	if (btfw_path[0] != '\0')
+		fw = btfw_path;
+
+	if (fw && fw[0] != '\0') {
+		fw_len = strlen(fw);
+		if (fw_len >= sizeof(dhdinfo->btfw_path)) {
+			DHD_ERROR(("fw path len exceeds max len of dhdinfo->btfw_path\n"));
+			return FALSE;
+		}
+		strlcpy(dhdinfo->btfw_path, fw, sizeof(dhdinfo->btfw_path));
+	}
+
+	/* clear the path in module parameter */
+	btfw_path[0] = '\0';
+
+	if (dhdinfo->btfw_path[0] == '\0') {
+		DHD_ERROR(("bt firmware path not found\n"));
+		return FALSE;
+	}
+
+	return TRUE;
+}
+#endif /* defined (BT_OVER_SDIO) */
+
+#if defined(BT_OVER_SDIO)
+wlan_bt_handle_t dhd_bt_get_pub_hndl(void)
+{
+	DHD_ERROR(("%s: g_dhd_pub %p\n", __FUNCTION__, g_dhd_pub));
+	/* assuming that dhd_pub_t type pointer is available from a global variable */
+	return (wlan_bt_handle_t) g_dhd_pub;
+} EXPORT_SYMBOL(dhd_bt_get_pub_hndl);
+
+int dhd_download_btfw(wlan_bt_handle_t handle, char* btfw_path)
+{
+	int ret = -1;
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+	dhd_info_t *dhd = (dhd_info_t*)dhdp->info;
+
+	/* Download BT firmware image to the dongle */
+	if (dhd->pub.busstate == DHD_BUS_DATA && dhd_update_btfw_path(dhd, btfw_path)) {
+		DHD_INFO(("%s: download btfw from: %s\n", __FUNCTION__, dhd->btfw_path));
+		ret = dhd_bus_download_btfw(dhd->pub.bus, dhd->pub.osh, dhd->btfw_path);
+		if (ret < 0) {
+			DHD_ERROR(("%s: failed to download btfw from: %s\n",
+				__FUNCTION__, dhd->btfw_path));
+			return ret;
+		}
+	}
+	return ret;
+} EXPORT_SYMBOL(dhd_download_btfw);
+#endif /* defined (BT_OVER_SDIO) */
+
+#ifndef BCMDBUS
+int
+dhd_bus_start(dhd_pub_t *dhdp)
+{
+	int ret = -1;
+	dhd_info_t *dhd = (dhd_info_t*)dhdp->info;
+	unsigned long flags;
+
+#if defined(DHD_DEBUG) && defined(BCMSDIO)
+	int fw_download_start = 0, fw_download_end = 0, f2_sync_start = 0, f2_sync_end = 0;
+#endif /* DHD_DEBUG && BCMSDIO */
+	ASSERT(dhd);
+
+	DHD_TRACE(("Enter %s:\n", __FUNCTION__));
+	dhdp->memdump_type = 0;
+	dhdp->dongle_trap_occured = 0;
+	dhd->pub.dsack_hc_due_to_isr_delay = 0;
+	dhd->pub.dsack_hc_due_to_dpc_delay = 0;
+	dhdp->dongle_trap_during_wifi_onoff = 0;
+#ifdef DHD_SSSR_DUMP
+	dhdp->collect_sssr = FALSE;
+#endif /* DHD_SSSR_DUMP */
+#ifdef DHD_SDTC_ETB_DUMP
+	dhdp->collect_sdtc = FALSE;
+#endif /* DHD_SDTC_ETB_DUMP */
+#if defined(BCMPCIE)
+	if (dhdp->extended_trap_data) {
+		memset(dhdp->extended_trap_data, 0, BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+	}
+#endif /* BCMPCIE */
+#ifdef DHD_SSSR_DUMP
+	/* Flag to indicate sssr dump is collected */
+	dhdp->sssr_dump_collected = 0;
+#endif /* DHD_SSSR_DUMP */
+	dhdp->iovar_timeout_occured = 0;
+#ifdef PCIE_FULL_DONGLE
+	dhdp->d3ack_timeout_occured = 0;
+	dhdp->livelock_occured = 0;
+	dhdp->pktid_audit_failed = 0;
+	dhdp->pktid_invalid_occured = 0;
+#endif /* PCIE_FULL_DONGLE */
+	dhd->pub.iface_op_failed = 0;
+	dhd->pub.scan_timeout_occurred = 0;
+	dhd->pub.scan_busy_occurred = 0;
+	dhd->pub.p2p_disc_busy_occurred = 0;
+	/* Retain BH induced errors and clear induced error during initialize */
+	if (dhd->pub.dhd_induce_error) {
+		dhd->pub.dhd_induce_bh_error = dhd->pub.dhd_induce_error;
+	}
+	dhd->pub.dhd_induce_error = DHD_INDUCE_ERROR_CLEAR;
+#ifdef DHD_MAP_PKTID_LOGGING
+	dhd->pub.enable_pktid_log_dump = FALSE;
+#endif /* DHD_MAP_PKTID_LOGGING */
+#ifdef DHD_SUPPORT_SPMI_MODE
+	dhd->pub.dhd_spmi_mode = spmi_mode;
+#endif /* DHD_SUPPORT_SPMI_MODE */
+	dhd->pub.tput_test_done = FALSE;
+
+	dhd->pub.p2p_disc_busy_cnt = 0;
+	/* try to download image and nvram to the dongle */
+	if  (dhd->pub.busstate == DHD_BUS_DOWN && dhd_update_fw_nv_path(dhd)) {
+		/* Indicate FW Download has not yet done */
+		dhd->pub.fw_download_status = FW_DOWNLOAD_IN_PROGRESS;
+		DHD_INFO(("%s download fw %s, nv %s, conf %s\n",
+			__FUNCTION__, dhd->fw_path, dhd->nv_path, dhd->conf_path));
+#if defined(DHD_DEBUG) && defined(BCMSDIO)
+		fw_download_start = OSL_SYSUPTIME();
+#endif /* DHD_DEBUG && BCMSDIO */
+		dhd_bus_set_signature_path(dhd->pub.bus, dhd->sig_path);
+		ret = dhd_bus_download_firmware(dhd->pub.bus, dhd->pub.osh,
+			dhd->fw_path, dhd->nv_path, dhd->clm_path, dhd->conf_path);
+#if defined(DHD_DEBUG) && defined(BCMSDIO)
+		fw_download_end = OSL_SYSUPTIME();
+#endif /* DHD_DEBUG && BCMSDIO */
+		if (ret < 0) {
+			DHD_ERROR(("%s: failed to download firmware %s\n",
+			          __FUNCTION__, dhd->fw_path));
+			return ret;
+		}
+		/* Indicate FW Download has succeeded */
+		dhd->pub.fw_download_status = FW_DOWNLOAD_DONE;
+	}
+	if (dhd->pub.busstate != DHD_BUS_LOAD) {
+		return -ENETDOWN;
+	}
+
+#ifdef BCMSDIO
+	dhd_os_sdlock(dhdp);
+#endif /* BCMSDIO */
+
+	/* Start the watchdog timer */
+	dhd->pub.tickcnt = 0;
+	dhd_os_wd_timer(&dhd->pub, dhd_watchdog_ms);
+
+	/* Bring up the bus */
+	if ((ret = dhd_bus_init(&dhd->pub, FALSE)) != 0) {
+
+		DHD_ERROR(("%s, dhd_bus_init failed %d\n", __FUNCTION__, ret));
+#ifdef BCMSDIO
+		dhd_os_sdunlock(dhdp);
+#endif /* BCMSDIO */
+		return ret;
+	}
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) || defined(BCMPCIE_OOB_HOST_WAKE)
+	/* Host registration for OOB interrupt */
+	if (dhd_bus_oob_intr_register(dhdp)) {
+		/* deactivate timer and wait for the handler to finish */
+#if !defined(BCMPCIE_OOB_HOST_WAKE)
+		DHD_GENERAL_LOCK(&dhd->pub, flags);
+		dhd->wd_timer_valid = FALSE;
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		del_timer_sync(&dhd->timer);
+
+#endif /* !BCMPCIE_OOB_HOST_WAKE */
+		DHD_STOP_RPM_TIMER(&dhd->pub);
+
+		DHD_ERROR(("%s Host failed to register for OOB\n", __FUNCTION__));
+		DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+		return -ENODEV;
+	}
+
+#if defined(BCMPCIE_OOB_HOST_WAKE)
+	dhd_bus_oob_intr_set(dhdp, TRUE);
+#else
+	/* Enable oob at firmware */
+	dhd_enable_oob_intr(dhd->pub.bus, TRUE);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#elif defined(FORCE_WOWLAN)
+	/* Enable oob at firmware */
+	dhd_enable_oob_intr(dhd->pub.bus, TRUE);
+#endif /* OOB_INTR_ONLY || BCMSPI_ANDROID || BCMPCIE_OOB_HOST_WAKE */
+#ifdef PCIE_FULL_DONGLE
+	{
+		/* max_h2d_rings includes H2D common rings */
+		uint32 max_h2d_rings = dhd_bus_max_h2d_queues(dhd->pub.bus);
+
+		DHD_INFO(("%s: Initializing %u h2drings\n", __FUNCTION__,
+			max_h2d_rings));
+		if ((ret = dhd_flow_rings_init(&dhd->pub, max_h2d_rings)) != BCME_OK) {
+#ifdef BCMSDIO
+			dhd_os_sdunlock(dhdp);
+#endif /* BCMSDIO */
+			return ret;
+		}
+	}
+#endif /* PCIE_FULL_DONGLE */
+
+	/* set default value for now. Will be updated again in dhd_preinit_ioctls()
+	 * after querying FW
+	 */
+	dhdp->event_log_max_sets = NUM_EVENT_LOG_SETS;
+	dhdp->event_log_max_sets_queried = FALSE;
+
+	dhdp->smmu_fault_occurred = 0;
+#ifdef DNGL_AXI_ERROR_LOGGING
+	dhdp->axi_error = FALSE;
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+	/* Do protocol initialization necessary for IOCTL/IOVAR */
+	ret = dhd_prot_init(&dhd->pub);
+	if (unlikely(ret) != BCME_OK) {
+		DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+		return ret;
+	}
+
+	/* If bus is not ready, can't come up */
+	if (dhd->pub.busstate != DHD_BUS_DATA) {
+		DHD_GENERAL_LOCK(&dhd->pub, flags);
+		dhd->wd_timer_valid = FALSE;
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		del_timer_sync(&dhd->timer);
+		DHD_ERROR(("%s failed bus is not ready\n", __FUNCTION__));
+		DHD_STOP_RPM_TIMER(&dhd->pub);
+#ifdef BCMSDIO
+		dhd_os_sdunlock(dhdp);
+#endif /* BCMSDIO */
+		DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+		return -ENODEV;
+	}
+
+#ifdef BCMSDIO
+	dhd_os_sdunlock(dhdp);
+#endif /* BCMSDIO */
+
+	/* Bus is ready, query any dongle information */
+	/* XXX Since dhd_sync_with_dongle can sleep, should module count surround it? */
+#if defined(DHD_DEBUG) && defined(BCMSDIO)
+	f2_sync_start = OSL_SYSUPTIME();
+#endif /* DHD_DEBUG && BCMSDIO */
+	if ((ret = dhd_sync_with_dongle(&dhd->pub)) < 0) {
+		DHD_GENERAL_LOCK(&dhd->pub, flags);
+		dhd->wd_timer_valid = FALSE;
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		del_timer_sync(&dhd->timer);
+		DHD_ERROR(("%s failed to sync with dongle\n", __FUNCTION__));
+		DHD_OS_WD_WAKE_UNLOCK(&dhd->pub);
+		return ret;
+	}
+
+#if defined(BCMPCIE)
+	/* XXX: JIRA SWWLAN-139454: Added L1ss enable
+	 * after firmware download completion due to link down issue
+	 * JIRA SWWLAN-142236: Amendment - Changed L1ss enable point
+	 */
+	DHD_RPM(("%s: Enable L1ss EP side\n", __FUNCTION__));
+	dhd_plat_l1ss_ctrl(1);
+#endif /* BCMPCIE */
+
+#if defined(DHD_DEBUG) && defined(BCMSDIO)
+	f2_sync_end = OSL_SYSUPTIME();
+	DHD_ERROR(("Time taken for FW download and F2 ready is: %d msec\n",
+			(fw_download_end - fw_download_start) + (f2_sync_end - f2_sync_start)));
+#endif /* DHD_DEBUG && BCMSDIO */
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	if (dhd->pend_ipaddr) {
+#ifdef AOE_IP_ALIAS_SUPPORT
+		/* XXX Assume pending ip address is belong to primary interface */
+		aoe_update_host_ipv4_table(&dhd->pub, dhd->pend_ipaddr, TRUE, 0);
+#endif /* AOE_IP_ALIAS_SUPPORT */
+		dhd->pend_ipaddr = 0;
+	}
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#if defined(BCM_ROUTER_DHD)
+	bzero(&dhd->pub.dhd_tm_dwm_tbl, sizeof(dhd_trf_mgmt_dwm_tbl_t));
+#endif /* BCM_ROUTER_DHD */
+	return 0;
+}
+#endif /* !BCMDBUS */
+
+#ifdef WLTDLS
+int _dhd_tdls_enable(dhd_pub_t *dhd, bool tdls_on, bool auto_on, struct ether_addr *mac)
+{
+	uint32 tdls = tdls_on;
+	int ret = 0;
+	uint32 tdls_auto_op = 0;
+	uint32 tdls_idle_time = CUSTOM_TDLS_IDLE_MODE_SETTING;
+	int32 tdls_rssi_high = CUSTOM_TDLS_RSSI_THRESHOLD_HIGH;
+	int32 tdls_rssi_low = CUSTOM_TDLS_RSSI_THRESHOLD_LOW;
+	uint32 tdls_pktcnt_high = CUSTOM_TDLS_PCKTCNT_THRESHOLD_HIGH;
+	uint32 tdls_pktcnt_low = CUSTOM_TDLS_PCKTCNT_THRESHOLD_LOW;
+
+	BCM_REFERENCE(mac);
+	if (!FW_SUPPORTED(dhd, tdls))
+		return BCME_ERROR;
+
+	if (dhd->tdls_enable == tdls_on)
+		goto auto_mode;
+	ret = dhd_iovar(dhd, 0, "tdls_enable", (char *)&tdls, sizeof(tdls), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: tdls %d failed %d\n", __FUNCTION__, tdls, ret));
+		goto exit;
+	}
+	dhd->tdls_enable = tdls_on;
+auto_mode:
+
+	tdls_auto_op = auto_on;
+	ret = dhd_iovar(dhd, 0, "tdls_auto_op", (char *)&tdls_auto_op, sizeof(tdls_auto_op), NULL,
+			0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: tdls_auto_op failed %d\n", __FUNCTION__, ret));
+		goto exit;
+	}
+
+	if (tdls_auto_op) {
+		ret = dhd_iovar(dhd, 0, "tdls_idle_time", (char *)&tdls_idle_time,
+				sizeof(tdls_idle_time), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: tdls_idle_time failed %d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+		ret = dhd_iovar(dhd, 0, "tdls_rssi_high", (char *)&tdls_rssi_high,
+				sizeof(tdls_rssi_high), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: tdls_rssi_high failed %d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+		ret = dhd_iovar(dhd, 0, "tdls_rssi_low", (char *)&tdls_rssi_low,
+				sizeof(tdls_rssi_low), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: tdls_rssi_low failed %d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+		ret = dhd_iovar(dhd, 0, "tdls_trigger_pktcnt_high", (char *)&tdls_pktcnt_high,
+				sizeof(tdls_pktcnt_high), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: tdls_trigger_pktcnt_high failed %d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+		ret = dhd_iovar(dhd, 0, "tdls_trigger_pktcnt_low", (char *)&tdls_pktcnt_low,
+				sizeof(tdls_pktcnt_low), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: tdls_trigger_pktcnt_low failed %d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+	}
+
+exit:
+	return ret;
+}
+
+int dhd_tdls_enable(struct net_device *dev, bool tdls_on, bool auto_on, struct ether_addr *mac)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+	if (dhd)
+		ret = _dhd_tdls_enable(&dhd->pub, tdls_on, auto_on, mac);
+	else
+		ret = BCME_ERROR;
+	return ret;
+}
+
+int
+dhd_tdls_set_mode(dhd_pub_t *dhd, bool wfd_mode)
+{
+	int ret = 0;
+	bool auto_on = false;
+	uint32 mode =  wfd_mode;
+
+#ifdef ENABLE_TDLS_AUTO_MODE
+	if (wfd_mode) {
+		auto_on = false;
+	} else {
+		auto_on = true;
+	}
+#else
+	auto_on = false;
+#endif /* ENABLE_TDLS_AUTO_MODE */
+	ret = _dhd_tdls_enable(dhd, false, auto_on, NULL);
+	if (ret < 0) {
+		DHD_ERROR(("Disable tdls_auto_op failed. %d\n", ret));
+		return ret;
+	}
+
+	ret = dhd_iovar(dhd, 0, "tdls_wfd_mode", (char *)&mode, sizeof(mode), NULL, 0, TRUE);
+	if ((ret < 0) && (ret != BCME_UNSUPPORTED)) {
+		DHD_ERROR(("%s: tdls_wfd_mode faile_wfd_mode %d\n", __FUNCTION__, ret));
+		return ret;
+	}
+
+	ret = _dhd_tdls_enable(dhd, true, auto_on, NULL);
+	if (ret < 0) {
+		DHD_ERROR(("enable tdls_auto_op failed. %d\n", ret));
+		return ret;
+	}
+
+	dhd->tdls_mode = mode;
+	return ret;
+}
+#ifdef PCIE_FULL_DONGLE
+int dhd_tdls_update_peer_info(dhd_pub_t *dhdp, wl_event_msg_t *event)
+{
+	dhd_pub_t *dhd_pub = dhdp;
+	tdls_peer_node_t *cur = dhd_pub->peer_tbl.node;
+	tdls_peer_node_t *new = NULL, *prev = NULL;
+	int ifindex = dhd_ifname2idx(dhd_pub->info, event->ifname);
+	uint8 *da = (uint8 *)&event->addr.octet[0];
+	bool connect = FALSE;
+	uint32 reason = ntoh32(event->reason);
+	unsigned long flags;
+
+	/* No handling needed for peer discovered reason */
+	if (reason == WLC_E_TDLS_PEER_DISCOVERED) {
+		return BCME_ERROR;
+	}
+	if (reason == WLC_E_TDLS_PEER_CONNECTED)
+		connect = TRUE;
+	else if (reason == WLC_E_TDLS_PEER_DISCONNECTED)
+		connect = FALSE;
+	else
+	{
+		DHD_ERROR(("%s: TDLS Event reason is unknown\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	if (ifindex == DHD_BAD_IF)
+		return BCME_ERROR;
+
+	if (connect) {
+		while (cur != NULL) {
+			if (!memcmp(da, cur->addr, ETHER_ADDR_LEN)) {
+				DHD_ERROR(("%s: TDLS Peer exist already %d\n",
+					__FUNCTION__, __LINE__));
+				return BCME_ERROR;
+			}
+			cur = cur->next;
+		}
+
+		new = MALLOC(dhd_pub->osh, sizeof(tdls_peer_node_t));
+		if (new == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+		memcpy(new->addr, da, ETHER_ADDR_LEN);
+		DHD_TDLS_LOCK(&dhdp->tdls_lock, flags);
+		new->next = dhd_pub->peer_tbl.node;
+		dhd_pub->peer_tbl.node = new;
+		dhd_pub->peer_tbl.tdls_peer_count++;
+		DHD_ERROR(("%s: Add TDLS peer, count=%d " MACDBG "\n",
+			__FUNCTION__, dhd_pub->peer_tbl.tdls_peer_count,
+			MAC2STRDBG((char *)da)));
+		DHD_TDLS_UNLOCK(&dhdp->tdls_lock, flags);
+
+	} else {
+		while (cur != NULL) {
+			if (!memcmp(da, cur->addr, ETHER_ADDR_LEN)) {
+				dhd_flow_rings_delete_for_peer(dhd_pub, (uint8)ifindex, da);
+				DHD_TDLS_LOCK(&dhdp->tdls_lock, flags);
+				if (prev)
+					prev->next = cur->next;
+				else
+					dhd_pub->peer_tbl.node = cur->next;
+				MFREE(dhd_pub->osh, cur, sizeof(tdls_peer_node_t));
+				dhd_pub->peer_tbl.tdls_peer_count--;
+				DHD_ERROR(("%s: Remove TDLS peer, count=%d " MACDBG "\n",
+					__FUNCTION__, dhd_pub->peer_tbl.tdls_peer_count,
+					MAC2STRDBG((char *)da)));
+				DHD_TDLS_UNLOCK(&dhdp->tdls_lock, flags);
+				return BCME_OK;
+			}
+			prev = cur;
+			cur = cur->next;
+		}
+		DHD_ERROR(("%s: TDLS Peer Entry Not found\n", __FUNCTION__));
+	}
+	return BCME_OK;
+}
+#endif /* PCIE_FULL_DONGLE */
+#endif /* BCMDBUS */
+
+bool dhd_is_concurrent_mode(dhd_pub_t *dhd)
+{
+	if (!dhd)
+		return FALSE;
+
+	if (dhd->op_mode & DHD_FLAG_CONCURR_MULTI_CHAN_MODE)
+		return TRUE;
+	else if ((dhd->op_mode & DHD_FLAG_CONCURR_SINGLE_CHAN_MODE) ==
+		DHD_FLAG_CONCURR_SINGLE_CHAN_MODE)
+		return TRUE;
+	else
+		return FALSE;
+}
+
+#if defined(WLADPS)
+#ifdef WL_BAM
+static int
+dhd_check_adps_bad_ap(dhd_pub_t *dhd)
+{
+	struct net_device *ndev;
+	struct bcm_cfg80211 *cfg;
+	struct wl_profile *profile;
+	struct ether_addr bssid;
+
+	if (!dhd_is_associated(dhd, 0, NULL)) {
+		DHD_ERROR(("%s - not associated\n", __FUNCTION__));
+		return BCME_OK;
+	}
+
+	ndev = dhd_linux_get_primary_netdev(dhd);
+	if (!ndev) {
+		DHD_ERROR(("%s: Cannot find primary netdev\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	cfg = wl_get_cfg(ndev);
+	if (!cfg) {
+		DHD_ERROR(("%s: Cannot find cfg\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	profile = wl_get_profile_by_netdev(cfg, ndev);
+	if (!profile) {
+		DHD_ERROR(("%s : Cannot get profile\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	memcpy(bssid.octet, profile->bssid, ETHER_ADDR_LEN);
+	if (wl_adps_bad_ap_check(cfg, &bssid)) {
+		if (wl_adps_enabled(cfg, ndev)) {
+			wl_adps_set_suspend(cfg, ndev, ADPS_SUSPEND);
+		}
+	}
+
+	return BCME_OK;
+}
+#endif	/* WL_BAM */
+
+int
+dhd_enable_adps(dhd_pub_t *dhd, uint8 on)
+{
+	int i;
+	int len;
+	int ret = BCME_OK;
+
+	bcm_iov_buf_t *iov_buf = NULL;
+	wl_adps_params_v1_t *data = NULL;
+
+	len = OFFSETOF(bcm_iov_buf_t, data) + sizeof(*data);
+	iov_buf = MALLOC(dhd->osh, len);
+	if (iov_buf == NULL) {
+		DHD_ERROR(("%s - failed to allocate %d bytes for iov_buf\n", __FUNCTION__, len));
+		ret = BCME_NOMEM;
+		goto exit;
+	}
+
+	iov_buf->version = WL_ADPS_IOV_VER;
+	iov_buf->len = sizeof(*data);
+	iov_buf->id = WL_ADPS_IOV_MODE;
+
+	data = (wl_adps_params_v1_t *)iov_buf->data;
+	data->version = ADPS_SUB_IOV_VERSION_1;
+	data->length = sizeof(*data);
+	data->mode = on;
+
+	for (i = 1; i <= MAX_BANDS; i++) {
+		data->band = i;
+		ret = dhd_iovar(dhd, 0, "adps", (char *)iov_buf, len, NULL, 0, TRUE);
+		if (ret < 0) {
+			if (ret == BCME_UNSUPPORTED) {
+				DHD_ERROR(("%s adps, UNSUPPORTED\n", __FUNCTION__));
+				ret = BCME_OK;
+				goto exit;
+			}
+			else {
+				DHD_ERROR(("%s fail to set adps %s for band %d (%d)\n",
+					__FUNCTION__, on ? "On" : "Off", i, ret));
+				goto exit;
+			}
+		}
+	}
+
+#ifdef WL_BAM
+	if (on) {
+		dhd_check_adps_bad_ap(dhd);
+	}
+#endif	/* WL_BAM */
+
+exit:
+	if (iov_buf) {
+		MFREE(dhd->osh, iov_buf, len);
+	}
+	return ret;
+}
+#endif /* WLADPS */
+
+int
+dhd_get_preserve_log_numbers(dhd_pub_t *dhd, uint64 *logset_mask)
+{
+	wl_el_set_type_t logset_type, logset_op;
+	wl_el_set_all_type_v1_t *logset_all_type_op = NULL;
+	bool use_logset_all_type = FALSE;
+	int ret = BCME_ERROR;
+	int err = 0;
+	uint8 i = 0;
+	int el_set_all_type_len;
+
+	if (!dhd || !logset_mask)
+		return BCME_BADARG;
+
+	el_set_all_type_len = OFFSETOF(wl_el_set_all_type_v1_t, set_type) +
+		(sizeof(wl_el_set_type_v1_t) * dhd->event_log_max_sets);
+
+	logset_all_type_op = (wl_el_set_all_type_v1_t *) MALLOC(dhd->osh, el_set_all_type_len);
+	if (logset_all_type_op == NULL) {
+		DHD_ERROR(("%s: failed to allocate %d bytes for logset_all_type_op\n",
+			__FUNCTION__, el_set_all_type_len));
+		return BCME_NOMEM;
+	}
+
+	*logset_mask = 0;
+	memset(&logset_type, 0, sizeof(logset_type));
+	memset(&logset_op, 0, sizeof(logset_op));
+#if defined(EVENT_LOG_SET_TYPE_VERSION_0)
+	logset_type.version = htod16(EVENT_LOG_SET_TYPE_VERSION_0);
+#else
+	logset_type.version = htod16(EVENT_LOG_SET_TYPE_CURRENT_VERSION);
+#endif
+	logset_type.len = htod16(sizeof(wl_el_set_type_t));
+
+	/* Try with set = event_log_max_sets, if fails, use legacy event_log_set_type */
+	logset_type.set = dhd->event_log_max_sets;
+	err = dhd_iovar(dhd, 0, "event_log_set_type", (char *)&logset_type, sizeof(logset_type),
+		(char *)logset_all_type_op, el_set_all_type_len, FALSE);
+	if (err == BCME_OK) {
+		DHD_ERROR(("%s: use optimised use_logset_all_type\n", __FUNCTION__));
+		use_logset_all_type = TRUE;
+	}
+
+	for (i = 0; i < dhd->event_log_max_sets; i++) {
+		if (use_logset_all_type) {
+			logset_op.type = logset_all_type_op->set_type[i].type_val;
+		} else {
+			logset_type.set = i;
+			err = dhd_iovar(dhd, 0, "event_log_set_type", (char *)&logset_type,
+				sizeof(logset_type), (char *)&logset_op, sizeof(logset_op), FALSE);
+		}
+		/* the iovar may return 'unsupported' error if a log set number is not present
+		* in the fw, so we should not return on error !
+		*/
+		if (err == BCME_OK &&
+				logset_op.type == EVENT_LOG_SET_TYPE_PRSRV) {
+			*logset_mask |= 0x01ULL << i;
+			ret = BCME_OK;
+			DHD_INFO(("[INIT] logset:%d is preserve/chatty\n", i));
+		}
+	}
+
+	MFREE(dhd->osh, logset_all_type_op, el_set_all_type_len);
+	return ret;
+}
+
+#ifndef OEM_ANDROID
+/* For non-android FC modular builds, override firmware preinited values */
+void
+dhd_override_fwprenit(dhd_pub_t * dhd)
+{
+	int ret = 0;
+
+	{
+		/* Disable bcn_li_bcn */
+		uint32 bcn_li_bcn = 0;
+		ret = dhd_iovar(dhd, 0, "bcn_li_bcn", (char *)&bcn_li_bcn,
+				sizeof(bcn_li_bcn), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: bcn_li_bcn failed:%d\n",
+				__FUNCTION__, ret));
+		}
+	}
+
+	{
+		/* Disable apsta */
+		uint32 apsta = 0;
+		ret = dhd_iovar(dhd, 0, "apsta", (char *)&apsta,
+				sizeof(apsta), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: apsta failed:%d\n",
+				__FUNCTION__, ret));
+		}
+	}
+
+	{
+		int ap_mode = 0;
+		if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_AP, (char *)&ap_mode,
+			sizeof(ap_mode), TRUE, 0)) < 0) {
+			DHD_ERROR(("%s: set apmode failed :%d\n", __FUNCTION__, ret));
+		}
+	}
+}
+#endif /* !OEM_ANDROID */
+
+int
+dhd_get_fw_capabilities(dhd_pub_t * dhd)
+{
+
+	int ret = 0;
+	uint32 cap_buf_size = sizeof(dhd->fw_capabilities);
+	memset(dhd->fw_capabilities, 0, cap_buf_size);
+	ret = dhd_iovar(dhd, 0, "cap", NULL, 0, dhd->fw_capabilities, (cap_buf_size - 1),
+		FALSE);
+
+	if (ret < 0) {
+		DHD_ERROR(("%s: Get Capability failed (error=%d)\n",
+		__FUNCTION__, ret));
+		return ret;
+	}
+
+	memmove(&dhd->fw_capabilities[1], dhd->fw_capabilities, (cap_buf_size - 1));
+	dhd->fw_capabilities[0] = ' ';
+	dhd->fw_capabilities[cap_buf_size - 2] = ' ';
+	dhd->fw_capabilities[cap_buf_size - 1] = '\0';
+
+	return 0;
+}
+
+int
+dhd_optimised_preinit_ioctls(dhd_pub_t * dhd)
+{
+	int ret = 0;
+	/*  Room for "event_msgs_ext" + '\0' + bitvec  */
+	char iovbuf[WL_EVENTING_MASK_EXT_LEN + EVENTMSGS_EXT_STRUCT_SIZE + 16];
+	uint32 event_log_max_sets = 0;
+	char* iov_buf = NULL;
+	/* XXX: Use ret2 for return check of IOVARS that might return BCME_UNSUPPORTED,
+	*	based on FW build tag.
+	*/
+	int ret2 = 0;
+#if defined(WL_MONITOR) && defined(HOST_RADIOTAP_CONV)
+	uint monitor = 0;
+	dhd_info_t *dhdinfo = (dhd_info_t*)dhd->info;
+#endif /* WL_MONITOR */
+#if defined(BCMSUP_4WAY_HANDSHAKE)
+	uint32 sup_wpa = 1;
+#endif /* BCMSUP_4WAY_HANDSHAKE */
+
+	uint32 frameburst = CUSTOM_FRAMEBURST_SET;
+	uint wnm_bsstrans_resp = 0;
+#ifdef DHD_BUS_MEM_ACCESS
+	uint32 enable_memuse = 1;
+#endif /* DHD_BUS_MEM_ACCESS */
+#ifdef DHD_PM_CONTROL_FROM_FILE
+	uint power_mode = PM_FAST;
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+	char buf[WLC_IOCTL_SMLEN];
+	char *ptr;
+#ifdef ROAM_ENABLE
+	uint roamvar = 0;
+#ifdef ROAM_AP_ENV_DETECTION
+	int roam_env_mode = 0;
+#endif /* ROAM_AP_ENV_DETECTION */
+#endif /* ROAM_ENABLE */
+#if defined(SOFTAP)
+	uint dtim = 1;
+#endif
+/* xxx andrey tmp fix for dk8000 build error  */
+	struct ether_addr p2p_ea;
+#ifdef GET_CUSTOM_MAC_ENABLE
+	struct ether_addr ea_addr;
+#endif /* GET_CUSTOM_MAC_ENABLE */
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	uint32 hostwake_oob = 0;
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+	wl_wlc_version_t wlc_ver;
+#ifdef CUSTOM_OCL_RSSI_VAL
+	int ocl_rssi_threshold = CUSTOM_OCL_RSSI_VAL;
+#endif /* CUSTOM_OCL_RSSI_VAL */
+
+#if defined(WBTEXT) && defined(RRM_BCNREQ_MAX_CHAN_TIME)
+	uint32 rrm_bcn_req_thrtl_win = RRM_BCNREQ_MAX_CHAN_TIME * 2;
+	uint32 rrm_bcn_req_max_off_chan_time = RRM_BCNREQ_MAX_CHAN_TIME;
+#endif /* WBTEXT && RRM_BCNREQ_MAX_CHAN_TIME */
+
+	uint32 d3_hostwake_delay = D3_HOSTWAKE_DELAY;
+#ifdef SUPPORT_MULTIPLE_CLMBLOB
+	char customer_clm_file_name[MAX_FILE_LEN] = {0, };
+#endif /* SUPPORT_MULTIPLE_CLMBLOB */
+	char* apply_clm;
+
+#ifdef PKT_FILTER_SUPPORT
+	dhd_pkt_filter_enable = TRUE;
+#endif /* PKT_FILTER_SUPPORT */
+#ifdef APF
+	dhd->apf_set = FALSE;
+#endif /* APF */
+#ifdef CUSTOM_SET_OCLOFF
+	dhd->ocl_off = FALSE;
+#endif /* CUSTOM_SET_OCLOFF */
+#ifdef SUPPORT_SET_TID
+	dhd->tid_mode = SET_TID_OFF;
+	dhd->target_uid = 0;
+	dhd->target_tid = 0;
+#endif /* SUPPORT_SET_TID */
+	DHD_TRACE(("Enter %s\n", __FUNCTION__));
+	dhd->op_mode = 0;
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	/* arpoe will be applied from the supsend context */
+	dhd->arpoe_enable = TRUE;
+	dhd->arpol_configured = FALSE;
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+	/* clear AP flags */
+#if defined(CUSTOM_COUNTRY_CODE)
+	dhd->dhd_cflags &= ~WLAN_PLAT_AP_FLAG;
+#endif /* CUSTOM_COUNTRY_CODE */
+
+	/* query for 'ver' to get version info from firmware */
+	memset(buf, 0, sizeof(buf));
+	ptr = buf;
+	ret = dhd_iovar(dhd, 0, "ver", NULL, 0, (char *)&buf, sizeof(buf), FALSE);
+	if (ret < 0)
+		DHD_ERROR(("%s failed %d\n", __FUNCTION__, ret));
+	else {
+		bcmstrtok(&ptr, "\n", 0);
+		/* Print fw version info */
+		strncpy(fw_version, buf, FW_VER_STR_LEN);
+		fw_version[FW_VER_STR_LEN-1] = '\0';
+	}
+
+	/* query for 'wlc_ver' to get version info from firmware */
+	/* memsetting to zero */
+	memset_s(&wlc_ver, sizeof(wl_wlc_version_t), 0,
+		sizeof(wl_wlc_version_t));
+	ret = dhd_iovar(dhd, 0, "wlc_ver", NULL, 0, (char *)&wlc_ver,
+			sizeof(wl_wlc_version_t), FALSE);
+	if (ret < 0)
+		DHD_ERROR(("%s failed %d\n", __FUNCTION__, ret));
+	else {
+		dhd->wlc_ver_major = wlc_ver.wlc_ver_major;
+		dhd->wlc_ver_minor = wlc_ver.wlc_ver_minor;
+	}
+// mark flag for later blob checking
+	/* Set op_mode as MFG_MODE if WLTEST is present in "wl ver" */
+	if (strstr(fw_version, "WLTEST") != NULL) {
+		DHD_ERROR(("%s: wl ver has WLTEST, setting op_mode as DHD_FLAG_MFG_MODE\n",
+			__FUNCTION__));
+		op_mode = DHD_FLAG_MFG_MODE;
+	}
+	/* get a capabilities from firmware */
+	ret = dhd_get_fw_capabilities(dhd);
+
+	if (ret < 0) {
+		DHD_ERROR(("%s: Get Capability failed (error=%d)\n",
+			__FUNCTION__, ret));
+		goto done;
+	}
+
+	if ((!op_mode && dhd_get_fw_mode(dhd->info) == DHD_FLAG_MFG_MODE) ||
+		(op_mode == DHD_FLAG_MFG_MODE)) {
+		dhd->op_mode = DHD_FLAG_MFG_MODE;
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+		/* disable runtimePM by default in MFG mode. */
+		pm_runtime_disable(dhd_bus_to_dev(dhd->bus));
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+		/* Check and adjust IOCTL response timeout for Manufactring firmware */
+		dhd_os_set_ioctl_resp_timeout(MFG_IOCTL_RESP_TIMEOUT);
+		DHD_ERROR(("%s : Set IOCTL response time for Manufactring Firmware\n",
+			__FUNCTION__));
+
+#if defined(ARP_OFFLOAD_SUPPORT)
+		dhd->arpoe_enable = FALSE;
+#endif /* ARP_OFFLOAD_SUPPORT */
+#ifdef PKT_FILTER_SUPPORT
+		dhd_pkt_filter_enable = FALSE;
+#endif /* PKT_FILTER_SUPPORT */
+#ifndef CUSTOM_SET_ANTNPM
+		if (FW_SUPPORTED(dhd, rsdb)) {
+			wl_config_t rsdb_mode;
+			memset(&rsdb_mode, 0, sizeof(rsdb_mode));
+			ret = dhd_iovar(dhd, 0, "rsdb_mode", (char *)&rsdb_mode, sizeof(rsdb_mode),
+				NULL, 0, TRUE);
+			if (ret < 0) {
+				DHD_ERROR(("%s Disable rsdb_mode is failed ret= %d\n",
+					__FUNCTION__, ret));
+			}
+		}
+#endif /* !CUSTOM_SET_ANTNPM */
+	} else {
+		uint32 concurrent_mode = 0;
+		dhd_os_set_ioctl_resp_timeout(IOCTL_RESP_TIMEOUT);
+		DHD_INFO(("%s : Set IOCTL response time.\n", __FUNCTION__));
+
+		BCM_REFERENCE(concurrent_mode);
+
+		dhd->op_mode = DHD_FLAG_STA_MODE;
+
+		BCM_REFERENCE(p2p_ea);
+#if defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P)
+		if ((concurrent_mode = dhd_get_concurrent_capabilites(dhd))) {
+			dhd->op_mode |= concurrent_mode;
+		}
+
+		/* Check if we are enabling p2p */
+		if (dhd->op_mode & DHD_FLAG_P2P_MODE) {
+			memcpy(&p2p_ea, &dhd->mac, ETHER_ADDR_LEN);
+			ETHER_SET_LOCALADDR(&p2p_ea);
+			ret = dhd_iovar(dhd, 0, "p2p_da_override", (char *)&p2p_ea, sizeof(p2p_ea),
+					NULL, 0, TRUE);
+			if (ret < 0)
+				DHD_ERROR(("%s p2p_da_override ret= %d\n", __FUNCTION__, ret));
+			else
+				DHD_INFO(("dhd_preinit_ioctls: p2p_da_override succeeded\n"));
+		}
+#endif /* defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P) */
+
+	}
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	ret = dhd_iovar(dhd, 0, "bus:hostwake_oob", NULL, 0, (char *)&hostwake_oob,
+		sizeof(hostwake_oob), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: hostwake_oob IOVAR not present, proceed\n", __FUNCTION__));
+	} else {
+		if (hostwake_oob == 0) {
+			DHD_ERROR(("%s: hostwake_oob is not enabled in the NVRAM, STOP\n",
+				__FUNCTION__));
+			ret = BCME_UNSUPPORTED;
+			goto done;
+		} else {
+			DHD_ERROR(("%s: hostwake_oob enabled\n", __FUNCTION__));
+		}
+	}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+	ret = dhd_iovar(dhd, 0, "axierror_logbuf_addr", NULL, 0, (char *)&dhd->axierror_logbuf_addr,
+		sizeof(dhd->axierror_logbuf_addr), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: axierror_logbuf_addr IOVAR not present, proceed\n", __FUNCTION__));
+		dhd->axierror_logbuf_addr = 0;
+	} else {
+		DHD_ERROR(("%s: axierror_logbuf_addr : 0x%x\n",
+			__FUNCTION__, dhd->axierror_logbuf_addr));
+	}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#ifdef GET_CUSTOM_MAC_ENABLE
+	ret = wifi_platform_get_mac_addr(dhd->info->adapter, ea_addr.octet, 0);
+	if (!ret) {
+		ret = dhd_iovar(dhd, 0, "cur_etheraddr", (char *)&ea_addr, ETHER_ADDR_LEN, NULL, 0,
+				TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: can't set MAC address , error=%d\n", __FUNCTION__, ret));
+			ret = BCME_NOTUP;
+			goto done;
+		}
+		memcpy(dhd->mac.octet, ea_addr.octet, ETHER_ADDR_LEN);
+	} else
+#endif /* GET_CUSTOM_MAC_ENABLE */
+	{
+		/* Get the default device MAC address directly from firmware */
+		ret = dhd_iovar(dhd, 0, "cur_etheraddr", NULL, 0, (char *)&buf, sizeof(buf), FALSE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: can't get MAC address , error=%d\n", __FUNCTION__, ret));
+			ret = BCME_NOTUP;
+			goto done;
+		}
+
+		DHD_ERROR(("%s: use firmware generated mac_address "MACDBG"\n",
+			__FUNCTION__, MAC2STRDBG(&buf)));
+
+#ifdef MACADDR_PROVISION_ENFORCED
+		if (ETHER_IS_LOCALADDR(buf)) {
+			DHD_ERROR(("%s: error! not using provision mac addr!\n", __FUNCTION__));
+			ret = BCME_BADADDR;
+			goto done;
+		}
+#endif /* MACADDR_PROVISION_ENFORCED */
+
+		/* Update public MAC address after reading from Firmware */
+		memcpy(dhd->mac.octet, buf, ETHER_ADDR_LEN);
+	}
+
+	if (ETHER_ISNULLADDR(dhd->mac.octet)) {
+		DHD_ERROR(("%s: NULL MAC address during pre-init\n", __FUNCTION__));
+		ret = BCME_BADADDR;
+		goto done;
+	} else {
+		(void)memcpy_s(dhd_linux_get_primary_netdev(dhd)->perm_addr, ETHER_ADDR_LEN,
+			dhd->mac.octet, ETHER_ADDR_LEN);
+	}
+
+#ifdef SUPPORT_MULTIPLE_CLMBLOB
+	if (dhd_get_platform_naming_for_nvram_clmblob_file(CLM_BLOB,
+			customer_clm_file_name) == BCME_OK) {
+		apply_clm = customer_clm_file_name;
+	}
+	else
+#endif /* SUPPORT_MULTIPLE_CLMBLOB */
+	{
+		apply_clm = dhd->info->clm_path;
+	}
+
+	if ((ret = dhd_apply_default_clm(dhd, apply_clm)) < 0) {
+		DHD_ERROR(("%s: CLM set failed. Ignore and continue initialization.\n",
+			__FUNCTION__));
+	}
+
+	DHD_ERROR(("Firmware up: op_mode=0x%04x, MAC="MACDBG"\n",
+		dhd->op_mode, MAC2STRDBG(dhd->mac.octet)));
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+	if (!dhd->is_blob)
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+	{
+		/* get a ccode and revision for the country code */
+#if defined(CUSTOM_COUNTRY_CODE)
+		get_customized_country_code(dhd->info->adapter, dhd->dhd_cspec.country_abbrev,
+			&dhd->dhd_cspec, dhd->dhd_cflags);
+#else
+		get_customized_country_code(dhd->info->adapter, dhd->dhd_cspec.country_abbrev,
+			&dhd->dhd_cspec);
+#endif /* CUSTOM_COUNTRY_CODE */
+	}
+
+#if defined(RXFRAME_THREAD) && defined(RXTHREAD_ONLYSTA)
+	if (dhd->op_mode == DHD_FLAG_HOSTAP_MODE)
+		dhd->info->rxthread_enabled = FALSE;
+	else
+		dhd->info->rxthread_enabled = TRUE;
+#endif
+
+#if defined(CUSTOM_COUNTRY_CODE_XZ)
+	/* Set initial country code to XZ */
+	strlcpy(dhd->dhd_cspec.country_abbrev, "XZ", WLC_CNTRY_BUF_SZ);
+	strlcpy(dhd->dhd_cspec.ccode, "XZ", WLC_CNTRY_BUF_SZ);
+	DHD_ERROR(("%s: Set initial country code to XZ(World Wide Safe)\n", __FUNCTION__));
+#endif /* CUSTOM_COUNTRY_CODE_XZ */
+	/* Set Country code  */
+	if (dhd->dhd_cspec.ccode[0] != 0) {
+		ret = dhd_iovar(dhd, 0, "country", (char *)&dhd->dhd_cspec, sizeof(wl_country_t),
+				NULL, 0, TRUE);
+		if (ret < 0)
+			DHD_ERROR(("%s: country code setting failed\n", __FUNCTION__));
+	}
+
+#if defined(ROAM_ENABLE)
+#ifdef USE_WFA_CERT_CONF
+	if (sec_get_param_wfa_cert(dhd, SET_PARAM_ROAMOFF, &roamvar) == BCME_OK) {
+		DHD_ERROR(("%s: read roam_off param =%d\n", __FUNCTION__, roamvar));
+	}
+#endif /* USE_WFA_CERT_CONF */
+	/* Disable built-in roaming to allowed ext supplicant to take care of roaming */
+	ret = dhd_iovar(dhd, 0, "roam_off", (char *)&roamvar, sizeof(roamvar), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s roam_off failed %d\n", __FUNCTION__, ret));
+	}
+#ifdef WL_CFG80211
+	ROAMOFF_DBG_SAVE(dhd_linux_get_primary_netdev(dhd), SET_ROAM_PREINIT, roamvar);
+#endif /* WL_CFG80211 */
+#ifdef ROAM_AP_ENV_DETECTION
+	/* Changed to GET iovar to read roam_env_mode */
+	dhd->roam_env_detection = FALSE;
+	ret = dhd_iovar(dhd, 0, "roam_env_detection", NULL, 0, (char *)&roam_env_mode,
+			sizeof(roam_env_mode), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: roam_env_detection IOVAR not present\n", __FUNCTION__));
+	} else {
+		if (roam_env_mode == AP_ENV_INDETERMINATE) {
+			dhd->roam_env_detection = TRUE;
+		}
+	}
+#endif /* ROAM_AP_ENV_DETECTION */
+#ifdef CONFIG_ROAM_RSSI_LIMIT
+	ret = dhd_roam_rssi_limit_set(dhd, CUSTOM_ROAMRSSI_2G, CUSTOM_ROAMRSSI_5G);
+	if (ret < 0) {
+		DHD_ERROR(("%s set roam_rssi_limit failed ret %d\n", __FUNCTION__, ret));
+	}
+#endif /* CONFIG_ROAM_RSSI_LIMIT */
+#ifdef CONFIG_ROAM_MIN_DELTA
+	ret = dhd_roam_min_delta_set(dhd, CUSTOM_ROAM_MIN_DELTA, CUSTOM_ROAM_MIN_DELTA);
+	if (ret < 0) {
+		DHD_ERROR(("%s set roam_min_delta failed ret %d\n", __FUNCTION__, ret));
+	}
+#endif /* CONFIG_ROAM_MIN_DELTA */
+#endif /* ROAM_ENABLE */
+#ifdef CUSTOM_EVENT_PM_WAKE
+	dhd_init_excess_pm_awake(dhd);
+#endif	/* CUSTOM_EVENT_PM_WAKE */
+
+#ifdef WLTDLS
+	dhd->tdls_enable = FALSE;
+	/* query tdls_eable */
+	ret = dhd_iovar(dhd, 0, "tdls_enable", NULL, 0, (char *)&dhd->tdls_enable,
+		sizeof(dhd->tdls_enable), FALSE);
+	DHD_ERROR(("%s: tdls_enable=%d ret=%d\n", __FUNCTION__, dhd->tdls_enable, ret));
+#endif /* WLTDLS */
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+	sec_control_pm(dhd, &power_mode);
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+
+#ifdef MIMO_ANT_SETTING
+	dhd_sel_ant_from_file(dhd);
+#endif /* MIMO_ANT_SETTING */
+
+#if defined(OEM_ANDROID) && defined(SOFTAP)
+	if (ap_fw_loaded == TRUE) {
+		dhd_wl_ioctl_cmd(dhd, WLC_SET_DTIMPRD, (char *)&dtim, sizeof(dtim), TRUE, 0);
+	}
+#endif /* defined(OEM_ANDROID) && defined(SOFTAP) */
+
+#if defined(KEEP_ALIVE)
+	/* Set Keep Alive : be sure to use FW with -keepalive */
+	if (!(dhd->op_mode &
+		(DHD_FLAG_HOSTAP_MODE | DHD_FLAG_MFG_MODE))) {
+		if ((ret = dhd_keep_alive_onoff(dhd)) < 0)
+			DHD_ERROR(("%s set keeplive failed %d\n",
+			__FUNCTION__, ret));
+	}
+#endif /* defined(KEEP_ALIVE) */
+
+	ret = dhd_iovar(dhd, 0, "event_log_max_sets", NULL, 0, (char *)&event_log_max_sets,
+		sizeof(event_log_max_sets), FALSE);
+	if (ret == BCME_OK) {
+		dhd->event_log_max_sets = event_log_max_sets;
+	} else {
+		dhd->event_log_max_sets = NUM_EVENT_LOG_SETS;
+	}
+	BCM_REFERENCE(iovbuf);
+	/* Make sure max_sets is set first with wmb and then sets_queried,
+	 * this will be used during parsing the logsets in the reverse order.
+	 */
+	OSL_SMP_WMB();
+	dhd->event_log_max_sets_queried = TRUE;
+	DHD_ERROR(("%s: event_log_max_sets: %d ret: %d\n",
+		__FUNCTION__, dhd->event_log_max_sets, ret));
+#ifdef DHD_BUS_MEM_ACCESS
+	ret = dhd_iovar(dhd, 0, "enable_memuse", (char *)&enable_memuse,
+			sizeof(enable_memuse), iovbuf, sizeof(iovbuf), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: enable_memuse is failed ret=%d\n",
+			__FUNCTION__, ret));
+	} else {
+		DHD_ERROR(("%s: enable_memuse = %d\n",
+			__FUNCTION__, enable_memuse));
+	}
+#endif /* DHD_BUS_MEM_ACCESS */
+
+#ifdef USE_WFA_CERT_CONF
+#ifdef USE_WL_FRAMEBURST
+	 if (sec_get_param_wfa_cert(dhd, SET_PARAM_FRAMEBURST, &frameburst) == BCME_OK) {
+		DHD_ERROR(("%s, read frameburst param=%d\n", __FUNCTION__, frameburst));
+	 }
+#endif /* USE_WL_FRAMEBURST */
+	 g_frameburst = frameburst;
+#endif /* USE_WFA_CERT_CONF */
+
+#ifdef DISABLE_WL_FRAMEBURST_SOFTAP
+	/* Disable Framebursting for SofAP */
+	if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+		frameburst = 0;
+	}
+#endif /* DISABLE_WL_FRAMEBURST_SOFTAP */
+
+	BCM_REFERENCE(frameburst);
+#if defined(USE_WL_FRAMEBURST) || defined(DISABLE_WL_FRAMEBURST_SOFTAP)
+	/* frameburst is set to 1 by preinit fw, change if otherwise */
+	if (frameburst != 1) {
+		/* Set frameburst to value */
+		if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_FAKEFRAG, (char *)&frameburst,
+			sizeof(frameburst), TRUE, 0)) < 0) {
+			DHD_INFO(("%s frameburst not supported	%d\n", __FUNCTION__, ret));
+		}
+	}
+#endif /* USE_WL_FRAMEBURST || DISABLE_WL_FRAMEBURST_SOFTAP */
+
+	iov_buf = (char*)MALLOC(dhd->osh, WLC_IOCTL_SMLEN);
+	if (iov_buf == NULL) {
+		DHD_ERROR(("failed to allocate %d bytes for iov_buf\n", WLC_IOCTL_SMLEN));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+
+#if defined(BCMSUP_4WAY_HANDSHAKE)
+	/* Read 4-way handshake requirements */
+	if (dhd_use_idsup == 1) {
+		ret = dhd_iovar(dhd, 0, "sup_wpa", (char *)&sup_wpa, sizeof(sup_wpa),
+				(char *)&iovbuf, sizeof(iovbuf), FALSE);
+		/* sup_wpa iovar returns NOTREADY status on some platforms using modularized
+		 * in-dongle supplicant.
+		 */
+		if (ret >= 0 || ret == BCME_NOTREADY)
+			dhd->fw_4way_handshake = TRUE;
+		DHD_TRACE(("4-way handshake mode is: %d\n", dhd->fw_4way_handshake));
+	}
+#endif /* BCMSUP_4WAY_HANDSHAKE */
+
+#if defined(PCIE_FULL_DONGLE) && defined(DHD_LOSSLESS_ROAMING)
+	dhd_update_flow_prio_map(dhd, DHD_FLOW_PRIO_LLR_MAP);
+#endif /* defined(PCIE_FULL_DONGLE) && defined(DHD_LOSSLESS_ROAMING) */
+
+#if defined(BCMPCIE) && defined(EAPOL_PKT_PRIO)
+	dhd_update_flow_prio_map(dhd, DHD_FLOW_PRIO_LLR_MAP);
+#endif /* defined(BCMPCIE) && defined(EAPOL_PKT_PRIO) */
+
+#if defined(BCMSDIO) && defined(DHD_LOSSLESS_ROAMING)
+	dhd_update_sdio_data_prio_map(dhd);
+#endif /* BCMSDIO && DHD_LOSSLESS_ROAMING */
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	DHD_ERROR(("arp_enable:%d arp_ol:%d\n",
+		dhd->arpoe_enable, dhd->arpol_configured));
+#endif /* ARP_OFFLOAD_SUPPORT */
+	/*
+	 * Retaining pktfilter fotr temporary, once fw preinit includes this,
+	 * this will be removed. Caution is to skip the pktfilter check during
+	 * each pktfilter removal.
+	 */
+#ifdef PKT_FILTER_SUPPORT
+	/* Setup default defintions for pktfilter , enable in suspend */
+	if (dhd_master_mode) {
+		dhd->pktfilter_count = 6;
+		dhd->pktfilter[DHD_BROADCAST_FILTER_NUM] = NULL;
+		if (!FW_SUPPORTED(dhd, pf6)) {
+			dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = NULL;
+			dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = NULL;
+		} else {
+			/* Immediately pkt filter TYPE 6 Discard IPv4/IPv6 Multicast Packet */
+			dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = DISCARD_IPV4_MCAST;
+			dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = DISCARD_IPV6_MCAST;
+		}
+		/* apply APP pktfilter */
+		dhd->pktfilter[DHD_ARP_FILTER_NUM] = "105 0 0 12 0xFFFF 0x0806";
+
+#ifdef BLOCK_IPV6_PACKET
+		/* Setup filter to allow only IPv4 unicast frames */
+		dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = "100 0 0 0 "
+			HEX_PREF_STR UNI_FILTER_STR ZERO_ADDR_STR ETHER_TYPE_STR IPV6_FILTER_STR
+			" "
+			HEX_PREF_STR ZERO_ADDR_STR ZERO_ADDR_STR ETHER_TYPE_STR ZERO_TYPE_STR;
+#else
+		/* Setup filter to allow only unicast */
+		dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = "100 0 0 0 0x01 0x00";
+#endif /* BLOCK_IPV6_PACKET */
+
+#ifdef PASS_IPV4_SUSPEND
+		/* XXX customer want to get IPv4 multicast packets */
+		dhd->pktfilter[DHD_MDNS_FILTER_NUM] = "104 0 0 0 0xFFFFFF 0x01005E";
+#else
+		/* Add filter to pass multicastDNS packet and NOT filter out as Broadcast */
+		dhd->pktfilter[DHD_MDNS_FILTER_NUM] = NULL;
+#endif /* PASS_IPV4_SUSPEND */
+		if (FW_SUPPORTED(dhd, pf6)) {
+			/* Immediately pkt filter TYPE 6 Dicard Broadcast IP packet */
+			dhd->pktfilter[DHD_IP4BCAST_DROP_FILTER_NUM] = DISCARD_IPV4_BCAST;
+			/* Immediately pkt filter TYPE 6 Dicard Cisco STP packet */
+			dhd->pktfilter[DHD_LLC_STP_DROP_FILTER_NUM] = DISCARD_LLC_STP;
+			/* Immediately pkt filter TYPE 6 Dicard Cisco XID protocol */
+			dhd->pktfilter[DHD_LLC_XID_DROP_FILTER_NUM] = DISCARD_LLC_XID;
+			/* Immediately pkt filter TYPE 6 Dicard NETBIOS packet(port 137) */
+			dhd->pktfilter[DHD_UDPNETBIOS_DROP_FILTER_NUM] = DISCARD_UDPNETBIOS;
+			dhd->pktfilter_count = 11;
+		}
+
+#ifdef GAN_LITE_NAT_KEEPALIVE_FILTER
+		dhd->pktfilter_count = 4;
+		/* Setup filter to block broadcast and NAT Keepalive packets */
+		/* discard all broadcast packets */
+		dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = "100 0 0 0 0xffffff 0xffffff";
+		/* discard NAT Keepalive packets */
+		dhd->pktfilter[DHD_BROADCAST_FILTER_NUM] = "102 0 0 36 0xffffffff 0x11940009";
+		/* discard NAT Keepalive packets */
+		dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = "104 0 0 38 0xffffffff 0x11940009";
+		dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = NULL;
+#endif /* GAN_LITE_NAT_KEEPALIVE_FILTER */
+	} else
+		dhd_conf_discard_pkt_filter(dhd);
+	dhd_conf_add_pkt_filter(dhd);
+
+#if defined(SOFTAP)
+	if (ap_fw_loaded) {
+		/* XXX Andrey: fo SOFTAP disable pkt filters (if there were any )  */
+		dhd_enable_packet_filter(0, dhd);
+	}
+#endif /* defined(SOFTAP) */
+	dhd_set_packet_filter(dhd);
+#endif /* PKT_FILTER_SUPPORT */
+
+	/* query for 'clmver' to get clm version info from firmware */
+	bzero(buf, sizeof(buf));
+	ret = dhd_iovar(dhd, 0, "clmver", NULL, 0, buf, sizeof(buf), FALSE);
+	if (ret < 0)
+		DHD_ERROR(("%s clmver failed %d\n", __FUNCTION__, ret));
+	else {
+		char *ver_temp_buf = NULL, *ver_date_buf = NULL;
+		int len;
+
+		if ((ver_temp_buf = bcmstrstr(buf, "Data:")) == NULL) {
+			DHD_ERROR(("Couldn't find \"Data:\"\n"));
+		} else {
+			ver_date_buf = bcmstrstr(buf, "Creation:");
+			ptr = (ver_temp_buf + strlen("Data:"));
+			if ((ver_temp_buf = bcmstrtok(&ptr, "\n", 0)) == NULL) {
+				DHD_ERROR(("Couldn't find New line character\n"));
+			} else {
+				memset(clm_version, 0, CLM_VER_STR_LEN);
+				len = snprintf(clm_version, CLM_VER_STR_LEN - 1, "%s", ver_temp_buf);
+				if (ver_date_buf) {
+					ptr = (ver_date_buf + strlen("Creation:"));
+					ver_date_buf = bcmstrtok(&ptr, "\n", 0);
+					if (ver_date_buf)
+						snprintf(clm_version+len, CLM_VER_STR_LEN-1-len,
+							" (%s)", ver_date_buf);
+				}
+				DHD_INFO(("CLM version = %s\n", clm_version));
+			}
+		}
+
+		if (strlen(clm_version)) {
+			DHD_INFO(("CLM version = %s\n", clm_version));
+		} else {
+			DHD_ERROR(("Couldn't find CLM version!\n"));
+		}
+
+	}
+	dhd_set_version_info(dhd, fw_version);
+
+#ifdef WRITE_WLANINFO
+	do {
+		uint len = MAX_NVRAMBUF_SIZE;
+		char *memblock = NULL;
+
+		dhd_get_download_buffer(dhd, dhd->info->nv_path, NVRAM, &memblock, (int *)&len);
+		if ((len <= 0 || len > MAX_NVRAMBUF_SIZE)) {
+			DHD_ERROR(("Can't open nvram info, skip save wifiver info\n"));
+			break;
+		}
+
+		sec_save_wlinfo(fw_version, EPI_VERSION_STR, memblock, clm_version);
+		if (memblock) {
+			dhd_free_download_buffer(dhd, memblock, len);
+		}
+	} while (0);
+#endif /* WRITE_WLANINFO */
+
+#ifdef GEN_SOFTAP_INFO_FILE
+	sec_save_softap_info();
+#endif /* GEN_SOFTAP_INFO_FILE */
+
+#ifdef PNO_SUPPORT
+	if (!dhd->pno_state) {
+		dhd_pno_init(dhd);
+	}
+#endif
+
+#ifdef RTT_SUPPORT
+	if (dhd->rtt_state) {
+		ret = dhd_rtt_init(dhd);
+		if (ret < 0) {
+			DHD_ERROR(("%s failed to initialize RTT\n", __FUNCTION__));
+		}
+	}
+#endif
+
+#ifdef FILTER_IE
+	/* Failure to configure filter IE is not a fatal error, ignore it. */
+	if (FW_SUPPORTED(dhd, fie) &&
+		!(dhd->op_mode & (DHD_FLAG_HOSTAP_MODE | DHD_FLAG_MFG_MODE))) {
+		dhd_read_from_file(dhd);
+	}
+#endif /* FILTER_IE */
+
+#ifdef NDO_CONFIG_SUPPORT
+	dhd->ndo_enable = FALSE;
+	dhd->ndo_host_ip_overflow = FALSE;
+	dhd->ndo_max_host_ip = NDO_MAX_HOST_IP_ENTRIES;
+#endif /* NDO_CONFIG_SUPPORT */
+
+	/* ND offload version supported */
+	dhd->ndo_version = dhd_ndo_get_version(dhd);
+
+	/* check dongle supports wbtext (product policy) or not */
+	dhd->wbtext_support = FALSE;
+	if (dhd_wl_ioctl_get_intiovar(dhd, "wnm_bsstrans_resp", &wnm_bsstrans_resp,
+			WLC_GET_VAR, FALSE, 0) != BCME_OK) {
+		DHD_ERROR(("failed to get wnm_bsstrans_resp\n"));
+	}
+	dhd->wbtext_policy = wnm_bsstrans_resp;
+	if (dhd->wbtext_policy == WL_BSSTRANS_POLICY_PRODUCT_WBTEXT) {
+		dhd->wbtext_support = TRUE;
+	}
+#ifndef WBTEXT
+	/* driver can turn off wbtext feature through makefile */
+	if (dhd->wbtext_support) {
+		if (dhd_wl_ioctl_set_intiovar(dhd, "wnm_bsstrans_resp",
+				WL_BSSTRANS_POLICY_ROAM_ALWAYS,
+				WLC_SET_VAR, FALSE, 0) != BCME_OK) {
+			DHD_ERROR(("failed to disable WBTEXT\n"));
+		}
+	}
+#endif /* !WBTEXT */
+
+#ifdef DHD_NON_DMA_M2M_CORRUPTION
+	/* check pcie non dma loopback */
+	if (dhd->op_mode == DHD_FLAG_MFG_MODE &&
+		(dhd_bus_dmaxfer_lpbk(dhd, M2M_NON_DMA_LPBK) < 0)) {
+			goto done;
+	}
+#endif /* DHD_NON_DMA_M2M_CORRUPTION */
+
+#ifdef CUSTOM_ASSOC_TIMEOUT
+	/* set recreate_bi_timeout to increase assoc timeout :
+	* 20 * 100TU * 1024 / 1000 = 2 secs
+	* (beacon wait time = recreate_bi_timeout * beacon_period * 1024 / 1000)
+	*/
+	if (dhd_wl_ioctl_set_intiovar(dhd, "recreate_bi_timeout",
+			CUSTOM_ASSOC_TIMEOUT,
+			WLC_SET_VAR, TRUE, 0) != BCME_OK) {
+		DHD_ERROR(("failed to set assoc timeout\n"));
+	}
+#endif /* CUSTOM_ASSOC_TIMEOUT */
+
+	BCM_REFERENCE(ret2);
+#if defined(WBTEXT) && defined(RRM_BCNREQ_MAX_CHAN_TIME)
+	if (dhd_iovar(dhd, 0, "rrm_bcn_req_thrtl_win",
+		(char *)&rrm_bcn_req_thrtl_win, sizeof(rrm_bcn_req_thrtl_win),
+		NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("failed to set RRM BCN request thrtl_win\n"));
+	}
+	if (dhd_iovar(dhd, 0, "rrm_bcn_req_max_off_chan_time",
+		(char *)&rrm_bcn_req_max_off_chan_time, sizeof(rrm_bcn_req_max_off_chan_time),
+		NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("failed to set RRM BCN Request max_off_chan_time\n"));
+	}
+#endif /* WBTEXT && RRM_BCNREQ_MAX_CHAN_TIME */
+#ifdef WL_MONITOR
+#ifdef HOST_RADIOTAP_CONV
+	/* 'Wl monitor' IOVAR is fired to check whether the FW supports radiotap conversion or not.
+	 * This is indicated through MSB(1<<31) bit, based on which host radiotap conversion
+	 * will be enabled or disabled.
+	 * 0 - Host supports Radiotap conversion.
+	 * 1 - FW supports Radiotap conversion.
+	 */
+	bcm_mkiovar("monitor", (char *)&monitor, sizeof(monitor), iovbuf, sizeof(iovbuf));
+	if ((ret2 = dhd_wl_ioctl_cmd(dhd, WLC_GET_MONITOR, iovbuf,
+		sizeof(iovbuf), FALSE, 0)) == 0) {
+		memcpy(&monitor, iovbuf, sizeof(monitor));
+		dhdinfo->host_radiotap_conv = (monitor & HOST_RADIOTAP_CONV_BIT) ? TRUE : FALSE;
+	} else {
+		DHD_ERROR(("%s Failed to get monitor mode, err %d\n",
+			__FUNCTION__, ret2));
+	}
+#endif /* HOST_RADIOTAP_CONV */
+	if (FW_SUPPORTED(dhd, monitor)) {
+		dhd->monitor_enable = TRUE;
+		DHD_ERROR(("%s: Monitor mode is enabled in FW cap\n", __FUNCTION__));
+	} else {
+		dhd->monitor_enable = FALSE;
+		DHD_ERROR(("%s: Monitor mode is not enabled in FW cap\n", __FUNCTION__));
+	}
+#endif /* WL_MONITOR */
+
+	/* store the preserve log set numbers */
+	if (dhd_get_preserve_log_numbers(dhd, &dhd->logset_prsrv_mask)
+			!= BCME_OK) {
+		DHD_ERROR(("%s: Failed to get preserve log # !\n", __FUNCTION__));
+	}
+
+#ifdef CUSTOM_OCL_RSSI_VAL
+	if (ocl_rssi_threshold != FW_OCL_RSSI_THRESH_INITVAL) {
+		ret = dhd_iovar(dhd, 0, "ocl_rssi_threshold", (char *)&ocl_rssi_threshold,
+			sizeof(ocl_rssi_threshold), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("failed to set ocl_rssi_threshold ret %d\n", ret));
+		}
+	}
+#endif /* CUSTOM_OCL_RSSI_VAL */
+
+	if (dhd_iovar(dhd, 0, "bus:d3_hostwake_delay", (char *)&d3_hostwake_delay,
+		sizeof(d3_hostwake_delay), NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("%s: d3_hostwake_delay IOVAR not present, proceed\n", __FUNCTION__));
+	} else {
+		DHD_ERROR(("%s: d3_hostwake_delay enabled\n", __FUNCTION__));
+	}
+
+	dhd->fr_phase_nibble = TRUE;
+	if (dhd_iovar(dhd, 0, "bus:fr_phase_nibble", (char *)&dhd->fr_phase_nibble,
+		sizeof(dhd->fr_phase_nibble), NULL, 0, TRUE) < 0) {
+		dhd->fr_phase_nibble = FALSE;
+		DHD_ERROR(("%s: fr_phase_nibble IOVAR not present, proceed\n", __FUNCTION__));
+	} else {
+		DHD_ERROR(("%s: fr_phase_nibble enabled\n", __FUNCTION__));
+	}
+
+#ifndef OEM_ANDROID
+	/* For non-android FC modular builds, override firmware preinited values */
+	dhd_override_fwprenit(dhd);
+#endif /* !OEM_ANDROID */
+	dhd_set_bandlock(dhd);
+
+#ifdef DHD_WAKE_EVENT_STATUS
+#ifdef CUSTOM_WAKE_REASON_STATS
+	/* Initialization */
+	if (dhd_bus_get_wakecount(dhd)) {
+		int i = 0;
+		wake_counts_t *wcp = dhd_bus_get_wakecount(dhd);
+
+		wcp->rc_event_idx = 0;
+		for (i = 0; i < MAX_WAKE_REASON_STATS; i++) {
+			wcp->rc_event[i] = -1;
+		}
+	}
+#endif /* CUSTOM_WAKE_REASON_STATS */
+#endif /* DHD_WAKE_EVENT_STATUS */
+
+#ifdef WL_UWB_COEX
+#ifdef WL_UWB_COEX_DEF_ENABLE
+	ret = wl_cfg_uwb_coex_enable(dhd_linux_get_primary_netdev(dhd),
+		TRUE, UWB_COEX_CH_MIN, UWB_COEX_CH_MAX);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("Failed to set UWB Coex (%d)\n", ret));
+	}
+#endif /* WL_UWB_COEX_DEF_ENABLE */
+#endif /* WL_UWB_COEX */
+
+done:
+	dhd_conf_postinit_ioctls(dhd);
+	if (iov_buf) {
+		MFREE(dhd->osh, iov_buf, WLC_IOCTL_SMLEN);
+	}
+	return ret;
+}
+
+int
+dhd_legacy_preinit_ioctls(dhd_pub_t *dhd)
+{
+	int ret = 0;
+	/*  Room for "event_msgs_ext" + '\0' + bitvec  */
+	char iovbuf[WL_EVENTING_MASK_EXT_LEN + EVENTMSGS_EXT_STRUCT_SIZE + 16];
+	char *mask;
+	uint32 buf_key_b4_m4 = 1;
+	uint8 msglen;
+	eventmsgs_ext_t *eventmask_msg = NULL;
+	uint32 event_log_max_sets = 0;
+	char* iov_buf = NULL;
+	/* XXX: Use ret2 for return check of IOVARS that might return BCME_UNSUPPORTED,
+	*	based on FW build tag.
+	*/
+	int ret2 = 0;
+	uint32 wnm_cap = 0;
+#if defined(WL_MONITOR) && defined(HOST_RADIOTAP_CONV)
+	uint monitor = 0;
+	dhd_info_t *dhdinfo = (dhd_info_t*)dhd->info;
+#endif /* WL_MONITOR */
+#if defined(BCMSUP_4WAY_HANDSHAKE)
+	uint32 sup_wpa = 1;
+#endif /* BCMSUP_4WAY_HANDSHAKE */
+#if defined(CUSTOM_AMPDU_BA_WSIZE)
+	uint32 ampdu_ba_wsize = 0;
+#endif
+#if defined(CUSTOM_AMPDU_MPDU)
+	int32 ampdu_mpdu = 0;
+#endif
+#if defined(CUSTOM_AMPDU_RELEASE)
+	int32 ampdu_release = 0;
+#endif
+#if defined(CUSTOM_AMSDU_AGGSF)
+	int32 amsdu_aggsf = 0;
+#endif
+
+#if defined(BCMSDIO) || defined(BCMDBUS)
+#ifdef PROP_TXSTATUS
+	int wlfc_enable = TRUE;
+#ifndef DISABLE_11N
+	uint32 hostreorder = 1;
+	uint wl_down = 1;
+#endif /* DISABLE_11N */
+#endif /* PROP_TXSTATUS */
+#endif /* defined(BCMSDIO) || defined(BCMDBUS) */
+
+#ifndef PCIE_FULL_DONGLE
+	uint32 wl_ap_isolate;
+#endif /* PCIE_FULL_DONGLE */
+	uint32 frameburst = CUSTOM_FRAMEBURST_SET;
+	uint wnm_bsstrans_resp = 0;
+#ifdef SUPPORT_SET_CAC
+#ifdef CUSTOMER_HW4
+	/* Off by default */
+	uint32 cac = 0;
+#else
+	uint32 cac = 1;
+#endif /* CUSTOMER_HW4 */
+#endif /* SUPPORT_SET_CAC */
+#ifdef DHD_BUS_MEM_ACCESS
+	uint32 enable_memuse = 1;
+#endif /* DHD_BUS_MEM_ACCESS */
+#if defined(SUPPORT_2G_VHT) || defined(SUPPORT_5G_1024QAM_VHT)
+	uint32 vht_features = 0; /* init to 0, will be set based on each support */
+#endif /* SUPPORT_2G_VHT || SUPPORT_5G_1024QAM_VHT */
+
+#ifdef OEM_ANDROID
+#ifdef DHD_ENABLE_LPC
+	uint32 lpc = 1;
+#endif /* DHD_ENABLE_LPC */
+	uint power_mode = PM_FAST;
+#ifdef ASSOC_PREFER_RSSI_THRESH
+	int assoc_pref_rssi_thresh = ASSOC_PREFER_RSSI_THRESH;
+#endif /* ASSOC_PREFER_RSSI_THRESH */
+#if defined(BCMSDIO)
+	uint32 dongle_align = DHD_SDALIGN;
+	uint32 glom = CUSTOM_GLOM_SETTING;
+#endif /* defined(BCMSDIO) */
+	uint bcn_timeout = CUSTOM_BCN_TIMEOUT;
+	uint scancache_enab = TRUE;
+#ifdef ENABLE_BCN_LI_BCN_WAKEUP
+	uint32 bcn_li_bcn = 1;
+#endif /* ENABLE_BCN_LI_BCN_WAKEUP */
+	uint retry_max = CUSTOM_ASSOC_RETRY_MAX;
+	int scan_assoc_time = DHD_SCAN_ASSOC_ACTIVE_TIME;
+	int scan_unassoc_time = DHD_SCAN_UNASSOC_ACTIVE_TIME;
+	int scan_passive_time = DHD_SCAN_PASSIVE_TIME;
+	char buf[WLC_IOCTL_SMLEN];
+	char *ptr;
+	uint32 listen_interval = CUSTOM_LISTEN_INTERVAL; /* Default Listen Interval in Beacons */
+#ifdef DHD_RANDMAC_LOGGING
+	uint privacy_mask = 0;
+#endif /* DHD_RANDMAC_LOGGING */
+#ifdef ROAM_ENABLE
+	uint roamvar = 0;
+	int roam_trigger[2] = {CUSTOM_ROAM_TRIGGER_SETTING, WLC_BAND_ALL};
+	int roam_scan_period[2] = {10, WLC_BAND_ALL};
+	int roam_delta[2] = {CUSTOM_ROAM_DELTA_SETTING, WLC_BAND_ALL};
+#ifdef ROAM_AP_ENV_DETECTION
+	int roam_env_mode = AP_ENV_INDETERMINATE;
+#endif /* ROAM_AP_ENV_DETECTION */
+#ifdef FULL_ROAMING_SCAN_PERIOD_60_SEC
+	int roam_fullscan_period = 60;
+#else /* FULL_ROAMING_SCAN_PERIOD_60_SEC */
+	int roam_fullscan_period = 120;
+#endif /* FULL_ROAMING_SCAN_PERIOD_60_SEC */
+#ifdef DISABLE_BCNLOSS_ROAM
+	uint roam_bcnloss_off = 1;
+#endif /* DISABLE_BCNLOSS_ROAM */
+#else
+#ifdef DISABLE_BUILTIN_ROAM
+	uint roamvar = 1;
+#endif /* DISABLE_BUILTIN_ROAM */
+#endif /* ROAM_ENABLE */
+
+#if defined(SOFTAP)
+	uint dtim = 1;
+#endif
+/* xxx andrey tmp fix for dk8000 build error  */
+#if (defined(AP) && !defined(WLP2P)) || (!defined(AP) && defined(WL_CFG80211))
+	struct ether_addr p2p_ea;
+#endif
+#ifdef BCMCCX
+	uint32 ccx = 1;
+#endif
+#ifdef SOFTAP_UAPSD_OFF
+	uint32 wme_apsd = 0;
+#endif /* SOFTAP_UAPSD_OFF */
+#if (defined(AP) || defined(WLP2P)) && !defined(SOFTAP_AND_GC)
+	uint32 apsta = 1; /* Enable APSTA mode */
+#elif defined(SOFTAP_AND_GC)
+	uint32 apsta = 0;
+	int ap_mode = 1;
+#endif /* (defined(AP) || defined(WLP2P)) && !defined(SOFTAP_AND_GC) */
+	struct ether_addr ea_addr;
+	char hw_ether[62];
+	dhd_if_t *ifp = dhd->info->iflist[0];
+	bool set_mac = FALSE;
+#ifdef OKC_SUPPORT
+	uint32 okc = 1;
+#endif
+
+#ifdef DISABLE_11N
+	uint32 nmode = 0;
+#endif /* DISABLE_11N */
+
+#if defined(DISABLE_11AC)
+	uint32 vhtmode = 0;
+#endif /* DISABLE_11AC */
+#ifdef USE_WL_TXBF
+	uint32 txbf = 1;
+#endif /* USE_WL_TXBF */
+#ifdef DISABLE_TXBFR
+	uint32 txbf_bfr_cap = 0;
+#endif /* DISABLE_TXBFR */
+#ifdef AMPDU_VO_ENABLE
+	/* XXX: Enabling VO AMPDU to reduce FER */
+	struct ampdu_tid_control tid;
+#endif
+#if defined(PROP_TXSTATUS)
+#ifdef USE_WFA_CERT_CONF
+	uint32 proptx = 0;
+#endif /* USE_WFA_CERT_CONF */
+#endif /* PROP_TXSTATUS */
+#ifdef DHD_SET_FW_HIGHSPEED
+	uint32 ack_ratio = 250;
+	uint32 ack_ratio_depth = 64;
+#endif /* DHD_SET_FW_HIGHSPEED */
+#ifdef DISABLE_11N_PROPRIETARY_RATES
+	uint32 ht_features = 0;
+#endif /* DISABLE_11N_PROPRIETARY_RATES */
+#ifdef DISABLE_PRUNED_SCAN
+	uint32 scan_features = 0;
+#endif /* DISABLE_PRUNED_SCAN */
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	uint32 hostwake_oob = 0;
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef EVENT_LOG_RATE_HC
+	/* threshold number of lines per second */
+	uint32 event_log_rate_hc = EVENT_LOG_RATE_HC_THRESHOLD;
+#endif /* EVENT_LOG_RATE_HC */
+#endif  /* OEM_ANDROID */
+#if defined(WBTEXT) && defined(WBTEXT_BTMDELTA)
+	uint32 btmdelta = WBTEXT_BTMDELTA;
+#endif /* WBTEXT && WBTEXT_BTMDELTA */
+#if defined(WBTEXT) && defined(RRM_BCNREQ_MAX_CHAN_TIME)
+	uint32 rrm_bcn_req_thrtl_win = RRM_BCNREQ_MAX_CHAN_TIME * 2;
+	uint32 rrm_bcn_req_max_off_chan_time = RRM_BCNREQ_MAX_CHAN_TIME;
+#endif /* WBTEXT && RRM_BCNREQ_MAX_CHAN_TIME */
+#ifdef CUSTOM_PSPRETEND_THR
+	uint32 pspretend_thr = CUSTOM_PSPRETEND_THR;
+#endif
+#if (defined(DHD_8021X_DUMP) && defined(SHOW_LOGTRACE)) || defined(DEBUGABILITY)
+	wl_el_tag_params_t *el_tag = NULL;
+#endif /* DHD_8021X_DUMP */
+
+	uint32 d3_hostwake_delay = D3_HOSTWAKE_DELAY;
+
+	BCM_REFERENCE(iovbuf);
+	DHD_TRACE(("Enter %s\n", __FUNCTION__));
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	/* arpoe will be applied from the supsend context */
+	dhd->arpoe_enable = TRUE;
+	dhd->arpol_configured = FALSE;
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#ifdef OEM_ANDROID
+#ifdef PKT_FILTER_SUPPORT
+	dhd_pkt_filter_enable = TRUE;
+#endif /* PKT_FILTER_SUPPORT */
+#ifdef APF
+	dhd->apf_set = FALSE;
+#endif /* APF */
+#ifdef CUSTOM_SET_OCLOFF
+	dhd->ocl_off = FALSE;
+#endif /* CUSTOM_SET_OCLOFF */
+#ifdef SUPPORT_SET_TID
+	dhd->tid_mode = SET_TID_OFF;
+	dhd->target_uid = 0;
+	dhd->target_tid = 0;
+#endif /* SUPPORT_SET_TID */
+#ifdef DHDTCPACK_SUPPRESS
+	dhd_tcpack_suppress_set(dhd, dhd->conf->tcpack_sup_mode);
+#endif
+	dhd->op_mode = 0;
+
+	/* clear AP flags */
+#if defined(CUSTOM_COUNTRY_CODE)
+	dhd->dhd_cflags &= ~WLAN_PLAT_AP_FLAG;
+#endif /* CUSTOM_COUNTRY_CODE */
+
+	/* query for 'ver' to get version info from firmware */
+	memset(buf, 0, sizeof(buf));
+	ptr = buf;
+	ret = dhd_iovar(dhd, 0, "ver", NULL, 0, (char *)&buf, sizeof(buf), FALSE);
+	if (ret < 0)
+		DHD_ERROR(("%s failed %d\n", __FUNCTION__, ret));
+	else {
+		bcmstrtok(&ptr, "\n", 0);
+		/* Print fw version info */
+		strncpy(fw_version, buf, FW_VER_STR_LEN);
+		fw_version[FW_VER_STR_LEN-1] = '\0';
+	}
+
+// mark flag for later blob checking
+	/* Set op_mode as MFG_MODE if WLTEST is present in "wl ver" */
+	if (strstr(fw_version, "WLTEST") != NULL) {
+		DHD_INFO(("%s: wl ver has WLTEST, setting op_mode as DHD_FLAG_MFG_MODE\n",
+			__FUNCTION__));
+		op_mode = DHD_FLAG_MFG_MODE;
+	}
+
+	if ((!op_mode && dhd_get_fw_mode(dhd->info) == DHD_FLAG_MFG_MODE) ||
+		(op_mode == DHD_FLAG_MFG_MODE)) {
+		dhd->op_mode = DHD_FLAG_MFG_MODE;
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+		/* disable runtimePM by default in MFG mode. */
+		pm_runtime_disable(dhd_bus_to_dev(dhd->bus));
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+		/* Check and adjust IOCTL response timeout for Manufactring firmware */
+		dhd_os_set_ioctl_resp_timeout(MFG_IOCTL_RESP_TIMEOUT);
+		DHD_INFO(("%s : Set IOCTL response time for Manufactring Firmware\n",
+			__FUNCTION__));
+	} else {
+		dhd_os_set_ioctl_resp_timeout(IOCTL_RESP_TIMEOUT);
+		DHD_INFO(("%s : Set IOCTL response time.\n", __FUNCTION__));
+	}
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	ret = dhd_iovar(dhd, 0, "bus:hostwake_oob", NULL, 0, (char *)&hostwake_oob,
+		sizeof(hostwake_oob), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: hostwake_oob IOVAR not present, proceed\n", __FUNCTION__));
+	} else {
+		if (hostwake_oob == 0) {
+			DHD_ERROR(("%s: hostwake_oob is not enabled in the NVRAM, STOP\n",
+				__FUNCTION__));
+			ret = BCME_UNSUPPORTED;
+			goto done;
+		} else {
+			DHD_INFO(("%s: hostwake_oob enabled\n", __FUNCTION__));
+		}
+	}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+	ret = dhd_iovar(dhd, 0, "axierror_logbuf_addr", NULL, 0, (char *)&dhd->axierror_logbuf_addr,
+		sizeof(dhd->axierror_logbuf_addr), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: axierror_logbuf_addr IOVAR not present, proceed\n", __FUNCTION__));
+		dhd->axierror_logbuf_addr = 0;
+	} else {
+		DHD_INFO(("%s: axierror_logbuf_addr : 0x%x\n",
+			__FUNCTION__, dhd->axierror_logbuf_addr));
+	}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#ifdef EVENT_LOG_RATE_HC
+	ret = dhd_iovar(dhd, 0, "event_log_rate_hc", (char *)&event_log_rate_hc,
+		sizeof(event_log_rate_hc), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s event_log_rate_hc set failed %d\n", __FUNCTION__, ret));
+	} else  {
+		DHD_INFO(("%s event_log_rate_hc set with threshold:%d\n", __FUNCTION__,
+			event_log_rate_hc));
+	}
+#endif /* EVENT_LOG_RATE_HC */
+
+	memset(hw_ether, 0, sizeof(hw_ether));
+	if (ifp->set_macaddress) {
+		memcpy(hw_ether, ifp->mac_addr, ETHER_ADDR_LEN);
+		set_mac = TRUE;
+	}
+#ifdef GET_CUSTOM_MAC_ENABLE
+#ifdef GET_CUSTOM_MAC_FROM_CONFIG
+	else if (memcmp(&ether_null, &dhd->conf->hw_ether, ETHER_ADDR_LEN)) {
+		set_mac = TRUE;
+		memcpy(hw_ether, &dhd->conf->hw_ether, sizeof(dhd->conf->hw_ether));
+	}
+#endif
+	else {
+		ret = wifi_platform_get_mac_addr(dhd->info->adapter, hw_ether, 0);
+		if (!ret)
+			set_mac = TRUE;
+	}
+#endif /* GET_CUSTOM_MAC_ENABLE */
+	if (set_mac) {
+		memset(buf, 0, sizeof(buf));
+		bcopy(hw_ether, ea_addr.octet, sizeof(struct ether_addr));
+		bcm_mkiovar("cur_etheraddr", (void *)&ea_addr, ETHER_ADDR_LEN, buf, sizeof(buf));
+		ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, sizeof(buf), TRUE, 0);
+		if (ret < 0) {
+			memset(buf, 0, sizeof(buf));
+			bcm_mkiovar("hw_ether", hw_ether, sizeof(hw_ether), buf, sizeof(buf));
+			ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, sizeof(buf), TRUE, 0);
+			if (ret) {
+				DHD_ERROR(("%s: can't set MAC address MAC="MACDBG", error=%d\n",
+					__FUNCTION__, MAC2STRDBG(hw_ether), ret));
+				prhex("MACPAD", &hw_ether[ETHER_ADDR_LEN], sizeof(hw_ether)-ETHER_ADDR_LEN);
+				ret = BCME_NOTUP;
+				goto done;
+			}
+		}
+	}
+#ifdef GET_CUSTOM_MAC_ENABLE
+	else {
+		DHD_ERROR(("%s: can't get custom MAC address, ret=%d\n", __FUNCTION__, ret));
+		ret = BCME_NOTUP;
+		goto done;
+	}
+#endif /* GET_CUSTOM_MAC_ENABLE */
+	/* Get the default device MAC address directly from firmware */
+	ret = dhd_iovar(dhd, 0, "cur_etheraddr", NULL, 0, (char *)&buf, sizeof(buf), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: can't get MAC address , error=%d\n", __FUNCTION__, ret));
+		ret = BCME_NOTUP;
+		goto done;
+	}
+
+	DHD_INFO(("%s: use firmware generated mac_address "MACDBG"\n",
+		__FUNCTION__, MAC2STRDBG(&buf)));
+
+#ifdef MACADDR_PROVISION_ENFORCED
+	if (ETHER_IS_LOCALADDR(buf)) {
+		DHD_ERROR(("%s: error! not using provision mac addr!\n", __FUNCTION__));
+		ret = BCME_BADADDR;
+		goto done;
+	}
+#endif /* MACADDR_PROVISION_ENFORCED */
+
+	/* Update public MAC address after reading from Firmware */
+	memcpy(dhd->mac.octet, buf, ETHER_ADDR_LEN);
+
+	if (ETHER_ISNULLADDR(dhd->mac.octet)) {
+		DHD_ERROR(("%s: NULL MAC address during pre-init\n", __FUNCTION__));
+		ret = BCME_BADADDR;
+		goto done;
+	} else {
+		(void)memcpy_s(dhd_linux_get_primary_netdev(dhd)->perm_addr, ETHER_ADDR_LEN,
+			dhd->mac.octet, ETHER_ADDR_LEN);
+	}
+#if defined(WL_STA_ASSOC_RAND) && defined(WL_STA_INIT_RAND)
+	/* Set cur_etheraddr of primary interface to randomized address to ensure
+	 * that any action frame transmission will happen using randomized macaddr
+	 * primary netdev->perm_addr will hold the original factory MAC.
+	 */
+	{
+		if ((ret = dhd_update_rand_mac_addr(dhd)) < 0) {
+			DHD_ERROR(("%s: failed to set macaddress\n", __FUNCTION__));
+			goto done;
+		}
+	}
+#endif /* WL_STA_ASSOC_RAND && WL_STA_INIT_RAND */
+#ifdef BCMDBUS
+	if (dhd->info->clm_path[0] == '\0') {
+		if ((strlen(clm_path) > MOD_PARAM_PATHLEN) ||
+			(strlen(clm_path) >= sizeof(dhd->info->clm_path))) {
+			DHD_ERROR(("%s clm path exceeds max len\n", __FUNCTION__));
+			ret = BCME_BADLEN;
+			goto done;
+		}
+		strlcpy(dhd->info->clm_path, clm_path, sizeof(dhd->info->clm_path));
+	}
+#endif /* BCMDBUS */
+
+	if ((ret = dhd_apply_default_clm(dhd, dhd->info->clm_path)) < 0) {
+		DHD_INFO(("%s: CLM set failed. Ignore and continue initialization.\n",
+			__FUNCTION__));
+	}
+
+	/* get a capabilities from firmware */
+	{
+		uint32 cap_buf_size = sizeof(dhd->fw_capabilities);
+		memset(dhd->fw_capabilities, 0, cap_buf_size);
+		ret = dhd_iovar(dhd, 0, "cap", NULL, 0, dhd->fw_capabilities, (cap_buf_size - 1),
+				FALSE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: Get Capability failed (error=%d)\n",
+				__FUNCTION__, ret));
+			return 0;
+		}
+
+		memmove(&dhd->fw_capabilities[1], dhd->fw_capabilities, (cap_buf_size - 1));
+		dhd->fw_capabilities[0] = ' ';
+		dhd->fw_capabilities[cap_buf_size - 2] = ' ';
+		dhd->fw_capabilities[cap_buf_size - 1] = '\0';
+	}
+
+	if ((!op_mode && dhd_get_fw_mode(dhd->info) == DHD_FLAG_HOSTAP_MODE) ||
+		(op_mode == DHD_FLAG_HOSTAP_MODE)) {
+#ifdef SET_RANDOM_MAC_SOFTAP
+		uint rand_mac;
+#endif /* SET_RANDOM_MAC_SOFTAP */
+		dhd->op_mode = DHD_FLAG_HOSTAP_MODE;
+#ifdef PKT_FILTER_SUPPORT
+		if (dhd_conf_get_insuspend(dhd, AP_FILTER_IN_SUSPEND))
+			dhd_pkt_filter_enable = TRUE;
+		else
+			dhd_pkt_filter_enable = FALSE;
+#endif
+#ifdef SET_RANDOM_MAC_SOFTAP
+		rand_mac = RANDOM32();
+		iovbuf[0] = (unsigned char)(vendor_oui >> 16) | 0x02;	/* local admin bit */
+		iovbuf[1] = (unsigned char)(vendor_oui >> 8);
+		iovbuf[2] = (unsigned char)vendor_oui;
+		iovbuf[3] = (unsigned char)(rand_mac & 0x0F) | 0xF0;
+		iovbuf[4] = (unsigned char)(rand_mac >> 8);
+		iovbuf[5] = (unsigned char)(rand_mac >> 16);
+
+		ret = dhd_iovar(dhd, 0, "cur_etheraddr", (char *)&iovbuf, ETHER_ADDR_LEN, NULL, 0,
+				TRUE);
+		if (ret < 0) {
+			DHD_INFO(("%s: can't set MAC address , error=%d\n", __FUNCTION__, ret));
+		} else
+			memcpy(dhd->mac.octet, iovbuf, ETHER_ADDR_LEN);
+#endif /* SET_RANDOM_MAC_SOFTAP */
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+		dhdsdio_func_blocksize(dhd, 2, DYNAMIC_F2_BLKSIZE_FOR_NONLEGACY);
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+#ifdef SUPPORT_AP_POWERSAVE
+		dhd_set_ap_powersave(dhd, 0, TRUE);
+#endif /* SUPPORT_AP_POWERSAVE */
+#ifdef SOFTAP_UAPSD_OFF
+		ret = dhd_iovar(dhd, 0, "wme_apsd", (char *)&wme_apsd, sizeof(wme_apsd), NULL, 0,
+				TRUE);
+		if (ret < 0) {
+			DHD_INFO(("%s: set wme_apsd 0 fail (error=%d)\n",
+				__FUNCTION__, ret));
+		}
+#endif /* SOFTAP_UAPSD_OFF */
+
+		/* set AP flag for specific country code of SOFTAP */
+#if defined(CUSTOM_COUNTRY_CODE)
+		dhd->dhd_cflags |= WLAN_PLAT_AP_FLAG | WLAN_PLAT_NODFS_FLAG;
+#endif /* CUSTOM_COUNTRY_CODE */
+	} else if ((!op_mode && dhd_get_fw_mode(dhd->info) == DHD_FLAG_MFG_MODE) ||
+		(op_mode == DHD_FLAG_MFG_MODE)) {
+#if defined(ARP_OFFLOAD_SUPPORT)
+		dhd->arpoe_enable = FALSE;
+#endif /* ARP_OFFLOAD_SUPPORT */
+#ifdef PKT_FILTER_SUPPORT
+		dhd_pkt_filter_enable = FALSE;
+#endif /* PKT_FILTER_SUPPORT */
+		dhd->op_mode = DHD_FLAG_MFG_MODE;
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+		/* XXX The 'wl counters' command triggers SDIO bus error
+		 * if F2 block size is greater than 128 bytes using 4354A1
+		 * manufacturing firmware. To avoid this problem, F2 block
+		 * size is set to 128 bytes only for DHD_FLAG_MFG_MODE.
+		 * There is no problem for other chipset since big data
+		 * transcation through SDIO bus is not happened during
+		 * manufacturing test.
+		 */
+		dhdsdio_func_blocksize(dhd, 2, DYNAMIC_F2_BLKSIZE_FOR_NONLEGACY);
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+#ifndef CUSTOM_SET_ANTNPM
+		if (FW_SUPPORTED(dhd, rsdb)) {
+			wl_config_t rsdb_mode;
+			memset(&rsdb_mode, 0, sizeof(rsdb_mode));
+			ret = dhd_iovar(dhd, 0, "rsdb_mode", (char *)&rsdb_mode, sizeof(rsdb_mode),
+				NULL, 0, TRUE);
+			if (ret < 0) {
+				DHD_INFO(("%s Disable rsdb_mode is failed ret= %d\n",
+					__FUNCTION__, ret));
+			}
+		}
+#endif /* !CUSTOM_SET_ANTNPM */
+	} else {
+		uint32 concurrent_mode = 0;
+		if ((!op_mode && dhd_get_fw_mode(dhd->info) == DHD_FLAG_P2P_MODE) ||
+			(op_mode == DHD_FLAG_P2P_MODE)) {
+#ifdef PKT_FILTER_SUPPORT
+			dhd_pkt_filter_enable = FALSE;
+#endif
+			dhd->op_mode = DHD_FLAG_P2P_MODE;
+		} else if ((!op_mode && dhd_get_fw_mode(dhd->info) == DHD_FLAG_IBSS_MODE) ||
+			(op_mode == DHD_FLAG_IBSS_MODE)) {
+			dhd->op_mode = DHD_FLAG_IBSS_MODE;
+		} else
+			dhd->op_mode = DHD_FLAG_STA_MODE;
+#if defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P)
+		if (dhd->op_mode != DHD_FLAG_IBSS_MODE &&
+			(concurrent_mode = dhd_get_concurrent_capabilites(dhd))) {
+			dhd->op_mode |= concurrent_mode;
+		}
+
+		/* Check if we are enabling p2p */
+		if (dhd->op_mode & DHD_FLAG_P2P_MODE) {
+			ret = dhd_iovar(dhd, 0, "apsta", (char *)&apsta, sizeof(apsta), NULL, 0,
+					TRUE);
+			if (ret < 0)
+				DHD_INFO(("%s APSTA for P2P failed ret= %d\n", __FUNCTION__, ret));
+
+#if defined(SOFTAP_AND_GC)
+		if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_AP,
+			(char *)&ap_mode, sizeof(ap_mode), TRUE, 0)) < 0) {
+				DHD_INFO(("%s WLC_SET_AP failed %d\n", __FUNCTION__, ret));
+		}
+#endif
+			memcpy(&p2p_ea, &dhd->mac, ETHER_ADDR_LEN);
+			ETHER_SET_LOCALADDR(&p2p_ea);
+			ret = dhd_iovar(dhd, 0, "p2p_da_override", (char *)&p2p_ea, sizeof(p2p_ea),
+					NULL, 0, TRUE);
+			if (ret < 0)
+				DHD_INFO(("%s p2p_da_override ret= %d\n", __FUNCTION__, ret));
+			else
+				DHD_INFO(("dhd_preinit_ioctls: p2p_da_override succeeded\n"));
+		}
+#else
+	(void)concurrent_mode;
+#endif /* defined(OEM_ANDROID) && !defined(AP) && defined(WLP2P) */
+	}
+
+#ifdef DISABLE_PRUNED_SCAN
+	if (FW_SUPPORTED(dhd, rsdb)) {
+		ret = dhd_iovar(dhd, 0, "scan_features", (char *)&scan_features,
+				sizeof(scan_features), iovbuf, sizeof(iovbuf), FALSE);
+		if (ret < 0) {
+			if (ret == BCME_UNSUPPORTED) {
+				DHD_ERROR(("%s get scan_features, UNSUPPORTED\n",
+					__FUNCTION__));
+			} else {
+				DHD_ERROR(("%s get scan_features err(%d)\n",
+					__FUNCTION__, ret));
+			}
+
+		} else {
+			memcpy(&scan_features, iovbuf, 4);
+			scan_features &= ~RSDB_SCAN_DOWNGRADED_CH_PRUNE_ROAM;
+			ret = dhd_iovar(dhd, 0, "scan_features", (char *)&scan_features,
+					sizeof(scan_features), NULL, 0, TRUE);
+			if (ret < 0) {
+				DHD_ERROR(("%s set scan_features err(%d)\n",
+					__FUNCTION__, ret));
+			}
+		}
+	}
+#endif /* DISABLE_PRUNED_SCAN */
+
+	DHD_INFO(("Firmware up: op_mode=0x%04x, MAC="MACDBG"\n",
+		dhd->op_mode, MAC2STRDBG(dhd->mac.octet)));
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+	if (!dhd->is_blob)
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+	{
+		/* get a ccode and revision for the country code */
+#if defined(CUSTOM_COUNTRY_CODE)
+		get_customized_country_code(dhd->info->adapter, dhd->dhd_cspec.country_abbrev,
+			&dhd->dhd_cspec, dhd->dhd_cflags);
+#else
+		get_customized_country_code(dhd->info->adapter, dhd->dhd_cspec.country_abbrev,
+			&dhd->dhd_cspec);
+#endif /* CUSTOM_COUNTRY_CODE */
+	}
+
+#if defined(RXFRAME_THREAD) && defined(RXTHREAD_ONLYSTA)
+	if (dhd->op_mode == DHD_FLAG_HOSTAP_MODE)
+		dhd->info->rxthread_enabled = FALSE;
+	else
+		dhd->info->rxthread_enabled = TRUE;
+#endif
+
+#if defined(CUSTOM_COUNTRY_CODE_XZ)
+	/* Set initial country code to XZ */
+	strlcpy(dhd->dhd_cspec.country_abbrev, "XZ", WLC_CNTRY_BUF_SZ);
+	strlcpy(dhd->dhd_cspec.ccode, "XZ", WLC_CNTRY_BUF_SZ);
+	DHD_INFO(("%s: Set initial country code to XZ(World Wide Safe)\n", __FUNCTION__));
+#endif /* CUSTOM_COUNTRY_CODE_XZ */
+	/* Set Country code  */
+	if (dhd->dhd_cspec.ccode[0] != 0) {
+		ret = dhd_iovar(dhd, 0, "country", (char *)&dhd->dhd_cspec, sizeof(wl_country_t),
+				NULL, 0, TRUE);
+		if (ret < 0)
+			DHD_ERROR(("%s: country code setting failed\n", __FUNCTION__));
+	}
+
+#if defined(DISABLE_11AC)
+	ret = dhd_iovar(dhd, 0, "vhtmode", (char *)&vhtmode, sizeof(vhtmode), NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s wl vhtmode 0 failed %d\n", __FUNCTION__, ret));
+#endif /* DISABLE_11AC */
+
+	/* Set Listen Interval */
+	ret = dhd_iovar(dhd, 0, "assoc_listen", (char *)&listen_interval, sizeof(listen_interval),
+			NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s assoc_listen failed %d\n", __FUNCTION__, ret));
+
+#if defined(ROAM_ENABLE) || defined(DISABLE_BUILTIN_ROAM)
+#ifdef USE_WFA_CERT_CONF
+	if (sec_get_param_wfa_cert(dhd, SET_PARAM_ROAMOFF, &roamvar) == BCME_OK) {
+		DHD_ERROR(("%s: read roam_off param =%d\n", __FUNCTION__, roamvar));
+	}
+#endif /* USE_WFA_CERT_CONF */
+	/* Disable built-in roaming to allowed ext supplicant to take care of roaming */
+	ret = dhd_iovar(dhd, 0, "roam_off", (char *)&roamvar, sizeof(roamvar), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s roam_off failed %d\n", __FUNCTION__, ret));
+	}
+	ROAMOFF_DBG_SAVE(dhd_linux_get_primary_netdev(dhd), SET_ROAM_PREINIT, roamvar);
+#endif /* ROAM_ENABLE || DISABLE_BUILTIN_ROAM */
+#if defined(ROAM_ENABLE)
+#ifdef DISABLE_BCNLOSS_ROAM
+	ret = dhd_iovar(dhd, 0, "roam_bcnloss_off", (char *)&roam_bcnloss_off,
+			sizeof(roam_bcnloss_off), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s roam_bcnloss_off failed %d\n", __FUNCTION__, ret));
+	}
+#endif /* DISABLE_BCNLOSS_ROAM */
+	if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_ROAM_TRIGGER, roam_trigger,
+		sizeof(roam_trigger), TRUE, 0)) < 0)
+		DHD_ERROR(("%s: roam trigger set failed %d\n", __FUNCTION__, ret));
+	if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_ROAM_SCAN_PERIOD, roam_scan_period,
+		sizeof(roam_scan_period), TRUE, 0)) < 0)
+		DHD_ERROR(("%s: roam scan period set failed %d\n", __FUNCTION__, ret));
+	if ((dhd_wl_ioctl_cmd(dhd, WLC_SET_ROAM_DELTA, roam_delta,
+		sizeof(roam_delta), TRUE, 0)) < 0)
+		DHD_ERROR(("%s: roam delta set failed %d\n", __FUNCTION__, ret));
+	ret = dhd_iovar(dhd, 0, "fullroamperiod", (char *)&roam_fullscan_period,
+			sizeof(roam_fullscan_period), NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s: roam fullscan period set failed %d\n", __FUNCTION__, ret));
+#ifdef ROAM_AP_ENV_DETECTION
+	if (roam_trigger[0] == WL_AUTO_ROAM_TRIGGER) {
+		if (dhd_iovar(dhd, 0, "roam_env_detection", (char *)&roam_env_mode,
+				sizeof(roam_env_mode), NULL, 0, TRUE) == BCME_OK)
+			dhd->roam_env_detection = TRUE;
+		else
+			dhd->roam_env_detection = FALSE;
+	}
+#endif /* ROAM_AP_ENV_DETECTION */
+#ifdef CONFIG_ROAM_RSSI_LIMIT
+	ret = dhd_roam_rssi_limit_set(dhd, CUSTOM_ROAMRSSI_2G, CUSTOM_ROAMRSSI_5G);
+	if (ret < 0) {
+		DHD_ERROR(("%s set roam_rssi_limit failed ret %d\n", __FUNCTION__, ret));
+	}
+#endif /* CONFIG_ROAM_RSSI_LIMIT */
+#ifdef CONFIG_ROAM_MIN_DELTA
+	ret = dhd_roam_min_delta_set(dhd, CUSTOM_ROAM_MIN_DELTA, CUSTOM_ROAM_MIN_DELTA);
+	if (ret < 0) {
+		DHD_ERROR(("%s set roam_min_delta failed ret %d\n", __FUNCTION__, ret));
+	}
+#endif /* CONFIG_ROAM_MIN_DELTA */
+#endif /* ROAM_ENABLE */
+
+#ifdef CUSTOM_EVENT_PM_WAKE
+	dhd_init_excess_pm_awake(dhd);
+#endif	/* CUSTOM_EVENT_PM_WAKE */
+#ifdef OKC_SUPPORT
+	dhd_iovar(dhd, 0, "okc_enable", (char *)&okc, sizeof(okc), NULL, 0, TRUE);
+#endif
+#ifdef BCMCCX
+	dhd_iovar(dhd, 0, "ccx_enable", (char *)&ccx, sizeof(ccx), NULL, 0, TRUE);
+#endif /* BCMCCX */
+
+#ifdef WLTDLS
+	dhd->tdls_enable = FALSE;
+	dhd_tdls_set_mode(dhd, false);
+#endif /* WLTDLS */
+
+#ifdef DHD_ENABLE_LPC
+	/* Set lpc 1 */
+	ret = dhd_iovar(dhd, 0, "lpc", (char *)&lpc, sizeof(lpc), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Set lpc failed  %d\n", __FUNCTION__, ret));
+
+		if (ret == BCME_NOTDOWN) {
+			uint wl_down = 1;
+			ret = dhd_wl_ioctl_cmd(dhd, WLC_DOWN,
+				(char *)&wl_down, sizeof(wl_down), TRUE, 0);
+			DHD_ERROR(("%s lpc fail WL_DOWN : %d, lpc = %d\n", __FUNCTION__, ret, lpc));
+
+			ret = dhd_iovar(dhd, 0, "lpc", (char *)&lpc, sizeof(lpc), NULL, 0, TRUE);
+			DHD_ERROR(("%s Set lpc ret --> %d\n", __FUNCTION__, ret));
+		}
+	}
+#endif /* DHD_ENABLE_LPC */
+
+#ifdef WLADPS
+	if (dhd->op_mode & DHD_FLAG_STA_MODE) {
+		if ((ret = dhd_enable_adps(dhd, ADPS_MODE_PM2_ONLY)) != BCME_OK &&
+			(ret != BCME_UNSUPPORTED)) {
+			DHD_ERROR(("%s dhd_enable_adps failed %d\n",
+				__FUNCTION__, ret));
+		}
+	}
+#endif /* WLADPS */
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+	sec_control_pm(dhd, &power_mode);
+#else
+	/* Set PowerSave mode */
+	(void) dhd_wl_ioctl_cmd(dhd, WLC_SET_PM, (char *)&power_mode, sizeof(power_mode), TRUE, 0);
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+#ifdef ASSOC_PREFER_RSSI_THRESH
+	ret = dhd_iovar(dhd, 0, "assoc_pref_rssi_thresh", (char *)&assoc_pref_rssi_thresh,
+		sizeof(assoc_pref_rssi_thresh), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set assoc_pref_rssi_thresh failed %d\n", __FUNCTION__, ret));
+	}
+#endif /* ASSOC_PREFER_RSSI_THRESH */
+#if defined(BCMSDIO)
+	/* Match Host and Dongle rx alignment */
+	ret = dhd_iovar(dhd, 0, "bus:txglomalign", (char *)&dongle_align, sizeof(dongle_align),
+			NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set bus:txglomalign failed %d\n", __FUNCTION__, ret));
+	}
+
+#ifdef USE_WFA_CERT_CONF
+	if (sec_get_param_wfa_cert(dhd, SET_PARAM_BUS_TXGLOM_MODE, &glom) == BCME_OK) {
+		DHD_ERROR(("%s, read txglom param =%d\n", __FUNCTION__, glom));
+	}
+#endif /* USE_WFA_CERT_CONF */
+	if (glom != DEFAULT_GLOM_VALUE) {
+		DHD_INFO(("%s set glom=0x%X\n", __FUNCTION__, glom));
+		ret = dhd_iovar(dhd, 0, "bus:txglom", (char *)&glom, sizeof(glom), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s set bus:txglom failed %d\n", __FUNCTION__, ret));
+		}
+	}
+#endif /* defined(BCMSDIO) */
+
+	/* Setup timeout if Beacons are lost and roam is off to report link down */
+	ret = dhd_iovar(dhd, 0, "bcn_timeout", (char *)&bcn_timeout, sizeof(bcn_timeout),
+			NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set bcn_timeout failed %d\n", __FUNCTION__, ret));
+	}
+
+	/* Setup assoc_retry_max count to reconnect target AP in dongle */
+	ret = dhd_iovar(dhd, 0, "assoc_retry_max", (char *)&retry_max, sizeof(retry_max),
+			NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set assoc_retry_max failed %d\n", __FUNCTION__, ret));
+	}
+
+#if defined(AP) && !defined(WLP2P)
+	ret = dhd_iovar(dhd, 0, "apsta", (char *)&apsta, sizeof(apsta), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set apsta failed %d\n", __FUNCTION__, ret));
+	}
+
+#endif /* defined(AP) && !defined(WLP2P) */
+
+#ifdef MIMO_ANT_SETTING
+	dhd_sel_ant_from_file(dhd);
+#endif /* MIMO_ANT_SETTING */
+
+#if defined(OEM_ANDROID) && defined(SOFTAP)
+	if (ap_fw_loaded == TRUE) {
+		dhd_wl_ioctl_cmd(dhd, WLC_SET_DTIMPRD, (char *)&dtim, sizeof(dtim), TRUE, 0);
+	}
+#endif /* defined(OEM_ANDROID) && defined(SOFTAP) */
+
+#if defined(KEEP_ALIVE)
+	{
+	/* Set Keep Alive : be sure to use FW with -keepalive */
+	int res;
+
+#if defined(OEM_ANDROID) && defined(SOFTAP)
+	if (ap_fw_loaded == FALSE)
+#endif /* defined(OEM_ANDROID) && defined(SOFTAP) */
+		if (!(dhd->op_mode &
+			(DHD_FLAG_HOSTAP_MODE | DHD_FLAG_MFG_MODE))) {
+			if ((res = dhd_keep_alive_onoff(dhd)) < 0)
+				DHD_ERROR(("%s set keeplive failed %d\n",
+				__FUNCTION__, res));
+		}
+	}
+#endif /* defined(KEEP_ALIVE) */
+
+#ifdef USE_WL_TXBF
+	ret = dhd_iovar(dhd, 0, "txbf", (char *)&txbf, sizeof(txbf), NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s Set txbf failed  %d\n", __FUNCTION__, ret));
+
+#endif /* USE_WL_TXBF */
+
+	ret = dhd_iovar(dhd, 0, "scancache", (char *)&scancache_enab, sizeof(scancache_enab), NULL,
+			0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Set scancache failed %d\n", __FUNCTION__, ret));
+	}
+
+#else /* OEM_ANDROID */
+#ifdef BCMDBUS
+	if (dhd->info->clm_path[0] == '\0') {
+		if (strlen(clm_path) > MOD_PARAM_PATHLEN) {
+			DHD_ERROR(("%s clm path exceeds max len\n", __FUNCTION__));
+			ret = BCME_BADLEN;
+			goto done;
+		}
+		strlcpy(dhd->info->clm_path, clm_path, strlen(clm_path)+1);
+	}
+#endif /* BCMDBUS */
+	if ((ret = dhd_apply_default_clm(dhd, dhd->info->clm_path)) < 0) {
+		DHD_ERROR(("%s: CLM set failed. Ignore and continue initialization.\n",
+			__FUNCTION__));
+	}
+
+#if defined(KEEP_ALIVE)
+	if (!(dhd->op_mode &
+		(DHD_FLAG_HOSTAP_MODE | DHD_FLAG_MFG_MODE))) {
+		if ((ret = dhd_keep_alive_onoff(dhd)) < 0)
+			DHD_ERROR(("%s set keeplive failed %d\n",
+			__FUNCTION__, ret));
+	}
+#endif
+
+	/* get a capabilities from firmware */
+	memset(dhd->fw_capabilities, 0, sizeof(dhd->fw_capabilities));
+	ret = dhd_iovar(dhd, 0, "cap", NULL, 0, dhd->fw_capabilities, sizeof(dhd->fw_capabilities),
+			FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: Get Capability failed (error=%d)\n",
+			__FUNCTION__, ret));
+		goto done;
+	}
+#endif  /* OEM_ANDROID */
+
+	ret = dhd_iovar(dhd, 0, "event_log_max_sets", NULL, 0, (char *)&event_log_max_sets,
+		sizeof(event_log_max_sets), FALSE);
+	if (ret == BCME_OK) {
+		dhd->event_log_max_sets = event_log_max_sets;
+	} else {
+		dhd->event_log_max_sets = NUM_EVENT_LOG_SETS;
+	}
+	/* Make sure max_sets is set first with wmb and then sets_queried,
+	 * this will be used during parsing the logsets in the reverse order.
+	 */
+	OSL_SMP_WMB();
+	dhd->event_log_max_sets_queried = TRUE;
+	DHD_INFO(("%s: event_log_max_sets: %d ret: %d\n",
+		__FUNCTION__, dhd->event_log_max_sets, ret));
+#ifdef DHD_BUS_MEM_ACCESS
+	ret = dhd_iovar(dhd, 0, "enable_memuse", (char *)&enable_memuse,
+			sizeof(enable_memuse), iovbuf, sizeof(iovbuf), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: enable_memuse is failed ret=%d\n",
+			__FUNCTION__, ret));
+	} else {
+		DHD_ERROR(("%s: enable_memuse = %d\n",
+			__FUNCTION__, enable_memuse));
+	}
+#endif /* DHD_BUS_MEM_ACCESS */
+
+#ifdef DISABLE_TXBFR
+	ret = dhd_iovar(dhd, 0, "txbf_bfr_cap", (char *)&txbf_bfr_cap, sizeof(txbf_bfr_cap), NULL,
+			0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Clear txbf_bfr_cap failed  %d\n", __FUNCTION__, ret));
+	}
+#endif /* DISABLE_TXBFR */
+
+#ifdef USE_WFA_CERT_CONF
+#ifdef USE_WL_FRAMEBURST
+	 if (sec_get_param_wfa_cert(dhd, SET_PARAM_FRAMEBURST, &frameburst) == BCME_OK) {
+		DHD_ERROR(("%s, read frameburst param=%d\n", __FUNCTION__, frameburst));
+	 }
+#endif /* USE_WL_FRAMEBURST */
+	 g_frameburst = frameburst;
+#endif /* USE_WFA_CERT_CONF */
+#ifdef DISABLE_WL_FRAMEBURST_SOFTAP
+	/* Disable Framebursting for SofAP */
+	if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+		frameburst = 0;
+	}
+#endif /* DISABLE_WL_FRAMEBURST_SOFTAP */
+	/* Set frameburst to value */
+	if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_FAKEFRAG, (char *)&frameburst,
+		sizeof(frameburst), TRUE, 0)) < 0) {
+		DHD_INFO(("%s frameburst not supported  %d\n", __FUNCTION__, ret));
+	}
+#ifdef DHD_SET_FW_HIGHSPEED
+	/* Set ack_ratio */
+	ret = dhd_iovar(dhd, 0, "ack_ratio", (char *)&ack_ratio, sizeof(ack_ratio), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Set ack_ratio failed  %d\n", __FUNCTION__, ret));
+	}
+
+	/* Set ack_ratio_depth */
+	ret = dhd_iovar(dhd, 0, "ack_ratio_depth", (char *)&ack_ratio_depth,
+			sizeof(ack_ratio_depth), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Set ack_ratio_depth failed  %d\n", __FUNCTION__, ret));
+	}
+#endif /* DHD_SET_FW_HIGHSPEED */
+
+	iov_buf = (char*)MALLOC(dhd->osh, WLC_IOCTL_SMLEN);
+	if (iov_buf == NULL) {
+		DHD_ERROR(("failed to allocate %d bytes for iov_buf\n", WLC_IOCTL_SMLEN));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+
+	BCM_REFERENCE(ret2);
+
+#if defined(CUSTOM_AMPDU_BA_WSIZE)
+	/* Set ampdu ba wsize to 64 or 16 */
+#ifdef CUSTOM_AMPDU_BA_WSIZE
+	ampdu_ba_wsize = CUSTOM_AMPDU_BA_WSIZE;
+#endif
+	if (ampdu_ba_wsize != 0) {
+		ret = dhd_iovar(dhd, 0, "ampdu_ba_wsize", (char *)&ampdu_ba_wsize,
+				sizeof(ampdu_ba_wsize), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s Set ampdu_ba_wsize to %d failed  %d\n",
+				__FUNCTION__, ampdu_ba_wsize, ret));
+		}
+	}
+#endif /* CUSTOM_AMPDU_BA_WSIZE */
+
+#if defined(CUSTOM_AMPDU_MPDU)
+	ampdu_mpdu = CUSTOM_AMPDU_MPDU;
+	if (ampdu_mpdu != 0 && (ampdu_mpdu <= ampdu_ba_wsize)) {
+		ret = dhd_iovar(dhd, 0, "ampdu_mpdu", (char *)&ampdu_mpdu, sizeof(ampdu_mpdu),
+				NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s Set ampdu_mpdu to %d failed  %d\n",
+				__FUNCTION__, CUSTOM_AMPDU_MPDU, ret));
+		}
+	}
+#endif /* CUSTOM_AMPDU_MPDU */
+
+#if defined(CUSTOM_AMPDU_RELEASE)
+	ampdu_release = CUSTOM_AMPDU_RELEASE;
+	if (ampdu_release != 0 && (ampdu_release <= ampdu_ba_wsize)) {
+		ret = dhd_iovar(dhd, 0, "ampdu_release", (char *)&ampdu_release,
+				sizeof(ampdu_release), NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s Set ampdu_release to %d failed  %d\n",
+				__FUNCTION__, CUSTOM_AMPDU_RELEASE, ret));
+		}
+	}
+#endif /* CUSTOM_AMPDU_RELEASE */
+
+#if defined(CUSTOM_AMSDU_AGGSF)
+	amsdu_aggsf = CUSTOM_AMSDU_AGGSF;
+	if (amsdu_aggsf != 0) {
+		ret = dhd_iovar(dhd, 0, "amsdu_aggsf", (char *)&amsdu_aggsf, sizeof(amsdu_aggsf),
+				NULL, 0, TRUE);
+		if (ret < 0) {
+			DHD_ERROR(("%s Set amsdu_aggsf to %d failed  %d\n",
+				__FUNCTION__, CUSTOM_AMSDU_AGGSF, ret));
+		}
+	}
+#endif /* CUSTOM_AMSDU_AGGSF */
+
+#if defined(BCMSUP_4WAY_HANDSHAKE)
+	/* Read 4-way handshake requirements */
+	if (dhd_use_idsup == 1) {
+		ret = dhd_iovar(dhd, 0, "sup_wpa", (char *)&sup_wpa, sizeof(sup_wpa),
+				(char *)&iovbuf, sizeof(iovbuf), FALSE);
+		/* sup_wpa iovar returns NOTREADY status on some platforms using modularized
+		 * in-dongle supplicant.
+		 */
+		if (ret >= 0 || ret == BCME_NOTREADY)
+			dhd->fw_4way_handshake = TRUE;
+		DHD_TRACE(("4-way handshake mode is: %d\n", dhd->fw_4way_handshake));
+	}
+#endif /* BCMSUP_4WAY_HANDSHAKE */
+#if defined(SUPPORT_2G_VHT) || defined(SUPPORT_5G_1024QAM_VHT)
+	ret = dhd_iovar(dhd, 0, "vht_features", (char *)&vht_features, sizeof(vht_features),
+			(char *)&vht_features, sizeof(vht_features), FALSE);
+	if (ret < 0) {
+		DHD_ERROR(("%s vht_features get failed %d\n", __FUNCTION__, ret));
+		vht_features = 0;
+	} else {
+#ifdef SUPPORT_2G_VHT
+		vht_features |= 0x3; /* 2G support */
+#endif /* SUPPORT_2G_VHT */
+#ifdef SUPPORT_5G_1024QAM_VHT
+		vht_features |= 0x6; /* 5G 1024 QAM support */
+#endif /* SUPPORT_5G_1024QAM_VHT */
+	}
+	if (vht_features) {
+		ret = dhd_iovar(dhd, 0, "vht_features", (char *)&vht_features, sizeof(vht_features),
+				NULL, 0, TRUE);
+		if (ret < 0) {
+			if (ret == BCME_NOTDOWN) {
+				uint wl_down = 1;
+				ret = dhd_wl_ioctl_cmd(dhd, WLC_DOWN,
+					(char *)&wl_down, sizeof(wl_down), TRUE, 0);
+				DHD_ERROR(("%s vht_features fail WL_DOWN : %d,"
+					" vht_features = 0x%x\n",
+					__FUNCTION__, ret, vht_features));
+
+				ret = dhd_iovar(dhd, 0, "vht_features", (char *)&vht_features,
+						sizeof(vht_features), NULL, 0, TRUE);
+
+				DHD_ERROR(("%s vht_features set. ret --> %d\n", __FUNCTION__, ret));
+			}
+			if (ret != BCME_BADOPTION) {
+				DHD_ERROR(("%s vht_features set failed %d\n", __FUNCTION__, ret));
+			} else {
+				DHD_INFO(("%s vht_features ret(%d) - need to check BANDLOCK\n",
+					__FUNCTION__, ret));
+			}
+		}
+	}
+#endif /* SUPPORT_2G_VHT || SUPPORT_5G_1024QAM_VHT */
+#ifdef DISABLE_11N_PROPRIETARY_RATES
+	ret = dhd_iovar(dhd, 0, "ht_features", (char *)&ht_features, sizeof(ht_features), NULL, 0,
+			TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s ht_features set failed %d\n", __FUNCTION__, ret));
+	}
+#endif /* DISABLE_11N_PROPRIETARY_RATES */
+#if defined(DISABLE_HE_ENAB) || defined(CUSTOM_CONTROL_HE_ENAB)
+#if defined(DISABLE_HE_ENAB)
+	/* XXX DISABLE_HE_ENAB has higher priority than CUSTOM_CONTROL_HE_ENAB */
+	control_he_enab = 0;
+#endif /* DISABLE_HE_ENAB */
+	dhd_control_he_enab(dhd, control_he_enab);
+#endif /* DISABLE_HE_ENAB || CUSTOM_CONTROL_HE_ENAB */
+
+#ifdef CUSTOM_PSPRETEND_THR
+	/* Turn off MPC in AP mode */
+	ret = dhd_iovar(dhd, 0, "pspretend_threshold", (char *)&pspretend_thr,
+			sizeof(pspretend_thr), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s pspretend_threshold for HostAPD failed  %d\n",
+			__FUNCTION__, ret));
+	}
+#endif
+
+	/* XXX Enable firmware key buffering before sent 4-way M4 */
+	ret = dhd_iovar(dhd, 0, "buf_key_b4_m4", (char *)&buf_key_b4_m4, sizeof(buf_key_b4_m4),
+			NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s buf_key_b4_m4 set failed %d\n", __FUNCTION__, ret));
+	}
+#ifdef SUPPORT_SET_CAC
+	ret = dhd_iovar(dhd, 0, "cac", (char *)&cac, sizeof(cac), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s Failed to set cac to %d, %d\n", __FUNCTION__, cac, ret));
+	}
+#endif /* SUPPORT_SET_CAC */
+	/* make up event mask ext message iovar for event larger than 128 */
+	msglen = WL_EVENTING_MASK_EXT_LEN + EVENTMSGS_EXT_STRUCT_SIZE;
+	eventmask_msg = (eventmsgs_ext_t*)MALLOC(dhd->osh, msglen);
+	if (eventmask_msg == NULL) {
+		DHD_ERROR(("failed to allocate %d bytes for event_msg_ext\n", msglen));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+	bzero(eventmask_msg, msglen);
+	eventmask_msg->ver = EVENTMSGS_VER;
+	eventmask_msg->len = ROUNDUP(WLC_E_LAST, NBBY)/NBBY;
+
+	/* Read event_msgs_ext mask */
+	ret = dhd_iovar(dhd, 0, "event_msgs_ext", (char *)eventmask_msg, msglen, iov_buf,
+			WLC_IOCTL_SMLEN, FALSE);
+
+	/* event_msgs_ext must be supported */
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s read event mask ext failed %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+	bcopy(iov_buf, eventmask_msg, msglen);
+	/* make up event mask ext message iovar for event larger than 128 */
+	mask = eventmask_msg->mask;
+
+	/* Setup event_msgs */
+	setbit(mask, WLC_E_SET_SSID);
+	setbit(mask, WLC_E_PRUNE);
+	setbit(mask, WLC_E_AUTH);
+	setbit(mask, WLC_E_AUTH_IND);
+	setbit(mask, WLC_E_ASSOC);
+	setbit(mask, WLC_E_REASSOC);
+	setbit(mask, WLC_E_REASSOC_IND);
+	if (!(dhd->op_mode & DHD_FLAG_IBSS_MODE))
+		setbit(mask, WLC_E_DEAUTH);
+	setbit(mask, WLC_E_DEAUTH_IND);
+	setbit(mask, WLC_E_DISASSOC_IND);
+	setbit(mask, WLC_E_DISASSOC);
+	setbit(mask, WLC_E_JOIN);
+	setbit(mask, WLC_E_START);
+	setbit(mask, WLC_E_ASSOC_IND);
+	setbit(mask, WLC_E_PSK_SUP);
+	setbit(mask, WLC_E_LINK);
+	setbit(mask, WLC_E_MIC_ERROR);
+	setbit(mask, WLC_E_ASSOC_REQ_IE);
+	setbit(mask, WLC_E_ASSOC_RESP_IE);
+#ifdef LIMIT_BORROW
+	setbit(mask, WLC_E_ALLOW_CREDIT_BORROW);
+#endif
+#ifndef WL_CFG80211
+	setbit(mask, WLC_E_PMKID_CACHE);
+//	setbit(mask, WLC_E_TXFAIL); // terence 20181106: remove unnecessary event
+#endif
+	setbit(mask, WLC_E_JOIN_START);
+	setbit(mask, WLC_E_OWE_INFO);
+//	setbit(mask, WLC_E_SCAN_COMPLETE); // terence 20150628: remove redundant event
+#ifdef DHD_DEBUG
+	setbit(mask, WLC_E_SCAN_CONFIRM_IND);
+#endif
+#ifdef PNO_SUPPORT
+	setbit(mask, WLC_E_PFN_NET_FOUND);
+	setbit(mask, WLC_E_PFN_BEST_BATCHING);
+	setbit(mask, WLC_E_PFN_BSSID_NET_FOUND);
+	setbit(mask, WLC_E_PFN_BSSID_NET_LOST);
+#endif /* PNO_SUPPORT */
+	/* enable dongle roaming event */
+#ifdef WL_CFG80211
+#if !defined(ROAM_EVT_DISABLE)
+	setbit(mask, WLC_E_ROAM);
+#endif /* !ROAM_EVT_DISABLE */
+	setbit(mask, WLC_E_BSSID);
+#endif /* WL_CFG80211 */
+#ifdef BCMCCX
+	setbit(mask, WLC_E_ADDTS_IND);
+	setbit(mask, WLC_E_DELTS_IND);
+#endif /* BCMCCX */
+#ifdef WLTDLS
+	setbit(mask, WLC_E_TDLS_PEER_EVENT);
+#endif /* WLTDLS */
+#ifdef RTT_SUPPORT
+	setbit(mask, WLC_E_PROXD);
+#endif /* RTT_SUPPORT */
+#ifdef WL_ESCAN
+	setbit(mask, WLC_E_ESCAN_RESULT);
+#endif /* WL_ESCAN */
+#if !defined(WL_CFG80211) && !defined(OEM_ANDROID)
+	setbit(mask, WLC_E_ESCAN_RESULT);
+#endif
+#ifdef WL_CFG80211
+	setbit(mask, WLC_E_ESCAN_RESULT);
+	setbit(mask, WLC_E_AP_STARTED);
+	setbit(mask, WLC_E_ACTION_FRAME_RX);
+	if (dhd->op_mode & DHD_FLAG_P2P_MODE) {
+		setbit(mask, WLC_E_P2P_DISC_LISTEN_COMPLETE);
+	}
+#endif /* WL_CFG80211 */
+
+#if defined(SHOW_LOGTRACE) && defined(LOGTRACE_FROM_FILE)
+	if (dhd_logtrace_from_file(dhd)) {
+		setbit(mask, WLC_E_TRACE);
+	} else {
+		clrbit(mask, WLC_E_TRACE);
+	}
+#elif defined(SHOW_LOGTRACE)
+	setbit(mask, WLC_E_TRACE);
+#else
+	clrbit(mask, WLC_E_TRACE);
+#endif /* defined(SHOW_LOGTRACE) && defined(LOGTRACE_FROM_FILE) */
+
+	setbit(mask, WLC_E_CSA_COMPLETE_IND);
+#ifdef DHD_WMF
+	setbit(mask, WLC_E_PSTA_PRIMARY_INTF_IND);
+#endif
+#ifdef CUSTOM_EVENT_PM_WAKE
+	setbit(mask, WLC_E_EXCESS_PM_WAKE_EVENT);
+#endif	/* CUSTOM_EVENT_PM_WAKE */
+#ifdef DHD_LOSSLESS_ROAMING
+	setbit(mask, WLC_E_ROAM_PREP);
+#endif
+	/* nan events */
+	setbit(mask, WLC_E_NAN);
+#if defined(PCIE_FULL_DONGLE) && defined(DHD_LOSSLESS_ROAMING)
+	dhd_update_flow_prio_map(dhd, DHD_FLOW_PRIO_LLR_MAP);
+#endif /* defined(PCIE_FULL_DONGLE) && defined(DHD_LOSSLESS_ROAMING) */
+
+#if defined(BCMPCIE) && defined(EAPOL_PKT_PRIO)
+	dhd_update_flow_prio_map(dhd, DHD_FLOW_PRIO_LLR_MAP);
+#endif /* defined(BCMPCIE) && defined(EAPOL_PKT_PRIO) */
+
+#if defined(BCMSDIO) && defined(DHD_LOSSLESS_ROAMING)
+	dhd_update_sdio_data_prio_map(dhd);
+#endif /* BCMSDIO && DHD_LOSSLESS_ROAMING */
+
+#ifdef RSSI_MONITOR_SUPPORT
+	setbit(mask, WLC_E_RSSI_LQM);
+#endif /* RSSI_MONITOR_SUPPORT */
+#ifdef GSCAN_SUPPORT
+	setbit(mask, WLC_E_PFN_GSCAN_FULL_RESULT);
+	setbit(mask, WLC_E_PFN_SCAN_COMPLETE);
+	setbit(mask, WLC_E_PFN_SSID_EXT);
+	setbit(mask, WLC_E_ROAM_EXP_EVENT);
+#endif /* GSCAN_SUPPORT */
+	setbit(mask, WLC_E_RSSI_LQM);
+#ifdef DBG_PKT_MON
+	setbit(mask, WLC_E_ROAM_PREP);
+#endif /* DBG_PKT_MON */
+#ifdef BCM_ROUTER_DHD
+	setbit(mask, WLC_E_DPSTA_INTF_IND);
+#endif /* BCM_ROUTER_DHD */
+	setbit(mask, WLC_E_SLOTTED_BSS_PEER_OP);
+#ifdef WL_BCNRECV
+	setbit(mask, WLC_E_BCNRECV_ABORTED);
+#endif /* WL_BCNRECV */
+#ifdef WL_MBO
+	setbit(mask, WLC_E_MBO);
+#endif /* WL_MBO */
+#ifdef WL_CAC_TS
+	setbit(mask, WLC_E_ADDTS_IND);
+	setbit(mask, WLC_E_DELTS_IND);
+#endif /* WL_BCNRECV */
+	setbit(mask, WLC_E_COUNTRY_CODE_CHANGED);
+#ifdef CSI_SUPPORT
+	setbit(mask, WLC_E_CSI);
+#endif /* CSI_SUPPORT */
+
+	/* Write updated Event mask */
+	eventmask_msg->ver = EVENTMSGS_VER;
+	eventmask_msg->command = EVENTMSGS_SET_MASK;
+	eventmask_msg->len = WL_EVENTING_MASK_EXT_LEN;
+	ret = dhd_iovar(dhd, 0, "event_msgs_ext", (char *)eventmask_msg, msglen, NULL, 0,
+			TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s write event mask ext failed %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+#if defined(DHD_8021X_DUMP) && defined(SHOW_LOGTRACE)
+	/* Enabling event log trace for EAP events */
+	el_tag = (wl_el_tag_params_t *)MALLOC(dhd->osh, sizeof(wl_el_tag_params_t));
+	if (el_tag == NULL) {
+		DHD_ERROR(("failed to allocate %d bytes for event_msg_ext\n",
+				(int)sizeof(wl_el_tag_params_t)));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+	el_tag->tag = EVENT_LOG_TAG_4WAYHANDSHAKE;
+	el_tag->set = 1;
+	el_tag->flags = EVENT_LOG_TAG_FLAG_LOG;
+	ret = dhd_iovar(dhd, 0, "event_log_tag_control", (char *)el_tag, sizeof(*el_tag), NULL,
+			0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set event_log_tag_control fail %d\n", __FUNCTION__, ret));
+	}
+#endif /* DHD_8021X_DUMP */
+
+#ifdef DEBUGABILITY
+	/* Enabling event log trace for roam stats events */
+	el_tag = (wl_el_tag_params_t *)MALLOC(dhd->osh, sizeof(wl_el_tag_params_t));
+	if (el_tag == NULL) {
+		DHD_ERROR(("failed to allocate %d bytes for event_msg_ext\n",
+			(int)sizeof(wl_el_tag_params_t)));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+	el_tag->tag = EVENT_LOG_TAG_ROAM_ENHANCED_LOG;
+	el_tag->set = 1;
+	el_tag->flags = EVENT_LOG_TAG_FLAG_LOG;
+	ret = dhd_iovar(dhd, 0, "event_log_tag_control", (char *)el_tag, sizeof(*el_tag), NULL,
+			0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s set event_log_tag_control fail %d\n", __FUNCTION__, ret));
+	}
+#endif /* DEBUGABILITY */
+
+#ifdef DHD_RANDMAC_LOGGING
+	if (FW_SUPPORTED((dhd), event_log)) {
+		if (dhd_iovar(dhd, 0, "privacy_mask", (char *)&privacy_mask, sizeof(privacy_mask),
+			NULL, 0, TRUE) < 0) {
+			DHD_ERROR(("failed to set privacy mask\n"));
+		}
+	} else {
+		/* Don't enable feature to prevent macaddr print in clr text */
+		DHD_ERROR(("skip privacy_mask set. event_log not enabled\n"));
+	}
+#endif /* DHD_RANDMAC_LOGGING */
+
+#ifdef OEM_ANDROID
+	dhd_wl_ioctl_cmd(dhd, WLC_SET_SCAN_CHANNEL_TIME, (char *)&scan_assoc_time,
+			sizeof(scan_assoc_time), TRUE, 0);
+	dhd_wl_ioctl_cmd(dhd, WLC_SET_SCAN_UNASSOC_TIME, (char *)&scan_unassoc_time,
+			sizeof(scan_unassoc_time), TRUE, 0);
+	dhd_wl_ioctl_cmd(dhd, WLC_SET_SCAN_PASSIVE_TIME, (char *)&scan_passive_time,
+			sizeof(scan_passive_time), TRUE, 0);
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	DHD_INFO(("arp_enable:%d arp_ol:%d\n",
+		dhd->arpoe_enable, dhd->arpol_configured));
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#ifdef PKT_FILTER_SUPPORT
+	/* Setup default defintions for pktfilter , enable in suspend */
+	if (dhd_master_mode) {
+		dhd->pktfilter_count = 6;
+		dhd->pktfilter[DHD_BROADCAST_FILTER_NUM] = NULL;
+		if (!FW_SUPPORTED(dhd, pf6)) {
+			dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = NULL;
+			dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = NULL;
+		} else {
+			/* Immediately pkt filter TYPE 6 Discard IPv4/IPv6 Multicast Packet */
+			dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = DISCARD_IPV4_MCAST;
+			dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = DISCARD_IPV6_MCAST;
+		}
+		/* apply APP pktfilter */
+		dhd->pktfilter[DHD_ARP_FILTER_NUM] = "105 0 0 12 0xFFFF 0x0806";
+
+#ifdef BLOCK_IPV6_PACKET
+		/* Setup filter to allow only IPv4 unicast frames */
+		dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = "100 0 0 0 "
+			HEX_PREF_STR UNI_FILTER_STR ZERO_ADDR_STR ETHER_TYPE_STR IPV6_FILTER_STR
+			" "
+			HEX_PREF_STR ZERO_ADDR_STR ZERO_ADDR_STR ETHER_TYPE_STR ZERO_TYPE_STR;
+#else
+		/* Setup filter to allow only unicast */
+		dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = "100 0 0 0 0x01 0x00";
+#endif /* BLOCK_IPV6_PACKET */
+
+#ifdef PASS_IPV4_SUSPEND
+		/* XXX customer want to get IPv4 multicast packets */
+		dhd->pktfilter[DHD_MDNS_FILTER_NUM] = "104 0 0 0 0xFFFFFF 0x01005E";
+#else
+		/* Add filter to pass multicastDNS packet and NOT filter out as Broadcast */
+		dhd->pktfilter[DHD_MDNS_FILTER_NUM] = NULL;
+#endif /* PASS_IPV4_SUSPEND */
+		if (FW_SUPPORTED(dhd, pf6)) {
+			/* Immediately pkt filter TYPE 6 Dicard Broadcast IP packet */
+			dhd->pktfilter[DHD_IP4BCAST_DROP_FILTER_NUM] = DISCARD_IPV4_BCAST;
+			dhd->pktfilter_count = 8;
+		}
+
+#ifdef GAN_LITE_NAT_KEEPALIVE_FILTER
+		dhd->pktfilter_count = 4;
+		/* Setup filter to block broadcast and NAT Keepalive packets */
+		/* discard all broadcast packets */
+		dhd->pktfilter[DHD_UNICAST_FILTER_NUM] = "100 0 0 0 0xffffff 0xffffff";
+		/* discard NAT Keepalive packets */
+		dhd->pktfilter[DHD_BROADCAST_FILTER_NUM] = "102 0 0 36 0xffffffff 0x11940009";
+		/* discard NAT Keepalive packets */
+		dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = "104 0 0 38 0xffffffff 0x11940009";
+		dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = NULL;
+#endif /* GAN_LITE_NAT_KEEPALIVE_FILTER */
+	} else
+		dhd_conf_discard_pkt_filter(dhd);
+	dhd_conf_add_pkt_filter(dhd);
+
+#if defined(SOFTAP)
+	if (ap_fw_loaded) {
+		/* XXX Andrey: fo SOFTAP disable pkt filters (if there were any )  */
+		dhd_enable_packet_filter(0, dhd);
+	}
+#endif /* defined(SOFTAP) */
+	dhd_set_packet_filter(dhd);
+#endif /* PKT_FILTER_SUPPORT */
+#ifdef DISABLE_11N
+	ret = dhd_iovar(dhd, 0, "nmode", (char *)&nmode, sizeof(nmode), NULL, 0, TRUE);
+	if (ret < 0)
+		DHD_ERROR(("%s wl nmode 0 failed %d\n", __FUNCTION__, ret));
+#endif /* DISABLE_11N */
+
+#ifdef ENABLE_BCN_LI_BCN_WAKEUP
+	ret = dhd_iovar(dhd, 0, "bcn_li_bcn", (char *)&bcn_li_bcn, sizeof(bcn_li_bcn),
+			NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s: set bcn_li_bcn failed %d\n", __FUNCTION__, ret));
+	}
+#endif /* ENABLE_BCN_LI_BCN_WAKEUP */
+#ifdef AMPDU_VO_ENABLE
+	/* XXX: Enabling VO AMPDU to reduce FER */
+	tid.tid = PRIO_8021D_VO; /* Enable TID(6) for voice */
+	tid.enable = TRUE;
+	ret = dhd_iovar(dhd, 0, "ampdu_tid", (char *)&tid, sizeof(tid), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s ampdu_tid %d\n", __FUNCTION__, ret));
+	}
+
+	tid.tid = PRIO_8021D_NC; /* Enable TID(7) for voice */
+	tid.enable = TRUE;
+	ret = dhd_iovar(dhd, 0, "ampdu_tid", (char *)&tid, sizeof(tid), NULL, 0, TRUE);
+	if (ret < 0) {
+		DHD_ERROR(("%s ampdu_tid %d\n", __FUNCTION__, ret));
+	}
+#endif
+#if defined(SOFTAP_TPUT_ENHANCE)
+	if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE) {
+#if defined(BCMSDIO)
+		dhd_bus_setidletime(dhd, (int)100);
+#endif /* BCMSDIO */
+#ifdef DHDTCPACK_SUPPRESS
+		dhd_tcpack_suppress_set(dhd, TCPACK_SUP_OFF);
+#endif
+
+#if defined(BCMSDIO)
+		memset(buf, 0, sizeof(buf));
+		ret = dhd_iovar(dhd, 0, "bus:txglom_auto_control", NULL, 0, buf, sizeof(buf),
+				FALSE);
+		if (ret < 0) {
+			glom = 0;
+			ret = dhd_iovar(dhd, 0, "bus:txglom", (char *)&glom, sizeof(glom),
+					NULL, 0, TRUE);
+			if (ret < 0) {
+				DHD_ERROR(("%s bus:txglom failed %d\n", __FUNCTION__, ret));
+			}
+		} else {
+			if (buf[0] == 0) {
+				glom = 1;
+				ret = dhd_iovar(dhd, 0, "bus:txglom_auto_control", (char *)&glom,
+						sizeof(glom), NULL, 0, TRUE);
+				if (ret < 0) {
+					DHD_ERROR(("%s bus:txglom_auto_control failed %d\n",
+						__FUNCTION__, ret));
+				}
+			}
+		}
+#endif /* BCMSDIO */
+	}
+#endif /* SOFTAP_TPUT_ENHANCE */
+	/* query for 'clmver' to get clm version info from firmware */
+	bzero(buf, sizeof(buf));
+	ret = dhd_iovar(dhd, 0, "clmver", NULL, 0, buf, sizeof(buf), FALSE);
+	if (ret < 0)
+		DHD_ERROR(("%s clmver failed %d\n", __FUNCTION__, ret));
+	else {
+		char *ver_temp_buf = NULL, *ver_date_buf = NULL;
+		int len;
+
+		if ((ver_temp_buf = bcmstrstr(buf, "Data:")) == NULL) {
+			DHD_ERROR(("Couldn't find \"Data:\"\n"));
+		} else {
+			ver_date_buf = bcmstrstr(buf, "Creation:");
+			ptr = (ver_temp_buf + strlen("Data:"));
+			if ((ver_temp_buf = bcmstrtok(&ptr, "\n", 0)) == NULL) {
+				DHD_ERROR(("Couldn't find New line character\n"));
+			} else {
+				memset(clm_version, 0, CLM_VER_STR_LEN);
+				len = snprintf(clm_version, CLM_VER_STR_LEN - 1, "%s", ver_temp_buf);
+				if (ver_date_buf) {
+					ptr = (ver_date_buf + strlen("Creation:"));
+					ver_date_buf = bcmstrtok(&ptr, "\n", 0);
+					if (ver_date_buf)
+						snprintf(clm_version+len, CLM_VER_STR_LEN-1-len,
+							" (%s)", ver_date_buf);
+				}
+				DHD_INFO(("CLM version = %s\n", clm_version));
+			}
+		}
+
+		if (strlen(clm_version)) {
+			DHD_INFO(("CLM version = %s\n", clm_version));
+		} else {
+			DHD_ERROR(("Couldn't find CLM version!\n"));
+		}
+	}
+	dhd_set_version_info(dhd, fw_version);
+
+#ifdef WRITE_WLANINFO
+	do {
+		uint len = MAX_NVRAMBUF_SIZE;
+		char *memblock = NULL;
+
+		dhd_get_download_buffer(dhd, dhd->info->nv_path, NVRAM, &memblock, (int *)&len);
+		if ((len <= 0 || len > MAX_NVRAMBUF_SIZE)) {
+			DHD_ERROR(("Can't open nvram info, skip save wifiver info\n"));
+			break;
+		}
+
+		sec_save_wlinfo(fw_version, EPI_VERSION_STR, memblock, clm_version);
+		if (memblock) {
+			dhd_free_download_buffer(dhd, memblock, len);
+		}
+	} while (0);
+#endif /* WRITE_WLANINFO */
+
+#endif /* defined(OEM_ANDROID) */
+#ifdef GEN_SOFTAP_INFO_FILE
+	sec_save_softap_info();
+#endif /* GEN_SOFTAP_INFO_FILE */
+
+#if defined(BCMSDIO)
+	dhd_txglom_enable(dhd, dhd->conf->bus_rxglom);
+#endif /* defined(BCMSDIO) */
+
+#if defined(BCMSDIO) || defined(BCMDBUS)
+#ifdef PROP_TXSTATUS
+	if (disable_proptx ||
+#ifdef PROP_TXSTATUS_VSDB
+		/* enable WLFC only if the firmware is VSDB when it is in STA mode */
+		(dhd->op_mode != DHD_FLAG_HOSTAP_MODE &&
+		 dhd->op_mode != DHD_FLAG_IBSS_MODE) ||
+#endif /* PROP_TXSTATUS_VSDB */
+		FALSE) {
+		wlfc_enable = FALSE;
+	}
+	ret = dhd_conf_get_disable_proptx(dhd);
+	if (ret == 0){
+		disable_proptx = 0;
+		wlfc_enable = TRUE;
+	} else if (ret >= 1) {
+		disable_proptx = 1;
+		wlfc_enable = FALSE;
+		/* terence 20161229: we should set ampdu_hostreorder=0 when disable_proptx=1 */
+		hostreorder = 0;
+	}
+
+#if defined(PROP_TXSTATUS)
+#ifdef USE_WFA_CERT_CONF
+	if (sec_get_param_wfa_cert(dhd, SET_PARAM_PROPTX, &proptx) == BCME_OK) {
+		DHD_ERROR(("%s , read proptx param=%d\n", __FUNCTION__, proptx));
+		wlfc_enable = proptx;
+	}
+#endif /* USE_WFA_CERT_CONF */
+#endif /* PROP_TXSTATUS */
+
+#ifndef DISABLE_11N
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_DOWN, (char *)&wl_down, sizeof(wl_down), TRUE, 0);
+	ret2 = dhd_iovar(dhd, 0, "ampdu_hostreorder", (char *)&hostreorder, sizeof(hostreorder),
+			NULL, 0, TRUE);
+	if (ret2 < 0) {
+		DHD_ERROR(("%s wl ampdu_hostreorder failed %d\n", __FUNCTION__, ret2));
+		if (ret2 != BCME_UNSUPPORTED)
+			ret = ret2;
+
+		if (ret == BCME_NOTDOWN) {
+			uint wl_down = 1;
+			ret2 = dhd_wl_ioctl_cmd(dhd, WLC_DOWN, (char *)&wl_down,
+				sizeof(wl_down), TRUE, 0);
+			DHD_ERROR(("%s ampdu_hostreorder fail WL_DOWN : %d, hostreorder :%d\n",
+				__FUNCTION__, ret2, hostreorder));
+
+			ret2 = dhd_iovar(dhd, 0, "ampdu_hostreorder", (char *)&hostreorder,
+					sizeof(hostreorder), NULL, 0, TRUE);
+			DHD_ERROR(("%s wl ampdu_hostreorder. ret --> %d\n", __FUNCTION__, ret2));
+			if (ret2 != BCME_UNSUPPORTED)
+					ret = ret2;
+		}
+		if (ret2 != BCME_OK)
+			hostreorder = 0;
+	}
+#endif /* DISABLE_11N */
+
+#ifdef READ_CONFIG_FROM_FILE
+	dhd_preinit_config(dhd, 0, CONFIG_BCMDHD_CONFIG_PATH);
+#endif /* READ_CONFIG_FROM_FILE */
+
+	if (wlfc_enable) {
+		dhd_wlfc_init(dhd);
+		/* terence 20161229: enable ampdu_hostreorder if tlv enabled */
+		dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "ampdu_hostreorder", 1, 0, TRUE);
+	}
+#ifndef DISABLE_11N
+	else if (hostreorder)
+		dhd_wlfc_hostreorder_init(dhd);
+#endif /* DISABLE_11N */
+#else
+	/* terence 20161229: disable ampdu_hostreorder if PROP_TXSTATUS not defined */
+	printf("%s: not define PROP_TXSTATUS\n", __FUNCTION__);
+	dhd_conf_set_intiovar(dhd, 0, WLC_SET_VAR, "ampdu_hostreorder", 0, 0, TRUE);
+#endif /* PROP_TXSTATUS */
+#endif /* BCMSDIO || BCMDBUS */
+
+#if defined(SYNA_SAR_CUSTOMER_PARAMETER) && defined(READ_CONFIG_FROM_FILE)
+	dhd_sar_reset_parameter();
+	dhd_preinit_config(dhd, 0, config_sar_path);
+#endif /* SYNA_SAR_CUSTOMER_PARAMETER && READ_CONFIG_FROM_FILE */
+
+#ifndef PCIE_FULL_DONGLE
+	/* For FD we need all the packets at DHD to handle intra-BSS forwarding */
+	if (FW_SUPPORTED(dhd, ap)) {
+		wl_ap_isolate = AP_ISOLATE_SENDUP_ALL;
+		ret = dhd_iovar(dhd, 0, "ap_isolate", (char *)&wl_ap_isolate, sizeof(wl_ap_isolate),
+				NULL, 0, TRUE);
+		if (ret < 0)
+			DHD_ERROR(("%s failed %d\n", __FUNCTION__, ret));
+	}
+#endif /* PCIE_FULL_DONGLE */
+#ifdef PNO_SUPPORT
+	if (!dhd->pno_state) {
+		dhd_pno_init(dhd);
+	}
+#endif
+
+#ifdef RTT_SUPPORT
+	if (dhd->rtt_state) {
+		ret = dhd_rtt_init(dhd);
+		if (ret < 0) {
+			DHD_ERROR(("%s failed to initialize RTT\n", __FUNCTION__));
+		}
+	}
+#endif
+#ifdef FILTER_IE
+	/* Failure to configure filter IE is not a fatal error, ignore it. */
+	if (FW_SUPPORTED(dhd, fie) &&
+		!(dhd->op_mode & (DHD_FLAG_HOSTAP_MODE | DHD_FLAG_MFG_MODE))) {
+		dhd_read_from_file(dhd);
+	}
+#endif /* FILTER_IE */
+#ifdef WL11U
+	dhd_interworking_enable(dhd);
+#endif /* WL11U */
+
+#ifdef NDO_CONFIG_SUPPORT
+	dhd->ndo_enable = FALSE;
+	dhd->ndo_host_ip_overflow = FALSE;
+	dhd->ndo_max_host_ip = NDO_MAX_HOST_IP_ENTRIES;
+#endif /* NDO_CONFIG_SUPPORT */
+
+	/* ND offload version supported */
+	dhd->ndo_version = dhd_ndo_get_version(dhd);
+	if (dhd->ndo_version > 0) {
+		DHD_INFO(("%s: ndo version %d\n", __FUNCTION__, dhd->ndo_version));
+
+#ifdef NDO_CONFIG_SUPPORT
+		/* enable Unsolicited NA filter */
+		ret = dhd_ndo_unsolicited_na_filter_enable(dhd, 1);
+		if (ret < 0) {
+			DHD_ERROR(("%s failed to enable Unsolicited NA filter\n", __FUNCTION__));
+		}
+#endif /* NDO_CONFIG_SUPPORT */
+	}
+
+	/* check dongle supports wbtext (product policy) or not */
+	dhd->wbtext_support = FALSE;
+	if (dhd_wl_ioctl_get_intiovar(dhd, "wnm_bsstrans_resp", &wnm_bsstrans_resp,
+			WLC_GET_VAR, FALSE, 0) != BCME_OK) {
+		DHD_ERROR(("failed to get wnm_bsstrans_resp\n"));
+	}
+	dhd->wbtext_policy = wnm_bsstrans_resp;
+	if (dhd->wbtext_policy == WL_BSSTRANS_POLICY_PRODUCT_WBTEXT) {
+		dhd->wbtext_support = TRUE;
+	}
+#ifndef WBTEXT
+	/* driver can turn off wbtext feature through makefile */
+	if (dhd->wbtext_support) {
+		if (dhd_wl_ioctl_set_intiovar(dhd, "wnm_bsstrans_resp",
+				WL_BSSTRANS_POLICY_ROAM_ALWAYS,
+				WLC_SET_VAR, FALSE, 0) != BCME_OK) {
+			DHD_ERROR(("failed to disable WBTEXT\n"));
+		}
+	}
+#endif /* !WBTEXT */
+
+#ifdef DHD_NON_DMA_M2M_CORRUPTION
+	/* check pcie non dma loopback */
+	if (dhd->op_mode == DHD_FLAG_MFG_MODE &&
+		(dhd_bus_dmaxfer_lpbk(dhd, M2M_NON_DMA_LPBK) < 0)) {
+			goto done;
+	}
+#endif /* DHD_NON_DMA_M2M_CORRUPTION */
+
+	/* WNM capabilities */
+	wnm_cap = 0
+#ifdef WL11U
+		| WL_WNM_BSSTRANS | WL_WNM_NOTIF
+#endif
+#ifdef WBTEXT
+		| WL_WNM_BSSTRANS | WL_WNM_MAXIDLE
+#endif
+		;
+#if defined(WL_MBO) && defined(WL_OCE)
+	if (FW_SUPPORTED(dhd, estm)) {
+		wnm_cap |= WL_WNM_ESTM;
+	}
+#endif /* WL_MBO && WL_OCE */
+	if (dhd_iovar(dhd, 0, "wnm", (char *)&wnm_cap, sizeof(wnm_cap), NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("failed to set WNM capabilities\n"));
+	}
+
+#ifdef CUSTOM_ASSOC_TIMEOUT
+	/* set recreate_bi_timeout to increase assoc timeout :
+	* 20 * 100TU * 1024 / 1000 = 2 secs
+	* (beacon wait time = recreate_bi_timeout * beacon_period * 1024 / 1000)
+	*/
+	if (dhd_wl_ioctl_set_intiovar(dhd, "recreate_bi_timeout",
+			CUSTOM_ASSOC_TIMEOUT,
+			WLC_SET_VAR, TRUE, 0) != BCME_OK) {
+		DHD_ERROR(("failed to set assoc timeout\n"));
+	}
+#endif /* CUSTOM_ASSOC_TIMEOUT */
+
+#if defined(WBTEXT) && defined(WBTEXT_BTMDELTA)
+	if (dhd_iovar(dhd, 0, "wnm_btmdelta", (char *)&btmdelta, sizeof(btmdelta),
+			NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("failed to set BTM delta\n"));
+	}
+#endif /* WBTEXT && WBTEXT_BTMDELTA */
+#if defined(WBTEXT) && defined(RRM_BCNREQ_MAX_CHAN_TIME)
+	if (dhd_iovar(dhd, 0, "rrm_bcn_req_thrtl_win",
+		(char *)&rrm_bcn_req_thrtl_win, sizeof(rrm_bcn_req_thrtl_win),
+		NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("failed to set RRM BCN request thrtl_win\n"));
+	}
+	if (dhd_iovar(dhd, 0, "rrm_bcn_req_max_off_chan_time",
+		(char *)&rrm_bcn_req_max_off_chan_time, sizeof(rrm_bcn_req_max_off_chan_time),
+		NULL, 0, TRUE) < 0) {
+		DHD_ERROR(("failed to set RRM BCN Request max_off_chan_time\n"));
+	}
+#endif /* WBTEXT && RRM_BCNREQ_MAX_CHAN_TIME */
+
+#ifdef WL_MONITOR
+#ifdef HOST_RADIOTAP_CONV
+	/* 'Wl monitor' IOVAR is fired to check whether the FW supports radiotap conversion or not.
+	 * This is indicated through MSB(1<<31) bit, based on which host radiotap conversion
+	 * will be enabled or disabled.
+	 * 0 - Host supports Radiotap conversion.
+	 * 1 - FW supports Radiotap conversion.
+	 */
+	bcm_mkiovar("monitor", (char *)&monitor, sizeof(monitor), iovbuf, sizeof(iovbuf));
+	if ((ret2 = dhd_wl_ioctl_cmd(dhd, WLC_GET_MONITOR, iovbuf,
+		sizeof(iovbuf), FALSE, 0)) == 0) {
+		memcpy(&monitor, iovbuf, sizeof(monitor));
+		dhdinfo->host_radiotap_conv = (monitor & HOST_RADIOTAP_CONV_BIT) ? TRUE : FALSE;
+	} else {
+		DHD_ERROR(("%s Failed to get monitor mode, err %d\n",
+			__FUNCTION__, ret2));
+	}
+#endif /* HOST_RADIOTAP_CONV */
+	if (FW_SUPPORTED(dhd, monitor)) {
+		dhd->monitor_enable = TRUE;
+		DHD_ERROR(("%s: Monitor mode is enabled in FW cap\n", __FUNCTION__));
+	} else {
+		dhd->monitor_enable = FALSE;
+		DHD_ERROR(("%s: Monitor mode is not enabled in FW cap\n", __FUNCTION__));
+	}
+#endif /* WL_MONITOR */
+
+	/* store the preserve log set numbers */
+	if (dhd_get_preserve_log_numbers(dhd, &dhd->logset_prsrv_mask)
+			!= BCME_OK) {
+		DHD_INFO(("%s: Failed to get preserve log # !\n", __FUNCTION__));
+	}
+
+	if (FW_SUPPORTED(dhd, ecounters) && enable_ecounter) {
+		dhd_ecounter_configure(dhd, TRUE);
+	}
+
+#ifndef OEM_ANDROID
+	/* For non-android FC modular builds, override firmware preinited values */
+	dhd_override_fwprenit(dhd);
+#endif /* !OEM_ANDROID */
+
+	dhd_set_bandlock(dhd);
+
+#ifdef WL_UWB_COEX
+#ifdef WL_UWB_COEX_DEF_ENABLE
+	ret = wl_cfg_uwb_coex_enable(dhd_linux_get_primary_netdev(dhd),
+		TRUE, UWB_COEX_CH_MIN, UWB_COEX_CH_MAX);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("Failed to set UWB Coex (%d)\n", ret));
+	}
+#endif /* WL_UWB_COEX_DEF_ENABLE */
+#endif /* WL_UWB_COEX */
+
+	if (dhd_iovar(dhd, 0, "bus:d3_hostwake_delay", (char *)&d3_hostwake_delay,
+		sizeof(d3_hostwake_delay), NULL, 0, TRUE) < 0) {
+		DHD_INFO(("%s: d3_hostwake_delay IOVAR not present, proceed\n", __FUNCTION__));
+	} else {
+		DHD_INFO(("%s: d3_hostwake_delay enabled\n", __FUNCTION__));
+	}
+
+#if defined(BCMSDIO)
+	/* check if srmemsize consitant at fw and dhd side */
+	dhd_bus_check_srmemsize(dhd);
+#endif /* BCMSDIO */
+
+	dhd_conf_postinit_ioctls(dhd);
+done:
+
+	if (eventmask_msg) {
+		MFREE(dhd->osh, eventmask_msg, msglen);
+	}
+	if (iov_buf) {
+		MFREE(dhd->osh, iov_buf, WLC_IOCTL_SMLEN);
+	}
+#if (defined(DHD_8021X_DUMP) && defined(SHOW_LOGTRACE)) || defined(DEBUGABILITY)
+	if (el_tag) {
+		MFREE(dhd->osh, el_tag, sizeof(wl_el_tag_params_t));
+	}
+#endif /* DHD_8021X_DUMP */
+	return ret;
+}
+
+#if defined(BCMPCIE)
+/* Deafult enable preinit optimisation */
+#define DHD_PREINIT_OPTIMISATION
+#endif /* BCMPCIE */
+
+int
+dhd_preinit_ioctls(dhd_pub_t *dhd)
+{
+	int ret = 0;
+
+#ifdef DHD_PREINIT_OPTIMISATION
+	int preinit_status = 0;
+	ret = dhd_iovar(dhd, 0, "preinit_status", NULL, 0, (char *)&preinit_status,
+		sizeof(preinit_status), FALSE);
+
+	if (ret == BCME_OK) {
+		DHD_ERROR(("%s: preinit_status IOVAR present, use optimised preinit\n",
+			__FUNCTION__));
+		dhd->fw_preinit = TRUE;
+		ret = dhd_optimised_preinit_ioctls(dhd);
+	} else if (ret == BCME_UNSUPPORTED) {
+		DHD_ERROR(("%s: preinit_status IOVAR not supported, use legacy preinit\n",
+			__FUNCTION__));
+		dhd->fw_preinit = FALSE;
+		ret = dhd_legacy_preinit_ioctls(dhd);
+	} else {
+		DHD_ERROR(("%s: preinit_status IOVAR returned err(%d), ABORT\n",
+			__FUNCTION__, ret));
+	}
+#else
+	dhd->fw_preinit = FALSE;
+	ret = dhd_legacy_preinit_ioctls(dhd);
+#endif /* DHD_PREINIT_OPTIMISATION */
+
+	if (!ret && dhd_query_bus_erros(dhd)) {
+		DHD_ERROR(("%s: retrun error due to query errors\n", __FUNCTION__));
+		ret = BCME_ERROR;
+	}
+
+	return ret;
+}
+
+int
+dhd_getiovar(dhd_pub_t *pub, int ifidx, char *name, char *cmd_buf,
+	uint cmd_len, char **resptr, uint resp_len)
+{
+	int len = resp_len;
+	int ret;
+	char *buf = *resptr;
+	wl_ioctl_t ioc;
+	if (resp_len > WLC_IOCTL_MAXLEN)
+		return BCME_BADARG;
+
+	memset(buf, 0, resp_len);
+
+	ret = bcm_mkiovar(name, cmd_buf, cmd_len, buf, len);
+	if (ret == 0) {
+		return BCME_BUFTOOSHORT;
+	}
+
+	memset(&ioc, 0, sizeof(ioc));
+
+	ioc.cmd = WLC_GET_VAR;
+	ioc.buf = buf;
+	ioc.len = len;
+	ioc.set = 0;
+
+	ret = dhd_wl_ioctl(pub, ifidx, &ioc, ioc.buf, ioc.len);
+
+	return ret;
+}
+
+int dhd_change_mtu(dhd_pub_t *dhdp, int new_mtu, int ifidx)
+{
+	struct dhd_info *dhd = dhdp->info;
+	struct net_device *dev = NULL;
+
+	ASSERT(dhd && dhd->iflist[ifidx]);
+	dev = dhd->iflist[ifidx]->net;
+	ASSERT(dev);
+
+#ifndef DHD_TPUT_PATCH
+	if (netif_running(dev)) {
+		DHD_ERROR(("%s: Must be down to change its MTU\n", dev->name));
+		return BCME_NOTDOWN;
+	}
+#endif
+
+#define DHD_MIN_MTU 1500
+#define DHD_MAX_MTU 1752
+
+	if ((new_mtu < DHD_MIN_MTU) || (new_mtu > DHD_MAX_MTU)) {
+		DHD_ERROR(("%s: MTU size %d is invalid.\n", __FUNCTION__, new_mtu));
+		return BCME_BADARG;
+	}
+
+	dev->mtu = new_mtu;
+	return 0;
+}
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+static int dhd_wait_for_file_dump(dhd_pub_t *dhdp)
+{
+	int ret = BCME_OK;
+	struct net_device *primary_ndev;
+#ifdef WL_CFG80211
+	struct bcm_cfg80211 *cfg;
+#endif /* WL_CFG80211 */
+	unsigned long flags = 0;
+	primary_ndev = dhd_linux_get_primary_netdev(dhdp);
+
+	if (!primary_ndev) {
+		DHD_ERROR(("%s: Cannot find primary netdev\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+#ifdef WL_CFG80211
+	cfg = wl_get_cfg(primary_ndev);
+
+	if (!cfg) {
+		DHD_ERROR(("%s: Cannot find cfg\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+#endif /* WL_CFG80211 */
+
+	if (dhdp->stop_in_progress) {
+		DHD_ERROR(("%s: dhd_stop in progress\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_BUS_BUSY_CLEAR_IN_HALDUMP(dhdp);
+		dhd_os_busbusy_wake(dhdp);
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s: bus is down! can't collect log dump. \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	DHD_BUS_BUSY_SET_IN_HALDUMP(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+#ifdef WL_CFG80211
+	DHD_OS_WAKE_LOCK(dhdp);
+	/* check for hal started and only then send event if not clear dump state here */
+	if (wl_cfg80211_is_hal_started(cfg)) {
+		int timeleft = 0;
+
+		DHD_ERROR(("[DUMP] %s: HAL started. send urgent event\n", __FUNCTION__));
+#ifdef DHD_MAP_PKTID_LOGGING
+		dhd_pktid_logging_dump(dhdp);
+#endif /* DHD_MAP_PKTID_LOGGING */
+		dhd_dbg_send_urgent_evt(dhdp, NULL, 0);
+
+		DHD_ERROR(("%s: wait to clear dhd_bus_busy_state: 0x%x\n",
+			__FUNCTION__, dhdp->dhd_bus_busy_state));
+		timeleft = dhd_os_busbusy_wait_bitmask(dhdp,
+				&dhdp->dhd_bus_busy_state, DHD_BUS_BUSY_IN_HALDUMP, 0);
+		if ((dhdp->dhd_bus_busy_state & DHD_BUS_BUSY_IN_HALDUMP) != 0) {
+			DHD_ERROR(("%s: Timed out(%d) dhd_bus_busy_state=0x%x\n",
+					__FUNCTION__, timeleft, dhdp->dhd_bus_busy_state));
+			dhd_set_dump_status(dhdp, DUMP_FAILURE);
+			ret = BCME_ERROR;
+		}
+	} else {
+		DHD_ERROR(("[DUMP] %s: HAL Not started. skip urgent event\n", __FUNCTION__));
+		ret = BCME_ERROR;
+	}
+	DHD_OS_WAKE_UNLOCK(dhdp);
+#endif /* WL_CFG80211 */
+
+	/* In case of dhd_os_busbusy_wait_bitmask() timeout,
+	 * hal dump bit will not be cleared. Hence clearing it here.
+	 */
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_HALDUMP(dhdp);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	return ret;
+}
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_CORE_DUMP */
+
+#ifdef ARP_OFFLOAD_SUPPORT
+/* add or remove AOE host ip(s) (up to 8 IPs on the interface)  */
+/* XXX add operation is more efficent */
+void
+aoe_update_host_ipv4_table(dhd_pub_t *dhd_pub, u32 ipa, bool add, int idx)
+{
+	u32 ipv4_buf[MAX_IPV4_ENTRIES]; /* temp save for AOE host_ip table */
+	int i;
+	int ret;
+
+	bzero(ipv4_buf, sizeof(ipv4_buf));
+
+	/* display what we've got */
+	ret = dhd_arp_get_arp_hostip_table(dhd_pub, ipv4_buf, sizeof(ipv4_buf), idx);
+	DHD_ARPOE(("%s: hostip table read from Dongle:\n", __FUNCTION__));
+#ifdef AOE_DBG
+	dhd_print_buf(ipv4_buf, 32, 4); /* max 8 IPs 4b each */
+#endif
+	/* now we saved hoste_ip table, clr it in the dongle AOE */
+	dhd_aoe_hostip_clr(dhd_pub, idx);
+
+	if (ret) {
+		DHD_ERROR(("%s failed\n", __FUNCTION__));
+		return;
+	}
+
+	for (i = 0; i < MAX_IPV4_ENTRIES; i++) {
+		if (add && (ipv4_buf[i] == 0)) {
+				ipv4_buf[i] = ipa;
+				add = FALSE; /* added ipa to local table  */
+				DHD_ARPOE(("%s: Saved new IP in temp arp_hostip[%d]\n",
+				__FUNCTION__, i));
+		} else if (ipv4_buf[i] == ipa) {
+			ipv4_buf[i]	= 0;
+			DHD_ARPOE(("%s: removed IP:%x from temp table %d\n",
+				__FUNCTION__, ipa, i));
+		}
+
+		if (ipv4_buf[i] != 0) {
+			/* add back host_ip entries from our local cache */
+			dhd_arp_offload_add_ip(dhd_pub, ipv4_buf[i], idx);
+			DHD_ARPOE(("%s: added IP:%x to dongle arp_hostip[%d]\n\n",
+				__FUNCTION__, ipv4_buf[i], i));
+		}
+	}
+#ifdef AOE_DBG
+	/* see the resulting hostip table */
+	dhd_arp_get_arp_hostip_table(dhd_pub, ipv4_buf, sizeof(ipv4_buf), idx);
+	DHD_ARPOE(("%s: read back arp_hostip table:\n", __FUNCTION__));
+	dhd_print_buf(ipv4_buf, 32, 4); /* max 8 IPs 4b each */
+#endif
+}
+
+/* XXX this function is only for IP address */
+/*
+ * Notification mechanism from kernel to our driver. This function is called by the Linux kernel
+ * whenever there is an event related to an IP address.
+ * ptr : kernel provided pointer to IP address that has changed
+ */
+static int dhd_inetaddr_notifier_call(struct notifier_block *this,
+	unsigned long event,
+	void *ptr)
+{
+	struct in_ifaddr *ifa = (struct in_ifaddr *)ptr;
+
+	dhd_info_t *dhd;
+	dhd_pub_t *dhd_pub;
+	int idx;
+
+	if (!ifa || !(ifa->ifa_dev->dev))
+		return NOTIFY_DONE;
+
+	/* Filter notifications meant for non Broadcom devices */
+	if ((ifa->ifa_dev->dev->netdev_ops != &dhd_ops_pri) &&
+	    (ifa->ifa_dev->dev->netdev_ops != &dhd_ops_virt)) {
+#if defined(WL_ENABLE_P2P_IF)
+		if (!wl_cfgp2p_is_ifops(ifa->ifa_dev->dev->netdev_ops))
+#endif /* WL_ENABLE_P2P_IF */
+			return NOTIFY_DONE;
+	}
+
+	dhd = DHD_DEV_INFO(ifa->ifa_dev->dev);
+	if (!dhd)
+		return NOTIFY_DONE;
+
+	dhd_pub = &dhd->pub;
+
+	if (!dhd_pub->arpoe_enable) {
+		DHD_ERROR(("arpoe_enable not set"));
+		return NOTIFY_DONE;
+	}
+
+	if (dhd_pub->arp_version == 1) {
+		idx = 0;
+	} else {
+		for (idx = 0; idx < DHD_MAX_IFS; idx++) {
+			if (dhd->iflist[idx] && dhd->iflist[idx]->net == ifa->ifa_dev->dev)
+			break;
+		}
+		if (idx < DHD_MAX_IFS)
+			DHD_TRACE(("ifidx : %p %s %d\n", dhd->iflist[idx]->net,
+				dhd->iflist[idx]->name, dhd->iflist[idx]->idx));
+		else {
+			DHD_ERROR(("Cannot find ifidx for(%s) set to 0\n", ifa->ifa_label));
+			idx = 0;
+		}
+	}
+
+	switch (event) {
+		case NETDEV_UP:
+			DHD_ARPOE(("%s: [%s] Up IP: 0x%x\n",
+				__FUNCTION__, ifa->ifa_label, ifa->ifa_address));
+
+			/*
+			 * Skip if Bus is not in a state to transport the IOVAR
+			 * (or) the Dongle is not ready.
+			 */
+			if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(&dhd->pub) ||
+				dhd->pub.busstate ==  DHD_BUS_LOAD) {
+				DHD_ERROR(("%s: bus not ready, exit NETDEV_UP : %d\n",
+					__FUNCTION__, dhd->pub.busstate));
+				if (dhd->pend_ipaddr) {
+					DHD_ERROR(("%s: overwrite pending ipaddr: 0x%x\n",
+						__FUNCTION__, dhd->pend_ipaddr));
+				}
+				dhd->pend_ipaddr = ifa->ifa_address;
+				break;
+			}
+
+#ifdef AOE_IP_ALIAS_SUPPORT
+			/* XXX HOSTAPD will be rerturned at first */
+			DHD_ARPOE(("%s:add aliased IP to AOE hostip cache\n",
+				__FUNCTION__));
+			aoe_update_host_ipv4_table(dhd_pub, ifa->ifa_address, TRUE, idx);
+#endif /* AOE_IP_ALIAS_SUPPORT */
+			dhd_conf_set_garp(dhd_pub, idx, ifa->ifa_address, TRUE);
+			break;
+
+		case NETDEV_DOWN:
+			DHD_ARPOE(("%s: [%s] Down IP: 0x%x\n",
+				__FUNCTION__, ifa->ifa_label, ifa->ifa_address));
+			dhd->pend_ipaddr = 0;
+#ifdef AOE_IP_ALIAS_SUPPORT
+			/* XXX HOSTAPD will be rerturned at first */
+			DHD_ARPOE(("%s:interface is down, AOE clr all for this if\n",
+				__FUNCTION__));
+			if ((dhd_pub->op_mode & DHD_FLAG_HOSTAP_MODE) ||
+				(ifa->ifa_dev->dev != dhd_linux_get_primary_netdev(dhd_pub))) {
+				aoe_update_host_ipv4_table(dhd_pub, ifa->ifa_address, FALSE, idx);
+			} else
+#endif /* AOE_IP_ALIAS_SUPPORT */
+			{
+				/* XXX clear ALL arp and hostip tables */
+				dhd_aoe_hostip_clr(&dhd->pub, idx);
+				dhd_aoe_arp_clr(&dhd->pub, idx);
+			}
+			dhd_conf_set_garp(dhd_pub, idx, ifa->ifa_address, FALSE);
+			break;
+
+		default:
+			DHD_ARPOE(("%s: do noting for [%s] Event: %lu\n",
+				__func__, ifa->ifa_label, event));
+			break;
+	}
+	return NOTIFY_DONE;
+}
+#endif /* ARP_OFFLOAD_SUPPORT */
+
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+/* Neighbor Discovery Offload: defered handler */
+static void
+dhd_inet6_work_handler(void *dhd_info, void *event_data, u8 event)
+{
+	struct ipv6_work_info_t *ndo_work = (struct ipv6_work_info_t *)event_data;
+	dhd_info_t *dhd = (dhd_info_t *)dhd_info;
+	dhd_pub_t *dhdp;
+	int ret;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: invalid dhd_info\n", __FUNCTION__));
+		goto done;
+	}
+	dhdp = &dhd->pub;
+
+	if (event != DHD_WQ_WORK_IPV6_NDO) {
+		DHD_ERROR(("%s: unexpected event\n", __FUNCTION__));
+		goto done;
+	}
+
+	if (!ndo_work) {
+		DHD_ERROR(("%s: ipv6 work info is not initialized\n", __FUNCTION__));
+		return;
+	}
+
+	switch (ndo_work->event) {
+		case NETDEV_UP:
+#ifndef NDO_CONFIG_SUPPORT
+			DHD_TRACE(("%s: Enable NDO \n ", __FUNCTION__));
+			ret = dhd_ndo_enable(dhdp, TRUE);
+			if (ret < 0) {
+				DHD_ERROR(("%s: Enabling NDO Failed %d\n", __FUNCTION__, ret));
+			}
+#endif /* !NDO_CONFIG_SUPPORT */
+			DHD_TRACE(("%s: Add a host ip for NDO\n", __FUNCTION__));
+			if (dhdp->ndo_version > 0) {
+				/* inet6 addr notifier called only for unicast address */
+				ret = dhd_ndo_add_ip_with_type(dhdp, &ndo_work->ipv6_addr[0],
+					WL_ND_IPV6_ADDR_TYPE_UNICAST, ndo_work->if_idx);
+			} else {
+				ret = dhd_ndo_add_ip(dhdp, &ndo_work->ipv6_addr[0],
+					ndo_work->if_idx);
+			}
+			if (ret < 0) {
+				DHD_ERROR(("%s: Adding a host ip for NDO failed %d\n",
+					__FUNCTION__, ret));
+			}
+			break;
+		case NETDEV_DOWN:
+			if (dhdp->ndo_version > 0) {
+				DHD_TRACE(("%s: Remove a host ip for NDO\n", __FUNCTION__));
+				ret = dhd_ndo_remove_ip_by_addr(dhdp,
+					&ndo_work->ipv6_addr[0], ndo_work->if_idx);
+			} else {
+				DHD_TRACE(("%s: Clear host ip table for NDO \n", __FUNCTION__));
+				ret = dhd_ndo_remove_ip(dhdp, ndo_work->if_idx);
+			}
+			if (ret < 0) {
+				DHD_ERROR(("%s: Removing host ip for NDO failed %d\n",
+					__FUNCTION__, ret));
+				goto done;
+			}
+#ifdef NDO_CONFIG_SUPPORT
+			if (dhdp->ndo_host_ip_overflow) {
+				ret = dhd_dev_ndo_update_inet6addr(
+					dhd_idx2net(dhdp, ndo_work->if_idx));
+				if ((ret < 0) && (ret != BCME_NORESOURCE)) {
+					DHD_ERROR(("%s: Updating host ip for NDO failed %d\n",
+						__FUNCTION__, ret));
+					goto done;
+				}
+			}
+#else /* !NDO_CONFIG_SUPPORT */
+			DHD_TRACE(("%s: Disable NDO\n ", __FUNCTION__));
+			ret = dhd_ndo_enable(dhdp, FALSE);
+			if (ret < 0) {
+				DHD_ERROR(("%s: disabling NDO Failed %d\n", __FUNCTION__, ret));
+				goto done;
+			}
+#endif /* NDO_CONFIG_SUPPORT */
+			break;
+
+		default:
+			DHD_ERROR(("%s: unknown notifier event \n", __FUNCTION__));
+			break;
+	}
+done:
+
+	/* free ndo_work. alloced while scheduling the work */
+	if (ndo_work) {
+		kfree(ndo_work);
+	}
+
+	return;
+} /* dhd_init_logstrs_array */
+
+/*
+ * Neighbor Discovery Offload: Called when an interface
+ * is assigned with ipv6 address.
+ * Handles only primary interface
+ */
+int dhd_inet6addr_notifier_call(struct notifier_block *this, unsigned long event, void *ptr)
+{
+	dhd_info_t *dhd;
+	dhd_pub_t *dhdp;
+	struct inet6_ifaddr *inet6_ifa = ptr;
+	struct ipv6_work_info_t *ndo_info;
+	int idx = 0;
+
+	/* Filter notifications meant for non Broadcom devices */
+	if (inet6_ifa->idev->dev->netdev_ops != &dhd_ops_pri) {
+			return NOTIFY_DONE;
+	}
+
+	dhd = DHD_DEV_INFO(inet6_ifa->idev->dev);
+	if (!dhd) {
+		return NOTIFY_DONE;
+	}
+	dhdp = &dhd->pub;
+
+	idx = dhd_net2idx(dhd, inet6_ifa->idev->dev);
+	if ((idx >= 0 && idx < DHD_MAX_IFS)
+#if defined(WL_CFG80211)
+		&& IS_STA_IFACE(ndev_to_wdev(inet6_ifa->idev->dev))
+#endif /* WL_CFG80211 */
+		&& 1) {
+		DHD_TRACE(("ifidx : %p %s %d\n", dhd->iflist[idx]->net,
+			dhd->iflist[idx]->name, dhd->iflist[idx]->idx));
+	} else {
+		return NOTIFY_DONE;
+	}
+
+	/* FW capability */
+	if (!FW_SUPPORTED(dhdp, ndoe)) {
+		return NOTIFY_DONE;
+	}
+
+	ndo_info = (struct ipv6_work_info_t *)kzalloc(sizeof(struct ipv6_work_info_t), GFP_ATOMIC);
+	if (!ndo_info) {
+		DHD_ERROR(("%s: ipv6 work alloc failed\n", __FUNCTION__));
+		return NOTIFY_DONE;
+	}
+
+	/* fill up ndo_info */
+	ndo_info->event = event;
+	ndo_info->if_idx = idx;
+	memcpy(ndo_info->ipv6_addr, &inet6_ifa->addr, IPV6_ADDR_LEN);
+
+	/* defer the work to thread as it may block kernel */
+	dhd_deferred_schedule_work(dhd->dhd_deferred_wq, (void *)ndo_info, DHD_WQ_WORK_IPV6_NDO,
+		dhd_inet6_work_handler, DHD_WQ_WORK_PRIORITY_LOW);
+	return NOTIFY_DONE;
+}
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+
+/* Network attach to be invoked from the bus probe handlers */
+int
+dhd_attach_net(dhd_pub_t *dhdp, bool need_rtnl_lock)
+{
+	struct net_device *primary_ndev;
+#ifdef GET_CUSTOM_MAC_ENABLE
+	char hw_ether[62];
+#endif /* GET_CUSTOM_MAC_ENABLE */
+#if defined(GET_CUSTOM_MAC_ENABLE) || defined(GET_OTP_MAC_ENABLE)
+	int ret = BCME_ERROR;
+#endif /* GET_CUSTOM_MAC_ENABLE || GET_OTP_MAC_ENABLE */
+
+	BCM_REFERENCE(primary_ndev);
+
+#ifdef GET_CUSTOM_MAC_ENABLE
+	ret = wifi_platform_get_mac_addr(dhdp->adapter, hw_ether, 0);
+	if (!ret)
+		bcopy(hw_ether, dhdp->mac.octet, ETHER_ADDR_LEN);
+#endif /* GET_CUSTOM_MAC_ENABLE */
+
+#ifdef GET_OTP_MAC_ENABLE
+	if (ret && memcmp(&ether_null, &dhdp->conf->otp_mac, ETHER_ADDR_LEN))
+		bcopy(&dhdp->conf->otp_mac, &dhdp->mac, ETHER_ADDR_LEN);
+#endif /* GET_OTP_MAC_ENABLE */
+
+	/* Register primary net device */
+	if (dhd_register_if(dhdp, 0, need_rtnl_lock) != 0) {
+		return BCME_ERROR;
+	}
+
+#if defined(WL_CFG80211)
+	primary_ndev =  dhd_linux_get_primary_netdev(dhdp);
+	if (wl_cfg80211_net_attach(primary_ndev) < 0) {
+		/* fail the init */
+		dhd_remove_if(dhdp, 0, TRUE);
+		return BCME_ERROR;
+	}
+#endif /* WL_CFG80211 */
+	return BCME_OK;
+}
+
+int
+dhd_register_if(dhd_pub_t *dhdp, int ifidx, bool need_rtnl_lock)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	dhd_if_t *ifp;
+	struct net_device *net = NULL;
+	int err = 0;
+	uint8 temp_addr[ETHER_ADDR_LEN] = { 0x00, 0x90, 0x4c, 0x11, 0x22, 0x33 };
+	unsigned long flags;
+
+	DHD_TRACE(("%s: ifidx %d\n", __FUNCTION__, ifidx));
+
+	if (dhd == NULL || dhd->iflist[ifidx] == NULL) {
+		DHD_ERROR(("%s: Invalid Interface\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	ASSERT(dhd && dhd->iflist[ifidx]);
+	ifp = dhd->iflist[ifidx];
+	net = ifp->net;
+	ASSERT(net && (ifp->idx == ifidx));
+
+	ASSERT(!net->netdev_ops);
+	net->netdev_ops = &dhd_ops_virt;
+
+	/* Ok, link into the network layer... */
+	if (ifidx == 0) {
+		/*
+		 * device functions for the primary interface only
+		 */
+		net->netdev_ops = &dhd_ops_pri;
+		if (!ETHER_ISNULLADDR(dhd->pub.mac.octet))
+			memcpy(temp_addr, dhd->pub.mac.octet, ETHER_ADDR_LEN);
+	} else {
+		/*
+		 * We have to use the primary MAC for virtual interfaces
+		 */
+		memcpy(temp_addr, ifp->mac_addr, ETHER_ADDR_LEN);
+#if defined(OEM_ANDROID)
+		/*
+		 * Android sets the locally administered bit to indicate that this is a
+		 * portable hotspot.  This will not work in simultaneous AP/STA mode,
+		 * nor with P2P.  Need to set the Donlge's MAC address, and then use that.
+		 */
+		if (!memcmp(temp_addr, dhd->iflist[0]->mac_addr,
+			ETHER_ADDR_LEN)) {
+			DHD_ERROR(("%s interface [%s]: set locally administered bit in MAC\n",
+			__func__, net->name));
+			temp_addr[0] |= 0x02;
+		}
+#endif /* defined(OEM_ANDROID) */
+	}
+
+	net->hard_header_len = ETH_HLEN + dhd->pub.hdrlen;
+
+	net->ethtool_ops = &dhd_ethtool_ops;
+
+#if defined(WL_WIRELESS_EXT)
+#if WIRELESS_EXT < 19
+	net->get_wireless_stats = dhd_get_wireless_stats;
+#endif /* WIRELESS_EXT < 19 */
+#if WIRELESS_EXT > 12
+	net->wireless_handlers = &wl_iw_handler_def;
+#endif /* WIRELESS_EXT > 12 */
+#endif /* defined(WL_WIRELESS_EXT) */
+
+	/* XXX Set up an MTU change notifier as per linux/notifier.h? */
+	dhd->pub.rxsz = DBUS_RX_BUFFER_SIZE_DHD(net);
+
+#ifdef WLMESH
+	if (ifidx >= 2 && dhdp->conf->fw_type == FW_TYPE_MESH) {
+		temp_addr[4] ^= 0x80;
+		temp_addr[4] += ifidx;
+		temp_addr[5] += ifidx;
+	}
+#endif
+	/*
+	 * XXX Linux 2.6.25 does not like a blank MAC address, so use a
+	 * dummy address until the interface is brought up.
+	 */
+	DEV_ADDR_SET(net, temp_addr);
+
+	if (ifidx == 0)
+		DHD_INFO(("%s\n", dhd_version));
+	else {
+#ifdef WL_EXT_IAPSTA
+		wl_ext_iapsta_update_net_device(net, ifidx);
+#endif /* WL_EXT_IAPSTA */
+		if (dhd->pub.up == 1) {
+			memcpy(temp_addr, net->dev_addr, ETHER_ADDR_LEN);
+			if (_dhd_set_mac_address(dhd, ifidx, temp_addr, FALSE) == 0)
+				DHD_INFO(("%s: MACID is overwritten\n", __FUNCTION__));
+			else
+				DHD_ERROR(("%s: _dhd_set_mac_address() failed\n", __FUNCTION__));
+		}
+	}
+
+	err = dhd_register_net(net, need_rtnl_lock);
+	if (err != 0) {
+		DHD_ERROR(("couldn't register the net device [%s], err %d\n", net->name, err));
+		goto fail;
+	}
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+	if ((ctf_dev_register(dhd->cih, net, FALSE) != BCME_OK) ||
+	    (ctf_enable(dhd->cih, net, TRUE, &dhd->brc_hot) != BCME_OK)) {
+		DHD_ERROR(("%s:%d: ctf_dev_register/ctf_enable failed for interface %d\n",
+			__FUNCTION__, __LINE__, ifidx));
+		goto fail;
+	}
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#if defined(WLDWDS) && defined(WL_EXT_IAPSTA)
+	if (ifp->dwds) {
+		wl_ext_iapsta_attach_dwds_netdev(net, ifidx, ifp->bssidx);
+	} else
+#endif /* WLDWDS && WL_EXT_IAPSTA */
+	{
+#ifdef WL_EVENT
+		wl_ext_event_attach_netdev(net, ifidx, ifp->bssidx);
+#endif /* WL_EVENT */
+#ifdef WL_ESCAN
+		wl_escan_event_attach(net, ifidx);
+#endif /* WL_ESCAN */
+#ifdef WL_EXT_IAPSTA
+		wl_ext_iapsta_attach_netdev(net, ifidx, ifp->bssidx);
+		wl_ext_iapsta_attach_name(net, ifidx);
+#endif /* WL_EXT_IAPSTA */
+	}
+
+#if defined(CONFIG_TIZEN)
+	net_stat_tizen_register(net);
+#endif /* CONFIG_TIZEN */
+
+	DHD_INFO(("Register interface [%s]  MAC: "MACDBG"\n\n", net->name,
+		MAC2STRDBG(net->dev_addr)));
+
+#if defined(OEM_ANDROID) && (defined(BCMPCIE) || defined(BCMLXSDMMC) || defined(BCMDBUS))
+	if (ifidx == 0) {
+#if defined(BCMLXSDMMC) && !defined(DHD_PRELOAD)
+		up(&dhd_registration_sem);
+#endif /* BCMLXSDMMC */
+		if (!dhd_download_fw_on_driverload) {
+#ifdef WL_CFG80211
+			wl_terminate_event_handler(net);
+#endif /* WL_CFG80211 */
+#if defined(DHD_LB_RXP)
+			__skb_queue_purge(&dhd->rx_pend_queue);
+			skb_queue_purge(&dhd->rx_emerge_queue);
+#endif /* DHD_LB_RXP */
+
+#if defined(DHD_LB_TXP)
+			skb_queue_purge(&dhd->tx_pend_queue);
+#endif /* DHD_LB_TXP */
+
+#ifdef SHOW_LOGTRACE
+			/* Release the skbs from queue for WLC_E_TRACE event */
+			dhd_event_logtrace_flush_queue(dhdp);
+#endif /* SHOW_LOGTRACE */
+
+#if defined(BCMPCIE) && defined(DHDTCPACK_SUPPRESS)
+			dhd_tcpack_suppress_set(dhdp, TCPACK_SUP_OFF);
+#endif /* BCMPCIE && DHDTCPACK_SUPPRESS */
+
+#if defined(WLAN_ACCEL_BOOT)
+			/*
+			 * Make Wifi suspended after boot up with forge_reg_on true
+			 * This will download fw when ifconfig wlan0 up sequence
+			 */
+			wl_android_wifi_off(net, TRUE);
+			dhd->wl_accel_force_reg_on = TRUE;
+			/* Module init time clear do_chip_bighammer */
+			dhd->pub.do_chip_bighammer = FALSE;
+#else
+			/* Turn off Wifi after boot up */
+#if defined(BT_OVER_SDIO)
+			dhd_bus_put(&dhd->pub, WLAN_MODULE);
+			wl_android_set_wifi_on_flag(FALSE);
+#else
+			wl_android_wifi_off(net, TRUE);
+#endif /* BT_OVER_SDIO */
+#endif /* WLAN_ACCEL_BOOT */
+
+		}
+	}
+#endif /* OEM_ANDROID && (BCMPCIE || (BCMLXSDMMC) */
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	ifp->del_in_progress = FALSE;
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	return 0;
+
+fail:
+	net->netdev_ops = NULL;
+	return err;
+}
+
+void
+dhd_bus_detach(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (dhdp) {
+		dhd = (dhd_info_t *)dhdp->info;
+		if (dhd) {
+
+			/*
+			 * In case of Android cfg80211 driver, the bus is down in dhd_stop,
+			 *  calling stop again will cuase SD read/write errors.
+			 */
+			if (dhd->pub.busstate != DHD_BUS_DOWN && dhd_download_fw_on_driverload) {
+				/* Stop the protocol module */
+				dhd_prot_stop(&dhd->pub);
+
+				/* Stop the bus module */
+#ifdef BCMDBUS
+				/* Force Dongle terminated */
+				if (dhd_wl_ioctl_cmd(dhdp, WLC_TERMINATED, NULL, 0, TRUE, 0) < 0)
+					DHD_ERROR(("%s Setting WLC_TERMINATED failed\n",
+						__FUNCTION__));
+				dbus_stop(dhd->pub.bus);
+				DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+				dhd->pub.busstate = DHD_BUS_DOWN;
+#else
+				dhd_bus_stop(dhd->pub.bus, TRUE);
+#endif /* BCMDBUS */
+			}
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) || defined(BCMPCIE_OOB_HOST_WAKE)
+			dhd_bus_oob_intr_unregister(dhdp);
+#endif /* OOB_INTR_ONLY || BCMSPI_ANDROID || BCMPCIE_OOB_HOST_WAKE */
+		}
+	}
+}
+
+void dhd_detach(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+	unsigned long flags;
+	int timer_valid = FALSE;
+	struct net_device *dev = NULL;
+	dhd_if_t *ifp;
+#ifdef WL_CFG80211
+	struct bcm_cfg80211 *cfg = NULL;
+#endif
+	if (!dhdp)
+		return;
+
+	dhd = (dhd_info_t *)dhdp->info;
+	if (!dhd)
+		return;
+
+	/* primary interface 0 */
+	ifp = dhd->iflist[0];
+	if (ifp && ifp->net) {
+		dev = ifp->net;
+	}
+
+	if (dev) {
+		rtnl_lock();
+#if defined(WL_CFG80211) && defined(WL_STATIC_IF)
+		if (dhd->dhd_state & DHD_ATTACH_STATE_CFG80211) {
+			wl_cfg80211_static_if_dev_close(dev);
+		}
+#endif /* WL_CFG80211 && WL_STATIC_IF */
+		if (dev->flags & IFF_UP) {
+			/* If IFF_UP is still up, it indicates that
+			 * "ifconfig wlan0 down" hasn't been called.
+			 * So invoke dev_close explicitly here to
+			 * bring down the interface.
+			 */
+			DHD_TRACE(("IFF_UP flag is up. Enforcing dev_close from detach \n"));
+			dev_close(dev);
+		}
+		rtnl_unlock();
+	}
+
+	DHD_TRACE(("%s: Enter state 0x%x\n", __FUNCTION__, dhd->dhd_state));
+
+	/* XXX	kernel panic issue when first bootup time,
+	 *	 rmmod without interface down make unnecessary hang event.
+	 */
+	DHD_INFO(("%s: making dhdpub up FALSE\n", __FUNCTION__));
+	dhd->pub.up = 0;
+	if (!(dhd->dhd_state & DHD_ATTACH_STATE_DONE)) {
+		/* Give sufficient time for threads to start running in case
+		 * dhd_attach() has failed
+		 */
+		OSL_SLEEP(100);
+	}
+#ifdef DHD_WET
+	dhd_free_wet_info(&dhd->pub, dhd->pub.wet_info);
+#endif /* DHD_WET */
+
+#ifdef PROP_TXSTATUS
+#ifdef DHD_WLFC_THREAD
+	if (dhd->pub.wlfc_thread) {
+		kthread_stop(dhd->pub.wlfc_thread);
+		dhdp->wlfc_thread_go = TRUE;
+		wake_up_interruptible(&dhdp->wlfc_wqhead);
+	}
+	dhd->pub.wlfc_thread = NULL;
+#endif /* DHD_WLFC_THREAD */
+#endif /* PROP_TXSTATUS */
+
+	if (dhd->dhd_state & DHD_ATTACH_STATE_PROT_ATTACH) {
+
+#if defined(OEM_ANDROID) || !defined(BCMSDIO)
+		dhd_bus_detach(dhdp);
+#endif /* OEM_ANDROID || !BCMSDIO */
+#ifndef PCIE_FULL_DONGLE
+#if defined(OEM_ANDROID) || !defined(BCMSDIO)
+		if (dhdp->prot)
+			dhd_prot_detach(dhdp);
+#endif /* OEM_ANDROID || !BCMSDIO */
+#endif /* !PCIE_FULL_DONGLE */
+	}
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	if (dhd_inetaddr_notifier_registered) {
+		dhd_inetaddr_notifier_registered = FALSE;
+		unregister_inetaddr_notifier(&dhd_inetaddr_notifier);
+	}
+#endif /* ARP_OFFLOAD_SUPPORT */
+#if defined(CONFIG_IPV6) && defined(IPV6_NDO_SUPPORT)
+	if (dhd_inet6addr_notifier_registered) {
+		dhd_inet6addr_notifier_registered = FALSE;
+		unregister_inet6addr_notifier(&dhd_inet6addr_notifier);
+	}
+#endif /* CONFIG_IPV6 && IPV6_NDO_SUPPORT */
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+	if (dhd->dhd_state & DHD_ATTACH_STATE_EARLYSUSPEND_DONE) {
+		if (dhd->early_suspend.suspend)
+			unregister_early_suspend(&dhd->early_suspend);
+	}
+#endif /* CONFIG_HAS_EARLYSUSPEND && DHD_USE_EARLYSUSPEND */
+
+#if defined(WL_WIRELESS_EXT)
+	if (dhd->dhd_state & DHD_ATTACH_STATE_WL_ATTACH) {
+		/* Detatch and unlink in the iw */
+		wl_iw_detach(dev);
+	}
+#endif /* defined(WL_WIRELESS_EXT) */
+#ifdef WL_EXT_GENL
+	wl_ext_genl_deinit(dev);
+#endif
+#ifdef WL_EXT_IAPSTA
+	wl_ext_iapsta_dettach(dev);
+#endif /* WL_EXT_IAPSTA */
+#ifdef WL_ESCAN
+	wl_escan_detach(dev);
+#endif /* WL_ESCAN */
+#ifdef WL_TIMER
+	wl_timer_dettach(dhdp);
+#endif /* WL_TIMER */
+#ifdef WL_EVENT
+	wl_ext_event_dettach(dhdp);
+#endif /* WL_EVENT */
+
+	/* delete all interfaces, start with virtual  */
+	if (dhd->dhd_state & DHD_ATTACH_STATE_ADD_IF) {
+		int i = 1;
+
+		/* Cleanup virtual interfaces */
+		dhd_net_if_lock_local(dhd);
+		for (i = 1; i < DHD_MAX_IFS; i++) {
+			if (dhd->iflist[i]) {
+				dhd_remove_if(&dhd->pub, i, TRUE);
+			}
+		}
+		dhd_net_if_unlock_local(dhd);
+
+		/* 'ifp' indicates primary interface 0, clean it up. */
+		if (ifp && ifp->net) {
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+			if (dhd->cih)
+				ctf_dev_unregister(dhd->cih, ifp->net);
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#ifdef WL_CFG80211
+			cfg = wl_get_cfg(ifp->net);
+#endif
+			/* in unregister_netdev case, the interface gets freed by net->destructor
+			 * (which is set to free_netdev)
+			 */
+			if (ifp->net->reg_state == NETREG_UNINITIALIZED) {
+				free_netdev(ifp->net);
+			} else {
+#ifdef SET_RPS_CPUS
+				custom_rps_map_clear(ifp->net->_rx);
+#endif /* SET_RPS_CPUS */
+				netif_tx_disable(ifp->net);
+				dhd_unregister_net(ifp->net, true);
+			}
+#ifdef PCIE_FULL_DONGLE
+			ifp->net = DHD_NET_DEV_NULL;
+#else
+			ifp->net = NULL;
+#endif /* PCIE_FULL_DONGLE */
+#if defined(BCMSDIO) && !defined(OEM_ANDROID)
+			dhd_bus_detach(dhdp);
+
+			if (dhdp->prot)
+				dhd_prot_detach(dhdp);
+#endif /* BCMSDIO && !OEM_ANDROID */
+
+#ifdef DHD_WMF
+			dhd_wmf_cleanup(dhdp, 0);
+#endif /* DHD_WMF */
+#ifdef DHD_L2_FILTER
+			bcm_l2_filter_arp_table_update(dhdp->osh, ifp->phnd_arp_table, TRUE,
+				NULL, FALSE, dhdp->tickcnt);
+			deinit_l2_filter_arp_table(dhdp->osh, ifp->phnd_arp_table);
+			ifp->phnd_arp_table = NULL;
+#endif /* DHD_L2_FILTER */
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+			MFREE(dhdp->osh, ifp->qosmap_up_table, UP_TABLE_MAX);
+			ifp->qosmap_up_table_enable = FALSE;
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+
+			dhd_if_del_sta_list(ifp);
+
+			MFREE(dhd->pub.osh, ifp, sizeof(*ifp));
+			ifp = NULL;
+#ifdef WL_CFG80211
+			if (cfg && cfg->wdev)
+				cfg->wdev->netdev = NULL;
+#endif
+		}
+	}
+
+	/* Clear the watchdog timer */
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	timer_valid = dhd->wd_timer_valid;
+	dhd->wd_timer_valid = FALSE;
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+	if (timer_valid)
+		del_timer_sync(&dhd->timer);
+	DHD_STOP_RPM_TIMER(&dhd->pub);
+
+#ifdef BCMDBUS
+	tasklet_kill(&dhd->tasklet);
+#else
+	if (dhd->dhd_state & DHD_ATTACH_STATE_THREADS_CREATED) {
+		if (dhd->thr_wdt_ctl.thr_pid >= 0) {
+			PROC_STOP(&dhd->thr_wdt_ctl);
+		}
+
+		if (dhd->rxthread_enabled && dhd->thr_rxf_ctl.thr_pid >= 0) {
+			PROC_STOP(&dhd->thr_rxf_ctl);
+		}
+
+		if (dhd->thr_dpc_ctl.thr_pid >= 0) {
+			PROC_STOP(&dhd->thr_dpc_ctl);
+		} else
+		{
+			tasklet_kill(&dhd->tasklet);
+		}
+	}
+#endif /* BCMDBUS */
+
+	cancel_delayed_work_sync(&dhd->dhd_dpc_dispatcher_work);
+#ifdef DHD_LB
+	if (dhd->dhd_state & DHD_ATTACH_STATE_LB_ATTACH_DONE) {
+		/* Clear the flag first to avoid calling the cpu notifier */
+		dhd->dhd_state &= ~DHD_ATTACH_STATE_LB_ATTACH_DONE;
+
+		/* Kill the Load Balancing Tasklets */
+#ifdef DHD_LB_RXP
+		cancel_work_sync(&dhd->rx_napi_dispatcher_work);
+		__skb_queue_purge(&dhd->rx_pend_queue);
+		skb_queue_purge(&dhd->rx_emerge_queue);
+#endif /* DHD_LB_RXP */
+#ifdef DHD_LB_TXP
+		cancel_work_sync(&dhd->tx_dispatcher_work);
+		tasklet_kill(&dhd->tx_tasklet);
+		__skb_queue_purge(&dhd->tx_pend_queue);
+#endif /* DHD_LB_TXP */
+
+		/* Unregister from CPU Hotplug framework */
+		dhd_unregister_cpuhp_callback(dhd);
+
+		dhd_cpumasks_deinit(dhd);
+		DHD_LB_STATS_DEINIT(&dhd->pub);
+	}
+#endif /* DHD_LB */
+
+#if defined(DNGL_AXI_ERROR_LOGGING) && defined(DHD_USE_WQ_FOR_DNGL_AXI_ERROR)
+	cancel_work_sync(&dhd->axi_error_dispatcher_work);
+#endif /* DNGL_AXI_ERROR_LOGGING && DHD_USE_WQ_FOR_DNGL_AXI_ERROR */
+
+	DHD_SSSR_REG_INFO_DEINIT(&dhd->pub);
+	DHD_SSSR_MEMPOOL_DEINIT(&dhd->pub);
+	DHD_COREDUMP_MEMPOOL_DEINIT(&dhd->pub);
+
+#ifdef DHD_SDTC_ETB_DUMP
+	dhd_sdtc_etb_mempool_deinit(&dhd->pub);
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#ifdef RX_PKT_POOL
+	dhd_rx_pktpool_deinit(dhd);
+#endif
+
+#ifdef WL_CFG80211
+	if (dhd->dhd_state & DHD_ATTACH_STATE_CFG80211) {
+		if (!cfg) {
+			DHD_ERROR(("cfg NULL!\n"));
+			ASSERT(0);
+		} else {
+			wl_cfg80211_detach(cfg);
+			dhd_monitor_uninit();
+		}
+	}
+#endif
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	destroy_workqueue(dhd->tx_wq);
+	dhd->tx_wq = NULL;
+	destroy_workqueue(dhd->rx_wq);
+	dhd->rx_wq = NULL;
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+#ifdef DEBUGABILITY
+	if (dhdp->dbg) {
+#ifdef DBG_PKT_MON
+		dhd_os_dbg_detach_pkt_monitor(dhdp);
+		osl_spin_lock_deinit(dhd->pub.osh, dhd->pub.dbg->pkt_mon_lock);
+#endif /* DBG_PKT_MON */
+	}
+#endif /* DEBUGABILITY */
+	if (dhdp->dbg) {
+		dhd_os_dbg_detach(dhdp);
+	}
+#ifdef DHD_MEM_STATS
+	osl_spin_lock_deinit(dhd->pub.osh, dhd->pub.mem_stats_lock);
+#endif /* DHD_MEM_STATS */
+
+#ifdef DHD_PKT_LOGGING
+	dhd_os_detach_pktlog(dhdp);
+#endif /* DHD_PKT_LOGGING */
+#ifdef DHD_STATUS_LOGGING
+	dhd_detach_statlog(dhdp);
+#endif /* DHD_STATUS_LOGGING */
+#ifdef DHD_PKTDUMP_ROAM
+	dhd_dump_pkt_deinit(dhdp);
+#endif /* DHD_PKTDUMP_ROAM */
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+	if (dhd->pub.hang_info) {
+		MFREE(dhd->pub.osh, dhd->pub.hang_info, VENDOR_SEND_HANG_EXT_INFO_LEN);
+	}
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+#ifdef SHOW_LOGTRACE
+	/* Release the skbs from queue for WLC_E_TRACE event */
+	dhd_event_logtrace_flush_queue(dhdp);
+
+	/* Wait till event logtrace context finishes */
+	dhd_cancel_logtrace_process_sync(dhd);
+
+	/* Remove ring proc entries */
+	dhd_dbg_ring_proc_destroy(&dhd->pub);
+
+	if (dhd->dhd_state & DHD_ATTACH_LOGTRACE_INIT) {
+		if (dhd->event_data.fmts) {
+			MFREE(dhd->pub.osh, dhd->event_data.fmts,
+					dhd->event_data.fmts_size);
+		}
+		if (dhd->event_data.raw_fmts) {
+			MFREE(dhd->pub.osh, dhd->event_data.raw_fmts,
+					dhd->event_data.raw_fmts_size);
+		}
+		if (dhd->event_data.raw_sstr) {
+			MFREE(dhd->pub.osh, dhd->event_data.raw_sstr,
+					dhd->event_data.raw_sstr_size);
+		}
+		if (dhd->event_data.rom_raw_sstr) {
+			MFREE(dhd->pub.osh, dhd->event_data.rom_raw_sstr,
+					dhd->event_data.rom_raw_sstr_size);
+		}
+		dhd->dhd_state &= ~DHD_ATTACH_LOGTRACE_INIT;
+	}
+#endif /* SHOW_LOGTRACE */
+#ifdef PNO_SUPPORT
+	if (dhdp->pno_state)
+		dhd_pno_deinit(dhdp);
+#endif
+#ifdef RTT_SUPPORT
+	if (dhdp->rtt_state) {
+		dhd_rtt_detach(dhdp);
+	}
+#endif
+#if defined(CONFIG_PM_SLEEP)
+	if (dhd_pm_notifier_registered) {
+		unregister_pm_notifier(&dhd->pm_notifier);
+		dhd_pm_notifier_registered = FALSE;
+	}
+#endif /* CONFIG_PM_SLEEP */
+
+#ifdef DEBUG_CPU_FREQ
+		if (dhd->new_freq)
+			free_percpu(dhd->new_freq);
+		dhd->new_freq = NULL;
+		cpufreq_unregister_notifier(&dhd->freq_trans, CPUFREQ_TRANSITION_NOTIFIER);
+#endif
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd->wakelock_wd_counter = 0;
+	dhd_wake_lock_destroy(dhd->wl_wdwake);
+#endif /* CONFIG_HAS_WAKELOCK */
+	if (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT) {
+		DHD_TRACE(("wd wakelock count:%d\n", dhd->wakelock_wd_counter));
+		DHD_OS_WAKE_LOCK_DESTROY(dhd);
+	}
+
+#ifdef DHDTCPACK_SUPPRESS
+	/* This will free all MEM allocated for TCPACK SUPPRESS */
+	dhd_tcpack_suppress_set(&dhd->pub, TCPACK_SUP_OFF);
+#endif /* DHDTCPACK_SUPPRESS */
+
+#ifdef PCIE_FULL_DONGLE
+	dhd_flow_rings_deinit(dhdp);
+	if (dhdp->prot)
+		dhd_prot_detach(dhdp);
+#endif
+
+#ifdef EWP_EDL
+	if (host_edl_support) {
+		DHD_EDL_MEM_DEINIT(dhdp);
+		host_edl_support = FALSE;
+	}
+#endif /* EWP_EDL */
+
+#if defined(WLTDLS) && defined(PCIE_FULL_DONGLE)
+		dhd_free_tdls_peer_list(dhdp);
+#endif
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+	/* Release CTF pool ONLY after the prot layer is dettached and
+	 * pkts, possibly from fast ctfpool are freed into ctfpool/kernel
+	 */
+#ifdef CTFPOOL
+	/* free the buffers in fast pool */
+	osl_ctfpool_cleanup(dhd->pub.osh);
+#endif /* CTFPOOL */
+
+	/* free ctf resources */
+	if (dhd->cih)
+		ctf_detach(dhd->cih);
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+#if defined(BCMDBG) && defined(DHD_MACDBG)
+	dhd_macdbg_detach(dhdp);
+#endif /* BCMDBG && DHD_MACDBG */
+
+#ifdef DUMP_IOCTL_IOV_LIST
+	dhd_iov_li_delete(dhdp, &(dhdp->dump_iovlist_head));
+#endif /* DUMP_IOCTL_IOV_LIST */
+#ifdef DHD_DEBUG
+	/* memory waste feature list initilization */
+	dhd_mw_list_delete(dhdp, &(dhdp->mw_list_head));
+#endif /* DHD_DEBUG */
+#ifdef WL_MONITOR
+	dhd_del_monitor_if(dhd);
+#endif /* WL_MONITOR */
+
+	cancel_work_sync(&dhd->dhd_hang_process_work);
+
+	/* Prefer adding de-init code above this comment unless necessary.
+	 * The idea is to cancel work queue, sysfs and flags at the end.
+	 */
+	dhd_deferred_work_deinit(dhd->dhd_deferred_wq);
+	dhd->dhd_deferred_wq = NULL;
+
+	/* log dump related buffers should be freed after wq is purged */
+#ifdef DHD_LOG_DUMP
+	dhd_log_dump_deinit(&dhd->pub);
+#endif /* DHD_LOG_DUMP */
+#if defined(BCMPCIE)
+	if (dhdp->extended_trap_data)
+	{
+		MFREE(dhdp->osh, dhdp->extended_trap_data, BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+		dhdp->extended_trap_data = NULL;
+	}
+#ifdef DNGL_AXI_ERROR_LOGGING
+	if (dhdp->axi_err_dump)
+	{
+		MFREE(dhdp->osh, dhdp->axi_err_dump, sizeof(dhd_axi_error_dump_t));
+		dhdp->axi_err_dump = NULL;
+	}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+#endif /* BCMPCIE */
+
+#ifdef EWP_EDL
+	cancel_delayed_work_sync(&dhd->edl_dispatcher_work);
+#endif
+
+	(void)dhd_deinit_sock_flows_buf(dhd);
+
+#ifdef CSI_SUPPORT
+	dhd_csi_deinit(dhdp);
+#endif /* CSI_SUPPORT */
+
+#ifdef DHD_DUMP_MNGR
+	if (dhd->pub.dump_file_manage) {
+		MFREE(dhd->pub.osh, dhd->pub.dump_file_manage,
+				sizeof(dhd_dump_file_manage_t));
+	}
+#endif /* DHD_DUMP_MNGR */
+
+	dhd_sysfs_exit(dhd);
+	dhd->pub.fw_download_status = FW_UNLOADED;
+
+#if defined(BT_OVER_SDIO)
+	mutex_destroy(&dhd->bus_user_lock);
+#endif /* BT_OVER_SDIO */
+
+#ifdef DHD_TX_PROFILE
+	(void)dhd_tx_profile_detach(dhdp);
+#endif /* defined(DHD_TX_PROFILE) */
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+	cancel_work_sync(&dhd->dhd_alert_process_work);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	cancel_work_sync(&dhd->dhd_dump_proc_work);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+	dhd_conf_detach(dhdp);
+} /* dhd_detach */
+
+void
+dhd_free(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (dhdp) {
+		int i;
+		for (i = 0; i < ARRAYSIZE(dhdp->reorder_bufs); i++) {
+			if (dhdp->reorder_bufs[i]) {
+				reorder_info_t *ptr;
+				uint32 buf_size = sizeof(struct reorder_info);
+
+				ptr = dhdp->reorder_bufs[i];
+
+				buf_size += ((ptr->max_idx + 1) * sizeof(void*));
+				DHD_REORDER(("free flow id buf %d, maxidx is %d, buf_size %d\n",
+					i, ptr->max_idx, buf_size));
+
+				MFREE(dhdp->osh, dhdp->reorder_bufs[i], buf_size);
+			}
+		}
+
+		dhd_sta_pool_fini(dhdp, DHD_MAX_STA);
+
+		dhd = (dhd_info_t *)dhdp->info;
+		if (dhdp->soc_ram) {
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+			DHD_OS_PREFREE(dhdp, dhdp->soc_ram, dhdp->soc_ram_length);
+#else
+			if (is_vmalloc_addr(dhdp->soc_ram)) {
+				VMFREE(dhdp->osh, dhdp->soc_ram, dhdp->soc_ram_length);
+			}
+			else {
+				MFREE(dhdp->osh, dhdp->soc_ram, dhdp->soc_ram_length);
+			}
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_USE_STATIC_MEMDUMP */
+			dhdp->soc_ram = NULL;
+		}
+#ifdef CACHE_FW_IMAGES
+		if (dhdp->cached_fw) {
+			MFREE(dhdp->osh, dhdp->cached_fw, dhdp->bus->ramsize);
+		}
+
+		if (dhdp->cached_nvram) {
+			MFREE(dhdp->osh, dhdp->cached_nvram, MAX_NVRAMBUF_SIZE);
+		}
+#endif
+#ifdef SUPPORT_OTA_UPDATE
+		(void)dhd_ota_buf_clean(dhdp);
+#endif /* SUPPORT_OTA_UPDATE */
+		if (dhd != NULL) {
+#ifdef REPORT_FATAL_TIMEOUTS
+			deinit_dhd_timeouts(&dhd->pub);
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+			/* Free Platform Layer allocations */
+			if (dhd->pub.plat_info) {
+				MFREE(dhdp->osh, dhdp->plat_info, dhdp->plat_info_size);
+			}
+
+			/* If pointer is allocated by dhd_os_prealloc then avoid MFREE */
+			if (dhd != (dhd_info_t *)dhd_os_prealloc(dhdp,
+					DHD_PREALLOC_DHD_INFO, 0, FALSE))
+				MFREE(dhd->pub.osh, dhd, sizeof(*dhd));
+			dhd = NULL;
+		}
+	}
+}
+
+void
+dhd_clear(dhd_pub_t *dhdp)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (dhdp) {
+		int i;
+#ifdef DHDTCPACK_SUPPRESS
+		/* Clean up timer/data structure for any remaining/pending packet or timer. */
+		dhd_tcpack_info_tbl_clean(dhdp);
+#endif /* DHDTCPACK_SUPPRESS */
+		for (i = 0; i < ARRAYSIZE(dhdp->reorder_bufs); i++) {
+			if (dhdp->reorder_bufs[i]) {
+				reorder_info_t *ptr;
+				uint32 buf_size = sizeof(struct reorder_info);
+
+				ptr = dhdp->reorder_bufs[i];
+
+				buf_size += ((ptr->max_idx + 1) * sizeof(void*));
+				DHD_REORDER(("free flow id buf %d, maxidx is %d, buf_size %d\n",
+					i, ptr->max_idx, buf_size));
+
+				MFREE(dhdp->osh, dhdp->reorder_bufs[i], buf_size);
+			}
+		}
+
+		dhd_sta_pool_clear(dhdp, DHD_MAX_STA);
+
+		if (dhdp->soc_ram) {
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+			DHD_OS_PREFREE(dhdp, dhdp->soc_ram, dhdp->soc_ram_length);
+#else
+			if (is_vmalloc_addr(dhdp->soc_ram)) {
+				VMFREE(dhdp->osh, dhdp->soc_ram, dhdp->soc_ram_length);
+			}
+			else {
+				MFREE(dhdp->osh, dhdp->soc_ram, dhdp->soc_ram_length);
+			}
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_USE_STATIC_MEMDUMP */
+			dhdp->soc_ram = NULL;
+		}
+	}
+}
+
+static void
+dhd_module_cleanup(void)
+{
+#ifdef DHD_SR_FIX
+	uint wl_down = 1;
+	int ret = 0;
+#endif /* DHD_SR_FIX */
+
+#if defined(ENABLE_NOT_LOAD_DHD_MODULE)
+	DHD_ERROR(("%s ##### Do not clean-up due to secondary build\n", __FUNCTION__));
+	return;
+#endif /* ENABLE_NOT_LOAD_DHD_MODULE */
+	printf("%s: Enter\n", __FUNCTION__);
+
+#ifdef DHD_SR_FIX
+	ret = dhd_wl_ioctl_cmd(g_dhd_pub, WLC_DOWN,
+		(char *)&wl_down, sizeof(wl_down), TRUE, 0);
+	DHD_ERROR(("%s WL_DOWN : %d\n", __FUNCTION__, ret));
+#endif /* DHD_SR_FIX */
+
+	dhd_bus_unregister();
+
+#if defined(OEM_ANDROID)
+	wl_android_exit();
+#endif /* OEM_ANDROID */
+
+	dhd_wifi_platform_unregister_drv();
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_STATIC_IN_DRIVER)
+	dhd_static_buf_exit();
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_STATIC_IN_DRIVER */
+	printf("%s: Exit\n", __FUNCTION__);
+}
+
+static void __exit
+dhd_module_exit(void)
+{
+#if defined(ENABLE_NOT_LOAD_DHD_MODULE)
+	DHD_ERROR(("%s ##### Do not unload driver due to secondary build\n", __FUNCTION__));
+	return;
+#endif /* ENABLE_NOT_LOAD_DHD_MODULE */
+	atomic_set(&exit_in_progress, 1);
+#ifdef DHD_BUZZZ_LOG_ENABLED
+	dhd_buzzz_detach();
+#endif /* DHD_BUZZZ_LOG_ENABLED */
+	dhd_module_cleanup();
+	unregister_reboot_notifier(&dhd_reboot_notifier);
+	dhd_destroy_to_notifier_skt();
+}
+
+#if defined(CONFIG_ARCH_WAIPIO) || defined(CONFIG_SOC_S5E9925)
+extern bool devid_mismatch;
+#endif /* CONFIG_ARCH_WAIPIO || CONFIG_SOC_S5E9925 */
+
+static int
+_dhd_module_init(void)
+{
+	int err;
+	int retry = POWERUP_MAX_RETRY;
+
+	DHD_INFO((KERN_ERR PERCENT_S DHD_LOG_PREFIXS "%s: in %s\n",
+		PRINTF_SYSTEM_TIME, __FUNCTION__, dhd_version));
+	if (ANDROID_VERSION > 0)
+		DHD_INFO(("ANDROID_VERSION = %d\n", ANDROID_VERSION));
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_STATIC_IN_DRIVER)
+	err = dhd_static_buf_init();
+	if (err) {
+		goto exit;
+	}
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_STATIC_IN_DRIVER */
+
+#ifdef DHD_BUZZZ_LOG_ENABLED
+	dhd_buzzz_attach();
+#endif /* DHD_BUZZZ_LOG_ENABLED */
+
+#if defined(BCM_ROUTER_DHD)
+	{	/* XXX Should we maintain nvram budget/thresholds per 5G|2G radio? */
+		char * var;
+		if ((var = getvar(NULL, "dhd_queue_budget")) != NULL) {
+			dhd_queue_budget = bcm_strtoul(var, NULL, 0);
+		}
+		DHD_ERROR(("dhd_queue_budget = %d\n", dhd_queue_budget));
+
+		if ((var = getvar(NULL, "dhd_sta_threshold")) != NULL) {
+			dhd_sta_threshold = bcm_strtoul(var, NULL, 0);
+		}
+		DHD_ERROR(("dhd_sta_threshold = %d\n", dhd_sta_threshold));
+
+		if ((var = getvar(NULL, "dhd_if_threshold")) != NULL) {
+			dhd_if_threshold = bcm_strtoul(var, NULL, 0);
+		}
+		DHD_ERROR(("dhd_if_threshold = %d\n", dhd_if_threshold));
+	}
+#endif /* BCM_ROUTER_DHD */
+
+	if (firmware_path[0] != '\0') {
+		strlcpy(fw_bak_path, firmware_path, sizeof(fw_bak_path));
+	}
+
+	if (nvram_path[0] != '\0') {
+		strlcpy(nv_bak_path, nvram_path, sizeof(nv_bak_path));
+	}
+
+	do {
+		err = dhd_wifi_platform_register_drv();
+		if (!err) {
+			register_reboot_notifier(&dhd_reboot_notifier);
+			dhd_create_to_notifier_skt();
+			break;
+		} else {
+			if (err == -ENXIO) {
+				DHD_ERROR(("%s: driver is disabled\n", __FUNCTION__));
+				break;
+			}
+#if defined(CONFIG_ARCH_WAIPIO) || defined(CONFIG_SOC_S5E9925)
+			if (devid_mismatch == TRUE) {
+				DHD_ERROR(("%s: Devid is mismatch, don't retry\n",
+						__FUNCTION__));
+				break;
+			}
+#endif /* CONFIG_ARCH_WAIPIO || CONFIG_SOC_S5E9925 */
+			DHD_ERROR(("%s: Failed to load the driver, try cnt %d\n",
+				__FUNCTION__, retry));
+			strlcpy(firmware_path, fw_bak_path, sizeof(firmware_path));
+			strlcpy(nvram_path, nv_bak_path, sizeof(nvram_path));
+		}
+	} while (retry--);
+
+#if defined(CONFIG_ARCH_WAIPIO) || defined(CONFIG_SOC_S5E9925)
+	if (devid_mismatch == TRUE) {
+		DHD_ERROR(("%s: Return true if devid is mismatch \n", __FUNCTION__));
+		return 0;
+	}
+#endif /* CONFIG_ARCH_WAIPIO || CONFIG_SOC_S5E9925 */
+
+	if (err) {
+		DHD_ERROR(("%s: Failed to load driver max retry reached**\n", __FUNCTION__));
+	} else {
+		if (!dhd_download_fw_on_driverload) {
+			dhd_driver_init_done = TRUE;
+		}
+	}
+
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_STATIC_IN_DRIVER)
+exit:
+	if (err)
+		dhd_static_buf_exit();
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_STATIC_IN_DRIVER */
+	DHD_INFO(("%s: Exit err=%d\n", __FUNCTION__, err));
+	return err;
+}
+
+static int
+dhd_module_init(void)
+{
+	int err;
+
+#if defined(ENABLE_NOT_LOAD_DHD_MODULE)
+	DHD_ERROR(("%s ##### Do not load driver due to secondary build\n", __FUNCTION__));
+	return 0;
+#endif /* ENABLE_NOT_LOAD_DHD_MODULE */
+	err = _dhd_module_init();
+#ifdef DHD_SUPPORT_HDM
+	if (hdm_wifi_support && err && !dhd_download_fw_on_driverload) {
+		dhd_hdm_wlan_sysfs_init();
+		err = 0;
+	}
+#endif /* DHD_SUPPORT_HDM */
+#if defined(BCMDHD_MODULAR) && defined(DHD_MODULE_INIT_FORCE_SUCCESS)
+	err = 0;
+#endif /* BCMDHD_MODULAR && DHD_MODULE_INIT_FORCE_SUCCESS */
+	return err;
+
+}
+
+#ifdef DHD_SUPPORT_HDM
+bool hdm_trigger_init = FALSE;
+struct delayed_work hdm_sysfs_wq;
+
+int
+dhd_module_init_hdm(void)
+{
+	int err = 0;
+
+	hdm_trigger_init = TRUE;
+
+	if (dhd_driver_init_done) {
+		DHD_INFO(("%s : Module is already inited\n", __FUNCTION__));
+		return err;
+	}
+
+	err = _dhd_module_init();
+
+	/* remove sysfs file after module load properly */
+	if (!err && !dhd_download_fw_on_driverload) {
+		INIT_DELAYED_WORK(&hdm_sysfs_wq, dhd_hdm_wlan_sysfs_deinit);
+		schedule_delayed_work(&hdm_sysfs_wq, msecs_to_jiffies(SYSFS_DEINIT_MS));
+	}
+
+	hdm_trigger_init = FALSE;
+	return err;
+}
+#endif /* DHD_SUPPORT_HDM */
+
+static int
+dhd_reboot_callback(struct notifier_block *this, unsigned long code, void *unused)
+{
+	dhd_pub_t *dhdp = g_dhd_pub;
+
+	BCM_REFERENCE(dhdp);
+	DHD_ERROR(("%s: code = %ld\n", __FUNCTION__, code));
+#ifdef OEM_ANDROID
+	if (!OSL_ATOMIC_INC_AND_TEST(dhdp->osh, &reboot_in_progress)) {
+		DHD_ERROR(("%s: Skip duplicated reboot callback!\n", __FUNCTION__));
+		return NOTIFY_DONE;
+	}
+
+	dhd_module_cleanup();
+#else
+	if (code == SYS_RESTART) {
+		dhd_module_cleanup();
+	}
+#endif /* OEM_ANDROID */
+	return NOTIFY_DONE;
+}
+
+#if defined(CONFIG_DEFERRED_INITCALLS) && !defined(EXYNOS_PCIE_MODULE_PATCH)
+/* XXX To decrease the device boot time, deferred_module_init() macro can be
+ * used. The detailed principle and implemenation of deferred_module_init()
+ * is found at http://elinux.org/Deferred_Initcalls
+ * To enable this feature for module build, it needs to add another
+ * deferred_module_init() definition to include/linux/init.h in Linux Kernel.
+ * #define deferred_module_init(fn)	module_init(fn)
+ */
+#if defined(CONFIG_ARCH_MSM) || defined(CONFIG_ARCH_EXYNOS)
+deferred_module_init_sync(dhd_module_init);
+#else
+deferred_module_init(dhd_module_init);
+#endif /* CONFIG_ARCH_MSM || CONFIG_ARCH_EXYNOS */
+#elif defined(USE_LATE_INITCALL_SYNC)
+late_initcall_sync(dhd_module_init);
+#else
+late_initcall(dhd_module_init);
+#endif /* USE_LATE_INITCALL_SYNC */
+
+module_exit(dhd_module_exit);
+
+/* For the Exynos platform, the WLAN_REG_ON pin is connected to an expander GPIO such as PMIC.
+ * The WLAN_REG_ON pin initialization failed
+ * if the Wi-Fi module is loaded before the PMIC module is loaded.
+ * Add the MODULE_SOFTDEP macro to load the Wi-Fi module after the PMIC module is loaded.
+ */
+#if defined(CONFIG_SOC_S5E9925)
+MODULE_SOFTDEP("pre: acpm-mfd-bus");
+#endif /* CONFIG_SOC_S5E9925 */
+
+/*
+ * OS specific functions required to implement DHD driver in OS independent way
+ */
+int
+dhd_os_proto_block(dhd_pub_t *pub)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		down(&dhd->proto_sem);
+
+		return 1;
+	}
+
+	return 0;
+}
+
+int
+dhd_os_proto_unblock(dhd_pub_t *pub)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		up(&dhd->proto_sem);
+		return 1;
+	}
+
+	return 0;
+}
+
+void
+dhd_os_dhdiovar_lock(dhd_pub_t *pub)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		mutex_lock(&dhd->dhd_iovar_mutex);
+	}
+}
+
+void
+dhd_os_dhdiovar_unlock(dhd_pub_t *pub)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		mutex_unlock(&dhd->dhd_iovar_mutex);
+	}
+}
+
+void
+dhd_os_logdump_lock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = NULL;
+
+	if (!pub)
+		return;
+
+	dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		mutex_lock(&dhd->logdump_lock);
+	}
+}
+
+void
+dhd_os_logdump_unlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = NULL;
+
+	if (!pub)
+		return;
+
+	dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		mutex_unlock(&dhd->logdump_lock);
+	}
+}
+
+unsigned int
+dhd_os_get_ioctl_resp_timeout(void)
+{
+	return ((unsigned int)dhd_ioctl_timeout_msec);
+}
+
+void
+dhd_os_set_ioctl_resp_timeout(unsigned int timeout_msec)
+{
+	dhd_ioctl_timeout_msec = (int)timeout_msec;
+}
+
+int
+dhd_os_ioctl_resp_wait(dhd_pub_t *pub, uint *condition)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(dhd_ioctl_timeout_msec);
+
+#ifdef BCMQT_HW
+	DHD_ERROR(("%s, Timeout wait until %d mins (%d ms) in QT mode\n",
+		__FUNCTION__, (dhd_ioctl_timeout_msec / (60 * 1000)), dhd_ioctl_timeout_msec));
+#endif /* BCMQT_HW */
+
+	timeout = wait_event_timeout(dhd->ioctl_resp_wait, (*condition), timeout);
+
+	return timeout;
+}
+
+int
+dhd_os_ioctl_resp_wake(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	wake_up(&dhd->ioctl_resp_wait);
+	return 0;
+}
+
+int
+dhd_os_d3ack_wait(dhd_pub_t *pub, uint *condition)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(D3_ACK_RESP_TIMEOUT);
+#ifdef BCMSLTGT
+	timeout *= htclkratio;
+#endif /* BCMSLTGT */
+
+	timeout = wait_event_timeout(dhd->d3ack_wait, (*condition), timeout);
+
+	return timeout;
+}
+
+#ifdef PCIE_INB_DW
+int
+dhd_os_ds_exit_wait(dhd_pub_t *pub, uint *condition)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(ds_exit_timeout_msec);
+#ifdef BCMSLTGT
+	timeout *= htclkratio;
+#endif /* BCMSLTGT */
+
+	timeout = wait_event_timeout(dhd->ds_exit_wait, (*condition), timeout);
+
+	return timeout;
+}
+
+int
+dhd_os_ds_exit_wake(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	wake_up_all(&dhd->ds_exit_wait);
+	return 0;
+}
+
+#endif /* PCIE_INB_DW */
+
+int
+dhd_os_d3ack_wake(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	wake_up(&dhd->d3ack_wait);
+	return 0;
+}
+
+int
+dhd_os_busbusy_wait_negation(dhd_pub_t *pub, uint *condition)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	/* Wait for bus usage contexts to gracefully exit within some timeout value
+	 * Set time out to little higher than dhd_ioctl_timeout_msec,
+	 * so that IOCTL timeout should not get affected.
+	 */
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(DHD_BUS_BUSY_TIMEOUT);
+
+	timeout = wait_event_timeout(dhd->dhd_bus_busy_state_wait, !(*condition), timeout);
+
+	return timeout;
+}
+
+/*
+ * Wait until the condition *var == condition is met.
+ * Returns 0 if the @condition evaluated to false after the timeout elapsed
+ * Returns 1 if the @condition evaluated to true
+ */
+int
+dhd_os_busbusy_wait_condition(dhd_pub_t *pub, uint *var, uint condition)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(DHD_BUS_BUSY_TIMEOUT);
+
+	timeout = wait_event_timeout(dhd->dhd_bus_busy_state_wait, (*var == condition), timeout);
+
+	return timeout;
+}
+
+/*
+ * Wait until the '(*var & bitmask) == condition' is met.
+ * Returns 0 if the @condition evaluated to false after the timeout elapsed
+ * Returns 1 if the @condition evaluated to true
+ */
+int
+dhd_os_busbusy_wait_bitmask(dhd_pub_t *pub, uint *var,
+		uint bitmask, uint condition)
+{
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	/* Convert timeout in millsecond to jiffies */
+	timeout = msecs_to_jiffies(DHD_BUS_BUSY_TIMEOUT);
+
+	timeout = wait_event_timeout(dhd->dhd_bus_busy_state_wait,
+			((*var & bitmask) == condition), timeout);
+
+	return timeout;
+}
+
+int
+dhd_os_dmaxfer_wait(dhd_pub_t *pub, uint *condition)
+{
+	int ret = 0;
+	dhd_info_t * dhd = (dhd_info_t *)(pub->info);
+	int timeout;
+
+	timeout = msecs_to_jiffies(IOCTL_DMAXFER_TIMEOUT);
+
+	ret = wait_event_timeout(dhd->dmaxfer_wait, (*condition), timeout);
+
+	return ret;
+
+}
+
+int
+dhd_os_dmaxfer_wake(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	wake_up(&dhd->dmaxfer_wait);
+	return 0;
+}
+
+/* Fix compilation error for FC11 */
+INLINE int
+dhd_os_busbusy_wake(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	/* Call wmb() to make sure before waking up the other event value gets updated */
+	OSL_SMP_WMB();
+	wake_up(&dhd->dhd_bus_busy_state_wait);
+	return 0;
+}
+
+void
+dhd_os_wd_timer_extend(void *bus, bool extend)
+{
+#ifndef BCMDBUS
+	dhd_pub_t *pub = bus;
+	dhd_info_t *dhd = (dhd_info_t *)pub->info;
+
+	if (extend)
+		dhd_os_wd_timer(bus, WATCHDOG_EXTEND_INTERVAL);
+	else
+		dhd_os_wd_timer(bus, dhd->default_wd_interval);
+#endif /* !BCMDBUS */
+}
+
+void
+dhd_os_wd_timer(void *bus, uint wdtick)
+{
+#ifndef BCMDBUS
+	dhd_pub_t *pub = bus;
+	dhd_info_t *dhd = (dhd_info_t *)pub->info;
+	unsigned long flags;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd NULL\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_GENERAL_LOCK(pub, flags);
+
+	/* don't start the wd until fw is loaded */
+	if (pub->busstate == DHD_BUS_DOWN) {
+		DHD_GENERAL_UNLOCK(pub, flags);
+#ifdef BCMSDIO
+		if (!wdtick) {
+			DHD_OS_WD_WAKE_UNLOCK(pub);
+		}
+#endif /* BCMSDIO */
+		return;
+	}
+
+	/* Totally stop the timer */
+	if (!wdtick && dhd->wd_timer_valid == TRUE) {
+		dhd->wd_timer_valid = FALSE;
+		DHD_GENERAL_UNLOCK(pub, flags);
+		del_timer_sync(&dhd->timer);
+#ifdef BCMSDIO
+		DHD_OS_WD_WAKE_UNLOCK(pub);
+#endif /* BCMSDIO */
+		return;
+	}
+
+	if (wdtick) {
+#ifdef BCMSDIO
+		DHD_OS_WD_WAKE_LOCK(pub);
+		dhd_watchdog_ms = (uint)wdtick;
+#endif /* BCMSDIO */
+		/* Re arm the timer, at last watchdog period */
+		mod_timer(&dhd->timer, jiffies + msecs_to_jiffies(dhd_watchdog_ms));
+		dhd->wd_timer_valid = TRUE;
+	}
+	DHD_GENERAL_UNLOCK(pub, flags);
+#endif /* BCMDBUS */
+}
+
+#ifdef DHD_LINUX_STD_FW_API
+int
+dhd_os_get_img_fwreq(const struct firmware **fw, char *file_path)
+{
+	int ret = BCME_ERROR;
+
+	ret = request_firmware(fw, file_path, dhd_bus_to_dev(g_dhd_pub->bus));
+	if (ret < 0) {
+		DHD_ERROR(("%s: request_firmware %s err: %d\n", __FUNCTION__, file_path, ret));
+		/* convert to BCME_NOTFOUND error for error handling */
+		ret = BCME_NOTFOUND;
+	} else
+		DHD_ERROR(("%s: %s (%zu bytes) open success\n", __FUNCTION__, file_path, (*fw)->size));
+
+	return ret;
+}
+
+void
+dhd_os_close_img_fwreq(const struct firmware *fw)
+{
+	release_firmware(fw);
+}
+#endif /* DHD_LINUX_STD_FW_API */
+
+void *
+dhd_os_open_image1(dhd_pub_t *pub, char *filename)
+{
+	struct file *fp;
+	int size;
+
+	fp = dhd_filp_open(filename, O_RDONLY, 0);
+
+	/*
+	 * 2.6.11 (FC4) supports dhd_filp_open() but later revs don't?
+	 * Alternative:
+	 * fp = open_namei(AT_FDCWD, filename, O_RD, 0);
+	 * ???
+	 */
+	 if (IS_ERR(fp) || (fp == NULL)) {
+		 fp = NULL;
+		 goto err;
+	 }
+
+	 if (!S_ISREG(file_inode(fp)->i_mode)) {
+		 DHD_ERROR(("%s: %s is not regular file\n", __FUNCTION__, filename));
+		 fp = NULL;
+		 goto err;
+	 }
+
+	 size = dhd_i_size_read(file_inode(fp));
+	 if (size <= 0) {
+		 DHD_ERROR(("%s: %s file size invalid %d\n", __FUNCTION__, filename, size));
+		 fp = NULL;
+		 goto err;
+	 }
+
+	 DHD_INFO(("%s: %s (%d bytes) open success\n", __FUNCTION__, filename, size));
+
+err:
+	 return fp;
+}
+
+int
+dhd_os_get_image_block(char *buf, int len, void *image)
+{
+	struct file *fp = (struct file *)image;
+	int rdlen;
+	int size;
+
+	if (!image) {
+		return 0;
+	}
+
+	size = dhd_i_size_read(file_inode(fp));
+	rdlen = dhd_kernel_read_compat(fp, fp->f_pos, buf, MIN(len, size));
+
+	if (len >= size && size != rdlen) {
+		return -EIO;
+	}
+
+	if (rdlen > 0) {
+		fp->f_pos += rdlen;
+	}
+
+	return rdlen;
+}
+
+#if defined(BT_OVER_SDIO)
+int
+dhd_os_gets_image(dhd_pub_t *pub, char *str, int len, void *image)
+{
+	struct file *fp = (struct file *)image;
+	int rd_len;
+	uint str_len = 0;
+	char *str_end = NULL;
+
+	if (!image)
+		return 0;
+
+	rd_len = dhd_kernel_read_compat(fp, fp->f_pos, str, len);
+	str_end = strnchr(str, len, '\n');
+	if (str_end == NULL) {
+		goto err;
+	}
+	str_len = (uint)(str_end - str);
+
+	/* Advance file pointer past the string length */
+	fp->f_pos += str_len + 1;
+	bzero(str_end, rd_len - str_len);
+
+err:
+	return str_len;
+}
+#endif /* defined (BT_OVER_SDIO) */
+
+int
+dhd_os_get_image_size(void *image)
+{
+	struct file *fp = (struct file *)image;
+	int size;
+	if (!image) {
+		return 0;
+	}
+
+	size = dhd_i_size_read(file_inode(fp));
+
+	return size;
+}
+
+void
+dhd_os_close_image1(dhd_pub_t *pub, void *image)
+{
+	if (image) {
+		dhd_filp_close((struct file *)image, NULL);
+	}
+}
+
+void
+dhd_os_sdlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+
+#ifdef BCMDBUS
+	spin_lock_bh(&dhd->sdlock);
+#else
+	if (dhd_dpc_prio >= 0)
+		down(&dhd->sdsem);
+	else
+		spin_lock_bh(&dhd->sdlock);
+#endif /* !BCMDBUS */
+}
+
+void
+dhd_os_sdunlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+
+#ifdef BCMDBUS
+	spin_unlock_bh(&dhd->sdlock);
+#else
+	if (dhd_dpc_prio >= 0)
+		up(&dhd->sdsem);
+	else
+		spin_unlock_bh(&dhd->sdlock);
+#endif /* !BCMDBUS */
+}
+
+#ifdef DHDTCPACK_SUPPRESS
+unsigned long
+dhd_os_tcpacklock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+	unsigned long flags = 0;
+
+	dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+#ifdef BCMSDIO
+		spin_lock_bh(&dhd->tcpack_lock);
+#else
+		flags = osl_spin_lock(&dhd->tcpack_lock);
+#endif /* BCMSDIO */
+	}
+
+	return flags;
+}
+
+void
+dhd_os_tcpackunlock(dhd_pub_t *pub, unsigned long flags)
+{
+	dhd_info_t *dhd;
+
+#ifdef BCMSDIO
+	BCM_REFERENCE(flags);
+#endif /* BCMSDIO */
+
+	dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+#ifdef BCMSDIO
+		spin_unlock_bh(&dhd->tcpack_lock);
+#else
+		osl_spin_unlock(&dhd->tcpack_lock, flags);
+#endif /* BCMSDIO */
+	}
+}
+#endif /* DHDTCPACK_SUPPRESS */
+
+uint8* dhd_os_prealloc(dhd_pub_t *dhdpub, int section, uint size, bool kmalloc_if_fail)
+{
+	uint8* buf;
+	gfp_t flags = CAN_SLEEP() ? GFP_KERNEL: GFP_ATOMIC;
+
+	buf = (uint8*)wifi_platform_prealloc(dhdpub->info->adapter, section, size);
+	if (buf == NULL && kmalloc_if_fail)
+		buf = kmalloc(size, flags);
+
+	return buf;
+}
+
+void dhd_os_prefree(dhd_pub_t *dhdpub, void *addr, uint size)
+{
+}
+
+#if defined(WL_WIRELESS_EXT)
+struct iw_statistics *
+dhd_get_wireless_stats(struct net_device *dev)
+{
+	int res = 0;
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	if (!dhd->pub.up) {
+		return NULL;
+	}
+
+	if (!(dev->flags & IFF_UP)) {
+		return NULL;
+	}
+
+	res = wl_iw_get_wireless_stats(dev, &dhd->iw.wstats);
+
+	if (res == 0)
+		return &dhd->iw.wstats;
+	else
+		return NULL;
+}
+#endif /* defined(WL_WIRELESS_EXT) */
+
+/* send up locally generated event */
+void
+dhd_sendup_event(dhd_pub_t *dhdp, wl_event_msg_t *event, void *data)
+{
+	switch (ntoh32(event->event_type)) {
+	/* Handle error case or further events here */
+	default:
+		break;
+	}
+}
+
+#ifdef LOG_INTO_TCPDUMP
+void
+dhd_sendup_log(dhd_pub_t *dhdp, void *data, int data_len)
+{
+	struct sk_buff *p, *skb;
+	uint32 pktlen;
+	int len;
+	dhd_if_t *ifp;
+	dhd_info_t *dhd;
+	uchar *skb_data;
+	int ifidx = 0;
+	struct ether_header eth;
+
+	pktlen = sizeof(eth) + data_len;
+	dhd = dhdp->info;
+
+	if ((p = PKTGET(dhdp->osh, pktlen, FALSE))) {
+		ASSERT(ISALIGNED((uintptr)PKTDATA(dhdp->osh, p), sizeof(uint32)));
+
+		bcopy(&dhdp->mac, &eth.ether_dhost, ETHER_ADDR_LEN);
+		bcopy(&dhdp->mac, &eth.ether_shost, ETHER_ADDR_LEN);
+		ETHER_TOGGLE_LOCALADDR(&eth.ether_shost);
+		eth.ether_type = hton16(ETHER_TYPE_BRCM);
+
+		bcopy((void *)&eth, PKTDATA(dhdp->osh, p), sizeof(eth));
+		bcopy(data, PKTDATA(dhdp->osh, p) + sizeof(eth), data_len);
+		skb = PKTTONATIVE(dhdp->osh, p);
+		skb_data = skb->data;
+		len = skb->len;
+
+		ifidx = dhd_ifname2idx(dhd, "wlan0");
+		ifp = dhd->iflist[ifidx];
+		if (ifp == NULL)
+			 ifp = dhd->iflist[0];
+
+		ASSERT(ifp);
+		skb->dev = ifp->net;
+		skb->protocol = eth_type_trans(skb, skb->dev);
+		skb->data = skb_data;
+		skb->len = len;
+
+		/* Strip header, count, deliver upward */
+		skb_pull(skb, ETH_HLEN);
+
+		bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+			__FUNCTION__, __LINE__);
+		/* Send the packet */
+		if (in_interrupt()) {
+			netif_rx(skb);
+		} else {
+			dhd_netif_rx_ni(skb);
+		}
+	} else {
+		/* Could not allocate a sk_buf */
+		DHD_ERROR(("%s: unable to alloc sk_buf\n", __FUNCTION__));
+	}
+}
+#endif /* LOG_INTO_TCPDUMP */
+
+void dhd_wait_for_event(dhd_pub_t *dhd, bool *lockvar)
+{
+#if defined(BCMSDIO)
+	struct dhd_info *dhdinfo =  dhd->info;
+
+	int timeout = msecs_to_jiffies(IOCTL_RESP_TIMEOUT);
+
+	dhd_os_sdunlock(dhd);
+	wait_event_timeout(dhdinfo->ctrl_wait, (*lockvar == FALSE), timeout);
+	dhd_os_sdlock(dhd);
+#endif /* defined(BCMSDIO) */
+	return;
+}
+
+void dhd_wait_event_wakeup(dhd_pub_t *dhd)
+{
+#if defined(BCMSDIO)
+	struct dhd_info *dhdinfo =  dhd->info;
+	if (waitqueue_active(&dhdinfo->ctrl_wait))
+		wake_up(&dhdinfo->ctrl_wait);
+#endif
+	return;
+}
+
+#if defined(BCMSDIO) || defined(BCMPCIE) || defined(BCMDBUS)
+int
+dhd_net_bus_devreset(struct net_device *dev, uint8 flag)
+{
+	int ret;
+
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	if (pm_runtime_get_sync(dhd_bus_to_dev(dhd->pub.bus)) < 0)
+		return BCME_ERROR;
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	if (flag == TRUE) {
+#ifndef WL_CFG80211
+		/* Issue wl down command for non-cfg before resetting the chip */
+		if (dhd_wl_ioctl_cmd(&dhd->pub, WLC_DOWN, NULL, 0, TRUE, 0) < 0) {
+			DHD_TRACE(("%s: wl down failed\n", __FUNCTION__));
+		}
+#endif /* !WL_CFG80211 */
+#ifdef PROP_TXSTATUS
+		if (dhd->pub.wlfc_enabled) {
+			dhd_wlfc_deinit(&dhd->pub);
+		}
+#endif /* PROP_TXSTATUS */
+#ifdef PNO_SUPPORT
+		if (dhd->pub.pno_state) {
+			dhd_pno_deinit(&dhd->pub);
+		}
+#endif
+#ifdef RTT_SUPPORT
+		if (dhd->pub.rtt_state) {
+			dhd_rtt_deinit(&dhd->pub);
+		}
+#endif /* RTT_SUPPORT */
+
+		DHD_SSSR_DUMP_DEINIT(&dhd->pub);
+#ifdef DHD_SDTC_ETB_DUMP
+		if (dhd->pub.sdtc_etb_inited) {
+			dhd_sdtc_etb_deinit(&dhd->pub);
+		}
+#endif /* DHD_SDTC_ETB_DUMP */
+/*
+ * XXX Detach only if the module is not attached by default at dhd_attach.
+ * If attached by default, we need to keep it till dhd_detach, so that
+ * module is not detached at wifi on/off
+ */
+#if defined(DBG_PKT_MON) && !defined(DBG_PKT_MON_INIT_DEFAULT)
+		dhd_os_dbg_detach_pkt_monitor(&dhd->pub);
+#endif /* DBG_PKT_MON */
+	}
+
+#ifdef BCMSDIO
+	/* XXX Some DHD modules (e.g. cfg80211) configures operation mode based on firmware name.
+	 * This is indeed a hack but we have to make it work properly before we have a better
+	 * solution
+	 */
+	if (!flag) {
+		dhd_update_fw_nv_path(dhd);
+		/* update firmware and nvram path to sdio bus */
+		dhd_bus_update_fw_nv_path(dhd->pub.bus,
+			dhd->fw_path, dhd->nv_path, dhd->clm_path, dhd->conf_path);
+	}
+#endif /* BCMSDIO */
+#if defined(BCMPCIE)
+	/* XXX: JIRA SWWLAN-139454: Added L1ss enable
+	 * after firmware download completion due to link down issue
+	 * JIRA SWWLAN-142236: Amendment - Changed L1ss enable point
+	 */
+	if (flag == FALSE && dhd->pub.busstate == DHD_BUS_DOWN) {
+		DHD_RPM(("%s Disable L1ss EP side\n", __FUNCTION__));
+		dhd_plat_l1ss_ctrl(0);
+	}
+#endif /* BCMPCIE */
+
+	ret = dhd_bus_devreset(&dhd->pub, flag);
+
+	if (!ret && dhd_query_bus_erros(&dhd->pub)) {
+		DHD_ERROR(("%s: retrun error due to query errors\n", __FUNCTION__));
+		ret = BCME_ERROR;
+	}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	pm_runtime_mark_last_busy(dhd_bus_to_dev(dhd->pub.bus));
+	pm_runtime_put_autosuspend(dhd_bus_to_dev(dhd->pub.bus));
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	if (flag) {
+		/* Clear some flags for recovery logic */
+		dhd->pub.dongle_trap_occured = 0;
+		dhd->pub.dsack_hc_due_to_isr_delay = 0;
+		dhd->pub.dsack_hc_due_to_dpc_delay = 0;
+		dhd->pub.iovar_timeout_occured = 0;
+#ifdef PCIE_FULL_DONGLE
+		dhd->pub.d3ack_timeout_occured = 0;
+		dhd->pub.livelock_occured = 0;
+		dhd->pub.pktid_audit_failed = 0;
+		dhd->pub.pktid_invalid_occured = 0;
+#endif /* PCIE_FULL_DONGLE */
+		dhd->pub.smmu_fault_occurred = 0;
+		dhd->pub.iface_op_failed = 0;
+		dhd->pub.scan_timeout_occurred = 0;
+		dhd->pub.scan_busy_occurred = 0;
+		dhd->pub.p2p_disc_busy_occurred = 0;
+	}
+
+	dhd->pub.p2p_disc_busy_cnt = 0;
+
+	if (ret == BCME_NOMEM) {
+		DHD_ERROR(("%s: skip collect dump in case of BCME_NOMEM\n",
+				__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if ((ret == BCME_NOMEM) || (ret == BCME_NOTFOUND)) {
+		DHD_ERROR(("%s: skip collect dump in case of BCME_NOMEM or BCME_NOTFOUND\n",
+				__FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (ret) {
+		DHD_ERROR(("%s: dhd_bus_devreset: %d\n", __FUNCTION__, ret));
+		dhd_force_collect_init_fail_dumps(&dhd->pub);
+	}
+
+	return ret;
+}
+
+#if defined(BCMSDIO) || defined(BCMPCIE)
+int
+dhd_net_bus_suspend(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	uint bitmask = 0xFFFFFFFF;
+	int timeleft = 0;
+	unsigned long flags = 0;
+	int ret = 0;
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	if (!DHD_BUS_BUSY_CHECK_IDLE(&dhd->pub)) {
+		DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		DHD_ERROR(("%s: wait to clear dhd_bus_busy_state: 0x%x\n",
+			__FUNCTION__, dhd->pub.dhd_bus_busy_state));
+		timeleft = dhd_os_busbusy_wait_bitmask(&dhd->pub,
+				&dhd->pub.dhd_bus_busy_state, bitmask,
+				DHD_BUS_BUSY_SUSPEND_IN_PROGRESS);
+		if ((timeleft == 0) || (timeleft == 1)) {
+			DHD_ERROR(("%s: Timed out dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhd->pub.dhd_bus_busy_state));
+			ASSERT(0);
+		}
+	} else {
+		DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+	}
+
+	ret = dhd_bus_suspend(&dhd->pub);
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	DHD_BUS_BUSY_CLEAR_SUSPEND_IN_PROGRESS(&dhd->pub);
+	dhd_os_busbusy_wake(&dhd->pub);
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+
+	return ret;
+}
+
+int
+dhd_net_bus_resume(struct net_device *dev, uint8 stage)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	unsigned long flags = 0;
+	int ret = 0;
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	DHD_BUS_BUSY_SET_RESUME_IN_PROGRESS(&dhd->pub);
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+
+	ret = dhd_bus_resume(&dhd->pub, stage);
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	DHD_BUS_BUSY_CLEAR_RESUME_IN_PROGRESS(&dhd->pub);
+	dhd_os_busbusy_wake(&dhd->pub);
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+
+	return ret;
+}
+
+#endif /* BCMSDIO || BCMPCIE */
+#endif /* BCMSDIO || BCMPCIE || BCMDBUS */
+
+int net_os_set_suspend_disable(struct net_device *dev, int val)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd) {
+		ret = dhd->pub.suspend_disable_flag;
+		dhd->pub.suspend_disable_flag = val;
+	}
+	return ret;
+}
+
+int net_os_set_suspend(struct net_device *dev, int val, int force)
+{
+	int ret = 0;
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	if (dhd && (dhd->pub.conf->suspend_mode == EARLY_SUSPEND ||
+			dhd->pub.conf->suspend_mode == SUSPEND_MODE_2)) {
+		if (dhd->pub.conf->suspend_mode == EARLY_SUSPEND && !val)
+			dhd_conf_set_suspend_resume(&dhd->pub, val);
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+		ret = dhd_set_suspend(val, &dhd->pub);
+#else
+		ret = dhd_suspend_resume_helper(dhd, val, force);
+#endif
+#ifdef WL_CFG80211
+		wl_cfg80211_update_power_mode(dev);
+#endif
+		if (dhd->pub.conf->suspend_mode == EARLY_SUSPEND && val)
+			dhd_conf_set_suspend_resume(&dhd->pub, val);
+	}
+	return ret;
+}
+
+#ifdef PKT_FILTER_SUPPORT
+/* XXX RB:4238 Change net_os_set_packet_filter() function name to net_os_enable_packet_filter()
+ * previous code do 'set' & 'enable' in one fucntion.
+ * but from now on, we are going to separate 'set' and 'enable' feature.
+ *  - set : net_os_rxfilter_add_remove() -> dhd_set_packet_filter() -> dhd_pktfilter_offload_set()
+ *  - enable : net_os_enable_packet_filter() -> dhd_enable_packet_filter()
+ *                                                              -> dhd_pktfilter_offload_enable()
+ */
+int dhd_os_enable_packet_filter(dhd_pub_t *dhdp, int val)
+
+{
+	int ret = 0;
+
+	/* Packet filtering is set only if we still in early-suspend and
+	 * we need either to turn it ON or turn it OFF
+	 * We can always turn it OFF in case of early-suspend, but we turn it
+	 * back ON only if suspend_disable_flag was not set
+	*/
+	if (dhdp && dhdp->up) {
+		if (dhdp->in_suspend) {
+			if (!val || (val && !dhdp->suspend_disable_flag))
+				dhd_enable_packet_filter(val, dhdp);
+		}
+	}
+	return ret;
+}
+
+/* function to enable/disable packet for Network device */
+int net_os_enable_packet_filter(struct net_device *dev, int val)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	DHD_INFO(("%s: val = %d\n", __FUNCTION__, val));
+	return dhd_os_enable_packet_filter(&dhd->pub, val);
+}
+#endif /* PKT_FILTER_SUPPORT */
+
+int
+dhd_dev_init_ioctl(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret;
+
+	if ((ret = dhd_sync_with_dongle(&dhd->pub)) < 0)
+		goto done;
+
+done:
+	return ret;
+}
+
+int
+dhd_dev_get_feature_set(struct net_device *dev)
+{
+	dhd_info_t *ptr = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhd = (&ptr->pub);
+	int feature_set = 0;
+
+	/* tdls capability or othters can be missed because of initialization */
+	if (dhd_get_fw_capabilities(dhd) < 0) {
+		DHD_ERROR(("Capabilities rechecking fail\n"));
+	}
+
+	if (FW_SUPPORTED(dhd, sta))
+		feature_set |= WIFI_FEATURE_INFRA;
+	if (FW_SUPPORTED(dhd, dualband))
+		feature_set |= WIFI_FEATURE_INFRA_5G;
+	if (FW_SUPPORTED(dhd, p2p))
+		feature_set |= WIFI_FEATURE_P2P;
+	if (dhd->op_mode & DHD_FLAG_HOSTAP_MODE)
+		feature_set |= WIFI_FEATURE_SOFT_AP;
+	if (FW_SUPPORTED(dhd, tdls))
+		feature_set |= WIFI_FEATURE_TDLS;
+	if (FW_SUPPORTED(dhd, vsdb))
+		feature_set |= WIFI_FEATURE_TDLS_OFFCHANNEL;
+	if (FW_SUPPORTED(dhd, nan)) {
+		feature_set |= WIFI_FEATURE_NAN;
+		/* NAN is essentail for d2d rtt */
+		if (FW_SUPPORTED(dhd, rttd2d))
+			feature_set |= WIFI_FEATURE_D2D_RTT;
+	}
+#ifdef RTT_SUPPORT
+	if (dhd->rtt_supported) {
+		feature_set |= WIFI_FEATURE_D2D_RTT;
+		feature_set |= WIFI_FEATURE_D2AP_RTT;
+	}
+#endif /* RTT_SUPPORT */
+#ifdef LINKSTAT_SUPPORT
+	feature_set |= WIFI_FEATURE_LINKSTAT;
+#endif /* LINKSTAT_SUPPORT */
+
+#if defined(PNO_SUPPORT) && !defined(DISABLE_ANDROID_PNO)
+	if (dhd_is_pno_supported(dhd)) {
+		feature_set |= WIFI_FEATURE_PNO;
+#ifdef BATCH_SCAN
+		/* Deprecated */
+		feature_set |= WIFI_FEATURE_BATCH_SCAN;
+#endif /* BATCH_SCAN */
+#ifdef GSCAN_SUPPORT
+		/* terence 20171115: remove to get GTS PASS
+		 * com.google.android.gts.wifi.WifiHostTest#testWifiScannerBatchTimestamp
+		 */
+#ifndef DISABLE_ANDROID_GSCAN
+//		feature_set |= WIFI_FEATURE_GSCAN;
+#endif /* DISABLE_ANDROID_GSCAN */
+//		feature_set |= WIFI_FEATURE_HAL_EPNO;
+#endif /* GSCAN_SUPPORT */
+	}
+#endif /* PNO_SUPPORT && !DISABLE_ANDROID_PNO */
+#ifdef RSSI_MONITOR_SUPPORT
+	if (FW_SUPPORTED(dhd, rssi_mon)) {
+		feature_set |= WIFI_FEATURE_RSSI_MONITOR;
+	}
+#endif /* RSSI_MONITOR_SUPPORT */
+#ifdef WL11U
+	feature_set |= WIFI_FEATURE_HOTSPOT;
+#endif /* WL11U */
+#ifdef KEEP_ALIVE
+	feature_set |= WIFI_FEATURE_MKEEP_ALIVE;
+#endif /* KEEP_ALIVE */
+#ifdef NDO_CONFIG_SUPPORT
+	feature_set |= WIFI_FEATURE_CONFIG_NDO;
+#endif /* NDO_CONFIG_SUPPORT */
+#ifdef SUPPORT_RANDOM_MAC_SCAN
+	feature_set |= WIFI_FEATURE_SCAN_RAND;
+#endif /* SUPPORT_RANDOM_MAC_SCAN */
+#ifdef FILTER_IE
+	if (FW_SUPPORTED(dhd, fie)) {
+		feature_set |= WIFI_FEATURE_FILTER_IE;
+	}
+#endif /* FILTER_IE */
+#ifdef ROAMEXP_SUPPORT
+	 feature_set |= WIFI_FEATURE_CONTROL_ROAMING;
+#endif /* ROAMEXP_SUPPORT */
+#ifdef WL_P2P_RAND
+	feature_set |= WIFI_FEATURE_P2P_RAND_MAC;
+#endif /* WL_P2P_RAND */
+#ifdef WL_SAR_TX_POWER
+	feature_set |= WIFI_FEATURE_SET_TX_POWER_LIMIT;
+	feature_set |= WIFI_FEATURE_USE_BODY_HEAD_SAR;
+#endif /* WL_SAR_TX_POWER */
+#ifdef WL_STATIC_IF
+	feature_set |= WIFI_FEATURE_AP_STA;
+#endif /* WL_STATIC_IF */
+#ifdef WL_LATENCY_MODE
+	feature_set |= WIFI_FEATURE_SET_LATENCY_MODE;
+#endif /* WL_LATENCY_MODE */
+	return feature_set;
+}
+
+int
+dhd_dev_indoor_cfg(struct net_device *dev, u8 enable)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+    dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0, ifidx = 0;
+	uint indoor = enable;
+
+	ifidx = dhd_net2idx(dhd, dev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx\n", __FUNCTION__));
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = dhd_wl_ioctl_set_intiovar(dhdp, "indoor", indoor, WLC_SET_VAR,
+			TRUE, ifidx);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s: Unable to set INDOOR %d, ret=%d\n", __FUNCTION__, indoor, ret));
+	}
+
+exit:
+	return ret;
+}
+
+int
+dhd_dev_get_feature_set_matrix(struct net_device *dev, int num)
+{
+	int feature_set_full;
+	int ret = 0;
+
+	feature_set_full = dhd_dev_get_feature_set(dev);
+
+	/* Common feature set for all interface */
+	ret = (feature_set_full & WIFI_FEATURE_INFRA) |
+		(feature_set_full & WIFI_FEATURE_INFRA_5G) |
+		(feature_set_full & WIFI_FEATURE_D2D_RTT) |
+		(feature_set_full & WIFI_FEATURE_D2AP_RTT) |
+		(feature_set_full & WIFI_FEATURE_RSSI_MONITOR) |
+		(feature_set_full & WIFI_FEATURE_EPR);
+
+	/* Specific feature group for each interface */
+	switch (num) {
+	case 0:
+		ret |= (feature_set_full & WIFI_FEATURE_P2P) |
+			/* Not supported yet */
+			/* (feature_set_full & WIFI_FEATURE_NAN) | */
+			(feature_set_full & WIFI_FEATURE_TDLS) |
+			(feature_set_full & WIFI_FEATURE_PNO) |
+			(feature_set_full & WIFI_FEATURE_HAL_EPNO) |
+			(feature_set_full & WIFI_FEATURE_BATCH_SCAN) |
+			(feature_set_full & WIFI_FEATURE_GSCAN) |
+			(feature_set_full & WIFI_FEATURE_HOTSPOT) |
+			(feature_set_full & WIFI_FEATURE_ADDITIONAL_STA);
+		break;
+
+	case 1:
+		ret |= (feature_set_full & WIFI_FEATURE_P2P);
+		/* Not yet verified NAN with P2P */
+		/* (feature_set_full & WIFI_FEATURE_NAN) | */
+		break;
+
+	case 2:
+		ret |= (feature_set_full & WIFI_FEATURE_NAN) |
+			(feature_set_full & WIFI_FEATURE_TDLS) |
+			(feature_set_full & WIFI_FEATURE_TDLS_OFFCHANNEL);
+		break;
+
+	default:
+		ret = WIFI_FEATURE_INVALID;
+		DHD_ERROR(("%s: Out of index(%d) for get feature set\n", __FUNCTION__, num));
+		break;
+	}
+
+	return ret;
+}
+
+#if defined(CUSTOM_FORCE_NODFS_FLAG) && defined(CUSTOM_COUNTRY_CODE)
+int
+dhd_dev_set_nodfs(struct net_device *dev, u32 nodfs)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	bool force_ccode_change = FALSE;
+	UNUSED_PARAMETER(force_ccode_change);
+
+	if (nodfs && !(dhd->pub.dhd_cflags & WLAN_PLAT_NODFS_FLAG)) {
+		dhd->pub.dhd_cflags |= WLAN_PLAT_NODFS_FLAG;
+		force_ccode_change = TRUE;
+	} else if (!nodfs && (dhd->pub.dhd_cflags & WLAN_PLAT_NODFS_FLAG)) {
+		dhd->pub.dhd_cflags &= ~WLAN_PLAT_NODFS_FLAG;
+		force_ccode_change = TRUE;
+	}
+
+#ifdef SKIP_CCODE_FOR_SAME_DFS_FLAG
+	if (force_ccode_change == FALSE) {
+		DHD_ERROR(("No change in dfs flags. dfs:%d\n", !!nodfs));
+		return 0;
+	}
+#endif /* SKIP_CCODE_FOR_SAME_DFS_FLAG */
+
+	DHD_ERROR(("Force country change. dfs:%d \n", !!nodfs));
+	dhd->pub.force_country_change = TRUE;
+	return 0;
+}
+#endif /* CUSTOM_FORCE_NODFS_FLAG && CUSTOM_COUNTRY_CODE */
+
+#ifdef NDO_CONFIG_SUPPORT
+int
+dhd_dev_ndo_cfg(struct net_device *dev, u8 enable)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0;
+
+	if (enable) {
+		/* enable ND offload feature (will be enabled in FW on suspend) */
+		dhdp->ndo_enable = TRUE;
+
+		/* Update changes of anycast address & DAD failed address */
+		ret = dhd_dev_ndo_update_inet6addr(dev);
+		if ((ret < 0) && (ret != BCME_NORESOURCE)) {
+			DHD_ERROR(("%s: failed to update host ip addr: %d\n", __FUNCTION__, ret));
+			return ret;
+		}
+	} else {
+		/* disable ND offload feature */
+		dhdp->ndo_enable = FALSE;
+
+		/* disable ND offload in FW */
+		ret = dhd_ndo_enable(dhdp, FALSE);
+		if (ret < 0) {
+			DHD_ERROR(("%s: failed to disable NDO: %d\n", __FUNCTION__, ret));
+		}
+	}
+	return ret;
+}
+
+static int
+dhd_dev_ndo_get_valid_inet6addr_count(struct inet6_dev *inet6)
+{
+	struct inet6_ifaddr *ifa;
+	struct ifacaddr6 *acaddr = NULL;
+	int addr_count = 0;
+
+	/* lock */
+	read_lock_bh(&inet6->lock);
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	/* Count valid unicast address */
+	list_for_each_entry(ifa, &inet6->addr_list, if_list) {
+		GCC_DIAGNOSTIC_POP();
+		if ((ifa->flags & IFA_F_DADFAILED) == 0) {
+			addr_count++;
+		}
+	}
+
+	/* Count anycast address */
+	acaddr = inet6->ac_list;
+	while (acaddr) {
+		addr_count++;
+		acaddr = acaddr->aca_next;
+	}
+
+	/* unlock */
+	read_unlock_bh(&inet6->lock);
+
+	return addr_count;
+}
+
+int
+dhd_dev_ndo_update_inet6addr(struct net_device *dev)
+{
+	dhd_info_t *dhd;
+	dhd_pub_t *dhdp;
+	struct inet6_dev *inet6;
+	struct inet6_ifaddr *ifa;
+	struct ifacaddr6 *acaddr = NULL;
+	struct in6_addr *ipv6_addr = NULL;
+	int cnt, i;
+	int ret = BCME_OK;
+
+	/*
+	 * this function evaulates host ip address in struct inet6_dev
+	 * unicast addr in inet6_dev->addr_list
+	 * anycast addr in inet6_dev->ac_list
+	 * while evaluating inet6_dev, read_lock_bh() is required to prevent
+	 * access on null(freed) pointer.
+	 */
+
+	if (dev) {
+		inet6 = dev->ip6_ptr;
+		if (!inet6) {
+			DHD_ERROR(("%s: Invalid inet6_dev\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+
+		dhd = DHD_DEV_INFO(dev);
+		if (!dhd) {
+			DHD_ERROR(("%s: Invalid dhd_info\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+		dhdp = &dhd->pub;
+
+		if (dhd_net2idx(dhd, dev) != 0) {
+			DHD_ERROR(("%s: Not primary interface\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+	} else {
+		DHD_ERROR(("%s: Invalid net_device\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Check host IP overflow */
+	cnt = dhd_dev_ndo_get_valid_inet6addr_count(inet6);
+	if (cnt > dhdp->ndo_max_host_ip) {
+		if (!dhdp->ndo_host_ip_overflow) {
+			dhdp->ndo_host_ip_overflow = TRUE;
+			/* Disable ND offload in FW */
+			DHD_INFO(("%s: Host IP overflow, disable NDO\n", __FUNCTION__));
+			ret = dhd_ndo_enable(dhdp, FALSE);
+		}
+
+		return ret;
+	}
+
+	/*
+	 * Allocate ipv6 addr buffer to store addresses to be added/removed.
+	 * driver need to lock inet6_dev while accessing structure. but, driver
+	 * cannot use ioctl while inet6_dev locked since it requires scheduling
+	 * hence, copy addresses to the buffer and do ioctl after unlock.
+	 */
+	ipv6_addr = (struct in6_addr *)MALLOC(dhdp->osh,
+		sizeof(struct in6_addr) * dhdp->ndo_max_host_ip);
+	if (!ipv6_addr) {
+		DHD_ERROR(("%s: failed to alloc ipv6 addr buffer\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+
+	/* Find DAD failed unicast address to be removed */
+	cnt = 0;
+	read_lock_bh(&inet6->lock);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry(ifa, &inet6->addr_list, if_list) {
+		GCC_DIAGNOSTIC_POP();
+		/* DAD failed unicast address */
+		if ((ifa->flags & IFA_F_DADFAILED) &&
+			(cnt < dhdp->ndo_max_host_ip)) {
+				memcpy(&ipv6_addr[cnt], &ifa->addr, sizeof(struct in6_addr));
+				cnt++;
+		}
+	}
+	read_unlock_bh(&inet6->lock);
+
+	/* Remove DAD failed unicast address */
+	for (i = 0; i < cnt; i++) {
+		DHD_INFO(("%s: Remove DAD failed addr\n", __FUNCTION__));
+		ret = dhd_ndo_remove_ip_by_addr(dhdp, (char *)&ipv6_addr[i], 0);
+		if (ret < 0) {
+			goto done;
+		}
+	}
+
+	/* Remove all anycast address */
+	ret = dhd_ndo_remove_ip_by_type(dhdp, WL_ND_IPV6_ADDR_TYPE_ANYCAST, 0);
+	if (ret < 0) {
+		goto done;
+	}
+
+	/*
+	 * if ND offload was disabled due to host ip overflow,
+	 * attempt to add valid unicast address.
+	 */
+	if (dhdp->ndo_host_ip_overflow) {
+		/* Find valid unicast address */
+		cnt = 0;
+		read_lock_bh(&inet6->lock);
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		list_for_each_entry(ifa, &inet6->addr_list, if_list) {
+			GCC_DIAGNOSTIC_POP();
+			/* valid unicast address */
+			if (!(ifa->flags & IFA_F_DADFAILED) &&
+				(cnt < dhdp->ndo_max_host_ip)) {
+					memcpy(&ipv6_addr[cnt], &ifa->addr,
+						sizeof(struct in6_addr));
+					cnt++;
+			}
+		}
+		read_unlock_bh(&inet6->lock);
+
+		/* Add valid unicast address */
+		for (i = 0; i < cnt; i++) {
+			ret = dhd_ndo_add_ip_with_type(dhdp,
+				(char *)&ipv6_addr[i], WL_ND_IPV6_ADDR_TYPE_UNICAST, 0);
+			if (ret < 0) {
+				goto done;
+			}
+		}
+	}
+
+	/* Find anycast address */
+	cnt = 0;
+	read_lock_bh(&inet6->lock);
+	acaddr = inet6->ac_list;
+	while (acaddr) {
+		if (cnt < dhdp->ndo_max_host_ip) {
+			memcpy(&ipv6_addr[cnt], &acaddr->aca_addr, sizeof(struct in6_addr));
+			cnt++;
+		}
+		acaddr = acaddr->aca_next;
+	}
+	read_unlock_bh(&inet6->lock);
+
+	/* Add anycast address */
+	for (i = 0; i < cnt; i++) {
+		ret = dhd_ndo_add_ip_with_type(dhdp,
+			(char *)&ipv6_addr[i], WL_ND_IPV6_ADDR_TYPE_ANYCAST, 0);
+		if (ret < 0) {
+			goto done;
+		}
+	}
+
+	/* Now All host IP addr were added successfully */
+	if (dhdp->ndo_host_ip_overflow) {
+		dhdp->ndo_host_ip_overflow = FALSE;
+		if (dhdp->in_suspend) {
+			/* drvier is in (early) suspend state, need to enable ND offload in FW */
+			DHD_INFO(("%s: enable NDO\n", __FUNCTION__));
+			ret = dhd_ndo_enable(dhdp, TRUE);
+		}
+	}
+
+done:
+	if (ipv6_addr) {
+		MFREE(dhdp->osh, ipv6_addr, sizeof(struct in6_addr) * dhdp->ndo_max_host_ip);
+	}
+
+	return ret;
+}
+
+#endif /* NDO_CONFIG_SUPPORT */
+
+#ifdef PNO_SUPPORT
+/* Linux wrapper to call common dhd_pno_stop_for_ssid */
+int
+dhd_dev_pno_stop_for_ssid(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	return (dhd_pno_stop_for_ssid(&dhd->pub));
+}
+/* Linux wrapper to call common dhd_pno_set_for_ssid */
+int
+dhd_dev_pno_set_for_ssid(struct net_device *dev, wlc_ssid_ext_t* ssids_local, int nssid,
+	uint16  scan_fr, int pno_repeat, int pno_freq_expo_max, uint16 *channel_list, int nchan)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	return (dhd_pno_set_for_ssid(&dhd->pub, ssids_local, nssid, scan_fr,
+		pno_repeat, pno_freq_expo_max, channel_list, nchan));
+}
+
+/* Linux wrapper to call common dhd_pno_enable */
+int
+dhd_dev_pno_enable(struct net_device *dev, int enable)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	return (dhd_pno_enable(&dhd->pub, enable));
+}
+
+/* Linux wrapper to call common dhd_pno_set_for_hotlist */
+int
+dhd_dev_pno_set_for_hotlist(struct net_device *dev, wl_pfn_bssid_t *p_pfn_bssid,
+	struct dhd_pno_hotlist_params *hotlist_params)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return (dhd_pno_set_for_hotlist(&dhd->pub, p_pfn_bssid, hotlist_params));
+}
+/* Linux wrapper to call common dhd_dev_pno_stop_for_batch */
+int
+dhd_dev_pno_stop_for_batch(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return (dhd_pno_stop_for_batch(&dhd->pub));
+}
+
+/* Linux wrapper to call common dhd_dev_pno_set_for_batch */
+int
+dhd_dev_pno_set_for_batch(struct net_device *dev, struct dhd_pno_batch_params *batch_params)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return (dhd_pno_set_for_batch(&dhd->pub, batch_params));
+}
+
+/* Linux wrapper to call common dhd_dev_pno_get_for_batch */
+int
+dhd_dev_pno_get_for_batch(struct net_device *dev, char *buf, int bufsize)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return (dhd_pno_get_for_batch(&dhd->pub, buf, bufsize, PNO_STATUS_NORMAL));
+}
+#endif /* PNO_SUPPORT */
+
+#if defined(PNO_SUPPORT)
+#ifdef GSCAN_SUPPORT
+bool
+dhd_dev_is_legacy_pno_enabled(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_is_legacy_pno_enabled(&dhd->pub));
+}
+
+int
+dhd_dev_set_epno(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	if (!dhd) {
+		return BCME_ERROR;
+	}
+	return dhd_pno_set_epno(&dhd->pub);
+}
+int
+dhd_dev_flush_fw_epno(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	if (!dhd) {
+		return BCME_ERROR;
+	}
+	return dhd_pno_flush_fw_epno(&dhd->pub);
+}
+
+/* Linux wrapper to call common dhd_pno_set_cfg_gscan */
+int
+dhd_dev_pno_set_cfg_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type,
+ void *buf, bool flush)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_set_cfg_gscan(&dhd->pub, type, buf, flush));
+}
+
+/* Linux wrapper to call common dhd_pno_get_gscan */
+void *
+dhd_dev_pno_get_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type,
+                      void *info, uint32 *len)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_get_gscan(&dhd->pub, type, info, len));
+}
+
+/* Linux wrapper to call common dhd_wait_batch_results_complete */
+int
+dhd_dev_wait_batch_results_complete(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_wait_batch_results_complete(&dhd->pub));
+}
+
+/* Linux wrapper to call common dhd_pno_lock_batch_results */
+int
+dhd_dev_pno_lock_access_batch_results(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_lock_batch_results(&dhd->pub));
+}
+/* Linux wrapper to call common dhd_pno_unlock_batch_results */
+void
+dhd_dev_pno_unlock_access_batch_results(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_unlock_batch_results(&dhd->pub));
+}
+
+/* Linux wrapper to call common dhd_pno_initiate_gscan_request */
+int
+dhd_dev_pno_run_gscan(struct net_device *dev, bool run, bool flush)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_initiate_gscan_request(&dhd->pub, run, flush));
+}
+
+/* Linux wrapper to call common dhd_pno_enable_full_scan_result */
+int
+dhd_dev_pno_enable_full_scan_result(struct net_device *dev, bool real_time_flag)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_enable_full_scan_result(&dhd->pub, real_time_flag));
+}
+
+/* Linux wrapper to call common dhd_handle_hotlist_scan_evt */
+void *
+dhd_dev_hotlist_scan_event(struct net_device *dev,
+      const void  *data, int *send_evt_bytes, hotlist_type_t type, u32 *buf_len)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_handle_hotlist_scan_evt(&dhd->pub, data, send_evt_bytes, type, buf_len));
+}
+
+/* Linux wrapper to call common dhd_process_full_gscan_result */
+void *
+dhd_dev_process_full_gscan_result(struct net_device *dev,
+const void  *data, uint32 len, int *send_evt_bytes)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_process_full_gscan_result(&dhd->pub, data, len, send_evt_bytes));
+}
+
+void
+dhd_dev_gscan_hotlist_cache_cleanup(struct net_device *dev, hotlist_type_t type)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	dhd_gscan_hotlist_cache_cleanup(&dhd->pub, type);
+
+	return;
+}
+
+int
+dhd_dev_gscan_batch_cache_cleanup(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_gscan_batch_cache_cleanup(&dhd->pub));
+}
+
+/* Linux wrapper to call common dhd_retreive_batch_scan_results */
+int
+dhd_dev_retrieve_batch_scan(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_retreive_batch_scan_results(&dhd->pub));
+}
+
+/* Linux wrapper to call common dhd_pno_process_epno_result */
+void * dhd_dev_process_epno_result(struct net_device *dev,
+	const void  *data, uint32 event, int *send_evt_bytes)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_pno_process_epno_result(&dhd->pub, data, event, send_evt_bytes));
+}
+
+int
+dhd_dev_set_lazy_roam_cfg(struct net_device *dev,
+             wlc_roam_exp_params_t *roam_param)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	wl_roam_exp_cfg_t roam_exp_cfg;
+	int err;
+
+	if (!roam_param) {
+		return BCME_BADARG;
+	}
+
+	DHD_INFO(("a_band_boost_thr %d a_band_penalty_thr %d\n",
+	      roam_param->a_band_boost_threshold, roam_param->a_band_penalty_threshold));
+	DHD_INFO(("a_band_boost_factor %d a_band_penalty_factor %d cur_bssid_boost %d\n",
+	      roam_param->a_band_boost_factor, roam_param->a_band_penalty_factor,
+	      roam_param->cur_bssid_boost));
+	DHD_INFO(("alert_roam_trigger_thr %d a_band_max_boost %d\n",
+	      roam_param->alert_roam_trigger_threshold, roam_param->a_band_max_boost));
+
+	memcpy(&roam_exp_cfg.params, roam_param, sizeof(*roam_param));
+	roam_exp_cfg.version = ROAM_EXP_CFG_VERSION_1;
+	roam_exp_cfg.flags = ROAM_EXP_CFG_PRESENT;
+	if (dhd->pub.lazy_roam_enable) {
+		roam_exp_cfg.flags |= ROAM_EXP_ENABLE_FLAG;
+	}
+	err = dhd_iovar(&dhd->pub, 0, "roam_exp_params",
+			(char *)&roam_exp_cfg, sizeof(roam_exp_cfg), NULL, 0,
+			TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : Failed to execute roam_exp_params %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+
+int
+dhd_dev_lazy_roam_enable(struct net_device *dev, uint32 enable)
+{
+	int err;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	wl_roam_exp_cfg_t roam_exp_cfg;
+
+	memset(&roam_exp_cfg, 0, sizeof(roam_exp_cfg));
+	roam_exp_cfg.version = ROAM_EXP_CFG_VERSION_1;
+	if (enable) {
+		roam_exp_cfg.flags = ROAM_EXP_ENABLE_FLAG;
+	}
+
+	err = dhd_iovar(&dhd->pub, 0, "roam_exp_params",
+			(char *)&roam_exp_cfg, sizeof(roam_exp_cfg), NULL, 0,
+			TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : Failed to execute roam_exp_params %d\n", __FUNCTION__, err));
+	} else {
+		dhd->pub.lazy_roam_enable = (enable != 0);
+	}
+	return err;
+}
+
+int
+dhd_dev_set_lazy_roam_bssid_pref(struct net_device *dev,
+       wl_bssid_pref_cfg_t *bssid_pref, uint32 flush)
+{
+	int err;
+	uint len;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	bssid_pref->version = BSSID_PREF_LIST_VERSION_1;
+	/* By default programming bssid pref flushes out old values */
+	bssid_pref->flags = (flush && !bssid_pref->count) ? ROAM_EXP_CLEAR_BSSID_PREF: 0;
+	len = sizeof(wl_bssid_pref_cfg_t);
+	if (bssid_pref->count) {
+		len += (bssid_pref->count - 1) * sizeof(wl_bssid_pref_list_t);
+	}
+	err = dhd_iovar(&dhd->pub, 0, "roam_exp_bssid_pref",
+			(char *)bssid_pref, len, NULL, 0, TRUE);
+	if (err != BCME_OK) {
+		DHD_ERROR(("%s : Failed to execute roam_exp_bssid_pref %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+#endif /* GSCAN_SUPPORT */
+
+#if defined(GSCAN_SUPPORT) || defined(ROAMEXP_SUPPORT)
+int
+dhd_dev_set_blacklist_bssid(struct net_device *dev, maclist_t *blacklist,
+    uint32 len, uint32 flush)
+{
+	int err;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	int macmode;
+
+	if (blacklist) {
+		err = dhd_wl_ioctl_cmd(&(dhd->pub), WLC_SET_MACLIST, (char *)blacklist,
+				len, TRUE, 0);
+		if (err != BCME_OK) {
+			DHD_ERROR(("%s : WLC_SET_MACLIST failed %d\n", __FUNCTION__, err));
+			return err;
+		}
+	}
+	/* By default programming blacklist flushes out old values */
+	macmode = (flush && !blacklist) ? WLC_MACMODE_DISABLED : WLC_MACMODE_DENY;
+	err = dhd_wl_ioctl_cmd(&(dhd->pub), WLC_SET_MACMODE, (char *)&macmode,
+	              sizeof(macmode), TRUE, 0);
+	if (err != BCME_OK) {
+		DHD_ERROR(("%s : WLC_SET_MACMODE failed %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+
+int
+dhd_dev_set_whitelist_ssid(struct net_device *dev, wl_ssid_whitelist_t *ssid_whitelist,
+    uint32 len, uint32 flush)
+{
+	int err;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	wl_ssid_whitelist_t whitelist_ssid_flush;
+
+	if (!ssid_whitelist) {
+		if (flush) {
+			ssid_whitelist = &whitelist_ssid_flush;
+			ssid_whitelist->ssid_count = 0;
+		} else {
+			DHD_ERROR(("%s : Nothing to do here\n", __FUNCTION__));
+			return BCME_BADARG;
+		}
+	}
+	ssid_whitelist->version = SSID_WHITELIST_VERSION_1;
+	ssid_whitelist->flags = flush ? ROAM_EXP_CLEAR_SSID_WHITELIST : 0;
+	err = dhd_iovar(&dhd->pub, 0, "roam_exp_ssid_whitelist", (char *)ssid_whitelist, len, NULL,
+			0, TRUE);
+	if (err != BCME_OK) {
+		if (err == BCME_UNSUPPORTED) {
+			DHD_ERROR(("%s : roam_exp_bssid_pref, UNSUPPORTED \n", __FUNCTION__));
+		} else {
+			DHD_ERROR(("%s : Failed to execute roam_exp_bssid_pref %d\n",
+				__FUNCTION__, err));
+		}
+	}
+	return err;
+}
+#endif /* GSCAN_SUPPORT || ROAMEXP_SUPPORT */
+#endif /* defined(OEM_ANDROID) && defined(PNO_SUPPORT) */
+
+#ifdef RSSI_MONITOR_SUPPORT
+int
+dhd_dev_set_rssi_monitor_cfg(struct net_device *dev, int start,
+             int8 max_rssi, int8 min_rssi)
+{
+	int err;
+	wl_rssi_monitor_cfg_t rssi_monitor;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	rssi_monitor.version = RSSI_MONITOR_VERSION_1;
+	rssi_monitor.max_rssi = max_rssi;
+	rssi_monitor.min_rssi = min_rssi;
+	rssi_monitor.flags = start ? 0: RSSI_MONITOR_STOP;
+	err = dhd_iovar(&dhd->pub, 0, "rssi_monitor", (char *)&rssi_monitor, sizeof(rssi_monitor),
+			NULL, 0, TRUE);
+	if (err < 0 && err != BCME_UNSUPPORTED) {
+		DHD_ERROR(("%s : Failed to execute rssi_monitor %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+#endif /* RSSI_MONITOR_SUPPORT */
+
+#ifdef DHDTCPACK_SUPPRESS
+int
+dhd_dev_set_tcpack_sup_mode_cfg(struct net_device *dev, uint8 enable)
+{
+	int err;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	err = dhd_tcpack_suppress_set(&dhd->pub, enable);
+	if (err != BCME_OK) {
+		DHD_ERROR(("%s : Failed to set tcpack_suppress mode: %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+#endif /* DHDTCPACK_SUPPRESS */
+
+int
+dhd_dev_cfg_rand_mac_oui(struct net_device *dev, uint8 *oui)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	if (!dhdp || !oui) {
+		DHD_ERROR(("NULL POINTER : %s\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	if (ETHER_ISMULTI(oui)) {
+		DHD_ERROR(("Expected unicast OUI\n"));
+		return BCME_ERROR;
+	} else {
+		uint8 *rand_mac_oui = dhdp->rand_mac_oui;
+		memcpy(rand_mac_oui, oui, DOT11_OUI_LEN);
+		DHD_ERROR(("Random MAC OUI to be used - "MACOUIDBG"\n",
+			MACOUI2STRDBG(rand_mac_oui)));
+	}
+	return BCME_OK;
+}
+
+int
+dhd_set_rand_mac_oui(dhd_pub_t *dhd)
+{
+	int err;
+	wl_pfn_macaddr_cfg_t wl_cfg;
+	uint8 *rand_mac_oui = dhd->rand_mac_oui;
+
+	memset(&wl_cfg.macaddr, 0, ETHER_ADDR_LEN);
+	memcpy(&wl_cfg.macaddr, rand_mac_oui, DOT11_OUI_LEN);
+	wl_cfg.version = WL_PFN_MACADDR_CFG_VER;
+	if (ETHER_ISNULLADDR(&wl_cfg.macaddr)) {
+		wl_cfg.flags = 0;
+	} else {
+		wl_cfg.flags = (WL_PFN_MAC_OUI_ONLY_MASK | WL_PFN_SET_MAC_UNASSOC_MASK);
+	}
+
+	DHD_ERROR(("Setting rand mac oui to FW - "MACOUIDBG"\n",
+		MACOUI2STRDBG(rand_mac_oui)));
+
+	err = dhd_iovar(dhd, 0, "pfn_macaddr", (char *)&wl_cfg, sizeof(wl_cfg), NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_macaddr %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+
+#if defined(RTT_SUPPORT) && defined(WL_CFG80211)
+/* Linux wrapper to call common dhd_pno_set_cfg_gscan */
+int
+dhd_dev_rtt_set_cfg(struct net_device *dev, void *buf)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_rtt_set_cfg(&dhd->pub, buf));
+}
+
+int
+dhd_dev_rtt_cancel_cfg(struct net_device *dev, struct ether_addr *mac_list, int mac_cnt)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_rtt_stop(&dhd->pub, mac_list, mac_cnt));
+}
+
+int
+dhd_dev_rtt_register_noti_callback(struct net_device *dev, void *ctx, dhd_rtt_compl_noti_fn noti_fn)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_rtt_register_noti_callback(&dhd->pub, ctx, noti_fn));
+}
+
+int
+dhd_dev_rtt_unregister_noti_callback(struct net_device *dev, dhd_rtt_compl_noti_fn noti_fn)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_rtt_unregister_noti_callback(&dhd->pub, noti_fn));
+}
+
+int
+dhd_dev_rtt_capability(struct net_device *dev, rtt_capabilities_t *capa)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+
+	return (dhd_rtt_capability(&dhd->pub, capa));
+}
+
+int
+dhd_dev_rtt_avail_channel(struct net_device *dev, wifi_channel_info *channel_info)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	return (dhd_rtt_avail_channel(&dhd->pub, channel_info));
+}
+
+int
+dhd_dev_rtt_enable_responder(struct net_device *dev, wifi_channel_info *channel_info)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	return (dhd_rtt_enable_responder(&dhd->pub, channel_info));
+}
+
+int dhd_dev_rtt_cancel_responder(struct net_device *dev)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	return (dhd_rtt_cancel_responder(&dhd->pub));
+}
+
+#endif /* RTT_SUPPORT */
+
+#if defined(APF)
+static void _dhd_apf_lock_local(dhd_info_t *dhd)
+{
+	if (dhd) {
+		mutex_lock(&dhd->dhd_apf_mutex);
+	}
+}
+
+static void _dhd_apf_unlock_local(dhd_info_t *dhd)
+{
+	if (dhd) {
+		mutex_unlock(&dhd->dhd_apf_mutex);
+	}
+}
+
+static int
+_dhd_apf_add_filter(struct net_device *ndev, uint32 filter_id, u8* program, uint32 program_len)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	wl_pkt_filter_t * pkt_filterp;
+	wl_apf_program_t *apf_program;
+	char *buf = NULL;
+	u32 cmd_len, buf_len, max_len;
+	int ifidx, ret = BCME_OK;
+	char cmd[] = "pkt_filter_add";
+
+	ifidx = dhd_net2idx(dhd, ndev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx\n", __FUNCTION__));
+		ret = -ENODEV;
+		goto exit;
+	}
+	if (dhd_conf_del_pkt_filter(dhdp, filter_id))
+		goto exit;
+
+	cmd_len = sizeof(cmd);
+
+	/* Check if the program_len is more than the expected len or if the program is NULL,
+	 * then return from here.
+	 */
+	ret = dhd_wl_ioctl_get_intiovar(dhdp, "apf_size_limit", &max_len, WLC_GET_VAR,
+		FALSE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to get APF size limit, ret=%d\n", __FUNCTION__, ret));
+		goto exit;
+	}
+	if ((program_len > max_len) || (program == NULL)) {
+		DHD_ERROR(("%s Invalid program_len: %d, program: %pK\n", __FUNCTION__,
+			program_len, program));
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	buf_len = cmd_len + WL_PKT_FILTER_FIXED_LEN + WL_APF_PROGRAM_FIXED_LEN + program_len;
+	buf = MALLOCZ(dhdp->osh, buf_len);
+	if (unlikely(!buf)) {
+		DHD_ERROR(("%s: MALLOC failure, %d bytes\n", __FUNCTION__, buf_len));
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	ret = memcpy_s(buf, buf_len, cmd, cmd_len);
+	if (unlikely(ret)) {
+		goto exit;
+	}
+
+	pkt_filterp = (wl_pkt_filter_t *) (buf + cmd_len);
+	pkt_filterp->id = htod32(filter_id);
+	pkt_filterp->negate_match = htod32(FALSE);
+	pkt_filterp->type = htod32(WL_PKT_FILTER_TYPE_APF_MATCH);
+
+	apf_program = &pkt_filterp->u.apf_program;
+	apf_program->version = htod16(WL_APF_INTERNAL_VERSION);
+	apf_program->instr_len = htod16(program_len);
+	ret = memcpy_s(apf_program->instrs, program_len, program, program_len);
+	if (unlikely(ret)) {
+		goto exit;
+	}
+
+	ret = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, buf, buf_len, TRUE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to add APF filter, id=%d, ret=%d\n", __FUNCTION__,
+			filter_id, ret));
+		goto exit;
+	}
+
+exit:
+	if (buf) {
+		MFREE(dhdp->osh, buf, buf_len);
+	}
+	return ret;
+}
+
+static int
+_dhd_apf_config_filter(struct net_device *ndev, uint32 filter_id, uint32 mode, uint32 enable)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	wl_pkt_filter_enable_t * pkt_filterp;
+	char *buf = NULL;
+	u32 cmd_len, buf_len;
+	int ifidx, ret;
+	char cmd[] = "pkt_filter_enable";
+
+	ifidx = dhd_net2idx(dhd, ndev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx\n", __FUNCTION__));
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	cmd_len = sizeof(cmd);
+	buf_len = cmd_len + (u32)sizeof(*pkt_filterp);
+
+	buf = MALLOCZ(dhdp->osh, buf_len);
+	if (unlikely(!buf)) {
+		DHD_ERROR(("%s: MALLOC failure, %d bytes\n", __FUNCTION__, buf_len));
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	ret = memcpy_s(buf, buf_len, cmd, cmd_len);
+	if (unlikely(ret)) {
+		goto exit;
+	}
+
+	pkt_filterp = (wl_pkt_filter_enable_t *) (buf + cmd_len);
+	pkt_filterp->id = htod32(filter_id);
+	pkt_filterp->enable = htod32(enable);
+
+	ret = dhd_wl_ioctl_cmd(dhdp, WLC_SET_VAR, buf, buf_len, TRUE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to enable APF filter, id=%d, ret=%d\n", __FUNCTION__,
+			filter_id, ret));
+		goto exit;
+	}
+
+	ret = dhd_wl_ioctl_set_intiovar(dhdp, "pkt_filter_mode", dhd_master_mode, WLC_SET_VAR,
+		TRUE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to set APF filter mode, id=%d, ret=%d\n", __FUNCTION__,
+			filter_id, ret));
+		goto exit;
+	}
+
+exit:
+	if (buf) {
+		MFREE(dhdp->osh, buf, buf_len);
+	}
+	return ret;
+}
+
+static int
+_dhd_apf_delete_filter(struct net_device *ndev, uint32 filter_id)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ifidx, ret;
+
+	ifidx = dhd_net2idx(dhd, ndev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	ret = dhd_wl_ioctl_set_intiovar(dhdp, "pkt_filter_delete", htod32(filter_id),
+		WLC_SET_VAR, TRUE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to delete APF filter, id=%d, ret=%d\n", __FUNCTION__,
+			filter_id, ret));
+		return ret;
+	}
+
+	return ret;
+}
+
+static int
+_dhd_apf_read_filter_data(struct net_device *ndev, uint32 filter_id, u8* buf, uint32 buf_len)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	wl_apf_program_t *apf_program;
+	u32 cmd_len, max_len;
+	int ifidx, ret;
+	char cmd[] = "apf_buf_get";
+
+	ifidx = dhd_net2idx(dhd, ndev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	/* Check if the provided buffer is large enough to store the filter data or
+	 * if the buffer is NULL, then return from here.
+	 */
+	ret = dhd_wl_ioctl_get_intiovar(dhdp, "apf_size_limit", &max_len,
+		WLC_GET_VAR, FALSE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to get APF size limit, ret=%d\n",
+			__FUNCTION__, ret));
+		return ret;
+	}
+	if ((buf_len < (WL_APF_PROGRAM_FIXED_LEN + max_len)) || (buf == NULL)) {
+		DHD_ERROR(("%s: Invalid buf_len=%d or buf=%pK\n", __FUNCTION__, buf_len, buf));
+		return -EINVAL;
+	}
+
+	/* Get APF filter data from the dongle */
+	cmd_len = sizeof(cmd);
+	ret = memcpy_s(buf, buf_len, cmd, cmd_len);
+	if (unlikely(ret)) {
+		return ret;
+	}
+	*(buf + cmd_len) = htod32(filter_id);
+	ret = dhd_wl_ioctl_cmd(dhdp, WLC_GET_VAR, buf, buf_len, FALSE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to read APF filter data, id=%d, ret=%d\n", __FUNCTION__,
+			filter_id, ret));
+		return ret;
+	}
+
+	/* Validate structure version */
+	apf_program = (wl_apf_program_t *)buf;
+	if (apf_program->version != WL_APF_INTERNAL_VERSION) {
+		DHD_ERROR(("%s: incorrect version, version=%d, expected version=%d\n",
+			__FUNCTION__, apf_program->version, WL_APF_INTERNAL_VERSION));
+		return BCME_VERSION;
+	}
+
+	return ret;
+}
+
+void dhd_apf_lock(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	_dhd_apf_lock_local(dhd);
+}
+
+void dhd_apf_unlock(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	_dhd_apf_unlock_local(dhd);
+}
+
+int
+dhd_dev_apf_get_version(struct net_device *ndev, uint32 *version)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ifidx, ret;
+
+	if (!FW_SUPPORTED(dhdp, apf)) {
+		DHD_ERROR(("%s: firmware doesn't support APF\n", __FUNCTION__));
+		/* Notify Android framework that APF is not supported by setting version as zero. */
+		*version = 0;
+		return BCME_OK;
+	}
+
+	ifidx = dhd_net2idx(dhd, ndev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	ret = dhd_wl_ioctl_get_intiovar(dhdp, "apf_ver", version, WLC_GET_VAR, FALSE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to get APF version, ret=%d\n", __FUNCTION__, ret));
+		return ret;
+	}
+
+	return ret;
+}
+
+int
+dhd_dev_apf_get_max_len(struct net_device *ndev, uint32 *max_len)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ifidx, ret;
+
+	if (!FW_SUPPORTED(dhdp, apf)) {
+		DHD_ERROR(("%s: firmware doesn't support APF\n", __FUNCTION__));
+		*max_len = 0;
+		return BCME_OK;
+	}
+
+	ifidx = dhd_net2idx(dhd, ndev);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	ret = dhd_wl_ioctl_get_intiovar(dhdp, "apf_size_limit", max_len, WLC_GET_VAR, FALSE, ifidx);
+	if (unlikely(ret)) {
+		DHD_ERROR(("%s: failed to get APF size limit, ret=%d\n", __FUNCTION__, ret));
+		return ret;
+	}
+
+	return ret;
+}
+
+int
+dhd_dev_apf_add_filter(struct net_device *ndev, u8* program,
+	uint32 program_len)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret;
+
+	DHD_ERROR(("%s: id %d\n", __func__, PKT_FILTER_APF_ID));
+
+	DHD_APF_LOCK(ndev);
+
+	/* delete, if filter already exists */
+	if (dhdp->apf_set) {
+		ret = _dhd_apf_delete_filter(ndev, PKT_FILTER_APF_ID);
+		if (unlikely(ret)) {
+			goto exit;
+		}
+		dhdp->apf_set = FALSE;
+	}
+
+	ret = _dhd_apf_add_filter(ndev, PKT_FILTER_APF_ID, program, program_len);
+	if (ret) {
+		goto exit;
+	}
+	dhdp->apf_set = TRUE;
+
+	if (dhdp->in_suspend && dhdp->apf_set && !(dhdp->op_mode & DHD_FLAG_HOSTAP_MODE)) {
+		/* Driver is still in (early) suspend state and during this time Android
+		 * framework updated the filter program. As there will not be another
+		 * early suspend notification, enable back the APF filter with the new
+		 * filter program to avoid unnecessary host wake-ups.
+		 */
+		ret = _dhd_apf_config_filter(ndev, PKT_FILTER_APF_ID,
+			PKT_FILTER_MODE_FORWARD_ON_MATCH, TRUE);
+		if (ret) {
+			goto exit;
+		}
+	}
+exit:
+	DHD_APF_UNLOCK(ndev);
+
+	return ret;
+}
+
+void
+dhd_dev_pkt_filter_stats(struct net_device *ndev, int pkt_filter_id) {
+	int getbuf[64];
+	wl_pkt_filter_stats_t   *stats;
+	wldev_iovar_getbuf_bsscfg(ndev, "pkt_filter_stats", &pkt_filter_id, sizeof(pkt_filter_id), getbuf, sizeof(getbuf), 0, NULL);
+	stats = (wl_pkt_filter_stats_t *) getbuf;
+
+	DHD_ERROR(("%s: id %d, matched %d, discarded %d, forwarded %d\n",
+			__func__, pkt_filter_id,
+			dtoh32(stats->num_pkts_matched),
+			dtoh32(stats->num_pkts_discarded),
+			dtoh32(stats->num_pkts_forwarded)));
+}
+
+int
+dhd_dev_apf_enable_filter(struct net_device *ndev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0;
+	bool nan_dp_active = false;
+
+	DHD_ERROR(("%s: apf_set %d, op_mode 0x%x, nan_dp_active %d\n", __func__,
+		dhdp->apf_set, dhdp->op_mode, nan_dp_active));
+
+	DHD_APF_LOCK(ndev);
+
+#ifdef WL_NAN
+	nan_dp_active = wl_cfgnan_is_dp_active(ndev);
+#endif /* WL_NAN */
+
+	if (dhdp->apf_set && (!(dhdp->op_mode & DHD_FLAG_HOSTAP_MODE) && !nan_dp_active)) {
+		DHD_ERROR(("%s: id %d Enable\n", __func__, PKT_FILTER_APF_ID));
+		ret = _dhd_apf_config_filter(ndev, PKT_FILTER_APF_ID,
+			PKT_FILTER_MODE_FORWARD_ON_MATCH, TRUE);
+		dhd_dev_pkt_filter_stats(ndev, PKT_FILTER_APF_ID);
+	}
+
+	DHD_APF_UNLOCK(ndev);
+
+	return ret;
+}
+
+int
+dhd_dev_apf_disable_filter(struct net_device *ndev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0;
+
+	DHD_ERROR(("%s: apf_set %d\n", __func__, dhdp->apf_set));
+
+	DHD_APF_LOCK(ndev);
+
+	if (dhdp->apf_set) {
+		DHD_ERROR(("%s: id %d Disable\n", __func__, PKT_FILTER_APF_ID));
+		ret = _dhd_apf_config_filter(ndev, PKT_FILTER_APF_ID,
+			PKT_FILTER_MODE_FORWARD_ON_MATCH, FALSE);
+		dhd_dev_pkt_filter_stats(ndev, PKT_FILTER_APF_ID);
+	}
+
+	DHD_APF_UNLOCK(ndev);
+
+	return ret;
+}
+
+int
+dhd_dev_apf_delete_filter(struct net_device *ndev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0;
+
+	DHD_ERROR(("%s: id %d\n", __func__, PKT_FILTER_APF_ID));
+
+	DHD_APF_LOCK(ndev);
+
+	if (dhdp->apf_set) {
+		ret = _dhd_apf_delete_filter(ndev, PKT_FILTER_APF_ID);
+		if (!ret) {
+			dhdp->apf_set = FALSE;
+		}
+	}
+
+	DHD_APF_UNLOCK(ndev);
+
+	return ret;
+}
+
+int
+dhd_dev_apf_read_filter_data(struct net_device *ndev, u8* buf, uint32 buf_len)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	int ret = 0;
+	bool apf_disabled = FALSE;
+
+	DHD_APF_LOCK(ndev);
+
+	if (!dhdp->apf_set) {
+		DHD_ERROR(("%s: APF filter is not programmed\n", __FUNCTION__));
+		goto exit;
+	}
+
+	/* As per the requirement, APF filter needs to be stopped while reading the filter data.
+	 * Otherwise the APF counters could be wrong. Therefore disable the APF filter, if driver
+	 * is in (early) suspend state.
+	 */
+	if (dhdp->in_suspend) {
+		ret = _dhd_apf_config_filter(ndev, PKT_FILTER_APF_ID,
+			PKT_FILTER_MODE_FORWARD_ON_MATCH, FALSE);
+		if (unlikely(ret)) {
+			DHD_ERROR(("%s: failed to disable APF filter, ret=%d\n",
+				__FUNCTION__, ret));
+			goto exit;
+		}
+		apf_disabled = TRUE;
+	}
+
+	ret = _dhd_apf_read_filter_data(ndev, PKT_FILTER_APF_ID, buf, buf_len);
+	if (unlikely(ret)) {
+		goto exit;
+	}
+
+exit:
+	/* Re-enable the filter if got disabled above */
+	if (apf_disabled && _dhd_apf_config_filter(ndev, PKT_FILTER_APF_ID,
+			PKT_FILTER_MODE_FORWARD_ON_MATCH, TRUE)) {
+		DHD_ERROR(("%s: failed to enable APF filter, ret=%d\n", __FUNCTION__, ret));
+	}
+
+	DHD_APF_UNLOCK(ndev);
+
+	return ret;
+}
+#endif /* APF */
+
+#if defined(OEM_ANDROID)
+static void dhd_hang_process(struct work_struct *work_data)
+{
+	struct net_device *dev;
+#ifdef IFACE_HANG_FORCE_DEV_CLOSE
+	struct net_device *ndev;
+	uint8 i = 0;
+#endif /* IFACE_HANG_FORCE_DEV_CLOSE */
+	struct dhd_info *dhd;
+	dhd_pub_t *dhdp;
+#ifdef WL_CFG80211
+	struct bcm_cfg80211 *cfg;
+#endif /* WL_CFG80211 */
+
+	/* Ignore compiler warnings due to -Werror=cast-qual */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(work_data, dhd_info_t, dhd_hang_process_work);
+	GCC_DIAGNOSTIC_POP();
+	dhdp = &dhd->pub;
+
+	dev = dhd_linux_get_primary_netdev(dhdp);
+
+	if (!dev) {
+		DHD_ERROR(("%s: Cannot find primary netdev\n", __FUNCTION__));
+		dhdp->hang_was_sent = 0;
+		return;
+	}
+#ifdef WL_CFG80211
+	cfg = wl_get_cfg(dev);
+	if (!cfg) {
+		DHD_ERROR(("%s: Cannot find cfg\n", __FUNCTION__));
+		dhdp->hang_was_sent = 0;
+		return;
+	}
+	/* Skip sending HANG event to framework if driver is not ready */
+	if (!wl_get_drv_status(cfg, READY, dev)) {
+		DHD_ERROR(("%s: device is not ready\n", __FUNCTION__));
+		dhdp->hang_was_sent = 0;
+		return;
+	}
+#endif /* WL_CFG80211 */
+#if defined(WL_WIRELESS_EXT)
+	wl_iw_send_priv_event(dev, "HANG");
+#endif
+#if defined(WL_CFG80211)
+	wl_cfg80211_hang(dev, WLAN_REASON_UNSPECIFIED);
+#endif
+
+#ifdef IFACE_HANG_FORCE_DEV_CLOSE
+	/*
+	 * In case of wif scan only mode, upper layer doesn't handle hang
+	 * So dev_close need to be called explicitly
+	 */
+#ifdef HANG_DELAY_BEFORE_DEV_CLOSE
+	{
+		int wait_cnt = WAIT_FOR_DEV_CLOSE_MAX;
+		while (dev && (dev->flags & IFF_UP) && (wait_cnt > 0)) {
+			wait_cnt--;
+			OSL_SLEEP(10);
+		}
+		DHD_ERROR(("dev->name : %s wait for interface down done, wait_cnt:%d\n",
+			((dev == NULL) ? "null" : dev->name), wait_cnt));
+	}
+#endif /* HANG_DELAY_BEFORE_DEV_CLOSE */
+
+	rtnl_lock();
+	for (i = 0; i < DHD_MAX_IFS && dhd; i++) {
+		ndev = dhd->iflist[i] ? dhd->iflist[i]->net : NULL;
+		if (ndev && (ndev->flags & IFF_UP)) {
+			DHD_ERROR(("ndev->name : %s dev close\n",
+					ndev->name));
+#ifdef ENABLE_INSMOD_NO_FW_LOAD
+			dhd_download_fw_on_driverload = FALSE;
+#endif
+			dev_close(ndev);
+		}
+	}
+	rtnl_unlock();
+#endif /* IFACE_HANG_FORCE_DEV_CLOSE */
+}
+
+#if defined(CONFIG_ARCH_EXYNOS) && defined(BCMPCIE)
+extern dhd_pub_t *link_recovery;
+void dhd_host_recover_link(void)
+{
+	DHD_ERROR(("****** %s ******\n", __FUNCTION__));
+	link_recovery->hang_reason = HANG_REASON_PCIE_LINK_DOWN_RC_DETECT;
+	dhd_bus_set_linkdown(link_recovery, TRUE);
+	dhd_os_send_hang_message(link_recovery);
+}
+EXPORT_SYMBOL(dhd_host_recover_link);
+#endif /* CONFIG_ARCH_EXYNOS && BCMPCIE */
+
+/*
+ * Note that this function should be called in both interrupt and process context
+ * Therefore, Keep it simple, and use workqueue for complex/time-consuming work
+ */
+int dhd_os_send_hang_message(dhd_pub_t *dhdp)
+{
+	int ret = 0;
+	dhd_info_t *dhd_info = NULL;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is null\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	dhd_info = (dhd_info_t *)dhdp->info;
+	BCM_REFERENCE(dhd_info);
+#ifdef DHD_FW_COREDUMP
+	if (dhdp->memdump_enabled == DUMP_MEMFILE_BUGON &&
+		(dhdp->hang_reason == HANG_REASON_PCIE_LINK_DOWN_RC_DETECT ||
+		dhdp->hang_reason == HANG_REASON_PCIE_LINK_DOWN_EP_DETECT)) {
+			DHD_ERROR(("%s: BUG_ON for pcie link down memdump_enabled:%d\n",
+				__FUNCTION__, dhdp->memdump_enabled));
+			BUG_ON(1);
+	}
+#endif /* DHD_FW_COREDUMP */
+#if defined(WLAN_ACCEL_BOOT)
+	if (!dhd_info->wl_accel_force_reg_on) {
+		DHD_ERROR(("%s: set force reg on\n", __FUNCTION__));
+		dhd_info->wl_accel_force_reg_on = TRUE;
+	}
+#endif /* WLAN_ACCEL_BOOT */
+
+	if (!dhdp->hang_report) {
+		DHD_ERROR(("%s: hang_report is disabled\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhdp->stop_in_progress) {
+		DHD_ERROR(("%s: dhd_stop in progress\n", __FUNCTION__));
+		return BCME_OK;
+	}
+
+#if defined(WL_CFG80211) && (defined(DHD_FILE_DUMP_EVENT) || defined(DHD_COREDUMP))
+	if (dhd_info->scheduled_memdump) {
+		DHD_ERROR_RLMT(("[DUMP]:%s, memdump in progress. return\n", __FUNCTION__));
+		dhdp->hang_was_pending = 1;
+		return BCME_OK;
+	}
+#endif /* WL_CFG80211 && (DHD_FILE_DUMP_EVENT || DHD_COREDUMP) */
+
+#if defined(DHD_HANG_SEND_UP_TEST)
+	if (dhdp->req_hang_type) {
+		DHD_ERROR(("%s, Clear HANG test request 0x%x\n",
+			__FUNCTION__, dhdp->req_hang_type));
+		dhdp->req_hang_type = 0;
+	}
+#endif /* DHD_HANG_SEND_UP_TEST */
+
+	if (!dhdp->hang_was_sent) {
+#ifdef DHD_DETECT_CONSECUTIVE_MFG_HANG
+		if (dhdp->op_mode & DHD_FLAG_MFG_MODE) {
+			dhdp->hang_count++;
+			if (dhdp->hang_count >= MAX_CONSECUTIVE_MFG_HANG_COUNT) {
+				DHD_ERROR(("%s, Consecutive hang from Dongle :%u\n",
+					__FUNCTION__, dhdp->hang_count));
+				BUG_ON(1);
+			}
+		}
+#endif /* DHD_DETECT_CONSECUTIVE_MFG_HANG */
+#ifdef DHD_DEBUG_UART
+		/* If PCIe lane has broken, execute the debug uart application
+		 * to gether a ramdump data from dongle via uart
+		 */
+		if (!dhdp->info->duart_execute) {
+			dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq,
+					(void *)dhdp, DHD_WQ_WORK_DEBUG_UART_DUMP,
+					dhd_debug_uart_exec_rd, DHD_WQ_WORK_PRIORITY_HIGH);
+		}
+#endif	/* DHD_DEBUG_UART */
+		dhdp->hang_was_sent = 1;
+#ifdef BT_OVER_SDIO
+		dhdp->is_bt_recovery_required = TRUE;
+#endif
+		schedule_work(&dhdp->info->dhd_hang_process_work);
+		DHD_ERROR(("%s: Event HANG send up due to  re=%d te=%d s=%d\n", __FUNCTION__,
+			dhdp->rxcnt_timeout, dhdp->txcnt_timeout, dhdp->busstate));
+		printf("%s\n", info_string);
+		printf("MAC %pM\n", &dhdp->mac);
+	}
+	return ret;
+}
+
+int net_os_send_hang_message(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd) {
+		/* Report FW problem when enabled */
+		if (dhd->pub.hang_report) {
+#ifdef BT_OVER_SDIO
+			if (netif_running(dev))
+#endif /* BT_OVER_SDIO */
+			{
+				ret = dhd_os_send_hang_message(&dhd->pub);
+			}
+#ifdef BT_OVER_SDIO
+			DHD_ERROR(("%s: HANG -> Reset BT\n", __FUNCTION__));
+			bcmsdh_btsdio_process_dhd_hang_notification(!netif_running(dev));
+#endif /* BT_OVER_SDIO */
+		} else {
+			DHD_ERROR(("%s: FW HANG ignored (for testing purpose) and not sent up\n",
+				__FUNCTION__));
+		}
+	}
+	return ret;
+}
+
+int net_os_send_hang_message_reason(struct net_device *dev, const char *string_num)
+{
+	dhd_info_t *dhd = NULL;
+	dhd_pub_t *dhdp = NULL;
+	int reason;
+
+	dhd = DHD_DEV_INFO(dev);
+	if (dhd) {
+		dhdp = &dhd->pub;
+	}
+
+	if (!dhd || !dhdp) {
+		return 0;
+	}
+
+	reason = bcm_strtoul(string_num, NULL, 0);
+	DHD_INFO(("%s: Enter, reason=0x%x\n", __FUNCTION__, reason));
+
+	if ((reason <= HANG_REASON_MASK) || (reason >= HANG_REASON_MAX)) {
+		reason = 0;
+	}
+
+	dhdp->hang_reason = reason;
+
+	return net_os_send_hang_message(dev);
+}
+#endif /* OEM_ANDROID */
+
+int dhd_net_wifi_platform_set_power(struct net_device *dev, bool on, unsigned long delay_msec)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	return wifi_platform_set_power(dhd->adapter, on, delay_msec);
+}
+
+int dhd_wifi_platform_set_power(dhd_pub_t *pub, bool on)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long delay_msec = on ? WIFI_TURNON_DELAY : WIFI_TURNOFF_DELAY;
+	return wifi_platform_set_power(dhd->adapter, on, delay_msec);
+}
+
+bool dhd_force_country_change(struct net_device *dev)
+{
+#if defined(CUSTOM_FORCE_NODFS_FLAG) && defined(CUSTOM_COUNTRY_CODE)
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	if (dhd && dhd->pub.up && dhd->pub.force_country_change) {
+		DHD_ERROR(("force country change\n"));
+		return TRUE;
+	}
+#endif /* CUSTOM_FORCE_NODFS_FLAG && CUSTOM_COUNTRY_CODE */
+	DHD_ERROR(("skip force country change\n"));
+	return FALSE;
+}
+
+void dhd_get_customized_country_code(struct net_device *dev, char *country_iso_code,
+	wl_country_t *cspec)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+	if (!dhd->pub.is_blob)
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+	{
+#if defined(CUSTOM_COUNTRY_CODE)
+		get_customized_country_code(dhd->adapter, country_iso_code, cspec,
+			dhd->pub.dhd_cflags);
+#else
+		get_customized_country_code(dhd->adapter, country_iso_code, cspec);
+#endif /* CUSTOM_COUNTRY_CODE */
+	}
+#if defined(DHD_BLOB_EXISTENCE_CHECK) && !defined(CUSTOM_COUNTRY_CODE)
+	else {
+		/* Replace the ccode to XZ if ccode is undefined country or world
+		 * domain
+		 */
+		if ((strncmp(country_iso_code, "", WLC_CNTRY_BUF_SZ) == 0) ||
+				(strncmp(country_iso_code, "00", WLC_CNTRY_BUF_SZ) == 0)) {
+			strlcpy(country_iso_code, "XZ", WLC_CNTRY_BUF_SZ);
+			strlcpy(cspec->country_abbrev, country_iso_code, WLC_CNTRY_BUF_SZ);
+			strlcpy(cspec->ccode, country_iso_code, WLC_CNTRY_BUF_SZ);
+			DHD_ERROR(("%s: ccode change to %s\n", __FUNCTION__, country_iso_code));
+		}
+	}
+#endif /* DHD_BLOB_EXISTENCE_CHECK && !CUSTOM_COUNTRY_CODE */
+
+#ifdef KEEP_JP_REGREV
+/* XXX Needed by customer's request */
+	if (strncmp(country_iso_code, "JP", 3) == 0) {
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+		if (dhd->pub.is_blob) {
+			if (strncmp(dhd->pub.vars_ccode, "J1", 3) == 0) {
+				memcpy(cspec->ccode, dhd->pub.vars_ccode,
+					sizeof(dhd->pub.vars_ccode));
+			}
+		} else
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+		{
+			if (strncmp(dhd->pub.vars_ccode, "JP", 3) == 0) {
+				cspec->rev = dhd->pub.vars_regrev;
+			}
+		}
+	}
+#endif /* KEEP_JP_REGREV */
+	BCM_REFERENCE(dhd);
+}
+
+void dhd_bus_country_set(struct net_device *dev, wl_country_t *cspec, bool notify)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+#ifdef WL_CFG80211
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+#endif
+
+	if (dhd && dhd->pub.up) {
+		memcpy(&dhd->pub.dhd_cspec, cspec, sizeof(wl_country_t));
+		dhd->pub.force_country_change = FALSE;
+#ifdef WL_CFG80211
+		wl_update_wiphybands(cfg, notify);
+#endif
+	}
+}
+
+void dhd_bus_band_set(struct net_device *dev, uint band)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+#ifdef WL_CFG80211
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+#endif
+	if (dhd && dhd->pub.up) {
+#ifdef WL_CFG80211
+		wl_update_wiphybands(cfg, true);
+#endif
+	}
+}
+
+int dhd_net_set_fw_path(struct net_device *dev, char *fw)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	if (!fw || fw[0] == '\0')
+		return -EINVAL;
+
+	strlcpy(dhd->fw_path, fw, sizeof(dhd->fw_path));
+
+#if defined(OEM_ANDROID) && defined(SOFTAP)
+	if (strstr(fw, "apsta") != NULL) {
+		DHD_INFO(("GOT APSTA FIRMWARE\n"));
+		ap_fw_loaded = TRUE;
+	} else {
+		DHD_INFO(("GOT STA FIRMWARE\n"));
+		ap_fw_loaded = FALSE;
+	}
+#endif /* defined(OEM_ANDROID) && defined(SOFTAP) */
+	return 0;
+}
+
+void dhd_net_if_lock(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	dhd_net_if_lock_local(dhd);
+}
+
+void dhd_net_if_unlock(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	dhd_net_if_unlock_local(dhd);
+}
+
+bool dhd_net_if_locked(struct net_device *dev)
+{
+	bool locked = FALSE;
+
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	if (dhd) {
+		if (mutex_is_locked(&dhd->dhd_net_if_mutex))
+			locked = TRUE;
+	}
+	return locked;
+}
+
+static void dhd_net_if_lock_local(dhd_info_t *dhd)
+{
+#if defined(OEM_ANDROID)
+	if (dhd)
+		mutex_lock(&dhd->dhd_net_if_mutex);
+#endif
+}
+
+static void dhd_net_if_unlock_local(dhd_info_t *dhd)
+{
+#if defined(OEM_ANDROID)
+	if (dhd)
+		mutex_unlock(&dhd->dhd_net_if_mutex);
+#endif
+}
+
+static void dhd_suspend_lock(dhd_pub_t *pub)
+{
+#if defined(OEM_ANDROID)
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	if (dhd)
+		mutex_lock(&dhd->dhd_suspend_mutex);
+#endif
+}
+
+static void dhd_suspend_unlock(dhd_pub_t *pub)
+{
+#if defined(OEM_ANDROID)
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	if (dhd)
+		mutex_unlock(&dhd->dhd_suspend_mutex);
+#endif
+}
+
+unsigned long dhd_os_general_spin_lock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags = 0;
+
+	if (dhd) {
+		flags = osl_spin_lock(&dhd->dhd_lock);
+	}
+
+	return flags;
+}
+
+void dhd_os_general_spin_unlock(dhd_pub_t *pub, unsigned long flags)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		osl_spin_unlock(&dhd->dhd_lock, flags);
+	}
+}
+
+static int
+dhd_get_pend_8021x_cnt(dhd_info_t *dhd)
+{
+	return (atomic_read(&dhd->pend_8021x_cnt));
+}
+
+#define MAX_WAIT_FOR_8021X_TX	100
+
+int
+dhd_wait_pend8021x(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int timeout = msecs_to_jiffies(10);
+	int ntimes = MAX_WAIT_FOR_8021X_TX;
+	int pend = dhd_get_pend_8021x_cnt(dhd);
+
+	while (ntimes && pend) {
+		if (pend) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(timeout);
+			set_current_state(TASK_RUNNING);
+			ntimes--;
+		}
+		pend = dhd_get_pend_8021x_cnt(dhd);
+	}
+	if (ntimes == 0)
+	{
+		atomic_set(&dhd->pend_8021x_cnt, 0);
+		WL_MSG(dev->name, "TIMEOUT\n");
+	}
+	return pend;
+}
+
+#if defined(BCM_ROUTER_DHD) || defined(DHD_DEBUG)
+int write_file(const char * file_name, uint32 flags, uint8 *buf, int size)
+{
+	int ret = 0;
+	struct file *fp = NULL;
+#ifdef get_fs
+	mm_segment_t old_fs;
+#endif /* get_fs */
+	loff_t pos = 0;
+
+#ifdef get_fs
+	/* change to KERNEL_DS address limit */
+	old_fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	/* open file to write */
+	fp = dhd_filp_open(file_name, flags, 0664);
+	if (IS_ERR(fp) || (fp == NULL)) {
+		DHD_ERROR(("open file error, err = %ld\n", PTR_ERR(fp)));
+		goto exit;
+	}
+
+	/* Write buf to file */
+	ret = dhd_vfs_write(fp, buf, size, &pos);
+	if (ret < 0) {
+		DHD_ERROR(("write file error, err = %d\n", ret));
+		goto exit;
+	}
+
+	/* Sync file from filesystem to physical media */
+	ret = dhd_vfs_fsync(fp, 0);
+	if (ret < 0) {
+		DHD_ERROR(("sync file error, error = %d\n", ret));
+		goto exit;
+	}
+	ret = BCME_OK;
+
+exit:
+	/* close file before return */
+	if (!IS_ERR(fp))
+		dhd_filp_close(fp, current->files);
+
+#ifdef get_fs
+	/* restore previous address limit */
+	set_fs(old_fs);
+#endif /* get_fs */
+
+	return ret;
+}
+#endif /* BCM_ROUTER_DHD || DHD_DEBUG */
+
+#ifdef DHD_DEBUG
+void
+dhd_get_memdump_filename(struct net_device *ndev, char *memdump_path, int len, char *fname)
+{
+	char memdump_type[DHD_MEMDUMP_TYPE_STR_LEN];
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(ndev);
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	/* Init file name */
+	memset(memdump_path, 0, len);
+	memset(memdump_type, 0, DHD_MEMDUMP_TYPE_STR_LEN);
+#ifdef DHD_LOG_DUMP
+	dhd_convert_memdump_type_to_str(dhdp->memdump_type, memdump_type, DHD_MEMDUMP_TYPE_STR_LEN,
+		dhdp->debug_dump_subcmd);
+	clear_debug_dump_time(dhdp->debug_dump_time_str);
+	get_debug_dump_time(dhdp->debug_dump_time_str);
+#endif /* DHD_LOG_DUMP */
+	snprintf(memdump_path, len, "%s%s_%s_" "%s",
+		DHD_COMMON_DUMP_PATH, fname, memdump_type,  dhdp->debug_dump_time_str);
+
+	if (strstr(fname, "sssr_dump")) {
+		DHD_SSSR_PRINT_FILEPATH(dhdp, memdump_path);
+	} else {
+		DHD_ERROR(("%s: file_path = %s%s\n", __FUNCTION__,
+			memdump_path, FILE_NAME_HAL_TAG));
+	}
+}
+
+int
+write_dump_to_file(dhd_pub_t *dhd, uint8 *buf, int size, char *fname)
+{
+	int ret = 0;
+	char memdump_path[DHD_MEMDUMP_PATH_STR_LEN];
+	char memdump_type[DHD_MEMDUMP_TYPE_STR_LEN];
+	uint32 file_mode;
+
+	/* Init file name */
+	memset(memdump_path, 0, DHD_MEMDUMP_PATH_STR_LEN);
+	memset(memdump_type, 0, DHD_MEMDUMP_TYPE_STR_LEN);
+#ifdef DHD_LOG_DUMP
+	dhd_convert_memdump_type_to_str(dhd->memdump_type, memdump_type, DHD_MEMDUMP_TYPE_STR_LEN,
+			dhd->debug_dump_subcmd);
+	clear_debug_dump_time(dhd->debug_dump_time_str);
+	get_debug_dump_time(dhd->debug_dump_time_str);
+#endif /* DHD_LOG_DUMP */
+
+	snprintf(memdump_path, sizeof(memdump_path), "%s%s_%s_" "%s",
+		DHD_COMMON_DUMP_PATH, fname, memdump_type, dhd->debug_dump_time_str);
+#if defined(BOARD_HIKEY)
+	file_mode = O_CREAT | O_WRONLY | O_SYNC;
+#elif defined(OEM_ANDROID) && defined(__ARM_ARCH_7A__)
+	file_mode = O_CREAT | O_WRONLY;
+#elif defined(OEM_ANDROID)
+	/* Extra flags O_DIRECT and O_SYNC are required for Brix Android, as we are
+	 * calling BUG_ON immediately after collecting the socram dump.
+	 * So the file write operation should directly write the contents into the
+	 * file instead of caching it. O_TRUNC flag ensures that file will be re-written
+	 * instead of appending.
+	 */
+	file_mode = O_CREAT | O_WRONLY | O_SYNC;
+	{
+		struct file *fp = dhd_filp_open(memdump_path, file_mode, 0664);
+		/* Check if it is live Brix image having /installmedia, else use /data */
+		if (IS_ERR(fp) || (fp == NULL)) {
+			DHD_ERROR(("open file %s, try /data/\n", memdump_path));
+			snprintf(memdump_path, sizeof(memdump_path), "%s%s_%s_" "%s",
+				"/data/", fname, memdump_type,  dhd->debug_dump_time_str);
+		} else {
+			dhd_filp_close(fp, NULL);
+		}
+	}
+#else
+	file_mode = O_CREAT | O_WRONLY;
+#endif /* BOARD_HIKEY */
+
+	/* print SOCRAM dump file path */
+	DHD_ERROR(("%s: file_path = %s\n", __FUNCTION__, memdump_path));
+
+#ifdef DHD_LOG_DUMP
+	dhd_print_buf_addr(dhd, "write_dump_to_file", buf, size);
+#endif /* DHD_LOG_DUMP */
+
+	/* Write file */
+	ret = write_file(memdump_path, file_mode, buf, size);
+
+#ifdef DHD_DUMP_MNGR
+	if (ret == BCME_OK) {
+		dhd_dump_file_manage_enqueue(dhd, memdump_path, fname);
+	}
+#endif /* DHD_DUMP_MNGR */
+
+	return ret;
+}
+#endif /* DHD_DEBUG */
+
+int dhd_os_wake_lock_timeout(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		ret = dhd->wakelock_rx_timeout_enable > dhd->wakelock_ctrl_timeout_enable ?
+			dhd->wakelock_rx_timeout_enable : dhd->wakelock_ctrl_timeout_enable;
+#ifdef CONFIG_HAS_WAKELOCK
+#ifdef DHD_DEBUG_WAKE_LOCK
+		printf("%s: rx_timeout=%dms, ctrl_timeout=%dms\n", __FUNCTION__,
+		dhd->wakelock_rx_timeout_enable, dhd->wakelock_ctrl_timeout_enable);
+#endif
+		if (dhd->wakelock_rx_timeout_enable)
+			dhd_wake_lock_timeout(dhd->wl_rxwake,
+				msecs_to_jiffies(dhd->wakelock_rx_timeout_enable));
+		if (dhd->wakelock_ctrl_timeout_enable)
+			dhd_wake_lock_timeout(dhd->wl_ctrlwake,
+				msecs_to_jiffies(dhd->wakelock_ctrl_timeout_enable));
+#endif
+		dhd->wakelock_rx_timeout_enable = 0;
+		dhd->wakelock_ctrl_timeout_enable = 0;
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return ret;
+}
+
+int net_os_wake_lock_timeout(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd)
+		ret = dhd_os_wake_lock_timeout(&dhd->pub);
+	return ret;
+}
+
+int dhd_os_wake_lock_ctrl_timeout_enable(dhd_pub_t *pub, int val)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		if (val > dhd->wakelock_ctrl_timeout_enable)
+			dhd->wakelock_ctrl_timeout_enable = val;
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return 0;
+}
+
+int dhd_os_wake_lock_ctrl_timeout_cancel(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		dhd->wakelock_ctrl_timeout_enable = 0;
+#ifdef CONFIG_HAS_WAKELOCK
+		if (dhd_wake_lock_active(dhd->wl_ctrlwake))
+			dhd_wake_unlock(dhd->wl_ctrlwake);
+#endif
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return 0;
+}
+
+int net_os_wake_lock_ctrl_timeout_enable(struct net_device *dev, int val)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd)
+		ret = dhd_os_wake_lock_ctrl_timeout_enable(&dhd->pub, val);
+	return ret;
+}
+
+#if defined(DHD_TRACE_WAKE_LOCK)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+#include <linux/hashtable.h>
+/* Define 2^5 = 32 bucket size hash table */
+DEFINE_HASHTABLE(wklock_history, 5);
+#else
+#include <linux/hash.h>
+/* Define 2^5 = 32 bucket size hash table */
+struct hlist_head wklock_history[32] = { [0 ... 31] = HLIST_HEAD_INIT };
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0) */
+
+atomic_t trace_wklock_onoff;
+static struct wk_trace_record *find_wklock_entry(unsigned long addr)
+{
+	struct wk_trace_record *wklock_info;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	hash_for_each_possible(wklock_history, wklock_info, wklock_node, addr)
+#else
+	struct hlist_node *entry;
+	int index = hash_long(addr, ilog2(ARRAY_SIZE(wklock_history)));
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	hlist_for_each_entry(wklock_info, entry, &wklock_history[index], wklock_node)
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0) */
+	{
+		GCC_DIAGNOSTIC_POP();
+		if (wklock_info->addr == addr) {
+			return wklock_info;
+		}
+	}
+	return NULL;
+}
+
+static inline void dhd_wk_lock_rec_dump(void)
+{
+	int bkt;
+	struct wk_trace_record *wklock_info;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	hash_for_each(wklock_history, bkt, wklock_info, wklock_node)
+#else
+	struct hlist_node *entry = NULL;
+	int max_index = ARRAY_SIZE(wklock_history);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	for (bkt = 0; bkt < max_index; bkt++)
+		hlist_for_each_entry(wklock_info, entry, &wklock_history[bkt], wklock_node)
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0) */
+		{
+			GCC_DIAGNOSTIC_POP();
+			switch (wklock_info->lock_type) {
+				case DHD_WAKE_LOCK:
+					DHD_ERROR(("wakelock lock : %pS  lock_counter : %llu \n",
+						(void *)wklock_info->addr, wklock_info->counter));
+					break;
+				case DHD_WAKE_UNLOCK:
+					DHD_ERROR(("wakelock unlock : %pS,"
+						" unlock_counter : %llu \n",
+						(void *)wklock_info->addr, wklock_info->counter));
+					break;
+				case DHD_WAIVE_LOCK:
+					DHD_ERROR(("wakelock waive : %pS  before_waive : %llu \n",
+						(void *)wklock_info->addr, wklock_info->counter));
+					break;
+				case DHD_RESTORE_LOCK:
+					DHD_ERROR(("wakelock restore : %pS, after_waive : %llu \n",
+						(void *)wklock_info->addr, wklock_info->counter));
+					break;
+			}
+		}
+}
+
+static void dhd_wk_lock_trace_init(struct dhd_info *dhd)
+{
+	unsigned long flags;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0))
+	int i;
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0) */
+
+	DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+	hash_init(wklock_history);
+#else
+	for (i = 0; i < ARRAY_SIZE(wklock_history); i++)
+		INIT_HLIST_HEAD(&wklock_history[i]);
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0) */
+	DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	atomic_set(&trace_wklock_onoff, 1);
+}
+
+static void dhd_wk_lock_trace_deinit(struct dhd_info *dhd)
+{
+	int bkt;
+	struct wk_trace_record *wklock_info;
+	struct hlist_node *tmp;
+	unsigned long flags;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0))
+	struct hlist_node *entry = NULL;
+	int max_index = ARRAY_SIZE(wklock_history);
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0) */
+
+	DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+	hash_for_each_safe(wklock_history, bkt, tmp, wklock_info, wklock_node)
+#else
+	for (bkt = 0; bkt < max_index; bkt++)
+		hlist_for_each_entry_safe(wklock_info, entry, tmp,
+			&wklock_history[bkt], wklock_node)
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0)) */
+		{
+			GCC_DIAGNOSTIC_POP();
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+			hash_del(&wklock_info->wklock_node);
+#else
+			hlist_del_init(&wklock_info->wklock_node);
+#endif /* KERNEL_VER >= KERNEL_VERSION(3, 7, 0)) */
+			kfree(wklock_info);
+		}
+	DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+}
+
+void dhd_wk_lock_stats_dump(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(dhdp->info);
+	unsigned long flags;
+
+	DHD_ERROR(("DHD Printing wl_wake Lock/Unlock Record \r\n"));
+	DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+	dhd_wk_lock_rec_dump();
+	DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+
+}
+#endif /* ! DHD_TRACE_WAKE_LOCK */
+
+int dhd_os_wake_lock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		if (dhd->wakelock_counter == 0 && !dhd->waive_wakelock) {
+#ifdef CONFIG_HAS_WAKELOCK
+			dhd_wake_lock(dhd->wl_wifi);
+#elif defined(BCMSDIO)
+			dhd_bus_dev_pm_stay_awake(pub);
+#endif
+		}
+#ifdef DHD_TRACE_WAKE_LOCK
+		if (atomic_read(&trace_wklock_onoff)) {
+			STORE_WKLOCK_RECORD(DHD_WAKE_LOCK);
+		}
+#endif /* DHD_TRACE_WAKE_LOCK */
+		dhd->wakelock_counter++;
+		ret = dhd->wakelock_counter;
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+
+	return ret;
+}
+
+void dhd_event_wake_lock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+#ifdef CONFIG_HAS_WAKELOCK
+		dhd_wake_lock(dhd->wl_evtwake);
+#elif defined(BCMSDIO)
+		dhd_bus_dev_pm_stay_awake(pub);
+#endif
+	}
+}
+
+void
+dhd_pm_wake_lock_timeout(dhd_pub_t *pub, int val)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		dhd_wake_lock_timeout(dhd->wl_pmwake, msecs_to_jiffies(val));
+	}
+#endif /* CONFIG_HAS_WAKE_LOCK */
+}
+
+void
+dhd_nan_wake_lock_timeout(dhd_pub_t *pub, int val)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		dhd_wake_lock_timeout(dhd->wl_nanwake, msecs_to_jiffies(val));
+	}
+#endif /* CONFIG_HAS_WAKE_LOCK */
+}
+
+int net_os_wake_lock(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd)
+		ret = dhd_os_wake_lock(&dhd->pub);
+	return ret;
+}
+
+int dhd_os_wake_unlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	dhd_os_wake_lock_timeout(pub);
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+
+		if (dhd->wakelock_counter > 0) {
+			dhd->wakelock_counter--;
+#ifdef DHD_TRACE_WAKE_LOCK
+			if (atomic_read(&trace_wklock_onoff)) {
+				STORE_WKLOCK_RECORD(DHD_WAKE_UNLOCK);
+			}
+#endif /* DHD_TRACE_WAKE_LOCK */
+			if (dhd->wakelock_counter == 0 && !dhd->waive_wakelock) {
+#ifdef CONFIG_HAS_WAKELOCK
+				dhd_wake_unlock(dhd->wl_wifi);
+#elif defined(BCMSDIO)
+				dhd_bus_dev_pm_relax(pub);
+#endif
+			}
+			ret = dhd->wakelock_counter;
+		}
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return ret;
+}
+
+void dhd_event_wake_unlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+#ifdef CONFIG_HAS_WAKELOCK
+		dhd_wake_unlock(dhd->wl_evtwake);
+#elif defined(BCMSDIO)
+		dhd_bus_dev_pm_relax(pub);
+#endif
+	}
+}
+
+void dhd_pm_wake_unlock(dhd_pub_t *pub)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		/* if wl_pmwake is active, unlock it */
+		if (dhd_wake_lock_active(dhd->wl_pmwake)) {
+			dhd_wake_unlock(dhd->wl_pmwake);
+		}
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+
+void dhd_txfl_wake_unlock(dhd_pub_t *pub)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		/* if wl_txflwake is active, unlock it */
+		if (dhd_wake_lock_active(dhd->wl_txflwake)) {
+			dhd_wake_unlock(dhd->wl_txflwake);
+		}
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+
+void dhd_nan_wake_unlock(dhd_pub_t *pub)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		/* if wl_nanwake is active, unlock it */
+		if (dhd_wake_lock_active(dhd->wl_nanwake)) {
+			dhd_wake_unlock(dhd->wl_nanwake);
+		}
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+
+int dhd_os_check_wakelock(dhd_pub_t *pub)
+{
+#if defined(CONFIG_HAS_WAKELOCK) || defined(BCMSDIO)
+#if defined(CONFIG_HAS_WAKELOCK)
+	int l1, l2;
+	int c, lock_active;
+#endif /* CONFIG_HAS_WAKELOCK */
+	dhd_info_t *dhd;
+
+	if (!pub)
+		return 0;
+	dhd = (dhd_info_t *)(pub->info);
+	if (!dhd) {
+		return 0;
+	}
+#endif /* CONFIG_HAS_WAKELOCK || BCMSDIO */
+
+#ifdef CONFIG_HAS_WAKELOCK
+	c = dhd->wakelock_counter;
+	l1 = dhd_wake_lock_active(dhd->wl_wifi);
+	l2 = dhd_wake_lock_active(dhd->wl_wdwake);
+	lock_active = (l1 || l2);
+	/* Indicate to the SD Host to avoid going to suspend if internal locks are up */
+	if (lock_active) {
+		DHD_ERROR(("%s wakelock c-%d wl-%d wd-%d\n",
+			__FUNCTION__, c, l1, l2));
+		return 1;
+	}
+#elif defined(BCMSDIO)
+	if (dhd && (dhd->wakelock_counter > 0) && dhd_bus_dev_pm_enabled(pub)) {
+		DHD_ERROR(("%s wakelock c-%d\n", __FUNCTION__, dhd->wakelock_counter));
+		return 1;
+	}
+#endif
+	return 0;
+}
+
+int
+dhd_os_check_wakelock_all(dhd_pub_t *pub)
+{
+#if defined(CONFIG_HAS_WAKELOCK) || defined(BCMSDIO)
+#if defined(CONFIG_HAS_WAKELOCK)
+	int l1, l2, l3, l4, l7, l8, l9, l10;
+	int l5 = 0, l6 = 0;
+	int c, lock_active;
+#endif /* CONFIG_HAS_WAKELOCK */
+	dhd_info_t *dhd;
+
+	if (!pub) {
+		return 0;
+	}
+	if (pub->up == 0) {
+		DHD_ERROR(("%s: skip as down in progress\n", __FUNCTION__));
+		return 0;
+	}
+	dhd = (dhd_info_t *)(pub->info);
+	if (!dhd) {
+		return 0;
+	}
+#endif /* CONFIG_HAS_WAKELOCK || BCMSDIO */
+
+#ifdef CONFIG_HAS_WAKELOCK
+	c = dhd->wakelock_counter;
+	l1 = dhd_wake_lock_active(dhd->wl_wifi);
+	l2 = dhd_wake_lock_active(dhd->wl_wdwake);
+	l3 = dhd_wake_lock_active(dhd->wl_rxwake);
+	l4 = dhd_wake_lock_active(dhd->wl_ctrlwake);
+	l7 = dhd_wake_lock_active(dhd->wl_evtwake);
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	l5 = dhd_wake_lock_active(dhd->wl_intrwake);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef DHD_USE_SCAN_WAKELOCK
+	l6 = dhd_wake_lock_active(dhd->wl_scanwake);
+#endif /* DHD_USE_SCAN_WAKELOCK */
+	l8 = dhd_wake_lock_active(dhd->wl_pmwake);
+	l9 = dhd_wake_lock_active(dhd->wl_txflwake);
+	l10 = dhd_wake_lock_active(dhd->wl_nanwake);
+	lock_active = (l1 || l2 || l3 || l4 || l5 || l6 || l7 || l8 || l9 || l10);
+
+	/* Indicate to the Host to avoid going to suspend if internal locks are up */
+	DHD_ERROR(("%s wakelock c-%d wl-%d wd-%d rx-%d "
+		"ctl-%d intr-%d scan-%d evt-%d, pm-%d, txfl-%d nan-%d\n",
+		__FUNCTION__, c, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10));
+	if (lock_active) {
+#ifdef RPM_FAST_TRIGGER
+		if (pub->rpm_fast_trigger && l4) {
+			DHD_ERROR(("%s : reset rpm_fast_trigger becasue of wl_ctrlwake activated\n",
+					__FUNCTION__));
+			pub->rpm_fast_trigger = FALSE;
+		}
+#endif /* RPM_FAST_TRIGGER */
+		return 1;
+	}
+#elif defined(BCMSDIO)
+	if (dhd && (dhd->wakelock_counter > 0) && dhd_bus_dev_pm_enabled(pub)) {
+		DHD_ERROR(("%s wakelock c-%d\n", __FUNCTION__, dhd->wakelock_counter));
+		return 1;
+	}
+#endif /* defined(BCMSDIO) */
+	return 0;
+}
+
+int net_os_wake_unlock(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd)
+		ret = dhd_os_wake_unlock(&dhd->pub);
+	return ret;
+}
+
+int dhd_os_wd_wake_lock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	if (dhd) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		if (dhd->wakelock_wd_counter == 0 && !dhd->waive_wakelock) {
+#ifdef CONFIG_HAS_WAKELOCK
+			/* if wakelock_wd_counter was never used : lock it at once */
+			dhd_wake_lock(dhd->wl_wdwake);
+#endif
+		}
+		dhd->wakelock_wd_counter++;
+		ret = dhd->wakelock_wd_counter;
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return ret;
+}
+
+int dhd_os_wd_wake_unlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	if (dhd) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		if (dhd->wakelock_wd_counter > 0) {
+			dhd->wakelock_wd_counter = 0;
+			if (!dhd->waive_wakelock) {
+#ifdef CONFIG_HAS_WAKELOCK
+				dhd_wake_unlock(dhd->wl_wdwake);
+#endif
+			}
+		}
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return ret;
+}
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+void
+dhd_os_oob_irq_wake_lock_timeout(dhd_pub_t *pub, int val)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		dhd_wake_lock_timeout(dhd->wl_intrwake, msecs_to_jiffies(val));
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+
+void
+dhd_os_oob_irq_wake_unlock(dhd_pub_t *pub)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		/* if wl_intrwake is active, unlock it */
+		if (dhd_wake_lock_active(dhd->wl_intrwake)) {
+			dhd_wake_unlock(dhd->wl_intrwake);
+		}
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef DHD_USE_SCAN_WAKELOCK
+void
+dhd_os_scan_wake_lock_timeout(dhd_pub_t *pub, int val)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		dhd_wake_lock_timeout(dhd->wl_scanwake, msecs_to_jiffies(val));
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+
+void
+dhd_os_scan_wake_unlock(dhd_pub_t *pub)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		/* if wl_scanwake is active, unlock it */
+		if (dhd_wake_lock_active(dhd->wl_scanwake)) {
+			dhd_wake_unlock(dhd->wl_scanwake);
+		}
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+#endif /* DHD_USE_SCAN_WAKELOCK */
+
+/* waive wakelocks for operations such as IOVARs in suspend function, must be closed
+ * by a paired function call to dhd_wakelock_restore. returns current wakelock counter
+ */
+int dhd_os_wake_lock_waive(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+
+		/* dhd_wakelock_waive/dhd_wakelock_restore must be paired */
+		if (dhd->waive_wakelock == FALSE) {
+#ifdef DHD_TRACE_WAKE_LOCK
+			if (atomic_read(&trace_wklock_onoff)) {
+				STORE_WKLOCK_RECORD(DHD_WAIVE_LOCK);
+			}
+#endif /* DHD_TRACE_WAKE_LOCK */
+			/* record current lock status */
+			dhd->wakelock_before_waive = dhd->wakelock_counter;
+			dhd->waive_wakelock = TRUE;
+		}
+		ret = dhd->wakelock_wd_counter;
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return ret;
+}
+
+int dhd_os_wake_lock_restore(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+	int ret = 0;
+
+	if (!dhd)
+		return 0;
+	if ((dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT) == 0)
+		return 0;
+
+	DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+
+	/* dhd_wakelock_waive/dhd_wakelock_restore must be paired */
+	if (!dhd->waive_wakelock)
+		goto exit;
+
+	dhd->waive_wakelock = FALSE;
+	/* if somebody else acquires wakelock between dhd_wakelock_waive/dhd_wakelock_restore,
+	 * we need to make it up by calling dhd_wake_lock or pm_stay_awake. or if somebody releases
+	 * the lock in between, do the same by calling dhd_wake_unlock or pm_relax
+	 */
+#ifdef DHD_TRACE_WAKE_LOCK
+	if (atomic_read(&trace_wklock_onoff)) {
+		STORE_WKLOCK_RECORD(DHD_RESTORE_LOCK);
+	}
+#endif /* DHD_TRACE_WAKE_LOCK */
+
+	if (dhd->wakelock_before_waive == 0 && dhd->wakelock_counter > 0) {
+#ifdef CONFIG_HAS_WAKELOCK
+		dhd_wake_lock(dhd->wl_wifi);
+#elif defined(BCMSDIO)
+		dhd_bus_dev_pm_stay_awake(&dhd->pub);
+#endif
+	} else if (dhd->wakelock_before_waive > 0 && dhd->wakelock_counter == 0) {
+#ifdef CONFIG_HAS_WAKELOCK
+		dhd_wake_unlock(dhd->wl_wifi);
+#elif defined(BCMSDIO)
+		dhd_bus_dev_pm_relax(&dhd->pub);
+#endif
+	}
+	dhd->wakelock_before_waive = 0;
+exit:
+	ret = dhd->wakelock_wd_counter;
+	DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	return ret;
+}
+
+void dhd_os_wake_lock_init(struct dhd_info *dhd)
+{
+	DHD_TRACE(("%s: initialize wake_lock_counters\n", __FUNCTION__));
+	dhd->wakelock_counter = 0;
+	dhd->wakelock_rx_timeout_enable = 0;
+	dhd->wakelock_ctrl_timeout_enable = 0;
+	/* wakelocks prevent a system from going into a low power state */
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_wake_lock_init(dhd->wl_wifi, dhd_bus_to_dev(dhd->pub.bus), "wlan_wake");
+	dhd_wake_lock_init(dhd->wl_rxwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_rx_wake");
+	dhd_wake_lock_init(dhd->wl_ctrlwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_ctrl_wake");
+	dhd_wake_lock_init(dhd->wl_evtwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_evt_wake");
+	dhd_wake_lock_init(dhd->wl_pmwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_pm_wake");
+	dhd_wake_lock_init(dhd->wl_txflwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_txfl_wake");
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	dhd_wake_lock_init(dhd->wl_intrwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_oob_irq_wake");
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef DHD_USE_SCAN_WAKELOCK
+	dhd_wake_lock_init(dhd->wl_scanwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_scan_wake");
+#endif /* DHD_USE_SCAN_WAKELOCK */
+	dhd_wake_lock_init(dhd->wl_nanwake, dhd_bus_to_dev(dhd->pub.bus), "wlan_nan_wake");
+#endif /* CONFIG_HAS_WAKELOCK */
+#ifdef DHD_TRACE_WAKE_LOCK
+	dhd_wk_lock_trace_init(dhd);
+#endif /* DHD_TRACE_WAKE_LOCK */
+}
+
+void dhd_os_wake_lock_destroy(struct dhd_info *dhd)
+{
+	DHD_TRACE(("%s: deinit wake_lock_counters\n", __FUNCTION__));
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd->wakelock_counter = 0;
+	dhd->wakelock_rx_timeout_enable = 0;
+	dhd->wakelock_ctrl_timeout_enable = 0;
+	dhd_wake_lock_destroy(dhd->wl_wifi);
+	dhd_wake_lock_destroy(dhd->wl_rxwake);
+	dhd_wake_lock_destroy(dhd->wl_ctrlwake);
+	dhd_wake_lock_destroy(dhd->wl_evtwake);
+	dhd_wake_lock_destroy(dhd->wl_pmwake);
+	dhd_wake_lock_destroy(dhd->wl_txflwake);
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	dhd_wake_lock_destroy(dhd->wl_intrwake);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef DHD_USE_SCAN_WAKELOCK
+	dhd_wake_lock_destroy(dhd->wl_scanwake);
+#endif /* DHD_USE_SCAN_WAKELOCK */
+	dhd_wake_lock_destroy(dhd->wl_nanwake);
+#ifdef DHD_TRACE_WAKE_LOCK
+	dhd_wk_lock_trace_deinit(dhd);
+#endif /* DHD_TRACE_WAKE_LOCK */
+#else /* !CONFIG_HAS_WAKELOCK */
+	if (dhd->wakelock_counter > 0) {
+		DHD_ERROR(("%s: wake lock count=%d\n",
+			__FUNCTION__, dhd->wakelock_counter));
+		while (dhd_os_wake_unlock(&dhd->pub));
+	}
+#endif /* CONFIG_HAS_WAKELOCK */
+}
+
+bool dhd_os_check_if_up(dhd_pub_t *pub)
+{
+	if (!pub)
+		return FALSE;
+	return pub->up;
+}
+
+/* function to collect firmware, chip id and chip version info */
+void dhd_set_version_info(dhd_pub_t *dhdp, char *fw)
+{
+	int i;
+
+	i = snprintf(info_string, sizeof(info_string),
+		"  Driver: %s\n%s  Firmware: %s\n%s  CLM: %s ",
+		EPI_VERSION_STR,
+		DHD_LOG_PREFIXS, fw,
+		DHD_LOG_PREFIXS, clm_version);
+	DHD_INFO(("%s\n", info_string));
+
+	if (!dhdp)
+		return;
+
+	i += snprintf(&info_string[i], sizeof(info_string) - i,
+		"\n%s  Chip: %x Rev %x", DHD_LOG_PREFIXS, dhd_conf_get_chip(dhdp),
+		dhd_conf_get_chiprev(dhdp));
+#if defined(USE_CID_CHECK)
+	i += snprintf(&info_string[i], sizeof(info_string) - i,
+		" VID %x", cur_vid_info);
+#endif /* USE_CID_CHECK */
+}
+
+int dhd_ioctl_entry_local(struct net_device *net, wl_ioctl_t *ioc, int cmd)
+{
+	int ifidx;
+	int ret = 0;
+	dhd_info_t *dhd = NULL;
+
+	if (!net || !DEV_PRIV(net)) {
+		DHD_TRACE(("%s invalid parameter net %p dev_priv %p\n",
+			__FUNCTION__, net, DEV_PRIV(net)));
+		return -EINVAL;
+	}
+
+	dhd = DHD_DEV_INFO(net);
+	if (!dhd)
+		return -EINVAL;
+
+	ifidx = dhd_net2idx(dhd, net);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+
+	ret = dhd_wl_ioctl(&dhd->pub, ifidx, ioc, ioc->buf, ioc->len);
+	dhd_check_hang(net, &dhd->pub, ret);
+
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+
+	return ret;
+}
+
+bool dhd_os_check_hang(dhd_pub_t *dhdp, int ifidx, int ret)
+{
+	struct net_device *net;
+
+	net = dhd_idx2net(dhdp, ifidx);
+	if (!net) {
+		DHD_ERROR(("%s : Invalid index : %d\n", __FUNCTION__, ifidx));
+		return -EINVAL;
+	}
+
+	return dhd_check_hang(net, dhdp, ret);
+}
+
+/* Return instance */
+int dhd_get_instance(dhd_pub_t *dhdp)
+{
+	return dhdp->info->unit;
+}
+
+#if defined(WL_CFG80211) && defined(SUPPORT_DEEP_SLEEP)
+#define MAX_TRY_CNT             5 /* Number of tries to disable deepsleep */
+int dhd_deepsleep(struct net_device *dev, int flag)
+{
+	char iovbuf[20];
+	uint powervar = 0;
+	dhd_info_t *dhd;
+	dhd_pub_t *dhdp;
+	int cnt = 0;
+	int ret = 0;
+
+	dhd = DHD_DEV_INFO(dev);
+	dhdp = &dhd->pub;
+
+	switch (flag) {
+		case 1 :  /* Deepsleep on */
+			DHD_ERROR(("[WiFi] Deepsleep On\n"));
+			/* give some time to sysioc_work before deepsleep */
+			OSL_SLEEP(200);
+#ifdef PKT_FILTER_SUPPORT
+		/* disable pkt filter */
+		dhd_enable_packet_filter(0, dhdp);
+#endif /* PKT_FILTER_SUPPORT */
+			/* Disable MPC */
+			powervar = 0;
+			ret = dhd_iovar(dhdp, 0, "mpc", (char *)&powervar, sizeof(powervar), NULL,
+					0, TRUE);
+			if (ret) {
+				DHD_ERROR(("%s: mpc failed:%d\n", __FUNCTION__, ret));
+			}
+			/* Enable Deepsleep */
+			powervar = 1;
+			ret = dhd_iovar(dhdp, 0, "deepsleep", (char *)&powervar, sizeof(powervar),
+					NULL, 0, TRUE);
+			if (ret) {
+				DHD_ERROR(("%s: deepsleep failed:%d\n", __FUNCTION__, ret));
+			}
+			break;
+
+		case 0: /* Deepsleep Off */
+			DHD_ERROR(("[WiFi] Deepsleep Off\n"));
+
+			/* Disable Deepsleep */
+			for (cnt = 0; cnt < MAX_TRY_CNT; cnt++) {
+				powervar = 0;
+				ret = dhd_iovar(dhdp, 0, "deepsleep", (char *)&powervar,
+						sizeof(powervar), NULL, 0, TRUE);
+				if (ret) {
+					DHD_ERROR(("%s: deepsleep failed:%d\n", __FUNCTION__, ret));
+				}
+
+				ret = dhd_iovar(dhdp, 0, "deepsleep", (char *)&powervar,
+						sizeof(powervar), iovbuf, sizeof(iovbuf), FALSE);
+				if (ret < 0) {
+					DHD_ERROR(("the error of dhd deepsleep status"
+						" ret value :%d\n", ret));
+				} else {
+					if (!(*(int *)iovbuf)) {
+						DHD_ERROR(("deepsleep mode is 0,"
+							" count: %d\n", cnt));
+						break;
+					}
+				}
+			}
+
+			/* Enable MPC */
+			powervar = 1;
+			ret = dhd_iovar(dhdp, 0, "mpc", (char *)&powervar, sizeof(powervar),
+					NULL, 0, TRUE);
+			if (ret) {
+				DHD_ERROR(("%s: mpc failed:%d\n", __FUNCTION__, ret));
+			}
+			break;
+	}
+
+	return 0;
+}
+#endif /* WL_CFG80211 && SUPPORT_DEEP_SLEEP */
+
+#ifdef PROP_TXSTATUS
+
+void dhd_wlfc_plat_init(void *dhd)
+{
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+	dhdsdio_func_blocksize((dhd_pub_t *)dhd, 2, DYNAMIC_F2_BLKSIZE_FOR_NONLEGACY);
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+	return;
+}
+
+void dhd_wlfc_plat_deinit(void *dhd)
+{
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+	dhdsdio_func_blocksize((dhd_pub_t *)dhd, 2, sd_f2_blocksize);
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+	return;
+}
+
+bool dhd_wlfc_skip_fc(void * dhdp, uint8 idx)
+{
+#ifdef SKIP_WLFC_ON_CONCURRENT
+
+#ifdef WL_CFG80211
+	struct net_device * net =  dhd_idx2net((dhd_pub_t *)dhdp, idx);
+	if (net)
+	/* enable flow control in vsdb mode */
+	return !(wl_cfg80211_is_concurrent_mode(net));
+#else
+	return TRUE; /* skip flow control */
+#endif /* WL_CFG80211 */
+
+#else
+	return FALSE;
+#endif /* SKIP_WLFC_ON_CONCURRENT */
+	return FALSE;
+}
+#endif /* PROP_TXSTATUS */
+
+#ifdef BCMDBGFS
+#include <linux/debugfs.h>
+
+typedef struct dhd_dbgfs {
+	struct dentry	*debugfs_dir;
+	struct dentry	*debugfs_mem;
+	dhd_pub_t	*dhdp;
+	uint32		size;
+} dhd_dbgfs_t;
+
+dhd_dbgfs_t g_dbgfs;
+
+extern uint32 dhd_readregl(void *bp, uint32 addr);
+extern uint32 dhd_writeregl(void *bp, uint32 addr, uint32 data);
+
+static int
+dhd_dbg_state_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+static ssize_t
+dhd_dbg_state_read(struct file *file, char __user *ubuf,
+                       size_t count, loff_t *ppos)
+{
+	ssize_t rval;
+	uint32 tmp;
+	loff_t pos = *ppos;
+	size_t ret;
+
+	if (pos < 0)
+		return -EINVAL;
+	if (pos >= g_dbgfs.size || !count)
+		return 0;
+	if (count > g_dbgfs.size - pos)
+		count = g_dbgfs.size - pos;
+
+	/* XXX: The user can request any length they want, but they are getting 4 bytes */
+	/* Basically enforce aligned 4 byte reads. It's up to the user to work out the details */
+	tmp = dhd_readregl(g_dbgfs.dhdp->bus, file->f_pos & (~3));
+
+	ret = copy_to_user(ubuf, &tmp, 4);
+	if (ret == count)
+		return -EFAULT;
+
+	count -= ret;
+	*ppos = pos + count;
+	rval = count;
+
+	return rval;
+}
+
+static ssize_t
+dhd_debugfs_write(struct file *file, const char __user *ubuf, size_t count, loff_t *ppos)
+{
+	loff_t pos = *ppos;
+	size_t ret;
+	uint32 buf;
+
+	if (pos < 0)
+		return -EINVAL;
+	if (pos >= g_dbgfs.size || !count)
+		return 0;
+	if (count > g_dbgfs.size - pos)
+		count = g_dbgfs.size - pos;
+
+	ret = copy_from_user(&buf, ubuf, sizeof(uint32));
+	if (ret == count)
+		return -EFAULT;
+
+	/* XXX: The user can request any length they want, but they are getting 4 bytes */
+	/* Basically enforce aligned 4 byte writes. It's up to the user to work out the details */
+	dhd_writeregl(g_dbgfs.dhdp->bus, file->f_pos & (~3), buf);
+
+	return count;
+}
+
+loff_t
+dhd_debugfs_lseek(struct file *file, loff_t off, int whence)
+{
+	loff_t pos = -1;
+
+	switch (whence) {
+		case 0:
+			pos = off;
+			break;
+		case 1:
+			pos = file->f_pos + off;
+			break;
+		case 2:
+			pos = g_dbgfs.size - off;
+	}
+	return (pos < 0 || pos > g_dbgfs.size) ? -EINVAL : (file->f_pos = pos);
+}
+
+static const struct file_operations dhd_dbg_state_ops = {
+	.read   = dhd_dbg_state_read,
+	.write	= dhd_debugfs_write,
+	.open   = dhd_dbg_state_open,
+	.llseek	= dhd_debugfs_lseek
+};
+
+static void dhd_dbgfs_create(void)
+{
+	if (g_dbgfs.debugfs_dir) {
+		g_dbgfs.debugfs_mem = debugfs_create_file("mem", 0644, g_dbgfs.debugfs_dir,
+			NULL, &dhd_dbg_state_ops);
+	}
+}
+
+void dhd_dbgfs_init(dhd_pub_t *dhdp)
+{
+	g_dbgfs.dhdp = dhdp;
+	g_dbgfs.size = 0x20000000; /* Allow access to various cores regs */
+
+	g_dbgfs.debugfs_dir = debugfs_create_dir("dhd", 0);
+	if (IS_ERR(g_dbgfs.debugfs_dir)) {
+		g_dbgfs.debugfs_dir = NULL;
+		return;
+	}
+
+	dhd_dbgfs_create();
+
+	return;
+}
+
+void dhd_dbgfs_remove(void)
+{
+	debugfs_remove(g_dbgfs.debugfs_mem);
+	debugfs_remove(g_dbgfs.debugfs_dir);
+
+	bzero((unsigned char *) &g_dbgfs, sizeof(g_dbgfs));
+}
+#endif /* BCMDBGFS */
+
+#ifdef CUSTOM_SET_CPUCORE
+void dhd_set_cpucore(dhd_pub_t *dhd, int set)
+{
+	int e_dpc = 0, e_rxf = 0, retry_set = 0;
+
+	if (!(dhd->chan_isvht80)) {
+		DHD_ERROR(("%s: chan_status(%d) cpucore!!!\n", __FUNCTION__, dhd->chan_isvht80));
+		return;
+	}
+
+	if (DPC_CPUCORE && dhd->current_dpc) {
+		do {
+			if (set == TRUE) {
+				e_dpc = set_cpus_allowed_ptr(dhd->current_dpc,
+					cpumask_of(DPC_CPUCORE));
+			} else {
+				e_dpc = set_cpus_allowed_ptr(dhd->current_dpc,
+					cpumask_of(PRIMARY_CPUCORE));
+			}
+			if (retry_set++ > MAX_RETRY_SET_CPUCORE) {
+				DHD_ERROR(("%s: dpc(%d) invalid cpu!\n", __FUNCTION__, e_dpc));
+				return;
+			}
+			if (e_dpc < 0)
+				OSL_SLEEP(1);
+		} while (e_dpc < 0);
+	}
+	if (RXF_CPUCORE && dhd->current_rxf) {
+		do {
+			if (set == TRUE) {
+				e_rxf = set_cpus_allowed_ptr(dhd->current_rxf,
+					cpumask_of(RXF_CPUCORE));
+			} else {
+				e_rxf = set_cpus_allowed_ptr(dhd->current_rxf,
+					cpumask_of(PRIMARY_CPUCORE));
+			}
+			if (retry_set++ > MAX_RETRY_SET_CPUCORE) {
+				DHD_ERROR(("%s: rxf(%d) invalid cpu!\n", __FUNCTION__, e_rxf));
+				return;
+			}
+			if (e_rxf < 0)
+				OSL_SLEEP(1);
+		} while (e_rxf < 0);
+	}
+	DHD_TRACE(("%s: set(%d) cpucore success!\n", __FUNCTION__, set));
+
+	return;
+}
+#endif /* CUSTOM_SET_CPUCORE */
+
+#ifdef DHD_MCAST_REGEN
+/* Get interface specific ap_isolate configuration */
+int dhd_get_mcast_regen_bss_enable(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	return ifp->mcast_regen_bss_enable;
+}
+
+/* Set interface specific mcast_regen configuration */
+int dhd_set_mcast_regen_bss_enable(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ifp->mcast_regen_bss_enable = val;
+
+	/* Disable rx_pkt_chain feature for interface, if mcast_regen feature
+	 * is enabled
+	 */
+	dhd_update_rx_pkt_chainable_state(dhdp, idx);
+	return BCME_OK;
+}
+#endif	/* DHD_MCAST_REGEN */
+
+/* Get interface specific ap_isolate configuration */
+int dhd_get_ap_isolate(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	return ifp->ap_isolate;
+}
+
+/* Set interface specific ap_isolate configuration */
+int dhd_set_ap_isolate(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	if (ifp)
+		ifp->ap_isolate = val;
+
+	return 0;
+}
+
+#ifdef DHD_FW_COREDUMP
+bool dhd_memdump_is_scheduled(dhd_pub_t *dhdp)
+{
+	return dhdp->info->scheduled_memdump;
+}
+
+void dhd_schedule_memdump(dhd_pub_t *dhdp, uint8 *buf, uint32 size)
+{
+	dhd_dump_t *dump = NULL;
+	unsigned long flags = 0;
+	dhd_info_t *dhd_info = NULL;
+#if defined(DHD_LOG_DUMP) && !defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	log_dump_type_t type = DLD_BUF_TYPE_ALL;
+#endif /* DHD_LOG_DUMP && !DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+
+	dhd_info = (dhd_info_t *)dhdp->info;
+	dump = (dhd_dump_t *)MALLOC(dhdp->osh, sizeof(dhd_dump_t));
+	if (dump == NULL) {
+		DHD_ERROR(("%s: dhd dump memory allocation failed\n", __FUNCTION__));
+		return;
+	}
+	dump->buf = buf;
+	dump->bufsize = size;
+#ifdef BCMPCIE
+	dhd_get_hscb_info(dhdp, (void*)(&dump->hscb_buf),
+			(uint32 *)(&dump->hscb_bufsize));
+#else
+	dump->hscb_bufsize = 0;
+#endif /* BCMPCIE */
+
+#ifdef DHD_LOG_DUMP
+	dhd_print_buf_addr(dhdp, "memdump", buf, size);
+#if !defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	/* Print out buffer infomation */
+	dhd_log_dump_buf_addr(dhdp, &type);
+#endif /* !DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+#endif /* DHD_LOG_DUMP */
+
+	if ((dhdp->memdump_type == DUMP_TYPE_DONGLE_INIT_FAILURE) ||
+		(dhdp->memdump_type == DUMP_TYPE_DUE_TO_BT) ||
+		(dhdp->memdump_type == DUMP_TYPE_SMMU_FAULT) ||
+		(dhdp->memdump_type == DUMP_TYPE_DONGLE_TRAP_DURING_WIFI_ONOFF))
+	{
+		dhd_info->scheduled_memdump = FALSE;
+		dhd_mem_dump((void *)dhdp->info, (void *)dump, 0);
+		/* No need to collect debug dump for init failure */
+		if (dhdp->memdump_type == DUMP_TYPE_DONGLE_INIT_FAILURE ||
+			dhdp->memdump_type == DUMP_TYPE_DONGLE_TRAP_DURING_WIFI_ONOFF) {
+			return;
+		}
+#ifdef DHD_LOG_DUMP
+		{
+			log_dump_type_t *flush_type = NULL;
+			/* for dongle init fail cases, 'dhd_mem_dump' does
+			 * not call 'dhd_log_dump', so call it here.
+			*/
+			flush_type = MALLOCZ(dhdp->osh,
+				sizeof(log_dump_type_t));
+			if (flush_type) {
+				*flush_type = DLD_BUF_TYPE_ALL;
+				DHD_ERROR(("%s: calling log dump.. \n", __FUNCTION__));
+				dhd_log_dump(dhdp->info, flush_type, 0);
+			}
+		}
+#endif /* DHD_LOG_DUMP */
+		return;
+	}
+
+	dhd_info->scheduled_memdump = TRUE;
+
+	/* bus busy bit for mem dump will be cleared in mem dump
+	* work item context, after mem dump file is written
+	*/
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_SET_IN_MEMDUMP(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	DHD_ERROR(("%s: scheduling mem dump.. \n", __FUNCTION__));
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq, (void *)dump,
+		DHD_WQ_WORK_SOC_RAM_DUMP, dhd_mem_dump, DHD_WQ_WORK_PRIORITY_HIGH);
+}
+#ifdef DHD_SSSR_DUMP
+#define DUMP_SSSR_DUMP_MAX_COUNT	8
+#endif
+#ifdef DHD_COREDUMP
+#ifdef DHD_LINUX_STD_FW_API
+char map_path[PATH_MAX] = DHD_MAP_NAME;
+#else
+#ifndef CONFIG_BCMDHD_MAP_PATH
+char map_path[PATH_MAX] = PLATFORM_PATH"rtecdc.map";
+#else
+char map_path[PATH_MAX] = VENDOR_PATH CONFIG_BCMDHD_MAP_PATH;
+#endif /* CONFIG_BCMDHD_MAP_PATH */
+#endif /* DHD_LINUX_STD_FW_API */
+extern int dhd_collect_coredump(dhd_pub_t *dhdp, dhd_dump_t *dump);
+#endif /* DHD_COREDUMP */
+
+static void
+dhd_mem_dump(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	dhd_pub_t *dhdp = NULL;
+	unsigned long flags = 0;
+#if defined(DHD_LOG_DUMP)
+#if defined(DHD_FILE_DUMP_EVENT) || defined(DHD_DEBUGABILITY_DEBUG_DUMP)
+	log_dump_type_t type = DLD_BUF_TYPE_ALL;
+#endif /* DHD_FILE_DUMP_EVENT || DHD_DEBUGABILITY_DEBUG_DUMP */
+#endif /* DHD_LOG_DUMP */
+
+#if defined(DHD_FILE_DUMP_EVENT) || (defined(DHD_SSSR_COREDUMP) && \
+	defined(DHD_COREDUMP))
+	int ret = 0;
+#endif /* DHD_FILE_DUMP_EVENT || DHD_SSSR_COREDUMP && DHD_COREDUMP */
+	dhd_dump_t *dump = NULL;
+#ifdef DHD_COREDUMP
+	char pc_fn[DHD_FUNC_STR_LEN] = "\0";
+	char lr_fn[DHD_FUNC_STR_LEN] = "\0";
+	trap_t *tr;
+#endif /* DHD_COREDUMP */
+	uint32 memdump_type;
+
+	DHD_ERROR(("%s: ENTER \n", __FUNCTION__));
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	dhdp = &dhd->pub;
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+	/* to keep it in case other memdump overwrites it */
+	memdump_type = dhdp->memdump_type;
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s: bus is down! can't collect mem dump. \n", __FUNCTION__));
+		goto exit;
+	}
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	dump = (dhd_dump_t *)event_info;
+	if (!dump) {
+		DHD_ERROR(("%s: dump is NULL\n", __FUNCTION__));
+		goto exit;
+	}
+
+#ifdef DHD_SSSR_DUMP
+	DHD_ERROR(("%s: sssr_enab=%d dhdp->sssr_inited=%d dhdp->collect_sssr=%d\n",
+		__FUNCTION__, sssr_enab, dhdp->sssr_inited, dhdp->collect_sssr));
+	if (sssr_enab && dhdp->sssr_inited && dhdp->collect_sssr) {
+		uint32 arr_len[DUMP_SSSR_DUMP_MAX_COUNT];
+
+		if (fis_enab && dhdp->sssr_reg_info->rev3.fis_enab) {
+			int bcmerror = dhd_bus_fis_trigger(dhdp);
+
+			if (bcmerror == BCME_OK) {
+				dhd_bus_fis_dump(dhdp);
+			} else {
+				DHD_ERROR(("%s: FIS trigger failed: %d\n",
+					__FUNCTION__, bcmerror));
+			}
+		} else	{
+			DHD_ERROR(("%s: FIS not enabled (%d:%d), collect legacy sssr\n",
+				__FUNCTION__, fis_enab, dhdp->sssr_reg_info->rev3.fis_enab));
+			dhdpcie_sssr_dump(dhdp);
+		}
+#ifndef DHD_FIS_DUMP
+		/* Reset fis_enab flag after fis collection which was set for init failure debug */
+		fis_enab = FALSE;
+#endif /* DHD_FIS_DUMP */
+		/* Print sssr buffer address for debugging */
+		if (dhdp->sssr_dump_collected) {
+			dhdpcie_sssr_dump_get_before_after_len(dhdp, arr_len);
+		}
+	}
+#endif /* DHD_SSSR_DUMP */
+
+#ifdef DHD_SDTC_ETB_DUMP
+	if (dhdp->collect_sdtc) {
+		dhd_sdtc_etb_dump(dhdp);
+		dhdp->collect_sdtc = FALSE;
+	}
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#if (defined(DHD_FILE_DUMP_EVENT) || defined(DHD_DEBUGABILITY_DEBUG_DUMP))
+	if (dhdp->memdump_enabled == DUMP_MEMONLY) {
+		DHD_ERROR(("%s: Force BUG_ON for memdump_enabled:%d\n",
+			__FUNCTION__, dhdp->memdump_enabled));
+		BUG_ON(1);
+	}
+
+#ifdef DHD_LOG_DUMP
+	if (dhd_log_flush(dhdp, &type) < 0) {
+		DHD_ERROR(("%s: Failed to flush Preserve/FW logs\n",
+			__FUNCTION__));
+	}
+#endif /* DHD_LOG_DUMP */
+
+#if defined(DHD_FILE_DUMP_EVENT)
+	ret = dhd_wait_for_file_dump(dhdp);
+	if (ret) {
+		DHD_ERROR(("%s: file_dump failed.\n", __FUNCTION__));
+#ifdef BOARD_HIKEY
+		/* For Hikey do force kernel write of socram if HAL dump fails */
+		if (write_dump_to_file(&dhd->pub, dump->buf, dump->bufsize, "mem_dump")) {
+			DHD_ERROR(("%s: writing SoC_RAM dump to the file failed\n", __FUNCTION__));
+		}
+#endif /* BOARD_HIKEY */
+	}
+	dhdp->skip_memdump_map_read = FALSE;
+#elif defined(DHD_DEBUGABILITY_DEBUG_DUMP)
+	dhd_debug_dump_to_ring(dhdp);
+#endif /* DHD_FILE_DUMP_EVENT */
+#endif /* (DHD_FILE_DUMP_EVENT || DHD_DEBUGABILITY_DEBUG_DUMP) */
+
+#ifdef DHD_COREDUMP
+	memset_s(dhdp->memdump_str, DHD_MEMDUMP_LONGSTR_LEN, 0, DHD_MEMDUMP_LONGSTR_LEN);
+
+	if (dhdp->dsack_hc_due_to_isr_delay) {
+		memdump_type = DUMP_TYPE_BY_DSACK_HC_DUE_TO_ISR_DELAY;
+	} else if (dhdp->dsack_hc_due_to_dpc_delay) {
+		memdump_type = DUMP_TYPE_BY_DSACK_HC_DUE_TO_DPC_DELAY;
+	}
+
+#ifdef DHD_LOG_DUMP
+	dhd_convert_memdump_type_to_str(memdump_type, dhdp->memdump_str,
+		DHD_MEMDUMP_LONGSTR_LEN, dhdp->debug_dump_subcmd);
+#endif /* DHD_LOG_DUMP */
+
+	if (memdump_type == DUMP_TYPE_DONGLE_TRAP &&
+		dhdp->dongle_trap_occured == TRUE) {
+		if (!dhdp->dsack_hc_due_to_isr_delay &&
+				!dhdp->dsack_hc_due_to_dpc_delay) {
+			tr = &dhdp->last_trap_info;
+			dhd_lookup_map(dhdp->osh, map_path,
+					ltoh32(tr->epc), pc_fn, ltoh32(tr->r14), lr_fn);
+			sprintf(&dhdp->memdump_str[strlen(dhdp->memdump_str)],
+				"_%.79s_%.79s", pc_fn, lr_fn);
+		}
+	}
+	DHD_ERROR(("%s: dump reason: %s\n", __FUNCTION__, dhdp->memdump_str));
+
+#ifdef DHD_SSSR_COREDUMP
+	ret = dhd_collect_coredump(dhdp, dump);
+	if (ret == BCME_ERROR) {
+		DHD_ERROR(("%s: dhd_collect_coredump() failed.\n", __FUNCTION__));
+		goto exit;
+	} else if (ret == BCME_UNSUPPORTED) {
+		DHD_LOG_MEM(("%s: Unable to collect SSSR dumps. Skip it.\n",
+			__FUNCTION__));
+	}
+#endif /* DHD_SSSR_COREDUMP */
+	if (memdump_type == DUMP_TYPE_BY_SYSDUMP) {
+		DHD_LOG_MEM(("%s: coredump is not supported for BY_SYSDUMP/non trap cases\n",
+			__FUNCTION__));
+	} else {
+		DHD_ERROR(("%s: writing SoC_RAM dump\n", __FUNCTION__));
+		if (wifi_platform_set_coredump(dhd->adapter, dump->buf,
+			dump->bufsize, dhdp->memdump_str)) {
+			DHD_ERROR(("%s: writing SoC_RAM dump failed\n", __FUNCTION__));
+#ifdef DHD_DEBUG_UART
+			dhd->pub.memdump_success = FALSE;
+#endif  /* DHD_DEBUG_UART */
+		}
+	}
+#endif /* DHD_COREDUMP */
+
+	/*
+	 * If kernel does not have file write access enabled
+	 * then skip writing dumps to files.
+	 * The dumps will be pushed to HAL layer which will
+	 * write into files
+	 */
+#ifdef DHD_DUMP_FILE_WRITE_FROM_KERNEL
+
+#ifdef D2H_MINIDUMP
+	/* dump minidump */
+	if (dhd_bus_is_minidump_enabled(dhdp)) {
+		dhd_d2h_minidump(&dhd->pub);
+	} else {
+		DHD_ERROR(("minidump is not enabled\n"));
+	}
+#endif /* D2H_MINIDUMP */
+
+	if (write_dump_to_file(&dhd->pub, dump->buf, dump->bufsize, "mem_dump")) {
+		DHD_ERROR(("%s: writing SoC_RAM dump to the file failed\n", __FUNCTION__));
+#ifdef DHD_DEBUG_UART
+		dhd->pub.memdump_success = FALSE;
+#endif	/* DHD_DEBUG_UART */
+	}
+
+	if (dump->hscb_buf && dump->hscb_bufsize) {
+		if (write_dump_to_file(&dhd->pub, dump->hscb_buf,
+			dump->hscb_bufsize, "mem_dump_hscb")) {
+			DHD_ERROR(("%s: writing HSCB dump to the file failed\n", __FUNCTION__));
+#ifdef DHD_DEBUG_UART
+			dhd->pub.memdump_success = FALSE;
+#endif	/* DHD_DEBUG_UART */
+		}
+	}
+
+#if !defined(DHD_PKT_LOGGING) && defined(DHD_LOG_DUMP)
+	clear_debug_dump_time(dhdp->debug_dump_time_str);
+#endif /* !DHD_PKT_LOGGING && DHD_LOG_DUMP */
+
+	/* directly call dhd_log_dump for debug_dump collection from the mem_dump work queue
+	* context, no need to schedule another work queue for log dump. In case of
+	* user initiated DEBUG_DUMP wpa_cli command (DUMP_TYPE_BY_SYSDUMP),
+	* cfg layer is itself scheduling the log_dump work queue.
+	* that path is not disturbed. If 'dhd_mem_dump' is called directly then we will not
+	* collect debug_dump as it may be called from non-sleepable context.
+	*/
+#ifdef DHD_LOG_DUMP
+	if (dhd->scheduled_memdump &&
+		memdump_type != DUMP_TYPE_BY_SYSDUMP) {
+		log_dump_type_t *flush_type = MALLOCZ(dhdp->osh,
+				sizeof(log_dump_type_t));
+		if (flush_type) {
+			*flush_type = DLD_BUF_TYPE_ALL;
+			DHD_ERROR(("%s: calling log dump.. \n", __FUNCTION__));
+			dhd_log_dump(dhd, flush_type, 0);
+		}
+	}
+#endif /* DHD_LOG_DUMP */
+
+	/* before calling bug on, wait for other logs to be dumped.
+	* we cannot wait in case dhd_mem_dump is called directly
+	* as it may not be from a sleepable context
+	*/
+	if (dhd->scheduled_memdump) {
+		uint bitmask = 0;
+		int timeleft = 0;
+#ifdef DHD_SSSR_DUMP
+		bitmask |= DHD_BUS_BUSY_IN_SSSRDUMP;
+#endif
+		if (bitmask != 0) {
+			DHD_ERROR(("%s: wait to clear dhd_bus_busy_state: 0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+			timeleft = dhd_os_busbusy_wait_bitmask(dhdp,
+					&dhdp->dhd_bus_busy_state, bitmask, 0);
+			if ((timeleft == 0) || (timeleft == 1)) {
+				DHD_ERROR(("%s: Timed out dhd_bus_busy_state=0x%x\n",
+					__FUNCTION__, dhdp->dhd_bus_busy_state));
+			}
+		}
+	}
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+
+	if (dhd->pub.memdump_enabled == DUMP_MEMFILE_BUGON &&
+#ifdef DHD_LOG_DUMP
+		memdump_type != DUMP_TYPE_BY_SYSDUMP &&
+#endif /* DHD_LOG_DUMP */
+		memdump_type != DUMP_TYPE_BY_USER &&
+#ifdef DHD_DEBUG_UART
+		dhd->pub.memdump_success == TRUE &&
+#endif	/* DHD_DEBUG_UART */
+#ifdef DNGL_EVENT_SUPPORT
+		memdump_type != DUMP_TYPE_DONGLE_HOST_EVENT &&
+#endif /* DNGL_EVENT_SUPPORT */
+		memdump_type != DUMP_TYPE_CFG_VENDOR_TRIGGERED) {
+#ifdef SHOW_LOGTRACE
+		/* Wait till logtrace context is flushed */
+		dhd_flush_logtrace_process(dhd);
+#endif /* SHOW_LOGTRACE */
+
+#ifdef EWP_EDL
+		cancel_delayed_work_sync(&dhd->edl_dispatcher_work);
+#endif
+
+		printf("%s\n", info_string);
+		printf("MAC %pM\n", &dhdp->mac);
+		DHD_ERROR(("%s: call BUG_ON \n", __FUNCTION__));
+#ifdef DHD_USE_WARN_RATHER_THAN_BUG_ON
+		WARN_ON(1);
+#else // DHD_USE_WARN_RATHER_THAN_BUG_ON
+//		BUG_ON(1);
+#endif // DHD_USE_WARN_RATHER_THAN_BUG_ON
+	}
+
+exit:
+	if (dump) {
+		MFREE(dhd->pub.osh, dump, sizeof(dhd_dump_t));
+	}
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_MEMDUMP(&dhd->pub);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	dhd->scheduled_memdump = FALSE;
+
+#ifdef OEM_ANDROID
+	if (dhdp->hang_was_pending) {
+		DHD_ERROR(("%s: Send pending HANG event...\n", __FUNCTION__));
+		dhd_os_send_hang_message(dhdp);
+		dhdp->hang_was_pending = 0;
+	}
+#endif /* OEM_ANDROID */
+	DHD_ERROR(("%s: EXIT \n", __FUNCTION__));
+
+	return;
+}
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef D2H_MINIDUMP
+void
+dhd_d2h_minidump(dhd_pub_t *dhdp)
+{
+	char d2h_minidump[128];
+	dhd_dma_buf_t *minidump_buf;
+
+	minidump_buf = dhd_prot_get_minidump_buf(dhdp);
+	if (minidump_buf->va == NULL) {
+		DHD_ERROR(("%s: minidump_buf is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	/* Init file name */
+	memset(d2h_minidump, 0, sizeof(d2h_minidump));
+	snprintf(d2h_minidump, sizeof(d2h_minidump), "%s", "d2h_minidump");
+
+	if (write_dump_to_file(dhdp, (uint8 *)minidump_buf->va, minidump_buf->len, d2h_minidump)) {
+		DHD_ERROR(("%s: failed to dump d2h_minidump to file\n", __FUNCTION__));
+	}
+}
+#endif /* D2H_MINIDUMP */
+
+#ifdef DHD_SSSR_DUMP
+uint
+dhd_sssr_dig_buf_size(dhd_pub_t *dhdp)
+{
+	uint dig_buf_size = 0;
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhdp->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3:
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			if ((dhdp->sssr_reg_info->rev2.length >
+			 OFFSETOF(sssr_reg_info_v2_t, dig_mem_info)) &&
+			 dhdp->sssr_reg_info->rev2.dig_mem_info.dig_sr_size) {
+				dig_buf_size = dhdp->sssr_reg_info->rev2.dig_mem_info.dig_sr_size;
+			}
+			break;
+		case SSSR_REG_INFO_VER_1 :
+			if (dhdp->sssr_reg_info->rev1.vasip_regs.vasip_sr_size) {
+				dig_buf_size = dhdp->sssr_reg_info->rev1.vasip_regs.vasip_sr_size;
+			} else if ((dhdp->sssr_reg_info->rev1.length >
+			 OFFSETOF(sssr_reg_info_v1_t, dig_mem_info)) &&
+			 dhdp->sssr_reg_info->rev1.dig_mem_info.dig_sr_size) {
+				dig_buf_size = dhdp->sssr_reg_info->rev1.dig_mem_info.dig_sr_size;
+			}
+			break;
+		case SSSR_REG_INFO_VER_0 :
+			if (dhdp->sssr_reg_info->rev0.vasip_regs.vasip_sr_size) {
+				dig_buf_size = dhdp->sssr_reg_info->rev0.vasip_regs.vasip_sr_size;
+			}
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	return dig_buf_size;
+}
+
+uint
+dhd_sssr_dig_buf_addr(dhd_pub_t *dhdp)
+{
+	uint dig_buf_addr = 0;
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhdp->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			if ((dhdp->sssr_reg_info->rev2.length >
+			 OFFSETOF(sssr_reg_info_v2_t, dig_mem_info)) &&
+			 dhdp->sssr_reg_info->rev2.dig_mem_info.dig_sr_size) {
+				dig_buf_addr = dhdp->sssr_reg_info->rev2.dig_mem_info.dig_sr_addr;
+			}
+			break;
+		case SSSR_REG_INFO_VER_1 :
+			if (dhdp->sssr_reg_info->rev1.vasip_regs.vasip_sr_size) {
+				dig_buf_addr = dhdp->sssr_reg_info->rev1.vasip_regs.vasip_sr_addr;
+			} else if ((dhdp->sssr_reg_info->rev1.length >
+			 OFFSETOF(sssr_reg_info_v1_t, dig_mem_info)) &&
+			 dhdp->sssr_reg_info->rev1.dig_mem_info.dig_sr_size) {
+				dig_buf_addr = dhdp->sssr_reg_info->rev1.dig_mem_info.dig_sr_addr;
+			}
+			break;
+		case SSSR_REG_INFO_VER_0 :
+			if (dhdp->sssr_reg_info->rev0.vasip_regs.vasip_sr_size) {
+				dig_buf_addr = dhdp->sssr_reg_info->rev0.vasip_regs.vasip_sr_addr;
+			}
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	return dig_buf_addr;
+}
+
+uint
+dhd_sssr_mac_buf_size(dhd_pub_t *dhdp, uint8 core_idx)
+{
+	uint mac_buf_size = 0;
+	uint8 num_d11cores;
+
+	num_d11cores = dhd_d11_slices_num_get(dhdp);
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	if (core_idx < num_d11cores) {
+		switch (dhdp->sssr_reg_info->rev2.version) {
+			case SSSR_REG_INFO_VER_3 :
+				/* intentional fall through */
+			case SSSR_REG_INFO_VER_2 :
+				mac_buf_size = dhdp->sssr_reg_info->rev2.mac_regs[core_idx].sr_size;
+				break;
+			case SSSR_REG_INFO_VER_1 :
+				mac_buf_size = dhdp->sssr_reg_info->rev1.mac_regs[core_idx].sr_size;
+				break;
+			case SSSR_REG_INFO_VER_0 :
+				mac_buf_size = dhdp->sssr_reg_info->rev0.mac_regs[core_idx].sr_size;
+				break;
+			default :
+				DHD_ERROR(("invalid sssr_reg_ver"));
+				return BCME_UNSUPPORTED;
+		}
+	}
+
+	return mac_buf_size;
+}
+
+uint
+dhd_sssr_mac_xmtaddress(dhd_pub_t *dhdp, uint8 core_idx)
+{
+	uint xmtaddress = 0;
+	uint8 num_d11cores;
+
+	num_d11cores = dhd_d11_slices_num_get(dhdp);
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	if (core_idx < num_d11cores) {
+		switch (dhdp->sssr_reg_info->rev2.version) {
+			case SSSR_REG_INFO_VER_3 :
+				/* intentional fall through */
+			case SSSR_REG_INFO_VER_2 :
+				xmtaddress = dhdp->sssr_reg_info->rev2.
+					mac_regs[core_idx].base_regs.xmtaddress;
+				break;
+			case SSSR_REG_INFO_VER_1 :
+				xmtaddress = dhdp->sssr_reg_info->rev1.
+					mac_regs[core_idx].base_regs.xmtaddress;
+				break;
+			case SSSR_REG_INFO_VER_0 :
+				xmtaddress = dhdp->sssr_reg_info->rev0.
+					mac_regs[core_idx].base_regs.xmtaddress;
+				break;
+			default :
+				DHD_ERROR(("invalid sssr_reg_ver"));
+				return BCME_UNSUPPORTED;
+		}
+	}
+
+	return xmtaddress;
+}
+
+uint
+dhd_sssr_mac_xmtdata(dhd_pub_t *dhdp, uint8 core_idx)
+{
+	uint xmtdata = 0;
+	uint8 num_d11cores;
+
+	num_d11cores = dhd_d11_slices_num_get(dhdp);
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	if (core_idx < num_d11cores) {
+		switch (dhdp->sssr_reg_info->rev2.version) {
+			case SSSR_REG_INFO_VER_3 :
+				/* intentional fall through */
+			case SSSR_REG_INFO_VER_2 :
+				xmtdata = dhdp->sssr_reg_info->rev2.
+					mac_regs[core_idx].base_regs.xmtdata;
+				break;
+			case SSSR_REG_INFO_VER_1 :
+				xmtdata = dhdp->sssr_reg_info->rev1.
+					mac_regs[core_idx].base_regs.xmtdata;
+				break;
+			case SSSR_REG_INFO_VER_0 :
+				xmtdata = dhdp->sssr_reg_info->rev0.
+					mac_regs[core_idx].base_regs.xmtdata;
+				break;
+			default :
+				DHD_ERROR(("invalid sssr_reg_ver"));
+				return BCME_UNSUPPORTED;
+		}
+	}
+
+	return xmtdata;
+}
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+int
+dhd_sssr_dump_dig_buf_before(void *dev, const void *user_buf, uint32 len)
+{
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+	int pos = 0, ret = BCME_ERROR;
+	uint dig_buf_size = 0;
+
+	dig_buf_size = dhd_sssr_dig_buf_size(dhdp);
+
+	if (dhdp->sssr_dig_buf_before && (dhdp->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+		ret = dhd_export_debug_data((char *)dhdp->sssr_dig_buf_before,
+			NULL, user_buf, dig_buf_size, &pos);
+	}
+	return ret;
+}
+
+int
+dhd_sssr_dump_d11_buf_before(void *dev, const void *user_buf, uint32 len, int core)
+{
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+	int pos = 0, ret = BCME_ERROR;
+
+	if (dhdp->sssr_d11_before[core] &&
+		dhdp->sssr_d11_outofreset[core] &&
+		(dhdp->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+		ret = dhd_export_debug_data((char *)dhdp->sssr_d11_before[core],
+			NULL, user_buf, len, &pos);
+	}
+	return ret;
+}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+int
+dhd_sssr_dump_dig_buf_after(void *dev, const void *user_buf, uint32 len)
+{
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+	int pos = 0, ret = BCME_ERROR;
+	uint dig_buf_size = 0;
+
+	dig_buf_size = dhd_sssr_dig_buf_size(dhdp);
+
+	if (dhdp->sssr_dig_buf_after) {
+		ret = dhd_export_debug_data((char *)dhdp->sssr_dig_buf_after,
+			NULL, user_buf, dig_buf_size, &pos);
+	}
+	return ret;
+}
+
+int
+dhd_sssr_dump_d11_buf_after(void *dev, const void *user_buf, uint32 len, int core)
+{
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+	int pos = 0, ret = BCME_ERROR;
+
+	if (dhdp->sssr_d11_after[core] &&
+		dhdp->sssr_d11_outofreset[core]) {
+		ret = dhd_export_debug_data((char *)dhdp->sssr_d11_after[core],
+			NULL, user_buf, len, &pos);
+	}
+	return ret;
+}
+
+void
+dhd_sssr_dump_to_file(dhd_info_t* dhdinfo)
+{
+	dhd_info_t *dhd = dhdinfo;
+	dhd_pub_t *dhdp;
+	int i;
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	char before_sr_dump[128];
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	char after_sr_dump[128];
+	unsigned long flags = 0;
+	uint dig_buf_size = 0;
+	uint8 num_d11cores = 0;
+	uint d11_buf_size = 0;
+
+	DHD_ERROR(("%s: ENTER \n", __FUNCTION__));
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	dhdp = &dhd->pub;
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_SET_IN_SSSRDUMP(dhdp);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s: bus is down! can't collect sssr dump. \n", __FUNCTION__));
+		goto exit;
+	}
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	num_d11cores = dhd_d11_slices_num_get(dhdp);
+
+	for (i = 0; i < num_d11cores; i++) {
+		/* Init file name */
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		memset(before_sr_dump, 0, sizeof(before_sr_dump));
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+		memset(after_sr_dump, 0, sizeof(after_sr_dump));
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		snprintf(before_sr_dump, sizeof(before_sr_dump), "%s_%d_%s",
+			"sssr_dump_core", i, "before_SR");
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+		snprintf(after_sr_dump, sizeof(after_sr_dump), "%s_%d_%s",
+			"sssr_dump_core", i, "after_SR");
+
+		d11_buf_size = dhd_sssr_mac_buf_size(dhdp, i);
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		if (dhdp->sssr_d11_before[i] && dhdp->sssr_d11_outofreset[i] &&
+			(dhdp->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+			if (write_dump_to_file(dhdp, (uint8 *)dhdp->sssr_d11_before[i],
+				d11_buf_size, before_sr_dump)) {
+				DHD_ERROR(("%s: writing SSSR MAIN dump before to the file failed\n",
+					__FUNCTION__));
+			}
+		}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+		if (dhdp->sssr_d11_after[i] && dhdp->sssr_d11_outofreset[i]) {
+			if (write_dump_to_file(dhdp, (uint8 *)dhdp->sssr_d11_after[i],
+				d11_buf_size, after_sr_dump)) {
+				DHD_ERROR(("%s: writing SSSR AUX dump after to the file failed\n",
+					__FUNCTION__));
+			}
+		}
+	}
+
+	dig_buf_size = dhd_sssr_dig_buf_size(dhdp);
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	if (dhdp->sssr_dig_buf_before && (dhdp->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+		if (write_dump_to_file(dhdp, (uint8 *)dhdp->sssr_dig_buf_before,
+			dig_buf_size, "sssr_dump_dig_before_SR")) {
+			DHD_ERROR(("%s: writing SSSR Dig dump before to the file failed\n",
+				__FUNCTION__));
+		}
+	}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+	if (dhdp->sssr_dig_buf_after) {
+		if (write_dump_to_file(dhdp, (uint8 *)dhdp->sssr_dig_buf_after,
+			dig_buf_size, "sssr_dump_dig_after_SR")) {
+			DHD_ERROR(("%s: writing SSSR Dig VASIP dump after to the file failed\n",
+			 __FUNCTION__));
+		}
+	}
+
+exit:
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_SSSRDUMP(dhdp);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+}
+
+void
+dhd_write_sssr_dump(dhd_pub_t *dhdp, uint32 dump_mode)
+{
+#if defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	dhdp->sssr_dump_mode = dump_mode;
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+
+	/*
+	 * If kernel does not have file write access enabled
+	 * then skip writing dumps to files.
+	 * The dumps will be pushed to HAL layer which will
+	 * write into files
+	 */
+#if !defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	return;
+#else
+	/*
+	 * dhd_mem_dump -> dhd_sssr_dump -> dhd_write_sssr_dump
+	 * Without workqueue -
+	 * DUMP_TYPE_DONGLE_INIT_FAILURE/DUMP_TYPE_DUE_TO_BT/DUMP_TYPE_SMMU_FAULT
+	 * : These are called in own handler, not in the interrupt context
+	 * With workqueue - all other DUMP_TYPEs : dhd_mem_dump is called in workqueue
+	 * Thus, it doesn't neeed to dump SSSR in workqueue
+	 */
+	DHD_ERROR(("%s: writing sssr dump to file... \n", __FUNCTION__));
+	dhd_sssr_dump_to_file(dhdp->info);
+#endif /* !DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+}
+#endif /* DHD_SSSR_DUMP */
+
+#ifdef DHD_SDTC_ETB_DUMP
+void
+dhd_sdtc_etb_dump(dhd_pub_t *dhd)
+{
+	etb_info_t etb_info;
+	uint8 *sdtc_etb_dump;
+	uint8 *sdtc_etb_mempool;
+	int ret = 0;
+
+	if (!dhd->sdtc_etb_inited) {
+		DHD_ERROR(("%s, SDTC ETB dump not supported\n", __FUNCTION__));
+		return;
+	}
+
+	bzero(&etb_info, sizeof(etb_info));
+
+	if ((ret = dhd_bus_get_etb_info(dhd, dhd->etb_addr_info.etbinfo_addr, &etb_info))) {
+		DHD_ERROR(("%s: failed to get etb info %d\n", __FUNCTION__, ret));
+		return;
+	}
+
+	if (etb_info.addr == (uint32)-1) {
+		DHD_ERROR(("%s: invalid etbinfo.addr 0x%x Hence donot collect SDTC ETB\n",
+			__FUNCTION__, etb_info.addr));
+		return;
+	}
+
+	if (etb_info.read_bytes == 0) {
+		DHD_ERROR(("%s ETB is of zero size. Hence donot collect SDTC ETB\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("%s etb_info ver:%d len:%d rwp:%d etb_full:%d etb:addr:0x%x, len:%d\n",
+		__FUNCTION__, etb_info.version, etb_info.len,
+		etb_info.read_write_p, etb_info.etb_full,
+		etb_info.addr, etb_info.read_bytes));
+
+	/*
+	 * etb mempool format = etb_info + etb
+	 */
+	dhd->sdtc_etb_dump_len = etb_info.read_bytes + sizeof(etb_info);
+	if (dhd->sdtc_etb_dump_len > DHD_SDTC_ETB_MEMPOOL_SIZE) {
+		DHD_ERROR(("%s etb_dump_len: %d is more than the alloced %d.Hence cannot collect\n",
+			__FUNCTION__, dhd->sdtc_etb_dump_len, DHD_SDTC_ETB_MEMPOOL_SIZE));
+		return;
+	}
+	sdtc_etb_mempool = dhd->sdtc_etb_mempool;
+	memcpy(sdtc_etb_mempool, &etb_info, sizeof(etb_info));
+	sdtc_etb_dump = sdtc_etb_mempool + sizeof(etb_info);
+	if ((ret = dhd_bus_get_sdtc_etb(dhd, sdtc_etb_dump, etb_info.addr, etb_info.read_bytes))) {
+		DHD_ERROR(("%s: error to get SDTC ETB ret: %d\n", __FUNCTION__, ret));
+		return;
+	}
+
+	dhd_print_buf_addr(dhd, "sdtc_etb_dump", (uint8 *)sdtc_etb_mempool, dhd->sdtc_etb_dump_len);
+	/*
+	 * If kernel does not have file write access enabled
+	 * then skip writing dumps to files.
+	 * The dumps will be pushed to HAL layer which will
+	 * write into files
+	 */
+#ifdef DHD_DUMP_FILE_WRITE_FROM_KERNEL
+	if (write_dump_to_file(dhd, (uint8 *)sdtc_etb_mempool,
+		dhd->sdtc_etb_dump_len, "sdtc_etb_dump")) {
+		DHD_ERROR(("%s: failed to dump sdtc_etb to file\n",
+			__FUNCTION__));
+	}
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+}
+
+int
+dhd_sdtc_etb_hal_file_dump(void *dev, const void *user_buf, uint32 len)
+{
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+	int pos = 0, ret = BCME_ERROR;
+
+	if (dhdp->sdtc_etb_dump_len) {
+		ret = dhd_export_debug_data((char *)dhdp->sdtc_etb_mempool,
+			NULL, user_buf, dhdp->sdtc_etb_dump_len, &pos);
+	} else {
+		DHD_ERROR(("%s ETB is of zero size. Hence donot collect SDTC ETB\n", __FUNCTION__));
+	}
+	DHD_ERROR(("%s, done ret: %d\n", __FUNCTION__, ret));
+	return ret;
+}
+#endif /* DHD_SDTC_ETB_DUMP */
+
+/* This function writes data to the file pointed by fp, OR
+ * copies data to the user buffer sent by upper layer(HAL).
+ */
+int
+dhd_export_debug_data(void *mem_buf, void *fp, const void *user_buf, uint32 buf_len, void *pos)
+{
+	int ret = BCME_OK;
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+	struct dhd_dbg_ring_buf *ring_buf;
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+
+	if (fp) {
+		ret = dhd_vfs_write(fp, mem_buf, buf_len, (loff_t *)pos);
+		if (ret < 0) {
+			DHD_ERROR(("write file error, err = %d\n", ret));
+			goto exit;
+		}
+	} else if (user_buf) {
+#ifdef CONFIG_COMPAT
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0))
+		if (in_compat_syscall())
+#else
+		if (is_compat_task())
+#endif /* LINUX_VER >= 4.6 */
+		{
+			void * usr_ptr =  compat_ptr((uintptr_t) user_buf);
+			ret = copy_to_user((void *)((uintptr_t)usr_ptr + (*(int *)pos)),
+				mem_buf, buf_len);
+			if (ret) {
+				DHD_ERROR(("failed to copy into user buffer : %d\n", ret));
+				goto exit;
+			}
+		}
+		else
+#endif /* CONFIG_COMPAT */
+		{
+			ret = copy_to_user((void *)((uintptr_t)user_buf + (*(int *)pos)),
+				mem_buf, buf_len);
+			if (ret) {
+				DHD_ERROR(("failed to copy into user buffer : %d\n", ret));
+				goto exit;
+			}
+		}
+		(*(int *)pos) += buf_len;
+	}
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+	else {
+		ring_buf = &g_ring_buf;
+		if (ring_buf->dhd_pub) {
+			ret = dhd_debug_dump_ring_push(ring_buf->dhd_pub, (*(int *)pos),
+				buf_len, mem_buf);
+			if (ret != BCME_OK) {
+				DHD_ERROR(("%s: ring push failed ret:%d\n", __func__, ret));
+			}
+		}
+	}
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+exit:
+	return ret;
+}
+
+#ifdef BCM_ROUTER_DHD
+void dhd_schedule_trap_log_dump(dhd_pub_t *dhdp,
+	uint8 *buf, uint32 size)
+{
+	dhd_write_file_t *wf = NULL;
+	wf = (dhd_write_file_t *)MALLOC(dhdp->osh, sizeof(dhd_write_file_t));
+	if (wf == NULL) {
+		DHD_ERROR(("%s: dhd write file memory allocation failed\n", __FUNCTION__));
+		return;
+	}
+	snprintf(wf->file_path, sizeof(wf->file_path), "%s", "/tmp/failed_if.txt");
+	wf->file_flags = O_CREAT | O_WRONLY | O_SYNC;
+	wf->buf = buf;
+	wf->bufsize = size;
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq, (void *)wf,
+		DHD_WQ_WORK_INFORM_DHD_MON, dhd_inform_dhd_monitor_handler,
+		DHD_WQ_WORK_PRIORITY_HIGH);
+}
+
+/* Returns the pid of a the userspace process running with the given name */
+static struct task_struct *
+_get_task_info(const char *pname)
+{
+	struct task_struct *task;
+	if (!pname)
+		return NULL;
+
+	for_each_process(task) {
+		if (strcmp(pname, task->comm) == 0)
+			return task;
+	}
+
+	return NULL;
+}
+
+#define DHD_MONITOR_NS	"dhd_monitor"
+extern void emergency_restart(void);
+
+static void
+dhd_inform_dhd_monitor_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	dhd_write_file_t *wf = event_info;
+	struct task_struct *monitor_task;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+	if (!event_info) {
+		DHD_ERROR(("%s: File info is NULL\n", __FUNCTION__));
+		return;
+	}
+	if (!wf->buf) {
+		DHD_ERROR(("%s: Unable to get failed interface name\n", __FUNCTION__));
+		goto exit;
+	}
+	if (write_file(wf->file_path, wf->file_flags, wf->buf, wf->bufsize)) {
+		DHD_ERROR(("%s: writing to the file failed\n", __FUNCTION__));
+	}
+exit:
+	MFREE(dhd->pub.osh, wf, sizeof(dhd_write_file_t));
+
+	/* check if dhd_monitor is running */
+	monitor_task = _get_task_info(DHD_MONITOR_NS);
+	if (monitor_task == NULL) {
+		/* If dhd_monitor is not running, handle recovery from here */
+
+		char *val = nvram_get("watchdog");
+		if (val && bcm_atoi(val)) {
+			/* watchdog enabled, so reboot */
+			DHD_ERROR(("%s: Dongle(wl%d) trap detected. Restarting the system\n",
+				__FUNCTION__, dhd->unit));
+
+			mdelay(1000);
+			emergency_restart();
+			while (1)
+				cpu_relax();
+		} else {
+			DHD_ERROR(("%s: Dongle(wl%d) trap detected. No watchdog.\n",
+			   __FUNCTION__, dhd->unit));
+		}
+
+		return;
+	}
+
+	/* If monitor daemon is running, let's signal the monitor for recovery */
+	DHD_ERROR(("%s: Dongle(wl%d) trap detected. Send signal to dhd_monitor.\n",
+		__FUNCTION__, dhd->unit));
+
+	send_sig_info(SIGUSR1, (void *)1L, monitor_task);
+}
+#endif /* BCM_ROUTER_DHD */
+
+#if defined(BCMDBG) && defined(DHD_MACDBG)
+#define DUMPMAC_BUF_SZ (128 * 1024)
+#define DUMPMAC_FILENAME_SZ 32
+
+static void
+_dhd_schedule_macdbg_dump(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	dhd_pub_t *dhdp = &dhd->pub;
+#ifndef BCM_ROUTER_DHD
+	char *dumpbuf = NULL;
+	int dumpbuf_len = 0;
+	uint16 dump_signature;
+	char dumpfilename[DUMPMAC_FILENAME_SZ] = {0, };
+#endif /* BCM_ROUTER_DHD */
+
+	ASSERT(event == DHD_WQ_WORK_MACDBG);
+	BCM_REFERENCE(event_info);
+
+	DHD_ERROR(("%s: Dongle(wl%d) macreg dump scheduled\n",
+		__FUNCTION__, dhd->unit));
+
+	DHD_OS_WAKE_LOCK(dhdp);
+
+	/* Make sure dongle stops running to avoid race condition in reading mac registers */
+	(void) dhd_wl_ioctl_set_intiovar(dhdp, "bus:disconnect", 99, WLC_SET_VAR, TRUE, 0);
+
+	/* In router, skip macregs dump as dhd_monitor will dump them */
+#ifndef BCM_ROUTER_DHD
+	dumpbuf = (char *)MALLOCZ(dhdp->osh, DUMPMAC_BUF_SZ);
+	if (dumpbuf) {
+		/* Write macdump to a file */
+
+		/* Get dump file signature */
+		dump_signature = (uint16)OSL_RAND();
+
+		/* PSMr */
+		if (dhd_macdbg_dumpmac(dhdp, dumpbuf, DUMPMAC_BUF_SZ,
+			&dumpbuf_len, FALSE) == BCME_OK) {
+			snprintf(dumpfilename, DUMPMAC_FILENAME_SZ,
+				"/tmp/d11reg_dump_%04X.txt", dump_signature);
+			DHD_ERROR(("%s: PSMr macreg dump to %s\n", __FUNCTION__, dumpfilename));
+			/* Write to a file */
+			if (write_file(dumpfilename, (O_CREAT | O_WRONLY | O_SYNC),
+				dumpbuf, dumpbuf_len)) {
+				DHD_ERROR(("%s: writing mac dump to the file failed\n",
+					__FUNCTION__));
+			}
+			memset(dumpbuf, 0, DUMPMAC_BUF_SZ);
+			memset(dumpfilename, 0, DUMPMAC_FILENAME_SZ);
+			dumpbuf_len = 0;
+		}
+
+		/* PSMx */
+		if (dhd_macdbg_dumpmac(dhdp, dumpbuf, DUMPMAC_BUF_SZ,
+			&dumpbuf_len, TRUE) == BCME_OK) {
+			snprintf(dumpfilename, DUMPMAC_FILENAME_SZ,
+				"/tmp/d11regx_dump_%04X.txt", dump_signature);
+			DHD_ERROR(("%s: PSMx macreg dump to %s\n", __FUNCTION__, dumpfilename));
+			/* Write to a file */
+			if (write_file(dumpfilename, (O_CREAT | O_WRONLY | O_SYNC),
+				dumpbuf, dumpbuf_len)) {
+				DHD_ERROR(("%s: writing mac dump to the file failed\n",
+					__FUNCTION__));
+			}
+			memset(dumpbuf, 0, DUMPMAC_BUF_SZ);
+			memset(dumpfilename, 0, DUMPMAC_FILENAME_SZ);
+			dumpbuf_len = 0;
+		}
+
+		/* SVMP */
+		if (dhd_macdbg_dumpsvmp(dhdp, dumpbuf, DUMPMAC_BUF_SZ,
+			&dumpbuf_len) == BCME_OK) {
+			snprintf(dumpfilename, DUMPMAC_FILENAME_SZ,
+				"/tmp/svmp_dump_%04X.txt", dump_signature);
+			DHD_ERROR(("%s: SVMP mems dump to %s\n", __FUNCTION__, dumpfilename));
+			/* Write to a file */
+			if (write_file(dumpfilename, (O_CREAT | O_WRONLY | O_SYNC),
+				dumpbuf, dumpbuf_len)) {
+				DHD_ERROR(("%s: writing svmp dump to the file failed\n",
+					__FUNCTION__));
+			}
+			memset(dumpbuf, 0, DUMPMAC_BUF_SZ);
+			memset(dumpfilename, 0, DUMPMAC_FILENAME_SZ);
+			dumpbuf_len = 0;
+		}
+
+		MFREE(dhdp->osh, dumpbuf, DUMPMAC_BUF_SZ);
+	} else {
+		DHD_ERROR(("%s: print macdump\n", __FUNCTION__));
+		/* Just printf the dumps */
+		(void) dhd_macdbg_dumpmac(dhdp, NULL, 0, NULL, FALSE); /* PSMr */
+		(void) dhd_macdbg_dumpmac(dhdp, NULL, 0, NULL, TRUE); /* PSMx */
+		(void) dhd_macdbg_dumpsvmp(dhdp, NULL, 0, NULL);
+	}
+#endif /* BCM_ROUTER_DHD */
+
+	DHD_OS_WAKE_UNLOCK(dhdp);
+	dhd_deferred_work_set_skip(dhd->dhd_deferred_wq,
+		DHD_WQ_WORK_MACDBG, FALSE);
+}
+
+void
+dhd_schedule_macdbg_dump(dhd_pub_t *dhdp)
+{
+	DHD_ERROR(("%s: Dongle(wl%d) schedule macreg dump\n",
+		__FUNCTION__, dhdp->info->unit));
+
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq, NULL,
+		DHD_WQ_WORK_MACDBG, _dhd_schedule_macdbg_dump, DHD_WQ_WORK_PRIORITY_LOW);
+	dhd_deferred_work_set_skip(dhdp->info->dhd_deferred_wq,
+		DHD_WQ_WORK_MACDBG, TRUE);
+}
+#endif /* BCMDBG && DHD_MACDBG */
+
+/*
+ * This call is to get the memdump size so that,
+ * halutil can alloc that much buffer in user space.
+ */
+int
+dhd_os_socram_dump(struct net_device *dev, uint32 *dump_size)
+{
+	int ret = BCME_OK;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	if (dhdp->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s: bus is down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(dhdp)) {
+		DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state, so skip\n",
+			__FUNCTION__, dhdp->busstate, dhdp->dhd_bus_busy_state));
+		return BCME_ERROR;
+	}
+	ret = dhd_common_socram_dump(dhdp);
+	if (ret == BCME_OK) {
+		*dump_size = dhdp->soc_ram_length;
+	}
+	return ret;
+}
+
+/*
+ * This is to get the actual memdup after getting the memdump size
+ */
+int
+dhd_os_get_socram_dump(struct net_device *dev, char **buf, uint32 *size)
+{
+	int ret = BCME_OK;
+	int orig_len = 0;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	if (buf == NULL)
+		return BCME_ERROR;
+	orig_len = *size;
+	if (dhdp->soc_ram) {
+		if (orig_len >= dhdp->soc_ram_length) {
+			*buf = dhdp->soc_ram;
+			*size = dhdp->soc_ram_length;
+		} else {
+			ret = BCME_BUFTOOSHORT;
+			DHD_ERROR(("The length of the buffer is too short"
+				" to save the memory dump with %d\n", dhdp->soc_ram_length));
+		}
+	} else {
+		DHD_ERROR(("socram_dump is not ready to get\n"));
+		ret = BCME_NOTREADY;
+	}
+	return ret;
+}
+
+#ifdef DHD_MAP_PKTID_LOGGING
+uint32
+dhd_get_pktid_map_logging_len(void *ndev, dhd_pub_t *dhdp, bool is_map)
+{
+	dhd_info_t *dhd_info;
+	log_dump_section_hdr_t sec_hdr;
+	uint32 length = 0;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp || !dhdp->enable_pktid_log_dump)
+		return length;
+
+	length = dhd_pktid_buf_len(dhdp, is_map) + sizeof(sec_hdr);
+	return length;
+}
+
+int
+dhd_print_pktid_map_log_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos, bool is_map)
+{
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp) {
+		return BCME_ERROR;
+	}
+
+	return dhd_write_pktid_log_dump(dhdp, user_buf, fp, len, pos, is_map);
+}
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+int
+dhd_os_get_version(struct net_device *dev, bool dhd_ver, char **buf, uint32 size)
+{
+	char *fw_str;
+
+	if (size == 0)
+		return BCME_BADARG;
+
+	fw_str = strstr(info_string, "Firmware: ");
+	if (fw_str == NULL) {
+		return BCME_ERROR;
+	}
+
+	bzero(*buf, size);
+	if (dhd_ver) {
+		strlcpy(*buf, dhd_version, size);
+	} else {
+		strlcpy(*buf, fw_str, size);
+	}
+	return BCME_OK;
+}
+
+#ifdef DHD_PKT_LOGGING
+int
+dhd_os_get_pktlog_dump(void *dev, const void *user_buf, uint32 len)
+{
+#ifdef DHD_PKT_LOGGING_DBGRING
+	/* With DHD_PKT_LOGGING_DBGRING, this path is not allowed. */
+	return BCME_UNSUPPORTED;
+#else
+	int ret = BCME_OK;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	if (user_buf == NULL) {
+		DHD_ERROR(("%s(): user buffer is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	ret = dhd_pktlog_dump_write_memory(dhdp, user_buf, len);
+	if (ret < 0) {
+		DHD_ERROR(("%s(): fail to dump pktlog, err = %d\n", __FUNCTION__, ret));
+		return ret;
+	}
+	return ret;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+}
+
+spinlock_t *
+dhd_os_get_pktlog_lock(dhd_pub_t *dhdp)
+{
+	return dhdp->pktlog->pktlog_ring->pktlog_ring_lock;
+}
+
+uint32
+dhd_os_get_pktlog_dump_size(struct net_device *dev)
+{
+	int ret = 0;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	ret = dhd_pktlog_get_dump_length(dhdp);
+	if (ret < 0) {
+		DHD_ERROR(("%s(): fail to get pktlog size, err = %d\n",
+			__FUNCTION__, ret));
+		return 0;
+	}
+	return ret;
+}
+
+void
+dhd_os_get_pktlogdump_filename(struct net_device *dev, char *dump_path, int len)
+{
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	dhd_pktlog_get_filename(dhdp, dump_path, len);
+}
+#endif /* DHD_PKT_LOGGING */
+#ifdef DNGL_AXI_ERROR_LOGGING
+int
+dhd_os_get_axi_error_dump(void *dev, const void *user_buf, uint32 len)
+{
+	int ret = BCME_OK;
+	dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev);
+	dhd_pub_t *dhdp = &dhd->pub;
+	loff_t pos = 0;
+	if (user_buf == NULL) {
+		DHD_ERROR(("%s(): user buffer is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	ret = dhd_export_debug_data((char *)dhdp->axi_err_dump,
+			NULL, user_buf, sizeof(dhd_axi_error_dump_t), &pos);
+
+	if (ret < 0) {
+		DHD_ERROR(("%s(): fail to dump pktlog, err = %d\n", __FUNCTION__, ret));
+		return ret;
+	}
+	return ret;
+}
+
+int
+dhd_os_get_axi_error_dump_size(struct net_device *dev)
+{
+	int size = -1;
+
+	size = sizeof(dhd_axi_error_dump_t);
+	if (size < 0) {
+		DHD_ERROR(("%s(): fail to get axi error size, err = %d\n", __FUNCTION__, size));
+	}
+	return size;
+}
+
+void
+dhd_os_get_axi_error_filename(struct net_device *dev, char *dump_path, int len)
+{
+	snprintf(dump_path, len, "%s",
+		DHD_COMMON_DUMP_PATH DHD_DUMP_AXI_ERROR_FILENAME);
+}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+#ifdef DHD_WMF
+/* Returns interface specific WMF configuration */
+dhd_wmf_t* dhd_wmf_conf(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+	return &ifp->wmf;
+}
+#endif /* DHD_WMF */
+
+#if defined(BCM_ROUTER_DHD)
+void traffic_mgmt_pkt_set_prio(dhd_pub_t *dhdp, void * pktbuf)
+{
+	struct ether_header *eh;
+	struct ethervlan_header *evh;
+	uint8 *pktdata, *ip_body;
+	uint8  dwm_filter;
+	uint8 tos_tc = 0;
+	uint8 dscp   = 0;
+	pktdata = (uint8 *)PKTDATA(dhdp->osh, pktbuf);
+	eh = (struct ether_header *) pktdata;
+	ip_body = NULL;
+
+	if (dhdp->dhd_tm_dwm_tbl.dhd_dwm_enabled) {
+		if (eh->ether_type == hton16(ETHER_TYPE_8021Q)) {
+			evh = (struct ethervlan_header *)eh;
+			if ((evh->ether_type == hton16(ETHER_TYPE_IP)) ||
+				(evh->ether_type == hton16(ETHER_TYPE_IPV6))) {
+				ip_body = pktdata + sizeof(struct ethervlan_header);
+			}
+		} else if ((eh->ether_type == hton16(ETHER_TYPE_IP)) ||
+			(eh->ether_type == hton16(ETHER_TYPE_IPV6))) {
+			ip_body = pktdata + sizeof(struct ether_header);
+		}
+		if (ip_body) {
+			tos_tc = IP_TOS46(ip_body);
+			dscp = tos_tc >> IPV4_TOS_DSCP_SHIFT;
+		}
+
+		if (dscp < DHD_DWM_TBL_SIZE) {
+			dwm_filter = dhdp->dhd_tm_dwm_tbl.dhd_dwm_tbl[dscp];
+			if (DHD_TRF_MGMT_DWM_IS_FILTER_SET(dwm_filter)) {
+				PKTSETPRIO(pktbuf, DHD_TRF_MGMT_DWM_PRIO(dwm_filter));
+			}
+		}
+	}
+}
+#endif /* BCM_ROUTER_DHD */
+
+bool dhd_sta_associated(dhd_pub_t *dhdp, uint32 bssidx, uint8 *mac)
+{
+	return dhd_find_sta(dhdp, bssidx, mac) ? TRUE : FALSE;
+}
+
+#ifdef DHD_L2_FILTER
+arp_table_t*
+dhd_get_ifp_arp_table_handle(dhd_pub_t *dhdp, uint32 bssidx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(bssidx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[bssidx];
+	return ifp->phnd_arp_table;
+}
+
+int dhd_get_parp_status(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	if (ifp)
+		return ifp->parp_enable;
+	else
+		return FALSE;
+}
+
+/* Set interface specific proxy arp configuration */
+int dhd_set_parp_status(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+
+	if (!ifp)
+	    return BCME_ERROR;
+
+	/* At present all 3 variables are being
+	 * handled at once
+	 */
+	ifp->parp_enable = val;
+	ifp->parp_discard = val;
+	ifp->parp_allnode = val;
+
+	/* Flush ARP entries when disabled */
+	if (val == FALSE) {
+		bcm_l2_filter_arp_table_update(dhdp->osh, ifp->phnd_arp_table, TRUE, NULL,
+			FALSE, dhdp->tickcnt);
+	}
+	return BCME_OK;
+}
+
+bool dhd_parp_discard_is_enabled(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+	return ifp->parp_discard;
+}
+
+bool
+dhd_parp_allnode_is_enabled(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	return ifp->parp_allnode;
+}
+
+int dhd_get_dhcp_unicast_status(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	return ifp->dhcp_unicast;
+}
+
+int dhd_set_dhcp_unicast_status(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	ifp->dhcp_unicast = val;
+	return BCME_OK;
+}
+
+int dhd_get_block_ping_status(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	return ifp->block_ping;
+}
+
+int dhd_set_block_ping_status(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	ifp->block_ping = val;
+	/* Disable rx_pkt_chain feature for interface if block_ping option is
+	 * enabled
+	 */
+	dhd_update_rx_pkt_chainable_state(dhdp, idx);
+	return BCME_OK;
+}
+
+int dhd_get_grat_arp_status(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	return ifp->grat_arp;
+}
+
+int dhd_set_grat_arp_status(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	ifp->grat_arp = val;
+
+	return BCME_OK;
+}
+
+int dhd_get_block_tdls_status(dhd_pub_t *dhdp, uint32 idx)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+
+	ASSERT(idx < DHD_MAX_IFS);
+
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	return ifp->block_tdls;
+}
+
+int dhd_set_block_tdls_status(dhd_pub_t *dhdp, uint32 idx, int val)
+{
+	dhd_info_t *dhd = dhdp->info;
+	dhd_if_t *ifp;
+	ASSERT(idx < DHD_MAX_IFS);
+	ifp = dhd->iflist[idx];
+
+	ASSERT(ifp);
+
+	ifp->block_tdls = val;
+
+	return BCME_OK;
+}
+#endif /* DHD_L2_FILTER */
+
+#if defined(SET_XPS_CPUS)
+int dhd_xps_cpus_enable(struct net_device *net, int enable)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+	dhd_if_t *ifp;
+	int ifidx;
+	char * XPS_CPU_SETBUF;
+
+	ifidx = dhd_net2idx(dhd, net);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	if (!dhd->pub.conf->xps_cpus)
+		return -ENODEV;
+
+	if (ifidx == PRIMARY_INF) {
+		if (dhd->pub.op_mode == DHD_FLAG_IBSS_MODE) {
+			DHD_INFO(("%s : set for IBSS.\n", __FUNCTION__));
+			XPS_CPU_SETBUF = RPS_CPUS_MASK_IBSS;
+		} else {
+			DHD_INFO(("%s : set for BSS.\n", __FUNCTION__));
+			XPS_CPU_SETBUF = RPS_CPUS_MASK;
+		}
+	} else if (ifidx == VIRTUAL_INF) {
+		DHD_INFO(("%s : set for P2P.\n", __FUNCTION__));
+		XPS_CPU_SETBUF = RPS_CPUS_MASK_P2P;
+	} else {
+		DHD_ERROR(("%s : Invalid index : %d.\n", __FUNCTION__, ifidx));
+		return -EINVAL;
+	}
+
+	ifp = dhd->iflist[ifidx];
+	if (ifp) {
+		if (enable) {
+			DHD_INFO(("%s : set xps_cpus as [%s]\n", __FUNCTION__, XPS_CPU_SETBUF));
+			custom_xps_map_set(ifp->net, XPS_CPU_SETBUF, strlen(XPS_CPU_SETBUF));
+		} else {
+			custom_xps_map_clear(ifp->net);
+		}
+	} else {
+		DHD_ERROR(("%s : ifp is NULL!!\n", __FUNCTION__));
+		return -ENODEV;
+	}
+	return BCME_OK;
+}
+
+int custom_xps_map_set(struct net_device *net, char *buf, size_t len)
+{
+	cpumask_var_t mask;
+	int err;
+
+	DHD_INFO(("%s : Entered.\n", __FUNCTION__));
+
+	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
+		DHD_ERROR(("%s : alloc_cpumask_var fail.\n", __FUNCTION__));
+		return -ENOMEM;
+	}
+
+	err = bitmap_parse(buf, len, cpumask_bits(mask), nr_cpumask_bits);
+	if (err) {
+		free_cpumask_var(mask);
+		DHD_ERROR(("%s : bitmap_parse fail.\n", __FUNCTION__));
+		return err;
+	}
+
+	err = netif_set_xps_queue(net, mask, 0);
+
+	free_cpumask_var(mask);
+
+	if (0 == err)
+		WL_MSG(net->name, "Done. mapping cpu\n");
+
+	return err;
+}
+
+void custom_xps_map_clear(struct net_device *net)
+{
+    struct xps_dev_maps *dev_maps;
+
+	DHD_INFO(("%s : Entered.\n", __FUNCTION__));
+
+    rcu_read_lock();
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+	dev_maps = rcu_dereference(net->xps_maps[XPS_CPUS]);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0))
+	dev_maps = rcu_dereference(net->xps_cpus_map);
+#else
+    dev_maps = rcu_dereference(net->xps_maps);
+#endif
+    rcu_read_unlock();
+
+	if (dev_maps) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+		RCU_INIT_POINTER(net->xps_maps[XPS_CPUS], NULL);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0))
+		RCU_INIT_POINTER(net->xps_cpus_map, NULL);
+#else
+		RCU_INIT_POINTER(net->xps_maps, NULL);
+#endif
+		kfree_rcu(dev_maps, rcu);
+		DHD_INFO(("%s : xps_cpus map clear.\n", __FUNCTION__));
+	}
+}
+#endif // endif
+
+#if defined(SET_RPS_CPUS)
+int dhd_rps_cpus_enable(struct net_device *net, int enable)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+	dhd_if_t *ifp;
+	int ifidx;
+	char * RPS_CPU_SETBUF;
+
+	ifidx = dhd_net2idx(dhd, net);
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s bad ifidx\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	if (!dhd->pub.conf->rps_cpus)
+		return -ENODEV;
+
+	if (ifidx == PRIMARY_INF) {
+		if (dhd->pub.op_mode == DHD_FLAG_IBSS_MODE) {
+			DHD_INFO(("%s : set for IBSS.\n", __FUNCTION__));
+			RPS_CPU_SETBUF = RPS_CPUS_MASK_IBSS;
+		} else {
+			DHD_INFO(("%s : set for BSS.\n", __FUNCTION__));
+			RPS_CPU_SETBUF = RPS_CPUS_MASK;
+		}
+	} else if (ifidx == VIRTUAL_INF) {
+		DHD_INFO(("%s : set for P2P.\n", __FUNCTION__));
+		RPS_CPU_SETBUF = RPS_CPUS_MASK_P2P;
+	} else {
+		DHD_ERROR(("%s : Invalid index : %d.\n", __FUNCTION__, ifidx));
+		return -EINVAL;
+	}
+
+	ifp = dhd->iflist[ifidx];
+	if (ifp) {
+		if (enable) {
+			DHD_INFO(("%s : set rps_cpus as [%s]\n", __FUNCTION__, RPS_CPU_SETBUF));
+			custom_rps_map_set(ifp->net->_rx, RPS_CPU_SETBUF, strlen(RPS_CPU_SETBUF));
+		} else {
+			custom_rps_map_clear(ifp->net->_rx);
+		}
+	} else {
+		DHD_ERROR(("%s : ifp is NULL!!\n", __FUNCTION__));
+		return -ENODEV;
+	}
+	return BCME_OK;
+}
+
+int custom_rps_map_set(struct netdev_rx_queue *queue, char *buf, size_t len)
+{
+	struct rps_map *old_map, *map;
+	cpumask_var_t mask;
+	int err, cpu, i;
+	static DEFINE_SPINLOCK(rps_map_lock);
+
+	DHD_INFO(("%s : Entered.\n", __FUNCTION__));
+
+	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
+		DHD_ERROR(("%s : alloc_cpumask_var fail.\n", __FUNCTION__));
+		return -ENOMEM;
+	}
+
+	err = bitmap_parse(buf, len, cpumask_bits(mask), nr_cpumask_bits);
+	if (err) {
+		free_cpumask_var(mask);
+		DHD_ERROR(("%s : bitmap_parse fail.\n", __FUNCTION__));
+		return err;
+	}
+
+	map = kzalloc(max_t(unsigned int,
+		RPS_MAP_SIZE(cpumask_weight(mask)), L1_CACHE_BYTES),
+		GFP_KERNEL);
+	if (!map) {
+		free_cpumask_var(mask);
+		DHD_ERROR(("%s : map malloc fail.\n", __FUNCTION__));
+		return -ENOMEM;
+	}
+
+	i = 0;
+	for_each_cpu(cpu, mask) {
+		map->cpus[i++] = cpu;
+	}
+
+	if (i) {
+		map->len = i;
+	} else {
+		kfree(map);
+		map = NULL;
+		free_cpumask_var(mask);
+		DHD_ERROR(("%s : mapping cpu fail.\n", __FUNCTION__));
+		return -1;
+	}
+
+	spin_lock(&rps_map_lock);
+	old_map = rcu_dereference_protected(queue->rps_map,
+		lockdep_is_held(&rps_map_lock));
+	rcu_assign_pointer(queue->rps_map, map);
+	spin_unlock(&rps_map_lock);
+
+	if (map) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0))
+		static_key_slow_inc(&rps_needed.key);
+#else
+		static_key_slow_inc(&rps_needed);
+#endif
+	}
+	if (old_map) {
+		kfree_rcu(old_map, rcu);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0))
+		static_key_slow_dec(&rps_needed.key);
+#else
+		static_key_slow_dec(&rps_needed);
+#endif
+	}
+	free_cpumask_var(mask);
+
+	DHD_INFO(("%s : Done. mapping cpu nummber : %d\n", __FUNCTION__, map->len));
+	return map->len;
+}
+
+void custom_rps_map_clear(struct netdev_rx_queue *queue)
+{
+	struct rps_map *map;
+
+	DHD_INFO(("%s : Entered.\n", __FUNCTION__));
+
+	map = rcu_dereference_protected(queue->rps_map, 1);
+	if (map) {
+		RCU_INIT_POINTER(queue->rps_map, NULL);
+		kfree_rcu(map, rcu);
+		DHD_INFO(("%s : rps_cpus map clear.\n", __FUNCTION__));
+	}
+}
+#endif // endif
+
+#ifdef DHD_DEBUG_PAGEALLOC
+/* XXX Additional Kernel implemenation is needed to use this function at
+ * the top of the check_poison_mem() function in mm/debug-pagealloc.c file.
+ * Please check if below codes are implemenated your Linux Kernel first.
+ *
+ * - mm/debug-pagealloc.c
+ *
+ * // for DHD_DEBUG_PAGEALLOC
+ * typedef void (*page_corrupt_cb_t)(void *handle, void *addr_corrupt, uint addr_len);
+ * page_corrupt_cb_t corrupt_cb = NULL;
+ * void *corrupt_cb_handle = NULL;
+ *
+ * void register_page_corrupt_cb(page_corrupt_cb_t cb, void *handle)
+ * {
+ *      corrupt_cb = cb;
+ *      corrupt_cb_handle = handle;
+ * }
+ * EXPORT_SYMBOL(register_page_corrupt_cb);
+ *
+ * extern void dhd_page_corrupt_cb(void *handle, void *addr_corrupt, size_t len);
+ *
+ * static void check_poison_mem(unsigned char *mem, size_t bytes)
+ * {
+ * ......
+ *
+ *      if (!__ratelimit(&ratelimit))
+ *               return;
+ *      else if (start == end && single_bit_flip(*start, PAGE_POISON))
+ *              printk(KERN_ERR "pagealloc: single bit error\n");
+ *      else
+ *              printk(KERN_ERR "pagealloc: memory corruption\n");
+ *
+ *      print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start,
+ *              end - start + 1, 1);
+ *
+ *      // for DHD_DEBUG_PAGEALLOC
+ *      dhd_page_corrupt_cb(corrupt_cb_handle, start, end - start + 1);
+ *
+ *      dump_stack();
+ * }
+ *
+ */
+
+void
+dhd_page_corrupt_cb(void *handle, void *addr_corrupt, size_t len)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)handle;
+
+	DHD_ERROR(("%s: Got dhd_page_corrupt_cb 0x%p %d\n",
+		__FUNCTION__, addr_corrupt, (uint32)len));
+
+	DHD_OS_WAKE_LOCK(dhdp);
+	prhex("Page Corruption:", addr_corrupt, len);
+	dhd_dump_to_kernelog(dhdp);
+#if defined(BCMPCIE) && defined(DHD_FW_COREDUMP)
+	/* Load the dongle side dump to host memory and then BUG_ON() */
+	dhdp->memdump_enabled = DUMP_MEMONLY;
+	dhdp->memdump_type = DUMP_TYPE_MEMORY_CORRUPTION;
+	dhd_bus_mem_dump(dhdp);
+#endif /* BCMPCIE && DHD_FW_COREDUMP */
+	DHD_OS_WAKE_UNLOCK(dhdp);
+}
+EXPORT_SYMBOL(dhd_page_corrupt_cb);
+#endif /* DHD_DEBUG_PAGEALLOC */
+
+#if defined(BCMPCIE) && defined(DHD_PKTID_AUDIT_ENABLED)
+void
+dhd_pktid_error_handler(dhd_pub_t *dhdp)
+{
+	unsigned long flags = 0;
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s: bus is down!\n", __FUNCTION__));
+		return;
+	}
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	DHD_ERROR(("%s: Got Pkt Id Audit failure \n", __FUNCTION__));
+	DHD_OS_WAKE_LOCK(dhdp);
+	dhd_dump_to_kernelog(dhdp);
+#ifdef DHD_FW_COREDUMP
+	/* Load the dongle side dump to host memory */
+	if (dhdp->memdump_enabled == DUMP_DISABLED) {
+		dhdp->memdump_enabled = DUMP_MEMFILE;
+	}
+	dhdp->memdump_type = DUMP_TYPE_PKTID_AUDIT_FAILURE;
+	dhd_bus_mem_dump(dhdp);
+#endif /* DHD_FW_COREDUMP */
+#ifdef OEM_ANDROID
+	/* XXX Send HANG event to Android Framework for recovery */
+	dhdp->hang_reason = HANG_REASON_PCIE_PKTID_ERROR;
+	dhd_os_check_hang(dhdp, 0, -EREMOTEIO);
+#endif /* OEM_ANDROID */
+	DHD_OS_WAKE_UNLOCK(dhdp);
+}
+#endif /* BCMPCIE && DHD_PKTID_AUDIT_ENABLED */
+
+struct net_device *
+dhd_linux_get_primary_netdev(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+
+	if (dhd->iflist[0] && dhd->iflist[0]->net)
+		return dhd->iflist[0]->net;
+	else
+		return NULL;
+}
+
+static int
+dhd_create_to_notifier_skt(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+	/* Kernel 3.7 onwards this API accepts only 3 arguments. */
+	/* Kernel version 3.6 is a special case which accepts 4 arguments */
+	nl_to_event_sk = netlink_kernel_create(&init_net, BCM_NL_USER, &dhd_netlink_cfg);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+	/* Kernel version 3.5 and below use this old API format */
+	nl_to_event_sk = netlink_kernel_create(&init_net, BCM_NL_USER, 0,
+			dhd_process_daemon_msg, NULL, THIS_MODULE);
+#else
+	nl_to_event_sk = netlink_kernel_create(&init_net, BCM_NL_USER, THIS_MODULE,
+			&dhd_netlink_cfg);
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) */
+	if (!nl_to_event_sk)
+	{
+		DHD_CONS_ONLY(("Error creating socket.\n"));
+		return -1;
+	}
+	DHD_INFO(("nl_to socket created successfully...\n"));
+	return 0;
+}
+
+void
+dhd_destroy_to_notifier_skt(void)
+{
+	DHD_INFO(("Destroying nl_to socket\n"));
+	netlink_kernel_release(nl_to_event_sk);
+}
+
+static void
+dhd_recv_msg_from_daemon(struct sk_buff *skb)
+{
+	struct nlmsghdr *nlh;
+	bcm_to_info_t *cmd;
+
+	nlh = (struct nlmsghdr *)skb->data;
+	cmd = (bcm_to_info_t *)nlmsg_data(nlh);
+	if ((cmd->magic == BCM_TO_MAGIC) && (cmd->reason == REASON_DAEMON_STARTED)) {
+		sender_pid = ((struct nlmsghdr *)(skb->data))->nlmsg_pid;
+		DHD_INFO(("DHD Daemon Started\n"));
+	}
+}
+
+int
+dhd_send_msg_to_daemon(struct sk_buff *skb, void *data, int size)
+{
+	struct nlmsghdr *nlh;
+	struct sk_buff *skb_out;
+	int ret = BCME_ERROR;
+
+	BCM_REFERENCE(skb);
+	if (sender_pid == 0) {
+		DHD_INFO(("Invalid PID 0\n"));
+		skb_out = NULL;
+		goto err;
+	}
+
+	if ((skb_out = nlmsg_new(size, 0)) == NULL) {
+		DHD_ERROR(("%s: skb alloc failed\n", __FUNCTION__));
+		ret = BCME_NOMEM;
+		goto err;
+	}
+	nlh = nlmsg_put(skb_out, 0, 0, NLMSG_DONE, size, 0);
+	if (nlh == NULL) {
+		DHD_ERROR(("%s: nlmsg_put failed\n", __FUNCTION__));
+		goto err;
+	}
+	NETLINK_CB(skb_out).dst_group = 0; /* Unicast */
+	(void)memcpy_s(nlmsg_data(nlh), size, (char *)data, size);
+
+	if ((ret = nlmsg_unicast(nl_to_event_sk, skb_out, sender_pid)) < 0) {
+		DHD_ERROR(("Error sending message, ret:%d\n", ret));
+		/* skb is already freed inside nlmsg_unicast() on error case */
+		/* explicitly making skb_out to NULL to avoid double free */
+		skb_out = NULL;
+		goto err;
+	}
+	return BCME_OK;
+err:
+	if (skb_out) {
+		nlmsg_free(skb_out);
+	}
+	return ret;
+}
+
+static void
+dhd_process_daemon_msg(struct sk_buff *skb)
+{
+	bcm_to_info_t to_info;
+
+	to_info.magic = BCM_TO_MAGIC;
+	to_info.reason = REASON_DAEMON_STARTED;
+	to_info.trap = NO_TRAP;
+
+	dhd_recv_msg_from_daemon(skb);
+	dhd_send_msg_to_daemon(skb, &to_info, sizeof(to_info));
+}
+
+#ifdef REPORT_FATAL_TIMEOUTS
+static void
+dhd_send_trap_to_fw(dhd_pub_t * pub, int reason, int trap)
+{
+	bcm_to_info_t to_info;
+
+	to_info.magic = BCM_TO_MAGIC;
+	to_info.reason = reason;
+	to_info.trap = trap;
+
+	DHD_ERROR(("Sending Event reason:%d trap:%d\n", reason, trap));
+	dhd_send_msg_to_daemon(NULL, (void *)&to_info, sizeof(bcm_to_info_t));
+}
+
+void
+dhd_send_trap_to_fw_for_timeout(dhd_pub_t * pub, timeout_reasons_t reason)
+{
+	int to_reason;
+	int trap = NO_TRAP;
+	switch (reason) {
+		case DHD_REASON_COMMAND_TO:
+			to_reason = REASON_COMMAND_TO;
+			trap = DO_TRAP;
+			break;
+		case DHD_REASON_JOIN_TO:
+			to_reason = REASON_JOIN_TO;
+			trap = DO_TRAP;
+			break;
+		case DHD_REASON_SCAN_TO:
+			to_reason = REASON_SCAN_TO;
+			trap = DO_TRAP;
+			break;
+		case DHD_REASON_OQS_TO:
+			to_reason = REASON_OQS_TO;
+			trap = DO_TRAP;
+			break;
+		default:
+			to_reason = REASON_UNKOWN;
+	}
+	dhd_send_trap_to_fw(pub, to_reason, trap);
+}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+char*
+dhd_dbg_get_system_timestamp(void)
+{
+	static char timebuf[DEBUG_DUMP_TIME_BUF_LEN];
+	struct osl_timespec tv;
+	unsigned long local_time;
+	struct rtc_time tm;
+
+	memset_s(timebuf, DEBUG_DUMP_TIME_BUF_LEN, 0, DEBUG_DUMP_TIME_BUF_LEN);
+	osl_do_gettimeofday(&tv);
+	local_time = (u32)(tv.tv_sec - (sys_tz.tz_minuteswest * 60));
+	rtc_time_to_tm(local_time, &tm);
+	snprintf(timebuf, DEBUG_DUMP_TIME_BUF_LEN,
+			"%02d-%02d-%02d/%02d:%02d:%02d.%04lu",
+			tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
+			tm.tm_sec, tv.tv_usec);
+	return timebuf;
+}
+
+char*
+dhd_log_dump_get_timestamp(void)
+{
+	static char buf[32];
+	u64 ts_nsec;
+	unsigned long rem_nsec;
+
+	ts_nsec = local_clock();
+	rem_nsec = DIV_AND_MOD_U64_BY_U32(ts_nsec, NSEC_PER_SEC);
+	snprintf(buf, sizeof(buf), "%5lu.%06lu",
+		(unsigned long)ts_nsec, rem_nsec / NSEC_PER_USEC);
+
+	return buf;
+}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+void
+dhd_flush_rx_tx_wq(dhd_pub_t *dhdp)
+{
+	dhd_info_t * dhd;
+
+	if (dhdp) {
+		dhd = dhdp->info;
+		if (dhd) {
+			flush_workqueue(dhd->tx_wq);
+			flush_workqueue(dhd->rx_wq);
+		}
+	}
+
+	return;
+}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef DHD_DEBUG_UART
+bool
+dhd_debug_uart_is_running(struct net_device *dev)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	if (dhd->duart_execute) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static void
+dhd_debug_uart_exec_rd(void *handle, void *event_info, u8 event)
+{
+	dhd_pub_t *dhdp = handle;
+	dhd_debug_uart_exec(dhdp, "rd");
+}
+
+static void
+dhd_debug_uart_exec(dhd_pub_t *dhdp, char *cmd)
+{
+	int ret;
+
+	char *argv[] = {DHD_DEBUG_UART_EXEC_PATH, cmd, NULL};
+	char *envp[] = {"HOME=/", "TERM=linux", "PATH=/sbin:/system/bin", NULL};
+
+#ifdef DHD_FW_COREDUMP
+	if (dhdp->memdump_enabled == DUMP_MEMFILE_BUGON)
+#endif
+	{
+		if (dhdp->hang_reason == HANG_REASON_PCIE_LINK_DOWN_RC_DETECT ||
+			dhdp->hang_reason == HANG_REASON_PCIE_LINK_DOWN_EP_DETECT ||
+#ifdef DHD_FW_COREDUMP
+			dhdp->memdump_success == FALSE ||
+#endif
+			FALSE) {
+			dhdp->info->duart_execute = TRUE;
+			DHD_ERROR(("DHD: %s - execute %s %s\n",
+				__FUNCTION__, DHD_DEBUG_UART_EXEC_PATH, cmd));
+			ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
+			DHD_ERROR(("DHD: %s - %s %s ret = %d\n",
+				__FUNCTION__, DHD_DEBUG_UART_EXEC_PATH, cmd, ret));
+			dhdp->info->duart_execute = FALSE;
+
+#ifdef DHD_LOG_DUMP
+			if (dhdp->memdump_type != DUMP_TYPE_BY_SYSDUMP)
+#endif
+			{
+				BUG_ON(1);
+			}
+		}
+	}
+}
+#endif	/* DHD_DEBUG_UART */
+
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+#ifdef DHD_LINUX_STD_FW_API
+void
+dhd_set_blob_support(dhd_pub_t *dhdp, char *fw_path)
+{
+	char *filepath = DHD_CLM_NAME;
+	const struct firmware *fw = NULL;
+	int ret = 0;
+
+	ret = dhd_os_get_img_fwreq(&fw, filepath);
+	if (ret < 0) {
+		DHD_ERROR(("%s: ----- blob file doesn't exist (%s) -----\n", __FUNCTION__,
+			filepath));
+		dhdp->is_blob = FALSE;
+	} else {
+		DHD_ERROR(("%s: ----- blob file exists (%s) -----\n", __FUNCTION__, filepath));
+		dhdp->is_blob = TRUE;
+#if defined(CONCATE_BLOB)
+		strncat(fw_path, "_blob", strlen("_blob"));
+#else
+		BCM_REFERENCE(fw_path);
+#endif /* SKIP_CONCATE_BLOB */
+		dhd_os_close_img_fwreq(fw);
+	}
+}
+#else
+void
+dhd_set_blob_support(dhd_pub_t *dhdp, char *fw_path)
+{
+	struct file *fp;
+	char *filepath = dhdp->info->clm_path;
+
+	fp = dhd_filp_open(filepath, O_RDONLY, 0);
+	if (IS_ERR(fp) || (fp == NULL)) {
+		DHD_ERROR(("%s: ----- blob file doesn't exist (%s) -----\n", __FUNCTION__,
+			filepath));
+		dhdp->is_blob = FALSE;
+	} else {
+		DHD_ERROR(("%s: ----- blob file exists (%s) -----\n", __FUNCTION__, filepath));
+		dhdp->is_blob = TRUE;
+#if defined(CONCATE_BLOB)
+		strncat(fw_path, "_blob", strlen("_blob"));
+#else
+		BCM_REFERENCE(fw_path);
+#endif /* SKIP_CONCATE_BLOB */
+		dhd_filp_close(fp, NULL);
+	}
+}
+#endif /* DHD_LINUX_STD_FW_API */
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+
+#if defined(PCIE_FULL_DONGLE)
+/** test / loopback */
+void
+dmaxfer_free_dmaaddr_handler(void *handle, void *event_info, u8 event)
+{
+	dmaxref_mem_map_t *dmmap = (dmaxref_mem_map_t *)event_info;
+	dhd_info_t *dhd_info = (dhd_info_t *)handle;
+
+	if (event != DHD_WQ_WORK_DMA_LB_MEM_REL) {
+		DHD_ERROR(("%s: unexpected event \n", __FUNCTION__));
+		return;
+	}
+	if (dhd_info == NULL) {
+		DHD_ERROR(("%s: invalid dhd_info\n", __FUNCTION__));
+		return;
+	}
+	if (dmmap == NULL) {
+		DHD_ERROR(("%s: dmmap is null\n", __FUNCTION__));
+		return;
+	}
+	dmaxfer_free_prev_dmaaddr(&dhd_info->pub, dmmap);
+}
+
+void
+dhd_schedule_dmaxfer_free(dhd_pub_t *dhdp, dmaxref_mem_map_t *dmmap)
+{
+	dhd_info_t *dhd_info = dhdp->info;
+
+	dhd_deferred_schedule_work(dhd_info->dhd_deferred_wq, (void *)dmmap,
+		DHD_WQ_WORK_DMA_LB_MEM_REL, dmaxfer_free_dmaaddr_handler, DHD_WQ_WORK_PRIORITY_LOW);
+}
+#endif /* PCIE_FULL_DONGLE */
+/* ---------------------------- End of sysfs implementation ------------------------------------- */
+
+int
+dhd_write_file(const char *filepath, char *buf, int buf_len)
+{
+	struct file *fp = NULL;
+#ifdef get_fs
+	mm_segment_t old_fs;
+#endif /* get_fs */
+	int ret = 0;
+
+#ifdef get_fs
+	/* change to KERNEL_DS address limit */
+	old_fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	/* File is always created. */
+	fp = dhd_filp_open(filepath, O_RDWR | O_CREAT, 0664);
+	if (IS_ERR(fp) || (fp == NULL)) {
+		DHD_ERROR(("%s: Couldn't open file '%s' err %ld\n",
+			__FUNCTION__, filepath, PTR_ERR(fp)));
+		ret = BCME_ERROR;
+	} else {
+		if (fp->f_mode & FMODE_WRITE) {
+			ret = dhd_vfs_write(fp, buf, buf_len, &fp->f_pos);
+			if (ret < 0) {
+				DHD_ERROR(("%s: Couldn't write file '%s'\n",
+					__FUNCTION__, filepath));
+				ret = BCME_ERROR;
+			} else {
+				ret = BCME_OK;
+			}
+		}
+		dhd_filp_close(fp, NULL);
+	}
+
+#ifdef get_fs
+	/* restore previous address limit */
+	set_fs(old_fs);
+#endif /* get_fs */
+
+	return ret;
+}
+
+int
+dhd_read_file(const char *filepath, char *buf, int buf_len)
+{
+	struct file *fp = NULL;
+#ifdef get_fs
+	mm_segment_t old_fs;
+#endif /* get_fs */
+	int ret;
+
+#ifdef get_fs
+	/* change to KERNEL_DS address limit */
+	old_fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	fp = dhd_filp_open(filepath, O_RDONLY, 0);
+	if (IS_ERR(fp) || (fp == NULL)) {
+#ifdef get_fs
+		set_fs(old_fs);
+#endif /* get_fs */
+		DHD_ERROR(("%s: File %s doesn't exist\n", __FUNCTION__, filepath));
+		return BCME_ERROR;
+	}
+
+	ret = dhd_kernel_read_compat(fp, 0, buf, buf_len);
+	dhd_filp_close(fp, NULL);
+
+#ifdef get_fs
+	/* restore previous address limit */
+	set_fs(old_fs);
+#endif /* get_fs */
+
+	/* Return the number of bytes read */
+	if (ret > 0) {
+		/* Success to read */
+		ret = 0;
+	} else {
+		DHD_ERROR(("%s: Couldn't read the file %s, ret=%d\n",
+			__FUNCTION__, filepath, ret));
+		ret = BCME_ERROR;
+	}
+
+	return ret;
+}
+
+int
+dhd_write_file_and_check(const char *filepath, char *buf, int buf_len)
+{
+	int ret;
+
+	ret = dhd_write_file(filepath, buf, buf_len);
+	if (ret < 0) {
+		return ret;
+	}
+
+	/* Read the file again and check if the file size is not zero */
+	memset(buf, 0, buf_len);
+	ret = dhd_read_file(filepath, buf, buf_len);
+
+	return ret;
+}
+
+#ifdef FILTER_IE
+int dhd_read_from_file(dhd_pub_t *dhd)
+{
+	int ret = 0;
+#ifdef DHD_LINUX_STD_FW_API
+	const struct firmware *fw = NULL;
+	char *filepath = FILTER_IE_PATH;
+	int filelen = 0;
+#else
+	int nread = 0;
+	void *fd;
+#endif /* DHD_LINUX_STD_FW_API */
+	uint8 *buf;
+	NULL_CHECK(dhd, "dhd is NULL", ret);
+
+	buf = MALLOCZ(dhd->osh, FILE_BLOCK_READ_SIZE);
+	if (!buf) {
+		DHD_ERROR(("error: failed to alllocate buf.\n"));
+		return BCME_NOMEM;
+	}
+
+	/* open file to read */
+#ifdef DHD_LINUX_STD_FW_API
+	ret = dhd_os_get_img_fwreq(&fw, filepath);
+	if (ret < 0) {
+		DHD_ERROR(("dhd_os_get_img_fwreq(%s) error : %d\n",
+			filepath, ret));
+		goto exit;
+	}
+
+	filelen = fw->size;
+	if (filelen == 0) {
+		DHD_ERROR(("error: zero length file.failed to read\n"));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	ret = memcpy_s(buf, FILE_BLOCK_READ_SIZE, fw->data, fw->size);
+	if (ret < 0) {
+		DHD_ERROR((" memcpy_s() error : %d\n", ret));
+		goto exit;
+	}
+
+	ret = dhd_parse_filter_ie(dhd, buf);
+	if (ret < 0) {
+		DHD_ERROR(("error: failed to parse filter ie\n"));
+	}
+#else
+	fd = dhd_os_open_image1(dhd, FILTER_IE_PATH);
+	if (!fd) {
+		DHD_ERROR(("No filter file(not an error), filter path%s\n", FILTER_IE_PATH));
+		ret = BCME_EPERM;
+		goto exit;
+	}
+	nread = dhd_os_get_image_block(buf, (FILE_BLOCK_READ_SIZE - 1), fd);
+	if (nread > 0) {
+		buf[nread] = '\0';
+		if ((ret = dhd_parse_filter_ie(dhd, buf)) < 0) {
+			DHD_ERROR(("error: failed to parse filter ie\n"));
+		}
+	} else {
+		DHD_ERROR(("error: zero length file.failed to read\n"));
+		ret = BCME_ERROR;
+	}
+#endif /* DHD_LINUX_STD_FW_API */
+exit:
+#ifdef DHD_LINUX_STD_FW_API
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+#else
+	dhd_os_close_image1(dhd, fd);
+#endif /* DHD_LINUX_STD_FW_API */
+	if (buf) {
+		MFREE(dhd->osh, buf, FILE_BLOCK_READ_SIZE);
+	}
+	return ret;
+}
+
+int dhd_get_filter_ie_count(dhd_pub_t *dhdp, uint8* buf)
+{
+	uint8* pstr = buf;
+	int element_count = 0;
+
+	if (buf == NULL) {
+		return BCME_ERROR;
+	}
+
+	while (*pstr != '\0') {
+		if (*pstr == '\n') {
+			element_count++;
+		}
+		pstr++;
+	}
+	/*
+	 * New line character must not be present after last line.
+	 * To count last line
+	 */
+	element_count++;
+
+	return element_count;
+}
+
+int dhd_parse_oui(dhd_pub_t *dhd, uint8 *inbuf, uint8 *oui, int len)
+{
+	uint8 i, j, msb, lsb, oui_len = 0;
+	/*
+	 * OUI can vary from 3 bytes to 5 bytes.
+	 * While reading from file as ascii input it can
+	 * take maximum size of 14 bytes and minumum size of
+	 * 8 bytes including ":"
+	 * Example 5byte OUI <AB:DE:BE:CD:FA>
+	 * Example 3byte OUI <AB:DC:EF>
+	 */
+
+	if ((inbuf == NULL) || (len < 8) || (len > 14)) {
+		DHD_ERROR(("error: failed to parse OUI \n"));
+		return BCME_ERROR;
+	}
+
+	for (j = 0, i = 0; i < len; i += 3, ++j) {
+		if (!bcm_isxdigit(inbuf[i]) || !bcm_isxdigit(inbuf[i + 1])) {
+			DHD_ERROR(("error: invalid OUI format \n"));
+			return BCME_ERROR;
+		}
+		msb = inbuf[i] > '9' ? bcm_toupper(inbuf[i]) - 'A' + 10 : inbuf[i] - '0';
+		lsb = inbuf[i + 1] > '9' ? bcm_toupper(inbuf[i + 1]) -
+			'A' + 10 : inbuf[i + 1] - '0';
+		oui[j] = (msb << 4) | lsb;
+	}
+	/* Size of oui.It can vary from 3/4/5 */
+	oui_len = j;
+
+	return oui_len;
+}
+
+int dhd_check_valid_ie(dhd_pub_t *dhdp, uint8* buf, int len)
+{
+	int i = 0;
+
+	while (i < len) {
+		if (!bcm_isdigit(buf[i])) {
+			DHD_ERROR(("error: non digit value found in filter_ie \n"));
+			return BCME_ERROR;
+		}
+		i++;
+	}
+
+	return BCME_OK;
+}
+
+int dhd_parse_filter_ie(dhd_pub_t *dhd, uint8 *buf)
+{
+	int element_count = 0, i = 0, oui_size = 0, ret = 0;
+	uint16 bufsize, buf_space_left, id = 0, len = 0;
+	uint16 filter_iovsize, all_tlvsize;
+	wl_filter_ie_tlv_t *p_ie_tlv = NULL;
+	wl_filter_ie_iov_v1_t *p_filter_iov = (wl_filter_ie_iov_v1_t *) NULL;
+	char *token = NULL, *ele_token = NULL, *oui_token = NULL, *type = NULL;
+	uint8 data[20];
+
+	element_count = dhd_get_filter_ie_count(dhd, buf);
+	DHD_INFO(("total element count %d \n", element_count));
+	/* Calculate the whole buffer size */
+	filter_iovsize = sizeof(wl_filter_ie_iov_v1_t) + FILTER_IE_BUFSZ;
+	p_filter_iov = MALLOCZ(dhd->osh, filter_iovsize);
+
+	if (p_filter_iov == NULL) {
+		DHD_ERROR(("error: failed to allocate %d bytes of memory\n", filter_iovsize));
+		return BCME_ERROR;
+	}
+
+	/* setup filter iovar header */
+	p_filter_iov->version = WL_FILTER_IE_VERSION_1;
+	p_filter_iov->len = filter_iovsize;
+	p_filter_iov->fixed_length = p_filter_iov->len - FILTER_IE_BUFSZ;
+	p_filter_iov->pktflag = FC_PROBE_REQ;
+	p_filter_iov->option = WL_FILTER_IE_CHECK_SUB_OPTION;
+	/* setup TLVs */
+	bufsize = filter_iovsize - WL_FILTER_IE_IOV_HDR_SIZE; /* adjust available size for TLVs */
+	p_ie_tlv = (wl_filter_ie_tlv_t *)&p_filter_iov->tlvs[0];
+	buf_space_left = bufsize;
+
+	while ((i < element_count) && (buf != NULL)) {
+		len = 0;
+		/* token contains one line of input data */
+		token = bcmstrtok((char**)&buf, "\n", NULL);
+		if (token == NULL) {
+			break;
+		}
+		if ((ele_token = bcmstrstr(token, ",")) == NULL) {
+		/* only element id is present */
+			if (dhd_check_valid_ie(dhd, token, strlen(token)) == BCME_ERROR) {
+				DHD_ERROR(("error: Invalid element id \n"));
+				ret = BCME_ERROR;
+				goto exit;
+			}
+			id = bcm_atoi((char*)token);
+			data[len++] = WL_FILTER_IE_SET;
+		} else {
+			/* oui is present */
+			ele_token = bcmstrtok(&token, ",", NULL);
+			if ((ele_token == NULL) || (dhd_check_valid_ie(dhd, ele_token,
+				strlen(ele_token)) == BCME_ERROR)) {
+				DHD_ERROR(("error: Invalid element id \n"));
+				ret = BCME_ERROR;
+				goto exit;
+			}
+			id =  bcm_atoi((char*)ele_token);
+			data[len++] = WL_FILTER_IE_SET;
+			if ((oui_token = bcmstrstr(token, ",")) == NULL) {
+				oui_size = dhd_parse_oui(dhd, token, &(data[len]), strlen(token));
+				if (oui_size == BCME_ERROR) {
+					DHD_ERROR(("error: Invalid OUI \n"));
+					ret = BCME_ERROR;
+					goto exit;
+				}
+				len += oui_size;
+			} else {
+				/* type is present */
+				oui_token = bcmstrtok(&token, ",", NULL);
+				if ((oui_token == NULL) || ((oui_size =
+					dhd_parse_oui(dhd, oui_token,
+					&(data[len]), strlen(oui_token))) == BCME_ERROR)) {
+					DHD_ERROR(("error: Invalid OUI \n"));
+					ret = BCME_ERROR;
+					goto exit;
+				}
+				len += oui_size;
+				if ((type = bcmstrstr(token, ",")) == NULL) {
+					if (dhd_check_valid_ie(dhd, token,
+						strlen(token)) == BCME_ERROR) {
+						DHD_ERROR(("error: Invalid type \n"));
+						ret = BCME_ERROR;
+						goto exit;
+					}
+					data[len++] = bcm_atoi((char*)token);
+				} else {
+					/* subtype is present */
+					type = bcmstrtok(&token, ",", NULL);
+					if ((type == NULL) || (dhd_check_valid_ie(dhd, type,
+						strlen(type)) == BCME_ERROR)) {
+						DHD_ERROR(("error: Invalid type \n"));
+						ret = BCME_ERROR;
+						goto exit;
+					}
+					data[len++] = bcm_atoi((char*)type);
+					/* subtype is last element */
+					if ((token == NULL) || (*token == '\0') ||
+						(dhd_check_valid_ie(dhd, token,
+						strlen(token)) == BCME_ERROR)) {
+						DHD_ERROR(("error: Invalid subtype \n"));
+						ret = BCME_ERROR;
+						goto exit;
+					}
+					data[len++] = bcm_atoi((char*)token);
+				}
+			}
+		}
+		ret = bcm_pack_xtlv_entry((uint8 **)&p_ie_tlv,
+			&buf_space_left, id, len, data, BCM_XTLV_OPTION_ALIGN32);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s : bcm_pack_xtlv_entry() failed ,"
+				"status=%d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+		i++;
+	}
+	if (i == 0) {
+		/* file is empty or first line is blank */
+		DHD_ERROR(("error: filter_ie file is empty or first line is blank \n"));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+	/* update the iov header, set len to include all TLVs + header */
+	all_tlvsize = (bufsize - buf_space_left);
+	p_filter_iov->len = htol16(all_tlvsize + WL_FILTER_IE_IOV_HDR_SIZE);
+	ret = dhd_iovar(dhd, 0, "filter_ie", (void *)p_filter_iov,
+			p_filter_iov->len, NULL, 0, TRUE);
+	if (ret != BCME_OK) {
+		DHD_ERROR(("error: IOVAR failed, status=%d\n", ret));
+	}
+exit:
+	/* clean up */
+	if (p_filter_iov) {
+		MFREE(dhd->osh, p_filter_iov, filter_iovsize);
+	}
+	return ret;
+}
+#endif /* FILTER_IE */
+#ifdef DHD_WAKE_STATUS
+wake_counts_t*
+dhd_get_wakecount(dhd_pub_t *dhdp)
+{
+#ifdef BCMDBUS
+	return NULL;
+#else
+	return dhd_bus_get_wakecount(dhdp);
+#endif /* BCMDBUS */
+}
+#endif /* DHD_WAKE_STATUS */
+
+int
+dhd_get_random_bytes(uint8 *buf, uint len)
+{
+#ifdef BCMPCIE
+	get_random_bytes(buf, len);
+#endif /* BCMPCIE */
+	return BCME_OK;
+}
+
+#if defined(DHD_HANG_SEND_UP_TEST)
+void
+dhd_make_hang_with_reason(struct net_device *dev, const char *string_num)
+{
+	dhd_info_t *dhd = NULL;
+	dhd_pub_t *dhdp = NULL;
+	uint reason = HANG_REASON_MAX;
+	uint32 fw_test_code = 0;
+	dhd = DHD_DEV_INFO(dev);
+
+	if (dhd) {
+		dhdp = &dhd->pub;
+	}
+
+	if (!dhd || !dhdp) {
+		return;
+	}
+
+	reason = (uint) bcm_strtoul(string_num, NULL, 0);
+	DHD_ERROR(("Enter %s, reason=0x%x\n", __FUNCTION__,  reason));
+
+	if (reason == 0) {
+		if (dhdp->req_hang_type) {
+			DHD_ERROR(("%s, Clear HANG test request 0x%x\n",
+				__FUNCTION__, dhdp->req_hang_type));
+			dhdp->req_hang_type = 0;
+			return;
+		} else {
+			DHD_ERROR(("%s, No requested HANG test\n", __FUNCTION__));
+			return;
+		}
+	} else if ((reason <= HANG_REASON_MASK) || (reason >= HANG_REASON_MAX)) {
+		DHD_ERROR(("Invalid HANG request, reason 0x%x\n", reason));
+		return;
+	}
+
+	if (dhdp->req_hang_type != 0) {
+		DHD_ERROR(("Already HANG requested for test\n"));
+		return;
+	}
+
+	switch (reason) {
+		case HANG_REASON_IOCTL_RESP_TIMEOUT:
+			DHD_ERROR(("Make HANG!!!: IOCTL response timeout(0x%x)\n", reason));
+			dhdp->req_hang_type = reason;
+			fw_test_code = 102; /* resumed on timeour */
+			(void) dhd_wl_ioctl_set_intiovar(dhdp, "bus:disconnect", fw_test_code,
+				WLC_SET_VAR, TRUE, 0);
+			break;
+		case HANG_REASON_DONGLE_TRAP:
+			DHD_ERROR(("Make HANG!!!: Dongle trap (0x%x)\n", reason));
+			dhdp->req_hang_type = reason;
+			fw_test_code = 99; /* dongle trap */
+			(void) dhd_wl_ioctl_set_intiovar(dhdp, "bus:disconnect", fw_test_code,
+				WLC_SET_VAR, TRUE, 0);
+			break;
+		case HANG_REASON_D3_ACK_TIMEOUT:
+			DHD_ERROR(("Make HANG!!!: D3 ACK timeout (0x%x)\n", reason));
+			dhdp->req_hang_type = reason;
+			break;
+		case HANG_REASON_BUS_DOWN:
+			DHD_ERROR(("Make HANG!!!: BUS down(0x%x)\n", reason));
+			dhdp->req_hang_type = reason;
+			break;
+		case HANG_REASON_PCIE_LINK_DOWN_RC_DETECT:
+		case HANG_REASON_PCIE_LINK_DOWN_EP_DETECT:
+		case HANG_REASON_MSGBUF_LIVELOCK:
+			dhdp->req_hang_type = 0;
+			DHD_ERROR(("Does not support requested HANG(0x%x)\n", reason));
+			break;
+		case HANG_REASON_IFACE_DEL_FAILURE:
+			dhdp->req_hang_type = 0;
+			DHD_ERROR(("Does not support requested HANG(0x%x)\n", reason));
+			break;
+		case HANG_REASON_HT_AVAIL_ERROR:
+			dhdp->req_hang_type = 0;
+			DHD_ERROR(("PCIe does not support requested HANG(0x%x)\n", reason));
+			break;
+		case HANG_REASON_PCIE_RC_LINK_UP_FAIL:
+			DHD_ERROR(("Make HANG!!!:Link Up(0x%x)\n", reason));
+			dhdp->req_hang_type = reason;
+			break;
+		default:
+			dhdp->req_hang_type = 0;
+			DHD_ERROR(("Unknown HANG request (0x%x)\n", reason));
+			break;
+	}
+}
+#endif /* DHD_HANG_SEND_UP_TEST */
+
+#ifdef DHD_TX_PROFILE
+static int
+process_layer2_headers(uint8 **p, int *plen, uint16 *type, bool is_host_sfhllc)
+{
+	int err = BCME_OK;
+
+	if (*type < ETHER_TYPE_MIN) {
+		struct dot3_mac_llc_snap_header *sh = (struct dot3_mac_llc_snap_header *)*p;
+
+		if (bcmp(&sh->dsap, llc_snap_hdr, SNAP_HDR_LEN) == 0) {
+			*type = ntoh16(sh->type);
+			if (*type == ETHER_TYPE_8021Q ||
+				(is_host_sfhllc && *type != ETHER_TYPE_8021Q)) {
+				*p += sizeof(struct dot3_mac_llc_snap_header);
+				if ((*plen -= sizeof(struct dot3_mac_llc_snap_header)) <= 0) {
+					err = BCME_ERROR;
+				}
+			}
+			else {
+				struct dot3_mac_llc_snapvlan_header *svh = (struct
+					dot3_mac_llc_snapvlan_header *)*p;
+
+				*type = ntoh16(svh->ether_type);
+				*p += sizeof(struct dot3_mac_llc_snapvlan_header);
+				if ((*plen -= sizeof(struct dot3_mac_llc_snapvlan_header)) <= 0) {
+					err = BCME_ERROR;
+				}
+			}
+		}
+		else {
+			err = BCME_ERROR;
+		}
+	}
+	else {
+		if (*type == ETHER_TYPE_8021Q) {
+			struct ethervlan_header *evh = (struct ethervlan_header *)*p;
+
+			*type = ntoh16(evh->ether_type);
+			*p += ETHERVLAN_HDR_LEN;
+			if ((*plen -= ETHERVLAN_HDR_LEN) <= 0) {
+				err = BCME_ERROR;
+			}
+		}
+		else {
+			*p += ETHER_HDR_LEN;
+			if ((*plen -= ETHER_HDR_LEN) <= 0) {
+				err = BCME_ERROR;
+			}
+		}
+	}
+
+	return err;
+}
+
+static int
+process_layer3_headers(uint8 **p, int plen, uint16 *type)
+{
+	int err = BCME_OK;
+
+	if (*type == ETHER_TYPE_IP) {
+		struct ipv4_hdr *iph = (struct ipv4_hdr *)*p;
+		uint16 len = IPV4_HLEN(iph);
+		if ((plen -= len) <= 0) {
+			err = BCME_ERROR;
+		} else if (IP_VER(iph) == IP_VER_4 && len >= IPV4_MIN_HEADER_LEN) {
+			*type = IPV4_PROT(iph);
+			*p += len;
+		} else {
+			err = BCME_ERROR;
+		}
+	} else if (*type == ETHER_TYPE_IPV6) {
+		struct ipv6_hdr *ip6h = (struct ipv6_hdr *)*p;
+		if ((plen -= IPV6_MIN_HLEN) <= 0) {
+			err = BCME_ERROR;
+		} else if (IP_VER(ip6h) == IP_VER_6) {
+			*type = IPV6_PROT(ip6h);
+			*p += IPV6_MIN_HLEN;
+			if (IPV6_EXTHDR(*type)) {
+				uint8 proto_6 = 0;
+				int32 exth_len = ipv6_exthdr_len(*p, &proto_6);
+				if (exth_len < 0 || ((plen -= exth_len) <= 0)) {
+					err = BCME_ERROR;
+				} else {
+					*type = proto_6;
+					*p += exth_len;
+				}
+			}
+		} else {
+			err = BCME_ERROR;
+		}
+	}
+
+	return err;
+}
+
+bool
+dhd_protocol_matches_profile(uint8 *p, int plen, const dhd_tx_profile_protocol_t
+		*proto, bool is_host_sfhllc)
+{
+	struct ether_header *eh = NULL;
+	bool result = FALSE;
+	uint16 type = 0, ether_type = 0;
+
+	ASSERT(proto != NULL);
+	ASSERT(p != NULL);
+
+	if (plen <= 0) {
+		result = FALSE;
+	} else {
+		eh = (struct ether_header *)p;
+		type = ntoh16(eh->ether_type);
+		if (type < ETHER_TYPE_MIN && is_host_sfhllc) {
+			struct dot3_mac_llc_snap_header *dot3 =
+				(struct dot3_mac_llc_snap_header *)p;
+			ether_type = ntoh16(dot3->type);
+		} else {
+			ether_type = type;
+		}
+
+		if (proto->layer == DHD_TX_PROFILE_DATA_LINK_LAYER &&
+				proto->protocol_number == ether_type) {
+			result = TRUE;
+		} else if (process_layer2_headers(&p, &plen, &type, is_host_sfhllc) != BCME_OK) {
+			/* pass 'type' instead of 'ether_type' to process_layer2_headers
+			 * because process_layer2_headers will take care of extraction
+			 * of protocol types if llc snap header is present, based on
+			 * the condition (type < ETHER_TYPE_MIN)
+			 */
+			result = FALSE;
+		} else if (proto->layer == DHD_TX_PROFILE_DATA_LINK_LAYER) {
+			result = proto->protocol_number == type;
+		} else if (proto->layer != DHD_TX_PROFILE_NETWORK_LAYER) {
+			result = FALSE;
+		} else if (process_layer3_headers(&p, plen, &type) != BCME_OK) {
+			result = FALSE;
+		} else if (proto->protocol_number == type) {
+			/* L4, only check TCP/UDP case */
+			if ((type == IP_PROT_TCP) || (type == IP_PROT_UDP)) {
+				/* src/dst port are the first two uint16 fields in both tcp/udp
+				 * hdr
+				 */
+				struct bcmudp_hdr *hdr = (struct bcmudp_hdr *)p;
+
+				/* note that a src_port or dest_port of zero counts as a match
+				 */
+				result = ((proto->src_port == 0) || (proto->src_port ==
+					ntoh16(hdr->src_port))) && ((proto->dest_port == 0) ||
+					(proto->dest_port == ntoh16(hdr->dst_port)));
+			} else {
+				/* at this point we know we are dealing with layer 3, and we
+				 * know we are not dealing with TCP or UDP; this is considered a
+				 * match
+				 */
+				result = TRUE;
+			}
+		}
+	}
+
+	return result;
+}
+#endif /* defined(DHD_TX_PROFILE) */
+
+#ifdef BCMPCIE
+#define KIRQ_PRINT_BUF_LEN 256
+
+void
+dhd_print_kirqstats(dhd_pub_t *dhd, unsigned int irq_num)
+{
+	unsigned long flags = 0;
+	struct irq_desc *desc;
+	int i;          /* cpu iterator */
+	struct bcmstrbuf strbuf;
+	char tmp_buf[KIRQ_PRINT_BUF_LEN];
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	desc = irq_to_desc(irq_num);
+#else // (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	desc = irq_data_to_desc(irq_get_irq_data(irq_num));
+#endif // (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	if (!desc) {
+		DHD_ERROR(("%s : irqdesc is not found \n", __FUNCTION__));
+		return;
+	}
+	bcm_binit(&strbuf, tmp_buf, KIRQ_PRINT_BUF_LEN);
+	raw_spin_lock_irqsave(&desc->lock, flags);
+	bcm_bprintf(&strbuf, "dhd irq %u:", irq_num);
+	for_each_online_cpu(i)
+		bcm_bprintf(&strbuf, "%10u ",
+			desc->kstat_irqs ? *per_cpu_ptr(desc->kstat_irqs, i) : 0);
+	if (desc->irq_data.chip) {
+		if (desc->irq_data.chip->name)
+			bcm_bprintf(&strbuf, " %8s", desc->irq_data.chip->name);
+		else
+			bcm_bprintf(&strbuf, " %8s", "-");
+	} else {
+		bcm_bprintf(&strbuf, " %8s", "None");
+	}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0))
+	if (desc->irq_data.domain)
+		bcm_bprintf(&strbuf, " %d", (int)desc->irq_data.hwirq);
+#ifdef CONFIG_GENERIC_IRQ_SHOW_LEVEL
+	bcm_bprintf(&strbuf, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge");
+#endif
+#endif /* LINUX VERSION > 3.1.0 */
+
+	if (desc->name)
+		bcm_bprintf(&strbuf, "-%-8s", desc->name);
+
+	DHD_ERROR(("%s\n", strbuf.origbuf));
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
+#endif /* LINUX VERSION > 2.6.28 */
+}
+#endif /* BCMPCIE */
+
+void
+dhd_show_kirqstats(dhd_pub_t *dhd)
+{
+	unsigned int irq = -1;
+#ifdef BCMPCIE
+	dhdpcie_get_pcieirq(dhd->bus, &irq);
+#endif /* BCMPCIE */
+#ifdef BCMSDIO
+	irq = ((wifi_adapter_info_t *)dhd->info->adapter)->irq_num;
+#endif /* BCMSDIO */
+	if (irq != -1) {
+#ifdef BCMPCIE
+		DHD_ERROR(("DUMP data kernel irq stats : \n"));
+		dhd_print_kirqstats(dhd, irq);
+#endif /* BCMPCIE */
+#ifdef BCMSDIO
+		DHD_ERROR(("DUMP data/host wakeup kernel irq stats : \n"));
+#endif /* BCMSDIO */
+	}
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	irq = dhd_bus_get_oob_irq_num(dhd);
+	if (irq) {
+		DHD_ERROR(("DUMP PCIE host wakeup kernel irq stats : \n"));
+		dhd_print_kirqstats(dhd, irq);
+	}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+}
+
+void
+dhd_print_tasklet_status(dhd_pub_t *dhd)
+{
+	dhd_info_t *dhdinfo;
+
+	if (!dhd) {
+		DHD_ERROR(("%s : DHD is null\n", __FUNCTION__));
+		return;
+	}
+
+	dhdinfo = dhd->info;
+
+	if (!dhdinfo) {
+		DHD_ERROR(("%s : DHD INFO is null \n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("DHD Tasklet status : 0x%lx\n", dhdinfo->tasklet.state));
+#ifndef BCMDBUS
+	DHD_ERROR(("DPC thread thr_pid: %ld\n", dhdinfo->thr_dpc_ctl.thr_pid));
+#endif /* BCMDBUS */
+}
+
+#if defined(DHD_MQ) && defined(DHD_MQ_STATS)
+void
+dhd_mqstats_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	dhd_info_t *dhd = NULL;
+	int i = 0, j = 0;
+
+	if (!dhdp || !strbuf)
+		return;
+
+	dhd = dhdp->info;
+	bcm_bprintf(strbuf, "\nMQ STATS:\n=========\n");
+
+	bcm_bprintf(strbuf, "\nTx packet arrival AC histogram:\n");
+	bcm_bprintf(strbuf, "AC_BE    \tAC_BK    \tAC_VI    \tAC_VO\n");
+	bcm_bprintf(strbuf, "-----    \t-----    \t-----    \t-----\n");
+	for (i = 0; i < AC_COUNT; i++)
+		bcm_bprintf(strbuf, "%-10d\t", dhd->pktcnt_per_ac[i]);
+
+	bcm_bprintf(strbuf, "\n\nTx packet arrival Q-AC histogram:\n");
+	bcm_bprintf(strbuf, "\tAC_BE    \tAC_BK    \tAC_VI    \tAC_VO\n");
+	bcm_bprintf(strbuf, "\t-----    \t-----    \t-----    \t-----");
+	for (i = 0; i < MQ_MAX_QUEUES; i++) {
+		bcm_bprintf(strbuf, "\nQ%d\t", i);
+		for (j = 0; j < AC_COUNT; j++)
+			bcm_bprintf(strbuf, "%-8d\t", dhd->pktcnt_qac_histo[i][j]);
+	}
+
+	bcm_bprintf(strbuf, "\n\nTx Q-CPU scheduling histogram:\n");
+	bcm_bprintf(strbuf, "\t");
+	for (i = 0; i < nr_cpu_ids; i++)
+		bcm_bprintf(strbuf, "CPU%d    \t", i);
+	for (i = 0; i < MQ_MAX_QUEUES; i++) {
+		bcm_bprintf(strbuf, "\nQ%d\t", i);
+		for (j = 0; j < nr_cpu_ids; j++)
+			bcm_bprintf(strbuf, "%-8d\t", dhd->cpu_qstats[i][j]);
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+#endif /* DHD_MQ && DHD_MQ_STATS */
+
+#ifdef DHD_MAP_LOGGING
+/* Will be called from SMMU fault handler */
+void
+dhd_smmu_fault_handler(uint32 axid, ulong fault_addr)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)g_dhd_pub;
+	uint32 irq = (uint32)-1;
+
+	DHD_ERROR(("%s: Trigger SMMU Fault\n", __FUNCTION__));
+	DHD_ERROR(("%s: axid:0x%x, fault_addr:0x%lx", __FUNCTION__, axid, fault_addr));
+	dhdp->smmu_fault_occurred = TRUE;
+#ifdef DNGL_AXI_ERROR_LOGGING
+	dhdp->axi_error = TRUE;
+	dhdp->axi_err_dump->axid = axid;
+	dhdp->axi_err_dump->fault_address = fault_addr;
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+	/* Disable PCIe IRQ */
+	dhdpcie_get_pcieirq(dhdp->bus, &irq);
+	if (irq != (uint32)-1) {
+		disable_irq_nosync(irq);
+	}
+
+	/* Take debug information first */
+	DHD_OS_WAKE_LOCK(dhdp);
+	dhd_prot_smmu_fault_dump(dhdp);
+	DHD_OS_WAKE_UNLOCK(dhdp);
+
+	/* Take AXI information if possible */
+#ifdef DNGL_AXI_ERROR_LOGGING
+#ifdef DHD_USE_WQ_FOR_DNGL_AXI_ERROR
+	dhd_axi_error_dispatch(dhdp);
+#else
+	dhd_axi_error(dhdp);
+#endif /* DHD_USE_WQ_FOR_DNGL_AXI_ERROR */
+#endif /* DNGL_AXI_ERROR_LOGGING */
+}
+EXPORT_SYMBOL(dhd_smmu_fault_handler);
+#endif /* DHD_MAP_LOGGING */
+
+#ifdef DHD_PKT_LOGGING
+int
+dhd_pktlog_debug_dump(dhd_pub_t *dhdp)
+{
+	struct net_device *primary_ndev;
+	struct bcm_cfg80211 *cfg;
+	unsigned long flags = 0;
+
+	primary_ndev = dhd_linux_get_primary_netdev(dhdp);
+	if (!primary_ndev) {
+		DHD_ERROR(("%s: Cannot find primary netdev\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	cfg = wl_get_cfg(primary_ndev);
+	if (!cfg) {
+		DHD_ERROR(("%s: Cannot find cfg\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_BUSY_CHECK_IN_HALDUMP(dhdp)) {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s: HAL dump is already triggered \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_BUS_BUSY_SET_IN_HALDUMP(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	DHD_OS_WAKE_LOCK(dhdp);
+
+	if (wl_cfg80211_is_hal_started(cfg)) {
+		dhdp->pktlog_debug = TRUE;
+		dhd_dbg_send_urgent_evt(dhdp, NULL, 0);
+	} else {
+		DHD_ERROR(("[DUMP] %s: HAL Not started. skip urgent event\n", __FUNCTION__));
+	}
+	DHD_OS_WAKE_UNLOCK(dhdp);
+	/* In case of dhd_os_busbusy_wait_bitmask() timeout,
+	 * hal dump bit will not be cleared. Hence clearing it here.
+	 */
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_HALDUMP(dhdp);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	return BCME_OK;
+}
+
+void
+dhd_pktlog_dump(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (dhd_pktlog_dump_write_file(&dhd->pub)) {
+		DHD_ERROR(("%s: writing pktlog dump file failed\n", __FUNCTION__));
+		return;
+	}
+}
+
+void
+dhd_schedule_pktlog_dump(dhd_pub_t *dhdp)
+{
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq,
+			(void*)NULL, DHD_WQ_WORK_PKTLOG_DUMP,
+			dhd_pktlog_dump, DHD_WQ_WORK_PRIORITY_HIGH);
+}
+#endif /* DHD_PKT_LOGGING */
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+static void dhd_blk_tsfl_handler(struct work_struct * work)
+{
+	dhd_if_t *ifp = NULL;
+	dhd_pub_t *dhdp = NULL;
+	/* Ignore compiler warnings due to -Werror=cast-qual */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	ifp = container_of(work, dhd_if_t, blk_tsfl_work);
+	GCC_DIAGNOSTIC_POP();
+
+	if (ifp) {
+		dhdp = &ifp->info->pub;
+		if (dhdp) {
+			if ((dhdp->op_mode & DHD_FLAG_P2P_GO_MODE)||
+				(dhdp->op_mode & DHD_FLAG_HOSTAP_MODE)) {
+				DHD_ERROR(("Disassoc due to TCP SYNC FLOOD ATTACK\n"));
+				wl_cfg80211_del_all_sta(ifp->net, WLAN_REASON_UNSPECIFIED);
+			} else if ((dhdp->op_mode & DHD_FLAG_P2P_GC_MODE)||
+				(dhdp->op_mode & DHD_FLAG_STA_MODE)) {
+				DHD_ERROR(("Diconnect due to TCP SYNC FLOOD ATTACK\n"));
+				wl_cfg80211_disassoc(ifp->net, WLAN_REASON_UNSPECIFIED);
+			}
+			ifp->disconnect_tsync_flood = TRUE;
+		}
+	}
+}
+void dhd_reset_tcpsync_info_by_ifp(dhd_if_t *ifp)
+{
+	ifp->tsync_rcvd = 0;
+	ifp->tsyncack_txed = 0;
+	ifp->last_sync = DIV_U64_BY_U32(OSL_LOCALTIME_NS(), NSEC_PER_SEC);
+}
+void dhd_reset_tcpsync_info_by_dev(struct net_device *dev)
+{
+	dhd_if_t *ifp = NULL;
+	if (dev) {
+		ifp = DHD_DEV_IFP(dev);
+	}
+	if (ifp) {
+		ifp->tsync_rcvd = 0;
+		ifp->tsyncack_txed = 0;
+		ifp->last_sync = DIV_U64_BY_U32(OSL_LOCALTIME_NS(), NSEC_PER_SEC);
+		ifp->tsync_per_sec = 0;
+		ifp->disconnect_tsync_flood = FALSE;
+	}
+}
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+static void dhd_m4_state_handler(struct work_struct *work)
+{
+	dhd_if_t *ifp = NULL;
+	/* Ignore compiler warnings due to -Werror=cast-qual */
+	struct delayed_work *dw = to_delayed_work(work);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	ifp = container_of(dw, dhd_if_t, m4state_work);
+	GCC_DIAGNOSTIC_POP();
+
+	if (ifp && ifp->net &&
+		(OSL_ATOMIC_READ(ifp->info->pub->osh, &ifp->m4state) == M4_TXFAILED)) {
+		DHD_ERROR(("Disassoc for 4WAY_HANDSHAKE_TIMEOUT at %s\n",
+				ifp->net->name));
+		wl_cfg80211_disassoc(ifp->net, WLAN_REASON_4WAY_HANDSHAKE_TIMEOUT);
+	}
+}
+
+void
+dhd_cleanup_m4_state_work(dhd_pub_t *dhdp, int ifidx)
+{
+	dhd_info_t *dhdinfo;
+	dhd_if_t *ifp;
+
+	if ((ifidx < 0) || (ifidx >= DHD_MAX_IFS)) {
+		DHD_ERROR(("%s: invalid ifidx %d\n", __FUNCTION__, ifidx));
+		return;
+	}
+
+	dhdinfo = (dhd_info_t *)(dhdp->info);
+	if (!dhdinfo) {
+		DHD_ERROR(("%s: dhdinfo is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	ifp = dhdinfo->iflist[ifidx];
+	if (ifp) {
+		cancel_delayed_work_sync(&ifp->m4state_work);
+	}
+}
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+/*
+ * DHD RING
+ */
+#define DHD_RING_ERR_INTERNAL(fmt, ...) DHD_ERROR(("EWPF-" fmt, ##__VA_ARGS__))
+#define DHD_RING_TRACE_INTERNAL(fmt, ...) DHD_INFO(("EWPF-" fmt, ##__VA_ARGS__))
+
+#define DHD_RING_ERR(x) DHD_RING_ERR_INTERNAL x
+#define DHD_RING_TRACE(x) DHD_RING_TRACE_INTERNAL x
+
+#define DHD_RING_MAGIC 0x20170910
+#define DHD_RING_IDX_INVALID	0xffffffff
+
+#define DHD_RING_SYNC_LOCK_INIT(osh)		osl_spin_lock_init(osh)
+#define DHD_RING_SYNC_LOCK_DEINIT(osh, lock)	osl_spin_lock_deinit(osh, lock)
+#define DHD_RING_SYNC_LOCK(lock, flags)		(flags) = osl_spin_lock(lock)
+#define DHD_RING_SYNC_UNLOCK(lock, flags)	osl_spin_unlock(lock, flags)
+
+typedef struct {
+	uint32 elem_size;
+	uint32 elem_cnt;
+	uint32 write_idx;	/* next write index, -1 : not started */
+	uint32 read_idx;	/* next read index, -1 : not start */
+
+	/* protected elements during serialization */
+	int lock_idx;	/* start index of locked, element will not be overried */
+	int lock_count; /* number of locked, from lock idx */
+
+	/* saved data elements */
+	void *elem;
+} dhd_fixed_ring_info_t;
+
+typedef struct {
+	uint32 elem_size;
+	uint32 elem_cnt;
+	uint32 idx;		/* -1 : not started */
+	uint32 rsvd;		/* reserved for future use */
+
+	/* protected elements during serialization */
+	atomic_t ring_locked;
+	/* check the overwriting */
+	uint32 ring_overwrited;
+
+	/* saved data elements */
+	void *elem;
+} dhd_singleidx_ring_info_t;
+
+typedef struct {
+	uint32 magic;
+	uint32 type;
+	void *ring_sync; /* spinlock for sync */
+	union {
+		dhd_fixed_ring_info_t fixed;
+		dhd_singleidx_ring_info_t single;
+	};
+} dhd_ring_info_t;
+
+uint32
+dhd_ring_get_hdr_size(void)
+{
+	return sizeof(dhd_ring_info_t);
+}
+
+void *
+dhd_ring_init(dhd_pub_t *dhdp, uint8 *buf, uint32 buf_size, uint32 elem_size,
+	uint32 elem_cnt, uint32 type)
+{
+	dhd_ring_info_t *ret_ring;
+
+	if (!buf) {
+		DHD_RING_ERR(("NO RING BUFFER\n"));
+		return NULL;
+	}
+
+	if (buf_size < dhd_ring_get_hdr_size() + elem_size * elem_cnt) {
+		DHD_RING_ERR(("RING SIZE IS TOO SMALL\n"));
+		return NULL;
+	}
+
+	if (type != DHD_RING_TYPE_FIXED && type != DHD_RING_TYPE_SINGLE_IDX) {
+		DHD_RING_ERR(("UNSUPPORTED RING TYPE\n"));
+		return NULL;
+	}
+
+	ret_ring = (dhd_ring_info_t *)buf;
+	ret_ring->type = type;
+	ret_ring->ring_sync = (void *)DHD_RING_SYNC_LOCK_INIT(dhdp->osh);
+	ret_ring->magic = DHD_RING_MAGIC;
+
+	if (type == DHD_RING_TYPE_FIXED) {
+		ret_ring->fixed.read_idx = DHD_RING_IDX_INVALID;
+		ret_ring->fixed.write_idx = DHD_RING_IDX_INVALID;
+		ret_ring->fixed.lock_idx = DHD_RING_IDX_INVALID;
+		ret_ring->fixed.elem = buf + sizeof(dhd_ring_info_t);
+		ret_ring->fixed.elem_size = elem_size;
+		ret_ring->fixed.elem_cnt = elem_cnt;
+	} else {
+		ret_ring->single.idx = DHD_RING_IDX_INVALID;
+		atomic_set(&ret_ring->single.ring_locked, 0);
+		ret_ring->single.ring_overwrited = 0;
+		ret_ring->single.rsvd = 0;
+		ret_ring->single.elem = buf + sizeof(dhd_ring_info_t);
+		ret_ring->single.elem_size = elem_size;
+		ret_ring->single.elem_cnt = elem_cnt;
+	}
+
+	return ret_ring;
+}
+
+void
+dhd_ring_deinit(dhd_pub_t *dhdp, void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	if (!ring) {
+		return;
+	}
+
+	if (ring->magic != DHD_RING_MAGIC) {
+		return;
+	}
+
+	if (ring->type != DHD_RING_TYPE_FIXED &&
+		ring->type != DHD_RING_TYPE_SINGLE_IDX) {
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK_DEINIT(dhdp->osh, ring->ring_sync);
+	ring->ring_sync = NULL;
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		dhd_fixed_ring_info_t *fixed = &ring->fixed;
+		memset(fixed->elem, 0, fixed->elem_size * fixed->elem_cnt);
+		fixed->elem_size = fixed->elem_cnt = 0;
+	} else {
+		dhd_singleidx_ring_info_t *single = &ring->single;
+		memset(single->elem, 0, single->elem_size * single->elem_cnt);
+		single->elem_size = single->elem_cnt = 0;
+	}
+	ring->type = 0;
+	ring->magic = 0;
+}
+
+static inline uint32
+__dhd_ring_ptr2idx(void *ring, void *ptr, char *sig, uint32 type)
+{
+	uint32 diff;
+	uint32 ret_idx = (uint32)DHD_RING_IDX_INVALID;
+	uint32 elem_size, elem_cnt;
+	void *elem;
+
+	if (type == DHD_RING_TYPE_FIXED) {
+		dhd_fixed_ring_info_t *fixed = (dhd_fixed_ring_info_t *)ring;
+		elem_size = fixed->elem_size;
+		elem_cnt = fixed->elem_cnt;
+		elem = fixed->elem;
+	} else if (type == DHD_RING_TYPE_SINGLE_IDX) {
+		dhd_singleidx_ring_info_t *single = (dhd_singleidx_ring_info_t *)ring;
+		elem_size = single->elem_size;
+		elem_cnt = single->elem_cnt;
+		elem = single->elem;
+	} else {
+		DHD_RING_ERR(("UNSUPPORTED RING TYPE %d\n", type));
+		return ret_idx;
+	}
+
+	if (ptr < elem) {
+		DHD_RING_ERR(("INVALID POINTER %s:%p, ring->elem:%p\n", sig, ptr, elem));
+		return ret_idx;
+	}
+	diff = (uint32)((uint8 *)ptr - (uint8 *)elem);
+	if (diff % elem_size != 0) {
+		DHD_RING_ERR(("INVALID POINTER %s:%p, ring->elem:%p\n", sig, ptr, elem));
+		return ret_idx;
+	}
+	ret_idx = diff / elem_size;
+	if (ret_idx >= elem_cnt) {
+		DHD_RING_ERR(("INVALID POINTER max:%d cur:%d\n", elem_cnt, ret_idx));
+	}
+	return ret_idx;
+}
+
+/* Sub functions for fixed ring */
+/* get counts between two indexes of ring buffer (internal only) */
+static inline int
+__dhd_fixed_ring_get_count(dhd_fixed_ring_info_t *ring, int start, int end)
+{
+	if (start == DHD_RING_IDX_INVALID || end == DHD_RING_IDX_INVALID) {
+		return 0;
+	}
+
+	return (ring->elem_cnt + end - start) % ring->elem_cnt + 1;
+}
+
+static inline int
+__dhd_fixed_ring_get_cur_size(dhd_fixed_ring_info_t *ring)
+{
+	return __dhd_fixed_ring_get_count(ring, ring->read_idx, ring->write_idx);
+}
+
+static inline void *
+__dhd_fixed_ring_get_first(dhd_fixed_ring_info_t *ring)
+{
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		return NULL;
+	}
+	return (uint8 *)ring->elem + (ring->elem_size * ring->read_idx);
+}
+
+static inline void
+__dhd_fixed_ring_free_first(dhd_fixed_ring_info_t *ring)
+{
+	uint32 next_idx;
+
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return;
+	}
+
+	next_idx = (ring->read_idx + 1) % ring->elem_cnt;
+	if (ring->read_idx == ring->write_idx) {
+		/* Become empty */
+		ring->read_idx = ring->write_idx = DHD_RING_IDX_INVALID;
+		return;
+	}
+
+	ring->read_idx = next_idx;
+	return;
+}
+
+static inline void *
+__dhd_fixed_ring_get_last(dhd_fixed_ring_info_t *ring)
+{
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		return NULL;
+	}
+	return (uint8 *)ring->elem + (ring->elem_size * ring->write_idx);
+}
+
+static inline void *
+__dhd_fixed_ring_get_empty(dhd_fixed_ring_info_t *ring)
+{
+	uint32 tmp_idx;
+
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		ring->read_idx = ring->write_idx = 0;
+		return (uint8 *)ring->elem;
+	}
+
+	/* check next index is not locked */
+	tmp_idx = (ring->write_idx + 1) % ring->elem_cnt;
+	if (ring->lock_idx == tmp_idx) {
+		return NULL;
+	}
+
+	ring->write_idx = tmp_idx;
+	if (ring->write_idx == ring->read_idx) {
+		/* record is full, drop oldest one */
+		ring->read_idx = (ring->read_idx + 1) % ring->elem_cnt;
+
+	}
+	return (uint8 *)ring->elem + (ring->elem_size * ring->write_idx);
+}
+
+static inline void *
+__dhd_fixed_ring_get_next(dhd_fixed_ring_info_t *ring, void *prev, uint32 type)
+{
+	uint32 cur_idx;
+
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return NULL;
+	}
+
+	cur_idx = __dhd_ring_ptr2idx(ring, prev, "NEXT", type);
+	if (cur_idx >= ring->elem_cnt) {
+		return NULL;
+	}
+
+	if (cur_idx == ring->write_idx) {
+		/* no more new record */
+		return NULL;
+	}
+
+	cur_idx = (cur_idx + 1) % ring->elem_cnt;
+	return (uint8 *)ring->elem + ring->elem_size * cur_idx;
+}
+
+static inline void *
+__dhd_fixed_ring_get_prev(dhd_fixed_ring_info_t *ring, void *prev, uint32 type)
+{
+	uint32 cur_idx;
+
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return NULL;
+	}
+	cur_idx = __dhd_ring_ptr2idx(ring, prev, "PREV", type);
+	if (cur_idx >= ring->elem_cnt) {
+		return NULL;
+	}
+	if (cur_idx == ring->read_idx) {
+		/* no more new record */
+		return NULL;
+	}
+
+	cur_idx = (cur_idx + ring->elem_cnt - 1) % ring->elem_cnt;
+	return (uint8 *)ring->elem + ring->elem_size * cur_idx;
+}
+
+static inline void
+__dhd_fixed_ring_lock(dhd_fixed_ring_info_t *ring, void *first_ptr, void *last_ptr, uint32 type)
+{
+	uint32 first_idx;
+	uint32 last_idx;
+	uint32 ring_filled_cnt;
+	uint32 tmp_cnt;
+
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return;
+	}
+
+	if (first_ptr) {
+		first_idx = __dhd_ring_ptr2idx(ring, first_ptr, "LCK FIRST", type);
+		if (first_idx >= ring->elem_cnt) {
+			return;
+		}
+	} else {
+		first_idx = ring->read_idx;
+	}
+
+	if (last_ptr) {
+		last_idx = __dhd_ring_ptr2idx(ring, last_ptr, "LCK LAST", type);
+		if (last_idx >= ring->elem_cnt) {
+			return;
+		}
+	} else {
+		last_idx = ring->write_idx;
+	}
+
+	ring_filled_cnt = __dhd_fixed_ring_get_count(ring, ring->read_idx, ring->write_idx);
+	tmp_cnt = __dhd_fixed_ring_get_count(ring, ring->read_idx, first_idx);
+	if (tmp_cnt > ring_filled_cnt) {
+		DHD_RING_ERR(("LOCK FIRST IS TO EMPTY ELEM: write: %d read: %d cur:%d\n",
+			ring->write_idx, ring->read_idx, first_idx));
+		return;
+	}
+
+	tmp_cnt = __dhd_fixed_ring_get_count(ring, ring->read_idx, last_idx);
+	if (tmp_cnt > ring_filled_cnt) {
+		DHD_RING_ERR(("LOCK LAST IS TO EMPTY ELEM: write: %d read: %d cur:%d\n",
+			ring->write_idx, ring->read_idx, last_idx));
+		return;
+	}
+
+	ring->lock_idx = first_idx;
+	ring->lock_count = __dhd_fixed_ring_get_count(ring, first_idx, last_idx);
+	return;
+}
+
+static inline void
+__dhd_fixed_ring_lock_free(dhd_fixed_ring_info_t *ring)
+{
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return;
+	}
+
+	ring->lock_idx = DHD_RING_IDX_INVALID;
+	ring->lock_count = 0;
+	return;
+}
+static inline void *
+__dhd_fixed_ring_lock_get_first(dhd_fixed_ring_info_t *ring)
+{
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return NULL;
+	}
+	if (ring->lock_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("NO LOCK POINT\n"));
+		return NULL;
+	}
+	return (uint8 *)ring->elem + ring->elem_size * ring->lock_idx;
+}
+
+static inline void *
+__dhd_fixed_ring_lock_get_last(dhd_fixed_ring_info_t *ring)
+{
+	int lock_last_idx;
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return NULL;
+	}
+	if (ring->lock_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("NO LOCK POINT\n"));
+		return NULL;
+	}
+
+	lock_last_idx = (ring->lock_idx + ring->lock_count - 1) % ring->elem_cnt;
+	return (uint8 *)ring->elem + ring->elem_size * lock_last_idx;
+}
+
+static inline int
+__dhd_fixed_ring_lock_get_count(dhd_fixed_ring_info_t *ring)
+{
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return BCME_ERROR;
+	}
+	if (ring->lock_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("NO LOCK POINT\n"));
+		return BCME_ERROR;
+	}
+	return ring->lock_count;
+}
+
+static inline void
+__dhd_fixed_ring_lock_free_first(dhd_fixed_ring_info_t *ring)
+{
+	if (ring->read_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return;
+	}
+	if (ring->lock_idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("NO LOCK POINT\n"));
+		return;
+	}
+
+	ring->lock_count--;
+	if (ring->lock_count <= 0) {
+		ring->lock_idx = DHD_RING_IDX_INVALID;
+	} else {
+		ring->lock_idx = (ring->lock_idx + 1) % ring->elem_cnt;
+	}
+	return;
+}
+
+static inline void
+__dhd_fixed_ring_set_read_idx(dhd_fixed_ring_info_t *ring, uint32 idx)
+{
+	ring->read_idx = idx;
+}
+
+static inline void
+__dhd_fixed_ring_set_write_idx(dhd_fixed_ring_info_t *ring, uint32 idx)
+{
+	ring->write_idx = idx;
+}
+
+static inline uint32
+__dhd_fixed_ring_get_read_idx(dhd_fixed_ring_info_t *ring)
+{
+	return ring->read_idx;
+}
+
+static inline uint32
+__dhd_fixed_ring_get_write_idx(dhd_fixed_ring_info_t *ring)
+{
+	return ring->write_idx;
+}
+
+/* Sub functions for single index ring */
+static inline void *
+__dhd_singleidx_ring_get_first(dhd_singleidx_ring_info_t *ring)
+{
+	uint32 tmp_idx = 0;
+
+	if (ring->idx == DHD_RING_IDX_INVALID) {
+		return NULL;
+	}
+
+	if (ring->ring_overwrited) {
+		tmp_idx = (ring->idx + 1) % ring->elem_cnt;
+	}
+
+	return (uint8 *)ring->elem + (ring->elem_size * tmp_idx);
+}
+
+static inline void *
+__dhd_singleidx_ring_get_last(dhd_singleidx_ring_info_t *ring)
+{
+	if (ring->idx == DHD_RING_IDX_INVALID) {
+		return NULL;
+	}
+
+	return (uint8 *)ring->elem + (ring->elem_size * ring->idx);
+}
+
+static inline void *
+__dhd_singleidx_ring_get_empty(dhd_singleidx_ring_info_t *ring)
+{
+	if (ring->idx == DHD_RING_IDX_INVALID) {
+		ring->idx = 0;
+		return (uint8 *)ring->elem;
+	}
+
+	/* check the lock is held */
+	if (atomic_read(&ring->ring_locked)) {
+		return NULL;
+	}
+
+	/* check the index rollover */
+	if (!ring->ring_overwrited && ring->idx == (ring->elem_cnt - 1)) {
+		ring->ring_overwrited = 1;
+	}
+
+	ring->idx = (ring->idx + 1) % ring->elem_cnt;
+
+	return (uint8 *)ring->elem + (ring->elem_size * ring->idx);
+}
+
+static inline void *
+__dhd_singleidx_ring_get_next(dhd_singleidx_ring_info_t *ring, void *prev, uint32 type)
+{
+	uint32 cur_idx;
+
+	if (ring->idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return NULL;
+	}
+
+	cur_idx = __dhd_ring_ptr2idx(ring, prev, "NEXT", type);
+	if (cur_idx >= ring->elem_cnt) {
+		return NULL;
+	}
+
+	if (cur_idx == ring->idx) {
+		/* no more new record */
+		return NULL;
+	}
+
+	cur_idx = (cur_idx + 1) % ring->elem_cnt;
+
+	return (uint8 *)ring->elem + ring->elem_size * cur_idx;
+}
+
+static inline void *
+__dhd_singleidx_ring_get_prev(dhd_singleidx_ring_info_t *ring, void *prev, uint32 type)
+{
+	uint32 cur_idx;
+
+	if (ring->idx == DHD_RING_IDX_INVALID) {
+		DHD_RING_ERR(("EMPTY RING\n"));
+		return NULL;
+	}
+	cur_idx = __dhd_ring_ptr2idx(ring, prev, "PREV", type);
+	if (cur_idx >= ring->elem_cnt) {
+		return NULL;
+	}
+
+	if (!ring->ring_overwrited && cur_idx == 0) {
+		/* no more new record */
+		return NULL;
+	}
+
+	cur_idx = (cur_idx + ring->elem_cnt - 1) % ring->elem_cnt;
+	if (ring->ring_overwrited && cur_idx == ring->idx) {
+		/* no more new record */
+		return NULL;
+	}
+
+	return (uint8 *)ring->elem + ring->elem_size * cur_idx;
+}
+
+static inline void
+__dhd_singleidx_ring_whole_lock(dhd_singleidx_ring_info_t *ring)
+{
+	if (!atomic_read(&ring->ring_locked)) {
+		atomic_set(&ring->ring_locked, 1);
+	}
+}
+
+static inline void
+__dhd_singleidx_ring_whole_unlock(dhd_singleidx_ring_info_t *ring)
+{
+	if (atomic_read(&ring->ring_locked)) {
+		atomic_set(&ring->ring_locked, 0);
+	}
+}
+
+/* Get first element : oldest element */
+void *
+dhd_ring_get_first(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_get_first(&ring->fixed);
+	}
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		ret = __dhd_singleidx_ring_get_first(&ring->single);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+/* Free first element : oldest element */
+void
+dhd_ring_free_first(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		__dhd_fixed_ring_free_first(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+void
+dhd_ring_set_read_idx(void *_ring, uint32 read_idx)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		__dhd_fixed_ring_set_read_idx(&ring->fixed, read_idx);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+void
+dhd_ring_set_write_idx(void *_ring, uint32 write_idx)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		__dhd_fixed_ring_set_write_idx(&ring->fixed, write_idx);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+uint32
+dhd_ring_get_read_idx(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	uint32 read_idx = DHD_RING_IDX_INVALID;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return read_idx;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		read_idx = __dhd_fixed_ring_get_read_idx(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+
+	return read_idx;
+}
+
+uint32
+dhd_ring_get_write_idx(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	uint32 write_idx = DHD_RING_IDX_INVALID;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return write_idx;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		write_idx = __dhd_fixed_ring_get_write_idx(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+
+	return write_idx;
+}
+
+/* Get latest element */
+void *
+dhd_ring_get_last(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_get_last(&ring->fixed);
+	}
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		ret = __dhd_singleidx_ring_get_last(&ring->single);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+/* Get next point can be written
+ * will overwrite which doesn't read
+ * will return NULL if next pointer is locked.
+ */
+void *
+dhd_ring_get_empty(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_get_empty(&ring->fixed);
+	}
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		ret = __dhd_singleidx_ring_get_empty(&ring->single);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+void *
+dhd_ring_get_next(void *_ring, void *cur)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_get_next(&ring->fixed, cur, ring->type);
+	}
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		ret = __dhd_singleidx_ring_get_next(&ring->single, cur, ring->type);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+void *
+dhd_ring_get_prev(void *_ring, void *cur)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_get_prev(&ring->fixed, cur, ring->type);
+	}
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		ret = __dhd_singleidx_ring_get_prev(&ring->single, cur, ring->type);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+int
+dhd_ring_get_cur_size(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	int cnt = 0;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return cnt;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		cnt = __dhd_fixed_ring_get_cur_size(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return cnt;
+}
+
+/* protect element between lock_ptr and write_idx */
+void
+dhd_ring_lock(void *_ring, void *first_ptr, void *last_ptr)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		__dhd_fixed_ring_lock(&ring->fixed, first_ptr, last_ptr, ring->type);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+/* free all lock */
+void
+dhd_ring_lock_free(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		__dhd_fixed_ring_lock_free(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+void *
+dhd_ring_lock_get_first(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_lock_get_first(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+void *
+dhd_ring_lock_get_last(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	void *ret = NULL;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return NULL;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_lock_get_last(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+int
+dhd_ring_lock_get_count(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	int ret = BCME_ERROR;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return ret;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		ret = __dhd_fixed_ring_lock_get_count(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+	return ret;
+}
+
+/* free first locked element */
+void
+dhd_ring_lock_free_first(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_FIXED) {
+		__dhd_fixed_ring_lock_free_first(&ring->fixed);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+void
+dhd_ring_whole_lock(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		__dhd_singleidx_ring_whole_lock(&ring->single);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+
+void
+dhd_ring_whole_unlock(void *_ring)
+{
+	dhd_ring_info_t *ring = (dhd_ring_info_t *)_ring;
+	unsigned long flags;
+
+	if (!ring || ring->magic != DHD_RING_MAGIC) {
+		DHD_RING_ERR(("%s :INVALID RING INFO\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_RING_SYNC_LOCK(ring->ring_sync, flags);
+	if (ring->type == DHD_RING_TYPE_SINGLE_IDX) {
+		__dhd_singleidx_ring_whole_unlock(&ring->single);
+	}
+	DHD_RING_SYNC_UNLOCK(ring->ring_sync, flags);
+}
+/* END of DHD RING */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))
+#define DHD_VFS_INODE(dir) (dir->d_inode)
+#else
+#define DHD_VFS_INODE(dir) d_inode(dir)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) */
+
+#if ((defined DHD_DUMP_MNGR) || (defined DNGL_AXI_ERROR_LOGGING))
+int
+dhd_file_delete(char *path)
+{
+	struct path file_path = {.dentry = 0};
+	int err;
+	struct dentry *dir;
+
+	err = dhd_kern_path(path, 0, &file_path);
+
+	if (err < 0) {
+		DHD_ERROR(("Failed to get kern-path delete file: %s error: %d\n", path, err));
+		return err;
+	}
+	if (
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
+		!d_is_file(file_path.dentry) ||
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 1, 0))
+		d_really_is_negative(file_path.dentry) ||
+#endif /* LINUX_VERSION_CODE > KERNEL_VERSION(4, 1, 0) */
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) */
+		FALSE)
+	{
+		err = -EINVAL;
+	} else {
+		dir = dget_parent(file_path.dentry);
+
+		if (!IS_ERR(dir)) {
+			err = DHD_VFS_UNLINK(dir, file_path, NULL);
+			dput(dir);
+		} else {
+			err = PTR_ERR(dir);
+		}
+	}
+
+	path_put(&file_path);
+
+	if (err < 0) {
+		DHD_ERROR(("Failed to delete file: %s error: %d\n", path, err));
+	}
+
+	return err;
+}
+#endif
+
+#ifdef DHD_DUMP_MNGR
+static int
+dhd_dump_file_manage_idx(dhd_dump_file_manage_t *fm_ptr, char *fname)
+{
+	int i;
+	int fm_idx = -1;
+
+	for (i = 0; i < DHD_DUMP_TYPE_COUNT_MAX; i++) {
+		/* XXX dump file manager enqueues the type name to empty slot,
+		 * so it's impossible that empty slot is in the middle.
+		 */
+		if (strlen(fm_ptr->elems[i].type_name) == 0) {
+			fm_idx = i;
+			break;
+		}
+		if (!(strncmp(fname, fm_ptr->elems[i].type_name, strlen(fname)))) {
+			fm_idx = i;
+			break;
+		}
+	}
+
+	if (fm_idx == -1) {
+		return fm_idx;
+	}
+
+	if (strlen(fm_ptr->elems[fm_idx].type_name) == 0) {
+		strncpy(fm_ptr->elems[fm_idx].type_name, fname, DHD_DUMP_TYPE_NAME_SIZE);
+		fm_ptr->elems[fm_idx].type_name[DHD_DUMP_TYPE_NAME_SIZE - 1] = '\0';
+		fm_ptr->elems[fm_idx].file_idx = 0;
+	}
+
+	return fm_idx;
+}
+
+/*
+ * dhd_dump_file_manage_enqueue - enqueue dump file path
+ * and delete odest file if file count is max.
+ */
+void
+dhd_dump_file_manage_enqueue(dhd_pub_t *dhd, char *dump_path, char *fname)
+{
+	int fm_idx;
+	int fp_idx;
+	dhd_dump_file_manage_t *fm_ptr;
+	DFM_elem_t *elem;
+
+	if (!dhd || !dhd->dump_file_manage) {
+		DHD_ERROR(("%s(): dhdp=%p dump_file_manage=%p\n",
+			__FUNCTION__, dhd, (dhd ? dhd->dump_file_manage : NULL)));
+		return;
+	}
+
+	fm_ptr = dhd->dump_file_manage;
+
+	/* find file_manage idx */
+	DHD_INFO(("%s(): fname: %s dump_path: %s\n", __FUNCTION__, fname, dump_path));
+	if ((fm_idx = dhd_dump_file_manage_idx(fm_ptr, fname)) < 0) {
+		DHD_ERROR(("%s(): Out of file manager entries, fname: %s\n",
+			__FUNCTION__, fname));
+		return;
+	}
+
+	elem = &fm_ptr->elems[fm_idx];
+	fp_idx = elem->file_idx;
+	DHD_INFO(("%s(): fm_idx: %d fp_idx: %d path: %s\n",
+		__FUNCTION__, fm_idx, fp_idx, elem->file_path[fp_idx]));
+
+	/* delete oldest file */
+	if (strlen(elem->file_path[fp_idx]) != 0) {
+		if (dhd_file_delete(elem->file_path[fp_idx]) < 0) {
+			DHD_ERROR(("%s(): Failed to delete file: %s\n",
+				__FUNCTION__, elem->file_path[fp_idx]));
+		} else {
+			DHD_ERROR(("%s(): Successed to delete file: %s\n",
+				__FUNCTION__, elem->file_path[fp_idx]));
+		}
+	}
+
+	/* save dump file path */
+	strncpy(elem->file_path[fp_idx], dump_path, DHD_DUMP_FILE_PATH_SIZE);
+	elem->file_path[fp_idx][DHD_DUMP_FILE_PATH_SIZE - 1] = '\0';
+
+	/* change file index to next file index */
+	elem->file_idx = (elem->file_idx + 1) % DHD_DUMP_FILE_COUNT_MAX;
+}
+#endif /* DHD_DUMP_MNGR */
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+static void
+dhd_axi_error_dump(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = (dhd_info_t *)handle;
+	dhd_pub_t *dhdp = NULL;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		goto exit;
+	}
+
+	dhdp = &dhd->pub;
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		goto exit;
+	}
+
+	/**
+	 * First save axi error information to a file
+	 * because panic should happen right after this.
+	 * After dhd reset, dhd reads the file, and do hang event process
+	 * to send axi error stored on the file to Bigdata server
+	 */
+	if (dhdp->axi_err_dump->etd_axi_error_v1.version != HND_EXT_TRAP_AXIERROR_VERSION_1) {
+		DHD_ERROR(("%s: Invalid AXI version: 0x%x\n",
+			__FUNCTION__, dhdp->axi_err_dump->etd_axi_error_v1.version));
+	}
+
+	DHD_OS_WAKE_LOCK(dhdp);
+#ifdef DHD_FW_COREDUMP
+#ifdef DHD_SSSR_DUMP
+	DHD_ERROR(("%s : Set collect_sssr as TRUE\n", __FUNCTION__));
+	dhdp->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+	DHD_ERROR(("%s: scheduling mem dump.. \n", __FUNCTION__));
+	dhd_schedule_memdump(dhdp, dhdp->soc_ram, dhdp->soc_ram_length);
+#endif /* DHD_FW_COREDUMP */
+	DHD_OS_WAKE_UNLOCK(dhdp);
+
+exit:
+	/* Trigger kernel panic after taking necessary dumps */
+	BUG_ON(1);
+}
+
+void dhd_schedule_axi_error_dump(dhd_pub_t *dhdp, void *type)
+{
+	DHD_ERROR(("%s: scheduling axi_error_dump.. \n", __FUNCTION__));
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq,
+		type, DHD_WQ_WORK_AXI_ERROR_DUMP,
+		dhd_axi_error_dump, DHD_WQ_WORK_PRIORITY_HIGH);
+}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#ifdef SUPPORT_SET_TID
+/*
+ * Set custom TID value for UDP frame based on UID value.
+ * This will be triggered by android private command below.
+ * DRIVER SET_TID <Mode:uint8> <Target UID:uint32> <Custom TID:uint8>
+ * Mode 0(SET_TID_OFF) : Disable changing TID
+ * Mode 1(SET_TID_ALL_UDP) : Change TID for all UDP frames
+ * Mode 2(SET_TID_BASED_ON_UID) : Change TID for UDP frames based on target UID
+*/
+void
+dhd_set_tid_based_on_uid(dhd_pub_t *dhdp, void *pkt)
+{
+	struct ether_header *eh = NULL;
+	struct sock *sk = NULL;
+	uint8 *pktdata = NULL;
+	uint8 *ip_hdr = NULL;
+	uint8 cur_prio;
+	uint8 prio;
+	uint32 uid;
+
+	if (dhdp->tid_mode == SET_TID_OFF) {
+		return;
+	}
+
+	pktdata = (uint8 *)PKTDATA(dhdp->osh, pkt);
+	eh = (struct ether_header *) pktdata;
+	ip_hdr = (uint8 *)eh + ETHER_HDR_LEN;
+
+	if (IPV4_PROT(ip_hdr) != IP_PROT_UDP) {
+		return;
+	}
+
+	cur_prio = PKTPRIO(pkt);
+	prio = dhdp->target_tid;
+	uid = dhdp->target_uid;
+
+	if ((cur_prio == prio) ||
+		(cur_prio != PRIO_8021D_BE)) {
+			return;
+	}
+
+	sk = ((struct sk_buff*)(pkt))->sk;
+
+	if ((dhdp->tid_mode == SET_TID_ALL_UDP) ||
+		(sk && (uid == __kuid_val(sock_i_uid(sk))))) {
+		PKTSETPRIO(pkt, prio);
+	}
+}
+#endif /* SUPPORT_SET_TID */
+
+#ifdef BCMPCIE
+static void
+dhd_cto_recovery_handler(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	dhd_pub_t *dhdp = NULL;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		BUG_ON(1);
+		return;
+	}
+
+	dhdp = &dhd->pub;
+	if (dhdp->dhd_induce_error == DHD_INDUCE_BH_CBP_HANG) {
+		DHD_ERROR(("%s: skip cto recovery for DHD_INDUCE_BH_CBP_HANG\n",
+			__FUNCTION__));
+		return;
+	}
+	dhdpcie_cto_recovery_handler(dhdp);
+}
+
+void
+dhd_schedule_cto_recovery(dhd_pub_t *dhdp)
+{
+	if (dhdp->up == FALSE) {
+		DHD_ERROR(("%s : skip scheduling cto because dhd is not up\n",
+				__FUNCTION__));
+		return;
+	}
+
+	if (dhdp->info->scheduled_memdump) {
+		DHD_ERROR(("%s, memdump in progress\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("%s: scheduling cto recovery.. \n", __FUNCTION__));
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq,
+		NULL, DHD_WQ_WORK_CTO_RECOVERY,
+		dhd_cto_recovery_handler, DHD_WQ_WORK_PRIORITY_HIGH);
+}
+#endif /* BCMPCIE */
+
+#ifdef DHD_WIFI_SHUTDOWN
+void wifi_plat_dev_drv_shutdown(struct platform_device *pdev)
+{
+	dhd_pub_t *dhd_pub = NULL;
+	dhd_info_t *dhd_info = NULL;
+	dhd_if_t *dhd_if = NULL;
+
+	DHD_ERROR(("%s enter\n", __FUNCTION__));
+	dhd_pub = g_dhd_pub;
+
+	if (dhd_os_check_if_up(dhd_pub)) {
+		dhd_info = (dhd_info_t *)dhd_pub->info;
+		dhd_if = dhd_info->iflist[0];
+		ASSERT(dhd_if);
+		ASSERT(dhd_if->net);
+		if (dhd_if && dhd_if->net) {
+			dhd_stop(dhd_if->net);
+		}
+	}
+}
+#endif /* DHD_WIFI_SHUTDOWN */
+#if defined(WL_AUTO_QOS) && defined(DHD_QOS_ON_SOCK_FLOW)
+void
+dhd_wl_sock_qos_set_status(dhd_pub_t *dhdp, unsigned long on_off)
+{
+	dhd_sock_qos_set_status(dhdp->info, on_off);
+}
+#endif /* WL_AUTO_QOS && DHD_QOS_ON_SOCK_FLOW */
+
+#ifdef DHD_CFG80211_SUSPEND_RESUME
+void
+dhd_cfg80211_suspend(dhd_pub_t *dhdp)
+{
+	struct net_device *net = dhd_idx2net((dhd_pub_t *)dhdp, 0);
+	struct bcm_cfg80211 *cfg = wl_get_cfg(net);
+	wl_cfg80211_suspend(cfg);
+}
+
+void
+dhd_cfg80211_resume(dhd_pub_t *dhdp)
+{
+	struct net_device * net = dhd_idx2net((dhd_pub_t *)dhdp, 0);
+	struct bcm_cfg80211 *cfg = wl_get_cfg(net);
+	wl_cfg80211_resume(cfg);
+}
+#endif /* DHD_CFG80211_SUSPEND_RESUME */
+
+void
+dhd_generate_rand_mac_addr(struct ether_addr *ea_addr)
+{
+	RANDOM_BYTES(ea_addr->octet, ETHER_ADDR_LEN);
+	/* restore mcast and local admin bits to 0 and 1 */
+	ETHER_SET_UNICAST(ea_addr->octet);
+	ETHER_SET_LOCALADDR(ea_addr->octet);
+	DHD_ERROR(("%s:generated new MAC="MACDBG" \n",
+		__FUNCTION__, MAC2STRDBG(ea_addr->octet)));
+	return;
+}
+
+void *
+dhd_get_roam_evt(dhd_pub_t *dhdp)
+{
+#if defined(DHD_PUB_ROAM_EVT)
+	return (void *)&(dhdp->roam_evt);
+#else
+	return NULL;
+#endif /* DHD_PUB_ROAM_EVT */
+}
+
+/* BANDLOCK_FILE is for Hikey only and BANDLOCK has a priority than BANDLOCK_FILE */
+static void dhd_set_bandlock(dhd_pub_t * dhd)
+{
+#if defined(BANDLOCK)
+	int band = BANDLOCK;
+	if (dhd_wl_ioctl_cmd(dhd, WLC_SET_BAND, &band, sizeof(band), TRUE, 0) < 0) {
+		DHD_ERROR(("%s: set band(%d) error\n", __FUNCTION__, band));
+	}
+#elif defined(BANDLOCK_FILE)
+	int band;
+	char val[2] = {0, 0};
+	if (dhd_read_file(PATH_BANDLOCK_INFO, (char *)val, sizeof(char)) == BCME_OK) {
+		band = bcm_atoi(val);
+		if (dhd_wl_ioctl_cmd(dhd, WLC_SET_BAND, &band, sizeof(band), TRUE, 0) < 0) {
+			DHD_ERROR(("%s: set band(%d) error\n", __FUNCTION__, band));
+		}
+	}
+#endif /* BANDLOCK */
+}
+
+void
+dhd_set_del_in_progress(dhd_pub_t *dhdp, struct net_device *ndev)
+{
+	dhd_if_t *ifp = NULL;
+	unsigned long flags;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+	ifp = dhd_get_ifp_by_ndev(dhdp, ndev);
+	if (ifp == NULL) {
+		DHD_ERROR(("DHD Iface Info corresponding to %s not found\n", ndev->name));
+		return;
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	ifp->del_in_progress = TRUE;
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+}
+
+void
+dhd_clear_del_in_progress(dhd_pub_t *dhdp, struct net_device *ndev)
+{
+	dhd_if_t *ifp = NULL;
+	unsigned long flags;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+	ifp = dhd_get_ifp_by_ndev(dhdp, ndev);
+	if (ifp == NULL) {
+		DHD_ERROR(("DHD Iface Info corresponding to %s not found\n", ndev->name));
+		return;
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	ifp->del_in_progress = FALSE;
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+}
+
+#ifdef PCIE_FULL_DONGLE
+/* API to delete flowings and Stations
+* corresponding to the interface(ndev)
+*/
+void
+dhd_net_del_flowrings_sta(dhd_pub_t *dhd, struct net_device *ndev)
+{
+	dhd_if_t *ifp = NULL;
+
+	ifp = dhd_get_ifp_by_ndev(dhd, ndev);
+	if (ifp == NULL) {
+		DHD_ERROR(("DHD Iface Info corresponding to %s not found\n", ndev->name));
+		return;
+	}
+
+	/* For now called only in iface delete path..
+	* Add reason codes if this API need to be reused in any other paths.
+	*/
+	DHD_ERROR(("%s:Clean up IFACE idx %d due to interface delete\n",
+		__FUNCTION__, ifp->idx));
+
+	dhd_del_all_sta(dhd, ifp->idx);
+	/* Try to resume if already suspended or suspend in progress */
+	dhd_flow_rings_delete(dhd, ifp->idx);
+}
+#if defined(WBRC) && defined(WBRC_BT2WL_RESET)
+void
+dhd_wbrc_wl_trap(void)
+{
+	dhd_pub_t *dhdp = g_dhd_pub;
+
+	dhdpcie_db7_trap(dhdp->bus);
+	/* set that dongle trap has occured to stop other contests from firing IOVARS */
+	dhdp->dongle_trap_occured = TRUE;
+#ifdef OEM_ANDROID
+	/* Set chip big hammer */
+	dhdp->do_chip_bighammer = TRUE;
+	dhdp->hang_reason = HANG_REASON_BT2WL_REG_RESET;
+	dhd_os_send_hang_message(dhdp);
+#endif /* OEM_ANDROID */
+}
+#endif /* WBRC && WBRC_BT2WL_RESET */
+#endif /* PCIE_FULL_DONGLE */
+
+#ifndef BCMDBUS
+static void
+dhd_deferred_socram_dump(void *handle, void *event_info, u8 event)
+{
+#ifndef BCMDBUS
+	dhd_pub_t *dhdp = (dhd_pub_t *)event_info;
+	DHD_ERROR(("%s ... scheduled to collect memdump over bus\n", __FUNCTION__));
+	dhd_socram_dump(dhdp->bus);
+#else
+	DHD_ERROR(("%s Not collecting socram dump for BCMDBUS\n", __FUNCTION__));
+	return;
+#endif /* BCMDBUS */
+}
+
+int
+dhd_schedule_socram_dump(dhd_pub_t *dhdp)
+{
+	int ret = 0;
+	ret = dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq, (void *)dhdp,
+		DHD_WQ_WORK_SOC_RAM_COLLECT, dhd_deferred_socram_dump, DHD_WQ_WORK_PRIORITY_HIGH);
+	return ret;
+}
+#endif
+
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+void dhd_alert_process(struct work_struct *work_data)
+{
+	struct net_device *dev;
+	dhd_info_t *dhd_info = NULL;
+	dhd_pub_t *dhdp = NULL;
+
+	/* Ignore compiler warnings due to -Werror=cast-qual */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd_info = container_of(work_data, dhd_info_t, dhd_alert_process_work);
+	GCC_DIAGNOSTIC_POP();
+	dhdp = &dhd_info->pub;
+
+	if (!dhdp) {
+		DHD_ERROR(("dhd is NULL\n"));
+		return;
+	}
+	dev = dhd_linux_get_primary_netdev(dhdp);
+
+	if (dev) {
+		wl_cfg80211_alert(dev);
+	}
+}
+
+int dhd_os_send_alert_message(dhd_pub_t *dhdp)
+{
+	int ret = 0;
+	dhd_info_t *dhd_info = NULL;
+#ifdef WL_CFG80211
+	struct net_device *primary_ndev;
+	struct bcm_cfg80211 *cfg;
+#endif /* WL_CFG80211 */
+
+	if (!dhdp) {
+		DHD_ERROR(("dhdp is null\n"));
+		return -EINVAL;
+	}
+
+	dhd_info = (dhd_info_t *)dhdp->info;
+	BCM_REFERENCE(dhd_info);
+
+#ifdef WL_CFG80211
+	primary_ndev = dhd_linux_get_primary_netdev(dhdp);
+	if (!primary_ndev) {
+		DHD_ERROR(("Cannot find primary netdev\n"));
+		return -ENODEV;
+	}
+	cfg = wl_get_cfg(primary_ndev);
+	if (!cfg) {
+		DHD_ERROR(("Cannot find cfg\n"));
+		return -EINVAL;
+	}
+
+	/* Skip sending ALERT event to framework if driver is not ready */
+	if (!wl_get_drv_status(cfg, READY, primary_ndev)) {
+		DHD_ERROR(("device is not ready\n"));
+		return -ENODEV;
+	}
+
+	schedule_work(&dhdp->info->dhd_alert_process_work);
+#endif /* WL_CFG80211 */
+
+	return ret;
+}
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+int
+dhd_dev_set_accel_force_reg_on(struct net_device *dev)
+{
+#if defined(WLAN_ACCEL_BOOT)
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv(dev);
+
+	if (dhd_info && !dhd_info->wl_accel_force_reg_on) {
+		DHD_ERROR(("%s: set force reg on\n", __FUNCTION__));
+		dhd_info->wl_accel_force_reg_on = TRUE;
+	}
+#endif /* WLAN_ACCEL_BOOT */
+
+	return BCME_OK;
+}
+#ifdef TPUT_DEBUG_DUMP
+bool dhd_get_napi_sched_cnt(dhd_pub_t * dhdp,
+	uint32 **napi_cnts, uint32 **txp_cnts, uint32 **tx_start_cnts)
+{
+	struct dhd_info  *dhd = NULL;
+	if (!dhdp) {
+		WL_ERR(("Invaslid dhd_pub\n"));
+		return FALSE;
+	}
+
+	dhd = dhdp->info;
+
+	if (!dhd) {
+		WL_ERR(("Invaslid dhd_info\n"));
+		return FALSE;
+	}
+
+	*napi_cnts =  dhd->napi_percpu_run_cnt;
+	*txp_cnts = dhd->txp_percpu_run_cnt;
+	*tx_start_cnts = dhd->tx_start_percpu_run_cnt;
+
+	return TRUE;
+}
+#endif /* TPUT_DEBUG_DUMP */
+
+#if defined(DHD_ENABLE_S2MPU)
+/*
+ * return
+ * S2MPUFD_NOTIFY_BAD : watchdog reset
+ * S2MPUFD_NOTIFY_OK  : No watchdog reset
+ */
+int s2mpufd_notifier_callback(struct s2mpufd_notifier_block *block,
+		struct s2mpufd_notifier_info *info)
+{
+	int ret = S2MPUFD_NOTIFY_OK;
+
+	DHD_ERROR(("%s: fault_addr:0x%lx, rw:%d, len:%d, type:%d \n",
+			__FUNCTION__, info->fault_addr, info->fault_rw,
+			info->fault_len, info->fault_type));
+	dhd_smmu_fault_handler(0, info->fault_addr);
+
+	DHD_ERROR(("%s: return : %d\n", __FUNCTION__, ret));
+
+	return ret;
+}
+
+static void dhd_module_s2mpu_register(struct device *dev)
+{
+	struct s2mpufd_notifier_block *s2mpu_nb = NULL;
+
+	DHD_ERROR(("%s: Enter\n", __FUNCTION__));
+
+	s2mpu_nb = devm_kzalloc(dev, sizeof(struct s2mpufd_notifier_block), GFP_KERNEL);
+	if (!s2mpu_nb) {
+		DHD_ERROR(("%s: devm_kzalloc fail\n", __FUNCTION__));
+		return;
+	}
+
+	s2mpu_nb->subsystem = "PCIE_GEN2";
+	s2mpu_nb->notifier_call = s2mpufd_notifier_callback;
+	s2mpufd_notifier_call_register(s2mpu_nb);
+}
+#endif /* DHD_ENABLE_S2MPU */
+
+#ifdef DHD_REGISTER_SMMU_FAULT_HANDLER
+#define SYSMMU_NOTIFY_SKIP_PANIC 3
+static int sysmmu_notifier_callback(struct notifier_block *nb, unsigned long addr,
+		void *flag)
+{
+	int fault_info = *(int *)flag;
+	DHD_ERROR(("%s: Fault ADDR 0x%lx - %s\n", __FUNCTION__, addr,
+			(fault_info & 0x1) ? "WRITE" : "READ"));
+	dhd_smmu_fault_handler(0, addr);
+	return SYSMMU_NOTIFY_SKIP_PANIC;
+}
+#endif /* DHD_REGISTER_SMMU_FAULT_HANDLER */
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+dhd_dongledump_status_t dhd_get_dump_status(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	return OSL_ATOMIC_READ(pub->osh, &dhd->dump_status);
+}
+
+void dhd_set_dump_status(dhd_pub_t *pub, dhd_dongledump_status_t dump_status)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	OSL_ATOMIC_SET(pub->osh, &dhd->dump_status, dump_status);
+}
+
+static void dhd_dump_proc(struct work_struct *work_data)
+{
+	dhd_info_t *dhd_info = NULL;
+	dhd_pub_t *dhdp = NULL;
+	unsigned long flags = 0;
+
+	/* Ignore compiler warnings due to -Werror=cast-qual */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd_info = container_of(work_data, dhd_info_t, dhd_dump_proc_work);
+	GCC_DIAGNOSTIC_POP();
+
+	if (!dhd_info || ((dhdp = &dhd_info->pub) == NULL)) {
+		DHD_ERROR(("dhd is NULL\n"));
+		return;
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_SET_IN_SYSFS_DUMP(dhdp);
+#ifdef DHD_LOG_DUMP
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	dhd_log_dump_trigger(dhdp, CMD_DEFAULT);
+	DHD_GENERAL_LOCK(dhdp, flags);
+#endif /* DHD_LOG_DUMP */
+	DHD_BUS_BUSY_CLEAR_IN_SYSFS_DUMP(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+}
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+void *dhd_get_pub(struct net_device *dev)
+{
+	dhd_info_t *dhdinfo = *(dhd_info_t **)netdev_priv(dev);
+	if (dhdinfo)
+		return (void *)&dhdinfo->pub;
+	else {
+		printf("%s: null dhdinfo\n", __FUNCTION__);
+		return NULL;
+	}
+}
+
+void *dhd_get_conf(struct net_device *dev)
+{
+	dhd_info_t *dhdinfo = *(dhd_info_t **)netdev_priv(dev);
+	if (dhdinfo)
+		return (void *)dhdinfo->pub.conf;
+	else {
+		printf("%s: null dhdinfo\n", __FUNCTION__);
+		return NULL;
+	}
+}
+
+bool dhd_os_wd_timer_enabled(void *bus)
+{
+	dhd_pub_t *pub = bus;
+	dhd_info_t *dhd = (dhd_info_t *)pub->info;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd NULL\n", __FUNCTION__));
+		return FALSE;
+	}
+	return dhd->wd_timer_valid;
+}
+
+#if defined(WLDWDS) && defined(FOURADDR_AUTO_BRG)
+/* This function is to automatically add/del interface to the bridged dev that priamy dev is in */
+static void
+dhd_bridge_dev_set(dhd_info_t *dhd, int ifidx, struct net_device *dev)
+{
+	struct net_device *pdev = NULL, *br_dev = NULL;
+	int i, err = 0;
+
+	if (dev) {
+		/* search the primary interface wlan1(wl0.1) with same bssidx */
+		for (i = 0; i < ifidx; i++) {
+			if (dhd->iflist[i]->bssidx == dhd->iflist[ifidx]->bssidx) {
+				pdev = dhd->pub.info->iflist[i]->net;
+				WL_MSG(dev->name, "found primary dev %s\n", pdev->name);
+				break;
+			}
+		}
+		if (!pdev) {
+			WL_MSG(dev->name, "can not find primary dev\n");
+			return;
+		}
+	} else {
+		pdev = dhd->iflist[ifidx]->net;
+	}
+
+	/* if primary net device is bridged */
+	if (pdev->priv_flags & IFF_BRIDGE_PORT) {
+		rtnl_lock();
+		/* get bridge device */
+		br_dev = netdev_master_upper_dev_get(pdev);
+		if (br_dev) {
+			const struct net_device_ops *ops = br_dev->netdev_ops;
+			if (ops) {
+				if (dev) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+					err = ops->ndo_add_slave(br_dev, dev, NULL);
+#else
+					err = ops->ndo_add_slave(br_dev, dev);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0) */
+					if (err)
+						WL_MSG(dev->name, "add to %s failed %d\n", br_dev->name, err);
+					else
+						WL_MSG(dev->name, "slave added to %s\n", br_dev->name);
+					/* Also bring wds0.x interface up automatically */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+					dev_change_flags(dev, dev->flags | IFF_UP, NULL);
+#else
+					dev_change_flags(dev, dev->flags | IFF_UP);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) */
+				} else {
+					err = ops->ndo_del_slave(br_dev, pdev);
+					if (err)
+						WL_MSG(pdev->name, "del from %s failed %d\n", br_dev->name, err);
+					else
+						WL_MSG(pdev->name, "slave deleted from %s\n", br_dev->name);
+				}
+			}
+		}
+		else {
+			WL_MSG(pdev->name, "not bridged\n");
+		}
+		rtnl_unlock();
+	}
+	else {
+		WL_MSG(pdev->name, "not bridged\n");
+	}
+}
+#endif /* WLDWDS && FOURADDR_AUTO_BRG */
+
+#ifdef DHD_DEBUG_WAKE_LOCK
+int
+dhd_wakelock_counter_get(dhd_pub_t *pub)
+{
+	return ((dhd_info_t *)pub)->wakelock_counter;
+}
+
+int
+dhd_wakelock_wd_counter_get(dhd_pub_t *pub)
+{
+	return ((dhd_info_t *)pub)->wakelock_wd_counter;
+}
+#endif /* DHD_DEBUG_WAKE_LOCK */
diff --git a/drivers/net/wireless/ap12275/dhd_linux.h b/drivers/net/wireless/ap12275/dhd_linux.h
new file mode 100644
index 000000000000..364aa8216201
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux.h
@@ -0,0 +1,609 @@
+/*
+ * DHD Linux header file (dhd_linux exports for cfg80211 and other components)
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/* wifi platform functions for power, interrupt and pre-alloc, either
+ * from Android-like platform device data, or Broadcom wifi platform
+ * device data.
+ *
+ */
+#ifndef __DHD_LINUX_H__
+#define __DHD_LINUX_H__
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <dngl_stats.h>
+#ifdef DHD_LOG_DUMP
+#include <dhd_log_dump.h>
+#endif
+#include <dhd.h>
+#ifdef DHD_WMF
+#include <dhd_wmf_linux.h>
+#endif
+/* Linux wireless extension support */
+#if defined(WL_WIRELESS_EXT)
+#include <wl_iw.h>
+#endif /* defined(WL_WIRELESS_EXT) */
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+#include <linux/earlysuspend.h>
+#endif /* defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND) */
+
+#ifdef BCMPCIE
+#include <bcmmsgbuf.h>
+#endif /* BCMPCIE */
+
+#ifdef PCIE_FULL_DONGLE
+#include <etd.h>
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef CONFIG_IRQ_HISTORY
+#include <linux/power/irq_history.h>
+#endif /* CONFIG_IRQ_HISTORY */
+
+#if defined(OEM_ANDROID)
+#include <linux/nl80211.h>
+#endif /* OEM_ANDROID */
+
+#ifdef WL_MONITOR
+#ifdef HOST_RADIOTAP_CONV
+#include <bcmwifi_monitor.h>
+#else
+#define MAX_RADIOTAP_SIZE      256 /* Maximum size to hold HE Radiotap header format */
+#define MAX_MON_PKT_SIZE       (4096 + MAX_RADIOTAP_SIZE)
+#endif /* HOST_RADIOTAP_CONV */
+#endif /* WL_MONITOR */
+
+/* dongle status */
+enum wifi_adapter_status {
+	WIFI_STATUS_POWER_ON = 0,
+	WIFI_STATUS_FW_READY,
+	WIFI_STATUS_NET_ATTACHED,
+	WIFI_STATUS_BUS_DISCONNECTED
+};
+#define wifi_chk_adapter_status(adapter, stat) (test_bit(stat, &(adapter)->status))
+#define wifi_get_adapter_status(adapter, stat) (test_bit(stat, &(adapter)->status))
+#define wifi_set_adapter_status(adapter, stat) (set_bit(stat, &(adapter)->status))
+#define wifi_clr_adapter_status(adapter, stat) (clear_bit(stat, &(adapter)->status))
+#define wifi_chg_adapter_status(adapter, stat) (change_bit(stat, &(adapter)->status))
+
+#ifdef DHD_COREDUMP
+#define PC_FOUND_BIT 0x01
+#define LR_FOUND_BIT 0x02
+#define ALL_ADDR_VAL (PC_FOUND_BIT | LR_FOUND_BIT)
+#define READ_NUM_BYTES 1000
+#define DHD_FUNC_STR_LEN 80
+
+#define DHD_COREDUMP_MAGIC 0xDDCEDACF
+#define TLV_TYPE_LENGTH_SIZE	(8u)
+/* coredump is composed as following TLV format.
+ * Type(32bit) | Length(32bit) | Value(x bit)
+ * e.g) socram type | length | socram dump
+ *      sssr core1 type | length | sssr core1 dump
+ *      ...
+ */
+enum coredump_types {
+	DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_BEFORE = 0,
+	DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_AFTER,
+	DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_BEFORE,
+	DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_AFTER,
+	DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_BEFORE,
+	DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_AFTER,
+	DHD_COREDUMP_TYPE_SSSRDUMP_DIG_BEFORE,
+	DHD_COREDUMP_TYPE_SSSRDUMP_DIG_AFTER,
+	DHD_COREDUMP_TYPE_SOCRAMDUMP
+};
+
+#ifdef DHD_SSSR_DUMP
+typedef struct dhd_coredump {
+	uint32 type;
+	uint32 length;
+	void *bufptr;
+} dhd_coredump_t;
+#endif /* DHD_SSSR_DUMP */
+#endif /* DHD_COREDUMP */
+
+#ifdef BCMDBUS
+#define DBUS_NRXQ	50
+#define DBUS_NTXQ	100
+#endif /* BCMDBUS */
+
+#ifdef DHD_WAKE_RX_STATUS
+#define ETHER_ICMP6_HEADER	20
+#define ETHER_IPV6_SADDR (ETHER_ICMP6_HEADER + 2)
+#define ETHER_IPV6_DAADR (ETHER_IPV6_SADDR + IPV6_ADDR_LEN)
+#define ETHER_ICMPV6_TYPE (ETHER_IPV6_DAADR + IPV6_ADDR_LEN)
+#endif /* DHD_WAKE_RX_STATUS */
+
+#ifdef SUPPORT_AP_POWERSAVE
+#define RXCHAIN_PWRSAVE_PPS			10
+#define RXCHAIN_PWRSAVE_QUIET_TIME		10
+#define RXCHAIN_PWRSAVE_STAS_ASSOC_CHECK	0
+#endif /* SUPPORT_AP_POWERSAVE */
+
+#define DHD_REGISTRATION_TIMEOUT  12000  /* msec : allowed time to finished dhd registration */
+#define DHD_FW_READY_TIMEOUT  5000  /* msec : allowed time to finished fw download */
+
+/* FW initialised value for ocl_rssi_threshold */
+#define FW_OCL_RSSI_THRESH_INITVAL -75
+
+#if defined(DHD_TRACE_WAKE_LOCK)
+typedef enum dhd_wklock_type {
+	DHD_WAKE_LOCK,
+	DHD_WAKE_UNLOCK,
+	DHD_WAIVE_LOCK,
+	DHD_RESTORE_LOCK
+} dhd_wklock_t;
+
+struct wk_trace_record {
+	unsigned long addr;	            /* Address of the instruction */
+	dhd_wklock_t lock_type;         /* lock_type */
+	unsigned long long counter;		/* counter information */
+	struct hlist_node wklock_node;  /* hash node */
+};
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+#define HASH_ADD(hashtable, node, key) \
+	do { \
+		hash_add(hashtable, node, key); \
+	} while (0);
+#else
+#define HASH_ADD(hashtable, node, key) \
+	do { \
+		int index = hash_long(key, ilog2(ARRAY_SIZE(hashtable))); \
+		hlist_add_head(node, &hashtable[index]); \
+	} while (0);
+#endif /* KERNEL_VER < KERNEL_VERSION(3, 7, 0) */
+
+#define STORE_WKLOCK_RECORD(wklock_type) \
+	do { \
+		struct wk_trace_record *wklock_info = NULL; \
+		unsigned long func_addr = (unsigned long)__builtin_return_address(0); \
+		wklock_info = find_wklock_entry(func_addr); \
+		if (wklock_info) { \
+			if (wklock_type == DHD_WAIVE_LOCK || wklock_type == DHD_RESTORE_LOCK) { \
+				wklock_info->counter = dhd->wakelock_counter; \
+			} else { \
+				wklock_info->counter++; \
+			} \
+		} else { \
+			wklock_info = kzalloc(sizeof(*wklock_info), GFP_ATOMIC); \
+			if (!wklock_info) {\
+				DHD_ERROR(("Can't allocate wk_trace_record \n")); \
+			} else { \
+				wklock_info->addr = func_addr; \
+				wklock_info->lock_type = wklock_type; \
+				if (wklock_type == DHD_WAIVE_LOCK || \
+						wklock_type == DHD_RESTORE_LOCK) { \
+					wklock_info->counter = dhd->wakelock_counter; \
+				} else { \
+					wklock_info->counter++; \
+				} \
+				HASH_ADD(wklock_history, &wklock_info->wklock_node, func_addr); \
+			} \
+		} \
+	} while (0);
+
+#else
+#define STORE_WKLOCK_RECORD(wklock_type)
+#endif /* DHD_TRACE_WAKE_LOCK */
+
+#ifdef ENABLE_ADAPTIVE_SCHED
+#define DEFAULT_CPUFREQ_THRESH		1000000	/* threshold frequency : 1000000 = 1GHz */
+#ifndef CUSTOM_CPUFREQ_THRESH
+#define CUSTOM_CPUFREQ_THRESH	DEFAULT_CPUFREQ_THRESH
+#endif /* CUSTOM_CPUFREQ_THRESH */
+#endif /* ENABLE_ADAPTIVE_SCHED */
+
+/* enable HOSTIP cache update from the host side when an eth0:N is up */
+#define AOE_IP_ALIAS_SUPPORT 1
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_LIMIT_OUTPUT)
+#ifndef DHD_TCP_LIMIT_OUTPUT_BYTES
+#define DHD_TCP_LIMIT_OUTPUT_BYTES (4 * 1024 * 1024)
+#endif /* DHD_TCP_LIMIT_OUTPUT_BYTES */
+#ifndef TCP_DEFAULT_LIMIT_OUTPUT
+#define TCP_DEFAULT_LIMIT_OUTPUT (256 * 1024)
+#endif /* TSQ_DEFAULT_LIMIT_OUTPUT */
+#endif /* LINUX_VERSION_CODE > 4.19.0 && DHD_TCP_LIMIT_OUTPUT */
+
+#define DHD_MEMDUMP_TYPE_STR_LEN 32
+#define DHD_MEMDUMP_PATH_STR_LEN 128
+
+typedef struct wifi_adapter_info {
+	const char	*name;
+	uint		irq_num;
+	uint		intr_flags;
+	const char	*fw_path;
+	const char	*nv_path;
+	const char	*clm_path;
+	const char	*conf_path;
+	void		*wifi_plat_data;	/* wifi ctrl func, for backward compatibility */
+	uint		bus_type;
+	uint		bus_num;
+	uint		slot_num;
+	int			index;
+	int 		gpio_wl_reg_on;
+#ifdef CUSTOMER_OOB
+	int 		gpio_wl_host_wake;
+#endif
+	wait_queue_head_t status_event;
+	unsigned long status;
+#if defined(BT_OVER_SDIO)
+	const char	*btfw_path;
+#endif /* defined (BT_OVER_SDIO) */
+#if defined(BCMSDIO)
+	struct sdio_func *sdio_func;
+#endif /* BCMSDIO */
+#if defined(BCMPCIE)
+	struct pci_dev *pci_dev;
+	struct pci_saved_state *pci_saved_state;
+#endif /* BCMPCIE */
+#ifdef BCMDHD_PLATDEV
+	struct platform_device *pdev;
+#endif /* BCMDHD_PLATDEV */
+} wifi_adapter_info_t;
+
+#if defined(CONFIG_WIFI_CONTROL_FUNC)
+#include <linux/wlan_plat.h>
+#else
+#include <dhd_plat.h>
+#endif /* CONFIG_WIFI_CONTROL_FUNC */
+
+typedef struct bcmdhd_wifi_platdata {
+	uint				num_adapters;
+	wifi_adapter_info_t	*adapters;
+} bcmdhd_wifi_platdata_t;
+
+/** Per STA params. A list of dhd_sta objects are managed in dhd_if */
+typedef struct dhd_sta {
+	cumm_ctr_t cumm_ctr;    /* cummulative queue length of child flowrings */
+	uint16 flowid[NUMPRIO]; /* allocated flow ring ids (by priority) */
+	void * ifp;             /* associated dhd_if */
+	struct ether_addr ea;   /* stations ethernet mac address */
+	struct list_head list;  /* link into dhd_if::sta_list */
+	int idx;                /* index of self in dhd_pub::sta_pool[] */
+	int ifidx;              /* index of interface in dhd */
+#ifdef DHD_WMF
+	struct dhd_sta *psta_prim; /* primary index of psta interface */
+#endif /* DHD_WMF */
+	chanspec_t chanspec;	/* sta chanspec info */
+} dhd_sta_t;
+typedef dhd_sta_t dhd_sta_pool_t;
+
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+typedef enum {
+	NONE_4WAY,
+	M1_4WAY,
+	M2_4WAY,
+	M3_4WAY,
+	M4_4WAY
+} msg_4way_t;
+typedef enum {
+	M3_RXED,
+	M4_TXFAILED
+} msg_4way_state_t;
+#define MAX_4WAY_TIMEOUT_MS 2000
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#if defined(DHD_LB)
+/* Dynamic CPU selection for load balancing. */
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/notifier.h>
+#include <linux/workqueue.h>
+#include <asm/atomic.h>
+
+/* FIXME: Make this a module param or a sysfs. */
+#if !defined(DHD_LB_PRIMARY_CPUS)
+#define DHD_LB_PRIMARY_CPUS     0x0 /* Big CPU coreids mask */
+#endif
+#if !defined(DHD_LB_SECONDARY_CPUS)
+#define DHD_LB_SECONDARY_CPUS   0xFE /* Little CPU coreids mask */
+#endif
+
+#define HIST_BIN_SIZE	9
+
+#if defined(DHD_LB_TXP)
+/* Pkttag not compatible with PROP_TXSTATUS or WLFC */
+typedef struct dhd_tx_lb_pkttag_fr {
+	struct net_device *net;
+	int ifidx;
+} dhd_tx_lb_pkttag_fr_t;
+
+#define DHD_LB_TX_PKTTAG_SET_NETDEV(tag, netdevp)	((tag)->net = netdevp)
+#define DHD_LB_TX_PKTTAG_NETDEV(tag)			((tag)->net)
+
+#define DHD_LB_TX_PKTTAG_SET_IFIDX(tag, ifidx)	((tag)->ifidx = ifidx)
+#define DHD_LB_TX_PKTTAG_IFIDX(tag)		((tag)->ifidx)
+#endif /* DHD_LB_TXP */
+#endif /* DHD_LB */
+
+#define FILE_DUMP_MAX_WAIT_TIME 4000
+
+#ifdef IL_BIGENDIAN
+#include <bcmendian.h>
+#define htod32(i) (bcmswap32(i))
+#define htod16(i) (bcmswap16(i))
+#define dtoh32(i) (bcmswap32(i))
+#define dtoh16(i) (bcmswap16(i))
+#define htodchanspec(i) htod16(i)
+#define dtohchanspec(i) dtoh16(i)
+#else
+#define htod32(i) (i)
+#define htod16(i) (i)
+#define dtoh32(i) (i)
+#define dtoh16(i) (i)
+#define htodchanspec(i) (i)
+#define dtohchanspec(i) (i)
+#endif /* IL_BIGENDINA */
+
+#ifdef BLOCK_IPV6_PACKET
+#define HEX_PREF_STR	"0x"
+#define UNI_FILTER_STR	"010000000000"
+#define ZERO_ADDR_STR	"000000000000"
+#define ETHER_TYPE_STR	"0000"
+#define IPV6_FILTER_STR	"20"
+#define ZERO_TYPE_STR	"00"
+#endif /* BLOCK_IPV6_PACKET */
+
+#if defined(OEM_ANDROID) && defined(SOFTAP)
+extern bool ap_cfg_running;
+extern bool ap_fw_loaded;
+#endif
+
+#if defined(OEM_ANDROID) && defined(BCMPCIE)
+extern int dhd_get_suspend_bcn_li_dtim(dhd_pub_t *dhd, int *dtim_period, int *bcn_interval);
+#else
+extern int dhd_get_suspend_bcn_li_dtim(dhd_pub_t *dhd);
+#endif /* OEM_ANDROID && BCMPCIE */
+
+#ifdef DHD_SEND_HANG_PRIVCMD_ERRORS
+extern uint32 report_hang_privcmd_err;
+#endif /* DHD_SEND_HANG_PRIVCMD_ERRORS */
+
+#if defined(SOFTAP_TPUT_ENHANCE)
+extern void dhd_bus_setidletime(dhd_pub_t *dhdp, int idle_time);
+extern void dhd_bus_getidletime(dhd_pub_t *dhdp, int* idle_time);
+#endif /* SOFTAP_TPUT_ENHANCE */
+
+#if defined(BCM_ROUTER_DHD)
+void traffic_mgmt_pkt_set_prio(dhd_pub_t *dhdp, void * pktbuf);
+#endif /* BCM_ROUTER_DHD */
+
+typedef struct dhd_if_event {
+	struct list_head	list;
+	wl_event_data_if_t	event;
+	char			name[IFNAMSIZ+1];
+	uint8			mac[ETHER_ADDR_LEN];
+} dhd_if_event_t;
+
+/* Interface control information */
+typedef struct dhd_if {
+	struct dhd_info *info;			/* back pointer to dhd_info */
+	/* OS/stack specifics */
+	struct net_device *net;
+	int				idx;			/* iface idx in dongle */
+	uint			subunit;		/* subunit */
+	uint8			mac_addr[ETHER_ADDR_LEN];	/* assigned MAC address */
+	bool			set_macaddress;
+	bool			set_multicast;
+	uint8			bssidx;			/* bsscfg index for the interface */
+	bool			attached;		/* Delayed attachment when unset */
+	bool			txflowcontrol;	/* Per interface flow control indicator */
+	char			name[IFNAMSIZ+1]; /* linux interface name */
+	char			dngl_name[IFNAMSIZ+1]; /* corresponding dongle interface name */
+	struct net_device_stats stats;
+#ifdef DHD_WMF
+	dhd_wmf_t		wmf;		/* per bsscfg wmf setting */
+	bool	wmf_psta_disable;		/* enable/disable MC pkt to each mac
+						 * of MC group behind PSTA
+						 */
+#endif /* DHD_WMF */
+#ifdef PCIE_FULL_DONGLE
+	struct list_head sta_list;		/* sll of associated stations */
+	spinlock_t	sta_list_lock;		/* lock for manipulating sll */
+#endif /* PCIE_FULL_DONGLE */
+	uint32  ap_isolate;			/* ap-isolation settings */
+#ifdef DHD_L2_FILTER
+	bool parp_enable;
+	bool parp_discard;
+	bool parp_allnode;
+	arp_table_t *phnd_arp_table;
+	/* for Per BSS modification */
+	bool dhcp_unicast;
+	bool block_ping;
+	bool grat_arp;
+	bool block_tdls;
+#endif /* DHD_L2_FILTER */
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+	uint8	 *qosmap_up_table;		/* user priority table, size is UP_TABLE_MAX */
+	bool qosmap_up_table_enable;	/* flag set only when app want to set additional UP */
+#endif /* BCM_ROUTER_DHD && QOS_MAP_SET */
+#ifdef DHD_MCAST_REGEN
+	bool mcast_regen_bss_enable;
+#endif
+	bool rx_pkt_chainable;		/* set all rx packet to chainable config by default */
+	cumm_ctr_t cumm_ctr;		/* cummulative queue length of child flowrings */
+#ifdef BCM_ROUTER_DHD
+	bool	primsta_dwds;		/* DWDS status of primary sta interface */
+#endif /* BCM_ROUTER_DHD */
+	uint8 tx_paths_active;
+	bool del_in_progress;
+	bool static_if;			/* used to avoid some operations on static_if */
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+	struct delayed_work m4state_work;
+	atomic_t m4state;
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+#ifdef DHDTCPSYNC_FLOOD_BLK
+	uint32 tsync_rcvd;
+	uint32 tsyncack_txed;
+	u64 last_sync;
+	struct work_struct  blk_tsfl_work;
+	uint32 tsync_per_sec;
+	bool disconnect_tsync_flood;
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+#ifdef DHD_POST_EAPOL_M1_AFTER_ROAM_EVT
+	bool recv_reassoc_evt;
+	bool post_roam_evt;
+#endif /* DHD_POST_EAPOL_M1_AFTER_ROAM_EVT */
+#ifdef WLDWDS
+	bool dwds;	/* DWDS interface */
+#endif /* WLDWDS */
+#ifdef WLEASYMESH
+	uint8 _1905_al_ucast[ETHER_ADDR_LEN];
+	uint8 _1905_al_mcast[ETHER_ADDR_LEN];
+#endif /* WLEASYMESH */
+} dhd_if_t;
+
+struct ipv6_work_info_t {
+	uint8			if_idx;
+	char			ipv6_addr[IPV6_ADDR_LEN];
+	unsigned long		event;
+};
+
+typedef struct dhd_dump {
+	uint8 *buf;
+	int bufsize;
+	uint8 *hscb_buf;
+	int hscb_bufsize;
+} dhd_dump_t;
+#ifdef DNGL_AXI_ERROR_LOGGING
+typedef struct dhd_axi_error_dump {
+	ulong fault_address;
+	uint32 axid;
+	struct hnd_ext_trap_axi_error_v1 etd_axi_error_v1;
+} dhd_axi_error_dump_t;
+#endif /* DNGL_AXI_ERROR_LOGGING */
+#ifdef BCM_ROUTER_DHD
+typedef struct dhd_write_file {
+	char file_path[64];
+	uint32 file_flags;
+	uint8 *buf;
+	int bufsize;
+} dhd_write_file_t;
+#endif
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+struct dhd_rx_tx_work {
+	struct work_struct work;
+	struct sk_buff *skb;
+	struct net_device *net;
+	struct dhd_pub *pub;
+};
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef FILTER_IE
+#ifdef DHD_LINUX_STD_FW_API
+#define FILTER_IE_PATH "filter_ie"
+#else
+#define FILTER_IE_PATH "/vendor/etc/wifi/filter_ie"
+#endif /* DHD_LINUX_STD_FW_API */
+#define FILTER_IE_BUFSZ 1024 /* ioc buffsize for FILTER_IE */
+#define FILE_BLOCK_READ_SIZE 256
+#define WL_FILTER_IE_IOV_HDR_SIZE OFFSETOF(wl_filter_ie_iov_v1_t, tlvs)
+#endif /* FILTER_IE */
+
+#define NULL_CHECK(p, s, err)  \
+do { \
+	if (!(p)) { \
+		DHD_ERROR(("NULL POINTER (%s) : %s\n", __FUNCTION__, (s))); \
+		err = BCME_ERROR; \
+		return err; \
+	} \
+} while (0)
+
+int dhd_wifi_platform_register_drv(void);
+void dhd_wifi_platform_unregister_drv(void);
+wifi_adapter_info_t* dhd_wifi_platform_attach_adapter(uint32 bus_type,
+	uint32 bus_num, uint32 slot_num, unsigned long status);
+wifi_adapter_info_t* dhd_wifi_platform_get_adapter(uint32 bus_type, uint32 bus_num,
+	uint32 slot_num);
+int wifi_platform_set_power(wifi_adapter_info_t *adapter, bool on, unsigned long msec);
+int wifi_platform_bus_enumerate(wifi_adapter_info_t *adapter, bool device_present);
+int wifi_platform_get_irq_number(wifi_adapter_info_t *adapter, unsigned long *irq_flags_ptr);
+int wifi_platform_get_mac_addr(wifi_adapter_info_t *adapter, unsigned char *buf, int ifidx);
+#ifdef DHD_COREDUMP
+int wifi_platform_set_coredump(wifi_adapter_info_t *adapter, const char *buf, int buf_len,
+	const char *info);
+#endif /* DHD_COREDUMP */
+#ifdef CUSTOM_COUNTRY_CODE
+void *wifi_platform_get_country_code(wifi_adapter_info_t *adapter, char *ccode,
+	u32 flags);
+#else
+void *wifi_platform_get_country_code(wifi_adapter_info_t *adapter, char *ccode);
+#endif /* CUSTOM_COUNTRY_CODE */
+void* wifi_platform_prealloc(wifi_adapter_info_t *adapter, int section, unsigned long size);
+void* wifi_platform_get_prealloc_func_ptr(wifi_adapter_info_t *adapter);
+
+int dhd_get_fw_mode(struct dhd_info *dhdinfo);
+bool dhd_update_fw_nv_path(struct dhd_info *dhdinfo);
+#ifdef BCM_ROUTER_DHD
+void dhd_update_dpsta_interface_for_sta(dhd_pub_t* dhdp, int ifidx, void* event_data);
+#endif /* BCM_ROUTER_DHD */
+#ifdef DHD_WMF
+dhd_wmf_t* dhd_wmf_conf(dhd_pub_t *dhdp, uint32 idx);
+int dhd_get_wmf_psta_disable(dhd_pub_t *dhdp, uint32 idx);
+int dhd_set_wmf_psta_disable(dhd_pub_t *dhdp, uint32 idx, int val);
+void dhd_update_psta_interface_for_sta(dhd_pub_t *dhdp, char* ifname,
+		void* mac_addr, void* event_data);
+#endif /* DHD_WMF */
+
+#if defined(BT_OVER_SDIO)
+int dhd_net_bus_get(struct net_device *dev);
+int dhd_net_bus_put(struct net_device *dev);
+#endif /* BT_OVER_SDIO */
+#if defined(WLADPS)
+#define ADPS_ENABLE	1
+#define ADPS_DISABLE	0
+#define ADPS_MODE_PM2_ONLY	3
+
+int dhd_enable_adps(dhd_pub_t *dhd, uint8 on);
+#endif
+#ifdef DHDTCPSYNC_FLOOD_BLK
+extern void dhd_reset_tcpsync_info_by_ifp(dhd_if_t *ifp);
+extern void dhd_reset_tcpsync_info_by_dev(struct net_device *dev);
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+extern void dhd_set_del_in_progress(dhd_pub_t *dhdp, struct net_device * ndev);
+extern void dhd_clear_del_in_progress(dhd_pub_t *dhdp, struct net_device * ndev);
+#ifdef PCIE_FULL_DONGLE
+extern void dhd_net_del_flowrings_sta(dhd_pub_t * dhd, struct net_device * ndev);
+#endif /* PCIE_FULL_DONGLE */
+int dhd_get_fw_capabilities(dhd_pub_t * dhd);
+#ifdef BCMDBUS
+int dhd_dbus_txdata(dhd_pub_t *dhdp, void *pktbuf);
+#endif
+void dhd_event_logtrace_enqueue(dhd_pub_t *dhdp, int ifidx, void *pktbuf);
+#if defined(SUPPORT_MULTIPLE_NVRAM) || defined(SUPPORT_MULTIPLE_CLMBLOB)
+int dhd_get_platform_naming_for_nvram_clmblob_file(download_type_t component, char* file_name);
+#ifdef USE_CID_CHECK
+void dhd_set_platform_ext_name_for_chip_version(char* chip_version);
+#endif /* USE_CID_CHECK */
+#endif /* SUPPORT_MULTIPLE_NVRAM || SUPPORT_MULTIPLE_CLMBLOB */
+void dhd_netif_rx_ni(struct sk_buff * skb);
+#if defined(DBG_PKT_MON) && !defined(PCIE_FULL_DONGLE)
+extern bool dhd_80211_mon_pkt(dhd_pub_t *dhdp, void *pkt, int ifidx);
+#endif /* DBG_PKT_MON */
+
+#endif /* __DHD_LINUX_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_exportfs.c b/drivers/net/wireless/ap12275/dhd_linux_exportfs.c
new file mode 100644
index 000000000000..002fc3d8e998
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_exportfs.c
@@ -0,0 +1,3292 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Linux-specific network interface
+ * Basically selected code segments from usb-cdc.c and usb-rndis.c
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <linux/kobject.h>
+#include <linux/proc_fs.h>
+#include <linux/sysfs.h>
+#include <osl.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_linux_priv.h>
+#include <dhd_proto.h>
+#if defined(WL_BAM)
+#include <wl_bam.h>
+#endif	/* WL_BAM */
+#ifdef PWRSTATS_SYSFS
+#include <wldev_common.h>
+#endif /* PWRSTATS_SYSFS */
+#ifdef WL_CFG80211
+#include <wl_cfg80211.h>
+#endif /* WL_CFG80211 */
+#ifdef CSI_SUPPORT
+#include <dhd_csi.h>
+#endif /* CSI_SUPPORT */
+
+#ifdef SHOW_LOGTRACE
+extern dhd_pub_t* g_dhd_pub;
+#if defined(DEBUGABILITY) || defined(EWP_ECNTRS_LOGGING) || defined(EWP_RTT_LOGGING)
+static int dhd_ring_proc_open(struct inode *inode, struct file *file);
+ssize_t dhd_ring_proc_read(struct file *file, char *buffer, size_t tt, loff_t *loff);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static const struct file_operations dhd_ring_proc_ops = {
+	.open = dhd_ring_proc_open,
+	.read = dhd_ring_proc_read,
+	.release = single_release,
+};
+#else
+static const struct proc_ops dhd_ring_proc_ops = {
+	.proc_open = dhd_ring_proc_open,
+	.proc_read = dhd_ring_proc_read,
+	.proc_release = single_release,
+};
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0) */
+
+static int
+dhd_ring_proc_open(struct inode *inode, struct file *file)
+{
+	int ret = BCME_ERROR;
+	if (inode) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0))
+		ret = single_open(file, 0, pde_data(inode));
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+		ret = single_open(file, 0, PDE_DATA(inode));
+#else
+		/* This feature is not supported for lower kernel versions */
+		ret = single_open(file, 0, NULL);
+#endif
+	} else {
+		DHD_ERROR(("%s: inode is NULL\n", __FUNCTION__));
+	}
+	return ret;
+}
+
+ssize_t
+dhd_ring_proc_read(struct file *file, char __user *buffer, size_t tt, loff_t *loff)
+{
+	trace_buf_info_t *trace_buf_info;
+	int ret = BCME_ERROR;
+	dhd_dbg_ring_t *ring = (dhd_dbg_ring_t *)((struct seq_file *)(file->private_data))->private;
+
+	if (ring == NULL) {
+		DHD_ERROR(("%s: ring is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ASSERT(g_dhd_pub);
+
+	trace_buf_info = (trace_buf_info_t *)MALLOCZ(g_dhd_pub->osh, sizeof(trace_buf_info_t));
+	if (trace_buf_info) {
+		dhd_dbg_read_ring_into_trace_buf(ring, trace_buf_info);
+		if (!buffer || (MIN(trace_buf_info->size, tt) > TRACE_LOG_BUF_MAX_SIZE)) {
+			DHD_ERROR(("%s: size %zu tt %zu trace_buf_sz %d\n", __FUNCTION__,
+				MIN(trace_buf_info->size, tt), tt, trace_buf_info->size));
+			ret = -ENOMEM;
+			goto exit;
+		}
+		if (copy_to_user(buffer, (void*)trace_buf_info->buf,
+				MIN(trace_buf_info->size, tt))) {
+			ret = -EFAULT;
+			goto exit;
+		}
+		if (trace_buf_info->availability == BUF_NOT_AVAILABLE)
+			ret = BUF_NOT_AVAILABLE;
+		else
+			ret = trace_buf_info->size;
+	} else
+		DHD_ERROR(("%s: Memory allocation Failed\n", __FUNCTION__));
+
+exit:
+	if (trace_buf_info) {
+		MFREE(g_dhd_pub->osh, trace_buf_info, sizeof(trace_buf_info_t));
+	}
+	return ret;
+}
+#endif /* DEBUGABILITY || EWP_ECNTRS_LOGGING || EWP_RTT_LOGGING */
+
+void
+dhd_dbg_ring_proc_create(dhd_pub_t *dhdp)
+{
+#ifdef DEBUGABILITY
+	dhd_dbg_ring_t *dbg_verbose_ring = NULL;
+
+	dbg_verbose_ring = dhd_dbg_get_ring_from_ring_id(dhdp, FW_VERBOSE_RING_ID);
+	if (dbg_verbose_ring) {
+		if (!proc_create_data("dhd_trace", S_IRUSR, NULL, &dhd_ring_proc_ops,
+			dbg_verbose_ring)) {
+			DHD_ERROR(("Failed to create /proc/dhd_trace procfs interface\n"));
+		} else {
+			DHD_INFO(("Created /proc/dhd_trace procfs interface\n"));
+		}
+	} else {
+		DHD_ERROR(("dbg_verbose_ring is NULL, /proc/dhd_trace not created\n"));
+	}
+#endif /* DEBUGABILITY */
+
+#ifdef EWP_ECNTRS_LOGGING
+	if (!proc_create_data("dhd_ecounters", S_IRUSR, NULL, &dhd_ring_proc_ops,
+		dhdp->ecntr_dbg_ring)) {
+		DHD_ERROR(("Failed to create /proc/dhd_ecounters procfs interface\n"));
+	} else {
+		DHD_INFO(("Created /proc/dhd_ecounters procfs interface\n"));
+	}
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+	if (!proc_create_data("dhd_rtt", S_IRUSR, NULL, &dhd_ring_proc_ops,
+		dhdp->rtt_dbg_ring)) {
+		DHD_ERROR(("Failed to create /proc/dhd_rtt procfs interface\n"));
+	} else {
+		DHD_INFO(("Created /proc/dhd_rtt procfs interface\n"));
+	}
+#endif /* EWP_RTT_LOGGING */
+}
+
+void
+dhd_dbg_ring_proc_destroy(dhd_pub_t *dhdp)
+{
+#ifdef DEBUGABILITY
+	remove_proc_entry("dhd_trace", NULL);
+#endif /* DEBUGABILITY */
+
+#ifdef EWP_ECNTRS_LOGGING
+	remove_proc_entry("dhd_ecounters", NULL);
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+	remove_proc_entry("dhd_rtt", NULL);
+#endif /* EWP_RTT_LOGGING */
+
+}
+#endif /* SHOW_LOGTRACE */
+
+/* ----------------------------------------------------------------------------
+ * Infrastructure code for sysfs interface support for DHD
+ *
+ * What is sysfs interface?
+ * https://www.kernel.org/doc/Documentation/filesystems/sysfs.txt
+ *
+ * Why sysfs interface?
+ * This is the Linux standard way of changing/configuring Run Time parameters
+ * for a driver. We can use this interface to control "linux" specific driver
+ * parameters.
+ *
+ * -----------------------------------------------------------------------------
+ */
+
+#if defined(DHD_TRACE_WAKE_LOCK)
+extern atomic_t trace_wklock_onoff;
+
+/* Function to show the history buffer */
+static ssize_t
+show_wklock_trace(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+
+	buf[ret] = '\n';
+	buf[ret+1] = 0;
+
+	dhd_wk_lock_stats_dump(&dhd->pub);
+	return ret+1;
+}
+
+/* Function to enable/disable wakelock trace */
+static ssize_t
+wklock_trace_onoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	BCM_REFERENCE(dhd);
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+	if (onoff != 0 && onoff != 1) {
+		return -EINVAL;
+	}
+
+	atomic_set(&trace_wklock_onoff, onoff);
+	if (atomic_read(&trace_wklock_onoff)) {
+		DHD_ERROR(("ENABLE WAKLOCK TRACE\n"));
+	} else {
+		DHD_ERROR(("DISABLE WAKELOCK TRACE\n"));
+	}
+
+	return (ssize_t)(onoff+1);
+}
+#endif /* DHD_TRACE_WAKE_LOCK */
+
+#ifdef DHD_LOG_DUMP
+extern int logdump_periodic_flush;
+extern int logdump_ecntr_enable;
+static ssize_t
+show_logdump_periodic_flush(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long val;
+
+	val = logdump_periodic_flush;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n", val);
+	return ret;
+}
+
+static ssize_t
+logdump_periodic_flush_onoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long val;
+
+	val = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &val);
+	if (val != 0 && val != 1) {
+		 return -EINVAL;
+	}
+	logdump_periodic_flush = val;
+	return count;
+}
+
+static ssize_t
+show_logdump_ecntr(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long val;
+
+	val = logdump_ecntr_enable;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n", val);
+	return ret;
+}
+
+static ssize_t
+logdump_ecntr_onoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long val;
+
+	val = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &val);
+	if (val != 0 && val != 1) {
+		 return -EINVAL;
+	}
+	logdump_ecntr_enable = val;
+	return count;
+}
+
+#endif /* DHD_LOG_DUMP */
+
+extern uint enable_ecounter;
+static ssize_t
+show_enable_ecounter(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long onoff;
+
+	onoff = enable_ecounter;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n",
+		onoff);
+	return ret;
+}
+
+static ssize_t
+ecounter_onoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	dhd_pub_t *dhdp;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+	dhdp = &dhd->pub;
+	if (!FW_SUPPORTED(dhdp, ecounters)) {
+		DHD_ERROR(("%s: ecounters not supported by FW\n", __FUNCTION__));
+		return count;
+	}
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &onoff);
+	if (onoff != 0 && onoff != 1) {
+		return -EINVAL;
+	}
+
+	if (enable_ecounter == onoff) {
+		DHD_ERROR(("%s: ecounters already %d\n", __FUNCTION__, enable_ecounter));
+		return count;
+	}
+
+	enable_ecounter = onoff;
+	dhd_ecounter_configure(dhdp, enable_ecounter);
+
+	return count;
+}
+
+#ifdef DHD_SSSR_DUMP
+static ssize_t
+show_sssr_enab(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long onoff;
+
+	onoff = sssr_enab;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n",
+		onoff);
+	return ret;
+}
+
+static ssize_t
+set_sssr_enab(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &onoff);
+	if (onoff != 0 && onoff != 1) {
+		return -EINVAL;
+	}
+
+	sssr_enab = (uint)onoff;
+
+	return count;
+}
+
+static ssize_t
+show_fis_enab(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long onoff;
+
+	onoff = fis_enab;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n",
+		onoff);
+	return ret;
+}
+
+static ssize_t
+set_fis_enab(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &onoff);
+	if (onoff != 0 && onoff != 1) {
+		return -EINVAL;
+	}
+
+	fis_enab = (uint)onoff;
+
+	return count;
+}
+#endif /* DHD_SSSR_DUMP */
+
+#define FMT_BUFSZ	32
+extern char firmware_path[];
+
+static ssize_t
+show_firmware_path(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%s\n", firmware_path);
+
+	return ret;
+}
+
+static ssize_t
+store_firmware_path(struct dhd_info *dev, const char *buf, size_t count)
+{
+	char fmt_spec[FMT_BUFSZ] = "";
+
+	if ((int)strlen(buf) >= MOD_PARAM_PATHLEN) {
+		return -EINVAL;
+	}
+
+	snprintf(fmt_spec, FMT_BUFSZ, "%%%ds", MOD_PARAM_PATHLEN - 1);
+	sscanf(buf, fmt_spec, firmware_path);
+
+	return count;
+}
+
+extern char nvram_path[];
+
+static ssize_t
+show_nvram_path(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%s\n", nvram_path);
+
+	return ret;
+}
+
+static ssize_t
+store_nvram_path(struct dhd_info *dev, const char *buf, size_t count)
+{
+	char fmt_spec[FMT_BUFSZ] = "";
+
+	if ((int)strlen(buf) >= MOD_PARAM_PATHLEN) {
+		return -EINVAL;
+	}
+
+	snprintf(fmt_spec, FMT_BUFSZ, "%%%ds", MOD_PARAM_PATHLEN - 1);
+	sscanf(buf, fmt_spec, nvram_path);
+
+	return count;
+}
+
+extern char signature_path[];
+
+static ssize_t
+show_signature_path(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%s\n", signature_path);
+
+	return ret;
+}
+
+static ssize_t
+store_signature_path(struct dhd_info *dev, const char *buf, size_t count)
+{
+	char fmt_spec[FMT_BUFSZ] = "";
+
+	if ((int)strlen(buf) >= MOD_PARAM_PATHLEN) {
+		return -EINVAL;
+	}
+
+	snprintf(fmt_spec, FMT_BUFSZ, "%%%ds", MOD_PARAM_PATHLEN - 1);
+	sscanf(buf, fmt_spec, signature_path);
+
+	return count;
+}
+
+#ifdef PWRSTATS_SYSFS
+typedef struct wl_pwrstats_sysfs {
+	uint64	current_ts;
+	uint64	pm_cnt;
+	uint64	pm_dur;
+	uint64	pm_last_entry_us;
+	uint64	awake_cnt;
+	uint64	awake_dur;
+	uint64	awake_last_entry_us;
+	uint64	l0_cnt;
+	uint64	l0_dur_us;
+	uint64	l1_cnt;
+	uint64	l1_dur_us;
+	uint64	l1_1_cnt;
+	uint64	l1_1_dur_us;
+	uint64	l1_2_cnt;
+	uint64	l1_2_dur_us;
+	uint64	l2_cnt;
+	uint64	l2_dur_us;
+} wl_pwrstats_sysfs_t;
+
+uint64 last_delta = 0;
+wl_pwrstats_sysfs_t accumstats = {0, };
+wl_pwrstats_sysfs_t laststats = {0, };
+static const char pwrstr_cnt[] = "count:";
+static const char pwrstr_dur[] = "duration_usec:";
+static const char pwrstr_ts[] = "last_entry_timestamp_usec:";
+
+ssize_t print_pwrstats_cum(char *buf)
+{
+	ssize_t ret = 0;
+
+	ret += scnprintf(buf, PAGE_SIZE, "WIFI\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "AWAKE:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.awake_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.awake_dur);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_ts,
+		laststats.awake_last_entry_us);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "ASLEEP:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.pm_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.pm_dur);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_ts,
+		laststats.pm_last_entry_us);
+
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\nWIFI-PCIE\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "L0:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.l0_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.l0_dur_us);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "L1:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.l1_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.l1_dur_us);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "L1_1:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.l1_1_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.l1_1_dur_us);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "L1_2:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.l1_2_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.l1_2_dur_us);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "L2:\n");
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_cnt,
+		accumstats.l2_cnt);
+	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s 0x%0llx\n", pwrstr_dur,
+		accumstats.l2_dur_us);
+
+	return ret;
+}
+
+void update_pwrstats_cum(uint64 *accum, uint64 *last, uint64 *now, bool force)
+{
+	if (accum) { /* accumulation case, ex; counts, duration */
+		if (*now < *last) {
+			if (force) {
+				/* force update for pm_cnt/awake_cnt and PCIE stats */
+				*accum += *now;
+				*last = *now;
+			}
+		} else {
+			*accum += (*now - *last);
+			*last = *now;
+		}
+	} else if (*now != 0) { /* last entry timestamp case */
+		*last = *now + last_delta;
+	}
+}
+
+static const uint16 pwrstats_req_type[] = {
+	WL_PWRSTATS_TYPE_PCIE,
+	WL_PWRSTATS_TYPE_PM_ACCUMUL
+};
+#define PWRSTATS_REQ_TYPE_NUM	sizeof(pwrstats_req_type) / sizeof(uint16)
+#define PWRSTATS_IOV_BUF_LEN	OFFSETOF(wl_pwrstats_t, data) \
+	+ sizeof(uint32) * PWRSTATS_REQ_TYPE_NUM \
+	+ sizeof(wl_pwr_pcie_stats_t) \
+	+ sizeof(wl_pwr_pm_accum_stats_v1_t) \
+	+ (uint)strlen("pwrstats") + 1
+
+#ifdef BCMPCIE
+extern uint64 dhdpcie_get_last_suspend_time(dhd_pub_t *dhdp);
+#endif /* BCMPCIE */
+
+#ifdef BCMSDIO
+extern uint64 dhdsdio_get_last_suspend_time(dhd_pub_t *dhdp);
+#endif /* BCMSDIO */
+
+uint64
+dhd_get_last_suspend_time(dhd_pub_t *dhdp)
+{
+#if defined(BCMPCIE)
+	return dhdpcie_get_last_suspend_time(dhdp);
+#elif defined(BCMSDIO)
+	return dhdsdio_get_last_suspend_time(dhdp);
+#else
+	return 0;
+#endif
+}
+
+static ssize_t
+show_pwrstats_path(struct dhd_info *dev, char *buf)
+{
+	int err = 0;
+	void *p_data = NULL;
+	ssize_t ret = 0;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	struct net_device *ndev = dhd_linux_get_primary_netdev(&dhd->pub);
+	char *iovar_buf = NULL;
+	wl_pwrstats_query_t *p_query = NULL;
+	wl_pwrstats_sysfs_t pwrstats_sysfs = {0, };
+	wl_pwrstats_t *pwrstats;
+	uint len, taglen, i;
+	uint16 type;
+	uint64 ts_sec, ts_usec, time_delta;
+
+	ASSERT(g_dhd_pub);
+
+	/* Even if dongle is in D3 state,
+	 * ASLEEP duration should be updated with estimated value.
+	 */
+	if (!dhd_os_check_if_up(&dhd->pub)) {
+		if (dhd->pub.busstate == DHD_BUS_SUSPEND) {
+			static uint64 last_suspend_end_time = 0;
+			uint64 curr_time = 0, estimated_pm_dur = 0;
+
+			if (last_suspend_end_time <
+					dhd_get_last_suspend_time(&dhd->pub)) {
+				last_suspend_end_time = dhd_get_last_suspend_time(&dhd->pub);
+			}
+
+			curr_time = OSL_LOCALTIME_NS();
+			if (curr_time >= last_suspend_end_time) {
+				estimated_pm_dur = DIV_U64_BY_U32(
+					(curr_time - last_suspend_end_time), NSEC_PER_USEC);
+				estimated_pm_dur += laststats.pm_dur;
+
+				update_pwrstats_cum(&accumstats.pm_dur, &laststats.pm_dur,
+					&estimated_pm_dur, FALSE);
+				last_suspend_end_time = curr_time;
+			}
+		}
+		ret = print_pwrstats_cum(buf);
+		goto done; /* Not ready yet */
+	}
+
+	len = PWRSTATS_IOV_BUF_LEN;
+	iovar_buf = (char *)MALLOCZ(g_dhd_pub->osh, len);
+	if (iovar_buf == NULL) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		goto done;
+	}
+
+	/* Alloc req buffer */
+	len = OFFSETOF(wl_pwrstats_query_t, type) +
+		PWRSTATS_REQ_TYPE_NUM * sizeof(uint16);
+	p_query = (wl_pwrstats_query_t *)MALLOCZ(g_dhd_pub->osh, len);
+	if (p_query == NULL) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		goto done;
+	}
+
+	/* Build a list of types */
+	p_query->length = PWRSTATS_REQ_TYPE_NUM;
+	for (i = 0; i < PWRSTATS_REQ_TYPE_NUM; i++) {
+		p_query->type[i] = pwrstats_req_type[i];
+	}
+
+	/* Query with desired type list */
+	err = wldev_iovar_getbuf(ndev, "pwrstats", p_query, len,
+		iovar_buf, PWRSTATS_IOV_BUF_LEN, NULL);
+	if (err != BCME_OK) {
+		DHD_ERROR(("error (%d) - size = %zu\n", err, sizeof(wl_pwrstats_t)));
+		ret = print_pwrstats_cum(buf);
+		goto done;
+	}
+
+	/* Check version */
+	pwrstats = (wl_pwrstats_t *) iovar_buf;
+	if (dtoh16(pwrstats->version) != WL_PWRSTATS_VERSION) {
+		DHD_ERROR(("PWRSTATS Version mismatch\n"));
+		goto done;
+	}
+
+	/* Parse TLVs */
+	len = dtoh16(pwrstats->length) - WL_PWR_STATS_HDRLEN;
+	p_data = pwrstats->data;
+	do {
+		type = dtoh16(((uint16*)p_data)[0]);
+		taglen = dtoh16(((uint16*)p_data)[1]);
+
+		if ((taglen < BCM_XTLV_HDR_SIZE) || (taglen > len)) {
+			DHD_ERROR(("Bad len %d for tag %d, remaining len %d\n",
+					taglen, type, len));
+			goto done;
+		}
+
+		if (taglen & 0xF000) {
+			DHD_ERROR(("Resrved bits in len %d for tag %d, remaining len %d\n",
+					taglen, type, len));
+			goto done;
+		}
+
+		switch (type) {
+		case WL_PWRSTATS_TYPE_PCIE:
+		{
+			wl_pwr_pcie_stats_t *stats =
+				(wl_pwr_pcie_stats_t *)p_data;
+
+			if (taglen < sizeof(wl_pwr_pcie_stats_t)) {
+				DHD_ERROR(("Short len for %d: %d < %d\n",
+					type, taglen, (int)sizeof(wl_pwr_pcie_stats_t)));
+				goto done;
+			}
+
+			if (dtoh32(stats->pcie.l0_cnt) == 0) {
+				DHD_ERROR(("link stats are not supported for this pcie core\n"));
+			}
+
+			pwrstats_sysfs.l0_cnt = dtoh32(stats->pcie.l0_cnt);
+			pwrstats_sysfs.l0_dur_us = dtoh32(stats->pcie.l0_usecs);
+			pwrstats_sysfs.l1_cnt = dtoh32(stats->pcie.l1_cnt);
+			pwrstats_sysfs.l1_dur_us = dtoh32(stats->pcie.l1_usecs);
+			pwrstats_sysfs.l1_1_cnt = dtoh32(stats->pcie.l1_1_cnt);
+			pwrstats_sysfs.l1_1_dur_us = dtoh32(stats->pcie.l1_1_usecs);
+			pwrstats_sysfs.l1_2_cnt = dtoh32(stats->pcie.l1_2_cnt);
+			pwrstats_sysfs.l1_2_dur_us = dtoh32(stats->pcie.l1_2_usecs);
+			pwrstats_sysfs.l2_cnt = dtoh32(stats->pcie.l2_cnt);
+			pwrstats_sysfs.l2_dur_us = dtoh32(stats->pcie.l2_usecs);
+		}
+		break;
+
+		case WL_PWRSTATS_TYPE_PM_ACCUMUL:
+		{
+			wl_pwr_pm_accum_stats_v1_t *stats =
+				(wl_pwr_pm_accum_stats_v1_t *)p_data;
+
+			if (taglen < sizeof(wl_pwr_pm_accum_stats_v1_t)) {
+				DHD_ERROR(("Short len for %d: %d < %d\n", type,
+					taglen, (int)sizeof(wl_pwr_pm_accum_stats_v1_t)));
+				goto done;
+			}
+
+			pwrstats_sysfs.current_ts =
+				dtoh64(stats->accum_data.current_ts);
+			pwrstats_sysfs.pm_cnt =
+				dtoh64(stats->accum_data.pm_cnt);
+			pwrstats_sysfs.pm_dur =
+				dtoh64(stats->accum_data.pm_dur);
+			pwrstats_sysfs.pm_last_entry_us =
+				dtoh64(stats->accum_data.pm_last_entry_us);
+			pwrstats_sysfs.awake_cnt =
+				dtoh64(stats->accum_data.awake_cnt);
+			pwrstats_sysfs.awake_dur =
+				dtoh64(stats->accum_data.awake_dur);
+			pwrstats_sysfs.awake_last_entry_us =
+				dtoh64(stats->accum_data.awake_last_entry_us);
+		}
+		break;
+
+		default:
+			DHD_ERROR(("Skipping uknown %d-byte tag %d\n", taglen, type));
+			break;
+		}
+
+		/* Adjust length to account for padding, but don't exceed total len */
+		taglen = (ROUNDUP(taglen, 4) > len) ? len : ROUNDUP(taglen, 4);
+		len -= taglen;
+		*(uint8**)&p_data += taglen;
+	} while (len >= BCM_XTLV_HDR_SIZE);
+
+	/* [awake|pm]_last_entry_us are provided based on host timestamp.
+	 * These are calculated by dongle timestamp + (delta time of host & dongle)
+	 * If the newly calculated delta time is more than 1 second gap from
+	 * the existing delta time, it is updated to compensate more accurately.
+	 */
+	OSL_GET_LOCALTIME(&ts_sec, &ts_usec);
+	time_delta = ts_sec * USEC_PER_SEC + ts_usec - pwrstats_sysfs.current_ts;
+	if ((time_delta > last_delta) &&
+			((time_delta - last_delta) > USEC_PER_SEC)) {
+		last_delta = time_delta;
+	}
+
+	update_pwrstats_cum(&accumstats.awake_cnt, &laststats.awake_cnt,
+			&pwrstats_sysfs.awake_cnt, TRUE);
+	update_pwrstats_cum(&accumstats.awake_dur, &laststats.awake_dur,
+			&pwrstats_sysfs.awake_dur, FALSE);
+	update_pwrstats_cum(&accumstats.pm_cnt, &laststats.pm_cnt, &pwrstats_sysfs.pm_cnt,
+			TRUE);
+	update_pwrstats_cum(&accumstats.pm_dur, &laststats.pm_dur, &pwrstats_sysfs.pm_dur,
+			FALSE);
+	update_pwrstats_cum(NULL, &laststats.awake_last_entry_us,
+			&pwrstats_sysfs.awake_last_entry_us, FALSE);
+	update_pwrstats_cum(NULL, &laststats.pm_last_entry_us,
+			&pwrstats_sysfs.pm_last_entry_us, FALSE);
+
+	update_pwrstats_cum(&accumstats.l0_cnt, &laststats.l0_cnt, &pwrstats_sysfs.l0_cnt,
+			TRUE);
+	update_pwrstats_cum(&accumstats.l0_dur_us, &laststats.l0_dur_us,
+			&pwrstats_sysfs.l0_dur_us, TRUE);
+	update_pwrstats_cum(&accumstats.l1_cnt, &laststats.l1_cnt, &pwrstats_sysfs.l1_cnt,
+			TRUE);
+	update_pwrstats_cum(&accumstats.l1_dur_us, &laststats.l1_dur_us,
+			&pwrstats_sysfs.l1_dur_us, TRUE);
+	update_pwrstats_cum(&accumstats.l1_1_cnt, &laststats.l1_1_cnt,
+			&pwrstats_sysfs.l1_1_cnt, TRUE);
+	update_pwrstats_cum(&accumstats.l1_1_dur_us, &laststats.l1_1_dur_us,
+			&pwrstats_sysfs.l1_1_dur_us, TRUE);
+	update_pwrstats_cum(&accumstats.l1_2_cnt, &laststats.l1_2_cnt,
+			&pwrstats_sysfs.l1_2_cnt, TRUE);
+	update_pwrstats_cum(&accumstats.l1_2_dur_us, &laststats.l1_2_dur_us,
+			&pwrstats_sysfs.l1_2_dur_us, TRUE);
+	update_pwrstats_cum(&accumstats.l2_cnt, &laststats.l2_cnt, &pwrstats_sysfs.l2_cnt,
+			TRUE);
+	update_pwrstats_cum(&accumstats.l2_dur_us, &laststats.l2_dur_us,
+			&pwrstats_sysfs.l2_dur_us, TRUE);
+
+	ret = print_pwrstats_cum(buf);
+done:
+	if (p_query) {
+		MFREE(g_dhd_pub->osh, p_query, len);
+	}
+	if (iovar_buf) {
+		MFREE(g_dhd_pub->osh, iovar_buf, PWRSTATS_IOV_BUF_LEN);
+	}
+
+	return ret;
+}
+#endif /* PWRSTATS_SYSFS */
+
+static ssize_t
+show_tcm_test_mode(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long mode;
+
+	mode = dhd_tcm_test_mode;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n",
+		mode);
+	return ret;
+}
+
+static ssize_t
+set_tcm_test_mode(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long mode;
+
+	mode = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &mode);
+	if (mode > TCM_TEST_MODE_ALWAYS) {
+		return -EINVAL;
+	}
+
+	/* reset with the mode change */
+	if (dhd_tcm_test_mode != mode) {
+		dhd_tcm_test_status = TCM_TEST_NOT_RUN;
+	}
+
+	dhd_tcm_test_mode = (uint)mode;
+
+	return count;
+}
+
+/*
+ * Generic Attribute Structure for DHD.
+ * If we have to add a new sysfs entry under /sys/bcm-dhd/, we have
+ * to instantiate an object of type dhd_attr,  populate it with
+ * the required show/store functions (ex:- dhd_attr_cpumask_primary)
+ * and add the object to default_attrs[] array, that gets registered
+ * to the kobject of dhd (named bcm-dhd).
+ */
+
+struct dhd_attr {
+	struct attribute attr;
+	ssize_t(*show)(struct dhd_info *, char *);
+	ssize_t(*store)(struct dhd_info *, const char *, size_t count);
+};
+
+#if defined(DHD_TRACE_WAKE_LOCK)
+static struct dhd_attr dhd_attr_wklock =
+	__ATTR(wklock_trace, 0660, show_wklock_trace, wklock_trace_onoff);
+#endif /* defined(DHD_TRACE_WAKE_LOCK */
+
+#ifdef DHD_LOG_DUMP
+static struct dhd_attr dhd_attr_logdump_periodic_flush =
+     __ATTR(logdump_periodic_flush, 0660, show_logdump_periodic_flush,
+		logdump_periodic_flush_onoff);
+static struct dhd_attr dhd_attr_logdump_ecntr =
+	__ATTR(logdump_ecntr_enable, 0660, show_logdump_ecntr,
+		logdump_ecntr_onoff);
+#endif /* DHD_LOG_DUMP */
+
+static struct dhd_attr dhd_attr_ecounters =
+	__ATTR(ecounters, 0660, show_enable_ecounter, ecounter_onoff);
+
+#ifdef DHD_SSSR_DUMP
+static struct dhd_attr dhd_attr_sssr_enab =
+	__ATTR(sssr_enab, 0660, show_sssr_enab, set_sssr_enab);
+static struct dhd_attr dhd_attr_fis_enab =
+	__ATTR(fis_enab, 0660, show_fis_enab, set_fis_enab);
+#endif /* DHD_SSSR_DUMP */
+
+static struct dhd_attr dhd_attr_firmware_path =
+	__ATTR(firmware_path, 0660, show_firmware_path, store_firmware_path);
+
+static struct dhd_attr dhd_attr_nvram_path =
+	__ATTR(nvram_path, 0660, show_nvram_path, store_nvram_path);
+
+static struct dhd_attr dhd_attr_sig_path =
+	__ATTR(signature_path, 0660, show_signature_path, store_signature_path);
+
+#ifdef PWRSTATS_SYSFS
+static struct dhd_attr dhd_attr_pwrstats_path =
+	__ATTR(power_stats, 0664, show_pwrstats_path, NULL);
+#endif /* PWRSTATS_SYSFS */
+
+static struct dhd_attr dhd_attr_tcm_test_mode =
+	__ATTR(tcm_test_mode, 0660, show_tcm_test_mode, set_tcm_test_mode);
+
+#define to_dhd(k) container_of(k, struct dhd_info, dhd_kobj)
+#define to_attr(a) container_of(a, struct dhd_attr, attr)
+
+#ifdef DHD_MAC_ADDR_EXPORT
+struct ether_addr sysfs_mac_addr;
+static ssize_t
+show_mac_addr(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, MACF,
+		(uint32)sysfs_mac_addr.octet[0], (uint32)sysfs_mac_addr.octet[1],
+		(uint32)sysfs_mac_addr.octet[2], (uint32)sysfs_mac_addr.octet[3],
+		(uint32)sysfs_mac_addr.octet[4], (uint32)sysfs_mac_addr.octet[5]);
+
+	return ret;
+}
+
+static ssize_t
+set_mac_addr(struct dhd_info *dev, const char *buf, size_t count)
+{
+	if (!bcm_ether_atoe(buf, &sysfs_mac_addr)) {
+		DHD_ERROR(("Invalid Mac Address \n"));
+		return -EINVAL;
+	}
+
+	DHD_ERROR(("Mac Address set with "MACDBG"\n", MAC2STRDBG(&sysfs_mac_addr)));
+
+	return count;
+}
+
+static struct dhd_attr dhd_attr_macaddr =
+	__ATTR(mac_addr, 0660, show_mac_addr, set_mac_addr);
+#endif /* DHD_MAC_ADDR_EXPORT */
+
+#ifdef DHD_FW_COREDUMP
+/*
+ * XXX The filename to store memdump is defined for each platform.
+ * - The default path of CUSTOMER_HW4 device is "PLATFORM_PATH/.memdump.info"
+ * - Brix platform will take default path "/installmedia/.memdump.info"
+ * New platforms can add their ifdefs accordingly below.
+ */
+
+#ifdef CONFIG_X86
+#if defined(OEM_ANDROID)
+#define MEMDUMPINFO_LIVE PLATFORM_PATH".memdump.info"
+#define MEMDUMPINFO_INST "/data/.memdump.info"
+#define MEMDUMPINFO MEMDUMPINFO_LIVE
+#else /* FC19 and Others */
+#define MEMDUMPINFO PLATFORM_PATH".memdump.info"
+#endif /* OEM_ANDROID */
+#else /* For non x86 platforms */
+#define MEMDUMPINFO PLATFORM_PATH".memdump.info"
+#endif /* CONFIG_X86 */
+
+uint32
+get_mem_val_from_file(void)
+{
+	struct file *fp = NULL;
+	uint32 mem_val = DUMP_MEMFILE_MAX;
+	char *p_mem_val = NULL;
+	char *filepath = MEMDUMPINFO;
+	int ret = 0;
+
+	/* Read memdump info from the file */
+	fp = dhd_filp_open(filepath, O_RDONLY, 0);
+	if (IS_ERR(fp) || (fp == NULL)) {
+		DHD_ERROR(("%s: File [%s] doesn't exist\n", __FUNCTION__, filepath));
+#if defined(CONFIG_X86) && defined(OEM_ANDROID)
+		/* Check if it is Live Brix Image */
+		if (strcmp(filepath, MEMDUMPINFO_LIVE) != 0) {
+			goto done;
+		}
+		/* Try if it is Installed Brix Image */
+		filepath = MEMDUMPINFO_INST;
+		DHD_ERROR(("%s: Try File [%s]\n", __FUNCTION__, filepath));
+		fp = dhd_filp_open(filepath, O_RDONLY, 0);
+		if (IS_ERR(fp) || (fp == NULL)) {
+			DHD_ERROR(("%s: File [%s] doesn't exist\n", __FUNCTION__, filepath));
+			goto done;
+		}
+#else /* Non Brix Android platform */
+		goto done;
+#endif /* CONFIG_X86 && OEM_ANDROID */
+	}
+
+	/* Handle success case */
+	ret = dhd_kernel_read_compat(fp, 0, (char *)&mem_val, sizeof(uint32));
+	if (ret < 0) {
+		DHD_ERROR(("%s: File read error, ret=%d\n", __FUNCTION__, ret));
+		dhd_filp_close(fp, NULL);
+		goto done;
+	}
+
+	p_mem_val = (char*)&mem_val;
+	p_mem_val[sizeof(uint32) - 1] = '\0';
+	mem_val = bcm_atoi(p_mem_val);
+
+	dhd_filp_close(fp, NULL);
+
+done:
+	return mem_val;
+}
+
+void dhd_get_memdump_info(dhd_pub_t *dhd)
+{
+#ifndef DHD_EXPORT_CNTL_FILE
+	uint32 mem_val = DUMP_MEMFILE_MAX;
+
+	mem_val = get_mem_val_from_file();
+	if (mem_val != DUMP_MEMFILE_MAX)
+		dhd->memdump_enabled = mem_val;
+#ifdef DHD_INIT_DEFAULT_MEMDUMP
+	if (mem_val == 0 || mem_val == DUMP_MEMFILE_MAX)
+		mem_val = DUMP_MEMFILE_BUGON;
+#endif /* DHD_INIT_DEFAULT_MEMDUMP */
+#else
+#ifdef DHD_INIT_DEFAULT_MEMDUMP
+	if (dhd->memdump_enabled == 0 || dhd->memdump_enabled == DUMP_MEMFILE_MAX)
+		dhd->memdump_enabled = DUMP_MEMFILE;
+#endif /* DHD_INIT_DEFAULT_MEMDUMP */
+#endif /* !DHD_EXPORT_CNTL_FILE */
+#ifdef BCMQT
+	/* In QT environment collecting memdump on FW TRAP, IOVAR timeouts,
+	 * is taking more time and makes system unresponsive so disabling it.
+	 * if needed memdump can be collected through 'dhd upload' command.
+	*/
+	dhd->memdump_enabled = DUMP_DISABLED;
+#endif
+#ifdef DHD_DETECT_CONSECUTIVE_MFG_HANG
+	/* override memdump_enabled value to avoid once trap issues */
+	if (dhd_bus_get_fw_mode(dhd) == DHD_FLAG_MFG_MODE &&
+			(dhd->memdump_enabled == DUMP_MEMONLY ||
+			dhd->memdump_enabled == DUMP_MEMFILE_BUGON)) {
+		dhd->memdump_enabled = DUMP_MEMFILE;
+		DHD_ERROR(("%s : Override memdump_value to %d\n",
+				__FUNCTION__, dhd->memdump_enabled));
+	}
+#endif /* DHD_DETECT_CONSECUTIVE_MFG_HANG */
+	DHD_ERROR(("%s: MEMDUMP ENABLED = %u\n", __FUNCTION__, dhd->memdump_enabled));
+}
+
+#ifdef DHD_EXPORT_CNTL_FILE
+static ssize_t
+show_memdump_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	dhd_pub_t *dhdp;
+
+	if (!dev) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	dhdp = &dev->pub;
+	ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", dhdp->memdump_enabled);
+	return ret;
+}
+
+static ssize_t
+set_memdump_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long memval;
+	dhd_pub_t *dhdp;
+
+	if (!dev) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+	dhdp = &dev->pub;
+
+	memval = bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%lu", &memval);
+
+	dhdp->memdump_enabled = (uint32)memval;
+
+	DHD_ERROR(("%s: MEMDUMP ENABLED = %u\n", __FUNCTION__, dhdp->memdump_enabled));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_memdump =
+	__ATTR(memdump, 0660, show_memdump_info, set_memdump_info);
+#endif /* DHD_EXPORT_CNTL_FILE */
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef BCMASSERT_LOG
+/*
+ * XXX The filename to store assert type is defined for each platform.
+ * New platforms can add their ifdefs accordingly below.
+ */
+#define ASSERTINFO PLATFORM_PATH".assert.info"
+
+int
+get_assert_val_from_file(void)
+{
+	struct file *fp = NULL;
+	char *filepath = ASSERTINFO;
+	char *p_mem_val = NULL;
+	int mem_val = -1;
+
+	/*
+	 * Read assert info from the file
+	 * 0: Trigger Kernel crash by panic()
+	 * 1: Print out the logs and don't trigger Kernel panic. (default)
+	 * 2: Trigger Kernel crash by BUG()
+	 * File doesn't exist: Keep default value (1).
+	 */
+	fp = dhd_filp_open(filepath, O_RDONLY, 0);
+	if (IS_ERR(fp) || (fp == NULL)) {
+		DHD_ERROR(("%s: File [%s] doesn't exist\n", __FUNCTION__, filepath));
+	} else {
+		int ret = dhd_kernel_read_compat(fp, 0, (char *)&mem_val, sizeof(uint32));
+		if (ret < 0) {
+			DHD_ERROR(("%s: File read error, ret=%d\n", __FUNCTION__, ret));
+		} else {
+			p_mem_val = (char *)&mem_val;
+			p_mem_val[sizeof(uint32) - 1] = '\0';
+			mem_val = bcm_atoi(p_mem_val);
+			DHD_ERROR(("%s: ASSERT ENABLED = %d\n", __FUNCTION__, mem_val));
+		}
+		dhd_filp_close(fp, NULL);
+	}
+
+	mem_val = (mem_val >= 0) ? mem_val : 0;
+	return mem_val;
+}
+
+void dhd_get_assert_info(dhd_pub_t *dhd)
+{
+#ifndef DHD_EXPORT_CNTL_FILE
+	int mem_val = -1;
+
+	mem_val = get_assert_val_from_file();
+
+	g_assert_type = mem_val;
+#endif /* !DHD_EXPORT_CNTL_FILE */
+}
+
+#ifdef DHD_EXPORT_CNTL_FILE
+static ssize_t
+show_assert_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (!dev) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = scnprintf(buf, PAGE_SIZE -1, "%d\n", g_assert_type);
+	return ret;
+
+}
+
+static ssize_t
+set_assert_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long assert_val;
+
+	assert_val = bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%lu", &assert_val);
+
+	g_assert_type = (uint32)assert_val;
+
+	DHD_ERROR(("%s: ASSERT ENABLED = %lu\n", __FUNCTION__, assert_val));
+	return count;
+
+}
+
+static struct dhd_attr dhd_attr_assert =
+	__ATTR(assert, 0660, show_assert_info, set_assert_info);
+#endif /* DHD_EXPORT_CNTL_FILE */
+#endif /* BCMASSERT_LOG */
+
+#ifdef DHD_EXPORT_CNTL_FILE
+#if defined(WRITE_WLANINFO)
+static ssize_t
+show_wifiver_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE -1, "%s", version_info);
+	return ret;
+}
+
+static ssize_t
+set_wifiver_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	DHD_ERROR(("Do not set version info\n"));
+	return -EINVAL;
+}
+
+static struct dhd_attr dhd_attr_wifiver =
+	__ATTR(wifiver, 0660, show_wifiver_info, set_wifiver_info);
+#endif /* WRITE_WLANINFO */
+
+#if defined(USE_CID_CHECK) || defined(USE_DIRECT_VID_TAG)
+char cidinfostr[MAX_VNAME_LEN];
+
+static ssize_t
+show_cid_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+#ifdef USE_DIRECT_VID_TAG
+	ret = scnprintf(buf, PAGE_SIZE -1, "%x%x", cidinfostr[VENDOR_OFF], cidinfostr[MD_REV_OFF]);
+#endif /* USE_DIRECT_VID_TAG */
+#ifdef USE_CID_CHECK
+	ret = scnprintf(buf, PAGE_SIZE -1, "%s", cidinfostr);
+#endif /* USE_CID_CHECK */
+	return ret;
+}
+
+static ssize_t
+set_cid_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+#ifdef USE_DIRECT_VID_TAG
+	uint32 stored_vid = 0, md_rev = 0, vendor = 0;
+	uint32 vendor_mask = 0x00FF;
+
+	stored_vid = bcm_strtoul(buf, NULL, 16);
+
+	DHD_ERROR(("%s : stored_vid : 0x%x\n", __FUNCTION__, stored_vid));
+	md_rev = stored_vid & vendor_mask;
+	vendor = stored_vid >> 8;
+
+	memset(cidinfostr, 0, sizeof(cidinfostr));
+
+	cidinfostr[MD_REV_OFF] = (char)md_rev;
+	cidinfostr[VENDOR_OFF] = (char)vendor;
+	DHD_INFO(("CID string %x%x\n", cidinfostr[VENDOR_OFF], cidinfostr[MD_REV_OFF]));
+#endif /* USE_DIRECT_VID_TAG */
+#ifdef USE_CID_CHECK
+	int len = strlen(buf) + 1;
+	int maxstrsz;
+	maxstrsz = MAX_VNAME_LEN;
+
+	scnprintf(cidinfostr, ((len > maxstrsz) ? maxstrsz : len), "%s", buf);
+	DHD_INFO(("%s : CID info string\n", cidinfostr));
+#endif /* USE_CID_CHECK */
+	return count;
+}
+
+static struct dhd_attr dhd_attr_cidinfo =
+	__ATTR(cid, 0660, show_cid_info, set_cid_info);
+#endif /* USE_CID_CHECK || USE_DIRECT_VID_TAG */
+
+#if defined(GEN_SOFTAP_INFO_FILE)
+char softapinfostr[SOFTAP_INFO_BUF_SZ];
+static ssize_t
+show_softap_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE -1, "%s", softapinfostr);
+	return ret;
+}
+
+static ssize_t
+set_softap_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	DHD_ERROR(("Do not set sofap related info\n"));
+	return -EINVAL;
+}
+
+static struct dhd_attr dhd_attr_softapinfo =
+	__ATTR(softap, 0660, show_softap_info, set_softap_info);
+#endif /* GEN_SOFTAP_INFO_FILE */
+
+#if defined(MIMO_ANT_SETTING)
+unsigned long antsel;
+
+static ssize_t
+show_ant_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE -1, "%lu\n", antsel);
+	return ret;
+}
+
+static ssize_t
+set_ant_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long ant_val;
+
+	ant_val = bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%lu", &ant_val);
+
+	/*
+	 * Check value
+	 * 0 - Not set, handle same as file not exist
+	 */
+	if (ant_val > 3) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %lu \n",
+			__FUNCTION__, ant_val));
+		return -EINVAL;
+	}
+
+	antsel = ant_val;
+	DHD_ERROR(("[WIFI_SEC] %s: Set Antinfo val = %lu \n", __FUNCTION__, antsel));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_antinfo =
+	__ATTR(ant, 0660, show_ant_info, set_ant_info);
+#endif /* MIMO_ANT_SETTING */
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+extern uint32 pmmode_val;
+static ssize_t
+show_pm_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (pmmode_val == 0xFF) {
+		ret = scnprintf(buf, PAGE_SIZE -1, "PM mode is not set\n");
+	} else {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", pmmode_val);
+	}
+	return ret;
+}
+
+static ssize_t
+set_pm_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long pm_val;
+
+	pm_val = bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%lu", &pm_val);
+
+	if (pm_val > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %lu \n",
+			__FUNCTION__, pm_val));
+		return -EINVAL;
+	}
+
+	pmmode_val = (uint32)pm_val;
+	DHD_ERROR(("[WIFI_SEC] %s: Set pminfo val = %u\n", __FUNCTION__, pmmode_val));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_pminfo =
+	__ATTR(pm, 0660, show_pm_info, set_pm_info);
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+
+#ifdef LOGTRACE_FROM_FILE
+unsigned long logtrace_val = 1;
+
+static ssize_t
+show_logtrace_info(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE -1, "%lu\n", logtrace_val);
+	return ret;
+}
+
+static ssize_t
+set_logtrace_info(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%lu", &onoff);
+
+	if (onoff > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %lu \n",
+			__FUNCTION__, onoff));
+		return -EINVAL;
+	}
+
+	logtrace_val = onoff;
+	DHD_ERROR(("[WIFI_SEC] %s: LOGTRACE On/Off from sysfs = %lu\n",
+		__FUNCTION__, logtrace_val));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_logtraceinfo =
+	__ATTR(logtrace, 0660, show_logtrace_info, set_logtrace_info);
+#endif /* LOGTRACE_FROM_FILE */
+
+#ifdef  USE_WFA_CERT_CONF
+#ifdef BCMSDIO
+uint32 bus_txglom = VALUENOTSET;
+
+static ssize_t
+show_bustxglom(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (bus_txglom == VALUENOTSET) {
+		ret = scnprintf(buf, PAGE_SIZE - 1, "%s\n", "bustxglom not set from sysfs");
+	} else {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", bus_txglom);
+	}
+	return ret;
+}
+
+static ssize_t
+set_bustxglom(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 onoff;
+
+	onoff = (uint32)bcm_atoi(buf);
+	sscanf(buf, "%u", &onoff);
+
+	if (onoff > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %u \n",
+			__FUNCTION__, onoff));
+		return -EINVAL;
+	}
+
+	bus_txglom = onoff;
+	DHD_ERROR(("[WIFI_SEC] %s: BUS TXGLOM On/Off from sysfs = %u\n",
+			__FUNCTION__, bus_txglom));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_bustxglom =
+	__ATTR(bustxglom, 0660, show_bustxglom, set_bustxglom);
+#endif /* BCMSDIO */
+
+#if defined(ROAM_ENABLE) || defined(DISABLE_BUILTIN_ROAM)
+uint32 roam_off = VALUENOTSET;
+
+static ssize_t
+show_roamoff(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (roam_off == VALUENOTSET) {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%s\n", "roam_off not set from sysfs");
+	} else {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", roam_off);
+	}
+	return ret;
+}
+
+static ssize_t
+set_roamoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 onoff;
+
+	onoff = bcm_atoi(buf);
+	sscanf(buf, "%u", &onoff);
+
+	if (onoff > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %u \n",
+			__FUNCTION__, onoff));
+		return -EINVAL;
+	}
+
+	roam_off = onoff;
+	DHD_ERROR(("[WIFI_SEC] %s: ROAM On/Off from sysfs = %u\n",
+		__FUNCTION__, roam_off));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_roamoff =
+	__ATTR(roamoff, 0660, show_roamoff, set_roamoff);
+#endif /* ROAM_ENABLE || DISABLE_BUILTIN_ROAM */
+
+#ifdef USE_WL_FRAMEBURST
+uint32 frameburst = VALUENOTSET;
+
+static ssize_t
+show_frameburst(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (frameburst == VALUENOTSET) {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%s\n", "frameburst not set from sysfs");
+	} else {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", frameburst);
+	}
+	return ret;
+}
+
+static ssize_t
+set_frameburst(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 onoff;
+
+	onoff = bcm_atoi(buf);
+	sscanf(buf, "%u", &onoff);
+
+	if (onoff > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %u \n",
+			__FUNCTION__, onoff));
+		return -EINVAL;
+	}
+
+	frameburst = onoff;
+	DHD_ERROR(("[WIFI_SEC] %s: FRAMEBURST On/Off from sysfs = %u\n",
+		__FUNCTION__, frameburst));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_frameburst =
+	__ATTR(frameburst, 0660, show_frameburst, set_frameburst);
+#endif /* USE_WL_FRAMEBURST */
+
+#ifdef USE_WL_TXBF
+uint32 txbf = VALUENOTSET;
+
+static ssize_t
+show_txbf(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (txbf == VALUENOTSET) {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%s\n", "txbf not set from sysfs");
+	} else {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", txbf);
+	}
+	return ret;
+}
+
+static ssize_t
+set_txbf(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 onoff;
+
+	onoff = bcm_atoi(buf);
+	sscanf(buf, "%u", &onoff);
+
+	if (onoff > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %u \n",
+			__FUNCTION__, onoff));
+		return -EINVAL;
+	}
+
+	txbf = onoff;
+	DHD_ERROR(("[WIFI_SEC] %s: FRAMEBURST On/Off from sysfs = %u\n",
+		__FUNCTION__, txbf));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_txbf =
+	__ATTR(txbf, 0660, show_txbf, set_txbf);
+#endif /* USE_WL_TXBF */
+
+#ifdef PROP_TXSTATUS
+uint32 proptx = VALUENOTSET;
+
+static ssize_t
+show_proptx(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	if (proptx == VALUENOTSET) {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%s\n", "proptx not set from sysfs");
+	} else {
+		ret = scnprintf(buf, PAGE_SIZE -1, "%u\n", proptx);
+	}
+	return ret;
+}
+
+static ssize_t
+set_proptx(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 onoff;
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%u", &onoff);
+
+	if (onoff > 2) {
+		DHD_ERROR(("[WIFI_SEC] %s: Set Invalid value %u \n",
+			__FUNCTION__, onoff));
+		return -EINVAL;
+	}
+
+	proptx = onoff;
+	DHD_ERROR(("[WIFI_SEC] %s: FRAMEBURST On/Off from sysfs = %u\n",
+		__FUNCTION__, txbf));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_proptx =
+	__ATTR(proptx, 0660, show_proptx, set_proptx);
+
+#endif /* PROP_TXSTATUS */
+#endif /* USE_WFA_CERT_CONF */
+#endif /* DHD_EXPORT_CNTL_FILE */
+
+#if defined(WL_BAM)
+#define BAD_AP_MAC_ADDR_ELEMENT_NUM	6
+#define MACF_READ	"%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx"
+wl_bad_ap_mngr_t *g_bad_ap_mngr = NULL;
+
+static ssize_t
+show_adps_bam_list(struct dhd_info *dev, char *buf)
+{
+	int offset = 0;
+	ssize_t ret = 0;
+
+	wl_bad_ap_info_t *bad_ap;
+	wl_bad_ap_info_entry_t *entry;
+
+	if (g_bad_ap_mngr == NULL)
+		return ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry(entry, &g_bad_ap_mngr->list, list) {
+		bad_ap = &entry->bad_ap;
+
+		ret = scnprintf(buf + offset, PAGE_SIZE - 1, MACF"\n",
+			bad_ap->bssid.octet[0], bad_ap->bssid.octet[1],
+			bad_ap->bssid.octet[2], bad_ap->bssid.octet[3],
+			bad_ap->bssid.octet[4], bad_ap->bssid.octet[5]);
+
+		offset += ret;
+	}
+	GCC_DIAGNOSTIC_POP();
+
+	return offset;
+}
+
+static ssize_t
+store_adps_bam_list(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int ret;
+	size_t len;
+	int offset;
+	char tmp[128];
+	wl_bad_ap_info_t bad_ap;
+
+	if (g_bad_ap_mngr == NULL)
+		return count;
+
+	len = count;
+	offset = 0;
+	do {
+		ret = sscanf(buf + offset, MACF_READ"\n",
+			&bad_ap.bssid.octet[0], &bad_ap.bssid.octet[1],
+			&bad_ap.bssid.octet[2], &bad_ap.bssid.octet[3],
+			&bad_ap.bssid.octet[4], &bad_ap.bssid.octet[5]);
+		if (ret != BAD_AP_MAC_ADDR_ELEMENT_NUM) {
+			DHD_ERROR(("%s - fail to parse bad ap data\n", __FUNCTION__));
+			return -EINVAL;
+		}
+
+		ret = wl_bad_ap_mngr_add(g_bad_ap_mngr, &bad_ap);
+		if (ret < 0)
+			return ret;
+
+		ret = snprintf(tmp, ARRAYSIZE(tmp), MACF"\n",
+			bad_ap.bssid.octet[0], bad_ap.bssid.octet[1],
+			bad_ap.bssid.octet[2], bad_ap.bssid.octet[3],
+			bad_ap.bssid.octet[4], bad_ap.bssid.octet[5]);
+		if (ret < 0) {
+			DHD_ERROR(("%s - fail to get bad ap data length(%d)\n", __FUNCTION__, ret));
+			return ret;
+		}
+
+		len -= ret;
+		offset += ret;
+	} while (len > 0);
+
+	return count;
+}
+
+static struct dhd_attr dhd_attr_adps_bam =
+	__ATTR(bad_ap_list, 0660, show_adps_bam_list, store_adps_bam_list);
+#endif	/* WL_BAM */
+
+uint32 report_hang_privcmd_err = 1;
+
+#ifdef DHD_SEND_HANG_PRIVCMD_ERRORS
+static ssize_t
+show_hang_privcmd_err(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%u\n", report_hang_privcmd_err);
+	return ret;
+}
+
+static ssize_t
+set_hang_privcmd_err(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 val;
+
+	val = bcm_atoi(buf);
+	sscanf(buf, "%u", &val);
+
+	report_hang_privcmd_err = val ? 1 : 0;
+	DHD_INFO(("%s: Set report HANG for private cmd error: %d\n",
+		__FUNCTION__, report_hang_privcmd_err));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_hang_privcmd_err =
+	__ATTR(hang_privcmd_err, 0660, show_hang_privcmd_err, set_hang_privcmd_err);
+#endif /* DHD_SEND_HANG_PRIVCMD_ERRORS */
+
+#if defined(SHOW_LOGTRACE)
+static ssize_t
+show_control_logtrace(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", control_logtrace);
+	return ret;
+}
+
+static ssize_t
+set_control_logtrace(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 val;
+
+	val = bcm_atoi(buf);
+
+	control_logtrace = val;
+	DHD_ERROR(("%s: Set control logtrace: %d\n", __FUNCTION__, control_logtrace));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_control_logtrace =
+__ATTR(control_logtrace, 0660, show_control_logtrace, set_control_logtrace);
+#endif /* SHOW_LOGTRACE */
+
+#if defined(DISABLE_HE_ENAB) || defined(CUSTOM_CONTROL_HE_ENAB)
+uint8 control_he_enab = 1;
+#endif /* DISABLE_HE_ENAB || CUSTOM_CONTROL_HE_ENAB */
+
+#ifdef RX_PKT_POOL
+static ssize_t
+show_max_rx_pkt_pool(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", dhd->rx_pkt_pool.max_size);
+	return ret;
+}
+
+static ssize_t
+set_max_rx_pkt_pool(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+
+	val = bcm_atoi(buf);
+
+	dhd->rx_pkt_pool.max_size = ((val > MAX_RX_PKT_POOL) &&
+		(val <= (MAX_RX_PKT_POOL * 8))) ? val : MAX_RX_PKT_POOL;
+	DHD_ERROR(("%s: MAX_RX_PKT_POOL: %d\n", __FUNCTION__, dhd->rx_pkt_pool.max_size));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_max_rx_pkt_pool=
+__ATTR(dhd_max_rx_pkt_pool, 0660, show_max_rx_pkt_pool, set_max_rx_pkt_pool);
+#endif /* RX_PKT_POOL */
+
+#ifdef PCIE_FULL_DONGLE
+static ssize_t
+dhd_set_aspm_enab(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	unsigned long aspm_enab;
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	aspm_enab = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &aspm_enab);
+	if (aspm_enab != 0 && aspm_enab != 1) {
+		return -EINVAL;
+	}
+	dhd_bus_aspm_enable_rc_ep(dhdp->bus, aspm_enab);
+
+	return count;
+}
+
+static ssize_t
+show_aspm_enab(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	bool aspm_enab;
+	dhd_pub_t *dhdp = &dhd->pub;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	aspm_enab = dhd_bus_is_aspm_enab_rc_ep(dhdp->bus);
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", aspm_enab);
+
+	return ret;
+}
+
+static struct dhd_attr dhd_attr_aspm_enab =
+__ATTR(aspm_enab, 0660, show_aspm_enab, dhd_set_aspm_enab);
+
+static ssize_t
+dhd_set_l1ss_enab(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	unsigned long l1ss_enab;
+	dhd_pub_t *dhdp = &dhd->pub;
+
+	l1ss_enab = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &l1ss_enab);
+	if (l1ss_enab != 0 && l1ss_enab != 1) {
+		return -EINVAL;
+	}
+	dhd_bus_l1ss_enable_rc_ep(dhdp->bus, l1ss_enab);
+	return count;
+}
+
+static ssize_t
+show_l1ss_enab(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	bool l1ss_enab;
+	dhd_pub_t *dhdp = &dhd->pub;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	l1ss_enab = dhd_bus_is_l1ss_enab_rc_ep(dhdp->bus);
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", l1ss_enab);
+
+	return ret;
+}
+
+static struct dhd_attr dhd_attr_l1ss_enab =
+__ATTR(l1ss_enab, 0660, show_l1ss_enab, dhd_set_l1ss_enab);
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef RPM_FAST_TRIGGER
+static ssize_t
+dhd_show_fast_rpm_thresh(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long val;
+
+	val = dhd_fast_runtimepm_ms;
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n", val);
+	return ret;
+}
+
+static ssize_t
+dhd_set_fast_rpm_thresh(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long val;
+
+	val = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &val);
+	/* Ensure fast RPM threashold is lesser than regular RPM threshold */
+	if (val == 0 || val >= dhd_runtimepm_ms) {
+		 return -EINVAL;
+	}
+	dhd_fast_runtimepm_ms = val;
+	return count;
+}
+static struct dhd_attr dhd_attr_fast_rpm_thresh =
+__ATTR(fast_rpm_thresh, 0660, dhd_show_fast_rpm_thresh, dhd_set_fast_rpm_thresh);
+#endif /* RPM_FAST_TRIGGER */
+
+#if defined(CUSTOM_CONTROL_HE_ENAB)
+static ssize_t
+show_control_he_enab(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", control_he_enab);
+	return ret;
+}
+
+static ssize_t
+set_control_he_enab(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 val;
+
+	val = bcm_atoi(buf);
+
+	control_he_enab = val ? 1 : 0;
+	DHD_ERROR(("%s: Set control he enab: %d\n", __FUNCTION__, control_he_enab));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_control_he_enab=
+__ATTR(control_he_enab, 0660, show_control_he_enab, set_control_he_enab);
+#endif /* CUSTOM_CONTROL_HE_ENAB */
+
+#if defined(WLAN_ACCEL_BOOT)
+static ssize_t
+show_wl_accel_force_reg_on(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", dhd->wl_accel_force_reg_on);
+	return ret;
+}
+
+static ssize_t
+set_wl_accel_force_reg_on(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+
+	val = bcm_atoi(buf);
+
+	dhd->wl_accel_force_reg_on = val ? 1 : 0;
+	DHD_ERROR(("%s: wl_accel_force_reg_on: %d\n", __FUNCTION__, dhd->wl_accel_force_reg_on));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_wl_accel_force_reg_on=
+__ATTR(wl_accel_force_reg_on, 0660, show_wl_accel_force_reg_on, set_wl_accel_force_reg_on);
+#endif /* WLAN_ACCEL_BOOT */
+
+#if defined(AGG_H2D_DB)
+extern bool agg_h2d_db_enab;
+extern uint32 agg_h2d_db_timeout;
+extern uint32 agg_h2d_db_inflight_thresh;
+
+static ssize_t
+show_agg_h2d_db_enab(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", agg_h2d_db_enab);
+	return ret;
+}
+
+static ssize_t
+set_agg_h2d_db_enab(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+
+	val = bcm_atoi(buf);
+
+	agg_h2d_db_enab = val ? TRUE : FALSE;
+	DHD_ERROR(("%s: agg_h2d_db_timeout: %d\n", __FUNCTION__, agg_h2d_db_enab));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_agg_h2d_db_enab =
+__ATTR(agg_h2d_db_enab, 0660, show_agg_h2d_db_enab, set_agg_h2d_db_enab);
+
+static ssize_t
+show_agg_h2d_db_inflight_thresh(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", agg_h2d_db_inflight_thresh);
+	return ret;
+}
+
+static ssize_t
+set_agg_h2d_db_inflight_thresh(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+
+	val = bcm_atoi(buf);
+
+	agg_h2d_db_inflight_thresh = val;
+	DHD_ERROR(("%s: agg_h2d_db_timeout: %d\n", __FUNCTION__, agg_h2d_db_inflight_thresh));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_agg_h2d_db_inflight_thresh =
+__ATTR(agg_h2d_db_inflight_thresh, 0660, show_agg_h2d_db_inflight_thresh,
+	set_agg_h2d_db_inflight_thresh);
+
+static ssize_t
+show_agg_h2d_db_timeout(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", agg_h2d_db_timeout);
+	return ret;
+}
+
+static ssize_t
+set_agg_h2d_db_timeout(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+
+	val = bcm_atoi(buf);
+
+	agg_h2d_db_timeout = val;
+	DHD_ERROR(("%s: agg_h2d_db_timeout: %d\n", __FUNCTION__, agg_h2d_db_timeout));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_agg_h2d_db_timeout =
+__ATTR(agg_h2d_db_timeout, 0660, show_agg_h2d_db_timeout, set_agg_h2d_db_timeout);
+#endif /* WLAN_ACCEL_BOOT */
+/*
+ * Dumps the lock and other state information useful for debug
+ *
+ */
+static ssize_t
+dhd_debug_dump_stateinfo(struct dhd_info *dhd, char *buf)
+{
+	u32 buf_size = PAGE_SIZE - 1;
+	u8 *ptr = buf;
+	ssize_t len = 0;
+
+	len += scnprintf(ptr, buf_size, "[DHD]\nlock info:\n");
+#ifdef BT_OVER_SDIO
+	len += scnprintf((ptr+len), (buf_size-len), "bus_user_lock:\n",
+			mutex_is_locked(&dhd->bus_user_lock));
+#endif /* BT_OVER_SDIO */
+
+#ifdef WL_CFG80211
+	len += wl_cfg80211_debug_data_dump(dhd_linux_get_primary_netdev(&dhd->pub),
+			(ptr + len), (buf_size - len));
+#endif /* WL_CFG80211 */
+
+	/* Ensure buffer ends with null char */
+	buf[len] = '\0';
+	return len + 1;
+}
+static struct dhd_attr dhd_attr_dhd_debug_data =
+__ATTR(dump_stateinfo, 0660, dhd_debug_dump_stateinfo, NULL);
+
+#ifdef WL_CFG80211
+#define SUBLOGLEVEL 20
+#define SUBLOGLEVELZ ((SUBLOGLEVEL) + (1))
+
+static const struct {
+	u32 log_level;
+	char *sublogname;
+} sublogname_map[] = {
+	{WL_DBG_ERR, "ERR"},
+	{WL_DBG_INFO, "INFO"},
+	{WL_DBG_DBG, "DBG"},
+	{WL_DBG_SCAN, "SCAN"},
+	{WL_DBG_TRACE, "TRACE"},
+	{WL_DBG_P2P_ACTION, "P2PACTION"},
+	{WL_DBG_PNO, "PNO"}
+};
+
+/**
+* Format : echo "SCAN:1 DBG:1" > /sys/wifi/wl_dbg_level
+* to turn on SCAN and DBG log.
+* To turn off SCAN partially, echo "SCAN:0" > /sys/wifi/wl_dbg_level
+* To see current setting of debug level,
+* cat /sys/wifi/wl_dbg_level
+*/
+static ssize_t
+show_wl_debug_level(struct dhd_info *dhd, char *buf)
+{
+	char *param;
+	char tbuf[SUBLOGLEVELZ * ARRAYSIZE(sublogname_map)];
+	uint i;
+	ssize_t ret = 0;
+
+	bzero(tbuf, sizeof(tbuf));
+	param = &tbuf[0];
+	for (i = 0; i < ARRAYSIZE(sublogname_map); i++) {
+		param += snprintf(param, sizeof(tbuf) - 1, "%s:%d ",
+			sublogname_map[i].sublogname,
+			(wl_dbg_level & sublogname_map[i].log_level) ? 1 : 0);
+	}
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%s \n", tbuf);
+	return ret;
+}
+
+static ssize_t
+set_wl_debug_level(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	char tbuf[SUBLOGLEVELZ * ARRAYSIZE(sublogname_map)], sublog[SUBLOGLEVELZ];
+	char *params, *token, *colon;
+	uint i, tokens, log_on = 0;
+	size_t minsize = min_t(size_t, (sizeof(tbuf) - 1), count);
+
+	bzero(tbuf, sizeof(tbuf));
+	bzero(sublog, sizeof(sublog));
+	strlcpy(tbuf, buf, minsize);
+
+	DHD_INFO(("current wl_dbg_level %d \n", wl_dbg_level));
+
+	tbuf[minsize] = '\0';
+	params = &tbuf[0];
+	colon = strchr(params, '\n');
+	if (colon != NULL)
+		*colon = '\0';
+	while ((token = strsep(&params, " ")) != NULL) {
+		bzero(sublog, sizeof(sublog));
+		if (token == NULL || !*token)
+			break;
+		if (*token == '\0')
+			continue;
+		colon = strchr(token, ':');
+		if (colon != NULL) {
+			*colon = ' ';
+		}
+		tokens = sscanf(token, "%"SIZE_CONST_STRING(SUBLOGLEVEL)"s %u", sublog, &log_on);
+		if (colon != NULL)
+			*colon = ':';
+
+		if (tokens == 2) {
+				for (i = 0; i < ARRAYSIZE(sublogname_map); i++) {
+					if (!strncmp(sublog, sublogname_map[i].sublogname,
+						strlen(sublogname_map[i].sublogname))) {
+						if (log_on) {
+							wl_dbg_level |=
+							(sublogname_map[i].log_level);
+							wl_log_level |=
+							(sublogname_map[i].log_level);
+						} else {
+							wl_dbg_level &=
+							~(sublogname_map[i].log_level);
+							wl_log_level &=
+							~(sublogname_map[i].log_level);
+						}
+					}
+				}
+		} else
+			WL_ERR(("%s: can't parse '%s' as a "
+			       "SUBMODULE:LEVEL (%d tokens)\n",
+			       tbuf, token, tokens));
+
+	}
+	DHD_INFO(("changed wl_dbg_level %d \n", wl_dbg_level));
+	return count;
+}
+
+static struct dhd_attr dhd_attr_wl_dbg_level =
+__ATTR(wl_dbg_level, 0660, show_wl_debug_level, set_wl_debug_level);
+#endif /* WL_CFG80211 */
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+#define DUMP_TRIGGER	1
+
+static ssize_t
+show_dhd_dump_in_progress(struct dhd_info *dhd, char *buf)
+{
+	size_t ret = 0;
+	dhd_dongledump_status_t dump_status;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dump_status = dhd_get_dump_status(&dhd->pub);
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d \n", dump_status);
+
+	return ret;
+}
+
+static ssize_t
+set_dhd_dump_in_progress(struct dhd_info *dhd, const char *buf, size_t count)
+{
+	uint32 input;
+	dhd_dongledump_status_t dump_status;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return count;
+	}
+
+	dump_status = dhd_get_dump_status(&dhd->pub);
+	if (dump_status == DUMP_NOT_READY || dump_status == DUMP_IN_PROGRESS) {
+		DHD_ERROR(("%s: Could not start dongle dump: %d\n",
+			__FUNCTION__, dump_status));
+		goto exit;
+	}
+
+	input = bcm_atoi(buf);
+	if (input == DUMP_TRIGGER) {
+		DHD_INFO(("%s: Trigger dongle dump\n", __FUNCTION__));
+		dhd_set_dump_status(&dhd->pub, DUMP_IN_PROGRESS);
+		schedule_work(&dhd->dhd_dump_proc_work);
+	}
+	else {
+		DHD_ERROR(("%s: Invalid value %d\n", __FUNCTION__, input));
+	}
+
+exit:
+	return count;
+}
+
+static struct dhd_attr dhd_attr_dump_in_progress =
+__ATTR(dump_in_progress, 0660, show_dhd_dump_in_progress, set_dhd_dump_in_progress);
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+#ifdef DHD_DUMP_START_COMMAND
+static ssize_t
+trigger_dhd_dump_start_command(struct dhd_info *dhd, char *buf)
+{
+	ssize_t ret = 0;
+	DHD_ERROR(("%s: dump_start command delivered.\n", __FUNCTION__));
+	return ret;
+}
+
+static struct dhd_attr dhd_attr_dump_start_command =
+	__ATTR(dump_start, 0664, trigger_dhd_dump_start_command, NULL);
+#endif /* DHD_DUMP_START_COMMAND */
+
+/* Attribute object that gets registered with "wifi" kobject tree */
+static struct attribute *default_file_attrs[] = {
+#ifdef DHD_MAC_ADDR_EXPORT
+	&dhd_attr_macaddr.attr,
+#endif /* DHD_MAC_ADDR_EXPORT */
+#ifdef DHD_EXPORT_CNTL_FILE
+#ifdef DHD_FW_COREDUMP
+	&dhd_attr_memdump.attr,
+#endif /* DHD_FW_COREDUMP */
+#ifdef BCMASSERT_LOG
+	&dhd_attr_assert.attr,
+#endif /* BCMASSERT_LOG */
+#ifdef WRITE_WLANINFO
+	&dhd_attr_wifiver.attr,
+#endif /* WRITE_WLANINFO */
+#if defined(USE_CID_CHECK) || defined(USE_DIRECT_VID_TAG)
+	&dhd_attr_cidinfo.attr,
+#endif /* USE_CID_CHECK || USE_DIRECT_VID_TAG */
+#ifdef GEN_SOFTAP_INFO_FILE
+	&dhd_attr_softapinfo.attr,
+#endif /* GEN_SOFTAP_INFO_FILE */
+#ifdef MIMO_ANT_SETTING
+	&dhd_attr_antinfo.attr,
+#endif /* MIMO_ANT_SETTING */
+#ifdef DHD_PM_CONTROL_FROM_FILE
+	&dhd_attr_pminfo.attr,
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+#ifdef LOGTRACE_FROM_FILE
+	&dhd_attr_logtraceinfo.attr,
+#endif /* LOGTRACE_FROM_FILE */
+#ifdef USE_WFA_CERT_CONF
+#ifdef BCMSDIO
+	&dhd_attr_bustxglom.attr,
+#endif /* BCMSDIO */
+	&dhd_attr_roamoff.attr,
+#ifdef USE_WL_FRAMEBURST
+	&dhd_attr_frameburst.attr,
+#endif /* USE_WL_FRAMEBURST */
+#ifdef USE_WL_TXBF
+	&dhd_attr_txbf.attr,
+#endif /* USE_WL_TXBF */
+#ifdef PROP_TXSTATUS
+	&dhd_attr_proptx.attr,
+#endif /* PROP_TXSTATUS */
+#endif /* USE_WFA_CERT_CONF */
+#endif /* DHD_EXPORT_CNTL_FILE */
+#if defined(WL_BAM)
+	&dhd_attr_adps_bam.attr,
+#endif	/* WL_BAM */
+#ifdef DHD_SEND_HANG_PRIVCMD_ERRORS
+	&dhd_attr_hang_privcmd_err.attr,
+#endif /* DHD_SEND_HANG_PRIVCMD_ERRORS */
+#if defined(SHOW_LOGTRACE)
+	&dhd_attr_control_logtrace.attr,
+#endif /* SHOW_LOGTRACE */
+#if defined(DHD_TRACE_WAKE_LOCK)
+	&dhd_attr_wklock.attr,
+#endif
+#ifdef DHD_LOG_DUMP
+	&dhd_attr_logdump_periodic_flush.attr,
+	&dhd_attr_logdump_ecntr.attr,
+#endif
+	&dhd_attr_ecounters.attr,
+#ifdef DHD_SSSR_DUMP
+	&dhd_attr_sssr_enab.attr,
+	&dhd_attr_fis_enab.attr,
+#endif /* DHD_SSSR_DUMP */
+	&dhd_attr_firmware_path.attr,
+	&dhd_attr_nvram_path.attr,
+#if defined(CUSTOM_CONTROL_HE_ENAB)
+	&dhd_attr_control_he_enab.attr,
+#endif /* CUSTOM_CONTROL_HE_ENAB */
+#if defined(WLAN_ACCEL_BOOT)
+	&dhd_attr_wl_accel_force_reg_on.attr,
+#endif /* WLAN_ACCEL_BOOT */
+#ifdef PWRSTATS_SYSFS
+	&dhd_attr_pwrstats_path.attr,
+#endif /* PWRSTATS_SYSFS */
+#if defined(WL_CFG80211)
+	&dhd_attr_wl_dbg_level.attr,
+#endif /* WL_CFG80211 */
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	&dhd_attr_dump_in_progress.attr,
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+#ifdef DHD_DUMP_START_COMMAND
+	&dhd_attr_dump_start_command.attr,
+#endif /* DHD_DUMP_START_COMMAND */
+	&dhd_attr_dhd_debug_data.attr,
+#if defined(AGG_H2D_DB)
+	&dhd_attr_agg_h2d_db_enab.attr,
+	&dhd_attr_agg_h2d_db_inflight_thresh.attr,
+	&dhd_attr_agg_h2d_db_timeout.attr,
+#endif /* AGG_H2D_DB */
+#if defined(RX_PKT_POOL)
+	&dhd_attr_max_rx_pkt_pool.attr,
+#endif /* RX_PKT_POOL */
+#ifdef PCIE_FULL_DONGLE
+	&dhd_attr_aspm_enab.attr,
+	&dhd_attr_l1ss_enab.attr,
+#endif /* PCIE_FULL_DONGLE */
+#ifdef RPM_FAST_TRIGGER
+	&dhd_attr_fast_rpm_thresh.attr,
+#endif /* RPM_FAST_TRIGGER */
+	&dhd_attr_sig_path.attr,
+	&dhd_attr_tcm_test_mode.attr,
+	NULL
+};
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+static const struct attribute_group default_group = {
+	.attrs = default_file_attrs
+};
+static const struct attribute_group *default_groups[] = {
+	&default_group,
+	NULL
+};
+#endif /* LINUX_VER >= 5.18 */
+/*
+ * wifi kobject show function, the "attr" attribute specifices to which
+ * node under "sys/wifi" the show function is called.
+ */
+static ssize_t dhd_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+	dhd_info_t *dhd;
+	struct dhd_attr *d_attr;
+	int ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = to_dhd(kobj);
+	d_attr = to_attr(attr);
+	GCC_DIAGNOSTIC_POP();
+
+	if (d_attr->show)
+		ret = d_attr->show(dhd, buf);
+	else
+		ret = -EIO;
+
+	return ret;
+}
+
+/*
+ * wifi kobject show function, the "attr" attribute specifices to which
+ * node under "sys/wifi" the store function is called.
+ */
+static ssize_t dhd_store(struct kobject *kobj, struct attribute *attr,
+	const char *buf, size_t count)
+{
+	dhd_info_t *dhd;
+	struct dhd_attr *d_attr;
+	int ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = to_dhd(kobj);
+	d_attr = to_attr(attr);
+	GCC_DIAGNOSTIC_POP();
+
+	if (d_attr->store)
+		ret = d_attr->store(dhd, buf, count);
+	else
+		ret = -EIO;
+
+	return ret;
+
+}
+
+static const struct sysfs_ops dhd_sysfs_ops = {
+	.show = dhd_show,
+	.store = dhd_store,
+};
+
+static struct kobj_type dhd_ktype = {
+	.sysfs_ops = &dhd_sysfs_ops,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+	.default_groups = default_groups,
+#else
+	.default_attrs = default_file_attrs,
+#endif /* LINUX_VER >= 5.18 */
+};
+
+/*
+ * sysfs for dhd_lb
+ */
+#ifdef DHD_LB
+#if defined(DHD_LB_TXP)
+static ssize_t
+show_lbtxp(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long onoff;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+
+	onoff = atomic_read(&dhd->lb_txp_active);
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n",
+		onoff);
+	return ret;
+}
+
+static ssize_t
+lbtxp_onoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	int i;
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &onoff);
+	if (onoff != 0 && onoff != 1) {
+		return -EINVAL;
+	}
+	atomic_set(&dhd->lb_txp_active, onoff);
+
+	/* Since the scheme is changed clear the counters */
+	for (i = 0; i < NR_CPUS; i++) {
+		DHD_LB_STATS_CLR(dhd->txp_percpu_run_cnt[i]);
+		DHD_LB_STATS_CLR(dhd->tx_start_percpu_run_cnt[i]);
+	}
+
+	return count;
+}
+
+static struct dhd_attr dhd_attr_lbtxp =
+	__ATTR(lbtxp, 0660, show_lbtxp, lbtxp_onoff);
+#endif /* DHD_LB_TXP */
+
+#if defined(DHD_LB_RXP)
+static ssize_t
+show_lbrxp(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+	unsigned long onoff;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+
+	onoff = atomic_read(&dhd->lb_rxp_active);
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%lu \n",
+		onoff);
+	return ret;
+}
+
+static ssize_t
+lbrxp_onoff(struct dhd_info *dev, const char *buf, size_t count)
+{
+	unsigned long onoff;
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+
+	onoff = bcm_strtoul(buf, NULL, 10);
+
+	sscanf(buf, "%lu", &onoff);
+	if (onoff != 0 && onoff != 1) {
+		return -EINVAL;
+	}
+	atomic_set(&dhd->lb_rxp_active, onoff);
+
+	return count;
+}
+static struct dhd_attr dhd_attr_lbrxp =
+	__ATTR(lbrxp, 0660, show_lbrxp, lbrxp_onoff);
+
+static ssize_t
+get_lb_rxp_stop_thr(struct dhd_info *dev, char *buf)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	dhd_pub_t *dhdp;
+	ssize_t ret = 0;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+	dhdp = &dhd->pub;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%u \n",
+		(dhdp->lb_rxp_stop_thr / D2HRING_RXCMPLT_MAX_ITEM));
+	return ret;
+}
+
+#define ONE_GB (1024 * 1024 * 1024)
+
+static ssize_t
+set_lb_rxp_stop_thr(struct dhd_info *dev, const char *buf, size_t count)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	dhd_pub_t *dhdp;
+	uint32 lb_rxp_stop_thr;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+	dhdp = &dhd->pub;
+
+	lb_rxp_stop_thr = (uint32)bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%u", &lb_rxp_stop_thr);
+
+	/* disable lb_rxp flow ctrl */
+	if (lb_rxp_stop_thr == 0) {
+		dhdp->lb_rxp_stop_thr = 0;
+		dhdp->lb_rxp_strt_thr = 0;
+		atomic_set(&dhd->pub.lb_rxp_flow_ctrl, FALSE);
+		return count;
+	}
+	/* 1. by the time lb_rxp_stop_thr gets into picture,
+	 * DHD RX path should not consume more than 1GB
+	 * 2. lb_rxp_stop_thr should always be more than dhdp->lb_rxp_strt_thr
+	 */
+	if (((lb_rxp_stop_thr *
+		D2HRING_RXCMPLT_MAX_ITEM *
+		dhd_prot_get_rxbufpost_sz(dhdp)) > ONE_GB) ||
+		(lb_rxp_stop_thr <= (dhdp->lb_rxp_strt_thr / D2HRING_RXCMPLT_MAX_ITEM))) {
+		return -EINVAL;
+	}
+
+	dhdp->lb_rxp_stop_thr = (D2HRING_RXCMPLT_MAX_ITEM * lb_rxp_stop_thr);
+	return count;
+}
+
+static struct dhd_attr dhd_attr_lb_rxp_stop_thr =
+	__ATTR(lbrxp_stop_thr, 0660, get_lb_rxp_stop_thr, set_lb_rxp_stop_thr);
+
+static ssize_t
+get_lb_rxp_strt_thr(struct dhd_info *dev, char *buf)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	dhd_pub_t *dhdp;
+	ssize_t ret = 0;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+	dhdp = &dhd->pub;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%u \n",
+		(dhdp->lb_rxp_strt_thr / D2HRING_RXCMPLT_MAX_ITEM));
+	return ret;
+}
+
+static ssize_t
+set_lb_rxp_strt_thr(struct dhd_info *dev, const char *buf, size_t count)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dev;
+	dhd_pub_t *dhdp;
+	uint32 lb_rxp_strt_thr;
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+	dhdp = &dhd->pub;
+
+	lb_rxp_strt_thr = (uint32)bcm_strtoul(buf, NULL, 10);
+	sscanf(buf, "%u", &lb_rxp_strt_thr);
+
+	/* disable lb_rxp flow ctrl */
+	if (lb_rxp_strt_thr == 0) {
+		dhdp->lb_rxp_strt_thr = 0;
+		dhdp->lb_rxp_stop_thr = 0;
+		atomic_set(&dhd->pub.lb_rxp_flow_ctrl, FALSE);
+		return count;
+	}
+	/* should be less than dhdp->lb_rxp_stop_thr */
+	if ((lb_rxp_strt_thr <= 0) ||
+		(lb_rxp_strt_thr >= (dhdp->lb_rxp_stop_thr / D2HRING_RXCMPLT_MAX_ITEM))) {
+		return -EINVAL;
+	}
+	dhdp->lb_rxp_strt_thr = (D2HRING_RXCMPLT_MAX_ITEM * lb_rxp_strt_thr);
+	return count;
+}
+static struct dhd_attr dhd_attr_lb_rxp_strt_thr =
+	__ATTR(lbrxp_strt_thr, 0660, get_lb_rxp_strt_thr, set_lb_rxp_strt_thr);
+
+#endif /* DHD_LB_RXP */
+
+static ssize_t
+show_candidacy_override(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1,
+			"%d\n", (int)dev->dhd_lb_candidacy_override);
+	return ret;
+}
+
+static ssize_t
+set_candidacy_override(struct dhd_info *dev, const char *buf, size_t count)
+{
+
+	int val = 0;
+	val = bcm_atoi(buf);
+
+	if (val > 0) {
+		dev->dhd_lb_candidacy_override = TRUE;
+	} else {
+		dev->dhd_lb_candidacy_override = FALSE;
+	}
+
+	DHD_ERROR(("set dhd_lb_candidacy_override %d\n", dev->dhd_lb_candidacy_override));
+	return count;
+}
+
+static struct dhd_attr dhd_candidacy_override =
+__ATTR(candidacy_override, 0660, show_candidacy_override, set_candidacy_override);
+
+static ssize_t
+show_primary_mask(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1,
+			"%02lx\n", *cpumask_bits(dev->cpumask_primary));
+	return ret;
+}
+
+static ssize_t
+set_primary_mask(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int ret;
+
+	cpumask_var_t primary_mask;
+
+	if (!alloc_cpumask_var(&primary_mask, GFP_KERNEL)) {
+		DHD_ERROR(("Can't allocate cpumask vars\n"));
+		return count;
+	}
+
+	cpumask_clear(primary_mask);
+	ret = cpumask_parse(buf, primary_mask);
+	if (ret < 0) {
+		DHD_ERROR(("Setting cpumask failed ret = %d\n", ret));
+		return count;
+	}
+
+	cpumask_clear(dev->cpumask_primary);
+	cpumask_or(dev->cpumask_primary, dev->cpumask_primary, primary_mask);
+
+	DHD_ERROR(("set cpumask results cpumask_primary 0x%2lx\n",
+		*cpumask_bits(dev->cpumask_primary)));
+
+	dhd_select_cpu_candidacy(dev);
+	return count;
+}
+
+static struct dhd_attr dhd_primary_mask =
+__ATTR(primary_mask, 0660, show_primary_mask, set_primary_mask);
+
+static ssize_t
+show_secondary_mask(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1,
+			"%02lx\n", *cpumask_bits(dev->cpumask_secondary));
+	return ret;
+}
+
+static ssize_t
+set_secondary_mask(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int ret;
+
+	cpumask_var_t secondary_mask;
+
+	if (!alloc_cpumask_var(&secondary_mask, GFP_KERNEL)) {
+		DHD_ERROR(("Can't allocate cpumask vars\n"));
+		return count;
+	}
+
+	cpumask_clear(secondary_mask);
+
+	ret = cpumask_parse(buf, secondary_mask);
+
+	if (ret < 0) {
+		DHD_ERROR(("Setting cpumask failed ret = %d\n", ret));
+		return count;
+	}
+
+	cpumask_clear(dev->cpumask_secondary);
+	cpumask_or(dev->cpumask_secondary, dev->cpumask_secondary, secondary_mask);
+
+	DHD_ERROR(("set cpumask results cpumask_secondary 0x%2lx\n",
+		*cpumask_bits(dev->cpumask_secondary)));
+
+	dhd_select_cpu_candidacy(dev);
+
+	return count;
+}
+
+static struct dhd_attr dhd_secondary_mask =
+__ATTR(secondary_mask, 0660, show_secondary_mask, set_secondary_mask);
+
+static ssize_t
+show_rx_cpu(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", atomic_read(&dev->rx_napi_cpu));
+	return ret;
+}
+
+static ssize_t
+set_rx_cpu(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dev->dhd_lb_candidacy_override) {
+		DHD_ERROR(("dhd_lb_candidacy_override is required %d\n",
+			dev->dhd_lb_candidacy_override));
+		return count;
+	}
+
+	val = (uint32)bcm_atoi(buf);
+	if (val >= nr_cpu_ids)
+	{
+		DHD_ERROR(("%s : can't set the value out of number of cpus, val = %u\n",
+			__FUNCTION__, val));
+	}
+
+	atomic_set(&dev->rx_napi_cpu, val);
+	DHD_ERROR(("%s: rx_napi_cpu = %d\n", __FUNCTION__, atomic_read(&dev->rx_napi_cpu)));
+	return count;
+}
+
+static struct dhd_attr dhd_rx_cpu =
+__ATTR(rx_cpu, 0660, show_rx_cpu, set_rx_cpu);
+
+static ssize_t
+show_tx_cpu(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", atomic_read(&dev->tx_cpu));
+	return ret;
+}
+
+static ssize_t
+set_tx_cpu(struct dhd_info *dev, const char *buf, size_t count)
+{
+	uint32 val;
+
+	if (!dev->dhd_lb_candidacy_override) {
+		DHD_ERROR(("dhd_lb_candidacy_override is required %d\n",
+			dev->dhd_lb_candidacy_override));
+		return count;
+	}
+
+	val = (uint32)bcm_atoi(buf);
+	if (val >= nr_cpu_ids)
+	{
+		DHD_ERROR(("%s : can't set the value out of number of cpus, val = %u\n",
+			__FUNCTION__, val));
+		return count;
+	}
+
+	atomic_set(&dev->tx_cpu, val);
+	DHD_ERROR(("%s: tx_cpu = %d\n", __FUNCTION__, atomic_read(&dev->tx_cpu)));
+	return count;
+}
+
+static struct dhd_attr dhd_tx_cpu =
+__ATTR(tx_cpu, 0660, show_tx_cpu, set_tx_cpu);
+
+static struct attribute *debug_lb_attrs[] = {
+#if defined(DHD_LB_TXP)
+	&dhd_attr_lbtxp.attr,
+#endif /* DHD_LB_TXP */
+#if defined(DHD_LB_RXP)
+	&dhd_attr_lbrxp.attr,
+	&dhd_attr_lb_rxp_stop_thr.attr,
+	&dhd_attr_lb_rxp_strt_thr.attr,
+#endif /* DHD_LB_RXP */
+	&dhd_candidacy_override.attr,
+	&dhd_primary_mask.attr,
+	&dhd_secondary_mask.attr,
+	&dhd_rx_cpu.attr,
+	&dhd_tx_cpu.attr,
+	NULL
+};
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+static const struct attribute_group debug_lb_group = {
+	.attrs = debug_lb_attrs
+};
+static const struct attribute_group *debug_lb_groups[] = {
+	&debug_lb_group,
+	NULL
+};
+#endif /* LINUX_VER >= 5.18 */
+#define to_dhd_lb(k) container_of(k, struct dhd_info, dhd_lb_kobj)
+
+/*
+ * wifi/lb kobject show function, the "attr" attribute specifices to which
+ * node under "sys/wifi/lb" the show function is called.
+ */
+static ssize_t dhd_lb_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+	dhd_info_t *dhd;
+	struct dhd_attr *d_attr;
+	int ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = to_dhd_lb(kobj);
+	d_attr = to_attr(attr);
+	GCC_DIAGNOSTIC_POP();
+
+	if (d_attr->show)
+		ret = d_attr->show(dhd, buf);
+	else
+		ret = -EIO;
+
+	return ret;
+}
+
+/*
+ * wifi kobject show function, the "attr" attribute specifices to which
+ * node under "sys/wifi/lb" the store function is called.
+ */
+static ssize_t dhd_lb_store(struct kobject *kobj, struct attribute *attr,
+		const char *buf, size_t count)
+{
+	dhd_info_t *dhd;
+	struct dhd_attr *d_attr;
+	int ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = to_dhd_lb(kobj);
+	d_attr = to_attr(attr);
+	GCC_DIAGNOSTIC_POP();
+
+	if (d_attr->store)
+		ret = d_attr->store(dhd, buf, count);
+	else
+		ret = -EIO;
+
+	return ret;
+
+}
+
+static const struct sysfs_ops dhd_sysfs_lb_ops = {
+	.show = dhd_lb_show,
+	.store = dhd_lb_store,
+};
+
+static struct kobj_type dhd_lb_ktype = {
+	.sysfs_ops = &dhd_sysfs_lb_ops,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+	.default_groups = debug_lb_groups,
+#else
+	.default_attrs = debug_lb_attrs,
+#endif /* LINUX_VER >= 5.18 */
+};
+#endif /* DHD_LB */
+
+#ifdef BCMPCIE
+#define CONST_SYNA_DHD_KOBJ_NAME       "wifi_pcie"
+#define CONST_SYNA_DHD_CSI_OBJ_NAME    "csi"
+#define CONST_SYNA_DHD_LB_OBJ_NAME     "lb_pcie"
+#else /* BCMSDIO */
+#define CONST_SYNA_DHD_KOBJ_NAME       "wifi_sdio"
+#define CONST_SYNA_DHD_CSI_OBJ_NAME    "csi"
+#define CONST_SYNA_DHD_LB_OBJ_NAME     "lb_sdio"
+#endif /* BCMPCIE */
+
+/*
+ * ************ DPC BOUNDS *************
+ */
+
+static ssize_t
+show_ctrl_cpl_post_bound(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n",
+		dhd_prot_get_ctrl_cpl_post_bound(&dev->pub));
+	return ret;
+}
+
+static ssize_t
+set_ctrl_cpl_post_bound(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int val;
+
+	val = (uint32)bcm_atoi(buf);
+	if (val <= 0)
+	{
+		DHD_ERROR(("%s : invalid ctrl_cpl_post_bound %u\n",
+			__FUNCTION__, val));
+		return count;
+	}
+
+	dhd_prot_set_ctrl_cpl_post_bound(&dev->pub, val);
+	return count;
+}
+
+static ssize_t
+show_tx_post_bound(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", dhd_prot_get_tx_post_bound(&dev->pub));
+	return ret;
+}
+
+static ssize_t
+set_tx_post_bound(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int val;
+
+	val = (uint32)bcm_atoi(buf);
+	if (val <= 0)
+	{
+		DHD_ERROR(("%s : invalid tx_post_bound %u\n",
+			__FUNCTION__, val));
+		return count;
+	}
+
+	dhd_prot_set_tx_post_bound(&dev->pub, val);
+	return count;
+}
+
+static ssize_t
+show_rx_cpl_post_bound(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n",
+		dhd_prot_get_rx_cpl_post_bound(&dev->pub));
+	return ret;
+}
+
+static ssize_t
+set_rx_cpl_post_bound(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int val;
+
+	val = (uint32)bcm_atoi(buf);
+	if (val <= 0)
+	{
+		DHD_ERROR(("%s : invalid rx_cpl_post_bound %u\n",
+			__FUNCTION__, val));
+		return count;
+	}
+
+	dhd_prot_set_rx_cpl_post_bound(&dev->pub, val);
+	return count;
+}
+
+static ssize_t
+show_tx_cpl_bound(struct dhd_info *dev, char *buf)
+{
+	ssize_t ret = 0;
+
+	ret = scnprintf(buf, PAGE_SIZE - 1, "%d\n", dhd_prot_get_tx_cpl_bound(&dev->pub));
+	return ret;
+}
+
+static ssize_t
+set_tx_cpl_bound(struct dhd_info *dev, const char *buf, size_t count)
+{
+	int val;
+
+	val = (uint32)bcm_atoi(buf);
+	if (val <= 0)
+	{
+		DHD_ERROR(("%s : invalid tx_cpl_bound %u\n",
+			__FUNCTION__, val));
+		return count;
+	}
+
+	dhd_prot_set_tx_cpl_bound(&dev->pub, val);
+	return count;
+}
+
+static struct dhd_attr dhd_attr_ctrl_cpl_post_bound =
+__ATTR(ctrl_cpl_post_bound, 0660, show_ctrl_cpl_post_bound, set_ctrl_cpl_post_bound);
+static struct dhd_attr dhd_attr_tx_post_bound =
+__ATTR(tx_post_bound, 0660, show_tx_post_bound, set_tx_post_bound);
+static struct dhd_attr dhd_attr_rx_cpl_post_bound =
+__ATTR(rx_cpl_post_bound, 0660, show_rx_cpl_post_bound, set_rx_cpl_post_bound);
+static struct dhd_attr dhd_attr_tx_cpl_bound =
+__ATTR(tx_cpl_bound, 0660, show_tx_cpl_bound, set_tx_cpl_bound);
+
+static struct attribute *debug_dpc_bounds_attrs[] = {
+	&dhd_attr_tx_cpl_bound.attr,
+	&dhd_attr_rx_cpl_post_bound.attr,
+	&dhd_attr_tx_post_bound.attr,
+	&dhd_attr_ctrl_cpl_post_bound.attr,
+	NULL
+};
+
+#define to_dhd_dpc_bounds(k) container_of(k, struct dhd_info, dhd_dpc_bounds_kobj)
+
+/*
+ * wifi/dpc_bounds kobject show function, the "attr" attribute specifices to which
+ * node under "sys/wifi/dpc_bounds" the show function is called.
+ */
+static ssize_t dhd_dpc_bounds_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+	dhd_info_t *dhd;
+	struct dhd_attr *d_attr;
+	int ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = to_dhd_dpc_bounds(kobj);
+	d_attr = to_attr(attr);
+	GCC_DIAGNOSTIC_POP();
+
+	if (d_attr->show)
+		ret = d_attr->show(dhd, buf);
+	else
+		ret = -EIO;
+
+	return ret;
+}
+
+/*
+ * wifi/dpc_bounds kobject store function, the "attr" attribute specifices to which
+ * node under "sys/wifi/dpc_bounds" the store function is called.
+ */
+static ssize_t dhd_dpc_bounds_store(struct kobject *kobj, struct attribute *attr,
+		const char *buf, size_t count)
+{
+	dhd_info_t *dhd;
+	struct dhd_attr *d_attr;
+	int ret;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = to_dhd_dpc_bounds(kobj);
+	d_attr = to_attr(attr);
+	GCC_DIAGNOSTIC_POP();
+
+	if (d_attr->store)
+		ret = d_attr->store(dhd, buf, count);
+	else
+		ret = -EIO;
+
+	return ret;
+
+}
+
+static struct sysfs_ops dhd_sysfs_dpc_bounds_ops = {
+	.show = dhd_dpc_bounds_show,
+	.store = dhd_dpc_bounds_store,
+};
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+static const struct attribute_group debug_dpc_bounds_group = {
+	.attrs = debug_dpc_bounds_attrs
+};
+static const struct attribute_group *debug_dpc_bounds_groups[] = {
+	&debug_dpc_bounds_group,
+	NULL
+};
+#endif /* LINUX_VER >= 5.18 */
+
+static struct kobj_type dhd_dpc_bounds_ktype = {
+	.sysfs_ops = &dhd_sysfs_dpc_bounds_ops,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+	.default_groups = debug_dpc_bounds_groups,
+#else
+	.default_attrs = debug_dpc_bounds_attrs,
+#endif /* LINUX_VER >= 5.18 */
+};
+
+/*
+ * *************************************
+ */
+
+#ifdef CSI_SUPPORT
+/* Function to show current ccode */
+static ssize_t read_csi_data(struct file *filp, struct kobject *kobj,
+	struct bin_attribute *bin_attr, char *buf, loff_t off, size_t count)
+{
+	dhd_info_t *dhd = to_dhd(kobj);
+	int n = 0;
+
+	n = dhd_csi_retrieve_queue_data(&dhd->pub, buf, (uint)count);
+	DHD_TRACE(("Dump data to file, size %d\n", n));
+
+	return n;
+}
+
+static struct bin_attribute dhd_attr_csi = {
+	.attr = {
+		.name = CONST_SYNA_DHD_CSI_OBJ_NAME,
+		.mode = 0660
+	},
+	.size = MAX_CSI_FILESZ,
+	.read = read_csi_data,
+};
+#endif /* CSI_SUPPORT */
+
+/* Create a kobject and attach to sysfs interface */
+int dhd_sysfs_init(dhd_info_t *dhd)
+{
+	int ret = -1;
+
+	dhd->flag_kobj = 0x0;
+
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): dhd is NULL \r\n", __FUNCTION__));
+		return ret;
+	}
+
+	/* Initialize the kobject */
+	ret = kobject_init_and_add(&dhd->dhd_kobj, &dhd_ktype, NULL, "wifi" BUS_TYPE);
+	if (ret) {
+		kobject_put(&dhd->dhd_kobj);
+		DHD_ERROR(("%s(): Unable to allocate kobject \r\n", __FUNCTION__));
+		return ret;
+	} else {
+		dhd->flag_kobj |= 0x01;
+	}
+
+	/*
+	 * We are always responsible for sending the uevent that the kobject
+	 * was added to the system.
+	 */
+	kobject_uevent(&dhd->dhd_kobj, KOBJ_ADD);
+
+#ifdef DHD_LB
+	ret  = kobject_init_and_add(&dhd->dhd_lb_kobj,
+			&dhd_lb_ktype, &dhd->dhd_kobj, "lb");
+	if (ret) {
+		kobject_put(&dhd->dhd_lb_kobj);
+		DHD_ERROR(("%s(): Unable to allocate kobject for 'lb'\r\n", __FUNCTION__));
+		return ret;
+	} else {
+		dhd->flag_kobj |= 0x02;
+	}
+
+	kobject_uevent(&dhd->dhd_lb_kobj, KOBJ_ADD);
+#endif /* DHD_LB */
+
+	/* DPC bounds */
+	ret  = kobject_init_and_add(&dhd->dhd_dpc_bounds_kobj,
+			&dhd_dpc_bounds_ktype, &dhd->dhd_kobj, "dpc_bounds");
+	if (ret) {
+		kobject_put(&dhd->dhd_dpc_bounds_kobj);
+		DHD_ERROR(("%s(): Unable to allocate kobject for 'dpc_bounds'\r\n", __FUNCTION__));
+		return ret;
+	}
+	kobject_uevent(&dhd->dhd_dpc_bounds_kobj, KOBJ_ADD);
+
+#ifdef CSI_SUPPORT
+	ret = sysfs_create_bin_file(&dhd->dhd_kobj, &dhd_attr_csi);
+	if (ret) {
+		DHD_ERROR(("%s: can't create %s\n", __func__, dhd_attr_csi.attr.name));
+		return ret;
+	} else {
+		dhd->flag_kobj |= 0x04;
+	}
+#endif /* CSI_SUPPORT */
+
+	return ret;
+}
+
+/* Done with the kobject and detach the sysfs interface */
+void dhd_sysfs_exit(dhd_info_t *dhd)
+{
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): dhd is NULL \r\n", __FUNCTION__));
+		return;
+	}
+
+#ifdef CSI_SUPPORT
+	if (0x04 & dhd->flag_kobj) {
+		sysfs_remove_bin_file(&dhd->dhd_kobj, &dhd_attr_csi);
+	}
+#endif /* CSI_SUPPORT */
+
+#ifdef DHD_LB
+	if (0X02 & dhd->flag_kobj) {
+		kobject_put(&dhd->dhd_lb_kobj);
+	}
+#endif /* DHD_LB */
+
+	/* DPC bounds */
+	kobject_put(&dhd->dhd_dpc_bounds_kobj);
+
+	/* Release the kobject */
+	if (0X01 & dhd->flag_kobj) {
+		kobject_put(&dhd->dhd_kobj);
+	}
+
+	dhd->flag_kobj = 0X0;
+}
+
+#ifdef DHD_SUPPORT_HDM
+static ssize_t
+hdm_load_module(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	int val = bcm_atoi(buf);
+
+	if (val == 1) {
+		DHD_ERROR(("%s : Load module from the hdm %d\n", __FUNCTION__, val));
+		dhd_module_init_hdm();
+	} else {
+		DHD_ERROR(("Module load triggered with invalid value : %d\n", val));
+	}
+
+	return count;
+}
+
+static struct kobj_attribute hdm_wlan_attr =
+	__ATTR(hdm_wlan_loader, 0660, NULL, hdm_load_module);
+
+void
+dhd_hdm_wlan_sysfs_init()
+{
+	DHD_ERROR(("export hdm_wlan_loader\n"));
+	if (sysfs_create_file(kernel_kobj, &hdm_wlan_attr.attr)) {
+		DHD_ERROR(("export hdm_load failed\n"));
+	}
+}
+
+void
+dhd_hdm_wlan_sysfs_deinit(struct work_struct *work)
+{
+	sysfs_remove_file(kernel_kobj,  &hdm_wlan_attr.attr);
+
+}
+#endif /* DHD_SUPPORT_HDM */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_lb.c b/drivers/net/wireless/ap12275/dhd_linux_lb.c
new file mode 100644
index 000000000000..dfb5ab6edb89
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_lb.c
@@ -0,0 +1,1526 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Linux-specific network interface
+ * Basically selected code segments from usb-cdc.c and usb-rndis.c
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <dhd_linux_priv.h>
+
+extern dhd_pub_t* g_dhd_pub;
+
+#if defined(DHD_LB)
+
+#ifdef DHD_LB_STATS
+#define DHD_NUM_NAPI_LATENCY_ROWS (17u)
+#define DHD_NAPI_LATENCY_SIZE (sizeof(uint64) * DHD_NUM_NAPI_LATENCY_ROWS)
+#endif /* DHD_LB_STATS */
+
+void
+dhd_lb_set_default_cpus(dhd_info_t *dhd)
+{
+	/* Default CPU allocation for the jobs */
+	atomic_set(&dhd->rx_napi_cpu, 1);
+	atomic_set(&dhd->tx_cpu, 2);
+	atomic_set(&dhd->net_tx_cpu, 0);
+	atomic_set(&dhd->dpc_cpu, 0);
+}
+
+void
+dhd_cpumasks_deinit(dhd_info_t *dhd)
+{
+	free_cpumask_var(dhd->cpumask_curr_avail);
+	free_cpumask_var(dhd->cpumask_primary);
+	free_cpumask_var(dhd->cpumask_primary_new);
+	free_cpumask_var(dhd->cpumask_secondary);
+	free_cpumask_var(dhd->cpumask_secondary_new);
+}
+
+int
+dhd_cpumasks_init(dhd_info_t *dhd)
+{
+	int id;
+	uint32 cpus, num_cpus = num_possible_cpus();
+	int ret = 0;
+
+	DHD_ERROR(("%s CPU masks primary(big)=0x%x secondary(little)=0x%x\n", __FUNCTION__,
+		DHD_LB_PRIMARY_CPUS, DHD_LB_SECONDARY_CPUS));
+
+	/* FIXME: If one alloc fails we must free_cpumask_var the previous */
+	if (!alloc_cpumask_var(&dhd->cpumask_curr_avail, GFP_KERNEL) ||
+	    !alloc_cpumask_var(&dhd->cpumask_primary, GFP_KERNEL) ||
+	    !alloc_cpumask_var(&dhd->cpumask_primary_new, GFP_KERNEL) ||
+	    !alloc_cpumask_var(&dhd->cpumask_secondary, GFP_KERNEL) ||
+	    !alloc_cpumask_var(&dhd->cpumask_secondary_new, GFP_KERNEL)) {
+		DHD_ERROR(("%s Failed to init cpumasks\n", __FUNCTION__));
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	cpumask_copy(dhd->cpumask_curr_avail, cpu_online_mask);
+	cpumask_clear(dhd->cpumask_primary);
+	cpumask_clear(dhd->cpumask_secondary);
+
+	if (num_cpus > 32) {
+		DHD_ERROR(("%s max cpus must be 32, %d too big\n", __FUNCTION__, num_cpus));
+		ASSERT(0);
+	}
+
+	cpus = DHD_LB_PRIMARY_CPUS;
+	for (id = 0; id < num_cpus; id++) {
+		if (isset(&cpus, id))
+			cpumask_set_cpu(id, dhd->cpumask_primary);
+	}
+
+	cpus = DHD_LB_SECONDARY_CPUS;
+	for (id = 0; id < num_cpus; id++) {
+		if (isset(&cpus, id))
+			cpumask_set_cpu(id, dhd->cpumask_secondary);
+	}
+
+	return ret;
+fail:
+	dhd_cpumasks_deinit(dhd);
+	return ret;
+}
+
+/*
+ * The CPU Candidacy Algorithm
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * The available CPUs for selection are divided into two groups
+ *  Primary Set - A CPU mask that carries the First Choice CPUs
+ *  Secondary Set - A CPU mask that carries the Second Choice CPUs.
+ *
+ * There are two types of Job, that needs to be assigned to
+ * the CPUs, from one of the above mentioned CPU group. The Jobs are
+ * 1) Rx Packet Processing - napi_cpu
+ *
+ * To begin with napi_cpu is on CPU0. Whenever a CPU goes
+ * on-line/off-line the CPU candidacy algorithm is triggerd. The candidacy
+ * algo tries to pickup the first available non boot CPU (CPU0) for napi_cpu.
+ *
+ */
+void dhd_select_cpu_candidacy(dhd_info_t *dhd)
+{
+	uint32 primary_available_cpus; /* count of primary available cpus */
+	uint32 secondary_available_cpus; /* count of secondary available cpus */
+	uint32 napi_cpu = 0; /* cpu selected for napi rx processing */
+	uint32 tx_cpu = 0; /* cpu selected for tx processing job */
+	uint32 dpc_cpu = atomic_read(&dhd->dpc_cpu);
+	uint32 net_tx_cpu = atomic_read(&dhd->net_tx_cpu);
+
+	cpumask_clear(dhd->cpumask_primary_new);
+	cpumask_clear(dhd->cpumask_secondary_new);
+
+	/*
+	 * Now select from the primary mask. Even if a Job is
+	 * already running on a CPU in secondary group, we still move
+	 * to primary CPU. So no conditional checks.
+	 */
+	cpumask_and(dhd->cpumask_primary_new, dhd->cpumask_primary,
+		dhd->cpumask_curr_avail);
+
+	cpumask_and(dhd->cpumask_secondary_new, dhd->cpumask_secondary,
+		dhd->cpumask_curr_avail);
+
+	/* Clear DPC cpu from new masks so that dpc cpu is not chosen for LB */
+	cpumask_clear_cpu(dpc_cpu, dhd->cpumask_primary_new);
+	cpumask_clear_cpu(dpc_cpu, dhd->cpumask_secondary_new);
+
+	/* Clear net_tx_cpu from new masks so that same is not chosen for LB */
+	cpumask_clear_cpu(net_tx_cpu, dhd->cpumask_primary_new);
+	cpumask_clear_cpu(net_tx_cpu, dhd->cpumask_secondary_new);
+
+	primary_available_cpus = cpumask_weight(dhd->cpumask_primary_new);
+
+#if defined(DHD_LB_HOST_CTRL)
+	/* Does not use promary cpus if DHD received affinity off cmd
+	*  from framework
+	*/
+	if (primary_available_cpus > 0 && dhd->permitted_primary_cpu)
+#else
+	if (primary_available_cpus > 0)
+#endif /* DHD_LB_HOST_CTRL */
+	{
+		napi_cpu = cpumask_first(dhd->cpumask_primary_new);
+
+		/* If no further CPU is available,
+		 * cpumask_next returns >= nr_cpu_ids
+		 */
+		tx_cpu = cpumask_next(napi_cpu, dhd->cpumask_primary_new);
+		if (tx_cpu >= nr_cpu_ids) {
+			/* If no CPU is available for tx processing in primary CPUs,
+			 * choose the same CPU with net_tx_cpu
+			 * in case net_tx_cpu is in primary CPUs.
+			 */
+			cpumask_and(dhd->cpumask_primary_new, dhd->cpumask_primary,
+					dhd->cpumask_curr_avail);
+
+			if (cpumask_test_cpu(net_tx_cpu, dhd->cpumask_primary_new)) {
+				tx_cpu = net_tx_cpu;
+				DHD_INFO(("%s If no CPU is for tx cpu, use net_tx_cpu %d\n",
+					__FUNCTION__, net_tx_cpu));
+			} else {
+				tx_cpu = 0;
+			}
+		}
+	}
+
+	DHD_INFO(("%s After primary CPU check napi_cpu %d tx_cpu %d\n",
+		__FUNCTION__, napi_cpu, tx_cpu));
+
+	/* -- Now check for the CPUs from the secondary mask -- */
+	secondary_available_cpus = cpumask_weight(dhd->cpumask_secondary_new);
+
+	DHD_INFO(("%s Available secondary cpus %d nr_cpu_ids %d\n",
+		__FUNCTION__, secondary_available_cpus, nr_cpu_ids));
+
+	if (secondary_available_cpus > 0) {
+		/* At this point if napi_cpu is unassigned it means no CPU
+		 * is online from Primary Group
+		 */
+#if defined(DHD_LB_TXP_LITTLE_CORE_CTRL)
+		/* Clear tx_cpu, so that it can be picked from little core */
+		tx_cpu = 0;
+#endif /* DHD_LB_TXP_LITTLE_CORE_CTRL */
+		if (napi_cpu == 0) {
+			napi_cpu = cpumask_first(dhd->cpumask_secondary_new);
+			tx_cpu = cpumask_next(napi_cpu, dhd->cpumask_secondary_new);
+		} else if (tx_cpu == 0) {
+			tx_cpu = cpumask_first(dhd->cpumask_secondary_new);
+		}
+	}
+
+	if ((primary_available_cpus == 0) &&
+		(secondary_available_cpus == 0)) {
+		/* No CPUs available from primary or secondary mask */
+		tx_cpu = napi_cpu = nr_cpu_ids - 1;
+	}
+
+	/* If no CPU was available for napi processing, choose CPU 0 */
+	if (napi_cpu >= nr_cpu_ids)
+		napi_cpu = 0;
+
+	/* If no CPU was available for tx processing, choose CPU 0 */
+	if (tx_cpu >= nr_cpu_ids)
+		tx_cpu = 0;
+
+	DHD_INFO(("%s After secondary CPU check napi_cpu %d tx_cpu %d nr cpu ids %d\n",
+		__FUNCTION__, napi_cpu, tx_cpu, nr_cpu_ids));
+
+	if (!cpu_online(napi_cpu)) {
+		napi_cpu = 0;
+	}
+	if (!cpu_online(tx_cpu)) {
+		tx_cpu = 0;
+	}
+
+	atomic_set(&dhd->rx_napi_cpu, napi_cpu);
+	atomic_set(&dhd->tx_cpu, tx_cpu);
+
+	return;
+}
+
+/*
+ * Function to handle CPU Hotplug notifications.
+ * One of the task it does is to trigger the CPU Candidacy algorithm
+ * for load balancing.
+ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+
+int dhd_cpu_startup_callback(unsigned int cpu)
+{
+	dhd_info_t *dhd = g_dhd_pub->info;
+
+	if (!dhd || !(dhd->dhd_state & DHD_ATTACH_STATE_LB_ATTACH_DONE)) {
+		DHD_ERROR(("%s(): LB data is not initialized yet.\n",
+			__FUNCTION__));
+		return 0;
+	}
+
+	DHD_INFO(("%s(): \r\n cpu:%d", __FUNCTION__, cpu));
+	DHD_LB_STATS_INCR(dhd->cpu_online_cnt[cpu]);
+	cpumask_set_cpu(cpu, dhd->cpumask_curr_avail);
+	dhd_select_cpu_candidacy(dhd);
+
+	return 0;
+}
+
+int dhd_cpu_teardown_callback(unsigned int cpu)
+{
+	dhd_info_t *dhd = g_dhd_pub->info;
+
+	if (!dhd || !(dhd->dhd_state & DHD_ATTACH_STATE_LB_ATTACH_DONE)) {
+		DHD_ERROR(("%s(): LB data is not initialized yet.\n",
+			__FUNCTION__));
+		return 0;
+	}
+
+	DHD_INFO(("%s(): \r\n cpu:%d", __FUNCTION__, cpu));
+	DHD_LB_STATS_INCR(dhd->cpu_offline_cnt[cpu]);
+	cpumask_clear_cpu(cpu, dhd->cpumask_curr_avail);
+	dhd_select_cpu_candidacy(dhd);
+
+	return 0;
+}
+#else
+int
+dhd_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+	unsigned long int cpu = (unsigned long int)hcpu;
+	dhd_info_t *dhd;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(nfb, dhd_info_t, cpu_notifier);
+	GCC_DIAGNOSTIC_POP();
+
+	if (!dhd || !(dhd->dhd_state & DHD_ATTACH_STATE_LB_ATTACH_DONE)) {
+		DHD_INFO(("%s(): LB data is not initialized yet.\n",
+			__FUNCTION__));
+		return NOTIFY_BAD;
+	}
+
+	/* XXX: Do we need other action types ? */
+	switch (action)
+	{
+		case CPU_ONLINE:
+		case CPU_ONLINE_FROZEN:
+			DHD_LB_STATS_INCR(dhd->cpu_online_cnt[cpu]);
+			cpumask_set_cpu(cpu, dhd->cpumask_curr_avail);
+			dhd_select_cpu_candidacy(dhd);
+			break;
+
+		case CPU_DOWN_PREPARE:
+		case CPU_DOWN_PREPARE_FROZEN:
+			DHD_LB_STATS_INCR(dhd->cpu_offline_cnt[cpu]);
+			cpumask_clear_cpu(cpu, dhd->cpumask_curr_avail);
+			dhd_select_cpu_candidacy(dhd);
+			break;
+		default:
+			break;
+	}
+
+	return NOTIFY_OK;
+}
+#endif /* LINUX_VERSION_CODE < 4.10.0 */
+
+int dhd_register_cpuhp_callback(dhd_info_t *dhd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	dhd->dhd_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "dhd",
+		dhd_cpu_startup_callback, dhd_cpu_teardown_callback);
+
+	if (dhd->dhd_cpuhp_state < 0) {
+		DHD_ERROR(("%s(): cpuhp_setup_state failed %d RX LB won't happen \r\n",
+			__FUNCTION__, dhd->dhd_cpuhp_state));
+	}
+#else
+	/*
+	 * If we are able to initialize CPU masks, lets register to the
+	 * CPU Hotplug framework to change the CPU for each job dynamically
+	 * using candidacy algorithm.
+	 */
+	dhd->cpu_notifier.notifier_call = dhd_cpu_callback;
+	register_hotcpu_notifier(&dhd->cpu_notifier); /* Register a callback */
+#endif /* LINUX_VERSION_CODE < 4.10.0 */
+	return dhd->dhd_cpuhp_state;
+}
+
+int dhd_unregister_cpuhp_callback(dhd_info_t *dhd)
+{
+	int ret = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	/* Don't want to call tear down while unregistering */
+	if (dhd->dhd_cpuhp_state >= 0) {
+		cpuhp_remove_state_nocalls(dhd->dhd_cpuhp_state);
+	}
+#else
+	if (dhd->cpu_notifier.notifier_call != NULL) {
+		unregister_cpu_notifier(&dhd->cpu_notifier);
+	}
+#endif /* LINUX_VERSION_CODE < 4.10.0 */
+	return ret;
+}
+
+#if defined(DHD_LB_STATS)
+void dhd_lb_stats_reset(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+	int i, j, num_cpus = num_possible_cpus();
+
+	if (dhdp == NULL) {
+		DHD_ERROR(("%s dhd pub pointer is NULL \n",
+			__FUNCTION__));
+		return;
+	}
+
+	dhd = dhdp->info;
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__));
+		return;
+	}
+
+	DHD_LB_STATS_CLR(dhd->dhd_dpc_cnt);
+	DHD_LB_STATS_CLR(dhd->napi_sched_cnt);
+
+	/* reset NAPI latency stats */
+	if (dhd->napi_latency) {
+		bzero(dhd->napi_latency, DHD_NAPI_LATENCY_SIZE);
+	}
+	/* reset NAPI per cpu stats */
+	if (dhd->napi_percpu_run_cnt) {
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->napi_percpu_run_cnt[i]);
+		}
+	}
+
+	DHD_LB_STATS_CLR(dhd->rxc_sched_cnt);
+
+	if (dhd->rxc_percpu_run_cnt) {
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->rxc_percpu_run_cnt[i]);
+		}
+	}
+
+	DHD_LB_STATS_CLR(dhd->txc_sched_cnt);
+
+	if (dhd->txc_percpu_run_cnt) {
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->txc_percpu_run_cnt[i]);
+		}
+	}
+
+	if (dhd->txp_percpu_run_cnt) {
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->txp_percpu_run_cnt[i]);
+		}
+	}
+
+	if (dhd->tx_start_percpu_run_cnt) {
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->tx_start_percpu_run_cnt[i]);
+		}
+	}
+
+	for (j = 0; j < HIST_BIN_SIZE; j++) {
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->napi_rx_hist[j][i]);
+		}
+	}
+
+	dhd->pub.lb_rxp_strt_thr_hitcnt = 0;
+	dhd->pub.lb_rxp_stop_thr_hitcnt = 0;
+
+	dhd->pub.lb_rxp_napi_sched_cnt = 0;
+	dhd->pub.lb_rxp_napi_complete_cnt = 0;
+	return;
+}
+
+void dhd_lb_stats_init(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+	int i, j, num_cpus = num_possible_cpus();
+	int alloc_size = sizeof(uint32) * num_cpus;
+
+	if (dhdp == NULL) {
+		DHD_ERROR(("%s(): Invalid argument dhd pubb pointer is NULL \n",
+			__FUNCTION__));
+		return;
+	}
+
+	dhd = dhdp->info;
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__));
+		return;
+	}
+
+	DHD_LB_STATS_CLR(dhd->dhd_dpc_cnt);
+	DHD_LB_STATS_CLR(dhd->napi_sched_cnt);
+
+	/* NAPI latency stats */
+	dhd->napi_latency = (uint64 *)MALLOCZ(dhdp->osh, DHD_NAPI_LATENCY_SIZE);
+	/* NAPI per cpu stats */
+	dhd->napi_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->napi_percpu_run_cnt) {
+		DHD_ERROR(("%s(): napi_percpu_run_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->napi_percpu_run_cnt[i]);
+
+	DHD_LB_STATS_CLR(dhd->rxc_sched_cnt);
+
+	dhd->rxc_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->rxc_percpu_run_cnt) {
+		DHD_ERROR(("%s(): rxc_percpu_run_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->rxc_percpu_run_cnt[i]);
+
+	DHD_LB_STATS_CLR(dhd->txc_sched_cnt);
+
+	dhd->txc_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->txc_percpu_run_cnt) {
+		DHD_ERROR(("%s(): txc_percpu_run_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->txc_percpu_run_cnt[i]);
+
+	dhd->cpu_online_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->cpu_online_cnt) {
+		DHD_ERROR(("%s(): cpu_online_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->cpu_online_cnt[i]);
+
+	dhd->cpu_offline_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->cpu_offline_cnt) {
+		DHD_ERROR(("%s(): cpu_offline_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->cpu_offline_cnt[i]);
+
+	dhd->txp_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->txp_percpu_run_cnt) {
+		DHD_ERROR(("%s(): txp_percpu_run_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->txp_percpu_run_cnt[i]);
+
+	dhd->tx_start_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+	if (!dhd->tx_start_percpu_run_cnt) {
+		DHD_ERROR(("%s(): tx_start_percpu_run_cnt malloc failed \n",
+			__FUNCTION__));
+		return;
+	}
+	for (i = 0; i < num_cpus; i++)
+		DHD_LB_STATS_CLR(dhd->tx_start_percpu_run_cnt[i]);
+
+	for (j = 0; j < HIST_BIN_SIZE; j++) {
+		dhd->napi_rx_hist[j] = (uint32 *)MALLOC(dhdp->osh, alloc_size);
+		if (!dhd->napi_rx_hist[j]) {
+			DHD_ERROR(("%s(): dhd->napi_rx_hist[%d] malloc failed \n",
+				__FUNCTION__, j));
+			return;
+		}
+		for (i = 0; i < num_cpus; i++) {
+			DHD_LB_STATS_CLR(dhd->napi_rx_hist[j][i]);
+		}
+	}
+
+	dhd->pub.lb_rxp_strt_thr_hitcnt = 0;
+	dhd->pub.lb_rxp_stop_thr_hitcnt = 0;
+
+	dhd->pub.lb_rxp_napi_sched_cnt = 0;
+	dhd->pub.lb_rxp_napi_complete_cnt = 0;
+	return;
+}
+
+void dhd_lb_stats_deinit(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd;
+	int j, num_cpus = num_possible_cpus();
+	int alloc_size = sizeof(uint32) * num_cpus;
+
+	if (dhdp == NULL) {
+		DHD_ERROR(("%s(): Invalid argument dhd pubb pointer is NULL \n",
+			__FUNCTION__));
+		return;
+	}
+
+	dhd = dhdp->info;
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__));
+		return;
+	}
+
+	if (dhd->napi_percpu_run_cnt) {
+		MFREE(dhdp->osh, dhd->napi_percpu_run_cnt, alloc_size);
+	}
+	if (dhd->rxc_percpu_run_cnt) {
+		MFREE(dhdp->osh, dhd->rxc_percpu_run_cnt, alloc_size);
+	}
+	if (dhd->txc_percpu_run_cnt) {
+		MFREE(dhdp->osh, dhd->txc_percpu_run_cnt, alloc_size);
+	}
+	if (dhd->cpu_online_cnt) {
+		MFREE(dhdp->osh, dhd->cpu_online_cnt, alloc_size);
+	}
+	if (dhd->cpu_offline_cnt) {
+		MFREE(dhdp->osh, dhd->cpu_offline_cnt, alloc_size);
+	}
+
+	if (dhd->txp_percpu_run_cnt) {
+		MFREE(dhdp->osh, dhd->txp_percpu_run_cnt, alloc_size);
+	}
+	if (dhd->tx_start_percpu_run_cnt) {
+		MFREE(dhdp->osh, dhd->tx_start_percpu_run_cnt, alloc_size);
+	}
+	if (dhd->napi_latency) {
+		MFREE(dhdp->osh, dhd->napi_latency, DHD_NAPI_LATENCY_SIZE);
+	}
+
+	for (j = 0; j < HIST_BIN_SIZE; j++) {
+		if (dhd->napi_rx_hist[j]) {
+			MFREE(dhdp->osh, dhd->napi_rx_hist[j], alloc_size);
+		}
+	}
+
+	return;
+}
+
+void dhd_lb_stats_dump_napi_latency(dhd_pub_t *dhdp,
+	struct bcmstrbuf *strbuf, uint64 *napi_latency)
+{
+	uint32 i;
+
+	bcm_bprintf(strbuf, "napi-latency(us): \t count\n");
+	for (i = 0; i < DHD_NUM_NAPI_LATENCY_ROWS; i++) {
+		bcm_bprintf(strbuf, "%16u: \t %llu\n", 1U<<i, napi_latency[i]);
+	}
+}
+
+void dhd_lb_stats_dump_histo(dhd_pub_t *dhdp,
+	struct bcmstrbuf *strbuf, uint32 **hist)
+{
+	int i, j;
+	uint32 *per_cpu_total;
+	uint32 total = 0;
+	uint32 num_cpus = num_possible_cpus();
+
+	per_cpu_total = (uint32 *)MALLOC(dhdp->osh, sizeof(uint32) * num_cpus);
+	if (!per_cpu_total) {
+		DHD_ERROR(("%s(): dhd->per_cpu_total malloc failed \n", __FUNCTION__));
+		return;
+	}
+	bzero(per_cpu_total, sizeof(uint32) * num_cpus);
+
+	bcm_bprintf(strbuf, "CPU: \t\t");
+	for (i = 0; i < num_cpus; i++)
+		bcm_bprintf(strbuf, "%d\t", i);
+	bcm_bprintf(strbuf, "\nBin\n");
+
+	for (i = 0; i < HIST_BIN_SIZE; i++) {
+		bcm_bprintf(strbuf, "%d:\t\t", 1<<i);
+		for (j = 0; j < num_cpus; j++) {
+			bcm_bprintf(strbuf, "%d\t", hist[i][j]);
+		}
+		bcm_bprintf(strbuf, "\n");
+	}
+	bcm_bprintf(strbuf, "Per CPU Total \t");
+	total = 0;
+	for (i = 0; i < num_cpus; i++) {
+		for (j = 0; j < HIST_BIN_SIZE; j++) {
+			per_cpu_total[i] += (hist[j][i] * (1<<j));
+		}
+		bcm_bprintf(strbuf, "%d\t", per_cpu_total[i]);
+		total += per_cpu_total[i];
+	}
+	bcm_bprintf(strbuf, "\nTotal\t\t%d \n", total);
+
+	if (per_cpu_total) {
+		MFREE(dhdp->osh, per_cpu_total, sizeof(uint32) * num_cpus);
+	}
+	return;
+}
+
+void dhd_lb_stats_dump_cpu_array(struct bcmstrbuf *strbuf, uint32 *p)
+{
+	int i, num_cpus = num_possible_cpus();
+
+	bcm_bprintf(strbuf, "CPU: \t\t");
+	for (i = 0; i < num_cpus; i++)
+		bcm_bprintf(strbuf, "%d\t", i);
+	bcm_bprintf(strbuf, "\n");
+
+	bcm_bprintf(strbuf, "Val: \t\t");
+	for (i = 0; i < num_cpus; i++)
+		bcm_bprintf(strbuf, "%u\t", *(p+i));
+	bcm_bprintf(strbuf, "\n");
+	return;
+}
+
+#ifdef DHD_MEM_STATS
+uint64 dhd_lb_mem_usage(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	dhd_info_t *dhd;
+	uint16 rxbufpost_alloc_sz;
+	uint16 rx_post_active = 0;
+	uint16 rx_cmpl_active = 0;
+	uint64 rx_path_memory_usage = 0;
+
+	if (dhdp == NULL || strbuf == NULL) {
+		DHD_ERROR(("%s(): Invalid argument dhdp %p strbuf %p \n",
+			__FUNCTION__, dhdp, strbuf));
+		return 0;
+	}
+
+	dhd = dhdp->info;
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__));
+		return 0;
+	}
+	rxbufpost_alloc_sz = dhd_prot_get_rxbufpost_alloc_sz(dhdp);
+	if (rxbufpost_alloc_sz == 0) {
+		rxbufpost_alloc_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
+	}
+	rx_path_memory_usage = rxbufpost_alloc_sz * (skb_queue_len(&dhd->rx_emerge_queue) +
+		skb_queue_len(&dhd->rx_pend_queue) +
+		skb_queue_len(&dhd->rx_napi_queue) +
+		skb_queue_len(&dhd->rx_process_queue));
+	rx_post_active = dhd_prot_get_h2d_rx_post_active(dhdp);
+	if (rx_post_active != 0) {
+		rx_path_memory_usage += (rxbufpost_alloc_sz * rx_post_active);
+	}
+
+	rx_cmpl_active = dhd_prot_get_d2h_rx_cpln_active(dhdp);
+	if (rx_cmpl_active != 0) {
+		rx_path_memory_usage += (rxbufpost_alloc_sz * rx_cmpl_active);
+	}
+
+	dhdp->rxpath_mem = rx_path_memory_usage;
+	bcm_bprintf(strbuf, "\nrxbufpost_alloc_sz: %d rx_post_active: %d rx_cmpl_active: %d "
+		"emerge_queue_len: %d pend_queue_len: %d napi_queue_len: %d"
+		" process_queue_len: %d\n",
+		rxbufpost_alloc_sz, rx_post_active, rx_cmpl_active,
+		skb_queue_len(&dhd->rx_emerge_queue), skb_queue_len(&dhd->rx_pend_queue),
+		skb_queue_len(&dhd->rx_napi_queue), skb_queue_len(&dhd->rx_process_queue));
+	bcm_bprintf(strbuf, "DHD rx-path memory_usage: %llubytes %lluKB \n",
+		rx_path_memory_usage, (rx_path_memory_usage/ 1024));
+	return rx_path_memory_usage;
+}
+#endif /* DHD_MEM_STATS */
+
+void dhd_lb_stats_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	dhd_info_t *dhd;
+
+	if (dhdp == NULL || strbuf == NULL) {
+		DHD_ERROR(("%s(): Invalid argument dhdp %p strbuf %p \n",
+			__FUNCTION__, dhdp, strbuf));
+		return;
+	}
+
+	dhd = dhdp->info;
+	if (dhd == NULL) {
+		DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__));
+		return;
+	}
+
+	bcm_bprintf(strbuf, "\nLoad Balancing/NAPI stats:\n==========================\n");
+	bcm_bprintf(strbuf, "\ncpu_online_cnt:\n");
+	dhd_lb_stats_dump_cpu_array(strbuf, dhd->cpu_online_cnt);
+
+	bcm_bprintf(strbuf, "\ncpu_offline_cnt:\n");
+	dhd_lb_stats_dump_cpu_array(strbuf, dhd->cpu_offline_cnt);
+
+	bcm_bprintf(strbuf, "\nsched_cnt: dhd_dpc %u napi %u rxc %u txc %u\n",
+		dhd->dhd_dpc_cnt, dhd->napi_sched_cnt, dhd->rxc_sched_cnt,
+		dhd->txc_sched_cnt);
+
+	bcm_bprintf(strbuf, "\nCPUs: dpc_cpu %u napi_cpu %u net_tx_cpu %u tx_cpu %u\n",
+		atomic_read(&dhd->dpc_cpu),
+		atomic_read(&dhd->rx_napi_cpu),
+		atomic_read(&dhd->net_tx_cpu),
+		atomic_read(&dhd->tx_cpu));
+
+#ifdef DHD_LB_RXP
+	bcm_bprintf(strbuf, "\nnapi_percpu_run_cnt:\n");
+	dhd_lb_stats_dump_cpu_array(strbuf, dhd->napi_percpu_run_cnt);
+	bcm_bprintf(strbuf, "\nNAPI Packets Received Histogram:\n");
+	dhd_lb_stats_dump_histo(dhdp, strbuf, dhd->napi_rx_hist);
+	bcm_bprintf(strbuf, "\nNAPI poll latency stats ie from napi schedule to napi execution\n");
+	dhd_lb_stats_dump_napi_latency(dhdp, strbuf, dhd->napi_latency);
+
+	bcm_bprintf(strbuf, "\nlb_rxp_stop_thr_hitcnt: %llu lb_rxp_strt_thr_hitcnt: %llu"
+		" rx_dma_stall_hc_ignore_cnt: %llu\n",
+		dhdp->lb_rxp_stop_thr_hitcnt, dhdp->lb_rxp_strt_thr_hitcnt,
+		dhdp->rx_dma_stall_hc_ignore_cnt);
+	bcm_bprintf(strbuf, "\nlb_rxp_napi_sched_cnt: %llu lb_rxp_napi_omplete_cnt: %llu\n",
+		dhdp->lb_rxp_napi_sched_cnt, dhdp->lb_rxp_napi_complete_cnt);
+#endif /* DHD_LB_RXP */
+
+#ifdef DHD_LB_TXP
+	bcm_bprintf(strbuf, "\ntxp_percpu_run_cnt:\n");
+	dhd_lb_stats_dump_cpu_array(strbuf, dhd->txp_percpu_run_cnt);
+
+	bcm_bprintf(strbuf, "\ntx_start_percpu_run_cnt:\n");
+	dhd_lb_stats_dump_cpu_array(strbuf, dhd->tx_start_percpu_run_cnt);
+#endif /* DHD_LB_TXP */
+	bcm_bprintf(strbuf, "\n");
+}
+
+void dhd_lb_stats_update_napi_latency(uint64 *bin, uint32 latency)
+{
+	uint64 *p;
+	uint32 bin_power;
+	bin_power = next_larger_power2(latency);
+
+	switch (bin_power) {
+		case   1: p = bin + 0; break;
+		case   2: p = bin + 1; break;
+		case   4: p = bin + 2; break;
+		case   8: p = bin + 3; break;
+		case  16: p = bin + 4; break;
+		case  32: p = bin + 5; break;
+		case  64: p = bin + 6; break;
+		case 128: p = bin + 7; break;
+		case 256: p = bin + 8; break;
+		case 512: p = bin + 9; break;
+		case 1024: p = bin + 10; break;
+		case 2048: p = bin + 11; break;
+		case 4096: p = bin + 12; break;
+		case 8192: p = bin + 13; break;
+		case 16384: p = bin + 14; break;
+		case 32768: p = bin + 15; break;
+		default : p = bin + 16; break;
+	}
+	ASSERT((p - bin) < DHD_NUM_NAPI_LATENCY_ROWS);
+	*p = *p + 1;
+	return;
+
+}
+
+void dhd_lb_stats_update_histo(uint32 **bin, uint32 count, uint32 cpu)
+{
+	uint32 bin_power;
+	uint32 *p;
+	bin_power = next_larger_power2(count);
+
+	switch (bin_power) {
+		case   1: p = bin[0] + cpu; break;
+		case   2: p = bin[1] + cpu; break;
+		case   4: p = bin[2] + cpu; break;
+		case   8: p = bin[3] + cpu; break;
+		case  16: p = bin[4] + cpu; break;
+		case  32: p = bin[5] + cpu; break;
+		case  64: p = bin[6] + cpu; break;
+		case 128: p = bin[7] + cpu; break;
+		default : p = bin[8] + cpu; break;
+	}
+
+	*p = *p + 1;
+	return;
+}
+
+void dhd_lb_stats_update_napi_histo(dhd_pub_t *dhdp, uint32 count)
+{
+	int cpu;
+	dhd_info_t *dhd = dhdp->info;
+
+	cpu = get_cpu();
+	put_cpu();
+	dhd_lb_stats_update_histo(dhd->napi_rx_hist, count, cpu);
+
+	return;
+}
+
+void dhd_lb_stats_update_txc_histo(dhd_pub_t *dhdp, uint32 count)
+{
+	int cpu;
+	dhd_info_t *dhd = dhdp->info;
+
+	cpu = get_cpu();
+	put_cpu();
+	dhd_lb_stats_update_histo(dhd->txc_hist, count, cpu);
+
+	return;
+}
+
+void dhd_lb_stats_update_rxc_histo(dhd_pub_t *dhdp, uint32 count)
+{
+	int cpu;
+	dhd_info_t *dhd = dhdp->info;
+
+	cpu = get_cpu();
+	put_cpu();
+	dhd_lb_stats_update_histo(dhd->rxc_hist, count, cpu);
+
+	return;
+}
+
+void dhd_lb_stats_txc_percpu_cnt_incr(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	DHD_LB_STATS_PERCPU_ARR_INCR(dhd->txc_percpu_run_cnt);
+}
+
+void dhd_lb_stats_rxc_percpu_cnt_incr(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	DHD_LB_STATS_PERCPU_ARR_INCR(dhd->rxc_percpu_run_cnt);
+}
+#endif /* DHD_LB_STATS */
+
+/**
+ * dhd_tasklet_schedule - Function that runs in IPI context of the destination
+ * CPU and schedules a tasklet.
+ * @tasklet: opaque pointer to the tasklet
+ */
+INLINE void
+dhd_tasklet_schedule(void *tasklet)
+{
+	tasklet_schedule((struct tasklet_struct *)tasklet);
+}
+
+/**
+ * dhd_work_schedule_on - Executes the passed work in a given CPU
+ * @work: work to be scheduled
+ * @on_cpu: cpu core id
+ *
+ * If the requested cpu is online, then an IPI is sent to this cpu via the
+ * schedule_work_on and the work function
+ * will be invoked to schedule the specified work on the requested CPU.
+ */
+
+INLINE void
+dhd_work_schedule_on(struct work_struct *work, int on_cpu)
+{
+	schedule_work_on(on_cpu, work);
+}
+
+INLINE void
+dhd_delayed_work_schedule_on(struct delayed_work *dwork, int on_cpu, ulong delay)
+{
+	schedule_delayed_work_on(on_cpu, dwork, delay);
+}
+
+#if defined(DHD_LB_TXP)
+void dhd_tx_dispatcher_work(struct work_struct * work)
+{
+	struct dhd_info *dhd;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(work, struct dhd_info, tx_dispatcher_work);
+	GCC_DIAGNOSTIC_POP();
+
+	dhd_tasklet_schedule(&dhd->tx_tasklet);
+}
+
+/**
+ * dhd_lb_tx_dispatch - load balance by dispatching the tx_tasklet
+ * on another cpu. The tx_tasklet will take care of actually putting
+ * the skbs into appropriate flow ring and ringing H2D interrupt
+ *
+ * @dhdp: pointer to dhd_pub object
+ */
+void
+dhd_lb_tx_dispatch(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	int curr_cpu;
+	int tx_cpu;
+	int prev_net_tx_cpu;
+
+	/*
+	 * Get cpu will disable pre-ermption and will not allow any cpu to go offline
+	 * and call put_cpu() only after scheduling rx_napi_dispatcher_work.
+	 */
+	curr_cpu = get_cpu();
+
+	/* Record the CPU in which the TX request from Network stack came */
+	prev_net_tx_cpu = atomic_read(&dhd->net_tx_cpu);
+	atomic_set(&dhd->net_tx_cpu, curr_cpu);
+
+	tx_cpu = atomic_read(&dhd->tx_cpu);
+
+	/*
+	 * Avoid cpu candidacy, if override is set via sysfs for changing cpu mannually
+	 */
+	if (dhd->dhd_lb_candidacy_override) {
+		if (!cpu_online(tx_cpu)) {
+			tx_cpu = curr_cpu;
+		}
+	} else {
+		/*
+		 * Now if the NET TX has scheduled in the same CPU
+		 * that is chosen for Tx processing
+		 * OR scheduled on different cpu than previously it was scheduled,
+		 * OR if tx_cpu is offline,
+		 * Call cpu candidacy algorithm to recompute tx_cpu.
+		 */
+		if ((curr_cpu == tx_cpu) || (curr_cpu != prev_net_tx_cpu) ||
+			!cpu_online(tx_cpu)) {
+			/* Re compute LB CPUs */
+			dhd_select_cpu_candidacy(dhd);
+			/* Use updated tx cpu */
+			tx_cpu = atomic_read(&dhd->tx_cpu);
+		}
+	}
+	/*
+	 * Schedule tx_dispatcher_work to on the cpu which
+	 * in turn will schedule tx_tasklet.
+	 */
+	dhd_work_schedule_on(&dhd->tx_dispatcher_work, tx_cpu);
+
+	put_cpu();
+}
+#endif /* DHD_LB_TXP */
+
+#if defined(DHD_LB_RXP)
+
+/**
+ * dhd_napi_poll - Load balance napi poll function to process received
+ * packets and send up the network stack using netif_receive_skb()
+ *
+ * @napi: napi object in which context this poll function is invoked
+ * @budget: number of packets to be processed.
+ *
+ * Fetch the dhd_info given the rx_napi_struct. Move all packets from the
+ * rx_napi_queue into a local rx_process_queue (lock and queue move and unlock).
+ * Dequeue each packet from head of rx_process_queue, fetch the ifid from the
+ * packet tag and sendup.
+ */
+int
+dhd_napi_poll(struct napi_struct *napi, int budget)
+{
+	int ifid;
+	const int pkt_count = 1;
+	const int chan = 0;
+	struct sk_buff * skb;
+	unsigned long flags;
+	struct dhd_info *dhd;
+	int processed = 0;
+	int dpc_cpu;
+#ifdef DHD_LB_STATS
+	uint32 napi_latency;
+#endif /* DHD_LB_STATS */
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(napi, struct dhd_info, rx_napi_struct);
+	GCC_DIAGNOSTIC_POP();
+
+#ifdef DHD_LB_STATS
+	napi_latency = (uint32)(OSL_SYSUPTIME_US() - dhd->napi_schedule_time);
+	dhd_lb_stats_update_napi_latency(dhd->napi_latency, napi_latency);
+#endif /* DHD_LB_STATS */
+	DHD_INFO(("%s napi_queue<%d> budget<%d>\n",
+		__FUNCTION__, skb_queue_len(&dhd->rx_napi_queue), budget));
+
+	/*
+	 * Extract the entire rx_napi_queue into another rx_process_queue
+	 * and process only 'budget' number of skbs from rx_process_queue.
+	 * If there are more items to be processed, napi poll will be rescheduled
+	 * During the next iteration, next set of skbs from
+	 * rx_napi_queue will be extracted and attached to the tail of rx_process_queue.
+	 * Again budget number of skbs will be processed from rx_process_queue.
+	 * If there are less than budget number of skbs in rx_process_queue,
+	 * call napi_complete to stop rescheduling napi poll.
+	 */
+	DHD_RX_NAPI_QUEUE_LOCK(&dhd->rx_napi_queue.lock, flags);
+	skb_queue_splice_tail_init(&dhd->rx_napi_queue, &dhd->rx_process_queue);
+	DHD_RX_NAPI_QUEUE_UNLOCK(&dhd->rx_napi_queue.lock, flags);
+
+	while ((processed < budget) && (skb = __skb_dequeue(&dhd->rx_process_queue)) != NULL) {
+		OSL_PREFETCH(skb->data);
+
+		ifid = DHD_PKTTAG_IFID((dhd_pkttag_fr_t *)PKTTAG(skb));
+
+		DHD_INFO(("%s dhd_rx_frame pkt<%p> ifid<%d>\n",
+			__FUNCTION__, skb, ifid));
+
+		dhd_rx_frame(&dhd->pub, ifid, skb, pkt_count, chan);
+		processed++;
+	}
+
+	if (atomic_read(&dhd->pub.lb_rxp_flow_ctrl) &&
+		(dhd_lb_rxp_process_qlen(&dhd->pub) <= dhd->pub.lb_rxp_strt_thr)) {
+		/*
+		 * If the dpc CPU is online Schedule dhd_dpc_dispatcher_work on the dpc cpu which
+		 * in turn will schedule dpc tasklet. Else schedule dpc takslet.
+		 */
+		get_cpu();
+		dpc_cpu = atomic_read(&dhd->dpc_cpu);
+		if (!cpu_online(dpc_cpu)) {
+			dhd_tasklet_schedule(&dhd->tasklet);
+		} else {
+			dhd_delayed_work_schedule_on(&dhd->dhd_dpc_dispatcher_work, dpc_cpu, 0);
+		}
+		put_cpu();
+	}
+	DHD_LB_STATS_UPDATE_NAPI_HISTO(&dhd->pub, processed);
+
+	DHD_INFO(("%s processed %d\n", __FUNCTION__, processed));
+
+	/*
+	 * Signal napi complete only when no more packets are processed and
+	 * none are left in the enqueued queue.
+	 */
+	if ((processed == 0) && (skb_queue_len(&dhd->rx_napi_queue) == 0)) {
+		napi_complete(napi);
+#ifdef DHD_LB_STATS
+		dhd->pub.lb_rxp_napi_complete_cnt++;
+#endif /* DHD_LB_STATS */
+		DHD_GENERAL_LOCK(&dhd->pub, flags);
+		DHD_BUS_BUSY_CLEAR_IN_NAPI(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		return 0;
+	}
+
+#ifdef DHD_LB_STATS
+	dhd->napi_schedule_time = OSL_SYSUPTIME_US();
+#endif /* DHD_LB_STATS */
+
+	/* Return budget so that it gets rescheduled immediately */
+	return budget;
+}
+
+/**
+ * dhd_napi_schedule - Place the napi struct into the current cpus softnet napi
+ * poll list. This function may be invoked via the smp_call_function_single
+ * from a remote CPU.
+ *
+ * This function will essentially invoke __raise_softirq_irqoff(NET_RX_SOFTIRQ)
+ * after the napi_struct is added to the softnet data's poll_list
+ *
+ * @info: pointer to a dhd_info struct
+ */
+static void
+dhd_napi_schedule(void *info)
+{
+	dhd_info_t *dhd = (dhd_info_t *)info;
+	unsigned long flags;
+
+	DHD_INFO(("%s rx_napi_struct<%p> on cpu<%d>\n",
+		__FUNCTION__, &dhd->rx_napi_struct, atomic_read(&dhd->rx_napi_cpu)));
+
+	/* On Android platform, napi prevention during suspend in progress causes
+	 * rx performance drop of ~5Mbs(SWWLAN-349763).
+	 * So, excludes this prevention for Android platform.
+	 */
+#ifndef OEM_ANDROID
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+
+	if (DHD_BUS_BUSY_CHECK_SUSPEND_IN_PROGRESS(&dhd->pub)) {
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		return;
+	}
+
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+#endif /* OEM_ANDROID */
+
+	/* add napi_struct to softnet data poll list and raise NET_RX_SOFTIRQ */
+	if (napi_schedule_prep(&dhd->rx_napi_struct)) {
+
+		/*
+		 * Set busbusystate in NAPI, which will be cleared after
+		 * napi_complete from napi_poll context
+		 */
+		DHD_GENERAL_LOCK(&dhd->pub, flags);
+		DHD_BUS_BUSY_SET_IN_NAPI(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+
+#ifdef DHD_LB_STATS
+		dhd->napi_schedule_time = OSL_SYSUPTIME_US();
+		dhd->pub.lb_rxp_napi_sched_cnt++;
+#endif /* DHD_LB_STATS */
+		__napi_schedule(&dhd->rx_napi_struct);
+#ifdef WAKEUP_KSOFTIRQD_POST_NAPI_SCHEDULE
+		raise_softirq(NET_RX_SOFTIRQ);
+#endif /* WAKEUP_KSOFTIRQD_POST_NAPI_SCHEDULE */
+	}
+
+	/*
+	 * If the rx_napi_struct was already running, then we let it complete
+	 * processing all its packets. The rx_napi_struct may only run on one
+	 * core at a time, to avoid out-of-order handling.
+	 */
+}
+
+/*
+ * Call get_online_cpus/put_online_cpus around dhd_napi_schedule_on
+ * Why should we do this?
+ * The candidacy algorithm is run from the call back function
+ * registered to CPU hotplug notifier. This call back happens from Worker
+ * context. The dhd_napi_schedule_on is also from worker context.
+ * Note that both of this can run on two different CPUs at the same time.
+ * So we can possibly have a window where a given CPUn is being brought
+ * down from CPUm while we try to run a function on CPUn.
+ * To prevent this its better have the whole code to execute an SMP
+ * function under get_online_cpus.
+ * This function call ensures that hotplug mechanism does not kick-in
+ * until we are done dealing with online CPUs
+ * If the hotplug worker is already running, no worries because the
+ * candidacy algo would then reflect the same in dhd->rx_napi_cpu.
+ *
+ * The below mentioned code structure is proposed in
+ * https://www.kernel.org/doc/Documentation/cpu-hotplug.txt
+ * for the question
+ * Q: I need to ensure that a particular cpu is not removed when there is some
+ *    work specific to this cpu is in progress
+ *
+ * According to the documentation calling get_online_cpus is NOT required, if
+ * we are running from tasklet context. Since dhd_rx_napi_dispatcher_work can
+ * run from Work Queue context we have to call these functions
+ */
+void dhd_rx_napi_dispatcher_work(struct work_struct * work)
+{
+	struct dhd_info *dhd;
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	dhd = container_of(work, struct dhd_info, rx_napi_dispatcher_work);
+	GCC_DIAGNOSTIC_POP();
+
+	dhd_napi_schedule(dhd);
+}
+
+/**
+ * dhd_lb_rx_napi_dispatch - load balance by dispatching the rx_napi_struct
+ * to run on another CPU. The rx_napi_struct's poll function will retrieve all
+ * the packets enqueued into the rx_napi_queue and sendup.
+ * The producer's rx packet queue is appended to the rx_napi_queue before
+ * dispatching the rx_napi_struct.
+ */
+void
+dhd_lb_rx_napi_dispatch(dhd_pub_t *dhdp)
+{
+	unsigned long flags;
+	dhd_info_t *dhd = dhdp->info;
+	int curr_cpu;
+	int rx_napi_cpu;
+	int prev_dpc_cpu;
+
+	if (dhd->rx_napi_netdev == NULL) {
+		DHD_ERROR(("%s: dhd->rx_napi_netdev is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_INFO(("%s append napi_queue<%d> pend_queue<%d>\n", __FUNCTION__,
+		skb_queue_len(&dhd->rx_napi_queue), skb_queue_len(&dhd->rx_pend_queue)));
+
+	/* append the producer's queue of packets to the napi's rx process queue */
+	DHD_RX_NAPI_QUEUE_LOCK(&dhd->rx_napi_queue.lock, flags);
+	skb_queue_splice_tail_init(&dhd->rx_pend_queue, &dhd->rx_napi_queue);
+	DHD_RX_NAPI_QUEUE_UNLOCK(&dhd->rx_napi_queue.lock, flags);
+
+	/* If sysfs lb_rxp_active is not set, schedule on current cpu */
+	if (!atomic_read(&dhd->lb_rxp_active))
+	{
+		dhd_napi_schedule(dhd);
+		return;
+	}
+
+	/*
+	 * Get cpu will disable pre-ermption and will not allow any cpu to go offline
+	 * and call put_cpu() only after scheduling rx_napi_dispatcher_work.
+	 */
+	curr_cpu = get_cpu();
+
+	prev_dpc_cpu = atomic_read(&dhd->prev_dpc_cpu);
+
+	rx_napi_cpu = atomic_read(&dhd->rx_napi_cpu);
+
+	/*
+	 * Avoid cpu candidacy, if override is set via sysfs for changing cpu mannually
+	 */
+	if (dhd->dhd_lb_candidacy_override) {
+		if (!cpu_online(rx_napi_cpu)) {
+			rx_napi_cpu = curr_cpu;
+		}
+	} else {
+		/*
+		 * Now if the DPC has scheduled in the same CPU
+		 * that is chosen for Rx napi processing
+		 * OR scheduled on different cpu than previously it was scheduled,
+		 * OR if rx_napi_cpu is offline,
+		 * Call cpu candidacy algorithm to recompute napi_cpu.
+		 */
+		if ((curr_cpu == rx_napi_cpu) || (curr_cpu != prev_dpc_cpu) ||
+			!cpu_online(rx_napi_cpu)) {
+			/* Re compute LB CPUs */
+			dhd_select_cpu_candidacy(dhd);
+			/* Use updated napi cpu */
+			rx_napi_cpu = atomic_read(&dhd->rx_napi_cpu);
+		}
+
+	}
+
+	DHD_INFO(("%s : schedule to curr_cpu : %d, rx_napi_cpu : %d\n",
+		__FUNCTION__, curr_cpu, rx_napi_cpu));
+	dhd_work_schedule_on(&dhd->rx_napi_dispatcher_work, rx_napi_cpu);
+	DHD_LB_STATS_INCR(dhd->napi_sched_cnt);
+
+	put_cpu();
+}
+
+/**
+ * dhd_rx_emerge_enqueue - Enqueue the packet into the ememrgency queue for repost
+ */
+void
+dhd_rx_emerge_enqueue(dhd_pub_t *dhdp, void *pkt)
+{
+	dhd_info_t *dhd = dhdp->info;
+	skb_queue_tail(&dhd->rx_emerge_queue, pkt);
+}
+
+/**
+ * dhd_rx_emerge_dequeue - Deueue the packet from the emergency queue for repost
+ */
+void *
+dhd_rx_emerge_dequeue(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	return skb_dequeue(&dhd->rx_emerge_queue);
+}
+
+/**
+ * dhd_lb_rx_pkt_enqueue - Enqueue the packet into the producer's queue
+ */
+void
+dhd_lb_rx_pkt_enqueue(dhd_pub_t *dhdp, void *pkt, int ifidx)
+{
+	dhd_info_t *dhd = dhdp->info;
+
+	DHD_INFO(("%s enqueue pkt<%p> ifidx<%d> pend_queue<%d>\n", __FUNCTION__,
+		pkt, ifidx, skb_queue_len(&dhd->rx_pend_queue)));
+	DHD_PKTTAG_SET_IFID((dhd_pkttag_fr_t *)PKTTAG(pkt), ifidx);
+	__skb_queue_tail(&dhd->rx_pend_queue, pkt);
+	DHD_LB_STATS_PERCPU_ARR_INCR(dhd->napi_percpu_run_cnt);
+}
+
+unsigned long
+dhd_read_lb_rxp(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	return atomic_read(&dhd->lb_rxp_active);
+}
+
+uint32
+dhd_lb_rxp_process_qlen(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+	return skb_queue_len(&dhd->rx_process_queue);
+}
+#endif /* DHD_LB_RXP */
+
+#if defined(DHD_LB_TXP)
+int
+BCMFASTPATH(dhd_lb_sendpkt)(dhd_info_t *dhd, struct net_device *net,
+	int ifidx, void *skb)
+{
+	DHD_LB_STATS_PERCPU_ARR_INCR(dhd->tx_start_percpu_run_cnt);
+
+	/* If the feature is disabled run-time do TX from here */
+	if (atomic_read(&dhd->lb_txp_active) == 0) {
+		DHD_LB_STATS_PERCPU_ARR_INCR(dhd->txp_percpu_run_cnt);
+		 return __dhd_sendpkt(&dhd->pub, ifidx, skb);
+	}
+
+	/* Store the address of net device and interface index in the Packet tag */
+	DHD_LB_TX_PKTTAG_SET_NETDEV((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb), net);
+	DHD_LB_TX_PKTTAG_SET_IFIDX((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb), ifidx);
+
+	/* Enqueue the skb into tx_pend_queue */
+	skb_queue_tail(&dhd->tx_pend_queue, skb);
+
+	DHD_TRACE(("%s(): Added skb %p for netdev %p \r\n", __FUNCTION__, skb, net));
+
+	/* Dispatch the Tx job to be processed by the tx_tasklet */
+	dhd_lb_tx_dispatch(&dhd->pub);
+
+	return NETDEV_TX_OK;
+}
+#endif /* DHD_LB_TXP */
+
+#ifdef DHD_LB_TXP
+#ifndef DHD_LB_TXBOUND
+#define DHD_LB_TXBOUND	64
+#endif
+/*
+ * Function that performs the TX processing on a given CPU
+ */
+bool
+dhd_lb_tx_process(dhd_info_t *dhd)
+{
+	struct sk_buff *skb;
+	int cnt = 0;
+	struct net_device *net;
+	int ifidx;
+	bool resched = FALSE;
+
+	DHD_TRACE(("%s(): TX Processing \r\n", __FUNCTION__));
+	if (dhd == NULL) {
+		DHD_ERROR((" Null pointer DHD \r\n"));
+		return resched;
+	}
+
+	BCM_REFERENCE(net);
+
+	DHD_LB_STATS_PERCPU_ARR_INCR(dhd->txp_percpu_run_cnt);
+
+	/* Base Loop to perform the actual Tx */
+	do {
+		skb = skb_dequeue(&dhd->tx_pend_queue);
+		if (skb == NULL) {
+			DHD_TRACE(("Dequeued a Null Packet \r\n"));
+			break;
+		}
+		cnt++;
+
+		net =  DHD_LB_TX_PKTTAG_NETDEV((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb));
+		ifidx = DHD_LB_TX_PKTTAG_IFIDX((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb));
+
+		DHD_TRACE(("Processing skb %p for net %p index %d \r\n", skb,
+			net, ifidx));
+
+		__dhd_sendpkt(&dhd->pub, ifidx, skb);
+
+		if (cnt >= DHD_LB_TXBOUND) {
+			resched = TRUE;
+			break;
+		}
+
+	} while (1);
+
+	DHD_INFO(("%s(): Processed %d packets \r\n", __FUNCTION__, cnt));
+
+	return resched;
+}
+
+void
+dhd_lb_tx_handler(unsigned long data)
+{
+	dhd_info_t *dhd = (dhd_info_t *)data;
+
+	if (dhd_lb_tx_process(dhd)) {
+		dhd_tasklet_schedule(&dhd->tx_tasklet);
+	}
+}
+
+#endif /* DHD_LB_TXP */
+#endif /* DHD_LB */
+
+#if defined(SET_PCIE_IRQ_CPU_CORE) || defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON)
+void
+dhd_irq_set_affinity(dhd_pub_t *dhdp, const struct cpumask *cpumask)
+{
+	unsigned int irq = (unsigned int)-1;
+	int err = BCME_OK;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s : dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhdp->bus) {
+		DHD_ERROR(("%s : bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("%s : irq set affinity cpu:0x%lx\n",
+			__FUNCTION__, *cpumask_bits(cpumask)));
+
+	dhdpcie_get_pcieirq(dhdp->bus, &irq);
+#ifdef BCMDHD_MODULAR
+	err = irq_set_affinity_hint(irq, cpumask);
+#else
+	err = irq_set_affinity(irq, cpumask);
+#endif /* BCMDHD_MODULAR */
+	if (err)
+		DHD_ERROR(("%s : irq set affinity is failed cpu:0x%lx\n",
+				__FUNCTION__, *cpumask_bits(cpumask)));
+}
+#endif /* SET_PCIE_IRQ_CPU_CORE ||  DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON */
+
+#ifdef SET_PCIE_IRQ_CPU_CORE
+void
+dhd_set_irq_cpucore(dhd_pub_t *dhdp, int affinity_cmd)
+{
+#if defined(DHD_LB) && defined(DHD_LB_HOST_CTRL)
+	struct dhd_info  *dhd = NULL;
+#endif /* DHD_LB && DHD_LB_HOST_CTRL */
+
+	if (!dhdp) {
+		DHD_ERROR(("%s : dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (!dhdp->bus) {
+		DHD_ERROR(("%s : dhd->bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (affinity_cmd < DHD_AFFINITY_OFF || affinity_cmd > DHD_AFFINITY_LAST) {
+		DHD_ERROR(("Wrong Affinity cmds:%d, %s\n", affinity_cmd, __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("Enter %s, PCIe affinity cmd=0x%x\n", __FUNCTION__, affinity_cmd));
+
+#if defined(DHD_LB) && defined(DHD_LB_HOST_CTRL)
+	dhd = dhdp->info;
+
+	if (affinity_cmd == DHD_AFFINITY_OFF) {
+		dhd->permitted_primary_cpu = FALSE;
+	} else if (affinity_cmd == DHD_AFFINITY_TPUT_150MBPS ||
+		affinity_cmd == DHD_AFFINITY_TPUT_300MBPS) {
+		dhd->permitted_primary_cpu = TRUE;
+	}
+	dhd_select_cpu_candidacy(dhd);
+	/*
+	* It needs to NAPI disable -> enable to raise NET_RX napi CPU core
+	* during Rx traffic
+	* NET_RX does not move to NAPI CPU core if continusly calling napi polling
+	* function
+	*/
+	napi_disable(&dhd->rx_napi_struct);
+	napi_enable(&dhd->rx_napi_struct);
+#endif /* DHD_LB && DHD_LB_HOST_CTRL */
+
+	/*
+		irq_set_affinity() assign dedicated CPU core PCIe interrupt
+		If dedicated CPU core is not on-line,
+		PCIe interrupt scheduled on CPU core 0
+	*/
+	switch (affinity_cmd) {
+		case DHD_AFFINITY_OFF:
+#if defined(DHD_LB) && defined(DHD_LB_HOST_CTRL)
+			dhd_irq_set_affinity(dhdp, dhdp->info->cpumask_secondary);
+#endif /* DHD_LB && DHD_LB_HOST_CTRL */
+			break;
+		case DHD_AFFINITY_TPUT_150MBPS:
+			dhd_irq_set_affinity(dhdp, dhdp->info->cpumask_primary);
+			break;
+		case DHD_AFFINITY_TPUT_300MBPS:
+#ifdef CONFIG_ARCH_EXYNOS
+			dhd_irq_set_affinity(dhdp, cpumask_of(PCIE_IRQ_CPU_CORE));
+#else
+			dhd_irq_set_affinity(dhdp, dhdp->info->cpumask_primary);
+#endif /* CONFIG_ARCH_EXYNOS */
+			break;
+		default:
+			DHD_ERROR(("%s, Unknown PCIe affinity cmd=0x%x\n",
+				__FUNCTION__, affinity_cmd));
+	}
+}
+#endif /* SET_PCIE_IRQ_CPU_CORE */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_pktdump.c b/drivers/net/wireless/ap12275/dhd_linux_pktdump.c
new file mode 100644
index 000000000000..57091830117b
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_pktdump.c
@@ -0,0 +1,2262 @@
+/*
+ * Packet dump helper functions
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <ethernet.h>
+#include <bcmutils.h>
+#include <bcmevent.h>
+#include <bcmendian.h>
+#include <bcmtlv.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <bcmip.h>
+#include <bcmudp.h>
+#include <bcmdhcp.h>
+#include <bcmarp.h>
+#include <bcmicmp.h>
+#include <bcmigmp.h>
+#include <dhd_linux_pktdump.h>
+#ifdef WL_CFGVENDOR_CUST_ADVLOG
+#include <wl_cfg80211.h>
+#include <wl_cfgvendor.h>
+#include <dhd_flowring.h>
+#endif /* WL_CFGVENDOR_CUST_ADVLOG */
+#include <dhd_config.h>
+#include <wl_android.h>
+
+#define DHD_PKTDUMP(arg)	printf arg
+#define DHD_PKTDUMP_MEM(arg)	printf arg
+#define PACKED_STRUCT __attribute__ ((packed))
+
+#define EAPOL_HDR_LEN		4
+
+/* EAPOL types */
+#define EAP_PACKET		0
+#define EAPOL_START		1
+#define EAPOL_LOGOFF		2
+#define EAPOL_KEY		3
+#define EAPOL_ASF		4
+
+/* EAPOL-Key types */
+#define EAPOL_RC4_KEY		1
+#define EAPOL_WPA2_KEY		2	/* 802.11i/WPA2 */
+#define EAPOL_WPA_KEY		254	/* WPA */
+
+/* EAPOL-Key header field size */
+#define AKW_BLOCK_LEN		8
+#define WPA_KEY_REPLAY_LEN	8
+#define WPA_KEY_NONCE_LEN	32
+#define WPA_KEY_IV_LEN		16
+#define WPA_KEY_RSC_LEN		8
+#define WPA_KEY_ID_LEN		8
+#define WPA_KEY_MIC_LEN		16
+#define WPA_MAX_KEY_SIZE	32
+#define WPA_KEY_DATA_LEN	(WPA_MAX_KEY_SIZE + AKW_BLOCK_LEN)
+
+/* Key information bit */
+#define KEYINFO_TYPE_MASK	(1 << 3)
+#define KEYINFO_INSTALL_MASK	(1 << 6)
+#define KEYINFO_KEYACK_MASK	(1 << 7)
+#define KEYINFO_KEYMIC_MASK	(1 << 8)
+#define KEYINFO_SECURE_MASK	(1 << 9)
+#define KEYINFO_ERROR_MASK	(1 << 10)
+#define KEYINFO_REQ_MASK	(1 << 11)
+
+/* EAP Code */
+#define EAP_CODE_REQUEST	1	/* Request */
+#define EAP_CODE_RESPONSE	2	/* Response */
+#define EAP_CODE_SUCCESS	3	/* Success */
+#define EAP_CODE_FAILURE	4	/* Failure */
+
+/* EAP Type */
+#define EAP_TYPE_RSVD		0	/* Reserved */
+#define EAP_TYPE_IDENT		1	/* Identify */
+#define EAP_TYPE_NOTI		2	/* Notification */
+#define EAP_TYPE_TLS		13	/* EAP-TLS */
+#define EAP_TYPE_LEAP		17	/* Cisco-LEAP */
+#define EAP_TYPE_TTLS		21	/* EAP-TTLS */
+#define EAP_TYPE_AKA		23	/* EAP-AKA */
+#define EAP_TYPE_PEAP		25	/* EAP-PEAP */
+#define EAP_TYPE_FAST		43	/* EAP-FAST */
+#define EAP_TYPE_PSK		47	/* EAP-PSK */
+#define EAP_TYPE_AKAP		50	/* EAP-AKA' */
+#define EAP_TYPE_EXP		254	/* Reserved for Expended Type */
+
+/* WSC */
+#define EAP_HDR_LEN		5
+#define EAP_WSC_NONCE_OFFSET	10
+#define EAP_WSC_DATA_OFFSET	(OFFSETOF(eap_wsc_fmt_t, data))
+#define EAP_WSC_MIN_DATA_LEN	((EAP_HDR_LEN) + (EAP_WSC_DATA_OFFSET))
+#define WFA_VID			"\x00\x37\x2A"	/* WFA SMI code */
+#define WFA_VID_LEN		3		/* WFA VID length */
+#define WFA_VTYPE		1u		/* WFA Vendor type */
+
+/* WSC opcode */
+#define WSC_OPCODE_UPNP		0
+#define WSC_OPCODE_START	1
+#define WSC_OPCODE_ACK		2
+#define WSC_OPCODE_NACK		3
+#define WSC_OPCODE_MSG		4
+#define WSC_OPCODE_DONE		5
+#define WSC_OPCODE_FRAG_ACK	6
+
+/* WSC flag */
+#define WSC_FLAG_MF		1	/* more fragements */
+#define WSC_FLAG_LF		2	/* length field */
+
+/* WSC message code */
+#define WSC_ATTR_MSG		0x1022
+#define WSC_MSG_M1		0x04
+#define WSC_MSG_M2		0x05
+#define WSC_MSG_M3		0x07
+#define WSC_MSG_M4		0x08
+#define WSC_MSG_M5		0x09
+#define WSC_MSG_M6		0x0A
+#define WSC_MSG_M7		0x0B
+#define WSC_MSG_M8		0x0C
+
+/* Debug prints */
+typedef enum pkt_cnt_type {
+	PKT_CNT_TYPE_INVALID	= 0,
+	PKT_CNT_TYPE_ARP	= 1,
+	PKT_CNT_TYPE_DNS	= 2,
+	PKT_CNT_TYPE_MAX	= 3
+} pkt_cnt_type_t;
+
+typedef struct pkt_cnt {
+	uint32 tx_cnt;
+	uint32 tx_err_cnt;
+	uint32 rx_cnt;
+} pkt_cnt_t;
+
+typedef struct pkt_cnt_log {
+	bool enabled;
+	uint16 reason;
+	timer_list_compat_t pktcnt_timer;
+	pkt_cnt_t arp_cnt;
+	pkt_cnt_t dns_cnt;
+} pkt_cnts_log_t;
+
+#define PKT_CNT_TIMER_INTERNVAL_MS		5000	/* packet count timeout(ms) */
+#define PKT_CNT_RSN_VALID(rsn)	\
+	(((rsn) > (PKT_CNT_RSN_INVALID)) && ((rsn) < (PKT_CNT_RSN_MAX)))
+
+#ifdef DHD_PKTDUMP_ROAM
+static const char pkt_cnt_msg[][20] = {
+	"INVALID",
+	"ROAM_SUCCESS",
+	"GROUP_KEY_UPDATE",
+	"CONNECT_SUCCESS",
+	"INVALID"
+};
+#endif
+
+#if defined(DHD_DNS_DUMP) || defined(DHD_ARP_DUMP) || defined(DHD_ICMP_DUMP) || \
+	defined(DHD_DHCP_DUMP) || defined(DHD_8021X_DUMP) || defined(DHD_PKTDUMP_ROAM)
+#define DHD_PKTFATE_STR_MAX 30
+#define DHD_PKTFATE_EX_STR_MAX 10
+typedef struct dhd_tx_pktfate_entry {
+	const char reason[DHD_PKTFATE_STR_MAX];
+	const char ex[DHD_PKTFATE_EX_STR_MAX];
+} dhd_tx_pktfate_entry_t;
+
+static dhd_tx_pktfate_entry_t tx_pktfate[] = {
+	{"TX_PKT_FATE_ACKED", "ACK"},			/* 0: WLFC_CTL_PKTFLAG_DISCARD */
+	{"TX_PKT_FATE_FW_D11SUPPRESS", "TX_FAIL"},	/* 1: WLFC_CTL_PKTFLAG_D11SUPPRESS */
+	{"TX_PKT_FATE_FW_WLSUPPRESS", "TX_FAIL"},	/* 2: WLFC_CTL_PKTFLAG_WLSUPPRESS */
+	{"TX_PKT_FATE_FW_TOSSED_BYWLC", "TX_FAIL"},	/* 3: WLFC_CTL_PKTFLAG_TOSSED_BYWLC */
+	{"TX_PKT_FATE_SENT_NOACK", "NO_ACK"},		/* 4: WLFC_CTL_PKTFLAG_DISCARD_NOACK */
+	{"TX_PKT_FATE_FW_SUPPRESS_ACKED", "TX_FAIL"},	/* 5: WLFC_CTL_PKTFLAG_SUPPRESS_ACKED */
+	{"TX_PKT_FATE_FW_DROP_EXPTIME",	"TX_FAIL"},	/* 6: WLFC_CTL_PKTFLAG_EXPIRED */
+	{"TX_PKT_FATE_FW_DROP_OTHER", "TX_FAIL"},	/* 7: WLFC_CTL_PKTFLAG_DROPPED */
+	{"TX_PKT_FATE_FW_PKT_FREE", "TX_FAIL"},		/* 8: WLFC_CTL_PKTFLAG_MKTFREE */
+	{"TX_PKT_FATE_FW_MAX_SUP_RETR", "TX_FAIL"},	/* 9: WLFC_CTL_PKTFLAG_MAX_SUP_RETR */
+	{"TX_PKT_FATE_FW_FORCED_EXPIRED", "TX_FAIL"},	/* 10: WLFC_CTL_PKTFLAG_FORCED_EXPIRED */
+};
+
+#define DBGREPLAY		" Replay Counter: %02x%02x%02x%02x%02x%02x%02x%02x"
+#define REPLAY_FMT(key)		((const eapol_key_hdr_t *)(key))->replay[0], \
+				((const eapol_key_hdr_t *)(key))->replay[1], \
+				((const eapol_key_hdr_t *)(key))->replay[2], \
+				((const eapol_key_hdr_t *)(key))->replay[3], \
+				((const eapol_key_hdr_t *)(key))->replay[4], \
+				((const eapol_key_hdr_t *)(key))->replay[5], \
+				((const eapol_key_hdr_t *)(key))->replay[6], \
+				((const eapol_key_hdr_t *)(key))->replay[7]
+#define TXFATE_FMT		" TX_PKTHASH:0x%X TX_PKT_FATE:%s"
+#define TX_PKTHASH(pkthash)		((pkthash) ? (*pkthash) : (0))
+#define TX_FATE_STR(fate)	(((*fate) <= (WLFC_CTL_PKTFLAG_FORCED_EXPIRED)) ? \
+				(tx_pktfate[(*fate)].reason) : "TX_PKT_FATE_UNKNOWN")
+#define TX_FATE_STR_EX(fate)	(((*fate) <= (WLFC_CTL_PKTFLAG_FORCED_EXPIRED)) ? \
+				(tx_pktfate[(*fate)].ex) : NULL)
+#define TX_FATE(fate)		((fate) ? (TX_FATE_STR(fate)) : "N/A")
+#define TX_FATE_EX(fate)	((fate) ? (TX_FATE_STR_EX(fate)) : NULL)
+#define TX_FATE_ACKED(fate)	((fate) ? ((*fate) == (WLFC_CTL_PKTFLAG_DISCARD)) : (0))
+#endif /*
+	* DHD_DNS_DUMP || DHD_ARP_DUMP || DHD_ICMP_DUMP ||
+	* DHD_DHCP_DUMP || DHD_8021X_DUMP || DHD_PKTDUMP_ROAM
+	*/
+
+#define EAP_PRINT(x, args...) \
+	do { \
+		if (dump_msg_level & DUMP_EAPOL_VAL) { \
+			if (tx) { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [TX] : (%s) %s (%s)"TXFATE_FMT"\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [RX] : (%s) %s (%s)\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf)); \
+			} \
+		} \
+	} while (0)
+
+#define EAP_PRINT_REPLAY(x, args...) \
+	do { \
+		if (dump_msg_level & DUMP_EAPOL_VAL) { \
+			if (tx) { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [TX] : (%s) %s (%s)"DBGREPLAY TXFATE_FMT"\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					REPLAY_FMT(eap_key), TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [RX] : (%s) %s (%s)"DBGREPLAY"\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					REPLAY_FMT(eap_key))); \
+			} \
+		} \
+	} while (0)
+
+#define EAP_PRINT_OTHER(x, args...) \
+	do { \
+		if (dump_msg_level & DUMP_EAPOL_VAL) { \
+			if (tx) { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [TX] : (%s) %s (%s) " \
+					"ver %d, type %d"TXFATE_FMT"\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					eapol_hdr->version, eapol_hdr->type, \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [RX] : (%s) %s (%s) " \
+					"ver %d, type %d\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					eapol_hdr->version, eapol_hdr->type)); \
+			} \
+		} \
+	} while (0)
+
+#define EAP_PRINT_OTHER_4WAY(x, args...) \
+	do { \
+		if (dump_msg_level & DUMP_EAPOL_VAL) { \
+			if (tx) { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [TX] : (%s) %s (%s) " \
+					"ver %d type %d keytype %d keyinfo 0x%02X"TXFATE_FMT"\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					eapol_hdr->version, eapol_hdr->type, eap_key->type, \
+					(uint32)hton16(eap_key->key_info), \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP(("[%s] 802_1X " x " [RX] : (%s) %s (%s) " \
+					"ver %d type %d keytype %d keyinfo 0x%02X\n", \
+					ifname, ## args, \
+					tx?seabuf:deabuf, tx?"->":"<-", tx?deabuf:seabuf, \
+					eapol_hdr->version, eapol_hdr->type, eap_key->type, \
+					(uint32)hton16(eap_key->key_info))); \
+			} \
+		} \
+	} while (0)
+
+#define UDP_PORT_DNS	53	/* UDP DNS port */
+
+/* EAPOL header */
+typedef struct eapol_header {
+	struct ether_header eth;	/* 802.3/Ethernet header */
+	uint8 version;			/* EAPOL protocol version */
+	uint8 type;			/* EAPOL type */
+	uint16 length;			/* Length of body */
+	uint8 body[1];			/* Body (optional) */
+} PACKED_STRUCT eapol_header_t;
+
+/* EAP header */
+typedef struct eap_header_fmt {
+	uint8 code;
+	uint8 id;
+	uint16 len;
+	uint8 type;
+	uint8 data[1];
+} PACKED_STRUCT eap_header_fmt_t;
+
+/* WSC EAP format */
+typedef struct eap_wsc_fmt {
+	uint8 oui[3];
+	uint32 ouitype;
+	uint8 opcode;
+	uint8 flags;
+	uint8 data[1];
+} PACKED_STRUCT eap_wsc_fmt_t;
+
+/* EAPOL-Key */
+typedef struct eapol_key_hdr {
+	uint8 type;				/* Key Descriptor Type */
+	uint16 key_info;			/* Key Information (unaligned) */
+	uint16 key_len;				/* Key Length (unaligned) */
+	uint8 replay[WPA_KEY_REPLAY_LEN];	/* Replay Counter */
+	uint8 nonce[WPA_KEY_NONCE_LEN];		/* Nonce */
+	uint8 iv[WPA_KEY_IV_LEN];		/* Key IV */
+	uint8 rsc[WPA_KEY_RSC_LEN];		/* Key RSC */
+	uint8 id[WPA_KEY_ID_LEN];		/* WPA:Key ID, 802.11i/WPA2: Reserved */
+	uint8 mic[WPA_KEY_MIC_LEN];		/* Key MIC */
+	uint16 data_len;			/* Key Data Length */
+	uint8 data[WPA_KEY_DATA_LEN];		/* Key data */
+} PACKED_STRUCT eapol_key_hdr_t;
+
+typedef struct hdr_fmt {
+	struct ipv4_hdr iph;
+	struct bcmudp_hdr udph;
+} PACKED_STRUCT hdr_fmt_t;
+
+#ifdef DHD_IPV6_DUMP
+typedef struct ipv6hdr_fmt {
+	struct ipv6_hdr ip6h;
+	struct bcmudp_hdr udph;
+} PACKED_STRUCT ipv6hdr_fmt_t;
+
+typedef struct dhcp6_fmt {
+	struct ipv6_hdr iph;
+	struct bcmudp_hdr udph;
+	uint8 msgtype;
+} PACKED_STRUCT dhcp6_fmt_t;
+
+typedef struct icmpv6 {
+	uint8   icmp6_type;
+	uint8   icmp6_code;
+	uint16  icmp6_cksum;
+	uint16  id;
+	uint16  seq;
+} PACKED_STRUCT icmpv6_hdr_t;
+
+typedef struct dns6_fmt {
+	struct ipv6_hdr ip6h;
+	struct bcmudp_hdr udph;
+	uint16 id;
+	uint16 flags;
+	uint16 qdcount;
+	uint16 ancount;
+	uint16 nscount;
+	uint16 arcount;
+} PACKED_STRUCT dns6_fmt_t;
+#endif /* DHD_IPV6_DUMP */
+
+msg_eapol_t
+dhd_is_4way_msg(uint8 *pktdata)
+{
+	eapol_header_t *eapol_hdr;
+	eapol_key_hdr_t *eap_key;
+	msg_eapol_t type = EAPOL_OTHER;
+	bool pair, ack, mic, kerr, req, sec, install;
+	uint16 key_info;
+
+	if (!pktdata) {
+		DHD_PKTDUMP(("%s: pktdata is NULL\n", __FUNCTION__));
+		return type;
+	}
+
+	eapol_hdr = (eapol_header_t *)pktdata;
+	eap_key = (eapol_key_hdr_t *)(eapol_hdr->body);
+	if (eap_key->type != EAPOL_WPA2_KEY && eap_key->type != EAPOL_WPA_KEY) {
+		return type;
+	}
+
+	key_info = hton16(eap_key->key_info);
+	pair = !!(key_info & KEYINFO_TYPE_MASK);
+	ack = !!(key_info & KEYINFO_KEYACK_MASK);
+	mic = !!(key_info & KEYINFO_KEYMIC_MASK);
+	kerr = !!(key_info & KEYINFO_ERROR_MASK);
+	req = !!(key_info & KEYINFO_REQ_MASK);
+	sec = !!(key_info & KEYINFO_SECURE_MASK);
+	install = !!(key_info & KEYINFO_INSTALL_MASK);
+
+	if (eap_key->type == EAPOL_WPA2_KEY) {
+		if (pair && !install && ack && !mic && !sec && !kerr && !req) {
+			type = EAPOL_4WAY_M1;
+		} else if (pair && !install && !ack && mic && !sec && !kerr && !req) {
+			type = EAPOL_4WAY_M2;
+		} else if (pair && ack && mic && sec && !kerr && !req) {
+			type = EAPOL_4WAY_M3;
+		} else if (pair && !install && !ack && mic && sec && !req && !kerr) {
+			type = EAPOL_4WAY_M4;
+		} else if (!pair && !install && ack && mic && sec && !req && !kerr) {
+			type = EAPOL_GROUPKEY_M1;
+		} else if (!pair && !install && !ack && mic && sec && !req && !kerr) {
+			type = EAPOL_GROUPKEY_M2;
+		} else {
+			type = EAPOL_OTHER;
+			if (dump_msg_level & DUMP_EAPOL_VAL) {
+				printf("WPA2: key_info=0x%x, pair=%d, ack=%d, mic=%d, sec=%d, kerr=%d, req=%d\n",
+					key_info, pair, ack, mic, sec, kerr, req);
+			}
+		}
+	}
+	else if (eap_key->type == EAPOL_WPA_KEY) {
+		if (pair && !install && ack && !mic && !sec && !kerr && !req) {
+			type = EAPOL_4WAY_M1;
+		} else if (pair && !install && !ack && mic && !sec && !kerr && !req && eap_key->data_len) {
+			type = EAPOL_4WAY_M2;
+		} else if (pair && install && ack && mic && !sec && !kerr && !req) {
+			type = EAPOL_4WAY_M3;
+		} else if (pair && !install && !ack && mic && !sec && !req && !kerr) {
+			type = EAPOL_4WAY_M4;
+		} else if (!pair && !install && ack && mic && sec && !req && !kerr) {
+			type = EAPOL_GROUPKEY_M1;
+		} else if (!pair && !install && !ack && mic && sec && !req && !kerr) {
+			type = EAPOL_GROUPKEY_M2;
+		} else {
+			type = EAPOL_OTHER;
+			if (dump_msg_level & DUMP_EAPOL_VAL) {
+				printf("WPA: key_info=0x%x, pair=%d, ack=%d, mic=%d, sec=%d, kerr=%d, req=%d\n",
+					key_info, pair, ack, mic, sec, kerr, req);
+			}
+		}
+	}
+	else {
+		type = EAPOL_OTHER;
+		if (dump_msg_level & DUMP_EAPOL_VAL) {
+			printf("OTHER: key_info=0x%x, pair=%d, ack=%d, mic=%d, sec=%d, kerr=%d, req=%d\n",
+				key_info, pair, ack, mic, sec, kerr, req);
+		}
+	}
+
+	return type;
+}
+
+void
+dhd_dump_pkt(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, uint32 pktlen,
+	bool tx, uint32 *pkthash, uint16 *pktfate)
+{
+	struct ether_header *eh;
+	uint16 ether_type;
+
+	if (!pktdata || pktlen < ETHER_HDR_LEN) {
+		return;
+	}
+
+#if 0 // defined(BCMPCIE) && defined(DHD_PKT_LOGGING)
+	if (tx && !pkthash && !pktfate) {
+		return;
+	}
+#endif /* BCMPCIE && DHD_PKT_LOGGING */
+
+	eh = (struct ether_header *)pktdata;
+	ether_type = ntoh16(eh->ether_type);
+	if (ether_type == ETHER_TYPE_802_1X) {
+		dhd_dump_eapol_message(dhdp, ifidx, pktdata, pktlen,
+			tx, pkthash, pktfate);
+	}
+	if (ether_type == ETHER_TYPE_IP) {
+		if (dhd_check_dhcp(pktdata)) {
+			dhd_dhcp_dump(dhdp, ifidx, pktdata, tx, pkthash, pktfate);
+		} else if (dhd_check_icmp(pktdata)) {
+			dhd_icmp_dump(dhdp, ifidx, pktdata, tx, pkthash, pktfate);
+		} else if (dhd_check_dns(pktdata)) {
+			dhd_dns_dump(dhdp, ifidx, pktdata, tx, pkthash, pktfate);
+		}
+	}
+	if (ether_type == ETHER_TYPE_ARP) {
+		if (dhd_check_arp(pktdata, ether_type)) {
+			dhd_arp_dump(dhdp, ifidx, pktdata, tx, pkthash, pktfate);
+		}
+	}
+	dhd_trx_pkt_dump(dhdp, ifidx, pktdata, pktlen, tx);
+}
+
+#ifdef DHD_PKTDUMP_ROAM
+static void
+dhd_dump_pkt_cnts_inc(dhd_pub_t *dhdp, bool tx, uint16 *pktfate, uint16 pkttype)
+{
+	pkt_cnts_log_t *pktcnts;
+	pkt_cnt_t *cnt;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pktcnts = (pkt_cnts_log_t *)(dhdp->pktcnts);
+	if (!pktcnts) {
+		DHD_ERROR(("%s: pktcnts is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (!pktcnts->enabled || (tx && !pktfate)) {
+		return;
+	}
+
+	if (pkttype == PKT_CNT_TYPE_ARP) {
+		cnt = (pkt_cnt_t *)&pktcnts->arp_cnt;
+	} else if (pkttype == PKT_CNT_TYPE_DNS) {
+		cnt = (pkt_cnt_t *)&pktcnts->dns_cnt;
+	} else {
+		/* invalid packet type */
+		return;
+	}
+
+	if (tx) {
+		TX_FATE_ACKED(pktfate) ? cnt->tx_cnt++ : cnt->tx_err_cnt++;
+	} else {
+		cnt->rx_cnt++;
+	}
+}
+
+static void
+dhd_dump_pkt_timer(unsigned long data)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)data;
+	pkt_cnts_log_t *pktcnts = (pkt_cnts_log_t *)(dhdp->pktcnts);
+
+	pktcnts->enabled = FALSE;
+
+	/* print out the packet counter value */
+	DHD_PKTDUMP(("============= PACKET COUNT SUMMARY ============\n"));
+	DHD_PKTDUMP(("- Reason: %s\n", pkt_cnt_msg[pktcnts->reason]));
+	DHD_PKTDUMP(("- Duration: %d msec(s)\n", PKT_CNT_TIMER_INTERNVAL_MS));
+	DHD_PKTDUMP(("- ARP PACKETS: tx_success:%d tx_fail:%d rx_cnt:%d\n",
+		pktcnts->arp_cnt.tx_cnt, pktcnts->arp_cnt.tx_err_cnt,
+		pktcnts->arp_cnt.rx_cnt));
+	DHD_PKTDUMP(("- DNS PACKETS: tx_success:%d tx_fail:%d rx_cnt:%d\n",
+		pktcnts->dns_cnt.tx_cnt, pktcnts->dns_cnt.tx_err_cnt,
+		pktcnts->dns_cnt.rx_cnt));
+	DHD_PKTDUMP(("============= END OF COUNT SUMMARY ============\n"));
+}
+
+void
+dhd_dump_mod_pkt_timer(dhd_pub_t *dhdp, uint16 rsn)
+{
+	pkt_cnts_log_t *pktcnts;
+
+	if (!dhdp || !dhdp->pktcnts) {
+		DHD_ERROR(("%s: dhdp or dhdp->pktcnts is NULL\n",
+			__FUNCTION__));
+		return;
+	}
+
+	if (!PKT_CNT_RSN_VALID(rsn)) {
+		DHD_ERROR(("%s: invalid reason code %d\n",
+			__FUNCTION__, rsn));
+		return;
+	}
+
+	pktcnts = (pkt_cnts_log_t *)(dhdp->pktcnts);
+	if (timer_pending(&pktcnts->pktcnt_timer)) {
+		del_timer_sync(&pktcnts->pktcnt_timer);
+	}
+
+	bzero(&pktcnts->arp_cnt, sizeof(pkt_cnt_t));
+	bzero(&pktcnts->dns_cnt, sizeof(pkt_cnt_t));
+	pktcnts->reason = rsn;
+	pktcnts->enabled = TRUE;
+	mod_timer(&pktcnts->pktcnt_timer,
+		jiffies + msecs_to_jiffies(PKT_CNT_TIMER_INTERNVAL_MS));
+	DHD_PKTDUMP(("%s: Arm the pktcnt timer. reason=%d\n",
+		__FUNCTION__, rsn));
+}
+
+void
+dhd_dump_pkt_init(dhd_pub_t *dhdp)
+{
+	pkt_cnts_log_t *pktcnts;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pktcnts = (pkt_cnts_log_t *)MALLOCZ(dhdp->osh, sizeof(pkt_cnts_log_t));
+	if (!pktcnts) {
+		DHD_ERROR(("%s: failed to allocate memory for pktcnts\n",
+			__FUNCTION__));
+		return;
+	}
+
+	/* init timers */
+	init_timer_compat(&pktcnts->pktcnt_timer, dhd_dump_pkt_timer, dhdp);
+	dhdp->pktcnts = pktcnts;
+}
+
+void
+dhd_dump_pkt_deinit(dhd_pub_t *dhdp)
+{
+	pkt_cnts_log_t *pktcnts;
+
+	if (!dhdp || !dhdp->pktcnts) {
+		DHD_ERROR(("%s: dhdp or pktcnts is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pktcnts = (pkt_cnts_log_t *)(dhdp->pktcnts);
+	pktcnts->enabled = FALSE;
+	del_timer_sync(&pktcnts->pktcnt_timer);
+	MFREE(dhdp->osh, dhdp->pktcnts, sizeof(pkt_cnts_log_t));
+	dhdp->pktcnts = NULL;
+}
+
+void
+dhd_dump_pkt_clear(dhd_pub_t *dhdp)
+{
+	pkt_cnts_log_t *pktcnts;
+
+	if (!dhdp || !dhdp->pktcnts) {
+		DHD_ERROR(("%s: dhdp or pktcnts is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pktcnts = (pkt_cnts_log_t *)(dhdp->pktcnts);
+	pktcnts->enabled = FALSE;
+	del_timer_sync(&pktcnts->pktcnt_timer);
+	pktcnts->reason = 0;
+	bzero(&pktcnts->arp_cnt, sizeof(pkt_cnt_t));
+	bzero(&pktcnts->dns_cnt, sizeof(pkt_cnt_t));
+}
+
+bool
+dhd_dump_pkt_enabled(dhd_pub_t *dhdp)
+{
+	pkt_cnts_log_t *pktcnts;
+
+	if (!dhdp || !dhdp->pktcnts) {
+		return FALSE;
+	}
+
+	pktcnts = (pkt_cnts_log_t *)(dhdp->pktcnts);
+
+	return pktcnts->enabled;
+}
+#else
+static INLINE void
+dhd_dump_pkt_cnts_inc(dhd_pub_t *dhdp, bool tx, uint16 *pktfate, uint16 pkttype) { }
+static INLINE bool
+dhd_dump_pkt_enabled(dhd_pub_t *dhdp) { return FALSE; }
+#endif /* DHD_PKTDUMP_ROAM */
+
+#ifdef DHD_8021X_DUMP
+static void
+dhd_dump_wsc_message(dhd_pub_t *dhd, int ifidx, uint8 *pktdata,
+	uint32 pktlen, bool tx, uint32 *pkthash, uint16 *pktfate)
+{
+	eapol_header_t *eapol_hdr;
+	eap_header_fmt_t *eap_hdr;
+	eap_wsc_fmt_t *eap_wsc;
+	char *ifname;
+	uint16 eap_len;
+	bool cond;
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!pktdata) {
+		DHD_ERROR(("%s: pktdata is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (pktlen < (ETHER_HDR_LEN + EAPOL_HDR_LEN)) {
+		DHD_ERROR(("%s: invalid pkt length\n", __FUNCTION__));
+		return;
+	}
+
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+
+	eapol_hdr = (eapol_header_t *)pktdata;
+	eap_hdr = (eap_header_fmt_t *)(eapol_hdr->body);
+	if (eap_hdr->type != EAP_TYPE_EXP) {
+		return;
+	}
+
+	eap_len = ntoh16(eap_hdr->len);
+	if (eap_len < EAP_WSC_MIN_DATA_LEN) {
+		return;
+	}
+
+	eap_wsc = (eap_wsc_fmt_t *)(eap_hdr->data);
+	if (bcmp(eap_wsc->oui, (const uint8 *)WFA_VID, WFA_VID_LEN) ||
+		(ntoh32(eap_wsc->ouitype) != WFA_VTYPE)) {
+		return;
+	}
+
+	if (eap_wsc->flags) {
+		return;
+	}
+
+	ifname = dhd_ifname(dhd, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+
+	if (eap_wsc->opcode == WSC_OPCODE_MSG) {
+		const uint8 *tlv_buf = (const uint8 *)(eap_wsc->data);
+		const uint8 *msg;
+		uint16 msglen;
+		uint16 wsc_data_len = (uint16)(eap_len - EAP_HDR_LEN - EAP_WSC_DATA_OFFSET);
+		bcm_xtlv_opts_t opt = BCM_XTLV_OPTION_IDBE | BCM_XTLV_OPTION_LENBE;
+
+		msg = bcm_get_data_from_xtlv_buf(tlv_buf, wsc_data_len,
+			WSC_ATTR_MSG, &msglen, opt);
+		if (msg && msglen) {
+			switch (*msg) {
+			case WSC_MSG_M1:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M1);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M1), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M1");
+				break;
+			case WSC_MSG_M2:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M2);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M2), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M2");
+				break;
+			case WSC_MSG_M3:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M3);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M3), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M3");
+				break;
+			case WSC_MSG_M4:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M4);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M4), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M4");
+				break;
+			case WSC_MSG_M5:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M5);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M5), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M5");
+				break;
+			case WSC_MSG_M6:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M6);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M6), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M6");
+				break;
+			case WSC_MSG_M7:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M7);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M7), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M7");
+				break;
+			case WSC_MSG_M8:
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WPS_M8);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(WPS_M8), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, WPS M8");
+				break;
+			default:
+				EAP_PRINT("EAP Packet, WPS MSG TYPE %d", *msg);
+				break;
+			}
+		}
+	} else if (eap_wsc->opcode == WSC_OPCODE_START) {
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WSC_START);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(WSC_START), ifidx, tx, cond);
+		EAP_PRINT("EAP Packet, WSC Start");
+	} else if (eap_wsc->opcode == WSC_OPCODE_DONE) {
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_WSC_DONE);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(WSC_DONE), ifidx, tx, cond);
+		EAP_PRINT("EAP Packet, WSC Done");
+	} else {
+		EAP_PRINT("EAP Packet, WSC opcode=%d", eap_wsc->opcode);
+	}
+}
+
+static void
+dhd_dump_eap_packet(dhd_pub_t *dhd, int ifidx, uint8 *pktdata,
+	uint32 pktlen, bool tx, uint32 *pkthash, uint16 *pktfate)
+{
+	eapol_header_t *eapol_hdr;
+	eap_header_fmt_t *eap_hdr;
+	char *ifname;
+	bool cond;
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!pktdata) {
+		DHD_PKTDUMP(("%s: pktdata is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+
+	eapol_hdr = (eapol_header_t *)pktdata;
+	eap_hdr = (eap_header_fmt_t *)(eapol_hdr->body);
+	ifname = dhd_ifname(dhd, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+
+	if (eap_hdr->code == EAP_CODE_REQUEST ||
+		eap_hdr->code == EAP_CODE_RESPONSE) {
+		bool isreq = (eap_hdr->code == EAP_CODE_REQUEST);
+		switch (eap_hdr->type) {
+		case EAP_TYPE_IDENT:
+			if (isreq) {
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_REQID);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_IDENTITY), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, Identity");
+			} else {
+#ifdef WL_EXT_IAPSTA
+				wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_RSPID);
+#endif
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_IDENTITY), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, Identity");
+			}
+			break;
+		case EAP_TYPE_TLS:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_TLS), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, TLS");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_TLS), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, TLS");
+			}
+			break;
+		case EAP_TYPE_LEAP:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_LEAP), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, LEAP");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_LEAP), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, LEAP");
+			}
+			break;
+		case EAP_TYPE_TTLS:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_TTLS), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, TTLS");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_TTLS), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, TTLS");
+			}
+			break;
+		case EAP_TYPE_AKA:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_AKA), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, AKA");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_AKA), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, AKA");
+			}
+			break;
+		case EAP_TYPE_PEAP:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_PEAP), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, PEAP");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_PEAP), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, PEAP");
+			}
+			break;
+		case EAP_TYPE_FAST:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_FAST), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, FAST");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_FAST), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, FAST");
+			}
+			break;
+		case EAP_TYPE_PSK:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_PSK), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, PSK");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_PSK), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, PSK");
+			}
+			break;
+		case EAP_TYPE_AKAP:
+			if (isreq) {
+				DHD_STATLOG_DATA(dhd, ST(EAP_REQ_AKAP), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Request, AKAP");
+			} else {
+				DHD_STATLOG_DATA(dhd, ST(EAP_RESP_AKAP), ifidx, tx, cond);
+				EAP_PRINT("EAP Packet, Response, AKAP");
+			}
+			break;
+		case EAP_TYPE_EXP:
+			dhd_dump_wsc_message(dhd, ifidx, pktdata, pktlen, tx,
+				pkthash, pktfate);
+			break;
+		default:
+			EAP_PRINT("EAP Packet, EAP TYPE %d", eap_hdr->type);
+			break;
+		}
+	} else if (eap_hdr->code == EAP_CODE_SUCCESS) {
+		DHD_STATLOG_DATA(dhd, ST(EAP_SUCCESS), ifidx, tx, cond);
+		EAP_PRINT("EAP Packet, Success");
+	} else if (eap_hdr->code == EAP_CODE_FAILURE) {
+		DHD_STATLOG_DATA(dhd, ST(EAP_FAILURE), ifidx, tx, cond);
+		EAP_PRINT("EAP Packet, Failure");
+	} else {
+		EAP_PRINT("EAP Packet, EAP CODE %d", eap_hdr->code);
+	}
+}
+
+static void
+dhd_dump_eapol_4way_message(dhd_pub_t *dhd, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate)
+{
+	eapol_header_t *eapol_hdr;
+	eapol_key_hdr_t *eap_key;
+	msg_eapol_t type;
+	char *ifname;
+	bool cond;
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!pktdata) {
+		DHD_PKTDUMP(("%s: pktdata is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+
+	type = dhd_is_4way_msg(pktdata);
+	ifname = dhd_ifname(dhd, ifidx);
+	eapol_hdr = (eapol_header_t *)pktdata;
+	eap_key = (eapol_key_hdr_t *)(eapol_hdr->body);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+
+	if (eap_key->type != EAPOL_WPA2_KEY && eap_key->type != EAPOL_WPA_KEY) {
+		EAP_PRINT_OTHER("NON EAPOL_WPA2_KEY %d", eap_key->type);
+		return;
+	}
+
+	switch (type) {
+	case EAPOL_4WAY_M1:
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_4WAY_M1);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_M1), ifidx, tx, cond);
+		EAP_PRINT("EAPOL Packet, 4-way handshake, M1");
+		break;
+	case EAPOL_4WAY_M2:
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_4WAY_M2);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_M2), ifidx, tx, cond);
+		EAP_PRINT("EAPOL Packet, 4-way handshake, M2");
+		break;
+	case EAPOL_4WAY_M3:
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_4WAY_M3);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_M3), ifidx, tx, cond);
+		EAP_PRINT("EAPOL Packet, 4-way handshake, M3");
+		break;
+	case EAPOL_4WAY_M4:
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_4WAY_M4);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_M4), ifidx, tx, cond);
+		EAP_PRINT("EAPOL Packet, 4-way handshake, M4");
+		break;
+	case EAPOL_GROUPKEY_M1:
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_GROUPKEY_M1);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_GROUPKEY_M1), ifidx, tx, cond);
+		EAP_PRINT_REPLAY("EAPOL Packet, GROUP Key handshake, M1");
+		break;
+	case EAPOL_GROUPKEY_M2:
+#ifdef WL_EXT_IAPSTA
+		wl_ext_update_conn_state(dhd, ifidx, CONN_STATE_GROUPKEY_M2);
+#endif
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_GROUPKEY_M2), ifidx, tx, cond);
+		EAP_PRINT_REPLAY("EAPOL Packet, GROUP Key handshake, M2");
+		if (ifidx == 0 && tx && pktfate) {
+			dhd_dump_mod_pkt_timer(dhd, PKT_CNT_RSN_GRPKEY_UP);
+		}
+		break;
+	default:
+		DHD_STATLOG_DATA(dhd, ST(8021X_OTHER), ifidx, tx, cond);
+		EAP_PRINT_OTHER("OTHER 4WAY type=%d", type);
+		break;
+	}
+}
+
+void
+dhd_dump_eapol_message(dhd_pub_t *dhd, int ifidx, uint8 *pktdata,
+	uint32 pktlen, bool tx, uint32 *pkthash, uint16 *pktfate)
+{
+	char *ifname;
+	eapol_header_t *eapol_hdr = (eapol_header_t *)pktdata;
+	bool cond;
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!pktdata) {
+		DHD_ERROR(("%s: pktdata is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+
+	eapol_hdr = (eapol_header_t *)pktdata;
+	ifname = dhd_ifname(dhd, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+
+	if (eapol_hdr->type == EAP_PACKET) {
+		dhd_dump_eap_packet(dhd, ifidx, pktdata, pktlen, tx,
+			pkthash, pktfate);
+	} else if (eapol_hdr->type == EAPOL_START) {
+		DHD_STATLOG_DATA(dhd, ST(EAPOL_START), ifidx, tx, cond);
+		EAP_PRINT("EAP Packet, EAPOL-Start");
+	} else if (eapol_hdr->type == EAPOL_KEY) {
+		dhd_dump_eapol_4way_message(dhd, ifidx, pktdata, tx,
+			pkthash, pktfate);
+	} else {
+		DHD_STATLOG_DATA(dhd, ST(8021X_OTHER), ifidx, tx, cond);
+		EAP_PRINT_OTHER("OTHER 8021X");
+	}
+}
+#endif /* DHD_8021X_DUMP */
+
+bool
+dhd_check_ip_prot(uint8 *pktdata, uint16 ether_type)
+{
+	hdr_fmt_t *b = (hdr_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = &b->iph;
+
+	/* check IP header */
+	if ((ether_type != ETHER_TYPE_IP) ||
+		(IPV4_HLEN(iph) < IPV4_HLEN_MIN) ||
+		(IP_VER(iph) != IP_VER_4)) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+bool
+dhd_check_dhcp(uint8 *pktdata)
+{
+	hdr_fmt_t *b = (hdr_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = &b->iph;
+
+	if (IPV4_PROT(iph) != IP_PROT_UDP) {
+		return FALSE;
+	}
+
+	/* check UDP port for bootp (67, 68) */
+	if (b->udph.src_port != htons(DHCP_PORT_SERVER) &&
+	        b->udph.src_port != htons(DHCP_PORT_CLIENT) &&
+	        b->udph.dst_port != htons(DHCP_PORT_SERVER) &&
+	        b->udph.dst_port != htons(DHCP_PORT_CLIENT)) {
+	        return FALSE;
+	}
+
+	/* check header length */
+	if (ntohs(iph->tot_len) < ntohs(b->udph.len) + sizeof(struct bcmudp_hdr)) {
+	        return FALSE;
+	}
+	return TRUE;
+}
+
+#define BOOTP_CHADDR_LEN		16
+#define BOOTP_SNAME_LEN			64
+#define BOOTP_FILE_LEN			128
+#define BOOTP_MIN_DHCP_OPT_LEN		312
+#define BOOTP_MAGIC_COOKIE_LEN		4
+
+typedef struct bootp_fmt {
+	struct ipv4_hdr iph;
+	struct bcmudp_hdr udph;
+	uint8 op;
+	uint8 htype;
+	uint8 hlen;
+	uint8 hops;
+	uint32 transaction_id;
+	uint16 secs;
+	uint16 flags;
+	uint32 client_ip;
+	uint32 assigned_ip;
+	uint32 server_ip;
+	uint32 relay_ip;
+	uint8 hw_address[BOOTP_CHADDR_LEN];
+	uint8 server_name[BOOTP_SNAME_LEN];
+	uint8 file_name[BOOTP_FILE_LEN];
+	uint8 options[BOOTP_MIN_DHCP_OPT_LEN];
+} PACKED_STRUCT bootp_fmt_t;
+static const uint8 bootp_magic_cookie[4] = { 99, 130, 83, 99 };
+
+#ifdef DHD_DHCP_DUMP
+#if defined(DHD_DHCP_DUMP) || defined(WL_CFGVENDOR_CUST_ADVLOG)
+#define DHCP_MSGTYPE_DISCOVER		1
+#define DHCP_MSGTYPE_OFFER		2
+#define DHCP_MSGTYPE_REQUEST		3
+#define DHCP_MSGTYPE_DECLINE		4
+#define DHCP_MSGTYPE_ACK		5
+#define DHCP_MSGTYPE_NAK		6
+#define DHCP_MSGTYPE_RELEASE		7
+#define DHCP_MSGTYPE_INFORM		8
+#endif /* DHD_DHCP_DUMP || WL_CFGVENDOR_CUST_ADVLOG */
+
+#define DHCP_PRINT(str) \
+	do { \
+		if (tx) { \
+			DHD_PKTDUMP(("[%s] " str " %8s[%8s] [TX] : %s(%s) %s %s(%s)"TXFATE_FMT"\n", \
+				ifname, typestr, opstr, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, \
+				TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+		} else { \
+			DHD_PKTDUMP(("[%s] " str " %8s[%8s] [RX] : %s(%s) %s %s(%s)\n", \
+				ifname, typestr, opstr, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf)); \
+		} \
+	} while (0)
+
+#define MAX_DHCP_OPS_STR 3
+#define MAX_DHCP_TYPES_STR 9
+
+static char dhcp_ops[MAX_DHCP_OPS_STR][10] = {
+	"NA", "REQUEST", "REPLY"
+};
+static char dhcp_types[MAX_DHCP_TYPES_STR][10] = {
+	"NA", "DISCOVER", "OFFER", "REQUEST", "DECLINE", "ACK", "NAK", "RELEASE", "INFORM"
+};
+
+#define DHCP_OPS_STR(ops)	((ops < MAX_DHCP_OPS_STR) ? \
+				(dhcp_ops[ops]) : "UNKNOWN_DHCP_OPS")
+#define DHCP_TYPES_STR(type)	((type < MAX_DHCP_TYPES_STR) ? \
+				(dhcp_types[type]) : "UNKNOWN_DHCP_TYPE")
+
+#ifdef DHD_IPV6_DUMP
+#define DHCP6_PRINT(str) \
+		do { \
+			if (tx) { \
+				DHD_PKTDUMP_MEM((str " %s[%s] [TX] -" TXFATE_FMT "\n", \
+					typestr, ifname, \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP_MEM((str " %s[%s] [RX]\n", \
+					typestr, ifname)); \
+			} \
+		} while (0)
+
+#define ICMPv6_PRINT(str) \
+	do { \
+		if (tx) { \
+			DHD_PKTDUMP_MEM((str " %s[%s] [TX] -" TXFATE_FMT "\n", \
+				typestr, ifname, \
+				TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+		} else { \
+			if (msgtype == ICMPV6_PKT_TYPE_RA) { \
+				DHD_PKTDUMP_MEM((str " %s[%s] flags 0x%x [RX]\n", \
+					typestr, ifname, icmp6_ra_flag)); \
+			} else { \
+				DHD_PKTDUMP_MEM((str " %s[%s] [RX]\n", \
+					typestr, ifname)); \
+			} \
+		} \
+	} while (0)
+
+#define MAX_DHCP6_TYPES_STR 11
+#define MAX_DHCP6_TYPES_STAT 11
+
+static char dhcp6_types[MAX_DHCP6_TYPES_STR][10] = {
+	"NA", "SOLICIT", "ADVERTISE", "REQUEST", "CONFIRM",
+	"RENEW", "REBIND", "REPLY", "RELEASE", "DECLINE", "RECONFIG"
+};
+#define DHCP6_TYPES_STR(type)    ((type < MAX_DHCP6_TYPES_STR) ? \
+		(dhcp6_types[type]) : "UNKNOWN_DHCP_TYPE")
+
+static const int dhcp6_types_stat[MAX_DHCP6_TYPES_STAT] = {
+	ST(INVALID), ST(SOLICIT), ST(ADVERTISE), ST(REQUEST),
+	ST(CONFIRM), ST(RENEW), ST(REBIND), ST(REPLY),
+	ST(RELEASE), ST(DECLINE), ST(RECONFIGURE)
+};
+#define DHCP6_TYPES_STAT(type)   ((type < MAX_DHCP6_TYPES_STAT) ? \
+		(dhcp6_types_stat[type]) : ST(INVALID))
+
+#define MAX_ICMPV6_TYPES_STAT 10
+static const int icmpv6_types_stat[MAX_ICMPV6_TYPES_STAT] = {
+	ST(INVALID), ST(ECHO_REQ), ST(ECHO_REPLY), ST(MULTI_QUERY),
+	ST(MULTI_REPORT), ST(MULTI_DONE), ST(ROUTER_SOLIC), ST(ROUTER_ADV),
+	ST(NEIGHBOR_SOLIC), ST(NEIGHBOR_ADV)
+};
+#define ICMPV6_TYPES_STAT(type)   ((type < MAX_ICMPV6_TYPES_STAT) ? \
+		(icmpv6_types_stat[type]) : ST(INVALID))
+#endif /* DHD_IPV6_DUMP */
+
+#ifdef DHD_STATUS_LOGGING
+#define MAX_DHCP_TYPES_STAT	9
+#define DHCP_TYPES_STAT(type)	((type < MAX_DHCP_TYPES_STAT) ? \
+				(dhcp_types_stat[type]) : ST(INVALID))
+static const int dhcp_types_stat[MAX_DHCP_TYPES_STAT] = {
+	ST(INVALID), ST(DHCP_DISCOVER), ST(DHCP_OFFER), ST(DHCP_REQUEST),
+	ST(DHCP_DECLINE), ST(DHCP_ACK), ST(DHCP_NAK), ST(DHCP_RELEASE),
+	ST(DHCP_INFORM)
+};
+#endif /* DHD_STATUS_LOGGING */
+
+void
+dhd_dhcp_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate)
+{
+	bootp_fmt_t *b = (bootp_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = &b->iph;
+	uint8 *ptr, *opt, *end = (uint8 *) b + ntohs(b->iph.tot_len);
+	uint8 dhcp_type = 0;
+	int len, opt_len;
+	char *ifname = NULL, *typestr = NULL, *opstr = NULL;
+	bool cond;
+	char sabuf[20]="", dabuf[20]="";
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!(dump_msg_level & DUMP_DHCP_VAL))
+		return;
+	bcm_ip_ntoa((struct ipv4_addr *)iph->src_ip, sabuf);
+	bcm_ip_ntoa((struct ipv4_addr *)iph->dst_ip, dabuf);
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+
+	ifname = dhd_ifname(dhdp, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+	len = ntohs(b->udph.len) - sizeof(struct bcmudp_hdr);
+	opt_len = len - (sizeof(*b) - sizeof(struct ipv4_hdr) -
+		sizeof(struct bcmudp_hdr) - sizeof(b->options));
+
+	/* parse bootp options */
+	if (opt_len >= BOOTP_MAGIC_COOKIE_LEN &&
+		!memcmp(b->options, bootp_magic_cookie, BOOTP_MAGIC_COOKIE_LEN)) {
+		ptr = &b->options[BOOTP_MAGIC_COOKIE_LEN];
+		while (ptr < end && *ptr != 0xff) {
+			opt = ptr++;
+			if (*opt == 0) {
+				continue;
+			}
+			ptr += *ptr + 1;
+			if (ptr >= end) {
+				break;
+			}
+			if (*opt == DHCP_OPT_MSGTYPE) {
+				if (opt[1]) {
+					dhcp_type = opt[2];
+					typestr = DHCP_TYPES_STR(dhcp_type);
+					opstr = DHCP_OPS_STR(b->op);
+					DHD_STATLOG_DATA(dhdp, DHCP_TYPES_STAT(dhcp_type),
+						ifidx, tx, cond);
+					DHCP_PRINT("DHCP");
+					break;
+				}
+			}
+		}
+	}
+}
+#endif /* DHD_DHCP_DUMP */
+
+bool
+dhd_check_icmp(uint8 *pktdata)
+{
+	uint8 *pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = (struct ipv4_hdr *)pkt;
+
+	if (IPV4_PROT(iph) != IP_PROT_ICMP) {
+		return FALSE;
+	}
+
+	/* check header length */
+	if (ntohs(iph->tot_len) - IPV4_HLEN(iph) < sizeof(struct bcmicmp_hdr)) {
+		return FALSE;
+	}
+	return TRUE;
+}
+
+bool
+dhd_check_icmpv6(uint8 *pktdata, uint32 plen)
+{
+	uint8 *pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	struct ipv6_hdr *ip6h = (struct ipv6_hdr *)pkt;
+
+	if (IPV6_PROT(ip6h) != IP_PROT_ICMP6) {
+		return FALSE;
+	}
+
+	/* check header length */
+	if (plen <= IPV6_MIN_HLEN) {
+		return FALSE;
+	}
+	return TRUE;
+}
+
+#ifdef DHD_ICMP_DUMP
+#define ICMP_TYPE_DEST_UNREACH		3
+#define ICMP_ECHO_SEQ_OFFSET		6
+#define ICMP_ECHO_SEQ(h) (*(uint16 *)((uint8 *)(h) + (ICMP_ECHO_SEQ_OFFSET)))
+#define ICMP_PING_PRINT(str) \
+	do { \
+		if (tx) { \
+			DHD_PKTDUMP_MEM(("[%s] "str " [TX] : %s(%s) %s %s(%s) SEQNUM=%d" \
+				TXFATE_FMT"\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, seqnum, \
+				TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+		} else { \
+			DHD_PKTDUMP_MEM(("[%s] "str " [RX] : %s(%s) %s %s(%s) SEQNUM=%d\n", \
+				ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, seqnum)); \
+		} \
+	} while (0)
+
+#define ICMP_PRINT(str) \
+	do { \
+		if (tx) { \
+			DHD_PKTDUMP_MEM(("[%s] "str " [TX] : %s(%s) %s %s(%s) TYPE=%d, CODE=%d" \
+				TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, type, code, \
+				TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+		} else { \
+			DHD_PKTDUMP_MEM(("[%s] "str " [RX] : %s(%s) %s %s(%s) TYPE=%d," \
+				" CODE=%d\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, type, code)); \
+		} \
+	} while (0)
+
+void
+dhd_icmp_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate)
+{
+	uint8 *pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = (struct ipv4_hdr *)pkt;
+	struct bcmicmp_hdr *icmph;
+	char *ifname;
+	bool cond;
+	uint16 seqnum, type, code;
+	char sabuf[20]="", dabuf[20]="";
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!(dump_msg_level & DUMP_ICMP_VAL))
+		return;
+
+	ifname = dhd_ifname(dhdp, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+	icmph = (struct bcmicmp_hdr *)((uint8 *)pkt + sizeof(struct ipv4_hdr));
+	seqnum = 0;
+	type = icmph->type;
+	code = icmph->code;
+	bcm_ip_ntoa((struct ipv4_addr *)iph->src_ip, sabuf);
+	bcm_ip_ntoa((struct ipv4_addr *)iph->dst_ip, dabuf);
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+	if (type == ICMP_TYPE_ECHO_REQUEST) {
+		seqnum = ntoh16(ICMP_ECHO_SEQ(icmph));
+		DHD_STATLOG_DATA(dhdp, ST(ICMP_PING_REQ), ifidx, tx, cond);
+		ICMP_PING_PRINT("PING REQUEST");
+	} else if (type == ICMP_TYPE_ECHO_REPLY) {
+		seqnum = ntoh16(ICMP_ECHO_SEQ(icmph));
+		DHD_STATLOG_DATA(dhdp, ST(ICMP_PING_RESP), ifidx, tx, cond);
+		ICMP_PING_PRINT("PING REPLY  ");
+	} else if (type == ICMP_TYPE_DEST_UNREACH) {
+		DHD_STATLOG_DATA(dhdp, ST(ICMP_DEST_UNREACH), ifidx, tx, cond);
+		ICMP_PRINT("ICMP DEST UNREACH");
+	} else {
+		DHD_STATLOG_DATA(dhdp, ST(ICMP_OTHER), ifidx, tx, cond);
+		ICMP_PRINT("ICMP OTHER");
+	}
+}
+#endif /* DHD_ICMP_DUMP */
+
+bool
+dhd_check_arp(uint8 *pktdata, uint16 ether_type)
+{
+	uint8 *pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	struct bcmarp *arph = (struct bcmarp *)pkt;
+
+	/* validation check */
+	if ((ether_type != ETHER_TYPE_ARP) ||
+		(arph->htype != hton16(HTYPE_ETHERNET)) ||
+		(arph->hlen != ETHER_ADDR_LEN) ||
+		(arph->plen != 4)) {
+		return FALSE;
+	}
+	return TRUE;
+}
+
+#ifdef DHD_ARP_DUMP
+#ifdef BOARD_HIKEY
+/* On Hikey, due to continuous ARP prints
+ * DPC not scheduled. Hence rate limit the prints.
+ */
+#define DHD_PKTDUMP_ARP DHD_ERROR_RLMT
+#else
+#define DHD_PKTDUMP_ARP DHD_PKTDUMP
+#endif /* BOARD_HIKEY */
+
+#define ARP_PRINT(str) \
+	do { \
+		if (tx) { \
+			if (dump_enabled && pktfate && !TX_FATE_ACKED(pktfate)) { \
+				DHD_PKTDUMP(("[%s] "str " [TX] : %s(%s) %s %s(%s)"TXFATE_FMT"\n", \
+					ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP_MEM(("[%s] "str " [TX] : %s(%s) %s %s(%s)"TXFATE_FMT"\n", \
+					ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} \
+		} else { \
+			DHD_PKTDUMP_MEM(("[%s] "str " [RX] : %s(%s) %s %s(%s)\n", \
+				ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf)); \
+		} \
+	} while (0) \
+
+#define ARP_PRINT_OTHER(str) \
+	do { \
+		if (tx) { \
+			if (dump_enabled && pktfate && !TX_FATE_ACKED(pktfate)) { \
+				DHD_PKTDUMP(("[%s] "str " [TX] : %s(%s) %s %s(%s) op_code=%d" \
+					TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, opcode, \
+					TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP_MEM(("[%s] "str " [TX] : %s(%s) %s %s(%s) op_code=%d" \
+				TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, opcode, \
+				TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} \
+		} else { \
+			DHD_PKTDUMP_MEM(("[%s] "str " [RX] : %s(%s) %s %s(%s) op_code=%d\n", \
+				ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, opcode)); \
+		} \
+	} while (0)
+
+void
+dhd_arp_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate)
+{
+	uint8 *pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	struct bcmarp *arph = (struct bcmarp *)pkt;
+	char *ifname;
+	uint16 opcode;
+	bool cond, dump_enabled;
+	char sabuf[20]="", dabuf[20]="";
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!(dump_msg_level & DUMP_ARP_VAL))
+		return;
+
+	ifname = dhd_ifname(dhdp, ifidx);
+	opcode = ntoh16(arph->oper);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+	dump_enabled = dhd_dump_pkt_enabled(dhdp);
+	bcm_ip_ntoa((struct ipv4_addr *)arph->src_ip, sabuf);
+	bcm_ip_ntoa((struct ipv4_addr *)arph->dst_ip, dabuf);
+	bcm_ether_ntoa((struct ether_addr *)arph->dst_eth, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)arph->src_eth, seabuf);
+	if (opcode == ARP_OPC_REQUEST) {
+		DHD_STATLOG_DATA(dhdp, ST(ARP_REQ), ifidx, tx, cond);
+		ARP_PRINT("ARP REQUEST ");
+	} else if (opcode == ARP_OPC_REPLY) {
+		DHD_STATLOG_DATA(dhdp, ST(ARP_RESP), ifidx, tx, cond);
+		ARP_PRINT("ARP RESPONSE");
+	} else {
+		ARP_PRINT_OTHER("ARP OTHER");
+	}
+
+	if (ifidx == 0) {
+		dhd_dump_pkt_cnts_inc(dhdp, tx, pktfate, PKT_CNT_TYPE_ARP);
+	}
+}
+#endif /* DHD_ARP_DUMP */
+
+bool
+dhd_check_dns(uint8 *pktdata)
+{
+	hdr_fmt_t *dnsh = (hdr_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = &dnsh->iph;
+
+	if (IPV4_PROT(iph) != IP_PROT_UDP) {
+		return FALSE;
+	}
+
+	/* check UDP port for DNS */
+	if (dnsh->udph.src_port != hton16(UDP_PORT_DNS) &&
+		dnsh->udph.dst_port != hton16(UDP_PORT_DNS)) {
+		return FALSE;
+	}
+
+	/* check header length */
+	if (ntoh16(iph->tot_len) < (ntoh16(dnsh->udph.len) +
+		sizeof(struct bcmudp_hdr))) {
+		return FALSE;
+	}
+	return TRUE;
+}
+
+#ifdef DHD_DNS_DUMP
+typedef struct dns_fmt {
+	struct ipv4_hdr iph;
+	struct bcmudp_hdr udph;
+	uint16 id;
+	uint16 flags;
+	uint16 qdcount;
+	uint16 ancount;
+	uint16 nscount;
+	uint16 arcount;
+} PACKED_STRUCT dns_fmt_t;
+
+#define DNS_QR_LOC		15
+#define DNS_OPCODE_LOC		11
+#define DNS_RCODE_LOC		0
+#define DNS_QR_MASK		((0x1) << (DNS_QR_LOC))
+#define DNS_OPCODE_MASK		((0xF) << (DNS_OPCODE_LOC))
+#define DNS_RCODE_MASK		((0xF) << (DNS_RCODE_LOC))
+#define GET_DNS_QR(flags)	(((flags) & (DNS_QR_MASK)) >> (DNS_QR_LOC))
+#define GET_DNS_OPCODE(flags)	(((flags) & (DNS_OPCODE_MASK)) >> (DNS_OPCODE_LOC))
+#define GET_DNS_RCODE(flags)	(((flags) & (DNS_RCODE_MASK)) >> (DNS_RCODE_LOC))
+#define DNS_UNASSIGNED_OPCODE(flags) ((GET_DNS_OPCODE(flags) >= (6)))
+
+static const char dns_opcode_types[][11] = {
+	"QUERY", "IQUERY", "STATUS", "UNASSIGNED", "NOTIFY", "UPDATE"
+};
+
+#define DNSOPCODE(op)	\
+	(DNS_UNASSIGNED_OPCODE(flags) ? "UNASSIGNED" : dns_opcode_types[op])
+
+#define DNS_REQ_PRINT(str) \
+	do { \
+		if (tx) { \
+			if (dump_enabled && pktfate && !TX_FATE_ACKED(pktfate)) { \
+				DHD_PKTDUMP(("[%s] " str " [TX] : %s(%s) %s %s(%s) ID:0x%04X OPCODE:%s" \
+					TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, \
+					id, DNSOPCODE(opcode), TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP_MEM(("[%s] " str " [TX] : %s(%s) %s %s(%s) ID:0x%04X OPCODE:%s" \
+					TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, \
+					id, DNSOPCODE(opcode), TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} \
+		} else { \
+			DHD_PKTDUMP_MEM(("[%s] " str " [RX] : %s(%s) %s %s(%s) ID:0x%04X OPCODE:%s\n", \
+				ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, tx?"->":"<-", \
+				tx?dabuf:sabuf, tx?deabuf:seabuf, id, DNSOPCODE(opcode))); \
+		} \
+	} while (0)
+
+#define DNS_RESP_PRINT(str) \
+	do { \
+		if (tx) { \
+			if (dump_enabled && pktfate && !TX_FATE_ACKED(pktfate)) { \
+				DHD_PKTDUMP(("[%s] " str " [TX] : %s(%s) %s %s(%s) ID:0x%04X OPCODE:%s RCODE:%d" \
+					TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, id, DNSOPCODE(opcode), \
+					GET_DNS_RCODE(flags), TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} else { \
+				DHD_PKTDUMP_MEM(("[%s] " str " [TX] : %s(%s) %s %s(%s) ID:0x%04X OPCODE:%s RCODE:%d" \
+					TXFATE_FMT "\n", ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+					tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, id, DNSOPCODE(opcode), \
+					GET_DNS_RCODE(flags), TX_PKTHASH(pkthash), TX_FATE(pktfate))); \
+			} \
+		} else { \
+			DHD_PKTDUMP_MEM(("[%s] " str " [RX] : %s(%s) %s %s(%s) ID:0x%04X OPCODE:%s RCODE:%d\n", \
+				ifname, tx?sabuf:dabuf, tx?seabuf:deabuf, \
+				tx?"->":"<-", tx?dabuf:sabuf, tx?deabuf:seabuf, \
+				id, DNSOPCODE(opcode), GET_DNS_RCODE(flags))); \
+		} \
+	} while (0)
+
+void
+dhd_dns_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate)
+{
+	dns_fmt_t *dnsh = (dns_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = &dnsh->iph;
+	uint16 flags, opcode, id;
+	char *ifname;
+	bool cond, dump_enabled;
+	char sabuf[20]="", dabuf[20]="";
+	char seabuf[ETHER_ADDR_STR_LEN]="";
+	char deabuf[ETHER_ADDR_STR_LEN]="";
+
+	if (!(dump_msg_level & DUMP_DNS_VAL))
+		return;
+
+	ifname = dhd_ifname(dhdp, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+	dump_enabled = dhd_dump_pkt_enabled(dhdp);
+	flags = hton16(dnsh->flags);
+	opcode = GET_DNS_OPCODE(flags);
+	id = hton16(dnsh->id);
+	bcm_ip_ntoa((struct ipv4_addr *)iph->src_ip, sabuf);
+	bcm_ip_ntoa((struct ipv4_addr *)iph->dst_ip, dabuf);
+	bcm_ether_ntoa((struct ether_addr *)pktdata, deabuf);
+	bcm_ether_ntoa((struct ether_addr *)(pktdata+6), seabuf);
+	if (GET_DNS_QR(flags)) {
+		/* Response */
+		DHD_STATLOG_DATA(dhdp, ST(DNS_RESP), ifidx, tx, cond);
+		DNS_RESP_PRINT("DNS RESPONSE");
+	} else {
+		/* Request */
+		DHD_STATLOG_DATA(dhdp, ST(DNS_QUERY), ifidx, tx, cond);
+		DNS_REQ_PRINT("DNS REQUEST");
+	}
+
+	if (ifidx == 0) {
+		dhd_dump_pkt_cnts_inc(dhdp, tx, pktfate, PKT_CNT_TYPE_DNS);
+	}
+}
+#endif /* DHD_DNS_DUMP */
+
+#ifdef DHD_IPV6_DUMP
+void
+dhd_dhcp6_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+uint32 *pkthash, uint16 *pktfate)
+{
+	dhcp6_fmt_t *b = (dhcp6_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	char *ifname = NULL, *typestr = NULL;
+	uint8 msgtype;
+	bool cond;
+
+	ifname = dhd_ifname(dhdp, ifidx);
+	msgtype = ntoh16(b->msgtype);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+
+	/* only handle DHCPv6 msgtype from 1 ~ 10 */
+	if (msgtype > 10)
+		return;
+
+	typestr = DHCP6_TYPES_STR(msgtype);
+	DHD_STATLOG_DATA(dhdp, DHCP6_TYPES_STAT(msgtype),
+			ifidx, tx, cond);
+	DHCP6_PRINT("DHCPv6");
+	return;
+}
+
+void
+dhd_dns6_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate)
+{
+	dns6_fmt_t *dns6h = (dns6_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	uint16 flags, opcode, id;
+	char *ifname;
+	bool cond, dump_enabled;
+
+	ifname = dhd_ifname(dhdp, ifidx);
+	cond = (tx && pktfate) ? FALSE : TRUE;
+	dump_enabled = dhd_dump_pkt_enabled(dhdp);
+	flags = hton16(dns6h->flags);
+	opcode = GET_DNS_OPCODE(flags);
+	id = hton16(dns6h->id);
+	if (GET_DNS_QR(flags)) {
+		/* Response */
+		DHD_STATLOG_DATA(dhdp, ST(DNS_RESP), ifidx, tx, cond);
+		DNS_RESP_PRINT("DNS6 RESPONSE");
+	} else {
+		/* Request */
+		DHD_STATLOG_DATA(dhdp, ST(DNS_QUERY), ifidx, tx, cond);
+		DNS_REQ_PRINT("DNS6 REQUEST");
+	}
+}
+
+char*
+icmpv6_types(uint8 type)
+{
+	switch (type) {
+		case ICMPV6_PKT_TYPE_ECHO_REQ:
+			return "Echo Request";
+		case ICMPV6_PKT_TYPE_ECHO_REPLY:
+			return "Echo Reply";
+		case ICMPV6_PKT_TYPE_MULTI_LST_QUERY:
+			return "Multicast Listener Query";
+		case ICMPV6_PKT_TYPE_MULTI_LST_REPORT:
+			return "Multicast Listener Report";
+		case ICMPV6_PKT_TYPE_MULTI_LST_DONE:
+			return "Multicast Listener Done";
+		case ICMPV6_PKT_TYPE_RS:
+			return "Router Solicitation";
+		case ICMPV6_PKT_TYPE_RA:
+			return "Router Advertisement";
+		case ICMPV6_PKT_TYPE_NS:
+			return "Neighbor Solicitation";
+		case ICMPV6_PKT_TYPE_NA:
+			return "Neighbor Advertisement";
+		default:
+			return NULL;
+	}
+}
+
+void
+dhd_icmpv6_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+uint32 *pkthash, uint16 *pktfate)
+{
+	struct ipv6_hdr *ip6h = (struct ipv6_hdr *)&pktdata[ETHER_HDR_LEN];
+	struct icmp6_hdr *icmpv6;
+	char *ifname = NULL, *typestr = NULL;
+	uint8 icmp6_ra_flag;
+	uint8 msgtype;
+	bool cond;
+
+	uint8 nexthdr = ip6h->nexthdr;
+
+	if (nexthdr != ICMPV6_HEADER_TYPE)
+		return;
+
+	icmpv6 = (struct icmp6_hdr *) ((uint8 *)ip6h + sizeof(struct ipv6_hdr));
+	ifname = dhd_ifname(dhdp, ifidx);
+	msgtype = icmpv6->icmp6_type;
+	cond = (tx && pktfate) ? FALSE : TRUE;
+	typestr = icmpv6_types(msgtype);
+
+	/* only handle ICMPv6 msgtype from 128 ~ 136 */
+	if (!typestr)
+		return;
+
+	if (msgtype == ICMPV6_PKT_TYPE_RA) {
+		/* check IPv6 configuration is stateless or stateful DHCPv6 */
+		icmp6_ra_flag = *((uint8 *)ip6h + sizeof(struct ipv6_hdr) + ICMPV6_RA_FLAG_OFFSET);
+	}
+
+	DHD_STATLOG_DATA(dhdp, ICMPV6_TYPES_STAT(msgtype - 127),
+			ifidx, tx, cond);
+	ICMPv6_PRINT("ICMPv6");
+	return;
+}
+
+bool
+dhd_check_dhcp6(uint8 *pktdata, uint32 plen)
+{
+	ipv6hdr_fmt_t *b = (ipv6hdr_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv6_hdr *ip6h = &b->ip6h;
+
+	/* check header length */
+	if (plen <= IPV6_MIN_HLEN) {
+		return FALSE;
+	}
+
+	if (IPV6_PROT(ip6h) != IP_PROT_UDP) {
+		return FALSE;
+	}
+
+	/* check UDP port for bootp (546, 547) */
+	if (b->udph.src_port != htons(DHCP6_PORT_SERVER) &&
+			b->udph.src_port != htons(DHCP6_PORT_CLIENT) &&
+			b->udph.dst_port != htons(DHCP6_PORT_SERVER) &&
+			b->udph.dst_port != htons(DHCP6_PORT_CLIENT)) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+bool
+dhd_check_dns6(uint8 *pktdata, uint32 plen)
+{
+	ipv6hdr_fmt_t *b = (ipv6hdr_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv6_hdr *ip6h = &b->ip6h;
+
+	/* check header length */
+	if (plen <= IPV6_MIN_HLEN) {
+		return FALSE;
+	}
+
+	if (IPV6_PROT(ip6h) != IP_PROT_UDP) {
+		return FALSE;
+	}
+
+	/* check UDP port for DNS */
+	if (b->udph.src_port != hton16(UDP_PORT_DNS) &&
+		b->udph.dst_port != hton16(UDP_PORT_DNS)) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+#endif /* DHD_IPV6_DUMP */
+
+#ifdef DHD_TRX_DUMP
+void
+dhd_trx_pkt_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, uint32 pktlen, bool tx)
+{
+	struct ether_header *eh;
+	uint16 protocol;
+	char *pkttype = "UNKNOWN";
+
+	if (!(dump_msg_level & DUMP_TRX_VAL))
+		return;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (!pktdata) {
+		DHD_ERROR(("%s: pktdata is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	eh = (struct ether_header *)pktdata;
+	protocol = hton16(eh->ether_type);
+	BCM_REFERENCE(pktlen);
+
+	switch (protocol) {
+	case ETHER_TYPE_IP:
+		pkttype = "IP";
+		break;
+	case ETHER_TYPE_ARP:
+		pkttype = "ARP";
+		break;
+	case ETHER_TYPE_BRCM:
+		pkttype = "BRCM";
+		break;
+	case ETHER_TYPE_802_1X:
+		pkttype = "802.1X";
+		break;
+	case ETHER_TYPE_WAI:
+		pkttype = "WAPI";
+		break;
+	default:
+		break;
+	}
+
+	if (protocol != ETHER_TYPE_BRCM) {
+		if (pktdata[0] == 0xFF) {
+			DHD_PKTDUMP(("[%s] %s BROADCAST DUMP - %s\n",
+				dhd_ifname(dhdp, ifidx), tx?"TX":"RX", pkttype));
+		} else if (pktdata[0] & 1) {
+			DHD_PKTDUMP(("[%s] %s MULTICAST DUMP " MACDBG " - %s\n",
+				dhd_ifname(dhdp, ifidx), tx?"TX":"RX", MAC2STRDBG(pktdata), pkttype));
+		} else {
+			DHD_PKTDUMP(("[%s] %s DUMP - %s\n",
+				dhd_ifname(dhdp, ifidx), tx?"TX":"RX", pkttype));
+		}
+#ifdef DHD_RX_FULL_DUMP
+		prhex("Data", pktdata, pktlen);
+#endif /* DHD_RX_FULL_DUMP */
+	}
+	else {
+		DHD_PKTDUMP(("[%s] %s DUMP - %s\n",
+			dhd_ifname(dhdp, ifidx), tx?"TX":"RX", pkttype));
+	}
+}
+#endif /* DHD_RX_DUMP */
+
+#ifdef WL_CFGVENDOR_CUST_ADVLOG
+typedef struct dhd_advlog_arr_map_entry {
+	uint32 avglog_type;
+	dhd_advlog_map_entry_t *arr;
+	uint32 arr_len;
+} dhd_advlog_arr_map_entry_t;
+
+typedef enum dhd_advlog_type {
+	DHD_ADVLOG_DHCP_TX      = 0,
+	DHD_ADVLOG_DHCP_RX      = 1,
+	DHD_ADVLOG_EAP          = 2,
+	DHD_ADVLOG_EAPOL        = 3,
+	DHD_ADVLOG_LAST         = 4
+} dhd_advlog_type_t;
+
+static dhd_advlog_map_entry_t eap_type_map[] = {
+	{EAP_TYPE_IDENT, "Identity"},
+	{EAP_TYPE_TLS, "TLS"},
+	{EAP_TYPE_LEAP, "LEAP"},
+	{EAP_TYPE_TTLS, "TTLS"},
+	{EAP_TYPE_AKA, "AKA"},
+	{EAP_TYPE_PEAP, "PEAP"},
+	{EAP_TYPE_FAST,	"FAST"},
+	{EAP_TYPE_PSK, "PSK"},
+	{EAP_TYPE_AKAP, "AKAP"},
+};
+
+static dhd_advlog_map_entry_t eap_advlog_map[] = {
+	{EAP_CODE_REQUEST, "[EAP] REQ type=%s len=%d"},
+	{EAP_CODE_RESPONSE, "[EAP] RESP type=%s len=%d tx_status=%s"},
+	{EAP_CODE_SUCCESS, "[EAP] SUCC"},
+	{EAP_CODE_FAILURE, "[EAP] FAIL"},
+};
+
+static dhd_advlog_map_entry_t dhcp_tx_advlog_map[] = {
+	{DHCP_MSGTYPE_DISCOVER, "[DHCP] DISCOVER tx_status=%s"},
+	{DHCP_MSGTYPE_REQUEST, "[DHCP] REQUEST tx_status=%s"},
+};
+
+static dhd_advlog_map_entry_t dhcp_rx_advlog_map[] = {
+	{DHCP_MSGTYPE_OFFER, "[DHCP] OFFER"},
+	{DHCP_MSGTYPE_ACK, "[DHCP] ACK"},
+	{DHCP_MSGTYPE_NAK, "[DHCP] NAK"},
+};
+
+static dhd_advlog_map_entry_t eapol_advlog_map[] = {
+	{EAPOL_4WAY_M1, "[EAPOL] 4WAY M1"},
+	{EAPOL_4WAY_M2, "[EAPOL] 4WAY M2 tx_status=%s"},
+	{EAPOL_4WAY_M3, "[EAPOL] 4WAY M3"},
+	{EAPOL_4WAY_M4, "[EAPOL] 4WAY M4 tx_status=%s"},
+	{EAPOL_GROUPKEY_M1, "[EAPOL] GTK M1"},
+	{EAPOL_GROUPKEY_M2, "[EAPOL] GTK M2 tx_status=%s"},
+};
+
+static dhd_advlog_arr_map_entry_t advlog_map_arr[] = {
+	{DHD_ADVLOG_DHCP_TX, dhcp_tx_advlog_map, ARRAY_SIZE(dhcp_tx_advlog_map)},
+	{DHD_ADVLOG_DHCP_RX, dhcp_rx_advlog_map, ARRAY_SIZE(dhcp_rx_advlog_map)},
+	{DHD_ADVLOG_EAP, eap_advlog_map, ARRAY_SIZE(eap_advlog_map)},
+	{DHD_ADVLOG_EAPOL, eapol_advlog_map, ARRAY_SIZE(eapol_advlog_map)},
+};
+
+const char* get_advlog_val(dhd_advlog_map_entry_t *arr, uint32 arr_len, int tag)
+{
+	int i;
+	for (i = 0; i < arr_len; i++) {
+		if (tag == arr[i].key) {
+			return arr[i].val;
+		}
+	}
+	return NULL;
+}
+
+static int get_dhcp_type(uint8 *pktdata)
+{
+	bootp_fmt_t *b = (bootp_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	uint8 *ptr, *opt, *end = (uint8 *) b + ntohs(b->iph.tot_len);
+	int dhcp_type = 0, len, opt_len;
+
+	len = ntohs(b->udph.len) - sizeof(struct bcmudp_hdr);
+	opt_len = len - (sizeof(*b) - sizeof(struct ipv4_hdr) -
+		sizeof(struct bcmudp_hdr) - sizeof(b->options));
+
+	/* parse bootp options */
+	if (opt_len >= BOOTP_MAGIC_COOKIE_LEN &&
+		!memcmp(b->options, bootp_magic_cookie, BOOTP_MAGIC_COOKIE_LEN)) {
+		ptr = &b->options[BOOTP_MAGIC_COOKIE_LEN];
+		while (ptr < end && *ptr != 0xff) {
+			opt = ptr++;
+			if (*opt == 0) {
+				continue;
+			}
+			ptr += *ptr + 1;
+			if (ptr >= end) {
+				break;
+			}
+			if (*opt == DHCP_OPT_MSGTYPE) {
+				if (opt[1]) {
+					dhcp_type = opt[2];
+					return dhcp_type;
+				}
+			}
+		}
+	}
+	return BCME_ERROR;
+}
+
+static int dhd_send_supp(uint32 avglog_type, int arg_tag, int arg_type, uint32 pktlen,
+	bool tx, uint16 *pktfate)
+{
+	const char *type_str = NULL;
+	const char *fmt_str = NULL;
+	dhd_advlog_map_entry_t *fmt_arr;
+	uint32 fmt_arr_len;
+
+	if (avglog_type >= DHD_ADVLOG_LAST) {
+		DHD_ERROR(("%s incorrect array index type:%d tag:%d\n",
+			__func__, avglog_type, arg_tag));
+		return BCME_ERROR;
+	}
+	if ((arg_type && !pktlen) || (!arg_type && pktlen)) {
+		DHD_ERROR(("%s Not expected value pair type:%d tag:%d pktlen:%u\n",
+			__func__, avglog_type, arg_tag, pktlen));
+		return BCME_ERROR;
+	}
+	if (arg_tag <= 0) {
+		DHD_ERROR(("%s Invalid tag number type:%d tag:%d\n",
+			__func__, avglog_type, arg_tag));
+		return BCME_ERROR;
+	}
+
+	/* Get the format string corresponding to the type */
+	fmt_arr = advlog_map_arr[avglog_type].arr;
+	fmt_arr_len = advlog_map_arr[avglog_type].arr_len;
+	fmt_str = get_advlog_val(fmt_arr, fmt_arr_len, arg_tag);
+	if (!fmt_str) {
+		DHD_ERROR(("%s map arr not found type:%d tag:%d\n",
+			__func__, avglog_type, arg_tag));
+		return BCME_ERROR;
+	}
+
+	/* EAP REQ/RESP */
+	if (arg_type && pktlen) {
+		/* Get Identity/PEAP/FAST/etc string */
+		type_str = get_advlog_val(eap_type_map, ARRAY_SIZE(eap_type_map), arg_type);
+		if (!type_str) {
+			DHD_ERROR(("%s type string not found type:%d tag:%d\n",
+				__func__, avglog_type, arg_tag));
+			return BCME_ERROR;
+		}
+		if (tx && pktfate) {
+			/* EAP RESP */
+			SUPP_ADVLOG((fmt_str, type_str, pktlen, TX_FATE_EX(pktfate)));
+		} else if (!tx) {
+			/* EAP REQ */
+			SUPP_ADVLOG((fmt_str, type_str, pktlen));
+		}
+	} else {
+		if (tx && pktfate) {
+			/* EAPOL M2/M4, GTK M2, DHCP DISC/REQ */
+			SUPP_ADVLOG((fmt_str, TX_FATE_EX(pktfate)));
+		} else if (!tx) {
+			/* EAPOL M1/M3, GTK M1, DHCP OFFER/ACK/NAK, EAP SUCC/FAIL */
+			SUPP_ADVLOG((fmt_str));
+		}
+	}
+	return BCME_OK;
+}
+
+void dhd_send_supp_dhcp(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx, uint16 *pktfate)
+{
+	uint32 advlog_type;
+
+	/* Advanced Logging supports only STA mode */
+	if (!DHD_IF_ROLE_STA(dhdp, ifidx)) {
+		return;
+	}
+
+	if (tx) {
+		advlog_type = DHD_ADVLOG_DHCP_TX;
+	} else {
+		advlog_type = DHD_ADVLOG_DHCP_RX;
+	}
+	dhd_send_supp(advlog_type, get_dhcp_type(pktdata), 0, 0, tx, pktfate);
+}
+
+void dhd_send_supp_eap(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, uint32 pktlen,
+	bool tx, uint16 *pktfate)
+{
+	eapol_header_t *eapol_hdr;
+	eap_header_fmt_t *eap_hdr;
+	uint16 len = 0;
+	uint8 type = 0;
+
+	/* Advanced Logging supports only STA mode */
+	if (!DHD_IF_ROLE_STA(dhdp, ifidx)) {
+		return;
+	}
+
+	eapol_hdr = (eapol_header_t *)pktdata;
+	eap_hdr = (eap_header_fmt_t *)(eapol_hdr->body);
+
+	if (eapol_hdr->type == EAP_PACKET) {
+		/* EAP SUCC/FAIL has no arg to print */
+		if (eap_hdr->code != EAP_CODE_SUCCESS &&
+			eap_hdr->code != EAP_CODE_FAILURE) {
+			type = eap_hdr->type;
+			len = ntoh16(eap_hdr->len);
+		}
+		dhd_send_supp(DHD_ADVLOG_EAP, eap_hdr->code, type, len, tx, pktfate);
+	} else if (eapol_hdr->type == EAPOL_KEY) {
+		dhd_send_supp(DHD_ADVLOG_EAPOL, dhd_is_4way_msg(pktdata),
+			0, 0, tx, pktfate);
+	}
+}
+#endif /* WL_CFGVENDOR_CUST_ADVLOG */
+
+#ifdef BCMPCIE
+static bool
+dhd_is_eapol_pkt(dhd_pub_t *dhd, uint8 *pktdata, uint32 pktlen)
+{
+	eapol_header_t *eapol_hdr = (eapol_header_t *)pktdata;
+
+	eapol_hdr = (eapol_header_t *)pktdata;
+
+	if (eapol_hdr->type == EAP_PACKET) {
+		return TRUE;
+	} else if (eapol_hdr->type == EAPOL_START) {
+		return TRUE;
+	} else if (eapol_hdr->type == EAPOL_KEY) {
+		return TRUE;
+	} else {
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static bool
+dhd_is_arp_pkt(dhd_pub_t *dhdp, uint8 *pktdata)
+{
+	uint8 *pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	struct bcmarp *arph = (struct bcmarp *)pkt;
+	uint16 opcode;
+
+	/* validation check */
+	if (arph->htype != hton16(HTYPE_ETHERNET) ||
+		arph->hlen != ETHER_ADDR_LEN ||
+		arph->plen != 4) {
+		return FALSE;
+	}
+
+	opcode = ntoh16(arph->oper);
+	if (opcode == ARP_OPC_REQUEST) {
+		return TRUE;
+	} else if (opcode == ARP_OPC_REPLY) {
+		return TRUE;
+	} else {
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static bool
+dhd_is_dhcp_pkt(dhd_pub_t *dhdp, uint8 *pktdata)
+{
+	bootp_fmt_t *b = (bootp_fmt_t *)&pktdata[ETHER_HDR_LEN];
+	struct ipv4_hdr *iph = &b->iph;
+	uint8 *ptr, *opt, *end = (uint8 *) b + ntohs(b->iph.tot_len);
+	int len, opt_len;
+
+	/* check IP header */
+	if ((IPV4_HLEN(iph) < IPV4_HLEN_MIN) ||
+		IP_VER(iph) != IP_VER_4 ||
+		IPV4_PROT(iph) != IP_PROT_UDP) {
+		return FALSE;
+	}
+
+	/* check UDP port for bootp (67, 68) */
+	if (b->udph.src_port != htons(DHCP_PORT_SERVER) &&
+		b->udph.src_port != htons(DHCP_PORT_CLIENT) &&
+		b->udph.dst_port != htons(DHCP_PORT_SERVER) &&
+		b->udph.dst_port != htons(DHCP_PORT_CLIENT)) {
+		return FALSE;
+	}
+
+	/* check header length */
+	if (ntohs(iph->tot_len) < ntohs(b->udph.len) + sizeof(struct bcmudp_hdr)) {
+		return FALSE;
+	}
+
+	len = ntohs(b->udph.len) - sizeof(struct bcmudp_hdr);
+	opt_len = len - (sizeof(*b) - sizeof(struct ipv4_hdr) -
+		sizeof(struct bcmudp_hdr) - sizeof(b->options));
+
+	/* parse bootp options */
+	if (opt_len >= BOOTP_MAGIC_COOKIE_LEN &&
+		!memcmp(b->options, bootp_magic_cookie, BOOTP_MAGIC_COOKIE_LEN)) {
+		ptr = &b->options[BOOTP_MAGIC_COOKIE_LEN];
+		while (ptr < end && *ptr != 0xff) {
+			opt = ptr++;
+			if (*opt == 0) {
+				continue;
+			}
+			ptr += *ptr + 1;
+			if (ptr >= end) {
+				break;
+			}
+			if (*opt == DHCP_OPT_MSGTYPE) {
+				if (opt[1]) {
+					return TRUE;
+				}
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+static bool
+dhd_is_icmp_pkt(dhd_pub_t *dhd, uint8 *pktdata, uint32 pktlen)
+{
+	uint8 *pkt;
+	struct ipv4_hdr *iph;
+
+	pkt = (uint8 *)&pktdata[ETHER_HDR_LEN];
+	iph = (struct ipv4_hdr *)pkt;
+
+	/* check IP header */
+	if ((IPV4_HLEN(iph) < IPV4_HLEN_MIN) ||
+		IP_VER(iph) != IP_VER_4 ||
+		IPV4_PROT(iph) != IP_PROT_ICMP) {
+		return FALSE;
+	}
+
+	/* check header length */
+	if (ntohs(iph->tot_len) - IPV4_HLEN(iph) < sizeof(struct bcmicmp_hdr)) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+bool
+dhd_match_pkt_type(dhd_pub_t *dhd, uint8 *pktdata, uint32 pktlen)
+{
+	struct ether_header *eh;
+	uint16 ether_type;
+	bool match = FALSE;
+
+	if (!pktdata || pktlen < ETHER_HDR_LEN) {
+		return match;
+	}
+
+	eh = (struct ether_header *)pktdata;
+	ether_type = ntoh16(eh->ether_type);
+	if ((dhd->conf->enq_hdr_pkt & ENQ_PKT_TYPE_EAPOL) &&
+			ether_type == ETHER_TYPE_802_1X) {
+		match = dhd_is_eapol_pkt(dhd, pktdata, pktlen);
+	}
+	else if ((dhd->conf->enq_hdr_pkt & ENQ_PKT_TYPE_ARP) &&
+			ntoh16(eh->ether_type) == ETHER_TYPE_ARP) {
+		match = dhd_is_arp_pkt(dhd, pktdata);
+	}
+	else if (ntoh16(eh->ether_type) == ETHER_TYPE_IP) {
+		if (dhd->conf->enq_hdr_pkt & ENQ_PKT_TYPE_ICMP)
+			match = dhd_is_icmp_pkt(dhd, pktdata, pktlen);
+		if (!match && dhd->conf->enq_hdr_pkt & ENQ_PKT_TYPE_DHCP)
+			match = dhd_is_dhcp_pkt(dhd, pktdata);
+	}
+
+	return match;
+}
+#endif /* BCMPCIE */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_pktdump.h b/drivers/net/wireless/ap12275/dhd_linux_pktdump.h
new file mode 100644
index 000000000000..e1ae011725da
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_pktdump.h
@@ -0,0 +1,167 @@
+/*
+ * Header file for the Packet dump helper functions
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_LINUX_PKTDUMP_H_
+#define __DHD_LINUX_PKTDUMP_H_
+
+#include <typedefs.h>
+#include <dhd.h>
+
+typedef enum {
+	EAPOL_OTHER = 0,
+	EAPOL_4WAY_M1,
+	EAPOL_4WAY_M2,
+	EAPOL_4WAY_M3,
+	EAPOL_4WAY_M4,
+	EAPOL_GROUPKEY_M1,
+	EAPOL_GROUPKEY_M2
+} msg_eapol_t;
+
+typedef enum pkt_cnt_rsn {
+	PKT_CNT_RSN_INVALID	= 0,
+	PKT_CNT_RSN_ROAM	= 1,
+	PKT_CNT_RSN_GRPKEY_UP	= 2,
+	PKT_CNT_RSN_CONNECT	= 3,
+	PKT_CNT_RSN_MAX		= 4
+} pkt_cnt_rsn_t;
+
+enum pkt_type {
+	PKT_TYPE_DATA = 0,
+	PKT_TYPE_DHCP = 1,
+	PKT_TYPE_ICMP = 2,
+	PKT_TYPE_ICMPV6 = 3,
+	PKT_TYPE_DNS = 4,
+	PKT_TYPE_ARP = 5,
+	PKT_TYPE_EAP = 6,
+	PKT_TYPE_DHCP6 = 7,
+	PKT_TYPE_DNS6 = 8
+};
+
+#ifdef WL_CFGVENDOR_CUST_ADVLOG
+#define DHD_ADVLOG_FMT_MAX      256u
+typedef struct dhd_advlog_map_entry {
+	uint32 key;
+	const char val[DHD_ADVLOG_FMT_MAX];
+} dhd_advlog_map_entry_t;
+
+extern void dhd_send_supp_dhcp(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata,
+	bool tx, uint16 *pktfate);
+extern void dhd_send_supp_eap(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, uint32 pktlen,
+	bool tx, uint16 *pktfate);
+#else
+static INLINE void dhd_send_supp_dhcp(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata,
+	bool tx, uint16 *pktfate) {}
+static INLINE void dhd_send_supp_eap(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata,
+	uint32 pktlen, bool tx, uint16 *pktfate) {}
+#endif /* WL_CFGVENDOR_CUST_ADVLOG */
+
+extern msg_eapol_t dhd_is_4way_msg(uint8 *pktdata);
+extern void dhd_dump_pkt(dhd_pub_t *dhd, int ifidx, uint8 *pktdata,
+	uint32 pktlen, bool tx, uint32 *pkthash, uint16 *pktfate);
+#ifdef BCMPCIE
+extern bool dhd_match_pkt_type(dhd_pub_t *dhd, uint8 *pktdata, uint32 pktlen);
+#endif /* BCMPCIE */
+#ifdef DHD_PKTDUMP_ROAM
+extern void dhd_dump_mod_pkt_timer(dhd_pub_t *dhdp, uint16 rsn);
+extern void dhd_dump_pkt_init(dhd_pub_t *dhdp);
+extern void dhd_dump_pkt_deinit(dhd_pub_t *dhdp);
+extern void dhd_dump_pkt_clear(dhd_pub_t *dhdp);
+#else
+static INLINE void dhd_dump_mod_pkt_timer(dhd_pub_t *dhdp, uint16 rsn) { }
+static INLINE void dhd_dump_pkt_init(dhd_pub_t *dhdp) { }
+static INLINE void dhd_dump_pkt_deinit(dhd_pub_t *dhdp) { }
+static INLINE void dhd_dump_pkt_clear(dhd_pub_t *dhdp) { }
+#endif /* DHD_PKTDUMP_ROAM */
+
+/* Rx packet dump */
+#ifdef DHD_TRX_DUMP
+extern void dhd_trx_pkt_dump(dhd_pub_t *dhdp, int ifidx,
+	uint8 *pktdata, uint32 pktlen, bool tx);
+#else
+static INLINE void dhd_trx_pkt_dump(dhd_pub_t *dhdp, int ifidx,
+	uint8 *pktdata, uint32 pktlen, bool tx) { }
+#endif /* DHD_TRX_DUMP */
+
+/* DHCP packet dump */
+#ifdef DHD_DHCP_DUMP
+extern void dhd_dhcp_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate);
+#else
+static INLINE void dhd_dhcp_dump(dhd_pub_t *dhdp, int ifidx,
+	uint8 *pktdata, bool tx, uint32 *pkthash, uint16 *pktfate) { }
+#endif /* DHD_DHCP_DUMP */
+
+/* DNS packet dump */
+#ifdef DHD_DNS_DUMP
+extern void dhd_dns_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate);
+#else
+static INLINE void dhd_dns_dump(dhd_pub_t *dhdp, int ifidx,
+	uint8 *pktdata, bool tx, uint32 *pkthash, uint16 *pktfate) { }
+#endif /* DHD_DNS_DUMP */
+
+/* ICMP packet dump */
+#ifdef DHD_ICMP_DUMP
+extern void dhd_icmp_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate);
+#else
+static INLINE void dhd_icmp_dump(dhd_pub_t *dhdp, int ifidx,
+	uint8 *pktdata, bool tx, uint32 *pkthash, uint16 *pktfate) { }
+#endif /* DHD_ICMP_DUMP */
+
+/* ARP packet dump */
+#ifdef DHD_ARP_DUMP
+extern void dhd_arp_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate);
+#else
+static INLINE void dhd_arp_dump(dhd_pub_t *dhdp, int ifidx,
+	uint8 *pktdata, bool tx, uint32 *pkthash, uint16 *pktfate) { }
+#endif /* DHD_ARP_DUMP */
+
+/* 802.1X packet dump */
+#ifdef DHD_8021X_DUMP
+extern void dhd_dump_eapol_message(dhd_pub_t *dhd, int ifidx,
+        uint8 *pktdata, uint32 pktlen, bool tx, uint32 *pkthash, uint16 *pktfate);
+#else
+static INLINE void dhd_dump_eapol_message(dhd_pub_t *dhd, int ifidx,
+        uint8 *pktdata, uint32 pktlen, bool tx, uint32 *pkthash, uint16 *pktfate) { }
+#endif /* DHD_8021X_DUMP */
+#ifdef DHD_IPV6_DUMP
+extern void dhd_dhcp6_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate);
+extern void dhd_icmpv6_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+	uint32 *pkthash, uint16 *pktfate);
+extern void dhd_dns6_dump(dhd_pub_t *dhdp, int ifidx, uint8 *pktdata, bool tx,
+    uint32 *pkthash, uint16 *pktfate);
+extern bool dhd_check_dhcp6(uint8 *pktdata, uint32 plen);
+extern bool dhd_check_dns6(uint8 *pktdata, uint32 plen);
+#endif
+extern bool dhd_check_ip_prot(uint8 *pktdata, uint16 ether_type);
+extern bool dhd_check_arp(uint8 *pktdata, uint16 ether_type);
+extern bool dhd_check_dhcp(uint8 *pktdata);
+extern bool dhd_check_icmp(uint8 *pktdata);
+extern bool dhd_check_icmpv6(uint8 *pktdata, uint32 plen);
+extern bool dhd_check_dns(uint8 *pktdata);
+#endif /* __DHD_LINUX_PKTDUMP_H_ */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_platdev.c b/drivers/net/wireless/ap12275/dhd_linux_platdev.c
new file mode 100644
index 000000000000..c2ce2cc9b94e
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_platdev.c
@@ -0,0 +1,1294 @@
+/*
+ * Linux platform device for DHD WLAN adapter
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <typedefs.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <bcmutils.h>
+#include <linux_osl.h>
+#include <dhd_dbg.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_bus.h>
+#include <dhd_linux.h>
+#if defined(OEM_ANDROID)
+#include <wl_android.h>
+#endif
+#include <dhd_plat.h>
+#if defined(CONFIG_WIFI_CONTROL_FUNC)
+#include <linux/wlan_plat.h>
+#endif /* CONFIG_WIFI_CONTROL_FUNC */
+#ifdef CONFIG_DTS
+#include<linux/regulator/consumer.h>
+#include<linux/of_gpio.h>
+#endif /* CONFIG_DTS */
+
+#if defined(CUSTOMER_HW) || defined(BCMDHD_PLATDEV)
+extern int dhd_wlan_init_plat_data(wifi_adapter_info_t *adapter);
+extern void dhd_wlan_deinit_plat_data(wifi_adapter_info_t *adapter);
+#endif /* CUSTOMER_HW */
+
+#define WIFI_PLAT_NAME		"bcmdhd_wlan"
+#define WIFI_PLAT_NAME2		"bcm4329_wlan"
+#define WIFI_PLAT_EXT		"bcmdhd_wifi_platform"
+
+#if defined(SUPPORT_MULTIPLE_BOARD_REVISION)
+#include <linux/of.h>
+extern char* dhd_get_device_dt_name(void);
+#endif /* SUPPORT_MULTIPLE_BOARD_REVISION */
+
+#ifdef DHD_WIFI_SHUTDOWN
+extern void wifi_plat_dev_drv_shutdown(struct platform_device *pdev);
+#endif
+
+#ifdef CONFIG_DTS
+struct regulator *wifi_regulator = NULL;
+#endif /* CONFIG_DTS */
+
+bool cfg_multichip = FALSE;
+bcmdhd_wifi_platdata_t *dhd_wifi_platdata = NULL;
+static int wifi_plat_dev_probe_ret = 0;
+static bool is_power_on = FALSE;
+/* XXX Some Qualcomm based CUSTOMER_HW4 platforms are using platform
+ * device structure even if the Kernel uses device tree structure.
+ * Therefore, the CONFIG_ARCH_MSM condition is temporarly remained
+ * to support in this case.
+ */
+#if !defined(CONFIG_DTS)
+#if defined(DHD_OF_SUPPORT)
+static bool dts_enabled = TRUE;
+extern struct resource dhd_wlan_resources;
+extern struct wifi_platform_data dhd_wlan_control;
+#else
+static bool dts_enabled = FALSE;
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#endif
+struct resource dhd_wlan_resources = {0};
+extern struct wifi_platform_data dhd_wlan_control;
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+#endif /* CONFIG_OF && !defined(CONFIG_ARCH_MSM) */
+#endif /* !defind(CONFIG_DTS) */
+
+static int dhd_wifi_platform_load(void);
+
+extern void* wl_cfg80211_get_dhdp(struct net_device *dev);
+
+// modify for compatibility
+#if defined(BCMDHD_MODULAR) && defined(BOARD_MODULAR_INIT)
+//extern int dhd_wlan_init(void);
+//extern int dhd_wlan_deinit(void);
+#ifdef WBRC
+extern int wbrc_init(void);
+extern void wbrc_exit(void);
+#endif /* WBRC */
+#endif /* defined(BCMDHD_MODULAR) && defined(BOARD_MODULAR_INIT) */
+
+#ifdef ENABLE_4335BT_WAR
+extern int bcm_bt_lock(int cookie);
+extern void bcm_bt_unlock(int cookie);
+static int lock_cookie_wifi = 'W' | 'i'<<8 | 'F'<<16 | 'i'<<24;	/* cookie is "WiFi" */
+#endif /* ENABLE_4335BT_WAR */
+
+#ifdef BCM4335_XTAL_WAR
+extern bool check_bcm4335_rev(void);
+#endif /* BCM4335_XTAL_WAR */
+
+#if defined(CONFIG_X86)
+#define PCIE_RC_VENDOR_ID 0x8086
+#define PCIE_RC_DEVICE_ID 0x9c1a
+#elif defined(CONFIG_ARCH_TEGRA)
+#define PCIE_RC_VENDOR_ID 0x14e4
+#define PCIE_RC_DEVICE_ID 0x4347
+#else /* CONFIG_ARCH_TEGRA */
+/* Dummy defn */
+#define PCIE_RC_VENDOR_ID 0xffff
+#define PCIE_RC_DEVICE_ID 0xffff
+#endif /* CONFIG_X86 */
+
+wifi_adapter_info_t* dhd_wifi_platform_attach_adapter(uint32 bus_type,
+	uint32 bus_num, uint32 slot_num, unsigned long status)
+{
+	int i;
+
+	if (dhd_wifi_platdata == NULL)
+		return NULL;
+
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		wifi_adapter_info_t *adapter = &dhd_wifi_platdata->adapters[i];
+		if ((adapter->bus_type == -1 || adapter->bus_type == bus_type) &&
+			(adapter->bus_num == -1 || adapter->bus_num == bus_num) &&
+			(adapter->slot_num == -1 || adapter->slot_num == slot_num)
+#if defined(ENABLE_INSMOD_NO_FW_LOAD) && !defined(ENABLE_INSMOD_NO_POWER_OFF)
+			&& (wifi_chk_adapter_status(adapter, status))
+#endif
+		) {
+			DHD_ERROR(("attach adapter info '%s'\n", adapter->name));
+			return adapter;
+		}
+	}
+	return NULL;
+}
+
+wifi_adapter_info_t* dhd_wifi_platform_get_adapter(uint32 bus_type, uint32 bus_num, uint32 slot_num)
+{
+	int i;
+
+	if (dhd_wifi_platdata == NULL)
+		return NULL;
+
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		wifi_adapter_info_t *adapter = &dhd_wifi_platdata->adapters[i];
+		if ((adapter->bus_type == -1 || adapter->bus_type == bus_type) &&
+			(adapter->bus_num == -1 || adapter->bus_num == bus_num) &&
+			(adapter->slot_num == -1 || adapter->slot_num == slot_num)) {
+			DHD_TRACE(("found adapter info '%s'\n", adapter->name));
+			return adapter;
+		}
+	}
+	return NULL;
+}
+
+#if defined(CONFIG_WIFI_CONTROL_FUNC) && defined(CONFIG_DHD_USE_STATIC_BUF)
+extern void *dhd_wlan_mem_prealloc(int section, unsigned long size);
+#endif /* CONFIG_WIFI_CONTROL_FUNC && CONFIG_DHD_USE_STATIC_BUF */
+
+void* wifi_platform_prealloc(wifi_adapter_info_t *adapter, int section, unsigned long size)
+{
+	void *alloc_ptr = NULL;
+	struct wifi_platform_data *plat_data;
+
+	if (!adapter || !adapter->wifi_plat_data)
+		return NULL;
+	plat_data = adapter->wifi_plat_data;
+	if (plat_data->mem_prealloc) {
+#if defined(BCMDHD_MDRIVER) && !defined(DHD_STATIC_IN_DRIVER)
+		alloc_ptr = plat_data->mem_prealloc(adapter->bus_type, adapter->index, section, size);
+#else
+#if defined(CONFIG_WIFI_CONTROL_FUNC) && defined(CONFIG_DHD_USE_STATIC_BUF)
+		alloc_ptr = dhd_wlan_mem_prealloc(section, size);
+#else
+		alloc_ptr = plat_data->mem_prealloc(section, size);
+#endif
+#endif
+		if (alloc_ptr) {
+			DHD_INFO(("success alloc section %d\n", section));
+			if (size != 0L)
+				bzero(alloc_ptr, size);
+			return alloc_ptr;
+		}
+	} else
+		return NULL;
+
+	DHD_ERROR(("%s: failed to alloc static mem section %d\n", __FUNCTION__, section));
+	return NULL;
+}
+
+void* wifi_platform_get_prealloc_func_ptr(wifi_adapter_info_t *adapter)
+{
+	struct wifi_platform_data *plat_data;
+
+	if (!adapter || !adapter->wifi_plat_data)
+		return NULL;
+	plat_data = adapter->wifi_plat_data;
+	return plat_data->mem_prealloc;
+}
+
+int wifi_platform_get_irq_number(wifi_adapter_info_t *adapter, unsigned long *irq_flags_ptr)
+{
+	if (adapter == NULL)
+		return -1;
+	if (irq_flags_ptr)
+		*irq_flags_ptr = adapter->intr_flags;
+	return adapter->irq_num;
+}
+
+int wifi_platform_set_power(wifi_adapter_info_t *adapter, bool on, unsigned long msec)
+{
+	int err = 0;
+#ifndef CONFIG_DTS
+	struct wifi_platform_data *plat_data;
+#endif
+#ifdef BT_OVER_SDIO
+	if (is_power_on == on) {
+		return -EINVAL;
+	}
+#endif /* BT_OVER_SDIO */
+	if (on) {
+		wifi_set_adapter_status(adapter, WIFI_STATUS_POWER_ON);
+	} else {
+		wifi_clr_adapter_status(adapter, WIFI_STATUS_POWER_ON);
+	}
+#ifdef CONFIG_DTS
+	if (on) {
+		DHD_INFO(("======== PULL WL_REG_ON HIGH! ========\n"));
+		err = regulator_enable(wifi_regulator);
+		is_power_on = TRUE;
+	}
+	else {
+		DHD_INFO(("======== PULL WL_REG_ON LOW! ========\n"));
+		err = regulator_disable(wifi_regulator);
+		is_power_on = FALSE;
+	}
+	if (err < 0) {
+		DHD_ERROR(("%s: regulator enable/disable failed", __FUNCTION__));
+		goto fail;
+	}
+#else
+	if (!adapter->wifi_plat_data) {
+		err = -EINVAL;
+		goto fail;
+	}
+	plat_data = adapter->wifi_plat_data;
+
+	DHD_INFO(("%s = %d, delay: %lu msec\n", __FUNCTION__, on, msec));
+	if (plat_data->set_power) {
+#ifdef ENABLE_4335BT_WAR
+		if (on) {
+			DHD_INFO(("WiFi: trying to acquire BT lock\n"));
+			if (bcm_bt_lock(lock_cookie_wifi) != 0)
+				DHD_INFO(("** WiFi: timeout in acquiring bt lock**\n"));
+			DHD_INFO(("%s: btlock acquired\n", __FUNCTION__));
+		}
+		else {
+			/* For a exceptional case, release btlock */
+			bcm_bt_unlock(lock_cookie_wifi);
+		}
+#endif /* ENABLE_4335BT_WAR */
+
+#ifdef CONFIG_WIFI_CONTROL_FUNC
+		err = plat_data->set_power(on);
+#else
+		err = plat_data->set_power(on, adapter);
+#endif
+	}
+
+	if (msec && !err) {
+		OSL_SLEEP(msec);
+		DHD_INFO(("%s = %d, sleep done: %lu msec\n", __FUNCTION__, on, msec));
+	}
+
+	if (on && !err)
+		is_power_on = TRUE;
+	else
+		is_power_on = FALSE;
+
+#endif /* CONFIG_DTS */
+
+	return err;
+fail:
+	if (on) {
+		wifi_clr_adapter_status(adapter, WIFI_STATUS_POWER_ON);
+	} else {
+		wifi_set_adapter_status(adapter, WIFI_STATUS_POWER_ON);
+	}
+	return err;
+}
+
+int wifi_platform_bus_enumerate(wifi_adapter_info_t *adapter, bool device_present)
+{
+	int err = 0;
+	struct wifi_platform_data *plat_data;
+
+	if (!adapter || !adapter->wifi_plat_data)
+		return -EINVAL;
+	plat_data = adapter->wifi_plat_data;
+
+	DHD_INFO(("%s device present %d\n", __FUNCTION__, device_present));
+	if (plat_data->set_carddetect) {
+		err = plat_data->set_carddetect(device_present);
+	}
+	return err;
+
+}
+
+int wifi_platform_get_mac_addr(wifi_adapter_info_t *adapter, unsigned char *buf,
+	int ifidx)
+{
+	struct wifi_platform_data *plat_data;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+	if (!buf || !adapter || !adapter->wifi_plat_data)
+		return -EINVAL;
+	plat_data = adapter->wifi_plat_data;
+	if (plat_data->get_mac_addr) {
+#ifdef CONFIG_WIFI_CONTROL_FUNC
+		return plat_data->get_mac_addr(buf);
+#else
+		return plat_data->get_mac_addr(buf, ifidx);
+#endif
+	}
+	return -EOPNOTSUPP;
+}
+
+#ifdef DHD_COREDUMP
+int wifi_platform_set_coredump(wifi_adapter_info_t *adapter, const char *buf,
+	int buf_len, const char *info)
+{
+	struct wifi_platform_data *plat_data;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+	if (!buf || !adapter || !adapter->wifi_plat_data)
+		return -EINVAL;
+	plat_data = adapter->wifi_plat_data;
+	if (plat_data->set_coredump) {
+		return plat_data->set_coredump(buf, buf_len, info);
+	}
+	return -EOPNOTSUPP;
+}
+#endif /* DHD_COREDUMP */
+
+void *
+#ifdef CUSTOM_COUNTRY_CODE
+wifi_platform_get_country_code(wifi_adapter_info_t *adapter, char *ccode, u32 flags)
+#else
+wifi_platform_get_country_code(wifi_adapter_info_t *adapter, char *ccode)
+#endif /* CUSTOM_COUNTRY_CODE */
+{
+	/* get_country_code was added after 2.6.39 */
+#if	(LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	struct wifi_platform_data *plat_data;
+
+	if (!ccode || !adapter || !adapter->wifi_plat_data)
+		return NULL;
+	plat_data = adapter->wifi_plat_data;
+
+	DHD_TRACE(("%s\n", __FUNCTION__));
+	if (plat_data->get_country_code) {
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+		return plat_data->get_country_code(ccode, flags);
+#else
+		return plat_data->get_country_code(ccode);
+#endif /* CUSTOM_COUNTRY_CODE */
+	}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)) */
+
+	return NULL;
+}
+
+#ifndef CUSTOMER_HW
+static int wifi_plat_dev_drv_probe(struct platform_device *pdev)
+{
+	struct resource *resource;
+	wifi_adapter_info_t *adapter;
+#if defined(CONFIG_DTS) && defined(CUSTOMER_OOB)
+	int irq, gpio;
+#endif /* CONFIG_DTS */
+
+	/* Android style wifi platform data device ("bcmdhd_wlan" or "bcm4329_wlan")
+	 * is kept for backward compatibility and supports only 1 adapter
+	 */
+	ASSERT(dhd_wifi_platdata != NULL);
+	ASSERT(dhd_wifi_platdata->num_adapters == 1);
+	adapter = &dhd_wifi_platdata->adapters[0];
+#if defined(CONFIG_WIFI_CONTROL_FUNC)
+	adapter->wifi_plat_data = (struct wifi_platform_data *)(pdev->dev.platform_data);
+#else
+	adapter->wifi_plat_data = (void *)&dhd_wlan_control;
+#endif
+
+#ifdef BCMDHD_PLATDEV
+	adapter->pdev = pdev;
+	wifi_plat_dev_probe_ret = dhd_wlan_init_plat_data(adapter);
+	if (!wifi_plat_dev_probe_ret)
+		wifi_plat_dev_probe_ret = dhd_wifi_platform_load();
+	return wifi_plat_dev_probe_ret;
+#endif
+
+	resource = platform_get_resource_byname(pdev, IORESOURCE_IRQ, "bcmdhd_wlan_irq");
+	if (resource == NULL)
+		resource = platform_get_resource_byname(pdev, IORESOURCE_IRQ, "bcm4329_wlan_irq");
+	if (resource) {
+		adapter->irq_num = resource->start;
+		adapter->intr_flags = resource->flags & IRQF_TRIGGER_MASK;
+#ifdef DHD_ISR_NO_SUSPEND
+		adapter->intr_flags |= IRQF_NO_SUSPEND;
+#endif
+	}
+
+#ifdef CONFIG_DTS
+	wifi_regulator = regulator_get(&pdev->dev, "wlreg_on");
+	if (wifi_regulator == NULL) {
+		DHD_ERROR(("%s regulator is null\n", __FUNCTION__));
+		return -1;
+	}
+
+#if defined(CUSTOMER_OOB)
+	/* This is to get the irq for the OOB */
+	gpio = of_get_gpio(pdev->dev.of_node, 0);
+
+	if (gpio < 0) {
+		DHD_ERROR(("%s gpio information is incorrect\n", __FUNCTION__));
+		return -1;
+	}
+	irq = gpio_to_irq(gpio);
+	if (irq < 0) {
+		DHD_ERROR(("%s irq information is incorrect\n", __FUNCTION__));
+		return -1;
+	}
+	adapter->irq_num = irq;
+
+	/* need to change the flags according to our requirement */
+#ifdef HW_OOB
+	adapter->intr_flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL |
+		IORESOURCE_IRQ_SHAREABLE;
+#else
+	adapter->intr_flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHEDGE |
+		IORESOURCE_IRQ_SHAREABLE;
+#endif
+#endif
+#endif /* CONFIG_DTS */
+
+	wifi_plat_dev_probe_ret = dhd_wifi_platform_load();
+	return wifi_plat_dev_probe_ret;
+}
+
+static int wifi_plat_dev_drv_remove(struct platform_device *pdev)
+{
+	wifi_adapter_info_t *adapter;
+
+	/* Android style wifi platform data device ("bcmdhd_wlan" or "bcm4329_wlan")
+	 * is kept for backward compatibility and supports only 1 adapter
+	 */
+	ASSERT(dhd_wifi_platdata != NULL);
+	ASSERT(dhd_wifi_platdata->num_adapters == 1);
+	adapter = &dhd_wifi_platdata->adapters[0];
+	if (is_power_on) {
+#ifdef BCMPCIE
+		wifi_platform_bus_enumerate(adapter, FALSE);
+		wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+#else
+		wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+		wifi_platform_bus_enumerate(adapter, FALSE);
+#endif /* BCMPCIE */
+	}
+
+#ifdef CONFIG_DTS
+	regulator_put(wifi_regulator);
+#endif /* CONFIG_DTS */
+#ifdef BCMDHD_PLATDEV
+	dhd_wlan_deinit_plat_data(adapter);
+#endif
+	return 0;
+}
+
+static int wifi_plat_dev_drv_suspend(struct platform_device *pdev, pm_message_t state)
+{
+	DHD_TRACE(("##> %s\n", __FUNCTION__));
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 39)) && defined(OOB_INTR_ONLY) && \
+	defined(BCMSDIO)
+	bcmsdh_oob_intr_set(0);
+#endif /* (OOB_INTR_ONLY) */
+	return 0;
+}
+
+static int wifi_plat_dev_drv_resume(struct platform_device *pdev)
+{
+	DHD_TRACE(("##> %s\n", __FUNCTION__));
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 39)) && defined(OOB_INTR_ONLY) && \
+	defined(BCMSDIO)
+	if (dhd_os_check_if_up(wl_cfg80211_get_dhdp()))
+		bcmsdh_oob_intr_set(1);
+#endif /* (OOB_INTR_ONLY) */
+	return 0;
+}
+
+#ifdef CONFIG_DTS
+static const struct of_device_id wifi_device_dt_match[] = {
+	{ .compatible = "android,bcmdhd_wlan", },
+	{},
+};
+#endif /* CONFIG_DTS */
+
+static struct platform_driver wifi_platform_dev_driver = {
+	.probe          = wifi_plat_dev_drv_probe,
+	.remove         = wifi_plat_dev_drv_remove,
+	.suspend        = wifi_plat_dev_drv_suspend,
+	.resume         = wifi_plat_dev_drv_resume,
+#ifdef DHD_WIFI_SHUTDOWN
+	.shutdown       = wifi_plat_dev_drv_shutdown,
+#endif /* DHD_WIFI_SHUTDOWN */
+	.driver         = {
+	.name   = WIFI_PLAT_NAME,
+#ifdef CONFIG_DTS
+	.of_match_table = wifi_device_dt_match,
+#endif /* CONFIG_DTS */
+	}
+};
+
+static struct platform_driver wifi_platform_dev_driver_legacy = {
+	.probe          = wifi_plat_dev_drv_probe,
+	.remove         = wifi_plat_dev_drv_remove,
+	.suspend        = wifi_plat_dev_drv_suspend,
+	.resume         = wifi_plat_dev_drv_resume,
+#ifdef DHD_WIFI_SHUTDOWN
+	.shutdown       = wifi_plat_dev_drv_shutdown,
+#endif /* DHD_WIFI_SHUTDOWN */
+	.driver         = {
+	.name	= WIFI_PLAT_NAME2,
+	}
+};
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
+static int wifi_platdev_match(struct device *dev, const void *data)
+#else
+static int wifi_platdev_match(struct device *dev, void *data)
+#endif /* LINUX_VER >= 5.3.0 */
+{
+	char *name = NULL;
+	const struct platform_device *pdev;
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	name = (char*)data;
+	pdev = to_platform_device(dev);
+	GCC_DIAGNOSTIC_POP();
+
+	if (strcmp(pdev->name, name) == 0) {
+		DHD_ERROR(("found wifi platform device %s\n", name));
+		return TRUE;
+	}
+
+	return FALSE;
+}
+#endif
+
+static int wifi_ctrlfunc_register_drv(void)
+{
+	wifi_adapter_info_t *adapter;
+
+#ifndef CUSTOMER_HW
+	int err = 0;
+	struct device *dev1, *dev2;
+	dev1 = bus_find_device(&platform_bus_type, NULL, WIFI_PLAT_NAME, wifi_platdev_match);
+	dev2 = bus_find_device(&platform_bus_type, NULL, WIFI_PLAT_NAME2, wifi_platdev_match);
+#endif
+
+// modify for compaibility
+#if defined(BCMDHD_MODULAR) && defined(BOARD_MODULAR_INIT)
+//	if ((err = dhd_wlan_init())) {
+//		DHD_ERROR(("%s: dhd_wlan_init() failed(%d)\n", __FUNCTION__, err));
+//		return err;
+//	}
+#ifdef WBRC
+	wbrc_init();
+#endif /* WBRC */
+#endif /* defined(BCMDHD_MODULAR) && defined(BOARD_MODULAR_INIT) */
+
+#if !defined(CONFIG_DTS) && !defined(CUSTOMER_HW)
+	if (!dts_enabled) {
+		if (dev1 == NULL && dev2 == NULL) {
+			DHD_ERROR(("no wifi platform data, skip\n"));
+			return -ENXIO;
+		}
+	}
+#endif /* !defined(CONFIG_DTS) */
+
+	/* multi-chip support not enabled, build one adapter information for
+	 * DHD (either SDIO, USB or PCIe)
+	 */
+	adapter = kzalloc(sizeof(wifi_adapter_info_t), GFP_KERNEL);
+	if (adapter == NULL) {
+		DHD_ERROR(("%s:adapter alloc failed", __FUNCTION__));
+		return -ENOMEM;
+	}
+	adapter->name = "DHD generic adapter";
+	adapter->index = 0;
+	adapter->bus_type = -1;
+	adapter->bus_num = -1;
+	adapter->slot_num = -1;
+	adapter->irq_num = -1;
+	is_power_on = FALSE;
+	wifi_plat_dev_probe_ret = 0;
+	dhd_wifi_platdata = kzalloc(sizeof(bcmdhd_wifi_platdata_t), GFP_KERNEL);
+	if (dhd_wifi_platdata == NULL) {
+		DHD_ERROR(("%s:dhd_wifi_platdata alloc failed", __FUNCTION__));
+		kfree(adapter);
+		return -ENOMEM;
+	}
+	dhd_wifi_platdata->num_adapters = 1;
+	dhd_wifi_platdata->adapters = adapter;
+	init_waitqueue_head(&adapter->status_event);
+
+#ifndef CUSTOMER_HW
+	if (dev1) {
+		err = platform_driver_register(&wifi_platform_dev_driver);
+		if (err) {
+			DHD_ERROR(("%s: failed to register wifi ctrl func driver\n",
+				__FUNCTION__));
+			return err;
+		}
+	}
+	if (dev2) {
+		err = platform_driver_register(&wifi_platform_dev_driver_legacy);
+		if (err) {
+			DHD_ERROR(("%s: failed to register wifi ctrl func legacy driver\n",
+				__FUNCTION__));
+			return err;
+		}
+	}
+#endif
+
+#if !defined(CONFIG_DTS)
+	if (dts_enabled) {
+		adapter->wifi_plat_data = (void *)&dhd_wlan_control;
+#ifdef CUSTOMER_HW
+		wifi_plat_dev_probe_ret = dhd_wlan_init_plat_data(adapter);
+		if (wifi_plat_dev_probe_ret)
+			return wifi_plat_dev_probe_ret;
+#endif
+#ifdef DHD_ISR_NO_SUSPEND
+		adapter->intr_flags |= IRQF_NO_SUSPEND;
+#endif
+		wifi_plat_dev_probe_ret = dhd_wifi_platform_load();
+	}
+#endif /* !defined(CONFIG_DTS) */
+
+#if defined(CONFIG_DTS) && !defined(CUSTOMER_HW)
+	wifi_plat_dev_probe_ret = platform_driver_register(&wifi_platform_dev_driver);
+#endif /* CONFIG_DTS */
+
+	/* return probe function's return value if registeration succeeded */
+	return wifi_plat_dev_probe_ret;
+}
+
+void wifi_ctrlfunc_unregister_drv(void)
+{
+#ifndef CONFIG_DTS
+	wifi_adapter_info_t *adapter = NULL;
+#endif
+
+#if defined(CONFIG_DTS) && !defined(CUSTOMER_HW)
+	DHD_ERROR(("unregister wifi platform drivers\n"));
+	platform_driver_unregister(&wifi_platform_dev_driver);
+#else
+#ifndef CUSTOMER_HW
+	struct device *dev1, *dev2;
+	dev1 = bus_find_device(&platform_bus_type, NULL, WIFI_PLAT_NAME, wifi_platdev_match);
+	dev2 = bus_find_device(&platform_bus_type, NULL, WIFI_PLAT_NAME2, wifi_platdev_match);
+	if (!dts_enabled)
+		if (dev1 == NULL && dev2 == NULL)
+			return;
+#endif
+	DHD_ERROR(("unregister wifi platform drivers\n"));
+#ifndef CUSTOMER_HW
+	if (dev1)
+		platform_driver_unregister(&wifi_platform_dev_driver);
+	if (dev2)
+		platform_driver_unregister(&wifi_platform_dev_driver_legacy);
+#endif
+
+	if (!dhd_wifi_platdata) {
+		goto done;
+	}
+
+	if (dts_enabled) {
+		adapter = &dhd_wifi_platdata->adapters[0];
+		if (is_power_on) {
+			wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+		}
+		wifi_platform_bus_enumerate(adapter, FALSE);
+	}
+// modify for compaibility
+#if defined(BCMDHD_MODULAR) && defined(BOARD_MODULAR_INIT)
+//	dhd_wlan_deinit();
+	osl_static_mem_deinit(NULL, NULL);
+#ifdef WBRC
+	wbrc_exit();
+#endif /* WBRC */
+#endif /* defined(BCMDHD_MODULAR) && defined(BOARD_MODULAR_INIT)  */
+#endif /* !defined(CONFIG_DTS) */
+
+#if defined(CUSTOMER_HW)
+	if (adapter)
+		dhd_wlan_deinit_plat_data(adapter);
+#endif
+
+done:
+	if (dhd_wifi_platdata && dhd_wifi_platdata->adapters) {
+		kfree(dhd_wifi_platdata->adapters);
+		dhd_wifi_platdata->adapters = NULL;
+		dhd_wifi_platdata->num_adapters = 0;
+	}
+	if (dhd_wifi_platdata) {
+		kfree(dhd_wifi_platdata);
+		dhd_wifi_platdata = NULL;
+	}
+}
+
+#ifndef CUSTOMER_HW
+static int bcmdhd_wifi_plat_dev_drv_probe(struct platform_device *pdev)
+{
+	dhd_wifi_platdata = (bcmdhd_wifi_platdata_t *)(pdev->dev.platform_data);
+
+	return dhd_wifi_platform_load();
+}
+
+static int bcmdhd_wifi_plat_dev_drv_remove(struct platform_device *pdev)
+{
+	int i;
+	wifi_adapter_info_t *adapter;
+	ASSERT(dhd_wifi_platdata != NULL);
+
+	/* power down all adapters */
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		adapter = &dhd_wifi_platdata->adapters[i];
+		wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+		wifi_platform_bus_enumerate(adapter, FALSE);
+	}
+	return 0;
+}
+
+static struct platform_driver dhd_wifi_platform_dev_driver = {
+	.probe          = bcmdhd_wifi_plat_dev_drv_probe,
+	.remove         = bcmdhd_wifi_plat_dev_drv_remove,
+	.driver         = {
+	.name   = WIFI_PLAT_EXT,
+	}
+};
+#endif
+
+int dhd_wifi_platform_register_drv(void)
+{
+	int err = 0;
+#ifndef CUSTOMER_HW
+	struct device *dev;
+
+	/* register Broadcom wifi platform data driver if multi-chip is enabled,
+	 * otherwise use Android style wifi platform data (aka wifi control function)
+	 * if it exists
+	 *
+	 * to support multi-chip DHD, Broadcom wifi platform data device must
+	 * be added in kernel early boot (e.g. board config file).
+	 */
+	if (cfg_multichip) {
+		dev = bus_find_device(&platform_bus_type, NULL, WIFI_PLAT_EXT, wifi_platdev_match);
+		if (dev == NULL) {
+			DHD_ERROR(("bcmdhd wifi platform data device not found!!\n"));
+			return -ENXIO;
+		}
+		err = platform_driver_register(&dhd_wifi_platform_dev_driver);
+	} else
+#endif
+	{
+		err = wifi_ctrlfunc_register_drv();
+
+		/* no wifi ctrl func either, load bus directly and ignore this error */
+		if (err) {
+			if (err == -ENXIO) {
+				/* wifi ctrl function does not exist */
+				err = dhd_wifi_platform_load();
+			} else {
+				/* unregister driver due to initialization failure */
+				wifi_ctrlfunc_unregister_drv();
+			}
+		}
+	}
+
+	return err;
+}
+
+#ifdef BCMPCIE
+static int dhd_wifi_platform_load_pcie(void)
+{
+	int i;
+	int err;
+	int retry;
+	wifi_adapter_info_t *adapter;
+
+	if (dhd_wifi_platdata) {
+		/* enumerate PCIe RC */
+		for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+			adapter = &dhd_wifi_platdata->adapters[i];
+			err = wifi_platform_bus_enumerate(adapter, TRUE);
+			if (err) {
+				DHD_ERROR(("failed to enumerate bus %s err=%d",
+					adapter->name, err));
+				return err;
+			}
+		}
+#ifdef DHD_SUPPORT_HDM
+		if (dhd_download_fw_on_driverload || hdm_trigger_init)
+#else
+		if (dhd_download_fw_on_driverload)
+#endif /* DHD_SUPPORT_HDM */
+		{
+			/* power up all adapters */
+			for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+				retry = POWERUP_MAX_RETRY;
+				adapter = &dhd_wifi_platdata->adapters[i];
+
+				DHD_ERROR(("Power-up adapter '%s'\n", adapter->name));
+				DHD_INFO((" - irq %d [flags %d], firmware: %s, nvram: %s\n",
+					adapter->irq_num, adapter->intr_flags, adapter->fw_path,
+					adapter->nv_path));
+				DHD_INFO((" - bus type %d, bus num %d, slot num %d\n\n",
+					adapter->bus_type, adapter->bus_num, adapter->slot_num));
+
+				do {
+					err = wifi_platform_set_power(adapter,
+						TRUE, WIFI_TURNON_DELAY);
+					if (err) {
+						DHD_ERROR(("failed to power up %s,"
+							" %d retry left\n",
+							adapter->name, retry));
+						/* WL_REG_ON state unknown, Power off forcely */
+						wifi_platform_set_power(adapter,
+							FALSE, WIFI_TURNOFF_DELAY);
+						continue;
+					}
+
+					err = wifi_platform_bus_enumerate(adapter, TRUE);
+					if (err) {
+						DHD_ERROR(("failed to enumerate bus %s, "
+							"%d retry left\n",
+							adapter->name, retry));
+						wifi_platform_set_power(adapter, FALSE,
+							WIFI_TURNOFF_DELAY);
+					} else {
+						break;
+					}
+				} while (retry--);
+
+				if (retry < 0) {
+					DHD_ERROR(("failed to power up %s, max retry reached**\n",
+						adapter->name));
+					return -ENODEV;
+				}
+			}
+		}
+	}
+
+	err = dhd_bus_register();
+	if (err) {
+		DHD_ERROR(("%s: dhd_bus_register failed err=%d\n", __FUNCTION__, err));
+		if (dhd_wifi_platdata && dhd_download_fw_on_driverload) {
+			/* power down all adapters */
+			for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+				adapter = &dhd_wifi_platdata->adapters[i];
+				wifi_platform_bus_enumerate(adapter, FALSE);
+				wifi_platform_set_power(adapter,
+					FALSE, WIFI_TURNOFF_DELAY);
+			}
+		}
+	}
+
+	return err;
+}
+#else
+static int dhd_wifi_platform_load_pcie(void)
+{
+	return 0;
+}
+#endif /* BCMPCIE  */
+
+void dhd_wifi_platform_unregister_drv(void)
+{
+#ifndef CUSTOMER_HW
+	if (cfg_multichip)
+		platform_driver_unregister(&dhd_wifi_platform_dev_driver);
+	else
+#endif
+		wifi_ctrlfunc_unregister_drv();
+}
+
+extern int dhd_watchdog_prio;
+extern int dhd_dpc_prio;
+extern uint dhd_deferred_tx;
+#if defined(OEM_ANDROID) && (defined(BCMLXSDMMC) || defined(BCMDBUS))
+extern struct semaphore dhd_registration_sem;
+#endif /* defined(OEM_ANDROID) && defined(BCMLXSDMMC) */
+
+#ifdef BCMSDIO
+static int dhd_wifi_platform_load_sdio(void)
+{
+	int i;
+	int err = 0;
+	wifi_adapter_info_t *adapter;
+
+	BCM_REFERENCE(i);
+	BCM_REFERENCE(adapter);
+	/* Sanity check on the module parameters
+	 * - Both watchdog and DPC as tasklets are ok
+	 * - If both watchdog and DPC are threads, TX must be deferred
+	 */
+	if (!(dhd_watchdog_prio < 0 && dhd_dpc_prio < 0) &&
+		!(dhd_watchdog_prio >= 0 && dhd_dpc_prio >= 0 && dhd_deferred_tx))
+		return -EINVAL;
+
+#if defined(OEM_ANDROID) && defined(BCMLXSDMMC) && !defined(DHD_PRELOAD)
+	sema_init(&dhd_registration_sem, 0);
+#endif
+
+	if (dhd_wifi_platdata == NULL) {
+		DHD_INFO(("DHD wifi platform data is required for Android build\n"));
+		DHD_INFO(("DHD registering bus directly\n"));
+		/* x86 bring-up PC needs no power-up operations */
+		err = dhd_bus_register();
+		return err;
+	}
+
+#if defined(OEM_ANDROID) && defined(BCMLXSDMMC) && !defined(DHD_PRELOAD)
+	/* power up all adapters */
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		bool chip_up = FALSE;
+		int retry = POWERUP_MAX_RETRY;
+#ifndef DHD_INSMOD_NOWAIT
+		struct semaphore dhd_chipup_sem;
+#endif
+
+		adapter = &dhd_wifi_platdata->adapters[i];
+
+		DHD_INFO(("Power-up adapter '%s'\n", adapter->name));
+		DHD_INFO((" - irq %d [flags %d], firmware: %s, nvram: %s\n",
+			adapter->irq_num, adapter->intr_flags, adapter->fw_path, adapter->nv_path));
+		DHD_INFO((" - bus type %d, bus num %d, slot num %d\n\n",
+			adapter->bus_type, adapter->bus_num, adapter->slot_num));
+
+		do {
+#ifdef DHD_INSMOD_NOWAIT
+			err = wifi_platform_set_power(adapter, TRUE, WIFI_TURNON_DELAY);
+			if (err) {
+				DHD_TRACE(("%s: wifi pwr on error ! \n", __FUNCTION__));
+				wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+				continue;
+			} else {
+				wifi_platform_bus_enumerate(adapter, TRUE);
+				chip_up = TRUE;
+				break;
+			}
+#else
+			sema_init(&dhd_chipup_sem, 0);
+			err = dhd_bus_reg_sdio_notify(&dhd_chipup_sem);
+			if (err) {
+				DHD_ERROR(("%s dhd_bus_reg_sdio_notify fail(%d)\n\n",
+					__FUNCTION__, err));
+				return err;
+			}
+			err = wifi_platform_set_power(adapter, TRUE, WIFI_TURNON_DELAY);
+			if (err) {
+				DHD_ERROR(("%s: wifi pwr on error ! \n", __FUNCTION__));
+				dhd_bus_unreg_sdio_notify();
+				/* WL_REG_ON state unknown, Power off forcely */
+				wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+				continue;
+			} else {
+				wifi_platform_bus_enumerate(adapter, TRUE);
+			}
+
+			if (down_timeout(&dhd_chipup_sem, msecs_to_jiffies(POWERUP_WAIT_MS)) == 0) {
+				dhd_bus_unreg_sdio_notify();
+				chip_up = TRUE;
+				break;
+			}
+
+			DHD_ERROR(("failed to power up %s, %d retry left\n", adapter->name, retry));
+			dhd_bus_unreg_sdio_notify();
+			wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+			wifi_platform_bus_enumerate(adapter, FALSE);
+#endif
+		} while (retry--);
+
+		if (!chip_up) {
+			DHD_ERROR(("failed to power up %s, max retry reached**\n", adapter->name));
+			return -ENODEV;
+		}
+
+	}
+
+	err = dhd_bus_register();
+
+	if (err) {
+		DHD_ERROR(("%s: sdio_register_driver failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+#ifndef DHD_INSMOD_NOWAIT
+	/*
+	 * Wait till MMC sdio_register_driver callback called and made driver attach.
+	 * It's needed to make sync up exit from dhd insmod  and
+	 * Kernel MMC sdio device callback registration
+	 */
+	err = down_timeout(&dhd_registration_sem, msecs_to_jiffies(DHD_REGISTRATION_TIMEOUT));
+	if (err) {
+		DHD_ERROR(("%s: sdio_register_driver timeout or error \n", __FUNCTION__));
+		dhd_bus_unregister();
+		goto fail;
+	}
+#endif
+
+	return err;
+
+fail:
+	/* power down all adapters */
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		adapter = &dhd_wifi_platdata->adapters[i];
+		wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+		wifi_platform_bus_enumerate(adapter, FALSE);
+	}
+#else
+	/* x86 bring-up PC needs no power-up operations */
+	err = dhd_bus_register();
+#endif /* defined(OEM_ANDROID) && defined(BCMLXSDMMC) */
+
+	return err;
+}
+#else /* BCMSDIO */
+static int dhd_wifi_platform_load_sdio(void)
+{
+	return 0;
+}
+#endif /* BCMSDIO */
+
+#ifdef BCMDBUS
+static int dhd_wifi_platform_load_usb(void)
+{
+	int err = 0;
+#if !defined(DHD_PRELOAD)
+	wifi_adapter_info_t *adapter;
+	s32 timeout = -1;
+	int i;
+#endif
+
+#if !defined(DHD_PRELOAD) && !defined(ENABLE_INSMOD_NO_POWER_OFF)
+	/* power down all adapters */
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		adapter = &dhd_wifi_platdata->adapters[i];
+		wifi_platform_set_power(adapter, FALSE, 0);
+		if (err) {
+			DHD_ERROR(("failed to wifi_platform_set_power off %s\n", adapter->name));
+			goto exit;
+		}
+	}
+	OSL_SLEEP(200);
+#endif
+
+	err = dhd_bus_register();
+	if (err) {
+		DHD_ERROR(("%s: usb_register failed\n", __FUNCTION__));
+		goto exit;
+	}
+
+#if !defined(DHD_PRELOAD)
+	/* power up all adapters */
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		adapter = &dhd_wifi_platdata->adapters[i];
+		DHD_ERROR(("Power-up adapter '%s'\n", adapter->name));
+		DHD_INFO((" - irq %d [flags %d], firmware: %s, nvram: %s\n",
+			adapter->irq_num, adapter->intr_flags, adapter->fw_path, adapter->nv_path));
+		DHD_INFO((" - bus type %d, bus num %d, slot num %d\n\n",
+			adapter->bus_type, adapter->bus_num, adapter->slot_num));
+		err = wifi_platform_set_power(adapter, TRUE, WIFI_TURNON_DELAY);
+		if (err) {
+			DHD_ERROR(("failed to wifi_platform_set_power on %s\n", adapter->name));
+			goto fail;
+		}
+		timeout = wait_event_interruptible_timeout(adapter->status_event,
+			wifi_get_adapter_status(adapter, WIFI_STATUS_NET_ATTACHED),
+			msecs_to_jiffies(DHD_REGISTRATION_TIMEOUT));
+		if (timeout <= 0) {
+			err = -1;
+			DHD_ERROR(("%s: usb_register_driver timeout\n", __FUNCTION__));
+			goto fail;
+		}
+	}
+#endif
+
+exit:
+	return err;
+
+#if !defined(DHD_PRELOAD)
+fail:
+	dhd_bus_unregister();
+	/* power down all adapters */
+	for (i = 0; i < dhd_wifi_platdata->num_adapters; i++) {
+		adapter = &dhd_wifi_platdata->adapters[i];
+		wifi_platform_set_power(adapter, FALSE, WIFI_TURNOFF_DELAY);
+	}
+
+	return err;
+#endif
+}
+#else /* BCMDBUS */
+static int dhd_wifi_platform_load_usb(void)
+{
+	return 0;
+}
+#endif /* BCMDBUS */
+
+static int dhd_wifi_platform_load(void)
+{
+	int err = 0;
+	DHD_INFO(("%s: Enter\n", __FUNCTION__));
+
+#if defined(OEM_ANDROID)
+	wl_android_init();
+#endif /* OEM_ANDROID */
+
+	if ((err = dhd_wifi_platform_load_usb()))
+		goto end;
+	else if ((err = dhd_wifi_platform_load_sdio()))
+		goto end;
+	else
+		err = dhd_wifi_platform_load_pcie();
+
+end:
+#if defined(OEM_ANDROID)
+	if (err)
+		wl_android_exit();
+#if !defined(MULTIPLE_SUPPLICANT)
+	else
+		wl_android_post_init();
+#endif
+#endif /* OEM_ANDROID */
+
+	return err;
+}
+
+#if defined(SUPPORT_MULTIPLE_BOARD_REVISION)
+void
+concate_custom_board_revision(char *nv_path)
+{
+	uint32 board_revision = 0;
+	struct device_node *root_node = NULL;
+	char* wlan_node = NULL;
+
+	if (!nv_path) {
+		DHD_ERROR(("nv_path is null\n"));
+		return;
+	}
+
+	wlan_node = dhd_get_device_dt_name();
+	if (!wlan_node) {
+		DHD_ERROR(("Failed to dt name\n"));
+		return;
+	}
+
+	root_node = of_find_compatible_node(NULL, NULL, wlan_node);
+	if (!root_node) {
+		DHD_ERROR(("Failed to get device node\n"));
+		return;
+	}
+
+	if (of_property_read_u32(root_node, "nvram-ES", &board_revision)) {
+		DHD_ERROR(("No board revision property in dtsi\n"));
+		return;
+	}
+
+	DHD_INFO(("Board revision:%d\n", board_revision));
+
+	if (board_revision == 1) {
+		strcat(nv_path, "_ES10");
+		DHD_INFO(("Mached Board revision ES10: nvram name:%s\n", nv_path));
+	}
+
+}
+#endif /* SUPPORT_MULTIPLE_BOARD_REVISION */
+
+/* Weak functions that can be overridden in Platform specific implementation */
+char* __attribute__ ((weak)) dhd_get_device_dt_name(void)
+{
+	return NULL;
+}
+
+uint32 __attribute__ ((weak)) dhd_plat_get_info_size(void)
+{
+	return 0;
+}
+
+int __attribute__ ((weak)) dhd_plat_pcie_register_event(void *plat_info,
+		struct pci_dev *pdev, dhd_pcie_event_cb_t pfn)
+{
+	return 0;
+}
+
+void __attribute__ ((weak)) dhd_plat_pcie_deregister_event(void *plat_info)
+{
+	return;
+}
+
+void __attribute__ ((weak)) dhd_plat_l1ss_ctrl(bool ctrl)
+{
+	return;
+}
+
+void __attribute__ ((weak)) dhd_plat_l1_exit_io(void)
+{
+	return;
+}
+
+void __attribute__ ((weak)) dhd_plat_l1_exit(void)
+{
+	return;
+}
+
+void __attribute__ ((weak)) dhd_plat_report_bh_sched(void *plat_info, int resched)
+{
+	return;
+}
+
+int __attribute__ ((weak)) dhd_plat_pcie_suspend(void *plat_info)
+{
+	return 0;
+}
+
+int __attribute__ ((weak)) dhd_plat_pcie_resume(void *plat_info)
+{
+	return 0;
+}
+
+void __attribute__ ((weak)) dhd_plat_pcie_register_dump(void *plat_info)
+{
+	return;
+}
+
+void __attribute__ ((weak)) dhd_plat_pin_dbg_show(void *plat_info)
+{
+	return;
+}
+
+uint32 __attribute__ ((weak)) dhd_plat_get_rc_vendor_id(void)
+{
+	return PCIE_RC_VENDOR_ID;
+}
+
+uint32 __attribute__ ((weak)) dhd_plat_get_rc_device_id(void)
+{
+	return PCIE_RC_DEVICE_ID;
+}
+
+uint16 __attribute__ ((weak)) dhd_plat_align_rxbuf_size(uint16 rxbufpost_sz)
+{
+	return rxbufpost_sz;
+}
+
+int
+__attribute__ ((weak)) dhd_get_platform_naming_for_nvram_clmblob_file(download_type_t component,
+	char *file_name)
+{
+	return BCME_ERROR;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_linux_priv.h b/drivers/net/wireless/ap12275/dhd_linux_priv.h
new file mode 100644
index 000000000000..029217a885f4
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_priv.h
@@ -0,0 +1,550 @@
+/*
+ * DHD Linux header file - contains private structure definition of the Linux specific layer
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_LINUX_PRIV_H__
+#define __DHD_LINUX_PRIV_H__
+
+#include <osl.h>
+
+#ifdef SHOW_LOGTRACE
+#include <linux/syscalls.h>
+#include <event_log.h>
+#endif /* SHOW_LOGTRACE */
+#include <linux/skbuff.h>
+#include <linux/spinlock.h>
+#include <linux/interrupt.h>
+#ifdef CONFIG_COMPAT
+#include <linux/compat.h>
+#endif /* CONFIG COMPAT */
+#ifdef CONFIG_HAS_WAKELOCK
+#include <linux/pm_wakeup.h>
+#endif /* CONFIG_HAS_WAKELOCK */
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_debug.h>
+#include <dhd_linux.h>
+#include <dhd_bus.h>
+
+#ifdef PCIE_FULL_DONGLE
+#include <bcmmsgbuf.h>
+#include <dhd_flowring.h>
+#endif /* PCIE_FULL_DONGLE */
+
+#ifdef RX_PKT_POOL
+#define RX_PKTPOOL_RESCHED_DELAY_MS 500u
+#define RX_PKTPOOL_FETCH_MAX_ATTEMPTS 10u
+typedef struct pkt_pool {
+	struct sk_buff_head skb_q     ____cacheline_aligned;
+	uint32 max_size;
+	uint16 rxbuf_sz;
+} pkt_pool_t;
+#endif /* RX_PKT_POOL */
+
+/*
+ * Do not include this header except for the dhd_linux.c dhd_linux_sysfs.c
+ * Local private structure (extension of pub)
+ */
+typedef struct dhd_info {
+#if defined(WL_WIRELESS_EXT)
+	wl_iw_t		iw;		/* wireless extensions state (must be first) */
+#endif /* defined(WL_WIRELESS_EXT) */
+	dhd_pub_t pub;
+	/* for supporting multiple interfaces.
+	* static_ifs hold the net ifaces without valid FW IF
+	*/
+	dhd_if_t *iflist[DHD_MAX_IFS + DHD_MAX_STATIC_IFS];
+	wifi_adapter_info_t *adapter;			/* adapter information, interrupt, fw path etc. */
+	char fw_path[PATH_MAX];		/* path to firmware image */
+	char nv_path[PATH_MAX];		/* path to nvram vars file */
+	char clm_path[PATH_MAX];	/* path to CLM data */
+	char conf_path[PATH_MAX];	/* path to config vars file */
+	char sig_path[PATH_MAX];	/* path to rtecdc.sig file */
+#ifdef DHD_UCODE_DOWNLOAD
+	char uc_path[PATH_MAX];	/* path to ucode image */
+#endif /* DHD_UCODE_DOWNLOAD */
+
+	/* serialize dhd iovars */
+	struct mutex dhd_iovar_mutex;
+
+	struct semaphore proto_sem;
+#ifdef PROP_TXSTATUS
+	spinlock_t	wlfc_spinlock;
+
+#ifdef BCMDBUS
+	ulong		wlfc_lock_flags;
+	ulong		wlfc_pub_lock_flags;
+#endif /* BCMDBUS */
+#endif /* PROP_TXSTATUS */
+	wait_queue_head_t ioctl_resp_wait;
+	wait_queue_head_t d3ack_wait;
+	wait_queue_head_t dhd_bus_busy_state_wait;
+	wait_queue_head_t dmaxfer_wait;
+	uint32	default_wd_interval;
+
+	timer_list_compat_t timer;
+	bool wd_timer_valid;
+	struct tasklet_struct tasklet;
+	spinlock_t	sdlock;
+	spinlock_t	txqlock;
+	spinlock_t	dhd_lock;
+	spinlock_t	txoff_lock;
+#ifdef BCMDBUS
+	ulong		txqlock_flags;
+#endif /* BCMDBUS */
+
+#ifndef BCMDBUS
+	struct semaphore sdsem;
+	tsk_ctl_t	thr_dpc_ctl;
+	tsk_ctl_t	thr_wdt_ctl;
+#endif /* BCMDBUS */
+
+	tsk_ctl_t	thr_rxf_ctl;
+	spinlock_t	rxf_lock;
+	bool		rxthread_enabled;
+
+	/* Wakelocks */
+#if defined(CONFIG_HAS_WAKELOCK)
+	struct wakeup_source *wl_wifi;   /* Wifi wakelock */
+	struct wakeup_source *wl_rxwake; /* Wifi rx wakelock */
+	struct wakeup_source *wl_ctrlwake; /* Wifi ctrl wakelock */
+	struct wakeup_source *wl_wdwake; /* Wifi wd wakelock */
+	struct wakeup_source *wl_evtwake; /* Wifi event wakelock */
+	struct wakeup_source *wl_pmwake;   /* Wifi pm handler wakelock */
+	struct wakeup_source *wl_txflwake; /* Wifi tx flow wakelock */
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	struct wakeup_source *wl_intrwake; /* Host wakeup wakelock */
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef DHD_USE_SCAN_WAKELOCK
+	struct wakeup_source *wl_scanwake;  /* Wifi scan wakelock */
+#endif /* DHD_USE_SCAN_WAKELOCK */
+	struct wakeup_source *wl_nanwake; /* NAN wakelock */
+#endif /* CONFIG_HAS_WAKELOCK */
+
+#if defined(OEM_ANDROID)
+	/* net_device interface lock, prevent race conditions among net_dev interface
+	 * calls and wifi_on or wifi_off
+	 */
+	struct mutex dhd_net_if_mutex;
+	struct mutex dhd_suspend_mutex;
+#if defined(APF)
+	struct mutex dhd_apf_mutex;
+#endif /* APF */
+#else
+#if defined(APF)
+	struct mutex dhd_apf_mutex;
+#endif /* APF */
+#endif /* OEM_ANDROID */
+	spinlock_t wakelock_spinlock;
+	spinlock_t wakelock_evt_spinlock;
+	uint32 wakelock_counter;
+	int wakelock_wd_counter;
+	int wakelock_rx_timeout_enable;
+	int wakelock_ctrl_timeout_enable;
+	bool waive_wakelock;
+	uint32 wakelock_before_waive;
+
+	/* Thread to issue ioctl for multicast */
+	wait_queue_head_t ctrl_wait;
+	atomic_t pend_8021x_cnt;
+	dhd_attach_states_t dhd_state;
+#ifdef SHOW_LOGTRACE
+	dhd_event_log_t event_data;
+#endif /* SHOW_LOGTRACE */
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND)
+	struct early_suspend early_suspend;
+#endif /* CONFIG_HAS_EARLYSUSPEND && DHD_USE_EARLYSUSPEND */
+
+#ifdef ARP_OFFLOAD_SUPPORT
+	u32 pend_ipaddr;
+#endif /* ARP_OFFLOAD_SUPPORT */
+#ifdef DHDTCPACK_SUPPRESS
+	spinlock_t	tcpack_lock;
+#endif /* DHDTCPACK_SUPPRESS */
+#ifdef FIX_CPU_MIN_CLOCK
+	bool cpufreq_fix_status;
+	struct mutex cpufreq_fix;
+	struct pm_qos_request dhd_cpu_qos;
+#ifdef FIX_BUS_MIN_CLOCK
+	struct pm_qos_request dhd_bus_qos;
+#endif /* FIX_BUS_MIN_CLOCK */
+#endif /* FIX_CPU_MIN_CLOCK */
+	void			*dhd_deferred_wq;
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+	ctf_t		*cih;		/* ctf instance handle */
+	ctf_brc_hot_t *brc_hot;			/* hot ctf bridge cache entry */
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+#ifdef DEBUG_CPU_FREQ
+	struct notifier_block freq_trans;
+	int __percpu *new_freq;
+#endif
+	unsigned int unit;
+	struct notifier_block pm_notifier;
+#ifdef DHD_PSTA
+	uint32	psta_mode;	/* PSTA or PSR */
+#endif /* DHD_PSTA */
+#ifdef DHD_WET
+	        uint32  wet_mode;
+#endif /* DHD_WET */
+#ifdef DHD_DEBUG
+	dhd_dump_t *dump;
+	timer_list_compat_t join_timer;
+	u32 join_timeout_val;
+	bool join_timer_active;
+	uint scan_time_count;
+	timer_list_compat_t scan_timer;
+	bool scan_timer_active;
+#endif
+	struct delayed_work	dhd_dpc_dispatcher_work;
+
+	/* CPU on which the DHD DPC is running */
+	atomic_t	dpc_cpu;
+	atomic_t	prev_dpc_cpu;
+#if defined(DHD_LB)
+#if defined(DHD_LB_HOST_CTRL)
+	bool permitted_primary_cpu;
+#endif /* DHD_LB_HOST_CTRL */
+	/* CPU Load Balance dynamic CPU selection */
+
+	/* Variable that tracks the currect CPUs available for candidacy */
+	cpumask_var_t cpumask_curr_avail;
+
+	/* Primary and secondary CPU mask */
+	cpumask_var_t cpumask_primary, cpumask_secondary; /* configuration */
+	cpumask_var_t cpumask_primary_new, cpumask_secondary_new; /* temp */
+
+	struct notifier_block cpu_notifier;
+
+	/* Napi struct for handling rx packet sendup. Packets are removed from
+	 * H2D RxCompl ring and placed into rx_pend_queue. rx_pend_queue is then
+	 * appended to rx_napi_queue (w/ lock) and the rx_napi_struct is scheduled
+	 * to run to rx_napi_cpu.
+	 */
+	struct sk_buff_head   rx_pend_queue  ____cacheline_aligned;
+	struct sk_buff_head   rx_napi_queue  ____cacheline_aligned;
+	struct sk_buff_head   rx_process_queue  ____cacheline_aligned;
+	struct napi_struct    rx_napi_struct ____cacheline_aligned;
+	atomic_t                   rx_napi_cpu; /* cpu on which the napi is dispatched */
+	struct net_device    *rx_napi_netdev; /* netdev of primary interface */
+
+	struct work_struct    rx_napi_dispatcher_work;
+	struct work_struct    tx_compl_dispatcher_work;
+	struct work_struct    tx_dispatcher_work;
+	struct work_struct    rx_compl_dispatcher_work;
+
+	/* Emergency queue to hold pkts when flow control is enabled and
+	 * same pkts will be posted back to the dongle till flow control is disabled.
+	*/
+	struct sk_buff_head   rx_emerge_queue	____cacheline_aligned;
+
+	/* Number of times DPC Tasklet ran */
+	uint32	dhd_dpc_cnt;
+	/* Number of times NAPI processing got scheduled */
+	uint32	napi_sched_cnt;
+	/* NAPI latency stats */
+	uint64  *napi_latency;
+	uint64 napi_schedule_time;
+	/* Number of times NAPI processing ran on each available core */
+	uint32	*napi_percpu_run_cnt;
+	/* Number of times RX Completions got scheduled */
+	uint32	rxc_sched_cnt;
+	/* Number of times RX Completion ran on each available core */
+	uint32	*rxc_percpu_run_cnt;
+	/* Number of times TX Completions got scheduled */
+	uint32	txc_sched_cnt;
+	/* Number of times TX Completions ran on each available core */
+	uint32	*txc_percpu_run_cnt;
+	/* CPU status */
+	/* Number of times each CPU came online */
+	uint32	*cpu_online_cnt;
+	/* Number of times each CPU went offline */
+	uint32	*cpu_offline_cnt;
+
+	/* Number of times TX processing run on each core */
+	uint32	*txp_percpu_run_cnt;
+	/* Number of times TX start run on each core */
+	uint32	*tx_start_percpu_run_cnt;
+
+	/* Tx load balancing */
+
+	/* TODO: Need to see if batch processing is really required in case of TX
+	 * processing. In case of RX the Dongle can send a bunch of rx completions,
+	 * hence we took a 3 queue approach
+	 * enque - adds the skbs to rx_pend_queue
+	 * dispatch - uses a lock and adds the list of skbs from pend queue to
+	 *            napi queue
+	 * napi processing - copies the pend_queue into a local queue and works
+	 * on it.
+	 * But for TX its going to be 1 skb at a time, so we are just thinking
+	 * of using only one queue and use the lock supported skb queue functions
+	 * to add and process it. If its in-efficient we'll re-visit the queue
+	 * design.
+	 */
+
+	/* When the NET_TX tries to send a TX packet put it into tx_pend_queue */
+	/* struct sk_buff_head		tx_pend_queue  ____cacheline_aligned;  */
+	/*
+	 * From the Tasklet that actually sends out data
+	 * copy the list tx_pend_queue into tx_active_queue. There by we need
+	 * to spinlock to only perform the copy the rest of the code ie to
+	 * construct the tx_pend_queue and the code to process tx_active_queue
+	 * can be lockless. The concept is borrowed as is from RX processing
+	 */
+	/* struct sk_buff_head		tx_active_queue  ____cacheline_aligned; */
+
+	/* Control TXP in runtime, enable by default */
+	atomic_t                lb_txp_active;
+
+	/* Control RXP in runtime, enable by default */
+	atomic_t                lb_rxp_active;
+
+	/*
+	 * When the NET_TX tries to send a TX packet put it into tx_pend_queue
+	 * For now, the processing tasklet will also direcly operate on this
+	 * queue
+	 */
+	struct sk_buff_head	tx_pend_queue  ____cacheline_aligned;
+
+	/* cpu on which the DHD Tx is happenning */
+	atomic_t		tx_cpu;
+
+	/* CPU on which the Network stack is calling the DHD's xmit function */
+	atomic_t		net_tx_cpu;
+
+	/* Tasklet context from which the DHD's TX processing happens */
+	struct tasklet_struct tx_tasklet;
+
+	/*
+	 * Consumer Histogram - NAPI RX Packet processing
+	 * -----------------------------------------------
+	 * On Each CPU, when the NAPI RX Packet processing call back was invoked
+	 * how many packets were processed is captured in this data structure.
+	 * Now its difficult to capture the "exact" number of packets processed.
+	 * So considering the packet counter to be a 32 bit one, we have a
+	 * bucket with 8 bins (2^1, 2^2 ... 2^8). The "number" of packets
+	 * processed is rounded off to the next power of 2 and put in the
+	 * approriate "bin" the value in the bin gets incremented.
+	 * For example, assume that in CPU 1 if NAPI Rx runs 3 times
+	 * and the packet count processed is as follows (assume the bin counters are 0)
+	 * iteration 1 - 10 (the bin counter 2^4 increments to 1)
+	 * iteration 2 - 30 (the bin counter 2^5 increments to 1)
+	 * iteration 3 - 15 (the bin counter 2^4 increments by 1 to become 2)
+	 */
+	uint32 *napi_rx_hist[HIST_BIN_SIZE];
+	uint32 *txc_hist[HIST_BIN_SIZE];
+	uint32 *rxc_hist[HIST_BIN_SIZE];
+	struct kobject dhd_lb_kobj;
+	bool dhd_lb_candidacy_override;
+	enum cpuhp_state dhd_cpuhp_state;
+#endif /* DHD_LB */
+
+	/* DPC bounds sysfs */
+	struct kobject dhd_dpc_bounds_kobj;
+
+#if defined(DNGL_AXI_ERROR_LOGGING) && defined(DHD_USE_WQ_FOR_DNGL_AXI_ERROR)
+	struct work_struct	  axi_error_dispatcher_work;
+#endif /* DNGL_AXI_ERROR_LOGGING && DHD_USE_WQ_FOR_DNGL_AXI_ERROR */
+#ifdef SHOW_LOGTRACE
+#ifdef DHD_USE_KTHREAD_FOR_LOGTRACE
+	tsk_ctl_t	  thr_logtrace_ctl;
+#else
+	struct delayed_work	  event_log_dispatcher_work;
+#endif /* DHD_USE_KTHREAD_FOR_LOGTRACE */
+#endif /* SHOW_LOGTRACE */
+
+#ifdef EWP_EDL
+	struct delayed_work edl_dispatcher_work;
+#endif
+#if defined(WLAN_ACCEL_BOOT)
+	bool wl_accel_force_reg_on;
+#endif
+#if defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW)
+#if defined(BCMDBUS)
+	struct task_struct *fw_download_task;
+	struct semaphore fw_download_lock;
+	bool fw_download_thread_exit;
+#endif /* BCMDBUS */
+#endif /* defined(BCM_DNGL_EMBEDIMAGE) || defined(BCM_REQUEST_FW) */
+	uint32 flag_kobj;    /* Add for duplicate kobj processing */
+	struct kobject dhd_kobj;
+	timer_list_compat_t timesync_timer;
+#if defined(BT_OVER_SDIO)
+    char btfw_path[PATH_MAX];
+#endif /* defined (BT_OVER_SDIO) */
+#ifdef WL_MONITOR
+	struct net_device *monitor_dev; /* monitor pseudo device */
+	struct sk_buff *monitor_skb;
+	uint	monitor_len;
+	uint	monitor_type;   /* monitor pseudo device */
+#ifdef HOST_RADIOTAP_CONV
+	monitor_info_t *monitor_info;
+	uint host_radiotap_conv;
+#endif /* HOST_RADIOTAP_CONV */
+#endif /* WL_MONITOR */
+#if defined(BT_OVER_SDIO)
+    struct mutex bus_user_lock; /* lock for sdio bus apis shared between WLAN & BT */
+    int     bus_user_count; /* User counts of sdio bus shared between WLAN & BT */
+#endif /* BT_OVER_SDIO */
+#ifdef SHOW_LOGTRACE
+	struct sk_buff_head   evt_trace_queue     ____cacheline_aligned;
+#endif
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	struct workqueue_struct *tx_wq;
+	struct workqueue_struct *rx_wq;
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+#ifdef PCIE_INB_DW
+	wait_queue_head_t ds_exit_wait;
+#endif /* PCIE_INB_DW */
+#ifdef DHD_DEBUG_UART
+	bool duart_execute;
+#endif	/* DHD_DEBUG_UART */
+	struct mutex logdump_lock;
+#if defined(DHD_MQ) && defined(DHD_MQ_STATS)
+	uint64 pktcnt_qac_histo[MQ_MAX_QUEUES][AC_COUNT];
+	uint64 pktcnt_per_ac[AC_COUNT];
+	uint64 cpu_qstats[MQ_MAX_QUEUES][MQ_MAX_CPUS];
+#endif /* DHD_MQ && DHD_MQ_STATS */
+	/* indicates mem_dump was scheduled as work queue or called directly */
+	bool scheduled_memdump;
+	struct work_struct dhd_hang_process_work;
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+	struct work_struct dhd_alert_process_work;
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+#ifdef RX_PKT_POOL
+	pkt_pool_t rx_pkt_pool;
+	tsk_ctl_t rx_pktpool_thread;
+#endif
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	osl_atomic_t dump_status;
+	struct work_struct dhd_dump_proc_work;
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+} dhd_info_t;
+
+/** priv_link is the link between netdev and the dhdif and dhd_info structs. */
+typedef struct dhd_dev_priv {
+	dhd_info_t * dhd; /* cached pointer to dhd_info in netdevice priv */
+	dhd_if_t   * ifp; /* cached pointer to dhd_if in netdevice priv */
+	int          ifidx; /* interface index */
+	void       * lkup;
+} dhd_dev_priv_t;
+
+#define DHD_DEV_PRIV_SIZE       (sizeof(dhd_dev_priv_t))
+#define DHD_DEV_PRIV(dev)       ((dhd_dev_priv_t *)DEV_PRIV(dev))
+#define DHD_DEV_INFO(dev)       (((dhd_dev_priv_t *)DEV_PRIV(dev))->dhd)
+#define DHD_DEV_IFP(dev)        (((dhd_dev_priv_t *)DEV_PRIV(dev))->ifp)
+#define DHD_DEV_IFIDX(dev)      (((dhd_dev_priv_t *)DEV_PRIV(dev))->ifidx)
+#define DHD_DEV_LKUP(dev)		(((dhd_dev_priv_t *)DEV_PRIV(dev))->lkup)
+
+#ifdef WL_MONITOR
+#define MONPKT_EXTRA_LEN	48u
+#endif /* WL_MONITOR */
+
+extern int dhd_sysfs_init(dhd_info_t *dhd);
+extern void dhd_sysfs_exit(dhd_info_t *dhd);
+extern void dhd_dbg_ring_proc_create(dhd_pub_t *dhdp);
+extern void dhd_dbg_ring_proc_destroy(dhd_pub_t *dhdp);
+
+int __dhd_sendpkt(dhd_pub_t *dhdp, int ifidx, void *pktbuf);
+
+void dhd_dpc_tasklet_dispatcher_work(struct work_struct * work);
+#if defined(DHD_LB)
+#if defined(DHD_LB_TXP)
+int dhd_lb_sendpkt(dhd_info_t *dhd, struct net_device *net, int ifidx, void *skb);
+void dhd_tx_dispatcher_work(struct work_struct * work);
+void dhd_tx_dispatcher_fn(dhd_pub_t *dhdp);
+void dhd_lb_tx_dispatch(dhd_pub_t *dhdp);
+void dhd_lb_tx_handler(unsigned long data);
+#endif /* DHD_LB_TXP */
+
+#if defined(DHD_LB_RXP)
+int dhd_napi_poll(struct napi_struct *napi, int budget);
+void dhd_rx_napi_dispatcher_work(struct work_struct * work);
+void dhd_lb_rx_napi_dispatch(dhd_pub_t *dhdp);
+void dhd_lb_rx_pkt_enqueue(dhd_pub_t *dhdp, void *pkt, int ifidx);
+unsigned long dhd_read_lb_rxp(dhd_pub_t *dhdp);
+#endif /* DHD_LB_RXP */
+
+void dhd_lb_set_default_cpus(dhd_info_t *dhd);
+void dhd_cpumasks_deinit(dhd_info_t *dhd);
+int dhd_cpumasks_init(dhd_info_t *dhd);
+
+void dhd_select_cpu_candidacy(dhd_info_t *dhd);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int dhd_cpu_startup_callback(unsigned int cpu);
+int dhd_cpu_teardown_callback(unsigned int cpu);
+#else
+int dhd_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu);
+#endif /* LINUX_VERSION_CODE < 4.10.0 */
+
+int dhd_register_cpuhp_callback(dhd_info_t *dhd);
+int dhd_unregister_cpuhp_callback(dhd_info_t *dhd);
+#endif /* DHD_LB */
+
+#ifdef RX_PKT_POOL
+void dhd_rx_pktpool_init(dhd_info_t *dhd);
+void dhd_rx_pktpool_deinit(dhd_info_t *dhd);
+#endif /* RX_PKT_POOL */
+
+#if defined(SET_PCIE_IRQ_CPU_CORE) || defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON)
+void dhd_irq_set_affinity(dhd_pub_t *dhdp, const struct cpumask *cpumask);
+#endif /* SET_PCIE_IRQ_CPU_CORE ||  DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON */
+
+void dhd_flush_logtrace_process(dhd_info_t *dhd);
+
+#ifdef DHD_SSSR_DUMP
+extern uint sssr_enab;
+extern uint fis_enab;
+#endif /* DHD_SSSR_DUMP */
+
+#if defined(ANDROID_VERSION) && (LINUX_VERSION_CODE  >= KERNEL_VERSION(4, 14, 0))
+#define WAKELOCK_BACKPORT
+#endif
+
+#ifdef CONFIG_HAS_WAKELOCK
+#if ((LINUX_VERSION_CODE  >= KERNEL_VERSION(5, 4, 0)) || defined(WAKELOCK_BACKPORT))
+#define dhd_wake_lock_init(wakeup_source, dev, name) \
+do { \
+	wakeup_source = wakeup_source_register(dev, name); \
+} while (0);
+#else
+#define dhd_wake_lock_init(wakeup_source, dev, name) \
+do { \
+	wakeup_source = wakeup_source_register(name); \
+} while (0);
+#endif /* LINUX_VERSION >= 5.4.0 */
+#define dhd_wake_lock_destroy(wakeup_source) \
+do { \
+	wakeup_source_unregister(wakeup_source); \
+} while (0);
+#define dhd_wake_lock(wakeup_source)			__pm_stay_awake(wakeup_source)
+#define dhd_wake_unlock(wakeup_source)			__pm_relax(wakeup_source)
+#define dhd_wake_lock_active(wakeup_source)		((wakeup_source)->active)
+#define dhd_wake_lock_timeout(wakeup_source, timeout)	\
+	__pm_wakeup_event(wakeup_source, jiffies_to_msecs(timeout))
+#endif /* CONFIG_HAS_WAKELOCK */
+
+#endif /* __DHD_LINUX_PRIV_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_rx.c b/drivers/net/wireless/ap12275/dhd_linux_rx.c
new file mode 100644
index 000000000000..ae2044be6a42
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_rx.c
@@ -0,0 +1,1696 @@
+/*
+ * Broadcom Dongle Host Driver (DHD),
+ * Linux-specific network interface for receive(rx) path
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <linuxver.h>
+#include <osl.h>
+#ifdef SHOW_LOGTRACE
+#include <linux/syscalls.h>
+#include <event_log.h>
+#endif /* SHOW_LOGTRACE */
+
+#ifdef PCIE_FULL_DONGLE
+#include <bcmmsgbuf.h>
+#endif /* PCIE_FULL_DONGLE */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/rtnetlink.h>
+#include <linux/etherdevice.h>
+#include <linux/random.h>
+#include <linux/spinlock.h>
+#include <linux/ethtool.h>
+#include <linux/fcntl.h>
+#include <linux/fs.h>
+#include <linux/ip.h>
+#include <linux/reboot.h>
+#include <linux/notifier.h>
+#include <linux/irq.h>
+#if defined(CONFIG_TIZEN)
+#include <linux/net_stat_tizen.h>
+#endif /* CONFIG_TIZEN */
+#include <net/addrconf.h>
+#include <net/ndisc.h>
+#ifdef ENABLE_ADAPTIVE_SCHED
+#include <linux/cpufreq.h>
+#endif /* ENABLE_ADAPTIVE_SCHED */
+#include <linux/rtc.h>
+#include <linux/namei.h>
+#include <asm/uaccess.h>
+#include <asm/unaligned.h>
+#include <dhd_linux_priv.h>
+
+#include <epivers.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+#include <bcmdevs_legacy.h>    /* need to still support chips no longer in trunk firmware */
+#include <bcmiov.h>
+#include <bcmstdlib_s.h>
+
+#include <ethernet.h>
+#include <bcmevent.h>
+#include <vlan.h>
+#include <802.3.h>
+
+#include <dhd_linux_wq.h>
+#include <dhd.h>
+#include <dhd_linux.h>
+#include <dhd_linux_pktdump.h>
+#ifdef DHD_WET
+#include <dhd_wet.h>
+#endif /* DHD_WET */
+#ifdef PCIE_FULL_DONGLE
+#include <dhd_flowring.h>
+#endif
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <dhd_dbg.h>
+#include <dhd_dbg_ring.h>
+#include <dhd_debug.h>
+#if defined(WL_CFG80211)
+#include <wl_cfg80211.h>
+#ifdef WL_BAM
+#include <wl_bam.h>
+#endif	/* WL_BAM */
+#endif	/* WL_CFG80211 */
+#ifdef PNO_SUPPORT
+#include <dhd_pno.h>
+#endif
+#ifdef RTT_SUPPORT
+#include <dhd_rtt.h>
+#endif
+
+#include <dhd_linux_sock_qos.h>
+
+#ifdef CONFIG_COMPAT
+#include <linux/compat.h>
+#endif
+
+#ifdef DHD_WMF
+#include <dhd_wmf_linux.h>
+#endif /* DHD_WMF */
+
+#ifdef DHD_L2_FILTER
+#include <bcmicmp.h>
+#include <bcm_l2_filter.h>
+#include <dhd_l2_filter.h>
+#endif /* DHD_L2_FILTER */
+
+#ifdef DHD_PSTA
+#include <dhd_psta.h>
+#endif /* DHD_PSTA */
+
+#ifdef AMPDU_VO_ENABLE
+/* XXX: Enabling VO AMPDU to reduce FER */
+#include <802.1d.h>
+#endif /* AMPDU_VO_ENABLE */
+
+#if defined(DHDTCPACK_SUPPRESS) || defined(DHDTCPSYNC_FLOOD_BLK)
+#include <dhd_ip.h>
+#endif /* DHDTCPACK_SUPPRESS || DHDTCPSYNC_FLOOD_BLK */
+#include <dhd_daemon.h>
+#ifdef DHD_PKT_LOGGING
+#include <dhd_pktlog.h>
+#endif /* DHD_PKT_LOGGING */
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+#include <eapol.h>
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#ifdef ENABLE_DHD_GRO
+#include <net/sch_generic.h>
+#endif /* ENABLE_DHD_GRO */
+
+#ifdef FIX_CPU_MIN_CLOCK
+#include <linux/pm_qos.h>
+#endif /* FIX_CPU_MIN_CLOCK */
+
+#ifdef PROP_TXSTATUS
+#include <wlfc_proto.h>
+#include <dhd_wlfc.h>
+#endif
+
+#if defined(OEM_ANDROID)
+#include <wl_android.h>
+#endif
+#include <dhd_config.h>
+
+/* RX frame thread priority */
+int dhd_rxf_prio = CUSTOM_RXF_PRIO_SETTING;
+module_param(dhd_rxf_prio, int, 0);
+
+/* Request scheduling of the bus rx frame */
+static void dhd_os_rxflock(dhd_pub_t *pub);
+static void dhd_os_rxfunlock(dhd_pub_t *pub);
+
+static int dhd_wl_host_event(dhd_info_t *dhd, int ifidx, void *pktdata, uint16 pktlen,
+                wl_event_msg_t *event_ptr, void **data_ptr);
+
+static inline int dhd_rxf_enqueue(dhd_pub_t *dhdp, void* skb)
+{
+	uint32 store_idx;
+	uint32 sent_idx;
+
+	if (!skb) {
+		DHD_ERROR(("dhd_rxf_enqueue: NULL skb!!!\n"));
+		return BCME_ERROR;
+	}
+
+	dhd_os_rxflock(dhdp);
+	store_idx = dhdp->store_idx;
+	sent_idx = dhdp->sent_idx;
+	if (dhdp->skbbuf[store_idx] != NULL) {
+		/* Make sure the previous packets are processed */
+		dhd_os_rxfunlock(dhdp);
+#ifdef RXF_DEQUEUE_ON_BUSY
+		DHD_TRACE(("dhd_rxf_enqueue: pktbuf not consumed %p, store idx %d sent idx %d\n",
+			skb, store_idx, sent_idx));
+		return BCME_BUSY;
+#else /* RXF_DEQUEUE_ON_BUSY */
+		DHD_ERROR(("dhd_rxf_enqueue: pktbuf not consumed %p, store idx %d sent idx %d\n",
+			skb, store_idx, sent_idx));
+		/* removed msleep here, should use wait_event_timeout if we
+		 * want to give rx frame thread a chance to run
+		 */
+#if defined(WAIT_DEQUEUE)
+		OSL_SLEEP(1);
+#endif
+		return BCME_ERROR;
+#endif /* RXF_DEQUEUE_ON_BUSY */
+	}
+	DHD_TRACE(("dhd_rxf_enqueue: Store SKB %p. idx %d -> %d\n",
+		skb, store_idx, (store_idx + 1) & (MAXSKBPEND - 1)));
+	dhdp->skbbuf[store_idx] = skb;
+	dhdp->store_idx = (store_idx + 1) & (MAXSKBPEND - 1);
+	dhd_os_rxfunlock(dhdp);
+
+	return BCME_OK;
+}
+
+static inline void* dhd_rxf_dequeue(dhd_pub_t *dhdp)
+{
+	uint32 store_idx;
+	uint32 sent_idx;
+	void *skb;
+
+	dhd_os_rxflock(dhdp);
+
+	store_idx = dhdp->store_idx;
+	sent_idx = dhdp->sent_idx;
+	skb = dhdp->skbbuf[sent_idx];
+
+	if (skb == NULL) {
+		dhd_os_rxfunlock(dhdp);
+		DHD_ERROR(("dhd_rxf_dequeue: Dequeued packet is NULL, store idx %d sent idx %d\n",
+			store_idx, sent_idx));
+		return NULL;
+	}
+
+	dhdp->skbbuf[sent_idx] = NULL;
+	dhdp->sent_idx = (sent_idx + 1) & (MAXSKBPEND - 1);
+
+	DHD_TRACE(("dhd_rxf_dequeue: netif_rx_ni(%p), sent idx %d\n",
+		skb, sent_idx));
+
+	dhd_os_rxfunlock(dhdp);
+
+	return skb;
+}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+void dhd_rx_wq_wakeup(struct work_struct *ptr)
+{
+	struct dhd_rx_tx_work *work;
+	struct dhd_pub * pub;
+
+	work = container_of(ptr, struct dhd_rx_tx_work, work);
+
+	pub = work->pub;
+
+	DHD_RPM(("%s: ENTER. \n", __FUNCTION__));
+
+	if (atomic_read(&pub->block_bus) || pub->busstate == DHD_BUS_DOWN) {
+		return;
+	}
+
+	DHD_OS_WAKE_LOCK(pub);
+	if (pm_runtime_get_sync(dhd_bus_to_dev(pub->bus)) >= 0) {
+
+		// do nothing but wakeup the bus.
+		pm_runtime_mark_last_busy(dhd_bus_to_dev(pub->bus));
+		pm_runtime_put_autosuspend(dhd_bus_to_dev(pub->bus));
+	}
+	DHD_OS_WAKE_UNLOCK(pub);
+	kfree(work);
+}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef DHD_WMF
+bool
+dhd_is_rxthread_enabled(dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd = dhdp->info;
+
+	return dhd->rxthread_enabled;
+}
+#endif /* DHD_WMF */
+
+/** Called when a frame is received by the dongle on interface 'ifidx' */
+void
+dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+	struct sk_buff *skb;
+	uchar *eth;
+	uint len;
+	void *data, *pnext = NULL;
+	int i;
+	dhd_if_t *ifp;
+	wl_event_msg_t event;
+#if defined(OEM_ANDROID)
+	int tout_rx = 0;
+	int tout_ctrl = 0;
+#endif /* OEM_ANDROID */
+	void *skbhead = NULL;
+	void *skbprev = NULL;
+	uint16 protocol;
+	unsigned char *dump_data;
+#ifdef DHD_MCAST_REGEN
+	uint8 interface_role;
+	if_flow_lkup_t *if_flow_lkup;
+	unsigned long flags;
+#endif
+#ifdef DHD_WAKE_STATUS
+	wake_counts_t *wcp = NULL;
+#endif /* DHD_WAKE_STATUS */
+	int pkt_wake = 0;
+#ifdef ENABLE_DHD_GRO
+	bool dhd_gro_enable = TRUE;
+	struct Qdisc *qdisc = NULL;
+#endif /* ENABLE_DHD_GRO */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	BCM_REFERENCE(dump_data);
+	BCM_REFERENCE(pkt_wake);
+
+#ifdef DHD_TPUT_PATCH
+	if (dhdp->conf->pktsetsum)
+		PKTSETSUMGOOD(pktbuf, TRUE);
+#endif
+
+#ifdef ENABLE_DHD_GRO
+	if (ifidx < DHD_MAX_IFS) {
+		ifp = dhd->iflist[ifidx];
+		if (ifp && ifp->net->qdisc) {
+			if (ifp->net->qdisc->ops->cl_ops) {
+				dhd_gro_enable = FALSE;
+				DHD_TRACE(("%s: disable sw gro becasue of"
+						" qdisc tx traffic control\n", __FUNCTION__));
+			}
+
+			if (dev_ingress_queue(ifp->net)) {
+				qdisc = dev_ingress_queue(ifp->net)->qdisc_sleeping;
+				if (qdisc != NULL && (qdisc->flags & TCQ_F_INGRESS)) {
+					if (
+#if defined(CONFIG_NET_XGRESS)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0))
+						(ifp->net->tcx_ingress != NULL) ||
+#endif /* LINUX_VERSION >= 6.6.0 */
+#elif defined(CONFIG_NET_CLS_ACT)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+						(ifp->net->miniq_ingress != NULL) ||
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0))
+						(ifp->net->ingress_cl_list != NULL) ||
+#endif /* LINUX_VERSION >= 4.2.0 */
+#endif /* CONFIG_NET_CLS_ACT */
+						0 ) {
+						dhd_gro_enable = FALSE;
+						DHD_TRACE(("%s: disable sw gro because of"
+						" qdisc rx traffic control\n", __FUNCTION__));
+					}
+				}
+			}
+		}
+	}
+#ifdef DHD_GRO_ENABLE_HOST_CTRL
+	if (!dhdp->permitted_gro && dhd_gro_enable) {
+		dhd_gro_enable = FALSE;
+	}
+#endif /* DHD_GRO_ENABLE_HOST_CTRL */
+#endif /* ENABLE_DHD_GRO */
+
+	for (i = 0; pktbuf && i < numpkt; i++, pktbuf = pnext) {
+		struct ether_header *eh;
+
+		pnext = PKTNEXT(dhdp->osh, pktbuf);
+		PKTSETNEXT(dhdp->osh, pktbuf, NULL);
+
+		/* info ring "debug" data, which is not a 802.3 frame, is sent/hacked with a
+		 * special ifidx of DHD_DUMMY_INFO_IF.  This is just internal to dhd to get the data
+		 * from dhd_msgbuf.c:dhd_prot_infobuf_cmplt_process() to here (dhd_rx_frame).
+		 */
+		if (ifidx == DHD_DUMMY_INFO_IF) {
+			/* Event msg printing is called from dhd_rx_frame which is in Tasklet
+			 * context in case of PCIe FD, in case of other bus this will be from
+			 * DPC context. If we get bunch of events from Dongle then printing all
+			 * of them from Tasklet/DPC context that too in data path is costly.
+			 * Also in the new Dongle SW(4359, 4355 onwards) console prints too come as
+			 * events with type WLC_E_TRACE.
+			 * We'll print this console logs from the WorkQueue context by enqueing SKB
+			 * here and Dequeuing will be done in WorkQueue and will be freed only if
+			 * logtrace_pkt_sendup is TRUE
+			 */
+#ifdef SHOW_LOGTRACE
+			dhd_event_logtrace_enqueue(dhdp, ifidx, pktbuf);
+#else /* !SHOW_LOGTRACE */
+			/* If SHOW_LOGTRACE not defined and ifidx is DHD_DUMMY_INFO_IF,
+			 * free the PKT here itself
+			 */
+			PKTFREE_CTRLBUF(dhdp->osh, pktbuf, FALSE);
+#endif /* SHOW_LOGTRACE */
+			continue;
+		}
+
+		eh = (struct ether_header *)PKTDATA(dhdp->osh, pktbuf);
+
+		/* Check if dhd_stop is in progress */
+		if (dhdp->stop_in_progress) {
+			DHD_INFO(("%s: dhd_stop in progress ignore received packet\n",
+				__FUNCTION__));
+			RX_PKTFREE(dhdp->osh, eh->ether_type, pktbuf, FALSE);
+			continue;
+		}
+
+#ifdef DHD_WAKE_STATUS
+		/* Get pkt_wake and clear it */
+#if defined(BCMPCIE)
+		pkt_wake = dhd_bus_set_get_bus_wake_pkt_dump(dhdp, 0);
+#elif defined(BCMSDIO)
+		pkt_wake = dhd_bus_set_get_bus_wake(dhdp, 0);
+#endif /* BCMPCIE */
+#ifdef BCMDBUS
+		wcp = NULL;
+#else
+		wcp = dhd_bus_get_wakecount(dhdp);
+#endif /* BCMDBUS */
+		if (wcp == NULL) {
+			/* If wakeinfo count buffer is null do not  update wake count values */
+			pkt_wake = 0;
+		}
+#endif /* DHD_WAKE_STATUS */
+
+		if (dhd->pub.tput_data.tput_test_running &&
+			dhd->pub.tput_data.direction == TPUT_DIR_RX &&
+			ntoh16(eh->ether_type) == ETHER_TYPE_IP) {
+			dhd_tput_test_rx(dhdp, pktbuf);
+			PKTFREE(dhd->pub.osh, pktbuf, FALSE);
+			continue;
+		}
+
+		if (ifidx >= DHD_MAX_IFS) {
+			DHD_ERROR(("%s: ifidx(%d) Out of bound. drop packet\n",
+				__FUNCTION__, ifidx));
+			RX_PKTFREE(dhdp->osh, eh->ether_type, pktbuf, FALSE);
+			continue;
+		}
+
+		ifp = dhd->iflist[ifidx];
+		if (ifp == NULL) {
+			DHD_ERROR_RLMT(("%s: ifp is NULL. drop packet\n",
+				__FUNCTION__));
+			RX_PKTFREE(dhdp->osh, eh->ether_type, pktbuf, FALSE);
+			continue;
+		}
+
+		/* Dropping only data packets before registering net device to avoid kernel panic */
+#ifndef PROP_TXSTATUS_VSDB
+		if ((!ifp->net || ifp->net->reg_state != NETREG_REGISTERED) &&
+			(ntoh16(eh->ether_type) != ETHER_TYPE_BRCM))
+#else
+		if ((!ifp->net || ifp->net->reg_state != NETREG_REGISTERED || !dhd->pub.up) &&
+			(ntoh16(eh->ether_type) != ETHER_TYPE_BRCM))
+#endif /* PROP_TXSTATUS_VSDB */
+		{
+			DHD_ERROR(("%s: net device is NOT registered yet. drop packet\n",
+			__FUNCTION__));
+			PKTCFREE(dhdp->osh, pktbuf, FALSE);
+			continue;
+		}
+
+#ifdef PROP_TXSTATUS
+		if (dhd_wlfc_is_header_only_pkt(dhdp, pktbuf)) {
+			/* WLFC may send header only packet when
+			there is an urgent message but no packet to
+			piggy-back on
+			*/
+			PKTCFREE(dhdp->osh, pktbuf, FALSE);
+			continue;
+		}
+#endif
+#ifdef DHD_L2_FILTER
+		/* If block_ping is enabled drop the ping packet */
+		if (ifp->block_ping) {
+			if (bcm_l2_filter_block_ping(dhdp->osh, pktbuf) == BCME_OK) {
+				PKTCFREE(dhdp->osh, pktbuf, FALSE);
+				continue;
+			}
+		}
+		if (ifp->grat_arp && DHD_IF_ROLE_STA(dhdp, ifidx)) {
+		    if (bcm_l2_filter_gratuitous_arp(dhdp->osh, pktbuf) == BCME_OK) {
+				PKTCFREE(dhdp->osh, pktbuf, FALSE);
+				continue;
+		    }
+		}
+		if (ifp->parp_enable && DHD_IF_ROLE_AP(dhdp, ifidx)) {
+			int ret = dhd_l2_filter_pkt_handle(dhdp, ifidx, pktbuf, FALSE);
+
+			/* Drop the packets if l2 filter has processed it already
+			 * otherwise continue with the normal path
+			 */
+			if (ret == BCME_OK) {
+				PKTCFREE(dhdp->osh, pktbuf, TRUE);
+				continue;
+			}
+		}
+		if (ifp->block_tdls) {
+			if (bcm_l2_filter_block_tdls(dhdp->osh, pktbuf) == BCME_OK) {
+				PKTCFREE(dhdp->osh, pktbuf, FALSE);
+				continue;
+			}
+		}
+#endif /* DHD_L2_FILTER */
+
+#ifdef DHD_MCAST_REGEN
+		DHD_FLOWID_LOCK(dhdp->flowid_lock, flags);
+		if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+		ASSERT(if_flow_lkup);
+
+		interface_role = if_flow_lkup[ifidx].role;
+		DHD_FLOWID_UNLOCK(dhdp->flowid_lock, flags);
+
+		if (ifp->mcast_regen_bss_enable && (interface_role != WLC_E_IF_ROLE_WDS) &&
+				!DHD_IF_ROLE_AP(dhdp, ifidx) &&
+				ETHER_ISUCAST(eh->ether_dhost)) {
+			if (dhd_mcast_reverse_translation(eh) ==  BCME_OK) {
+#ifdef DHD_PSTA
+				/* Change bsscfg to primary bsscfg for unicast-multicast packets */
+				if ((dhd_get_psta_mode(dhdp) == DHD_MODE_PSTA) ||
+						(dhd_get_psta_mode(dhdp) == DHD_MODE_PSR)) {
+					if (ifidx != 0) {
+						/* Let the primary in PSTA interface handle this
+						 * frame after unicast to Multicast conversion
+						 */
+						ifp = dhd_get_ifp(dhdp, 0);
+						ASSERT(ifp);
+					}
+				}
+			}
+#endif /* PSTA */
+		}
+#endif /* MCAST_REGEN */
+
+#ifdef DHD_WMF
+		/* WMF processing for multicast packets */
+		if (ifp->wmf.wmf_enable && (ETHER_ISMULTI(eh->ether_dhost))) {
+			dhd_sta_t *sta;
+			int ret;
+
+			sta = dhd_find_sta(dhdp, ifidx, (void *)eh->ether_shost);
+			ret = dhd_wmf_packets_handle(dhdp, pktbuf, sta, ifidx, 1);
+			switch (ret) {
+				case WMF_TAKEN:
+					/* The packet is taken by WMF. Continue to next iteration */
+					continue;
+				case WMF_DROP:
+					/* Packet DROP decision by WMF. Toss it */
+					DHD_ERROR(("%s: WMF decides to drop packet\n",
+						__FUNCTION__));
+					PKTCFREE(dhdp->osh, pktbuf, FALSE);
+					continue;
+				default:
+					/* Continue the transmit path */
+					break;
+			}
+		}
+#endif /* DHD_WMF */
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+		if (dhd_tcpdata_get_flag(dhdp, pktbuf) == FLAG_SYNC) {
+			int delta_sec;
+			int delta_sync;
+			int sync_per_sec;
+			u64 curr_time = DIV_U64_BY_U32(OSL_LOCALTIME_NS(), NSEC_PER_SEC);
+			ifp->tsync_rcvd ++;
+			delta_sync = ifp->tsync_rcvd - ifp->tsyncack_txed;
+			delta_sec = curr_time - ifp->last_sync;
+			if (delta_sec > 1) {
+				sync_per_sec = delta_sync/delta_sec;
+				if (sync_per_sec > TCP_SYNC_FLOOD_LIMIT) {
+					schedule_work(&ifp->blk_tsfl_work);
+					DHD_ERROR(("ifx %d TCP SYNC Flood attack suspected! "
+						"sync recvied %d pkt/sec \n",
+						ifidx, sync_per_sec));
+					ifp->tsync_per_sec = sync_per_sec;
+				}
+				dhd_reset_tcpsync_info_by_ifp(ifp);
+			}
+
+		}
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+#ifdef DHDTCPACK_SUPPRESS
+		dhd_tcpdata_info_get(dhdp, pktbuf);
+#endif
+		skb = PKTTONATIVE(dhdp->osh, pktbuf);
+
+		ASSERT(ifp);
+		skb->dev = ifp->net;
+#ifdef DHD_WET
+		/* wet related packet proto manipulation should be done in DHD
+		 * since dongle doesn't have complete payload
+		 */
+		if (WET_ENABLED(&dhd->pub) && (dhd_wet_recv_proc(dhd->pub.wet_info,
+				pktbuf) < 0)) {
+			DHD_INFO(("%s:%s: wet recv proc failed\n",
+				__FUNCTION__, dhd_ifname(dhdp, ifidx)));
+		}
+#endif /* DHD_WET */
+
+#ifdef DHD_PSTA
+		if (PSR_ENABLED(dhdp) &&
+#ifdef BCM_ROUTER_DHD
+				!(ifp->primsta_dwds) &&
+#endif /* BCM_ROUTER_DHD */
+				(dhd_psta_proc(dhdp, ifidx, &pktbuf, FALSE) < 0)) {
+			DHD_ERROR(("%s:%s: psta recv proc failed\n", __FUNCTION__,
+				dhd_ifname(dhdp, ifidx)));
+		}
+#endif /* DHD_PSTA */
+
+#if defined(BCM_ROUTER_DHD)
+		/* XXX Use WOFA for both dhdap and dhdap-atlas router. */
+		/* XXX dhd_sendpkt verify pkt accounting (TO/FRM NATIVE) and PKTCFREE */
+
+		if (DHD_IF_ROLE_AP(dhdp, ifidx) && (!ifp->ap_isolate)) {
+			eh = (struct ether_header *)PKTDATA(dhdp->osh, pktbuf);
+			if (ETHER_ISUCAST(eh->ether_dhost)) {
+				if (dhd_find_sta(dhdp, ifidx, (void *)eh->ether_dhost)) {
+					dhd_sendpkt(dhdp, ifidx, pktbuf);
+					continue;
+				}
+			} else {
+				void *npkt;
+#if defined(HNDCTF)
+				if (PKTISCHAINED(pktbuf)) { /* XXX WAR */
+					DHD_ERROR(("Error: %s():%d Chained non unicast pkt<%p>\n",
+						__FUNCTION__, __LINE__, pktbuf));
+					PKTFRMNATIVE(dhdp->osh, pktbuf);
+					PKTCFREE(dhdp->osh, pktbuf, FALSE);
+					continue;
+				}
+#endif /* HNDCTF */
+				if ((ntoh16(eh->ether_type) != ETHER_TYPE_IAPP_L2_UPDATE) &&
+					((npkt = PKTDUP(dhdp->osh, pktbuf)) != NULL))
+					dhd_sendpkt(dhdp, ifidx, npkt);
+			}
+		}
+
+#if defined(HNDCTF)
+		/* try cut thru' before sending up */
+		if (dhd_ctf_forward(dhd, skb, &pnext) == BCME_OK) {
+			continue;
+		}
+#endif /* HNDCTF */
+
+#else /* !BCM_ROUTER_DHD */
+
+#if defined(DBG_PKT_MON) && !defined(PCIE_FULL_DONGLE)
+		if (dhd_80211_mon_pkt(dhdp, pktbuf, ifidx)) {
+			continue;
+		}
+#endif
+
+#ifdef PCIE_FULL_DONGLE
+		if ((DHD_IF_ROLE_AP(dhdp, ifidx) || DHD_IF_ROLE_P2PGO(dhdp, ifidx)) &&
+			(!ifp->ap_isolate)) {
+			eh = (struct ether_header *)PKTDATA(dhdp->osh, pktbuf);
+			if (ETHER_ISUCAST(eh->ether_dhost)) {
+				if (dhd_find_sta(dhdp, ifidx, (void *)eh->ether_dhost)) {
+					dhdp->rx_forward++; /* Local count */
+					dhd_sendpkt(dhdp, ifidx, pktbuf);
+					continue;
+				}
+			} else {
+				if ((ntoh16(eh->ether_type) != ETHER_TYPE_IAPP_L2_UPDATE)) {
+					void *npktbuf = NULL;
+					/*
+					* If host_sfhllc_supported enabled, do skb_copy as SFHLLC
+					* header will be inserted during Tx, due to which network
+					* stack will not decode the Rx packet.
+					* Else PKTDUP(skb_clone) is enough.
+					*/
+					if (dhdp->host_sfhllc_supported) {
+						npktbuf = skb_copy(skb, GFP_ATOMIC);
+					} else {
+						npktbuf = PKTDUP(dhdp->osh, pktbuf);
+					}
+					if (npktbuf != NULL) {
+						dhdp->rx_forward++; /* Local count */
+						dhd_sendpkt(dhdp, ifidx, npktbuf);
+					}
+				}
+			}
+		}
+#endif /* PCIE_FULL_DONGLE */
+#endif /* BCM_ROUTER_DHD */
+#ifdef DHD_POST_EAPOL_M1_AFTER_ROAM_EVT
+		if ((IS_STA_IFACE(ndev_to_wdev(ifp->net)) || (IS_P2P_GC(ndev_to_wdev(ifp->net)))) &&
+			(ifp->recv_reassoc_evt == TRUE) && (ifp->post_roam_evt == FALSE) &&
+			(dhd_is_4way_msg((char *)(skb->data)) == EAPOL_4WAY_M1)) {
+				DHD_ERROR(("%s: Reassoc is in progress. "
+					"Drop EAPOL M1 frame\n", __FUNCTION__));
+				PKTFREE(dhdp->osh, pktbuf, FALSE);
+				continue;
+		}
+#endif /* DHD_POST_EAPOL_M1_AFTER_ROAM_EVT */
+#ifdef WLEASYMESH
+		if ((dhdp->conf->fw_type == FW_TYPE_EZMESH) &&
+				(ntoh16(eh->ether_type) != ETHER_TYPE_BRCM)) {
+			uint16 * da = (uint16 *)(eh->ether_dhost);
+			ASSERT(ISALIGNED(da, 2));
+
+			/* XXX: Special handling for 1905 messages
+			 * if DA matches with configured 1905 AL MAC addresses
+			 * bypass fwder and foward it to linux stack
+			 */
+			if (ntoh16(eh->ether_type) == ETHER_TYPE_1905_1) {
+				if (!eacmp(da, ifp->_1905_al_ucast) || !eacmp(da, ifp->_1905_al_mcast)) {
+					//skb->fwr_flood = 0;
+				} else {
+					//skb->fwr_flood = 1;
+				}
+			}
+		}
+#endif /* WLEASYMESH */
+		/* Get the protocol, maintain skb around eth_type_trans()
+		 * The main reason for this hack is for the limitation of
+		 * Linux 2.4 where 'eth_type_trans' uses the 'net->hard_header_len'
+		 * to perform skb_pull inside vs ETH_HLEN. Since to avoid
+		 * coping of the packet coming from the network stack to add
+		 * BDC, Hardware header etc, during network interface registration
+		 * we set the 'net->hard_header_len' to ETH_HLEN + extra space required
+		 * for BDC, Hardware header etc. and not just the ETH_HLEN
+		 */
+		eth = skb->data;
+		len = skb->len;
+		dump_data = skb->data;
+		protocol = (skb->data[12] << 8) | skb->data[13];
+
+		if (protocol == ETHER_TYPE_802_1X) {
+			DBG_EVENT_LOG(dhdp, WIFI_EVENT_DRIVER_EAPOL_FRAME_RECEIVED);
+#if defined(WL_CFG80211) && defined(WL_WPS_SYNC)
+			wl_handle_wps_states(ifp->net, dump_data, len, FALSE);
+#endif /* WL_CFG80211 && WL_WPS_SYNC */
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+			if (dhd_is_4way_msg((uint8 *)(skb->data)) == EAPOL_4WAY_M3) {
+				OSL_ATOMIC_SET(dhdp->osh, &ifp->m4state, M3_RXED);
+			}
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+#ifdef EAPOL_RESEND
+			wl_ext_release_eapol_txpkt(dhdp, ifidx, TRUE);
+#endif /* EAPOL_RESEND */
+		}
+		dhd_handle_pktdata(dhdp, ifidx, skb, dump_data, FALSE,
+			len, NULL, NULL, NULL, FALSE, pkt_wake, TRUE);
+#if defined(DHD_WAKE_STATUS) && defined(DHD_WAKEPKT_DUMP)
+		if (pkt_wake) {
+			DHD_ERROR(("##### dhdpcie_host_wake caused by packets\n"));
+			dhd_prhex("[wakepkt_dump]", (char*)dump_data, MIN(len, 64), DHD_RPM_VAL);
+			DHD_ERROR(("config check in_suspend: %d\n", dhdp->in_suspend));
+#ifdef ARP_OFFLOAD_SUPPORT
+			DHD_ERROR(("arp hmac_update:%d \n", dhdp->hmac_updated));
+#endif /* ARP_OFFLOAD_SUPPORT */
+#if defined(DHD_WAKEPKT_SET_MARK) && defined(CONFIG_NF_CONNTRACK_MARK)
+			PKTMARK(skb) |= 0x80000000;
+#endif /* DHD_WAKEPKT_SET_MARK && CONFIG_NF_CONNTRACK_MARK */
+		}
+#endif /* DHD_WAKE_STATUS && DHD_WAKEPKT_DUMP */
+
+		skb->protocol = eth_type_trans(skb, skb->dev);
+
+		if (skb->pkt_type == PACKET_MULTICAST) {
+			dhd->pub.rx_multicast++;
+			ifp->stats.multicast++;
+		}
+
+		skb->data = eth;
+		skb->len = len;
+
+		/* TODO: XXX: re-look into dropped packets. */
+		DHD_DBG_PKT_MON_RX(dhdp, skb, FRAME_TYPE_ETHERNET_II);
+		/* Strip header, count, deliver upward */
+		skb_pull(skb, ETH_HLEN);
+
+#ifdef ENABLE_WAKEUP_PKT_DUMP
+		if (dhd_mmc_wake) {
+			DHD_INFO(("wake_pkt %s(%d)\n", __FUNCTION__, __LINE__));
+			prhex("wake_pkt", (char*) eth, MIN(len, 64));
+
+			update_wake_pkt_info(skb);
+			DHD_ERROR(("wake_pkt %s(%d), raw_info=0x%016llx\n",
+				__FUNCTION__, __LINE__, temp_raw))
+#ifdef CONFIG_IRQ_HISTORY
+			add_irq_history(0, "WIFI");
+#endif
+			dhd_mmc_wake = FALSE;
+		}
+#endif /* ENABLE_WAKEUP_PKT_DUMP */
+
+		/* Process special event packets and then discard them */
+		/* XXX Decide on a better way to fit this in */
+		memset(&event, 0, sizeof(event));
+
+		if (ntoh16(skb->protocol) == ETHER_TYPE_BRCM) {
+			bcm_event_msg_u_t evu;
+			int ret_event, event_type;
+			void *pkt_data = skb_mac_header(skb);
+
+			ret_event = wl_host_event_get_data(pkt_data, len, &evu);
+
+			if (ret_event != BCME_OK) {
+				DHD_ERROR(("%s: wl_host_event_get_data err = %d\n",
+					__FUNCTION__, ret_event));
+				PKTFREE_CTRLBUF(dhdp->osh, pktbuf, FALSE);
+				continue;
+			}
+
+			memcpy(&event, &evu.event, sizeof(wl_event_msg_t));
+			event_type = ntoh32_ua((void *)&event.event_type);
+#ifdef SHOW_LOGTRACE
+			/* Event msg printing is called from dhd_rx_frame which is in Tasklet
+			 * context in case of PCIe FD, in case of other bus this will be from
+			 * DPC context. If we get bunch of events from Dongle then printing all
+			 * of them from Tasklet/DPC context that too in data path is costly.
+			 * Also in the new Dongle SW(4359, 4355 onwards) console prints too come as
+			 * events with type WLC_E_TRACE.
+			 * We'll print this console logs from the WorkQueue context by enqueing SKB
+			 * here and Dequeuing will be done in WorkQueue and will be freed only if
+			 * logtrace_pkt_sendup is true
+			 */
+			if (event_type == WLC_E_TRACE) {
+				DHD_TRACE(("%s: WLC_E_TRACE\n", __FUNCTION__));
+				dhd_event_logtrace_enqueue(dhdp, ifidx, pktbuf);
+				continue;
+			}
+#endif /* SHOW_LOGTRACE */
+
+			ret_event = dhd_wl_host_event(dhd, ifidx, pkt_data, len, &event, &data);
+
+			wl_event_to_host_order(&event);
+#if defined(OEM_ANDROID)
+			if (!tout_ctrl)
+				tout_ctrl = DHD_PACKET_TIMEOUT_MS;
+#endif /* OEM_ANDROID */
+
+#if (defined(OEM_ANDROID) && defined(PNO_SUPPORT))
+			if (event_type == WLC_E_PFN_NET_FOUND) {
+				/* enforce custom wake lock to garantee that Kernel not suspended */
+				tout_ctrl = CUSTOM_PNO_EVENT_LOCK_xTIME * DHD_PACKET_TIMEOUT_MS;
+			}
+#endif /* PNO_SUPPORT */
+			if (numpkt != 1) {
+				DHD_TRACE(("%s: Got BRCM event packet in a chained packet.\n",
+				__FUNCTION__));
+			}
+
+#ifdef DHD_WAKE_STATUS
+			if (unlikely(pkt_wake)) {
+#ifdef DHD_WAKE_EVENT_STATUS
+				if (event.event_type < WLC_E_LAST) {
+#ifdef CUSTOM_WAKE_REASON_STATS
+					if (wcp->rc_event_idx == MAX_WAKE_REASON_STATS) {
+						wcp->rc_event_idx = 0;
+					}
+					DHD_ERROR(("[pkt_wake] event id %u = %s\n",
+						event.event_type,
+						bcmevent_get_name(event.event_type)));
+					wcp->rc_event[wcp->rc_event_idx] = event.event_type;
+					wcp->rc_event_idx++;
+#else
+					wcp->rc_event[event.event_type]++;
+#endif /* CUSTOM_WAKE_REASON_STATS */
+					wcp->rcwake++;
+					pkt_wake = 0;
+				}
+#endif /* DHD_WAKE_EVENT_STATUS */
+			}
+#endif /* DHD_WAKE_STATUS */
+
+			/* For delete virtual interface event, wl_host_event returns positive
+			 * i/f index, do not proceed. just free the pkt.
+			 */
+			if ((event_type == WLC_E_IF) && (ret_event > 0)) {
+				DHD_ERROR(("%s: interface is deleted. Free event packet\n",
+				__FUNCTION__));
+				PKTFREE_CTRLBUF(dhdp->osh, pktbuf, FALSE);
+				continue;
+			}
+
+			/*
+			 * For the event packets, there is a possibility
+			 * of ifidx getting modifed.Thus update the ifp
+			 * once again.
+			 */
+			ASSERT(ifidx < DHD_MAX_IFS);
+			ifp = dhd->iflist[ifidx];
+#ifndef PROP_TXSTATUS_VSDB
+			if (!(ifp && ifp->net && (ifp->net->reg_state == NETREG_REGISTERED)))
+#else
+			if (!(ifp && ifp->net && (ifp->net->reg_state == NETREG_REGISTERED) &&
+				dhd->pub.up))
+#endif /* PROP_TXSTATUS_VSDB */
+			{
+				DHD_ERROR(("%s: net device is NOT registered. drop event packet\n",
+				__FUNCTION__));
+				PKTFREE_CTRLBUF(dhdp->osh, pktbuf, FALSE);
+				continue;
+			}
+
+#ifdef SENDPROB
+			if (dhdp->wl_event_enabled ||
+				(dhdp->recv_probereq && (event.event_type == WLC_E_PROBREQ_MSG)))
+#else
+			if (dhdp->wl_event_enabled)
+#endif
+			{
+#ifdef DHD_USE_STATIC_CTRLBUF
+				/* If event bufs are allocated via static buf pool
+				 * and wl events are enabled, make a copy, free the
+				 * local one and send the copy up.
+				 */
+				struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
+				/* Copy event and send it up */
+				PKTFREE_STATIC(dhdp->osh, pktbuf, FALSE);
+				if (nskb) {
+					skb = nskb;
+				} else {
+					DHD_ERROR(("skb clone failed. dropping event.\n"));
+					continue;
+				}
+#endif /* DHD_USE_STATIC_CTRLBUF */
+			} else {
+				/* If event enabled not explictly set, drop events */
+				PKTFREE_CTRLBUF(dhdp->osh, pktbuf, FALSE);
+				continue;
+			}
+		} else {
+#if defined(OEM_ANDROID)
+			tout_rx = DHD_PACKET_TIMEOUT_MS;
+
+			/* Override rx wakelock timeout to give hostapd enough time
+			 * to process retry or error handling for handshake
+			 */
+			if (IS_AP_IFACE(ndev_to_wdev(ifp->net)) &&
+				ntoh16(skb->protocol) == ETHER_TYPE_802_1X) {
+				if (dhd_is_4way_msg(dump_data) == EAPOL_4WAY_M2) {
+					tout_rx = DHD_HANDSHAKE_TIMEOUT_MS;
+				}
+			}
+#endif /* OEM_ANDROID */
+
+#ifdef PROP_TXSTATUS
+			dhd_wlfc_save_rxpath_ac_time(dhdp, (uint8)PKTPRIO(skb));
+#endif /* PROP_TXSTATUS */
+
+#ifdef DHD_WAKE_STATUS
+			if (unlikely(pkt_wake)) {
+				wcp->rxwake++;
+#ifdef DHD_WAKE_RX_STATUS
+
+				if (ntoh16(skb->protocol) == ETHER_TYPE_ARP) /* ARP */
+					wcp->rx_arp++;
+				if (ntoh16(skb->protocol) == ETHER_TYPE_IP) {
+					struct iphdr *ipheader = NULL;
+					ipheader = (struct iphdr*)(skb->data);
+					if (ipheader && ipheader->protocol == IPPROTO_ICMP) {
+						wcp->rx_icmp++;
+					}
+				}
+				if (dump_data[0] == 0xFF) { /* Broadcast */
+					wcp->rx_bcast++;
+				} else if (dump_data[0] & 0x01) { /* Multicast */
+					wcp->rx_mcast++;
+					if (ntoh16(skb->protocol) == ETHER_TYPE_IPV6) {
+					    wcp->rx_multi_ipv6++;
+					    if ((skb->len > ETHER_ICMP6_HEADER) &&
+					        (dump_data[ETHER_ICMP6_HEADER] == IPPROTO_ICMPV6)) {
+					        wcp->rx_icmpv6++;
+					        if (skb->len > ETHER_ICMPV6_TYPE) {
+					            switch (dump_data[ETHER_ICMPV6_TYPE]) {
+					            case NDISC_ROUTER_ADVERTISEMENT:
+					                wcp->rx_icmpv6_ra++;
+					                break;
+					            case NDISC_NEIGHBOUR_ADVERTISEMENT:
+					                wcp->rx_icmpv6_na++;
+					                break;
+					            case NDISC_NEIGHBOUR_SOLICITATION:
+					                wcp->rx_icmpv6_ns++;
+					                break;
+					            }
+					        }
+					    }
+					} else if (dump_data[2] == 0x5E) {
+						wcp->rx_multi_ipv4++;
+					} else {
+						wcp->rx_multi_other++;
+					}
+				} else { /* Unicast */
+					wcp->rx_ucast++;
+				}
+#undef ETHER_ICMP6_HEADER
+#undef ETHER_IPV6_SADDR
+#undef ETHER_IPV6_DAADR
+#undef ETHER_ICMPV6_TYPE
+#ifdef DHD_WAKE_STATUS_PRINT
+				dhd_dump_wake_status(dhdp, wcp, (struct ether_header *)eth);
+#endif /* DHD_WAKE_STATUS_PRINT */
+#endif /* DHD_WAKE_RX_STATUS */
+				pkt_wake = 0;
+			}
+#endif /* DHD_WAKE_STATUS */
+		}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
+		ifp->net->last_rx = jiffies;
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) */
+
+		if (ntoh16(skb->protocol) != ETHER_TYPE_BRCM) {
+			dhdp->dstats.rx_bytes += skb->len;
+			dhdp->rx_bytes += skb->len;
+			dhdp->rx_packets++; /* Local count */
+			ifp->stats.rx_bytes += skb->len;
+			ifp->stats.rx_packets++;
+		}
+
+		/* XXX WL here makes sure data is 4-byte aligned? */
+		if (in_interrupt()) {
+			bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+				__FUNCTION__, __LINE__);
+#if defined(DHD_LB_RXP)
+#ifdef ENABLE_DHD_GRO
+			/* The pktlog module clones a skb using skb_clone and
+			 * stores the skb point to the ring buffer of the pktlog module.
+			 * Once the buffer is full,
+			 * the PKTFREE is called for removing the oldest skb.
+			 * The kernel panic occurred when the pktlog module free
+			 * the rx frame handled by napi_gro_receive().
+			 * It is a fix code that DHD don't use napi_gro_receive() for
+			 * the packet used in pktlog module.
+			 */
+			if (dhd_gro_enable && !skb_cloned(skb) &&
+				ntoh16(skb->protocol) != ETHER_TYPE_BRCM) {
+				napi_gro_receive(&dhd->rx_napi_struct, skb);
+			} else {
+#if defined(WL_MONITOR) && defined(BCMSDIO)
+				if (dhd_monitor_enabled(dhdp, ifidx))
+					dhd_rx_mon_pkt_sdio(dhdp, skb, ifidx);
+				else
+#endif /* WL_MONITOR && BCMSDIO */
+				netif_receive_skb(skb);
+			}
+#else
+#if defined(WL_MONITOR) && defined(BCMSDIO)
+			if (dhd_monitor_enabled(dhdp, ifidx))
+				dhd_rx_mon_pkt_sdio(dhdp, skb, ifidx);
+			else
+#endif /* WL_MONITOR && BCMSDIO */
+			netif_receive_skb(skb);
+#endif /* ENABLE_DHD_GRO */
+#else /* !defined(DHD_LB_RXP) */
+			netif_rx(skb);
+#endif /* !defined(DHD_LB_RXP) */
+		} else {
+			if (dhd->rxthread_enabled) {
+				if (!skbhead)
+					skbhead = skb;
+				else
+					PKTSETNEXT(dhdp->osh, skbprev, skb);
+				skbprev = skb;
+			} else {
+
+				/* If the receive is not processed inside an ISR,
+				 * the softirqd must be woken explicitly to service
+				 * the NET_RX_SOFTIRQ.	In 2.6 kernels, this is handled
+				 * by netif_rx_ni(), but in earlier kernels, we need
+				 * to do it manually.
+				 */
+				bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+					__FUNCTION__, __LINE__);
+
+#if defined(BCMPCIE) && defined(DHDTCPACK_SUPPRESS)
+				dhd_tcpack_suppress_set(&dhd->pub, TCPACK_SUP_OFF);
+#endif /* BCMPCIE && DHDTCPACK_SUPPRESS */
+#if defined(DHD_LB_RXP)
+#ifdef ENABLE_DHD_GRO
+				if (dhd_gro_enable && !skb_cloned(skb) &&
+					ntoh16(skb->protocol) != ETHER_TYPE_BRCM) {
+					napi_gro_receive(&dhd->rx_napi_struct, skb);
+				} else {
+					netif_receive_skb(skb);
+				}
+#else
+				netif_receive_skb(skb);
+#endif /* ENABLE_DHD_GRO */
+#else /* !defined(DHD_LB_RXP) */
+				dhd_netif_rx_ni(skb);
+#endif /* !defined(DHD_LB_RXP) */
+			}
+		}
+	}
+
+	if (dhd->rxthread_enabled && skbhead)
+		dhd_sched_rxf(dhdp, skbhead);
+
+#if defined(OEM_ANDROID)
+	DHD_OS_WAKE_LOCK_RX_TIMEOUT_ENABLE(dhdp, tout_rx);
+	DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_ENABLE(dhdp, tout_ctrl);
+
+	/* To immediately notify the host that timeout is enabled */
+	DHD_OS_WAKE_LOCK_TIMEOUT(dhdp);
+#endif /* OEM_ANDROID */
+}
+
+int
+dhd_rxf_thread(void *data)
+{
+	tsk_ctl_t *tsk = (tsk_ctl_t *)data;
+	dhd_info_t *dhd = (dhd_info_t *)tsk->parent;
+#if defined(WAIT_DEQUEUE)
+#define RXF_WATCHDOG_TIME 250 /* BARK_TIME(1000) /  */
+	ulong watchdogTime = OSL_SYSUPTIME(); /* msec */
+#endif
+	dhd_pub_t *pub = &dhd->pub;
+
+	/* This thread doesn't need any user-level access,
+	 * so get rid of all our resources
+	 */
+	if (dhd_rxf_prio > 0)
+	{
+		struct sched_param param;
+		param.sched_priority = (dhd_rxf_prio < MAX_RT_PRIO)?dhd_rxf_prio:(MAX_RT_PRIO-1);
+		setScheduler(current, SCHED_FIFO, &param);
+	}
+
+#ifdef CUSTOM_SET_CPUCORE
+	dhd->pub.current_rxf = current;
+#endif /* CUSTOM_SET_CPUCORE */
+	/* Run until signal received */
+	while (1) {
+		if (dhd->pub.conf->rxf_cpucore >= 0) {
+			printf("%s: set rxf_cpucore %d\n", __FUNCTION__, dhd->pub.conf->rxf_cpucore);
+			set_cpus_allowed_ptr(current, cpumask_of(dhd->pub.conf->rxf_cpucore));
+			dhd->pub.conf->rxf_cpucore = -1;
+		}
+		if (down_interruptible(&tsk->sema) == 0) {
+			void *skb;
+#ifdef ENABLE_ADAPTIVE_SCHED
+			dhd_sched_policy(dhd_rxf_prio);
+#endif /* ENABLE_ADAPTIVE_SCHED */
+
+			SMP_RD_BARRIER_DEPENDS();
+
+			if (tsk->terminated) {
+				DHD_OS_WAKE_UNLOCK(pub);
+				break;
+			}
+			skb = dhd_rxf_dequeue(pub);
+
+			if (skb == NULL) {
+				continue;
+			}
+			while (skb) {
+				void *skbnext = PKTNEXT(pub->osh, skb);
+				PKTSETNEXT(pub->osh, skb, NULL);
+				bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+					__FUNCTION__, __LINE__);
+#if defined(WL_MONITOR) && defined(BCMSDIO)
+				if (dhd_monitor_enabled(pub, 0))
+					dhd_rx_mon_pkt_sdio(pub, skb, 0);
+				else
+#endif /* WL_MONITOR && BCMSDIO */
+				dhd_netif_rx_ni(skb);
+				skb = skbnext;
+			}
+#if defined(WAIT_DEQUEUE)
+			if (OSL_SYSUPTIME() - watchdogTime > RXF_WATCHDOG_TIME) {
+				OSL_SLEEP(1);
+				watchdogTime = OSL_SYSUPTIME();
+			}
+#endif
+
+			DHD_OS_WAKE_UNLOCK(pub);
+		} else {
+			break;
+		}
+	}
+	complete_and_exit(&tsk->completed, 0);
+}
+
+void
+dhd_sched_rxf(dhd_pub_t *dhdp, void *skb)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+#ifdef RXF_DEQUEUE_ON_BUSY
+	int ret = BCME_OK;
+	int retry = 2;
+#endif /* RXF_DEQUEUE_ON_BUSY */
+
+	DHD_OS_WAKE_LOCK(dhdp);
+
+	DHD_TRACE(("dhd_sched_rxf: Enter\n"));
+#ifdef RXF_DEQUEUE_ON_BUSY
+	do {
+		ret = dhd_rxf_enqueue(dhdp, skb);
+		if (ret == BCME_OK || ret == BCME_ERROR)
+			break;
+		else
+			OSL_SLEEP(50); /* waiting for dequeueing */
+	} while (retry-- > 0);
+
+	if (retry <= 0 && ret == BCME_BUSY) {
+		void *skbp = skb;
+
+		while (skbp) {
+			void *skbnext = PKTNEXT(dhdp->osh, skbp);
+			PKTSETNEXT(dhdp->osh, skbp, NULL);
+			bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+				__FUNCTION__, __LINE__);
+			dhd_netif_rx_ni(skbp);
+			skbp = skbnext;
+		}
+		DHD_ERROR(("send skb to kernel backlog without rxf_thread\n"));
+	} else {
+		if (dhd->thr_rxf_ctl.thr_pid >= 0) {
+			up(&dhd->thr_rxf_ctl.sema);
+		}
+	}
+#else /* RXF_DEQUEUE_ON_BUSY */
+	do {
+		if (dhd_rxf_enqueue(dhdp, skb) == BCME_OK)
+			break;
+	} while (1);
+	if (dhd->thr_rxf_ctl.thr_pid >= 0) {
+		up(&dhd->thr_rxf_ctl.sema);
+	} else {
+		DHD_OS_WAKE_UNLOCK(dhdp);
+	}
+	return;
+#endif /* RXF_DEQUEUE_ON_BUSY */
+}
+
+#ifdef WL_MONITOR
+#ifdef BCMSDIO
+void
+dhd_rx_mon_pkt_sdio(dhd_pub_t *dhdp, void *pkt, int ifidx)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+
+	if (!dhd->monitor_skb) {
+		if ((dhd->monitor_skb = PKTTONATIVE(dhdp->osh, pkt)) == NULL)
+			return;
+	}
+
+	if (dhd->monitor_type && dhd->monitor_dev)
+		dhd->monitor_skb->dev = dhd->monitor_dev;
+	else {
+		PKTFREE(dhdp->osh, pkt, FALSE);
+		dhd->monitor_skb = NULL;
+		return;
+	}
+
+	dhd->monitor_skb->protocol =
+		eth_type_trans(dhd->monitor_skb, dhd->monitor_skb->dev);
+	dhd->monitor_len = 0;
+
+	netif_rx_ni(dhd->monitor_skb);
+
+	dhd->monitor_skb = NULL;
+}
+#elif defined(BCMPCIE)
+void
+dhd_rx_mon_pkt(dhd_pub_t *dhdp, host_rxbuf_cmpl_t* msg, void *pkt, int ifidx)
+{
+	dhd_info_t *dhd = (dhd_info_t *)dhdp->info;
+#ifdef HOST_RADIOTAP_CONV
+	if (dhd->host_radiotap_conv) {
+		uint16 len = 0, offset = 0;
+		monitor_pkt_info_t pkt_info;
+
+		memcpy(&pkt_info.marker, &msg->marker, sizeof(msg->marker));
+		memcpy(&pkt_info.ts, &msg->ts, sizeof(monitor_pkt_ts_t));
+
+		if (!dhd->monitor_skb) {
+			if ((dhd->monitor_skb = dev_alloc_skb(MAX_MON_PKT_SIZE)) == NULL)
+				return;
+		}
+
+		len = bcmwifi_monitor(dhd->monitor_info, &pkt_info, PKTDATA(dhdp->osh, pkt),
+			PKTLEN(dhdp->osh, pkt), PKTDATA(dhdp->osh, dhd->monitor_skb), &offset);
+
+		if (dhd->monitor_type && dhd->monitor_dev)
+			dhd->monitor_skb->dev = dhd->monitor_dev;
+		else {
+			PKTFREE(dhdp->osh, pkt, FALSE);
+			dev_kfree_skb(dhd->monitor_skb);
+			return;
+		}
+
+		PKTFREE(dhdp->osh, pkt, FALSE);
+
+		if (!len) {
+			return;
+		}
+
+		skb_put(dhd->monitor_skb, len);
+		skb_pull(dhd->monitor_skb, offset);
+
+		dhd->monitor_skb->protocol = eth_type_trans(dhd->monitor_skb,
+			dhd->monitor_skb->dev);
+	}
+	else
+#endif /* HOST_RADIOTAP_CONV */
+	{
+		uint8 amsdu_flag = (msg->flags & BCMPCIE_PKT_FLAGS_MONITOR_MASK) >>
+			BCMPCIE_PKT_FLAGS_MONITOR_SHIFT;
+		switch (amsdu_flag) {
+			case BCMPCIE_PKT_FLAGS_MONITOR_NO_AMSDU:
+			default:
+				if (!dhd->monitor_skb) {
+					if ((dhd->monitor_skb = PKTTONATIVE(dhdp->osh, pkt))
+						== NULL)
+						return;
+				}
+				if (dhd->monitor_type && dhd->monitor_dev)
+					dhd->monitor_skb->dev = dhd->monitor_dev;
+				else {
+					PKTFREE(dhdp->osh, pkt, FALSE);
+					dhd->monitor_skb = NULL;
+					return;
+				}
+				dhd->monitor_skb->protocol =
+					eth_type_trans(dhd->monitor_skb, dhd->monitor_skb->dev);
+				dhd->monitor_len = 0;
+				break;
+
+			case BCMPCIE_PKT_FLAGS_MONITOR_FIRST_PKT:
+				if (!dhd->monitor_skb) {
+					if ((dhd->monitor_skb = dev_alloc_skb(MAX_MON_PKT_SIZE))
+						== NULL)
+						return;
+					dhd->monitor_len = 0;
+				}
+				if (dhd->monitor_type && dhd->monitor_dev)
+					dhd->monitor_skb->dev = dhd->monitor_dev;
+				else {
+					PKTFREE(dhdp->osh, pkt, FALSE);
+					dev_kfree_skb(dhd->monitor_skb);
+					return;
+				}
+				memcpy(PKTDATA(dhdp->osh, dhd->monitor_skb),
+				PKTDATA(dhdp->osh, pkt), PKTLEN(dhdp->osh, pkt));
+				dhd->monitor_len = PKTLEN(dhdp->osh, pkt);
+				PKTFREE(dhdp->osh, pkt, FALSE);
+				return;
+
+			case BCMPCIE_PKT_FLAGS_MONITOR_INTER_PKT:
+				memcpy(PKTDATA(dhdp->osh, dhd->monitor_skb) + dhd->monitor_len,
+				PKTDATA(dhdp->osh, pkt), PKTLEN(dhdp->osh, pkt));
+				dhd->monitor_len += PKTLEN(dhdp->osh, pkt);
+				PKTFREE(dhdp->osh, pkt, FALSE);
+				return;
+
+			case BCMPCIE_PKT_FLAGS_MONITOR_LAST_PKT:
+				memcpy(PKTDATA(dhdp->osh, dhd->monitor_skb) + dhd->monitor_len,
+				PKTDATA(dhdp->osh, pkt), PKTLEN(dhdp->osh, pkt));
+				dhd->monitor_len += PKTLEN(dhdp->osh, pkt);
+				PKTFREE(dhdp->osh, pkt, FALSE);
+				skb_put(dhd->monitor_skb, dhd->monitor_len);
+				dhd->monitor_skb->protocol =
+					eth_type_trans(dhd->monitor_skb, dhd->monitor_skb->dev);
+				dhd->monitor_len = 0;
+				break;
+		}
+	}
+
+	if (skb_headroom(dhd->monitor_skb) < ETHER_HDR_LEN) {
+		struct sk_buff *skb2;
+
+		DHD_INFO(("%s: insufficient headroom\n",
+		          dhd_ifname(&dhd->pub, ifidx)));
+
+		skb2 = skb_realloc_headroom(dhd->monitor_skb, ETHER_HDR_LEN);
+
+		dev_kfree_skb(dhd->monitor_skb);
+		if ((dhd->monitor_skb = skb2) == NULL) {
+			DHD_ERROR(("%s: skb_realloc_headroom failed\n",
+			           dhd_ifname(&dhd->pub, ifidx)));
+			return;
+		}
+	}
+	PKTPUSH(dhd->pub.osh, dhd->monitor_skb, ETHER_HDR_LEN);
+
+	/* XXX WL here makes sure data is 4-byte aligned? */
+	if (in_interrupt()) {
+		bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE,
+			__FUNCTION__, __LINE__);
+		netif_rx(dhd->monitor_skb);
+	} else {
+		/* If the receive is not processed inside an ISR,
+		 * the softirqd must be woken explicitly to service
+		 * the NET_RX_SOFTIRQ.	In 2.6 kernels, this is handled
+		 * by netif_rx_ni(), but in earlier kernels, we need
+		 * to do it manually.
+		 */
+		bcm_object_trace_opr(dhd->monitor_skb, BCM_OBJDBG_REMOVE,
+			__FUNCTION__, __LINE__);
+
+		dhd_netif_rx_ni(dhd->monitor_skb);
+	}
+
+	dhd->monitor_skb = NULL;
+
+#if defined(OEM_ANDROID)
+	DHD_OS_WAKE_LOCK_RX_TIMEOUT_ENABLE(dhdp, DHD_MONITOR_TIMEOUT_MS);
+	DHD_OS_WAKE_LOCK_TIMEOUT(dhdp);
+#endif /* OEM_ANDROID */
+}
+#endif /* PCIE_FULL_DONGLE */
+#endif /* WL_MONITOR */
+
+static int
+dhd_wl_host_event(dhd_info_t *dhd, int ifidx, void *pktdata, uint16 pktlen,
+	wl_event_msg_t *event, void **data)
+{
+	int bcmerror = 0;
+#ifdef WL_CFG80211
+	unsigned long flags = 0;
+#endif /* WL_CFG80211 */
+	ASSERT(dhd != NULL);
+
+#ifdef SHOW_LOGTRACE
+	bcmerror = wl_process_host_event(&dhd->pub, &ifidx, pktdata, pktlen, event, data,
+		&dhd->event_data);
+#else
+	bcmerror = wl_process_host_event(&dhd->pub, &ifidx, pktdata, pktlen, event, data,
+		NULL);
+#endif /* SHOW_LOGTRACE */
+	if (unlikely(bcmerror != BCME_OK)) {
+		return bcmerror;
+	}
+
+	if (ntoh32(event->event_type) == WLC_E_IF) {
+		/* WLC_E_IF event types are consumed by wl_process_host_event.
+		 * For ifadd/del ops, the netdev ptr may not be valid at this
+		 * point. so return before invoking cfg80211/wext handlers.
+		 */
+		return BCME_OK;
+	}
+
+#ifdef WL_EVENT
+	wl_ext_event_send(dhd->pub.event_params, event, *data);
+#endif
+
+#ifdef WL_CFG80211
+	if (dhd->iflist[ifidx]->net) {
+		DHD_UP_LOCK(&dhd->pub.up_lock, flags);
+		if (dhd->pub.up) {
+			wl_cfg80211_event(dhd->iflist[ifidx]->net, event, *data);
+		}
+		DHD_UP_UNLOCK(&dhd->pub.up_lock, flags);
+	}
+#endif /* defined(WL_CFG80211) */
+
+	return (bcmerror);
+}
+
+void
+dhd_os_sdlock_rxq(dhd_pub_t *pub)
+{
+}
+
+void
+dhd_os_sdunlock_rxq(dhd_pub_t *pub)
+{
+}
+
+static void
+dhd_os_rxflock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+	spin_lock_bh(&dhd->rxf_lock);
+
+}
+
+static void
+dhd_os_rxfunlock(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+	spin_unlock_bh(&dhd->rxf_lock);
+}
+
+#ifdef PKT_FILTER_SUPPORT
+int net_os_rxfilter_add_remove(struct net_device *dev, int add_remove, int num)
+{
+	int ret = 0;
+
+#ifndef GAN_LITE_NAT_KEEPALIVE_FILTER
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+
+	if (!dhd_master_mode)
+		add_remove = !add_remove;
+	DHD_INFO(("%s: add_remove = %d, num = %d\n", __FUNCTION__, add_remove, num));
+	if (!dhd || (num == DHD_UNICAST_FILTER_NUM)) {
+		return 0;
+	}
+
+#ifdef BLOCK_IPV6_PACKET
+	/* customer want to use NO IPV6 packets only */
+	if (num == DHD_MULTICAST6_FILTER_NUM) {
+		return 0;
+	}
+#endif /* BLOCK_IPV6_PACKET */
+
+	if (num >= dhd->pub.pktfilter_count) {
+		return -EINVAL;
+	}
+
+	ret = dhd_packet_filter_add_remove(&dhd->pub, add_remove, num);
+#endif /* !GAN_LITE_NAT_KEEPALIVE_FILTER */
+
+	return ret;
+}
+#endif /* PKT_FILTER_SUPPORT */
+
+int dhd_os_wake_lock_rx_timeout_enable(dhd_pub_t *pub, int val)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+	unsigned long flags;
+
+	if (dhd && (dhd->dhd_state & DHD_ATTACH_STATE_WAKELOCKS_INIT)) {
+		DHD_WAKE_SPIN_LOCK(&dhd->wakelock_spinlock, flags);
+		if (val > dhd->wakelock_rx_timeout_enable)
+			dhd->wakelock_rx_timeout_enable = val;
+		DHD_WAKE_SPIN_UNLOCK(&dhd->wakelock_spinlock, flags);
+	}
+	return 0;
+}
+
+int net_os_wake_lock_rx_timeout_enable(struct net_device *dev, int val)
+{
+	dhd_info_t *dhd = DHD_DEV_INFO(dev);
+	int ret = 0;
+
+	if (dhd)
+		ret = dhd_os_wake_lock_rx_timeout_enable(&dhd->pub, val);
+	return ret;
+}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+void
+dhd_flush_rx_tx_wq(dhd_pub_t *dhdp)
+{
+	dhd_info_t * dhd;
+
+	if (dhdp) {
+		dhd = dhdp->info;
+		if (dhd) {
+			flush_workqueue(dhd->tx_wq);
+			flush_workqueue(dhd->rx_wq);
+		}
+	}
+
+	return;
+}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef RX_PKT_POOL
+void *
+BCMFASTPATH(dhd_rxpool_pktget)(osl_t *osh, dhd_info_t *dhd, uint16 len)
+{
+	void *p = NULL;
+	pkt_pool_t *rx_pool = &dhd->rx_pkt_pool;
+
+	if (dhd->rx_pktpool_thread.thr_pid >= 0) {
+		if (rx_pool->rxbuf_sz == len) {
+			p = skb_dequeue(&rx_pool->skb_q);
+			/* kick rx buffer mgmt thread to alloc more rx
+			 * buffers into the pool
+			 */
+			binary_sema_up(&dhd->rx_pktpool_thread);
+		} else {
+			DHD_ERROR_RLMT(("%s: PKTGET_RX_POOL is called with length %u, "
+				"but pkt pool created with length : %u\n", __FUNCTION__,
+				len, rx_pool->rxbuf_sz));
+		}
+	} else {
+		DHD_ERROR_RLMT(("%s: rx_pktpool_thread(%ld) not inited\n", __FUNCTION__,
+			dhd->rx_pktpool_thread.thr_pid));
+	}
+
+	return p;
+}
+
+static int
+dhd_rx_pktpool_thread(void *data)
+{
+	tsk_ctl_t *tsk = (tsk_ctl_t *)data;
+	dhd_info_t *dhd = (dhd_info_t *)tsk->parent;
+	dhd_pub_t *dhdp = (dhd_pub_t *)&dhd->pub;
+	pkt_pool_t *rx_pool = &dhd->rx_pkt_pool;
+	void *p = NULL;
+	int qlen = 0;
+	int num_attempts = 0;
+
+	DHD_TRACE(("%s: STARTED...\n", __FUNCTION__));
+	while (1) {
+		if (!binary_sema_down(tsk)) {
+			SMP_RD_BARRIER_DEPENDS();
+			/* Check terminated before processing the items */
+			if (tsk->terminated) {
+				DHD_ERROR(("%s: task terminated\n", __FUNCTION__));
+				goto exit;
+			}
+			/* Allocate rx buffer only after dongle
+			 * handshake the actual rxbuf size.
+			 */
+			if (rx_pool->rxbuf_sz) {
+				qlen = skb_queue_len(&rx_pool->skb_q);
+				DHD_TRACE(("%s: before alloc - skb_q len=%u, max_size=%u "
+					"rxbuf_sz : %u\n", __FUNCTION__, qlen, rx_pool->max_size,
+					rx_pool->rxbuf_sz));
+				num_attempts = 0;
+				while (qlen < rx_pool->max_size) {
+					p = PKTGET(dhdp->osh, rx_pool->rxbuf_sz, FALSE);
+					if (!p) {
+						DHD_ERROR_RLMT(("%s: pktget fails, resched...\n",
+							__FUNCTION__));
+						/* retry after some time to fetch packets
+						 * if maximum attempts hit, stop
+						 */
+						num_attempts++;
+						if (num_attempts >= RX_PKTPOOL_FETCH_MAX_ATTEMPTS) {
+							DHD_ERROR_RLMT(("%s: max attempts to fetch"
+								" exceeded.\n", __FUNCTION__));
+							break;
+						}
+						OSL_SLEEP(RX_PKTPOOL_RESCHED_DELAY_MS);
+					} else {
+						skb_queue_tail(&rx_pool->skb_q, p);
+					}
+					qlen = skb_queue_len(&rx_pool->skb_q);
+				}
+				DHD_TRACE(("%s: after alloc - skb_q len=%u, max_size=%u \n",
+					__FUNCTION__, qlen, rx_pool->max_size));
+			}
+		} else {
+			DHD_ERROR_RLMT(("%s: unexpected break\n", __FUNCTION__));
+			break;
+		}
+	}
+exit:
+	DHD_TRACE(("%s: EXITED...\n", __FUNCTION__));
+	complete_and_exit(&tsk->completed, 0);
+}
+
+void
+dhd_rx_pktpool_create(dhd_info_t *dhd, uint16 rxbuf_sz)
+{
+	pkt_pool_t *rx_pool = &dhd->rx_pkt_pool;
+	rx_pool->rxbuf_sz = rxbuf_sz;
+	binary_sema_up(&dhd->rx_pktpool_thread);
+}
+
+void
+dhd_rx_pktpool_init(dhd_info_t *dhd)
+{
+	pkt_pool_t *rx_pool = &dhd->rx_pkt_pool;
+
+	skb_queue_head_init(&rx_pool->skb_q);
+	rx_pool->max_size = MAX_RX_PKT_POOL;
+	rx_pool->rxbuf_sz = 0;
+
+	PROC_START(dhd_rx_pktpool_thread, dhd, &dhd->rx_pktpool_thread, 0, "dhd_rx_pktpool_thread");
+}
+
+void
+dhd_rx_pktpool_deinit(dhd_info_t *dhd)
+{
+	pkt_pool_t *rx_pool = &dhd->rx_pkt_pool;
+	tsk_ctl_t *tsk = &dhd->rx_pktpool_thread;
+
+	if (tsk->parent && tsk->thr_pid >= 0) {
+		PROC_STOP_USING_BINARY_SEMA(tsk);
+	} else {
+		DHD_ERROR(("%s: rx_pktpool_thread(%ld) not inited\n",
+			__FUNCTION__, tsk->thr_pid));
+	}
+	/* skb list manipulation functions use internal
+	 * spin lock, so no need to take separate lock
+	 */
+	skb_queue_purge(&rx_pool->skb_q);
+	rx_pool->max_size = 0;
+	DHD_ERROR(("%s: de-alloc'd rx buffers in pool \n",
+		__FUNCTION__));
+}
+#endif /* RX_PKT_POOL */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_sched.c b/drivers/net/wireless/ap12275/dhd_linux_sched.c
new file mode 100644
index 000000000000..4735c4bda897
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_sched.c
@@ -0,0 +1,60 @@
+/*
+ * Expose some of the kernel scheduler routines
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <typedefs.h>
+#include <linuxver.h>
+
+int setScheduler(struct task_struct *p, int policy, struct sched_param *param)
+{
+	int rc = 0;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	rc = sched_setscheduler(p, policy, param);
+#else
+	switch (policy) {
+		case SCHED_FIFO:
+			sched_set_fifo(p);
+			break;
+		case SCHED_RR:
+			sched_set_fifo_low(p);
+			break;
+		case SCHED_NORMAL:
+		default:
+			sched_set_normal(p, PRIO_TO_NICE(param->sched_priority));
+			break;
+	}
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) */
+
+	return rc;
+}
+
+int get_scheduler_policy(struct task_struct *p)
+{
+	int rc = SCHED_NORMAL;
+	rc = p->policy;
+	return rc;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_linux_sock_qos.h b/drivers/net/wireless/ap12275/dhd_linux_sock_qos.h
new file mode 100644
index 000000000000..f6745d80c1c1
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_sock_qos.h
@@ -0,0 +1,76 @@
+/*
+ * Header file for DHD TPA (Traffic Pattern Analyzer)
+ *
+ * Provides type definitions and function prototypes to call into
+ * DHD's QOS on Socket Flow module.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ *
+ */
+
+#ifndef _DHD_LINUX_TPA_H_
+#define _DHD_LINUX_TPA_H_
+
+struct dhd_sock_flow_info;
+
+/* Feature Disabled dummy implementations */
+
+static inline int
+dhd_init_sock_flows_buf(dhd_info_t *dhd, uint watchdog_ms)
+{
+		BCM_REFERENCE(dhd);
+		return BCME_UNSUPPORTED;
+}
+
+static inline int
+dhd_deinit_sock_flows_buf(dhd_info_t *dhd)
+{
+		BCM_REFERENCE(dhd);
+		return BCME_UNSUPPORTED;
+}
+
+static inline void
+dhd_update_sock_flows(dhd_info_t *dhd, struct sk_buff *skb)
+{
+		BCM_REFERENCE(dhd);
+		BCM_REFERENCE(skb);
+		return;
+}
+
+static inline void
+dhd_analyze_sock_flows(dhd_info_t *dhd, uint32 watchdog_ms)
+{
+	BCM_REFERENCE(dhd);
+	BCM_REFERENCE(dhd_watchdog_ms);
+	return;
+}
+
+static inline void
+dhd_sock_qos_update_bus_flowid(dhd_info_t *dhd, void *pktbuf,
+	uint32 bus_flow_id)
+{
+	BCM_REFERENCE(dhd);
+	BCM_REFERENCE(pktbuf);
+	BCM_REFERENCE(bus_flow_id);
+}
+
+#endif /* _DHD_LINUX_TPA_H_ */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_tx.c b/drivers/net/wireless/ap12275/dhd_linux_tx.c
new file mode 100644
index 000000000000..93a96a20cffd
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_tx.c
@@ -0,0 +1,1309 @@
+/*
+ * Broadcom Dongle Host Driver (DHD),
+ * Linux-specific network interface for transmit(tx) path
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <linuxver.h>
+#include <osl.h>
+#ifdef SHOW_LOGTRACE
+#include <linux/syscalls.h>
+#include <event_log.h>
+#endif /* SHOW_LOGTRACE */
+
+#ifdef PCIE_FULL_DONGLE
+#include <bcmmsgbuf.h>
+#endif /* PCIE_FULL_DONGLE */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/rtnetlink.h>
+#include <linux/etherdevice.h>
+#include <linux/random.h>
+#include <linux/spinlock.h>
+#include <linux/ethtool.h>
+#include <linux/fcntl.h>
+#include <linux/fs.h>
+#include <linux/ip.h>
+#include <linux/reboot.h>
+#include <linux/notifier.h>
+#include <linux/irq.h>
+#if defined(CONFIG_TIZEN)
+#include <linux/net_stat_tizen.h>
+#endif /* CONFIG_TIZEN */
+#include <net/addrconf.h>
+#ifdef ENABLE_ADAPTIVE_SCHED
+#include <linux/cpufreq.h>
+#endif /* ENABLE_ADAPTIVE_SCHED */
+#include <linux/rtc.h>
+#include <linux/namei.h>
+#include <asm/uaccess.h>
+#include <asm/unaligned.h>
+#include <dhd_linux_priv.h>
+
+#include <epivers.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+#include <bcmdevs_legacy.h>    /* need to still support chips no longer in trunk firmware */
+#include <bcmiov.h>
+#include <bcmstdlib_s.h>
+
+#include <ethernet.h>
+#include <bcmevent.h>
+#include <vlan.h>
+#include <802.3.h>
+
+#include <dhd_linux_wq.h>
+#include <dhd.h>
+#include <dhd_linux.h>
+#include <dhd_linux_pktdump.h>
+#ifdef DHD_WET
+#include <dhd_wet.h>
+#endif /* DHD_WET */
+#ifdef PCIE_FULL_DONGLE
+#include <dhd_flowring.h>
+#endif
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <dhd_dbg.h>
+#include <dhd_dbg_ring.h>
+#include <dhd_debug.h>
+#if defined(WL_CFG80211)
+#include <wl_cfg80211.h>
+#ifdef WL_BAM
+#include <wl_bam.h>
+#endif	/* WL_BAM */
+#endif	/* WL_CFG80211 */
+#ifdef PNO_SUPPORT
+#include <dhd_pno.h>
+#endif
+#ifdef RTT_SUPPORT
+#include <dhd_rtt.h>
+#endif
+
+#include <dhd_linux_sock_qos.h>
+
+#ifdef CONFIG_COMPAT
+#include <linux/compat.h>
+#endif
+
+#ifdef DHD_WMF
+#include <dhd_wmf_linux.h>
+#endif /* DHD_WMF */
+
+#ifdef DHD_L2_FILTER
+#include <bcmicmp.h>
+#include <bcm_l2_filter.h>
+#include <dhd_l2_filter.h>
+#endif /* DHD_L2_FILTER */
+
+#ifdef DHD_PSTA
+#include <dhd_psta.h>
+#endif /* DHD_PSTA */
+
+#ifdef AMPDU_VO_ENABLE
+/* XXX: Enabling VO AMPDU to reduce FER */
+#include <802.1d.h>
+#endif /* AMPDU_VO_ENABLE */
+
+#if defined(DHDTCPACK_SUPPRESS) || defined(DHDTCPSYNC_FLOOD_BLK)
+#include <dhd_ip.h>
+#endif /* DHDTCPACK_SUPPRESS || DHDTCPSYNC_FLOOD_BLK */
+#include <dhd_daemon.h>
+#ifdef DHD_PKT_LOGGING
+#include <dhd_pktlog.h>
+#endif /* DHD_PKT_LOGGING */
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+#include <eapol.h>
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+#ifdef ENABLE_DHD_GRO
+#include <net/sch_generic.h>
+#endif /* ENABLE_DHD_GRO */
+
+#ifdef FIX_CPU_MIN_CLOCK
+#include <linux/pm_qos.h>
+#endif /* FIX_CPU_MIN_CLOCK */
+
+#ifdef PROP_TXSTATUS
+#include <wlfc_proto.h>
+#include <dhd_wlfc.h>
+#endif
+
+#if defined(OEM_ANDROID)
+#include <wl_android.h>
+#endif
+#include <dhd_config.h>
+
+#define WME_PRIO2AC(prio)  wme_fifo2ac[prio2fifo[(prio)]]
+
+void
+dhd_tx_stop_queues(struct net_device *net)
+{
+#ifdef DHD_MQ
+	netif_tx_stop_all_queues(net);
+#else
+	netif_stop_queue(net);
+#endif
+}
+
+void
+dhd_tx_start_queues(struct net_device *net)
+{
+#ifdef DHD_MQ
+	netif_tx_wake_all_queues(net);
+#else
+	netif_wake_queue(net);
+#endif
+}
+
+int
+BCMFASTPATH(__dhd_sendpkt)(dhd_pub_t *dhdp, int ifidx, void *pktbuf)
+{
+	int ret = BCME_OK;
+	dhd_info_t *dhd = (dhd_info_t *)(dhdp->info);
+	struct ether_header *eh = NULL;
+	uint8 pkt_flow_prio;
+	dhd_if_t *ifp;
+	unsigned long flags;
+	uint datalen;
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	ifp = dhd_get_ifp(dhdp, ifidx);
+	if (!ifp || ifp->del_in_progress) {
+		DHD_ERROR(("%s: ifp:%p del_in_progress:%d\n",
+			__FUNCTION__, ifp, ifp ? ifp->del_in_progress : 0));
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return -ENODEV;
+	}
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	/* Reject if down */
+	if (!dhdp->up || (dhdp->busstate == DHD_BUS_DOWN)) {
+		/* free the packet here since the caller won't */
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return -ENODEV;
+	}
+
+#ifdef PCIE_FULL_DONGLE
+	if (dhdp->busstate == DHD_BUS_SUSPEND) {
+		DHD_ERROR(("%s : pcie is still in suspend state!!\n", __FUNCTION__));
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return NETDEV_TX_BUSY;
+	}
+#endif /* PCIE_FULL_DONGLE */
+
+	/* Reject if pktlen > MAX_MTU_SZ */
+	if (PKTLEN(dhdp->osh, pktbuf) > MAX_MTU_SZ) {
+		/* free the packet here since the caller won't */
+		dhdp->tx_big_packets++;
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return BCME_ERROR;
+	}
+
+	datalen = PKTLEN(dhdp->osh, pktbuf);
+
+#ifdef DHD_L2_FILTER
+	/* if dhcp_unicast is enabled, we need to convert the */
+	/* broadcast DHCP ACK/REPLY packets to Unicast. */
+	if (ifp->dhcp_unicast) {
+	    uint8* mac_addr;
+	    uint8* ehptr = NULL;
+	    int ret;
+	    ret = bcm_l2_filter_get_mac_addr_dhcp_pkt(dhdp->osh, pktbuf, ifidx, &mac_addr);
+	    if (ret == BCME_OK) {
+		/*  if given mac address having valid entry in sta list
+		 *  copy the given mac address, and return with BCME_OK
+		*/
+		if (dhd_find_sta(dhdp, ifidx, mac_addr)) {
+		    ehptr = PKTDATA(dhdp->osh, pktbuf);
+		    bcopy(mac_addr, ehptr + ETHER_DEST_OFFSET, ETHER_ADDR_LEN);
+		}
+	    }
+	}
+
+	if (ifp->grat_arp && DHD_IF_ROLE_AP(dhdp, ifidx)) {
+	    if (bcm_l2_filter_gratuitous_arp(dhdp->osh, pktbuf) == BCME_OK) {
+			PKTCFREE(dhdp->osh, pktbuf, TRUE);
+			return BCME_ERROR;
+	    }
+	}
+
+	if (ifp->parp_enable && DHD_IF_ROLE_AP(dhdp, ifidx)) {
+		ret = dhd_l2_filter_pkt_handle(dhdp, ifidx, pktbuf, TRUE);
+
+		/* Drop the packets if l2 filter has processed it already
+		 * otherwise continue with the normal path
+		 */
+		if (ret == BCME_OK) {
+			PKTCFREE(dhdp->osh, pktbuf, TRUE);
+			return BCME_ERROR;
+		}
+	}
+#endif /* DHD_L2_FILTER */
+	/* Update multicast statistic */
+	if (PKTLEN(dhdp->osh, pktbuf) >= ETHER_HDR_LEN) {
+		uint8 *pktdata = (uint8 *)PKTDATA(dhdp->osh, pktbuf);
+		eh = (struct ether_header *)pktdata;
+
+		if (ETHER_ISMULTI(eh->ether_dhost))
+			dhdp->tx_multicast++;
+		if (ntoh16(eh->ether_type) == ETHER_TYPE_802_1X) {
+#ifdef DHD_LOSSLESS_ROAMING
+			uint8 prio = (uint8)PKTPRIO(pktbuf);
+
+			/* back up 802.1x's priority */
+			dhdp->prio_8021x = prio;
+#endif /* DHD_LOSSLESS_ROAMING */
+			DBG_EVENT_LOG(dhdp, WIFI_EVENT_DRIVER_EAPOL_FRAME_TRANSMIT_REQUESTED);
+			atomic_inc(&dhd->pend_8021x_cnt);
+#if defined(WL_CFG80211) && defined(WL_WPS_SYNC)
+			wl_handle_wps_states(dhd_idx2net(dhdp, ifidx),
+				pktdata, PKTLEN(dhdp->osh, pktbuf), TRUE);
+#endif /* WL_CFG80211 && WL_WPS_SYNC */
+#ifdef EAPOL_RESEND
+			wl_ext_backup_eapol_txpkt(dhdp, ifidx, pktbuf);
+#endif /* EAPOL_RESEND */
+		}
+		dhd_dump_pkt(dhdp, ifidx, pktdata,
+			(uint32)PKTLEN(dhdp->osh, pktbuf), TRUE, NULL, NULL);
+	} else {
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return BCME_ERROR;
+	}
+
+#if (defined(BCM_ROUTER_DHD) && defined(QOS_MAP_SET))
+	if (ifp->qosmap_up_table_enable) {
+		pktsetprio_qms(pktbuf, ifp->qosmap_up_table, FALSE);
+	}
+	else
+#endif
+	{
+		/* Look into the packet and update the packet priority */
+#ifndef PKTPRIO_OVERRIDE
+		/* XXX RB:6270 Ignore skb->priority from TCP/IP stack */
+		if (PKTPRIO(pktbuf) == 0)
+#endif /* !PKTPRIO_OVERRIDE */
+		{
+#if (!defined(BCM_ROUTER_DHD) && (defined(QOS_MAP_SET) || \
+	defined(WL_CUSTOM_MAPPING_OF_DSCP)))
+			u8 *up_table = wl_get_up_table(dhdp, ifidx);
+			pktsetprio_qms(pktbuf, up_table, FALSE);
+			if (PKTPRIO(pktbuf) > MAXPRIO) {
+				DHD_ERROR_RLMT(("wrong user prio:%d from qosmap ifidx:%d\n",
+					PKTPRIO(pktbuf), ifidx));
+				if (up_table) {
+					prhex("up_table", up_table, UP_TABLE_MAX);
+				}
+			}
+#else
+			/* For LLR, pkt prio will be changed to 7(NC) here */
+			pktsetprio(pktbuf, FALSE);
+#endif /* QOS_MAP_SET || WL_CUSTOM_MAPPING_OF_DSCP */
+		}
+#ifndef PKTPRIO_OVERRIDE
+		else {
+			/* Some protocols like OZMO use priority values from 256..263.
+			 * these are magic values to indicate a specific 802.1d priority.
+			 * make sure that priority field is in range of 0..7
+			 */
+			PKTSETPRIO(pktbuf, PKTPRIO(pktbuf) & 0x7);
+		}
+#endif /* !PKTPRIO_OVERRIDE */
+	}
+
+#if defined(BCM_ROUTER_DHD)
+	traffic_mgmt_pkt_set_prio(dhdp, pktbuf);
+
+#endif /* BCM_ROUTER_DHD */
+
+	BCM_REFERENCE(pkt_flow_prio);
+	/* Intercept and create Socket level statistics */
+	/*
+	 * TODO: Some how moving this code block above the pktsetprio code
+	 * is resetting the priority back to 0, but this does not happen for
+	 * packets generated from iperf uisng -S option. Can't understand why.
+	 */
+	dhd_update_sock_flows(dhd, pktbuf);
+
+#ifdef SUPPORT_SET_TID
+	dhd_set_tid_based_on_uid(dhdp, pktbuf);
+#endif	/* SUPPORT_SET_TID */
+
+#ifdef PCIE_FULL_DONGLE
+	/*
+	 * Lkup the per interface hash table, for a matching flowring. If one is not
+	 * available, allocate a unique flowid and add a flowring entry.
+	 * The found or newly created flowid is placed into the pktbuf's tag.
+	 */
+
+#ifdef DHD_TX_PROFILE
+	if (dhdp->tx_profile_enab && dhdp->num_profiles > 0 &&
+		dhd_protocol_matches_profile(PKTDATA(dhdp->osh, pktbuf),
+		PKTLEN(dhdp->osh, pktbuf), dhdp->protocol_filters,
+		dhdp->host_sfhllc_supported)) {
+		/* we only have support for one tx_profile at the moment */
+
+		/* tagged packets must be put into TID 6 */
+		PKTSETPRIO(pktbuf, PRIO_8021D_VO);
+	}
+#endif /* defined(DHD_TX_PROFILE) */
+
+	if (PKTPRIO(pktbuf) > MAXPRIO) {
+		DHD_ERROR_RLMT(("Wrong user prio:%d ifidx:%d\n", PKTPRIO(pktbuf), ifidx));
+		/* non-assert build, print ratelimit error, free packet and exit */
+		ASSERT(0);
+		PKTCFREE(dhd->pub.osh, pktbuf, TRUE);
+		return BCME_ERROR;
+	}
+
+	pkt_flow_prio = dhdp->flow_prio_map[(PKTPRIO(pktbuf))];
+
+	ret = dhd_flowid_update(dhdp, ifidx, pkt_flow_prio, pktbuf);
+	if (ret != BCME_OK) {
+		PKTCFREE(dhd->pub.osh, pktbuf, TRUE);
+		return ret;
+	}
+#endif /* PCIE_FULL_DONGLE */
+	/* terence 20150901: Micky add to ajust the 802.1X priority */
+	/* Set the 802.1X packet with the highest priority 7 */
+	if (dhdp->conf->pktprio8021x >= 0)
+		pktset8021xprio(pktbuf, dhdp->conf->pktprio8021x);
+
+#ifdef PROP_TXSTATUS
+	if (dhd_wlfc_is_supported(dhdp)) {
+		/* store the interface ID */
+		DHD_PKTTAG_SETIF(PKTTAG(pktbuf), ifidx);
+
+		/* store destination MAC in the tag as well */
+		DHD_PKTTAG_SETDSTN(PKTTAG(pktbuf), eh->ether_dhost);
+
+		/* decide which FIFO this packet belongs to */
+		if (ETHER_ISMULTI(eh->ether_dhost))
+			/* one additional queue index (highest AC + 1) is used for bc/mc queue */
+			DHD_PKTTAG_SETFIFO(PKTTAG(pktbuf), AC_COUNT);
+		else
+			DHD_PKTTAG_SETFIFO(PKTTAG(pktbuf), WME_PRIO2AC(PKTPRIO(pktbuf)));
+	} else
+#endif /* PROP_TXSTATUS */
+	{
+		/* If the protocol uses a data header, apply it */
+		dhd_prot_hdrpush(dhdp, ifidx, pktbuf);
+	}
+
+	/* Use bus module to send data frame */
+#ifdef PROP_TXSTATUS
+	{
+		if (dhd_wlfc_commit_packets(dhdp, (f_commitpkt_t)dhd_bus_txdata,
+			dhdp->bus, pktbuf, TRUE) == WLFC_UNSUPPORTED) {
+			/* non-proptxstatus way */
+#ifdef BCMPCIE
+			ret = dhd_bus_txdata(dhdp->bus, pktbuf, (uint8)ifidx);
+#else
+			ret = dhd_bus_txdata(dhdp->bus, pktbuf);
+#endif /* BCMPCIE */
+		}
+	}
+#else
+#ifdef BCMPCIE
+	ret = dhd_bus_txdata(dhdp->bus, pktbuf, (uint8)ifidx);
+#else
+	ret = dhd_bus_txdata(dhdp->bus, pktbuf);
+#endif /* BCMPCIE */
+#endif /* PROP_TXSTATUS */
+
+	/* Update the packet counters here, as it is called for LB Tx and non-LB Tx too */
+	if (ret) {
+#ifdef BCMDBUS
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+#endif /* BCMDBUS */
+		ifp->stats.tx_dropped++;
+		dhdp->tx_dropped++;
+	} else {
+#ifdef PROP_TXSTATUS
+		/* tx_packets counter can counted only when wlfc is disabled */
+		if (!dhd_wlfc_is_supported(dhdp))
+#endif
+		{
+			dhdp->tx_packets++;
+			dhdp->tx_bytes += datalen;
+			ifp->stats.tx_packets++;
+			ifp->stats.tx_bytes += datalen;
+		}
+		dhdp->actual_tx_pkts++;
+	}
+
+	return ret;
+}
+
+int
+BCMFASTPATH(dhd_sendpkt)(dhd_pub_t *dhdp, int ifidx, void *pktbuf)
+{
+	int ret = 0;
+	unsigned long flags;
+	dhd_if_t *ifp;
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	ifp = dhd_get_ifp(dhdp, ifidx);
+	if (!ifp || ifp->del_in_progress) {
+		DHD_ERROR(("%s: ifp:%p del_in_progress:%d\n",
+			__FUNCTION__, ifp, ifp ? ifp->del_in_progress : 0));
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return -ENODEV;
+	}
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_ERROR(("%s: returning as busstate=%d\n",
+			__FUNCTION__, dhdp->busstate));
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return -ENODEV;
+	}
+	DHD_IF_SET_TX_ACTIVE(ifp, DHD_TX_SEND_PKT);
+	DHD_BUS_BUSY_SET_IN_SEND_PKT(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(dhdp)) {
+		DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state!!\n",
+			__FUNCTION__, dhdp->busstate, dhdp->dhd_bus_busy_state));
+		DHD_BUS_BUSY_CLEAR_IN_SEND_PKT(dhdp);
+		DHD_IF_CLR_TX_ACTIVE(ifp, DHD_TX_SEND_PKT);
+		dhd_os_tx_completion_wake(dhdp);
+		dhd_os_busbusy_wake(dhdp);
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		PKTCFREE(dhdp->osh, pktbuf, TRUE);
+		return -ENODEV;
+	}
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	ret = __dhd_sendpkt(dhdp, ifidx, pktbuf);
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_SEND_PKT(dhdp);
+	DHD_IF_CLR_TX_ACTIVE(ifp, DHD_TX_SEND_PKT);
+	dhd_os_tx_completion_wake(dhdp);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	return ret;
+}
+
+#ifdef DHD_MQ
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0))
+static uint16
+BCMFASTPATH(dhd_select_queue)(struct net_device *net, struct sk_buff *skb,
+       void *accel_priv, select_queue_fallback_t fallback)
+#else
+static uint16
+BCMFASTPATH(dhd_select_queue)(struct net_device *net, struct sk_buff *skb)
+#endif /* LINUX_VERSION_CODE */
+{
+	dhd_info_t *dhd_info = DHD_DEV_INFO(net);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+	uint16 prio = 0;
+
+	BCM_REFERENCE(dhd_info);
+	BCM_REFERENCE(dhdp);
+	BCM_REFERENCE(prio);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0))
+	if (mq_select_disable) {
+		/* if driver side queue selection is disabled via sysfs, call the kernel
+		* supplied fallback function to select the queue, which is usually
+		* '__netdev_pick_tx()' in net/core/dev.c
+		*/
+		return fallback(net, skb);
+	}
+#endif /* LINUX_VERSION */
+
+	prio = dhdp->flow_prio_map[skb->priority];
+	if (prio < AC_COUNT)
+		return prio;
+	else
+		return AC_BK;
+}
+#endif /* DHD_MQ */
+
+netdev_tx_t
+BCMFASTPATH(dhd_start_xmit)(struct sk_buff *skb, struct net_device *net)
+{
+	int ret;
+	void *pktbuf;
+	dhd_info_t *dhd = DHD_DEV_INFO(net);
+	dhd_if_t *ifp = NULL;
+	int ifidx;
+	unsigned long flags;
+#if !defined(BCM_ROUTER_DHD)
+	uint8 htsfdlystat_sz = 0;
+#endif /* ! BCM_ROUTER_DHD */
+#ifdef DHD_WMF
+	struct ether_header *eh;
+	uint8 *iph;
+#endif /* DHD_WMF */
+#if defined(DHD_MQ) && defined(DHD_MQ_STATS)
+	int qidx = 0;
+	int cpuid = 0;
+	int prio = 0;
+#endif /* DHD_MQ && DHD_MQ_STATS */
+#if defined(WL_CFG80211)
+	struct bcm_cfg80211 *cfg = wl_get_cfg(net);
+#endif /* defined(WL_CFG80211) */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+#if defined(WL_CFG80211)
+	UNUSED_PARAMETER(cfg);
+#endif /* defined(WL_CFG80211) */
+
+#if defined(DHD_MQ) && defined(DHD_MQ_STATS)
+	qidx = skb_get_queue_mapping(skb);
+	/* if in a non pre-emptable context, smp_processor_id can be used
+	* else get_cpu and put_cpu should be used
+	*/
+	if (!CAN_SLEEP()) {
+		cpuid = smp_processor_id();
+	}
+	else {
+		cpuid = get_cpu();
+		put_cpu();
+	}
+	prio = dhd->pub.flow_prio_map[skb->priority];
+	DHD_TRACE(("%s: Q idx = %d, CPU = %d, prio = %d \n", __FUNCTION__,
+		qidx, cpuid, prio));
+	dhd->pktcnt_qac_histo[qidx][prio]++;
+	dhd->pktcnt_per_ac[prio]++;
+	dhd->cpu_qstats[qidx][cpuid]++;
+#endif /* DHD_MQ && DHD_MQ_STATS */
+
+	if (dhd_query_bus_erros(&dhd->pub)) {
+		return -ENODEV;
+	}
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	DHD_BUS_BUSY_SET_IN_TX(&dhd->pub);
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+#ifdef BCMPCIE
+	if (DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(&dhd->pub)) {
+		DHD_ERROR_RLMT(("%s: bus is in suspend(%d) or suspending(0x%x) state!!\n",
+			__FUNCTION__, dhd->pub.busstate, dhd->pub.dhd_bus_busy_state));
+		DHD_BUS_BUSY_CLEAR_IN_TX(&dhd->pub);
+#ifdef PCIE_FULL_DONGLE
+		/* Stop tx queues if suspend is in progress or suspended */
+		if (DHD_BUS_CHECK_ANY_SUSPEND_IN_PROGRESS(&dhd->pub) ||
+			(dhd->pub.busstate == DHD_BUS_SUSPEND &&
+			!DHD_BUS_BUSY_CHECK_RESUME_IN_PROGRESS(&dhd->pub))) {
+			dhd_bus_stop_queue(dhd->pub.bus);
+		}
+#endif /* PCIE_FULL_DONGLE */
+		dhd_os_busbusy_wake(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		return NETDEV_TX_BUSY;
+	}
+#else
+	if (DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(&dhd->pub)) {
+		DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state!!\n",
+			__FUNCTION__, dhd->pub.busstate, dhd->pub.dhd_bus_busy_state));
+	}
+#endif
+
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+
+#if defined(DHD_HANG_SEND_UP_TEST)
+	if (dhd->pub.req_hang_type == HANG_REASON_BUS_DOWN) {
+		DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+		dhd->pub.busstate = DHD_BUS_DOWN;
+	}
+#endif /* DHD_HANG_SEND_UP_TEST */
+
+	/* Reject if down */
+	/* XXX kernel panic issue when first bootup time,
+	 *	 rmmod without interface down make unnecessary hang event.
+	 */
+	if (dhd->pub.hang_was_sent || DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(&dhd->pub)) {
+		DHD_ERROR(("%s: xmit rejected pub.up=%d busstate=%d \n",
+			__FUNCTION__, dhd->pub.up, dhd->pub.busstate));
+		dhd_tx_stop_queues(net);
+#if defined(OEM_ANDROID)
+		/* Send Event when bus down detected during data session */
+		if (dhd->pub.up && !dhd->pub.hang_was_sent && !DHD_BUS_CHECK_REMOVE(&dhd->pub)) {
+			DHD_ERROR(("%s: Event HANG sent up\n", __FUNCTION__));
+			dhd->pub.hang_reason = HANG_REASON_BUS_DOWN;
+			net_os_send_hang_message(net);
+		}
+#endif /* OEM_ANDROID */
+		DHD_BUS_BUSY_CLEAR_IN_TX(&dhd->pub);
+		dhd_os_busbusy_wake(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return NETDEV_TX_BUSY;
+	}
+
+	ifp = DHD_DEV_IFP(net);
+	ifidx = DHD_DEV_IFIDX(net);
+#ifdef DHD_BUZZZ_LOG_ENABLED
+	BUZZZ_LOG(START_XMIT_BGN, 2, (uint32)ifidx, (uintptr)skb);
+#endif /* DHD_BUZZZ_LOG_ENABLED */
+	if (ifidx == DHD_BAD_IF) {
+		DHD_ERROR(("%s: bad ifidx %d\n", __FUNCTION__, ifidx));
+		dhd_tx_stop_queues(net);
+		DHD_BUS_BUSY_CLEAR_IN_TX(&dhd->pub);
+		dhd_os_busbusy_wake(&dhd->pub);
+		DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+		DHD_OS_WAKE_UNLOCK(&dhd->pub);
+		return NETDEV_TX_BUSY;
+	}
+#ifdef RPM_FAST_TRIGGER
+	/* Xmit is running reset RPM fast trigger */
+	if (dhd->pub.rpm_fast_trigger) {
+		DHD_ERROR(("%s: Reset RPM fast trigger\n", __FUNCTION__));
+		dhd->pub.rpm_fast_trigger = FALSE;
+	}
+#endif /* RPM_FAST_TRIGGER */
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+
+	/* If tput test is in progress */
+	if (dhd->pub.tput_data.tput_test_running) {
+		return NETDEV_TX_BUSY;
+	}
+
+	ASSERT(ifidx == dhd_net2idx(dhd, net));
+	ASSERT((ifp != NULL) && ((ifidx < DHD_MAX_IFS) && (ifp == dhd->iflist[ifidx])));
+
+	bcm_object_trace_opr(skb, BCM_OBJDBG_ADD_PKT, __FUNCTION__, __LINE__);
+
+	/* re-align socket buffer if "skb->data" is odd address */
+	if (((unsigned long)(skb->data)) & 0x1) {
+		unsigned char *data = skb->data;
+		uint32 length = skb->len;
+		PKTPUSH(dhd->pub.osh, skb, 1);
+		memmove(skb->data, data, length);
+		PKTSETLEN(dhd->pub.osh, skb, length);
+	}
+
+#ifdef TPUT_MONITOR
+	if (dhd->pub.conf->tput_monitor_ms) {
+		dhd_os_sdlock_txq(&dhd->pub);
+		dhd->pub.conf->net_len += PKTLEN(dhd->pub.osh, skb);
+		dhd_os_sdunlock_txq(&dhd->pub);
+		if ((dhd->pub.conf->data_drop_mode == XMIT_DROP) &&
+				(PKTLEN(dhd->pub.osh, skb) > 500)) {
+			dev_kfree_skb(skb);
+			return NETDEV_TX_OK;
+		}
+	}
+#endif
+	/* Make sure there's enough room for any header */
+#if !defined(BCM_ROUTER_DHD)
+	if (skb_cow(skb, (dhd->pub.hdrlen + htsfdlystat_sz))) {
+		DHD_ERROR(("%s: skb_cow failed\n",
+		           dhd_ifname(&dhd->pub, ifidx)));
+		bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE, __FUNCTION__, __LINE__);
+		dev_kfree_skb_any(skb);
+		ret = -ENOMEM;
+		goto done;
+	}
+#endif /* !BCM_ROUTER_DHD */
+
+	/* Convert to packet */
+	if (!(pktbuf = PKTFRMNATIVE(dhd->pub.osh, skb))) {
+		DHD_ERROR(("%s: PKTFRMNATIVE failed\n",
+		           dhd_ifname(&dhd->pub, ifidx)));
+		bcm_object_trace_opr(skb, BCM_OBJDBG_REMOVE, __FUNCTION__, __LINE__);
+		dev_kfree_skb_any(skb);
+		ret = -ENOMEM;
+		goto done;
+	}
+
+#ifdef DHD_WET
+	/* wet related packet proto manipulation should be done in DHD
+	   since dongle doesn't have complete payload
+	 */
+	if (WET_ENABLED(&dhd->pub) &&
+			(dhd_wet_send_proc(dhd->pub.wet_info, pktbuf, &pktbuf) < 0)) {
+		DHD_INFO(("%s:%s: wet send proc failed\n",
+				__FUNCTION__, dhd_ifname(&dhd->pub, ifidx)));
+		PKTFREE(dhd->pub.osh, pktbuf, FALSE);
+		ret =  -EFAULT;
+		goto done;
+	}
+#endif /* DHD_WET */
+
+#ifdef DHD_WMF
+	eh = (struct ether_header *)PKTDATA(dhd->pub.osh, pktbuf);
+	iph = (uint8 *)eh + ETHER_HDR_LEN;
+
+	/* WMF processing for multicast packets
+	 * Only IPv4 packets are handled
+	 */
+	if (ifp->wmf.wmf_enable && (ntoh16(eh->ether_type) == ETHER_TYPE_IP) &&
+		(IP_VER(iph) == IP_VER_4) && (ETHER_ISMULTI(eh->ether_dhost) ||
+		((IPV4_PROT(iph) == IP_PROT_IGMP) && dhd->pub.wmf_ucast_igmp))) {
+#if defined(DHD_IGMP_UCQUERY) || defined(DHD_UCAST_UPNP)
+		void *sdu_clone;
+		bool ucast_convert = FALSE;
+#ifdef DHD_UCAST_UPNP
+		uint32 dest_ip;
+
+		dest_ip = ntoh32(*((uint32 *)(iph + IPV4_DEST_IP_OFFSET)));
+		ucast_convert = dhd->pub.wmf_ucast_upnp && MCAST_ADDR_UPNP_SSDP(dest_ip);
+#endif /* DHD_UCAST_UPNP */
+#ifdef DHD_IGMP_UCQUERY
+		ucast_convert |= dhd->pub.wmf_ucast_igmp_query &&
+			(IPV4_PROT(iph) == IP_PROT_IGMP) &&
+			(*(iph + IPV4_HLEN(iph)) == IGMPV2_HOST_MEMBERSHIP_QUERY);
+#endif /* DHD_IGMP_UCQUERY */
+		if (ucast_convert) {
+			dhd_sta_t *sta;
+			unsigned long flags;
+			struct list_head snapshot_list;
+			struct list_head *wmf_ucforward_list;
+
+			ret = NETDEV_TX_OK;
+
+			/* For non BCM_GMAC3 platform we need a snapshot sta_list to
+			 * resolve double DHD_IF_STA_LIST_LOCK call deadlock issue.
+			 */
+			wmf_ucforward_list = DHD_IF_WMF_UCFORWARD_LOCK(dhd, ifp, &snapshot_list);
+
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			/* Convert upnp/igmp query to unicast for each assoc STA */
+			list_for_each_entry(sta, wmf_ucforward_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				/* Skip sending to proxy interfaces of proxySTA */
+				if (sta->psta_prim != NULL && !ifp->wmf_psta_disable) {
+					continue;
+				}
+				if ((sdu_clone = PKTDUP(dhd->pub.osh, pktbuf)) == NULL) {
+					ret = WMF_NOP;
+					break;
+				}
+				dhd_wmf_forward(ifp->wmf.wmfh, sdu_clone, 0, sta, 1);
+			}
+			DHD_IF_WMF_UCFORWARD_UNLOCK(dhd, wmf_ucforward_list);
+
+			DHD_GENERAL_LOCK(&dhd->pub, flags);
+			DHD_BUS_BUSY_CLEAR_IN_TX(&dhd->pub);
+			dhd_os_busbusy_wake(&dhd->pub);
+			DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+			DHD_OS_WAKE_UNLOCK(&dhd->pub);
+
+			if (ret == NETDEV_TX_OK)
+				PKTFREE(dhd->pub.osh, pktbuf, TRUE);
+
+			return ret;
+		} else
+#endif /* defined(DHD_IGMP_UCQUERY) || defined(DHD_UCAST_UPNP) */
+		{
+			/* There will be no STA info if the packet is coming from LAN host
+			 * Pass as NULL
+			 */
+			ret = dhd_wmf_packets_handle(&dhd->pub, pktbuf, NULL, ifidx, 0);
+			switch (ret) {
+			case WMF_TAKEN:
+			case WMF_DROP:
+				/* Either taken by WMF or we should drop it.
+				 * Exiting send path
+				 */
+
+				DHD_GENERAL_LOCK(&dhd->pub, flags);
+				DHD_BUS_BUSY_CLEAR_IN_TX(&dhd->pub);
+				dhd_os_busbusy_wake(&dhd->pub);
+				DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+				DHD_OS_WAKE_UNLOCK(&dhd->pub);
+				return NETDEV_TX_OK;
+			default:
+				/* Continue the transmit path */
+				break;
+			}
+		}
+	}
+#endif /* DHD_WMF */
+#ifdef DHD_PSTA
+	/* PSR related packet proto manipulation should be done in DHD
+	 * since dongle doesn't have complete payload
+	 */
+	if (PSR_ENABLED(&dhd->pub) &&
+#ifdef BCM_ROUTER_DHD
+		!(ifp->primsta_dwds) &&
+#endif /* BCM_ROUTER_DHD */
+		(dhd_psta_proc(&dhd->pub, ifidx, &pktbuf, TRUE) < 0)) {
+
+			DHD_ERROR(("%s:%s: psta send proc failed\n", __FUNCTION__,
+				dhd_ifname(&dhd->pub, ifidx)));
+	}
+#endif /* DHD_PSTA */
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_PACING_SHIFT)
+#ifndef DHD_DEFAULT_TCP_PACING_SHIFT
+#define DHD_DEFAULT_TCP_PACING_SHIFT 7
+#endif /* DHD_DEFAULT_TCP_PACING_SHIFT */
+	if (skb->sk) {
+		sk_pacing_shift_update(skb->sk, DHD_DEFAULT_TCP_PACING_SHIFT);
+	}
+#else
+#if defined(BCMPCIE) && defined(DHD_VSDB_SKIP_ORPHAN) && defined(WL_CFG80211)
+	if (!cfg->vsdb_mode)
+#endif /* (BCMPCIE) && (DHD_VSDB_SKIP_ORPHAN) && defined(WL_CFG80211) */
+	skb_orphan(skb);
+#endif /* LINUX_VERSION_CODE >= 4.19.0 && DHD_TCP_PACING_SHIFT */
+
+	/* move from dhdsdio_sendfromq(), try to orphan skb early */
+	if (dhd->pub.conf->orphan_move == 2)
+		PKTORPHAN(skb, dhd->pub.conf->tsq);
+	else if (dhd->pub.conf->orphan_move == 3)
+		skb_orphan(skb);
+
+#ifdef DHDTCPSYNC_FLOOD_BLK
+	if (dhd_tcpdata_get_flag(&dhd->pub, pktbuf) == FLAG_SYNCACK) {
+		ifp->tsyncack_txed ++;
+	}
+#endif /* DHDTCPSYNC_FLOOD_BLK */
+
+#ifdef DHDTCPACK_SUPPRESS
+	if (dhd->pub.tcpack_sup_mode == TCPACK_SUP_HOLD) {
+		/* If this packet has been hold or got freed, just return */
+		if (dhd_tcpack_hold(&dhd->pub, pktbuf, ifidx)) {
+			ret = 0;
+			goto done;
+		}
+	} else {
+		/* If this packet has replaced another packet and got freed, just return */
+		if (dhd_tcpack_suppress(&dhd->pub, pktbuf)) {
+			ret = 0;
+			goto done;
+		}
+	}
+#endif /* DHDTCPACK_SUPPRESS */
+
+	/*
+	 * If Load Balance is enabled queue the packet
+	 * else send directly from here.
+	 */
+#if defined(DHD_LB_TXP)
+	ret = dhd_lb_sendpkt(dhd, net, ifidx, pktbuf);
+#else
+	ret = __dhd_sendpkt(&dhd->pub, ifidx, pktbuf);
+#endif
+	BCM_REFERENCE(ret);
+
+done:
+	DHD_GENERAL_LOCK(&dhd->pub, flags);
+	DHD_BUS_BUSY_CLEAR_IN_TX(&dhd->pub);
+	DHD_IF_CLR_TX_ACTIVE(ifp, DHD_TX_START_XMIT);
+	dhd_os_tx_completion_wake(&dhd->pub);
+	dhd_os_busbusy_wake(&dhd->pub);
+	DHD_GENERAL_UNLOCK(&dhd->pub, flags);
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+#ifdef DHD_BUZZZ_LOG_ENABLED
+	BUZZZ_LOG(START_XMIT_END, 0);
+#endif /* DHD_BUZZZ_LOG_ENABLED */
+	/* Return ok: we always eat the packet */
+	return NETDEV_TX_OK;
+}
+
+static void
+__dhd_txflowcontrol(dhd_pub_t *dhdp, struct net_device *net, bool state)
+{
+
+	if (net->reg_state != NETREG_REGISTERED) {
+		return;
+	}
+
+	if (state == ON) {
+		if (!netif_queue_stopped(net)) {
+			DHD_TXFLOWCTL(("%s: Stop Netif Queue\n", __FUNCTION__));
+			netif_stop_queue(net);
+		} else {
+			DHD_INFO(("%s: Netif Queue already stopped\n", __FUNCTION__));
+		}
+	}
+
+	if (state == OFF) {
+		if (netif_queue_stopped(net)) {
+			DHD_TXFLOWCTL(("%s: Start Netif Queue\n", __FUNCTION__));
+			netif_wake_queue(net);
+		} else {
+			DHD_INFO(("%s: Netif Queue already started\n", __FUNCTION__));
+		}
+	}
+}
+
+void
+dhd_txflowcontrol(dhd_pub_t *dhdp, int ifidx, bool state)
+{
+	struct net_device *net;
+	dhd_info_t *dhd = dhdp->info;
+	unsigned long flags;
+	int i;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(dhd);
+
+#ifdef DHD_LOSSLESS_ROAMING
+	/* block flowcontrol during roaming */
+	if ((dhdp->dequeue_prec_map == (1 << dhdp->flow_prio_map[PRIO_8021D_NC])) && (state == ON))
+	{
+		DHD_ERROR_RLMT(("%s: Roaming in progress, cannot stop network queue (0x%x:%d)\n",
+			__FUNCTION__, dhdp->dequeue_prec_map, dhdp->flow_prio_map[PRIO_8021D_NC]));
+		return;
+	}
+#endif
+
+	flags = dhd_os_sdlock_txoff(&dhd->pub);
+	if (ifidx == ALL_INTERFACES) {
+		for (i = 0; i < DHD_MAX_IFS; i++) {
+			if (dhd->iflist[i]) {
+				net = dhd->iflist[i]->net;
+				__dhd_txflowcontrol(dhdp, net, state);
+			}
+		}
+	} else {
+		if (dhd->iflist[ifidx]) {
+			net = dhd->iflist[ifidx]->net;
+			__dhd_txflowcontrol(dhdp, net, state);
+		}
+	}
+	dhdp->txoff = state;
+	dhd_os_sdunlock_txoff(&dhd->pub, flags);
+}
+
+void
+dhd_txcomplete(dhd_pub_t *dhdp, void *txp, bool success)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(dhdp->info);
+	struct ether_header *eh;
+	uint16 type;
+
+	if (dhdp->tput_data.tput_test_running) {
+
+		dhdp->batch_tx_pkts_cmpl++;
+
+		/* don't count the stop pkt */
+		if (success &&
+			dhdp->batch_tx_pkts_cmpl <= dhdp->batch_tx_num_pkts)
+			dhdp->tput_data.pkts_good++;
+		else if (!success)
+			dhdp->tput_data.pkts_bad++;
+
+		/* we dont care for the stop packet in tput test */
+		if (dhdp->batch_tx_pkts_cmpl == dhdp->batch_tx_num_pkts) {
+			dhdp->tput_stop_ts = OSL_SYSUPTIME_US();
+			dhdp->tput_data.pkts_cmpl += dhdp->batch_tx_pkts_cmpl;
+			dhdp->tput_data.num_pkts += dhdp->batch_tx_num_pkts;
+			dhd_os_tput_test_wake(dhdp);
+		}
+	}
+	/* XXX where does this stuff belong to? */
+	dhd_prot_hdrpull(dhdp, NULL, txp, NULL, NULL);
+
+	/* XXX Use packet tag when it is available to identify its type */
+
+	eh = (struct ether_header *)PKTDATA(dhdp->osh, txp);
+	type  = ntoh16(eh->ether_type);
+
+	if (type == ETHER_TYPE_802_1X) {
+		atomic_dec(&dhd->pend_8021x_cnt);
+	}
+
+#ifdef PROP_TXSTATUS
+	if (dhdp->wlfc_state && (dhdp->proptxstatus_mode != WLFC_FCMODE_NONE)) {
+		dhd_if_t *ifp = dhd->iflist[DHD_PKTTAG_IF(PKTTAG(txp))];
+		uint datalen  = PKTLEN(dhd->pub.osh, txp);
+		if (ifp != NULL) {
+			if (success) {
+				dhd->pub.tx_packets++;
+				dhd->pub.tx_bytes += datalen;
+				ifp->stats.tx_packets++;
+				ifp->stats.tx_bytes += datalen;
+			} else {
+				ifp->stats.tx_dropped++;
+			}
+		}
+	}
+#endif
+	if (success) {
+		dhd->pub.tot_txcpl++;
+	}
+}
+
+void
+dhd_os_tx_completion_wake(dhd_pub_t *dhd)
+{
+	/* Call wmb() to make sure before waking up the other event value gets updated */
+	OSL_SMP_WMB();
+	wake_up(&dhd->tx_completion_wait);
+}
+
+void
+dhd_os_sdlock_txq(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+#ifdef BCMDBUS
+	spin_lock_irqsave(&dhd->txqlock, dhd->txqlock_flags);
+#else
+	spin_lock_bh(&dhd->txqlock);
+#endif
+}
+
+void
+dhd_os_sdunlock_txq(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+#ifdef BCMDBUS
+	spin_unlock_irqrestore(&dhd->txqlock, dhd->txqlock_flags);
+#else
+	spin_unlock_bh(&dhd->txqlock);
+#endif
+}
+
+unsigned long
+dhd_os_sdlock_txoff(dhd_pub_t *pub)
+{
+	dhd_info_t *dhd;
+	unsigned long flags = 0;
+
+	dhd = (dhd_info_t *)(pub->info);
+	spin_lock_irqsave(&dhd->txoff_lock, flags);
+
+	return flags;
+}
+
+void
+dhd_os_sdunlock_txoff(dhd_pub_t *pub, unsigned long flags)
+{
+	dhd_info_t *dhd;
+
+	dhd = (dhd_info_t *)(pub->info);
+	spin_unlock_irqrestore(&dhd->txoff_lock, flags);
+}
+
+void
+dhd_txfl_wake_lock_timeout(dhd_pub_t *pub, int val)
+{
+#ifdef CONFIG_HAS_WAKELOCK
+	dhd_info_t *dhd = (dhd_info_t *)(pub->info);
+
+	if (dhd) {
+		dhd_wake_lock_timeout(dhd->wl_txflwake, msecs_to_jiffies(val));
+	}
+#endif /* CONFIG_HAS_WAKE_LOCK */
+}
+
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+void
+dhd_eap_txcomplete(dhd_pub_t *dhdp, void *txp, bool success, int ifidx)
+{
+	dhd_info_t *dhd = (dhd_info_t *)(dhdp->info);
+	struct ether_header *eh;
+	uint16 type;
+
+	/* XXX where does this stuff belong to? */
+	dhd_prot_hdrpull(dhdp, NULL, txp, NULL, NULL);
+
+	/* XXX Use packet tag when it is available to identify its type */
+	eh = (struct ether_header *)PKTDATA(dhdp->osh, txp);
+	type  = ntoh16(eh->ether_type);
+	if (type == ETHER_TYPE_802_1X) {
+		if (dhd_is_4way_msg((uint8 *)eh) == EAPOL_4WAY_M4) {
+			dhd_if_t *ifp = NULL;
+			ifp = dhd->iflist[ifidx];
+			if (!ifp || !ifp->net) {
+				return;
+			}
+			if (!success) {
+				DHD_INFO(("%s: M4 TX failed on %d.\n",
+						__FUNCTION__, ifidx));
+
+				OSL_ATOMIC_SET(dhdp->osh, &ifp->m4state, M4_TXFAILED);
+				schedule_delayed_work(&ifp->m4state_work,
+						msecs_to_jiffies(MAX_4WAY_TIMEOUT_MS));
+			} else {
+				cancel_delayed_work(&ifp->m4state_work);
+			}
+		}
+	}
+}
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+
+void
+dhd_handle_pktdata(dhd_pub_t *dhdp, int ifidx, void *pkt, uint8 *pktdata, uint32 pktid,
+	uint32 pktlen, uint16 *pktfate, uint8 *dhd_udr, uint8 *dhd_igmp,
+	bool tx, int pkt_wake, bool pkt_log)
+{
+	struct ether_header *eh;
+	uint16 ether_type;
+	uint32 pkthash;
+	uint8 pkt_type = PKT_TYPE_DATA;
+#ifdef DHD_PKT_LOGGING_DBGRING
+	bool verbose_logging = FALSE;
+	dhd_dbg_ring_t *ring;
+	ring = &dhdp->dbg->dbg_rings[PACKET_LOG_RING_ID];
+	UNUSED_PARAMETER(verbose_logging);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	if (!pktdata || pktlen < ETHER_HDR_LEN) {
+		return;
+	}
+
+	eh = (struct ether_header *)pktdata;
+	ether_type = ntoh16(eh->ether_type);
+
+	/* Check packet type */
+	if (dhd_check_ip_prot(pktdata, ether_type)) {
+		if (dhd_check_dhcp(pktdata)) {
+			pkt_type = PKT_TYPE_DHCP;
+		} else if (dhd_check_icmp(pktdata)) {
+			pkt_type = PKT_TYPE_ICMP;
+		} else if (dhd_check_dns(pktdata)) {
+			pkt_type = PKT_TYPE_DNS;
+		}
+	}
+	else if (ether_type == ETHER_TYPE_IPV6) {
+		if (dhd_check_icmpv6(pktdata, pktlen)) {
+			pkt_type = PKT_TYPE_ICMPV6;
+		}
+#ifdef DHD_IPV6_DUMP
+		else if (dhd_check_dhcp6(pktdata, pktlen)) {
+			pkt_type = PKT_TYPE_DHCP6;
+		} else if (dhd_check_dns6(pktdata, pktlen)) {
+			pkt_type = PKT_TYPE_DNS6;
+		}
+#endif
+	}
+	else if (dhd_check_arp(pktdata, ether_type)) {
+		pkt_type = PKT_TYPE_ARP;
+	}
+	else if (ether_type == ETHER_TYPE_802_1X) {
+		pkt_type = PKT_TYPE_EAP;
+	}
+#ifdef DHD_PKT_LOGGING_DBGRING
+	do {
+		if (!OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->enable)) {
+			struct dhd_pktlog_ring *pktlog_ring;
+
+			pktlog_ring = dhdp->pktlog->pktlog_ring;
+			if (pktlog_ring->pktcount <= DHD_PACKET_LOG_RING_RESUME_THRESHOLD) {
+				dhd_pktlog_resume(dhdp);
+			} else {
+				/* If pktlog disabled(suspeneded), only allowed TXS update */
+				if (tx && pktfate) {
+					DHD_PKTLOG_TXS(dhdp, pkt, pktdata, pktid, *pktfate);
+					pkthash = __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
+				}
+				dhd_os_dbg_urgent_pullreq(dhdp->dbg->private,
+					PACKET_LOG_RING_ID);
+				break;
+			}
+		}
+
+		/* Allow logging for all packets without pktlog filter */
+		if (ring->log_level == RING_LOG_LEVEL_EXCESSIVE) {
+			verbose_logging = TRUE;
+			pkt_log = TRUE;
+		/* Not allow logging for any packets */
+		} else if (ring->log_level == RING_LOG_LEVEL_NONE) {
+			verbose_logging = FALSE;
+			pkt_log = FALSE;
+		}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+#ifdef DHD_SBN
+		/* Set UDR based on packet type */
+		if (dhd_udr && (pkt_type == PKT_TYPE_DHCP ||
+			pkt_type == PKT_TYPE_DNS ||
+			pkt_type == PKT_TYPE_ARP)) {
+			*dhd_udr = TRUE;
+		}
+#endif /* DHD_SBN */
+
+#ifdef DHD_PKT_LOGGING
+#ifdef DHD_SKIP_PKTLOGGING_FOR_DATA_PKTS
+		if (pkt_type != PKT_TYPE_DATA)
+#else
+#ifdef DHD_PKT_LOGGING_DBGRING
+		if ((verbose_logging == TRUE) || (pkt_type != PKT_TYPE_DATA))
+#endif /* DHD_PKT_LOGGING_DBGRING */
+#endif /* DHD_PKT_LOGGING */
+		{
+			if (pkt_log) {
+				if (tx) {
+					if (pktfate) {
+						/* Tx status */
+						DHD_PKTLOG_TXS(dhdp, pkt, pktdata, pktid, *pktfate);
+					} else {
+						/* Tx packet */
+						DHD_PKTLOG_TX(dhdp, pkt, pktdata, pktid);
+					}
+					pkthash = __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
+				} else {
+					struct sk_buff *skb = (struct sk_buff *)pkt;
+					if (pkt_wake) {
+						DHD_PKTLOG_WAKERX(dhdp, skb, pktdata);
+					} else {
+						DHD_PKTLOG_RX(dhdp, skb, pktdata);
+					}
+				}
+			}
+		}
+#endif /* DHD_PKT_LOGGING */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	} while (FALSE);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	/* Dump packet data */
+	if (!tx)
+		dhd_dump_pkt(dhdp, ifidx, pktdata, pktlen, FALSE, &pkthash, NULL);
+	switch (pkt_type) {
+		case PKT_TYPE_DHCP:
+			dhd_send_supp_dhcp(dhdp, ifidx, pktdata, tx, pktfate);
+			break;
+		case PKT_TYPE_EAP:
+			dhd_send_supp_eap(dhdp, ifidx, pktdata, pktlen, tx, pktfate);
+			break;
+#ifdef DHD_IPV6_DUMP
+		case PKT_TYPE_ICMPV6:
+			dhd_icmpv6_dump(dhdp, ifidx, pktdata, tx, &pkthash, pktfate);
+			break;
+		case PKT_TYPE_DHCP6:
+			dhd_dhcp6_dump(dhdp, ifidx, pktdata, tx, &pkthash, pktfate);
+			break;
+		case PKT_TYPE_DNS6:
+			dhd_dns6_dump(dhdp, ifidx, pktdata, tx, &pkthash, pktfate);
+			break;
+#endif
+		default:
+			break;
+	}
+}
diff --git a/drivers/net/wireless/ap12275/dhd_linux_tx.h b/drivers/net/wireless/ap12275/dhd_linux_tx.h
new file mode 100644
index 000000000000..1484f7b5c31b
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_tx.h
@@ -0,0 +1,49 @@
+/*
+ * Broadcom Dongle Host Driver (DHD),
+ * Linux-specific network interface for transmit(tx) path
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef _dhd_linux_tx_h_
+#define _dhd_linux_tx_h_
+void dhd_tx_stop_queues(struct net_device *net);
+void dhd_tx_start_queues(struct net_device *net);
+int BCMFASTPATH(__dhd_sendpkt)(dhd_pub_t *dhdp, int ifidx, void *pktbuf);
+int BCMFASTPATH(dhd_sendpkt)(dhd_pub_t *dhdp, int ifidx, void *pktbuf);
+#ifdef DHD_MQ
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0))
+uint16
+BCMFASTPATH(dhd_select_queue)(struct net_device *net, struct sk_buff *skb,
+       void *accel_priv, select_queue_fallback_t fallback);
+#else
+uint16
+BCMFASTPATH(dhd_select_queue)(struct net_device *net, struct sk_buff *skb);
+#endif /* LINUX_VERSION_CODE */
+#endif /* DHD_MQ */
+netdev_tx_t BCMFASTPATH(dhd_start_xmit)(struct sk_buff *skb, struct net_device *net);
+void dhd_txflowcontrol(dhd_pub_t *dhdp, int ifidx, bool state);
+void dhd_txcomplete(dhd_pub_t *dhdp, void *txp, bool success);
+void dhd_os_tx_completion_wake(dhd_pub_t *dhd);
+void dhd_os_sdlock_txq(dhd_pub_t *pub);
+void dhd_os_sdunlock_txq(dhd_pub_t *pub);
+#endif /* _dhd_linux_tx_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_linux_wq.c b/drivers/net/wireless/ap12275/dhd_linux_wq.c
new file mode 100644
index 000000000000..1904d0e24b8b
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_wq.c
@@ -0,0 +1,413 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Generic work queue framework
+ * Generic interface to handle dhd deferred work events
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/fcntl.h>
+#include <linux/fs.h>
+#include <linux/ip.h>
+#include <linux/kfifo.h>
+
+#include <linuxver.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_linux_wq.h>
+
+/*
+ * XXX: always make sure that the size of this structure is aligned to
+ * the power of 2 (2^n) i.e, if any new variable has to be added then
+ * modify the padding accordingly
+ */
+typedef struct dhd_deferred_event {
+	u8 event;		/* holds the event */
+	void *event_data;	/* holds event specific data */
+	event_handler_t event_handler;
+	unsigned long pad;	/* for memory alignment to power of 2 */
+} dhd_deferred_event_t;
+
+#define DEFRD_EVT_SIZE	(sizeof(dhd_deferred_event_t))
+
+/*
+ * work events may occur simultaneously.
+ * can hold upto 64 low priority events and 16 high priority events
+ */
+#define DHD_PRIO_WORK_FIFO_SIZE	(16 * DEFRD_EVT_SIZE)
+#define DHD_WORK_FIFO_SIZE	(64 * DEFRD_EVT_SIZE)
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 32))
+#define kfifo_avail(fifo) (fifo->size - kfifo_len(fifo))
+#endif /* (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 32)) */
+
+#define DHD_FIFO_HAS_FREE_SPACE(fifo) \
+	((fifo) && (kfifo_avail(fifo) >= DEFRD_EVT_SIZE))
+#define DHD_FIFO_HAS_ENOUGH_DATA(fifo) \
+	((fifo) && (kfifo_len(fifo) >= DEFRD_EVT_SIZE))
+
+struct dhd_deferred_wq {
+	struct work_struct deferred_work; /* should be the first member */
+
+	struct kfifo *prio_fifo;
+	struct kfifo			*work_fifo;
+	u8				*prio_fifo_buf;
+	u8				*work_fifo_buf;
+	spinlock_t			work_lock;
+	void				*dhd_info; /* review: does it require */
+	u32				event_skip_mask;
+};
+
+static inline struct kfifo*
+dhd_kfifo_init(u8 *buf, int size, spinlock_t *lock)
+{
+	struct kfifo *fifo;
+	gfp_t flags = CAN_SLEEP()? GFP_KERNEL : GFP_ATOMIC;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33))
+	fifo = kfifo_init(buf, size, flags, lock);
+#else
+	fifo = (struct kfifo *)kzalloc(sizeof(struct kfifo), flags);
+	if (!fifo) {
+		return NULL;
+	}
+	kfifo_init(fifo, buf, size);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33)) */
+	return fifo;
+}
+
+static inline void
+dhd_kfifo_free(struct kfifo *fifo)
+{
+	kfifo_free(fifo);
+	kfree(fifo);
+}
+
+/* deferred work functions */
+static void dhd_deferred_work_handler(struct work_struct *data);
+
+void*
+dhd_deferred_work_init(void *dhd_info)
+{
+	struct dhd_deferred_wq	*work = NULL;
+	u8*	buf;
+	unsigned long	fifo_size = 0;
+	gfp_t	flags = CAN_SLEEP()? GFP_KERNEL : GFP_ATOMIC;
+
+	if (!dhd_info) {
+		DHD_ERROR(("%s: dhd info not initialized\n", __FUNCTION__));
+		goto return_null;
+	}
+
+	work = (struct dhd_deferred_wq *)kzalloc(sizeof(struct dhd_deferred_wq),
+		flags);
+	if (!work) {
+		DHD_ERROR(("%s: work queue creation failed\n", __FUNCTION__));
+		goto return_null;
+	}
+
+	INIT_WORK((struct work_struct *)work, dhd_deferred_work_handler);
+
+	/* initialize event fifo */
+	spin_lock_init(&work->work_lock);
+
+	/* allocate buffer to hold prio events */
+	fifo_size = DHD_PRIO_WORK_FIFO_SIZE;
+	fifo_size = is_power_of_2(fifo_size) ? fifo_size :
+			roundup_pow_of_two(fifo_size);
+	buf = (u8*)kzalloc(fifo_size, flags);
+	if (!buf) {
+		DHD_ERROR(("%s: prio work fifo allocation failed\n",
+			__FUNCTION__));
+		goto return_null;
+	}
+
+	/* Initialize prio event fifo */
+	work->prio_fifo = dhd_kfifo_init(buf, fifo_size, &work->work_lock);
+	if (!work->prio_fifo) {
+		kfree(buf);
+		goto return_null;
+	}
+
+	/* allocate buffer to hold work events */
+	fifo_size = DHD_WORK_FIFO_SIZE;
+	fifo_size = is_power_of_2(fifo_size) ? fifo_size :
+			roundup_pow_of_two(fifo_size);
+	buf = (u8*)kzalloc(fifo_size, flags);
+	if (!buf) {
+		DHD_ERROR(("%s: work fifo allocation failed\n", __FUNCTION__));
+		goto return_null;
+	}
+
+	/* Initialize event fifo */
+	work->work_fifo = dhd_kfifo_init(buf, fifo_size, &work->work_lock);
+	if (!work->work_fifo) {
+		kfree(buf);
+		goto return_null;
+	}
+
+	work->dhd_info = dhd_info;
+	work->event_skip_mask = 0;
+	DHD_INFO(("%s: work queue initialized\n", __FUNCTION__));
+	return work;
+
+return_null:
+	if (work) {
+		dhd_deferred_work_deinit(work);
+	}
+
+	return NULL;
+}
+
+void
+dhd_deferred_work_deinit(void *work)
+{
+	struct dhd_deferred_wq *deferred_work = work;
+
+	if (!deferred_work) {
+		DHD_ERROR(("%s: deferred work has been freed already\n",
+			__FUNCTION__));
+		return;
+	}
+
+	/* cancel the deferred work handling */
+	cancel_work_sync((struct work_struct *)deferred_work);
+
+	/*
+	 * free work event fifo.
+	 * kfifo_free frees locally allocated fifo buffer
+	 */
+	if (deferred_work->prio_fifo) {
+		dhd_kfifo_free(deferred_work->prio_fifo);
+	}
+
+	if (deferred_work->work_fifo) {
+		dhd_kfifo_free(deferred_work->work_fifo);
+	}
+
+	kfree(deferred_work);
+}
+
+/* select kfifo according to priority */
+static inline struct kfifo *
+dhd_deferred_work_select_kfifo(struct dhd_deferred_wq *deferred_wq,
+	u8 priority)
+{
+	if (priority == DHD_WQ_WORK_PRIORITY_HIGH) {
+		return deferred_wq->prio_fifo;
+	} else if (priority == DHD_WQ_WORK_PRIORITY_LOW) {
+		return deferred_wq->work_fifo;
+	} else {
+		return NULL;
+	}
+}
+
+/*
+ *	Prepares event to be queued
+ *	Schedules the event
+ */
+int
+dhd_deferred_schedule_work(void *workq, void *event_data, u8 event,
+	event_handler_t event_handler, u8 priority)
+{
+	struct dhd_deferred_wq *deferred_wq = (struct dhd_deferred_wq *)workq;
+	struct kfifo *fifo;
+	dhd_deferred_event_t deferred_event;
+	int bytes_copied = 0;
+
+	if (!deferred_wq) {
+		DHD_ERROR(("%s: work queue not initialized\n", __FUNCTION__));
+		ASSERT(0);
+		return DHD_WQ_STS_UNINITIALIZED;
+	}
+
+	if (!event || (event >= DHD_MAX_WQ_EVENTS)) {
+		DHD_ERROR(("%s: unknown event, event=%d\n", __FUNCTION__,
+			event));
+		return DHD_WQ_STS_UNKNOWN_EVENT;
+	}
+
+	if (!priority || (priority >= DHD_WQ_MAX_PRIORITY)) {
+		DHD_ERROR(("%s: unknown priority, priority=%d\n",
+			__FUNCTION__, priority));
+		return DHD_WQ_STS_UNKNOWN_PRIORITY;
+	}
+
+	if ((deferred_wq->event_skip_mask & (1 << event))) {
+		DHD_ERROR(("%s: Skip event requested. Mask = 0x%x\n",
+			__FUNCTION__, deferred_wq->event_skip_mask));
+		return DHD_WQ_STS_EVENT_SKIPPED;
+	}
+
+	/*
+	 * default element size is 1, which can be changed
+	 * using kfifo_esize(). Older kernel(FC11) doesn't support
+	 * changing element size. For compatibility changing
+	 * element size is not prefered
+	 */
+	ASSERT(kfifo_esize(deferred_wq->prio_fifo) == 1);
+	ASSERT(kfifo_esize(deferred_wq->work_fifo) == 1);
+
+	deferred_event.event = event;
+	deferred_event.event_data = event_data;
+	deferred_event.event_handler = event_handler;
+
+	fifo = dhd_deferred_work_select_kfifo(deferred_wq, priority);
+	if (DHD_FIFO_HAS_FREE_SPACE(fifo)) {
+		bytes_copied = kfifo_in_spinlocked(fifo, &deferred_event,
+			DEFRD_EVT_SIZE, &deferred_wq->work_lock);
+	}
+	if (bytes_copied != DEFRD_EVT_SIZE) {
+		DHD_ERROR(("%s: failed to schedule deferred work, "
+			"priority=%d, bytes_copied=%d\n", __FUNCTION__,
+			priority, bytes_copied));
+		return DHD_WQ_STS_SCHED_FAILED;
+	}
+	schedule_work((struct work_struct *)deferred_wq);
+	return DHD_WQ_STS_OK;
+}
+
+static bool
+dhd_get_scheduled_work(struct dhd_deferred_wq *deferred_wq,
+	dhd_deferred_event_t *event)
+{
+	int bytes_copied = 0;
+
+	if (!deferred_wq) {
+		DHD_ERROR(("%s: work queue not initialized\n", __FUNCTION__));
+		return DHD_WQ_STS_UNINITIALIZED;
+	}
+
+	/*
+	 * default element size is 1 byte, which can be changed
+	 * using kfifo_esize(). Older kernel(FC11) doesn't support
+	 * changing element size. For compatibility changing
+	 * element size is not prefered
+	 */
+	ASSERT(kfifo_esize(deferred_wq->prio_fifo) == 1);
+	ASSERT(kfifo_esize(deferred_wq->work_fifo) == 1);
+
+	/* handle priority work */
+	if (DHD_FIFO_HAS_ENOUGH_DATA(deferred_wq->prio_fifo)) {
+		bytes_copied = kfifo_out_spinlocked(deferred_wq->prio_fifo,
+			event, DEFRD_EVT_SIZE, &deferred_wq->work_lock);
+	}
+
+	/* handle normal work if priority work doesn't have enough data */
+	if ((bytes_copied != DEFRD_EVT_SIZE) &&
+		DHD_FIFO_HAS_ENOUGH_DATA(deferred_wq->work_fifo)) {
+		bytes_copied = kfifo_out_spinlocked(deferred_wq->work_fifo,
+			event, DEFRD_EVT_SIZE, &deferred_wq->work_lock);
+	}
+
+	return (bytes_copied == DEFRD_EVT_SIZE);
+}
+
+static inline void
+dhd_deferred_dump_work_event(dhd_deferred_event_t *work_event)
+{
+	if (!work_event) {
+		DHD_ERROR(("%s: work_event is null\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("%s: work_event->event = %d\n", __FUNCTION__,
+		work_event->event));
+	DHD_ERROR(("%s: work_event->event_data = %p\n", __FUNCTION__,
+		work_event->event_data));
+	DHD_ERROR(("%s: work_event->event_handler = %p\n", __FUNCTION__,
+		work_event->event_handler));
+}
+
+/*
+ *	Called when work is scheduled
+ */
+static void
+dhd_deferred_work_handler(struct work_struct *work)
+{
+	struct dhd_deferred_wq *deferred_work = (struct dhd_deferred_wq *)work;
+	dhd_deferred_event_t work_event;
+
+	if (!deferred_work) {
+		DHD_ERROR(("%s: work queue not initialized\n", __FUNCTION__));
+		return;
+	}
+
+	do {
+		if (!dhd_get_scheduled_work(deferred_work, &work_event)) {
+			DHD_TRACE(("%s: no event to handle\n", __FUNCTION__));
+			break;
+		}
+
+		if (work_event.event >= DHD_MAX_WQ_EVENTS) {
+			DHD_ERROR(("%s: unknown event\n", __FUNCTION__));
+			dhd_deferred_dump_work_event(&work_event);
+			ASSERT(work_event.event < DHD_MAX_WQ_EVENTS);
+			continue;
+		}
+
+		/*
+		 * XXX: don't do NULL check for 'work_event.event_data'
+		 * as for some events like DHD_WQ_WORK_DHD_LOG_DUMP the
+		 * event data is always NULL even though rest of the
+		 * event parameters are valid
+		 */
+
+		if (work_event.event_handler) {
+			work_event.event_handler(deferred_work->dhd_info,
+				work_event.event_data, work_event.event);
+		} else {
+			DHD_ERROR(("%s: event handler is null\n",
+				__FUNCTION__));
+			dhd_deferred_dump_work_event(&work_event);
+			ASSERT(work_event.event_handler != NULL);
+		}
+	} while (1);
+
+	return;
+}
+
+void
+dhd_deferred_work_set_skip(void *work, u8 event, bool set)
+{
+	struct dhd_deferred_wq *deferred_wq = (struct dhd_deferred_wq *)work;
+
+	if (!deferred_wq || !event || (event >= DHD_MAX_WQ_EVENTS)) {
+		DHD_ERROR(("%s: Invalid!!\n", __FUNCTION__));
+		return;
+	}
+
+	if (set) {
+		/* Set */
+		deferred_wq->event_skip_mask |= (1 << event);
+	} else {
+		/* Clear */
+		deferred_wq->event_skip_mask &= ~(1 << event);
+	}
+}
diff --git a/drivers/net/wireless/ap12275/dhd_linux_wq.h b/drivers/net/wireless/ap12275/dhd_linux_wq.h
new file mode 100644
index 000000000000..2c097e6c5753
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_linux_wq.h
@@ -0,0 +1,88 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), Generic work queue framework
+ * Generic interface to handle dhd deferred work events
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#ifndef _dhd_linux_wq_h_
+#define _dhd_linux_wq_h_
+/*
+ *	Work event definitions
+ */
+enum _wq_event {
+	DHD_WQ_WORK_IF_ADD = 1,
+	DHD_WQ_WORK_IF_DEL,
+	DHD_WQ_WORK_SET_MAC,
+	DHD_WQ_WORK_SET_MCAST_LIST,
+	DHD_WQ_WORK_IPV6_NDO,
+	DHD_WQ_WORK_HANG_MSG,
+	DHD_WQ_WORK_DHD_LOG_DUMP,
+	DHD_WQ_WORK_PKTLOG_DUMP,
+	DHD_WQ_WORK_INFORM_DHD_MON,
+	DHD_WQ_WORK_EVENT_LOGTRACE,
+	DHD_WQ_WORK_DMA_LB_MEM_REL,
+	DHD_WQ_WORK_NATOE_EVENT,
+	DHD_WQ_WORK_NATOE_IOCTL,
+	DHD_WQ_WORK_MACDBG,
+	DHD_WQ_WORK_DEBUG_UART_DUMP,
+	DHD_WQ_WORK_GET_BIGDATA_AP,
+	DHD_WQ_WORK_SOC_RAM_DUMP,
+	DHD_WQ_WORK_SOC_RAM_COLLECT,
+	DHD_WQ_WORK_H2D_CONSOLE_TIME_STAMP_MATCH,
+	DHD_WQ_WORK_AXI_ERROR_DUMP,
+	DHD_WQ_WORK_CTO_RECOVERY,
+#ifdef DHD_UPDATE_INTF_MAC
+	DHD_WQ_WORK_IF_UPDATE,
+#endif /* DHD_UPDATE_INTF_MAC */
+	DHD_WQ_WORK_AML_IOVAR,
+	DHD_WQ_WORK_NDEV_UPD_FEATURES,
+	DHD_MAX_WQ_EVENTS
+};
+
+/*
+ *	Work event priority
+ */
+enum wq_priority {
+	DHD_WQ_WORK_PRIORITY_LOW = 1,
+	DHD_WQ_WORK_PRIORITY_HIGH,
+	DHD_WQ_MAX_PRIORITY
+};
+
+/*
+ *	Error definitions
+ */
+#define DHD_WQ_STS_OK			 0
+#define DHD_WQ_STS_FAILED		-1	/* General failure */
+#define DHD_WQ_STS_UNINITIALIZED	-2
+#define DHD_WQ_STS_SCHED_FAILED		-3
+#define DHD_WQ_STS_UNKNOWN_EVENT	-4
+#define DHD_WQ_STS_UNKNOWN_PRIORITY	-5
+#define DHD_WQ_STS_EVENT_SKIPPED	-6
+
+typedef void (*event_handler_t)(void *handle, void *event_data, u8 event);
+
+void *dhd_deferred_work_init(void *dhd);
+void dhd_deferred_work_deinit(void *workq);
+int dhd_deferred_schedule_work(void *workq, void *event_data, u8 event,
+	event_handler_t evt_handler, u8 priority);
+void dhd_deferred_work_set_skip(void *work, u8 event, bool set);
+#endif /* _dhd_linux_wq_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_log_dump.c b/drivers/net/wireless/ap12275/dhd_log_dump.c
new file mode 100644
index 000000000000..cb8fd58838df
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_log_dump.c
@@ -0,0 +1,2480 @@
+/*
+ * log_dump - debugability support for dumping logs to file
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+#ifdef DHD_LOG_DUMP
+
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmstdlib_s.h>
+#include <dngl_stats.h>
+#include <dhd_linux_priv.h>
+#include <dhd_linux_wq.h>
+#include <dhd.h>
+#include <dhd_proto.h>
+#include <dhd_log_dump.h>
+#ifdef DHD_EVENT_LOG_FILTER
+#include <dhd_event_log_filter.h>
+#endif /* DHD_EVENT_LOG_FILTER */
+#ifdef DHD_PKT_LOGGING
+#include <dhd_pktlog.h>
+#endif /* DHD_PKT_LOGGING */
+#if defined(WL_CFG80211)
+#include <wl_cfg80211.h>
+#endif
+
+extern char dhd_version[];
+extern char fw_version[];
+struct dhd_log_dump_buf g_dld_buf[DLD_BUFFER_NUM];
+
+/* Only header for log dump buffers is stored in array
+ * header for sections like 'dhd dump', 'ext trap'
+ * etc, is not in the array, because they are not log
+ * ring buffers
+ */
+dld_hdr_t dld_hdrs[DLD_BUFFER_NUM] = {
+		{GENERAL_LOG_HDR, LOG_DUMP_SECTION_GENERAL},
+		{PRESERVE_LOG_HDR, LOG_DUMP_SECTION_PRESERVE},
+		{SPECIAL_LOG_HDR, LOG_DUMP_SECTION_SPECIAL}
+};
+static int dld_buf_size[DLD_BUFFER_NUM] = {
+		LOG_DUMP_GENERAL_MAX_BUFSIZE,	/* DLD_BUF_TYPE_GENERAL */
+		LOG_DUMP_PRESERVE_MAX_BUFSIZE,	/* DLD_BUF_TYPE_PRESERVE */
+		LOG_DUMP_SPECIAL_MAX_BUFSIZE,	/* DLD_BUF_TYPE_SPECIAL */
+};
+
+int logdump_max_filesize = LOG_DUMP_MAX_FILESIZE;
+module_param(logdump_max_filesize, int, 0644);
+int logdump_max_bufsize = LOG_DUMP_GENERAL_MAX_BUFSIZE;
+module_param(logdump_max_bufsize, int, 0644);
+int logdump_periodic_flush = FALSE;
+module_param(logdump_periodic_flush, int, 0644);
+#ifdef EWP_ECNTRS_LOGGING
+int logdump_ecntr_enable = TRUE;
+#else
+int logdump_ecntr_enable = FALSE;
+#endif /* EWP_ECNTRS_LOGGING */
+module_param(logdump_ecntr_enable, int, 0644);
+#ifdef EWP_RTT_LOGGING
+int logdump_rtt_enable = TRUE;
+#else
+int logdump_rtt_enable = FALSE;
+#endif /* EWP_RTT_LOGGING */
+
+int logdump_prsrv_tailsize = DHD_LOG_DUMP_MAX_TAIL_FLUSH_SIZE;
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+static dhd_debug_dump_ring_entry_t dhd_debug_dump_ring_map[] = {
+	{LOG_DUMP_SECTION_TIMESTAMP, DEBUG_DUMP_RING1_ID},
+	{LOG_DUMP_SECTION_ECNTRS, DEBUG_DUMP_RING2_ID},
+	{LOG_DUMP_SECTION_STATUS, DEBUG_DUMP_RING1_ID},
+	{LOG_DUMP_SECTION_RTT, DEBUG_DUMP_RING2_ID},
+	{LOG_DUMP_SECTION_PKTID_MAP_LOG, DEBUG_DUMP_RING2_ID},
+	{LOG_DUMP_SECTION_PKTID_UNMAP_LOG, DEBUG_DUMP_RING2_ID},
+	{LOG_DUMP_SECTION_DHD_DUMP, DEBUG_DUMP_RING1_ID},
+	{LOG_DUMP_SECTION_EXT_TRAP, DEBUG_DUMP_RING1_ID},
+	{LOG_DUMP_SECTION_HEALTH_CHK, DEBUG_DUMP_RING1_ID},
+	{LOG_DUMP_SECTION_COOKIE, DEBUG_DUMP_RING1_ID},
+	{LOG_DUMP_SECTION_RING, DEBUG_DUMP_RING1_ID},
+};
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+
+int
+dhd_log_flush(dhd_pub_t *dhdp, log_dump_type_t *type)
+{
+	unsigned long flags = 0;
+#ifdef EWP_EDL
+	int i = 0;
+#endif /* EWP_EDL */
+	dhd_info_t *dhd_info = NULL;
+
+	BCM_REFERENCE(dhd_info);
+
+	/* if dhdp is null, its extremely unlikely that log dump will be scheduled
+	 * so not freeing 'type' here is ok, even if we want to free 'type'
+	 * we cannot do so, since 'dhdp->osh' is unavailable
+	 * as dhdp is null
+	 */
+	if (!dhdp || !type) {
+		if (dhdp) {
+			DHD_GENERAL_LOCK(dhdp, flags);
+			DHD_BUS_BUSY_CLEAR_IN_LOGDUMP(dhdp);
+			dhd_os_busbusy_wake(dhdp);
+			DHD_GENERAL_UNLOCK(dhdp, flags);
+		}
+		return BCME_ERROR;
+	}
+
+#if defined(BCMPCIE)
+	if (dhd_bus_get_linkdown(dhdp)) {
+		/* As link is down donot collect any data over PCIe.
+		 * Also return BCME_OK to caller, so that caller can
+		 * dump all the outstanding data to file
+		 */
+		return BCME_OK;
+	}
+#endif /* BCMPCIE */
+
+	dhd_info = (dhd_info_t *)dhdp->info;
+	/* in case of trap get preserve logs from ETD */
+#if defined(BCMPCIE) && defined(EWP_ETD_PRSRV_LOGS)
+	if (dhdp->dongle_trap_occured &&
+			dhdp->extended_trap_data) {
+		dhdpcie_get_etd_preserve_logs(dhdp, (uint8 *)dhdp->extended_trap_data,
+				&dhd_info->event_data);
+	}
+#endif /* BCMPCIE */
+
+	/* flush the event work items to get any fw events/logs
+	 * flush_work is a blocking call
+	 */
+#ifdef SHOW_LOGTRACE
+#ifdef EWP_EDL
+	if (dhd_info->pub.dongle_edl_support) {
+		/* wait till existing edl items are processed */
+		dhd_flush_logtrace_process(dhd_info);
+		/* dhd_flush_logtrace_process will ensure the work items in the ring
+		* (EDL ring) from rd to wr are processed. But if wr had
+		* wrapped around, only the work items from rd to ring-end are processed.
+		* So to ensure that the work items at the
+		* beginning of ring are also processed in the wrap around case, call
+		* it twice
+		*/
+		for (i = 0; i < 2; i++) {
+			/* blocks till the edl items are processed */
+			dhd_flush_logtrace_process(dhd_info);
+		}
+	} else {
+		dhd_flush_logtrace_process(dhd_info);
+	}
+#else
+	dhd_flush_logtrace_process(dhd_info);
+#endif /* EWP_EDL */
+#endif /* SHOW_LOGTRACE */
+
+	return BCME_OK;
+}
+
+void
+dhd_log_dump(void *handle, void *event_info, u8 event)
+{
+	dhd_info_t *dhd = handle;
+	log_dump_type_t *type = (log_dump_type_t *)event_info;
+
+	if (!dhd || !type) {
+		DHD_ERROR(("%s: dhd/type is NULL\n", __FUNCTION__));
+		return;
+	}
+
+#ifdef WL_CFG80211
+	/* flush the fw preserve logs */
+	wl_flush_fw_log_buffer(dhd_linux_get_primary_netdev(&dhd->pub),
+		FW_LOGSET_MASK_ALL);
+#endif
+
+	/* there are currently 3 possible contexts from which
+	 * log dump can be scheduled -
+	 * 1.TRAP 2.supplicant DEBUG_DUMP pvt driver command
+	 * 3.HEALTH CHECK event
+	 * The concise debug info buffer is a shared resource
+	 * and in case a trap is one of the contexts then both the
+	 * scheduled work queues need to run because trap data is
+	 * essential for debugging. Hence a mutex lock is acquired
+	 * before calling do_dhd_log_dump().
+	 */
+	DHD_ERROR(("%s: calling log dump.. \n", __FUNCTION__));
+	dhd_os_logdump_lock(&dhd->pub);
+	DHD_OS_WAKE_LOCK(&dhd->pub);
+	if (do_dhd_log_dump(&dhd->pub, type) != BCME_OK) {
+		DHD_ERROR(("%s: writing debug dump to the file failed\n", __FUNCTION__));
+	}
+	DHD_OS_WAKE_UNLOCK(&dhd->pub);
+	dhd_os_logdump_unlock(&dhd->pub);
+}
+
+void dhd_schedule_log_dump(dhd_pub_t *dhdp, void *type)
+{
+	DHD_ERROR(("%s: scheduling log dump.. \n", __FUNCTION__));
+
+	dhd_deferred_schedule_work(dhdp->info->dhd_deferred_wq,
+		type, DHD_WQ_WORK_DHD_LOG_DUMP,
+		dhd_log_dump, DHD_WQ_WORK_PRIORITY_HIGH);
+}
+
+void
+dhd_print_buf_addr(dhd_pub_t *dhdp, char *name, void *buf, unsigned int size)
+{
+#ifdef DHD_FW_COREDUMP
+	if ((dhdp->memdump_enabled == DUMP_MEMONLY) ||
+		(dhdp->memdump_enabled == DUMP_MEMFILE_BUGON) ||
+		(dhdp->memdump_type == DUMP_TYPE_SMMU_FAULT) ||
+#ifdef DHD_DETECT_CONSECUTIVE_MFG_HANG
+		(dhdp->op_mode & DHD_FLAG_MFG_MODE &&
+			(dhdp->hang_count >= MAX_CONSECUTIVE_MFG_HANG_COUNT-1)) ||
+#endif /* DHD_DETECT_CONSECUTIVE_MFG_HANG */
+		FALSE)
+#else
+	if (dhdp->memdump_type == DUMP_TYPE_SMMU_FAULT)
+#endif
+	{
+#if defined(CONFIG_ARM64)
+		DHD_ERROR(("-------- %s: buf(va)=%llx, buf(pa)=%llx, bufsize=%d\n",
+			name, (uint64)buf, (uint64)__virt_to_phys((ulong)buf), size));
+#elif defined(__ARM_ARCH_7A__)
+		DHD_ERROR(("-------- %s: buf(va)=%x, buf(pa)=%x, bufsize=%d\n",
+			name, (uint32)buf, (uint32)__virt_to_phys((ulong)buf), size));
+#endif /* __ARM_ARCH_7A__ */
+	}
+}
+
+void
+dhd_log_dump_buf_addr(dhd_pub_t *dhdp, log_dump_type_t *type)
+{
+	int i;
+	unsigned long wr_size = 0;
+	struct dhd_log_dump_buf *dld_buf = &g_dld_buf[0];
+	size_t log_size = 0;
+	char buf_name[DHD_PRINT_BUF_NAME_LEN];
+	dhd_dbg_ring_t *ring = NULL;
+
+	BCM_REFERENCE(ring);
+
+	for (i = 0; i < DLD_BUFFER_NUM; i++) {
+		dld_buf = &g_dld_buf[i];
+		log_size = (unsigned long)dld_buf->max -
+			(unsigned long)dld_buf->buffer;
+		if (dld_buf->wraparound) {
+			wr_size = log_size;
+		} else {
+			wr_size = (unsigned long)dld_buf->present -
+				(unsigned long)dld_buf->front;
+		}
+		scnprintf(buf_name, sizeof(buf_name), "dlb_buf[%d]", i);
+		dhd_print_buf_addr(dhdp, buf_name, dld_buf, dld_buf_size[i]);
+		scnprintf(buf_name, sizeof(buf_name), "dlb_buf[%d] buffer", i);
+		dhd_print_buf_addr(dhdp, buf_name, dld_buf->buffer, wr_size);
+		scnprintf(buf_name, sizeof(buf_name), "dlb_buf[%d] present", i);
+		dhd_print_buf_addr(dhdp, buf_name, dld_buf->present, wr_size);
+		scnprintf(buf_name, sizeof(buf_name), "dlb_buf[%d] front", i);
+		dhd_print_buf_addr(dhdp, buf_name, dld_buf->front, wr_size);
+	}
+
+#ifdef DEBUGABILITY_ECNTRS_LOGGING
+	/* periodic flushing of ecounters is NOT supported */
+	if (*type == DLD_BUF_TYPE_ALL &&
+			logdump_ecntr_enable &&
+			dhdp->ecntr_dbg_ring) {
+
+		ring = (dhd_dbg_ring_t *)dhdp->ecntr_dbg_ring;
+		dhd_print_buf_addr(dhdp, "ecntr_dbg_ring", ring, LOG_DUMP_ECNTRS_MAX_BUFSIZE);
+		dhd_print_buf_addr(dhdp, "ecntr_dbg_ring ring_buf", ring->ring_buf,
+				LOG_DUMP_ECNTRS_MAX_BUFSIZE);
+	}
+#endif /* DEBUGABILITY_ECNTRS_LOGGING */
+
+#if defined(BCMPCIE)
+	if (dhdp->dongle_trap_occured && dhdp->extended_trap_data) {
+		dhd_print_buf_addr(dhdp, "extended_trap_data", dhdp->extended_trap_data,
+				BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+	}
+#endif /* BCMPCIE */
+
+#if defined(DHD_FW_COREDUMP) && defined(DNGL_EVENT_SUPPORT)
+	/* if health check event was received */
+	if (dhdp->memdump_type == DUMP_TYPE_DONGLE_HOST_EVENT) {
+		dhd_print_buf_addr(dhdp, "health_chk_event_data", dhdp->health_chk_event_data,
+				HEALTH_CHK_BUF_SIZE);
+	}
+#endif /* DHD_FW_COREDUMP && DNGL_EVENT_SUPPORT */
+
+	/* append the concise debug information */
+	if (dhdp->concise_dbg_buf) {
+		dhd_print_buf_addr(dhdp, "concise_dbg_buf", dhdp->concise_dbg_buf,
+				CONCISE_DUMP_BUFLEN);
+	}
+}
+
+#ifdef DHD_SSSR_DUMP
+#ifdef DHD_COREDUMP
+extern dhd_coredump_t dhd_coredump_types[];
+#endif /* DHD_COREDUMP */
+int
+dhdpcie_sssr_dump_get_before_after_len(dhd_pub_t *dhd, uint32 *arr_len)
+{
+	int i = 0;
+	uint dig_buf_size = 0;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* core 0 */
+	i = 0;
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	if (dhd->sssr_d11_before[i] && dhd->sssr_d11_outofreset[i] &&
+		(dhd->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_BEFORE].length =
+#endif /* DHD_COREDUMP */
+			arr_len[SSSR_C0_D11_BEFORE] = dhd_sssr_mac_buf_size(dhd, i);
+		DHD_ERROR(("%s: arr_len[SSSR_C0_D11_BEFORE] : %d\n", __FUNCTION__,
+			arr_len[SSSR_C0_D11_BEFORE]));
+#ifdef DHD_LOG_DUMP
+		dhd_print_buf_addr(dhd, "SSSR_C0_D11_BEFORE",
+			dhd->sssr_d11_before[i], arr_len[SSSR_C0_D11_BEFORE]);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_BEFORE].bufptr =
+			dhd->sssr_d11_before[i];
+#endif /* DHD_COREDUMP */
+	}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	if (dhd->sssr_d11_after[i] && dhd->sssr_d11_outofreset[i]) {
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_AFTER].length =
+#endif /* DHD_COREDUMP */
+			arr_len[SSSR_C0_D11_AFTER] = dhd_sssr_mac_buf_size(dhd, i);
+		DHD_ERROR(("%s: arr_len[SSSR_C0_D11_AFTER] : %d\n", __FUNCTION__,
+			arr_len[SSSR_C0_D11_AFTER]));
+#ifdef DHD_LOG_DUMP
+		dhd_print_buf_addr(dhd, "SSSR_C0_D11_AFTER",
+			dhd->sssr_d11_after[i], arr_len[SSSR_C0_D11_AFTER]);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE0_AFTER].bufptr =
+			dhd->sssr_d11_after[i];
+#endif /* DHD_COREDUMP */
+	}
+
+	/* core 1 */
+	i = 1;
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	if (dhd->sssr_d11_before[i] && dhd->sssr_d11_outofreset[i] &&
+		(dhd->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_BEFORE].length =
+#endif /* DHD_COREDUMP */
+			arr_len[SSSR_C1_D11_BEFORE] = dhd_sssr_mac_buf_size(dhd, i);
+		DHD_ERROR(("%s: arr_len[SSSR_C1_D11_BEFORE] : %d\n", __FUNCTION__,
+			arr_len[SSSR_C1_D11_BEFORE]));
+#ifdef DHD_LOG_DUMP
+		dhd_print_buf_addr(dhd, "SSSR_C1_D11_BEFORE",
+			dhd->sssr_d11_before[i], arr_len[SSSR_C1_D11_BEFORE]);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_BEFORE].bufptr =
+			dhd->sssr_d11_before[i];
+#endif /* DHD_COREDUMP */
+	}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+	if (dhd->sssr_d11_after[i] && dhd->sssr_d11_outofreset[i]) {
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_AFTER].length =
+#endif /* DHD_COREDUMP */
+			arr_len[SSSR_C1_D11_AFTER] = dhd_sssr_mac_buf_size(dhd, i);
+		DHD_ERROR(("%s: arr_len[SSSR_C1_D11_AFTER] : %d\n", __FUNCTION__,
+			arr_len[SSSR_C1_D11_AFTER]));
+#ifdef DHD_LOG_DUMP
+		dhd_print_buf_addr(dhd, "SSSR_C1_D11_AFTER",
+			dhd->sssr_d11_after[i], arr_len[SSSR_C1_D11_AFTER]);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE1_AFTER].bufptr =
+			dhd->sssr_d11_after[i];
+#endif /* DHD_COREDUMP */
+	}
+
+	/* core 2 scan core */
+	if (dhd->sssr_reg_info->rev2.version >= SSSR_REG_INFO_VER_2) {
+		i = 2;
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+		if (dhd->sssr_d11_before[i] && dhd->sssr_d11_outofreset[i] &&
+			(dhd->sssr_dump_mode == SSSR_DUMP_MODE_SSSR)) {
+#ifdef DHD_COREDUMP
+			dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_BEFORE].length =
+#endif /* DHD_COREDUMP */
+				arr_len[SSSR_C2_D11_BEFORE] = dhd_sssr_mac_buf_size(dhd, i);
+			DHD_ERROR(("%s: arr_len[SSSR_C2_D11_BEFORE] : %d\n", __FUNCTION__,
+				arr_len[SSSR_C2_D11_BEFORE]));
+#ifdef DHD_LOG_DUMP
+			dhd_print_buf_addr(dhd, "SSSR_C2_D11_BEFORE",
+				dhd->sssr_d11_before[i], arr_len[SSSR_C2_D11_BEFORE]);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+		dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_BEFORE].bufptr =
+			dhd->sssr_d11_before[i];
+#endif /* DHD_COREDUMP */
+		}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+		if (dhd->sssr_d11_after[i] && dhd->sssr_d11_outofreset[i]) {
+#ifdef DHD_COREDUMP
+			dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_AFTER].length =
+#endif /* DHD_COREDUMP */
+				arr_len[SSSR_C2_D11_AFTER]  = dhd_sssr_mac_buf_size(dhd, i);
+			DHD_ERROR(("%s: arr_len[SSSR_C2_D11_AFTER] : %d\n", __FUNCTION__,
+				arr_len[SSSR_C2_D11_AFTER]));
+#ifdef DHD_LOG_DUMP
+			dhd_print_buf_addr(dhd, "SSSR_C2_D11_AFTER",
+				dhd->sssr_d11_after[i], arr_len[SSSR_C2_D11_AFTER]);
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+			dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_CORE2_AFTER].bufptr =
+				dhd->sssr_d11_after[i];
+#endif /* DHD_COREDUMP */
+		}
+	}
+
+	/* DIG core or VASIP */
+	dig_buf_size = dhd_sssr_dig_buf_size(dhd);
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+#ifdef DHD_COREDUMP
+	dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_DIG_BEFORE].length =
+#endif /* DHD_COREDUMP */
+		arr_len[SSSR_DIG_BEFORE] = (dhd->sssr_dig_buf_before) ? dig_buf_size : 0;
+	DHD_ERROR(("%s: arr_len[SSSR_DIG_BEFORE] : %d\n", __FUNCTION__,
+		arr_len[SSSR_DIG_BEFORE]));
+#ifdef DHD_LOG_DUMP
+	if (dhd->sssr_dig_buf_before) {
+		dhd_print_buf_addr(dhd, "SSSR_DIG_BEFORE",
+			dhd->sssr_dig_buf_before, arr_len[SSSR_DIG_BEFORE]);
+	}
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+	dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_DIG_BEFORE].bufptr =
+		dhd->sssr_dig_buf_before;
+#endif /* DHD_COREDUMP */
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+#ifdef DHD_COREDUMP
+	dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_DIG_AFTER].length =
+#endif /* DHD_COREDUMP */
+		arr_len[SSSR_DIG_AFTER] = (dhd->sssr_dig_buf_after) ? dig_buf_size : 0;
+	DHD_ERROR(("%s: arr_len[SSSR_DIG_AFTER] : %d\n", __FUNCTION__,
+		arr_len[SSSR_DIG_AFTER]));
+#ifdef DHD_LOG_DUMP
+	if (dhd->sssr_dig_buf_after) {
+		dhd_print_buf_addr(dhd, "SSSR_DIG_AFTER",
+			dhd->sssr_dig_buf_after, arr_len[SSSR_DIG_AFTER]);
+	}
+#endif /* DHD_LOG_DUMP */
+#ifdef DHD_COREDUMP
+	dhd_coredump_types[DHD_COREDUMP_TYPE_SSSRDUMP_DIG_AFTER].bufptr =
+		dhd->sssr_dig_buf_after;
+#endif /* DHD_COREDUMP */
+
+	return BCME_OK;
+}
+
+void
+dhd_nla_put_sssr_dump_len(void *ndev, uint32 *arr_len)
+{
+	dhd_info_t *dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+	dhd_pub_t *dhdp = &dhd_info->pub;
+
+	if (dhdp->sssr_dump_collected) {
+		dhdpcie_sssr_dump_get_before_after_len(dhdp, arr_len);
+	}
+}
+#endif /* DHD_SSSR_DUMP */
+
+uint32
+dhd_get_time_str_len(void)
+{
+	char *ts = NULL, time_str[128];
+
+	ts = dhd_log_dump_get_timestamp();
+	snprintf(time_str, sizeof(time_str),
+			"\n\n ========== LOG DUMP TAKEN AT : %s =========\n", ts);
+	return strlen(time_str);
+}
+
+#if defined(BCMPCIE)
+uint32
+dhd_get_ext_trap_len(void *ndev, dhd_pub_t *dhdp)
+{
+	int length = 0;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	if (dhdp->extended_trap_data) {
+		length = (strlen(EXT_TRAP_LOG_HDR)
+					+ sizeof(sec_hdr) + BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+	}
+	return length;
+}
+#endif /* BCMPCIE */
+
+#if defined(DHD_FW_COREDUMP) && defined(DNGL_EVENT_SUPPORT)
+uint32
+dhd_get_health_chk_len(void *ndev, dhd_pub_t *dhdp)
+{
+	int length = 0;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	if (dhdp->memdump_type == DUMP_TYPE_DONGLE_HOST_EVENT) {
+		length = (strlen(HEALTH_CHK_LOG_HDR)
+			+ sizeof(sec_hdr) + HEALTH_CHK_BUF_SIZE);
+	}
+	return length;
+}
+#endif /* DHD_FW_COREDUMP && DNGL_EVENT_SUPPORT */
+
+uint32
+dhd_get_dhd_dump_len(void *ndev, dhd_pub_t *dhdp)
+{
+	uint32 length = 0;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+	int remain_len = 0;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	if (dhdp->concise_dbg_buf) {
+		remain_len = dhd_dump(dhdp, (char *)dhdp->concise_dbg_buf, CONCISE_DUMP_BUFLEN);
+		 if (remain_len <= 0 || remain_len >= CONCISE_DUMP_BUFLEN) {
+			DHD_ERROR(("%s: error getting concise debug info ! remain_len: %d\n",
+				__FUNCTION__, remain_len));
+			return length;
+		}
+
+		length += (uint32)(CONCISE_DUMP_BUFLEN - remain_len);
+	}
+
+	length += (uint32)(strlen(DHD_DUMP_LOG_HDR) + sizeof(sec_hdr));
+	return length;
+}
+
+#ifdef EWP_RTT_LOGGING
+uint32
+dhd_get_rtt_len(void *ndev, dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd_info;
+	log_dump_section_hdr_t sec_hdr;
+	int length = 0;
+	dhd_dbg_ring_t *ring;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	if (logdump_rtt_enable && dhdp->rtt_dbg_ring) {
+		ring = (dhd_dbg_ring_t *)dhdp->rtt_dbg_ring;
+		length = ring->ring_size + strlen(RTT_LOG_HDR) + sizeof(sec_hdr);
+	}
+	return length;
+}
+#endif /* EWP_RTT_LOGGING */
+
+uint32
+dhd_get_cookie_log_len(void *ndev, dhd_pub_t *dhdp)
+{
+	int length = 0;
+	dhd_info_t *dhd_info;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	if (dhdp->logdump_cookie && dhd_logdump_cookie_count(dhdp) > 0) {
+		length = dhd_log_dump_cookie_len(dhdp);
+	}
+	return length;
+
+}
+
+#ifdef DHD_DUMP_PCIE_RINGS
+uint32
+dhd_get_flowring_len(void *ndev, dhd_pub_t *dhdp)
+{
+	uint32 length = 0;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+	uint16 max_tx_flowrings;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	length += (uint32) strlen(RING_DUMP_HDR);
+	length += (uint32) sizeof(sec_hdr);
+	length += (uint32) sizeof(max_tx_flowrings);
+	max_tx_flowrings = dhd_get_max_flow_rings(dhdp);
+	/* max_item and item_size value which is of 4bytes is dumped at
+	 * start of each ring dump, so adding 4bytes to total length.
+	 */
+	length += (uint32) ((D2HRING_TXCMPLT_ITEMSIZE * d2h_max_txcpl)
+				+ (sizeof(uint16) * 2)
+				+ (H2DRING_RXPOST_ITEMSIZE * h2d_max_rxpost)
+				+ (sizeof(uint16) * 2)
+				+ (D2HRING_RXCMPLT_ITEMSIZE * d2h_max_rxcpl)
+				+ (sizeof(uint16) * 2)
+				+ (H2DRING_CTRL_SUB_ITEMSIZE * h2d_max_ctrlpost)
+				+ (sizeof(uint16) * 2)
+				+ (D2HRING_CTRL_CMPLT_ITEMSIZE * d2h_max_ctrlcpl)
+				+ (sizeof(uint16) * 2)
+#ifdef EWP_EDL
+				/* EDL ring doesn't have max_item and item_size */
+				+ (D2HRING_EDL_HDR_SIZE * D2HRING_EDL_MAX_ITEM));
+#else
+				+ (H2DRING_INFO_BUFPOST_ITEMSIZE * H2DRING_DYNAMIC_INFO_MAX_ITEM)
+				+ (sizeof(uint16) * 2)
+				+ (D2HRING_INFO_BUFCMPLT_ITEMSIZE * D2HRING_DYNAMIC_INFO_MAX_ITEM)
+				+ (sizeof(uint16) * 2));
+#endif /* EWP_EDL */
+
+	if (dhdp->htput_support) {
+		/* flowring lengths are different for HTPUT rings, handle accordingly */
+		length += ((H2DRING_TXPOST_ITEMSIZE * h2d_htput_max_txpost *
+			HTPUT_TOTAL_FLOW_RINGS) +
+			(H2DRING_TXPOST_ITEMSIZE * h2d_max_txpost *
+			(max_tx_flowrings - HTPUT_TOTAL_FLOW_RINGS)));
+	} else {
+		length += (H2DRING_TXPOST_ITEMSIZE * h2d_max_txpost *
+			max_tx_flowrings);
+	}
+	length += max_tx_flowrings * (sizeof(uint16) * 2);
+
+	return length;
+}
+#endif /* DHD_DUMP_PCIE_RINGS */
+
+#ifdef EWP_ECNTRS_LOGGING
+uint32
+dhd_get_ecntrs_len(void *ndev, dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd_info;
+	log_dump_section_hdr_t sec_hdr;
+	int length = 0;
+	dhd_dbg_ring_t *ring;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return length;
+
+	if (logdump_ecntr_enable && dhdp->ecntr_dbg_ring) {
+		ring = (dhd_dbg_ring_t *)dhdp->ecntr_dbg_ring;
+		length = ring->ring_size + strlen(ECNTRS_LOG_HDR) + sizeof(sec_hdr);
+	}
+	return length;
+}
+#endif /* EWP_ECNTRS_LOGGING */
+
+int
+dhd_get_dld_log_dump(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, int type, void *pos)
+{
+	int ret = BCME_OK;
+	struct dhd_log_dump_buf *dld_buf;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+
+	dld_buf = &g_dld_buf[type];
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	} else if (!dhdp) {
+		return BCME_ERROR;
+	}
+
+	DHD_ERROR(("%s: ENTER \n", __FUNCTION__));
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	/* write the section header first */
+	ret = dhd_export_debug_data(dld_hdrs[type].hdr_str, fp, user_buf,
+		strlen(dld_hdrs[type].hdr_str), pos);
+	if (ret < 0)
+		goto exit;
+	len -= (uint32)strlen(dld_hdrs[type].hdr_str);
+	len -= (uint32)sizeof(sec_hdr);
+	sec_hdr.type = dld_hdrs[type].sec_type;
+	sec_hdr.length = len;
+	ret = dhd_export_debug_data((char *)&sec_hdr, fp, user_buf, sizeof(sec_hdr), pos);
+	if (ret < 0)
+		goto exit;
+	ret = dhd_export_debug_data(dld_buf->buffer, fp, user_buf, len, pos);
+	if (ret < 0)
+		goto exit;
+
+exit:
+	return ret;
+}
+
+int
+dhd_get_debug_dump_file_name(void *dev, dhd_pub_t *dhdp, char *dump_path, int size)
+{
+	int ret;
+	int len = 0;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	memset(dump_path, 0, size);
+
+	ret = snprintf(dump_path, size, "%s",
+			DHD_COMMON_DUMP_PATH DHD_DEBUG_DUMP_TYPE);
+	len += ret;
+
+	/* Keep the same timestamp across different dump logs */
+	if (!dhdp->logdump_periodic_flush) {
+		struct rtc_time tm;
+#ifdef DHD_LOG_DUMP
+		clear_debug_dump_time(dhdp->debug_dump_time_str);
+		get_debug_dump_time(dhdp->debug_dump_time_str);
+#endif /* DHD_LOG_DUMP */
+		sscanf(dhdp->debug_dump_time_str, DHD_LOG_DUMP_TS_FMT_YYMMDDHHMMSS,
+			&tm.tm_year, &tm.tm_mon, &tm.tm_mday,
+			&tm.tm_hour, &tm.tm_min, &tm.tm_sec);
+		ret = snprintf(dump_path + len, size - len, "_" DHD_LOG_DUMP_TS_FMT_YYMMDDHHMMSS,
+				tm.tm_year, tm.tm_mon, tm.tm_mday,
+				tm.tm_hour, tm.tm_min, tm.tm_sec);
+		len += ret;
+	}
+
+	ret = 0;
+	switch (dhdp->debug_dump_subcmd) {
+	case CMD_UNWANTED:
+		ret = snprintf(dump_path + len, size - len, "%s", DHD_DUMP_SUBSTR_UNWANTED);
+		break;
+	case CMD_DISCONNECTED:
+		ret = snprintf(dump_path + len, size - len, "%s", DHD_DUMP_SUBSTR_DISCONNECTED);
+		break;
+	default:
+		break;
+	}
+	len += ret;
+
+	return BCME_OK;
+}
+
+uint32
+dhd_get_dld_len(int log_type)
+{
+	unsigned long wr_size = 0;
+	unsigned long buf_size = 0;
+	unsigned long flags = 0;
+	struct dhd_log_dump_buf *dld_buf;
+	log_dump_section_hdr_t sec_hdr;
+
+	/* calculate the length of the log */
+	dld_buf = &g_dld_buf[log_type];
+	buf_size = (unsigned long)dld_buf->max -
+			(unsigned long)dld_buf->buffer;
+
+	if (dld_buf->wraparound) {
+		wr_size = buf_size;
+	} else {
+		/* need to hold the lock before accessing 'present' and 'remain' ptrs */
+		DHD_LOG_DUMP_BUF_LOCK(&dld_buf->lock, flags);
+		wr_size = (unsigned long)dld_buf->present -
+				(unsigned long)dld_buf->front;
+		DHD_LOG_DUMP_BUF_UNLOCK(&dld_buf->lock, flags);
+	}
+	return (wr_size + sizeof(sec_hdr) + strlen(dld_hdrs[log_type].hdr_str));
+}
+
+static void
+dhd_get_time_str(dhd_pub_t *dhdp, char *time_str, int size)
+{
+	char *ts = NULL;
+	memset(time_str, 0, size);
+	ts = dhd_log_dump_get_timestamp();
+	snprintf(time_str, size,
+			"\n\n ========== LOG DUMP TAKEN AT : %s =========\n", ts);
+}
+
+int
+dhd_print_time_str(const void *user_buf, void *fp, uint32 len, void *pos)
+{
+	char *ts = NULL;
+	int ret = 0;
+	char time_str[128];
+
+	memset_s(time_str, sizeof(time_str), 0, sizeof(time_str));
+	ts = dhd_log_dump_get_timestamp();
+	snprintf(time_str, sizeof(time_str),
+			"\n\n ========== LOG DUMP TAKEN AT : %s =========\n", ts);
+
+	/* write the timestamp hdr to the file first */
+	ret = dhd_export_debug_data(time_str, fp, user_buf, strlen(time_str), pos);
+	if (ret < 0) {
+		DHD_ERROR(("write file error, err = %d\n", ret));
+	}
+	return ret;
+}
+
+#if defined(DHD_FW_COREDUMP) && defined(DNGL_EVENT_SUPPORT)
+int
+dhd_print_health_chk_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos)
+{
+	int ret = BCME_OK;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	if (dhdp->memdump_type == DUMP_TYPE_DONGLE_HOST_EVENT) {
+		/* write the section header first */
+		ret = dhd_export_debug_data(HEALTH_CHK_LOG_HDR, fp, user_buf,
+			strlen(HEALTH_CHK_LOG_HDR), pos);
+		if (ret < 0)
+			goto exit;
+
+		len -= (uint32)strlen(HEALTH_CHK_LOG_HDR);
+		sec_hdr.type = LOG_DUMP_SECTION_HEALTH_CHK;
+		sec_hdr.length = HEALTH_CHK_BUF_SIZE;
+		ret = dhd_export_debug_data((char *)&sec_hdr, fp, user_buf, sizeof(sec_hdr), pos);
+		if (ret < 0)
+			goto exit;
+
+		len -= (uint32)sizeof(sec_hdr);
+		/* write the log */
+		ret = dhd_export_debug_data((char *)dhdp->health_chk_event_data, fp,
+			user_buf, len, pos);
+		if (ret < 0)
+			goto exit;
+	}
+exit:
+	return ret;
+}
+#endif /* DHD_FW_COREDUMP && DNGL_EVENT_SUPPORT */
+
+#if defined(BCMPCIE)
+int
+dhd_print_ext_trap_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos)
+{
+	int ret = BCME_OK;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	/* append extended trap data to the file in case of traps */
+	if (dhdp->dongle_trap_occured &&
+			dhdp->extended_trap_data) {
+		/* write the section header first */
+		ret = dhd_export_debug_data(EXT_TRAP_LOG_HDR, fp, user_buf,
+			strlen(EXT_TRAP_LOG_HDR), pos);
+		if (ret < 0)
+			goto exit;
+
+		len -= (uint32)strlen(EXT_TRAP_LOG_HDR);
+		sec_hdr.type = LOG_DUMP_SECTION_EXT_TRAP;
+		sec_hdr.length = BCMPCIE_EXT_TRAP_DATA_MAXLEN;
+		ret = dhd_export_debug_data((uint8 *)&sec_hdr, fp, user_buf, sizeof(sec_hdr), pos);
+		if (ret < 0)
+			goto exit;
+
+		len -= (uint32)sizeof(sec_hdr);
+		/* write the log */
+		ret = dhd_export_debug_data((uint8 *)dhdp->extended_trap_data, fp,
+			user_buf, len, pos);
+		if (ret < 0)
+			goto exit;
+	}
+exit:
+	return ret;
+}
+#endif /* BCMPCIE */
+
+int
+dhd_print_dump_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos)
+{
+	int ret = BCME_OK;
+	log_dump_section_hdr_t sec_hdr;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	ret = dhd_export_debug_data(DHD_DUMP_LOG_HDR, fp, user_buf, strlen(DHD_DUMP_LOG_HDR), pos);
+	if (ret < 0)
+		goto exit;
+
+	len -= (uint32)strlen(DHD_DUMP_LOG_HDR);
+	sec_hdr.type = LOG_DUMP_SECTION_DHD_DUMP;
+	sec_hdr.length = len;
+	ret = dhd_export_debug_data((char *)&sec_hdr, fp, user_buf, sizeof(sec_hdr), pos);
+	if (ret < 0)
+		goto exit;
+
+	len -= (uint32)sizeof(sec_hdr);
+
+	if (dhdp->concise_dbg_buf) {
+		dhd_dump(dhdp, (char *)dhdp->concise_dbg_buf, CONCISE_DUMP_BUFLEN);
+		ret = dhd_export_debug_data(dhdp->concise_dbg_buf, fp, user_buf, len, pos);
+		if (ret < 0)
+			goto exit;
+	}
+
+exit:
+	return ret;
+}
+
+int
+dhd_print_cookie_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos)
+{
+	int ret = BCME_OK;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	if (dhdp->logdump_cookie && dhd_logdump_cookie_count(dhdp) > 0) {
+		ret = dhd_log_dump_cookie_to_file(dhdp, fp, user_buf, (unsigned long *)pos);
+	}
+	return ret;
+}
+
+#ifdef DHD_DUMP_PCIE_RINGS
+int
+dhd_print_flowring_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+		void *fp, uint32 len, void *pos)
+{
+	log_dump_section_hdr_t sec_hdr;
+	int ret = BCME_OK;
+	uint16 max_tx_flowrings;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	/* write the section header first */
+	ret = dhd_export_debug_data(RING_DUMP_HDR, fp, user_buf,
+		strlen(RING_DUMP_HDR), pos);
+	if (ret < 0)
+		goto exit;
+	len -= strlen(RING_DUMP_HDR);
+
+	sec_hdr.type = LOG_DUMP_SECTION_RING;
+	sec_hdr.length = len - sizeof(sec_hdr);
+	ret = dhd_export_debug_data((char *)&sec_hdr, fp, user_buf, sizeof(sec_hdr), pos);
+	if (ret < 0)
+		goto exit;
+
+	max_tx_flowrings =  dhd_get_max_flow_rings(dhdp);
+	/* write the number of max_tx_flowrings after section header */
+	ret = dhd_export_debug_data((char *)&max_tx_flowrings, fp, user_buf,
+		sizeof(max_tx_flowrings), pos);
+	if (ret < 0)
+		goto exit;
+
+	/* write the log */
+	ret = dhd_d2h_h2d_ring_dump(dhdp, fp, user_buf, (unsigned long *)pos, TRUE);
+	if (ret < 0)
+		goto exit;
+exit:
+	return ret;
+}
+#endif /* DHD_DUMP_PCIE_RINGS */
+
+#ifdef EWP_ECNTRS_LOGGING
+int
+dhd_print_ecntrs_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+		void *fp, uint32 len, void *pos)
+{
+	log_dump_section_hdr_t sec_hdr;
+	int ret = BCME_OK;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	if (logdump_ecntr_enable &&
+			dhdp->ecntr_dbg_ring) {
+		sec_hdr.type = LOG_DUMP_SECTION_ECNTRS;
+		ret = dhd_dump_debug_ring(dhdp, dhdp->ecntr_dbg_ring,
+				user_buf, &sec_hdr, ECNTRS_LOG_HDR, len, LOG_DUMP_SECTION_ECNTRS);
+	}
+	return ret;
+
+}
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+int
+dhd_print_rtt_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+		void *fp, uint32 len, void *pos)
+{
+	log_dump_section_hdr_t sec_hdr;
+	int ret = BCME_OK;
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp)
+		return BCME_ERROR;
+
+	dhd_init_sec_hdr(&sec_hdr);
+
+	if (logdump_rtt_enable && dhdp->rtt_dbg_ring) {
+		ret = dhd_dump_debug_ring(dhdp, dhdp->rtt_dbg_ring,
+				user_buf, &sec_hdr, RTT_LOG_HDR, len, LOG_DUMP_SECTION_RTT);
+	}
+	return ret;
+
+}
+#endif /* EWP_RTT_LOGGING */
+
+#ifdef DHD_STATUS_LOGGING
+int
+dhd_print_status_log_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos)
+{
+	dhd_info_t *dhd_info;
+
+	if (dev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)dev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (!dhdp) {
+		return BCME_ERROR;
+	}
+
+	return dhd_statlog_write_logdump(dhdp, user_buf, fp, len, pos);
+}
+
+uint32
+dhd_get_status_log_len(void *ndev, dhd_pub_t *dhdp)
+{
+	dhd_info_t *dhd_info;
+	uint32 length = 0;
+
+	if (ndev) {
+		dhd_info = *(dhd_info_t **)netdev_priv((struct net_device *)ndev);
+		dhdp = &dhd_info->pub;
+	}
+
+	if (dhdp) {
+		length = dhd_statlog_get_logbuf_len(dhdp);
+	}
+
+	return length;
+}
+#endif /* DHD_STATUS_LOGGING */
+
+void
+dhd_init_sec_hdr(log_dump_section_hdr_t *sec_hdr)
+{
+	/* prep the section header */
+	memset(sec_hdr, 0, sizeof(*sec_hdr));
+	sec_hdr->magic = LOG_DUMP_MAGIC;
+	sec_hdr->timestamp = local_clock();
+}
+
+/* Must hold 'dhd_os_logdump_lock' before calling this function ! */
+int
+do_dhd_log_dump(dhd_pub_t *dhdp, log_dump_type_t *type)
+{
+	int ret = 0, i = 0;
+	struct file *fp = NULL;
+#ifdef get_fs
+	mm_segment_t old_fs;
+#endif /* get_fs */
+	loff_t pos = 0;
+	char dump_path[128];
+	uint32 file_mode;
+	unsigned long flags = 0;
+	size_t log_size = 0;
+	size_t fspace_remain = 0;
+	struct file *filep = NULL;
+	int logstrs_size = 0;
+	char time_str[128];
+	unsigned int len = 0;
+	log_dump_section_hdr_t sec_hdr;
+
+	DHD_ERROR(("%s: ENTER \n", __FUNCTION__));
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s: bus is down! can't collect log dump. \n", __FUNCTION__));
+		goto exit1;
+	}
+	DHD_BUS_BUSY_SET_IN_LOGDUMP(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	if ((ret = dhd_log_flush(dhdp, type)) < 0) {
+		goto exit1;
+	}
+#ifdef get_fs
+	/* change to KERNEL_DS address limit */
+	old_fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+
+	dhd_get_debug_dump_file_name(NULL, dhdp, dump_path, sizeof(dump_path));
+
+	DHD_ERROR(("debug_dump_path = %s\n", dump_path));
+	DHD_ERROR(("DHD version: %s\n", dhd_version));
+	DHD_ERROR(("F/W version: %s\n", fw_version));
+
+	dhd_log_dump_buf_addr(dhdp, type);
+
+	dhd_get_time_str(dhdp, time_str, 128);
+
+	/* if this is the first time after dhd is loaded,
+	 * or, if periodic flush is disabled, clear the log file
+	 */
+	if (!dhdp->logdump_periodic_flush || dhdp->last_file_posn == 0)
+		file_mode = O_CREAT | O_WRONLY | O_SYNC | O_TRUNC;
+	else
+		file_mode = O_CREAT | O_RDWR | O_SYNC;
+
+	fp = dhd_filp_open(dump_path, file_mode, 0664);
+	if (IS_ERR(fp) || (fp == NULL)) {
+		/* If android installed image, try '/data' directory */
+#if defined(CONFIG_X86) && defined(OEM_ANDROID)
+		DHD_ERROR(("%s: File open error on Installed android image, trying /data...\n",
+			__FUNCTION__));
+		snprintf(dump_path, sizeof(dump_path), "/data/" DHD_DEBUG_DUMP_TYPE);
+		if (!dhdp->logdump_periodic_flush) {
+			snprintf(dump_path + strlen(dump_path),
+				sizeof(dump_path) - strlen(dump_path),
+				"_%s", dhdp->debug_dump_time_str);
+		}
+		fp = dhd_filp_open(dump_path, file_mode, 0664);
+		if (IS_ERR(fp) || (fp == NULL)) {
+			ret = PTR_ERR(fp);
+			DHD_ERROR(("open file error, err = %d\n", ret));
+			goto exit2;
+		}
+		DHD_ERROR(("debug_dump_path = %s\n", dump_path));
+#else
+		ret = PTR_ERR(fp);
+		DHD_ERROR(("open file error, err = %d\n", ret));
+		goto exit2;
+#endif /* CONFIG_X86 && OEM_ANDROID */
+	}
+
+	filep = dhd_filp_open(dump_path, O_RDONLY, 0);
+
+	if (IS_ERR(filep) || (filep == NULL)) {
+		DHD_ERROR(("file stat error, err = %d\n", ret));
+		goto exit2;
+	}
+	logstrs_size = dhd_vfs_size_read(filep);
+	/* if some one else has changed the file */
+	if (dhdp->last_file_posn != 0 &&
+			logstrs_size < dhdp->last_file_posn) {
+		dhdp->last_file_posn = 0;
+	}
+
+	/* XXX: periodic flush is disabled by default, if enabled
+	 * only periodic flushing of 'GENERAL' log dump buffer
+	 * is supported, its not recommended to turn on periodic
+	 * flushing, except for developer unit test.
+	 */
+	if (dhdp->logdump_periodic_flush) {
+		log_size = strlen(time_str) + strlen(DHD_DUMP_LOG_HDR) + sizeof(sec_hdr);
+		/* calculate the amount of space required to dump all logs */
+		for (i = 0; i < DLD_BUFFER_NUM; ++i) {
+			if (*type != DLD_BUF_TYPE_ALL && i != *type)
+				continue;
+
+			if (g_dld_buf[i].wraparound) {
+				log_size += (unsigned long)g_dld_buf[i].max
+						- (unsigned long)g_dld_buf[i].buffer;
+			} else {
+				DHD_LOG_DUMP_BUF_LOCK(&g_dld_buf[i].lock, flags);
+				log_size += (unsigned long)g_dld_buf[i].present -
+						(unsigned long)g_dld_buf[i].front;
+				DHD_LOG_DUMP_BUF_UNLOCK(&g_dld_buf[i].lock, flags);
+			}
+			log_size += strlen(dld_hdrs[i].hdr_str) + sizeof(sec_hdr);
+
+			if (*type != DLD_BUF_TYPE_ALL && i == *type)
+				break;
+		}
+
+		ret = generic_file_llseek(fp, dhdp->last_file_posn, SEEK_CUR);
+		if (ret < 0) {
+			DHD_ERROR(("file seek last posn error ! err = %d \n", ret));
+			goto exit2;
+		}
+		pos = fp->f_pos;
+
+		/* if the max file size is reached, wrap around to beginning of the file
+		 * we're treating the file as a large ring buffer
+		 */
+		fspace_remain = logdump_max_filesize - pos;
+		if (log_size > fspace_remain) {
+			fp->f_pos -= pos;
+			pos = fp->f_pos;
+		}
+	}
+
+	dhd_print_time_str(0, fp, len, &pos);
+
+	for (i = 0; i < DLD_BUFFER_NUM; ++i) {
+
+		if (*type != DLD_BUF_TYPE_ALL && i != *type)
+			continue;
+
+		len = dhd_get_dld_len(i);
+		dhd_get_dld_log_dump(NULL, dhdp, 0, fp, len, i, &pos);
+		if (*type != DLD_BUF_TYPE_ALL)
+			break;
+	}
+
+#ifdef EWP_ECNTRS_LOGGING
+	if (*type == DLD_BUF_TYPE_ALL &&
+			logdump_ecntr_enable &&
+			dhdp->ecntr_dbg_ring) {
+		dhd_log_dump_ring_to_file(dhdp, dhdp->ecntr_dbg_ring,
+				fp, (unsigned long *)&pos,
+				&sec_hdr, ECNTRS_LOG_HDR, LOG_DUMP_SECTION_ECNTRS);
+	}
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef DHD_STATUS_LOGGING
+	if (dhdp->statlog) {
+		/* write the statlog */
+		len = dhd_get_status_log_len(NULL, dhdp);
+		if (len) {
+			if (dhd_print_status_log_data(NULL, dhdp, 0, fp,
+				len, &pos) < 0) {
+				goto exit2;
+			}
+		}
+	}
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef DHD_STATUS_LOGGING
+	if (dhdp->statlog) {
+		dhd_print_buf_addr(dhdp, "statlog_logbuf", dhd_statlog_get_logbuf(dhdp),
+			dhd_statlog_get_logbuf_len(dhdp));
+	}
+#endif /* DHD_STATUS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+	if (*type == DLD_BUF_TYPE_ALL &&
+			logdump_rtt_enable &&
+			dhdp->rtt_dbg_ring) {
+		dhd_log_dump_ring_to_file(dhdp, dhdp->rtt_dbg_ring,
+				fp, (unsigned long *)&pos,
+				&sec_hdr, RTT_LOG_HDR, LOG_DUMP_SECTION_RTT);
+	}
+#endif /* EWP_RTT_LOGGING */
+
+#ifdef EWP_BCM_TRACE
+	if (*type == DLD_BUF_TYPE_ALL &&
+		dhdp->bcm_trace_dbg_ring) {
+		dhd_log_dump_ring_to_file(dhdp, dhdp->bcm_trace_dbg_ring,
+				fp, (unsigned long *)&pos,
+				&sec_hdr, BCM_TRACE_LOG_HDR, LOG_DUMP_SECTION_BCM_TRACE);
+	}
+#endif /* EWP_BCM_TRACE */
+
+#ifdef BCMPCIE
+	len = dhd_get_ext_trap_len(NULL, dhdp);
+	if (len) {
+		if (dhd_print_ext_trap_data(NULL, dhdp, 0, fp, len, &pos) < 0)
+			goto exit2;
+	}
+#endif /* BCMPCIE */
+
+#if defined(DHD_FW_COREDUMP) && defined(DNGL_EVENT_SUPPORT)
+	len = dhd_get_health_chk_len(NULL, dhdp);
+	if (len) {
+		if (dhd_print_health_chk_data(NULL, dhdp, 0, fp, len, &pos) < 0)
+			goto exit2;
+	}
+#endif /* DHD_FW_COREDUMP && DNGL_EVENT_SUPPORT */
+
+	len = dhd_get_dhd_dump_len(NULL, dhdp);
+	if (len) {
+		if (dhd_print_dump_data(NULL, dhdp, 0, fp, len, &pos) < 0)
+			goto exit2;
+	}
+
+	len = dhd_get_cookie_log_len(NULL, dhdp);
+	if (len) {
+		if (dhd_print_cookie_data(NULL, dhdp, 0, fp, len, &pos) < 0)
+			goto exit2;
+	}
+
+#ifdef DHD_DUMP_PCIE_RINGS
+	len = dhd_get_flowring_len(NULL, dhdp);
+	if (len) {
+		if (dhd_print_flowring_data(NULL, dhdp, 0, fp, len, &pos) < 0)
+			goto exit2;
+	}
+#endif
+
+	if (dhdp->logdump_periodic_flush) {
+		/* store the last position written to in the file for future use */
+		dhdp->last_file_posn = pos;
+	}
+
+exit2:
+	if (!IS_ERR(fp) && fp != NULL) {
+		dhd_filp_close(fp, NULL);
+		DHD_ERROR(("%s: Finished writing log dump to file - '%s' \n",
+				__FUNCTION__, dump_path));
+	}
+#ifdef get_fs
+	set_fs(old_fs);
+#endif /* get_fs */
+exit1:
+	if (type) {
+		MFREE(dhdp->osh, type, sizeof(*type));
+	}
+	DHD_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_LOGDUMP(dhdp);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+#ifdef DHD_DUMP_MNGR
+	if (ret >= 0) {
+		dhd_dump_file_manage_enqueue(dhdp, dump_path, DHD_DEBUG_DUMP_TYPE);
+	}
+#endif /* DHD_DUMP_MNGR */
+
+	return (ret < 0) ? BCME_ERROR : BCME_OK;
+}
+
+bool
+dhd_log_dump_ecntr_enabled(void)
+{
+	return (bool)logdump_ecntr_enable;
+}
+
+bool
+dhd_log_dump_rtt_enabled(void)
+{
+	return (bool)logdump_rtt_enable;
+}
+
+void
+dhd_log_dump_init(dhd_pub_t *dhd)
+{
+	struct dhd_log_dump_buf *dld_buf, *dld_buf_special;
+	int i = 0;
+	uint8 *prealloc_buf = NULL, *bufptr = NULL;
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+	int prealloc_idx = DHD_PREALLOC_DHD_LOG_DUMP_BUF;
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_USE_STATIC_MEMDUMP */
+	int ret;
+	dhd_dbg_ring_t *ring = NULL;
+	unsigned long flags = 0;
+	dhd_info_t *dhd_info = dhd->info;
+#if defined(EWP_ECNTRS_LOGGING)
+	void *cookie_buf = NULL;
+#endif
+
+	BCM_REFERENCE(ret);
+	BCM_REFERENCE(ring);
+	BCM_REFERENCE(flags);
+
+	/* sanity check */
+	if (logdump_prsrv_tailsize <= 0 ||
+		logdump_prsrv_tailsize > DHD_LOG_DUMP_MAX_TAIL_FLUSH_SIZE) {
+		logdump_prsrv_tailsize = DHD_LOG_DUMP_MAX_TAIL_FLUSH_SIZE;
+	}
+	/* now adjust the preserve log flush size based on the
+	* kernel printk log buffer size
+	*/
+#ifdef CONFIG_LOG_BUF_SHIFT
+	DHD_ERROR(("%s: kernel log buf size = %uKB; logdump_prsrv_tailsize = %uKB;"
+		" limit prsrv tail size to = %uKB\n",
+		__FUNCTION__, (1 << CONFIG_LOG_BUF_SHIFT)/1024,
+		logdump_prsrv_tailsize/1024, LOG_DUMP_KERNEL_TAIL_FLUSH_SIZE/1024));
+
+	if (logdump_prsrv_tailsize > LOG_DUMP_KERNEL_TAIL_FLUSH_SIZE) {
+		logdump_prsrv_tailsize = LOG_DUMP_KERNEL_TAIL_FLUSH_SIZE;
+	}
+#else
+	DHD_ERROR(("%s: logdump_prsrv_tailsize = %uKB \n",
+		__FUNCTION__, logdump_prsrv_tailsize/1024));
+#endif /* CONFIG_LOG_BUF_SHIFT */
+
+	mutex_init(&dhd_info->logdump_lock);
+	/* initialize log dump buf structures */
+	memset(g_dld_buf, 0, sizeof(struct dhd_log_dump_buf) * DLD_BUFFER_NUM);
+
+	/* set the log dump buffer size based on the module_param */
+	if (logdump_max_bufsize > LOG_DUMP_GENERAL_MAX_BUFSIZE ||
+			logdump_max_bufsize <= 0)
+		dld_buf_size[DLD_BUF_TYPE_GENERAL] = LOG_DUMP_GENERAL_MAX_BUFSIZE;
+	else
+		dld_buf_size[DLD_BUF_TYPE_GENERAL] = logdump_max_bufsize;
+
+	/* pre-alloc the memory for the log buffers & 'special' buffer */
+	dld_buf_special = &g_dld_buf[DLD_BUF_TYPE_SPECIAL];
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+	prealloc_buf = DHD_OS_PREALLOC(dhd, prealloc_idx++, LOG_DUMP_TOTAL_BUFSIZE);
+	dld_buf_special->buffer = DHD_OS_PREALLOC(dhd, prealloc_idx++,
+			dld_buf_size[DLD_BUF_TYPE_SPECIAL]);
+#else
+	prealloc_buf = MALLOCZ(dhd->osh, LOG_DUMP_TOTAL_BUFSIZE);
+	dld_buf_special->buffer = MALLOCZ(dhd->osh, dld_buf_size[DLD_BUF_TYPE_SPECIAL]);
+#endif /* CONFIG_DHD_USE_STATIC_BUF && DHD_USE_STATIC_MEMDUMP */
+
+	if (!prealloc_buf) {
+		DHD_ERROR(("Failed to allocate memory for log buffers\n"));
+		goto fail;
+	}
+	if (!dld_buf_special->buffer) {
+		DHD_ERROR(("Failed to allocate memory for special buffer\n"));
+		goto fail;
+	}
+
+	bufptr = prealloc_buf;
+	for (i = 0; i < DLD_BUFFER_NUM; i++) {
+		dld_buf = &g_dld_buf[i];
+		dld_buf->dhd_pub = dhd;
+		spin_lock_init(&dld_buf->lock);
+		dld_buf->wraparound = 0;
+		if (i != DLD_BUF_TYPE_SPECIAL) {
+			dld_buf->buffer = bufptr;
+			dld_buf->max = (unsigned long)dld_buf->buffer + dld_buf_size[i];
+			bufptr = (uint8 *)dld_buf->max;
+		} else {
+			dld_buf->max = (unsigned long)dld_buf->buffer + dld_buf_size[i];
+		}
+		dld_buf->present = dld_buf->front = dld_buf->buffer;
+		dld_buf->remain = dld_buf_size[i];
+		dld_buf->enable = 1;
+	}
+
+	/* now use the rest of the pre-alloc'd memory for other rings */
+#ifdef EWP_ECNTRS_LOGGING
+	dhd->ecntr_dbg_ring = dhd_dbg_ring_alloc_init(dhd,
+			ECNTR_RING_ID, ECNTR_RING_NAME,
+			LOG_DUMP_ECNTRS_MAX_BUFSIZE,
+			bufptr, TRUE);
+	if (!dhd->ecntr_dbg_ring) {
+		DHD_ERROR(("%s: unable to init ecounters dbg ring !\n",
+				__FUNCTION__));
+		goto fail;
+	}
+	bufptr += LOG_DUMP_ECNTRS_MAX_BUFSIZE;
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+	dhd->rtt_dbg_ring = dhd_dbg_ring_alloc_init(dhd,
+			RTT_RING_ID, RTT_RING_NAME,
+			LOG_DUMP_RTT_MAX_BUFSIZE,
+			bufptr, TRUE);
+	if (!dhd->rtt_dbg_ring) {
+		DHD_ERROR(("%s: unable to init rtt dbg ring !\n",
+				__FUNCTION__));
+		goto fail;
+	}
+	bufptr += LOG_DUMP_RTT_MAX_BUFSIZE;
+#endif /* EWP_RTT_LOGGING */
+
+#ifdef EWP_BCM_TRACE
+	dhd->bcm_trace_dbg_ring = dhd_dbg_ring_alloc_init(dhd,
+			BCM_TRACE_RING_ID, BCM_TRACE_RING_NAME,
+			LOG_DUMP_BCM_TRACE_MAX_BUFSIZE,
+			bufptr, TRUE);
+	if (!dhd->bcm_trace_dbg_ring) {
+		DHD_ERROR(("%s: unable to init bcm trace dbg ring !\n",
+				__FUNCTION__));
+		goto fail;
+	}
+	bufptr += LOG_DUMP_BCM_TRACE_MAX_BUFSIZE;
+#endif /* EWP_BCM_TRACE */
+
+	/* Concise buffer is used as intermediate buffer for following purposes
+	* a) pull ecounters records temporarily before
+	*  writing it to file
+	* b) to store dhd dump data before putting it to file
+	* It should have a size equal to
+	* MAX(largest possible ecntr record, 'dhd dump' data size)
+	*/
+	dhd->concise_dbg_buf = MALLOC(dhd->osh, CONCISE_DUMP_BUFLEN);
+	if (!dhd->concise_dbg_buf) {
+		DHD_ERROR(("%s: unable to alloc mem for concise debug info !\n",
+				__FUNCTION__));
+		goto fail;
+	}
+
+#if defined(DHD_EVENT_LOG_FILTER)
+	/* XXX init filter last, because filter use buffer which alloced by log dump */
+	ret = dhd_event_log_filter_init(dhd,
+		bufptr,
+		LOG_DUMP_FILTER_MAX_BUFSIZE);
+	if (ret != BCME_OK) {
+		goto fail;
+	}
+#endif /* DHD_EVENT_LOG_FILTER */
+
+#if defined(EWP_ECNTRS_LOGGING)
+	cookie_buf = MALLOC(dhd->osh, LOG_DUMP_COOKIE_BUFSIZE);
+	if (!cookie_buf) {
+		DHD_ERROR(("%s: unable to alloc mem for logdump cookie buffer\n",
+			__FUNCTION__));
+		goto fail;
+	}
+
+	ret = dhd_logdump_cookie_init(dhd, cookie_buf, LOG_DUMP_COOKIE_BUFSIZE);
+	if (ret != BCME_OK) {
+		MFREE(dhd->osh, cookie_buf, LOG_DUMP_COOKIE_BUFSIZE);
+		goto fail;
+	}
+#endif /* EWP_ECNTRS_LOGGING */
+	return;
+
+fail:
+
+#if defined(DHD_EVENT_LOG_FILTER)
+	/* XXX deinit filter first, because filter use buffer which alloced by log dump */
+	if (dhd->event_log_filter) {
+		dhd_event_log_filter_deinit(dhd);
+	}
+#endif /* DHD_EVENT_LOG_FILTER */
+
+	if (dhd->concise_dbg_buf) {
+		MFREE(dhd->osh, dhd->concise_dbg_buf, CONCISE_DUMP_BUFLEN);
+	}
+
+#ifdef EWP_ECNTRS_LOGGING
+	if (dhd->logdump_cookie) {
+		dhd_logdump_cookie_deinit(dhd);
+		MFREE(dhd->osh, dhd->logdump_cookie, LOG_DUMP_COOKIE_BUFSIZE);
+		dhd->logdump_cookie = NULL;
+	}
+#endif /* EWP_ECNTRS_LOGGING */
+
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+	if (prealloc_buf) {
+		DHD_OS_PREFREE(dhd, prealloc_buf, LOG_DUMP_TOTAL_BUFSIZE);
+	}
+	if (dld_buf_special->buffer) {
+		DHD_OS_PREFREE(dhd, dld_buf_special->buffer,
+				dld_buf_size[DLD_BUF_TYPE_SPECIAL]);
+	}
+#else
+	if (prealloc_buf) {
+		MFREE(dhd->osh, prealloc_buf, LOG_DUMP_TOTAL_BUFSIZE);
+	}
+	if (dld_buf_special->buffer) {
+		MFREE(dhd->osh, dld_buf_special->buffer,
+				dld_buf_size[DLD_BUF_TYPE_SPECIAL]);
+	}
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+	for (i = 0; i < DLD_BUFFER_NUM; i++) {
+		dld_buf = &g_dld_buf[i];
+		dld_buf->enable = 0;
+		dld_buf->buffer = NULL;
+	}
+	mutex_destroy(&dhd_info->logdump_lock);
+}
+
+void
+dhd_log_dump_deinit(dhd_pub_t *dhd)
+{
+	struct dhd_log_dump_buf *dld_buf = NULL, *dld_buf_special = NULL;
+	int i = 0;
+	dhd_info_t *dhd_info = dhd->info;
+	dhd_dbg_ring_t *ring = NULL;
+
+	BCM_REFERENCE(ring);
+
+	if (dhd->concise_dbg_buf) {
+		MFREE(dhd->osh, dhd->concise_dbg_buf, CONCISE_DUMP_BUFLEN);
+		dhd->concise_dbg_buf = NULL;
+	}
+
+#ifdef EWP_ECNTRS_LOGGING
+	if (dhd->logdump_cookie) {
+		dhd_logdump_cookie_deinit(dhd);
+		MFREE(dhd->osh, dhd->logdump_cookie, LOG_DUMP_COOKIE_BUFSIZE);
+		dhd->logdump_cookie = NULL;
+	}
+
+	if (dhd->ecntr_dbg_ring) {
+		dhd_dbg_ring_dealloc_deinit(&dhd->ecntr_dbg_ring, dhd);
+	}
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+	if (dhd->rtt_dbg_ring) {
+		dhd_dbg_ring_dealloc_deinit(&dhd->rtt_dbg_ring, dhd);
+	}
+#endif /* EWP_RTT_LOGGING */
+
+#ifdef EWP_BCM_TRACE
+	if (dhd->bcm_trace_dbg_ring) {
+		dhd_dbg_ring_dealloc_deinit(&dhd->bcm_trace_dbg_ring, dhd);
+	}
+#endif /* EWP_BCM_TRACE */
+
+	/* 'general' buffer points to start of the pre-alloc'd memory */
+	dld_buf = &g_dld_buf[DLD_BUF_TYPE_GENERAL];
+	dld_buf_special = &g_dld_buf[DLD_BUF_TYPE_SPECIAL];
+#if defined(CONFIG_DHD_USE_STATIC_BUF) && defined(DHD_USE_STATIC_MEMDUMP)
+	if (dld_buf->buffer) {
+		DHD_OS_PREFREE(dhd, dld_buf->buffer, LOG_DUMP_TOTAL_BUFSIZE);
+	}
+	if (dld_buf_special->buffer) {
+		DHD_OS_PREFREE(dhd, dld_buf_special->buffer,
+				dld_buf_size[DLD_BUF_TYPE_SPECIAL]);
+	}
+#else
+	if (dld_buf->buffer) {
+		MFREE(dhd->osh, dld_buf->buffer, LOG_DUMP_TOTAL_BUFSIZE);
+	}
+	if (dld_buf_special->buffer) {
+		MFREE(dhd->osh, dld_buf_special->buffer,
+				dld_buf_size[DLD_BUF_TYPE_SPECIAL]);
+	}
+#endif /* CONFIG_DHD_USE_STATIC_BUF */
+	for (i = 0; i < DLD_BUFFER_NUM; i++) {
+		dld_buf = &g_dld_buf[i];
+		dld_buf->enable = 0;
+		dld_buf->buffer = NULL;
+	}
+	mutex_destroy(&dhd_info->logdump_lock);
+}
+
+void
+dhd_log_dump_write(int type, char *binary_data,
+		int binary_len, const char *fmt, ...)
+{
+	int len = 0;
+	char tmp_buf[DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE] = {0, };
+	va_list args;
+	unsigned long flags = 0;
+	struct dhd_log_dump_buf *dld_buf = NULL;
+	bool flush_log = FALSE;
+
+	if (type < 0 || type >= DLD_BUFFER_NUM) {
+		DHD_INFO(("%s: Unsupported DHD_LOG_DUMP_BUF_TYPE(%d).\n",
+			__FUNCTION__, type));
+		return;
+	}
+
+	dld_buf = &g_dld_buf[type];
+	if (dld_buf->enable != 1) {
+		return;
+	}
+
+	va_start(args, fmt);
+	len = vsnprintf(tmp_buf, DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE, fmt, args);
+	/* Non ANSI C99 compliant returns -1,
+	 * ANSI compliant return len >= DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE
+	 */
+	va_end(args);
+	if (len < 0) {
+		return;
+	}
+
+	if (len >= DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE) {
+		len = DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE - 1;
+		tmp_buf[len] = '\0';
+	}
+
+	/* make a critical section to eliminate race conditions */
+	DHD_LOG_DUMP_BUF_LOCK(&dld_buf->lock, flags);
+	if (dld_buf->remain < len) {
+		dld_buf->wraparound = 1;
+		dld_buf->present = dld_buf->front;
+		dld_buf->remain = dld_buf_size[type];
+		/* if wrap around happens, flush the ring buffer to the file */
+		flush_log = TRUE;
+	}
+
+	memcpy(dld_buf->present, tmp_buf, len);
+	dld_buf->remain -= len;
+	dld_buf->present += len;
+	DHD_LOG_DUMP_BUF_UNLOCK(&dld_buf->lock, flags);
+
+	/* double check invalid memory operation */
+	ASSERT((unsigned long)dld_buf->present <= dld_buf->max);
+
+	if (dld_buf->dhd_pub) {
+		dhd_pub_t *dhdp = (dhd_pub_t *)dld_buf->dhd_pub;
+		dhdp->logdump_periodic_flush =
+			logdump_periodic_flush;
+		if (logdump_periodic_flush && flush_log) {
+			log_dump_type_t *flush_type = MALLOCZ(dhdp->osh,
+					sizeof(log_dump_type_t));
+			if (flush_type) {
+				*flush_type = type;
+				dhd_schedule_log_dump(dld_buf->dhd_pub, flush_type);
+			}
+		}
+	}
+}
+
+void
+dhd_log_dump_vendor_trigger(dhd_pub_t *dhd_pub)
+{
+	unsigned long flags = 0;
+	DHD_GENERAL_LOCK(dhd_pub, flags);
+	DHD_BUS_BUSY_SET_IN_DUMP_DONGLE_MEM(dhd_pub);
+	DHD_GENERAL_UNLOCK(dhd_pub, flags);
+
+	dhd_log_dump_trigger(dhd_pub, CMD_DEFAULT);
+
+	DHD_GENERAL_LOCK(dhd_pub, flags);
+	DHD_BUS_BUSY_CLEAR_IN_DUMP_DONGLE_MEM(dhd_pub);
+	dhd_os_busbusy_wake(dhd_pub);
+	DHD_GENERAL_UNLOCK(dhd_pub, flags);
+
+	return;
+}
+
+void
+dhd_log_dump_trigger(dhd_pub_t *dhdp, int subcmd)
+{
+#if defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	log_dump_type_t *flush_type;
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+	uint64 current_time_sec;
+
+	if (!dhdp) {
+		DHD_ERROR(("dhdp is NULL !\n"));
+		return;
+	}
+
+	if (subcmd >= CMD_MAX || subcmd < CMD_DEFAULT) {
+		DHD_ERROR(("%s : Invalid subcmd \n", __FUNCTION__));
+		return;
+	}
+
+	current_time_sec = DIV_U64_BY_U32(OSL_LOCALTIME_NS(), NSEC_PER_SEC);
+
+	DHD_ERROR(("%s: current_time_sec=%lld debug_dump_time_sec=%lld interval=%d\n",
+		__FUNCTION__, current_time_sec, dhdp->debug_dump_time_sec,
+		DEBUG_DUMP_TRIGGER_INTERVAL_SEC));
+
+	if ((current_time_sec - dhdp->debug_dump_time_sec) < DEBUG_DUMP_TRIGGER_INTERVAL_SEC) {
+		DHD_ERROR(("%s : Last debug dump triggered(%lld) within %d seconds, so SKIP\n",
+			__FUNCTION__, dhdp->debug_dump_time_sec, DEBUG_DUMP_TRIGGER_INTERVAL_SEC));
+		return;
+	}
+
+	clear_debug_dump_time(dhdp->debug_dump_time_str);
+	/*  */
+
+	dhdp->debug_dump_subcmd = subcmd;
+
+	dhdp->debug_dump_time_sec = DIV_U64_BY_U32(OSL_LOCALTIME_NS(), NSEC_PER_SEC);
+
+#if defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	/* flush_type is freed at do_dhd_log_dump function */
+	flush_type = MALLOCZ(dhdp->osh, sizeof(log_dump_type_t));
+	if (flush_type) {
+		*flush_type = DLD_BUF_TYPE_ALL;
+		dhd_schedule_log_dump(dhdp, flush_type);
+	} else {
+		DHD_ERROR(("%s Fail to malloc flush_type\n", __FUNCTION__));
+		return;
+	}
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+
+#if defined(DHD_SDTC_ETB_DUMP) && defined(BCMQT_HW)
+	/* For QT/zebu memdump collection is disabled,
+	 * but set collect_sdtc true here, so that
+	 * developers can issue 'log_dump' dhd iovar
+	 * to get etb/sdtc dumps
+	 */
+	DHD_ERROR(("%s : QT/FPGA set collect_sdtc as TRUE\n", __FUNCTION__));
+	dhdp->collect_sdtc = TRUE;
+#endif /* DHD_SDTC_ETB_DUMP && BCMQT_HW */
+
+	/* Inside dhd_mem_dump, event notification will be sent to HAL and
+	 * from other context DHD pushes memdump, debug_dump and pktlog dump
+	 * to HAL and HAL will write into file
+	 */
+#if (defined(BCMPCIE) || defined(BCMSDIO)) && defined(DHD_FW_COREDUMP)
+	dhdp->memdump_type = DUMP_TYPE_BY_SYSDUMP;
+	dhd_bus_mem_dump(dhdp);
+#endif /* BCMPCIE && DHD_FW_COREDUMP */
+
+#if defined(DHD_PKT_LOGGING) && defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL)
+	dhd_schedule_pktlog_dump(dhdp);
+#endif /* DHD_PKT_LOGGING && DHD_DUMP_FILE_WRITE_FROM_KERNEL */
+}
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+int dhd_debug_dump_get_ring_num(int sec_type)
+{
+	int idx = 0;
+	for (idx = 0; idx < ARRAYSIZE(dhd_debug_dump_ring_map); idx++) {
+		if (dhd_debug_dump_ring_map[idx].type == sec_type) {
+			idx = dhd_debug_dump_ring_map[idx].debug_dump_ring;
+			return idx;
+		}
+	}
+	return idx;
+}
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+
+int
+dhd_dump_debug_ring(dhd_pub_t *dhdp, void *ring_ptr, const void *user_buf,
+		log_dump_section_hdr_t *sec_hdr,
+		char *text_hdr, int buflen, uint32 sec_type)
+{
+	uint32 rlen = 0;
+	uint32 data_len = 0;
+	void *data = NULL;
+	unsigned long flags = 0;
+	int ret = 0;
+	dhd_dbg_ring_t *ring = (dhd_dbg_ring_t *)ring_ptr;
+#ifndef DHD_DEBUGABILITY_DEBUG_DUMP
+	int pos = 0;
+#endif /* !DHD_DEBUGABILITY_DEBUG_DUMP */
+	int fpos_sechdr = 0;
+	int tot_len = 0;
+	char *tmp_buf = NULL;
+	int idx;
+	int ring_num = 0;
+
+	BCM_REFERENCE(idx);
+	BCM_REFERENCE(tot_len);
+	BCM_REFERENCE(fpos_sechdr);
+	BCM_REFERENCE(data_len);
+	BCM_REFERENCE(tmp_buf);
+	BCM_REFERENCE(ring_num);
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+	if (!dhdp || !ring || !sec_hdr || !text_hdr) {
+		return BCME_BADARG;
+	}
+
+	tmp_buf = (char *)VMALLOCZ(dhdp->osh, ring->ring_size);
+	if (!tmp_buf) {
+		DHD_ERROR(("%s: VMALLOC Fail id:%d size:%u\n",
+			__func__, ring->id, ring->ring_size));
+		return BCME_NOMEM;
+	}
+#else
+	if (!dhdp || !ring || !user_buf || !sec_hdr || !text_hdr) {
+		return BCME_BADARG;
+	}
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+	/* do not allow further writes to the ring
+	 * till we flush it
+	 */
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->state = RING_SUSPEND;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+	ring_num = dhd_debug_dump_get_ring_num(sec_type);
+	dhd_export_debug_data(text_hdr, NULL, NULL, strlen(text_hdr), &ring_num);
+
+	data = tmp_buf;
+	do {
+		rlen = dhd_dbg_ring_pull_single(ring, data, ring->ring_size, TRUE);
+		if (rlen > 0) {
+			tot_len += rlen;
+			data += rlen;
+		}
+	} while ((rlen > 0));
+
+	sec_hdr->type = sec_type;
+	sec_hdr->length = tot_len;
+	DHD_ERROR(("%s: DUMP id:%d type:%u tot_len:%d\n", __func__, ring->id, sec_type, tot_len));
+
+	dhd_export_debug_data((char *)sec_hdr, NULL, NULL, sizeof(*sec_hdr), &ring_num);
+	dhd_export_debug_data(tmp_buf, NULL, NULL, tot_len, &ring_num);
+
+	VMFREE(dhdp->osh, tmp_buf, ring->ring_size);
+#else
+	if (dhdp->concise_dbg_buf) {
+		/* re-use concise debug buffer temporarily
+		 * to pull ring data, to write
+		 * record by record to file
+		 */
+		data_len = CONCISE_DUMP_BUFLEN;
+		data = dhdp->concise_dbg_buf;
+		ret = dhd_export_debug_data(text_hdr, NULL, user_buf, strlen(text_hdr), &pos);
+		/* write the section header now with zero length,
+		 * once the correct length is found out, update
+		 * it later
+		 */
+		fpos_sechdr = pos;
+		sec_hdr->type = sec_type;
+		sec_hdr->length = 0;
+		ret = dhd_export_debug_data((char *)sec_hdr, NULL, user_buf,
+			sizeof(*sec_hdr), &pos);
+		do {
+			rlen = dhd_dbg_ring_pull_single(ring, data, data_len, TRUE);
+			if (rlen > 0) {
+				/* write the log */
+				ret = dhd_export_debug_data(data, NULL, user_buf, rlen, &pos);
+			}
+			DHD_DBGIF(("%s: rlen : %d\n", __FUNCTION__, rlen));
+		} while ((rlen > 0));
+		/* now update the section header length in the file */
+		/* Complete ring size is dumped by HAL, hence updating length to ring size */
+		sec_hdr->length = ring->ring_size;
+		ret = dhd_export_debug_data((char *)sec_hdr, NULL, user_buf,
+			sizeof(*sec_hdr), &fpos_sechdr);
+	} else {
+		DHD_ERROR(("%s: No concise buffer available !\n", __FUNCTION__));
+	}
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->state = RING_ACTIVE;
+	/* Resetting both read and write pointer,
+	 * since all items are read.
+	 */
+	ring->rp = ring->wp = 0;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	return ret;
+}
+
+int
+dhd_log_dump_ring_to_file(dhd_pub_t *dhdp, void *ring_ptr, void *file,
+		unsigned long *file_posn, log_dump_section_hdr_t *sec_hdr,
+		char *text_hdr, uint32 sec_type)
+{
+	uint32 rlen = 0;
+	uint32 data_len = 0, total_len = 0;
+	void *data = NULL;
+	unsigned long fpos_sechdr = 0;
+	unsigned long flags = 0;
+	int ret = 0;
+	dhd_dbg_ring_t *ring = (dhd_dbg_ring_t *)ring_ptr;
+
+	if (!dhdp || !ring || !file || !sec_hdr ||
+		!file_posn || !text_hdr)
+		return BCME_BADARG;
+
+	/* do not allow further writes to the ring
+	 * till we flush it
+	 */
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->state = RING_SUSPEND;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+
+	if (dhdp->concise_dbg_buf) {
+		/* re-use concise debug buffer temporarily
+		 * to pull ring data, to write
+		 * record by record to file
+		 */
+		data_len = CONCISE_DUMP_BUFLEN;
+		data = dhdp->concise_dbg_buf;
+		dhd_os_write_file_posn(file, file_posn, text_hdr,
+				strlen(text_hdr));
+		/* write the section header now with zero length,
+		 * once the correct length is found out, update
+		 * it later
+		 */
+		dhd_init_sec_hdr(sec_hdr);
+		fpos_sechdr = *file_posn;
+		sec_hdr->type = sec_type;
+		sec_hdr->length = 0;
+		dhd_os_write_file_posn(file, file_posn, (char *)sec_hdr,
+				sizeof(*sec_hdr));
+		do {
+			rlen = dhd_dbg_ring_pull_single(ring, data, data_len, TRUE);
+			if (rlen > 0) {
+				/* write the log */
+				ret = dhd_os_write_file_posn(file, file_posn, data, rlen);
+				if (ret < 0) {
+					DHD_ERROR(("%s: write file error !\n", __FUNCTION__));
+					DHD_DBG_RING_LOCK(ring->lock, flags);
+					ring->state = RING_ACTIVE;
+					DHD_DBG_RING_UNLOCK(ring->lock, flags);
+					return BCME_ERROR;
+				}
+			}
+			total_len += rlen;
+		} while (rlen > 0);
+		/* now update the section header length in the file */
+		sec_hdr->length = total_len;
+		dhd_os_write_file_posn(file, &fpos_sechdr, (char *)sec_hdr, sizeof(*sec_hdr));
+	} else {
+		DHD_ERROR(("%s: No concise buffer available !\n", __FUNCTION__));
+	}
+
+	DHD_DBG_RING_LOCK(ring->lock, flags);
+	ring->state = RING_ACTIVE;
+	/* Resetting both read and write pointer,
+	 * since all items are read.
+	 */
+	ring->rp = ring->wp = 0;
+	DHD_DBG_RING_UNLOCK(ring->lock, flags);
+	return BCME_OK;
+}
+
+/* logdump cookie */
+#define MAX_LOGUDMP_COOKIE_CNT	10u
+#define LOGDUMP_COOKIE_STR_LEN	50u
+int
+dhd_logdump_cookie_init(dhd_pub_t *dhdp, uint8 *buf, uint32 buf_size)
+{
+	uint32 ring_size;
+
+	if (!dhdp || !buf) {
+		DHD_ERROR(("INVALID PTR: dhdp:%p buf:%p\n", dhdp, buf));
+		return BCME_ERROR;
+	}
+
+	ring_size = dhd_ring_get_hdr_size() + LOGDUMP_COOKIE_STR_LEN * MAX_LOGUDMP_COOKIE_CNT;
+	if (buf_size < ring_size) {
+		DHD_ERROR(("BUF SIZE IS TO SHORT: req:%d buf_size:%d\n",
+			ring_size, buf_size));
+		return BCME_ERROR;
+	}
+
+	dhdp->logdump_cookie = dhd_ring_init(dhdp, buf, buf_size,
+		LOGDUMP_COOKIE_STR_LEN, MAX_LOGUDMP_COOKIE_CNT,
+		DHD_RING_TYPE_FIXED);
+	if (!dhdp->logdump_cookie) {
+		DHD_ERROR(("FAIL TO INIT COOKIE RING\n"));
+		return BCME_ERROR;
+	}
+
+	return BCME_OK;
+}
+
+void
+dhd_logdump_cookie_deinit(dhd_pub_t *dhdp)
+{
+	if (!dhdp) {
+		return;
+	}
+	if (dhdp->logdump_cookie) {
+		dhd_ring_deinit(dhdp, dhdp->logdump_cookie);
+	}
+
+	return;
+}
+
+void
+dhd_logdump_cookie_save(dhd_pub_t *dhdp, char *cookie, char *type)
+{
+	char *ptr;
+
+	if (!dhdp || !cookie || !type || !dhdp->logdump_cookie) {
+		DHD_ERROR(("%s: At least one buffer ptr is NULL dhdp=%p cookie=%p"
+			" type = %p, cookie_cfg:%p\n", __FUNCTION__,
+			dhdp, cookie, type, dhdp?dhdp->logdump_cookie: NULL));
+		return;
+	}
+	ptr = (char *)dhd_ring_get_empty(dhdp->logdump_cookie);
+	if (ptr == NULL) {
+		DHD_ERROR(("%s : Skip to save due to locking\n", __FUNCTION__));
+		return;
+	}
+	scnprintf(ptr, LOGDUMP_COOKIE_STR_LEN, "%s: %s\n", type, cookie);
+	return;
+}
+
+int
+dhd_logdump_cookie_get(dhd_pub_t *dhdp, char *ret_cookie, uint32 buf_size)
+{
+	char *ptr;
+
+	if (!dhdp || !ret_cookie || !dhdp->logdump_cookie) {
+		DHD_ERROR(("%s: At least one buffer ptr is NULL dhdp=%p"
+			"cookie=%p cookie_cfg:%p\n", __FUNCTION__,
+			dhdp, ret_cookie, dhdp?dhdp->logdump_cookie: NULL));
+		return BCME_ERROR;
+	}
+	ptr = (char *)dhd_ring_get_first(dhdp->logdump_cookie);
+	if (ptr == NULL) {
+		DHD_ERROR(("%s : Skip to save due to locking\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	memcpy(ret_cookie, ptr, MIN(buf_size, strlen(ptr)));
+	dhd_ring_free_first(dhdp->logdump_cookie);
+	return BCME_OK;
+}
+
+int
+dhd_logdump_cookie_count(dhd_pub_t *dhdp)
+{
+	if (!dhdp || !dhdp->logdump_cookie) {
+		DHD_ERROR(("%s: At least one buffer ptr is NULL dhdp=%p cookie=%p\n",
+			__FUNCTION__, dhdp, dhdp?dhdp->logdump_cookie: NULL));
+		return 0;
+	}
+	return dhd_ring_get_cur_size(dhdp->logdump_cookie);
+}
+
+static inline int
+__dhd_log_dump_cookie_to_file(
+	dhd_pub_t *dhdp, void *fp, const void *user_buf, unsigned long *f_pos,
+	char *buf, uint32 buf_size)
+{
+
+	uint32 remain = buf_size;
+	int ret = BCME_ERROR;
+	char tmp_buf[LOGDUMP_COOKIE_STR_LEN];
+	log_dump_section_hdr_t sec_hdr;
+	uint32 read_idx;
+	uint32 write_idx;
+
+	read_idx = dhd_ring_get_read_idx(dhdp->logdump_cookie);
+	write_idx = dhd_ring_get_write_idx(dhdp->logdump_cookie);
+	while (dhd_logdump_cookie_count(dhdp) > 0) {
+		memset(tmp_buf, 0, sizeof(tmp_buf));
+		ret = dhd_logdump_cookie_get(dhdp, tmp_buf, LOGDUMP_COOKIE_STR_LEN);
+		if (ret != BCME_OK) {
+			return ret;
+		}
+		remain -= scnprintf(&buf[buf_size - remain], remain, "%s", tmp_buf);
+	}
+	dhd_ring_set_read_idx(dhdp->logdump_cookie, read_idx);
+	dhd_ring_set_write_idx(dhdp->logdump_cookie, write_idx);
+
+	ret = dhd_export_debug_data(COOKIE_LOG_HDR, fp, user_buf, strlen(COOKIE_LOG_HDR), f_pos);
+	if (ret < 0) {
+		DHD_ERROR(("%s : Write file Error for cookie hdr\n", __FUNCTION__));
+		return ret;
+	}
+	sec_hdr.magic = LOG_DUMP_MAGIC;
+	sec_hdr.timestamp = local_clock();
+	sec_hdr.type = LOG_DUMP_SECTION_COOKIE;
+	sec_hdr.length = buf_size - remain;
+
+	ret = dhd_export_debug_data((char *)&sec_hdr, fp, user_buf, sizeof(sec_hdr), f_pos);
+	if (ret < 0) {
+		DHD_ERROR(("%s : Write file Error for section hdr\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = dhd_export_debug_data(buf, fp, user_buf, sec_hdr.length, f_pos);
+	if (ret < 0) {
+		DHD_ERROR(("%s : Write file Error for cookie data\n", __FUNCTION__));
+	}
+
+	return ret;
+}
+
+uint32
+dhd_log_dump_cookie_len(dhd_pub_t *dhdp)
+{
+	int len = 0;
+	char tmp_buf[LOGDUMP_COOKIE_STR_LEN];
+	log_dump_section_hdr_t sec_hdr;
+	char *buf = NULL;
+	int ret = BCME_ERROR;
+	uint32 buf_size = MAX_LOGUDMP_COOKIE_CNT * LOGDUMP_COOKIE_STR_LEN;
+	uint32 read_idx;
+	uint32 write_idx;
+	uint32 remain;
+
+	remain = buf_size;
+
+	if (!dhdp || !dhdp->logdump_cookie) {
+		DHD_ERROR(("%s At least one ptr is NULL "
+			"dhdp = %p cookie %p\n",
+			__FUNCTION__, dhdp, dhdp?dhdp->logdump_cookie:NULL));
+		goto exit;
+	}
+
+	buf = (char *)MALLOCZ(dhdp->osh, buf_size);
+	if (!buf) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		goto exit;
+	}
+
+	read_idx = dhd_ring_get_read_idx(dhdp->logdump_cookie);
+	write_idx = dhd_ring_get_write_idx(dhdp->logdump_cookie);
+	while (dhd_logdump_cookie_count(dhdp) > 0) {
+		memset(tmp_buf, 0, sizeof(tmp_buf));
+		ret = dhd_logdump_cookie_get(dhdp, tmp_buf, LOGDUMP_COOKIE_STR_LEN);
+		if (ret != BCME_OK) {
+			goto exit;
+		}
+		remain -= (uint32)strlen(tmp_buf);
+	}
+	dhd_ring_set_read_idx(dhdp->logdump_cookie, read_idx);
+	dhd_ring_set_write_idx(dhdp->logdump_cookie, write_idx);
+	len += strlen(COOKIE_LOG_HDR);
+	len += sizeof(sec_hdr);
+	len += (buf_size - remain);
+exit:
+	if (buf)
+		MFREE(dhdp->osh, buf, buf_size);
+	return len;
+}
+
+int
+dhd_log_dump_cookie(dhd_pub_t *dhdp, const void *user_buf)
+{
+	int ret = BCME_ERROR;
+	char tmp_buf[LOGDUMP_COOKIE_STR_LEN];
+	log_dump_section_hdr_t sec_hdr;
+	char *buf = NULL;
+	uint32 buf_size = MAX_LOGUDMP_COOKIE_CNT * LOGDUMP_COOKIE_STR_LEN;
+	int pos = 0;
+	uint32 read_idx;
+	uint32 write_idx;
+	uint32 remain;
+
+	remain = buf_size;
+
+	if (!dhdp || !dhdp->logdump_cookie) {
+		DHD_ERROR(("%s At least one ptr is NULL "
+			"dhdp = %p cookie %p\n",
+			__FUNCTION__, dhdp, dhdp?dhdp->logdump_cookie:NULL));
+		goto exit;
+	}
+
+	buf = (char *)MALLOCZ(dhdp->osh, buf_size);
+	if (!buf) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		goto exit;
+	}
+
+	read_idx = dhd_ring_get_read_idx(dhdp->logdump_cookie);
+	write_idx = dhd_ring_get_write_idx(dhdp->logdump_cookie);
+	while (dhd_logdump_cookie_count(dhdp) > 0) {
+		memset(tmp_buf, 0, sizeof(tmp_buf));
+		ret = dhd_logdump_cookie_get(dhdp, tmp_buf, LOGDUMP_COOKIE_STR_LEN);
+		if (ret != BCME_OK) {
+			goto exit;
+		}
+		remain -= scnprintf(&buf[buf_size - remain], remain, "%s", tmp_buf);
+	}
+	dhd_ring_set_read_idx(dhdp->logdump_cookie, read_idx);
+	dhd_ring_set_write_idx(dhdp->logdump_cookie, write_idx);
+	ret = dhd_export_debug_data(COOKIE_LOG_HDR, NULL, user_buf, strlen(COOKIE_LOG_HDR), &pos);
+	sec_hdr.magic = LOG_DUMP_MAGIC;
+	sec_hdr.timestamp = local_clock();
+	sec_hdr.type = LOG_DUMP_SECTION_COOKIE;
+	sec_hdr.length = buf_size - remain;
+	ret = dhd_export_debug_data((char *)&sec_hdr, NULL, user_buf, sizeof(sec_hdr), &pos);
+	ret = dhd_export_debug_data(buf, NULL, user_buf, sec_hdr.length, &pos);
+exit:
+	if (buf)
+		MFREE(dhdp->osh, buf, buf_size);
+	return ret;
+}
+
+int
+dhd_log_dump_cookie_to_file(dhd_pub_t *dhdp, void *fp, const void *user_buf, unsigned long *f_pos)
+{
+	char *buf;
+	int ret = BCME_ERROR;
+	uint32 buf_size = MAX_LOGUDMP_COOKIE_CNT * LOGDUMP_COOKIE_STR_LEN;
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+	if (!dhdp || !dhdp->logdump_cookie)
+#else
+	if (!dhdp || !dhdp->logdump_cookie || (!fp && !user_buf) || !f_pos)
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+	{
+		DHD_ERROR(("%s At least one ptr is NULL "
+			"dhdp = %p cookie %p fp = %p f_pos = %p\n",
+			__FUNCTION__, dhdp, dhdp?dhdp->logdump_cookie:NULL, fp, f_pos));
+		return ret;
+	}
+
+	buf = (char *)MALLOCZ(dhdp->osh, buf_size);
+	if (!buf) {
+		DHD_ERROR(("%s Fail to malloc buffer\n", __FUNCTION__));
+		return ret;
+	}
+	ret = __dhd_log_dump_cookie_to_file(dhdp, fp, user_buf, f_pos, buf, buf_size);
+	MFREE(dhdp->osh, buf, buf_size);
+
+	return ret;
+}
+
+void
+get_debug_dump_time(char *str)
+{
+	struct osl_timespec curtime;
+	unsigned long long local_time;
+	struct rtc_time tm;
+
+	if (!strlen(str)) {
+		osl_do_gettimeofday(&curtime);
+		local_time = (u64)(curtime.tv_sec -
+				(sys_tz.tz_minuteswest * DHD_LOG_DUMP_TS_MULTIPLIER_VALUE));
+		rtc_time_to_tm(local_time, &tm);
+		snprintf(str, DEBUG_DUMP_TIME_BUF_LEN, DHD_LOG_DUMP_TS_FMT_YYMMDDHHMMSSMSMS,
+				tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
+				tm.tm_sec, curtime.tv_nsec/NSEC_PER_MSEC);
+	}
+}
+
+void
+clear_debug_dump_time(char *str)
+{
+	memset(str, 0, DEBUG_DUMP_TIME_BUF_LEN);
+}
+#if defined(WL_CFGVENDOR_SEND_HANG_EVENT) || defined(DHD_PKT_LOGGING)
+void
+copy_debug_dump_time(char *dest, char *src)
+{
+	memcpy(dest, src, DEBUG_DUMP_TIME_BUF_LEN);
+}
+#endif
+
+#ifdef DHD_IOVAR_LOG_FILTER_DUMP
+typedef struct iovar_log_filter_table {
+	char command[64];
+	int  enable;
+} iovar_log_filter_table_t;
+
+/* WLC_GET_VAR is not being logged by default. Add for logging in debug_dump. */
+static const iovar_log_filter_table_t iovar_get_filter_params[] = {
+	{"cur_etheraddr", TRUE},
+	{"\0", TRUE}
+};
+
+/* WLC_SET_VAR is being logged by default. Add for not logging in debug_dump. */
+static const iovar_log_filter_table_t iovar_set_filter_params[] = {
+	{"pkt_filter_enable", FALSE},
+	{"pkt_filter_mode", FALSE},
+	{"\0", FALSE}
+};
+
+bool
+dhd_iovar_log_dump_check(dhd_pub_t *dhd_pub, uint32 cmd, char *msg)
+{
+	int cnt = 0;
+	const iovar_log_filter_table_t *table;
+	bool ret_val = TRUE;
+
+	/* Logging all IOVARs with DHD_IOVAR_MEM() in debug_dump file
+	 * during during Wifi ON.
+	 */
+	if (dhd_pub->up == FALSE) {
+		return TRUE;
+	}
+
+	if (cmd == WLC_GET_VAR) {
+		ret_val = FALSE;
+		table = iovar_get_filter_params;
+	} else if (cmd == WLC_SET_VAR) {
+		ret_val = TRUE;
+		table = iovar_set_filter_params;
+	} else {
+		return TRUE;
+	}
+	while (strlen(table[cnt].command) > 0) {
+		if (!strncmp(msg, table[cnt].command,
+				strlen(table[cnt].command))) {
+			return table[cnt].enable;
+		}
+
+		cnt++;
+	}
+
+	return ret_val;
+}
+#endif /* DHD_IOVAR_LOG_FILTER_DUMP */
+#endif /* DHD_LOG_DUMP */
diff --git a/drivers/net/wireless/ap12275/dhd_log_dump.h b/drivers/net/wireless/ap12275/dhd_log_dump.h
new file mode 100644
index 000000000000..97d48461634e
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_log_dump.h
@@ -0,0 +1,402 @@
+/*
+ * log_dump - debugability support for dumping logs to file - header file
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_LOG_DUMP_H__
+#define __DHD_LOG_DUMP_H__
+
+#ifdef DHD_LOG_DUMP
+
+#include <dhd.h>
+
+#define LOG_DUMP_MAGIC 0xDEB3DEB3
+#ifdef EWP_ECNTRS_LOGGING
+#define ECNTR_RING_ID 0xECDB
+#define	ECNTR_RING_NAME	"ewp_ecntr_ring"
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_RTT_LOGGING
+#define	RTT_RING_ID 0xADCD
+#define	RTT_RING_NAME	"ewp_rtt_ring"
+#endif /* EWP_ECNTRS_LOGGING */
+
+#ifdef EWP_BCM_TRACE
+#define	BCM_TRACE_RING_ID 0xBCBC
+#define	BCM_TRACE_RING_NAME "ewp_bcm_trace_ring"
+#endif /* EWP_BCM_TRACE */
+
+#define DHD_LOG_DUMP_MAX_TEMP_BUFFER_SIZE	512
+#define DHD_LOG_DUMP_MAX_TAIL_FLUSH_SIZE (80 * 1024)
+#define DHD_LOG_DUMP_TS_MULTIPLIER_VALUE    60
+#define DHD_LOG_DUMP_TS_FMT_YYMMDDHHMMSSMSMS    "%02d%02d%02d%02d%02d%02d%04lu"
+#define DHD_LOG_DUMP_TS_FMT_YYMMDDHHMMSS        "%02d%02d%02d%02d%02d%02d"
+#define DHD_DEBUG_DUMP_TYPE		"debug_dump"
+#define DEBUG_DUMP_TRIGGER_INTERVAL_SEC	4
+
+#ifndef _DHD_LOG_DUMP_DEFINITIONS_
+#define _DHD_LOG_DUMP_DEFINITIONS_
+#define GENERAL_LOG_HDR "\n-------------------- General log ---------------------------\n"
+#define PRESERVE_LOG_HDR "\n-------------------- Preserve log ---------------------------\n"
+#define SPECIAL_LOG_HDR "\n-------------------- Special log ---------------------------\n"
+#define DHD_DUMP_LOG_HDR "\n-------------------- 'dhd dump' log -----------------------\n"
+#define EXT_TRAP_LOG_HDR "\n-------------------- Extended trap data -------------------\n"
+#define HEALTH_CHK_LOG_HDR "\n-------------------- Health check data --------------------\n"
+#ifdef DHD_DUMP_PCIE_RINGS
+#define RING_DUMP_HDR "\n-------------------- Ring dump --------------------\n"
+#endif /* DHD_DUMP_PCIE_RINGS */
+#define DHD_LOG_DUMP_DLD(fmt, ...) \
+	dhd_log_dump_write(DLD_BUF_TYPE_GENERAL, NULL, 0, fmt, ##__VA_ARGS__)
+#define DHD_LOG_DUMP_DLD_EX(fmt, ...) \
+	dhd_log_dump_write(DLD_BUF_TYPE_SPECIAL, NULL, 0, fmt, ##__VA_ARGS__)
+#define DHD_LOG_DUMP_DLD_PRSRV(fmt, ...) \
+	dhd_log_dump_write(DLD_BUF_TYPE_PRESERVE, NULL, 0, fmt, ##__VA_ARGS__)
+#endif /* !_DHD_LOG_DUMP_DEFINITIONS_ */
+
+#ifndef DHD_LOG_DUMP_RING_DEFINITIONS
+#define DHD_LOG_DUMP_RING_DEFINITIONS
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+/* Enabled DHD_DEBUGABILITY_LOG_DUMP_RING */
+#define DHD_DBG_RING(fmt, ...) \
+	dhd_dbg_ring_write(DRIVER_LOG_RING_ID, NULL, 0, fmt, ##__VA_ARGS__)
+#define DHD_DBG_RING_EX(fmt, ...) \
+	dhd_dbg_ring_write(FW_VERBOSE_RING_ID, NULL, 0, fmt, ##__VA_ARGS__)
+#define DHD_DBG_RING_ROAM(fmt, ...) \
+	dhd_dbg_ring_write(ROAM_STATS_RING_ID, NULL, 0, fmt, ##__VA_ARGS__)
+
+#define DHD_LOG_DUMP_WRITE		DHD_DBG_RING
+#define DHD_LOG_DUMP_WRITE_EX		DHD_DBG_RING_EX
+#define DHD_LOG_DUMP_WRITE_PRSRV	DHD_DBG_RING_EX
+#define DHD_LOG_DUMP_WRITE_ROAM		DHD_DBG_RING_ROAM
+
+#define DHD_PREFIX_TS "[%s][%s] ",	\
+	OSL_GET_RTCTIME(), dhd_log_dump_get_timestamp()
+#define DHD_PREFIX_TS_FN "[%s][%s] %s: ",	\
+	OSL_GET_RTCTIME(), dhd_log_dump_get_timestamp(), __func__
+#define DHD_LOG_DUMP_WRITE_TS		DHD_DBG_RING(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_TS_FN	DHD_DBG_RING(DHD_PREFIX_TS_FN)
+#define DHD_LOG_DUMP_WRITE_EX_TS	DHD_DBG_RING_EX(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_EX_TS_FN	DHD_DBG_RING_EX(DHD_PREFIX_TS_FN)
+#define DHD_LOG_DUMP_WRITE_PRSRV_TS	DHD_DBG_RING_EX(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_PRSRV_TS_FN	DHD_DBG_RING_EX(DHD_PREFIX_TS_FN)
+#define DHD_LOG_DUMP_WRITE_ROAM_TS	DHD_DBG_RING_ROAM(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_ROAM_TS_FN	DHD_DBG_RING_ROAM(DHD_PREFIX_TS_FN)
+#else
+/* Not enabled DHD_DEBUGABILITY_LOG_DUMP_RING */
+#define DHD_LOG_DUMP_WRITE             DHD_LOG_DUMP_DLD
+#define DHD_LOG_DUMP_WRITE_EX          DHD_LOG_DUMP_DLD_EX
+#define DHD_LOG_DUMP_WRITE_PRSRV       DHD_LOG_DUMP_DLD_PRSRV
+#define DHD_LOG_DUMP_WRITE_ROAM                DHD_LOG_DUMP_DLD
+
+#define DHD_PREFIX_TS "[%s]: ", dhd_log_dump_get_timestamp()
+#define DHD_PREFIX_TS_FN "[%s] %s: ", dhd_log_dump_get_timestamp(), __func__
+#define DHD_LOG_DUMP_WRITE_TS          DHD_LOG_DUMP_DLD(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_TS_FN       DHD_LOG_DUMP_DLD(DHD_PREFIX_TS_FN)
+#define DHD_LOG_DUMP_WRITE_EX_TS       DHD_LOG_DUMP_DLD_EX(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_EX_TS_FN    DHD_LOG_DUMP_DLD_EX(DHD_PREFIX_TS_FN)
+#define DHD_LOG_DUMP_WRITE_PRSRV_TS    DHD_LOG_DUMP_DLD_PRSRV(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_PRSRV_TS_FN DHD_LOG_DUMP_DLD_PRSRV(DHD_PREFIX_TS_FN)
+#define DHD_LOG_DUMP_WRITE_ROAM_TS     DHD_LOG_DUMP_DLD(DHD_PREFIX_TS)
+#define DHD_LOG_DUMP_WRITE_ROAM_TS_FN  DHD_LOG_DUMP_DLD(DHD_PREFIX_TS_FN)
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+#endif /* DHD_LOG_DUMP_RING_DEFINITIONS */
+
+#define CONCISE_DUMP_BUFLEN 32 * 1024
+#define ECNTRS_LOG_HDR "\n-------------------- Ecounters log --------------------------\n"
+#ifdef DHD_STATUS_LOGGING
+#define STATUS_LOG_HDR "\n-------------------- Status log -----------------------\n"
+#endif /* DHD_STATUS_LOGGING */
+#define RTT_LOG_HDR "\n-------------------- RTT log --------------------------\n"
+#define BCM_TRACE_LOG_HDR "\n-------------------- BCM Trace log --------------------------\n"
+#define COOKIE_LOG_HDR "\n-------------------- Cookie List ----------------------------\n"
+#define DHD_PKTID_MAP_LOG_HDR "\n---------------- PKTID MAP log -----------------------\n"
+#define DHD_PKTID_UNMAP_LOG_HDR "\n------------------ PKTID UNMAP log -----------------------\n"
+#define PKTID_LOG_DUMP_FMT \
+	"\nIndex \t\tTimestamp \tPktaddr(PA) \tPktid \tSize \tPkttype\n(Current=%d)\n"
+
+/* 0: DLD_BUF_TYPE_GENERAL, 1: DLD_BUF_TYPE_PRESERVE
+* 2: DLD_BUF_TYPE_SPECIAL
+*/
+#define DLD_BUFFER_NUM 3
+
+#ifndef CUSTOM_LOG_DUMP_BUFSIZE_MB
+#define CUSTOM_LOG_DUMP_BUFSIZE_MB	4 /* DHD_LOG_DUMP_BUF_SIZE 4 MB static memory in kernel */
+#endif /* CUSTOM_LOG_DUMP_BUFSIZE_MB */
+
+#define LOG_DUMP_TOTAL_BUFSIZE (1024 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+
+/*
+ * Below are different sections that use the prealloced buffer
+ * and sum of the sizes of these should not cross LOG_DUMP_TOTAL_BUFSIZE
+ */
+#ifdef EWP_BCM_TRACE
+#define LOG_DUMP_GENERAL_MAX_BUFSIZE (192 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+#define LOG_DUMP_BCM_TRACE_MAX_BUFSIZE (64 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+#else
+#define LOG_DUMP_GENERAL_MAX_BUFSIZE (256 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+#define LOG_DUMP_BCM_TRACE_MAX_BUFSIZE 0
+#endif /* EWP_BCM_TRACE */
+#define LOG_DUMP_PRESERVE_MAX_BUFSIZE (128 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+#define LOG_DUMP_ECNTRS_MAX_BUFSIZE (256 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+#define LOG_DUMP_RTT_MAX_BUFSIZE (256 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+#define LOG_DUMP_FILTER_MAX_BUFSIZE (128 * 1024 * CUSTOM_LOG_DUMP_BUFSIZE_MB)
+
+#if LOG_DUMP_TOTAL_BUFSIZE < (LOG_DUMP_GENERAL_MAX_BUFSIZE + \
+	LOG_DUMP_PRESERVE_MAX_BUFSIZE + LOG_DUMP_ECNTRS_MAX_BUFSIZE + LOG_DUMP_RTT_MAX_BUFSIZE \
+	+ LOG_DUMP_BCM_TRACE_MAX_BUFSIZE + LOG_DUMP_FILTER_MAX_BUFSIZE)
+#error "LOG_DUMP_TOTAL_BUFSIZE is lesser than sum of all rings"
+#endif
+
+/* Special buffer is allocated as separately in prealloc */
+#define LOG_DUMP_SPECIAL_MAX_BUFSIZE (8 * 1024)
+
+#define LOG_DUMP_MAX_FILESIZE (8 *1024 * 1024) /* 8 MB default */
+
+#ifdef CONFIG_LOG_BUF_SHIFT
+/* 15% of kernel log buf size, if for example klog buf size is 512KB
+* 15% of 512KB ~= 80KB
+*/
+#define LOG_DUMP_KERNEL_TAIL_FLUSH_SIZE \
+	(15 * ((1 << CONFIG_LOG_BUF_SHIFT)/100))
+#endif /* CONFIG_LOG_BUF_SHIFT */
+
+#define LOG_DUMP_COOKIE_BUFSIZE	1024u
+#define DHD_PRINT_BUF_NAME_LEN 30
+
+#ifdef DHD_SSSR_DUMP
+#define DUMP_SSSR_ATTR_START	2
+#define DUMP_SSSR_ATTR_COUNT	10
+
+typedef enum {
+	SSSR_C0_D11_BEFORE = 0,
+	SSSR_C0_D11_AFTER = 1,
+	SSSR_C1_D11_BEFORE = 2,
+	SSSR_C1_D11_AFTER = 3,
+	SSSR_C2_D11_BEFORE = 4,
+	SSSR_C2_D11_AFTER = 5,
+	SSSR_DIG_BEFORE = 6,
+	SSSR_DIG_AFTER = 7
+} EWP_SSSR_DUMP;
+#endif /* DHD_SSSR_DUMP */
+
+typedef enum {
+	DLD_BUF_TYPE_GENERAL = 0,
+	DLD_BUF_TYPE_PRESERVE = 1,
+	DLD_BUF_TYPE_SPECIAL = 2,
+	DLD_BUF_TYPE_ECNTRS = 3,
+	DLD_BUF_TYPE_FILTER = 4,
+	DLD_BUF_TYPE_ALL = 5
+} log_dump_type_t;
+
+/*
+ * XXX: Always add new enums at the end to compatible with parser,
+ * also add new section in split_ret of EWP_config.py
+ */
+typedef enum {
+	LOG_DUMP_SECTION_GENERAL = 0,
+	LOG_DUMP_SECTION_ECNTRS,
+	LOG_DUMP_SECTION_SPECIAL,
+	LOG_DUMP_SECTION_DHD_DUMP,
+	LOG_DUMP_SECTION_EXT_TRAP,
+	LOG_DUMP_SECTION_HEALTH_CHK,
+	LOG_DUMP_SECTION_PRESERVE,
+	LOG_DUMP_SECTION_COOKIE,
+	LOG_DUMP_SECTION_RING,
+	LOG_DUMP_SECTION_STATUS,
+	LOG_DUMP_SECTION_RTT,
+	LOG_DUMP_SECTION_BCM_TRACE,
+	LOG_DUMP_SECTION_PKTID_MAP_LOG,
+	LOG_DUMP_SECTION_PKTID_UNMAP_LOG,
+	LOG_DUMP_SECTION_TIMESTAMP,
+	LOG_DUMP_SECTION_MAX
+} log_dump_section_type_t;
+
+/* Each section in the debug_dump log file shall begin with a header */
+typedef struct {
+	uint32 magic;  /* 0xDEB3DEB3 */
+	uint32 type;   /* of type log_dump_section_type_t */
+	uint64 timestamp;
+	uint32 length;  /* length of the section that follows */
+} log_dump_section_hdr_t;
+
+#ifdef DHD_DEBUGABILITY_LOG_DUMP_RING
+struct dhd_dbg_ring_buf
+{
+	void *dhd_pub;
+};
+extern struct dhd_dbg_ring_buf g_ring_buf;
+#endif /* DHD_DEBUGABILITY_LOG_DUMP_RING */
+
+typedef struct dhd_debug_dump_ring_entry {
+	log_dump_section_type_t type;
+	uint32 debug_dump_ring;
+} dhd_debug_dump_ring_entry_t;
+
+/* below structure describe ring buffer. */
+struct dhd_log_dump_buf
+{
+#if defined(LINUX) || defined(linux) || defined(ANDROID) || defined(OEM_ANDROID)
+	spinlock_t lock;
+#endif
+	void *dhd_pub;
+	unsigned int enable;
+	unsigned int wraparound;
+	unsigned long max;
+	unsigned int remain;
+	char* present;
+	char* front;
+	char* buffer;
+};
+
+typedef struct {
+	char *hdr_str;
+	log_dump_section_type_t sec_type;
+} dld_hdr_t;
+
+extern void dhd_log_dump_write(int type, char *binary_data,
+		int binary_len, const char *fmt, ...);
+void dhd_schedule_log_dump(dhd_pub_t *dhdp, void *type);
+void dhd_log_dump_trigger(dhd_pub_t *dhdp, int subcmd);
+void dhd_log_dump_vendor_trigger(dhd_pub_t *dhd_pub);
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+int dhd_debug_dump_get_ring_num(int sec_type);
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+int dhd_log_dump_ring_to_file(dhd_pub_t *dhdp, void *ring_ptr, void *file,
+		unsigned long *file_posn, log_dump_section_hdr_t *sec_hdr, char *text_hdr,
+		uint32 sec_type);
+int dhd_dump_debug_ring(dhd_pub_t *dhdp, void *ring_ptr, const void *user_buf,
+		log_dump_section_hdr_t *sec_hdr, char *text_hdr, int buflen, uint32 sec_type);
+int dhd_log_dump_cookie_to_file(dhd_pub_t *dhdp, void *fp,
+	const void *user_buf, unsigned long *f_pos);
+int dhd_log_dump_cookie(dhd_pub_t *dhdp, const void *user_buf);
+uint32 dhd_log_dump_cookie_len(dhd_pub_t *dhdp);
+int dhd_logdump_cookie_init(dhd_pub_t *dhdp, uint8 *buf, uint32 buf_size);
+void dhd_logdump_cookie_deinit(dhd_pub_t *dhdp);
+void dhd_logdump_cookie_save(dhd_pub_t *dhdp, char *cookie, char *type);
+int dhd_logdump_cookie_get(dhd_pub_t *dhdp, char *ret_cookie, uint32 buf_size);
+int dhd_logdump_cookie_count(dhd_pub_t *dhdp);
+int dhd_get_dld_log_dump(void *dev, dhd_pub_t *dhdp, const void *user_buf, void *fp,
+	uint32 len, int type, void *pos);
+#if defined(BCMPCIE)
+int dhd_print_ext_trap_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+uint32 dhd_get_ext_trap_len(void *ndev, dhd_pub_t *dhdp);
+#endif /* BCMPCIE */
+int dhd_print_dump_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+int dhd_print_cookie_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+int dhd_print_health_chk_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+int dhd_print_time_str(const void *user_buf, void *fp, uint32 len, void *pos);
+#ifdef DHD_DUMP_PCIE_RINGS
+int dhd_print_flowring_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+uint32 dhd_get_flowring_len(void *ndev, dhd_pub_t *dhdp);
+#endif /* DHD_DUMP_PCIE_RINGS */
+#ifdef DHD_STATUS_LOGGING
+extern int dhd_print_status_log_data(void *dev, dhd_pub_t *dhdp,
+	const void *user_buf, void *fp, uint32 len, void *pos);
+extern uint32 dhd_get_status_log_len(void *ndev, dhd_pub_t *dhdp);
+#endif /* DHD_STATUS_LOGGING */
+#ifdef DHD_MAP_PKTID_LOGGING
+extern uint32 dhd_pktid_buf_len(dhd_pub_t *dhd, bool is_map);
+extern int dhd_print_pktid_map_log_data(void *dev, dhd_pub_t *dhdp,
+	const void *user_buf, void *fp, uint32 len, void *pos, bool is_map);
+extern int dhd_write_pktid_log_dump(dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, unsigned long *pos, bool is_map);
+extern uint32 dhd_get_pktid_map_logging_len(void *ndev, dhd_pub_t *dhdp, bool is_map);
+#endif /* DHD_MAP_PKTID_LOGGING */
+int dhd_print_ecntrs_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+int dhd_print_rtt_data(void *dev, dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, void *pos);
+int dhd_get_debug_dump_file_name(void *dev, dhd_pub_t *dhdp,
+	char *dump_path, int size);
+uint32 dhd_get_time_str_len(void);
+uint32 dhd_get_health_chk_len(void *ndev, dhd_pub_t *dhdp);
+uint32 dhd_get_dhd_dump_len(void *ndev, dhd_pub_t *dhdp);
+uint32 dhd_get_cookie_log_len(void *ndev, dhd_pub_t *dhdp);
+uint32 dhd_get_ecntrs_len(void *ndev, dhd_pub_t *dhdp);
+uint32 dhd_get_rtt_len(void *ndev, dhd_pub_t *dhdp);
+uint32 dhd_get_dld_len(int log_type);
+void dhd_init_sec_hdr(log_dump_section_hdr_t *sec_hdr);
+extern char *dhd_log_dump_get_timestamp(void);
+bool dhd_log_dump_ecntr_enabled(void);
+bool dhd_log_dump_rtt_enabled(void);
+void dhd_nla_put_sssr_dump_len(void *ndev, uint32 *arr_len);
+int dhd_get_debug_dump(void *dev, const void *user_buf, uint32 len, int type);
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+int
+dhd_sssr_dump_d11_buf_before(void *dev, const void *user_buf, uint32 len, int core);
+int
+dhd_sssr_dump_dig_buf_before(void *dev, const void *user_buf, uint32 len);
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+int
+dhd_sssr_dump_d11_buf_after(void *dev, const void *user_buf, uint32 len, int core);
+int
+dhd_sssr_dump_dig_buf_after(void *dev, const void *user_buf, uint32 len);
+#ifdef DHD_PKT_LOGGING
+extern int dhd_os_get_pktlog_dump(void *dev, const void *user_buf, uint32 len);
+extern spinlock_t* dhd_os_get_pktlog_lock(dhd_pub_t *dhdp);
+extern uint32 dhd_os_get_pktlog_dump_size(struct net_device *dev);
+extern void dhd_os_get_pktlogdump_filename(struct net_device *dev, char *dump_path, int len);
+#endif /* DHD_PKT_LOGGING */
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+extern int dhd_os_get_axi_error_dump(void *dev, const void *user_buf, uint32 len);
+extern int dhd_os_get_axi_error_dump_size(struct net_device *dev);
+extern void dhd_os_get_axi_error_filename(struct net_device *dev, char *dump_path, int len);
+#endif /*  DNGL_AXI_ERROR_LOGGING */
+
+#ifdef DHD_DEBUGABILITY_DEBUG_DUMP
+extern int dhd_debug_dump_to_ring(dhd_pub_t *dhdp);
+#endif /* DHD_DEBUGABILITY_DEBUG_DUMP */
+
+extern char *dhd_log_dump_get_timestamp(void);
+extern void dhd_dbg_ring_write(int type, char *binary_data,
+	int binary_len, const char *fmt, ...);
+
+void dhd_log_dump_init(dhd_pub_t *dhd);
+void dhd_log_dump_deinit(dhd_pub_t *dhd);
+void dhd_log_dump(void *handle, void *event_info, uint8 event);
+int do_dhd_log_dump(dhd_pub_t *dhdp, log_dump_type_t *type);
+void dhd_print_buf_addr(dhd_pub_t *dhdp, char *name, void *buf, unsigned int size);
+void dhd_log_dump_buf_addr(dhd_pub_t *dhdp, log_dump_type_t *type);
+int dhd_log_flush(dhd_pub_t *dhdp, log_dump_type_t *type);
+
+extern void get_debug_dump_time(char *str);
+extern void clear_debug_dump_time(char *str);
+#if defined(WL_CFGVENDOR_SEND_HANG_EVENT) || defined(DHD_PKT_LOGGING)
+extern void copy_debug_dump_time(char *dest, char *src);
+#endif
+
+void dhd_get_debug_dump_len(void *handle, struct sk_buff *skb, void *event_info, uint8 event);
+void cfgvendor_log_dump_len(dhd_pub_t *dhdp, log_dump_type_t *type, struct sk_buff *skb);
+#ifdef DHD_IOVAR_LOG_FILTER_DUMP
+bool dhd_iovar_log_dump_check(dhd_pub_t *dhd_pub, uint32 cmd, char *msg);
+#endif /* DHD_IOVAR_LOG_FILTER_DUMP */
+#endif /* DHD_LOG_DUMP */
+
+#endif /* !__DHD_LOG_DUMP_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_mschdbg.c b/drivers/net/wireless/ap12275/dhd_mschdbg.c
new file mode 100644
index 000000000000..167fa19e08fc
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_mschdbg.c
@@ -0,0 +1,796 @@
+/*
+ * DHD debugability support
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id: dhd_mschdbg.c 639872 2016-05-25 05:39:30Z $
+ */
+#ifdef SHOW_LOGTRACE
+#include <typedefs.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_debug.h>
+#include <dhd_mschdbg.h>
+
+#include <event_log.h>
+#include <event_trace.h>
+#include <msgtrace.h>
+
+static const char *head_log = "";
+#define MSCH_EVENT_HEAD(space) \
+	do { \
+		MSCH_EVENT(("%s_E:  ", head_log)); \
+		if (space > 0) { \
+			int ii; \
+			for (ii = 0; ii < space; ii += 4) MSCH_EVENT(("    ")); \
+		} \
+	} while (0)
+
+#if defined(CUSTOM_PREFIX)
+#define MSCH_EVENT(args) \
+do {	\
+	if (dhd_msg_level & DHD_EVENT_VAL) {	\
+		DBG_PRINT_SYSTEM_TIME;  \
+		pr_cont args;	\
+	}	\
+} while (0)
+#else
+#define MSCH_EVENT(args) do {if (dhd_msg_level & DHD_EVENT_VAL) printf args;} while (0)
+#endif
+
+static uint64 solt_start_time[4], req_start_time[4], profiler_start_time[4];
+static uint32 solt_chanspec[4] = {0, }, req_start[4] = {0, };
+static bool lastMessages = FALSE;
+
+#define US_PRE_SEC		1000000
+#define DATA_UNIT_FOR_LOG_CNT	4
+
+static void dhd_mschdbg_us_to_sec(uint32 time_h, uint32 time_l, uint32 *sec, uint32 *remain)
+{
+	uint64 cur_time = ((uint64)(ntoh32(time_h)) << 32) | ntoh32(time_l);
+	uint64 r, u = 0;
+
+	r = cur_time;
+	while (time_h != 0) {
+		u += (uint64)((0xffffffff / US_PRE_SEC)) * time_h;
+		r = cur_time - u * US_PRE_SEC;
+		time_h = (uint32)(r >> 32);
+	}
+
+	*sec = (uint32)(u + ((uint32)(r) / US_PRE_SEC));
+	*remain = (uint32)(r) % US_PRE_SEC;
+}
+
+static char *dhd_mschdbg_display_time(uint32 time_h, uint32 time_l)
+{
+	static char display_time[32];
+	uint32 s, ss;
+
+	if (time_h == 0xffffffff && time_l == 0xffffffff) {
+		snprintf(display_time, 31, "-1");
+	} else {
+		dhd_mschdbg_us_to_sec(time_h, time_l, &s, &ss);
+		snprintf(display_time, 31, "%d.%06d", s, ss);
+	}
+	return display_time;
+}
+
+static void
+dhd_mschdbg_chanspec_list(int sp, char *data, uint16 ptr, uint16 chanspec_cnt)
+{
+	int i, cnt = (int)ntoh16(chanspec_cnt);
+	uint16 *chanspec_list = (uint16 *)(data + ntoh16(ptr));
+	char buf[CHANSPEC_STR_LEN];
+	chanspec_t c;
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<chanspec_list>:"));
+	for (i = 0; i < cnt; i++) {
+		c = (chanspec_t)ntoh16(chanspec_list[i]);
+		MSCH_EVENT((" %s", wf_chspec_ntoa(c, buf)));
+	}
+	MSCH_EVENT(("\n"));
+}
+
+static void
+dhd_mschdbg_elem_list(int sp, char *title, char *data, uint16 ptr, uint16 list_cnt)
+{
+	int i, cnt = (int)ntoh16(list_cnt);
+	uint32 *list = (uint32 *)(data + ntoh16(ptr));
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("%s_list: ", title));
+	for (i = 0; i < cnt; i++) {
+		MSCH_EVENT(("0x%08x->", ntoh32(list[i])));
+	}
+	MSCH_EVENT(("null\n"));
+}
+
+static void
+dhd_mschdbg_req_param_profiler_event_data(int sp, int ver, char *data, uint16 ptr)
+{
+	int sn = sp + 4;
+	msch_req_param_profiler_event_data_t *p =
+		(msch_req_param_profiler_event_data_t *)(data + ntoh16(ptr));
+	uint32 type, flags;
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<request parameters>\n"));
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("req_type: "));
+
+	type = p->req_type;
+	if (type < 4) {
+		char *req_type[] = {"fixed", "start-flexible", "duration-flexible",
+			"both-flexible"};
+		MSCH_EVENT(("%s", req_type[type]));
+	}
+	else
+		MSCH_EVENT(("unknown(%d)", type));
+
+	flags = ntoh16(p->flags);
+	if (flags & WL_MSCH_REQ_FLAGS_CHAN_CONTIGUOUS)
+		MSCH_EVENT((", CHAN_CONTIGUOUS"));
+	if (flags & WL_MSCH_REQ_FLAGS_MERGE_CONT_SLOTS)
+		MSCH_EVENT((", MERGE_CONT_SLOTS"));
+	if (flags & WL_MSCH_REQ_FLAGS_PREMTABLE)
+		MSCH_EVENT((", PREMTABLE"));
+	if (flags & WL_MSCH_REQ_FLAGS_PREMT_CURTS)
+		MSCH_EVENT((", PREMT_CURTS"));
+	if (flags & WL_MSCH_REQ_FLAGS_PREMT_IMMEDIATE)
+		MSCH_EVENT((", PREMT_IMMEDIATE"));
+	MSCH_EVENT((", priority: %d\n", p->priority));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("start-time: %s, duration: %d(us), interval: %d(us)\n",
+		dhd_mschdbg_display_time(p->start_time_h, p->start_time_l),
+		ntoh32(p->duration), ntoh32(p->interval)));
+
+	if (type == WL_MSCH_RT_DUR_FLEX) {
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("dur_flex: %d(us)\n", ntoh32(p->flex.dur_flex)));
+	} else if (type == WL_MSCH_RT_BOTH_FLEX) {
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("min_dur: %d(us), max_away_dur: %d(us)\n",
+			ntoh32(p->flex.bf.min_dur), ntoh32(p->flex.bf.max_away_dur)));
+
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("hi_prio_time: %s, hi_prio_interval: %d(us)\n",
+			dhd_mschdbg_display_time(p->flex.bf.hi_prio_time_h,
+			p->flex.bf.hi_prio_time_l),
+			ntoh32(p->flex.bf.hi_prio_interval)));
+	}
+}
+
+static void
+dhd_mschdbg_timeslot_profiler_event_data(int sp, int ver, char *title, char *data,
+	uint16 ptr, bool empty)
+{
+	int s, sn = sp + 4;
+	msch_timeslot_profiler_event_data_t *p =
+		(msch_timeslot_profiler_event_data_t *)(data + ntoh16(ptr));
+	char *state[] = {"NONE", "CHN_SW", "ONCHAN_FIRE", "OFF_CHN_PREP",
+		"OFF_CHN_DONE", "TS_COMPLETE"};
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<%s timeslot>: ", title));
+	if (empty) {
+		MSCH_EVENT((" null\n"));
+		return;
+	}
+	else
+		MSCH_EVENT(("0x%08x\n", ntoh32(p->p_timeslot)));
+
+	s = (int)(ntoh32(p->state));
+	if (s < 0 || s > 5) s = 0;
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("id: %d, state[%d]: %s, chan_ctxt: [0x%08x]\n",
+		ntoh32(p->timeslot_id), ntoh32(p->state), state[s], ntoh32(p->p_chan_ctxt)));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("fire_time: %s",
+		dhd_mschdbg_display_time(p->fire_time_h, p->fire_time_l)));
+
+	MSCH_EVENT((", pre_start_time: %s",
+		dhd_mschdbg_display_time(p->pre_start_time_h, p->pre_start_time_l)));
+
+	MSCH_EVENT((", end_time: %s",
+		dhd_mschdbg_display_time(p->end_time_h, p->end_time_l)));
+
+	MSCH_EVENT((", sch_dur: %s\n",
+		dhd_mschdbg_display_time(p->sch_dur_h, p->sch_dur_l)));
+}
+
+static void
+dhd_mschdbg_req_timing_profiler_event_data(int sp, int ver, char *title, char *data,
+	uint16 ptr, bool empty)
+{
+	int sn = sp + 4;
+	msch_req_timing_profiler_event_data_t *p =
+		(msch_req_timing_profiler_event_data_t *)(data + ntoh16(ptr));
+	uint32 type;
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<%s req_timing>: ", title));
+	if (empty) {
+		MSCH_EVENT((" null\n"));
+		return;
+	}
+	else
+		MSCH_EVENT(("0x%08x (prev 0x%08x, next 0x%08x)\n",
+			ntoh32(p->p_req_timing), ntoh32(p->p_prev), ntoh32(p->p_next)));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("flags:"));
+	type = ntoh16(p->flags);
+	if ((type & 0x7f) == 0)
+		MSCH_EVENT((" NONE"));
+	else {
+		if (type & WL_MSCH_RC_FLAGS_ONCHAN_FIRE)
+			MSCH_EVENT((" ONCHAN_FIRE"));
+		if (type & WL_MSCH_RC_FLAGS_START_FIRE_DONE)
+			MSCH_EVENT((" START_FIRE"));
+		if (type & WL_MSCH_RC_FLAGS_END_FIRE_DONE)
+			MSCH_EVENT((" END_FIRE"));
+		if (type & WL_MSCH_RC_FLAGS_ONFIRE_DONE)
+			MSCH_EVENT((" ONFIRE_DONE"));
+		if (type & WL_MSCH_RC_FLAGS_SPLIT_SLOT_START)
+			MSCH_EVENT((" SPLIT_SLOT_START"));
+		if (type & WL_MSCH_RC_FLAGS_SPLIT_SLOT_END)
+			MSCH_EVENT((" SPLIT_SLOT_END"));
+		if (type & WL_MSCH_RC_FLAGS_PRE_ONFIRE_DONE)
+			MSCH_EVENT((" PRE_ONFIRE_DONE"));
+	}
+	MSCH_EVENT(("\n"));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("pre_start_time: %s",
+		dhd_mschdbg_display_time(p->pre_start_time_h, p->pre_start_time_l)));
+
+	MSCH_EVENT((", start_time: %s",
+		dhd_mschdbg_display_time(p->start_time_h, p->start_time_l)));
+
+	MSCH_EVENT((", end_time: %s\n",
+		dhd_mschdbg_display_time(p->end_time_h, p->end_time_l)));
+
+	if (p->p_timeslot && (p->timeslot_ptr == 0)) {
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("<%s timeslot>: 0x%08x\n", title, ntoh32(p->p_timeslot)));
+	} else
+		dhd_mschdbg_timeslot_profiler_event_data(sn, ver, title, data, p->timeslot_ptr,
+			(p->timeslot_ptr == 0));
+}
+
+static void
+dhd_mschdbg_chan_ctxt_profiler_event_data(int sp, int ver, char *data, uint16 ptr, bool empty)
+{
+	int sn = sp + 4;
+	msch_chan_ctxt_profiler_event_data_t *p =
+		(msch_chan_ctxt_profiler_event_data_t *)(data + ntoh16(ptr));
+	chanspec_t c;
+	char buf[CHANSPEC_STR_LEN];
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<chan_ctxt>: "));
+	if (empty) {
+		MSCH_EVENT((" null\n"));
+		return;
+	}
+	else
+		MSCH_EVENT(("0x%08x (prev 0x%08x, next 0x%08x)\n",
+			ntoh32(p->p_chan_ctxt), ntoh32(p->p_prev), ntoh32(p->p_next)));
+
+	c = (chanspec_t)ntoh16(p->chanspec);
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("channel: %s, bf_sch_pending: %s, bf_skipped: %d\n",
+		wf_chspec_ntoa(c, buf), p->bf_sch_pending? "TRUE" : "FALSE",
+		ntoh32(p->bf_skipped_count)));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("bf_link: prev 0x%08x, next 0x%08x\n",
+		ntoh32(p->bf_link_prev), ntoh32(p->bf_link_next)));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("onchan_time: %s",
+		dhd_mschdbg_display_time(p->onchan_time_h, p->onchan_time_l)));
+	MSCH_EVENT((", actual_onchan_dur: %s",
+		dhd_mschdbg_display_time(p->actual_onchan_dur_h, p->actual_onchan_dur_l)));
+	MSCH_EVENT((", pend_onchan_dur: %s\n",
+		dhd_mschdbg_display_time(p->pend_onchan_dur_h, p->pend_onchan_dur_l)));
+
+	dhd_mschdbg_elem_list(sn, "req_entity", data, p->req_entity_list_ptr,
+		p->req_entity_list_cnt);
+	dhd_mschdbg_elem_list(sn, "bf_entity", data, p->bf_entity_list_ptr,
+		p->bf_entity_list_cnt);
+}
+
+static void
+dhd_mschdbg_req_entity_profiler_event_data(int sp, int ver, char *data, uint16 ptr, bool empty)
+{
+	int sn = sp + 4;
+	msch_req_entity_profiler_event_data_t *p =
+		(msch_req_entity_profiler_event_data_t *)(data + ntoh16(ptr));
+	char buf[CHANSPEC_STR_LEN];
+	chanspec_t c;
+	uint32 flags;
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<req_entity>: "));
+	if (empty) {
+		MSCH_EVENT((" null\n"));
+		return;
+	}
+	else
+		MSCH_EVENT(("0x%08x (prev 0x%08x, next 0x%08x)\n",
+			ntoh32(p->p_req_entity), ntoh32(p->req_hdl_link_prev),
+			ntoh32(p->req_hdl_link_next)));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("req_hdl: [0x%08x]\n", ntoh32(p->p_req_hdl)));
+
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("chan_ctxt_link: prev 0x%08x, next 0x%08x\n",
+		ntoh32(p->chan_ctxt_link_prev), ntoh32(p->chan_ctxt_link_next)));
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("rt_specific_link: prev 0x%08x, next 0x%08x\n",
+		ntoh32(p->rt_specific_link_prev), ntoh32(p->rt_specific_link_next)));
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("start_fixed_link: prev 0x%08x, next 0x%08x\n",
+		ntoh32(p->start_fixed_link_prev), ntoh32(p->start_fixed_link_next)));
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("both_flex_list: prev 0x%08x, next 0x%08x\n",
+		ntoh32(p->both_flex_list_prev), ntoh32(p->both_flex_list_next)));
+
+	c = (chanspec_t)ntoh16(p->chanspec);
+	MSCH_EVENT_HEAD(sn);
+	if (ver >= 2) {
+		MSCH_EVENT(("channel: %s, onchan Id %d, current chan Id %d, priority %d",
+			wf_chspec_ntoa(c, buf), ntoh16(p->onchan_chn_idx), ntoh16(p->cur_chn_idx),
+			ntoh16(p->priority)));
+		flags = ntoh32(p->flags);
+		if (flags & WL_MSCH_ENTITY_FLAG_MULTI_INSTANCE)
+			MSCH_EVENT((" : MULTI_INSTANCE\n"));
+		else
+			MSCH_EVENT(("\n"));
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("actual_start_time: %s, ",
+			dhd_mschdbg_display_time(p->actual_start_time_h, p->actual_start_time_l)));
+		MSCH_EVENT(("curts_fire_time: %s, ",
+			dhd_mschdbg_display_time(p->curts_fire_time_h, p->curts_fire_time_l)));
+	} else {
+		MSCH_EVENT(("channel: %s, priority %d, ", wf_chspec_ntoa(c, buf),
+			ntoh16(p->priority)));
+	}
+	MSCH_EVENT(("bf_last_serv_time: %s\n",
+		dhd_mschdbg_display_time(p->bf_last_serv_time_h, p->bf_last_serv_time_l)));
+
+	dhd_mschdbg_req_timing_profiler_event_data(sn, ver, "current", data, p->cur_slot_ptr,
+		(p->cur_slot_ptr == 0));
+	dhd_mschdbg_req_timing_profiler_event_data(sn, ver, "pending", data, p->pend_slot_ptr,
+		(p->pend_slot_ptr == 0));
+
+	if (p->p_chan_ctxt && (p->chan_ctxt_ptr == 0)) {
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("<chan_ctxt>: 0x%08x\n", ntoh32(p->p_chan_ctxt)));
+	}
+	else
+		dhd_mschdbg_chan_ctxt_profiler_event_data(sn, ver, data, p->chan_ctxt_ptr,
+			(p->chan_ctxt_ptr == 0));
+}
+
+static void
+dhd_mschdbg_req_handle_profiler_event_data(int sp, int ver, char *data, uint16 ptr, bool empty)
+{
+	int sn = sp + 4;
+	msch_req_handle_profiler_event_data_t *p =
+		(msch_req_handle_profiler_event_data_t *)(data + ntoh16(ptr));
+	uint32 flags;
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("<req_handle>: "));
+	if (empty) {
+		MSCH_EVENT((" null\n"));
+		return;
+	}
+	else
+		MSCH_EVENT(("0x%08x (prev 0x%08x, next 0x%08x)\n",
+			ntoh32(p->p_req_handle), ntoh32(p->p_prev), ntoh32(p->p_next)));
+
+	dhd_mschdbg_elem_list(sn, "req_entity", data, p->req_entity_list_ptr,
+		p->req_entity_list_cnt);
+	MSCH_EVENT_HEAD(sn);
+	MSCH_EVENT(("cb_func: [0x%08x], cb_func: [0x%08x]",
+		ntoh32(p->cb_func), ntoh32(p->cb_ctxt)));
+	if (ver < 2) {
+		MSCH_EVENT((", chan_cnt: %d", ntoh16(p->chan_cnt)));
+	}
+	flags = ntoh32(p->flags);
+	if (flags & WL_MSCH_REQ_HDL_FLAGS_NEW_REQ)
+		MSCH_EVENT((", NEW_REQ"));
+	MSCH_EVENT(("\n"));
+
+	dhd_mschdbg_req_param_profiler_event_data(sn, ver, data, p->req_param_ptr);
+
+	if (ver >= 2) {
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("req_time: %s\n",
+			dhd_mschdbg_display_time(p->req_time_h, p->req_time_l)));
+		MSCH_EVENT_HEAD(sn);
+		MSCH_EVENT(("chan_cnt: %d, chan idx %d, last chan idx %d\n",
+			ntoh16(p->chan_cnt), ntoh16(p->chan_idx), ntoh16(p->last_chan_idx)));
+		if (p->chanspec_list && p->chanspec_cnt) {
+			dhd_mschdbg_chanspec_list(sn, data, p->chanspec_list, p->chanspec_cnt);
+		}
+	}
+}
+
+static void
+dhd_mschdbg_profiler_profiler_event_data(int sp, int ver, char *data, uint16 ptr)
+{
+	msch_profiler_profiler_event_data_t *p =
+		(msch_profiler_profiler_event_data_t *)(data + ntoh16(ptr));
+	uint32 flags;
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("free list: req_hdl 0x%08x, req_entity 0x%08x,"
+		" chan_ctxt 0x%08x, chanspec 0x%08x\n",
+		ntoh32(p->free_req_hdl_list), ntoh32(p->free_req_entity_list),
+		ntoh32(p->free_chan_ctxt_list), ntoh32(p->free_chanspec_list)));
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("alloc count: chanspec %d, req_entity %d, req_hdl %d, "
+		"chan_ctxt %d, timeslot %d\n",
+		ntoh16(p->msch_chanspec_alloc_cnt), ntoh16(p->msch_req_entity_alloc_cnt),
+		ntoh16(p->msch_req_hdl_alloc_cnt), ntoh16(p->msch_chan_ctxt_alloc_cnt),
+		ntoh16(p->msch_timeslot_alloc_cnt)));
+
+	dhd_mschdbg_elem_list(sp, "req_hdl", data, p->msch_req_hdl_list_ptr,
+		p->msch_req_hdl_list_cnt);
+	dhd_mschdbg_elem_list(sp, "chan_ctxt", data, p->msch_chan_ctxt_list_ptr,
+		p->msch_chan_ctxt_list_cnt);
+	dhd_mschdbg_elem_list(sp, "req_timing", data, p->msch_req_timing_list_ptr,
+		p->msch_req_timing_list_cnt);
+	dhd_mschdbg_elem_list(sp, "start_fixed", data, p->msch_start_fixed_list_ptr,
+		p->msch_start_fixed_list_cnt);
+	dhd_mschdbg_elem_list(sp, "both_flex_req_entity", data,
+		p->msch_both_flex_req_entity_list_ptr,
+		p->msch_both_flex_req_entity_list_cnt);
+	dhd_mschdbg_elem_list(sp, "start_flex", data, p->msch_start_flex_list_ptr,
+		p->msch_start_flex_list_cnt);
+	dhd_mschdbg_elem_list(sp, "both_flex", data, p->msch_both_flex_list_ptr,
+		p->msch_both_flex_list_cnt);
+
+	if (p->p_cur_msch_timeslot && (p->cur_msch_timeslot_ptr == 0)) {
+		MSCH_EVENT_HEAD(sp);
+		MSCH_EVENT(("<cur_msch timeslot>: 0x%08x\n",
+			ntoh32(p->p_cur_msch_timeslot)));
+	} else
+		dhd_mschdbg_timeslot_profiler_event_data(sp, ver, "cur_msch", data,
+			p->cur_msch_timeslot_ptr, (p->cur_msch_timeslot_ptr == 0));
+
+	if (p->p_next_timeslot && (p->next_timeslot_ptr == 0)) {
+		MSCH_EVENT_HEAD(sp);
+		MSCH_EVENT(("<next timeslot>: 0x%08x\n",
+			ntoh32(p->p_next_timeslot)));
+	} else
+		dhd_mschdbg_timeslot_profiler_event_data(sp, ver, "next", data,
+			p->next_timeslot_ptr, (p->next_timeslot_ptr == 0));
+
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("ts_id: %d, ", ntoh32(p->ts_id)));
+	flags = ntoh32(p->flags);
+	if (flags & WL_MSCH_STATE_IN_TIEMR_CTXT)
+		MSCH_EVENT(("IN_TIEMR_CTXT, "));
+	if (flags & WL_MSCH_STATE_SCHD_PENDING)
+		MSCH_EVENT(("SCHD_PENDING, "));
+	MSCH_EVENT(("slotskip_flags: %d, cur_armed_timeslot: 0x%08x\n",
+		(ver >= 2)? ntoh32(p->slotskip_flag) : 0, ntoh32(p->cur_armed_timeslot)));
+	MSCH_EVENT_HEAD(sp);
+	MSCH_EVENT(("flex_list_cnt: %d, service_interval: %d, "
+		"max_lo_prio_interval: %d\n",
+		ntoh16(p->flex_list_cnt), ntoh32(p->service_interval),
+		ntoh32(p->max_lo_prio_interval)));
+}
+
+static void dhd_mschdbg_dump_data(dhd_pub_t *dhdp, void *raw_event_ptr, int type,
+	char *data, int len)
+{
+	uint64 t = 0, tt = 0;
+	uint32 s = 0, ss = 0;
+	int wlc_index, ver;
+
+	ver = (type & WL_MSCH_PROFILER_VER_MASK) >> WL_MSCH_PROFILER_VER_SHIFT;
+	wlc_index = (type & WL_MSCH_PROFILER_WLINDEX_MASK) >> WL_MSCH_PROFILER_WLINDEX_SHIFT;
+	if (wlc_index >= 4)
+		return;
+
+	type &= WL_MSCH_PROFILER_TYPE_MASK;
+	if (type <= WL_MSCH_PROFILER_PROFILE_END) {
+		msch_profiler_event_data_t *pevent = (msch_profiler_event_data_t *)data;
+		tt = ((uint64)(ntoh32(pevent->time_hi)) << 32) | ntoh32(pevent->time_lo);
+		dhd_mschdbg_us_to_sec(pevent->time_hi, pevent->time_lo, &s, &ss);
+	}
+
+	if (lastMessages && (type != WL_MSCH_PROFILER_MESSAGE) &&
+		(type != WL_MSCH_PROFILER_EVENT_LOG)) {
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("\n"));
+		lastMessages = FALSE;
+	}
+
+	switch (type) {
+	case WL_MSCH_PROFILER_START:
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d START\n", s, ss));
+		break;
+
+	case WL_MSCH_PROFILER_EXIT:
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d EXIT\n", s, ss));
+		break;
+
+	case WL_MSCH_PROFILER_REQ:
+	{
+		msch_req_profiler_event_data_t *p = (msch_req_profiler_event_data_t *)data;
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("\n"));
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("===============================\n"));
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d [wl%d] REGISTER:\n", s, ss, wlc_index));
+		dhd_mschdbg_req_param_profiler_event_data(4, ver, data, p->req_param_ptr);
+		dhd_mschdbg_chanspec_list(4, data, p->chanspec_ptr, p->chanspec_cnt);
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("===============================\n"));
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("\n"));
+	}
+		break;
+
+	case WL_MSCH_PROFILER_CALLBACK:
+	{
+		msch_callback_profiler_event_data_t *p =
+			(msch_callback_profiler_event_data_t *)data;
+		char buf[CHANSPEC_STR_LEN];
+		chanspec_t chanspec;
+		uint16 cbtype;
+
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d [wl%d] CALLBACK: ", s, ss, wlc_index));
+		chanspec = (chanspec_t)ntoh16(p->chanspec);
+		MSCH_EVENT(("req_hdl[0x%08x], channel %s --",
+			ntoh32(p->p_req_hdl), wf_chspec_ntoa(chanspec, buf)));
+
+		cbtype = ntoh16(p->type);
+		if (cbtype & WL_MSCH_CT_ON_CHAN)
+			MSCH_EVENT((" ON_CHAN"));
+		if (cbtype & WL_MSCH_CT_OFF_CHAN)
+			MSCH_EVENT((" OFF_CHAN"));
+		if (cbtype & WL_MSCH_CT_REQ_START)
+			MSCH_EVENT((" REQ_START"));
+		if (cbtype & WL_MSCH_CT_REQ_END)
+			MSCH_EVENT((" REQ_END"));
+		if (cbtype & WL_MSCH_CT_SLOT_START)
+			MSCH_EVENT((" SLOT_START"));
+		if (cbtype & WL_MSCH_CT_SLOT_SKIP)
+			MSCH_EVENT((" SLOT_SKIP"));
+		if (cbtype & WL_MSCH_CT_SLOT_END)
+			MSCH_EVENT((" SLOT_END"));
+		if (cbtype & WL_MSCH_CT_OFF_CHAN_DONE)
+			MSCH_EVENT((" OFF_CHAN_DONE"));
+		if (cbtype & WL_MSCH_CT_PARTIAL)
+			MSCH_EVENT((" PARTIAL"));
+		if (cbtype & WL_MSCH_CT_PRE_ONCHAN)
+			MSCH_EVENT((" PRE_ONCHAN"));
+		if (cbtype & WL_MSCH_CT_PRE_REQ_START)
+			MSCH_EVENT((" PRE_REQ_START"));
+
+		if (cbtype & WL_MSCH_CT_REQ_START) {
+			req_start[wlc_index] = 1;
+			req_start_time[wlc_index] = tt;
+		} else if (cbtype & WL_MSCH_CT_REQ_END) {
+			if (req_start[wlc_index]) {
+				MSCH_EVENT((" : REQ duration %d",
+					(uint32)(tt - req_start_time[wlc_index])));
+				req_start[wlc_index] = 0;
+			}
+		}
+
+		if (cbtype & WL_MSCH_CT_SLOT_START) {
+			solt_chanspec[wlc_index] = p->chanspec;
+			solt_start_time[wlc_index] = tt;
+		} else if (cbtype & WL_MSCH_CT_SLOT_END) {
+			if (p->chanspec == solt_chanspec[wlc_index]) {
+				MSCH_EVENT((" : SLOT duration %d",
+					(uint32)(tt - solt_start_time[wlc_index])));
+				solt_chanspec[wlc_index] = 0;
+			}
+		}
+		MSCH_EVENT(("\n"));
+
+		if (cbtype & (WL_MSCH_CT_ON_CHAN | WL_MSCH_CT_SLOT_SKIP)) {
+			MSCH_EVENT_HEAD(4);
+			if (cbtype & WL_MSCH_CT_ON_CHAN) {
+				MSCH_EVENT(("ID %d onchan idx %d cur_chan_seq_start %s ",
+					ntoh32(p->timeslot_id), ntoh32(p->onchan_idx),
+					dhd_mschdbg_display_time(p->cur_chan_seq_start_time_h,
+					p->cur_chan_seq_start_time_l)));
+			}
+			t = ((uint64)(ntoh32(p->start_time_h)) << 32) |
+				ntoh32(p->start_time_l);
+			MSCH_EVENT(("start %s ",
+				dhd_mschdbg_display_time(p->start_time_h,
+				p->start_time_l)));
+			tt = ((uint64)(ntoh32(p->end_time_h)) << 32) | ntoh32(p->end_time_l);
+			MSCH_EVENT(("end %s duration %d\n",
+				dhd_mschdbg_display_time(p->end_time_h, p->end_time_l),
+				(p->end_time_h == 0xffffffff && p->end_time_l == 0xffffffff)?
+				-1 : (int)(tt - t)));
+		}
+
+	}
+		break;
+
+	case WL_MSCH_PROFILER_EVENT_LOG:
+	{
+		while (len >= (int)WL_MSCH_EVENT_LOG_HEAD_SIZE) {
+			msch_event_log_profiler_event_data_t *p =
+				(msch_event_log_profiler_event_data_t *)data;
+			/* TODO: How to parse MSCH if extended event tag is present ??? */
+			prcd_event_log_hdr_t hdr;
+			int size = WL_MSCH_EVENT_LOG_HEAD_SIZE + p->hdr.count * sizeof(uint32);
+			if (len < size || size > sizeof(msch_event_log_profiler_event_data_t)) {
+				break;
+			}
+			data += size;
+			len -= size;
+			dhd_mschdbg_us_to_sec(p->time_hi, p->time_lo, &s, &ss);
+			MSCH_EVENT_HEAD(0);
+			MSCH_EVENT(("%06d.%06d [wl%d]: ", s, ss, p->hdr.tag));
+			bzero(&hdr, sizeof(hdr));
+			hdr.tag = EVENT_LOG_TAG_MSCHPROFILE;
+			hdr.count = p->hdr.count + 1;
+			/* exclude LSB 2 bits which indicate binary/non-binary data */
+			hdr.fmt_num = ntoh16(p->hdr.fmt_num) >> 2;
+			hdr.fmt_num_raw = ntoh16(p->hdr.fmt_num);
+			if (ntoh16(p->hdr.fmt_num) == DHD_OW_BI_RAW_EVENT_LOG_FMT) {
+				hdr.binary_payload = TRUE;
+			}
+			dhd_dbg_verboselog_printf(dhdp, &hdr, raw_event_ptr, p->data, 0, 0);
+		}
+		lastMessages = TRUE;
+		break;
+	}
+
+	case WL_MSCH_PROFILER_MESSAGE:
+	{
+		msch_message_profiler_event_data_t *p = (msch_message_profiler_event_data_t *)data;
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d [wl%d]: %s", s, ss, wlc_index, p->message));
+		lastMessages = TRUE;
+		break;
+	}
+
+	case WL_MSCH_PROFILER_PROFILE_START:
+		profiler_start_time[wlc_index] = tt;
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("-------------------------------\n"));
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d [wl%d] PROFILE DATA:\n", s, ss, wlc_index));
+		dhd_mschdbg_profiler_profiler_event_data(4, ver, data, 0);
+		break;
+
+	case WL_MSCH_PROFILER_PROFILE_END:
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d [wl%d] PROFILE END: take time %d\n", s, ss,
+			wlc_index, (uint32)(tt - profiler_start_time[wlc_index])));
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("-------------------------------\n"));
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("\n"));
+		break;
+
+	case WL_MSCH_PROFILER_REQ_HANDLE:
+		dhd_mschdbg_req_handle_profiler_event_data(4, ver, data, 0, FALSE);
+		break;
+
+	case WL_MSCH_PROFILER_REQ_ENTITY:
+		dhd_mschdbg_req_entity_profiler_event_data(4, ver, data, 0, FALSE);
+		break;
+
+	case WL_MSCH_PROFILER_CHAN_CTXT:
+		dhd_mschdbg_chan_ctxt_profiler_event_data(4, ver, data, 0, FALSE);
+		break;
+
+	case WL_MSCH_PROFILER_REQ_TIMING:
+		dhd_mschdbg_req_timing_profiler_event_data(4, ver, "msch", data, 0, FALSE);
+		break;
+
+	default:
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("[wl%d] ERROR: unsupported EVENT reason code:%d; ",
+			wlc_index, type));
+		break;
+	}
+}
+
+void
+wl_mschdbg_event_handler(dhd_pub_t *dhdp, void *raw_event_ptr, int type, void *data, int len)
+{
+	head_log = "MSCH";
+	dhd_mschdbg_dump_data(dhdp, raw_event_ptr, type, (char *)data, len);
+}
+
+void
+wl_mschdbg_verboselog_handler(dhd_pub_t *dhdp, void *raw_event_ptr, prcd_event_log_hdr_t *plog_hdr,
+	uint32 *log_ptr)
+{
+	uint32 log_pyld_len;
+	head_log = "CONSOLE";
+
+	if (plog_hdr->count == 0) {
+		return;
+	}
+	log_pyld_len = (plog_hdr->count - 1) * DATA_UNIT_FOR_LOG_CNT;
+
+	if (plog_hdr->tag == EVENT_LOG_TAG_MSCHPROFILE) {
+		msch_event_log_profiler_event_data_t *p =
+			(msch_event_log_profiler_event_data_t *)log_ptr;
+		/* TODO: How to parse MSCH if extended event tag is present ??? */
+		prcd_event_log_hdr_t hdr;
+		uint32 s, ss;
+
+		if (log_pyld_len < OFFSETOF(msch_event_log_profiler_event_data_t, data) ||
+			log_pyld_len > sizeof(msch_event_log_profiler_event_data_t)) {
+			return;
+		}
+
+		dhd_mschdbg_us_to_sec(p->time_hi, p->time_lo, &s, &ss);
+		MSCH_EVENT_HEAD(0);
+		MSCH_EVENT(("%06d.%06d [wl%d]: ", s, ss, p->hdr.tag));
+		bzero(&hdr, sizeof(hdr));
+		hdr.tag = EVENT_LOG_TAG_MSCHPROFILE;
+		hdr.count = p->hdr.count + 1;
+		/* exclude LSB 2 bits which indicate binary/non-binary data */
+		hdr.fmt_num = ntoh16(p->hdr.fmt_num) >> 2;
+		hdr.fmt_num_raw = ntoh16(p->hdr.fmt_num);
+		if (ntoh16(p->hdr.fmt_num) == DHD_OW_BI_RAW_EVENT_LOG_FMT) {
+			hdr.binary_payload = TRUE;
+		}
+		dhd_dbg_verboselog_printf(dhdp, &hdr, raw_event_ptr, p->data, 0, 0);
+	} else {
+		msch_collect_tlv_t *p = (msch_collect_tlv_t *)log_ptr;
+		int type = ntoh16(p->type);
+		int len = ntoh16(p->size);
+
+		if (log_pyld_len < OFFSETOF(msch_collect_tlv_t, value) + len) {
+			return;
+		}
+
+		dhd_mschdbg_dump_data(dhdp, raw_event_ptr, type, p->value, len);
+	}
+}
+#endif /* SHOW_LOGTRACE */
diff --git a/drivers/net/wireless/ap12275/dhd_mschdbg.h b/drivers/net/wireless/ap12275/dhd_mschdbg.h
new file mode 100644
index 000000000000..533b4ebd5119
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_mschdbg.h
@@ -0,0 +1,36 @@
+/*
+ * DHD debugability header file
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id: dhd_mschdbg.h 571265 2015-07-14 20:50:18Z $
+ */
+
+#ifndef _dhd_mschdbg_h_
+#define _dhd_mschdbg_h_
+
+#ifdef SHOW_LOGTRACE
+extern void wl_mschdbg_event_handler(dhd_pub_t *dhdp, void *raw_event_ptr, int type,
+	void *data, int len);
+extern void wl_mschdbg_verboselog_handler(dhd_pub_t *dhdp, void *raw_event_ptr,
+	prcd_event_log_hdr_t *plog_hdr, uint32 *log_ptr);
+#endif /* SHOW_LOGTRACE */
+
+#endif /* _dhd_mschdbg_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_msgbuf.c b/drivers/net/wireless/ap12275/dhd_msgbuf.c
new file mode 100644
index 000000000000..3624247d80be
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_msgbuf.c
@@ -0,0 +1,14807 @@
+/**
+ * @file definition of host message ring functionality
+ * Provides type definitions and function prototypes used to link the
+ * DHD OS, bus, and protocol modules.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/** XXX Twiki: [PCIeFullDongleArchitecture] */
+
+#include <typedefs.h>
+#include <osl.h>
+
+#include <bcmutils.h>
+#include <bcmmsgbuf.h>
+#include <bcmendian.h>
+#include <bcmstdlib_s.h>
+
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_proto.h>
+
+#include <dhd_bus.h>
+
+#include <dhd_dbg.h>
+#include <siutils.h>
+#include <dhd_debug.h>
+#ifdef EXT_STA
+#include <wlc_cfg.h>
+#include <wlc_pub.h>
+#include <wl_port_if.h>
+#endif /* EXT_STA */
+
+#include <dhd_flowring.h>
+
+#include <pcie_core.h>
+#include <bcmpcie.h>
+#include <dhd_pcie.h>
+#include <dhd_config.h>
+#if defined(DHD_LB)
+#if !defined(LINUX) && !defined(linux) && !defined(OEM_ANDROID)
+#error "DHD Loadbalancing only supported on LINUX | OEM_ANDROID"
+#endif /* !LINUX && !OEM_ANDROID */
+#include <linux/cpu.h>
+#include <bcm_ring.h>
+#define DHD_LB_WORKQ_SZ			    (8192)
+#define DHD_LB_WORKQ_SYNC           (16)
+#define DHD_LB_WORK_SCHED           (DHD_LB_WORKQ_SYNC * 2)
+#endif /* DHD_LB */
+
+#include <etd.h>
+#include <hnd_debug.h>
+#include <bcmtlv.h>
+#include <hnd_armtrap.h>
+#include <dnglevent.h>
+
+#ifdef DHD_PKT_LOGGING
+#include <dhd_pktlog.h>
+#include <dhd_linux_pktdump.h>
+#endif /* DHD_PKT_LOGGING */
+#ifdef DHD_EWPR_VER2
+#include <dhd_bitpack.h>
+#endif /* DHD_EWPR_VER2 */
+
+extern char dhd_version[];
+extern char fw_version[];
+
+/**
+ * Host configures a soft doorbell for d2h rings, by specifying a 32bit host
+ * address where a value must be written. Host may also interrupt coalescing
+ * on this soft doorbell.
+ * Use Case: Hosts with network processors, may register with the dongle the
+ * network processor's thread wakeup register and a value corresponding to the
+ * core/thread context. Dongle will issue a write transaction <address,value>
+ * to the PCIE RC which will need to be routed to the mapped register space, by
+ * the host.
+ */
+/* #define DHD_D2H_SOFT_DOORBELL_SUPPORT */
+
+/* Dependency Check */
+#if defined(IOCTLRESP_USE_CONSTMEM) && defined(DHD_USE_STATIC_CTRLBUF)
+#error "DHD_USE_STATIC_CTRLBUF is NOT working with DHD_USE_OSLPKT_FOR_RESPBUF"
+#endif /* IOCTLRESP_USE_CONSTMEM && DHD_USE_STATIC_CTRLBUF */
+
+#define RETRIES 2		/* # of retries to retrieve matching ioctl response */
+
+/* For legacy firmware where pcie shared structure does not have max_host_rxbufs, use this */
+#define LEGACY_MAX_RXBUFPOST		256u
+
+/* Read index update Magic sequence */
+#define DHD_DMA_INDX_SEQ_H2D_DB_MAGIC	0xDDDDDDDDAu
+#define DHD_RDPTR_UPDATE_H2D_DB_MAGIC(ring)	(0xDD000000 | (ring->idx << 16u) | ring->rd)
+/* Write index update Magic sequence */
+#define DHD_WRPTR_UPDATE_H2D_DB_MAGIC(ring)	(0xFF000000 | (ring->idx << 16u) | ring->wr)
+#define DHD_AGGR_H2D_DB_MAGIC	0xFFFFFFFAu
+
+#define DHD_STOP_QUEUE_THRESHOLD	200
+#define DHD_START_QUEUE_THRESHOLD	100
+
+#define RX_DMA_OFFSET		8 /* Mem2mem DMA inserts an extra 8 */
+#define IOCT_RETBUF_SIZE	(RX_DMA_OFFSET + WLC_IOCTL_MAXLEN)
+
+/* flags for ioctl pending status */
+#define MSGBUF_IOCTL_ACK_PENDING	(1<<0)
+#define MSGBUF_IOCTL_RESP_PENDING	(1<<1)
+
+#define DHD_IOCTL_REQ_PKTBUFSZ		2048
+#define MSGBUF_IOCTL_MAX_RQSTLEN	(DHD_IOCTL_REQ_PKTBUFSZ - H2DRING_CTRL_SUB_ITEMSIZE)
+
+/**
+ * XXX: DMA_ALIGN_LEN use is overloaded:
+ * - as align bits: in DMA_ALLOC_CONSISTENT 1 << 4
+ * - in ensuring that a buffer's va is 4 Byte aligned
+ * - in rounding up a buffer length to 4 Bytes.
+ */
+#define DMA_ALIGN_LEN		4
+
+#define DMA_D2H_SCRATCH_BUF_LEN	8
+#define DMA_XFER_LEN_LIMIT	0x400000
+
+#ifdef BCM_HOST_BUF
+#ifndef DMA_HOST_BUFFER_LEN
+#define DMA_HOST_BUFFER_LEN	0x200000
+#endif
+#endif /* BCM_HOST_BUF */
+
+#define DHD_FLOWRING_IOCTL_BUFPOST_PKTSZ		8192
+
+#define DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D		1
+#define DHD_FLOWRING_MAX_EVENTBUF_POST			32
+#define DHD_FLOWRING_MAX_IOCTLRESPBUF_POST		8
+#define DHD_H2D_INFORING_MAX_BUF_POST			32
+#define DHD_MAX_TSBUF_POST			8
+
+#define DHD_PROT_FUNCS	43
+
+/* Length of buffer in host for bus throughput measurement */
+#define DHD_BUS_TPUT_BUF_LEN 2048
+
+#define TXP_FLUSH_NITEMS
+
+/* optimization to write "n" tx items at a time to ring */
+#define TXP_FLUSH_MAX_ITEMS_FLUSH_CNT	48
+
+#define RING_NAME_MAX_LENGTH		24
+#define CTRLSUB_HOSTTS_MEESAGE_SIZE		1024
+/* Giving room before ioctl_trans_id rollsover. */
+#define BUFFER_BEFORE_ROLLOVER 300
+
+/* 512K memory + 32K registers */
+#define SNAPSHOT_UPLOAD_BUF_SIZE	((512 + 32) * 1024)
+
+struct msgbuf_ring; /* ring context for common and flow rings */
+
+#ifdef DHD_HMAPTEST
+/* 5 * DMA_CONSISTENT_ALIGN as different tests use upto 4th page */
+#define HMAP_SANDBOX_BUFFER_LEN	(DMA_CONSISTENT_ALIGN * 5) /* for a 4k page this is 20K */
+/**
+ * for D11 DMA HMAPTEST thes states are as follows
+ * iovar sets ACTIVE state
+ * next TXPOST / RXPOST sets POSTED state
+ * on TXCPL / RXCPL POSTED + pktid match does buffer free nd state changed to INACTIVE
+ * This ensures that on an iovar only one buffer is replaced from sandbox area
+ */
+#define HMAPTEST_D11_TX_INACTIVE 0
+#define HMAPTEST_D11_TX_ACTIVE 1
+#define HMAPTEST_D11_TX_POSTED 2
+
+#define HMAPTEST_D11_RX_INACTIVE 0
+#define HMAPTEST_D11_RX_ACTIVE 1
+#define HMAPTEST_D11_RX_POSTED 2
+#endif /* DHD_HMAPTEST */
+
+#define PCIE_DMA_LOOPBACK	0
+#define D11_DMA_LOOPBACK	1
+#define BMC_DMA_LOOPBACK	2
+
+/**
+ * PCIE D2H DMA Complete Sync Modes
+ *
+ * Firmware may interrupt the host, prior to the D2H Mem2Mem DMA completes into
+ * Host system memory. A WAR using one of 3 approaches is needed:
+ * 1. Dongle places a modulo-253 seqnum in last word of each D2H message
+ * 2. XOR Checksum, with epoch# in each work item. Dongle builds an XOR checksum
+ *    writes in the last word of each work item. Each work item has a seqnum
+ *    number = sequence num % 253.
+ *
+ * 3. Read Barrier: Dongle does a host memory read access prior to posting an
+ *    interrupt, ensuring that D2H data transfer indeed completed.
+ * 4. Dongle DMA's all indices after producing items in the D2H ring, flushing
+ *    ring contents before the indices.
+ *
+ * Host does not sync for DMA to complete with option #3 or #4, and a noop sync
+ * callback (see dhd_prot_d2h_sync_none) may be bound.
+ *
+ * Dongle advertizes host side sync mechanism requirements.
+ */
+
+#ifdef PCIE_D2H_SYNC_RETRY_CNT_ONE
+#define PCIE_D2H_SYNC_WAIT_TRIES    (1U)
+#define PCIE_D2H_SYNC_NUM_OF_STEPS  (1U)
+#define PCIE_D2H_SYNC_DELAY         (1UL)	/* in terms of usecs */
+#else
+#define PCIE_D2H_SYNC_WAIT_TRIES    (512U)
+#define PCIE_D2H_SYNC_NUM_OF_STEPS  (5U)
+#define PCIE_D2H_SYNC_DELAY         (100UL)	/* in terms of usecs */
+#endif /* PCIE_D2H_SYNC_RETRY_CNT_ONE */
+
+/**
+ * Custom callback attached based upon D2H DMA Sync mode advertized by dongle.
+ *
+ * On success: return cmn_msg_hdr_t::msg_type
+ * On failure: return 0 (invalid msg_type)
+ */
+typedef uint8 (* d2h_sync_cb_t)(dhd_pub_t *dhd, struct msgbuf_ring *ring,
+                                volatile cmn_msg_hdr_t *msg, int msglen);
+
+void dhd_prot_debug_ring_info(dhd_pub_t *dhd);
+
+/**
+ * Custom callback attached based upon D2H DMA Sync mode advertized by dongle.
+ * For EDL messages.
+ *
+ * On success: return cmn_msg_hdr_t::msg_type
+ * On failure: return 0 (invalid msg_type)
+ */
+#ifdef EWP_EDL
+typedef int (* d2h_edl_sync_cb_t)(dhd_pub_t *dhd, struct msgbuf_ring *ring,
+                                volatile cmn_msg_hdr_t *msg);
+#endif /* EWP_EDL */
+
+/*
+ * +----------------------------------------------------------------------------
+ *
+ * RingIds and FlowId are not equivalent as ringids include D2H rings whereas
+ * flowids do not.
+ *
+ * Dongle advertizes the max H2D rings, as max_sub_queues = 'N' which includes
+ * the H2D common rings as well as the (N-BCMPCIE_H2D_COMMON_MSGRINGS) flowrings
+ *
+ * Here is a sample mapping for (based on PCIE Full Dongle Rev5) where,
+ *  BCMPCIE_H2D_COMMON_MSGRINGS = 2, i.e. 2 H2D common rings,
+ *  BCMPCIE_COMMON_MSGRINGS     = 5, i.e. include 3 D2H common rings.
+ *
+ *  H2D Control  Submit   RingId = 0        FlowId = 0 reserved never allocated
+ *  H2D RxPost   Submit   RingId = 1        FlowId = 1 reserved never allocated
+ *
+ *  D2H Control  Complete RingId = 2
+ *  D2H Transmit Complete RingId = 3
+ *  D2H Receive  Complete RingId = 4
+ *
+ *  H2D TxPost   FLOWRING RingId = 5         FlowId = 2     (1st flowring)
+ *  H2D TxPost   FLOWRING RingId = 6         FlowId = 3     (2nd flowring)
+ *  H2D TxPost   FLOWRING RingId = 5 + (N-1) FlowId = (N-1) (Nth flowring)
+ *
+ * When TxPost FlowId(s) are allocated, the FlowIds [0..FLOWID_RESERVED) are
+ * unused, where FLOWID_RESERVED is BCMPCIE_H2D_COMMON_MSGRINGS.
+ *
+ * Example: when a system supports 4 bc/mc and 128 uc flowrings, with
+ * BCMPCIE_H2D_COMMON_MSGRINGS = 2, and BCMPCIE_H2D_COMMON_MSGRINGS = 5, and the
+ * FlowId values would be in the range [2..133] and the corresponding
+ * RingId values would be in the range [5..136].
+ *
+ * The flowId allocator, may chose to, allocate Flowids:
+ *   bc/mc (per virtual interface) in one consecutive range [2..(2+VIFS))
+ *   X# of uc flowids in consecutive ranges (per station Id), where X is the
+ *   packet's access category (e.g. 4 uc flowids per station).
+ *
+ * CAUTION:
+ * When DMA indices array feature is used, RingId=5, corresponding to the 0th
+ * FLOWRING, will actually use the FlowId as index into the H2D DMA index,
+ * since the FlowId truly represents the index in the H2D DMA indices array.
+ *
+ * Likewise, in the D2H direction, the RingId - BCMPCIE_H2D_COMMON_MSGRINGS,
+ * will represent the index in the D2H DMA indices array.
+ *
+ * +----------------------------------------------------------------------------
+ */
+
+/* First TxPost Flowring Id */
+#define DHD_FLOWRING_START_FLOWID   BCMPCIE_H2D_COMMON_MSGRINGS
+
+/* Determine whether a ringid belongs to a TxPost flowring */
+#define DHD_IS_FLOWRING(ringid, max_flow_rings) \
+	((ringid) >= BCMPCIE_COMMON_MSGRINGS && \
+	(ringid) < ((max_flow_rings) + BCMPCIE_COMMON_MSGRINGS))
+
+/* Convert a H2D TxPost FlowId to a MsgBuf RingId */
+#define DHD_FLOWID_TO_RINGID(flowid) \
+	(BCMPCIE_COMMON_MSGRINGS + ((flowid) - BCMPCIE_H2D_COMMON_MSGRINGS))
+
+/* Convert a MsgBuf RingId to a H2D TxPost FlowId */
+#define DHD_RINGID_TO_FLOWID(ringid) \
+	(BCMPCIE_H2D_COMMON_MSGRINGS + ((ringid) - BCMPCIE_COMMON_MSGRINGS))
+
+/* Convert a H2D MsgBuf RingId to an offset index into the H2D DMA indices array
+ * This may be used for the H2D DMA WR index array or H2D DMA RD index array or
+ * any array of H2D rings.
+ */
+#define DHD_H2D_RING_OFFSET(ringid) \
+	(((ringid) >= BCMPCIE_COMMON_MSGRINGS) ? DHD_RINGID_TO_FLOWID(ringid) : (ringid))
+
+/* Convert a H2D MsgBuf Flowring Id to an offset index into the H2D DMA indices array
+ * This may be used for IFRM.
+ */
+#define DHD_H2D_FRM_FLOW_RING_OFFSET(ringid) \
+	((ringid) - BCMPCIE_COMMON_MSGRINGS)
+
+/* Convert a D2H MsgBuf RingId to an offset index into the D2H DMA indices array
+ * This may be used for the D2H DMA WR index array or D2H DMA RD index array or
+ * any array of D2H rings.
+ * d2h debug ring is located at the end, i.e. after all the tx flow rings and h2d debug ring
+ * max_h2d_rings: total number of h2d rings
+ */
+#define DHD_D2H_RING_OFFSET(ringid, max_h2d_rings) \
+	((ringid) > (max_h2d_rings) ? \
+		((ringid) - max_h2d_rings) : \
+		((ringid) - BCMPCIE_H2D_COMMON_MSGRINGS))
+
+/* Convert a D2H DMA Indices Offset to a RingId */
+#define DHD_D2H_RINGID(offset) \
+	((offset) + BCMPCIE_H2D_COMMON_MSGRINGS)
+
+/* XXX: The ringid and flowid and dma indices array index idiosyncracy is error
+ * prone. While a simplification is possible, the backward compatability
+ * requirement (DHD should operate with any PCIE rev version of firmware),
+ * limits what may be accomplished.
+ *
+ * At the minimum, implementation should use macros for any conversions
+ * facilitating introduction of future PCIE FD revs that need more "common" or
+ * other dynamic rings.
+ */
+
+/* XXX: Presently there is no need for maintaining both a dmah and a secdmah */
+#define DHD_DMAH_NULL      ((void*)NULL)
+
+/*
+ * Pad a DMA-able buffer by an additional cachline. If the end of the DMA-able
+ * buffer does not occupy the entire cacheline, and another object is placed
+ * following the DMA-able buffer, data corruption may occur if the DMA-able
+ * buffer is used to DMAing into (e.g. D2H direction), when HW cache coherency
+ * is not available.
+ */
+#if defined(L1_CACHE_BYTES)
+#define DHD_DMA_PAD        (L1_CACHE_BYTES)
+#else
+#define DHD_DMA_PAD        (128)
+#endif
+
+/*
+ * +----------------------------------------------------------------------------
+ * Flowring Pool
+ *
+ * Unlike common rings, which are attached very early on (dhd_prot_attach),
+ * flowrings are dynamically instantiated. Moreover, flowrings may require a
+ * larger DMA-able buffer. To avoid issues with fragmented cache coherent
+ * DMA-able memory, a pre-allocated pool of msgbuf_ring_t is allocated once.
+ * The DMA-able buffers are attached to these pre-allocated msgbuf_ring.
+ *
+ * Each DMA-able buffer may be allocated independently, or may be carved out
+ * of a single large contiguous region that is registered with the protocol
+ * layer into flowrings_dma_buf. On a 64bit platform, this contiguous region
+ * may not span 0x00000000FFFFFFFF (avoid dongle side 64bit ptr arithmetic).
+ *
+ * No flowring pool action is performed in dhd_prot_attach(), as the number
+ * of h2d rings is not yet known.
+ *
+ * In dhd_prot_init(), the dongle advertized number of h2d rings is used to
+ * determine the number of flowrings required, and a pool of msgbuf_rings are
+ * allocated and a DMA-able buffer (carved or allocated) is attached.
+ * See: dhd_prot_flowrings_pool_attach()
+ *
+ * A flowring msgbuf_ring object may be fetched from this pool during flowring
+ * creation, using the flowid. Likewise, flowrings may be freed back into the
+ * pool on flowring deletion.
+ * See: dhd_prot_flowrings_pool_fetch(), dhd_prot_flowrings_pool_release()
+ *
+ * In dhd_prot_detach(), the flowring pool is detached. The DMA-able buffers
+ * are detached (returned back to the carved region or freed), and the pool of
+ * msgbuf_ring and any objects allocated against it are freed.
+ * See: dhd_prot_flowrings_pool_detach()
+ *
+ * In dhd_prot_reset(), the flowring pool is simply reset by returning it to a
+ * state as-if upon an attach. All DMA-able buffers are retained.
+ * Following a dhd_prot_reset(), in a subsequent dhd_prot_init(), the flowring
+ * pool attach will notice that the pool persists and continue to use it. This
+ * will avoid the case of a fragmented DMA-able region.
+ *
+ * +----------------------------------------------------------------------------
+ */
+
+/* Conversion of a flowid to a flowring pool index */
+#define DHD_FLOWRINGS_POOL_OFFSET(flowid) \
+	((flowid) - BCMPCIE_H2D_COMMON_MSGRINGS)
+
+/* Fetch the msgbuf_ring_t from the flowring pool given a flowid */
+#define DHD_RING_IN_FLOWRINGS_POOL(prot, flowid) \
+	(msgbuf_ring_t*)((prot)->h2d_flowrings_pool) + \
+	    DHD_FLOWRINGS_POOL_OFFSET(flowid)
+
+/* Traverse each flowring in the flowring pool, assigning ring and flowid */
+#define FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, total_flowrings) \
+	for ((flowid) = DHD_FLOWRING_START_FLOWID, \
+		(ring) = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid); \
+		 (flowid) < ((total_flowrings) + DHD_FLOWRING_START_FLOWID); \
+		 (ring)++, (flowid)++)
+
+/* Used in loopback tests */
+typedef struct dhd_dmaxfer {
+	dhd_dma_buf_t srcmem;
+	dhd_dma_buf_t dstmem;
+	uint32        srcdelay;
+	uint32        destdelay;
+	uint32        len;
+	bool          in_progress;
+	uint64        start_usec;
+	uint64        time_taken;
+	uint32        d11_lpbk;
+	int           status;
+} dhd_dmaxfer_t;
+
+#ifdef DHD_HMAPTEST
+/* Used in HMAP test */
+typedef struct dhd_hmaptest {
+	dhd_dma_buf_t	mem;
+	uint32		len;
+	bool	in_progress;
+	uint32	is_write;
+	uint32	accesstype;
+	uint64  start_usec;
+	uint32	offset;
+} dhd_hmaptest_t;
+#endif /* DHD_HMAPTEST */
+/**
+ * msgbuf_ring : This object manages the host side ring that includes a DMA-able
+ * buffer, the WR and RD indices, ring parameters such as max number of items
+ * an length of each items, and other miscellaneous runtime state.
+ * A msgbuf_ring may be used to represent a H2D or D2H common ring or a
+ * H2D TxPost ring as specified in the PCIE FullDongle Spec.
+ * Ring parameters are conveyed to the dongle, which maintains its own peer end
+ * ring state. Depending on whether the DMA Indices feature is supported, the
+ * host will update the WR/RD index in the DMA indices array in host memory or
+ * directly in dongle memory.
+ */
+typedef struct msgbuf_ring {
+	bool           inited;
+	uint16         idx;       /* ring id */
+	uint16         rd;        /* read index */
+	uint16         curr_rd;   /* read index for debug */
+	uint16         wr;        /* write index */
+	uint16         max_items; /* maximum number of items in ring */
+	uint16         item_len;  /* length of each item in the ring */
+	sh_addr_t      base_addr; /* LITTLE ENDIAN formatted: base address */
+	dhd_dma_buf_t  dma_buf;   /* DMA-able buffer: pa, va, len, dmah, secdma */
+	uint32         seqnum;    /* next expected item's sequence number */
+#ifdef TXP_FLUSH_NITEMS
+	void           *start_addr;
+	/* # of messages on ring not yet announced to dongle */
+	uint16         pend_items_count;
+#ifdef AGG_H2D_DB
+	osl_atomic_t	inflight;
+#endif /* AGG_H2D_DB */
+#endif /* TXP_FLUSH_NITEMS */
+
+	uint8           ring_type;
+	uint8           n_completion_ids;
+	bool            create_pending;
+	uint16          create_req_id;
+	uint8           current_phase;
+	uint16	        compeltion_ring_ids[MAX_COMPLETION_RING_IDS_ASSOCIATED];
+	uchar		name[RING_NAME_MAX_LENGTH];
+	uint32		ring_mem_allocated;
+	void	        *ring_lock;
+	struct msgbuf_ring *linked_ring; /* Ring Associated to metadata ring */
+} msgbuf_ring_t;
+
+#define DHD_RING_BGN_VA(ring)           ((ring)->dma_buf.va)
+#define DHD_RING_END_VA(ring) \
+	((uint8 *)(DHD_RING_BGN_VA((ring))) + \
+	 (((ring)->max_items - 1) * (ring)->item_len))
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+/** D2H WLAN Rx Packet Chaining context */
+typedef struct rxchain_info {
+	uint		pkt_count;
+	uint		ifidx;
+	void		*pkthead;
+	void		*pkttail;
+	uint8		*h_da;	/* pointer to da of chain head */
+	uint8		*h_sa;	/* pointer to sa of chain head */
+	uint8		h_prio; /* prio of chain head */
+} rxchain_info_t;
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+/* Legacy ring sizes for 1Gbps and lesser */
+#define H2DRING_TXPOST_SIZE_V1		512u
+#define H2DRING_HTPUT_TXPOST_SIZE_V1	512u	/* Keep it same as H2DRING_TXPOST_SIZE_V1 */
+#define D2HRING_TXCPL_SIZE_V1		1024u
+
+#define H2DRING_RXPOST_SIZE_V1		512u
+#define D2HRING_RXCPL_SIZE_V1		512u
+
+#define H2DRING_CTRLPOST_SIZE_V1	64u
+#define D2HRING_CTRLCPL_SIZE_V1		64u
+
+#define RX_BUF_BURST_V1			32u	/* Rx buffers for MSDU Data */
+#define RX_BUFPOST_THRESHOLD_V1		32u	/* Rxbuf post threshold */
+
+/* Ring sizes version 2 for 2Gbps */
+#define H2DRING_TXPOST_SIZE_V2		512u
+#define H2DRING_HTPUT_TXPOST_SIZE_V2	2048u	/* ONLY for 4 160 MHz BE flowrings */
+#define D2HRING_TXCPL_SIZE_V2		2048u
+
+#define H2DRING_RXPOST_SIZE_V2		2048u
+#define D2HRING_RXCPL_SIZE_V2		1024u
+
+#define H2DRING_CTRLPOST_SIZE_V2	128u
+#define D2HRING_CTRLCPL_SIZE_V2		64u
+
+#define RX_BUF_BURST_V2			64u	/* Rx buffers for MSDU Data */
+#define RX_BUFPOST_THRESHOLD_V2		64u	/* Rxbuf post threshold */
+
+int ring_size_version;
+int ring_size_alloc_version;
+
+uint h2d_max_txpost;
+uint h2d_htput_max_txpost;
+uint d2h_max_txcpl;
+
+uint h2d_max_rxpost;
+uint d2h_max_rxcpl;
+
+uint h2d_max_ctrlpost;
+uint d2h_max_ctrlcpl;
+
+uint rx_buf_burst;
+uint rx_bufpost_threshold;
+
+#ifndef DHD_RX_CPL_POST_BOUND
+#define DHD_RX_CPL_POST_BOUND		1024
+#endif
+#ifndef DHD_TX_POST_BOUND
+#define DHD_TX_POST_BOUND		256
+#endif
+#ifndef DHD_CTRL_CPL_POST_BOUND
+#define DHD_CTRL_CPL_POST_BOUND		64
+#endif
+#ifndef DHD_TX_CPL_BOUND
+#define DHD_TX_CPL_BOUND		2048
+#endif
+
+uint dhd_rx_cpl_post_bound = DHD_RX_CPL_POST_BOUND;
+uint dhd_tx_post_bound = DHD_TX_POST_BOUND;
+uint dhd_tx_cpl_bound = DHD_TX_CPL_BOUND;
+uint dhd_ctrl_cpl_post_bound = DHD_CTRL_CPL_POST_BOUND;
+
+#ifdef AGG_H2D_DB
+bool agg_h2d_db_enab = TRUE;
+
+#define AGG_H2D_DB_TIMEOUT_USEC		(1000u)	/* 1 msec */
+uint32 agg_h2d_db_timeout = AGG_H2D_DB_TIMEOUT_USEC;
+
+#ifndef AGG_H2D_DB_INFLIGHT_THRESH
+/* Keep inflight threshold same as txp_threshold */
+#define AGG_H2D_DB_INFLIGHT_THRESH TXP_FLUSH_MAX_ITEMS_FLUSH_CNT
+#endif /* !AGG_H2D_DB_INFLIGHT_THRESH */
+
+uint32 agg_h2d_db_inflight_thresh = AGG_H2D_DB_INFLIGHT_THRESH;
+
+#define DHD_NUM_INFLIGHT_HISTO_ROWS (14u)
+#define DHD_INFLIGHT_HISTO_SIZE (sizeof(uint64) * DHD_NUM_INFLIGHT_HISTO_ROWS)
+
+typedef struct _agg_h2d_db_info {
+	void *dhd;
+	struct hrtimer timer;
+	bool init;
+	uint32 direct_db_cnt;
+	uint32 timer_db_cnt;
+	uint64  *inflight_histo;
+} agg_h2d_db_info_t;
+#endif /* AGG_H2D_DB */
+
+#define DHD_DEBUG_INVALID_PKTID
+
+/** DHD protocol handle. Is an opaque type to other DHD software layers. */
+typedef struct dhd_prot {
+	osl_t *osh;		/* OSL handle */
+	uint16 rxbufpost_sz;		/* Size of rx buffer posted to dongle */
+	uint16 rxbufpost_alloc_sz;	/* Actual rx buffer packet allocated in the host */
+	uint16 rxbufpost;
+	uint16 rx_buf_burst;
+	uint16 rx_bufpost_threshold;
+	uint16 max_rxbufpost;
+	uint32 tot_rxbufpost;
+	uint32 tot_rxcpl;
+	uint16 max_eventbufpost;
+	uint16 max_ioctlrespbufpost;
+	uint16 max_tsbufpost;
+	uint16 max_infobufpost;
+	uint16 infobufpost;
+	uint16 cur_event_bufs_posted;
+	uint16 cur_ioctlresp_bufs_posted;
+	uint16 cur_ts_bufs_posted;
+
+	/* Flow control mechanism based on active transmits pending */
+	osl_atomic_t active_tx_count; /* increments/decrements on every packet tx/tx_status */
+	uint16 h2d_max_txpost;
+	uint16 h2d_htput_max_txpost;
+	uint16 txp_threshold;  /* optimization to write "n" tx items at a time to ring */
+
+	/* MsgBuf Ring info: has a dhd_dma_buf that is dynamically allocated */
+	msgbuf_ring_t h2dring_ctrl_subn; /* H2D ctrl message submission ring */
+	msgbuf_ring_t h2dring_rxp_subn; /* H2D RxBuf post ring */
+	msgbuf_ring_t d2hring_ctrl_cpln; /* D2H ctrl completion ring */
+	msgbuf_ring_t d2hring_tx_cpln; /* D2H Tx complete message ring */
+	msgbuf_ring_t d2hring_rx_cpln; /* D2H Rx complete message ring */
+	msgbuf_ring_t *h2dring_info_subn; /* H2D info submission ring */
+	msgbuf_ring_t *d2hring_info_cpln; /* D2H info completion ring */
+	msgbuf_ring_t *d2hring_edl; /* D2H Enhanced Debug Lane (EDL) ring */
+
+	msgbuf_ring_t *h2d_flowrings_pool; /* Pool of preallocated flowings */
+	dhd_dma_buf_t flowrings_dma_buf; /* Contiguous DMA buffer for flowrings */
+	uint16        h2d_rings_total; /* total H2D (common rings + flowrings) */
+
+	uint32		rx_dataoffset;
+
+	dhd_mb_ring_t	mb_ring_fn;	/* called when dongle needs to be notified of new msg */
+	dhd_mb_ring_2_t	mb_2_ring_fn;	/* called when dongle needs to be notified of new msg */
+
+	/* ioctl related resources */
+	uint8 ioctl_state;
+	int16 ioctl_status;		/* status returned from dongle */
+	uint16 ioctl_resplen;
+	dhd_ioctl_recieved_status_t ioctl_received;
+	uint curr_ioctl_cmd;
+	dhd_dma_buf_t	retbuf;		/* For holding ioctl response */
+	dhd_dma_buf_t	ioctbuf;	/* For holding ioctl request */
+
+	dhd_dma_buf_t	d2h_dma_scratch_buf;	/* For holding d2h scratch */
+
+	/* DMA-able arrays for holding WR and RD indices */
+	uint32          rw_index_sz; /* Size of a RD or WR index in dongle */
+	dhd_dma_buf_t   h2d_dma_indx_wr_buf;	/* Array of H2D WR indices */
+	dhd_dma_buf_t	h2d_dma_indx_rd_buf;	/* Array of H2D RD indices */
+	dhd_dma_buf_t	d2h_dma_indx_wr_buf;	/* Array of D2H WR indices */
+	dhd_dma_buf_t	d2h_dma_indx_rd_buf;	/* Array of D2H RD indices */
+	dhd_dma_buf_t h2d_ifrm_indx_wr_buf;	/* Array of H2D WR indices for ifrm */
+
+	dhd_dma_buf_t	host_bus_throughput_buf; /* bus throughput measure buffer */
+
+	dhd_dma_buf_t   *flowring_buf;    /* pool of flow ring buf */
+#ifdef DHD_DMA_INDICES_SEQNUM
+	char *h2d_dma_indx_rd_copy_buf; /* Local copy of H2D WR indices array */
+	char *d2h_dma_indx_wr_copy_buf; /* Local copy of D2H WR indices array */
+	uint32 h2d_dma_indx_rd_copy_bufsz; /* H2D WR indices array size */
+	uint32 d2h_dma_indx_wr_copy_bufsz; /* D2H WR indices array size */
+	uint32 host_seqnum;	/* Seqence number for D2H DMA Indices sync */
+#endif /* DHD_DMA_INDICES_SEQNUM */
+	uint32			flowring_num;
+
+	d2h_sync_cb_t d2h_sync_cb; /* Sync on D2H DMA done: SEQNUM or XORCSUM */
+#ifdef EWP_EDL
+	d2h_edl_sync_cb_t d2h_edl_sync_cb; /* Sync on EDL D2H DMA done: SEQNUM or XORCSUM */
+#endif /* EWP_EDL */
+	ulong d2h_sync_wait_max; /* max number of wait loops to receive one msg */
+	ulong d2h_sync_wait_tot; /* total wait loops */
+
+	dhd_dmaxfer_t	dmaxfer; /* for test/DMA loopback */
+
+	uint16		ioctl_seq_no;
+	uint16		data_seq_no;  /* XXX this field is obsolete */
+	uint16		ioctl_trans_id;
+	void		*pktid_ctrl_map; /* a pktid maps to a packet and its metadata */
+	void		*pktid_rx_map;	/* pktid map for rx path */
+	void		*pktid_tx_map;	/* pktid map for tx path */
+	bool		metadata_dbg;
+	void		*pktid_map_handle_ioctl;
+#ifdef DHD_MAP_PKTID_LOGGING
+	void		*pktid_dma_map;	/* pktid map for DMA MAP */
+	void		*pktid_dma_unmap; /* pktid map for DMA UNMAP */
+#endif /* DHD_MAP_PKTID_LOGGING */
+	uint32		pktid_depleted_cnt;	/* pktid depleted count */
+	/* netif tx queue stop count */
+	uint8		pktid_txq_stop_cnt;
+	/* netif tx queue start count */
+	uint8		pktid_txq_start_cnt;
+	uint64		ioctl_fillup_time;	/* timestamp for ioctl fillup */
+	uint64		ioctl_ack_time;		/* timestamp for ioctl ack */
+	uint64		ioctl_cmplt_time;	/* timestamp for ioctl completion */
+
+	/* Applications/utilities can read tx and rx metadata using IOVARs */
+	uint16		rx_metadata_offset;
+	uint16		tx_metadata_offset;
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+	rxchain_info_t	rxchain;	/* chain of rx packets */
+#endif
+
+#if defined(DHD_D2H_SOFT_DOORBELL_SUPPORT)
+	/* Host's soft doorbell configuration */
+	bcmpcie_soft_doorbell_t soft_doorbell[BCMPCIE_D2H_COMMON_MSGRINGS];
+#endif /* DHD_D2H_SOFT_DOORBELL_SUPPORT */
+
+	/* Work Queues to be used by the producer and the consumer, and threshold
+	 * when the WRITE index must be synced to consumer's workq
+	 */
+	dhd_dma_buf_t	fw_trap_buf; /* firmware trap buffer */
+#ifdef FLOW_RING_PREALLOC
+	/* pre-allocation htput ring buffer */
+	dhd_dma_buf_t	htput_ring_buf[HTPUT_TOTAL_FLOW_RINGS];
+	/* pre-allocation folw ring(non htput rings) */
+	dhd_dma_buf_t	flow_ring_buf[MAX_FLOW_RINGS];
+#endif /* FLOW_RING_PREALLOC */
+	uint32  host_ipc_version; /* Host sypported IPC rev */
+	uint32  device_ipc_version; /* FW supported IPC rev */
+	uint32  active_ipc_version; /* Host advertised IPC rev */
+	dhd_dma_buf_t   hostts_req_buf; /* For holding host timestamp request buf */
+	bool    hostts_req_buf_inuse;
+	bool    rx_ts_log_enabled;
+	bool    tx_ts_log_enabled;
+#ifdef DHD_HMAPTEST
+	uint32 hmaptest_rx_active;
+	uint32 hmaptest_rx_pktid;
+	char *hmap_rx_buf_va;
+	dmaaddr_t hmap_rx_buf_pa;
+	uint32 hmap_rx_buf_len;
+
+	uint32 hmaptest_tx_active;
+	uint32 hmaptest_tx_pktid;
+	char *hmap_tx_buf_va;
+	dmaaddr_t hmap_tx_buf_pa;
+	uint32	  hmap_tx_buf_len;
+	dhd_hmaptest_t	hmaptest; /* for hmaptest */
+	bool hmap_enabled; /* TRUE = hmap is enabled */
+#endif /* DHD_HMAPTEST */
+	bool no_retry;
+	bool no_aggr;
+	bool fixed_rate;
+	dhd_dma_buf_t	host_scb_buf; /* scb host offload buffer */
+	bool no_tx_resource;
+	uint32 txcpl_db_cnt;
+#ifdef AGG_H2D_DB
+	agg_h2d_db_info_t agg_h2d_db_info;
+#endif /* AGG_H2D_DB */
+	uint64 tx_h2d_db_cnt;
+#ifdef DHD_DEBUG_INVALID_PKTID
+	uint8 ctrl_cpl_snapshot[D2HRING_CTRL_CMPLT_ITEMSIZE];
+#endif /* DHD_DEBUG_INVALID_PKTID */
+	uint32 event_wakeup_pkt; /* Number of event wakeup packet rcvd */
+	uint32 rx_wakeup_pkt;    /* Number of Rx wakeup packet rcvd */
+	uint32 info_wakeup_pkt;  /* Number of info cpl wakeup packet rcvd */
+	msgbuf_ring_t *d2hring_md_cpl; /* D2H metadata completion ring */
+	/* no. which controls how many rx cpl/post items are processed per dpc */
+	uint32 rx_cpl_post_bound;
+	/*
+	 * no. which controls how many tx post items are processed per dpc,
+	 * i.e, how many tx pkts are posted to flowring from the bkp queue
+	 * from dpc context
+	 */
+	uint32 tx_post_bound;
+	/* no. which controls how many tx cpl items are processed per dpc */
+	uint32 tx_cpl_bound;
+	/* no. which controls how many ctrl cpl/post items are processed per dpc */
+	uint32 ctrl_cpl_post_bound;
+} dhd_prot_t;
+
+#ifdef DHD_EWPR_VER2
+#define HANG_INFO_BASE64_BUFFER_SIZE 640
+#endif
+
+#ifdef DHD_DUMP_PCIE_RINGS
+static
+int dhd_ring_write(dhd_pub_t *dhd, msgbuf_ring_t *ring, void *file,
+	const void *user_buf, unsigned long *file_posn);
+#ifdef EWP_EDL
+static
+int dhd_edl_ring_hdr_write(dhd_pub_t *dhd, msgbuf_ring_t *ring, void *file, const void *user_buf,
+	unsigned long *file_posn);
+#endif /* EWP_EDL */
+#endif /* DHD_DUMP_PCIE_RINGS */
+extern bool dhd_timesync_delay_post_bufs(dhd_pub_t *dhdp);
+extern void dhd_schedule_dmaxfer_free(dhd_pub_t* dhdp, dmaxref_mem_map_t *dmmap);
+/* Convert a dmaaddr_t to a base_addr with htol operations */
+static INLINE void dhd_base_addr_htolpa(sh_addr_t *base_addr, dmaaddr_t pa);
+
+/* APIs for managing a DMA-able buffer */
+static int  dhd_dma_buf_audit(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf);
+static void dhd_dma_buf_reset(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf);
+
+/* msgbuf ring management */
+static int dhd_prot_allocate_bufs(dhd_pub_t *dhd, dhd_prot_t *prot);
+static int dhd_prot_ring_attach(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+	const char *name, uint16 max_items, uint16 len_item, uint16 ringid);
+static void dhd_prot_ring_init(dhd_pub_t *dhd, msgbuf_ring_t *ring);
+static void dhd_prot_ring_reset(dhd_pub_t *dhd, msgbuf_ring_t *ring);
+static void dhd_prot_ring_detach(dhd_pub_t *dhd, msgbuf_ring_t *ring);
+static void dhd_prot_process_fw_timestamp(dhd_pub_t *dhd, void* buf);
+
+/* Pool of pre-allocated msgbuf_ring_t with DMA-able buffers for Flowrings */
+static int  dhd_prot_flowrings_pool_attach(dhd_pub_t *dhd);
+static void dhd_prot_flowrings_pool_reset(dhd_pub_t *dhd);
+static void dhd_prot_flowrings_pool_detach(dhd_pub_t *dhd);
+
+/* Fetch and Release a flowring msgbuf_ring from flowring  pool */
+static msgbuf_ring_t *dhd_prot_flowrings_pool_fetch(dhd_pub_t *dhd,
+	uint16 flowid);
+/* see also dhd_prot_flowrings_pool_release() in dhd_prot.h */
+
+/* Producer: Allocate space in a msgbuf ring */
+static void* dhd_prot_alloc_ring_space(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+	uint16 nitems, uint16 *alloced, bool exactly_nitems);
+static void* dhd_prot_get_ring_space(msgbuf_ring_t *ring, uint16 nitems,
+	uint16 *alloced, bool exactly_nitems);
+
+/* Consumer: Determine the location where the next message may be consumed */
+static uint8* dhd_prot_get_read_addr(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+	uint32 *available_len, uint32 bound);
+
+/* Producer (WR index update) or Consumer (RD index update) indication */
+static void dhd_prot_ring_write_complete(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+	void *p, uint16 len);
+
+#ifdef AGG_H2D_DB
+static void dhd_prot_agg_db_ring_write(dhd_pub_t *dhd, msgbuf_ring_t * ring,
+		void* p, uint16 len);
+static void dhd_prot_aggregate_db_ring_door_bell(dhd_pub_t *dhd, uint16 flowid, bool ring_db);
+static void dhd_prot_txdata_aggr_db_write_flush(dhd_pub_t *dhd, uint16 flowid);
+#endif /* AGG_H2D_DB */
+static void dhd_prot_ring_doorbell(dhd_pub_t *dhd, uint32 value);
+static void dhd_prot_upd_read_idx(dhd_pub_t *dhd, msgbuf_ring_t *ring);
+
+static INLINE int dhd_prot_dma_indx_alloc(dhd_pub_t *dhd, uint8 type,
+	dhd_dma_buf_t *dma_buf, uint32 bufsz);
+
+/* Set/Get a RD or WR index in the array of indices */
+/* See also: dhd_prot_dma_indx_init() */
+void dhd_prot_dma_indx_set(dhd_pub_t *dhd, uint16 new_index, uint8 type,
+	uint16 ringid);
+static uint16 dhd_prot_dma_indx_get(dhd_pub_t *dhd, uint8 type, uint16 ringid);
+
+/* Locate a packet given a pktid */
+static INLINE void *dhd_prot_packet_get(dhd_pub_t *dhd, uint32 pktid, uint8 pkttype,
+	bool free_pktid);
+/* Locate a packet given a PktId and free it. */
+static INLINE void dhd_prot_packet_free(dhd_pub_t *dhd, void *pkt, uint8 pkttype, bool send);
+
+static int dhd_msgbuf_query_ioctl(dhd_pub_t *dhd, int ifidx, uint cmd,
+	void *buf, uint len, uint8 action);
+static int dhd_msgbuf_set_ioctl(dhd_pub_t *dhd, int ifidx, uint cmd,
+	void *buf, uint len, uint8 action);
+static int dhd_msgbuf_wait_ioctl_cmplt(dhd_pub_t *dhd, uint32 len, void *buf);
+static int dhd_fillup_ioct_reqst(dhd_pub_t *dhd, uint16 len, uint cmd,
+	void *buf, int ifidx);
+static void dhd_msgbuf_dump_iovar_name(dhd_pub_t *dhd);
+
+/* Post buffers for Rx, control ioctl response and events */
+static uint16 dhd_msgbuf_rxbuf_post_ctrlpath(dhd_pub_t *dhd, uint8 msgid, uint32 max_to_post);
+static void dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd_pub_t *pub);
+static void dhd_msgbuf_rxbuf_post_event_bufs(dhd_pub_t *pub);
+static void dhd_msgbuf_rxbuf_post(dhd_pub_t *dhd, bool use_rsv_pktid);
+static int dhd_prot_rxbuf_post(dhd_pub_t *dhd, uint16 count, bool use_rsv_pktid);
+static int dhd_msgbuf_rxbuf_post_ts_bufs(dhd_pub_t *pub);
+
+static void dhd_prot_return_rxbuf(dhd_pub_t *dhd, msgbuf_ring_t *ring, uint32 pktid, uint32 rxcnt);
+
+/* D2H Message handling */
+static int dhd_prot_process_msgtype(dhd_pub_t *dhd, msgbuf_ring_t *ring, uint8 *buf, uint32 len);
+
+/* D2H Message handlers */
+static void dhd_prot_noop(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_txstatus_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_ioctcmplt_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_ioctack_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_ringstatus_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_genstatus_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_event_process(dhd_pub_t *dhd, void *msg);
+
+/* Loopback test with dongle */
+static void dmaxfer_free_dmaaddr(dhd_pub_t *dhd, dhd_dmaxfer_t *dma);
+static int dmaxfer_prepare_dmaaddr(dhd_pub_t *dhd, uint len, uint srcdelay,
+	uint destdelay, dhd_dmaxfer_t *dma);
+static void dhd_msgbuf_dmaxfer_process(dhd_pub_t *dhd, void *msg);
+
+/* Flowring management communication with dongle */
+static void dhd_prot_flow_ring_create_response_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_flow_ring_delete_response_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_flow_ring_flush_response_process(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_process_flow_ring_resume_response(dhd_pub_t *dhd, void* msg);
+static void dhd_prot_process_flow_ring_suspend_response(dhd_pub_t *dhd, void* msg);
+
+/* Monitor Mode */
+#if defined(WL_MONITOR)
+extern bool dhd_monitor_enabled(dhd_pub_t *dhd, int ifidx);
+extern void dhd_rx_mon_pkt(dhd_pub_t *dhdp, host_rxbuf_cmpl_t* msg, void *pkt, int ifidx);
+#if defined(DBG_PKT_MON)
+extern void dhd_80211_mon_pkt(dhd_pub_t *dhdp, host_rxbuf_cmpl_t* msg, void *pkt, int ifidx);
+#endif /* DBG_PKT_MON */
+#endif /* WL_MONITOR */
+
+/* Configure a soft doorbell per D2H ring */
+static void dhd_msgbuf_ring_config_d2h_soft_doorbell(dhd_pub_t *dhd);
+static void dhd_prot_process_d2h_ring_config_complete(dhd_pub_t *dhd, void *msg);
+static void dhd_prot_process_d2h_ring_create_complete(dhd_pub_t *dhd, void *buf);
+#if !defined(BCM_ROUTER_DHD)
+static void dhd_prot_process_h2d_ring_create_complete(dhd_pub_t *dhd, void *buf);
+static void dhd_prot_process_infobuf_complete(dhd_pub_t *dhd, void* buf);
+#endif /* !BCM_ROUTER_DHD */
+static void dhd_prot_process_d2h_mb_data(dhd_pub_t *dhd, void* buf);
+static void dhd_prot_detach_info_rings(dhd_pub_t *dhd);
+#ifdef EWP_EDL
+static void dhd_prot_detach_edl_rings(dhd_pub_t *dhd);
+#endif
+static void dhd_prot_detach_md_rings(dhd_pub_t *dhd);
+static void dhd_prot_process_d2h_host_ts_complete(dhd_pub_t *dhd, void* buf);
+static void dhd_prot_process_snapshot_complete(dhd_pub_t *dhd, void *buf);
+
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+void dhd_dump_bus_flow_ring_status_isr_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf);
+void dhd_dump_bus_flow_ring_status_dpc_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf);
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+
+#ifdef DHD_TX_PROFILE
+extern bool dhd_protocol_matches_profile(uint8 *p, int plen, const
+		dhd_tx_profile_protocol_t *proto, bool is_host_sfhllc);
+#endif /* defined(DHD_TX_PROFILE) */
+
+static void dhd_update_rxstats(dhd_pub_t *dhd, host_rxbuf_cmpl_t *rxstatus);
+typedef void (*dhd_msgbuf_func_t)(dhd_pub_t *dhd, void *msg);
+
+/** callback functions for messages generated by the dongle */
+#define MSG_TYPE_INVALID 0
+
+static dhd_msgbuf_func_t table_lookup[DHD_PROT_FUNCS] = {
+	dhd_prot_noop, /* 0 is MSG_TYPE_INVALID */
+	dhd_prot_genstatus_process, /* MSG_TYPE_GEN_STATUS */
+	dhd_prot_ringstatus_process, /* MSG_TYPE_RING_STATUS */
+	NULL,
+	dhd_prot_flow_ring_create_response_process, /* MSG_TYPE_FLOW_RING_CREATE_CMPLT */
+	NULL,
+	dhd_prot_flow_ring_delete_response_process, /* MSG_TYPE_FLOW_RING_DELETE_CMPLT */
+	NULL,
+	dhd_prot_flow_ring_flush_response_process, /* MSG_TYPE_FLOW_RING_FLUSH_CMPLT */
+	NULL,
+	dhd_prot_ioctack_process, /* MSG_TYPE_IOCTLPTR_REQ_ACK */
+	NULL,
+	dhd_prot_ioctcmplt_process, /* MSG_TYPE_IOCTL_CMPLT */
+	NULL,
+	dhd_prot_event_process, /* MSG_TYPE_WL_EVENT */
+	NULL,
+	dhd_prot_txstatus_process, /* MSG_TYPE_TX_STATUS */
+	NULL,
+	NULL,	/* MSG_TYPE_RX_CMPLT use dedicated handler */
+	NULL,
+	dhd_msgbuf_dmaxfer_process, /* MSG_TYPE_LPBK_DMAXFER_CMPLT */
+	NULL, /* MSG_TYPE_FLOW_RING_RESUME */
+	dhd_prot_process_flow_ring_resume_response, /* MSG_TYPE_FLOW_RING_RESUME_CMPLT */
+	NULL, /* MSG_TYPE_FLOW_RING_SUSPEND */
+	dhd_prot_process_flow_ring_suspend_response, /* MSG_TYPE_FLOW_RING_SUSPEND_CMPLT */
+	NULL, /* MSG_TYPE_INFO_BUF_POST */
+#if defined(BCM_ROUTER_DHD)
+	NULL, /* MSG_TYPE_INFO_BUF_CMPLT */
+#else
+	dhd_prot_process_infobuf_complete, /* MSG_TYPE_INFO_BUF_CMPLT */
+#endif /* BCM_ROUTER_DHD */
+	NULL, /* MSG_TYPE_H2D_RING_CREATE */
+	NULL, /* MSG_TYPE_D2H_RING_CREATE */
+#if defined(BCM_ROUTER_DHD)
+	NULL, /* MSG_TYPE_H2D_RING_CREATE_CMPLT */
+#else
+	dhd_prot_process_h2d_ring_create_complete, /* MSG_TYPE_H2D_RING_CREATE_CMPLT */
+#endif /* BCM_ROUTER_DHD */
+	dhd_prot_process_d2h_ring_create_complete, /* MSG_TYPE_D2H_RING_CREATE_CMPLT */
+	NULL, /* MSG_TYPE_H2D_RING_CONFIG */
+	NULL, /* MSG_TYPE_D2H_RING_CONFIG */
+	NULL, /* MSG_TYPE_H2D_RING_CONFIG_CMPLT */
+	dhd_prot_process_d2h_ring_config_complete, /* MSG_TYPE_D2H_RING_CONFIG_CMPLT */
+	NULL, /* MSG_TYPE_H2D_MAILBOX_DATA */
+	dhd_prot_process_d2h_mb_data, /* MSG_TYPE_D2H_MAILBOX_DATA */
+	NULL,	/* MSG_TYPE_TIMSTAMP_BUFPOST */
+	NULL,	/* MSG_TYPE_HOSTTIMSTAMP */
+	dhd_prot_process_d2h_host_ts_complete,	/* MSG_TYPE_HOSTTIMSTAMP_CMPLT */
+	dhd_prot_process_fw_timestamp,	/* MSG_TYPE_FIRMWARE_TIMESTAMP */
+	NULL,	/* MSG_TYPE_SNAPSHOT_UPLOAD */
+	dhd_prot_process_snapshot_complete,	/* MSG_TYPE_SNAPSHOT_CMPLT */
+};
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+/* Related to router CPU mapping per radio core */
+#define DHD_RX_CHAINING
+#endif /* BCM_ROUTER_DHD && HNDCTF */
+
+#ifdef DHD_RX_CHAINING
+
+#define PKT_CTF_CHAINABLE(dhd, ifidx, evh, prio, h_sa, h_da, h_prio) \
+	(dhd_wet_chainable(dhd) && \
+	dhd_rx_pkt_chainable((dhd), (ifidx)) && \
+	!ETHER_ISNULLDEST(((struct ether_header *)(evh))->ether_dhost) && \
+	!ETHER_ISMULTI(((struct ether_header *)(evh))->ether_dhost) && \
+	!eacmp((h_da), ((struct ether_header *)(evh))->ether_dhost) && \
+	!eacmp((h_sa), ((struct ether_header *)(evh))->ether_shost) && \
+	((h_prio) == (prio)) && (dhd_ctf_hotbrc_check((dhd), (evh), (ifidx))) && \
+	((((struct ether_header *)(evh))->ether_type == HTON16(ETHER_TYPE_IP)) || \
+	(((struct ether_header *)(evh))->ether_type == HTON16(ETHER_TYPE_IPV6))))
+
+static INLINE void dhd_rxchain_reset(rxchain_info_t *rxchain);
+static void dhd_rxchain_frame(dhd_pub_t *dhd, void *pkt, uint ifidx);
+static void dhd_rxchain_commit(dhd_pub_t *dhd);
+
+#define DHD_PKT_CTF_MAX_CHAIN_LEN	64
+
+#endif /* DHD_RX_CHAINING */
+
+#define DHD_LPBKDTDUMP_ON()	(dhd_msg_level & DHD_LPBKDTDUMP_VAL)
+
+static void dhd_prot_h2d_sync_init(dhd_pub_t *dhd);
+
+uint32
+dhd_prot_get_tx_post_bound(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	return prot->tx_post_bound;
+}
+
+uint32
+dhd_prot_get_ctrl_cpl_post_bound(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	return prot->ctrl_cpl_post_bound;
+}
+
+uint32
+dhd_prot_get_tx_cpl_bound(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	return prot->tx_cpl_bound;
+}
+
+uint32
+dhd_prot_get_rx_cpl_post_bound(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	return prot->rx_cpl_post_bound;
+}
+
+void
+dhd_prot_set_tx_post_bound(dhd_pub_t *dhd, uint32 val)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->tx_post_bound = val;
+}
+
+void
+dhd_prot_set_ctrl_cpl_post_bound(dhd_pub_t *dhd, uint32 val)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->ctrl_cpl_post_bound = val;
+}
+
+void
+dhd_prot_set_tx_cpl_bound(dhd_pub_t *dhd, uint32 val)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->tx_cpl_bound = val;
+}
+
+void dhd_prot_set_rx_cpl_post_bound(dhd_pub_t *dhd, uint32 val)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->rx_cpl_post_bound = val;
+}
+
+void
+dhd_prot_set_ring_size_ver(dhd_pub_t *dhd, int version)
+{
+	if (ring_size_alloc_version < version) {
+		DHD_ERROR(("%s: Ring alloced version(%d) is lesser than requested(%d), ABORT\n",
+			__FUNCTION__, ring_size_alloc_version, version));
+		return;
+	}
+	ring_size_version = version;
+
+	/* Change each parameters only if they are 0s, non-zero means,
+	 * it is overridden via module parameter.
+	 */
+	switch (version) {
+		case 1:
+			if (!h2d_max_txpost) {
+				h2d_max_txpost = H2DRING_TXPOST_SIZE_V1;
+			}
+			if (!h2d_htput_max_txpost) {
+				h2d_htput_max_txpost = H2DRING_HTPUT_TXPOST_SIZE_V1;
+			}
+			if (!d2h_max_txcpl) {
+				d2h_max_txcpl = D2HRING_TXCPL_SIZE_V1;
+			}
+
+			if (!h2d_max_rxpost) {
+				h2d_max_rxpost = H2DRING_RXPOST_SIZE_V1;
+			}
+			if (!d2h_max_rxcpl) {
+				d2h_max_rxcpl = D2HRING_RXCPL_SIZE_V1;
+			}
+
+			if (!h2d_max_ctrlpost) {
+				h2d_max_ctrlpost = H2DRING_CTRLPOST_SIZE_V1;
+			}
+			if (!d2h_max_ctrlcpl) {
+				d2h_max_ctrlcpl = D2HRING_CTRLCPL_SIZE_V1;
+			}
+
+			if (!rx_buf_burst) {
+				rx_buf_burst = RX_BUF_BURST_V1;
+			}
+			if (!rx_bufpost_threshold) {
+				rx_bufpost_threshold = RX_BUFPOST_THRESHOLD_V1;
+			}
+			break;
+		case 2:
+			if (!h2d_max_txpost) {
+				h2d_max_txpost = H2DRING_TXPOST_SIZE_V2;
+			}
+			if (!h2d_htput_max_txpost) {
+				h2d_htput_max_txpost = H2DRING_HTPUT_TXPOST_SIZE_V2;
+			}
+			if (!d2h_max_txcpl) {
+				d2h_max_txcpl = D2HRING_TXCPL_SIZE_V2;
+			}
+
+			if (!h2d_max_rxpost) {
+				h2d_max_rxpost = H2DRING_RXPOST_SIZE_V2;
+			}
+			if (!d2h_max_rxcpl) {
+				d2h_max_rxcpl = D2HRING_RXCPL_SIZE_V2;
+			}
+
+			if (!h2d_max_ctrlpost) {
+				h2d_max_ctrlpost = H2DRING_CTRLPOST_SIZE_V2;
+			}
+			if (!d2h_max_ctrlcpl) {
+				d2h_max_ctrlcpl = D2HRING_CTRLCPL_SIZE_V2;
+			}
+
+			if (!rx_buf_burst) {
+				rx_buf_burst = RX_BUF_BURST_V2;
+			}
+			if (!rx_bufpost_threshold) {
+				rx_bufpost_threshold = RX_BUFPOST_THRESHOLD_V2;
+			}
+			break;
+		default:
+			DHD_ERROR(("%s: invalid override version:%d", __FUNCTION__, version));
+	}
+}
+
+#ifdef D2H_MINIDUMP
+dhd_dma_buf_t *
+dhd_prot_get_minidump_buf(dhd_pub_t *dhd)
+{
+	return &dhd->prot->fw_trap_buf;
+}
+#endif /* D2H_MINIDUMP */
+
+uint16
+dhd_prot_get_rxbufpost_sz(dhd_pub_t *dhd)
+{
+	return dhd->prot->rxbufpost_sz;
+}
+
+uint16
+dhd_prot_get_rxbufpost_alloc_sz(dhd_pub_t *dhd)
+{
+	return dhd->prot->rxbufpost_alloc_sz;
+}
+uint16
+dhd_prot_get_h2d_rx_post_active(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *flow_ring = &prot->h2dring_rxp_subn;
+	uint16 rd, wr;
+
+	/* Since wr is owned by host in h2d direction, directly read wr */
+	wr = flow_ring->wr;
+
+	if (dhd->dma_d2h_ring_upd_support) {
+		rd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, flow_ring->idx);
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, flow_ring->idx);
+	}
+	return NTXPACTIVE(rd, wr, flow_ring->max_items);
+}
+
+uint16
+dhd_prot_get_d2h_rx_cpln_active(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *flow_ring = &prot->d2hring_rx_cpln;
+	uint16 rd, wr;
+
+	if (dhd->dma_d2h_ring_upd_support) {
+		wr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, flow_ring->idx);
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, flow_ring->idx);
+	}
+
+	/* Since rd is owned by host in d2h direction, directly read rd */
+	rd = flow_ring->rd;
+
+	return NTXPACTIVE(rd, wr, flow_ring->max_items);
+}
+
+bool
+dhd_prot_is_h2d_ring_empty(dhd_pub_t *dhd, void *prot_info)
+{
+	msgbuf_ring_t *ring = (msgbuf_ring_t *)prot_info;
+	uint16 rd, wr;
+	bool ret;
+
+	/* For h2d ring, WR is owned by host, RD is owned by dongle */
+	wr = ring->wr;
+	if (dhd->dma_h2d_ring_upd_support) {
+		rd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+	}
+	ret = (wr == rd) ? TRUE : FALSE;
+	return ret;
+}
+
+/* returns TRUE if there are pending ctrl completions to be processed at host */
+bool
+dhd_prot_check_pending_ctrl_cmpls(dhd_pub_t *dhd)
+{
+	msgbuf_ring_t *ring = &dhd->prot->d2hring_ctrl_cpln;
+	uint16 wr, host_rd;
+	bool ret;
+
+	/* For Completion ring, WR is owned by dongle, RD is owned by Host */
+	if (dhd->dma_d2h_ring_upd_support) {
+		wr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+	}
+
+	if (wr > ring->max_items) {
+		DHD_ERROR(("%s(): ring %p, ring->name %s, wr %d ring->max_items %d\n",
+			__FUNCTION__, ring, ring->name, wr, ring->max_items));
+		return FALSE;
+	}
+
+	host_rd = ring->rd;
+
+	/* Consider the ring as processed if host rd and dongle wr are same */
+	ret = (host_rd != wr) ? TRUE : FALSE;
+	return ret;
+}
+
+void
+dhd_prot_dump_ring_ptrs(void *prot_info)
+{
+	msgbuf_ring_t *ring = (msgbuf_ring_t *)prot_info;
+	DHD_ERROR(("%s curr_rd: %d rd: %d wr: %d \n", __FUNCTION__,
+		ring->curr_rd, ring->rd, ring->wr));
+}
+
+/**
+ * D2H DMA to completion callback handlers. Based on the mode advertised by the
+ * dongle through the PCIE shared region, the appropriate callback will be
+ * registered in the proto layer to be invoked prior to precessing any message
+ * from a D2H DMA ring. If the dongle uses a read barrier or another mode that
+ * does not require host participation, then a noop callback handler will be
+ * bound that simply returns the msg_type.
+ */
+static void dhd_prot_d2h_sync_livelock(dhd_pub_t *dhd, uint32 msg_seqnum, msgbuf_ring_t *ring,
+                                       uint32 tries, volatile uchar *msg, int msglen);
+static uint8 dhd_prot_d2h_sync_seqnum(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                                      volatile cmn_msg_hdr_t *msg, int msglen);
+static uint8 dhd_prot_d2h_sync_xorcsum(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                                       volatile cmn_msg_hdr_t *msg, int msglen);
+static uint8 dhd_prot_d2h_sync_none(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                                    volatile cmn_msg_hdr_t *msg, int msglen);
+static void dhd_prot_d2h_sync_init(dhd_pub_t *dhd);
+static int dhd_send_d2h_ringcreate(dhd_pub_t *dhd, msgbuf_ring_t *ring_to_create,
+	uint16 ring_type, uint32 id);
+static int dhd_send_h2d_ringcreate(dhd_pub_t *dhd, msgbuf_ring_t *ring_to_create,
+	uint8 type, uint32 id);
+
+/**
+ * dhd_prot_d2h_sync_livelock - when the host determines that a DMA transfer has
+ * not completed, a livelock condition occurs. Host will avert this livelock by
+ * dropping this message and moving to the next. This dropped message can lead
+ * to a packet leak, or even something disastrous in the case the dropped
+ * message happens to be a control response.
+ * Here we will log this condition. One may choose to reboot the dongle.
+ *
+ */
+static void
+dhd_prot_d2h_sync_livelock(dhd_pub_t *dhd, uint32 msg_seqnum, msgbuf_ring_t *ring, uint32 tries,
+                           volatile uchar *msg, int msglen)
+{
+	uint32 ring_seqnum = ring->seqnum;
+
+	if (dhd_query_bus_erros(dhd)) {
+		return;
+	}
+
+	DHD_ERROR((
+		"LIVELOCK DHD<%p> ring<%s> msg_seqnum<%u> ring_seqnum<%u:%u> tries<%u> max<%lu>"
+		" tot<%lu> dma_buf va<%p> msg<%p> curr_rd<%d> rd<%d> wr<%d>\n",
+		dhd, ring->name, msg_seqnum, ring_seqnum, ring_seqnum% D2H_EPOCH_MODULO, tries,
+		dhd->prot->d2h_sync_wait_max, dhd->prot->d2h_sync_wait_tot,
+		ring->dma_buf.va, msg, ring->curr_rd, ring->rd, ring->wr));
+
+	dhd_prhex("D2H MsgBuf Failure", msg, msglen, DHD_ERROR_VAL);
+
+	/* Try to resume if already suspended or suspend in progress */
+
+	/* Skip if still in suspended or suspend in progress */
+	if (DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhd)) {
+		DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state, so skip\n",
+			__FUNCTION__, dhd->busstate, dhd->dhd_bus_busy_state));
+		goto exit;
+	}
+
+	dhd_bus_dump_console_buffer(dhd->bus);
+	dhd_prot_debug_info_print(dhd);
+
+#ifdef DHD_FW_COREDUMP
+	if (dhd->memdump_enabled) {
+		/* collect core dump */
+		dhd->memdump_type = DUMP_TYPE_BY_LIVELOCK;
+		dhd_bus_mem_dump(dhd);
+	}
+#endif /* DHD_FW_COREDUMP */
+
+exit:
+	dhd_schedule_reset(dhd);
+
+#ifdef OEM_ANDROID
+#endif /* OEM_ANDROID */
+	dhd->livelock_occured = TRUE;
+}
+
+/**
+ * dhd_prot_d2h_sync_seqnum - Sync on a D2H DMA completion using the SEQNUM
+ * mode. Sequence number is always in the last word of a message.
+ */
+static uint8
+BCMFASTPATH(dhd_prot_d2h_sync_seqnum)(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                         volatile cmn_msg_hdr_t *msg, int msglen)
+{
+	uint32 tries;
+	uint32 ring_seqnum = ring->seqnum % D2H_EPOCH_MODULO;
+	int num_words = msglen / sizeof(uint32); /* num of 32bit words */
+	volatile uint32 *marker = (volatile uint32 *)msg + (num_words - 1); /* last word */
+	dhd_prot_t *prot = dhd->prot;
+	uint32 msg_seqnum;
+	uint32 step = 0;
+	uint32 delay = PCIE_D2H_SYNC_DELAY;
+	uint32 total_tries = 0;
+
+	ASSERT(msglen == ring->item_len);
+
+	BCM_REFERENCE(delay);
+	/*
+	 * For retries we have to make some sort of stepper algorithm.
+	 * We see that every time when the Dongle comes out of the D3
+	 * Cold state, the first D2H mem2mem DMA takes more time to
+	 * complete, leading to livelock issues.
+	 *
+	 * Case 1 - Apart from Host CPU some other bus master is
+	 * accessing the DDR port, probably page close to the ring
+	 * so, PCIE does not get a change to update the memory.
+	 * Solution - Increase the number of tries.
+	 *
+	 * Case 2 - The 50usec delay given by the Host CPU is not
+	 * sufficient for the PCIe RC to start its work.
+	 * In this case the breathing time of 50usec given by
+	 * the Host CPU is not sufficient.
+	 * Solution: Increase the delay in a stepper fashion.
+	 * This is done to ensure that there are no
+	 * unwanted extra delay introdcued in normal conditions.
+	 */
+	for (step = 1; step <= PCIE_D2H_SYNC_NUM_OF_STEPS; step++) {
+		for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
+			msg_seqnum = *marker;
+			if (ltoh32(msg_seqnum) == ring_seqnum) { /* dma upto last word done */
+				ring->seqnum++; /* next expected sequence number */
+				/* Check for LIVELOCK induce flag, which is set by firing
+				 * dhd iovar to induce LIVELOCK error. If flag is set,
+				 * MSG_TYPE_INVALID is returned, which results in to LIVELOCK error.
+				 */
+				if (dhd->dhd_induce_error != DHD_INDUCE_LIVELOCK) {
+					goto dma_completed;
+				}
+			}
+
+			total_tries = (uint32)(((step-1) * PCIE_D2H_SYNC_WAIT_TRIES) + tries);
+
+			if (total_tries > prot->d2h_sync_wait_max)
+				prot->d2h_sync_wait_max = total_tries;
+
+			OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
+			OSL_CPU_RELAX(); /* CPU relax for msg_seqnum  value to update */
+			OSL_DELAY(delay * step); /* Add stepper delay */
+
+		} /* for PCIE_D2H_SYNC_WAIT_TRIES */
+	} /* for PCIE_D2H_SYNC_NUM_OF_STEPS */
+
+	dhd_prot_d2h_sync_livelock(dhd, msg_seqnum, ring, total_tries,
+		(volatile uchar *) msg, msglen);
+
+	ring->seqnum++; /* skip this message ... leak of a pktid */
+	return MSG_TYPE_INVALID; /* invalid msg_type 0 -> noop callback */
+
+dma_completed:
+
+	prot->d2h_sync_wait_tot += tries;
+	return msg->msg_type;
+}
+
+/**
+ * dhd_prot_d2h_sync_xorcsum - Sync on a D2H DMA completion using the XORCSUM
+ * mode. The xorcsum is placed in the last word of a message. Dongle will also
+ * place a seqnum in the epoch field of the cmn_msg_hdr.
+ */
+static uint8
+BCMFASTPATH(dhd_prot_d2h_sync_xorcsum)(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                          volatile cmn_msg_hdr_t *msg, int msglen)
+{
+	uint32 tries;
+	uint32 prot_checksum = 0; /* computed checksum */
+	int num_words = msglen / sizeof(uint32); /* num of 32bit words */
+	uint8 ring_seqnum = ring->seqnum % D2H_EPOCH_MODULO;
+	dhd_prot_t *prot = dhd->prot;
+	uint32 step = 0;
+	uint32 delay = PCIE_D2H_SYNC_DELAY;
+	uint32 total_tries = 0;
+
+	ASSERT(msglen == ring->item_len);
+
+	BCM_REFERENCE(delay);
+	/*
+	 * For retries we have to make some sort of stepper algorithm.
+	 * We see that every time when the Dongle comes out of the D3
+	 * Cold state, the first D2H mem2mem DMA takes more time to
+	 * complete, leading to livelock issues.
+	 *
+	 * Case 1 - Apart from Host CPU some other bus master is
+	 * accessing the DDR port, probably page close to the ring
+	 * so, PCIE does not get a change to update the memory.
+	 * Solution - Increase the number of tries.
+	 *
+	 * Case 2 - The 50usec delay given by the Host CPU is not
+	 * sufficient for the PCIe RC to start its work.
+	 * In this case the breathing time of 50usec given by
+	 * the Host CPU is not sufficient.
+	 * Solution: Increase the delay in a stepper fashion.
+	 * This is done to ensure that there are no
+	 * unwanted extra delay introdcued in normal conditions.
+	 */
+	for (step = 1; step <= PCIE_D2H_SYNC_NUM_OF_STEPS; step++) {
+		for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
+			/* First verify if the seqnumber has been update,
+			 * if yes, then only check xorcsum.
+			 * Once seqnum and xorcsum is proper that means
+			 * complete message has arrived.
+			 */
+			if (msg->epoch == ring_seqnum) {
+				prot_checksum = bcm_compute_xor32((volatile uint32 *)msg,
+					num_words);
+				if (prot_checksum == 0U) { /* checksum is OK */
+					ring->seqnum++; /* next expected sequence number */
+					/* Check for LIVELOCK induce flag, which is set by firing
+					 * dhd iovar to induce LIVELOCK error. If flag is set,
+					 * MSG_TYPE_INVALID is returned, which results in to
+					 * LIVELOCK error.
+					 */
+					if (dhd->dhd_induce_error != DHD_INDUCE_LIVELOCK) {
+						goto dma_completed;
+					}
+				}
+			}
+
+			total_tries = ((step-1) * PCIE_D2H_SYNC_WAIT_TRIES) + tries;
+
+			if (total_tries > prot->d2h_sync_wait_max)
+				prot->d2h_sync_wait_max = total_tries;
+
+			OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
+			OSL_CPU_RELAX(); /* CPU relax for msg_seqnum  value to update */
+			OSL_DELAY(delay * step); /* Add stepper delay */
+
+		} /* for PCIE_D2H_SYNC_WAIT_TRIES */
+	} /* for PCIE_D2H_SYNC_NUM_OF_STEPS */
+
+	DHD_ERROR(("%s: prot_checksum = 0x%x\n", __FUNCTION__, prot_checksum));
+	dhd_prot_d2h_sync_livelock(dhd, msg->epoch, ring, total_tries,
+		(volatile uchar *) msg, msglen);
+
+	ring->seqnum++; /* skip this message ... leak of a pktid */
+	return MSG_TYPE_INVALID; /* invalid msg_type 0 -> noop callback */
+
+dma_completed:
+
+	prot->d2h_sync_wait_tot += tries;
+	return msg->msg_type;
+}
+
+/**
+ * dhd_prot_d2h_sync_none - Dongle ensure that the DMA will complete and host
+ * need to try to sync. This noop sync handler will be bound when the dongle
+ * advertises that neither the SEQNUM nor XORCSUM mode of DMA sync is required.
+ */
+static uint8
+BCMFASTPATH(dhd_prot_d2h_sync_none)(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                       volatile cmn_msg_hdr_t *msg, int msglen)
+{
+	/* Check for LIVELOCK induce flag, which is set by firing
+	* dhd iovar to induce LIVELOCK error. If flag is set,
+	* MSG_TYPE_INVALID is returned, which results in to LIVELOCK error.
+	*/
+	if (dhd->dhd_induce_error == DHD_INDUCE_LIVELOCK) {
+		DHD_ERROR(("%s: Inducing livelock\n", __FUNCTION__));
+		return MSG_TYPE_INVALID;
+	} else {
+		return msg->msg_type;
+	}
+}
+
+#ifdef EWP_EDL
+/**
+ * dhd_prot_d2h_sync_edl - Sync on a D2H DMA completion by validating the cmn_msg_hdr_t
+ * header values at both the beginning and end of the payload.
+ * The cmn_msg_hdr_t is placed at the start and end of the payload
+ * in each work item in the EDL ring.
+ * Dongle will place a seqnum inside the cmn_msg_hdr_t 'epoch' field
+ * and the length of the payload in the 'request_id' field.
+ * Structure of each work item in the EDL ring:
+ * | cmn_msg_hdr_t | payload (var len) | cmn_msg_hdr_t |
+ * NOTE: - it was felt that calculating xorcsum for the entire payload (max length of 1648 bytes) is
+ * too costly on the dongle side and might take up too many ARM cycles,
+ * hence the xorcsum sync method is not being used for EDL ring.
+ */
+static int
+BCMFASTPATH(dhd_prot_d2h_sync_edl)(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                          volatile cmn_msg_hdr_t *msg)
+{
+	uint32 tries;
+	int msglen = 0, len = 0;
+	uint32 ring_seqnum = ring->seqnum % D2H_EPOCH_MODULO;
+	dhd_prot_t *prot = dhd->prot;
+	uint32 step = 0;
+	uint32 delay = PCIE_D2H_SYNC_DELAY;
+	uint32 total_tries = 0;
+	volatile cmn_msg_hdr_t *trailer = NULL;
+	volatile uint8 *buf = NULL;
+	bool valid_msg = FALSE;
+
+	BCM_REFERENCE(delay);
+	/*
+	 * For retries we have to make some sort of stepper algorithm.
+	 * We see that every time when the Dongle comes out of the D3
+	 * Cold state, the first D2H mem2mem DMA takes more time to
+	 * complete, leading to livelock issues.
+	 *
+	 * Case 1 - Apart from Host CPU some other bus master is
+	 * accessing the DDR port, probably page close to the ring
+	 * so, PCIE does not get a change to update the memory.
+	 * Solution - Increase the number of tries.
+	 *
+	 * Case 2 - The 50usec delay given by the Host CPU is not
+	 * sufficient for the PCIe RC to start its work.
+	 * In this case the breathing time of 50usec given by
+	 * the Host CPU is not sufficient.
+	 * Solution: Increase the delay in a stepper fashion.
+	 * This is done to ensure that there are no
+	 * unwanted extra delay introdcued in normal conditions.
+	 */
+	for (step = 1; step <= PCIE_D2H_SYNC_NUM_OF_STEPS; step++) {
+		for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
+			/* First verify if the seqnumber has been updated,
+			 * if yes, only then validate the header and trailer.
+			 * Once seqnum, header and trailer have been validated, it means
+			 * that the complete message has arrived.
+			 */
+			valid_msg = FALSE;
+			if (msg->epoch == ring_seqnum &&
+				msg->msg_type == MSG_TYPE_INFO_PYLD &&
+				msg->request_id > 0 &&
+				msg->request_id <= ring->item_len) {
+				/* proceed to check trailer only if header is valid */
+				buf = (volatile uint8 *)msg;
+				msglen = sizeof(cmn_msg_hdr_t) + msg->request_id;
+				buf += msglen;
+				if (msglen + sizeof(cmn_msg_hdr_t) <= ring->item_len) {
+					trailer = (volatile cmn_msg_hdr_t *)buf;
+					valid_msg = (trailer->epoch == ring_seqnum) &&
+						(trailer->msg_type == msg->msg_type) &&
+						(trailer->request_id == msg->request_id);
+					if (!valid_msg) {
+						DHD_TRACE(("%s:invalid trailer! seqnum=%u;reqid=%u"
+						" expected, seqnum=%u; reqid=%u. Retrying... \n",
+						__FUNCTION__, trailer->epoch, trailer->request_id,
+						msg->epoch, msg->request_id));
+					}
+				} else {
+					DHD_TRACE(("%s: invalid payload length (%u)! Retrying.. \n",
+						__FUNCTION__, msg->request_id));
+				}
+
+				if (valid_msg) {
+					/* data is OK */
+					ring->seqnum++; /* next expected sequence number */
+					if (dhd->dhd_induce_error != DHD_INDUCE_LIVELOCK) {
+						goto dma_completed;
+					}
+				}
+			} else {
+				DHD_TRACE(("%s: wrong hdr, seqnum expected %u, got %u."
+					" msg_type=0x%x, request_id=%u."
+					" Retrying...\n",
+					__FUNCTION__, ring_seqnum, msg->epoch,
+					msg->msg_type, msg->request_id));
+			}
+
+			total_tries = ((step-1) * PCIE_D2H_SYNC_WAIT_TRIES) + tries;
+
+			if (total_tries > prot->d2h_sync_wait_max)
+				prot->d2h_sync_wait_max = total_tries;
+
+			OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
+#if !(defined(BCM_ROUTER_DHD) && defined(BCM_GMAC3))
+			OSL_CPU_RELAX(); /* CPU relax for msg_seqnum  value to update */
+			OSL_DELAY(delay * step); /* Add stepper delay */
+#endif /* !(defined(BCM_ROUTER_DHD) && defined(BCM_GMAC3)) */
+
+		} /* for PCIE_D2H_SYNC_WAIT_TRIES */
+	} /* for PCIE_D2H_SYNC_NUM_OF_STEPS */
+
+	DHD_ERROR(("%s: EDL header check fails !\n", __FUNCTION__));
+	DHD_ERROR(("%s: header: seqnum=%u; expected-seqnum=%u"
+		" msgtype=0x%x; expected-msgtype=0x%x"
+		" length=%u; expected-max-length=%u", __FUNCTION__,
+		msg->epoch, ring_seqnum, msg->msg_type, MSG_TYPE_INFO_PYLD,
+		msg->request_id, ring->item_len));
+	dhd_prhex("msg header bytes: ", (volatile uchar *)msg, sizeof(*msg), DHD_ERROR_VAL);
+	if (trailer && msglen > 0 &&
+			(msglen + sizeof(cmn_msg_hdr_t)) <= ring->item_len) {
+		DHD_ERROR(("%s: trailer: seqnum=%u; expected-seqnum=%u"
+			" msgtype=0x%x; expected-msgtype=0x%x"
+			" length=%u; expected-length=%u", __FUNCTION__,
+			trailer->epoch, ring_seqnum, trailer->msg_type, MSG_TYPE_INFO_PYLD,
+			trailer->request_id, msg->request_id));
+		dhd_prhex("msg trailer bytes: ", (volatile uchar *)trailer,
+			sizeof(*trailer), DHD_ERROR_VAL);
+	}
+
+	if ((msglen + sizeof(cmn_msg_hdr_t)) <= ring->item_len)
+		len = msglen + sizeof(cmn_msg_hdr_t);
+	else
+		len = ring->item_len;
+
+	dhd_prot_d2h_sync_livelock(dhd, msg->epoch, ring, total_tries,
+		(volatile uchar *) msg, len);
+
+	ring->seqnum++; /* skip this message */
+	return BCME_ERROR; /* invalid msg_type 0 -> noop callback */
+
+dma_completed:
+	DHD_TRACE(("%s: EDL header check pass, seqnum=%u; reqid=%u\n", __FUNCTION__,
+		msg->epoch, msg->request_id));
+
+	prot->d2h_sync_wait_tot += tries;
+	return BCME_OK;
+}
+
+/**
+ * dhd_prot_d2h_sync_edl_none - Dongle ensure that the DMA will complete and host
+ * need to try to sync. This noop sync handler will be bound when the dongle
+ * advertises that neither the SEQNUM nor XORCSUM mode of DMA sync is required.
+ */
+static int BCMFASTPATH
+(dhd_prot_d2h_sync_edl_none)(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+                       volatile cmn_msg_hdr_t *msg)
+{
+	/* Check for LIVELOCK induce flag, which is set by firing
+	* dhd iovar to induce LIVELOCK error. If flag is set,
+	* MSG_TYPE_INVALID is returned, which results in to LIVELOCK error.
+	*/
+	if (dhd->dhd_induce_error == DHD_INDUCE_LIVELOCK) {
+		DHD_ERROR(("%s: Inducing livelock\n", __FUNCTION__));
+		return BCME_ERROR;
+	} else {
+		if (msg->msg_type == MSG_TYPE_INFO_PYLD)
+			return BCME_OK;
+		else
+			return msg->msg_type;
+	}
+}
+#endif /* EWP_EDL */
+
+INLINE void
+dhd_wakeup_ioctl_event(dhd_pub_t *dhd, dhd_ioctl_recieved_status_t reason)
+{
+	/* To synchronize with the previous memory operations call wmb() */
+	OSL_SMP_WMB();
+	dhd->prot->ioctl_received = reason;
+	/* Call another wmb() to make sure before waking up the other event value gets updated */
+	OSL_SMP_WMB();
+	dhd_os_ioctl_resp_wake(dhd);
+}
+
+/**
+ * dhd_prot_d2h_sync_init - Setup the host side DMA sync mode based on what
+ * dongle advertizes.
+ */
+static void
+dhd_prot_d2h_sync_init(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->d2h_sync_wait_max = 0UL;
+	prot->d2h_sync_wait_tot = 0UL;
+
+	prot->d2hring_ctrl_cpln.seqnum = D2H_EPOCH_INIT_VAL;
+	prot->d2hring_ctrl_cpln.current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+
+	prot->d2hring_tx_cpln.seqnum = D2H_EPOCH_INIT_VAL;
+	prot->d2hring_tx_cpln.current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+
+	prot->d2hring_rx_cpln.seqnum = D2H_EPOCH_INIT_VAL;
+	prot->d2hring_rx_cpln.current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+
+	if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_SEQNUM) {
+		prot->d2h_sync_cb = dhd_prot_d2h_sync_seqnum;
+#ifdef EWP_EDL
+		prot->d2h_edl_sync_cb = dhd_prot_d2h_sync_edl;
+#endif /* EWP_EDL */
+		DHD_ERROR(("%s(): D2H sync mechanism is SEQNUM \r\n", __FUNCTION__));
+	} else if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_XORCSUM) {
+		prot->d2h_sync_cb = dhd_prot_d2h_sync_xorcsum;
+#ifdef EWP_EDL
+		prot->d2h_edl_sync_cb = dhd_prot_d2h_sync_edl;
+#endif /* EWP_EDL */
+		DHD_ERROR(("%s(): D2H sync mechanism is XORCSUM \r\n", __FUNCTION__));
+	} else {
+		prot->d2h_sync_cb = dhd_prot_d2h_sync_none;
+#ifdef EWP_EDL
+		prot->d2h_edl_sync_cb = dhd_prot_d2h_sync_edl_none;
+#endif /* EWP_EDL */
+		DHD_ERROR(("%s(): D2H sync mechanism is NONE \r\n", __FUNCTION__));
+	}
+}
+
+/**
+ * dhd_prot_h2d_sync_init - Per H2D common ring, setup the msgbuf ring seqnum
+ */
+static void
+dhd_prot_h2d_sync_init(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->h2dring_rxp_subn.seqnum = H2D_EPOCH_INIT_VAL;
+
+	prot->h2dring_rxp_subn.current_phase = 0;
+
+	prot->h2dring_ctrl_subn.seqnum = H2D_EPOCH_INIT_VAL;
+	prot->h2dring_ctrl_subn.current_phase = 0;
+}
+
+/* +-----------------  End of PCIE DHD H2D DMA SYNC ------------------------+ */
+
+/*
+ * +---------------------------------------------------------------------------+
+ * PCIE DMA-able buffer. Sets up a dhd_dma_buf_t object, which includes the
+ * virtual and physical address, the buffer lenght and the DMA handler.
+ * A secdma handler is also included in the dhd_dma_buf object.
+ * +---------------------------------------------------------------------------+
+ */
+
+static INLINE void
+dhd_base_addr_htolpa(sh_addr_t *base_addr, dmaaddr_t pa)
+{
+	base_addr->low_addr = htol32(PHYSADDRLO(pa));
+	base_addr->high_addr = htol32(PHYSADDRHI(pa));
+}
+
+/**
+ * dhd_dma_buf_audit - Any audits on a DHD DMA Buffer.
+ */
+static int
+dhd_dma_buf_audit(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf)
+{
+	uint32 pa_lowaddr, end; /* dongle uses 32bit ptr arithmetic */
+	ASSERT(dma_buf);
+	pa_lowaddr = PHYSADDRLO(dma_buf->pa);
+	ASSERT(PHYSADDRLO(dma_buf->pa) || PHYSADDRHI(dma_buf->pa));
+	ASSERT(ISALIGNED(pa_lowaddr, DMA_ALIGN_LEN));
+	ASSERT(dma_buf->len != 0);
+
+	/* test 32bit offset arithmetic over dma buffer for loss of carry-over */
+	end = (pa_lowaddr + dma_buf->len); /* end address */
+
+	if ((end & 0xFFFFFFFF) < (pa_lowaddr & 0xFFFFFFFF)) { /* exclude carryover */
+		DHD_ERROR(("%s: dma_buf %x len %d spans dongle 32bit ptr arithmetic\n",
+			__FUNCTION__, pa_lowaddr, dma_buf->len));
+		return BCME_ERROR;
+	}
+
+	return BCME_OK;
+}
+
+/**
+ * dhd_dma_buf_alloc - Allocate a cache coherent DMA-able buffer.
+ * returns BCME_OK=0 on success
+ * returns non-zero negative error value on failure.
+ */
+int
+dhd_dma_buf_alloc(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf, uint32 buf_len)
+{
+	uint32 dma_pad = 0;
+	osl_t *osh = dhd->osh;
+	uint16 dma_align = DMA_ALIGN_LEN;
+	uint32 rem = 0;
+
+	ASSERT(dma_buf != NULL);
+	ASSERT(dma_buf->va == NULL);
+	ASSERT(dma_buf->len == 0);
+
+	/* Pad the buffer length to align to cacheline size. */
+	rem = (buf_len % DHD_DMA_PAD);
+	dma_pad = rem ? (DHD_DMA_PAD - rem) : 0;
+
+	dma_buf->va = DMA_ALLOC_CONSISTENT(osh, buf_len + dma_pad,
+		dma_align, &dma_buf->_alloced, &dma_buf->pa, &dma_buf->dmah);
+
+	if (dma_buf->va == NULL) {
+		DHD_ERROR(("%s: buf_len %d, no memory available\n",
+			__FUNCTION__, buf_len));
+		return BCME_NOMEM;
+	}
+
+	dma_buf->len = buf_len; /* not including padded len */
+
+	if (dhd_dma_buf_audit(dhd, dma_buf) != BCME_OK) { /* audit dma buf */
+		dhd_dma_buf_free(dhd, dma_buf);
+		return BCME_ERROR;
+	}
+
+	dhd_dma_buf_reset(dhd, dma_buf); /* zero out and cache flush */
+
+	return BCME_OK;
+}
+
+/**
+ * dhd_dma_buf_reset - Reset a cache coherent DMA-able buffer.
+ */
+static void
+dhd_dma_buf_reset(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf)
+{
+	if ((dma_buf == NULL) || (dma_buf->va == NULL))
+		return;
+
+	(void)dhd_dma_buf_audit(dhd, dma_buf);
+
+	/* Zero out the entire buffer and cache flush */
+	memset((void*)dma_buf->va, 0, dma_buf->len);
+	OSL_CACHE_FLUSH((void *)dma_buf->va, dma_buf->len);
+}
+
+void
+dhd_local_buf_reset(char *buf, uint32 len)
+{
+	/* Zero out the entire buffer and cache flush */
+	memset((void*)buf, 0, len);
+	OSL_CACHE_FLUSH((void *)buf, len);
+}
+
+/**
+ * dhd_dma_buf_free - Free a DMA-able buffer that was previously allocated using
+ * dhd_dma_buf_alloc().
+ */
+void
+dhd_dma_buf_free(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf)
+{
+	osl_t *osh = dhd->osh;
+
+	ASSERT(dma_buf);
+
+	if (dma_buf->va == NULL)
+		return; /* Allow for free invocation, when alloc failed */
+
+	/* DEBUG: dhd_dma_buf_reset(dhd, dma_buf) */
+	(void)dhd_dma_buf_audit(dhd, dma_buf);
+
+	/* dma buffer may have been padded at allocation */
+	DMA_FREE_CONSISTENT(osh, dma_buf->va, dma_buf->_alloced,
+		dma_buf->pa, dma_buf->dmah);
+
+	memset(dma_buf, 0, sizeof(dhd_dma_buf_t));
+}
+
+/**
+ * dhd_dma_buf_init - Initialize a dhd_dma_buf with speicifed values.
+ * Do not use dhd_dma_buf_init to zero out a dhd_dma_buf_t object. Use memset 0.
+ */
+void
+dhd_dma_buf_init(dhd_pub_t *dhd, void *dhd_dma_buf,
+	void *va, uint32 len, dmaaddr_t pa, void *dmah, void *secdma)
+{
+	dhd_dma_buf_t *dma_buf;
+	ASSERT(dhd_dma_buf);
+	dma_buf = (dhd_dma_buf_t *)dhd_dma_buf;
+	dma_buf->va = va;
+	dma_buf->len = len;
+	dma_buf->pa = pa;
+	dma_buf->dmah = dmah;
+	dma_buf->secdma = secdma;
+
+	/* Audit user defined configuration */
+	(void)dhd_dma_buf_audit(dhd, dma_buf);
+}
+
+/* +------------------  End of PCIE DHD DMA BUF ADT ------------------------+ */
+
+/*
+ * +---------------------------------------------------------------------------+
+ * DHD_MAP_PKTID_LOGGING
+ * Logging the PKTID and DMA map/unmap information for the SMMU fault issue
+ * debugging in customer platform.
+ * +---------------------------------------------------------------------------+
+ */
+
+#ifdef DHD_MAP_PKTID_LOGGING
+typedef struct dhd_pktid_log_item {
+	dmaaddr_t pa;		/* DMA bus address */
+	uint64 ts_nsec;		/* Timestamp: nsec */
+	uint32 size;		/* DMA map/unmap size */
+	uint32 pktid;		/* Packet ID */
+	uint8 pkttype;		/* Packet Type */
+	uint8 rsvd[7];		/* Reserved for future use */
+} dhd_pktid_log_item_t;
+
+typedef struct dhd_pktid_log {
+	uint32 items;		/* number of total items */
+	uint32 index;		/* index of pktid_log_item */
+	dhd_pktid_log_item_t map[0];	/* metadata storage */
+} dhd_pktid_log_t;
+
+typedef void * dhd_pktid_log_handle_t; /* opaque handle to pktid log */
+
+#define	MAX_PKTID_LOG				(2048u)
+/*
+ * index timestamp pktaddr(pa) pktid size
+ * index(5) + timestamp (<= 20) + dummy(4) + pa (<=11) + pktid (10) + size(7) +
+ * pkttype(7) + space(6) + null/lf(2)
+ * ex) 1367          78348654722      0xabbe69040       3317    1920       1
+ */
+#define PKTID_LOG_STR_SZ			72u
+#define MAX_PKTID_LOG_BUF_SZ			MAX_PKTID_LOG * PKTID_LOG_STR_SZ + \
+	strlen(PKTID_LOG_DUMP_FMT)
+#define DHD_PKTID_LOG_ITEM_SZ			(sizeof(dhd_pktid_log_item_t))
+#define DHD_PKTID_LOG_SZ(items)			(uint32)((sizeof(dhd_pktid_log_t)) + \
+					((DHD_PKTID_LOG_ITEM_SZ) * (items)))
+
+#define DHD_PKTID_LOG_INIT(dhd, hdl)		dhd_pktid_logging_init((dhd), (hdl))
+#define DHD_PKTID_LOG_FINI(dhd, hdl)		dhd_pktid_logging_fini((dhd), (hdl))
+#define DHD_PKTID_LOG(dhd, hdl, pa, pktid, len, pkttype)	\
+	dhd_pktid_logging((dhd), (hdl), (pa), (pktid), (len), (pkttype))
+#ifndef DHD_IGNORE_PKTID_AUDIT_ERROR
+#define DHD_PKTID_LOG_DUMP(dhd)			dhd_pktid_logging_dump((dhd))
+#else
+#define DHD_PKTID_LOG_DUMP(dhd)
+#endif /* DHD_IGNORE_PKTID_AUDIT_ERROR */
+
+static dhd_pktid_log_handle_t *
+dhd_pktid_logging_init(dhd_pub_t *dhd, uint32 num_items)
+{
+	dhd_pktid_log_t *log;
+	uint32 log_size;
+
+	log_size = DHD_PKTID_LOG_SZ(num_items);
+	log = (dhd_pktid_log_t *)KVMALLOCZ(dhd->osh, log_size);
+	if (log == NULL) {
+		DHD_ERROR(("%s: KVMALLOC failed for size %d\n",
+			__FUNCTION__, log_size));
+		return (dhd_pktid_log_handle_t *)NULL;
+	}
+
+	log->items = num_items;
+	log->index = 0;
+
+	return (dhd_pktid_log_handle_t *)log; /* opaque handle */
+}
+
+static void
+dhd_pktid_logging_fini(dhd_pub_t *dhd, dhd_pktid_log_handle_t *handle)
+{
+	dhd_pktid_log_t *log;
+	uint32 log_size;
+
+	if (handle == NULL) {
+		DHD_ERROR(("%s: handle is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	log = (dhd_pktid_log_t *)handle;
+	log_size = DHD_PKTID_LOG_SZ(log->items);
+	KVMFREE(dhd->osh, handle, log_size);
+}
+
+static void
+dhd_pktid_logging(dhd_pub_t *dhd, dhd_pktid_log_handle_t *handle, dmaaddr_t pa,
+	uint32 pktid, uint32 len, uint8 pkttype)
+{
+	dhd_pktid_log_t *log;
+	uint32 idx;
+
+	if (handle == NULL) {
+		DHD_ERROR(("%s: handle is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	log = (dhd_pktid_log_t *)handle;
+	idx = log->index;
+	log->map[idx].ts_nsec = OSL_LOCALTIME_NS();
+	log->map[idx].pa = pa;
+	log->map[idx].pktid = pktid;
+	log->map[idx].size = len;
+	log->map[idx].pkttype = pkttype;
+	log->index = (idx + 1) % (log->items);	/* update index */
+}
+
+void
+dhd_pktid_logging_dump(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	dhd_pktid_log_t *map_log, *unmap_log;
+	uint64 ts_sec, ts_usec;
+
+	if (prot == NULL) {
+		DHD_ERROR(("%s: prot is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	dhd->enable_pktid_log_dump = TRUE;
+
+	map_log = (dhd_pktid_log_t *)(prot->pktid_dma_map);
+	unmap_log = (dhd_pktid_log_t *)(prot->pktid_dma_unmap);
+	OSL_GET_LOCALTIME(&ts_sec, &ts_usec);
+	if (map_log && unmap_log) {
+		DHD_ERROR(("%s: map_log(%s), unmap_log(%s)\n", __FUNCTION__,
+			is_vmalloc_addr(map_log) ? "vmalloc" : "kmalloc",
+			is_vmalloc_addr(unmap_log) ? "vmalloc" : "kmalloc"));
+		DHD_ERROR(("%s: map_idx=%d unmap_idx=%d "
+			"current time=[%5lu.%06lu]\n", __FUNCTION__,
+			map_log->index, unmap_log->index,
+			(unsigned long)ts_sec, (unsigned long)ts_usec));
+		DHD_ERROR(("%s: pktid_map_log(pa)=0x%llx size=%d, "
+			"pktid_unmap_log(pa)=0x%llx size=%d\n", __FUNCTION__,
+			(uint64)__virt_to_phys((ulong)(map_log->map)),
+			(uint32)(DHD_PKTID_LOG_ITEM_SZ * map_log->items),
+			(uint64)__virt_to_phys((ulong)(unmap_log->map)),
+			(uint32)(DHD_PKTID_LOG_ITEM_SZ * unmap_log->items)));
+	}
+}
+
+uint32
+dhd_pktid_buf_len(dhd_pub_t *dhd, bool is_map)
+{
+	dhd_prot_t *prot = dhd->prot;
+	dhd_pktid_log_t *pktlog_buf;
+	uint32 len = 0;
+
+	if (is_map == TRUE) {
+		pktlog_buf = (dhd_pktid_log_t *)(prot->pktid_dma_map);
+		len += strlen(DHD_PKTID_MAP_LOG_HDR);
+	} else {
+		pktlog_buf = (dhd_pktid_log_t *)(prot->pktid_dma_unmap);
+		len += strlen(DHD_PKTID_UNMAP_LOG_HDR);
+	}
+
+	if (!pktlog_buf) {
+		return 0; /* len should be 0. */
+	}
+
+	/* Adding Format string + LineFeed */
+	len += strlen(PKTID_LOG_DUMP_FMT) + 2;
+	len += (uint32)PKTID_LOG_STR_SZ * pktlog_buf->items;
+
+	return len;
+}
+
+int
+dhd_write_pktid_log_dump(dhd_pub_t *dhdp, const void *user_buf,
+	void *fp, uint32 len, unsigned long *pos, bool is_map)
+{
+	dhd_prot_t *prot = dhdp->prot;
+	dhd_pktid_log_t *log_buf;
+	log_dump_section_hdr_t sec_hdr;
+	char *buf;
+	uint32 buflen = 0, remain_len = 0;
+	unsigned long long addr = 0;
+	int i = 0;
+	int ret = BCME_OK;
+
+	if (prot == NULL) {
+		DHD_ERROR(("%s: prot is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	buf = (char *)VMALLOCZ(dhdp->osh, MAX_PKTID_LOG_BUF_SZ);
+	if (!buf) {
+		DHD_ERROR(("%s: buf alloc fails!\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+	remain_len = buflen = MAX_PKTID_LOG_BUF_SZ;
+
+	dhd_init_sec_hdr(&sec_hdr);
+	sec_hdr.magic = LOG_DUMP_MAGIC;
+
+	if (is_map) {
+		log_buf = prot->pktid_dma_map;
+		ret = dhd_export_debug_data(DHD_PKTID_MAP_LOG_HDR, fp, user_buf,
+			strlen(DHD_PKTID_MAP_LOG_HDR), pos);
+		if (ret < 0) {
+			DHD_ERROR(("%s :  PKTID_MAP_LOG_HDR write error\n",
+				__FUNCTION__));
+			goto exit;
+		}
+		sec_hdr.type = LOG_DUMP_SECTION_PKTID_MAP_LOG;
+	} else {
+		log_buf = prot->pktid_dma_unmap;
+		ret = dhd_export_debug_data(DHD_PKTID_UNMAP_LOG_HDR, fp, user_buf,
+			strlen(DHD_PKTID_UNMAP_LOG_HDR), pos);
+		if (ret < 0) {
+			DHD_ERROR(("%s :  PKTID_UNMAP_LOG_HDR write error\n",
+				__FUNCTION__));
+			goto exit;
+		}
+		sec_hdr.type = LOG_DUMP_SECTION_PKTID_UNMAP_LOG;
+	}
+
+	remain_len -= scnprintf(&buf[buflen - remain_len], remain_len,
+			PKTID_LOG_DUMP_FMT, log_buf->index);
+
+	for (i = 0; i < log_buf->items; i++) {
+#ifdef BCMDMA64OSL
+		PHYSADDRTOULONG(log_buf->map[i].pa, addr);
+#else
+		addr = PHYSADDRLO(log_buf->map[i].pa);
+#endif /* BCMDMA64OSL */
+		remain_len -= scnprintf(&buf[buflen - remain_len], remain_len,
+			"%5u %20llu %4s 0x%llx %10u %7u %7u\n",
+			i, log_buf->map[i].ts_nsec, " ", addr,
+			log_buf->map[i].pktid, log_buf->map[i].size,
+			log_buf->map[i].pkttype);
+	}
+	remain_len -= scnprintf(&buf[buflen - remain_len], remain_len, "\n");
+
+	sec_hdr.timestamp = local_clock();
+	sec_hdr.length = buflen - remain_len;
+
+	ret = dhd_export_debug_data((char *)&sec_hdr, NULL, user_buf, sizeof(sec_hdr), pos);
+	ret = dhd_export_debug_data(buf, NULL, user_buf, sec_hdr.length, pos);
+
+exit:
+	if (buf) {
+		VMFREE(dhdp->osh, buf, buflen);
+	}
+
+	return ret;
+}
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+/* +-----------------  End of DHD_MAP_PKTID_LOGGING -----------------------+ */
+
+/*
+ * +---------------------------------------------------------------------------+
+ * PktId Map: Provides a native packet pointer to unique 32bit PktId mapping.
+ * Main purpose is to save memory on the dongle, has other purposes as well.
+ * The packet id map, also includes storage for some packet parameters that
+ * may be saved. A native packet pointer along with the parameters may be saved
+ * and a unique 32bit pkt id will be returned. Later, the saved packet pointer
+ * and the metadata may be retrieved using the previously allocated packet id.
+ * +---------------------------------------------------------------------------+
+ */
+#define DHD_PCIE_PKTID
+
+/* On Router, the pktptr serves as a pktid. */
+#if defined(BCM_ROUTER_DHD) && !defined(BCA_HNDROUTER)
+#undef DHD_PCIE_PKTID		/* Comment this undef, to reenable PKTIDMAP */
+#endif /* BCM_ROUTER_DHD && !BCA_HNDROUTER */
+
+#if defined(BCM_ROUTER_DHD) && defined(DHD_PCIE_PKTID)
+#undef MAX_TX_PKTID
+#define MAX_TX_PKTID     ((36 * 1024) - 1) /* Extend for 64 clients support. */
+#endif /* BCM_ROUTER_DHD && DHD_PCIE_PKTID */
+
+/* XXX: PROP_TXSTATUS: WLFS defines a private pkttag layout.
+ * Hence cannot store the dma parameters in the pkttag and the pktidmap locker
+ * is required.
+ */
+#if defined(PROP_TXSTATUS) && !defined(DHD_PCIE_PKTID)
+#error "PKTIDMAP must be supported with PROP_TXSTATUS/WLFC"
+#endif
+
+/* Enum for marking the buffer color based on usage */
+typedef enum dhd_pkttype {
+	PKTTYPE_DATA_TX = 0,
+	PKTTYPE_DATA_RX,
+	PKTTYPE_IOCTL_RX,
+	PKTTYPE_EVENT_RX,
+	PKTTYPE_INFO_RX,
+	/* dhd_prot_pkt_free no check, if pktid reserved and no space avail case */
+	PKTTYPE_NO_CHECK,
+	PKTTYPE_TSBUF_RX
+} dhd_pkttype_t;
+
+#define DHD_PKTID_MIN_AVAIL_COUNT		512U
+#define DHD_PKTID_DEPLETED_MAX_COUNT		(DHD_PKTID_MIN_AVAIL_COUNT * 2U)
+#define DHD_PKTID_INVALID			(0U)
+#define DHD_IOCTL_REQ_PKTID			(0xFFFE)
+#define DHD_FAKE_PKTID				(0xFACE)
+#define DHD_H2D_DBGRING_REQ_PKTID		0xFFFD
+#define DHD_D2H_DBGRING_REQ_PKTID		0xFFFC
+#define DHD_H2D_HOSTTS_REQ_PKTID		0xFFFB
+#define DHD_H2D_BTLOGRING_REQ_PKTID		0xFFFA
+#define DHD_D2H_BTLOGRING_REQ_PKTID		0xFFF9
+#define DHD_H2D_SNAPSHOT_UPLOAD_REQ_PKTID	0xFFF8
+#define DHD_D2H_MDRING_REQ_PKTID		0xFFF8
+
+#define IS_FLOWRING(ring) \
+	((strncmp(ring->name, "h2dflr", sizeof("h2dflr"))) == (0))
+
+typedef void * dhd_pktid_map_handle_t; /* opaque handle to a pktid map */
+
+/* Construct a packet id mapping table, returning an opaque map handle */
+static dhd_pktid_map_handle_t *dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items);
+
+/* Destroy a packet id mapping table, freeing all packets active in the table */
+static void dhd_pktid_map_fini(dhd_pub_t *dhd, dhd_pktid_map_handle_t *map);
+
+#define DHD_NATIVE_TO_PKTID_INIT(dhd, items) dhd_pktid_map_init((dhd), (items))
+#define DHD_NATIVE_TO_PKTID_RESET(dhd, map)  dhd_pktid_map_reset((dhd), (map))
+#define DHD_NATIVE_TO_PKTID_FINI(dhd, map)   dhd_pktid_map_fini((dhd), (map))
+#define DHD_NATIVE_TO_PKTID_FINI_IOCTL(osh, map)  dhd_pktid_map_fini_ioctl((osh), (map))
+
+#if defined(DHD_PCIE_PKTID)
+#if defined(NDIS)
+/* XXX: for NDIS, using consistent memory instead of buffer from PKTGET for
+ * up to 8K ioctl response
+ */
+#define IOCTLRESP_USE_CONSTMEM
+static void free_ioctl_return_buffer(dhd_pub_t *dhd, dhd_dma_buf_t *retbuf);
+static int  alloc_ioctl_return_buffer(dhd_pub_t *dhd, dhd_dma_buf_t *retbuf);
+#endif
+
+/* Determine number of pktids that are available */
+static INLINE uint32 dhd_pktid_map_avail_cnt(dhd_pktid_map_handle_t *handle);
+
+/* Allocate a unique pktid against which a pkt and some metadata is saved */
+static INLINE uint32 dhd_pktid_map_reserve(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle,
+	void *pkt, dhd_pkttype_t pkttype);
+static INLINE void dhd_pktid_map_save(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle,
+	void *pkt, uint32 nkey, dmaaddr_t pa, uint32 len, uint8 dma,
+	void *dmah, void *secdma, dhd_pkttype_t pkttype);
+static uint32 dhd_pktid_map_alloc(dhd_pub_t *dhd, dhd_pktid_map_handle_t *map,
+	void *pkt, dmaaddr_t pa, uint32 len, uint8 dma,
+	void *dmah, void *secdma, dhd_pkttype_t pkttype);
+
+/* Return an allocated pktid, retrieving previously saved pkt and metadata */
+static void *dhd_pktid_map_free(dhd_pub_t *dhd, dhd_pktid_map_handle_t *map,
+	uint32 id, dmaaddr_t *pa, uint32 *len, void **dmah,
+	void **secdma, dhd_pkttype_t pkttype, bool rsv_locker);
+
+/* Metadata d2h completion ring linking/unlink ring types */
+typedef enum dhd_mdata_linked_ring_idx {
+	DHD_METADATA_NO_RING = 0,
+	DHD_METADATA_D2H_TXCPL,
+	DHD_METADATA_D2H_RXCPL,
+	DHD_METADATA_D2H_HP2PTX,
+	DHD_METADATA_D2H_HP2PRX
+} dhd_mdata_linked_ring_idx_t;
+
+/*
+ * DHD_PKTID_AUDIT_ENABLED: Audit of PktIds in DHD for duplicate alloc and frees
+ *
+ * DHD_PKTID_AUDIT_MAP: Audit the LIFO or FIFO PktIdMap allocator
+ * DHD_PKTID_AUDIT_RING: Audit the pktid during producer/consumer ring operation
+ *
+ * CAUTION: When DHD_PKTID_AUDIT_ENABLED is defined,
+ *    either DHD_PKTID_AUDIT_MAP or DHD_PKTID_AUDIT_RING may be selected.
+ */
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+#define USE_DHD_PKTID_AUDIT_LOCK 1
+/* Audit the pktidmap allocator */
+/* #define DHD_PKTID_AUDIT_MAP */
+
+/* Audit the pktid during production/consumption of workitems */
+#define DHD_PKTID_AUDIT_RING
+
+#if defined(DHD_PKTID_AUDIT_MAP) && defined(DHD_PKTID_AUDIT_RING)
+#error "May only enabled audit of MAP or RING, at a time."
+#endif /* DHD_PKTID_AUDIT_MAP && DHD_PKTID_AUDIT_RING */
+
+#define DHD_DUPLICATE_ALLOC     1
+#define DHD_DUPLICATE_FREE      2
+#define DHD_TEST_IS_ALLOC       3
+#define DHD_TEST_IS_FREE        4
+
+typedef enum dhd_pktid_map_type {
+	DHD_PKTID_MAP_TYPE_CTRL = 1,
+	DHD_PKTID_MAP_TYPE_TX,
+	DHD_PKTID_MAP_TYPE_RX,
+	DHD_PKTID_MAP_TYPE_UNKNOWN
+} dhd_pktid_map_type_t;
+
+#ifdef USE_DHD_PKTID_AUDIT_LOCK
+#define DHD_PKTID_AUDIT_LOCK_INIT(osh)          osl_spin_lock_init(osh)
+#define DHD_PKTID_AUDIT_LOCK_DEINIT(osh, lock)  osl_spin_lock_deinit(osh, lock)
+#define DHD_PKTID_AUDIT_LOCK(lock)              osl_spin_lock(lock)
+#define DHD_PKTID_AUDIT_UNLOCK(lock, flags)     osl_spin_unlock(lock, flags)
+#else
+#define DHD_PKTID_AUDIT_LOCK_INIT(osh)          (void *)(1)
+#define DHD_PKTID_AUDIT_LOCK_DEINIT(osh, lock)  do { /* noop */ } while (0)
+#define DHD_PKTID_AUDIT_LOCK(lock)              0
+#define DHD_PKTID_AUDIT_UNLOCK(lock, flags)     do { /* noop */ } while (0)
+#endif /* !USE_DHD_PKTID_AUDIT_LOCK */
+
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+#define USE_DHD_PKTID_LOCK   1
+
+#ifdef USE_DHD_PKTID_LOCK
+#define DHD_PKTID_LOCK_INIT(osh)                osl_spin_lock_init(osh)
+#define DHD_PKTID_LOCK_DEINIT(osh, lock)        osl_spin_lock_deinit(osh, lock)
+#define DHD_PKTID_LOCK(lock, flags)             (flags) = osl_spin_lock(lock)
+#define DHD_PKTID_UNLOCK(lock, flags)           osl_spin_unlock(lock, flags)
+#else
+#define DHD_PKTID_LOCK_INIT(osh)                (void *)(1)
+#define DHD_PKTID_LOCK_DEINIT(osh, lock)	\
+	do { \
+		BCM_REFERENCE(osh); \
+		BCM_REFERENCE(lock); \
+	} while (0)
+#define DHD_PKTID_LOCK(lock)                    0
+#define DHD_PKTID_UNLOCK(lock, flags)           \
+	do { \
+		BCM_REFERENCE(lock); \
+		BCM_REFERENCE(flags); \
+	} while (0)
+#endif /* !USE_DHD_PKTID_LOCK */
+
+typedef enum dhd_locker_state {
+	LOCKER_IS_FREE,
+	LOCKER_IS_BUSY,
+	LOCKER_IS_RSVD
+} dhd_locker_state_t;
+
+/* Packet metadata saved in packet id mapper */
+
+typedef struct dhd_pktid_item {
+	dhd_locker_state_t state;  /* tag a locker to be free, busy or reserved */
+	uint8       dir;      /* dma map direction (Tx=flush or Rx=invalidate) */
+	dhd_pkttype_t pkttype; /* pktlists are maintained based on pkttype */
+	uint16      len;      /* length of mapped packet's buffer */
+	void        *pkt;     /* opaque native pointer to a packet */
+	dmaaddr_t   pa;       /* physical address of mapped packet's buffer */
+	void        *dmah;    /* handle to OS specific DMA map */
+	void		*secdma;
+} dhd_pktid_item_t;
+
+typedef uint32 dhd_pktid_key_t;
+
+typedef struct dhd_pktid_map {
+	uint32      items;    /* total items in map */
+	uint32      avail;    /* total available items */
+	int         failures; /* lockers unavailable count */
+	/* Spinlock to protect dhd_pktid_map in process/tasklet context */
+	void        *pktid_lock; /* Used when USE_DHD_PKTID_LOCK is defined */
+
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+	void		*pktid_audit_lock;
+	struct bcm_mwbmap *pktid_audit; /* multi word bitmap based audit */
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+	dhd_pktid_key_t	*keys; /* map_items +1 unique pkt ids */
+	dhd_pktid_item_t lockers[0];           /* metadata storage */
+} dhd_pktid_map_t;
+
+/*
+ * PktId (Locker) #0 is never allocated and is considered invalid.
+ *
+ * On request for a pktid, a value DHD_PKTID_INVALID must be treated as a
+ * depleted pktid pool and must not be used by the caller.
+ *
+ * Likewise, a caller must never free a pktid of value DHD_PKTID_INVALID.
+ */
+
+#define DHD_PKTID_FREE_LOCKER           (FALSE)
+#define DHD_PKTID_RSV_LOCKER            (TRUE)
+
+#define DHD_PKTID_ITEM_SZ               (sizeof(dhd_pktid_item_t))
+#define DHD_PKIDMAP_ITEMS(items)        (items)
+#define DHD_PKTID_MAP_SZ(items)         (sizeof(dhd_pktid_map_t) + \
+	                                     (DHD_PKTID_ITEM_SZ * ((items) + 1)))
+#define DHD_PKTIDMAP_KEYS_SZ(items)     (sizeof(dhd_pktid_key_t) * ((items) + 1))
+
+#define DHD_NATIVE_TO_PKTID_RESET_IOCTL(dhd, map)  dhd_pktid_map_reset_ioctl((dhd), (map))
+
+/* Convert a packet to a pktid, and save pkt pointer in busy locker */
+#define DHD_NATIVE_TO_PKTID_RSV(dhd, map, pkt, pkttype)    \
+	dhd_pktid_map_reserve((dhd), (map), (pkt), (pkttype))
+/* Reuse a previously reserved locker to save packet params */
+#define DHD_NATIVE_TO_PKTID_SAVE(dhd, map, pkt, nkey, pa, len, dir, dmah, secdma, pkttype) \
+	dhd_pktid_map_save((dhd), (map), (void *)(pkt), (nkey), (pa), (uint32)(len), \
+		(uint8)(dir), (void *)(dmah), (void *)(secdma), \
+		(dhd_pkttype_t)(pkttype))
+/* Convert a packet to a pktid, and save packet params in locker */
+#define DHD_NATIVE_TO_PKTID(dhd, map, pkt, pa, len, dir, dmah, secdma, pkttype) \
+	dhd_pktid_map_alloc((dhd), (map), (void *)(pkt), (pa), (uint32)(len), \
+		(uint8)(dir), (void *)(dmah), (void *)(secdma), \
+		(dhd_pkttype_t)(pkttype))
+
+/* Convert pktid to a packet, and free the locker */
+#define DHD_PKTID_TO_NATIVE(dhd, map, pktid, pa, len, dmah, secdma, pkttype) \
+	dhd_pktid_map_free((dhd), (map), (uint32)(pktid), \
+		(dmaaddr_t *)&(pa), (uint32 *)&(len), (void **)&(dmah), \
+		(void **)&(secdma), (dhd_pkttype_t)(pkttype), DHD_PKTID_FREE_LOCKER)
+
+/* Convert the pktid to a packet, empty locker, but keep it reserved */
+#define DHD_PKTID_TO_NATIVE_RSV(dhd, map, pktid, pa, len, dmah, secdma, pkttype) \
+	dhd_pktid_map_free((dhd), (map), (uint32)(pktid), \
+	                   (dmaaddr_t *)&(pa), (uint32 *)&(len), (void **)&(dmah), \
+	                   (void **)&(secdma), (dhd_pkttype_t)(pkttype), DHD_PKTID_RSV_LOCKER)
+
+#define DHD_PKTID_AVAIL(map)                 dhd_pktid_map_avail_cnt(map)
+
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+
+#ifndef DHD_IGNORE_PKTID_AUDIT_ERROR
+#define DHD_PKTID_ERROR_HANDLER(dhdp)	dhd_pktid_error_handler((dhdp))
+#else
+#define DHD_PKTID_ERROR_HANDLER(dhdp)
+#endif /* DHD_IGNORE_PKTID_AUDIT_ERROR */
+
+static int
+dhd_get_pktid_map_type(dhd_pub_t *dhd, dhd_pktid_map_t *pktid_map)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int pktid_map_type;
+
+	if (pktid_map == prot->pktid_ctrl_map) {
+		pktid_map_type = DHD_PKTID_MAP_TYPE_CTRL;
+	} else if (pktid_map == prot->pktid_tx_map) {
+		pktid_map_type = DHD_PKTID_MAP_TYPE_TX;
+	} else if (pktid_map == prot->pktid_rx_map) {
+		pktid_map_type = DHD_PKTID_MAP_TYPE_RX;
+	} else {
+		pktid_map_type = DHD_PKTID_MAP_TYPE_UNKNOWN;
+	}
+
+	return pktid_map_type;
+}
+
+/**
+* __dhd_pktid_audit - Use the mwbmap to audit validity of a pktid.
+*/
+static int
+__dhd_pktid_audit(dhd_pub_t *dhd, dhd_pktid_map_t *pktid_map, uint32 pktid,
+	const int test_for, const char *errmsg)
+{
+#define DHD_PKT_AUDIT_STR "ERROR: %16s Host PktId Audit: "
+	struct bcm_mwbmap *handle;
+	uint32	flags;
+	bool ignore_audit;
+	int error = BCME_OK;
+
+	if (pktid_map == (dhd_pktid_map_t *)NULL) {
+		DHD_ERROR((DHD_PKT_AUDIT_STR "Pkt id map NULL\n", errmsg));
+		return BCME_OK;
+	}
+
+	flags = DHD_PKTID_AUDIT_LOCK(pktid_map->pktid_audit_lock);
+
+	handle = pktid_map->pktid_audit;
+	if (handle == (struct bcm_mwbmap *)NULL) {
+		DHD_ERROR((DHD_PKT_AUDIT_STR "Handle NULL\n", errmsg));
+		goto out;
+	}
+
+	/* Exclude special pktids from audit */
+	ignore_audit = (pktid == DHD_IOCTL_REQ_PKTID) | (pktid == DHD_FAKE_PKTID);
+	if (ignore_audit) {
+		goto out;
+	}
+
+	if ((pktid == DHD_PKTID_INVALID) || (pktid > pktid_map->items)) {
+		DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> invalid\n", errmsg, pktid));
+		error = BCME_ERROR;
+		goto out;
+	}
+
+	/* Perform audit */
+	switch (test_for) {
+		case DHD_DUPLICATE_ALLOC:
+			if (!bcm_mwbmap_isfree(handle, pktid)) {
+				DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> alloc duplicate\n",
+				           errmsg, pktid));
+				error = BCME_ERROR;
+			} else {
+				bcm_mwbmap_force(handle, pktid);
+			}
+			break;
+
+		case DHD_DUPLICATE_FREE:
+			if (bcm_mwbmap_isfree(handle, pktid)) {
+				DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> free duplicate\n",
+				           errmsg, pktid));
+				error = BCME_ERROR;
+			} else {
+				bcm_mwbmap_free(handle, pktid);
+			}
+			break;
+
+		case DHD_TEST_IS_ALLOC:
+			if (bcm_mwbmap_isfree(handle, pktid)) {
+				DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> is not allocated\n",
+				           errmsg, pktid));
+				error = BCME_ERROR;
+			}
+			break;
+
+		case DHD_TEST_IS_FREE:
+			if (!bcm_mwbmap_isfree(handle, pktid)) {
+				DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> is not free",
+				           errmsg, pktid));
+				error = BCME_ERROR;
+			}
+			break;
+
+		default:
+			DHD_ERROR(("%s: Invalid test case: %d\n", __FUNCTION__, test_for));
+			error = BCME_ERROR;
+			break;
+	}
+
+out:
+	DHD_PKTID_AUDIT_UNLOCK(pktid_map->pktid_audit_lock, flags);
+
+	if (error != BCME_OK) {
+#ifndef DHD_IGNORE_PKTID_AUDIT_ERROR
+		dhd->pktid_audit_failed = TRUE;
+#endif /* !DHD_IGNORE_PKTID_AUDIT_ERROR */
+	}
+
+	return error;
+}
+
+static int
+dhd_pktid_audit(dhd_pub_t *dhdp, dhd_pktid_map_t *pktid_map, uint32 pktid,
+	const int test_for, const char *errmsg)
+{
+	int ret = BCME_OK;
+	ret = __dhd_pktid_audit(dhdp, pktid_map, pktid, test_for, errmsg);
+	if (ret == BCME_ERROR) {
+		DHD_ERROR(("%s: Got Pkt Id Audit failure: PKTID<%d> PKTID MAP TYPE<%d>\n",
+			__FUNCTION__, pktid, dhd_get_pktid_map_type(dhdp, pktid_map)));
+		DHD_PKTID_ERROR_HANDLER(dhdp);
+#ifdef DHD_MAP_PKTID_LOGGING
+		DHD_PKTID_LOG_DUMP(dhdp);
+#endif /* DHD_MAP_PKTID_LOGGING */
+	}
+
+	return ret;
+}
+
+#define DHD_PKTID_AUDIT(dhdp, map, pktid, test_for) \
+	dhd_pktid_audit((dhdp), (dhd_pktid_map_t *)(map), (pktid), (test_for), __FUNCTION__)
+
+static int
+dhd_pktid_audit_ring_debug(dhd_pub_t *dhdp, dhd_pktid_map_t *map, uint32 pktid,
+	const int test_for, void *msg, uint32 msg_len, const char *func)
+{
+	int ret = BCME_OK;
+
+	if (dhd_query_bus_erros(dhdp)) {
+		return BCME_ERROR;
+	}
+
+	ret = __dhd_pktid_audit(dhdp, map, pktid, test_for, func);
+	if (ret == BCME_ERROR) {
+		DHD_ERROR(("%s: Got Pkt Id Audit failure: PKTID<%d> PKTID MAP TYPE<%d>\n",
+			__FUNCTION__, pktid, dhd_get_pktid_map_type(dhdp, map)));
+		prhex(func, (uchar *)msg, msg_len);
+		if (map == dhdp->prot->pktid_ctrl_map) {
+#ifdef DHD_DEBUG_INVALID_PKTID
+			prhex("ctrl_cpl_snapshot", dhdp->prot->ctrl_cpl_snapshot,
+				D2HRING_CTRL_CMPLT_ITEMSIZE);
+#endif /* DHD_DEBUG_INVALID_PKTID */
+		}
+		DHD_ERROR(("%s: ### For pktid audit failures, read curr_rd as curr_rd - 1,"
+			" as it is updated early before item processing ###\n",
+			__FUNCTION__));
+		dhd_prot_debug_ring_info(dhdp);
+		DHD_PKTID_ERROR_HANDLER(dhdp);
+#ifdef DHD_MAP_PKTID_LOGGING
+		DHD_PKTID_LOG_DUMP(dhdp);
+#endif /* DHD_MAP_PKTID_LOGGING */
+	}
+	return ret;
+}
+#define DHD_PKTID_AUDIT_RING_DEBUG(dhdp, map, pktid, test_for, msg, msg_len) \
+	dhd_pktid_audit_ring_debug((dhdp), (dhd_pktid_map_t *)(map), \
+		(pktid), (test_for), msg, msg_len, __FUNCTION__)
+
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+/**
+ * +---------------------------------------------------------------------------+
+ * Packet to Packet Id mapper using a <numbered_key, locker> paradigm.
+ *
+ * dhd_pktid_map manages a set of unique Packet Ids range[1..MAX_xxx_PKTID].
+ *
+ * dhd_pktid_map_alloc() may be used to save some packet metadata, and a unique
+ * packet id is returned. This unique packet id may be used to retrieve the
+ * previously saved packet metadata, using dhd_pktid_map_free(). On invocation
+ * of dhd_pktid_map_free(), the unique packet id is essentially freed. A
+ * subsequent call to dhd_pktid_map_alloc() may reuse this packet id.
+ *
+ * Implementation Note:
+ * Convert this into a <key,locker> abstraction and place into bcmutils !
+ * Locker abstraction should treat contents as opaque storage, and a
+ * callback should be registered to handle busy lockers on destructor.
+ *
+ * +---------------------------------------------------------------------------+
+ */
+
+/** Allocate and initialize a mapper of num_items <numbered_key, locker> */
+
+static dhd_pktid_map_handle_t *
+dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items)
+{
+	void* osh;
+	uint32 nkey;
+	dhd_pktid_map_t *map;
+	uint32 dhd_pktid_map_sz;
+	uint32 map_items;
+	uint32 map_keys_sz;
+	osh = dhd->osh;
+
+	dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(num_items);
+
+	map = (dhd_pktid_map_t *)VMALLOC(osh, dhd_pktid_map_sz);
+	if (map == NULL) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for size %d\n",
+			__FUNCTION__, __LINE__, dhd_pktid_map_sz));
+		return (dhd_pktid_map_handle_t *)NULL;
+	}
+
+	map->items = num_items;
+	map->avail = num_items;
+
+	map_items = DHD_PKIDMAP_ITEMS(map->items);
+
+	map_keys_sz = DHD_PKTIDMAP_KEYS_SZ(map->items);
+
+	/* Initialize the lock that protects this structure */
+	map->pktid_lock = DHD_PKTID_LOCK_INIT(osh);
+	if (map->pktid_lock == NULL) {
+		DHD_ERROR(("%s:%d: Lock init failed \r\n", __FUNCTION__, __LINE__));
+		goto error;
+	}
+
+	map->keys = (dhd_pktid_key_t *)VMALLOC(osh, map_keys_sz);
+	if (map->keys == NULL) {
+		DHD_ERROR(("%s:%d: VMALLOC failed for map->keys size %d\n",
+			__FUNCTION__, __LINE__, map_keys_sz));
+		goto error;
+	}
+
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+		/* Incarnate a hierarchical multiword bitmap for auditing pktid allocator */
+		map->pktid_audit = bcm_mwbmap_init(osh, map_items + 1);
+		if (map->pktid_audit == (struct bcm_mwbmap *)NULL) {
+			DHD_ERROR(("%s:%d: pktid_audit init failed\r\n", __FUNCTION__, __LINE__));
+			goto error;
+		} else {
+			DHD_ERROR(("%s:%d: pktid_audit init succeeded %d\n",
+				__FUNCTION__, __LINE__, map_items + 1));
+		}
+		map->pktid_audit_lock = DHD_PKTID_AUDIT_LOCK_INIT(osh);
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+	for (nkey = 1; nkey <= map_items; nkey++) { /* locker #0 is reserved */
+		map->keys[nkey] = nkey; /* populate with unique keys */
+		map->lockers[nkey].state = LOCKER_IS_FREE;
+		map->lockers[nkey].pkt   = NULL; /* bzero: redundant */
+		map->lockers[nkey].len   = 0;
+	}
+
+	/* Reserve pktid #0, i.e. DHD_PKTID_INVALID to be inuse */
+	map->lockers[DHD_PKTID_INVALID].state = LOCKER_IS_BUSY; /* tag locker #0 as inuse */
+	map->lockers[DHD_PKTID_INVALID].pkt   = NULL; /* bzero: redundant */
+	map->lockers[DHD_PKTID_INVALID].len   = 0;
+
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+	/* do not use dhd_pktid_audit() here, use bcm_mwbmap_force directly */
+	bcm_mwbmap_force(map->pktid_audit, DHD_PKTID_INVALID);
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+	return (dhd_pktid_map_handle_t *)map; /* opaque handle */
+
+error:
+	if (map) {
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+		if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
+			bcm_mwbmap_fini(osh, map->pktid_audit); /* Destruct pktid_audit */
+			map->pktid_audit = (struct bcm_mwbmap *)NULL;
+			if (map->pktid_audit_lock)
+				DHD_PKTID_AUDIT_LOCK_DEINIT(osh, map->pktid_audit_lock);
+		}
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+		if (map->keys) {
+			VMFREE(osh, map->keys, map_keys_sz);
+		}
+
+		if (map->pktid_lock) {
+			DHD_PKTID_LOCK_DEINIT(osh, map->pktid_lock);
+		}
+
+		VMFREE(osh, map, dhd_pktid_map_sz);
+	}
+	return (dhd_pktid_map_handle_t *)NULL;
+}
+
+/**
+ * Retrieve all allocated keys and free all <numbered_key, locker>.
+ * Freeing implies: unmapping the buffers and freeing the native packet
+ * This could have been a callback registered with the pktid mapper.
+ */
+static void
+dhd_pktid_map_reset(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle)
+{
+	void *osh;
+	uint32 nkey;
+	dhd_pktid_map_t *map;
+	dhd_pktid_item_t *locker;
+	uint32 map_items;
+	unsigned long flags;
+	bool data_tx = FALSE;
+
+	if (handle == NULL) {
+		return;
+	}
+
+	map = (dhd_pktid_map_t *)handle;
+	DHD_PKTID_LOCK(map->pktid_lock, flags);
+	osh = dhd->osh;
+
+	map_items = DHD_PKIDMAP_ITEMS(map->items);
+	/* skip reserved KEY #0, and start from 1 */
+
+	for (nkey = 1; nkey <= map_items; nkey++) {
+		if (map->lockers[nkey].state == LOCKER_IS_BUSY) {
+			locker = &map->lockers[nkey];
+			locker->state = LOCKER_IS_FREE;
+			data_tx = (locker->pkttype == PKTTYPE_DATA_TX);
+			if (data_tx) {
+				OSL_ATOMIC_DEC(dhd->osh, &dhd->prot->active_tx_count);
+			}
+
+#ifdef DHD_PKTID_AUDIT_RING
+			DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* duplicate frees */
+#endif /* DHD_PKTID_AUDIT_RING */
+#ifdef DHD_MAP_PKTID_LOGGING
+			DHD_PKTID_LOG(dhd, dhd->prot->pktid_dma_unmap,
+				locker->pa, nkey, locker->len,
+				locker->pkttype);
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+			DMA_UNMAP(osh, locker->pa, locker->len, locker->dir, 0, locker->dmah);
+			dhd_prot_packet_free(dhd, (ulong*)locker->pkt,
+				locker->pkttype, data_tx);
+		}
+		else {
+#ifdef DHD_PKTID_AUDIT_RING
+			DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_FREE);
+#endif /* DHD_PKTID_AUDIT_RING */
+		}
+		map->keys[nkey] = nkey; /* populate with unique keys */
+	}
+
+	map->avail = map_items;
+	memset(&map->lockers[1], 0, sizeof(dhd_pktid_item_t) * map_items);
+	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+}
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+/** Called in detach scenario. Releasing IOCTL buffers. */
+static void
+dhd_pktid_map_reset_ioctl(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle)
+{
+	uint32 nkey;
+	dhd_pktid_map_t *map;
+	dhd_pktid_item_t *locker;
+	uint32 map_items;
+	unsigned long flags;
+
+	if (handle == NULL) {
+		return;
+	}
+
+	map = (dhd_pktid_map_t *)handle;
+	DHD_PKTID_LOCK(map->pktid_lock, flags);
+
+	map_items = DHD_PKIDMAP_ITEMS(map->items);
+	/* skip reserved KEY #0, and start from 1 */
+	for (nkey = 1; nkey <= map_items; nkey++) {
+		if (map->lockers[nkey].state == LOCKER_IS_BUSY) {
+			dhd_dma_buf_t retbuf;
+
+#ifdef DHD_PKTID_AUDIT_RING
+			DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* duplicate frees */
+#endif /* DHD_PKTID_AUDIT_RING */
+
+			locker = &map->lockers[nkey];
+			retbuf.va = locker->pkt;
+			retbuf.len = locker->len;
+			retbuf.pa = locker->pa;
+			retbuf.dmah = locker->dmah;
+			retbuf.secdma = locker->secdma;
+
+			free_ioctl_return_buffer(dhd, &retbuf);
+		}
+		else {
+#ifdef DHD_PKTID_AUDIT_RING
+			DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_FREE);
+#endif /* DHD_PKTID_AUDIT_RING */
+		}
+		map->keys[nkey] = nkey; /* populate with unique keys */
+	}
+
+	map->avail = map_items;
+	memset(&map->lockers[1], 0, sizeof(dhd_pktid_item_t) * map_items);
+	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+}
+#endif /* IOCTLRESP_USE_CONSTMEM */
+
+/**
+ * Free the pktid map.
+ */
+static void
+dhd_pktid_map_fini(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle)
+{
+	dhd_pktid_map_t *map;
+	uint32 dhd_pktid_map_sz;
+	uint32 map_keys_sz;
+
+	if (handle == NULL)
+		return;
+
+	/* Free any pending packets */
+	dhd_pktid_map_reset(dhd, handle);
+
+	map = (dhd_pktid_map_t *)handle;
+	dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(map->items);
+	map_keys_sz = DHD_PKTIDMAP_KEYS_SZ(map->items);
+
+	DHD_PKTID_LOCK_DEINIT(dhd->osh, map->pktid_lock);
+
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+	if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
+		bcm_mwbmap_fini(dhd->osh, map->pktid_audit); /* Destruct pktid_audit */
+		map->pktid_audit = (struct bcm_mwbmap *)NULL;
+		if (map->pktid_audit_lock) {
+			DHD_PKTID_AUDIT_LOCK_DEINIT(dhd->osh, map->pktid_audit_lock);
+		}
+	}
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+	VMFREE(dhd->osh, map->keys, map_keys_sz);
+	VMFREE(dhd->osh, handle, dhd_pktid_map_sz);
+}
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+static void
+dhd_pktid_map_fini_ioctl(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle)
+{
+	dhd_pktid_map_t *map;
+	uint32 dhd_pktid_map_sz;
+	uint32 map_keys_sz;
+
+	if (handle == NULL)
+		return;
+
+	/* Free any pending packets */
+	dhd_pktid_map_reset_ioctl(dhd, handle);
+
+	map = (dhd_pktid_map_t *)handle;
+	dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(map->items);
+	map_keys_sz = DHD_PKTIDMAP_KEYS_SZ(map->items);
+
+	DHD_PKTID_LOCK_DEINIT(dhd->osh, map->pktid_lock);
+
+#if defined(DHD_PKTID_AUDIT_ENABLED)
+	if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
+		bcm_mwbmap_fini(dhd->osh, map->pktid_audit); /* Destruct pktid_audit */
+		map->pktid_audit = (struct bcm_mwbmap *)NULL;
+		if (map->pktid_audit_lock) {
+			DHD_PKTID_AUDIT_LOCK_DEINIT(dhd->osh, map->pktid_audit_lock);
+		}
+	}
+#endif /* DHD_PKTID_AUDIT_ENABLED */
+
+	VMFREE(dhd->osh, map->keys, map_keys_sz);
+	VMFREE(dhd->osh, handle, dhd_pktid_map_sz);
+}
+#endif /* IOCTLRESP_USE_CONSTMEM */
+
+/** Get the pktid free count */
+static INLINE uint32
+BCMFASTPATH(dhd_pktid_map_avail_cnt)(dhd_pktid_map_handle_t *handle)
+{
+	dhd_pktid_map_t *map;
+	uint32	avail;
+	unsigned long flags;
+
+	ASSERT(handle != NULL);
+	map = (dhd_pktid_map_t *)handle;
+
+	DHD_PKTID_LOCK(map->pktid_lock, flags);
+	avail = map->avail;
+	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+
+	return avail;
+}
+
+/**
+ * dhd_pktid_map_reserve - reserve a unique numbered key. Reserved locker is not
+ * yet populated. Invoke the pktid save api to populate the packet parameters
+ * into the locker. This function is not reentrant, and is the caller's
+ * responsibility. Caller must treat a returned value DHD_PKTID_INVALID as
+ * a failure case, implying a depleted pool of pktids.
+ */
+static INLINE uint32
+dhd_pktid_map_reserve(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle,
+	void *pkt, dhd_pkttype_t pkttype)
+{
+	uint32 nkey;
+	dhd_pktid_map_t *map;
+	dhd_pktid_item_t *locker;
+	unsigned long flags;
+
+	ASSERT(handle != NULL);
+	map = (dhd_pktid_map_t *)handle;
+
+	DHD_PKTID_LOCK(map->pktid_lock, flags);
+
+	if ((int)(map->avail) <= 0) { /* no more pktids to allocate */
+		map->failures++;
+		DHD_INFO(("%s:%d: failed, no free keys\n", __FUNCTION__, __LINE__));
+		DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+		return DHD_PKTID_INVALID; /* failed alloc request */
+	}
+
+	ASSERT(map->avail <= map->items);
+	nkey = map->keys[map->avail]; /* fetch a free locker, pop stack */
+
+	if ((map->avail > map->items) || (nkey > map->items)) {
+		map->failures++;
+		DHD_ERROR(("%s:%d: failed to allocate a new pktid,"
+			" map->avail<%u>, nkey<%u>, pkttype<%u>\n",
+			__FUNCTION__, __LINE__, map->avail, nkey,
+			pkttype));
+		DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+		return DHD_PKTID_INVALID; /* failed alloc request */
+	}
+
+	locker = &map->lockers[nkey]; /* save packet metadata in locker */
+	map->avail--;
+	locker->pkt = pkt; /* pkt is saved, other params not yet saved. */
+	locker->len = 0;
+	locker->state = LOCKER_IS_BUSY; /* reserve this locker */
+
+	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+
+	ASSERT(nkey != DHD_PKTID_INVALID);
+
+	return nkey; /* return locker's numbered key */
+}
+
+/*
+ * dhd_pktid_map_save - Save a packet's parameters into a locker
+ * corresponding to a previously reserved unique numbered key.
+ */
+static INLINE void
+dhd_pktid_map_save(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle, void *pkt,
+	uint32 nkey, dmaaddr_t pa, uint32 len, uint8 dir, void *dmah, void *secdma,
+	dhd_pkttype_t pkttype)
+{
+	dhd_pktid_map_t *map;
+	dhd_pktid_item_t *locker;
+	unsigned long flags;
+
+	ASSERT(handle != NULL);
+	map = (dhd_pktid_map_t *)handle;
+
+	DHD_PKTID_LOCK(map->pktid_lock, flags);
+
+	if ((nkey == DHD_PKTID_INVALID) || (nkey > DHD_PKIDMAP_ITEMS(map->items)) ||
+			(dhd->dhd_induce_error == DHD_INDUCE_PKTID_INVALID_SAVE)) {
+		DHD_ERROR(("%s:%d: Error! saving invalid pktid<%u> pkttype<%u>\n",
+			__FUNCTION__, __LINE__, nkey, pkttype));
+		DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			dhd->pktid_invalid_occured = TRUE;
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
+			dhd_bus_mem_dump(dhd);
+#ifdef OEM_ANDROID
+			dhd->hang_reason = HANG_REASON_PCIE_PKTID_ERROR;
+			dhd_os_send_hang_message(dhd);
+#endif /* OEM_ANDROID */
+
+		}
+#else
+		ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+		return;
+	}
+
+	locker = &map->lockers[nkey];
+
+	ASSERT(((locker->state == LOCKER_IS_BUSY) && (locker->pkt == pkt)) ||
+		((locker->state == LOCKER_IS_RSVD) && (locker->pkt == NULL)));
+
+	/* store contents in locker */
+	locker->dir = dir;
+	locker->pa = pa;
+	locker->len = (uint16)len; /* 16bit len */
+	locker->dmah = dmah; /* 16bit len */
+	locker->secdma = secdma;
+	locker->pkttype = pkttype;
+	locker->pkt = pkt;
+	locker->state = LOCKER_IS_BUSY; /* make this locker busy */
+#ifdef DHD_MAP_PKTID_LOGGING
+	DHD_PKTID_LOG(dhd, dhd->prot->pktid_dma_map, pa, nkey, len, pkttype);
+#endif /* DHD_MAP_PKTID_LOGGING */
+	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+}
+
+/**
+ * dhd_pktid_map_alloc - Allocate a unique numbered key and save the packet
+ * contents into the corresponding locker. Return the numbered key.
+ */
+static uint32
+BCMFASTPATH(dhd_pktid_map_alloc)(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle, void *pkt,
+	dmaaddr_t pa, uint32 len, uint8 dir, void *dmah, void *secdma,
+	dhd_pkttype_t pkttype)
+{
+	uint32 nkey;
+
+	nkey = dhd_pktid_map_reserve(dhd, handle, pkt, pkttype);
+	if (nkey != DHD_PKTID_INVALID) {
+		dhd_pktid_map_save(dhd, handle, pkt, nkey, pa,
+			len, dir, dmah, secdma, pkttype);
+	}
+
+	return nkey;
+}
+
+/**
+ * dhd_pktid_map_free - Given a numbered key, return the locker contents.
+ * dhd_pktid_map_free() is not reentrant, and is the caller's responsibility.
+ * Caller may not free a pktid value DHD_PKTID_INVALID or an arbitrary pktid
+ * value. Only a previously allocated pktid may be freed.
+ */
+static void *
+BCMFASTPATH(dhd_pktid_map_free)(dhd_pub_t *dhd, dhd_pktid_map_handle_t *handle, uint32 nkey,
+	dmaaddr_t *pa, uint32 *len, void **dmah, void **secdma, dhd_pkttype_t pkttype,
+	bool rsv_locker)
+{
+	dhd_pktid_map_t *map;
+	dhd_pktid_item_t *locker;
+	void * pkt;
+	unsigned long long locker_addr;
+	unsigned long flags;
+
+	ASSERT(handle != NULL);
+
+	map = (dhd_pktid_map_t *)handle;
+
+	DHD_PKTID_LOCK(map->pktid_lock, flags);
+
+	/* XXX PLEASE DO NOT remove this ASSERT, fix the bug in caller. */
+	if ((nkey == DHD_PKTID_INVALID) || (nkey > DHD_PKIDMAP_ITEMS(map->items)) ||
+			(dhd->dhd_induce_error == DHD_INDUCE_PKTID_INVALID_FREE)) {
+		DHD_ERROR(("%s:%d: Error! Try to free invalid pktid<%u>, pkttype<%d>\n",
+		           __FUNCTION__, __LINE__, nkey, pkttype));
+		DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			dhd->pktid_invalid_occured = TRUE;
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
+			dhd_bus_mem_dump(dhd);
+#ifdef OEM_ANDROID
+			dhd->hang_reason = HANG_REASON_PCIE_PKTID_ERROR;
+			dhd_os_send_hang_message(dhd);
+#endif /* OEM_ANDROID */
+		}
+#else
+		ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+		return NULL;
+	}
+
+	locker = &map->lockers[nkey];
+
+#if defined(DHD_PKTID_AUDIT_MAP)
+	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* Audit duplicate FREE */
+#endif /* DHD_PKTID_AUDIT_MAP */
+
+	/* Debug check for cloned numbered key */
+	if (locker->state == LOCKER_IS_FREE) {
+		DHD_ERROR(("%s:%d: Error! freeing already freed invalid pktid<%u>\n",
+		           __FUNCTION__, __LINE__, nkey));
+		DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+		/* XXX PLEASE DO NOT remove this ASSERT, fix the bug in caller. */
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			dhd->pktid_invalid_occured = TRUE;
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
+			dhd_bus_mem_dump(dhd);
+#ifdef OEM_ANDROID
+			dhd->hang_reason = HANG_REASON_PCIE_PKTID_ERROR;
+			dhd_os_send_hang_message(dhd);
+#endif /* OEM_ANDROID */
+		}
+#else
+		ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+		return NULL;
+	}
+
+	/* Check for the colour of the buffer i.e The buffer posted for TX,
+	 * should be freed for TX completion. Similarly the buffer posted for
+	 * IOCTL should be freed for IOCT completion etc.
+	 */
+	if ((pkttype != PKTTYPE_NO_CHECK) && (locker->pkttype != pkttype)) {
+
+		DHD_ERROR(("%s:%d: Error! Invalid Buffer Free for pktid<%u> \n",
+			__FUNCTION__, __LINE__, nkey));
+#ifdef BCMDMA64OSL
+		PHYSADDRTOULONG(locker->pa, locker_addr);
+#else
+		locker_addr = PHYSADDRLO(locker->pa);
+#endif /* BCMDMA64OSL */
+		DHD_ERROR(("%s:%d: locker->state <%d>, locker->pkttype <%d>,"
+			"pkttype <%d> locker->pa <0x%llx> \n",
+			__FUNCTION__, __LINE__, locker->state, locker->pkttype,
+			pkttype, locker_addr));
+		DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			dhd->pktid_invalid_occured = TRUE;
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
+			dhd_bus_mem_dump(dhd);
+#ifdef OEM_ANDROID
+			dhd->hang_reason = HANG_REASON_PCIE_PKTID_ERROR;
+			dhd_os_send_hang_message(dhd);
+#endif /* OEM_ANDROID */
+		}
+#else
+		ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+		return NULL;
+	}
+
+	if (rsv_locker == DHD_PKTID_FREE_LOCKER) {
+		map->avail++;
+		map->keys[map->avail] = nkey; /* make this numbered key available */
+		locker->state = LOCKER_IS_FREE; /* open and free Locker */
+	} else {
+		/* pktid will be reused, but the locker does not have a valid pkt */
+		locker->state = LOCKER_IS_RSVD;
+	}
+
+#if defined(DHD_PKTID_AUDIT_MAP)
+	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_FREE);
+#endif /* DHD_PKTID_AUDIT_MAP */
+#ifdef DHD_MAP_PKTID_LOGGING
+	DHD_PKTID_LOG(dhd, dhd->prot->pktid_dma_unmap, locker->pa, nkey,
+		(uint32)locker->len, pkttype);
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+	*pa = locker->pa; /* return contents of locker */
+	*len = (uint32)locker->len;
+	*dmah = locker->dmah;
+	*secdma = locker->secdma;
+
+	pkt = locker->pkt;
+	locker->pkt = NULL; /* Clear pkt */
+	locker->len = 0;
+
+	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
+
+	return pkt;
+}
+
+#else /* ! DHD_PCIE_PKTID */
+
+#ifndef linux
+#error "DHD_PCIE_PKTID has to be defined for non-linux/android platforms"
+#endif
+
+typedef struct pktlist {
+	PKT_LIST *tx_pkt_list;		/* list for tx packets */
+	PKT_LIST *rx_pkt_list;		/* list for rx packets */
+	PKT_LIST *ctrl_pkt_list;	/* list for ioctl/event buf post */
+} pktlists_t;
+
+/*
+ * Given that each workitem only uses a 32bit pktid, only 32bit hosts may avail
+ * of a one to one mapping 32bit pktptr and a 32bit pktid.
+ *
+ * - When PKTIDMAP is not used, DHD_NATIVE_TO_PKTID variants will never fail.
+ * - Neither DHD_NATIVE_TO_PKTID nor DHD_PKTID_TO_NATIVE need to be protected by
+ *   a lock.
+ * - Hence DHD_PKTID_INVALID is not defined when DHD_PCIE_PKTID is undefined.
+ */
+#define DHD_PKTID32(pktptr32)	((uint32)(pktptr32))
+#define DHD_PKTPTR32(pktid32)	((void *)(pktid32))
+
+static INLINE uint32 dhd_native_to_pktid(dhd_pktid_map_handle_t *map, void *pktptr32,
+	dmaaddr_t pa, uint32 dma_len, void *dmah, void *secdma,
+	dhd_pkttype_t pkttype);
+static INLINE void * dhd_pktid_to_native(dhd_pktid_map_handle_t *map, uint32 pktid32,
+	dmaaddr_t *pa, uint32 *dma_len, void **dmah, void **secdma,
+	dhd_pkttype_t pkttype);
+
+static dhd_pktid_map_handle_t *
+dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items)
+{
+	osl_t *osh = dhd->osh;
+	pktlists_t *handle = NULL;
+
+	if ((handle = (pktlists_t *) MALLOCZ(osh, sizeof(pktlists_t))) == NULL) {
+		DHD_ERROR(("%s:%d: MALLOC failed for lists allocation, size=%d\n",
+		           __FUNCTION__, __LINE__, sizeof(pktlists_t)));
+		goto error_done;
+	}
+
+	if ((handle->tx_pkt_list = (PKT_LIST *) MALLOC(osh, sizeof(PKT_LIST))) == NULL) {
+		DHD_ERROR(("%s:%d: MALLOC failed for list allocation, size=%d\n",
+		           __FUNCTION__, __LINE__, sizeof(PKT_LIST)));
+		goto error;
+	}
+
+	if ((handle->rx_pkt_list = (PKT_LIST *) MALLOC(osh, sizeof(PKT_LIST))) == NULL) {
+		DHD_ERROR(("%s:%d: MALLOC failed for list allocation, size=%d\n",
+		           __FUNCTION__, __LINE__, sizeof(PKT_LIST)));
+		goto error;
+	}
+
+	if ((handle->ctrl_pkt_list = (PKT_LIST *) MALLOC(osh, sizeof(PKT_LIST))) == NULL) {
+		DHD_ERROR(("%s:%d: MALLOC failed for list allocation, size=%d\n",
+		           __FUNCTION__, __LINE__, sizeof(PKT_LIST)));
+		goto error;
+	}
+
+	PKTLIST_INIT(handle->tx_pkt_list);
+	PKTLIST_INIT(handle->rx_pkt_list);
+	PKTLIST_INIT(handle->ctrl_pkt_list);
+
+	return (dhd_pktid_map_handle_t *) handle;
+
+error:
+	if (handle->ctrl_pkt_list) {
+		MFREE(osh, handle->ctrl_pkt_list, sizeof(PKT_LIST));
+	}
+
+	if (handle->rx_pkt_list) {
+		MFREE(osh, handle->rx_pkt_list, sizeof(PKT_LIST));
+	}
+
+	if (handle->tx_pkt_list) {
+		MFREE(osh, handle->tx_pkt_list, sizeof(PKT_LIST));
+	}
+
+	if (handle) {
+		MFREE(osh, handle, sizeof(pktlists_t));
+	}
+
+error_done:
+	return (dhd_pktid_map_handle_t *)NULL;
+}
+
+static void
+dhd_pktid_map_reset(dhd_pub_t *dhd, pktlists_t *handle)
+{
+	osl_t *osh = dhd->osh;
+
+	if (handle == NULL) {
+		return;
+	}
+
+	if (handle->ctrl_pkt_list) {
+		PKTLIST_FINI(handle->ctrl_pkt_list);
+		MFREE(osh, handle->ctrl_pkt_list, sizeof(PKT_LIST));
+	}
+
+	if (handle->rx_pkt_list) {
+		PKTLIST_FINI(handle->rx_pkt_list);
+		MFREE(osh, handle->rx_pkt_list, sizeof(PKT_LIST));
+	}
+
+	if (handle->tx_pkt_list) {
+		PKTLIST_FINI(handle->tx_pkt_list);
+		MFREE(osh, handle->tx_pkt_list, sizeof(PKT_LIST));
+	}
+}
+
+static void
+dhd_pktid_map_fini(dhd_pub_t *dhd, dhd_pktid_map_handle_t *map)
+{
+	osl_t *osh = dhd->osh;
+	pktlists_t *handle = (pktlists_t *) map;
+
+	ASSERT(handle != NULL);
+	if (handle == (pktlists_t *)NULL) {
+		return;
+	}
+
+	dhd_pktid_map_reset(dhd, handle);
+
+	if (handle) {
+		MFREE(osh, handle, sizeof(pktlists_t));
+	}
+}
+
+/** Save dma parameters into the packet's pkttag and convert a pktptr to pktid */
+static INLINE uint32
+dhd_native_to_pktid(dhd_pktid_map_handle_t *map, void *pktptr32,
+	dmaaddr_t pa, uint32 dma_len, void *dmah, void *secdma,
+	dhd_pkttype_t pkttype)
+{
+	pktlists_t *handle = (pktlists_t *) map;
+	ASSERT(pktptr32 != NULL);
+	DHD_PKT_SET_DMA_LEN(pktptr32, dma_len);
+	DHD_PKT_SET_DMAH(pktptr32, dmah);
+	DHD_PKT_SET_PA(pktptr32, pa);
+	DHD_PKT_SET_SECDMA(pktptr32, secdma);
+
+	/* XXX optimize these branch conditionals */
+	if (pkttype == PKTTYPE_DATA_TX) {
+		PKTLIST_ENQ(handle->tx_pkt_list,  pktptr32);
+	} else if (pkttype == PKTTYPE_DATA_RX) {
+		PKTLIST_ENQ(handle->rx_pkt_list,  pktptr32);
+	} else {
+		PKTLIST_ENQ(handle->ctrl_pkt_list,  pktptr32);
+	}
+
+	return DHD_PKTID32(pktptr32);
+}
+
+/** Convert a pktid to pktptr and retrieve saved dma parameters from packet */
+static INLINE void *
+dhd_pktid_to_native(dhd_pktid_map_handle_t *map, uint32 pktid32,
+	dmaaddr_t *pa, uint32 *dma_len, void **dmah, void **secdma,
+	dhd_pkttype_t pkttype)
+{
+	pktlists_t *handle = (pktlists_t *) map;
+	void *pktptr32;
+
+	ASSERT(pktid32 != 0U);
+	pktptr32 = DHD_PKTPTR32(pktid32);
+	*dma_len = DHD_PKT_GET_DMA_LEN(pktptr32);
+	*dmah = DHD_PKT_GET_DMAH(pktptr32);
+	*pa = DHD_PKT_GET_PA(pktptr32);
+	*secdma = DHD_PKT_GET_SECDMA(pktptr32);
+
+	/* XXX optimize these branch conditionals */
+	if (pkttype == PKTTYPE_DATA_TX) {
+		PKTLIST_UNLINK(handle->tx_pkt_list,  pktptr32);
+	} else if (pkttype == PKTTYPE_DATA_RX) {
+		PKTLIST_UNLINK(handle->rx_pkt_list,  pktptr32);
+	} else {
+		PKTLIST_UNLINK(handle->ctrl_pkt_list,  pktptr32);
+	}
+
+	return pktptr32;
+}
+
+#define DHD_NATIVE_TO_PKTID_RSV(dhd, map, pkt, pkttype)  DHD_PKTID32(pkt)
+
+#define DHD_NATIVE_TO_PKTID_SAVE(dhd, map, pkt, nkey, pa, len, dma_dir, dmah, secdma, pkttype) \
+	({ BCM_REFERENCE(dhd); BCM_REFERENCE(nkey); BCM_REFERENCE(dma_dir); \
+	   dhd_native_to_pktid((dhd_pktid_map_handle_t *) map, (pkt), (pa), (len), \
+			   (dmah), (secdma), (dhd_pkttype_t)(pkttype)); \
+	})
+
+#define DHD_NATIVE_TO_PKTID(dhd, map, pkt, pa, len, dma_dir, dmah, secdma, pkttype) \
+	({ BCM_REFERENCE(dhd); BCM_REFERENCE(dma_dir); \
+	   dhd_native_to_pktid((dhd_pktid_map_handle_t *) map, (pkt), (pa), (len), \
+			   (dmah), (secdma), (dhd_pkttype_t)(pkttype)); \
+	})
+
+#define DHD_PKTID_TO_NATIVE(dhd, map, pktid, pa, len, dmah, secdma, pkttype) \
+	({ BCM_REFERENCE(dhd); BCM_REFERENCE(pkttype);	\
+		dhd_pktid_to_native((dhd_pktid_map_handle_t *) map, (uint32)(pktid), \
+				(dmaaddr_t *)&(pa), (uint32 *)&(len), (void **)&(dmah), \
+				(void **)&secdma, (dhd_pkttype_t)(pkttype)); \
+	})
+
+#define DHD_PKTID_AVAIL(map)  (~0)
+
+#endif /* ! DHD_PCIE_PKTID */
+
+/* +------------------ End of PCIE DHD PKTID MAPPER  -----------------------+ */
+
+/*
+ * Allocating buffers for common rings.
+ * also allocating Buffers for hmaptest, Scratch buffer for dma rx offset,
+ * bus_throughput_measurement and snapshot upload
+ */
+static int
+dhd_prot_allocate_bufs(dhd_pub_t *dhd, dhd_prot_t *prot)
+{
+	/* Common Ring Allocations */
+
+	/* Ring  0: H2D Control Submission */
+	if (dhd_prot_ring_attach(dhd, &prot->h2dring_ctrl_subn, "h2dctrl",
+	        (uint16)h2d_max_ctrlpost, H2DRING_CTRL_SUB_ITEMSIZE,
+	        BCMPCIE_H2D_MSGRING_CONTROL_SUBMIT) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_prot_ring_attach H2D Ctrl Submission failed\n",
+			__FUNCTION__));
+		goto fail;
+	}
+
+	/* Ring  1: H2D Receive Buffer Post */
+	if (dhd_prot_ring_attach(dhd, &prot->h2dring_rxp_subn, "h2drxp",
+	        (uint16)h2d_max_rxpost, H2DRING_RXPOST_ITEMSIZE,
+	        BCMPCIE_H2D_MSGRING_RXPOST_SUBMIT) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_prot_ring_attach H2D RxPost failed\n",
+			__FUNCTION__));
+		goto fail;
+	}
+
+	/* Ring  2: D2H Control Completion */
+	if (dhd_prot_ring_attach(dhd, &prot->d2hring_ctrl_cpln, "d2hctrl",
+	        (uint16)d2h_max_ctrlcpl, D2HRING_CTRL_CMPLT_ITEMSIZE,
+	        BCMPCIE_D2H_MSGRING_CONTROL_COMPLETE) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_prot_ring_attach D2H Ctrl Completion failed\n",
+			__FUNCTION__));
+		goto fail;
+	}
+
+	/* Ring  3: D2H Transmit Complete */
+	if (dhd_prot_ring_attach(dhd, &prot->d2hring_tx_cpln, "d2htxcpl",
+	        (uint16)d2h_max_txcpl, D2HRING_TXCMPLT_ITEMSIZE,
+	        BCMPCIE_D2H_MSGRING_TX_COMPLETE) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_prot_ring_attach D2H Tx Completion failed\n",
+			__FUNCTION__));
+		goto fail;
+
+	}
+
+	/* Ring  4: D2H Receive Complete */
+	if (dhd_prot_ring_attach(dhd, &prot->d2hring_rx_cpln, "d2hrxcpl",
+	        (uint16)d2h_max_rxcpl, D2HRING_RXCMPLT_ITEMSIZE,
+	        BCMPCIE_D2H_MSGRING_RX_COMPLETE) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_prot_ring_attach D2H Rx Completion failed\n",
+			__FUNCTION__));
+		goto fail;
+
+	}
+
+	/*
+	 * Max number of flowrings is not yet known. msgbuf_ring_t with DMA-able
+	 * buffers for flowrings will be instantiated, in dhd_prot_init() .
+	 * See dhd_prot_flowrings_pool_attach()
+	 */
+	/* ioctl response buffer */
+	if (dhd_dma_buf_alloc(dhd, &prot->retbuf, IOCT_RETBUF_SIZE)) {
+		goto fail;
+	}
+
+	/* IOCTL request buffer */
+	if (dhd_dma_buf_alloc(dhd, &prot->ioctbuf, IOCT_RETBUF_SIZE)) {
+		goto fail;
+	}
+
+	/* Host TS request buffer one buffer for now */
+	if (dhd_dma_buf_alloc(dhd, &prot->hostts_req_buf, CTRLSUB_HOSTTS_MEESAGE_SIZE)) {
+		goto fail;
+	}
+	prot->hostts_req_buf_inuse = FALSE;
+
+	/* Scratch buffer for dma rx offset */
+#ifdef BCM_HOST_BUF
+	if (dhd_dma_buf_alloc(dhd, &prot->d2h_dma_scratch_buf,
+		ROUNDUP(DMA_D2H_SCRATCH_BUF_LEN, 16) + DMA_HOST_BUFFER_LEN))
+#else
+	if (dhd_dma_buf_alloc(dhd, &prot->d2h_dma_scratch_buf, DMA_D2H_SCRATCH_BUF_LEN))
+
+#endif /* BCM_HOST_BUF */
+	{
+
+		goto fail;
+	}
+
+#ifdef DHD_HMAPTEST
+	/* Allocate buffer for hmaptest  */
+	DHD_ERROR(("allocating memory for hmaptest \n"));
+	if (dhd_dma_buf_alloc(dhd, &prot->hmaptest.mem, HMAP_SANDBOX_BUFFER_LEN)) {
+
+		goto fail;
+	} else {
+		uint32 scratch_len;
+		uint64 scratch_lin, w1_start;
+		dmaaddr_t scratch_pa;
+
+		scratch_pa = prot->hmaptest.mem.pa;
+		scratch_len = prot->hmaptest.mem.len;
+		scratch_lin  = (uint64)(PHYSADDRLO(scratch_pa) & 0xffffffff)
+			| (((uint64)PHYSADDRHI(scratch_pa)& 0xffffffff) << 32);
+		w1_start  = scratch_lin +  scratch_len;
+		DHD_ERROR(("hmap: NOTE Buffer alloc for HMAPTEST Start=0x%0llx len=0x%08x"
+			"End=0x%0llx\n", (uint64) scratch_lin, scratch_len, (uint64) w1_start));
+	}
+#endif /* DHD_HMAPTEST */
+
+	/* scratch buffer bus throughput measurement */
+	if (dhd_dma_buf_alloc(dhd, &prot->host_bus_throughput_buf, DHD_BUS_TPUT_BUF_LEN)) {
+		goto fail;
+	}
+
+	return BCME_OK;
+
+fail:
+	return BCME_NOMEM;
+}
+
+/**
+ * The PCIE FD protocol layer is constructed in two phases:
+ *    Phase 1. dhd_prot_attach()
+ *    Phase 2. dhd_prot_init()
+ *
+ * dhd_prot_attach() - Allocates a dhd_prot_t object and resets all its fields.
+ * All Common rings are also attached (msgbuf_ring_t objects are allocated
+ * with DMA-able buffers).
+ * All dhd_dma_buf_t objects are also allocated here.
+ *
+ * As dhd_prot_attach is invoked prior to the pcie_shared object is read, any
+ * initialization of objects that requires information advertized by the dongle
+ * may not be performed here.
+ * E.g. the number of TxPost flowrings is not know at this point, neither do
+ * we know shich form of D2H DMA sync mechanism is advertized by the dongle, or
+ * whether the dongle supports DMA-ing of WR/RD indices for the H2D and/or D2H
+ * rings (common + flow).
+ *
+ * dhd_prot_init() is invoked after the bus layer has fetched the information
+ * advertized by the dongle in the pcie_shared_t.
+ */
+int
+dhd_prot_attach(dhd_pub_t *dhd)
+{
+	osl_t *osh = dhd->osh;
+	dhd_prot_t *prot;
+	uint32 trap_buf_len;
+#ifdef FLOW_RING_PREALLOC
+	int i = 0;
+#endif /* FLOW_RING_PREALLOC */
+	/* bus is required in chipid checks and flowring checks, abort, if it is NULL */
+	if (dhd->bus == NULL) {
+		DHD_ERROR(("%s: Abort as dhd->bus is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dhd->htput_support = dhd_check_htput_chip(dhd->bus);
+
+	/* Allocate prot structure */
+	if (!(prot = (dhd_prot_t *)DHD_OS_PREALLOC(dhd, DHD_PREALLOC_PROT,
+		sizeof(dhd_prot_t)))) {
+		DHD_ERROR(("%s: kmalloc failed\n", __FUNCTION__));
+		goto fail;
+	}
+	memset(prot, 0, sizeof(*prot));
+
+	prot->osh = osh;
+	dhd->prot = prot;
+
+	if (dhd->htput_support) {
+		ring_size_alloc_version = 2;
+	} else {
+		ring_size_alloc_version = 1;
+	}
+
+	dhd_prot_set_ring_size_ver(dhd, ring_size_alloc_version);
+
+	/* DMAing ring completes supported? FALSE by default  */
+	dhd->dma_d2h_ring_upd_support = FALSE;
+	dhd->dma_h2d_ring_upd_support = FALSE;
+	dhd->dma_ring_upd_overwrite = FALSE;
+
+	dhd->idma_inited = 0;
+	dhd->ifrm_inited = 0;
+	dhd->dar_inited = 0;
+
+	if (dhd_prot_allocate_bufs(dhd, prot) != BCME_OK) {
+		goto fail;
+	}
+
+#ifdef DHD_RX_CHAINING
+	dhd_rxchain_reset(&prot->rxchain);
+#endif
+
+	prot->pktid_ctrl_map = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_PKTID_CTRL);
+	if (prot->pktid_ctrl_map == NULL) {
+		goto fail;
+	}
+
+	prot->pktid_rx_map = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_PKTID_RX);
+	if (prot->pktid_rx_map == NULL)
+		goto fail;
+
+	prot->pktid_tx_map = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_PKTID_TX);
+	if (prot->pktid_rx_map == NULL)
+		goto fail;
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+	prot->pktid_map_handle_ioctl = DHD_NATIVE_TO_PKTID_INIT(dhd,
+		DHD_FLOWRING_MAX_IOCTLRESPBUF_POST);
+	if (prot->pktid_map_handle_ioctl == NULL) {
+		goto fail;
+	}
+#endif /* IOCTLRESP_USE_CONSTMEM */
+
+#ifdef DHD_MAP_PKTID_LOGGING
+	prot->pktid_dma_map = DHD_PKTID_LOG_INIT(dhd, MAX_PKTID_LOG);
+	if (prot->pktid_dma_map == NULL) {
+		DHD_ERROR(("%s: failed to allocate pktid_dma_map\n",
+			__FUNCTION__));
+	}
+
+	prot->pktid_dma_unmap = DHD_PKTID_LOG_INIT(dhd, MAX_PKTID_LOG);
+	if (prot->pktid_dma_unmap == NULL) {
+		DHD_ERROR(("%s: failed to allocate pktid_dma_unmap\n",
+			__FUNCTION__));
+	}
+#endif /* DHD_MAP_PKTID_LOGGING */
+
+#ifdef D2H_MINIDUMP
+	if (dhd->bus->sih->buscorerev < 71) {
+		trap_buf_len = BCMPCIE_HOST_EXT_TRAP_DBGBUF_LEN_MIN;
+	} else {
+		/* buscorerev >= 71, supports minimdump of len 96KB */
+		trap_buf_len = BCMPCIE_HOST_EXT_TRAP_DBGBUF_LEN;
+	}
+#else
+	/* FW going to DMA extended trap data,
+	 * allocate buffer for the maximum extended trap data.
+	 */
+	trap_buf_len = BCMPCIE_EXT_TRAP_DATA_MAXLEN;
+#endif /* D2H_MINIDUMP */
+
+#ifdef FLOW_RING_PREALLOC
+	if (dhd->htput_support) {
+		/* pre-allocation htput ring */
+		for (i = 0; i < HTPUT_TOTAL_FLOW_RINGS; i++) {
+			if (dhd_dma_buf_alloc(dhd, &dhd->prot->htput_ring_buf[i],
+				(uint32)(h2d_htput_max_txpost * H2DRING_TXPOST_ITEMSIZE)))
+			{
+				DHD_ERROR(("%s : dhd_prealloc_htput_ring_buffer failed\n",
+					__FUNCTION__));
+				goto fail;
+			}
+		}
+	}
+
+	if (dhd->htput_support) {
+		dhd->non_htput_total_flow_rings = MAX_FLOW_RINGS - HTPUT_TOTAL_FLOW_RINGS;
+	} else {
+		dhd->non_htput_total_flow_rings = MAX_FLOW_RINGS;
+	}
+
+	/* pre-allocation flow ring */
+	for (i = 0; i < dhd->non_htput_total_flow_rings; i++) {
+		if (dhd_dma_buf_alloc(dhd, &dhd->prot->flow_ring_buf[i],
+			(uint32)(H2DRING_TXPOST_MAX_ITEM * H2DRING_TXPOST_ITEMSIZE))) {
+			DHD_ERROR(("%s : dhd_prealloc_flowring_buffer failed\n", __FUNCTION__));
+			goto fail;
+		}
+	}
+#endif /* FLOW_RING_PREALLOC */
+
+	/* Initialize trap buffer */
+	if (dhd_dma_buf_alloc(dhd, &dhd->prot->fw_trap_buf, trap_buf_len)) {
+		DHD_ERROR(("%s: dhd_init_trap_buffer falied\n", __FUNCTION__));
+		goto fail;
+	}
+
+	return BCME_OK;
+
+fail:
+
+	if (prot) {
+		/* Free up all allocated memories */
+		dhd_prot_detach(dhd);
+	}
+
+	return BCME_NOMEM;
+} /* dhd_prot_attach */
+
+static int
+dhd_alloc_host_scbs(dhd_pub_t *dhd)
+{
+	int ret = BCME_OK;
+	sh_addr_t base_addr;
+	dhd_prot_t *prot = dhd->prot;
+	uint32 host_scb_size = 0;
+
+	if (dhd->hscb_enable) {
+		/* read number of bytes to allocate from F/W */
+		dhd_bus_cmn_readshared(dhd->bus, &host_scb_size, HOST_SCB_ADDR, 0);
+		if (host_scb_size) {
+			/* In fw reload scenario the buffer could have been allocated for previous
+			 * run. Check the existing buffer if there is one that can accommodate
+			 * the new firmware requirement and reuse the buffer is possible.
+			 */
+			if (prot->host_scb_buf.va) {
+				if (prot->host_scb_buf.len >= host_scb_size) {
+					prot->host_scb_buf.len = host_scb_size;
+				} else {
+					dhd_dma_buf_free(dhd, &prot->host_scb_buf);
+				}
+			}
+			/* alloc array of host scbs */
+			if (prot->host_scb_buf.va == NULL) {
+				ret = dhd_dma_buf_alloc(dhd, &prot->host_scb_buf, host_scb_size);
+			}
+			/* write host scb address to F/W */
+			if (ret == BCME_OK) {
+				dhd_base_addr_htolpa(&base_addr, prot->host_scb_buf.pa);
+				dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+					HOST_SCB_ADDR, 0);
+			}
+		}
+	} else {
+		DHD_TRACE(("%s: Host scb not supported in F/W. \n", __FUNCTION__));
+	}
+
+	if (ret != BCME_OK) {
+		DHD_ERROR(("%s dhd_alloc_host_scbs, alloc failed: Err Code %d\n",
+			__FUNCTION__, ret));
+	}
+	return ret;
+}
+
+#ifdef DHD_PCIE_PTM
+/*
+ * Currently used to set PCIE PTM capability only.
+ */
+void
+dhd_set_host_cap2(dhd_pub_t *dhd)
+{
+	uint32 data = 0;
+
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_7) {
+		/* Advertise PTM capability */
+		data |= HOSTCAP2_PCIE_PTM;
+		dhd_bus_cmn_writeshared(dhd->bus, &data, sizeof(uint32), HOST_CAP2, 0);
+		DHD_INFO(("Set host_cap2 0x%x\n", data));
+	}
+}
+#endif /* DHD_PCIE_PTM */
+
+void
+dhd_set_host_cap(dhd_pub_t *dhd)
+{
+	uint32 data = 0;
+	dhd_prot_t *prot = dhd->prot;
+#ifdef D2H_MINIDUMP
+	uint16 host_trap_addr_len;
+#endif /* D2H_MINIDUMP */
+
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6) {
+		if (dhd->h2d_phase_supported) {
+			data |= HOSTCAP_H2D_VALID_PHASE;
+			if (dhd->force_dongletrap_on_bad_h2d_phase)
+				data |= HOSTCAP_H2D_ENABLE_TRAP_ON_BADPHASE;
+		}
+		if (prot->host_ipc_version > prot->device_ipc_version)
+			prot->active_ipc_version = prot->device_ipc_version;
+		else
+			prot->active_ipc_version = prot->host_ipc_version;
+
+		data |= prot->active_ipc_version;
+
+		if (dhdpcie_bus_get_pcie_hostready_supported(dhd->bus)) {
+			DHD_INFO(("Advertise Hostready Capability\n"));
+			data |= HOSTCAP_H2D_ENABLE_HOSTRDY;
+		}
+#ifdef PCIE_INB_DW
+		if (dhdpcie_bus_get_pcie_inband_dw_supported(dhd->bus)) {
+			DHD_INFO(("Advertise Inband-DW Capability\n"));
+			data |= HOSTCAP_DS_INBAND_DW;
+			data |= HOSTCAP_DS_NO_OOB_DW;
+			dhdpcie_bus_enab_pcie_dw(dhd->bus, DEVICE_WAKE_INB);
+			if (!dhd->dma_h2d_ring_upd_support || !dhd->dma_d2h_ring_upd_support) {
+				dhd_init_dongle_ds_lock(dhd->bus);
+				dhdpcie_set_dongle_deepsleep(dhd->bus, FALSE);
+			}
+		} else
+#endif /* PCIE_INB_DW */
+#ifdef PCIE_OOB
+		if (dhdpcie_bus_get_pcie_oob_dw_supported(dhd->bus)) {
+			dhdpcie_bus_enab_pcie_dw(dhd->bus, DEVICE_WAKE_OOB);
+		} else
+#endif /* PCIE_OOB */
+		{
+			/* Disable DS altogether */
+			data |= HOSTCAP_DS_NO_OOB_DW;
+			dhdpcie_bus_enab_pcie_dw(dhd->bus, DEVICE_WAKE_NONE);
+		}
+
+		/* Indicate support for extended trap data */
+		data |= HOSTCAP_EXTENDED_TRAP_DATA;
+
+		/* Indicate support for TX status metadata */
+		if (dhd->pcie_txs_metadata_enable != 0)
+			data |= HOSTCAP_TXSTATUS_METADATA;
+
+		/* Enable fast delete ring in firmware if supported */
+		if (dhd->fast_delete_ring_support) {
+			data |= HOSTCAP_FAST_DELETE_RING;
+		}
+
+		if (dhdpcie_bus_get_pcie_idma_supported(dhd->bus)) {
+			DHD_ERROR_MEM(("IDMA inited\n"));
+			data |= HOSTCAP_H2D_IDMA;
+			dhd->idma_inited = TRUE;
+		} else {
+			DHD_ERROR_MEM(("IDMA not enabled in FW !!\n"));
+			dhd->idma_inited = FALSE;
+		}
+
+		if (dhdpcie_bus_get_pcie_ifrm_supported(dhd->bus)) {
+			DHD_ERROR_MEM(("IFRM Inited\n"));
+			data |= HOSTCAP_H2D_IFRM;
+			dhd->ifrm_inited = TRUE;
+			dhd->dma_h2d_ring_upd_support = FALSE;
+			dhd_prot_dma_indx_free(dhd);
+		} else {
+			DHD_ERROR_MEM(("IFRM not enabled in FW !!\n"));
+			dhd->ifrm_inited = FALSE;
+		}
+
+		if (dhdpcie_bus_get_pcie_dar_supported(dhd->bus)) {
+			DHD_ERROR(("DAR doorbell Use\n"));
+			data |= HOSTCAP_H2D_DAR;
+			dhd->dar_inited = TRUE;
+		} else {
+			DHD_ERROR_MEM(("DAR not enabled in FW !!\n"));
+			dhd->dar_inited = FALSE;
+		}
+
+		/* FW Checks for HOSTCAP_UR_FW_NO_TRAP and Does not TRAP if set
+		 * Radar 36403220 JIRA SWWLAN-182145
+		 */
+		data |= HOSTCAP_UR_FW_NO_TRAP;
+
+		if (dhd->hscb_enable) {
+			data |= HOSTCAP_HSCB;
+		}
+
+#ifdef EWP_EDL
+		if (dhd->dongle_edl_support) {
+			data |= HOSTCAP_EDL_RING;
+			DHD_ERROR(("Enable EDL host cap\n"));
+		} else {
+			DHD_ERROR(("DO NOT SET EDL host cap\n"));
+		}
+#endif /* EWP_EDL */
+
+#ifdef D2H_MINIDUMP
+		if (dhd_bus_is_minidump_enabled(dhd)) {
+			data |= HOSTCAP_EXT_TRAP_DBGBUF;
+			DHD_ERROR(("ALLOW D2H MINIDUMP!!\n"));
+		}
+#endif /* D2H_MINIDUMP */
+
+		if (dhd->extdtxs_in_txcpl) {
+			DHD_ERROR_MEM(("Enable hostcap: EXTD TXS in txcpl\n"));
+			data |= HOSTCAP_PKT_TXSTATUS;
+		}
+		else {
+			DHD_ERROR_MEM(("Enable hostcap: EXTD TXS in txcpl\n"));
+		}
+
+		DHD_INFO(("%s:Active Ver:%d, Host Ver:%d, FW Ver:%d\n",
+			__FUNCTION__,
+			prot->active_ipc_version, prot->host_ipc_version,
+			prot->device_ipc_version));
+
+		dhd_bus_cmn_writeshared(dhd->bus, &data, sizeof(uint32), HOST_API_VERSION, 0);
+		dhd_bus_cmn_writeshared(dhd->bus, &prot->fw_trap_buf.pa,
+			sizeof(prot->fw_trap_buf.pa), DNGL_TO_HOST_TRAP_ADDR, 0);
+#ifdef D2H_MINIDUMP
+		if (dhd_bus_is_minidump_enabled(dhd)) {
+			/* Dongle expects the host_trap_addr_len in terms of words */
+			host_trap_addr_len = prot->fw_trap_buf.len / 4;
+			dhd_bus_cmn_writeshared(dhd->bus, &host_trap_addr_len,
+				sizeof(host_trap_addr_len), DNGL_TO_HOST_TRAP_ADDR_LEN, 0);
+		}
+#endif /* D2H_MINIDUMP */
+	}
+
+}
+
+#ifdef AGG_H2D_DB
+void dhd_agg_inflight_stats_dump(dhd_pub_t *dhd, struct bcmstrbuf *strbuf)
+{
+	uint64 *inflight_histo = dhd->prot->agg_h2d_db_info.inflight_histo;
+	uint32 i;
+	uint64 total_inflight_histo = 0;
+
+	if (inflight_histo == NULL) {
+		return;
+	}
+
+	bcm_bprintf(strbuf, "inflight: \t count\n");
+	for (i = 0; i < DHD_NUM_INFLIGHT_HISTO_ROWS; i++) {
+		bcm_bprintf(strbuf, "%16u: \t %llu\n", 1U<<i, inflight_histo[i]);
+		total_inflight_histo += inflight_histo[i];
+	}
+	bcm_bprintf(strbuf, "total_inflight_histo: %llu\n", total_inflight_histo);
+}
+
+void dhd_agg_inflights_stats_update(dhd_pub_t *dhd, uint32 inflight)
+{
+	uint64 *bin = dhd->prot->agg_h2d_db_info.inflight_histo;
+	uint64 *p;
+	uint32 bin_power;
+	bin_power = next_larger_power2(inflight);
+
+	switch (bin_power) {
+		case   1: p = bin + 0; break;
+		case   2: p = bin + 1; break;
+		case   4: p = bin + 2; break;
+		case   8: p = bin + 3; break;
+		case  16: p = bin + 4; break;
+		case  32: p = bin + 5; break;
+		case  64: p = bin + 6; break;
+		case 128: p = bin + 7; break;
+		case 256: p = bin + 8; break;
+		case 512: p = bin + 9; break;
+		case 1024: p = bin + 10; break;
+		case 2048: p = bin + 11; break;
+		case 4096: p = bin + 12; break;
+		case 8192: p = bin + 13; break;
+		default : p = bin + 13; break;
+	}
+	ASSERT((p - bin) < DHD_NUM_INFLIGHT_HISTO_ROWS);
+	*p = *p + 1;
+	return;
+}
+
+/*
+ * dhd_msgbuf_agg_h2d_db_timer_fn:
+ * Timer callback function for ringing h2d DB.
+ * This is run in isr context (HRTIMER_MODE_REL),
+ * do not hold any spin_lock_bh().
+ * Using HRTIMER_MODE_REL_SOFT causing TPUT regressions.
+ */
+enum hrtimer_restart
+dhd_msgbuf_agg_h2d_db_timer_fn(struct hrtimer *timer)
+{
+	agg_h2d_db_info_t *agg_db_info;
+	dhd_pub_t *dhd;
+	dhd_prot_t *prot;
+	uint32 db_index;
+	uint corerev;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	agg_db_info = container_of(timer, agg_h2d_db_info_t, timer);
+	GCC_DIAGNOSTIC_POP();
+
+	dhd = agg_db_info->dhd;
+	prot = dhd->prot;
+
+	prot->agg_h2d_db_info.timer_db_cnt++;
+	if (IDMA_ACTIVE(dhd)) {
+		db_index = IDMA_IDX0;
+		if (dhd->bus->sih) {
+			corerev = dhd->bus->sih->buscorerev;
+			if (corerev >= 24) {
+				db_index |= (DMA_TYPE_IDMA << DMA_TYPE_SHIFT);
+			}
+		}
+		prot->mb_2_ring_fn(dhd->bus, db_index, TRUE);
+	} else {
+		prot->mb_ring_fn(dhd->bus, DHD_AGGR_H2D_DB_MAGIC);
+	}
+
+	return HRTIMER_NORESTART;
+}
+
+void
+dhd_msgbuf_agg_h2d_db_timer_start(dhd_prot_t *prot)
+{
+	agg_h2d_db_info_t *agg_db_info = &prot->agg_h2d_db_info;
+
+	/* Queue the timer only when it is not in the queue */
+	if (!hrtimer_active(&agg_db_info->timer)) {
+		hrtimer_start(&agg_db_info->timer, ns_to_ktime(agg_h2d_db_timeout * NSEC_PER_USEC),
+				HRTIMER_MODE_REL);
+	}
+}
+
+static void
+dhd_msgbuf_agg_h2d_db_timer_init(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	agg_h2d_db_info_t *agg_db_info = &prot->agg_h2d_db_info;
+
+	agg_db_info->dhd = dhd;
+	hrtimer_init(&agg_db_info->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	/* The timer function will run from ISR context, ensure no spin_lock_bh are used */
+	agg_db_info->timer.function = &dhd_msgbuf_agg_h2d_db_timer_fn;
+	agg_db_info->init = TRUE;
+	agg_db_info->timer_db_cnt = 0;
+	agg_db_info->direct_db_cnt = 0;
+	agg_db_info->inflight_histo = (uint64 *)MALLOCZ(dhd->osh, DHD_INFLIGHT_HISTO_SIZE);
+}
+
+static void
+dhd_msgbuf_agg_h2d_db_timer_reset(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	agg_h2d_db_info_t *agg_db_info = &prot->agg_h2d_db_info;
+	if (agg_db_info->init) {
+		if (agg_db_info->inflight_histo) {
+			MFREE(dhd->osh, agg_db_info->inflight_histo, DHD_INFLIGHT_HISTO_SIZE);
+		}
+		hrtimer_try_to_cancel(&agg_db_info->timer);
+		agg_db_info->init = FALSE;
+	}
+}
+
+static void
+dhd_msgbuf_agg_h2d_db_timer_cancel(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	agg_h2d_db_info_t *agg_db_info = &prot->agg_h2d_db_info;
+	hrtimer_try_to_cancel(&agg_db_info->timer);
+}
+#endif /* AGG_H2D_DB */
+
+void
+dhd_prot_clearcounts(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+#ifdef AGG_H2D_DB
+	agg_h2d_db_info_t *agg_db_info = &prot->agg_h2d_db_info;
+	if (agg_db_info->inflight_histo) {
+		memset(agg_db_info->inflight_histo, 0, DHD_INFLIGHT_HISTO_SIZE);
+	}
+	agg_db_info->direct_db_cnt = 0;
+	agg_db_info->timer_db_cnt = 0;
+#endif /* AGG_H2D_DB */
+	prot->txcpl_db_cnt = 0;
+	prot->tx_h2d_db_cnt = 0;
+}
+
+void
+dhd_prot_update_rings_size(dhd_prot_t *prot)
+{
+	uint16 max_items;
+
+	/* Update ring max_items to MIN of allocated vs iovar overridden */
+	max_items = MIN(d2h_max_txcpl, prot->d2hring_tx_cpln.max_items);
+	d2h_max_txcpl = prot->d2hring_tx_cpln.max_items = max_items;
+
+	max_items = MIN(h2d_max_rxpost, prot->h2dring_rxp_subn.max_items);
+	h2d_max_rxpost = prot->h2dring_rxp_subn.max_items = max_items;
+
+	max_items = MIN(d2h_max_rxcpl, prot->d2hring_rx_cpln.max_items);
+	d2h_max_rxcpl = prot->d2hring_rx_cpln.max_items = max_items;
+
+	max_items = MIN(h2d_max_ctrlpost, prot->h2dring_ctrl_subn.max_items);
+	h2d_max_ctrlpost = prot->h2dring_ctrl_subn.max_items = max_items;
+
+	max_items = MIN(d2h_max_ctrlcpl, prot->d2hring_ctrl_cpln.max_items);
+	d2h_max_ctrlcpl = prot->d2hring_ctrl_cpln.max_items = max_items;
+
+}
+
+/**
+ * dhd_prot_init - second stage of dhd_prot_attach. Now that the dongle has
+ * completed it's initialization of the pcie_shared structure, we may now fetch
+ * the dongle advertized features and adjust the protocol layer accordingly.
+ *
+ * dhd_prot_init() may be invoked again after a dhd_prot_reset().
+ */
+int
+dhd_prot_init(dhd_pub_t *dhd)
+{
+	sh_addr_t base_addr;
+	dhd_prot_t *prot = dhd->prot;
+	int ret = 0;
+	uint32 idmacontrol;
+	uint32 waitcount = 0;
+	uint16 max_eventbufpost = 0;
+
+	/**
+	 * A user defined value can be assigned to global variable h2d_max_txpost via
+	 * 1. DHD IOVAR h2d_max_txpost, before firmware download
+	 * 2. module parameter h2d_max_txpost
+	 * prot->h2d_max_txpost is assigned with DHD_H2DRING_TXPOST_MAX_ITEM,
+	 * if user has not defined any buffers by one of the above methods.
+	 */
+	prot->h2d_max_txpost = (uint16)h2d_max_txpost;
+	DHD_ERROR(("%s:%d: h2d_max_txpost = %d\n", __FUNCTION__, __LINE__,
+		prot->h2d_max_txpost));
+
+	if (dhd->htput_support) {
+		prot->h2d_htput_max_txpost = (uint16)h2d_htput_max_txpost;
+		DHD_ERROR(("%s:%d: h2d_htput_max_txpost = %d\n",
+			__FUNCTION__, __LINE__, prot->h2d_htput_max_txpost));
+	}
+
+	/* Read max rx packets supported by dongle */
+	dhd_bus_cmn_readshared(dhd->bus, &prot->max_rxbufpost, MAX_HOST_RXBUFS, 0);
+	if (prot->max_rxbufpost == 0) {
+		/* This would happen if the dongle firmware is not */
+		/* using the latest shared structure template */
+		prot->max_rxbufpost = LEGACY_MAX_RXBUFPOST;
+	}
+	prot->rx_buf_burst = (uint16)rx_buf_burst;
+	prot->rx_bufpost_threshold = (uint16)rx_bufpost_threshold;
+	DHD_ERROR(("%s: max_rxbufpost:%d rx_buf_burst:%d rx_bufpost_threshold:%d\n",
+		__FUNCTION__, prot->max_rxbufpost, prot->rx_buf_burst, prot->rx_bufpost_threshold));
+
+	/* Update static rings sizes if overridden by iovar */
+	dhd_prot_update_rings_size(prot);
+
+	/* Initialize.  bzero() would blow away the dma pointers. */
+	max_eventbufpost = (uint16)dhdpcie_get_max_eventbufpost(dhd->bus);
+	prot->max_eventbufpost = (((max_eventbufpost + DHD_FLOWRING_MAX_IOCTLRESPBUF_POST)) >=
+		H2DRING_CTRL_SUB_MAX_ITEM) ? DHD_FLOWRING_MAX_EVENTBUF_POST : max_eventbufpost;
+	prot->max_ioctlrespbufpost = DHD_FLOWRING_MAX_IOCTLRESPBUF_POST;
+	prot->max_infobufpost = DHD_H2D_INFORING_MAX_BUF_POST;
+	prot->max_tsbufpost = DHD_MAX_TSBUF_POST;
+
+	prot->cur_ioctlresp_bufs_posted = 0;
+	OSL_ATOMIC_INIT(dhd->osh, &prot->active_tx_count);
+	prot->data_seq_no = 0;
+	prot->ioctl_seq_no = 0;
+	prot->rxbufpost = 0;
+	prot->tot_rxbufpost = 0;
+	prot->tot_rxcpl = 0;
+	prot->cur_event_bufs_posted = 0;
+	prot->ioctl_state = 0;
+	prot->curr_ioctl_cmd = 0;
+	prot->cur_ts_bufs_posted = 0;
+	prot->infobufpost = 0;
+
+	prot->dmaxfer.srcmem.va = NULL;
+	prot->dmaxfer.dstmem.va = NULL;
+	prot->dmaxfer.in_progress = FALSE;
+
+#ifdef DHD_HMAPTEST
+	prot->hmaptest.in_progress = FALSE;
+#endif /* DHD_HMAPTEST */
+	prot->metadata_dbg = FALSE;
+	prot->rx_metadata_offset = 0;
+	prot->tx_metadata_offset = 0;
+	prot->txp_threshold = TXP_FLUSH_MAX_ITEMS_FLUSH_CNT;
+
+	/* To catch any rollover issues fast, starting with higher ioctl_trans_id */
+	prot->ioctl_trans_id = MAXBITVAL(NBITS(prot->ioctl_trans_id)) - BUFFER_BEFORE_ROLLOVER;
+	prot->ioctl_state = 0;
+	prot->ioctl_status = 0;
+	prot->ioctl_resplen = 0;
+	prot->ioctl_received = IOCTL_WAIT;
+	prot->rx_wakeup_pkt = 0;
+	prot->event_wakeup_pkt = 0;
+	prot->info_wakeup_pkt = 0;
+
+	/* Initialize Common MsgBuf Rings */
+
+	prot->device_ipc_version = dhd->bus->api.fw_rev;
+	prot->host_ipc_version = PCIE_SHARED_VERSION;
+	prot->no_tx_resource = FALSE;
+
+	prot->rx_cpl_post_bound =
+		(dhd_rx_cpl_post_bound) ? dhd_rx_cpl_post_bound : DHD_RX_CPL_POST_BOUND;
+	prot->tx_post_bound =
+		(dhd_tx_post_bound) ? dhd_tx_post_bound : DHD_TX_POST_BOUND;
+	prot->tx_cpl_bound =
+		(dhd_tx_cpl_bound) ? dhd_tx_cpl_bound : DHD_TX_CPL_BOUND;
+	prot->ctrl_cpl_post_bound =
+		(dhd_ctrl_cpl_post_bound) ? dhd_ctrl_cpl_post_bound : DHD_CTRL_CPL_POST_BOUND;
+
+	/* Init the host API version */
+	dhd_set_host_cap(dhd);
+
+#ifdef DHD_PCIE_PTM
+	/* Set host capability 2 flags; for PCIE PTM */
+	dhd_set_host_cap2(dhd);
+#endif /* DHD_PCIE_PTM */
+
+	/* alloc and configure scb host address for dongle */
+	if ((ret = dhd_alloc_host_scbs(dhd))) {
+		return ret;
+	}
+
+	/* Register the interrupt function upfront */
+	/* remove corerev checks in data path */
+	/* do this after host/fw negotiation for DAR */
+	prot->mb_ring_fn = dhd_bus_get_mbintr_fn(dhd->bus);
+	prot->mb_2_ring_fn = dhd_bus_get_mbintr_2_fn(dhd->bus);
+
+	prot->tx_h2d_db_cnt = 0;
+#ifdef AGG_H2D_DB
+	dhd_msgbuf_agg_h2d_db_timer_init(dhd);
+#endif /* AGG_H2D_DB */
+
+	dhd->bus->_dar_war = (dhd->bus->sih->buscorerev < 64) ? TRUE : FALSE;
+
+	/* If supported by the host, indicate the memory block
+	 * for completion writes / submission reads to shared space
+	 */
+	if (dhd->dma_d2h_ring_upd_support) {
+		dhd_base_addr_htolpa(&base_addr, prot->d2h_dma_indx_wr_buf.pa);
+		dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+			D2H_DMA_INDX_WR_BUF, 0);
+		dhd_base_addr_htolpa(&base_addr, prot->h2d_dma_indx_rd_buf.pa);
+		dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+			H2D_DMA_INDX_RD_BUF, 0);
+	}
+
+	if (dhd->dma_h2d_ring_upd_support || IDMA_ENAB(dhd)) {
+		dhd_base_addr_htolpa(&base_addr, prot->h2d_dma_indx_wr_buf.pa);
+		dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+			H2D_DMA_INDX_WR_BUF, 0);
+		dhd_base_addr_htolpa(&base_addr, prot->d2h_dma_indx_rd_buf.pa);
+		dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+			D2H_DMA_INDX_RD_BUF, 0);
+	}
+
+	dhd_prot_ring_init(dhd, &prot->h2dring_ctrl_subn);
+	dhd_prot_ring_init(dhd, &prot->h2dring_rxp_subn);
+	dhd_prot_ring_init(dhd, &prot->d2hring_ctrl_cpln);
+
+	/* Make it compatibile with pre-rev7 Firmware */
+	if (prot->active_ipc_version < PCIE_SHARED_VERSION_7) {
+		prot->d2hring_tx_cpln.item_len =
+			D2HRING_TXCMPLT_ITEMSIZE_PREREV7;
+		prot->d2hring_rx_cpln.item_len =
+			D2HRING_RXCMPLT_ITEMSIZE_PREREV7;
+	}
+	dhd_prot_ring_init(dhd, &prot->d2hring_tx_cpln);
+	dhd_prot_ring_init(dhd, &prot->d2hring_rx_cpln);
+
+	dhd_prot_d2h_sync_init(dhd);
+
+	dhd_prot_h2d_sync_init(dhd);
+
+#ifdef PCIE_INB_DW
+	/* Set the initial DS state */
+	if (INBAND_DW_ENAB(dhd->bus)) {
+		dhdpcie_bus_set_pcie_inband_dw_state(dhd->bus,
+			DW_DEVICE_DS_ACTIVE);
+	}
+#endif /* PCIE_INB_DW */
+
+	/* init the scratch buffer */
+	dhd_base_addr_htolpa(&base_addr, prot->d2h_dma_scratch_buf.pa);
+	dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+		D2H_DMA_SCRATCH_BUF, 0);
+	dhd_bus_cmn_writeshared(dhd->bus, &prot->d2h_dma_scratch_buf.len,
+		sizeof(prot->d2h_dma_scratch_buf.len), D2H_DMA_SCRATCH_BUF_LEN, 0);
+#ifdef DHD_DMA_INDICES_SEQNUM
+	prot->host_seqnum = D2H_EPOCH_INIT_VAL % D2H_EPOCH_MODULO;
+#endif /* DHD_DMA_INDICES_SEQNUM */
+	/* Signal to the dongle that common ring init is complete */
+	if (dhd->hostrdy_after_init)
+		dhd_bus_hostready(dhd->bus);
+
+	/*
+	 * If the DMA-able buffers for flowring needs to come from a specific
+	 * contiguous memory region, then setup prot->flowrings_dma_buf here.
+	 * dhd_prot_flowrings_pool_attach() will carve out DMA-able buffers from
+	 * this contiguous memory region, for each of the flowrings.
+	 */
+
+	/* Pre-allocate pool of msgbuf_ring for flowrings */
+	if ((ret = dhd_prot_flowrings_pool_attach(dhd))) {
+		return ret;
+	}
+
+	dhd->ring_attached = TRUE;
+
+	/* If IFRM is enabled, wait for FW to setup the DMA channel */
+	if (IFRM_ENAB(dhd)) {
+		dhd_base_addr_htolpa(&base_addr, prot->h2d_ifrm_indx_wr_buf.pa);
+		dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
+			H2D_IFRM_INDX_WR_BUF, 0);
+	}
+
+	/* If IDMA is enabled and initied, wait for FW to setup the IDMA descriptors
+	 * Waiting just before configuring doorbell
+	 */
+#ifdef BCMQT
+#define	IDMA_ENABLE_WAIT  100
+#else
+#define	IDMA_ENABLE_WAIT  10
+#endif
+	if (IDMA_ACTIVE(dhd)) {
+		/* wait for idma_en bit in IDMAcontrol register to be set */
+		/* Loop till idma_en is not set */
+		uint buscorerev = dhd->bus->sih->buscorerev;
+		idmacontrol = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			IDMAControl(buscorerev), 0, 0);
+		while (!(idmacontrol & PCIE_IDMA_MODE_EN(buscorerev)) &&
+			(waitcount++ < IDMA_ENABLE_WAIT)) {
+
+			DHD_ERROR(("iDMA not enabled yet,waiting 1 ms "
+				"c=%d IDMAControl = %08x\n",
+				waitcount, idmacontrol));
+			OSL_DELAY(1000); /* 1ms as its onetime only */
+			idmacontrol = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				IDMAControl(buscorerev), 0, 0);
+		}
+
+		if (waitcount < IDMA_ENABLE_WAIT) {
+			DHD_ERROR(("iDMA enabled PCIEControl = %08x\n", idmacontrol));
+		} else {
+			DHD_ERROR(("Error: wait for iDMA timed out wait=%d IDMAControl = %08x\n",
+				waitcount, idmacontrol));
+			return BCME_ERROR;
+		}
+		// add delay to fix bring up issue
+		OSL_SLEEP(1);
+	}
+
+	/* Host should configure soft doorbells if needed ... here */
+
+	/* Post to dongle host configured soft doorbells */
+	dhd_msgbuf_ring_config_d2h_soft_doorbell(dhd);
+
+	dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd);
+	dhd_msgbuf_rxbuf_post_event_bufs(dhd);
+
+	prot->no_retry = FALSE;
+	prot->no_aggr = FALSE;
+	prot->fixed_rate = FALSE;
+
+	/*
+	 * Note that any communication with the Dongle should be added
+	 * below this point. Any other host data structure initialiation that
+	 * needs to be done prior to the DPC starts executing should be done
+	 * befor this point.
+	 * Because once we start sending H2D requests to Dongle, the Dongle
+	 * respond immediately. So the DPC context to handle this
+	 * D2H response could preempt the context in which dhd_prot_init is running.
+	 * We want to ensure that all the Host part of dhd_prot_init is
+	 * done before that.
+	 */
+
+	/* See if info rings could be created, info rings should be created
+	* only if dongle does not support EDL
+	*/
+#ifdef EWP_EDL
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6 && !dhd->dongle_edl_support)
+#else
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6)
+#endif /* EWP_EDL */
+	{
+		if ((ret = dhd_prot_init_info_rings(dhd)) != BCME_OK) {
+			/* For now log and proceed, further clean up action maybe necessary
+			 * when we have more clarity.
+			 */
+			DHD_ERROR(("%s Info rings couldn't be created: Err Code%d",
+				__FUNCTION__, ret));
+		}
+	}
+
+#ifdef EWP_EDL
+		/* Create Enhanced Debug Lane rings (EDL) if dongle supports it */
+		if (dhd->dongle_edl_support) {
+			if ((ret = dhd_prot_init_edl_rings(dhd)) != BCME_OK) {
+				DHD_ERROR(("%s EDL rings couldn't be created: Err Code%d",
+					__FUNCTION__, ret));
+			}
+		}
+#endif /* EWP_EDL */
+
+	/* create MD cpl rings */
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_7 && dhd->mdring_capable) {
+		if ((ret = dhd_prot_init_md_rings(dhd)) != BCME_OK) {
+			/* For now log and proceed, further clean up action maybe necessary
+			 * when we have more clarity.
+			 */
+			DHD_ERROR(("%s MD rings couldn't be created: Err Code%d",
+				__FUNCTION__, ret));
+		}
+	}
+
+#ifdef DHD_LB_RXP
+	/* defualt rx flow ctrl thresholds. Can be changed at run time through sysfs */
+	dhd->lb_rxp_stop_thr = (D2HRING_RXCMPLT_MAX_ITEM * LB_RXP_STOP_THR);
+	dhd->lb_rxp_strt_thr = (D2HRING_RXCMPLT_MAX_ITEM * LB_RXP_STRT_THR);
+	atomic_set(&dhd->lb_rxp_flow_ctrl, FALSE);
+#endif /* DHD_LB_RXP */
+	return BCME_OK;
+} /* dhd_prot_init */
+
+/**
+ * dhd_prot_detach - PCIE FD protocol layer destructor.
+ * Unlink, frees allocated protocol memory (including dhd_prot)
+ */
+void dhd_prot_detach(dhd_pub_t *dhd)
+{
+#ifdef FLOW_RING_PREALLOC
+	int i = 0;
+#endif /* FLOW_RING_PREALLOC */
+	dhd_prot_t *prot = dhd->prot;
+
+	/* Stop the protocol module */
+	if (prot) {
+		/* For non-android platforms, devreset will not be called,
+		 * so call prot_reset here. It is harmless if called twice.
+		 */
+		dhd_prot_reset(dhd);
+
+		/* free up all DMA-able buffers allocated during prot attach/init */
+
+		dhd_dma_buf_free(dhd, &prot->d2h_dma_scratch_buf);
+#ifdef DHD_HMAPTEST
+		dhd_dma_buf_free(dhd, &prot->hmaptest.mem);
+#endif /* DHD_HMAPTEST */
+		dhd_dma_buf_free(dhd, &prot->retbuf);
+		dhd_dma_buf_free(dhd, &prot->ioctbuf);
+		dhd_dma_buf_free(dhd, &prot->host_bus_throughput_buf);
+		dhd_dma_buf_free(dhd, &prot->hostts_req_buf);
+		dhd_dma_buf_free(dhd, &prot->fw_trap_buf);
+		dhd_dma_buf_free(dhd, &prot->host_scb_buf);
+
+		/* DMA-able buffers for DMAing H2D/D2H WR/RD indices */
+		dhd_dma_buf_free(dhd, &prot->h2d_dma_indx_wr_buf);
+		dhd_dma_buf_free(dhd, &prot->h2d_dma_indx_rd_buf);
+		dhd_dma_buf_free(dhd, &prot->d2h_dma_indx_wr_buf);
+		dhd_dma_buf_free(dhd, &prot->d2h_dma_indx_rd_buf);
+
+		dhd_dma_buf_free(dhd, &prot->h2d_ifrm_indx_wr_buf);
+
+		/* Common MsgBuf Rings */
+		dhd_prot_ring_detach(dhd, &prot->h2dring_ctrl_subn);
+		dhd_prot_ring_detach(dhd, &prot->h2dring_rxp_subn);
+		dhd_prot_ring_detach(dhd, &prot->d2hring_ctrl_cpln);
+		dhd_prot_ring_detach(dhd, &prot->d2hring_tx_cpln);
+		dhd_prot_ring_detach(dhd, &prot->d2hring_rx_cpln);
+
+		/* Detach each DMA-able buffer and free the pool of msgbuf_ring_t */
+		dhd_prot_flowrings_pool_detach(dhd);
+
+#ifdef FLOW_RING_PREALLOC
+		if (dhd->htput_support) {
+			for (i = 0; i < HTPUT_TOTAL_FLOW_RINGS; i++) {
+				dhd_dma_buf_free(dhd, &prot->htput_ring_buf[i]);
+			}
+		}
+
+		for (i = 0; i < dhd->non_htput_total_flow_rings; i++) {
+			dhd_dma_buf_free(dhd, &prot->flow_ring_buf[i]);
+		}
+#endif /* FLOW_RING_PREALLOC */
+
+		/* detach info rings */
+		dhd_prot_detach_info_rings(dhd);
+
+#ifdef EWP_EDL
+		dhd_prot_detach_edl_rings(dhd);
+#endif
+		dhd_prot_detach_md_rings(dhd);
+
+		/* if IOCTLRESP_USE_CONSTMEM is defined IOCTL PKTs use pktid_map_handle_ioctl
+		 * handler and PKT memory is allocated using alloc_ioctl_return_buffer(), Otherwise
+		 * they will be part of pktid_ctrl_map handler and PKT memory is allocated using
+		 * PKTGET_STATIC (if DHD_USE_STATIC_CTRLBUF is defined) OR PKGET.
+		 * Similarly for freeing PKT buffers DHD_NATIVE_TO_PKTID_FINI will be used
+		 * which calls PKTFREE_STATIC (if DHD_USE_STATIC_CTRLBUF is defined) OR PKFREE.
+		 * Else if IOCTLRESP_USE_CONSTMEM is defined IOCTL PKTs will be freed using
+		 * DHD_NATIVE_TO_PKTID_FINI_IOCTL which calls free_ioctl_return_buffer.
+		 */
+		DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_ctrl_map);
+		DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_rx_map);
+		DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_tx_map);
+#ifdef IOCTLRESP_USE_CONSTMEM
+		DHD_NATIVE_TO_PKTID_FINI_IOCTL(dhd, prot->pktid_map_handle_ioctl);
+#endif
+#ifdef DHD_MAP_PKTID_LOGGING
+		DHD_PKTID_LOG_FINI(dhd, prot->pktid_dma_map);
+		DHD_PKTID_LOG_FINI(dhd, prot->pktid_dma_unmap);
+#endif /* DHD_MAP_PKTID_LOGGING */
+#ifdef DHD_DMA_INDICES_SEQNUM
+		if (prot->h2d_dma_indx_rd_copy_buf) {
+			MFREE(dhd->osh, prot->h2d_dma_indx_rd_copy_buf,
+				prot->h2d_dma_indx_rd_copy_bufsz);
+		}
+		if (prot->d2h_dma_indx_wr_copy_buf) {
+			MFREE(dhd->osh, prot->d2h_dma_indx_wr_copy_buf,
+				prot->d2h_dma_indx_wr_copy_bufsz);
+		}
+#endif /* DHD_DMA_INDICES_SEQNUM */
+		DHD_OS_PREFREE(dhd, dhd->prot, sizeof(dhd_prot_t));
+
+		dhd->prot = NULL;
+	}
+} /* dhd_prot_detach */
+
+/**
+ * dhd_prot_reset - Reset the protocol layer without freeing any objects.
+ * This may be invoked to soft reboot the dongle, without having to
+ * detach and attach the entire protocol layer.
+ *
+ * After dhd_prot_reset(), dhd_prot_init() may be invoked
+ * without going througha dhd_prot_attach() phase.
+ */
+void
+dhd_prot_reset(dhd_pub_t *dhd)
+{
+#ifdef FLOW_RING_PREALLOC
+	int i = 0;
+#endif /* FLOW_RING_PREALLOC */
+	struct dhd_prot *prot = dhd->prot;
+
+	DHD_TRACE(("%s\n", __FUNCTION__));
+
+	if (prot == NULL) {
+		return;
+	}
+
+	dhd->ring_attached = FALSE;
+
+	dhd_prot_flowrings_pool_reset(dhd);
+
+	/* Reset Common MsgBuf Rings */
+	dhd_prot_ring_reset(dhd, &prot->h2dring_ctrl_subn);
+	dhd_prot_ring_reset(dhd, &prot->h2dring_rxp_subn);
+	dhd_prot_ring_reset(dhd, &prot->d2hring_ctrl_cpln);
+	dhd_prot_ring_reset(dhd, &prot->d2hring_tx_cpln);
+	dhd_prot_ring_reset(dhd, &prot->d2hring_rx_cpln);
+
+	/* Reset info rings */
+	if (prot->h2dring_info_subn) {
+		dhd_prot_ring_reset(dhd, prot->h2dring_info_subn);
+	}
+
+	if (prot->d2hring_info_cpln) {
+		dhd_prot_ring_reset(dhd, prot->d2hring_info_cpln);
+	}
+
+#ifdef EWP_EDL
+	if (prot->d2hring_edl) {
+		dhd_prot_ring_reset(dhd, prot->d2hring_edl);
+	}
+#endif /* EWP_EDL */
+
+	/* Reset all DMA-able buffers allocated during prot attach */
+	dhd_dma_buf_reset(dhd, &prot->d2h_dma_scratch_buf);
+#ifdef DHD_HMAPTEST
+	dhd_dma_buf_reset(dhd, &prot->hmaptest.mem);
+#endif /* DHD_HMAPTEST */
+	dhd_dma_buf_reset(dhd, &prot->retbuf);
+	dhd_dma_buf_reset(dhd, &prot->ioctbuf);
+	dhd_dma_buf_reset(dhd, &prot->host_bus_throughput_buf);
+	dhd_dma_buf_reset(dhd, &prot->hostts_req_buf);
+	dhd_dma_buf_reset(dhd, &prot->fw_trap_buf);
+	dhd_dma_buf_reset(dhd, &prot->host_scb_buf);
+
+	dhd_dma_buf_reset(dhd, &prot->h2d_ifrm_indx_wr_buf);
+
+	/* Rest all DMA-able buffers for DMAing H2D/D2H WR/RD indices */
+	dhd_dma_buf_reset(dhd, &prot->h2d_dma_indx_rd_buf);
+	dhd_dma_buf_reset(dhd, &prot->h2d_dma_indx_wr_buf);
+	dhd_dma_buf_reset(dhd, &prot->d2h_dma_indx_rd_buf);
+	dhd_dma_buf_reset(dhd, &prot->d2h_dma_indx_wr_buf);
+
+#ifdef FLOW_RING_PREALLOC
+	if (dhd->htput_support) {
+		for (i = 0; i < HTPUT_TOTAL_FLOW_RINGS; i++) {
+			dhd_dma_buf_reset(dhd, &prot->htput_ring_buf[i]);
+		}
+	}
+
+	for (i = 0; i < dhd->non_htput_total_flow_rings; i++) {
+		dhd_dma_buf_reset(dhd, &prot->flow_ring_buf[i]);
+	}
+#endif /* FLOW_RING_PREALLOC */
+
+#ifdef DHD_DMA_INDICES_SEQNUM
+		if (prot->d2h_dma_indx_wr_copy_buf) {
+			dhd_local_buf_reset(prot->h2d_dma_indx_rd_copy_buf,
+				prot->h2d_dma_indx_rd_copy_bufsz);
+			dhd_local_buf_reset(prot->d2h_dma_indx_wr_copy_buf,
+				prot->d2h_dma_indx_wr_copy_bufsz);
+		}
+#endif /* DHD_DMA_INDICES_SEQNUM */
+
+	/* XXX: dmaxfer src and dst? */
+
+	prot->rx_metadata_offset = 0;
+	prot->tx_metadata_offset = 0;
+
+	prot->rxbufpost = 0;
+	prot->cur_event_bufs_posted = 0;
+	prot->cur_ioctlresp_bufs_posted = 0;
+
+	OSL_ATOMIC_INIT(dhd->osh, &prot->active_tx_count);
+	prot->data_seq_no = 0;
+	prot->ioctl_seq_no = 0;
+	prot->ioctl_state = 0;
+	prot->curr_ioctl_cmd = 0;
+	prot->ioctl_received = IOCTL_WAIT;
+	/* To catch any rollover issues fast, starting with higher ioctl_trans_id */
+	prot->ioctl_trans_id = MAXBITVAL(NBITS(prot->ioctl_trans_id)) - BUFFER_BEFORE_ROLLOVER;
+	prot->txcpl_db_cnt = 0;
+
+	/* dhd_flow_rings_init is located at dhd_bus_start,
+	 * so when stopping bus, flowrings shall be deleted
+	 */
+	if (dhd->flow_rings_inited) {
+		dhd_flow_rings_deinit(dhd);
+	}
+
+	if (prot->d2hring_md_cpl) {
+		dhd_prot_ring_reset(dhd, prot->d2hring_md_cpl);
+	}
+
+	/* Reset PKTID map */
+	DHD_NATIVE_TO_PKTID_RESET(dhd, prot->pktid_ctrl_map);
+	DHD_NATIVE_TO_PKTID_RESET(dhd, prot->pktid_rx_map);
+	DHD_NATIVE_TO_PKTID_RESET(dhd, prot->pktid_tx_map);
+#ifdef IOCTLRESP_USE_CONSTMEM
+	DHD_NATIVE_TO_PKTID_RESET_IOCTL(dhd, prot->pktid_map_handle_ioctl);
+#endif /* IOCTLRESP_USE_CONSTMEM */
+#ifdef DMAMAP_STATS
+	dhd->dma_stats.txdata = dhd->dma_stats.txdata_sz = 0;
+	dhd->dma_stats.rxdata = dhd->dma_stats.rxdata_sz = 0;
+#ifndef IOCTLRESP_USE_CONSTMEM
+	dhd->dma_stats.ioctl_rx = dhd->dma_stats.ioctl_rx_sz = 0;
+#endif /* IOCTLRESP_USE_CONSTMEM */
+	dhd->dma_stats.event_rx = dhd->dma_stats.event_rx_sz = 0;
+	dhd->dma_stats.info_rx = dhd->dma_stats.info_rx_sz = 0;
+	dhd->dma_stats.tsbuf_rx = dhd->dma_stats.tsbuf_rx_sz = 0;
+#endif /* DMAMAP_STATS */
+
+#ifdef AGG_H2D_DB
+	dhd_msgbuf_agg_h2d_db_timer_reset(dhd);
+#endif /* AGG_H2D_DB */
+
+} /* dhd_prot_reset */
+
+#if defined(DHD_LB_RXP)
+#define DHD_LB_DISPATCH_RX_PROCESS(dhdp)	dhd_lb_dispatch_rx_process(dhdp)
+#else /* !DHD_LB_RXP */
+#define DHD_LB_DISPATCH_RX_PROCESS(dhdp)	do { /* noop */ } while (0)
+#endif /* !DHD_LB_RXP */
+
+#if defined(DHD_LB)
+/* DHD load balancing: deferral of work to another online CPU */
+/* DHD_LB_RXP dispatchers, in dhd_linux.c */
+extern void dhd_lb_rx_napi_dispatch(dhd_pub_t *dhdp);
+extern void dhd_lb_rx_pkt_enqueue(dhd_pub_t *dhdp, void *pkt, int ifidx);
+extern unsigned long dhd_read_lb_rxp(dhd_pub_t *dhdp);
+extern void dhd_rx_emerge_enqueue(dhd_pub_t *dhdp, void *pkt);
+extern void * dhd_rx_emerge_dequeue(dhd_pub_t *dhdp);
+
+#if defined(DHD_LB_RXP)
+/**
+ * dhd_lb_dispatch_rx_process - load balance by dispatch Rx processing work
+ * to other CPU cores
+ */
+static INLINE void
+dhd_lb_dispatch_rx_process(dhd_pub_t *dhdp)
+{
+	dhd_lb_rx_napi_dispatch(dhdp); /* dispatch rx_process_napi */
+}
+#endif /* DHD_LB_RXP */
+#else
+static INLINE void *
+dhd_rx_emerge_dequeue(dhd_pub_t *dhdp)
+{
+	return NULL;
+}
+#endif /* DHD_LB */
+
+void
+dhd_prot_rx_dataoffset(dhd_pub_t *dhd, uint32 rx_offset)
+{
+	dhd_prot_t *prot = dhd->prot;
+	prot->rx_dataoffset = rx_offset;
+}
+
+static int
+dhd_check_create_info_rings(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_ERROR;
+	uint16 ringid;
+
+	{
+		/* dongle may increase max_submission_rings so keep
+		 * ringid at end of dynamic rings
+		 */
+		ringid = dhd->bus->max_tx_flowrings +
+			(dhd->bus->max_submission_rings - dhd->bus->max_tx_flowrings) +
+			BCMPCIE_H2D_COMMON_MSGRINGS;
+	}
+
+	if (prot->d2hring_info_cpln) {
+		/* for d2hring re-entry case, clear inited flag */
+		prot->d2hring_info_cpln->inited = FALSE;
+	}
+
+	if (prot->h2dring_info_subn && prot->d2hring_info_cpln) {
+		return BCME_OK; /* dhd_prot_init rentry after a dhd_prot_reset */
+	}
+
+	if (prot->h2dring_info_subn == NULL) {
+		prot->h2dring_info_subn = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
+
+		if (prot->h2dring_info_subn == NULL) {
+			DHD_ERROR(("%s: couldn't alloc memory for h2dring_info_subn\n",
+				__FUNCTION__));
+			return BCME_NOMEM;
+		}
+
+		DHD_INFO(("%s: about to create debug submit ring\n", __FUNCTION__));
+		ret = dhd_prot_ring_attach(dhd, prot->h2dring_info_subn, "h2dinfo",
+			H2DRING_DYNAMIC_INFO_MAX_ITEM, H2DRING_INFO_BUFPOST_ITEMSIZE,
+			ringid);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: couldn't alloc resources for dbg submit ring\n",
+				__FUNCTION__));
+			goto err;
+		}
+	}
+
+	if (prot->d2hring_info_cpln == NULL) {
+		prot->d2hring_info_cpln = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
+
+		if (prot->d2hring_info_cpln == NULL) {
+			DHD_ERROR(("%s: couldn't alloc memory for h2dring_info_subn\n",
+				__FUNCTION__));
+			return BCME_NOMEM;
+		}
+
+		/* create the debug info completion ring next to debug info submit ring
+		* ringid = id next to debug info submit ring
+		*/
+		ringid = ringid + 1;
+
+		DHD_INFO(("%s: about to create debug cpl ring\n", __FUNCTION__));
+		ret = dhd_prot_ring_attach(dhd, prot->d2hring_info_cpln, "d2hinfo",
+			D2HRING_DYNAMIC_INFO_MAX_ITEM, D2HRING_INFO_BUFCMPLT_ITEMSIZE,
+			ringid);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: couldn't alloc resources for dbg cpl ring\n",
+				__FUNCTION__));
+			dhd_prot_ring_detach(dhd, prot->h2dring_info_subn);
+			goto err;
+		}
+	}
+
+	return ret;
+err:
+	MFREE(prot->osh, prot->h2dring_info_subn, sizeof(msgbuf_ring_t));
+
+	if (prot->d2hring_info_cpln) {
+		MFREE(prot->osh, prot->d2hring_info_cpln, sizeof(msgbuf_ring_t));
+	}
+	return ret;
+} /* dhd_check_create_info_rings */
+
+int
+dhd_prot_init_info_rings(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_OK;
+
+	if ((ret = dhd_check_create_info_rings(dhd)) != BCME_OK) {
+		DHD_ERROR(("%s: info rings aren't created! \n",
+			__FUNCTION__));
+		return ret;
+	}
+
+	if ((prot->d2hring_info_cpln->inited) || (prot->d2hring_info_cpln->create_pending)) {
+		DHD_INFO(("Info completion ring was created!\n"));
+		return ret;
+	}
+
+	DHD_ERROR(("trying to send create d2h info ring: id %d\n",
+		prot->d2hring_info_cpln->idx));
+	ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_info_cpln,
+		BCMPCIE_D2H_RING_TYPE_DBGBUF_CPL, DHD_D2H_DBGRING_REQ_PKTID);
+	if (ret != BCME_OK)
+		return ret;
+
+	prot->h2dring_info_subn->seqnum = H2D_EPOCH_INIT_VAL;
+	prot->h2dring_info_subn->current_phase = 0;
+	prot->d2hring_info_cpln->seqnum = D2H_EPOCH_INIT_VAL;
+	prot->d2hring_info_cpln->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+
+	DHD_ERROR_MEM(("trying to send create h2d info ring id %d\n",
+		prot->h2dring_info_subn->idx));
+	prot->h2dring_info_subn->n_completion_ids = 1;
+	prot->h2dring_info_subn->compeltion_ring_ids[0] = prot->d2hring_info_cpln->idx;
+
+	ret = dhd_send_h2d_ringcreate(dhd, prot->h2dring_info_subn,
+		BCMPCIE_H2D_RING_TYPE_DBGBUF_SUBMIT, DHD_H2D_DBGRING_REQ_PKTID);
+
+	/* Note that there is no way to delete d2h or h2d ring deletion incase either fails,
+	 * so can not cleanup if one ring was created while the other failed
+	 */
+	return ret;
+} /* dhd_prot_init_info_rings */
+
+static void
+dhd_prot_detach_info_rings(dhd_pub_t *dhd)
+{
+	if (dhd->prot->h2dring_info_subn) {
+		dhd_prot_ring_detach(dhd, dhd->prot->h2dring_info_subn);
+		MFREE(dhd->prot->osh, dhd->prot->h2dring_info_subn, sizeof(msgbuf_ring_t));
+	}
+	if (dhd->prot->d2hring_info_cpln) {
+		dhd_prot_ring_detach(dhd, dhd->prot->d2hring_info_cpln);
+		MFREE(dhd->prot->osh, dhd->prot->d2hring_info_cpln, sizeof(msgbuf_ring_t));
+	}
+}
+
+static int
+dhd_check_create_md_rings(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_ERROR;
+	uint16 ringid;
+
+	/* Last dynamic ring indices are used by metadata rings */
+	ringid = dhd->bus->max_submission_rings + dhd->bus->max_completion_rings - 1;
+
+	if (prot->d2hring_md_cpl == NULL) {
+		prot->d2hring_md_cpl = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
+
+		if (prot->d2hring_md_cpl == NULL) {
+			DHD_ERROR(("%s: couldn't alloc memory for d2hring_md_cpl\n",
+				__FUNCTION__));
+			return BCME_NOMEM;
+		}
+
+		DHD_INFO(("%s: about to create md cpl ring\n", __FUNCTION__));
+		ret = dhd_prot_ring_attach(dhd, prot->d2hring_md_cpl, "d2hmd_cpl",
+			D2HRING_MDRING_MAX_ITEM, D2HRING_MDCMPLT_ITEMSIZE,
+			ringid);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: couldn't alloc resources for md txcpl ring\n",
+				__FUNCTION__));
+			goto err;
+		}
+		dhd->md_item_count = 0;
+		if ((dhd->mdring_info = MALLOCZ(prot->osh,
+			MAX_MDRING_ITEM_DUMP * D2HRING_MDCMPLT_ITEMSIZE)) == NULL) {
+			DHD_ERROR(("%s: couldn't alloc resources for md ring dump\n",
+				__FUNCTION__));
+		}
+	}
+	if (prot->d2hring_md_cpl != NULL) {
+		/* dhd_prot_init rentry after a dhd_prot_reset */
+		ret = BCME_OK;
+	}
+
+	return ret;
+err:
+	MFREE(prot->osh, prot->d2hring_md_cpl, sizeof(msgbuf_ring_t));
+	prot->d2hring_md_cpl = NULL;
+	return ret;
+} /* dhd_check_create_md_rings */
+
+int
+dhd_prot_init_md_rings(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_OK;
+
+	if ((ret = dhd_check_create_md_rings(dhd)) != BCME_OK) {
+		DHD_ERROR(("%s: md rings aren't created! \n",
+			__FUNCTION__));
+		return ret;
+	}
+
+	if ((prot->d2hring_md_cpl->inited) || (prot->d2hring_md_cpl->create_pending)) {
+		DHD_INFO(("md completion ring was created!\n"));
+		return ret;
+	}
+
+	DHD_ERROR(("trying to send create d2h md cpl ring: id %d\n",
+		prot->d2hring_md_cpl->idx));
+	ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_md_cpl,
+		BCMPCIE_D2H_RING_TYPE_MDATA_CPL, DHD_D2H_MDRING_REQ_PKTID);
+	if (ret != BCME_OK)
+		return ret;
+
+	prot->d2hring_md_cpl->seqnum = D2H_EPOCH_INIT_VAL;
+	prot->d2hring_md_cpl->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+
+	/* Note that there is no way to delete d2h or h2d ring deletion incase either fails,
+	 * so can not cleanup if one ring was created while the other failed
+	 */
+	return BCME_OK;
+} /* dhd_prot_init_md_rings */
+
+static void
+dhd_prot_detach_md_rings(dhd_pub_t *dhd)
+{
+	if (dhd->prot->d2hring_md_cpl) {
+		dhd_prot_ring_detach(dhd, dhd->prot->d2hring_md_cpl);
+		MFREE(dhd->prot->osh, dhd->prot->d2hring_md_cpl, sizeof(msgbuf_ring_t));
+		dhd->prot->d2hring_md_cpl = NULL;
+	}
+}
+
+#ifdef EWP_EDL
+static int
+dhd_check_create_edl_rings(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_ERROR;
+	uint16 ringid;
+
+	{
+		/* dongle may increase max_submission_rings so keep
+		 * ringid at end of dynamic rings (re-use info ring cpl ring id)
+		 */
+		ringid = dhd->bus->max_tx_flowrings +
+			(dhd->bus->max_submission_rings - dhd->bus->max_tx_flowrings) +
+			BCMPCIE_H2D_COMMON_MSGRINGS + 1;
+	}
+
+	if (prot->d2hring_edl) {
+		prot->d2hring_edl->inited = FALSE;
+		return BCME_OK; /* dhd_prot_init rentry after a dhd_prot_reset */
+	}
+
+	if (prot->d2hring_edl == NULL) {
+		prot->d2hring_edl = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
+
+		if (prot->d2hring_edl == NULL) {
+			DHD_ERROR(("%s: couldn't alloc memory for d2hring_edl\n",
+				__FUNCTION__));
+			return BCME_NOMEM;
+		}
+
+		DHD_ERROR(("%s: about to create EDL ring, ringid: %u \n", __FUNCTION__,
+			ringid));
+		ret = dhd_prot_ring_attach(dhd, prot->d2hring_edl, "d2hring_edl",
+			D2HRING_EDL_MAX_ITEM, D2HRING_EDL_ITEMSIZE,
+			ringid);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: couldn't alloc resources for EDL ring\n",
+				__FUNCTION__));
+			goto err;
+		}
+	}
+
+	return ret;
+err:
+	MFREE(prot->osh, prot->d2hring_edl, sizeof(msgbuf_ring_t));
+	prot->d2hring_edl = NULL;
+
+	return ret;
+} /* dhd_check_create_btlog_rings */
+
+int
+dhd_prot_init_edl_rings(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_ERROR;
+
+	if ((ret = dhd_check_create_edl_rings(dhd)) != BCME_OK) {
+		DHD_ERROR(("%s: EDL rings aren't created! \n",
+			__FUNCTION__));
+		return ret;
+	}
+
+	if ((prot->d2hring_edl->inited) || (prot->d2hring_edl->create_pending)) {
+		DHD_INFO(("EDL completion ring was created!\n"));
+		return ret;
+	}
+
+	DHD_ERROR_MEM(("trying to send create d2h edl ring: idx %d\n", prot->d2hring_edl->idx));
+	ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_edl,
+		BCMPCIE_D2H_RING_TYPE_EDL, DHD_D2H_DBGRING_REQ_PKTID);
+	if (ret != BCME_OK)
+		return ret;
+
+	prot->d2hring_edl->seqnum = D2H_EPOCH_INIT_VAL;
+	prot->d2hring_edl->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+
+	return BCME_OK;
+} /* dhd_prot_init_btlog_rings */
+
+static void
+dhd_prot_detach_edl_rings(dhd_pub_t *dhd)
+{
+	if (dhd->prot->d2hring_edl) {
+		dhd_prot_ring_detach(dhd, dhd->prot->d2hring_edl);
+		MFREE(dhd->prot->osh, dhd->prot->d2hring_edl, sizeof(msgbuf_ring_t));
+		dhd->prot->d2hring_edl = NULL;
+	}
+}
+#endif	/* EWP_EDL */
+
+/**
+ * Initialize protocol: sync w/dongle state.
+ * Sets dongle media info (iswl, drv_version, mac address).
+ */
+int dhd_sync_with_dongle(dhd_pub_t *dhd)
+{
+	int ret = 0;
+	uint len = 0;
+	wlc_rev_info_t revinfo;
+	char buf[128];
+	dhd_prot_t *prot = dhd->prot;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	dhd_os_set_ioctl_resp_timeout(IOCTL_RESP_TIMEOUT);
+
+	/* Post ts buffer after shim layer is attached */
+	ret = dhd_msgbuf_rxbuf_post_ts_bufs(dhd);
+
+	/* query for 'wlc_ver' to get version info from firmware */
+	/* memsetting to zero */
+	bzero(buf, sizeof(buf));
+	len = bcm_mkiovar("wlc_ver", NULL, 0, buf, sizeof(buf));
+	if (len == 0) {
+		DHD_ERROR(("%s failed in calling bcm_mkiovar %u\n", __FUNCTION__, len));
+		ret = BCME_ERROR;
+		goto done;
+	}
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
+	if (ret < 0) {
+		DHD_ERROR(("%s failed %d\n", __FUNCTION__, ret));
+	} else {
+		dhd->wlc_ver_major = ((wl_wlc_version_t*)buf)->wlc_ver_major;
+		dhd->wlc_ver_minor = ((wl_wlc_version_t*)buf)->wlc_ver_minor;
+	}
+
+	DHD_INFO(("wlc_ver_major %d, wlc_ver_minor %d\n",
+		dhd->wlc_ver_major, dhd->wlc_ver_minor));
+#ifndef OEM_ANDROID
+	/* Get the device MAC address */
+	bzero(buf, sizeof(buf));
+	strlcpy(buf, "cur_etheraddr", sizeof(buf));
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
+	if (ret < 0) {
+		DHD_ERROR(("%s: GET iovar cur_etheraddr FAILED\n", __FUNCTION__));
+		goto done;
+	}
+	eacopy(&buf, &dhd->mac.octet);
+	if (dhd_msg_level & DHD_INFO_VAL) {
+		bcm_print_bytes("CUR_ETHERADDR ", (uchar *)buf, ETHER_ADDR_LEN);
+	}
+#endif /* OEM_ANDROID */
+
+#ifdef DHD_FW_COREDUMP
+	/* Check the memdump capability */
+	dhd_get_memdump_info(dhd);
+#endif /* DHD_FW_COREDUMP */
+#ifdef BCMASSERT_LOG
+	dhd_get_assert_info(dhd);
+#endif /* BCMASSERT_LOG */
+
+	/* Get the device rev info */
+	memset(&revinfo, 0, sizeof(revinfo));
+	ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_REVINFO, &revinfo, sizeof(revinfo), FALSE, 0);
+	if (ret < 0) {
+		DHD_ERROR(("%s: GET revinfo FAILED\n", __FUNCTION__));
+		goto done;
+	}
+	DHD_ERROR(("%s: GET_REVINFO device 0x%x, vendor 0x%x, chipnum 0x%x\n", __FUNCTION__,
+		revinfo.deviceid, revinfo.vendorid, revinfo.chipnum));
+
+	/* Get the RxBuf post size */
+	/* Use default value in case of failure */
+	prot->rxbufpost_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
+	memset(buf, 0, sizeof(buf));
+	len = bcm_mkiovar("rxbufpost_sz", NULL, 0, buf, sizeof(buf));
+	if (len == 0) {
+		DHD_ERROR(("%s failed in calling bcm_mkiovar %u\n", __FUNCTION__, len));
+	} else {
+		ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
+		if (ret < 0) {
+			DHD_ERROR(("%s: GET RxBuf post FAILED, use default %d\n",
+				__FUNCTION__, DHD_FLOWRING_RX_BUFPOST_PKTSZ));
+		} else {
+			if (memcpy_s(&(prot->rxbufpost_sz), sizeof(prot->rxbufpost_sz),
+					buf, sizeof(uint16)) != BCME_OK) {
+				DHD_ERROR(("%s: rxbufpost_sz memcpy failed\n", __FUNCTION__));
+			}
+
+			if (prot->rxbufpost_sz > DHD_FLOWRING_RX_BUFPOST_PKTSZ_MAX) {
+				DHD_ERROR(("%s: Invalid RxBuf post size : %d, default to %d\n",
+					__FUNCTION__, prot->rxbufpost_sz,
+					DHD_FLOWRING_RX_BUFPOST_PKTSZ));
+					prot->rxbufpost_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
+			} else {
+				DHD_ERROR(("%s: RxBuf Post : %d\n",
+					__FUNCTION__, prot->rxbufpost_sz));
+			}
+		}
+	}
+
+	prot->rxbufpost_alloc_sz = dhd_plat_align_rxbuf_size(prot->rxbufpost_sz);
+	DHD_ERROR(("%s: RxBuf Post Alloc : %d\n", __FUNCTION__, prot->rxbufpost_alloc_sz));
+
+#ifdef RX_PKT_POOL
+	/* Rx pkt pool creation after rxbuf size is shared by dongle */
+	dhd_rx_pktpool_create(dhd->info, prot->rxbufpost_alloc_sz);
+#endif /* RX_PKT_POOL */
+
+	/* Post buffers for packet reception */
+	dhd_msgbuf_rxbuf_post(dhd, FALSE); /* alloc pkt ids */
+
+	DHD_SSSR_DUMP_INIT(dhd);
+
+	dhd_process_cid_mac(dhd, TRUE);
+	ret = dhd_preinit_ioctls(dhd);
+	dhd_process_cid_mac(dhd, FALSE);
+#if defined(DHD_SDTC_ETB_DUMP)
+	dhd_sdtc_etb_init(dhd);
+#endif /* DHD_SDTC_ETB_DUMP */
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+	if (FW_SUPPORTED(dhd, h2dlogts)) {
+		dhd->dhd_rte_time_sync_ms = DHD_H2D_LOG_TIME_STAMP_MATCH;
+		dhd->bus->dhd_rte_time_sync_count = OSL_SYSUPTIME_US();
+		/* This is during initialization. */
+		dhd_h2d_log_time_sync(dhd);
+	} else {
+		dhd->dhd_rte_time_sync_ms = 0;
+	}
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+	/* Always assumes wl for now */
+	dhd->iswl = TRUE;
+done:
+	return ret;
+} /* dhd_sync_with_dongle */
+
+#define DHD_DBG_SHOW_METADATA	0
+
+#if DHD_DBG_SHOW_METADATA
+static void
+BCMFASTPATH(dhd_prot_print_metadata)(dhd_pub_t *dhd, void *ptr, int len)
+{
+	uint8 tlv_t;
+	uint8 tlv_l;
+	uint8 *tlv_v = (uint8 *)ptr;
+	int ret = 0;
+
+	if (len <= BCMPCIE_D2H_METADATA_HDRLEN)
+		return;
+
+	len -= BCMPCIE_D2H_METADATA_HDRLEN;
+	tlv_v += BCMPCIE_D2H_METADATA_HDRLEN;
+
+	while (len > TLV_HDR_LEN) {
+		tlv_t = tlv_v[TLV_TAG_OFF];
+		tlv_l = tlv_v[TLV_LEN_OFF];
+
+		len -= TLV_HDR_LEN;
+		tlv_v += TLV_HDR_LEN;
+		if (len < tlv_l)
+			break;
+		if ((tlv_t == 0) || (tlv_t == WLFC_CTL_TYPE_FILLER))
+			break;
+
+		switch (tlv_t) {
+		case WLFC_CTL_TYPE_TXSTATUS: {
+			uint32 txs;
+			(void)memcpy_s(&txs, sizeof(txs), tlv_v, sizeof(uint32));
+			if (tlv_l < (sizeof(wl_txstatus_additional_info_t) + sizeof(uint32))) {
+				DHD_CONS_ONLY(("METADATA TX_STATUS: %08x\n", txs));
+			} else {
+				wl_txstatus_additional_info_t tx_add_info;
+				ret = memcpy_s(&tx_add_info, sizeof(wl_txstatus_additional_info_t),
+					tlv_v + sizeof(uint32), tlv_l);
+				if (ret) {
+					DHD_ERROR(("tx status memcpy failed:%d, destsz:%lu, n:%d\n",
+						ret, sizeof(wl_txstatus_additional_info_t), tlv_l));
+					break;
+				}
+				DHD_CONS_ONLY(("METADATA TX_STATUS: %08x"
+					" WLFCTS[%04x | %08x - %08x - %08x]"
+					" rate = %08x tries = %d - %d\n", txs,
+					tx_add_info.seq, tx_add_info.entry_ts,
+					tx_add_info.enq_ts, tx_add_info.last_ts,
+					tx_add_info.rspec, tx_add_info.rts_cnt,
+					tx_add_info.tx_cnt));
+			}
+			} break;
+
+		case WLFC_CTL_TYPE_RSSI: {
+			if (tlv_l == 1)
+				DHD_CONS_ONLY(("METADATA RX_RSSI: rssi = %d\n", *tlv_v));
+			else
+				DHD_CONS_ONLY(("METADATA RX_RSSI[%04x]: rssi = %d snr = %d\n",
+					(*(tlv_v + 3) << 8) | *(tlv_v + 2),
+					(int8)(*tlv_v), *(tlv_v + 1))_;
+			} break;
+
+		case WLFC_CTL_TYPE_FIFO_CREDITBACK:
+			bcm_print_bytes("METADATA FIFO_CREDITBACK", tlv_v, tlv_l);
+			break;
+
+		case WLFC_CTL_TYPE_TX_ENTRY_STAMP:
+			bcm_print_bytes("METADATA TX_ENTRY", tlv_v, tlv_l);
+			break;
+
+		case WLFC_CTL_TYPE_RX_STAMP: {
+			struct {
+				uint32 rspec;
+				uint32 bus_time;
+				uint32 wlan_time;
+			} rx_tmstamp;
+			ret = memcpy_s(&rx_tmstamp, sizeof(rx_tmstamp), tlv_v, tlv_l);
+			if (ret) {
+				DHD_ERROR(("rx tmstamp memcpy failed:%d, destsz:%lu, n:%d\n",
+					ret, sizeof(rx_tmstamp), tlv_l));
+				break;
+			}
+			DHD_CONS_ONLY(("METADATA RX TIMESTMAP: WLFCTS[%08x - %08x] rate = %08x\n",
+				rx_tmstamp.wlan_time, rx_tmstamp.bus_time, rx_tmstamp.rspec));
+			}
+			break;
+
+		case WLFC_CTL_TYPE_TRANS_ID:
+			bcm_print_bytes("METADATA TRANS_ID", tlv_v, tlv_l);
+			break;
+
+		case WLFC_CTL_TYPE_COMP_TXSTATUS:
+			bcm_print_bytes("METADATA COMP_TXSTATUS", tlv_v, tlv_l);
+			break;
+
+		default:
+			bcm_print_bytes("METADATA UNKNOWN", tlv_v, tlv_l);
+			break;
+		}
+
+		len -= tlv_l;
+		tlv_v += tlv_l;
+	}
+}
+#endif /* DHD_DBG_SHOW_METADATA */
+
+static INLINE void
+BCMFASTPATH(dhd_prot_packet_free)(dhd_pub_t *dhd, void *pkt, uint8 pkttype, bool send)
+{
+	if (pkt) {
+		if (pkttype == PKTTYPE_IOCTL_RX ||
+			pkttype == PKTTYPE_EVENT_RX ||
+			pkttype == PKTTYPE_INFO_RX ||
+			pkttype == PKTTYPE_TSBUF_RX) {
+			PKTFREE_CTRLBUF(dhd->osh, pkt, send);
+		} else {
+			PKTFREE(dhd->osh, pkt, send);
+		}
+	}
+}
+
+/**
+ * dhd_prot_packet_get should be called only for items having pktid_ctrl_map handle
+ * and all the bottom most functions like dhd_pktid_map_free hold separate DHD_PKTID_LOCK
+ * to ensure thread safety, so no need to hold any locks for this function
+ */
+static INLINE void *
+BCMFASTPATH(dhd_prot_packet_get)(dhd_pub_t *dhd, uint32 pktid, uint8 pkttype, bool free_pktid)
+{
+	void *PKTBUF;
+	dmaaddr_t pa;
+	uint32 len;
+	void *dmah;
+	void *secdma;
+
+#ifdef DHD_PCIE_PKTID
+	if (free_pktid) {
+		PKTBUF = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_ctrl_map,
+			pktid, pa, len, dmah, secdma, pkttype);
+	} else {
+		PKTBUF = DHD_PKTID_TO_NATIVE_RSV(dhd, dhd->prot->pktid_ctrl_map,
+			pktid, pa, len, dmah, secdma, pkttype);
+	}
+#else
+	PKTBUF = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_ctrl_map, pktid, pa,
+		len, dmah, secdma, pkttype);
+#endif /* DHD_PCIE_PKTID */
+	if (PKTBUF) {
+		DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0, dmah);
+#ifdef DMAMAP_STATS
+		switch (pkttype) {
+#ifndef IOCTLRESP_USE_CONSTMEM
+			case PKTTYPE_IOCTL_RX:
+				dhd->dma_stats.ioctl_rx--;
+				dhd->dma_stats.ioctl_rx_sz -= len;
+				break;
+#endif /* IOCTLRESP_USE_CONSTMEM */
+			case PKTTYPE_EVENT_RX:
+				dhd->dma_stats.event_rx--;
+				dhd->dma_stats.event_rx_sz -= len;
+				break;
+			case PKTTYPE_INFO_RX:
+				dhd->dma_stats.info_rx--;
+				dhd->dma_stats.info_rx_sz -= len;
+				break;
+			case PKTTYPE_TSBUF_RX:
+				dhd->dma_stats.tsbuf_rx--;
+				dhd->dma_stats.tsbuf_rx_sz -= len;
+				break;
+		}
+#endif /* DMAMAP_STATS */
+	}
+
+	return PKTBUF;
+}
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+static INLINE void
+BCMFASTPATH(dhd_prot_ioctl_ret_buffer_get)(dhd_pub_t *dhd, uint32 pktid, dhd_dma_buf_t *retbuf)
+{
+	memset(retbuf, 0, sizeof(dhd_dma_buf_t));
+	retbuf->va = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_map_handle_ioctl, pktid,
+		retbuf->pa, retbuf->len, retbuf->dmah, retbuf->secdma, PKTTYPE_IOCTL_RX);
+
+	return;
+}
+#endif
+
+#ifdef PCIE_INB_DW
+static int
+dhd_prot_inc_hostactive_devwake_assert(dhd_bus_t *bus)
+{
+	unsigned long flags = 0;
+
+	if (INBAND_DW_ENAB(bus)) {
+		DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+		bus->host_active_cnt++;
+		DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+		if (dhd_bus_set_device_wake(bus, TRUE) != BCME_OK) {
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			bus->host_active_cnt--;
+			dhd_bus_inb_ack_pending_ds_req(bus);
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+			return BCME_ERROR;
+		}
+	}
+
+	return BCME_OK;
+}
+
+static void
+dhd_prot_dec_hostactive_ack_pending_dsreq(dhd_bus_t *bus)
+{
+	unsigned long flags = 0;
+	if (INBAND_DW_ENAB(bus)) {
+		DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+		bus->host_active_cnt--;
+		dhd_bus_inb_ack_pending_ds_req(bus);
+		DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+	}
+}
+#endif /* PCIE_INB_DW */
+
+static void
+BCMFASTPATH(dhd_msgbuf_rxbuf_post)(dhd_pub_t *dhd, bool use_rsv_pktid)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int16 fillbufs;
+	int retcount = 0;
+
+	fillbufs = prot->max_rxbufpost - prot->rxbufpost;
+	while (fillbufs >= prot->rx_buf_burst) {
+		/* Post in a burst of 32 buffers at a time */
+		fillbufs = MIN(fillbufs, prot->rx_buf_burst);
+
+		/* Post buffers */
+		retcount = dhd_prot_rxbuf_post(dhd, fillbufs, use_rsv_pktid);
+
+		if (retcount > 0) {
+			prot->rxbufpost += (uint16)retcount;
+			/* how many more to post */
+			fillbufs = prot->max_rxbufpost - prot->rxbufpost;
+		} else {
+			/* Make sure we don't run loop any further */
+			fillbufs = 0;
+		}
+	}
+}
+
+/** Post 'count' no of rx buffers to dongle */
+static int
+BCMFASTPATH(dhd_prot_rxbuf_post)(dhd_pub_t *dhd, uint16 count, bool use_rsv_pktid)
+{
+	void *p, **pktbuf;
+	uint8 *rxbuf_post_tmp;
+	host_rxbuf_post_t *rxbuf_post;
+	void *msg_start;
+	dmaaddr_t pa, *pktbuf_pa;
+	uint32 *pktlen;
+	uint16 i = 0, alloced = 0;
+	unsigned long flags;
+	uint32 pktid;
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring = &prot->h2dring_rxp_subn;
+	void *lcl_buf;
+	uint16 lcl_buf_size;
+
+#ifdef BCM_ROUTER_DHD
+	prot->rxbufpost_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ + BCMEXTRAHDROOM;
+	prot->rxbufpost_alloc_sz = prot->rxbufpost_sz;
+#endif /* BCM_ROUTER_DHD */
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	/* allocate a local buffer to store pkt buffer va, pa and length */
+	lcl_buf_size = (sizeof(void *) + sizeof(dmaaddr_t) + sizeof(uint32)) *
+		prot->rx_buf_burst;
+	lcl_buf = MALLOC(dhd->osh, lcl_buf_size);
+	if (!lcl_buf) {
+		DHD_ERROR(("%s: local scratch buffer allocation failed\n", __FUNCTION__));
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return 0;
+	}
+	pktbuf = lcl_buf;
+	pktbuf_pa = (dmaaddr_t *)((uint8 *)pktbuf + sizeof(void *) * prot->rx_buf_burst);
+	pktlen = (uint32 *)((uint8 *)pktbuf_pa + sizeof(dmaaddr_t) * prot->rx_buf_burst);
+
+	for (i = 0; i < count; i++) {
+		/* First try to dequeue from emergency queue which will be filled
+		 * during rx flow control.
+		*/
+		p = dhd_rx_emerge_dequeue(dhd);
+		if ((p == NULL) &&
+			((p = PKTGET(dhd->osh, prot->rxbufpost_alloc_sz, FALSE)) == NULL)) {
+			dhd->rx_pktgetfail++;
+			DHD_ERROR_RLMT(("%s:%d: PKTGET for rxbuf failed, rx_pktget_fail :%lu\n",
+				__FUNCTION__, __LINE__, dhd->rx_pktgetfail));
+			/* Try to get pkt from Rx reserve pool if monitor mode is not enabled as
+			 * the buffer size for monitor mode is larger(4k) than normal rx pkt(1920)
+			 */
+#if defined(WL_MONITOR)
+			if (dhd_monitor_enabled(dhd, 0)) {
+				break;
+			} else
+#endif /* WL_MONITOR */
+			{
+#ifdef RX_PKT_POOL
+				if ((p = PKTGET_RX_POOL(dhd->osh, dhd->info,
+					prot->rxbufpost_alloc_sz, FALSE)) == NULL) {
+					dhd->rx_pktgetpool_fail++;
+					DHD_ERROR_RLMT(("%s:%d: PKTGET_RX_POOL for rxbuf failed, "
+						"rx_pktgetpool_fail : %lu\n",
+						__FUNCTION__, __LINE__, dhd->rx_pktgetpool_fail));
+					break;
+				}
+#endif /* RX_PKT_POOL */
+			}
+		}
+
+		/* Set pktlen to the actual bufferpost size */
+		if (prot->rxbufpost_sz != prot->rxbufpost_alloc_sz) {
+			DHD_TRACE(("%s: pktlen before: %d\n", __FUNCTION__, PKTLEN(dhd->osh, p)));
+			PKTSETLEN(dhd->osh, p, prot->rxbufpost_sz);
+			DHD_TRACE(("%s: pktlen after: %d\n", __FUNCTION__, PKTLEN(dhd->osh, p)));
+		}
+
+#ifdef BCM_ROUTER_DHD
+		/* Reserve extra headroom for router builds */
+		PKTPULL(dhd->osh, p, BCMEXTRAHDROOM);
+#endif /* BCM_ROUTER_DHD */
+		pktlen[i] = PKTLEN(dhd->osh, p);
+		pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen[i], DMA_RX, p, 0);
+
+		if (PHYSADDRISZERO(pa)) {
+			PKTFREE(dhd->osh, p, FALSE);
+			DHD_ERROR(("Invalid phyaddr 0\n"));
+			ASSERT(0);
+			break;
+		}
+#ifdef DMAMAP_STATS
+		dhd->dma_stats.rxdata++;
+		dhd->dma_stats.rxdata_sz += pktlen[i];
+#endif /* DMAMAP_STATS */
+
+		PKTPULL(dhd->osh, p, prot->rx_metadata_offset);
+		pktlen[i] = PKTLEN(dhd->osh, p);
+		pktbuf[i] = p;
+		pktbuf_pa[i] = pa;
+	}
+
+	/* only post what we have */
+	count = i;
+
+	/* grab the ring lock to allocate pktid and post on ring */
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	/* Claim space for exactly 'count' no of messages, for mitigation purpose */
+	msg_start = (void *)
+		dhd_prot_alloc_ring_space(dhd, ring, count, &alloced, TRUE);
+	if (msg_start == NULL) {
+		DHD_INFO(("%s:%d: Rxbufpost Msgbuf Not available\n", __FUNCTION__, __LINE__));
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		goto cleanup;
+	}
+	/* if msg_start !=  NULL, we should have alloced space for atleast 1 item */
+	ASSERT(alloced > 0);
+
+	rxbuf_post_tmp = (uint8*)msg_start;
+
+	for (i = 0; i < alloced; i++) {
+		rxbuf_post = (host_rxbuf_post_t *)rxbuf_post_tmp;
+		p = pktbuf[i];
+		pa = pktbuf_pa[i];
+
+		pktid = DHD_NATIVE_TO_PKTID(dhd, dhd->prot->pktid_rx_map, p, pa,
+			pktlen[i], DMA_RX, NULL, ring->dma_buf.secdma, PKTTYPE_DATA_RX);
+#if defined(DHD_PCIE_PKTID)
+		if (pktid == DHD_PKTID_INVALID) {
+			break;
+		}
+#endif /* DHD_PCIE_PKTID */
+
+#ifdef DHD_HMAPTEST
+	if (dhd->prot->hmaptest_rx_active == HMAPTEST_D11_RX_ACTIVE) {
+		/* scratchbuf area */
+		dhd->prot->hmap_rx_buf_va = (char *)dhd->prot->hmaptest.mem.va
+			+ dhd->prot->hmaptest.offset;
+
+		dhd->prot->hmap_rx_buf_len = pktlen[i] + prot->rx_metadata_offset;
+		if ((dhd->prot->hmap_rx_buf_va +  dhd->prot->hmap_rx_buf_len) >
+			((char *)dhd->prot->hmaptest.mem.va + dhd->prot->hmaptest.mem.len)) {
+			DHD_ERROR(("hmaptest: ERROR Rxpost outside HMAPTEST buffer\n"));
+			DHD_ERROR(("hmaptest: NOT Replacing Rx Buffer\n"));
+			dhd->prot->hmaptest_rx_active = HMAPTEST_D11_RX_INACTIVE;
+			dhd->prot->hmaptest.in_progress = FALSE;
+		} else {
+			pa = DMA_MAP(dhd->osh, dhd->prot->hmap_rx_buf_va,
+				dhd->prot->hmap_rx_buf_len, DMA_RX, p, 0);
+
+			dhd->prot->hmap_rx_buf_pa = pa;
+			dhd->prot->hmaptest_rx_pktid = pktid;
+			dhd->prot->hmaptest_rx_active = HMAPTEST_D11_RX_POSTED;
+			DHD_ERROR(("hmaptest: d11write rxpost scratch rxbuf pktid=0x%08x\n",
+				pktid));
+			DHD_ERROR(("hmaptest: d11write rxpost scratch rxbuf va=0x%p pa.lo=0x%08x\n",
+				dhd->prot->hmap_rx_buf_va, (uint32)PHYSADDRLO(pa)));
+			DHD_ERROR(("hmaptest: d11write rxpost orig pktdata va=0x%p pa.lo=0x%08x\n",
+				PKTDATA(dhd->osh, p), (uint32)PHYSADDRLO(pktbuf_pa[i])));
+		}
+	}
+#endif /* DHD_HMAPTEST */
+		dhd->prot->tot_rxbufpost++;
+		/* Common msg header */
+		rxbuf_post->cmn_hdr.msg_type = MSG_TYPE_RXBUF_POST;
+		rxbuf_post->cmn_hdr.if_id = 0;
+		rxbuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+		rxbuf_post->cmn_hdr.flags = ring->current_phase;
+		ring->seqnum++;
+		rxbuf_post->data_buf_len = htol16((uint16)pktlen[i]);
+		rxbuf_post->data_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
+		rxbuf_post->data_buf_addr.low_addr =
+			htol32(PHYSADDRLO(pa) + prot->rx_metadata_offset);
+
+		if (prot->rx_metadata_offset) {
+			rxbuf_post->metadata_buf_len = prot->rx_metadata_offset;
+			rxbuf_post->metadata_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
+			rxbuf_post->metadata_buf_addr.low_addr  = htol32(PHYSADDRLO(pa));
+		} else {
+			rxbuf_post->metadata_buf_len = 0;
+			rxbuf_post->metadata_buf_addr.high_addr = 0;
+			rxbuf_post->metadata_buf_addr.low_addr  = 0;
+		}
+
+#ifdef DHD_PKTID_AUDIT_RING
+		DHD_PKTID_AUDIT(dhd, prot->pktid_rx_map, pktid, DHD_DUPLICATE_ALLOC);
+#endif /* DHD_PKTID_AUDIT_RING */
+
+		rxbuf_post->cmn_hdr.request_id = htol32(pktid);
+
+		/* Move rxbuf_post_tmp to next item */
+		rxbuf_post_tmp = rxbuf_post_tmp + ring->item_len;
+#ifdef DHD_LBUF_AUDIT
+		PKTAUDIT(dhd->osh, p);
+#endif
+	}
+
+	if (i < alloced) {
+		if (ring->wr < (alloced - i))
+			ring->wr = ring->max_items - (alloced - i);
+		else
+			ring->wr -= (alloced - i);
+
+		if (ring->wr == 0) {
+			DHD_INFO(("%s: flipping the phase now\n", ring->name));
+				ring->current_phase = ring->current_phase ?
+				0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+		}
+
+		alloced = i;
+	}
+
+	/* update ring's WR index and ring doorbell to dongle */
+	if (alloced > 0) {
+		dhd_prot_ring_write_complete(dhd, ring, msg_start, alloced);
+	}
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+cleanup:
+	for (i = alloced; i < count; i++) {
+		p = pktbuf[i];
+		pa = pktbuf_pa[i];
+
+		DMA_UNMAP(dhd->osh, pa, pktlen[i], DMA_RX, 0, DHD_DMAH_NULL);
+		PKTFREE(dhd->osh, p, FALSE);
+	}
+
+	MFREE(dhd->osh, lcl_buf, lcl_buf_size);
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return alloced;
+} /* dhd_prot_rxbufpost */
+
+#if !defined(BCM_ROUTER_DHD)
+static int
+dhd_prot_infobufpost(dhd_pub_t *dhd, msgbuf_ring_t *ring)
+{
+	unsigned long flags;
+	uint32 pktid;
+	dhd_prot_t *prot = dhd->prot;
+	uint16 alloced = 0;
+	uint16 pktsz = DHD_INFOBUF_RX_BUFPOST_PKTSZ;
+	uint32 pktlen;
+	info_buf_post_msg_t *infobuf_post;
+	uint8 *infobuf_post_tmp;
+	void *p;
+	void* msg_start;
+	uint8 i = 0;
+	dmaaddr_t pa;
+	int16 count = 0;
+
+	if (ring == NULL)
+		return 0;
+
+	if (ring->inited != TRUE)
+		return 0;
+	if (ring == dhd->prot->h2dring_info_subn) {
+		if (prot->max_infobufpost == 0)
+			return 0;
+
+		count = prot->max_infobufpost - prot->infobufpost;
+	}
+	else {
+		DHD_ERROR(("Unknown ring\n"));
+		return 0;
+	}
+
+	if (count <= 0) {
+		DHD_INFO(("%s: Cannot post more than max info resp buffers\n",
+			__FUNCTION__));
+		return 0;
+	}
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	/* grab the ring lock to allocate pktid and post on ring */
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	/* Claim space for exactly 'count' no of messages, for mitigation purpose */
+	msg_start = (void *) dhd_prot_alloc_ring_space(dhd, ring, count, &alloced, FALSE);
+
+	if (msg_start == NULL) {
+		DHD_INFO(("%s:%d: infobufpost Msgbuf Not available\n", __FUNCTION__, __LINE__));
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return -1;
+	}
+
+	/* if msg_start !=  NULL, we should have alloced space for atleast 1 item */
+	ASSERT(alloced > 0);
+
+	infobuf_post_tmp = (uint8*) msg_start;
+
+	/* loop through each allocated message in the host ring */
+	for (i = 0; i < alloced; i++) {
+		infobuf_post = (info_buf_post_msg_t *) infobuf_post_tmp;
+		/* Create a rx buffer */
+#ifdef DHD_USE_STATIC_CTRLBUF
+		p = PKTGET_STATIC(dhd->osh, pktsz, FALSE);
+#else
+		p = PKTGET(dhd->osh, pktsz, FALSE);
+#endif /* DHD_USE_STATIC_CTRLBUF */
+		if (p == NULL) {
+			DHD_ERROR(("%s:%d: PKTGET for infobuf failed\n", __FUNCTION__, __LINE__));
+			dhd->rx_pktgetfail++;
+			break;
+		}
+		pktlen = PKTLEN(dhd->osh, p);
+		pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen, DMA_RX, p, 0);
+		if (PHYSADDRISZERO(pa)) {
+			DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
+			PKTFREE_CTRLBUF(dhd->osh, p, FALSE);
+			DHD_ERROR(("Invalid phyaddr 0\n"));
+			ASSERT(0);
+			break;
+		}
+#ifdef DMAMAP_STATS
+		dhd->dma_stats.info_rx++;
+		dhd->dma_stats.info_rx_sz += pktlen;
+#endif /* DMAMAP_STATS */
+		pktlen = PKTLEN(dhd->osh, p);
+
+		/* Common msg header */
+		infobuf_post->cmn_hdr.msg_type = MSG_TYPE_INFO_BUF_POST;
+		infobuf_post->cmn_hdr.if_id = 0;
+		infobuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+		infobuf_post->cmn_hdr.flags = ring->current_phase;
+		ring->seqnum++;
+
+		pktid = DHD_NATIVE_TO_PKTID(dhd, dhd->prot->pktid_ctrl_map, p, pa,
+			pktlen, DMA_RX, NULL, ring->dma_buf.secdma, PKTTYPE_INFO_RX);
+
+#if defined(DHD_PCIE_PKTID)
+		if (pktid == DHD_PKTID_INVALID) {
+			DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, 0);
+			PKTFREE_CTRLBUF(dhd->osh, p, FALSE);
+			DHD_ERROR_RLMT(("%s: Pktid pool depleted.\n", __FUNCTION__));
+			break;
+		}
+#endif /* DHD_PCIE_PKTID */
+
+		infobuf_post->host_buf_len = htol16((uint16)pktlen);
+		infobuf_post->host_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
+		infobuf_post->host_buf_addr.low_addr = htol32(PHYSADDRLO(pa));
+
+#ifdef DHD_PKTID_AUDIT_RING
+		DHD_PKTID_AUDIT(dhd, prot->pktid_ctrl_map, pktid, DHD_DUPLICATE_ALLOC);
+#endif /* DHD_PKTID_AUDIT_RING */
+
+		DHD_INFO(("ID %d, low_addr 0x%08x, high_addr 0x%08x\n",
+			infobuf_post->cmn_hdr.request_id,  infobuf_post->host_buf_addr.low_addr,
+			infobuf_post->host_buf_addr.high_addr));
+
+		infobuf_post->cmn_hdr.request_id = htol32(pktid);
+		/* Move rxbuf_post_tmp to next item */
+		infobuf_post_tmp = infobuf_post_tmp + ring->item_len;
+#ifdef DHD_LBUF_AUDIT
+		PKTAUDIT(dhd->osh, p);
+#endif
+	}
+
+	if (i < alloced) {
+		if (ring->wr < (alloced - i))
+			ring->wr = ring->max_items - (alloced - i);
+		else
+			ring->wr -= (alloced - i);
+
+		alloced = i;
+		if (alloced && ring->wr == 0) {
+			DHD_INFO(("%s: flipping the phase now\n", ring->name));
+			ring->current_phase = ring->current_phase ?
+				0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+		}
+	}
+
+	/* Update the write pointer in TCM & ring bell */
+	if (alloced > 0) {
+		if (ring == dhd->prot->h2dring_info_subn) {
+			prot->infobufpost += alloced;
+		}
+		dhd_prot_ring_write_complete(dhd, ring, msg_start, alloced);
+	}
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return alloced;
+} /* dhd_prot_infobufpost */
+#endif /* !BCM_ROUTER_DHD */
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+static int
+alloc_ioctl_return_buffer(dhd_pub_t *dhd, dhd_dma_buf_t *retbuf)
+{
+	int err;
+	memset(retbuf, 0, sizeof(dhd_dma_buf_t));
+
+	if ((err = dhd_dma_buf_alloc(dhd, retbuf, IOCT_RETBUF_SIZE)) != BCME_OK) {
+		DHD_ERROR(("%s: dhd_dma_buf_alloc err %d\n", __FUNCTION__, err));
+		ASSERT(0);
+		return BCME_NOMEM;
+	}
+
+	return BCME_OK;
+}
+
+static void
+free_ioctl_return_buffer(dhd_pub_t *dhd, dhd_dma_buf_t *retbuf)
+{
+	/* retbuf (declared on stack) not fully populated ...  */
+	if (retbuf->va) {
+		uint32 dma_pad;
+		dma_pad = (IOCT_RETBUF_SIZE % DHD_DMA_PAD) ? DHD_DMA_PAD : 0;
+		retbuf->len = IOCT_RETBUF_SIZE;
+		retbuf->_alloced = retbuf->len + dma_pad;
+	}
+
+	dhd_dma_buf_free(dhd, retbuf);
+	return;
+}
+#endif /* IOCTLRESP_USE_CONSTMEM */
+
+static int
+dhd_prot_rxbufpost_ctrl(dhd_pub_t *dhd, uint8 msg_type)
+{
+	void *p;
+	uint16 pktsz;
+	ioctl_resp_evt_buf_post_msg_t *rxbuf_post;
+	dmaaddr_t pa;
+	uint32 pktlen;
+	dhd_prot_t *prot = dhd->prot;
+	uint16 alloced = 0;
+	unsigned long flags;
+	dhd_dma_buf_t retbuf;
+	void *dmah = NULL;
+	uint32 pktid;
+	void *map_handle;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+	bool non_ioctl_resp_buf = 0;
+	dhd_pkttype_t buf_type;
+
+	if (dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s: bus is already down.\n", __FUNCTION__));
+		return -1;
+	}
+	memset(&retbuf, 0, sizeof(dhd_dma_buf_t));
+
+	if (msg_type == MSG_TYPE_IOCTLRESP_BUF_POST)
+		buf_type = PKTTYPE_IOCTL_RX;
+	else if (msg_type == MSG_TYPE_EVENT_BUF_POST)
+		buf_type = PKTTYPE_EVENT_RX;
+	else if (msg_type == MSG_TYPE_TIMSTAMP_BUFPOST)
+		buf_type = PKTTYPE_TSBUF_RX;
+	else {
+		DHD_ERROR(("invalid message type to be posted to Ctrl ring %d\n", msg_type));
+		/* XXX: may be add an assert */
+		return -1;
+	}
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK) {
+		 return BCME_ERROR;
+	}
+#endif /* PCIE_INB_DW */
+
+	if ((msg_type == MSG_TYPE_EVENT_BUF_POST) || (msg_type == MSG_TYPE_TIMSTAMP_BUFPOST))
+		non_ioctl_resp_buf = TRUE;
+	else
+		non_ioctl_resp_buf = FALSE;
+
+	if (non_ioctl_resp_buf) {
+		/* Allocate packet for not ioctl resp buffer post */
+		pktsz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
+	} else {
+		/* Allocate packet for ctrl/ioctl buffer post */
+		pktsz = DHD_FLOWRING_IOCTL_BUFPOST_PKTSZ;
+	}
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+	if (!non_ioctl_resp_buf) {
+		if (alloc_ioctl_return_buffer(dhd, &retbuf) != BCME_OK) {
+			DHD_ERROR(("Could not allocate IOCTL response buffer\n"));
+			goto fail;
+		}
+		ASSERT(retbuf.len == IOCT_RETBUF_SIZE);
+		p = retbuf.va;
+		pktlen = retbuf.len;
+		pa = retbuf.pa;
+		dmah = retbuf.dmah;
+	} else
+#endif /* IOCTLRESP_USE_CONSTMEM */
+	{
+#ifdef DHD_USE_STATIC_CTRLBUF
+		p = PKTGET_STATIC(dhd->osh, pktsz, FALSE);
+#else
+		p = PKTGET(dhd->osh, pktsz, FALSE);
+#endif /* DHD_USE_STATIC_CTRLBUF */
+		if (p == NULL) {
+			DHD_ERROR(("%s:%d: PKTGET for %s buf failed\n",
+				__FUNCTION__, __LINE__, non_ioctl_resp_buf ?
+				"EVENT" : "IOCTL RESP"));
+			dhd->rx_pktgetfail++;
+			goto fail;
+		}
+
+		pktlen = PKTLEN(dhd->osh, p);
+		pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen, DMA_RX, p, 0);
+
+		if (PHYSADDRISZERO(pa)) {
+			DHD_ERROR(("Invalid physaddr 0\n"));
+			ASSERT(0);
+			goto free_pkt_return;
+		}
+
+#ifdef DMAMAP_STATS
+		switch (buf_type) {
+#ifndef IOCTLRESP_USE_CONSTMEM
+			case PKTTYPE_IOCTL_RX:
+				dhd->dma_stats.ioctl_rx++;
+				dhd->dma_stats.ioctl_rx_sz += pktlen;
+				break;
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+			case PKTTYPE_EVENT_RX:
+				dhd->dma_stats.event_rx++;
+				dhd->dma_stats.event_rx_sz += pktlen;
+				break;
+			case PKTTYPE_TSBUF_RX:
+				dhd->dma_stats.tsbuf_rx++;
+				dhd->dma_stats.tsbuf_rx_sz += pktlen;
+				break;
+			default:
+				break;
+		}
+#endif /* DMAMAP_STATS */
+
+	}
+
+	/* grab the ring lock to allocate pktid and post on ring */
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	rxbuf_post = (ioctl_resp_evt_buf_post_msg_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+
+	if (rxbuf_post == NULL) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		DHD_ERROR(("%s:%d: Ctrl submit Msgbuf Not available to post buffer \n",
+			__FUNCTION__, __LINE__));
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+		if (non_ioctl_resp_buf)
+#endif /* IOCTLRESP_USE_CONSTMEM */
+		{
+			DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
+		}
+		goto free_pkt_return;
+	}
+
+	/* CMN msg header */
+	rxbuf_post->cmn_hdr.msg_type = msg_type;
+
+#ifdef IOCTLRESP_USE_CONSTMEM
+	if (!non_ioctl_resp_buf) {
+		map_handle = dhd->prot->pktid_map_handle_ioctl;
+		pktid = DHD_NATIVE_TO_PKTID(dhd, map_handle, p, pa, pktlen, DMA_RX, dmah,
+			ring->dma_buf.secdma, buf_type);
+	} else
+#endif /* IOCTLRESP_USE_CONSTMEM */
+	{
+		map_handle = dhd->prot->pktid_ctrl_map;
+		pktid = DHD_NATIVE_TO_PKTID(dhd, map_handle,
+			p, pa, pktlen, DMA_RX, dmah, ring->dma_buf.secdma,
+			buf_type);
+	}
+
+	if (pktid == DHD_PKTID_INVALID) {
+		if (ring->wr == 0) {
+			ring->wr = ring->max_items - 1;
+		} else {
+			ring->wr--;
+			if (ring->wr == 0) {
+				ring->current_phase = ring->current_phase ? 0 :
+					BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+			}
+		}
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
+		DHD_ERROR_RLMT(("%s: Pktid pool depleted.\n", __FUNCTION__));
+		goto free_pkt_return;
+	}
+
+#ifdef DHD_PKTID_AUDIT_RING
+	DHD_PKTID_AUDIT(dhd, map_handle, pktid, DHD_DUPLICATE_ALLOC);
+#endif /* DHD_PKTID_AUDIT_RING */
+
+	rxbuf_post->cmn_hdr.request_id = htol32(pktid);
+	rxbuf_post->cmn_hdr.if_id = 0;
+	rxbuf_post->cmn_hdr.epoch =  ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+	rxbuf_post->cmn_hdr.flags = ring->current_phase;
+
+#if defined(DHD_PCIE_PKTID)
+	if (rxbuf_post->cmn_hdr.request_id == DHD_PKTID_INVALID) {
+		if (ring->wr == 0) {
+			ring->wr = ring->max_items - 1;
+		} else {
+			if (ring->wr == 0) {
+				ring->current_phase = ring->current_phase ? 0 :
+					BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+			}
+		}
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef IOCTLRESP_USE_CONSTMEM
+		if (non_ioctl_resp_buf)
+#endif /* IOCTLRESP_USE_CONSTMEM */
+		{
+			DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
+		}
+		goto free_pkt_return;
+	}
+#endif /* DHD_PCIE_PKTID */
+
+#ifndef IOCTLRESP_USE_CONSTMEM
+	rxbuf_post->host_buf_len = htol16((uint16)PKTLEN(dhd->osh, p));
+#else
+	rxbuf_post->host_buf_len = htol16((uint16)pktlen);
+#endif /* IOCTLRESP_USE_CONSTMEM */
+	rxbuf_post->host_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
+	rxbuf_post->host_buf_addr.low_addr  = htol32(PHYSADDRLO(pa));
+#ifdef DHD_LBUF_AUDIT
+	if (non_ioctl_resp_buf)
+		PKTAUDIT(dhd->osh, p);
+#endif
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, rxbuf_post, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return 1;
+
+free_pkt_return:
+	if (!non_ioctl_resp_buf) {
+#ifdef IOCTLRESP_USE_CONSTMEM
+		free_ioctl_return_buffer(dhd, &retbuf);
+#else
+		dhd_prot_packet_free(dhd, p, buf_type, FALSE);
+#endif /* IOCTLRESP_USE_CONSTMEM */
+	} else {
+		dhd_prot_packet_free(dhd, p, buf_type, FALSE);
+	}
+
+fail:
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return -1;
+} /* dhd_prot_rxbufpost_ctrl */
+
+static uint16
+dhd_msgbuf_rxbuf_post_ctrlpath(dhd_pub_t *dhd, uint8 msg_type, uint32 max_to_post)
+{
+	uint32 i = 0;
+	int32 ret_val;
+
+	DHD_INFO(("max to post %d, event %d \n", max_to_post, msg_type));
+
+	if (dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s: bus is already down.\n", __FUNCTION__));
+		return 0;
+	}
+
+	while (i < max_to_post) {
+		ret_val  = dhd_prot_rxbufpost_ctrl(dhd, msg_type);
+		if (ret_val < 0)
+			break;
+		i++;
+	}
+	DHD_INFO(("posted %d buffers of type %d\n", i, msg_type));
+	return (uint16)i;
+}
+
+static void
+dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int max_to_post;
+
+	DHD_INFO(("ioctl resp buf post\n"));
+	max_to_post = prot->max_ioctlrespbufpost - prot->cur_ioctlresp_bufs_posted;
+	if (max_to_post <= 0) {
+		DHD_INFO(("%s: Cannot post more than max IOCTL resp buffers\n",
+			__FUNCTION__));
+		return;
+	}
+	prot->cur_ioctlresp_bufs_posted += dhd_msgbuf_rxbuf_post_ctrlpath(dhd,
+		MSG_TYPE_IOCTLRESP_BUF_POST, max_to_post);
+}
+
+static void
+dhd_msgbuf_rxbuf_post_event_bufs(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int max_to_post;
+
+	max_to_post = prot->max_eventbufpost - prot->cur_event_bufs_posted;
+	if (max_to_post <= 0) {
+		DHD_ERROR(("%s: Cannot post more than max event buffers\n",
+			__FUNCTION__));
+		return;
+	}
+	prot->cur_event_bufs_posted += dhd_msgbuf_rxbuf_post_ctrlpath(dhd,
+		MSG_TYPE_EVENT_BUF_POST, max_to_post);
+}
+
+static int
+dhd_msgbuf_rxbuf_post_ts_bufs(dhd_pub_t *dhd)
+{
+	return 0;
+}
+
+bool
+BCMFASTPATH(dhd_prot_process_msgbuf_infocpl)(dhd_pub_t *dhd, uint bound,
+	uint32 *info_items)
+{
+	dhd_prot_t *prot = dhd->prot;
+	bool more = TRUE;
+	uint n = 0;
+	msgbuf_ring_t *ring = prot->d2hring_info_cpln;
+	unsigned long flags;
+
+	if (ring == NULL)
+		return FALSE;
+	if (ring->inited != TRUE)
+		return FALSE;
+
+	/* Process all the messages - DTOH direction */
+	while (!dhd_is_device_removed(dhd)) {
+		uint8 *msg_addr;
+		uint32 msg_len;
+
+		if (dhd->hang_was_sent) {
+			more = FALSE;
+			break;
+		}
+
+		if (dhd->smmu_fault_occurred) {
+			more = FALSE;
+			break;
+		}
+
+		DHD_RING_LOCK(ring->ring_lock, flags);
+		/* Get the message from ring */
+		/* must pass 'bound - n' rather than just 'bound', because
+		 * within this loop, dhd_prot_get_read_addr is called multiple
+		 * times, so if just 'bound' is passed we may end up reading
+		 * more than 'bound' items, ex:- for a bound of 2048,
+		 * during the first iteration let us say n = 2000, so the loop
+		 * continues and for the second iteration n = 1000 items may be read,
+		 * so the total items read will be 3000 which is > 2048
+		 */
+		msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len, bound - n);
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		if (msg_addr == NULL) {
+			more = FALSE;
+			break;
+		}
+
+		*info_items = msg_len / ring->item_len;
+
+		/* Prefetch data to populate the cache */
+		OSL_PREFETCH(msg_addr);
+
+		if (dhd_prot_process_msgtype(dhd, ring, msg_addr, msg_len) != BCME_OK) {
+			DHD_ERROR(("%s: Error at  process rxpl msgbuf of len %d\n",
+				__FUNCTION__, msg_len));
+		}
+
+		/* Update read pointer */
+		dhd_prot_upd_read_idx(dhd, ring);
+
+		/* After batch processing, check RX bound */
+		n += msg_len / ring->item_len;
+		if (n >= bound) {
+			break;
+		}
+	}
+
+	return more;
+}
+
+#ifdef EWP_EDL
+bool
+dhd_prot_process_msgbuf_edl(dhd_pub_t *dhd, uint32 *evtlog_items)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring = prot->d2hring_edl;
+	unsigned long flags = 0;
+	uint32 items = 0;
+	uint16 rd = 0;
+	uint16 depth = 0;
+
+	if (ring == NULL)
+		return FALSE;
+	if (ring->inited != TRUE)
+		return FALSE;
+	if (ring->item_len == 0) {
+		DHD_ERROR(("%s: Bad ring ! ringidx %d, item_len %d \n",
+			__FUNCTION__, ring->idx, ring->item_len));
+		return FALSE;
+	}
+
+	if (dhd_query_bus_erros(dhd)) {
+		return FALSE;
+	}
+
+	if (dhd->hang_was_sent) {
+		return FALSE;
+	}
+
+	/* in this DPC context just check if wr index has moved
+	 * and schedule deferred context to actually process the
+	 * work items.
+	*/
+
+	/* update the write index */
+	DHD_RING_LOCK(ring->ring_lock, flags);
+	if (dhd->dma_d2h_ring_upd_support) {
+		/* DMAing write/read indices supported */
+		ring->wr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &ring->wr, RING_WR_UPD, ring->idx);
+	}
+	rd = ring->rd;
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+	depth = ring->max_items;
+	/* check for avail space, in number of ring items */
+	items = READ_AVAIL_SPACE(ring->wr, rd, depth);
+	*evtlog_items = items;
+	if (items == 0) {
+		/* no work items in edl ring */
+		return FALSE;
+	}
+	if (items > ring->max_items) {
+		DHD_ERROR(("\r\n======================= \r\n"));
+		DHD_ERROR(("%s(): ring %p, ring->name %s, ring->max_items %d, items %d \r\n",
+			__FUNCTION__, ring, ring->name, ring->max_items, items));
+		DHD_ERROR(("wr: %d,  rd: %d,  depth: %d  \r\n",
+			ring->wr, ring->rd, depth));
+		DHD_ERROR(("dhd->busstate %d bus->wait_for_d3_ack %d \r\n",
+			dhd->busstate, dhd->bus->wait_for_d3_ack));
+		DHD_ERROR(("\r\n======================= \r\n"));
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_RESUMED_ON_INVALID_RING_RDWR;
+			dhd_bus_mem_dump(dhd);
+
+		}
+#endif /* DHD_FW_COREDUMP */
+		dhd_schedule_reset(dhd);
+
+		return FALSE;
+	}
+
+	if (items > D2HRING_EDL_WATERMARK) {
+		DHD_ERROR_RLMT(("%s: WARNING! EDL watermark hit, num items=%u;"
+			" rd=%u; wr=%u; depth=%u;\n", __FUNCTION__, items,
+			ring->rd, ring->wr, depth));
+	}
+
+	dhd_schedule_logtrace(dhd->info);
+
+	return FALSE;
+}
+
+/*
+ * This is called either from work queue context of 'event_log_dispatcher_work' or
+ * from the kthread context of dhd_logtrace_thread
+ */
+int
+dhd_prot_process_edl_complete(dhd_pub_t *dhd, void *evt_decode_data)
+{
+	dhd_prot_t *prot = NULL;
+	msgbuf_ring_t *ring = NULL;
+	int err = 0;
+	unsigned long flags = 0;
+	cmn_msg_hdr_t *msg = NULL;
+	uint8 *msg_addr = NULL;
+	uint32 max_items_to_process = 0, n = 0;
+	uint32 num_items = 0, new_items = 0;
+	uint16 depth = 0;
+	volatile uint16 wr = 0;
+
+	if (!dhd || !dhd->prot)
+		return 0;
+
+	prot = dhd->prot;
+	ring = prot->d2hring_edl;
+
+	if (!ring || !evt_decode_data) {
+		return 0;
+	}
+
+	if (dhd->hang_was_sent) {
+		return FALSE;
+	}
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+	ring->curr_rd = ring->rd;
+	wr = ring->wr;
+	depth = ring->max_items;
+	/* check for avail space, in number of ring items
+	 * Note, that this will only give the # of items
+	 * from rd to wr if wr>=rd, or from rd to ring end
+	 * if wr < rd. So in the latter case strictly speaking
+	 * not all the items are read. But this is OK, because
+	 * these will be processed in the next doorbell as rd
+	 * would have wrapped around. Processing in the next
+	 * doorbell is acceptable since EDL only contains debug data
+	 */
+	num_items = READ_AVAIL_SPACE(wr, ring->rd, depth);
+
+	if (num_items == 0) {
+		/* no work items in edl ring */
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		return 0;
+	}
+
+	DHD_INFO(("%s: EDL work items [%u] available \n",
+			__FUNCTION__, num_items));
+
+	/* if space is available, calculate address to be read */
+	msg_addr = (char*)ring->dma_buf.va + (ring->rd * ring->item_len);
+
+	max_items_to_process = MIN(num_items, DHD_EVENT_LOGTRACE_BOUND);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+	/* Prefetch data to populate the cache */
+	OSL_PREFETCH(msg_addr);
+
+	n = max_items_to_process;
+	while (n > 0) {
+		msg = (cmn_msg_hdr_t *)msg_addr;
+		/* wait for DMA of work item to complete */
+		if ((err = dhd->prot->d2h_edl_sync_cb(dhd, ring, msg)) != BCME_OK) {
+			DHD_ERROR(("%s: Error waiting for DMA to cmpl in EDL ring; err = %d\n",
+				__FUNCTION__, err));
+		}
+		/*
+		 * Update the curr_rd to the current index in the ring, from where
+		 * the work item is fetched. This way if the fetched work item
+		 * fails in LIVELOCK, we can print the exact read index in the ring
+		 * that shows up the corrupted work item.
+		 */
+		if ((ring->curr_rd + 1) >= ring->max_items) {
+			ring->curr_rd = 0;
+		} else {
+			ring->curr_rd += 1;
+		}
+
+		if (err != BCME_OK) {
+			return 0;
+		}
+
+		/* process the edl work item, i.e, the event log */
+		err = dhd_event_logtrace_process_edl(dhd, msg_addr, evt_decode_data);
+
+		/* Dummy sleep so that scheduler kicks in after processing any logprints */
+		OSL_SLEEP(0);
+
+		/* Prefetch data to populate the cache */
+		OSL_PREFETCH(msg_addr + ring->item_len);
+
+		msg_addr += ring->item_len;
+		--n;
+	}
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+	/* update host ring read pointer */
+	if ((ring->rd + max_items_to_process) >= ring->max_items)
+		ring->rd = 0;
+	else
+		ring->rd += max_items_to_process;
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+	/* Now after processing max_items_to_process update dongle rd index.
+	 * The TCM rd index is updated only if bus is not
+	 * in D3. Else, the rd index is updated from resume
+	 * context in - 'dhdpcie_bus_suspend'
+	 */
+	DHD_GENERAL_LOCK(dhd, flags);
+	if (DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhd)) {
+		DHD_INFO(("%s: bus is in suspend(%d) or suspending(0x%x) state!!\n",
+			__FUNCTION__, dhd->busstate, dhd->dhd_bus_busy_state));
+		DHD_GENERAL_UNLOCK(dhd, flags);
+	} else {
+		DHD_GENERAL_UNLOCK(dhd, flags);
+		DHD_EDL_RING_TCM_RD_UPDATE(dhd);
+	}
+
+	/* if num_items > bound, then anyway we will reschedule and
+	 * this function runs again, so that if in between the DPC has
+	 * updated the wr index, then the updated wr is read. But if
+	 * num_items <= bound, and if DPC executes and updates the wr index
+	 * when the above while loop is running, then the updated 'wr' index
+	 * needs to be re-read from here, If we don't do so, then till
+	 * the next time this function is scheduled
+	 * the event logs will not be processed.
+	*/
+	if (num_items <= DHD_EVENT_LOGTRACE_BOUND) {
+		/* read the updated wr index if reqd. and update num_items */
+		DHD_RING_LOCK(ring->ring_lock, flags);
+		if (wr != (volatile uint16)ring->wr) {
+			wr = (volatile uint16)ring->wr;
+			new_items = READ_AVAIL_SPACE(wr, ring->rd, depth);
+			DHD_INFO(("%s: new items [%u] avail in edl\n",
+				__FUNCTION__, new_items));
+			num_items += new_items;
+		}
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+	}
+
+	/* if # of items processed is less than num_items, need to re-schedule
+	* the deferred ctx
+	*/
+	if (max_items_to_process < num_items) {
+		DHD_INFO(("%s: EDL bound hit / new items found, "
+				"items processed=%u; remaining=%u, "
+				"resched deferred ctx...\n",
+				__FUNCTION__, max_items_to_process,
+				num_items - max_items_to_process));
+		return (num_items - max_items_to_process);
+	}
+
+	return 0;
+
+}
+
+void
+dhd_prot_edl_ring_tcm_rd_update(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = NULL;
+	unsigned long flags = 0;
+	msgbuf_ring_t *ring = NULL;
+
+	if (!dhd)
+		return;
+
+	prot = dhd->prot;
+	if (!prot || !prot->d2hring_edl)
+		return;
+
+	ring = prot->d2hring_edl;
+	DHD_RING_LOCK(ring->ring_lock, flags);
+	dhd_prot_upd_read_idx(dhd, ring);
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+}
+#endif /* EWP_EDL */
+
+static void
+dhd_prot_rx_frame(dhd_pub_t *dhd, void *pkt, int ifidx, uint pkt_count)
+{
+
+#ifdef DHD_LB_RXP
+	if (dhd_read_lb_rxp(dhd) == 1) {
+		dhd_lb_rx_pkt_enqueue(dhd, pkt, ifidx);
+		return;
+	}
+#endif /* DHD_LB_RXP */
+	dhd_bus_rx_frame(dhd->bus, pkt, ifidx, pkt_count);
+}
+
+#ifdef DHD_LB_RXP
+static int dhd_prot_lb_rxp_flow_ctrl(dhd_pub_t *dhd)
+{
+	if ((dhd->lb_rxp_stop_thr == 0) || (dhd->lb_rxp_strt_thr == 0)) {
+		/* when either of stop and start thresholds are zero flow ctrl is not enabled */
+		atomic_set(&dhd->lb_rxp_flow_ctrl, FALSE);
+		return FALSE;
+	}
+
+	if ((dhd_lb_rxp_process_qlen(dhd) >= dhd->lb_rxp_stop_thr) &&
+			(!atomic_read(&dhd->lb_rxp_flow_ctrl))) {
+		atomic_set(&dhd->lb_rxp_flow_ctrl, TRUE);
+#ifdef DHD_LB_STATS
+		dhd->lb_rxp_stop_thr_hitcnt++;
+#endif /* DHD_LB_STATS */
+		DHD_INFO(("lb_rxp_process_qlen %d lb_rxp_stop_thr %d\n",
+			dhd_lb_rxp_process_qlen(dhd), dhd->lb_rxp_stop_thr));
+	} else if ((dhd_lb_rxp_process_qlen(dhd) <= dhd->lb_rxp_strt_thr) &&
+			(atomic_read(&dhd->lb_rxp_flow_ctrl))) {
+		atomic_set(&dhd->lb_rxp_flow_ctrl, FALSE);
+#ifdef DHD_LB_STATS
+		dhd->lb_rxp_strt_thr_hitcnt++;
+#endif /* DHD_LB_STATS */
+		DHD_INFO(("lb_rxp_process_qlen %d lb_rxp_strt_thr %d\n",
+			dhd_lb_rxp_process_qlen(dhd), dhd->lb_rxp_strt_thr));
+	}
+
+	return atomic_read(&dhd->lb_rxp_flow_ctrl);
+}
+#endif /* DHD_LB_RXP */
+
+/** called when DHD needs to check for 'receive complete' messages from the dongle */
+bool
+BCMFASTPATH(dhd_prot_process_msgbuf_rxcpl)(dhd_pub_t *dhd, int ringtype, uint32 *rxcpl_items)
+{
+	bool more = FALSE;
+	uint n = 0;
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring;
+	uint16 item_len;
+	host_rxbuf_cmpl_t *msg = NULL;
+	uint8 *msg_addr;
+	uint32 msg_len;
+	uint16 pkt_cnt, pkt_cnt_newidx;
+	unsigned long flags;
+	dmaaddr_t pa;
+	uint32 len;
+	void *dmah;
+	void *secdma;
+	int ifidx = 0, if_newidx = 0;
+	void *pkt, *pktqhead = NULL, *prevpkt = NULL, *pkt_newidx, *nextpkt;
+	uint32 pktid;
+	int i;
+	uint8 sync;
+	unsigned long rx_lock_flags = 0;
+
+#ifdef DHD_LB_RXP
+	/* must be the first check in this function */
+	(void)dhd_prot_lb_rxp_flow_ctrl(dhd);
+#endif /* DHD_LB_RXP */
+
+		ring = &prot->d2hring_rx_cpln;
+	item_len = ring->item_len;
+	while (1) {
+		if (dhd_is_device_removed(dhd))
+			break;
+
+		if (dhd_query_bus_erros(dhd))
+			break;
+
+		if (dhd->hang_was_sent)
+			break;
+
+		if (dhd->smmu_fault_occurred) {
+			break;
+		}
+
+		pkt_cnt = 0;
+		pktqhead = pkt_newidx = NULL;
+		pkt_cnt_newidx = 0;
+
+		DHD_RING_LOCK(ring->ring_lock, flags);
+
+		/* Get the address of the next message to be read from ring */
+		/* must pass 'bound - n' rather than just 'bound', because
+		 * within this loop, dhd_prot_get_read_addr is called multiple
+		 * times, so if just 'bound' is passed we may end up reading
+		 * more than 'bound' items, ex:- for a bound of 2048,
+		 * during the first iteration let us say n = 2000, so the loop
+		 * continues and for the second iteration n = 1000 items may be read,
+		 * so the total items read will be 3000 which is > 2048
+		 */
+		msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len,
+			prot->rx_cpl_post_bound - n);
+		if (msg_addr == NULL) {
+			DHD_RING_UNLOCK(ring->ring_lock, flags);
+			break;
+		}
+
+		*rxcpl_items = msg_len / ring->item_len;
+
+		while (msg_len > 0) {
+			msg = (host_rxbuf_cmpl_t *)msg_addr;
+
+			/* Wait until DMA completes, then fetch msg_type */
+			sync = prot->d2h_sync_cb(dhd, ring, &msg->cmn_hdr, item_len);
+			/*
+			 * Update the curr_rd to the current index in the ring, from where
+			 * the work item is fetched. This way if the fetched work item
+			 * fails in LIVELOCK, we can print the exact read index in the ring
+			 * that shows up the corrupted work item.
+			 */
+			if ((ring->curr_rd + 1) >= ring->max_items) {
+				ring->curr_rd = 0;
+			} else {
+				ring->curr_rd += 1;
+			}
+
+			if (!sync) {
+				msg_len -= item_len;
+				msg_addr += item_len;
+				continue;
+			}
+
+			pktid = ltoh32(msg->cmn_hdr.request_id);
+			if (msg->cmn_hdr.flags & BCMPCIE_CMNHDR_FLAGS_WAKE_PACKET) {
+				DHD_ERROR(("%s:Rx: Wakeup Packet received\n", __FUNCTION__));
+				prot->rx_wakeup_pkt ++;
+			}
+
+#ifdef DHD_PKTID_AUDIT_RING
+			if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, prot->pktid_rx_map, pktid,
+				DHD_DUPLICATE_FREE, msg, D2HRING_RXCMPLT_ITEMSIZE) != BCME_OK) {
+				msg_len -= item_len;
+				msg_addr += item_len;
+				continue;
+			}
+#endif /* DHD_PKTID_AUDIT_RING */
+
+			pkt = DHD_PKTID_TO_NATIVE(dhd, prot->pktid_rx_map, pktid, pa,
+			        len, dmah, secdma, PKTTYPE_DATA_RX);
+			/* Sanity check of shinfo nrfrags */
+			if (!pkt || (dhd_check_shinfo_nrfrags(dhd, pkt, &pa, pktid) != BCME_OK)) {
+				msg_len -= item_len;
+				msg_addr += item_len;
+				continue;
+			}
+			prot->tot_rxcpl++;
+
+			DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0, dmah);
+
+#ifdef DMAMAP_STATS
+			dhd->dma_stats.rxdata--;
+			dhd->dma_stats.rxdata_sz -= len;
+#endif /* DMAMAP_STATS */
+#ifdef DHD_HMAPTEST
+			if ((prot->hmaptest_rx_active == HMAPTEST_D11_RX_POSTED) &&
+				(pktid == prot->hmaptest_rx_pktid)) {
+
+				uchar *ptr;
+				ptr = PKTDATA(dhd->osh, pkt) - (prot->rx_metadata_offset);
+				DMA_UNMAP(dhd->osh, prot->hmap_rx_buf_pa,
+					(uint)prot->hmap_rx_buf_len, DMA_RX, 0, dmah);
+				DHD_ERROR(("hmaptest: d11write rxcpl rcvd sc rxbuf pktid=0x%08x\n",
+					pktid));
+				DHD_ERROR(("hmaptest: d11write rxcpl r0_st=0x%08x r1_stat=0x%08x\n",
+					msg->rx_status_0, msg->rx_status_1));
+				DHD_ERROR(("hmaptest: d11write rxcpl rxbuf va=0x%p pa=0x%08x\n",
+					prot->hmap_rx_buf_va,
+					(uint32)PHYSADDRLO(prot->hmap_rx_buf_pa)));
+				DHD_ERROR(("hmaptest: d11write rxcpl pktdata va=0x%p pa=0x%08x\n",
+					PKTDATA(dhd->osh, pkt), (uint32)PHYSADDRLO(pa)));
+				memcpy(ptr, prot->hmap_rx_buf_va, prot->hmap_rx_buf_len);
+				prot->hmaptest_rx_active = HMAPTEST_D11_RX_INACTIVE;
+				prot->hmap_rx_buf_va = NULL;
+				prot->hmap_rx_buf_len = 0;
+				PHYSADDRHISET(prot->hmap_rx_buf_pa, 0);
+				PHYSADDRLOSET(prot->hmap_rx_buf_pa, 0);
+				prot->hmaptest.in_progress = FALSE;
+			}
+#endif /* DHD_HMAPTEST */
+			DHD_INFO(("id 0x%04x, offset %d, len %d, idx %d, phase 0x%02x, "
+				"pktdata %p, metalen %d\n",
+				ltoh32(msg->cmn_hdr.request_id),
+				ltoh16(msg->data_offset),
+				ltoh16(msg->data_len), msg->cmn_hdr.if_id,
+				msg->cmn_hdr.flags, PKTDATA(dhd->osh, pkt),
+				ltoh16(msg->metadata_len)));
+
+			pkt_cnt++;
+			msg_len -= item_len;
+			msg_addr += item_len;
+
+#if !defined(BCM_ROUTER_DHD)
+#if DHD_DBG_SHOW_METADATA
+			if (prot->metadata_dbg && prot->rx_metadata_offset &&
+			        msg->metadata_len) {
+				uchar *ptr;
+				ptr = PKTDATA(dhd->osh, pkt) - (prot->rx_metadata_offset);
+				/* header followed by data */
+				bcm_print_bytes("rxmetadata", ptr, msg->metadata_len);
+				dhd_prot_print_metadata(dhd, ptr, msg->metadata_len);
+			}
+#endif /* DHD_DBG_SHOW_METADATA */
+#endif /* !BCM_ROUTER_DHD */
+
+#ifdef DHD_LB_RXP
+			/* If flow control is hit, do not enqueue the pkt into napi queue,
+			 * rather enque it in emergency queue and same will be dequeued first
+			 * during PKTGET. This rxcpl packet will be dropped and will not be sent
+			 * to network stack.
+			 */
+			if (atomic_read(&dhd->lb_rxp_flow_ctrl)) {
+				/* Put back rx_metadata_offset which was pulled during rxpost */
+				PKTPUSH(dhd->osh, pkt, prot->rx_metadata_offset);
+				dhd_rx_emerge_enqueue(dhd, pkt);
+				continue;
+			}
+#endif /* DHD_LB_RXP */
+			/* data_offset from buf start */
+			if (ltoh16(msg->data_offset)) {
+				/* data offset given from dongle after split rx */
+				PKTPULL(dhd->osh, pkt, ltoh16(msg->data_offset));
+			}
+			else if (prot->rx_dataoffset) {
+				/* DMA RX offset updated through shared area */
+				PKTPULL(dhd->osh, pkt, prot->rx_dataoffset);
+			}
+			/* Actual length of the packet */
+			PKTSETLEN(dhd->osh, pkt, ltoh16(msg->data_len));
+
+#if defined(WL_MONITOR)
+			if (dhd_monitor_enabled(dhd, ifidx)) {
+				if (msg->flags & BCMPCIE_PKT_FLAGS_FRAME_802_11) {
+					dhd_rx_mon_pkt(dhd, msg, pkt, ifidx);
+					continue;
+				} else {
+					DHD_ERROR(("Received non 802.11 packet, "
+						"when monitor mode is enabled\n"));
+				}
+			} else if (dhd->op_mode == DHD_FLAG_MFG_MODE &&
+					msg->flags & BCMPCIE_PKT_FLAGS_FRAME_802_11) {
+				DHD_TRACE(("Monitor disable, PKTFREE\n"));
+				PKTFREE(dhd->osh, pkt, TRUE);
+				continue;
+#ifdef DBG_PKT_MON
+			} else {
+				if (msg->flags & BCMPCIE_PKT_FLAGS_FRAME_802_11) {
+					DHD_TRACE(("Received 802.11 packet for PKT MON\n"));
+					dhd_80211_mon_pkt(dhd, msg, pkt, ifidx);
+					continue;
+				}
+#endif /* DBG_PKT_MON */
+			}
+#endif /* WL_MONITOR */
+
+			if (!pktqhead) {
+				pktqhead = prevpkt = pkt;
+				ifidx = msg->cmn_hdr.if_id;
+			} else {
+				if (ifidx != msg->cmn_hdr.if_id) {
+					pkt_newidx = pkt;
+					if_newidx = msg->cmn_hdr.if_id;
+					pkt_cnt--;
+					pkt_cnt_newidx = 1;
+					break;
+				} else {
+					PKTSETNEXT(dhd->osh, prevpkt, pkt);
+					prevpkt = pkt;
+				}
+			}
+
+			if (dhd->rx_cpl_lat_capable) {
+				DHD_GENERAL_LOCK(dhd, rx_lock_flags);
+				dhd_update_rxstats(dhd, msg);
+				DHD_GENERAL_UNLOCK(dhd, rx_lock_flags);
+			}
+
+#ifdef DHD_LBUF_AUDIT
+			PKTAUDIT(dhd->osh, pkt);
+#endif
+		}
+
+		/* roll back read pointer for unprocessed message */
+		if (msg_len > 0) {
+			if (ring->rd < msg_len / item_len)
+				ring->rd = ring->max_items - msg_len / item_len;
+			else
+				ring->rd -= msg_len / item_len;
+		}
+
+		/* Update read pointer */
+		dhd_prot_upd_read_idx(dhd, ring);
+
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+		pkt = pktqhead;
+		for (i = 0; pkt && i < pkt_cnt; i++, pkt = nextpkt) {
+			nextpkt = PKTNEXT(dhd->osh, pkt);
+			PKTSETNEXT(dhd->osh, pkt, NULL);
+#ifdef DHD_RX_CHAINING
+			dhd_rxchain_frame(dhd, pkt, ifidx);
+#else
+			dhd_prot_rx_frame(dhd, pkt, ifidx, 1);
+#endif /* DHD_LB_RXP */
+		}
+
+		if (pkt_newidx) {
+#ifdef DHD_RX_CHAINING
+			dhd_rxchain_frame(dhd, pkt_newidx, if_newidx);
+#else
+			dhd_prot_rx_frame(dhd, pkt_newidx, if_newidx, 1);
+#endif /* DHD_LB_RXP */
+		}
+
+		pkt_cnt += pkt_cnt_newidx;
+
+		/* Post another set of rxbufs to the device */
+		dhd_prot_return_rxbuf(dhd, ring, 0, pkt_cnt);
+
+#ifdef DHD_RX_CHAINING
+		dhd_rxchain_commit(dhd);
+#endif
+
+		/* After batch processing, check RX bound */
+		n += pkt_cnt;
+		if (n >= prot->rx_cpl_post_bound) {
+			more = TRUE;
+			break;
+		}
+	}
+
+	/* Call lb_dispatch only if packets are queued */
+	if (n &&
+#ifdef WL_MONITOR
+	!(dhd_monitor_enabled(dhd, ifidx)) &&
+#endif /* WL_MONITOR */
+	TRUE) {
+		DHD_LB_DISPATCH_RX_PROCESS(dhd);
+	}
+
+	return more;
+
+}
+
+/**
+ * Hands transmit packets (with a caller provided flow_id) over to dongle territory (the flow ring)
+ */
+bool
+dhd_prot_update_txflowring(dhd_pub_t *dhd, uint16 flowid, void *msgring)
+{
+	msgbuf_ring_t *ring = (msgbuf_ring_t *)msgring;
+	dhd_prot_t *prot = dhd->prot;
+	bool is_qempty = FALSE;
+	int ret = 0;
+
+	if (ring == NULL) {
+		DHD_ERROR(("%s: NULL txflowring. exiting...\n",  __FUNCTION__));
+		return FALSE;
+	}
+
+	DHD_TRACE(("ringid %d flowid %d write %d read %d \n\n",
+		ring->idx, flowid, ring->wr, ring->rd));
+
+	/* Need more logic here, but for now use it directly */
+	ret = dhd_bus_schedule_queue(dhd->bus, flowid, TRUE,
+		prot->tx_post_bound, &is_qempty); /* from queue to flowring */
+	if (ret == BCME_OK) {
+		return !is_qempty;
+	} else {
+		return FALSE;
+	}
+}
+
+/** called when DHD needs to check for 'transmit complete' messages from the dongle */
+bool
+BCMFASTPATH(dhd_prot_process_msgbuf_txcpl)(dhd_pub_t *dhd, int ringtype, uint32 *txcpl_items)
+{
+	bool more = TRUE;
+	uint n = 0;
+	msgbuf_ring_t *ring;
+	unsigned long flags;
+	dhd_prot_t *prot = dhd->prot;
+
+		ring = &prot->d2hring_tx_cpln;
+
+	/* Process all the messages - DTOH direction */
+	while (!dhd_is_device_removed(dhd)) {
+		uint8 *msg_addr;
+		uint32 msg_len;
+
+		if (dhd_query_bus_erros(dhd)) {
+			more = FALSE;
+			break;
+		}
+
+		if (dhd->hang_was_sent) {
+			more = FALSE;
+			break;
+		}
+
+		if (dhd->smmu_fault_occurred) {
+			more = FALSE;
+			break;
+		}
+
+		DHD_RING_LOCK(ring->ring_lock, flags);
+
+		/* Get the address of the next message to be read from ring */
+		/* must pass 'bound - n' rather than just 'bound', because
+		 * within this loop, dhd_prot_get_read_addr is called multiple
+		 * times, so if just 'bound' is passed we may end up reading
+		 * more than 'bound' items, ex:- for a bound of 2048,
+		 * during the first iteration let us say n = 2000, so the loop
+		 * continues and for the second iteration n = 1000 items may be read,
+		 * so the total items read will be 3000 which is > 2048
+		 */
+		msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len, prot->tx_cpl_bound - n);
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+		if (msg_addr == NULL) {
+			more = FALSE;
+			break;
+		}
+
+		*txcpl_items = msg_len / ring->item_len;
+
+		/* Prefetch data to populate the cache */
+		OSL_PREFETCH(msg_addr);
+
+		if (dhd_prot_process_msgtype(dhd, ring, msg_addr, msg_len) != BCME_OK) {
+			DHD_ERROR(("%s: process %s msg addr %p len %d\n",
+				__FUNCTION__, ring->name, msg_addr, msg_len));
+		}
+
+		/* Write to dngl rd ptr */
+		dhd_prot_upd_read_idx(dhd, ring);
+
+		/* After batch processing, check bound */
+		n += msg_len / ring->item_len;
+
+		if (n >= prot->tx_cpl_bound) {
+			break;
+		}
+	}
+
+	return more;
+}
+
+int
+BCMFASTPATH(dhd_prot_process_trapbuf)(dhd_pub_t *dhd)
+{
+	uint32 data, copylen;
+	dhd_dma_buf_t *trap_addr = &dhd->prot->fw_trap_buf;
+	int ret = 0;
+
+	/* Interrupts can come in before this struct
+	 *  has been initialized.
+	 */
+	if (trap_addr->va == NULL) {
+		DHD_ERROR(("%s: trap_addr->va is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	OSL_CACHE_INV((void *)trap_addr->va, sizeof(uint32));
+	data = *(uint32 *)(trap_addr->va);
+
+	if (data & D2H_DEV_FWHALT) {
+		if (dhd->db7_trap.fw_db7w_trap_inprogress) {
+			DHD_ERROR(("DB7 FW responded 0x%04x\n", data));
+			dhd->db7_trap.fw_db7w_trap_recieved = TRUE;
+		} else {
+			DHD_ERROR(("Firmware trapped and trap_data is 0x%04x\n", data));
+		}
+
+#if defined(PCIE_INB_DW) && defined(PCIE_INB_DSACK_EXT_WAIT)
+		if (INBAND_DW_ENAB(dhd->bus)) {
+			if (data & D2H_DEV_TRAP_DS_ACK_TIMEOUT) {
+				dhd_bus_ds_ack_debug_dump(dhd->bus);
+			}
+		}
+#endif /* PCIE_INB_DW && PCIE_INB_DSACK_EXT_WAIT */
+
+		if (data & D2H_DEV_EXT_TRAP_DATA)
+		{
+			if (dhd->extended_trap_data) {
+				OSL_CACHE_INV((void *)trap_addr->va,
+				       BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+				copylen = MIN(trap_addr->len, BCMPCIE_EXT_TRAP_DATA_MAXLEN);
+				ret = memcpy_s(dhd->extended_trap_data,
+					BCMPCIE_EXT_TRAP_DATA_MAXLEN,
+					(uint32 *)trap_addr->va, copylen);
+				if (ret) {
+					DHD_ERROR(("trap data memcpy failed:%d, destsz:%d, n:%u\n",
+						ret, BCMPCIE_EXT_TRAP_DATA_MAXLEN,
+						copylen));
+					return 0;
+				}
+			}
+			if (dhd->db7_trap.fw_db7w_trap_inprogress == FALSE) {
+				DHD_ERROR(("Extended trap data available\n"));
+			}
+		}
+		return data;
+	}
+	return 0;
+}
+
+/** called when DHD needs to check for 'ioctl complete' messages from the dongle */
+bool
+BCMFASTPATH(dhd_prot_process_ctrlbuf)(dhd_pub_t *dhd, uint32 *ctrlcpl_items)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring = &prot->d2hring_ctrl_cpln;
+	unsigned long flags;
+	uint32 n = 0;
+	bool more = TRUE;
+
+	/* Process all the messages - DTOH direction */
+	while (!dhd_is_device_removed(dhd)) {
+		uint8 *msg_addr;
+		uint32 msg_len;
+
+		if (dhd_query_bus_erros(dhd)) {
+			more = FALSE;
+			break;
+		}
+
+		if (dhd->hang_was_sent) {
+			more = FALSE;
+			break;
+		}
+
+		if (dhd->smmu_fault_occurred) {
+			more = FALSE;
+			break;
+		}
+
+		DHD_RING_LOCK(ring->ring_lock, flags);
+		/* Get the address of the next message to be read from ring */
+		/* must pass 'bound - n' rather than just 'bound', because
+		 * within this loop, dhd_prot_get_read_addr is called multiple
+		 * times, so if just 'bound' is passed we may end up reading
+		 * more than 'bound' items, ex:- for a bound of 2048,
+		 * during the first iteration let us say n = 2000, so the loop
+		 * continues and for the second iteration n = 1000 items may be read,
+		 * so the total items read will be 3000 which is > 2048
+		 */
+		msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len,
+			prot->ctrl_cpl_post_bound - n);
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+		if (msg_addr == NULL) {
+			more = FALSE;
+			break;
+		}
+
+		*ctrlcpl_items = msg_len / ring->item_len;
+
+		/* Prefetch data to populate the cache */
+		OSL_PREFETCH(msg_addr);
+		if (dhd_prot_process_msgtype(dhd, ring, msg_addr, msg_len) != BCME_OK) {
+			DHD_ERROR(("%s: process %s msg addr %p len %d\n",
+				__FUNCTION__, ring->name, msg_addr, msg_len));
+		}
+
+		/* Write to dngl rd ptr */
+		dhd_prot_upd_read_idx(dhd, ring);
+
+		n += msg_len / ring->item_len;
+		if (n >= prot->ctrl_cpl_post_bound) {
+			break;
+		}
+	}
+
+	return more;
+}
+
+/**
+ * Consume messages out of the D2H ring. Ensure that the message's DMA to host
+ * memory has completed, before invoking the message handler via a table lookup
+ * of the cmn_msg_hdr::msg_type.
+ */
+static int
+BCMFASTPATH(dhd_prot_process_msgtype)(dhd_pub_t *dhd, msgbuf_ring_t *ring, uint8 *buf, uint32 len)
+{
+	uint32 buf_len = len;
+	uint16 item_len;
+	uint8 msg_type;
+	cmn_msg_hdr_t *msg = NULL;
+	int ret = BCME_OK;
+
+	ASSERT(ring);
+	item_len = ring->item_len;
+	if (item_len == 0) {
+		DHD_ERROR(("%s: ringidx %d, item_len %d buf_len %d \n",
+			__FUNCTION__, ring->idx, item_len, buf_len));
+		return BCME_ERROR;
+	}
+
+	while (buf_len > 0) {
+		if (dhd->hang_was_sent) {
+			ret = BCME_ERROR;
+			goto done;
+		}
+
+		if (dhd->smmu_fault_occurred) {
+			ret = BCME_ERROR;
+			goto done;
+		}
+
+#ifdef DHD_DEBUG_INVALID_PKTID
+		if (ring == &dhd->prot->d2hring_ctrl_cpln) {
+			(void)memcpy_s(dhd->prot->ctrl_cpl_snapshot, D2HRING_CTRL_CMPLT_ITEMSIZE,
+				buf, D2HRING_CTRL_CMPLT_ITEMSIZE);
+		}
+#endif /* DHD_DEBUG_INVALID_PKTID */
+
+		msg = (cmn_msg_hdr_t *)buf;
+
+		/* Wait until DMA completes, then fetch msg_type */
+		msg_type = dhd->prot->d2h_sync_cb(dhd, ring, msg, item_len);
+
+		/*
+		 * Update the curr_rd to the current index in the ring, from where
+		 * the work item is fetched. This way if the fetched work item
+		 * fails in LIVELOCK, we can print the exact read index in the ring
+		 * that shows up the corrupted work item.
+		 */
+		if ((ring->curr_rd + 1) >= ring->max_items) {
+			ring->curr_rd = 0;
+		} else {
+			ring->curr_rd += 1;
+		}
+
+		/* Prefetch data to populate the cache */
+		OSL_PREFETCH(buf + item_len);
+
+		DHD_INFO(("msg_type %d item_len %d buf_len %d\n",
+			msg_type, item_len, buf_len));
+
+		if (msg_type == MSG_TYPE_LOOPBACK) {
+			bcm_print_bytes("LPBK RESP: ", (uint8 *)msg, item_len);
+			DHD_ERROR((" MSG_TYPE_LOOPBACK, len %d\n", item_len));
+		}
+
+		ASSERT(msg_type < DHD_PROT_FUNCS);
+		if (msg_type >= DHD_PROT_FUNCS) {
+			DHD_ERROR(("%s: msg_type %d, item_len %d buf_len %d\n",
+				__FUNCTION__, msg_type, item_len, buf_len));
+			ret = BCME_ERROR;
+			goto done;
+		}
+
+#if !defined(BCM_ROUTER_DHD)
+		if (msg_type == MSG_TYPE_INFO_BUF_CMPLT) {
+			if (ring == dhd->prot->d2hring_info_cpln) {
+				if (!dhd->prot->infobufpost) {
+					DHD_ERROR(("infobuf posted are zero,"
+						   "but there is a completion\n"));
+					goto done;
+				}
+				dhd->prot->infobufpost--;
+				dhd_prot_infobufpost(dhd, dhd->prot->h2dring_info_subn);
+				dhd_prot_process_infobuf_complete(dhd, buf);
+			}
+		} else
+#endif	/* !defined(BCM_ROUTER_DHD) */
+		if (table_lookup[msg_type]) {
+			table_lookup[msg_type](dhd, buf);
+		}
+
+		if (buf_len < item_len) {
+			ret = BCME_ERROR;
+			goto done;
+		}
+		buf_len = buf_len - item_len;
+		buf = buf + item_len;
+	}
+
+done:
+
+#ifdef DHD_RX_CHAINING
+	dhd_rxchain_commit(dhd);
+#endif
+
+	return ret;
+} /* dhd_prot_process_msgtype */
+
+static void
+dhd_prot_noop(dhd_pub_t *dhd, void *msg)
+{
+	return;
+}
+
+/** called on MSG_TYPE_RING_STATUS message received from dongle */
+static void
+dhd_prot_ringstatus_process(dhd_pub_t *dhd, void *msg)
+{
+	pcie_ring_status_t *ring_status = (pcie_ring_status_t *) msg;
+	uint32 request_id = ltoh32(ring_status->cmn_hdr.request_id);
+	uint16 status = ltoh16(ring_status->compl_hdr.status);
+	uint16 ring_id = ltoh16(ring_status->compl_hdr.flow_ring_id);
+
+	DHD_ERROR(("ring status: request_id %d, status 0x%04x, flow ring %d, write_idx %d \n",
+		request_id, status, ring_id, ltoh16(ring_status->write_idx)));
+
+	if (ltoh16(ring_status->compl_hdr.ring_id) != BCMPCIE_H2D_MSGRING_CONTROL_SUBMIT)
+		return;
+	if (status == BCMPCIE_BAD_PHASE) {
+		/* bad phase report from */
+		/* XXX: if the request is ioctl request finish the ioctl, rather than timing out */
+		DHD_ERROR(("Bad phase\n"));
+	}
+	if (status != BCMPCIE_BADOPTION)
+		return;
+
+	if (request_id == DHD_H2D_DBGRING_REQ_PKTID) {
+		/* XXX: see if the debug ring create is pending */
+		if (dhd->prot->h2dring_info_subn != NULL) {
+			if (dhd->prot->h2dring_info_subn->create_pending == TRUE) {
+				DHD_ERROR(("H2D ring create failed for info ring\n"));
+				dhd->prot->h2dring_info_subn->create_pending = FALSE;
+			}
+			else
+				DHD_ERROR(("ring create ID for a ring, create not pending\n"));
+		} else {
+			DHD_ERROR(("%s info submit ring doesn't exist\n", __FUNCTION__));
+		}
+	}
+	else if (request_id == DHD_D2H_DBGRING_REQ_PKTID) {
+		/* XXX: see if the debug ring create is pending */
+		if (dhd->prot->d2hring_info_cpln != NULL) {
+			if (dhd->prot->d2hring_info_cpln->create_pending == TRUE) {
+				DHD_ERROR(("D2H ring create failed for info ring\n"));
+				dhd->prot->d2hring_info_cpln->create_pending = FALSE;
+			}
+			else
+				DHD_ERROR_MEM(("ring create ID for info ring, "
+					"create not pending\n"));
+		} else {
+			DHD_ERROR(("%s info cpl ring doesn't exist\n", __FUNCTION__));
+		}
+	}
+	else {
+		DHD_ERROR(("don;t know how to pair with original request\n"));
+	}
+	/* How do we track this to pair it with ??? */
+	return;
+}
+
+/** called on MSG_TYPE_GEN_STATUS ('general status') message received from dongle */
+static void
+dhd_prot_genstatus_process(dhd_pub_t *dhd, void *msg)
+{
+	pcie_gen_status_t *gen_status = (pcie_gen_status_t *)msg;
+	DHD_ERROR(("ERROR: gen status: request_id %d, STATUS 0x%04x, flow ring %d \n",
+		gen_status->cmn_hdr.request_id, gen_status->compl_hdr.status,
+		gen_status->compl_hdr.flow_ring_id));
+
+	/* How do we track this to pair it with ??? */
+	return;
+}
+
+/**
+ * Called on MSG_TYPE_IOCTLPTR_REQ_ACK ('ioctl ack') message received from dongle, meaning that the
+ * dongle received the ioctl message in dongle memory.
+ */
+static void
+dhd_prot_ioctack_process(dhd_pub_t *dhd, void *msg)
+{
+	ioctl_req_ack_msg_t *ioct_ack = (ioctl_req_ack_msg_t *)msg;
+	unsigned long flags;
+#if defined(DHD_PKTID_AUDIT_RING) && !defined(BCM_ROUTER_DHD)
+	uint32 pktid = ltoh32(ioct_ack->cmn_hdr.request_id);
+#endif /* DHD_PKTID_AUDIT_RING && !BCM_ROUTER_DHD */
+
+#if defined(DHD_PKTID_AUDIT_RING) && !defined(BCM_ROUTER_DHD)
+	/* Skip audit for ADHD_IOCTL_REQ_PKTID = 0xFFFE */
+	if (pktid != DHD_IOCTL_REQ_PKTID) {
+#ifndef IOCTLRESP_USE_CONSTMEM
+		if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_ctrl_map, pktid,
+			DHD_TEST_IS_ALLOC, msg, D2HRING_CTRL_CMPLT_ITEMSIZE) != BCME_OK)
+#else
+		if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_map_handle_ioctl, pktid,
+			DHD_TEST_IS_ALLOC, msg, D2HRING_CTRL_CMPLT_ITEMSIZE) != BCME_OK)
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+		{
+			return;
+		}
+	}
+#endif /* DHD_PKTID_AUDIT_RING && !BCM_ROUTER_DHD */
+
+	dhd->prot->ioctl_ack_time = OSL_LOCALTIME_NS();
+
+	DHD_GENERAL_LOCK(dhd, flags);
+	if ((dhd->prot->ioctl_state & MSGBUF_IOCTL_ACK_PENDING) &&
+		(dhd->prot->ioctl_state & MSGBUF_IOCTL_RESP_PENDING)) {
+		dhd->prot->ioctl_state &= ~MSGBUF_IOCTL_ACK_PENDING;
+	} else {
+		DHD_ERROR(("%s: received ioctl ACK with state %02x trans_id = %d\n",
+			__FUNCTION__, dhd->prot->ioctl_state, dhd->prot->ioctl_trans_id));
+		prhex("dhd_prot_ioctack_process:",
+			(uchar *)msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
+	}
+	DHD_GENERAL_UNLOCK(dhd, flags);
+
+	DHD_CTL(("ioctl req ack: request_id %d, status 0x%04x, flow ring %d \n",
+		ioct_ack->cmn_hdr.request_id, ioct_ack->compl_hdr.status,
+		ioct_ack->compl_hdr.flow_ring_id));
+	if (ioct_ack->compl_hdr.status != 0)  {
+		DHD_ERROR(("got an error status for the ioctl request...need to handle that\n"));
+		/* FIXME: should we fail the pending IOCTL compelteion wait process... */
+	}
+#ifdef REPORT_FATAL_TIMEOUTS
+	else {
+		dhd_stop_bus_timer(dhd);
+	}
+#endif /* REPORT_FATAL_TIMEOUTS */
+}
+
+/** called on MSG_TYPE_IOCTL_CMPLT message received from dongle */
+static void
+dhd_prot_ioctcmplt_process(dhd_pub_t *dhd, void *msg)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint32 pkt_id, xt_id;
+	ioctl_comp_resp_msg_t *ioct_resp = (ioctl_comp_resp_msg_t *)msg;
+	void *pkt;
+	unsigned long flags;
+	dhd_dma_buf_t retbuf;
+#ifdef REPORT_FATAL_TIMEOUTS
+	uint16	dhd_xt_id;
+#endif
+	int ret = 0;
+
+	/* Check for ioctl timeout induce flag, which is set by firing
+	 * dhd iovar to induce IOCTL timeout. If flag is set,
+	 * return from here, which results in to IOCTL timeout.
+	 */
+	if (dhd->dhd_induce_error == DHD_INDUCE_IOCTL_TIMEOUT) {
+		DHD_ERROR(("%s: Inducing resumed on timeout\n", __FUNCTION__));
+		return;
+	}
+
+	memset(&retbuf, 0, sizeof(dhd_dma_buf_t));
+
+	pkt_id = ltoh32(ioct_resp->cmn_hdr.request_id);
+
+#if defined(DHD_PKTID_AUDIT_RING) && !defined(BCM_ROUTER_DHD)
+#ifndef IOCTLRESP_USE_CONSTMEM
+	if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, prot->pktid_ctrl_map, pkt_id,
+		DHD_DUPLICATE_FREE, msg, D2HRING_CTRL_CMPLT_ITEMSIZE) != BCME_OK)
+#else
+	if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, prot->pktid_map_handle_ioctl, pkt_id,
+		DHD_DUPLICATE_FREE, msg, D2HRING_CTRL_CMPLT_ITEMSIZE) != BCME_OK)
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+	{
+		return;
+	}
+#endif /* DHD_PKTID_AUDIT_RING && !BCM_ROUTER_DHD */
+
+	DHD_GENERAL_LOCK(dhd, flags);
+	if ((prot->ioctl_state & MSGBUF_IOCTL_ACK_PENDING) ||
+		!(prot->ioctl_state & MSGBUF_IOCTL_RESP_PENDING)) {
+		DHD_ERROR(("%s: received ioctl response with state %02x trans_id = %d\n",
+			__FUNCTION__, dhd->prot->ioctl_state, dhd->prot->ioctl_trans_id));
+		prhex("dhd_prot_ioctcmplt_process:",
+			(uchar *)msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
+		DHD_GENERAL_UNLOCK(dhd, flags);
+		return;
+	}
+
+	dhd->prot->ioctl_cmplt_time = OSL_LOCALTIME_NS();
+
+	/* Clear Response pending bit */
+	prot->ioctl_state &= ~MSGBUF_IOCTL_RESP_PENDING;
+	DHD_GENERAL_UNLOCK(dhd, flags);
+
+#ifndef IOCTLRESP_USE_CONSTMEM
+	pkt = dhd_prot_packet_get(dhd, pkt_id, PKTTYPE_IOCTL_RX, TRUE);
+#else
+	dhd_prot_ioctl_ret_buffer_get(dhd, pkt_id, &retbuf);
+	pkt = retbuf.va;
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+	if (!pkt) {
+		DHD_ERROR(("%s: received ioctl response with NULL pkt\n", __FUNCTION__));
+		prhex("dhd_prot_ioctcmplt_process:",
+			(uchar *)msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
+		return;
+	}
+
+	prot->ioctl_resplen = ltoh16(ioct_resp->resp_len);
+	prot->ioctl_status = ltoh16(ioct_resp->compl_hdr.status);
+	xt_id = ltoh16(ioct_resp->trans_id);
+
+	if (xt_id != prot->ioctl_trans_id || prot->curr_ioctl_cmd != ioct_resp->cmd) {
+		DHD_ERROR(("%s: transaction id(%d %d) or cmd(%d %d) mismatch\n",
+			__FUNCTION__, xt_id, prot->ioctl_trans_id,
+			prot->curr_ioctl_cmd, ioct_resp->cmd));
+#ifdef REPORT_FATAL_TIMEOUTS
+		dhd_stop_cmd_timer(dhd);
+#endif /* REPORT_FATAL_TIMEOUTS */
+		dhd_wakeup_ioctl_event(dhd, IOCTL_RETURN_ON_ERROR);
+		dhd_prot_debug_info_print(dhd);
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_TRANS_ID_MISMATCH;
+			dhd_bus_mem_dump(dhd);
+		}
+#else
+		ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+		dhd_schedule_reset(dhd);
+		goto exit;
+	}
+#ifdef REPORT_FATAL_TIMEOUTS
+	dhd_xt_id = dhd_get_request_id(dhd);
+	if (xt_id == dhd_xt_id) {
+		dhd_stop_cmd_timer(dhd);
+	} else {
+		DHD_ERROR(("%s: Cmd timer not stopped received xt_id %d stored xt_id %d",
+			__FUNCTION__, xt_id, dhd_xt_id));
+	}
+#endif /* REPORT_FATAL_TIMEOUTS */
+	DHD_CTL(("IOCTL_COMPLETE: req_id %x transid %d status %x resplen %d\n",
+		pkt_id, xt_id, prot->ioctl_status, prot->ioctl_resplen));
+
+	if (prot->ioctl_resplen > 0) {
+		uint16 copy_len = MIN(prot->ioctl_resplen, prot->retbuf.len);
+#ifndef IOCTLRESP_USE_CONSTMEM
+		ret = memcpy_s(prot->retbuf.va, prot->retbuf.len, PKTDATA(dhd->osh, pkt), copy_len);
+#else
+		ret = memcpy_s(prot->retbuf.va, prot->retbuf.len, pkt, copy_len);
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+		if (ret) {
+			DHD_ERROR(("memcpy failed:%d, destsz:%d, n:%u\n",
+				ret, prot->retbuf.len, copy_len));
+			dhd_wakeup_ioctl_event(dhd, IOCTL_RETURN_ON_ERROR);
+			goto exit;
+		}
+	}
+
+	/* wake up any dhd_os_ioctl_resp_wait() */
+	dhd_wakeup_ioctl_event(dhd, IOCTL_RETURN_ON_SUCCESS);
+
+exit:
+#ifndef IOCTLRESP_USE_CONSTMEM
+	dhd_prot_packet_free(dhd, pkt,
+		PKTTYPE_IOCTL_RX, FALSE);
+#else
+	free_ioctl_return_buffer(dhd, &retbuf);
+#endif /* !IOCTLRESP_USE_CONSTMEM */
+
+	/* Post another ioctl buf to the device */
+	if (prot->cur_ioctlresp_bufs_posted > 0) {
+		prot->cur_ioctlresp_bufs_posted--;
+	}
+
+	dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd);
+}
+
+int
+dhd_prot_check_tx_resource(dhd_pub_t *dhd)
+{
+	return dhd->prot->no_tx_resource;
+}
+
+/** called on MSG_TYPE_TX_STATUS message received from dongle */
+static void
+BCMFASTPATH(dhd_prot_txstatus_process)(dhd_pub_t *dhd, void *msg)
+{
+	dhd_prot_t *prot = dhd->prot;
+	host_txbuf_cmpl_t * txstatus;
+	unsigned long flags;
+	uint32 pktid;
+	void *pkt;
+	dmaaddr_t pa;
+	uint32 len;
+	void *dmah;
+	void *secdma;
+	bool pkt_fate;
+	msgbuf_ring_t *ring = &dhd->prot->d2hring_tx_cpln;
+#if defined(TX_STATUS_LATENCY_STATS)
+	flow_info_t *flow_info;
+	uint64 tx_status_latency;
+#endif /* TX_STATUS_LATENCY_STATS */
+#ifdef AGG_H2D_DB
+	msgbuf_ring_t *flow_ring;
+#endif /* AGG_H2D_DB */
+	flow_ring_node_t *flow_ring_node;
+	uint16 flowid;
+	tx_cpl_info_t *txcpl_info = &dhd->txcpl_info;
+	ts_timestamp_t *ts;
+	txstatus = (host_txbuf_cmpl_t *)msg;
+
+	flowid = txstatus->compl_hdr.flow_ring_id;
+	flow_ring_node = DHD_FLOW_RING(dhd, flowid);
+#ifdef AGG_H2D_DB
+	flow_ring = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid);
+	OSL_ATOMIC_DEC(dhd->osh, &flow_ring->inflight);
+#endif /* AGG_H2D_DB */
+
+	BCM_REFERENCE(flow_ring_node);
+
+#ifdef DEVICE_TX_STUCK_DETECT
+	/**
+	 * Since we got a completion message on this flowid,
+	 * update tx_cmpl time stamp
+	 */
+	flow_ring_node->tx_cmpl = OSL_SYSUPTIME();
+	/* update host copy of rd pointer */
+	ring->curr_rd++;
+	if (ring->curr_rd >= ring->max_items) {
+		ring->curr_rd = 0;
+	}
+#endif /* DEVICE_TX_STUCK_DETECT */
+
+	/* locks required to protect circular buffer accesses */
+	DHD_RING_LOCK(ring->ring_lock, flags);
+	pktid = ltoh32(txstatus->cmn_hdr.request_id);
+
+	if (dhd->pcie_txs_metadata_enable > 1) {
+		/* Return metadata format (little endian):
+		 * |<--- txstatus --->|<- metadatalen ->|
+		 * |____|____|________|________|________|
+		 * |    |    |        |        |> total delay from fetch to report (8-bit 1 = 4ms)
+		 * |    |    |        |> ucode delay from enqueue to completion (8-bit 1 = 4ms)
+		 * |    |    |> 8-bit reserved (pre-filled with original TX status by caller)
+		 * |    |> delay time first fetch to the last fetch (4-bit 1 = 32ms)
+		 * |> fetch count (4-bit)
+		 */
+		DHD_CONS_ONLY(("TX status[%d] = %04x-%04x -> status = %d (%d/%dms + %d/%dms)\n",
+			pktid,
+			ltoh16(txstatus->tx_status_ext), ltoh16(txstatus->tx_status),
+			(txstatus->tx_status & WLFC_CTL_PKTFLAG_MASK),
+			((txstatus->tx_status >> 12) & 0xf),
+			((txstatus->tx_status >> 8) & 0xf) * 32,
+			((txstatus->tx_status_ext & 0xff) * 4),
+			((txstatus->tx_status_ext >> 8) & 0xff) * 4));
+	}
+	pkt_fate = TRUE;
+
+#if defined(DHD_PKTID_AUDIT_RING) && !defined(BCM_ROUTER_DHD)
+	if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_tx_map, pktid,
+			DHD_DUPLICATE_FREE, msg, D2HRING_TXCMPLT_ITEMSIZE) != BCME_OK) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		return;
+	}
+#endif /* DHD_PKTID_AUDIT_RING && !BCM_ROUTER_DHD */
+
+	DHD_INFO(("txstatus for pktid 0x%04x\n", pktid));
+	if (OSL_ATOMIC_DEC_RETURN(dhd->osh, &prot->active_tx_count) < 0) {
+		DHD_ERROR(("Extra packets are freed\n"));
+	}
+	ASSERT(pktid != 0);
+
+#ifdef DHD_HMAPTEST
+
+	if ((dhd->prot->hmaptest_tx_active == HMAPTEST_D11_TX_POSTED) &&
+		(pktid == dhd->prot->hmaptest_tx_pktid)) {
+		DHD_ERROR(("hmaptest: d11read txcpl received sc txbuf pktid=0x%08x\n", pktid));
+		DHD_ERROR(("hmaptest: d11read txcpl txstatus=0x%08x\n", txstatus->tx_status));
+		DHD_ERROR(("hmaptest: d11read txcpl sc txbuf va=0x%p pa=0x%08x\n",
+			dhd->prot->hmap_tx_buf_va, (uint32)PHYSADDRLO(dhd->prot->hmap_tx_buf_pa)));
+		dhd->prot->hmaptest_tx_active = HMAPTEST_D11_TX_INACTIVE;
+		dhd->prot->hmap_tx_buf_va = NULL;
+		dhd->prot->hmap_tx_buf_len = 0;
+		PHYSADDRHISET(dhd->prot->hmap_tx_buf_pa, 0);
+		PHYSADDRLOSET(dhd->prot->hmap_tx_buf_pa, 0);
+		prot->hmaptest.in_progress = FALSE;
+	}
+	/* original skb is kept as it is because its going to be freed  later in this path */
+#endif /* DHD_HMAPTEST */
+
+	pkt = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_tx_map, pktid,
+		pa, len, dmah, secdma, PKTTYPE_DATA_TX);
+	if (!pkt) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		DHD_ERROR(("%s: received txstatus with NULL pkt\n", __FUNCTION__));
+		prhex("dhd_prot_txstatus_process:", (uchar *)msg, D2HRING_TXCMPLT_ITEMSIZE);
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			dhd->pktid_invalid_occured = TRUE;
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
+			dhd_bus_mem_dump(dhd);
+#ifdef OEM_ANDROID
+			dhd->hang_reason = HANG_REASON_PCIE_PKTID_ERROR;
+			dhd_os_send_hang_message(dhd);
+#endif /* OEM_ANDROID */
+		}
+#else
+		ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+		return;
+	}
+
+	if (DHD_PKTID_AVAIL(dhd->prot->pktid_tx_map) == DHD_PKTID_MIN_AVAIL_COUNT) {
+		DHD_ERROR_RLMT(("%s: start tx queue as min pktids are available\n",
+			__FUNCTION__));
+		prot->pktid_txq_stop_cnt--;
+		dhd->prot->no_tx_resource = FALSE;
+		dhd_bus_start_queue(dhd->bus);
+	}
+
+	DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_TX, 0, dmah);
+
+#ifdef TX_STATUS_LATENCY_STATS
+	/* update the tx status latency for flowid */
+	flow_info = &flow_ring_node->flow_info;
+	tx_status_latency = OSL_SYSUPTIME_US() - DHD_PKT_GET_QTIME(pkt);
+	flow_info->cum_tx_status_latency += tx_status_latency;
+	flow_info->num_tx_status++;
+#endif /* TX_STATUS_LATENCY_STATS */
+
+#ifdef DMAMAP_STATS
+	dhd->dma_stats.txdata--;
+	dhd->dma_stats.txdata_sz -= len;
+#endif /* DMAMAP_STATS */
+	pkt_fate = dhd_dbg_process_tx_status(dhd, pkt, pktid,
+		ltoh16(txstatus->compl_hdr.status) & WLFC_CTL_PKTFLAG_MASK);
+#ifdef DHD_PKT_LOGGING
+	if (dhd->d11_tx_status) {
+		uint16 status = ltoh16(txstatus->compl_hdr.status) &
+			WLFC_CTL_PKTFLAG_MASK;
+		dhd_handle_pktdata(dhd, ltoh32(txstatus->cmn_hdr.if_id),
+			pkt, (uint8 *)PKTDATA(dhd->osh, pkt), pktid, len,
+			&status, NULL, NULL, TRUE, FALSE, TRUE);
+	}
+#endif /* DHD_PKT_LOGGING */
+#if defined(BCMPCIE) && (defined(LINUX) || defined(OEM_ANDROID))
+	dhd_txcomplete(dhd, pkt, pkt_fate);
+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
+	dhd_eap_txcomplete(dhd, pkt, pkt_fate, txstatus->cmn_hdr.if_id);
+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
+#endif /* BCMPCIE && (defined(LINUX) || defined(OEM_ANDROID)) */
+
+#if DHD_DBG_SHOW_METADATA
+	if (dhd->prot->metadata_dbg &&
+			dhd->prot->tx_metadata_offset && txstatus->metadata_len) {
+		uchar *ptr;
+		/* The Ethernet header of TX frame was copied and removed.
+		 * Here, move the data pointer forward by Ethernet header size.
+		 */
+		PKTPULL(dhd->osh, pkt, ETHER_HDR_LEN);
+		ptr = PKTDATA(dhd->osh, pkt)  - (dhd->prot->tx_metadata_offset);
+		bcm_print_bytes("txmetadata", ptr, txstatus->metadata_len);
+		dhd_prot_print_metadata(dhd, ptr, txstatus->metadata_len);
+	}
+#endif /* DHD_DBG_SHOW_METADATA */
+
+	/* Store PTM timestamps */
+	ts = (ts_timestamp_t *)&txstatus->ts;
+	bzero(&txcpl_info->tx_history[txcpl_info->txcpl_hist_count],
+		sizeof(tx_cpl_history_t));
+	if (DHD_PTM_CLKID(ts->high)) {
+		txcpl_info->tx_history[txcpl_info->txcpl_hist_count].ptm_high = ts->high;
+		txcpl_info->tx_history[txcpl_info->txcpl_hist_count].ptm_low = ts->low;
+	}
+	/* Tx Latency for successful xmission of non HPP packet */
+	if ((txstatus->tx_status == 0) &&
+		(ring == &dhd->prot->d2hring_tx_cpln)) {
+		txcpl_info->tx_history[txcpl_info->txcpl_hist_count].latency =
+			txstatus->metadata_len & BCMPCIE_TX_PKT_LATENCY_MASK;
+	}
+	txcpl_info->tx_history[txcpl_info->txcpl_hist_count].host_time =
+		(uint32)OSL_SYSUPTIME_US();
+	txcpl_info->tx_history[txcpl_info->txcpl_hist_count].tid =
+		flow_ring_node->flow_info.tid;
+	txcpl_info->tx_history[txcpl_info->txcpl_hist_count].flowid = flowid;
+	txcpl_info->txcpl_hist_count =
+		(txcpl_info->txcpl_hist_count +1) % MAX_TXCPL_HISTORY;
+
+#ifdef DHD_LBUF_AUDIT
+	PKTAUDIT(dhd->osh, pkt);
+#endif
+	DHD_FLOWRING_TXSTATUS_CNT_UPDATE(dhd->bus, txstatus->compl_hdr.flow_ring_id,
+		txstatus->tx_status);
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef DHD_MEM_STATS
+	DHD_MEM_STATS_LOCK(dhd->mem_stats_lock, flags);
+	DHD_INFO(("%s txpath_mem: %llu PKTLEN: %d\n",
+		__FUNCTION__, dhd->txpath_mem, PKTLEN(dhd->osh, pkt)));
+	dhd->txpath_mem -= PKTLEN(dhd->osh, pkt);
+	DHD_MEM_STATS_UNLOCK(dhd->mem_stats_lock, flags);
+#endif /* DHD_MEM_STATS */
+	PKTFREE(dhd->osh, pkt, TRUE);
+
+	return;
+} /* dhd_prot_txstatus_process */
+
+/* FIXME: assuming that it is getting inline data related to the event data */
+/** called on MSG_TYPE_WL_EVENT message received from dongle */
+static void
+dhd_prot_event_process(dhd_pub_t *dhd, void *msg)
+{
+	wlevent_req_msg_t *evnt;
+	uint32 bufid;
+	uint16 buflen;
+	int ifidx = 0;
+	void* pkt;
+	dhd_prot_t *prot = dhd->prot;
+
+	/* Event complete header */
+	evnt = (wlevent_req_msg_t *)msg;
+	bufid = ltoh32(evnt->cmn_hdr.request_id);
+
+#if defined(DHD_PKTID_AUDIT_RING) && !defined(BCM_ROUTER_DHD)
+	if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_ctrl_map, bufid,
+			DHD_DUPLICATE_FREE, msg, D2HRING_CTRL_CMPLT_ITEMSIZE) != BCME_OK) {
+		return;
+	}
+#endif /* DHD_PKTID_AUDIT_RING && !BCM_ROUTER_DHD */
+
+	buflen = ltoh16(evnt->event_data_len);
+
+	ifidx = BCMMSGBUF_API_IFIDX(&evnt->cmn_hdr);
+	if (evnt->cmn_hdr.flags & BCMPCIE_CMNHDR_FLAGS_WAKE_PACKET) {
+		DHD_ERROR(("%s:Event: Wakeup Packet received\n", __FUNCTION__));
+		prot->event_wakeup_pkt ++;
+	}
+	/* FIXME: check the event status */
+
+	/* Post another rxbuf to the device */
+	if (prot->cur_event_bufs_posted)
+		prot->cur_event_bufs_posted--;
+	dhd_msgbuf_rxbuf_post_event_bufs(dhd);
+
+	pkt = dhd_prot_packet_get(dhd, bufid, PKTTYPE_EVENT_RX, TRUE);
+
+	if (!pkt) {
+		DHD_ERROR(("%s: pkt is NULL for pktid %d\n", __FUNCTION__, bufid));
+		return;
+	}
+
+#if !defined(BCM_ROUTER_DHD)
+	/* FIXME: make sure the length is more than dataoffset */
+	/* DMA RX offset updated through shared area */
+	if (dhd->prot->rx_dataoffset)
+		PKTPULL(dhd->osh, pkt, dhd->prot->rx_dataoffset);
+#endif /* !BCM_ROUTER_DHD */
+
+	PKTSETLEN(dhd->osh, pkt, buflen);
+#ifdef DHD_LBUF_AUDIT
+	PKTAUDIT(dhd->osh, pkt);
+#endif
+	dhd_bus_rx_frame(dhd->bus, pkt, ifidx, 1);
+}
+
+#if !defined(BCM_ROUTER_DHD)
+/** called on MSG_TYPE_INFO_BUF_CMPLT message received from dongle */
+static void
+BCMFASTPATH(dhd_prot_process_infobuf_complete)(dhd_pub_t *dhd, void* buf)
+{
+	info_buf_resp_t *resp;
+	uint32 pktid;
+	uint16 buflen;
+	void * pkt;
+
+	resp = (info_buf_resp_t *)buf;
+	pktid = ltoh32(resp->cmn_hdr.request_id);
+	buflen = ltoh16(resp->info_data_len);
+
+#ifdef DHD_PKTID_AUDIT_RING
+	if (DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_ctrl_map, pktid,
+			DHD_DUPLICATE_FREE, buf, D2HRING_INFO_BUFCMPLT_ITEMSIZE) != BCME_OK) {
+			return;
+	}
+#endif /* DHD_PKTID_AUDIT_RING */
+
+	DHD_INFO(("id 0x%04x, len %d, phase 0x%02x, seqnum %d, rx_dataoffset %d\n",
+		pktid, buflen, resp->cmn_hdr.flags, ltoh16(resp->seqnum),
+		dhd->prot->rx_dataoffset));
+
+	if (dhd->debug_buf_dest_support) {
+		if (resp->dest < DEBUG_BUF_DEST_MAX) {
+			dhd->debug_buf_dest_stat[resp->dest]++;
+		}
+	}
+	if (resp->cmn_hdr.flags & BCMPCIE_CMNHDR_FLAGS_WAKE_PACKET) {
+		DHD_ERROR(("%s:Infobuf: Wakeup Packet received\n", __FUNCTION__));
+		dhd->prot->info_wakeup_pkt ++;
+	}
+
+	pkt = dhd_prot_packet_get(dhd, pktid, PKTTYPE_INFO_RX, TRUE);
+	if (!pkt)
+		return;
+
+#if !defined(BCM_ROUTER_DHD)
+	/* FIXME: make sure the length is more than dataoffset */
+	/* DMA RX offset updated through shared area */
+	if (dhd->prot->rx_dataoffset)
+		PKTPULL(dhd->osh, pkt, dhd->prot->rx_dataoffset);
+#endif /* !BCM_ROUTER_DHD */
+
+	PKTSETLEN(dhd->osh, pkt, buflen);
+#ifdef DHD_LBUF_AUDIT
+	PKTAUDIT(dhd->osh, pkt);
+#endif
+	/* info ring "debug" data, which is not a 802.3 frame, is sent/hacked with a
+	 * special ifidx of -1.  This is just internal to dhd to get the data to
+	 * dhd_linux.c:dhd_rx_frame() from here (dhd_prot_infobuf_cmplt_process).
+	 */
+	dhd_bus_rx_frame(dhd->bus, pkt, DHD_DUMMY_INFO_IF /* ifidx HACK */, 1);
+}
+#endif /* !BCM_ROUTER_DHD */
+
+/** called on MSG_TYPE_SNAPSHOT_CMPLT message received from dongle */
+static void
+BCMFASTPATH(dhd_prot_process_snapshot_complete)(dhd_pub_t *dhd, void *buf)
+{
+}
+
+/** Stop protocol: sync w/dongle state. */
+void dhd_prot_stop(dhd_pub_t *dhd)
+{
+	ASSERT(dhd);
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#if defined(NDIS)
+	if (dhd->prot) {
+		DHD_NATIVE_TO_PKTID_RESET(dhd, dhd->prot->pktid_ctrl_map);
+		DHD_NATIVE_TO_PKTID_RESET(dhd, dhd->prot->pktid_rx_map);
+		DHD_NATIVE_TO_PKTID_RESET(dhd, dhd->prot->pktid_tx_map);
+#if defined(IOCTLRESP_USE_CONSTMEM)
+		DHD_NATIVE_TO_PKTID_RESET_IOCTL(dhd, dhd->prot->pktid_map_handle_ioctl);
+#endif /* DHD_PCIE_PKTID */
+	}
+#endif /* NDIS */
+}
+
+/* Add any protocol-specific data header.
+ * Caller must reserve prot_hdrlen prepend space.
+ */
+void
+BCMFASTPATH(dhd_prot_hdrpush)(dhd_pub_t *dhd, int ifidx, void *PKTBUF)
+{
+	return;
+}
+
+uint
+dhd_prot_hdrlen(dhd_pub_t *dhd, void *PKTBUF)
+{
+	return 0;
+}
+
+#define PKTBUF pktbuf
+
+/**
+ * Called when a tx ethernet packet has been dequeued from a flow queue, and has to be inserted in
+ * the corresponding flow ring.
+ */
+int
+BCMFASTPATH(dhd_prot_txdata)(dhd_pub_t *dhd, void *PKTBUF, uint8 ifidx)
+{
+	unsigned long flags;
+	dhd_prot_t *prot = dhd->prot;
+	host_txbuf_post_t *txdesc = NULL;
+	dmaaddr_t pa, meta_pa;
+	uint8 *pktdata;
+	uint32 pktlen;
+	uint32 pktid;
+	uint8	prio;
+	uint16 flowid = 0;
+	uint16 alloced = 0;
+	uint16	headroom;
+	msgbuf_ring_t *ring;
+	flow_ring_table_t *flow_ring_table;
+	flow_ring_node_t *flow_ring_node;
+	void *big_pktbuf = NULL;
+	uint8 dhd_udr = FALSE;
+	uint8 dhd_igmp = FALSE;
+	bool host_sfh_llc_reqd = dhd->host_sfhllc_supported;
+	bool llc_inserted = FALSE;
+
+	BCM_REFERENCE(llc_inserted);
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK) {
+		DHD_ERROR(("failed to increment hostactive_devwake\n"));
+		return BCME_ERROR;
+	}
+#endif /* PCIE_INB_DW */
+
+	if (dhd->flow_ring_table == NULL) {
+		DHD_ERROR(("dhd flow_ring_table is NULL\n"));
+		goto fail;
+	}
+
+#ifdef DHD_PCIE_PKTID
+		if (!DHD_PKTID_AVAIL(dhd->prot->pktid_tx_map)) {
+			if (dhd->prot->pktid_depleted_cnt == DHD_PKTID_DEPLETED_MAX_COUNT) {
+				DHD_ERROR(("%s: stop tx queue as pktid_depleted_cnt maxed\n",
+					__FUNCTION__));
+				prot->pktid_txq_stop_cnt++;
+				dhd_bus_stop_queue(dhd->bus);
+				dhd->prot->no_tx_resource = TRUE;
+			}
+			dhd->prot->pktid_depleted_cnt++;
+			goto fail;
+		} else {
+			dhd->prot->pktid_depleted_cnt = 0;
+		}
+#endif /* DHD_PCIE_PKTID */
+
+	if (dhd->dhd_induce_error == DHD_INDUCE_TX_BIG_PKT) {
+		if ((big_pktbuf = PKTGET(dhd->osh, DHD_FLOWRING_TX_BIG_PKT_SIZE, TRUE)) == NULL) {
+			DHD_ERROR(("%s:%d: PKTGET for txbuf failed\n", __FUNCTION__, __LINE__));
+			goto fail;
+		}
+
+		memset(PKTDATA(dhd->osh, big_pktbuf), 0xff, DHD_FLOWRING_TX_BIG_PKT_SIZE);
+		DHD_ERROR(("PKTBUF len = %d big_pktbuf len = %d\n", PKTLEN(dhd->osh, PKTBUF),
+				PKTLEN(dhd->osh, big_pktbuf)));
+		if (memcpy_s(PKTDATA(dhd->osh, big_pktbuf), DHD_FLOWRING_TX_BIG_PKT_SIZE,
+				PKTDATA(dhd->osh, PKTBUF), PKTLEN(dhd->osh, PKTBUF)) != BCME_OK) {
+			DHD_ERROR(("%s:%d: memcpy_s big_pktbuf failed\n", __FUNCTION__, __LINE__));
+			ASSERT(0);
+		}
+	}
+
+	flowid = DHD_PKT_GET_FLOWID(PKTBUF);
+	flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
+	flow_ring_node = (flow_ring_node_t *)&flow_ring_table[flowid];
+
+	ring = (msgbuf_ring_t *)flow_ring_node->prot_info;
+
+	/*
+	 * XXX:
+	 * JIRA SW4349-436:
+	 * Copying the TX Buffer to an SKB that lives in the DMA Zone
+	 * is done here. Previously this was done from dhd_stat_xmit
+	 * On conditions where the Host is pumping heavy traffic to
+	 * the dongle, we see that the Queue that is backing up the
+	 * flow rings is getting full and holds the precious memory
+	 * from DMA Zone, leading the host to run out of memory in DMA
+	 * Zone. So after this change the back up queue would continue to
+	 * hold the pointers from Network Stack, just before putting
+	 * the PHY ADDR in the flow rings, we'll do the copy.
+	 */
+
+	if (dhd->dhd_induce_error == DHD_INDUCE_TX_BIG_PKT && big_pktbuf) {
+		PKTFREE(dhd->osh, PKTBUF, TRUE);
+		PKTBUF = big_pktbuf;
+	}
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	/* Create a unique 32-bit packet id */
+	pktid = DHD_NATIVE_TO_PKTID_RSV(dhd, dhd->prot->pktid_tx_map,
+		PKTBUF, PKTTYPE_DATA_TX);
+#if defined(DHD_PCIE_PKTID)
+	if (pktid == DHD_PKTID_INVALID) {
+		DHD_ERROR_RLMT(("%s: Pktid pool depleted.\n", __FUNCTION__));
+		/*
+		 * If we return error here, the caller would queue the packet
+		 * again. So we'll just free the skb allocated in DMA Zone.
+		 * Since we have not freed the original SKB yet the caller would
+		 * requeue the same.
+		 */
+		goto err_no_res_pktfree;
+	}
+#endif /* DHD_PCIE_PKTID */
+
+	/* Reserve space in the circular buffer */
+	txdesc = (host_txbuf_post_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+	if (txdesc == NULL) {
+		DHD_INFO(("%s:%d: HTOD Msgbuf Not available TxCount = %d\n",
+			__FUNCTION__, __LINE__, OSL_ATOMIC_READ(dhd->osh, &prot->active_tx_count)));
+		goto err_free_pktid;
+	}
+	txdesc->flags = 0;
+
+	/* Extract the data pointer and length information */
+	pktdata = PKTDATA(dhd->osh, PKTBUF);
+	pktlen  = PKTLEN(dhd->osh, PKTBUF);
+
+	/* TODO: XXX: re-look into dropped packets */
+	DHD_DBG_PKT_MON_TX(dhd, PKTBUF, pktid, FRAME_TYPE_ETHERNET_II, 0);
+
+	dhd_handle_pktdata(dhd, ifidx, PKTBUF, pktdata, pktid,
+		pktlen, NULL, &dhd_udr, &dhd_igmp, TRUE, FALSE, TRUE);
+
+	/* Ethernet header - contains ethertype field
+	* Copy before we cache flush packet using DMA_MAP
+	*/
+	bcopy(pktdata, txdesc->txhdr, ETHER_HDR_LEN);
+
+	{
+		/* Extract the ethernet header and adjust the data pointer and length */
+		pktlen = PKTLEN(dhd->osh, PKTBUF) - ETHER_HDR_LEN;
+		pktdata = PKTPULL(dhd->osh, PKTBUF, ETHER_HDR_LEN);
+	}
+
+	/* Map the data pointer to a DMA-able address */
+	pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, PKTBUF), pktlen, DMA_TX, PKTBUF, 0);
+
+	if (PHYSADDRISZERO(pa)) {
+		DHD_ERROR(("%s: Something really bad, unless 0 is "
+			"a valid phyaddr for pa\n", __FUNCTION__));
+		ASSERT(0);
+		/* XXX if ASSERT() doesn't work like as Android platform,
+		 * try to requeue the packet to the backup queue.
+		 */
+		goto err_rollback_idx;
+	}
+
+#ifdef DMAMAP_STATS
+	dhd->dma_stats.txdata++;
+	dhd->dma_stats.txdata_sz += pktlen;
+#endif /* DMAMAP_STATS */
+	/* No need to lock. Save the rest of the packet's metadata */
+	DHD_NATIVE_TO_PKTID_SAVE(dhd, dhd->prot->pktid_tx_map, PKTBUF, pktid,
+	    pa, pktlen, DMA_TX, NULL, ring->dma_buf.secdma, PKTTYPE_DATA_TX);
+
+#ifdef TXP_FLUSH_NITEMS
+	if (ring->pend_items_count == 0)
+		ring->start_addr = (void *)txdesc;
+	ring->pend_items_count++;
+#endif
+#ifdef DHD_HMAPTEST
+	if (dhd->prot->hmaptest_tx_active == HMAPTEST_D11_TX_ACTIVE) {
+		/* scratch area */
+		dhd->prot->hmap_tx_buf_va = (char *)dhd->prot->hmaptest.mem.va
+			+ dhd->prot->hmaptest.offset;
+		/* replace pa with our pa for txbuf post only */
+		dhd->prot->hmap_tx_buf_len = pktlen;
+		if ((dhd->prot->hmap_tx_buf_va + dhd->prot->hmap_tx_buf_len) >
+			((char *)dhd->prot->hmaptest.mem.va + dhd->prot->hmaptest.mem.len)) {
+			DHD_ERROR(("hmaptest: ERROR Txpost outside HMAPTEST buffer\n"));
+			DHD_ERROR(("hmaptest: NOT Replacing Rx Buffer\n"));
+			dhd->prot->hmaptest_tx_active = HMAPTEST_D11_TX_INACTIVE;
+			dhd->prot->hmaptest.in_progress = FALSE;
+		} else {
+			int ret = 0;
+			/* copy pktdata to our va */
+			ret = memcpy_s(dhd->prot->hmap_tx_buf_va, dhd->prot->hmap_tx_buf_len,
+				PKTDATA(dhd->osh, PKTBUF), pktlen);
+			if (ret) {
+				DHD_ERROR(("memcpy hmap_tx_buf_va failed:%d, destsz:%d, n:%d\n",
+					ret, dhd->prot->hmap_tx_buf_len, pktlen));
+				ASSERT(0);
+				goto err_rollback_idx;
+			}
+			pa = DMA_MAP(dhd->osh, dhd->prot->hmap_tx_buf_va,
+				dhd->prot->hmap_tx_buf_len, DMA_TX, PKTBUF, 0);
+
+			dhd->prot->hmap_tx_buf_pa = pa;
+			/* store pktid for later mapping in txcpl */
+			dhd->prot->hmaptest_tx_pktid = pktid;
+			dhd->prot->hmaptest_tx_active = HMAPTEST_D11_TX_POSTED;
+			DHD_ERROR(("hmaptest: d11read txpost scratch txbuf pktid=0x%08x\n", pktid));
+			DHD_ERROR(("hmaptest: d11read txpost txbuf va=0x%p pa.lo=0x%08x len=%d\n",
+				dhd->prot->hmap_tx_buf_va, (uint32)PHYSADDRLO(pa), pktlen));
+		}
+	}
+#endif /* DHD_HMAPTEST */
+
+	/* Form the Tx descriptor message buffer */
+
+	/* Common message hdr */
+	txdesc->cmn_hdr.msg_type = MSG_TYPE_TX_POST;
+	txdesc->cmn_hdr.if_id = ifidx;
+	txdesc->cmn_hdr.flags = ring->current_phase;
+
+	txdesc->flags |= BCMPCIE_PKT_FLAGS_FRAME_802_3;
+	prio = (uint8)PKTPRIO(PKTBUF);
+
+#ifdef EXT_STA
+	txdesc->flags &= ~BCMPCIE_PKT_FLAGS_FRAME_EXEMPT_MASK <<
+		BCMPCIE_PKT_FLAGS_FRAME_EXEMPT_SHIFT;
+	txdesc->flags |= (WLPKTFLAG_EXEMPT_GET(WLPKTTAG(PKTBUF)) &
+		BCMPCIE_PKT_FLAGS_FRAME_EXEMPT_MASK)
+		<< BCMPCIE_PKT_FLAGS_FRAME_EXEMPT_SHIFT;
+#endif
+
+	txdesc->flags |= (prio & 0x7) << BCMPCIE_PKT_FLAGS_PRIO_SHIFT;
+	txdesc->seg_cnt = 1;
+
+	txdesc->data_len = htol16((uint16) pktlen);
+	txdesc->data_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
+	txdesc->data_buf_addr.low_addr  = htol32(PHYSADDRLO(pa));
+
+	if (!host_sfh_llc_reqd)
+	{
+		/* Move data pointer to keep ether header in local PKTBUF for later reference */
+		PKTPUSH(dhd->osh, PKTBUF, ETHER_HDR_LEN);
+	}
+
+	txdesc->ext_flags = 0;
+
+#ifdef DHD_SBN
+	if (dhd_udr) {
+		txdesc->ext_flags |= BCMPCIE_PKT_FLAGS_FRAME_UDR;
+	}
+#endif /* DHD_SBN */
+
+	if (dhd_igmp) {
+		txdesc->ext_flags |= BCMPCIE_PKT_FLAGS_IGMP;
+	}
+
+#ifdef DHD_TX_PROFILE
+	if (!llc_inserted &&
+		dhd->tx_profile_enab && dhd->num_profiles > 0)
+	{
+		uint8 offset;
+
+		for (offset = 0; offset < dhd->num_profiles; offset++) {
+			if (dhd_protocol_matches_profile((uint8 *)PKTDATA(dhd->osh, PKTBUF),
+				PKTLEN(dhd->osh, PKTBUF), &(dhd->protocol_filters[offset]),
+				host_sfh_llc_reqd)) {
+				/* mask so other reserved bits are not modified. */
+				txdesc->rate |=
+					(((uint8)dhd->protocol_filters[offset].profile_index) &
+					BCMPCIE_TXPOST_RATE_PROFILE_IDX_MASK);
+
+				/* so we can use the rate field for our purposes */
+				txdesc->rate |= BCMPCIE_TXPOST_RATE_EXT_USAGE;
+
+				break;
+			}
+		}
+	}
+#endif /* defined(DHD_TX_PROFILE) */
+
+	/* Handle Tx metadata */
+	headroom = (uint16)PKTHEADROOM(dhd->osh, PKTBUF);
+	if (prot->tx_metadata_offset && (headroom < prot->tx_metadata_offset))
+		DHD_ERROR(("No headroom for Metadata tx %d %d\n",
+		prot->tx_metadata_offset, headroom));
+
+	if (prot->tx_metadata_offset && (headroom >= prot->tx_metadata_offset)) {
+		DHD_TRACE(("Metadata in tx %d\n", prot->tx_metadata_offset));
+
+		/* Adjust the data pointer to account for meta data in DMA_MAP */
+		PKTPUSH(dhd->osh, PKTBUF, prot->tx_metadata_offset);
+
+		meta_pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, PKTBUF),
+			prot->tx_metadata_offset, DMA_RX, PKTBUF, 0);
+
+		if (PHYSADDRISZERO(meta_pa)) {
+			/* Unmap the data pointer to a DMA-able address */
+			DMA_UNMAP(dhd->osh, pa, pktlen, DMA_TX, 0, DHD_DMAH_NULL);
+#ifdef TXP_FLUSH_NITEMS
+			/* update pend_items_count */
+			ring->pend_items_count--;
+#endif /* TXP_FLUSH_NITEMS */
+
+			DHD_ERROR(("%s: Something really bad, unless 0 is "
+				"a valid phyaddr for meta_pa\n", __FUNCTION__));
+			ASSERT(0);
+			/* XXX if ASSERT() doesn't work like as Android platform,
+			 * try to requeue the packet to the backup queue.
+			 */
+			goto err_rollback_idx;
+		}
+
+		/* Adjust the data pointer back to original value */
+		PKTPULL(dhd->osh, PKTBUF, prot->tx_metadata_offset);
+
+		txdesc->metadata_buf_len = prot->tx_metadata_offset;
+		txdesc->metadata_buf_addr.high_addr = htol32(PHYSADDRHI(meta_pa));
+		txdesc->metadata_buf_addr.low_addr = htol32(PHYSADDRLO(meta_pa));
+	} else {
+		if (1) {
+			txdesc->metadata_buf_len = htol16(0);
+			txdesc->metadata_buf_addr.high_addr = 0;
+			txdesc->metadata_buf_addr.low_addr = 0;
+		}
+	}
+
+#ifdef AGG_H2D_DB
+	OSL_ATOMIC_INC(dhd->osh, &ring->inflight);
+#endif /* AGG_H2D_DB */
+
+#ifdef DHD_PKTID_AUDIT_RING
+	DHD_PKTID_AUDIT(dhd, prot->pktid_tx_map, pktid, DHD_DUPLICATE_ALLOC);
+#endif /* DHD_PKTID_AUDIT_RING */
+
+	txdesc->cmn_hdr.request_id = htol32(pktid);
+
+	DHD_TRACE(("txpost: data_len %d, pktid 0x%04x\n", txdesc->data_len,
+		txdesc->cmn_hdr.request_id));
+
+#ifdef DHD_LBUF_AUDIT
+	PKTAUDIT(dhd->osh, PKTBUF);
+#endif
+
+	/* Update the write pointer in TCM & ring bell */
+#if defined(TXP_FLUSH_NITEMS)
+	{
+		if ((ring->pend_items_count == prot->txp_threshold) ||
+				((uint8 *) txdesc == (uint8 *) DHD_RING_END_VA(ring))) {
+#ifdef AGG_H2D_DB
+			if (agg_h2d_db_enab) {
+				dhd_prot_txdata_aggr_db_write_flush(dhd, flowid);
+				if ((uint8 *) txdesc == (uint8 *) DHD_RING_END_VA(ring)) {
+					dhd_prot_aggregate_db_ring_door_bell(dhd, flowid, TRUE);
+				}
+			} else
+#endif /* AGG_H2D_DB */
+			{
+				dhd_prot_txdata_write_flush(dhd, flowid);
+			}
+
+		}
+	}
+#else
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, txdesc, 1);
+#endif /* TXP_FLUSH_NITEMS */
+
+#ifdef TX_STATUS_LATENCY_STATS
+	/* set the time when pkt is queued to flowring */
+	DHD_PKT_SET_QTIME(PKTBUF, OSL_SYSUPTIME_US());
+#endif /* TX_STATUS_LATENCY_STATS */
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+	OSL_ATOMIC_INC(dhd->osh, &prot->active_tx_count);
+
+	/*
+	 * Take a wake lock, do not sleep if we have atleast one packet
+	 * to finish.
+	 */
+	DHD_TXFL_WAKE_LOCK_TIMEOUT(dhd, MAX_TX_TIMEOUT);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+#ifdef TX_STATUS_LATENCY_STATS
+	flow_ring_node->flow_info.num_tx_pkts++;
+#endif /* TX_STATUS_LATENCY_STATS */
+	return BCME_OK;
+
+err_rollback_idx:
+	/* roll back write pointer for unprocessed message */
+	if (ring->wr == 0) {
+		ring->wr = ring->max_items - 1;
+	} else {
+		ring->wr--;
+		if (ring->wr == 0) {
+			DHD_INFO(("%s: flipping the phase now\n", ring->name));
+			ring->current_phase = ring->current_phase ?
+				0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+		}
+	}
+
+err_free_pktid:
+#if defined(DHD_PCIE_PKTID)
+	{
+		void *dmah;
+		void *secdma;
+		/* Free up the PKTID. physaddr and pktlen will be garbage. */
+		DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_tx_map, pktid,
+			pa, pktlen, dmah, secdma, PKTTYPE_NO_CHECK);
+	}
+
+err_no_res_pktfree:
+#endif /* DHD_PCIE_PKTID */
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+fail:
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return BCME_NORESOURCE;
+} /* dhd_prot_txdata */
+
+#ifdef AGG_H2D_DB
+static void
+dhd_prot_txdata_aggr_db_write_flush(dhd_pub_t *dhd, uint16 flowid)
+{
+	flow_ring_table_t *flow_ring_table;
+	flow_ring_node_t *flow_ring_node;
+	msgbuf_ring_t *ring;
+
+	if (dhd->flow_ring_table == NULL) {
+		return;
+	}
+
+	flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
+	flow_ring_node = (flow_ring_node_t *)&flow_ring_table[flowid];
+	ring = (msgbuf_ring_t *)flow_ring_node->prot_info;
+
+	if (ring->pend_items_count) {
+		dhd_prot_agg_db_ring_write(dhd, ring, ring->start_addr,
+				ring->pend_items_count);
+		ring->pend_items_count = 0;
+		ring->start_addr = NULL;
+	}
+
+}
+#endif /* AGG_H2D_DB */
+
+/* called with a ring_lock */
+/** optimization to write "n" tx items at a time to ring */
+void
+BCMFASTPATH(dhd_prot_txdata_write_flush)(dhd_pub_t *dhd, uint16 flowid)
+{
+#ifdef TXP_FLUSH_NITEMS
+	flow_ring_table_t *flow_ring_table;
+	flow_ring_node_t *flow_ring_node;
+	msgbuf_ring_t *ring;
+
+	if (dhd->flow_ring_table == NULL) {
+		return;
+	}
+
+	flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
+	flow_ring_node = (flow_ring_node_t *)&flow_ring_table[flowid];
+	ring = (msgbuf_ring_t *)flow_ring_node->prot_info;
+
+	if (ring->pend_items_count) {
+		/* update ring's WR index and ring doorbell to dongle */
+		dhd_prot_ring_write_complete(dhd, ring, ring->start_addr,
+			ring->pend_items_count);
+		ring->pend_items_count = 0;
+		ring->start_addr = NULL;
+		dhd->prot->tx_h2d_db_cnt++;
+	}
+#endif /* TXP_FLUSH_NITEMS */
+}
+
+#undef PKTBUF	/* Only defined in the above routine */
+
+int
+BCMFASTPATH(dhd_prot_hdrpull)(dhd_pub_t *dhd, int *ifidx, void *pkt, uchar *buf, uint *len)
+{
+	return 0;
+}
+
+/** post a set of receive buffers to the dongle */
+static void
+BCMFASTPATH(dhd_prot_return_rxbuf)(dhd_pub_t *dhd, msgbuf_ring_t *ring, uint32 pktid,
+	uint32 rxcnt)
+/* XXX function name could be more descriptive, eg dhd_prot_post_rxbufs */
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	if (prot->rxbufpost >= rxcnt) {
+		prot->rxbufpost -= (uint16)rxcnt;
+	} else {
+		/* XXX: I have seen this assert hitting.
+		 * Will be removed once rootcaused.
+		 */
+		/* ASSERT(0); */
+		prot->rxbufpost = 0;
+	}
+
+	if (prot->rxbufpost <= (prot->max_rxbufpost - prot->rx_bufpost_threshold)) {
+		dhd_msgbuf_rxbuf_post(dhd, FALSE); /* alloc pkt ids */
+	} else if (dhd->dma_h2d_ring_upd_support && !IDMA_ACTIVE(dhd)) {
+		/* Ring DoorBell after processing the rx packets,
+		 * so that dongle will sync the DMA indices.
+		 */
+		dhd_prot_ring_doorbell(dhd, DHD_RDPTR_UPDATE_H2D_DB_MAGIC(ring));
+	}
+
+	return;
+}
+
+#ifdef DHD_HMAPTEST
+
+static void
+dhd_msgbuf_hmaptest_cmplt(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint64 end_usec;
+	char *readbuf;
+	uint32 len = dhd->prot->hmaptest.len;
+	uint32 i;
+
+	end_usec = OSL_SYSUPTIME_US();
+	end_usec -= prot->hmaptest.start_usec;
+	DHD_ERROR(("hmaptest cmplt: %d bytes in %llu usec, %u kBps\n",
+		len, end_usec, (len * (1000 * 1000 / 1024) / (uint32)(end_usec + 1))));
+
+	prot->hmaptest.in_progress = FALSE;
+	if (prot->hmaptest.accesstype == HMAPTEST_ACCESS_M2M) {
+			DHD_ERROR(("HMAPTEST_ACCESS_M2M\n"));
+	} else if (prot->hmaptest.accesstype == HMAPTEST_ACCESS_ARM) {
+			DHD_ERROR(("HMAPTEST_ACCESS_ARM\n"));
+	} else {
+		return;
+	}
+	readbuf = (char *)dhd->prot->hmaptest.mem.va + dhd->prot->hmaptest.offset;
+	OSL_CACHE_FLUSH(dhd->prot->hmaptest.mem.va,
+		dhd->prot->hmaptest.mem.len);
+	if (prot->hmaptest.is_write) {
+		DHD_ERROR(("hmaptest cmplt: FW has written at 0x%p\n", readbuf));
+		DHD_ERROR(("hmaptest cmplt: pattern = \n"));
+		len = ALIGN_SIZE(len, (sizeof(int32)));
+		for (i = 0; i < len; i += (sizeof(int32))) {
+			DHD_ERROR(("0x%08x\n", *(int *)(readbuf + i)));
+		}
+		DHD_ERROR(("\n\n"));
+	}
+
+}
+/* program HMAPTEST window and window config registers
+ * Reference for HMAP implementation in OS's that can easily leverage it
+ * this function can be used as reference for programming HMAP windows
+ * the function to program HMAP windows and enable it
+ * can be called at init time or hmap iovar
+ */
+static void
+dhdmsgbuf_set_hmaptest_windows(dhd_pub_t *dhd)
+{
+	uint32 nwindows = 0;
+	uint32 scratch_len;
+	uint64 scratch_lin, w1_start;
+	dmaaddr_t scratch_pa;
+	pcie_hmapwindow_t *hmapwindows; /* 8 windows 0-7 */
+	dhd_prot_t *prot = dhd->prot;
+	uint corerev = dhd->bus->sih->buscorerev;
+
+	scratch_pa = prot->hmaptest.mem.pa;
+	scratch_len = prot->hmaptest.mem.len;
+	scratch_lin  = (uint64)(PHYSADDRLO(scratch_pa) & 0xffffffff)
+		| (((uint64)PHYSADDRHI(scratch_pa)& 0xffffffff) << 32);
+	hmapwindows = (pcie_hmapwindow_t *)((uintptr_t)PCI_HMAP_WINDOW_BASE(corerev));
+	/* windows are 4kb aligned and window length is 512 byte aligned
+	 * window start ends with 0x1000 and window length ends with 0xe00
+	 * make the sandbox buffer 4kb aligned and size also 4kb aligned for hmap test
+	 * window0 = 0 - sandbox_start
+	 * window1 = sandbox_end + 1 - 0xffffffff
+	 * window2 = 0x100000000 - 0x1fffffe00
+	 * window 3 is programmed only for valid test cases
+	 * window3 = sandbox_start - sandbox_end
+	 */
+	w1_start  = scratch_lin +  scratch_len;
+		DHD_ERROR(("hmaptest: window 0 offset lower=0x%p upper=0x%p length=0x%p\n",
+		&(hmapwindows[0].baseaddr_lo), &(hmapwindows[0].baseaddr_hi),
+		&(hmapwindows[0].windowlength)));
+	DHD_ERROR(("hmaptest: window 1 offset lower=0x%p upper=0x%p length=0x%p\n",
+		&(hmapwindows[1].baseaddr_lo), &(hmapwindows[1].baseaddr_hi),
+		&(hmapwindows[1].windowlength)));
+	DHD_ERROR(("hmaptest: window 2 offset lower=0x%p upper=0x%p length=0x%p\n",
+		&(hmapwindows[2].baseaddr_lo), &(hmapwindows[2].baseaddr_hi),
+			&(hmapwindows[2].windowlength)));
+	DHD_ERROR(("hmaptest: window 3 offset lower=0x%p upper=0x%p length=0x%p\n",
+		&(hmapwindows[3].baseaddr_lo), &(hmapwindows[3].baseaddr_hi),
+		&(hmapwindows[3].windowlength)));
+		DHD_ERROR(("hmaptest: w0 base_lo=0x%08x base_hi=0x%08x len=0x%0llx\n",
+			0, 0, (uint64) scratch_lin));
+		DHD_ERROR(("hmaptest: w1 base_lo=0x%08x base_hi=0x%08x len=0x%0llx\n",
+			(uint32)(w1_start & 0xffffffff),
+			(uint32)((w1_start >> 32) & 0xffffffff),
+			(uint64)(0x100000000 - w1_start)));
+		DHD_ERROR(("hmaptest: w2 base_lo=0x%08x base_hi=0x%08x len=0x%0llx\n",
+			0, 1, (uint64)0xfffffe00));
+		/* setting window0 */
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[0].baseaddr_lo)), ~0, 0x0);
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[0].baseaddr_hi)), ~0, 0x0);
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[0].windowlength)), ~0,
+			(uint64)scratch_lin);
+		/* setting window1 */
+		w1_start  = scratch_lin +  scratch_len;
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[1].baseaddr_lo)), ~0,
+			(uint32)(w1_start & 0xffffffff));
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[1].baseaddr_hi)), ~0,
+			(uint32)((w1_start >> 32) & 0xffffffff));
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[1].windowlength)), ~0,
+			(0x100000000 - w1_start));
+		/* setting window2 */
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[2].baseaddr_lo)), ~0, 0x0);
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[2].baseaddr_hi)), ~0, 0x1);
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[2].windowlength)), ~0, 0xfffffe00);
+		nwindows = 3;
+		/* program only windows 0-2 with section1 +section2 */
+		/* setting window config */
+		/* set bit 8:15 in windowconfig to enable n windows in order */
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uint)PCI_HMAP_WINDOW_CONFIG(corerev), ~0, (nwindows << 8));
+}
+
+/* stop HMAPTEST does not check corerev
+ * caller has to ensure corerev check
+ */
+int
+dhdmsgbuf_hmaptest_stop(dhd_pub_t *dhd)
+{
+	uint32 window_config, nwindows, i;
+	pcie_hmapwindow_t *hmapwindows; /* 8 windows 0-7 */
+	uint corerev = dhd->bus->sih->buscorerev;
+
+	hmapwindows = (pcie_hmapwindow_t *)((uintptr_t)PCI_HMAP_WINDOW_BASE(corerev));
+	dhd->prot->hmaptest.in_progress = FALSE;
+
+	/* Reference for HMAP Implementation
+	 * Disable HMAP windows.
+	 * As windows were programmed in bus:hmap set call
+	 * disabling in hmaptest_stop.
+	 */
+	DHD_ERROR(("hmap: disable hmap windows\n"));
+	window_config = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		(uint)PCI_HMAP_WINDOW_CONFIG(corerev), 0, 0);
+	nwindows = (window_config & PCI_HMAP_NWINDOWS_MASK) >> PCI_HMAP_NWINDOWS_SHIFT;
+	si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		(uint)PCI_HMAP_WINDOW_CONFIG(corerev), ~0, 0);
+	/* clear all windows */
+	for (i = 0; i < nwindows; i++) {
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[i].baseaddr_lo)), ~0, 0);
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[i].baseaddr_hi)), ~0, 0);
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[i].windowlength)), ~0, 0);
+	}
+
+	return BCME_OK;
+}
+
+/* HMAP iovar intercept process */
+int
+dhdmsgbuf_hmap(dhd_pub_t *dhd, pcie_hmap_t *hmap_params, bool set)
+{
+
+	uint32 scratch_len;
+	uint64 scratch_lin, w1_start;
+	dmaaddr_t scratch_pa;
+	uint32 addr_lo, addr_hi, window_length, window_config, nwindows, i;
+	pcie_hmapwindow_t *hmapwindows; /* 8 windows 0-7 */
+
+	dhd_prot_t *prot = dhd->prot;
+	dhd_bus_t *bus = dhd->bus;
+	uint corerev = bus->sih->buscorerev;
+	scratch_pa = prot->hmaptest.mem.pa;
+	scratch_len = prot->hmaptest.mem.len;
+	scratch_lin  = (uint64)(PHYSADDRLO(scratch_pa) & 0xffffffff)
+		| (((uint64)PHYSADDRHI(scratch_pa)& 0xffffffff) << 32);
+	w1_start  = scratch_lin +  scratch_len;
+	DHD_ERROR(("HMAP:  pcicorerev = %d\n", corerev));
+
+	if (corerev < 24) {
+		DHD_ERROR(("HMAP not available on pci corerev = %d\n", corerev));
+		return BCME_UNSUPPORTED;
+	}
+	if (set) {
+		if (hmap_params->enable) {
+			dhdmsgbuf_set_hmaptest_windows(dhd);
+		} else {
+			dhdmsgbuf_hmaptest_stop(dhd); /* stop will clear all programmed windows */
+		}
+	}
+
+	OSL_CACHE_FLUSH(dhd->prot->hmaptest.mem.va,
+		dhd->prot->hmaptest.mem.len);
+
+	window_config = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		(uint)PCI_HMAP_WINDOW_CONFIG(corerev), 0, 0);
+	nwindows = (window_config & PCI_HMAP_NWINDOWS_MASK) >> PCI_HMAP_NWINDOWS_SHIFT;
+	prot->hmap_enabled = nwindows ? TRUE : FALSE;
+
+	/* getting window config */
+	/* set bit 8:15 in windowconfig to enable n windows in order */
+	DHD_ERROR(("hmap: hmap status = %s\n", (prot->hmap_enabled ? "Enabled" : "Disabled")));
+	DHD_ERROR(("hmap: window config = 0x%08x\n", window_config));
+	DHD_ERROR(("hmap: Windows\n"));
+
+	hmapwindows = (pcie_hmapwindow_t *)((uintptr_t)PCI_HMAP_WINDOW_BASE(corerev));
+	/* getting windows */
+	if (nwindows > 8)
+		return BCME_ERROR;
+	for (i = 0; i < nwindows; i++) {
+		addr_lo = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[i].baseaddr_lo)), 0, 0);
+		addr_hi = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[i].baseaddr_hi)), 0, 0);
+		window_length = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			(uintptr_t)(&(hmapwindows[i].windowlength)), 0, 0);
+
+		DHD_ERROR(("hmap: window %d address lower=0x%08x upper=0x%08x length=0x%08x\n",
+			i, addr_lo, addr_hi, window_length));
+	}
+	addr_hi = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		(uint)(PCI_HMAP_VIOLATION_ADDR_U(corerev)), 0, 0);
+	addr_lo = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		(uint)(PCI_HMAP_VIOLATION_ADDR_L(corerev)), 0, 0);
+	window_length = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		(uint)(PCI_HMAP_VIOLATION_INFO(corerev)), 0, 0);
+	DHD_ERROR(("hmap: violation regs\n"));
+	DHD_ERROR(("hmap: violationaddr_hi =0x%08x\n", addr_hi));
+	DHD_ERROR(("hmap: violationaddr_lo =0x%08x\n", addr_lo));
+	DHD_ERROR(("hmap: violation_info   =0x%08x\n", window_length));
+	DHD_ERROR(("hmap: Buffer allocated for HMAPTEST Start=0x%0llx len =0x%08x End =0x%0llx\n",
+		(uint64) scratch_lin, scratch_len, (uint64) w1_start));
+
+	return BCME_OK;
+}
+
+/* hmaptest iovar process
+ * This iovar triggers HMAPTEST with given params
+ * on chips that have HMAP
+ * DHD programs hmap window registers with host addresses here.
+ */
+int
+dhdmsgbuf_hmaptest(dhd_pub_t *dhd, pcie_hmaptest_t *hmaptest_params)
+{
+
+	dhd_prot_t *prot = dhd->prot;
+	int ret = BCME_OK;
+	uint32 offset = 0;
+	uint64 scratch_lin;
+	dhd_bus_t *bus = dhd->bus;
+	uint corerev = bus->sih->buscorerev;
+
+	if (prot->hmaptest.in_progress) {
+		DHD_ERROR(("HMAPTEST already running. Try again.\n"));
+		return BCME_BUSY;
+	}
+
+	prot->hmaptest.in_progress = TRUE;
+
+	if (corerev < 24) {
+		DHD_ERROR(("HMAP not available on pci corerev = %d\n", corerev));
+		return BCME_UNSUPPORTED;
+	}
+	prot->hmaptest.accesstype = hmaptest_params->accesstype;
+	prot->hmaptest.is_write = hmaptest_params->is_write;
+	prot->hmaptest.len = hmaptest_params->xfer_len;
+	prot->hmaptest.offset = hmaptest_params->host_offset;
+	offset = prot->hmaptest.offset;
+
+	DHD_ERROR(("hmaptest: is_write =%d accesstype=%d offset =%d len=%d value=0x%08x\n",
+		prot->hmaptest.is_write, prot->hmaptest.accesstype,
+		offset, prot->hmaptest.len, hmaptest_params->value));
+
+	DHD_ERROR(("hmaptest  dma_lo=0x%08x hi=0x%08x pa\n",
+		(uint32)PHYSADDRLO(prot->hmaptest.mem.pa),
+		(uint32)PHYSADDRHI(prot->hmaptest.mem.pa)));
+
+	if (prot->hmaptest.accesstype == HMAPTEST_ACCESS_D11) {
+		if (prot->hmaptest.is_write) {
+			/* if d11 is writing then post rxbuf from scratch area */
+			dhd->prot->hmaptest_rx_active = HMAPTEST_D11_RX_ACTIVE;
+		} else {
+			/* if d11 is reading then post txbuf from scratch area */
+			dhd->prot->hmaptest_tx_active = HMAPTEST_D11_TX_ACTIVE;
+		}
+
+	} else {
+		uint32 pattern = 0xdeadbeef;
+		uint32 i;
+		uint32 maxbuflen = MIN(prot->hmaptest.len, (PKTBUFSZ));
+		char *fillbuf = (char *)dhd->prot->hmaptest.mem.va
+			+ offset;
+		if ((fillbuf + maxbuflen) >
+			((char *)dhd->prot->hmaptest.mem.va + dhd->prot->hmaptest.mem.len)) {
+			DHD_ERROR(("hmaptest: M2m/ARM ERROR offset + len outside buffer\n"));
+			dhd->prot->hmaptest.in_progress = FALSE;
+			return BCME_BADARG;
+		}
+
+		if (prot->hmaptest.accesstype == HMAPTEST_ACCESS_M2M) {
+			DHD_ERROR(("HMAPTEST_ACCESS_M2M\n"));
+		} else if (prot->hmaptest.accesstype == HMAPTEST_ACCESS_ARM) {
+			DHD_ERROR(("HMAPTEST_ACCESS_ARM\n"));
+		} else {
+			prot->hmaptest.in_progress = FALSE;
+			DHD_ERROR(("hmaptest: accesstype error\n"));
+			return BCME_BADARG;
+		}
+
+		/* fill a pattern at offset */
+		maxbuflen = ALIGN_SIZE(maxbuflen, (sizeof(uint32)));
+		memset(fillbuf, 0, maxbuflen);
+		DHD_ERROR(("hmaptest: dhd write pattern at addr=0x%p\n",
+			fillbuf));
+		DHD_ERROR(("pattern = %08x, %u times",
+			pattern, (uint32)(maxbuflen / sizeof(uint32))));
+		for (i = 0; i < maxbuflen; i += sizeof(uint32)) {
+			*(uint32 *)(fillbuf + i) = pattern;
+		}
+		OSL_CACHE_FLUSH(dhd->prot->hmaptest.mem.va,
+			dhd->prot->hmaptest.mem.len);
+		DHD_ERROR(("\n\n"));
+
+	}
+
+	/*
+	 * Do not calculate address from scratch buffer + offset,
+	 * if user supplied absolute address
+	 */
+	if (hmaptest_params->host_addr_lo || hmaptest_params->host_addr_hi) {
+		if (prot->hmaptest.accesstype == HMAPTEST_ACCESS_D11) {
+			DHD_ERROR(("hmaptest: accesstype D11 does not support absolute addr\n"));
+			return BCME_UNSUPPORTED;
+		}
+	} else {
+		scratch_lin  = (uint64)(PHYSADDRLO(prot->hmaptest.mem.pa) & 0xffffffff)
+			| (((uint64)PHYSADDRHI(prot->hmaptest.mem.pa) & 0xffffffff) << 32);
+		scratch_lin += offset;
+		hmaptest_params->host_addr_lo = htol32((uint32)(scratch_lin & 0xffffffff));
+		hmaptest_params->host_addr_hi = htol32((uint32)((scratch_lin >> 32) & 0xffffffff));
+	}
+
+	DHD_INFO(("HMAPTEST Started...\n"));
+	prot->hmaptest.start_usec = OSL_SYSUPTIME_US();
+	return ret;
+
+}
+
+#endif /* DHD_HMAPTEST */
+
+/* called before an ioctl is sent to the dongle */
+static void
+dhd_prot_wlioctl_intercept(dhd_pub_t *dhd, wl_ioctl_t * ioc, void * buf)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int slen = 0;
+
+	if (ioc->cmd == WLC_SET_VAR && buf != NULL && !strcmp(buf, "pcie_bus_tput")) {
+		pcie_bus_tput_params_t *tput_params;
+
+		slen = strlen("pcie_bus_tput") + 1;
+		tput_params = (pcie_bus_tput_params_t*)((char *)buf + slen);
+		bcopy(&prot->host_bus_throughput_buf.pa, &tput_params->host_buf_addr,
+			sizeof(tput_params->host_buf_addr));
+		tput_params->host_buf_len = DHD_BUS_TPUT_BUF_LEN;
+	}
+
+#ifdef DHD_HMAPTEST
+	if (buf != NULL && !strcmp(buf, "bus:hmap")) {
+		pcie_hmap_t *hmap_params;
+		slen = strlen("bus:hmap") + 1;
+		hmap_params = (pcie_hmap_t*)((char *)buf + slen);
+		dhdmsgbuf_hmap(dhd, hmap_params, (ioc->cmd == WLC_SET_VAR));
+	}
+
+	if (ioc->cmd == WLC_SET_VAR && buf != NULL && !strcmp(buf, "bus:hmaptest")) {
+		pcie_hmaptest_t *hmaptest_params;
+
+		slen = strlen("bus:hmaptest") + 1;
+		hmaptest_params = (pcie_hmaptest_t*)((char *)buf + slen);
+		dhdmsgbuf_hmaptest(dhd, hmaptest_params);
+	}
+#endif /* DHD_HMAPTEST */
+}
+
+/* called after an ioctl returns from dongle */
+static void
+dhd_prot_wl_ioctl_ret_intercept(dhd_pub_t *dhd, wl_ioctl_t * ioc, void * buf,
+	int ifidx, int ret, int len)
+{
+
+#ifdef DHD_HMAPTEST
+	if (ioc->cmd == WLC_SET_VAR && buf != NULL && !strcmp(buf, "bus:hmaptest")) {
+		dhd_msgbuf_hmaptest_cmplt(dhd);
+	}
+#endif /* DHD_HMAPTEST */
+
+	if (!ret && ioc->cmd == WLC_SET_VAR && buf != NULL) {
+		int slen;
+		/* Intercept the wme_dp ioctl here */
+		if (!strcmp(buf, "wme_dp")) {
+			int val = 0;
+			slen = strlen("wme_dp") + 1;
+			if (len >= (int)(slen + sizeof(int)))
+				bcopy(((char *)buf + slen), &val, sizeof(int));
+			dhd->wme_dp = (uint8) ltoh32(val);
+		}
+
+	}
+
+}
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+extern bool g_pm_control;
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+
+/** Use protocol to issue ioctl to dongle. Only one ioctl may be in transit. */
+int dhd_prot_ioctl(dhd_pub_t *dhd, int ifidx, wl_ioctl_t * ioc, void * buf, int len)
+{
+	int ret = -1;
+	uint8 action;
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR_RLMT(("%s : PCIe link is down. we have nothing to do\n", __FUNCTION__));
+		goto done;
+	}
+
+	if (dhd_query_bus_erros(dhd)) {
+		DHD_ERROR_RLMT(("%s : some BUS error. we have nothing to do\n", __FUNCTION__));
+		goto done;
+	}
+
+	if ((dhd->busstate == DHD_BUS_DOWN) || dhd->hang_was_sent) {
+		DHD_ERROR_RLMT(("%s : bus is down. we have nothing to do -"
+			" bus state: %d, sent hang: %d\n", __FUNCTION__,
+			dhd->busstate, dhd->hang_was_sent));
+		goto done;
+	}
+
+	if (dhd->busstate == DHD_BUS_SUSPEND) {
+		DHD_ERROR(("%s : bus is suspended\n", __FUNCTION__));
+		goto done;
+	}
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#ifdef DHD_PCIE_REG_ACCESS
+#ifdef BOARD_HIKEY
+#ifndef PCIE_LNK_SPEED_GEN1
+#define PCIE_LNK_SPEED_GEN1		0x1
+#endif
+	/* BUG_ON if link speed is GEN1 in Hikey for 4389B0 */
+	if (dhd->bus->sih->buscorerev == 72) {
+		if (dhd_get_pcie_linkspeed(dhd) == PCIE_LNK_SPEED_GEN1) {
+			DHD_ERROR(("%s: ******* Link Speed is GEN1 *********\n", __FUNCTION__));
+			BUG_ON(1);
+		}
+	}
+#endif /* BOARD_HIKEY */
+#endif /* DHD_PCIE_REG_ACCESS */
+
+	if (ioc->cmd == WLC_SET_PM) {
+#ifdef DHD_PM_CONTROL_FROM_FILE
+		if (g_pm_control == TRUE) {
+			DHD_ERROR(("%s: SET PM ignored!(Requested:%d)\n",
+				__FUNCTION__, buf ? *(char *)buf : 0));
+			goto done;
+		}
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+#ifdef DHD_PM_OVERRIDE
+		{
+			extern bool g_pm_override;
+			if (g_pm_override == TRUE) {
+				DHD_ERROR(("%s: PM override SET PM ignored!(Requested:%d)\n",
+					__FUNCTION__, buf ? *(char *)buf : 0));
+				goto done;
+			}
+		}
+#endif /* DHD_PM_OVERRIDE */
+		DHD_TRACE_HW4(("%s: SET PM to %d\n", __FUNCTION__, buf ? *(char *)buf : 0));
+	}
+
+	ASSERT(len <= WLC_IOCTL_MAXLEN);
+
+	if (len > WLC_IOCTL_MAXLEN)
+		goto done;
+
+	action = ioc->set;
+
+	dhd_prot_wlioctl_intercept(dhd, ioc, buf);
+
+#if defined(EXT_STA)
+	wl_dbglog_ioctl_add(ioc, len, NULL);
+#endif
+	if (action & WL_IOCTL_ACTION_SET) {
+		ret = dhd_msgbuf_set_ioctl(dhd, ifidx, ioc->cmd, buf, len, action);
+	} else {
+		ret = dhd_msgbuf_query_ioctl(dhd, ifidx, ioc->cmd, buf, len, action);
+		if (ret > 0)
+			ioc->used = ret;
+	}
+
+	/* Too many programs assume ioctl() returns 0 on success */
+	if (ret >= 0) {
+		ret = 0;
+	} else {
+#ifndef DETAIL_DEBUG_LOG_FOR_IOCTL
+		DHD_INFO(("%s: status ret value is %d \n", __FUNCTION__, ret));
+#endif /* !DETAIL_DEBUG_LOG_FOR_IOCTL */
+		dhd->dongle_error = ret;
+	}
+
+	dhd_prot_wl_ioctl_ret_intercept(dhd, ioc, buf, ifidx, ret, len);
+
+done:
+	return ret;
+
+} /* dhd_prot_ioctl */
+
+/** test / loopback */
+
+/*
+ * XXX: This will fail with new PCIe Split header Full Dongle using fixed
+ * sized messages in control submission ring. We seem to be sending the lpbk
+ * data via the control message, wherein the lpbk data may be larger than 1
+ * control message that is being committed.
+ */
+int
+dhdmsgbuf_lpbk_req(dhd_pub_t *dhd, uint len)
+{
+	unsigned long flags;
+	dhd_prot_t *prot = dhd->prot;
+	uint16 alloced = 0;
+
+	ioct_reqst_hdr_t *ioct_rqst;
+
+	uint16 hdrlen = sizeof(ioct_reqst_hdr_t);
+	uint16 msglen = len + hdrlen;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+
+	msglen = ALIGN_SIZE(msglen, DMA_ALIGN_LEN);
+	msglen = LIMIT_TO_MAX(msglen, MSGBUF_MAX_MSG_SIZE);
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	ioct_rqst = (ioct_reqst_hdr_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+
+	if (ioct_rqst == NULL) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return 0;
+	}
+
+	{
+		uint8 *ptr;
+		uint16 i;
+
+		ptr = (uint8 *)ioct_rqst; /* XXX: failure!!! */
+		for (i = 0; i < msglen; i++) {
+			ptr[i] = i % 256;
+		}
+	}
+
+	/* Common msg buf hdr */
+	ioct_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+
+	ioct_rqst->msg.msg_type = MSG_TYPE_LOOPBACK;
+	ioct_rqst->msg.if_id = 0;
+	ioct_rqst->msg.flags = ring->current_phase;
+
+	bcm_print_bytes("LPBK REQ: ", (uint8 *)ioct_rqst, msglen);
+
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, ioct_rqst, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return 0;
+}
+
+/** test / loopback */
+void dmaxfer_free_dmaaddr(dhd_pub_t *dhd, dhd_dmaxfer_t *dmaxfer)
+{
+	if (dmaxfer == NULL)
+		return;
+
+	dhd_dma_buf_free(dhd, &dmaxfer->srcmem);
+	dhd_dma_buf_free(dhd, &dmaxfer->dstmem);
+}
+
+/** test / loopback */
+int
+dhd_prepare_schedule_dmaxfer_free(dhd_pub_t *dhdp)
+{
+	dhd_prot_t *prot = dhdp->prot;
+	dhd_dmaxfer_t *dmaxfer = &prot->dmaxfer;
+	dmaxref_mem_map_t *dmap = NULL;
+
+	dmap = MALLOCZ(dhdp->osh, sizeof(dmaxref_mem_map_t));
+	if (!dmap) {
+		DHD_ERROR(("%s: dmap alloc failed\n", __FUNCTION__));
+		goto mem_alloc_fail;
+	}
+	dmap->srcmem = &(dmaxfer->srcmem);
+	dmap->dstmem = &(dmaxfer->dstmem);
+
+	DMAXFER_FREE(dhdp, dmap);
+	return BCME_OK;
+
+mem_alloc_fail:
+	if (dmap) {
+		MFREE(dhdp->osh, dmap, sizeof(dmaxref_mem_map_t));
+	}
+	return BCME_NOMEM;
+} /* dhd_prepare_schedule_dmaxfer_free */
+
+/** test / loopback */
+void
+dmaxfer_free_prev_dmaaddr(dhd_pub_t *dhdp, dmaxref_mem_map_t *dmmap)
+{
+
+	dhd_dma_buf_free(dhdp, dmmap->srcmem);
+	dhd_dma_buf_free(dhdp, dmmap->dstmem);
+
+	MFREE(dhdp->osh, dmmap, sizeof(dmaxref_mem_map_t));
+
+	dhdp->bus->dmaxfer_complete = TRUE;
+	dhd_os_dmaxfer_wake(dhdp);
+} /* dmaxfer_free_prev_dmaaddr */
+
+/** test / loopback */
+int dmaxfer_prepare_dmaaddr(dhd_pub_t *dhd, uint len,
+	uint srcdelay, uint destdelay, dhd_dmaxfer_t *dmaxfer)
+{
+	uint i = 0, j = 0;
+	if (!dmaxfer)
+		return BCME_ERROR;
+
+	/* First free up existing buffers */
+	dmaxfer_free_dmaaddr(dhd, dmaxfer);
+
+	if (dhd_dma_buf_alloc(dhd, &dmaxfer->srcmem, len)) {
+		return BCME_NOMEM;
+	}
+
+	if (dhd_dma_buf_alloc(dhd, &dmaxfer->dstmem, len + 8)) {
+		dhd_dma_buf_free(dhd, &dmaxfer->srcmem);
+		return BCME_NOMEM;
+	}
+
+	dmaxfer->len = len;
+	DHD_ERROR(("using pattern %d, for loopback test\n", dhd->bus->lpbk_xfer_data_pattern_type));
+
+	/* Populate source with a pattern like below
+	 * 0x00000000
+	 * 0x01010101
+	 * 0x02020202
+	 * 0x03030303
+	 * 0x04040404
+	 * 0x05050505
+	 * ...
+	 * 0xFFFFFFFF
+	 */
+	while (i < dmaxfer->len) {
+		if (dhd->bus->lpbk_xfer_data_pattern_type == LPBK_DMA_XFER_DTPTRN_0x00) {
+			((uint8*)dmaxfer->srcmem.va)[i] = 0x00;
+			i++;
+		}
+		else if (dhd->bus->lpbk_xfer_data_pattern_type == LPBK_DMA_XFER_DTPTRN_0xFF) {
+			((uint8*)dmaxfer->srcmem.va)[i] = 0xFF;
+			i++;
+		}
+		else if (dhd->bus->lpbk_xfer_data_pattern_type == LPBK_DMA_XFER_DTPTRN_0x55) {
+			((uint8*)dmaxfer->srcmem.va)[i] = 0x55;
+			i++;
+		}
+		else if (dhd->bus->lpbk_xfer_data_pattern_type == LPBK_DMA_XFER_DTPTRN_0xAA) {
+			((uint8*)dmaxfer->srcmem.va)[i] = 0xAA;
+			i++;
+		}
+		else {
+			((uint8*)dmaxfer->srcmem.va)[i] = j % 256;
+			i++;
+			if (i % 4 == 0) {
+				j++;
+			}
+		}
+	}
+
+	OSL_CACHE_FLUSH(dmaxfer->srcmem.va, dmaxfer->len);
+
+	dmaxfer->srcdelay = srcdelay;
+	dmaxfer->destdelay = destdelay;
+
+	return BCME_OK;
+} /* dmaxfer_prepare_dmaaddr */
+
+static void
+dhd_msgbuf_dmaxfer_process(dhd_pub_t *dhd, void *msg)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint64 end_usec;
+	pcie_dmaxfer_cmplt_t *cmplt = (pcie_dmaxfer_cmplt_t *)msg;
+	int buf_free_scheduled;
+	int err = 0;
+
+	BCM_REFERENCE(cmplt);
+	end_usec = OSL_SYSUPTIME_US();
+
+	DHD_ERROR(("DMA loopback status: %d\n", cmplt->compl_hdr.status));
+	prot->dmaxfer.status = cmplt->compl_hdr.status;
+	OSL_CACHE_INV(prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
+	if (prot->dmaxfer.d11_lpbk != M2M_WRITE_TO_RAM &&
+		prot->dmaxfer.d11_lpbk != M2M_READ_FROM_RAM &&
+		prot->dmaxfer.d11_lpbk != D11_WRITE_TO_RAM &&
+		prot->dmaxfer.d11_lpbk != D11_READ_FROM_RAM) {
+		err = memcmp(prot->dmaxfer.srcmem.va,
+			prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
+	}
+	if (prot->dmaxfer.srcmem.va && prot->dmaxfer.dstmem.va) {
+		if (err ||
+		        cmplt->compl_hdr.status != BCME_OK) {
+		        DHD_ERROR(("DMA loopback failed\n"));
+			/* it is observed that some times the completion
+			 * header status is set as OK, but the memcmp fails
+			 * hence always explicitly set the dmaxfer status
+			 * as error if this happens.
+			 */
+			prot->dmaxfer.status = BCME_ERROR;
+			prhex("XFER SRC: ",
+			    prot->dmaxfer.srcmem.va, prot->dmaxfer.len);
+			prhex("XFER DST: ",
+			    prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
+		}
+		else {
+			switch (prot->dmaxfer.d11_lpbk) {
+			case M2M_DMA_LPBK: {
+				DHD_ERROR(("DMA successful pcie m2m DMA loopback\n"));
+				} break;
+			case D11_LPBK: {
+				DHD_ERROR(("DMA successful with d11 loopback\n"));
+				} break;
+			case BMC_LPBK: {
+				DHD_ERROR(("DMA successful with bmc loopback\n"));
+				} break;
+			case M2M_NON_DMA_LPBK: {
+				DHD_ERROR(("DMA successful pcie m2m NON DMA loopback\n"));
+				} break;
+			case D11_HOST_MEM_LPBK: {
+				DHD_ERROR(("DMA successful d11 host mem loopback\n"));
+				} break;
+			case BMC_HOST_MEM_LPBK: {
+				DHD_ERROR(("DMA successful bmc host mem loopback\n"));
+				} break;
+			case M2M_WRITE_TO_RAM: {
+				DHD_ERROR(("DMA successful pcie m2m write to ram\n"));
+				} break;
+			case M2M_READ_FROM_RAM: {
+				DHD_ERROR(("DMA successful pcie m2m read from ram\n"));
+				prhex("XFER DST: ",
+					prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
+				} break;
+			case D11_WRITE_TO_RAM: {
+				DHD_ERROR(("DMA successful D11 write to ram\n"));
+				} break;
+			case D11_READ_FROM_RAM: {
+				DHD_ERROR(("DMA successful D11 read from ram\n"));
+				prhex("XFER DST: ",
+					prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
+				} break;
+			default: {
+				DHD_ERROR(("Invalid loopback option\n"));
+				} break;
+			}
+
+			if (DHD_LPBKDTDUMP_ON()) {
+				/* debug info print of the Tx and Rx buffers */
+				dhd_prhex("XFER SRC: ", prot->dmaxfer.srcmem.va,
+					prot->dmaxfer.len, DHD_INFO_VAL);
+				dhd_prhex("XFER DST: ", prot->dmaxfer.dstmem.va,
+					prot->dmaxfer.len, DHD_INFO_VAL);
+			}
+		}
+	}
+
+	buf_free_scheduled = dhd_prepare_schedule_dmaxfer_free(dhd);
+	end_usec -= prot->dmaxfer.start_usec;
+	if (end_usec) {
+		prot->dmaxfer.time_taken = end_usec;
+		DHD_ERROR(("DMA loopback %d bytes in %lu usec, %u kBps\n",
+			prot->dmaxfer.len, (unsigned long)end_usec,
+			(prot->dmaxfer.len * (1000 * 1000 / 1024) / (uint32)end_usec)));
+	}
+	dhd->prot->dmaxfer.in_progress = FALSE;
+
+	if (buf_free_scheduled != BCME_OK) {
+		dhd->bus->dmaxfer_complete = TRUE;
+		dhd_os_dmaxfer_wake(dhd);
+	}
+}
+
+/** Test functionality.
+ * Transfers bytes from host to dongle and to host again using DMA
+ * This function is not reentrant, as prot->dmaxfer.in_progress is not protected
+ * by a spinlock.
+ */
+int
+dhdmsgbuf_dmaxfer_req(dhd_pub_t *dhd, uint len, uint srcdelay, uint destdelay,
+	uint d11_lpbk, uint core_num, uint32 mem_addr)
+{
+	unsigned long flags;
+	int ret = BCME_OK;
+	dhd_prot_t *prot = dhd->prot;
+	pcie_dma_xfer_params_t *dmap;
+	uint32 xferlen = LIMIT_TO_MAX(len, DMA_XFER_LEN_LIMIT);
+	uint16 alloced = 0;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+
+	/* XXX: prot->dmaxfer.in_progress is not protected by lock */
+	if (prot->dmaxfer.in_progress) {
+		DHD_ERROR(("DMA is in progress...\n"));
+		return BCME_ERROR;
+	}
+
+	if (d11_lpbk >= MAX_LPBK) {
+		DHD_ERROR(("loopback mode should be either"
+			" 0-PCIE_M2M_DMA, 1-D11, 2-BMC or 3-PCIE_M2M_NonDMA\n"));
+		return BCME_ERROR;
+	}
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK) {
+		return BCME_ERROR;
+	}
+#endif /* PCIE_INB_DW */
+
+	prot->dmaxfer.in_progress = TRUE;
+	if ((ret = dmaxfer_prepare_dmaaddr(dhd, xferlen, srcdelay, destdelay,
+	        &prot->dmaxfer)) != BCME_OK) {
+		prot->dmaxfer.in_progress = FALSE;
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return ret;
+	}
+	DHD_RING_LOCK(ring->ring_lock, flags);
+	dmap = (pcie_dma_xfer_params_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+
+	if (dmap == NULL) {
+		dmaxfer_free_dmaaddr(dhd, &prot->dmaxfer);
+		prot->dmaxfer.in_progress = FALSE;
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_NOMEM;
+	}
+
+	/* Common msg buf hdr */
+	dmap->cmn_hdr.msg_type = MSG_TYPE_LPBK_DMAXFER;
+	dmap->cmn_hdr.request_id = htol32(DHD_FAKE_PKTID);
+	dmap->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	dmap->cmn_hdr.flags = ring->current_phase;
+	ring->seqnum++;
+
+	dmap->host_input_buf_addr.high = htol32(PHYSADDRHI(prot->dmaxfer.srcmem.pa));
+	dmap->host_input_buf_addr.low = htol32(PHYSADDRLO(prot->dmaxfer.srcmem.pa));
+	dmap->host_ouput_buf_addr.high = htol32(PHYSADDRHI(prot->dmaxfer.dstmem.pa));
+	dmap->host_ouput_buf_addr.low = htol32(PHYSADDRLO(prot->dmaxfer.dstmem.pa));
+	dmap->xfer_len = htol32(prot->dmaxfer.len);
+	dmap->srcdelay = htol32(prot->dmaxfer.srcdelay);
+	dmap->destdelay = htol32(prot->dmaxfer.destdelay);
+	prot->dmaxfer.d11_lpbk = d11_lpbk;
+	if (d11_lpbk == M2M_WRITE_TO_RAM) {
+		dmap->host_ouput_buf_addr.high = 0x0;
+		dmap->host_ouput_buf_addr.low = mem_addr;
+	} else if (d11_lpbk == M2M_READ_FROM_RAM) {
+		dmap->host_input_buf_addr.high = 0x0;
+		dmap->host_input_buf_addr.low = mem_addr;
+	} else if (d11_lpbk == D11_WRITE_TO_RAM) {
+		dmap->host_ouput_buf_addr.high = 0x0;
+		dmap->host_ouput_buf_addr.low = mem_addr;
+	} else if (d11_lpbk == D11_READ_FROM_RAM) {
+		dmap->host_input_buf_addr.high = 0x0;
+		dmap->host_input_buf_addr.low = mem_addr;
+	}
+	dmap->flags = (((core_num & PCIE_DMA_XFER_FLG_CORE_NUMBER_MASK)
+			<< PCIE_DMA_XFER_FLG_CORE_NUMBER_SHIFT) |
+			((prot->dmaxfer.d11_lpbk & PCIE_DMA_XFER_FLG_D11_LPBK_MASK)
+			 << PCIE_DMA_XFER_FLG_D11_LPBK_SHIFT));
+	prot->dmaxfer.start_usec = OSL_SYSUPTIME_US();
+
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, dmap, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+	DHD_ERROR(("DMA loopback Started... on core[%d]\n", core_num));
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return BCME_OK;
+} /* dhdmsgbuf_dmaxfer_req */
+
+int
+dhdmsgbuf_dmaxfer_status(dhd_pub_t *dhd, dma_xfer_info_t *result)
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	if (prot->dmaxfer.in_progress)
+		result->status = DMA_XFER_IN_PROGRESS;
+	else if (prot->dmaxfer.status == 0)
+		result->status = DMA_XFER_SUCCESS;
+	else
+		result->status = DMA_XFER_FAILED;
+
+	result->type = prot->dmaxfer.d11_lpbk;
+	result->error_code = prot->dmaxfer.status;
+	result->num_bytes = prot->dmaxfer.len;
+	result->time_taken = prot->dmaxfer.time_taken;
+	if (prot->dmaxfer.time_taken) {
+		/* throughput in kBps */
+		result->tput =
+			(prot->dmaxfer.len * (1000 * 1000 / 1024)) /
+			(uint32)prot->dmaxfer.time_taken;
+	}
+
+	return BCME_OK;
+}
+
+/** Called in the process of submitting an ioctl to the dongle */
+static int
+dhd_msgbuf_query_ioctl(dhd_pub_t *dhd, int ifidx, uint cmd, void *buf, uint len, uint8 action)
+{
+	int ret = 0;
+	uint copylen = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s : PCIe link is down. we have nothing to do\n",
+			__FUNCTION__));
+		return -EIO;
+	}
+
+	if (dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
+		return -EIO;
+	}
+
+	/* don't talk to the dongle if fw is about to be reloaded */
+	if (dhd->hang_was_sent) {
+		DHD_ERROR(("%s: HANG was sent up earlier. Not talking to the chip\n",
+			__FUNCTION__));
+		return -EIO;
+	}
+
+	if (cmd == WLC_GET_VAR && buf)
+	{
+		if (!len || !*(uint8 *)buf) {
+			DHD_ERROR(("%s(): Zero length bailing\n", __FUNCTION__));
+			ret = BCME_BADARG;
+			goto done;
+		}
+
+		/* Respond "bcmerror" and "bcmerrorstr" with local cache */
+		copylen = MIN(len, BCME_STRLEN);
+
+		if ((len >= strlen("bcmerrorstr")) &&
+			(!strcmp((char *)buf, "bcmerrorstr"))) {
+			strlcpy((char *)buf, bcmerrorstr(dhd->dongle_error), copylen);
+			goto done;
+		} else if ((len >= strlen("bcmerror")) &&
+			!strcmp((char *)buf, "bcmerror")) {
+			*(uint32 *)(uint32 *)buf = dhd->dongle_error;
+			goto done;
+		}
+	}
+
+	DHD_CTL(("query_ioctl: ACTION %d ifdix %d cmd %d len %d \n",
+	    action, ifidx, cmd, len));
+#ifdef REPORT_FATAL_TIMEOUTS
+	/*
+	 * These timers "should" be started before sending H2D interrupt.
+	 * Think of the scenario where H2D interrupt is fired and the Dongle
+	 * responds back immediately. From the DPC we would stop the cmd, bus
+	 * timers. But the process context could have switched out leading to
+	 * a situation where the timers are Not started yet, but are actually stopped.
+	 *
+	 * Disable preemption from the time we start the timer until we are done
+	 * with seding H2D interrupts.
+	 */
+	OSL_DISABLE_PREEMPTION(dhd->osh);
+	dhd_set_request_id(dhd, dhd->prot->ioctl_trans_id+1, cmd);
+	dhd_start_cmd_timer(dhd);
+	dhd_start_bus_timer(dhd);
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+	ret = dhd_fillup_ioct_reqst(dhd, (uint16)len, cmd, buf, ifidx);
+
+#ifdef REPORT_FATAL_TIMEOUTS
+	/* For some reason if we fail to ring door bell, stop the timers */
+	if (ret < 0) {
+		DHD_ERROR(("%s(): dhd_fillup_ioct_reqst failed \r\n", __FUNCTION__));
+		dhd_stop_cmd_timer(dhd);
+		dhd_stop_bus_timer(dhd);
+		OSL_ENABLE_PREEMPTION(dhd->osh);
+		goto done;
+	}
+	OSL_ENABLE_PREEMPTION(dhd->osh);
+#else
+	if (ret < 0) {
+		DHD_ERROR(("%s(): dhd_fillup_ioct_reqst failed \r\n", __FUNCTION__));
+		goto done;
+	}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+	/* wait for IOCTL completion message from dongle and get first fragment */
+	ret = dhd_msgbuf_wait_ioctl_cmplt(dhd, len, buf);
+
+done:
+	return ret;
+}
+
+static void
+dhd_msgbuf_dump_iovar_name(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	dhd->rxcnt_timeout++;
+	dhd->rx_ctlerrs++;
+	DHD_ERROR(("%s: resumed on timeout rxcnt_timeout %d ioctl_cmd %d "
+		"trans_id %d state %d busstate=%d ioctl_received=%d\n",	__FUNCTION__,
+		dhd->rxcnt_timeout, prot->curr_ioctl_cmd, prot->ioctl_trans_id,
+		prot->ioctl_state, dhd->busstate, prot->ioctl_received));
+
+	if (prot->curr_ioctl_cmd == WLC_SET_VAR ||
+			prot->curr_ioctl_cmd == WLC_GET_VAR) {
+		char iovbuf[32];
+		int dump_size = 128;
+		uint8 *ioctl_buf = (uint8 *)prot->ioctbuf.va;
+		memset(iovbuf, 0, sizeof(iovbuf));
+		strncpy(iovbuf, ioctl_buf, sizeof(iovbuf) - 1);
+		iovbuf[sizeof(iovbuf) - 1] = '\0';
+		DHD_ERROR(("Current IOVAR (%s): %s\n",
+			prot->curr_ioctl_cmd == WLC_SET_VAR ?
+			"WLC_SET_VAR" : "WLC_GET_VAR", iovbuf));
+		DHD_ERROR(("========== START IOCTL REQBUF DUMP ==========\n"));
+		dhd_prhex(NULL, ioctl_buf, dump_size, DHD_ERROR_VAL);
+		DHD_ERROR(("========== END IOCTL REQBUF DUMP ==========\n"));
+	}
+}
+
+void
+dhd_msgbuf_iovar_timeout_dump(dhd_pub_t *dhd)
+{
+	uint32 intstatus;
+	if (dhd->is_sched_error) {
+		DHD_ERROR(("%s: ROT due to scheduling problem\n", __FUNCTION__));
+	}
+#if defined(DHD_KERNEL_SCHED_DEBUG) && defined(DHD_FW_COREDUMP)
+		/* XXX DHD triggers Kernel panic if the resumed on timeout occurrs
+		 * due to tasklet or workqueue scheduling problems in the Linux Kernel.
+		 * Customer informs that it is hard to find any clue from the
+		 * host memory dump since the important tasklet or workqueue information
+		 * is already disappered due the latency while printing out the timestamp
+		 * logs for debugging scan timeout issue.
+		 * For this reason, customer requestes us to trigger Kernel Panic rather than
+		 * taking a SOCRAM dump.
+		 */
+		if (dhd->is_sched_error && dhd->memdump_enabled == DUMP_MEMFILE_BUGON) {
+			/* change g_assert_type to trigger Kernel panic */
+			g_assert_type = 2;
+			/* use ASSERT() to trigger panic */
+			ASSERT(0);
+			return;
+		}
+#endif /* DHD_KERNEL_SCHED_DEBUG && DHD_FW_COREDUMP */
+
+	/* Check the PCIe link status by reading intstatus register */
+	if (!dhd || !dhd->bus || !dhd->bus->sih) {
+		DHD_ERROR(("%s: skip due to invalid parameter\n", __FUNCTION__));
+		ASSERT(0);
+		return;
+	}
+	intstatus = si_corereg(dhd->bus->sih,
+		dhd->bus->sih->buscoreidx, dhd->bus->pcie_mailbox_int, 0, 0);
+	if (intstatus == (uint32)-1) {
+		DHD_ERROR(("%s : PCIe link might be down\n", __FUNCTION__));
+		dhd->bus->is_linkdown = TRUE;
+	}
+
+	dhd_bus_dump_console_buffer(dhd->bus);
+	dhd_prot_debug_info_print(dhd);
+}
+
+extern void dhd_validate_pcie_link_cbp_wlbp(dhd_bus_t *bus);
+/**
+ * Waits for IOCTL completion message from the dongle, copies this into caller
+ * provided parameter 'buf'.
+ */
+static int
+dhd_msgbuf_wait_ioctl_cmplt(dhd_pub_t *dhd, uint32 len, void *buf)
+{
+	dhd_prot_t *prot = dhd->prot;
+	int timeleft;
+	unsigned long flags;
+	int ret = 0;
+	static uint cnt = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (dhd_query_bus_erros(dhd)) {
+		ret = -EIO;
+		goto out;
+	}
+#ifdef GDB_PROXY
+	/* Loop while timeout is caused by firmware stop in GDB */
+	GDB_PROXY_TIMEOUT_DO(dhd) {
+			timeleft = dhd_os_ioctl_resp_wait(dhd, (uint *)&prot->ioctl_received);
+	} GDB_PROXY_TIMEOUT_WHILE(timeleft == 0);
+#else /* GDB_PROXY */
+	timeleft = dhd_os_ioctl_resp_wait(dhd, (uint *)&prot->ioctl_received);
+#endif /* else GDB_PROXY */
+
+#ifdef DHD_RECOVER_TIMEOUT
+	if (prot->ioctl_received == 0) {
+		uint32 intstatus = si_corereg(dhd->bus->sih,
+			dhd->bus->sih->buscoreidx, dhd->bus->pcie_mailbox_int, 0, 0);
+		int host_irq_disbled = dhdpcie_irq_disabled(dhd->bus);
+		if ((intstatus) && (intstatus != (uint32)-1) &&
+			(timeleft == 0) && (!dhd_query_bus_erros(dhd))) {
+			DHD_ERROR(("%s: resumed on timeout for IOVAR happened. intstatus=%x"
+				" host_irq_disabled=%d\n",
+				__FUNCTION__, intstatus, host_irq_disbled));
+			dhd_pcie_intr_count_dump(dhd);
+			dhd_print_tasklet_status(dhd);
+			dhd_prot_ctrl_info_print(dhd);
+			/* Clear Interrupts */
+			dhdpcie_bus_clear_intstatus(dhd->bus);
+			if (dhd_prot_check_pending_ctrl_cmpls(dhd)) {
+				DHD_ERROR(("##### %s: iovar timeout trying again #####\n",
+					__FUNCTION__));
+				dhd_schedule_delayed_dpc_on_dpc_cpu(dhd, 0);
+				timeleft = dhd_os_ioctl_resp_wait(dhd,
+						(uint *)&prot->ioctl_received);
+			}
+		}
+	}
+#endif /* DHD_RECOVER_TIMEOUT */
+
+	if (dhd->conf->ctrl_resched > 0 && timeleft == 0 && (!dhd_query_bus_erros(dhd))) {
+		cnt++;
+		if (cnt <= dhd->conf->ctrl_resched) {
+			uint buscorerev = dhd->bus->sih->buscorerev;
+			uint32 intstatus = 0, intmask = 0;
+			intstatus = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, PCIMailBoxInt(buscorerev), 0, 0);
+			intmask = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, PCIMailBoxMask(buscorerev), 0, 0);
+			if (intstatus) {
+				DHD_ERROR(("%s: reschedule dhd_dpc, cnt=%d, intstatus=0x%x, intmask=0x%x\n",
+					__FUNCTION__, cnt, intstatus, intmask));
+				dhd->bus->intstatus = intstatus;
+				dhd->bus->ipend = TRUE;
+				dhd->bus->dpc_sched = TRUE;
+				dhd_sched_dpc(dhd);
+				timeleft = dhd_os_ioctl_resp_wait(dhd, &prot->ioctl_received);
+			}
+		}
+	} else {
+		cnt = 0;
+	}
+
+	if (timeleft == 0 && (!dhd->dongle_trap_data) && (!dhd_query_bus_erros(dhd))) {
+		/* Dump iovar name */
+		dhd_msgbuf_dump_iovar_name(dhd);
+		dhd_validate_pcie_link_cbp_wlbp(dhd->bus);
+		if (dhd->bus->link_state != DHD_PCIE_ALL_GOOD) {
+			DHD_ERROR(("%s: bus->link_state:%d\n",
+				__FUNCTION__, dhd->bus->link_state));
+			ret = -EREMOTEIO;
+			goto out;
+		}
+
+		if (dhd->check_trap_rot) {
+			/* check dongle trap first */
+			DHD_ERROR(("Check dongle trap in the case of iovar timeout\n"));
+			dhd_bus_checkdied(dhd->bus, NULL, 0);
+
+			if (dhd->dongle_trap_occured) {
+				ret = -EREMOTEIO;
+				goto out;
+			}
+		}
+		/* check if resumed on time out related to scheduling issue */
+		dhd->is_sched_error = dhd_bus_query_dpc_sched_errors(dhd);
+
+		dhd->iovar_timeout_occured = TRUE;
+		dhd_msgbuf_iovar_timeout_dump(dhd);
+
+#ifdef DHD_FW_COREDUMP
+		/* Collect socram dump */
+		if (dhd->memdump_enabled) {
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_RESUMED_ON_TIMEOUT;
+			dhd_bus_mem_dump(dhd);
+		}
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+		dhd->alert_reason = ALERT_IOCTL_TIMEOUT;
+		dhd_os_send_alert_message(dhd);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+		ret = -ETIMEDOUT;
+		goto out;
+	} else {
+		if (prot->ioctl_received != IOCTL_RETURN_ON_SUCCESS) {
+			DHD_ERROR(("%s: IOCTL failure due to ioctl_received = %d\n",
+				__FUNCTION__, prot->ioctl_received));
+			DHD_ERROR(("%s: setting iovar_timeout_occured\n", __FUNCTION__));
+			dhd->iovar_timeout_occured = TRUE;
+			ret = -EINVAL;
+			goto out;
+		}
+		dhd->rxcnt_timeout = 0;
+		dhd->rx_ctlpkts++;
+		DHD_CTL(("%s: ioctl resp resumed, got %d\n",
+			__FUNCTION__, prot->ioctl_resplen));
+	}
+
+	if (dhd->prot->ioctl_resplen > len)
+		dhd->prot->ioctl_resplen = (uint16)len;
+	if (buf)
+		bcopy(dhd->prot->retbuf.va, buf, dhd->prot->ioctl_resplen);
+
+	ret = (int)(dhd->prot->ioctl_status);
+
+out:
+	DHD_GENERAL_LOCK(dhd, flags);
+	dhd->prot->ioctl_state = 0;
+	dhd->prot->ioctl_resplen = 0;
+	dhd->prot->ioctl_received = IOCTL_WAIT;
+	dhd->prot->curr_ioctl_cmd = 0;
+	DHD_GENERAL_UNLOCK(dhd, flags);
+
+	return ret;
+} /* dhd_msgbuf_wait_ioctl_cmplt */
+
+static int
+dhd_msgbuf_set_ioctl(dhd_pub_t *dhd, int ifidx, uint cmd, void *buf, uint len, uint8 action)
+{
+	int ret = 0;
+
+	DHD_TRACE(("%s: Enter \n", __FUNCTION__));
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s : PCIe link is down. we have nothing to do\n",
+			__FUNCTION__));
+		return -EIO;
+	}
+
+	if (dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
+		return -EIO;
+	}
+
+	/* don't talk to the dongle if fw is about to be reloaded */
+	if (dhd->hang_was_sent) {
+		DHD_ERROR(("%s: HANG was sent up earlier. Not talking to the chip\n",
+			__FUNCTION__));
+		return -EIO;
+	}
+
+	DHD_CTL(("ACTION %d ifdix %d cmd %d len %d \n",
+		action, ifidx, cmd, len));
+
+#ifdef REPORT_FATAL_TIMEOUTS
+	/*
+	 * These timers "should" be started before sending H2D interrupt.
+	 * Think of the scenario where H2D interrupt is fired and the Dongle
+	 * responds back immediately. From the DPC we would stop the cmd, bus
+	 * timers. But the process context could have switched out leading to
+	 * a situation where the timers are Not started yet, but are actually stopped.
+	 *
+	 * Disable preemption from the time we start the timer until we are done
+	 * with seding H2D interrupts.
+	 */
+	OSL_DISABLE_PREEMPTION(dhd->osh);
+	dhd_set_request_id(dhd, dhd->prot->ioctl_trans_id+1, cmd);
+	dhd_start_cmd_timer(dhd);
+	dhd_start_bus_timer(dhd);
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+	/* Fill up msgbuf for ioctl req */
+	ret = dhd_fillup_ioct_reqst(dhd, (uint16)len, cmd, buf, ifidx);
+
+#ifdef REPORT_FATAL_TIMEOUTS
+	/* For some reason if we fail to ring door bell, stop the timers */
+	if (ret < 0) {
+		DHD_ERROR(("%s(): dhd_fillup_ioct_reqst failed \r\n", __FUNCTION__));
+		dhd_stop_cmd_timer(dhd);
+		dhd_stop_bus_timer(dhd);
+		OSL_ENABLE_PREEMPTION(dhd->osh);
+		goto done;
+	}
+
+	OSL_ENABLE_PREEMPTION(dhd->osh);
+#else
+	if (ret < 0) {
+		DHD_ERROR(("%s(): dhd_fillup_ioct_reqst failed \r\n", __FUNCTION__));
+		goto done;
+	}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+	ret = dhd_msgbuf_wait_ioctl_cmplt(dhd, len, buf);
+
+done:
+	return ret;
+}
+
+/** Called by upper DHD layer. Handles a protocol control response asynchronously. */
+int dhd_prot_ctl_complete(dhd_pub_t *dhd)
+{
+	return 0;
+}
+
+/** Called by upper DHD layer. Check for and handle local prot-specific iovar commands */
+int dhd_prot_iovar_op(dhd_pub_t *dhd, const char *name,
+                             void *params, int plen, void *arg, int len, bool set)
+{
+	return BCME_UNSUPPORTED;
+}
+
+#ifdef DHD_DUMP_PCIE_RINGS
+int dhd_d2h_h2d_ring_dump(dhd_pub_t *dhd, void *file, const void *user_buf,
+	unsigned long *file_posn, bool file_write)
+{
+	dhd_prot_t *prot;
+	msgbuf_ring_t *ring;
+	int ret = 0;
+	uint16 h2d_flowrings_total;
+	uint16 flowid;
+
+	if (!(dhd) || !(dhd->prot)) {
+		goto exit;
+	}
+	prot = dhd->prot;
+
+	/* Below is the same ring dump sequence followed in parser as well. */
+	ring = &prot->h2dring_ctrl_subn;
+	if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+		goto exit;
+
+	ring = &prot->h2dring_rxp_subn;
+	if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+		goto exit;
+
+	ring = &prot->d2hring_ctrl_cpln;
+	if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+		goto exit;
+
+	ring = &prot->d2hring_tx_cpln;
+	if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+		goto exit;
+
+	ring = &prot->d2hring_rx_cpln;
+	if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+		goto exit;
+
+	h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
+	FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
+		if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0) {
+			goto exit;
+		}
+	}
+
+#ifdef EWP_EDL
+	if (dhd->dongle_edl_support) {
+		ring = prot->d2hring_edl;
+		if ((ret = dhd_edl_ring_hdr_write(dhd, ring, file, user_buf, file_posn)) < 0)
+			goto exit;
+	}
+	else if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6 && !dhd->dongle_edl_support)
+#else
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6)
+#endif /* EWP_EDL */
+	{
+		ring = prot->h2dring_info_subn;
+		if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+			goto exit;
+
+		ring = prot->d2hring_info_cpln;
+		if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
+			goto exit;
+	}
+
+exit :
+	return ret;
+}
+
+/* Write to file */
+static
+int dhd_ring_write(dhd_pub_t *dhd, msgbuf_ring_t *ring, void *file,
+	const void *user_buf, unsigned long *file_posn)
+{
+	int ret = BCME_ERROR;
+
+	if (ring == NULL) {
+		DHD_ERROR(("%s: Ring not initialised, failed to dump ring contents\n",
+			__FUNCTION__));
+		goto exit;
+	}
+	if (file) {
+		ret = dhd_os_write_file_posn(file, file_posn, &ring->max_items,
+				sizeof(ring->max_items));
+		if (ret < 0) {
+			DHD_ERROR(("%s: Error writing ring max_items to file !\n",
+				__FUNCTION__));
+			goto exit;
+		}
+		ret = dhd_os_write_file_posn(file, file_posn, &ring->item_len,
+				sizeof(ring->item_len));
+		if (ret < 0) {
+			DHD_ERROR(("%s: Error writing ring length to file !\n",
+				__FUNCTION__));
+			goto exit;
+		}
+		ret = dhd_os_write_file_posn(file, file_posn, (char *)(ring->dma_buf.va),
+				((unsigned long)(ring->max_items) * (ring->item_len)));
+		if (ret < 0) {
+			DHD_ERROR(("%s: write file error !\n", __FUNCTION__));
+			goto exit;
+		}
+	} else {
+		ret = dhd_export_debug_data(&ring->max_items, NULL, user_buf,
+			sizeof(ring->max_items), (int *)file_posn);
+		if (ret < 0) {
+			goto exit;
+		}
+		ret = dhd_export_debug_data(&ring->item_len, NULL, user_buf,
+			sizeof(ring->item_len), (int *)file_posn);
+		if (ret < 0) {
+			goto exit;
+		}
+		ret = dhd_export_debug_data((char *)(ring->dma_buf.va), NULL, user_buf,
+			((unsigned long)(ring->max_items) * (ring->item_len)), (int *)file_posn);
+		if (ret < 0) {
+			goto exit;
+		}
+	}
+exit:
+	return ret;
+}
+
+#ifdef EWP_EDL
+/* Write to file */
+static
+int dhd_edl_ring_hdr_write(dhd_pub_t *dhd, msgbuf_ring_t *ring, void *file, const void *user_buf,
+	unsigned long *file_posn)
+{
+	int ret = 0, nitems = 0;
+	char *buf = NULL, *ptr = NULL;
+	uint8 *msg_addr = NULL;
+	uint16	rd = 0;
+
+	if (ring == NULL) {
+		DHD_ERROR(("%s: Ring not initialised, failed to dump ring contents\n",
+			__FUNCTION__));
+		ret = BCME_ERROR;
+		goto done;
+	}
+
+	buf = MALLOCZ(dhd->osh, (D2HRING_EDL_MAX_ITEM * D2HRING_EDL_HDR_SIZE));
+	if (buf == NULL) {
+		DHD_ERROR(("%s: buffer allocation failed\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		goto done;
+	}
+	ptr = buf;
+
+	for (; nitems < D2HRING_EDL_MAX_ITEM; nitems++, rd++) {
+		msg_addr = (uint8 *)ring->dma_buf.va + (rd * ring->item_len);
+		ret = memcpy_s(ptr, D2HRING_EDL_HDR_SIZE, (char *)msg_addr, ring->item_len);
+		if (ret) {
+			DHD_ERROR(("D2HRING_EDL_HDR(%d) memcpy failed:%d, destsz:%d, n:%d\n",
+				rd, ret, D2HRING_EDL_HDR_SIZE, ring->item_len));
+		}
+		ptr += D2HRING_EDL_HDR_SIZE;
+	}
+	if (file) {
+		ret = dhd_os_write_file_posn(file, file_posn, buf,
+				(D2HRING_EDL_HDR_SIZE * D2HRING_EDL_MAX_ITEM));
+		if (ret < 0) {
+			DHD_ERROR(("%s: write file error !\n", __FUNCTION__));
+			goto done;
+		}
+	}
+	else {
+		ret = dhd_export_debug_data(buf, NULL, user_buf,
+			(D2HRING_EDL_HDR_SIZE * D2HRING_EDL_MAX_ITEM), file_posn);
+	}
+
+done:
+	if (buf) {
+		MFREE(dhd->osh, buf, (D2HRING_EDL_MAX_ITEM * D2HRING_EDL_HDR_SIZE));
+	}
+	return ret;
+}
+#endif /* EWP_EDL */
+#endif /* DHD_DUMP_PCIE_RINGS */
+
+static void
+dhd_prot_print_ring_info(dhd_pub_t *dhd, struct bcmstrbuf *strbuf)
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	bcm_bprintf(strbuf, "max RX bufs to post: %d, \t posted %d \n",
+		dhd->prot->max_rxbufpost, dhd->prot->rxbufpost);
+
+	bcm_bprintf(strbuf, "Total RX bufs posted: %d, \t RX cpl got %d \n",
+		dhd->prot->tot_rxbufpost, dhd->prot->tot_rxcpl);
+
+	bcm_bprintf(strbuf, "Total TX packets: %lu, \t TX cpl got %lu \n",
+		dhd->actual_tx_pkts, dhd->tot_txcpl);
+
+	bcm_bprintf(strbuf,
+		"%14s %18s %18s %17s %17s %14s %14s %10s\n",
+		"Type", "TRD: HLRD: HDRD", "TWR: HLWR: HDWR", "BASE(VA)", "BASE(PA)",
+		"WORK_ITEM_SIZE", "MAX_WORK_ITEMS", "TOTAL_SIZE");
+	bcm_bprintf(strbuf, "%14s", "H2DCtrlPost");
+	dhd_prot_print_flow_ring(dhd, &prot->h2dring_ctrl_subn, TRUE, strbuf,
+		" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	bcm_bprintf(strbuf, "%14s", "D2HCtrlCpl");
+	dhd_prot_print_flow_ring(dhd, &prot->d2hring_ctrl_cpln, FALSE, strbuf,
+		" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	bcm_bprintf(strbuf, "%14s", "H2DRxPost");
+	dhd_prot_print_flow_ring(dhd, &prot->h2dring_rxp_subn, TRUE, strbuf,
+		" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	bcm_bprintf(strbuf, "%14s", "D2HRxCpl");
+	dhd_prot_print_flow_ring(dhd, &prot->d2hring_rx_cpln, FALSE, strbuf,
+		" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	bcm_bprintf(strbuf, "%14s", "D2HTxCpl");
+	dhd_prot_print_flow_ring(dhd, &prot->d2hring_tx_cpln, FALSE, strbuf,
+		" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	if (dhd->prot->h2dring_info_subn != NULL && dhd->prot->d2hring_info_cpln != NULL) {
+		bcm_bprintf(strbuf, "%14s", "H2DRingInfoSub");
+		dhd_prot_print_flow_ring(dhd, prot->h2dring_info_subn, TRUE, strbuf,
+			" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+		bcm_bprintf(strbuf, "%14s", "D2HRingInfoCpl");
+		dhd_prot_print_flow_ring(dhd, prot->d2hring_info_cpln, FALSE, strbuf,
+			" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	}
+	if (dhd->prot->d2hring_edl != NULL) {
+		bcm_bprintf(strbuf, "%14s", "D2HRingEDL");
+		dhd_prot_print_flow_ring(dhd, prot->d2hring_edl, FALSE, strbuf,
+			" %5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d\n");
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+
+/** Add prot dump output to a buffer */
+void dhd_prot_dump(dhd_pub_t *dhd, struct bcmstrbuf *b)
+{
+#if defined(BCM_ROUTER_DHD)
+	bcm_bprintf(b, "DHD Router: 1GMAC HotBRC forwarding mode\n");
+#endif /* BCM_ROUTER_DHD */
+
+	if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_SEQNUM)
+		bcm_bprintf(b, "\nd2h_sync: SEQNUM:");
+	else if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_XORCSUM)
+		bcm_bprintf(b, "\nd2h_sync: XORCSUM:");
+	else
+		bcm_bprintf(b, "\nd2h_sync: NONE:");
+	bcm_bprintf(b, " d2h_sync_wait max<%lu> tot<%lu>\n",
+		dhd->prot->d2h_sync_wait_max, dhd->prot->d2h_sync_wait_tot);
+
+	bcm_bprintf(b, "\nDongle DMA Indices: h2d %d  d2h %d index size %d bytes\n",
+		dhd->dma_h2d_ring_upd_support,
+		dhd->dma_d2h_ring_upd_support,
+		dhd->prot->rw_index_sz);
+#ifdef DHD_DMA_INDICES_SEQNUM
+	bcm_bprintf(b, "host_seqnum %u dngl_seqnum %u\n", dhd_prot_read_seqnum(dhd, TRUE),
+		dhd_prot_read_seqnum(dhd, FALSE));
+#endif /* DHD_DMA_INDICES_SEQNUM */
+
+	dhd_prot_counters(dhd, b, FALSE, FALSE);
+}
+
+void dhd_prot_counters(dhd_pub_t *dhd, struct bcmstrbuf *b,
+	bool print_ringinfo, bool print_pktidinfo)
+{
+	bcm_bprintf(b, "\nTX Post, doorbell(DB) stats:\n======================\n");
+
+	bcm_bprintf(b, "h2d_max_txpost: %d, prot->h2d_max_txpost: %d\n",
+		h2d_max_txpost, dhd->prot->h2d_max_txpost);
+	if (dhd->htput_support) {
+		bcm_bprintf(b, "h2d_htput_max_txpost: %d, prot->h2d_htput_max_txpost: %d\n",
+			h2d_htput_max_txpost, dhd->prot->h2d_htput_max_txpost);
+	}
+	bcm_bprintf(b, "pktid_txq_start_cnt: %d\n", dhd->prot->pktid_txq_start_cnt);
+	bcm_bprintf(b, "pktid_txq_stop_cnt: %d\n", dhd->prot->pktid_txq_stop_cnt);
+	bcm_bprintf(b, "pktid_depleted_cnt: %d\n", dhd->prot->pktid_depleted_cnt);
+	bcm_bprintf(b, "txcpl_db_cnt: %d\n", dhd->prot->txcpl_db_cnt);
+	bcm_bprintf(b, "tx_h2d_db_cnt:%llu\n", dhd->prot->tx_h2d_db_cnt);
+	bcm_bprintf(b, "\n");
+
+#ifdef AGG_H2D_DB
+	bcm_bprintf(b, "Aggregated DB, workitem stats:\n======================\n");
+	bcm_bprintf(b, "agg_h2d_db_enab:%d agg_h2d_db_timeout:%d agg_h2d_db_inflight_thresh:%d\n",
+		agg_h2d_db_enab, agg_h2d_db_timeout, agg_h2d_db_inflight_thresh);
+	bcm_bprintf(b, "agg_h2d_db: timer_db_cnt:%d direct_db_cnt:%d\n",
+		dhd->prot->agg_h2d_db_info.timer_db_cnt, dhd->prot->agg_h2d_db_info.direct_db_cnt);
+	dhd_agg_inflight_stats_dump(dhd, b);
+#endif /* AGG_H2D_DB */
+	bcm_bprintf(b, "\nrx_buf_burst: %d\n", dhd->prot->rx_buf_burst);
+	bcm_bprintf(b, "\n");
+
+	if (print_ringinfo) {
+		bcm_bprintf(b, "\nRing Info:\n=========\n");
+		bcm_bprintf(b, "[RD=read ptr; WR=write ptr; T=TCM; H=Host;"
+			" L=Local; D=DMA index'd]\n");
+		dhd_prot_print_ring_info(dhd, b);
+	}
+
+	if (print_pktidinfo) {
+		bcm_bprintf(b, "active_tx_count %d	 pktidmap_avail(ctrl/rx/tx) %d %d %d\n",
+			OSL_ATOMIC_READ(dhd->osh, &dhd->prot->active_tx_count),
+			DHD_PKTID_AVAIL(dhd->prot->pktid_ctrl_map),
+			DHD_PKTID_AVAIL(dhd->prot->pktid_rx_map),
+			DHD_PKTID_AVAIL(dhd->prot->pktid_tx_map));
+	}
+
+	bcm_bprintf(b, "\n");
+}
+
+/* Update local copy of dongle statistics */
+void dhd_prot_dstats(dhd_pub_t *dhd)
+{
+	return;
+}
+
+/** Called by upper DHD layer */
+int dhd_process_pkt_reorder_info(dhd_pub_t *dhd, uchar *reorder_info_buf,
+	uint reorder_info_len, void **pkt, uint32 *free_buf_count)
+{
+	return 0;
+}
+
+/** Debug related, post a dummy message to interrupt dongle. Used to process cons commands. */
+int
+dhd_post_dummy_msg(dhd_pub_t *dhd)
+{
+	unsigned long flags;
+	hostevent_hdr_t *hevent = NULL;
+	uint16 alloced = 0;
+
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	hevent = (hostevent_hdr_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+
+	if (hevent == NULL) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return -1;
+	}
+
+	/* CMN msg header */
+	hevent->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+	hevent->msg.msg_type = MSG_TYPE_HOST_EVNT;
+	hevent->msg.if_id = 0;
+	hevent->msg.flags = ring->current_phase;
+
+	/* Event payload */
+	hevent->evnt_pyld = htol32(HOST_EVENT_CONS_CMD);
+
+	/* Since, we are filling the data directly into the bufptr obtained
+	 * from the msgbuf, we can directly call the write_complete
+	 */
+	dhd_prot_ring_write_complete(dhd, ring, hevent, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return 0;
+}
+
+/**
+ * If exactly_nitems is true, this function will allocate space for nitems or fail
+ * If exactly_nitems is false, this function will allocate space for nitems or less
+ */
+static void *
+BCMFASTPATH(dhd_prot_alloc_ring_space)(dhd_pub_t *dhd, msgbuf_ring_t *ring,
+	uint16 nitems, uint16 * alloced, bool exactly_nitems)
+{
+	void * ret_buf;
+
+	if (nitems == 0) {
+		DHD_ERROR(("%s: nitems is 0 - ring(%s)\n", __FUNCTION__, ring->name));
+		return NULL;
+	}
+
+	/* Alloc space for nitems in the ring */
+	ret_buf = dhd_prot_get_ring_space(ring, nitems, alloced, exactly_nitems);
+
+	if (ret_buf == NULL) {
+		/* if alloc failed , invalidate cached read ptr */
+		if (dhd->dma_d2h_ring_upd_support) {
+			ring->rd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &(ring->rd), RING_RD_UPD, ring->idx);
+		}
+
+		/* Try allocating once more */
+		ret_buf = dhd_prot_get_ring_space(ring, nitems, alloced, exactly_nitems);
+
+		if (ret_buf == NULL) {
+			DHD_INFO(("%s: Ring space not available  \n", ring->name));
+			return NULL;
+		}
+	}
+
+	if (ret_buf == HOST_RING_BASE(ring)) {
+		DHD_INFO(("%s: setting the phase now\n", ring->name));
+		if (DHD_IS_FLOWRING(ring->idx, dhd->bus->max_tx_flowrings) &&
+			dhd->fr_phase_nibble) {
+			ring->current_phase =
+				((ring->current_phase == BCMPCIE_FLOWRING_PHASE_NIBBLE_INIT)
+				? BCMPCIE_FLOWRING_PHASE_NIBBLE_WRAP :
+				BCMPCIE_FLOWRING_PHASE_NIBBLE_INIT);
+		} else {
+			ring->current_phase = ring->current_phase ?
+				0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
+		}
+	}
+
+	/* Return alloced space */
+	return ret_buf;
+}
+
+/**
+ * Non inline ioct request.
+ * Form a ioctl request first as per ioctptr_reqst_hdr_t header in the circular buffer
+ * Form a separate request buffer where a 4 byte cmn header is added in the front
+ * buf contents from parent function is copied to remaining section of this buffer
+ */
+static int
+dhd_fillup_ioct_reqst(dhd_pub_t *dhd, uint16 len, uint cmd, void* buf, int ifidx)
+{
+	dhd_prot_t *prot = dhd->prot;
+	ioctl_req_msg_t *ioct_rqst;
+	void * ioct_buf;	/* For ioctl payload */
+	uint32	ioct_buf_len;
+	uint16  rqstlen, resplen;
+	unsigned long flags;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+#ifdef DBG_DW_CHK_PCIE_READ_LATENCY
+	uint16 data;
+	ktime_t begin_time, end_time;
+	s64 diff_ns;
+#endif /* DBG_DW_CHK_PCIE_READ_LATENCY */
+
+	if (dhd_query_bus_erros(dhd)) {
+		return -EIO;
+	}
+
+	rqstlen = len;
+	resplen = len;
+
+	/* Limit ioct request to MSGBUF_MAX_MSG_SIZE bytes including hdrs */
+	/* 8K allocation of dongle buffer fails */
+	/* dhd doesnt give separate input & output buf lens */
+	/* so making the assumption that input length can never be more than 2k */
+	rqstlen = MIN(rqstlen, MSGBUF_IOCTL_MAX_RQSTLEN);
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+
+#ifdef DBG_DW_CHK_PCIE_READ_LATENCY
+	preempt_disable();
+	begin_time = ktime_get();
+	dhd_bus_cmn_readshared(dhd->bus, &data, RING_RD_UPD, ring->idx);
+	end_time = ktime_get();
+	preempt_enable();
+	diff_ns = ktime_to_ns(ktime_sub(end_time, begin_time));
+	/* Check if the delta is greater than 1 msec */
+	if (diff_ns > (1 * NSEC_PER_MSEC)) {
+		DHD_ERROR(("%s: found latency over 1ms (%lld ns), ds state=%d\n", __func__,
+		       diff_ns, dhdpcie_bus_get_pcie_inband_dw_state(dhd->bus)));
+	}
+#endif /* DBG_DW_CHK_PCIE_READ_LATENCY */
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	if (prot->ioctl_state) {
+		DHD_ERROR(("%s: pending ioctl %02x\n", __FUNCTION__, prot->ioctl_state));
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_BUSY;
+	} else {
+		prot->ioctl_state = MSGBUF_IOCTL_ACK_PENDING | MSGBUF_IOCTL_RESP_PENDING;
+	}
+
+	/* Request for cbuf space */
+	ioct_rqst = (ioctl_req_msg_t*)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+	if (ioct_rqst == NULL) {
+		DHD_ERROR(("couldn't allocate space on msgring to send ioctl request\n"));
+		prot->ioctl_state = 0;
+		prot->curr_ioctl_cmd = 0;
+		prot->ioctl_received = IOCTL_WAIT;
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return -1;
+	}
+
+	/* Common msg buf hdr */
+	ioct_rqst->cmn_hdr.msg_type = MSG_TYPE_IOCTLPTR_REQ;
+	ioct_rqst->cmn_hdr.if_id = (uint8)ifidx;
+	ioct_rqst->cmn_hdr.flags = ring->current_phase;
+	ioct_rqst->cmn_hdr.request_id = htol32(DHD_IOCTL_REQ_PKTID);
+	ioct_rqst->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+
+	ioct_rqst->cmd = htol32(cmd);
+	prot->curr_ioctl_cmd = cmd;
+	ioct_rqst->output_buf_len = htol16(resplen);
+	prot->ioctl_trans_id++;
+	ioct_rqst->trans_id = prot->ioctl_trans_id;
+
+	/* populate ioctl buffer info */
+	ioct_rqst->input_buf_len = htol16(rqstlen);
+	ioct_rqst->host_input_buf_addr.high = htol32(PHYSADDRHI(prot->ioctbuf.pa));
+	ioct_rqst->host_input_buf_addr.low = htol32(PHYSADDRLO(prot->ioctbuf.pa));
+	/* copy ioct payload */
+	ioct_buf = (void *) prot->ioctbuf.va;
+	ioct_buf_len = prot->ioctbuf.len;
+
+	prot->ioctl_fillup_time = OSL_LOCALTIME_NS();
+
+	if (buf) {
+		int ret = 0;
+		ret = memcpy_s(ioct_buf, ioct_buf_len, buf, len);
+		if (ret) {
+			DHD_ERROR(("ioct_buf memcopy failed:%d, destsz:%d, n:%d\n",
+				ret, ioct_buf_len, len));
+			return BCME_ERROR;
+		}
+	}
+
+	OSL_CACHE_FLUSH((void *) prot->ioctbuf.va, len);
+
+	if (!ISALIGNED(ioct_buf, DMA_ALIGN_LEN))
+		DHD_ERROR(("host ioct address unaligned !!!!! \n"));
+
+	DHD_CTL(("submitted IOCTL request request_id %d, cmd %d, output_buf_len %d, tx_id %d\n",
+		ioct_rqst->cmn_hdr.request_id, cmd, ioct_rqst->output_buf_len,
+		ioct_rqst->trans_id));
+
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, ioct_rqst, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return 0;
+} /* dhd_fillup_ioct_reqst */
+
+/**
+ * dhd_prot_ring_attach - Initialize the msgbuf_ring object and attach a
+ * DMA-able buffer to it. The ring is NOT tagged as inited until all the ring
+ * information is posted to the dongle.
+ *
+ * Invoked in dhd_prot_attach for the common rings, and in dhd_prot_init for
+ * each flowring in pool of flowrings.
+ *
+ * returns BCME_OK=0 on success
+ * returns non-zero negative error value on failure.
+ */
+static int
+dhd_prot_ring_attach(dhd_pub_t *dhd, msgbuf_ring_t *ring, const char *name,
+	uint16 max_items, uint16 item_len, uint16 ringid)
+{
+	int dma_buf_alloced = BCME_NOMEM;
+	uint32 dma_buf_len;
+	dhd_prot_t *prot = dhd->prot;
+	uint16 max_flowrings = dhd->bus->max_tx_flowrings;
+	dhd_dma_buf_t *dma_buf = NULL;
+#if defined(EWP_EDL) || defined(FLOW_RING_PREALLOC)
+	int ret;
+#endif /* EWP_EDL || FLOW_RING_PREALLOC */
+
+	ASSERT(ring);
+	ASSERT(name);
+	ASSERT((max_items < 0xFFFF) && (item_len < 0xFFFF) && (ringid < 0xFFFF));
+
+	/* Init name */
+	strlcpy((char *)ring->name, name, sizeof(ring->name));
+
+	ring->idx = ringid;
+
+	/* Use HTPUT max items */
+	if (dhd->htput_support && DHD_IS_FLOWRING(ringid, max_flowrings) &&
+		DHD_IS_FLOWID_HTPUT(dhd, DHD_RINGID_TO_FLOWID(ringid))) {
+		max_items = prot->h2d_htput_max_txpost;
+	}
+
+	dma_buf_len = max_items * item_len;
+
+	ring->max_items = max_items;
+	ring->item_len = item_len;
+
+	/* A contiguous space may be reserved for all flowrings */
+	if (DHD_IS_FLOWRING(ringid, max_flowrings) && (prot->flowrings_dma_buf.va)) {
+		/* Carve out from the contiguous DMA-able flowring buffer */
+		uint16 flowid;
+		uint32 base_offset;
+		dhd_dma_buf_t *rsv_buf = &prot->flowrings_dma_buf;
+
+		dma_buf = &ring->dma_buf;
+
+		flowid = DHD_RINGID_TO_FLOWID(ringid);
+		base_offset = (flowid - BCMPCIE_H2D_COMMON_MSGRINGS) * dma_buf_len;
+
+		ASSERT(base_offset + dma_buf_len <= rsv_buf->len);
+
+		dma_buf->len = dma_buf_len;
+		dma_buf->va = (void *)((uintptr)rsv_buf->va + base_offset);
+		PHYSADDRHISET(dma_buf->pa, PHYSADDRHI(rsv_buf->pa));
+		PHYSADDRLOSET(dma_buf->pa, PHYSADDRLO(rsv_buf->pa) + base_offset);
+
+		/* On 64bit, contiguous space may not span across 0x00000000FFFFFFFF */
+		ASSERT(PHYSADDRLO(dma_buf->pa) >= PHYSADDRLO(rsv_buf->pa));
+
+		dma_buf->dmah   = rsv_buf->dmah;
+		dma_buf->secdma = rsv_buf->secdma;
+
+		(void)dhd_dma_buf_audit(dhd, &ring->dma_buf);
+	} else {
+#ifdef EWP_EDL
+		if (ring == dhd->prot->d2hring_edl) {
+			/* For EDL ring, memory is alloced during attach,
+			* so just need to copy the dma_buf to the ring's dma_buf
+			*/
+			ret = memcpy_s(&ring->dma_buf, sizeof(ring->dma_buf),
+				&dhd->edl_ring_mem, sizeof(dhd->edl_ring_mem));
+			if (ret != BCME_OK) {
+				DHD_ERROR(("%s: memcpy_s edl_ring_mem failed\n", __FUNCTION__));
+				ASSERT(0);
+				return BCME_ERROR;
+			}
+			dma_buf = &ring->dma_buf;
+			if (dma_buf->va == NULL) {
+				return BCME_NOMEM;
+			}
+		} else
+#endif /* EWP_EDL */
+
+#ifdef FLOW_RING_PREALLOC
+		if (dhd->htput_support && DHD_IS_FLOWRING(ringid, max_flowrings) &&
+			DHD_IS_FLOWID_HTPUT(dhd, DHD_RINGID_TO_FLOWID(ringid))) {
+			int htput_ringid = ringid % HTPUT_TOTAL_FLOW_RINGS;
+			/* copy pre-allocated mem with ringid */
+			ret = memcpy_s(&ring->dma_buf, sizeof(ring->dma_buf),
+				&dhd->prot->htput_ring_buf[htput_ringid],
+				sizeof(dhd->prot->htput_ring_buf[htput_ringid]));
+			if (ret != BCME_OK) {
+				DHD_ERROR(("%s: memcpy_s htput_ring_buf failed\n", __FUNCTION__));
+				ASSERT(0);
+				return BCME_ERROR;
+			}
+			dma_buf = &ring->dma_buf;
+			if (dma_buf->va == NULL) {
+				return BCME_NOMEM;
+			}
+		} else if (DHD_IS_FLOWRING(ringid, max_flowrings)) {
+			int non_htput_ringid = ringid % dhd->non_htput_total_flow_rings;
+			/* copy pre-allocated mem with ringid : Non-HTPUT ring */
+			ret = memcpy_s(&ring->dma_buf, sizeof(ring->dma_buf),
+				&dhd->prot->flow_ring_buf[non_htput_ringid],
+				sizeof(dhd->prot->flow_ring_buf[non_htput_ringid]));
+			if (ret != BCME_OK) {
+				DHD_ERROR(("%s: memcpy_s non_htput_ring_buf failed\n",
+					__FUNCTION__));
+				ASSERT(0);
+				return BCME_ERROR;
+			}
+			dma_buf = &ring->dma_buf;
+			if (dma_buf->va == NULL) {
+				return BCME_NOMEM;
+			}
+		} else
+#endif /* FLOW_RING_PREALLOC */
+		{
+			/* Allocate a dhd_dma_buf for non-htput flowrings and static rings */
+			dma_buf_alloced = dhd_dma_buf_alloc(dhd, &ring->dma_buf, dma_buf_len);
+			if (dma_buf_alloced != BCME_OK) {
+				return BCME_NOMEM;
+			}
+		}
+	}
+
+	/* CAUTION: Save ring::base_addr in little endian format! */
+	dhd_base_addr_htolpa(&ring->base_addr, ring->dma_buf.pa);
+
+	ring->ring_lock = osl_spin_lock_init(dhd->osh);
+
+	DHD_INFO(("RING_ATTACH : %s Max item %d len item %d total size %d "
+		"ring start %p buf phys addr  %x:%x \n",
+		ring->name, ring->max_items, ring->item_len,
+		dma_buf_len, ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+		ltoh32(ring->base_addr.low_addr)));
+
+	return BCME_OK;
+} /* dhd_prot_ring_attach */
+
+/**
+ * dhd_prot_ring_init - Post the common ring information to dongle.
+ *
+ * Used only for common rings.
+ *
+ * The flowrings information is passed via the create flowring control message
+ * (tx_flowring_create_request_t) sent over the H2D control submission common
+ * ring.
+ */
+static void
+dhd_prot_ring_init(dhd_pub_t *dhd, msgbuf_ring_t *ring)
+{
+	ring->wr = 0;
+	ring->rd = 0;
+	ring->curr_rd = 0;
+
+	/* CAUTION: ring::base_addr already in Little Endian */
+	dhd_bus_cmn_writeshared(dhd->bus, &ring->base_addr,
+		sizeof(sh_addr_t), RING_BUF_ADDR, ring->idx);
+	dhd_bus_cmn_writeshared(dhd->bus, &ring->max_items,
+		sizeof(uint16), RING_MAX_ITEMS, ring->idx);
+	dhd_bus_cmn_writeshared(dhd->bus, &ring->item_len,
+		sizeof(uint16), RING_ITEM_LEN, ring->idx);
+
+	dhd_bus_cmn_writeshared(dhd->bus, &(ring->wr),
+		sizeof(uint16), RING_WR_UPD, ring->idx);
+	dhd_bus_cmn_writeshared(dhd->bus, &(ring->rd),
+		sizeof(uint16), RING_RD_UPD, ring->idx);
+
+	/* ring inited */
+	ring->inited = TRUE;
+
+} /* dhd_prot_ring_init */
+
+/**
+ * dhd_prot_ring_reset - bzero a ring's DMA-ble buffer and cache flush
+ * Reset WR and RD indices to 0.
+ */
+static void
+dhd_prot_ring_reset(dhd_pub_t *dhd, msgbuf_ring_t *ring)
+{
+	DHD_TRACE(("%s\n", __FUNCTION__));
+
+	dhd_dma_buf_reset(dhd, &ring->dma_buf);
+
+	ring->rd = ring->wr = 0;
+	ring->curr_rd = 0;
+	ring->inited = FALSE;
+	ring->create_pending = FALSE;
+}
+
+/**
+ * dhd_prot_ring_detach - Detach the DMA-able buffer and any other objects
+ * hanging off the msgbuf_ring.
+ */
+static void
+dhd_prot_ring_detach(dhd_pub_t *dhd, msgbuf_ring_t *ring)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint16 max_flowrings = dhd->bus->max_tx_flowrings;
+	ASSERT(ring);
+
+	ring->inited = FALSE;
+	/* rd = ~0, wr = ring->rd - 1, max_items = 0, len_item = ~0 */
+
+	/* If the DMA-able buffer was carved out of a pre-reserved contiguous
+	 * memory, then simply stop using it.
+	 */
+	if (DHD_IS_FLOWRING(ring->idx, max_flowrings) && (prot->flowrings_dma_buf.va)) {
+		(void)dhd_dma_buf_audit(dhd, &ring->dma_buf);
+		memset(&ring->dma_buf, 0, sizeof(dhd_dma_buf_t));
+	} else {
+#ifdef EWP_EDL
+		if (ring == dhd->prot->d2hring_edl) {
+			/* For EDL ring, do not free ring mem here,
+			* it is done in dhd_detach
+			*/
+			memset(&ring->dma_buf, 0, sizeof(ring->dma_buf));
+		} else
+#endif /* EWP_EDL */
+#ifdef FLOW_RING_PREALLOC
+		if (dhd->htput_support &&
+			DHD_IS_FLOWID_HTPUT(dhd, DHD_RINGID_TO_FLOWID(ring->idx)) &&
+			(ring->dma_buf.va)) {
+			/* htput ring is freed in dhd_detach */
+			memset(&ring->dma_buf, 0, sizeof(ring->dma_buf));
+		} else if (DHD_IS_FLOWRING(ring->idx, max_flowrings) &&
+			(ring->dma_buf.va)) {
+			/* flow ring is freed in dhd_detach */
+			memset(&ring->dma_buf, 0, sizeof(ring->dma_buf));
+		} else
+#endif /* FLOW_RING_PREALLOC */
+		{
+			/* Free non-htput flowrings and static rings */
+			dhd_dma_buf_free(dhd, &ring->dma_buf);
+		}
+	}
+
+	osl_spin_lock_deinit(dhd->osh, ring->ring_lock);
+
+} /* dhd_prot_ring_detach */
+
+/* Fetch number of H2D flowrings given the total number of h2d rings */
+uint16
+dhd_get_max_flow_rings(dhd_pub_t *dhd)
+{
+	if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6)
+		return dhd->bus->max_tx_flowrings;
+	else
+		return (dhd->bus->max_tx_flowrings - BCMPCIE_H2D_COMMON_MSGRINGS);
+}
+
+/**
+ * dhd_prot_flowrings_pool_attach - Initialize a pool of flowring msgbuf_ring_t.
+ *
+ * Allocate a pool of msgbuf_ring along with DMA-able buffers for flowrings.
+ * Dongle includes common rings when it advertizes the number of H2D rings.
+ * Allocates a pool of msgbuf_ring_t and invokes dhd_prot_ring_attach to
+ * allocate the DMA-able buffer and initialize each msgbuf_ring_t object.
+ *
+ * dhd_prot_ring_attach is invoked to perform the actual initialization and
+ * attaching the DMA-able buffer.
+ *
+ * Later dhd_prot_flowrings_pool_fetch() may be used to fetch a preallocated and
+ * initialized msgbuf_ring_t object.
+ *
+ * returns BCME_OK=0 on success
+ * returns non-zero negative error value on failure.
+ */
+static int
+dhd_prot_flowrings_pool_attach(dhd_pub_t *dhd)
+{
+	uint16 flowid;
+	msgbuf_ring_t *ring;
+	uint16 h2d_flowrings_total; /* exclude H2D common rings */
+	dhd_prot_t *prot = dhd->prot;
+	char ring_name[RING_NAME_MAX_LENGTH];
+
+	if (prot->h2d_flowrings_pool != NULL)
+		return BCME_OK; /* dhd_prot_init rentry after a dhd_prot_reset */
+
+	ASSERT(prot->h2d_rings_total == 0);
+
+	/* h2d_rings_total includes H2D common rings: ctrl and rxbuf subn */
+	prot->h2d_rings_total = (uint16)dhd_bus_max_h2d_queues(dhd->bus);
+
+	if (prot->h2d_rings_total < BCMPCIE_H2D_COMMON_MSGRINGS) {
+		DHD_ERROR(("%s: h2d_rings_total advertized as %u\n",
+			__FUNCTION__, prot->h2d_rings_total));
+		return BCME_ERROR;
+	}
+
+	/* Subtract number of H2D common rings, to determine number of flowrings */
+	h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
+
+	DHD_ERROR(("Attach flowrings pool for %d rings\n", h2d_flowrings_total));
+
+	/* Allocate pool of msgbuf_ring_t objects for all flowrings */
+	prot->h2d_flowrings_pool = (msgbuf_ring_t *)MALLOCZ(prot->osh,
+		(h2d_flowrings_total * sizeof(msgbuf_ring_t)));
+
+	if (prot->h2d_flowrings_pool == NULL) {
+		DHD_ERROR(("%s: flowrings pool for %d flowrings, alloc failure\n",
+			__FUNCTION__, h2d_flowrings_total));
+		goto fail;
+	}
+
+#ifdef FLOW_RING_PREALLOC
+	if (h2d_flowrings_total > MAX_FLOW_RINGS) {
+		DHD_ERROR(("%s: requested size %d is bigger than pre-alloc flowrings %d,\n",
+			__FUNCTION__, h2d_flowrings_total, MAX_FLOW_RINGS));
+		goto attach_fail;
+	}
+#endif /* FLOW_RING_PREALLOC */
+
+	/* Setup & Attach a DMA-able buffer to each flowring in the flowring pool */
+	FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
+		snprintf(ring_name, sizeof(ring_name), "h2dflr_%03u", flowid);
+		/* For HTPUT case max_items will be changed inside dhd_prot_ring_attach */
+		if (dhd_prot_ring_attach(dhd, ring, ring_name,
+		        prot->h2d_max_txpost, H2DRING_TXPOST_ITEMSIZE,
+		        DHD_FLOWID_TO_RINGID(flowid)) != BCME_OK) {
+			goto attach_fail;
+		}
+	}
+
+	return BCME_OK;
+
+attach_fail:
+	/* XXX: On a per project basis, one may decide whether to continue with
+	 * "fewer" flowrings, and what value of fewer suffices.
+	 */
+	dhd_prot_flowrings_pool_detach(dhd); /* Free entire pool of flowrings */
+
+fail:
+	prot->h2d_rings_total = 0;
+	return BCME_NOMEM;
+
+} /* dhd_prot_flowrings_pool_attach */
+
+/**
+ * dhd_prot_flowrings_pool_reset - Reset all msgbuf_ring_t objects in the pool.
+ * Invokes dhd_prot_ring_reset to perform the actual reset.
+ *
+ * The DMA-able buffer is not freed during reset and neither is the flowring
+ * pool freed.
+ *
+ * dhd_prot_flowrings_pool_reset will be invoked in dhd_prot_reset. Following
+ * the dhd_prot_reset, dhd_prot_init will be re-invoked, and the flowring pool
+ * from a previous flowring pool instantiation will be reused.
+ *
+ * This will avoid a fragmented DMA-able memory condition, if multiple
+ * dhd_prot_reset were invoked to reboot the dongle without a full detach/attach
+ * cycle.
+ */
+static void
+dhd_prot_flowrings_pool_reset(dhd_pub_t *dhd)
+{
+	uint16 flowid, h2d_flowrings_total;
+	msgbuf_ring_t *ring;
+	dhd_prot_t *prot = dhd->prot;
+
+	if (prot->h2d_flowrings_pool == NULL) {
+		ASSERT(prot->h2d_rings_total == 0);
+		return;
+	}
+	h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
+	/* Reset each flowring in the flowring pool */
+	FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
+		dhd_prot_ring_reset(dhd, ring);
+		ring->inited = FALSE;
+	}
+
+	/* Flowring pool state must be as-if dhd_prot_flowrings_pool_attach */
+}
+
+/**
+ * dhd_prot_flowrings_pool_detach - Free pool of msgbuf_ring along with
+ * DMA-able buffers for flowrings.
+ * dhd_prot_ring_detach is invoked to free the DMA-able buffer and perform any
+ * de-initialization of each msgbuf_ring_t.
+ */
+static void
+dhd_prot_flowrings_pool_detach(dhd_pub_t *dhd)
+{
+	int flowid;
+	msgbuf_ring_t *ring;
+	uint16 h2d_flowrings_total; /* exclude H2D common rings */
+	dhd_prot_t *prot = dhd->prot;
+
+	if (prot->h2d_flowrings_pool == NULL) {
+		ASSERT(prot->h2d_rings_total == 0);
+		return;
+	}
+
+	h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
+	/* Detach the DMA-able buffer for each flowring in the flowring pool */
+	FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
+		dhd_prot_ring_detach(dhd, ring);
+	}
+
+	MFREE(prot->osh, prot->h2d_flowrings_pool,
+		(h2d_flowrings_total * sizeof(msgbuf_ring_t)));
+
+	prot->h2d_rings_total = 0;
+
+} /* dhd_prot_flowrings_pool_detach */
+
+/**
+ * dhd_prot_flowrings_pool_fetch - Fetch a preallocated and initialized
+ * msgbuf_ring from the flowring pool, and assign it.
+ *
+ * Unlike common rings, which uses a dhd_prot_ring_init() to pass the common
+ * ring information to the dongle, a flowring's information is passed via a
+ * flowring create control message.
+ *
+ * Only the ring state (WR, RD) index are initialized.
+ */
+static msgbuf_ring_t *
+dhd_prot_flowrings_pool_fetch(dhd_pub_t *dhd, uint16 flowid)
+{
+	msgbuf_ring_t *ring;
+	dhd_prot_t *prot = dhd->prot;
+
+	ASSERT(flowid >= DHD_FLOWRING_START_FLOWID);
+	ASSERT(flowid < prot->h2d_rings_total);
+	ASSERT(prot->h2d_flowrings_pool != NULL);
+
+	ring = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid);
+
+	/* ASSERT flow_ring->inited == FALSE */
+
+	ring->wr = 0;
+	ring->rd = 0;
+	ring->curr_rd = 0;
+	ring->inited = TRUE;
+	/**
+	 * Every time a flowring starts dynamically, initialize current_phase with 0
+	 * then flip to BCMPCIE_CMNHDR_PHASE_BIT_INIT
+	 */
+	ring->current_phase = 0;
+	return ring;
+}
+
+/**
+ * dhd_prot_flowrings_pool_release - release a previously fetched flowring's
+ * msgbuf_ring back to the flow_ring pool.
+ */
+void
+dhd_prot_flowrings_pool_release(dhd_pub_t *dhd, uint16 flowid, void *flow_ring)
+{
+	msgbuf_ring_t *ring;
+	dhd_prot_t *prot = dhd->prot;
+
+	ASSERT(flowid >= DHD_FLOWRING_START_FLOWID);
+	ASSERT(flowid < prot->h2d_rings_total);
+	ASSERT(prot->h2d_flowrings_pool != NULL);
+
+	ring = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid);
+
+	ASSERT(ring == (msgbuf_ring_t*)flow_ring);
+	/* ASSERT flow_ring->inited == TRUE */
+
+	(void)dhd_dma_buf_audit(dhd, &ring->dma_buf);
+
+	ring->wr = 0;
+	ring->rd = 0;
+	ring->inited = FALSE;
+
+	ring->curr_rd = 0;
+}
+
+#ifdef AGG_H2D_DB
+void
+dhd_prot_schedule_aggregate_h2d_db(dhd_pub_t *dhd, uint16 flowid)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring;
+	uint16 inflight;
+	bool db_req = FALSE;
+	bool flush;
+
+	ring = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid);
+	flush = !!ring->pend_items_count;
+	dhd_prot_txdata_aggr_db_write_flush(dhd, flowid);
+
+	inflight = OSL_ATOMIC_READ(dhd->osh, &ring->inflight);
+	if (flush && inflight) {
+		if (inflight <= agg_h2d_db_inflight_thresh) {
+			db_req = TRUE;
+		}
+		dhd_agg_inflights_stats_update(dhd, inflight);
+		dhd_prot_aggregate_db_ring_door_bell(dhd, flowid, db_req);
+	}
+}
+#endif /* AGG_H2D_DB */
+
+/* Assumes only one index is updated at a time */
+/* FIXME Need to fix it */
+/* If exactly_nitems is true, this function will allocate space for nitems or fail */
+/*    Exception: when wrap around is encountered, to prevent hangup (last nitems of ring buffer) */
+/* If exactly_nitems is false, this function will allocate space for nitems or less */
+static void *
+BCMFASTPATH(dhd_prot_get_ring_space)(msgbuf_ring_t *ring, uint16 nitems, uint16 * alloced,
+	bool exactly_nitems)
+{
+	void *ret_ptr = NULL;
+	uint16 ring_avail_cnt;
+
+	ASSERT(nitems <= ring->max_items);
+
+	ring_avail_cnt = CHECK_WRITE_SPACE(ring->rd, ring->wr, ring->max_items);
+
+	if ((ring_avail_cnt == 0) ||
+	       (exactly_nitems && (ring_avail_cnt < nitems) &&
+	       ((ring->max_items - ring->wr) >= nitems))) {
+		DHD_INFO(("Space not available: ring %s items %d write %d read %d\n",
+			ring->name, nitems, ring->wr, ring->rd));
+		return NULL;
+	}
+	*alloced = MIN(nitems, ring_avail_cnt);
+
+	/* Return next available space */
+	ret_ptr = (char *)DHD_RING_BGN_VA(ring) + (ring->wr * ring->item_len);
+
+	/* Update write index */
+	if ((ring->wr + *alloced) == ring->max_items)
+		ring->wr = 0;
+	else if ((ring->wr + *alloced) < ring->max_items)
+		ring->wr += *alloced;
+	else {
+		/* Should never hit this */
+		ASSERT(0);
+		return NULL;
+	}
+
+	return ret_ptr;
+} /* dhd_prot_get_ring_space */
+
+#ifdef AGG_H2D_DB
+
+static void
+dhd_prot_agg_db_ring_write(dhd_pub_t *dhd, msgbuf_ring_t * ring, void* p,
+		uint16 nitems)
+{
+	uint16 max_flowrings = dhd->bus->max_tx_flowrings;
+	unsigned long flags_bus;
+
+#ifdef DHD_FAKE_TX_STATUS
+	/* if fake tx status is enabled, we should not update
+	 * dongle side rd/wr index for the tx flowring
+	 * and also should not ring the doorbell
+	 */
+	if (DHD_IS_FLOWRING(ring->idx, max_flowrings)) {
+		return;
+	}
+#endif /* DHD_FAKE_TX_STATUS */
+
+	DHD_BUS_LP_STATE_LOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+
+	/* cache flush */
+	OSL_CACHE_FLUSH(p, ring->item_len * nitems);
+
+	if (IDMA_ACTIVE(dhd) || dhd->dma_h2d_ring_upd_support) {
+			dhd_prot_dma_indx_set(dhd, ring->wr,
+			                      H2D_DMA_INDX_WR_UPD, ring->idx);
+	} else if (IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(ring->idx, max_flowrings)) {
+			dhd_prot_dma_indx_set(dhd, ring->wr,
+			H2D_IFRM_INDX_WR_UPD, ring->idx);
+	} else {
+			dhd_bus_cmn_writeshared(dhd->bus, &(ring->wr),
+				sizeof(uint16), RING_WR_UPD, ring->idx);
+	}
+
+	DHD_BUS_LP_STATE_UNLOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+}
+
+static void
+dhd_prot_aggregate_db_ring_door_bell(dhd_pub_t *dhd, uint16 flowid, bool ring_db)
+{
+	dhd_prot_t *prot = dhd->prot;
+	flow_ring_table_t *flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
+	flow_ring_node_t *flow_ring_node = (flow_ring_node_t *)&flow_ring_table[flowid];
+	msgbuf_ring_t *ring = (msgbuf_ring_t *)flow_ring_node->prot_info;
+	uint32 db_index;
+	uint corerev;
+
+	if (ring_db == TRUE) {
+		dhd_msgbuf_agg_h2d_db_timer_cancel(dhd);
+		prot->agg_h2d_db_info.direct_db_cnt++;
+		/* raise h2d interrupt */
+		if (IDMA_ACTIVE(dhd) || (IFRM_ACTIVE(dhd))) {
+			db_index = IDMA_IDX0;
+			/* this api is called in wl down path..in that case sih is freed already */
+			if (dhd->bus->sih) {
+				corerev = dhd->bus->sih->buscorerev;
+				/* We need to explictly configure the type of DMA for
+				 * core rev >= 24
+				 */
+				if (corerev >= 24) {
+					db_index |= (DMA_TYPE_IDMA << DMA_TYPE_SHIFT);
+				}
+			}
+			prot->mb_2_ring_fn(dhd->bus, db_index, TRUE);
+		} else {
+			prot->mb_ring_fn(dhd->bus, DHD_WRPTR_UPDATE_H2D_DB_MAGIC(ring));
+		}
+	} else {
+		dhd_msgbuf_agg_h2d_db_timer_start(prot);
+	}
+}
+
+#endif /* AGG_H2D_DB */
+
+/**
+ * dhd_prot_ring_write_complete - Host updates the new WR index on producing
+ * new messages in a H2D ring. The messages are flushed from cache prior to
+ * posting the new WR index. The new WR index will be updated in the DMA index
+ * array or directly in the dongle's ring state memory.
+ * A PCIE doorbell will be generated to wake up the dongle.
+ * This is a non-atomic function, make sure the callers
+ * always hold appropriate locks.
+ */
+static void
+BCMFASTPATH(__dhd_prot_ring_write_complete)(dhd_pub_t *dhd, msgbuf_ring_t * ring, void* p,
+	uint16 nitems)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint32 db_index;
+	uint16 max_flowrings = dhd->bus->max_tx_flowrings;
+	uint corerev;
+
+	/* cache flush */
+	OSL_CACHE_FLUSH(p, ring->item_len * nitems);
+
+	if (IDMA_ACTIVE(dhd) || dhd->dma_h2d_ring_upd_support) {
+			dhd_prot_dma_indx_set(dhd, ring->wr,
+			                      H2D_DMA_INDX_WR_UPD, ring->idx);
+	} else if (IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(ring->idx, max_flowrings)) {
+			dhd_prot_dma_indx_set(dhd, ring->wr,
+			H2D_IFRM_INDX_WR_UPD, ring->idx);
+	} else {
+			dhd_bus_cmn_writeshared(dhd->bus, &(ring->wr),
+				sizeof(uint16), RING_WR_UPD, ring->idx);
+	}
+
+	/* raise h2d interrupt */
+	if (IDMA_ACTIVE(dhd) ||
+		(IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(ring->idx, max_flowrings))) {
+		db_index = IDMA_IDX0;
+		/* this api is called in wl down path..in that case sih is freed already */
+		if (dhd->bus->sih) {
+			corerev = dhd->bus->sih->buscorerev;
+			/* We need to explictly configure the type of DMA for core rev >= 24 */
+			if (corerev >= 24) {
+				db_index |= (DMA_TYPE_IDMA << DMA_TYPE_SHIFT);
+			}
+		}
+		prot->mb_2_ring_fn(dhd->bus, db_index, TRUE);
+	} else {
+		prot->mb_ring_fn(dhd->bus, DHD_WRPTR_UPDATE_H2D_DB_MAGIC(ring));
+	}
+}
+
+static void
+BCMFASTPATH(dhd_prot_ring_write_complete)(dhd_pub_t *dhd, msgbuf_ring_t * ring, void* p,
+	uint16 nitems)
+{
+	unsigned long flags_bus;
+	DHD_BUS_LP_STATE_LOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+	__dhd_prot_ring_write_complete(dhd, ring, p, nitems);
+	DHD_BUS_LP_STATE_UNLOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+}
+
+static void
+BCMFASTPATH(__dhd_prot_ring_doorbell)(dhd_pub_t *dhd, uint32 value)
+{
+	dhd->prot->mb_ring_fn(dhd->bus, value);
+}
+
+static void
+BCMFASTPATH(dhd_prot_ring_doorbell)(dhd_pub_t *dhd, uint32 value)
+{
+	unsigned long flags_bus;
+	DHD_BUS_LP_STATE_LOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+	__dhd_prot_ring_doorbell(dhd, value);
+	DHD_BUS_LP_STATE_UNLOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+}
+
+/**
+ * dhd_prot_ring_write_complete_mbdata - will be called from dhd_prot_h2d_mbdata_send_ctrlmsg,
+ * which will hold DHD_BUS_LP_STATE_LOCK to update WR pointer, Ring DB and also update
+ * bus_low_power_state to indicate D3_INFORM sent in the same BUS_LP_STATE_LOCK.
+ */
+static void
+BCMFASTPATH(dhd_prot_ring_write_complete_mbdata)(dhd_pub_t *dhd, msgbuf_ring_t * ring, void *p,
+	uint16 nitems, uint32 mb_data)
+{
+	unsigned long flags_bus;
+
+	DHD_BUS_LP_STATE_LOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+
+	__dhd_prot_ring_write_complete(dhd, ring, p, nitems);
+
+	/* Mark D3_INFORM in the same context to skip ringing H2D DB after D3_INFORM */
+	if (mb_data == H2D_HOST_D3_INFORM) {
+		__DHD_SET_BUS_LPS_D3_INFORMED(dhd->bus);
+	}
+
+	DHD_BUS_LP_STATE_UNLOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+}
+
+/**
+ * dhd_prot_upd_read_idx - Host updates the new RD index on consuming messages
+ * from a D2H ring. The new RD index will be updated in the DMA Index array or
+ * directly in dongle's ring state memory.
+ */
+static void
+dhd_prot_upd_read_idx(dhd_pub_t *dhd, msgbuf_ring_t * ring)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint32 db_index;
+	uint corerev;
+	unsigned long flags_bus;
+
+	/* Hold lp state lock before ringing DB for synchronization
+	 * purpose, to avoid rininging after D3.
+	 */
+	DHD_BUS_LP_STATE_LOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+
+	/* update read index */
+	/* If dma'ing h2d indices supported
+	 * update the r -indices in the
+	 * host memory o/w in TCM
+	 */
+	if (IDMA_ACTIVE(dhd)) {
+		dhd_prot_dma_indx_set(dhd, ring->rd,
+			D2H_DMA_INDX_RD_UPD, ring->idx);
+		db_index = IDMA_IDX1;
+		if (dhd->bus->sih) {
+			corerev = dhd->bus->sih->buscorerev;
+			/* We need to explictly configure the type of DMA for core rev >= 24 */
+			if (corerev >= 24) {
+				db_index |= (DMA_TYPE_IDMA << DMA_TYPE_SHIFT);
+			}
+		}
+		prot->mb_2_ring_fn(dhd->bus, db_index, FALSE);
+	} else if (dhd->dma_h2d_ring_upd_support) {
+		dhd_prot_dma_indx_set(dhd, ring->rd,
+		                      D2H_DMA_INDX_RD_UPD, ring->idx);
+		/* For IDMA and HWA case, doorbell is sent along with read index update.
+		 * For DMA indices case ring doorbell once n items are read to sync with dongle.
+		 */
+		__dhd_prot_ring_doorbell(dhd, DHD_RDPTR_UPDATE_H2D_DB_MAGIC(ring));
+	} else {
+		dhd_bus_cmn_writeshared(dhd->bus, &(ring->rd),
+			sizeof(uint16), RING_RD_UPD, ring->idx);
+	}
+
+	DHD_BUS_LP_STATE_UNLOCK(dhd->bus->bus_lp_state_lock, flags_bus);
+}
+
+static int
+dhd_send_d2h_ringcreate(dhd_pub_t *dhd, msgbuf_ring_t *ring_to_create,
+	uint16 ring_type, uint32 req_id)
+{
+	unsigned long flags;
+	d2h_ring_create_req_t  *d2h_ring;
+	uint16 alloced = 0;
+	int ret = BCME_OK;
+	uint16 max_h2d_rings = dhd->bus->max_submission_rings;
+	msgbuf_ring_t *ctrl_ring = &dhd->prot->h2dring_ctrl_subn;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+
+	DHD_TRACE(("%s trying to send D2H ring create Req\n", __FUNCTION__));
+
+	if (ring_to_create == NULL) {
+		DHD_ERROR(("%s: FATAL: ring_to_create is NULL\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		goto err;
+	}
+
+	/* Request for ring buffer space */
+	d2h_ring = (d2h_ring_create_req_t *) dhd_prot_alloc_ring_space(dhd,
+		ctrl_ring, DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,
+		&alloced, FALSE);
+
+	if (d2h_ring == NULL) {
+		DHD_ERROR(("%s: FATAL: No space in control ring to send D2H ring create\n",
+			__FUNCTION__));
+		ret = BCME_NOMEM;
+		goto err;
+	}
+	ring_to_create->create_req_id = (uint16)req_id;
+	ring_to_create->create_pending = TRUE;
+
+	/* Common msg buf hdr */
+	d2h_ring->msg.msg_type = MSG_TYPE_D2H_RING_CREATE;
+	d2h_ring->msg.if_id = 0;
+	d2h_ring->msg.flags = ctrl_ring->current_phase;
+	d2h_ring->msg.request_id = htol32(ring_to_create->create_req_id);
+	d2h_ring->ring_id = htol16(DHD_D2H_RING_OFFSET(ring_to_create->idx, max_h2d_rings));
+	DHD_ERROR(("%s ringid: %d idx: %d max_h2d: %d\n", __FUNCTION__, d2h_ring->ring_id,
+			ring_to_create->idx, max_h2d_rings));
+
+	d2h_ring->ring_type = ring_type;
+	d2h_ring->max_items = htol16(ring_to_create->max_items);
+	d2h_ring->len_item = htol16(ring_to_create->item_len);
+	d2h_ring->ring_ptr.low_addr = ring_to_create->base_addr.low_addr;
+	d2h_ring->ring_ptr.high_addr = ring_to_create->base_addr.high_addr;
+
+	d2h_ring->flags = 0;
+	d2h_ring->msg.epoch =
+		ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+	ctrl_ring->seqnum++;
+
+#ifdef EWP_EDL
+	if (ring_type == BCMPCIE_D2H_RING_TYPE_EDL) {
+		DHD_ERROR(("%s: sending d2h EDL ring create: "
+			"\n max items=%u; len_item=%u; ring_id=%u; low_addr=0x%x; high_addr=0x%x\n",
+			__FUNCTION__, ltoh16(d2h_ring->max_items),
+			ltoh16(d2h_ring->len_item),
+			ltoh16(d2h_ring->ring_id),
+			d2h_ring->ring_ptr.low_addr,
+			d2h_ring->ring_ptr.high_addr));
+	}
+#endif /* EWP_EDL */
+
+	/* Update the flow_ring's WRITE index */
+	dhd_prot_ring_write_complete(dhd, ctrl_ring, d2h_ring,
+		DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return ret;
+err:
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return ret;
+}
+
+static int
+dhd_send_h2d_ringcreate(dhd_pub_t *dhd, msgbuf_ring_t *ring_to_create, uint8 ring_type, uint32 id)
+{
+	unsigned long flags;
+	h2d_ring_create_req_t  *h2d_ring;
+	uint16 alloced = 0;
+	uint8 i = 0;
+	int ret = BCME_OK;
+	msgbuf_ring_t *ctrl_ring = &dhd->prot->h2dring_ctrl_subn;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+
+	DHD_TRACE(("%s trying to send H2D ring create Req\n", __FUNCTION__));
+
+	if (ring_to_create == NULL) {
+		DHD_ERROR(("%s: FATAL: ring_to_create is NULL\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		goto err;
+	}
+
+	/* Request for ring buffer space */
+	h2d_ring = (h2d_ring_create_req_t *)dhd_prot_alloc_ring_space(dhd,
+		ctrl_ring, DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,
+		&alloced, FALSE);
+
+	if (h2d_ring == NULL) {
+		DHD_ERROR(("%s: FATAL: No space in control ring to send H2D ring create\n",
+			__FUNCTION__));
+		ret = BCME_NOMEM;
+		goto err;
+	}
+	ring_to_create->create_req_id = (uint16)id;
+	ring_to_create->create_pending = TRUE;
+
+	/* Common msg buf hdr */
+	h2d_ring->msg.msg_type = MSG_TYPE_H2D_RING_CREATE;
+	h2d_ring->msg.if_id = 0;
+	h2d_ring->msg.request_id = htol32(ring_to_create->create_req_id);
+	h2d_ring->msg.flags = ctrl_ring->current_phase;
+	h2d_ring->ring_id = htol16(DHD_H2D_RING_OFFSET(ring_to_create->idx));
+	h2d_ring->ring_type = ring_type;
+	h2d_ring->max_items = htol16(H2DRING_DYNAMIC_INFO_MAX_ITEM);
+	h2d_ring->n_completion_ids = ring_to_create->n_completion_ids;
+	h2d_ring->len_item = htol16(H2DRING_INFO_BUFPOST_ITEMSIZE);
+	h2d_ring->ring_ptr.low_addr = ring_to_create->base_addr.low_addr;
+	h2d_ring->ring_ptr.high_addr = ring_to_create->base_addr.high_addr;
+
+	for (i = 0; i < ring_to_create->n_completion_ids; i++) {
+		h2d_ring->completion_ring_ids[i] = htol16(ring_to_create->compeltion_ring_ids[i]);
+	}
+
+	h2d_ring->flags = 0;
+	h2d_ring->msg.epoch =
+		ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+	ctrl_ring->seqnum++;
+
+	/* Update the flow_ring's WRITE index */
+	dhd_prot_ring_write_complete(dhd, ctrl_ring, h2d_ring,
+		DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return ret;
+err:
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return ret;
+}
+
+/**
+ * dhd_prot_dma_indx_set - set a new WR or RD index in the DMA index array.
+ * Dongle will DMA the entire array (if DMA_INDX feature is enabled).
+ * See dhd_prot_dma_indx_init()
+ */
+void
+dhd_prot_dma_indx_set(dhd_pub_t *dhd, uint16 new_index, uint8 type, uint16 ringid)
+{
+	uint8 *ptr;
+	uint16 offset;
+	dhd_prot_t *prot = dhd->prot;
+	uint16 max_h2d_rings = dhd->bus->max_submission_rings;
+
+	/* Ensure that instruction operates on workitem are
+	 * completed before dma indices are updated.
+	 * For non-dma indices case, this is handled
+	 * using W_REG() which inturn has wmb().
+	 */
+	OSL_MB();
+	switch (type) {
+		case H2D_DMA_INDX_WR_UPD:
+			ptr = (uint8 *)(prot->h2d_dma_indx_wr_buf.va);
+			offset = DHD_H2D_RING_OFFSET(ringid);
+			break;
+
+		case D2H_DMA_INDX_RD_UPD:
+			ptr = (uint8 *)(prot->d2h_dma_indx_rd_buf.va);
+			offset = DHD_D2H_RING_OFFSET(ringid, max_h2d_rings);
+			break;
+
+		case H2D_IFRM_INDX_WR_UPD:
+			ptr = (uint8 *)(prot->h2d_ifrm_indx_wr_buf.va);
+			offset = DHD_H2D_FRM_FLOW_RING_OFFSET(ringid);
+			break;
+
+		default:
+			DHD_ERROR(("%s: Invalid option for DMAing read/write index\n",
+				__FUNCTION__));
+			return;
+	}
+
+	ASSERT(prot->rw_index_sz != 0);
+	ptr += offset * prot->rw_index_sz;
+
+	/* XXX: Test casting ptr to uint16* for 32bit indices case on Big Endian */
+	*(uint16*)ptr = htol16(new_index);
+
+	OSL_CACHE_FLUSH((void *)ptr, prot->rw_index_sz);
+
+	/* Ensure that instruction operates on workitem are
+	 * performed after dma indices are updated.
+	 */
+	OSL_MB();
+
+	DHD_TRACE(("%s: data %d type %d ringid %d ptr 0x%p offset %d\n",
+		__FUNCTION__, new_index, type, ringid, ptr, offset));
+
+} /* dhd_prot_dma_indx_set */
+
+/**
+ * dhd_prot_dma_indx_get - Fetch a WR or RD index from the dongle DMA-ed index
+ * array.
+ * Dongle DMAes an entire array to host memory (if the feature is enabled).
+ * See dhd_prot_dma_indx_init()
+ */
+static uint16
+dhd_prot_dma_indx_get(dhd_pub_t *dhd, uint8 type, uint16 ringid)
+{
+	uint8 *ptr;
+	uint16 data;
+	uint16 offset;
+	dhd_prot_t *prot = dhd->prot;
+	uint16 max_h2d_rings = dhd->bus->max_submission_rings;
+
+	/* Ensure that instruction operates on workitem are
+	 * completed before dma indices are read.
+	 */
+	OSL_MB();
+
+	switch (type) {
+		case H2D_DMA_INDX_WR_UPD:
+			ptr = (uint8 *)(prot->h2d_dma_indx_wr_buf.va);
+			offset = DHD_H2D_RING_OFFSET(ringid);
+			break;
+
+		case H2D_DMA_INDX_RD_UPD:
+#ifdef DHD_DMA_INDICES_SEQNUM
+			if (prot->h2d_dma_indx_rd_copy_buf) {
+				ptr = (uint8 *)(prot->h2d_dma_indx_rd_copy_buf);
+			} else
+#endif /* DHD_DMA_INDICES_SEQNUM */
+			{
+				ptr = (uint8 *)(prot->h2d_dma_indx_rd_buf.va);
+			}
+			offset = DHD_H2D_RING_OFFSET(ringid);
+			break;
+
+		case D2H_DMA_INDX_WR_UPD:
+#ifdef DHD_DMA_INDICES_SEQNUM
+			if (prot->d2h_dma_indx_wr_copy_buf) {
+				ptr = (uint8 *)(prot->d2h_dma_indx_wr_copy_buf);
+			} else
+#endif /* DHD_DMA_INDICES_SEQNUM */
+			{
+				ptr = (uint8 *)(prot->d2h_dma_indx_wr_buf.va);
+			}
+			offset = DHD_D2H_RING_OFFSET(ringid, max_h2d_rings);
+			break;
+
+		case D2H_DMA_INDX_RD_UPD:
+			ptr = (uint8 *)(prot->d2h_dma_indx_rd_buf.va);
+			offset = DHD_D2H_RING_OFFSET(ringid, max_h2d_rings);
+			break;
+
+		default:
+			DHD_ERROR(("%s: Invalid option for DMAing read/write index\n",
+				__FUNCTION__));
+			return 0;
+	}
+
+	ASSERT(prot->rw_index_sz != 0);
+	ptr += offset * prot->rw_index_sz;
+
+	OSL_CACHE_INV((void *)ptr, prot->rw_index_sz);
+
+	/* XXX: Test casting ptr to uint16* for 32bit indices case on Big Endian */
+	data = LTOH16(*((uint16*)ptr));
+
+	DHD_TRACE(("%s: data %d type %d ringid %d ptr 0x%p offset %d\n",
+		__FUNCTION__, data, type, ringid, ptr, offset));
+
+	/* Ensure that instruction operates on workitem are
+	 * performed after dma indices are read.
+	 */
+	OSL_MB();
+	return (data);
+
+} /* dhd_prot_dma_indx_get */
+
+#ifdef DHD_DMA_INDICES_SEQNUM
+void
+dhd_prot_write_host_seqnum(dhd_pub_t *dhd, uint32 seq_num)
+{
+	uint8 *ptr;
+	dhd_prot_t *prot = dhd->prot;
+
+	/* Update host sequence number in first four bytes of scratchbuf */
+	ptr = (uint8 *)(prot->d2h_dma_scratch_buf.va);
+	*(uint32*)ptr = htol32(seq_num);
+	OSL_CACHE_FLUSH((void *)ptr, prot->d2h_dma_scratch_buf.len);
+
+	DHD_TRACE(("%s: data %d ptr 0x%p\n", __FUNCTION__, seq_num, ptr));
+
+} /* dhd_prot_dma_indx_set */
+
+uint32
+dhd_prot_read_seqnum(dhd_pub_t *dhd, bool host)
+{
+	uint8 *ptr;
+	dhd_prot_t *prot = dhd->prot;
+	uint32 data;
+
+	OSL_CACHE_INV((void *)ptr, d2h_dma_scratch_buf.len);
+
+	/* First four bytes of scratchbuf contains the host sequence number.
+	 * Next four bytes of scratchbuf contains the Dongle sequence number.
+	 */
+	if (host) {
+		ptr = (uint8 *)(prot->d2h_dma_scratch_buf.va);
+		data = LTOH32(*((uint32*)ptr));
+	} else {
+		ptr = ((uint8 *)(prot->d2h_dma_scratch_buf.va) + sizeof(uint32));
+		data = LTOH32(*((uint32*)ptr));
+	}
+	DHD_TRACE(("%s: data %d ptr 0x%p\n", __FUNCTION__, data, ptr));
+	return data;
+} /* dhd_prot_dma_indx_set */
+
+void
+dhd_prot_save_dmaidx(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint32 dngl_seqnum;
+
+	dngl_seqnum = dhd_prot_read_seqnum(dhd, FALSE);
+
+	DHD_TRACE(("%s: host_seqnum %u dngl_seqnum %u\n", __FUNCTION__,
+			prot->host_seqnum, dngl_seqnum));
+	if (prot->d2h_dma_indx_wr_copy_buf && prot->h2d_dma_indx_rd_copy_buf) {
+		if (prot->host_seqnum == dngl_seqnum) {
+			memcpy_s(prot->d2h_dma_indx_wr_copy_buf, prot->d2h_dma_indx_wr_copy_bufsz,
+				prot->d2h_dma_indx_wr_buf.va, prot->d2h_dma_indx_wr_copy_bufsz);
+			memcpy_s(prot->h2d_dma_indx_rd_copy_buf, prot->h2d_dma_indx_rd_copy_bufsz,
+				prot->h2d_dma_indx_rd_buf.va, prot->h2d_dma_indx_rd_copy_bufsz);
+			dhd_prot_write_host_seqnum(dhd, prot->host_seqnum);
+			/* Ring DoorBell */
+			dhd_prot_ring_doorbell(dhd, DHD_DMA_INDX_SEQ_H2D_DB_MAGIC);
+			prot->host_seqnum++;
+			prot->host_seqnum %= D2H_EPOCH_MODULO;
+		}
+	}
+}
+
+int
+dhd_prot_dma_indx_copybuf_init(dhd_pub_t *dhd, uint32 buf_sz, uint8 type)
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	switch (type) {
+		case D2H_DMA_INDX_WR_BUF:
+			prot->d2h_dma_indx_wr_copy_buf = MALLOCZ(dhd->osh, buf_sz);
+			if (prot->d2h_dma_indx_wr_copy_buf == NULL) {
+				DHD_ERROR(("%s: MALLOC failed for size %d\n",
+					__FUNCTION__, buf_sz));
+				goto ret_no_mem;
+			}
+			prot->d2h_dma_indx_wr_copy_bufsz = buf_sz;
+		break;
+
+		case H2D_DMA_INDX_RD_BUF:
+			prot->h2d_dma_indx_rd_copy_buf = MALLOCZ(dhd->osh, buf_sz);
+			if (prot->h2d_dma_indx_rd_copy_buf == NULL) {
+				DHD_ERROR(("%s: MALLOC failed for size %d\n",
+					__FUNCTION__, buf_sz));
+				goto ret_no_mem;
+			}
+			prot->h2d_dma_indx_rd_copy_bufsz = buf_sz;
+			break;
+
+		default:
+			break;
+	}
+	return BCME_OK;
+ret_no_mem:
+	return BCME_NOMEM;
+
+}
+#endif /* DHD_DMA_INDICES_SEQNUM */
+
+/**
+ * An array of DMA read/write indices, containing information about host rings, can be maintained
+ * either in host memory or in device memory, dependent on preprocessor options. This function is,
+ * dependent on these options, called during driver initialization. It reserves and initializes
+ * blocks of DMA'able host memory containing an array of DMA read or DMA write indices. The physical
+ * address of these host memory blocks are communicated to the dongle later on. By reading this host
+ * memory, the dongle learns about the state of the host rings.
+ */
+
+static INLINE int
+dhd_prot_dma_indx_alloc(dhd_pub_t *dhd, uint8 type,
+	dhd_dma_buf_t *dma_buf, uint32 bufsz)
+{
+	int rc;
+
+	if ((dma_buf->len == bufsz) || (dma_buf->va != NULL))
+		return BCME_OK;
+
+	rc = dhd_dma_buf_alloc(dhd, dma_buf, bufsz);
+
+	return rc;
+}
+
+int
+dhd_prot_dma_indx_init(dhd_pub_t *dhd, uint32 rw_index_sz, uint8 type, uint32 length)
+{
+	uint32 bufsz;
+	dhd_prot_t *prot = dhd->prot;
+	dhd_dma_buf_t *dma_buf;
+
+	if (prot == NULL) {
+		DHD_ERROR(("prot is not inited\n"));
+		return BCME_ERROR;
+	}
+
+	/* Dongle advertizes 2B or 4B RW index size */
+	ASSERT(rw_index_sz != 0);
+	prot->rw_index_sz = rw_index_sz;
+
+	bufsz = rw_index_sz * length;
+
+	switch (type) {
+		case H2D_DMA_INDX_WR_BUF:
+			dma_buf = &prot->h2d_dma_indx_wr_buf;
+			if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
+				goto ret_no_mem;
+			DHD_ERROR(("H2D DMA WR INDX : array size %d = %d * %d\n",
+				dma_buf->len, rw_index_sz, length));
+			break;
+
+		case H2D_DMA_INDX_RD_BUF:
+			dma_buf = &prot->h2d_dma_indx_rd_buf;
+			if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
+				goto ret_no_mem;
+			DHD_ERROR(("H2D DMA RD INDX : array size %d = %d * %d\n",
+				dma_buf->len, rw_index_sz, length));
+			break;
+
+		case D2H_DMA_INDX_WR_BUF:
+			dma_buf = &prot->d2h_dma_indx_wr_buf;
+			if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
+				goto ret_no_mem;
+			DHD_ERROR(("D2H DMA WR INDX : array size %d = %d * %d\n",
+				dma_buf->len, rw_index_sz, length));
+			break;
+
+		case D2H_DMA_INDX_RD_BUF:
+			dma_buf = &prot->d2h_dma_indx_rd_buf;
+			if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
+				goto ret_no_mem;
+			DHD_ERROR(("D2H DMA RD INDX : array size %d = %d * %d\n",
+				dma_buf->len, rw_index_sz, length));
+			break;
+
+		case H2D_IFRM_INDX_WR_BUF:
+			dma_buf = &prot->h2d_ifrm_indx_wr_buf;
+			if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
+				goto ret_no_mem;
+			DHD_ERROR(("H2D IFRM WR INDX : array size %d = %d * %d\n",
+				dma_buf->len, rw_index_sz, length));
+			break;
+
+		default:
+			DHD_ERROR(("%s: Unexpected option\n", __FUNCTION__));
+			return BCME_BADOPTION;
+	}
+
+	return BCME_OK;
+
+ret_no_mem:
+	DHD_ERROR(("%s: dhd_prot_dma_indx_alloc type %d buf_sz %d failure\n",
+		__FUNCTION__, type, bufsz));
+	return BCME_NOMEM;
+
+} /* dhd_prot_dma_indx_init */
+
+/**
+ * Called on checking for 'completion' messages from the dongle. Returns next host buffer to read
+ * from, or NULL if there are no more messages to read. If 'bound' is non-zero, limits
+ * the no. of items read to bound
+ */
+static uint8*
+dhd_prot_get_read_addr(dhd_pub_t *dhd, msgbuf_ring_t *ring, uint32 *available_len, uint32 bound)
+{
+	uint16 wr;
+	uint16 rd;
+	uint16 depth;
+	uint16 items;
+	void  *read_addr = NULL; /* address of next msg to be read in ring */
+	uint16 d2h_wr = 0;
+	void  *md_read_addr = NULL; /* address of next msg to be read in ring */
+	int i;
+	uint8 *ptr = NULL;
+	uint32 total_md_len = 0;
+	UNUSED_PARAMETER(total_md_len);
+
+	DHD_TRACE(("%s: d2h_dma_indx_rd_buf %p, d2h_dma_indx_wr_buf %p\n",
+		__FUNCTION__, (uint32 *)(dhd->prot->d2h_dma_indx_rd_buf.va),
+		(uint32 *)(dhd->prot->d2h_dma_indx_wr_buf.va)));
+
+	/* Remember the read index in a variable.
+	 * This is becuase ring->rd gets updated in the end of this function
+	 * So if we have to print the exact read index from which the
+	 * message is read its not possible.
+	 */
+	ring->curr_rd = ring->rd;
+
+	/* update write pointer */
+	if (dhd->dma_d2h_ring_upd_support) {
+		/* DMAing write/read indices supported */
+		d2h_wr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+		ring->wr = d2h_wr;
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &(ring->wr), RING_WR_UPD, ring->idx);
+	}
+
+	wr = ring->wr;
+	rd = ring->rd;
+	depth = ring->max_items;
+
+	/* check for avail space, in number of ring items */
+	items = READ_AVAIL_SPACE(wr, rd, depth);
+	if (items == 0)
+		return NULL;
+
+	/*
+	 * Note that there are builds where Assert translates to just printk
+	 * so, even if we had hit this condition we would never halt. Now
+	 * dhd_prot_process_msgtype can get into an big loop if this
+	 * happens.
+	 */
+	if (items > ring->max_items) {
+		DHD_ERROR(("\r\n======================= \r\n"));
+		DHD_ERROR(("%s(): ring %p, ring->name %s, ring->max_items %d, items %d \r\n",
+			__FUNCTION__, ring, ring->name, ring->max_items, items));
+		DHD_ERROR(("wr: %d,  rd: %d,  depth: %d  \r\n", wr, rd, depth));
+		DHD_ERROR(("dhd->busstate %d bus->wait_for_d3_ack %d \r\n",
+			dhd->busstate, dhd->bus->wait_for_d3_ack));
+		DHD_ERROR(("\r\n======================= \r\n"));
+#ifdef DHD_FW_COREDUMP
+		if (dhd->memdump_enabled) {
+			/* collect core dump */
+			dhd->memdump_type = DUMP_TYPE_RESUMED_ON_INVALID_RING_RDWR;
+			dhd_bus_mem_dump(dhd);
+
+		}
+#endif /* DHD_FW_COREDUMP */
+
+		*available_len = 0;
+		dhd_schedule_reset(dhd);
+
+		return NULL;
+	}
+
+	/* if space is available, calculate address to be read */
+	read_addr = (char*)ring->dma_buf.va + (rd * ring->item_len);
+
+	/* limit no. of items to bound */
+	if (bound) {
+		items = MIN(items, bound);
+	}
+
+	/* update read pointer */
+	if ((ring->rd + items) >= ring->max_items)
+		ring->rd = 0;
+	else
+		ring->rd += items;
+
+	ASSERT(ring->rd < ring->max_items);
+
+	/* convert items to bytes : available_len must be 32bits */
+	*available_len = (uint32)(items * ring->item_len);
+
+	/* XXX Double cache invalidate for ARM with L2 cache/prefetch */
+	OSL_CACHE_INV(read_addr, *available_len);
+
+	if (ring->linked_ring) {
+		md_read_addr = (char*)ring->linked_ring->dma_buf.va +
+			(rd * ring->linked_ring->item_len);
+		total_md_len = (uint32)(items * ring->linked_ring->item_len);
+		OSL_CACHE_INV(md_read_addr, total_md_len);
+		/* Prefetch data to populate the cache */
+		DHD_INFO(("ring address:0x%p, mdring address:0x%p, item:%d:%d\n",
+			read_addr, md_read_addr, items, total_md_len));
+		if (dhd->mdring_info) {
+			ptr = (uint8 *)md_read_addr;
+			for (i = 0; i < items; i ++) {
+				ptr += ring->linked_ring->item_len;
+				DHD_INFO(("md:0x%x:0x%x:0x%x:0x%x   0x%x:0x%x:0x%x:0x%x\n",
+					ptr[0], ptr[1], ptr[2], ptr[3],
+					ptr[4], ptr[5], ptr[6], ptr[7]));
+				if (memcpy_s(&dhd->mdring_info[dhd->md_item_count *
+					ring->linked_ring->item_len],
+					ring->linked_ring->item_len, ptr,
+					ring->linked_ring->item_len) != BCME_OK) {
+						DHD_ERROR(("%s:Couldn't copy md item:%d,len:%d\n",
+							__FUNCTION__,
+							i, ring->linked_ring->item_len));
+						break;
+				}
+				dhd->md_item_count =
+					(dhd->md_item_count + 1) % MAX_MDRING_ITEM_DUMP;
+			}
+		}
+	}
+
+	/* return read address */
+	return read_addr;
+
+} /* dhd_prot_get_read_addr */
+
+/**
+ * dhd_prot_h2d_mbdata_send_ctrlmsg is a non-atomic function,
+ * make sure the callers always hold appropriate locks.
+ */
+int dhd_prot_h2d_mbdata_send_ctrlmsg(dhd_pub_t *dhd, uint32 mb_data)
+{
+	h2d_mailbox_data_t *h2d_mb_data;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ctrl_ring = &dhd->prot->h2dring_ctrl_subn;
+	unsigned long flags;
+	int num_post = 1;
+	int i;
+
+	DHD_INFO(("%s Sending H2D MB data Req data 0x%04x\n",
+		__FUNCTION__, mb_data));
+	if (!ctrl_ring->inited) {
+		DHD_ERROR(("%s: Ctrl Submit Ring: not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+#ifdef PCIE_INB_DW
+	if ((INBAND_DW_ENAB(dhd->bus)) &&
+		(dhdpcie_bus_get_pcie_inband_dw_state(dhd->bus) ==
+			DW_DEVICE_DS_DEV_SLEEP)) {
+		if (mb_data == H2D_HOST_CONS_INT || mb_data == H2D_HOST_PTM_ENABLE ||
+			mb_data == H2D_HOST_PTM_DISABLE) {
+			/* One additional device_wake post needed */
+			DHD_INFO(("%s One additional device_wake post needed\n", __FUNCTION__));
+			num_post = 2;
+		}
+	}
+#endif /* PCIE_INB_DW */
+
+	for (i = 0; i < num_post; i ++) {
+		DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+		/* Request for ring buffer space */
+		h2d_mb_data = (h2d_mailbox_data_t *)dhd_prot_alloc_ring_space(dhd,
+			ctrl_ring, DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,
+			&alloced, FALSE);
+
+		if (h2d_mb_data == NULL) {
+			DHD_ERROR(("%s: FATAL: No space in control ring to send H2D Mb data\n",
+				__FUNCTION__));
+			DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+			return BCME_NOMEM;
+		}
+
+		memset(h2d_mb_data, 0, sizeof(h2d_mailbox_data_t));
+		/* Common msg buf hdr */
+		h2d_mb_data->msg.msg_type = MSG_TYPE_H2D_MAILBOX_DATA;
+		h2d_mb_data->msg.flags = ctrl_ring->current_phase;
+
+		h2d_mb_data->msg.epoch =
+			ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+		ctrl_ring->seqnum++;
+
+		/* Update flow create message */
+		h2d_mb_data->mail_box_data = htol32(mb_data);
+#ifdef PCIE_INB_DW
+		/* post device_wake first */
+		if ((num_post == 2) && (i == 0)) {
+			h2d_mb_data->mail_box_data = htol32(H2DMB_DS_DEVICE_WAKE);
+		} else
+#endif /* PCIE_INB_DW */
+		{
+			h2d_mb_data->mail_box_data = htol32(mb_data);
+		}
+
+		DHD_INFO(("%s Send H2D MB data Req data 0x%04x\n", __FUNCTION__,
+			h2d_mb_data->mail_box_data));
+
+		/* upd wrt ptr and raise interrupt */
+		dhd_prot_ring_write_complete_mbdata(dhd, ctrl_ring, h2d_mb_data,
+			DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D, mb_data);
+
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+		/* Add a delay if device_wake is posted */
+		if ((num_post == 2) && (i == 0)) {
+			OSL_DELAY(1000);
+		}
+#endif /* PCIE_INB_DW */
+	}
+	return 0;
+}
+
+/** Creates a flow ring and informs dongle of this event */
+int
+dhd_prot_flow_ring_create(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node)
+{
+	tx_flowring_create_request_t *flow_create_rqst;
+	msgbuf_ring_t *flow_ring;
+	dhd_prot_t *prot = dhd->prot;
+	unsigned long flags;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
+	uint16 max_flowrings = dhd->bus->max_tx_flowrings;
+	int ret = 0;
+
+	/* Fetch a pre-initialized msgbuf_ring from the flowring pool */
+	flow_ring = dhd_prot_flowrings_pool_fetch(dhd, flow_ring_node->flowid);
+	if (flow_ring == NULL) {
+		DHD_ERROR(("%s: dhd_prot_flowrings_pool_fetch TX Flowid %d failed\n",
+			__FUNCTION__, flow_ring_node->flowid));
+		return BCME_NOMEM;
+	}
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+
+	/* Request for ctrl_ring buffer space */
+	flow_create_rqst = (tx_flowring_create_request_t *)
+		dhd_prot_alloc_ring_space(dhd, ctrl_ring, 1, &alloced, FALSE);
+
+	if (flow_create_rqst == NULL) {
+		dhd_prot_flowrings_pool_release(dhd, flow_ring_node->flowid, flow_ring);
+		DHD_ERROR(("%s: Flow Create Req flowid %d - failure ring space\n",
+			__FUNCTION__, flow_ring_node->flowid));
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_NOMEM;
+	}
+
+	flow_ring_node->prot_info = (void *)flow_ring;
+
+	/* Common msg buf hdr */
+	flow_create_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_CREATE;
+	flow_create_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
+	flow_create_rqst->msg.request_id = htol32(0); /* TBD */
+	flow_create_rqst->msg.flags = ctrl_ring->current_phase;
+
+	flow_create_rqst->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+	ctrl_ring->seqnum++;
+
+	/* Update flow create message */
+	flow_create_rqst->tid = flow_ring_node->flow_info.tid;
+	flow_create_rqst->flow_ring_id = htol16((uint16)flow_ring_node->flowid);
+	ret = memcpy_s(flow_create_rqst->sa, sizeof(flow_create_rqst->sa),
+		flow_ring_node->flow_info.sa, sizeof(flow_ring_node->flow_info.sa));
+	if (ret) {
+		DHD_ERROR(("flow message sa memcpy failed:%d, destsz:%zu, n:%zu\n",
+			ret, sizeof(flow_create_rqst->sa), sizeof(flow_ring_node->flow_info.sa)));
+		return BCME_ERROR;
+	}
+	ret = memcpy_s(flow_create_rqst->da, sizeof(flow_create_rqst->da),
+		flow_ring_node->flow_info.da, sizeof(flow_ring_node->flow_info.da));
+	if (ret) {
+		DHD_ERROR(("flow message da memcpy failed:%d, destsz:%zu, n:%zu\n",
+			ret, sizeof(flow_create_rqst->da), sizeof(flow_ring_node->flow_info.da)));
+		return BCME_ERROR;
+	}
+	/* CAUTION: ring::base_addr already in Little Endian */
+	flow_create_rqst->flow_ring_ptr.low_addr = flow_ring->base_addr.low_addr;
+	flow_create_rqst->flow_ring_ptr.high_addr = flow_ring->base_addr.high_addr;
+	flow_create_rqst->max_items = htol16(flow_ring->max_items);
+	flow_create_rqst->len_item = htol16(H2DRING_TXPOST_ITEMSIZE);
+	flow_create_rqst->if_flags = 0;
+
+	/* definition for ifrm mask : bit0:d11ac core, bit1:d11ad core
+	 * currently it is not used for priority. so uses solely for ifrm mask
+	 */
+	if (IFRM_ACTIVE(dhd))
+		flow_create_rqst->priority_ifrmmask = (1 << IFRM_DEV_0);
+
+	DHD_ERROR(("%s: Send Flow Create Req flow ID %d for peer " MACDBG
+		" prio %d ifindex %d items %d\n", __FUNCTION__, flow_ring_node->flowid,
+		MAC2STRDBG(flow_ring_node->flow_info.da), flow_ring_node->flow_info.tid,
+		flow_ring_node->flow_info.ifindex, flow_ring->max_items));
+
+	/* Update the flow_ring's WRITE index */
+	if (IDMA_ACTIVE(dhd) || dhd->dma_h2d_ring_upd_support) {
+		dhd_prot_dma_indx_set(dhd, flow_ring->wr,
+		                      H2D_DMA_INDX_WR_UPD, flow_ring->idx);
+	} else if (IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(flow_ring->idx, max_flowrings)) {
+		dhd_prot_dma_indx_set(dhd, flow_ring->wr,
+			H2D_IFRM_INDX_WR_UPD, flow_ring->idx);
+	} else {
+		dhd_bus_cmn_writeshared(dhd->bus, &(flow_ring->wr),
+			sizeof(uint16), RING_WR_UPD, flow_ring->idx);
+	}
+
+	/* update control subn ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ctrl_ring, flow_create_rqst, 1);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return BCME_OK;
+} /* dhd_prot_flow_ring_create */
+
+/** called on receiving MSG_TYPE_FLOW_RING_CREATE_CMPLT message from dongle */
+static void
+dhd_prot_flow_ring_create_response_process(dhd_pub_t *dhd, void *msg)
+{
+	tx_flowring_create_response_t *flow_create_resp = (tx_flowring_create_response_t *)msg;
+
+	DHD_ERROR(("%s: Flow Create Response status = %d Flow %d\n", __FUNCTION__,
+		ltoh16(flow_create_resp->cmplt.status),
+		ltoh16(flow_create_resp->cmplt.flow_ring_id)));
+
+	dhd_bus_flow_ring_create_response(dhd->bus,
+		ltoh16(flow_create_resp->cmplt.flow_ring_id),
+		ltoh16(flow_create_resp->cmplt.status));
+}
+
+#if !defined(BCM_ROUTER_DHD)
+static void
+dhd_prot_process_h2d_ring_create_complete(dhd_pub_t *dhd, void *buf)
+{
+	h2d_ring_create_response_t *resp = (h2d_ring_create_response_t *)buf;
+	DHD_INFO(("%s ring create Response status = %d ring %d, id 0x%04x\n", __FUNCTION__,
+		ltoh16(resp->cmplt.status),
+		ltoh16(resp->cmplt.ring_id),
+		ltoh32(resp->cmn_hdr.request_id)));
+	if ((ltoh32(resp->cmn_hdr.request_id) != DHD_H2D_DBGRING_REQ_PKTID) &&
+		(ltoh32(resp->cmn_hdr.request_id) != DHD_H2D_BTLOGRING_REQ_PKTID)) {
+		DHD_ERROR(("invalid request ID with h2d ring create complete\n"));
+		return;
+	}
+	if (dhd->prot->h2dring_info_subn->create_req_id == ltoh32(resp->cmn_hdr.request_id) &&
+		!dhd->prot->h2dring_info_subn->create_pending) {
+		DHD_ERROR(("info ring create status for not pending submit ring\n"));
+	}
+
+	if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
+		DHD_ERROR(("info/btlog ring create failed with status %d\n",
+			ltoh16(resp->cmplt.status)));
+		return;
+	}
+	if (dhd->prot->h2dring_info_subn->create_req_id == ltoh32(resp->cmn_hdr.request_id)) {
+		dhd->prot->h2dring_info_subn->create_pending = FALSE;
+		dhd->prot->h2dring_info_subn->inited = TRUE;
+		DHD_ERROR(("info buffer post after ring create\n"));
+		dhd_prot_infobufpost(dhd, dhd->prot->h2dring_info_subn);
+	}
+}
+#endif /* !BCM_ROUTER_DHD */
+
+static void
+dhd_prot_process_d2h_ring_create_complete(dhd_pub_t *dhd, void *buf)
+{
+	d2h_ring_create_response_t *resp = (d2h_ring_create_response_t *)buf;
+	DHD_ERROR(("%s ring create Response status = %d ring %d, id 0x%04x\n", __FUNCTION__,
+		ltoh16(resp->cmplt.status),
+		ltoh16(resp->cmplt.ring_id),
+		ltoh32(resp->cmn_hdr.request_id)));
+	if ((ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_DBGRING_REQ_PKTID) &&
+		(ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_BTLOGRING_REQ_PKTID) &&
+		(ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_MDRING_REQ_PKTID) &&
+		TRUE) {
+		DHD_ERROR(("invalid request ID with d2h ring create complete\n"));
+		return;
+	}
+	if (ltoh32(resp->cmn_hdr.request_id) == DHD_D2H_DBGRING_REQ_PKTID) {
+#ifdef EWP_EDL
+		if (!dhd->dongle_edl_support)
+#endif
+		{
+
+			if (!dhd->prot->d2hring_info_cpln->create_pending) {
+				DHD_ERROR(("info ring create status for not pending cpl ring\n"));
+				return;
+			}
+
+			if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
+				DHD_ERROR(("info cpl ring create failed with status %d\n",
+					ltoh16(resp->cmplt.status)));
+				return;
+			}
+			dhd->prot->d2hring_info_cpln->create_pending = FALSE;
+			dhd->prot->d2hring_info_cpln->inited = TRUE;
+		}
+#ifdef EWP_EDL
+		else {
+			if (!dhd->prot->d2hring_edl->create_pending) {
+				DHD_ERROR(("edl ring create status for not pending cpl ring\n"));
+				return;
+			}
+
+			if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
+				DHD_ERROR(("edl cpl ring create failed with status %d\n",
+					ltoh16(resp->cmplt.status)));
+				return;
+			}
+			dhd->prot->d2hring_edl->create_pending = FALSE;
+			dhd->prot->d2hring_edl->inited = TRUE;
+		}
+#endif /* EWP_EDL */
+	}
+
+	if (dhd->prot->d2hring_md_cpl &&
+		ltoh32(resp->cmn_hdr.request_id) == DHD_D2H_MDRING_REQ_PKTID) {
+		if (!dhd->prot->d2hring_md_cpl->create_pending) {
+			DHD_ERROR(("metadata ring create status for not pending cpl ring\n"));
+			return;
+		}
+
+		if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
+			DHD_ERROR(("metadata cpl ring create failed with status %d\n",
+				ltoh16(resp->cmplt.status)));
+			return;
+		}
+		dhd->prot->d2hring_md_cpl->create_pending = FALSE;
+		dhd->prot->d2hring_md_cpl->inited = TRUE;
+	}
+}
+
+static void
+dhd_prot_process_d2h_mb_data(dhd_pub_t *dhd, void* buf)
+{
+	d2h_mailbox_data_t *d2h_data;
+
+	d2h_data = (d2h_mailbox_data_t *)buf;
+	DHD_INFO(("%s dhd_prot_process_d2h_mb_data, 0x%04x\n", __FUNCTION__,
+		d2h_data->d2h_mailbox_data));
+	dhd_bus_handle_mb_data(dhd->bus, d2h_data->d2h_mailbox_data);
+}
+
+static void
+dhd_prot_process_d2h_host_ts_complete(dhd_pub_t *dhd, void* buf)
+{
+	DHD_ERROR(("Timesunc feature not compiled in but GOT HOST_TS_COMPLETE\n"));
+
+}
+
+/** called on e.g. flow ring delete */
+void dhd_prot_clean_flow_ring(dhd_pub_t *dhd, void *msgbuf_flow_info)
+{
+	msgbuf_ring_t *flow_ring = (msgbuf_ring_t *)msgbuf_flow_info;
+	dhd_prot_ring_detach(dhd, flow_ring);
+	DHD_INFO(("%s Cleaning up Flow \n", __FUNCTION__));
+}
+
+void dhd_prot_print_flow_ring(dhd_pub_t *dhd, void *msgbuf_flow_info, bool h2d,
+	struct bcmstrbuf *strbuf, const char * fmt)
+{
+	const char *default_fmt =
+		"TRD:%d HLRD:%d HDRD:%d TWR:%d HLWR:%d HDWR:%d  BASE(VA) %p BASE(PA) %x:%x SIZE %d "
+		"WORK_ITEM_SIZE %d MAX_WORK_ITEMS %d TOTAL_SIZE %d\n";
+	msgbuf_ring_t *flow_ring = (msgbuf_ring_t *)msgbuf_flow_info;
+	uint16 rd, wr, drd = 0, dwr = 0;
+	uint32 dma_buf_len = flow_ring->max_items * flow_ring->item_len;
+
+	if (fmt == NULL) {
+		fmt = default_fmt;
+	}
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s: Skip dumping flowring due to Link down\n", __FUNCTION__));
+		return;
+	}
+
+	dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, flow_ring->idx);
+	dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, flow_ring->idx);
+	if (dhd->dma_d2h_ring_upd_support) {
+		if (h2d) {
+			drd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, flow_ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, flow_ring->idx);
+		} else {
+			drd = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, flow_ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, flow_ring->idx);
+		}
+	}
+	bcm_bprintf(strbuf, fmt, rd, flow_ring->rd, drd, wr, flow_ring->wr, dwr,
+		flow_ring->dma_buf.va,
+		ltoh32(flow_ring->base_addr.high_addr),
+		ltoh32(flow_ring->base_addr.low_addr),
+		flow_ring->item_len, flow_ring->max_items,
+		dma_buf_len);
+}
+
+void dhd_prot_print_info(dhd_pub_t *dhd, struct bcmstrbuf *strbuf)
+{
+	dhd_prot_t *prot = dhd->prot;
+	bcm_bprintf(strbuf, "IPCrevs: Dev %d, \t Host %d, \tactive %d\n",
+		prot->device_ipc_version,
+		prot->host_ipc_version,
+		prot->active_ipc_version);
+
+	bcm_bprintf(strbuf, "max Host TS bufs to post: %d, \t posted %d \n",
+		prot->max_tsbufpost, prot->cur_ts_bufs_posted);
+	bcm_bprintf(strbuf, "max INFO bufs to post: %d, \t posted %d \n",
+		prot->max_infobufpost, prot->infobufpost);
+	bcm_bprintf(strbuf, "max event bufs to post: %d, \t posted %d \n",
+		prot->max_eventbufpost, prot->cur_event_bufs_posted);
+	bcm_bprintf(strbuf, "max ioctlresp bufs to post: %d, \t posted %d \n",
+		prot->max_ioctlrespbufpost, prot->cur_ioctlresp_bufs_posted);
+
+	dhd_prot_print_ring_info(dhd, strbuf);
+
+	bcm_bprintf(strbuf, "active_tx_count %d	 pktidmap_avail(ctrl/rx/tx) %d %d %d\n",
+		OSL_ATOMIC_READ(dhd->osh, &prot->active_tx_count),
+		DHD_PKTID_AVAIL(prot->pktid_ctrl_map),
+		DHD_PKTID_AVAIL(prot->pktid_rx_map),
+		DHD_PKTID_AVAIL(prot->pktid_tx_map));
+
+	bcm_bprintf(strbuf, "Total wakeup packet rcvd: Event:%d,\t RX:%d,\t Info:%d\n",
+		prot->event_wakeup_pkt, prot->rx_wakeup_pkt,
+		prot->info_wakeup_pkt);
+
+#ifdef DHD_MMIO_TRACE
+	dhd_dump_bus_mmio_trace(dhd->bus, strbuf);
+#endif /* DHD_MMIO_TRACE */
+	dhd_dump_bus_ds_trace(dhd->bus, strbuf);
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+	dhd_dump_bus_flow_ring_status_isr_trace(dhd->bus, strbuf);
+	dhd_dump_bus_flow_ring_status_dpc_trace(dhd->bus, strbuf);
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+}
+
+int
+dhd_prot_flow_ring_delete(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node)
+{
+	tx_flowring_delete_request_t *flow_delete_rqst;
+	dhd_prot_t *prot = dhd->prot;
+	unsigned long flags;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	/* Request for ring buffer space */
+	flow_delete_rqst = (tx_flowring_delete_request_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+
+	if (flow_delete_rqst == NULL) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		DHD_ERROR(("%s: Flow Delete Req - failure ring space\n", __FUNCTION__));
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_NOMEM;
+	}
+
+	/* Common msg buf hdr */
+	flow_delete_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_DELETE;
+	flow_delete_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
+	flow_delete_rqst->msg.request_id = htol32(0); /* TBD */
+	flow_delete_rqst->msg.flags = ring->current_phase;
+
+	flow_delete_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+
+	/* Update Delete info */
+	flow_delete_rqst->flow_ring_id = htol16((uint16)flow_ring_node->flowid);
+	flow_delete_rqst->reason = htol16(BCME_OK);
+
+	DHD_ERROR(("%s: Send Flow Delete Req RING ID %d for peer " MACDBG
+		" prio %d ifindex %d\n", __FUNCTION__, flow_ring_node->flowid,
+		MAC2STRDBG(flow_ring_node->flow_info.da), flow_ring_node->flow_info.tid,
+		flow_ring_node->flow_info.ifindex));
+
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, flow_delete_rqst, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return BCME_OK;
+}
+
+static void
+BCMFASTPATH(dhd_prot_flow_ring_fastdelete)(dhd_pub_t *dhd, uint16 flowid, uint16 rd_idx)
+{
+	flow_ring_node_t *flow_ring_node = DHD_FLOW_RING(dhd, flowid);
+	msgbuf_ring_t *ring = (msgbuf_ring_t *)flow_ring_node->prot_info;
+	host_txbuf_cmpl_t txstatus;
+	host_txbuf_post_t *txdesc;
+	uint16 wr_idx;
+
+	DHD_INFO(("%s: FAST delete ring, flowid=%d, rd_idx=%d, wr_idx=%d\n",
+		__FUNCTION__, flowid, rd_idx, ring->wr));
+
+	memset(&txstatus, 0, sizeof(txstatus));
+	txstatus.compl_hdr.flow_ring_id = flowid;
+	txstatus.cmn_hdr.if_id = flow_ring_node->flow_info.ifindex;
+	wr_idx = ring->wr;
+
+	while (wr_idx != rd_idx) {
+		if (wr_idx)
+			wr_idx--;
+		else
+			wr_idx = ring->max_items - 1;
+		txdesc = (host_txbuf_post_t *)((char *)DHD_RING_BGN_VA(ring) +
+			(wr_idx * ring->item_len));
+		txstatus.cmn_hdr.request_id = txdesc->cmn_hdr.request_id;
+		dhd_prot_txstatus_process(dhd, &txstatus);
+	}
+}
+
+static void
+dhd_prot_flow_ring_delete_response_process(dhd_pub_t *dhd, void *msg)
+{
+	tx_flowring_delete_response_t *flow_delete_resp = (tx_flowring_delete_response_t *)msg;
+
+	DHD_ERROR(("%s: Flow Delete Response status = %d Flow %d\n", __FUNCTION__,
+		flow_delete_resp->cmplt.status, flow_delete_resp->cmplt.flow_ring_id));
+
+	if (dhd->fast_delete_ring_support) {
+		dhd_prot_flow_ring_fastdelete(dhd, flow_delete_resp->cmplt.flow_ring_id,
+			flow_delete_resp->read_idx);
+	}
+	dhd_bus_flow_ring_delete_response(dhd->bus, flow_delete_resp->cmplt.flow_ring_id,
+		flow_delete_resp->cmplt.status);
+}
+
+static void
+dhd_prot_process_flow_ring_resume_response(dhd_pub_t *dhd, void* msg)
+{
+#ifdef IDLE_TX_FLOW_MGMT
+	tx_idle_flowring_resume_response_t	*flow_resume_resp =
+		(tx_idle_flowring_resume_response_t *)msg;
+
+	DHD_ERROR(("%s Flow resume Response status = %d Flow %d\n", __FUNCTION__,
+		flow_resume_resp->cmplt.status, flow_resume_resp->cmplt.flow_ring_id));
+
+	dhd_bus_flow_ring_resume_response(dhd->bus, flow_resume_resp->cmplt.flow_ring_id,
+		flow_resume_resp->cmplt.status);
+#endif /* IDLE_TX_FLOW_MGMT */
+}
+
+static void
+dhd_prot_process_flow_ring_suspend_response(dhd_pub_t *dhd, void* msg)
+{
+#ifdef IDLE_TX_FLOW_MGMT
+	int16 status;
+	tx_idle_flowring_suspend_response_t	*flow_suspend_resp =
+		(tx_idle_flowring_suspend_response_t *)msg;
+	status = flow_suspend_resp->cmplt.status;
+
+	DHD_ERROR(("%s Flow id %d suspend Response status = %d\n",
+		__FUNCTION__, flow_suspend_resp->cmplt.flow_ring_id,
+		status));
+
+	if (status != BCME_OK) {
+
+		DHD_ERROR(("%s Error in Suspending Flow rings!!"
+			"Dongle will still be polling idle rings!!Status = %d \n",
+			__FUNCTION__, status));
+	}
+#endif /* IDLE_TX_FLOW_MGMT */
+}
+
+int
+dhd_prot_flow_ring_flush(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node)
+{
+	tx_flowring_flush_request_t *flow_flush_rqst;
+	dhd_prot_t *prot = dhd->prot;
+	unsigned long flags;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	/* Request for ring buffer space */
+	flow_flush_rqst = (tx_flowring_flush_request_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+	if (flow_flush_rqst == NULL) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		DHD_ERROR(("%s: Flow Flush Req - failure ring space\n", __FUNCTION__));
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_NOMEM;
+	}
+
+	/* Common msg buf hdr */
+	flow_flush_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_FLUSH;
+	flow_flush_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
+	flow_flush_rqst->msg.request_id = htol32(0); /* TBD */
+	flow_flush_rqst->msg.flags = ring->current_phase;
+	flow_flush_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+
+	flow_flush_rqst->flow_ring_id = htol16((uint16)flow_ring_node->flowid);
+	flow_flush_rqst->reason = htol16(BCME_OK);
+
+	DHD_INFO(("%s: Send Flow Flush Req\n", __FUNCTION__));
+
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, flow_flush_rqst, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return BCME_OK;
+} /* dhd_prot_flow_ring_flush */
+
+static void
+dhd_prot_flow_ring_flush_response_process(dhd_pub_t *dhd, void *msg)
+{
+	tx_flowring_flush_response_t *flow_flush_resp = (tx_flowring_flush_response_t *)msg;
+
+	DHD_INFO(("%s: Flow Flush Response status = %d\n", __FUNCTION__,
+		flow_flush_resp->cmplt.status));
+
+	dhd_bus_flow_ring_flush_response(dhd->bus, flow_flush_resp->cmplt.flow_ring_id,
+		flow_flush_resp->cmplt.status);
+}
+
+/**
+ * Request dongle to configure soft doorbells for D2H rings. Host populated soft
+ * doorbell information is transferred to dongle via the d2h ring config control
+ * message.
+ */
+void
+dhd_msgbuf_ring_config_d2h_soft_doorbell(dhd_pub_t *dhd)
+{
+#if defined(DHD_D2H_SOFT_DOORBELL_SUPPORT)
+	uint16 ring_idx;
+	uint8 *msg_next;
+	void *msg_start;
+	uint16 alloced = 0;
+	unsigned long flags;
+	dhd_prot_t *prot = dhd->prot;
+	ring_config_req_t *ring_config_req;
+	bcmpcie_soft_doorbell_t *soft_doorbell;
+	msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
+	const uint16 d2h_rings = BCMPCIE_D2H_COMMON_MSGRINGS;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	/* Claim space for d2h_ring number of d2h_ring_config_req_t messages */
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+	msg_start = dhd_prot_alloc_ring_space(dhd, ctrl_ring, d2h_rings, &alloced, TRUE);
+
+	if (msg_start == NULL) {
+		DHD_ERROR(("%s Msgbuf no space for %d D2H ring config soft doorbells\n",
+			__FUNCTION__, d2h_rings));
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return;
+	}
+
+	msg_next = (uint8*)msg_start;
+
+	for (ring_idx = 0; ring_idx < d2h_rings; ring_idx++) {
+
+		/* position the ring_config_req into the ctrl subm ring */
+		ring_config_req = (ring_config_req_t *)msg_next;
+
+		/* Common msg header */
+		ring_config_req->msg.msg_type = MSG_TYPE_D2H_RING_CONFIG;
+		ring_config_req->msg.if_id = 0;
+		ring_config_req->msg.flags = 0;
+
+		ring_config_req->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+		ctrl_ring->seqnum++;
+
+		ring_config_req->msg.request_id = htol32(DHD_FAKE_PKTID); /* unused */
+
+		/* Ring Config subtype and d2h ring_id */
+		ring_config_req->subtype = htol16(D2H_RING_CONFIG_SUBTYPE_SOFT_DOORBELL);
+		ring_config_req->ring_id = htol16(DHD_D2H_RINGID(ring_idx));
+
+		/* Host soft doorbell configuration */
+		soft_doorbell = &prot->soft_doorbell[ring_idx];
+
+		ring_config_req->soft_doorbell.value = htol32(soft_doorbell->value);
+		ring_config_req->soft_doorbell.haddr.high =
+			htol32(soft_doorbell->haddr.high);
+		ring_config_req->soft_doorbell.haddr.low =
+			htol32(soft_doorbell->haddr.low);
+		ring_config_req->soft_doorbell.items = htol16(soft_doorbell->items);
+		ring_config_req->soft_doorbell.msecs = htol16(soft_doorbell->msecs);
+
+		DHD_INFO(("%s: Soft doorbell haddr 0x%08x 0x%08x value 0x%08x\n",
+			__FUNCTION__, ring_config_req->soft_doorbell.haddr.high,
+			ring_config_req->soft_doorbell.haddr.low,
+			ring_config_req->soft_doorbell.value));
+
+		msg_next = msg_next + ctrl_ring->item_len;
+	}
+
+	/* update control subn ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ctrl_ring, msg_start, d2h_rings);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+#endif /* DHD_D2H_SOFT_DOORBELL_SUPPORT */
+}
+
+static void
+dhd_prot_process_d2h_ring_config_complete(dhd_pub_t *dhd, void *msg)
+{
+	ring_config_resp_t *ring_config_resp = (ring_config_resp_t *)msg;
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *cpl_ring = NULL;
+
+	DHD_INFO(("%s: Ring Config Response - status %d ringid %d subtype:%d\n",
+		__FUNCTION__, ltoh16(((ring_config_resp_t *)msg)->compl_hdr.status),
+		ltoh16(((ring_config_resp_t *)msg)->compl_hdr.flow_ring_id),
+		ltoh16(((ring_config_resp_t *)msg)->subtype)));
+
+	if (ltoh16(ring_config_resp->compl_hdr.status) == 0) {
+		if (ring_config_resp->subtype ==
+			htol16(D2H_RING_CONFIG_SUBTYPE_MDATA_LINK)) {
+			cpl_ring = prot->d2hring_md_cpl->linked_ring;
+			cpl_ring->linked_ring = prot->d2hring_md_cpl;
+		}
+		else if (ring_config_resp->subtype ==
+			htol16(D2H_RING_CONFIG_SUBTYPE_MDATA_UNLINK)) {
+			cpl_ring = prot->d2hring_md_cpl->linked_ring;
+			cpl_ring->linked_ring = NULL;
+			prot->d2hring_md_cpl->linked_ring = NULL;
+			bzero(dhd->mdring_info,
+				(MAX_MDRING_ITEM_DUMP * D2HRING_MDCMPLT_ITEMSIZE));
+		}
+	} else {
+		DHD_ERROR(("%s: Ring Config Failed - status %d ringid %d subtype:%d\n",
+			__FUNCTION__, ltoh16(((ring_config_resp_t *)msg)->compl_hdr.status),
+			ltoh16(((ring_config_resp_t *)msg)->compl_hdr.flow_ring_id),
+			ltoh16(((ring_config_resp_t *)msg)->subtype)));
+		prot->d2hring_md_cpl->linked_ring = NULL;
+	}
+}
+
+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
+void
+copy_ext_trap_sig(dhd_pub_t *dhd, trap_t *tr)
+{
+	uint32 *ext_data = dhd->extended_trap_data;
+	hnd_ext_trap_hdr_t *hdr;
+	const bcm_tlv_t *tlv;
+
+	if (ext_data == NULL) {
+		return;
+	}
+	/* First word is original trap_data */
+	ext_data++;
+
+	/* Followed by the extended trap data header */
+	hdr = (hnd_ext_trap_hdr_t *)ext_data;
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_SIGNATURE);
+	if (tlv) {
+		memcpy(tr, &tlv->data, sizeof(struct _trap_struct));
+	}
+}
+#define TRAP_T_NAME_OFFSET(var) {#var, OFFSETOF(trap_t, var)}
+
+typedef struct {
+	char name[HANG_INFO_TRAP_T_NAME_MAX];
+	uint32 offset;
+} hang_info_trap_t;
+
+#ifdef DHD_EWPR_VER2
+static hang_info_trap_t hang_info_trap_tbl[] = {
+	{"reason", 0},
+	{"ver", VENDOR_SEND_HANG_EXT_INFO_VER},
+	{"stype", 0},
+	TRAP_T_NAME_OFFSET(type),
+	TRAP_T_NAME_OFFSET(epc),
+	{"resrvd", 0},
+	{"resrvd", 0},
+	{"resrvd", 0},
+	{"resrvd", 0},
+	{"", 0}
+};
+#else
+static hang_info_trap_t hang_info_trap_tbl[] = {
+	{"reason", 0},
+	{"ver", VENDOR_SEND_HANG_EXT_INFO_VER},
+	{"stype", 0},
+	TRAP_T_NAME_OFFSET(type),
+	TRAP_T_NAME_OFFSET(epc),
+	TRAP_T_NAME_OFFSET(cpsr),
+	TRAP_T_NAME_OFFSET(spsr),
+	TRAP_T_NAME_OFFSET(r0),
+	TRAP_T_NAME_OFFSET(r1),
+	TRAP_T_NAME_OFFSET(r2),
+	TRAP_T_NAME_OFFSET(r3),
+	TRAP_T_NAME_OFFSET(r4),
+	TRAP_T_NAME_OFFSET(r5),
+	TRAP_T_NAME_OFFSET(r6),
+	TRAP_T_NAME_OFFSET(r7),
+	TRAP_T_NAME_OFFSET(r8),
+	TRAP_T_NAME_OFFSET(r9),
+	TRAP_T_NAME_OFFSET(r10),
+	TRAP_T_NAME_OFFSET(r11),
+	TRAP_T_NAME_OFFSET(r12),
+	TRAP_T_NAME_OFFSET(r13),
+	TRAP_T_NAME_OFFSET(r14),
+	TRAP_T_NAME_OFFSET(pc),
+	{"", 0}
+};
+#endif /* DHD_EWPR_VER2 */
+
+#define TAG_TRAP_IS_STATE(tag) \
+	((tag == TAG_TRAP_MEMORY) || (tag == TAG_TRAP_PCIE_Q) || \
+	(tag == TAG_TRAP_WLC_STATE) || (tag == TAG_TRAP_LOG_DATA) || \
+	(tag == TAG_TRAP_CODE))
+
+static void
+copy_hang_info_head(char *dest, trap_t *src, int len, int field_name,
+		int *bytes_written, int *cnt, char *cookie)
+{
+	uint8 *ptr;
+	int remain_len;
+	int i;
+
+	ptr = (uint8 *)src;
+
+	memset(dest, 0, len);
+	remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+
+	/* hang reason, hang info ver */
+	for (i = 0; (i < HANG_INFO_TRAP_T_SUBTYPE_IDX) && (*cnt < HANG_FIELD_CNT_MAX);
+			i++, (*cnt)++) {
+		if (field_name) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s:%c",
+					hang_info_trap_tbl[i].name, HANG_KEY_DEL);
+		}
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%d%c",
+				hang_info_trap_tbl[i].offset, HANG_KEY_DEL);
+
+	}
+
+	if (*cnt < HANG_FIELD_CNT_MAX) {
+		if (field_name) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s:%c",
+					"cookie", HANG_KEY_DEL);
+		}
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s%c",
+				cookie, HANG_KEY_DEL);
+		(*cnt)++;
+	}
+
+	if (*cnt < HANG_FIELD_CNT_MAX) {
+		if (field_name) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s:%c",
+					hang_info_trap_tbl[HANG_INFO_TRAP_T_SUBTYPE_IDX].name,
+					HANG_KEY_DEL);
+		}
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%08x%c",
+				hang_info_trap_tbl[HANG_INFO_TRAP_T_SUBTYPE_IDX].offset,
+				HANG_KEY_DEL);
+		(*cnt)++;
+	}
+
+	if (*cnt < HANG_FIELD_CNT_MAX) {
+		if (field_name) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s:%c",
+					hang_info_trap_tbl[HANG_INFO_TRAP_T_EPC_IDX].name,
+					HANG_KEY_DEL);
+		}
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%08x%c",
+				*(uint32 *)
+				(ptr + hang_info_trap_tbl[HANG_INFO_TRAP_T_EPC_IDX].offset),
+				HANG_KEY_DEL);
+		(*cnt)++;
+	}
+#ifdef DHD_EWPR_VER2
+	/* put 0 for HG03 ~ HG06 (reserved for future use) */
+	for (i = 0; (i < HANG_INFO_BIGDATA_EXTRA_KEY) && (*cnt < HANG_FIELD_CNT_MAX);
+			i++, (*cnt)++) {
+		if (field_name) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s:%c",
+				hang_info_trap_tbl[HANG_INFO_TRAP_T_EXTRA_KEY_IDX+i].name,
+				HANG_KEY_DEL);
+		}
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%d%c",
+			hang_info_trap_tbl[HANG_INFO_TRAP_T_EXTRA_KEY_IDX+i].offset,
+			HANG_KEY_DEL);
+	}
+#endif /* DHD_EWPR_VER2 */
+}
+#ifndef DHD_EWPR_VER2
+static void
+copy_hang_info_trap_t(char *dest, trap_t *src, int len, int field_name,
+		int *bytes_written, int *cnt, char *cookie)
+{
+	uint8 *ptr;
+	int remain_len;
+	int i;
+
+	ptr = (uint8 *)src;
+
+	remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+
+	for (i = HANG_INFO_TRAP_T_OFFSET_IDX;
+			(hang_info_trap_tbl[i].name[0] != 0) && (*cnt < HANG_FIELD_CNT_MAX);
+			i++, (*cnt)++) {
+		if (field_name) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%c%s:",
+					HANG_RAW_DEL, hang_info_trap_tbl[i].name);
+		}
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%c%08x",
+				HANG_RAW_DEL, *(uint32 *)(ptr + hang_info_trap_tbl[i].offset));
+	}
+}
+
+static void
+copy_hang_info_stack(dhd_pub_t *dhd, char *dest, int *bytes_written, int *cnt)
+{
+	int remain_len;
+	int i = 0;
+	const uint32 *stack;
+	uint32 *ext_data = dhd->extended_trap_data;
+	hnd_ext_trap_hdr_t *hdr;
+	const bcm_tlv_t *tlv;
+	int remain_stack_cnt = 0;
+	uint32 dummy_data = 0;
+	int bigdata_key_stack_cnt = 0;
+
+	if (ext_data == NULL) {
+		return;
+	}
+	/* First word is original trap_data */
+	ext_data++;
+
+	/* Followed by the extended trap data header */
+	hdr = (hnd_ext_trap_hdr_t *)ext_data;
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_STACK);
+
+	remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	if (tlv) {
+		stack = (const uint32 *)tlv->data;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len,
+				"%08x", *(uint32 *)(stack++));
+		(*cnt)++;
+		if (*cnt >= HANG_FIELD_CNT_MAX) {
+			return;
+		}
+		for (i = 1; i < (uint32)(tlv->len / sizeof(uint32)); i++, bigdata_key_stack_cnt++) {
+			remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+			/* Raw data for bigdata use '_' and Key data for bigdata use space */
+			*bytes_written += scnprintf(&dest[*bytes_written], remain_len,
+				"%c%08x",
+				i <= HANG_INFO_BIGDATA_KEY_STACK_CNT ? HANG_KEY_DEL : HANG_RAW_DEL,
+				*(uint32 *)(stack++));
+
+			(*cnt)++;
+			if ((*cnt >= HANG_FIELD_CNT_MAX) ||
+					(i >= HANG_FIELD_TRAP_T_STACK_CNT_MAX)) {
+				return;
+			}
+		}
+	}
+
+	remain_stack_cnt = HANG_FIELD_TRAP_T_STACK_CNT_MAX - i;
+
+	for (i = 0; i < remain_stack_cnt; i++) {
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%c%08x",
+				HANG_RAW_DEL, dummy_data);
+		(*cnt)++;
+		if (*cnt >= HANG_FIELD_CNT_MAX) {
+			return;
+		}
+	}
+	GCC_DIAGNOSTIC_POP();
+
+}
+
+static void
+copy_hang_info_specific(dhd_pub_t *dhd, char *dest, int *bytes_written, int *cnt)
+{
+	int remain_len;
+	int i;
+	const uint32 *data;
+	uint32 *ext_data = dhd->extended_trap_data;
+	hnd_ext_trap_hdr_t *hdr;
+	const bcm_tlv_t *tlv;
+	int remain_trap_data = 0;
+	uint8 buf_u8[sizeof(uint32)] = { 0, };
+	const uint8 *p_u8;
+
+	if (ext_data == NULL) {
+		return;
+	}
+	/* First word is original trap_data */
+	ext_data++;
+
+	/* Followed by the extended trap data header */
+	hdr = (hnd_ext_trap_hdr_t *)ext_data;
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_SIGNATURE);
+	if (tlv) {
+		/* header include tlv hader */
+		remain_trap_data = (hdr->len - tlv->len - sizeof(uint16));
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_STACK);
+	if (tlv) {
+		/* header include tlv hader */
+		remain_trap_data -= (tlv->len + sizeof(uint16));
+	}
+
+	data = (const uint32 *)(hdr->data + (hdr->len  - remain_trap_data));
+
+	remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+
+	for (i = 0; i < (uint32)(remain_trap_data / sizeof(uint32)) && *cnt < HANG_FIELD_CNT_MAX;
+			i++, (*cnt)++) {
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%c%08x",
+				HANG_RAW_DEL, *(uint32 *)(data++));
+		GCC_DIAGNOSTIC_POP();
+	}
+
+	if (*cnt >= HANG_FIELD_CNT_MAX) {
+		return;
+	}
+
+	remain_trap_data -= (sizeof(uint32) * i);
+
+	if (remain_trap_data > sizeof(buf_u8)) {
+		DHD_ERROR(("%s: resize remain_trap_data\n", __FUNCTION__));
+		remain_trap_data =  sizeof(buf_u8);
+	}
+
+	if (remain_trap_data) {
+		p_u8 = (const uint8 *)data;
+		for (i = 0; i < remain_trap_data; i++) {
+			buf_u8[i] = *(const uint8 *)(p_u8++);
+		}
+
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+		*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%c%08x",
+				HANG_RAW_DEL, ltoh32_ua(buf_u8));
+		(*cnt)++;
+	}
+}
+#endif /* DHD_EWPR_VER2 */
+
+static void
+get_hang_info_trap_subtype(dhd_pub_t *dhd, uint32 *subtype)
+{
+	uint32 i;
+	uint32 *ext_data = dhd->extended_trap_data;
+	hnd_ext_trap_hdr_t *hdr;
+	const bcm_tlv_t *tlv;
+
+	/* First word is original trap_data */
+	ext_data++;
+
+	/* Followed by the extended trap data header */
+	hdr = (hnd_ext_trap_hdr_t *)ext_data;
+
+	/* Dump a list of all tags found  before parsing data */
+	for (i = TAG_TRAP_DEEPSLEEP; i < TAG_TRAP_LAST; i++) {
+		tlv = bcm_parse_tlvs(hdr->data, hdr->len, i);
+		if (tlv) {
+			if (!TAG_TRAP_IS_STATE(i)) {
+				*subtype = i;
+				return;
+			}
+		}
+	}
+}
+#ifdef DHD_EWPR_VER2
+static void
+copy_hang_info_etd_base64(dhd_pub_t *dhd, char *dest, int *bytes_written, int *cnt)
+{
+	int remain_len;
+	uint32 *ext_data = dhd->extended_trap_data;
+	hnd_ext_trap_hdr_t *hdr;
+	char *base64_out = NULL;
+	int base64_cnt;
+	int max_base64_len = HANG_INFO_BASE64_BUFFER_SIZE;
+
+	if (ext_data == NULL) {
+		return;
+	}
+	/* First word is original trap_data */
+	ext_data++;
+
+	/* Followed by the extended trap data header */
+	hdr = (hnd_ext_trap_hdr_t *)ext_data;
+
+	remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
+
+	if (remain_len <= 0) {
+		DHD_ERROR(("%s: no space to put etd\n", __FUNCTION__));
+		return;
+	}
+
+	if (remain_len < max_base64_len) {
+		DHD_ERROR(("%s: change max base64 length to remain length %d\n", __FUNCTION__,
+			remain_len));
+		max_base64_len = remain_len;
+	}
+
+	base64_out = MALLOCZ(dhd->osh, HANG_INFO_BASE64_BUFFER_SIZE);
+	if (base64_out == NULL) {
+		DHD_ERROR(("%s: MALLOC failed for size %d\n",
+			__FUNCTION__, HANG_INFO_BASE64_BUFFER_SIZE));
+		return;
+	}
+
+	if (hdr->len > 0) {
+		base64_cnt = dhd_base64_encode(hdr->data, hdr->len, base64_out, max_base64_len);
+		if (base64_cnt == 0) {
+			DHD_ERROR(("%s: base64 encoding error\n", __FUNCTION__));
+		}
+	}
+
+	*bytes_written += scnprintf(&dest[*bytes_written], remain_len, "%s",
+			base64_out);
+	(*cnt)++;
+	MFREE(dhd->osh, base64_out, HANG_INFO_BASE64_BUFFER_SIZE);
+}
+#endif /* DHD_EWPR_VER2 */
+
+void
+copy_hang_info_trap(dhd_pub_t *dhd)
+{
+	trap_t tr;
+	int bytes_written;
+	int trap_subtype = 0;
+
+	if (!dhd || !dhd->hang_info) {
+		DHD_ERROR(("%s dhd=%p hang_info=%p\n", __FUNCTION__,
+			dhd, (dhd ? dhd->hang_info : NULL)));
+		return;
+	}
+
+	if (!dhd->dongle_trap_occured) {
+		DHD_ERROR(("%s: dongle_trap_occured is FALSE\n", __FUNCTION__));
+		return;
+	}
+
+	memset(&tr, 0x00, sizeof(struct _trap_struct));
+
+	copy_ext_trap_sig(dhd, &tr);
+	get_hang_info_trap_subtype(dhd, &trap_subtype);
+
+	hang_info_trap_tbl[HANG_INFO_TRAP_T_REASON_IDX].offset = HANG_REASON_DONGLE_TRAP;
+	hang_info_trap_tbl[HANG_INFO_TRAP_T_SUBTYPE_IDX].offset = trap_subtype;
+
+	bytes_written = 0;
+	dhd->hang_info_cnt = 0;
+	get_debug_dump_time(dhd->debug_dump_time_hang_str);
+	copy_debug_dump_time(dhd->debug_dump_time_str, dhd->debug_dump_time_hang_str);
+
+	copy_hang_info_head(dhd->hang_info, &tr, VENDOR_SEND_HANG_EXT_INFO_LEN, FALSE,
+			&bytes_written, &dhd->hang_info_cnt, dhd->debug_dump_time_hang_str);
+
+	DHD_INFO(("hang info head cnt: %d len: %d data: %s\n",
+		dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
+
+	clear_debug_dump_time(dhd->debug_dump_time_hang_str);
+
+#ifdef DHD_EWPR_VER2
+	/* stack info & trap info are included in etd data */
+
+	/* extended trap data dump */
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		copy_hang_info_etd_base64(dhd, dhd->hang_info, &bytes_written, &dhd->hang_info_cnt);
+		DHD_INFO(("hang info specific cnt: %d len: %d data: %s\n",
+			dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
+	}
+#else
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		copy_hang_info_stack(dhd, dhd->hang_info, &bytes_written, &dhd->hang_info_cnt);
+		DHD_INFO(("hang info stack cnt: %d len: %d data: %s\n",
+			dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
+	}
+
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		copy_hang_info_trap_t(dhd->hang_info, &tr, VENDOR_SEND_HANG_EXT_INFO_LEN, FALSE,
+				&bytes_written, &dhd->hang_info_cnt, dhd->debug_dump_time_hang_str);
+		DHD_INFO(("hang info trap_t cnt: %d len: %d data: %s\n",
+			dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
+	}
+
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		copy_hang_info_specific(dhd, dhd->hang_info, &bytes_written, &dhd->hang_info_cnt);
+		DHD_INFO(("hang info specific cnt: %d len: %d data: %s\n",
+			dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
+	}
+#endif /* DHD_EWPR_VER2 */
+}
+
+void
+copy_hang_info_linkdown(dhd_pub_t *dhd)
+{
+	int bytes_written = 0;
+	int remain_len;
+
+	if (!dhd || !dhd->hang_info) {
+		DHD_ERROR(("%s dhd=%p hang_info=%p\n", __FUNCTION__,
+			dhd, (dhd ? dhd->hang_info : NULL)));
+		return;
+	}
+
+	if (!dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s: link down is not happened\n", __FUNCTION__));
+		return;
+	}
+
+	dhd->hang_info_cnt = 0;
+
+	get_debug_dump_time(dhd->debug_dump_time_hang_str);
+	copy_debug_dump_time(dhd->debug_dump_time_str, dhd->debug_dump_time_hang_str);
+
+#ifdef BCMPCIE
+	/* hang reason code (0x8808) */
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - bytes_written;
+		bytes_written += scnprintf(&dhd->hang_info[bytes_written], remain_len, "%d%c",
+				HANG_REASON_PCIE_LINK_DOWN_EP_DETECT, HANG_KEY_DEL);
+		dhd->hang_info_cnt++;
+	}
+#endif
+
+	/* EWP version */
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - bytes_written;
+		bytes_written += scnprintf(&dhd->hang_info[bytes_written], remain_len, "%d%c",
+				VENDOR_SEND_HANG_EXT_INFO_VER, HANG_KEY_DEL);
+		dhd->hang_info_cnt++;
+	}
+
+	/* cookie - dump time stamp */
+	if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
+		remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - bytes_written;
+		bytes_written += scnprintf(&dhd->hang_info[bytes_written], remain_len, "%s%c",
+				dhd->debug_dump_time_hang_str, HANG_KEY_DEL);
+		dhd->hang_info_cnt++;
+	}
+
+	clear_debug_dump_time(dhd->debug_dump_time_hang_str);
+
+	DHD_INFO(("hang info haed cnt: %d len: %d data: %s\n",
+		dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
+
+}
+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
+
+void
+dhd_prot_ctrl_info_print(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring;
+	uint16 rd, wr, drd, dwr;
+	uint32 dma_buf_len;
+	ulong ring_tcm_rd_addr; /* dongle address */
+	ulong ring_tcm_wr_addr; /* dongle address */
+
+	ring = &prot->h2dring_ctrl_subn;
+	dma_buf_len = ring->max_items * ring->item_len;
+	ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+	ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+	DHD_ERROR(("CtrlPost: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+		"SIZE %d \r\n",
+		ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+		ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr, dma_buf_len));
+	DHD_ERROR(("CtrlPost: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+	if (dhd->dma_d2h_ring_upd_support) {
+		drd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+		dwr = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, ring->idx);
+		DHD_ERROR(("CtrlPost: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+	}
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("CtrlPost: From Shared Mem: RD and WR are invalid"
+			" due to PCIe link down\r\n"));
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+		dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+		DHD_ERROR(("CtrlPost: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+	}
+	DHD_ERROR(("CtrlPost: seq num: %d \r\n", ring->seqnum % H2D_EPOCH_MODULO));
+
+	ring = &prot->d2hring_ctrl_cpln;
+	dma_buf_len = ring->max_items * ring->item_len;
+	ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+	ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+	DHD_ERROR(("CtrlCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+		"SIZE %d \r\n",
+		ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+		ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr, dma_buf_len));
+	DHD_ERROR(("CtrlCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+	DHD_ERROR(("CtrlCpl: From Host mem: CURR_RD: %d\r\n", ring->curr_rd));
+	if (dhd->dma_d2h_ring_upd_support) {
+		drd = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+		dwr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+		DHD_ERROR(("CtrlCpl: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+	}
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("CtrlCpl: From Shared Mem: RD and WR are invalid"
+			" due to PCIe link down\r\n"));
+	} else {
+		dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+		dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+		DHD_ERROR(("CtrlCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+	}
+	DHD_ERROR(("CtrlCpl: Expected seq num: %d \r\n", ring->seqnum % H2D_EPOCH_MODULO));
+
+}
+
+void
+dhd_prot_debug_ring_info(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring;
+	uint16 rd, wr, drd, dwr;
+	uint32 dma_buf_len;
+	ulong ring_tcm_rd_addr; /* dongle address */
+	ulong ring_tcm_wr_addr; /* dongle address */
+
+	DHD_ERROR(("\n ------- DUMPING IOCTL RING RD WR Pointers ------- \r\n"));
+
+	dhd_prot_ctrl_info_print(dhd);
+
+	ring = prot->h2dring_info_subn;
+	if (ring) {
+		dma_buf_len = ring->max_items * ring->item_len;
+		ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+		ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+		DHD_ERROR(("InfoSub: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+			"SIZE %d \r\n",
+			ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+			ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
+			dma_buf_len));
+		DHD_ERROR(("InfoSub: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+		if (dhd->dma_d2h_ring_upd_support) {
+			drd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, ring->idx);
+			DHD_ERROR(("InfoSub: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+		}
+		if (dhd->bus->is_linkdown) {
+			DHD_ERROR(("InfoSub: From Shared Mem: RD and WR are invalid"
+				" due to PCIe link down\r\n"));
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+			dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+			DHD_ERROR(("InfoSub: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+		}
+		DHD_ERROR(("InfoSub: seq num: %d \r\n", ring->seqnum % H2D_EPOCH_MODULO));
+	}
+	ring = prot->d2hring_info_cpln;
+	if (ring) {
+		dma_buf_len = ring->max_items * ring->item_len;
+		ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+		ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+		DHD_ERROR(("InfoCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+			"SIZE %d \r\n",
+			ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+			ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
+			dma_buf_len));
+		DHD_ERROR(("InfoCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+		DHD_ERROR(("InfoCpl: From Host mem: CURR_RD: %d\r\n", ring->curr_rd));
+		if (dhd->dma_d2h_ring_upd_support) {
+			drd = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+			DHD_ERROR(("InfoCpl: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+		}
+		if (dhd->bus->is_linkdown) {
+			DHD_ERROR(("InfoCpl: From Shared Mem: RD and WR are invalid"
+				" due to PCIe link down\r\n"));
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+			dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+			DHD_ERROR(("InfoCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+		}
+		DHD_ERROR(("InfoCpl: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
+	}
+#ifdef EWP_EDL
+	ring = prot->d2hring_edl;
+	if (ring) {
+		ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+		ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+		dma_buf_len = ring->max_items * ring->item_len;
+		DHD_ERROR(("EdlRing: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+			"SIZE %d \r\n",
+			ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+			ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
+			dma_buf_len));
+		DHD_ERROR(("EdlRing: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+		DHD_ERROR(("EdlRing: From Host mem: CURR_RD: %d\r\n", ring->curr_rd));
+		if (dhd->dma_d2h_ring_upd_support) {
+			drd = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+			DHD_ERROR(("EdlRing: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+		}
+		if (dhd->bus->is_linkdown) {
+			DHD_ERROR(("EdlRing: From Shared Mem: RD and WR are invalid"
+				" due to PCIe link down\r\n"));
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+			dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+			DHD_ERROR(("EdlRing: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+		}
+		DHD_ERROR(("EdlRing: Expected seq num: %d \r\n",
+			ring->seqnum % D2H_EPOCH_MODULO));
+	}
+#endif /* EWP_EDL */
+
+	ring = &prot->d2hring_tx_cpln;
+	if (ring) {
+		ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+		ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+		dma_buf_len = ring->max_items * ring->item_len;
+		DHD_ERROR(("TxCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+			"SIZE %d \r\n",
+			ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+			ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
+			dma_buf_len));
+		DHD_ERROR(("TxCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+		DHD_ERROR(("TxCpl: From Host mem: CURR_RD: %d\r\n", ring->curr_rd));
+		if (dhd->dma_d2h_ring_upd_support) {
+			drd = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+			DHD_ERROR(("TxCpl: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+		}
+		if (dhd->bus->is_linkdown) {
+			DHD_ERROR(("TxCpl: From Shared Mem: RD and WR are invalid"
+				" due to PCIe link down\r\n"));
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+			dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+			DHD_ERROR(("TxCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+		}
+		DHD_ERROR(("TxCpl: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
+	}
+
+	ring = &prot->d2hring_rx_cpln;
+	if (ring) {
+		ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+		ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+		dma_buf_len = ring->max_items * ring->item_len;
+		DHD_ERROR(("RxCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+			"SIZE %d \r\n",
+			ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+			ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
+			dma_buf_len));
+		DHD_ERROR(("RxCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+		DHD_ERROR(("RxCpl: From Host mem: CURR_RD: %d\r\n", ring->curr_rd));
+		if (dhd->dma_d2h_ring_upd_support) {
+			drd = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+			DHD_ERROR(("RxCpl: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+		}
+		if (dhd->bus->is_linkdown) {
+			DHD_ERROR(("RxCpl: From Shared Mem: RD and WR are invalid"
+				" due to PCIe link down\r\n"));
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+			dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+			DHD_ERROR(("RxCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+		}
+		DHD_ERROR(("RxCpl: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
+	}
+
+	ring = &prot->h2dring_rxp_subn;
+	if (ring) {
+		ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
+		ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
+		dma_buf_len = ring->max_items * ring->item_len;
+		DHD_ERROR(("RxSub: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
+			"SIZE %d \r\n",
+			ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
+			ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
+			dma_buf_len));
+		DHD_ERROR(("RxSub: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
+		if (dhd->dma_d2h_ring_upd_support) {
+			drd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+			dwr = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, ring->idx);
+			DHD_ERROR(("RxSub: From Host DMA mem: RD: %d WR %d \r\n", drd, dwr));
+		}
+		if (dhd->bus->is_linkdown) {
+			DHD_ERROR(("RxSub: From Shared Mem: RD and WR are invalid"
+				" due to PCIe link down\r\n"));
+		} else {
+			dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
+			dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
+			DHD_ERROR(("RxSub: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
+		}
+		DHD_ERROR(("RxSub: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
+	}
+
+}
+
+int
+dhd_prot_debug_info_print(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	uint64 current_time;
+
+	DHD_ERROR(("\n ------- DUMPING VERSION INFORMATION ------- \r\n"));
+	DHD_ERROR(("DHD: %s\n", dhd_version));
+	DHD_ERROR(("Firmware: %s\n", fw_version));
+
+#ifdef DHD_FW_COREDUMP
+	DHD_ERROR(("\n ------- DUMPING CONFIGURATION INFORMATION ------ \r\n"));
+	DHD_ERROR(("memdump mode: %d\n", dhd->memdump_enabled));
+#endif /* DHD_FW_COREDUMP */
+
+	DHD_ERROR(("\n ------- DUMPING PROTOCOL INFORMATION ------- \r\n"));
+	DHD_ERROR(("ICPrevs: Dev %d, Host %d, active %d\n",
+		prot->device_ipc_version,
+		prot->host_ipc_version,
+		prot->active_ipc_version));
+	DHD_ERROR(("d2h_intr_method -> %s d2h_intr_control -> %s\n",
+			dhd->bus->d2h_intr_method ? "PCIE_MSI" : "PCIE_INTX",
+			dhd->bus->d2h_intr_control ? "HOST_IRQ" : "D2H_INTMASK"));
+	DHD_ERROR(("max Host TS bufs to post: %d, posted %d\n",
+		prot->max_tsbufpost, prot->cur_ts_bufs_posted));
+	DHD_ERROR(("max INFO bufs to post: %d, posted %d\n",
+		prot->max_infobufpost, prot->infobufpost));
+	DHD_ERROR(("max event bufs to post: %d, posted %d\n",
+		prot->max_eventbufpost, prot->cur_event_bufs_posted));
+	DHD_ERROR(("max ioctlresp bufs to post: %d, posted %d\n",
+		prot->max_ioctlrespbufpost, prot->cur_ioctlresp_bufs_posted));
+	DHD_ERROR(("max RX bufs to post: %d, posted %d\n",
+		prot->max_rxbufpost, prot->rxbufpost));
+	DHD_ERROR(("h2d_max_txpost: %d, prot->h2d_max_txpost: %d\n",
+		h2d_max_txpost, prot->h2d_max_txpost));
+	if (dhd->htput_support) {
+		DHD_ERROR(("h2d_max_txpost: %d, prot->h2d_max_txpost: %d\n",
+			h2d_htput_max_txpost, prot->h2d_htput_max_txpost));
+	}
+
+	current_time = OSL_LOCALTIME_NS();
+	DHD_ERROR(("current_time="SEC_USEC_FMT"\n", GET_SEC_USEC(current_time)));
+	DHD_ERROR(("ioctl_fillup_time="SEC_USEC_FMT
+		" ioctl_ack_time="SEC_USEC_FMT
+		" ioctl_cmplt_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(prot->ioctl_fillup_time),
+		GET_SEC_USEC(prot->ioctl_ack_time),
+		GET_SEC_USEC(prot->ioctl_cmplt_time)));
+
+	/* Check PCIe INT registers */
+	if (!dhd_pcie_dump_int_regs(dhd)) {
+		DHD_ERROR(("%s : PCIe link might be down\n", __FUNCTION__));
+		dhd->bus->is_linkdown = TRUE;
+	}
+
+	dhd_prot_debug_ring_info(dhd);
+
+	DHD_ERROR(("%s: cur_ioctlresp_bufs_posted %d cur_event_bufs_posted %d\n",
+		__FUNCTION__, prot->cur_ioctlresp_bufs_posted, prot->cur_event_bufs_posted));
+#ifdef DHD_LIMIT_MULTI_CLIENT_FLOWRINGS
+	DHD_ERROR(("%s: multi_client_flow_rings:%u max_multi_client_flow_rings:%d\n",
+		__FUNCTION__, OSL_ATOMIC_READ(dhd->osh, &dhd->multi_client_flow_rings),
+		dhd->max_multi_client_flow_rings));
+#endif /* DHD_LIMIT_MULTI_CLIENT_FLOWRINGS */
+
+	DHD_ERROR(("pktid_txq_start_cnt: %d\n", prot->pktid_txq_start_cnt));
+	DHD_ERROR(("pktid_txq_stop_cnt: %d\n", prot->pktid_txq_stop_cnt));
+	DHD_ERROR(("pktid_depleted_cnt: %d\n", prot->pktid_depleted_cnt));
+	dhd_pcie_debug_info_dump(dhd);
+#ifdef DHD_LB_STATS
+	DHD_ERROR(("\nlb_rxp_stop_thr_hitcnt: %llu lb_rxp_strt_thr_hitcnt: %llu\n",
+		dhd->lb_rxp_stop_thr_hitcnt, dhd->lb_rxp_strt_thr_hitcnt));
+	DHD_ERROR(("\nlb_rxp_napi_sched_cnt: %llu lb_rxp_napi_complete_cnt: %llu\n",
+		dhd->lb_rxp_napi_sched_cnt, dhd->lb_rxp_napi_complete_cnt));
+#endif /* DHD_LB_STATS */
+	return 0;
+}
+
+int
+dhd_prot_ringupd_dump(dhd_pub_t *dhd, struct bcmstrbuf *b)
+{
+	uint32 *ptr;
+	uint32 value;
+
+	if (dhd->prot->d2h_dma_indx_wr_buf.va) {
+		uint32 i;
+		uint32 max_h2d_queues = dhd_bus_max_h2d_queues(dhd->bus);
+
+		OSL_CACHE_INV((void *)dhd->prot->d2h_dma_indx_wr_buf.va,
+			dhd->prot->d2h_dma_indx_wr_buf.len);
+
+		ptr = (uint32 *)(dhd->prot->d2h_dma_indx_wr_buf.va);
+
+		bcm_bprintf(b, "\n max_tx_queues %d\n", max_h2d_queues);
+
+		bcm_bprintf(b, "\nRPTR block H2D common rings, 0x%4p\n", ptr);
+		value = ltoh32(*ptr);
+		bcm_bprintf(b, "\tH2D CTRL: value 0x%04x\n", value);
+		ptr++;
+		value = ltoh32(*ptr);
+		bcm_bprintf(b, "\tH2D RXPOST: value 0x%04x\n", value);
+
+		ptr++;
+		bcm_bprintf(b, "RPTR block Flow rings , 0x%4p\n", ptr);
+		for (i = BCMPCIE_H2D_COMMON_MSGRINGS; i < max_h2d_queues; i++) {
+			value = ltoh32(*ptr);
+			bcm_bprintf(b, "\tflowring ID %d: value 0x%04x\n", i, value);
+			ptr++;
+		}
+	}
+
+	if (dhd->prot->h2d_dma_indx_rd_buf.va) {
+		OSL_CACHE_INV((void *)dhd->prot->h2d_dma_indx_rd_buf.va,
+			dhd->prot->h2d_dma_indx_rd_buf.len);
+
+		ptr = (uint32 *)(dhd->prot->h2d_dma_indx_rd_buf.va);
+
+		bcm_bprintf(b, "\nWPTR block D2H common rings, 0x%4p\n", ptr);
+		value = ltoh32(*ptr);
+		bcm_bprintf(b, "\tD2H CTRLCPLT: value 0x%04x\n", value);
+		ptr++;
+		value = ltoh32(*ptr);
+		bcm_bprintf(b, "\tD2H TXCPLT: value 0x%04x\n", value);
+		ptr++;
+		value = ltoh32(*ptr);
+		bcm_bprintf(b, "\tD2H RXCPLT: value 0x%04x\n", value);
+	}
+
+	return 0;
+}
+
+uint32
+dhd_prot_metadata_dbg_set(dhd_pub_t *dhd, bool val)
+{
+	dhd_prot_t *prot = dhd->prot;
+#if DHD_DBG_SHOW_METADATA
+	prot->metadata_dbg = val;
+#endif
+	return (uint32)prot->metadata_dbg;
+}
+
+uint32
+dhd_prot_metadata_dbg_get(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	return (uint32)prot->metadata_dbg;
+}
+
+uint32
+dhd_prot_metadatalen_set(dhd_pub_t *dhd, uint32 val, bool rx)
+{
+#if !defined(BCM_ROUTER_DHD)
+	dhd_prot_t *prot = dhd->prot;
+	if (rx)
+		prot->rx_metadata_offset = (uint16)val;
+	else
+		prot->tx_metadata_offset = (uint16)val;
+#endif /* ! BCM_ROUTER_DHD */
+	return dhd_prot_metadatalen_get(dhd, rx);
+}
+
+uint32
+dhd_prot_metadatalen_get(dhd_pub_t *dhd, bool rx)
+{
+	dhd_prot_t *prot = dhd->prot;
+	if (rx)
+		return prot->rx_metadata_offset;
+	else
+		return prot->tx_metadata_offset;
+}
+
+/** optimization to write "n" tx items at a time to ring */
+uint32
+dhd_prot_txp_threshold(dhd_pub_t *dhd, bool set, uint32 val)
+{
+	dhd_prot_t *prot = dhd->prot;
+	if (set)
+		prot->txp_threshold = (uint16)val;
+	val = prot->txp_threshold;
+	return val;
+}
+
+#ifdef DHD_RX_CHAINING
+
+static INLINE void
+BCMFASTPATH(dhd_rxchain_reset)(rxchain_info_t *rxchain)
+{
+	rxchain->pkt_count = 0;
+}
+
+static void
+BCMFASTPATH(dhd_rxchain_frame)(dhd_pub_t *dhd, void *pkt, uint ifidx)
+{
+	uint8 *eh;
+	uint8 prio;
+	dhd_prot_t *prot = dhd->prot;
+	rxchain_info_t *rxchain = &prot->rxchain;
+
+	ASSERT(!PKTISCHAINED(pkt));
+	ASSERT(PKTCLINK(pkt) == NULL);
+	ASSERT(PKTCGETATTR(pkt) == 0);
+
+	eh = PKTDATA(dhd->osh, pkt);
+	prio = IP_TOS46(eh + ETHER_HDR_LEN) >> IPV4_TOS_PREC_SHIFT;
+
+	if (rxchain->pkt_count && !(PKT_CTF_CHAINABLE(dhd, ifidx, eh, prio, rxchain->h_sa,
+		rxchain->h_da, rxchain->h_prio))) {
+		/* Different flow - First release the existing chain */
+		dhd_rxchain_commit(dhd);
+	}
+
+	/* For routers, with HNDCTF, link the packets using PKTSETCLINK, */
+	/* so that the chain can be handed off to CTF bridge as is. */
+	if (rxchain->pkt_count == 0) {
+		/* First packet in chain */
+		rxchain->pkthead = rxchain->pkttail = pkt;
+
+		/* Keep a copy of ptr to ether_da, ether_sa and prio */
+		rxchain->h_da = ((struct ether_header *)eh)->ether_dhost;
+		rxchain->h_sa = ((struct ether_header *)eh)->ether_shost;
+		rxchain->h_prio = prio;
+		rxchain->ifidx = ifidx;
+		rxchain->pkt_count++;
+	} else {
+		/* Same flow - keep chaining */
+		PKTSETCLINK(rxchain->pkttail, pkt);
+		rxchain->pkttail = pkt;
+		rxchain->pkt_count++;
+	}
+
+	if ((dhd_rx_pkt_chainable(dhd, ifidx)) && (!ETHER_ISMULTI(rxchain->h_da)) &&
+		((((struct ether_header *)eh)->ether_type == HTON16(ETHER_TYPE_IP)) ||
+		(((struct ether_header *)eh)->ether_type == HTON16(ETHER_TYPE_IPV6)))) {
+		PKTSETCHAINED(dhd->osh, pkt);
+		PKTCINCRCNT(rxchain->pkthead);
+		PKTCADDLEN(rxchain->pkthead, PKTLEN(dhd->osh, pkt));
+	} else {
+		dhd_rxchain_commit(dhd);
+		return;
+	}
+
+	/* If we have hit the max chain length, dispatch the chain and reset */
+	if (rxchain->pkt_count >= DHD_PKT_CTF_MAX_CHAIN_LEN) {
+		dhd_rxchain_commit(dhd);
+	}
+}
+
+static void
+BCMFASTPATH(dhd_rxchain_commit)(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	rxchain_info_t *rxchain = &prot->rxchain;
+
+	if (rxchain->pkt_count == 0)
+		return;
+
+	/* Release the packets to dhd_linux */
+	dhd_bus_rx_frame(dhd->bus, rxchain->pkthead, rxchain->ifidx, rxchain->pkt_count);
+
+	/* Reset the chain */
+	dhd_rxchain_reset(rxchain);
+}
+
+#endif /* DHD_RX_CHAINING */
+
+#ifdef IDLE_TX_FLOW_MGMT
+int
+dhd_prot_flow_ring_resume(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node)
+{
+	tx_idle_flowring_resume_request_t *flow_resume_rqst;
+	msgbuf_ring_t *flow_ring;
+	dhd_prot_t *prot = dhd->prot;
+	unsigned long flags;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
+
+	/* Fetch a pre-initialized msgbuf_ring from the flowring pool */
+	flow_ring = dhd_prot_flowrings_pool_fetch(dhd, flow_ring_node->flowid);
+	if (flow_ring == NULL) {
+		DHD_ERROR(("%s: dhd_prot_flowrings_pool_fetch TX Flowid %d failed\n",
+			__FUNCTION__, flow_ring_node->flowid));
+		return BCME_NOMEM;
+	}
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+
+	/* Request for ctrl_ring buffer space */
+	flow_resume_rqst = (tx_idle_flowring_resume_request_t *)
+		dhd_prot_alloc_ring_space(dhd, ctrl_ring, 1, &alloced, FALSE);
+
+	if (flow_resume_rqst == NULL) {
+		dhd_prot_flowrings_pool_release(dhd, flow_ring_node->flowid, flow_ring);
+		DHD_ERROR(("%s: Flow resume Req flowid %d - failure ring space\n",
+			__FUNCTION__, flow_ring_node->flowid));
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_NOMEM;
+	}
+
+	flow_ring_node->prot_info = (void *)flow_ring;
+
+	/* Common msg buf hdr */
+	flow_resume_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_RESUME;
+	flow_resume_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
+	flow_resume_rqst->msg.request_id = htol32(0); /* TBD */
+
+	flow_resume_rqst->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+	ctrl_ring->seqnum++;
+
+	flow_resume_rqst->flow_ring_id = htol16((uint16)flow_ring_node->flowid);
+	DHD_ERROR(("%s Send Flow resume Req flow ID %d\n",
+		__FUNCTION__, flow_ring_node->flowid));
+
+	/* Update the flow_ring's WRITE index */
+	if (IDMA_ACTIVE(dhd) || dhd->dma_h2d_ring_upd_support) {
+		dhd_prot_dma_indx_set(dhd, flow_ring->wr,
+		                      H2D_DMA_INDX_WR_UPD, flow_ring->idx);
+	} else if (IFRM_ACTIVE(dhd) && (flow_ring->idx >= BCMPCIE_H2D_MSGRING_TXFLOW_IDX_START)) {
+		dhd_prot_dma_indx_set(dhd, flow_ring->wr,
+			H2D_IFRM_INDX_WR_UPD,
+			(flow_ring->idx - BCMPCIE_H2D_MSGRING_TXFLOW_IDX_START));
+	} else {
+		dhd_bus_cmn_writeshared(dhd->bus, &(flow_ring->wr),
+			sizeof(uint16), RING_WR_UPD, flow_ring->idx);
+	}
+
+	/* update control subn ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ctrl_ring, flow_resume_rqst, 1);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return BCME_OK;
+} /* dhd_prot_flow_ring_create */
+
+int
+dhd_prot_flow_ring_batch_suspend_request(dhd_pub_t *dhd, uint16 *ringid, uint16 count)
+{
+	tx_idle_flowring_suspend_request_t *flow_suspend_rqst;
+	dhd_prot_t *prot = dhd->prot;
+	unsigned long flags;
+	uint16 index;
+	uint16 alloced = 0;
+	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ring->ring_lock, flags);
+
+	/* Request for ring buffer space */
+	flow_suspend_rqst = (tx_idle_flowring_suspend_request_t *)
+		dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
+
+	if (flow_suspend_rqst == NULL) {
+		DHD_RING_UNLOCK(ring->ring_lock, flags);
+		DHD_ERROR(("%s: Flow suspend Req - failure ring space\n", __FUNCTION__));
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_NOMEM;
+	}
+
+	/* Common msg buf hdr */
+	flow_suspend_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_SUSPEND;
+	/* flow_suspend_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex; */
+	flow_suspend_rqst->msg.request_id = htol32(0); /* TBD */
+
+	flow_suspend_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
+	ring->seqnum++;
+
+	/* Update flow id  info */
+	for (index = 0; index < count; index++)
+	{
+		flow_suspend_rqst->ring_id[index] = ringid[index];
+	}
+	flow_suspend_rqst->num = count;
+
+	DHD_ERROR(("%s sending batch suspend!! count is %d\n", __FUNCTION__, count));
+
+	/* update ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ring, flow_suspend_rqst, 1);
+
+	DHD_RING_UNLOCK(ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+
+	return BCME_OK;
+}
+#endif /* IDLE_TX_FLOW_MGMT */
+
+static void dump_psmwd_v1(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_psmwd_v1_t* psmwd = (const hnd_ext_trap_psmwd_v1_t *)tlv->data;
+	uint32 i;
+
+	for (i = 0; i < PSMDBG_REG_READ_CNT_FOR_PSMWDTRAP_V1; i++) {
+		bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, psmwd->i32_psmdebug[i]);
+	}
+	bcm_bprintf(b, " gated clock en: 0x%x\n", psmwd->i16_0x1a8);
+	bcm_bprintf(b, " Rcv Fifo Ctrl: 0x%x\n", psmwd->i16_0x406);
+	bcm_bprintf(b, " Rx ctrl 1: 0x%x\n", psmwd->i16_0x408);
+	bcm_bprintf(b, " Rxe Status 1: 0x%x\n", psmwd->i16_0x41a);
+	bcm_bprintf(b, " Rxe Status 2: 0x%x\n", psmwd->i16_0x41c);
+	bcm_bprintf(b, " rcv wrd count 0: 0x%x\n", psmwd->i16_0x424);
+	bcm_bprintf(b, " rcv wrd count 1: 0x%x\n", psmwd->i16_0x426);
+	bcm_bprintf(b, " RCV_LFIFO_STS: 0x%x\n", psmwd->i16_0x456);
+	bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", psmwd->i16_0x480);
+	bcm_bprintf(b, " TXE CTRL: 0x%x\n", psmwd->i16_0x500);
+	bcm_bprintf(b, " TXE Status: 0x%x\n", psmwd->i16_0x50e);
+	bcm_bprintf(b, " TXE_xmtdmabusy: 0x%x\n", psmwd->i16_0x55e);
+	bcm_bprintf(b, " TXE_XMTfifosuspflush: 0x%x\n", psmwd->i16_0x566);
+	bcm_bprintf(b, " IFS Stat: 0x%x\n", psmwd->i16_0x690);
+	bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", psmwd->i16_0x692);
+	bcm_bprintf(b, " IFS_TX_DUR: 0x%x\n", psmwd->i16_0x694);
+	bcm_bprintf(b, " SLow_CTL: 0x%x\n", psmwd->i16_0x6a0);
+	bcm_bprintf(b, " PSM BRC: 0x%x\n", psmwd->i16_0x490);
+	bcm_bprintf(b, " TXE_AQM fifo Ready: 0x%x\n", psmwd->i16_0x838);
+	bcm_bprintf(b, " Dagg ctrl: 0x%x\n", psmwd->i16_0x8c0);
+	bcm_bprintf(b, " shm_prewds_cnt: 0x%x\n", psmwd->shm_prewds_cnt);
+	bcm_bprintf(b, " shm_txtplufl_cnt: 0x%x\n", psmwd->shm_txtplufl_cnt);
+	bcm_bprintf(b, " shm_txphyerr_cnt: 0x%x\n", psmwd->shm_txphyerr_cnt);
+
+}
+
+static void dump_psmwd_v2(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_psmwd_v2_t* psmwd = (const hnd_ext_trap_psmwd_v2_t *)tlv->data;
+	uint32 i;
+
+	for (i = 0; i < PSMDBG_REG_READ_CNT_FOR_PSMWDTRAP_V2; i++) {
+		bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, psmwd->i32_psmdebug[i]);
+	}
+
+	bcm_bprintf(b, " psm_brwk0: 0x%x\n", psmwd->i16_0x4b8);
+	bcm_bprintf(b, " psm_brwk1: 0x%x\n", psmwd->i16_0x4ba);
+	bcm_bprintf(b, " psm_brwk2: 0x%x\n", psmwd->i16_0x4bc);
+	bcm_bprintf(b, " psm_brwk3: 0x%x\n", psmwd->i16_0x4be);
+	bcm_bprintf(b, " PSM BRC_1: 0x%x\n", psmwd->i16_0x4da);
+	bcm_bprintf(b, " gated clock en: 0x%x\n", psmwd->i16_0x1a8);
+	bcm_bprintf(b, " Rcv Fifo Ctrl: 0x%x\n", psmwd->i16_0x406);
+	bcm_bprintf(b, " Rx ctrl 1: 0x%x\n", psmwd->i16_0x408);
+	bcm_bprintf(b, " Rxe Status 1: 0x%x\n", psmwd->i16_0x41a);
+	bcm_bprintf(b, " Rxe Status 2: 0x%x\n", psmwd->i16_0x41c);
+	bcm_bprintf(b, " rcv wrd count 0: 0x%x\n", psmwd->i16_0x424);
+	bcm_bprintf(b, " rcv wrd count 1: 0x%x\n", psmwd->i16_0x426);
+	bcm_bprintf(b, " RCV_LFIFO_STS: 0x%x\n", psmwd->i16_0x456);
+	bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", psmwd->i16_0x480);
+	bcm_bprintf(b, " TXE CTRL: 0x%x\n", psmwd->i16_0x500);
+	bcm_bprintf(b, " TXE Status: 0x%x\n", psmwd->i16_0x50e);
+	bcm_bprintf(b, " TXE_xmtdmabusy: 0x%x\n", psmwd->i16_0x55e);
+	bcm_bprintf(b, " TXE_XMTfifosuspflush: 0x%x\n", psmwd->i16_0x566);
+	bcm_bprintf(b, " IFS Stat: 0x%x\n", psmwd->i16_0x690);
+	bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", psmwd->i16_0x692);
+	bcm_bprintf(b, " IFS_TX_DUR: 0x%x\n", psmwd->i16_0x694);
+	bcm_bprintf(b, " SLow_CTL: 0x%x\n", psmwd->i16_0x6a0);
+	bcm_bprintf(b, " PSM BRC: 0x%x\n", psmwd->i16_0x490);
+	bcm_bprintf(b, " TXE_AQM fifo Ready: 0x%x\n", psmwd->i16_0x838);
+	bcm_bprintf(b, " Dagg ctrl: 0x%x\n", psmwd->i16_0x8c0);
+	bcm_bprintf(b, " shm_prewds_cnt: 0x%x\n", psmwd->shm_prewds_cnt);
+	bcm_bprintf(b, " shm_txtplufl_cnt: 0x%x\n", psmwd->shm_txtplufl_cnt);
+	bcm_bprintf(b, " shm_txphyerr_cnt: 0x%x\n", psmwd->shm_txphyerr_cnt);
+}
+
+static void dump_psmwd_v3(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_psmwd_v3_t* psmwd = (const hnd_ext_trap_psmwd_v3_t *)tlv->data;
+	uint32 i;
+
+	for (i = 0; i < PSMDBG_REG_READ_CNT_FOR_PSMWDTRAP_V1; i++) {
+		bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, psmwd->i32_psmdebug[i]);
+	}
+
+	bcm_bprintf(b, " i32_gated_clock_en: 0x%x\n", psmwd->i32_gated_clock_en);
+	bcm_bprintf(b, " Rcv Fifo Ctrl: 0x%x\n", psmwd->rcv_fifo_ctrl);
+	bcm_bprintf(b, " Rx ctrl 1: 0x%x\n", psmwd->rx_ctrl1);
+	bcm_bprintf(b, " Rxe Status 1: 0x%x\n", psmwd->rxe_status1);
+	bcm_bprintf(b, " Rxe Status 2: 0x%x\n", psmwd->rxe_status2);
+
+	bcm_bprintf(b, " rcv wrd count 0: 0x%x\n", psmwd->rcv_wrd_count0);
+	bcm_bprintf(b, " rcv wrd count 1: 0x%x\n", psmwd->rcv_wrd_count1);
+	bcm_bprintf(b, " RCV_LFIFO_STS: 0x%x\n", psmwd->rcv_lfifo_sts);
+	bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", psmwd->psm_slp_tmr);
+	bcm_bprintf(b, " PSM BRC: 0x%x\n", psmwd->psm_brc);
+
+	bcm_bprintf(b, " TXE CTRL: 0x%x\n", psmwd->txe_ctrl);
+	bcm_bprintf(b, " TXE Status: 0x%x\n", psmwd->txe_status);
+	bcm_bprintf(b, " TXE_xmtdmabusy: 0x%x\n", psmwd->txe_xmtdmabusy);
+	bcm_bprintf(b, " TXE_XMTfifosuspflush: 0x%x\n", psmwd->txe_xmt_fifo_susp_flush);
+	bcm_bprintf(b, " IFS Stat: 0x%x\n", psmwd->ifs_stat);
+
+	bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", psmwd->ifs_medbusy_ctr);
+	bcm_bprintf(b, " IFS_TX_DUR: 0x%x\n", psmwd->ifs_tx_dur);
+	bcm_bprintf(b, " SLow_CTL: 0x%x\n", psmwd->slow_ctl);
+	bcm_bprintf(b, " TXE_AQM fifo Ready: 0x%x\n", psmwd->txe_aqm_fifo_ready);
+	bcm_bprintf(b, " Dagg ctrl: 0x%x\n", psmwd->dagg_ctrl);
+
+	bcm_bprintf(b, " shm_prewds_cnt: 0x%x\n", psmwd->shm_prewds_cnt);
+	bcm_bprintf(b, " shm_txtplufl_cnt: 0x%x\n", psmwd->shm_txtplufl_cnt);
+	bcm_bprintf(b, " shm_txphyerr_cnt: 0x%x\n", psmwd->shm_txphyerr_cnt);
+}
+
+static void dump_psmwd(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_psmwd_v1_t * psmwd = (const hnd_ext_trap_psmwd_v1_t*)tlv->data;
+
+	if (psmwd->version == 0 || psmwd->version > HND_EXT_TRAP_PSMWD_INFO_VER_3) {
+		bcm_bprintf(b, " Bad version: 0x%x\n", psmwd->version);
+		return;
+	}
+	bcm_bprintf(b, " version: 0x%x\n", psmwd->version);
+	bcm_bprintf(b, " maccontrol: 0x%x\n", psmwd->i32_maccontrol);
+	bcm_bprintf(b, " maccommand: 0x%x\n", psmwd->i32_maccommand);
+	bcm_bprintf(b, " macintstatus: 0x%x\n", psmwd->i32_macintstatus);
+	bcm_bprintf(b, " phydebug: 0x%x\n", psmwd->i32_phydebug);
+	bcm_bprintf(b, " clk_ctl_st: 0x%x\n", psmwd->i32_clk_ctl_st);
+	if (psmwd->version == HND_EXT_TRAP_PSMWD_INFO_VER_1) {
+		dump_psmwd_v1(tlv, b);
+	} else if (psmwd->version == HND_EXT_TRAP_PSMWD_INFO_VER_2) {
+		dump_psmwd_v2(tlv, b);
+	} else if (psmwd->version == HND_EXT_TRAP_PSMWD_INFO_VER_3) {
+		dump_psmwd_v3(tlv, b);
+	}
+	return;
+}
+
+static void dump_macwake_v1(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_macenab_v1_t* macwake = (const hnd_ext_trap_macenab_v1_t *)tlv->data;
+
+	bcm_bprintf(b, " gated clock en: 0x%x\n", macwake->i16_0x1a8);
+	bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", macwake->i16_0x480);
+	bcm_bprintf(b, " PSM BRC: 0x%x\n", macwake->i16_0x490);
+	bcm_bprintf(b, " TSF CTL: 0x%x\n", macwake->i16_0x600);
+	bcm_bprintf(b, " IFS Stat: 0x%x\n", macwake->i16_0x690);
+	bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", macwake->i16_0x692);
+	bcm_bprintf(b, " Slow_CTL: 0x%x\n", macwake->i16_0x6a0);
+	bcm_bprintf(b, " Slow_FRAC: 0x%x\n", macwake->i16_0x6a6);
+	bcm_bprintf(b, " fast power up delay: 0x%x\n", macwake->i16_0x6a8);
+	bcm_bprintf(b, " Slow_PER: 0x%x\n", macwake->i16_0x6aa);
+	bcm_bprintf(b, " shm_ucode_dbgst: 0x%x\n", macwake->shm_ucode_dbgst);
+}
+
+static void dump_macwake_v2(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_macenab_v2_t* macwake = (const hnd_ext_trap_macenab_v2_t *)tlv->data;
+
+	bcm_bprintf(b, " gated clock en: 0x%x\n", macwake->i32_gated_clock_en);
+	bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", macwake->psm_slp_tmr);
+	bcm_bprintf(b, " PSM BRC: 0x%x\n", macwake->psm_brc);
+	bcm_bprintf(b, " TSF CTL: 0x%x\n", macwake->tsf_ctl);
+	bcm_bprintf(b, " IFS Stat: 0x%x\n", macwake->ifs_stat);
+	bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", macwake->ifs_medbusy_ctr);
+	bcm_bprintf(b, " Slow_CTL: 0x%x\n", macwake->slow_ctl);
+	bcm_bprintf(b, " Slow_FRAC: 0x%x\n", macwake->slow_frac);
+	bcm_bprintf(b, " fast power up delay: 0x%x\n", macwake->fast_powerup_delay);
+	bcm_bprintf(b, " Slow_PER: 0x%x\n", macwake->slow_per);
+	bcm_bprintf(b, " shm_ucode_dbgst: 0x%x\n", macwake->shm_ucode_dbgst);
+}
+
+static void dump_macwake(const bcm_tlv_t *tlv, struct bcmstrbuf *b)
+{
+	const hnd_ext_trap_macenab_v1_t* macwake = (const hnd_ext_trap_macenab_v1_t *)tlv->data;
+	uint32 i;
+
+	if (macwake->version == 0 || macwake->version > HND_EXT_TRAP_MACENAB_INFO_VER_2) {
+		bcm_bprintf(b, " Bad version: 0x%x\n", macwake->version);
+		return;
+	}
+
+	bcm_bprintf(b, " version: 0x%x\n", macwake->version);
+	bcm_bprintf(b, " trap_reason: 0x%x\n", macwake->trap_reason);
+	bcm_bprintf(b, " maccontrol: 0x%x\n", macwake->i32_maccontrol);
+	bcm_bprintf(b, " maccommand: 0x%x\n", macwake->i32_maccommand);
+	bcm_bprintf(b, " macintstatus: 0x%x\n", macwake->i32_macintstatus);
+	for (i = 0; i < 8; i++)
+		bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, macwake->i32_psmdebug[i]);
+	bcm_bprintf(b, " clk_ctl_st: 0x%x\n", macwake->i32_clk_ctl_st);
+	bcm_bprintf(b, " powerctl: 0x%x\n", macwake->i32_powerctl);
+
+	if (macwake->version == HND_EXT_TRAP_MACENAB_INFO_VER_1) {
+		dump_macwake_v1(tlv, b);
+	} else if (macwake->version == HND_EXT_TRAP_MACENAB_INFO_VER_2) {
+		dump_macwake_v2(tlv, b);
+	}
+	return;
+}
+
+static const char* etd_trap_name(hnd_ext_tag_trap_t tag)
+{
+	switch (tag) {
+	case TAG_TRAP_SIGNATURE: return "TAG_TRAP_SIGNATURE";
+	case TAG_TRAP_STACK: return "TAG_TRAP_STACK";
+	case TAG_TRAP_MEMORY: return "TAG_TRAP_MEMORY";
+	case TAG_TRAP_DEEPSLEEP: return "TAG_TRAP_DEEPSLEEP";
+	case TAG_TRAP_PSM_WD: return "TAG_TRAP_PSM_WD";
+	case TAG_TRAP_PHY: return "TAG_TRAP_PHY";
+	case TAG_TRAP_BUS: return "TAG_TRAP_BUS";
+	case TAG_TRAP_MAC_SUSP: return "TAG_TRAP_MAC_SUSP";
+	case TAG_TRAP_BACKPLANE: return "TAG_TRAP_BACKPLANE";
+	case TAG_TRAP_PCIE_Q: return "TAG_TRAP_PCIE_Q";
+	case TAG_TRAP_WLC_STATE: return "TAG_TRAP_WLC_STATE";
+	case TAG_TRAP_MAC_WAKE: return "TAG_TRAP_MAC_WAKE";
+	case TAG_TRAP_HMAP: return "TAG_TRAP_HMAP";
+	case TAG_TRAP_PHYTXERR_THRESH: return "TAG_TRAP_PHYTXERR_THRESH";
+	case TAG_TRAP_HC_DATA: return "TAG_TRAP_HC_DATA";
+	case TAG_TRAP_LOG_DATA: return "TAG_TRAP_LOG_DATA";
+	case TAG_TRAP_CODE: return "TAG_TRAP_CODE";
+	case TAG_TRAP_MEM_BIT_FLIP: return "TAG_TRAP_MEM_BIT_FLIP";
+	case TAG_TRAP_LAST:
+	default:
+		return "Unknown";
+	}
+	return "Unknown";
+}
+
+int dhd_prot_dump_extended_trap(dhd_pub_t *dhdp, struct bcmstrbuf *b, bool raw)
+{
+	uint32 i;
+	uint32 *ext_data;
+	hnd_ext_trap_hdr_t *hdr;
+	const bcm_tlv_t *tlv;
+	const trap_t *tr;
+	const uint32 *stack;
+	const hnd_ext_trap_bp_err_t *bpe;
+	uint32 raw_len;
+
+	ext_data = dhdp->extended_trap_data;
+
+	/* return if there is no extended trap data */
+	if (!ext_data || !(dhdp->dongle_trap_data & D2H_DEV_EXT_TRAP_DATA)) {
+		bcm_bprintf(b, "%d (0x%x)", dhdp->dongle_trap_data, dhdp->dongle_trap_data);
+		return BCME_OK;
+	}
+
+	bcm_bprintf(b, "Extended trap data\n");
+
+	/* First word is original trap_data */
+	bcm_bprintf(b, "trap_data = 0x%08x\n", *ext_data);
+	ext_data++;
+
+	/* Followed by the extended trap data header */
+	hdr = (hnd_ext_trap_hdr_t *)ext_data;
+	bcm_bprintf(b, "version: %d, len: %d\n", hdr->version, hdr->len);
+
+	/* Dump a list of all tags found  before parsing data */
+	bcm_bprintf(b, "\nTags Found:\n");
+	for (i = 0; i < TAG_TRAP_LAST; i++) {
+		tlv = bcm_parse_tlvs(hdr->data, hdr->len, i);
+		if (tlv)
+			bcm_bprintf(b, "Tag: %d (%s), Length: %d\n", i, etd_trap_name(i), tlv->len);
+	}
+
+	/* XXX debug dump */
+	if (raw) {
+		raw_len = sizeof(hnd_ext_trap_hdr_t) + (hdr->len / 4) + (hdr->len % 4 ? 1 : 0);
+		for (i = 0; i < raw_len; i++)
+		{
+			bcm_bprintf(b, "0x%08x ", ext_data[i]);
+			if (i % 4 == 3)
+				bcm_bprintf(b, "\n");
+		}
+		return BCME_OK;
+	}
+
+	/* Extract the various supported TLVs from the extended trap data */
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_CODE);
+	if (tlv) {
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_CODE), tlv->len);
+		bcm_bprintf(b, "ETD TYPE: %d\n", tlv->data[0]);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_SIGNATURE);
+	if (tlv) {
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_SIGNATURE), tlv->len);
+		tr = (const trap_t *)tlv->data;
+
+		bcm_bprintf(b, "TRAP %x: pc %x, lr %x, sp %x, cpsr %x, spsr %x\n",
+		       tr->type, tr->pc, tr->r14, tr->r13, tr->cpsr, tr->spsr);
+		bcm_bprintf(b, "  r0 %x, r1 %x, r2 %x, r3 %x, r4 %x, r5 %x, r6 %x\n",
+		       tr->r0, tr->r1, tr->r2, tr->r3, tr->r4, tr->r5, tr->r6);
+		bcm_bprintf(b, "  r7 %x, r8 %x, r9 %x, r10 %x, r11 %x, r12 %x\n",
+		       tr->r7, tr->r8, tr->r9, tr->r10, tr->r11, tr->r12);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_STACK);
+	if (tlv) {
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_STACK), tlv->len);
+		stack = (const uint32 *)tlv->data;
+		for (i = 0; i < (uint32)(tlv->len / 4); i++)
+		{
+			bcm_bprintf(b, "  0x%08x\n", *stack);
+			stack++;
+		}
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_BACKPLANE);
+	if (tlv) {
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_BACKPLANE), tlv->len);
+		bpe = (const hnd_ext_trap_bp_err_t *)tlv->data;
+		bcm_bprintf(b, " error: %x\n", bpe->error);
+		bcm_bprintf(b, " coreid: %x\n", bpe->coreid);
+		bcm_bprintf(b, " baseaddr: %x\n", bpe->baseaddr);
+		bcm_bprintf(b, " ioctrl: %x\n", bpe->ioctrl);
+		bcm_bprintf(b, " iostatus: %x\n", bpe->iostatus);
+		bcm_bprintf(b, " resetctrl: %x\n", bpe->resetctrl);
+		bcm_bprintf(b, " resetstatus: %x\n", bpe->resetstatus);
+		bcm_bprintf(b, " errlogctrl: %x\n", bpe->errlogctrl);
+		bcm_bprintf(b, " errlogdone: %x\n", bpe->errlogdone);
+		bcm_bprintf(b, " errlogstatus: %x\n", bpe->errlogstatus);
+		bcm_bprintf(b, " errlogaddrlo: %x\n", bpe->errlogaddrlo);
+		bcm_bprintf(b, " errlogaddrhi: %x\n", bpe->errlogaddrhi);
+		bcm_bprintf(b, " errlogid: %x\n", bpe->errlogid);
+		bcm_bprintf(b, " errloguser: %x\n", bpe->errloguser);
+		bcm_bprintf(b, " errlogflags: %x\n", bpe->errlogflags);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MEMORY);
+	if (tlv) {
+		const hnd_ext_trap_heap_err_t* hme;
+
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MEMORY), tlv->len);
+		hme = (const hnd_ext_trap_heap_err_t *)tlv->data;
+		bcm_bprintf(b, " arena total: %d\n", hme->arena_total);
+		bcm_bprintf(b, " heap free: %d\n", hme->heap_free);
+		bcm_bprintf(b, " heap in use: %d\n", hme->heap_inuse);
+		bcm_bprintf(b, " mf count: %d\n", hme->mf_count);
+		bcm_bprintf(b, " stack LWM: %x\n", hme->stack_lwm);
+
+		bcm_bprintf(b, " Histogram:\n");
+		for (i = 0; i < (HEAP_HISTOGRAM_DUMP_LEN * 2); i += 2) {
+			if (hme->heap_histogm[i] == 0xfffe)
+				bcm_bprintf(b, " Others\t%d\t?\n", hme->heap_histogm[i + 1]);
+			else if (hme->heap_histogm[i] == 0xffff)
+				bcm_bprintf(b, " >= 256K\t%d\t?\n", hme->heap_histogm[i + 1]);
+			else
+				bcm_bprintf(b, " %d\t%d\t%d\n", hme->heap_histogm[i] << 2,
+					hme->heap_histogm[i + 1], (hme->heap_histogm[i] << 2)
+					* hme->heap_histogm[i + 1]);
+		}
+
+		bcm_bprintf(b, " Max free block: %d\n", hme->max_sz_free_blk[0] << 2);
+		for (i = 1; i < HEAP_MAX_SZ_BLKS_LEN; i++) {
+			bcm_bprintf(b, " Next lgst free block: %d\n", hme->max_sz_free_blk[i] << 2);
+		}
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PCIE_Q);
+	if (tlv) {
+		const hnd_ext_trap_pcie_mem_err_t* pqme;
+
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PCIE_Q), tlv->len);
+		pqme = (const hnd_ext_trap_pcie_mem_err_t *)tlv->data;
+		bcm_bprintf(b, " d2h queue len: %x\n", pqme->d2h_queue_len);
+		bcm_bprintf(b, " d2h req queue len: %x\n", pqme->d2h_req_queue_len);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_WLC_STATE);
+	if (tlv) {
+		const hnd_ext_trap_wlc_mem_err_t* wsme;
+
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_WLC_STATE), tlv->len);
+		wsme = (const hnd_ext_trap_wlc_mem_err_t *)tlv->data;
+		bcm_bprintf(b, " instance: %d\n", wsme->instance);
+		bcm_bprintf(b, " associated: %d\n", wsme->associated);
+		bcm_bprintf(b, " peer count: %d\n", wsme->peer_cnt);
+		bcm_bprintf(b, " client count: %d\n", wsme->soft_ap_client_cnt);
+		bcm_bprintf(b, " TX_AC_BK_FIFO: %d\n", wsme->txqueue_len[0]);
+		bcm_bprintf(b, " TX_AC_BE_FIFO: %d\n", wsme->txqueue_len[1]);
+		bcm_bprintf(b, " TX_AC_VI_FIFO: %d\n", wsme->txqueue_len[2]);
+		bcm_bprintf(b, " TX_AC_VO_FIFO: %d\n", wsme->txqueue_len[3]);
+
+		if (tlv->len >= (sizeof(*wsme) * 2)) {
+			wsme++;
+			bcm_bprintf(b, "\n instance: %d\n", wsme->instance);
+			bcm_bprintf(b, " associated: %d\n", wsme->associated);
+			bcm_bprintf(b, " peer count: %d\n", wsme->peer_cnt);
+			bcm_bprintf(b, " client count: %d\n", wsme->soft_ap_client_cnt);
+			bcm_bprintf(b, " TX_AC_BK_FIFO: %d\n", wsme->txqueue_len[0]);
+			bcm_bprintf(b, " TX_AC_BE_FIFO: %d\n", wsme->txqueue_len[1]);
+			bcm_bprintf(b, " TX_AC_VI_FIFO: %d\n", wsme->txqueue_len[2]);
+			bcm_bprintf(b, " TX_AC_VO_FIFO: %d\n", wsme->txqueue_len[3]);
+		}
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PHY);
+	if (tlv) {
+		const hnd_ext_trap_phydbg_t* phydbg;
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PHY), tlv->len);
+		phydbg = (const hnd_ext_trap_phydbg_t *)tlv->data;
+		bcm_bprintf(b, " err: 0x%x\n", phydbg->err);
+		bcm_bprintf(b, " RxFeStatus: 0x%x\n", phydbg->RxFeStatus);
+		bcm_bprintf(b, " TxFIFOStatus0: 0x%x\n", phydbg->TxFIFOStatus0);
+		bcm_bprintf(b, " TxFIFOStatus1: 0x%x\n", phydbg->TxFIFOStatus1);
+		bcm_bprintf(b, " RfseqMode: 0x%x\n", phydbg->RfseqMode);
+		bcm_bprintf(b, " RfseqStatus0: 0x%x\n", phydbg->RfseqStatus0);
+		bcm_bprintf(b, " RfseqStatus1: 0x%x\n", phydbg->RfseqStatus1);
+		bcm_bprintf(b, " RfseqStatus_Ocl: 0x%x\n", phydbg->RfseqStatus_Ocl);
+		bcm_bprintf(b, " RfseqStatus_Ocl1: 0x%x\n", phydbg->RfseqStatus_Ocl1);
+		bcm_bprintf(b, " OCLControl1: 0x%x\n", phydbg->OCLControl1);
+		bcm_bprintf(b, " TxError: 0x%x\n", phydbg->TxError);
+		bcm_bprintf(b, " bphyTxError: 0x%x\n", phydbg->bphyTxError);
+		bcm_bprintf(b, " TxCCKError: 0x%x\n", phydbg->TxCCKError);
+		bcm_bprintf(b, " TxCtrlWrd0: 0x%x\n", phydbg->TxCtrlWrd0);
+		bcm_bprintf(b, " TxCtrlWrd1: 0x%x\n", phydbg->TxCtrlWrd1);
+		bcm_bprintf(b, " TxCtrlWrd2: 0x%x\n", phydbg->TxCtrlWrd2);
+		bcm_bprintf(b, " TxLsig0: 0x%x\n", phydbg->TxLsig0);
+		bcm_bprintf(b, " TxLsig1: 0x%x\n", phydbg->TxLsig1);
+		bcm_bprintf(b, " TxVhtSigA10: 0x%x\n", phydbg->TxVhtSigA10);
+		bcm_bprintf(b, " TxVhtSigA11: 0x%x\n", phydbg->TxVhtSigA11);
+		bcm_bprintf(b, " TxVhtSigA20: 0x%x\n", phydbg->TxVhtSigA20);
+		bcm_bprintf(b, " TxVhtSigA21: 0x%x\n", phydbg->TxVhtSigA21);
+		bcm_bprintf(b, " txPktLength: 0x%x\n", phydbg->txPktLength);
+		bcm_bprintf(b, " txPsdulengthCtr: 0x%x\n", phydbg->txPsdulengthCtr);
+		bcm_bprintf(b, " gpioClkControl: 0x%x\n", phydbg->gpioClkControl);
+		bcm_bprintf(b, " gpioSel: 0x%x\n", phydbg->gpioSel);
+		bcm_bprintf(b, " pktprocdebug: 0x%x\n", phydbg->pktprocdebug);
+		for (i = 0; i < 3; i++)
+			bcm_bprintf(b, " gpioOut[%d]: 0x%x\n", i, phydbg->gpioOut[i]);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PSM_WD);
+	if (tlv) {
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PSM_WD), tlv->len);
+		dump_psmwd(tlv, b);
+	}
+/* PHY TxErr MacDump */
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PHYTXERR_THRESH);
+	if (tlv) {
+		const hnd_ext_trap_macphytxerr_t* phytxerr = NULL;
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PHYTXERR_THRESH), tlv->len);
+		phytxerr = (const hnd_ext_trap_macphytxerr_t *)tlv->data;
+		bcm_bprintf(b, " version: 0x%x\n", phytxerr->version);
+		bcm_bprintf(b, " trap_reason: %d\n", phytxerr->trap_reason);
+		bcm_bprintf(b, " Tsf_rx_ts_0x63E: 0x%x\n", phytxerr->i16_0x63E);
+		bcm_bprintf(b, " Tsf_tx_ts_0x640: 0x%x\n", phytxerr->i16_0x640);
+		bcm_bprintf(b, " tsf_tmr_rx_end_ts_0x642: 0x%x\n", phytxerr->i16_0x642);
+		bcm_bprintf(b, " TDC_FrmLen0_0x846: 0x%x\n", phytxerr->i16_0x846);
+		bcm_bprintf(b, " TDC_FrmLen1_0x848: 0x%x\n", phytxerr->i16_0x848);
+		bcm_bprintf(b, " TDC_Txtime_0x84a: 0x%x\n", phytxerr->i16_0x84a);
+		bcm_bprintf(b, " TXE_BytCntInTxFrmLo_0xa5a: 0x%x\n", phytxerr->i16_0xa5a);
+		bcm_bprintf(b, " TXE_BytCntInTxFrmHi_0xa5c: 0x%x\n", phytxerr->i16_0xa5c);
+		bcm_bprintf(b, " TDC_VhtPsduLen0_0x856: 0x%x\n", phytxerr->i16_0x856);
+		bcm_bprintf(b, " TDC_VhtPsduLen1_0x858: 0x%x\n", phytxerr->i16_0x858);
+		bcm_bprintf(b, " PSM_BRC: 0x%x\n", phytxerr->i16_0x490);
+		bcm_bprintf(b, " PSM_BRC_1: 0x%x\n", phytxerr->i16_0x4d8);
+		bcm_bprintf(b, " shm_txerr_reason: 0x%x\n", phytxerr->shm_txerr_reason);
+		bcm_bprintf(b, " shm_pctl0: 0x%x\n", phytxerr->shm_pctl0);
+		bcm_bprintf(b, " shm_pctl1: 0x%x\n", phytxerr->shm_pctl1);
+		bcm_bprintf(b, " shm_pctl2: 0x%x\n", phytxerr->shm_pctl2);
+		bcm_bprintf(b, " shm_lsig0: 0x%x\n", phytxerr->shm_lsig0);
+		bcm_bprintf(b, " shm_lsig1: 0x%x\n", phytxerr->shm_lsig1);
+		bcm_bprintf(b, " shm_plcp0: 0x%x\n", phytxerr->shm_plcp0);
+		bcm_bprintf(b, " shm_plcp1: 0x%x\n", phytxerr->shm_plcp1);
+		bcm_bprintf(b, " shm_plcp2: 0x%x\n", phytxerr->shm_plcp2);
+		bcm_bprintf(b, " shm_vht_sigb0: 0x%x\n", phytxerr->shm_vht_sigb0);
+		bcm_bprintf(b, " shm_vht_sigb1: 0x%x\n", phytxerr->shm_vht_sigb1);
+		bcm_bprintf(b, " shm_tx_tst: 0x%x\n", phytxerr->shm_tx_tst);
+		bcm_bprintf(b, " shm_txerr_tm: 0x%x\n", phytxerr->shm_txerr_tm);
+		bcm_bprintf(b, " shm_curchannel: 0x%x\n", phytxerr->shm_curchannel);
+		bcm_bprintf(b, " shm_blk_crx_rxtsf_pos: 0x%x\n", phytxerr->shm_crx_rxtsf_pos);
+		bcm_bprintf(b, " shm_lasttx_tsf: 0x%x\n", phytxerr->shm_lasttx_tsf);
+		bcm_bprintf(b, " shm_s_rxtsftmrval: 0x%x\n", phytxerr->shm_s_rxtsftmrval);
+		bcm_bprintf(b, " Phy_0x29: 0x%x\n", phytxerr->i16_0x29);
+		bcm_bprintf(b, " Phy_0x2a: 0x%x\n", phytxerr->i16_0x2a);
+	}
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MAC_SUSP);
+	if (tlv) {
+		const hnd_ext_trap_macsusp_t* macsusp;
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MAC_SUSP), tlv->len);
+		macsusp = (const hnd_ext_trap_macsusp_t *)tlv->data;
+		bcm_bprintf(b, " version: %d\n", macsusp->version);
+		bcm_bprintf(b, " trap_reason: %d\n", macsusp->trap_reason);
+		bcm_bprintf(b, " maccontrol: 0x%x\n", macsusp->i32_maccontrol);
+		bcm_bprintf(b, " maccommand: 0x%x\n", macsusp->i32_maccommand);
+		bcm_bprintf(b, " macintstatus: 0x%x\n", macsusp->i32_macintstatus);
+		for (i = 0; i < 4; i++)
+			bcm_bprintf(b, " phydebug[%d]: 0x%x\n", i, macsusp->i32_phydebug[i]);
+		for (i = 0; i < 8; i++)
+			bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, macsusp->i32_psmdebug[i]);
+		bcm_bprintf(b, " Rxe Status_1: 0x%x\n", macsusp->i16_0x41a);
+		bcm_bprintf(b, " Rxe Status_2: 0x%x\n", macsusp->i16_0x41c);
+		bcm_bprintf(b, " PSM BRC: 0x%x\n", macsusp->i16_0x490);
+		bcm_bprintf(b, " TXE Status: 0x%x\n", macsusp->i16_0x50e);
+		bcm_bprintf(b, " TXE xmtdmabusy: 0x%x\n", macsusp->i16_0x55e);
+		bcm_bprintf(b, " TXE XMTfifosuspflush: 0x%x\n", macsusp->i16_0x566);
+		bcm_bprintf(b, " IFS Stat: 0x%x\n", macsusp->i16_0x690);
+		bcm_bprintf(b, " IFS MEDBUSY CTR: 0x%x\n", macsusp->i16_0x692);
+		bcm_bprintf(b, " IFS TX DUR: 0x%x\n", macsusp->i16_0x694);
+		bcm_bprintf(b, " WEP CTL: 0x%x\n", macsusp->i16_0x7c0);
+		bcm_bprintf(b, " TXE AQM fifo Ready: 0x%x\n", macsusp->i16_0x838);
+		bcm_bprintf(b, " MHP status: 0x%x\n", macsusp->i16_0x880);
+		bcm_bprintf(b, " shm_prewds_cnt: 0x%x\n", macsusp->shm_prewds_cnt);
+		bcm_bprintf(b, " shm_ucode_dbgst: 0x%x\n", macsusp->shm_ucode_dbgst);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MAC_WAKE);
+	if (tlv) {
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MAC_WAKE), tlv->len);
+		dump_macwake(tlv, b);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_BUS);
+	if (tlv) {
+		const bcm_dngl_pcie_hc_t* hc;
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_BUS), tlv->len);
+		hc = (const bcm_dngl_pcie_hc_t *)tlv->data;
+		bcm_bprintf(b, " version: 0x%x\n", hc->version);
+		bcm_bprintf(b, " reserved: 0x%x\n", hc->reserved);
+		bcm_bprintf(b, " pcie_err_ind_type: 0x%x\n", hc->pcie_err_ind_type);
+		bcm_bprintf(b, " pcie_flag: 0x%x\n", hc->pcie_flag);
+		bcm_bprintf(b, " pcie_control_reg: 0x%x\n", hc->pcie_control_reg);
+		for (i = 0; i < HC_PCIEDEV_CONFIG_REGLIST_MAX; i++)
+			bcm_bprintf(b, " pcie_config_regs[%d]: 0x%x\n", i, hc->pcie_config_regs[i]);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_HMAP);
+	if (tlv) {
+		const pcie_hmapviolation_t* hmap;
+		hmap = (const pcie_hmapviolation_t *)tlv->data;
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_HMAP), tlv->len);
+		bcm_bprintf(b, " HMAP Vio Addr Low: 0x%x\n", hmap->hmap_violationaddr_lo);
+		bcm_bprintf(b, " HMAP Vio Addr Hi: 0x%x\n", hmap->hmap_violationaddr_hi);
+		bcm_bprintf(b, " HMAP Vio Info: 0x%x\n", hmap->hmap_violation_info);
+	}
+
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MEM_BIT_FLIP);
+	if (tlv) {
+		const hnd_ext_trap_fb_mem_err_t* fbit;
+		bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MEM_BIT_FLIP), tlv->len);
+		fbit = (const hnd_ext_trap_fb_mem_err_t *)tlv->data;
+		bcm_bprintf(b, " version: %d\n", fbit->version);
+		bcm_bprintf(b, " flip_bit_err_time: %d\n", fbit->flip_bit_err_time);
+	}
+
+	return BCME_OK;
+}
+
+#ifdef BCMPCIE
+int
+dhd_prot_send_host_timestamp(dhd_pub_t *dhdp, uchar *tlvs, uint16 tlv_len,
+	uint16 seqnum, uint16 xt_id)
+{
+	dhd_prot_t *prot = dhdp->prot;
+	host_timestamp_msg_t *ts_req;
+	unsigned long flags;
+	uint16 alloced = 0;
+	uchar *ts_tlv_buf;
+	msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
+	int ret;
+
+	if ((tlvs == NULL) || (tlv_len == 0)) {
+		DHD_ERROR(("%s: argument error tlv: %p, tlv_len %d\n",
+			__FUNCTION__, tlvs, tlv_len));
+		return -1;
+	}
+
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhdp->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+
+	/* if Host TS req already pending go away */
+	if (prot->hostts_req_buf_inuse == TRUE) {
+		DHD_ERROR(("one host TS request already pending at device\n"));
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhdp->bus);
+#endif
+		return -1;
+	}
+
+	/* Request for cbuf space */
+	ts_req = (host_timestamp_msg_t*)dhd_prot_alloc_ring_space(dhdp, ctrl_ring,
+		DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,	&alloced, FALSE);
+	if (ts_req == NULL) {
+		DHD_ERROR(("couldn't allocate space on msgring to send host TS request\n"));
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhdp->bus);
+#endif
+		return -1;
+	}
+
+	/* Common msg buf hdr */
+	ts_req->msg.msg_type = MSG_TYPE_HOSTTIMSTAMP;
+	ts_req->msg.if_id = 0;
+	ts_req->msg.flags =  ctrl_ring->current_phase;
+	ts_req->msg.request_id = DHD_H2D_HOSTTS_REQ_PKTID;
+
+	ts_req->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+	ctrl_ring->seqnum++;
+
+	ts_req->xt_id = xt_id;
+	ts_req->seqnum = seqnum;
+	/* populate TS req buffer info */
+	ts_req->input_data_len = htol16(tlv_len);
+	ts_req->host_buf_addr.high = htol32(PHYSADDRHI(prot->hostts_req_buf.pa));
+	ts_req->host_buf_addr.low = htol32(PHYSADDRLO(prot->hostts_req_buf.pa));
+	/* copy ioct payload */
+	ts_tlv_buf = (void *) prot->hostts_req_buf.va;
+	prot->hostts_req_buf_inuse = TRUE;
+	ret = memcpy_s(ts_tlv_buf, prot->hostts_req_buf.len, tlvs, tlv_len);
+	if (ret) {
+		DHD_ERROR(("copy ioct payload failed:%d, destsz:%d, n:%d\n",
+			ret, prot->hostts_req_buf.len, tlv_len));
+		return BCME_ERROR;
+	}
+
+	OSL_CACHE_FLUSH((void *) prot->hostts_req_buf.va, tlv_len);
+
+	if (ISALIGNED(ts_tlv_buf, DMA_ALIGN_LEN) == FALSE) {
+		DHD_ERROR(("host TS req buffer address unaligned !!!!! \n"));
+	}
+
+	DHD_CTL(("submitted Host TS request request_id %d, data_len %d, tx_id %d, seq %d\n",
+		ts_req->msg.request_id, ts_req->input_data_len,
+		ts_req->xt_id, ts_req->seqnum));
+
+	/* upd wrt ptr and raise interrupt */
+	dhd_prot_ring_write_complete(dhdp, ctrl_ring, ts_req,
+		DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhdp->bus);
+#endif
+	return 0;
+} /* dhd_prot_send_host_timestamp */
+
+bool
+dhd_prot_data_path_tx_timestamp_logging(dhd_pub_t *dhd,  bool enable, bool set)
+{
+	if (set)
+		dhd->prot->tx_ts_log_enabled = enable;
+
+	return dhd->prot->tx_ts_log_enabled;
+}
+
+bool
+dhd_prot_data_path_rx_timestamp_logging(dhd_pub_t *dhd,  bool enable, bool set)
+{
+	if (set)
+		dhd->prot->rx_ts_log_enabled = enable;
+
+	return dhd->prot->rx_ts_log_enabled;
+}
+
+bool
+dhd_prot_pkt_noretry(dhd_pub_t *dhd, bool enable, bool set)
+{
+	if (set)
+		dhd->prot->no_retry = enable;
+
+	return dhd->prot->no_retry;
+}
+
+bool
+dhd_prot_pkt_noaggr(dhd_pub_t *dhd, bool enable, bool set)
+{
+	if (set)
+		dhd->prot->no_aggr = enable;
+
+	return dhd->prot->no_aggr;
+}
+
+bool
+dhd_prot_pkt_fixed_rate(dhd_pub_t *dhd, bool enable, bool set)
+{
+	if (set)
+		dhd->prot->fixed_rate = enable;
+
+	return dhd->prot->fixed_rate;
+}
+#endif /* BCMPCIE */
+
+void
+dhd_prot_dma_indx_free(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	dhd_dma_buf_free(dhd, &prot->h2d_dma_indx_wr_buf);
+	dhd_dma_buf_free(dhd, &prot->d2h_dma_indx_rd_buf);
+}
+
+void
+dhd_msgbuf_delay_post_ts_bufs(dhd_pub_t *dhd)
+{
+	if (dhd->prot->max_tsbufpost > 0)
+		dhd_msgbuf_rxbuf_post_ts_bufs(dhd);
+}
+
+static void
+BCMFASTPATH(dhd_prot_process_fw_timestamp)(dhd_pub_t *dhd, void* buf)
+{
+	DHD_ERROR(("Timesunc feature not compiled in but GOT FW TS message\n"));
+
+}
+
+uint16
+dhd_prot_get_ioctl_trans_id(dhd_pub_t *dhdp)
+{
+	return dhdp->prot->ioctl_trans_id;
+}
+
+int dhd_get_hscb_info(dhd_pub_t *dhd, void ** va, uint32 *len)
+{
+	if (!dhd->hscb_enable) {
+		if (len) {
+			/* prevent "Operation not supported" dhd message */
+			*len = 0;
+			return BCME_OK;
+		}
+		return BCME_UNSUPPORTED;
+	}
+
+	if (va) {
+		*va = dhd->prot->host_scb_buf.va;
+	}
+	if (len) {
+		*len = dhd->prot->host_scb_buf.len;
+	}
+
+	return BCME_OK;
+}
+
+#ifdef DHD_BUS_MEM_ACCESS
+int dhd_get_hscb_buff(dhd_pub_t *dhd, uint32 offset, uint32 length, void * buff)
+{
+	if (!dhd->hscb_enable) {
+		return BCME_UNSUPPORTED;
+	}
+
+	if (dhd->prot->host_scb_buf.va == NULL ||
+		((uint64)offset + length > (uint64)dhd->prot->host_scb_buf.len)) {
+		return BCME_BADADDR;
+	}
+
+	memcpy(buff, (char*)dhd->prot->host_scb_buf.va + offset, length);
+
+	return BCME_OK;
+}
+#endif /* DHD_BUS_MEM_ACCESS */
+
+static uint16
+dhd_d2h_cpl_ring_id(dhd_pub_t *dhd, msgbuf_ring_t *ring)
+{
+	uint16 max_h2d_rings = dhd->bus->max_submission_rings;
+	dhd_prot_t *prot = dhd->prot;
+
+	if (ring == &prot->d2hring_tx_cpln)
+		return prot->d2hring_tx_cpln.idx;
+	if (ring == &prot->d2hring_rx_cpln)
+		return prot->d2hring_rx_cpln.idx;
+
+	return (DHD_D2H_RING_OFFSET(ring->idx, max_h2d_rings) +
+		BCMPCIE_D2H_COMMON_MSGRINGS);
+}
+
+static int
+dhd_d2h_id_from_cpl_ring_idx(dhd_pub_t *dhd, uint16 idx)
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	if (prot->d2hring_tx_cpln.idx == idx) {
+		return DHD_METADATA_D2H_TXCPL;
+	}
+	if (prot->d2hring_rx_cpln.idx == idx) {
+		return DHD_METADATA_D2H_RXCPL;
+	}
+	return DHD_METADATA_NO_RING;
+}
+
+static msgbuf_ring_t *
+dhd_d2h_cpl_ring_from_id(dhd_pub_t *dhd, int idx)
+{
+	dhd_prot_t *prot = dhd->prot;
+
+	switch (idx) {
+		case DHD_METADATA_D2H_TXCPL: /* D2H Tx Completion Ring */
+		if (prot->d2hring_tx_cpln.inited) {
+			return &prot->d2hring_tx_cpln;
+		}
+		break;
+		case DHD_METADATA_D2H_RXCPL: /* D2H Rx Completion Ring */
+		if (prot->d2hring_rx_cpln.inited) {
+			return &prot->d2hring_rx_cpln;
+		}
+		break;
+		default:
+			return NULL;
+	}
+	return NULL;
+}
+
+int
+dhd_prot_mdring_link_unlink(dhd_pub_t *dhd, int idx, bool link)
+{
+	void *msg_start;
+	uint16 alloced = 0;
+	unsigned long flags;
+	dhd_prot_t *prot = dhd->prot;
+	ring_config_req_t *ring_config_req;
+	msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
+	msgbuf_ring_t *linked_ring = NULL;
+
+	if (!prot->d2hring_md_cpl) {
+		DHD_ERROR(("%s No metadata ring exists\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	if (link && prot->d2hring_md_cpl->linked_ring) {
+		DHD_ERROR(("%s Link:metadata ring already linked\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	linked_ring = dhd_d2h_cpl_ring_from_id(dhd, idx);
+	if (!link && linked_ring != prot->d2hring_md_cpl->linked_ring) {
+		DHD_ERROR(("%s Unlink:metadata ring not linked\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	prot->d2hring_md_cpl->linked_ring = linked_ring;
+#ifdef PCIE_INB_DW
+	if (dhd_prot_inc_hostactive_devwake_assert(dhd->bus) != BCME_OK)
+		return BCME_ERROR;
+#endif /* PCIE_INB_DW */
+	/* Claim space for 1  d2h_ring_config_req_t messages */
+	DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
+	msg_start = dhd_prot_alloc_ring_space(dhd, ctrl_ring, 1, &alloced, TRUE);
+
+	if (msg_start == NULL) {
+		DHD_ERROR(("%s Msgbuf no space for D2H ring config metadta ring link/unlink\n",
+			__FUNCTION__));
+		DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+#ifdef PCIE_INB_DW
+		dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+		return BCME_ERROR;
+	}
+
+	/* position the ring_config_req into the ctrl subm ring */
+	ring_config_req = (ring_config_req_t *)msg_start;
+
+	/* Common msg header */
+	ring_config_req->msg.msg_type = MSG_TYPE_D2H_RING_CONFIG;
+	ring_config_req->msg.if_id = 0;
+	ring_config_req->msg.request_id = htol32(0); /* TBD */
+	ring_config_req->msg.flags = ctrl_ring->current_phase;
+
+	ring_config_req->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
+	ctrl_ring->seqnum++;
+
+	ring_config_req->msg.request_id = htol32(DHD_FAKE_PKTID); /* unused */
+
+	/* Ring Config subtype and d2h ring_id */
+	if (link) {
+		ring_config_req->subtype = htol16(D2H_RING_CONFIG_SUBTYPE_MDATA_LINK);
+	} else {
+		ring_config_req->subtype = htol16(D2H_RING_CONFIG_SUBTYPE_MDATA_UNLINK);
+	}
+	ring_config_req->ring_id =
+		htol16(dhd_d2h_cpl_ring_id(dhd, prot->d2hring_md_cpl));
+
+	ring_config_req->mdata_assoc.ringid =
+		htol16(dhd_d2h_cpl_ring_id(dhd, prot->d2hring_md_cpl->linked_ring));
+
+	DHD_ERROR(("%s: metadata:%d link to ring:%d\n",
+		__FUNCTION__, ring_config_req->ring_id,
+		ring_config_req->mdata_assoc.ringid));
+
+	/* update control subn ring's WR index and ring doorbell to dongle */
+	dhd_prot_ring_write_complete(dhd, ctrl_ring, msg_start, 1);
+
+	DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
+
+#ifdef PCIE_INB_DW
+	dhd_prot_dec_hostactive_ack_pending_dsreq(dhd->bus);
+#endif
+	return BCME_OK;
+}
+
+int
+dhd_prot_mdring_linked_ring(dhd_pub_t *dhd)
+{
+	dhd_prot_t *prot = dhd->prot;
+	if (prot->d2hring_md_cpl && prot->d2hring_md_cpl->linked_ring) {
+		return dhd_d2h_id_from_cpl_ring_idx
+			(dhd, prot->d2hring_md_cpl->linked_ring->idx);
+	}
+	return 0;
+}
+
+/*
+* rx_status1:
+* Bit [31:0] - Packet reception time
+*
+* rx_status_ext (marker):
+* Bit[7:0] - RSSI for the received packet
+* Bit[23:8] - Set to the duration of the packet inside the dongle -
+* measured in units of 32 microseconds
+* Bit[24] - Packet received on 2G (0) or 5G (1)
+*
+* flags:
+* Bit [7:5] Packet priority
+*/
+
+static void
+dhd_update_rxstats(dhd_pub_t *dhd, host_rxbuf_cmpl_t *rxstatus)
+{
+	uint32 marker = rxstatus->marker;
+	uint16 flags = rxstatus->flags;
+	rx_cpl_lat_info_t *rxcpl_info;
+	uint32 dur;
+	uint8 slice;
+	uint8 prio;
+	int8 rssi;
+	uint32 rx_t0;
+
+	ts_timestamp_t *ts = (ts_timestamp_t *)&rxstatus->ts;
+
+	rxcpl_info = &dhd->rxcpl_lat_info;
+	rssi = (marker & BCMPCIE_RX_PKT_RSSI_MASK);
+	rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].rssi = rssi;
+	dur = (((marker & BCMPCIE_RX_PKT_DUR0_MASK) >>
+		BCMPCIE_RX_PKT_DUR0_SHIFT) * RX_LAT_TIME_SCALE);
+	rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].latency = dur;
+	slice = (marker & BCMPCIE_RX_PKT_BAND_MASK) >> BCMPCIE_RX_PKT_BAND_SHIFT;
+	rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].slice = slice;
+
+	prio = ((flags & BCMPCIE_PKT_FLAGS_PRIO_MASK) >> BCMPCIE_PKT_FLAGS_PRIO_SHIFT);
+	if (prio >= MAX_RX_LAT_PRIO) {
+		DHD_ERROR(("dhd_update_rxstats:Prio exceeded Max:%d\n", prio));
+		return;
+	}
+	rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].priority = prio;
+
+	rx_t0 = rxstatus->rx_status_1;
+	rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].rx_t0 = rx_t0;
+
+	/* store PTM timestamps */
+	if (DHD_PTM_CLKID(ts->high)) {
+		rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].ptm_high = ts->high;
+		rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].ptm_low = ts->low;
+		rxcpl_info->rx_history[rxcpl_info->rxcpl_hist_count].host_time =
+			(uint32)OSL_SYSUPTIME_US();
+	}
+
+	DHD_INFO(("%s:0x%x ptm_high:0x%x, ptm_low:0x%x, t0:0x%x band:%s, prio:%d, rssi:%d, "
+		"dur:%d useconds\n", __FUNCTION__, marker, ts->high, ts->low, rx_t0,
+		slice ? "5G":"2G", prio, rssi, dur));
+	rxcpl_info->rxcpl_hist_count =
+		(rxcpl_info->rxcpl_hist_count +1) % MAX_RXCPL_HISTORY;
+
+	dur /= RX_LAT_BIN_SCALE; /* scaled down for each bin */
+	if (dur >= MAX_RX_LAT_HIST_BIN) {
+		dur = MAX_RX_LAT_HIST_BIN -1;
+	}
+	if (slice) {
+		rxcpl_info->rx_dur_5g[prio][dur]++;
+	} else {
+		rxcpl_info->rx_dur_2g[prio][dur]++;
+	}
+}
+
+#ifdef DHD_MAP_LOGGING
+void
+dhd_prot_smmu_fault_dump(dhd_pub_t *dhdp)
+{
+	dhd_prot_debug_info_print(dhdp);
+	OSL_DMA_MAP_DUMP(dhdp->osh);
+#ifdef DHD_MAP_PKTID_LOGGING
+	dhd_pktid_logging_dump(dhdp);
+#endif /* DHD_MAP_PKTID_LOGGING */
+#ifdef DHD_FW_COREDUMP
+	dhdp->memdump_type = DUMP_TYPE_SMMU_FAULT;
+#ifdef DNGL_AXI_ERROR_LOGGING
+	dhdp->memdump_enabled = DUMP_MEMFILE;
+	dhd_bus_get_mem_dump(dhdp);
+#else
+	dhdp->memdump_enabled = DUMP_MEMONLY;
+	dhd_bus_mem_dump(dhdp);
+#endif /* DNGL_AXI_ERROR_LOGGING */
+#endif /* DHD_FW_COREDUMP */
+}
+#endif /* DHD_MAP_LOGGING */
+
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+void
+dhd_dump_bus_flow_ring_status_trace(
+	dhd_bus_t *bus, struct bcmstrbuf *strbuf, dhd_frs_trace_t *frs_trace, int dumpsz, char *str)
+{
+	int i;
+	dhd_prot_t *prot = bus->dhd->prot;
+	uint32 isr_cnt = bus->frs_isr_count % FRS_TRACE_SIZE;
+	uint32 dpc_cnt = bus->frs_dpc_count % FRS_TRACE_SIZE;
+
+	bcm_bprintf(strbuf, "---- %s ------ isr_cnt: %d dpc_cnt %d\n",
+		str, isr_cnt, dpc_cnt);
+	bcm_bprintf(strbuf, "%s\t%s\t%s\t%s\t%s\t%s\t",
+		"Timestamp ns", "H2DCtrlPost", "D2HCtrlCpl",
+		"H2DRxPost", "D2HRxCpl", "D2HTxCpl");
+	if (prot->h2dring_info_subn != NULL && prot->d2hring_info_cpln != NULL) {
+		bcm_bprintf(strbuf, "%s\t%s\t", "H2DRingInfoPost", "D2HRingInfoCpl");
+	}
+	if (prot->d2hring_edl != NULL) {
+		bcm_bprintf(strbuf, "%s", "D2HRingEDL");
+	}
+	bcm_bprintf(strbuf, "\n");
+	for (i = 0; i < dumpsz; i ++) {
+		bcm_bprintf(strbuf, "%llu\t%6u-%u\t%6u-%u\t%6u-%u\t%6u-%u\t%6u-%u\t",
+				frs_trace[i].timestamp,
+				frs_trace[i].h2d_ctrl_post_drd,
+				frs_trace[i].h2d_ctrl_post_dwr,
+				frs_trace[i].d2h_ctrl_cpln_drd,
+				frs_trace[i].d2h_ctrl_cpln_dwr,
+				frs_trace[i].h2d_rx_post_drd,
+				frs_trace[i].h2d_rx_post_dwr,
+				frs_trace[i].d2h_rx_cpln_drd,
+				frs_trace[i].d2h_rx_cpln_dwr,
+				frs_trace[i].d2h_tx_cpln_drd,
+				frs_trace[i].d2h_tx_cpln_dwr);
+		if (prot->h2dring_info_subn != NULL && prot->d2hring_info_cpln != NULL) {
+			bcm_bprintf(strbuf, "%6u-%u\t%6u-%u\t",
+				frs_trace[i].h2d_info_post_drd,
+				frs_trace[i].h2d_info_post_dwr,
+				frs_trace[i].d2h_info_cpln_drd,
+				frs_trace[i].d2h_info_cpln_dwr);
+		}
+		if (prot->d2hring_edl != NULL) {
+			bcm_bprintf(strbuf, "%6u-%u",
+				frs_trace[i].d2h_ring_edl_drd,
+				frs_trace[i].d2h_ring_edl_dwr);
+
+		}
+		bcm_bprintf(strbuf, "\n");
+	}
+	bcm_bprintf(strbuf, "--------------------------\n");
+}
+
+void
+dhd_dump_bus_flow_ring_status_isr_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	int dumpsz;
+
+	dumpsz = bus->frs_isr_count < FRS_TRACE_SIZE ?
+		bus->frs_isr_count : FRS_TRACE_SIZE;
+	if (dumpsz == 0) {
+		bcm_bprintf(strbuf, "\nEMPTY ISR FLOW RING TRACE\n");
+		return;
+	}
+	dhd_dump_bus_flow_ring_status_trace(bus, strbuf, bus->frs_isr_trace,
+		dumpsz, "ISR FLOW RING TRACE DRD-DWR");
+}
+
+void
+dhd_dump_bus_flow_ring_status_dpc_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	int dumpsz;
+
+	dumpsz = bus->frs_dpc_count < FRS_TRACE_SIZE ?
+		bus->frs_dpc_count : FRS_TRACE_SIZE;
+	if (dumpsz == 0) {
+		bcm_bprintf(strbuf, "\nEMPTY ISR FLOW RING TRACE\n");
+		return;
+	}
+	dhd_dump_bus_flow_ring_status_trace(bus, strbuf, bus->frs_dpc_trace,
+		dumpsz, "DPC FLOW RING TRACE DRD-DWR");
+}
+static void
+dhd_bus_flow_ring_status_trace(dhd_pub_t *dhd, dhd_frs_trace_t *frs_trace)
+{
+	dhd_prot_t *prot = dhd->prot;
+	msgbuf_ring_t *ring;
+
+	ring = &prot->h2dring_ctrl_subn;
+	frs_trace->h2d_ctrl_post_drd =
+		dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+	frs_trace->h2d_ctrl_post_dwr =
+		dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, ring->idx);
+
+	ring = &prot->d2hring_ctrl_cpln;
+	frs_trace->d2h_ctrl_cpln_drd =
+		dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+	frs_trace->d2h_ctrl_cpln_dwr =
+		dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+
+	ring = &prot->h2dring_rxp_subn;
+	frs_trace->h2d_rx_post_drd =
+		dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+	frs_trace->h2d_rx_post_dwr =
+		dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, ring->idx);
+
+	ring = &prot->d2hring_rx_cpln;
+	frs_trace->d2h_rx_cpln_drd =
+		dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+	frs_trace->d2h_rx_cpln_dwr =
+		dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+
+	ring = &prot->d2hring_tx_cpln;
+	frs_trace->d2h_tx_cpln_drd =
+		dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+	frs_trace->d2h_tx_cpln_dwr =
+		dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+
+	if ((dhd->prot->h2dring_info_subn != NULL) && (dhd->prot->d2hring_info_cpln != NULL)) {
+		ring = prot->h2dring_info_subn;
+		if (ring->inited) {
+			frs_trace->h2d_info_post_drd =
+				dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
+			frs_trace->h2d_info_post_dwr =
+				dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_WR_UPD, ring->idx);
+		}
+		ring = prot->d2hring_info_cpln;
+		if (ring->inited) {
+			frs_trace->d2h_info_cpln_drd =
+				dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+			frs_trace->d2h_info_cpln_dwr =
+				dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+		}
+	}
+	if (prot->d2hring_edl != NULL) {
+		ring = prot->d2hring_edl;
+		if (ring->inited) {
+			frs_trace->d2h_ring_edl_drd =
+				dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_RD_UPD, ring->idx);
+			frs_trace->d2h_ring_edl_dwr =
+				dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
+		}
+	}
+
+}
+
+void
+dhd_bus_flow_ring_status_isr_trace(dhd_pub_t *dhd)
+{
+	uint32 cnt = dhd->bus->frs_isr_count % FRS_TRACE_SIZE;
+	dhd_frs_trace_t *frs_isr_trace = &dhd->bus->frs_isr_trace[cnt];
+	uint64 time_ns_prev = frs_isr_trace->timestamp;
+	uint64 time_ns_now = OSL_LOCALTIME_NS();
+
+	if ((time_ns_now - time_ns_prev) < 250000) { /* delta less than 250us */
+		return;
+	}
+
+	dhd_bus_flow_ring_status_trace(dhd, frs_isr_trace);
+
+	frs_isr_trace->timestamp = OSL_LOCALTIME_NS();
+	dhd->bus->frs_isr_count ++;
+}
+
+void
+dhd_bus_flow_ring_status_dpc_trace(dhd_pub_t *dhd)
+{
+	uint32 cnt = dhd->bus->frs_dpc_count % FRS_TRACE_SIZE;
+	dhd_frs_trace_t *frs_dpc_trace = &dhd->bus->frs_dpc_trace[cnt];
+	uint64 time_ns_prev = frs_dpc_trace->timestamp;
+	uint64 time_ns_now = OSL_LOCALTIME_NS();
+
+	if ((time_ns_now - time_ns_prev) < 250000) { /* delta less than 250us */
+		return;
+	}
+
+	dhd_bus_flow_ring_status_trace(dhd, frs_dpc_trace);
+
+	frs_dpc_trace->timestamp = OSL_LOCALTIME_NS();
+	dhd->bus->frs_dpc_count ++;
+}
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
diff --git a/drivers/net/wireless/ap12275/dhd_pcie.c b/drivers/net/wireless/ap12275/dhd_pcie.c
new file mode 100644
index 000000000000..bd9f5a2d8191
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pcie.c
@@ -0,0 +1,18411 @@
+/*
+ * DHD Bus Module for PCIE
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/** XXX Twiki: [PCIeFullDongleArchitecture] */
+
+/* include files */
+#include <typedefs.h>
+#include <bcmutils.h>
+#include <bcmrand.h>
+#include <bcmdevs.h>
+#include <bcmdevs_legacy.h>    /* need to still support chips no longer in trunk firmware */
+#include <siutils.h>
+#include <hndoobr.h>
+#include <hndsoc.h>
+#include <hndpmu.h>
+#include <etd.h>
+#include <hnd_debug.h>
+#include <sbchipc.h>
+#include <sbhndarm.h>
+#include <sbsysmem.h>
+#include <hnd_armtrap.h>
+#if defined(DHD_DEBUG)
+#include <hnd_cons.h>
+#endif /* defined(DHD_DEBUG) */
+#include <dngl_stats.h>
+#include <pcie_core.h>
+#include <dhd.h>
+#include <dhd_bus.h>
+#include <dhd_flowring.h>
+#include <dhd_proto.h>
+#include <dhd_dbg.h>
+#include <dhd_debug.h>
+#if defined(__linux__)
+#include <dhd_daemon.h>
+#include <dhd_linux.h>
+#include <dhd_plat.h>
+#endif /* __linux__ */
+#include <dhdioctl.h>
+#include <sdiovar.h>
+#include <bcmmsgbuf.h>
+#include <pcicfg.h>
+#include <dhd_pcie.h>
+#include <bcmpcie.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmstdlib_s.h>
+#ifdef DHDTCPACK_SUPPRESS
+#include <dhd_ip.h>
+#endif /* DHDTCPACK_SUPPRESS */
+#include <bcmevent.h>
+#include <dhd_config.h>
+
+#ifdef BCM_ROUTER_DHD
+#include <bcmnvram.h>
+#define STR_END		"END\0\0"
+#define BOARDREV_PROMOTABLE_STR	"0xff"
+#endif
+#if defined(BCMEMBEDIMAGE)
+#include BCMEMBEDIMAGE
+#endif /* BCMEMBEDIMAGE */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#include <linux/pm_runtime.h>
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+#include <debugger.h>
+#endif /* DEBUGGER || DHD_DSCOPE */
+
+#include <dngl_rtlv.h>
+#if defined(FW_SIGNATURE)
+#include <bootrommem.h>
+#include <fwpkg_utils.h>
+#endif /* FW_SIGNATURE */
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+#include <dhd_linux_wq.h>
+#include <dhd_linux.h>
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+#if defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON)
+#include <dhd_linux_priv.h>
+#endif /* DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON */
+#include <dhd_linux_pktdump.h>
+
+#define EXTENDED_PCIE_DEBUG_DUMP 1	/* Enable Extended pcie registers dump */
+
+#define MEMBLOCK	2048		/* Block size used for downloading of dongle image */
+#ifdef LINUX
+#define MAX_WKLK_IDLE_CHECK	3 /* times dhd_wake_lock checked before deciding not to suspend */
+#endif /* LINUX */
+
+#define	DHD_MAX_ITEMS_HPP_TXCPL_RING	512
+#define	DHD_MAX_ITEMS_HPP_RXCPL_RING	512
+#define MAX_HP2P_CMPL_RINGS		2u
+
+/* XXX defines for 4378 */
+#define ARMCR4REG_CORECAP	(0x4/sizeof(uint32))
+#define ARMCR4REG_MPUCTRL	(0x90/sizeof(uint32))
+#define ACC_MPU_SHIFT		25
+#define ACC_MPU_MASK		(0x1u << ACC_MPU_SHIFT)
+
+/* XXX Offset for 4375 work around register */
+#define REG_WORK_AROUND		(0x1e4/sizeof(uint32))
+
+/* XXX defines for 43602a0 workaround JIRA CRWLARMCR4-53 */
+#define ARMCR4REG_BANKIDX	(0x40/sizeof(uint32))
+#define ARMCR4REG_BANKPDA	(0x4C/sizeof(uint32))
+/* Temporary war to fix precommit till sync issue between trunk & precommit branch is resolved */
+
+/* CTO Prevention Recovery */
+#define CTO_TO_CLEAR_WAIT_MS 50
+#define CTO_TO_CLEAR_WAIT_MAX_CNT 200
+
+/* FLR setting */
+#define PCIE_FLR_CAPAB_BIT		28
+#define PCIE_FUNCTION_LEVEL_RESET_BIT	15
+
+#ifdef BCMQT_HW
+extern int qt_flr_reset;
+/* FLR takes longer on QT Z boards so increasing the delay by 30% */
+#define DHD_FUNCTION_LEVEL_RESET_DELAY	70u
+#define DHD_SSRESET_STATUS_RETRY_DELAY	55u
+#else
+#define DHD_FUNCTION_LEVEL_RESET_DELAY	70u	/* 70 msec delay */
+#define DHD_SSRESET_STATUS_RETRY_DELAY	40u
+#endif /* BCMQT_HW */
+/*
+ * Increase SSReset de-assert time to 8ms.
+ * since it takes longer time if re-scan time on 4378B0.
+ */
+#define DHD_SSRESET_STATUS_RETRIES	200u
+
+/* Fetch address of a member in the pciedev_shared structure in dongle memory */
+#define DHD_PCIE_SHARED_MEMBER_ADDR(bus, member) \
+	(bus)->shared_addr + OFFSETOF(pciedev_shared_t, member)
+
+/* Fetch address of a member in rings_info_ptr structure in dongle memory */
+#define DHD_RING_INFO_MEMBER_ADDR(bus, member) \
+	(bus)->pcie_sh->rings_info_ptr + OFFSETOF(ring_info_t, member)
+
+/* Fetch address of a member in the ring_mem structure in dongle memory */
+#define DHD_RING_MEM_MEMBER_ADDR(bus, ringid, member) \
+	(bus)->ring_sh[ringid].ring_mem_addr + OFFSETOF(ring_mem_t, member)
+
+/* Mpu conntrol */
+#define MPU_REG(regs, reg)	(&((sysmemregs_t *)regs)->reg)
+#define BUS_MPU_ENABLE_MASK	0x1u
+#define BUS_MPU_ROM_PR_MASK	0x2u
+#define BUS_MPU_VEC_PR_MASK	0x4u
+#define BUS_MPU_LOCK_MASK	0x8u
+
+#if defined(SUPPORT_MULTIPLE_BOARD_REV)
+extern unsigned int system_rev;
+#endif /* SUPPORT_MULTIPLE_BOARD_REV */
+
+#ifdef EWP_EDL
+extern int host_edl_support;
+#endif
+
+#ifdef BCMQT_HW
+extern int qt_dngl_timeout;
+#endif /* BCMQT_HW */
+
+#define CONS_ADDR_SIGNATURE	0xac0ffee
+/* This can be overwritten by module parameter(dma_ring_indices) defined in dhd_linux.c */
+uint dma_ring_indices = 0;
+/* This can be overwritten by module parameter(h2d_phase) defined in dhd_linux.c */
+bool h2d_phase = 0;
+/* This can be overwritten by module parameter(force_trap_bad_h2d_phase)
+ * defined in dhd_linux.c
+ */
+bool force_trap_bad_h2d_phase = 0;
+
+int dhd_dongle_ramsize;
+struct dhd_bus *g_dhd_bus = NULL;
+#ifdef DNGL_AXI_ERROR_LOGGING
+static void dhd_log_dump_axi_error(uint8 *axi_err);
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+static int dhdpcie_checkdied(dhd_bus_t *bus, char *data, uint size);
+static int dhdpcie_bus_readconsole(dhd_bus_t *bus);
+#if defined(DHD_FW_COREDUMP)
+static int dhdpcie_mem_dump(dhd_bus_t *bus);
+static int dhdpcie_get_mem_dump(dhd_bus_t *bus);
+#endif /* DHD_FW_COREDUMP */
+
+static int dhdpcie_bus_membytes(dhd_bus_t *bus, bool write, ulong address, uint8 *data, uint size);
+static int dhdpcie_bus_doiovar(dhd_bus_t *bus, const bcm_iovar_t *vi, uint32 actionid,
+	const char *name, void *params,
+	uint plen, void *arg, uint len, int val_size);
+static int dhdpcie_bus_lpback_req(struct  dhd_bus *bus, uint32 intval);
+static int dhdpcie_bus_dmaxfer_req(struct  dhd_bus *bus,
+	uint32 len, uint32 srcdelay, uint32 destdelay,
+	uint32 d11_lpbk, uint32 core_num, uint32 wait,
+	uint32 mem_addr);
+static uint serialized_backplane_access(dhd_bus_t* bus, uint addr, uint size, uint* val, bool read);
+static int dhdpcie_bus_download_state(dhd_bus_t *bus, bool enter);
+static int _dhdpcie_download_firmware(struct dhd_bus *bus);
+static int dhdpcie_download_firmware(dhd_bus_t *bus, osl_t *osh);
+
+#if defined(FW_SIGNATURE)
+static int dhdpcie_bus_download_fw_signature(dhd_bus_t *bus, bool *do_write);
+static int dhdpcie_bus_write_fw_signature(dhd_bus_t *bus);
+static int dhdpcie_bus_download_ram_bootloader(dhd_bus_t *bus);
+static int dhdpcie_bus_write_fws_status(dhd_bus_t *bus);
+static int dhdpcie_bus_write_fws_mem_info(dhd_bus_t *bus);
+static int dhdpcie_bus_write_fwsig(dhd_bus_t *bus, char *fwsig_path, char *nvsig_path);
+static int dhdpcie_download_rtlv_end(dhd_bus_t *bus);
+static int dhdpcie_bus_save_download_info(dhd_bus_t *bus, uint32 download_addr,
+	uint32 download_size, const char *signature_fname,
+	const char *bloader_fname, uint32 bloader_download_addr);
+static int dhdpcie_read_fwstatus(dhd_bus_t *bus, bl_verif_status_t *status);
+#endif /* FW_SIGNATURE */
+
+static int dhdpcie_bus_write_vars(dhd_bus_t *bus);
+static bool dhdpcie_bus_process_mailbox_intr(dhd_bus_t *bus, uint32 intstatus);
+static bool dhdpci_bus_read_frames(dhd_bus_t *bus);
+static int dhdpcie_readshared(dhd_bus_t *bus);
+static void dhdpcie_init_shared_addr(dhd_bus_t *bus);
+static bool dhdpcie_dongle_attach(dhd_bus_t *bus);
+static void dhdpcie_bus_dongle_setmemsize(dhd_bus_t *bus, int mem_size);
+static void dhdpcie_bus_release_dongle(dhd_bus_t *bus, osl_t *osh,
+	bool dongle_isolation, bool reset_flag);
+static void dhdpcie_bus_release_malloc(dhd_bus_t *bus, osl_t *osh);
+static int dhdpcie_downloadvars(dhd_bus_t *bus, void *arg, int len);
+static void dhdpcie_setbar1win(dhd_bus_t *bus, uint32 addr);
+static void dhd_deinit_bar1_switch_lock(dhd_bus_t *bus);
+static void dhd_init_pwr_req_lock(dhd_bus_t *bus);
+static void dhd_deinit_pwr_req_lock(dhd_bus_t *bus);
+static void dhd_init_bus_lp_state_lock(dhd_bus_t *bus);
+static void dhd_deinit_bus_lp_state_lock(dhd_bus_t *bus);
+static void dhd_init_backplane_access_lock(dhd_bus_t *bus);
+static void dhd_deinit_backplane_access_lock(dhd_bus_t *bus);
+static uint8 dhdpcie_bus_rtcm8(dhd_bus_t *bus, ulong offset);
+static void dhdpcie_bus_wtcm8(dhd_bus_t *bus, ulong offset, uint8 data);
+static void dhdpcie_bus_wtcm16(dhd_bus_t *bus, ulong offset, uint16 data);
+static uint16 dhdpcie_bus_rtcm16(dhd_bus_t *bus, ulong offset);
+static void dhdpcie_bus_wtcm32(dhd_bus_t *bus, ulong offset, uint32 data);
+static uint32 dhdpcie_bus_rtcm32(dhd_bus_t *bus, ulong offset);
+#ifdef DHD_SUPPORT_64BIT
+static void dhdpcie_bus_wtcm64(dhd_bus_t *bus, ulong offset, uint64 data) __attribute__ ((used));
+static uint64 dhdpcie_bus_rtcm64(dhd_bus_t *bus, ulong offset) __attribute__ ((used));
+#endif /* DHD_SUPPORT_64BIT */
+static void dhdpcie_bus_cfg_set_bar0_win(dhd_bus_t *bus, uint32 data);
+static void dhdpcie_bus_reg_unmap(osl_t *osh, volatile char *addr, int size);
+static int dhdpcie_cc_nvmshadow(dhd_bus_t *bus, struct bcmstrbuf *b);
+static void dhd_bus_dump_rxlat_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+static void dhd_bus_dump_rxlat_histo(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+static void dhd_bus_dump_txcpl_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+static void dhd_bus_dump_mdring_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+static bool dhd_bus_support_dar_sec_status(dhd_bus_t *bus);
+static int dhd_bus_security_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+static int dhd_bus_sboot_disable(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+static int dhd_bus_get_security_status(dhd_bus_t *bus, uint32 *status);
+static void dhdpcie_fw_trap(dhd_bus_t *bus);
+static void dhd_fillup_ring_sharedptr_info(dhd_bus_t *bus, ring_info_t *ring_info);
+static void dhdpcie_handle_mb_data(dhd_bus_t *bus);
+extern void dhd_dpc_enable(dhd_pub_t *dhdp);
+#ifdef PCIE_INB_DW
+static void dhd_bus_ds_trace(dhd_bus_t *bus, uint32 dsval,
+	bool d2h, enum dhd_bus_ds_state inbstate);
+#else
+static void dhd_bus_ds_trace(dhd_bus_t *bus, uint32 dsval, bool d2h);
+#endif /* PCIE_INB_DW */
+#ifdef DHD_MMIO_TRACE
+static void dhd_bus_mmio_trace(dhd_bus_t *bus, uint32 addr, uint32 value, bool set);
+#endif /* defined(DHD_MMIO_TRACE) */
+
+#ifdef IDLE_TX_FLOW_MGMT
+static void dhd_bus_check_idle_scan(dhd_bus_t *bus);
+static void dhd_bus_idle_scan(dhd_bus_t *bus);
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#ifdef BCMEMBEDIMAGE
+static int dhdpcie_download_code_array(dhd_bus_t *bus);
+#endif /* BCMEMBEDIMAGE */
+#ifdef BCM_ROUTER_DHD
+extern char * nvram_get(const char *name);
+#endif
+#if defined(BCMEMBEDIMAGE) && defined(BCM_ROUTER_DHD)
+static void  select_fd_image(
+		struct dhd_bus *bus, unsigned char **p_dlarray,
+		char **p_dlimagename, char **p_dlimagever,
+		char **p_dlimagedate, int *image_size);
+#endif /* defined(BCMEMBEDIMAGE) && defined (BCM_ROUTER_DHD) */
+
+#ifdef BCM_ROUTER_DHD
+int dbushost_initvars_flash(si_t *sih, osl_t *osh, char **base, uint len);
+#endif
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+static void dhdpci_bus_rte_log_time_sync_poll(dhd_bus_t *bus);
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+#define PCI_VENDOR_ID_BROADCOM          0x14e4
+#define PCI_VENDOR_ID_SYNAPTICS         0x1e01
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#define MAX_D3_ACK_TIMEOUT	100
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef BCMQT
+#define DHD_DEFAULT_DOORBELL_TIMEOUT 40	/* ms */
+#else
+#define DHD_DEFAULT_DOORBELL_TIMEOUT 200	/* ms */
+#endif
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+static uint dhd_doorbell_timeout = DHD_DEFAULT_DOORBELL_TIMEOUT;
+#endif /* PCIE_OOB || PCIE_INB_DW */
+
+static bool dhdpcie_check_firmware_compatible(uint32 f_api_version, uint32 h_api_version);
+static int dhdpcie_cto_error_recovery(struct dhd_bus *bus);
+
+static int dhdpcie_init_d11status(struct dhd_bus *bus);
+
+static int dhdpcie_wrt_rnd(struct dhd_bus *bus);
+
+#if defined(linux) || defined(LINUX)
+#if !defined(DHD_FW_MEM_CORRUPTION) && defined(BCM4375_CHIP)
+#define NUM_PATTERNS 6
+#else
+#define NUM_PATTERNS 2
+#endif /* !DHD_FW_MEM_CORRUPTION && BCM4375_CHIP */
+static bool dhd_bus_tcm_test(struct dhd_bus *bus);
+#endif /* LINUX || linux */
+
+#if defined(FW_SIGNATURE)
+static int dhd_bus_dump_fws(dhd_bus_t *bus, struct bcmstrbuf *strbuf);
+#endif
+static void dhdpcie_pme_stat_clear(dhd_bus_t *bus);
+
+static int dhdpcie_download_rtlv(dhd_bus_t *bus, dngl_rtlv_type_t type, dngl_rtlv_len_t len,
+	uint8 *value);
+#if defined(SUPPORT_MULTIPLE_BOARD_REVISION)
+extern void concate_custom_board_revision(char *nv_path);
+#endif /* SUPPORT_MULTIPLE_BOARD_REVISION */
+
+/* IOVar table */
+enum {
+	IOV_INTR = 1,
+#ifdef DHD_BUS_MEM_ACCESS
+	IOV_MEMBYTES,
+#endif /* DHD_BUS_MEM_ACCESS */
+	IOV_MEMSIZE,
+	IOV_SET_DOWNLOAD_STATE,
+	IOV_SET_DOWNLOAD_INFO,
+	IOV_DEVRESET,
+	IOV_VARS,
+	IOV_MSI_SIM,
+	IOV_PCIE_LPBK,
+	IOV_CC_NVMSHADOW,
+	IOV_RAMSIZE,
+	IOV_RAMSTART,
+	IOV_SLEEP_ALLOWED,
+	IOV_PCIE_DMAXFER,
+	IOV_PCIE_SUSPEND,
+#ifdef DHD_PCIE_REG_ACCESS
+	IOV_PCIEREG,
+	IOV_PCIECFGREG,
+	IOV_PCIECOREREG,
+	IOV_PCIESERDESREG,
+	IOV_PCIEASPM,
+	IOV_BAR0_SECWIN_REG,
+	IOV_SBREG,
+#endif /* DHD_PCIE_REG_ACCESS */
+	IOV_DONGLEISOLATION,
+	IOV_LTRSLEEPON_UNLOOAD,
+	IOV_METADATA_DBG,
+	IOV_RX_METADATALEN,
+	IOV_TX_METADATALEN,
+	IOV_TXP_THRESHOLD,
+	IOV_BUZZZ_DUMP,
+	IOV_DUMP_RINGUPD_BLOCK,
+	IOV_DMA_RINGINDICES,
+	IOV_FORCE_FW_TRAP,
+	IOV_DB1_FOR_MB,
+	IOV_FLOW_PRIO_MAP,
+	IOV_RX_CPL_POST_BOUND,
+	IOV_TX_CPL_BOUND,
+	IOV_CTRL_CPL_POST_BOUND,
+	IOV_TX_POST_BOUND,
+	IOV_HANGREPORT,
+	IOV_H2D_MAILBOXDATA,
+	IOV_INFORINGS,
+	IOV_H2D_PHASE,
+	IOV_H2D_ENABLE_TRAP_BADPHASE,
+	IOV_RING_SIZE_VER,
+	IOV_H2D_TXPOST_MAX_ITEM,
+	IOV_H2D_HTPUT_TXPOST_MAX_ITEM,
+	IOV_D2H_TXCPL_MAX_ITEM,
+	IOV_H2D_RXPOST_MAX_ITEM,
+	IOV_D2H_RXCPL_MAX_ITEM,
+	IOV_H2D_CTRLPOST_MAX_ITEM,
+	IOV_D2H_CTRLCPL_MAX_ITEM,
+	IOV_RX_BUF_BURST,
+	IOV_RX_BUFPOST_THRESHOLD,
+	IOV_TRAPDATA,
+	IOV_TRAPDATA_RAW,
+	IOV_CTO_PREVENTION,
+#ifdef PCIE_OOB
+	IOV_OOB_BT_REG_ON,
+	IOV_OOB_ENABLE,
+#endif /* PCIE_OOB */
+#ifdef DEVICE_TX_STUCK_DETECT
+	IOV_DEVICE_TX_STUCK_DETECT,
+#endif /* DEVICE_TX_STUCK_DETECT */
+	IOV_PCIE_WD_RESET,
+	IOV_DUMP_DONGLE,
+	IOV_IDMA_ENABLE,
+	IOV_IFRM_ENABLE,
+	IOV_CLEAR_RING,
+	IOV_DAR_ENABLE,
+	IOV_DHD_CAPS,   /**< returns string with dhd capabilities */
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+	IOV_GDB_SERVER,  /**< starts gdb server on given interface */
+#endif /* DEBUGGER || DHD_DSCOPE */
+#if defined(GDB_PROXY)
+	IOV_GDB_PROXY_PROBE, /**< gdb proxy support presence check */
+	IOV_GDB_PROXY_STOP_COUNT, /**< gdb proxy firmware stop count */
+#endif /* GDB_PROXY */
+	IOV_INB_DW_ENABLE,
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	IOV_DEEP_SLEEP,
+#endif /* PCIE_OOB || PCIE_INB_DW */
+	IOV_CTO_THRESHOLD,
+#ifdef D2H_MINIDUMP
+	IOV_MINIDUMP_OVERRIDE,
+#endif /* D2H_MINIDUMP */
+	IOV_HSCBSIZE, /* get HSCB buffer size */
+#ifdef DHD_BUS_MEM_ACCESS
+	IOV_HSCBBYTES, /* copy HSCB buffer */
+#endif
+	IOV_HP2P_ENABLE,
+	IOV_HP2P_PKT_THRESHOLD,
+	IOV_HP2P_TIME_THRESHOLD,
+	IOV_HP2P_PKT_EXPIRY,
+	IOV_HP2P_TXCPL_MAXITEMS,
+	IOV_HP2P_RXCPL_MAXITEMS,
+	IOV_EXTDTXS_IN_TXCPL,
+	IOV_HOSTRDY_AFTER_INIT,
+	IOV_HP2P_MF_ENABLE,
+
+	IOV_DONGLE_RXLAT_INFO,
+	IOV_DONGLE_RXLAT_HISTO,
+	IOV_DONGLE_TXCPL_INFO,
+	IOV_DONGLE_SEC_INFO,
+	IOV_DONGLE_SBOOT_DIS,
+	IOV_LPM_MODE,
+	IOV_MDRING_LINK,
+	IOV_MDRING_UNLINK,
+	IOV_MDRING_DUMP,
+	IOV_PTM_ENABLE,
+	IOV_PCIE_DMAXFER_PTRN,
+
+	IOV_PCIE_LAST /**< unused IOVAR */
+};
+
+const bcm_iovar_t dhdpcie_iovars[] = {
+	{"intr",	IOV_INTR,	0, 	0, IOVT_BOOL,	0 },
+#ifdef DHD_BUS_MEM_ACCESS
+	{"membytes",	IOV_MEMBYTES,	0, 	0, IOVT_BUFFER,	2 * sizeof(int) },
+#endif /* DHD_BUS_MEM_ACCESS */
+	{"memsize",	IOV_MEMSIZE,	0, 	0, IOVT_UINT32,	0 },
+	{"dwnldstate",	IOV_SET_DOWNLOAD_STATE,	0, 	0, IOVT_BOOL,	0 },
+	{"dwnldinfo",	IOV_SET_DOWNLOAD_INFO,	0, 	0, IOVT_BUFFER,
+	sizeof(fw_download_info_t) },
+	{"vars",	IOV_VARS,	0, 	0, IOVT_BUFFER,	0 },
+	{"devreset",	IOV_DEVRESET,	0, 	0, IOVT_UINT8,	0 },
+	{"pcie_device_trap", IOV_FORCE_FW_TRAP, 0, 	0, 0,	0 },
+	{"pcie_lpbk",	IOV_PCIE_LPBK,	0,	0, IOVT_UINT32,	0 },
+	{"cc_nvmshadow", IOV_CC_NVMSHADOW, 0,	0, IOVT_BUFFER, 0 },
+	{"ramsize",	IOV_RAMSIZE,	0, 	0, IOVT_UINT32,	0 },
+	{"ramstart",	IOV_RAMSTART,	0, 	0, IOVT_UINT32,	0 },
+#ifdef DHD_PCIE_REG_ACCESS
+	{"pciereg",	IOV_PCIEREG,	0, 	0, IOVT_BUFFER,	2 * sizeof(int32) },
+	{"pciecfgreg",	IOV_PCIECFGREG,	DHD_IOVF_PWRREQ_BYPASS,	0, IOVT_BUFFER,	2 * sizeof(int32) },
+	{"pciecorereg",	IOV_PCIECOREREG,	0, 	0, IOVT_BUFFER,	2 * sizeof(int32) },
+	{"pcieserdesreg",	IOV_PCIESERDESREG,	0, 	0, IOVT_BUFFER,	3 * sizeof(int32) },
+	{"bar0secwinreg",	IOV_BAR0_SECWIN_REG,	0, 	0, IOVT_BUFFER,	sizeof(sdreg_t) },
+	{"sbreg",	IOV_SBREG,	0,	0, IOVT_BUFFER,	sizeof(uint8) },
+#endif /* DHD_PCIE_REG_ACCESS */
+	{"pcie_dmaxfer", IOV_PCIE_DMAXFER, 0, 0, IOVT_BUFFER, sizeof(dma_xfer_info_t)},
+	{"lpbk_dmaxfer_data_pattern", IOV_PCIE_DMAXFER_PTRN, 0, 0, IOVT_UINT32, 0},
+	{"pcie_suspend", IOV_PCIE_SUSPEND,	DHD_IOVF_PWRREQ_BYPASS,	0, IOVT_UINT32,	0 },
+#ifdef PCIE_OOB
+	{"oob_bt_reg_on", IOV_OOB_BT_REG_ON,    0,	0, IOVT_UINT32,    0 },
+	{"oob_enable",   IOV_OOB_ENABLE,    0,	0, IOVT_UINT32,    0 },
+#endif /* PCIE_OOB */
+	{"sleep_allowed",	IOV_SLEEP_ALLOWED,	0,	0, IOVT_BOOL,	0 },
+	{"dngl_isolation", IOV_DONGLEISOLATION,	0, 	0, IOVT_UINT32,	0 },
+	{"ltrsleep_on_unload", IOV_LTRSLEEPON_UNLOOAD,	0,	0, IOVT_UINT32,	0 },
+	{"dump_ringupdblk", IOV_DUMP_RINGUPD_BLOCK,	0, 	0, IOVT_BUFFER,	0 },
+	{"dma_ring_indices", IOV_DMA_RINGINDICES,	0, 	0, IOVT_UINT32,	0},
+	{"metadata_dbg", IOV_METADATA_DBG,	0,	0, IOVT_BOOL,	0 },
+	{"rx_metadata_len", IOV_RX_METADATALEN,	0, 	0, IOVT_UINT32,	0 },
+	{"tx_metadata_len", IOV_TX_METADATALEN,	0, 	0, IOVT_UINT32,	0 },
+	{"db1_for_mb", IOV_DB1_FOR_MB,	0, 	0, IOVT_UINT32,	0 },
+	{"txp_thresh", IOV_TXP_THRESHOLD,	0,	0, IOVT_UINT32,	0 },
+	{"buzzz_dump", IOV_BUZZZ_DUMP,		0, 	0, IOVT_UINT32,	0 },
+	{"flow_prio_map", IOV_FLOW_PRIO_MAP,	0, 	0, IOVT_UINT32,	0 },
+	{"rx_cpl_post_bound",     IOV_RX_CPL_POST_BOUND,    0, 0,	IOVT_UINT32,    0 },
+	{"tx_cpl_bound",     IOV_TX_CPL_BOUND,    0, 0,	IOVT_UINT32,    0 },
+	{"ctrl_cpl_post_bound",     IOV_CTRL_CPL_POST_BOUND,    0, 0,	IOVT_UINT32,    0 },
+	{"tx_post_bound",     IOV_TX_POST_BOUND,    0, 0,	IOVT_UINT32,    0 },
+#ifdef DHD_PCIE_REG_ACCESS
+	{"aspm", IOV_PCIEASPM, 0, 0, IOVT_INT32, 0 },
+#endif /* DHD_PCIE_REG_ACCESS */
+	{"fw_hang_report", IOV_HANGREPORT,	0, 0,	IOVT_BOOL,	0 },
+	{"h2d_mb_data",     IOV_H2D_MAILBOXDATA,    0, 0,      IOVT_UINT32,    0 },
+	{"inforings",   IOV_INFORINGS,    0, 0,      IOVT_UINT32,    0 },
+	{"h2d_phase",   IOV_H2D_PHASE,    0, 0,      IOVT_UINT32,    0 },
+	{"force_trap_bad_h2d_phase", IOV_H2D_ENABLE_TRAP_BADPHASE,    0, 0,
+	IOVT_UINT32,    0 },
+	{"ring_size_ver",   IOV_RING_SIZE_VER,    0, 0,      IOVT_UINT32,    0 },
+	{"h2d_max_txpost",   IOV_H2D_TXPOST_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"h2d_htput_max_txpost", IOV_H2D_HTPUT_TXPOST_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"d2h_max_txcpl",   IOV_D2H_TXCPL_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"h2d_max_rxpost",   IOV_H2D_RXPOST_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"d2h_max_rxcpl",   IOV_D2H_RXCPL_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"h2d_max_ctrlpost",   IOV_H2D_CTRLPOST_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"d2h_max_ctrlcpl",   IOV_D2H_CTRLCPL_MAX_ITEM,    0, 0,      IOVT_UINT32,    0 },
+	{"rx_buf_burst",   IOV_RX_BUF_BURST,    0, 0,      IOVT_UINT32,    0 },
+	{"rx_bufpost_threshold",   IOV_RX_BUFPOST_THRESHOLD,    0, 0,      IOVT_UINT32,    0 },
+	{"trap_data",	IOV_TRAPDATA,	0, 0,	IOVT_BUFFER,	0 },
+	{"trap_data_raw",	IOV_TRAPDATA_RAW,	0, 0,	IOVT_BUFFER,	0 },
+	{"cto_prevention",	IOV_CTO_PREVENTION,	0, 0,	IOVT_UINT32,	0 },
+	{"pcie_wd_reset",	IOV_PCIE_WD_RESET,	0,	0, IOVT_BOOL,	0 },
+#ifdef DEVICE_TX_STUCK_DETECT
+	{"dev_tx_stuck_monitor", IOV_DEVICE_TX_STUCK_DETECT, 0, 0, IOVT_UINT32, 0 },
+#endif /* DEVICE_TX_STUCK_DETECT */
+	{"dump_dongle", IOV_DUMP_DONGLE, 0, 0, IOVT_BUFFER,
+	MAX(sizeof(dump_dongle_in_t), sizeof(dump_dongle_out_t))},
+	{"clear_ring",   IOV_CLEAR_RING,    0, 0,  IOVT_UINT32,    0 },
+	{"idma_enable",   IOV_IDMA_ENABLE,    0, 0,  IOVT_UINT32,    0 },
+	{"ifrm_enable",   IOV_IFRM_ENABLE,    0, 0,  IOVT_UINT32,    0 },
+	{"dar_enable",   IOV_DAR_ENABLE,    0, 0,  IOVT_UINT32,    0 },
+	{"cap", IOV_DHD_CAPS,	0, 0, IOVT_BUFFER,	0},
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+	{"gdb_server", IOV_GDB_SERVER,    0, 0,      IOVT_UINT32,    0 },
+#endif /* DEBUGGER || DHD_DSCOPE */
+#if defined(GDB_PROXY)
+	{"gdb_proxy_probe", IOV_GDB_PROXY_PROBE, 0, 0, IOVT_BUFFER, 2 * sizeof(int32) },
+	{"gdb_proxy_stop_count", IOV_GDB_PROXY_STOP_COUNT, 0, 0, IOVT_UINT32, 0 },
+#endif /* GDB_PROXY */
+	{"inb_dw_enable",   IOV_INB_DW_ENABLE,    0, 0,  IOVT_UINT32,    0 },
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	{"deep_sleep", IOV_DEEP_SLEEP, 0, 0, IOVT_UINT32,    0},
+#endif /* PCIE_OOB || PCIE_INB_DW */
+	{"cto_threshold",	IOV_CTO_THRESHOLD,	0,	0, IOVT_UINT32,	0 },
+#ifdef D2H_MINIDUMP
+	{"minidump_override", IOV_MINIDUMP_OVERRIDE, 0, 0, IOVT_UINT32, 0 },
+#endif /* D2H_MINIDUMP */
+	{"hscbsize",	IOV_HSCBSIZE,	0,	0,	IOVT_UINT32,	0 },
+#ifdef DHD_BUS_MEM_ACCESS
+	{"hscbbytes",	IOV_HSCBBYTES,	0,	0,	IOVT_BUFFER,	2 * sizeof(int32) },
+#endif
+
+	{"extdtxs_in_txcpl", IOV_EXTDTXS_IN_TXCPL,	0,	0, IOVT_UINT32,	0 },
+	{"hostrdy_after_init", IOV_HOSTRDY_AFTER_INIT,	0,	0, IOVT_UINT32,	0 },
+	{"hp2p_mf_enable", IOV_HP2P_MF_ENABLE,	0,	0, IOVT_UINT32,	0 },
+
+	{"dump_rxlat", IOV_DONGLE_RXLAT_INFO, 0, 0, IOVT_BUFFER,
+	sizeof(rx_cpl_history_t) * MAX_RXCPL_HISTORY + 128 },
+	{"dump_rxlat_hist", IOV_DONGLE_RXLAT_HISTO, 0, 0, IOVT_BUFFER,
+	(MAX_RX_LAT_PRIO * MAX_RX_LAT_HIST_BIN * 2) + 128 },
+	{"lpm_mode",	IOV_LPM_MODE,	0,      0,	IOVT_UINT32,	0 },
+
+	{"dump_txcpl", IOV_DONGLE_TXCPL_INFO, 0, 0, IOVT_BUFFER,
+	sizeof(tx_cpl_history_t) * MAX_TXCPL_HISTORY + 128 },
+
+	{"security_status", IOV_DONGLE_SEC_INFO, 0, 0, IOVT_BUFFER,
+	sizeof(tx_cpl_history_t) * MAX_TXCPL_HISTORY + 128 },
+
+	{"sboot_disable", IOV_DONGLE_SBOOT_DIS, 0, 0, IOVT_BUFFER,
+	sizeof(tx_cpl_history_t) * MAX_TXCPL_HISTORY + 128 },
+
+	{"mdring_link", IOV_MDRING_LINK,	0,	0, IOVT_UINT32,	0 },
+	{"mdring_unlink", IOV_MDRING_UNLINK,	0,	0, IOVT_UINT32,	0 },
+	{"dump_mdring", IOV_MDRING_DUMP,	0,	0, IOVT_BUFFER,
+	MAX_MDRING_ITEM_DUMP * D2HRING_MDCMPLT_ITEMSIZE },
+	{"ptm_enable", IOV_PTM_ENABLE,	0,	0, IOVT_UINT32,	0 },
+
+	{NULL, 0, 0, 0, 0, 0 }
+};
+
+#ifdef BCMQT_HW
+#define MAX_READ_TIMEOUT	100 * 1000	/* 100 ms in dongle time */
+#elif defined(NDIS)
+#define MAX_READ_TIMEOUT	5 * 1000 * 1000
+#else
+#define MAX_READ_TIMEOUT	2 * 1000 * 1000
+#endif
+
+#define DHD_INFORING_BOUND	32
+#define DHD_BTLOGRING_BOUND	32
+
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+/** the GDB debugger layer will call back into this (bus) layer to read/write dongle memory */
+static struct dhd_gdb_bus_ops_s  bus_ops = {
+	.read_u16 = dhdpcie_bus_rtcm16,
+	.read_u32 = dhdpcie_bus_rtcm32,
+	.write_u32 = dhdpcie_bus_wtcm32,
+};
+#endif /* DEBUGGER || DHD_DSCOPE */
+
+bool
+dhd_bus_get_flr_force_fail(struct dhd_bus *bus)
+{
+	return bus->flr_force_fail;
+}
+
+/**
+ * Register/Unregister functions are called by the main DHD entry point (eg module insertion) to
+ * link with the bus driver, in order to look for or await the device.
+ */
+int
+dhd_bus_register(void)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	return dhdpcie_bus_register();
+}
+
+void
+dhd_bus_unregister(void)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	dhdpcie_bus_unregister();
+	return;
+}
+
+/** returns a host virtual address */
+uint32 *
+dhdpcie_bus_reg_map(osl_t *osh, ulong addr, int size)
+{
+	return (uint32 *)REG_MAP(addr, size);
+}
+
+void
+dhdpcie_bus_reg_unmap(osl_t *osh, volatile char *addr, int size)
+{
+	REG_UNMAP(addr);
+	return;
+}
+
+/**
+ * retrun H2D Doorbell registers address
+ * use DAR registers instead of enum register for corerev >= 23 (4347B0)
+ */
+static INLINE uint
+dhd_bus_db0_addr_get(struct dhd_bus *bus)
+{
+	uint addr = PCIH2D_MailBox;
+	uint dar_addr = DAR_PCIH2D_DB0_0(bus->sih->buscorerev);
+
+	/* Note: For DAR doorbell registers, the values is invalid. */
+	return ((DAR_ACTIVE(bus->dhd)) ? dar_addr : addr);
+}
+
+static INLINE uint
+dhd_bus_db0_addr_2_get(struct dhd_bus *bus)
+{
+	/* Note: For DAR doorbell registers, the values is invalid. */
+	return ((DAR_ACTIVE(bus->dhd)) ? DAR_PCIH2D_DB2_0(bus->sih->buscorerev) : PCIH2D_MailBox_2);
+}
+
+static INLINE uint
+dhd_bus_db1_addr_get(struct dhd_bus *bus)
+{
+	/* Note: For DB1, don't use DAR register */
+	return PCIH2D_DB1;
+}
+
+static INLINE uint
+dhd_bus_db1_addr_3_get(struct dhd_bus *bus)
+{
+	/* Note: For DB7, don't use DAR register */
+	return PCIH2D_DB1_3;
+}
+
+static void
+dhd_init_pwr_req_lock(dhd_bus_t *bus)
+{
+	if (!bus->pwr_req_lock) {
+		bus->pwr_req_lock = osl_spin_lock_init(bus->osh);
+	}
+}
+
+static void
+dhd_deinit_pwr_req_lock(dhd_bus_t *bus)
+{
+	if (bus->pwr_req_lock) {
+		osl_spin_lock_deinit(bus->osh, bus->pwr_req_lock);
+		bus->pwr_req_lock = NULL;
+	}
+}
+
+#ifdef PCIE_INB_DW
+void
+dhdpcie_set_dongle_deepsleep(dhd_bus_t *bus, bool val)
+{
+	ulong flags_ds;
+	if (INBAND_DW_ENAB(bus)) {
+		DHD_BUS_DONGLE_DS_LOCK(bus->dongle_ds_lock, flags_ds);
+		bus->dongle_in_deepsleep = val;
+		DHD_BUS_DONGLE_DS_UNLOCK(bus->dongle_ds_lock, flags_ds);
+	}
+}
+void
+dhd_init_dongle_ds_lock(dhd_bus_t *bus)
+{
+	if (!bus->dongle_ds_lock) {
+		bus->dongle_ds_lock = osl_spin_lock_init(bus->osh);
+	}
+}
+void
+dhd_deinit_dongle_ds_lock(dhd_bus_t *bus)
+{
+	if (bus->dongle_ds_lock) {
+		osl_spin_lock_deinit(bus->osh, bus->dongle_ds_lock);
+		bus->dongle_ds_lock = NULL;
+	}
+}
+#endif /* PCIE_INB_DW */
+
+/*
+ * WAR for SWWLAN-215055 - [4378B0] ARM fails to boot without DAR WL domain request
+ */
+static INLINE void
+dhd_bus_pcie_pwr_req_wl_domain(struct dhd_bus *bus, uint offset, bool enable)
+{
+	if (enable) {
+		si_corereg(bus->sih, bus->sih->buscoreidx, offset,
+			SRPWR_DMN1_ARMBPSD_MASK << SRPWR_REQON_SHIFT,
+			SRPWR_DMN1_ARMBPSD_MASK << SRPWR_REQON_SHIFT);
+	} else {
+		si_corereg(bus->sih, bus->sih->buscoreidx, offset,
+			SRPWR_DMN1_ARMBPSD_MASK << SRPWR_REQON_SHIFT, 0);
+	}
+}
+
+static INLINE void
+_dhd_bus_pcie_pwr_req_clear_cmn(struct dhd_bus *bus)
+{
+	uint mask;
+
+	/*
+	 * If multiple de-asserts, decrement ref and return
+	 * Clear power request when only one pending
+	 * so initial request is not removed unexpectedly
+	 */
+	if (bus->pwr_req_ref > 1) {
+		bus->pwr_req_ref--;
+		return;
+	}
+
+	ASSERT(bus->pwr_req_ref == 1);
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		/* Common BP controlled by HW so only need to toggle WL/ARM backplane */
+		mask = SRPWR_DMN1_ARMBPSD_MASK;
+	} else {
+		mask = SRPWR_DMN0_PCIE_MASK | SRPWR_DMN1_ARMBPSD_MASK;
+	}
+
+	si_srpwr_request(bus->sih, mask, 0);
+	bus->pwr_req_ref = 0;
+}
+
+static INLINE void
+dhd_bus_pcie_pwr_req_clear(struct dhd_bus *bus)
+{
+	unsigned long flags = 0;
+
+	DHD_BUS_PWR_REQ_LOCK(bus->pwr_req_lock, flags);
+	_dhd_bus_pcie_pwr_req_clear_cmn(bus);
+	DHD_BUS_PWR_REQ_UNLOCK(bus->pwr_req_lock, flags);
+}
+
+static INLINE void
+dhd_bus_pcie_pwr_req_clear_nolock(struct dhd_bus *bus)
+{
+	_dhd_bus_pcie_pwr_req_clear_cmn(bus);
+}
+
+static INLINE void
+_dhd_bus_pcie_pwr_req_cmn(struct dhd_bus *bus)
+{
+	uint mask, val;
+
+	/* If multiple request entries, increment reference and return */
+	if (bus->pwr_req_ref > 0) {
+		bus->pwr_req_ref++;
+		return;
+	}
+
+	ASSERT(bus->pwr_req_ref == 0);
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		/* Common BP controlled by HW so only need to toggle WL/ARM backplane */
+		mask = SRPWR_DMN1_ARMBPSD_MASK;
+		val = SRPWR_DMN1_ARMBPSD_MASK;
+	} else {
+		mask = SRPWR_DMN0_PCIE_MASK | SRPWR_DMN1_ARMBPSD_MASK;
+		val = SRPWR_DMN0_PCIE_MASK | SRPWR_DMN1_ARMBPSD_MASK;
+	}
+
+	si_srpwr_request(bus->sih, mask, val);
+
+	bus->pwr_req_ref = 1;
+}
+
+static INLINE void
+dhd_bus_pcie_pwr_req(struct dhd_bus *bus)
+{
+	unsigned long flags = 0;
+
+	DHD_BUS_PWR_REQ_LOCK(bus->pwr_req_lock, flags);
+	_dhd_bus_pcie_pwr_req_cmn(bus);
+	DHD_BUS_PWR_REQ_UNLOCK(bus->pwr_req_lock, flags);
+}
+
+static INLINE void
+_dhd_bus_pcie_pwr_req_pd0123_cmn(struct dhd_bus *bus)
+{
+	uint mask, val;
+
+	mask = SRPWR_DMN_ALL_MASK(bus->sih);
+	val = SRPWR_DMN_ALL_MASK(bus->sih);
+
+	si_srpwr_request(bus->sih, mask, val);
+}
+
+void
+dhd_bus_pcie_pwr_req_reload_war(struct dhd_bus *bus)
+{
+	unsigned long flags = 0;
+
+	/*
+	 * Few corerevs need the power domain to be active for FLR.
+	 * Return if the pwr req is not applicable for the corerev
+	 */
+	if (!(PCIE_PWR_REQ_RELOAD_WAR_ENAB(bus->sih->buscorerev))) {
+		return;
+	}
+
+	DHD_BUS_PWR_REQ_LOCK(bus->pwr_req_lock, flags);
+	_dhd_bus_pcie_pwr_req_pd0123_cmn(bus);
+	DHD_BUS_PWR_REQ_UNLOCK(bus->pwr_req_lock, flags);
+}
+
+static INLINE void
+_dhd_bus_pcie_pwr_req_clear_pd0123_cmn(struct dhd_bus *bus)
+{
+	uint mask;
+
+	mask = SRPWR_DMN_ALL_MASK(bus->sih);
+
+	si_srpwr_request(bus->sih, mask, 0);
+}
+
+void
+dhd_bus_pcie_pwr_req_clear_reload_war(struct dhd_bus *bus)
+{
+	unsigned long flags = 0;
+
+	/* return if the pwr clear is not applicable for the corerev */
+	if (!(PCIE_PWR_REQ_RELOAD_WAR_ENAB(bus->sih->buscorerev))) {
+		return;
+	}
+	DHD_BUS_PWR_REQ_LOCK(bus->pwr_req_lock, flags);
+	_dhd_bus_pcie_pwr_req_clear_pd0123_cmn(bus);
+	DHD_BUS_PWR_REQ_UNLOCK(bus->pwr_req_lock, flags);
+}
+
+static INLINE void
+dhd_bus_pcie_pwr_req_nolock(struct dhd_bus *bus)
+{
+	_dhd_bus_pcie_pwr_req_cmn(bus);
+}
+
+bool
+dhdpcie_chip_support_msi(dhd_bus_t *bus)
+{
+	/* XXX For chips with buscorerev <= 14 intstatus
+	 * is not getting cleared from these firmwares.
+	 * Either host can read and clear intstatus for these
+	 * or not enable MSI at all.
+	 * Here option 2 of not enabling MSI is choosen.
+	 * Also for hw4 chips, msi is not enabled.
+	 */
+	DHD_INFO(("%s: buscorerev=%d chipid=0x%x\n",
+		__FUNCTION__, bus->sih->buscorerev, si_chipid(bus->sih)));
+	if (bus->sih->buscorerev <= 14 ||
+		si_chipid(bus->sih) == BCM4381_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4382_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4385_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4375_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4376_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4362_CHIP_ID ||
+		si_chipid(bus->sih) == BCM43751_CHIP_ID ||
+		si_chipid(bus->sih) == BCM43752_CHIP_ID ||
+		si_chipid(bus->sih) == BCM43756_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4361_CHIP_ID ||
+		si_chipid(bus->sih) == BCM4359_CHIP_ID) {
+		return FALSE;
+	} else {
+		return TRUE;
+	}
+}
+
+/**
+ * Called once for each hardware (dongle) instance that this DHD manages.
+ *
+ * 'regs' is the host virtual address that maps to the start of the PCIe BAR0 window. The first 4096
+ * bytes in this window are mapped to the backplane address in the PCIEBAR0Window register. The
+ * precondition is that the PCIEBAR0Window register 'points' at the PCIe core.
+ *
+ * 'tcm' is the *host* virtual address at which tcm is mapped.
+ */
+int dhdpcie_bus_attach(osl_t *osh, dhd_bus_t **bus_ptr,
+	volatile char *regs, volatile char *tcm, void *pci_dev, wifi_adapter_info_t *adapter)
+{
+	dhd_bus_t *bus = NULL;
+	int ret = BCME_OK;
+
+	DHD_TRACE(("%s: ENTER\n", __FUNCTION__));
+
+	do {
+		if (!(bus = MALLOCZ(osh, sizeof(dhd_bus_t)))) {
+			DHD_ERROR(("%s: MALLOC of dhd_bus_t failed\n", __FUNCTION__));
+			ret = BCME_NORESOURCE;
+			break;
+		}
+		bus->bus = adapter->bus_type;
+		bus->bus_num = adapter->bus_num;
+		bus->slot_num = adapter->slot_num;
+
+		bus->regs = regs;
+		bus->tcm = tcm;
+		bus->osh = osh;
+#ifndef NDIS
+		/* Save pci_dev into dhd_bus, as it may be needed in dhd_attach */
+		bus->dev = (struct pci_dev *)pci_dev;
+#endif
+
+		dll_init(&bus->flowring_active_list);
+#ifdef IDLE_TX_FLOW_MGMT
+		bus->active_list_last_process_ts = OSL_SYSUPTIME();
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#ifdef DEVICE_TX_STUCK_DETECT
+		/* Enable the Device stuck detection feature by default */
+		bus->dev_tx_stuck_monitor = TRUE;
+		bus->device_tx_stuck_check = OSL_SYSUPTIME();
+#endif /* DEVICE_TX_STUCK_DETECT */
+
+		/* Attach pcie shared structure */
+		if (!(bus->pcie_sh = MALLOCZ(osh, sizeof(pciedev_shared_t)))) {
+			DHD_ERROR(("%s: MALLOC of bus->pcie_sh failed\n", __FUNCTION__));
+			ret = BCME_NORESOURCE;
+			break;
+		}
+
+		/* dhd_common_init(osh); */
+
+		if (dhdpcie_dongle_attach(bus)) {
+			DHD_ERROR(("%s: dhdpcie_probe_attach failed\n", __FUNCTION__));
+			ret = BCME_NOTREADY;
+			break;
+		}
+
+		/* software resources */
+		if (!(bus->dhd = dhd_attach(osh, bus, PCMSGBUF_HDRLEN))) {
+			DHD_ERROR(("%s: dhd_attach failed\n", __FUNCTION__));
+			ret = BCME_NORESOURCE;
+			break;
+		}
+#if defined(GET_OTP_MAC_ENABLE) || defined(GET_OTP_MODULE_NAME)
+		dhd_conf_get_otp(bus->dhd, bus->sih);
+#endif
+		DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+		bus->dhd->busstate = DHD_BUS_DOWN;
+		bus->dhd->hostrdy_after_init = TRUE;
+		bus->db1_for_mb = TRUE;
+		bus->dhd->hang_report = TRUE;
+		bus->use_mailbox = FALSE;
+		bus->use_d0_inform = FALSE;
+		bus->intr_enabled = FALSE;
+		bus->flr_force_fail = FALSE;
+		/* update the dma indices if set through module parameter. */
+		if (dma_ring_indices != 0) {
+			dhdpcie_set_dma_ring_indices(bus->dhd, dma_ring_indices);
+		}
+		/* update h2d phase support if set through module parameter */
+		bus->dhd->h2d_phase_supported = h2d_phase ? TRUE : FALSE;
+		/* update force trap on bad phase if set through module parameter */
+		bus->dhd->force_dongletrap_on_bad_h2d_phase =
+			force_trap_bad_h2d_phase ? TRUE : FALSE;
+#ifdef IDLE_TX_FLOW_MGMT
+		bus->enable_idle_flowring_mgmt = FALSE;
+#endif /* IDLE_TX_FLOW_MGMT */
+		bus->irq_registered = FALSE;
+
+#ifdef DHD_MSI_SUPPORT
+		bus->d2h_intr_method = enable_msi && dhdpcie_chip_support_msi(bus) ?
+			PCIE_MSI : PCIE_INTX;
+		if (bus->dhd->conf->d2h_intr_method >= 0)
+			bus->d2h_intr_method = bus->dhd->conf->d2h_intr_method;
+#else
+		bus->d2h_intr_method = PCIE_INTX;
+#endif /* DHD_MSI_SUPPORT */
+
+#ifdef NDIS
+		/* For NDIS use D2H INTMASK based control */
+		bus->d2h_intr_control = PCIE_D2H_INTMASK_CTRL;
+#else
+		/* For MSI, use host irq based control and for INTX use D2H INTMASK based control */
+		if (bus->d2h_intr_method == PCIE_MSI) {
+			bus->d2h_intr_control = PCIE_HOST_IRQ_CTRL;
+		} else {
+			bus->d2h_intr_control = PCIE_D2H_INTMASK_CTRL;
+		}
+#endif /* NDIS */
+
+		DHD_TRACE(("%s: EXIT SUCCESS\n",
+			__FUNCTION__));
+		g_dhd_bus = bus;
+		*bus_ptr = bus;
+		return ret;
+	} while (0);
+
+	DHD_TRACE(("%s: EXIT FAILURE\n", __FUNCTION__));
+	if (bus && bus->pcie_sh) {
+		MFREE(osh, bus->pcie_sh, sizeof(pciedev_shared_t));
+	}
+
+	if (bus) {
+		MFREE(osh, bus, sizeof(dhd_bus_t));
+	}
+
+	return ret;
+}
+
+bool
+dhd_bus_skip_clm(dhd_pub_t *dhdp)
+{
+	switch (dhd_bus_chip_id(dhdp)) {
+		case BCM4369_CHIP_ID:
+			return TRUE;
+		default:
+			return FALSE;
+	}
+}
+
+uint
+dhd_bus_chip(struct dhd_bus *bus)
+{
+	ASSERT(bus->sih != NULL);
+	return bus->sih->chip;
+}
+
+uint
+dhd_bus_chiprev(struct dhd_bus *bus)
+{
+	ASSERT(bus);
+	ASSERT(bus->sih != NULL);
+	return bus->sih->chiprev;
+}
+
+void *
+dhd_bus_pub(struct dhd_bus *bus)
+{
+	return bus->dhd;
+}
+
+void *
+dhd_bus_sih(struct dhd_bus *bus)
+{
+	return (void *)bus->sih;
+}
+
+void *
+dhd_bus_txq(struct dhd_bus *bus)
+{
+	return &bus->txq;
+}
+
+/** Get Chip ID version */
+uint dhd_bus_chip_id(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	return  bus->sih->chip;
+}
+
+/** Get Chip Rev ID version */
+uint dhd_bus_chiprev_id(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	return bus->sih->chiprev;
+}
+
+/** Get Chip Pkg ID version */
+uint dhd_bus_chippkg_id(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	return bus->sih->chippkg;
+}
+
+int dhd_bus_get_ids(struct dhd_bus *bus, uint32 *bus_type, uint32 *bus_num, uint32 *slot_num)
+{
+	*bus_type = bus->bus;
+	*bus_num = bus->bus_num;
+	*slot_num = bus->slot_num;
+	return 0;
+}
+
+/** Conduct Loopback test */
+int
+dhd_bus_dmaxfer_lpbk(dhd_pub_t *dhdp, uint32 type)
+{
+	dma_xfer_info_t dmaxfer_lpbk;
+	int ret = BCME_OK;
+
+#define PCIE_DMAXFER_LPBK_LENGTH	4096
+	memset(&dmaxfer_lpbk, 0, sizeof(dma_xfer_info_t));
+	dmaxfer_lpbk.version = DHD_DMAXFER_VERSION;
+	dmaxfer_lpbk.length = (uint16)sizeof(dma_xfer_info_t);
+	dmaxfer_lpbk.num_bytes = PCIE_DMAXFER_LPBK_LENGTH;
+	dmaxfer_lpbk.type = type;
+	dmaxfer_lpbk.should_wait = TRUE;
+
+	ret = dhd_bus_iovar_op(dhdp, "pcie_dmaxfer", NULL, 0,
+		(char *)&dmaxfer_lpbk, sizeof(dma_xfer_info_t), IOV_SET);
+	if (ret < 0) {
+		DHD_ERROR(("failed to start PCIe Loopback Test!!! "
+			"Type:%d Reason:%d\n", type, ret));
+		return ret;
+	}
+
+	if (dmaxfer_lpbk.status != DMA_XFER_SUCCESS) {
+		DHD_ERROR(("failed to check PCIe Loopback Test!!! "
+			"Type:%d Status:%d Error code:%d\n", type,
+			dmaxfer_lpbk.status, dmaxfer_lpbk.error_code));
+		ret = BCME_ERROR;
+	} else {
+		DHD_ERROR(("successful to check PCIe Loopback Test"
+			" Type:%d\n", type));
+	}
+#undef PCIE_DMAXFER_LPBK_LENGTH
+
+	return ret;
+}
+
+/* Check if there is DPC scheduling errors */
+bool
+dhd_bus_query_dpc_sched_errors(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	bool sched_err;
+
+	if ((bus->dpc_entry_time < bus->isr_exit_time) && bus->dpc_sched) {
+		/* Kernel doesn't schedule the DPC after processing PCIe IRQ */
+		sched_err = TRUE;
+	} else if (bus->dpc_entry_time < bus->resched_dpc_time) {
+		/* Kernel doesn't schedule the DPC after DHD tries to reschedule
+		 * the DPC due to pending work items to be processed.
+		 */
+		sched_err = TRUE;
+	} else {
+		sched_err = FALSE;
+	}
+
+	if (sched_err) {
+		/* print out minimum timestamp info */
+		DHD_ERROR(("isr_entry_time="SEC_USEC_FMT
+			" prev_isr_entry_time="SEC_USEC_FMT
+			" isr_exit_time="SEC_USEC_FMT
+			" dpc_entry_time="SEC_USEC_FMT
+			"\ndpc_exit_time="SEC_USEC_FMT
+			" isr_sched_dpc_time="SEC_USEC_FMT
+			" resched_dpc_time="SEC_USEC_FMT
+			" dpc_sched=%u\n",
+			GET_SEC_USEC(bus->isr_entry_time),
+			GET_SEC_USEC(bus->prev_isr_entry_time),
+			GET_SEC_USEC(bus->isr_exit_time),
+			GET_SEC_USEC(bus->dpc_entry_time),
+			GET_SEC_USEC(bus->dpc_exit_time),
+			GET_SEC_USEC(bus->isr_sched_dpc_time),
+			GET_SEC_USEC(bus->resched_dpc_time),
+			bus->dpc_sched));
+	}
+
+	return sched_err;
+}
+
+/** Read and clear intstatus. This should be called with interrupts disabled or inside isr */
+uint32
+dhdpcie_bus_intstatus(dhd_bus_t *bus)
+{
+	uint32 intstatus = 0;
+	uint32 intmask = 0;
+
+	if (__DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+		DHD_ERROR(("%s: trying to clear intstatus after D3 Ack\n", __FUNCTION__));
+		return intstatus;
+	}
+	/* XXX: check for PCIE Gen2 also */
+	if ((bus->sih->buscorerev == 6) || (bus->sih->buscorerev == 4) ||
+		(bus->sih->buscorerev == 2)) {
+		intstatus = dhdpcie_bus_cfg_read_dword(bus, PCIIntstatus, 4);
+		dhdpcie_bus_cfg_write_dword(bus, PCIIntstatus, 4, intstatus);
+		intstatus &= I_MB;
+	} else {
+		/* this is a PCIE core register..not a config register... */
+		intstatus = si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_int, 0, 0);
+
+#ifdef DHD_MMIO_TRACE
+		dhd_bus_mmio_trace(bus, bus->pcie_mailbox_int, intstatus, FALSE);
+#endif /* defined(DHD_MMIO_TRACE) */
+
+		/* this is a PCIE core register..not a config register... */
+		intmask = si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_mask, 0, 0);
+		/* Is device removed. intstatus & intmask read 0xffffffff */
+		if (intstatus == (uint32)-1 || intmask == (uint32)-1) {
+			DHD_ERROR(("%s: Device is removed or Link is down.\n", __FUNCTION__));
+			DHD_ERROR(("%s: INTSTAT : 0x%x INTMASK : 0x%x.\n",
+			    __FUNCTION__, intstatus, intmask));
+			bus->is_linkdown = TRUE;
+			dhd_pcie_debug_info_dump(bus->dhd);
+			return intstatus;
+		}
+
+#ifndef DHD_READ_INTSTATUS_IN_DPC
+		intstatus &= intmask;
+#endif /* DHD_READ_INTSTATUS_IN_DPC */
+
+#ifdef DHD_MMIO_TRACE
+		dhd_bus_mmio_trace(bus, bus->pcie_mailbox_mask, intmask, FALSE);
+#endif /* defined(DHD_MMIO_TRACE) */
+
+		/* XXX: define the mask in a .h file  */
+		/*
+		 * The fourth argument to si_corereg is the "mask" fields of the register to update
+		 * and the fifth field is the "value" to update. Now if we are interested in only
+		 * few fields of the "mask" bit map, we should not be writing back what we read
+		 * By doing so, we might clear/ack interrupts that are not handled yet.
+		 */
+#ifdef DHD_MMIO_TRACE
+		dhd_bus_mmio_trace(bus, bus->pcie_mailbox_int, intstatus, TRUE);
+#endif /* defined(DHD_MMIO_TRACE) */
+		si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_int, bus->def_intmask,
+			intstatus);
+
+		intstatus &= bus->def_intmask;
+	}
+
+	return intstatus;
+}
+
+void
+dhdpcie_cto_recovery_handler(dhd_pub_t *dhd)
+{
+	dhd_bus_t *bus = dhd->bus;
+	int ret;
+
+	if (dhd->up == FALSE) {
+		DHD_ERROR(("%s : dhd is not up\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: link is down\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->sih == NULL) {
+		DHD_ERROR(("%s: The address of sih is invalid\n", __FUNCTION__));
+		return;
+	}
+
+	/* Disable PCIe Runtime PM to avoid D3_ACK timeout.
+	 */
+	DHD_DISABLE_RUNTIME_PM(dhd);
+
+	/* Sleep for 1 seconds so that any AXI timeout
+	 * if running on ALP clock also will be captured
+	 */
+	OSL_SLEEP(1000);
+
+	/* reset backplane and cto,
+	 * then access through pcie is recovered.
+	 */
+	ret = dhdpcie_cto_error_recovery(bus);
+	if (!ret) {
+		/* Waiting for backplane reset */
+		OSL_SLEEP(10);
+		/* Dump debug Info */
+		dhd_prot_debug_info_print(bus->dhd);
+		/* Dump console buffer */
+		dhd_bus_dump_console_buffer(bus);
+#if defined(DHD_FW_COREDUMP)
+		/* save core dump or write to a file */
+		if (!bus->is_linkdown && bus->dhd->memdump_enabled) {
+#ifdef DHD_SSSR_DUMP
+			DHD_ERROR(("%s : Set collect_sssr as TRUE\n", __FUNCTION__));
+			bus->dhd->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+			bus->dhd->memdump_type = DUMP_TYPE_CTO_RECOVERY;
+			dhdpcie_mem_dump(bus);
+		}
+#endif /* DHD_FW_COREDUMP */
+	}
+#ifdef OEM_ANDROID
+	bus->is_linkdown = TRUE;
+	bus->dhd->hang_reason = HANG_REASON_PCIE_CTO_DETECT;
+	/* Send HANG event */
+	dhd_os_send_hang_message(bus->dhd);
+#endif /* OEM_ANDROID */
+}
+
+void
+dhd_bus_dump_imp_cfg_registers(struct dhd_bus *bus)
+{
+	uint32 status_cmd = dhd_pcie_config_read(bus, PCIECFGREG_STATUS_CMD, sizeof(uint32));
+	uint32 pmcsr = dhd_pcie_config_read(bus, PCIE_CFG_PMCSR, sizeof(uint32));
+	uint32 base_addr0 = dhd_pcie_config_read(bus, PCIECFGREG_BASEADDR0, sizeof(uint32));
+	uint32 base_addr1 = dhd_pcie_config_read(bus, PCIECFGREG_BASEADDR1, sizeof(uint32));
+	uint32 linkctl = dhd_pcie_config_read(bus, PCIECFGREG_LINK_STATUS_CTRL, sizeof(uint32));
+	uint32 l1ssctrl =
+		dhd_pcie_config_read(bus, PCIECFGREG_PML1_SUB_CTRL1, sizeof(uint32));
+	uint32 devctl = dhd_pcie_config_read(bus, PCIECFGREG_DEV_STATUS_CTRL, sizeof(uint32));
+	uint32 devctl2 = dhd_pcie_config_read(bus, PCIECFGGEN_DEV_STATUS_CTRL2, sizeof(uint32));
+
+	DHD_ERROR(("status_cmd(0x%x)=0x%x, pmcsr(0x%x)=0x%x "
+		"base_addr0(0x%x)=0x%x base_addr1(0x%x)=0x%x "
+		"linkctl(0x%x)=0x%x l1ssctrl(0x%x)=0x%x "
+		"devctl(0x%x)=0x%x devctl2(0x%x)=0x%x \n",
+		PCIECFGREG_STATUS_CMD, status_cmd,
+		PCIE_CFG_PMCSR, pmcsr,
+		PCIECFGREG_BASEADDR0, base_addr0,
+		PCIECFGREG_BASEADDR1, base_addr1,
+		PCIECFGREG_LINK_STATUS_CTRL, linkctl,
+		PCIECFGREG_PML1_SUB_CTRL1, l1ssctrl,
+		PCIECFGREG_DEV_STATUS_CTRL, devctl,
+		PCIECFGGEN_DEV_STATUS_CTRL2, devctl2));
+}
+
+/**
+ * Name:  dhdpcie_bus_isr
+ * Parameters:
+ * 1: IN int irq   -- interrupt vector
+ * 2: IN void *arg      -- handle to private data structure
+ * Return value:
+ * Status (TRUE or FALSE)
+ *
+ * Description:
+ * Interrupt Service routine checks for the status register,
+ * disable interrupt and queue DPC if mail box interrupts are raised.
+ */
+int32
+dhdpcie_bus_isr(dhd_bus_t *bus)
+{
+	uint32 intstatus = 0;
+
+	do {
+		DHD_INTR(("%s: Enter\n", __FUNCTION__));
+		/* verify argument */
+		if (!bus) {
+			DHD_LOG_MEM(("%s : bus is null pointer, exit \n", __FUNCTION__));
+			break;
+		}
+
+		if (bus->dhd->dongle_reset) {
+			DHD_LOG_MEM(("%s : dongle is reset\n", __FUNCTION__));
+			break;
+		}
+
+		if (bus->dhd->busstate == DHD_BUS_DOWN) {
+			DHD_LOG_MEM(("%s : bus is down \n", __FUNCTION__));
+			break;
+		}
+
+		/* avoid processing of interrupts until msgbuf prot is inited */
+		if (!bus->intr_enabled) {
+			DHD_INFO(("%s, not ready to receive interrupts\n", __FUNCTION__));
+			break;
+		}
+
+		/* Do not process any ISR after receiving D3_ACK */
+		if (__DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+			DHD_LOG_MEM(("%s: D3 Ack Received, skip\n", __FUNCTION__));
+			break;
+		}
+
+		if (PCIECTO_ENAB(bus)) {
+			/* read pci_intstatus */
+			intstatus = dhdpcie_bus_cfg_read_dword(bus, PCI_INT_STATUS, 4);
+
+			if (intstatus == (uint32)-1 ||
+				bus->dhd->dhd_induce_error == DHD_INDUCE_PCIE_LINK_DOWN_IN_ISR) {
+				DHD_ERROR(("%s: Invalid cfg intstatus(0x%x):0x%x, pcie link down,"
+					"induce %u\n", __FUNCTION__, PCI_INT_STATUS, intstatus,
+					bus->dhd->dhd_induce_error));
+				bus->is_linkdown = 1;
+				dhdpcie_disable_irq_nosync(bus);
+				dhd_pcie_debug_info_dump(bus->dhd);
+#ifdef OEM_ANDROID
+				bus->dhd->hang_reason = HANG_REASON_PCIE_LINK_DOWN_EP_DETECT;
+				dhd_os_send_hang_message(bus->dhd);
+#endif /* OEM_ANDROID */
+				break;
+			}
+
+			if (intstatus & PCI_CTO_INT_MASK) {
+				DHD_ERROR(("%s: ##### CTO RECOVERY REPORTED BY DONGLE "
+					"intstat=0x%x enab=%d\n", __FUNCTION__,
+					intstatus, bus->cto_enable));
+				bus->cto_triggered = 1;
+				bus->dhd->do_chip_bighammer = TRUE;
+				dhd_bus_dump_imp_cfg_registers(bus);
+				/*
+				 * DAR still accessible
+				 */
+				dhd_bus_dump_dar_registers(bus);
+
+				/* Disable further PCIe interrupts */
+#ifndef NDIS
+				dhdpcie_disable_irq_nosync(bus); /* Disable interrupt!! */
+#endif
+				/* Stop Tx flow */
+				dhd_bus_stop_queue(bus);
+
+				/* Schedule CTO recovery */
+				dhd_schedule_cto_recovery(bus->dhd);
+
+				return TRUE;
+			}
+		}
+
+#ifndef DHD_READ_INTSTATUS_IN_DPC
+		if (bus->d2h_intr_method == PCIE_MSI &&
+				!dhd_conf_legacy_msi_chip(bus->dhd)) {
+			/* For MSI, as intstatus is cleared by firmware, no need to read */
+			goto skip_intstatus_read;
+		}
+
+		intstatus = dhdpcie_bus_intstatus(bus);
+
+		/* Check if the interrupt is ours or not */
+		if (intstatus == 0) {
+			bus->non_ours_irq_count++;
+			bus->last_non_ours_irq_time = OSL_LOCALTIME_NS();
+			break;
+		}
+
+		/* save the intstatus */
+		/* read interrupt status register!! Status bits will be cleared in DPC !! */
+		bus->intstatus = intstatus;
+
+		/* return error for 0xFFFFFFFF */
+		if (intstatus == (uint32)-1) {
+			DHD_LOG_MEM(("%s : wrong interrupt status val : 0x%x\n",
+				__FUNCTION__, intstatus));
+			bus->is_linkdown = 1;
+			dhdpcie_disable_irq_nosync(bus);
+			break;
+		}
+
+skip_intstatus_read:
+		/*  Overall operation:
+		 *    - Mask further interrupts
+		 *    - Read/ack intstatus
+		 *    - Take action based on bits and state
+		 *    - Reenable interrupts (as per state)
+		 */
+
+		/* Count the interrupt call */
+		bus->intrcount++;
+#endif /* DHD_READ_INTSTATUS_IN_DPC */
+
+		bus->ipend = TRUE;
+
+		/* Due to irq mismatch WARNING in linux, currently keeping it disabled and
+		 * using dongle intmask to control INTR enable/disable
+		 */
+		if (bus->d2h_intr_control == PCIE_HOST_IRQ_CTRL) {
+			if (!dhdpcie_irq_disabled(bus)) {
+				bus->host_irq_disable_count++;
+				dhdpcie_disable_irq_nosync(bus); /* Disable interrupt!! */
+			}
+		} else {
+			dhdpcie_bus_intr_disable(bus); /* Disable interrupt using IntMask!! */
+			bus->dngl_intmask_disable_count++;
+		}
+
+		bus->intdis = TRUE;
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+		if (bus->dhd->dma_h2d_ring_upd_support && bus->dhd->dma_d2h_ring_upd_support &&
+			(bus->dhd->ring_attached == TRUE)) {
+			dhd_bus_flow_ring_status_isr_trace(bus->dhd);
+		}
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+#if defined(PCIE_ISR_THREAD)
+		DHD_TRACE(("Calling dhd_bus_dpc() from %s\n", __FUNCTION__));
+		DHD_OS_WAKE_LOCK(bus->dhd);
+		while (dhd_bus_dpc(bus));
+		DHD_OS_WAKE_UNLOCK(bus->dhd);
+#else // defined(PCIE_ISR_THREAD)
+		bus->dpc_sched = TRUE;
+		bus->isr_sched_dpc_time = OSL_LOCALTIME_NS();
+#ifndef NDIS
+		dhd_sched_dpc(bus->dhd);     /* queue DPC now!! */
+#endif /* !NDIS */
+#endif /* defined(PCIE_ISR_THREAD) */
+
+		DHD_INTR(("%s: Exit Success DPC Queued\n", __FUNCTION__));
+		return TRUE;
+
+	} while (0);
+
+	DHD_INTR(("%s: Exit Failure\n", __FUNCTION__));
+	return FALSE;
+}
+
+int
+dhdpcie_set_pwr_state(dhd_bus_t *bus, uint state)
+{
+	uint32 cur_state = 0;
+	uint32 pm_csr = 0;
+	osl_t *osh = bus->osh;
+
+	pm_csr = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_PM_CSR, sizeof(uint32));
+	cur_state = pm_csr & PCIECFGREG_PM_CSR_STATE_MASK;
+
+	if (cur_state == state) {
+		DHD_ERROR(("%s: Already in state %u \n", __FUNCTION__, cur_state));
+		return BCME_OK;
+	}
+
+	if (state > PCIECFGREG_PM_CSR_STATE_D3_HOT)
+		return BCME_ERROR;
+
+	/* Validate the state transition
+	* if already in a lower power state, return error
+	*/
+	if (state != PCIECFGREG_PM_CSR_STATE_D0 &&
+			cur_state <= PCIECFGREG_PM_CSR_STATE_D3_COLD &&
+			cur_state > state) {
+		DHD_ERROR(("%s: Invalid power state transition !\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	pm_csr &= ~PCIECFGREG_PM_CSR_STATE_MASK;
+	pm_csr |= state;
+
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_PM_CSR, sizeof(uint32), pm_csr);
+
+	/* need to wait for the specified mandatory pcie power transition delay time */
+	if (state == PCIECFGREG_PM_CSR_STATE_D3_HOT ||
+			cur_state == PCIECFGREG_PM_CSR_STATE_D3_HOT)
+			OSL_DELAY(DHDPCIE_PM_D3_DELAY);
+	else if (state == PCIECFGREG_PM_CSR_STATE_D2 ||
+			cur_state == PCIECFGREG_PM_CSR_STATE_D2)
+			OSL_DELAY(DHDPCIE_PM_D2_DELAY);
+
+	/* read back the power state and verify */
+	pm_csr = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_PM_CSR, sizeof(uint32));
+	cur_state = pm_csr & PCIECFGREG_PM_CSR_STATE_MASK;
+	if (cur_state != state) {
+		DHD_ERROR(("%s: power transition failed ! Current state is %u \n",
+				__FUNCTION__, cur_state));
+		return BCME_ERROR;
+	} else {
+		DHD_ERROR(("%s: power transition to %u success \n",
+				__FUNCTION__, cur_state));
+	}
+
+	return BCME_OK;
+}
+
+int
+dhdpcie_config_check(dhd_bus_t *bus)
+{
+	uint32 i, val;
+	int ret = BCME_ERROR;
+
+	for (i = 0; i < DHDPCIE_CONFIG_CHECK_RETRY_COUNT; i++) {
+		val = OSL_PCI_READ_CONFIG(bus->osh, PCI_CFG_VID, sizeof(uint32));
+		if (((val & 0xFFFF) == VENDOR_BROADCOM) || ((val & 0xFFFF) == VENDOR_SYNAPTICS)) {
+			ret = BCME_OK;
+			break;
+		}
+		OSL_DELAY(DHDPCIE_CONFIG_CHECK_DELAY_MS * 1000);
+	}
+
+	return ret;
+}
+
+int
+dhdpcie_config_restore(dhd_bus_t *bus, bool restore_pmcsr)
+{
+	uint32 i;
+	osl_t *osh = bus->osh;
+
+	if (BCME_OK != dhdpcie_config_check(bus)) {
+		return BCME_ERROR;
+	}
+
+	for (i = PCI_CFG_REV >> 2; i < DHDPCIE_CONFIG_HDR_SIZE; i++) {
+		OSL_PCI_WRITE_CONFIG(osh, i << 2, sizeof(uint32), bus->saved_config.header[i]);
+	}
+	OSL_PCI_WRITE_CONFIG(osh, PCI_CFG_CMD, sizeof(uint32), bus->saved_config.header[1]);
+
+	if (restore_pmcsr)
+		OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_PM_CSR,
+			sizeof(uint32), bus->saved_config.pmcsr);
+
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_MSI_CAP, sizeof(uint32), bus->saved_config.msi_cap);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_MSI_ADDR_L, sizeof(uint32),
+			bus->saved_config.msi_addr0);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_MSI_ADDR_H,
+			sizeof(uint32), bus->saved_config.msi_addr1);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_MSI_DATA,
+			sizeof(uint32), bus->saved_config.msi_data);
+
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_DEV_STATUS_CTRL,
+			sizeof(uint32), bus->saved_config.exp_dev_ctrl_stat);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGGEN_DEV_STATUS_CTRL2,
+			sizeof(uint32), bus->saved_config.exp_dev_ctrl_stat2);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_LINK_STATUS_CTRL,
+			sizeof(uint32), bus->saved_config.exp_link_ctrl_stat);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_LINK_STATUS_CTRL2,
+			sizeof(uint32), bus->saved_config.exp_link_ctrl_stat2);
+
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_PML1_SUB_CTRL1,
+			sizeof(uint32), bus->saved_config.l1pm0);
+	OSL_PCI_WRITE_CONFIG(osh, PCIECFGREG_PML1_SUB_CTRL2,
+			sizeof(uint32), bus->saved_config.l1pm1);
+
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCI_BAR0_WIN, sizeof(uint32),
+			bus->saved_config.bar0_win);
+	dhdpcie_setbar1win(bus, bus->saved_config.bar1_win);
+
+	return BCME_OK;
+}
+
+int
+dhdpcie_config_save(dhd_bus_t *bus)
+{
+	uint32 i;
+	osl_t *osh = bus->osh;
+
+	if (BCME_OK != dhdpcie_config_check(bus)) {
+		return BCME_ERROR;
+	}
+
+	for (i = 0; i < DHDPCIE_CONFIG_HDR_SIZE; i++) {
+		bus->saved_config.header[i] = OSL_PCI_READ_CONFIG(osh, i << 2, sizeof(uint32));
+	}
+
+	bus->saved_config.pmcsr = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_PM_CSR, sizeof(uint32));
+
+	bus->saved_config.msi_cap = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_MSI_CAP,
+			sizeof(uint32));
+	bus->saved_config.msi_addr0 = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_MSI_ADDR_L,
+			sizeof(uint32));
+	bus->saved_config.msi_addr1 = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_MSI_ADDR_H,
+			sizeof(uint32));
+	bus->saved_config.msi_data = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_MSI_DATA,
+			sizeof(uint32));
+
+	bus->saved_config.exp_dev_ctrl_stat = OSL_PCI_READ_CONFIG(osh,
+			PCIECFGREG_DEV_STATUS_CTRL, sizeof(uint32));
+	bus->saved_config.exp_dev_ctrl_stat2 = OSL_PCI_READ_CONFIG(osh,
+			PCIECFGGEN_DEV_STATUS_CTRL2, sizeof(uint32));
+	bus->saved_config.exp_link_ctrl_stat = OSL_PCI_READ_CONFIG(osh,
+			PCIECFGREG_LINK_STATUS_CTRL, sizeof(uint32));
+	bus->saved_config.exp_link_ctrl_stat2 = OSL_PCI_READ_CONFIG(osh,
+			PCIECFGREG_LINK_STATUS_CTRL2, sizeof(uint32));
+
+	bus->saved_config.l1pm0 = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_PML1_SUB_CTRL1,
+			sizeof(uint32));
+	bus->saved_config.l1pm1 = OSL_PCI_READ_CONFIG(osh, PCIECFGREG_PML1_SUB_CTRL2,
+			sizeof(uint32));
+
+	bus->saved_config.bar0_win = OSL_PCI_READ_CONFIG(osh, PCI_BAR0_WIN,
+			sizeof(uint32));
+	bus->saved_config.bar1_win = OSL_PCI_READ_CONFIG(osh, PCI_BAR1_WIN,
+			sizeof(uint32));
+
+	return BCME_OK;
+}
+
+#ifdef CONFIG_ARCH_EXYNOS
+dhd_pub_t *link_recovery = NULL;
+#endif /* CONFIG_ARCH_EXYNOS */
+
+static void
+dhdpcie_bus_intr_init(dhd_bus_t *bus)
+{
+	uint buscorerev = bus->sih->buscorerev;
+	bus->pcie_mailbox_int = PCIMailBoxInt(buscorerev);
+	bus->pcie_mailbox_mask = PCIMailBoxMask(buscorerev);
+	bus->d2h_mb_mask = PCIE_MB_D2H_MB_MASK(buscorerev);
+	bus->def_intmask = PCIE_MB_D2H_MB_MASK(buscorerev);
+	if (buscorerev < 64) {
+		bus->def_intmask |= PCIE_MB_TOPCIE_FN0_0 | PCIE_MB_TOPCIE_FN0_1;
+	}
+}
+
+static void
+dhdpcie_cc_watchdog_reset(dhd_bus_t *bus)
+{
+	uint32 wd_en = (bus->sih->buscorerev >= 66) ? WD_SSRESET_PCIE_F0_EN :
+		(WD_SSRESET_PCIE_F0_EN | WD_SSRESET_PCIE_ALL_FN_EN);
+	pcie_watchdog_reset(bus->osh, bus->sih, WD_ENABLE_MASK, wd_en);
+}
+
+void
+dhdpcie_dongle_reset(dhd_bus_t *bus)
+{
+
+	/* if the pcie link is down, watchdog reset
+	 * should not be done, as it may hang
+	 */
+	if (bus->is_linkdown) {
+		return;
+	}
+
+	/* Currently BP reset using CFG reg is done only for android platforms */
+#ifdef DHD_USE_BP_RESET_SPROM
+	/* This is for architectures that does NOT control subsystem reset */
+	(void)dhd_bus_cfg_sprom_ctrl_bp_reset(bus);
+	return;
+#elif defined(DHD_USE_BP_RESET_SS_CTRL)
+	/* This is for architectures that supports Subsystem Control */
+	(void)dhd_bus_cfg_ss_ctrl_bp_reset(bus);
+	return;
+#else
+
+#ifdef BCMQT_HW
+	/* flr takes a long time on qt and is only required when testing with BT
+	 * included database. Fall back to watchdog reset by default and only perform
+	 * flr if enabled through module parameter
+	 */
+	if (qt_flr_reset && (dhd_bus_perform_flr(bus, FALSE) != BCME_UNSUPPORTED)) {
+		return;
+	}
+#else
+	/* dhd_bus_perform_flr will return BCME_UNSUPPORTED if chip is not FLR capable */
+	if (dhd_bus_perform_flr(bus, FALSE) == BCME_UNSUPPORTED)
+#endif
+	{
+		/* Legacy chipcommon watchdog reset */
+		dhdpcie_cc_watchdog_reset(bus);
+	}
+	return;
+#endif /* DHD_USE_BP_RESET */
+}
+
+#ifdef BCMQT_HW
+/* Xtal to ALP clock ratio can change from chip to chip and this functions returns the
+ * Xtal to ALP clock ratio by extracting the value for the current chip.
+ */
+static uint32
+dhdpcie_get_xtal_to_alp_ratio(dhd_bus_t *bus)
+{
+	uint16 chipid = si_chipid(bus->sih);
+	uint32 xtal_to_alp_ratio;
+
+	switch (chipid) {
+		case BCM4397_CHIP_ID:
+			xtal_to_alp_ratio = 2;
+			break;
+		default:
+			xtal_to_alp_ratio = 1;
+			break;
+	}
+
+	return xtal_to_alp_ratio;
+}
+
+/* Calculate dongle/host clock ratio for QT so the waiting period in host driver can be scaled
+ * properly. The dongle uses ALP clock by default which can't be read directly. But ILP and
+ * ALP clocks are scaled disproportionally in QT. So DHD must know the preset crystal frequency
+ * for ALP clock in order to calculate the scale ratio. The logic below takes 3 sources of xtal
+ * frequency as following priority:
+ * 1 module parameter
+ * 2 nvram "xtalfreq" line (not available for the first dongle reset)
+ * 3 Hard coded 37.4MHz
+ * If the QT simulation of a chip uses a different frequency xtal other than 37.4MHz, it's
+ * strongly recommended to expend the hard coded value to per chip basis or override with module
+ * parameter.
+ */
+#define XTAL_FREQ_37M4		37400000u
+void dhdpcie_htclkratio_cal(dhd_bus_t *bus)
+{
+	uint cur_coreidx, pmu_idx, ilp_on_fast_lpo = 0;
+	uint32 ilp_start, ilp_tick = 0, alp_to_ilp_ratio = 0;
+	int xtalfreq = 0;
+	uint32 alp_freq = 0;
+	uint32 xtal_to_alp_ratio = dhdpcie_get_xtal_to_alp_ratio(bus);
+
+	/* If a larger than 1 htclkratio is set through module parameter, use it directly */
+	if (htclkratio > 1) {
+		goto exit;
+	}
+
+	/* Store current core id */
+	cur_coreidx = si_coreidx(bus->sih);
+	if (!si_setcore(bus->sih, PMU_CORE_ID, 0)) {
+		htclkratio = 2000;
+		goto exit;
+	}
+
+	pmu_idx = si_coreidx(bus->sih);
+
+	/* Count IPL ticks in 1 second of host domain clock */
+	ilp_start = si_corereg(bus->sih, pmu_idx, offsetof(pmuregs_t, pmutimer), 0, 0);
+	osl_sleep(1000);
+	ilp_tick = si_corereg(bus->sih, pmu_idx, offsetof(pmuregs_t, pmutimer), 0, 0);
+	/* -1 to compensate the incomplete cycle at the beginning */
+	ilp_tick -= ilp_start - 1;
+
+	ilp_on_fast_lpo = si_corereg(bus->sih, pmu_idx, offsetof(pmuregs_t, pmustatus), 0, 0) &
+		PST_ILPFASTLPO;
+
+	if (PMUREV(bus->sih->pmurev) <= 45) {
+		/* If ILP is running on Fast LPO is enabled, XtalFreqRatio uses 1 Mhz ILP
+		 * instead of 32 KHz. But PMUTimer runs at 32 KHz only irrespective of Fast
+		 * LPO is enabled or not. Thereby scale up ilp_tick when Fast LPO is enabled.
+		 * In case of Zebu/QT, Fast LPO is exactly 32 times of ILP clock.
+		 * [Reference: CRWLPMU-284]
+		 */
+		if (ilp_on_fast_lpo) {
+			ilp_tick *= 32;
+		}
+	}
+	/* Get xtal vs ILP ratio from XtalFreqRatio(0x66c) */
+	alp_to_ilp_ratio = si_corereg(bus->sih, pmu_idx, offsetof(pmuregs_t, pmu_xtalfreq), 0, 0);
+	alp_to_ilp_ratio = (alp_to_ilp_ratio & PMU_XTALFREQ_REG_ILPCTR_MASK) / 4;
+
+	/* Go back to original core */
+	si_setcoreidx(bus->sih, cur_coreidx);
+
+	/* Use module parameter if one is provided. Otherwise use default 37.4MHz */
+	if (dngl_xtalfreq) {
+		xtalfreq = dngl_xtalfreq;
+	} else {
+		xtalfreq = XTAL_FREQ_37M4;
+	}
+
+	alp_freq = xtalfreq / xtal_to_alp_ratio;
+
+	/* htclkratio = Expected ALP Freq / Real ALP Freq
+	 *            = Expected ALP Freq / (Real ILP Tick * Real alp_to_ilp_ratio)
+	 */
+	htclkratio = alp_freq / (ilp_tick * alp_to_ilp_ratio);
+	bus->xtalfreq = xtalfreq;
+	bus->ilp_tick = ilp_tick;
+	bus->alp_to_ilp_ratio = alp_to_ilp_ratio;
+	bus->xtal_to_alp_ratio = xtal_to_alp_ratio;
+
+exit:
+	DHD_ERROR(("Dongle/Host clock ratio %u with %dHz xtal frequency, ilp_tick: %d, "
+			"alp_to_ilp_ratio: %d, ilp_on_fast_lpo: %d\n",
+			htclkratio, xtalfreq, ilp_tick, alp_to_ilp_ratio, !!ilp_on_fast_lpo));
+}
+
+/* Re-calculate htclkratio if nvram provides a different xtalfreq */
+void dhdpcie_htclkratio_recal(dhd_bus_t *bus, char *nvram, uint nvram_sz)
+{
+	char *freq_c = NULL;
+	uint len, p;
+	int xtalfreq = 0;
+	uint cur_coreidx, pmu_idx;
+	uint32 alp_to_ilp_ratio = 0;
+	uint32 alp_freq = 0;
+
+	/* Do not re-calculate if xtalfreq is overridden by module parameter */
+	if (dngl_xtalfreq)
+		return;
+
+	/* look for "xtalfreq=xxxx" line in nvram */
+	len = strlen("xtalfreq");
+	for (p = 0; p < (nvram_sz - len) && nvram[p]; ) {
+		if ((bcmp(&nvram[p], "xtalfreq", len) == 0) && (nvram[p + len] == '=')) {
+			freq_c = &nvram[p + len + 1u];
+			break;
+		}
+		/* jump to next line */
+		while (nvram[p++]);
+	}
+
+	if (freq_c) {
+		xtalfreq = bcm_strtoul(freq_c, NULL, 0);
+		if (xtalfreq > (INT_MAX / 1000u)) {
+			DHD_ERROR(("xtalfreq %d in nvram is too big\n", xtalfreq));
+			xtalfreq = 0;
+		}
+		xtalfreq *= 1000;
+	}
+
+	/* Skip recalculation if:
+	 *   nvram doesn't provide "xtalfreq", or
+	 *   first calculation was not performed because module parameter override, or
+	 *   xtalfreq in nvram is the same as the one used in first calculation
+	 */
+	if (xtalfreq == 0 || bus->xtalfreq == 0 || xtalfreq == bus->xtalfreq) {
+		return;
+	}
+
+	/* Print out a error message here. Even if the ratio is corrected with nvram setting, dongle
+	 * reset has been performed before DHD has access to NVRAM. Insufficient waiting period
+	 * for reset might cause unexpected behavior.
+	 */
+	DHD_ERROR(("Re-calculating htclkratio because nvram xtalfreq %dHz is different from %dHz\n",
+		xtalfreq, bus->xtalfreq));
+
+	alp_freq = xtalfreq / bus->xtal_to_alp_ratio;
+	htclkratio = alp_freq / (bus->ilp_tick * bus->alp_to_ilp_ratio);
+	bus->xtalfreq = xtalfreq;
+
+	/* Recalculate alp_to_ilp_ratio just to aid in debuggability. Mainly to compare with the
+	 * previously calculated alp_to_ilp_ratio.
+	 */
+
+	cur_coreidx = si_coreidx(bus->sih);
+	si_setcore(bus->sih, PMU_CORE_ID, 0);
+	pmu_idx = si_coreidx(bus->sih);
+
+	alp_to_ilp_ratio = si_corereg(bus->sih, pmu_idx, offsetof(pmuregs_t, pmu_xtalfreq), 0, 0);
+	alp_to_ilp_ratio = (alp_to_ilp_ratio & PMU_XTALFREQ_REG_ILPCTR_MASK) / 4;
+
+	si_setcoreidx(bus->sih, cur_coreidx);
+
+	DHD_ERROR(("Corrected Host/Dongle clock ratio %u with %dHz xtal frequency, ilp_tick: %d, "
+			"alp_to_ilp_ratio: %d, alp_to_ilp_ratio (recalculated): %d\n", htclkratio,
+			xtalfreq, bus->ilp_tick, bus->alp_to_ilp_ratio, alp_to_ilp_ratio));
+}
+#endif /* BCMQT_HW */
+
+static bool
+is_bmpu_supported(dhd_bus_t *bus)
+{
+	if (BCM4378_CHIP(bus->sih->chip) ||
+		BCM4376_CHIP(bus->sih->chip) ||
+		BCM4387_CHIP(bus->sih->chip) ||
+		BCM4385_CHIP(bus->sih->chip)) {
+		return TRUE;
+	}
+	return FALSE;
+}
+
+#define CHIP_COMMON_SCR_DHD_TO_BL_ADDR(sih) (SI_ENUM_BASE(sih) + CC_SCR_DHD_TO_BL)
+#define CHIP_COMMON_SCR_BL_TO_DHD_ADDR(sih) (SI_ENUM_BASE(sih) + CC_SCR_BL_TO_DHD)
+void
+dhdpcie_bus_mpu_disable(dhd_bus_t *bus)
+{
+	volatile uint32 *cr4_regs;
+	uint val = 0;
+
+	if (is_bmpu_supported(bus) == FALSE) {
+		return;
+	}
+
+	/* reset to default values dhd_to_bl and bl_to_dhd regs */
+	(void)serialized_backplane_access(bus, CHIP_COMMON_SCR_DHD_TO_BL_ADDR(bus->sih),
+		sizeof(val), &val, FALSE);
+	(void)serialized_backplane_access(bus, CHIP_COMMON_SCR_BL_TO_DHD_ADDR(bus->sih),
+		sizeof(val), &val, FALSE);
+
+	cr4_regs = si_setcore(bus->sih, ARMCR4_CORE_ID, 0);
+	if (cr4_regs == NULL) {
+		DHD_ERROR(("%s: Failed to find ARM core!\n", __FUNCTION__));
+		return;
+	}
+	if (R_REG(bus->osh, cr4_regs + ARMCR4REG_CORECAP) & ACC_MPU_MASK) {
+		/* bus mpu is supported */
+		W_REG(bus->osh, cr4_regs + ARMCR4REG_MPUCTRL, 0);
+	}
+}
+
+uint32 pcie_slave_wrapper_offsets[] = {
+	AI_ERRLOGCTRL,
+	AI_ERRLOGDONE,
+	AI_ERRLOGSTATUS,
+	AI_ERRLOGADDRLO,
+	AI_ERRLOGADDRHI,
+	AI_ERRLOGID,
+	AI_ERRLOGUSER,
+	AI_ERRLOGFLAGS
+};
+
+#define PCIE_SLAVER_WRAPPER_BASE	0x18102000u
+
+static INLINE void dhd_sbreg_op(dhd_pub_t *dhd, uint addr, uint *val, bool read);
+
+static uint32
+dhd_get_pcie_slave_wrapper(si_t *sih)
+{
+	uint32 pcie_slave_wrapper = 0;
+	uint16 chipid = si_chipid(sih);
+
+	switch (chipid) {
+		case BCM4389_CHIP_ID:
+		case BCM4387_CHIP_ID:
+			pcie_slave_wrapper = PCIE_SLAVER_WRAPPER_BASE;
+			break;
+		default:
+			pcie_slave_wrapper = 0;
+			break;
+	}
+
+	return pcie_slave_wrapper;
+}
+
+static void
+dhd_dump_pcie_slave_wrapper_regs(dhd_bus_t *bus)
+{
+	uint32 i, val;
+	uint32 pcie_slave_wrapper_base = 0;
+	uint32 total_wrapper_regs;
+
+	if (bus->dhd == NULL) {
+		return;
+	}
+
+	pcie_slave_wrapper_base = dhd_get_pcie_slave_wrapper(bus->sih);
+	if (!pcie_slave_wrapper_base) {
+		DHD_ERROR(("%s pcie slave wrapper base not populated\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("%s: ##### Dumping PCIe slave wrapper regs #####\n", __FUNCTION__));
+
+	total_wrapper_regs =
+		sizeof(pcie_slave_wrapper_offsets) / sizeof(pcie_slave_wrapper_offsets[0]);
+
+	for (i = 0; i < total_wrapper_regs; i++) {
+		dhd_sbreg_op(bus->dhd, (pcie_slave_wrapper_base + pcie_slave_wrapper_offsets[i]),
+			&val, TRUE);
+	}
+
+	DHD_ERROR(("%s: ##### ##### #####\n", __FUNCTION__));
+}
+
+bool
+dhd_check_htput_chip(dhd_bus_t *bus)
+{
+	bool htput_support = FALSE;
+	uint16 chipid = si_chipid(bus->sih);
+
+	switch (chipid) {
+		/* Enable htput support for all 160Mhz chips */
+		case BCM4388_CHIP_ID:
+		case BCM4389_CHIP_ID:
+		case BCM4397_CHIP_ID:
+			htput_support = TRUE;
+			break;
+		default:
+			htput_support = TRUE;
+	}
+
+	DHD_ERROR(("%s: htput_support:%d\n", __FUNCTION__, htput_support));
+
+	return htput_support;
+}
+
+static bool
+dhdpcie_dongle_attach(dhd_bus_t *bus)
+{
+	osl_t *osh = bus->osh;
+	volatile void *regsva = (volatile void*)bus->regs;
+	uint16 devid;
+	uint32 val;
+	sbpcieregs_t *sbpcieregs;
+	bool dongle_reset_needed;
+	uint16 chipid;
+
+	BCM_REFERENCE(chipid);
+
+	DHD_TRACE(("%s: ENTER\n", __FUNCTION__));
+
+	chipid = dhd_get_chipid(bus);
+
+	if (chipid == (uint16)-1)
+		goto fail;
+
+	/* Configure CTO Prevention functionality */
+#if defined(BCMFPGA_HW) || defined(BCMQT_HW)
+	DHD_ERROR(("Disable CTO\n"));
+	bus->cto_enable = FALSE;
+#else
+#if defined(BCMPCIE_CTO_PREVENTION)
+
+	if (BCM4349_CHIP(chipid) || BCM4350_CHIP(chipid) || BCM4345_CHIP(chipid)) {
+		DHD_ERROR(("Disable CTO\n"));
+		bus->cto_enable = FALSE;
+	} else {
+		DHD_ERROR(("Enable CTO\n"));
+		bus->cto_enable = TRUE;
+	}
+#else
+	DHD_ERROR(("Disable CTO\n"));
+	bus->cto_enable = FALSE;
+#endif /* BCMPCIE_CTO_PREVENTION */
+#endif /* BCMFPGA_HW || BCMQT_HW */
+
+	if (PCIECTO_ENAB(bus)) {
+		/* WAR - Delay the CTO init in 4397A0. */
+		if (!(BCM4397_CHIP(chipid) && dhd_get_chiprev(bus) == 0)) {
+			dhdpcie_cto_init(bus, TRUE);
+		}
+	}
+
+#ifdef CONFIG_ARCH_EXYNOS
+	link_recovery = bus->dhd;
+#endif /* CONFIG_ARCH_EXYNOS */
+
+	dhd_init_pwr_req_lock(bus);
+	dhd_init_bus_lp_state_lock(bus);
+	dhd_init_backplane_access_lock(bus);
+
+	bus->alp_only = TRUE;
+	bus->sih = NULL;
+
+	/* Checking PCIe bus status with reading configuration space */
+	val = OSL_PCI_READ_CONFIG(osh, PCI_CFG_VID, sizeof(uint32));
+	if (((val & 0xFFFF) != VENDOR_BROADCOM) && ((val & 0xFFFF) != VENDOR_SYNAPTICS)) {
+		DHD_ERROR(("%s : failed to read PCI configuration space!\n", __FUNCTION__));
+		goto fail;
+	}
+	devid = (val >> 16) & 0xFFFF;
+	bus->cl_devid = devid;
+
+	/* Set bar0 window to si_enum_base */
+	dhdpcie_bus_cfg_set_bar0_win(bus, si_enum_base(devid));
+
+	/*
+	 * Checking PCI_SPROM_CONTROL register for preventing invalid address access
+	 * due to switch address space from PCI_BUS to SI_BUS.
+	 */
+	val = OSL_PCI_READ_CONFIG(osh, PCI_SPROM_CONTROL, sizeof(uint32));
+	if (val == 0xffffffff) {
+		DHD_ERROR(("%s : failed to read SPROM control register\n", __FUNCTION__));
+		goto fail;
+	}
+
+#if defined(NDIS)
+	/* Save good copy of PCIe config space */
+	if (BCME_OK != dhdpcie_config_save(bus)) {
+		DHD_ERROR(("%s : failed to save PCI configuration space!\n", __FUNCTION__));
+		goto fail;
+	}
+#endif
+
+	/* si_attach() will provide an SI handle and scan the backplane */
+	if (!(bus->sih = si_attach((uint)devid, osh, regsva, PCI_BUS, bus,
+	                           &bus->vars, &bus->varsz))) {
+		DHD_ERROR(("%s: si_attach failed!\n", __FUNCTION__));
+		goto fail;
+	}
+
+	if (MULTIBP_ENAB(bus->sih) && (bus->sih->buscorerev >= 66)) {
+		/*
+		 * HW JIRA - CRWLPCIEGEN2-672
+		 * Producer Index Feature which is used by F1 gets reset on F0 FLR
+		 * fixed in REV68
+		 */
+		if (PCIE_ENUM_RESET_WAR_ENAB(bus->sih->buscorerev)) {
+			dhdpcie_ssreset_dis_enum_rst(bus);
+		}
+
+		/* IOV_DEVRESET could exercise si_detach()/si_attach() again so reset
+		*   dhdpcie_bus_release_dongle() --> si_detach()
+		*   dhdpcie_dongle_attach() --> si_attach()
+		*/
+		bus->pwr_req_ref = 0;
+	}
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_nolock(bus);
+	}
+
+	/* Calling after requesting dm1 (Wl) power as there are mac wrapper regs */
+	dhd_dump_pcie_slave_wrapper_regs(bus);
+
+	/* Get info on the ARM and SOCRAM cores... */
+	/* Should really be qualified by device id */
+	if ((si_setcore(bus->sih, ARM7S_CORE_ID, 0)) ||
+	    (si_setcore(bus->sih, ARMCM3_CORE_ID, 0)) ||
+	    (si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) ||
+	    (si_setcore(bus->sih, ARMCA7_CORE_ID, 0))) {
+		bus->armrev = si_corerev(bus->sih);
+		bus->coreid = si_coreid(bus->sih);
+	} else {
+		DHD_ERROR(("%s: failed to find ARM core!\n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* CA7 requires coherent bits on */
+	if (bus->coreid == ARMCA7_CORE_ID) {
+		val = dhdpcie_bus_cfg_read_dword(bus, PCIE_CFG_SUBSYSTEM_CONTROL, 4);
+		dhdpcie_bus_cfg_write_dword(bus, PCIE_CFG_SUBSYSTEM_CONTROL, 4,
+			(val | PCIE_BARCOHERENTACCEN_MASK));
+	}
+
+	/* EFI requirement - stop driver load if FW is already running
+	*  need to do this here before pcie_watchdog_reset, because
+	*  pcie_watchdog_reset will put the ARM back into halt state
+	*/
+	if (!dhdpcie_is_arm_halted(bus)) {
+		DHD_ERROR(("%s: ARM is not halted,FW is already running! Abort.\n",
+				__FUNCTION__));
+		goto fail;
+	}
+
+	BCM_REFERENCE(dongle_reset_needed);
+
+	/* For inbuilt drivers pcie clk req will be done by RC,
+	 * so do not do clkreq from dhd
+	 */
+#if defined(linux) || defined(LINUX)
+	if (dhd_download_fw_on_driverload)
+#endif /* linux || LINUX */
+	{
+		/* Enable CLKREQ# */
+		dhdpcie_clkreq(bus->osh, 1, 1);
+	}
+
+	/* Calculate htclkratio only for QT, for FPGA it is fixed at 30 */
+#ifdef BCMQT_HW
+	dhdpcie_htclkratio_cal(bus);
+#endif /* BCMQT_HW */
+
+	/*
+	 * bus->dhd will be NULL if it is called from dhd_bus_attach, so need to reset
+	 * without checking dongle_isolation flag, but if it is called via some other path
+	 * like quiesce FLR, then based on dongle_isolation flag, watchdog_reset should
+	 * be called.
+	 */
+	if (bus->dhd == NULL) {
+		/* dhd_attach not yet happened, do dongle reset */
+#ifdef DHD_SKIP_DONGLE_RESET_IN_ATTACH
+		dongle_reset_needed = FALSE;
+#else
+		dongle_reset_needed = TRUE;
+#endif /* DHD_SKIP_DONGLE_RESET_IN_ATTACH */
+	} else {
+		/* Based on dongle_isolationflag, reset dongle */
+		dongle_reset_needed = !(bus->dhd->dongle_isolation);
+	}
+
+	/* Fix for FLR reset & CTO init specific to 4397a0. Write a value 0x1E in PMU CC reg18 */
+	if (BCM4397_CHIP(bus->sih->chip) && (bus->sih->chiprev == 0)) {
+		uint origidx = 0;
+		chipcregs_t *cc;
+
+		origidx = si_coreidx(bus->sih);
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_addr, ~0, PMU_CHIPCTL18);
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_data,
+			(PMU_CC18_WL_BOOKER_FORCEPWRDWN_EN | PMU_CC18_WL_P_CHAN_TIMER_SEL_MASK),
+			(PMU_CC18_WL_BOOKER_FORCEPWRDWN_EN |
+			((PMU_CC18_WL_P_CHAN_TIMER_SEL_8ms << PMU_CC18_WL_P_CHAN_TIMER_SEL_OFF) &
+			PMU_CC18_WL_P_CHAN_TIMER_SEL_MASK)));
+
+		/* 4397A0 WAR. Set max rsrc mask to 0x7fffffff and force HT before CTO init */
+		pmu_corereg(bus->sih, SI_CC_IDX, max_res_mask, ~0, 0x7fffffff);
+
+		cc = (chipcregs_t*)si_setcore(bus->sih, CC_CORE_ID, 0);
+		if (cc) {
+			OR_REG(osh, &cc->clk_ctl_st, CCS_FORCEHT);
+			SPINWAIT(((R_REG(osh, &cc->clk_ctl_st) & CCS_HTAVAIL) == 0),
+				PMU_MAX_TRANSITION_DLY);
+		} else {
+			DHD_ERROR(("CC is NULL\n"));
+		}
+		si_setcore(bus->sih, origidx, 0);
+		if (PCIECTO_ENAB(bus)) {
+			dhdpcie_cto_init(bus, TRUE);
+		}
+	}
+
+	/* need to set the force_bt_quiesce flag here
+	 * before calling dhdpcie_dongle_flr_or_pwr_toggle
+	 */
+	bus->force_bt_quiesce = TRUE;
+	/*
+	 * For buscorerev = 66 and after, F0 FLR should be done independent from F1.
+	 * So don't need BT quiesce.
+	 */
+	if (bus->sih->buscorerev >= 66) {
+		bus->force_bt_quiesce = FALSE;
+	}
+
+	/*
+	 * Issue dongle to reset all the cores on the chip - similar to rmmod dhd
+	 * This is required to avoid spurious interrupts to the Host and bring back
+	 * dongle to a sane state (on host soft-reboot / watchdog-reboot).
+	 */
+	if (dongle_reset_needed) {
+		dhdpcie_dongle_reset(bus);
+	}
+
+	dhdpcie_dongle_flr_or_pwr_toggle(bus);
+
+	dhdpcie_bus_mpu_disable(bus);
+
+	si_setcore(bus->sih, PCIE2_CORE_ID, 0);
+	sbpcieregs = (sbpcieregs_t*)(bus->regs);
+
+	/* WAR where the BAR1 window may not be sized properly */
+	W_REG(osh, &sbpcieregs->configaddr, 0x4e0);
+	val = R_REG(osh, &sbpcieregs->configdata);
+	W_REG(osh, &sbpcieregs->configdata, val);
+
+	/* if chip uses sysmem instead of tcm, typically ARM CA chips */
+	if (si_setcore(bus->sih, SYSMEM_CORE_ID, 0)) {
+		if (!(bus->orig_ramsize = si_sysmem_size(bus->sih))) {
+			DHD_ERROR(("%s: failed to find SYSMEM memory!\n", __FUNCTION__));
+			goto fail;
+		}
+		/* also populate base address */
+		switch ((uint16)bus->sih->chip) {
+			case BCM4385_CHIP_ID:
+				bus->dongle_ram_base = CA7_4385_RAM_BASE;
+				break;
+			case BCM4388_CHIP_ID:
+			case BCM4389_CHIP_ID:
+				bus->dongle_ram_base = CA7_4389_RAM_BASE;
+				break;
+			default:
+				/* also populate base address */
+				bus->dongle_ram_base = 0x200000;
+				DHD_ERROR(("%s: WARNING: Using default ram base at 0x%x\n",
+					__FUNCTION__, bus->dongle_ram_base));
+				break;
+		}
+	} else if (!si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+		if (!(bus->orig_ramsize = si_socram_size(bus->sih))) {
+			DHD_ERROR(("%s: failed to find SOCRAM memory!\n", __FUNCTION__));
+			goto fail;
+		}
+	} else {
+		/* cr4 has a different way to find the RAM size from TCM's */
+		if (!(bus->orig_ramsize = si_tcm_size(bus->sih))) {
+			DHD_ERROR(("%s: failed to find CR4-TCM memory!\n", __FUNCTION__));
+			goto fail;
+		}
+		/* also populate base address */
+		switch ((uint16)bus->sih->chip) {
+		case BCM4339_CHIP_ID:
+		case BCM4335_CHIP_ID:
+			bus->dongle_ram_base = CR4_4335_RAM_BASE;
+			break;
+		case BCM4358_CHIP_ID:
+		case BCM4354_CHIP_ID:
+		case BCM4356_CHIP_ID:
+		case BCM43567_CHIP_ID:
+		case BCM43569_CHIP_ID:
+		case BCM4350_CHIP_ID:
+		case BCM43570_CHIP_ID:
+			bus->dongle_ram_base = CR4_4350_RAM_BASE;
+			break;
+		case BCM4360_CHIP_ID:
+			bus->dongle_ram_base = CR4_4360_RAM_BASE;
+			break;
+
+		case BCM4364_CHIP_ID:
+			bus->dongle_ram_base = CR4_4364_RAM_BASE;
+			break;
+
+		CASE_BCM4345_CHIP:
+			bus->dongle_ram_base = (bus->sih->chiprev < 6)  /* changed at 4345C0 */
+				? CR4_4345_LT_C0_RAM_BASE : CR4_4345_GE_C0_RAM_BASE;
+			break;
+		CASE_BCM43602_CHIP:
+			bus->dongle_ram_base = CR4_43602_RAM_BASE;
+			break;
+		case BCM4349_CHIP_GRPID:
+			/* RAM based changed from 4349c0(revid=9) onwards */
+			bus->dongle_ram_base = ((bus->sih->chiprev < 9) ?
+				CR4_4349_RAM_BASE : CR4_4349_RAM_BASE_FROM_REV_9);
+			break;
+		case BCM4347_CHIP_ID:
+		case BCM4357_CHIP_ID:
+		case BCM4361_CHIP_ID:
+			bus->dongle_ram_base = CR4_4347_RAM_BASE;
+			break;
+		case BCM43751_CHIP_ID:
+			bus->dongle_ram_base = CR4_43751_RAM_BASE;
+			break;
+		case BCM43752_CHIP_ID:
+			bus->dongle_ram_base = CR4_43752_RAM_BASE;
+			break;
+		case BCM43756_CHIP_ID:
+			bus->dongle_ram_base = CR4_43756_RAM_BASE;
+			break;
+		case BCM43711_CHIP_ID:
+			bus->dongle_ram_base = CR4_43711_RAM_BASE;
+			break;
+		case BCM4376_CHIP_GRPID:
+			bus->dongle_ram_base = CR4_4376_RAM_BASE;
+			break;
+		case BCM4378_CHIP_GRPID:
+			bus->dongle_ram_base = CR4_4378_RAM_BASE;
+			break;
+		case BCM4362_CHIP_ID:
+			bus->dongle_ram_base = CR4_4362_RAM_BASE;
+			break;
+		case BCM4375_CHIP_ID:
+		case BCM4369_CHIP_ID:
+			bus->dongle_ram_base = CR4_4369_RAM_BASE;
+			break;
+		case BCM4377_CHIP_ID:
+			bus->dongle_ram_base = CR4_4377_RAM_BASE;
+			break;
+		case BCM4381_CHIP_ID:
+			bus->dongle_ram_base = CR4_4381_RAM_BASE;
+			break;
+		case BCM4382_CHIP_ID:
+			bus->dongle_ram_base = CR4_4382_RAM_BASE;
+			break;
+		case BCM4387_CHIP_GRPID:
+			bus->dongle_ram_base = CR4_4387_RAM_BASE;
+			break;
+		case BCM4385_CHIP_ID:
+			bus->dongle_ram_base = CR4_4385_RAM_BASE;
+			break;
+		default:
+			bus->dongle_ram_base = 0;
+			DHD_ERROR(("%s: WARNING: Using default ram base at 0x%x\n",
+			           __FUNCTION__, bus->dongle_ram_base));
+		}
+	}
+	bus->ramsize = bus->orig_ramsize;
+	if (dhd_dongle_ramsize) {
+		dhdpcie_bus_dongle_setmemsize(bus, dhd_dongle_ramsize);
+	}
+
+	if (bus->ramsize > DONGLE_TCM_MAP_SIZE) {
+		DHD_ERROR(("%s : invalid ramsize %d(0x%x) is returned from dongle\n",
+				__FUNCTION__, bus->ramsize, bus->ramsize));
+		goto fail;
+	}
+
+	DHD_INFO(("DHD: dongle ram size is set to %d(orig %d) at 0x%x\n",
+	           bus->ramsize, bus->orig_ramsize, bus->dongle_ram_base));
+
+	dhdpcie_bar1_window_switch_enab(bus);
+
+	/* Init bar1_switch_lock only after bar1_switch_enab is inited */
+	dhd_init_bar1_switch_lock(bus);
+
+	bus->srmemsize = si_socram_srmem_size(bus->sih);
+
+	dhdpcie_bus_intr_init(bus);
+
+	/* Set the poll and/or interrupt flags */
+	bus->intr = (bool)dhd_intr;
+	if ((bus->poll = (bool)dhd_poll))
+		bus->pollrate = 1;
+#ifdef DHD_DISABLE_ASPM
+	dhd_bus_aspm_enable_rc_ep(bus, FALSE);
+#endif /* DHD_DISABLE_ASPM */
+#ifdef PCIE_OOB
+	dhdpcie_oob_init(bus);
+#endif /* PCIE_OOB */
+#ifdef PCIE_INB_DW
+	bus->inb_enabled = TRUE;
+#endif /* PCIE_INB_DW */
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	bus->ds_enabled = TRUE;
+	bus->deep_sleep = TRUE;
+#endif
+
+	/*
+	 * This check is to prevent to go back to enable during fw download
+	 * after forced to disable by dhd commend(eg. "dhd dar_enable 0") after insmod
+	 */
+	if ((bus->dhd == NULL) || (bus->dhd->dongle_reset == FALSE)) {
+		bus->dar_enabled = TRUE;
+		bus->idma_enabled = TRUE;
+		bus->ifrm_enabled = TRUE;
+	}
+
+	dhdpcie_pme_stat_clear(bus);
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear_nolock(bus);
+
+		/*
+		 * One time clearing of Common Power Domain since HW default is set
+		 * Needs to be after FLR because FLR resets PCIe enum back to HW defaults
+		 * for 4378B0 (rev 68).
+		 * On 4378A0 (rev 66), PCIe enum reset is disabled due to CRWLPCIEGEN2-672
+		 */
+		si_srpwr_request(bus->sih, SRPWR_DMN0_PCIE_MASK, 0);
+
+		/*
+		 * WAR to fix ARM cold boot;
+		 * Assert WL domain in DAR helps but not enum
+		 */
+		if (bus->sih->buscorerev >= 68) {
+			dhd_bus_pcie_pwr_req_wl_domain(bus,
+				DAR_PCIE_PWR_CTRL((bus->sih)->buscorerev), TRUE);
+		}
+	}
+
+	DHD_TRACE(("%s: EXIT: SUCCESS\n", __FUNCTION__));
+
+	return 0;
+
+fail:
+/* for EFI even if there is an error, load still succeeds
+* so si_detach should not be called here, it is called during unload
+*/
+	/*
+	 * As request irq is done later, till then CTO will not be detected,
+	 * so unconditionally dump cfg and DAR registers.
+	 */
+	dhd_bus_dump_imp_cfg_registers(bus);
+	/* Check if CTO has happened */
+	if (PCIECTO_ENAB(bus)) {
+		/* read pci_intstatus */
+		uint32 pci_intstatus =
+			dhdpcie_bus_cfg_read_dword(bus, PCI_INT_STATUS, 4);
+		if (pci_intstatus == (uint32)-1) {
+			DHD_ERROR(("%s : Invalid pci_intstatus(0x%x)\n",
+				__FUNCTION__, pci_intstatus));
+		} else if (pci_intstatus & PCI_CTO_INT_MASK) {
+			DHD_ERROR(("%s: ##### CTO REPORTED BY DONGLE "
+				"intstat=0x%x enab=%d\n", __FUNCTION__,
+				pci_intstatus, bus->cto_enable));
+		}
+		dhd_bus_dump_dar_registers(bus);
+	}
+	dhd_deinit_pwr_req_lock(bus);
+	dhd_deinit_bus_lp_state_lock(bus);
+	dhd_deinit_backplane_access_lock(bus);
+
+	if (bus->sih != NULL) {
+		/* Dump DAR registers only if si_attach has succeeded */
+		dhd_bus_dump_dar_registers(bus);
+		if (MULTIBP_ENAB(bus->sih)) {
+			dhd_bus_pcie_pwr_req_clear_nolock(bus);
+		}
+
+		si_detach(bus->sih);
+		bus->sih = NULL;
+	}
+	DHD_TRACE(("%s: EXIT: FAILURE\n", __FUNCTION__));
+	return -1;
+}
+
+int
+dhpcie_bus_unmask_interrupt(dhd_bus_t *bus)
+{
+	dhdpcie_bus_cfg_write_dword(bus, PCIIntmask, 4, I_MB);
+	return 0;
+}
+int
+dhpcie_bus_mask_interrupt(dhd_bus_t *bus)
+{
+	dhdpcie_bus_cfg_write_dword(bus, PCIIntmask, 4, 0x0);
+	return 0;
+}
+
+/* Non atomic function, caller should hold appropriate lock */
+void
+dhdpcie_bus_intr_enable(dhd_bus_t *bus)
+{
+	DHD_TRACE(("%s Enter\n", __FUNCTION__));
+	if (bus) {
+		if (bus->sih && !bus->is_linkdown) {
+			/* Skip after receiving D3 ACK */
+			if (__DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+				return;
+			}
+			if ((bus->sih->buscorerev == 2) || (bus->sih->buscorerev == 6) ||
+				(bus->sih->buscorerev == 4)) {
+				dhpcie_bus_unmask_interrupt(bus);
+			} else {
+#ifdef DHD_MMIO_TRACE
+				dhd_bus_mmio_trace(bus, bus->pcie_mailbox_mask,
+					bus->def_intmask, TRUE);
+#endif /* defined(DHD_MMIO_TRACE) */
+				si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_mask,
+					bus->def_intmask, bus->def_intmask);
+			}
+		}
+
+#if defined(NDIS)
+		dhd_msix_message_set(bus->dhd, 0, 0, TRUE);
+#endif
+	}
+
+	DHD_TRACE(("%s Exit\n", __FUNCTION__));
+}
+
+/* Non atomic function, caller should hold appropriate lock */
+void
+dhdpcie_bus_intr_disable(dhd_bus_t *bus)
+{
+	DHD_TRACE(("%s Enter\n", __FUNCTION__));
+	if (bus && bus->sih && !bus->is_linkdown) {
+		/* Skip after receiving D3 ACK */
+		if (__DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+			return;
+		}
+
+		if ((bus->sih->buscorerev == 2) || (bus->sih->buscorerev == 6) ||
+			(bus->sih->buscorerev == 4)) {
+			dhpcie_bus_mask_interrupt(bus);
+		} else {
+#ifdef DHD_MMIO_TRACE
+			dhd_bus_mmio_trace(bus, bus->pcie_mailbox_mask, 0, TRUE);
+#endif /* defined(DHD_MMIO_TRACE) */
+			si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_mask,
+				bus->def_intmask, 0);
+		}
+	}
+#if defined(NDIS)
+	/*
+	 * dhdpcie_bus_intr_disable may get called from
+	 * dhdpcie_dongle_attach -> dhdpcie_dongle_reset
+	 * with dhd = NULL during attach time. So check for bus->dhd NULL before
+	 * calling dhd_msix_message_set
+	 */
+	if (bus && bus->dhd) {
+		dhd_msix_message_set(bus->dhd, 0, 0, FALSE);
+	}
+#endif
+
+	DHD_TRACE(("%s Exit\n", __FUNCTION__));
+}
+
+/*
+ *  dhdpcie_advertise_bus_cleanup advertises that clean up is under progress
+ * to other bus user contexts like Tx, Rx, IOVAR, WD etc and it waits for other contexts
+ * to gracefully exit. All the bus usage contexts before marking busstate as busy, will check for
+ * whether the busstate is DHD_BUS_DOWN or DHD_BUS_DOWN_IN_PROGRESS, if so
+ * they will exit from there itself without marking dhd_bus_busy_state as BUSY.
+ */
+void
+dhdpcie_advertise_bus_cleanup(dhd_pub_t *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	dhdp->dhd_watchdog_ms_backup = dhd_watchdog_ms;
+	if (dhdp->dhd_watchdog_ms_backup) {
+		DHD_ERROR(("%s: Disabling wdtick before dhd deinit\n",
+			__FUNCTION__));
+		dhd_os_wd_timer(dhdp, 0);
+	}
+	if (dhdp->busstate != DHD_BUS_DOWN) {
+#ifdef DHD_DONGLE_TRAP_IN_DETACH
+		/*
+		 * For x86 platforms, rmmod/insmod is failing due to some power
+		 * resources are not held high.
+		 * Hence induce DB7 trap during detach and in FW trap handler all
+		 * power resources are held high.
+		 */
+		if (!dhd_query_bus_erros(dhdp) && dhdp->db7_trap.fw_db7w_trap) {
+			dhdp->db7_trap.fw_db7w_trap_inprogress = TRUE;
+			dhdpcie_fw_trap(dhdp->bus);
+			OSL_DELAY(100 * 1000); // wait 100 msec
+			if (!dhdp->db7_trap.fw_db7w_trap_recieved) {
+				DHD_ERROR(("%s : Did not receive DB7 Ack\n", __FUNCTION__));
+#ifdef WBRC
+				if (dhdp->fw_mode_changed == FALSE) {
+					DHD_ERROR(("%s : Set do_chip_bighammer\n", __FUNCTION__));
+					dhdp->do_chip_bighammer = TRUE;
+				}
+#endif /* WBRC */
+			}
+			dhdp->db7_trap.fw_db7w_trap_inprogress = FALSE;
+		} else {
+			DHD_ERROR(("%s: DB7 Not sent!!!\n",
+				__FUNCTION__));
+		}
+#endif /* DHD_DONGLE_TRAP_IN_DETACH */
+		DHD_GENERAL_LOCK(dhdp, flags);
+		dhdp->busstate = DHD_BUS_DOWN_IN_PROGRESS;
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+	}
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+#ifdef LINUX
+	if ((timeleft == 0) || (timeleft == 1))
+#else
+	if (timeleft == 0)
+#endif
+	{
+		/* XXX This condition ideally should not occur, this means some
+		 * bus usage context is not clearing the respective usage bit, print
+		 * dhd_bus_busy_state and crash the host for further debugging.
+		 */
+		DHD_ERROR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+		ASSERT(0);
+	}
+
+	return;
+}
+
+static void
+dhdpcie_advertise_bus_remove(dhd_pub_t	 *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	dhdp->busstate = DHD_BUS_REMOVE;
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+	if ((timeleft == 0) || (timeleft == 1)) {
+		DHD_ERROR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+		ASSERT(0);
+	}
+
+	return;
+}
+
+/*
+ * dhdpcie_busbusy_wait mark busstate as DHD_BUS_DOWN_IN_PROGRESS and waits
+ * for all the contexts to garacefully exit. All the bus usage contexts before
+ * marking busstate as busy, will check for whether the busstate is DHD_BUS_DOWN
+ * or DHD_BUS_DOWN_IN_PROGRESS, if so they will exit from there itself without
+ * marking dhd_bus_busy_state as BUSY.
+ */
+void
+dhdpcie_busbusy_wait(dhd_pub_t *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	dhdp->dhd_watchdog_ms_backup = dhd_watchdog_ms;
+	if (dhdp->dhd_watchdog_ms_backup) {
+		DHD_ERROR(("%s: Disabling wdtick\n", __FUNCTION__));
+		dhd_os_wd_timer(dhdp, 0);
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	dhdp->busstate = DHD_BUS_DOWN_IN_PROGRESS;
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+	if ((timeleft == 0) || (timeleft == 1)) {
+		DHD_ERROR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+	}
+
+	DHD_GENERAL_LOCK(dhdp, flags);
+	dhdp->busstate = DHD_BUS_DOWN;
+	DHD_GENERAL_UNLOCK(dhdp, flags);
+	return;
+}
+
+static void
+dhdpcie_bus_remove_prep(dhd_bus_t *bus)
+{
+	unsigned long flags;
+	DHD_TRACE(("%s Enter\n", __FUNCTION__));
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+	bus->dhd->busstate = DHD_BUS_DOWN;
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+#ifdef PCIE_INB_DW
+	/* De-Initialize the lock to serialize Device Wake Inband activities */
+	if (bus->inb_lock) {
+		osl_spin_lock_deinit(bus->dhd->osh, bus->inb_lock);
+		bus->inb_lock = NULL;
+	}
+#endif
+
+	dhd_os_sdlock(bus->dhd);
+
+	if (bus->sih && !bus->dhd->dongle_isolation) {
+
+		dhd_bus_pcie_pwr_req_reload_war(bus);
+
+		/* Skip below WARs for Android as insmod fails after rmmod in Brix Android */
+#if !defined(OEM_ANDROID)
+		/* HW4347-909, Set PCIE TRefUp time to 100us for 4347/4377 */
+		if ((bus->sih->buscorerev == 19) || (bus->sih->buscorerev == 23)) {
+			pcie_set_trefup_time_100us(bus->sih);
+		}
+
+		/* disable fast lpo from 4347/4377 */
+		/* Do not disable fast lpo for 4378/4387/4389/4381/4382.
+		 * Because we always enable fast lpo. it causes insmod/rmmod reload failure.
+		 */
+		if ((PMUREV(bus->sih->pmurev) > 31) &&
+			!(PCIE_FASTLPO_ENABLED(bus->sih->buscorerev))) {
+			si_pmu_fast_lpo_disable(bus->sih);
+		}
+#endif /* !OEM_ANDROID */
+
+		/* if the pcie link is down, watchdog reset
+		* should not be done, as it may hang
+		*/
+
+		if (!bus->is_linkdown) {
+			/* For Non-EFI modular builds, do dongle reset during rmmod */
+			/* For EFI-DHD this compile flag will be defined.
+			 * In EFI, depending on bt over pcie mode
+			 * we either power toggle or do F0 FLR
+			 * from dhdpcie_bus_release dongle. So no need to
+			 * do dongle reset from here
+			 */
+			dhdpcie_dongle_reset(bus);
+		}
+
+		bus->dhd->is_pcie_watchdog_reset = TRUE;
+	}
+
+	dhd_os_sdunlock(bus->dhd);
+
+	DHD_TRACE(("%s Exit\n", __FUNCTION__));
+}
+
+void
+dhd_init_bus_lp_state_lock(dhd_bus_t *bus)
+{
+	if (!bus->bus_lp_state_lock) {
+		bus->bus_lp_state_lock = osl_spin_lock_init(bus->osh);
+	}
+}
+
+void
+dhd_deinit_bus_lp_state_lock(dhd_bus_t *bus)
+{
+	if (bus->bus_lp_state_lock) {
+		osl_spin_lock_deinit(bus->osh, bus->bus_lp_state_lock);
+		bus->bus_lp_state_lock = NULL;
+	}
+}
+
+void
+dhd_init_backplane_access_lock(dhd_bus_t *bus)
+{
+	if (!bus->backplane_access_lock) {
+		bus->backplane_access_lock = osl_spin_lock_init(bus->osh);
+	}
+}
+
+void
+dhd_deinit_backplane_access_lock(dhd_bus_t *bus)
+{
+	if (bus->backplane_access_lock) {
+		osl_spin_lock_deinit(bus->osh, bus->backplane_access_lock);
+		bus->backplane_access_lock = NULL;
+	}
+}
+
+void
+dhd_init_dpc_histos(dhd_pub_t *dhd)
+{
+	dhd_bus_t *bus = dhd->bus;
+	if (!bus->dpc_time_histo) {
+		bus->dpc_time_histo = dhd_histo_init(dhd);
+	}
+	if (!bus->ctrl_cpl_post_time_histo) {
+		bus->ctrl_cpl_post_time_histo = dhd_histo_init(dhd);
+	}
+	if (!bus->tx_post_time_histo) {
+		bus->tx_post_time_histo = dhd_histo_init(dhd);
+	}
+	if (!bus->tx_cpl_time_histo) {
+		bus->tx_cpl_time_histo = dhd_histo_init(dhd);
+	}
+	if (!bus->rx_cpl_post_time_histo) {
+		bus->rx_cpl_post_time_histo = dhd_histo_init(dhd);
+	}
+}
+
+void
+dhd_deinit_dpc_histos(dhd_pub_t *dhd)
+{
+	dhd_bus_t *bus = dhd->bus;
+	if (bus->dpc_time_histo) {
+		dhd_histo_deinit(dhd, bus->dpc_time_histo);
+	}
+	if (bus->ctrl_cpl_post_time_histo) {
+		dhd_histo_deinit(dhd, bus->ctrl_cpl_post_time_histo);
+	}
+	if (bus->tx_post_time_histo) {
+		dhd_histo_deinit(dhd, bus->tx_post_time_histo);
+	}
+	if (bus->tx_cpl_time_histo) {
+		dhd_histo_deinit(dhd, bus->tx_cpl_time_histo);
+	}
+	if (bus->rx_cpl_post_time_histo) {
+		dhd_histo_deinit(dhd, bus->rx_cpl_post_time_histo);
+	}
+}
+
+void
+dhd_dump_dpc_histos(dhd_pub_t *dhd, struct bcmstrbuf *strbuf)
+{
+	dhd_bus_t *bus = dhd->bus;
+	bcm_bprintf(strbuf, "==== DPC Histograms in Usec ====\n");
+	dhd_histo_tag_dump(dhd, strbuf, "usec/histo");
+	dhd_histo_dump(dhd, strbuf, bus->dpc_time_histo, "dpc");
+	dhd_histo_dump(dhd, strbuf, bus->ctrl_cpl_post_time_histo, "ctrl_cpl_post");
+	dhd_histo_dump(dhd, strbuf, bus->tx_post_time_histo, "tx_post");
+	dhd_histo_dump(dhd, strbuf, bus->tx_cpl_time_histo, "tx_cpl");
+	dhd_histo_dump(dhd, strbuf, bus->rx_cpl_post_time_histo, "rx_cpl_post");
+	bcm_bprintf(strbuf, "================================\n");
+}
+
+void
+dhd_clear_dpc_histos(dhd_pub_t *dhd)
+{
+	dhd_bus_t *bus = dhd->bus;
+	dhd_histo_clear(dhd, bus->dpc_time_histo);
+	dhd_histo_clear(dhd, bus->ctrl_cpl_post_time_histo);
+	dhd_histo_clear(dhd, bus->tx_post_time_histo);
+	dhd_histo_clear(dhd, bus->tx_cpl_time_histo);
+	dhd_histo_clear(dhd, bus->rx_cpl_post_time_histo);
+}
+/** Detach and free everything */
+void
+dhdpcie_bus_release(dhd_bus_t *bus)
+{
+	bool dongle_isolation = FALSE;
+#ifdef BCMQT
+	uint buscorerev = 0;
+#endif /* BCMQT */
+	osl_t *osh = NULL;
+#if defined(__linux__)
+	int bcmerror = 0;
+#endif /* __linux__ */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus) {
+
+		osh = bus->osh;
+		ASSERT(osh);
+
+		if (bus->dhd) {
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+			debugger_close();
+#endif /* DEBUGGER || DHD_DSCOPE */
+			dhdpcie_advertise_bus_remove(bus->dhd);
+			dongle_isolation = bus->dhd->dongle_isolation;
+			bus->dhd->is_pcie_watchdog_reset = FALSE;
+			dhdpcie_bus_remove_prep(bus);
+
+			if (bus->intr) {
+				dhdpcie_bus_intr_disable(bus);
+				dhdpcie_free_irq(bus);
+			}
+			dhd_deinit_dpc_histos(bus->dhd);
+
+			dhd_deinit_bus_lp_state_lock(bus);
+			dhd_deinit_bar1_switch_lock(bus);
+			dhd_deinit_backplane_access_lock(bus);
+			dhd_deinit_pwr_req_lock(bus);
+#ifdef PCIE_INB_DW
+			dhd_deinit_dongle_ds_lock(bus);
+#endif /* PCIE_INB_DW */
+#ifdef BCMQT
+			if (IDMA_ACTIVE(bus->dhd)) {
+			/**
+			 * On FPGA during exit path force set "IDMA Control Register"
+			 * to default value 0x0. Otherwise host dongle syc for IDMA fails
+			 * during next IDMA initilization(without system reboot)
+			 */
+				buscorerev = bus->sih->buscorerev;
+				si_corereg(bus->sih, bus->sih->buscoreidx,
+					IDMAControl(buscorerev), ~0, 0);
+			}
+#endif /* BCMQT */
+			/**
+			 * dhdpcie_bus_release_dongle free bus->sih  handle, which is needed to
+			 * access Dongle registers.
+			 * dhd_detach will communicate with dongle to delete flowring ..etc.
+			 * So dhdpcie_bus_release_dongle should be called only after the dhd_detach.
+			 */
+			dhd_detach(bus->dhd);
+			dhdpcie_bus_release_dongle(bus, osh, dongle_isolation, TRUE);
+#if defined(__linux__)
+			DHD_ERROR(("%s: disable pcie dev\n", __FUNCTION__));
+			bcmerror = dhdpcie_bus_disable_device(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_disable_device: %d\n",
+					__FUNCTION__, bcmerror));
+			}
+			DHD_ERROR(("%s: stop host dev dev\n", __FUNCTION__));
+			bcmerror = dhdpcie_bus_stop_host_dev(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_stop_host_dev failed: %d\n",
+					__FUNCTION__, bcmerror));
+			}
+#endif /* __linux__ */
+			dhd_free(bus->dhd);
+			bus->dhd = NULL;
+		}
+		/* unmap the regs and tcm here!! */
+		if (bus->regs) {
+			dhdpcie_bus_reg_unmap(osh, bus->regs, DONGLE_REG_MAP_SIZE);
+			bus->regs = NULL;
+		}
+		if (bus->tcm) {
+			dhdpcie_bus_reg_unmap(osh, bus->tcm, DONGLE_TCM_MAP_SIZE);
+			bus->tcm = NULL;
+		}
+
+		dhdpcie_bus_release_malloc(bus, osh);
+		/* Detach pcie shared structure */
+		if (bus->pcie_sh) {
+			MFREE(osh, bus->pcie_sh, sizeof(pciedev_shared_t));
+		}
+
+		if (bus->console.buf != NULL) {
+			MFREE(osh, bus->console.buf, bus->console.bufsize);
+		}
+
+		/* Finally free bus info */
+		MFREE(osh, bus, sizeof(dhd_bus_t));
+
+		g_dhd_bus = NULL;
+	}
+
+	DHD_TRACE(("%s: Exit\n", __FUNCTION__));
+} /* dhdpcie_bus_release */
+
+void
+dhdpcie_bus_release_dongle(dhd_bus_t *bus, osl_t *osh, bool dongle_isolation, bool reset_flag)
+{
+	DHD_TRACE(("%s: Enter bus->dhd %p bus->dhd->dongle_reset %d \n", __FUNCTION__,
+		bus->dhd, bus->dhd->dongle_reset));
+
+	if ((bus->dhd && bus->dhd->dongle_reset) && reset_flag) {
+		goto fail;
+	}
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s : Skip release dongle due to linkdown \n", __FUNCTION__));
+		goto fail;
+	}
+
+	if (bus->sih) {
+
+		/*
+		 * Perform dongle reset only if dongle isolation is not enabled.
+		 * In android platforms, dongle isolation will be enabled and
+		 * quiescing dongle will be done using DB7 trap.
+		 */
+		if (!dongle_isolation &&
+			bus->dhd && !bus->dhd->is_pcie_watchdog_reset) {
+			dhdpcie_dongle_reset(bus);
+		}
+
+		/* Only for EFI this will be effective */
+		dhdpcie_dongle_flr_or_pwr_toggle(bus);
+
+		if (bus->ltrsleep_on_unload) {
+			si_corereg(bus->sih, bus->sih->buscoreidx,
+				OFFSETOF(sbpcieregs_t, u.pcie2.ltr_state), ~0, 0);
+		}
+
+		if (bus->sih->buscorerev == 13)
+			 pcie_serdes_iddqdisable(bus->osh, bus->sih,
+			                         (sbpcieregs_t *) bus->regs);
+
+		/* For inbuilt drivers pcie clk req will be done by RC,
+		 * so do not do clkreq from dhd
+		 */
+#if defined(linux) || defined(LINUX)
+		if (dhd_download_fw_on_driverload)
+#endif /* linux || LINUX */
+		{
+			/* Disable CLKREQ# */
+			dhdpcie_clkreq(bus->osh, 1, 0);
+		}
+	}
+fail:
+	/* Resources should be freed */
+	if (bus->sih) {
+		si_detach(bus->sih);
+		bus->sih = NULL;
+	}
+	if (bus->vars && bus->varsz) {
+		MFREE(osh, bus->vars, bus->varsz);
+		bus->vars = NULL;
+	}
+
+	DHD_TRACE(("%s Exit\n", __FUNCTION__));
+}
+
+uint32
+dhdpcie_bus_cfg_read_dword(dhd_bus_t *bus, uint32 addr, uint32 size)
+{
+	uint32 data = OSL_PCI_READ_CONFIG(bus->osh, addr, size);
+	return data;
+}
+
+/** 32 bit config write */
+void
+dhdpcie_bus_cfg_write_dword(dhd_bus_t *bus, uint32 addr, uint32 size, uint32 data)
+{
+	OSL_PCI_WRITE_CONFIG(bus->osh, addr, size, data);
+}
+
+void
+dhdpcie_bus_cfg_set_bar0_win(dhd_bus_t *bus, uint32 data)
+{
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCI_BAR0_WIN, 4, data);
+}
+
+void
+dhdpcie_bus_dongle_setmemsize(struct dhd_bus *bus, int mem_size)
+{
+	int32 min_size =  DONGLE_MIN_MEMSIZE;
+	/* Restrict the memsize to user specified limit */
+	DHD_ERROR(("user: Restrict the dongle ram size to %d, min accepted %d max accepted %d\n",
+		mem_size, min_size, (int32)bus->orig_ramsize));
+	if ((mem_size > min_size) &&
+		(mem_size < (int32)bus->orig_ramsize)) {
+		bus->ramsize = mem_size;
+	} else {
+		DHD_ERROR(("%s: Invalid mem_size %d\n", __FUNCTION__, mem_size));
+	}
+}
+
+void
+dhdpcie_bus_release_malloc(dhd_bus_t *bus, osl_t *osh)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd && bus->dhd->dongle_reset)
+		return;
+
+	if (bus->vars && bus->varsz) {
+		MFREE(osh, bus->vars, bus->varsz);
+	}
+
+	DHD_TRACE(("%s: Exit\n", __FUNCTION__));
+	return;
+
+}
+
+/** Stop bus module: clear pending frames, disable data flow */
+void dhd_bus_stop(struct dhd_bus *bus, bool enforce_mutex)
+{
+	unsigned long flags;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (!bus->dhd)
+		return;
+
+	if (bus->dhd->busstate == DHD_BUS_DOWN) {
+		DHD_INFO(("%s: already down by net_dev_reset\n", __FUNCTION__));
+		goto done;
+	}
+
+	DHD_STOP_RPM_TIMER(bus->dhd);
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_INFO(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+	bus->dhd->busstate = DHD_BUS_DOWN;
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	atomic_set(&bus->dhd->block_bus, TRUE);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	dhdpcie_bus_intr_disable(bus);
+
+	if (!bus->is_linkdown) {
+		uint32 status;
+		status = dhdpcie_bus_cfg_read_dword(bus, PCIIntstatus, 4);
+		dhdpcie_bus_cfg_write_dword(bus, PCIIntstatus, 4, status);
+	}
+
+#if defined(linux) || defined(LINUX)
+	if (!dhd_download_fw_on_driverload) {
+		dhd_dpc_kill(bus->dhd);
+	}
+#endif /* linux || LINUX */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	pm_runtime_disable(dhd_bus_to_dev(bus));
+	pm_runtime_set_suspended(dhd_bus_to_dev(bus));
+	pm_runtime_enable(dhd_bus_to_dev(bus));
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	/* Clear rx control and wake any waiters */
+	/* XXX More important in disconnect, but no context? */
+	dhd_os_set_ioctl_resp_timeout(IOCTL_DISABLE_TIMEOUT);
+	dhd_wakeup_ioctl_event(bus->dhd, IOCTL_RETURN_ON_BUS_STOP);
+
+done:
+	return;
+}
+
+#ifdef DEVICE_TX_STUCK_DETECT
+void
+dhd_bus_send_msg_to_daemon(int reason)
+{
+	bcm_to_info_t to_info;
+
+	to_info.magic = BCM_TO_MAGIC;
+	to_info.reason = reason;
+
+	dhd_send_msg_to_daemon(NULL, (void *)&to_info, sizeof(bcm_to_info_t));
+	return;
+}
+
+#define DHD_MEMORY_SET_PATTERN 0xAA
+
+/**
+ * scan the flow rings in active list to check if stuck and notify application
+ * The conditions for warn/stuck detection are
+ * 1. Flow ring is active
+ * 2. There are packets to be consumed by the consumer (wr != rd)
+ * If 1 and 2 are true, then
+ * 3. Warn, if Tx completion is not received for a duration of DEVICE_TX_STUCK_WARN_DURATION
+ * 4. Trap FW, if Tx completion is not received for a duration of DEVICE_TX_STUCK_DURATION
+ */
+static void
+dhd_bus_device_tx_stuck_scan(dhd_bus_t *bus)
+{
+	uint32 tx_cmpl;
+	unsigned long list_lock_flags;
+	unsigned long ring_lock_flags;
+	dll_t *item, *prev;
+	flow_ring_node_t *flow_ring_node;
+	if_flow_lkup_t *if_flow_lkup = (if_flow_lkup_t *)bus->dhd->if_flow_lkup;
+	uint8 ifindex;
+#ifndef FW_HAS_AGING_LOGIC_ALL_IF
+	/**
+	 * Since the aging logic is implemented only for INFRA in FW,
+	 * DHD should monitor only INFRA for stuck detection.
+	 */
+	uint8 role;
+#endif /* FW_HAS_AGING_LOGIC_ALL_IF */
+	bool ring_empty;
+	bool active;
+	uint8 status;
+
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, list_lock_flags);
+
+	for (item = dll_tail_p(&bus->flowring_active_list);
+			!dll_end(&bus->flowring_active_list, item); item = prev) {
+
+		prev = dll_prev_p(item);
+
+		flow_ring_node = dhd_constlist_to_flowring(item);
+		ifindex = flow_ring_node->flow_info.ifindex;
+#ifndef FW_HAS_AGING_LOGIC_ALL_IF
+		role = if_flow_lkup[ifindex].role;
+		if (role != WLC_E_IF_ROLE_STA) {
+			continue;
+		}
+#endif /* FW_HAS_AGING_LOGIC_ALL_IF */
+		DHD_FLOWRING_LOCK(flow_ring_node->lock, ring_lock_flags);
+		tx_cmpl = flow_ring_node->tx_cmpl;
+		active = flow_ring_node->active;
+		status = flow_ring_node->status;
+		ring_empty = dhd_prot_is_h2d_ring_empty(bus->dhd, flow_ring_node->prot_info);
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, ring_lock_flags);
+		/*
+		 * Need not monitor the flow ring if,
+		 * 1. flow ring is empty
+		 * 2. LINK is down
+		 * 3. flow ring is not in FLOW_RING_STATUS_OPEN state
+		 */
+		if ((ring_empty) || !(if_flow_lkup[ifindex].status) ||
+			(status != FLOW_RING_STATUS_OPEN)) {
+			/* reset conters... etc */
+			flow_ring_node->stuck_count = 0;
+			flow_ring_node->tx_cmpl_prev = tx_cmpl;
+			continue;
+		}
+		/**
+		 * DEVICE_TX_STUCK_WARN_DURATION, DEVICE_TX_STUCK_DURATION are integer
+		 * representation of time, to decide if a flow is in warn state or stuck.
+		 *
+		 * flow_ring_node->stuck_count is an integer counter representing how long
+		 * tx_cmpl is not received though there are pending packets in the ring
+		 * to be consumed by the dongle for that particular flow.
+		 *
+		 * This method of determining time elapsed is helpful in sleep/wake scenarios.
+		 * If host sleeps and wakes up, that sleep time is not considered into
+		 * stuck duration.
+		 */
+		if ((tx_cmpl == flow_ring_node->tx_cmpl_prev) && active) {
+
+			flow_ring_node->stuck_count++;
+
+			DHD_ERROR(("%s: flowid: %d tx_cmpl: %u tx_cmpl_prev: %u stuck_count: %d\n",
+				__func__, flow_ring_node->flowid, tx_cmpl,
+				flow_ring_node->tx_cmpl_prev, flow_ring_node->stuck_count));
+			dhd_prot_dump_ring_ptrs(flow_ring_node->prot_info);
+
+			switch (flow_ring_node->stuck_count) {
+				case DEVICE_TX_STUCK_WARN_DURATION:
+					/**
+					 * Notify Device Tx Stuck Notification App about the
+					 * device Tx stuck warning for this flowid.
+					 * App will collect the logs required.
+					 */
+					DHD_ERROR(("stuck warning for flowid: %d sent to app\n",
+						flow_ring_node->flowid));
+					dhd_bus_send_msg_to_daemon(REASON_DEVICE_TX_STUCK_WARNING);
+					break;
+				case DEVICE_TX_STUCK_DURATION:
+					/**
+					 * Notify Device Tx Stuck Notification App about the
+					 * device Tx stuck info for this flowid.
+					 * App will collect the logs required.
+					 */
+					DHD_ERROR(("stuck information for flowid: %d sent to app\n",
+						flow_ring_node->flowid));
+					dhd_bus_send_msg_to_daemon(REASON_DEVICE_TX_STUCK);
+					break;
+				default:
+					break;
+			}
+		} else {
+			flow_ring_node->tx_cmpl_prev = tx_cmpl;
+			flow_ring_node->stuck_count = 0;
+		}
+	}
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, list_lock_flags);
+}
+/**
+ * schedules dhd_bus_device_tx_stuck_scan after DEVICE_TX_STUCK_CKECK_TIMEOUT,
+ * to determine if any flowid is stuck.
+ */
+static void
+dhd_bus_device_stuck_scan(dhd_bus_t *bus)
+{
+	uint32 time_stamp; /* in millisec */
+	uint32 diff;
+
+	/* Need not run the algorith if Dongle has trapped */
+	if (bus->dhd->dongle_trap_occured) {
+		return;
+	}
+	time_stamp = OSL_SYSUPTIME();
+	diff = time_stamp - bus->device_tx_stuck_check;
+	if (diff > DEVICE_TX_STUCK_CKECK_TIMEOUT) {
+		dhd_bus_device_tx_stuck_scan(bus);
+		bus->device_tx_stuck_check = OSL_SYSUPTIME();
+	}
+	return;
+}
+#if defined(ASSOC_CHECK_SR)
+void
+dhd_assoc_check_sr(dhd_pub_t *dhd, bool state)
+{
+	int ret = -1;
+	/* if stuck monitor is disabled */
+	if (!dhd->bus->dev_tx_stuck_monitor) {
+		return;
+	}
+
+	if (state) { /* suspend */
+		dhd->assoc_at_suspend = dhd_is_associated(dhd, 0, NULL);
+		DHD_ERROR(("%s assoc_at_suspend: %d\n", __FUNCTION__, dhd->assoc_at_suspend));
+	} else if ((dhd->assoc_at_suspend) && (!dhd_is_associated(dhd, 0, NULL))) { /* resume */
+		DHD_ERROR(("%s - assoc lost in D3. Issue disassoc\n", __FUNCTION__));
+		ret = dhd_wl_ioctl_cmd(dhd, WLC_DISASSOC, NULL, 0, IOV_SET, 0);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s error issuing disassoc, %d\n", __FUNCTION__, ret));
+		}
+	}
+}
+#endif /* ASSOC_CHECK_SR */
+#endif /* DEVICE_TX_STUCK_DETECT */
+
+/**
+ * Watchdog timer function.
+ * @param dhd   Represents a specific hardware (dongle) instance that this DHD manages
+ */
+bool dhd_bus_watchdog(dhd_pub_t *dhd)
+{
+	unsigned long flags;
+	dhd_bus_t *bus = dhd->bus;
+
+	if (dhd_query_bus_erros(bus->dhd)) {
+		return FALSE;
+	}
+
+	DHD_GENERAL_LOCK(dhd, flags);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhd) ||
+			DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(dhd)) {
+		DHD_GENERAL_UNLOCK(dhd, flags);
+		return FALSE;
+	}
+	DHD_BUS_BUSY_SET_IN_WD(dhd);
+	DHD_GENERAL_UNLOCK(dhd, flags);
+
+	/* Poll for console output periodically */
+	if (dhd->busstate == DHD_BUS_DATA &&
+		dhd->dhd_console_ms != 0 &&
+		DHD_CHK_BUS_NOT_IN_LPS(bus)) {
+		bus->console.count += dhd_watchdog_ms;
+		if (bus->console.count >= dhd->dhd_console_ms) {
+			bus->console.count -= dhd->dhd_console_ms;
+
+			if (MULTIBP_ENAB(bus->sih)) {
+				dhd_bus_pcie_pwr_req(bus);
+			}
+
+			/* Make sure backplane clock is on */
+			if (dhd->db7_trap.fw_db7w_trap_inprogress == FALSE) {
+				if (dhdpcie_bus_readconsole(bus) < 0) {
+					DHD_ERROR(("%s: disable dconpoll\n", __FUNCTION__));
+					dhd->dhd_console_ms = 0; /* On error, stop trying */
+				}
+			}
+
+			if (MULTIBP_ENAB(bus->sih)) {
+				dhd_bus_pcie_pwr_req_clear(bus);
+			}
+		}
+	}
+
+#ifdef DHD_READ_INTSTATUS_IN_DPC
+	if (bus->poll) {
+		bus->ipend = TRUE;
+		bus->dpc_sched = TRUE;
+		dhd_sched_dpc(bus->dhd);     /* queue DPC now!! */
+	}
+#endif /* DHD_READ_INTSTATUS_IN_DPC */
+
+#ifdef DEVICE_TX_STUCK_DETECT
+	if (dhd->bus->dev_tx_stuck_monitor == TRUE) {
+		dhd_bus_device_stuck_scan(bus);
+	}
+#endif /* DEVICE_TX_STUCK_DETECT */
+
+	DHD_GENERAL_LOCK(dhd, flags);
+	DHD_BUS_BUSY_CLEAR_IN_WD(dhd);
+	dhd_os_busbusy_wake(dhd);
+	DHD_GENERAL_UNLOCK(dhd, flags);
+#if (defined(PCIE_OOB) || defined(PCIE_INB_DW))
+	dhd->bus->inb_dw_deassert_cnt += dhd_watchdog_ms;
+	if (dhd->bus->inb_dw_deassert_cnt >=
+		DHD_INB_DW_DEASSERT_MS) {
+		dhd->bus->inb_dw_deassert_cnt = 0;
+		/* Inband device wake is deasserted from DPC context after DS_Exit is received,
+		 * but if at all there is no d2h interrupt received, dpc will not be scheduled
+		 * and inband DW is not deasserted, hence DW is deasserted from watchdog thread
+		 * for every 250ms.
+		 */
+		dhd_bus_dw_deassert(dhd);
+	}
+#endif
+	return TRUE;
+} /* dhd_bus_watchdog */
+
+#if defined(SUPPORT_MULTIPLE_REVISION)
+static int concate_revision_bcm4358(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	uint32 chiprev;
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+	char chipver_tag[20] = "_4358";
+#else
+	char chipver_tag[10] = {0, };
+#endif /* SUPPORT_MULTIPLE_CHIPS */
+
+	chiprev = dhd_bus_chiprev(bus);
+	if (chiprev == 0) {
+		DHD_ERROR(("----- CHIP 4358 A0 -----\n"));
+		strcat(chipver_tag, "_a0");
+	} else if (chiprev == 1) {
+		DHD_ERROR(("----- CHIP 4358 A1 -----\n"));
+#if defined(SUPPORT_MULTIPLE_CHIPS) || defined(SUPPORT_MULTIPLE_MODULE_CIS)
+		strcat(chipver_tag, "_a1");
+#endif /* defined(SUPPORT_MULTIPLE_CHIPS) || defined(SUPPORT_MULTIPLE_MODULE_CIS) */
+	} else if (chiprev == 3) {
+		DHD_ERROR(("----- CHIP 4358 A3 -----\n"));
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+		strcat(chipver_tag, "_a3");
+#endif /* SUPPORT_MULTIPLE_CHIPS */
+	} else {
+		DHD_ERROR(("----- Unknown chip version, ver=%x -----\n", chiprev));
+	}
+
+	strcat(fw_path, chipver_tag);
+
+#if defined(SUPPORT_MULTIPLE_MODULE_CIS) && defined(USE_CID_CHECK)
+	if (chiprev == 1 || chiprev == 3) {
+		int ret = dhd_check_module_b85a();
+		if ((chiprev == 1) && (ret < 0)) {
+			memset(chipver_tag, 0x00, sizeof(chipver_tag));
+			strcat(chipver_tag, "_b85");
+			strcat(chipver_tag, "_a1");
+		}
+	}
+
+	DHD_ERROR(("%s: chipver_tag %s \n", __FUNCTION__, chipver_tag));
+#endif /* defined(SUPPORT_MULTIPLE_MODULE_CIS) && defined(USE_CID_CHECK) */
+
+#if defined(SUPPORT_MULTIPLE_BOARD_REV)
+	if (system_rev >= 10) {
+		DHD_ERROR(("----- Board Rev  [%d]-----\n", system_rev));
+		strcat(chipver_tag, "_r10");
+	}
+#endif /* SUPPORT_MULTIPLE_BOARD_REV */
+	strcat(nv_path, chipver_tag);
+
+	return 0;
+}
+
+static int concate_revision_bcm4359(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	uint32 chip_ver;
+	char chipver_tag[10] = {0, };
+#if defined(SUPPORT_MULTIPLE_MODULE_CIS) && defined(USE_CID_CHECK) && \
+	defined(SUPPORT_BCM4359_MIXED_MODULES)
+	int module_type = -1;
+#endif /* SUPPORT_MULTIPLE_MODULE_CIS && USE_CID_CHECK && SUPPORT_BCM4359_MIXED_MODULES */
+
+	chip_ver = bus->sih->chiprev;
+	if (chip_ver == 4) {
+		DHD_ERROR(("----- CHIP 4359 B0 -----\n"));
+		strncat(chipver_tag, "_b0", strlen("_b0"));
+	} else if (chip_ver == 5) {
+		DHD_ERROR(("----- CHIP 4359 B1 -----\n"));
+		strncat(chipver_tag, "_b1", strlen("_b1"));
+	} else if (chip_ver == 9) {
+		DHD_ERROR(("----- CHIP 4359 C0 -----\n"));
+		strncat(chipver_tag, "_c0", strlen("_c0"));
+	} else {
+		DHD_ERROR(("----- Unknown chip version, ver=%x -----\n", chip_ver));
+		return -1;
+	}
+
+#if defined(SUPPORT_MULTIPLE_MODULE_CIS) && defined(USE_CID_CHECK) && \
+	defined(SUPPORT_BCM4359_MIXED_MODULES)
+	module_type =  dhd_check_module_b90();
+
+	switch (module_type) {
+		case BCM4359_MODULE_TYPE_B90B:
+			strcat(fw_path, chipver_tag);
+			break;
+		case BCM4359_MODULE_TYPE_B90S:
+		default:
+			/*
+			 * .cid.info file not exist case,
+			 * loading B90S FW force for initial MFG boot up.
+			*/
+			if (chip_ver == 5) {
+				strncat(fw_path, "_b90s", strlen("_b90s"));
+			}
+			strcat(fw_path, chipver_tag);
+			strcat(nv_path, chipver_tag);
+			break;
+	}
+#else /* SUPPORT_MULTIPLE_MODULE_CIS && USE_CID_CHECK && SUPPORT_BCM4359_MIXED_MODULES */
+	strcat(fw_path, chipver_tag);
+	strcat(nv_path, chipver_tag);
+#endif /* SUPPORT_MULTIPLE_MODULE_CIS && USE_CID_CHECK && SUPPORT_BCM4359_MIXED_MODULES */
+
+	return 0;
+}
+
+#define NVRAM_FEM_MURATA	"_murata"
+#if defined(DHD_FW_COREDUMP) && defined(DHD_COREDUMP)
+extern char map_path[PATH_MAX];
+#endif /* DHD_FW_COREDUMP && DHD_COREDUMP */
+
+static int
+concate_revision_from_cisinfo(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	int ret = BCME_OK;
+#if defined(SUPPORT_MIXED_MODULES)
+#if defined(USE_CID_CHECK)
+	char module_type[MAX_VNAME_LEN];
+	naming_info_t *info = NULL;
+	bool is_murata_fem = FALSE;
+
+	memset(module_type, 0, sizeof(module_type));
+
+	if (dhd_check_module_bcm(module_type,
+			MODULE_NAME_INDEX_MAX, &is_murata_fem) == BCME_OK) {
+		info = dhd_find_naming_info(bus->dhd, module_type);
+	} else {
+		/* in case of .cid.info doesn't exists */
+		info = dhd_find_naming_info_by_chip_rev(bus->dhd, &is_murata_fem);
+	}
+
+#ifdef BCM4361_CHIP
+	if (bcmstrnstr(nv_path, PATH_MAX,  "_murata", 7)) {
+		is_murata_fem = FALSE;
+	}
+#endif /* BCM4361_CHIP */
+
+	if (info) {
+#ifdef BCM4361_CHIP
+		if (is_murata_fem) {
+			strncat(nv_path, NVRAM_FEM_MURATA, strlen(NVRAM_FEM_MURATA));
+		}
+#endif /* BCM4361_CHIP */
+		strncat(nv_path, info->nvram_ext, strlen(info->nvram_ext));
+#if defined(SUPPORT_MULTIPLE_NVRAM) || defined(SUPPORT_MULTIPLE_CLMBLOB)
+		dhd_set_platform_ext_name_for_chip_version(info->nvram_ext);
+#endif /* SUPPORT_MULTIPLE_NVRAM || SUPPORT_MULTIPLE_CLMBLOB */
+		strncat(fw_path, info->fw_ext, strlen(info->fw_ext));
+#if defined(DHD_COREDUMP) && defined(SUPPORT_MULTIPLE_REVISION_MAP)
+		if (!bcmstrnstr(map_path, PATH_MAX, info->fw_ext, strlen(info->fw_ext)))
+			strncat(map_path, info->fw_ext, strlen(info->fw_ext));
+#endif /* DHD_COREDUMP  && SUPPORT_MULTIPLE_REVISION_MAP */
+	} else {
+		DHD_ERROR(("%s:failed to find extension for nvram and firmware\n", __FUNCTION__));
+		ret = BCME_ERROR;
+	}
+#endif /* USE_CID_CHECK */
+#ifdef USE_DIRECT_VID_TAG
+	int revid = bus->sih->chiprev;
+	unsigned char chipstr[MAX_VID_LEN];
+
+	memset(chipstr, 0, sizeof(chipstr));
+	snprintf(chipstr, sizeof(chipstr), "_4389");
+
+	/* write chipstr/vid into nvram tag */
+	ret = concate_nvram_by_vid(bus->dhd, nv_path, chipstr);
+	/* write chiprev into FW tag */
+	if (ret == BCME_OK) {
+		if (revid == 1) {
+			strncat(fw_path, B0_REV, strlen(fw_path));
+			DHD_ERROR(("%s: fw_path : %s\n", __FUNCTION__, fw_path));
+		} else if (revid == 2) {
+			strncat(fw_path, C0_REV, strlen(fw_path));
+			DHD_ERROR(("%s: fw_path : %s\n", __FUNCTION__, fw_path));
+		} else if (revid == 3) {
+			strncat(fw_path, C1_REV, strlen(fw_path));
+			DHD_ERROR(("%s: fw_path : %s\n", __FUNCTION__, fw_path));
+
+		} else {
+			DHD_ERROR(("%s: INVALID CHIPREV %d\n", __FUNCTION__, revid));
+		}
+	}
+#endif /* USE_DIRECT_VID_TAG */
+#else /* SUPPORT_MIXED_MODULE */
+	char chipver_tag[10] = {0, };
+
+	strcat(fw_path, chipver_tag);
+	strcat(nv_path, chipver_tag);
+#endif /* SUPPORT_MIXED_MODULE */
+
+	return ret;
+}
+
+#ifdef CONCAT_DEF_REV_FOR_NOMATCH_VID
+#define DEF_REVISION "_c0"
+static int
+vendor_concat_revision(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	int ret = BCME_OK;
+	char *tag = DEF_REVISION;
+
+	DHD_ERROR(("%s:failed to find extension, use %s as default\n", __FUNCTION__, DEF_REVISION));
+	/* Failed to find info, use _c0 as default */
+	strlcat(nv_path, tag, PATH_MAX);
+	strlcat(fw_path, tag, PATH_MAX);
+#if defined(DHD_COREDUMP) && defined(SUPPORT_MULTIPLE_REVISION_MAP)
+	if (!bcmstrnstr(map_path, PATH_MAX, tag, strlen(tag))) {
+		strlcat(map_path, tag, PATH_MAX);
+	}
+#endif /* DHD_COREDUMP && SUPPORT_MULTIPLE_REVISION_MAP */
+	return ret;
+}
+#endif /* CONCAT_DEF_REV_FOR_NOMATCH_VID */
+
+int
+concate_revision(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	int res = 0;
+
+	if (!bus || !bus->sih) {
+		DHD_ERROR(("%s:Bus is Invalid\n", __FUNCTION__));
+		return -1;
+	}
+
+	if (!fw_path || !nv_path) {
+		DHD_ERROR(("fw_path or nv_path is null.\n"));
+		return res;
+	}
+
+	switch (si_chipid(bus->sih)) {
+
+	case BCM43569_CHIP_ID:
+	case BCM4358_CHIP_ID:
+		res = concate_revision_bcm4358(bus, fw_path, nv_path);
+		break;
+	case BCM4355_CHIP_ID:
+	case BCM4359_CHIP_ID:
+		res = concate_revision_bcm4359(bus, fw_path, nv_path);
+		break;
+	case BCM4361_CHIP_ID:
+	case BCM4347_CHIP_ID:
+	case BCM4375_CHIP_ID:
+	case BCM4389_CHIP_ID:
+		res = concate_revision_from_cisinfo(bus, fw_path, nv_path);
+		break;
+	default:
+		DHD_ERROR(("REVISION SPECIFIC feature is not required\n"));
+		return res;
+	}
+
+#ifdef CONCAT_DEF_REV_FOR_NOMATCH_VID
+	if (res != BCME_OK) {
+		res = vendor_concat_revision(bus, fw_path, nv_path);
+	}
+#endif /* CONCAT_DEF_REV_FOR_NOMATCH_VID */
+	return res;
+}
+#endif /* SUPPORT_MULTIPLE_REVISION */
+
+uint16
+dhd_get_chipid(struct dhd_bus *bus)
+{
+	if (bus && bus->sih) {
+		return (uint16)si_chipid(bus->sih);
+	} else if (bus && bus->regs) {
+		chipcregs_t *cc = (chipcregs_t *)bus->regs;
+		uint w, chipid;
+
+		/* Set bar0 window to si_enum_base */
+		dhdpcie_bus_cfg_set_bar0_win(bus, si_enum_base(0));
+
+		w = R_REG(bus->osh, &cc->chipid);
+		chipid = w & CID_ID_MASK;
+
+		return (uint16)chipid;
+	} else {
+		return 0;
+	}
+}
+
+uint16
+dhd_get_chiprev(struct dhd_bus *bus)
+{
+	if (bus && bus->sih) {
+		return bus->sih->chiprev;
+	} else if (bus && bus->regs) {
+		chipcregs_t *cc = (chipcregs_t *)bus->regs;
+		uint w, chiprev;
+
+		/* Set bar0 window to si_enum_base */
+		dhdpcie_bus_cfg_set_bar0_win(bus, si_enum_base(0));
+
+		w = R_REG(bus->osh, &cc->chipid);
+		chiprev = (w & CID_REV_MASK) >> CID_REV_SHIFT;
+
+		return (uint16)chiprev;
+	} else {
+		return 0xffff;
+	}
+}
+
+/**
+ * Loads firmware given by caller supplied path and nvram image into PCIe dongle.
+ *
+ * BCM_REQUEST_FW specific :
+ * Given the chip type, determines the to be used file paths within /lib/firmware/brcm/ containing
+ * firmware and nvm for that chip. If the download fails, retries download with a different nvm file
+ *
+ * BCMEMBEDIMAGE specific:
+ * If bus->fw_path is empty, or if the download of bus->fw_path failed, firmware contained in header
+ * file will be used instead.
+ *
+ * @return BCME_OK on success
+ */
+int
+dhd_bus_download_firmware(struct dhd_bus *bus, osl_t *osh,
+                          char *pfw_path, char *pnv_path,
+                          char *pclm_path, char *pconf_path)
+{
+	int ret;
+
+	bus->fw_path = pfw_path;
+	bus->nv_path = pnv_path;
+	bus->dhd->clm_path = pclm_path;
+	bus->dhd->conf_path = pconf_path;
+
+#if defined(SUPPORT_MULTIPLE_REVISION)
+	if (concate_revision(bus, bus->fw_path, bus->nv_path) != 0) {
+		DHD_ERROR(("%s: fail to concatnate revison \n",
+			__FUNCTION__));
+		/* Proceed if SUPPORT_MULTIPLE_CHIPS is enabled */
+#ifndef SUPPORT_MULTIPLE_CHIPS
+		return BCME_BADARG;
+#endif /* !SUPPORT_MULTIPLE_CHIPS */
+	}
+#endif /* SUPPORT_MULTIPLE_REVISION */
+
+#if defined(SUPPORT_MULTIPLE_NVRAM)
+	if (dhd_get_platform_naming_for_nvram_clmblob_file(NVRAM, bus->nv_path) != BCME_OK) {
+		DHD_ERROR(("%s: Platform's NVRAMs file don't exist."
+			" Download default NVRAM file.\n", __FUNCTION__));
+	}
+#endif /* SUPPORT_MULTIPLE_NVRAM */
+
+#if defined(SUPPORT_MULTIPLE_BOARD_REVISION)
+	concate_custom_board_revision(bus->nv_path);
+#endif /* SUPPORT_MULTIPLE_BOARD_REVISION */
+
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+	dhd_set_blob_support(bus->dhd, bus->fw_path);
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+
+	DHD_ERROR(("%s: firmware path=%s, nvram path=%s\n",
+		__FUNCTION__, bus->fw_path, bus->nv_path));
+#if defined(LINUX) || defined(linux)
+	dhdpcie_dump_resource(bus);
+#endif /* LINUX || linux */
+
+	ret = dhdpcie_download_firmware(bus, osh);
+
+	return ret;
+}
+
+void
+dhd_set_bus_params(struct dhd_bus *bus)
+{
+	struct dhd_conf *conf = bus->dhd->conf;
+
+	if (conf->dhd_poll >= 0) {
+		bus->poll = conf->dhd_poll;
+		if (!bus->pollrate)
+			bus->pollrate = 1;
+		printf("%s: set polling mode %d\n", __FUNCTION__, conf->dhd_poll);
+	}
+	if (conf->d2h_intr_control >= 0)
+		bus->d2h_intr_control = conf->d2h_intr_control;
+	printf("d2h_intr_method -> %s(%d); d2h_intr_control -> %s(%d)\n",
+		bus->d2h_intr_method ? "PCIE_MSI" : "PCIE_INTX", bus->d2h_intr_method,
+		bus->d2h_intr_control ? "HOST_IRQ" : "D2H_INTMASK", bus->d2h_intr_control);
+
+	if (conf->aspm != -1) {
+		bool aspm = conf->aspm ? TRUE : FALSE;
+		dhd_bus_aspm_enable_rc_ep(bus, aspm);
+	}
+	if (conf->l1ss != -1) {
+		bool l1ss = conf->l1ss ? TRUE : FALSE;
+		dhd_bus_l1ss_enable_rc_ep(bus, l1ss);
+	}
+}
+
+/**
+ * Loads firmware given by 'bus->fw_path' into PCIe dongle.
+ *
+ * BCM_REQUEST_FW specific :
+ * Given the chip type, determines the to be used file paths within /lib/firmware/brcm/ containing
+ * firmware and nvm for that chip. If the download fails, retries download with a different nvm file
+ *
+ * BCMEMBEDIMAGE specific:
+ * If bus->fw_path is empty, or if the download of bus->fw_path failed, firmware contained in header
+ * file will be used instead.
+ *
+ * @return BCME_OK on success
+ */
+#if defined(BCM_REQUEST_FW)
+extern char clm_path[MOD_PARAM_PATHLEN];
+#endif /* BCM_REQUEST_FW */
+static int
+dhdpcie_download_firmware(struct dhd_bus *bus, osl_t *osh)
+{
+	int ret = 0;
+#if defined(BCM_REQUEST_FW)
+	uint chipid = bus->sih->chip;
+	uint revid = bus->sih->chiprev;
+#if defined(DHD_LINUX_STD_FW_API)
+	char fw_path[64] = "brcm/bcm";	/* path to firmware image */
+#else
+	char fw_path[64] = "/lib/firmware/brcm/bcm";	/* path to firmware image */
+#endif /* DHD_LINUX_STD_FW_API */
+	char nv_path[64];		/* path to nvram vars file */
+
+	bus->fw_path = fw_path;
+	bus->nv_path = nv_path;
+	switch (chipid) {
+	case BCM43570_CHIP_ID:
+		bcmstrncat(fw_path, "43570", 5);
+		switch (revid) {
+		case 0:
+			bcmstrncat(fw_path, "a0", 2);
+			break;
+		case 2:
+			bcmstrncat(fw_path, "a2", 2);
+			break;
+		default:
+			DHD_ERROR(("%s: revid is not found %x\n", __FUNCTION__,
+			revid));
+			break;
+		}
+		break;
+	case BCM4381_CHIP_ID:
+		bcmstrncat(fw_path, "4381", 4);
+		switch (revid) {
+		case 1:
+			bcmstrncat(fw_path, "a0-pcie", 7);
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		DHD_ERROR(("%s: unsupported device %x\n", __FUNCTION__,
+		chipid));
+		return 0;
+	}
+	/* load board specific nvram file */
+	snprintf(bus->nv_path, sizeof(nv_path), "%s.nvm", fw_path);
+	/* load signature */
+	snprintf(bus->fwsig_filename, sizeof(bus->fwsig_filename), "%s.sig", fw_path);
+	/* load clm_blob */
+	snprintf(clm_path, sizeof(clm_path), "%s.clm_blob", fw_path);
+	/* load firmware */
+	snprintf(bus->fw_path, sizeof(fw_path), "%s.bin", fw_path);
+#endif /* BCM_REQUEST_FW */
+
+	DHD_OS_WAKE_LOCK(bus->dhd);
+
+	dhd_conf_set_path_params(bus->dhd, bus->fw_path, bus->nv_path);
+	dhd_set_bus_params(bus);
+
+	ret = _dhdpcie_download_firmware(bus);
+
+	DHD_OS_WAKE_UNLOCK(bus->dhd);
+	return ret;
+} /* dhdpcie_download_firmware */
+
+#ifdef DHD_LINUX_STD_FW_API
+static int
+dhdpcie_download_code_file(struct dhd_bus *bus, char *pfw_path)
+{
+	int bcmerror = BCME_ERROR;
+	int offset = 0;
+	int len = 0;
+	bool store_reset;
+	int offset_end = bus->ramsize;
+	const struct firmware *fw = NULL;
+	int buf_offset = 0, residual_len = 0;
+#ifdef CHECK_DOWNLOAD_FW
+	uint8 *memptr_tmp = NULL; // terence: check downloaded firmware is correct
+#endif
+
+#if defined(DHD_FW_MEM_CORRUPTION)
+	if (dhd_bus_get_fw_mode(bus->dhd) == DHD_FLAG_MFG_MODE) {
+		dhd_tcm_test_enable = TRUE;
+	} else {
+		dhd_tcm_test_enable = FALSE;
+	}
+#endif /* DHD_FW_MEM_CORRUPTION */
+	DHD_ERROR(("%s: dhd_tcm_test_enable %u, dhd_tcm_test_status %u, dhd_tcm_test_mode %u\n",
+		__FUNCTION__, dhd_tcm_test_enable, dhd_tcm_test_status, dhd_tcm_test_mode));
+
+	if (dhd_tcm_test_enable && dhd_tcm_test_mode != TCM_TEST_MODE_DISABLE) {
+		if (((dhd_tcm_test_mode == TCM_TEST_MODE_ONCE) &&
+				(dhd_tcm_test_status == TCM_TEST_NOT_RUN)) ||
+				(dhd_tcm_test_mode == TCM_TEST_MODE_ALWAYS)) {
+			if (dhd_tcm_test_status != TCM_TEST_PASSED) {
+				if (dhd_bus_tcm_test(bus) == FALSE) {
+					DHD_ERROR(("dhd_bus_tcm_test failed\n"));
+					dhd_tcm_test_status = TCM_TEST_FAILED;
+					bcmerror = BCME_ERROR;
+					goto err;
+				} else {
+					dhd_tcm_test_status = TCM_TEST_PASSED;
+				}
+			}
+		}
+	}
+
+	DHD_ERROR(("%s: download firmware %s\n", __FUNCTION__, pfw_path));
+
+	/* check if CR4/CA7 */
+	store_reset = (si_setcore(bus->sih, ARMCR4_CORE_ID, 0) ||
+			si_setcore(bus->sih, ARMCA7_CORE_ID, 0));
+
+	bcmerror = dhd_os_get_img_fwreq(&fw, bus->fw_path);
+	if (bcmerror < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			bcmerror));
+		goto err;
+	}
+#ifdef CHECK_DOWNLOAD_FW
+	if (bus->dhd->conf->fwchk) {
+		memptr_tmp = MALLOC(bus->dhd->osh, MEMBLOCK + DHD_SDALIGN);
+		if (memptr_tmp == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__, MEMBLOCK));
+			goto err;
+		}
+	}
+#endif
+
+	bus->fw_download_len = fw->size;
+	bus->fw_download_addr = bus->dongle_ram_base;
+	residual_len = fw->size;
+	while (residual_len) {
+		len = MIN(residual_len, MEMBLOCK);
+
+		/* if address is 0, store the reset instruction to be written in 0 */
+		if (store_reset) {
+			ASSERT(offset == 0);
+			bus->resetinstr = *(((const uint32*)fw->data + buf_offset));
+			/* Add start of RAM address to the address given by user */
+			offset += bus->dongle_ram_base;
+			offset_end += offset;
+			store_reset = FALSE;
+		}
+
+		/* Ignore compiler warnings due to -Werror=cast-qual */
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		bcmerror = dhdpcie_bus_membytes(bus, TRUE, offset,
+			(uint8 *)fw->data + buf_offset, len);
+		GCC_DIAGNOSTIC_POP();
+
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+				__FUNCTION__, bcmerror, MEMBLOCK, offset));
+			goto err;
+		}
+#ifdef CHECK_DOWNLOAD_FW
+		if (bus->dhd->conf->fwchk) {
+			bcmerror = dhdpcie_bus_membytes(bus, FALSE, offset, memptr_tmp, len);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+				        __FUNCTION__, bcmerror, MEMBLOCK, offset));
+				goto err;
+			}
+			if (memcmp(memptr_tmp, (uint8 *)fw->data + buf_offset, len)) {
+				DHD_ERROR(("%s: Downloaded image is corrupted at 0x%08x\n", __FUNCTION__, offset));
+				bcmerror = BCME_ERROR;
+				goto err;
+			} else
+				DHD_INFO(("%s: Download, Upload and compare succeeded.\n", __FUNCTION__));
+		}
+#endif
+		offset += MEMBLOCK;
+
+		if (offset >= offset_end) {
+			DHD_ERROR(("%s: invalid address access to %x (offset end: %x)\n",
+				__FUNCTION__, offset, offset_end));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+		residual_len -= len;
+		buf_offset += len;
+	}
+err:
+#ifdef CHECK_DOWNLOAD_FW
+	if (memptr_tmp)
+		MFREE(bus->dhd->osh, memptr_tmp, MEMBLOCK + DHD_SDALIGN);
+#endif
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+	return bcmerror;
+} /* dhdpcie_download_code_file */
+
+#else
+
+/**
+ * Downloads a file containing firmware into dongle memory. In case of a .bea file, the DHD
+ * is updated with the event logging partitions within that file as well.
+ *
+ * @param pfw_path    Path to .bin or .bea file
+ */
+static int
+dhdpcie_download_code_file(struct dhd_bus *bus, char *pfw_path)
+{
+	int bcmerror = BCME_ERROR;
+	int offset = 0;
+	int len = 0;
+	bool store_reset;
+	void *filep = NULL;
+	uint8 *memblock = NULL, *memptr = NULL;
+#ifdef CHECK_DOWNLOAD_FW
+	uint8 *memptr_tmp = NULL; // terence: check downloaded firmware is correct
+#endif
+	int offset_end = bus->ramsize;
+	uint32 file_size = 0, read_len = 0;
+	fwpkg_info_t *fwpkg;
+
+#if defined(CACHE_FW_IMAGES)
+	int buf_offset, total_len, residual_len;
+	char * dnld_buf = NULL;
+#endif /* CACHE_FW_IMAGES */
+
+#if defined(linux) || defined(LINUX)
+#if defined(DHD_FW_MEM_CORRUPTION)
+	if (dhd_bus_get_fw_mode(bus->dhd) == DHD_FLAG_MFG_MODE) {
+		dhd_tcm_test_enable = TRUE;
+	} else {
+		dhd_tcm_test_enable = FALSE;
+	}
+#endif /* DHD_FW_MEM_CORRUPTION */
+	DHD_ERROR(("%s: dhd_tcm_test_enable %u, dhd_tcm_test_status %u\n", __FUNCTION__,
+		dhd_tcm_test_enable, dhd_tcm_test_status));
+	/* run TCM test if not passed yet */
+	if (dhd_tcm_test_enable && dhd_tcm_test_status != TCM_TEST_PASSED) {
+		if (dhd_bus_tcm_test(bus) == FALSE) {
+			DHD_ERROR(("dhd_bus_tcm_test failed\n"));
+			dhd_tcm_test_status = TCM_TEST_FAILED;
+			bcmerror = BCME_ERROR;
+			goto err;
+		} else {
+			dhd_tcm_test_status = TCM_TEST_PASSED;
+		}
+	}
+#endif /* LINUX || linux */
+	DHD_ERROR_MEM(("%s: download firmware %s\n", __FUNCTION__, pfw_path));
+
+	bcmerror = fwpkg_init(&bus->fwpkg, pfw_path);
+	if (bcmerror == BCME_ERROR) {
+		printf("%s: fwpkg_init failed %s\n", __FUNCTION__, pfw_path);
+		goto err;
+	}
+	fwpkg = &bus->fwpkg;
+	/* Should succeed in opening image if it is actually given through registry
+	 * entry or in module param.
+	 */
+	bcmerror = fwpkg_open_firmware_img(fwpkg, pfw_path, &filep);
+	if (bcmerror == BCME_ERROR) {
+		printf("%s: Open firmware file failed %s\n", __FUNCTION__, pfw_path);
+		goto err;
+	}
+
+	if (bcmerror == BCME_UNSUPPORTED) {
+		file_size = fwpkg->file_size;
+		DHD_ERROR(("%s Using SINGLE image (size %d)\n",
+			__FUNCTION__, file_size));
+	} else {
+		file_size = fwpkg_get_firmware_img_size(fwpkg);
+		strlcpy(bus->fwsig_filename, pfw_path, sizeof(bus->fwsig_filename));
+		DHD_ERROR(("%s Using COMBINED image (size %d)\n",
+			__FUNCTION__, file_size));
+	}
+	bus->fw_download_len = file_size;
+	bus->fw_download_addr = bus->dongle_ram_base;
+
+	memptr = memblock = MALLOC(bus->dhd->osh, MEMBLOCK + DHD_SDALIGN);
+	if (memblock == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__, MEMBLOCK));
+		bcmerror = BCME_NOMEM;
+		goto err;
+	}
+#ifdef CHECK_DOWNLOAD_FW
+	if (bus->dhd->conf->fwchk) {
+		memptr_tmp = MALLOC(bus->dhd->osh, MEMBLOCK + DHD_SDALIGN);
+		if (memptr_tmp == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__, MEMBLOCK));
+			goto err;
+		}
+	}
+#endif
+	if ((uint32)(uintptr)memblock % DHD_SDALIGN) {
+		memptr += (DHD_SDALIGN - ((uint32)(uintptr)memblock % DHD_SDALIGN));
+	}
+
+	/* check if CR4/CA7 */
+	store_reset = (si_setcore(bus->sih, ARMCR4_CORE_ID, 0) ||
+			si_setcore(bus->sih, ARMCA7_CORE_ID, 0));
+#if defined(CACHE_FW_IMAGES)
+	total_len = bus->ramsize;
+	dhd_os_close_image(filep);
+	filep = NULL;
+	buf_offset = 0;
+	bcmerror = dhd_get_download_buffer(bus->dhd, pfw_path, FW, &dnld_buf, &total_len);
+	if (bcmerror != BCME_OK) {
+		DHD_ERROR(("%s: dhd_get_download_buffer failed (%d)\n", __FUNCTION__, bcmerror));
+		goto err;
+	}
+	residual_len = total_len;
+	/* Download image with MEMBLOCK size */
+	while (residual_len)
+#else
+	/* Download image with MEMBLOCK size */
+	while ((len = dhd_os_get_image_block((char*)memptr, MEMBLOCK, filep)))
+#endif /* CACHE_FW_IMAGES */
+	{
+#if defined(CACHE_FW_IMAGES)
+		len = MIN(residual_len, MEMBLOCK);
+		memcpy(memptr, dnld_buf + buf_offset, len);
+		residual_len -= len;
+		buf_offset += len;
+#else
+		if (len < 0) {
+			DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+#endif /* CACHE_FW_IMAGES */
+
+		read_len += len;
+		if (read_len > file_size) {
+			DHD_ERROR(("%s: WARNING! reading beyond EOF, len=%d; read_len=%u;"
+				" file_size=%u truncating len to %d \n", __FUNCTION__,
+				len, read_len, file_size, (len - (read_len - file_size))));
+			len -= (read_len - file_size);
+		}
+
+		/* if address is 0, store the reset instruction to be written in 0 */
+		if (store_reset) {
+			ASSERT(offset == 0);
+			bus->resetinstr = *(((uint32*)memptr));
+			/* Add start of RAM address to the address given by user */
+			offset += bus->dongle_ram_base;
+			offset_end += offset;
+			store_reset = FALSE;
+		}
+
+		bcmerror = dhdpcie_bus_membytes(bus, TRUE, offset, (uint8 *)memptr, len);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+				__FUNCTION__, bcmerror, MEMBLOCK, offset));
+			goto err;
+		}
+
+#ifdef CHECK_DOWNLOAD_FW
+		if (bus->dhd->conf->fwchk) {
+			bcmerror = dhdpcie_bus_membytes(bus, FALSE, offset, memptr_tmp, len);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+				        __FUNCTION__, bcmerror, MEMBLOCK, offset));
+				goto err;
+			}
+			if (memcmp(memptr_tmp, memptr, len)) {
+				DHD_ERROR(("%s: Downloaded image is corrupted at 0x%08x\n", __FUNCTION__, offset));
+				bcmerror = BCME_ERROR;
+				goto err;
+			} else
+				DHD_INFO(("%s: Download, Upload and compare succeeded.\n", __FUNCTION__));
+		}
+#endif
+		offset += MEMBLOCK;
+
+		if (offset >= offset_end) {
+			DHD_ERROR(("%s: invalid address access to %x (offset end: %x)\n",
+				__FUNCTION__, offset, offset_end));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+
+		if (read_len >= file_size) {
+			break;
+		}
+	}
+#if defined(DHD_DEBUG) && defined(DHD_FW_MEM_CORRUPTION)
+	/* Upload and compare the downloaded code */
+	{
+		unsigned char *ulblock = NULL;
+		unsigned int uploded_len;
+		read_len = 0;
+		uploded_len = 0;
+		bcmerror = BCME_ERROR;
+		ulblock = MALLOC(bus->dhd->osh, MEMBLOCK);
+		if (ulblock == NULL)
+			goto upload_err;
+		/* Close and re-open the image file to reset the file pointer.
+		*/
+		dhd_os_close_image1(bus->dhd, imgbuf);
+		imgbuf = dhd_os_open_image1(bus->dhd, pfw_path);
+		if (imgbuf == NULL) {
+			goto upload_err;
+		}
+		/* Upload image to verify downloaded contents. */
+		offset = bus->dongle_ram_base;
+		offset_end = offset + bus->ramsize;
+		/* Upload image with MEMBLOCK size */
+		while ((len = dhd_os_get_image_block((char*)memptr, MEMBLOCK, imgbuf))) {
+			if (len < 0) {
+				DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+				bcmerror = BCME_ERROR;
+				goto upload_err;
+			}
+
+			read_len += len;
+			if (read_len > file_size) {
+				DHD_ERROR(("%s: WARNING! reading beyond EOF, len=%d; read_len=%u;"
+					" file_size=%u truncating len to %d \n", __FUNCTION__,
+					len, read_len, file_size, (len - (read_len - file_size))));
+				len -= (read_len - file_size);
+			}
+			memset(ulblock, 0xaa, MEMBLOCK);
+			bcmerror = dhdpcie_bus_membytes(bus, FALSE, offset,
+				(uint8 *)ulblock, len);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+					__FUNCTION__, bcmerror, len, offset));
+				goto upload_err;
+			}
+			if (memcmp(memptr, ulblock, len)) {
+				DHD_ERROR(("%s: Downloaded image is corrupted! \n",
+					__FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto upload_err;
+			}
+
+			uploded_len += len;
+			offset += MEMBLOCK;
+
+			if (offset >= offset_end) {
+				DHD_ERROR(("%s: invalid address access to %x (offset end: %x)\n",
+					__FUNCTION__, offset, offset_end));
+				bcmerror = BCME_ERROR;
+				goto upload_err;
+			}
+
+			if (read_len >= file_size) {
+				break;
+			}
+		}
+		DHD_INFO(("%s: Download, Upload and compare succeeded.\n", __FUNCTION__));
+
+upload_err:
+		if (ulblock)
+			MFREE(bus->dhd->osh, ulblock, MEMBLOCK);
+	}
+#endif /* DHD_DEBUG && DHD_FW_MEM_CORRUPTION */
+err:
+	if (memblock) {
+		MFREE(bus->dhd->osh, memblock, MEMBLOCK + DHD_SDALIGN);
+#ifdef CHECK_DOWNLOAD_FW
+		if (memptr_tmp)
+			MFREE(bus->dhd->osh, memptr_tmp, MEMBLOCK + DHD_SDALIGN);
+#endif
+	}
+#if defined(CACHE_FW_IMAGES)
+	if (dnld_buf) {
+		dhd_free_download_buffer(bus->dhd, dnld_buf, total_len);
+	}
+#endif /* CACHE_FW_IMAGES */
+
+	if (filep) {
+		dhd_os_close_image1(bus->dhd, filep);
+	}
+
+	return bcmerror;
+} /* dhdpcie_download_code_file */
+#endif /* DHD_LINUX_STD_FW_API */
+
+static int
+dhdpcie_download_nvram(struct dhd_bus *bus)
+{
+	int bcmerror = BCME_ERROR;
+	uint len, memblock_len = 0;
+	char * memblock = NULL;
+	char *bufp;
+	char *pnv_path;
+	bool nvram_file_exists;
+	bool nvram_uefi_exists = FALSE;
+	bool local_alloc = FALSE;
+#ifdef SUPPORT_OTA_UPDATE
+	ota_update_info_t *ota_info = &bus->dhd->ota_update_info;
+#endif /* SUPPORT_OTA_UPDATE */
+	pnv_path = bus->nv_path;
+
+	nvram_file_exists = ((pnv_path != NULL) && (pnv_path[0] != '\0'));
+
+	/* First try UEFI */
+	len = MAX_NVRAMBUF_SIZE;
+	bcmerror = dhd_get_download_buffer(bus->dhd, NULL, NVRAM, &memblock, (int *)&len);
+
+	/* If UEFI empty, then read from file system */
+	if ((len <= 0) || (memblock == NULL)) {
+
+		if (nvram_file_exists) {
+			len = MAX_NVRAMBUF_SIZE;
+			bcmerror = dhd_get_download_buffer(bus->dhd, pnv_path, NVRAM, &memblock,
+					(int *)&len);
+			if ((len <= 0 || len > MAX_NVRAMBUF_SIZE)) {
+				goto err;
+			}
+		}
+#ifdef BCM_ROUTER_DHD
+		else if (bus->nvram_params_len) {
+			memblock = MALLOCZ(bus->dhd->osh, MAX_NVRAMBUF_SIZE);
+			if (memblock == NULL) {
+				DHD_ERROR(("%s: Failed to allocate memory %d bytes\n",
+					__FUNCTION__, MAX_NVRAMBUF_SIZE));
+				goto err;
+			}
+			local_alloc = TRUE;
+			/* nvram is string with null terminated. cannot use strlen */
+			len = bus->nvram_params_len;
+			ASSERT(len <= MAX_NVRAMBUF_SIZE);
+			memcpy(memblock, bus->nvram_params, len);
+		}
+#endif /* BCM_ROUTER_DHD */
+		else {
+			/* For SROM OTP no external file or UEFI required */
+			bcmerror = BCME_OK;
+		}
+	} else {
+		nvram_uefi_exists = TRUE;
+	}
+	memblock_len = MAX_NVRAMBUF_SIZE;
+
+	DHD_ERROR_MEM(("%s: dhd_get_download_buffer len %d\n", __FUNCTION__, len));
+
+	if (len > 0 && len < MAX_NVRAMBUF_SIZE && memblock != NULL) {
+		bufp = (char *) memblock;
+
+#ifdef CACHE_FW_IMAGES
+		if (bus->processed_nvram_params_len) {
+			len = bus->processed_nvram_params_len;
+		}
+
+		if (!bus->processed_nvram_params_len) {
+			bufp[len-1] = 0;
+			if (nvram_uefi_exists || nvram_file_exists) {
+				len = process_nvram_vars(bufp, len);
+				bus->processed_nvram_params_len = len;
+			}
+		} else
+#else
+		{
+			bufp[len-1] = 0;
+			if (nvram_uefi_exists || nvram_file_exists) {
+				len = process_nvram_vars(bufp, len);
+			}
+		}
+#endif /* CACHE_FW_IMAGES */
+
+		DHD_ERROR_MEM(("%s: process_nvram_vars len %d\n", __FUNCTION__, len));
+
+		if (len % 4) {
+			len += 4 - (len % 4);
+		}
+		bufp += len;
+		*bufp++ = 0;
+		if (len)
+			bcmerror = dhdpcie_downloadvars(bus, memblock, len + 1);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error downloading vars: %d\n",
+				__FUNCTION__, bcmerror));
+		}
+	}
+
+err:
+	if (memblock) {
+#ifdef SUPPORT_OTA_UPDATE
+		if (ota_info->nvram_len) {
+			MFREE(bus->dhd->osh, ota_info->nvram_buf, ota_info->nvram_len);
+			ota_info->nvram_len = 0;
+		}
+		else
+#endif /* SUPPORT_OTA_UPDATE */
+		{
+			if (local_alloc) {
+				MFREE(bus->dhd->osh, memblock, MAX_NVRAMBUF_SIZE);
+			} else {
+				dhd_free_download_buffer(bus->dhd, memblock, memblock_len);
+			}
+		}
+	}
+
+	return bcmerror;
+}
+
+#if defined(BCMEMBEDIMAGE) && defined(BCM_ROUTER_DHD)
+
+#ifdef DLIMAGE_43602a1
+#define CHIPID_43602            BCM43602_CHIP_ID
+#define CHIPID_43462            BCM43462_CHIP_ID
+#define CHIPID_43522            BCM43522_CHIP_ID
+#define CHIP_43602_CHIPREV_A0   0
+#define CHIP_43602_CHIPREV_A1   1
+#define CHIP_43602_PKG_OPT      1
+#endif
+
+#define CHIPID_NONE            -1
+
+struct fd_chip_image
+{
+	unsigned char *dlarray;
+	int dlimagesize;
+	char *dlimagename;
+	char *dlimagever;
+	char *dliamgedate;
+} static chip_dl_image_array[] __initdata =
+{
+#ifdef DLIMAGE_43602a1
+	{dlarray_43602a1, sizeof(dlarray_43602a1), dlimagename_43602a1,
+	dlimagever_43602a1, dlimagedate_43602a1},
+#endif
+	/* {image attributes for other chips, only if image is compiled} */
+};
+
+enum chip_image_rev
+{
+#ifdef DLIMAGE_43602a1
+	CHIP_43602_A1_CHIP_IMAGE,
+#endif
+	/* index in the above array */
+};
+
+struct chip_image_map
+{
+	uint32 chipid;
+	uint32 chiprev;
+	uint32 chippkg;
+	uint32 image_idx;
+} static chip_image_index_map_table [] __initdata  =
+{
+#ifdef DLIMAGE_43602a1
+	{CHIPID_43602, CHIP_43602_CHIPREV_A1, CHIP_43602_PKG_OPT, CHIP_43602_A1_CHIP_IMAGE},
+	{CHIPID_43462, CHIP_43602_CHIPREV_A1, CHIP_43602_PKG_OPT, CHIP_43602_A1_CHIP_IMAGE},
+	{CHIPID_43522, CHIP_43602_CHIPREV_A1, CHIP_43602_PKG_OPT, CHIP_43602_A1_CHIP_IMAGE},
+#endif
+	/* {for a given chipid, chiprev, chippkg, what is the index (the above enum) */
+	{CHIPID_NONE, 0, 0, 0} /* CHIPID_NONE is -1, used to mark end of list */
+};
+
+static void  __init select_fd_image(
+		struct dhd_bus *bus, unsigned char **p_dlarray,
+		char **p_dlimagename, char **p_dlimagever,
+		char **p_dlimagedate, int *image_size) {
+
+	uint32 chipid, chiprev, chippkg_opt;
+	int image_index;
+	struct chip_image_map *p_image_index;
+
+	chipid = 0;
+	image_index = -1;
+	p_image_index = &chip_image_index_map_table[0];
+	while (chipid != CHIPID_NONE) {
+		chipid = p_image_index->chipid;
+		chiprev = p_image_index->chiprev;
+		chippkg_opt = p_image_index->chippkg;
+
+		if ((chipid == bus->sih->chip) && (chiprev == bus->sih->chiprev) &&
+			(chippkg_opt == bus->sih->chippkg)) {
+			image_index = p_image_index->image_idx;
+			break;
+		}
+		p_image_index++;
+	}
+
+	if (image_index != -1) {
+		*p_dlarray     = chip_dl_image_array[image_index].dlarray;
+		*p_dlimagename = chip_dl_image_array[image_index].dlimagename;
+		*p_dlimagever  = chip_dl_image_array[image_index].dlimagever;
+		*p_dlimagedate = chip_dl_image_array[image_index].dliamgedate;
+		*image_size    = chip_dl_image_array[image_index].dlimagesize;
+	} else {
+		*p_dlarray     = 0;
+		DHD_ERROR(("####################################################################\n"
+			"# %s: Dongle image not available for chipid = 0x%x"
+			"  chiprev = %d  chippkg = %d\n"
+			"####################################################################\n",
+			__FUNCTION__, bus->sih->chip, bus->sih->chiprev, bus->sih->chippkg));
+	}
+}
+#endif /* defined(BCMEMBEDIMAGE) && defined (BCM_ROUTER_DHD) */
+
+#ifdef BCMEMBEDIMAGE
+int
+dhdpcie_download_code_array(struct dhd_bus *bus)
+{
+	int bcmerror = -1;
+	int offset = 0;
+	unsigned char *p_dlarray  = NULL;
+	unsigned int dlarray_size = 0;
+	unsigned int downloded_len, remaining_len, len;
+	char *p_dlimagename, *p_dlimagever, *p_dlimagedate;
+	uint8 *memblock = NULL, *memptr;
+
+	downloded_len = 0;
+	remaining_len = 0;
+	len = 0;
+
+#ifndef BCM_ROUTER_DHD
+	p_dlarray = dlarray;
+	dlarray_size = sizeof(dlarray);
+	p_dlimagename = dlimagename;
+	p_dlimagever  = dlimagever;
+	p_dlimagedate = dlimagedate;
+#else
+	select_fd_image(bus, &p_dlarray, &p_dlimagename, &p_dlimagever,
+		&p_dlimagedate, &dlarray_size);
+#endif /* endif for BCM_ROUTER_DHD */
+
+	if ((p_dlarray == 0) ||	(dlarray_size == 0) ||(dlarray_size > bus->ramsize) ||
+		(p_dlimagename == 0) ||	(p_dlimagever  == 0) ||	(p_dlimagedate == 0))
+		goto err;
+
+	memptr = memblock = MALLOC(bus->dhd->osh, MEMBLOCK + DHD_SDALIGN);
+	if (memblock == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__, MEMBLOCK));
+		goto err;
+	}
+	if ((uint32)(uintptr)memblock % DHD_SDALIGN)
+		memptr += (DHD_SDALIGN - ((uint32)(uintptr)memblock % DHD_SDALIGN));
+
+	while (downloded_len  < dlarray_size) {
+		remaining_len = dlarray_size - downloded_len;
+		if (remaining_len >= MEMBLOCK)
+			len = MEMBLOCK;
+		else
+			len = remaining_len;
+
+		memcpy(memptr, (p_dlarray + downloded_len), len);
+		/* check if CR4/CA7 */
+		if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0) ||
+			si_setcore(bus->sih, SYSMEM_CORE_ID, 0)) {
+			/* if address is 0, store the reset instruction to be written in 0 */
+			if (offset == 0) {
+				bus->resetinstr = *(((uint32*)memptr));
+				/* Add start of RAM address to the address given by user */
+				offset += bus->dongle_ram_base;
+			}
+		}
+		bcmerror = dhdpcie_bus_membytes(bus, TRUE, offset, (uint8 *)memptr, len);
+		downloded_len += len;
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+				__FUNCTION__, bcmerror, MEMBLOCK, offset));
+			goto err;
+		}
+		offset += MEMBLOCK;
+	}
+
+#ifdef DHD_DEBUG
+	/* Upload and compare the downloaded code */
+	{
+		unsigned char *ularray = NULL;
+		unsigned int uploded_len;
+		uploded_len = 0;
+		bcmerror = -1;
+		ularray = MALLOC(bus->dhd->osh, dlarray_size);
+		if (ularray == NULL)
+			goto upload_err;
+		/* Upload image to verify downloaded contents. */
+		offset = bus->dongle_ram_base;
+		memset(ularray, 0xaa, dlarray_size);
+		while (uploded_len  < dlarray_size) {
+			remaining_len = dlarray_size - uploded_len;
+			if (remaining_len >= MEMBLOCK)
+				len = MEMBLOCK;
+			else
+				len = remaining_len;
+			bcmerror = dhdpcie_bus_membytes(bus, FALSE, offset,
+				(uint8 *)(ularray + uploded_len), len);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+					__FUNCTION__, bcmerror, MEMBLOCK, offset));
+				goto upload_err;
+			}
+
+			uploded_len += len;
+			offset += MEMBLOCK;
+		}
+		if (memcmp(p_dlarray, ularray, dlarray_size)) {
+			DHD_ERROR(("%s: Downloaded image is corrupted (%s, %s, %s).\n",
+				__FUNCTION__, p_dlimagename, p_dlimagever, p_dlimagedate));
+			goto upload_err;
+
+		} else
+			DHD_ERROR(("%s: Download, Upload and compare succeeded (%s, %s, %s).\n",
+				__FUNCTION__, p_dlimagename, p_dlimagever, p_dlimagedate));
+
+upload_err:
+		if (ularray)
+			MFREE(bus->dhd->osh, ularray, dlarray_size);
+	}
+#endif /* DHD_DEBUG */
+err:
+
+	if (memblock)
+		MFREE(bus->dhd->osh, memblock, MEMBLOCK + DHD_SDALIGN);
+
+	return bcmerror;
+} /* dhdpcie_download_code_array */
+#endif /* BCMEMBEDIMAGE */
+
+#ifdef BCM_ROUTER_DHD
+static int
+_dhdpcie_get_nvram_params(struct dhd_bus *bus)
+{
+	int nvram_len = MAX_NVRAMBUF_SIZE;
+	int tmp_nvram_len, boardrev_str_len;
+	char *boardrev_str;
+	char *boardtype_str;
+	char *ptr;
+
+	bus->nvram_params = MALLOC(bus->dhd->osh, nvram_len);
+	if (!bus->nvram_params) {
+		DHD_ERROR(("%s: fail to get nvram buffer to download.\n", __FUNCTION__));
+		return -1;
+	}
+
+	bus->nvram_params[0] = 0;
+	ptr = bus->nvram_params;
+	/*
+	 * For full dongle router platforms, we would have two dhd instances running,
+	 * serving two radios, one for 5G and another for 2G. But, both dongle instances
+	 * would come up as wl0, as one is not aware of the other. In order to avoid
+	 * this situation, we pass the dhd instance number through nvram parameter
+	 * wlunit=0 and wlunit=1 to the dongle and make sure the two dongle instances
+	 * come up as wl0 and wl1.
+	 */
+
+	tmp_nvram_len = strlen("wlunit=xx\n\n") + 1;
+	tmp_nvram_len =
+		snprintf(ptr, tmp_nvram_len, "wlunit=%d", dhd_get_instance(bus->dhd));
+	ptr += (tmp_nvram_len + 1); /* leave NULL */
+	tmp_nvram_len++;
+
+	if ((boardrev_str = si_getdevpathvar(bus->sih, "boardrev")) == NULL)
+		boardrev_str = nvram_get("boardrev");
+
+	boardrev_str_len = strlen("boardrev=0xXXXX") + 1;
+	boardrev_str_len = snprintf(ptr, boardrev_str_len, "boardrev=%s",
+		boardrev_str? boardrev_str : BOARDREV_PROMOTABLE_STR);
+	ptr += (boardrev_str_len + 1); /* leave NULL */
+	tmp_nvram_len += (boardrev_str_len + 1);
+
+	/* If per device boardtype is not available, use global boardtype */
+	if ((boardtype_str = si_getdevpathvar(bus->sih, "boardtype")) == NULL) {
+		if ((boardtype_str = nvram_get("boardtype")) != NULL) {
+			int boardtype_str_len = 0;
+
+			boardtype_str_len = strlen("boardtype=0xXXXX") + 1;
+			boardtype_str_len = snprintf(ptr, boardtype_str_len,
+					"boardtype=%s", boardtype_str);
+			ptr += (boardtype_str_len + 1); /* leave NULL */
+			tmp_nvram_len += (boardtype_str_len + 1);
+		}
+	}
+
+	if (dbushost_initvars_flash(bus->sih,
+		bus->osh, &ptr,
+		(nvram_len - tmp_nvram_len)) != 0) {
+		DHD_ERROR(("%s: fail to read nvram from flash.\n", __FUNCTION__));
+	}
+
+	tmp_nvram_len = (int)(ptr - bus->nvram_params);
+
+	bcopy(STR_END, ptr, sizeof(STR_END));
+	tmp_nvram_len += sizeof(STR_END);
+	bus->nvram_params_len  = tmp_nvram_len;
+	return 0;
+}
+
+static void
+_dhdpcie_free_nvram_params(struct dhd_bus *bus)
+{
+	if (bus->nvram_params) {
+		MFREE(bus->dhd->osh, bus->nvram_params, MAX_NVRAMBUF_SIZE);
+	}
+}
+
+/** Handler to send a signal to the dhdmonitor process to notify of firmware traps */
+void
+dhdpcie_handle_dongle_trap(struct dhd_bus *bus)
+{
+	char *failed_if;
+
+	/* Call the bus module watchdog */
+	dhd_bus_watchdog(bus->dhd);
+
+	/* Get the failed interface name to be later used by
+	 * dhd_monitor to capture the required logs
+	 */
+	failed_if = dhd_ifname(bus->dhd, 0);
+	dhd_schedule_trap_log_dump(bus->dhd, (uint8 *)failed_if, strlen(failed_if));
+}
+
+#endif /* BCM_ROUTER_DHD */
+
+/**
+ * Downloads firmware file given by 'bus->fw_path' into PCIe dongle
+ *
+ * BCMEMBEDIMAGE specific:
+ * If bus->fw_path is empty, or if the download of bus->fw_path failed, firmware contained in header
+ * file will be used instead.
+ *
+ */
+static int
+_dhdpcie_download_firmware(struct dhd_bus *bus)
+{
+	int bcmerror = -1;
+
+	bool embed = FALSE;	/* download embedded firmware */
+	bool dlok = FALSE;	/* download firmware succeeded */
+
+	/* Out immediately if no image to download */
+	if ((bus->fw_path == NULL) || (bus->fw_path[0] == '\0')) {
+#ifdef BCMEMBEDIMAGE
+		embed = TRUE;
+#else
+		DHD_ERROR(("%s: no fimrware file\n", __FUNCTION__));
+		return 0;
+#endif
+	}
+#ifdef BCM_ROUTER_DHD
+	if (_dhdpcie_get_nvram_params(bus) < 0) {
+		DHD_ERROR(("%s: fail to get nvram from system.\n", __FUNCTION__));
+		return 0;
+	}
+#endif
+	/* Keep arm in reset */
+	if (dhdpcie_bus_download_state(bus, TRUE)) {
+		DHD_ERROR(("%s: error placing ARM core in reset\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* External image takes precedence if specified */
+	if ((bus->fw_path != NULL) && (bus->fw_path[0] != '\0')) {
+		if ((bcmerror = dhdpcie_download_code_file(bus, bus->fw_path))) {
+			DHD_ERROR(("%s:%d dongle image file download failed\n", __FUNCTION__,
+				__LINE__));
+#ifdef BCMEMBEDIMAGE
+			embed = TRUE;
+#else
+			goto err;
+#endif
+		} else {
+			embed = FALSE;
+			dlok = TRUE;
+		}
+	}
+
+#ifdef BCMEMBEDIMAGE
+	if (embed) {
+		if (dhdpcie_download_code_array(bus)) {
+			DHD_ERROR(("%s: dongle image array download failed\n", __FUNCTION__));
+			goto err;
+		} else {
+			dlok = TRUE;
+		}
+	}
+#else
+	BCM_REFERENCE(embed);
+#endif
+	if (!dlok) {
+		DHD_ERROR(("%s:%d dongle image download failed\n", __FUNCTION__, __LINE__));
+		goto err;
+	}
+
+	/* EXAMPLE: nvram_array */
+	/* If a valid nvram_arry is specified as above, it can be passed down to dongle */
+	/* dhd_bus_set_nvram_params(bus, (char *)&nvram_array); */
+
+	/* External nvram takes precedence if specified */
+	if ((bcmerror = dhdpcie_download_nvram(bus))) {
+		DHD_ERROR(("%s:%d dongle nvram file download failed\n", __FUNCTION__, __LINE__));
+		goto err;
+	}
+
+	/* Take arm out of reset */
+	if (dhdpcie_bus_download_state(bus, FALSE)) {
+		DHD_ERROR(("%s: error getting out of ARM core reset\n", __FUNCTION__));
+		goto err;
+	}
+
+	bcmerror = 0;
+
+err:
+#ifdef BCM_ROUTER_DHD
+	_dhdpcie_free_nvram_params(bus);
+#endif /* BCM_ROUTER_DHD */
+	return bcmerror;
+} /* _dhdpcie_download_firmware */
+
+static int
+dhdpcie_bus_readconsole(dhd_bus_t *bus)
+{
+	dhd_console_t *c = &bus->console;
+	uint8 line[CONSOLE_LINE_MAX], ch;
+	uint32 n, idx, addr;
+	int rv;
+	uint readlen = 0;
+	uint i = 0;
+
+	/* Don't do anything until FWREADY updates console address */
+	if (bus->console_addr == 0)
+		return -1;
+
+	/* Read console log struct */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, log);
+
+	if ((rv = dhdpcie_bus_membytes(bus, FALSE, addr, (uint8 *)&c->log, sizeof(c->log))) < 0)
+		return rv;
+
+	/* Allocate console buffer (one time only) */
+	if (c->buf == NULL) {
+		c->bufsize = ltoh32(c->log.buf_size);
+		if ((c->buf = MALLOC(bus->dhd->osh, c->bufsize)) == NULL)
+			return BCME_NOMEM;
+		DHD_INFO(("conlog: bufsize=0x%x\n", c->bufsize));
+	}
+	idx = ltoh32(c->log.idx);
+
+	/* Protect against corrupt value */
+	if (idx > c->bufsize)
+		return BCME_ERROR;
+
+	/* Skip reading the console buffer if the index pointer has not moved */
+	if (idx == c->last)
+		return BCME_OK;
+
+	DHD_INFO(("conlog: addr=0x%x, idx=0x%x, last=0x%x \n", c->log.buf,
+	   idx, c->last));
+
+	/* Read the console buffer data to a local buffer
+	 * optimize and read only the portion of the buffer needed, but
+	 * important to handle wrap-around. Read ptr is 'c->last',
+	 * write ptr is 'idx'
+	 */
+	addr = ltoh32(c->log.buf);
+
+	/* wrap around case - write ptr < read ptr */
+	if (idx < c->last) {
+		/* from read ptr to end of buffer */
+		readlen = c->bufsize - c->last;
+		if ((rv = dhdpcie_bus_membytes(bus, FALSE,
+				addr + c->last, c->buf, readlen)) < 0) {
+			DHD_ERROR(("conlog: read error[1] ! \n"));
+			return rv;
+		}
+		/* from beginning of buffer to write ptr */
+		if ((rv = dhdpcie_bus_membytes(bus, FALSE,
+				addr, c->buf + readlen,
+				idx)) < 0) {
+			DHD_ERROR(("conlog: read error[2] ! \n"));
+			return rv;
+		}
+		readlen += idx;
+	} else {
+		/* non-wraparound case, write ptr > read ptr */
+		readlen = (uint)idx - c->last;
+		if ((rv = dhdpcie_bus_membytes(bus, FALSE,
+				addr + c->last, c->buf, readlen)) < 0) {
+			DHD_ERROR(("conlog: read error[3] ! \n"));
+			return rv;
+		}
+	}
+	/* update read ptr */
+	c->last = idx;
+
+	/* now output the read data from the local buffer to the host console */
+	while (i < readlen) {
+		for (n = 0; n < CONSOLE_LINE_MAX - 2 && i < readlen; n++) {
+			ch = c->buf[i];
+			++i;
+			if (ch == '\n')
+				break;
+			line[n] = ch;
+		}
+
+		if (n > 0) {
+			if (line[n - 1] == '\r')
+				n--;
+			line[n] = 0;
+			printf("CONSOLE: %s\n", line);
+		}
+	}
+
+	return BCME_OK;
+
+} /* dhdpcie_bus_readconsole */
+
+void
+dhd_bus_dump_console_buffer(dhd_bus_t *bus)
+{
+	uint32 n, i;
+	uint32 addr;
+	char *console_buffer = NULL;
+	uint32 console_ptr, console_size, console_index;
+	uint8 line[CONSOLE_LINE_MAX], ch;
+	int rv;
+
+	DHD_ERROR(("%s: Dump Complete Console Buffer\n", __FUNCTION__));
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: Skip dump Console Buffer due to PCIe link down\n", __FUNCTION__));
+		return;
+	}
+
+	addr =	bus->pcie_sh->console_addr + OFFSETOF(hnd_cons_t, log);
+	if ((rv = dhdpcie_bus_membytes(bus, FALSE, addr,
+		(uint8 *)&console_ptr, sizeof(console_ptr))) < 0) {
+		goto exit;
+	}
+
+	addr =	bus->pcie_sh->console_addr + OFFSETOF(hnd_cons_t, log.buf_size);
+	if ((rv = dhdpcie_bus_membytes(bus, FALSE, addr,
+		(uint8 *)&console_size, sizeof(console_size))) < 0) {
+		goto exit;
+	}
+
+	addr =	bus->pcie_sh->console_addr + OFFSETOF(hnd_cons_t, log.idx);
+	if ((rv = dhdpcie_bus_membytes(bus, FALSE, addr,
+		(uint8 *)&console_index, sizeof(console_index))) < 0) {
+		goto exit;
+	}
+
+	console_ptr = ltoh32(console_ptr);
+	console_size = ltoh32(console_size);
+	console_index = ltoh32(console_index);
+
+	if (console_size > CONSOLE_BUFFER_MAX ||
+		!(console_buffer = MALLOC(bus->dhd->osh, console_size))) {
+		goto exit;
+	}
+
+	if ((rv = dhdpcie_bus_membytes(bus, FALSE, console_ptr,
+		(uint8 *)console_buffer, console_size)) < 0) {
+		goto exit;
+	}
+
+	for (i = 0, n = 0; i < console_size; i += n + 1) {
+		for (n = 0; n < CONSOLE_LINE_MAX - 2; n++) {
+			ch = console_buffer[(console_index + i + n) % console_size];
+			if (ch == '\n')
+				break;
+			line[n] = ch;
+		}
+
+		if (n > 0) {
+			if (line[n - 1] == '\r')
+				n--;
+			line[n] = 0;
+			/* Don't use DHD_ERROR macro since we print
+			 * a lot of information quickly. The macro
+			 * will truncate a lot of the printfs
+			 */
+
+			printf("CONSOLE: %s\n", line);
+		}
+	}
+
+exit:
+	if (console_buffer)
+		MFREE(bus->dhd->osh, console_buffer, console_size);
+	return;
+}
+
+static void dhdpcie_force_console_dump(dhd_bus_t *bus)
+{
+	uint32 cons_sign = 0;
+	uint32 cons_addr = 0;
+	uint32 val = 0;
+
+	cons_sign = bus->dongle_ram_base + bus->ramsize - 12;
+
+	val = LTOH32(dhdpcie_bus_rtcm32(bus, cons_sign));
+
+	if (val == CONS_ADDR_SIGNATURE) {
+		cons_addr = bus->dongle_ram_base + bus->ramsize - 8;
+		val = LTOH32(dhdpcie_bus_rtcm32(bus, cons_addr));
+		bus->console_addr = val;
+		dhd_msg_level |= DHD_FWLOG_VAL;
+		dhdpcie_bus_readconsole(bus);
+	}
+	return;
+}
+
+static void
+dhdpcie_schedule_log_dump(dhd_bus_t *bus)
+{
+#if defined(DHD_DUMP_FILE_WRITE_FROM_KERNEL) && defined(DHD_LOG_DUMP)
+	log_dump_type_t *flush_type;
+
+	/* flush_type is freed at do_dhd_log_dump function */
+	flush_type = MALLOCZ(bus->dhd->osh, sizeof(log_dump_type_t));
+	if (flush_type) {
+		*flush_type = DLD_BUF_TYPE_ALL;
+		dhd_schedule_log_dump(bus->dhd, flush_type);
+	} else {
+		DHD_ERROR(("%s Fail to malloc flush_type\n", __FUNCTION__));
+	}
+#endif /* DHD_DUMP_FILE_WRITE_FROM_KERNEL && DHD_LOG_DUMP */
+}
+
+#ifdef DHD_SSSR_DUMP
+extern uint fis_enab;
+#endif /* DHD_SSSR_DUMP */
+
+dhd_pcie_link_state_type_t
+dhdpcie_get_link_state(dhd_bus_t *bus)
+{
+	uint32 intstatus;
+	uint origidx;
+	dhd_pcie_link_state_type_t link_state = DHD_PCIE_ALL_GOOD;
+	uint32 status_cmd = dhd_pcie_config_read(bus, PCIECFGREG_STATUS_CMD, sizeof(uint32));
+	if (status_cmd == (uint32) -1) {
+		DHD_ERROR(("%s: pcie link down: status_cmd:0x%x\n", __FUNCTION__, status_cmd));
+		link_state = DHD_PCIE_LINK_DOWN;
+		bus->is_linkdown = 1;
+		goto exit;
+	}
+
+	/* Check the common backplane status */
+	intstatus = si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_int, 0, 0);
+	if (intstatus == (uint32) -1) {
+		DHD_ERROR(("%s: common backplane is down, intstatus:0x%x\n",
+			__FUNCTION__, intstatus));
+		link_state = DHD_PCIE_COMMON_BP_DOWN;
+		goto exit;
+	}
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	origidx = si_coreidx(bus->sih);
+
+	/* Switch to SYSMEM core and check for wlan backplane status */
+	if (si_setcore(bus->sih, SYSMEM_CORE_ID, 0)) {
+		uint32 ioctrl = si_wrapperreg(bus->sih, AI_IOCTRL, 0, 0);
+		uint32 resetctrl = si_wrapperreg(bus->sih, AI_RESETCTRL, 0, 0);
+		DHD_ERROR(("%s: sysmem ioctrl:0x%x resetctrl:0x%x\n",
+			__FUNCTION__, ioctrl, resetctrl));
+		if (ioctrl == (uint32) -1) {
+			DHD_ERROR(("%s: wlan backplane is down ioctrl:0x%x\n",
+				__FUNCTION__, ioctrl));
+			link_state = DHD_PCIE_WLAN_BP_DOWN;
+		}
+	} else {
+		DHD_ERROR(("%s: Failed to find SYSMEM core!\n", __FUNCTION__));
+	}
+
+	/* Restore back to original core */
+	si_setcoreidx(bus->sih, origidx);
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+
+exit:
+	return link_state;
+}
+
+void
+dhd_validate_pcie_link_cbp_wlbp(dhd_bus_t *bus)
+{
+	bus->link_state = dhdpcie_get_link_state(bus);
+
+	if (bus->link_state == DHD_PCIE_ALL_GOOD) {
+		dhd_bus_dump_imp_cfg_registers(bus);
+		dhd_bus_dump_dar_registers(bus);
+	} else if (bus->link_state == DHD_PCIE_LINK_DOWN) {
+		DHD_ERROR(("%s: pcie link down\n", __FUNCTION__));
+		bus->is_linkdown = 1;
+	} else if (bus->link_state == DHD_PCIE_COMMON_BP_DOWN) {
+		dhd_bus_dump_imp_cfg_registers(bus);
+		dhd_bus_dump_dar_registers(bus);
+		bus->is_linkdown = 1;
+	} else if (bus->link_state == DHD_PCIE_WLAN_BP_DOWN) {
+		dhd_bus_dump_imp_cfg_registers(bus);
+		dhd_bus_dump_dar_registers(bus);
+#if defined(DHD_FW_COREDUMP)
+#ifdef DHD_SSSR_DUMP
+#ifdef OEM_ANDROID
+			DHD_ERROR(("%s : Set collect_sssr and fis_enab as TRUE\n", __FUNCTION__));
+			bus->dhd->collect_sssr = TRUE;
+			fis_enab = TRUE;
+#endif /* OEM_ANDROID */
+#endif /* DHD_SSSR_DUMP */
+			/* save core dump or write to a file */
+			if (bus->dhd->memdump_enabled) {
+				bus->dhd->memdump_type = DUMP_TYPE_READ_SHM_FAIL;
+				dhdpcie_mem_dump(bus);
+			}
+#endif /* DHD_FW_COREDUMP */
+	}
+	return;
+}
+
+/**
+ * Opens the file given by bus->fw_path, reads part of the file into a buffer and closes the file.
+ *
+ * @return BCME_OK on success
+ */
+static int
+dhdpcie_checkdied(dhd_bus_t *bus, char *data, uint size)
+{
+	int bcmerror = 0;
+	uint msize = 512;
+	char *mbuffer = NULL;
+	uint maxstrlen = 256;
+	char *str = NULL;
+	pciedev_shared_t *local_pciedev_shared = bus->pcie_sh;
+	struct bcmstrbuf strbuf;
+	unsigned long flags;
+	bool dongle_trap_occured = FALSE;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (DHD_NOCHECKDIED_ON() || dhd_query_bus_erros(bus->dhd)) {
+		return 0;
+	}
+
+	if (data == NULL) {
+		/*
+		 * Called after a rx ctrl timeout. "data" is NULL.
+		 * allocate memory to trace the trap or assert.
+		 */
+		size = msize;
+		mbuffer = data = MALLOC(bus->dhd->osh, msize);
+
+		if (mbuffer == NULL) {
+			DHD_ERROR(("%s: MALLOC(%d) failed \n", __FUNCTION__, msize));
+			bcmerror = BCME_NOMEM;
+			goto done2;
+		}
+	}
+
+	if ((str = MALLOC(bus->dhd->osh, maxstrlen)) == NULL) {
+		DHD_ERROR(("%s: MALLOC(%d) failed \n", __FUNCTION__, maxstrlen));
+		bcmerror = BCME_NOMEM;
+		goto done2;
+	}
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_SET_IN_CHECKDIED(bus->dhd);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+	if ((bcmerror = dhdpcie_readshared(bus)) < 0) {
+		goto done1;
+	}
+
+	bcm_binit(&strbuf, data, size);
+
+	bcm_bprintf(&strbuf, "msgtrace address : 0x%08X\nconsole address  : 0x%08X\n",
+	            local_pciedev_shared->msgtrace_addr, local_pciedev_shared->console_addr);
+
+	if ((local_pciedev_shared->flags & PCIE_SHARED_ASSERT_BUILT) == 0) {
+		/* NOTE: Misspelled assert is intentional - DO NOT FIX.
+		 * (Avoids conflict with real asserts for programmatic parsing of output.)
+		 */
+		bcm_bprintf(&strbuf, "Assrt not built in dongle\n");
+	}
+
+	if ((bus->pcie_sh->flags & (PCIE_SHARED_ASSERT|PCIE_SHARED_TRAP)) == 0) {
+		/* NOTE: Misspelled assert is intentional - DO NOT FIX.
+		 * (Avoids conflict with real asserts for programmatic parsing of output.)
+		 */
+		bcm_bprintf(&strbuf, "No trap%s in dongle",
+		          (bus->pcie_sh->flags & PCIE_SHARED_ASSERT_BUILT)
+		          ?"/assrt" :"");
+	} else {
+		if (bus->pcie_sh->flags & PCIE_SHARED_ASSERT) {
+			/* Download assert */
+			bcm_bprintf(&strbuf, "Dongle assert");
+			if (bus->pcie_sh->assert_exp_addr != 0) {
+				str[0] = '\0';
+				if ((bcmerror = dhdpcie_bus_membytes(bus, FALSE,
+					bus->pcie_sh->assert_exp_addr,
+					(uint8 *)str, maxstrlen)) < 0) {
+					goto done1;
+				}
+
+				str[maxstrlen - 1] = '\0';
+				bcm_bprintf(&strbuf, " expr \"%s\"", str);
+			}
+
+			if (bus->pcie_sh->assert_file_addr != 0) {
+				str[0] = '\0';
+				if ((bcmerror = dhdpcie_bus_membytes(bus, FALSE,
+					bus->pcie_sh->assert_file_addr,
+					(uint8 *)str, maxstrlen)) < 0) {
+					goto done1;
+				}
+
+				str[maxstrlen - 1] = '\0';
+				bcm_bprintf(&strbuf, " file \"%s\"", str);
+			}
+
+			bcm_bprintf(&strbuf, " line %d ",  bus->pcie_sh->assert_line);
+		}
+
+		if (bus->pcie_sh->flags & PCIE_SHARED_TRAP) {
+			trap_t *tr = &bus->dhd->last_trap_info;
+			dongle_trap_occured = TRUE;
+			if ((bcmerror = dhdpcie_bus_membytes(bus, FALSE,
+				bus->pcie_sh->trap_addr, (uint8*)tr, sizeof(trap_t))) < 0) {
+				bus->dhd->dongle_trap_occured = TRUE;
+				goto done1;
+			}
+			dhd_bus_dump_trap_info(bus, &strbuf);
+		}
+	}
+
+	if (bus->pcie_sh->flags & (PCIE_SHARED_ASSERT | PCIE_SHARED_TRAP)) {
+		DHD_ERROR(("%s: %s\n", __FUNCTION__, strbuf.origbuf));
+#ifdef REPORT_FATAL_TIMEOUTS
+		/**
+		 * stop the timers as FW trapped
+		 */
+		if (dhd_stop_scan_timer(bus->dhd, FALSE, 0)) {
+			DHD_ERROR(("dhd_stop_scan_timer failed\n"));
+			ASSERT(0);
+		}
+		if (dhd_stop_bus_timer(bus->dhd)) {
+			DHD_ERROR(("dhd_stop_bus_timer failed\n"));
+			ASSERT(0);
+		}
+		if (dhd_stop_cmd_timer(bus->dhd)) {
+			DHD_ERROR(("dhd_stop_cmd_timer failed\n"));
+			ASSERT(0);
+		}
+		if (dhd_stop_join_timer(bus->dhd)) {
+			DHD_ERROR(("dhd_stop_join_timer failed\n"));
+			ASSERT(0);
+		}
+#endif /* REPORT_FATAL_TIMEOUTS */
+
+		/* wake up IOCTL wait event */
+		dhd_wakeup_ioctl_event(bus->dhd, IOCTL_RETURN_ON_TRAP);
+
+		dhd_bus_dump_console_buffer(bus);
+		dhd_prot_debug_info_print(bus->dhd);
+
+#ifdef DHD_FW_COREDUMP
+#ifdef OEM_ANDROID
+		if (bus->dhd->up == 0) {
+			DHD_ERROR(("%s: socram will be collected in dhd_net_bus_devreset failure\n",
+				__FUNCTION__));
+			bus->dhd->dongle_trap_during_wifi_onoff = 1;
+		} else
+#endif /* OEM_ANDROID */
+		{
+			/* save core dump or write to a file */
+			if (bus->dhd->memdump_enabled) {
+#ifdef DHD_SSSR_DUMP
+				DHD_ERROR(("%s : Set collect_sssr as TRUE\n", __FUNCTION__));
+				bus->dhd->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+#ifdef DHD_SDTC_ETB_DUMP
+				DHD_ERROR(("%s : Set collect_sdtc as TRUE\n", __FUNCTION__));
+				bus->dhd->collect_sdtc = TRUE;
+#endif /* DHD_SDTC_ETB_DUMP */
+				bus->dhd->memdump_type = DUMP_TYPE_DONGLE_TRAP;
+				dhdpcie_mem_dump(bus);
+			}
+		}
+#endif /* DHD_FW_COREDUMP */
+
+		/* set the trap occured flag only after all the memdump,
+		* logdump and sssr dump collection has been scheduled
+		*/
+		if (dongle_trap_occured) {
+			bus->dhd->dongle_trap_occured = TRUE;
+			if (bus->dhd->check_trap_rot &&
+				bus->dhd->ext_trap_data_supported &&
+				bus->pcie_sh->flags2 & PCIE_SHARED2_ETD_ADDR_SUPPORT) {
+				uint32 trap_data = *(uint32 *)bus->dhd->extended_trap_data;
+				DHD_ERROR(("%s : etd data : %x\n", __FUNCTION__, trap_data));
+				if (!(trap_data & D2H_DEV_EXT_TRAP_DATA)) {
+					uint32 *ext_data = bus->dhd->extended_trap_data;
+					/* Skip the first word which is trap_data */
+					ext_data++;
+					DHD_ERROR(("Dongle trap but no etd\n"));
+					if (dhdpcie_bus_membytes(bus, FALSE,
+						local_pciedev_shared->etd_addr,
+						(uint8 *)ext_data,
+						BCMPCIE_EXT_TRAP_DATA_MAXLEN -
+							sizeof(trap_data)) < 0) {
+						DHD_ERROR(("Error to read etd from dongle\n"));
+					}
+				} else {
+					DHD_ERROR(("Dongle trap with etd\n"));
+				}
+			}
+
+		}
+
+#ifdef WL_CFGVENDOR_SEND_ALERT_EVENT
+		bus->dhd->alert_reason = ALERT_DONGLE_TRAP;
+		dhd_os_send_alert_message(bus->dhd);
+#endif /* WL_CFGVENDOR_SEND_ALERT_EVENT */
+		dhd_schedule_reset(bus->dhd);
+
+#ifdef NDIS
+		/* ASSERT only if hang detection/recovery is disabled. If enabled then let
+		 * windows HDR mechansim trigger FW download via surprise removal
+		 */
+		dhd_bus_check_died(bus);
+#endif
+
+	} else if (bus->dhd->dongle_trap_data) {
+		DHD_ERROR(("%s: TRAP/ASSERT flag not set in shared flags:0x%x\n",
+			__FUNCTION__, bus->pcie_sh->flags));
+		bus->dhd->dongle_trap_occured = TRUE;
+#ifdef DHD_FW_COREDUMP
+#ifdef OEM_ANDROID
+		if (bus->dhd->up == 0) {
+			DHD_ERROR(("%s: socram will be collected in dhd_net_bus_devreset failure\n",
+				__FUNCTION__));
+		} else
+#endif /* OEM_ANDROID */
+		{
+			/* save core dump or write to a file */
+			if (bus->dhd->memdump_enabled) {
+#ifdef DHD_SSSR_DUMP
+				DHD_ERROR(("%s : Set collect_sssr as TRUE\n", __FUNCTION__));
+				bus->dhd->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+#ifdef DHD_SDTC_ETB_DUMP
+				DHD_ERROR(("%s : Set collect_sdtc as TRUE\n", __FUNCTION__));
+				bus->dhd->collect_sdtc = TRUE;
+#endif /* DHD_SDTC_ETB_DUMP */
+				bus->dhd->memdump_type = DUMP_TYPE_DONGLE_TRAP;
+				dhdpcie_mem_dump(bus);
+			}
+		}
+#endif /* DHD_FW_COREDUMP */
+	}
+
+done1:
+	if (bcmerror) {
+		/* dhdpcie_checkdied is invoked only when dongle has trapped
+		 * or after PCIe link down..etc. so set dongle_trap_occured so that
+		 * log_dump logic can rely on only one flag dongle_trap_occured.
+		 */
+		bus->dhd->dongle_trap_occured = TRUE;
+		dhdpcie_schedule_log_dump(bus);
+	}
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_IN_CHECKDIED(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+done2:
+	if (mbuffer)
+		MFREE(bus->dhd->osh, mbuffer, msize);
+	if (str)
+		MFREE(bus->dhd->osh, str, maxstrlen);
+
+	return bcmerror;
+} /* dhdpcie_checkdied */
+
+/* Custom copy of dhdpcie_mem_dump() that can be called at interrupt level */
+void dhdpcie_mem_dump_bugcheck(dhd_bus_t *bus, uint8 *buf)
+{
+	int ret = 0;
+	int size; /* Full mem size */
+	int start; /* Start address */
+	int read_size = 0; /* Read size of each iteration */
+	uint8 *databuf = buf;
+
+	if (bus == NULL) {
+		return;
+	}
+
+	start = bus->dongle_ram_base;
+	read_size = 4;
+	/* check for dead bus */
+	{
+		uint test_word = 0;
+		ret = dhdpcie_bus_membytes(bus, FALSE, start, (uint8*)&test_word, read_size);
+		/* if read error or bus timeout */
+		if (ret || (test_word == 0xFFFFFFFF)) {
+			return;
+		}
+	}
+
+	/* Get full mem size */
+	size = bus->ramsize;
+	/* Read mem content */
+	while (size)
+	{
+		read_size = MIN(MEMBLOCK, size);
+		if ((ret = dhdpcie_bus_membytes(bus, FALSE, start, databuf, read_size))) {
+			return;
+		}
+
+		/* Decrement size and increment start address */
+		size -= read_size;
+		start += read_size;
+		databuf += read_size;
+	}
+	bus->dhd->soc_ram = buf;
+	bus->dhd->soc_ram_length = bus->ramsize;
+	return;
+}
+
+#if defined(DHD_FW_COREDUMP)
+static int
+dhdpcie_get_mem_dump(dhd_bus_t *bus)
+{
+	int ret = BCME_OK;
+	int size = 0;
+	int start = 0;
+	int read_size = 0; /* Read size of each iteration */
+	uint8 *p_buf = NULL, *databuf = NULL;
+#ifdef BOARD_HIKEY
+	unsigned long flags_bus;
+#endif /* BOARD_HIKEY */
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	dhd_dongledump_status_t dump_status;
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	if (!bus) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	if (!bus->dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	if (bus->link_state == DHD_PCIE_WLAN_BP_DOWN) {
+		DHD_ERROR(("%s: DHD_PCIE_WLAN_BP_DOWN return success and collect only FIS if set\n",
+			__FUNCTION__));
+		ret = BCME_OK;
+		goto exit;
+	}
+
+	size = bus->ramsize; /* Full mem size */
+	start = bus->dongle_ram_base; /* Start address */
+
+	/* Get full mem size */
+	p_buf = dhd_get_fwdump_buf(bus->dhd, size);
+	if (!p_buf) {
+		DHD_ERROR(("%s: Out of memory (%d bytes)\n",
+			__FUNCTION__, size));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	dump_status = dhd_get_dump_status(bus->dhd);
+	if (dump_status != DUMP_IN_PROGRESS) {
+		dhd_set_dump_status(bus->dhd, DUMP_IN_PROGRESS);
+	}
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	/* Read mem content */
+	DHD_TRACE_HW4(("Dump dongle memory\n"));
+	databuf = p_buf;
+
+	while (size > 0) {
+		read_size = MIN(MEMBLOCK, size);
+#ifdef BOARD_HIKEY
+		/* Hold BUS_LP_STATE_LOCK to avoid simultaneous bus access */
+		DHD_BUS_LP_STATE_LOCK(bus->bus_lp_state_lock, flags_bus);
+#endif /* BOARD_HIKEY */
+		ret = dhdpcie_bus_membytes(bus, FALSE, start, databuf, read_size);
+#ifdef BOARD_HIKEY
+		DHD_BUS_LP_STATE_UNLOCK(bus->bus_lp_state_lock, flags_bus);
+#endif /* BOARD_HIKEY */
+		if (ret) {
+			DHD_ERROR(("%s: Error membytes %d\n", __FUNCTION__, ret));
+#ifdef DHD_DEBUG_UART
+			bus->dhd->memdump_success = FALSE;
+#endif	/* DHD_DEBUG_UART */
+			break;
+		}
+		DHD_TRACE(("."));
+
+		/* Decrement size and increment start address */
+		size -= read_size;
+		start += read_size;
+		databuf += read_size;
+	}
+exit:
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	if (ret != BCME_OK) {
+		dhd_set_dump_status(bus->dhd, DUMP_FAILURE);
+	}
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	return ret;
+}
+
+static int
+dhdpcie_mem_dump(dhd_bus_t *bus)
+{
+	dhd_pub_t *dhdp;
+	int ret = BCME_OK;
+	uint32 dhd_console_ms_prev = 0;
+
+#ifdef GDB_PROXY
+	bus->gdb_proxy_mem_dump_count++;
+#endif /* GDB_PROXY */
+	dhdp = bus->dhd;
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dhd_console_ms_prev = dhdp->dhd_console_ms;
+	if (dhd_console_ms_prev) {
+		DHD_ERROR(("%s: Disabling console msgs(0x%d) before mem dump to local buf\n",
+			__FUNCTION__, dhd_console_ms_prev));
+		dhdp->dhd_console_ms = 0;
+	}
+
+#if defined(__linux__)
+	dhd_plat_pcie_register_dump(dhdp->plat_info);
+#endif /* __linux__ */
+
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp)) {
+		DHD_ERROR(("%s: bus is down! can't collect mem dump. \n", __FUNCTION__));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	/* Induce DB7 trap for below non-trap cases */
+	switch (dhdp->memdump_type) {
+		case DUMP_TYPE_RESUMED_ON_TIMEOUT:
+			/* intentional fall through */
+		case DUMP_TYPE_D3_ACK_TIMEOUT:
+			/* intentional fall through */
+		case DUMP_TYPE_PKTID_AUDIT_FAILURE:
+			/* intentional fall through */
+		case DUMP_TYPE_PKTID_INVALID:
+			/* intentional fall through */
+		case DUMP_TYPE_SCAN_TIMEOUT:
+			/* intentional fall through */
+		case DUMP_TYPE_SCAN_BUSY:
+			/* intentional fall through */
+		case DUMP_TYPE_BY_LIVELOCK:
+			/* intentional fall through */
+		case DUMP_TYPE_TRANS_ID_MISMATCH:
+			/* intentional fall through */
+		case DUMP_TYPE_IFACE_OP_FAILURE:
+			/* intentional fall through */
+		case DUMP_TYPE_PKTID_POOL_DEPLETED:
+			/* intentional fall through */
+		case DUMP_TYPE_ESCAN_SYNCID_MISMATCH:
+			/* intentional fall through */
+		case DUMP_TYPE_PROXD_TIMEOUT:
+			/* intentional fall through */
+		case DUMP_TYPE_INBAND_DEVICE_WAKE_FAILURE:
+			/* intentional fall through */
+		case DUMP_TYPE_INVALID_SHINFO_NRFRAGS:
+			/* intentional fall through */
+		case DUMP_TYPE_P2P_DISC_BUSY:
+			/* intentional fall through */
+			if (dhdp->db7_trap.fw_db7w_trap) {
+				/* Set fw_db7w_trap_inprogress here and clear from DPC */
+				dhdp->db7_trap.fw_db7w_trap_inprogress = TRUE;
+				dhdpcie_fw_trap(dhdp->bus);
+				OSL_DELAY(100 * 1000); // wait 100 msec
+				/*
+				 * Since dpc may have triggered db7, it will not pre-empt itself
+				 * so if trap_data is 0, re-read trap_data before giving up
+				 */
+				if (dhdp->dongle_trap_data == 0) {
+					dhdp->dongle_trap_data = dhd_prot_process_trapbuf(dhdp);
+				}
+				if (dhdp->db7_trap.fw_db7w_trap_recieved) {
+					/* Collect SSSR dump for all non trap cases
+					 * only after DB7 ack
+					 */
+#ifdef DHD_SSSR_DUMP
+					DHD_ERROR(("%s : Set collect_sssr as TRUE after DB7\n",
+						__FUNCTION__));
+					dhdp->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+#ifdef DHD_SDTC_ETB_DUMP
+					DHD_ERROR(("%s : Set collect_sdtc as TRUE after DB7\n",
+						__FUNCTION__));
+					dhdp->collect_sdtc = TRUE;
+#endif /* DHD_SDTC_ETB_DUMP */
+				} else {
+					DHD_ERROR(("%s : Did not receive DB7 Ack\n", __FUNCTION__));
+#ifdef WBRC
+					DHD_ERROR(("%s : Set do_chip_bighammer\n", __FUNCTION__));
+					bus->dhd->do_chip_bighammer = TRUE;
+#endif /* WBRC */
+				}
+				DHD_ERROR(("Function_Intstatus(0x%x)=0x%x "
+					"Function_Intmask(0x%x)=0x%x\n",
+					PCIFunctionIntstatus(bus->sih->buscorerev),
+					si_corereg(bus->sih, bus->sih->buscoreidx,
+					PCIFunctionIntstatus(bus->sih->buscorerev), 0, 0),
+					PCIFunctionIntmask(bus->sih->buscorerev),
+					si_corereg(bus->sih, bus->sih->buscoreidx,
+					PCIFunctionIntmask(bus->sih->buscorerev), 0, 0)));
+			} else {
+				DHD_ERROR(("%s: DB7 Not supported!!!\n",
+					__FUNCTION__));
+			}
+			break;
+		default:
+			break;
+	}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	if (pm_runtime_get_sync(dhd_bus_to_dev(bus)) < 0)
+		return BCME_ERROR;
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	if (dhdp->skip_memdump_map_read == FALSE) {
+		ret = dhdpcie_get_mem_dump(bus);
+		if (ret) {
+			DHD_ERROR(("%s: failed to get mem dump, err=%d\n", __FUNCTION__, ret));
+			goto exit;
+		}
+	} else {
+		DHD_ERROR(("%s: Skipped to get mem dump, err=%d\n", __FUNCTION__, ret));
+		dhdp->skip_memdump_map_read = FALSE;
+	}
+#ifdef DHD_DEBUG_UART
+	bus->dhd->memdump_success = TRUE;
+#endif	/* DHD_DEBUG_UART */
+
+	dhd_schedule_memdump(dhdp, dhdp->soc_ram, dhdp->soc_ram_length);
+	/* buf, actually soc_ram free handled in dhd_{free,clear} */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	pm_runtime_mark_last_busy(dhd_bus_to_dev(bus));
+	pm_runtime_put_autosuspend(dhd_bus_to_dev(bus));
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+exit:
+	if (dhd_console_ms_prev) {
+		DHD_ERROR(("%s: enable console msgs(0x%d) after collecting memdump to local buf\n",
+			__FUNCTION__, dhd_console_ms_prev));
+		dhdp->dhd_console_ms = dhd_console_ms_prev;
+	}
+
+	return ret;
+}
+
+int
+dhd_bus_get_mem_dump(dhd_pub_t *dhdp)
+{
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	return dhdpcie_get_mem_dump(dhdp->bus);
+}
+
+int
+dhd_bus_mem_dump(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	int ret = BCME_ERROR;
+
+	if (dhdp->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s bus is down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Try to resume if already suspended or suspend in progress */
+
+	/* Skip if still in suspended or suspend in progress */
+	if (DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhdp)) {
+		DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state, so skip\n",
+			__FUNCTION__, dhdp->busstate, dhdp->dhd_bus_busy_state));
+		return BCME_ERROR;
+	}
+
+	DHD_OS_WAKE_LOCK(dhdp);
+	ret = dhdpcie_mem_dump(bus);
+	DHD_OS_WAKE_UNLOCK(dhdp);
+	return ret;
+}
+#endif	/* DHD_FW_COREDUMP */
+
+int
+dhd_socram_dump(dhd_bus_t *bus)
+{
+#if defined(DHD_FW_COREDUMP)
+	DHD_OS_WAKE_LOCK(bus->dhd);
+	dhd_bus_mem_dump(bus->dhd);
+	DHD_OS_WAKE_UNLOCK(bus->dhd);
+	return 0;
+#else
+	return -1;
+#endif
+}
+
+/**
+ * Transfers bytes from host to dongle using pio mode.
+ * Parameter 'address' is a backplane address.
+ */
+static int
+dhdpcie_bus_membytes(dhd_bus_t *bus, bool write, ulong address, uint8 *data, uint size)
+{
+	uint dsize;
+	int detect_endian_flag = 0x01;
+	bool little_endian;
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+	/* Detect endianness. */
+	little_endian = *(char *)&detect_endian_flag;
+
+	/* In remap mode, adjust address beyond socram and redirect
+	 * to devram at SOCDEVRAM_BP_ADDR since remap address > orig_ramsize
+	 * is not backplane accessible
+	 */
+
+	/* Determine initial transfer parameters */
+#ifdef DHD_SUPPORT_64BIT
+	dsize = sizeof(uint64);
+#else /* !DHD_SUPPORT_64BIT */
+	dsize = sizeof(uint32);
+#endif /* DHD_SUPPORT_64BIT */
+
+	/* Do the transfer(s) */
+	DHD_INFO(("%s: %s %d bytes in window 0x%08lx\n",
+	          __FUNCTION__, (write ? "write" : "read"), size, address));
+	if (write) {
+		while (size) {
+#ifdef DHD_SUPPORT_64BIT
+			if (size >= sizeof(uint64) && little_endian &&	!(address % 8)) {
+				dhdpcie_bus_wtcm64(bus, address, *((uint64 *)data));
+			}
+#else /* !DHD_SUPPORT_64BIT */
+			if (size >= sizeof(uint32) && little_endian &&	!(address % 4)) {
+				dhdpcie_bus_wtcm32(bus, address, *((uint32*)data));
+			}
+#endif /* DHD_SUPPORT_64BIT */
+			else {
+				dsize = sizeof(uint8);
+				dhdpcie_bus_wtcm8(bus, address, *data);
+			}
+
+			/* Adjust for next transfer (if any) */
+			if ((size -= dsize)) {
+				data += dsize;
+				address += dsize;
+			}
+		}
+	} else {
+		while (size) {
+#ifdef DHD_SUPPORT_64BIT
+			if (size >= sizeof(uint64) && little_endian &&	!(address % 8))
+			{
+				*(uint64 *)data = dhdpcie_bus_rtcm64(bus, address);
+			}
+#else /* !DHD_SUPPORT_64BIT */
+			if (size >= sizeof(uint32) && little_endian &&	!(address % 4))
+			{
+				*(uint32 *)data = dhdpcie_bus_rtcm32(bus, address);
+			}
+#endif /* DHD_SUPPORT_64BIT */
+			else {
+				dsize = sizeof(uint8);
+				*data = dhdpcie_bus_rtcm8(bus, address);
+			}
+
+			/* Adjust for next transfer (if any) */
+			if ((size -= dsize) > 0) {
+				data += dsize;
+				address += dsize;
+			}
+		}
+	}
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+	return BCME_OK;
+} /* dhdpcie_bus_membytes */
+
+extern bool agg_h2d_db_enab;
+/**
+ * Transfers one transmit (ethernet) packet that was queued in the (flow controlled) flow ring queue
+ * to the (non flow controlled) flow ring.
+ */
+int
+BCMFASTPATH(dhd_bus_schedule_queue)(struct dhd_bus  *bus, uint16 flow_id, bool txs,
+	uint32 bound, bool *is_qempty)
+/** XXX function name could be more descriptive, eg use 'tx' and 'flow ring' in name */
+{
+	flow_ring_node_t *flow_ring_node;
+	int ret = BCME_OK;
+#ifdef DHD_LOSSLESS_ROAMING
+	dhd_pub_t *dhdp = bus->dhd;
+#endif /* DHD_LOSSLESS_ROAMING */
+	uint32 cnt = 0;
+
+	DHD_INFO(("%s: flow_id is %d\n", __FUNCTION__, flow_id));
+
+	/* ASSERT on flow_id */
+	if (flow_id >= bus->max_submission_rings) {
+		DHD_ERROR(("%s: flow_id is invalid %d, max %d\n", __FUNCTION__,
+			flow_id, bus->max_submission_rings));
+		return 0;
+	}
+
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flow_id);
+
+	if (flow_ring_node->prot_info == NULL) {
+	    DHD_ERROR((" %s : invalid flow_ring_node \n", __FUNCTION__));
+	    return BCME_NOTREADY;
+	}
+
+#ifdef DHD_LOSSLESS_ROAMING
+	if ((dhdp->dequeue_prec_map & (1 << flow_ring_node->flow_info.tid)) == 0) {
+		DHD_ERROR_RLMT(("%s: roam in progress, tid %d is not in precedence map 0x%x."
+			" block scheduling\n",
+			__FUNCTION__, flow_ring_node->flow_info.tid, dhdp->dequeue_prec_map));
+		return BCME_OK;
+	}
+#endif /* DHD_LOSSLESS_ROAMING */
+
+	{
+		unsigned long flags;
+		void *txp = NULL;
+		flow_queue_t *queue;
+#ifdef TPUT_MONITOR
+		int pktlen;
+#endif
+
+		queue = &flow_ring_node->queue; /* queue associated with flow ring */
+
+		DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+
+		if (flow_ring_node->status != FLOW_RING_STATUS_OPEN) {
+			DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+			return BCME_NOTREADY;
+		}
+
+		if (queue->len == 0) {
+			DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+			return BCME_NOTREADY;
+		}
+
+		while ((txp = dhd_flow_queue_dequeue(bus->dhd, queue)) != NULL) {
+			if (bus->dhd->conf->orphan_move <= 1)
+				PKTORPHAN(txp, bus->dhd->conf->tsq);
+
+			/*
+			 * Modifying the packet length caused P2P cert failures.
+			 * Specifically on test cases where a packet of size 52 bytes
+			 * was injected, the sniffer capture showed 62 bytes because of
+			 * which the cert tests failed. So making the below change
+			 * only Router specific.
+			 */
+#if defined(BCM_ROUTER_DHD)
+			if (PKTLEN(bus->dhd->osh, txp) < (ETHER_MIN_LEN - ETHER_CRC_LEN)) {
+				PKTSETLEN(bus->dhd->osh, txp, (ETHER_MIN_LEN - ETHER_CRC_LEN));
+			}
+#endif /* BCM_ROUTER_DHD */
+
+#ifdef DHDTCPACK_SUPPRESS
+			if (bus->dhd->tcpack_sup_mode != TCPACK_SUP_HOLD) {
+				ret = dhd_tcpack_check_xmit(bus->dhd, txp);
+				if (ret != BCME_OK) {
+					DHD_ERROR(("%s: dhd_tcpack_check_xmit() error.\n",
+						__FUNCTION__));
+				}
+			}
+#endif /* DHDTCPACK_SUPPRESS */
+			/* Attempt to transfer packet over flow ring */
+			/* XXX: ifidx is wrong */
+			++cnt;
+#ifdef TPUT_MONITOR
+			pktlen  = PKTLEN(OSH_NULL, txp);
+			if ((bus->dhd->conf->data_drop_mode == TXPKT_DROP) && (pktlen > 500))
+				ret = BCME_OK;
+			else
+#endif
+			ret = dhd_prot_txdata(bus->dhd, txp, flow_ring_node->flow_info.ifindex);
+			if (ret != BCME_OK) { /* may not have resources in flow ring */
+				DHD_INFO(("%s: Reinserrt %d\n", __FUNCTION__, ret));
+#ifdef AGG_H2D_DB
+				if (agg_h2d_db_enab) {
+					dhd_prot_schedule_aggregate_h2d_db(bus->dhd, flow_id);
+				} else
+#endif /* AGG_H2D_DB */
+				{
+					dhd_prot_txdata_write_flush(bus->dhd, flow_id);
+				}
+				/* reinsert at head */
+				dhd_flow_queue_reinsert(bus->dhd, queue, txp);
+				DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+				if (is_qempty) {
+					*is_qempty = FALSE;
+				}
+				/* If we are able to requeue back, return success */
+				return BCME_OK;
+			}
+
+#ifdef DHD_MEM_STATS
+			DHD_MEM_STATS_LOCK(bus->dhd->mem_stats_lock, flags);
+			bus->dhd->txpath_mem += PKTLEN(bus->dhd->osh, txp);
+			DHD_INFO(("%s txpath_mem: %llu PKTLEN: %d\n",
+				__FUNCTION__, bus->dhd->txpath_mem, PKTLEN(bus->dhd->osh, txp)));
+			DHD_MEM_STATS_UNLOCK(bus->dhd->mem_stats_lock, flags);
+#endif /* DHD_MEM_STATS */
+
+			/* check bound and break if exceeded */
+			if (bound && cnt >= bound) {
+				break;
+			}
+		}
+
+		{
+#ifdef AGG_H2D_DB
+			if (agg_h2d_db_enab) {
+				dhd_prot_schedule_aggregate_h2d_db(bus->dhd, flow_id);
+			} else
+#endif /* AGG_H2D_DB */
+			{
+				dhd_prot_txdata_write_flush(bus->dhd, flow_id);
+			}
+		}
+
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+		if (is_qempty) {
+			*is_qempty = queue->len > 0 ? FALSE : TRUE;
+		}
+	}
+
+	return ret;
+} /* dhd_bus_schedule_queue */
+
+/** Sends an (ethernet) data frame (in 'txp') to the dongle. Callee disposes of txp. */
+int
+BCMFASTPATH(dhd_bus_txdata)(struct dhd_bus *bus, void *txp, uint8 ifidx)
+{
+	uint16 flowid;
+#ifdef IDLE_TX_FLOW_MGMT
+	uint8	node_status;
+#endif /* IDLE_TX_FLOW_MGMT */
+	flow_queue_t *queue;
+	flow_ring_node_t *flow_ring_node;
+	unsigned long flags;
+	int ret = BCME_OK;
+	void *txp_pend = NULL;
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+	void *ntxp = NULL;
+	uint8 prio = PKTPRIO(txp);
+#endif
+	uint8 *pktdata = (uint8 *)PKTDATA(bus->dhd->osh, txp);
+
+	if (!bus->dhd->flowid_allocator) {
+		DHD_ERROR(("%s: Flow ring not intited yet  \n", __FUNCTION__));
+		goto toss;
+	}
+
+	flowid = DHD_PKT_GET_FLOWID(txp);
+
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flowid);
+
+	DHD_TRACE(("%s: pkt flowid %d, status %d active %d\n",
+		__FUNCTION__, flowid, flow_ring_node->status, flow_ring_node->active));
+
+	DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+	if ((flowid > bus->dhd->max_tx_flowid) ||
+#ifdef IDLE_TX_FLOW_MGMT
+		(!flow_ring_node->active))
+#else
+		(!flow_ring_node->active) ||
+		(flow_ring_node->status == FLOW_RING_STATUS_DELETE_PENDING) ||
+		(flow_ring_node->status == FLOW_RING_STATUS_STA_FREEING))
+#endif /* IDLE_TX_FLOW_MGMT */
+	{
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+		DHD_INFO(("%s: Dropping pkt flowid %d, status %d active %d\n",
+			__FUNCTION__, flowid, flow_ring_node->status,
+			flow_ring_node->active));
+		ret = BCME_ERROR;
+			goto toss;
+	}
+
+#ifdef IDLE_TX_FLOW_MGMT
+	node_status = flow_ring_node->status;
+
+	/* handle diffrent status states here!! */
+	switch (node_status)
+	{
+		case FLOW_RING_STATUS_OPEN:
+
+			if (bus->enable_idle_flowring_mgmt) {
+				/* Move the node to the head of active list */
+				dhd_flow_ring_move_to_active_list_head(bus, flow_ring_node);
+			}
+			break;
+
+		case FLOW_RING_STATUS_SUSPENDED:
+			DHD_INFO(("Need to Initiate TX Flow resume\n"));
+			/* Issue resume_ring request */
+			dhd_bus_flow_ring_resume_request(bus,
+					flow_ring_node);
+			break;
+
+		case FLOW_RING_STATUS_CREATE_PENDING:
+		case FLOW_RING_STATUS_RESUME_PENDING:
+			/* Dont do anything here!! */
+			DHD_INFO(("Waiting for Flow create/resume! status is %u\n",
+				node_status));
+			break;
+
+		case FLOW_RING_STATUS_DELETE_PENDING:
+		default:
+			DHD_ERROR(("Dropping packet!! flowid %u status is %u\n",
+				flowid, node_status));
+			/* error here!! */
+			ret = BCME_ERROR;
+			DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+			goto toss;
+	}
+	/* Now queue the packet */
+#endif /* IDLE_TX_FLOW_MGMT */
+
+	queue = &flow_ring_node->queue; /* queue associated with flow ring */
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+	FOREACH_CHAINED_PKT(txp, ntxp) {
+		/* Tag the packet with flowid - Remember, only the head packet */
+		/* of the chain has been tagged with the FlowID in dhd_sendpkt */
+		/* Also set the priority */
+		DHD_PKT_SET_FLOWID(txp, flowid);
+		PKTSETPRIO(txp, prio);
+
+		if ((ret = dhd_flow_queue_enqueue(bus->dhd, queue, txp)) != BCME_OK) {
+			txp_pend = txp;
+			PKTSETCLINK((txp), ntxp);
+			break;
+		}
+	}
+#else  /* !(defined(BCM_ROUTER_DHD) && defined(HNDCTF)) */
+	if (dhd_match_pkt_type(bus->dhd, pktdata, (uint32)PKTLEN(bus->dhd->osh, txp))) {
+		if ((ret = dhd_flow_queue_enqueue_head(bus->dhd, queue, txp)) != BCME_OK)
+			txp_pend = txp;
+	} else
+	if ((ret = dhd_flow_queue_enqueue(bus->dhd, queue, txp)) != BCME_OK)
+		txp_pend = txp;
+#endif /* defined(BCM_ROUTER_DHD) && defined(HNDCTF */
+
+	DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+	if (flow_ring_node->status) {
+		DHD_INFO(("%s: Enq pkt flowid %d, status %d active %d\n",
+		    __FUNCTION__, flowid, flow_ring_node->status,
+		    flow_ring_node->active));
+		if (txp_pend) {
+			txp = txp_pend;
+			goto toss;
+		}
+		return BCME_OK;
+	}
+	ret = dhd_bus_schedule_queue(bus, flowid, FALSE, 0, NULL); /* from queue to flowring */
+
+	/* If we have anything pending, try to push into q */
+	if (txp_pend) {
+		DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+
+#if (defined(BCM_ROUTER_DHD) && defined(HNDCTF))
+		FOREACH_CHAINED_PKT(txp_pend, ntxp) {
+			/* Tag the packet with flowid and set packet priority */
+			DHD_PKT_SET_FLOWID(txp_pend, flowid);
+			PKTSETPRIO(txp_pend, prio);
+
+			if ((ret = dhd_flow_queue_enqueue(bus->dhd, queue, txp_pend))
+				     != BCME_OK) {
+				DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+				PKTSETCLINK((txp_pend), ntxp);
+				txp = txp_pend;
+				goto toss;
+			}
+		}
+#else  /* !(defined(BCM_ROUTER_DHD) && defined(HNDCTF)) */
+		if (dhd_match_pkt_type(bus->dhd, pktdata, (uint32)PKTLEN(bus->dhd->osh, txp))) {
+			if ((ret = dhd_flow_queue_enqueue_head(bus->dhd, queue, txp_pend)) != BCME_OK) {
+				DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+				txp = txp_pend;
+				goto toss;
+			}
+		} else
+		if ((ret = dhd_flow_queue_enqueue(bus->dhd, queue, txp_pend)) != BCME_OK) {
+			DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+			txp = txp_pend;
+			goto toss;
+		}
+#endif /* defined(BCM_ROUTER_DHD) && defined(HNDCTF) */
+
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+	}
+
+	return ret;
+
+toss:
+	DHD_INFO(("%s: Toss %d\n", __FUNCTION__, ret));
+	PKTCFREE(bus->dhd->osh, txp, TRUE);
+	return ret;
+} /* dhd_bus_txdata */
+
+void
+dhd_bus_stop_queue(struct dhd_bus *bus)
+{
+	dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, ON);
+}
+
+void
+dhd_bus_start_queue(struct dhd_bus *bus)
+{
+	/*
+	 * Tx queue has been stopped due to resource shortage (or)
+	 * bus is not in a state to turn on.
+	 *
+	 * Note that we try to re-start network interface only
+	 * when we have enough resources, one has to first change the
+	 * flag indicating we have all the resources.
+	 */
+	if (dhd_prot_check_tx_resource(bus->dhd)) {
+		DHD_ERROR(("%s: Interface NOT started, previously stopped "
+			"due to resource shortage\n", __FUNCTION__));
+		return;
+	}
+	dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+}
+
+/* Device console input function */
+int dhd_bus_console_in(dhd_pub_t *dhd, uchar *msg, uint msglen)
+{
+	dhd_bus_t *bus = dhd->bus;
+	uint32 addr, val;
+	int rv;
+#ifdef PCIE_INB_DW
+	unsigned long flags = 0;
+#endif /* PCIE_INB_DW */
+
+	/* Address could be zero if CONSOLE := 0 in dongle Makefile */
+	if (bus->console_addr == 0)
+		return BCME_UNSUPPORTED;
+
+	/* Don't allow input if dongle is in reset */
+	if (bus->dhd->dongle_reset) {
+		return BCME_NOTREADY;
+	}
+
+	/* Zero cbuf_index */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, cbuf_idx);
+	/* handle difference in definition of hnd_log_t in certain branches */
+	if (dhd->wlc_ver_major < 14) {
+		addr -= (uint32)sizeof(uint32);
+	}
+	val = htol32(0);
+	if ((rv = dhdpcie_bus_membytes(bus, TRUE, addr, (uint8 *)&val, sizeof(val))) < 0)
+		goto done;
+
+	/* Write message into cbuf */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, cbuf);
+	/* handle difference in definition of hnd_log_t in certain branches */
+	if (dhd->wlc_ver_major < 14) {
+		addr -= sizeof(uint32);
+	}
+	if ((rv = dhdpcie_bus_membytes(bus, TRUE, addr, (uint8 *)msg, msglen)) < 0)
+		goto done;
+
+	/* Write length into vcons_in */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, vcons_in);
+	val = htol32(msglen);
+	if ((rv = dhdpcie_bus_membytes(bus, TRUE, addr, (uint8 *)&val, sizeof(val))) < 0)
+		goto done;
+
+#ifdef PCIE_INB_DW
+	/* Use a lock to ensure this tx DEVICE_WAKE + tx H2D_HOST_CONS_INT sequence is
+	 * mutually exclusive with the rx D2H_DEV_DS_ENTER_REQ + tx H2D_HOST_DS_ACK sequence.
+	 */
+	DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+#endif /* PCIE_INB_DW */
+
+	/* generate an interrupt to dongle to indicate that it needs to process cons command */
+	dhdpcie_send_mb_data(bus, H2D_HOST_CONS_INT);
+
+#ifdef PCIE_INB_DW
+	DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+#endif /* PCIE_INB_DW */
+done:
+	return rv;
+} /* dhd_bus_console_in */
+
+/**
+ * Called on frame reception, the frame was received from the dongle on interface 'ifidx' and is
+ * contained in 'pkt'. Processes rx frame, forwards up the layer to netif.
+ */
+void
+BCMFASTPATH(dhd_bus_rx_frame)(struct dhd_bus *bus, void* pkt, int ifidx, uint pkt_count)
+{
+	dhd_rx_frame(bus->dhd, ifidx, pkt, pkt_count, 0);
+}
+
+/* Aquire/Release bar1_switch_lock only if the chip supports bar1 switching */
+#define DHD_BUS_BAR1_SWITCH_LOCK(bus, flags) \
+	((bus)->bar1_switch_enab) ? DHD_BAR1_SWITCH_LOCK((bus)->bar1_switch_lock, flags) : \
+		BCM_REFERENCE(flags)
+
+#define DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags) \
+	((bus)->bar1_switch_enab) ? DHD_BAR1_SWITCH_UNLOCK((bus)->bar1_switch_lock, flags) : \
+		BCM_REFERENCE(flags)
+
+/* Init/Deinit bar1_switch_lock only if the chip supports bar1 switching */
+void
+dhd_init_bar1_switch_lock(dhd_bus_t *bus)
+{
+	if (bus->bar1_switch_enab && !bus->bar1_switch_lock) {
+		bus->bar1_switch_lock = osl_spin_lock_init(bus->osh);
+	}
+}
+
+static void
+dhd_deinit_bar1_switch_lock(dhd_bus_t *bus)
+{
+	if (bus->bar1_switch_enab && bus->bar1_switch_lock) {
+		osl_spin_lock_deinit(bus->osh, bus->bar1_switch_lock);
+		bus->bar1_switch_lock = NULL;
+	}
+}
+
+/*
+ * The bpwindow for any address will be lower bound of multiples of bar1_size.
+ * For eg, if addr=0x938fff and bar1_size is 0x400000, then
+ * address will fall in the window of 0x800000-0xbfffff, so need
+ * to select bpwindow as 0x800000.
+ * To achieve this mask the LSB nibbles of bar1_size of the given addr.
+ */
+#define DHD_BUS_BAR1_BPWIN(addr, bar1_size) \
+	(uint32)((addr) & ~((bar1_size) - 1))
+
+/**
+ * dhdpcie_bar1_window_switch_enab
+ *
+ * Check if the chip requires BAR1 window switching based on
+ * dongle_ram_base, ramsize and mapped bar1_size and sets
+ * bus->bar1_switch_enab accordingly
+ * @bus: dhd bus context
+ *
+ */
+void
+dhdpcie_bar1_window_switch_enab(dhd_bus_t *bus)
+{
+	uint32 ramstart = bus->dongle_ram_base;
+	uint32 ramend = bus->dongle_ram_base + bus->ramsize - 1;
+	uint32 bpwinstart = DHD_BUS_BAR1_BPWIN(ramstart, bus->bar1_size);
+	uint32 bpwinend = DHD_BUS_BAR1_BPWIN(ramend, bus->bar1_size);
+
+	bus->bar1_switch_enab = FALSE;
+
+	/*
+	 * Window switch is needed to access complete BAR1
+	 * if bpwinstart and bpwinend are different
+	 */
+	if (bpwinstart != bpwinend) {
+		bus->bar1_switch_enab = TRUE;
+	}
+
+	DHD_ERROR_MEM(("%s: bar1_switch_enab=%d ramstart=0x%x ramend=0x%x bar1_size=0x%x\n",
+		__FUNCTION__, bus->bar1_switch_enab, ramstart, ramend, bus->bar1_size));
+}
+
+/**
+ * dhdpcie_setbar1win
+ *
+ * os independendent function for setting bar1 window in order to allow
+ * also set current window positon.
+ *
+ * @bus: dhd bus context
+ * @addr: new backplane windows address for BAR1
+ */
+static void
+dhdpcie_setbar1win(dhd_bus_t *bus, uint32 addr)
+{
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCI_BAR1_WIN, 4, addr);
+	bus->curr_bar1_win = addr;
+}
+
+/**
+ * dhdpcie_bus_chkandshift_bpoffset
+ *
+ * Check the provided address is within the current BAR1 window,
+ * if not, shift the window
+ *
+ * @bus: dhd bus context
+ * @offset: back plane address that the caller wants to access
+ *
+ * Return: new offset for access
+ */
+static ulong
+dhdpcie_bus_chkandshift_bpoffset(dhd_bus_t *bus, ulong offset)
+{
+
+	uint32 bpwin;
+	/* Determine BAR1 backplane window using window size
+	 * Window address mask should be ~(size - 1)
+	 */
+	bpwin = DHD_BUS_BAR1_BPWIN(offset, bus->bar1_size);
+
+	if (bpwin != bus->curr_bar1_win) {
+		DHD_INFO(("%s: move BAR1 window curr_bar1_win=0x%x bpwin=0x%x offset=0x%lx\n",
+			__FUNCTION__, bus->curr_bar1_win, bpwin, offset));
+		/* Move BAR1 window */
+		dhdpcie_setbar1win(bus, bpwin);
+	}
+
+	return offset - bpwin;
+}
+
+/** 'offset' is a backplane address */
+void
+dhdpcie_bus_wtcm8(dhd_bus_t *bus, ulong offset, uint8 data)
+{
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if defined(linux) || defined(LINUX)
+	W_REG(bus->dhd->osh, (volatile uint8 *)(bus->tcm + offset), data);
+#else
+	*(volatile uint8 *)(bus->tcm + offset) = (uint8)data;
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+}
+
+void
+dhdpcie_bus_wtcm16(dhd_bus_t *bus, ulong offset, uint16 data)
+{
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if defined(linux) || defined(LINUX)
+	W_REG(bus->dhd->osh, (volatile uint16 *)(bus->tcm + offset), data);
+#else
+	*(volatile uint16 *)(bus->tcm + offset) = (uint16)data;
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+}
+
+void
+dhdpcie_bus_wtcm32(dhd_bus_t *bus, ulong offset, uint32 data)
+{
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if defined(linux) || defined(LINUX)
+	W_REG(bus->dhd->osh, (volatile uint32 *)(bus->tcm + offset), data);
+#else
+	*(volatile uint32 *)(bus->tcm + offset) = (uint32)data;
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+}
+
+#ifdef DHD_SUPPORT_64BIT
+void
+dhdpcie_bus_wtcm64(dhd_bus_t *bus, ulong offset, uint64 data)
+{
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if defined(linux) || defined(LINUX)
+	W_REG(bus->dhd->osh, (volatile uint64 *)(bus->tcm + offset), data);
+#else
+	*(volatile uint64 *)(bus->tcm + offset) = (uint64)data;
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+}
+#endif /* DHD_SUPPORT_64BIT */
+
+uint8
+dhdpcie_bus_rtcm8(dhd_bus_t *bus, ulong offset)
+{
+	volatile uint8 data;
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		data = (uint8)-1;
+		return data;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if defined(linux) || defined(LINUX)
+	data = R_REG(bus->dhd->osh, (volatile uint8 *)(bus->tcm + offset));
+#else
+	data = *(volatile uint8 *)(bus->tcm + offset);
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+	return data;
+}
+
+uint16
+dhdpcie_bus_rtcm16(dhd_bus_t *bus, ulong offset)
+{
+	volatile uint16 data;
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		data = (uint16)-1;
+		return data;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if (defined(linux) || defined(LINUX))
+	data = R_REG(bus->dhd->osh, (volatile uint16 *)(bus->tcm + offset));
+#else
+	data = *(volatile uint16 *)(bus->tcm + offset);
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+	return data;
+}
+
+uint32
+dhdpcie_bus_rtcm32(dhd_bus_t *bus, ulong offset)
+{
+	volatile uint32 data;
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		data = (uint32)-1;
+		return data;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if (defined(linux) || defined(LINUX))
+	data = R_REG(bus->dhd->osh, (volatile uint32 *)(bus->tcm + offset));
+#else
+	data = *(volatile uint32 *)(bus->tcm + offset);
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+	return data;
+}
+
+#ifdef DHD_SUPPORT_64BIT
+uint64
+dhdpcie_bus_rtcm64(dhd_bus_t *bus, ulong offset)
+{
+	volatile uint64 data;
+	ulong flags = 0;
+
+	if (bus->is_linkdown) {
+		DHD_LOG_MEM(("%s: PCIe link was down\n", __FUNCTION__));
+		data = (uint64)-1;
+		return data;
+	}
+
+	DHD_BUS_BAR1_SWITCH_LOCK(bus, flags);
+
+	offset = dhdpcie_bus_chkandshift_bpoffset(bus, offset);
+
+#if (defined(linux) || defined(LINUX))
+	data = R_REG(bus->dhd->osh, (volatile uint64 *)(bus->tcm + offset));
+#else
+	data = *(volatile uint64 *)(bus->tcm + offset);
+#endif /* linux || LINUX */
+
+	DHD_BUS_BAR1_SWITCH_UNLOCK(bus, flags);
+	return data;
+}
+#endif /* DHD_SUPPORT_64BIT */
+
+void
+dhdpcie_update_ring_ptrs_in_tcm_with_req_pwr(dhd_bus_t *bus, void *data, uint8 type,
+	uint16 ringid, bool read, bool req_pwr)
+{
+	ulong addr;
+
+	if (type == RING_WR_UPD) {
+		addr = bus->ring_sh[ringid].ring_state_w;
+	} else if (type == RING_RD_UPD) {
+		addr = bus->ring_sh[ringid].ring_state_r;
+	} else {
+		DHD_ERROR(("%s: invalid type:%d\n", __FUNCTION__, type));
+		return;
+	}
+
+	if (req_pwr && MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	if (read) {
+		/* Read */
+		*(uint16*)data = LTOH16(dhdpcie_bus_rtcm16(bus, addr));
+	} else {
+		/* Write */
+		dhdpcie_bus_wtcm16(bus, addr, (uint16) HTOL16(*(uint16 *)data));
+	}
+
+	if (req_pwr && MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+}
+
+void
+dhdpcie_update_ring_ptrs_in_tcm(dhd_bus_t *bus, void *data, uint8 type,	uint16 ringid,
+	bool read)
+{
+#ifdef PCIE_INB_DW
+	if (INBAND_DW_ENAB(bus)) {
+		ulong flags_ds;
+		DHD_BUS_DONGLE_DS_LOCK(bus->dongle_ds_lock, flags_ds);
+		dhdpcie_update_ring_ptrs_in_tcm_with_req_pwr(bus, data, type, ringid, read,
+			bus->dongle_in_deepsleep);
+		DHD_BUS_DONGLE_DS_UNLOCK(bus->dongle_ds_lock, flags_ds);
+	} else
+#endif /* PCIE_INB_DW */
+	{
+		/* Request power explicitly */
+		dhdpcie_update_ring_ptrs_in_tcm_with_req_pwr(bus, data, type, ringid, read, TRUE);
+	}
+}
+
+/** A snippet of dongle memory is shared between host and dongle */
+void
+dhd_bus_cmn_writeshared(dhd_bus_t *bus, void *data, uint32 len, uint8 type, uint16 ringid)
+{
+	uint64 long_data;
+	ulong addr; /* dongle address */
+
+	DHD_INFO(("%s: writing to dongle type %d len %d\n", __FUNCTION__, type, len));
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	/*
+	 * Use explicit tcm ring ptr update functions when DMA indices are not enabled to
+	 * as backplane power request calls are causing TPUT drops
+	 */
+	if (!(bus->dhd->dma_d2h_ring_upd_support || bus->dhd->dma_h2d_ring_upd_support)) {
+		if ((type == RING_WR_UPD) || (type == RING_RD_UPD)) {
+			dhdpcie_update_ring_ptrs_in_tcm(bus, data, type, ringid, FALSE);
+			return;
+		}
+	}
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+	switch (type) {
+		case RING_WR_UPD :
+			addr = bus->ring_sh[ringid].ring_state_w;
+			dhdpcie_bus_wtcm16(bus, addr, (uint16) HTOL16(*(uint16 *)data));
+			break;
+
+		case RING_RD_UPD :
+			addr = bus->ring_sh[ringid].ring_state_r;
+			dhdpcie_bus_wtcm16(bus, addr, (uint16) HTOL16(*(uint16 *)data));
+			break;
+
+		case D2H_DMA_SCRATCH_BUF:
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_dma_scratch_buffer);
+			long_data = HTOL64(*(uint64 *)data);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8*) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case D2H_DMA_SCRATCH_BUF_LEN :
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_dma_scratch_buffer_len);
+			dhdpcie_bus_wtcm32(bus, addr, (uint32) HTOL32(*(uint32 *)data));
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case H2D_DMA_INDX_WR_BUF:
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_RING_INFO_MEMBER_ADDR(bus, h2d_w_idx_hostaddr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8*) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case H2D_DMA_INDX_RD_BUF:
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_RING_INFO_MEMBER_ADDR(bus, h2d_r_idx_hostaddr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8*) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case D2H_DMA_INDX_WR_BUF:
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_RING_INFO_MEMBER_ADDR(bus, d2h_w_idx_hostaddr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8*) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case D2H_DMA_INDX_RD_BUF:
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_RING_INFO_MEMBER_ADDR(bus, d2h_r_idx_hostaddr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8*) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case H2D_IFRM_INDX_WR_BUF:
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_RING_INFO_MEMBER_ADDR(bus, ifrm_w_idx_hostaddr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8*) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case RING_ITEM_LEN :
+			addr = DHD_RING_MEM_MEMBER_ADDR(bus, ringid, len_items);
+			dhdpcie_bus_wtcm16(bus, addr, (uint16) HTOL16(*(uint16 *)data));
+			break;
+
+		case RING_MAX_ITEMS :
+			addr = DHD_RING_MEM_MEMBER_ADDR(bus, ringid, max_item);
+			dhdpcie_bus_wtcm16(bus, addr, (uint16) HTOL16(*(uint16 *)data));
+			break;
+
+		case RING_BUF_ADDR :
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_RING_MEM_MEMBER_ADDR(bus, ringid, base_addr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8 *) &long_data, len);
+			if (dhd_msg_level & DHD_INFO_VAL) {
+				prhex(__FUNCTION__, data, len);
+			}
+			break;
+
+		case D2H_MB_DATA:
+			addr = bus->d2h_mb_data_ptr_addr;
+			dhdpcie_bus_wtcm32(bus, addr, (uint32) HTOL32(*(uint32 *)data));
+			break;
+
+		case H2D_MB_DATA:
+			addr = bus->h2d_mb_data_ptr_addr;
+			dhdpcie_bus_wtcm32(bus, addr, (uint32) HTOL32(*(uint32 *)data));
+			break;
+
+		case HOST_API_VERSION:
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_cap);
+			dhdpcie_bus_wtcm32(bus, addr, (uint32) HTOL32(*(uint32 *)data));
+			break;
+
+		case DNGL_TO_HOST_TRAP_ADDR:
+			long_data = HTOL64(*(uint64 *)data);
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_trap_addr);
+			dhdpcie_bus_membytes(bus, TRUE, addr, (uint8 *) &long_data, len);
+			DHD_INFO(("Wrote trap addr:0x%x\n", (uint32) HTOL32(*(uint32 *)data)));
+			break;
+
+#ifdef D2H_MINIDUMP
+		case DNGL_TO_HOST_TRAP_ADDR_LEN:
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, device_trap_debug_buffer_len);
+			dhdpcie_bus_wtcm16(bus, addr, (uint16) HTOL16(*(uint16 *)data));
+			break;
+#endif /* D2H_MINIDUMP */
+
+		case HOST_SCB_ADDR:
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_scb_addr);
+#ifdef DHD_SUPPORT_64BIT
+			dhdpcie_bus_wtcm64(bus, addr, (uint64) HTOL64(*(uint64 *)data));
+#else /* !DHD_SUPPORT_64BIT */
+			dhdpcie_bus_wtcm32(bus, addr, *((uint32*)data));
+#endif /* DHD_SUPPORT_64BIT */
+			DHD_INFO(("Wrote host_scb_addr:0x%x\n",
+				(uint32) HTOL32(*(uint32 *)data)));
+			break;
+
+#ifdef DHD_PCIE_PTM
+		case HOST_CAP2:
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_cap2);
+			dhdpcie_bus_wtcm32(bus, addr, (uint32) HTOL32(*(uint32 *)data));
+			DHD_INFO(("Wrote host_cap2 0x%x\n", (uint32) HTOL32(*(uint32 *)data)));
+			break;
+#endif /* DHD_PCIE_PTM */
+
+		default:
+			break;
+	}
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+} /* dhd_bus_cmn_writeshared */
+
+/** A snippet of dongle memory is shared between host and dongle */
+void
+dhd_bus_cmn_readshared(dhd_bus_t *bus, void* data, uint8 type, uint16 ringid)
+{
+	ulong addr; /* dongle address */
+
+	/*
+	 * Use explicit tcm ring ptr update functions when DMA indices are not enabled to
+	 * as backplane power request calls are causing TPUT drops
+	 */
+	if (!(bus->dhd->dma_d2h_ring_upd_support || bus->dhd->dma_h2d_ring_upd_support)) {
+		if ((type == RING_WR_UPD) || (type == RING_RD_UPD)) {
+			dhdpcie_update_ring_ptrs_in_tcm(bus, data, type, ringid, TRUE);
+			return;
+		}
+	}
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+	switch (type) {
+		case RING_WR_UPD :
+			addr = bus->ring_sh[ringid].ring_state_w;
+			*(uint16*)data = LTOH16(dhdpcie_bus_rtcm16(bus, addr));
+			break;
+
+		case RING_RD_UPD :
+			addr = bus->ring_sh[ringid].ring_state_r;
+			*(uint16*)data = LTOH16(dhdpcie_bus_rtcm16(bus, addr));
+			break;
+
+		case TOTAL_LFRAG_PACKET_CNT :
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, total_lfrag_pkt_cnt);
+			*(uint16*)data = LTOH16(dhdpcie_bus_rtcm16(bus, addr));
+			break;
+
+		case H2D_MB_DATA:
+			addr = bus->h2d_mb_data_ptr_addr;
+			*(uint32*)data = LTOH32(dhdpcie_bus_rtcm32(bus, addr));
+			break;
+
+		case D2H_MB_DATA:
+			addr = bus->d2h_mb_data_ptr_addr;
+			*(uint32*)data = LTOH32(dhdpcie_bus_rtcm32(bus, addr));
+			break;
+
+		case MAX_HOST_RXBUFS :
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, max_host_rxbufs);
+			*(uint16*)data = LTOH16(dhdpcie_bus_rtcm16(bus, addr));
+			break;
+
+		case HOST_SCB_ADDR:
+			addr = DHD_PCIE_SHARED_MEMBER_ADDR(bus, host_scb_size);
+			*(uint32*)data = LTOH32(dhdpcie_bus_rtcm32(bus, addr));
+			break;
+
+		default :
+			break;
+	}
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+}
+
+uint32 dhd_bus_get_sharedflags(dhd_bus_t *bus)
+{
+	return ((pciedev_shared_t*)bus->pcie_sh)->flags;
+}
+
+void dhd_prot_clearcounts(dhd_pub_t *dhd);
+
+void
+dhd_bus_clearcounts(dhd_pub_t *dhdp)
+{
+	dhd_prot_clearcounts(dhdp);
+	dhdp->rx_pktgetpool_fail = 0;
+
+	dhd_clear_dpc_histos(dhdp);
+}
+
+/**
+ * @param params    input buffer, NULL for 'set' operation.
+ * @param plen      length of 'params' buffer, 0 for 'set' operation.
+ * @param arg       output buffer
+ */
+int
+dhd_bus_iovar_op(dhd_pub_t *dhdp, const char *name,
+                 void *params, uint plen, void *arg, uint len, bool set)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	const bcm_iovar_t *vi = NULL;
+	int bcmerror = BCME_UNSUPPORTED;
+	uint val_size;
+	uint32 actionid;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(name);
+	if (!name)
+		return BCME_BADARG;
+
+	/* Get MUST have return space */
+	ASSERT(set || (arg && len));
+	if (!(set || (arg && len)))
+		return BCME_BADARG;
+
+	/* Set does NOT take qualifiers */
+	ASSERT(!set || (!params && !plen));
+	if (!(!set || (!params && !plen)))
+		return BCME_BADARG;
+
+	DHD_INFO(("%s: %s %s, len %d plen %d\n", __FUNCTION__,
+	         name, (set ? "set" : "get"), len, plen));
+
+	/* Look up var locally; if not found pass to host driver */
+	if ((vi = bcm_iovar_lookup(dhdpcie_iovars, name)) == NULL) {
+		goto exit;
+	}
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		if (vi->flags & DHD_IOVF_PWRREQ_BYPASS) {
+			DHD_ERROR(("%s: Bypass pwr request\n", __FUNCTION__));
+		} else {
+			dhd_bus_pcie_pwr_req(bus);
+		}
+	}
+
+	/* set up 'params' pointer in case this is a set command so that
+	 * the convenience int and bool code can be common to set and get
+	 */
+	if (params == NULL) {
+		params = arg;
+		plen = len;
+	}
+
+	if (vi->type == IOVT_VOID)
+		val_size = 0;
+	else if (vi->type == IOVT_BUFFER)
+		val_size = len;
+	else
+		/* all other types are integer sized */
+		val_size = sizeof(int);
+
+	actionid = set ? IOV_SVAL(vi->varid) : IOV_GVAL(vi->varid);
+	bcmerror = dhdpcie_bus_doiovar(bus, vi, actionid, name, params, plen, arg, len, val_size);
+
+exit:
+	/* In DEVRESET_QUIESCE/DEVRESET_ON,
+	 * this includes dongle re-attach which initialize pwr_req_ref count to 0 and
+	 * causes pwr_req_ref count miss-match in pwr req clear function and hang.
+	 * In this case, bypass pwr req clear.
+	 */
+	if (bcmerror == BCME_DNGL_DEVRESET) {
+		bcmerror = BCME_OK;
+	} else {
+		if (MULTIBP_ENAB(bus->sih)) {
+			if (vi != NULL) {
+				if (vi->flags & DHD_IOVF_PWRREQ_BYPASS) {
+					DHD_ERROR(("%s: Bypass pwr request clear\n", __FUNCTION__));
+				} else {
+					dhd_bus_pcie_pwr_req_clear(bus);
+				}
+			}
+		}
+	}
+	return bcmerror;
+} /* dhd_bus_iovar_op */
+
+#ifdef BCM_BUZZZ
+#include <bcm_buzzz.h>
+
+int
+dhd_buzzz_dump_cntrs(char *p, uint32 *core, uint32 *log,
+	const int num_counters)
+{
+	int bytes = 0;
+	uint32 ctr;
+	uint32 curr[BCM_BUZZZ_COUNTERS_MAX], prev[BCM_BUZZZ_COUNTERS_MAX];
+	uint32 delta[BCM_BUZZZ_COUNTERS_MAX];
+
+	/* Compute elapsed counter values per counter event type */
+	for (ctr = 0U; ctr < num_counters; ctr++) {
+		prev[ctr] = core[ctr];
+		curr[ctr] = *log++;
+		core[ctr] = curr[ctr];  /* saved for next log */
+
+		if (curr[ctr] < prev[ctr])
+			delta[ctr] = curr[ctr] + (~0U - prev[ctr]);
+		else
+			delta[ctr] = (curr[ctr] - prev[ctr]);
+
+		bytes += sprintf(p + bytes, "%12u ", delta[ctr]);
+	}
+
+	return bytes;
+}
+
+typedef union cm3_cnts { /* export this in bcm_buzzz.h */
+	uint32 u32;
+	uint8  u8[4];
+	struct {
+		uint8 cpicnt;
+		uint8 exccnt;
+		uint8 sleepcnt;
+		uint8 lsucnt;
+	};
+} cm3_cnts_t;
+
+int
+dhd_bcm_buzzz_dump_cntrs6(char *p, uint32 *core, uint32 *log)
+{
+	int bytes = 0;
+
+	uint32 cyccnt, instrcnt;
+	cm3_cnts_t cm3_cnts;
+	uint8 foldcnt;
+
+	{   /* 32bit cyccnt */
+		uint32 curr, prev, delta;
+		prev = core[0]; curr = *log++; core[0] = curr;
+		if (curr < prev)
+			delta = curr + (~0U - prev);
+		else
+			delta = (curr - prev);
+
+		bytes += sprintf(p + bytes, "%12u ", delta);
+		cyccnt = delta;
+	}
+
+	{	/* Extract the 4 cnts: cpi, exc, sleep and lsu */
+		int i;
+		uint8 max8 = ~0;
+		cm3_cnts_t curr, prev, delta;
+		prev.u32 = core[1]; curr.u32 = * log++; core[1] = curr.u32;
+		for (i = 0; i < 4; i++) {
+			if (curr.u8[i] < prev.u8[i])
+				delta.u8[i] = curr.u8[i] + (max8 - prev.u8[i]);
+			else
+				delta.u8[i] = (curr.u8[i] - prev.u8[i]);
+			bytes += sprintf(p + bytes, "%4u ", delta.u8[i]);
+		}
+		cm3_cnts.u32 = delta.u32;
+	}
+
+	{   /* Extract the foldcnt from arg0 */
+		uint8 curr, prev, delta, max8 = ~0;
+		bcm_buzzz_arg0_t arg0; arg0.u32 = *log;
+		prev = core[2]; curr = arg0.klog.cnt; core[2] = curr;
+		if (curr < prev)
+			delta = curr + (max8 - prev);
+		else
+			delta = (curr - prev);
+		bytes += sprintf(p + bytes, "%4u ", delta);
+		foldcnt = delta;
+	}
+
+	instrcnt = cyccnt - (cm3_cnts.u8[0] + cm3_cnts.u8[1] + cm3_cnts.u8[2]
+		                 + cm3_cnts.u8[3]) + foldcnt;
+	if (instrcnt > 0xFFFFFF00)
+		bytes += sprintf(p + bytes, "[%10s] ", "~");
+	else
+		bytes += sprintf(p + bytes, "[%10u] ", instrcnt);
+	return bytes;
+}
+
+int
+dhd_buzzz_dump_log(char *p, uint32 *core, uint32 *log, bcm_buzzz_t *buzzz)
+{
+	int bytes = 0;
+	bcm_buzzz_arg0_t arg0;
+	static uint8 * fmt[] = BCM_BUZZZ_FMT_STRINGS;
+
+	if (buzzz->counters == 6) {
+		bytes += dhd_bcm_buzzz_dump_cntrs6(p, core, log);
+		log += 2; /* 32bit cyccnt + (4 x 8bit) CM3 */
+	} else {
+		bytes += dhd_buzzz_dump_cntrs(p, core, log, buzzz->counters);
+		log += buzzz->counters; /* (N x 32bit) CR4=3, CA7=4 */
+	}
+
+	/* Dump the logged arguments using the registered formats */
+	arg0.u32 = *log++;
+
+	switch (arg0.klog.args) {
+		case 0:
+			bytes += sprintf(p + bytes, fmt[arg0.klog.id]);
+			break;
+		case 1:
+		{
+			uint32 arg1 = *log++;
+			bytes += sprintf(p + bytes, fmt[arg0.klog.id], arg1);
+			break;
+		}
+		case 2:
+		{
+			uint32 arg1, arg2;
+			arg1 = *log++; arg2 = *log++;
+			bytes += sprintf(p + bytes, fmt[arg0.klog.id], arg1, arg2);
+			break;
+		}
+		case 3:
+		{
+			uint32 arg1, arg2, arg3;
+			arg1 = *log++; arg2 = *log++; arg3 = *log++;
+			bytes += sprintf(p + bytes, fmt[arg0.klog.id], arg1, arg2, arg3);
+			break;
+		}
+		case 4:
+		{
+			uint32 arg1, arg2, arg3, arg4;
+			arg1 = *log++; arg2 = *log++;
+			arg3 = *log++; arg4 = *log++;
+			bytes += sprintf(p + bytes, fmt[arg0.klog.id], arg1, arg2, arg3, arg4);
+			break;
+		}
+		default:
+			DHD_CONS_ONLY(("%s: Maximum one argument supported\n", __FUNCTION__));
+			break;
+	}
+
+	bytes += sprintf(p + bytes, "\n");
+
+	return bytes;
+}
+
+void dhd_buzzz_dump(bcm_buzzz_t *buzzz_p, void *buffer_p, char *p)
+{
+	int i;
+	uint32 total, part1, part2, log_sz, core[BCM_BUZZZ_COUNTERS_MAX];
+	void * log;
+
+	for (i = 0; i < BCM_BUZZZ_COUNTERS_MAX; i++) {
+		core[i] = 0;
+	}
+
+	log_sz = buzzz_p->log_sz;
+
+	part1 = ((uint32)buzzz_p->cur - (uint32)buzzz_p->log) / log_sz;
+
+	if (buzzz_p->wrap == TRUE) {
+		part2 = ((uint32)buzzz_p->end - (uint32)buzzz_p->cur) / log_sz;
+		total = (buzzz_p->buffer_sz - BCM_BUZZZ_LOGENTRY_MAXSZ) / log_sz;
+	} else {
+		part2 = 0U;
+		total = buzzz_p->count;
+	}
+
+	if (total == 0U) {
+		DHD_CONS_ONLY(("%s: bcm_buzzz_dump total<%u> done\n", __FUNCTION__, total));
+		return;
+	} else {
+		DHD_CONS_ONLY(("%s: bcm_buzzz_dump total<%u> : part2<%u> + part1<%u>\n",
+			__FUNCTION__, total, part2, part1));
+	}
+
+	if (part2) {   /* with wrap */
+		log = (void*)((size_t)buffer_p + (buzzz_p->cur - buzzz_p->log));
+		while (part2--) {   /* from cur to end : part2 */
+			p[0] = '\0';
+			dhd_buzzz_dump_log(p, core, (uint32 *)log, buzzz_p);
+			printf("%s", p);
+			log = (void*)((size_t)log + buzzz_p->log_sz);
+		}
+	}
+
+	log = (void*)buffer_p;
+	while (part1--) {
+		p[0] = '\0';
+		dhd_buzzz_dump_log(p, core, (uint32 *)log, buzzz_p);
+		printf("%s", p);
+		log = (void*)((size_t)log + buzzz_p->log_sz);
+	}
+
+	DHD_CONS_ONLY(("%s: bcm_buzzz_dump done.\n", __FUNCTION__));
+}
+
+int dhd_buzzz_dump_dngl(dhd_bus_t *bus)
+{
+	bcm_buzzz_t * buzzz_p = NULL;
+	void * buffer_p = NULL;
+	char * page_p = NULL;
+	pciedev_shared_t *sh;
+	int ret = 0;
+
+	if (bus->dhd->busstate != DHD_BUS_DATA) {
+		return BCME_UNSUPPORTED;
+	}
+	if ((page_p = (char *)MALLOC(bus->dhd->osh, 4096)) == NULL) {
+		DHD_CONS_ONLY(("%s: Page memory allocation failure\n", __FUNCTION__));
+		goto done;
+	}
+	if ((buzzz_p = MALLOC(bus->dhd->osh, sizeof(bcm_buzzz_t))) == NULL) {
+		DHD_CONS_ONLY(("%s: BCM BUZZZ memory allocation failure\n", __FUNCTION__));
+		goto done;
+	}
+
+	ret = dhdpcie_readshared(bus);
+	if (ret < 0) {
+		DHD_ERROR(("%s :Shared area read failed \n", __FUNCTION__));
+		goto done;
+	}
+
+	sh = bus->pcie_sh;
+
+	DHD_INFO(("%s buzzz:%08x\n", __FUNCTION__, sh->buzz_dbg_ptr));
+
+	if (sh->buzz_dbg_ptr != 0U) {	/* Fetch and display dongle BUZZZ Trace */
+
+		dhdpcie_bus_membytes(bus, FALSE, (ulong)sh->buzz_dbg_ptr,
+		                     (uint8 *)buzzz_p, sizeof(bcm_buzzz_t));
+
+		DHD_CONS_ONLY(("BUZZZ[0x%08x]: log<0x%08x> cur<0x%08x> end<0x%08x> "
+			"count<%u> status<%u> wrap<%u>\n"
+			"cpu<0x%02X> counters<%u> group<%u> buffer_sz<%u> log_sz<%u>\n",
+			(int)sh->buzz_dbg_ptr,
+			(int)buzzz_p->log, (int)buzzz_p->cur, (int)buzzz_p->end,
+			buzzz_p->count, buzzz_p->status, buzzz_p->wrap,
+			buzzz_p->cpu_idcode, buzzz_p->counters, buzzz_p->group,
+			buzzz_p->buffer_sz, buzzz_p->log_sz));
+
+		if (buzzz_p->count == 0) {
+			DHD_CONS_ONLY(("%s: Empty dongle BUZZZ trace\n\n", __FUNCTION__));
+			goto done;
+		}
+
+		/* Allocate memory for trace buffer and format strings */
+		buffer_p = MALLOC(bus->dhd->osh, buzzz_p->buffer_sz);
+		if (buffer_p == NULL) {
+			DHD_CONS_ONLY(("%s: Buffer memory allocation failure\n", __FUNCTION__));
+			goto done;
+		}
+
+		/* Fetch the trace. format strings are exported via bcm_buzzz.h */
+		dhdpcie_bus_membytes(bus, FALSE, (uint32)buzzz_p->log,   /* Trace */
+		                     (uint8 *)buffer_p, buzzz_p->buffer_sz);
+
+		/* Process and display the trace using formatted output */
+
+		{
+			int ctr;
+			for (ctr = 0; ctr < buzzz_p->counters; ctr++) {
+				printf("<Evt[%02X]> ", buzzz_p->eventid[ctr]);
+			}
+			DHD_CONS_ONLY(("<code execution point>\n"));
+		}
+
+		dhd_buzzz_dump(buzzz_p, buffer_p, page_p);
+
+		DHD_CONS_ONLY(("%s: ----- End of dongle BCM BUZZZ Trace -----\n\n", __FUNCTION__));
+
+		MFREE(bus->dhd->osh, buffer_p, buzzz_p->buffer_sz); buffer_p = NULL;
+	}
+
+done:
+
+	if (page_p)   MFREE(bus->dhd->osh, page_p, 4096);
+	if (buzzz_p)  MFREE(bus->dhd->osh, buzzz_p, sizeof(bcm_buzzz_t));
+	if (buffer_p) MFREE(bus->dhd->osh, buffer_p, buzzz_p->buffer_sz);
+
+	return BCME_OK;
+}
+#endif /* BCM_BUZZZ */
+
+#define PCIE_GEN2(sih) ((BUSTYPE((sih)->bustype) == PCI_BUS) &&	\
+	((sih)->buscoretype == PCIE2_CORE_ID))
+#ifdef DHD_PCIE_REG_ACCESS
+static bool
+pcie2_mdiosetblock(dhd_bus_t *bus, uint blk)
+{
+	uint mdiodata, mdioctrl, i = 0;
+	uint pcie_serdes_spinwait = 200;
+
+	mdioctrl = MDIOCTL2_DIVISOR_VAL | (0x1F << MDIOCTL2_REGADDR_SHF);
+	mdiodata = (blk << MDIODATA2_DEVADDR_SHF) | MDIODATA2_DONE;
+
+	si_corereg(bus->sih, bus->sih->buscoreidx, PCIE2_MDIO_CONTROL, ~0, mdioctrl);
+	si_corereg(bus->sih, bus->sih->buscoreidx, PCIE2_MDIO_WR_DATA, ~0, mdiodata);
+
+	OSL_DELAY(10);
+	/* retry till the transaction is complete */
+	while (i < pcie_serdes_spinwait) {
+		uint mdioctrl_read = si_corereg(bus->sih, bus->sih->buscoreidx, PCIE2_MDIO_WR_DATA,
+			0, 0);
+		if (!(mdioctrl_read & MDIODATA2_DONE)) {
+			break;
+		}
+		OSL_DELAY(1000);
+		i++;
+	}
+
+	if (i >= pcie_serdes_spinwait) {
+		DHD_ERROR(("pcie_mdiosetblock: timed out\n"));
+		return FALSE;
+	}
+
+	return TRUE;
+}
+#endif /* DHD_PCIE_REG_ACCESS */
+
+static void
+dhdpcie_enum_reg_init(dhd_bus_t *bus)
+{
+	/* initialize Function control register (clear bit 4) to HW init value */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.control), ~0,
+		PCIE_CPLCA_ENABLE | PCIE_DLY_PERST_TO_COE);
+
+	/* clear IntMask */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.intmask), ~0, 0);
+	/* clear IntStatus */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.intstatus), ~0,
+		si_corereg(bus->sih, bus->sih->buscoreidx,
+			OFFSETOF(sbpcieregs_t, ftn_ctrl.intstatus), 0, 0));
+
+	/* clear MSIVector */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.msi_vector), ~0, 0);
+	/* clear MSIIntMask */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.msi_intmask), ~0, 0);
+	/* clear MSIIntStatus */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.msi_intstatus), ~0,
+		si_corereg(bus->sih, bus->sih->buscoreidx,
+			OFFSETOF(sbpcieregs_t, ftn_ctrl.msi_intstatus), 0, 0));
+
+	/* clear PowerIntMask */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.pwr_intmask), ~0, 0);
+	/* clear PowerIntStatus */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.pwr_intstatus), ~0,
+		si_corereg(bus->sih, bus->sih->buscoreidx,
+			OFFSETOF(sbpcieregs_t, ftn_ctrl.pwr_intstatus), 0, 0));
+
+	/* clear MailboxIntMask */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.mbox_intmask), ~0, 0);
+	/* clear MailboxInt */
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, ftn_ctrl.mbox_intstatus), ~0,
+		si_corereg(bus->sih, bus->sih->buscoreidx,
+			OFFSETOF(sbpcieregs_t, ftn_ctrl.mbox_intstatus), 0, 0));
+}
+
+int
+dhd_bus_perform_flr(dhd_bus_t *bus, bool force_fail)
+{
+	uint flr_capab;
+	uint val;
+	int retry = 0;
+	bool in_flr_already = FALSE;
+
+	DHD_ERROR(("******** Perform FLR ********\n"));
+
+	/* Kernel Panic for 4378Ax during traptest/devreset4 reload case:
+	 * For 4378Ax, enum registers will not be reset with FLR (producer index WAR).
+	 * So, the MailboxIntMask is left as 0xffff during fw boot-up,
+	 * and the fw trap handling during fw boot causes Kernel Panic.
+	 * Jira: SWWLAN-212578: [4378A0 PCIe DVT] :
+	 *       Kernel Panic seen in F0 FLR with BT Idle/Traffic/DMA
+	 */
+	if (bus->sih && PCIE_ENUM_RESET_WAR_ENAB(bus->sih->buscorerev)) {
+		if (bus->pcie_mailbox_mask != 0) {
+			dhdpcie_bus_intr_disable(bus);
+		}
+		/* initialize F0 enum registers before FLR for rev66/67 */
+		dhdpcie_enum_reg_init(bus);
+	}
+
+	/* Read PCIE_CFG_DEVICE_CAPABILITY bit 28 to check FLR capability */
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_DEVICE_CAPABILITY, sizeof(val));
+	flr_capab =  val & (1 << PCIE_FLR_CAPAB_BIT);
+	DHD_INFO(("Read Device Capability: reg=0x%x read val=0x%x flr_capab=0x%x\n",
+		PCIE_CFG_DEVICE_CAPABILITY, val, flr_capab));
+	if (!flr_capab) {
+	       DHD_ERROR(("Chip does not support FLR\n"));
+	       return BCME_UNSUPPORTED;
+	}
+	/* Check if we are already in FLR, if we are then just go to restore part of it  */
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val));
+	DHD_ERROR(("config space 0x88 is 0x%x\n", val));
+	if (val & (1 << PCIE_SSRESET_STATUS_BIT)) {
+		DHD_ERROR(("******** device already in FLR, so clear only ********\n"));
+		in_flr_already = TRUE;
+	}
+	else {
+		DHD_ERROR(("******** device not in FLR ********\n"));
+	}
+	if (!in_flr_already) {
+
+		bus->ptm_ctrl_reg = dhdpcie_bus_cfg_read_dword(bus, PCI_CFG_PTM_CTRL, 4);
+
+		/* Save pcie config space */
+		DHD_INFO(("Save Pcie Config Space\n"));
+		DHD_PCIE_CONFIG_SAVE(bus);
+
+		/* Set bit 15 of PCIE_CFG_DEVICE_CONTROL */
+		DHD_INFO(("Set PCIE_FUNCTION_LEVEL_RESET_BIT(%d) of"
+			" PCIE_CFG_DEVICE_CONTROL(0x%x)\n",
+			PCIE_FUNCTION_LEVEL_RESET_BIT, PCIE_CFG_DEVICE_CONTROL));
+		val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_DEVICE_CONTROL, sizeof(val));
+		DHD_INFO(("read_config: reg=0x%x read val=0x%x\n", PCIE_CFG_DEVICE_CONTROL, val));
+		val = val | (1 << PCIE_FUNCTION_LEVEL_RESET_BIT);
+		DHD_INFO(("write_config: reg=0x%x write val=0x%x\n", PCIE_CFG_DEVICE_CONTROL, val));
+		OSL_PCI_WRITE_CONFIG(bus->osh, PCIE_CFG_DEVICE_CONTROL, sizeof(val), val);
+		if (bus->lpm_keep_in_reset) {
+			/* Keep the device in FLR */
+			DHD_ERROR(("******** device force FLR only ********\n"));
+			return BCME_OK;
+		}
+		else {
+			DHD_ERROR(("******** device force FLR only not set ********\n"));
+		}
+
+		/* wait for DHD_FUNCTION_LEVEL_RESET_DELAY msec */
+#ifdef BCMSLTGT
+		DHD_ERROR_MEM(("Delay of %d msec\n", DHD_FUNCTION_LEVEL_RESET_DELAY * htclkratio));
+#else
+		DHD_ERROR_MEM(("Delay of %d msec\n", DHD_FUNCTION_LEVEL_RESET_DELAY));
+#endif /* BCMSLTGT */
+
+		CAN_SLEEP() ? OSL_SLEEP(DHD_FUNCTION_LEVEL_RESET_DELAY) :
+			OSL_DELAY(DHD_FUNCTION_LEVEL_RESET_DELAY * USEC_PER_MSEC);
+#ifdef USE_ISB_IN_FLR
+		OSL_ISB();
+#endif /* USE_ISB_IN_FLR */
+
+		if (force_fail) {
+			DHD_ERROR(("Set PCIE_SSRESET_DISABLE_BIT(%d) of"
+				" PCIE_CFG_SUBSYSTEM_CONTROL(0x%x)\n",
+				PCIE_SSRESET_DISABLE_BIT, PCIE_CFG_SUBSYSTEM_CONTROL));
+			val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL,
+				sizeof(val));
+			DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n",
+				PCIE_CFG_SUBSYSTEM_CONTROL, val));
+			val = val | (1 << PCIE_SSRESET_DISABLE_BIT);
+			DHD_ERROR(("write_config: reg=0x%x write val=0x%x\n",
+				PCIE_CFG_SUBSYSTEM_CONTROL, val));
+			OSL_PCI_WRITE_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL,
+				sizeof(val), val);
+
+			val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL,
+				sizeof(val));
+			DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n",
+				PCIE_CFG_SUBSYSTEM_CONTROL, val));
+		}
+	}
+
+	/* Clear bit 15 of PCIE_CFG_DEVICE_CONTROL */
+	DHD_INFO(("Clear PCIE_FUNCTION_LEVEL_RESET_BIT(%d) of PCIE_CFG_DEVICE_CONTROL(0x%x)\n",
+		PCIE_FUNCTION_LEVEL_RESET_BIT, PCIE_CFG_DEVICE_CONTROL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_DEVICE_CONTROL, sizeof(val));
+	DHD_INFO(("read_config: reg=0x%x read val=0x%x\n", PCIE_CFG_DEVICE_CONTROL, val));
+	val = val & ~(1 << PCIE_FUNCTION_LEVEL_RESET_BIT);
+	DHD_INFO(("write_config: reg=0x%x write val=0x%x\n", PCIE_CFG_DEVICE_CONTROL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIE_CFG_DEVICE_CONTROL, sizeof(val), val);
+
+	/* Wait till bit 13 of PCIE_CFG_SUBSYSTEM_CONTROL is cleared */
+	DHD_INFO(("Wait till PCIE_SSRESET_STATUS_BIT(%d) of PCIE_CFG_SUBSYSTEM_CONTROL(0x%x)"
+		"is cleared\n",	PCIE_SSRESET_STATUS_BIT, PCIE_CFG_SUBSYSTEM_CONTROL));
+	do {
+		val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val));
+		DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n",
+			PCIE_CFG_SUBSYSTEM_CONTROL, val));
+		val = val & (1 << PCIE_SSRESET_STATUS_BIT);
+		OSL_DELAY(DHD_SSRESET_STATUS_RETRY_DELAY);
+	} while (val && (retry++ < DHD_SSRESET_STATUS_RETRIES));
+
+	if (val) {
+		DHD_ERROR_MEM(("ERROR: reg=0x%x bit %d is not cleared\n",
+			PCIE_CFG_SUBSYSTEM_CONTROL, PCIE_SSRESET_STATUS_BIT));
+		/* User has to fire the IOVAR again, if force_fail is needed */
+		if (force_fail) {
+			bus->flr_force_fail = FALSE;
+			DHD_ERROR(("%s cleared flr_force_fail flag\n", __FUNCTION__));
+		}
+		return BCME_DONGLE_DOWN;
+	}
+
+	dhdpcie_bus_cfg_write_dword(bus, PCI_CFG_PTM_CTRL, 4, bus->ptm_ctrl_reg);
+
+	/* Restore pcie config space */
+	DHD_INFO(("Restore Pcie Config Space\n"));
+	DHD_PCIE_CONFIG_RESTORE(bus);
+
+	DHD_ERROR(("******** FLR Succedeed ********\n"));
+
+	return BCME_OK;
+}
+
+#define DHD_BP_RESET_ASPM_DISABLE_DELAY		500u	/* usec */
+
+#define DHD_BP_RESET_STATUS_RETRY_DELAY		40u	/* usec */
+#define DHD_BP_RESET_STATUS_RETRIES		50u
+
+#define PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT		10
+#define PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT	12
+
+int
+dhd_bus_cfg_ss_ctrl_bp_reset(struct dhd_bus *bus)
+{
+	uint val;
+	int retry = 0;
+	int ret = BCME_OK;
+	bool reset_stat_bit;
+
+	DHD_ERROR(("******** Perform BP reset ********\n"));
+
+	/* Disable ASPM */
+	DHD_ERROR(("Disable ASPM: Clear bits(1-0) of PCIECFGREG_LINK_STATUS_CTRL(0x%x)\n",
+		PCIECFGREG_LINK_STATUS_CTRL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val));
+	DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	val = val & (~PCIE_ASPM_ENAB);
+	DHD_ERROR(("write_config: reg=0x%x write val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val), val);
+
+	/* wait for delay usec */
+	DHD_ERROR(("Delay of %d usec\n", DHD_BP_RESET_ASPM_DISABLE_DELAY));
+	OSL_DELAY(DHD_BP_RESET_ASPM_DISABLE_DELAY);
+
+	/* Set bp reset bit 10 of PCIE_CFG_SUBSYSTEM_CONTROL */
+	DHD_ERROR(("Set PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT(%d)"
+		" of PCIE_CFG_SUBSYSTEM_CONTROL(0x%x)\n",
+		PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT, PCIE_CFG_SUBSYSTEM_CONTROL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val));
+	DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n", PCIE_CFG_SUBSYSTEM_CONTROL, val));
+	val = val | (1 << PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT);
+	DHD_ERROR(("write_config: reg=0x%x write val=0x%x\n", PCIE_CFG_SUBSYSTEM_CONTROL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val), val);
+
+	/* Wait till bp reset status bit 12 of PCIE_CFG_SUBSYSTEM_CONTROL is set */
+	DHD_ERROR(("Wait till PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT(%d) of "
+		"PCIE_CFG_SUBSYSTEM_CONTROL(0x%x) is set\n",
+		PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT, PCIE_CFG_SUBSYSTEM_CONTROL));
+	do {
+		val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val));
+		DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n",
+			PCIE_CFG_SUBSYSTEM_CONTROL, val));
+		reset_stat_bit = val & (1 << PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT);
+		OSL_DELAY(DHD_BP_RESET_STATUS_RETRY_DELAY);
+	} while (!reset_stat_bit && (retry++ < DHD_BP_RESET_STATUS_RETRIES));
+
+	if (!reset_stat_bit) {
+		DHD_ERROR(("ERROR: reg=0x%x bit %d is not set\n",
+			PCIE_CFG_SUBSYSTEM_CONTROL,
+			PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT));
+		ret = BCME_ERROR;
+		goto aspm_enab;
+	}
+
+	/* Clear bp reset bit 10 of PCIE_CFG_SUBSYSTEM_CONTROL */
+	DHD_ERROR(("Clear PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT(%d)"
+		" of PCIECFGREG_SPROM_CTRL(0x%x)\n",
+		PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT, PCIE_CFG_SUBSYSTEM_CONTROL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val));
+	DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n", PCIE_CFG_SUBSYSTEM_CONTROL, val));
+	val = val & ~(1 << PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_BIT);
+	DHD_ERROR(("write_config: reg=0x%x write val=0x%x\n", PCIE_CFG_SUBSYSTEM_CONTROL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val), val);
+
+	/* Wait till bp reset status bit 12 of PCIE_CFG_SUBSYSTEM_CONTROL is cleared */
+	DHD_ERROR(("Wait till PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT(%d) of "
+		"PCIE_CFG_SUBSYSTEM_CONTROL(0x%x) is cleared\n",
+		PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT, PCIE_CFG_SUBSYSTEM_CONTROL));
+	do {
+		val = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_SUBSYSTEM_CONTROL, sizeof(val));
+		DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n",
+			PCIE_CFG_SUBSYSTEM_CONTROL, val));
+		reset_stat_bit = val & (1 << PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT);
+		OSL_DELAY(DHD_BP_RESET_STATUS_RETRY_DELAY);
+	} while (reset_stat_bit && (retry++ < DHD_BP_RESET_STATUS_RETRIES));
+
+	if (reset_stat_bit) {
+		DHD_ERROR(("ERROR: reg=0x%x bit %d is not cleared\n",
+			PCIE_CFG_SUBSYSTEM_CONTROL,
+			PCIE_CFG_SUBSYSTEM_CONTROL_BP_RESET_STATUS_BIT));
+		ret = BCME_ERROR;
+	}
+
+aspm_enab:
+	/* Enable ASPM */
+	DHD_ERROR(("Enable ASPM: set bit 1 of PCIECFGREG_LINK_STATUS_CTRL(0x%x)\n",
+		PCIECFGREG_LINK_STATUS_CTRL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val));
+	DHD_ERROR(("read_config: reg=0x%x read val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	val = val | (PCIE_ASPM_L1_ENAB);
+	DHD_ERROR(("write_config: reg=0x%x write val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val), val);
+
+	if (ret) {
+		DHD_ERROR(("******** BP reset Failed ********\n"));
+	} else {
+		DHD_ERROR(("******** BP reset Succedeed ********\n"));
+	}
+
+	return ret;
+}
+
+#define PCIE_CFG_SPROM_CTRL_SB_RESET_BIT	10
+#define PCIE_CFG_CLOCK_CTRL_STATUS_BP_RESET_BIT	21
+
+int
+dhd_bus_cfg_sprom_ctrl_bp_reset(struct dhd_bus *bus)
+{
+	uint val;
+	int retry = 0;
+	uint dar_clk_ctrl_status_reg = DAR_CLK_CTRL(bus->sih->buscorerev);
+	int ret = BCME_OK;
+	bool cond;
+
+	DHD_ERROR(("******** Perform BP reset ********\n"));
+
+	/* Disable ASPM */
+	DHD_INFO(("Disable ASPM: Clear bits(1-0) of PCIECFGREG_LINK_STATUS_CTRL(0x%x)\n",
+		PCIECFGREG_LINK_STATUS_CTRL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val));
+	DHD_INFO(("read_config: reg=0x%x read val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	val = val & (~PCIE_ASPM_ENAB);
+	DHD_INFO(("write_config: reg=0x%x write val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val), val);
+
+	/* wait for delay usec */
+	DHD_INFO(("Delay of %d usec\n", DHD_BP_RESET_ASPM_DISABLE_DELAY));
+	OSL_DELAY(DHD_BP_RESET_ASPM_DISABLE_DELAY);
+
+	/* Set bit 10 of PCIECFGREG_SPROM_CTRL */
+	DHD_INFO(("Set PCIE_CFG_SPROM_CTRL_SB_RESET_BIT(%d) of PCIECFGREG_SPROM_CTRL(0x%x)\n",
+		PCIE_CFG_SPROM_CTRL_SB_RESET_BIT, PCIECFGREG_SPROM_CTRL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIECFGREG_SPROM_CTRL, sizeof(val));
+	DHD_INFO(("read_config: reg=0x%x read val=0x%x\n", PCIECFGREG_SPROM_CTRL, val));
+	val = val | (1 << PCIE_CFG_SPROM_CTRL_SB_RESET_BIT);
+	DHD_INFO(("write_config: reg=0x%x write val=0x%x\n", PCIECFGREG_SPROM_CTRL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIECFGREG_SPROM_CTRL, sizeof(val), val);
+
+	/* Wait till bit backplane reset is ASSERTED i,e
+	 * bit 10 of PCIECFGREG_SPROM_CTRL is cleared.
+	 * Only after this, poll for 21st bit of DAR reg 0xAE0 is valid
+	 * else DAR register will read previous old value
+	 */
+	DHD_INFO(("Wait till PCIE_CFG_SPROM_CTRL_SB_RESET_BIT(%d) of "
+		"PCIECFGREG_SPROM_CTRL(0x%x) is cleared\n",
+		PCIE_CFG_SPROM_CTRL_SB_RESET_BIT, PCIECFGREG_SPROM_CTRL));
+	do {
+		val = OSL_PCI_READ_CONFIG(bus->osh, PCIECFGREG_SPROM_CTRL, sizeof(val));
+		DHD_INFO(("read_config: reg=0x%x read val=0x%x\n", PCIECFGREG_SPROM_CTRL, val));
+		cond = val & (1 << PCIE_CFG_SPROM_CTRL_SB_RESET_BIT);
+		OSL_DELAY(DHD_BP_RESET_STATUS_RETRY_DELAY);
+	} while (cond && (retry++ < DHD_BP_RESET_STATUS_RETRIES));
+
+	if (cond) {
+		DHD_ERROR(("ERROR: reg=0x%x bit %d is not cleared\n",
+			PCIECFGREG_SPROM_CTRL, PCIE_CFG_SPROM_CTRL_SB_RESET_BIT));
+		ret = BCME_ERROR;
+		goto aspm_enab;
+	}
+
+	/* Wait till bit 21 of dar_clk_ctrl_status_reg is cleared */
+	DHD_INFO(("Wait till PCIE_CFG_CLOCK_CTRL_STATUS_BP_RESET_BIT(%d) of "
+		"dar_clk_ctrl_status_reg(0x%x) is cleared\n",
+		PCIE_CFG_CLOCK_CTRL_STATUS_BP_RESET_BIT, dar_clk_ctrl_status_reg));
+	do {
+		val = si_corereg(bus->sih, bus->sih->buscoreidx,
+			dar_clk_ctrl_status_reg, 0, 0);
+		DHD_INFO(("read_dar si_corereg: reg=0x%x read val=0x%x\n",
+			dar_clk_ctrl_status_reg, val));
+		cond = val & (1 << PCIE_CFG_CLOCK_CTRL_STATUS_BP_RESET_BIT);
+		OSL_DELAY(DHD_BP_RESET_STATUS_RETRY_DELAY);
+	} while (cond && (retry++ < DHD_BP_RESET_STATUS_RETRIES));
+
+	if (cond) {
+		DHD_ERROR(("ERROR: reg=0x%x bit %d is not cleared\n",
+			dar_clk_ctrl_status_reg, PCIE_CFG_CLOCK_CTRL_STATUS_BP_RESET_BIT));
+		ret = BCME_ERROR;
+	}
+
+aspm_enab:
+	/* Enable ASPM */
+	DHD_INFO(("Enable ASPM: set bit 1 of PCIECFGREG_LINK_STATUS_CTRL(0x%x)\n",
+		PCIECFGREG_LINK_STATUS_CTRL));
+	val = OSL_PCI_READ_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val));
+	DHD_INFO(("read_config: reg=0x%x read val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	val = val | (PCIE_ASPM_L1_ENAB);
+	DHD_INFO(("write_config: reg=0x%x write val=0x%x\n", PCIECFGREG_LINK_STATUS_CTRL, val));
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIECFGREG_LINK_STATUS_CTRL, sizeof(val), val);
+
+	DHD_ERROR(("******** BP reset Succedeed ********\n"));
+
+	return ret;
+}
+
+#if defined(LINUX) || defined(linux)
+int
+dhd_bus_devreset(dhd_pub_t *dhdp, uint8 flag)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	int bcmerror = 0;
+	unsigned long flags;
+	int retry = POWERUP_MAX_RETRY;
+
+	if (flag == TRUE) { /* Turn off WLAN */
+		/* Removing Power */
+		DHD_ERROR(("%s: == Power OFF ==\n", __FUNCTION__));
+
+		/* wait for other contexts to finish -- if required a call
+		* to OSL_DELAY for 1s can be added to give other contexts
+		* a chance to finish
+		*/
+		dhdpcie_advertise_bus_cleanup(bus->dhd);
+
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+			atomic_set(&bus->dhd->block_bus, TRUE);
+			dhd_flush_rx_tx_wq(bus->dhd);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+			/* Clean up any pending host wake IRQ */
+			dhd_bus_oob_intr_set(bus->dhd, FALSE);
+			dhd_bus_oob_intr_unregister(bus->dhd);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+			dhd_os_wd_timer(dhdp, 0);
+			dhd_bus_stop(bus, TRUE);
+			if (bus->intr) {
+				dhdpcie_bus_intr_disable(bus);
+				dhdpcie_free_irq(bus);
+			}
+			dhd_deinit_bus_lp_state_lock(bus);
+			dhd_deinit_bar1_switch_lock(bus);
+			dhd_deinit_backplane_access_lock(bus);
+			dhd_deinit_pwr_req_lock(bus);
+#ifdef PCIE_INB_DW
+			dhd_deinit_dongle_ds_lock(bus);
+#endif /* PCIE_INB_DW */
+			dhd_bus_release_dongle(bus);
+			dhdpcie_bus_free_resource(bus);
+			bcmerror = dhdpcie_bus_disable_device(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_disable_device: %d\n",
+					__FUNCTION__, bcmerror));
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+				atomic_set(&bus->dhd->block_bus, FALSE);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+			}
+			/* Clean up protocol data after Bus Master Enable bit clear
+			 * so that host can safely unmap DMA and remove the allocated buffers
+			 * from the PKTID MAP. Some Applicantion Processors supported
+			 * System MMU triggers Kernel panic when they detect to attempt to
+			 * DMA-unmapped memory access from the devices which use the
+			 * System MMU. Therefore, Kernel panic can be happened since it is
+			 * possible that dongle can access to DMA-unmapped memory after
+			 * calling the dhd_prot_reset().
+			 * For this reason, the dhd_prot_reset() and dhd_clear() functions
+			 * should be located after the dhdpcie_bus_disable_device().
+			 */
+			dhd_prot_reset(dhdp);
+			/* XXX Reset dhd_pub_t instance to initial status
+			 * for built-in type driver
+			 */
+			dhd_clear(dhdp);
+
+			bcmerror = dhdpcie_bus_stop_host_dev(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_stop host_dev failed: %d\n",
+					__FUNCTION__, bcmerror));
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+				atomic_set(&bus->dhd->block_bus, FALSE);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+				goto done;
+			}
+
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+			atomic_set(&bus->dhd->block_bus, FALSE);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+		} else {
+			if (bus->intr) {
+				dhdpcie_free_irq(bus);
+			}
+#ifdef BCMPCIE_OOB_HOST_WAKE
+			/* Clean up any pending host wake IRQ */
+			dhd_bus_oob_intr_set(bus->dhd, FALSE);
+			dhd_bus_oob_intr_unregister(bus->dhd);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+			dhd_dpc_kill(bus->dhd);
+			if (!bus->no_bus_init) {
+				dhd_bus_release_dongle(bus);
+				dhdpcie_bus_free_resource(bus);
+				bcmerror = dhdpcie_bus_disable_device(bus);
+				if (bcmerror) {
+					DHD_ERROR(("%s: dhdpcie_bus_disable_device: %d\n",
+						__FUNCTION__, bcmerror));
+				}
+
+				/* Clean up protocol data after Bus Master Enable bit clear
+				 * so that host can safely unmap DMA and remove the allocated
+				 * buffers from the PKTID MAP. Some Applicantion Processors
+				 * supported System MMU triggers Kernel panic when they detect
+				 * to attempt to DMA-unmapped memory access from the devices
+				 * which use the System MMU.
+				 * Therefore, Kernel panic can be happened since it is possible
+				 * that dongle can access to DMA-unmapped memory after calling
+				 * the dhd_prot_reset().
+				 * For this reason, the dhd_prot_reset() and dhd_clear() functions
+				 * should be located after the dhdpcie_bus_disable_device().
+				 */
+				dhd_prot_reset(dhdp);
+				/* XXX Reset dhd_pub_t instance to initial status
+				 * for built-in type driver
+				 */
+				dhd_clear(dhdp);
+			} else {
+				bus->no_bus_init = FALSE;
+			}
+
+			bcmerror = dhdpcie_bus_stop_host_dev(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_stop_host_dev failed: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+		}
+
+		bus->dhd->dongle_reset = TRUE;
+		DHD_ERROR(("%s:  WLAN OFF Done\n", __FUNCTION__));
+
+	} else { /* Turn on WLAN */
+		if (bus->dhd->busstate == DHD_BUS_DOWN) {
+			/* Powering On */
+			DHD_ERROR(("%s: == Power ON ==\n", __FUNCTION__));
+			/* PCIe RC Turn on */
+			do {
+				bcmerror = dhdpcie_bus_start_host_dev(bus);
+				if (!bcmerror) {
+					DHD_ERROR_MEM(("%s: dhdpcie_bus_start_host_dev OK\n",
+						__FUNCTION__));
+					break;
+				} else {
+					OSL_SLEEP(10);
+				}
+			} while (retry--);
+
+			if (bcmerror) {
+				DHD_ERROR(("%s: host pcie clock enable failed: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+#if defined(DHD_CONTROL_PCIE_ASPM_WIFI_TURNON)
+			dhd_bus_aspm_enable_rc_ep(bus, FALSE);
+#endif /* DHD_CONTROL_PCIE_ASPM_WIFI_TURNON */
+			bus->is_linkdown = 0;
+			bus->cto_triggered = 0;
+			bcmerror = dhdpcie_bus_enable_device(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: host configuration restore failed: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+
+			bcmerror = dhdpcie_bus_alloc_resource(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_resource_alloc failed: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+
+#ifdef FORCE_DONGLE_RESET_IN_DEVRESET_ON
+			/*
+			 * This will be enabled from phone platforms to
+			 * reset dongle during Wifi ON
+			 */
+			dhdpcie_dongle_reset(bus);
+#endif /* FORCE_DONGLE_RESET_IN_DEVRESET_ON */
+
+			bcmerror = dhdpcie_bus_dongle_attach(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhdpcie_bus_dongle_attach failed: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+
+			bcmerror = dhd_bus_request_irq(bus);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhd_bus_request_irq failed: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+
+			bus->dhd->dongle_reset = FALSE;
+
+#if defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON)
+			dhd_irq_set_affinity(bus->dhd, cpumask_of(1));
+#endif /* DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON */
+
+			bcmerror = dhd_bus_start(dhdp);
+			if (bcmerror) {
+				DHD_ERROR(("%s: dhd_bus_start: %d\n",
+					__FUNCTION__, bcmerror));
+				goto done;
+			}
+
+			/* Renabling watchdog which is disabled in dhdpcie_advertise_bus_cleanup */
+			if (bus->dhd->dhd_watchdog_ms_backup) {
+				DHD_ERROR(("%s: Enabling wdtick after dhd init\n",
+					__FUNCTION__));
+				dhd_os_wd_timer(bus->dhd, bus->dhd->dhd_watchdog_ms_backup);
+			}
+			DHD_ERROR(("%s: WLAN Power On Done\n", __FUNCTION__));
+		} else {
+			DHD_ERROR(("%s: what should we do here\n", __FUNCTION__));
+			goto done;
+		}
+	}
+
+done:
+	return bcmerror;
+}
+#else
+int
+dhd_bus_devreset(dhd_pub_t *dhdp, uint8 flag)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	int bcmerror = 0;
+	unsigned long flags;
+
+	if (flag == TRUE) {
+		/* Removing Power */
+		if (!dhdp->dongle_reset) {
+			DHD_ERROR(("%s: == Power OFF ==\n", __FUNCTION__));
+			dhdpcie_advertise_bus_cleanup(bus->dhd);
+			dhd_os_sdlock(dhdp);
+			dhd_os_wd_timer(dhdp, 0);
+			dhd_bus_stop(bus, FALSE);
+			dhd_prot_reset(dhdp);
+
+			dhdpcie_bus_release_dongle(bus, bus->dhd->osh,
+				bus->dhd->dongle_isolation, TRUE);
+			bus->dhd->dongle_reset = TRUE;
+
+			dhd_os_sdunlock(dhdp);
+
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+			DHD_ERROR(("%s:  WLAN OFF Done\n", __FUNCTION__));
+		} else {
+			DHD_ERROR(("%s: Dongle is already in RESET!\n", __FUNCTION__));
+			bcmerror = BCME_DONGLE_DOWN;
+		}
+	} else {
+		/* Powering On */
+		DHD_ERROR(("%s: == Power ON ==\n", __FUNCTION__));
+
+		if (bus->dhd->dongle_reset) {
+			dhd_os_sdlock(dhdp); /* Turn on WLAN */
+
+			if (dhdpcie_dongle_attach(bus)) {
+				DHD_ERROR(("%s: dhdpcie_probe_attach failed\n", __FUNCTION__));
+				dhd_os_sdunlock(dhdp);
+				return BCME_DONGLE_DOWN;
+			}
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+			DHD_INFO(("%s: About to download firmware\n", __FUNCTION__));
+			if (dhd_bus_download_firmware(bus, bus->dhd->osh,
+				bus->fw_path, bus->nv_path) == 0) {
+
+				bcmerror = dhd_bus_init((dhd_pub_t *) bus->dhd, FALSE);
+				if (bcmerror == BCME_OK) {
+					bus->dhd->dongle_reset = FALSE;
+
+					dhd_os_wd_timer(dhdp, dhd_watchdog_ms);
+					DHD_ERROR(("%s: WLAN Power On Done\n", __FUNCTION__));
+				} else {
+					DHD_ERROR(("%s: dhd_bus_init FAILed\n", __FUNCTION__));
+					dhd_bus_stop(bus, FALSE);
+				}
+			} else {
+				DHD_ERROR(("%s: dhd_bus_download_firmware FAILed\n", __FUNCTION__));
+				bcmerror = BCME_DONGLE_DOWN;
+			}
+
+			dhd_os_sdunlock(dhdp);
+		} else {
+			bcmerror = BCME_DONGLE_DOWN;
+			DHD_ERROR(("%s called when dongle is not in reset\n", __FUNCTION__));
+		}
+	}
+	return bcmerror;
+}
+#endif /* LINUX || linux */
+
+#ifdef DHD_PCIE_REG_ACCESS
+static int
+pcie2_mdioop(dhd_bus_t *bus, uint physmedia, uint regaddr, bool write, uint *val,
+	bool slave_bypass)
+{
+	uint pcie_serdes_spinwait = 200, i = 0, mdio_ctrl;
+	uint32 reg32;
+
+	pcie2_mdiosetblock(bus, physmedia);
+
+	/* enable mdio access to SERDES */
+	mdio_ctrl = MDIOCTL2_DIVISOR_VAL;
+	mdio_ctrl |= (regaddr << MDIOCTL2_REGADDR_SHF);
+
+	if (slave_bypass)
+		mdio_ctrl |= MDIOCTL2_SLAVE_BYPASS;
+
+	if (!write)
+		mdio_ctrl |= MDIOCTL2_READ;
+
+	si_corereg(bus->sih, bus->sih->buscoreidx, PCIE2_MDIO_CONTROL, ~0, mdio_ctrl);
+
+	if (write) {
+		reg32 =  PCIE2_MDIO_WR_DATA;
+		si_corereg(bus->sih, bus->sih->buscoreidx, PCIE2_MDIO_WR_DATA, ~0,
+			*val | MDIODATA2_DONE);
+	} else
+		reg32 =  PCIE2_MDIO_RD_DATA;
+
+	/* retry till the transaction is complete */
+	while (i < pcie_serdes_spinwait) {
+		uint done_val =  si_corereg(bus->sih, bus->sih->buscoreidx, reg32, 0, 0);
+		if (!(done_val & MDIODATA2_DONE)) {
+			if (!write) {
+				*val = si_corereg(bus->sih, bus->sih->buscoreidx,
+					PCIE2_MDIO_RD_DATA, 0, 0);
+				*val = *val & MDIODATA2_MASK;
+			}
+			return 0;
+		}
+		OSL_DELAY(1000);
+		i++;
+	}
+	return -1;
+}
+#endif /* DHD_PCIE_REG_ACCESS */
+
+static int
+dhdpcie_get_dma_ring_indices(dhd_pub_t *dhd)
+{
+	int h2d_support, d2h_support;
+
+	d2h_support = dhd->dma_d2h_ring_upd_support ? 1 : 0;
+	h2d_support = dhd->dma_h2d_ring_upd_support ? 1 : 0;
+	return (d2h_support | (h2d_support << 1));
+
+}
+int
+dhdpcie_set_dma_ring_indices(dhd_pub_t *dhd, int32 int_val)
+{
+	int bcmerror = 0;
+	/* Can change it only during initialization/FW download */
+	if (dhd->busstate == DHD_BUS_DOWN) {
+		if ((int_val > 3) || (int_val < 0)) {
+			DHD_ERROR(("Bad argument. Possible values: 0, 1, 2 & 3\n"));
+			bcmerror = BCME_BADARG;
+		} else {
+			dhd->dma_d2h_ring_upd_support = (int_val & 1) ? TRUE : FALSE;
+			dhd->dma_h2d_ring_upd_support = (int_val & 2) ? TRUE : FALSE;
+			dhd->dma_ring_upd_overwrite = TRUE;
+		}
+	} else {
+		DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+			__FUNCTION__));
+		bcmerror = BCME_NOTDOWN;
+	}
+
+	return bcmerror;
+
+}
+
+/* si_backplane_access() manages a shared resource - BAR0 mapping, hence its
+ * calls shall be serialized. This wrapper function provides such serialization
+ * and shall be used everywjer einstead of direct call of si_backplane_access()
+ *
+ * Linux DHD driver calls si_backplane_access() from 3 three contexts: tasklet
+ * (that may call dhdpcie_sssr_dump() from dhdpcie_sssr_dump()), iovar
+ * ("sbreg", "membyres", etc.) and procfs (used by GDB proxy). To avoid race
+ * conditions calls of si_backplane_access() shall be serialized. Presence of
+ * tasklet context implies that serialization shall b ebased on spinlock. Hence
+ * Linux implementation of dhd_pcie_backplane_access_[un]lock() is
+ * spinlock-based.
+ *
+ * Other platforms may add their own implementations of
+ * dhd_pcie_backplane_access_[un]lock() as needed (e.g. if serialization is not
+ * needed implementation might be empty)
+ */
+static uint
+serialized_backplane_access(dhd_bus_t *bus, uint addr, uint size, uint *val, bool read)
+{
+	uint ret;
+	unsigned long flags;
+	DHD_BACKPLANE_ACCESS_LOCK(bus->backplane_access_lock, flags);
+	ret = si_backplane_access(bus->sih, addr, size, val, read);
+	DHD_BACKPLANE_ACCESS_UNLOCK(bus->backplane_access_lock, flags);
+	return ret;
+}
+
+#ifndef DHD_CAP_PLATFORM
+#define DHD_CAP_PLATFORM "x86 "
+#endif
+
+#ifndef DHD_CAP_CUSTOMER
+#define DHD_CAP_CUSTOMER "brcm "
+#endif
+
+void
+BCMRAMFN(dhd_cap_bcmstrbuf)(dhd_pub_t *dhd, struct bcmstrbuf *b)
+{
+	bcm_bprintf(b, DHD_CAP_PLATFORM);
+	bcm_bprintf(b, DHD_CAP_CUSTOMER);
+#ifdef PCIE_FULL_DONGLE
+	bcm_bprintf(b, "pcie ");
+#endif /* PCIE_FULL_DONGLE */
+	/* regaccess and memaccess will be present only for internal reference builds @brcm */
+#ifdef DHD_NO_MOG
+	bcm_bprintf(b, "internal ");
+#else
+	bcm_bprintf(b, "external ");
+#endif /* DHD_NO_MOG */
+#ifdef WLAN_ACCEL_BOOT
+	bcm_bprintf(b, "wlan-accel ");
+#endif /* WLAN_ACCEL_BOOT */
+#ifdef ENABLE_DHD_GRO
+	bcm_bprintf(b, "gro ");
+#endif /* ENABLE_DHD_GRO */
+#ifdef WBRC
+	bcm_bprintf(b, "wbrc ");
+#endif /* ENABLE_DHD_GRO */
+#ifdef WL_CFG80211
+	bcm_bprintf(b, "cfg80211 ");
+#endif /* WL_CFG80211 */
+#ifdef DHD_FILE_DUMP_EVENT
+	bcm_bprintf(b, "haldump ");
+#endif /* DHD_FILE_DUMP_EVENT */
+#ifdef DHD_LB_RXP
+	bcm_bprintf(b, "lbrxp ");
+#endif /* DHD_LB_RXP */
+#ifdef DHD_LB_TXP
+#ifdef DHD_LB_TXP_DEFAULT_ENAB
+	bcm_bprintf(b, "lbtxp ");
+#endif /* DHD_LB_TXP_DEFAULT_ENAB */
+#endif /* DHD_LB_TXP */
+	if (dhd->htput_support) {
+		bcm_bprintf(b, "htput ");
+	}
+}
+
+/** Return dhd capability string */
+static char*
+dhd_cap(dhd_pub_t *dhd, char *buf, uint bufsize)
+{
+	struct bcmstrbuf b;
+
+	bcm_binit(&b, buf, bufsize);
+
+	dhd_cap_bcmstrbuf(dhd, &b);
+
+	/* this is either full or overflow. return error */
+	if (b.size <= 1)
+		return NULL;
+
+	return (buf);
+}
+
+/**
+ * IOVAR handler of the DHD bus layer (in this case, the PCIe bus).
+ *
+ * @param actionid  e.g. IOV_SVAL(IOV_PCIEREG)
+ * @param params    input buffer
+ * @param plen      length in [bytes] of input buffer 'params'
+ * @param arg       output buffer
+ * @param len       length in [bytes] of output buffer 'arg'
+ */
+static int
+dhdpcie_bus_doiovar(dhd_bus_t *bus, const bcm_iovar_t *vi, uint32 actionid, const char *name,
+                void *params, uint plen, void *arg, uint len, int val_size)
+{
+	int bcmerror = 0;
+	int32 int_val = 0;
+	int32 int_val2 = 0;
+	int32 int_val3 = 0;
+	bool bool_val = 0;
+
+	DHD_TRACE(("%s: Enter, action %d name %s params %p plen %d arg %p len %d val_size %d\n",
+	           __FUNCTION__, actionid, name, params, plen, arg, len, val_size));
+
+	if ((bcmerror = bcm_iovar_lencheck(vi, arg, len, IOV_ISSET(actionid))) != 0)
+		goto exit;
+
+	if (plen >= sizeof(int_val))
+		bcopy(params, &int_val, sizeof(int_val));
+
+	if (plen >= sizeof(int_val) * 2)
+		bcopy((void*)((uintptr)params + sizeof(int_val)), &int_val2, sizeof(int_val2));
+
+	if (plen >= sizeof(int_val) * 3)
+		bcopy((void*)((uintptr)params + 2 * sizeof(int_val)), &int_val3, sizeof(int_val3));
+
+	bool_val = (int_val != 0) ? TRUE : FALSE;
+
+	/* Check if dongle is in reset. If so, only allow DEVRESET iovars */
+	if (bus->dhd->dongle_reset && !(actionid == IOV_SVAL(IOV_DEVRESET) ||
+	                                actionid == IOV_GVAL(IOV_DEVRESET))) {
+		bcmerror = BCME_NOTREADY;
+		goto exit;
+	}
+
+	switch (actionid) {
+
+	case IOV_SVAL(IOV_VARS):
+		bcmerror = dhdpcie_downloadvars(bus, arg, len);
+		break;
+#ifdef DHD_PCIE_REG_ACCESS
+	case IOV_SVAL(IOV_PCIEREG):
+		/* XXX: validate int_val ??? */
+		si_corereg(bus->sih, bus->sih->buscoreidx, OFFSETOF(sbpcieregs_t, configaddr), ~0,
+			int_val);
+		si_corereg(bus->sih, bus->sih->buscoreidx, OFFSETOF(sbpcieregs_t, configdata), ~0,
+			int_val2);
+		break;
+
+	case IOV_GVAL(IOV_PCIEREG):
+		/* XXX: validate int_val ??? */
+		si_corereg(bus->sih, bus->sih->buscoreidx, OFFSETOF(sbpcieregs_t, configaddr), ~0,
+			int_val);
+		int_val = si_corereg(bus->sih, bus->sih->buscoreidx,
+			OFFSETOF(sbpcieregs_t, configdata), 0, 0);
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+
+	case IOV_SVAL(IOV_PCIECOREREG):
+		/* XXX: validate int_val ??? */
+		si_corereg(bus->sih, bus->sih->buscoreidx, int_val, ~0, int_val2);
+		break;
+	case IOV_GVAL(IOV_BAR0_SECWIN_REG):
+	{
+		sdreg_t sdreg;
+		uint32 addr, size;
+
+		bcopy(params, &sdreg, sizeof(sdreg));
+
+		addr = sdreg.offset;
+		size = sdreg.func;
+
+		if (serialized_backplane_access(bus, addr, size, (uint *)&int_val, TRUE) != BCME_OK)
+		{
+			DHD_ERROR(("Invalid size/addr combination \n"));
+			bcmerror = BCME_ERROR;
+			break;
+		}
+		bcopy(&int_val, arg, sizeof(int32));
+		break;
+	}
+
+	case IOV_SVAL(IOV_BAR0_SECWIN_REG):
+	{
+		sdreg_t sdreg;
+		uint32 addr, size;
+
+		bcopy(params, &sdreg, sizeof(sdreg));
+
+		addr = sdreg.offset;
+		size = sdreg.func;
+		if (serialized_backplane_access(bus, addr, size,
+			(uint *)(&sdreg.value), FALSE) != BCME_OK) {
+			DHD_ERROR(("Invalid size/addr combination \n"));
+			bcmerror = BCME_ERROR;
+		}
+		break;
+	}
+
+	case IOV_GVAL(IOV_SBREG):
+	{
+		sdreg_t sdreg;
+		uint32 addr, size;
+
+		bcopy(params, &sdreg, sizeof(sdreg));
+
+		addr = sdreg.offset | SI_ENUM_BASE(bus->sih);
+		size = sdreg.func;
+
+		if (serialized_backplane_access(bus, addr, size, (uint *)&int_val, TRUE) != BCME_OK)
+		{
+			DHD_ERROR(("Invalid size/addr combination \n"));
+			bcmerror = BCME_ERROR;
+			break;
+		}
+		bcopy(&int_val, arg, size);
+		break;
+	}
+
+	case IOV_SVAL(IOV_SBREG):
+	{
+		sdreg_t sdreg;
+		uint32 addr, size;
+
+		bcopy(params, &sdreg, sizeof(sdreg));
+
+		addr = sdreg.offset | SI_ENUM_BASE(bus->sih);
+		size = sdreg.func;
+		if (serialized_backplane_access(bus, addr, size,
+			(uint *)(&sdreg.value), FALSE) != BCME_OK) {
+			DHD_ERROR(("Invalid size/addr combination \n"));
+			bcmerror = BCME_ERROR;
+		}
+		break;
+	}
+
+	case IOV_GVAL(IOV_PCIESERDESREG):
+	{
+		uint val;
+		if (!PCIE_GEN2(bus->sih)) {
+			DHD_ERROR(("supported only in pcie gen2\n"));
+			bcmerror = BCME_ERROR;
+			break;
+		}
+
+		if (!pcie2_mdioop(bus, int_val, int_val2, FALSE, &val, FALSE)) {
+			bcopy(&val, arg, sizeof(int32));
+		} else {
+			DHD_ERROR(("pcie2_mdioop failed.\n"));
+			bcmerror = BCME_ERROR;
+		}
+		break;
+	}
+
+	case IOV_SVAL(IOV_PCIESERDESREG):
+		if (!PCIE_GEN2(bus->sih)) {
+			DHD_ERROR(("supported only in pcie gen2\n"));
+			bcmerror = BCME_ERROR;
+			break;
+		}
+		if (pcie2_mdioop(bus, int_val, int_val2, TRUE, (uint *)&int_val3, FALSE)) {
+			DHD_ERROR(("pcie2_mdioop failed.\n"));
+			bcmerror = BCME_ERROR;
+		}
+		break;
+	case IOV_GVAL(IOV_PCIECOREREG):
+		/* XXX: validate int_val ??? */
+		int_val = si_corereg(bus->sih, bus->sih->buscoreidx, int_val, 0, 0);
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+
+	case IOV_SVAL(IOV_PCIECFGREG):
+		/* XXX: validate int_val ??? */
+		OSL_PCI_WRITE_CONFIG(bus->osh, int_val, 4, int_val2);
+		break;
+
+	case IOV_GVAL(IOV_PCIECFGREG):
+		/* XXX: validate int_val ??? */
+		int_val = OSL_PCI_READ_CONFIG(bus->osh, int_val, 4);
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+#endif /* DHD_PCIE_REG_ACCESS */
+	case IOV_SVAL(IOV_PCIE_LPBK):
+		bcmerror = dhdpcie_bus_lpback_req(bus, int_val);
+		break;
+
+	case IOV_SVAL(IOV_PCIE_DMAXFER): {
+		dma_xfer_info_t *dmaxfer = (dma_xfer_info_t *)arg;
+		uint32 mem_addr;
+
+		if (!dmaxfer)
+			return BCME_BADARG;
+		if (dmaxfer->version != DHD_DMAXFER_VERSION)
+			return BCME_VERSION;
+		if (dmaxfer->length != sizeof(dma_xfer_info_t)) {
+			return BCME_BADLEN;
+		}
+
+		mem_addr = (uint32)dmaxfer->tput;
+		dmaxfer->tput = 0;
+		bcmerror = dhdpcie_bus_dmaxfer_req(bus, dmaxfer->num_bytes,
+				dmaxfer->src_delay, dmaxfer->dest_delay,
+				dmaxfer->type, dmaxfer->core_num,
+				dmaxfer->should_wait, mem_addr);
+
+		if (dmaxfer->should_wait && bcmerror >= 0) {
+			bcmerror = dhdmsgbuf_dmaxfer_status(bus->dhd, dmaxfer);
+		}
+		break;
+	}
+
+	case IOV_GVAL(IOV_PCIE_DMAXFER): {
+		dma_xfer_info_t *dmaxfer = (dma_xfer_info_t *)params;
+		if (!dmaxfer)
+			return BCME_BADARG;
+		if (dmaxfer->version != DHD_DMAXFER_VERSION)
+			return BCME_VERSION;
+		if (dmaxfer->length != sizeof(dma_xfer_info_t)) {
+			return BCME_BADLEN;
+		}
+		bcmerror = dhdmsgbuf_dmaxfer_status(bus->dhd, dmaxfer);
+		break;
+	}
+	case IOV_GVAL(IOV_PCIE_DMAXFER_PTRN): {
+		int_val = bus->lpbk_xfer_data_pattern_type;
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+	case IOV_SVAL(IOV_PCIE_DMAXFER_PTRN): {
+		bus->lpbk_xfer_data_pattern_type = int_val;
+		break;
+	}
+
+#ifdef PCIE_OOB
+	case IOV_GVAL(IOV_OOB_BT_REG_ON):
+		int_val = dhd_oob_get_bt_reg_on(bus);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_OOB_BT_REG_ON):
+		dhd_oob_set_bt_reg_on(bus, (uint8)int_val);
+		break;
+	case IOV_GVAL(IOV_OOB_ENABLE):
+		int_val = bus->oob_enabled;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_OOB_ENABLE):
+		bus->oob_enabled = (bool)int_val;
+		break;
+#endif /* PCIE_OOB */
+#ifdef PCIE_INB_DW
+	case IOV_GVAL(IOV_INB_DW_ENABLE):
+		int_val = bus->inb_enabled;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_INB_DW_ENABLE):
+		bus->inb_enabled = (bool)int_val;
+		break;
+#endif /* PCIE_INB_DW */
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	case IOV_GVAL(IOV_DEEP_SLEEP):
+		int_val = bus->ds_enabled;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DEEP_SLEEP):
+		if (int_val == 1) {
+			bus->deep_sleep = TRUE;
+			if (!bus->ds_enabled) {
+				bus->ds_enabled = TRUE;
+				/* Deassert */
+				if (dhd_bus_set_device_wake(bus, FALSE) == BCME_OK) {
+#ifdef PCIE_INB_DW
+					if (INBAND_DW_ENAB(bus)) {
+						int timeleft;
+						timeleft = dhd_os_ds_enter_wait(bus->dhd, NULL);
+						if (timeleft == 0) {
+							DHD_ERROR(("DS-ENTER timeout\n"));
+							bus->ds_enabled = FALSE;
+							break;
+						}
+					}
+#endif /* PCIE_INB_DW */
+				}
+				else {
+					DHD_ERROR(("%s: Enable Deep Sleep failed !\n",
+							__FUNCTION__));
+					bus->ds_enabled = FALSE;
+				}
+			} else {
+				DHD_ERROR(("%s: Deep Sleep already enabled !\n", __FUNCTION__));
+			}
+		}
+		else if (int_val == 0) {
+			bus->deep_sleep = FALSE;
+			if (bus->ds_enabled) {
+				bus->calc_ds_exit_latency = TRUE;
+				/* Assert */
+				if (dhd_bus_set_device_wake(bus, TRUE) == BCME_OK) {
+					bus->ds_enabled = FALSE;
+					if (INBAND_DW_ENAB(bus)) {
+						if (bus->ds_exit_latency != 0) {
+							DHD_ERROR(("DS-EXIT latency = %llu us\n",
+								bus->ds_exit_latency));
+						} else {
+							DHD_ERROR(("Failed to measure DS-EXIT"
+								" latency!(Possibly a non"
+								" waitable context)\n"));
+						}
+					}
+				} else {
+					DHD_ERROR(("%s: Disable Deep Sleep failed !\n",
+						__FUNCTION__));
+				}
+				bus->calc_ds_exit_latency = FALSE;
+			} else {
+				DHD_ERROR(("%s: Deep Sleep already disabled !\n", __FUNCTION__));
+			}
+		}
+		else
+			DHD_ERROR(("%s: Invalid number, allowed only 0|1\n", __FUNCTION__));
+
+		break;
+#endif /* PCIE_OOB || PCIE_INB_DW */
+#ifdef DEVICE_TX_STUCK_DETECT
+	case IOV_GVAL(IOV_DEVICE_TX_STUCK_DETECT):
+		int_val = bus->dev_tx_stuck_monitor;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_DEVICE_TX_STUCK_DETECT):
+		bus->dev_tx_stuck_monitor = (bool)int_val;
+		break;
+#endif /* DEVICE_TX_STUCK_DETECT */
+	case IOV_GVAL(IOV_PCIE_SUSPEND):
+		int_val = (bus->dhd->busstate == DHD_BUS_SUSPEND) ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PCIE_SUSPEND):
+		if (bool_val) { /* Suspend */
+			int ret;
+			unsigned long flags;
+
+			/*
+			 * If some other context is busy, wait until they are done,
+			 * before starting suspend
+			 */
+			ret = dhd_os_busbusy_wait_condition(bus->dhd,
+				&bus->dhd->dhd_bus_busy_state, DHD_BUS_BUSY_IN_DHD_IOVAR);
+			if (ret == 0) {
+				DHD_ERROR(("%s:Wait Timedout, dhd_bus_busy_state = 0x%x\n",
+					__FUNCTION__, bus->dhd->dhd_bus_busy_state));
+				return BCME_BUSY;
+			}
+
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(bus->dhd);
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+			dhdpcie_bus_suspend(bus, TRUE, TRUE);
+#else
+			dhdpcie_bus_suspend(bus, TRUE);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_BUS_BUSY_CLEAR_SUSPEND_IN_PROGRESS(bus->dhd);
+			dhd_os_busbusy_wake(bus->dhd);
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		} else { /* Resume */
+			unsigned long flags;
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_BUS_BUSY_SET_RESUME_IN_PROGRESS(bus->dhd);
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+			dhdpcie_bus_suspend(bus, FALSE);
+
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			DHD_BUS_BUSY_CLEAR_RESUME_IN_PROGRESS(bus->dhd);
+			dhd_os_busbusy_wake(bus->dhd);
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		}
+		break;
+
+	case IOV_GVAL(IOV_MEMSIZE):
+		int_val = (int32)bus->ramsize;
+		bcopy(&int_val, arg, val_size);
+		break;
+#ifdef DHD_BUS_MEM_ACCESS
+	case IOV_SVAL(IOV_MEMBYTES):
+	case IOV_GVAL(IOV_MEMBYTES):
+	{
+		uint32 address;		/* absolute backplane address */
+		uint size, dsize;
+		uint8 *data;
+
+		bool set = (actionid == IOV_SVAL(IOV_MEMBYTES));
+
+		ASSERT(plen >= 2*sizeof(int));
+
+		address = (uint32)int_val;
+		bcopy((char *)params + sizeof(int_val), &int_val, sizeof(int_val));
+		size = (uint)int_val;
+
+		/* Do some validation */
+		dsize = set ? plen - (2 * sizeof(int)) : len;
+		if (dsize < size) {
+			DHD_ERROR(("%s: error on %s membytes, addr 0x%08x size %d dsize %d\n",
+			           __FUNCTION__, (set ? "set" : "get"), address, size, dsize));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		DHD_INFO(("%s: Request to %s %d bytes at address 0x%08x\n dsize %d ", __FUNCTION__,
+		          (set ? "write" : "read"), size, address, dsize));
+
+		/* check if CR4 */
+		if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0) ||
+		    si_setcore(bus->sih, ARMCA7_CORE_ID, 0) ||
+		    si_setcore(bus->sih, SYSMEM_CORE_ID, 0)) {
+			/* if address is 0, store the reset instruction to be written in 0 */
+			if (set && address == bus->dongle_ram_base) {
+				bus->resetinstr = *(((uint32*)params) + 2);
+			}
+		}
+
+		/* Generate the actual data pointer */
+		data = set ? (uint8*)params + 2 * sizeof(int): (uint8*)arg;
+
+		/* Call to do the transfer */
+		bcmerror = dhdpcie_bus_membytes(bus, set, address, data, size);
+
+		break;
+	}
+#endif /* DHD_BUS_MEM_ACCESS */
+
+	/* Debug related. Dumps core registers or one of the dongle memory */
+	case IOV_GVAL(IOV_DUMP_DONGLE):
+	{
+		dump_dongle_in_t ddi = *(dump_dongle_in_t*)params;
+		dump_dongle_out_t *ddo = (dump_dongle_out_t*)arg;
+		uint32 *p = ddo->val;
+		const uint max_offset = 4096 - 1; /* one core contains max 4096/4 registers */
+
+		if (plen < sizeof(ddi) || len < sizeof(ddo)) {
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		switch (ddi.type) {
+		case DUMP_DONGLE_COREREG:
+			ddo->n_bytes = 0;
+
+			if (si_setcoreidx(bus->sih, ddi.index) == NULL) {
+				break; // beyond last core: core enumeration ended
+			}
+
+			ddo->address = si_addrspace(bus->sih, CORE_SLAVE_PORT_0, CORE_BASE_ADDR_0);
+			ddo->address += ddi.offset; // BP address at which this dump starts
+
+			ddo->id = si_coreid(bus->sih);
+			ddo->rev = si_corerev(bus->sih);
+
+			while (ddi.offset < max_offset &&
+				sizeof(dump_dongle_out_t) + ddo->n_bytes < (uint)len) {
+				*p++ = si_corereg(bus->sih, ddi.index, ddi.offset, 0, 0);
+				ddi.offset += sizeof(uint32);
+				ddo->n_bytes += sizeof(uint32);
+			}
+			break;
+		default:
+			// TODO: implement d11 SHM/TPL dumping
+			bcmerror = BCME_BADARG;
+			break;
+		}
+		break;
+	}
+
+	/* Debug related. Returns a string with dongle capabilities */
+	case IOV_GVAL(IOV_DHD_CAPS):
+	{
+		if (dhd_cap(bus->dhd, (char*)arg, len) == NULL) {
+			bcmerror = BCME_BUFTOOSHORT;
+		}
+		break;
+	}
+
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+	case IOV_SVAL(IOV_GDB_SERVER):
+		/* debugger_*() functions may sleep, so cannot hold spinlock */
+		if (int_val > 0) {
+			debugger_init((void *) bus, &bus_ops, int_val, SI_ENUM_BASE(bus->sih));
+		} else {
+			debugger_close();
+		}
+		break;
+#endif /* DEBUGGER || DHD_DSCOPE */
+#if defined(GDB_PROXY)
+	case IOV_GVAL(IOV_GDB_PROXY_PROBE):
+	{
+		dhd_gdb_proxy_probe_data_t ret;
+		ret.data_len = (uint32)sizeof(ret);
+		ret.magic = DHD_IOCTL_MAGIC;
+		ret.flags = 0;
+		if (bus->gdb_proxy_access_enabled) {
+			ret.flags |= DHD_GDB_PROXY_PROBE_ACCESS_ENABLED;
+			if (bus->dhd->busstate < DHD_BUS_LOAD) {
+				ret.flags |= DHD_GDB_PROXY_PROBE_FIRMWARE_NOT_RUNNING;
+			} else {
+				ret.flags |= DHD_GDB_PROXY_PROBE_FIRMWARE_RUNNING;
+			}
+		}
+		if (bus->gdb_proxy_bootloader_mode) {
+			ret.flags |= DHD_GDB_PROXY_PROBE_BOOTLOADER_MODE;
+		}
+		ret.last_id = bus->gdb_proxy_last_id;
+		if (bus->hostfw_buf.va) {
+			ret.flags |= DHD_GDB_PROXY_PROBE_HOSTMEM_CODE;
+			ret.hostmem_code_win_base =
+				(uint32)PCIEDEV_ARM_ADDR(PHYSADDRLO(bus->hostfw_buf.pa),
+				PCIEDEV_TRANS_WIN_HOSTMEM);
+			ret.hostmem_code_win_length = bus->hostfw_buf.len;
+		}
+		ret.flags |= DHD_GDB_PROXY_PROBE_MEMDUMP_COUNT;
+		ret.mem_dump_count = bus->gdb_proxy_mem_dump_count;
+		if (plen && int_val) {
+			bus->gdb_proxy_last_id = (uint32)int_val;
+		}
+		if (len >= sizeof(ret)) {
+			bcopy(&ret, arg, sizeof(ret));
+			bus->dhd->gdb_proxy_active = TRUE;
+		} else {
+			bcmerror = BCME_BADARG;
+		}
+		break;
+	}
+	case IOV_GVAL(IOV_GDB_PROXY_STOP_COUNT):
+		int_val = (int32)bus->dhd->gdb_proxy_stop_count;
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	case IOV_SVAL(IOV_GDB_PROXY_STOP_COUNT):
+		bus->dhd->gdb_proxy_stop_count = (uint32)int_val;
+		break;
+#endif /* GDB_PROXY */
+
+#ifdef BCM_BUZZZ
+	/* Dump dongle side buzzz trace to console */
+	case IOV_GVAL(IOV_BUZZZ_DUMP):
+		bcmerror = dhd_buzzz_dump_dngl(bus);
+		break;
+#endif /* BCM_BUZZZ */
+
+	case IOV_SVAL(IOV_SET_DOWNLOAD_STATE):
+		bcmerror = dhdpcie_bus_download_state(bus, bool_val);
+		break;
+
+#if defined(FW_SIGNATURE)
+	case IOV_SVAL(IOV_SET_DOWNLOAD_INFO):
+	{
+		fw_download_info_t *info = (fw_download_info_t*)params;
+		DHD_INFO(("dwnldinfo: sig=%s fw=%x,%u bl=%s,0x%x\n",
+			info->fw_signature_fname,
+			info->fw_start_addr, info->fw_size,
+			info->bootloader_fname, info->bootloader_start_addr));
+		bcmerror = dhdpcie_bus_save_download_info(bus,
+			info->fw_start_addr, info->fw_size, info->fw_signature_fname,
+			info->bootloader_fname, info->bootloader_start_addr);
+		break;
+	}
+#endif /* FW_SIGNATURE */
+
+	case IOV_GVAL(IOV_RAMSIZE):
+		int_val = (int32)bus->ramsize;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_RAMSIZE):
+		bus->ramsize = int_val;
+		bus->orig_ramsize = int_val;
+		break;
+
+	case IOV_GVAL(IOV_RAMSTART):
+		int_val = (int32)bus->dongle_ram_base;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_CC_NVMSHADOW):
+	{
+		struct bcmstrbuf dump_b;
+
+		bcm_binit(&dump_b, arg, len);
+		bcmerror = dhdpcie_cc_nvmshadow(bus, &dump_b);
+		break;
+	}
+
+	case IOV_GVAL(IOV_SLEEP_ALLOWED):
+		bool_val = bus->sleep_allowed;
+		bcopy(&bool_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SLEEP_ALLOWED):
+		bus->sleep_allowed = bool_val;
+		break;
+
+	case IOV_GVAL(IOV_DONGLEISOLATION):
+		int_val = bus->dhd->dongle_isolation;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DONGLEISOLATION):
+		bus->dhd->dongle_isolation = bool_val;
+		break;
+
+	case IOV_GVAL(IOV_LTRSLEEPON_UNLOOAD):
+		int_val = bus->ltrsleep_on_unload;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_LTRSLEEPON_UNLOOAD):
+		bus->ltrsleep_on_unload = bool_val;
+		break;
+
+	case IOV_GVAL(IOV_DUMP_RINGUPD_BLOCK):
+	{
+		struct bcmstrbuf dump_b;
+		bcm_binit(&dump_b, arg, len);
+		bcmerror = dhd_prot_ringupd_dump(bus->dhd, &dump_b);
+		break;
+	}
+	case IOV_GVAL(IOV_DMA_RINGINDICES):
+	{
+		int_val = dhdpcie_get_dma_ring_indices(bus->dhd);
+		bcopy(&int_val, arg, sizeof(int_val));
+		break;
+	}
+	case IOV_SVAL(IOV_DMA_RINGINDICES):
+		bcmerror = dhdpcie_set_dma_ring_indices(bus->dhd, int_val);
+		break;
+
+	case IOV_GVAL(IOV_METADATA_DBG):
+		int_val = dhd_prot_metadata_dbg_get(bus->dhd);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_METADATA_DBG):
+		dhd_prot_metadata_dbg_set(bus->dhd, (int_val != 0));
+		break;
+
+	case IOV_GVAL(IOV_RX_METADATALEN):
+		int_val = dhd_prot_metadatalen_get(bus->dhd, TRUE);
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_RX_METADATALEN):
+#if !defined(BCM_ROUTER_DHD)
+		if (int_val > 64) {
+			bcmerror = BCME_BUFTOOLONG;
+			break;
+		}
+		dhd_prot_metadatalen_set(bus->dhd, int_val, TRUE);
+#else
+		bcmerror = BCME_UNSUPPORTED;
+#endif /* BCM_ROUTER_DHD */
+		break;
+
+	case IOV_SVAL(IOV_TXP_THRESHOLD):
+		dhd_prot_txp_threshold(bus->dhd, TRUE, int_val);
+		break;
+
+	case IOV_GVAL(IOV_TXP_THRESHOLD):
+		int_val = dhd_prot_txp_threshold(bus->dhd, FALSE, int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DB1_FOR_MB):
+		if (int_val)
+			bus->db1_for_mb = TRUE;
+		else
+			bus->db1_for_mb = FALSE;
+		break;
+
+	case IOV_GVAL(IOV_DB1_FOR_MB):
+		if (bus->db1_for_mb)
+			int_val = 1;
+		else
+			int_val = 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_TX_METADATALEN):
+		int_val = dhd_prot_metadatalen_get(bus->dhd, FALSE);
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_TX_METADATALEN):
+#if !defined(BCM_ROUTER_DHD)
+		if (int_val > 64) {
+			bcmerror = BCME_BUFTOOLONG;
+			break;
+		}
+		dhd_prot_metadatalen_set(bus->dhd, int_val, FALSE);
+#else
+		bcmerror = BCME_UNSUPPORTED;
+#endif /* BCM_ROUTER_DHD */
+		break;
+
+	case IOV_SVAL(IOV_DEVRESET):
+	{
+		devreset_info_t *devreset = (devreset_info_t *)arg;
+
+		if (!devreset) {
+			return BCME_BADARG;
+		}
+
+		if (devreset->length == sizeof(devreset_info_t)) {
+			if (devreset->version != DHD_DEVRESET_VERSION) {
+				return BCME_VERSION;
+			}
+			int_val = devreset->mode;
+		}
+
+		switch (int_val) {
+			case DHD_BUS_DEVRESET_ON:
+				bcmerror = dhd_bus_devreset(bus->dhd, (uint8)int_val);
+				break;
+			case DHD_BUS_DEVRESET_OFF:
+				bcmerror = dhd_bus_devreset(bus->dhd, (uint8)int_val);
+				break;
+#if !defined(NDIS)
+			case DHD_BUS_DEVRESET_FLR:
+				bcmerror = dhd_bus_perform_flr(bus, bus->flr_force_fail);
+				break;
+			case DHD_BUS_DEVRESET_FLR_FORCE_FAIL:
+				bus->flr_force_fail = TRUE;
+				break;
+#endif /* !defined(NDIS) */
+			default:
+				DHD_ERROR(("%s: invalid argument for devreset\n", __FUNCTION__));
+				break;
+		}
+		break;
+	}
+	case IOV_SVAL(IOV_FORCE_FW_TRAP):
+		if (bus->dhd->busstate == DHD_BUS_DATA)
+			dhdpcie_fw_trap(bus);
+		else {
+			DHD_ERROR(("%s: Bus is NOT up\n", __FUNCTION__));
+			bcmerror = BCME_NOTUP;
+		}
+		break;
+	case IOV_GVAL(IOV_FLOW_PRIO_MAP):
+		int_val = bus->dhd->flow_prio_map_type;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_FLOW_PRIO_MAP):
+		int_val = (int32)dhd_update_flow_prio_map(bus->dhd, (uint8)int_val);
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_H2D_MAILBOXDATA):
+		dhdpcie_send_mb_data(bus, (uint)int_val);
+		break;
+
+	case IOV_SVAL(IOV_INFORINGS):
+		dhd_prot_init_info_rings(bus->dhd);
+		break;
+
+	case IOV_SVAL(IOV_H2D_PHASE):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		if (int_val)
+			bus->dhd->h2d_phase_supported = TRUE;
+		else
+			bus->dhd->h2d_phase_supported = FALSE;
+		break;
+
+	case IOV_GVAL(IOV_H2D_PHASE):
+		int_val = (int32) bus->dhd->h2d_phase_supported;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_H2D_ENABLE_TRAP_BADPHASE):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		if (int_val)
+			bus->dhd->force_dongletrap_on_bad_h2d_phase = TRUE;
+		else
+			bus->dhd->force_dongletrap_on_bad_h2d_phase = FALSE;
+		break;
+
+	case IOV_GVAL(IOV_H2D_ENABLE_TRAP_BADPHASE):
+		int_val = (int32) bus->dhd->force_dongletrap_on_bad_h2d_phase;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_RING_SIZE_VER):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		dhd_prot_set_ring_size_ver(bus->dhd, int_val);
+		break;
+
+	case IOV_GVAL(IOV_RING_SIZE_VER):
+		int_val = ring_size_version;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_H2D_TXPOST_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		h2d_max_txpost = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_H2D_TXPOST_MAX_ITEM):
+		int_val = h2d_max_txpost;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_H2D_HTPUT_TXPOST_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		h2d_htput_max_txpost = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_H2D_HTPUT_TXPOST_MAX_ITEM):
+		int_val = h2d_htput_max_txpost;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_D2H_TXCPL_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		d2h_max_txcpl = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_D2H_TXCPL_MAX_ITEM):
+		int_val = d2h_max_txcpl;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_H2D_RXPOST_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		h2d_max_rxpost = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_H2D_RXPOST_MAX_ITEM):
+		int_val = h2d_max_rxpost;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_D2H_RXCPL_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		d2h_max_rxcpl = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_D2H_RXCPL_MAX_ITEM):
+		int_val = d2h_max_rxcpl;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_H2D_CTRLPOST_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		h2d_max_ctrlpost = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_H2D_CTRLPOST_MAX_ITEM):
+		int_val = h2d_max_ctrlpost;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_D2H_CTRLCPL_MAX_ITEM):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		d2h_max_ctrlcpl = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_D2H_CTRLCPL_MAX_ITEM):
+		int_val = d2h_max_ctrlcpl;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_RX_BUF_BURST):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		rx_buf_burst = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_RX_BUF_BURST):
+		int_val = rx_buf_burst;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_RX_BUFPOST_THRESHOLD):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		rx_bufpost_threshold = (uint16)int_val;
+		break;
+
+	case IOV_GVAL(IOV_RX_BUFPOST_THRESHOLD):
+		int_val = rx_bufpost_threshold;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_TX_CPL_BOUND):
+		int_val = (int32)dhd_prot_get_tx_cpl_bound(bus->dhd);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_TX_CPL_BOUND):
+		if (int_val <= 0) {
+			bcmerror = BCME_BADARG;
+			DHD_ERROR(("%s: invalid tx_cpl_bound value %d !\n",
+				__FUNCTION__, int_val));
+		} else {
+			dhd_prot_set_tx_cpl_bound(bus->dhd, (uint)int_val);
+		}
+		break;
+	case IOV_GVAL(IOV_RX_CPL_POST_BOUND):
+		int_val = (int32)dhd_prot_get_rx_cpl_post_bound(bus->dhd);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_RX_CPL_POST_BOUND):
+		if (int_val <= 0) {
+			bcmerror = BCME_BADARG;
+			DHD_ERROR(("%s: invalid rx_cpl_post_bound value %d !\n",
+				__FUNCTION__, int_val));
+		} else {
+			dhd_prot_set_rx_cpl_post_bound(bus->dhd, (uint)int_val);
+		}
+		break;
+	case IOV_GVAL(IOV_CTRL_CPL_POST_BOUND):
+		int_val = (int32)dhd_prot_get_ctrl_cpl_post_bound(bus->dhd);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_CTRL_CPL_POST_BOUND):
+		if (int_val <= 0) {
+			bcmerror = BCME_BADARG;
+			DHD_ERROR(("%s: invalid ctrl_cpl_post_bound value %d !\n",
+				__FUNCTION__, int_val));
+		} else {
+			dhd_prot_set_ctrl_cpl_post_bound(bus->dhd, (uint)int_val);
+		}
+		break;
+	case IOV_GVAL(IOV_TX_POST_BOUND):
+		int_val = (int32)dhd_prot_get_tx_post_bound(bus->dhd);
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_TX_POST_BOUND):
+		if (int_val <= 0) {
+			bcmerror = BCME_BADARG;
+			DHD_ERROR(("%s: invalid tx_post_bound value %d !\n",
+				__FUNCTION__, int_val));
+		} else {
+			dhd_prot_set_tx_post_bound(bus->dhd, (uint)int_val);
+		}
+		break;
+
+	case IOV_GVAL(IOV_TRAPDATA):
+	{
+		struct bcmstrbuf dump_b;
+		bcm_binit(&dump_b, arg, len);
+		bcmerror = dhd_prot_dump_extended_trap(bus->dhd, &dump_b, FALSE);
+		break;
+	}
+
+	case IOV_GVAL(IOV_TRAPDATA_RAW):
+	{
+		struct bcmstrbuf dump_b;
+		bcm_binit(&dump_b, arg, len);
+		bcmerror = dhd_prot_dump_extended_trap(bus->dhd, &dump_b, TRUE);
+		break;
+	}
+
+#ifdef DHD_PCIE_REG_ACCESS
+	case IOV_GVAL(IOV_PCIEASPM): {
+		uint8 clkreq = 0;
+		uint32 aspm = 0;
+
+		/* this command is to hide the details, but match the lcreg
+		#define PCIE_CLKREQ_ENAB		0x100
+		#define PCIE_ASPM_L1_ENAB        	2
+		#define PCIE_ASPM_L0s_ENAB       	1
+		*/
+
+		clkreq = dhdpcie_clkreq(bus->dhd->osh, 0, 0);
+		aspm = dhdpcie_lcreg(bus->dhd->osh, 0, 0);
+
+		int_val = ((clkreq & 0x1) << 8) | (aspm & PCIE_ASPM_ENAB);
+		bcopy(&int_val, arg, val_size);
+		break;
+	}
+
+	case IOV_SVAL(IOV_PCIEASPM): {
+		uint32 tmp;
+
+		tmp = dhdpcie_lcreg(bus->dhd->osh, 0, 0);
+		dhdpcie_lcreg(bus->dhd->osh, PCIE_ASPM_ENAB,
+			(tmp & ~PCIE_ASPM_ENAB) | (int_val & PCIE_ASPM_ENAB));
+
+		dhdpcie_clkreq(bus->dhd->osh, 1, ((int_val & 0x100) >> 8));
+		break;
+	}
+#endif /* DHD_PCIE_REG_ACCESS */
+	case IOV_SVAL(IOV_HANGREPORT):
+		bus->dhd->hang_report = bool_val;
+		DHD_ERROR(("%s: Set hang_report as %d\n",
+			__FUNCTION__, bus->dhd->hang_report));
+		break;
+
+	case IOV_GVAL(IOV_HANGREPORT):
+		int_val = (int32)bus->dhd->hang_report;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_CTO_PREVENTION):
+		bcmerror = dhdpcie_cto_init(bus, bool_val);
+		break;
+
+	case IOV_GVAL(IOV_CTO_PREVENTION):
+		if (bus->sih->buscorerev < 19) {
+			bcmerror = BCME_UNSUPPORTED;
+			break;
+		}
+		int_val = (int32)bus->cto_enable;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_CTO_THRESHOLD):
+		{
+			if (bus->sih->buscorerev < 19) {
+				bcmerror = BCME_UNSUPPORTED;
+				break;
+			}
+			bus->cto_threshold = (uint32)int_val;
+		}
+		break;
+
+	case IOV_GVAL(IOV_CTO_THRESHOLD):
+		if (bus->sih->buscorerev < 19) {
+			bcmerror = BCME_UNSUPPORTED;
+			break;
+		}
+		if (bus->cto_threshold) {
+			int_val = (int32)bus->cto_threshold;
+		} else {
+			int_val = pcie_cto_to_thresh_default(bus->sih->buscorerev);
+		}
+
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_PCIE_WD_RESET):
+		if (bool_val) {
+			/* Legacy chipcommon watchdog reset */
+			dhdpcie_cc_watchdog_reset(bus);
+		}
+		break;
+
+	case IOV_GVAL(IOV_IDMA_ENABLE):
+		int_val = bus->idma_enabled;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_IDMA_ENABLE):
+		bus->idma_enabled = (bool)int_val;
+		break;
+	case IOV_GVAL(IOV_IFRM_ENABLE):
+		int_val = bus->ifrm_enabled;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_IFRM_ENABLE):
+		bus->ifrm_enabled = (bool)int_val;
+		break;
+	case IOV_GVAL(IOV_CLEAR_RING):
+		bcopy(&int_val, arg, val_size);
+		dhd_flow_rings_flush(bus->dhd, 0);
+		break;
+	case IOV_GVAL(IOV_DAR_ENABLE):
+		int_val = bus->dar_enabled;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_DAR_ENABLE):
+		bus->dar_enabled = (bool)int_val;
+		break;
+	case IOV_GVAL(IOV_HSCBSIZE):
+		bcmerror = dhd_get_hscb_info(bus->dhd, NULL, (uint32 *)arg);
+		break;
+#ifdef DHD_BUS_MEM_ACCESS
+	case IOV_GVAL(IOV_HSCBBYTES):
+		bcmerror = dhd_get_hscb_buff(bus->dhd, int_val, int_val2, (void*)arg);
+		break;
+#endif
+#ifdef D2H_MINIDUMP
+	case IOV_GVAL(IOV_MINIDUMP_OVERRIDE):
+		int_val = bus->d2h_minidump_override;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_MINIDUMP_OVERRIDE):
+		/* Can change it only before FW download */
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			DHD_ERROR(("%s: Can change only when bus down (before FW download)\n",
+				__FUNCTION__));
+			bcmerror = BCME_NOTDOWN;
+			break;
+		}
+		bus->d2h_minidump_override = (bool)int_val;
+		break;
+#endif /* D2H_MINIDUMP */
+
+	case IOV_SVAL(IOV_EXTDTXS_IN_TXCPL):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			return BCME_NOTDOWN;
+		}
+		if (int_val)
+			bus->dhd->extdtxs_in_txcpl = TRUE;
+		else
+			bus->dhd->extdtxs_in_txcpl = FALSE;
+		break;
+
+	case IOV_GVAL(IOV_EXTDTXS_IN_TXCPL):
+		int_val = bus->dhd->extdtxs_in_txcpl;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_HOSTRDY_AFTER_INIT):
+		if (bus->dhd->busstate != DHD_BUS_DOWN) {
+			return BCME_NOTDOWN;
+		}
+		if (int_val)
+			bus->dhd->hostrdy_after_init = TRUE;
+		else
+			bus->dhd->hostrdy_after_init = FALSE;
+		break;
+
+	case IOV_GVAL(IOV_HOSTRDY_AFTER_INIT):
+		int_val = bus->dhd->hostrdy_after_init;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_DONGLE_RXLAT_INFO):
+	{
+		unsigned long flags = 0;
+		struct bcmstrbuf strbuf;
+
+		bcm_binit(&strbuf, arg, len);
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		dhd_bus_dump_rxlat_info(bus->dhd, &strbuf);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		break;
+	}
+
+	case IOV_GVAL(IOV_DONGLE_RXLAT_HISTO):
+	{
+		unsigned long flags = 0;
+		struct bcmstrbuf strbuf;
+
+		bcm_binit(&strbuf, arg, len);
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		dhd_bus_dump_rxlat_histo(bus->dhd, &strbuf);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		break;
+	}
+
+	case IOV_GVAL(IOV_DONGLE_TXCPL_INFO):
+	{
+		unsigned long flags = 0;
+		struct bcmstrbuf strbuf;
+
+		bcm_binit(&strbuf, arg, len);
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		dhd_bus_dump_txcpl_info(bus->dhd, &strbuf);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		break;
+	}
+
+	case IOV_GVAL(IOV_DONGLE_SEC_INFO):
+	{
+		unsigned long flags = 0;
+		struct bcmstrbuf strbuf;
+
+		bcm_binit(&strbuf, arg, len);
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		bcmerror = dhd_bus_security_info(bus->dhd, &strbuf);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		break;
+	}
+
+	case IOV_GVAL(IOV_DONGLE_SBOOT_DIS):
+	{
+		unsigned long flags = 0;
+		struct bcmstrbuf strbuf;
+
+		bcm_binit(&strbuf, arg, len);
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		bcmerror = dhd_bus_sboot_disable(bus->dhd, &strbuf);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		break;
+	}
+	case IOV_GVAL(IOV_LPM_MODE):
+		int_val = bus->lpm_mode;
+		if (bus->lpm_mode) {
+			if (!bus->lpm_mem_kill) {
+				int_val |= LPM_MODE_NO_MEMKILL;
+			}
+			if (!bus->lpm_force_flr) {
+				int_val |= LPM_MODE_NO_FLR;
+			}
+		}
+		(void)memcpy_s(arg, val_size, &int_val, sizeof(int_val));
+		break;
+	case IOV_SVAL(IOV_LPM_MODE):
+		if (int_val) {
+			bus->lpm_mode = TRUE;
+			if (int_val & LPM_MODE_NO_MEMKILL) {
+				bus->lpm_mem_kill = FALSE;
+			}
+			else {
+				bus->lpm_mem_kill = TRUE;
+			}
+			if (int_val & LPM_MODE_NO_FLR) {
+				bus->lpm_force_flr = FALSE;
+			}
+			else {
+				bus->lpm_force_flr = TRUE;
+			}
+		}
+		else {
+			bus->lpm_mode = FALSE;
+			bus->lpm_mem_kill = FALSE;
+			bus->lpm_force_flr = FALSE;
+		}
+		break;
+	case IOV_GVAL(IOV_MDRING_DUMP):
+	{
+		struct bcmstrbuf strbuf;
+
+		if (bus->dhd->mdring_capable) {
+			bcm_binit(&strbuf, arg, len);
+			dhd_bus_dump_mdring_info(bus->dhd, &strbuf);
+		}
+		break;
+	}
+	case IOV_SVAL(IOV_MDRING_LINK):
+		if (bus->dhd->mdring_capable && int_val) {
+			dhd_prot_mdring_link_unlink(bus->dhd, int_val, TRUE);
+		}
+		break;
+	case IOV_GVAL(IOV_MDRING_LINK):
+		if (bus->dhd->mdring_capable) {
+			int_val = dhd_prot_mdring_linked_ring(bus->dhd);
+			bcopy(&int_val, arg, val_size);
+		}
+		break;
+	case IOV_SVAL(IOV_MDRING_UNLINK):
+		if (bus->dhd->mdring_capable) {
+			dhd_prot_mdring_link_unlink(bus->dhd, int_val, FALSE);
+		}
+		break;
+	case IOV_GVAL(IOV_MDRING_UNLINK):
+		if (bus->dhd->mdring_capable) {
+			int_val = dhd_prot_mdring_linked_ring(bus->dhd);
+			bcopy(&int_val, arg, val_size);
+		}
+		break;
+	case IOV_SVAL(IOV_PTM_ENABLE):
+		if (int_val) {
+			dhdpcie_send_mb_data(bus, H2D_HOST_PTM_ENABLE);
+		} else {
+			dhdpcie_send_mb_data(bus, H2D_HOST_PTM_DISABLE);
+		}
+		break;
+	default:
+		bcmerror = BCME_UNSUPPORTED;
+		break;
+	}
+
+exit:
+	return bcmerror;
+} /* dhdpcie_bus_doiovar */
+
+/** Transfers bytes from host to dongle using pio mode */
+static int
+dhdpcie_bus_lpback_req(struct  dhd_bus *bus, uint32 len)
+{
+	if (bus->dhd == NULL) {
+		DHD_ERROR(("%s: bus not inited\n", __FUNCTION__));
+		return 0;
+	}
+	if (bus->dhd->prot == NULL) {
+		DHD_ERROR(("%s: prot is not inited\n", __FUNCTION__));
+		return 0;
+	}
+	if (bus->dhd->busstate != DHD_BUS_DATA) {
+		DHD_ERROR(("%s: not in a readystate to LPBK  is not inited\n", __FUNCTION__));
+		return 0;
+	}
+	dhdmsgbuf_lpbk_req(bus->dhd, len);
+	return 0;
+}
+
+void
+dhd_bus_dump_dar_registers(struct dhd_bus *bus)
+{
+	uint32 dar_clk_ctrl_val, dar_pwr_ctrl_val, dar_intstat_val,
+		dar_errlog_val, dar_erraddr_val, dar_pcie_mbint_val;
+	uint32 dar_clk_ctrl_reg, dar_pwr_ctrl_reg, dar_intstat_reg,
+		dar_errlog_reg, dar_erraddr_reg, dar_pcie_mbint_reg;
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: link is down\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->sih == NULL) {
+		DHD_ERROR(("%s: si_attach has not happened, cannot dump DAR registers\n",
+			__FUNCTION__));
+		return;
+	}
+
+	bus->dar_err_set = FALSE;
+
+	if (DAR_PWRREQ(bus)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	dar_clk_ctrl_reg = (uint32)DAR_CLK_CTRL(bus->sih->buscorerev);
+	dar_pwr_ctrl_reg = (uint32)DAR_PCIE_PWR_CTRL(bus->sih->buscorerev);
+	dar_intstat_reg = (uint32)DAR_INTSTAT(bus->sih->buscorerev);
+	dar_errlog_reg = (uint32)DAR_ERRLOG(bus->sih->buscorerev);
+	dar_erraddr_reg = (uint32)DAR_ERRADDR(bus->sih->buscorerev);
+	dar_pcie_mbint_reg = (uint32)DAR_PCIMailBoxInt(bus->sih->buscorerev);
+
+	if (bus->sih->buscorerev < 24) {
+		DHD_ERROR(("%s: DAR not supported for corerev(%d) < 24\n",
+			__FUNCTION__, bus->sih->buscorerev));
+		return;
+	}
+
+	dar_clk_ctrl_val = si_corereg(bus->sih, bus->sih->buscoreidx, dar_clk_ctrl_reg, 0, 0);
+	dar_pwr_ctrl_val = si_corereg(bus->sih, bus->sih->buscoreidx, dar_pwr_ctrl_reg, 0, 0);
+	dar_intstat_val = si_corereg(bus->sih, bus->sih->buscoreidx, dar_intstat_reg, 0, 0);
+	dar_errlog_val = si_corereg(bus->sih, bus->sih->buscoreidx, dar_errlog_reg, 0, 0);
+	dar_erraddr_val = si_corereg(bus->sih, bus->sih->buscoreidx, dar_erraddr_reg, 0, 0);
+	dar_pcie_mbint_val = si_corereg(bus->sih, bus->sih->buscoreidx, dar_pcie_mbint_reg, 0, 0);
+
+	DHD_RPM(("%s: dar_clk_ctrl(0x%x:0x%x) dar_pwr_ctrl(0x%x:0x%x) "
+		"dar_intstat(0x%x:0x%x)\n",
+		__FUNCTION__, dar_clk_ctrl_reg, dar_clk_ctrl_val,
+		dar_pwr_ctrl_reg, dar_pwr_ctrl_val, dar_intstat_reg, dar_intstat_val));
+
+	DHD_RPM(("%s: dar_errlog(0x%x:0x%x) dar_erraddr(0x%x:0x%x) "
+		"dar_pcie_mbint(0x%x:0x%x)\n",
+		__FUNCTION__, dar_errlog_reg, dar_errlog_val,
+		dar_erraddr_reg, dar_erraddr_val, dar_pcie_mbint_reg, dar_pcie_mbint_val));
+
+	if (dar_errlog_val || dar_erraddr_val) {
+		bus->dar_err_set = TRUE;
+	}
+
+}
+
+/* Ring DoorBell1 to indicate Hostready i.e. D3 Exit */
+void
+dhd_bus_hostready(struct  dhd_bus *bus)
+{
+	if (!bus->dhd->d2h_hostrdy_supported) {
+		return;
+	}
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_ERROR(("%s : Read PCICMD Reg: 0x%08X\n", __FUNCTION__,
+		dhd_pcie_config_read(bus, PCI_CFG_CMD, sizeof(uint32))));
+
+	dhd_bus_dump_dar_registers(bus);
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down before DB1\n", __FUNCTION__));
+		return;
+	}
+
+#ifdef DHD_MMIO_TRACE
+	dhd_bus_mmio_trace(bus, dhd_bus_db1_addr_get(bus), 0x1, TRUE);
+#endif /* defined(DHD_MMIO_TRACE) */
+	si_corereg(bus->sih, bus->sih->buscoreidx, dhd_bus_db1_addr_get(bus), ~0, 0x12345678);
+	bus->hostready_count ++;
+	DHD_ERROR(("%s: Ring Hostready:%d\n", __FUNCTION__, bus->hostready_count));
+}
+
+/* Clear INTSTATUS */
+void
+dhdpcie_bus_clear_intstatus(struct dhd_bus *bus)
+{
+	uint32 intstatus = 0;
+	/* Skip after receiving D3 ACK */
+	if (DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+		return;
+	}
+	/* XXX: check for PCIE Gen2 also */
+	if ((bus->sih->buscorerev == 6) || (bus->sih->buscorerev == 4) ||
+		(bus->sih->buscorerev == 2)) {
+		intstatus = dhdpcie_bus_cfg_read_dword(bus, PCIIntstatus, 4);
+		dhdpcie_bus_cfg_write_dword(bus, PCIIntstatus, 4, intstatus);
+	} else {
+		/* this is a PCIE core register..not a config register... */
+		intstatus = si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_int, 0, 0);
+		si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_int, bus->def_intmask,
+			intstatus);
+	}
+}
+
+#ifdef PWRSTATS_SYSFS
+uint64
+dhdpcie_get_last_suspend_time(dhd_pub_t *dhdp)
+{
+	return dhdp->bus->last_suspend_end_time;
+}
+#endif /* PWRSTATS_SYSFS */
+
+#define BCM4387_WLAN_SYS_MEMDOWN_FORCE_PMU      0x444
+#define BCM4378_WLAN_SYS_MEMDOWN_FORCE_PMU      0x133
+
+static void
+dhd_pcie_handle_lpm_memkill(struct dhd_bus *bus)
+{
+	uint16 chipid = si_chipid(bus->sih);
+
+	if (bus->lpm_mem_kill == FALSE) {
+		DHD_ERROR(("LPM MODE: MEM Kill not enabled\n"));
+		return;
+	}
+	if (chipid == BCM4378_CHIP_ID) {
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_addr, ~0, PMU_CHIPCTL13);
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_data,
+			BCM4378_WLAN_SYS_MEMDOWN_FORCE_PMU, BCM4378_WLAN_SYS_MEMDOWN_FORCE_PMU);
+	}
+	else if (chipid == BCM4387_CHIP_ID) {
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_addr, ~0, PMU_CHIPCTL13);
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_data,
+			BCM4387_WLAN_SYS_MEMDOWN_FORCE_PMU, BCM4387_WLAN_SYS_MEMDOWN_FORCE_PMU);
+	}
+	else {
+		DHD_ERROR(("LPM MODE: Mem kill support not added for chip %d\n", chipid));
+		return;
+	}
+	DHD_ERROR(("PMU chipcontrol addr %d value is 0x%04x\n",
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_addr, 0, 0),
+		pmu_corereg(bus->sih, SI_CC_IDX, chipcontrol_data, 0, 0)));
+}
+
+int
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+dhdpcie_bus_suspend(struct dhd_bus *bus, bool state, bool byint)
+#else
+dhdpcie_bus_suspend(struct dhd_bus *bus, bool state)
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+{
+	int timeleft;
+	int rc = 0;
+	unsigned long flags;
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	int d3_read_retry = 0;
+	uint32 d2h_mb_data = 0;
+	uint32 zero = 0;
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	printf("%s: state=%d\n", __FUNCTION__, state);
+	if (bus->dhd == NULL) {
+		DHD_ERROR(("%s: bus not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	if (bus->dhd->prot == NULL) {
+		DHD_ERROR(("%s: prot is not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhd_query_bus_erros(bus->dhd)) {
+		return BCME_ERROR;
+	}
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	if (!(bus->dhd->busstate == DHD_BUS_DATA || bus->dhd->busstate == DHD_BUS_SUSPEND)) {
+		DHD_ERROR(("%s: not in a readystate\n", __FUNCTION__));
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		return BCME_ERROR;
+	}
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+	if (bus->dhd->dongle_reset) {
+		DHD_ERROR(("Dongle is in reset state.\n"));
+		return -EIO;
+	}
+
+	/* Check whether we are already in the requested state.
+	 * state=TRUE means Suspend
+	 * state=FALSE meanse Resume
+	 */
+	if (state == TRUE && bus->dhd->busstate == DHD_BUS_SUSPEND) {
+		DHD_ERROR(("Bus is already in SUSPEND state.\n"));
+		return BCME_OK;
+	} else if (state == FALSE && bus->dhd->busstate == DHD_BUS_DATA) {
+		DHD_ERROR(("Bus is already in RESUME state.\n"));
+		return BCME_OK;
+	}
+
+	if (state) {
+#ifdef OEM_ANDROID
+		int idle_retry = 0;
+		int active;
+#endif /* OEM_ANDROID */
+
+		if (bus->is_linkdown) {
+			DHD_ERROR(("%s: PCIe link was down, state=%d\n",
+				__FUNCTION__, state));
+			return BCME_ERROR;
+		}
+
+		/* Suspend */
+		DHD_RPM(("%s: Entering suspend state\n", __FUNCTION__));
+
+		bus->dhd->dhd_watchdog_ms_backup = dhd_watchdog_ms;
+		if (bus->dhd->dhd_watchdog_ms_backup) {
+			DHD_ERROR(("%s: Disabling wdtick before going to suspend\n",
+				__FUNCTION__));
+			dhd_os_wd_timer(bus->dhd, 0);
+		}
+
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+#if defined(LINUX) || defined(linux)
+		if (DHD_BUS_BUSY_CHECK_IN_TX(bus->dhd)) {
+			DHD_ERROR(("Tx Request is not ended\n"));
+			bus->dhd->busstate = DHD_BUS_DATA;
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+			return -EBUSY;
+		}
+#endif /* LINUX || linux */
+
+		bus->last_suspend_start_time = OSL_LOCALTIME_NS();
+
+		/* stop all interface network queue. */
+		dhd_bus_stop_queue(bus);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+		if (byint) {
+			DHD_OS_WAKE_LOCK_WAIVE(bus->dhd);
+			/* Clear wait_for_d3_ack before sending D3_INFORM */
+			bus->wait_for_d3_ack = 0;
+			dhdpcie_send_mb_data(bus, H2D_HOST_D3_INFORM);
+
+			timeleft = dhd_os_d3ack_wait(bus->dhd, &bus->wait_for_d3_ack);
+			DHD_OS_WAKE_LOCK_RESTORE(bus->dhd);
+		} else {
+			/* Clear wait_for_d3_ack before sending D3_INFORM */
+			bus->wait_for_d3_ack = 0;
+			dhdpcie_send_mb_data(bus, H2D_HOST_D3_INFORM | H2D_HOST_ACK_NOINT);
+			while (!bus->wait_for_d3_ack && d3_read_retry < MAX_D3_ACK_TIMEOUT) {
+				dhdpcie_handle_mb_data(bus);
+				usleep_range(1000, 1500);
+				d3_read_retry++;
+			}
+		}
+#else
+		DHD_OS_WAKE_LOCK_WAIVE(bus->dhd);
+
+#ifdef PCIE_INB_DW
+		/* As D3_INFORM will be sent after De-assert,
+		 * skip sending DS-ACK for DS-REQ.
+		 */
+	DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+	bus->skip_ds_ack = TRUE;
+	DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+#endif /* PCIE_INB_DW */
+
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+		dhd_bus_set_device_wake(bus, TRUE);
+#endif /* defined(PCIE_OOB) || defined(PCIE_INB_DW) */
+#ifdef PCIE_OOB
+		bus->oob_presuspend = TRUE;
+#endif
+#ifdef PCIE_INB_DW
+		/* De-assert at this point for In-band device_wake */
+		if (INBAND_DW_ENAB(bus)) {
+			dhd_bus_set_device_wake(bus, FALSE);
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			dhdpcie_bus_set_pcie_inband_dw_state(bus, DW_DEVICE_HOST_SLEEP_WAIT);
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+		}
+#endif /* PCIE_INB_DW */
+		/* Clear wait_for_d3_ack before sending D3_INFORM */
+		bus->wait_for_d3_ack = 0;
+		/*
+		 * Send H2D_HOST_D3_INFORM to dongle and mark bus->bus_low_power_state
+		 * to DHD_BUS_D3_INFORM_SENT in dhd_prot_ring_write_complete_mbdata
+		 * inside atomic context, so that no more DBs will be
+		 * rung after sending D3_INFORM
+		 */
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			dhdpcie_send_mb_data(bus, H2D_HOST_D3_INFORM);
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+		} else
+#endif /* PCIE_INB_DW */
+		{
+			dhdpcie_send_mb_data(bus, H2D_HOST_D3_INFORM);
+		}
+
+		/* Wait for D3 ACK for D3_ACK_RESP_TIMEOUT seconds */
+
+		timeleft = dhd_os_d3ack_wait(bus->dhd, &bus->wait_for_d3_ack);
+#ifdef DHD_RECOVER_TIMEOUT
+		/* XXX: WAR for missing D3 ACK MB interrupt */
+		if (bus->wait_for_d3_ack == 0) {
+			/* If wait_for_d3_ack was not updated because D2H MB was not received */
+			uint32 intstatus = si_corereg(bus->sih, bus->sih->buscoreidx,
+				bus->pcie_mailbox_int, 0, 0);
+			int host_irq_disabled = dhdpcie_irq_disabled(bus);
+			if ((intstatus) && (intstatus != (uint32)-1) &&
+				(timeleft == 0) && (!dhd_query_bus_erros(bus->dhd))) {
+				DHD_ERROR(("%s: resumed on timeout for D3 ack. intstatus=%x"
+					" host_irq_disabled=%d\n",
+					__FUNCTION__, intstatus, host_irq_disabled));
+				dhd_pcie_intr_count_dump(bus->dhd);
+				dhd_print_tasklet_status(bus->dhd);
+				dhd_prot_ctrl_info_print(bus->dhd);
+				/* Clear Interrupts */
+				dhdpcie_bus_clear_intstatus(bus);
+				if (dhd_prot_check_pending_ctrl_cmpls(bus->dhd)) {
+					DHD_ERROR(("##### %s: D3 ack timeout trying again #####\n",
+						__FUNCTION__));
+					dhd_schedule_delayed_dpc_on_dpc_cpu(bus->dhd, 0);
+					timeleft = dhd_os_d3ack_wait(bus->dhd,
+						&bus->wait_for_d3_ack);
+				}
+			}
+		} /* bus->wait_for_d3_ack was 0 */
+#endif /* DHD_RECOVER_TIMEOUT */
+
+		DHD_OS_WAKE_LOCK_RESTORE(bus->dhd);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef OEM_ANDROID
+		/* To allow threads that got pre-empted to complete.
+		 */
+		while ((active = dhd_os_check_wakelock_all(bus->dhd)) &&
+			(idle_retry < MAX_WKLK_IDLE_CHECK)) {
+			OSL_SLEEP(1);
+			idle_retry++;
+		}
+#endif /* OEM_ANDROID */
+
+		if (bus->wait_for_d3_ack) {
+			DHD_RPM(("%s: Got D3 Ack \n", __FUNCTION__));
+			/* Got D3 Ack. Suspend the bus */
+#ifdef OEM_ANDROID
+			if (active) {
+				DHD_ERROR(("%s():Suspend failed because of wakelock "
+					"restoring Dongle to D0\n", __FUNCTION__));
+
+				if (bus->dhd->dhd_watchdog_ms_backup) {
+					DHD_ERROR(("%s: Enabling wdtick due to wakelock active\n",
+						__FUNCTION__));
+					dhd_os_wd_timer(bus->dhd,
+						bus->dhd->dhd_watchdog_ms_backup);
+				}
+
+				/*
+				 * Dongle still thinks that it has to be in D3 state until
+				 * it gets a D0 Inform, but we are backing off from suspend.
+				 * Ensure that Dongle is brought back to D0.
+				 *
+				 * Bringing back Dongle from D3 Ack state to D0 state is a
+				 * 2 step process. Dongle would want to know that D0 Inform
+				 * would be sent as a MB interrupt to bring it out of D3 Ack
+				 * state to D0 state. So we have to send both this message.
+				 */
+
+				/* Clear wait_for_d3_ack to send D0_INFORM or host_ready */
+				bus->wait_for_d3_ack = 0;
+
+				DHD_SET_BUS_NOT_IN_LPS(bus);
+#ifdef PCIE_INB_DW
+				if (INBAND_DW_ENAB(bus)) {
+					DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+					/* Since suspend has failed due to wakelock is held,
+					 * update the DS state to DW_DEVICE_HOST_WAKE_WAIT.
+					 * So that host sends the DS-ACK for DS-REQ.
+					 */
+					DHD_ERROR(("Suspend failed due to wakelock is held, "
+					 "set inband dw state to DW_DEVICE_HOST_WAKE_WAIT\n"));
+					dhdpcie_bus_set_pcie_inband_dw_state(bus,
+						DW_DEVICE_HOST_WAKE_WAIT);
+					dhd_bus_ds_trace(bus, 0, TRUE,
+						dhdpcie_bus_get_pcie_inband_dw_state(bus));
+					bus->skip_ds_ack = FALSE;
+					DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				}
+#endif /* PCIE_INB_DW */
+				/* Enable back the intmask which was cleared in DPC
+				 * after getting D3_ACK.
+				 */
+				dhdpcie_bus_intr_enable(bus);
+				bus->dngl_intmask_enable_count++;
+
+				if (bus->use_d0_inform) {
+					DHD_OS_WAKE_LOCK_WAIVE(bus->dhd);
+					dhdpcie_send_mb_data(bus,
+						(H2D_HOST_D0_INFORM_IN_USE | H2D_HOST_D0_INFORM));
+					DHD_OS_WAKE_LOCK_RESTORE(bus->dhd);
+				}
+				/* ring doorbell 1 (hostready) */
+				dhd_bus_hostready(bus);
+
+				DHD_GENERAL_LOCK(bus->dhd, flags);
+				bus->dhd->busstate = DHD_BUS_DATA;
+				/* resume all interface network queue. */
+				dhd_bus_start_queue(bus);
+				DHD_GENERAL_UNLOCK(bus->dhd, flags);
+				rc = BCME_ERROR;
+			} else {
+				/* Actual Suspend after no wakelock */
+#endif /* OEM_ANDROID */
+				/* At this time bus->bus_low_power_state will be
+				 * made to DHD_BUS_D3_ACK_RECIEVED after receiving D3_ACK
+				 * in dhd_bus_handle_d3_ack()
+				 */
+#ifdef PCIE_OOB
+				bus->oob_presuspend = FALSE;
+				if (OOB_DW_ENAB(bus)) {
+					dhd_bus_set_device_wake(bus, FALSE);
+				}
+#endif
+#ifdef PCIE_OOB
+				bus->oob_presuspend = TRUE;
+#endif
+#ifdef PCIE_INB_DW
+				if (INBAND_DW_ENAB(bus)) {
+					DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+					if (dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+						DW_DEVICE_HOST_SLEEP_WAIT) {
+						dhdpcie_bus_set_pcie_inband_dw_state(bus,
+							DW_DEVICE_HOST_SLEEP);
+#ifdef PCIE_INB_DW
+						dhd_bus_ds_trace(bus, 0, TRUE,
+							dhdpcie_bus_get_pcie_inband_dw_state(bus));
+#else
+						dhd_bus_ds_trace(bus, 0, TRUE);
+#endif /* PCIE_INB_DW */
+					}
+					DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				}
+#endif /* PCIE_INB_DW */
+				if (bus->use_d0_inform &&
+					(bus->api.fw_rev < PCIE_SHARED_VERSION_6)) {
+					DHD_OS_WAKE_LOCK_WAIVE(bus->dhd);
+					dhdpcie_send_mb_data(bus, (H2D_HOST_D0_INFORM_IN_USE));
+					DHD_OS_WAKE_LOCK_RESTORE(bus->dhd);
+				}
+#ifdef PCIE_OOB
+				dhd_bus_set_device_wake(bus, FALSE);
+#endif /* PCIE_OOB */
+
+#if defined(BCMPCIE_OOB_HOST_WAKE)
+				if (bus->dhd->dhd_induce_error == DHD_INDUCE_DROP_OOB_IRQ) {
+					DHD_ERROR(("%s: Inducing DROP OOB IRQ\n", __FUNCTION__));
+				} else {
+					dhdpcie_oob_intr_set(bus, TRUE);
+				}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+				DHD_GENERAL_LOCK(bus->dhd, flags);
+				/* The Host cannot process interrupts now so disable the same.
+				 * No need to disable the dongle INTR using intmask, as we are
+				 * already calling disabling INTRs from DPC context after
+				 * getting D3_ACK in dhd_bus_handle_d3_ack.
+				 * Code may not look symmetric between Suspend and
+				 * Resume paths but this is done to close down the timing window
+				 * between DPC and suspend context and bus->bus_low_power_state
+				 * will be set to DHD_BUS_D3_ACK_RECIEVED in DPC.
+				 */
+				bus->dhd->d3ackcnt_timeout = 0;
+				bus->dhd->busstate = DHD_BUS_SUSPEND;
+				DHD_GENERAL_UNLOCK(bus->dhd, flags);
+#if defined(LINUX) || defined(linux)
+				dhdpcie_dump_resource(bus);
+#endif /* LINUX || linux */
+				DHD_INFO(("lpm_mode %d, fw_lpm_support %d\n", bus->lpm_mode,
+					bus->dhd->fw_lpm_support));
+				/* Once LPM mode is entered, recovery is only through WL_REG_ON */
+				if (bus->lpm_mode && bus->dhd->fw_lpm_support) {
+					/* send DB7 with PCIE_DB7_MAGIC_NUMBER_DPC_LPM */
+					int trap_data = 0;
+					uint32 addr = dhd_bus_db1_addr_3_get(bus);
+
+					DHD_ERROR(("forcing LPM mode\n"));
+					si_corereg(bus->sih, bus->sih->buscoreidx, addr, ~0,
+						PCIE_DB7_MAGIC_NUMBER_DPC_LPM);
+					/* wait 500ms */
+					OSL_DELAY(500000);
+					/* read trap data to confirm */
+					trap_data = dhd_prot_process_trapbuf(bus->dhd);
+					DHD_ERROR(("trap data in response to LPM mode entry"
+						" is 0x%x\n", trap_data));
+					/* force memory bits down */
+					if (trap_data) {
+						dhd_pcie_handle_lpm_memkill(bus);
+					}
+				}
+				DHD_INFO(("Doing the D3\n"));
+				rc = dhdpcie_pci_suspend_resume(bus, state);
+				if (!rc) {
+					bus->last_suspend_end_time = OSL_LOCALTIME_NS();
+				}
+				DHD_INFO(("Doing the FLR, lpm_mode %d, lpm_force_flr %d,"
+					" fwsupport %d\n",
+					bus->lpm_mode, bus->lpm_force_flr,
+					bus->dhd->fw_lpm_support));
+				if (bus->lpm_mode && bus->dhd->fw_lpm_support &&
+					bus->lpm_force_flr) {
+					bus->lpm_keep_in_reset = TRUE;
+					dhd_bus_perform_flr(bus, FALSE);
+					bus->lpm_keep_in_reset = FALSE;
+					DHD_ERROR(("FLR done\n"));
+				}
+#ifdef OEM_ANDROID
+			}
+#endif /* OEM_ANDROID */
+		} else if (timeleft == 0) { /* D3 ACK Timeout */
+#ifdef DHD_FW_COREDUMP
+			uint32 cur_memdump_mode = bus->dhd->memdump_enabled;
+#endif /* DHD_FW_COREDUMP */
+
+			/* check if the D3 ACK timeout due to scheduling issue */
+			bus->dhd->is_sched_error = !dhd_query_bus_erros(bus->dhd) &&
+				dhd_bus_query_dpc_sched_errors(bus->dhd);
+			DHD_ERROR(("%s: resumed on timeout for D3 ACK%s d3_inform_cnt %d\n",
+				__FUNCTION__, bus->dhd->is_sched_error ?
+				" due to scheduling problem" : "", bus->dhd->d3ackcnt_timeout));
+#if defined(DHD_KERNEL_SCHED_DEBUG) && defined(DHD_FW_COREDUMP)
+			/* XXX DHD triggers Kernel panic if the resumed on timeout occurrs
+			 * due to tasklet or workqueue scheduling problems in the Linux Kernel.
+			 * Customer informs that it is hard to find any clue from the
+			 * host memory dump since the important tasklet or workqueue information
+			 * is already disappered due the latency while printing out the timestamp
+			 * logs for debugging scan timeout issue.
+			 * For this reason, customer requestes us to trigger Kernel Panic rather
+			 * than taking a SOCRAM dump.
+			 */
+			if (bus->dhd->is_sched_error && cur_memdump_mode == DUMP_MEMFILE_BUGON) {
+				/* change g_assert_type to trigger Kernel panic */
+				g_assert_type = 2;
+				/* use ASSERT() to trigger panic */
+				ASSERT(0);
+			}
+#endif /* DHD_KERNEL_SCHED_DEBUG && DHD_FW_COREDUMP */
+			DHD_SET_BUS_NOT_IN_LPS(bus);
+
+			DHD_GENERAL_LOCK(bus->dhd, flags);
+			bus->dhd->busstate = DHD_BUS_DATA;
+			/* resume all interface network queue. */
+			dhd_bus_start_queue(bus);
+			DHD_GENERAL_UNLOCK(bus->dhd, flags);
+			/* Dump important config space registers */
+			dhd_bus_dump_imp_cfg_registers(bus);
+			/* XXX : avoid multiple socram dump from dongle trap and
+			 * invalid PCIe bus assceess due to PCIe link down
+			 */
+			if (bus->dhd->check_trap_rot) {
+				DHD_ERROR(("Check dongle trap in the case of d3 ack timeout\n"));
+				dhdpcie_checkdied(bus, NULL, 0);
+			}
+			/* Set d3_ack_timeout_occurred after checkdied,
+			 * as checkdied returns for any bus error
+			 */
+			bus->dhd->d3ack_timeout_occured = TRUE;
+			/* If the D3 Ack has timeout */
+			bus->dhd->d3ackcnt_timeout++;
+
+			dhd_validate_pcie_link_cbp_wlbp(bus);
+			if (bus->link_state != DHD_PCIE_ALL_GOOD) {
+				DHD_ERROR(("%s: bus->link_state:%d\n",
+					__FUNCTION__, bus->link_state));
+#ifdef OEM_ANDROID
+				dhd_os_check_hang(bus->dhd, 0, -ETIMEDOUT);
+#endif /* OEM_ANDROID */
+			}
+
+			if (bus->dhd->dongle_trap_occured) {
+#ifdef OEM_ANDROID
+				dhd_os_check_hang(bus->dhd, 0, -EREMOTEIO);
+#endif /* OEM_ANDROID */
+			} else if (!bus->is_linkdown &&
+				!bus->cto_triggered && (bus->link_state == DHD_PCIE_ALL_GOOD)) {
+				uint32 intstatus = 0;
+
+				/* Check if PCIe bus status is valid */
+				intstatus = si_corereg(bus->sih, bus->sih->buscoreidx,
+					bus->pcie_mailbox_int, 0, 0);
+				if (intstatus == (uint32)-1) {
+					/* Invalidate PCIe bus status */
+					bus->is_linkdown = 1;
+				}
+
+				dhd_bus_dump_console_buffer(bus);
+				dhd_prot_debug_info_print(bus->dhd);
+#ifdef DHD_FW_COREDUMP
+				if (cur_memdump_mode) {
+					/* write core dump to file */
+					bus->dhd->memdump_type = DUMP_TYPE_D3_ACK_TIMEOUT;
+					dhdpcie_mem_dump(bus);
+				}
+#endif /* DHD_FW_COREDUMP */
+
+#ifdef NDIS
+				/* ASSERT only if hang detection/recovery is disabled.
+				 * If enabled then let
+				 * windows HDR mechansim trigger FW download via surprise removal
+				 */
+				dhd_bus_check_died(bus);
+#endif
+#ifdef OEM_ANDROID
+				DHD_ERROR(("%s: Event HANG send up due to D3_ACK timeout\n",
+					__FUNCTION__));
+				dhd_os_check_hang(bus->dhd, 0, -ETIMEDOUT);
+#endif /* OEM_ANDROID */
+			}
+			rc = -ETIMEDOUT;
+		}
+#ifdef PCIE_OOB
+		bus->oob_presuspend = FALSE;
+#endif /* PCIE_OOB */
+	} else {
+		/* Resume */
+		DHD_RPM(("%s: Entering resume state\n", __FUNCTION__));
+		bus->last_resume_start_time = OSL_LOCALTIME_NS();
+
+		/**
+		 * PCIE2_BAR0_CORE2_WIN gets reset after D3 cold.
+		 * si_backplane_access(function to read/write backplane)
+		 * updates the window(PCIE2_BAR0_CORE2_WIN) only if
+		 * window being accessed is different form the window
+		 * being pointed by second_bar0win.
+		 * Since PCIE2_BAR0_CORE2_WIN is already reset because of D3 cold,
+		 * invalidating second_bar0win after resume updates
+		 * PCIE2_BAR0_CORE2_WIN with right window.
+		 */
+		si_invalidate_second_bar0win(bus->sih);
+#if defined(linux) && defined(OEM_ANDROID)
+#if defined(BCMPCIE_OOB_HOST_WAKE)
+		DHD_OS_OOB_IRQ_WAKE_UNLOCK(bus->dhd);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#endif /* linux && OEM_ANDROID */
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			if (dhdpcie_bus_get_pcie_inband_dw_state(bus) == DW_DEVICE_HOST_SLEEP) {
+				dhdpcie_bus_set_pcie_inband_dw_state(bus, DW_DEVICE_HOST_WAKE_WAIT);
+#ifdef PCIE_INB_DW
+				dhd_bus_ds_trace(bus, 0, TRUE,
+					dhdpcie_bus_get_pcie_inband_dw_state(bus));
+#else
+				dhd_bus_ds_trace(bus, 0, TRUE);
+#endif /* PCIE_INB_DW */
+			}
+			bus->skip_ds_ack = FALSE;
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+		}
+#endif /* PCIE_INB_DW */
+		rc = dhdpcie_pci_suspend_resume(bus, state);
+#if defined(LINUX) || defined(linux)
+		dhdpcie_dump_resource(bus);
+#endif /* LINUX || linux */
+
+		/* set bus_low_power_state to DHD_BUS_NO_LOW_POWER_STATE */
+		DHD_SET_BUS_NOT_IN_LPS(bus);
+
+		if (!rc && bus->dhd->busstate == DHD_BUS_SUSPEND) {
+			if (bus->use_d0_inform) {
+				DHD_OS_WAKE_LOCK_WAIVE(bus->dhd);
+				dhdpcie_send_mb_data(bus, (H2D_HOST_D0_INFORM));
+				DHD_OS_WAKE_LOCK_RESTORE(bus->dhd);
+			}
+			/* ring doorbell 1 (hostready) */
+			dhd_bus_hostready(bus);
+		}
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		bus->dhd->busstate = DHD_BUS_DATA;
+#ifdef PCIE_OOB
+		/*
+		 * Assert & Deassert the Device Wake. The following is the explanation for doing so.
+		 * 0) At this point,
+		 *    Host is in suspend state, Link is in L2/L3, Dongle is in D3 Cold
+		 *    Device Wake is enabled.
+		 * 1) When the Host comes out of Suspend, it first sends PERST# in the Link.
+		 *    Looking at this the Dongle moves from D3 Cold to NO DS State
+		 * 2) Now The Host OS calls the "resume" function of DHD. From here the DHD first
+		 *    Asserts the Device Wake.
+		 *    From the defn, when the Device Wake is asserted, The dongle FW will ensure
+		 *    that the Dongle is out of deep sleep IF the device is already in deep sleep.
+		 *    But note that now the Dongle is NOT in Deep sleep and is actually in
+		 *    NO DS state. So just driving the Device Wake high does not trigger any state
+		 *    transitions. The Host should actually "Toggle" the Device Wake to ensure
+		 *    that Dongle synchronizes with the Host and starts the State Transition to D0.
+		 * 4) Note that the above explanation is applicable Only when the Host comes out of
+		 *    suspend and the Dongle comes out of D3 Cold
+		 */
+		/* This logic is not required when hostready is enabled */
+
+		if (!bus->dhd->d2h_hostrdy_supported) {
+			dhd_bus_set_device_wake(bus, TRUE);
+			OSL_DELAY(1000);
+			dhd_bus_set_device_wake(bus, FALSE);
+		}
+
+#endif /* PCIE_OOB */
+		/* resume all interface network queue. */
+		dhd_bus_start_queue(bus);
+
+		/* Enable back the intmask which was cleared in DPC
+		 * after getting D3_ACK.
+		 */
+		dhdpcie_bus_intr_enable(bus);
+		bus->dngl_intmask_enable_count++;
+
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+		if (bus->dhd->dhd_watchdog_ms_backup) {
+			DHD_ERROR(("%s: Enabling wdtick after resume\n",
+				__FUNCTION__));
+			dhd_os_wd_timer(bus->dhd, bus->dhd->dhd_watchdog_ms_backup);
+		}
+
+		bus->last_resume_end_time = OSL_LOCALTIME_NS();
+
+		/* Update TCM rd index for EDL ring */
+		DHD_EDL_RING_TCM_RD_UPDATE(bus->dhd);
+
+	}
+	return rc;
+}
+
+#define BUS_SUSPEND	TRUE
+#define BUS_RESUME	FALSE
+int dhd_bus_suspend(dhd_pub_t *dhd)
+{
+	int ret;
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	/* TODO: Check whether the arguments are correct */
+	ret = dhdpcie_bus_suspend(dhd->bus, TRUE, BUS_SUSPEND);
+#else
+	ret = dhdpcie_bus_suspend(dhd->bus, BUS_SUSPEND);
+#endif
+	return ret;
+}
+
+int dhd_bus_resume(dhd_pub_t *dhd, int stage)
+{
+	int ret;
+	BCM_REFERENCE(stage);
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	/* TODO: Check whether the arguments are correct */
+	ret = dhdpcie_bus_suspend(dhd->bus, FALSE, BUS_RESUME);
+#else
+	ret = dhdpcie_bus_suspend(dhd->bus, BUS_RESUME);
+#endif
+	return ret;
+}
+
+uint32
+dhdpcie_force_alp(struct dhd_bus *bus, bool enable)
+{
+	ASSERT(bus && bus->sih);
+	if (enable) {
+	si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, u.pcie2.clk_ctl_st), CCS_FORCEALP, CCS_FORCEALP);
+	} else {
+		si_corereg(bus->sih, bus->sih->buscoreidx,
+			OFFSETOF(sbpcieregs_t, u.pcie2.clk_ctl_st), CCS_FORCEALP, 0);
+	}
+	return 0;
+}
+
+/* set pcie l1 entry time: dhd pciereg 0x1004[22:16] */
+uint32
+dhdpcie_set_l1_entry_time(struct dhd_bus *bus, int l1_entry_time)
+{
+	uint reg_val;
+
+	ASSERT(bus && bus->sih);
+
+	si_corereg(bus->sih, bus->sih->buscoreidx, OFFSETOF(sbpcieregs_t, configaddr), ~0,
+		0x1004);
+	reg_val = si_corereg(bus->sih, bus->sih->buscoreidx,
+		OFFSETOF(sbpcieregs_t, configdata), 0, 0);
+	reg_val = (reg_val & ~(0x7f << 16)) | ((l1_entry_time & 0x7f) << 16);
+	si_corereg(bus->sih, bus->sih->buscoreidx, OFFSETOF(sbpcieregs_t, configdata), ~0,
+		reg_val);
+
+	return 0;
+}
+
+static uint32
+dhd_apply_d11_war_length(struct  dhd_bus *bus, uint32 len, uint32 d11_lpbk)
+{
+	uint16 chipid = si_chipid(bus->sih);
+	/*
+	 * XXX: WAR for CRWLDOT11M-3011
+	 * program the DMA descriptor Buffer length as the expected frame length
+	 *  + 8 bytes extra for corerev 82 when buffer length % 128 is equal to 4
+	 */
+	if ((chipid == BCM4375_CHIP_ID ||
+		chipid == BCM4362_CHIP_ID ||
+		chipid == BCM4377_CHIP_ID ||
+		chipid == BCM43751_CHIP_ID ||
+		chipid == BCM43752_CHIP_ID ||
+		chipid == BCM43756_CHIP_ID ||
+		chipid == BCM43711_CHIP_ID) &&
+		(d11_lpbk != M2M_DMA_LPBK && d11_lpbk != M2M_NON_DMA_LPBK)) {
+			len += 8;
+	}
+	DHD_ERROR(("%s: len %d\n", __FUNCTION__, len));
+	return len;
+}
+
+/** Transfers bytes from host to dongle and to host again using DMA */
+static int
+dhdpcie_bus_dmaxfer_req(struct  dhd_bus *bus,
+		uint32 len, uint32 srcdelay, uint32 destdelay,
+		uint32 d11_lpbk, uint32 core_num, uint32 wait,
+		uint32 mem_addr)
+{
+	int ret = 0;
+
+	if (bus->dhd == NULL) {
+		DHD_ERROR(("%s: bus not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	if (bus->dhd->prot == NULL) {
+		DHD_ERROR(("%s: prot is not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	if (bus->dhd->busstate != DHD_BUS_DATA) {
+		DHD_ERROR(("%s: not in a readystate to LPBK  is not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (len < 5 || len > 4194296) {
+		DHD_ERROR(("%s: len is too small or too large\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	len = dhd_apply_d11_war_length(bus, len, d11_lpbk);
+
+	bus->dmaxfer_complete = FALSE;
+	ret = dhdmsgbuf_dmaxfer_req(bus->dhd, len, srcdelay, destdelay,
+		d11_lpbk, core_num, mem_addr);
+	if (ret != BCME_OK || !wait) {
+		DHD_INFO(("%s: dmaxfer req returns status %u; wait = %u\n", __FUNCTION__,
+				ret, wait));
+	} else {
+		ret = dhd_os_dmaxfer_wait(bus->dhd, &bus->dmaxfer_complete);
+		if (ret < 0)
+			ret = BCME_NOTREADY;
+	}
+
+	return ret;
+
+}
+
+bool
+dhd_bus_is_multibp_capable(struct dhd_bus *bus)
+{
+	return MULTIBP_CAP(bus->sih);
+}
+
+#define PCIE_REV_FOR_4378A0	66	/* dhd_bus_perform_flr_with_quiesce() causes problems */
+#define PCIE_REV_FOR_4378B0	68
+
+static int
+dhdpcie_bus_download_state(dhd_bus_t *bus, bool enter)
+{
+	int bcmerror = 0;
+	volatile uint32 *cr4_regs;
+	bool do_flr;
+	bool do_wr_flops = TRUE;
+
+	if (!bus->sih) {
+		DHD_ERROR(("%s: NULL sih!!\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	do_flr = ((bus->sih->buscorerev != PCIE_REV_FOR_4378A0) &&
+			(bus->sih->buscorerev != PCIE_REV_FOR_4378B0));
+
+	/*
+	 * Jira SWWLAN-214966: 4378B0 BToverPCIe: fails to download firmware
+	 *   with "insmod dhd.ko firmware_path=rtecdc.bin nvram_path=nvram.txt" format
+	 *   CTO is seen during autoload case.
+	 * Need to assert PD1 power req during ARM out of reset.
+	 * And doing FLR after this would conflict as FLR resets PCIe enum space.
+	 */
+	if (MULTIBP_ENAB(bus->sih) && !do_flr) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	/* To enter download state, disable ARM and reset SOCRAM.
+	 * To exit download state, simply reset ARM (default is RAM boot).
+	 */
+	if (enter) {
+#ifndef BCMQT	/* for performance reasons, skip the FLR for QT */
+#endif /* !BCMQT */
+
+		/* Make sure BAR1 maps to backplane address 0 */
+		dhdpcie_setbar1win(bus, 0x00000000);
+		bus->alp_only = TRUE;
+#ifdef GDB_PROXY
+		bus->gdb_proxy_access_enabled = TRUE;
+		bus->gdb_proxy_bootloader_mode = FALSE;
+#endif /* GDB_PROXY */
+
+		/* some chips (e.g. 43602) have two ARM cores, the CR4 is receives the firmware. */
+		cr4_regs = si_setcore(bus->sih, ARMCR4_CORE_ID, 0);
+
+		if (cr4_regs == NULL && !(si_setcore(bus->sih, ARM7S_CORE_ID, 0)) &&
+		    !(si_setcore(bus->sih, ARMCM3_CORE_ID, 0)) &&
+		    !(si_setcore(bus->sih, ARMCA7_CORE_ID, 0))) {
+			DHD_ERROR(("%s: Failed to find ARM core!\n", __FUNCTION__));
+			bcmerror = BCME_ERROR;
+			goto fail;
+		}
+
+		if (si_setcore(bus->sih, ARMCA7_CORE_ID, 0)) {
+			/* Halt ARM & remove reset */
+			si_core_reset(bus->sih, SICF_CPUHALT, SICF_CPUHALT);
+			if (!(si_setcore(bus->sih, SYSMEM_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find SYSMEM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+			si_core_reset(bus->sih, 0, 0);
+			/* reset last 4 bytes of RAM address. to be used for shared area */
+			dhdpcie_init_shared_addr(bus);
+		} else if (cr4_regs == NULL) { /* no CR4 present on chip */
+			si_core_disable(bus->sih, 0);
+
+			if (!(si_setcore(bus->sih, SOCRAM_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find SOCRAM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			si_core_reset(bus->sih, 0, 0);
+
+			/* Clear the top bit of memory */
+			if (bus->ramsize) {
+				uint32 zeros = 0;
+				if (dhdpcie_bus_membytes(bus, TRUE, bus->ramsize - 4,
+				                     (uint8*)&zeros, 4) < 0) {
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+			}
+		} else {
+			/* For CR4,
+			 * Halt ARM
+			 * Remove ARM reset
+			 * Read RAM base address [0x18_0000]
+			 * [next] Download firmware
+			 * [done at else] Populate the reset vector
+			 * [done at else] Remove ARM halt
+			*/
+			/* Halt ARM & remove reset */
+			si_core_reset(bus->sih, SICF_CPUHALT, SICF_CPUHALT);
+			if (BCM43602_CHIP(bus->sih->chip)) {
+				/* XXX CRWLARMCR4-53 43602a0 HW bug when banks are powered down */
+				W_REG(bus->pcie_mb_intr_osh, cr4_regs + ARMCR4REG_BANKIDX, 5);
+				W_REG(bus->pcie_mb_intr_osh, cr4_regs + ARMCR4REG_BANKPDA, 0);
+				W_REG(bus->pcie_mb_intr_osh, cr4_regs + ARMCR4REG_BANKIDX, 7);
+				W_REG(bus->pcie_mb_intr_osh, cr4_regs + ARMCR4REG_BANKPDA, 0);
+			}
+			/* reset last 4 bytes of RAM address. to be used for shared area */
+			dhdpcie_init_shared_addr(bus);
+		}
+	} else {
+		if (si_setcore(bus->sih, ARMCA7_CORE_ID, 0)) {
+			/* write vars */
+			if ((bcmerror = dhdpcie_bus_write_vars(bus))) {
+				DHD_ERROR(("%s: could not write vars to RAM\n", __FUNCTION__));
+				goto fail;
+			}
+			/* write random numbers to sysmem for the purpose of
+			 * randomizing heap address space.
+			 */
+			if ((bcmerror = dhdpcie_wrt_rnd(bus)) != BCME_OK) {
+				DHD_ERROR(("%s: Failed to get random seed to write to TCM !\n",
+					__FUNCTION__));
+				goto fail;
+			}
+
+#if defined(FW_SIGNATURE)
+			if ((bcmerror = dhdpcie_bus_download_fw_signature(bus, &do_wr_flops))
+				!= BCME_OK) {
+				goto fail;
+			}
+#endif /* FW_SIGNATURE */
+
+			if (do_wr_flops) {
+				uint32 resetinstr_data = 0;
+
+				/* switch back to arm core again */
+				if (!(si_setcore(bus->sih, ARMCA7_CORE_ID, 0))) {
+					DHD_ERROR(("%s: Failed to find ARM CA7 core!\n",
+						__FUNCTION__));
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+
+				bcmerror = dhd_bus_get_security_status(bus, &resetinstr_data);
+				if (bcmerror == BCME_UNSUPPORTED) {
+					/*
+					 * read address 0 with reset instruction,
+					 * to validate that is not secured
+					 */
+					bcmerror = dhdpcie_bus_membytes(bus, FALSE, 0,
+						(uint8 *)&resetinstr_data, sizeof(resetinstr_data));
+				}
+				if ((resetinstr_data == 0xFFFFFFFF) ||
+					(resetinstr_data & DAR_SEC_SBOOT_MASK)) {
+					DHD_ERROR(("%s: **** FLOPS Vector is secured, "
+						"Signature file is missing! ***\n", __FUNCTION__));
+					bcmerror = BCME_NO_SIG_FILE;
+					goto fail;
+				}
+				/* write address 0 with reset instruction */
+				bcmerror = dhdpcie_bus_membytes(bus, TRUE, 0,
+					(uint8 *)&bus->resetinstr, sizeof(bus->resetinstr));
+				/* now remove reset and halt and continue to run CA7 */
+			}
+		} else if (!si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			if (!(si_setcore(bus->sih, SOCRAM_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find SOCRAM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			if (!si_iscoreup(bus->sih)) {
+				DHD_ERROR(("%s: SOCRAM core is down after reset?\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			/* Enable remap before ARM reset but after vars.
+			 * No backplane access in remap mode
+			 */
+			if (!si_setcore(bus->sih, PCMCIA_CORE_ID, 0) &&
+			    !si_setcore(bus->sih, SDIOD_CORE_ID, 0)) {
+				DHD_ERROR(("%s: Can't change back to SDIO core?\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			/* XXX Change standby configuration here if necessary */
+
+			if (!(si_setcore(bus->sih, ARM7S_CORE_ID, 0)) &&
+			    !(si_setcore(bus->sih, ARMCM3_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find ARM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+		} else {
+			if (BCM43602_CHIP(bus->sih->chip)) {
+				/* Firmware crashes on SOCSRAM access when core is in reset */
+				if (!(si_setcore(bus->sih, SOCRAM_CORE_ID, 0))) {
+					DHD_ERROR(("%s: Failed to find SOCRAM core!\n",
+						__FUNCTION__));
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+				si_core_reset(bus->sih, 0, 0);
+				si_setcore(bus->sih, ARMCR4_CORE_ID, 0);
+			}
+
+			/* write vars */
+			if ((bcmerror = dhdpcie_bus_write_vars(bus))) {
+				DHD_ERROR(("%s: could not write vars to RAM\n", __FUNCTION__));
+				goto fail;
+			}
+
+			/* write a random number rTLV to TCM for the purpose of
+			 * randomizing heap address space.
+			 */
+			if ((bcmerror = dhdpcie_wrt_rnd(bus)) != BCME_OK) {
+				DHD_ERROR(("%s: Failed to get random seed to write to TCM !\n",
+					__FUNCTION__));
+				goto fail;
+			}
+
+#if defined(FW_SIGNATURE)
+			if ((bcmerror = dhdpcie_bus_download_fw_signature(bus, &do_wr_flops))
+				!= BCME_OK) {
+				goto fail;
+			}
+#endif /* FW_SIGNATURE */
+			if (do_wr_flops) {
+				uint32 resetinstr_data;
+
+				/* switch back to arm core again */
+				if (!(si_setcore(bus->sih, ARMCR4_CORE_ID, 0))) {
+					DHD_ERROR(("%s: Failed to find ARM CR4 core!\n",
+						__FUNCTION__));
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+
+				/*
+				 * read address 0 with reset instruction,
+				 * to validate that is not secured
+				 */
+				bcmerror = dhdpcie_bus_membytes(bus, FALSE, 0,
+					(uint8 *)&resetinstr_data, sizeof(resetinstr_data));
+
+				if (resetinstr_data == 0xFFFFFFFF) {
+					DHD_ERROR(("%s: **** FLOPS Vector is secured, "
+						"Signature file is missing! ***\n", __FUNCTION__));
+					bcmerror = BCME_NO_SIG_FILE;
+					goto fail;
+				}
+
+				/* write address 0 with reset instruction */
+				bcmerror = dhdpcie_bus_membytes(bus, TRUE, 0,
+					(uint8 *)&bus->resetinstr, sizeof(bus->resetinstr));
+
+				if (bcmerror == BCME_OK) {
+					uint32 tmp;
+
+					bcmerror = dhdpcie_bus_membytes(bus, FALSE, 0,
+					                                (uint8 *)&tmp, sizeof(tmp));
+
+					if (bcmerror == BCME_OK && tmp != bus->resetinstr) {
+						DHD_ERROR(("%s: Failed to write 0x%08x to addr 0\n",
+						          __FUNCTION__, bus->resetinstr));
+						DHD_ERROR(("%s: contents of addr 0 is 0x%08x\n",
+						          __FUNCTION__, tmp));
+						bcmerror = BCME_ERROR;
+						goto fail;
+					}
+				}
+				/* now remove reset and halt and continue to run CR4 */
+			}
+		}
+
+		bus->arm_oor_time = OSL_LOCALTIME_NS();
+		si_core_reset(bus->sih, 0, 0);
+
+		DHD_ERROR(("%s: Took ARM out of Reset\n", __FUNCTION__));
+
+		/* Allow HT Clock now that the ARM is running. */
+		bus->alp_only = FALSE;
+
+		bus->dhd->busstate = DHD_BUS_LOAD;
+	}
+
+fail:
+	/* Always return to PCIE core */
+	si_setcore(bus->sih, PCIE2_CORE_ID, 0);
+
+	if (MULTIBP_ENAB(bus->sih) && !do_flr) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+
+	return bcmerror;
+} /* dhdpcie_bus_download_state */
+
+#if defined(FW_SIGNATURE)
+
+static int
+dhdpcie_bus_download_fw_signature(dhd_bus_t *bus, bool *do_write)
+{
+	int bcmerror = BCME_OK;
+
+	DHD_INFO(("FWSIG: bl=%s,%x fw=%x,%u sig=%s,%x,%u"
+		" stat=%x,%u ram=%x,%x\n",
+		bus->bootloader_filename, bus->bootloader_addr,
+		bus->fw_download_addr, bus->fw_download_len,
+		bus->fwsig_filename, bus->fwsig_download_addr,
+		bus->fwsig_download_len,
+		bus->fwstat_download_addr, bus->fwstat_download_len,
+		bus->dongle_ram_base, bus->ramtop_addr));
+
+	if (bus->fwsig_filename[0] == 0) {
+		DHD_INFO(("%s: missing signature file\n", __FUNCTION__));
+		goto exit;
+	}
+
+	/* Write RAM Bootloader to TCM if requested */
+	if ((bcmerror = dhdpcie_bus_download_ram_bootloader(bus))
+		!= BCME_OK) {
+		DHD_ERROR(("%s: could not write RAM BL to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write FW signature to memory */
+	if ((bcmerror = dhdpcie_bus_write_fw_signature(bus))) {
+		DHD_ERROR(("%s: could not write FWsig , err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* In case of BL RAM, do write flops */
+	if (bus->bootloader_filename[0] != 0) {
+		*do_write = TRUE;
+	} else {
+		*do_write = FALSE;
+	}
+
+exit:
+	return bcmerror;
+}
+
+/* Complete RAM structure, write the followings
+ * signature image
+ * signature verification status
+ * FW memory map
+ * end-of-TLVs marker
+ */
+static int
+dhdpcie_bus_write_fw_signature(dhd_bus_t *bus)
+{
+	int bcmerror = BCME_OK;
+
+	/* Write FW signature rTLV to TCM */
+	if ((bcmerror = dhdpcie_bus_write_fwsig(bus, bus->fwsig_filename,
+		NULL))) {
+		DHD_ERROR(("%s: could not write FWsig to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write FW signature verification status rTLV to TCM */
+	if ((bcmerror = dhdpcie_bus_write_fws_status(bus)) != BCME_OK) {
+		DHD_ERROR(("%s: could not write FWinfo to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write FW memory map rTLV to TCM */
+	if ((bcmerror = dhdpcie_bus_write_fws_mem_info(bus)) != BCME_OK) {
+		DHD_ERROR(("%s: could not write FWinfo to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write a end-of-TLVs marker to TCM */
+	if ((bcmerror = dhdpcie_download_rtlv_end(bus)) != BCME_OK) {
+		DHD_ERROR(("%s: could not write rTLV-end marker to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+exit:
+	return bcmerror;
+}
+#endif /* defined(FW_SIGNATURE) */
+
+/* Download a reversed-TLV to the top of dongle RAM without overlapping any existing rTLVs */
+static int
+dhdpcie_download_rtlv(dhd_bus_t *bus, dngl_rtlv_type_t type, dngl_rtlv_len_t len, uint8 *value)
+{
+	int bcmerror = BCME_OK;
+#ifdef DHD_DEBUG
+	uint8 *readback_buf = NULL;
+	uint32 readback_val = 0;
+#endif /* DHD_DEBUG */
+	uint32 dest_addr = 0;		/* dongle RAM dest address */
+	uint32 dest_size = 0;		/* dongle RAM dest size */
+	uint32 dest_raw_size = 0;	/* dest size with added checksum */
+
+	/* Calculate the destination dongle RAM address and size */
+	dest_size = ROUNDUP(len, 4);
+	dest_addr = bus->ramtop_addr - sizeof(dngl_rtlv_type_t) - sizeof(dngl_rtlv_len_t)
+		- dest_size;
+	bus->ramtop_addr = dest_addr;
+
+	/* Create the rTLV size field.  This consists of 2 16-bit fields:
+	 * The lower 16 bits is the size.  The higher 16 bits is a checksum
+	 * consisting of the size with all bits reversed.
+	 *     +-------------+-------------+
+	 *     |   checksum  |   size      |
+	 *     +-------------+-------------+
+	 *      High 16 bits    Low 16 bits
+	 */
+	if (type == DNGL_RTLV_TYPE_RNG_SIGNATURE) {
+		/* Random number seed TLV (DNGL_RTLV_TYPE_RNG_SIGNATURE) was the very
+		 * first TLV, its length does not have checksum, firmware crashes if
+		 * checksum is present
+		 */
+		dest_raw_size = dest_size & 0x0000FFFF;
+	} else {
+		dest_raw_size = (~dest_size << 16) | (dest_size & 0x0000FFFF);
+	}
+	/* Write the value block */
+	if (dest_size > 0) {
+		bcmerror = dhdpcie_bus_membytes(bus, TRUE, dest_addr, value, dest_size);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes to 0x%08x\n",
+				__FUNCTION__, bcmerror, dest_size, dest_addr));
+			goto exit;
+		}
+	}
+
+	/* Write the length word */
+	bcmerror = dhdpcie_bus_membytes(bus, TRUE, dest_addr + dest_size,
+		(uint8*)&dest_raw_size, sizeof(dngl_rtlv_len_t));
+
+	/* Write the type word */
+	bcmerror = dhdpcie_bus_membytes(bus, TRUE,
+		dest_addr + dest_size + sizeof(dngl_rtlv_len_t),
+		(uint8*)&type, sizeof(dngl_rtlv_type_t));
+
+#ifdef DHD_DEBUG
+	/* Read back and compare the downloaded data */
+	if (dest_size > 0) {
+		readback_buf = (uint8*)MALLOC(bus->dhd->osh, dest_size);
+		if (!readback_buf) {
+			bcmerror = BCME_NOMEM;
+			goto exit;
+		}
+		memset(readback_buf, 0xaa, dest_size);
+		bcmerror = dhdpcie_bus_membytes(bus, FALSE, dest_addr, readback_buf, dest_size);
+		if (bcmerror) {
+			DHD_ERROR(("%s: readback error %d, %d bytes from 0x%08x\n",
+				__FUNCTION__, bcmerror, dest_size, dest_addr));
+			goto exit;
+		}
+		if (memcmp(value, readback_buf, dest_size) != 0) {
+			DHD_ERROR(("%s: Downloaded data mismatch.\n", __FUNCTION__));
+			bcmerror = BCME_ERROR;
+			goto exit;
+		} else {
+			DHD_ERROR(("Download and compare of TLV 0x%x succeeded"
+				" (size %u, addr %x).\n", type, dest_size, dest_addr));
+		}
+	}
+
+	/* Read back and compare the downloaded len field */
+	bcmerror = dhdpcie_bus_membytes(bus, FALSE, dest_addr + dest_size,
+		(uint8*)&readback_val, sizeof(dngl_rtlv_len_t));
+	if (!bcmerror) {
+		if (readback_val != dest_raw_size) {
+			bcmerror = BCME_BADLEN;
+		}
+	}
+	if (bcmerror) {
+		DHD_ERROR(("%s: Downloaded len error %d\n", __FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Read back and compare the downloaded type field */
+	bcmerror = dhdpcie_bus_membytes(bus, FALSE,
+		dest_addr + dest_size + sizeof(dngl_rtlv_len_t),
+		(uint8*)&readback_val, sizeof(dngl_rtlv_type_t));
+	if (!bcmerror) {
+		if (readback_val != type) {
+			bcmerror = BCME_BADOPTION;
+		}
+	}
+	if (bcmerror) {
+		DHD_ERROR(("%s: Downloaded type error %d\n", __FUNCTION__, bcmerror));
+		goto exit;
+	}
+#endif /* DHD_DEBUG */
+
+	bus->ramtop_addr = dest_addr;
+
+exit:
+#ifdef DHD_DEBUG
+	if (readback_buf) {
+		MFREE(bus->dhd->osh, readback_buf, dest_size);
+	}
+#endif /* DHD_DEBUG */
+
+	return bcmerror;
+} /* dhdpcie_download_rtlv */
+
+#if defined(FW_SIGNATURE)
+/* Download a reversed-TLV END marker to the top of dongle RAM */
+static int
+dhdpcie_download_rtlv_end(dhd_bus_t *bus)
+{
+	return dhdpcie_download_rtlv(bus, DNGL_RTLV_TYPE_END_MARKER, 0, NULL);
+}
+
+/* Write the FW signature verification status to dongle memory */
+static int
+dhdpcie_bus_write_fws_status(dhd_bus_t *bus)
+{
+	bl_verif_status_t vstatus;
+	int ret;
+
+	bzero(&vstatus, sizeof(vstatus));
+
+	ret = dhdpcie_download_rtlv(bus, DNGL_RTLV_TYPE_FWSIGN_STATUS, sizeof(vstatus),
+		(uint8*)&vstatus);
+	bus->fwstat_download_addr = bus->ramtop_addr;
+	bus->fwstat_download_len = sizeof(vstatus);
+
+	return ret;
+} /* dhdpcie_bus_write_fws_status */
+
+/* Write the FW signature verification memory map to dongle memory */
+static int
+dhdpcie_bus_write_fws_mem_info(dhd_bus_t *bus)
+{
+	bl_mem_info_t memmap;
+	int ret;
+
+	bzero(&memmap, sizeof(memmap));
+	memmap.firmware.start = bus->fw_download_addr;
+	memmap.firmware.end = memmap.firmware.start + bus->fw_download_len;
+	memmap.heap.start = ROUNDUP(memmap.firmware.end + BL_HEAP_START_GAP_SIZE, 4);
+	memmap.heap.end = memmap.heap.start + BL_HEAP_SIZE;
+	memmap.signature.start = bus->fwsig_download_addr;
+	memmap.signature.end = memmap.signature.start + bus->fwsig_download_len;
+	memmap.vstatus.start = bus->fwstat_download_addr;
+	memmap.vstatus.end = memmap.vstatus.start + bus->fwstat_download_len;
+	DHD_INFO(("%s: mem_info: fw=%x-%x heap=%x-%x sig=%x-%x vst=%x-%x res=%x\n",
+		__FUNCTION__,
+		memmap.firmware.start, memmap.firmware.end,
+		memmap.heap.start, memmap.heap.end,
+		memmap.signature.start, memmap.signature.end,
+		memmap.vstatus.start, memmap.vstatus.end,
+		memmap.reset_vec.start));
+
+	ret = dhdpcie_download_rtlv(bus, DNGL_RTLV_TYPE_FWSIGN_MEM_MAP, sizeof(memmap),
+		(uint8*)&memmap);
+	bus->fw_memmap_download_addr = bus->ramtop_addr;
+	bus->fw_memmap_download_len = sizeof(memmap);
+
+	return ret;
+} /* dhdpcie_bus_write_fws_mem_info */
+
+/* Download a bootloader image to dongle RAM */
+static int
+dhdpcie_bus_download_ram_bootloader(dhd_bus_t *bus)
+{
+	int ret = BCME_OK;
+	uint32 dongle_ram_base_save;
+
+	DHD_INFO(("download_bloader: %s,0x%x. ramtop=0x%x\n",
+		bus->bootloader_filename, bus->bootloader_addr, bus->ramtop_addr));
+	if (bus->bootloader_filename[0] == '\0') {
+		return ret;
+	}
+
+	/* Save ram base */
+	dongle_ram_base_save = bus->dongle_ram_base;
+
+	/* Set ram base to bootloader download start address */
+	bus->dongle_ram_base = bus->bootloader_addr;
+
+	/* Download the bootloader image to TCM */
+	ret = dhdpcie_download_code_file(bus, bus->bootloader_filename);
+
+	/* Restore ram base */
+	bus->dongle_ram_base = dongle_ram_base_save;
+
+	return ret;
+} /* dhdpcie_bus_download_ram_bootloader */
+
+/* Save the FW download address and size */
+static int
+dhdpcie_bus_save_download_info(dhd_bus_t *bus, uint32 download_addr,
+	uint32 download_size, const char *signature_fname,
+	const char *bloader_fname, uint32 bloader_download_addr)
+{
+	/* DHD EXE may not have size of the downloaded FW image in case of swpaging binary */
+	if (download_size) {
+		bus->fw_download_len = download_size;
+	}
+	bus->fw_download_addr = download_addr;
+	strlcpy(bus->fwsig_filename, signature_fname, sizeof(bus->fwsig_filename));
+	strlcpy(bus->bootloader_filename, bloader_fname, sizeof(bus->bootloader_filename));
+	bus->bootloader_addr = bloader_download_addr;
+#ifdef GDB_PROXY
+	/* GDB proxy bootloader mode - if signature file specified (i.e.
+	 * bootloader is used), but bootloader is not specified (i.e. ROM
+	 * bootloader is uses).
+	 * Bootloader mode is significant only for for preattachment debugging
+	 * of chips, in which debug cell can't be initialized before ARM CPU
+	 * start
+	 */
+	bus->gdb_proxy_bootloader_mode = bus->fwsig_filename[0] != 0;
+#endif /* GDB_PROXY */
+	return BCME_OK;
+} /* dhdpcie_bus_save_download_info */
+
+/* Read a binary file and write it to the specified socram dest address */
+#ifdef DHD_LINUX_STD_FW_API
+static int
+dhdpcie_download_sig_file(dhd_bus_t *bus, char *path, uint32 type)
+{
+	int bcmerror = BCME_ERROR;
+	int srcsize = 0;
+	uint32 dest_size = 0;	/* dongle RAM dest size */
+	const struct firmware *sig = NULL;
+
+	if (path == NULL || path[0] == '\0') {
+		DHD_ERROR(("%s: no file\n", __FUNCTION__));
+		bcmerror = BCME_NOTFOUND;
+		goto exit;
+	}
+
+	bcmerror = dhd_os_get_img_fwreq(&sig, bus->fwsig_filename);
+	if (bcmerror < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			bcmerror));
+		goto exit;
+	}
+	DHD_ERROR(("%s: dhd_os_get_img(Request Firmware API) success. size %d.\n",
+		__func__, sig->size));
+
+	srcsize = sig->size;
+	if (srcsize <= 0 || srcsize > MEMBLOCK) {
+		DHD_ERROR(("%s: invalid fwsig size %u\n", __FUNCTION__, srcsize));
+		bcmerror = BCME_BUFTOOSHORT;
+		goto exit;
+	}
+
+	dest_size = ROUNDUP(srcsize, 4);
+	/* Write the src buffer as a rTLV to the dongle */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	bcmerror = dhdpcie_download_rtlv(bus, type, dest_size, (uint8 *)sig->data);
+	GCC_DIAGNOSTIC_POP();
+	if (bcmerror) {
+		DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			__FUNCTION__, bcmerror, srcsize, bus->ramtop_addr));
+		goto exit;
+	}
+
+	bus->fwsig_download_addr = bus->ramtop_addr;
+	bus->fwsig_download_len = dest_size;
+exit:
+	if (sig) {
+		dhd_os_close_img_fwreq(sig);
+	}
+
+	return bcmerror;
+}
+#else
+static int
+dhdpcie_download_sig_file(dhd_bus_t *bus, char *path, uint32 type)
+{
+	int bcmerror = BCME_OK;
+	void *filep = NULL;
+	uint8 *srcbuf = NULL;
+	int srcsize = 0;
+	int len;
+	uint32 dest_size = 0;	/* dongle RAM dest size */
+	fwpkg_info_t *fwpkg = NULL;
+
+	if (path == NULL || path[0] == '\0') {
+		DHD_ERROR(("%s: no file\n", __FUNCTION__));
+		bcmerror = BCME_NOTFOUND;
+		goto exit;
+	}
+
+	bcmerror = fwpkg_init(&bus->fwpkg, path);
+	if (bcmerror == BCME_ERROR) {
+		goto exit;
+	}
+	fwpkg = &bus->fwpkg;
+
+	/* Open file, get size */
+	bcmerror = fwpkg_open_signature_img(fwpkg, path, &filep);
+	if (bcmerror == BCME_ERROR) {
+		DHD_ERROR(("%s: error opening file %s\n", __FUNCTION__, path));
+		goto exit;
+	}
+
+	srcsize = fwpkg_get_signature_img_size(fwpkg);
+
+	if (srcsize <= 0 || srcsize > MEMBLOCK) {
+		DHD_ERROR(("%s: invalid fwsig size %u\n", __FUNCTION__, srcsize));
+		bcmerror = BCME_BUFTOOSHORT;
+		goto exit;
+	}
+	dest_size = ROUNDUP(srcsize, 4);
+
+	/* Allocate src buffer, read in the entire file */
+	srcbuf = (uint8 *)MALLOCZ(bus->dhd->osh, dest_size);
+	if (!srcbuf) {
+		bcmerror = BCME_NOMEM;
+		goto exit;
+	}
+	len = dhd_os_get_image_block(srcbuf, srcsize, filep);
+	if (len != srcsize) {
+		DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+		bcmerror = BCME_BADLEN;
+		goto exit;
+	}
+
+	/* Write the src buffer as a rTLV to the dongle */
+	bcmerror = dhdpcie_download_rtlv(bus, type, dest_size, srcbuf);
+
+	bus->fwsig_download_addr = bus->ramtop_addr;
+	bus->fwsig_download_len = dest_size;
+
+exit:
+	if (filep) {
+		dhd_os_close_image1(bus->dhd, filep);
+	}
+	if (srcbuf) {
+		MFREE(bus->dhd->osh, srcbuf, dest_size);
+	}
+
+	return bcmerror;
+} /* dhdpcie_download_sig_file */
+#endif /* DHD_LINUX_STD_FW_API */
+
+static int
+dhdpcie_bus_write_fwsig(dhd_bus_t *bus, char *fwsig_path, char *nvsig_path)
+{
+	int bcmerror = BCME_OK;
+
+	/* Download the FW signature file to the chip */
+	bcmerror = dhdpcie_download_sig_file(bus, fwsig_path, DNGL_RTLV_TYPE_FW_SIGNATURE);
+	if (bcmerror) {
+		goto exit;
+	}
+
+exit:
+	if (bcmerror) {
+		DHD_ERROR(("%s: error %d\n", __FUNCTION__, bcmerror));
+	}
+	return bcmerror;
+} /* dhdpcie_bus_write_fwsig */
+
+static int
+dhdpcie_read_fwstatus(dhd_bus_t *bus, bl_verif_status_t *status)
+{
+	int ret = BCME_OK;
+
+	bzero(status, sizeof(*status));
+	if (bus->fwstat_download_addr != 0) {
+		ret = dhdpcie_bus_membytes(bus, FALSE, bus->fwstat_download_addr,
+			(uint8 *)status, sizeof(*status));
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: error %d on reading %zu membytes at 0x%08x\n",
+				__FUNCTION__, ret, sizeof(*status), bus->fwstat_download_addr));
+		}
+	}
+
+	return ret;
+}
+
+/* Dump secure firmware status. */
+static int
+dhd_bus_dump_fws(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	bl_verif_status_t status;
+	bl_mem_info_t     meminfo;
+	int               err = BCME_OK;
+
+	err = dhdpcie_read_fwstatus(bus, &status);
+	if (err != BCME_OK) {
+		return (err);
+	}
+
+	bzero(&meminfo, sizeof(meminfo));
+	if (bus->fw_memmap_download_addr != 0) {
+		err = dhdpcie_bus_membytes(bus, FALSE, bus->fw_memmap_download_addr,
+			(uint8 *)&meminfo, sizeof(meminfo));
+		if (err != BCME_OK) {
+			DHD_ERROR(("%s: error %d on reading %zu membytes at 0x%08x\n",
+				__FUNCTION__, err, sizeof(meminfo), bus->fw_memmap_download_addr));
+			return (err);
+		}
+	}
+
+	bcm_bprintf(strbuf, "Firmware signing\nSignature: (%08x) len (%d)\n",
+		bus->fwsig_download_addr, bus->fwsig_download_len);
+
+	bcm_bprintf(strbuf,
+		"Verification status: (%08x)\n"
+		"\tstatus: %d\n"
+		"\tstate: %u\n"
+		"\talloc_bytes: %u\n"
+		"\tmax_alloc_bytes: %u\n"
+		"\ttotal_alloc_bytes: %u\n"
+		"\ttotal_freed_bytes: %u\n"
+		"\tnum_allocs: %u\n"
+		"\tmax_allocs: %u\n"
+		"\tmax_alloc_size: %u\n"
+		"\talloc_failures: %u\n",
+		bus->fwstat_download_addr,
+		status.status,
+		status.state,
+		status.alloc_bytes,
+		status.max_alloc_bytes,
+		status.total_alloc_bytes,
+		status.total_freed_bytes,
+		status.num_allocs,
+		status.max_allocs,
+		status.max_alloc_size,
+		status.alloc_failures);
+
+	bcm_bprintf(strbuf,
+		"Memory info: (%08x)\n"
+		"\tfw   %08x-%08x\n\theap %08x-%08x\n\tsig  %08x-%08x\n\tvst  %08x-%08x\n",
+		bus->fw_memmap_download_addr,
+		meminfo.firmware.start,  meminfo.firmware.end,
+		meminfo.heap.start,      meminfo.heap.end,
+		meminfo.signature.start, meminfo.signature.end,
+		meminfo.vstatus.start,   meminfo.vstatus.end);
+
+	return (err);
+}
+#endif /* FW_SIGNATURE */
+
+/* Write nvram data to the top of dongle RAM, ending with a size in # of 32-bit words */
+static int
+dhdpcie_bus_write_vars(dhd_bus_t *bus)
+{
+	int bcmerror = 0;
+	uint32 varsize, phys_size;
+	uint32 varaddr;
+	uint8 *vbuffer;
+	uint32 varsizew;
+	uint32 rd_varsizew;
+#ifdef DHD_DEBUG
+	uint8 *nvram_ularray;
+#endif /* DHD_DEBUG */
+
+	/* Even if there are no vars are to be written, we still need to set the ramsize. */
+	varsize = bus->varsz ? ROUNDUP(bus->varsz, 4) : 0;
+	varaddr = (bus->ramsize - 4) - varsize;
+
+	varaddr += bus->dongle_ram_base;
+	bus->ramtop_addr = varaddr;
+
+	if (bus->vars) {
+
+		/* XXX In case the controller has trouble with odd bytes... */
+		vbuffer = (uint8 *)MALLOC(bus->dhd->osh, varsize);
+		if (!vbuffer)
+			return BCME_NOMEM;
+
+		bzero(vbuffer, varsize);
+		bcopy(bus->vars, vbuffer, bus->varsz);
+		/* Write the vars list */
+		bcmerror = dhdpcie_bus_membytes(bus, TRUE, varaddr, vbuffer, varsize);
+
+		/* Implement read back and verify later */
+#ifdef DHD_DEBUG
+		/* Verify NVRAM bytes */
+		DHD_INFO(("%s: Compare NVRAM dl & ul; varsize=%d\n", __FUNCTION__, varsize));
+		nvram_ularray = (uint8*)MALLOC(bus->dhd->osh, varsize);
+		if (!nvram_ularray) {
+			MFREE(bus->dhd->osh, vbuffer, varsize);
+			return BCME_NOMEM;
+		}
+
+		/* Upload image to verify downloaded contents. */
+		memset(nvram_ularray, 0xaa, varsize);
+
+		/* Read the vars list to temp buffer for comparison */
+		bcmerror = dhdpcie_bus_membytes(bus, FALSE, varaddr, nvram_ularray, varsize);
+		if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d nvram bytes at 0x%08x\n",
+					__FUNCTION__, bcmerror, varsize, varaddr));
+		}
+
+		/* Compare the org NVRAM with the one read from RAM */
+		if (memcmp(vbuffer, nvram_ularray, varsize)) {
+			DHD_ERROR(("%s: Downloaded NVRAM image is corrupted.\n", __FUNCTION__));
+			prhex("nvram file", vbuffer, varsize);
+			prhex("downloaded nvram", nvram_ularray, varsize);
+			MFREE(bus->dhd->osh, nvram_ularray, varsize);
+			MFREE(bus->dhd->osh, vbuffer, varsize);
+			return BCME_ERROR;
+		} else
+			DHD_ERROR(("%s: Download, Upload and compare of NVRAM succeeded.\n",
+			__FUNCTION__));
+
+		MFREE(bus->dhd->osh, nvram_ularray, varsize);
+#endif /* DHD_DEBUG */
+
+		MFREE(bus->dhd->osh, vbuffer, varsize);
+	}
+
+	phys_size = REMAP_ENAB(bus) ? bus->ramsize : bus->orig_ramsize;
+
+	phys_size += bus->dongle_ram_base;
+
+	/* adjust to the user specified RAM */
+	DHD_INFO(("%s: Physical memory size: %d, usable memory size: %d\n", __FUNCTION__,
+		phys_size, bus->ramsize));
+	DHD_INFO(("%s: Vars are at %d, orig varsize is %d\n", __FUNCTION__,
+		varaddr, varsize));
+	varsize = ((phys_size - 4) - varaddr);
+
+	/*
+	 * Determine the length token:
+	 * Varsize, converted to words, in lower 16-bits, checksum in upper 16-bits.
+	 */
+	if (bcmerror) {
+		varsizew = 0;
+		bus->nvram_csm = varsizew;
+	} else {
+		varsizew = varsize / 4;
+		varsizew = (~varsizew << 16) | (varsizew & 0x0000FFFF);
+		bus->nvram_csm = varsizew;
+		varsizew = htol32(varsizew);
+	}
+
+	DHD_ERROR(("%s: New varsize is %d, length token(nvram_csm)=0x%08x\n", __FUNCTION__, varsize, varsizew));
+
+	/* Write the length token to the last word */
+	bcmerror = dhdpcie_bus_membytes(bus, TRUE, (phys_size - 4),
+		(uint8*)&varsizew, 4);
+
+	/* Read back the last word */
+	bcmerror = dhdpcie_bus_membytes(bus, FALSE, (phys_size - 4),
+		(uint8*)&rd_varsizew, 4);
+	if (rd_varsizew != varsizew) {
+		DHD_ERROR(("%s: readback nvram_csm(0x%08x:0x%08x) is wrong at 0x%x\n",
+			__FUNCTION__, rd_varsizew, varsizew, (phys_size - 4)));
+	}
+
+	return bcmerror;
+} /* dhdpcie_bus_write_vars */
+
+int
+dhdpcie_downloadvars(dhd_bus_t *bus, void *arg, int len)
+{
+	int bcmerror = BCME_OK;
+#ifdef KEEP_JP_REGREV
+	/* XXX Needed by customer's request */
+	char *tmpbuf;
+	uint tmpidx;
+#endif /* KEEP_JP_REGREV */
+#ifdef GDB_PROXY
+	const char nodeadman_record[] = "deadman_to=0";
+#endif /* GDB_PROXY */
+#ifdef DHD_SUPPORT_SPMI_MODE
+	char tag_spmi_mode[16] = {0};
+#endif /* DHD_SUPPORT_SPMI_MODE */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (!len) {
+		bcmerror = BCME_BUFTOOSHORT;
+		goto err;
+	}
+
+	/* Free the old ones and replace with passed variables */
+	if (bus->vars)
+		MFREE(bus->dhd->osh, bus->vars, bus->varsz);
+#ifdef GDB_PROXY
+	if (bus->dhd->gdb_proxy_nodeadman) {
+		len += sizeof(nodeadman_record);
+	}
+#endif /* GDB_PROXY */
+#ifdef DHD_SUPPORT_SPMI_MODE
+	if (bus->dhd->dhd_spmi_mode) {
+		snprintf(tag_spmi_mode, sizeof(tag_spmi_mode), "spmi_mode=%d\n",
+			bus->dhd->dhd_spmi_mode);
+		len += sizeof(tag_spmi_mode);
+	}
+#endif /* DHD_SUPPORT_SPMI_MODE */
+
+	bus->vars = MALLOC(bus->dhd->osh, len);
+	bus->varsz = bus->vars ? len : 0;
+	if (bus->vars == NULL) {
+		bcmerror = BCME_NOMEM;
+		goto err;
+	}
+
+	/* Copy the passed variables, which should include the terminating double-null */
+	bcopy(arg, bus->vars, bus->varsz);
+#ifdef GDB_PROXY
+	if (bus->dhd->gdb_proxy_nodeadman &&
+		!replace_nvram_variable(bus->vars, bus->varsz, nodeadman_record, NULL))
+	{
+		bcmerror = BCME_NOMEM;
+		goto err;
+	}
+#endif /* GDB_PROXY */
+#ifdef DHD_SUPPORT_SPMI_MODE
+	if (bus->dhd->dhd_spmi_mode) {
+		char *pos = NULL;
+		char tag_spmi[] = "spmi=";
+		uint spmi = 0;
+
+		/* check if the spmi feature is enabled */
+		pos = bcmstrnstr(bus->vars, bus->varsz, tag_spmi, strlen(tag_spmi));
+		if (pos) {
+			sscanf(pos, "spmi=%u\n", &spmi);
+			DHD_INFO(("%s: spmi feature is %s\n", __FUNCTION__,
+				spmi ? "enabled" : "disabled"));
+		}
+
+		if (spmi && !replace_nvram_variable(bus->vars,
+			bus->varsz, tag_spmi_mode, NULL)) {
+			bcmerror = BCME_NOMEM;
+			goto err;
+		}
+	}
+#endif /* DHD_SUPPORT_SPMI_MODE */
+
+	/* Re-Calculate htclkratio only for QT, for FPGA it is fixed at 30 */
+#ifdef BCMQT_HW
+	dhdpcie_htclkratio_recal(bus, bus->vars, bus->varsz);
+#endif
+
+#ifdef DHD_USE_SINGLE_NVRAM_FILE
+	/* XXX Change the default country code only for MFG firmware */
+	if (dhd_bus_get_fw_mode(bus->dhd) == DHD_FLAG_MFG_MODE) {
+		char *sp = NULL;
+		char *ep = NULL;
+		int i;
+		char tag[2][8] = {"ccode=", "regrev="};
+
+		/* Find ccode and regrev info */
+		for (i = 0; i < 2; i++) {
+			sp = strnstr(bus->vars, tag[i], bus->varsz);
+			if (!sp) {
+				DHD_ERROR(("%s: Could not find ccode info from the nvram %s\n",
+					__FUNCTION__, bus->nv_path));
+				bcmerror = BCME_ERROR;
+				goto err;
+			}
+			sp = strchr(sp, '=');
+			ep = strchr(sp, '\0');
+			/* We assumed that string length of both ccode and
+			 * regrev values should not exceed WLC_CNTRY_BUF_SZ
+			 */
+			if (ep && ((ep - sp) <= WLC_CNTRY_BUF_SZ)) {
+				sp++;
+				while (*sp != '\0') {
+					DHD_INFO(("%s: parse '%s', current sp = '%c'\n",
+						__FUNCTION__, tag[i], *sp));
+					*sp++ = '0';
+				}
+			} else {
+				DHD_ERROR(("%s: Invalid parameter format when parsing for %s\n",
+					__FUNCTION__, tag[i]));
+				bcmerror = BCME_ERROR;
+				goto err;
+			}
+		}
+	}
+#endif /* DHD_USE_SINGLE_NVRAM_FILE */
+
+#ifdef KEEP_JP_REGREV
+	/* XXX Needed by customer's request */
+#ifdef DHD_USE_SINGLE_NVRAM_FILE
+	if (dhd_bus_get_fw_mode(bus->dhd) != DHD_FLAG_MFG_MODE)
+#endif /* DHD_USE_SINGLE_NVRAM_FILE */
+	{
+		char *pos = NULL;
+		tmpbuf = MALLOCZ(bus->dhd->osh, bus->varsz + 1);
+		if (tmpbuf == NULL) {
+			goto err;
+		}
+		memcpy(tmpbuf, bus->vars, bus->varsz);
+		for (tmpidx = 0; tmpidx < bus->varsz; tmpidx++) {
+			if (tmpbuf[tmpidx] == 0) {
+				tmpbuf[tmpidx] = '\n';
+			}
+		}
+		bus->dhd->vars_ccode[0] = 0;
+		bus->dhd->vars_regrev = 0;
+		if ((pos = strstr(tmpbuf, "ccode"))) {
+			sscanf(pos, "ccode=%3s\n", bus->dhd->vars_ccode);
+		}
+		if ((pos = strstr(tmpbuf, "regrev"))) {
+			sscanf(pos, "regrev=%u\n", &(bus->dhd->vars_regrev));
+		}
+		MFREE(bus->dhd->osh, tmpbuf, bus->varsz + 1);
+	}
+#endif /* KEEP_JP_REGREV */
+
+err:
+	return bcmerror;
+}
+
+/* loop through the capability list and see if the pcie capabilty exists */
+uint8
+dhdpcie_find_pci_capability(osl_t *osh, uint8 req_cap_id)
+{
+	uint8 cap_id;
+	uint8 cap_ptr = 0;
+	uint8 byte_val;
+
+	/* check for Header type 0 */
+	byte_val = read_pci_cfg_byte(PCI_CFG_HDR);
+	if ((byte_val & 0x7f) != PCI_HEADER_NORMAL) {
+		DHD_ERROR(("%s : PCI config header not normal.\n", __FUNCTION__));
+		goto end;
+	}
+
+	/* check if the capability pointer field exists */
+	byte_val = read_pci_cfg_byte(PCI_CFG_STAT);
+	if (!(byte_val & PCI_CAPPTR_PRESENT)) {
+		DHD_ERROR(("%s : PCI CAP pointer not present.\n", __FUNCTION__));
+		goto end;
+	}
+
+	cap_ptr = read_pci_cfg_byte(PCI_CFG_CAPPTR);
+	/* check if the capability pointer is 0x00 */
+	if (cap_ptr == 0x00) {
+		DHD_ERROR(("%s : PCI CAP pointer is 0x00.\n", __FUNCTION__));
+		goto end;
+	}
+
+	/* loop thr'u the capability list and see if the pcie capabilty exists */
+
+	cap_id = read_pci_cfg_byte(cap_ptr);
+
+	while (cap_id != req_cap_id) {
+		cap_ptr = read_pci_cfg_byte((cap_ptr + 1));
+		if (cap_ptr == 0x00) break;
+		cap_id = read_pci_cfg_byte(cap_ptr);
+	}
+
+end:
+	return cap_ptr;
+}
+
+void
+dhdpcie_pme_active(osl_t *osh, bool enable)
+{
+	uint8 cap_ptr;
+	uint32 pme_csr;
+
+	cap_ptr = dhdpcie_find_pci_capability(osh, PCI_CAP_POWERMGMTCAP_ID);
+
+	if (!cap_ptr) {
+		DHD_ERROR(("%s : Power Management Capability not present\n", __FUNCTION__));
+		return;
+	}
+
+	pme_csr = OSL_PCI_READ_CONFIG(osh, cap_ptr + PME_CSR_OFFSET, sizeof(uint32));
+	DHD_ERROR(("%s : pme_sts_ctrl 0x%x\n", __FUNCTION__, pme_csr));
+
+	pme_csr |= PME_CSR_PME_STAT;
+	if (enable) {
+		pme_csr |= PME_CSR_PME_EN;
+	} else {
+		pme_csr &= ~PME_CSR_PME_EN;
+	}
+
+	OSL_PCI_WRITE_CONFIG(osh, cap_ptr + PME_CSR_OFFSET, sizeof(uint32), pme_csr);
+}
+
+bool
+dhdpcie_pme_cap(osl_t *osh)
+{
+	uint8 cap_ptr;
+	uint32 pme_cap;
+
+	cap_ptr = dhdpcie_find_pci_capability(osh, PCI_CAP_POWERMGMTCAP_ID);
+
+	if (!cap_ptr) {
+		DHD_ERROR(("%s : Power Management Capability not present\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	pme_cap = OSL_PCI_READ_CONFIG(osh, cap_ptr, sizeof(uint32));
+
+	DHD_ERROR(("%s : pme_cap 0x%x\n", __FUNCTION__, pme_cap));
+
+	return ((pme_cap & PME_CAP_PM_STATES) != 0);
+}
+
+static void
+dhdpcie_pme_stat_clear(dhd_bus_t *bus)
+{
+	uint32 pmcsr = dhd_pcie_config_read(bus, PCIE_CFG_PMCSR, sizeof(uint32));
+
+	OSL_PCI_WRITE_CONFIG(bus->osh, PCIE_CFG_PMCSR, sizeof(uint32), pmcsr | PCIE_PMCSR_PMESTAT);
+}
+
+uint32
+dhdpcie_lcreg(osl_t *osh, uint32 mask, uint32 val)
+{
+
+	uint8	pcie_cap;
+	uint8	lcreg_offset;	/* PCIE capability LCreg offset in the config space */
+	uint32	reg_val;
+
+	pcie_cap = dhdpcie_find_pci_capability(osh, PCI_CAP_PCIECAP_ID);
+
+	if (!pcie_cap) {
+		DHD_ERROR(("%s : PCIe Capability not present\n", __FUNCTION__));
+		return 0;
+	}
+
+	lcreg_offset = pcie_cap + PCIE_CAP_LINKCTRL_OFFSET;
+
+	/* set operation */
+	if (mask) {
+		/* read */
+		reg_val = OSL_PCI_READ_CONFIG(osh, lcreg_offset, sizeof(uint32));
+
+		/* modify */
+		reg_val &= ~mask;
+		reg_val |= (mask & val);
+
+		/* write */
+		OSL_PCI_WRITE_CONFIG(osh, lcreg_offset, sizeof(uint32), reg_val);
+	}
+	return OSL_PCI_READ_CONFIG(osh, lcreg_offset, sizeof(uint32));
+}
+
+#if defined(NDIS)
+/* set min res mask to highest value, preventing sleep */
+void
+dhdpcie_set_pmu_min_res_mask(struct dhd_bus *bus, uint min_res_mask)
+{
+	si_pmu_set_min_res_mask(bus->sih, bus->osh, min_res_mask);
+}
+#endif /* defined(NDIS) */
+
+uint8
+dhdpcie_clkreq(osl_t *osh, uint32 mask, uint32 val)
+{
+	uint8	pcie_cap;
+	uint32	reg_val;
+	uint8	lcreg_offset;	/* PCIE capability LCreg offset in the config space */
+
+	pcie_cap = dhdpcie_find_pci_capability(osh, PCI_CAP_PCIECAP_ID);
+
+	if (!pcie_cap) {
+		DHD_ERROR(("%s : PCIe Capability not present\n", __FUNCTION__));
+		return 0;
+	}
+
+	lcreg_offset = pcie_cap + PCIE_CAP_LINKCTRL_OFFSET;
+
+	reg_val = OSL_PCI_READ_CONFIG(osh, lcreg_offset, sizeof(uint32));
+	/* set operation */
+	if (mask) {
+		if (val)
+			reg_val |= PCIE_CLKREQ_ENAB;
+		else
+			reg_val &= ~PCIE_CLKREQ_ENAB;
+		OSL_PCI_WRITE_CONFIG(osh, lcreg_offset, sizeof(uint32), reg_val);
+		reg_val = OSL_PCI_READ_CONFIG(osh, lcreg_offset, sizeof(uint32));
+	}
+	if (reg_val & PCIE_CLKREQ_ENAB)
+		return 1;
+	else
+		return 0;
+}
+
+void dhd_dump_intr_counters(dhd_pub_t *dhd, struct bcmstrbuf *strbuf)
+{
+	dhd_bus_t *bus;
+	uint64 current_time = OSL_LOCALTIME_NS();
+
+	if (!dhd) {
+		DHD_ERROR(("%s: dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	bus = dhd->bus;
+	if (!bus) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	bcm_bprintf(strbuf, "\n ------- DUMPING INTR enable/disable counters-------\n");
+	bcm_bprintf(strbuf, "host_irq_disable_count=%lu host_irq_enable_count=%lu\n"
+		"dngl_intmask_disable_count=%lu dngl_intmask_enable_count=%lu\n"
+		"dpc_return_busdown_count=%lu non_ours_irq_count=%lu\n",
+		bus->host_irq_disable_count, bus->host_irq_enable_count,
+		bus->dngl_intmask_disable_count, bus->dngl_intmask_enable_count,
+		bus->dpc_return_busdown_count, bus->non_ours_irq_count);
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	bcm_bprintf(strbuf, "oob_intr_count=%lu oob_intr_enable_count=%lu"
+		" oob_intr_disable_count=%lu\noob_irq_num=%d"
+		" last_oob_irq_times="SEC_USEC_FMT":"SEC_USEC_FMT
+		" last_oob_irq_enable_time="SEC_USEC_FMT"\nlast_oob_irq_disable_time="SEC_USEC_FMT
+		" oob_irq_enabled=%d oob_gpio_level=%d\n",
+		bus->oob_intr_count, bus->oob_intr_enable_count,
+		bus->oob_intr_disable_count, dhdpcie_get_oob_irq_num(bus),
+		GET_SEC_USEC(bus->last_oob_irq_isr_time),
+		GET_SEC_USEC(bus->last_oob_irq_thr_time),
+		GET_SEC_USEC(bus->last_oob_irq_enable_time),
+		GET_SEC_USEC(bus->last_oob_irq_disable_time), dhdpcie_get_oob_irq_status(bus),
+		dhdpcie_get_oob_irq_level());
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+	bcm_bprintf(strbuf, "\ncurrent_time="SEC_USEC_FMT" isr_entry_time="SEC_USEC_FMT
+		" prev_isr_entry_time="SEC_USEC_FMT" isr_exit_time="SEC_USEC_FMT"\n"
+		"isr_sched_dpc_time="SEC_USEC_FMT" rpm_sched_dpc_time="SEC_USEC_FMT"\n"
+		" last_non_ours_irq_time="SEC_USEC_FMT" dpc_entry_time="SEC_USEC_FMT"\n"
+		"last_process_ctrlbuf_time="SEC_USEC_FMT " last_process_flowring_time="SEC_USEC_FMT
+		" last_process_txcpl_time="SEC_USEC_FMT"\nlast_process_rxcpl_time="SEC_USEC_FMT
+		" last_process_infocpl_time="SEC_USEC_FMT" last_process_edl_time="SEC_USEC_FMT
+		"\ndpc_exit_time="SEC_USEC_FMT" resched_dpc_time="SEC_USEC_FMT"\n"
+		"last_d3_inform_time="SEC_USEC_FMT" dpc_sched=%u\n",
+		GET_SEC_USEC(current_time), GET_SEC_USEC(bus->isr_entry_time),
+		GET_SEC_USEC(bus->prev_isr_entry_time),
+		GET_SEC_USEC(bus->isr_exit_time), GET_SEC_USEC(bus->isr_sched_dpc_time),
+		GET_SEC_USEC(bus->rpm_sched_dpc_time),
+		GET_SEC_USEC(bus->last_non_ours_irq_time), GET_SEC_USEC(bus->dpc_entry_time),
+		GET_SEC_USEC(bus->last_process_ctrlbuf_time),
+		GET_SEC_USEC(bus->last_process_flowring_time),
+		GET_SEC_USEC(bus->last_process_txcpl_time),
+		GET_SEC_USEC(bus->last_process_rxcpl_time),
+		GET_SEC_USEC(bus->last_process_infocpl_time),
+		GET_SEC_USEC(bus->last_process_edl_time),
+		GET_SEC_USEC(bus->dpc_exit_time), GET_SEC_USEC(bus->resched_dpc_time),
+		GET_SEC_USEC(bus->last_d3_inform_time), bus->dpc_sched);
+
+	bcm_bprintf(strbuf, "\nlast_suspend_start_time="SEC_USEC_FMT" last_suspend_end_time="
+		SEC_USEC_FMT" last_resume_start_time="SEC_USEC_FMT" last_resume_end_time="
+		SEC_USEC_FMT"\n", GET_SEC_USEC(bus->last_suspend_start_time),
+		GET_SEC_USEC(bus->last_suspend_end_time),
+		GET_SEC_USEC(bus->last_resume_start_time),
+		GET_SEC_USEC(bus->last_resume_end_time));
+
+#if defined(SHOW_LOGTRACE) && defined(DHD_USE_KTHREAD_FOR_LOGTRACE)
+	bcm_bprintf(strbuf, "logtrace_thread_entry_time="SEC_USEC_FMT
+		" logtrace_thread_sem_down_time="SEC_USEC_FMT
+		"\nlogtrace_thread_flush_time="SEC_USEC_FMT
+		" logtrace_thread_unexpected_break_time="SEC_USEC_FMT
+		"\nlogtrace_thread_complete_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(dhd->logtrace_thr_ts.entry_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.sem_down_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.flush_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.unexpected_break_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.complete_time));
+#endif /* SHOW_LOGTRACE && DHD_USE_KTHREAD_FOR_LOGTRACE */
+}
+
+void dhd_dump_intr_registers(dhd_pub_t *dhd, struct bcmstrbuf *strbuf)
+{
+	uint32 intstatus = 0;
+	uint32 intmask = 0;
+	uint32 d2h_db0 = 0;
+	uint32 d2h_mb_data = 0;
+
+	intstatus = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		dhd->bus->pcie_mailbox_int, 0, 0);
+#ifdef DHD_MMIO_TRACE
+	dhd_bus_mmio_trace(dhd->bus, dhd->bus->pcie_mailbox_int, intstatus, FALSE);
+#endif /* defined(DHD_MMIO_TRACE) */
+	intmask = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		dhd->bus->pcie_mailbox_mask, 0, 0);
+#ifdef DHD_MMIO_TRACE
+	dhd_bus_mmio_trace(dhd->bus, dhd->bus->pcie_mailbox_mask, intmask, FALSE);
+#endif /* defined(DHD_MMIO_TRACE) */
+	d2h_db0 = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, PCID2H_MailBox, 0, 0);
+	dhd_bus_cmn_readshared(dhd->bus, &d2h_mb_data, D2H_MB_DATA, 0);
+
+	bcm_bprintf(strbuf, "intstatus=0x%x intmask=0x%x d2h_db0=0x%x\n",
+		intstatus, intmask, d2h_db0);
+	bcm_bprintf(strbuf, "d2h_mb_data=0x%x def_intmask=0x%x\n",
+		d2h_mb_data, dhd->bus->def_intmask);
+}
+
+void dhd_bus_dump_flowring(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	uint16 flowid;
+	int ix = 0;
+	flow_ring_node_t *flow_ring_node;
+	flow_info_t *flow_info;
+#ifdef TX_STATUS_LATENCY_STATS
+	uint8 ifindex;
+	if_flow_lkup_t *if_flow_lkup;
+	dhd_if_tx_status_latency_t if_tx_status_latency[DHD_MAX_IFS];
+#endif /* TX_STATUS_LATENCY_STATS */
+	unsigned long flags;
+
+#ifdef TX_STATUS_LATENCY_STATS
+	bzero(if_tx_status_latency, sizeof(if_tx_status_latency));
+#endif /* TX_STATUS_LATENCY_STATS */
+
+	bcm_bprintf(strbuf, "Flowring info:\n==============\n");
+	bcm_bprintf(strbuf, "[RD=read ptr; WR=write ptr; T=TCM; H=Host; L=Local; D=DMA index'd]\n");
+#ifndef BCM_ROUTER_DHD
+	bcm_bprintf(strbuf,
+		"%4s %4s %2s %4s %17s %4s %10s %17s %17s %17s %17s %14s %14s %10s ",
+		"Num:", "Flow", "If", "Prio", ":Dest_MacAddress:", "Qlen",
+		" Overflows", "TRD: HLRD: HDRD", "TWR: HLWR: HDWR", "BASE(VA)", "BASE(PA)",
+		"WORK_ITEM_SIZE", "MAX_WORK_ITEMS", "TOTAL_SIZE");
+#else
+	bcm_bprintf(strbuf,
+		"%4s %4s %2s %4s %17s %4s %4s %6s %10s %17s %17s %17s %17s %14s %14s %10s ",
+		"Num:", "Flow", "If", "Prio", ":Dest_MacAddress:", "Qlen", "CLen", "L2CLen",
+		" Overflows", "TRD: HLRD: HDRD", "TWR: HLWR: HDWR", "BASE(VA)", "BASE(PA)",
+		"WORK_ITEM_SIZE", "MAX_WORK_ITEMS", "TOTAL_SIZE");
+#endif /* !BCM_ROUTER_DHD */
+
+#ifdef TX_STATUS_LATENCY_STATS
+	/* Average Tx status/Completion Latency in micro secs */
+	bcm_bprintf(strbuf, "%16s %16s ", "       NumTxPkts", "    AvgTxCmpL_Us");
+#endif /* TX_STATUS_LATENCY_STATS */
+
+	bcm_bprintf(strbuf, "\n");
+
+	for (flowid = 0; flowid < dhdp->num_h2d_rings; flowid++) {
+		flow_ring_node = DHD_FLOW_RING(dhdp, flowid);
+		DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+		if (flow_ring_node->status != FLOW_RING_STATUS_OPEN) {
+			DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+			continue;
+		}
+
+		flow_info = &flow_ring_node->flow_info;
+		bcm_bprintf(strbuf,
+			"%4d %4d %2d %4d "MACDBG" %4d"
+#ifdef BCM_ROUTER_DHD
+			"%4d %6d"
+#endif
+			"%10u ", ix++,
+			flow_ring_node->flowid, flow_info->ifindex, flow_info->tid,
+			MAC2STRDBG(flow_info->da),
+			DHD_FLOW_QUEUE_LEN(&flow_ring_node->queue),
+#ifdef BCM_ROUTER_DHD
+			DHD_CUMM_CTR_READ(DHD_FLOW_QUEUE_CLEN_PTR(&flow_ring_node->queue)),
+			DHD_CUMM_CTR_READ(DHD_FLOW_QUEUE_L2CLEN_PTR(&flow_ring_node->queue)),
+#endif
+			DHD_FLOW_QUEUE_FAILURES(&flow_ring_node->queue));
+		dhd_prot_print_flow_ring(dhdp, flow_ring_node->prot_info, TRUE, strbuf,
+			"%5d:%5d:%5d %5d:%5d:%5d %17p %8x:%8x %14d %14d %10d");
+
+#ifdef TX_STATUS_LATENCY_STATS
+		bcm_bprintf(strbuf, "%16llu %16llu ",
+			flow_info->num_tx_pkts,
+			flow_info->num_tx_status ?
+			DIV_U64_BY_U64(flow_info->cum_tx_status_latency,
+			flow_info->num_tx_status) : 0);
+		ifindex = flow_info->ifindex;
+		ASSERT(ifindex < DHD_MAX_IFS);
+		if (ifindex < DHD_MAX_IFS) {
+			if_tx_status_latency[ifindex].num_tx_status += flow_info->num_tx_status;
+			if_tx_status_latency[ifindex].cum_tx_status_latency +=
+				flow_info->cum_tx_status_latency;
+		} else {
+			DHD_ERROR(("%s: Bad IF index: %d associated with flowid: %d\n",
+				__FUNCTION__, ifindex, flowid));
+		}
+#endif /* TX_STATUS_LATENCY_STATS */
+		bcm_bprintf(strbuf, "\n");
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+	}
+
+#ifdef TX_STATUS_LATENCY_STATS
+	bcm_bprintf(strbuf, "\n%s  %16s  %16s\n", "If", "AvgTxCmpL_Us", "NumTxStatus");
+	if_flow_lkup = (if_flow_lkup_t *)dhdp->if_flow_lkup;
+	for (ix = 0; ix < DHD_MAX_IFS; ix++) {
+		if (!if_flow_lkup[ix].status) {
+			continue;
+		}
+		bcm_bprintf(strbuf, "%2d  %16llu  %16llu\n",
+			ix,
+			if_tx_status_latency[ix].num_tx_status ?
+			DIV_U64_BY_U64(if_tx_status_latency[ix].cum_tx_status_latency,
+			if_tx_status_latency[ix].num_tx_status): 0,
+			if_tx_status_latency[ix].num_tx_status);
+	}
+#endif /* TX_STATUS_LATENCY_STATS */
+
+	bcm_bprintf(strbuf, "\n");
+}
+
+void
+dhd_bus_counters(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	dhd_prot_counters(dhdp, strbuf, TRUE, TRUE);
+
+	dhd_bus_dump_flowring(dhdp, strbuf);
+
+	dhd_dump_dpc_histos(dhdp, strbuf);
+}
+
+/** Add bus dump output to a buffer */
+void dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	uint16 flowid;
+#ifdef BCMDBG
+	int ix = 0;
+	flow_ring_node_t *flow_ring_node;
+	flow_info_t *flow_info;
+	flow_info_t *local_flow_info;
+#endif /* BCMDBG */
+
+	BCM_REFERENCE(flowid);
+
+#if defined(FW_SIGNATURE)
+	/* Dump secure firmware status. */
+	if (dhdp->busstate <= DHD_BUS_LOAD) {
+		dhd_bus_dump_fws(dhdp->bus, strbuf);
+	}
+#endif
+
+	if (dhdp->busstate != DHD_BUS_DATA)
+		return;
+
+#ifdef DHD_WAKE_STATUS
+	bcm_bprintf(strbuf, "wake %u rxwake %u readctrlwake %u\n",
+		bcmpcie_get_total_wake(dhdp->bus), dhdp->bus->wake_counts.rxwake,
+		dhdp->bus->wake_counts.rcwake);
+#ifdef DHD_WAKE_RX_STATUS
+	bcm_bprintf(strbuf, " unicast %u muticast %u broadcast %u arp %u\n",
+		dhdp->bus->wake_counts.rx_ucast, dhdp->bus->wake_counts.rx_mcast,
+		dhdp->bus->wake_counts.rx_bcast, dhdp->bus->wake_counts.rx_arp);
+	bcm_bprintf(strbuf, " multi4 %u multi6 %u icmp %u icmp6 %u multiother %u\n",
+		dhdp->bus->wake_counts.rx_multi_ipv4, dhdp->bus->wake_counts.rx_multi_ipv6,
+		dhdp->bus->wake_counts.rx_icmp, dhdp->bus->wake_counts.rx_icmpv6,
+		dhdp->bus->wake_counts.rx_multi_other);
+	bcm_bprintf(strbuf, " icmp6_ra %u, icmp6_na %u, icmp6_ns %u\n",
+		dhdp->bus->wake_counts.rx_icmpv6_ra, dhdp->bus->wake_counts.rx_icmpv6_na,
+		dhdp->bus->wake_counts.rx_icmpv6_ns);
+#endif /* DHD_WAKE_RX_STATUS */
+#ifdef DHD_WAKE_EVENT_STATUS
+#ifdef CUSTOM_WAKE_REASON_STATS
+	bcm_bprintf(strbuf, "rc_event_idx = %d, which indicates queue head\n",
+		dhdp->bus->wake_counts.rc_event_idx);
+	for (flowid = 0; flowid < MAX_WAKE_REASON_STATS; flowid++)
+		if (dhdp->bus->wake_counts.rc_event[flowid] != -1)
+#else
+	for (flowid = 0; flowid < WLC_E_LAST; flowid++)
+		if (dhdp->bus->wake_counts.rc_event[flowid] != 0)
+#endif /* CUSTOM_WAKE_REASON_STATS */
+			bcm_bprintf(strbuf, " %s = %u\n", bcmevent_get_name(flowid),
+				dhdp->bus->wake_counts.rc_event[flowid]);
+	bcm_bprintf(strbuf, "\n");
+#endif /* DHD_WAKE_EVENT_STATUS */
+#endif /* DHD_WAKE_STATUS */
+
+	dhd_prot_print_info(dhdp, strbuf);
+	dhd_dump_intr_registers(dhdp, strbuf);
+	dhd_dump_intr_counters(dhdp, strbuf);
+	bcm_bprintf(strbuf, "h2d_mb_data_ptr_addr 0x%x, d2h_mb_data_ptr_addr 0x%x\n",
+		dhdp->bus->h2d_mb_data_ptr_addr, dhdp->bus->d2h_mb_data_ptr_addr);
+	bcm_bprintf(strbuf, "dhd cumm_ctr %d\n", DHD_CUMM_CTR_READ(&dhdp->cumm_ctr));
+#ifdef DHD_LIMIT_MULTI_CLIENT_FLOWRINGS
+	bcm_bprintf(strbuf, "multi_client_flow_rings:%u max_multi_client_flow_rings:%d\n",
+		OSL_ATOMIC_READ(dhdp->osh, &dhdp->multi_client_flow_rings),
+		dhdp->max_multi_client_flow_rings);
+#endif /* DHD_LIMIT_MULTI_CLIENT_FLOWRINGS */
+
+	if (dhdp->htput_support) {
+		bcm_bprintf(strbuf, "htput_flow_ring_start:%d total_htput:%d client_htput=%d\n",
+			dhdp->htput_flow_ring_start, HTPUT_TOTAL_FLOW_RINGS,
+			dhdp->htput_client_flow_rings);
+	}
+
+	dhd_bus_dump_flowring(dhdp, strbuf);
+
+#ifdef BCMDBG
+	if (!dhdp->d2h_sync_mode) {
+		ix = 0;
+		bcm_bprintf(strbuf, "\n%4s %4s %2s %10s %7s %6s %5s %5s %10s %7s %7s %7s \n",
+			"Num:", "Flow", "If", "     ACKED", "D11SPRS", "WLSPRS", "TSDWL",
+			"NOACK", "SPRS_ACKED", "EXPIRED", "DROPPED", "FWFREED");
+		for (flowid = 0; flowid < dhdp->num_h2d_rings; flowid++) {
+			flow_ring_node = DHD_FLOW_RING(dhdp, flowid);
+			if (!flow_ring_node->active)
+				continue;
+
+			flow_info = &flow_ring_node->flow_info;
+			bcm_bprintf(strbuf, "%4d %4d %2d ",
+				ix++, flow_ring_node->flowid, flow_info->ifindex);
+			local_flow_info = &flow_ring_node->flow_info;
+			bcm_bprintf(strbuf, "%10d %7d %6d %5d %5d %10d %7d %7d %7d\n",
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_DISCARD],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_D11SUPPRESS],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_WLSUPPRESS],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_TOSSED_BYWLC],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_DISCARD_NOACK],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_SUPPRESS_ACKED],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_EXPIRED],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_DROPPED],
+				local_flow_info->tx_status[WLFC_CTL_PKTFLAG_MKTFREE]);
+		}
+	}
+#endif /* BCMDBG */
+
+	dhd_dump_dpc_histos(dhdp, strbuf);
+
+	bcm_bprintf(strbuf, "D3 inform cnt %d\n", dhdp->bus->d3_inform_cnt);
+	bcm_bprintf(strbuf, "D0 inform cnt %d\n", dhdp->bus->d0_inform_cnt);
+	bcm_bprintf(strbuf, "D0 inform in use cnt %d\n", dhdp->bus->d0_inform_in_use_cnt);
+	if (dhdp->d2h_hostrdy_supported) {
+		bcm_bprintf(strbuf, "hostready count:%d\n", dhdp->bus->hostready_count);
+	}
+#ifdef PCIE_INB_DW
+	/* Inband device wake counters */
+	if (INBAND_DW_ENAB(dhdp->bus)) {
+		bcm_bprintf(strbuf, "Inband device_wake assert count: %d\n",
+			dhdp->bus->inband_dw_assert_cnt);
+		bcm_bprintf(strbuf, "Inband device_wake deassert count: %d\n",
+			dhdp->bus->inband_dw_deassert_cnt);
+		bcm_bprintf(strbuf, "Inband DS-EXIT <host initiated> count: %d\n",
+			dhdp->bus->inband_ds_exit_host_cnt);
+		bcm_bprintf(strbuf, "Inband DS-EXIT <device initiated> count: %d\n",
+			dhdp->bus->inband_ds_exit_device_cnt);
+		bcm_bprintf(strbuf, "Inband DS-EXIT Timeout count: %d\n",
+			dhdp->bus->inband_ds_exit_to_cnt);
+		bcm_bprintf(strbuf, "Inband HOST_SLEEP-EXIT Timeout count: %d\n",
+			dhdp->bus->inband_host_sleep_exit_to_cnt);
+	}
+#endif /* PCIE_INB_DW */
+	bcm_bprintf(strbuf, "d2h_intr_method -> %s d2h_intr_control -> %s\n",
+		dhdp->bus->d2h_intr_method ? "PCIE_MSI" : "PCIE_INTX",
+		dhdp->bus->d2h_intr_control ? "HOST_IRQ" : "D2H_INTMASK");
+
+	bcm_bprintf(strbuf, "\n\nDB7 stats - db7_send_cnt: %d, db7_trap_cnt: %d, "
+		"max duration: %lld (%lld - %lld), db7_timing_error_cnt: %d\n",
+		dhdp->db7_trap.debug_db7_send_cnt,
+		dhdp->db7_trap.debug_db7_trap_cnt,
+		dhdp->db7_trap.debug_max_db7_dur,
+		dhdp->db7_trap.debug_max_db7_trap_time,
+		dhdp->db7_trap.debug_max_db7_send_time,
+		dhdp->db7_trap.debug_db7_timing_error_cnt);
+}
+
+int
+dhd_dump_flowrings(dhd_pub_t *dhdp, char *buf, int buflen)
+{
+	struct bcmstrbuf b;
+	struct bcmstrbuf *strbuf = &b;
+
+	if (!dhdp || !dhdp->prot || !buf) {
+		return BCME_ERROR;
+	}
+
+	bcm_binit(strbuf, buf, buflen);
+	dhd_bus_dump_flowring(dhdp, strbuf);
+	return (!strbuf->size ? BCME_BUFTOOSHORT : strbuf->size);
+}
+
+static void
+dhd_bus_dump_txcpl_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	tx_cpl_info_t *txcpl_info = &dhdp->txcpl_info;
+	int i;
+
+	bcm_bprintf(strbuf, "\nTx Completion History\n");
+	bcm_bprintf(strbuf, "Host(us)\t\tPTM_high(ns)\t\tPTM_low(ns)\t\t"
+		"Latency(ms)\t\tTID\t\tFlowID\n");
+	for (i = 0; i < MAX_TXCPL_HISTORY; i ++) {
+		bcm_bprintf(strbuf, "0x%x\t\t0x%x\t\t0x%x\t\t%u\t\t%u\t\t%u\n",
+			txcpl_info->tx_history[i].host_time,
+			txcpl_info->tx_history[i].ptm_high,
+			txcpl_info->tx_history[i].ptm_low,
+			txcpl_info->tx_history[i].latency,
+			txcpl_info->tx_history[i].tid,
+			txcpl_info->tx_history[i].flowid);
+	}
+	bzero(txcpl_info->tx_history,
+		sizeof(tx_cpl_history_t) * MAX_TXCPL_HISTORY);
+
+	bcm_bprintf(strbuf, "\n");
+}
+
+static void
+dhd_bus_dump_mdring_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	int i;
+	int count;
+
+	bcm_bprintf(strbuf, "\nMetadata Ring Dump\n");
+	for (i = 0, count = 0;
+		i < MAX_MDRING_ITEM_DUMP * D2HRING_MDCMPLT_ITEMSIZE; i += 4) {
+		if ((i % D2HRING_MDCMPLT_ITEMSIZE) == 0) {
+			bcm_bprintf(strbuf, "\nEntry:%d:", ++count);
+		}
+		bcm_bprintf(strbuf, "0x%x%x%x%x:",
+			dhdp->mdring_info[i], dhdp->mdring_info[i+1],
+			dhdp->mdring_info[i+2], dhdp->mdring_info[i+3]);
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+
+static bool
+dhd_bus_support_dar_sec_status(dhd_bus_t *bus)
+{
+	DHD_ERROR(("%s: buscorerev=%d chipid=0x%x\n",
+		__FUNCTION__, bus->sih->buscorerev, si_chipid(bus->sih)));
+
+	/* Support for DAR security status register provided for pcie core
+	 * generation2 at revisions > 73 and for generation3 at revisions > 128
+	 */
+	if ((bus->sih->buscorerev < 74) || (bus->sih->buscorerev == 128)) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static int
+dhd_bus_get_security_status(dhd_bus_t *bus, uint32 *status)
+{
+	uint32 dar_sec_reg;
+
+	if (!dhd_bus_support_dar_sec_status(bus)) {
+		return BCME_UNSUPPORTED;
+	}
+	/* DAR Security Status Register address */
+	dar_sec_reg = (uint32)DAR_SEC_STATUS(bus->sih->buscorerev);
+
+	/* read DAR Security Status Register */
+	*status = si_corereg(bus->sih, bus->sih->buscoreidx, dar_sec_reg, 0, 0);
+
+	return BCME_OK;
+}
+
+static int
+dhd_bus_security_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	int ret = BCME_OK;
+	dhd_bus_t *bus = dhdp->bus;
+	uint32 dar_sec_val;
+
+	ret = dhd_bus_get_security_status(bus, &dar_sec_val);
+	if (ret != BCME_OK) {
+		return ret;
+	}
+
+	bcm_bprintf(strbuf, "\nDAR Security Status Reg\n");
+	bcm_bprintf(strbuf, "  Jtag Disable:     %d\n",
+		(dar_sec_val & DAR_SEC_JTAG_MASK) >> DAR_SEC_JTAG_SHIFT);
+	bcm_bprintf(strbuf, "  Secure Boot:      %d\n",
+		(dar_sec_val & DAR_SEC_SBOOT_MASK) >> DAR_SEC_SBOOT_SHIFT);
+	bcm_bprintf(strbuf, "  Arm Dbg Disable:  %d\n",
+		(dar_sec_val & DAR_SEC_ARM_DBG_MASK) >> DAR_SEC_ARM_DBG_SHIFT);
+	bcm_bprintf(strbuf, "  Transient Unlock: %d\n",
+		(dar_sec_val & DAR_SEC_UNLOCK_MASK) >> DAR_SEC_UNLOCK_SHIFT);
+
+	/* The following bits in security status register provided for pcie core
+	 * generation2 at revisions > 76 and for generation3 at revisions > 129
+	 */
+	if (((bus->sih->buscorerev > 76) && (bus->sih->buscorerev < 128)) ||
+		(bus->sih->buscorerev > 129)) {
+		bcm_bprintf(strbuf, "  Rom Protect:      %d\n",
+			(dar_sec_val & DAR_SEC_ROM_PROT_MASK) >> DAR_SEC_ROM_PROT_SHIFT);
+		bcm_bprintf(strbuf, "  Non Secure Write: %d\n",
+			(dar_sec_val & DAR_SEC_NSEC_WR_MASK) >> DAR_SEC_NSEC_WR_SHIFT);
+		bcm_bprintf(strbuf, "  Non Secure Read:  %d\n",
+			(dar_sec_val & DAR_SEC_NSEC_RD_MASK) >> DAR_SEC_NSEC_RD_SHIFT);
+	}
+	return BCME_OK;
+}
+
+static int
+dhd_bus_sboot_disable(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	int ret = BCME_OK;
+	dhd_bus_t *bus = dhdp->bus;
+	uint32 save_idx, val;
+	sysmemregs_t *regs;
+	uint32 dar_sec_val;
+
+	save_idx = si_coreidx(bus->sih);
+	ret = dhd_bus_get_security_status(bus, &dar_sec_val);
+	if (ret != BCME_OK) {
+		goto exit;
+	}
+
+	/* check if chip is Transient Unlocked */
+	if (!(dar_sec_val & DAR_SEC_UNLOCK_MASK)) {
+		bcm_bprintf(strbuf, "\nFail: Secure Boot still enabled!\n");
+		goto exit;
+	}
+
+	if ((regs = si_setcore(bus->sih, SYSMEM_CORE_ID, 0)) != NULL) {
+		/* check if mpu control is unlocked */
+		val = R_REG(dhdp->osh, MPU_REG(regs, mpucontrol));
+		if (val & BUS_MPU_LOCK_MASK) {
+			bcm_bprintf(strbuf, "\nFail: Mpu Control is locked!\n");
+			goto exit;
+		}
+
+		/* clear RomProtect and VectorProtect */
+		W_REG(dhdp->osh, MPU_REG(regs, mpucontrol),
+			val & ~(BUS_MPU_ROM_PR_MASK | BUS_MPU_VEC_PR_MASK));
+
+		/* validate RomProtect and VectorProtect are cleared */
+		val = R_REG(dhdp->osh, MPU_REG(regs, mpucontrol));
+		if (val & (BUS_MPU_ROM_PR_MASK | BUS_MPU_VEC_PR_MASK)) {
+			bcm_bprintf(strbuf, "\nFail: Secure Boot still enabled!\n");
+		} else {
+			bcm_bprintf(strbuf, "\nSecure Boot disabled!\n");
+		}
+
+	}
+exit:
+	si_setcoreidx(bus->sih, save_idx);
+	return ret;
+}
+
+static void
+dhd_bus_dump_rxlat_info(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	if (dhdp->rx_cpl_lat_capable) {
+		rx_cpl_lat_info_t *rxcpl_info = &dhdp->rxcpl_lat_info;
+		int i;
+
+		bcm_bprintf(strbuf, "\nRx Completion History\n");
+		bcm_bprintf(strbuf, "Host(us)\t\tPTM_high(ns)\t\tPTM_low(ns)\t\tRx_t0\t\tBand"
+			"\t\tPrio\t\tRSSI\t\tLatency(us)\n");
+		for (i = 0; i < MAX_RXCPL_HISTORY; i ++) {
+			if (rxcpl_info->rx_history[i].rx_t0 || rxcpl_info->rx_history[i].ptm_low) {
+				bcm_bprintf(strbuf, "0x%x\t\t0x%x\t\t0x%x\t\t0x%x\t\t%s\t\t%d\t\t%d"
+					"\t\t%d\n",
+					rxcpl_info->rx_history[i].host_time,
+					rxcpl_info->rx_history[i].ptm_high,
+					rxcpl_info->rx_history[i].ptm_low,
+					rxcpl_info->rx_history[i].rx_t0,
+					rxcpl_info->rx_history[i].slice? "5G":"2G",
+					rxcpl_info->rx_history[i].priority,
+					rxcpl_info->rx_history[i].rssi,
+					rxcpl_info->rx_history[i].latency);
+			}
+		}
+		bzero(rxcpl_info->rx_history,
+			sizeof(rx_cpl_history_t) * MAX_RXCPL_HISTORY);
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+
+static void
+dhd_bus_dump_rxlat_histo(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	int bin, prio;
+
+	if (dhdp->rx_cpl_lat_capable) {
+		rx_cpl_lat_info_t *rxcpl_info = &dhdp->rxcpl_lat_info;
+
+		bcm_bprintf(strbuf,
+			"\nRx:5G Latency histogram, each bin:%d us\n", RX_LAT_BIN_SCALE);
+		bcm_bprintf(strbuf,
+			"Bin\tPrio-0\tPrio-1\tPrio-2\tPrio-3\tPrio-4\tPrio-5\tPrio-6\tPrio-7\n");
+		for (bin = 0; bin < MAX_RX_LAT_HIST_BIN; bin++) {
+			bcm_bprintf(strbuf, "\n%d", bin);
+			for (prio = 0; prio < MAX_RX_LAT_PRIO; prio++) {
+				bcm_bprintf(strbuf, "\t");
+				if (rxcpl_info->rx_dur_5g[prio][bin]) {
+					bcm_bprintf(strbuf, "%d",
+						rxcpl_info->rx_dur_5g[prio][bin]);
+				}
+			}
+		}
+		bcm_bprintf(strbuf,
+			"\nRx:2G Latency histogram, each bin:%d us\n", RX_LAT_BIN_SCALE);
+		bcm_bprintf(strbuf,
+			"Bin\tPrio-0\tPrio-1\tPrio-2\tPrio-3\tPrio-4\tPrio-5\tPrio-6\tPrio-7\n");
+		for (bin = 0; bin < MAX_RX_LAT_HIST_BIN; bin++) {
+			bcm_bprintf(strbuf, "\n%d", bin);
+			for (prio = 0; prio < MAX_RX_LAT_PRIO; prio++) {
+				bcm_bprintf(strbuf, "\t");
+				if (rxcpl_info->rx_dur_2g[prio][bin]) {
+					bcm_bprintf(strbuf, "%d",
+						rxcpl_info->rx_dur_2g[prio][bin]);
+				}
+			}
+		}
+		bzero(rxcpl_info->rx_dur_5g, MAX_RX_LAT_PRIO * MAX_RX_LAT_HIST_BIN);
+		bzero(rxcpl_info->rx_dur_2g, MAX_RX_LAT_PRIO * MAX_RX_LAT_HIST_BIN);
+	}
+	bcm_bprintf(strbuf, "\n");
+}
+
+#ifdef DNGL_AXI_ERROR_LOGGING
+bool
+dhd_axi_sig_match(dhd_pub_t *dhdp)
+{
+	uint32 axi_tcm_addr = dhdpcie_bus_rtcm32(dhdp->bus, dhdp->axierror_logbuf_addr);
+
+	if (dhdp->dhd_induce_error == DHD_INDUCE_DROP_AXI_SIG) {
+		DHD_ERROR(("%s: Induce AXI signature drop\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	DHD_ERROR(("%s: axi_tcm_addr: 0x%x, tcm range: 0x%x ~ 0x%x\n",
+		__FUNCTION__, axi_tcm_addr, dhdp->bus->dongle_ram_base,
+		dhdp->bus->dongle_ram_base + dhdp->bus->ramsize));
+	if (axi_tcm_addr >= dhdp->bus->dongle_ram_base &&
+	    axi_tcm_addr < dhdp->bus->dongle_ram_base + dhdp->bus->ramsize) {
+		uint32 axi_signature = dhdpcie_bus_rtcm32(dhdp->bus, (axi_tcm_addr +
+			OFFSETOF(hnd_ext_trap_axi_error_v1_t, signature)));
+		if (axi_signature == HND_EXT_TRAP_AXIERROR_SIGNATURE) {
+			return TRUE;
+		} else {
+			DHD_ERROR(("%s: No AXI signature: 0x%x\n",
+				__FUNCTION__, axi_signature));
+			return FALSE;
+		}
+	} else {
+		DHD_ERROR(("%s: No AXI shared tcm address debug info.\n", __FUNCTION__));
+		return FALSE;
+	}
+}
+
+void
+dhd_axi_error(dhd_pub_t *dhdp)
+{
+	dhd_axi_error_dump_t *axi_err_dump;
+	uint8 *axi_err_buf = NULL;
+	uint8 *p_axi_err = NULL;
+	uint32 axi_logbuf_addr;
+	uint32 axi_tcm_addr;
+	int err, size;
+
+	/* XXX: On the Dongle side, if an invalid Host Address is generated for a transaction
+	 * it results in SMMU Fault. Now the Host won't respond for the invalid transaction.
+	 * On the Dongle side, after 50msec this results in AXI Slave Error.
+	 * Hence introduce a delay higher than 50msec to ensure AXI Slave error happens and
+	 * the Dongle collects the required information.
+	 */
+	OSL_DELAY(75000);
+
+	axi_logbuf_addr = dhdp->axierror_logbuf_addr;
+	if (!axi_logbuf_addr) {
+		DHD_ERROR(("%s: No AXI TCM address debug info.\n", __FUNCTION__));
+		goto sched_axi;
+	}
+
+	axi_err_dump = dhdp->axi_err_dump;
+	if (!axi_err_dump) {
+		goto sched_axi;
+	}
+
+	if (!dhd_axi_sig_match(dhdp)) {
+		goto sched_axi;
+	}
+
+	/* Reading AXI error data for SMMU fault */
+	DHD_ERROR(("%s: Read AXI data from TCM address\n", __FUNCTION__));
+	axi_tcm_addr = dhdpcie_bus_rtcm32(dhdp->bus, axi_logbuf_addr);
+	size = sizeof(hnd_ext_trap_axi_error_v1_t);
+	axi_err_buf = MALLOCZ(dhdp->osh, size);
+	if (axi_err_buf == NULL) {
+		DHD_ERROR(("%s: out of memory !\n", __FUNCTION__));
+		goto sched_axi;
+	}
+
+	p_axi_err = axi_err_buf;
+	err = dhdpcie_bus_membytes(dhdp->bus, FALSE, axi_tcm_addr, p_axi_err, size);
+	if (err) {
+		DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+			__FUNCTION__, err, size, axi_tcm_addr));
+		goto sched_axi;
+	}
+
+	/* Dump data to Dmesg */
+	dhd_log_dump_axi_error(axi_err_buf);
+	err = memcpy_s(&axi_err_dump->etd_axi_error_v1, size, axi_err_buf, size);
+	if (err) {
+		DHD_ERROR(("%s: failed to copy etd axi error info, err=%d\n",
+			__FUNCTION__, err));
+	}
+
+sched_axi:
+	if (axi_err_buf) {
+		MFREE(dhdp->osh, axi_err_buf, size);
+	}
+	dhd_schedule_axi_error_dump(dhdp, NULL);
+}
+
+static void
+dhd_log_dump_axi_error(uint8 *axi_err)
+{
+	dma_dentry_v1_t dma_dentry;
+	dma_fifo_v1_t dma_fifo;
+	int i = 0, j = 0;
+
+	if (*(uint8 *)axi_err == HND_EXT_TRAP_AXIERROR_VERSION_1) {
+		hnd_ext_trap_axi_error_v1_t *axi_err_v1 = (hnd_ext_trap_axi_error_v1_t *)axi_err;
+		DHD_ERROR(("%s: signature : 0x%x\n", __FUNCTION__, axi_err_v1->signature));
+		DHD_ERROR(("%s: version : 0x%x\n", __FUNCTION__, axi_err_v1->version));
+		DHD_ERROR(("%s: length : 0x%x\n", __FUNCTION__, axi_err_v1->length));
+		DHD_ERROR(("%s: dma_fifo_valid_count : 0x%x\n",
+			__FUNCTION__, axi_err_v1->dma_fifo_valid_count));
+		DHD_ERROR(("%s: axi_errorlog_status : 0x%x\n",
+			__FUNCTION__, axi_err_v1->axi_errorlog_status));
+		DHD_ERROR(("%s: axi_errorlog_core : 0x%x\n",
+			__FUNCTION__, axi_err_v1->axi_errorlog_core));
+		DHD_ERROR(("%s: axi_errorlog_hi : 0x%x\n",
+			__FUNCTION__, axi_err_v1->axi_errorlog_hi));
+		DHD_ERROR(("%s: axi_errorlog_lo : 0x%x\n",
+			__FUNCTION__, axi_err_v1->axi_errorlog_lo));
+		DHD_ERROR(("%s: axi_errorlog_id : 0x%x\n",
+			__FUNCTION__, axi_err_v1->axi_errorlog_id));
+
+		for (i = 0; i < MAX_DMAFIFO_ENTRIES_V1; i++) {
+			dma_fifo = axi_err_v1->dma_fifo[i];
+			DHD_ERROR(("%s: valid:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.valid));
+			DHD_ERROR(("%s: direction:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.direction));
+			DHD_ERROR(("%s: index:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.index));
+			DHD_ERROR(("%s: dpa:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.dpa));
+			DHD_ERROR(("%s: desc_lo:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.desc_lo));
+			DHD_ERROR(("%s: desc_hi:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.desc_hi));
+			DHD_ERROR(("%s: din:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.din));
+			DHD_ERROR(("%s: dout:%d : 0x%x\n",
+				__FUNCTION__, i, dma_fifo.dout));
+			for (j = 0; j < MAX_DMAFIFO_DESC_ENTRIES_V1; j++) {
+				dma_dentry = axi_err_v1->dma_fifo[i].dentry[j];
+				DHD_ERROR(("%s: ctrl1:%d : 0x%x\n",
+					__FUNCTION__, i, dma_dentry.ctrl1));
+				DHD_ERROR(("%s: ctrl2:%d : 0x%x\n",
+					__FUNCTION__, i, dma_dentry.ctrl2));
+				DHD_ERROR(("%s: addrlo:%d : 0x%x\n",
+					__FUNCTION__, i, dma_dentry.addrlo));
+				DHD_ERROR(("%s: addrhi:%d : 0x%x\n",
+					__FUNCTION__, i, dma_dentry.addrhi));
+			}
+		}
+	}
+	else {
+		DHD_ERROR(("%s: Invalid AXI version: 0x%x\n", __FUNCTION__, (*(uint8 *)axi_err)));
+	}
+}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+
+/**
+ * Brings transmit packets on all flow rings closer to the dongle, by moving (a subset) from their
+ * flow queue to their flow ring.
+ */
+static bool
+dhd_update_txflowrings(dhd_pub_t *dhd)
+{
+	unsigned long flags;
+	dll_t *item, *next;
+	flow_ring_node_t *flow_ring_node;
+	struct dhd_bus *bus = dhd->bus;
+	int count = 0;
+	bool more = FALSE;
+
+	if (dhd_query_bus_erros(dhd)) {
+		return more;
+	}
+
+	/* Hold flowring_list_lock to ensure no race condition while accessing the List */
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+	for (item = dll_head_p(&bus->flowring_active_list);
+		(!dhd_is_device_removed(dhd) && !dll_end(&bus->flowring_active_list, item));
+		item = next, count++) {
+		if (dhd->hang_was_sent) {
+			more = FALSE;
+			break;
+		}
+
+		if (count > bus->max_tx_flowrings) {
+			DHD_ERROR(("%s : overflow max flowrings\n", __FUNCTION__));
+#ifdef OEM_ANDROID
+			dhd->hang_reason = HANG_REASON_UNKNOWN;
+			dhd_os_send_hang_message(dhd);
+#endif /* OEM_ANDROID */
+			more = FALSE;
+			break;
+		}
+
+		next = dll_next_p(item);
+		flow_ring_node = dhd_constlist_to_flowring(item);
+
+		/* Ensure that flow_ring_node in the list is Not Null */
+		ASSERT(flow_ring_node != NULL);
+
+		/* Ensure that the flowring node has valid contents */
+		ASSERT(flow_ring_node->prot_info != NULL);
+
+		more = dhd_prot_update_txflowring(dhd, flow_ring_node->flowid,
+			flow_ring_node->prot_info);
+	}
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	return more;
+}
+
+/** Mailbox ringbell Function */
+static void
+dhd_bus_gen_devmb_intr(struct dhd_bus *bus)
+{
+	if ((bus->sih->buscorerev == 2) || (bus->sih->buscorerev == 6) ||
+		(bus->sih->buscorerev == 4)) {
+		DHD_ERROR(("%s: mailbox communication not supported\n", __FUNCTION__));
+		return;
+	}
+	if (bus->db1_for_mb)  {
+		/* this is a pcie core register, not the config register */
+		/* XXX: make sure we are on PCIE */
+		DHD_INFO(("%s: writing a mail box interrupt to the device, through doorbell 1\n", __FUNCTION__));
+		if (DAR_PWRREQ(bus)) {
+			dhd_bus_pcie_pwr_req(bus);
+		}
+#ifdef DHD_MMIO_TRACE
+		dhd_bus_mmio_trace(bus, dhd_bus_db1_addr_get(bus), 0x2, TRUE);
+#endif /* defined(DHD_MMIO_TRACE) */
+		si_corereg(bus->sih, bus->sih->buscoreidx, dhd_bus_db1_addr_get(bus),
+			~0, 0x12345678);
+	} else {
+		DHD_INFO(("%s: writing a mail box interrupt to the device, through config space\n", __FUNCTION__));
+		dhdpcie_bus_cfg_write_dword(bus, PCISBMbx, 4, (1 << 0));
+		/* XXX CRWLPCIEGEN2-182 requires double write */
+		dhdpcie_bus_cfg_write_dword(bus, PCISBMbx, 4, (1 << 0));
+	}
+}
+
+/* Upon receiving a mailbox interrupt,
+ * if H2D_FW_TRAP bit is set in mailbox location
+ * device traps
+ */
+static void
+dhdpcie_fw_trap(dhd_bus_t *bus)
+{
+	if (bus->dhd->db7_trap.fw_db7w_trap) {
+		uint32 addr = dhd_bus_db1_addr_3_get(bus);
+
+		DHD_ERROR(("%s: send DB7 trap!!!\n", __FUNCTION__));
+		bus->dhd->db7_trap.fw_db7w_trap_recieved = FALSE;
+		bus->dhd->db7_trap.debug_db7_send_time = OSL_LOCALTIME_NS();
+		bus->dhd->db7_trap.debug_db7_send_cnt++;
+		si_corereg(bus->sih, bus->sih->buscoreidx, addr, ~0,
+			bus->dhd->db7_trap.db7_magic_number);
+		return;
+	}
+
+	DHD_ERROR(("%s: send trap!!!\n", __FUNCTION__));
+	/* Send the mailbox data and generate mailbox intr. */
+	dhdpcie_send_mb_data(bus, H2D_FW_TRAP);
+	/* For FWs that cannot interprete H2D_FW_TRAP */
+	(void)dhd_wl_ioctl_set_intiovar(bus->dhd, "bus:disconnect", 99, WLC_SET_VAR, TRUE, 0);
+}
+
+void
+dhdpcie_db7_trap(struct dhd_bus *bus)
+{
+	if (bus->dhd->busstate != DHD_BUS_DATA) {
+		DHD_ERROR(("%s: Bus is NOT up. busstate: %d\n", __FUNCTION__, bus->dhd->busstate));
+		return;
+	}
+	if (bus->dhd->db7_trap.fw_db7w_trap) {
+		unsigned long flags = 0;
+
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		bus->dhd->db7_trap.fw_db7w_trap_inprogress = TRUE;
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		dhdpcie_fw_trap(bus);
+		OSL_DELAY(100 * 1000); // wait 100 msec
+		if (!bus->dhd->db7_trap.fw_db7w_trap_recieved) {
+			DHD_ERROR(("%s : Did not receive DB7 Ack\n", __FUNCTION__));
+		}
+		DHD_GENERAL_LOCK(bus->dhd, flags);
+		bus->dhd->db7_trap.fw_db7w_trap_inprogress = FALSE;
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+	} else {
+		DHD_TRACE(("%s: DB7 Not supported!!!\n", __FUNCTION__));
+	}
+
+}
+
+#ifdef PCIE_INB_DW
+
+void
+dhd_bus_inb_ack_pending_ds_req(dhd_bus_t *bus)
+{
+	/* The DHD_BUS_INB_DW_LOCK must be held before
+	* calling this function !!
+	*/
+	if ((dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+		DW_DEVICE_DS_DEV_SLEEP_PEND) &&
+		(bus->host_active_cnt == 0) && (!bus->skip_ds_ack)) {
+		dhdpcie_bus_set_pcie_inband_dw_state(bus, DW_DEVICE_DS_DEV_SLEEP);
+		dhdpcie_send_mb_data(bus, H2D_HOST_DS_ACK);
+	}
+}
+
+int
+dhd_bus_inb_set_device_wake(struct dhd_bus *bus, bool val)
+{
+	int timeleft;
+	unsigned long flags;
+	int ret;
+
+	if (!INBAND_DW_ENAB(bus)) {
+		return BCME_ERROR;
+	}
+	if (val) {
+		DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+
+		/*
+		 * Reset the Door Bell Timeout value. So that the Watchdog
+		 * doesn't try to Deassert Device Wake, while we are in
+		 * the process of still Asserting the same.
+		 */
+		dhd_bus_doorbell_timeout_reset(bus);
+
+		if (dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+			DW_DEVICE_DS_DEV_SLEEP) {
+			/* Clear wait_for_ds_exit */
+			bus->wait_for_ds_exit = 0;
+			if (bus->calc_ds_exit_latency) {
+				bus->ds_exit_latency = 0;
+				bus->ds_exit_ts2 = 0;
+				bus->ds_exit_ts1 = OSL_SYSUPTIME_US();
+			}
+			ret = dhdpcie_send_mb_data(bus, H2DMB_DS_DEVICE_WAKE_ASSERT);
+			if (ret != BCME_OK) {
+				DHD_ERROR(("Failed: assert Inband device_wake\n"));
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				ret = BCME_ERROR;
+				goto exit;
+			}
+			dhdpcie_bus_set_pcie_inband_dw_state(bus,
+				DW_DEVICE_DS_DISABLED_WAIT);
+			bus->inband_dw_assert_cnt++;
+		} else if (dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+			DW_DEVICE_DS_DISABLED_WAIT) {
+			DHD_ERROR(("Inband device wake is already asserted, "
+				"waiting for DS-Exit\n"));
+		}
+		else {
+			DHD_INFO(("Not in DS SLEEP state \n"));
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+			ret = BCME_OK;
+			goto exit;
+		}
+
+		/*
+		 * Since we are going to wait/sleep .. release the lock.
+		 * The Device Wake sanity is still valid, because
+		 * a) If there is another context that comes in and tries
+		 *    to assert DS again and if it gets the lock, since
+		 *    ds_state would be now != DW_DEVICE_DS_DEV_SLEEP the
+		 *    context would return saying Not in DS Sleep.
+		 * b) If ther is another context that comes in and tries
+		 *    to de-assert DS and gets the lock,
+		 *    since the ds_state is != DW_DEVICE_DS_DEV_WAKE
+		 *    that context would return too. This can not happen
+		 *    since the watchdog is the only context that can
+		 *    De-Assert Device Wake and as the first step of
+		 *    Asserting the Device Wake, we have pushed out the
+		 *    Door Bell Timeout.
+		 *
+		 */
+
+		if (!CAN_SLEEP()) {
+			dhdpcie_bus_set_pcie_inband_dw_state(bus,
+				DW_DEVICE_DS_DEV_WAKE);
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+			/* Called from context that cannot sleep */
+			OSL_DELAY(1000);
+		} else {
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+			/* Wait for DS EXIT for DS_EXIT_TIMEOUT seconds */
+			timeleft = dhd_os_ds_exit_wait(bus->dhd, &bus->wait_for_ds_exit);
+			if (!bus->wait_for_ds_exit || timeleft == 0) {
+				DHD_ERROR(("dhd_bus_inb_set_device_wake:DS-EXIT timeout, "
+					"wait_for_ds_exit : %d\n", bus->wait_for_ds_exit));
+				bus->inband_ds_exit_to_cnt++;
+				bus->ds_exit_timeout = 0;
+#ifdef DHD_FW_COREDUMP
+				if (bus->dhd->memdump_enabled) {
+					/* collect core dump */
+					DHD_GENERAL_LOCK(bus->dhd, flags);
+					DHD_BUS_BUSY_CLEAR_RPM_SUSPEND_IN_PROGRESS(bus->dhd);
+					DHD_GENERAL_UNLOCK(bus->dhd, flags);
+					bus->dhd->memdump_type =
+						DUMP_TYPE_INBAND_DEVICE_WAKE_FAILURE;
+					dhd_bus_mem_dump(bus->dhd);
+				}
+#else
+				ASSERT(0);
+#endif /* DHD_FW_COREDUMP */
+				ret = BCME_ERROR;
+				goto exit;
+			}
+		}
+		ret = BCME_OK;
+	} else {
+		DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+		if ((dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+			DW_DEVICE_DS_DEV_WAKE)) {
+			ret = dhdpcie_send_mb_data(bus, H2DMB_DS_DEVICE_WAKE_DEASSERT);
+			if (ret != BCME_OK) {
+				DHD_ERROR(("Failed: deassert Inband device_wake\n"));
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				goto exit;
+			}
+			dhdpcie_bus_set_pcie_inband_dw_state(bus,
+				DW_DEVICE_DS_ACTIVE);
+			bus->inband_dw_deassert_cnt++;
+		} else if ((dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+			DW_DEVICE_DS_DEV_SLEEP_PEND) &&
+			(bus->host_active_cnt == 0) && (!bus->skip_ds_ack)) {
+			dhdpcie_bus_set_pcie_inband_dw_state(bus, DW_DEVICE_DS_DEV_SLEEP);
+			dhdpcie_send_mb_data(bus, H2D_HOST_DS_ACK);
+		}
+
+		ret = BCME_OK;
+		DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+	}
+
+exit:
+	return ret;
+}
+
+#ifdef PCIE_INB_DSACK_EXT_WAIT
+void
+dhd_bus_ds_ack_debug_dump(struct dhd_bus *bus)
+{
+	if ((DIV_U64_BY_U32(bus->isr_entry_time, NSEC_PER_SEC) -
+		DIV_U64_BY_U32(bus->prev_isr_entry_time, NSEC_PER_SEC)) >=
+		DW_DS_ACK_RETRY_THRESHOLD) {
+		bus->dhd->dsack_hc_due_to_isr_delay = TRUE;
+		DHD_ERROR(("ISR did not run causing DS ack timeout\n"));
+	} else if ((bus->dpc_entry_time > bus->isr_entry_time) &&
+		(DIV_U64_BY_U32(bus->dpc_entry_time, NSEC_PER_SEC) -
+		DIV_U64_BY_U32(bus->isr_entry_time, NSEC_PER_SEC)) >=
+		DW_DS_ACK_RETRY_THRESHOLD) {
+		bus->dhd->dsack_hc_due_to_dpc_delay = TRUE;
+		DHD_ERROR(("Delay in scheduling DPC causing DS ack timeout\n"));
+	} else {
+		DHD_ERROR(("Failed to send DS ack\n"));
+	}
+	DHD_ERROR(("prev_isr_entry_time="SEC_USEC_FMT
+		" isr_entry_time="SEC_USEC_FMT
+		" dpc_entry_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->prev_isr_entry_time),
+		GET_SEC_USEC(bus->isr_entry_time),
+		GET_SEC_USEC(bus->dpc_entry_time)));
+}
+#endif /* PCIE_INB_DSACK_EXT_WAIT */
+#endif /* PCIE_INB_DW */
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+void
+dhd_bus_doorbell_timeout_reset(struct dhd_bus *bus)
+{
+	if (dhd_doorbell_timeout) {
+#ifdef BCMQT
+		uint wd_scale = 1;
+#else
+		uint wd_scale = dhd_watchdog_ms;
+#endif
+		if (dhd_watchdog_ms) {
+			dhd_timeout_start(&bus->doorbell_timer,
+				(dhd_doorbell_timeout * 1000) / wd_scale);
+		}
+	}
+	else if (!(bus->dhd->busstate == DHD_BUS_SUSPEND)) {
+		dhd_bus_set_device_wake(bus, FALSE);
+	}
+}
+
+int
+dhd_bus_set_device_wake(struct dhd_bus *bus, bool val)
+{
+	if (bus->ds_enabled && bus->dhd->ring_attached) {
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			return dhd_bus_inb_set_device_wake(bus, val);
+		}
+#endif /* PCIE_INB_DW */
+#ifdef PCIE_OOB
+		if (OOB_DW_ENAB(bus)) {
+			return dhd_os_oob_set_device_wake(bus, val);
+		}
+#endif /* PCIE_OOB */
+	}
+	return BCME_OK;
+}
+
+void
+dhd_bus_dw_deassert(dhd_pub_t *dhd)
+{
+	dhd_bus_t *bus = dhd->bus;
+	unsigned long flags;
+
+	if (dhd_query_bus_erros(bus->dhd)) {
+		return;
+	}
+
+	/* If haven't communicated with device for a while, deassert the Device_Wake GPIO */
+	if (dhd_doorbell_timeout != 0 && bus->dhd->busstate == DHD_BUS_DATA &&
+		dhd_timeout_expired(&bus->doorbell_timer) &&
+		!dhd_query_bus_erros(bus->dhd)) {
+		DHD_GENERAL_LOCK(dhd, flags);
+		if (DHD_BUS_BUSY_CHECK_IDLE(dhd) &&
+			!DHD_CHECK_CFG_IN_PROGRESS(dhd)) {
+			DHD_BUS_BUSY_SET_IN_DS_DEASSERT(dhd);
+			DHD_GENERAL_UNLOCK(dhd, flags);
+			dhd_bus_set_device_wake(bus, FALSE);
+			DHD_GENERAL_LOCK(dhd, flags);
+			DHD_BUS_BUSY_CLEAR_IN_DS_DEASSERT(dhd);
+			dhd_os_busbusy_wake(bus->dhd);
+			DHD_GENERAL_UNLOCK(dhd, flags);
+		} else {
+			DHD_GENERAL_UNLOCK(dhd, flags);
+		}
+	}
+
+#ifdef PCIE_INB_DW
+	if (INBAND_DW_ENAB(bus)) {
+		if (bus->ds_exit_timeout) {
+			bus->ds_exit_timeout --;
+			if (bus->ds_exit_timeout == 1) {
+				DHD_ERROR(("DS-EXIT TIMEOUT\n"));
+				bus->ds_exit_timeout = 0;
+				bus->inband_ds_exit_to_cnt++;
+			}
+		}
+		if (bus->host_sleep_exit_timeout) {
+			bus->host_sleep_exit_timeout --;
+			if (bus->host_sleep_exit_timeout == 1) {
+				DHD_ERROR(("HOST_SLEEP-EXIT TIMEOUT\n"));
+				bus->host_sleep_exit_timeout = 0;
+				bus->inband_host_sleep_exit_to_cnt++;
+			}
+		}
+	}
+#endif /* PCIE_INB_DW */
+}
+#endif /* defined(PCIE_OOB) || defined(PCIE_INB_DW) */
+
+/** mailbox doorbell ring function */
+void
+dhd_bus_ringbell(struct dhd_bus *bus, uint32 value)
+{
+	/* Skip once bus enters low power state (D3_INFORM/D3_ACK) */
+	if (__DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_ERROR(("%s: trying to ring the doorbell after D3 inform %d\n",
+			__FUNCTION__, bus->bus_low_power_state));
+		return;
+	}
+
+	/* Skip in the case of link down */
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	if ((bus->sih->buscorerev == 2) || (bus->sih->buscorerev == 6) ||
+		(bus->sih->buscorerev == 4)) {
+		si_corereg(bus->sih, bus->sih->buscoreidx, bus->pcie_mailbox_int,
+			PCIE_INTB, PCIE_INTB);
+	} else {
+		/* this is a pcie core register, not the config regsiter */
+		/* XXX: makesure we are on PCIE */
+		DHD_INFO(("%s: writing a door bell to the device\n", __FUNCTION__));
+		if (IDMA_ACTIVE(bus->dhd)) {
+			if (DAR_PWRREQ(bus)) {
+				dhd_bus_pcie_pwr_req(bus);
+			}
+			si_corereg(bus->sih, bus->sih->buscoreidx, dhd_bus_db0_addr_2_get(bus),
+				~0, value);
+		} else {
+			if (DAR_PWRREQ(bus)) {
+				dhd_bus_pcie_pwr_req(bus);
+			}
+			si_corereg(bus->sih, bus->sih->buscoreidx,
+				dhd_bus_db0_addr_get(bus), ~0, 0x12345678);
+		}
+	}
+}
+
+/** mailbox doorbell ring function for IDMA/IFRM using dma channel2 */
+void
+dhd_bus_ringbell_2(struct dhd_bus *bus, uint32 value, bool devwake)
+{
+	/* this is a pcie core register, not the config regsiter */
+	/* XXX: makesure we are on PCIE */
+	/* Skip once bus enters low power state (D3_INFORM/D3_ACK) */
+	if (__DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_ERROR(("%s: trying to ring the doorbell after D3 inform %d\n",
+			__FUNCTION__, bus->bus_low_power_state));
+		return;
+	}
+
+	/* Skip in the case of link down */
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_INFO(("writing a door bell 2 to the device\n"));
+	if (DAR_PWRREQ(bus)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+	si_corereg(bus->sih, bus->sih->buscoreidx, dhd_bus_db0_addr_2_get(bus),
+		~0, value);
+}
+
+void
+dhdpcie_bus_ringbell_fast(struct dhd_bus *bus, uint32 value)
+{
+	/* Skip once bus enters low power state (D3_INFORM/D3_ACK) */
+	if (__DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_RPM(("%s: trying to ring the doorbell after D3 inform %d\n",
+			__FUNCTION__, bus->bus_low_power_state));
+		return;
+	}
+
+	/* Skip in the case of link down */
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	if (OOB_DW_ENAB(bus)) {
+		dhd_bus_set_device_wake(bus, TRUE);
+	}
+	dhd_bus_doorbell_timeout_reset(bus);
+#endif /* defined(PCIE_OOB) || defined(PCIE_INB_DW) */
+#ifdef DHD_MMIO_TRACE
+	dhd_bus_mmio_trace(bus, dhd_bus_db0_addr_get(bus), value,
+		((value >> 24u) == 0xFF) ? TRUE : FALSE);
+#endif /* defined(DHD_MMIO_TRACE) */
+	if (DAR_PWRREQ(bus)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	W_REG(bus->pcie_mb_intr_osh, bus->pcie_mb_intr_addr, value);
+}
+
+void
+dhdpcie_bus_ringbell_2_fast(struct dhd_bus *bus, uint32 value, bool devwake)
+{
+	/* Skip once bus enters low power state (D3_INFORM/D3_ACK) */
+	if (__DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_ERROR(("%s: trying to ring the doorbell after D3 inform %d\n",
+			__FUNCTION__, bus->bus_low_power_state));
+		return;
+	}
+
+	/* Skip in the case of link down */
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	if (devwake) {
+		if (OOB_DW_ENAB(bus)) {
+			dhd_bus_set_device_wake(bus, TRUE);
+		}
+	}
+	dhd_bus_doorbell_timeout_reset(bus);
+#endif /* defined(PCIE_OOB) || defined(PCIE_INB_DW) */
+
+#ifdef DHD_MMIO_TRACE
+	dhd_bus_mmio_trace(bus, dhd_bus_db0_addr_2_get(bus), value, TRUE);
+#endif /* defined(DHD_MMIO_TRACE) */
+	if (DAR_PWRREQ(bus)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+	W_REG(bus->pcie_mb_intr_osh, bus->pcie_mb_intr_2_addr, value);
+}
+
+static void
+dhd_bus_ringbell_oldpcie(struct dhd_bus *bus, uint32 value)
+{
+	uint32 w;
+	/* Skip once bus enters low power state (D3_INFORM/D3_ACK) */
+	if (__DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_ERROR(("%s: trying to ring the doorbell after D3 inform %d\n",
+			__FUNCTION__, bus->bus_low_power_state));
+		return;
+	}
+
+	/* Skip in the case of link down */
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return;
+	}
+
+	w = (R_REG(bus->pcie_mb_intr_osh, bus->pcie_mb_intr_addr) & ~PCIE_INTB) | PCIE_INTB;
+	W_REG(bus->pcie_mb_intr_osh, bus->pcie_mb_intr_addr, w);
+}
+
+dhd_mb_ring_t
+dhd_bus_get_mbintr_fn(struct dhd_bus *bus)
+{
+	if ((bus->sih->buscorerev == 2) || (bus->sih->buscorerev == 6) ||
+		(bus->sih->buscorerev == 4)) {
+		bus->pcie_mb_intr_addr = si_corereg_addr(bus->sih, bus->sih->buscoreidx,
+			bus->pcie_mailbox_int);
+		if (bus->pcie_mb_intr_addr) {
+			bus->pcie_mb_intr_osh = si_osh(bus->sih);
+			return dhd_bus_ringbell_oldpcie;
+		}
+	} else {
+		bus->pcie_mb_intr_addr = si_corereg_addr(bus->sih, bus->sih->buscoreidx,
+			dhd_bus_db0_addr_get(bus));
+		if (bus->pcie_mb_intr_addr) {
+			bus->pcie_mb_intr_osh = si_osh(bus->sih);
+			return dhdpcie_bus_ringbell_fast;
+		}
+	}
+	return dhd_bus_ringbell;
+}
+
+dhd_mb_ring_2_t
+dhd_bus_get_mbintr_2_fn(struct dhd_bus *bus)
+{
+	bus->pcie_mb_intr_2_addr = si_corereg_addr(bus->sih, bus->sih->buscoreidx,
+		dhd_bus_db0_addr_2_get(bus));
+	if (bus->pcie_mb_intr_2_addr) {
+		bus->pcie_mb_intr_osh = si_osh(bus->sih);
+		return dhdpcie_bus_ringbell_2_fast;
+	}
+	return dhd_bus_ringbell_2;
+}
+
+bool
+BCMFASTPATH(dhd_bus_dpc)(struct dhd_bus *bus)
+{
+	bool resched = FALSE;	  /* Flag indicating resched wanted */
+	unsigned long flags;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	bus->dpc_entry_time = OSL_LOCALTIME_NS();
+
+	if (dhd_query_bus_erros(bus->dhd)) {
+		if (bus->dhd->db7_trap.fw_db7w_trap_inprogress == FALSE) {
+			DHD_ERROR_RLMT(("%s: db7 not in progress, exit\n", __FUNCTION__));
+			return 0;
+		}
+	}
+
+	/* Due to irq mismatch WARNING in linux, currently keeping it disabled and
+	 * using dongle intmask to control INTR enable/disable
+	 */
+	if (bus->d2h_intr_control == PCIE_HOST_IRQ_CTRL) {
+		/*
+		 * Disable IRQ at start of DPC if it is not disabled and
+		 * Enable back at the end of dpc if irq is disabled.
+		 */
+		if (!dhdpcie_irq_disabled(bus)) {
+			bus->host_irq_disable_count++;
+			dhdpcie_disable_irq_nosync(bus); /* Disable host IRQ!! */
+		}
+	}
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	/* Check for only DHD_BUS_DOWN and not for DHD_BUS_DOWN_IN_PROGRESS
+	 * to avoid IOCTL Resumed On timeout when ioctl is waiting for response
+	 * and rmmod is fired in parallel, which will make DHD_BUS_DOWN_IN_PROGRESS
+	 * and if we return from here, then IOCTL response will never be handled
+	 */
+	if (bus->dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s: Bus down, ret\n", __FUNCTION__));
+		bus->intstatus = 0;
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		bus->dpc_return_busdown_count++;
+		return 0;
+	}
+	DHD_BUS_BUSY_SET_IN_DPC(bus->dhd);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+#ifdef DHD_READ_INTSTATUS_IN_DPC
+	if (bus->ipend) {
+		bus->ipend = FALSE;
+		bus->intstatus = dhdpcie_bus_intstatus(bus);
+		/* Check if the interrupt is ours or not */
+		if (bus->intstatus == 0) {
+			goto INTR_ON;
+		}
+		bus->intrcount++;
+	}
+#endif /* DHD_READ_INTSTATUS_IN_DPC */
+
+	resched = dhdpcie_bus_process_mailbox_intr(bus, bus->intstatus);
+	if (!resched) {
+		bus->intstatus = 0;
+#ifdef DHD_READ_INTSTATUS_IN_DPC
+INTR_ON:
+#endif /* DHD_READ_INTSTATUS_IN_DPC */
+		bus->dpc_exit_time = OSL_LOCALTIME_NS();
+		bus->dpc_time_usec = DIV_U64_BY_U64((bus->dpc_exit_time - bus->dpc_entry_time), NSEC_PER_USEC);
+		if (!dhd_query_bus_erros(bus->dhd)) {
+			/* Due to irq mismatch WARNING in linux, currently keeping it disabled and
+			 * using dongle intmask to control INTR enable/disable
+			 */
+			if (bus->d2h_intr_control == PCIE_HOST_IRQ_CTRL) {
+				bus->host_irq_enable_count += dhdpcie_irq_disabled(bus);
+				dhdpcie_enable_irq_loop(bus);
+			} else {
+				/* Enable back interrupt using Intmask! */
+				dhdpcie_bus_intr_enable(bus);
+				bus->dngl_intmask_enable_count++;
+			}
+		}
+	} else {
+		bus->resched_dpc_time = OSL_LOCALTIME_NS();
+		bus->dpc_time_usec = DIV_U64_BY_U64((bus->resched_dpc_time - bus->dpc_entry_time), NSEC_PER_USEC);
+	}
+	dhd_histo_update(bus->dhd, bus->dpc_time_histo, (uint32)bus->dpc_time_usec);
+
+	bus->dpc_sched = resched;
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+	if (bus->dhd->dma_h2d_ring_upd_support && bus->dhd->dma_d2h_ring_upd_support &&
+			(bus->dhd->ring_attached == TRUE)) {
+		dhd_bus_flow_ring_status_dpc_trace(bus->dhd);
+	}
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_IN_DPC(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+	return resched;
+
+}
+
+int
+dhdpcie_send_mb_data(dhd_bus_t *bus, uint32 h2d_mb_data)
+{
+	uint32 cur_h2d_mb_data = 0;
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link was down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_INFO(("%s: H2D_MB_DATA: 0x%08X\n", __FUNCTION__, h2d_mb_data));
+
+#ifdef PCIE_INB_DW
+	if (h2d_mb_data == H2D_HOST_DS_ACK) {
+		dhdpcie_set_dongle_deepsleep(bus, TRUE);
+	}
+	dhd_bus_ds_trace(bus, h2d_mb_data, FALSE, dhdpcie_bus_get_pcie_inband_dw_state(bus));
+#else
+	dhd_bus_ds_trace(bus, h2d_mb_data, FALSE);
+#endif /* PCIE_INB_DW */
+	if (bus->api.fw_rev >= PCIE_SHARED_VERSION_6 && !bus->use_mailbox) {
+		DHD_INFO(("API rev is 6, sending mb data as H2D Ctrl message to dongle, 0x%04x\n",
+			h2d_mb_data));
+		/* Prevent asserting device_wake during doorbell ring for mb data to avoid loop. */
+#ifdef PCIE_OOB
+		bus->oob_enabled = FALSE;
+#endif /* PCIE_OOB */
+		/* XXX: check the error return value here... */
+		if (dhd_prot_h2d_mbdata_send_ctrlmsg(bus->dhd, h2d_mb_data)) {
+			DHD_ERROR(("failure sending the H2D Mailbox message "
+					"to firmware\n"));
+			goto fail;
+		}
+#ifdef PCIE_OOB
+		bus->oob_enabled = TRUE;
+#endif /* PCIE_OOB */
+		goto done;
+	}
+
+	dhd_bus_cmn_readshared(bus, &cur_h2d_mb_data, H2D_MB_DATA, 0);
+
+	if (cur_h2d_mb_data != 0) {
+		uint32 i = 0;
+		DHD_INFO(("%s: GRRRRRRR: MB transaction is already pending 0x%04x\n", __FUNCTION__, cur_h2d_mb_data));
+		/* XXX: start a zero length timer to keep checking this to be zero */
+		while ((i++ < 100) && cur_h2d_mb_data) {
+			OSL_DELAY(10);
+			dhd_bus_cmn_readshared(bus, &cur_h2d_mb_data, H2D_MB_DATA, 0);
+		}
+		if (i >= 100) {
+			DHD_ERROR(("%s : waited 1ms for the dngl "
+				"to ack the previous mb transaction\n", __FUNCTION__));
+			DHD_ERROR(("%s : MB transaction is still pending 0x%04x\n",
+				__FUNCTION__, cur_h2d_mb_data));
+		}
+	}
+
+	dhd_bus_cmn_writeshared(bus, &h2d_mb_data, sizeof(uint32), H2D_MB_DATA, 0);
+	dhd_bus_gen_devmb_intr(bus);
+
+done:
+	if (h2d_mb_data == H2D_HOST_D3_INFORM) {
+		DHD_INFO_HW4(("%s: send H2D_HOST_D3_INFORM to dongle\n", __FUNCTION__));
+		bus->last_d3_inform_time = OSL_LOCALTIME_NS();
+		bus->d3_inform_cnt++;
+	}
+	if (h2d_mb_data == H2D_HOST_D0_INFORM_IN_USE) {
+		DHD_INFO_HW4(("%s: send H2D_HOST_D0_INFORM_IN_USE to dongle\n", __FUNCTION__));
+		bus->d0_inform_in_use_cnt++;
+	}
+	if (h2d_mb_data == H2D_HOST_D0_INFORM) {
+		DHD_INFO_HW4(("%s: send H2D_HOST_D0_INFORM to dongle\n", __FUNCTION__));
+		bus->d0_inform_cnt++;
+	}
+	if (h2d_mb_data == H2D_HOST_PTM_ENABLE) {
+		DHD_INFO(("%s: send H2D_HOST_PTM_ENABLE to dongle\n", __FUNCTION__));
+	}
+	if (h2d_mb_data == H2D_HOST_PTM_DISABLE) {
+		DHD_INFO(("%s: send H2D_HOST_PTM_DISABLE to dongle\n", __FUNCTION__));
+	}
+	return BCME_OK;
+fail:
+	return BCME_ERROR;
+}
+
+static void
+dhd_bus_handle_d3_ack(dhd_bus_t *bus)
+{
+	bus->dngl_intmask_disable_count++;
+
+	/* Disable dongle interrupts by intmask and clear the intstatus
+	 * if it is set to avoid unnecessary intrrupts after D3 ACK.
+	 */
+	dhdpcie_bus_intr_disable(bus); /* Disable interrupt using IntMask!! */
+	dhdpcie_bus_clear_intstatus(bus);
+
+	DHD_SET_BUS_LPS_D3_ACKED(bus);
+	DHD_RPM(("%s: D3_ACK Recieved\n", __FUNCTION__));
+
+	if (bus->dhd->dhd_induce_error == DHD_INDUCE_D3_ACK_TIMEOUT) {
+		/* Set bus_low_power_state to DHD_BUS_D3_ACK_RECIEVED */
+		DHD_ERROR(("%s: Due to d3ack induce error forcefully set "
+		"bus_low_power_state to DHD_BUS_D3_INFORM_SENT\n", __FUNCTION__));
+		DHD_SET_BUS_LPS_D3_INFORMED(bus);
+	}
+	/* Check for D3 ACK induce flag, which is set by firing dhd iovar to induce D3 Ack timeout.
+	 * If flag is set, D3 wake is skipped, which results in to D3 Ack timeout.
+	 */
+	if (bus->dhd->dhd_induce_error != DHD_INDUCE_D3_ACK_TIMEOUT) {
+		bus->wait_for_d3_ack = 1;
+		dhd_os_d3ack_wake(bus->dhd);
+	} else {
+		DHD_ERROR(("%s: Inducing D3 ACK timeout\n", __FUNCTION__));
+	}
+
+	bus->last_d3_ack_time = OSL_LOCALTIME_NS();
+}
+
+void
+dhd_bus_handle_mb_data(dhd_bus_t *bus, uint32 d2h_mb_data)
+{
+#ifdef PCIE_INB_DW
+	unsigned long flags = 0;
+#endif /* PCIE_INB_DW */
+
+	DHD_INFO(("D2H_MB_DATA: 0x%04x\n", d2h_mb_data));
+#ifdef PCIE_INB_DW
+	DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+	dhd_bus_ds_trace(bus, d2h_mb_data, TRUE, dhdpcie_bus_get_pcie_inband_dw_state(bus));
+	DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+#else
+	dhd_bus_ds_trace(bus, d2h_mb_data, TRUE);
+#endif /* PCIE_INB_DW */
+
+	if (d2h_mb_data & D2H_DEV_FWHALT) {
+		if (bus->dhd->db7_trap.fw_db7w_trap_inprogress == FALSE) {
+			DHD_ERROR(("FW trap has happened, dongle_trap_data 0x%8x\n",
+				bus->dhd->dongle_trap_data));
+		}
+
+		if (bus->dhd->dongle_trap_data & D2H_DEV_TRAP_HOSTDB) {
+			uint64 db7_dur;
+
+			bus->dhd->db7_trap.debug_db7_trap_time = OSL_LOCALTIME_NS();
+			bus->dhd->db7_trap.debug_db7_trap_cnt++;
+			db7_dur = bus->dhd->db7_trap.debug_db7_trap_time -
+				bus->dhd->db7_trap.debug_db7_send_time;
+			if (db7_dur > bus->dhd->db7_trap.debug_max_db7_dur) {
+				bus->dhd->db7_trap.debug_max_db7_send_time =
+					bus->dhd->db7_trap.debug_db7_send_time;
+				bus->dhd->db7_trap.debug_max_db7_trap_time =
+					bus->dhd->db7_trap.debug_db7_trap_time;
+			}
+			bus->dhd->db7_trap.debug_max_db7_dur =
+				MAX(bus->dhd->db7_trap.debug_max_db7_dur, db7_dur);
+			if (bus->dhd->db7_trap.fw_db7w_trap_inprogress == FALSE) {
+				bus->dhd->db7_trap.debug_db7_timing_error_cnt++;
+			}
+		} else {
+			dhdpcie_checkdied(bus, NULL, 0);
+#ifdef BCM_ROUTER_DHD
+			dhdpcie_handle_dongle_trap(bus);
+#endif
+#ifdef OEM_ANDROID
+			dhd_os_check_hang(bus->dhd, 0, -EREMOTEIO);
+#endif /* OEM_ANDROID */
+		}
+		if (bus->dhd->db7_trap.fw_db7w_trap_inprogress) {
+			bus->dhd->db7_trap.fw_db7w_trap_inprogress = FALSE;
+#if !defined(OEM_ANDROID)
+			/* Fix for UTF issue */
+			bus->dhd->dongle_trap_occured = TRUE;
+#endif
+		}
+		goto exit;
+	}
+	if (d2h_mb_data & D2H_DEV_DS_ENTER_REQ)  {
+		bool ds_acked = FALSE;
+		BCM_REFERENCE(ds_acked);
+		if (__DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+			DHD_ERROR(("DS-ENTRY AFTER D3-ACK!!!!! QUITING\n"));
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+			goto exit;
+		}
+		/* what should we do */
+		DHD_INFO(("D2H_MB_DATA: DEEP SLEEP REQ\n"));
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			/* As per inband state machine, host should not send DS-ACK
+			 * during suspend or suspend in progress, instead D3 inform will be sent.
+			 */
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			if (!bus->skip_ds_ack) {
+				if (dhdpcie_bus_get_pcie_inband_dw_state(bus)
+					== DW_DEVICE_DS_ACTIVE) {
+					dhdpcie_bus_set_pcie_inband_dw_state(bus,
+						DW_DEVICE_DS_DEV_SLEEP_PEND);
+					 if (bus->host_active_cnt == 0) {
+						dhdpcie_bus_set_pcie_inband_dw_state(bus,
+							DW_DEVICE_DS_DEV_SLEEP);
+						dhdpcie_send_mb_data(bus, H2D_HOST_DS_ACK);
+						ds_acked = TRUE;
+						DHD_INFO(("D2H_MB_DATA: sent DEEP SLEEP"
+							"ACK to DNGL\n"));
+					} else {
+						DHD_INFO(("%s: Host is active, "
+							"skip sending DS-ACK. "
+							"host_active_cnt is %d\n",
+							__FUNCTION__, bus->host_active_cnt));
+					}
+				}
+				/* Currently DW_DEVICE_HOST_SLEEP_WAIT is set only
+				 * under dhd_bus_suspend() function.
+				 */
+				else if (dhdpcie_bus_get_pcie_inband_dw_state(bus)
+				== DW_DEVICE_HOST_SLEEP_WAIT) {
+					DHD_ERROR(("%s: DS-ACK not sent due to suspend "
+						"in progress\n", __FUNCTION__));
+				} else {
+					DHD_ERROR_RLMT(("%s: Failed to send DS-ACK, DS state is %d",
+						__FUNCTION__,
+						dhdpcie_bus_get_pcie_inband_dw_state(bus)));
+				}
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				dhd_os_ds_enter_wake(bus->dhd);
+			} else {
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				DHD_INFO(("%s: Skip DS-ACK due to "
+					"suspend in progress\n", __FUNCTION__));
+			}
+		} else
+#endif /* PCIE_INB_DW */
+		{
+			dhdpcie_send_mb_data(bus, H2D_HOST_DS_ACK);
+			DHD_INFO(("D2H_MB_DATA: sent DEEP SLEEP ACK\n"));
+		}
+	}
+	if (d2h_mb_data & D2H_DEV_DS_EXIT_NOTE)  {
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			if (bus->calc_ds_exit_latency) {
+				bus->ds_exit_ts2 = OSL_SYSUPTIME_US();
+				if (bus->ds_exit_ts2 > bus->ds_exit_ts1 &&
+						bus->ds_exit_ts1 != 0)
+					bus->ds_exit_latency = bus->ds_exit_ts2 - bus->ds_exit_ts1;
+				else
+					bus->ds_exit_latency = 0;
+			}
+		}
+#endif /* PCIE_INB_DW */
+		/* what should we do */
+		DHD_INFO(("D2H_MB_DATA: DEEP SLEEP EXIT\n"));
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			if (dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+					DW_DEVICE_DS_DISABLED_WAIT) {
+				/* wake up only if some one is waiting in
+				* DW_DEVICE_DS_DISABLED_WAIT state
+				* in this case the waiter will change the state
+				* to DW_DEVICE_DS_DEV_WAKE
+				*/
+				bus->inband_ds_exit_host_cnt++;
+				/* To synchronize with the previous memory operations call wmb() */
+				OSL_SMP_WMB();
+				bus->wait_for_ds_exit = 1;
+				/* Call another wmb() to make sure before waking up the
+				 * other event value gets updated.
+				 */
+				OSL_SMP_WMB();
+				dhdpcie_bus_set_pcie_inband_dw_state(bus,
+					DW_DEVICE_DS_DEV_WAKE);
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+				dhd_os_ds_exit_wake(bus->dhd);
+			} else if (dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+					DW_DEVICE_DS_DEV_SLEEP) {
+				DHD_INFO(("recvd unsolicited DS-EXIT from dongle in DEV_SLEEP\n"));
+				/*
+				* unsolicited state change to DW_DEVICE_DS_DEV_WAKE if
+				* D2H_DEV_DS_EXIT_NOTE received in DW_DEVICE_DS_DEV_SLEEP state.
+				* This is need when dongle is woken by external events like
+				* WOW, ping ..etc
+				*/
+				bus->inband_ds_exit_device_cnt++;
+				dhdpcie_bus_set_pcie_inband_dw_state(bus,
+					DW_DEVICE_DS_DEV_WAKE);
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+			} else {
+				DHD_INFO(("D2H_MB_DATA: not in DS_DISABLED_WAIT/DS_DEV_SLEEP\n"));
+				bus->inband_ds_exit_host_cnt++;
+				DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+			}
+			/*
+			 * bus->deep_sleep is TRUE by default. deep_sleep is set to FALSE when the
+			 * dhd iovar deep_sleep is fired with value 0(user request to not enter
+			 * deep sleep). So donot attempt to go to deep sleep when user has
+			 * explicitly asked not to go to deep sleep. bus->deep_sleep is set to
+			 * TRUE when the dhd iovar deep_sleep is fired with value 1.
+			 */
+			if (bus->deep_sleep) {
+				dhd_bus_set_device_wake(bus, FALSE);
+				dhdpcie_set_dongle_deepsleep(bus, FALSE);
+			}
+		}
+#endif /* PCIE_INB_DW */
+	}
+	if (d2h_mb_data & D2HMB_DS_HOST_SLEEP_EXIT_ACK)  {
+		/* what should we do */
+		DHD_INFO(("D2H_MB_DATA: D0 ACK\n"));
+#ifdef PCIE_INB_DW
+		if (INBAND_DW_ENAB(bus)) {
+			DHD_BUS_INB_DW_LOCK(bus->inb_lock, flags);
+			if (dhdpcie_bus_get_pcie_inband_dw_state(bus) ==
+				DW_DEVICE_HOST_WAKE_WAIT) {
+				dhdpcie_bus_set_pcie_inband_dw_state(bus, DW_DEVICE_DS_ACTIVE);
+			}
+			DHD_BUS_INB_DW_UNLOCK(bus->inb_lock, flags);
+		}
+#endif /* PCIE_INB_DW */
+	}
+	if (d2h_mb_data & D2H_DEV_D3_ACK)  {
+		/* what should we do */
+		DHD_INFO_HW4(("D2H_MB_DATA: D3 ACK\n"));
+		if (!bus->wait_for_d3_ack) {
+#if defined(DHD_HANG_SEND_UP_TEST)
+			if (bus->dhd->req_hang_type == HANG_REASON_D3_ACK_TIMEOUT) {
+				DHD_ERROR(("TEST HANG: Skip to process D3 ACK\n"));
+			} else {
+				dhd_bus_handle_d3_ack(bus);
+			}
+#else /* DHD_HANG_SEND_UP_TEST */
+			dhd_bus_handle_d3_ack(bus);
+#endif /* DHD_HANG_SEND_UP_TEST */
+		}
+	}
+
+	if (d2h_mb_data & D2H_DEV_PTM_ENABLED)  {
+		DHD_INFO(("D2H_MB_DATA: PTM ENABLED\n"));
+	}
+
+	if (d2h_mb_data & D2H_DEV_PTM_DISABLED)  {
+		DHD_INFO(("D2H_MB_DATA: PTM DISABLED\n"));
+	}
+
+exit:
+	return;
+}
+
+static void
+dhdpcie_handle_mb_data(dhd_bus_t *bus)
+{
+	uint32 d2h_mb_data = 0;
+	uint32 zero = 0;
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link is down\n", __FUNCTION__));
+		return;
+	}
+
+	dhd_bus_cmn_readshared(bus, &d2h_mb_data, D2H_MB_DATA, 0);
+	if (D2H_DEV_MB_INVALIDATED(d2h_mb_data)) {
+		DHD_ERROR(("%s: Invalid D2H_MB_DATA: 0x%08x\n",
+			__FUNCTION__, d2h_mb_data));
+		goto exit;
+	}
+
+	dhd_bus_cmn_writeshared(bus, &zero, sizeof(uint32), D2H_MB_DATA, 0);
+
+	DHD_INFO_HW4(("%s: D2H_MB_DATA: 0x%04x\n", __FUNCTION__, d2h_mb_data));
+	if (d2h_mb_data & D2H_DEV_FWHALT)  {
+		DHD_ERROR(("FW trap has happened\n"));
+		dhdpcie_checkdied(bus, NULL, 0);
+		/* not ready yet dhd_os_ind_firmware_stall(bus->dhd); */
+#ifdef BCM_ROUTER_DHD
+		dhdpcie_handle_dongle_trap(bus);
+#endif
+		goto exit;
+	}
+	if (d2h_mb_data & D2H_DEV_DS_ENTER_REQ)  {
+		/* what should we do */
+		DHD_INFO(("%s: D2H_MB_DATA: DEEP SLEEP REQ\n", __FUNCTION__));
+		dhdpcie_send_mb_data(bus, H2D_HOST_DS_ACK);
+		DHD_INFO(("%s: D2H_MB_DATA: sent DEEP SLEEP ACK\n", __FUNCTION__));
+	}
+	if (d2h_mb_data & D2H_DEV_DS_EXIT_NOTE)  {
+		/* what should we do */
+		DHD_INFO(("%s: D2H_MB_DATA: DEEP SLEEP EXIT\n", __FUNCTION__));
+	}
+	if (d2h_mb_data & D2H_DEV_D3_ACK)  {
+		/* what should we do */
+		DHD_INFO_HW4(("%s: D2H_MB_DATA: D3 ACK\n", __FUNCTION__));
+		if (!bus->wait_for_d3_ack) {
+#if defined(DHD_HANG_SEND_UP_TEST)
+			if (bus->dhd->req_hang_type == HANG_REASON_D3_ACK_TIMEOUT) {
+				DHD_ERROR(("TEST HANG: Skip to process D3 ACK\n"));
+			} else {
+			dhd_bus_handle_d3_ack(bus);
+			}
+#else /* DHD_HANG_SEND_UP_TEST */
+			dhd_bus_handle_d3_ack(bus);
+#endif /* DHD_HANG_SEND_UP_TEST */
+		}
+	}
+
+exit:
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+}
+
+static void
+dhdpcie_read_handle_mb_data(dhd_bus_t *bus)
+{
+	uint32 d2h_mb_data = 0;
+	uint32 zero = 0;
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	dhd_bus_cmn_readshared(bus, &d2h_mb_data, D2H_MB_DATA, 0);
+	if (!d2h_mb_data) {
+		goto exit;
+	}
+
+	dhd_bus_cmn_writeshared(bus, &zero, sizeof(uint32), D2H_MB_DATA, 0);
+
+	dhd_bus_handle_mb_data(bus, d2h_mb_data);
+
+exit:
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+}
+
+#define DHD_SCHED_RETRY_DPC_DELAY_MS 100u
+
+static void
+dhd_bus_handle_intx_ahead_dma_indices(dhd_bus_t *bus)
+{
+	if (bus->d2h_intr_method == PCIE_MSI) {
+		DHD_INFO(("%s: not required for msi\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->dhd->dma_d2h_ring_upd_support == FALSE) {
+		DHD_INFO(("%s: not required for non-dma-indices\n", __FUNCTION__));
+		return;
+	}
+
+	if (dhd_query_bus_erros(bus->dhd)) {
+		return;
+	}
+
+#ifndef NDIS
+	/*
+	 * skip delayed dpc if tasklet is scheduled by non-ISR
+	 * From ISR context, if dpc is scheduled, the isr_sched_dpc_time
+	 * will be lesser than dpc_exit_time.
+	 */
+	if (bus->isr_sched_dpc_time < bus->dpc_exit_time) {
+		DHD_INFO(("%s: skip delayed dpc as tasklet is scheduled from non isr\n",
+			__FUNCTION__));
+		return;
+	}
+#endif /* NDIS */
+
+	if (DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+		DHD_INFO(("%s: skip delayed dpc as d3 ack is received\n", __FUNCTION__));
+		return;
+	}
+
+	DHD_INFO(("%s: schedule delayed DPC\n", __FUNCTION__));
+	dhd_schedule_delayed_dpc_on_dpc_cpu(bus->dhd, DHD_SCHED_RETRY_DPC_DELAY_MS);
+	return;
+}
+
+static bool
+dhdpcie_bus_process_mailbox_intr(dhd_bus_t *bus, uint32 intstatus)
+{
+	bool resched = FALSE;
+
+	/* Do not process dpc after receiving D3_ACK */
+	if (DHD_CHK_BUS_LPS_D3_ACKED(bus)) {
+		DHD_RPM(("%s: D3 Ack Received, skip dpc\n", __FUNCTION__));
+		return resched;
+	}
+
+	if ((bus->sih->buscorerev == 2) || (bus->sih->buscorerev == 6) ||
+		(bus->sih->buscorerev == 4)) {
+		/* Msg stream interrupt */
+		if (intstatus & I_BIT1) {
+			resched = dhdpci_bus_read_frames(bus);
+		} else if (intstatus & I_BIT0) {
+			/* do nothing for Now */
+		}
+	} else {
+		if (intstatus & (PCIE_MB_TOPCIE_FN0_0 | PCIE_MB_TOPCIE_FN0_1))
+			bus->api.handle_mb_data(bus);
+
+		/* The fact that we are here implies that dhdpcie_bus_intstatus( )
+		* retuned a non-zer0 status after applying the current mask.
+		* No further check required, in fact bus->instatus can be eliminated.
+		* Both bus->instatus, and bud->intdis are shared between isr and dpc.
+		*/
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+		if (pm_runtime_get(dhd_bus_to_dev(bus)) >= 0) {
+			resched = dhdpci_bus_read_frames(bus);
+			pm_runtime_mark_last_busy(dhd_bus_to_dev(bus));
+			pm_runtime_put_autosuspend(dhd_bus_to_dev(bus));
+		}
+#else
+		resched = dhdpci_bus_read_frames(bus);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+	}
+
+	dhd_bus_handle_intx_ahead_dma_indices(bus);
+
+	return resched;
+}
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+static void
+dhdpci_bus_rte_log_time_sync_poll(dhd_bus_t *bus)
+{
+	unsigned long time_elapsed;
+
+	/* Poll for timeout value periodically */
+	if ((bus->dhd->busstate == DHD_BUS_DATA) &&
+		(bus->dhd->dhd_rte_time_sync_ms != 0) &&
+		DHD_CHK_BUS_NOT_IN_LPS(bus)) {
+		/*
+		 * XXX OSL_SYSUPTIME_US() overflow should not happen.
+		 * As it is a unsigned 64 bit value 18446744073709551616L,
+		 * which needs 213503982334 days to overflow
+		 */
+		time_elapsed = OSL_SYSUPTIME_US() - bus->dhd_rte_time_sync_count;
+		/* Compare time is milli seconds */
+		if ((time_elapsed / 1000) >= bus->dhd->dhd_rte_time_sync_ms) {
+			/*
+			 * Its fine, if it has crossed the timeout value. No need to adjust the
+			 * elapsed time
+			 */
+			bus->dhd_rte_time_sync_count += time_elapsed;
+
+			/* Schedule deffered work. Work function will send IOVAR. */
+			dhd_h2d_log_time_sync_deferred_wq_schedule(bus->dhd);
+		}
+	}
+}
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+static bool
+dhdpci_bus_read_frames(dhd_bus_t *bus)
+{
+	bool more = FALSE;
+	uint32 ctrlcpl_items = 0;
+	uint32 txcpl_items = 0;
+	uint32 rxcpl_items = 0;
+	uint32 evtlog_items = 0;
+	uint64 read_frames_entry_time = OSL_LOCALTIME_NS();
+
+	/* First check if there a FW trap */
+	if ((bus->api.fw_rev >= PCIE_SHARED_VERSION_6) &&
+		(bus->dhd->dongle_trap_data = dhd_prot_process_trapbuf(bus->dhd))) {
+#ifdef DNGL_AXI_ERROR_LOGGING
+		if (bus->dhd->axi_error) {
+			DHD_ERROR(("AXI Error happened\n"));
+			return FALSE;
+		}
+#endif /* DNGL_AXI_ERROR_LOGGING */
+		dhd_bus_handle_mb_data(bus, D2H_DEV_FWHALT);
+		return FALSE;
+	}
+
+	if (dhd_query_bus_erros(bus->dhd)) {
+		DHD_ERROR(("%s: detected bus errors. Hence donot process msg rings\n",
+			__FUNCTION__));
+		return FALSE;
+	}
+#ifdef DHD_DMA_INDICES_SEQNUM
+	dhd_prot_save_dmaidx(bus->dhd);
+#endif /* DHD_DMA_INDICES_SEQNUM */
+	/* There may be frames in both ctrl buf and data buf; check ctrl buf first */
+	more |= dhd_prot_process_ctrlbuf(bus->dhd, &ctrlcpl_items);
+	bus->last_process_ctrlbuf_time = OSL_LOCALTIME_NS();
+
+	bus->ctrl_cpl_post_time_usec =
+		DIV_U64_BY_U64((bus->last_process_ctrlbuf_time - read_frames_entry_time), NSEC_PER_USEC);
+	dhd_histo_update(bus->dhd, bus->ctrl_cpl_post_time_histo,
+		(uint32)bus->ctrl_cpl_post_time_usec);
+
+	/* Do not process rest of ring buf once bus enters low power state (D3_INFORM/D3_ACK) */
+	if (DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_RPM(("%s: Bus is in power save state (%d). "
+			"Skip processing rest of ring buffers.\n",
+			__FUNCTION__, bus->bus_low_power_state));
+		return FALSE;
+	}
+
+	/* update the flow ring cpls */
+	more |= dhd_update_txflowrings(bus->dhd);
+	bus->last_process_flowring_time = OSL_LOCALTIME_NS();
+
+	bus->tx_post_time_usec =
+		DIV_U64_BY_U64((bus->last_process_flowring_time - bus->last_process_ctrlbuf_time), NSEC_PER_USEC);
+	dhd_histo_update(bus->dhd, bus->tx_post_time_histo, (uint32)bus->tx_post_time_usec);
+
+	/* With heavy TX traffic, we could get a lot of TxStatus
+	 * so add bound
+	 */
+	more |= dhd_prot_process_msgbuf_txcpl(bus->dhd, DHD_REGULAR_RING, &txcpl_items);
+	bus->last_process_txcpl_time = OSL_LOCALTIME_NS();
+
+	bus->tx_cpl_time_usec =
+		DIV_U64_BY_U64((bus->last_process_txcpl_time - bus->last_process_flowring_time), NSEC_PER_USEC);
+	dhd_histo_update(bus->dhd, bus->tx_cpl_time_histo, (uint32)bus->tx_cpl_time_usec);
+
+	/* With heavy RX traffic, this routine potentially could spend some time
+	 * processing RX frames without RX bound
+	 */
+	more |= dhd_prot_process_msgbuf_rxcpl(bus->dhd, DHD_REGULAR_RING, &rxcpl_items);
+	bus->last_process_rxcpl_time = OSL_LOCALTIME_NS();
+
+	bus->rx_cpl_post_time_usec =
+		DIV_U64_BY_U64((bus->last_process_rxcpl_time - bus->last_process_txcpl_time), NSEC_PER_USEC);
+	dhd_histo_update(bus->dhd, bus->rx_cpl_post_time_histo, (uint32)bus->rx_cpl_post_time_usec);
+
+	/* Process info ring completion messages */
+#ifdef EWP_EDL
+	if (!bus->dhd->dongle_edl_support)
+#endif
+	{
+		more |= dhd_prot_process_msgbuf_infocpl(bus->dhd, DHD_INFORING_BOUND,
+			&evtlog_items);
+		bus->last_process_infocpl_time = OSL_LOCALTIME_NS();
+	}
+#ifdef EWP_EDL
+	else {
+		more |= dhd_prot_process_msgbuf_edl(bus->dhd, &evtlog_items);
+		bus->last_process_edl_time = OSL_LOCALTIME_NS();
+	}
+#endif /* EWP_EDL */
+
+#ifdef IDLE_TX_FLOW_MGMT
+	if (bus->enable_idle_flowring_mgmt) {
+		/* Look for idle flow rings */
+		dhd_bus_check_idle_scan(bus);
+	}
+#endif /* IDLE_TX_FLOW_MGMT */
+
+	/* don't talk to the dongle if fw is about to be reloaded */
+	if (bus->dhd->hang_was_sent) {
+		more = FALSE;
+	}
+
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+	dhdpci_bus_rte_log_time_sync_poll(bus);
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+
+#if defined(DHD_WAKE_STATUS)
+	/* Check if host was woken up by any packets */
+	if (dhd_bus_get_bus_wake(bus->dhd) > 0) {
+		/*
+		 * If wake is due to Rx packets,
+		 * pktwake info will be printed and cleared from dhd_rx_frame()
+		 */
+		DHD_ERROR(("#### dhdpcie_host_wake: rxcpl:%d ctrlcpl:%d txcpl:%d evtlog:%d ####\n",
+			rxcpl_items, ctrlcpl_items, txcpl_items, evtlog_items));
+
+		dhd_bus_set_get_bus_wake(bus->dhd, 0);
+
+		if (rxcpl_items > 0) {
+			/* Request packet dump for first Rx packet */
+			dhd_bus_set_get_bus_wake_pkt_dump(bus->dhd, 1);
+		}
+	}
+#endif /* DHD_WAKE_STATUS */
+	return more;
+}
+
+bool
+dhdpcie_tcm_valid(dhd_bus_t *bus)
+{
+	uint32 addr = 0;
+	int rv;
+	uint32 shaddr = 0;
+	pciedev_shared_t sh;
+
+	shaddr = bus->dongle_ram_base + bus->ramsize - 4;
+
+	/* Read last word in memory to determine address of pciedev_shared structure */
+	addr = LTOH32(dhdpcie_bus_rtcm32(bus, shaddr));
+
+	if ((addr == 0) || (addr == bus->nvram_csm) || (addr < bus->dongle_ram_base) ||
+		(addr > shaddr)) {
+		DHD_ERROR(("%s: address (0x%08x) of pciedev_shared invalid addr\n",
+			__FUNCTION__, addr));
+		return FALSE;
+	}
+
+	/* Read hndrte_shared structure */
+	if ((rv = dhdpcie_bus_membytes(bus, FALSE, addr, (uint8 *)&sh,
+		sizeof(pciedev_shared_t))) < 0) {
+		DHD_ERROR(("Failed to read PCIe shared struct with %d\n", rv));
+		return FALSE;
+	}
+
+	/* Compare any field in pciedev_shared_t */
+	if (sh.console_addr != bus->pcie_sh->console_addr) {
+		DHD_ERROR(("Contents of pciedev_shared_t structure are not matching.\n"));
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static void
+dhdpcie_update_bus_api_revisions(uint32 firmware_api_version, uint32 host_api_version)
+{
+	snprintf(bus_api_revision, BUS_API_REV_STR_LEN, "\nBus API revisions:(FW rev%d)(DHD rev%d)",
+			firmware_api_version, host_api_version);
+	return;
+}
+
+static bool
+dhdpcie_check_firmware_compatible(uint32 firmware_api_version, uint32 host_api_version)
+{
+	bool retcode = FALSE;
+
+	DHD_INFO(("firmware api revision %d, host api revision %d\n",
+		firmware_api_version, host_api_version));
+
+	switch (firmware_api_version) {
+	case PCIE_SHARED_VERSION_7:
+	case PCIE_SHARED_VERSION_6:
+	case PCIE_SHARED_VERSION_5:
+		retcode = TRUE;
+		break;
+	default:
+		if (firmware_api_version <= host_api_version)
+			retcode = TRUE;
+	}
+	return retcode;
+}
+
+static int
+dhdpcie_readshared(dhd_bus_t *bus)
+{
+	uint32 addr = 0;
+	int dma_indx_wr_buf, dma_indx_rd_buf;
+	uint32 shaddr = 0;
+	pciedev_shared_t *sh = bus->pcie_sh;
+	dhd_timeout_t tmo;
+	bool idma_en = FALSE;
+#if defined(PCIE_INB_DW)
+	bool d2h_inband_dw = FALSE;
+#endif /* defined(PCIE_INB_DW) */
+#if defined(PCIE_OOB)
+	bool d2h_no_oob_dw = FALSE;
+#endif /* defined(PCIE_INB_DW) */
+	uint32 timeout = MAX_READ_TIMEOUT;
+	uint32 elapsed;
+	uint32 intstatus;
+	int ret = BCME_OK;
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	shaddr = bus->dongle_ram_base + bus->ramsize - 4;
+
+#ifdef BCMSLTGT
+#ifdef BCMQT_HW
+	if (qt_dngl_timeout) {
+		timeout = qt_dngl_timeout * 1000;
+	}
+#endif /* BCMQT_HW */
+	DHD_ERROR(("%s: host timeout in QT/FPGA mode %ld ms\n",
+		__FUNCTION__, (timeout * htclkratio) / USEC_PER_MSEC));
+#endif /* BCMSLTGT */
+#ifdef GDB_PROXY
+	if (bus->dhd->gdb_proxy_active) {
+		/* +3s ASLR delay timeout for case of preattachment debugging */
+		timeout += 3 * 1000 * 1000;
+	}
+#endif /* GDB_PROXY */
+
+#ifdef GDB_PROXY
+	/* Loop while timeout is caused by firmware stop in GDB */
+	GDB_PROXY_TIMEOUT_DO(bus->dhd)
+	{
+#endif /* GDB_PROXY */
+		/* start a timer for 5 seconds */
+		dhd_timeout_start(&tmo, timeout);
+
+		while (((addr == 0) || (addr == bus->nvram_csm)) && !dhd_timeout_expired(&tmo)) {
+			/* Read last word in memory to determine address of pciedev_shared structure
+			 */
+			addr = LTOH32(dhdpcie_bus_rtcm32(bus, shaddr));
+			if (addr != bus->nvram_csm) {
+				DHD_ERROR(("%s: addr=0x%x nvram_csm=0x%x\n",
+					__FUNCTION__, addr, bus->nvram_csm));
+			}
+		}
+#ifdef GDB_PROXY
+	}
+	GDB_PROXY_TIMEOUT_WHILE(dhd_timeout_expired(&tmo))
+#endif /* GDB_PROXY */
+	if (addr == (uint32)-1) {
+		DHD_ERROR(("%s: ##### pciedev shared address is 0xffffffff ####\n", __FUNCTION__));
+
+/* In phones sometimes NOC is seen for any further access after readshared fails */
+#ifdef DHD_NO_DUMP_FOR_READSHARED_FAIL
+		DHD_ERROR(("%s:### NO dumps will be colelcted and will be marked as linkdown ###\n",
+			__FUNCTION__));
+		bus->is_linkdown = 1;
+		ret = BCME_ERROR;
+		goto done;
+#endif /* DHD_NO_DUMP_FOR_READSHARED_FAIL */
+
+		dhd_bus_dump_imp_cfg_registers(bus);
+		dhd_bus_dump_dar_registers(bus);
+		/* Check the PCIe link status by reading intstatus register */
+		intstatus = si_corereg(bus->sih,
+				bus->sih->buscoreidx, bus->pcie_mailbox_int, 0, 0);
+		if (intstatus == (uint32)-1) {
+			DHD_ERROR(("%s : PCIe link might be down\n", __FUNCTION__));
+			bus->is_linkdown = TRUE;
+		} else {
+#if defined(DHD_FW_COREDUMP)
+#ifdef DHD_SSSR_DUMP
+#ifdef OEM_ANDROID
+			DHD_ERROR(("%s : Set collect_sssr and fis_enab as TRUE\n", __FUNCTION__));
+			bus->dhd->collect_sssr = TRUE;
+			fis_enab = TRUE;
+#endif /* OEM_ANDROID */
+#endif /* DHD_SSSR_DUMP */
+			/* save core dump or write to a file */
+			if (bus->dhd->memdump_enabled) {
+				bus->dhd->memdump_type = DUMP_TYPE_READ_SHM_FAIL;
+				dhdpcie_mem_dump(bus);
+			}
+#endif /* DHD_FW_COREDUMP */
+		}
+		ret = BCME_ERROR;
+		goto done;
+	}
+
+	if ((addr == 0) || (addr == bus->nvram_csm) || (addr < bus->dongle_ram_base) ||
+		(addr > shaddr)) {
+		uint origidx = si_coreidx(bus->sih);
+		if (si_setcore(bus->sih, ARMCA7_CORE_ID, 0)) {
+			uint32 ioctrl = si_wrapperreg(bus->sih, AI_IOCTRL, 0, 0);
+			uint32 resetctrl = si_wrapperreg(bus->sih, AI_RESETCTRL, 0, 0);
+			DHD_ERROR(("%s: ca7 ioctrl:0x%x resetctrl:0x%x\n",
+				__FUNCTION__, ioctrl, resetctrl));
+		} else {
+			DHD_ERROR(("%s: Failed to find CA7 core!\n", __FUNCTION__));
+		}
+
+		/* Restore back to original core */
+		si_setcoreidx(bus->sih, origidx);
+
+		elapsed = tmo.elapsed;
+#ifdef BCMSLTGT
+		elapsed *= htclkratio;
+#endif /* BCMSLTGT */
+		DHD_ERROR(("%s: address (0x%08x) of pciedev_shared invalid\n",
+			__FUNCTION__, addr));
+		DHD_ERROR(("%s: Waited %u usec, dongle is not ready\n", __FUNCTION__, elapsed));
+#ifndef OEM_ANDROID
+		if (addr != (uint32)-1) {	/* skip further PCIE reads if read this addr */
+#ifdef DHD_SSSR_DUMP
+			if (!bus->dhd->sssr_inited) {
+				bus->dhd->force_sssr_init = TRUE;
+				/* Hard code sssr_reg_info for 4389 */
+				DHD_SSSR_DUMP_INIT(bus->dhd);
+			}
+			bus->dhd->collect_sssr = TRUE;
+#endif /* DHD_SSSR_DUMP */
+			if (bus->dhd->memdump_enabled) {
+				bus->dhd->memdump_type = DUMP_TYPE_DONGLE_INIT_FAILURE;
+				dhdpcie_mem_dump(bus);
+			}
+		}
+#endif /* !OEM_ANDROID */
+#if defined(NDIS)
+		/* This is a very common code path to catch f/w init failures.
+		   Capture a socram dump.
+		*/
+		ASSERT(0);
+#endif /* defined(NDIS) */
+		ret = BCME_ERROR;
+		goto done;
+	} else {
+		bus->rd_shared_pass_time = OSL_LOCALTIME_NS();
+		elapsed = tmo.elapsed;
+#ifdef BCMSLTGT
+		elapsed *= htclkratio;
+#endif /* BCMSLTGT */
+		bus->shared_addr = (ulong)addr;
+		DHD_ERROR(("### Total time ARM OOR to Readshared pass took %llu usec ###\n",
+			DIV_U64_BY_U32((bus->rd_shared_pass_time - bus->arm_oor_time),
+			NSEC_PER_USEC)));
+		DHD_ERROR(("%s: PCIe shared addr (0x%08x) read took %u usec "
+			"before dongle is ready\n", __FUNCTION__, addr, elapsed));
+	}
+
+	/* Read hndrte_shared structure */
+	if ((ret = dhdpcie_bus_membytes(bus, FALSE, addr, (uint8 *)sh,
+		sizeof(pciedev_shared_t))) < 0) {
+		DHD_ERROR(("%s: Failed to read PCIe shared struct with %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+	/* Endianness */
+	sh->flags = ltoh32(sh->flags);
+	sh->trap_addr = ltoh32(sh->trap_addr);
+	sh->assert_exp_addr = ltoh32(sh->assert_exp_addr);
+	sh->assert_file_addr = ltoh32(sh->assert_file_addr);
+	sh->assert_line = ltoh32(sh->assert_line);
+	sh->console_addr = ltoh32(sh->console_addr);
+	sh->msgtrace_addr = ltoh32(sh->msgtrace_addr);
+	sh->dma_rxoffset = ltoh32(sh->dma_rxoffset);
+	sh->rings_info_ptr = ltoh32(sh->rings_info_ptr);
+	sh->flags2 = ltoh32(sh->flags2);
+
+	/* load bus console address */
+	bus->console_addr = sh->console_addr;
+
+	/* Read the dma rx offset */
+	bus->dma_rxoffset = bus->pcie_sh->dma_rxoffset;
+	dhd_prot_rx_dataoffset(bus->dhd, bus->dma_rxoffset);
+
+	DHD_INFO(("%s: DMA RX offset from shared Area %d\n", __FUNCTION__, bus->dma_rxoffset));
+
+	bus->api.fw_rev = sh->flags & PCIE_SHARED_VERSION_MASK;
+	if (!(dhdpcie_check_firmware_compatible(bus->api.fw_rev, PCIE_SHARED_VERSION)))
+	{
+		DHD_ERROR(("%s: pcie_shared version %d in dhd "
+		           "is older than pciedev_shared version %d in dongle\n",
+		           __FUNCTION__, PCIE_SHARED_VERSION,
+		           bus->api.fw_rev));
+		ret = BCME_ERROR;
+		goto done;
+	}
+	dhdpcie_update_bus_api_revisions(bus->api.fw_rev, PCIE_SHARED_VERSION);
+
+	bus->rw_index_sz = (sh->flags & PCIE_SHARED_2BYTE_INDICES) ?
+		sizeof(uint16) : sizeof(uint32);
+	DHD_INFO(("%s: Dongle advertizes %d size indices\n",
+		__FUNCTION__, bus->rw_index_sz));
+
+#ifdef IDLE_TX_FLOW_MGMT
+	if (sh->flags & PCIE_SHARED_IDLE_FLOW_RING) {
+		DHD_ERROR(("%s: FW Supports IdleFlow ring managment!\n",
+			__FUNCTION__));
+		bus->enable_idle_flowring_mgmt = TRUE;
+	}
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#ifdef PCIE_OOB
+	bus->dhd->d2h_no_oob_dw = (sh->flags & PCIE_SHARED_NO_OOB_DW) ? TRUE : FALSE;
+	d2h_no_oob_dw = bus->dhd->d2h_no_oob_dw;
+#endif /* PCIE_OOB */
+
+#ifdef PCIE_INB_DW
+	bus->dhd->d2h_inband_dw = (sh->flags & PCIE_SHARED_INBAND_DS) ? TRUE : FALSE;
+	d2h_inband_dw = bus->dhd->d2h_inband_dw;
+#endif /* PCIE_INB_DW */
+
+#if defined(PCIE_INB_DW)
+	DHD_ERROR(("FW supports Inband dw ? %s\n",
+		d2h_inband_dw ? "Y":"N"));
+#endif /* defined(PCIE_INB_DW) */
+
+#if defined(PCIE_OOB)
+	DHD_ERROR(("FW supports oob dw ? %s\n",
+		d2h_no_oob_dw ? "N":"Y"));
+#endif /* defined(PCIE_OOB) */
+
+	if (IDMA_CAPABLE(bus)) {
+		if (bus->sih->buscorerev == 23) {
+#ifdef PCIE_INB_DW
+			if (bus->dhd->d2h_inband_dw)
+			{
+				idma_en = TRUE;
+			}
+#endif /* PCIE_INB_DW */
+		} else {
+			idma_en = TRUE;
+		}
+	}
+
+	if (idma_en) {
+		bus->dhd->idma_enable = (sh->flags & PCIE_SHARED_IDMA) ? TRUE : FALSE;
+		bus->dhd->ifrm_enable = (sh->flags & PCIE_SHARED_IFRM) ? TRUE : FALSE;
+	}
+
+	bus->dhd->d2h_sync_mode = sh->flags & PCIE_SHARED_D2H_SYNC_MODE_MASK;
+
+	bus->dhd->dar_enable = (sh->flags & PCIE_SHARED_DAR) ? TRUE : FALSE;
+	DHD_ERROR(("FW supports DAR ? %s\n", bus->dhd->dar_enable ? "Y":"N"));
+
+	/* Does the FW support DMA'ing r/w indices */
+	if (sh->flags & PCIE_SHARED_DMA_INDEX) {
+		if (!bus->dhd->dma_ring_upd_overwrite) {
+#if defined(BCM_ROUTER_DHD)
+		/* Router platform does not use IOV_DMA_RINGINDICES */
+			if (sh->flags & PCIE_SHARED_2BYTE_INDICES)
+#endif /* BCM_ROUTER_DHD */
+			{
+				if (!IFRM_ENAB(bus->dhd)) {
+					bus->dhd->dma_h2d_ring_upd_support = TRUE;
+				}
+				bus->dhd->dma_d2h_ring_upd_support = TRUE;
+			}
+		}
+
+		DHD_INFO(("%s: Host support DMAing indices: H2D:%d - D2H:%d. FW supports it\n",
+			__FUNCTION__,
+			(bus->dhd->dma_h2d_ring_upd_support ? 1 : 0),
+			(bus->dhd->dma_d2h_ring_upd_support ? 1 : 0)));
+	} else if (!(sh->flags & PCIE_SHARED_D2H_SYNC_MODE_MASK)) {
+		DHD_ERROR(("%s FW has to support either dma indices or d2h sync\n",
+			__FUNCTION__));
+		ret = BCME_UNSUPPORTED;
+		goto done;
+	} else {
+		bus->dhd->dma_h2d_ring_upd_support = FALSE;
+		bus->dhd->dma_d2h_ring_upd_support = FALSE;
+	}
+
+	/* Does the firmware support fast delete ring? */
+	if (sh->flags2 & PCIE_SHARED2_FAST_DELETE_RING) {
+		DHD_INFO(("%s: Firmware supports fast delete ring\n",
+			__FUNCTION__));
+		bus->dhd->fast_delete_ring_support = TRUE;
+	} else {
+		DHD_INFO(("%s: Firmware does not support fast delete ring\n",
+			__FUNCTION__));
+		bus->dhd->fast_delete_ring_support = FALSE;
+	}
+
+	/* get ring_info, ring_state and mb data ptrs and store the addresses in bus structure */
+	{
+		ring_info_t  ring_info;
+
+		/* boundary check */
+		if ((sh->rings_info_ptr < bus->dongle_ram_base) || (sh->rings_info_ptr > shaddr)) {
+			DHD_ERROR(("%s: rings_info_ptr is invalid 0x%x, skip reading ring info\n",
+				__FUNCTION__, sh->rings_info_ptr));
+			ret = BCME_ERROR;
+			goto done;
+		}
+
+		if ((ret = dhdpcie_bus_membytes(bus, FALSE, sh->rings_info_ptr,
+			(uint8 *)&ring_info, sizeof(ring_info_t))) < 0) {
+			goto done;
+		}
+
+		bus->h2d_mb_data_ptr_addr = ltoh32(sh->h2d_mb_data_ptr);
+		bus->d2h_mb_data_ptr_addr = ltoh32(sh->d2h_mb_data_ptr);
+
+		if (bus->api.fw_rev >= PCIE_SHARED_VERSION_6) {
+			bus->max_tx_flowrings = ltoh16(ring_info.max_tx_flowrings);
+			bus->max_submission_rings = ltoh16(ring_info.max_submission_queues);
+			bus->max_completion_rings = ltoh16(ring_info.max_completion_rings);
+			bus->max_cmn_rings = bus->max_submission_rings - bus->max_tx_flowrings;
+			bus->api.handle_mb_data = dhdpcie_read_handle_mb_data;
+			bus->use_mailbox = sh->flags & PCIE_SHARED_USE_MAILBOX;
+		}
+		else {
+			bus->max_tx_flowrings = ltoh16(ring_info.max_tx_flowrings);
+			bus->max_submission_rings = bus->max_tx_flowrings;
+			bus->max_completion_rings = BCMPCIE_D2H_COMMON_MSGRINGS;
+			bus->max_cmn_rings = BCMPCIE_H2D_COMMON_MSGRINGS;
+			bus->api.handle_mb_data = dhdpcie_handle_mb_data;
+			bus->use_mailbox = TRUE;
+		}
+		if (bus->max_completion_rings == 0) {
+			DHD_ERROR(("dongle completion rings are invalid %d\n",
+				bus->max_completion_rings));
+			ret = BCME_ERROR;
+			goto done;
+		}
+		if (bus->max_submission_rings == 0) {
+			DHD_ERROR(("dongle submission rings are invalid %d\n",
+				bus->max_submission_rings));
+			ret = BCME_ERROR;
+			goto done;
+		}
+		if (bus->max_tx_flowrings == 0) {
+			DHD_ERROR(("dongle txflow rings are invalid %d\n", bus->max_tx_flowrings));
+			ret = BCME_ERROR;
+			goto done;
+		}
+
+		/* If both FW and Host support DMA'ing indices, allocate memory and notify FW
+		 * The max_sub_queues is read from FW initialized ring_info
+		 */
+		if (bus->dhd->dma_h2d_ring_upd_support || IDMA_ENAB(bus->dhd)) {
+			dma_indx_wr_buf = dhd_prot_dma_indx_init(bus->dhd, bus->rw_index_sz,
+				H2D_DMA_INDX_WR_BUF, bus->max_submission_rings);
+			dma_indx_rd_buf = dhd_prot_dma_indx_init(bus->dhd, bus->rw_index_sz,
+				D2H_DMA_INDX_RD_BUF, bus->max_completion_rings);
+
+			if ((dma_indx_wr_buf != BCME_OK) || (dma_indx_rd_buf != BCME_OK)) {
+				DHD_ERROR(("%s: Failed to allocate memory for dma'ing h2d indices"
+						"Host will use w/r indices in TCM\n",
+						__FUNCTION__));
+				bus->dhd->dma_h2d_ring_upd_support = FALSE;
+				bus->dhd->idma_enable = FALSE;
+			}
+		}
+
+		if (bus->dhd->dma_d2h_ring_upd_support) {
+			dma_indx_wr_buf = dhd_prot_dma_indx_init(bus->dhd, bus->rw_index_sz,
+				D2H_DMA_INDX_WR_BUF, bus->max_completion_rings);
+			dma_indx_rd_buf = dhd_prot_dma_indx_init(bus->dhd, bus->rw_index_sz,
+				H2D_DMA_INDX_RD_BUF, bus->max_submission_rings);
+
+			if ((dma_indx_wr_buf != BCME_OK) || (dma_indx_rd_buf != BCME_OK)) {
+				DHD_ERROR(("%s: Failed to allocate memory for dma'ing d2h indices"
+						"Host will use w/r indices in TCM\n",
+						__FUNCTION__));
+				bus->dhd->dma_d2h_ring_upd_support = FALSE;
+			}
+		}
+#ifdef DHD_DMA_INDICES_SEQNUM
+		if (bus->dhd->dma_d2h_ring_upd_support) {
+			uint32 bufsz = bus->rw_index_sz * bus->max_completion_rings;
+			if (dhd_prot_dma_indx_copybuf_init(bus->dhd, bufsz, D2H_DMA_INDX_WR_BUF)
+				!= BCME_OK) {
+				ret = BCME_NOMEM;
+				goto done;
+			}
+			bufsz = bus->rw_index_sz * bus->max_submission_rings;
+			if (dhd_prot_dma_indx_copybuf_init(bus->dhd, bufsz, H2D_DMA_INDX_RD_BUF)
+				!= BCME_OK) {
+				ret = BCME_NOMEM;
+				goto done;
+			}
+		}
+#endif /* DHD_DMA_INDICES_SEQNUM */
+		if (IFRM_ENAB(bus->dhd)) {
+			dma_indx_wr_buf = dhd_prot_dma_indx_init(bus->dhd, bus->rw_index_sz,
+				H2D_IFRM_INDX_WR_BUF, bus->max_tx_flowrings);
+
+			if (dma_indx_wr_buf != BCME_OK) {
+				DHD_ERROR(("%s: Failed to alloc memory for Implicit DMA\n",
+						__FUNCTION__));
+				bus->dhd->ifrm_enable = FALSE;
+			}
+		}
+
+		/* read ringmem and ringstate ptrs from shared area and store in host variables */
+		dhd_fillup_ring_sharedptr_info(bus, &ring_info);
+		if (dhd_msg_level & DHD_INFO_VAL) {
+			bcm_print_bytes("ring_info_raw", (uchar *)&ring_info, sizeof(ring_info_t));
+		}
+		DHD_INFO(("%s: ring_info\n", __FUNCTION__));
+
+		DHD_ERROR(("%s: max H2D queues %d\n",
+			__FUNCTION__, ltoh16(ring_info.max_tx_flowrings)));
+
+		DHD_INFO(("mail box address\n"));
+		DHD_INFO(("%s: h2d_mb_data_ptr_addr 0x%04x\n",
+			__FUNCTION__, bus->h2d_mb_data_ptr_addr));
+		DHD_INFO(("%s: d2h_mb_data_ptr_addr 0x%04x\n",
+			__FUNCTION__, bus->d2h_mb_data_ptr_addr));
+	}
+
+	DHD_INFO(("%s: d2h_sync_mode 0x%08x\n",
+		__FUNCTION__, bus->dhd->d2h_sync_mode));
+
+	bus->dhd->d2h_hostrdy_supported =
+		((sh->flags & PCIE_SHARED_HOSTRDY_SUPPORT) == PCIE_SHARED_HOSTRDY_SUPPORT);
+
+	bus->dhd->ext_trap_data_supported =
+		((sh->flags2 & PCIE_SHARED2_EXTENDED_TRAP_DATA) == PCIE_SHARED2_EXTENDED_TRAP_DATA);
+
+	if ((sh->flags2 & PCIE_SHARED2_TXSTATUS_METADATA) == 0)
+		bus->dhd->pcie_txs_metadata_enable = 0;
+
+	if (sh->flags2 & PCIE_SHARED2_TRAP_ON_HOST_DB7)  {
+		memset(&bus->dhd->db7_trap, 0, sizeof(bus->dhd->db7_trap));
+		bus->dhd->db7_trap.fw_db7w_trap = 1;
+		/* add an option to let the user select ?? */
+		bus->dhd->db7_trap.db7_magic_number = PCIE_DB7_MAGIC_NUMBER_DPC_TRAP;
+	}
+	if (sh->flags2 & PCIE_SHARED2_LPM_SUPPORT) {
+		bus->dhd->fw_lpm_support = TRUE;
+		DHD_ERROR(("FW supports LPM mode\n"));
+	}
+
+#ifdef D2H_MINIDUMP
+	bus->d2h_minidump = (sh->flags2 & PCIE_SHARED2_FW_SMALL_MEMDUMP) ? TRUE : FALSE;
+	DHD_ERROR(("FW supports minidump ? %s \n", bus->d2h_minidump ? "Y" : "N"));
+	if (bus->d2h_minidump_override) {
+		bus->d2h_minidump = FALSE;
+	}
+	DHD_ERROR(("d2h_minidump: %d d2h_minidump_override: %d\n",
+		bus->d2h_minidump, bus->d2h_minidump_override));
+#endif /* D2H_MINIDUMP */
+
+	bus->dhd->hscb_enable =
+		(sh->flags2 & PCIE_SHARED2_HSCB) == PCIE_SHARED2_HSCB;
+
+#ifdef EWP_EDL
+	if (host_edl_support) {
+		bus->dhd->dongle_edl_support = (sh->flags2 & PCIE_SHARED2_EDL_RING) ? TRUE : FALSE;
+		DHD_ERROR(("Dongle EDL support: %u\n", bus->dhd->dongle_edl_support));
+	}
+#endif /* EWP_EDL */
+
+	bus->dhd->debug_buf_dest_support =
+		(sh->flags2 & PCIE_SHARED2_DEBUG_BUF_DEST) ? TRUE : FALSE;
+	DHD_ERROR_MEM(("FW supports debug buf dest ? %s \n",
+		bus->dhd->debug_buf_dest_support ? "Y" : "N"));
+
+	bus->dhd->rx_cpl_lat_capable =
+		(sh->flags2 & PCIE_SHARED2_RX_CMPL_PRIO_VALID) ? TRUE : FALSE;
+
+	bus->dhd->mdring_capable =
+		(sh->flags2 & PCIE_SHARED2_METADATA_RING) ? TRUE : FALSE;
+
+	DHD_ERROR(("FW supports MD ring ? %s\n",
+			bus->dhd->mdring_capable ? "Y" : "N"));
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+
+		/*
+		 * WAR to fix ARM cold boot;
+		 * De-assert WL domain in DAR
+		 */
+		if (bus->sih->buscorerev >= 68) {
+			dhd_bus_pcie_pwr_req_wl_domain(bus,
+				DAR_PCIE_PWR_CTRL((bus->sih)->buscorerev), FALSE);
+		}
+	}
+done:
+#if defined(FW_SIGNATURE)
+	if ((ret == BCME_ERROR) && (bus->fwsig_filename[0] != 0)) {
+		bl_verif_status_t status;
+
+		(void)dhdpcie_read_fwstatus(bus, &status);
+		DHD_ERROR(("Verification status: (%08x)\n"
+			"\tstatus: %d\n"
+			"\tstate: %u\n"
+			"\talloc_bytes: %u\n"
+			"\tmax_alloc_bytes: %u\n"
+			"\ttotal_alloc_bytes: %u\n"
+			"\ttotal_freed_bytes: %u\n"
+			"\tnum_allocs: %u\n"
+			"\tmax_allocs: %u\n"
+			"\tmax_alloc_size: %u\n"
+			"\talloc_failures: %u\n",
+			bus->fwstat_download_addr,
+			status.status,
+			status.state,
+			status.alloc_bytes,
+			status.max_alloc_bytes,
+			status.total_alloc_bytes,
+			status.total_freed_bytes,
+			status.num_allocs,
+			status.max_allocs,
+			status.max_alloc_size,
+			status.alloc_failures));
+	}
+#endif /* FW_SIGNATURE */
+	return ret;
+} /* dhdpcie_readshared */
+
+/** Read ring mem and ring state ptr info from shared memory area in device memory */
+static void
+dhd_fillup_ring_sharedptr_info(dhd_bus_t *bus, ring_info_t *ring_info)
+{
+	uint16 i = 0;
+	uint16 j = 0;
+	uint32 tcm_memloc;
+	uint32 d2h_w_idx_ptr, d2h_r_idx_ptr, h2d_w_idx_ptr, h2d_r_idx_ptr;
+	uint16 max_tx_flowrings = bus->max_tx_flowrings;
+
+	/* Ring mem ptr info */
+	/* Alloated in the order
+		H2D_MSGRING_CONTROL_SUBMIT              0
+		H2D_MSGRING_RXPOST_SUBMIT               1
+		D2H_MSGRING_CONTROL_COMPLETE            2
+		D2H_MSGRING_TX_COMPLETE                 3
+		D2H_MSGRING_RX_COMPLETE                 4
+	*/
+
+	{
+		/* ringmemptr holds start of the mem block address space */
+		tcm_memloc = ltoh32(ring_info->ringmem_ptr);
+
+		/* Find out ringmem ptr for each ring common  ring */
+		for (i = 0; i <= BCMPCIE_COMMON_MSGRING_MAX_ID; i++) {
+			bus->ring_sh[i].ring_mem_addr = tcm_memloc;
+			/* Update mem block */
+			tcm_memloc = tcm_memloc + sizeof(ring_mem_t);
+			DHD_INFO(("%s: ring id %d ring mem addr 0x%04x \n", __FUNCTION__,
+				i, bus->ring_sh[i].ring_mem_addr));
+		}
+	}
+
+	/* Ring state mem ptr info */
+	{
+		d2h_w_idx_ptr = ltoh32(ring_info->d2h_w_idx_ptr);
+		d2h_r_idx_ptr = ltoh32(ring_info->d2h_r_idx_ptr);
+		h2d_w_idx_ptr = ltoh32(ring_info->h2d_w_idx_ptr);
+		h2d_r_idx_ptr = ltoh32(ring_info->h2d_r_idx_ptr);
+
+		/* Store h2d common ring write/read pointers */
+		for (i = 0; i < BCMPCIE_H2D_COMMON_MSGRINGS; i++) {
+			bus->ring_sh[i].ring_state_w = h2d_w_idx_ptr;
+			bus->ring_sh[i].ring_state_r = h2d_r_idx_ptr;
+
+			/* update mem block */
+			h2d_w_idx_ptr = h2d_w_idx_ptr + bus->rw_index_sz;
+			h2d_r_idx_ptr = h2d_r_idx_ptr + bus->rw_index_sz;
+
+			DHD_INFO(("%s: h2d w/r : idx %d write %x read %x \n", __FUNCTION__, i,
+				bus->ring_sh[i].ring_state_w, bus->ring_sh[i].ring_state_r));
+		}
+
+		/* Store d2h common ring write/read pointers */
+		for (j = 0; j < BCMPCIE_D2H_COMMON_MSGRINGS; j++, i++) {
+			bus->ring_sh[i].ring_state_w = d2h_w_idx_ptr;
+			bus->ring_sh[i].ring_state_r = d2h_r_idx_ptr;
+
+			/* update mem block */
+			d2h_w_idx_ptr = d2h_w_idx_ptr + bus->rw_index_sz;
+			d2h_r_idx_ptr = d2h_r_idx_ptr + bus->rw_index_sz;
+
+			DHD_INFO(("%s: d2h w/r : idx %d write %x read %x \n", __FUNCTION__, i,
+				bus->ring_sh[i].ring_state_w, bus->ring_sh[i].ring_state_r));
+		}
+
+		/* Store txflow ring write/read pointers */
+		if (bus->api.fw_rev < PCIE_SHARED_VERSION_6) {
+			max_tx_flowrings -= BCMPCIE_H2D_COMMON_MSGRINGS;
+		} else {
+			/* Account for Debug info h2d ring located after the last tx flow ring */
+			max_tx_flowrings = max_tx_flowrings + 1;
+		}
+		for (j = 0; j < max_tx_flowrings; i++, j++)
+		{
+			bus->ring_sh[i].ring_state_w = h2d_w_idx_ptr;
+			bus->ring_sh[i].ring_state_r = h2d_r_idx_ptr;
+
+			/* update mem block */
+			h2d_w_idx_ptr = h2d_w_idx_ptr + bus->rw_index_sz;
+			h2d_r_idx_ptr = h2d_r_idx_ptr + bus->rw_index_sz;
+
+			DHD_INFO(("%s: FLOW Rings h2d w/r : idx %d write %x read %x \n",
+				__FUNCTION__, i,
+				bus->ring_sh[i].ring_state_w,
+				bus->ring_sh[i].ring_state_r));
+		}
+		/* store wr/rd pointers for debug info completion or EDL ring */
+		bus->ring_sh[i].ring_state_w = d2h_w_idx_ptr;
+		bus->ring_sh[i].ring_state_r = d2h_r_idx_ptr;
+		d2h_w_idx_ptr = d2h_w_idx_ptr + bus->rw_index_sz;
+		d2h_r_idx_ptr = d2h_r_idx_ptr + bus->rw_index_sz;
+		DHD_INFO(("d2h w/r : idx %d write %x read %x \n", i,
+			bus->ring_sh[i].ring_state_w,
+			bus->ring_sh[i].ring_state_r));
+	}
+} /* dhd_fillup_ring_sharedptr_info */
+
+/**
+ * Initialize bus module: prepare for communication with the dongle. Called after downloading
+ * firmware into the dongle.
+ */
+int dhd_bus_init(dhd_pub_t *dhdp, bool enforce_mutex)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	int  ret = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(bus->dhd);
+	if (!bus->dhd)
+		return 0;
+
+	bus->link_state = DHD_PCIE_ALL_GOOD;
+
+	dhd_bus_pcie_pwr_req_clear_reload_war(bus);
+
+	dhd_init_dpc_histos(bus->dhd);
+
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req(bus);
+	}
+
+	/* Configure AER registers to log the TLP header */
+	dhd_bus_aer_config(bus);
+
+	/* Make sure we're talking to the core. */
+	bus->reg = si_setcore(bus->sih, PCIE2_CORE_ID, 0);
+	ASSERT(bus->reg != NULL);
+
+	/* before opening up bus for data transfer, check if shared are is intact */
+	ret = dhdpcie_readshared(bus);
+	if (ret < 0) {
+		DHD_ERROR(("%s :Shared area read failed \n", __FUNCTION__));
+		dhdpcie_force_console_dump(bus);
+		goto exit;
+	}
+
+	/* Make sure we're talking to the core. */
+	bus->reg = si_setcore(bus->sih, PCIE2_CORE_ID, 0);
+	ASSERT(bus->reg != NULL);
+
+	/* Set bus state according to enable result */
+	dhdp->busstate = DHD_BUS_DATA;
+	DHD_SET_BUS_NOT_IN_LPS(bus);
+	dhdp->dhd_bus_busy_state = 0;
+
+	/* D11 status via PCIe completion header */
+	if ((ret = dhdpcie_init_d11status(bus)) < 0) {
+		goto exit;
+	}
+
+#if defined(OEM_ANDROID) || defined(LINUX)
+	if (!dhd_download_fw_on_driverload)
+		dhd_dpc_enable(bus->dhd);
+#endif /* OEM_ANDROID || LINUX */
+	/* Enable the interrupt after device is up */
+	dhdpcie_bus_intr_enable(bus);
+
+	DHD_ERROR(("%s: Enabling bus->intr_enabled\n", __FUNCTION__));
+	bus->intr_enabled = TRUE;
+
+	/* XXX These need to change w/API updates */
+	/* bcmsdh_intr_unmask(bus->sdh); */
+	bus->idletime = 0;
+#ifdef PCIE_INB_DW
+	bus->skip_ds_ack = FALSE;
+	/* Initialize the lock to serialize Device Wake Inband activities */
+	if (!bus->inb_lock) {
+		bus->inb_lock = osl_spin_lock_init(bus->dhd->osh);
+	}
+#endif
+
+	/* XXX Temp errnum workaround: return ok, caller checks bus state */
+
+	/* Make use_d0_inform TRUE for Rev 5 for backward compatibility */
+	if (bus->api.fw_rev < PCIE_SHARED_VERSION_6) {
+		bus->use_d0_inform = TRUE;
+	} else {
+		bus->use_d0_inform = FALSE;
+	}
+	bus->lpm_mode = FALSE;
+	bus->lpm_mem_kill = FALSE;
+
+	bus->hostready_count = 0;
+
+	bus->host_irq_enable_count = 0;
+	bus->host_irq_disable_count = 0;
+	bus->dngl_intmask_disable_count = 0;
+	bus->dngl_intmask_enable_count = 0;
+	if (dhdp->rx_cpl_lat_capable) {
+		bzero(&dhdp->rxcpl_lat_info, sizeof(rx_cpl_lat_info_t));
+	}
+exit:
+	if (MULTIBP_ENAB(bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(bus);
+	}
+	return ret;
+}
+
+static void
+dhdpcie_init_shared_addr(dhd_bus_t *bus)
+{
+	uint32 addr = 0;
+	uint32 val = 0;
+	addr = bus->dongle_ram_base + bus->ramsize - 4;
+	dhdpcie_bus_membytes(bus, TRUE, addr, (uint8 *)&val, sizeof(val));
+}
+
+bool
+dhdpcie_chipmatch(uint16 vendor, uint16 device)
+{
+	if ((vendor != PCI_VENDOR_ID_BROADCOM) && (vendor != PCI_VENDOR_ID_SYNAPTICS))  {
+		DHD_ERROR(("%s: Unsupported vendor %x device %x\n", __FUNCTION__,
+			vendor, device));
+		return (-ENODEV);
+	}
+
+	switch (device) {
+		case BCM4345_CHIP_ID:
+		case BCM43454_CHIP_ID:
+		case BCM43455_CHIP_ID:
+		case BCM43457_CHIP_ID:
+		case BCM43458_CHIP_ID:
+		case BCM4350_D11AC_ID:
+		case BCM4350_D11AC2G_ID:
+		case BCM4350_D11AC5G_ID:
+		case BCM4350_CHIP_ID:
+		case BCM4354_D11AC_ID:
+		case BCM4354_D11AC2G_ID:
+		case BCM4354_D11AC5G_ID:
+		case BCM4354_CHIP_ID:
+		case BCM4356_D11AC_ID:
+		case BCM4356_D11AC2G_ID:
+		case BCM4356_D11AC5G_ID:
+		case BCM4356_CHIP_ID:
+		case BCM4371_D11AC_ID:
+		case BCM4371_D11AC2G_ID:
+		case BCM4371_D11AC5G_ID:
+		case BCM4371_CHIP_ID:
+		case BCM4345_D11AC_ID:
+		case BCM4345_D11AC2G_ID:
+		case BCM4345_D11AC5G_ID:
+		case BCM43452_D11AC_ID:
+		case BCM43452_D11AC2G_ID:
+		case BCM43452_D11AC5G_ID:
+		case BCM4335_D11AC_ID:
+		case BCM4335_D11AC2G_ID:
+		case BCM4335_D11AC5G_ID:
+		case BCM4335_CHIP_ID:
+		case BCM43602_D11AC_ID:
+		case BCM43602_D11AC2G_ID:
+		case BCM43602_D11AC5G_ID:
+		case BCM43602_CHIP_ID:
+		case BCM43569_D11AC_ID:
+		case BCM43569_D11AC2G_ID:
+		case BCM43569_D11AC5G_ID:
+		case BCM43569_CHIP_ID:
+		/* XXX: For 4358, BCM4358_CHIP_ID is not checked intentionally as
+		 * this is not a real chip id, but propagated from the OTP.
+		 */
+		case BCM4358_D11AC_ID:
+		case BCM4358_D11AC2G_ID:
+		case BCM4358_D11AC5G_ID:
+		case BCM4349_D11AC_ID:
+		case BCM4349_D11AC2G_ID:
+		case BCM4349_D11AC5G_ID:
+		case BCM4355_D11AC_ID:
+		case BCM4355_D11AC2G_ID:
+		case BCM4355_D11AC5G_ID:
+		case BCM4355_CHIP_ID:
+		/* XXX: BCM4359_CHIP_ID is not checked intentionally as this is
+		 * not a real chip id, but propogated from the OTP.
+		 */
+		case BCM4359_D11AC_ID:
+		case BCM4359_D11AC2G_ID:
+		case BCM4359_D11AC5G_ID:
+		case BCM43596_D11AC_ID:
+		case BCM43596_D11AC2G_ID:
+		case BCM43596_D11AC5G_ID:
+		case BCM43597_D11AC_ID:
+		case BCM43597_D11AC2G_ID:
+		case BCM43597_D11AC5G_ID:
+		case BCM4364_D11AC_ID:
+		case BCM4364_D11AC2G_ID:
+		case BCM4364_D11AC5G_ID:
+		case BCM4364_CHIP_ID:
+		case BCM4361_D11AC_ID:
+		case BCM4361_D11AC2G_ID:
+		case BCM4361_D11AC5G_ID:
+		case BCM4361_CHIP_ID:
+		case BCM4347_D11AC_ID:
+		case BCM4347_D11AC2G_ID:
+		case BCM4347_D11AC5G_ID:
+		case BCM4347_CHIP_ID:
+		case BCM4369_D11AX_ID:
+		case BCM4369_D11AX2G_ID:
+		case BCM4369_D11AX5G_ID:
+		case BCM4369_CHIP_ID:
+		case BCM4376_D11AX_ID:
+		case BCM4376_D11AX2G_ID:
+		case BCM4376_D11AX5G_ID:
+		case BCM4376_CHIP_ID:
+		case BCM4377_M_D11AX_ID:
+		case BCM4377_D11AX_ID:
+		case BCM4377_D11AX2G_ID:
+		case BCM4377_D11AX5G_ID:
+		case BCM4377_CHIP_ID:
+		case BCM4378_D11AX_ID:
+		case BCM4378_D11AX2G_ID:
+		case BCM4378_D11AX5G_ID:
+		case BCM4378_CHIP_ID:
+		case BCM4387_D11AX_ID:
+		case BCM4387_CHIP_ID:
+		case BCM4362_D11AX_ID:
+		case BCM4362_D11AX2G_ID:
+		case BCM4362_D11AX5G_ID:
+		case BCM4362_CHIP_ID:
+		case BCM4375_D11AX_ID:
+		case BCM4375_D11AX2G_ID:
+		case BCM4375_D11AX5G_ID:
+		case BCM4375_CHIP_ID:
+		case BCM43751_D11AX_ID:
+		case BCM43751_D11AX2G_ID:
+		case BCM43751_D11AX5G_ID:
+		case BCM43751_CHIP_ID:
+		case BCM43752_D11AX_ID:
+		case BCM43752_D11AX2G_ID:
+		case BCM43752_D11AX5G_ID:
+		case BCM43752_CHIP_ID:
+		case BCM43756_D11AX_ID:
+		case BCM43756_CHIP_ID:
+		case BCM43756E_D11AX6E_ID:
+		case BCM43756E_D11AC_ID:
+		case BCM43756E_D11AX_ID:
+		case BCM43711_CHIP_ID:
+		case BCM43711_D11AX6E_ID:
+		case BCM43711_D11AC_ID:
+		case BCM43711_D11AX_ID:
+		case BCM4381_CHIP_ID:
+		case BCM4381_D11AX_ID:
+		case BCM4382_CHIP_ID:
+		case BCM4382_D11AX_ID:
+		case BCM4388_CHIP_ID:
+		case BCM4388_D11AX_ID:
+		case BCM4389_CHIP_ID:
+		case BCM4389_D11AX_ID:
+		case BCM4385_D11AX_ID:
+		case BCM4385_CHIP_ID:
+
+			return 0;
+		default:
+			DHD_ERROR(("%s: Unsupported vendor %x device %x\n",
+				__FUNCTION__, vendor, device));
+			return (-ENODEV);
+	}
+} /* dhdpcie_chipmatch */
+
+/**
+ * Name:  dhdpcie_cc_nvmshadow
+ *
+ * Description:
+ * A shadow of OTP/SPROM exists in ChipCommon Region
+ * betw. 0x800 and 0xBFF (Backplane Addr. 0x1800_0800 and 0x1800_0BFF).
+ * Strapping option (SPROM vs. OTP), presence of OTP/SPROM and its size
+ * can also be read from ChipCommon Registers.
+ */
+/* XXX So far tested with 4345 and 4350 (Hence the checks in the function.) */
+static int
+dhdpcie_cc_nvmshadow(dhd_bus_t *bus, struct bcmstrbuf *b)
+{
+	uint16 dump_offset = 0;
+	uint32 dump_size = 0, otp_size = 0, sprom_size = 0;
+
+	/* Table for 65nm OTP Size (in bits) */
+	int  otp_size_65nm[8] = {0, 2048, 4096, 8192, 4096, 6144, 512, 1024};
+
+	volatile uint16 *nvm_shadow;
+
+	uint cur_coreid;
+	uint chipc_corerev;
+	chipcregs_t *chipcregs;
+
+	/* Save the current core */
+	cur_coreid = si_coreid(bus->sih);
+	/* Switch to ChipC */
+	chipcregs = (chipcregs_t *)si_setcore(bus->sih, CC_CORE_ID, 0);
+	ASSERT(chipcregs != NULL);
+
+	chipc_corerev = si_corerev(bus->sih);
+
+	/* Check ChipcommonCore Rev */
+	if (chipc_corerev < 44) {
+		DHD_ERROR(("%s: ChipcommonCore Rev %d < 44\n", __FUNCTION__, chipc_corerev));
+		return BCME_UNSUPPORTED;
+	}
+
+	/* Check ChipID */
+	if (((uint16)bus->sih->chip != BCM4350_CHIP_ID) && !BCM4345_CHIP((uint16)bus->sih->chip) &&
+	        ((uint16)bus->sih->chip != BCM4355_CHIP_ID) &&
+	        ((uint16)bus->sih->chip != BCM4364_CHIP_ID)) {
+		DHD_ERROR(("%s: cc_nvmdump not supported\n", __FUNCTION__));
+		return BCME_UNSUPPORTED;
+	}
+
+	/* Check if SRC_PRESENT in SpromCtrl(0x190 in ChipCommon Regs) is set */
+	if (chipcregs->sromcontrol & SRC_PRESENT) {
+		/* SPROM Size: 1Kbits (0x0), 4Kbits (0x1), 16Kbits(0x2) */
+		sprom_size = (1 << (2 * ((chipcregs->sromcontrol & SRC_SIZE_MASK)
+					>> SRC_SIZE_SHIFT))) * 1024;
+		bcm_bprintf(b, "\nSPROM Present (Size %d bits)\n", sprom_size);
+	}
+
+	/* XXX Check if OTP exists. 2 possible approaches:
+	 * 1) Check if OtpPresent in SpromCtrl (0x190 in ChipCommon Regs) is set OR
+	 * 2) Check if OtpSize > 0
+	 */
+	if (chipcregs->sromcontrol & SRC_OTPPRESENT) {
+		bcm_bprintf(b, "\nOTP Present");
+
+		if (((chipcregs->otplayout & OTPL_WRAP_TYPE_MASK) >> OTPL_WRAP_TYPE_SHIFT)
+			== OTPL_WRAP_TYPE_40NM) {
+			/* 40nm OTP: Size = (OtpSize + 1) * 1024 bits */
+			/* Chipcommon rev51 is a variation on rev45 and does not support
+			 * the latest OTP configuration.
+			 */
+			if (chipc_corerev != 51 && chipc_corerev >= 49) {
+				otp_size = (((chipcregs->otplayout & OTPL_ROW_SIZE_MASK)
+					>> OTPL_ROW_SIZE_SHIFT) + 1) * 1024;
+				bcm_bprintf(b, "(Size %d bits)\n", otp_size);
+			} else {
+				otp_size =  (((chipcregs->capabilities & CC_CAP_OTPSIZE)
+				        >> CC_CAP_OTPSIZE_SHIFT) + 1) * 1024;
+				bcm_bprintf(b, "(Size %d bits)\n", otp_size);
+			}
+		} else {
+			/* This part is untested since newer chips have 40nm OTP */
+			/* Chipcommon rev51 is a variation on rev45 and does not support
+			 * the latest OTP configuration.
+			 */
+			if (chipc_corerev != 51 && chipc_corerev >= 49) {
+				otp_size = otp_size_65nm[(chipcregs->otplayout & OTPL_ROW_SIZE_MASK)
+						>> OTPL_ROW_SIZE_SHIFT];
+				bcm_bprintf(b, "(Size %d bits)\n", otp_size);
+			} else {
+				otp_size = otp_size_65nm[(chipcregs->capabilities & CC_CAP_OTPSIZE)
+					        >> CC_CAP_OTPSIZE_SHIFT];
+				bcm_bprintf(b, "(Size %d bits)\n", otp_size);
+				DHD_INFO(("%s: 65nm/130nm OTP Size not tested. \n",
+					__FUNCTION__));
+			}
+		}
+	}
+
+	/* Chipcommon rev51 is a variation on rev45 and does not support
+	 * the latest OTP configuration.
+	 */
+	if (chipc_corerev != 51 && chipc_corerev >= 49) {
+		if (((chipcregs->sromcontrol & SRC_PRESENT) == 0) &&
+			((chipcregs->otplayout & OTPL_ROW_SIZE_MASK) == 0)) {
+			DHD_ERROR(("%s: SPROM and OTP could not be found "
+				"sromcontrol = %x, otplayout = %x \n",
+				__FUNCTION__, chipcregs->sromcontrol, chipcregs->otplayout));
+			return BCME_NOTFOUND;
+		}
+	} else {
+		if (((chipcregs->sromcontrol & SRC_PRESENT) == 0) &&
+			((chipcregs->capabilities & CC_CAP_OTPSIZE) == 0)) {
+			DHD_ERROR(("%s: SPROM and OTP could not be found "
+				"sromcontrol = %x, capablities = %x \n",
+				__FUNCTION__, chipcregs->sromcontrol, chipcregs->capabilities));
+			return BCME_NOTFOUND;
+		}
+	}
+
+	/* Check the strapping option in SpromCtrl: Set = OTP otherwise SPROM */
+	if ((!(chipcregs->sromcontrol & SRC_PRESENT) || (chipcregs->sromcontrol & SRC_OTPSEL)) &&
+		(chipcregs->sromcontrol & SRC_OTPPRESENT)) {
+
+		bcm_bprintf(b, "OTP Strap selected.\n"
+		               "\nOTP Shadow in ChipCommon:\n");
+
+		dump_size = otp_size / 16 ; /* 16bit words */
+
+	} else if (((chipcregs->sromcontrol & SRC_OTPSEL) == 0) &&
+		(chipcregs->sromcontrol & SRC_PRESENT)) {
+
+		bcm_bprintf(b, "SPROM Strap selected\n"
+				"\nSPROM Shadow in ChipCommon:\n");
+
+		/* If SPROM > 8K only 8Kbits is mapped to ChipCommon (0x800 - 0xBFF) */
+		/* dump_size in 16bit words */
+		dump_size = sprom_size > 8 ? (8 * 1024) / 16 : sprom_size / 16;
+	} else {
+		DHD_ERROR(("%s: NVM Shadow does not exist in ChipCommon\n",
+			__FUNCTION__));
+		return BCME_NOTFOUND;
+	}
+
+	if (bus->regs == NULL) {
+		DHD_ERROR(("ChipCommon Regs. not initialized\n"));
+		return BCME_NOTREADY;
+	} else {
+		bcm_bprintf(b, "\n OffSet:");
+
+		/* Chipcommon rev51 is a variation on rev45 and does not support
+		 * the latest OTP configuration.
+		 */
+		if (chipc_corerev != 51 && chipc_corerev >= 49) {
+			/* Chip common can read only 8kbits,
+			* for ccrev >= 49 otp size is around 12 kbits so use GCI core
+			*/
+			nvm_shadow = (volatile uint16 *)si_setcore(bus->sih, GCI_CORE_ID, 0);
+		} else {
+			/* Point to the SPROM/OTP shadow in ChipCommon */
+			nvm_shadow = chipcregs->sromotp;
+		}
+
+		if (nvm_shadow == NULL) {
+			DHD_ERROR(("%s: NVM Shadow is not intialized\n", __FUNCTION__));
+			return BCME_NOTFOUND;
+		}
+
+		/*
+		* Read 16 bits / iteration.
+		* dump_size & dump_offset in 16-bit words
+		*/
+		while (dump_offset < dump_size) {
+			if (dump_offset % 2 == 0)
+				/* Print the offset in the shadow space in Bytes */
+				bcm_bprintf(b, "\n 0x%04x", dump_offset * 2);
+
+			bcm_bprintf(b, "\t0x%04x", *(nvm_shadow + dump_offset));
+			dump_offset += 0x1;
+		}
+	}
+
+	/* Switch back to the original core */
+	si_setcore(bus->sih, cur_coreid, 0);
+
+	return BCME_OK;
+} /* dhdpcie_cc_nvmshadow */
+
+/** Flow rings are dynamically created and destroyed */
+void dhd_bus_clean_flow_ring(dhd_bus_t *bus, void *node)
+{
+	void *pkt;
+	flow_queue_t *queue;
+	flow_ring_node_t *flow_ring_node = (flow_ring_node_t *)node;
+	unsigned long flags;
+
+	queue = &flow_ring_node->queue;
+
+#ifdef DHDTCPACK_SUPPRESS
+	/* Clean tcp_ack_info_tbl in order to prevent access to flushed pkt,
+	 * when there is a newly coming packet from network stack.
+	 */
+	dhd_tcpack_info_tbl_clean(bus->dhd);
+#endif /* DHDTCPACK_SUPPRESS */
+
+	/* clean up BUS level info */
+	DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+
+	/* Flush all pending packets in the queue, if any */
+	while ((pkt = dhd_flow_queue_dequeue(bus->dhd, queue)) != NULL) {
+		PKTFREE(bus->dhd->osh, pkt, TRUE);
+	}
+	ASSERT(DHD_FLOW_QUEUE_EMPTY(queue));
+
+	/* Reinitialise flowring's queue */
+	dhd_flow_queue_reinit(bus->dhd, queue, bus->dhd->conf->flow_ring_queue_threshold);
+	flow_ring_node->status = FLOW_RING_STATUS_CLOSED;
+	flow_ring_node->active = FALSE;
+
+	DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+	/* Hold flowring_list_lock to ensure no race condition while accessing the List */
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+	dll_delete(&flow_ring_node->list);
+	dll_init(&flow_ring_node->list);
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	/* Release the flowring object back into the pool */
+	dhd_prot_flowrings_pool_release(bus->dhd,
+		flow_ring_node->flowid, flow_ring_node->prot_info);
+
+	/* Free the flowid back to the flowid allocator */
+	dhd_flowid_free(bus->dhd, flow_ring_node->flow_info.ifindex,
+	                flow_ring_node->flowid);
+}
+
+/**
+ * Allocate a Flow ring buffer,
+ * Init Ring buffer, send Msg to device about flow ring creation
+*/
+int
+dhd_bus_flow_ring_create_request(dhd_bus_t *bus, void *arg)
+{
+	flow_ring_node_t *flow_ring_node = (flow_ring_node_t *)arg;
+
+	DHD_INFO(("%s :Flow create\n", __FUNCTION__));
+
+	/* Send Msg to device about flow ring creation */
+	if (dhd_prot_flow_ring_create(bus->dhd, flow_ring_node) != BCME_OK)
+		return BCME_NOMEM;
+
+	return BCME_OK;
+}
+
+/** Handle response from dongle on a 'flow ring create' request */
+void
+dhd_bus_flow_ring_create_response(dhd_bus_t *bus, uint16 flowid, int32 status)
+{
+	flow_ring_node_t *flow_ring_node;
+	unsigned long flags;
+
+	DHD_INFO(("%s :Flow Response %d \n", __FUNCTION__, flowid));
+
+	/* Boundary check of the flowid */
+	if (flowid > bus->dhd->max_tx_flowid) {
+		DHD_ERROR(("%s: flowid is invalid %d, max id %d\n", __FUNCTION__,
+			flowid, bus->dhd->max_tx_flowid));
+		return;
+	}
+
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flowid);
+	if (!flow_ring_node) {
+		DHD_ERROR(("%s: flow_ring_node is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	ASSERT(flow_ring_node->flowid == flowid);
+	if (flow_ring_node->flowid != flowid) {
+		DHD_ERROR(("%s: flowid %d is different from the flowid "
+			"of the flow_ring_node %d\n", __FUNCTION__, flowid,
+			flow_ring_node->flowid));
+		return;
+	}
+
+	if (status != BCME_OK) {
+		DHD_ERROR(("%s Flow create Response failure error status = %d \n",
+		     __FUNCTION__, status));
+		/* Call Flow clean up */
+		dhd_bus_clean_flow_ring(bus, flow_ring_node);
+		return;
+	}
+
+	DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+	flow_ring_node->status = FLOW_RING_STATUS_OPEN;
+	DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+	/* Now add the Flow ring node into the active list
+	 * Note that this code to add the newly created node to the active
+	 * list was living in dhd_flowid_lookup. But note that after
+	 * adding the node to the active list the contents of node is being
+	 * filled in dhd_prot_flow_ring_create.
+	 * If there is a D2H interrupt after the node gets added to the
+	 * active list and before the node gets populated with values
+	 * from the Bottom half dhd_update_txflowrings would be called.
+	 * which will then try to walk through the active flow ring list,
+	 * pickup the nodes and operate on them. Now note that since
+	 * the function dhd_prot_flow_ring_create is not finished yet
+	 * the contents of flow_ring_node can still be NULL leading to
+	 * crashes. Hence the flow_ring_node should be added to the
+	 * active list only after its truely created, which is after
+	 * receiving the create response message from the Host.
+	 */
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+	dll_prepend(&bus->flowring_active_list, &flow_ring_node->list);
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	dhd_bus_schedule_queue(bus, flowid, FALSE, 0, NULL); /* from queue to flowring */
+
+	return;
+}
+
+int
+dhd_bus_flow_ring_delete_request(dhd_bus_t *bus, void *arg)
+{
+	void * pkt;
+	flow_queue_t *queue;
+	flow_ring_node_t *flow_ring_node;
+	unsigned long flags;
+
+	DHD_INFO(("%s :Flow Delete\n", __FUNCTION__));
+
+	flow_ring_node = (flow_ring_node_t *)arg;
+
+#ifdef DHDTCPACK_SUPPRESS
+	/* Clean tcp_ack_info_tbl in order to prevent access to flushed pkt,
+	 * when there is a newly coming packet from network stack.
+	 */
+	dhd_tcpack_info_tbl_clean(bus->dhd);
+#endif /* DHDTCPACK_SUPPRESS */
+	DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+	if (flow_ring_node->status == FLOW_RING_STATUS_DELETE_PENDING) {
+		DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+		DHD_ERROR(("%s :Delete Pending flowid %u\n", __FUNCTION__, flow_ring_node->flowid));
+		return BCME_ERROR;
+	}
+	flow_ring_node->status = FLOW_RING_STATUS_DELETE_PENDING;
+
+	queue = &flow_ring_node->queue; /* queue associated with flow ring */
+
+	/* Flush all pending packets in the queue, if any */
+	while ((pkt = dhd_flow_queue_dequeue(bus->dhd, queue)) != NULL) {
+		PKTFREE(bus->dhd->osh, pkt, TRUE);
+	}
+	ASSERT(DHD_FLOW_QUEUE_EMPTY(queue));
+
+	DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+	/* Send Msg to device about flow ring deletion */
+	dhd_prot_flow_ring_delete(bus->dhd, flow_ring_node);
+
+	return BCME_OK;
+}
+
+void
+dhd_bus_flow_ring_delete_response(dhd_bus_t *bus, uint16 flowid, uint32 status)
+{
+	flow_ring_node_t *flow_ring_node;
+
+	DHD_INFO(("%s :Flow Delete Response %d \n", __FUNCTION__, flowid));
+
+	/* Boundary check of the flowid */
+	if (flowid > bus->dhd->max_tx_flowid) {
+		DHD_ERROR(("%s: flowid is invalid %d, max id %d\n", __FUNCTION__,
+			flowid, bus->dhd->max_tx_flowid));
+		return;
+	}
+
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flowid);
+	if (!flow_ring_node) {
+		DHD_ERROR(("%s: flow_ring_node is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	ASSERT(flow_ring_node->flowid == flowid);
+	if (flow_ring_node->flowid != flowid) {
+		DHD_ERROR(("%s: flowid %d is different from the flowid "
+			"of the flow_ring_node %d\n", __FUNCTION__, flowid,
+			flow_ring_node->flowid));
+		return;
+	}
+
+	if (status != BCME_OK) {
+		DHD_ERROR(("%s Flow Delete Response failure error status = %d \n",
+		    __FUNCTION__, status));
+		return;
+	}
+	/* Call Flow clean up */
+	dhd_bus_clean_flow_ring(bus, flow_ring_node);
+
+	return;
+
+}
+
+int dhd_bus_flow_ring_flush_request(dhd_bus_t *bus, void *arg)
+{
+	void *pkt;
+	flow_queue_t *queue;
+	flow_ring_node_t *flow_ring_node;
+	unsigned long flags;
+
+	DHD_INFO(("%s :Flow Flush\n", __FUNCTION__));
+
+	flow_ring_node = (flow_ring_node_t *)arg;
+
+	DHD_FLOWRING_LOCK(flow_ring_node->lock, flags);
+	queue = &flow_ring_node->queue; /* queue associated with flow ring */
+	/* Flow ring status will be set back to FLOW_RING_STATUS_OPEN
+	 * once flow ring flush response is received for this flowring node.
+	 */
+	flow_ring_node->status = FLOW_RING_STATUS_FLUSH_PENDING;
+
+#ifdef DHDTCPACK_SUPPRESS
+	/* Clean tcp_ack_info_tbl in order to prevent access to flushed pkt,
+	 * when there is a newly coming packet from network stack.
+	 */
+	dhd_tcpack_info_tbl_clean(bus->dhd);
+#endif /* DHDTCPACK_SUPPRESS */
+
+	/* Flush all pending packets in the queue, if any */
+	while ((pkt = dhd_flow_queue_dequeue(bus->dhd, queue)) != NULL) {
+		PKTFREE(bus->dhd->osh, pkt, TRUE);
+	}
+	ASSERT(DHD_FLOW_QUEUE_EMPTY(queue));
+
+	DHD_FLOWRING_UNLOCK(flow_ring_node->lock, flags);
+
+	/* Send Msg to device about flow ring flush */
+	dhd_prot_flow_ring_flush(bus->dhd, flow_ring_node);
+
+	return BCME_OK;
+}
+
+void
+dhd_bus_flow_ring_flush_response(dhd_bus_t *bus, uint16 flowid, uint32 status)
+{
+	flow_ring_node_t *flow_ring_node;
+
+	if (status != BCME_OK) {
+		DHD_ERROR(("%s Flow flush Response failure error status = %d \n",
+		    __FUNCTION__, status));
+		return;
+	}
+
+	/* Boundary check of the flowid */
+	if (flowid > bus->dhd->max_tx_flowid) {
+		DHD_ERROR(("%s: flowid is invalid %d, max id %d\n", __FUNCTION__,
+			flowid, bus->dhd->max_tx_flowid));
+		return;
+	}
+
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flowid);
+	if (!flow_ring_node) {
+		DHD_ERROR(("%s: flow_ring_node is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	ASSERT(flow_ring_node->flowid == flowid);
+	if (flow_ring_node->flowid != flowid) {
+		DHD_ERROR(("%s: flowid %d is different from the flowid "
+			"of the flow_ring_node %d\n", __FUNCTION__, flowid,
+			flow_ring_node->flowid));
+		return;
+	}
+
+	flow_ring_node->status = FLOW_RING_STATUS_OPEN;
+	return;
+}
+
+uint32
+dhd_bus_max_h2d_queues(struct dhd_bus *bus)
+{
+	return bus->max_submission_rings;
+}
+
+/* To be symmetric with SDIO */
+void
+dhd_bus_pktq_flush(dhd_pub_t *dhdp)
+{
+	return;
+}
+
+void
+dhd_bus_set_linkdown(dhd_pub_t *dhdp, bool val)
+{
+	dhdp->bus->is_linkdown = val;
+}
+
+int
+dhd_bus_get_linkdown(dhd_pub_t *dhdp)
+{
+	return dhdp->bus->is_linkdown;
+}
+
+int
+dhd_bus_get_cto(dhd_pub_t *dhdp)
+{
+	return dhdp->bus->cto_triggered;
+}
+
+bool
+dhd_bus_get_read_shm(dhd_pub_t *dhdp)
+{
+	return FALSE;
+}
+
+#ifdef IDLE_TX_FLOW_MGMT
+/* resume request */
+int
+dhd_bus_flow_ring_resume_request(dhd_bus_t *bus, void *arg)
+{
+	flow_ring_node_t *flow_ring_node = (flow_ring_node_t *)arg;
+
+	DHD_ERROR(("%s :Flow Resume Request flow id %u\n", __FUNCTION__, flow_ring_node->flowid));
+
+	flow_ring_node->status = FLOW_RING_STATUS_RESUME_PENDING;
+
+	/* Send Msg to device about flow ring resume */
+	dhd_prot_flow_ring_resume(bus->dhd, flow_ring_node);
+
+	return BCME_OK;
+}
+
+/* add the node back to active flowring */
+void
+dhd_bus_flow_ring_resume_response(dhd_bus_t *bus, uint16 flowid, int32 status)
+{
+
+	flow_ring_node_t *flow_ring_node;
+
+	DHD_TRACE(("%s :flowid %d \n", __FUNCTION__, flowid));
+
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flowid);
+	ASSERT(flow_ring_node->flowid == flowid);
+
+	if (status != BCME_OK) {
+		DHD_ERROR(("%s Error Status = %d \n",
+			__FUNCTION__, status));
+		return;
+	}
+
+	DHD_TRACE(("%s :Number of pkts queued in FlowId:%d is -> %u!!\n",
+		__FUNCTION__, flow_ring_node->flowid,  flow_ring_node->queue.len));
+
+	flow_ring_node->status = FLOW_RING_STATUS_OPEN;
+
+	dhd_bus_schedule_queue(bus, flowid, FALSE, 0, NULL);
+	return;
+}
+
+/* scan the flow rings in active list for idle time out */
+void
+dhd_bus_check_idle_scan(dhd_bus_t *bus)
+{
+	uint64 time_stamp; /* in millisec */
+	uint64 diff;
+
+	time_stamp = OSL_SYSUPTIME();
+	diff = time_stamp - bus->active_list_last_process_ts;
+
+	if (diff > IDLE_FLOW_LIST_TIMEOUT) {
+		dhd_bus_idle_scan(bus);
+		bus->active_list_last_process_ts = OSL_SYSUPTIME();
+	}
+
+	return;
+}
+
+/* scan the nodes in active list till it finds a non idle node */
+void
+dhd_bus_idle_scan(dhd_bus_t *bus)
+{
+	dll_t *item, *prev;
+	flow_ring_node_t *flow_ring_node;
+	uint64 time_stamp, diff;
+	unsigned long flags;
+	uint16 ringid[MAX_SUSPEND_REQ];
+	uint16 count = 0;
+
+	time_stamp = OSL_SYSUPTIME();
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+
+	for (item = dll_tail_p(&bus->flowring_active_list);
+	         !dll_end(&bus->flowring_active_list, item); item = prev) {
+		prev = dll_prev_p(item);
+
+		flow_ring_node = dhd_constlist_to_flowring(item);
+
+		if (flow_ring_node->flowid == (bus->max_submission_rings - 1))
+			continue;
+
+		if (flow_ring_node->status != FLOW_RING_STATUS_OPEN) {
+			/* Takes care of deleting zombie rings */
+			/* delete from the active list */
+			DHD_INFO(("deleting flow id %u from active list\n",
+				flow_ring_node->flowid));
+			__dhd_flow_ring_delete_from_active_list(bus, flow_ring_node);
+			continue;
+		}
+
+		diff = time_stamp - flow_ring_node->last_active_ts;
+
+		if ((diff > IDLE_FLOW_RING_TIMEOUT) && !(flow_ring_node->queue.len))  {
+			DHD_ERROR(("\nSuspending flowid %d\n", flow_ring_node->flowid));
+			/* delete from the active list */
+			__dhd_flow_ring_delete_from_active_list(bus, flow_ring_node);
+			flow_ring_node->status = FLOW_RING_STATUS_SUSPENDED;
+			ringid[count] = flow_ring_node->flowid;
+			count++;
+			if (count == MAX_SUSPEND_REQ) {
+				/* create a batch message now!! */
+				dhd_prot_flow_ring_batch_suspend_request(bus->dhd, ringid, count);
+				count = 0;
+			}
+
+		} else {
+
+			/* No more scanning, break from here! */
+			break;
+		}
+	}
+
+	if (count) {
+		dhd_prot_flow_ring_batch_suspend_request(bus->dhd, ringid, count);
+	}
+
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	return;
+}
+
+void dhd_flow_ring_move_to_active_list_head(struct dhd_bus *bus, flow_ring_node_t *flow_ring_node)
+{
+	unsigned long flags;
+	dll_t* list;
+
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+	/* check if the node is already at head, otherwise delete it and prepend */
+	list = dll_head_p(&bus->flowring_active_list);
+	if (&flow_ring_node->list != list) {
+		dll_delete(&flow_ring_node->list);
+		dll_prepend(&bus->flowring_active_list, &flow_ring_node->list);
+	}
+
+	/* update flow ring timestamp */
+	flow_ring_node->last_active_ts = OSL_SYSUPTIME();
+
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	return;
+}
+
+void dhd_flow_ring_add_to_active_list(struct dhd_bus *bus, flow_ring_node_t *flow_ring_node)
+{
+	unsigned long flags;
+
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+
+	dll_prepend(&bus->flowring_active_list, &flow_ring_node->list);
+	/* update flow ring timestamp */
+	flow_ring_node->last_active_ts = OSL_SYSUPTIME();
+
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	return;
+}
+void __dhd_flow_ring_delete_from_active_list(struct dhd_bus *bus, flow_ring_node_t *flow_ring_node)
+{
+	dll_delete(&flow_ring_node->list);
+}
+
+void dhd_flow_ring_delete_from_active_list(struct dhd_bus *bus, flow_ring_node_t *flow_ring_node)
+{
+	unsigned long flags;
+
+	DHD_FLOWRING_LIST_LOCK(bus->dhd->flowring_list_lock, flags);
+
+	__dhd_flow_ring_delete_from_active_list(bus, flow_ring_node);
+
+	DHD_FLOWRING_LIST_UNLOCK(bus->dhd->flowring_list_lock, flags);
+
+	return;
+}
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#if defined(LINUX) || defined(linux)
+int
+dhdpcie_bus_start_host_dev(struct dhd_bus *bus)
+{
+	return dhdpcie_start_host_dev(bus);
+}
+
+int
+dhdpcie_bus_stop_host_dev(struct dhd_bus *bus)
+{
+	return dhdpcie_stop_host_dev(bus);
+}
+
+int
+dhdpcie_bus_disable_device(struct dhd_bus *bus)
+{
+	return dhdpcie_disable_device(bus);
+}
+
+int
+dhdpcie_bus_enable_device(struct dhd_bus *bus)
+{
+	return dhdpcie_enable_device(bus);
+}
+
+int
+dhdpcie_bus_alloc_resource(struct dhd_bus *bus)
+{
+	return dhdpcie_alloc_resource(bus);
+}
+
+void
+dhdpcie_bus_free_resource(struct dhd_bus *bus)
+{
+	dhdpcie_free_resource(bus);
+}
+
+int
+dhd_bus_request_irq(struct dhd_bus *bus)
+{
+	return dhdpcie_bus_request_irq(bus);
+}
+
+bool
+dhdpcie_bus_dongle_attach(struct dhd_bus *bus)
+{
+	return dhdpcie_dongle_attach(bus);
+}
+
+int
+dhd_bus_release_dongle(struct dhd_bus *bus)
+{
+	bool dongle_isolation;
+	osl_t *osh;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus) {
+		osh = bus->osh;
+		ASSERT(osh);
+
+		if (bus->dhd) {
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+			debugger_close();
+#endif /* DEBUGGER || DHD_DSCOPE */
+
+			dongle_isolation = bus->dhd->dongle_isolation;
+			dhdpcie_bus_release_dongle(bus, osh, dongle_isolation, TRUE);
+		}
+	}
+
+	return 0;
+}
+#endif /* LINUX || linux */
+
+int
+dhdpcie_cto_cfg_init(struct dhd_bus *bus, bool enable)
+{
+	if (enable) {
+		dhdpcie_bus_cfg_write_dword(bus, PCI_INT_MASK, 4,
+			PCI_CTO_INT_MASK | PCI_SBIM_MASK_SERR);
+	} else {
+		dhdpcie_bus_cfg_write_dword(bus, PCI_INT_MASK, 4, 0);
+	}
+	return 0;
+}
+
+int
+dhdpcie_cto_init(struct dhd_bus *bus, bool enable)
+{
+	volatile void *regsva = (volatile void *)bus->regs;
+	uint32 val;
+	uint16 chipid = dhd_get_chipid(bus);
+	uint32 ctoctrl;
+
+	bus->cto_enable = enable;
+
+	dhdpcie_cto_cfg_init(bus, enable);
+
+	if (enable) {
+		if (bus->cto_threshold == 0) {
+			if ((chipid == BCM4387_CHIP_ID) ||
+			    (chipid == BCM4388_CHIP_ID) ||
+			    (chipid == BCM4389_CHIP_ID) ||
+			    (chipid == BCM4397_CHIP_ID)) {
+				bus->cto_threshold = PCIE_CTO_TO_THRESH_DEFAULT_REV69;
+			} else {
+				bus->cto_threshold = PCIE_CTO_TO_THRESH_DEFAULT;
+			}
+		}
+		val = ((bus->cto_threshold << PCIE_CTO_TO_THRESHOLD_SHIFT) &
+			PCIE_CTO_TO_THRESHHOLD_MASK) |
+			((PCIE_CTO_CLKCHKCNT_VAL << PCIE_CTO_CLKCHKCNT_SHIFT) &
+			PCIE_CTO_CLKCHKCNT_MASK) |
+			PCIE_CTO_ENAB_MASK;
+
+		pcie_corereg(bus->osh, regsva, OFFSETOF(sbpcieregs_t, ctoctrl), ~0, val);
+	} else {
+		pcie_corereg(bus->osh, regsva, OFFSETOF(sbpcieregs_t, ctoctrl), ~0, 0);
+	}
+
+	ctoctrl = pcie_corereg(bus->osh, regsva, OFFSETOF(sbpcieregs_t, ctoctrl), 0, 0);
+
+	DHD_ERROR(("%s: ctoctrl(0x%x) enable/disable %d for chipid(0x%x)\n",
+		__FUNCTION__, ctoctrl, bus->cto_enable, chipid));
+
+	return 0;
+}
+
+static int
+dhdpcie_cto_error_recovery(struct dhd_bus *bus)
+{
+	uint32 pci_intmask, err_status;
+	uint8 i = 0;
+	uint32 val;
+
+	pci_intmask = dhdpcie_bus_cfg_read_dword(bus, PCI_INT_MASK, 4);
+	dhdpcie_bus_cfg_write_dword(bus, PCI_INT_MASK, 4, pci_intmask & ~PCI_CTO_INT_MASK);
+
+	DHD_OS_WAKE_LOCK(bus->dhd);
+
+	DHD_ERROR(("--- CTO Triggered --- %d\n", bus->pwr_req_ref));
+
+	/*
+	 * DAR still accessible
+	 */
+	dhd_bus_dump_dar_registers(bus);
+
+	/* reset backplane */
+	val = dhdpcie_bus_cfg_read_dword(bus, PCI_SPROM_CONTROL, 4);
+	dhdpcie_bus_cfg_write_dword(bus, PCI_SPROM_CONTROL, 4, val | SPROM_CFG_TO_SB_RST);
+
+	/* clear timeout error */
+	while (1) {
+		err_status =  si_corereg(bus->sih, bus->sih->buscoreidx,
+			DAR_ERRLOG(bus->sih->buscorerev),
+			0, 0);
+		if (err_status & PCIE_CTO_ERR_MASK) {
+			si_corereg(bus->sih, bus->sih->buscoreidx,
+					DAR_ERRLOG(bus->sih->buscorerev),
+					~0, PCIE_CTO_ERR_MASK);
+		} else {
+			break;
+		}
+		OSL_DELAY(CTO_TO_CLEAR_WAIT_MS * 1000);
+		i++;
+		if (i > CTO_TO_CLEAR_WAIT_MAX_CNT) {
+			DHD_ERROR(("cto recovery fail\n"));
+
+			DHD_OS_WAKE_UNLOCK(bus->dhd);
+			return BCME_ERROR;
+		}
+	}
+
+	/* clear interrupt status */
+	dhdpcie_bus_cfg_write_dword(bus, PCI_INT_STATUS, 4, PCI_CTO_INT_MASK);
+
+	/* Halt ARM & remove reset */
+	/* TBD : we can add ARM Halt here in case */
+
+	/* reset SPROM_CFG_TO_SB_RST */
+	val = dhdpcie_bus_cfg_read_dword(bus, PCI_SPROM_CONTROL, 4);
+
+	DHD_ERROR(("cto recovery reset 0x%x:SPROM_CFG_TO_SB_RST(0x%x) 0x%x\n",
+		PCI_SPROM_CONTROL, SPROM_CFG_TO_SB_RST, val));
+	dhdpcie_bus_cfg_write_dword(bus, PCI_SPROM_CONTROL, 4, val & ~SPROM_CFG_TO_SB_RST);
+
+	val = dhdpcie_bus_cfg_read_dword(bus, PCI_SPROM_CONTROL, 4);
+	DHD_ERROR(("cto recovery success 0x%x:SPROM_CFG_TO_SB_RST(0x%x) 0x%x\n",
+		PCI_SPROM_CONTROL, SPROM_CFG_TO_SB_RST, val));
+
+	DHD_OS_WAKE_UNLOCK(bus->dhd);
+
+	return BCME_OK;
+}
+
+void
+dhdpcie_ssreset_dis_enum_rst(struct dhd_bus *bus)
+{
+	uint32 val;
+
+	val = dhdpcie_bus_cfg_read_dword(bus, PCIE_CFG_SUBSYSTEM_CONTROL, 4);
+	dhdpcie_bus_cfg_write_dword(bus, PCIE_CFG_SUBSYSTEM_CONTROL, 4,
+		val | (0x1 << PCIE_SSRESET_DIS_ENUM_RST_BIT));
+}
+
+#if defined(DBG_PKT_MON) || defined(DHD_PKT_LOGGING)
+/*
+ * XXX: WAR: Update dongle that driver supports sending of d11
+ * tx_status through unused status field of PCIe completion header
+ * if dongle also supports the same WAR.
+ */
+static int
+dhdpcie_init_d11status(struct dhd_bus *bus)
+{
+	uint32 addr;
+	uint32 flags2;
+	int ret = 0;
+
+	if (bus->pcie_sh->flags2 & PCIE_SHARED2_D2H_D11_TX_STATUS) {
+		flags2 = bus->pcie_sh->flags2;
+		addr = bus->shared_addr + OFFSETOF(pciedev_shared_t, flags2);
+		flags2 |= PCIE_SHARED2_H2D_D11_TX_STATUS;
+		ret = dhdpcie_bus_membytes(bus, TRUE, addr,
+			(uint8 *)&flags2, sizeof(flags2));
+		if (ret < 0) {
+			DHD_ERROR(("%s: update flag bit (H2D_D11_TX_STATUS) failed\n",
+				__FUNCTION__));
+			return ret;
+		}
+		bus->pcie_sh->flags2 = flags2;
+		bus->dhd->d11_tx_status = TRUE;
+	}
+	return ret;
+}
+
+#else
+static int
+dhdpcie_init_d11status(struct dhd_bus *bus)
+{
+	return 0;
+}
+#endif /* DBG_PKT_MON || DHD_PKT_LOGGING */
+
+int
+dhdpcie_get_max_eventbufpost(struct dhd_bus *bus)
+{
+	int evt_buf_pool = EVENT_BUF_POOL_LOW;
+	if (bus->pcie_sh->flags2 & (0x1 << PCIE_SHARED_EVENT_BUF_POOL_MAX_POS)) {
+		evt_buf_pool = EVENT_BUF_POOL_MEDIUM;
+	} else if (bus->pcie_sh->flags2 & (0x2 << PCIE_SHARED_EVENT_BUF_POOL_MAX_POS)) {
+		evt_buf_pool = EVENT_BUF_POOL_HIGH;
+	} else if (bus->pcie_sh->flags2 & (0x3 << PCIE_SHARED_EVENT_BUF_POOL_MAX_POS)) {
+		evt_buf_pool = EVENT_BUF_POOL_HIGHEST;
+	}
+	return evt_buf_pool;
+}
+
+int
+dhd_bus_oob_intr_register(dhd_pub_t *dhdp)
+{
+	int err = 0;
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	err = dhdpcie_oob_intr_register(dhdp->bus);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+	return err;
+}
+
+void
+dhd_bus_oob_intr_unregister(dhd_pub_t *dhdp)
+{
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	dhdpcie_oob_intr_unregister(dhdp->bus);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+}
+
+void
+dhd_bus_oob_intr_set(dhd_pub_t *dhdp, bool enable)
+{
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	dhdpcie_oob_intr_set(dhdp->bus, enable);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+}
+
+int
+dhd_bus_get_oob_irq_num(dhd_pub_t *dhdp)
+{
+	int irq_num = 0;
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	irq_num = dhdpcie_get_oob_irq_num(dhdp->bus);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+	return irq_num;
+}
+
+#ifdef BCMDBG
+void
+dhd_bus_flow_ring_cnt_update(dhd_bus_t *bus, uint16 flowid, uint32 txstatus)
+{
+	flow_ring_node_t *flow_ring_node;
+	/* If we have d2h sync enabled due to marker overloading, we cannot update this. */
+	if (bus->dhd->d2h_sync_mode)
+		return;
+	if (txstatus >= DHD_MAX_TX_STATUS_MSGS) {
+		/*
+		 * XXX: changed DHD_ERROR to DHD_INFO
+		 * There are flood of messages with P2P FW
+		 * It is being root-caused.
+		 */
+		DHD_INFO(("%s Unknown txtstatus = %d \n",
+		    __FUNCTION__, txstatus));
+		return;
+	}
+	flow_ring_node = DHD_FLOW_RING(bus->dhd, flowid);
+	ASSERT(flow_ring_node->flowid == flowid);
+	flow_ring_node->flow_info.tx_status[txstatus]++;
+	return;
+}
+#endif /* BCMDBG */
+
+bool
+dhdpcie_bus_get_pcie_hostready_supported(dhd_bus_t *bus)
+{
+	return bus->dhd->d2h_hostrdy_supported;
+}
+
+void
+dhd_pcie_dump_core_regs(dhd_pub_t * pub, uint32 index, uint32 first_addr, uint32 last_addr)
+{
+	dhd_bus_t *bus = pub->bus;
+	uint32	coreoffset = index << 12;
+	uint32	core_addr = SI_ENUM_BASE(bus->sih) + coreoffset;
+	uint32 value;
+
+	while (first_addr <= last_addr) {
+		core_addr = SI_ENUM_BASE(bus->sih) + coreoffset + first_addr;
+		if (serialized_backplane_access(bus, core_addr, 4, &value, TRUE) != BCME_OK) {
+			DHD_ERROR(("Invalid size/addr combination \n"));
+		}
+		DHD_ERROR(("[0x%08x]: 0x%08x\n", core_addr, value));
+		first_addr = first_addr + 4;
+	}
+}
+
+bool
+dhdpcie_bus_get_pcie_idma_supported(dhd_bus_t *bus)
+{
+	if (!bus->dhd)
+		return FALSE;
+	else if (bus->idma_enabled) {
+		return bus->dhd->idma_enable;
+	} else {
+		return FALSE;
+	}
+}
+
+bool
+dhdpcie_bus_get_pcie_ifrm_supported(dhd_bus_t *bus)
+{
+	if (!bus->dhd)
+		return FALSE;
+	else if (bus->ifrm_enabled) {
+		return bus->dhd->ifrm_enable;
+	} else {
+		return FALSE;
+	}
+}
+
+bool
+dhdpcie_bus_get_pcie_dar_supported(dhd_bus_t *bus)
+{
+	if (!bus->dhd) {
+		return FALSE;
+	} else if (bus->dar_enabled) {
+		return bus->dhd->dar_enable;
+	} else {
+		return FALSE;
+	}
+}
+
+#ifdef PCIE_OOB
+bool
+dhdpcie_bus_get_pcie_oob_dw_supported(dhd_bus_t *bus)
+{
+	if (!bus->dhd)
+		return FALSE;
+	if (bus->oob_enabled) {
+		return !bus->dhd->d2h_no_oob_dw;
+	} else {
+		return FALSE;
+	}
+}
+#endif /* PCIE_OOB */
+
+void
+dhdpcie_bus_enab_pcie_dw(dhd_bus_t *bus, uint8 dw_option)
+{
+	DHD_ERROR(("ENABLING DW:%d\n", dw_option));
+	bus->dw_option = dw_option;
+}
+
+#ifdef PCIE_INB_DW
+bool
+dhdpcie_bus_get_pcie_inband_dw_supported(dhd_bus_t *bus)
+{
+	if (!bus->dhd)
+		return FALSE;
+	if (bus->inb_enabled) {
+		return bus->dhd->d2h_inband_dw;
+	} else {
+		return FALSE;
+	}
+}
+
+void
+dhdpcie_bus_set_pcie_inband_dw_state(dhd_bus_t *bus, enum dhd_bus_ds_state state)
+{
+	if (!INBAND_DW_ENAB(bus))
+		return;
+
+	DHD_INFO(("%s:%d\n", __FUNCTION__, state));
+	bus->dhd->ds_state = state;
+	if (state == DW_DEVICE_DS_DISABLED_WAIT || state == DW_DEVICE_DS_D3_INFORM_WAIT) {
+		bus->ds_exit_timeout = 100;
+	}
+	if (state == DW_DEVICE_HOST_WAKE_WAIT) {
+		bus->host_sleep_exit_timeout = 100;
+	}
+	if (state == DW_DEVICE_DS_DEV_WAKE) {
+		bus->ds_exit_timeout = 0;
+	}
+	if (state == DW_DEVICE_DS_ACTIVE) {
+		bus->host_sleep_exit_timeout = 0;
+	}
+}
+
+enum dhd_bus_ds_state
+dhdpcie_bus_get_pcie_inband_dw_state(dhd_bus_t *bus)
+{
+	if (!INBAND_DW_ENAB(bus))
+		return DW_DEVICE_DS_INVALID;
+	return bus->dhd->ds_state;
+}
+#endif /* PCIE_INB_DW */
+
+#ifdef DHD_MMIO_TRACE
+static void
+dhd_bus_mmio_trace(dhd_bus_t *bus, uint32 addr, uint32 value, bool set)
+{
+	uint32 cnt = bus->mmio_trace_count % MAX_MMIO_TRACE_SIZE;
+	uint64 ts_cur = OSL_LOCALTIME_NS();
+	uint32 tmp_cnt;
+
+	tmp_cnt = (bus->mmio_trace_count) ? ((bus->mmio_trace_count - 1)
+		% MAX_MMIO_TRACE_SIZE) : cnt;
+
+	if (((DIV_U64_BY_U64(ts_cur, NSEC_PER_USEC) -
+		DIV_U64_BY_U64(bus->mmio_trace[tmp_cnt].timestamp, NSEC_PER_USEC))
+		> MIN_MMIO_TRACE_TIME) || (bus->mmio_trace[tmp_cnt].value !=
+		(value & DHD_RING_IDX))) {
+		bus->mmio_trace_count++;
+	} else {
+		cnt = tmp_cnt;
+	}
+	bus->mmio_trace[cnt].timestamp = ts_cur;
+	bus->mmio_trace[cnt].addr = addr;
+	bus->mmio_trace[cnt].set = set;
+	bus->mmio_trace[cnt].value = value;
+}
+
+void
+dhd_dump_bus_mmio_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	int dumpsz;
+	int i;
+
+	dumpsz = bus->mmio_trace_count < MAX_MMIO_TRACE_SIZE ?
+		bus->mmio_trace_count : MAX_MMIO_TRACE_SIZE;
+	if (dumpsz == 0) {
+		bcm_bprintf(strbuf, "\nEmpty MMIO TRACE\n");
+		return;
+	}
+	bcm_bprintf(strbuf, "---- MMIO TRACE ------\n");
+	bcm_bprintf(strbuf, "Decoding value field, Ex: 0xFF2C00E4, 0xFF->WR/0XDD->RD "
+		"0x2C->Ringid 0x00E4->RD/WR Value\n");
+	bcm_bprintf(strbuf, "Timestamp ns\t\tAddr\t\tW/R\tValue\n");
+	for (i = 0; i < dumpsz; i ++) {
+		bcm_bprintf(strbuf, SEC_USEC_FMT"\t0x%08x\t%s\t0x%08x\n",
+			GET_SEC_USEC(bus->mmio_trace[i].timestamp),
+			bus->mmio_trace[i].addr,
+			bus->mmio_trace[i].set ? "W" : "R",
+			bus->mmio_trace[i].value);
+	}
+}
+#endif /* defined(DHD_MMIO_TRACE) */
+
+static void
+#ifdef PCIE_INB_DW
+dhd_bus_ds_trace(dhd_bus_t *bus, uint32 dsval, bool d2h, enum dhd_bus_ds_state inbstate)
+#else
+dhd_bus_ds_trace(dhd_bus_t *bus, uint32 dsval, bool d2h)
+#endif /* PCIE_INB_DW */
+{
+	uint32 cnt = bus->ds_trace_count % MAX_DS_TRACE_SIZE;
+
+	bus->ds_trace[cnt].timestamp = OSL_LOCALTIME_NS();
+	bus->ds_trace[cnt].d2h = d2h;
+	bus->ds_trace[cnt].dsval = dsval;
+#ifdef PCIE_INB_DW
+	bus->ds_trace[cnt].inbstate = inbstate;
+#endif /* PCIE_INB_DW */
+	bus->ds_trace_count ++;
+}
+
+#ifdef PCIE_INB_DW
+const char *
+dhd_convert_dsval(uint32 val, bool d2h)
+{
+	if (d2h) {
+		switch (val) {
+			case D2H_DEV_D3_ACK:
+				return "D2H_DEV_D3_ACK";
+			case D2H_DEV_DS_ENTER_REQ:
+				return "D2H_DEV_DS_ENTER_REQ";
+			case D2H_DEV_DS_EXIT_NOTE:
+				return "D2H_DEV_DS_EXIT_NOTE";
+			case D2H_DEV_FWHALT:
+				return "D2H_DEV_FWHALT";
+			case D2HMB_DS_HOST_SLEEP_EXIT_ACK:
+				return "D2HMB_DS_HOST_SLEEP_EXIT_ACK";
+			default:
+				return "INVALID";
+		}
+	} else {
+		switch (val) {
+			case H2DMB_DS_DEVICE_WAKE_DEASSERT:
+				return "H2DMB_DS_DEVICE_WAKE_DEASSERT";
+			case H2DMB_DS_DEVICE_WAKE_ASSERT:
+				return "H2DMB_DS_DEVICE_WAKE_ASSERT";
+			case H2D_HOST_D3_INFORM:
+				return "H2D_HOST_D3_INFORM";
+			case H2D_HOST_DS_ACK:
+				return "H2D_HOST_DS_ACK";
+			case H2D_HOST_DS_NAK:
+				return "H2D_HOST_DS_NAK";
+			case H2D_HOST_CONS_INT:
+				return "H2D_HOST_CONS_INT";
+			case H2D_FW_TRAP:
+				return "H2D_FW_TRAP";
+			default:
+				return "INVALID";
+		}
+	}
+}
+
+const char *
+dhd_convert_inb_state_names(enum dhd_bus_ds_state inbstate)
+{
+	switch (inbstate) {
+		case DW_DEVICE_DS_DEV_SLEEP:
+			return "DW_DEVICE_DS_DEV_SLEEP";
+		break;
+		case DW_DEVICE_DS_DISABLED_WAIT:
+			return "DW_DEVICE_DS_DISABLED_WAIT";
+		break;
+		case DW_DEVICE_DS_DEV_WAKE:
+			return "DW_DEVICE_DS_DEV_WAKE";
+		break;
+		case DW_DEVICE_DS_ACTIVE:
+			return "DW_DEVICE_DS_ACTIVE";
+		break;
+		case DW_DEVICE_HOST_SLEEP_WAIT:
+			return "DW_DEVICE_HOST_SLEEP_WAIT";
+		break;
+		case DW_DEVICE_HOST_SLEEP:
+			return "DW_DEVICE_HOST_SLEEP";
+		break;
+		case DW_DEVICE_HOST_WAKE_WAIT:
+			return "DW_DEVICE_HOST_WAKE_WAIT";
+		break;
+		case DW_DEVICE_DS_D3_INFORM_WAIT:
+			return "DW_DEVICE_DS_D3_INFORM_WAIT";
+		break;
+		default:
+			return "INVALID";
+	}
+}
+#endif /* PCIE_INB_DW */
+
+void
+dhd_dump_bus_ds_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	int dumpsz;
+	int i;
+
+	dumpsz = bus->ds_trace_count < MAX_DS_TRACE_SIZE ?
+		bus->ds_trace_count : MAX_DS_TRACE_SIZE;
+	if (dumpsz == 0) {
+		bcm_bprintf(strbuf, "\nEmpty DS TRACE\n");
+		return;
+	}
+	bcm_bprintf(strbuf, "---- DS TRACE ------\n");
+#ifdef PCIE_INB_DW
+	bcm_bprintf(strbuf, "%s\t\t%s\t%-30s\t\t%s\n",
+		"Timestamp us", "Dir", "Value", "Inband-State");
+	for (i = 0; i < dumpsz; i ++) {
+		bcm_bprintf(strbuf, "%llu\t%s\t%-30s\t\t%s\n",
+		bus->ds_trace[i].timestamp,
+		bus->ds_trace[i].d2h ? "D2H":"H2D",
+		dhd_convert_dsval(bus->ds_trace[i].dsval, bus->ds_trace[i].d2h),
+		dhd_convert_inb_state_names(bus->ds_trace[i].inbstate));
+	}
+#else
+	bcm_bprintf(strbuf, "Timestamp us\t\tDir\tValue\n");
+	for (i = 0; i < dumpsz; i ++) {
+		bcm_bprintf(strbuf, "%llu\t%s\t%d\n",
+		bus->ds_trace[i].timestamp,
+		bus->ds_trace[i].d2h ? "D2H":"H2D",
+		bus->ds_trace[i].dsval);
+	}
+#endif /* PCIE_INB_DW */
+	bcm_bprintf(strbuf, "--------------------------\n");
+}
+
+void
+dhd_bus_dump_trap_info(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	trap_t *tr = &bus->dhd->last_trap_info;
+	bcm_bprintf(strbuf,
+		"\nTRAP type 0x%x @ epc 0x%x, cpsr 0x%x, spsr 0x%x, sp 0x%x,"
+		" lp 0x%x, rpc 0x%x"
+		"\nTrap offset 0x%x, r0 0x%x, r1 0x%x, r2 0x%x, r3 0x%x, "
+		"r4 0x%x, r5 0x%x, r6 0x%x, r7 0x%x, r8 0x%x, r9 0x%x, "
+		"r10 0x%x, r11 0x%x, r12 0x%x\n\n",
+		ltoh32(tr->type), ltoh32(tr->epc), ltoh32(tr->cpsr), ltoh32(tr->spsr),
+		ltoh32(tr->r13), ltoh32(tr->r14), ltoh32(tr->pc),
+		ltoh32(bus->pcie_sh->trap_addr),
+		ltoh32(tr->r0), ltoh32(tr->r1), ltoh32(tr->r2), ltoh32(tr->r3),
+		ltoh32(tr->r4), ltoh32(tr->r5), ltoh32(tr->r6), ltoh32(tr->r7),
+		ltoh32(tr->r8), ltoh32(tr->r9), ltoh32(tr->r10),
+		ltoh32(tr->r11), ltoh32(tr->r12));
+}
+
+int
+dhd_bus_readwrite_bp_addr(dhd_pub_t *dhdp, uint addr, uint size, uint* data, bool read)
+{
+	int bcmerror = 0;
+	struct dhd_bus *bus = dhdp->bus;
+
+	if (serialized_backplane_access(bus, addr, size, data, read) != BCME_OK) {
+			DHD_ERROR(("Invalid size/addr combination \n"));
+			bcmerror = BCME_ERROR;
+	}
+
+	return bcmerror;
+}
+
+int
+dhd_get_idletime(dhd_pub_t *dhd)
+{
+	return dhd->bus->idletime;
+}
+
+bool
+dhd_get_rpm_state(dhd_pub_t *dhd)
+{
+	return dhd->bus->rpm_enabled;
+}
+
+void
+dhd_set_rpm_state(dhd_pub_t *dhd, bool state)
+{
+	DHD_RPM(("%s: %d\n", __FUNCTION__, state));
+	dhd->bus->rpm_enabled = state;
+}
+
+static INLINE void
+dhd_sbreg_op(dhd_pub_t *dhd, uint addr, uint *val, bool read)
+{
+	OSL_DELAY(1);
+	if (serialized_backplane_access(dhd->bus, addr, sizeof(uint), val, read) != BCME_OK) {
+		DHD_ERROR(("sbreg: Invalid uint addr: 0x%x \n", addr));
+	} else {
+		DHD_ERROR(("sbreg: addr:0x%x val:0x%x read:%d\n", addr, *val, read));
+	}
+	return;
+}
+
+#ifdef DHD_SSSR_DUMP
+static int
+dhdpcie_get_sssr_fifo_dump(dhd_pub_t *dhd, uint *buf, uint fifo_size,
+	uint addr_reg, uint data_reg)
+{
+	uint addr;
+	uint val = 0;
+	int i;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	if (!buf) {
+		DHD_ERROR(("%s: buf is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (!fifo_size) {
+		DHD_ERROR(("%s: fifo_size is 0\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Set the base address offset to 0 */
+	addr = addr_reg;
+	val = 0;
+	dhd_sbreg_op(dhd, addr, &val, FALSE);
+
+	addr = data_reg;
+	/* Read 4 bytes at once and loop for fifo_size / 4 */
+	for (i = 0; i < fifo_size / 4; i++) {
+		if (serialized_backplane_access(dhd->bus, addr,
+				sizeof(uint), &val, TRUE) != BCME_OK) {
+			DHD_ERROR(("%s: error in serialized_backplane_access\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+		buf[i] = val;
+		OSL_DELAY(1);
+	}
+	return BCME_OK;
+}
+
+static int
+dhdpcie_get_sssr_dig_dump(dhd_pub_t *dhd, uint *buf, uint fifo_size,
+	uint addr_reg)
+{
+	uint addr;
+	uint val = 0;
+	int i;
+	si_t *sih = dhd->bus->sih;
+	bool vasip_enab, dig_mem_check;
+	uint32 ioctrl_addr = 0;
+
+	DHD_ERROR(("%s addr_reg=0x%x size=0x%x\n", __FUNCTION__, addr_reg, fifo_size));
+
+	if (!buf) {
+		DHD_ERROR(("%s: buf is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (!fifo_size) {
+		DHD_ERROR(("%s: fifo_size is 0\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	vasip_enab = FALSE;
+	dig_mem_check = FALSE;
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			if ((dhd->sssr_reg_info->rev2.length > OFFSETOF(sssr_reg_info_v2_t,
+			dig_mem_info)) && dhd->sssr_reg_info->rev2.dig_mem_info.dig_sr_size) {
+				dig_mem_check = TRUE;
+			}
+			break;
+		case SSSR_REG_INFO_VER_1 :
+			if (dhd->sssr_reg_info->rev1.vasip_regs.vasip_sr_size) {
+				vasip_enab = TRUE;
+			} else if ((dhd->sssr_reg_info->rev1.length > OFFSETOF(sssr_reg_info_v1_t,
+				dig_mem_info)) && dhd->sssr_reg_info->rev1.
+				dig_mem_info.dig_sr_size) {
+				dig_mem_check = TRUE;
+			}
+			ioctrl_addr = dhd->sssr_reg_info->rev1.vasip_regs.wrapper_regs.ioctrl;
+			break;
+		case SSSR_REG_INFO_VER_0 :
+			if (dhd->sssr_reg_info->rev0.vasip_regs.vasip_sr_size) {
+				vasip_enab = TRUE;
+			}
+			ioctrl_addr = dhd->sssr_reg_info->rev0.vasip_regs.wrapper_regs.ioctrl;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+	if (addr_reg) {
+		DHD_ERROR(("dig_mem_check=%d vasip_enab=%d\n", dig_mem_check, vasip_enab));
+		if (!vasip_enab && dig_mem_check) {
+			int err = dhdpcie_bus_membytes(dhd->bus, FALSE, addr_reg, (uint8 *)buf,
+				fifo_size);
+			if (err != BCME_OK) {
+				DHD_ERROR(("%s: Error reading dig dump from dongle !\n",
+					__FUNCTION__));
+			}
+		} else {
+			/* Check if vasip clk is disabled, if yes enable it */
+			addr = ioctrl_addr;
+			dhd_sbreg_op(dhd, addr, &val, TRUE);
+			if (!val) {
+				val = 1;
+				dhd_sbreg_op(dhd, addr, &val, FALSE);
+			}
+
+			addr = addr_reg;
+			/* Read 4 bytes at once and loop for fifo_size / 4 */
+			for (i = 0; i < fifo_size / 4; i++, addr += 4) {
+				if (serialized_backplane_access(dhd->bus, addr, sizeof(uint),
+					&val, TRUE) != BCME_OK) {
+					DHD_ERROR(("%s: Invalid uint addr: 0x%x \n", __FUNCTION__,
+						addr));
+					return BCME_ERROR;
+				}
+				buf[i] = val;
+				OSL_DELAY(1);
+			}
+		}
+	} else {
+		uint cur_coreid;
+		uint chipc_corerev;
+		chipcregs_t *chipcregs;
+
+		/* Save the current core */
+		cur_coreid = si_coreid(sih);
+
+		/* Switch to ChipC */
+		chipcregs = (chipcregs_t *)si_setcore(sih, CC_CORE_ID, 0);
+		if (!chipcregs) {
+			DHD_ERROR(("%s: si_setcore returns NULL for core id %u \n",
+				__FUNCTION__, CC_CORE_ID));
+			return BCME_ERROR;
+		}
+
+		chipc_corerev = si_corerev(sih);
+
+		if ((chipc_corerev == 64) || (chipc_corerev == 65)) {
+			W_REG(si_osh(sih), &chipcregs->sr_memrw_addr, 0);
+
+			/* Read 4 bytes at once and loop for fifo_size / 4 */
+			for (i = 0; i < fifo_size / 4; i++) {
+				buf[i] = R_REG(si_osh(sih), &chipcregs->sr_memrw_data);
+				OSL_DELAY(1);
+			}
+		}
+
+		/* Switch back to the original core */
+		si_setcore(sih, cur_coreid, 0);
+	}
+
+	return BCME_OK;
+}
+
+#if defined(BCMPCIE) && defined(EWP_ETD_PRSRV_LOGS)
+void
+dhdpcie_get_etd_preserve_logs(dhd_pub_t *dhd,
+		uint8 *ext_trap_data, void *event_decode_data)
+{
+	hnd_ext_trap_hdr_t *hdr = NULL;
+	bcm_tlv_t *tlv;
+	eventlog_trapdata_info_t *etd_evtlog = NULL;
+	eventlog_trap_buf_info_t *evtlog_buf_arr = NULL;
+	uint arr_size = 0;
+	int i = 0;
+	int err = 0;
+	uint32 seqnum = 0;
+
+	if (!ext_trap_data || !event_decode_data || !dhd)
+		return;
+
+	if (!dhd->concise_dbg_buf)
+		return;
+
+	/* First word is original trap_data, skip */
+	ext_trap_data += sizeof(uint32);
+
+	hdr = (hnd_ext_trap_hdr_t *)ext_trap_data;
+	tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_LOG_DATA);
+	if (tlv) {
+		uint32 baseaddr = 0;
+		uint32 endaddr = dhd->bus->dongle_ram_base + dhd->bus->ramsize - 4;
+
+		etd_evtlog = (eventlog_trapdata_info_t *)tlv->data;
+		DHD_ERROR(("%s: etd_evtlog tlv found, num_elements=%x; "
+			"seq_num=%x; log_arr_addr=%x\n", __FUNCTION__,
+			(etd_evtlog->num_elements),
+			ntoh32(etd_evtlog->seq_num), (etd_evtlog->log_arr_addr)));
+		if (!etd_evtlog->num_elements ||
+				etd_evtlog->num_elements > MAX_EVENTLOG_BUFFERS) {
+			DHD_ERROR(("%s: ETD has bad 'num_elements' !\n", __FUNCTION__));
+			return;
+		}
+		if (!etd_evtlog->log_arr_addr) {
+			DHD_ERROR(("%s: ETD has bad 'log_arr_addr' !\n", __FUNCTION__));
+			return;
+		}
+
+		arr_size = (uint32)sizeof(*evtlog_buf_arr) * (etd_evtlog->num_elements);
+		evtlog_buf_arr = MALLOCZ(dhd->osh, arr_size);
+		if (!evtlog_buf_arr) {
+			DHD_ERROR(("%s: out of memory !\n",	__FUNCTION__));
+			return;
+		}
+
+		/* boundary check */
+		baseaddr = etd_evtlog->log_arr_addr;
+		if ((baseaddr < dhd->bus->dongle_ram_base) || (baseaddr > endaddr) ||
+			((baseaddr + arr_size) > endaddr)) {
+			DHD_ERROR(("%s: Error reading invalid address\n",
+				__FUNCTION__));
+			goto err;
+		}
+
+		/* read the eventlog_trap_buf_info_t array from dongle memory */
+		err = dhdpcie_bus_membytes(dhd->bus, FALSE,
+				(ulong)(etd_evtlog->log_arr_addr),
+				(uint8 *)evtlog_buf_arr, arr_size);
+		if (err != BCME_OK) {
+			DHD_ERROR(("%s: Error reading event log array from dongle !\n",
+				__FUNCTION__));
+			goto err;
+		}
+		/* ntoh is required only for seq_num, because in the original
+		* case of event logs from info ring, it is sent from dongle in that way
+		* so for ETD also dongle follows same convention
+		*/
+		seqnum = ntoh32(etd_evtlog->seq_num);
+		memset(dhd->concise_dbg_buf, 0, CONCISE_DUMP_BUFLEN);
+		for (i = 0; i < (etd_evtlog->num_elements); ++i) {
+			/* boundary check */
+			baseaddr = evtlog_buf_arr[i].buf_addr;
+			if ((baseaddr < dhd->bus->dongle_ram_base) || (baseaddr > endaddr) ||
+				((baseaddr + evtlog_buf_arr[i].len) > endaddr)) {
+				DHD_ERROR(("%s: Error reading invalid address\n",
+					__FUNCTION__));
+				goto err;
+			}
+			/* read each individual event log buf from dongle memory */
+			err = dhdpcie_bus_membytes(dhd->bus, FALSE,
+					((ulong)evtlog_buf_arr[i].buf_addr),
+					dhd->concise_dbg_buf, (evtlog_buf_arr[i].len));
+			if (err != BCME_OK) {
+				DHD_ERROR(("%s: Error reading event log buffer from dongle !\n",
+					__FUNCTION__));
+				goto err;
+			}
+			dhd_dbg_msgtrace_log_parser(dhd, dhd->concise_dbg_buf,
+				event_decode_data, (evtlog_buf_arr[i].len),
+				FALSE, hton32(seqnum));
+			++seqnum;
+		}
+err:
+		MFREE(dhd->osh, evtlog_buf_arr, arr_size);
+	} else {
+		DHD_ERROR(("%s: Error getting trap log data in ETD !\n", __FUNCTION__));
+	}
+}
+#endif /* BCMPCIE && DHD_LOG_DUMP */
+
+static uint32
+dhdpcie_resume_chipcommon_powerctrl(dhd_pub_t *dhd, uint32 reg_val)
+{
+	uint addr;
+	uint val = 0;
+	uint powerctrl_mask;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			addr = dhd->sssr_reg_info->rev2.chipcommon_regs.base_regs.powerctrl;
+			powerctrl_mask = dhd->sssr_reg_info->rev2.
+				chipcommon_regs.base_regs.powerctrl_mask;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			addr = dhd->sssr_reg_info->rev1.chipcommon_regs.base_regs.powerctrl;
+			powerctrl_mask = dhd->sssr_reg_info->rev1.
+				chipcommon_regs.base_regs.powerctrl_mask;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* conditionally clear bits [11:8] of PowerCtrl */
+	dhd_sbreg_op(dhd, addr, &val, TRUE);
+
+	if (!(val & powerctrl_mask)) {
+		dhd_sbreg_op(dhd, addr, &reg_val, FALSE);
+	}
+	return BCME_OK;
+}
+
+static uint32
+dhdpcie_suspend_chipcommon_powerctrl(dhd_pub_t *dhd)
+{
+	uint addr;
+	uint val = 0, reg_val = 0;
+	uint powerctrl_mask;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			addr = dhd->sssr_reg_info->rev2.chipcommon_regs.base_regs.powerctrl;
+			powerctrl_mask = dhd->sssr_reg_info->rev2.
+				chipcommon_regs.base_regs.powerctrl_mask;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			addr = dhd->sssr_reg_info->rev1.chipcommon_regs.base_regs.powerctrl;
+			powerctrl_mask = dhd->sssr_reg_info->rev1.
+				chipcommon_regs.base_regs.powerctrl_mask;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* conditionally clear bits [11:8] of PowerCtrl */
+	dhd_sbreg_op(dhd, addr, &reg_val, TRUE);
+	if (reg_val & powerctrl_mask) {
+		val = 0;
+		dhd_sbreg_op(dhd, addr, &val, FALSE);
+	}
+	return reg_val;
+}
+
+static int
+dhdpcie_clear_intmask_and_timer(dhd_pub_t *dhd)
+{
+	uint addr;
+	uint val;
+	uint32 cc_intmask, pmuintmask0, pmuintmask1, resreqtimer, macresreqtimer,
+	 macresreqtimer1, vasip_sr_size = 0;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			cc_intmask = dhd->sssr_reg_info->rev2.chipcommon_regs.base_regs.intmask;
+			pmuintmask0 = dhd->sssr_reg_info->rev2.pmu_regs.base_regs.pmuintmask0;
+			pmuintmask1 = dhd->sssr_reg_info->rev2.pmu_regs.base_regs.pmuintmask1;
+			resreqtimer = dhd->sssr_reg_info->rev2.pmu_regs.base_regs.resreqtimer;
+			macresreqtimer = dhd->sssr_reg_info->rev2.pmu_regs.base_regs.macresreqtimer;
+			macresreqtimer1 = dhd->sssr_reg_info->rev2.
+				pmu_regs.base_regs.macresreqtimer1;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			cc_intmask = dhd->sssr_reg_info->rev1.chipcommon_regs.base_regs.intmask;
+			pmuintmask0 = dhd->sssr_reg_info->rev1.pmu_regs.base_regs.pmuintmask0;
+			pmuintmask1 = dhd->sssr_reg_info->rev1.pmu_regs.base_regs.pmuintmask1;
+			resreqtimer = dhd->sssr_reg_info->rev1.pmu_regs.base_regs.resreqtimer;
+			macresreqtimer = dhd->sssr_reg_info->rev1.pmu_regs.base_regs.macresreqtimer;
+			macresreqtimer1 = dhd->sssr_reg_info->rev1.
+				pmu_regs.base_regs.macresreqtimer1;
+			vasip_sr_size = dhd->sssr_reg_info->rev1.vasip_regs.vasip_sr_size;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* clear chipcommon intmask */
+	val = 0x0;
+	dhd_sbreg_op(dhd, cc_intmask, &val, FALSE);
+
+	/* clear PMUIntMask0 */
+	val = 0x0;
+	dhd_sbreg_op(dhd, pmuintmask0, &val, FALSE);
+
+	/* clear PMUIntMask1 */
+	val = 0x0;
+	dhd_sbreg_op(dhd, pmuintmask1, &val, FALSE);
+
+	/* clear res_req_timer */
+	val = 0x0;
+	dhd_sbreg_op(dhd, resreqtimer, &val, FALSE);
+
+	/* clear macresreqtimer */
+	val = 0x0;
+	dhd_sbreg_op(dhd, macresreqtimer, &val, FALSE);
+
+	/* clear macresreqtimer1 */
+	val = 0x0;
+	dhd_sbreg_op(dhd, macresreqtimer1, &val, FALSE);
+
+	/* clear VasipClkEn */
+	if (vasip_sr_size) {
+		addr = dhd->sssr_reg_info->rev1.vasip_regs.wrapper_regs.ioctrl;
+		val = 0x0;
+		dhd_sbreg_op(dhd, addr, &val, FALSE);
+	}
+
+	return BCME_OK;
+}
+
+static void
+dhdpcie_update_d11_status_from_trapdata(dhd_pub_t *dhd)
+{
+#define TRAP_DATA_MAIN_CORE_BIT_MASK	(1 << 1)
+#define TRAP_DATA_AUX_CORE_BIT_MASK	(1 << 4)
+	uint trap_data_mask[MAX_NUM_D11CORES] =
+		{TRAP_DATA_MAIN_CORE_BIT_MASK, TRAP_DATA_AUX_CORE_BIT_MASK};
+	int i;
+	/* Apply only for 4375 chip */
+	if (dhd_bus_chip_id(dhd) == BCM4375_CHIP_ID) {
+		for (i = 0; i < MAX_NUM_D11CORES; i++) {
+			if (dhd->sssr_d11_outofreset[i] &&
+				(dhd->dongle_trap_data & trap_data_mask[i])) {
+				dhd->sssr_d11_outofreset[i] = TRUE;
+			} else {
+				dhd->sssr_d11_outofreset[i] = FALSE;
+			}
+			DHD_ERROR(("%s: sssr_d11_outofreset[%d] : %d after AND with "
+				"trap_data:0x%x-0x%x\n",
+				__FUNCTION__, i, dhd->sssr_d11_outofreset[i],
+				dhd->dongle_trap_data, trap_data_mask[i]));
+		}
+	}
+}
+
+static int
+dhdpcie_d11_check_outofreset(dhd_pub_t *dhd)
+{
+	int i;
+	uint addr = 0;
+	uint val = 0;
+	uint8 num_d11cores;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	for (i = 0; i < num_d11cores; i++) {
+		/* Check if bit 0 of resetctrl is cleared */
+		/* SSSR register information structure v0 and
+		 * v1 shares most except dig_mem
+		 */
+		switch (dhd->sssr_reg_info->rev2.version) {
+			case SSSR_REG_INFO_VER_3 :
+				/* intentional fall through */
+			case SSSR_REG_INFO_VER_2 :
+				addr = dhd->sssr_reg_info->rev2.
+					mac_regs[i].wrapper_regs.resetctrl;
+				break;
+			case SSSR_REG_INFO_VER_1 :
+			case SSSR_REG_INFO_VER_0 :
+				addr = dhd->sssr_reg_info->rev1.
+					mac_regs[i].wrapper_regs.resetctrl;
+				break;
+			default :
+				DHD_ERROR(("invalid sssr_reg_ver"));
+				return BCME_UNSUPPORTED;
+		}
+		if (!addr) {
+			DHD_ERROR(("%s: skipping for core[%d] as 'addr' is NULL\n",
+				__FUNCTION__, i));
+			continue;
+		}
+		dhd_sbreg_op(dhd, addr, &val, TRUE);
+		if (!(val & 1)) {
+			dhd->sssr_d11_outofreset[i] = TRUE;
+		} else {
+			dhd->sssr_d11_outofreset[i] = FALSE;
+		}
+		DHD_ERROR(("%s: sssr_d11_outofreset[%d] : %d\n",
+			__FUNCTION__, i, dhd->sssr_d11_outofreset[i]));
+	}
+	/* XXX Temporary WAR for 4375 to handle AXI errors on bad core
+	 * to not collect SSSR dump for the core whose bit is not set in trap_data.
+	 * It will be reverted once AXI errors are fixed
+	 */
+	dhdpcie_update_d11_status_from_trapdata(dhd);
+
+	return BCME_OK;
+}
+
+static int
+dhdpcie_d11_clear_clk_req(dhd_pub_t *dhd)
+{
+	int i;
+	uint val = 0;
+	uint8 num_d11cores;
+	uint32 clockrequeststatus, clockcontrolstatus, clockcontrolstatus_val;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	for (i = 0; i < num_d11cores; i++) {
+		if (dhd->sssr_d11_outofreset[i]) {
+			/* clear request clk only if itopoobb/extrsrcreqs is non zero */
+			/* SSSR register information structure v0 and
+			 * v1 shares most except dig_mem
+			 */
+			switch (dhd->sssr_reg_info->rev2.version) {
+				case SSSR_REG_INFO_VER_3 :
+					/* intentional fall through */
+				case SSSR_REG_INFO_VER_2 :
+					clockrequeststatus = dhd->sssr_reg_info->rev2.
+						mac_regs[i].wrapper_regs.extrsrcreq;
+					clockcontrolstatus = dhd->sssr_reg_info->rev2.
+						mac_regs[i].base_regs.clockcontrolstatus;
+					clockcontrolstatus_val = dhd->sssr_reg_info->rev2.
+						mac_regs[i].base_regs.clockcontrolstatus_val;
+					break;
+				case SSSR_REG_INFO_VER_1 :
+				case SSSR_REG_INFO_VER_0 :
+					clockrequeststatus = dhd->sssr_reg_info->rev1.
+						mac_regs[i].wrapper_regs.itopoobb;
+					clockcontrolstatus = dhd->sssr_reg_info->rev1.
+						mac_regs[i].base_regs.clockcontrolstatus;
+					clockcontrolstatus_val = dhd->sssr_reg_info->rev1.
+						mac_regs[i].base_regs.clockcontrolstatus_val;
+					break;
+				default :
+					DHD_ERROR(("invalid sssr_reg_ver"));
+					return BCME_UNSUPPORTED;
+			}
+			dhd_sbreg_op(dhd, clockrequeststatus, &val, TRUE);
+			if (val != 0) {
+				/* clear clockcontrolstatus */
+				dhd_sbreg_op(dhd, clockcontrolstatus,
+				 &clockcontrolstatus_val, FALSE);
+			}
+		}
+	}
+	return BCME_OK;
+}
+
+static int
+dhdpcie_arm_clear_clk_req(dhd_pub_t *dhd)
+{
+	uint val = 0;
+	uint cfgval = 0;
+	uint32 resetctrl, clockrequeststatus, clockcontrolstatus, clockcontrolstatus_val;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			resetctrl = dhd->sssr_reg_info->rev2.
+				arm_regs.wrapper_regs.resetctrl;
+			clockrequeststatus = dhd->sssr_reg_info->rev2.
+				arm_regs.wrapper_regs.extrsrcreq;
+			clockcontrolstatus = dhd->sssr_reg_info->rev2.
+				arm_regs.base_regs.clockcontrolstatus;
+			clockcontrolstatus_val = dhd->sssr_reg_info->rev2.
+				arm_regs.base_regs.clockcontrolstatus_val;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			resetctrl = dhd->sssr_reg_info->rev1.
+				arm_regs.wrapper_regs.resetctrl;
+			clockrequeststatus = dhd->sssr_reg_info->rev1.
+				arm_regs.wrapper_regs.itopoobb;
+			clockcontrolstatus = dhd->sssr_reg_info->rev1.
+				arm_regs.base_regs.clockcontrolstatus;
+			clockcontrolstatus_val = dhd->sssr_reg_info->rev1.
+				arm_regs.base_regs.clockcontrolstatus_val;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* Check if bit 0 of resetctrl is cleared */
+	dhd_sbreg_op(dhd, resetctrl, &val, TRUE);
+	if (!(val & 1)) {
+		/* clear request clk only if itopoobb/extrsrcreqs is non zero */
+		dhd_sbreg_op(dhd, clockrequeststatus, &val, TRUE);
+		if (val != 0) {
+			/* clear clockcontrolstatus */
+			dhd_sbreg_op(dhd, clockcontrolstatus, &clockcontrolstatus_val, FALSE);
+		}
+
+		if (MULTIBP_ENAB(dhd->bus->sih)) {
+			/* Clear coherent bits for CA7 because CPU is halted */
+			if (dhd->bus->coreid == ARMCA7_CORE_ID) {
+				cfgval = dhdpcie_bus_cfg_read_dword(dhd->bus,
+					PCIE_CFG_SUBSYSTEM_CONTROL, 4);
+				dhdpcie_bus_cfg_write_dword(dhd->bus, PCIE_CFG_SUBSYSTEM_CONTROL, 4,
+					(cfgval & ~PCIE_BARCOHERENTACCEN_MASK));
+			}
+
+			/* Just halt ARM but do not reset the core */
+			resetctrl &= ~(SI_CORE_SIZE - 1);
+			resetctrl += OFFSETOF(aidmp_t, ioctrl);
+
+			dhd_sbreg_op(dhd, resetctrl, &val, TRUE);
+			val |= SICF_CPUHALT;
+			dhd_sbreg_op(dhd, resetctrl, &val, FALSE);
+		}
+	}
+
+	return BCME_OK;
+}
+
+static int
+dhdpcie_arm_resume_clk_req(dhd_pub_t *dhd)
+{
+	uint val = 0;
+	uint32 resetctrl;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			resetctrl = dhd->sssr_reg_info->rev2.
+				arm_regs.wrapper_regs.resetctrl;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			resetctrl = dhd->sssr_reg_info->rev1.
+				arm_regs.wrapper_regs.resetctrl;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* Check if bit 0 of resetctrl is cleared */
+	dhd_sbreg_op(dhd, resetctrl, &val, TRUE);
+	if (!(val & 1)) {
+		if (MULTIBP_ENAB(dhd->bus->sih) && (dhd->bus->coreid != ARMCA7_CORE_ID)) {
+			/* Take ARM out of halt but do not reset core */
+			resetctrl &= ~(SI_CORE_SIZE - 1);
+			resetctrl += OFFSETOF(aidmp_t, ioctrl);
+
+			dhd_sbreg_op(dhd, resetctrl, &val, TRUE);
+			val &= ~SICF_CPUHALT;
+			dhd_sbreg_op(dhd, resetctrl, &val, FALSE);
+			dhd_sbreg_op(dhd, resetctrl, &val, TRUE);
+		}
+	}
+
+	return BCME_OK;
+}
+
+static int
+dhdpcie_pcie_clear_clk_req(dhd_pub_t *dhd)
+{
+	uint val = 0;
+	uint32 clockrequeststatus, clockcontrolstatus_addr, clockcontrolstatus_val;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			clockrequeststatus = dhd->sssr_reg_info->rev2.
+				pcie_regs.wrapper_regs.extrsrcreq;
+			clockcontrolstatus_addr = dhd->sssr_reg_info->rev2.
+				pcie_regs.base_regs.clockcontrolstatus;
+			clockcontrolstatus_val = dhd->sssr_reg_info->rev2.
+				pcie_regs.base_regs.clockcontrolstatus_val;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			clockrequeststatus = dhd->sssr_reg_info->rev1.
+				pcie_regs.wrapper_regs.itopoobb;
+			clockcontrolstatus_addr = dhd->sssr_reg_info->rev1.
+				pcie_regs.base_regs.clockcontrolstatus;
+			clockcontrolstatus_val = dhd->sssr_reg_info->rev1.
+				pcie_regs.base_regs.clockcontrolstatus_val;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	/* clear request clk only if itopoobb/extrsrcreqs is non zero */
+	dhd_sbreg_op(dhd, clockrequeststatus, &val, TRUE);
+	if (val) {
+		/* clear clockcontrolstatus */
+		dhd_sbreg_op(dhd, clockcontrolstatus_addr, &clockcontrolstatus_val, FALSE);
+	}
+	return BCME_OK;
+}
+
+static int
+dhdpcie_pcie_send_ltrsleep(dhd_pub_t *dhd)
+{
+	uint addr;
+	uint val = 0;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	/* SSSR register information structure v0 and v1 shares most except dig_mem */
+	switch (dhd->sssr_reg_info->rev2.version) {
+		case SSSR_REG_INFO_VER_3 :
+			/* intentional fall through */
+		case SSSR_REG_INFO_VER_2 :
+			addr = dhd->sssr_reg_info->rev2.pcie_regs.base_regs.ltrstate;
+			break;
+		case SSSR_REG_INFO_VER_1 :
+		case SSSR_REG_INFO_VER_0 :
+			addr = dhd->sssr_reg_info->rev1.pcie_regs.base_regs.ltrstate;
+			break;
+		default :
+			DHD_ERROR(("invalid sssr_reg_ver"));
+			return BCME_UNSUPPORTED;
+	}
+
+	val = LTR_ACTIVE;
+	dhd_sbreg_op(dhd, addr, &val, FALSE);
+
+	val = LTR_SLEEP;
+	dhd_sbreg_op(dhd, addr, &val, FALSE);
+
+	return BCME_OK;
+}
+
+static int
+dhdpcie_clear_clk_req(dhd_pub_t *dhd)
+{
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	dhdpcie_arm_clear_clk_req(dhd);
+
+	dhdpcie_d11_clear_clk_req(dhd);
+
+	dhdpcie_pcie_clear_clk_req(dhd);
+
+	return BCME_OK;
+}
+
+static int
+dhdpcie_bring_d11_outofreset(dhd_pub_t *dhd)
+{
+	int i;
+	uint val = 0;
+	uint8 num_d11cores;
+	uint32 resetctrl_addr, ioctrl_addr, ioctrl_resetseq_val0, ioctrl_resetseq_val1,
+	 ioctrl_resetseq_val2, ioctrl_resetseq_val3, ioctrl_resetseq_val4;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	for (i = 0; i < num_d11cores; i++) {
+		if (dhd->sssr_d11_outofreset[i]) {
+			/* SSSR register information structure v0 and v1 shares
+			 * most except dig_mem
+			 */
+			switch (dhd->sssr_reg_info->rev2.version) {
+				case SSSR_REG_INFO_VER_3 :
+					/* intentional fall through */
+				case SSSR_REG_INFO_VER_2 :
+					resetctrl_addr = dhd->sssr_reg_info->rev2.mac_regs[i].
+						wrapper_regs.resetctrl;
+					ioctrl_addr = dhd->sssr_reg_info->rev2.mac_regs[i].
+						wrapper_regs.ioctrl;
+					ioctrl_resetseq_val0 = dhd->sssr_reg_info->rev2.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[0];
+					ioctrl_resetseq_val1 = dhd->sssr_reg_info->rev2.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[1];
+					ioctrl_resetseq_val2 = dhd->sssr_reg_info->rev2.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[2];
+					ioctrl_resetseq_val3 = dhd->sssr_reg_info->rev2.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[3];
+					ioctrl_resetseq_val4 = dhd->sssr_reg_info->rev2.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[4];
+					break;
+				case SSSR_REG_INFO_VER_1 :
+				case SSSR_REG_INFO_VER_0 :
+					resetctrl_addr = dhd->sssr_reg_info->rev1.mac_regs[i].
+						wrapper_regs.resetctrl;
+					ioctrl_addr = dhd->sssr_reg_info->rev1.mac_regs[i].
+						wrapper_regs.ioctrl;
+					ioctrl_resetseq_val0 = dhd->sssr_reg_info->rev1.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[0];
+					ioctrl_resetseq_val1 = dhd->sssr_reg_info->rev1.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[1];
+					ioctrl_resetseq_val2 = dhd->sssr_reg_info->rev1.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[2];
+					ioctrl_resetseq_val3 = dhd->sssr_reg_info->rev1.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[3];
+					ioctrl_resetseq_val4 = dhd->sssr_reg_info->rev1.
+						mac_regs[0].wrapper_regs.ioctrl_resetseq_val[4];
+					break;
+				default :
+					DHD_ERROR(("invalid sssr_reg_ver"));
+					return BCME_UNSUPPORTED;
+			}
+			/* disable core by setting bit 0 */
+			val = 1;
+			dhd_sbreg_op(dhd, resetctrl_addr, &val, FALSE);
+			OSL_DELAY(6000);
+
+			dhd_sbreg_op(dhd, ioctrl_addr, &ioctrl_resetseq_val0, FALSE);
+
+			dhd_sbreg_op(dhd, ioctrl_addr, &ioctrl_resetseq_val1, FALSE);
+
+			/* enable core by clearing bit 0 */
+			val = 0;
+			dhd_sbreg_op(dhd, resetctrl_addr, &val, FALSE);
+
+			dhd_sbreg_op(dhd, ioctrl_addr, &ioctrl_resetseq_val2, FALSE);
+
+			dhd_sbreg_op(dhd, ioctrl_addr, &ioctrl_resetseq_val3, FALSE);
+
+			dhd_sbreg_op(dhd, ioctrl_addr, &ioctrl_resetseq_val4, FALSE);
+		}
+	}
+	return BCME_OK;
+}
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+static int
+dhdpcie_sssr_dump_get_before_sr(dhd_pub_t *dhd)
+{
+	int i;
+	uint32 sr_size, xmtaddress, xmtdata, dig_buf_size, dig_buf_addr;
+	uint8 num_d11cores;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	for (i = 0; i < num_d11cores; i++) {
+		if (dhd->sssr_d11_outofreset[i]) {
+			sr_size = dhd_sssr_mac_buf_size(dhd, i);
+			xmtaddress = dhd_sssr_mac_xmtaddress(dhd, i);
+			xmtdata = dhd_sssr_mac_xmtdata(dhd, i);
+			dhdpcie_get_sssr_fifo_dump(dhd, dhd->sssr_d11_before[i],
+				sr_size, xmtaddress, xmtdata);
+		}
+	}
+
+	dig_buf_size = dhd_sssr_dig_buf_size(dhd);
+	dig_buf_addr = dhd_sssr_dig_buf_addr(dhd);
+	if (dig_buf_size) {
+		dhdpcie_get_sssr_dig_dump(dhd, dhd->sssr_dig_buf_before,
+			dig_buf_size, dig_buf_addr);
+	}
+
+	return BCME_OK;
+}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+static int
+dhdpcie_sssr_dump_get_after_sr(dhd_pub_t *dhd)
+{
+	int i;
+	uint32 sr_size, xmtaddress, xmtdata, dig_buf_size, dig_buf_addr;
+	uint8 num_d11cores;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	for (i = 0; i < num_d11cores; i++) {
+		if (dhd->sssr_d11_outofreset[i]) {
+			sr_size = dhd_sssr_mac_buf_size(dhd, i);
+			xmtaddress = dhd_sssr_mac_xmtaddress(dhd, i);
+			xmtdata = dhd_sssr_mac_xmtdata(dhd, i);
+			dhdpcie_get_sssr_fifo_dump(dhd, dhd->sssr_d11_after[i],
+				sr_size, xmtaddress, xmtdata);
+		}
+	}
+
+	dig_buf_size = dhd_sssr_dig_buf_size(dhd);
+	dig_buf_addr = dhd_sssr_dig_buf_addr(dhd);
+
+	if (dig_buf_size) {
+		dhdpcie_get_sssr_dig_dump(dhd, dhd->sssr_dig_buf_after, dig_buf_size, dig_buf_addr);
+	}
+
+	return BCME_OK;
+}
+
+#define GCI_CHIPSTATUS_AUX	GCI_CHIPSTATUS_10
+#define GCI_CHIPSTATUS_MAIN	GCI_CHIPSTATUS_11
+#define GCI_CHIPSTATUS_DIG	GCI_CHIPSTATUS_12
+#define GCI_CHIPSTATUS_SCAN	GCI_CHIPSTATUS_13
+
+#define GCI_CHIPSTATUS_ILLEGAL_INSTR_BITMASK	(1u << 3)
+int
+dhdpcie_validate_gci_chip_intstatus(dhd_pub_t *dhd)
+{
+	int gci_intstatus;
+	si_t *sih = dhd->bus->sih;
+
+	/* For now validate only for 4389 chip */
+	if (si_chipid(sih) != BCM4389_CHIP_ID) {
+		DHD_ERROR(("%s: skipping for chipid:0x%x\n", __FUNCTION__, si_chipid(sih)));
+		return BCME_OK;
+	}
+
+	gci_intstatus = si_gci_chipstatus(sih, GCI_CHIPSTATUS_MAIN);
+	if (gci_intstatus & GCI_CHIPSTATUS_ILLEGAL_INSTR_BITMASK) {
+		DHD_ERROR(("%s: Illegal instruction set for MAIN core 0x%x\n",
+			__FUNCTION__, gci_intstatus));
+		return BCME_ERROR;
+	}
+
+	gci_intstatus = si_gci_chipstatus(sih, GCI_CHIPSTATUS_AUX);
+	if (gci_intstatus & GCI_CHIPSTATUS_ILLEGAL_INSTR_BITMASK) {
+		DHD_ERROR(("%s: Illegal instruction set for AUX core 0x%x\n",
+			__FUNCTION__, gci_intstatus));
+		return BCME_ERROR;
+	}
+
+	gci_intstatus = si_gci_chipstatus(sih, GCI_CHIPSTATUS_SCAN);
+	if (gci_intstatus & GCI_CHIPSTATUS_ILLEGAL_INSTR_BITMASK) {
+		DHD_ERROR(("%s: Illegal instruction set for SCAN core 0x%x\n",
+			__FUNCTION__, gci_intstatus));
+		return BCME_ERROR;
+	}
+
+	gci_intstatus = si_gci_chipstatus(sih, GCI_CHIPSTATUS_DIG);
+	if (gci_intstatus & GCI_CHIPSTATUS_ILLEGAL_INSTR_BITMASK) {
+		DHD_ERROR(("%s: Illegal instruction set for DIG core 0x%x\n",
+			__FUNCTION__, gci_intstatus));
+		return BCME_ERROR;
+	}
+
+	return BCME_OK;
+}
+
+int
+dhdpcie_sssr_dump(dhd_pub_t *dhd)
+{
+	uint32 powerctrl_val;
+
+	if (!dhd->sssr_inited) {
+		DHD_ERROR(("%s: SSSR not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link is down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhd->bus->cto_triggered) {
+		DHD_ERROR(("%s: CTO Triggered\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhdpcie_validate_gci_chip_intstatus(dhd) != BCME_OK) {
+		DHD_ERROR(("%s: ## Invalid GCI Chip intstatus, Abort SSSR ##\n",
+			__FUNCTION__));
+		return BCME_ERROR;
+	}
+	DHD_ERROR(("%s: Before WL down (powerctl: pcie:0x%x chipc:0x%x) "
+		"PMU rctl:0x%x res_state:0x%x\n", __FUNCTION__,
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			OFFSETOF(chipcregs_t, powerctl), 0, 0),
+		si_corereg(dhd->bus->sih, 0, OFFSETOF(chipcregs_t, powerctl), 0, 0),
+		PMU_REG(dhd->bus->sih, retention_ctl, 0, 0),
+		PMU_REG(dhd->bus->sih, res_state, 0, 0)));
+
+	dhdpcie_d11_check_outofreset(dhd);
+
+#ifdef DHD_SSSR_DUMP_BEFORE_SR
+	DHD_ERROR(("%s: Collecting Dump before SR\n", __FUNCTION__));
+	if (dhdpcie_sssr_dump_get_before_sr(dhd) != BCME_OK) {
+		DHD_ERROR(("%s: dhdpcie_sssr_dump_get_before_sr failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+#endif /* DHD_SSSR_DUMP_BEFORE_SR */
+
+	dhdpcie_clear_intmask_and_timer(dhd);
+	dhdpcie_clear_clk_req(dhd);
+	powerctrl_val = dhdpcie_suspend_chipcommon_powerctrl(dhd);
+	dhdpcie_pcie_send_ltrsleep(dhd);
+
+	if (MULTIBP_ENAB(dhd->bus->sih)) {
+		dhd_bus_pcie_pwr_req_wl_domain(dhd->bus, OFFSETOF(chipcregs_t, powerctl), FALSE);
+	}
+
+	/* Wait for some time before Restore */
+	OSL_DELAY(6000);
+
+	DHD_ERROR(("%s: After WL down (powerctl: pcie:0x%x chipc:0x%x) "
+		"PMU rctl:0x%x res_state:0x%x\n", __FUNCTION__,
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			OFFSETOF(chipcregs_t, powerctl), 0, 0),
+		si_corereg(dhd->bus->sih, 0, OFFSETOF(chipcregs_t, powerctl), 0, 0),
+		PMU_REG(dhd->bus->sih, retention_ctl, 0, 0),
+		PMU_REG(dhd->bus->sih, res_state, 0, 0)));
+
+	if (MULTIBP_ENAB(dhd->bus->sih)) {
+		dhd_bus_pcie_pwr_req_wl_domain(dhd->bus, OFFSETOF(chipcregs_t, powerctl), TRUE);
+		/* Add delay for WL domain to power up */
+		OSL_DELAY(15000);
+
+		DHD_ERROR(("%s: After WL up again (powerctl: pcie:0x%x chipc:0x%x) "
+			"PMU rctl:0x%x res_state:0x%x\n", __FUNCTION__,
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				OFFSETOF(chipcregs_t, powerctl), 0, 0),
+			si_corereg(dhd->bus->sih, 0, OFFSETOF(chipcregs_t, powerctl), 0, 0),
+			PMU_REG(dhd->bus->sih, retention_ctl, 0, 0),
+			PMU_REG(dhd->bus->sih, res_state, 0, 0)));
+	}
+
+	dhdpcie_resume_chipcommon_powerctrl(dhd, powerctrl_val);
+	dhdpcie_arm_resume_clk_req(dhd);
+	dhdpcie_bring_d11_outofreset(dhd);
+
+	DHD_ERROR(("%s: Collecting Dump after SR\n", __FUNCTION__));
+	if (dhdpcie_sssr_dump_get_after_sr(dhd) != BCME_OK) {
+		DHD_ERROR(("%s: dhdpcie_sssr_dump_get_after_sr failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	dhd->sssr_dump_collected = TRUE;
+	dhd_write_sssr_dump(dhd, SSSR_DUMP_MODE_SSSR);
+
+	return BCME_OK;
+}
+
+#define PCIE_CFG_DSTATE_MASK	0x11u
+
+static int
+dhdpcie_fis_trigger(dhd_pub_t *dhd)
+{
+	uint32 fis_ctrl_status, fis_min_res_mask;
+	uint32 cfg_status_cmd;
+	uint32 cfg_pmcsr;
+
+	if (!dhd->sssr_inited) {
+		DHD_ERROR(("%s: SSSR not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link is down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* Set fis_triggered flag to ignore link down callback from RC */
+	dhd->fis_triggered = TRUE;
+
+	/* set fis_min_res_mask to 0x1 */
+	PMU_REG(dhd->bus->sih, fis_min_res_mask, ~0, 0x1);
+
+	/* Set FIS PwrswForceOnAll */
+	PMU_REG(dhd->bus->sih, fis_ctrl_status, PMU_FIS_FORCEON_ALL_MASK, PMU_FIS_FORCEON_ALL_MASK);
+
+	fis_ctrl_status = PMU_REG(dhd->bus->sih, fis_ctrl_status, 0, 0);
+	fis_min_res_mask = PMU_REG(dhd->bus->sih, fis_min_res_mask, 0, 0);
+	DHD_ERROR(("%s: fis_ctrl_status=0x%x fis_min_res_mask=0x%x\n",
+		__FUNCTION__, fis_ctrl_status, fis_min_res_mask));
+
+	cfg_status_cmd = dhd_pcie_config_read(dhd->bus, PCIECFGREG_STATUS_CMD, sizeof(uint32));
+	cfg_pmcsr = dhd_pcie_config_read(dhd->bus, PCIE_CFG_PMCSR, sizeof(uint32));
+	DHD_ERROR(("before save: Status Command(0x%x)=0x%x PCIE_CFG_PMCSR(0x%x)=0x%x\n",
+		PCIECFGREG_STATUS_CMD, cfg_status_cmd, PCIE_CFG_PMCSR, cfg_pmcsr));
+
+	DHD_PCIE_CONFIG_SAVE(dhd->bus);
+
+	/* Trigger FIS */
+	si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		DAR_FIS_CTRL(dhd->bus->sih->buscorerev), ~0, DAR_FIS_START_MASK);
+	OSL_DELAY(100 * 1000);
+
+#ifdef OEM_ANDROID
+	/*
+	 * For android built-in platforms need to perform REG ON/OFF
+	 * to restore pcie link.
+	 * dhd_download_fw_on_driverload will be FALSE for built-in.
+	 */
+	if (!dhd_download_fw_on_driverload) {
+		DHD_ERROR(("%s: Toggle REG_ON and restore config space\n", __FUNCTION__));
+		dhdpcie_bus_stop_host_dev(dhd->bus);
+		dhd_wifi_platform_set_power(dhd, FALSE);
+		dhd_wifi_platform_set_power(dhd, TRUE);
+		dhdpcie_bus_start_host_dev(dhd->bus);
+		/* Restore inited pcie cfg from pci_load_saved_state */
+		dhdpcie_bus_enable_device(dhd->bus);
+	}
+#endif /* OEM_ANDROID */
+
+	cfg_status_cmd = dhd_pcie_config_read(dhd->bus, PCIECFGREG_STATUS_CMD, sizeof(uint32));
+	cfg_pmcsr = dhd_pcie_config_read(dhd->bus, PCIE_CFG_PMCSR, sizeof(uint32));
+	DHD_ERROR(("after regon-restore: Status Command(0x%x)=0x%x PCIE_CFG_PMCSR(0x%x)=0x%x\n",
+		PCIECFGREG_STATUS_CMD, cfg_status_cmd, PCIE_CFG_PMCSR, cfg_pmcsr));
+
+	/* To-Do: below is debug code, remove this if EP is in D0 after REG-ON restore */
+	DHD_PCIE_CONFIG_RESTORE(dhd->bus);
+
+	cfg_status_cmd = dhd_pcie_config_read(dhd->bus, PCIECFGREG_STATUS_CMD, sizeof(uint32));
+	cfg_pmcsr = dhd_pcie_config_read(dhd->bus, PCIE_CFG_PMCSR, sizeof(uint32));
+	DHD_ERROR(("after normal-restore: Status Command(0x%x)=0x%x PCIE_CFG_PMCSR(0x%x)=0x%x\n",
+		PCIECFGREG_STATUS_CMD, cfg_status_cmd, PCIE_CFG_PMCSR, cfg_pmcsr));
+
+	/*
+	 * To-Do: below is debug code, remove this if EP is in D0 after REG-ON restore
+	 * in both MSM and LSI RCs
+	 */
+	if ((cfg_pmcsr & PCIE_CFG_DSTATE_MASK) != 0) {
+		int ret = dhdpcie_set_master_and_d0_pwrstate(dhd->bus);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: Setting D0 failed, ABORT FIS collection\n", __FUNCTION__));
+			return ret;
+		}
+		cfg_status_cmd =
+			dhd_pcie_config_read(dhd->bus, PCIECFGREG_STATUS_CMD, sizeof(uint32));
+		cfg_pmcsr = dhd_pcie_config_read(dhd->bus, PCIE_CFG_PMCSR, sizeof(uint32));
+		DHD_ERROR(("after force-d0: Status Command(0x%x)=0x%x PCIE_CFG_PMCSR(0x%x)=0x%x\n",
+			PCIECFGREG_STATUS_CMD, cfg_status_cmd, PCIE_CFG_PMCSR, cfg_pmcsr));
+	}
+
+	/* Clear fis_triggered as REG OFF/ON recovered link */
+	dhd->fis_triggered = FALSE;
+
+	return BCME_OK;
+}
+
+int
+dhd_bus_fis_trigger(dhd_pub_t *dhd)
+{
+	return dhdpcie_fis_trigger(dhd);
+}
+
+static int
+dhdpcie_reset_hwa(dhd_pub_t *dhd)
+{
+	int ret;
+	sssr_reg_info_cmn_t *sssr_reg_info_cmn = dhd->sssr_reg_info;
+	sssr_reg_info_v3_t *sssr_reg_info = (sssr_reg_info_v3_t *)&sssr_reg_info_cmn->rev3;
+
+	/* HWA wrapper registers */
+	uint32 ioctrl, resetctrl;
+	/* HWA base registers */
+	uint32 clkenable, clkgatingenable, clkext, clkctlstatus;
+	uint32 hwa_resetseq_val[SSSR_HWA_RESET_SEQ_STEPS];
+	int i = 0;
+
+	if (sssr_reg_info->version < SSSR_REG_INFO_VER_3) {
+		DHD_ERROR(("%s: not supported for version:%d\n",
+			__FUNCTION__, sssr_reg_info->version));
+		return BCME_UNSUPPORTED;
+	}
+
+	if (sssr_reg_info->hwa_regs.base_regs.clkenable == 0) {
+		DHD_ERROR(("%s: hwa regs are not set\n", __FUNCTION__));
+		return BCME_UNSUPPORTED;
+	}
+
+	DHD_ERROR(("%s: version:%d\n", __FUNCTION__, sssr_reg_info->version));
+
+	ioctrl = sssr_reg_info->hwa_regs.wrapper_regs.ioctrl;
+	resetctrl = sssr_reg_info->hwa_regs.wrapper_regs.resetctrl;
+
+	clkenable = sssr_reg_info->hwa_regs.base_regs.clkenable;
+	clkgatingenable = sssr_reg_info->hwa_regs.base_regs.clkgatingenable;
+	clkext = sssr_reg_info->hwa_regs.base_regs.clkext;
+	clkctlstatus = sssr_reg_info->hwa_regs.base_regs.clkctlstatus;
+
+	ret = memcpy_s(hwa_resetseq_val, sizeof(hwa_resetseq_val),
+		sssr_reg_info->hwa_regs.hwa_resetseq_val,
+		sizeof(sssr_reg_info->hwa_regs.hwa_resetseq_val));
+	if (ret) {
+		DHD_ERROR(("%s: hwa_resetseq_val memcpy_s failed: %d\n",
+			__FUNCTION__, ret));
+		return ret;
+	}
+
+	dhd_sbreg_op(dhd, ioctrl, &hwa_resetseq_val[i++], FALSE);
+	dhd_sbreg_op(dhd, resetctrl, &hwa_resetseq_val[i++], FALSE);
+	dhd_sbreg_op(dhd, resetctrl, &hwa_resetseq_val[i++], FALSE);
+	dhd_sbreg_op(dhd, ioctrl, &hwa_resetseq_val[i++], FALSE);
+
+	dhd_sbreg_op(dhd, clkenable, &hwa_resetseq_val[i++], FALSE);
+	dhd_sbreg_op(dhd, clkgatingenable, &hwa_resetseq_val[i++], FALSE);
+	dhd_sbreg_op(dhd, clkext, &hwa_resetseq_val[i++], FALSE);
+	dhd_sbreg_op(dhd, clkctlstatus, &hwa_resetseq_val[i++], FALSE);
+
+	return BCME_OK;
+}
+
+static int
+dhdpcie_fis_dump(dhd_pub_t *dhd)
+{
+	int i;
+	uint8 num_d11cores;
+	struct dhd_bus *bus = dhd->bus;
+	uint32 save_idx = 0;
+
+	DHD_ERROR(("%s\n", __FUNCTION__));
+
+	if (!dhd->sssr_inited) {
+		DHD_ERROR(("%s: SSSR not inited\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link is down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* bring up all pmu resources */
+	PMU_REG(dhd->bus->sih, min_res_mask, ~0,
+		PMU_REG(dhd->bus->sih, max_res_mask, 0, 0));
+	OSL_DELAY(10 * 1000);
+
+	num_d11cores = dhd_d11_slices_num_get(dhd);
+
+	for (i = 0; i < num_d11cores; i++) {
+		dhd->sssr_d11_outofreset[i] = TRUE;
+	}
+
+	dhdpcie_bring_d11_outofreset(dhd);
+	OSL_DELAY(6000);
+
+	/* clear FIS Done */
+	PMU_REG(dhd->bus->sih, fis_ctrl_status, PMU_CLEAR_FIS_DONE_MASK, PMU_CLEAR_FIS_DONE_MASK);
+
+	if (dhdpcie_reset_hwa(dhd) != BCME_OK) {
+		DHD_ERROR(("%s: dhdpcie_reset_hwa failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dhdpcie_d11_check_outofreset(dhd);
+
+	/* take sysmem out of reset - otherwise
+	 * socram collected again will read only
+	 * 0xffff
+	 */
+	save_idx = si_coreidx(bus->sih);
+	if (si_setcore(bus->sih, SYSMEM_CORE_ID, 0)) {
+		si_core_reset(bus->sih, 0, 0);
+		si_setcoreidx(bus->sih, save_idx);
+	}
+
+	DHD_ERROR(("%s: Collecting Dump after SR\n", __FUNCTION__));
+	if (dhdpcie_sssr_dump_get_after_sr(dhd) != BCME_OK) {
+		DHD_ERROR(("%s: dhdpcie_sssr_dump_get_after_sr failed\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+	dhd->sssr_dump_collected = TRUE;
+	dhd_write_sssr_dump(dhd, SSSR_DUMP_MODE_FIS);
+
+	if (dhd->bus->link_state != DHD_PCIE_ALL_GOOD) {
+		/* reset link state and collect socram */
+		dhd->bus->link_state = DHD_PCIE_ALL_GOOD;
+		DHD_ERROR(("%s: recollect socram\n", __FUNCTION__));
+		/* re-read socram into buffer */
+		dhdpcie_get_mem_dump(bus);
+	}
+
+	return BCME_OK;
+}
+
+int
+dhd_bus_fis_dump(dhd_pub_t *dhd)
+{
+	return dhdpcie_fis_dump(dhd);
+}
+#endif /* DHD_SSSR_DUMP */
+
+#ifdef DHD_SDTC_ETB_DUMP
+int
+dhd_bus_get_etb_info(dhd_pub_t *dhd, uint32 etbinfo_addr, etb_info_t *etb_info)
+{
+
+	int ret = 0;
+
+	if ((ret = dhdpcie_bus_membytes(dhd->bus, FALSE, etbinfo_addr,
+		(unsigned char *)etb_info, sizeof(*etb_info)))) {
+		DHD_ERROR(("%s: Read Error membytes %d\n", __FUNCTION__, ret));
+		return BCME_ERROR;
+	}
+
+	return BCME_OK;
+}
+
+int
+dhd_bus_get_sdtc_etb(dhd_pub_t *dhd, uint8 *sdtc_etb_mempool, uint addr, uint read_bytes)
+{
+	int ret = 0;
+
+	if ((ret = dhdpcie_bus_membytes(dhd->bus, FALSE, addr,
+		(unsigned char *)sdtc_etb_mempool, read_bytes))) {
+		DHD_ERROR(("%s: Read Error membytes %d\n", __FUNCTION__, ret));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+#endif /* DHD_SDTC_ETB_DUMP */
+
+#ifdef DHD_WAKE_STATUS
+wake_counts_t*
+dhd_bus_get_wakecount(dhd_pub_t *dhd)
+{
+	return &dhd->bus->wake_counts;
+}
+int
+dhd_bus_get_bus_wake(dhd_pub_t *dhd)
+{
+	return bcmpcie_get_wake(dhd->bus);
+}
+int
+dhd_bus_set_get_bus_wake(dhd_pub_t *dhd, int set)
+{
+	return bcmpcie_set_get_wake(dhd->bus, set);
+}
+
+int
+dhd_bus_set_get_bus_wake_pkt_dump(dhd_pub_t *dhd, int wake_pkt_dump)
+{
+	return bcmpcie_set_get_wake_pkt_dump(dhd->bus, wake_pkt_dump);
+}
+#endif /* DHD_WAKE_STATUS */
+
+/* Writes random number(s) to the TCM. FW upon initialization reads this register
+ * to fetch the random number, and uses it to randomize heap address space layout.
+ */
+static int
+dhdpcie_wrt_rnd(struct dhd_bus *bus)
+{
+	uint8 rand_buf[BCM_ENTROPY_HOST_NBYTES];
+	int ret = BCME_OK;
+	memset(rand_buf, 0, sizeof(rand_buf));
+
+	/* Now write the random number(s) */
+	ret = dhd_get_random_bytes(rand_buf, sizeof(rand_buf));
+	if (ret != BCME_OK) {
+		return ret;
+	}
+	dhdpcie_download_rtlv(bus, DNGL_RTLV_TYPE_RNG_SIGNATURE, sizeof(rand_buf), rand_buf);
+
+	return BCME_OK;
+}
+
+#ifdef D2H_MINIDUMP
+bool
+dhd_bus_is_minidump_enabled(dhd_pub_t *dhdp)
+{
+	return dhdp->bus->d2h_minidump;
+}
+#endif /* D2H_MINIDUMP */
+
+void
+dhd_pcie_intr_count_dump(dhd_pub_t *dhd)
+{
+	struct dhd_bus *bus = dhd->bus;
+	uint64 current_time;
+
+	DHD_ERROR(("\n ------- DUMPING INTR enable/disable counters  ------- \r\n"));
+	DHD_ERROR(("dngl_intmask_enable_count=%lu host_irq_enable_count=%lu\n",
+		bus->dngl_intmask_enable_count, bus->host_irq_enable_count));
+	DHD_ERROR(("host_irq_disable_count=%lu dngl_intmask_disable_count=%lu\n",
+		bus->host_irq_disable_count, bus->dngl_intmask_disable_count));
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	DHD_ERROR(("oob_intr_count=%lu oob_intr_enable_count=%lu oob_intr_disable_count=%lu\n",
+		bus->oob_intr_count, bus->oob_intr_enable_count,
+		bus->oob_intr_disable_count));
+	DHD_ERROR(("oob_irq_num=%d last_oob_irq_times="SEC_USEC_FMT":"SEC_USEC_FMT"\n",
+		dhdpcie_get_oob_irq_num(bus),
+		GET_SEC_USEC(bus->last_oob_irq_isr_time),
+		GET_SEC_USEC(bus->last_oob_irq_thr_time)));
+	DHD_ERROR(("last_oob_irq_enable_time="SEC_USEC_FMT
+		" last_oob_irq_disable_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->last_oob_irq_enable_time),
+		GET_SEC_USEC(bus->last_oob_irq_disable_time)));
+	DHD_ERROR(("oob_irq_enabled=%d oob_gpio_level=%d\n",
+		dhdpcie_get_oob_irq_status(bus),
+		dhdpcie_get_oob_irq_level()));
+
+#if defined(__linux__)
+	dhd_plat_pin_dbg_show(bus->dhd->plat_info);
+#endif /* __linux__ */
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+	DHD_ERROR(("dpc_return_busdown_count=%lu non_ours_irq_count=%lu\n",
+		bus->dpc_return_busdown_count, bus->non_ours_irq_count));
+
+	current_time = OSL_LOCALTIME_NS();
+	DHD_ERROR(("\ncurrent_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(current_time)));
+	DHD_ERROR(("isr_entry_time="SEC_USEC_FMT
+		" prev_isr_entry_time="SEC_USEC_FMT
+		" isr_exit_time="SEC_USEC_FMT" dpc_sched=%u\n",
+		GET_SEC_USEC(bus->isr_entry_time),
+		GET_SEC_USEC(bus->prev_isr_entry_time),
+		GET_SEC_USEC(bus->isr_exit_time), bus->dpc_sched));
+	DHD_ERROR(("isr_sched_dpc_time="SEC_USEC_FMT
+		" rpm_sched_dpc_time="SEC_USEC_FMT
+		" last_non_ours_irq_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->isr_sched_dpc_time),
+		GET_SEC_USEC(bus->rpm_sched_dpc_time),
+		GET_SEC_USEC(bus->last_non_ours_irq_time)));
+	DHD_ERROR(("dpc_entry_time="SEC_USEC_FMT
+		" last_process_ctrlbuf_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->dpc_entry_time),
+		GET_SEC_USEC(bus->last_process_ctrlbuf_time)));
+	DHD_ERROR(("last_process_flowring_time="SEC_USEC_FMT
+		" last_process_txcpl_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->last_process_flowring_time),
+		GET_SEC_USEC(bus->last_process_txcpl_time)));
+	DHD_ERROR(("last_process_rxcpl_time="SEC_USEC_FMT
+		" last_process_infocpl_time="SEC_USEC_FMT
+		" last_process_edl_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->last_process_rxcpl_time),
+		GET_SEC_USEC(bus->last_process_infocpl_time),
+		GET_SEC_USEC(bus->last_process_edl_time)));
+	DHD_ERROR(("dpc_exit_time="SEC_USEC_FMT
+		" resched_dpc_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->dpc_exit_time),
+		GET_SEC_USEC(bus->resched_dpc_time)));
+	DHD_ERROR(("last_d3_inform_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->last_d3_inform_time)));
+
+	DHD_ERROR(("\nlast_suspend_start_time="SEC_USEC_FMT
+		" last_suspend_end_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->last_suspend_start_time),
+		GET_SEC_USEC(bus->last_suspend_end_time)));
+	DHD_ERROR(("last_resume_start_time="SEC_USEC_FMT
+		" last_resume_end_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(bus->last_resume_start_time),
+		GET_SEC_USEC(bus->last_resume_end_time)));
+
+#if defined(SHOW_LOGTRACE) && defined(DHD_USE_KTHREAD_FOR_LOGTRACE)
+	DHD_ERROR(("logtrace_thread_entry_time="SEC_USEC_FMT
+		" logtrace_thread_sem_down_time="SEC_USEC_FMT
+		"\nlogtrace_thread_flush_time="SEC_USEC_FMT
+		" logtrace_thread_unexpected_break_time="SEC_USEC_FMT
+		"\nlogtrace_thread_complete_time="SEC_USEC_FMT"\n",
+		GET_SEC_USEC(dhd->logtrace_thr_ts.entry_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.sem_down_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.flush_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.unexpected_break_time),
+		GET_SEC_USEC(dhd->logtrace_thr_ts.complete_time)));
+#endif /* SHOW_LOGTRACE && DHD_USE_KTHREAD_FOR_LOGTRACE */
+}
+
+void
+dhd_bus_intr_count_dump(dhd_pub_t *dhd)
+{
+	dhd_pcie_intr_count_dump(dhd);
+}
+
+int
+dhd_pcie_dump_wrapper_regs(dhd_pub_t *dhd)
+{
+	uint32 save_idx, val;
+	si_t *sih = dhd->bus->sih;
+	uint32 oob_base, oob_base1;
+	uint32 wrapper_dump_list[] = {
+		AI_OOBSELOUTA30, AI_OOBSELOUTA74, AI_OOBSELOUTB30, AI_OOBSELOUTB74,
+		AI_OOBSELOUTC30, AI_OOBSELOUTC74, AI_OOBSELOUTD30, AI_OOBSELOUTD74,
+		AI_RESETSTATUS, AI_RESETCTRL,
+		AI_ITIPOOBA, AI_ITIPOOBB, AI_ITIPOOBC, AI_ITIPOOBD,
+		AI_ITIPOOBAOUT, AI_ITIPOOBBOUT, AI_ITIPOOBCOUT, AI_ITIPOOBDOUT
+	};
+	uint32 i;
+	hndoobr_reg_t *reg;
+	cr4regs_t *cr4regs;
+	ca7regs_t *ca7regs;
+
+	save_idx = si_coreidx(sih);
+
+	DHD_ERROR(("%s: Master wrapper Reg\n", __FUNCTION__));
+
+	if (si_setcore(sih, PCIE2_CORE_ID, 0) != NULL) {
+		for (i = 0; i < (uint32)sizeof(wrapper_dump_list) / 4; i++) {
+			val = si_wrapperreg(sih, wrapper_dump_list[i], 0, 0);
+			DHD_ERROR(("sbreg: addr:0x%x val:0x%x\n", wrapper_dump_list[i], val));
+		}
+	}
+
+	dhd_dump_pcie_slave_wrapper_regs(dhd->bus);
+
+	if ((cr4regs = si_setcore(sih, ARMCR4_CORE_ID, 0)) != NULL) {
+		DHD_ERROR(("%s: ARM CR4 wrapper Reg\n", __FUNCTION__));
+		for (i = 0; i < (uint32)sizeof(wrapper_dump_list) / 4; i++) {
+			val = si_wrapperreg(sih, wrapper_dump_list[i], 0, 0);
+			DHD_ERROR(("sbreg: addr:0x%x val:0x%x\n", wrapper_dump_list[i], val));
+		}
+		DHD_ERROR(("%s: ARM CR4 core Reg\n", __FUNCTION__));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, corecontrol));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, corecontrol), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, corecapabilities));
+		DHD_ERROR(("reg:0x%x val:0x%x\n",
+			(uint)OFFSETOF(cr4regs_t, corecapabilities), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, corestatus));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, corestatus), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, nmiisrst));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, nmiisrst), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, nmimask));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, nmimask), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, isrmask));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, isrmask), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, swintreg));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, swintreg), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, intstatus));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, intstatus), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, cyclecnt));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, cyclecnt), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, inttimer));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, inttimer), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, clk_ctl_st));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, clk_ctl_st), val));
+		val = R_REG(dhd->osh, ARM_CR4_REG(cr4regs, powerctl));
+		DHD_ERROR(("reg:0x%x val:0x%x\n", (uint)OFFSETOF(cr4regs_t, powerctl), val));
+	}
+	/* XXX: Currently dumping CA7 registers causing CTO, temporarily disabling it */
+	BCM_REFERENCE(ca7regs);
+
+	DHD_ERROR(("%s: OOBR Reg\n", __FUNCTION__));
+
+	oob_base = si_oobr_baseaddr(sih, FALSE);
+	oob_base1 = si_oobr_baseaddr(sih, TRUE);
+	if (oob_base) {
+		dhd_sbreg_op(dhd, oob_base + OOB_STATUSA, &val, TRUE);
+		dhd_sbreg_op(dhd, oob_base + OOB_STATUSB, &val, TRUE);
+		dhd_sbreg_op(dhd, oob_base + OOB_STATUSC, &val, TRUE);
+		dhd_sbreg_op(dhd, oob_base + OOB_STATUSD, &val, TRUE);
+	} else if ((reg = si_setcore(sih, HND_OOBR_CORE_ID, 0)) != NULL) {
+		val = R_REG(dhd->osh, &reg->intstatus[0]);
+		DHD_ERROR(("reg: addr:%p val:0x%x\n", reg, val));
+		val = R_REG(dhd->osh, &reg->intstatus[1]);
+		DHD_ERROR(("reg: addr:%p val:0x%x\n", reg, val));
+		val = R_REG(dhd->osh, &reg->intstatus[2]);
+		DHD_ERROR(("reg: addr:%p val:0x%x\n", reg, val));
+		val = R_REG(dhd->osh, &reg->intstatus[3]);
+		DHD_ERROR(("reg: addr:%p val:0x%x\n", reg, val));
+	}
+
+	if (oob_base1) {
+		DHD_ERROR(("%s: Second OOBR Reg\n", __FUNCTION__));
+
+		dhd_sbreg_op(dhd, oob_base1 + OOB_STATUSA, &val, TRUE);
+		dhd_sbreg_op(dhd, oob_base1 + OOB_STATUSB, &val, TRUE);
+		dhd_sbreg_op(dhd, oob_base1 + OOB_STATUSC, &val, TRUE);
+		dhd_sbreg_op(dhd, oob_base1 + OOB_STATUSD, &val, TRUE);
+	}
+
+	si_setcoreidx(dhd->bus->sih, save_idx);
+
+	return 0;
+}
+
+static void
+dhdpcie_hw_war_regdump(dhd_bus_t *bus)
+{
+	uint32 save_idx, val;
+	volatile uint32 *reg;
+
+	save_idx = si_coreidx(bus->sih);
+	if ((reg = si_setcore(bus->sih, CC_CORE_ID, 0)) != NULL) {
+		val = R_REG(bus->osh, reg + REG_WORK_AROUND);
+		DHD_ERROR(("CC HW_WAR :0x%x\n", val));
+	}
+
+	if ((reg = si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) != NULL) {
+		val = R_REG(bus->osh, reg + REG_WORK_AROUND);
+		DHD_ERROR(("ARM HW_WAR:0x%x\n", val));
+	}
+
+	if ((reg = si_setcore(bus->sih, PCIE2_CORE_ID, 0)) != NULL) {
+		val = R_REG(bus->osh, reg + REG_WORK_AROUND);
+		DHD_ERROR(("PCIE HW_WAR :0x%x\n", val));
+	}
+	si_setcoreidx(bus->sih, save_idx);
+
+	val = PMU_REG_NEW(bus->sih, min_res_mask, 0, 0);
+	DHD_ERROR(("MINRESMASK :0x%x\n", val));
+}
+
+int
+dhd_pcie_dma_info_dump(dhd_pub_t *dhd)
+{
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("\n ------- SKIP DUMPING DMA Registers "
+			"due to PCIe link down ------- \r\n"));
+		return 0;
+	}
+
+	DHD_ERROR(("\n ------- DUMPING DMA Registers ------- \r\n"));
+
+	//HostToDev
+	DHD_ERROR(("HostToDev TX: XmtCtrl=0x%08x XmtPtr=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x200, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x204, 0, 0)));
+	DHD_ERROR(("            : XmtAddrLow=0x%08x XmtAddrHigh=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x208, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x20C, 0, 0)));
+	DHD_ERROR(("            : XmtStatus0=0x%08x XmtStatus1=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x210, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x214, 0, 0)));
+
+	DHD_ERROR(("HostToDev RX: RcvCtrl=0x%08x RcvPtr=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x220, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x224, 0, 0)));
+	DHD_ERROR(("            : RcvAddrLow=0x%08x RcvAddrHigh=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x228, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x22C, 0, 0)));
+	DHD_ERROR(("            : RcvStatus0=0x%08x RcvStatus1=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x230, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x234, 0, 0)));
+
+	//DevToHost
+	DHD_ERROR(("DevToHost TX: XmtCtrl=0x%08x XmtPtr=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x240, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x244, 0, 0)));
+	DHD_ERROR(("            : XmtAddrLow=0x%08x XmtAddrHigh=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x248, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x24C, 0, 0)));
+	DHD_ERROR(("            : XmtStatus0=0x%08x XmtStatus1=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x250, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x254, 0, 0)));
+
+	DHD_ERROR(("DevToHost RX: RcvCtrl=0x%08x RcvPtr=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x260, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x264, 0, 0)));
+	DHD_ERROR(("            : RcvAddrLow=0x%08x RcvAddrHigh=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x268, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x26C, 0, 0)));
+	DHD_ERROR(("            : RcvStatus0=0x%08x RcvStatus1=0x%08x\n",
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x270, 0, 0),
+		si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, 0x274, 0, 0)));
+
+	return 0;
+}
+
+bool
+dhd_pcie_check_lps_d3_acked(dhd_pub_t *dhd)
+{
+	return DHD_CHK_BUS_LPS_D3_ACKED(dhd->bus);
+}
+
+bool
+dhd_pcie_dump_int_regs(dhd_pub_t *dhd)
+{
+	uint32 intstatus = 0;
+	uint32 intmask = 0;
+	uint32 d2h_db0 = 0;
+	uint32 d2h_mb_data = 0;
+
+	DHD_ERROR(("\n ------- DUMPING INTR Status and Masks ------- \r\n"));
+	intstatus = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		dhd->bus->pcie_mailbox_int, 0, 0);
+	if (intstatus == (uint32)-1) {
+		DHD_ERROR(("intstatus=0x%x \n", intstatus));
+		return FALSE;
+	}
+
+	intmask = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		dhd->bus->pcie_mailbox_mask, 0, 0);
+	if (intmask == (uint32) -1) {
+		DHD_ERROR(("intstatus=0x%x intmask=0x%x \n", intstatus, intmask));
+		return FALSE;
+	}
+
+	d2h_db0 = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+		PCID2H_MailBox, 0, 0);
+	if (d2h_db0 == (uint32)-1) {
+		DHD_ERROR(("intstatus=0x%x intmask=0x%x d2h_db0=0x%x\n",
+		intstatus, intmask, d2h_db0));
+		return FALSE;
+	}
+
+	DHD_ERROR(("intstatus=0x%x intmask=0x%x d2h_db0=0x%x\n",
+		intstatus, intmask, d2h_db0));
+	dhd_bus_cmn_readshared(dhd->bus, &d2h_mb_data, D2H_MB_DATA, 0);
+	DHD_ERROR(("d2h_mb_data=0x%x def_intmask=0x%x \r\n", d2h_mb_data,
+		dhd->bus->def_intmask));
+
+	return TRUE;
+}
+
+void
+dhd_pcie_dump_rc_conf_space_cap(dhd_pub_t *dhd)
+{
+	DHD_ERROR(("\n ------- DUMPING PCIE RC config space Registers ------- \r\n"));
+	DHD_ERROR(("Pcie RC Uncorrectable Error Status Val=0x%x\n",
+		dhdpcie_rc_access_cap(dhd->bus, PCIE_EXTCAP_ID_ERR,
+		PCIE_EXTCAP_AER_UCERR_OFFSET, TRUE, FALSE, 0)));
+#ifdef EXTENDED_PCIE_DEBUG_DUMP
+	DHD_ERROR(("hdrlog0 =0x%08x hdrlog1 =0x%08x hdrlog2 =0x%08x hdrlog3 =0x%08x\n",
+		dhdpcie_rc_access_cap(dhd->bus, PCIE_EXTCAP_ID_ERR,
+		PCIE_EXTCAP_ERR_HEADER_LOG_0, TRUE, FALSE, 0),
+		dhdpcie_rc_access_cap(dhd->bus, PCIE_EXTCAP_ID_ERR,
+		PCIE_EXTCAP_ERR_HEADER_LOG_1, TRUE, FALSE, 0),
+		dhdpcie_rc_access_cap(dhd->bus, PCIE_EXTCAP_ID_ERR,
+		PCIE_EXTCAP_ERR_HEADER_LOG_2, TRUE, FALSE, 0),
+		dhdpcie_rc_access_cap(dhd->bus, PCIE_EXTCAP_ID_ERR,
+		PCIE_EXTCAP_ERR_HEADER_LOG_3, TRUE, FALSE, 0)));
+#endif /* EXTENDED_PCIE_DEBUG_DUMP */
+}
+
+int
+dhd_pcie_debug_info_dump(dhd_pub_t *dhd)
+{
+	int host_irq_disabled;
+
+	DHD_ERROR(("bus->bus_low_power_state = %d\n", dhd->bus->bus_low_power_state));
+	host_irq_disabled = dhdpcie_irq_disabled(dhd->bus);
+	DHD_ERROR(("host pcie_irq disabled = %d\n", host_irq_disabled));
+	dhd_print_tasklet_status(dhd);
+	dhd_pcie_intr_count_dump(dhd);
+
+#if defined(LINUX) || defined(linux)
+	DHD_ERROR(("\n ------- DUMPING PCIE EP Resouce Info ------- \r\n"));
+	dhdpcie_dump_resource(dhd->bus);
+#endif /* LINUX || linux */
+
+	dhd_pcie_dump_rc_conf_space_cap(dhd);
+
+	DHD_ERROR(("RootPort PCIe linkcap=0x%08x\n",
+		dhd_debug_get_rc_linkcap(dhd->bus)));
+	DHD_ERROR(("\n ------- DUMPING PCIE EP config space Registers ------- \r\n"));
+	/* XXX: hwnbu-twiki.sj.broadcom.com/bin/view/Mwgroup/CurrentPcieGen2ProgramGuide */
+	dhd_bus_dump_imp_cfg_registers(dhd->bus);
+#ifdef EXTENDED_PCIE_DEBUG_DUMP
+	DHD_ERROR(("Pcie EP Uncorrectable Error Status Val=0x%x\n",
+		dhdpcie_ep_access_cap(dhd->bus, PCIE_EXTCAP_ID_ERR,
+		PCIE_EXTCAP_AER_UCERR_OFFSET, TRUE, FALSE, 0)));
+	DHD_ERROR(("hdrlog0(0x%x)=0x%08x hdrlog1(0x%x)=0x%08x hdrlog2(0x%x)=0x%08x "
+		"hdrlog3(0x%x)=0x%08x\n", PCI_TLP_HDR_LOG1,
+		dhd_pcie_config_read(dhd->bus, PCI_TLP_HDR_LOG1, sizeof(uint32)),
+		PCI_TLP_HDR_LOG2,
+		dhd_pcie_config_read(dhd->bus, PCI_TLP_HDR_LOG2, sizeof(uint32)),
+		PCI_TLP_HDR_LOG3,
+		dhd_pcie_config_read(dhd->bus, PCI_TLP_HDR_LOG3, sizeof(uint32)),
+		PCI_TLP_HDR_LOG4,
+		dhd_pcie_config_read(dhd->bus, PCI_TLP_HDR_LOG4, sizeof(uint32))));
+	if (dhd->bus->sih->buscorerev >= 24) {
+		DHD_ERROR(("DeviceStatusControl(0x%x)=0x%x SubsystemControl(0x%x)=0x%x "
+			"L1SSControl2(0x%x)=0x%x\n", PCIECFGREG_DEV_STATUS_CTRL,
+			dhd_pcie_config_read(dhd->bus, PCIECFGREG_DEV_STATUS_CTRL,
+			sizeof(uint32)), PCIE_CFG_SUBSYSTEM_CONTROL,
+			dhd_pcie_config_read(dhd->bus, PCIE_CFG_SUBSYSTEM_CONTROL,
+			sizeof(uint32)), PCIECFGREG_PML1_SUB_CTRL2,
+			dhd_pcie_config_read(dhd->bus, PCIECFGREG_PML1_SUB_CTRL2,
+			sizeof(uint32))));
+		dhd_bus_dump_dar_registers(dhd->bus);
+	}
+#endif /* EXTENDED_PCIE_DEBUG_DUMP */
+
+	if (dhd->bus->is_linkdown) {
+		DHD_ERROR(("Skip dumping the PCIe Core registers. link may be DOWN\n"));
+		return 0;
+	}
+
+	if (MULTIBP_ENAB(dhd->bus->sih)) {
+		dhd_bus_pcie_pwr_req(dhd->bus);
+	}
+
+	dhd_pcie_dump_wrapper_regs(dhd);
+
+	if (dhd->bus->dar_err_set) {
+		DHD_ERROR(("Skip dumping the PCIe Core registers. DAR error log set\n"));
+		goto exit;
+	}
+
+	DHD_ERROR(("\n ------- DUMPING PCIE core Registers ------- \r\n"));
+	/* XXX: hwnbu-twiki.sj.broadcom.com/twiki/pub/Mwgroup/
+	 * CurrentPcieGen2ProgramGuide/pcie_ep.htm
+	 */
+
+	DHD_ERROR(("ClkReq0(0x%x)=0x%x ClkReq1(0x%x)=0x%x ClkReq2(0x%x)=0x%x "
+		"ClkReq3(0x%x)=0x%x\n", PCIECFGREG_PHY_DBG_CLKREQ0,
+		dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_DBG_CLKREQ0),
+		PCIECFGREG_PHY_DBG_CLKREQ1,
+		dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_DBG_CLKREQ1),
+		PCIECFGREG_PHY_DBG_CLKREQ2,
+		dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_DBG_CLKREQ2),
+		PCIECFGREG_PHY_DBG_CLKREQ3,
+		dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_DBG_CLKREQ3)));
+
+#ifdef EXTENDED_PCIE_DEBUG_DUMP
+	if (dhd->bus->sih->buscorerev >= 24) {
+
+		DHD_ERROR(("ltssm_hist_0(0x%x)=0x%x ltssm_hist_1(0x%x)=0x%x "
+			"ltssm_hist_2(0x%x)=0x%x "
+			"ltssm_hist_3(0x%x)=0x%x\n", PCIECFGREG_PHY_LTSSM_HIST_0,
+			dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_LTSSM_HIST_0),
+			PCIECFGREG_PHY_LTSSM_HIST_1,
+			dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_LTSSM_HIST_1),
+			PCIECFGREG_PHY_LTSSM_HIST_2,
+			dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_LTSSM_HIST_2),
+			PCIECFGREG_PHY_LTSSM_HIST_3,
+			dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_PHY_LTSSM_HIST_3)));
+
+		DHD_ERROR(("trefup(0x%x)=0x%x trefup_ext(0x%x)=0x%x\n",
+			PCIECFGREG_TREFUP,
+			dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_TREFUP),
+			PCIECFGREG_TREFUP_EXT,
+			dhd_pcie_corereg_read(dhd->bus->sih, PCIECFGREG_TREFUP_EXT)));
+		DHD_ERROR(("errlog(0x%x)=0x%x errlog_addr(0x%x)=0x%x "
+			"Function_Intstatus(0x%x)=0x%x "
+			"Function_Intmask(0x%x)=0x%x Power_Intstatus(0x%x)=0x%x "
+			"Power_Intmask(0x%x)=0x%x\n",
+			PCIE_CORE_REG_ERRLOG,
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+			PCIE_CORE_REG_ERRLOG, 0, 0),
+			PCIE_CORE_REG_ERR_ADDR,
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				PCIE_CORE_REG_ERR_ADDR, 0, 0),
+			PCIFunctionIntstatus(dhd->bus->sih->buscorerev),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				PCIFunctionIntstatus(dhd->bus->sih->buscorerev), 0, 0),
+			PCIFunctionIntmask(dhd->bus->sih->buscorerev),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				PCIFunctionIntmask(dhd->bus->sih->buscorerev), 0, 0),
+			PCIPowerIntstatus(dhd->bus->sih->buscorerev),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				PCIPowerIntstatus(dhd->bus->sih->buscorerev), 0, 0),
+			PCIPowerIntmask(dhd->bus->sih->buscorerev),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				PCIPowerIntmask(dhd->bus->sih->buscorerev), 0, 0)));
+		DHD_ERROR(("err_hdrlog1(0x%x)=0x%x err_hdrlog2(0x%x)=0x%x "
+			"err_hdrlog3(0x%x)=0x%x err_hdrlog4(0x%x)=0x%x\n",
+			(uint)OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg1),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg1), 0, 0),
+			(uint)OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg2),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg2), 0, 0),
+			(uint)OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg3),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg3), 0, 0),
+			(uint)OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg4),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				OFFSETOF(sbpcieregs_t, u.pcie2.err_hdr_logreg4), 0, 0)));
+		DHD_ERROR(("err_code(0x%x)=0x%x PCIH2D_MailBox(%08x)=%08x\n",
+			(uint)OFFSETOF(sbpcieregs_t, u.pcie2.err_code_logreg),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				OFFSETOF(sbpcieregs_t, u.pcie2.err_code_logreg), 0, 0),
+			dhd_bus_db0_addr_get(dhd->bus),
+			si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
+				dhd_bus_db0_addr_get(dhd->bus), 0, 0)));
+
+		dhdpcie_hw_war_regdump(dhd->bus);
+	}
+#endif /* EXTENDED_PCIE_DEBUG_DUMP */
+
+	dhd_pcie_dma_info_dump(dhd);
+exit:
+	if (MULTIBP_ENAB(dhd->bus->sih)) {
+		dhd_bus_pcie_pwr_req_clear(dhd->bus);
+	}
+
+	return 0;
+}
+
+bool
+dhd_bus_force_bt_quiesce_enabled(struct dhd_bus *bus)
+{
+	return bus->force_bt_quiesce;
+}
+
+uint8
+dhd_d11_slices_num_get(dhd_pub_t *dhdp)
+{
+	return (dhdp->bus->sih && si_scan_core_present(dhdp->bus->sih)) ?
+		MAX_NUM_D11_CORES_WITH_SCAN : MAX_NUM_D11CORES;
+}
+
+#if defined(linux) || defined(LINUX)
+static bool
+dhd_bus_tcm_test(struct dhd_bus *bus)
+{
+	int ret = 0;
+	int size; /* Full mem size */
+	int start; /* Start address */
+	int read_size = 0; /* Read size of each iteration */
+	int num = 0;
+	uint8 *read_buf, *write_buf;
+	uint8 init_val[NUM_PATTERNS] = {
+		0xFFu, /* 11111111 */
+		0x00u, /* 00000000 */
+#if !defined(DHD_FW_MEM_CORRUPTION) && defined(BCM4375_CHIP)
+		0x77u, /* 01110111 */
+		0x22u, /* 00100010 */
+		0x27u, /* 00100111 */
+		0x72u, /* 01110010 */
+#endif /* !DHD_FW_MEM_CORRUPTION */
+	};
+
+	if (!bus) {
+		DHD_ERROR(("%s: bus is NULL !\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	read_buf = MALLOCZ(bus->dhd->osh, MEMBLOCK);
+
+	if (!read_buf) {
+		DHD_ERROR(("%s: MALLOC of read_buf failed\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	write_buf = MALLOCZ(bus->dhd->osh, MEMBLOCK);
+
+	if (!write_buf) {
+		MFREE(bus->dhd->osh, read_buf, MEMBLOCK);
+		DHD_ERROR(("%s: MALLOC of write_buf failed\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	DHD_ERROR(("%s: start %x,  size: %x\n", __FUNCTION__, bus->dongle_ram_base, bus->ramsize));
+	DHD_ERROR(("%s: memblock size %d,  #pattern %d\n", __FUNCTION__, MEMBLOCK, NUM_PATTERNS));
+
+	while (num < NUM_PATTERNS) {
+		start = bus->dongle_ram_base;
+		/* Get full mem size */
+		size = bus->ramsize;
+
+		memset(write_buf, init_val[num], MEMBLOCK);
+		while (size > 0) {
+			read_size = MIN(MEMBLOCK, size);
+			memset(read_buf, 0, read_size);
+
+			/* Write */
+			if ((ret = dhdpcie_bus_membytes(bus, TRUE, start, write_buf, read_size))) {
+				DHD_ERROR(("%s: Write Error membytes %d\n", __FUNCTION__, ret));
+				MFREE(bus->dhd->osh, read_buf, MEMBLOCK);
+				MFREE(bus->dhd->osh, write_buf, MEMBLOCK);
+				return FALSE;
+			}
+
+			/* Read */
+			if ((ret = dhdpcie_bus_membytes(bus, FALSE, start, read_buf, read_size))) {
+				DHD_ERROR(("%s: Read Error membytes %d\n", __FUNCTION__, ret));
+				MFREE(bus->dhd->osh, read_buf, MEMBLOCK);
+				MFREE(bus->dhd->osh, write_buf, MEMBLOCK);
+				return FALSE;
+			}
+
+			/* Compare */
+			if (memcmp(read_buf, write_buf, read_size)) {
+				DHD_ERROR(("%s: Mismatch at %x, iter : %d\n",
+					__FUNCTION__, start, num));
+				prhex("Readbuf", read_buf, read_size);
+				prhex("Writebuf", write_buf, read_size);
+				MFREE(bus->dhd->osh, read_buf, MEMBLOCK);
+				MFREE(bus->dhd->osh, write_buf, MEMBLOCK);
+				return FALSE;
+			}
+
+			/* Decrement size and increment start address */
+			size -= read_size;
+			start += read_size;
+		}
+		num++;
+	}
+
+	MFREE(bus->dhd->osh, read_buf, MEMBLOCK);
+	MFREE(bus->dhd->osh, write_buf, MEMBLOCK);
+
+	DHD_ERROR(("%s: Success iter : %d\n", __FUNCTION__, num));
+	return TRUE;
+}
+#endif /* LINUX || linux */
+
+#define PCI_CFG_LINK_SPEED_SHIFT	16
+int
+dhd_get_pcie_linkspeed(dhd_pub_t *dhd)
+{
+	uint32 pcie_lnkst;
+	uint32 pcie_lnkspeed;
+	pcie_lnkst = OSL_PCI_READ_CONFIG(dhd->osh, PCIECFGREG_LINK_STATUS_CTRL,
+		sizeof(pcie_lnkst));
+
+	pcie_lnkspeed = (pcie_lnkst >> PCI_CFG_LINK_SPEED_SHIFT) & PCI_LINK_SPEED_MASK;
+	DHD_INFO(("%s: Link speed: %d\n", __FUNCTION__, pcie_lnkspeed));
+	return pcie_lnkspeed;
+}
+
+int
+dhd_bus_checkdied(struct dhd_bus *bus, char *data, uint size)
+{
+	return dhdpcie_checkdied(bus, data, size);
+}
+
+/* Common backplane can be hung by butting APB2 bridge in reset */
+void
+dhdpcie_induce_cbp_hang(dhd_pub_t *dhd)
+{
+	uint32 addr, val;
+	uint32 apb2_wrapper_reg = 0x18106000;
+	uint32 apb2_reset_ctrl_offset = 0x800;
+	addr = apb2_wrapper_reg + apb2_reset_ctrl_offset;
+	val = 1;
+	dhd_sbreg_op(dhd, addr, &val, FALSE);
+}
+
+bool
+dhd_bus_init_done(struct dhd_bus *bus)
+{
+	return bus->intr_enabled;
+}
+
+#ifdef DHD_SSSR_DUMP
+void
+dhd_fill_sssr_reg_info_4389(dhd_pub_t *dhd)
+{
+	sssr_reg_info_cmn_t *sssr_reg_info_cmn = dhd->sssr_reg_info;
+	sssr_reg_info_v3_t *sssr_reg_info;
+
+	sssr_reg_info = (sssr_reg_info_v3_t *)&sssr_reg_info_cmn->rev3;
+
+	DHD_ERROR(("%s:\n", __FUNCTION__));
+	sssr_reg_info->version = SSSR_REG_INFO_VER_3;
+	sssr_reg_info->length = sizeof(sssr_reg_info_v3_t);
+
+	sssr_reg_info->pmu_regs.base_regs.pmuintmask0 = 0x18012700;
+	sssr_reg_info->pmu_regs.base_regs.pmuintmask1 = 0x18012704;
+	sssr_reg_info->pmu_regs.base_regs.resreqtimer = 0x18012644;
+	sssr_reg_info->pmu_regs.base_regs.macresreqtimer = 0x18012688;
+	sssr_reg_info->pmu_regs.base_regs.macresreqtimer1 = 0x180126f0;
+	sssr_reg_info->pmu_regs.base_regs.macresreqtimer2 = 0x18012738;
+
+	sssr_reg_info->chipcommon_regs.base_regs.intmask = 0x18000024;
+	sssr_reg_info->chipcommon_regs.base_regs.powerctrl = 0x180001e8;
+	sssr_reg_info->chipcommon_regs.base_regs.clockcontrolstatus = 0x180001e0;
+	sssr_reg_info->chipcommon_regs.base_regs.powerctrl_mask = 0x1f00;
+
+	sssr_reg_info->arm_regs.base_regs.clockcontrolstatus = 0x180201e0;
+	sssr_reg_info->arm_regs.base_regs.clockcontrolstatus_val = 0x20;
+	sssr_reg_info->arm_regs.wrapper_regs.resetctrl = 0x18120800;
+	sssr_reg_info->arm_regs.wrapper_regs.extrsrcreq = 0x18006234;
+
+	sssr_reg_info->pcie_regs.base_regs.ltrstate = 0x18001c38;
+	sssr_reg_info->pcie_regs.base_regs.clockcontrolstatus = 0x180011e0;
+	sssr_reg_info->pcie_regs.base_regs.clockcontrolstatus_val = 0x0;
+	sssr_reg_info->pcie_regs.wrapper_regs.extrsrcreq = 0x180061b4;
+
+	sssr_reg_info->mac_regs[0].base_regs.xmtaddress = 0x18021130;
+	sssr_reg_info->mac_regs[0].base_regs.xmtdata = 0x18021134;
+	sssr_reg_info->mac_regs[0].base_regs.clockcontrolstatus = 0x180211e0;
+	sssr_reg_info->mac_regs[0].base_regs.clockcontrolstatus_val = 0x20;
+	sssr_reg_info->mac_regs[0].wrapper_regs.resetctrl = 0x18121800;
+	sssr_reg_info->mac_regs[0].wrapper_regs.extrsrcreq = 0x180062b4;
+	sssr_reg_info->mac_regs[0].wrapper_regs.ioctrl = 0x18121408;
+	sssr_reg_info->mac_regs[0].wrapper_regs.ioctrl_resetseq_val[0] = 0xc7;
+	sssr_reg_info->mac_regs[0].wrapper_regs.ioctrl_resetseq_val[1] = 0x15f;
+	sssr_reg_info->mac_regs[0].wrapper_regs.ioctrl_resetseq_val[2] = 0x151;
+	sssr_reg_info->mac_regs[0].wrapper_regs.ioctrl_resetseq_val[3] = 0x155;
+	sssr_reg_info->mac_regs[0].wrapper_regs.ioctrl_resetseq_val[4] = 0xc5;
+	sssr_reg_info->mac_regs[0].sr_size = 0x40000;
+
+	sssr_reg_info->mac_regs[1].base_regs.xmtaddress = 0x18022130;
+	sssr_reg_info->mac_regs[1].base_regs.xmtdata = 0x18022134;
+	sssr_reg_info->mac_regs[1].base_regs.clockcontrolstatus = 0x180221e0;
+	sssr_reg_info->mac_regs[1].base_regs.clockcontrolstatus_val = 0x20;
+	sssr_reg_info->mac_regs[1].wrapper_regs.resetctrl = 0x18122800;
+	sssr_reg_info->mac_regs[1].wrapper_regs.extrsrcreq = 0x18006334;
+	sssr_reg_info->mac_regs[1].wrapper_regs.ioctrl = 0x18122408;
+	sssr_reg_info->mac_regs[1].wrapper_regs.ioctrl_resetseq_val[0] = 0xc7;
+	sssr_reg_info->mac_regs[1].wrapper_regs.ioctrl_resetseq_val[1] = 0x15f;
+	sssr_reg_info->mac_regs[1].wrapper_regs.ioctrl_resetseq_val[2] = 0x151;
+	sssr_reg_info->mac_regs[1].wrapper_regs.ioctrl_resetseq_val[3] = 0x155;
+	sssr_reg_info->mac_regs[1].wrapper_regs.ioctrl_resetseq_val[4] = 0xc5;
+	sssr_reg_info->mac_regs[1].sr_size = 0x30000;
+
+	sssr_reg_info->mac_regs[2].base_regs.xmtaddress = 0x18023130;
+	sssr_reg_info->mac_regs[2].base_regs.xmtdata = 0x18023134;
+	sssr_reg_info->mac_regs[2].base_regs.clockcontrolstatus = 0x180231e0;
+	sssr_reg_info->mac_regs[2].base_regs.clockcontrolstatus_val = 0x20;
+	sssr_reg_info->mac_regs[2].wrapper_regs.resetctrl = 0x18123800;
+	sssr_reg_info->mac_regs[2].wrapper_regs.extrsrcreq = 0x180063b4;
+	sssr_reg_info->mac_regs[2].wrapper_regs.ioctrl = 0x18123408;
+	sssr_reg_info->mac_regs[2].wrapper_regs.ioctrl_resetseq_val[0] = 0xc7;
+	sssr_reg_info->mac_regs[2].wrapper_regs.ioctrl_resetseq_val[1] = 0x15f;
+	sssr_reg_info->mac_regs[2].wrapper_regs.ioctrl_resetseq_val[2] = 0x151;
+	sssr_reg_info->mac_regs[2].wrapper_regs.ioctrl_resetseq_val[3] = 0x155;
+	sssr_reg_info->mac_regs[2].wrapper_regs.ioctrl_resetseq_val[4] = 0xc5;
+	sssr_reg_info->mac_regs[2].sr_size = 0x30000;
+
+	sssr_reg_info->dig_mem_info.dig_sr_addr = 0x18520000;
+	sssr_reg_info->dig_mem_info.dig_sr_size = 0x10000;
+
+	sssr_reg_info->fis_enab = 1;
+
+	sssr_reg_info->hwa_regs.base_regs.clkenable = 0x180242d0;
+	sssr_reg_info->hwa_regs.base_regs.clkgatingenable = 0x180242d4;
+	sssr_reg_info->hwa_regs.base_regs.clkext = 0x180242e0;
+	sssr_reg_info->hwa_regs.base_regs.clkctlstatus = 0x180241e0;
+	sssr_reg_info->hwa_regs.wrapper_regs.ioctrl = 0x18124408;
+	sssr_reg_info->hwa_regs.wrapper_regs.resetctrl = 0x18124800;
+
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[0] = 0x3;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[1] = 0x1;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[2] = 0x0;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[3] = 0x1;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[4] = 0x1ff;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[5] = 0x1ff;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[6] = 0x3;
+	sssr_reg_info->hwa_regs.hwa_resetseq_val[7] = 0x20;
+
+	dhd->sssr_inited = TRUE;
+}
+
+void
+dhdpcie_fill_sssr_reg_info(dhd_pub_t *dhd)
+{
+	if (dhd_get_chipid(dhd->bus) == BCM4389_CHIP_ID) {
+		dhd_fill_sssr_reg_info_4389(dhd);
+	}
+}
+
+#endif /* DHD_SSSR_DUMP */
+
+#if defined(NDIS)
+void *
+dhd_bus_get_socram_buf(struct dhd_bus *bus, struct dhd_pub *dhdp)
+{
+	return dhd_get_fwdump_buf(dhdp, bus->ramsize);
+}
+#endif /* NDIS */
+
+void
+dhd_bus_set_signature_path(struct dhd_bus *bus, char *sig_path)
+{
+	strlcpy(bus->fwsig_filename, sig_path, sizeof(bus->fwsig_filename));
+}
+
+#define BUS_SLEEP_WAIT_CNT	3
+#define BUS_SLEEP_WAIT_MS	20
+int
+dhd_bus_sleep(dhd_pub_t *dhdp, bool sleep, uint32 *intstatus)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	int active, cnt = 0;
+
+	if (bus) {
+		while ((active = dhd_os_check_wakelock_all(bus->dhd)) &&
+				(cnt < BUS_SLEEP_WAIT_CNT)) {
+			OSL_SLEEP(BUS_SLEEP_WAIT_MS);
+			cnt++;
+		}
+	} else {
+		DHD_ERROR(("bus is NULL\n"));
+		active = -1;
+	}
+
+	return active;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_pcie.h b/drivers/net/wireless/ap12275/dhd_pcie.h
new file mode 100644
index 000000000000..68a100912960
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pcie.h
@@ -0,0 +1,940 @@
+/*
+ * Linux DHD Bus Module for PCIE
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef dhd_pcie_h
+#define dhd_pcie_h
+
+#include <bcmpcie.h>
+#include <hnd_cons.h>
+#include <fwpkg_utils.h>
+#include <dhd_linux.h>
+
+/* defines */
+#define PCIE_SHARED_VERSION		PCIE_SHARED_VERSION_7
+
+#define PCMSGBUF_HDRLEN 0
+#define DONGLE_REG_MAP_SIZE (32 * 1024)
+#define DONGLE_TCM_MAP_SIZE (4096 * 1024)
+#define DONGLE_MIN_MEMSIZE (128 *1024)
+#ifdef DHD_DEBUG
+#define DHD_PCIE_SUCCESS 0
+#define DHD_PCIE_FAILURE 1
+#endif /* DHD_DEBUG */
+#define	REMAP_ENAB(bus)			((bus)->remap)
+#define	REMAP_ISADDR(bus, a)		(((a) >= ((bus)->orig_ramsize)) && ((a) < ((bus)->ramsize)))
+
+#define MAX_DHD_TX_FLOWS	320
+
+/* user defined data structures */
+/* Device console log buffer state */
+#define CONSOLE_LINE_MAX	192u
+#define CONSOLE_BUFFER_MAX	(8 * 1024)
+
+#ifdef IDLE_TX_FLOW_MGMT
+#define IDLE_FLOW_LIST_TIMEOUT 5000
+#define IDLE_FLOW_RING_TIMEOUT 5000
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#ifdef DEVICE_TX_STUCK_DETECT
+#define DEVICE_TX_STUCK_CKECK_TIMEOUT	1000 /* 1 sec */
+#define DEVICE_TX_STUCK_TIMEOUT		10000 /* 10 secs */
+#define DEVICE_TX_STUCK_WARN_DURATION (DEVICE_TX_STUCK_TIMEOUT / DEVICE_TX_STUCK_CKECK_TIMEOUT)
+#define DEVICE_TX_STUCK_DURATION (DEVICE_TX_STUCK_WARN_DURATION * 2)
+#endif /* DEVICE_TX_STUCK_DETECT */
+
+/* implicit DMA for h2d wr and d2h rd indice from Host memory to TCM */
+#define IDMA_ENAB(dhd)		((dhd) && (dhd)->idma_enable)
+#define IDMA_ACTIVE(dhd)	((dhd) && ((dhd)->idma_enable) && ((dhd)->idma_inited))
+
+#define IDMA_CAPABLE(bus)	(((bus)->sih->buscorerev == 19) || ((bus)->sih->buscorerev >= 23))
+
+/* IFRM (Implicit Flow Ring Manager enable and inited */
+#define IFRM_ENAB(dhd)		((dhd) && (dhd)->ifrm_enable)
+#define IFRM_ACTIVE(dhd)	((dhd) && ((dhd)->ifrm_enable) && ((dhd)->ifrm_inited))
+
+/* DAR registers use for h2d doorbell */
+#define DAR_ENAB(dhd)		((dhd) && (dhd)->dar_enable)
+#define DAR_ACTIVE(dhd)		((dhd) && ((dhd)->dar_enable) && ((dhd)->dar_inited))
+
+/* DAR WAR for revs < 64 */
+#define DAR_PWRREQ(bus)		(((bus)->_dar_war) && DAR_ACTIVE((bus)->dhd))
+
+/* PCIE CTO Prevention and Recovery */
+#define PCIECTO_ENAB(bus)	((bus)->cto_enable)
+
+/* Implicit DMA index usage :
+ * Index 0 for h2d write index transfer
+ * Index 1 for d2h read index transfer
+ */
+#define IDMA_IDX0 0
+#define IDMA_IDX1 1
+#define IDMA_IDX2 2
+#define IDMA_IDX3 3
+#define DMA_TYPE_SHIFT	4
+#define DMA_TYPE_IDMA	1
+
+#define DHDPCIE_CONFIG_HDR_SIZE 16
+#define DHDPCIE_CONFIG_CHECK_DELAY_MS 10 /* 10ms */
+#define DHDPCIE_CONFIG_CHECK_RETRY_COUNT 20
+#define DHDPCIE_DONGLE_PWR_TOGGLE_DELAY 1000 /* 1ms in units of us */
+#define DHDPCIE_PM_D3_DELAY 200000 /* 200ms in units of us */
+#define DHDPCIE_PM_D2_DELAY 200 /* 200us */
+
+typedef struct dhd_console {
+	 uint		count;	/* Poll interval msec counter */
+	 uint		log_addr;		 /* Log struct address (fixed) */
+	 hnd_log_t	 log;			 /* Log struct (host copy) */
+	 uint		 bufsize;		 /* Size of log buffer */
+	 uint8		 *buf;			 /* Log buffer (host copy) */
+	 uint		 last;			 /* Last buffer read index */
+} dhd_console_t;
+
+typedef struct ring_sh_info {
+	uint32 ring_mem_addr;
+	uint32 ring_state_w;
+	uint32 ring_state_r;
+	pcie_hwa_db_index_t ring_hwa_db_idx;		/* HWA DB index value per ring */
+} ring_sh_info_t;
+#define MAX_DS_TRACE_SIZE	50
+#ifdef DHD_MMIO_TRACE
+#define MAX_MMIO_TRACE_SIZE	256
+/* Minimum of 250us should be elapsed to add new entry */
+#define MIN_MMIO_TRACE_TIME 250
+#define DHD_RING_IDX 0x00FF0000
+typedef struct _dhd_mmio_trace_t {
+	uint64  timestamp;
+	uint32	addr;
+	uint32	value;
+	bool	set;
+} dhd_mmio_trace_t;
+#endif /* defined(DHD_MMIO_TRACE) */
+typedef struct _dhd_ds_trace_t {
+	uint64  timestamp;
+	bool	d2h;
+	uint32	dsval;
+#ifdef PCIE_INB_DW
+	enum dhd_bus_ds_state inbstate;
+#endif /* PCIE_INB_DW */
+} dhd_ds_trace_t;
+
+#define DEVICE_WAKE_NONE	0
+#define DEVICE_WAKE_OOB		1
+#define DEVICE_WAKE_INB		2
+
+#define INBAND_DW_ENAB(bus)		((bus)->dw_option == DEVICE_WAKE_INB)
+#define OOB_DW_ENAB(bus)		((bus)->dw_option == DEVICE_WAKE_OOB)
+#define NO_DW_ENAB(bus)			((bus)->dw_option == DEVICE_WAKE_NONE)
+
+#define PCIE_PWR_REQ_RELOAD_WAR_ENAB(buscorerev) \
+	((buscorerev == 66) || (buscorerev == 67) || (buscorerev == 68) || \
+	(buscorerev == 70) || (buscorerev == 72))
+
+#define PCIE_FASTLPO_ENABLED(buscorerev) \
+	((buscorerev == 66) || (buscorerev == 67) || (buscorerev == 68) || \
+	(buscorerev == 70) || (buscorerev == 72) || (buscorerev == 76))
+
+/*
+ * HW JIRA - CRWLPCIEGEN2-672
+ * Producer Index Feature which is used by F1 gets reset on F0 FLR
+ * fixed in REV68
+ */
+#define PCIE_ENUM_RESET_WAR_ENAB(buscorerev) \
+	((buscorerev == 66) || (buscorerev == 67))
+
+struct dhd_bus;
+
+struct dhd_pcie_rev {
+	uint8	fw_rev;
+	void (*handle_mb_data)(struct dhd_bus *);
+};
+
+typedef struct dhdpcie_config_save
+{
+	uint32 header[DHDPCIE_CONFIG_HDR_SIZE];
+	/* pmcsr save */
+	uint32 pmcsr;
+	/* express save */
+	uint32 exp_dev_ctrl_stat;
+	uint32 exp_link_ctrl_stat;
+	uint32 exp_dev_ctrl_stat2;
+	uint32 exp_link_ctrl_stat2;
+	/* msi save */
+	uint32 msi_cap;
+	uint32 msi_addr0;
+	uint32 msi_addr1;
+	uint32 msi_data;
+	/* l1pm save */
+	uint32 l1pm0;
+	uint32 l1pm1;
+	/* ltr save */
+	uint32 ltr;
+	/* aer save */
+	uint32 aer_caps_ctrl; /* 0x18 */
+	uint32 aer_severity;  /* 0x0C */
+	uint32 aer_umask;     /* 0x08 */
+	uint32 aer_cmask;     /* 0x14 */
+	uint32 aer_root_cmd;  /* 0x2c */
+	/* BAR0 and BAR1 windows */
+	uint32 bar0_win;
+	uint32 bar1_win;
+} dhdpcie_config_save_t;
+
+/* The level of bus communication with the dongle */
+enum dhd_bus_low_power_state {
+	DHD_BUS_NO_LOW_POWER_STATE,	/* Not in low power state */
+	DHD_BUS_D3_INFORM_SENT,		/* D3 INFORM sent */
+	DHD_BUS_D3_ACK_RECIEVED,	/* D3 ACK recieved */
+};
+
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+#define FRS_TRACE_SIZE 32 /* frs - flow_ring_status */
+typedef struct _dhd_flow_ring_status_trace_t {
+	uint64  timestamp;
+	uint16 h2d_ctrl_post_drd;
+	uint16 h2d_ctrl_post_dwr;
+	uint16 d2h_ctrl_cpln_drd;
+	uint16 d2h_ctrl_cpln_dwr;
+	uint16 h2d_rx_post_drd;
+	uint16 h2d_rx_post_dwr;
+	uint16 d2h_rx_cpln_drd;
+	uint16 d2h_rx_cpln_dwr;
+	uint16 d2h_tx_cpln_drd;
+	uint16 d2h_tx_cpln_dwr;
+	uint16 h2d_info_post_drd;
+	uint16 h2d_info_post_dwr;
+	uint16 d2h_info_cpln_drd;
+	uint16 d2h_info_cpln_dwr;
+	uint16 d2h_ring_edl_drd;
+	uint16 d2h_ring_edl_dwr;
+} dhd_frs_trace_t;
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+
+typedef enum dhd_pcie_link_state {
+	DHD_PCIE_ALL_GOOD = 0,
+	DHD_PCIE_LINK_DOWN = 1,
+	DHD_PCIE_COMMON_BP_DOWN = 2,
+	DHD_PCIE_WLAN_BP_DOWN = 3
+} dhd_pcie_link_state_type_t;
+
+/* Max length of filename in IOVAR or in module parameter */
+#define DHD_MAX_PATH	2048u
+
+/** Instantiated once for each hardware (dongle) instance that this DHD manages */
+typedef struct dhd_bus {
+	dhd_pub_t	*dhd;	/**< pointer to per hardware (dongle) unique instance */
+#if !defined(NDIS)
+	struct pci_dev  *rc_dev;	/* pci RC device handle */
+	struct pci_dev  *dev;		/* pci device handle */
+#endif /* !defined(NDIS) */
+	dll_t		flowring_active_list; /* constructed list of tx flowring queues */
+#ifdef IDLE_TX_FLOW_MGMT
+	uint64		active_list_last_process_ts;
+						/* stores the timestamp of active list processing */
+#endif /* IDLE_TX_FLOW_MGMT */
+
+#ifdef DEVICE_TX_STUCK_DETECT
+	/* Flag to enable/disable device tx stuck monitor by DHD IOVAR dev_tx_stuck_monitor */
+	uint32 dev_tx_stuck_monitor;
+	/* Stores the timestamp (msec) of the last device Tx stuck check */
+	uint32	device_tx_stuck_check;
+#endif /* DEVICE_TX_STUCK_DETECT */
+
+	si_t		*sih;			/* Handle for SI calls */
+	char		*vars;			/* Variables (from CIS and/or other) */
+	uint		varsz;			/* Size of variables buffer */
+	uint32		sbaddr;			/* Current SB window pointer (-1, invalid) */
+	sbpcieregs_t	*reg;			/* Registers for PCIE core */
+
+	uint		armrev;			/* CPU core revision */
+	uint		coreid;			/* CPU core id */
+	uint		ramrev;			/* SOCRAM core revision */
+	uint32		ramsize;		/* Size of RAM in SOCRAM (bytes) */
+	uint32		orig_ramsize;		/* Size of RAM in SOCRAM (bytes) */
+	uint32		srmemsize;		/* Size of SRMEM */
+
+	uint32		bus;			/* gSPI or SDIO bus */
+	uint32		bus_num;		/* bus number */
+	uint32		slot_num;		/* slot ID */
+	uint32		intstatus;		/* Intstatus bits (events) pending */
+	bool		dpc_sched;		/* Indicates DPC schedule (intrpt rcvd) */
+	bool		fcstate;		/* State of dongle flow-control */
+
+	uint16		cl_devid;		/* cached devid for dhdsdio_probe_attach() */
+	char		*fw_path;		/* module_param: path to firmware image */
+	char		*nv_path;		/* module_param: path to nvram vars file */
+#ifdef CACHE_FW_IMAGES
+	int			processed_nvram_params_len;	/* Modified len of NVRAM info */
+#endif
+
+#ifdef BCM_ROUTER_DHD
+	char		*nvram_params;		/* user specified nvram params. */
+	int			nvram_params_len;
+#endif /* BCM_ROUTER_DHD */
+
+	struct pktq	txq;			/* Queue length used for flow-control */
+
+	bool		intr;			/* Use interrupts */
+	bool		poll;			/* Use polling */
+	bool		ipend;			/* Device interrupt is pending */
+	bool		intdis;			/* Interrupts disabled by isr */
+	uint		intrcount;		/* Count of device interrupt callbacks */
+	uint		lastintrs;		/* Count as of last watchdog timer */
+
+	dhd_console_t	console;		/* Console output polling support */
+	uint		console_addr;		/* Console address from shared struct */
+
+	bool		alp_only;		/* Don't use HT clock (ALP only) */
+
+	bool		remap;		/* Contiguous 1MB RAM: 512K socram + 512K devram
+					 * Available with socram rev 16
+					 * Remap region not DMA-able
+					 */
+	uint32		resetinstr;
+	uint32		dongle_ram_base;
+	uint32		next_tlv;	/* Holds location of next available TLV */
+	ulong		shared_addr;
+	pciedev_shared_t	*pcie_sh;
+	uint32		dma_rxoffset;
+	volatile char	*regs;		/* pci device memory va */
+	volatile char	*tcm;		/* pci device memory va */
+	uint32		bar1_size;	/* pci device memory size */
+	uint32		curr_bar1_win;	/* current PCIEBar1Window setting */
+	osl_t		*osh;
+	uint32		nvram_csm;	/* Nvram checksum */
+	uint16		pollrate;
+	uint16  polltick;
+
+	volatile uint32  *pcie_mb_intr_addr;
+	volatile uint32  *pcie_mb_intr_2_addr;
+	void    *pcie_mb_intr_osh;
+	bool	sleep_allowed;
+
+	wake_counts_t	wake_counts;
+
+	/* version 3 shared struct related info start */
+	ring_sh_info_t	ring_sh[BCMPCIE_COMMON_MSGRINGS + MAX_DHD_TX_FLOWS];
+
+	uint8	h2d_ring_count;
+	uint8	d2h_ring_count;
+	uint32  ringmem_ptr;
+	uint32  ring_state_ptr;
+
+	uint32 d2h_dma_scratch_buffer_mem_addr;
+
+	uint32 h2d_mb_data_ptr_addr;
+	uint32 d2h_mb_data_ptr_addr;
+	/* version 3 shared struct related info end */
+
+	uint32 def_intmask;
+	uint32 d2h_mb_mask;
+	uint32 pcie_mailbox_mask;
+	uint32 pcie_mailbox_int;
+	bool	ltrsleep_on_unload;
+	uint	wait_for_d3_ack;
+	uint16	max_tx_flowrings;
+	uint16	max_submission_rings;
+	uint16	max_completion_rings;
+	uint16	max_cmn_rings;
+	uint32	rw_index_sz;
+	uint32	hwa_db_index_sz;
+	bool	db1_for_mb;
+
+	dhd_timeout_t doorbell_timer;
+	bool	device_wake_state;
+#ifdef PCIE_OOB
+	bool	oob_enabled;
+#endif /* PCIE_OOB */
+	bool	irq_registered;
+	bool	d2h_intr_method;
+	bool	d2h_intr_control;
+	int32 idletime;                 /* Control for activity timeout */
+	bool rpm_enabled;
+	uint32 d3_inform_cnt;
+	uint32 d0_inform_cnt;
+	uint32 d0_inform_in_use_cnt;
+	uint8 force_suspend;
+	uint8 is_linkdown;
+	uint8 no_bus_init;
+#ifdef IDLE_TX_FLOW_MGMT
+	bool enable_idle_flowring_mgmt;
+#endif /* IDLE_TX_FLOW_MGMT */
+	struct	dhd_pcie_rev api;
+	bool use_mailbox;
+	bool    use_d0_inform;
+	void	*bus_lp_state_lock;
+	void	*pwr_req_lock;
+	bool	dongle_in_deepsleep;
+	void	*dongle_ds_lock;
+	bool	bar1_switch_enab;
+	void	*bar1_switch_lock;
+	void *backplane_access_lock;
+	enum dhd_bus_low_power_state bus_low_power_state;
+#ifdef DHD_FLOW_RING_STATUS_TRACE
+	dhd_frs_trace_t frs_isr_trace[FRS_TRACE_SIZE]; /* frs - flow_ring_status */
+	dhd_frs_trace_t frs_dpc_trace[FRS_TRACE_SIZE]; /* frs - flow_ring_status */
+	uint32	frs_isr_count;
+	uint32	frs_dpc_count;
+#endif /* DHD_FLOW_RING_STATUS_TRACE */
+#ifdef DHD_MMIO_TRACE
+	dhd_mmio_trace_t   mmio_trace[MAX_MMIO_TRACE_SIZE];
+	uint32	mmio_trace_count;
+#endif /* defined(DHD_MMIO_TRACE) */
+	dhd_ds_trace_t   ds_trace[MAX_DS_TRACE_SIZE];
+	uint32	ds_trace_count;
+	uint32  hostready_count; /* Number of hostready issued */
+#if defined(PCIE_OOB) || defined(BCMPCIE_OOB_HOST_WAKE)
+	bool	oob_presuspend;
+#endif /* PCIE_OOB || BCMPCIE_OOB_HOST_WAKE */
+	dhdpcie_config_save_t saved_config;
+	ulong host_irq_enable_count;
+	ulong host_irq_disable_count;
+	ulong dngl_intmask_disable_count;
+	ulong dngl_intmask_enable_count;
+	ulong dpc_return_busdown_count;
+	ulong non_ours_irq_count;
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	ulong oob_intr_count;
+	ulong oob_intr_enable_count;
+	ulong oob_intr_disable_count;
+	uint64 last_oob_irq_isr_time;
+	uint64 last_oob_irq_thr_time;
+	uint64 last_oob_irq_enable_time;
+	uint64 last_oob_irq_disable_time;
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+	uint64 isr_entry_time;
+	uint64 prev_isr_entry_time;
+	uint64 isr_exit_time;
+	uint64 isr_sched_dpc_time;
+	uint64 rpm_sched_dpc_time;
+	uint64 dpc_entry_time;
+	uint64 dpc_exit_time;
+	uint64 resched_dpc_time;
+	uint64 last_d3_inform_time;
+	uint64 last_d3_ack_time;
+	uint64 last_process_ctrlbuf_time;
+	uint64 last_process_flowring_time;
+	uint64 last_process_txcpl_time;
+	uint64 last_process_rxcpl_time;
+	uint64 last_process_infocpl_time;
+	uint64 last_process_edl_time;
+	uint64 last_suspend_start_time;
+	uint64 last_suspend_end_time;
+	uint64 last_resume_start_time;
+	uint64 last_resume_end_time;
+	uint64 last_non_ours_irq_time;
+	uint64 dpc_time_usec;
+	uint64  *dpc_time_histo;
+	uint64 ctrl_cpl_post_time_usec;
+	uint64  *ctrl_cpl_post_time_histo;
+	uint64 tx_post_time_usec;
+	uint64  *tx_post_time_histo;
+	uint64 tx_cpl_time_usec;
+	uint64  *tx_cpl_time_histo;
+	uint64 rx_cpl_post_time_usec;
+	uint64  *rx_cpl_post_time_histo;
+	bool  hwa_enabled;
+	bool  idma_enabled;
+	bool  ifrm_enabled;
+	bool  dar_enabled;
+	uint32 dmaxfer_complete;
+	uint8	dw_option;
+#ifdef PCIE_INB_DW
+	bool	inb_enabled;
+	uint32	ds_exit_timeout;
+	uint32	host_sleep_exit_timeout;
+	uint	wait_for_ds_exit;
+	uint32	inband_dw_assert_cnt; /* # of inband device_wake assert */
+	uint32	inband_dw_deassert_cnt; /* # of inband device_wake deassert */
+	uint32	inband_ds_exit_host_cnt; /* # of DS-EXIT , host initiated */
+	uint32	inband_ds_exit_device_cnt; /* # of DS-EXIT , device initiated */
+	uint32	inband_ds_exit_to_cnt; /* # of DS-EXIT timeout */
+	uint32	inband_host_sleep_exit_to_cnt; /* # of Host_Sleep exit timeout */
+	void	*inb_lock;	/* Lock to serialize in band device wake activity */
+	/* # of contexts in the host which currently want a FW transaction */
+	uint32  host_active_cnt;
+	bool	skip_ds_ack; /* Skip DS-ACK during suspend in progress */
+#endif /* PCIE_INB_DW */
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+	bool  ds_enabled;
+#endif
+#if defined(PCIE_INB_DW)
+	bool calc_ds_exit_latency;
+	bool deep_sleep; /* Indicates deep_sleep set or unset by the DHD IOVAR deep_sleep */
+	uint64 ds_exit_latency;
+	uint64 ds_exit_ts1;
+	uint64 ds_exit_ts2;
+#endif /* PCIE_INB_DW */
+	bool _dar_war;
+#ifdef GDB_PROXY
+	/* True if firmware loaded and backplane accessible */
+	bool gdb_proxy_access_enabled;
+	/* ID set by last "gdb_proxy_probe" iovar */
+	uint32 gdb_proxy_last_id;
+	/* True if firmware was started in bootloader mode */
+	bool gdb_proxy_bootloader_mode;
+	/* Counter incremented at each generated memory dump */
+	uint32 gdb_proxy_mem_dump_count;
+#endif /* GDB_PROXY */
+	uint8  dma_chan;
+
+	bool    cto_enable;     /* enable PCIE CTO Prevention and recovery */
+	uint32  cto_threshold;  /* PCIE CTO timeout threshold */
+	bool	cto_triggered;	/* CTO is triggered */
+	bool intr_enabled; /* ready to receive interrupts from dongle */
+	int	pwr_req_ref;
+	bool flr_force_fail; /* user intends to simulate flr force fail */
+
+	/* Information used to compose the memory map and to write the memory map,
+	 * FW, and FW signature to dongle RAM.
+	 * This information is used by the bootloader.
+	 */
+	uint32 ramtop_addr;		/* Dongle address of unused space at top of RAM */
+	uint32 fw_download_addr;	/* Dongle address of FW download */
+	uint32 fw_download_len;		/* Length in bytes of FW download */
+	uint32 fwsig_download_addr;	/* Dongle address of FW signature download */
+	uint32 fwsig_download_len;	/* Length in bytes of FW signature download */
+	uint32 fwstat_download_addr;	/* Dongle address of FWS status download */
+	uint32 fwstat_download_len;	/* Length in bytes of FWS status download */
+	uint32 fw_memmap_download_addr;	/* Dongle address of FWS memory-info download */
+	uint32 fw_memmap_download_len;	/* Length in bytes of FWS memory-info download */
+
+	char fwsig_filename[DHD_MAX_PATH];		/* Name of FW signature file */
+	char bootloader_filename[DHD_FILENAME_MAX];	/* Name of bootloader image file */
+	uint32 bootloader_addr;		/* Dongle address of bootloader download */
+	bool force_bt_quiesce; /* send bt_quiesce command to BT driver. */
+	bool rc_ep_aspm_cap; /* RC and EP ASPM capable */
+	bool rc_ep_l1ss_cap; /* EC and EP L1SS capable */
+#if defined(DHD_H2D_LOG_TIME_SYNC)
+	ulong dhd_rte_time_sync_count; /* OSL_SYSUPTIME_US() */
+#endif /* DHD_H2D_LOG_TIME_SYNC */
+#ifdef D2H_MINIDUMP
+	bool d2h_minidump; /* This flag will be set if Host and FW handshake to collect minidump */
+	bool d2h_minidump_override; /* Force disable minidump through dhd IOVAR */
+#endif /* D2H_MINIDUMP */
+#ifdef BCMSLTGT
+	int xtalfreq;		/* Xtal frequency used for htclkratio calculation */
+	uint32 ilp_tick;	/* ILP ticks per second read from pmutimer */
+	uint32 alp_to_ilp_ratio;	/* ALP ticks per ILP ticks read from pmu_xtalfreq */
+	uint32 xtal_to_alp_ratio;	/* xtal to ALP ratio which can change from chip to chip */
+#endif /* BCMSLTGT */
+	uint16 hp2p_txcpl_max_items;
+	uint16 hp2p_rxcpl_max_items;
+	/* PCIE coherent status */
+	uint32 coherent_state;
+	uint32 inb_dw_deassert_cnt;
+	uint64 arm_oor_time;
+	uint64 rd_shared_pass_time;
+	uint32 hwa_mem_base;
+	uint32 hwa_mem_size;
+	dhd_pcie_link_state_type_t link_state;
+	bool dar_err_set;
+	uint32 ptm_ctrl_reg;
+	fwpkg_info_t fwpkg;	/* combined fw package info structure */
+	bool lpm_mode;	/* lpm enabled */
+	bool lpm_keep_in_reset; /* during LPM keep in FLR, if FLR force is enabled */
+	bool lpm_mem_kill; /* kill WLAN memories in LPM */
+	bool lpm_force_flr; /* Force F0 FLR on WLAN  when in LPM */
+	uint32 lpbk_xfer_data_pattern_type; /*  data Pattern type DMA lpbk */
+} dhd_bus_t;
+
+#define LPBK_DMA_XFER_DTPTRN_DEFAULT	0
+#define LPBK_DMA_XFER_DTPTRN_0x00	1
+#define LPBK_DMA_XFER_DTPTRN_0xFF	2
+#define LPBK_DMA_XFER_DTPTRN_0x55	3
+#define LPBK_DMA_XFER_DTPTRN_0xAA	4
+
+#define LPM_MODE_LPM_ALL	0x1
+#define LPM_MODE_NO_MEMKILL	0x010
+#define LPM_MODE_NO_FLR		0x100
+
+#ifdef DHD_MSI_SUPPORT
+extern uint enable_msi;
+#endif /* DHD_MSI_SUPPORT */
+
+enum {
+	PCIE_INTX = 0,
+	PCIE_MSI = 1
+};
+
+enum {
+	PCIE_D2H_INTMASK_CTRL = 0,
+	PCIE_HOST_IRQ_CTRL = 1
+};
+
+static INLINE bool
+__dhd_check_bus_in_lps(dhd_bus_t *bus)
+{
+	bool ret = (bus->bus_low_power_state == DHD_BUS_D3_INFORM_SENT) ||
+		(bus->bus_low_power_state == DHD_BUS_D3_ACK_RECIEVED);
+	return ret;
+}
+
+static INLINE bool
+dhd_check_bus_in_lps(dhd_bus_t *bus)
+{
+	unsigned long flags_bus;
+	bool ret;
+	DHD_BUS_LP_STATE_LOCK(bus->bus_lp_state_lock, flags_bus);
+	ret = __dhd_check_bus_in_lps(bus);
+	DHD_BUS_LP_STATE_UNLOCK(bus->bus_lp_state_lock, flags_bus);
+	return ret;
+}
+
+static INLINE bool
+__dhd_check_bus_lps_d3_acked(dhd_bus_t *bus)
+{
+	bool ret = (bus->bus_low_power_state == DHD_BUS_D3_ACK_RECIEVED);
+	return ret;
+}
+
+static INLINE bool
+dhd_check_bus_lps_d3_acked(dhd_bus_t *bus)
+{
+	unsigned long flags_bus;
+	bool ret;
+	DHD_BUS_LP_STATE_LOCK(bus->bus_lp_state_lock, flags_bus);
+	ret = __dhd_check_bus_lps_d3_acked(bus);
+	DHD_BUS_LP_STATE_UNLOCK(bus->bus_lp_state_lock, flags_bus);
+	return ret;
+}
+
+static INLINE void
+__dhd_set_bus_not_in_lps(dhd_bus_t *bus)
+{
+	bus->bus_low_power_state = DHD_BUS_NO_LOW_POWER_STATE;
+	return;
+}
+
+static INLINE void
+dhd_set_bus_not_in_lps(dhd_bus_t *bus)
+{
+	unsigned long flags_bus;
+	DHD_BUS_LP_STATE_LOCK(bus->bus_lp_state_lock, flags_bus);
+	__dhd_set_bus_not_in_lps(bus);
+	DHD_BUS_LP_STATE_UNLOCK(bus->bus_lp_state_lock, flags_bus);
+	return;
+}
+
+static INLINE void
+__dhd_set_bus_lps_d3_informed(dhd_bus_t *bus)
+{
+	bus->bus_low_power_state = DHD_BUS_D3_INFORM_SENT;
+	return;
+}
+
+static INLINE void
+dhd_set_bus_lps_d3_informed(dhd_bus_t *bus)
+{
+	unsigned long flags_bus;
+	DHD_BUS_LP_STATE_LOCK(bus->bus_lp_state_lock, flags_bus);
+	__dhd_set_bus_lps_d3_informed(bus);
+	DHD_BUS_LP_STATE_UNLOCK(bus->bus_lp_state_lock, flags_bus);
+	return;
+}
+
+static INLINE void
+__dhd_set_bus_lps_d3_acked(dhd_bus_t *bus)
+{
+	bus->bus_low_power_state = DHD_BUS_D3_ACK_RECIEVED;
+	return;
+}
+
+static INLINE void
+dhd_set_bus_lps_d3_acked(dhd_bus_t *bus)
+{
+	unsigned long flags_bus;
+	DHD_BUS_LP_STATE_LOCK(bus->bus_lp_state_lock, flags_bus);
+	__dhd_set_bus_lps_d3_acked(bus);
+	DHD_BUS_LP_STATE_UNLOCK(bus->bus_lp_state_lock, flags_bus);
+	return;
+}
+
+/* check routines */
+#define DHD_CHK_BUS_IN_LPS(bus)			dhd_check_bus_in_lps(bus)
+#define __DHD_CHK_BUS_IN_LPS(bus)		__dhd_check_bus_in_lps(bus)
+
+#define DHD_CHK_BUS_NOT_IN_LPS(bus)		!(DHD_CHK_BUS_IN_LPS(bus))
+#define __DHD_CHK_BUS_NOT_IN_LPS(bus)		!(__DHD_CHK_BUS_IN_LPS(bus))
+
+#define DHD_CHK_BUS_LPS_D3_INFORMED(bus)	DHD_CHK_BUS_IN_LPS(bus)
+#define __DHD_CHK_BUS_LPS_D3_INFORMED(bus)	__DHD_CHK_BUS_IN_LPS(bus)
+
+#define DHD_CHK_BUS_LPS_D3_ACKED(bus)		dhd_check_bus_lps_d3_acked(bus)
+#define __DHD_CHK_BUS_LPS_D3_ACKED(bus)		__dhd_check_bus_lps_d3_acked(bus)
+
+/* set routines */
+#define DHD_SET_BUS_NOT_IN_LPS(bus)		dhd_set_bus_not_in_lps(bus)
+#define __DHD_SET_BUS_NOT_IN_LPS(bus)		__dhd_set_bus_not_in_lps(bus)
+
+#define DHD_SET_BUS_LPS_D3_INFORMED(bus)	dhd_set_bus_lps_d3_informed(bus)
+#define __DHD_SET_BUS_LPS_D3_INFORMED(bus)	__dhd_set_bus_lps_d3_informed(bus)
+
+#define DHD_SET_BUS_LPS_D3_ACKED(bus)		dhd_set_bus_lps_d3_acked(bus)
+#define __DHD_SET_BUS_LPS_D3_ACKED(bus)		__dhd_set_bus_lps_d3_acked(bus)
+
+/* function declarations */
+
+extern uint32* dhdpcie_bus_reg_map(osl_t *osh, ulong addr, int size);
+extern int dhdpcie_bus_register(void);
+extern void dhdpcie_bus_unregister(void);
+extern bool dhdpcie_chipmatch(uint16 vendor, uint16 device);
+
+extern int dhdpcie_bus_attach(osl_t *osh, dhd_bus_t **bus_ptr,
+	volatile char *regs, volatile char *tcm, void *pci_dev, wifi_adapter_info_t *adapter);
+extern uint32 dhdpcie_bus_cfg_read_dword(struct dhd_bus *bus, uint32 addr, uint32 size);
+extern void dhdpcie_bus_cfg_write_dword(struct dhd_bus *bus, uint32 addr, uint32 size, uint32 data);
+extern void dhdpcie_bus_intr_enable(struct dhd_bus *bus);
+extern void dhdpcie_bus_intr_disable(struct dhd_bus *bus);
+extern int dhpcie_bus_mask_interrupt(dhd_bus_t *bus);
+extern void dhdpcie_bus_release(struct dhd_bus *bus);
+extern int32 dhdpcie_bus_isr(struct dhd_bus *bus);
+extern void dhdpcie_free_irq(dhd_bus_t *bus);
+extern void dhdpcie_bus_ringbell_fast(struct dhd_bus *bus, uint32 value);
+extern void dhdpcie_bus_ringbell_2_fast(struct dhd_bus *bus, uint32 value, bool devwake);
+extern void dhdpcie_dongle_reset(dhd_bus_t *bus);
+extern int dhd_bus_cfg_sprom_ctrl_bp_reset(struct dhd_bus *bus);
+extern int dhd_bus_cfg_ss_ctrl_bp_reset(struct dhd_bus *bus);
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+extern int dhdpcie_bus_suspend(struct  dhd_bus *bus, bool state, bool byint);
+#else
+extern int dhdpcie_bus_suspend(struct  dhd_bus *bus, bool state);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+extern int dhdpcie_pci_suspend_resume(struct  dhd_bus *bus, bool state);
+extern uint32 dhdpcie_force_alp(struct dhd_bus *bus, bool enable);
+extern uint32 dhdpcie_set_l1_entry_time(struct dhd_bus *bus, int force_l1_entry_time);
+extern bool dhdpcie_tcm_valid(dhd_bus_t *bus);
+extern void dhdpcie_pme_active(osl_t *osh, bool enable);
+extern bool dhdpcie_pme_cap(osl_t *osh);
+extern uint32 dhdpcie_lcreg(osl_t *osh, uint32 mask, uint32 val);
+extern void dhdpcie_set_pmu_min_res_mask(struct dhd_bus *bus, uint min_res_mask);
+extern uint8 dhdpcie_clkreq(osl_t *osh, uint32 mask, uint32 val);
+extern int dhdpcie_disable_irq(dhd_bus_t *bus);
+extern int dhdpcie_disable_irq_nosync(dhd_bus_t *bus);
+extern int dhdpcie_enable_irq(dhd_bus_t *bus);
+
+extern void dhd_bus_dump_dar_registers(struct dhd_bus *bus);
+extern void dhd_bus_dump_imp_cfg_registers(struct dhd_bus *bus);
+
+#if defined(linux) || defined(LINUX)
+extern uint32 dhdpcie_rc_config_read(dhd_bus_t *bus, uint offset);
+extern uint32 dhdpcie_rc_access_cap(dhd_bus_t *bus, int cap, uint offset, bool is_ext,
+		bool is_write, uint32 writeval);
+extern uint32 dhdpcie_ep_access_cap(dhd_bus_t *bus, int cap, uint offset, bool is_ext,
+		bool is_write, uint32 writeval);
+extern uint32 dhd_debug_get_rc_linkcap(dhd_bus_t *bus);
+extern void dhdpcie_enable_irq_loop(dhd_bus_t *bus);
+#else
+static INLINE uint32 dhdpcie_rc_config_read(dhd_bus_t *bus, uint offset) { return 0;}
+static INLINE uint32 dhdpcie_rc_access_cap(dhd_bus_t *bus, int cap, uint offset, bool is_ext,
+		bool is_write, uint32 writeval) { return -1;}
+static INLINE uint32 dhdpcie_ep_access_cap(dhd_bus_t *bus, int cap, uint offset, bool is_ext,
+		bool is_write, uint32 writeval) { return -1;}
+static INLINE uint32 dhd_debug_get_rc_linkcap(dhd_bus_t *bus) { return -1;}
+static INLINE void dhdpcie_enable_irq_loop(dhd_bus_t *bus) { return; }
+#endif /* LINUX || linux */
+#if defined(linux) || defined(LINUX)
+extern int dhdpcie_start_host_dev(dhd_bus_t *bus);
+extern int dhdpcie_stop_host_dev(dhd_bus_t *bus);
+extern int dhdpcie_disable_device(dhd_bus_t *bus);
+extern int dhdpcie_alloc_resource(dhd_bus_t *bus);
+extern void dhdpcie_free_resource(dhd_bus_t *bus);
+extern void dhdpcie_dump_resource(dhd_bus_t *bus);
+extern int dhdpcie_bus_request_irq(struct dhd_bus *bus);
+void dhdpcie_os_setbar1win(dhd_bus_t *bus, uint32 addr);
+void dhdpcie_os_wtcm8(dhd_bus_t *bus, ulong offset, uint8 data);
+uint8 dhdpcie_os_rtcm8(dhd_bus_t *bus, ulong offset);
+void dhdpcie_os_wtcm16(dhd_bus_t *bus, ulong offset, uint16 data);
+uint16 dhdpcie_os_rtcm16(dhd_bus_t *bus, ulong offset);
+void dhdpcie_os_wtcm32(dhd_bus_t *bus, ulong offset, uint32 data);
+uint32 dhdpcie_os_rtcm32(dhd_bus_t *bus, ulong offset);
+#ifdef DHD_SUPPORT_64BIT
+void dhdpcie_os_wtcm64(dhd_bus_t *bus, ulong offset, uint64 data);
+uint64 dhdpcie_os_rtcm64(dhd_bus_t *bus, ulong offset);
+#endif
+extern void dhd_dpc_kill(dhd_pub_t *dhdp);
+#endif /* LINUX || linux */
+
+#if defined(linux) || defined(LINUX)
+extern int dhdpcie_enable_device(dhd_bus_t *bus);
+#endif
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+extern int dhdpcie_oob_intr_register(dhd_bus_t *bus);
+extern void dhdpcie_oob_intr_unregister(dhd_bus_t *bus);
+extern void dhdpcie_oob_intr_set(dhd_bus_t *bus, bool enable);
+extern int dhdpcie_get_oob_irq_num(struct dhd_bus *bus);
+extern int dhdpcie_get_oob_irq_status(struct dhd_bus *bus);
+extern int dhdpcie_get_oob_irq_level(void);
+#ifdef PRINT_WAKEUP_GPIO_STATUS
+extern int dhdpcie_get_oob_gpio_number(void);
+#endif /* PRINT_WAKEUP_GPIO_STATUS */
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef PCIE_OOB
+extern void dhd_oob_set_bt_reg_on(struct dhd_bus *bus, bool val);
+extern int dhd_oob_get_bt_reg_on(struct dhd_bus *bus);
+extern void dhdpcie_oob_init(dhd_bus_t *bus);
+extern int dhd_os_oob_set_device_wake(struct dhd_bus *bus, bool val);
+extern void dhd_os_ib_set_device_wake(struct dhd_bus *bus, bool val);
+#endif /* PCIE_OOB */
+#if defined(PCIE_OOB) || defined(PCIE_INB_DW)
+extern void dhd_bus_doorbell_timeout_reset(struct dhd_bus *bus);
+#endif /* defined(PCIE_OOB) || defined(PCIE_INB_DW) */
+
+#define DHD_REGULAR_RING    0
+#define DHD_HP2P_RING    1
+
+#ifdef DHD_SET_PCIE_DMA_MASK_FOR_GS101
+/* This is only for GS101 platform. Others is done on RC side */
+#define DHD_PCIE_DMA_MASK_FOR_GS101 36
+#endif /* DHD_SET_PCIE_DMA_MASK_FOR_GS101 */
+
+#ifdef CONFIG_ARCH_TEGRA
+extern int tegra_pcie_pm_suspend(void);
+extern int tegra_pcie_pm_resume(void);
+#endif /* CONFIG_ARCH_TEGRA */
+
+extern int dhd_buzzz_dump_dngl(dhd_bus_t *bus);
+#ifdef IDLE_TX_FLOW_MGMT
+extern int dhd_bus_flow_ring_resume_request(struct dhd_bus *bus, void *arg);
+extern void dhd_bus_flow_ring_resume_response(struct dhd_bus *bus, uint16 flowid, int32 status);
+extern int dhd_bus_flow_ring_suspend_request(struct dhd_bus *bus, void *arg);
+extern void dhd_bus_flow_ring_suspend_response(struct dhd_bus *bus, uint16 flowid, uint32 status);
+extern void dhd_flow_ring_move_to_active_list_head(struct dhd_bus *bus,
+	flow_ring_node_t *flow_ring_node);
+extern void dhd_flow_ring_add_to_active_list(struct dhd_bus *bus,
+	flow_ring_node_t *flow_ring_node);
+extern void dhd_flow_ring_delete_from_active_list(struct dhd_bus *bus,
+	flow_ring_node_t *flow_ring_node);
+extern void __dhd_flow_ring_delete_from_active_list(struct dhd_bus *bus,
+	flow_ring_node_t *flow_ring_node);
+#endif /* IDLE_TX_FLOW_MGMT */
+
+extern int dhdpcie_send_mb_data(dhd_bus_t *bus, uint32 h2d_mb_data);
+
+#ifdef DHD_WAKE_STATUS
+int bcmpcie_get_total_wake(struct dhd_bus *bus);
+int bcmpcie_set_get_wake(struct dhd_bus *bus, int flag);
+int bcmpcie_get_wake(struct dhd_bus *bus);
+int bcmpcie_set_get_wake_pkt_dump(struct dhd_bus *bus, int wake_pkt_dump);
+#endif /* DHD_WAKE_STATUS */
+#ifdef DHD_MMIO_TRACE
+extern void dhd_dump_bus_mmio_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf);
+#endif /* defined(DHD_MMIO_TRACE) */
+extern void dhd_dump_bus_ds_trace(dhd_bus_t *bus, struct bcmstrbuf *strbuf);
+extern bool dhdpcie_bus_get_pcie_hostready_supported(dhd_bus_t *bus);
+extern void dhd_bus_hostready(struct  dhd_bus *bus);
+#ifdef PCIE_OOB
+extern bool dhdpcie_bus_get_pcie_oob_dw_supported(dhd_bus_t *bus);
+#endif /* PCIE_OOB */
+#ifdef PCIE_INB_DW
+extern bool dhdpcie_bus_get_pcie_inband_dw_supported(dhd_bus_t *bus);
+extern void dhdpcie_bus_set_pcie_inband_dw_state(dhd_bus_t *bus,
+	enum dhd_bus_ds_state state);
+extern enum dhd_bus_ds_state dhdpcie_bus_get_pcie_inband_dw_state(dhd_bus_t *bus);
+extern const char * dhd_convert_inb_state_names(enum dhd_bus_ds_state inbstate);
+extern const char * dhd_convert_dsval(uint32 val, bool d2h);
+extern int dhd_bus_inb_set_device_wake(struct dhd_bus *bus, bool val);
+#ifdef PCIE_INB_DSACK_EXT_WAIT
+extern void dhd_bus_ds_ack_debug_dump(struct dhd_bus *bus);
+#endif /* PCIE_INB_DSACK_EXT_WAIT */
+extern void dhd_bus_inb_ack_pending_ds_req(dhd_bus_t *bus);
+#endif /* PCIE_INB_DW */
+extern void dhdpcie_bus_enab_pcie_dw(dhd_bus_t *bus, uint8 dw_option);
+#if defined(LINUX) || defined(linux)
+extern int dhdpcie_irq_disabled(struct dhd_bus *bus);
+extern int dhdpcie_set_master_and_d0_pwrstate(struct dhd_bus *bus);
+#else
+static INLINE bool dhdpcie_irq_disabled(struct dhd_bus *bus) { return BCME_ERROR;}
+static INLINE int dhdpcie_set_master_and_d0_pwrstate(struct dhd_bus *bus)
+{ return BCME_ERROR;}
+#endif /* defined(LINUX) || defined(linux) */
+
+static INLINE bool dhdpcie_is_arm_halted(struct dhd_bus *bus) {return TRUE;}
+static INLINE int dhd_os_wifi_platform_set_power(uint32 value) {return BCME_OK; }
+static INLINE void
+dhdpcie_dongle_flr_or_pwr_toggle(dhd_bus_t *bus)
+{ return; }
+
+int dhdpcie_config_check(dhd_bus_t *bus);
+int dhdpcie_config_restore(dhd_bus_t *bus, bool restore_pmcsr);
+int dhdpcie_config_save(dhd_bus_t *bus);
+int dhdpcie_set_pwr_state(dhd_bus_t *bus, uint state);
+
+extern bool dhdpcie_bus_get_pcie_hwa_supported(dhd_bus_t *bus);
+extern bool dhdpcie_bus_get_pcie_idma_supported(dhd_bus_t *bus);
+extern bool dhdpcie_bus_get_pcie_ifrm_supported(dhd_bus_t *bus);
+extern bool dhdpcie_bus_get_pcie_dar_supported(dhd_bus_t *bus);
+extern bool dhdpcie_bus_get_hp2p_supported(dhd_bus_t *bus);
+
+static INLINE uint32
+dhd_pcie_config_read(dhd_bus_t *bus, uint offset, uint size)
+{
+	/* For 4375 or prior chips to 4375 */
+	if (bus->sih && bus->sih->buscorerev <= 64) {
+		OSL_DELAY(100);
+	}
+	return OSL_PCI_READ_CONFIG(bus->osh, offset, size);
+}
+
+static INLINE uint32
+dhd_pcie_corereg_read(si_t *sih, uint val)
+{
+	/* For 4375 or prior chips to 4375 */
+	if (sih->buscorerev <= 64) {
+		OSL_DELAY(100);
+	}
+	si_corereg(sih, sih->buscoreidx, OFFSETOF(sbpcieregs_t, configaddr), ~0, val);
+	return si_corereg(sih, sih->buscoreidx, OFFSETOF(sbpcieregs_t, configdata), 0, 0);
+}
+
+extern int dhdpcie_get_fwpath_otp(dhd_bus_t *bus, char *fw_path, char *nv_path,
+		char *clm_path, char *txcap_path);
+
+extern int dhd_pcie_debug_info_dump(dhd_pub_t *dhd);
+extern void dhd_pcie_intr_count_dump(dhd_pub_t *dhd);
+extern void dhdpcie_bus_clear_intstatus(dhd_bus_t *bus);
+
+#if defined(NDIS)
+extern int dhd_get_platform(dhd_pub_t* dhd, char *progname);
+extern bool dhdpcie_is_chip_supported(uint32 chipid, int *idx);
+#endif
+
+extern int dhd_get_pcie_linkspeed(dhd_pub_t *dhd);
+extern void dhdpcie_bar1_window_switch_enab(dhd_bus_t *bus);
+
+#ifdef PCIE_INB_DW
+extern void dhdpcie_set_dongle_deepsleep(dhd_bus_t *bus, bool val);
+extern void dhd_init_dongle_ds_lock(dhd_bus_t *bus);
+extern void dhd_deinit_dongle_ds_lock(dhd_bus_t *bus);
+#endif /* PCIE_INB_DW */
+
+extern bool dhd_check_htput_chip(dhd_bus_t *bus);
+
+#if defined(DEVICE_TX_STUCK_DETECT) && defined(ASSOC_CHECK_SR)
+void dhd_assoc_check_sr(dhd_pub_t *dhd, bool state);
+#endif /* DEVICE_TX_STUCK_DETECT && ASSOC_CHECK_SR */
+
+void dhd_init_bar1_switch_lock(dhd_bus_t *bus);
+#endif /* dhd_pcie_h */
diff --git a/drivers/net/wireless/ap12275/dhd_pcie_linux.c b/drivers/net/wireless/ap12275/dhd_pcie_linux.c
new file mode 100644
index 000000000000..9870ab28137d
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pcie_linux.c
@@ -0,0 +1,3114 @@
+/*
+ * Linux DHD Bus Module for PCIE
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+/* include files */
+#include <typedefs.h>
+#include <bcmutils.h>
+#include <bcmdevs.h>
+#include <bcmdevs_legacy.h>    /* need to still support chips no longer in trunk firmware */
+#include <siutils.h>
+#include <hndsoc.h>
+#include <hndpmu.h>
+#include <sbchipc.h>
+#if defined(DHD_DEBUG)
+#include <hnd_armtrap.h>
+#include <hnd_cons.h>
+#endif /* defined(DHD_DEBUG) */
+#include <dngl_stats.h>
+#include <pcie_core.h>
+#include <dhd.h>
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <dhd_dbg.h>
+#include <dhdioctl.h>
+#include <bcmmsgbuf.h>
+#include <pcicfg.h>
+#include <dhd_pcie.h>
+#include <dhd_linux.h>
+#ifdef OEM_ANDROID
+#ifdef CONFIG_ARCH_MSM
+#if IS_ENABLED(CONFIG_PCI_MSM) || defined(CONFIG_ARCH_MSM8996)
+#include <linux/msm_pcie.h>
+#else
+#include <mach/msm_pcie.h>
+#endif /* CONFIG_PCI_MSM */
+#endif /* CONFIG_ARCH_MSM */
+#endif /* OEM_ANDROID */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+#include <linux/pm_runtime.h>
+
+#ifndef AUTO_SUSPEND_TIMEOUT
+#define AUTO_SUSPEND_TIMEOUT 1000
+#endif /* AUTO_SUSPEND_TIMEOUT */
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#include <linux/irq.h>
+#ifdef USE_SMMU_ARCH_MSM
+#include <asm/dma-iommu.h>
+#include <linux/iommu.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#endif /* USE_SMMU_ARCH_MSM */
+#include <dhd_config.h>
+
+#ifdef PCIE_OOB
+#include "ftdi_sio_external.h"
+#endif /* PCIE_OOB */
+
+#if defined(WL_CFG80211)
+#include <wl_cfg80211.h>
+#endif	/* WL_CFG80211 */
+
+#include <dhd_plat.h>
+
+#define PCI_CFG_RETRY		10		/* PR15065: retry count for pci cfg accesses */
+#define OS_HANDLE_MAGIC		0x1234abcd	/* Magic # to recognize osh */
+#define BCM_MEM_FILENAME_LEN	24		/* Mem. filename length */
+
+#ifdef PCIE_OOB
+#define HOST_WAKE 4   /* GPIO_0 (HOST_WAKE) - Output from WLAN */
+#define DEVICE_WAKE 5  /* GPIO_1 (DEVICE_WAKE) - Input to WLAN */
+#define BIT_WL_REG_ON 6
+#define BIT_BT_REG_ON 7
+
+int gpio_handle_val = 0;
+unsigned char gpio_port = 0;
+unsigned char gpio_direction = 0;
+#define OOB_PORT "ttyUSB0"
+#endif /* PCIE_OOB */
+
+#ifndef BCMPCI_DEV_ID
+#define BCMPCI_DEV_ID PCI_ANY_ID
+#endif
+
+#ifndef SYNAPCI_DEV_ID
+#define SYNAPCI_DEV_ID PCI_ANY_ID
+#endif
+
+#ifdef FORCE_TPOWERON
+extern uint32 tpoweron_scale;
+#endif /* FORCE_TPOWERON */
+/* user defined data structures  */
+
+typedef bool (*dhdpcie_cb_fn_t)(void *);
+
+typedef struct dhdpcie_info
+{
+	dhd_bus_t	*bus;
+	osl_t		*osh;
+	struct pci_dev  *dev;		/* pci device handle */
+	volatile char	*regs;		/* pci device memory va */
+	volatile char	*tcm;		/* pci device memory va */
+	uint32		bar1_size;	/* pci device memory size */
+	struct pcos_info *pcos_info;
+	uint16		last_intrstatus;	/* to cache intrstatus */
+	int	irq;
+	char pciname[32];
+	struct pci_saved_state* default_state;
+	struct pci_saved_state* state;
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	void *os_cxt;			/* Pointer to per-OS private data */
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef DHD_WAKE_STATUS
+	spinlock_t	pkt_wake_lock;
+	unsigned int	total_wake_count;
+	int		pkt_wake;
+	int		wake_irq;
+	int		pkt_wake_dump;
+#endif /* DHD_WAKE_STATUS */
+#ifdef USE_SMMU_ARCH_MSM
+	void *smmu_cxt;
+#endif /* USE_SMMU_ARCH_MSM */
+} dhdpcie_info_t;
+
+struct pcos_info {
+	dhdpcie_info_t *pc;
+	spinlock_t lock;
+	wait_queue_head_t intr_wait_queue;
+	timer_list_compat_t tuning_timer;
+	int tuning_timer_exp;
+	atomic_t timer_enab;
+	struct tasklet_struct tuning_tasklet;
+};
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+typedef struct dhdpcie_os_info {
+	int			oob_irq_num;	/* valid when hardware or software oob in use */
+	unsigned long		oob_irq_flags;	/* valid when hardware or software oob in use */
+	bool			oob_irq_registered;
+	bool			oob_irq_enabled;
+	bool			oob_irq_wake_enabled;
+	spinlock_t		oob_irq_spinlock;
+	void			*dev;		/* handle to the underlying device */
+} dhdpcie_os_info_t;
+static irqreturn_t wlan_oob_irq(int irq, void *data);
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef USE_SMMU_ARCH_MSM
+typedef struct dhdpcie_smmu_info {
+	struct dma_iommu_mapping *smmu_mapping;
+	dma_addr_t smmu_iova_start;
+	size_t smmu_iova_len;
+} dhdpcie_smmu_info_t;
+#endif /* USE_SMMU_ARCH_MSM */
+
+/* function declarations */
+static int __devinit
+dhdpcie_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
+static void __devexit dhdpcie_pci_remove(struct pci_dev *pdev);
+static void __devexit dhdpcie_pci_shutdown(struct pci_dev *pdev);
+static void __devexit dhdpcie_pci_stop(struct pci_dev *pdev);
+static int dhdpcie_init(struct pci_dev *pdev);
+static irqreturn_t dhdpcie_isr(int irq, void *arg);
+/* OS Routine functions for PCI suspend/resume */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+static int dhdpcie_set_suspend_resume(struct pci_dev *dev, bool state, bool byint);
+#else
+static int dhdpcie_set_suspend_resume(dhd_bus_t *bus, bool state);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+static int dhdpcie_resume_host_dev(dhd_bus_t *bus);
+static int dhdpcie_suspend_host_dev(dhd_bus_t *bus);
+static int dhdpcie_resume_dev(struct pci_dev *dev);
+static int dhdpcie_suspend_dev(struct pci_dev *dev);
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+static int dhdpcie_pm_system_suspend_noirq(struct device * dev);
+static int dhdpcie_pm_system_resume_noirq(struct device * dev);
+#else
+static int dhdpcie_pci_suspend(struct device * dev);
+static int dhdpcie_pci_resume(struct device * dev);
+static int dhdpcie_pci_resume_early(struct device * dev);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+static int dhdpcie_pm_runtime_suspend(struct device * dev);
+static int dhdpcie_pm_runtime_resume(struct device * dev);
+static int dhdpcie_pm_system_suspend_noirq(struct device * dev);
+static int dhdpcie_pm_system_resume_noirq(struct device * dev);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+static void dhdpcie_config_save_restore_coherent(dhd_bus_t *bus, bool state);
+
+uint32
+dhdpcie_access_cap(struct pci_dev *pdev, int cap, uint offset, bool is_ext, bool is_write,
+	uint32 writeval);
+
+static struct pci_device_id dhdpcie_pci_devid[] __devinitdata = {
+	{ vendor: VENDOR_BROADCOM,
+	device: BCMPCI_DEV_ID,
+	subvendor: PCI_ANY_ID,
+	subdevice: PCI_ANY_ID,
+	class: PCI_CLASS_NETWORK_OTHER << 8,
+	class_mask: 0xffff00,
+	driver_data: 0,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+	override_only: 0,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) */
+	},
+	{ vendor: VENDOR_SYNAPTICS,
+	device: BCMPCI_DEV_ID,
+	subvendor: PCI_ANY_ID,
+	subdevice: PCI_ANY_ID,
+	class: PCI_CLASS_NETWORK_OTHER << 8,
+	class_mask: 0xffff00,
+	driver_data: 0,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+	override_only: 0,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) */
+	},
+#if (BCMPCI_DEV_ID != PCI_ANY_ID) && defined(BCMPCI_NOOTP_DEV_ID)
+	{ vendor: VENDOR_BROADCOM,
+	device: BCMPCI_NOOTP_DEV_ID,
+	subvendor: PCI_ANY_ID,
+	subdevice: PCI_ANY_ID,
+	class: PCI_CLASS_NETWORK_OTHER << 8,
+	class_mask: 0xffff00,
+	driver_data: 0,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+	override_only: 0,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) */
+	},
+#endif /* BCMPCI_DEV_ID != PCI_ANY_ID && BCMPCI_NOOTP_DEV_ID */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+	{ 0, 0, 0, 0, 0, 0, 0, 0}
+#else
+	{ 0, 0, 0, 0, 0, 0, 0}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) */
+};
+MODULE_DEVICE_TABLE(pci, dhdpcie_pci_devid);
+
+/* Power Management Hooks */
+#if defined(DHD_PCIE_NATIVE_RUNTIMEPM)
+static const struct dev_pm_ops dhdpcie_pm_ops = {
+	SET_RUNTIME_PM_OPS(dhdpcie_pm_runtime_suspend, dhdpcie_pm_runtime_resume, NULL)
+	.suspend_noirq = dhdpcie_pm_system_suspend_noirq,
+	.resume_noirq = dhdpcie_pm_system_resume_noirq
+};
+#else
+/* for linux platforms */
+static const struct dev_pm_ops dhd_pcie_pm_ops = {
+	.suspend = dhdpcie_pci_suspend,
+	.resume	= dhdpcie_pci_resume,
+	.resume_early = dhdpcie_pci_resume_early,
+};
+#endif
+
+static struct pci_driver dhdpcie_driver = {
+	node:		{&dhdpcie_driver.node, &dhdpcie_driver.node},
+	name:		"pcieh"BUS_TYPE,
+	id_table:	dhdpcie_pci_devid,
+	probe:		dhdpcie_pci_probe,
+	remove:		dhdpcie_pci_remove,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 0))
+	.driver = {.pm = &dhd_pcie_pm_ops, .coredump = NULL},
+#else
+	.driver = {.pm = &dhd_pcie_pm_ops, },
+#endif /* LINUX_VERSION_CODE >= 4.16.0 */
+	shutdown:	dhdpcie_pci_shutdown,
+};
+
+int dhdpcie_init_succeeded = FALSE;
+
+#ifdef USE_SMMU_ARCH_MSM
+static int dhdpcie_smmu_init(struct pci_dev *pdev, void *smmu_cxt)
+{
+	struct dma_iommu_mapping *mapping;
+	struct device_node *root_node = NULL;
+	dhdpcie_smmu_info_t *smmu_info = (dhdpcie_smmu_info_t *)smmu_cxt;
+	int smmu_iova_address[2];
+	char *wlan_node = "android,bcmdhd_wlan";
+	char *wlan_smmu_node = "wlan-smmu-iova-address";
+	int atomic_ctx = 1;
+	int s1_bypass = 1;
+	int ret = 0;
+
+	DHD_ERROR(("%s: SMMU initialize\n", __FUNCTION__));
+
+	root_node = of_find_compatible_node(NULL, NULL, wlan_node);
+	if (!root_node) {
+		WARN(1, "failed to get device node of BRCM WLAN\n");
+		return -ENODEV;
+	}
+
+	if (of_property_read_u32_array(root_node, wlan_smmu_node,
+		smmu_iova_address, 2) == 0) {
+		DHD_ERROR(("%s : get SMMU start address 0x%x, size 0x%x\n",
+			__FUNCTION__, smmu_iova_address[0], smmu_iova_address[1]));
+		smmu_info->smmu_iova_start = smmu_iova_address[0];
+		smmu_info->smmu_iova_len = smmu_iova_address[1];
+	} else {
+		DHD_CONS_ONLY(("%s : can't get smmu iova address property\n",
+			__FUNCTION__));
+		return -ENODEV;
+	}
+
+	if (smmu_info->smmu_iova_len <= 0) {
+		DHD_ERROR(("%s: Invalid smmu iova len %d\n",
+			__FUNCTION__, (int)smmu_info->smmu_iova_len));
+		return -EINVAL;
+	}
+
+	DHD_ERROR(("%s : SMMU init start\n", __FUNCTION__));
+
+	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) ||
+		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
+		DHD_ERROR(("%s: DMA set 64bit mask failed.\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	mapping = arm_iommu_create_mapping(&platform_bus_type,
+		smmu_info->smmu_iova_start, smmu_info->smmu_iova_len);
+	if (IS_ERR(mapping)) {
+		DHD_ERROR(("%s: create mapping failed, err = %d\n",
+			__FUNCTION__, ret));
+		ret = PTR_ERR(mapping);
+		goto map_fail;
+	}
+
+	ret = iommu_domain_set_attr(mapping->domain,
+		DOMAIN_ATTR_ATOMIC, &atomic_ctx);
+	if (ret) {
+		DHD_ERROR(("%s: set atomic_ctx attribute failed, err = %d\n",
+			__FUNCTION__, ret));
+		goto set_attr_fail;
+	}
+
+	ret = iommu_domain_set_attr(mapping->domain,
+		DOMAIN_ATTR_S1_BYPASS, &s1_bypass);
+	if (ret < 0) {
+		DHD_ERROR(("%s: set s1_bypass attribute failed, err = %d\n",
+			__FUNCTION__, ret));
+		goto set_attr_fail;
+	}
+
+	ret = arm_iommu_attach_device(&pdev->dev, mapping);
+	if (ret) {
+		DHD_ERROR(("%s: attach device failed, err = %d\n",
+			__FUNCTION__, ret));
+		goto attach_fail;
+	}
+
+	smmu_info->smmu_mapping = mapping;
+
+	return ret;
+
+attach_fail:
+set_attr_fail:
+	arm_iommu_release_mapping(mapping);
+map_fail:
+	return ret;
+}
+
+static void dhdpcie_smmu_remove(struct pci_dev *pdev, void *smmu_cxt)
+{
+	dhdpcie_smmu_info_t *smmu_info;
+
+	if (!smmu_cxt) {
+		return;
+	}
+
+	smmu_info = (dhdpcie_smmu_info_t *)smmu_cxt;
+	if (smmu_info->smmu_mapping) {
+		arm_iommu_detach_device(&pdev->dev);
+		arm_iommu_release_mapping(smmu_info->smmu_mapping);
+		smmu_info->smmu_mapping = NULL;
+	}
+}
+#endif /* USE_SMMU_ARCH_MSM */
+
+#ifdef FORCE_TPOWERON
+static void
+dhd_bus_get_tpoweron(dhd_bus_t *bus)
+{
+
+	uint32 tpoweron_rc;
+	uint32 tpoweron_ep;
+
+	tpoweron_rc = dhdpcie_rc_access_cap(bus, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL2_OFFSET, TRUE, FALSE, 0);
+	tpoweron_ep = dhdpcie_ep_access_cap(bus, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL2_OFFSET, TRUE, FALSE, 0);
+	DHD_ERROR(("%s: tpoweron_rc:0x%x tpoweron_ep:0x%x\n",
+		__FUNCTION__, tpoweron_rc, tpoweron_ep));
+}
+
+static void
+dhd_bus_set_tpoweron(dhd_bus_t *bus, uint16 tpoweron)
+{
+
+	dhd_bus_get_tpoweron(bus);
+	/* Set the tpoweron */
+	DHD_ERROR(("%s tpoweron: 0x%x\n", __FUNCTION__, tpoweron));
+	dhdpcie_rc_access_cap(bus, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL2_OFFSET, TRUE, TRUE, tpoweron);
+	dhdpcie_ep_access_cap(bus, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL2_OFFSET, TRUE, TRUE, tpoweron);
+
+	dhd_bus_get_tpoweron(bus);
+
+}
+
+static bool
+dhdpcie_chip_req_forced_tpoweron(dhd_bus_t *bus)
+{
+	/*
+	 * On Fire's reference platform, coming out of L1.2,
+	 * there is a constant delay of 45us between CLKREQ# and stable REFCLK
+	 * Due to this delay, with tPowerOn < 50
+	 * there is a chance of the refclk sense to trigger on noise.
+	 *
+	 * Which ever chip needs forced tPowerOn of 50us should be listed below.
+	 */
+	if (si_chipid(bus->sih) == BCM4377_CHIP_ID) {
+		return TRUE;
+	}
+	return FALSE;
+}
+#endif /* FORCE_TPOWERON */
+
+static bool
+dhd_bus_aspm_enable_dev(dhd_bus_t *bus, struct pci_dev *dev, bool enable)
+{
+	uint32 linkctrl_before;
+	uint32 linkctrl_after = 0;
+	uint8 linkctrl_aspm;
+	char *device;
+
+	device = (dev == bus->dev) ? "EP" : "RC";
+
+	linkctrl_before = dhdpcie_access_cap(dev, PCIE_CAP_ID_EXP, PCIE_CAP_LINKCTRL_OFFSET,
+		FALSE, FALSE, 0);
+	linkctrl_aspm = (linkctrl_before & PCIE_ASPM_CTRL_MASK);
+
+	if (enable) {
+		if (linkctrl_aspm == PCIE_ASPM_L1_ENAB) {
+			DHD_ERROR(("%s: %s already enabled  linkctrl: 0x%x\n",
+				__FUNCTION__, device, linkctrl_before));
+			return FALSE;
+		}
+		/* Enable only L1 ASPM (bit 1) */
+		dhdpcie_access_cap(dev, PCIE_CAP_ID_EXP, PCIE_CAP_LINKCTRL_OFFSET, FALSE,
+			TRUE, (linkctrl_before | PCIE_ASPM_L1_ENAB));
+	} else {
+		if (linkctrl_aspm == 0) {
+			DHD_ERROR(("%s: %s already disabled linkctrl: 0x%x\n",
+				__FUNCTION__, device, linkctrl_before));
+			return FALSE;
+		}
+		/* Disable complete ASPM (bit 1 and bit 0) */
+		dhdpcie_access_cap(dev, PCIE_CAP_ID_EXP, PCIE_CAP_LINKCTRL_OFFSET, FALSE,
+			TRUE, (linkctrl_before & (~PCIE_ASPM_ENAB)));
+	}
+
+	linkctrl_after = dhdpcie_access_cap(dev, PCIE_CAP_ID_EXP, PCIE_CAP_LINKCTRL_OFFSET,
+		FALSE, FALSE, 0);
+	DHD_ERROR(("%s: %s %s, linkctrl_before: 0x%x linkctrl_after: 0x%x\n",
+		__FUNCTION__, device, (enable ? "ENABLE " : "DISABLE"),
+		linkctrl_before, linkctrl_after));
+
+	return TRUE;
+}
+
+static bool
+dhd_bus_is_aspm_enab_dev(dhd_bus_t *bus, struct pci_dev *dev)
+{
+	uint32 linkctrl = dhdpcie_access_cap(dev, PCIE_CAP_ID_EXP,
+		PCIE_CAP_LINKCTRL_OFFSET, FALSE, FALSE, 0);
+
+	DHD_INFO(("%s: %s: linkctrl:0x%x\n",
+		__FUNCTION__, (dev == bus->dev) ? "EP" : "RC", linkctrl));
+
+	return ((linkctrl & PCIE_ASPM_CTRL_MASK) == PCIE_ASPM_L1_ENAB);
+}
+
+bool
+dhd_bus_is_aspm_enab_rc_ep(dhd_bus_t *bus)
+{
+	uint32 rc_aspm_enab;
+	uint32 ep_aspm_enab;
+
+	rc_aspm_enab = dhd_bus_is_aspm_enab_dev(bus, bus->rc_dev);
+	ep_aspm_enab = dhd_bus_is_aspm_enab_dev(bus, bus->dev);
+
+	return (rc_aspm_enab && ep_aspm_enab);
+}
+
+static bool
+dhd_bus_is_rc_ep_aspm_capable(dhd_bus_t *bus)
+{
+	uint32 rc_aspm_cap;
+	uint32 ep_aspm_cap;
+
+	/* RC ASPM capability */
+	rc_aspm_cap = dhdpcie_access_cap(bus->rc_dev, PCIE_CAP_ID_EXP, PCIE_CAP_LINKCTRL_OFFSET,
+		FALSE, FALSE, 0);
+	if (rc_aspm_cap == BCME_ERROR) {
+		DHD_ERROR(("%s RC is not ASPM capable\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	/* EP ASPM capability */
+	ep_aspm_cap = dhdpcie_access_cap(bus->dev, PCIE_CAP_ID_EXP, PCIE_CAP_LINKCTRL_OFFSET,
+		FALSE, FALSE, 0);
+	if (ep_aspm_cap == BCME_ERROR) {
+		DHD_ERROR(("%s EP is not ASPM capable\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+bool
+dhd_bus_aspm_enable_rc_ep(dhd_bus_t *bus, bool enable)
+{
+	bool ret;
+
+	if (!bus->rc_ep_aspm_cap) {
+		DHD_ERROR(("%s: NOT ASPM  CAPABLE rc_ep_aspm_cap: %d\n",
+			__FUNCTION__, bus->rc_ep_aspm_cap));
+		return FALSE;
+	}
+
+	if (enable) {
+		/* Enable only L1 ASPM first RC then EP */
+		ret = dhd_bus_aspm_enable_dev(bus, bus->rc_dev, enable);
+		ret = dhd_bus_aspm_enable_dev(bus, bus->dev, enable);
+	} else {
+		/* Disable complete ASPM first EP then RC */
+		ret = dhd_bus_aspm_enable_dev(bus, bus->dev, enable);
+		ret = dhd_bus_aspm_enable_dev(bus, bus->rc_dev, enable);
+	}
+
+	return ret;
+}
+
+static bool
+dhd_bus_is_l1ss_enab_dev(dhd_bus_t *bus, struct pci_dev *dev)
+{
+	uint32 l1ssctrl;
+
+	/* Extendend Capacility Reg */
+	l1ssctrl = dhdpcie_access_cap(dev, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL_OFFSET, TRUE, FALSE, 0);
+
+	DHD_INFO(("%s: %s: l1ssctrl:0x%x\n",
+		__FUNCTION__, (dev == bus->dev) ? "EP" : "RC", l1ssctrl));
+
+	return ((l1ssctrl & PCIE_EXT_L1SS_MASK) == PCIE_EXT_L1SS_ENAB);
+}
+
+bool
+dhd_bus_is_l1ss_enab_rc_ep(dhd_bus_t *bus)
+{
+	uint32 rc_l1ss_enab;
+	uint32 ep_l1ss_enab;
+
+	rc_l1ss_enab = dhd_bus_is_l1ss_enab_dev(bus, bus->rc_dev);
+	ep_l1ss_enab = dhd_bus_is_l1ss_enab_dev(bus, bus->dev);
+
+	return (rc_l1ss_enab && ep_l1ss_enab);
+}
+
+static void
+dhd_bus_l1ss_enable_dev(dhd_bus_t *bus, struct pci_dev *dev, bool enable)
+{
+	uint32 l1ssctrl_before;
+	uint32 l1ssctrl_after = 0;
+	uint8 l1ss_ep;
+	char *device;
+
+	device = (dev == bus->dev) ? "EP" : "RC";
+
+	/* Extendend Capacility Reg */
+	l1ssctrl_before = dhdpcie_access_cap(dev, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL_OFFSET, TRUE, FALSE, 0);
+	l1ss_ep = (l1ssctrl_before & PCIE_EXT_L1SS_MASK);
+
+	if (enable) {
+		if (l1ss_ep == PCIE_EXT_L1SS_ENAB) {
+			DHD_ERROR(("%s: %s already enabled,  l1ssctrl: 0x%x\n",
+				__FUNCTION__, device, l1ssctrl_before));
+			return;
+		}
+		dhdpcie_access_cap(dev, PCIE_EXTCAP_ID_L1SS, PCIE_EXTCAP_L1SS_CONTROL_OFFSET,
+			TRUE, TRUE, (l1ssctrl_before | PCIE_EXT_L1SS_ENAB));
+	} else {
+		if (l1ss_ep == 0) {
+			DHD_ERROR(("%s: %s already disabled, l1ssctrl: 0x%x\n",
+				__FUNCTION__, device, l1ssctrl_before));
+			return;
+		}
+		dhdpcie_access_cap(dev, PCIE_EXTCAP_ID_L1SS, PCIE_EXTCAP_L1SS_CONTROL_OFFSET,
+			TRUE, TRUE, (l1ssctrl_before & (~PCIE_EXT_L1SS_ENAB)));
+	}
+	l1ssctrl_after = dhdpcie_access_cap(dev, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL_OFFSET, TRUE, FALSE, 0);
+	DHD_ERROR(("%s: %s %s, l1ssctrl_before: 0x%x l1ssctrl_after: 0x%x\n",
+		__FUNCTION__, device, (enable ? "ENABLE " : "DISABLE"),
+		l1ssctrl_before, l1ssctrl_after));
+
+}
+
+static bool
+dhd_bus_is_rc_ep_l1ss_capable(dhd_bus_t *bus)
+{
+	uint32 rc_l1ss_cap;
+	uint32 ep_l1ss_cap;
+
+	/* RC Extendend Capacility */
+	rc_l1ss_cap = dhdpcie_access_cap(bus->rc_dev, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL_OFFSET, TRUE, FALSE, 0);
+	if (rc_l1ss_cap == BCME_ERROR) {
+		DHD_ERROR(("%s RC is not l1ss capable\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	/* EP Extendend Capacility */
+	ep_l1ss_cap = dhdpcie_access_cap(bus->dev, PCIE_EXTCAP_ID_L1SS,
+		PCIE_EXTCAP_L1SS_CONTROL_OFFSET, TRUE, FALSE, 0);
+	if (ep_l1ss_cap == BCME_ERROR) {
+		DHD_ERROR(("%s EP is not l1ss capable\n", __FUNCTION__));
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+void
+dhd_bus_l1ss_enable_rc_ep(dhd_bus_t *bus, bool enable)
+{
+	bool ret;
+
+	if ((!bus->rc_ep_aspm_cap) || (!bus->rc_ep_l1ss_cap)) {
+		DHD_ERROR(("%s: NOT L1SS CAPABLE rc_ep_aspm_cap: %d rc_ep_l1ss_cap: %d\n",
+			__FUNCTION__, bus->rc_ep_aspm_cap, bus->rc_ep_l1ss_cap));
+		return;
+	}
+
+	/* Disable ASPM of RC and EP */
+	ret = dhd_bus_aspm_enable_rc_ep(bus, FALSE);
+
+	if (enable) {
+		/* Enable RC then EP */
+		dhd_bus_l1ss_enable_dev(bus, bus->rc_dev, enable);
+		dhd_bus_l1ss_enable_dev(bus, bus->dev, enable);
+	} else {
+		/* Disable EP then RC */
+		dhd_bus_l1ss_enable_dev(bus, bus->dev, enable);
+		dhd_bus_l1ss_enable_dev(bus, bus->rc_dev, enable);
+	}
+
+	/* Enable ASPM of RC and EP only if this API disabled */
+	if (ret == TRUE) {
+		dhd_bus_aspm_enable_rc_ep(bus, TRUE);
+	}
+}
+
+void
+dhd_bus_aer_config(dhd_bus_t *bus)
+{
+	uint32 val;
+
+	DHD_ERROR_MEM(("%s: Configure AER registers for EP\n", __FUNCTION__));
+	val = dhdpcie_ep_access_cap(bus, PCIE_ADVERRREP_CAPID,
+		PCIE_ADV_CORR_ERR_MASK_OFFSET, TRUE, FALSE, 0);
+	if (val != (uint32)-1) {
+		val &= ~CORR_ERR_AE;
+		dhdpcie_ep_access_cap(bus, PCIE_ADVERRREP_CAPID,
+			PCIE_ADV_CORR_ERR_MASK_OFFSET, TRUE, TRUE, val);
+	} else {
+		DHD_ERROR(("%s: Invalid EP's PCIE_ADV_CORR_ERR_MASK: 0x%x\n",
+			__FUNCTION__, val));
+	}
+
+	DHD_ERROR_MEM(("%s: Configure AER registers for RC\n", __FUNCTION__));
+	val = dhdpcie_rc_access_cap(bus, PCIE_ADVERRREP_CAPID,
+		PCIE_ADV_CORR_ERR_MASK_OFFSET, TRUE, FALSE, 0);
+	if (val != (uint32)-1) {
+		val &= ~CORR_ERR_AE;
+		dhdpcie_rc_access_cap(bus, PCIE_ADVERRREP_CAPID,
+			PCIE_ADV_CORR_ERR_MASK_OFFSET, TRUE, TRUE, val);
+	} else {
+		DHD_ERROR(("%s: Invalid RC's PCIE_ADV_CORR_ERR_MASK: 0x%x\n",
+			__FUNCTION__, val));
+	}
+}
+
+static int dhdpcie_pci_suspend(struct device *dev)
+{
+	int ret = 0;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = NULL;
+	unsigned long flags;
+	int timeleft = 0;
+	uint bitmask = 0xFFFFFFFF;
+
+	printf("%s: Enter\n", __FUNCTION__);
+	if (pch) {
+		bus = pch->bus;
+	}
+	if (!bus) {
+		return ret;
+	}
+
+#if defined(DEVICE_TX_STUCK_DETECT) && defined(ASSOC_CHECK_SR)
+	dhd_assoc_check_sr(bus->dhd, TRUE);
+#endif /* DEVICE_TX_STUCK_DETECT && ASSOC_CHECK_SR */
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	if (!DHD_BUS_BUSY_CHECK_IDLE(bus->dhd)) {
+		DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(bus->dhd);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+		DHD_ERROR(("%s: wait to clear dhd_bus_busy_state: 0x%x\n",
+			__FUNCTION__, bus->dhd->dhd_bus_busy_state));
+		timeleft = dhd_os_busbusy_wait_bitmask(bus->dhd,
+				&bus->dhd->dhd_bus_busy_state, bitmask,
+				DHD_BUS_BUSY_SUSPEND_IN_PROGRESS);
+		if ((timeleft == 0) || (timeleft == 1)) {
+			DHD_ERROR(("%s: Timed out dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, bus->dhd->dhd_bus_busy_state));
+			return -EBUSY;
+		}
+	} else {
+		DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(bus->dhd);
+		DHD_GENERAL_UNLOCK(bus->dhd, flags);
+	}
+
+#ifdef DHD_CFG80211_SUSPEND_RESUME
+	dhd_cfg80211_suspend(bus->dhd);
+#endif /* DHD_CFG80211_SUSPEND_RESUME */
+
+	if (!bus->dhd->dongle_reset)
+		ret = dhdpcie_set_suspend_resume(bus, TRUE);
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_SUSPEND_IN_PROGRESS(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	printf("%s: Exit ret=%d\n", __FUNCTION__, ret);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+	return ret;
+}
+
+static int dhdpcie_pci_resume_early(struct device *dev)
+{
+	int ret = 0;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = NULL;
+	uint32 pmcsr;
+
+	if (pch) {
+		bus = pch->bus;
+	}
+	if (!bus) {
+		return ret;
+	}
+	if (bus->dhd->busstate == DHD_BUS_DOWN) {
+		return ret;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 9))
+	/* On fc30 (linux ver 5.0.9),
+	 *  PMEStat of PMCSR(cfg reg) is cleared before this callback by kernel.
+	 *  So, we use SwPme of FunctionControl(enum reg) instead of PMEStat without kernel change.
+	 */
+	if (bus->sih->buscorerev >= 64) {
+		uint32 ftnctrl;
+		volatile void *regsva = (volatile void *)bus->regs;
+
+		ftnctrl = pcie_corereg(bus->osh, regsva,
+				OFFSETOF(sbpcieregs_t, ftn_ctrl.control), 0, 0);
+		pmcsr = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_PMCSR, sizeof(pmcsr));
+
+		DHD_ERROR(("%s(): pmcsr is 0x%x, ftnctrl is 0x%8x \r\n",
+			__FUNCTION__, pmcsr, ftnctrl));
+		if (ftnctrl & PCIE_FTN_SWPME_MASK) {
+			DHD_ERROR(("%s(): Wakeup due to WLAN \r\n", __FUNCTION__));
+		}
+	} else
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 9)) */
+	{
+		pmcsr = OSL_PCI_READ_CONFIG(bus->osh, PCIE_CFG_PMCSR, sizeof(pmcsr));
+
+		DHD_ERROR(("%s(): pmcsr is 0x%x \r\n", __FUNCTION__, pmcsr));
+		if (pmcsr & PCIE_PMCSR_PMESTAT) {
+			DHD_ERROR(("%s(): Wakeup due to WLAN \r\n", __FUNCTION__));
+		}
+	}
+
+	/*
+	 * TODO: Add code to take adavantage of what is read from pmcsr
+	 */
+
+	return ret;
+}
+
+static int dhdpcie_pci_resume(struct device *dev)
+{
+	int ret = 0;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = NULL;
+	unsigned long flags;
+
+	printf("%s: Enter\n", __FUNCTION__);
+	if (pch) {
+		bus = pch->bus;
+	}
+	if (!bus) {
+		return ret;
+	}
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_SET_RESUME_IN_PROGRESS(bus->dhd);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+
+	if (!bus->dhd->dongle_reset)
+		ret = dhdpcie_set_suspend_resume(bus, FALSE);
+
+	DHD_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_RESUME_IN_PROGRESS(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	printf("%s: Exit ret=%d\n", __FUNCTION__, ret);
+	DHD_GENERAL_UNLOCK(bus->dhd, flags);
+#if defined(DEVICE_TX_STUCK_DETECT) && defined(ASSOC_CHECK_SR)
+	dhd_assoc_check_sr(bus->dhd, FALSE);
+#endif /* DEVICE_TX_STUCK_DETECT && ASSOC_CHECK_SR */
+
+#ifdef WL_TWT
+	dhd_config_twt_event_mask_in_suspend(bus->dhd, FALSE);
+	dhd_send_twt_info_suspend(bus->dhd, FALSE);
+#endif /* WL_TWT */
+
+#ifdef DHD_CFG80211_SUSPEND_RESUME
+	dhd_cfg80211_resume(bus->dhd);
+#endif /* DHD_CFG80211_SUSPEND_RESUME */
+	return ret;
+}
+
+static int
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+dhdpcie_set_suspend_resume(dhd_bus_t *bus, bool state, bool byint)
+#else
+dhdpcie_set_suspend_resume(dhd_bus_t *bus, bool state)
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+{
+	int ret = 0;
+
+	ASSERT(bus && !bus->dhd->dongle_reset);
+
+	/* When firmware is not loaded do the PCI bus */
+	/* suspend/resume only */
+	if (bus->dhd->busstate == DHD_BUS_DOWN) {
+		ret = dhdpcie_pci_suspend_resume(bus, state);
+		return ret;
+	}
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	ret = dhdpcie_bus_suspend(bus, state, byint);
+#else
+	ret = dhdpcie_bus_suspend(bus, state);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) && defined(DHD_TCP_LIMIT_OUTPUT)
+	if (ret == BCME_OK) {
+		/*
+		 * net.ipv4.tcp_limit_output_bytes is used for all ipv4 sockets
+		 * so, returning back to original value when there is no traffic(suspend)
+		 */
+		if (state == TRUE) {
+			dhd_ctrl_tcp_limit_output_bytes(0);
+		} else {
+			dhd_ctrl_tcp_limit_output_bytes(1);
+		}
+	}
+#endif /* LINUX_VERSION_CODE > 4.19.0 && DHD_TCP_LIMIT_OUTPUT */
+
+	return ret;
+}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+static int dhdpcie_pm_runtime_suspend(struct device * dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = NULL;
+	int ret = 0;
+
+	if (!pch)
+		return -EBUSY;
+
+	bus = pch->bus;
+
+	DHD_RPM(("%s Enter\n", __FUNCTION__));
+
+	if (atomic_read(&bus->dhd->block_bus))
+		return -EHOSTDOWN;
+
+	dhd_netif_stop_queue(bus);
+	atomic_set(&bus->dhd->block_bus, TRUE);
+
+	if (dhdpcie_set_suspend_resume(pdev, TRUE, TRUE)) {
+		pm_runtime_mark_last_busy(dev);
+		ret = -EAGAIN;
+	}
+
+	atomic_set(&bus->dhd->block_bus, FALSE);
+	dhd_bus_start_queue(bus);
+
+	return ret;
+}
+
+static int dhdpcie_pm_runtime_resume(struct device * dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = pch->bus;
+
+	DHD_RPM(("%s Enter\n", __FUNCTION__));
+
+	if (atomic_read(&bus->dhd->block_bus))
+		return -EHOSTDOWN;
+
+	if (dhdpcie_set_suspend_resume(pdev, FALSE, TRUE))
+		return -EAGAIN;
+
+	return 0;
+}
+
+static int dhdpcie_pm_system_suspend_noirq(struct device * dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = NULL;
+	int ret;
+
+	DHD_RPM(("%s Enter\n", __FUNCTION__));
+
+	if (!pch)
+		return -EBUSY;
+
+	bus = pch->bus;
+
+	if (atomic_read(&bus->dhd->block_bus))
+		return -EHOSTDOWN;
+
+	dhd_netif_stop_queue(bus);
+	atomic_set(&bus->dhd->block_bus, TRUE);
+
+	ret = dhdpcie_set_suspend_resume(pdev, TRUE, FALSE);
+
+	if (ret) {
+		dhd_bus_start_queue(bus);
+		atomic_set(&bus->dhd->block_bus, FALSE);
+	}
+
+	return ret;
+}
+
+static int dhdpcie_pm_system_resume_noirq(struct device * dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	dhdpcie_info_t *pch = pci_get_drvdata(pdev);
+	dhd_bus_t *bus = NULL;
+	int ret;
+
+	if (!pch)
+		return -EBUSY;
+
+	bus = pch->bus;
+
+	DHD_RPM(("%s Enter\n", __FUNCTION__));
+
+	ret = dhdpcie_set_suspend_resume(pdev, FALSE, FALSE);
+
+	atomic_set(&bus->dhd->block_bus, FALSE);
+	dhd_bus_start_queue(bus);
+	pm_runtime_mark_last_busy(dhd_bus_to_dev(bus));
+
+	return ret;
+}
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if defined(OEM_ANDROID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+extern void dhd_dpc_tasklet_kill(dhd_pub_t *dhdp);
+#endif /* OEM_ANDROID && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+static void
+dhdpcie_suspend_dump_cfgregs(struct dhd_bus *bus, char *suspend_state)
+{
+	DHD_RPM(("%s: BaseAddress0(0x%x)=0x%x, "
+		"BaseAddress1(0x%x)=0x%x PCIE_CFG_PMCSR(0x%x)=0x%x "
+		"PCI_BAR1_WIN(0x%x)=(0x%x)\n",
+		suspend_state,
+		PCIECFGREG_BASEADDR0,
+		dhd_pcie_config_read(bus,
+			PCIECFGREG_BASEADDR0, sizeof(uint32)),
+		PCIECFGREG_BASEADDR1,
+		dhd_pcie_config_read(bus,
+			PCIECFGREG_BASEADDR1, sizeof(uint32)),
+		PCIE_CFG_PMCSR,
+		dhd_pcie_config_read(bus,
+			PCIE_CFG_PMCSR, sizeof(uint32)),
+		PCI_BAR1_WIN,
+		dhd_pcie_config_read(bus,
+			PCI_BAR1_WIN, sizeof(uint32))));
+}
+
+static int dhdpcie_suspend_dev(struct pci_dev *dev)
+{
+	int ret;
+	dhdpcie_info_t *pch = pci_get_drvdata(dev);
+	dhd_bus_t *bus = pch->bus;
+
+#if defined(OEM_ANDROID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	if (bus->is_linkdown) {
+		DHD_ERROR(("%s: PCIe link is down\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+#endif /* OEM_ANDROID && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+	DHD_RPM(("%s: Enter\n", __FUNCTION__));
+
+	/*
+	 * Disable L1ss on EP and RC side ... defaults to NOP
+	 * If needed implement this function in the dhd_custom_xxx.c
+	 * accordingly.
+	 */
+	dhd_plat_l1ss_ctrl(0);
+
+	dhdpcie_suspend_dump_cfgregs(bus, "BEFORE_EP_SUSPEND");
+#if defined(OEM_ANDROID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	dhd_dpc_tasklet_kill(bus->dhd);
+#endif /* OEM_ANDROID && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+	pci_save_state(dev);
+#if defined(OEM_ANDROID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	pch->state = pci_store_saved_state(dev);
+#endif /* OEM_ANDROID && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+	pci_enable_wake(dev, PCI_D0, TRUE);
+	if (pci_is_enabled(dev))
+		pci_disable_device(dev);
+
+	ret = pci_set_power_state(dev, PCI_D3hot);
+	if (ret) {
+		DHD_ERROR(("%s: pci_set_power_state error %d\n",
+			__FUNCTION__, ret));
+	}
+#ifdef OEM_ANDROID
+//	dev->state_saved = FALSE;
+#endif /* OEM_ANDROID */
+	dhdpcie_suspend_dump_cfgregs(bus, "AFTER_EP_SUSPEND");
+	return ret;
+}
+
+#ifdef DHD_WAKE_STATUS
+int bcmpcie_get_total_wake(struct dhd_bus *bus)
+{
+	dhdpcie_info_t *pch = pci_get_drvdata(bus->dev);
+
+	return pch->total_wake_count;
+}
+
+int bcmpcie_set_get_wake(struct dhd_bus *bus, int flag)
+{
+	dhdpcie_info_t *pch = pci_get_drvdata(bus->dev);
+	unsigned long flags;
+	int ret;
+
+	DHD_PKT_WAKE_LOCK(&pch->pkt_wake_lock, flags);
+
+	ret = pch->pkt_wake;
+	pch->total_wake_count += flag;
+	pch->pkt_wake = flag;
+	DHD_PKT_WAKE_UNLOCK(&pch->pkt_wake_lock, flags);
+	return ret;
+}
+
+int bcmpcie_get_wake(struct dhd_bus *bus)
+{
+	dhdpcie_info_t *pch = pci_get_drvdata(bus->dev);
+	unsigned long flags;
+	int ret;
+
+	DHD_PKT_WAKE_LOCK(&pch->pkt_wake_lock, flags);
+	ret = pch->pkt_wake;
+	DHD_PKT_WAKE_UNLOCK(&pch->pkt_wake_lock, flags);
+	return ret;
+}
+
+int bcmpcie_set_get_wake_pkt_dump(struct dhd_bus *bus, int wake_pkt_dump)
+{
+	dhdpcie_info_t *pch = pci_get_drvdata(bus->dev);
+	unsigned long flags;
+	int ret;
+
+	DHD_PKT_WAKE_LOCK(&pch->pkt_wake_lock, flags);
+	ret = pch->pkt_wake_dump;
+	pch->pkt_wake_dump = wake_pkt_dump;
+	DHD_PKT_WAKE_UNLOCK(&pch->pkt_wake_lock, flags);
+	return ret;
+}
+#endif /* DHD_WAKE_STATUS */
+
+static int dhdpcie_resume_dev(struct pci_dev *dev)
+{
+	int err = 0;
+	dhdpcie_info_t *pch = pci_get_drvdata(dev);
+#if defined(OEM_ANDROID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	pci_load_and_free_saved_state(dev, &pch->state);
+#endif /* OEM_ANDROID && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+	DHD_RPM(("%s: Enter\n", __FUNCTION__));
+
+	/* Resture back current bar1 window */
+	OSL_PCI_WRITE_CONFIG(pch->bus->osh, PCI_BAR1_WIN, 4, pch->bus->curr_bar1_win);
+
+#ifdef FORCE_TPOWERON
+	if (dhdpcie_chip_req_forced_tpoweron(pch->bus)) {
+		dhd_bus_set_tpoweron(pch->bus, tpoweron_scale);
+	}
+#endif /* FORCE_TPOWERON */
+	err = pci_enable_device(dev);
+	if (err) {
+		DHD_CONS_ONLY(("%s:pci_enable_device error %d \n", __FUNCTION__, err));
+		goto out;
+	}
+	pci_set_master(dev);
+	err = pci_set_power_state(dev, PCI_D0);
+	if (err) {
+		DHD_CONS_ONLY(("%s:pci_set_power_state error %d \n", __FUNCTION__, err));
+		goto out;
+	}
+
+#ifdef OEM_ANDROID
+	dev->state_saved = TRUE;
+#endif /* OEM_ANDROID */
+	pci_restore_state(dev);
+
+	BCM_REFERENCE(pch);
+	dhdpcie_suspend_dump_cfgregs(pch->bus, "AFTER_EP_RESUME");
+
+	/*
+	 * Re-enable L1ss in Resume path. Implementation defalts to NOP
+	 * If need override in the paltform file
+	 */
+	dhd_plat_l1ss_ctrl(1);
+
+out:
+	return err;
+}
+
+static int dhdpcie_resume_host_dev(dhd_bus_t *bus)
+{
+	int bcmerror = 0;
+
+	bcmerror = dhdpcie_start_host_dev(bus);
+	if (bcmerror < 0) {
+		DHD_ERROR(("%s: PCIe RC resume failed!!! (%d)\n",
+			__FUNCTION__, bcmerror));
+		bus->is_linkdown = 1;
+	}
+
+	return bcmerror;
+}
+
+static int dhdpcie_suspend_host_dev(dhd_bus_t *bus)
+{
+	int bcmerror = 0;
+#ifdef CONFIG_ARCH_EXYNOS
+	/*
+	 * XXX : SWWLAN-82173, SWWLAN-82183 WAR for SS PCIe RC
+	 * SS PCIe RC/EP is 1 to 1 mapping using different channel
+	 * RC0 - LTE, RC1 - WiFi RC0-1 is working independently
+	 */
+
+	if (bus->rc_dev) {
+		pci_save_state(bus->rc_dev);
+	} else {
+		DHD_ERROR(("%s: RC %x:%x handle is NULL\n",
+			__FUNCTION__, dhd_plat_get_rc_vendor_id(), dhd_plat_get_rc_device_id()));
+	}
+#endif /* CONFIG_ARCH_EXYNOS */
+	bcmerror = dhdpcie_stop_host_dev(bus);
+	return bcmerror;
+}
+
+int
+dhdpcie_set_master_and_d0_pwrstate(dhd_bus_t *bus)
+{
+	int err;
+	pci_set_master(bus->dev);
+	err = pci_set_power_state(bus->dev, PCI_D0);
+	if (err) {
+		DHD_ERROR(("%s: pci_set_power_state error %d \n", __FUNCTION__, err));
+	}
+	return err;
+}
+
+uint32
+dhdpcie_rc_config_read(dhd_bus_t *bus, uint offset)
+{
+	uint val = -1; /* Initialise to 0xfffffff */
+	if (bus->rc_dev) {
+		pci_read_config_dword(bus->rc_dev, offset, &val);
+		OSL_DELAY(100);
+	} else {
+		DHD_ERROR(("%s: RC %x:%x handle is NULL\n",
+			__FUNCTION__, dhd_plat_get_rc_vendor_id(), dhd_plat_get_rc_device_id()));
+	}
+	DHD_ERROR(("%s: RC %x:%x offset 0x%x val 0x%x\n",
+		__FUNCTION__, dhd_plat_get_rc_vendor_id(), dhd_plat_get_rc_device_id(),
+		offset, val));
+	return (val);
+}
+
+/*
+ * Reads/ Writes the value of capability register
+ * from the given CAP_ID section of PCI Root Port
+ *
+ * Arguements
+ * @bus current dhd_bus_t pointer
+ * @cap Capability or Extended Capability ID to get
+ * @offset offset of Register to Read
+ * @is_ext TRUE if @cap is given for Extended Capability
+ * @is_write is set to TRUE to indicate write
+ * @val value to write
+ *
+ * Return Value
+ * Returns 0xffffffff on error
+ * on write success returns BCME_OK (0)
+ * on Read Success returns the value of register requested
+ * Note: caller shoud ensure valid capability ID and Ext. Capability ID.
+ */
+
+uint32
+dhdpcie_access_cap(struct pci_dev *pdev, int cap, uint offset, bool is_ext, bool is_write,
+	uint32 writeval)
+{
+	int cap_ptr = 0;
+	uint32 ret = -1;
+	uint32 readval;
+
+	if (!(pdev)) {
+		DHD_ERROR(("%s: pdev is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	/* Find Capability offset */
+	if (is_ext) {
+		/* removing max EXT_CAP_ID check as
+		 * linux kernel definition's max value is not upadted yet as per spec
+		 */
+		cap_ptr = pci_find_ext_capability(pdev, cap);
+
+	} else {
+		/* removing max PCI_CAP_ID_MAX check as
+		 * pervious kernel versions dont have this definition
+		 */
+		cap_ptr = pci_find_capability(pdev, cap);
+	}
+
+	/* Return if capability with given ID not found */
+	if (cap_ptr == 0) {
+		DHD_ERROR(("%s: PCI Cap(0x%02x) not supported.\n",
+			__FUNCTION__, cap));
+		return BCME_ERROR;
+	}
+
+	if (is_write) {
+		pci_write_config_dword(pdev, (cap_ptr + offset), writeval);
+		ret = BCME_OK;
+
+	} else {
+
+		pci_read_config_dword(pdev, (cap_ptr + offset), &readval);
+		ret = readval;
+	}
+
+	return ret;
+}
+
+uint32
+dhdpcie_rc_access_cap(dhd_bus_t *bus, int cap, uint offset, bool is_ext, bool is_write,
+	uint32 writeval)
+{
+	if (!(bus->rc_dev)) {
+		DHD_ERROR(("%s: RC %x:%x handle is NULL\n",
+			__FUNCTION__, dhd_plat_get_rc_vendor_id(), dhd_plat_get_rc_device_id()));
+		return BCME_ERROR;
+	}
+
+	return dhdpcie_access_cap(bus->rc_dev, cap, offset, is_ext, is_write, writeval);
+}
+
+uint32
+dhdpcie_ep_access_cap(dhd_bus_t *bus, int cap, uint offset, bool is_ext, bool is_write,
+	uint32 writeval)
+{
+	if (!(bus->dev)) {
+		DHD_ERROR(("%s: EP handle is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	return dhdpcie_access_cap(bus->dev, cap, offset, is_ext, is_write, writeval);
+}
+
+/* API wrapper to read Root Port link capability
+ * Returns 2 = GEN2 1 = GEN1 BCME_ERR on linkcap not found
+ */
+
+uint32 dhd_debug_get_rc_linkcap(dhd_bus_t *bus)
+{
+	uint32 linkcap = -1;
+	linkcap = dhdpcie_rc_access_cap(bus, PCIE_CAP_ID_EXP,
+			PCIE_CAP_LINKCAP_OFFSET, FALSE, FALSE, 0);
+	linkcap &= PCIE_CAP_LINKCAP_LNKSPEED_MASK;
+	return linkcap;
+}
+
+static void dhdpcie_config_save_restore_coherent(dhd_bus_t *bus, bool state)
+{
+	if (bus->coreid == ARMCA7_CORE_ID) {
+		if (state) {
+			/* Sleep */
+			bus->coherent_state = dhdpcie_bus_cfg_read_dword(bus,
+				PCIE_CFG_SUBSYSTEM_CONTROL, 4) & PCIE_BARCOHERENTACCEN_MASK;
+		} else {
+			uint32 val = (dhdpcie_bus_cfg_read_dword(bus, PCIE_CFG_SUBSYSTEM_CONTROL,
+				4) & ~PCIE_BARCOHERENTACCEN_MASK) | bus->coherent_state;
+			dhdpcie_bus_cfg_write_dword(bus, PCIE_CFG_SUBSYSTEM_CONTROL, 4, val);
+		}
+	}
+}
+
+int dhdpcie_pci_suspend_resume(dhd_bus_t *bus, bool state)
+{
+	int rc;
+
+	struct pci_dev *dev = bus->dev;
+
+	if (state) {
+		bus->ptm_ctrl_reg = dhdpcie_bus_cfg_read_dword(bus, PCI_CFG_PTM_CTRL, 4);
+
+		dhdpcie_config_save_restore_coherent(bus, state);
+
+#if !defined(BCMPCIE_OOB_HOST_WAKE) && !defined(PCIE_OOB)
+		dhdpcie_pme_active(bus->osh, state);
+#endif /* !BCMPCIE_OOB_HOST_WAKE && !PCIE_OOB */
+		rc = dhdpcie_suspend_dev(dev);
+		if (!rc) {
+			dhdpcie_suspend_host_dev(bus);
+		}
+	} else {
+		rc = dhdpcie_resume_host_dev(bus);
+		if (!rc) {
+			rc = dhdpcie_resume_dev(dev);
+			if (PCIECTO_ENAB(bus)) {
+				/* reinit CTO configuration
+				 * because cfg space got reset at D3 (PERST)
+				 */
+				dhdpcie_cto_cfg_init(bus, TRUE);
+			}
+
+			dhdpcie_bus_cfg_write_dword(bus, PCI_CFG_PTM_CTRL, 4, bus->ptm_ctrl_reg);
+
+			if (PCIE_ENUM_RESET_WAR_ENAB(bus->sih->buscorerev)) {
+				dhdpcie_ssreset_dis_enum_rst(bus);
+			}
+#if !defined(BCMPCIE_OOB_HOST_WAKE) && !defined(PCIE_OOB)
+			dhdpcie_pme_active(bus->osh, state);
+#endif /* !BCMPCIE_OOB_HOST_WAKE && !PCIE_OOB */
+			dhdpcie_config_save_restore_coherent(bus, state);
+		}
+#if defined(OEM_ANDROID)
+#if defined(DHD_HANG_SEND_UP_TEST)
+		if (bus->is_linkdown ||
+			bus->dhd->req_hang_type == HANG_REASON_PCIE_RC_LINK_UP_FAIL)
+#else /* DHD_HANG_SEND_UP_TEST */
+		if (bus->is_linkdown)
+#endif /* DHD_HANG_SEND_UP_TEST */
+		{
+			bus->dhd->hang_reason = HANG_REASON_PCIE_RC_LINK_UP_FAIL;
+			dhd_os_send_hang_message(bus->dhd);
+		}
+#endif /* OEM_ANDROID */
+	}
+	return rc;
+}
+
+static int dhdpcie_device_scan(struct device *dev, void *data)
+{
+	struct pci_dev *pcidev;
+	int *cnt = data;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	pcidev = container_of(dev, struct pci_dev, dev);
+	GCC_DIAGNOSTIC_POP();
+
+	if ((pcidev->vendor != VENDOR_BROADCOM) && (pcidev->vendor != VENDOR_SYNAPTICS))
+		return 0;
+
+	DHD_INFO(("Found Broadcom or Synaptics PCI device 0x%04x\n", pcidev->device));
+	*cnt += 1;
+	if (pcidev->driver && strcmp(pcidev->driver->name, dhdpcie_driver.name))
+		DHD_ERROR(("Broadcom or Synaptics PCI Device 0x%04x has allocated with driver %s\n",
+			pcidev->device, pcidev->driver->name));
+
+	return 0;
+}
+
+int
+dhdpcie_bus_register(void)
+{
+	int error = 0;
+
+	if (!(error = pci_register_driver(&dhdpcie_driver))) {
+		bus_for_each_dev(dhdpcie_driver.driver.bus, NULL, &error, dhdpcie_device_scan);
+		if (!error) {
+			DHD_ERROR(("No Broadcom or Synaptics PCI device enumerated!\n"));
+#ifdef DHD_PRELOAD
+			return 0;
+#endif
+		} else if (!dhdpcie_init_succeeded) {
+			DHD_ERROR(("%s: dhdpcie initialize failed.\n", __FUNCTION__));
+		} else {
+			return 0;
+		}
+
+		pci_unregister_driver(&dhdpcie_driver);
+		error = BCME_ERROR;
+	}
+
+	return error;
+}
+
+void
+dhdpcie_bus_unregister(void)
+{
+	unsigned long flags;
+	dhd_pub_t *dhdp = NULL;
+	if (!g_dhd_bus) {
+		DHD_ERROR(("%s: Bus is NULL\n", __FUNCTION__));
+		return;
+	}
+	dhdp = g_dhd_bus->dhd;
+	DHD_GENERAL_LOCK(dhdp, flags);
+	/* Check if bus is in suspend due to wifi off with accel boot */
+	if (dhdp->up == FALSE && dhdp->busstate == DHD_BUS_SUSPEND) {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+		DHD_ERROR(("%s Bus is in suspend state\n", __FUNCTION__));
+		dhdpcie_pci_suspend_resume(dhdp->bus, FALSE);
+	} else {
+		DHD_GENERAL_UNLOCK(dhdp, flags);
+	}
+	pci_unregister_driver(&dhdpcie_driver);
+}
+
+#if defined(CONFIG_ARCH_WAIPIO) || defined(CONFIG_SOC_S5E9925)
+bool devid_mismatch = FALSE;
+
+bool dhdpcie_validate_devid(uint16 device)
+{
+	uint16 config_devid;
+
+#ifdef BCM4389_CHIP_DEF
+	config_devid = BCM4389_D11AX_ID;
+#elif defined(BCM4375_CHIP)
+	config_devid = BCM4375_D11AX_ID;
+#else
+	DHD_ERROR(("%s: Unknown dev id, BCMXXXX_CHIP is not defined \n",
+			__FUNCTION__));
+	config_devid = 0;
+
+	return FALSE;
+#endif /* BCM4389_CHIP_DEF */
+
+	DHD_ERROR(("%s: config_devid %X device %X\n", __FUNCTION__, config_devid, device));
+
+	if ((config_devid == BCM4389_D11AX_ID) && (device == BCM4389_CHIP_ID)) {
+		/* The dev id is 4389 if OTP is not written. */
+		return TRUE;
+	}
+
+	return config_devid == device;
+}
+#endif /* CONFIG_ARCH_WAIPIO || CONFIG_SOC_S5E9925 */
+
+int __devinit
+dhdpcie_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	int err = 0;
+	DHD_MUTEX_LOCK();
+
+	if (dhdpcie_chipmatch (pdev->vendor, pdev->device)) {
+		DHD_ERROR(("%s: chipmatch failed!!\n", __FUNCTION__));
+		err = -ENODEV;
+		goto exit;
+	}
+
+	DHD_CONS_ONLY(("PCI_PROBE:  bus 0x%X, slot 0x%X,vendor 0x%X, device 0x%X"
+		"(good PCI location)\n", pdev->bus->number,
+		PCI_SLOT(pdev->devfn), pdev->vendor, pdev->device));
+
+#if defined(CONFIG_ARCH_WAIPIO) || defined(CONFIG_SOC_S5E9925)
+	if (dhdpcie_validate_devid(pdev->device) == FALSE) {
+		DHD_ERROR(("%s: dev id and BCMXXXX_CHIP definition is not matched\n",
+				__FUNCTION__));
+		devid_mismatch = TRUE;
+		err = -ENODEV;
+		goto exit;
+	}
+#endif /* CONFIG_ARCH_WAIPIO || CONFIG_SOC_S5E9925 */
+
+	if (dhdpcie_init_succeeded == TRUE) {
+		DHD_ERROR(("%s(): === Driver Already attached to a BRCM device === \r\n",
+			__FUNCTION__));
+		err = -ENODEV;
+		goto exit;
+	}
+
+	if (dhdpcie_init (pdev)) {
+		DHD_ERROR(("%s: PCIe Enumeration failed\n", __FUNCTION__));
+		err = -ENODEV;
+		goto exit;
+	}
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	/*
+	Since MSM PCIe RC dev usage conunt already incremented +2 even
+	before dhdpcie_pci_probe() called, then we inevitably to call
+	pm_runtime_put_noidle() two times to make the count start with zero.
+	*/
+
+	pm_runtime_put_noidle(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#ifdef BCMPCIE_DISABLE_ASYNC_SUSPEND
+	/* disable async suspend */
+	device_disable_async_suspend(&pdev->dev);
+#endif /* BCMPCIE_DISABLE_ASYNC_SUSPEND */
+
+	DHD_TRACE(("%s: PCIe Enumeration done!!\n", __FUNCTION__));
+exit:
+	DHD_MUTEX_UNLOCK();
+	return err;
+}
+
+int
+dhdpcie_detach(dhdpcie_info_t *pch)
+{
+	if (pch) {
+#if defined(OEM_ANDROID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+		if (!dhd_download_fw_on_driverload) {
+			pci_load_and_free_saved_state(pch->dev, &pch->default_state);
+		}
+#endif /* OEM_ANDROID && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+		MFREE(pch->osh, pch, sizeof(dhdpcie_info_t));
+	}
+	return 0;
+}
+
+void __devexit
+dhdpcie_pci_stop(struct pci_dev *pdev)
+{
+	osl_t *osh = NULL;
+	dhdpcie_info_t *pch = NULL;
+	dhd_bus_t *bus = NULL;
+
+	DHD_TRACE(("%s Enter\n", __FUNCTION__));
+	pch = pci_get_drvdata(pdev);
+	bus = pch->bus;
+	osh = pch->osh;
+
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_get_noresume(&pdev->dev);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+	if (bus) {
+
+		bus->rc_dev = NULL;
+
+		dhdpcie_bus_release(bus);
+	}
+
+	/*
+	 * For module type driver,
+	 * it needs to back up configuration space before rmmod
+	 * Since original backed up configuration space won't be restored if state_saved = false
+	 * This back up the configuration space again & state_saved = true
+	 */
+	pci_save_state(pdev);
+
+	if (pci_is_enabled(pdev))
+		pci_disable_device(pdev);
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	/* pcie os info detach */
+	MFREE(osh, pch->os_cxt, sizeof(dhdpcie_os_info_t));
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef USE_SMMU_ARCH_MSM
+	/* smmu info detach */
+	dhdpcie_smmu_remove(pdev, pch->smmu_cxt);
+	MFREE(osh, pch->smmu_cxt, sizeof(dhdpcie_smmu_info_t));
+#endif /* USE_SMMU_ARCH_MSM */
+	/* pcie info detach */
+	dhdpcie_detach(pch);
+	/* osl detach */
+	osl_detach(osh);
+
+	dhdpcie_init_succeeded = FALSE;
+
+	DHD_TRACE(("%s Exit\n", __FUNCTION__));
+
+	return;
+}
+
+void __devexit
+dhdpcie_pci_remove(struct pci_dev *pdev)
+{
+	DHD_MUTEX_LOCK();
+	DHD_ERROR(("%s Enter\n", __FUNCTION__));
+	dhdpcie_pci_stop(pdev);
+	DHD_MUTEX_UNLOCK();
+	return;
+}
+
+void __devexit
+dhdpcie_pci_shutdown(struct pci_dev *pdev)
+{
+	dhdpcie_info_t *pch = NULL;
+	dhd_bus_t *bus = NULL;
+
+	pch = pci_get_drvdata(pdev);
+	bus = pch->bus;
+	DHD_ERROR(("%s Enter\n", __FUNCTION__));
+
+	/* Stop all interface network queue */
+	dhd_bus_stop_queue(bus);
+	/* Disable IRQ */
+	dhdpcie_disable_irq(bus);
+	/* Kill dpc */
+	dhd_dpc_kill(bus->dhd);
+	/* Stop RPM timer */
+	DHD_STOP_RPM_TIMER(bus->dhd);
+	dhdpcie_busbusy_wait(bus->dhd);
+	return;
+}
+
+/* Enable Linux Msi */
+int
+dhdpcie_enable_msi(struct pci_dev *pdev, unsigned int min_vecs, unsigned int max_vecs)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
+	return pci_alloc_irq_vectors(pdev, min_vecs, max_vecs, PCI_IRQ_MSI);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	return pci_enable_msi_range(pdev, min_vecs, max_vecs);
+#else
+	return pci_enable_msi_block(pdev, max_vecs);
+#endif
+}
+
+/* Disable Linux Msi */
+void
+dhdpcie_disable_msi(struct pci_dev *pdev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
+	pci_free_irq_vectors(pdev);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+	pci_disable_msi(pdev);
+#else
+	pci_disable_msi(pdev);
+#endif
+	return;
+}
+
+/* Request Linux irq */
+int
+dhdpcie_request_irq(dhdpcie_info_t *dhdpcie_info)
+{
+	dhd_bus_t *bus = dhdpcie_info->bus;
+	struct pci_dev *pdev = dhdpcie_info->bus->dev;
+	int err = 0;
+
+	if (!bus->irq_registered) {
+		snprintf(dhdpcie_info->pciname, sizeof(dhdpcie_info->pciname),
+			"dhdpcie:%s", pci_name(pdev));
+
+		if (bus->d2h_intr_method == PCIE_MSI) {
+			if (dhdpcie_enable_msi(pdev, 1, 1) < 0) {
+				DHD_ERROR(("%s: dhdpcie_enable_msi() failed\n", __FUNCTION__));
+				dhdpcie_disable_msi(pdev);
+				bus->d2h_intr_method = PCIE_INTX;
+			}
+		}
+
+		if (bus->d2h_intr_method == PCIE_MSI)
+			printf("%s: MSI enabled, irq=%d\n", __FUNCTION__, pdev->irq);
+		else
+			printf("%s: INTx enabled, irq=%d\n", __FUNCTION__, pdev->irq);
+
+		err = request_irq(pdev->irq, dhdpcie_isr, IRQF_SHARED,
+			dhdpcie_info->pciname, bus);
+		if (err < 0) {
+			DHD_ERROR(("%s: request_irq() failed with %d\n", __FUNCTION__, err));
+			if (bus->d2h_intr_method == PCIE_MSI) {
+				dhdpcie_disable_msi(pdev);
+			}
+			return -1;
+		}
+		else {
+			bus->irq_registered = TRUE;
+		}
+	} else {
+		DHD_ERROR(("%s: PCI IRQ is already registered\n", __FUNCTION__));
+	}
+
+	dhdpcie_enable_irq_loop(bus);
+
+	DHD_TRACE(("%s %s\n", __FUNCTION__, dhdpcie_info->pciname));
+
+	return 0; /* SUCCESS */
+}
+
+/**
+ *	dhdpcie_get_pcieirq - return pcie irq number to linux-dhd
+ */
+int
+dhdpcie_get_pcieirq(struct dhd_bus *bus, unsigned int *irq)
+{
+	struct pci_dev *pdev = bus->dev;
+
+	if (!pdev) {
+		DHD_ERROR(("%s : bus->dev is NULL\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	*irq  = pdev->irq;
+
+	return 0; /* SUCCESS */
+}
+
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
+#define PRINTF_RESOURCE	"0x%016llx"
+#else
+#define PRINTF_RESOURCE	"0x%08x"
+#endif
+
+#ifdef EXYNOS_PCIE_MODULE_PATCH
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+extern struct pci_saved_state *bcm_pcie_default_state;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+#endif /* EXYNOS_MODULE_PATCH */
+
+/*
+
+Name:  osl_pci_get_resource
+
+Parametrs:
+
+1: struct pci_dev *pdev   -- pci device structure
+2: pci_res                       -- structure containing pci configuration space values
+
+Return value:
+
+int   - Status (TRUE or FALSE)
+
+Description:
+Access PCI configuration space, retrieve  PCI allocated resources , updates in resource structure.
+
+ */
+#ifdef DHD_DEBUG_REG_DUMP
+uint64 regs_addr;
+#endif /* DHD_DEBUG_REG_DUMP */
+int dhdpcie_get_resource(dhdpcie_info_t *dhdpcie_info)
+{
+	phys_addr_t  bar0_addr, bar1_addr;
+	ulong bar1_size;
+	struct pci_dev *pdev = NULL;
+	pdev = dhdpcie_info->dev;
+#ifdef EXYNOS_PCIE_MODULE_PATCH
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	if (bcm_pcie_default_state) {
+		pci_load_saved_state(pdev, bcm_pcie_default_state);
+		pci_restore_state(pdev);
+	}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+#endif /* EXYNOS_MODULE_PATCH */
+
+	/*
+	 * For built-in type driver,
+	 * it can't restore configuration backup because of state_saved = false at first load time
+	 * For module type driver,
+	 * it couldn't remap the BAR0/BAR1 address
+	 * without restoring configuration backup at second load,
+	 * and remains configuration backup in pci_dev, DHD didn't remove it from the bus
+	 * pci_restore_state() restores proper BAR0/BAR1 address
+	 */
+	pci_restore_state(pdev);
+
+	do {
+		if (pci_enable_device(pdev)) {
+			DHD_CONS_ONLY(("%s: Cannot enable PCI device\n", __FUNCTION__));
+			break;
+		}
+		pci_set_master(pdev);
+		bar0_addr = pci_resource_start(pdev, 0);	/* Bar-0 mapped address */
+		bar1_addr = pci_resource_start(pdev, 2);	/* Bar-1 mapped address */
+
+		/* read Bar-1 mapped memory range */
+		bar1_size = pci_resource_len(pdev, 2);
+
+		if ((bar1_size == 0) || (bar1_addr == 0)) {
+			DHD_CONS_ONLY(("%s: BAR1 Not enabled for this device  size(%ld),"
+				" addr(0x"PRINTF_RESOURCE")\n",
+				__FUNCTION__, bar1_size, bar1_addr));
+			goto err;
+		}
+
+		dhdpcie_info->regs = (volatile char *) REG_MAP(bar0_addr, DONGLE_REG_MAP_SIZE);
+		dhdpcie_info->bar1_size =
+			(bar1_size > DONGLE_TCM_MAP_SIZE) ? bar1_size : DONGLE_TCM_MAP_SIZE;
+		dhdpcie_info->tcm = (volatile char *) REG_MAP(bar1_addr, dhdpcie_info->bar1_size);
+
+		if (!dhdpcie_info->regs || !dhdpcie_info->tcm) {
+			DHD_ERROR(("%s:ioremap() failed\n", __FUNCTION__));
+			break;
+		}
+#ifdef EXYNOS_PCIE_MODULE_PATCH
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+		if (bcm_pcie_default_state == NULL) {
+			pci_save_state(pdev);
+			bcm_pcie_default_state = pci_store_saved_state(pdev);
+		}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+#endif /* EXYNOS_MODULE_PATCH */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	/* Backup PCIe configuration so as to use Wi-Fi on/off process
+	 * in case of built in driver
+	 */
+	pci_save_state(pdev);
+	dhdpcie_info->default_state = pci_store_saved_state(pdev);
+
+	if (dhdpcie_info->default_state == NULL) {
+		DHD_ERROR(("%s pci_store_saved_state returns NULL\n",
+			__FUNCTION__));
+		REG_UNMAP(dhdpcie_info->regs);
+		REG_UNMAP(dhdpcie_info->tcm);
+		pci_disable_device(pdev);
+		break;
+	}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+		DHD_TRACE(("%s:Phys addr : reg space = %p base addr 0x"PRINTF_RESOURCE" \n",
+			__FUNCTION__, dhdpcie_info->regs, bar0_addr));
+		DHD_TRACE(("%s:Phys addr : tcm_space = %p base addr 0x"PRINTF_RESOURCE" \n",
+			__FUNCTION__, dhdpcie_info->tcm, bar1_addr));
+#ifdef DHD_DEBUG_REG_DUMP
+		regs_addr = (uint64)dhdpcie_info->regs;
+		DHD_ERROR(("%s: saved regs_addr = 0x%llx\n", __FUNCTION__, regs_addr));
+#endif /* DHD_DEBUG_REG_DUMP */
+		return 0; /* SUCCESS  */
+	} while (0);
+err:
+	return -1;  /* FAILURE */
+}
+
+int dhdpcie_scan_resource(dhdpcie_info_t *dhdpcie_info)
+{
+
+	DHD_TRACE(("%s: ENTER\n", __FUNCTION__));
+
+	do {
+		/* define it here only!! */
+		if (dhdpcie_get_resource (dhdpcie_info)) {
+			DHD_ERROR(("%s: Failed to get PCI resources\n", __FUNCTION__));
+			break;
+		}
+		DHD_TRACE(("%s:Exit - SUCCESS \n",
+			__FUNCTION__));
+
+		return 0; /* SUCCESS */
+
+	} while (0);
+
+	DHD_TRACE(("%s:Exit - FAILURE \n", __FUNCTION__));
+
+	return -1; /* FAILURE */
+
+}
+
+void dhdpcie_dump_resource(dhd_bus_t *bus)
+{
+	dhdpcie_info_t *pch;
+
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		DHD_ERROR(("%s: pch is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	/* BAR0 */
+	DHD_RPM(("%s: BAR0(VA): 0x%pK, BAR0(PA): "PRINTF_RESOURCE", SIZE: %d\n",
+		__FUNCTION__, pch->regs, pci_resource_start(bus->dev, 0),
+		DONGLE_REG_MAP_SIZE));
+
+	/* BAR1 */
+	DHD_RPM(("%s: BAR1(VA): 0x%pK, BAR1(PA): "PRINTF_RESOURCE", SIZE: %d\n",
+		__FUNCTION__, pch->tcm, pci_resource_start(bus->dev, 2),
+		pch->bar1_size));
+}
+
+int dhdpcie_init(struct pci_dev *pdev)
+{
+
+	osl_t 				*osh = NULL;
+	dhd_bus_t 			*bus = NULL;
+	dhdpcie_info_t		*dhdpcie_info =  NULL;
+	wifi_adapter_info_t	*adapter = NULL;
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	dhdpcie_os_info_t	*dhdpcie_osinfo = NULL;
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+#ifdef USE_SMMU_ARCH_MSM
+	dhdpcie_smmu_info_t	*dhdpcie_smmu_info = NULL;
+#endif /* USE_SMMU_ARCH_MSM */
+	int ret = 0;
+
+	do {
+		/* osl attach */
+		if (!(osh = osl_attach(pdev, PCI_BUS, FALSE))) {
+			DHD_ERROR(("%s: osl_attach failed\n", __FUNCTION__));
+			break;
+		}
+
+		/* initialize static buffer */
+		adapter = dhd_wifi_platform_get_adapter(PCI_BUS, pdev->bus->number,
+			PCI_SLOT(pdev->devfn));
+		if (adapter != NULL) {
+			DHD_ERROR(("%s: found adapter info '%s'\n", __FUNCTION__, adapter->name));
+			adapter->bus_type = PCI_BUS;
+			adapter->bus_num = pdev->bus->number;
+			adapter->slot_num = PCI_SLOT(pdev->devfn);
+			adapter->pci_dev = pdev;
+		} else {
+			DHD_ERROR(("%s: can't find adapter info for this chip\n", __FUNCTION__));
+#ifdef ADAPTER_IDX
+			break;
+#endif
+		}
+		osl_static_mem_init(osh, adapter);
+
+		/*  allocate linux spcific pcie structure here */
+		if (!(dhdpcie_info = MALLOC(osh, sizeof(dhdpcie_info_t)))) {
+			DHD_ERROR(("%s: MALLOC of dhd_bus_t failed\n", __FUNCTION__));
+			break;
+		}
+		bzero(dhdpcie_info, sizeof(dhdpcie_info_t));
+		dhdpcie_info->osh = osh;
+		dhdpcie_info->dev = pdev;
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+		/* allocate OS speicific structure */
+		dhdpcie_osinfo = MALLOC(osh, sizeof(dhdpcie_os_info_t));
+		if (dhdpcie_osinfo == NULL) {
+			DHD_ERROR(("%s: MALLOC of dhdpcie_os_info_t failed\n",
+				__FUNCTION__));
+			break;
+		}
+		bzero(dhdpcie_osinfo, sizeof(dhdpcie_os_info_t));
+		dhdpcie_info->os_cxt = (void *)dhdpcie_osinfo;
+
+		/* Initialize host wake IRQ */
+		spin_lock_init(&dhdpcie_osinfo->oob_irq_spinlock);
+		/* Get customer specific host wake IRQ parametres: IRQ number as IRQ type */
+		dhdpcie_osinfo->oob_irq_num = wifi_platform_get_irq_number(adapter,
+			&dhdpcie_osinfo->oob_irq_flags);
+		if (dhdpcie_osinfo->oob_irq_num < 0) {
+			DHD_ERROR(("%s: Host OOB irq is not defined\n", __FUNCTION__));
+		}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef USE_SMMU_ARCH_MSM
+		/* allocate private structure for using SMMU */
+		dhdpcie_smmu_info = MALLOC(osh, sizeof(dhdpcie_smmu_info_t));
+		if (dhdpcie_smmu_info == NULL) {
+			DHD_ERROR(("%s: MALLOC of dhdpcie_smmu_info_t failed\n",
+				__FUNCTION__));
+			break;
+		}
+		bzero(dhdpcie_smmu_info, sizeof(dhdpcie_smmu_info_t));
+		dhdpcie_info->smmu_cxt = (void *)dhdpcie_smmu_info;
+
+		/* Initialize smmu structure */
+		if (dhdpcie_smmu_init(pdev, dhdpcie_info->smmu_cxt) < 0) {
+			DHD_ERROR(("%s: Failed to initialize SMMU\n",
+				__FUNCTION__));
+			break;
+		}
+#endif /* USE_SMMU_ARCH_MSM */
+
+#ifdef DHD_SET_PCIE_DMA_MASK_FOR_GS101
+		/* S.SLSI PCIe DMA engine cannot support 64 bit bus address. Hence, set 36 bit */
+		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(DHD_PCIE_DMA_MASK_FOR_GS101)) ||
+			pci_set_consistent_dma_mask(pdev,
+				DMA_BIT_MASK(DHD_PCIE_DMA_MASK_FOR_GS101))) {
+			DHD_ERROR(("%s: DMA set %d bit mask failed.\n",
+				__FUNCTION__, DHD_PCIE_DMA_MASK_FOR_GS101));
+			return -EINVAL;
+		}
+#endif /* DHD_SET_PCIE_DMA_MASK_FOR_GS101 */
+
+#ifdef DHD_WAKE_STATUS
+		/* Initialize pkt_wake_lock */
+		spin_lock_init(&dhdpcie_info->pkt_wake_lock);
+#endif /* DHD_WAKE_STATUS */
+
+		/* Find the PCI resources, verify the  */
+		/* vendor and device ID, map BAR regions and irq,  update in structures */
+		if (dhdpcie_scan_resource(dhdpcie_info)) {
+			DHD_ERROR(("%s: dhd_Scan_PCI_Res failed\n", __FUNCTION__));
+
+			break;
+		}
+
+		/* Bus initialization */
+		ret = dhdpcie_bus_attach(osh, &bus, dhdpcie_info->regs, dhdpcie_info->tcm, pdev, adapter);
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s:dhdpcie_bus_attach() failed\n", __FUNCTION__));
+			break;
+		}
+
+		dhdpcie_info->bus = bus;
+		bus->bar1_size = dhdpcie_info->bar1_size;
+		bus->is_linkdown = 0;
+		bus->no_bus_init = FALSE;
+		bus->cto_triggered = 0;
+
+		bus->rc_dev = NULL;
+
+		/* Get RC Device Handle */
+		if (bus->dev->bus) {
+			/* self member of structure pci_bus is bridge device as seen by parent */
+			bus->rc_dev = bus->dev->bus->self;
+			if (bus->rc_dev)
+				DHD_ERROR(("%s: rc_dev from dev->bus->self (%x:%x) is %pK\n", __FUNCTION__,
+					bus->rc_dev->vendor, bus->rc_dev->device, bus->rc_dev));
+			else
+				DHD_ERROR(("%s: bus->dev->bus->self is NULL\n", __FUNCTION__));
+		} else {
+			DHD_ERROR(("%s: unable to get rc_dev as dev->bus is NULL\n", __FUNCTION__));
+		}
+
+		/* if rc_dev is still NULL, try to get from vendor/device IDs */
+		if (bus->rc_dev == NULL) {
+			bus->rc_dev = pci_get_device(dhd_plat_get_rc_vendor_id(),
+					dhd_plat_get_rc_device_id(), NULL);
+			DHD_ERROR(("%s: rc_dev from pci_get_device (%x:%x) is %p\n", __FUNCTION__,
+				dhd_plat_get_rc_vendor_id(), dhd_plat_get_rc_device_id(),
+				bus->rc_dev));
+		}
+
+		bus->rc_ep_aspm_cap = dhd_bus_is_rc_ep_aspm_capable(bus);
+		bus->rc_ep_l1ss_cap = dhd_bus_is_rc_ep_l1ss_capable(bus);
+		DHD_ERROR(("%s: rc_ep_aspm_cap: %d rc_ep_l1ss_cap: %d\n",
+			__FUNCTION__, bus->rc_ep_aspm_cap, bus->rc_ep_l1ss_cap));
+
+#ifdef FORCE_TPOWERON
+		if (dhdpcie_chip_req_forced_tpoweron(bus)) {
+			dhd_bus_set_tpoweron(bus, tpoweron_scale);
+		}
+#endif /* FORCE_TPOWERON */
+
+#ifdef DONGLE_ENABLE_ISOLATION
+		bus->dhd->dongle_isolation = TRUE;
+#endif /* DONGLE_ENABLE_ISOLATION */
+
+		if (bus->intr) {
+			/* Register interrupt callback, but mask it (not operational yet). */
+			DHD_INTR(("%s: Registering and masking interrupts\n", __FUNCTION__));
+			bus->intr_enabled = FALSE;
+			dhdpcie_bus_intr_disable(bus);
+
+			if (dhdpcie_request_irq(dhdpcie_info)) {
+				DHD_ERROR(("%s: request_irq() failed\n", __FUNCTION__));
+				break;
+			}
+		} else {
+			bus->pollrate = 1;
+			DHD_INFO(("%s: PCIe interrupt function is NOT registered "
+				"due to polling mode\n", __FUNCTION__));
+		}
+
+#if defined(BCM_REQUEST_FW)
+		if (dhd_bus_download_firmware(bus, osh, NULL, NULL) < 0) {
+		DHD_ERROR(("%s: failed to download firmware\n", __FUNCTION__));
+		}
+		bus->nv_path = NULL;
+		bus->fw_path = NULL;
+#endif /* BCM_REQUEST_FW */
+
+		/* set private data for pci_dev */
+		pci_set_drvdata(pdev, dhdpcie_info);
+
+		/* Ensure BAR1 switch feature enable if needed before FW download */
+		dhdpcie_bar1_window_switch_enab(bus);
+		dhd_init_bar1_switch_lock(bus);
+
+#if defined(BCMDHD_MODULAR) && defined(INSMOD_FW_LOAD)
+		if (1)
+#else
+		if (dhd_download_fw_on_driverload)
+#endif
+		{
+			if (dhd_bus_start(bus->dhd)) {
+				DHD_ERROR(("%s: dhd_bus_start() failed\n", __FUNCTION__));
+				if (!allow_delay_fwdl)
+					break;
+			}
+		} else {
+			/* Set ramdom MAC address during boot time */
+			get_random_bytes(&bus->dhd->mac.octet[3], 3);
+			/* Adding BRCM OUI */
+			bus->dhd->mac.octet[0] = 0;
+			bus->dhd->mac.octet[1] = 0x90;
+			bus->dhd->mac.octet[2] = 0x4C;
+		}
+
+		/* Attach to the OS network interface */
+		DHD_TRACE(("%s(): Calling dhd_attach_net() \n", __FUNCTION__));
+		if (dhd_attach_net(bus->dhd, TRUE)) {
+			DHD_ERROR(("%s(): ERROR.. dhd_attach_net() failed\n", __FUNCTION__));
+			break;
+		}
+
+		dhdpcie_init_succeeded = TRUE;
+#if defined(CONFIG_ARCH_MSM) && defined(CONFIG_SEC_PCIE_L1SS)
+		sec_pcie_set_use_ep_loaded(bus->rc_dev);
+#endif /* CONFIG_ARCH_MSM && CONFIG_SEC_PCIE_L1SS */
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+		pm_runtime_set_autosuspend_delay(&pdev->dev, AUTO_SUSPEND_TIMEOUT);
+		pm_runtime_use_autosuspend(&pdev->dev);
+		atomic_set(&bus->dhd->block_bus, FALSE);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+
+#if defined(MULTIPLE_SUPPLICANT)
+		wl_android_post_init(); // terence 20120530: fix critical section in dhd_open and dhdsdio_probe
+#endif /* MULTIPLE_SUPPLICANT */
+
+		DHD_TRACE(("%s:Exit - SUCCESS \n", __FUNCTION__));
+		return 0;  /* return  SUCCESS  */
+
+	} while (0);
+	/* reverse the initialization in order in case of error */
+
+	if (bus)
+		dhdpcie_bus_release(bus);
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+	if (dhdpcie_osinfo) {
+		MFREE(osh, dhdpcie_osinfo, sizeof(dhdpcie_os_info_t));
+	}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef USE_SMMU_ARCH_MSM
+	if (dhdpcie_smmu_info) {
+		MFREE(osh, dhdpcie_smmu_info, sizeof(dhdpcie_smmu_info_t));
+		dhdpcie_info->smmu_cxt = NULL;
+	}
+#endif /* USE_SMMU_ARCH_MSM */
+
+	if (dhdpcie_info)
+		dhdpcie_detach(dhdpcie_info);
+	if (adapter)
+		pci_disable_device(pdev);
+	if (osh)
+		osl_detach(osh);
+	if (adapter != NULL) {
+		adapter->bus_type = -1;
+		adapter->bus_num = -1;
+		adapter->slot_num = -1;
+	}
+
+	dhdpcie_init_succeeded = FALSE;
+
+	DHD_TRACE(("%s:Exit - FAILURE \n", __FUNCTION__));
+
+	return -1; /* return FAILURE  */
+}
+
+/* Free Linux irq */
+void
+dhdpcie_free_irq(dhd_bus_t *bus)
+{
+	struct pci_dev *pdev = NULL;
+
+	DHD_TRACE(("%s: freeing up the IRQ\n", __FUNCTION__));
+	if (bus) {
+		pdev = bus->dev;
+		if (bus->irq_registered) {
+#if defined(SET_PCIE_IRQ_CPU_CORE) || defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON) || \
+	defined(CLEAN_IRQ_AFFINITY_HINT)
+			/* clean up the affinity_hint before
+			 * the unregistration of PCIe irq
+			 */
+			(void)irq_set_affinity_hint(pdev->irq, NULL);
+#endif /* SET_PCIE_IRQ_CPU_CORE || DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON ||
+		* CLEAN_IRQ_AFFINITY_HINT
+		*/
+			free_irq(pdev->irq, bus);
+			bus->irq_registered = FALSE;
+			if (bus->d2h_intr_method == PCIE_MSI) {
+				dhdpcie_disable_msi(pdev);
+			}
+		} else {
+			DHD_ERROR(("%s: PCIe IRQ is not registered\n", __FUNCTION__));
+		}
+	}
+	DHD_TRACE(("%s: Exit\n", __FUNCTION__));
+	return;
+}
+
+/*
+
+Name:  dhdpcie_isr
+
+Parametrs:
+
+1: IN int irq   -- interrupt vector
+2: IN void *arg      -- handle to private data structure
+
+Return value:
+
+Status (TRUE or FALSE)
+
+Description:
+Interrupt Service routine checks for the status register,
+disable interrupt and queue DPC if mail box interrupts are raised.
+*/
+
+irqreturn_t
+dhdpcie_isr(int irq, void *arg)
+{
+	dhd_bus_t *bus = (dhd_bus_t*)arg;
+	bus->prev_isr_entry_time = bus->isr_entry_time;
+	bus->isr_entry_time = OSL_LOCALTIME_NS();
+	if (!dhdpcie_bus_isr(bus)) {
+		DHD_LOG_MEM(("%s: dhdpcie_bus_isr returns with FALSE\n", __FUNCTION__));
+	}
+	bus->isr_exit_time = OSL_LOCALTIME_NS();
+	return IRQ_HANDLED;
+}
+
+int
+dhdpcie_disable_irq_nosync(dhd_bus_t *bus)
+{
+	struct pci_dev *dev;
+	if ((bus == NULL) || (bus->dev == NULL)) {
+		DHD_ERROR(("%s: bus or bus->dev is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dev = bus->dev;
+	disable_irq_nosync(dev->irq);
+	return BCME_OK;
+}
+
+int
+dhdpcie_disable_irq(dhd_bus_t *bus)
+{
+	struct pci_dev *dev;
+	if ((bus == NULL) || (bus->dev == NULL)) {
+		DHD_ERROR(("%s: bus or bus->dev is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dev = bus->dev;
+	disable_irq(dev->irq);
+	return BCME_OK;
+}
+
+int
+dhdpcie_enable_irq(dhd_bus_t *bus)
+{
+	struct pci_dev *dev;
+	if ((bus == NULL) || (bus->dev == NULL)) {
+		DHD_ERROR(("%s: bus or bus->dev is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dev = bus->dev;
+	enable_irq(dev->irq);
+	return BCME_OK;
+}
+
+void
+dhdpcie_enable_irq_loop(dhd_bus_t *bus)
+{
+	/* Enable IRQ in a loop till host_irq_disable_count becomes 0 */
+	uint host_irq_disable_count = dhdpcie_irq_disabled(bus);
+	while (host_irq_disable_count--) {
+		dhdpcie_enable_irq(bus); /* Enable back interrupt!! */
+	}
+}
+
+int
+dhdpcie_irq_disabled(dhd_bus_t *bus)
+{
+	struct irq_desc *desc = NULL;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	desc = irq_to_desc(bus->dev->irq);
+#else // (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	desc = irq_data_to_desc(irq_get_irq_data(bus->dev->irq));
+#endif // (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+
+	/* depth will be zero, if enabled */
+	return desc->depth;
+}
+
+int
+dhdpcie_start_host_dev(dhd_bus_t *bus)
+{
+	int ret = 0;
+#ifdef CONFIG_ARCH_MSM
+#endif /* CONFIG_ARCH_MSM */
+	DHD_TRACE(("%s Enter:\n", __FUNCTION__));
+
+	if (bus == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (bus->dev == NULL) {
+		return BCME_ERROR;
+	}
+
+	ret = dhd_plat_pcie_resume(bus->dhd->plat_info);
+
+#ifdef CONFIG_ARCH_MSM
+	ret = msm_pcie_pm_control(MSM_PCIE_RESUME, bus->dev->bus->number,
+		bus->dev, NULL, 0);
+#endif /* CONFIG_ARCH_MSM */
+#ifdef CONFIG_ARCH_TEGRA
+#ifndef CONFIG_ARCH_TEGRA_210_SOC
+	ret = tegra_pcie_pm_resume();
+#endif /* CONFIG_ARCH_TEGRA_210_SOC */
+#endif /* CONFIG_ARCH_TEGRA */
+
+	if (ret) {
+		DHD_ERROR(("%s Failed to bring up PCIe link\n", __FUNCTION__));
+		goto done;
+	}
+
+done:
+	DHD_TRACE(("%s Exit:\n", __FUNCTION__));
+	return ret;
+}
+
+int
+dhdpcie_stop_host_dev(dhd_bus_t *bus)
+{
+	int ret = 0;
+#ifdef CONFIG_ARCH_MSM
+#endif /* CONFIG_ARCH_MSM */
+
+	DHD_TRACE(("%s Enter:\n", __FUNCTION__));
+
+	if (bus == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (bus->dev == NULL) {
+		return BCME_ERROR;
+	}
+
+	dhd_plat_pcie_suspend(bus->dhd->plat_info);
+
+#ifdef CONFIG_ARCH_MSM
+	ret = msm_pcie_pm_control(MSM_PCIE_SUSPEND, bus->dev->bus->number,
+		bus->dev, NULL, 0);
+#endif /* CONFIG_ARCH_MSM */
+#ifdef CONFIG_ARCH_TEGRA
+#ifndef CONFIG_ARCH_TEGRA_210_SOC
+	ret = tegra_pcie_pm_suspend();
+#endif /* CONFIG_ARCH_TEGRA_210_SOC */
+#endif /* CONFIG_ARCH_TEGRA */
+	if (ret) {
+		DHD_ERROR(("Failed to stop PCIe link\n"));
+		goto done;
+	}
+done:
+	DHD_TRACE(("%s Exit:\n", __FUNCTION__));
+	return ret;
+}
+
+int
+dhdpcie_disable_device(dhd_bus_t *bus)
+{
+	DHD_TRACE(("%s Enter:\n", __FUNCTION__));
+
+	if (bus == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (bus->dev == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (pci_is_enabled(bus->dev))
+		pci_disable_device(bus->dev);
+
+	return 0;
+}
+
+int
+dhdpcie_enable_device(dhd_bus_t *bus)
+{
+	int ret = BCME_ERROR;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	dhdpcie_info_t *pch;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+
+	DHD_TRACE(("%s Enter:\n", __FUNCTION__));
+
+	if (bus == NULL) {
+		return BCME_ERROR;
+	}
+
+	if (bus->dev == NULL) {
+		return BCME_ERROR;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		return BCME_ERROR;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) && (LINUX_VERSION_CODE < \
+	KERNEL_VERSION(3, 19, 0)) && !defined(CONFIG_SOC_EXYNOS8890)
+	/* Updated with pci_load_and_free_saved_state to compatible
+	 * with Kernel version 3.14.0 to 3.18.41.
+	 */
+	pci_load_and_free_saved_state(bus->dev, &pch->default_state);
+	pch->default_state = pci_store_saved_state(bus->dev);
+#else
+	pci_load_saved_state(bus->dev, pch->default_state);
+#endif /* LINUX_VERSION >= 3.14.0 && LINUX_VERSION < 3.19.0 && !CONFIG_SOC_EXYNOS8890 */
+
+	/* Check if Device ID is valid */
+	if (bus->dev->state_saved) {
+		uint32 vid, saved_vid;
+		pci_read_config_dword(bus->dev, PCI_CFG_VID, &vid);
+		saved_vid = bus->dev->saved_config_space[PCI_CFG_VID];
+		if (vid != saved_vid) {
+			DHD_ERROR(("%s: VID(0x%x) is different from saved VID(0x%x) "
+				"Skip the bus init\n", __FUNCTION__, vid, saved_vid));
+			bus->no_bus_init = TRUE;
+			/* Check if the PCIe link is down */
+			if (vid == (uint32)-1) {
+				bus->is_linkdown = 1;
+			}
+			return BCME_ERROR;
+		}
+	}
+
+	pci_restore_state(bus->dev);
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)) */
+
+	ret = pci_enable_device(bus->dev);
+	if (ret) {
+		pci_disable_device(bus->dev);
+	} else {
+		pci_set_master(bus->dev);
+	}
+
+	return ret;
+}
+
+int
+dhdpcie_alloc_resource(dhd_bus_t *bus)
+{
+	dhdpcie_info_t *dhdpcie_info;
+	phys_addr_t bar0_addr, bar1_addr;
+	ulong bar1_size;
+
+	do {
+		if (bus == NULL) {
+			DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+			break;
+		}
+
+		if (bus->dev == NULL) {
+			DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+			break;
+		}
+
+		dhdpcie_info = pci_get_drvdata(bus->dev);
+		if (dhdpcie_info == NULL) {
+			DHD_ERROR(("%s: dhdpcie_info is NULL\n", __FUNCTION__));
+			break;
+		}
+
+		bar0_addr = pci_resource_start(bus->dev, 0);	/* Bar-0 mapped address */
+		bar1_addr = pci_resource_start(bus->dev, 2);	/* Bar-1 mapped address */
+
+		/* read Bar-1 mapped memory range */
+		bar1_size = pci_resource_len(bus->dev, 2);
+
+		if ((bar1_size == 0) || (bar1_addr == 0)) {
+			DHD_CONS_ONLY(("%s: BAR1 Not enabled for this device size(%ld),"
+				" addr(0x"PRINTF_RESOURCE")\n",
+				__FUNCTION__, bar1_size, bar1_addr));
+			break;
+		}
+
+		dhdpcie_info->regs = (volatile char *) REG_MAP(bar0_addr, DONGLE_REG_MAP_SIZE);
+		if (!dhdpcie_info->regs) {
+			DHD_ERROR(("%s: ioremap() for regs is failed\n", __FUNCTION__));
+			break;
+		}
+
+		bus->regs = dhdpcie_info->regs;
+		dhdpcie_info->bar1_size =
+			(bar1_size > DONGLE_TCM_MAP_SIZE) ? bar1_size : DONGLE_TCM_MAP_SIZE;
+		dhdpcie_info->tcm = (volatile char *) REG_MAP(bar1_addr, dhdpcie_info->bar1_size);
+		if (!dhdpcie_info->tcm) {
+			DHD_ERROR(("%s: ioremap() for regs is failed\n", __FUNCTION__));
+			REG_UNMAP(dhdpcie_info->regs);
+			bus->regs = NULL;
+			break;
+		}
+
+		bus->tcm = dhdpcie_info->tcm;
+		bus->bar1_size = dhdpcie_info->bar1_size;
+
+		DHD_TRACE(("%s:Phys addr : reg space = %p base addr 0x"PRINTF_RESOURCE" \n",
+			__FUNCTION__, dhdpcie_info->regs, bar0_addr));
+		DHD_TRACE(("%s:Phys addr : tcm_space = %p base addr 0x"PRINTF_RESOURCE" \n",
+			__FUNCTION__, dhdpcie_info->tcm, bar1_addr));
+#ifdef DHD_DEBUG_REG_DUMP
+		regs_addr = (uint64)dhdpcie_info->regs;
+		DHD_ERROR(("%s: saved regs_addr = 0x%llx\n", __FUNCTION__, regs_addr));
+#endif /* DHD_DEBUG_REG_DUMP */
+		return 0;
+	} while (0);
+
+	return BCME_ERROR;
+}
+
+void
+dhdpcie_free_resource(dhd_bus_t *bus)
+{
+	dhdpcie_info_t *dhdpcie_info;
+
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	dhdpcie_info = pci_get_drvdata(bus->dev);
+	if (dhdpcie_info == NULL) {
+		DHD_ERROR(("%s: dhdpcie_info is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->regs) {
+		REG_UNMAP(dhdpcie_info->regs);
+		bus->regs = NULL;
+	}
+
+	if (bus->tcm) {
+		REG_UNMAP(dhdpcie_info->tcm);
+		bus->tcm = NULL;
+	}
+}
+
+int
+dhdpcie_bus_request_irq(struct dhd_bus *bus)
+{
+	dhdpcie_info_t *dhdpcie_info;
+	int ret = 0;
+
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dhdpcie_info = pci_get_drvdata(bus->dev);
+	if (dhdpcie_info == NULL) {
+		DHD_ERROR(("%s: dhdpcie_info is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (bus->intr) {
+		/* Register interrupt callback, but mask it (not operational yet). */
+		DHD_INTR(("%s: Registering and masking interrupts\n", __FUNCTION__));
+		bus->intr_enabled = FALSE;
+		dhdpcie_bus_intr_disable(bus);
+		ret = dhdpcie_request_irq(dhdpcie_info);
+		if (ret) {
+			DHD_ERROR(("%s: request_irq() failed, ret=%d\n",
+				__FUNCTION__, ret));
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+#ifdef BCMPCIE_OOB_HOST_WAKE
+#ifdef CONFIG_BCMDHD_GET_OOB_STATE
+extern int dhd_get_wlan_oob_gpio(void);
+#ifdef PRINT_WAKEUP_GPIO_STATUS
+extern int dhd_get_wlan_oob_gpio_number(void);
+#endif /* PRINT_WAKEUP_GPIO_STATUS */
+#endif /* CONFIG_BCMDHD_GET_OOB_STATE */
+
+int dhdpcie_get_oob_irq_level(void)
+{
+	int gpio_level;
+
+#ifdef CONFIG_BCMDHD_GET_OOB_STATE
+	gpio_level = dhd_get_wlan_oob_gpio();
+#else
+	gpio_level = BCME_UNSUPPORTED;
+#endif /* CONFIG_BCMDHD_GET_OOB_STATE */
+	return gpio_level;
+}
+#ifdef PRINT_WAKEUP_GPIO_STATUS
+int dhdpcie_get_oob_gpio_number(void)
+{
+	int gpio_number = BCME_UNSUPPORTED;
+#ifdef CONFIG_BCMDHD_GET_OOB_STATE
+	gpio_number = dhd_get_wlan_oob_gpio_number();
+#endif /* CONFIG_BCMDHD_GET_OOB_STATE */
+	return gpio_number;
+}
+#endif /* PRINT_WAKEUP_GPIO_STATUS */
+int dhdpcie_get_oob_irq_status(struct dhd_bus *bus)
+{
+	dhdpcie_info_t *pch;
+	dhdpcie_os_info_t *dhdpcie_osinfo;
+
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		DHD_ERROR(("%s: pch is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	dhdpcie_osinfo = (dhdpcie_os_info_t *)pch->os_cxt;
+
+	return dhdpcie_osinfo ? dhdpcie_osinfo->oob_irq_enabled : 0;
+}
+
+int dhdpcie_get_oob_irq_num(struct dhd_bus *bus)
+{
+	dhdpcie_info_t *pch;
+	dhdpcie_os_info_t *dhdpcie_osinfo;
+
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		DHD_ERROR(("%s: pch is NULL\n", __FUNCTION__));
+		return 0;
+	}
+
+	dhdpcie_osinfo = (dhdpcie_os_info_t *)pch->os_cxt;
+
+	return dhdpcie_osinfo ? dhdpcie_osinfo->oob_irq_num : 0;
+}
+
+void dhdpcie_oob_intr_set(dhd_bus_t *bus, bool enable)
+{
+	unsigned long flags;
+	dhdpcie_info_t *pch;
+	dhdpcie_os_info_t *dhdpcie_osinfo;
+
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		DHD_ERROR(("%s: pch is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	dhdpcie_osinfo = (dhdpcie_os_info_t *)pch->os_cxt;
+	DHD_OOB_IRQ_LOCK(&dhdpcie_osinfo->oob_irq_spinlock, flags);
+	if ((dhdpcie_osinfo->oob_irq_enabled != enable) &&
+		(dhdpcie_osinfo->oob_irq_num > 0)) {
+		if (enable) {
+			enable_irq(dhdpcie_osinfo->oob_irq_num);
+			bus->oob_intr_enable_count++;
+			bus->last_oob_irq_enable_time = OSL_LOCALTIME_NS();
+		} else {
+			disable_irq_nosync(dhdpcie_osinfo->oob_irq_num);
+			bus->oob_intr_disable_count++;
+			bus->last_oob_irq_disable_time = OSL_LOCALTIME_NS();
+		}
+		dhdpcie_osinfo->oob_irq_enabled = enable;
+	}
+	DHD_OOB_IRQ_UNLOCK(&dhdpcie_osinfo->oob_irq_spinlock, flags);
+}
+
+#if defined(DHD_USE_SPIN_LOCK_BH) && !defined(DHD_USE_PCIE_OOB_THREADED_IRQ)
+#error "Cannot enable DHD_USE_SPIN_LOCK_BH without enabling DHD_USE_PCIE_OOB_THREADED_IRQ"
+#endif /* DHD_USE_SPIN_LOCK_BH && !DHD_USE_PCIE_OOB_THREADED_IRQ */
+
+#ifdef DHD_USE_PCIE_OOB_THREADED_IRQ
+static irqreturn_t wlan_oob_irq_isr(int irq, void *data)
+{
+	dhd_bus_t *bus = (dhd_bus_t *)data;
+	dhdpcie_oob_intr_set(bus, FALSE);
+	DHD_INTR(("%s: IRQ ISR\n", __FUNCTION__));
+	bus->last_oob_irq_isr_time = OSL_LOCALTIME_NS();
+	return IRQ_WAKE_THREAD;
+}
+#endif /* DHD_USE_PCIE_OOB_THREADED_IRQ */
+
+static irqreturn_t wlan_oob_irq(int irq, void *data)
+{
+	dhd_bus_t *bus;
+	bus = (dhd_bus_t *)data;
+#ifdef DHD_USE_PCIE_OOB_THREADED_IRQ
+	DHD_INTR(("%s: IRQ Thread\n", __FUNCTION__));
+	bus->last_oob_irq_thr_time = OSL_LOCALTIME_NS();
+#else
+	dhdpcie_oob_intr_set(bus, FALSE);
+	DHD_INTR(("%s: IRQ ISR\n", __FUNCTION__));
+	bus->last_oob_irq_isr_time = OSL_LOCALTIME_NS();
+#endif /* DHD_USE_PCIE_OOB_THREADED_IRQ */
+
+	if (bus->dhd->up == 0) {
+		DHD_ERROR(("%s: ########### IRQ during dhd pub up is 0 ############\n",
+			__FUNCTION__));
+	}
+
+	bus->oob_intr_count++;
+#ifdef DHD_WAKE_STATUS
+	{
+		bcmpcie_set_get_wake(bus, 1);
+	}
+#endif /* DHD_WAKE_STATUS */
+#ifdef DHD_PCIE_NATIVE_RUNTIMEPM
+	dhd_bus_wakeup_work(bus->dhd);
+#endif /* DHD_PCIE_NATIVE_RUNTIMEPM */
+	/* Hold wakelock if bus_low_power_state is
+	 * DHD_BUS_D3_INFORM_SENT OR DHD_BUS_D3_ACK_RECIEVED
+	 */
+	if (bus->dhd->up && DHD_CHK_BUS_IN_LPS(bus)) {
+		DHD_OS_OOB_IRQ_WAKE_LOCK_TIMEOUT(bus->dhd, OOB_WAKE_LOCK_TIMEOUT);
+	}
+	return IRQ_HANDLED;
+}
+
+int dhdpcie_oob_intr_register(dhd_bus_t *bus)
+{
+	int err = 0;
+	dhdpcie_info_t *pch;
+	dhdpcie_os_info_t *dhdpcie_osinfo;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		DHD_ERROR(("%s: pch is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	dhdpcie_osinfo = (dhdpcie_os_info_t *)pch->os_cxt;
+	if (dhdpcie_osinfo->oob_irq_registered) {
+		DHD_ERROR(("%s: irq is already registered\n", __FUNCTION__));
+		return -EBUSY;
+	}
+
+	if (dhdpcie_osinfo->oob_irq_num > 0) {
+		printf("%s OOB irq=%d flags=0x%X\n", __FUNCTION__,
+			(int)dhdpcie_osinfo->oob_irq_num,
+			(int)dhdpcie_osinfo->oob_irq_flags);
+#ifdef DHD_USE_PCIE_OOB_THREADED_IRQ
+		err = request_threaded_irq(dhdpcie_osinfo->oob_irq_num,
+			wlan_oob_irq_isr, wlan_oob_irq,
+			dhdpcie_osinfo->oob_irq_flags, "dhdpcie_host_wake",
+			bus);
+#else
+		err = request_irq(dhdpcie_osinfo->oob_irq_num, wlan_oob_irq,
+			dhdpcie_osinfo->oob_irq_flags, "dhdpcie_host_wake",
+			bus);
+#endif /* DHD_USE_THREADED_IRQ_PCIE_OOB */
+		if (err) {
+			DHD_ERROR(("%s: request_irq failed with %d\n",
+				__FUNCTION__, err));
+			return err;
+		}
+#if defined(DISABLE_WOWLAN)
+		printf("%s: disable_irq_wake\n", __FUNCTION__);
+		dhdpcie_osinfo->oob_irq_wake_enabled = FALSE;
+#else
+		printf("%s: enable_irq_wake\n", __FUNCTION__);
+		err = enable_irq_wake(dhdpcie_osinfo->oob_irq_num);
+		if (!err) {
+			dhdpcie_osinfo->oob_irq_wake_enabled = TRUE;
+		} else
+			printf("%s: enable_irq_wake failed with %d\n", __FUNCTION__, err);
+#endif
+		dhdpcie_osinfo->oob_irq_enabled = TRUE;
+	}
+
+	dhdpcie_osinfo->oob_irq_registered = TRUE;
+
+	return 0;
+}
+
+void dhdpcie_oob_intr_unregister(dhd_bus_t *bus)
+{
+	int err = 0;
+	dhdpcie_info_t *pch;
+	dhdpcie_os_info_t *dhdpcie_osinfo;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	if (bus == NULL) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	if (bus->dev == NULL) {
+		DHD_ERROR(("%s: bus->dev is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	pch = pci_get_drvdata(bus->dev);
+	if (pch == NULL) {
+		DHD_ERROR(("%s: pch is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	dhdpcie_osinfo = (dhdpcie_os_info_t *)pch->os_cxt;
+	if (!dhdpcie_osinfo->oob_irq_registered) {
+		DHD_ERROR(("%s: irq is not registered\n", __FUNCTION__));
+		return;
+	}
+	if (dhdpcie_osinfo->oob_irq_num > 0) {
+		if (dhdpcie_osinfo->oob_irq_wake_enabled) {
+			err = disable_irq_wake(dhdpcie_osinfo->oob_irq_num);
+			if (!err) {
+				dhdpcie_osinfo->oob_irq_wake_enabled = FALSE;
+			}
+		}
+		if (dhdpcie_osinfo->oob_irq_enabled) {
+			disable_irq(dhdpcie_osinfo->oob_irq_num);
+			dhdpcie_osinfo->oob_irq_enabled = FALSE;
+		}
+		free_irq(dhdpcie_osinfo->oob_irq_num, bus);
+	}
+	dhdpcie_osinfo->oob_irq_registered = FALSE;
+}
+#endif /* BCMPCIE_OOB_HOST_WAKE */
+
+#ifdef PCIE_OOB
+void dhdpcie_oob_init(dhd_bus_t *bus)
+{
+	/* XXX this should be passed in as a command line parameter */
+	gpio_handle_val = get_handle(OOB_PORT);
+	if (gpio_handle_val < 0)
+	{
+		DHD_ERROR(("%s: Could not get GPIO handle.\n", __FUNCTION__));
+		ASSERT(FALSE);
+	}
+
+	gpio_direction = 0;
+	ftdi_set_bitmode(gpio_handle_val, 0, BITMODE_BITBANG);
+
+	/* Note BT core is also enabled here */
+	gpio_port = 1 << BIT_WL_REG_ON | 1 << BIT_BT_REG_ON | 1 << DEVICE_WAKE;
+	gpio_write_port(gpio_handle_val, gpio_port);
+
+	gpio_direction = 1 << BIT_WL_REG_ON | 1 << BIT_BT_REG_ON | 1 << DEVICE_WAKE;
+	ftdi_set_bitmode(gpio_handle_val, gpio_direction, BITMODE_BITBANG);
+
+	bus->oob_enabled = TRUE;
+	bus->oob_presuspend = FALSE;
+
+	/* drive the Device_Wake GPIO low on startup */
+	bus->device_wake_state = TRUE;
+	dhd_bus_set_device_wake(bus, FALSE);
+	dhd_bus_doorbell_timeout_reset(bus);
+
+}
+
+void
+dhd_oob_set_bt_reg_on(struct dhd_bus *bus, bool val)
+{
+	DHD_INFO(("Set Device_Wake to %d\n", val));
+	if (val)
+	{
+		gpio_port = gpio_port | (1 << BIT_BT_REG_ON);
+		gpio_write_port(gpio_handle_val, gpio_port);
+	} else {
+		gpio_port = gpio_port & (0xff ^ (1 << BIT_BT_REG_ON));
+		gpio_write_port(gpio_handle_val, gpio_port);
+	}
+}
+
+int
+dhd_oob_get_bt_reg_on(struct dhd_bus *bus)
+{
+	int ret;
+	uint8 val;
+	ret = gpio_read_port(gpio_handle_val, &val);
+
+	if (ret < 0) {
+		/* XXX handle error properly */
+		DHD_ERROR(("gpio_read_port returns %d\n", ret));
+		return ret;
+	}
+
+	if (val & (1 << BIT_BT_REG_ON))
+	{
+		ret = 1;
+	} else {
+		ret = 0;
+	}
+
+	return ret;
+}
+
+int
+dhd_os_oob_set_device_wake(struct dhd_bus *bus, bool val)
+{
+	if (bus->device_wake_state != val)
+	{
+		DHD_INFO(("Set Device_Wake to %d\n", val));
+
+		if (bus->oob_enabled && !bus->oob_presuspend)
+		{
+			if (val)
+			{
+				gpio_port = gpio_port | (1 << DEVICE_WAKE);
+				gpio_write_port_non_block(gpio_handle_val, gpio_port);
+			} else {
+				gpio_port = gpio_port & (0xff ^ (1 << DEVICE_WAKE));
+				gpio_write_port_non_block(gpio_handle_val, gpio_port);
+			}
+		}
+
+		bus->device_wake_state = val;
+	}
+	return BCME_OK;
+}
+
+INLINE void
+dhd_os_ib_set_device_wake(struct dhd_bus *bus, bool val)
+{
+	/* TODO: Currently Inband implementation of Device_Wake is not supported,
+	 * so this function is left empty later this can be used to support the same.
+	 */
+}
+#endif /* PCIE_OOB */
+
+struct device * dhd_bus_to_dev(dhd_bus_t *bus)
+{
+	struct pci_dev *pdev;
+	pdev = bus->dev;
+
+	if (pdev)
+		return &pdev->dev;
+	else
+		return NULL;
+}
+
+#ifdef DHD_FW_COREDUMP
+int
+dhd_dongle_mem_dump(void)
+{
+	if (!g_dhd_bus) {
+		DHD_ERROR(("%s: Bus is NULL\n", __FUNCTION__));
+		return -ENODEV;
+	}
+
+	dhd_bus_dump_console_buffer(g_dhd_bus);
+	dhd_prot_debug_info_print(g_dhd_bus->dhd);
+
+	g_dhd_bus->dhd->memdump_enabled = DUMP_MEMFILE_BUGON;
+	g_dhd_bus->dhd->memdump_type = DUMP_TYPE_AP_ABNORMAL_ACCESS;
+
+	dhd_bus_mem_dump(g_dhd_bus->dhd);
+	return 0;
+}
+#ifndef BCMDHD_MDRIVER
+EXPORT_SYMBOL(dhd_dongle_mem_dump);
+#endif
+#endif /* DHD_FW_COREDUMP */
+
+#if defined(CONFIG_ARCH_MSM) && defined(CONFIG_SEC_PCIE_L1SS)
+void
+dhd_bus_inform_ep_loaded_to_rc(dhd_pub_t *dhdp, bool up)
+{
+	if (dhdp->bus->rc_dev) {
+		sec_pcie_set_ep_driver_loaded(dhdp->bus->rc_dev, up);
+	}
+}
+#endif /* CONFIG_ARCH_MSM && CONFIG_SEC_PCIE_L1SS */
+
+bool
+dhd_bus_check_driver_up(void)
+{
+	dhd_bus_t *bus;
+	dhd_pub_t *dhdp;
+	bool isup = FALSE;
+
+	bus = (dhd_bus_t *)g_dhd_bus;
+	if (!bus) {
+		DHD_ERROR(("%s: bus is NULL\n", __FUNCTION__));
+		return isup;
+	}
+
+	dhdp = bus->dhd;
+	if (dhdp) {
+		isup = dhdp->up;
+	}
+
+	return isup;
+}
+#ifndef BCMDHD_MDRIVER
+EXPORT_SYMBOL(dhd_bus_check_driver_up);
+#endif
diff --git a/drivers/net/wireless/ap12275/dhd_pktlog.c b/drivers/net/wireless/ap12275/dhd_pktlog.c
new file mode 100644
index 000000000000..2ddb493da11d
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pktlog.c
@@ -0,0 +1,2050 @@
+/*
+ * DHD debugability packet logging support
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+#include <bcmutils.h>
+#include <bcmstdlib_s.h>
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_dbg.h>
+#include <dhd_pktlog.h>
+#include <dhd_wlfc.h>
+#include <dhd_debug.h>
+#ifdef LINUX
+#include <linux/vmalloc.h>
+#endif /* LINUX */
+
+#ifdef DHD_COMPACT_PKT_LOG
+#include <bcmip.h>
+#include <bcmudp.h>
+#include <bcmdhcp.h>
+#include <bcmarp.h>
+#include <bcmicmp.h>
+#include <bcmtlv.h>
+#include <802.11.h>
+#include <802.11u.h>
+#include <eap.h>
+#include <eapol.h>
+#include <bcmendian.h>
+#include <bcm_l2_filter.h>
+#include <dhd_bitpack.h>
+#include <bcmipv6.h>
+#endif	/* DHD_COMPACT_PKT_LOG */
+
+#ifdef DHD_PKT_LOGGING
+#ifndef strtoul
+#define strtoul(nptr, endptr, base) bcm_strtoul((nptr), (endptr), (base))
+#endif /* strtoul */
+extern int wl_pattern_atoh(char *src, char *dst);
+extern int pattern_atoh_len(char *src, char *dst, int len);
+extern wifi_tx_packet_fate __dhd_dbg_map_tx_status_to_pkt_fate(uint16 status);
+
+#ifdef DHD_COMPACT_PKT_LOG
+#define CPKT_LOG_BITS_PER_BYTE		8
+
+#define CPKT_LOG_BIT_LEN_TYPE		4
+
+#define CPKT_LOG_BIT_OFFSET_TS		0
+#define CPKT_LOG_BIT_OFFSET_DIR		5
+#define CPKT_LOG_BIT_OFFSET_TYPE	6
+#define CPKT_LOG_BIT_OFFSET_SUBTYPE	10
+#define CPKT_LOG_BIT_OFFSET_PKT_FATE	18
+
+#define CPKT_LOG_BIT_MASK_TS		0x1f
+#define CPKT_LOG_BIT_MASK_DIR		0x01
+#define CPKT_LOG_BIT_MASK_TYPE		0x0f
+#define CPKT_LOG_BIT_MASK_SUBTYPE	0xff
+#define CPKT_LOG_BIT_MASK_PKT_FATE	0x0f
+
+#define CPKT_LOG_DNS_PORT_CLIENT	53
+#define CPKT_LOG_MDNS_PORT_CLIENT	5353
+
+#define CPKT_LOG_TYPE_DNS		0x0
+#define CPKT_LOG_TYPE_ARP		0x1
+#define CPKT_LOG_TYPE_ICMP_REQ		0x2
+#define CPKT_LOG_TYPE_ICMP_RES		0x3
+#define CPKT_LOG_TYPE_ICMP_UNREACHABLE	0x4
+#define CPKT_LOG_TYPE_DHCP		0x5
+#define CPKT_LOG_TYPE_802_1X		0x6
+#define CPKT_LOG_TYPE_ICMPv6		0x7
+#define CPKT_LOG_TYPE_OTHERS		0xf
+
+#define CPKT_LOG_802_1X_SUBTYPE_IDENTITY	0x0
+#define CPKT_LOG_802_1X_SUBTYPE_TLS		0x1
+#define CPKT_LOG_802_1X_SUBTYPE_TTLS		0x2
+#define CPKT_LOG_802_1X_SUBTYPE_PEAP		0x3
+#define CPKT_LOG_802_1X_SUBTYPE_FAST		0x4
+#define CPKT_LOG_802_1X_SUBTYPE_LEAP		0x5
+#define CPKT_LOG_802_1X_SUBTYPE_PWD		0x6
+#define CPKT_LOG_802_1X_SUBTYPE_SIM		0x7
+#define CPKT_LOG_802_1X_SUBTYPE_AKA		0x8
+#define CPKT_LOG_802_1X_SUBTYPE_AKAP		0x9
+#define CPKT_LOG_802_1X_SUBTYPE_SUCCESS		0xA
+#define CPKT_LOG_802_1X_SUBTYPE_4WAY_M1		0xB
+#define CPKT_LOG_802_1X_SUBTYPE_4WAY_M2		0xC
+#define CPKT_LOG_802_1X_SUBTYPE_4WAY_M3		0xD
+#define CPKT_LOG_802_1X_SUBTYPE_4WAY_M4		0xE
+#define CPKT_LOG_802_1X_SUBTYPE_OTHERS		0xF
+
+#define CPKT_LOG_DHCP_MAGIC_COOKIE_LEN			4
+
+#define CPKT_LOG_ICMP_TYPE_DEST_UNREACHABLE		3
+#define CPKT_LOG_ICMP_TYPE_DEST_UNREACHABLE_IPV4_OFFSET	4
+
+typedef struct dhd_cpkt_log_ts_node {
+	struct rb_node rb;
+
+	uint64 ts_diff;		/* key, usec */
+	int idx;
+} dhd_cpkt_log_ts_node_t;
+
+/* Compact Packet Log Timestamp values, unit: uSec */
+const uint64 dhd_cpkt_log_tt_idx[] = {
+	10000, 50000, 100000, 150000, 300000, 500000, 750000, 1000000, 3000000, 5000000, 7500000,
+	10000000, 12500000, 15000000, 17500000, 20000000, 22500000, 25000000, 27500000, 30000000,
+	32500000, 35000000, 37500000, 40000000, 50000000, 75000000, 150000000, 300000000, 400000000,
+	500000000, 600000000
+};
+#define CPKT_LOG_TT_IDX_ARR_SZ	ARRAYSIZE(dhd_cpkt_log_tt_idx)
+
+static int dhd_cpkt_log_init_tt(dhd_pub_t *dhdp);
+static void dhd_cpkt_log_deinit_tt(dhd_pub_t *dhdp);
+#endif	/* DHD_COMPACT_PKT_LOG */
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+#define DHD_PKT_LOGGING_DBGRING_MAX_SIZE	400u
+const char dhd_eap_pattern[] = "12 0xffff 0x888e";
+const char dhd_arp_pattern[] = "12 0xffff 0x0806";
+const char dhd_icmp_pattern[] =
+	"12 0xffffff0000000000000000ff 0x080045000000000000000001";
+const char dhd_icmpv6_pattern[] =
+	"12 0xfffff00000000000ff 0x86dd6000000000003a";
+const char dhd_dhcp_pattern_01[] =
+	"12 0xffffff0000000000000000ff00000000000000000000ffffffff"
+	" 0x0800450000000000000000110000000000000000000000430044";
+const char dhd_dhcp_pattern_02[] =
+	"12 0xffffff0000000000000000ff00000000000000000000ffffffff"
+	" 0x0800450000000000000000110000000000000000000000440043";
+const char dhd_dns_pattern_01[] =
+	"12 0xffffff0000000000000000ff000000000000000000000000ffff"
+	" 0x0800450000000000000000110000000000000000000000000035";
+const char dhd_dns_pattern_02[] =
+	"12 0xffffff0000000000000000ff00000000000000000000ffff0000"
+	" 0x0800450000000000000000110000000000000000000000350000";
+const char dhd_dns_pattern_03[] =
+	"12 0xffffff0000000000000000ff00000000000000000000ffffffff"
+	" 0x0800450000000000000000110000000000000000000014e914e9";
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+int
+dhd_os_attach_pktlog(dhd_pub_t *dhdp)
+{
+#ifdef DHD_PKT_LOGGING_DBGRING
+	char filter_pattern[MAX_FILTER_PATTERN_LEN];
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	dhd_pktlog_t *pktlog;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s(): dhdp is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	pktlog = (dhd_pktlog_t *)VMALLOCZ(dhdp->osh, sizeof(dhd_pktlog_t));
+	if (unlikely(!pktlog)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+					"dhd_pktlog_t\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	dhdp->pktlog = pktlog;
+	pktlog->dhdp = dhdp;
+
+	OSL_ATOMIC_INIT(dhdp->osh, &pktlog->pktlog_status);
+#ifdef DHD_PKT_LOGGING_DBGRING
+	OSL_ATOMIC_INIT(dhdp->osh, &pktlog->enable);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	/* pktlog ring */
+	dhdp->pktlog->pktlog_ring = dhd_pktlog_ring_init(dhdp, MIN_PKTLOG_LEN);
+	dhdp->pktlog->pktlog_filter = dhd_pktlog_filter_init(MAX_DHD_PKTLOG_FILTER_LEN);
+#ifdef DHD_COMPACT_PKT_LOG
+	dhd_cpkt_log_init_tt(dhdp);
+#endif
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+/* By default, EAP, ARP, ICMP, ICMPV6, DHCP packets are allowed. */
+	DHD_PKTLOG_FILTER_ADD(dhd_eap_pattern, filter_pattern, dhdp);
+	DHD_PKTLOG_FILTER_ADD(dhd_arp_pattern, filter_pattern, dhdp);
+	DHD_PKTLOG_FILTER_ADD(dhd_icmp_pattern, filter_pattern, dhdp);
+	DHD_PKTLOG_FILTER_ADD(dhd_icmpv6_pattern, filter_pattern, dhdp);
+	DHD_PKTLOG_FILTER_ADD(dhd_dhcp_pattern_01, filter_pattern, dhdp);
+	DHD_PKTLOG_FILTER_ADD(dhd_dhcp_pattern_02, filter_pattern, dhdp);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	DHD_ERROR(("%s(): dhd_os_attach_pktlog attach\n", __FUNCTION__));
+
+	return BCME_OK;
+}
+
+int
+dhd_os_detach_pktlog(dhd_pub_t *dhdp)
+{
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_PKT_LOG(("%s(): dhdp=%p pktlog=%p\n",
+			__FUNCTION__, dhdp, (dhdp ? dhdp->pktlog : NULL)));
+		return -EINVAL;
+	}
+
+	dhd_pktlog_ring_deinit(dhdp, dhdp->pktlog->pktlog_ring);
+	dhd_pktlog_filter_deinit(dhdp->pktlog->pktlog_filter);
+#ifdef DHD_COMPACT_PKT_LOG
+	dhd_cpkt_log_deinit_tt(dhdp);
+#endif	/* DHD_COMPACT_PKT_LOG */
+
+	DHD_INFO(("%s(): dhd_os_attach_pktlog detach\n", __FUNCTION__));
+
+	VMFREE(dhdp->osh, dhdp->pktlog, sizeof(dhd_pktlog_t));
+
+	return BCME_OK;
+}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+int dhd_pktlog_is_enabled(dhd_pub_t *dhdp)
+{
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_ERROR(("%s(): dhdp=%p pktlog=%p\n",
+			__FUNCTION__, dhdp, (dhdp ? dhdp->pktlog : NULL)));
+		return BCME_ERROR;
+	}
+
+	if (!dhdp->pktlog->pktlog_ring) {
+		DHD_ERROR(("%s(): pktlog_ring=%p\n",
+			__FUNCTION__, dhdp->pktlog->pktlog_ring));
+		return BCME_ERROR;
+	}
+
+	return OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->enable);
+}
+
+void dhd_pktlog_resume(dhd_pub_t *dhdp)
+{
+	struct dhd_pktlog_ring *pktlog_ring;
+
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_ERROR(("%s(): dhdp=%p pktlog=%p\n",
+			__FUNCTION__, dhdp, (dhdp ? dhdp->pktlog : NULL)));
+		return;
+	}
+
+	if (!dhdp->pktlog->pktlog_ring) {
+		DHD_ERROR(("%s(): pktlog_ring=%p\n",
+			__FUNCTION__, dhdp->pktlog->pktlog_ring));
+		return;
+	}
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	if (!OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->enable)) {
+		OSL_ATOMIC_SET(dhdp->osh, &dhdp->pktlog->enable, TRUE);
+		DHD_INFO(("pktlog ring is resumed. pktcount(%d)\n",
+			pktlog_ring->pktcount));
+	}
+}
+
+void dhd_pktlog_suspend(dhd_pub_t *dhdp)
+{
+	struct dhd_pktlog_ring *pktlog_ring;
+
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_ERROR(("%s(): dhdp=%p pktlog=%p\n",
+			__FUNCTION__, dhdp, (dhdp ? dhdp->pktlog : NULL)));
+		return;
+	}
+
+	if (!dhdp->pktlog->pktlog_ring) {
+		DHD_ERROR(("%s(): pktlog_ring=%p\n",
+			__FUNCTION__, dhdp->pktlog->pktlog_ring));
+		return;
+	}
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	if (OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->enable)) {
+		OSL_ATOMIC_SET(dhdp->osh, &dhdp->pktlog->enable, FALSE);
+		DHD_INFO(("pktlog ring is suspended. pktcount(%d)\n",
+			pktlog_ring->pktcount));
+	}
+}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+dhd_pktlog_ring_t*
+dhd_pktlog_ring_init(dhd_pub_t *dhdp, int size)
+{
+	dhd_pktlog_ring_t *ring;
+	int i = 0;
+
+	if (!dhdp) {
+		DHD_ERROR(("%s(): dhdp is NULL\n", __FUNCTION__));
+		return NULL;
+	}
+
+	ring = (dhd_pktlog_ring_t *)VMALLOCZ(dhdp->osh, sizeof(dhd_pktlog_ring_t));
+	if (unlikely(!ring)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+					"dhd_pktlog_ring_t\n", __FUNCTION__));
+		goto fail;
+	}
+
+	dll_init(&ring->ring_info_head);
+	dll_init(&ring->ring_info_free);
+
+	ring->ring_info_mem = (dhd_pktlog_ring_info_t *)VMALLOCZ(dhdp->osh,
+		sizeof(dhd_pktlog_ring_info_t) * size);
+	if (unlikely(!ring->ring_info_mem)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+					"dhd_pktlog_ring_info_t\n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* initialize free ring_info linked list */
+	for (i = 0; i < size; i++) {
+	    dll_append(&ring->ring_info_free, (dll_t *)&ring->ring_info_mem[i].p_info);
+	}
+
+	OSL_ATOMIC_SET(dhdp->osh, &ring->start, TRUE);
+	ring->pktlog_minmize = FALSE;
+	ring->pktlog_len = size;
+	ring->pktcount = 0;
+	ring->dhdp = dhdp;
+	ring->pktlog_ring_lock = osl_spin_lock_init(dhdp->osh);
+#ifdef DHD_PKT_LOGGING_DBGRING
+	OSL_ATOMIC_SET(dhdp->osh, &dhdp->pktlog->enable, TRUE);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	DHD_INFO(("%s(): pktlog ring init success\n", __FUNCTION__));
+
+	return ring;
+fail:
+	if (ring) {
+		VMFREE(dhdp->osh, ring, sizeof(dhd_pktlog_ring_t));
+	}
+
+	return NULL;
+}
+
+/* Maximum wait counts */
+#define DHD_PKTLOG_WAIT_MAXCOUNT 1000
+int
+dhd_pktlog_ring_deinit(dhd_pub_t *dhdp, dhd_pktlog_ring_t *ring)
+{
+	int ret = BCME_OK;
+	dhd_pktlog_ring_info_t *ring_info;
+	dll_t *item, *next_p;
+	int waitcounts = 0;
+
+	if (!ring) {
+		DHD_ERROR(("%s(): ring is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	if (!ring->dhdp) {
+		DHD_ERROR(("%s(): dhdp is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	/* stop pkt log */
+	OSL_ATOMIC_SET(dhdp->osh, &ring->start, FALSE);
+
+	/* waiting TX/RX/TXS context is done, max timeout 1 second */
+	while ((waitcounts++ < DHD_PKTLOG_WAIT_MAXCOUNT)) {
+		if (!OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->pktlog_status))
+			break;
+		OSL_SLEEP(1);
+	}
+
+	if (waitcounts >= DHD_PKTLOG_WAIT_MAXCOUNT) {
+		DHD_ERROR(("%s(): pktlog wait timeout pktlog_status : 0x%x \n",
+			__FUNCTION__,
+			OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->pktlog_status)));
+		ASSERT(0);
+		return -EINVAL;
+	}
+
+	/* free ring_info->info.pkt */
+	for (item = dll_head_p(&ring->ring_info_head); !dll_end(&ring->ring_info_head, item);
+		item = next_p) {
+		next_p = dll_next_p(item);
+
+		ring_info = (dhd_pktlog_ring_info_t *)item;
+
+		if (ring_info->info.pkt) {
+			PKTFREE(ring->dhdp->osh, ring_info->info.pkt, TRUE);
+			DHD_PKT_LOG(("%s(): pkt free pos %p\n",
+					__FUNCTION__, ring_info->info.pkt));
+		}
+	}
+
+	if (ring->ring_info_mem) {
+		VMFREE(ring->dhdp->osh, ring->ring_info_mem,
+			sizeof(dhd_pktlog_ring_info_t) * ring->pktlog_len);
+	}
+
+	if (ring->pktlog_ring_lock) {
+		osl_spin_lock_deinit(ring->dhdp->osh, ring->pktlog_ring_lock);
+	}
+
+	VMFREE(dhdp->osh, ring, sizeof(dhd_pktlog_ring_t));
+
+	DHD_INFO(("%s(): pktlog ring deinit\n", __FUNCTION__));
+
+	return ret;
+}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+int
+dhd_pktlog_ring_reinit(dhd_pub_t *dhdp)
+{
+	dhd_pktlog_ring_info_t *ring_info;
+	dhd_pktlog_ring_t *pktlog_ring;
+	dll_t *item, *next_p;
+	int waitcounts = 0;
+	dhd_dbg_ring_t *ring;
+
+	DHD_ERROR(("%s: ENTER\n", __FUNCTION__));
+
+	if (!dhdp) {
+		DHD_ERROR(("%s(): dhdp is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	ring = &dhdp->dbg->dbg_rings[PACKET_LOG_RING_ID];
+
+	if (!ring) {
+		DHD_ERROR(("%s(): ring is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	/* stop pkt log */
+	OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, FALSE);
+
+	/* waiting TX/RX/TXS context is done, max timeout 1 second */
+	while ((waitcounts++ < DHD_PKTLOG_WAIT_MAXCOUNT)) {
+		if (!OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->pktlog_status))
+			break;
+		OSL_SLEEP(1);
+	}
+
+	if (waitcounts >= DHD_PKTLOG_WAIT_MAXCOUNT) {
+		DHD_ERROR(("%s(): pktlog wait timeout pktlog_status : 0x%x \n",
+			__FUNCTION__,
+			OSL_ATOMIC_READ(dhdp->osh, &dhdp->pktlog->pktlog_status)));
+		ASSERT(0);
+		return -EINVAL;
+	}
+
+	/* free ring_info->info.pkt */
+	for (item = dll_head_p(&pktlog_ring->ring_info_head);
+		!dll_end(&pktlog_ring->ring_info_head, item);
+		item = next_p) {
+		next_p = dll_next_p(item);
+
+		ring_info = (dhd_pktlog_ring_info_t *)item;
+
+		dll_delete((dll_t *)item);
+		if (ring_info->info.pkt) {
+			PKTFREE(pktlog_ring->dhdp->osh, ring_info->info.pkt, TRUE);
+			DHD_PKT_LOG(("%s(): pkt free pos %p\n",
+				__FUNCTION__, ring_info->info.pkt));
+		}
+		dll_append(&pktlog_ring->ring_info_free, (dll_t *)item);
+	}
+
+	/* reset stats and pktcount */
+	pktlog_ring->pktcount = 0;
+	memset(&ring->stat, 0, sizeof(struct ring_statistics));
+	/* start pkt log */
+	OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, TRUE);
+#ifdef DHD_PKT_LOGGING_DBGRING
+	OSL_ATOMIC_SET(dhdp->osh, &dhdp->pktlog->enable, TRUE);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	DHD_ERROR(("%s: EXIT\n", __FUNCTION__));
+
+	return BCME_OK;
+}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+/*
+ * dhd_pktlog_ring_add_pkts : add filtered packets into pktlog ring
+ * pktid : incase of rx, pktid is not used (pass DHD_INVALID_PKID)
+ * direction :  1 - TX / 0 - RX / 2 - RX Wakeup Packet
+ */
+int
+dhd_pktlog_ring_add_pkts(dhd_pub_t *dhdp, void *pkt, void *pktdata, uint32 pktid, uint32 direction)
+{
+	dhd_pktlog_ring_info_t *pkts;
+	dhd_pktlog_ring_t *pktlog_ring;
+	dhd_pktlog_filter_t *pktlog_filter;
+	uint32 pktlog_case = 0;
+	unsigned long flags = 0;
+#ifdef DHD_PKT_LOGGING_DBGRING
+	struct timespec64 ts;
+	dhd_dbg_ring_t *ring;
+	ring = &dhdp->dbg->dbg_rings[PACKET_LOG_RING_ID];
+#else
+	u64 ts_nsec;
+	unsigned long rem_nsec;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	/*
+	 * dhdp, dhdp->pktlog, dhd->pktlog_ring, pktlog_ring->start
+	 * are validated from the DHD_PKTLOG_TX macro
+	 */
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	pktlog_filter = dhdp->pktlog->pktlog_filter;
+
+	if (direction == PKT_TX) {
+		pktlog_case = PKTLOG_TXPKT_CASE;
+	} else if ((direction == PKT_RX) || (direction == PKT_WAKERX)) {
+		pktlog_case = PKTLOG_RXPKT_CASE;
+	}
+
+	if (pktlog_filter && pktlog_filter->list_cnt == 0) {
+		/* There is no packet log list */
+		return BCME_OK;
+	}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (ring->log_level == RING_LOG_LEVEL_EXCESSIVE) {
+		/* In excessive log level, pass through without pattern matching */
+	} else
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	if ((direction != PKT_WAKERX) &&
+		dhd_pktlog_filter_matched(pktlog_filter, pktdata, pktlog_case)
+		== FALSE) {
+		return BCME_OK;
+	}
+
+	if (direction == PKT_TX && pktid == DHD_INVALID_PKTID) {
+		DHD_ERROR(("%s : Invalid PKTID \n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* get free ring_info and insert to ring_info_head */
+	DHD_PKT_LOG_LOCK(pktlog_ring->pktlog_ring_lock, flags);
+	/* if free_list is empty, use the oldest ring_info */
+	if (dll_empty(&pktlog_ring->ring_info_free)) {
+		pkts = (dhd_pktlog_ring_info_t *)dll_head_p(&pktlog_ring->ring_info_head);
+		dll_delete((dll_t *)pkts);
+		/* free the oldest packet */
+		PKTFREE(pktlog_ring->dhdp->osh, pkts->info.pkt, TRUE);
+		pktlog_ring->pktcount--;
+	} else {
+		pkts = (dhd_pktlog_ring_info_t *)dll_tail_p(&pktlog_ring->ring_info_free);
+		dll_delete((dll_t *)pkts);
+	}
+
+	/* Update packet information */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	ktime_get_real_ts64(&ts);
+
+	pkts->info.driver_ts_sec = (uint32)ts.tv_sec;
+	pkts->info.driver_ts_usec = (uint32)(ts.tv_nsec/NSEC_PER_USEC);
+#else
+	ts_nsec = local_clock();
+	rem_nsec = do_div(ts_nsec, NSEC_PER_SEC);
+
+	pkts->info.driver_ts_sec = (uint32)ts_nsec;
+	pkts->info.driver_ts_usec = (uint32)(rem_nsec/NSEC_PER_USEC);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	pkts->info.pkt = PKTDUP(dhdp->osh, pkt);
+	/*
+	 * skb clone can be NULL, but pktlog feature assume it alway  can be cloned
+	 * The dummy pkt info will be added in the list to fit pktcount & list items
+	 * and handled in the dhd_pktlog_dump_write() with ignoring info.pkt
+	 */
+	if (pkts->info.pkt == NULL) {
+		DHD_ERROR(("%s : skb clone returns NULL \n", __FUNCTION__));
+	}
+	pkts->info.pkt_len = PKTLEN(dhdp->osh, pkt);
+	pkts->info.firmware_ts = 0U;
+	pkts->info.payload_type = FRAME_TYPE_ETHERNET_II;
+	pkts->info.direction = direction;
+
+	pkts->info.tx_status_ts_sec = 0;
+	pkts->info.tx_status_ts_usec = 0;
+
+	if (direction == PKT_TX) {
+		pkts->info.pkt_hash =  __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
+		pkts->tx_fate = TX_PKT_FATE_DRV_QUEUED;
+	} else if (direction == PKT_RX) {
+		pkts->info.pkt_hash = 0U;
+		pkts->rx_fate = RX_PKT_FATE_SUCCESS;
+	} else if (direction == PKT_WAKERX) {
+		pkts->info.pkt_hash = 0U;
+		pkts->rx_fate = RX_PKT_FATE_WAKE_PKT;
+	}
+
+	DHD_PKT_LOG(("%s(): pkt hash %d\n", __FUNCTION__, pkts->info.pkt_hash));
+	DHD_PKT_LOG(("%s(): sec %d usec %d\n", __FUNCTION__,
+		pkts->info.driver_ts_sec, pkts->info.driver_ts_usec));
+
+	/* insert tx_pkts to the pktlog_ring->ring_info_head */
+	dll_append(&pktlog_ring->ring_info_head, (dll_t *)pkts);
+	pktlog_ring->pktcount++;
+	DHD_PKT_LOG_UNLOCK(pktlog_ring->pktlog_ring_lock, flags);
+#ifdef DHD_PKT_LOGGING_DBGRING
+	dhd_dbg_update_to_ring(dhdp, pktlog_ring->dbg_ring,
+		dhd_pktlog_get_item_length(pkts));
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	DHD_PKT_LOG(("%s(): pktcount=%d\n", __FUNCTION__, pktlog_ring->pktcount));
+	return BCME_OK;
+}
+
+int
+dhd_pktlog_ring_tx_status(dhd_pub_t *dhdp, void *pkt, void *pktdata, uint32 pktid,
+		uint16 status)
+{
+	dhd_pktlog_ring_info_t *tx_pkt;
+	wifi_tx_packet_fate pkt_fate;
+	uint32 pkt_hash, temp_hash;
+	dhd_pktlog_ring_t *pktlog_ring;
+	dhd_pktlog_filter_t *pktlog_filter;
+	dll_t *item_p, *next_p;
+	unsigned long flags = 0;
+	u64 ts_nsec;
+	unsigned long rem_nsec;
+#ifdef BDC
+	struct bdc_header *h;
+	BCM_REFERENCE(h);
+#endif /* BDC */
+	/*
+	 * dhdp, dhdp->pktlog, dhd->pktlog_ring, pktlog_ring->start
+	 * are validated from the DHD_PKTLOG_TXS macro
+	 */
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	pktlog_filter = dhdp->pktlog->pktlog_filter;
+
+	if (dhd_pktlog_filter_matched(pktlog_filter, pktdata,
+		PKTLOG_TXSTATUS_CASE) == FALSE) {
+		return BCME_OK;
+	}
+
+	pkt_hash = __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
+	pkt_fate = __dhd_dbg_map_tx_status_to_pkt_fate(status);
+
+	ts_nsec = local_clock();
+	rem_nsec = do_div(ts_nsec, NSEC_PER_SEC);
+
+	/* find the sent tx packet and adding pkt_fate info */
+	DHD_PKT_LOG_LOCK(pktlog_ring->pktlog_ring_lock, flags);
+	/* Inverse traverse from the last packets */
+	for (item_p = dll_tail_p(&pktlog_ring->ring_info_head);
+		!dll_end(&pktlog_ring->ring_info_head, item_p);
+		item_p = next_p)
+	{
+	    if (dll_empty(item_p)) {
+		break;
+	    }
+	    next_p = dll_prev_p(item_p);
+	    tx_pkt = (dhd_pktlog_ring_info_t *)item_p;
+	    temp_hash = tx_pkt->info.pkt_hash;
+	    if (temp_hash == pkt_hash) {
+		tx_pkt->tx_fate = pkt_fate;
+#ifdef BDC
+		h = (struct bdc_header *)PKTDATA(dhdp->osh, tx_pkt->info.pkt);
+		PKTPULL(dhdp->osh, tx_pkt->info.pkt, BDC_HEADER_LEN);
+		PKTPULL(dhdp->osh, tx_pkt->info.pkt, (h->dataOffset << DHD_WORD_TO_LEN_SHIFT));
+#endif /* BDC */
+		tx_pkt->info.tx_status_ts_sec = (uint32)ts_nsec;
+		tx_pkt->info.tx_status_ts_usec = (uint32)(rem_nsec/NSEC_PER_USEC);
+		DHD_PKT_LOG(("%s(): Found pkt hash in prev pos\n", __FUNCTION__));
+		break;
+	    }
+	}
+	DHD_PKT_LOG_UNLOCK(pktlog_ring->pktlog_ring_lock, flags);
+	return BCME_OK;
+}
+
+dhd_pktlog_filter_t*
+dhd_pktlog_filter_init(int size)
+{
+	int i;
+	gfp_t kflags;
+	uint32 alloc_len;
+	dhd_pktlog_filter_t *filter;
+	dhd_pktlog_filter_info_t *filter_info = NULL;
+
+	kflags = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+
+	/* allocate and initialze pktmon filter */
+	alloc_len = sizeof(dhd_pktlog_filter_t);
+	filter = (dhd_pktlog_filter_t *)kzalloc(alloc_len, kflags);
+	if (unlikely(!filter)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+					"dhd_pktlog_filter_t\n", __FUNCTION__));
+		goto fail;
+	}
+
+	alloc_len = (sizeof(dhd_pktlog_filter_info_t) * size);
+	filter_info = (dhd_pktlog_filter_info_t *)kzalloc(alloc_len, kflags);
+	if (unlikely(!filter_info)) {
+		DHD_ERROR(("%s(): could not allocate memory for - "
+					"dhd_pktlog_filter_info_t\n", __FUNCTION__));
+		goto fail;
+	}
+
+	filter->info = filter_info;
+	filter->list_cnt = 0;
+
+	for (i = 0; i < MAX_DHD_PKTLOG_FILTER_LEN; i++) {
+		filter->info[i].id = 0;
+	}
+
+	filter->enable = PKTLOG_TXPKT_CASE | PKTLOG_TXSTATUS_CASE | PKTLOG_RXPKT_CASE;
+
+	DHD_INFO(("%s(): pktlog filter init success\n", __FUNCTION__));
+
+	return filter;
+fail:
+	if (filter) {
+		kfree(filter);
+	}
+
+	return NULL;
+}
+
+int
+dhd_pktlog_filter_deinit(dhd_pktlog_filter_t *filter)
+{
+	int ret = BCME_OK;
+
+	if (!filter) {
+		DHD_ERROR(("%s(): filter is NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+
+	if (filter->info) {
+		kfree(filter->info);
+	}
+	kfree(filter);
+
+	DHD_INFO(("%s(): pktlog filter deinit\n", __FUNCTION__));
+
+	return ret;
+}
+
+bool
+dhd_pktlog_filter_existed(dhd_pktlog_filter_t *filter, char *arg, uint32 *id)
+{
+	char filter_pattern[MAX_FILTER_PATTERN_LEN];
+	char *p;
+	int i, j;
+	int nchar;
+	int len;
+
+	if  (!filter || !arg) {
+		DHD_ERROR(("%s(): filter=%p arg=%p\n", __FUNCTION__, filter, arg));
+		return TRUE;
+	}
+
+	for (i = 0; i < filter->list_cnt; i++) {
+		p = filter_pattern;
+		len = sizeof(filter_pattern);
+
+		nchar = snprintf(p, len, "%d ", filter->info[i].offset);
+		p += nchar;
+		len -= nchar;
+
+		nchar = snprintf(p, len, "0x");
+		p += nchar;
+		len -= nchar;
+
+		for (j = 0; j < filter->info[i].size_bytes; j++) {
+			nchar = snprintf(p, len, "%02x", filter->info[i].mask[j]);
+			p += nchar;
+			len -= nchar;
+		}
+
+		nchar = snprintf(p, len, " 0x");
+		p += nchar;
+		len -= nchar;
+
+		for (j = 0; j < filter->info[i].size_bytes; j++) {
+			nchar = snprintf(p, len, "%02x", filter->info[i].pattern[j]);
+			p += nchar;
+			len -= nchar;
+		}
+
+		if (strlen(arg) < strlen(filter_pattern)) {
+			continue;
+		}
+
+		DHD_PKT_LOG(("%s(): Pattern %s\n", __FUNCTION__, filter_pattern));
+
+		if (strncmp(filter_pattern, arg, strlen(filter_pattern)) == 0) {
+			*id = filter->info[i].id;
+			DHD_ERROR(("%s(): This pattern is existed\n", __FUNCTION__));
+			DHD_ERROR(("%s(): arg %s\n", __FUNCTION__, arg));
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+int
+dhd_pktlog_filter_add(dhd_pktlog_filter_t *filter, char *arg)
+{
+	int32 mask_size, pattern_size;
+	char *offset, *bitmask, *pattern;
+	uint32 id = 0;
+
+	if  (!filter || !arg) {
+		DHD_ERROR(("%s(): pktlog_filter =%p arg =%p\n", __FUNCTION__, filter, arg));
+		return BCME_ERROR;
+	}
+
+	DHD_PKT_LOG(("%s(): arg %s\n", __FUNCTION__, arg));
+
+	if (dhd_pktlog_filter_existed(filter, arg, &id) == TRUE) {
+		DHD_PKT_LOG(("%s(): This pattern id %d is existed\n", __FUNCTION__, id));
+		return BCME_OK;
+	}
+
+	if (filter->list_cnt >= MAX_DHD_PKTLOG_FILTER_LEN) {
+		DHD_ERROR(("%s(): pktlog filter full\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if ((offset = bcmstrtok(&arg, " ", 0)) == NULL) {
+		DHD_ERROR(("%s(): offset not found\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if ((bitmask = bcmstrtok(&arg, " ", 0)) == NULL) {
+		DHD_ERROR(("%s(): bitmask not found\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if ((pattern = bcmstrtok(&arg, " ", 0)) == NULL) {
+		DHD_ERROR(("%s(): pattern not found\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	/* parse filter bitmask */
+	mask_size = pattern_atoh_len(bitmask,
+			(char *) &filter->info[filter->list_cnt].mask[0],
+			MAX_MASK_PATTERN_FILTER_LEN);
+	if (mask_size == -1) {
+		DHD_ERROR(("Rejecting: %s\n", bitmask));
+		return BCME_ERROR;
+	}
+
+	/* parse filter pattern */
+	pattern_size = pattern_atoh_len(pattern,
+			(char *) &filter->info[filter->list_cnt].pattern[0],
+			MAX_MASK_PATTERN_FILTER_LEN);
+	if (pattern_size == -1) {
+		DHD_ERROR(("Rejecting: %s\n", pattern));
+		return BCME_ERROR;
+	}
+
+	prhex("mask", (char *)&filter->info[filter->list_cnt].mask[0],
+			mask_size);
+	prhex("pattern", (char *)&filter->info[filter->list_cnt].pattern[0],
+			pattern_size);
+
+	if (mask_size != pattern_size) {
+		DHD_ERROR(("%s(): Mask and pattern not the same size\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	filter->info[filter->list_cnt].offset = strtoul(offset, NULL, 0);
+	filter->info[filter->list_cnt].size_bytes = mask_size;
+	filter->info[filter->list_cnt].id = filter->list_cnt + 1;
+	filter->info[filter->list_cnt].enable = TRUE;
+
+	filter->list_cnt++;
+
+	return BCME_OK;
+}
+
+int
+dhd_pktlog_filter_del(dhd_pktlog_filter_t *filter, char *arg)
+{
+	uint32 id = 0;
+
+	if  (!filter || !arg) {
+		DHD_ERROR(("%s(): pktlog_filter =%p arg =%p\n", __FUNCTION__, filter, arg));
+		return BCME_ERROR;
+	}
+
+	DHD_PKT_LOG(("%s(): arg %s\n", __FUNCTION__, arg));
+
+	if (dhd_pktlog_filter_existed(filter, arg, &id) != TRUE) {
+		DHD_PKT_LOG(("%s(): This pattern id %d doesn't existed\n", __FUNCTION__, id));
+		return BCME_OK;
+	}
+
+	dhd_pktlog_filter_pull_forward(filter, id, filter->list_cnt);
+
+	filter->list_cnt--;
+
+	return BCME_OK;
+}
+
+int
+dhd_pktlog_filter_enable(dhd_pktlog_filter_t *filter, uint32 pktmon_case, uint32 enable)
+{
+	if  (!filter) {
+		DHD_ERROR(("%s(): filter is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_PKT_LOG(("%s(): pktlog_case %d enable %d\n", __FUNCTION__, pktmon_case, enable));
+
+	if (enable) {
+		filter->enable |=  pktmon_case;
+	} else {
+		filter->enable &= ~pktmon_case;
+	}
+
+	return BCME_OK;
+}
+
+int
+dhd_pktlog_filter_pattern_enable(dhd_pktlog_filter_t *filter, char *arg, uint32 enable)
+{
+	uint32 id = 0;
+
+	if  (!filter || !arg) {
+		DHD_ERROR(("%s(): pktlog_filter =%p arg =%p\n", __FUNCTION__, filter, arg));
+		return BCME_ERROR;
+	}
+
+	if (dhd_pktlog_filter_existed(filter, arg, &id) == TRUE) {
+		if (id > 0) {
+			filter->info[id-1].enable = enable;
+			DHD_ERROR(("%s(): This pattern id %d is %s\n",
+				__FUNCTION__, id, (enable ? "enabled" : "disabled")));
+		}
+	} else {
+		DHD_ERROR(("%s(): This pattern is not existed\n", __FUNCTION__));
+		DHD_ERROR(("%s(): arg %s\n", __FUNCTION__, arg));
+	}
+
+	return BCME_OK;
+}
+
+int
+dhd_pktlog_filter_info(dhd_pktlog_filter_t *filter)
+{
+	char filter_pattern[MAX_FILTER_PATTERN_LEN];
+	char *p;
+	int i, j;
+	int nchar;
+	int len;
+
+	if  (!filter) {
+		DHD_ERROR(("%s(): pktlog_filter is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_ERROR(("---- PKTLOG FILTER INFO ----\n\n"));
+
+	DHD_ERROR(("Filter list cnt %d Filter is %s\n",
+		filter->list_cnt, (filter->enable ? "enabled" : "disabled")));
+
+	for (i = 0; i < filter->list_cnt; i++) {
+		p = filter_pattern;
+		len = sizeof(filter_pattern);
+
+		nchar = snprintf(p, len, "%d ", filter->info[i].offset);
+		p += nchar;
+		len -= nchar;
+
+		nchar = snprintf(p, len, "0x");
+		p += nchar;
+		len -= nchar;
+
+		for (j = 0; j < filter->info[i].size_bytes; j++) {
+			nchar = snprintf(p, len, "%02x", filter->info[i].mask[j]);
+			p += nchar;
+			len -= nchar;
+		}
+
+		nchar = snprintf(p, len, " 0x");
+		p += nchar;
+		len -= nchar;
+
+		for (j = 0; j < filter->info[i].size_bytes; j++) {
+			nchar = snprintf(p, len, "%02x", filter->info[i].pattern[j]);
+			p += nchar;
+			len -= nchar;
+		}
+
+		DHD_ERROR(("ID:%d is %s\n",
+			filter->info[i].id, (filter->info[i].enable ? "enabled" : "disabled")));
+		DHD_ERROR(("Pattern %s\n", filter_pattern));
+	}
+
+	DHD_ERROR(("---- PKTLOG FILTER END ----\n"));
+
+	return BCME_OK;
+}
+bool
+dhd_pktlog_filter_matched(dhd_pktlog_filter_t *filter, char *data, uint32 pktlog_case)
+{
+	uint16 szbts;	/* pattern size */
+	uint16 offset;	/* pattern offset */
+	int i, j;
+	uint8 *mask = NULL;		/* bitmask */
+	uint8 *pattern = NULL;
+	uint8 *pkt_offset = NULL;	/* packet offset */
+	bool matched;
+
+	if  (!filter || !data) {
+		DHD_PKT_LOG(("%s(): filter=%p data=%p\n",
+			__FUNCTION__, filter, data));
+		return TRUE;
+	}
+
+	if (!(pktlog_case & filter->enable)) {
+		DHD_PKT_LOG(("%s(): pktlog_case %d return TRUE filter is disabled\n",
+			__FUNCTION__, pktlog_case));
+		return TRUE;
+	}
+
+	for (i = 0; i < filter->list_cnt; i++) {
+		if (&filter->info[i] && filter->info[i].id && filter->info[i].enable) {
+			szbts = filter->info[i].size_bytes;
+			offset = filter->info[i].offset;
+			mask = &filter->info[i].mask[0];
+			pkt_offset = &data[offset];
+			pattern = &filter->info[i].pattern[0];
+
+			matched = TRUE;
+			for (j = 0; j < szbts; j++) {
+				if ((mask[j] & pkt_offset[j]) != pattern[j]) {
+					matched = FALSE;
+					break;
+				}
+			}
+
+			if (matched) {
+				DHD_PKT_LOG(("%s(): pktlog_filter return TRUE id %d\n",
+					__FUNCTION__, filter->info[i].id));
+				return TRUE;
+			}
+		} else {
+			DHD_PKT_LOG(("%s(): filter ino is null %p\n",
+				__FUNCTION__, &filter->info[i]));
+		}
+	}
+
+	return FALSE;
+}
+
+/* Ethernet Type MAC Header 12 bytes + Frame payload 10 bytes */
+#define PKTLOG_MINIMIZE_REPORT_LEN 22
+
+#ifndef DHD_PKT_LOGGING_DBGRING
+static char pktlog_minmize_mask_table[] = {
+	0xff, 0x00, 0x00, 0x00, 0xff, 0x0f, /* Ethernet Type MAC Header - Destination MAC Address */
+	0xff, 0x00, 0x00, 0x00, 0xff, 0x0f, /* Ethernet Type MAC Header - Source MAC Address */
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* Ethernet Type MAC Header - Ether Type - 2 bytes */
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* Frame payload */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, /* UDP port number offset - bytes as 0xff */
+	0xff, 0xff,
+};
+
+static inline void
+dhd_pktlog_minimize_report(char *pkt, uint32 frame_len,
+		void *file, const void *user_buf, void *pos)
+{
+	int i;
+	int ret = 0;
+	int table_len;
+	int report_len;
+	char *p_table;
+	char *mem_buf = NULL;
+
+	table_len = sizeof(pktlog_minmize_mask_table);
+	report_len =  table_len;
+	p_table = &pktlog_minmize_mask_table[0];
+
+	if (frame_len < PKTLOG_MINIMIZE_REPORT_LEN) {
+		DHD_ERROR(("%s : frame_len is samller than min\n", __FUNCTION__));
+		return;
+	}
+
+	mem_buf = vmalloc(frame_len);
+	if (!mem_buf) {
+		DHD_ERROR(("%s : failed to alloc membuf\n", __FUNCTION__));
+		return;
+	}
+
+	bzero(mem_buf, frame_len);
+
+	if (frame_len < table_len) {
+		report_len = PKTLOG_MINIMIZE_REPORT_LEN;
+	}
+
+	for (i = 0; i < report_len; i++) {
+		mem_buf[i] = pkt[i] & p_table[i];
+	}
+
+	ret = dhd_export_debug_data(mem_buf,
+			file, user_buf, frame_len, pos);
+	if (ret < 0) {
+		DHD_ERROR(("%s : Write minimize report\n", __FUNCTION__));
+	}
+	vfree(mem_buf);
+}
+#endif /* !DHD_PKT_LOGGING_DBGRING */
+
+dhd_pktlog_ring_t*
+dhd_pktlog_ring_change_size(dhd_pktlog_ring_t *ringbuf, int size)
+{
+	uint32 alloc_len;
+	uint32 pktlog_minmize;
+	dhd_pktlog_ring_t *pktlog_ring = NULL;
+	dhd_pub_t *dhdp;
+
+	if  (!ringbuf) {
+		DHD_ERROR(("%s(): ringbuf is NULL\n", __FUNCTION__));
+		return NULL;
+	}
+
+	alloc_len = size;
+	if (alloc_len < MIN_PKTLOG_LEN) {
+		alloc_len = MIN_PKTLOG_LEN;
+	}
+	if (alloc_len > MAX_PKTLOG_LEN) {
+		alloc_len = MAX_PKTLOG_LEN;
+	}
+	DHD_ERROR(("ring size requested: %d alloc: %d\n", size, alloc_len));
+
+	/* backup variable */
+	pktlog_minmize = ringbuf->pktlog_minmize;
+	dhdp = ringbuf->dhdp;
+
+	/* free ring_info */
+	dhd_pktlog_ring_deinit(dhdp, ringbuf);
+
+	/* alloc ring_info */
+	pktlog_ring = dhd_pktlog_ring_init(dhdp, alloc_len);
+
+	/* restore variable */
+	if (pktlog_ring) {
+		OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, TRUE);
+		pktlog_ring->pktlog_minmize = pktlog_minmize;
+	}
+
+	return pktlog_ring;
+}
+
+void
+dhd_pktlog_filter_pull_forward(dhd_pktlog_filter_t *filter, uint32 del_filter_id, uint32 list_cnt)
+{
+	int ret = 0;
+	int pos = 0;
+	int move_list_cnt = 0;
+	int move_bytes = 0;
+
+	if ((del_filter_id > list_cnt) ||
+		(list_cnt > MAX_DHD_PKTLOG_FILTER_LEN)) {
+		DHD_ERROR(("Wrong id %d cnt %d tried to remove\n", del_filter_id, list_cnt));
+		return;
+	}
+
+	move_list_cnt = list_cnt - del_filter_id;
+
+	pos = del_filter_id -1;
+	move_bytes = sizeof(dhd_pktlog_filter_info_t) * move_list_cnt;
+	if (move_list_cnt) {
+		ret = memmove_s(&filter->info[pos], move_bytes + sizeof(dhd_pktlog_filter_info_t),
+				&filter->info[pos+1], move_bytes);
+		if (ret) {
+			DHD_ERROR(("filter moving failed\n"));
+			return;
+		}
+		for (; pos < list_cnt -1; pos++) {
+			filter->info[pos].id -= 1;
+		}
+	}
+	bzero(&filter->info[list_cnt-1], sizeof(dhd_pktlog_filter_info_t));
+}
+
+void dhd_pktlog_get_filename(dhd_pub_t *dhdp, char *dump_path, int len)
+{
+	/* Init file name */
+	bzero(dump_path, len);
+	clear_debug_dump_time(dhdp->debug_dump_time_pktlog_str);
+	get_debug_dump_time(dhdp->debug_dump_time_pktlog_str);
+
+	if (dhdp->memdump_type == DUMP_TYPE_BY_SYSDUMP) {
+		if (dhdp->debug_dump_subcmd == CMD_UNWANTED) {
+			snprintf(dump_path, len, "%s",
+					DHD_PKTLOG_DUMP_PATH DHD_PKTLOG_DUMP_TYPE
+					DHD_DUMP_SUBSTR_UNWANTED);
+		} else if (dhdp->debug_dump_subcmd == CMD_DISCONNECTED) {
+			snprintf(dump_path, len, "%s",
+					DHD_PKTLOG_DUMP_PATH DHD_PKTLOG_DUMP_TYPE
+					DHD_DUMP_SUBSTR_DISCONNECTED);
+		} else {
+			snprintf(dump_path, len, "%s",
+					DHD_PKTLOG_DUMP_PATH DHD_PKTLOG_DUMP_TYPE);
+		}
+	} else {
+		if (dhdp->pktlog_debug) {
+			snprintf(dump_path, len, "%s",
+					DHD_PKTLOG_DUMP_PATH DHD_PKTLOG_DEBUG_DUMP_TYPE);
+		} else {
+			snprintf(dump_path, len, "%s",
+					DHD_PKTLOG_DUMP_PATH DHD_PKTLOG_DUMP_TYPE);
+		}
+
+	}
+
+	snprintf(dump_path, len, "%s_%s.pcap", dump_path,
+			dhdp->debug_dump_time_pktlog_str);
+	DHD_ERROR(("%s: pktlog path = %s%s\n", __FUNCTION__, dump_path, FILE_NAME_HAL_TAG));
+	clear_debug_dump_time(dhdp->debug_dump_time_pktlog_str);
+}
+
+uint32
+dhd_pktlog_get_item_length(dhd_pktlog_ring_info_t *report_ptr)
+{
+	uint32 len = 0;
+	char buf[DHD_PKTLOG_FATE_INFO_STR_LEN];
+	int bytes_user_data = 0;
+	uint32 write_frame_len;
+	uint32 frame_len;
+
+	len += (uint32)sizeof(report_ptr->info.driver_ts_sec);
+	len += (uint32)sizeof(report_ptr->info.driver_ts_usec);
+
+	if (report_ptr->info.payload_type == FRAME_TYPE_ETHERNET_II) {
+		frame_len = (uint32)min(report_ptr->info.pkt_len, (size_t)MAX_FRAME_LEN_ETHERNET);
+	} else {
+		frame_len = (uint32)min(report_ptr->info.pkt_len, (size_t)MAX_FRAME_LEN_80211_MGMT);
+	}
+#ifdef DHD_PKT_LOGGING_DBGRING
+	frame_len = (uint32)min(frame_len, DHD_PKT_LOGGING_DBGRING_MAX_SIZE);
+	bytes_user_data = snprintf(buf, sizeof(buf), "%s:%s:%02d\n",
+		DHD_PKTLOG_FATE_INFO_FORMAT,
+		(report_ptr->tx_fate ? "Failure" : "Succeed"), report_ptr->tx_fate);
+#else
+	bytes_user_data = snprintf(buf, sizeof(buf), "%s:%s:%02d:%s:%d.%d s\n",
+			DHD_PKTLOG_FATE_INFO_FORMAT,
+			(report_ptr->tx_fate ? "Failure" : "Succeed"), report_ptr->tx_fate,
+			(report_ptr->info.direction == PKT_TX) ? "TX" : "RX",
+			report_ptr->info.tx_status_ts_sec, report_ptr->info.tx_status_ts_usec);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	write_frame_len = frame_len + bytes_user_data;
+
+	/* pcap pkt head has incl_len and orig_len */
+	len += (uint32)sizeof(write_frame_len);
+	len += (uint32)sizeof(write_frame_len);
+	len += write_frame_len;
+
+	return len;
+}
+
+int
+dhd_pktlog_get_dump_length(dhd_pub_t *dhdp)
+{
+	dhd_pktlog_ring_info_t *report_ptr;
+	dhd_pktlog_ring_t *pktlog_ring;
+	uint32 len = 0;
+	dll_t *item_p, *next_p;
+#ifdef DHD_PKT_LOGGING_DBGRING
+	dhd_dbg_ring_t *ring;
+	ring = &dhdp->dbg->dbg_rings[PACKET_LOG_RING_ID];
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_PKT_LOG(("%s(): dhdp=%p pktlog=%p\n",
+			__FUNCTION__, dhdp, (dhdp ? dhdp->pktlog : NULL)));
+		return -EINVAL;
+	}
+
+	if (!dhdp->pktlog->pktlog_ring) {
+		DHD_PKT_LOG(("%s(): pktlog_ring =%p\n",
+			__FUNCTION__, dhdp->pktlog->pktlog_ring));
+		return -EINVAL;
+	}
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, FALSE);
+#ifndef DHD_PKT_LOGGING_DBGRING
+	len = sizeof(dhd_pktlog_pcap_hdr_t);
+#endif /* !DHD_PKT_LOGGING_DBGRING */
+	for (item_p = dll_head_p(&pktlog_ring->ring_info_head);
+			!dll_end(&pktlog_ring->ring_info_head, item_p);
+			item_p = next_p) {
+		next_p = dll_next_p(item_p);
+		report_ptr = (dhd_pktlog_ring_info_t *)item_p;
+		len += dhd_pktlog_get_item_length(report_ptr);
+#ifdef DHD_PKT_LOGGING_DBGRING
+		if (len >= ring->threshold) {
+			break;
+		}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	}
+	OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, TRUE);
+	DHD_PKT_LOG(("calcuated pkt log dump len:%d\n", len));
+
+	return len;
+}
+
+int
+#ifdef DHD_PKT_LOGGING_DBGRING
+dhd_pktlog_dump_write(dhd_pub_t *dhdp, void *file, const void *user_buf,
+	uint32 *written_bytes)
+#else
+dhd_pktlog_dump_write(dhd_pub_t *dhdp, void *file, const void *user_buf, uint32 size)
+#endif /* DHD_PKT_LOGGING_DBGRING */
+{
+	dhd_pktlog_ring_info_t *report_ptr;
+	dhd_pktlog_ring_t *pktlog_ring;
+	char buf[DHD_PKTLOG_FATE_INFO_STR_LEN];
+#ifndef DHD_PKT_LOGGING_DBGRING
+	dhd_pktlog_pcap_hdr_t pcap_h;
+	loff_t pos = 0;
+#else
+	uint32 size = 0;
+	unsigned long flags = 0;
+#endif /* !DHD_PKT_LOGGING_DBGRING */
+	uint32 write_frame_len;
+	uint32 frame_len;
+	ulong len = 0;
+	int bytes_user_data = 0;
+	int ret = BCME_OK;
+	dll_t *item_p, *next_p;
+
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_PKT_LOG(("%s(): dhdp=%p pktlog=%p\n",
+			__FUNCTION__, dhdp, (dhdp ? dhdp->pktlog : NULL)));
+		return -EINVAL;
+	}
+
+	if (!dhdp->pktlog->pktlog_ring) {
+		DHD_PKT_LOG(("%s(): pktlog_ring =%p\n",
+			__FUNCTION__, dhdp->pktlog->pktlog_ring));
+		return -EINVAL;
+	}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+	if (user_buf) {
+		size = *written_bytes;
+	} else if (file) {
+		/* With DHD_PKT_LOGGING_DBGRING, this path is not allowed. */
+		return BCME_UNSUPPORTED;
+	}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+	if (file && !user_buf && (size == 0)) {
+		DHD_ERROR(("Local file pktlog dump requested\n"));
+	} else if (!file && user_buf && (size > 0)) {
+#ifdef DHD_PKT_LOGGING_DBGRING
+		DHD_INFO(("HAL file pktlog dump %d bytes requested\n", size));
+#else
+		DHD_ERROR(("HAL file pktlog dump %d bytes requested\n", size));
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	} else {
+		DHD_ERROR(("Wrong type pktlog dump requested\n"));
+		return -EINVAL;
+	}
+
+	pktlog_ring = dhdp->pktlog->pktlog_ring;
+	OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, FALSE);
+#ifndef DHD_PKT_LOGGING_DBGRING
+	pcap_h.magic_number = PKTLOG_PCAP_MAGIC_NUM;
+	pcap_h.version_major = PKTLOG_PCAP_MAJOR_VER;
+	pcap_h.version_minor = PKTLOG_PCAP_MINOR_VER;
+	pcap_h.thiszone = 0x0;
+	pcap_h.sigfigs = 0x0;
+	pcap_h.snaplen = PKTLOG_PCAP_SNAP_LEN;
+	pcap_h.network = PKTLOG_PCAP_NETWORK_TYPE;
+
+	ret = dhd_export_debug_data((char *)&pcap_h, file, user_buf, sizeof(pcap_h), &pos);
+	len = sizeof(pcap_h);
+#endif /* !DHD_PKT_LOGGING_DBGRING */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	DHD_PKT_LOG_LOCK(dhd_os_get_pktlog_lock(dhdp), flags);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	for (item_p = dll_head_p(&pktlog_ring->ring_info_head);
+			!dll_end(&pktlog_ring->ring_info_head, item_p);
+			item_p = next_p) {
+#ifdef DHD_PKT_LOGGING_DBGRING
+		uint32 captured_frame_len;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+
+		next_p = dll_next_p(item_p);
+		report_ptr = (dhd_pktlog_ring_info_t *)item_p;
+#ifdef DHD_PKT_LOGGING_DBGRING
+		if (report_ptr->info.direction == PKT_TX &&
+				report_ptr->tx_fate == TX_PKT_FATE_DRV_QUEUED) {
+			/* Not updated txfate yet, postpone some time. */
+			report_ptr->tx_fate |= TX_PKT_FATE_DRV_WAIT_UPDATE;
+			continue;
+		}
+#endif /* DHD_PKT_LOGGING_DBGRING */
+		if ((file == NULL) &&
+			(len + dhd_pktlog_get_item_length(report_ptr) > size)) {
+#ifdef DHD_PKT_LOGGING_DBGRING
+			DHD_INFO(("overflowed pkt logs are dropped\n"));
+#else
+			DHD_ERROR(("overflowed pkt logs are dropped\n"));
+#endif /* DHD_PKT_LOGGING_DBGRING */
+			break;
+		}
+
+		if (report_ptr->info.pkt == NULL) {
+			DHD_ERROR(("%s : pkt isn't located skip it\n", __FUNCTION__));
+			continue;
+		}
+
+#ifdef DHD_PKT_LOGGING_DBGRING
+		ret = memcpy_s((void*)(user_buf + len), size - len,
+			(char*)&report_ptr->info.driver_ts_sec,
+			sizeof(report_ptr->info.driver_ts_sec));
+		len += sizeof(report_ptr->info.driver_ts_sec);
+
+		ret = memcpy_s((void*)(user_buf + len), size - len,
+			(char*)&report_ptr->info.driver_ts_usec,
+			sizeof(report_ptr->info.driver_ts_usec));
+		len += sizeof(report_ptr->info.driver_ts_usec);
+
+		if (report_ptr->info.payload_type == FRAME_TYPE_ETHERNET_II) {
+			frame_len = (uint32)min(report_ptr->info.pkt_len,
+					(size_t)MAX_FRAME_LEN_ETHERNET);
+		} else {
+			frame_len = (uint32)min(report_ptr->info.pkt_len,
+					(size_t)MAX_FRAME_LEN_80211_MGMT);
+		}
+
+		bytes_user_data = snprintf(buf, sizeof(buf), "%s:%s:%02d\n",
+				DHD_PKTLOG_FATE_INFO_FORMAT,
+				(report_ptr->tx_fate ? "Failure" : "Succeed"),
+				(report_ptr->tx_fate & ~(TX_PKT_FATE_DRV_WAIT_UPDATE)));
+		write_frame_len = frame_len + bytes_user_data;
+		frame_len = (uint32)min(frame_len, DHD_PKT_LOGGING_DBGRING_MAX_SIZE);
+		captured_frame_len = frame_len + bytes_user_data;
+
+		/* pcap pkt head has incl_len and orig_len */
+		ret = memcpy_s((void*)(user_buf + len), size - len,
+			(char*)&captured_frame_len, sizeof(captured_frame_len));
+		len += sizeof(captured_frame_len);
+
+		ret = memcpy_s((void*)(user_buf + len), size - len,
+			(char*)&write_frame_len, sizeof(write_frame_len));
+		len += sizeof(write_frame_len);
+
+		ret = memcpy_s((void*)(user_buf + len), size - len, PKTDATA(pktlog_ring->dhdp->osh,
+			report_ptr->info.pkt), frame_len);
+		len += frame_len;
+
+		ret = memcpy_s((void*)(user_buf + len), size - len, buf, bytes_user_data);
+		len += bytes_user_data;
+
+		dll_delete((dll_t *)report_ptr);
+		PKTFREE(pktlog_ring->dhdp->osh, report_ptr->info.pkt, TRUE);
+		pktlog_ring->pktcount--;
+		dll_append(&pktlog_ring->ring_info_free, (dll_t *)report_ptr);
+#else
+		ret = dhd_export_debug_data((char*)&report_ptr->info.driver_ts_sec, file,
+				user_buf, sizeof(report_ptr->info.driver_ts_sec), &pos);
+		len += sizeof(report_ptr->info.driver_ts_sec);
+
+		ret = dhd_export_debug_data((char*)&report_ptr->info.driver_ts_usec, file,
+				user_buf, sizeof(report_ptr->info.driver_ts_usec), &pos);
+		len += sizeof(report_ptr->info.driver_ts_usec);
+
+		if (report_ptr->info.payload_type == FRAME_TYPE_ETHERNET_II) {
+			frame_len = (uint32)min(report_ptr->info.pkt_len,
+					(size_t)MAX_FRAME_LEN_ETHERNET);
+
+		} else {
+			frame_len = (uint32)min(report_ptr->info.pkt_len,
+					(size_t)MAX_FRAME_LEN_80211_MGMT);
+		}
+
+		bytes_user_data = snprintf(buf, sizeof(buf), "%s:%s:%02d:%s:%d.%d s\n",
+				DHD_PKTLOG_FATE_INFO_FORMAT,
+				(report_ptr->tx_fate ? "Failure" : "Succeed"),
+				report_ptr->tx_fate,
+				(report_ptr->info.direction == PKT_TX) ? "TX" : "RX",
+				report_ptr->info.tx_status_ts_sec,
+				report_ptr->info.tx_status_ts_usec);
+		write_frame_len = frame_len + bytes_user_data;
+
+		/* pcap pkt head has incl_len and orig_len */
+		ret = dhd_export_debug_data((char*)&write_frame_len, file, user_buf,
+				sizeof(write_frame_len), &pos);
+		len += sizeof(write_frame_len);
+
+		ret = dhd_export_debug_data((char*)&write_frame_len, file, user_buf,
+				sizeof(write_frame_len), &pos);
+		len += sizeof(write_frame_len);
+
+		if (pktlog_ring->pktlog_minmize) {
+			dhd_pktlog_minimize_report(PKTDATA(pktlog_ring->dhdp->osh,
+					report_ptr->info.pkt), frame_len, file, user_buf, &pos);
+		} else {
+			ret = dhd_export_debug_data(PKTDATA(pktlog_ring->dhdp->osh,
+					report_ptr->info.pkt), file, user_buf, frame_len, &pos);
+		}
+		len += frame_len;
+
+		ret = dhd_export_debug_data(buf, file, user_buf, bytes_user_data, &pos);
+		len += bytes_user_data;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	}
+#ifdef DHD_PKT_LOGGING_DBGRING
+	DHD_PKT_LOG_UNLOCK(dhd_os_get_pktlog_lock(dhdp), flags);
+	*written_bytes = len;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	OSL_ATOMIC_SET(dhdp->osh, &pktlog_ring->start, TRUE);
+
+	return ret;
+}
+
+int
+dhd_pktlog_dump_write_memory(dhd_pub_t *dhdp, const void *user_buf, uint32 size)
+{
+#ifdef DHD_PKT_LOGGING_DBGRING
+	int ret = dhd_pktlog_dump_write(dhdp, NULL, user_buf, &size);
+#else
+	int ret = dhd_pktlog_dump_write(dhdp, NULL, user_buf, size);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+	if (ret < 0) {
+		DHD_ERROR(("dhd_pktlog_dump_write_memory error\n"));
+	}
+	return ret;
+}
+
+int
+dhd_pktlog_dump_write_file(dhd_pub_t *dhdp)
+{
+	struct file *w_pcap_fp = NULL;
+	uint32 file_mode;
+#ifdef get_fs
+	mm_segment_t old_fs;
+#endif /* get_fs */
+	char pktlogdump_path[128];
+	int ret = BCME_OK;
+
+	dhd_pktlog_get_filename(dhdp, pktlogdump_path, 128);
+#ifdef get_fs
+	old_fs = get_fs();
+	set_fs(KERNEL_DS);
+#endif /* get_fs */
+	file_mode = O_CREAT | O_WRONLY;
+
+	w_pcap_fp = dhd_filp_open(pktlogdump_path, file_mode, 0664);
+	if (IS_ERR(w_pcap_fp) || (w_pcap_fp == NULL)) {
+		DHD_ERROR(("%s: Couldn't open file '%s' err %ld\n",
+			__FUNCTION__, pktlogdump_path, PTR_ERR(w_pcap_fp)));
+		ret = BCME_ERROR;
+		goto fail;
+	}
+
+	dhd_pktlog_dump_write(dhdp, w_pcap_fp, NULL, 0);
+	if (ret < 0) {
+		DHD_ERROR(("dhd_pktlog_dump_write error\n"));
+		goto fail;
+	}
+
+	/* Sync file from filesystem to physical media */
+	ret = dhd_vfs_fsync(w_pcap_fp, 0);
+	if (ret < 0) {
+		DHD_ERROR(("%s(): sync pcap file error, err = %d\n", __FUNCTION__, ret));
+		goto fail;
+	}
+fail:
+	if (!IS_ERR(w_pcap_fp)) {
+		dhd_filp_close(w_pcap_fp, NULL);
+	}
+#ifdef get_fs
+	set_fs(old_fs);
+#endif /* get_fs */
+#ifdef DHD_DUMP_MNGR
+	if (ret >= 0) {
+		dhd_dump_file_manage_enqueue(dhdp, pktlogdump_path, DHD_PKTLOG_DUMP_TYPE);
+	}
+#endif /* DHD_DUMP_MNGR */
+	return ret;
+}
+
+#ifdef DHD_COMPACT_PKT_LOG
+static uint64
+dhd_cpkt_log_calc_time_diff(dhd_pktlog_ring_info_t *pkt_info, uint64 curr_ts_nsec)
+{
+	uint64 pkt_ts_nsec = pkt_info->info.driver_ts_sec * NSEC_PER_SEC +
+		pkt_info->info.driver_ts_usec * NSEC_PER_USEC;
+
+	return (curr_ts_nsec - pkt_ts_nsec) / NSEC_PER_USEC;
+}
+
+static int
+dhd_cpkt_log_get_ts_idx(dhd_pktlog_t *pktlog, dhd_pktlog_ring_info_t *pkt_info, u64 curr_ts_nsec)
+{
+	struct rb_node *n = pktlog->cpkt_log_tt_rbt.rb_node;
+	dhd_cpkt_log_ts_node_t *node = NULL;
+
+	uint64 ts_diff = dhd_cpkt_log_calc_time_diff(pkt_info, curr_ts_nsec);
+
+	if (ts_diff > dhd_cpkt_log_tt_idx[CPKT_LOG_TT_IDX_ARR_SZ - 1])
+		return CPKT_LOG_TT_IDX_ARR_SZ;
+
+	while (n) {
+		node = rb_entry(n, dhd_cpkt_log_ts_node_t, rb);
+
+		if (ts_diff < node->ts_diff)
+			n = n->rb_left;
+		else if (ts_diff > node->ts_diff)
+			n = n->rb_right;
+		else
+			break;
+	}
+
+	if (node != NULL) {
+		if (node->idx && ts_diff < node->ts_diff)
+			return node->idx - 1;
+		return node->idx;
+	}
+
+	return BCME_NOTFOUND;
+}
+
+static int
+dhd_cpkt_log_get_direction(dhd_pktlog_ring_info_t *pkt_info)
+{
+	return pkt_info->info.direction == PKTLOG_TXPKT_CASE ? PKT_TX : PKT_RX;
+}
+
+static int
+dhd_cpkt_log_get_802_1x_subtype(eapol_header_t *eapol)
+{
+	int subtype;
+	eap_header_t *eap;
+	eapol_wpa_key_header_t *ek;
+
+	uint16 key_info;
+	int pair, ack, mic, kerr, req, sec, install;
+
+	subtype = CPKT_LOG_802_1X_SUBTYPE_OTHERS;
+	if (eapol->type != EAPOL_KEY) {
+		eap = (eap_header_t *)eapol->body;
+
+		switch (eap->type) {
+		case EAP_IDENTITY:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_IDENTITY;
+			break;
+		case REALM_EAP_TLS:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_TLS;
+			break;
+		case REALM_EAP_TTLS:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_TTLS;
+			break;
+		case REALM_EAP_FAST:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_FAST;
+			break;
+		case REALM_EAP_LEAP:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_LEAP;
+			break;
+		case REALM_EAP_PSK:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_PWD;
+			break;
+		case REALM_EAP_SIM:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_SIM;
+			break;
+		case REALM_EAP_AKA:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_AKA;
+			break;
+		case REALM_EAP_AKAP:
+			subtype = CPKT_LOG_802_1X_SUBTYPE_AKAP;
+			break;
+		default:
+			break;
+		}
+		if (eap->code == EAP_SUCCESS)
+			subtype = CPKT_LOG_802_1X_SUBTYPE_SUCCESS;
+	} else {
+		/* in case of 4 way handshake */
+		ek = (eapol_wpa_key_header_t *)(eapol->body);
+
+		if (ek->type == EAPOL_WPA2_KEY || ek->type == EAPOL_WPA_KEY) {
+			key_info = ntoh16_ua(&ek->key_info);
+
+			pair =  0 != (key_info & WPA_KEY_PAIRWISE);
+			ack = 0  != (key_info & WPA_KEY_ACK);
+			mic = 0  != (key_info & WPA_KEY_MIC);
+			kerr =  0 != (key_info & WPA_KEY_ERROR);
+			req = 0  != (key_info & WPA_KEY_REQ);
+			sec = 0  != (key_info & WPA_KEY_SECURE);
+			install  = 0 != (key_info & WPA_KEY_INSTALL);
+
+			if (!sec && !mic && ack && !install && pair && !kerr && !req)
+				subtype = CPKT_LOG_802_1X_SUBTYPE_4WAY_M1;
+			else if (pair && !install && !ack && mic && !sec && !kerr && !req)
+				subtype = CPKT_LOG_802_1X_SUBTYPE_4WAY_M2;
+			else if (pair && ack && mic && sec && !kerr && !req)
+				subtype = CPKT_LOG_802_1X_SUBTYPE_4WAY_M3;
+			else if (pair && !install && !ack && mic && sec && !req && !kerr)
+				subtype = CPKT_LOG_802_1X_SUBTYPE_4WAY_M4;
+		}
+	}
+
+	return subtype;
+}
+
+static int
+dhd_cpkt_log_get_pkt_info(dhd_pktlog_t *pktlog, dhd_pktlog_ring_info_t *pkt_info)
+{
+	int type;
+	int subtype = 0;
+
+	uint8 prot;
+	uint16 src_port, dst_port;
+	int len, offset;
+
+	uint8 *pdata;
+	uint8 *pkt_data;
+
+	uint16 eth_type;
+	struct bcmarp *arp;
+	struct bcmicmp_hdr *icmp;
+	struct ipv4_hdr *ipv4;
+	struct ether_header *eth_hdr;
+	bcm_tlv_t *dhcp_opt;
+
+	struct ipv6_hdr *ipv6;
+	struct icmp6_hdr *icmpv6_hdr;
+
+	pkt_data = (uint8 *)PKTDATA(pktlog->dhdp->osh, pkt_info->info.pkt);
+
+	eth_hdr = (struct ether_header *)pkt_data;
+	eth_type = ntoh16(eth_hdr->ether_type);
+
+	type = CPKT_LOG_TYPE_OTHERS;
+	switch (eth_type) {
+	case ETHER_TYPE_IP:
+		if (get_pkt_ip_type(pktlog->dhdp->osh, pkt_info->info.pkt,
+			&pdata, &len, &prot) != 0) {
+			DHD_PKT_LOG(("%s: fail to get pkt ip type\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+
+		if (prot == IP_PROT_ICMP) {
+			icmp = (struct bcmicmp_hdr *)(pdata);
+			if (!(icmp->type == ICMP_TYPE_ECHO_REQUEST ||
+				icmp->type == ICMP_TYPE_ECHO_REPLY ||
+				icmp->type == CPKT_LOG_ICMP_TYPE_DEST_UNREACHABLE)) {
+				return BCME_ERROR;
+			}
+
+			if (icmp->type == ICMP_TYPE_ECHO_REQUEST) {
+				type = CPKT_LOG_TYPE_ICMP_REQ;
+				/* Subtype = Last 8 bits of identifier */
+				subtype = ntoh16_ua(pdata + sizeof(*icmp)) & 0xFF;
+			} else if (icmp->type == ICMP_TYPE_ECHO_REPLY) {
+				type = CPKT_LOG_TYPE_ICMP_RES;
+				/* Subtype = Last 8 bits of identifier */
+				subtype = ntoh16_ua(pdata + sizeof(*icmp)) & 0xFF;
+			} else if (icmp->type == CPKT_LOG_ICMP_TYPE_DEST_UNREACHABLE) {
+				type = CPKT_LOG_TYPE_ICMP_UNREACHABLE;
+				/* Subtype = Last 8 bits of identifier */
+				ipv4 = (struct ipv4_hdr *)(pdata + sizeof(*icmp) +
+						CPKT_LOG_ICMP_TYPE_DEST_UNREACHABLE_IPV4_OFFSET);
+				subtype = ipv4->id & 0xFF;
+			}
+
+			DHD_PKT_LOG(("%s: type = ICMP(%d), subtype = %x \n",
+				__FUNCTION__, type, subtype));
+		} else if (prot == IP_PROT_UDP) {
+			if (len < UDP_HDR_LEN)
+				return BCME_ERROR;
+
+			src_port = ntoh16_ua(pdata);
+			dst_port = ntoh16_ua(pdata + UDP_DEST_PORT_OFFSET);
+
+			if (src_port == DHCP_PORT_SERVER || src_port == DHCP_PORT_CLIENT) {
+				type = CPKT_LOG_TYPE_DHCP;
+				/* Subtype = DHCP message type */
+				offset = DHCP_OPT_OFFSET + CPKT_LOG_DHCP_MAGIC_COOKIE_LEN;
+				if ((UDP_HDR_LEN + offset) >= len)
+					return BCME_ERROR;
+				len -= (UDP_HDR_LEN - offset);
+
+				dhcp_opt = bcm_parse_tlvs(pdata + UDP_HDR_LEN + offset,
+					len, DHCP_OPT_MSGTYPE);
+				if (dhcp_opt == NULL)
+					return BCME_NOTFOUND;
+				subtype = dhcp_opt->data[0];
+
+				DHD_PKT_LOG(("%s: type = DHCP(%d), subtype = %x \n",
+					__FUNCTION__, type, subtype));
+			} else if (src_port == CPKT_LOG_DNS_PORT_CLIENT ||
+				dst_port == CPKT_LOG_DNS_PORT_CLIENT ||
+				dst_port == CPKT_LOG_MDNS_PORT_CLIENT) {
+				type = CPKT_LOG_TYPE_DNS;
+				/* Subtype = Last 8 bits of DNS Transaction ID */
+				subtype = ntoh16_ua(pdata + UDP_HDR_LEN) & 0xFF;
+
+				DHD_PKT_LOG(("%s: type = DNS(%d), subtype = %x \n",
+					__FUNCTION__, type, subtype));
+			} else {
+				DHD_PKT_LOG(("%s: unsupported ports num (src:%d, dst:%d)\n",
+					__FUNCTION__, src_port, dst_port));
+			}
+		} else {
+			DHD_PKT_LOG(("%s: prot = %x\n", __FUNCTION__, prot));
+		}
+
+		break;
+	case ETHER_TYPE_ARP:
+		type = CPKT_LOG_TYPE_ARP;
+		/* Subtype = Last 8 bits of target IP address */
+		arp = (struct bcmarp *)(pkt_data + ETHER_HDR_LEN);
+		subtype = arp->dst_ip[IPV4_ADDR_LEN - 1];
+
+		DHD_PKT_LOG(("%s: type = ARP(%d), subtype = %x\n",
+			__FUNCTION__, type, subtype));
+
+		break;
+	case ETHER_TYPE_802_1X:
+		type = CPKT_LOG_TYPE_802_1X;
+		/* EAPOL for 802.3/Ethernet */
+		subtype = dhd_cpkt_log_get_802_1x_subtype((eapol_header_t *)pkt_data);
+
+		DHD_PKT_LOG(("%s: type = 802.1x(%d), subtype = %x\n",
+			__FUNCTION__, type, subtype));
+
+		break;
+	case ETHER_TYPE_IPV6:
+		ipv6 = (struct ipv6_hdr *)(pkt_data + ETHER_HDR_LEN);
+		if (ipv6->nexthdr == ICMPV6_HEADER_TYPE) {
+			type = CPKT_LOG_TYPE_ICMPv6;
+			icmpv6_hdr =
+			       (struct icmp6_hdr *)(pkt_data + ETHER_HDR_LEN + sizeof(*ipv6));
+			subtype = icmpv6_hdr->icmp6_type;
+
+			DHD_PKT_LOG(("%s: type = ICMPv6(%x), subtype = %x\n",
+				__FUNCTION__, type, subtype));
+		} else {
+			DHD_ERROR(("%s: unsupported ipv6 next header\n", __FUNCTION__));
+		}
+
+		break;
+	default:
+		DHD_ERROR(("%s: Invalid eth type (%x)\n", __FUNCTION__, eth_hdr->ether_type));
+		break;
+	}
+
+	return (subtype << CPKT_LOG_BIT_LEN_TYPE) | type;
+}
+
+static int
+dhd_cpkt_log_get_pkt_fate(dhd_pktlog_ring_info_t *pktlog_info)
+{
+	return pktlog_info->fate;
+}
+
+/*
+ * dhd_cpkt_log_build: prepare 22 bits of data as compact packet log format to report to big data
+ *
+ * pkt_info: one packet data from packet log
+ * curr_ts_nsec: current time (nano seconds)
+ * cpkt: pointer for output(22 bits compact packet log)
+ *
+ */
+static int
+dhd_cpkt_log_build(dhd_pktlog_t *pktlog, dhd_pktlog_ring_info_t *pkt_info,
+	u64 curr_ts_nsec, int *cpkt)
+{
+	int ret;
+	int mask;
+	int temp = 0;
+
+	/* Timestamp index */
+	ret = dhd_cpkt_log_get_ts_idx(pktlog, pkt_info, curr_ts_nsec);
+	if (ret < 0) {
+		DHD_ERROR(("%s: Invalid cpktlog ts, err = %d\n", __FUNCTION__, ret));
+		return ret;
+	}
+	mask = CPKT_LOG_BIT_MASK_TS;
+	temp |= ((ret & mask) << CPKT_LOG_BIT_OFFSET_TS);
+
+	/* Direction: Tx/Rx */
+	ret = dhd_cpkt_log_get_direction(pkt_info);
+	mask = CPKT_LOG_BIT_MASK_DIR;
+	temp |= ((ret & mask) << CPKT_LOG_BIT_OFFSET_DIR);
+
+	/* Info = Packet Type & Packet Subtype */
+	ret = dhd_cpkt_log_get_pkt_info(pktlog, pkt_info);
+	if (ret < 0) {
+		DHD_ERROR(("%s: Invalid cpktlog info, err = %d\n", __FUNCTION__, ret));
+		return ret;
+	}
+	mask = CPKT_LOG_BIT_MASK_SUBTYPE << CPKT_LOG_BIT_LEN_TYPE | CPKT_LOG_BIT_MASK_TYPE;
+	temp |= ((ret & mask) << CPKT_LOG_BIT_OFFSET_TYPE);
+
+	/* Packet Fate */
+	ret = dhd_cpkt_log_get_pkt_fate(pkt_info);
+	mask = CPKT_LOG_BIT_MASK_PKT_FATE;
+	temp |= ((ret & mask) << CPKT_LOG_BIT_OFFSET_PKT_FATE);
+
+	*cpkt = temp;
+
+	return BCME_OK;
+}
+
+int
+dhd_cpkt_log_proc(dhd_pub_t *dhdp, char *buf, int buf_len, int bit_offset, int req_pkt_num)
+{
+	int ret;
+	int cpkt;
+	int offset = bit_offset;
+	dll_t *item_p, *prev_p;
+
+	uint8 pkt_cnt;
+	u64 curr_ts_nsec;
+
+	dhd_pktlog_t *pktlog;
+	dhd_pktlog_ring_t *pktlog_rbuf;
+
+	if (!dhdp || !dhdp->pktlog) {
+		DHD_ERROR(("%s: dhdp or pktlog is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	if (!dhdp->pktlog->pktlog_ring) {
+		DHD_ERROR(("%s: pktlog_ring is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	DHD_PKT_LOG(("%s: start cpkt log\n", __FUNCTION__));
+
+	pktlog = dhdp->pktlog;
+	pktlog_rbuf = pktlog->pktlog_ring;
+
+	req_pkt_num = req_pkt_num > CPKT_LOG_MAX_NUM ?
+		CPKT_LOG_MAX_NUM : req_pkt_num;
+
+	pkt_cnt = 0;
+	curr_ts_nsec = local_clock();
+	for (item_p = dll_tail_p(&pktlog_rbuf->ring_info_head);
+		!dll_end(&pktlog_rbuf->ring_info_head, item_p);
+		item_p = prev_p) {
+		prev_p = dll_prev_p(item_p);
+		if (prev_p == NULL)
+			break;
+
+		ret = dhd_cpkt_log_build(pktlog, (dhd_pktlog_ring_info_t *)item_p,
+			curr_ts_nsec, &cpkt);
+		if (ret < 0)
+			continue;
+
+		offset = dhd_bit_pack(buf, buf_len, offset, cpkt, CPKT_LOG_BIT_SIZE);
+
+		pkt_cnt++;
+		if (pkt_cnt >= req_pkt_num)
+			break;
+	}
+
+	return offset;
+}
+
+static void
+dhd_cpkt_log_insert_ts(dhd_cpkt_log_ts_node_t *node, struct rb_root *root)
+{
+	struct rb_node **new = &root->rb_node, *parent = NULL;
+	u64 ts_diff = node->ts_diff;
+
+	while (*new) {
+		parent = *new;
+		if (ts_diff < rb_entry(parent, dhd_cpkt_log_ts_node_t, rb)->ts_diff)
+			new = &parent->rb_left;
+		else
+			new = &parent->rb_right;
+	}
+
+	rb_link_node(&node->rb, parent, new);
+	rb_insert_color(&node->rb, root);
+}
+
+static void
+dhd_cpkt_log_deinit_tt(dhd_pub_t *dhdp)
+{
+	struct rb_node *n;
+	dhd_pktlog_t *pktlog = dhdp->pktlog;
+
+	dhd_cpkt_log_ts_node_t *node;
+
+	while ((n = rb_first(&pktlog->cpkt_log_tt_rbt))) {
+		node = rb_entry(n, dhd_cpkt_log_ts_node_t, rb);
+		rb_erase(&node->rb, &pktlog->cpkt_log_tt_rbt);
+		MFREE(dhdp->osh, node, sizeof(*node));
+	}
+}
+
+static int
+dhd_cpkt_log_init_tt(dhd_pub_t *dhdp)
+{
+	int i;
+	int ret = BCME_OK;
+
+	dhd_pktlog_t *pktlog = dhdp->pktlog;
+
+	dhd_cpkt_log_ts_node_t *node;
+
+	for (i = 0; i < ARRAYSIZE(dhd_cpkt_log_tt_idx); i++) {
+		node = (dhd_cpkt_log_ts_node_t *)MALLOCZ(dhdp->osh, sizeof(*node));
+		if (!node) {
+			ret = BCME_NOMEM;
+			goto exit;
+		}
+		node->ts_diff = dhd_cpkt_log_tt_idx[i];
+		node->idx = i;
+
+		dhd_cpkt_log_insert_ts(node, &pktlog->cpkt_log_tt_rbt);
+	}
+
+	return BCME_OK;
+exit:
+	dhd_cpkt_log_deinit_tt(dhdp);
+
+	return ret;
+}
+#endif	/* DHD_COMPACT_PKT_LOG */
+#endif /* DHD_PKT_LOGGING */
diff --git a/drivers/net/wireless/ap12275/dhd_pktlog.h b/drivers/net/wireless/ap12275/dhd_pktlog.h
new file mode 100644
index 000000000000..f4f9f26681f6
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pktlog.h
@@ -0,0 +1,325 @@
+/*
+ * DHD debugability packet logging header file
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_PKTLOG_H_
+#define __DHD_PKTLOG_H_
+
+#include <dhd_debug.h>
+#include <dhd.h>
+#include <asm/atomic.h>
+#ifdef DHD_COMPACT_PKT_LOG
+#include <linux/rbtree.h>
+#endif	/* DHD_COMPACT_PKT_LOG */
+
+#ifdef DHD_PKT_LOGGING
+#define DHD_PKT_LOG(args)	DHD_INFO(args)
+#define DEFAULT_MULTIPLE_PKTLOG_BUF	1
+#ifndef CUSTOM_MULTIPLE_PKTLOG_BUF
+#define CUSTOM_MULTIPLE_PKTLOG_BUF	DEFAULT_MULTIPLE_PKTLOG_BUF
+#endif /* CUSTOM_MULTIPLE_PKTLOG_BUF */
+#define MIN_PKTLOG_LEN			(32 * 10 * 2 * CUSTOM_MULTIPLE_PKTLOG_BUF)
+#define MAX_PKTLOG_LEN			(32 * 10 * 2 * 10)
+#define MAX_DHD_PKTLOG_FILTER_LEN	14
+#define MAX_MASK_PATTERN_FILTER_LEN	64
+#define PKTLOG_TXPKT_CASE		0x0001
+#define PKTLOG_TXSTATUS_CASE		0x0002
+#define PKTLOG_RXPKT_CASE		0x0004
+/* MAX_FILTER_PATTERN_LEN is buf len to print bitmask/pattern with string */
+#define MAX_FILTER_PATTERN_LEN \
+	((MAX_MASK_PATTERN_FILTER_LEN * HD_BYTE_SIZE) + HD_PREFIX_SIZE + 1) * 2
+#define PKTLOG_DUMP_BUF_SIZE		(64 * 1024)
+
+typedef struct dhd_dbg_pktlog_info {
+	frame_type payload_type;
+	size_t pkt_len;
+	uint32 driver_ts_sec;
+	uint32 driver_ts_usec;
+	uint32 firmware_ts;
+	uint32 pkt_hash;
+	uint32 tx_status_ts_sec;
+	uint32 tx_status_ts_usec;
+	bool direction;
+	void *pkt;
+} dhd_dbg_pktlog_info_t;
+
+typedef struct dhd_pktlog_ring_info
+{
+	dll_t p_info;			/* list pointer */
+	union {
+		wifi_tx_packet_fate tx_fate;
+		wifi_rx_packet_fate rx_fate;
+		uint32 fate;
+	};
+	dhd_dbg_pktlog_info_t info;
+} dhd_pktlog_ring_info_t;
+
+typedef struct dhd_pktlog_ring
+{
+	dll_t ring_info_head;		/* ring_info list */
+	dll_t ring_info_free;		/* ring_info free list */
+	osl_atomic_t start;
+	uint32 pktlog_minmize;
+	uint32 pktlog_len;		/* size of pkts */
+	uint32 pktcount;
+	spinlock_t *pktlog_ring_lock;
+	dhd_pub_t *dhdp;
+	dhd_pktlog_ring_info_t *ring_info_mem; /* ring_info mem pointer */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	void *dbg_ring;
+#endif /* DHD_PKT_LOGGING_DBGRING */
+} dhd_pktlog_ring_t;
+
+typedef struct dhd_pktlog_filter_info
+{
+	uint32 id;
+	uint32 offset;
+	uint32 size_bytes; /* Size of pattern. */
+	uint32 enable;
+	uint8 mask[MAX_MASK_PATTERN_FILTER_LEN];
+	uint8 pattern[MAX_MASK_PATTERN_FILTER_LEN];
+} dhd_pktlog_filter_info_t;
+
+typedef struct dhd_pktlog_filter
+{
+	dhd_pktlog_filter_info_t *info;
+	uint32 list_cnt;
+	uint32 enable;
+} dhd_pktlog_filter_t;
+
+typedef struct dhd_pktlog
+{
+	struct dhd_pktlog_ring *pktlog_ring;
+	struct dhd_pktlog_filter *pktlog_filter;
+	osl_atomic_t pktlog_status;
+	dhd_pub_t *dhdp;
+#ifdef DHD_COMPACT_PKT_LOG
+	struct rb_root cpkt_log_tt_rbt;
+#endif  /* DHD_COMPACT_PKT_LOG */
+#ifdef DHD_PKT_LOGGING_DBGRING
+	osl_atomic_t enable; /* logging suspend/resume */
+#endif /* DHD_PKT_LOGGING_DBGRING */
+} dhd_pktlog_t;
+
+typedef struct dhd_pktlog_pcap_hdr
+{
+	uint32 magic_number;
+	uint16 version_major;
+	uint16 version_minor;
+	uint16 thiszone;
+	uint32 sigfigs;
+	uint32 snaplen;
+	uint32 network;
+} dhd_pktlog_pcap_hdr_t;
+
+#define PKTLOG_PCAP_MAGIC_NUM 0xa1b2c3d4
+#define PKTLOG_PCAP_MAJOR_VER 0x02
+#define PKTLOG_PCAP_MINOR_VER 0x04
+#define PKTLOG_PCAP_SNAP_LEN 0x40000
+#define PKTLOG_PCAP_NETWORK_TYPE 147
+
+extern int dhd_os_attach_pktlog(dhd_pub_t *dhdp);
+extern int dhd_os_detach_pktlog(dhd_pub_t *dhdp);
+#ifdef DHD_PKT_LOGGING_DBGRING
+extern int dhd_pktlog_is_enabled(dhd_pub_t *dhdp);
+extern void dhd_pktlog_suspend(dhd_pub_t *dhdp);
+extern void dhd_pktlog_resume(dhd_pub_t *dhdp);
+extern int dhd_pktlog_ring_reinit(dhd_pub_t *dhdp);
+#endif /* DHD_PKT_LOGGING_DBGRING */
+extern dhd_pktlog_ring_t* dhd_pktlog_ring_init(dhd_pub_t *dhdp, int size);
+extern int dhd_pktlog_ring_deinit(dhd_pub_t *dhdp, dhd_pktlog_ring_t *ring);
+extern int dhd_pktlog_ring_set_nextpos(dhd_pktlog_ring_t *ringbuf);
+extern int dhd_pktlog_ring_get_nextbuf(dhd_pktlog_ring_t *ringbuf, void **data);
+extern int dhd_pktlog_ring_set_prevpos(dhd_pktlog_ring_t *ringbuf);
+extern int dhd_pktlog_ring_get_prevbuf(dhd_pktlog_ring_t *ringbuf, void **data);
+extern int dhd_pktlog_ring_get_writebuf(dhd_pktlog_ring_t *ringbuf, void **data);
+extern int dhd_pktlog_ring_add_pkts(dhd_pub_t *dhdp, void *pkt, void *pktdata, uint32 pktid,
+		uint32 direction);
+extern int dhd_pktlog_ring_tx_status(dhd_pub_t *dhdp, void *pkt, void *pktdata, uint32 pktid,
+		uint16 status);
+extern dhd_pktlog_ring_t* dhd_pktlog_ring_change_size(dhd_pktlog_ring_t *ringbuf, int size);
+extern void dhd_pktlog_filter_pull_forward(dhd_pktlog_filter_t *filter,
+		uint32 del_filter_id, uint32 list_cnt);
+
+#define PKT_RX 0
+#define PKT_TX 1
+#define PKT_WAKERX 2
+#define DHD_INVALID_PKTID (0U)
+#define PKTLOG_TRANS_TX 0x01
+#define PKTLOG_TRANS_RX 0x02
+#define PKTLOG_TRANS_TXS 0x04
+
+#define PKTLOG_SET_IN_TX(dhdp) \
+{ \
+	do { \
+		OSL_ATOMIC_OR((dhdp)->osh, &(dhdp)->pktlog->pktlog_status, PKTLOG_TRANS_TX); \
+	} while (0); \
+}
+
+#define PKTLOG_SET_IN_RX(dhdp) \
+{ \
+	do { \
+		OSL_ATOMIC_OR((dhdp)->osh, &(dhdp)->pktlog->pktlog_status, PKTLOG_TRANS_RX); \
+	} while (0); \
+}
+
+#define PKTLOG_SET_IN_TXS(dhdp) \
+{ \
+	do { \
+		OSL_ATOMIC_OR((dhdp)->osh, &(dhdp)->pktlog->pktlog_status, PKTLOG_TRANS_TXS); \
+	} while (0); \
+}
+
+#define PKTLOG_CLEAR_IN_TX(dhdp) \
+{ \
+	do { \
+		OSL_ATOMIC_AND((dhdp)->osh, &(dhdp)->pktlog->pktlog_status, ~PKTLOG_TRANS_TX); \
+	} while (0); \
+}
+
+#define PKTLOG_CLEAR_IN_RX(dhdp) \
+{ \
+	do { \
+		OSL_ATOMIC_AND((dhdp)->osh, &(dhdp)->pktlog->pktlog_status, ~PKTLOG_TRANS_RX); \
+	} while (0); \
+}
+
+#define PKTLOG_CLEAR_IN_TXS(dhdp) \
+{ \
+	do { \
+		OSL_ATOMIC_AND((dhdp)->osh, &(dhdp)->pktlog->pktlog_status, ~PKTLOG_TRANS_TXS); \
+	} while (0); \
+}
+
+#define DHD_PKTLOG_TX(dhdp, pkt, pktdata, pktid) \
+{ \
+	do { \
+		if ((dhdp) && (dhdp)->pktlog && (pkt)) { \
+			PKTLOG_SET_IN_TX(dhdp); \
+			if ((dhdp)->pktlog->pktlog_ring && \
+				OSL_ATOMIC_READ((dhdp)->osh, \
+					(&(dhdp)->pktlog->pktlog_ring->start))) { \
+				dhd_pktlog_ring_add_pkts(dhdp, pkt, pktdata, pktid, PKT_TX); \
+			} \
+			PKTLOG_CLEAR_IN_TX(dhdp); \
+		} \
+	} while (0); \
+}
+
+#define DHD_PKTLOG_TXS(dhdp, pkt, pktdata, pktid, status) \
+{ \
+	do { \
+		if ((dhdp) && (dhdp)->pktlog && (pkt)) { \
+			PKTLOG_SET_IN_TXS(dhdp); \
+			if ((dhdp)->pktlog->pktlog_ring && \
+				OSL_ATOMIC_READ((dhdp)->osh, \
+					(&(dhdp)->pktlog->pktlog_ring->start))) { \
+				dhd_pktlog_ring_tx_status(dhdp, pkt, pktdata, pktid, status); \
+			} \
+			PKTLOG_CLEAR_IN_TXS(dhdp); \
+		} \
+	} while (0); \
+}
+
+#define DHD_PKTLOG_RX(dhdp, pkt, pktdata) \
+{ \
+	do { \
+		if ((dhdp) && (dhdp)->pktlog && (pkt)) { \
+			PKTLOG_SET_IN_RX(dhdp); \
+			if (ntoh16((pkt)->protocol) != ETHER_TYPE_BRCM) { \
+				if ((dhdp)->pktlog->pktlog_ring && \
+					OSL_ATOMIC_READ((dhdp)->osh, \
+						(&(dhdp)->pktlog->pktlog_ring->start))) { \
+					dhd_pktlog_ring_add_pkts(dhdp, pkt, pktdata, \
+						DHD_INVALID_PKTID, PKT_RX); \
+				} \
+			} \
+			PKTLOG_CLEAR_IN_RX(dhdp); \
+		} \
+	} while (0); \
+}
+
+#define DHD_PKTLOG_WAKERX(dhdp, pkt, pktdata) \
+{ \
+	do { \
+		if ((dhdp) && (dhdp)->pktlog && (pkt)) { \
+			PKTLOG_SET_IN_RX(dhdp); \
+			if (ntoh16((pkt)->protocol) != ETHER_TYPE_BRCM) { \
+				if ((dhdp)->pktlog->pktlog_ring && \
+					OSL_ATOMIC_READ((dhdp)->osh, \
+						(&(dhdp)->pktlog->pktlog_ring->start))) { \
+					dhd_pktlog_ring_add_pkts(dhdp, pkt, pktdata, \
+						DHD_INVALID_PKTID, PKT_WAKERX); \
+				} \
+			} \
+			PKTLOG_CLEAR_IN_RX(dhdp); \
+		} \
+	} while (0); \
+}
+
+extern dhd_pktlog_filter_t* dhd_pktlog_filter_init(int size);
+extern int dhd_pktlog_filter_deinit(dhd_pktlog_filter_t *filter);
+extern int dhd_pktlog_filter_add(dhd_pktlog_filter_t *filter, char *arg);
+extern int dhd_pktlog_filter_del(dhd_pktlog_filter_t *filter, char *arg);
+extern int dhd_pktlog_filter_enable(dhd_pktlog_filter_t *filter, uint32 pktlog_case, uint32 enable);
+extern int dhd_pktlog_filter_pattern_enable(dhd_pktlog_filter_t *filter, char *arg, uint32 enable);
+extern int dhd_pktlog_filter_info(dhd_pktlog_filter_t *filter);
+extern bool dhd_pktlog_filter_matched(dhd_pktlog_filter_t *filter, char *data, uint32 pktlog_case);
+extern bool dhd_pktlog_filter_existed(dhd_pktlog_filter_t *filter, char *arg, uint32 *id);
+
+#define DHD_PKTLOG_FILTER_ADD(pattern, filter_pattern, dhdp)	\
+{	\
+	do {	\
+		if ((strlen(pattern) + 1) < sizeof(filter_pattern)) {	\
+			strncpy(filter_pattern, pattern, sizeof(filter_pattern));	\
+			dhd_pktlog_filter_add(dhdp->pktlog->pktlog_filter, filter_pattern);	\
+		}	\
+	} while (0);	\
+}
+
+#define DHD_PKTLOG_DUMP_PATH	DHD_COMMON_DUMP_PATH
+extern int dhd_pktlog_debug_dump(dhd_pub_t *dhdp);
+extern void dhd_pktlog_dump(void *handle, void *event_info, u8 event);
+extern void dhd_schedule_pktlog_dump(dhd_pub_t *dhdp);
+extern int dhd_pktlog_dump_write_memory(dhd_pub_t *dhdp, const void *user_buf, uint32 size);
+extern int dhd_pktlog_dump_write_file(dhd_pub_t *dhdp);
+
+#define DHD_PKTLOG_FATE_INFO_STR_LEN 256
+#define DHD_PKTLOG_FATE_INFO_FORMAT	"BRCM_Packet_Fate"
+#define DHD_PKTLOG_DUMP_TYPE "pktlog_dump"
+#define DHD_PKTLOG_DEBUG_DUMP_TYPE "pktlog_debug_dump"
+
+extern void dhd_pktlog_get_filename(dhd_pub_t *dhdp, char *dump_path, int len);
+extern uint32 dhd_pktlog_get_item_length(dhd_pktlog_ring_info_t *report_ptr);
+extern int dhd_pktlog_get_dump_length(dhd_pub_t *dhdp);
+extern uint32 __dhd_dbg_pkt_hash(uintptr_t pkt, uint32 pktid);
+
+#ifdef DHD_COMPACT_PKT_LOG
+#define CPKT_LOG_BIT_SIZE		22
+#define CPKT_LOG_MAX_NUM		80
+extern int dhd_cpkt_log_proc(dhd_pub_t *dhdp, char *buf, int buf_len,
+        int bit_offset, int req_pkt_num);
+#endif  /* DHD_COMPACT_PKT_LOG */
+#endif /* DHD_PKT_LOGGING */
+#endif /* __DHD_PKTLOG_H_ */
diff --git a/drivers/net/wireless/ap12275/dhd_plat.h b/drivers/net/wireless/ap12275/dhd_plat.h
new file mode 100644
index 000000000000..ec19a28efc78
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_plat.h
@@ -0,0 +1,103 @@
+/*
+ * DHD Linux platform header file
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef __DHD_PLAT_H__
+#define __DHD_PLAT_H__
+
+#if defined(__linux__)
+
+#include <linuxver.h>
+
+#if !defined(CONFIG_WIFI_CONTROL_FUNC)
+#define WLAN_PLAT_NODFS_FLAG	0x01
+#define WLAN_PLAT_AP_FLAG	0x02
+struct wifi_platform_data {
+	int (*set_power)(int val, wifi_adapter_info_t *adapter);
+	int (*set_reset)(int val);
+	int (*set_carddetect)(int val);
+#ifdef DHD_COREDUMP
+	int (*set_coredump)(const char *buf, int buf_len, const char *info);
+#endif /* DHD_COREDUMP */
+#if defined(BCMDHD_MDRIVER) && !defined(DHD_STATIC_IN_DRIVER)
+	void *(*mem_prealloc)(uint bus_type, int index, int section, unsigned long size);
+#else
+	void *(*mem_prealloc)(int section, unsigned long size);
+#endif
+	int (*get_mac_addr)(unsigned char *buf, int ifidx);
+#ifdef BCMSDIO
+	int (*get_wake_irq)(void);
+#endif
+#ifdef CUSTOM_FORCE_NODFS_FLAG
+	void *(*get_country_code)(char *ccode, u32 flags);
+#else /* defined (CUSTOM_FORCE_NODFS_FLAG) */
+	void *(*get_country_code)(char *ccode);
+#endif
+};
+#endif
+
+#include <linux/pci.h>
+
+/*
+ * Plat Layer defines the interfaces that the BSP specific file should override
+ * The default implementation of the interfaces are present in dhd_linux_platdev.c
+ * The data structure/handle to be passed by the DHD to BSP specific file is opaque
+ * called plat_info. The data structure is to be maintained purely inside the
+ * BSP specific file and hence its kept opaque.
+ *
+ * There are two types of interface functions
+ * 1) Functions that need the interface structure plat_info to be passed down
+ *    from DHD.
+ * 2) Functions that queries for certain information in BSP specific way and just
+ *    returns to DHD - These functions does not take the plat_info as argument.
+ *
+ * The declarations are grouped accordingly. While adding a new interface function
+ * declaration based on the group it belongs to add it in the appropriate section
+ */
+
+typedef void (*dhd_pcie_event_cb_t) (struct pci_dev *pdev);
+extern int dhd_plat_pcie_register_event(void *plat_info,
+		struct pci_dev *pdev, dhd_pcie_event_cb_t pfn);
+extern void dhd_plat_pcie_deregister_event(void *plat_info);
+extern void dhd_plat_report_bh_sched(void *plat_info, int resched);
+extern int dhd_plat_pcie_suspend(void *plat_info);
+extern int dhd_plat_pcie_resume(void *plat_info);
+extern void dhd_plat_pcie_register_dump(void *plat_info);
+extern void dhd_plat_pin_dbg_show(void *plat_info);
+
+extern uint32 dhd_plat_get_info_size(void);
+extern void dhd_plat_l1ss_ctrl(bool ctrl);
+
+/* To be called when we intend to exit L1 while performing wreg, rreg operations */
+extern void dhd_plat_l1_exit_io(void);
+
+/* To be called when we intend to exit L1 in non-io case */
+extern void dhd_plat_l1_exit(void);
+
+extern uint32 dhd_plat_get_rc_vendor_id(void);
+extern uint32 dhd_plat_get_rc_device_id(void);
+
+extern uint16 dhd_plat_align_rxbuf_size(uint16 rxbufpost_sz);
+#endif /* __linux__ */
+#endif /* __DHD_PLAT_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_pno.c b/drivers/net/wireless/ap12275/dhd_pno.c
new file mode 100644
index 000000000000..b097a1de508d
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pno.c
@@ -0,0 +1,4906 @@
+/*
+ * Broadcom Dongle Host Driver (DHD)
+ * Prefered Network Offload and Wi-Fi Location Service(WLS) code.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#if defined(GSCAN_SUPPORT) && !defined(PNO_SUPPORT)
+#error "GSCAN needs PNO to be enabled!"
+#endif
+
+#ifdef PNO_SUPPORT
+#include <typedefs.h>
+#include <osl.h>
+
+#include <epivers.h>
+#include <bcmutils.h>
+
+#include <bcmendian.h>
+
+#include <linuxver.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/sort.h>
+
+#include <dngl_stats.h>
+#include <wlioctl.h>
+
+#include <bcmevent.h>
+#include <dhd.h>
+#include <dhd_pno.h>
+#include <dhd_dbg.h>
+#ifdef GSCAN_SUPPORT
+#include <linux/gcd.h>
+#endif /* GSCAN_SUPPORT */
+#ifdef WL_CFG80211
+#include <wl_cfg80211.h>
+#endif /* WL_CFG80211 */
+
+#ifdef __BIG_ENDIAN
+#include <bcmendian.h>
+#define htod32(i) (bcmswap32(i))
+#define htod16(i) (bcmswap16(i))
+#define dtoh32(i) (bcmswap32(i))
+#define dtoh16(i) (bcmswap16(i))
+#define htodchanspec(i) htod16(i)
+#define dtohchanspec(i) dtoh16(i)
+#else
+#define htod32(i) (i)
+#define htod16(i) (i)
+#define dtoh32(i) (i)
+#define dtoh16(i) (i)
+#define htodchanspec(i) (i)
+#define dtohchanspec(i) (i)
+#endif /* IL_BIGENDINA */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+#define COMPLETION_WAIT_QUEUE_ACTIVE(wait_queue) swait_active(wait_queue)
+#else
+#define COMPLETION_WAIT_QUEUE_ACTIVE(wait_queue) waitqueue_active(wait_queue)
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0) */
+
+#ifdef CUSTOM_PREFIX
+#define PNO_PRINT_PREFIX "[%s]"CUSTOM_PREFIX, OSL_GET_RTCTIME()
+#define PNO_PRINT_SYSTEM_TIME pr_cont(PNO_PRINT_PREFIX)
+#define PNO_CONS_ONLY(args)     \
+do {    \
+	PNO_PRINT_SYSTEM_TIME;  \
+	pr_cont args;           \
+} while (0)
+#else
+#define PNO_PRINT_SYSTEM_TIME
+#define PNO_CONS_ONLY(args) do { printf args;} while (0)
+#endif /* CUSTOM_PREFIX */
+
+#define NULL_CHECK(p, s, err)  \
+do { \
+	if (!(p)) { \
+		PNO_CONS_ONLY(("NULL POINTER (%s) : %s\n", __FUNCTION__, (s))); \
+		err = BCME_ERROR; \
+		return err; \
+	} \
+} while (0)
+
+#define PNO_GET_PNOSTATE(dhd) ((dhd_pno_status_info_t *)dhd->pno_state)
+
+#define PNO_BESTNET_LEN		WLC_IOCTL_MEDLEN
+
+#define PNO_ON 1
+#define PNO_OFF 0
+#define CHANNEL_2G_MIN 1
+#define CHANNEL_2G_MAX 14
+#define CHANNEL_5G_MIN 34
+#define CHANNEL_5G_MAX 165
+#define IS_2G_CHANNEL(ch) ((ch >= CHANNEL_2G_MIN) && \
+	(ch <= CHANNEL_2G_MAX))
+#define IS_5G_CHANNEL(ch) ((ch >= CHANNEL_5G_MIN) && \
+	(ch <= CHANNEL_5G_MAX))
+#define MAX_NODE_CNT 5
+#define WLS_SUPPORTED(pno_state) (pno_state->wls_supported == TRUE)
+#define TIME_DIFF(timestamp1, timestamp2) (abs((uint32)(timestamp1/1000)  \
+						- (uint32)(timestamp2/1000)))
+#define TIME_DIFF_MS(timestamp1, timestamp2) (abs((uint32)(timestamp1)  \
+						- (uint32)(timestamp2)))
+
+#define ENTRY_OVERHEAD strlen("bssid=\nssid=\nfreq=\nlevel=\nage=\ndist=\ndistSd=\n====")
+#define TIME_MIN_DIFF 5
+
+#define EVENT_DATABUF_MAXLEN	(512 - sizeof(bcm_event_t))
+#define EVENT_MAX_NETCNT_V1 \
+	((EVENT_DATABUF_MAXLEN - sizeof(wl_pfn_scanresults_v1_t)) \
+	/ sizeof(wl_pfn_net_info_v1_t) + 1)
+#define EVENT_MAX_NETCNT_V2 \
+	((EVENT_DATABUF_MAXLEN - sizeof(wl_pfn_scanresults_v2_t)) \
+	/ sizeof(wl_pfn_net_info_v2_t) + 1)
+#define EVENT_MAX_NETCNT_V3 \
+	((EVENT_DATABUF_MAXLEN - sizeof(wl_pfn_scanresults_v3_t)) \
+	/ sizeof(wl_pfn_net_info_v3_t) + 1)
+
+#ifdef GSCAN_SUPPORT
+static int _dhd_pno_flush_ssid(dhd_pub_t *dhd);
+static wl_pfn_gscan_ch_bucket_cfg_t *
+dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd, dhd_pno_status_info_t *pno_state,
+	uint16 *chan_list, uint32 *num_buckets, uint32 *num_buckets_to_fw);
+#endif /* GSCAN_SUPPORT */
+
+static int dhd_pno_set_legacy_pno(dhd_pub_t *dhd, uint16  scan_fr, int pno_repeat,
+	int pno_freq_expo_max, uint16 *channel_list, int nchan);
+
+static inline bool
+is_dfs(dhd_pub_t *dhd, uint16 channel)
+{
+	u32 ch;
+	s32 err;
+	u8 buf[32];
+
+	ch = wl_ch_host_to_driver(channel);
+	err = dhd_iovar(dhd, 0, "per_chan_info", (char *)&ch,
+		sizeof(u32), buf, sizeof(buf), FALSE);
+	if (unlikely(err)) {
+		DHD_ERROR(("get per chan info failed:%d\n", err));
+		return FALSE;
+	}
+	/* Check the channel flags returned by fw */
+	if (*((u32 *)buf) & WL_CHAN_PASSIVE) {
+		return TRUE;
+	}
+	return FALSE;
+}
+
+int
+dhd_pno_clean(dhd_pub_t *dhd)
+{
+	int pfn = 0;
+	int err;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	/* Disable PNO */
+	err = dhd_iovar(dhd, 0, "pfn", (char *)&pfn, sizeof(pfn), NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn(error : %d)\n",
+			__FUNCTION__, err));
+		goto exit;
+	}
+	_pno_state->pno_status = DHD_PNO_DISABLED;
+	err = dhd_iovar(dhd, 0, "pfnclear", NULL, 0, NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfnclear(error : %d)\n",
+			__FUNCTION__, err));
+	}
+exit:
+	return err;
+}
+
+bool
+dhd_is_pno_supported(dhd_pub_t *dhd)
+{
+	dhd_pno_status_info_t *_pno_state;
+
+	if (!dhd || !dhd->pno_state) {
+		DHD_ERROR(("NULL POINTER : %s\n",
+			__FUNCTION__));
+		return FALSE;
+	}
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	return WLS_SUPPORTED(_pno_state);
+}
+
+bool
+dhd_is_legacy_pno_enabled(dhd_pub_t *dhd)
+{
+	dhd_pno_status_info_t *_pno_state;
+
+	if (!dhd || !dhd->pno_state) {
+		DHD_ERROR(("NULL POINTER : %s\n",
+			__FUNCTION__));
+		return FALSE;
+	}
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	return ((_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) != 0);
+}
+
+#ifdef GSCAN_SUPPORT
+static uint64
+convert_fw_rel_time_to_systime(struct osl_timespec *ts, uint32 fw_ts_ms)
+{
+	return ((uint64)(TIMESPEC64_TO_US(*ts)) - (uint64)(fw_ts_ms * USEC_PER_MSEC));
+}
+
+static void
+dhd_pno_idx_to_ssid(struct dhd_pno_gscan_params *gscan_params,
+		dhd_epno_results_t *res, uint32 idx)
+{
+	dhd_pno_ssid_t *iter, *next;
+	int i;
+
+	/* If idx doesn't make sense */
+	if (idx >= gscan_params->epno_cfg.num_epno_ssid) {
+		DHD_ERROR(("No match, idx %d num_ssid %d\n", idx,
+			gscan_params->epno_cfg.num_epno_ssid));
+		goto exit;
+	}
+
+	if (gscan_params->epno_cfg.num_epno_ssid > 0) {
+		i = 0;
+
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		list_for_each_entry_safe(iter, next,
+			&gscan_params->epno_cfg.epno_ssid_list, list) {
+			GCC_DIAGNOSTIC_POP();
+			if (i++ == idx) {
+				memcpy(res->ssid, iter->SSID, iter->SSID_len);
+				res->ssid_len = iter->SSID_len;
+				return;
+			}
+		}
+	}
+exit:
+	/* If we are here then there was no match */
+	res->ssid[0] = '\0';
+	res->ssid_len = 0;
+	return;
+}
+
+/* Translate HAL flag bitmask to BRCM FW flag bitmask */
+void
+dhd_pno_translate_epno_fw_flags(uint32 *flags)
+{
+	uint32 in_flags, fw_flags = 0;
+	in_flags = *flags;
+
+	if (in_flags & DHD_EPNO_A_BAND_TRIG) {
+		fw_flags |= WL_PFN_SSID_A_BAND_TRIG;
+	}
+
+	if (in_flags & DHD_EPNO_BG_BAND_TRIG) {
+		fw_flags |= WL_PFN_SSID_BG_BAND_TRIG;
+	}
+
+	if (!(in_flags & DHD_EPNO_STRICT_MATCH) &&
+			!(in_flags & DHD_EPNO_HIDDEN_SSID)) {
+		fw_flags |= WL_PFN_SSID_IMPRECISE_MATCH;
+	}
+
+	if (in_flags & DHD_EPNO_SAME_NETWORK) {
+		fw_flags |= WL_PFN_SSID_SAME_NETWORK;
+	}
+
+	/* Add any hard coded flags needed */
+	fw_flags |= WL_PFN_SUPPRESS_AGING_MASK;
+	*flags = fw_flags;
+
+	return;
+}
+
+/* Translate HAL auth bitmask to BRCM FW bitmask */
+void
+dhd_pno_set_epno_auth_flag(uint32 *wpa_auth)
+{
+	switch (*wpa_auth) {
+		case DHD_PNO_AUTH_CODE_OPEN:
+			*wpa_auth = WPA_AUTH_DISABLED;
+			break;
+		case DHD_PNO_AUTH_CODE_PSK:
+			*wpa_auth = (WPA_AUTH_PSK | WPA2_AUTH_PSK);
+			break;
+		case DHD_PNO_AUTH_CODE_EAPOL:
+			*wpa_auth = ~WPA_AUTH_NONE;
+			break;
+		default:
+			DHD_ERROR(("%s: Unknown auth %d", __FUNCTION__, *wpa_auth));
+			*wpa_auth = WPA_AUTH_PFN_ANY;
+			break;
+	}
+	return;
+}
+
+/* Cleanup all results */
+static void
+dhd_gscan_clear_all_batch_results(dhd_pub_t *dhd)
+{
+	struct dhd_pno_gscan_params *gscan_params;
+	dhd_pno_status_info_t *_pno_state;
+	gscan_results_cache_t *iter;
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	gscan_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan;
+	iter = gscan_params->gscan_batch_cache;
+	/* Mark everything as consumed */
+	while (iter) {
+		iter->tot_consumed = iter->tot_count;
+		iter = iter->next;
+	}
+	dhd_gscan_batch_cache_cleanup(dhd);
+	return;
+}
+
+static int
+_dhd_pno_gscan_cfg(dhd_pub_t *dhd, wl_pfn_gscan_cfg_t *pfncfg_gscan_param, int size)
+{
+	int err = BCME_OK;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	err = dhd_iovar(dhd, 0, "pfn_gscan_cfg", (char *)pfncfg_gscan_param, size, NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfncfg_gscan_param\n", __FUNCTION__));
+		goto exit;
+	}
+exit:
+	return err;
+}
+
+static int
+_dhd_pno_flush_ssid(dhd_pub_t *dhd)
+{
+	int err;
+	wl_pfn_t pfn_elem;
+	memset(&pfn_elem, 0, sizeof(wl_pfn_t));
+	pfn_elem.flags = htod32(WL_PFN_FLUSH_ALL_SSIDS);
+
+	err = dhd_iovar(dhd, 0, "pfn_add", (char *)&pfn_elem, sizeof(wl_pfn_t), NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_add\n", __FUNCTION__));
+	}
+	return err;
+}
+
+static bool
+is_batch_retrieval_complete(struct dhd_pno_gscan_params *gscan_params)
+{
+	smp_rmb();
+	return (gscan_params->get_batch_flag == GSCAN_BATCH_RETRIEVAL_COMPLETE);
+}
+#endif /* GSCAN_SUPPORT */
+
+static int
+_dhd_pno_suspend(dhd_pub_t *dhd)
+{
+	int err;
+	int suspend = 1;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	err = dhd_iovar(dhd, 0, "pfn_suspend", (char *)&suspend, sizeof(suspend), NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to suspend pfn(error :%d)\n", __FUNCTION__, err));
+		goto exit;
+
+	}
+	_pno_state->pno_status = DHD_PNO_SUSPEND;
+exit:
+	return err;
+}
+
+static int
+_dhd_pno_enable(dhd_pub_t *dhd, int enable)
+{
+	int err = BCME_OK;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (enable & 0xfffe) {
+		DHD_ERROR(("%s invalid value\n", __FUNCTION__));
+		err = BCME_BADARG;
+		goto exit;
+	}
+	if (!dhd_support_sta_mode(dhd)) {
+		DHD_ERROR(("PNO is not allowed for non-STA mode"));
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	/* Enable/Disable PNO */
+	err = dhd_iovar(dhd, 0, "pfn", (char *)&enable, sizeof(enable), NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_set - %d\n", __FUNCTION__, err));
+		goto exit;
+	}
+	_pno_state->pno_status = (enable)?
+		DHD_PNO_ENABLED : DHD_PNO_DISABLED;
+	if (!enable)
+		_pno_state->pno_mode = DHD_PNO_NONE_MODE;
+
+	DHD_PNO(("%s set pno as %s\n",
+		__FUNCTION__, enable ? "Enable" : "Disable"));
+exit:
+	return err;
+}
+
+static int
+_dhd_pno_set(dhd_pub_t *dhd, const dhd_pno_params_t *pno_params, dhd_pno_mode_t mode)
+{
+	int err = BCME_OK;
+	wl_pfn_param_v3_t pfn_param;
+	dhd_pno_params_t *_params;
+	dhd_pno_status_info_t *_pno_state;
+	bool combined_scan = FALSE;
+	uint16 size;
+	bool use_v3 = FALSE;
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	/* Query pfn version */
+	bzero(&pfn_param, sizeof(pfn_param));
+	err = dhd_iovar(dhd, 0, "pfn_set", (char *)&pfn_param, sizeof(pfn_param),
+		(char *)&pfn_param, sizeof(pfn_param), FALSE);
+	if (err < 0) {
+		if (err == BCME_UNSUPPORTED) {
+			DHD_PNO(("%s : PFN versioning not supported. Use v2\n",
+				__FUNCTION__));
+			use_v3 = FALSE;
+		} else {
+			DHD_ERROR(("%s : failed to query pfn_set %d\n", __FUNCTION__, err));
+			goto exit;
+		}
+	} else {
+		if (pfn_param.version == PFN_VERSION_V3) {
+			DHD_ERROR(("%s : using pfn_param v3\n", __FUNCTION__));
+			use_v3 = TRUE;
+		} else if (pfn_param.version == PFN_VERSION_V2) {
+			DHD_ERROR(("%s : using pfn_param v2\n", __FUNCTION__));
+			use_v3 = FALSE;
+		}  else {
+			DHD_ERROR(("unsupported pfn ver:%d\n", pfn_param.version));
+			err = BCME_UNSUPPORTED;
+			goto exit;
+		}
+	}
+
+	/* set pfn parameters */
+	bzero(&pfn_param, sizeof(pfn_param));
+	if (use_v3) {
+		pfn_param.version = PFN_VERSION_V3;
+		pfn_param.version = htod32(pfn_param.version);
+		size = sizeof(wl_pfn_param_v3_t);
+		pfn_param.length = htod32(size);
+	} else {
+		wl_pfn_param_v2_t *pfn_param_v2 = (wl_pfn_param_v2_t *)&pfn_param;
+		pfn_param_v2->version = PFN_VERSION_V2;
+		pfn_param_v2->version = htod32(pfn_param_v2->version);
+		size = sizeof(wl_pfn_param_v2_t);
+	}
+
+	pfn_param.flags = ((PFN_LIST_ORDER << SORT_CRITERIA_BIT) |
+		(ENABLE << IMMEDIATE_SCAN_BIT) | (ENABLE << REPORT_SEPERATELY_BIT));
+#ifdef WL_SCHED_SCAN
+	/* bit to select the pfn partial scan result event logic */
+	pfn_param.flags |= htod16(ENABLE << PFN_FULL_SCAN_RESULT_BIT);
+#endif /* WL_SCHED_SCAN */
+	if (mode == DHD_PNO_LEGACY_MODE) {
+		pfn_param.repeat = (uchar) (pno_params->params_legacy.pno_repeat);
+		/* check and set extra pno params */
+		if ((pno_params->params_legacy.pno_repeat != 0) &&
+			(pno_params->params_legacy.pno_freq_expo_max != 0)) {
+			pfn_param.flags |= htod16(ENABLE << ENABLE_ADAPTSCAN_BIT);
+			pfn_param.exp = (uchar) (pno_params->params_legacy.pno_freq_expo_max);
+		}
+		/* set up pno scan fr */
+		if (pno_params->params_legacy.scan_fr != 0)
+			pfn_param.scan_freq = htod32(pno_params->params_legacy.scan_fr);
+		if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+			DHD_PNO(("will enable combined scan with BATCHIG SCAN MODE\n"));
+			mode |= DHD_PNO_BATCH_MODE;
+			combined_scan = TRUE;
+		} else if (_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE) {
+			DHD_PNO(("will enable combined scan with HOTLIST SCAN MODE\n"));
+			mode |= DHD_PNO_HOTLIST_MODE;
+			combined_scan = TRUE;
+		}
+#ifdef GSCAN_SUPPORT
+		else if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+			DHD_PNO(("will enable combined scan with GSCAN SCAN MODE\n"));
+			mode |= DHD_PNO_GSCAN_MODE;
+		}
+#endif /* GSCAN_SUPPORT */
+	}
+	if (mode & (DHD_PNO_BATCH_MODE | DHD_PNO_HOTLIST_MODE)) {
+		/* Scan frequency of 30 sec */
+		pfn_param.scan_freq = htod32(30);
+		/* slow adapt scan is off by default */
+		pfn_param.slow_freq = htod32(0);
+		/* RSSI margin of 30 dBm */
+		pfn_param.rssi_margin = htod16(PNO_RSSI_MARGIN_DBM);
+		/* Network timeout 60 sec */
+		pfn_param.lost_network_timeout = htod32(60);
+		/* best n = 2 by default */
+		pfn_param.bestn = DEFAULT_BESTN;
+		/* mscan m=0 by default, so not record best networks by default */
+		pfn_param.mscan = DEFAULT_MSCAN;
+		/*  default repeat = 10 */
+		pfn_param.repeat = DEFAULT_REPEAT;
+		/* by default, maximum scan interval = 2^2
+		 * scan_freq when adaptive scan is turned on
+		 */
+		pfn_param.exp = DEFAULT_EXP;
+		if (mode == DHD_PNO_BATCH_MODE) {
+			/* In case of BATCH SCAN */
+			if (pno_params->params_batch.bestn)
+				pfn_param.bestn = pno_params->params_batch.bestn;
+			if (pno_params->params_batch.scan_fr)
+				pfn_param.scan_freq = htod32(pno_params->params_batch.scan_fr);
+			if (pno_params->params_batch.mscan)
+				pfn_param.mscan = pno_params->params_batch.mscan;
+			/* enable broadcast scan */
+			pfn_param.flags |= (ENABLE << ENABLE_BD_SCAN_BIT);
+		} else if (mode == DHD_PNO_HOTLIST_MODE) {
+			/* In case of HOTLIST SCAN */
+			if (pno_params->params_hotlist.scan_fr)
+				pfn_param.scan_freq = htod32(pno_params->params_hotlist.scan_fr);
+			pfn_param.bestn = 0;
+			pfn_param.repeat = 0;
+			/* enable broadcast scan */
+			pfn_param.flags |= (ENABLE << ENABLE_BD_SCAN_BIT);
+		}
+		if (combined_scan) {
+			/* Disable Adaptive Scan */
+			pfn_param.flags &= ~(htod16(ENABLE << ENABLE_ADAPTSCAN_BIT));
+			pfn_param.flags |= (ENABLE << ENABLE_BD_SCAN_BIT);
+			pfn_param.repeat = 0;
+			pfn_param.exp = 0;
+			if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+				/* In case of Legacy PNO + BATCH SCAN */
+				_params = &(_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS]);
+				if (_params->params_batch.bestn)
+					pfn_param.bestn = _params->params_batch.bestn;
+				if (_params->params_batch.scan_fr)
+					pfn_param.scan_freq = htod32(_params->params_batch.scan_fr);
+				if (_params->params_batch.mscan)
+					pfn_param.mscan = _params->params_batch.mscan;
+			} else if (_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE) {
+				/* In case of Legacy PNO + HOTLIST SCAN */
+				_params = &(_pno_state->pno_params_arr[INDEX_OF_HOTLIST_PARAMS]);
+				if (_params->params_hotlist.scan_fr)
+				pfn_param.scan_freq = htod32(_params->params_hotlist.scan_fr);
+				pfn_param.bestn = 0;
+				pfn_param.repeat = 0;
+			}
+		}
+	}
+#ifdef GSCAN_SUPPORT
+	if (mode & DHD_PNO_GSCAN_MODE) {
+		uint32 lost_network_timeout;
+
+		pfn_param.scan_freq = htod32(pno_params->params_gscan.scan_fr);
+		if (pno_params->params_gscan.mscan) {
+			pfn_param.bestn = pno_params->params_gscan.bestn;
+			pfn_param.mscan =  pno_params->params_gscan.mscan;
+			pfn_param.flags |= (ENABLE << ENABLE_BD_SCAN_BIT);
+		}
+		/* RSSI margin of 30 dBm */
+		pfn_param.rssi_margin = htod16(PNO_RSSI_MARGIN_DBM);
+		pfn_param.repeat = 0;
+		pfn_param.exp = 0;
+		pfn_param.slow_freq = 0;
+		pfn_param.flags |= htod16(ENABLE << ENABLE_ADAPTSCAN_BIT);
+
+		if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+			dhd_pno_params_t *params;
+
+			params = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]);
+
+			pfn_param.scan_freq = gcd(pno_params->params_gscan.scan_fr,
+			                 params->params_legacy.scan_fr);
+
+			if ((params->params_legacy.pno_repeat != 0) ||
+				(params->params_legacy.pno_freq_expo_max != 0)) {
+				pfn_param.repeat = (uchar) (params->params_legacy.pno_repeat);
+				pfn_param.exp = (uchar) (params->params_legacy.pno_freq_expo_max);
+			}
+		}
+
+		lost_network_timeout = (pno_params->params_gscan.max_ch_bucket_freq *
+		                        pfn_param.scan_freq *
+		                        pno_params->params_gscan.lost_ap_window);
+		if (lost_network_timeout) {
+			pfn_param.lost_network_timeout = htod32(MIN(lost_network_timeout,
+			                                 GSCAN_MIN_BSSID_TIMEOUT));
+		} else {
+			pfn_param.lost_network_timeout = htod32(GSCAN_MIN_BSSID_TIMEOUT);
+		}
+	} else
+#endif /* GSCAN_SUPPORT */
+	{
+		if (pfn_param.scan_freq < htod32(PNO_SCAN_MIN_FW_SEC) ||
+			pfn_param.scan_freq > htod32(PNO_SCAN_MAX_FW_SEC)) {
+			DHD_ERROR(("%s pno freq(%d sec) is not valid \n",
+				__FUNCTION__, PNO_SCAN_MIN_FW_SEC));
+			err = BCME_BADARG;
+			goto exit;
+		}
+	}
+#if !defined(WL_USE_RANDOMIZED_SCAN)
+	err = dhd_set_rand_mac_oui(dhd);
+	/* Ignore if chip doesnt support the feature */
+	if (err < 0 && err != BCME_UNSUPPORTED) {
+		DHD_ERROR(("%s : failed to set random mac for PNO scan, %d\n", __FUNCTION__, err));
+		goto exit;
+	}
+#endif /* !defined(WL_USE_RANDOMIZED_SCAN */
+#ifdef GSCAN_SUPPORT
+	if (mode == DHD_PNO_BATCH_MODE ||
+	((mode & DHD_PNO_GSCAN_MODE) && pno_params->params_gscan.mscan))
+#else
+	if (mode == DHD_PNO_BATCH_MODE)
+#endif /* GSCAN_SUPPORT */
+	{
+		int _tmp = pfn_param.bestn;
+		/* set bestn to calculate the max mscan which firmware supports */
+		err = dhd_iovar(dhd, 0, "pfnmem", (char *)&_tmp, sizeof(_tmp), NULL, 0, TRUE);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to set pfnmem\n", __FUNCTION__));
+			goto exit;
+		}
+		/* get max mscan which the firmware supports */
+		err = dhd_iovar(dhd, 0, "pfnmem", NULL, 0, (char *)&_tmp, sizeof(_tmp), FALSE);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to get pfnmem\n", __FUNCTION__));
+			goto exit;
+		}
+		pfn_param.mscan = MIN(pfn_param.mscan, _tmp);
+		DHD_PNO((" returned mscan : %d, set bestn : %d mscan %d\n", _tmp, pfn_param.bestn,
+		        pfn_param.mscan));
+	}
+	err = dhd_iovar(dhd, 0, "pfn_set", (char *)&pfn_param, size, NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_set %d\n", __FUNCTION__, err));
+		goto exit;
+	}
+	/* need to return mscan if this is for batch scan instead of err */
+	err = (mode == DHD_PNO_BATCH_MODE)? pfn_param.mscan : err;
+exit:
+	return err;
+}
+
+static int
+_dhd_pno_add_ssid(dhd_pub_t *dhd, struct list_head* ssid_list, int nssid)
+{
+	int err = BCME_OK;
+	int i = 0, mem_needed;
+	wl_pfn_t *pfn_elem_buf;
+	struct dhd_pno_ssid *iter, *next;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	if (!nssid) {
+		NULL_CHECK(ssid_list, "ssid list is NULL", err);
+		return BCME_ERROR;
+	}
+	mem_needed = (sizeof(wl_pfn_t) * nssid);
+	pfn_elem_buf = (wl_pfn_t *) MALLOCZ(dhd->osh, mem_needed);
+	if (!pfn_elem_buf) {
+		DHD_ERROR(("%s: Can't malloc %d bytes!\n", __FUNCTION__, mem_needed));
+		return BCME_NOMEM;
+	}
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry_safe(iter, next, ssid_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		pfn_elem_buf[i].infra = htod32(1);
+		pfn_elem_buf[i].auth = htod32(DOT11_OPEN_SYSTEM);
+		pfn_elem_buf[i].wpa_auth = htod32(iter->wpa_auth);
+		pfn_elem_buf[i].flags = htod32(iter->flags);
+		if (iter->hidden)
+			pfn_elem_buf[i].flags |= htod32(ENABLE << WL_PFN_HIDDEN_BIT);
+		/* If a single RSSI threshold is defined, use that */
+#ifdef PNO_MIN_RSSI_TRIGGER
+		pfn_elem_buf[i].flags |= ((PNO_MIN_RSSI_TRIGGER & 0xFF) << WL_PFN_RSSI_SHIFT);
+#else
+		pfn_elem_buf[i].flags |= ((iter->rssi_thresh & 0xFF) << WL_PFN_RSSI_SHIFT);
+#endif /* PNO_MIN_RSSI_TRIGGER */
+		memcpy((char *)pfn_elem_buf[i].ssid.SSID, iter->SSID,
+			iter->SSID_len);
+		pfn_elem_buf[i].ssid.SSID_len = iter->SSID_len;
+		DHD_PNO(("%s size = %d hidden = %d flags = %x rssi_thresh %d\n",
+			iter->SSID, iter->SSID_len, iter->hidden,
+			iter->flags, iter->rssi_thresh));
+		if (++i >= nssid) {
+			/* shouldn't happen */
+			break;
+		}
+	}
+
+	err = dhd_iovar(dhd, 0, "pfn_add", (char *)pfn_elem_buf, mem_needed, NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_add\n", __FUNCTION__));
+	}
+	MFREE(dhd->osh, pfn_elem_buf, mem_needed);
+	return err;
+}
+
+/* qsort compare function */
+static int
+_dhd_pno_cmpfunc(const void *a, const void *b)
+{
+	return (*(const uint16*)a - *(const uint16*)b);
+}
+
+static int
+_dhd_pno_chan_merge(uint16 *d_chan_list, int *nchan,
+	uint16 *chan_list1, int nchan1, uint16 *chan_list2, int nchan2)
+{
+	int err = BCME_OK;
+	int i = 0, j = 0, k = 0;
+	uint16 tmp;
+	NULL_CHECK(d_chan_list, "d_chan_list is NULL", err);
+	NULL_CHECK(nchan, "nchan is NULL", err);
+	NULL_CHECK(chan_list1, "chan_list1 is NULL", err);
+	NULL_CHECK(chan_list2, "chan_list2 is NULL", err);
+	/* chan_list1 and chan_list2 should be sorted at first */
+	while (i < nchan1 && j < nchan2) {
+		tmp = chan_list1[i] < chan_list2[j]?
+			chan_list1[i++] : chan_list2[j++];
+		for (; i < nchan1 && chan_list1[i] == tmp; i++);
+		for (; j < nchan2 && chan_list2[j] == tmp; j++);
+		d_chan_list[k++] = tmp;
+	}
+
+	while (i < nchan1) {
+		tmp = chan_list1[i++];
+		for (; i < nchan1 && chan_list1[i] == tmp; i++);
+		d_chan_list[k++] = tmp;
+	}
+
+	while (j < nchan2) {
+		tmp = chan_list2[j++];
+		for (; j < nchan2 && chan_list2[j] == tmp; j++);
+		d_chan_list[k++] = tmp;
+
+	}
+	*nchan = k;
+	return err;
+}
+
+static int
+_dhd_pno_get_channels(dhd_pub_t *dhd, uint16 *d_chan_list,
+	int *nchan, uint8 band, bool skip_dfs)
+{
+	int err = BCME_OK;
+	int i, j;
+	uint32 chan_buf[WL_NUMCHANNELS + 1];
+	wl_uint32_list_t *list;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	if (*nchan) {
+		NULL_CHECK(d_chan_list, "d_chan_list is NULL", err);
+	}
+	memset(&chan_buf, 0, sizeof(chan_buf));
+	list = (wl_uint32_list_t *) (void *)chan_buf;
+	list->count = htod32(WL_NUMCHANNELS);
+	err = dhd_wl_ioctl_cmd(dhd, WLC_GET_VALID_CHANNELS, chan_buf, sizeof(chan_buf), FALSE, 0);
+	if (err < 0) {
+		DHD_ERROR(("failed to get channel list (err: %d)\n", err));
+		return err;
+	}
+	for (i = 0, j = 0; i < dtoh32(list->count) && i < *nchan; i++) {
+		if (IS_2G_CHANNEL(dtoh32(list->element[i]))) {
+			if (!(band & WLC_BAND_2G) && !(band & WLC_BAND_AUTO)) {
+				/* Skip, if not 2g */
+				continue;
+			}
+			/* fall through to include the channel */
+		} else if (IS_5G_CHANNEL(dtoh32(list->element[i]))) {
+			bool dfs_channel = is_dfs(dhd, dtoh32(list->element[i]));
+			if ((skip_dfs && dfs_channel) ||
+				(!(band & WLC_BAND_5G) && !(band & WLC_BAND_AUTO) && !dfs_channel)) {
+				/* Skip the channel if:
+				* the DFS bit is NOT set & the channel is a dfs channel
+				* the band 5G is not set & the channel is a non DFS 5G channel
+				*/
+				continue;
+			}
+			/* fall through to include the channel */
+		} else {
+			/* Not in range. Bad channel */
+			DHD_ERROR(("Not in range. bad channel\n"));
+			*nchan = 0;
+			return BCME_BADCHAN;
+		}
+
+		/* Include the channel */
+		d_chan_list[j++] = (uint16) dtoh32(list->element[i]);
+	}
+	*nchan = j;
+	return err;
+}
+
+static int
+_dhd_pno_convert_format(dhd_pub_t *dhd, struct dhd_pno_batch_params *params_batch,
+	char *buf, int nbufsize)
+{
+	int err = BCME_OK;
+	int bytes_written = 0, nreadsize = 0;
+	int t_delta = 0;
+	int nleftsize = nbufsize;
+	uint8 cnt = 0;
+	char *bp = buf;
+	char eabuf[ETHER_ADDR_STR_LEN];
+#ifdef PNO_DEBUG
+	char *_base_bp;
+	char msg[150];
+#endif
+	dhd_pno_bestnet_entry_t *iter, *next;
+	dhd_pno_scan_results_t *siter, *snext;
+	dhd_pno_best_header_t *phead, *pprev;
+	NULL_CHECK(params_batch, "params_batch is NULL", err);
+	if (nbufsize > 0)
+		NULL_CHECK(buf, "buf is NULL", err);
+	/* initialize the buffer */
+	memset(buf, 0, nbufsize);
+	DHD_PNO(("%s enter \n", __FUNCTION__));
+	/* # of scans */
+	if (!params_batch->get_batch.batch_started) {
+		bp += nreadsize = snprintf(bp, nleftsize, "scancount=%d\n",
+			params_batch->get_batch.expired_tot_scan_cnt);
+		nleftsize -= nreadsize;
+		params_batch->get_batch.batch_started = TRUE;
+	}
+	DHD_PNO(("%s scancount %d\n", __FUNCTION__, params_batch->get_batch.expired_tot_scan_cnt));
+	/* preestimate scan count until which scan result this report is going to end */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry_safe(siter, snext,
+		&params_batch->get_batch.expired_scan_results_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		phead = siter->bestnetheader;
+		while (phead != NULL) {
+			/* if left_size is less than bestheader total size , stop this */
+			if (nleftsize <=
+				(phead->tot_size + phead->tot_cnt * ENTRY_OVERHEAD))
+				goto exit;
+			/* increase scan count */
+			cnt++;
+			/* # best of each scan */
+			DHD_PNO(("\n<loop : %d, apcount %d>\n", cnt - 1, phead->tot_cnt));
+			/* attribute of the scan */
+			if (phead->reason & PNO_STATUS_ABORT_MASK) {
+				bp += nreadsize = snprintf(bp, nleftsize, "trunc\n");
+				nleftsize -= nreadsize;
+			}
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(iter, next,
+				&phead->entry_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				t_delta = jiffies_to_msecs(jiffies - iter->recorded_time);
+#ifdef PNO_DEBUG
+				_base_bp = bp;
+				memset(msg, 0, sizeof(msg));
+#endif
+				/* BSSID info */
+				bp += nreadsize = snprintf(bp, nleftsize, "bssid=%s\n",
+				bcm_ether_ntoa((const struct ether_addr *)&iter->BSSID, eabuf));
+				nleftsize -= nreadsize;
+				/* SSID */
+				bp += nreadsize = snprintf(bp, nleftsize, "ssid=%s\n", iter->SSID);
+				nleftsize -= nreadsize;
+				/* channel */
+				bp += nreadsize = snprintf(bp, nleftsize, "freq=%d\n",
+				wl_channel_to_frequency(wf_chspec_ctlchan(iter->channel),
+					CHSPEC_BAND(iter->channel)));
+				nleftsize -= nreadsize;
+				/* RSSI */
+				bp += nreadsize = snprintf(bp, nleftsize, "level=%d\n", iter->RSSI);
+				nleftsize -= nreadsize;
+				/* add the time consumed in Driver to the timestamp of firmware */
+				iter->timestamp += t_delta;
+				bp += nreadsize = snprintf(bp, nleftsize,
+					"age=%d\n", iter->timestamp);
+				nleftsize -= nreadsize;
+				/* RTT0 */
+				bp += nreadsize = snprintf(bp, nleftsize, "dist=%d\n",
+				(iter->rtt0 == 0)? -1 : iter->rtt0);
+				nleftsize -= nreadsize;
+				/* RTT1 */
+				bp += nreadsize = snprintf(bp, nleftsize, "distSd=%d\n",
+				(iter->rtt0 == 0)? -1 : iter->rtt1);
+				nleftsize -= nreadsize;
+				bp += nreadsize = snprintf(bp, nleftsize, "%s", AP_END_MARKER);
+				nleftsize -= nreadsize;
+				list_del(&iter->list);
+				MFREE(dhd->osh, iter, BESTNET_ENTRY_SIZE);
+#ifdef PNO_DEBUG
+				memcpy(msg, _base_bp, bp - _base_bp);
+				DHD_PNO(("Entry : \n%s", msg));
+#endif
+			}
+			bp += nreadsize = snprintf(bp, nleftsize, "%s", SCAN_END_MARKER);
+			DHD_PNO(("%s", SCAN_END_MARKER));
+			nleftsize -= nreadsize;
+			pprev = phead;
+			/* reset the header */
+			siter->bestnetheader = phead = phead->next;
+			MFREE(dhd->osh, pprev, BEST_HEADER_SIZE);
+
+			siter->cnt_header--;
+		}
+		if (phead == NULL) {
+			/* we store all entry in this scan , so it is ok to delete */
+			list_del(&siter->list);
+			MFREE(dhd->osh, siter, SCAN_RESULTS_SIZE);
+		}
+	}
+exit:
+	if (cnt < params_batch->get_batch.expired_tot_scan_cnt) {
+		DHD_ERROR(("Buffer size is small to save all batch entry,"
+			" cnt : %d (remained_scan_cnt): %d\n",
+			cnt, params_batch->get_batch.expired_tot_scan_cnt - cnt));
+	}
+	params_batch->get_batch.expired_tot_scan_cnt -= cnt;
+	/* set FALSE only if the link list  is empty after returning the data */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	if (list_empty(&params_batch->get_batch.expired_scan_results_list)) {
+		GCC_DIAGNOSTIC_POP();
+		params_batch->get_batch.batch_started = FALSE;
+		bp += snprintf(bp, nleftsize, "%s", RESULTS_END_MARKER);
+		DHD_PNO(("%s", RESULTS_END_MARKER));
+		DHD_PNO(("%s : Getting the batching data is complete\n", __FUNCTION__));
+	}
+	/* return used memory in buffer */
+	bytes_written = (int32)(bp - buf);
+	return bytes_written;
+}
+
+static int
+_dhd_pno_clear_all_batch_results(dhd_pub_t *dhd, struct list_head *head, bool only_last)
+{
+	int err = BCME_OK;
+	int removed_scan_cnt = 0;
+	dhd_pno_scan_results_t *siter, *snext;
+	dhd_pno_best_header_t *phead, *pprev;
+	dhd_pno_bestnet_entry_t *iter, *next;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(head, "head is NULL", err);
+	NULL_CHECK(head->next, "head->next is NULL", err);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry_safe(siter, snext,
+		head, list) {
+		if (only_last) {
+			/* in case that we need to delete only last one */
+			if (!list_is_last(&siter->list, head)) {
+				/* skip if the one is not last */
+				continue;
+			}
+		}
+		/* delete all data belong if the one is last */
+		phead = siter->bestnetheader;
+		while (phead != NULL) {
+			removed_scan_cnt++;
+			list_for_each_entry_safe(iter, next,
+			&phead->entry_list, list) {
+				list_del(&iter->list);
+				MFREE(dhd->osh, iter, BESTNET_ENTRY_SIZE);
+			}
+			pprev = phead;
+			phead = phead->next;
+			MFREE(dhd->osh, pprev, BEST_HEADER_SIZE);
+		}
+		if (phead == NULL) {
+			/* it is ok to delete top node */
+			list_del(&siter->list);
+			MFREE(dhd->osh, siter, SCAN_RESULTS_SIZE);
+		}
+	}
+	GCC_DIAGNOSTIC_POP();
+	return removed_scan_cnt;
+}
+
+static int
+_dhd_pno_cfg(dhd_pub_t *dhd, uint16 *channel_list, int nchan)
+{
+	int err = BCME_OK;
+	int i = 0;
+	wl_pfn_cfg_t pfncfg_param;
+	bool use_chanspec = FALSE;
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dhd_linux_get_primary_netdev(dhd));
+
+	/* When enable 6G, force to use chanspec list */
+	if (FW_SUPPORTED((dhd), 6g) ||
+		(cfg && FW_MAJOR_VER_PNO_CHSPEC_BACK_PORTED(cfg->wlc_ver))) {
+		use_chanspec = TRUE;
+	}
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	if (nchan) {
+		if (nchan > WL_NUMCHANNELS) {
+			return BCME_RANGE;
+		}
+		DHD_PNO(("%s enter :  nchan : %d\n", __FUNCTION__, nchan));
+		(void)memset_s(&pfncfg_param, sizeof(wl_pfn_cfg_t), 0, sizeof(wl_pfn_cfg_t));
+		pfncfg_param.channel_num = htod32(0);
+
+		for (i = 0; i < nchan; i++) {
+			if ((dhd->wlc_ver_major >= DHD_PNO_CHSPEC_SUPPORT_VER) || use_chanspec) {
+				pfncfg_param.channel_list[i] = wf_chspec_ctlchspec(channel_list[i]);
+			} else {
+				pfncfg_param.channel_list[i] = channel_list[i];
+			}
+		}
+	}
+
+	/* Setup default values */
+	pfncfg_param.reporttype = htod32(WL_PFN_REPORT_ALLNET);
+	pfncfg_param.channel_num = htod32(nchan);
+	err = dhd_iovar(dhd, 0, "pfn_cfg", (char *)&pfncfg_param, sizeof(pfncfg_param), NULL, 0,
+			TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_cfg\n", __FUNCTION__));
+		goto exit;
+	}
+exit:
+	return err;
+}
+
+static int
+_dhd_pno_reinitialize_prof(dhd_pub_t *dhd, dhd_pno_params_t *params, dhd_pno_mode_t mode)
+{
+	int err = BCME_OK;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL\n", err);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	mutex_lock(&_pno_state->pno_mutex);
+	switch (mode) {
+	case DHD_PNO_LEGACY_MODE: {
+		struct dhd_pno_ssid *iter, *next;
+		if (params->params_legacy.nssid > 0) {
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(iter, next,
+				&params->params_legacy.ssid_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				list_del(&iter->list);
+				MFREE(dhd->osh, iter, sizeof(struct dhd_pno_ssid));
+			}
+		}
+
+		params->params_legacy.nssid = 0;
+		params->params_legacy.scan_fr = 0;
+		params->params_legacy.pno_freq_expo_max = 0;
+		params->params_legacy.pno_repeat = 0;
+		params->params_legacy.nchan = 0;
+		memset(params->params_legacy.chan_list, 0,
+			sizeof(params->params_legacy.chan_list));
+		break;
+	}
+	case DHD_PNO_BATCH_MODE: {
+		params->params_batch.scan_fr = 0;
+		params->params_batch.mscan = 0;
+		params->params_batch.nchan = 0;
+		params->params_batch.rtt = 0;
+		params->params_batch.bestn = 0;
+		params->params_batch.nchan = 0;
+		params->params_batch.band = WLC_BAND_AUTO;
+		memset(params->params_batch.chan_list, 0,
+			sizeof(params->params_batch.chan_list));
+		params->params_batch.get_batch.batch_started = FALSE;
+		params->params_batch.get_batch.buf = NULL;
+		params->params_batch.get_batch.bufsize = 0;
+		params->params_batch.get_batch.reason = 0;
+		_dhd_pno_clear_all_batch_results(dhd,
+			&params->params_batch.get_batch.scan_results_list, FALSE);
+		_dhd_pno_clear_all_batch_results(dhd,
+			&params->params_batch.get_batch.expired_scan_results_list, FALSE);
+		params->params_batch.get_batch.tot_scan_cnt = 0;
+		params->params_batch.get_batch.expired_tot_scan_cnt = 0;
+		params->params_batch.get_batch.top_node_cnt = 0;
+		INIT_LIST_HEAD(&params->params_batch.get_batch.scan_results_list);
+		INIT_LIST_HEAD(&params->params_batch.get_batch.expired_scan_results_list);
+		break;
+	}
+	case DHD_PNO_HOTLIST_MODE: {
+		struct dhd_pno_bssid *iter, *next;
+		if (params->params_hotlist.nbssid > 0) {
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(iter, next,
+				&params->params_hotlist.bssid_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				list_del(&iter->list);
+				MFREE(dhd->osh, iter, sizeof(struct dhd_pno_ssid));
+			}
+		}
+		params->params_hotlist.scan_fr = 0;
+		params->params_hotlist.nbssid = 0;
+		params->params_hotlist.nchan = 0;
+		params->params_batch.band = WLC_BAND_AUTO;
+		memset(params->params_hotlist.chan_list, 0,
+			sizeof(params->params_hotlist.chan_list));
+		break;
+	}
+	default:
+		DHD_ERROR(("%s : unknown mode : %d\n", __FUNCTION__, mode));
+		break;
+	}
+	mutex_unlock(&_pno_state->pno_mutex);
+	return err;
+}
+
+static int
+_dhd_pno_add_bssid(dhd_pub_t *dhd, wl_pfn_bssid_t *p_pfn_bssid, int nbssid)
+{
+	int err = BCME_OK;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	if (nbssid) {
+		NULL_CHECK(p_pfn_bssid, "bssid list is NULL", err);
+	}
+	err = dhd_iovar(dhd, 0, "pfn_add_bssid", (char *)p_pfn_bssid,
+			sizeof(wl_pfn_bssid_t) * nbssid, NULL, 0, TRUE);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to execute pfn_cfg\n", __FUNCTION__));
+		goto exit;
+	}
+exit:
+	return err;
+}
+
+int
+dhd_pno_stop_for_ssid(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	uint32 mode = 0, cnt = 0;
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params = NULL;
+	wl_pfn_bssid_t *p_pfn_bssid = NULL, *tmp_bssid;
+
+	NULL_CHECK(dhd, "dev is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	if (!(_pno_state->pno_mode & DHD_PNO_LEGACY_MODE)) {
+		DHD_ERROR(("%s : LEGACY PNO MODE is not enabled\n", __FUNCTION__));
+		goto exit;
+	}
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	/* If pno mode is PNO_LEGACY_MODE clear the pno values and unset the DHD_PNO_LEGACY_MODE */
+	_params = &_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS];
+	_dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_LEGACY_MODE);
+	_pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE;
+
+#ifdef GSCAN_SUPPORT
+	if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+		struct dhd_pno_gscan_params *gscan_params;
+
+		_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+		gscan_params = &_params->params_gscan;
+		if (gscan_params->mscan) {
+			/* retrieve the batching data from firmware into host */
+			err = dhd_wait_batch_results_complete(dhd);
+			if (err != BCME_OK)
+				goto exit;
+		}
+		/* save current pno_mode before calling dhd_pno_clean */
+		mutex_lock(&_pno_state->pno_mutex);
+		mode = _pno_state->pno_mode;
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			mutex_unlock(&_pno_state->pno_mutex);
+			goto exit;
+		}
+		/* restore previous pno_mode */
+		_pno_state->pno_mode = mode;
+		mutex_unlock(&_pno_state->pno_mutex);
+		/* Restart gscan */
+		err = dhd_pno_initiate_gscan_request(dhd, 1, 0);
+		goto exit;
+	}
+#endif /* GSCAN_SUPPORT */
+	/* restart Batch mode  if the batch mode is on */
+	if (_pno_state->pno_mode & (DHD_PNO_BATCH_MODE | DHD_PNO_HOTLIST_MODE)) {
+		/* retrieve the batching data from firmware into host */
+		dhd_pno_get_for_batch(dhd, NULL, 0, PNO_STATUS_DISABLE);
+		/* save current pno_mode before calling dhd_pno_clean */
+		mode = _pno_state->pno_mode;
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			err = BCME_ERROR;
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+
+		/* restore previous pno_mode */
+		_pno_state->pno_mode = mode;
+		if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+			_params = &(_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS]);
+			/* restart BATCH SCAN */
+			err = dhd_pno_set_for_batch(dhd, &_params->params_batch);
+			if (err < 0) {
+				_pno_state->pno_mode &= ~DHD_PNO_BATCH_MODE;
+				DHD_ERROR(("%s : failed to restart batch scan(err: %d)\n",
+					__FUNCTION__, err));
+				goto exit;
+			}
+		} else if (_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE) {
+			/* restart HOTLIST SCAN */
+			struct dhd_pno_bssid *iter, *next;
+			_params = &(_pno_state->pno_params_arr[INDEX_OF_HOTLIST_PARAMS]);
+			p_pfn_bssid = MALLOCZ(dhd->osh, sizeof(wl_pfn_bssid_t) *
+			_params->params_hotlist.nbssid);
+			if (p_pfn_bssid == NULL) {
+				DHD_ERROR(("%s : failed to allocate wl_pfn_bssid_t array"
+				" (count: %d)",
+					__FUNCTION__, _params->params_hotlist.nbssid));
+				err = BCME_ERROR;
+				_pno_state->pno_mode &= ~DHD_PNO_HOTLIST_MODE;
+				goto exit;
+			}
+			/* convert dhd_pno_bssid to wl_pfn_bssid */
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			cnt = 0;
+			tmp_bssid = p_pfn_bssid;
+			list_for_each_entry_safe(iter, next,
+			&_params->params_hotlist.bssid_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				memcpy(&tmp_bssid->macaddr,
+				&iter->macaddr, ETHER_ADDR_LEN);
+				tmp_bssid->flags = iter->flags;
+				if (cnt < _params->params_hotlist.nbssid) {
+					tmp_bssid++;
+					cnt++;
+				} else {
+					DHD_ERROR(("%s: Allocated insufficient memory\n",
+						__FUNCTION__));
+					break;
+				}
+			}
+			err = dhd_pno_set_for_hotlist(dhd, p_pfn_bssid, &_params->params_hotlist);
+			if (err < 0) {
+				_pno_state->pno_mode &= ~DHD_PNO_HOTLIST_MODE;
+				DHD_ERROR(("%s : failed to restart hotlist scan(err: %d)\n",
+					__FUNCTION__, err));
+				goto exit;
+			}
+		}
+	} else {
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+	}
+exit:
+	if (p_pfn_bssid) {
+		MFREE(dhd->osh, p_pfn_bssid, sizeof(wl_pfn_bssid_t) *
+			_params->params_hotlist.nbssid);
+	}
+	return err;
+}
+
+int
+dhd_pno_enable(dhd_pub_t *dhd, int enable)
+{
+	int err = BCME_OK;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	return (_dhd_pno_enable(dhd, enable));
+}
+
+static int
+dhd_pno_add_to_ssid_list(dhd_pub_t *dhd, struct list_head *ptr, wlc_ssid_ext_t *ssid_list,
+    int nssid, int *num_ssid_added)
+{
+	int ret = BCME_OK;
+	int i;
+	struct dhd_pno_ssid *_pno_ssid;
+
+	for (i = 0; i < nssid; i++) {
+		if (ssid_list[i].SSID_len > DOT11_MAX_SSID_LEN) {
+			DHD_ERROR(("%s : Invalid SSID length %d\n",
+				__FUNCTION__, ssid_list[i].SSID_len));
+			ret = BCME_ERROR;
+			goto exit;
+		}
+		/* Check for broadcast ssid */
+		if (!ssid_list[i].SSID_len) {
+			DHD_ERROR(("%d: Broadcast SSID is illegal for PNO setting\n", i));
+			ret = BCME_ERROR;
+			goto exit;
+		}
+		_pno_ssid = (struct dhd_pno_ssid *)MALLOCZ(dhd->osh,
+			sizeof(struct dhd_pno_ssid));
+		if (_pno_ssid == NULL) {
+			DHD_ERROR(("%s : failed to allocate struct dhd_pno_ssid\n",
+				__FUNCTION__));
+			ret = BCME_ERROR;
+			goto exit;
+		}
+		_pno_ssid->SSID_len = ssid_list[i].SSID_len;
+		_pno_ssid->hidden = ssid_list[i].hidden;
+		_pno_ssid->rssi_thresh = ssid_list[i].rssi_thresh;
+		_pno_ssid->flags = ssid_list[i].flags;
+		_pno_ssid->wpa_auth = WPA_AUTH_PFN_ANY;
+
+		memcpy(_pno_ssid->SSID, ssid_list[i].SSID, _pno_ssid->SSID_len);
+		list_add_tail(&_pno_ssid->list, ptr);
+	}
+
+exit:
+	*num_ssid_added = i;
+	return ret;
+}
+
+int
+dhd_pno_set_for_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssid_list, int nssid,
+	uint16  scan_fr, int pno_repeat, int pno_freq_expo_max, uint16 *channel_list, int nchan)
+{
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	struct dhd_pno_legacy_params *params_legacy;
+	int err = BCME_OK;
+
+	if (!dhd || !dhd->pno_state) {
+		DHD_ERROR(("%s: PNO Not enabled/Not ready\n", __FUNCTION__));
+		return BCME_NOTREADY;
+	}
+
+	if (!dhd_support_sta_mode(dhd)) {
+		return BCME_BADOPTION;
+	}
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_params = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]);
+	params_legacy = &(_params->params_legacy);
+	err = _dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_LEGACY_MODE);
+
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to reinitialize profile (err %d)\n",
+			__FUNCTION__, err));
+		return err;
+	}
+
+	INIT_LIST_HEAD(&params_legacy->ssid_list);
+
+	if (dhd_pno_add_to_ssid_list(dhd, &params_legacy->ssid_list, ssid_list,
+		nssid, &params_legacy->nssid) < 0) {
+		_dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_LEGACY_MODE);
+		return BCME_ERROR;
+	}
+
+	DHD_PNO(("%s enter : nssid %d, scan_fr :%d, pno_repeat :%d,"
+		"pno_freq_expo_max: %d, nchan :%d\n", __FUNCTION__,
+		params_legacy->nssid, scan_fr, pno_repeat, pno_freq_expo_max, nchan));
+
+	return dhd_pno_set_legacy_pno(dhd, scan_fr, pno_repeat,
+		pno_freq_expo_max, channel_list, nchan);
+
+}
+
+static int
+dhd_pno_set_legacy_pno(dhd_pub_t *dhd, uint16  scan_fr, int pno_repeat,
+	int pno_freq_expo_max, uint16 *channel_list, int nchan)
+{
+	dhd_pno_params_t *_params;
+	dhd_pno_params_t *_params2;
+	dhd_pno_status_info_t *_pno_state;
+	uint16 _chan_list[WL_NUMCHANNELS];
+	int32 tot_nchan = 0;
+	int err = BCME_OK;
+	int i, nssid;
+	int mode = 0;
+	struct list_head *ssid_list;
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	_params = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]);
+	/* If GSCAN is also ON will handle this down below */
+#ifdef GSCAN_SUPPORT
+	if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE &&
+		!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE))
+#else
+	if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE)
+#endif /* GSCAN_SUPPORT */
+	{
+		DHD_ERROR(("%s : Legacy PNO mode was already started, "
+			"will disable previous one to start new one\n", __FUNCTION__));
+		err = dhd_pno_stop_for_ssid(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to stop legacy PNO (err %d)\n",
+				__FUNCTION__, err));
+			return err;
+		}
+	}
+	_pno_state->pno_mode |= DHD_PNO_LEGACY_MODE;
+	(void)memset_s(_chan_list, sizeof(_chan_list),
+		0, sizeof(_chan_list));
+	tot_nchan = MIN(nchan, WL_NUMCHANNELS);
+	if (tot_nchan > 0 && channel_list) {
+		for (i = 0; i < tot_nchan; i++)
+		_params->params_legacy.chan_list[i] = _chan_list[i] = channel_list[i];
+	}
+#ifdef GSCAN_SUPPORT
+	else {
+		/* FW scan module will include all valid channels when chan count
+		 * is set to 0
+		 */
+		tot_nchan = 0;
+	}
+#endif /* GSCAN_SUPPORT */
+
+	if (_pno_state->pno_mode & (DHD_PNO_BATCH_MODE | DHD_PNO_HOTLIST_MODE)) {
+		DHD_PNO(("BATCH SCAN is on progress in firmware\n"));
+		/* retrieve the batching data from firmware into host */
+		dhd_pno_get_for_batch(dhd, NULL, 0, PNO_STATUS_DISABLE);
+		/* store current pno_mode before disabling pno */
+		mode = _pno_state->pno_mode;
+		err = _dhd_pno_enable(dhd, PNO_OFF);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to disable PNO\n", __FUNCTION__));
+			goto exit;
+		}
+		/* restore the previous mode */
+		_pno_state->pno_mode = mode;
+		/* use superset of channel list between two mode */
+		if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+			_params2 = &(_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS]);
+			if (_params2->params_batch.nchan > 0 && tot_nchan > 0) {
+				err = _dhd_pno_chan_merge(_chan_list, &tot_nchan,
+					&_params2->params_batch.chan_list[0],
+					_params2->params_batch.nchan,
+					&channel_list[0], tot_nchan);
+				if (err < 0) {
+					DHD_ERROR(("%s : failed to merge channel list"
+					" between legacy and batch\n",
+						__FUNCTION__));
+					goto exit;
+				}
+			}  else {
+				DHD_PNO(("superset channel will use"
+				" all channels in firmware\n"));
+			}
+		} else if (_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE) {
+			_params2 = &(_pno_state->pno_params_arr[INDEX_OF_HOTLIST_PARAMS]);
+			if (_params2->params_hotlist.nchan > 0 && tot_nchan > 0) {
+				err = _dhd_pno_chan_merge(_chan_list, &tot_nchan,
+					&_params2->params_hotlist.chan_list[0],
+					_params2->params_hotlist.nchan,
+					&channel_list[0], tot_nchan);
+				if (err < 0) {
+					DHD_ERROR(("%s : failed to merge channel list"
+					" between legacy and hotlist\n",
+						__FUNCTION__));
+					goto exit;
+				}
+			}
+		}
+	}
+	_params->params_legacy.scan_fr = scan_fr;
+	_params->params_legacy.pno_repeat = pno_repeat;
+	_params->params_legacy.pno_freq_expo_max = pno_freq_expo_max;
+	_params->params_legacy.nchan = tot_nchan;
+	ssid_list = &_params->params_legacy.ssid_list;
+	nssid = _params->params_legacy.nssid;
+
+#ifdef GSCAN_SUPPORT
+	/* dhd_pno_initiate_gscan_request will handle simultaneous Legacy PNO and GSCAN */
+	if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+		struct dhd_pno_gscan_params *gscan_params;
+		gscan_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan;
+		/* ePNO and Legacy PNO do not co-exist */
+		if (gscan_params->epno_cfg.num_epno_ssid) {
+			DHD_PNO(("ePNO and Legacy PNO do not co-exist\n"));
+			err = BCME_EPERM;
+			goto exit;
+		}
+		DHD_PNO(("GSCAN mode is ON! Will restart GSCAN+Legacy PNO\n"));
+		err = dhd_pno_initiate_gscan_request(dhd, 1, 0);
+		goto exit;
+	}
+#endif /* GSCAN_SUPPORT */
+	if ((err = _dhd_pno_set(dhd, _params, DHD_PNO_LEGACY_MODE)) < 0) {
+		DHD_ERROR(("failed to set call pno_set (err %d) in firmware\n", err));
+		goto exit;
+	}
+	if ((err = _dhd_pno_add_ssid(dhd, ssid_list, nssid)) < 0) {
+		DHD_ERROR(("failed to add ssid list(err %d), %d in firmware\n", err, nssid));
+		goto exit;
+	}
+
+	if ((err = _dhd_pno_cfg(dhd, _chan_list, tot_nchan)) < 0) {
+		DHD_ERROR(("%s : failed to set call pno_cfg (err %d) in firmware\n",
+			__FUNCTION__, err));
+		goto exit;
+	}
+
+	if (_pno_state->pno_status == DHD_PNO_DISABLED) {
+		if ((err = _dhd_pno_enable(dhd, PNO_ON)) < 0)
+			DHD_ERROR(("%s : failed to enable PNO\n", __FUNCTION__));
+	}
+exit:
+	if (err < 0) {
+		_dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_LEGACY_MODE);
+	}
+	/* clear mode in case of error */
+	if (err < 0) {
+		int ret = dhd_pno_clean(dhd);
+
+		if (ret < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, ret));
+		} else {
+			_pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE;
+		}
+	}
+	return err;
+}
+
+int
+dhd_pno_set_for_batch(dhd_pub_t *dhd, struct dhd_pno_batch_params *batch_params)
+{
+	int err = BCME_OK;
+	uint16 _chan_list[WL_NUMCHANNELS];
+	int rem_nchan = 0, tot_nchan = 0;
+	int mode = 0, mscan = 0;
+	dhd_pno_params_t *_params;
+	dhd_pno_params_t *_params2;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	NULL_CHECK(batch_params, "batch_params is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+	_params = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS];
+	if (!(_pno_state->pno_mode & DHD_PNO_BATCH_MODE)) {
+		_pno_state->pno_mode |= DHD_PNO_BATCH_MODE;
+		err = _dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_BATCH_MODE);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call _dhd_pno_reinitialize_prof\n",
+				__FUNCTION__));
+			goto exit;
+		}
+	} else {
+		/* batch mode is already started */
+		return -EBUSY;
+	}
+	_params->params_batch.scan_fr = batch_params->scan_fr;
+	_params->params_batch.bestn = batch_params->bestn;
+	_params->params_batch.mscan = (batch_params->mscan)?
+		batch_params->mscan : DEFAULT_BATCH_MSCAN;
+	_params->params_batch.nchan = batch_params->nchan;
+	memcpy(_params->params_batch.chan_list, batch_params->chan_list,
+		sizeof(_params->params_batch.chan_list));
+
+	memset(_chan_list, 0, sizeof(_chan_list));
+
+	rem_nchan = ARRAYSIZE(batch_params->chan_list) - batch_params->nchan;
+	if (batch_params->band == WLC_BAND_2G ||
+#ifdef WL_6G_BAND
+		batch_params->band == WLC_BAND_6G ||
+#endif /* WL_6G_BAND */
+		batch_params->band == WLC_BAND_5G) {
+		/* get a valid channel list based on band B or A */
+		err = _dhd_pno_get_channels(dhd,
+		&_params->params_batch.chan_list[batch_params->nchan],
+		&rem_nchan, batch_params->band, FALSE);
+		if (err < 0) {
+			DHD_ERROR(("%s: failed to get valid channel list(band : %d)\n",
+				__FUNCTION__, batch_params->band));
+			goto exit;
+		}
+		/* now we need to update nchan because rem_chan has valid channel count */
+		_params->params_batch.nchan += rem_nchan;
+		/* need to sort channel list */
+		sort(_params->params_batch.chan_list, _params->params_batch.nchan,
+			sizeof(_params->params_batch.chan_list[0]), _dhd_pno_cmpfunc, NULL);
+	}
+#ifdef PNO_DEBUG
+	{
+		DHD_PNO(("Channel list : "));
+		for (i = 0; i < _params->params_batch.nchan; i++) {
+			DHD_PNO(("%d ", _params->params_batch.chan_list[i]));
+		}
+		DHD_PNO(("\n"));
+	}
+#endif
+	if (_params->params_batch.nchan) {
+		/* copy the channel list into local array */
+		memcpy(_chan_list, _params->params_batch.chan_list, sizeof(_chan_list));
+		tot_nchan = _params->params_batch.nchan;
+	}
+	if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+		DHD_PNO(("PNO SSID is on progress in firmware\n"));
+		/* store current pno_mode before disabling pno */
+		mode = _pno_state->pno_mode;
+		err = _dhd_pno_enable(dhd, PNO_OFF);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to disable PNO\n", __FUNCTION__));
+			goto exit;
+		}
+		/* restore the previous mode */
+		_pno_state->pno_mode = mode;
+		/* Use the superset for channelist between two mode */
+		_params2 = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]);
+		if (_params2->params_legacy.nchan > 0 && _params->params_batch.nchan > 0) {
+			err = _dhd_pno_chan_merge(_chan_list, &tot_nchan,
+				&_params2->params_legacy.chan_list[0],
+				_params2->params_legacy.nchan,
+				&_params->params_batch.chan_list[0], _params->params_batch.nchan);
+			if (err < 0) {
+				DHD_ERROR(("%s : failed to merge channel list"
+				" between legacy and batch\n",
+					__FUNCTION__));
+				goto exit;
+			}
+		} else {
+			DHD_PNO(("superset channel will use all channels in firmware\n"));
+		}
+		if ((err = _dhd_pno_add_ssid(dhd, &_params2->params_legacy.ssid_list,
+				_params2->params_legacy.nssid)) < 0) {
+			DHD_ERROR(("failed to add ssid list (err %d) in firmware\n", err));
+			goto exit;
+		}
+	}
+	if ((err = _dhd_pno_set(dhd, _params, DHD_PNO_BATCH_MODE)) < 0) {
+		DHD_ERROR(("%s : failed to set call pno_set (err %d) in firmware\n",
+			__FUNCTION__, err));
+		goto exit;
+	} else {
+		/* we need to return mscan */
+		mscan = err;
+	}
+	if (tot_nchan > 0) {
+		if ((err = _dhd_pno_cfg(dhd, _chan_list, tot_nchan)) < 0) {
+			DHD_ERROR(("%s : failed to set call pno_cfg (err %d) in firmware\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+	}
+	if (_pno_state->pno_status == DHD_PNO_DISABLED) {
+		if ((err = _dhd_pno_enable(dhd, PNO_ON)) < 0)
+			DHD_ERROR(("%s : failed to enable PNO\n", __FUNCTION__));
+	}
+exit:
+	/* clear mode in case of error */
+	if (err < 0)
+		_pno_state->pno_mode &= ~DHD_PNO_BATCH_MODE;
+	else {
+		/* return #max scan firmware can do */
+		err = mscan;
+	}
+	return err;
+}
+
+#ifdef GSCAN_SUPPORT
+
+static int
+dhd_set_epno_params(dhd_pub_t *dhd, wl_ssid_ext_params_t *params, bool set)
+{
+	wl_pfn_ssid_cfg_t cfg;
+	int err;
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+	memset(&cfg, 0, sizeof(wl_pfn_ssid_cfg_t));
+	cfg.version = WL_PFN_SSID_CFG_VERSION;
+
+	/* If asked to clear params (set == FALSE) just set the CLEAR bit */
+	if (!set)
+		cfg.flags |= WL_PFN_SSID_CFG_CLEAR;
+	else if (params)
+		memcpy(&cfg.params, params, sizeof(wl_ssid_ext_params_t));
+	err = dhd_iovar(dhd, 0, "pfn_ssid_cfg", (char *)&cfg,
+			sizeof(wl_pfn_ssid_cfg_t), NULL, 0, TRUE);
+	if (err != BCME_OK) {
+		DHD_ERROR(("%s : Failed to execute pfn_ssid_cfg %d\n", __FUNCTION__, err));
+	}
+	return err;
+}
+
+int
+dhd_pno_flush_fw_epno(dhd_pub_t *dhd)
+{
+	int err;
+
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+
+	err = dhd_set_epno_params(dhd, NULL, FALSE);
+	if (err < 0) {
+		DHD_ERROR(("failed to set ePNO params %d\n", err));
+		return err;
+	}
+	err = _dhd_pno_flush_ssid(dhd);
+	return err;
+}
+
+int
+dhd_pno_set_epno(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	dhd_pno_params_t *params;
+	dhd_pno_status_info_t *_pno_state;
+
+	struct dhd_pno_gscan_params *gscan_params;
+
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	gscan_params = &params->params_gscan;
+
+	if (gscan_params->epno_cfg.num_epno_ssid) {
+		DHD_PNO(("num_epno_ssid %d\n", gscan_params->epno_cfg.num_epno_ssid));
+		if ((err = _dhd_pno_add_ssid(dhd, &gscan_params->epno_cfg.epno_ssid_list,
+				gscan_params->epno_cfg.num_epno_ssid)) < 0) {
+			DHD_ERROR(("failed to add ssid list (err %d) to firmware\n", err));
+			return err;
+		}
+		err = dhd_set_epno_params(dhd, &gscan_params->epno_cfg.params, TRUE);
+		if (err < 0) {
+			DHD_ERROR(("failed to set ePNO params %d\n", err));
+		}
+	}
+	return err;
+}
+
+static void
+dhd_pno_reset_cfg_gscan(dhd_pub_t *dhd, dhd_pno_params_t *_params,
+            dhd_pno_status_info_t *_pno_state, uint8 flags)
+{
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (flags & GSCAN_FLUSH_SCAN_CFG) {
+		_params->params_gscan.bestn = 0;
+		_params->params_gscan.mscan = 0;
+		_params->params_gscan.buffer_threshold = GSCAN_BATCH_NO_THR_SET;
+		_params->params_gscan.scan_fr = 0;
+		_params->params_gscan.send_all_results_flag = 0;
+		memset(_params->params_gscan.channel_bucket, 0,
+		_params->params_gscan.nchannel_buckets *
+		 sizeof(struct dhd_pno_gscan_channel_bucket));
+		_params->params_gscan.nchannel_buckets = 0;
+		DHD_PNO(("Flush Scan config\n"));
+	}
+	if (flags & GSCAN_FLUSH_HOTLIST_CFG) {
+		struct dhd_pno_bssid *iter, *next;
+		if (_params->params_gscan.nbssid_hotlist > 0) {
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(iter, next,
+				&_params->params_gscan.hotlist_bssid_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				list_del(&iter->list);
+				MFREE(dhd->osh, iter, sizeof(struct dhd_pno_bssid));
+			}
+		}
+		_params->params_gscan.nbssid_hotlist = 0;
+		DHD_PNO(("Flush Hotlist Config\n"));
+	}
+	if (flags & GSCAN_FLUSH_EPNO_CFG) {
+		dhd_pno_ssid_t *iter, *next;
+		dhd_epno_ssid_cfg_t *epno_cfg = &_params->params_gscan.epno_cfg;
+
+		if (epno_cfg->num_epno_ssid > 0) {
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(iter, next,
+				&epno_cfg->epno_ssid_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				list_del(&iter->list);
+				MFREE(dhd->osh, iter, sizeof(struct dhd_pno_bssid));
+			}
+			epno_cfg->num_epno_ssid = 0;
+		}
+		memset(&epno_cfg->params, 0, sizeof(wl_ssid_ext_params_t));
+		DHD_PNO(("Flushed ePNO Config\n"));
+	}
+
+	return;
+}
+
+int
+dhd_pno_lock_batch_results(dhd_pub_t *dhd)
+{
+	dhd_pno_status_info_t *_pno_state;
+	int err = BCME_OK;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	mutex_lock(&_pno_state->pno_mutex);
+	return err;
+}
+
+void
+dhd_pno_unlock_batch_results(dhd_pub_t *dhd)
+{
+	dhd_pno_status_info_t *_pno_state;
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	mutex_unlock(&_pno_state->pno_mutex);
+	return;
+}
+
+int
+dhd_wait_batch_results_complete(dhd_pub_t *dhd)
+{
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	int err = BCME_OK;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+
+	/* Has the workqueue finished its job already?? */
+	if (_params->params_gscan.get_batch_flag == GSCAN_BATCH_RETRIEVAL_IN_PROGRESS) {
+		DHD_PNO(("%s: Waiting to complete retrieval..\n", __FUNCTION__));
+		wait_event_interruptible_timeout(_pno_state->batch_get_wait,
+		     is_batch_retrieval_complete(&_params->params_gscan),
+		     msecs_to_jiffies(GSCAN_BATCH_GET_MAX_WAIT));
+	} else { /* GSCAN_BATCH_RETRIEVAL_COMPLETE */
+		gscan_results_cache_t *iter;
+		uint16 num_results = 0;
+
+		mutex_lock(&_pno_state->pno_mutex);
+		iter = _params->params_gscan.gscan_batch_cache;
+		while (iter) {
+			num_results += iter->tot_count - iter->tot_consumed;
+			iter = iter->next;
+		}
+		mutex_unlock(&_pno_state->pno_mutex);
+
+		/* All results consumed/No results cached??
+		 * Get fresh results from FW
+		 */
+		if ((_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) && !num_results) {
+			DHD_PNO(("%s: No results cached, getting from FW..\n", __FUNCTION__));
+			err = dhd_retreive_batch_scan_results(dhd);
+			if (err == BCME_OK) {
+				wait_event_interruptible_timeout(_pno_state->batch_get_wait,
+				  is_batch_retrieval_complete(&_params->params_gscan),
+				  msecs_to_jiffies(GSCAN_BATCH_GET_MAX_WAIT));
+			}
+		}
+	}
+	DHD_PNO(("%s: Wait complete\n", __FUNCTION__));
+	return err;
+}
+
+int
+dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type,
+    void *buf, bool flush)
+{
+	int err = BCME_OK;
+	dhd_pno_params_t *_params;
+	int i;
+	dhd_pno_status_info_t *_pno_state;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	mutex_lock(&_pno_state->pno_mutex);
+
+	switch (type) {
+	case DHD_PNO_BATCH_SCAN_CFG_ID:
+		{
+			gscan_batch_params_t *ptr = (gscan_batch_params_t *)buf;
+			_params->params_gscan.bestn = ptr->bestn;
+			_params->params_gscan.mscan = ptr->mscan;
+			_params->params_gscan.buffer_threshold = ptr->buffer_threshold;
+		}
+		break;
+		case DHD_PNO_GEOFENCE_SCAN_CFG_ID:
+		{
+			gscan_hotlist_scan_params_t *ptr = (gscan_hotlist_scan_params_t *)buf;
+			struct dhd_pno_bssid *_pno_bssid;
+			struct bssid_t *bssid_ptr;
+			int8 flags;
+
+			if (flush) {
+				dhd_pno_reset_cfg_gscan(dhd, _params, _pno_state,
+				    GSCAN_FLUSH_HOTLIST_CFG);
+			}
+
+			if (!ptr->nbssid) {
+				break;
+			}
+			if (!_params->params_gscan.nbssid_hotlist) {
+				INIT_LIST_HEAD(&_params->params_gscan.hotlist_bssid_list);
+			}
+
+			if ((_params->params_gscan.nbssid_hotlist +
+					ptr->nbssid) > PFN_SWC_MAX_NUM_APS) {
+				DHD_ERROR(("Excessive number of hotlist APs programmed %d\n",
+					(_params->params_gscan.nbssid_hotlist +
+					ptr->nbssid)));
+				err = BCME_RANGE;
+				goto exit;
+			}
+
+			for (i = 0, bssid_ptr = ptr->bssid; i < ptr->nbssid; i++, bssid_ptr++) {
+				_pno_bssid = (struct dhd_pno_bssid *)MALLOCZ(dhd->osh,
+					sizeof(struct dhd_pno_bssid));
+				if (!_pno_bssid) {
+					DHD_ERROR(("_pno_bssid is NULL, cannot kalloc %zd bytes",
+					       sizeof(struct dhd_pno_bssid)));
+					err = BCME_NOMEM;
+					goto exit;
+				}
+				memcpy(&_pno_bssid->macaddr, &bssid_ptr->macaddr, ETHER_ADDR_LEN);
+
+				flags = (int8) bssid_ptr->rssi_reporting_threshold;
+				_pno_bssid->flags = flags  << WL_PFN_RSSI_SHIFT;
+				list_add_tail(&_pno_bssid->list,
+				   &_params->params_gscan.hotlist_bssid_list);
+			}
+
+			_params->params_gscan.nbssid_hotlist += ptr->nbssid;
+			_params->params_gscan.lost_ap_window = ptr->lost_ap_window;
+		}
+		break;
+	case DHD_PNO_SCAN_CFG_ID:
+		{
+			int k;
+			uint16 band;
+			gscan_scan_params_t *ptr = (gscan_scan_params_t *)buf;
+			struct dhd_pno_gscan_channel_bucket *ch_bucket;
+
+			if (ptr->nchannel_buckets <= GSCAN_MAX_CH_BUCKETS) {
+				_params->params_gscan.nchannel_buckets = ptr->nchannel_buckets;
+
+				memcpy(_params->params_gscan.channel_bucket, ptr->channel_bucket,
+				    _params->params_gscan.nchannel_buckets *
+				    sizeof(struct dhd_pno_gscan_channel_bucket));
+				ch_bucket = _params->params_gscan.channel_bucket;
+
+				for (i = 0; i < ptr->nchannel_buckets; i++) {
+					band = ch_bucket[i].band;
+					for (k = 0; k < ptr->channel_bucket[i].num_channels; k++)  {
+						ch_bucket[i].chan_list[k] =
+						wf_mhz2channel(ptr->channel_bucket[i].chan_list[k],
+							0);
+					}
+					ch_bucket[i].band = 0;
+					/* HAL and DHD use different bits for 2.4G and
+					 * 5G in bitmap. Hence translating it here...
+					 */
+					if (band & GSCAN_BG_BAND_MASK) {
+						ch_bucket[i].band |= WLC_BAND_2G;
+					}
+					if (band & GSCAN_A_BAND_MASK) {
+						ch_bucket[i].band |= WLC_BAND_6G | WLC_BAND_5G;
+					}
+					if (band & GSCAN_DFS_MASK) {
+						ch_bucket[i].band |= GSCAN_DFS_MASK;
+					}
+					DHD_PNO(("band %d report_flag %d\n", ch_bucket[i].band,
+					          ch_bucket[i].report_flag));
+				}
+
+				for (i = 0; i < ptr->nchannel_buckets; i++) {
+					ch_bucket[i].bucket_freq_multiple =
+					ch_bucket[i].bucket_freq_multiple/ptr->scan_fr;
+					ch_bucket[i].bucket_max_multiple =
+					ch_bucket[i].bucket_max_multiple/ptr->scan_fr;
+					DHD_PNO(("mult %d max_mult %d\n",
+					                 ch_bucket[i].bucket_freq_multiple,
+					                 ch_bucket[i].bucket_max_multiple));
+				}
+				_params->params_gscan.scan_fr = ptr->scan_fr;
+
+				DHD_PNO(("num_buckets %d scan_fr %d\n", ptr->nchannel_buckets,
+				        _params->params_gscan.scan_fr));
+			} else {
+				err = BCME_BADARG;
+			}
+		}
+		break;
+	case DHD_PNO_EPNO_CFG_ID:
+		if (flush) {
+			dhd_pno_reset_cfg_gscan(dhd, _params, _pno_state,
+				GSCAN_FLUSH_EPNO_CFG);
+		}
+		break;
+	case DHD_PNO_EPNO_PARAMS_ID:
+		if (flush) {
+			memset(&_params->params_gscan.epno_cfg.params, 0,
+				sizeof(wl_ssid_ext_params_t));
+		}
+		if (buf) {
+			memcpy(&_params->params_gscan.epno_cfg.params, buf,
+				sizeof(wl_ssid_ext_params_t));
+		}
+		break;
+	default:
+		err = BCME_BADARG;
+		DHD_ERROR(("%s: Unrecognized cmd type - %d\n", __FUNCTION__, type));
+		break;
+	}
+exit:
+	mutex_unlock(&_pno_state->pno_mutex);
+	return err;
+
+}
+
+static bool
+validate_gscan_params(struct dhd_pno_gscan_params *gscan_params)
+{
+	unsigned int i, k;
+
+	if (!gscan_params->scan_fr || !gscan_params->nchannel_buckets) {
+		DHD_ERROR(("%s : Scan freq - %d or number of channel buckets - %d is empty\n",
+		 __FUNCTION__, gscan_params->scan_fr, gscan_params->nchannel_buckets));
+		return false;
+	}
+
+	for (i = 0; i < gscan_params->nchannel_buckets; i++) {
+		if (!gscan_params->channel_bucket[i].band) {
+			for (k = 0; k < gscan_params->channel_bucket[i].num_channels; k++) {
+				if (gscan_params->channel_bucket[i].chan_list[k] > CHANNEL_5G_MAX) {
+					DHD_ERROR(("%s : Unknown channel %d\n", __FUNCTION__,
+					 gscan_params->channel_bucket[i].chan_list[k]));
+					return false;
+				}
+			}
+		}
+	}
+
+	return true;
+}
+
+static int
+dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params)
+{
+	int err = BCME_OK;
+	int mode, i = 0;
+	uint16 _chan_list[WL_NUMCHANNELS];
+	int tot_nchan = 0;
+	int num_buckets_to_fw, tot_num_buckets, gscan_param_size;
+	dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd);
+	wl_pfn_gscan_ch_bucket_cfg_t *ch_bucket = NULL;
+	wl_pfn_gscan_cfg_t *pfn_gscan_cfg_t = NULL;
+	wl_pfn_bssid_t *p_pfn_bssid = NULL;
+	dhd_pno_params_t	*_params;
+	bool fw_flushed = FALSE;
+
+	_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	NULL_CHECK(gscan_params, "gscan_params is NULL", err);
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	if (!validate_gscan_params(gscan_params)) {
+		DHD_ERROR(("%s : Cannot start gscan - bad params\n", __FUNCTION__));
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	if (!(ch_bucket = dhd_pno_gscan_create_channel_list(dhd, _pno_state,
+	    _chan_list, &tot_num_buckets, &num_buckets_to_fw))) {
+		goto exit;
+	}
+
+	mutex_lock(&_pno_state->pno_mutex);
+	/* Clear any pre-existing results in our cache
+	 * not consumed by framework
+	 */
+	dhd_gscan_clear_all_batch_results(dhd);
+	if (_pno_state->pno_mode & (DHD_PNO_GSCAN_MODE | DHD_PNO_LEGACY_MODE)) {
+		/* store current pno_mode before disabling pno */
+		mode = _pno_state->pno_mode;
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to disable PNO\n", __FUNCTION__));
+			mutex_unlock(&_pno_state->pno_mutex);
+			goto exit;
+		}
+		fw_flushed = TRUE;
+		/* restore the previous mode */
+		_pno_state->pno_mode = mode;
+	}
+	_pno_state->pno_mode |= DHD_PNO_GSCAN_MODE;
+	mutex_unlock(&_pno_state->pno_mutex);
+
+	if ((_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) &&
+		!gscan_params->epno_cfg.num_epno_ssid) {
+		struct dhd_pno_legacy_params *params_legacy;
+		params_legacy =
+			&(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS].params_legacy);
+
+		if ((err = _dhd_pno_add_ssid(dhd, &params_legacy->ssid_list,
+			params_legacy->nssid)) < 0) {
+			DHD_ERROR(("failed to add ssid list (err %d) in firmware\n", err));
+			goto exit;
+		}
+	}
+
+	if ((err = _dhd_pno_set(dhd, _params, DHD_PNO_GSCAN_MODE)) < 0) {
+		DHD_ERROR(("failed to set call pno_set (err %d) in firmware\n", err));
+		goto exit;
+	}
+
+	gscan_param_size = sizeof(wl_pfn_gscan_cfg_t) +
+	          (num_buckets_to_fw - 1) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t);
+	pfn_gscan_cfg_t = (wl_pfn_gscan_cfg_t *) MALLOCZ(dhd->osh, gscan_param_size);
+
+	if (!pfn_gscan_cfg_t) {
+		DHD_ERROR(("%s: failed to malloc memory of size %d\n",
+		   __FUNCTION__, gscan_param_size));
+		err = BCME_NOMEM;
+		goto exit;
+	}
+
+	pfn_gscan_cfg_t->version = WL_GSCAN_CFG_VERSION_1;
+	if (gscan_params->mscan)
+		pfn_gscan_cfg_t->buffer_threshold = gscan_params->buffer_threshold;
+	else
+		pfn_gscan_cfg_t->buffer_threshold = GSCAN_BATCH_NO_THR_SET;
+
+	pfn_gscan_cfg_t->flags =
+	         (gscan_params->send_all_results_flag & GSCAN_SEND_ALL_RESULTS_MASK);
+	pfn_gscan_cfg_t->flags |= GSCAN_ALL_BUCKETS_IN_FIRST_SCAN_MASK;
+	pfn_gscan_cfg_t->count_of_channel_buckets = num_buckets_to_fw;
+	pfn_gscan_cfg_t->retry_threshold = GSCAN_RETRY_THRESHOLD;
+
+	for (i = 0; i < num_buckets_to_fw; i++) {
+		pfn_gscan_cfg_t->channel_bucket[i].bucket_end_index =
+		           ch_bucket[i].bucket_end_index;
+		pfn_gscan_cfg_t->channel_bucket[i].bucket_freq_multiple =
+		           ch_bucket[i].bucket_freq_multiple;
+		pfn_gscan_cfg_t->channel_bucket[i].max_freq_multiple =
+		           ch_bucket[i].max_freq_multiple;
+		pfn_gscan_cfg_t->channel_bucket[i].repeat =
+		           ch_bucket[i].repeat;
+		pfn_gscan_cfg_t->channel_bucket[i].flag =
+		           ch_bucket[i].flag;
+	}
+
+	tot_nchan = pfn_gscan_cfg_t->channel_bucket[num_buckets_to_fw - 1].bucket_end_index + 1;
+	DHD_PNO(("Total channel num %d total ch_buckets  %d ch_buckets_to_fw %d \n", tot_nchan,
+	      tot_num_buckets, num_buckets_to_fw));
+
+	if ((err = _dhd_pno_cfg(dhd, _chan_list, tot_nchan)) < 0) {
+		DHD_ERROR(("%s : failed to set call pno_cfg (err %d) in firmware\n",
+			__FUNCTION__, err));
+		goto exit;
+	}
+
+	if ((err = _dhd_pno_gscan_cfg(dhd, pfn_gscan_cfg_t, gscan_param_size)) < 0) {
+		DHD_ERROR(("%s : failed to set call pno_gscan_cfg (err %d) in firmware\n",
+			__FUNCTION__, err));
+		goto exit;
+	}
+	/* Reprogram ePNO cfg from dhd cache if FW has been flushed */
+	if (fw_flushed) {
+		dhd_pno_set_epno(dhd);
+	}
+
+	if (gscan_params->nbssid_hotlist) {
+		struct dhd_pno_bssid *iter, *next;
+		wl_pfn_bssid_t *ptr;
+		p_pfn_bssid = (wl_pfn_bssid_t *)MALLOCZ(dhd->osh,
+			sizeof(wl_pfn_bssid_t) * gscan_params->nbssid_hotlist);
+		if (p_pfn_bssid == NULL) {
+			DHD_ERROR(("%s : failed to allocate wl_pfn_bssid_t array"
+			" (count: %d)",
+				__FUNCTION__, _params->params_hotlist.nbssid));
+			err = BCME_NOMEM;
+			_pno_state->pno_mode &= ~DHD_PNO_HOTLIST_MODE;
+			goto exit;
+		}
+		ptr = p_pfn_bssid;
+		/* convert dhd_pno_bssid to wl_pfn_bssid */
+		DHD_PNO(("nhotlist %d\n", gscan_params->nbssid_hotlist));
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		list_for_each_entry_safe(iter, next,
+		          &gscan_params->hotlist_bssid_list, list) {
+			char buffer_hotlist[64];
+			GCC_DIAGNOSTIC_POP();
+			memcpy(&ptr->macaddr,
+			&iter->macaddr, ETHER_ADDR_LEN);
+			BCM_REFERENCE(buffer_hotlist);
+			DHD_PNO(("%s\n", bcm_ether_ntoa(&ptr->macaddr, buffer_hotlist)));
+			ptr->flags = iter->flags;
+			ptr++;
+		}
+
+		err = _dhd_pno_add_bssid(dhd, p_pfn_bssid, gscan_params->nbssid_hotlist);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call _dhd_pno_add_bssid(err :%d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+	}
+
+	if ((err = _dhd_pno_enable(dhd, PNO_ON)) < 0) {
+		DHD_ERROR(("%s : failed to enable PNO err %d\n", __FUNCTION__, err));
+	}
+
+exit:
+	/* clear mode in case of error */
+	if (err < 0) {
+		int ret = dhd_pno_clean(dhd);
+
+		if (ret < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, ret));
+		} else {
+			_pno_state->pno_mode &= ~DHD_PNO_GSCAN_MODE;
+		}
+	}
+	MFREE(dhd->osh, p_pfn_bssid,
+		sizeof(wl_pfn_bssid_t) * gscan_params->nbssid_hotlist);
+	if (pfn_gscan_cfg_t) {
+		MFREE(dhd->osh, pfn_gscan_cfg_t, gscan_param_size);
+	}
+	if (ch_bucket) {
+		MFREE(dhd->osh, ch_bucket,
+		(tot_num_buckets * sizeof(wl_pfn_gscan_ch_bucket_cfg_t)));
+	}
+	return err;
+
+}
+
+static wl_pfn_gscan_ch_bucket_cfg_t *
+dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd,
+                                  dhd_pno_status_info_t *_pno_state,
+                                  uint16 *chan_list,
+                                  uint32 *num_buckets,
+                                  uint32 *num_buckets_to_fw)
+{
+	int i, num_channels, err, nchan = WL_NUMCHANNELS, ch_cnt;
+	uint16 *ptr = chan_list, max;
+	wl_pfn_gscan_ch_bucket_cfg_t *ch_bucket;
+	dhd_pno_params_t *_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	bool is_pno_legacy_running;
+	dhd_pno_gscan_channel_bucket_t *gscan_buckets = _params->params_gscan.channel_bucket;
+
+	/* ePNO and Legacy PNO do not co-exist */
+	is_pno_legacy_running = ((_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) &&
+		!_params->params_gscan.epno_cfg.num_epno_ssid);
+
+	if (is_pno_legacy_running)
+		*num_buckets = _params->params_gscan.nchannel_buckets + 1;
+	else
+		*num_buckets = _params->params_gscan.nchannel_buckets;
+
+	*num_buckets_to_fw = 0;
+
+	ch_bucket = (wl_pfn_gscan_ch_bucket_cfg_t *) MALLOC(dhd->osh,
+	   ((*num_buckets) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t)));
+
+	if (!ch_bucket) {
+		DHD_ERROR(("%s: failed to malloc memory of size %zd\n",
+			__FUNCTION__, (*num_buckets) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t)));
+		*num_buckets_to_fw = *num_buckets = 0;
+		return NULL;
+	}
+
+	max = gscan_buckets[0].bucket_freq_multiple;
+	num_channels = 0;
+	/* nchan is the remaining space left in chan_list buffer
+	 * So any overflow list of channels is ignored
+	 */
+	for (i = 0; i < _params->params_gscan.nchannel_buckets && nchan; i++) {
+		if (!gscan_buckets[i].band) {
+			ch_cnt = MIN(gscan_buckets[i].num_channels, (uint8)nchan);
+			num_channels += ch_cnt;
+			memcpy(ptr, gscan_buckets[i].chan_list,
+			    ch_cnt * sizeof(uint16));
+			ptr = ptr + ch_cnt;
+		} else {
+			/* get a valid channel list based on band B or A */
+			err = _dhd_pno_get_channels(dhd, ptr,
+			        &nchan, (gscan_buckets[i].band & GSCAN_ABG_BAND_MASK),
+			        !(gscan_buckets[i].band & GSCAN_DFS_MASK));
+
+			if (err < 0) {
+				DHD_ERROR(("%s: failed to get valid channel list(band : %d)\n",
+					__FUNCTION__, gscan_buckets[i].band));
+				MFREE(dhd->osh, ch_bucket,
+				      ((*num_buckets) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t)));
+				*num_buckets_to_fw = *num_buckets = 0;
+				return NULL;
+			}
+
+			num_channels += nchan;
+			ptr = ptr + nchan;
+		}
+
+		ch_bucket[i].bucket_end_index = num_channels - 1;
+		ch_bucket[i].bucket_freq_multiple = gscan_buckets[i].bucket_freq_multiple;
+		ch_bucket[i].repeat = gscan_buckets[i].repeat;
+		ch_bucket[i].max_freq_multiple = gscan_buckets[i].bucket_max_multiple;
+		ch_bucket[i].flag = gscan_buckets[i].report_flag;
+		/* HAL and FW interpretations are opposite for this bit */
+		ch_bucket[i].flag ^= DHD_PNO_REPORT_NO_BATCH;
+		if (max < gscan_buckets[i].bucket_freq_multiple)
+			max = gscan_buckets[i].bucket_freq_multiple;
+		nchan = WL_NUMCHANNELS - num_channels;
+		*num_buckets_to_fw = *num_buckets_to_fw + 1;
+		DHD_PNO(("end_idx  %d freq_mult - %d\n",
+		ch_bucket[i].bucket_end_index, ch_bucket[i].bucket_freq_multiple));
+	}
+
+	_params->params_gscan.max_ch_bucket_freq = max;
+	/* Legacy PNO maybe running, which means we need to create a legacy PNO bucket
+	 * Get GCF of Legacy PNO and Gscan scanfreq
+	 */
+	if (is_pno_legacy_running) {
+		dhd_pno_params_t *_params1 = &_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS];
+		uint16 *legacy_chan_list = _params1->params_legacy.chan_list;
+		uint16 common_freq;
+		uint32 legacy_bucket_idx = _params->params_gscan.nchannel_buckets;
+		/* If no space is left then only gscan buckets will be sent to FW */
+		if (nchan) {
+			common_freq = gcd(_params->params_gscan.scan_fr,
+			                  _params1->params_legacy.scan_fr);
+			max = gscan_buckets[0].bucket_freq_multiple;
+			/* GSCAN buckets */
+			for (i = 0; i < _params->params_gscan.nchannel_buckets; i++) {
+				ch_bucket[i].bucket_freq_multiple *= _params->params_gscan.scan_fr;
+				ch_bucket[i].bucket_freq_multiple /= common_freq;
+				if (max < gscan_buckets[i].bucket_freq_multiple)
+					max = gscan_buckets[i].bucket_freq_multiple;
+			}
+			/* Legacy PNO bucket */
+			ch_bucket[legacy_bucket_idx].bucket_freq_multiple =
+			           _params1->params_legacy.scan_fr;
+			ch_bucket[legacy_bucket_idx].bucket_freq_multiple /=
+			           common_freq;
+			_params->params_gscan.max_ch_bucket_freq = MAX(max,
+			         ch_bucket[legacy_bucket_idx].bucket_freq_multiple);
+			ch_bucket[legacy_bucket_idx].flag = CH_BUCKET_REPORT_REGULAR;
+			/* Now add channels to the legacy scan bucket */
+			for (i = 0; i < _params1->params_legacy.nchan && nchan; i++, nchan--) {
+				ptr[i] = legacy_chan_list[i];
+				num_channels++;
+			}
+			ch_bucket[legacy_bucket_idx].bucket_end_index = num_channels - 1;
+			*num_buckets_to_fw = *num_buckets_to_fw + 1;
+			DHD_PNO(("end_idx  %d freq_mult - %d\n",
+			            ch_bucket[legacy_bucket_idx].bucket_end_index,
+			            ch_bucket[legacy_bucket_idx].bucket_freq_multiple));
+		}
+	}
+	return ch_bucket;
+}
+
+static int
+dhd_pno_stop_for_gscan(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	int mode;
+	dhd_pno_status_info_t *_pno_state;
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n",
+			__FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	if (!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE)) {
+		DHD_ERROR(("%s : GSCAN is not enabled\n", __FUNCTION__));
+		goto exit;
+	}
+	if (_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan.mscan) {
+		/* retrieve the batching data from firmware into host */
+		err = dhd_wait_batch_results_complete(dhd);
+		if (err != BCME_OK)
+			goto exit;
+	}
+	mutex_lock(&_pno_state->pno_mutex);
+	mode = _pno_state->pno_mode & ~DHD_PNO_GSCAN_MODE;
+	err = dhd_pno_clean(dhd);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+			__FUNCTION__, err));
+		mutex_unlock(&_pno_state->pno_mutex);
+		return err;
+	}
+	_pno_state->pno_mode = mode;
+	mutex_unlock(&_pno_state->pno_mutex);
+
+	/* Reprogram Legacy PNO if it was running */
+	if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+		struct dhd_pno_legacy_params *params_legacy;
+		uint16 chan_list[WL_NUMCHANNELS];
+
+		params_legacy = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS].params_legacy);
+		_pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE;
+
+		DHD_PNO(("Restarting Legacy PNO SSID scan...\n"));
+		memcpy(chan_list, params_legacy->chan_list,
+			(params_legacy->nchan * sizeof(uint16)));
+		err = dhd_pno_set_legacy_pno(dhd, params_legacy->scan_fr,
+			params_legacy->pno_repeat, params_legacy->pno_freq_expo_max,
+			chan_list, params_legacy->nchan);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to restart legacy PNO scan(err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+
+	}
+
+exit:
+	return err;
+}
+
+int
+dhd_pno_initiate_gscan_request(dhd_pub_t *dhd, bool run, bool flush)
+{
+	int err = BCME_OK;
+	dhd_pno_params_t *params;
+	dhd_pno_status_info_t *_pno_state;
+	struct dhd_pno_gscan_params *gscan_params;
+
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	DHD_PNO(("%s enter - run %d flush %d\n", __FUNCTION__, run, flush));
+
+	params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	gscan_params = &params->params_gscan;
+
+	if (run) {
+		err = dhd_pno_set_for_gscan(dhd, gscan_params);
+	} else {
+		if (flush) {
+			mutex_lock(&_pno_state->pno_mutex);
+			dhd_pno_reset_cfg_gscan(dhd, params, _pno_state, GSCAN_FLUSH_ALL_CFG);
+			mutex_unlock(&_pno_state->pno_mutex);
+		}
+		/* Need to stop all gscan */
+		err = dhd_pno_stop_for_gscan(dhd);
+	}
+
+	return err;
+}
+
+int
+dhd_pno_enable_full_scan_result(dhd_pub_t *dhd, bool real_time_flag)
+{
+	int err = BCME_OK;
+	dhd_pno_params_t *params;
+	dhd_pno_status_info_t *_pno_state;
+	struct dhd_pno_gscan_params *gscan_params;
+	uint8 old_flag;
+
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	gscan_params = &params->params_gscan;
+
+	mutex_lock(&_pno_state->pno_mutex);
+
+	old_flag = gscan_params->send_all_results_flag;
+	gscan_params->send_all_results_flag = (uint8) real_time_flag;
+	if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+	    if (old_flag != gscan_params->send_all_results_flag) {
+			wl_pfn_gscan_cfg_t gscan_cfg;
+
+			gscan_cfg.version = WL_GSCAN_CFG_VERSION_1;
+			gscan_cfg.flags = (gscan_params->send_all_results_flag &
+			                           GSCAN_SEND_ALL_RESULTS_MASK);
+			gscan_cfg.flags |= GSCAN_CFG_FLAGS_ONLY_MASK;
+
+			if ((err = _dhd_pno_gscan_cfg(dhd, &gscan_cfg,
+			            sizeof(wl_pfn_gscan_cfg_t))) < 0) {
+				DHD_ERROR(("%s : pno_gscan_cfg failed (err %d) in firmware\n",
+					__FUNCTION__, err));
+				goto exit_mutex_unlock;
+			}
+		} else {
+			DHD_PNO(("No change in flag - %d\n", old_flag));
+		}
+	} else {
+		DHD_PNO(("Gscan not started\n"));
+	}
+exit_mutex_unlock:
+	mutex_unlock(&_pno_state->pno_mutex);
+exit:
+	return err;
+}
+
+/* Cleanup any consumed results
+ * Return TRUE if all results consumed else FALSE
+ */
+int dhd_gscan_batch_cache_cleanup(dhd_pub_t *dhd)
+{
+	int ret = 0;
+	dhd_pno_params_t *params;
+	struct dhd_pno_gscan_params *gscan_params;
+	dhd_pno_status_info_t *_pno_state;
+	gscan_results_cache_t *iter, *tmp;
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	gscan_params = &params->params_gscan;
+	iter = gscan_params->gscan_batch_cache;
+
+	while (iter) {
+		if (iter->tot_consumed == iter->tot_count) {
+			tmp = iter->next;
+			MFREE(dhd->osh, iter,
+				((iter->tot_count - 1) * sizeof(wifi_gscan_result_t))
+				+ sizeof(gscan_results_cache_t));
+			iter = tmp;
+		} else
+			break;
+	}
+	gscan_params->gscan_batch_cache = iter;
+	ret = (iter == NULL);
+	return ret;
+}
+
+static int
+_dhd_pno_get_gscan_batch_from_fw(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	uint32 timestamp = 0, ts = 0, i, j, timediff;
+	dhd_pno_params_t *params;
+	dhd_pno_status_info_t *_pno_state;
+	wl_pfn_lnet_info_v1_t *plnetinfo;
+	wl_pfn_lnet_info_v2_t *plnetinfo_v2;
+	struct dhd_pno_gscan_params *gscan_params;
+	wl_pfn_lscanresults_v1_t *plbestnet_v1 = NULL;
+	wl_pfn_lscanresults_v2_t *plbestnet_v2 = NULL;
+	gscan_results_cache_t *iter, *tail;
+	wifi_gscan_result_t *result;
+	uint8 *nAPs_per_scan = NULL;
+	uint8 num_scans_in_cur_iter;
+	uint16 count;
+	uint16 fwcount;
+	uint16 fwstatus = PFN_INCOMPLETE;
+	struct osl_timespec tm_spec;
+
+	/* Static asserts in _dhd_pno_get_for_batch() below guarantee the v1 and v2
+	 * net_info and subnet_info structures are compatible in size and SSID offset,
+	 * allowing v1 to be safely used in the code below except for lscanresults
+	 * fields themselves (status, count, offset to netinfo).
+	 */
+
+	NULL_CHECK(dhd, "dhd is NULL\n", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+	if (!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE)) {
+		DHD_ERROR(("%s: GSCAN is not enabled\n", __FUNCTION__));
+		goto exit;
+	}
+	gscan_params = &params->params_gscan;
+	nAPs_per_scan = (uint8 *) MALLOC(dhd->osh, gscan_params->mscan);
+
+	if (!nAPs_per_scan) {
+		DHD_ERROR(("%s :Out of memory!! Cant malloc %d bytes\n", __FUNCTION__,
+		gscan_params->mscan));
+		err = BCME_NOMEM;
+		goto exit;
+	}
+
+	plbestnet_v1 = (wl_pfn_lscanresults_v1_t *)MALLOC(dhd->osh, PNO_BESTNET_LEN);
+	if (!plbestnet_v1) {
+		DHD_ERROR(("%s :Out of memory!! Cant malloc %d bytes\n", __FUNCTION__,
+		      (int)PNO_BESTNET_LEN));
+		err = BCME_NOMEM;
+		goto exit;
+	}
+	plbestnet_v2 = (wl_pfn_lscanresults_v2_t *)plbestnet_v1;
+
+	mutex_lock(&_pno_state->pno_mutex);
+
+	dhd_gscan_clear_all_batch_results(dhd);
+
+	if (!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE)) {
+		DHD_ERROR(("%s : GSCAN is not enabled\n", __FUNCTION__));
+		goto exit_mutex_unlock;
+	}
+
+	timediff = gscan_params->scan_fr * 1000;
+	timediff = timediff >> 1;
+
+	/* Ok, now lets start getting results from the FW */
+	tail = gscan_params->gscan_batch_cache;
+	do {
+		err = dhd_iovar(dhd, 0, "pfnlbest", NULL, 0, (char *)plbestnet_v1, PNO_BESTNET_LEN,
+				FALSE);
+		if (err < 0) {
+			DHD_ERROR(("%s : Cannot get all the batch results, err :%d\n",
+				__FUNCTION__, err));
+			goto exit_mutex_unlock;
+		}
+		osl_get_monotonic_boottime(&tm_spec);
+
+		if (plbestnet_v1->version == PFN_LBEST_SCAN_RESULT_VERSION_V1) {
+			fwstatus = plbestnet_v1->status;
+			fwcount = plbestnet_v1->count;
+			plnetinfo = &plbestnet_v1->netinfo[0];
+
+			DHD_PNO(("ver %d, status : %d, count %d\n",
+				plbestnet_v1->version, fwstatus, fwcount));
+
+			if (fwcount == 0) {
+				DHD_PNO(("No more batch results\n"));
+				goto exit_mutex_unlock;
+			}
+			if (fwcount > BESTN_MAX) {
+				DHD_ERROR(("%s :fwcount %d is greater than BESTN_MAX %d \n",
+					__FUNCTION__, fwcount, (int)BESTN_MAX));
+				/* Process only BESTN_MAX number of results per batch */
+				fwcount = BESTN_MAX;
+			}
+			num_scans_in_cur_iter = 0;
+
+			timestamp = plnetinfo->timestamp;
+			/* find out how many scans' results did we get in
+			 * this batch of FW results
+			 */
+			for (i = 0, count = 0; i < fwcount; i++, count++, plnetinfo++) {
+				/* Unlikely to happen, but just in case the results from
+				 * FW doesnt make sense..... Assume its part of one single scan
+				 */
+				if (num_scans_in_cur_iter >= gscan_params->mscan) {
+					num_scans_in_cur_iter = 0;
+					count = fwcount;
+					break;
+				}
+				if (TIME_DIFF_MS(timestamp, plnetinfo->timestamp) > timediff) {
+					nAPs_per_scan[num_scans_in_cur_iter] = count;
+					count = 0;
+					num_scans_in_cur_iter++;
+				}
+				timestamp = plnetinfo->timestamp;
+			}
+			if (num_scans_in_cur_iter < gscan_params->mscan) {
+				nAPs_per_scan[num_scans_in_cur_iter] = count;
+				num_scans_in_cur_iter++;
+			}
+
+			DHD_PNO(("num_scans_in_cur_iter %d\n", num_scans_in_cur_iter));
+			/* reset plnetinfo to the first item for the next loop */
+			plnetinfo -= i;
+
+			for (i = 0; i < num_scans_in_cur_iter; i++) {
+				iter = (gscan_results_cache_t *)
+					MALLOCZ(dhd->osh, ((nAPs_per_scan[i] - 1) *
+					sizeof(wifi_gscan_result_t)) +
+					sizeof(gscan_results_cache_t));
+				if (!iter) {
+					DHD_ERROR(("%s :Out of memory!! Cant malloc %d bytes\n",
+						__FUNCTION__, gscan_params->mscan));
+					err = BCME_NOMEM;
+					goto exit_mutex_unlock;
+				}
+				/* Need this check because the new set of results from FW
+				 * maybe a continuation of previous sets' scan results
+				 */
+				if (TIME_DIFF_MS(ts, plnetinfo->timestamp) > timediff) {
+					iter->scan_id = ++gscan_params->scan_id;
+				} else {
+					iter->scan_id = gscan_params->scan_id;
+				}
+				DHD_PNO(("scan_id %d tot_count %d \n",
+					gscan_params->scan_id, nAPs_per_scan[i]));
+				iter->tot_count = nAPs_per_scan[i];
+				iter->tot_consumed = 0;
+				iter->flag = 0;
+				if (plnetinfo->flags & PFN_PARTIAL_SCAN_MASK) {
+					DHD_PNO(("This scan is aborted\n"));
+					iter->flag = (ENABLE << PNO_STATUS_ABORT);
+				} else if (gscan_params->reason) {
+					iter->flag = (ENABLE << gscan_params->reason);
+				}
+
+				if (!tail) {
+					gscan_params->gscan_batch_cache = iter;
+				} else {
+					tail->next = iter;
+				}
+				tail = iter;
+				iter->next = NULL;
+				for (j = 0; j < nAPs_per_scan[i]; j++, plnetinfo++) {
+					result = &iter->results[j];
+
+					result->channel = wl_channel_to_frequency(
+						wf_chspec_ctlchan(plnetinfo->pfnsubnet.channel),
+						CHSPEC_BAND(plnetinfo->pfnsubnet.channel));
+					result->rssi = (int32) plnetinfo->RSSI;
+					result->beacon_period = 0;
+					result->capability = 0;
+					result->rtt = (uint64) plnetinfo->rtt0;
+					result->rtt_sd = (uint64) plnetinfo->rtt1;
+					result->ts = convert_fw_rel_time_to_systime(&tm_spec,
+							plnetinfo->timestamp);
+					ts = plnetinfo->timestamp;
+					if (plnetinfo->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+						DHD_ERROR(("%s: Invalid SSID length %d\n",
+							__FUNCTION__,
+							plnetinfo->pfnsubnet.SSID_len));
+						plnetinfo->pfnsubnet.SSID_len = DOT11_MAX_SSID_LEN;
+					}
+					(void)memcpy_s(result->ssid, DOT11_MAX_SSID_LEN,
+						plnetinfo->pfnsubnet.SSID,
+						plnetinfo->pfnsubnet.SSID_len);
+					result->ssid[plnetinfo->pfnsubnet.SSID_len] = '\0';
+					(void)memcpy_s(&result->macaddr, ETHER_ADDR_LEN,
+						&plnetinfo->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+
+					DHD_PNO(("\tSSID : "));
+					DHD_PNO(("\n"));
+					DHD_PNO(("\tBSSID: "MACDBG"\n",
+						MAC2STRDBG(result->macaddr.octet)));
+					DHD_PNO(("\tchannel: %d, RSSI: %d, timestamp: %d ms\n",
+						plnetinfo->pfnsubnet.channel,
+						plnetinfo->RSSI, plnetinfo->timestamp));
+					DHD_PNO(("\tRTT0 : %d, RTT1: %d\n",
+						plnetinfo->rtt0, plnetinfo->rtt1));
+
+				}
+			}
+
+		} else if (plbestnet_v2->version == PFN_LBEST_SCAN_RESULT_VERSION_V2) {
+			fwstatus = plbestnet_v2->status;
+			fwcount = plbestnet_v2->count;
+			plnetinfo_v2 = (wl_pfn_lnet_info_v2_t*)&plbestnet_v2->netinfo[0];
+
+			DHD_PNO(("ver %d, status : %d, count %d\n",
+				plbestnet_v2->version, fwstatus, fwcount));
+
+			if (fwcount == 0) {
+				DHD_PNO(("No more batch results\n"));
+				goto exit_mutex_unlock;
+			}
+			if (fwcount > BESTN_MAX) {
+				DHD_ERROR(("%s :fwcount %d is greater than BESTN_MAX %d \n",
+					__FUNCTION__, fwcount, (int)BESTN_MAX));
+				/* Process only BESTN_MAX number of results per batch */
+				fwcount = BESTN_MAX;
+			}
+			num_scans_in_cur_iter = 0;
+
+			timestamp = plnetinfo_v2->timestamp;
+			/* find out how many scans' results did we get
+			 * in this batch of FW results
+			 */
+			for (i = 0, count = 0; i < fwcount; i++, count++, plnetinfo_v2++) {
+				/* Unlikely to happen, but just in case the results from
+				 * FW doesnt make sense..... Assume its part of one single scan
+				 */
+				if (num_scans_in_cur_iter >= gscan_params->mscan) {
+					num_scans_in_cur_iter = 0;
+					count = fwcount;
+					break;
+				}
+				if (TIME_DIFF_MS(timestamp, plnetinfo_v2->timestamp) > timediff) {
+					nAPs_per_scan[num_scans_in_cur_iter] = count;
+					count = 0;
+					num_scans_in_cur_iter++;
+				}
+				timestamp = plnetinfo_v2->timestamp;
+			}
+			if (num_scans_in_cur_iter < gscan_params->mscan) {
+				nAPs_per_scan[num_scans_in_cur_iter] = count;
+				num_scans_in_cur_iter++;
+			}
+
+			DHD_PNO(("num_scans_in_cur_iter %d\n", num_scans_in_cur_iter));
+			/* reset plnetinfo to the first item for the next loop */
+			plnetinfo_v2 -= i;
+
+			for (i = 0; i < num_scans_in_cur_iter; i++) {
+				iter = (gscan_results_cache_t *)
+					MALLOCZ(dhd->osh, ((nAPs_per_scan[i] - 1) *
+					sizeof(wifi_gscan_result_t)) +
+					sizeof(gscan_results_cache_t));
+				if (!iter) {
+					DHD_ERROR(("%s :Out of memory!! Cant malloc %d bytes\n",
+						__FUNCTION__, gscan_params->mscan));
+					err = BCME_NOMEM;
+					goto exit_mutex_unlock;
+				}
+				/* Need this check because the new set of results from FW
+				 * maybe a continuation of previous sets' scan results
+				 */
+				if (TIME_DIFF_MS(ts, plnetinfo_v2->timestamp) > timediff) {
+					iter->scan_id = ++gscan_params->scan_id;
+				} else {
+					iter->scan_id = gscan_params->scan_id;
+				}
+				DHD_PNO(("scan_id %d tot_count %d ch_bucket %x\n",
+					gscan_params->scan_id, nAPs_per_scan[i],
+					plbestnet_v2->scan_ch_buckets[i]));
+				iter->tot_count = nAPs_per_scan[i];
+				iter->scan_ch_bucket = plbestnet_v2->scan_ch_buckets[i];
+				iter->tot_consumed = 0;
+				iter->flag = 0;
+				if (plnetinfo_v2->flags & PFN_PARTIAL_SCAN_MASK) {
+					DHD_PNO(("This scan is aborted\n"));
+					iter->flag = (ENABLE << PNO_STATUS_ABORT);
+				} else if (gscan_params->reason) {
+					iter->flag = (ENABLE << gscan_params->reason);
+				}
+
+				if (!tail) {
+					gscan_params->gscan_batch_cache = iter;
+				} else {
+					tail->next = iter;
+				}
+				tail = iter;
+				iter->next = NULL;
+				for (j = 0; j < nAPs_per_scan[i]; j++, plnetinfo_v2++) {
+					result = &iter->results[j];
+
+					result->channel =
+						wl_channel_to_frequency(
+						wf_chspec_ctlchan(plnetinfo_v2->pfnsubnet.channel),
+						CHSPEC_BAND(plnetinfo_v2->pfnsubnet.channel));
+					result->rssi = (int32) plnetinfo_v2->RSSI;
+					/* Info not available & not expected */
+					result->beacon_period = 0;
+					result->capability = 0;
+					result->rtt = (uint64) plnetinfo_v2->rtt0;
+					result->rtt_sd = (uint64) plnetinfo_v2->rtt1;
+					result->ts = convert_fw_rel_time_to_systime(&tm_spec,
+						plnetinfo_v2->timestamp);
+					ts = plnetinfo_v2->timestamp;
+					if (plnetinfo_v2->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+						DHD_ERROR(("%s: Invalid SSID length %d\n",
+							__FUNCTION__,
+							plnetinfo_v2->pfnsubnet.SSID_len));
+						plnetinfo_v2->pfnsubnet.SSID_len =
+							DOT11_MAX_SSID_LEN;
+					}
+					(void)memcpy_s(result->ssid, DOT11_MAX_SSID_LEN,
+						plnetinfo_v2->pfnsubnet.u.SSID,
+						plnetinfo_v2->pfnsubnet.SSID_len);
+					result->ssid[plnetinfo_v2->pfnsubnet.SSID_len] = '\0';
+					(void)memcpy_s(&result->macaddr, ETHER_ADDR_LEN,
+						&plnetinfo_v2->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+
+					DHD_PNO(("\tSSID : "));
+					DHD_PNO(("\n"));
+					DHD_PNO(("\tBSSID: "MACDBG"\n",
+						MAC2STRDBG(result->macaddr.octet)));
+					DHD_PNO(("\tchannel: %d, RSSI: %d, timestamp: %d ms\n",
+						plnetinfo_v2->pfnsubnet.channel,
+						plnetinfo_v2->RSSI, plnetinfo_v2->timestamp));
+					DHD_PNO(("\tRTT0 : %d, RTT1: %d\n",
+						plnetinfo_v2->rtt0, plnetinfo_v2->rtt1));
+
+				}
+			}
+
+		} else {
+			err = BCME_VERSION;
+			DHD_ERROR(("bestnet fw version %d not supported\n",
+				plbestnet_v1->version));
+			goto exit_mutex_unlock;
+		}
+	} while (fwstatus == PFN_INCOMPLETE);
+
+exit_mutex_unlock:
+	mutex_unlock(&_pno_state->pno_mutex);
+exit:
+	params->params_gscan.get_batch_flag = GSCAN_BATCH_RETRIEVAL_COMPLETE;
+	smp_wmb();
+	wake_up_interruptible(&_pno_state->batch_get_wait);
+	if (nAPs_per_scan) {
+		MFREE(dhd->osh, nAPs_per_scan, gscan_params->mscan * sizeof(uint8));
+	}
+	if (plbestnet_v1) {
+		MFREE(dhd->osh, plbestnet_v1, PNO_BESTNET_LEN);
+	}
+	DHD_PNO(("Batch retrieval done!\n"));
+	return err;
+}
+#endif /* GSCAN_SUPPORT */
+
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+static void *
+dhd_get_gscan_batch_results(dhd_pub_t *dhd, uint32 *len)
+{
+	gscan_results_cache_t *iter, *results;
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	uint16 num_scan_ids = 0, num_results = 0;
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+
+	iter = results = _params->params_gscan.gscan_batch_cache;
+	while (iter) {
+		num_results += iter->tot_count - iter->tot_consumed;
+		num_scan_ids++;
+		iter = iter->next;
+	}
+
+	*len = ((num_results << 16) | (num_scan_ids));
+	return results;
+}
+
+void *
+dhd_pno_get_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type,
+         void *info, uint32 *len)
+{
+	void *ret = NULL;
+	dhd_pno_gscan_capabilities_t *ptr;
+	dhd_pno_ssid_t *ssid_elem;
+	dhd_pno_params_t *_params;
+	dhd_epno_ssid_cfg_t *epno_cfg;
+	dhd_pno_status_info_t *_pno_state;
+
+	if (!dhd || !dhd->pno_state) {
+		DHD_ERROR(("NULL POINTER : %s\n", __FUNCTION__));
+		return NULL;
+	}
+
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+
+	if (!len) {
+		DHD_ERROR(("%s: len is NULL\n", __FUNCTION__));
+		return NULL;
+	}
+
+	switch (type) {
+		case DHD_PNO_GET_CAPABILITIES:
+			ptr = (dhd_pno_gscan_capabilities_t *)
+			MALLOCZ(dhd->osh, sizeof(dhd_pno_gscan_capabilities_t));
+			if (!ptr)
+				break;
+			/* Hardcoding these values for now, need to get
+			 * these values from FW, will change in a later check-in
+			 */
+			ptr->max_scan_cache_size = GSCAN_MAX_AP_CACHE;
+			ptr->max_scan_buckets = GSCAN_MAX_CH_BUCKETS;
+			ptr->max_ap_cache_per_scan = GSCAN_MAX_AP_CACHE_PER_SCAN;
+			ptr->max_rssi_sample_size = PFN_SWC_RSSI_WINDOW_MAX;
+			ptr->max_scan_reporting_threshold = 100;
+			ptr->max_hotlist_bssids = PFN_HOTLIST_MAX_NUM_APS;
+			ptr->max_hotlist_ssids = 0;
+			ptr->max_significant_wifi_change_aps = 0;
+			ptr->max_bssid_history_entries = 0;
+			ptr->max_epno_ssid_crc32 = MAX_EPNO_SSID_NUM;
+			ptr->max_epno_hidden_ssid = MAX_EPNO_HIDDEN_SSID;
+			ptr->max_white_list_ssid = MAX_WHITELIST_SSID;
+			ret = (void *)ptr;
+			*len = sizeof(dhd_pno_gscan_capabilities_t);
+			break;
+
+		case DHD_PNO_GET_BATCH_RESULTS:
+			ret = dhd_get_gscan_batch_results(dhd, len);
+			break;
+		case DHD_PNO_GET_CHANNEL_LIST:
+			if (info) {
+				uint16 ch_list[WL_NUMCHANNELS];
+				uint32 *p, mem_needed, i;
+				int32 err, nchan = WL_NUMCHANNELS;
+				uint32 *gscan_band = (uint32 *) info;
+				uint8 band = 0;
+
+				/* No band specified?, nothing to do */
+				if ((*gscan_band & GSCAN_BAND_MASK) == 0) {
+					DHD_PNO(("No band specified\n"));
+					*len = 0;
+					break;
+				}
+
+				/* HAL and DHD use different bits for 2.4G and
+				 * 5G in bitmap. Hence translating it here...
+				 */
+				if (*gscan_band & GSCAN_BG_BAND_MASK) {
+					band |= WLC_BAND_2G;
+				}
+				if (*gscan_band & GSCAN_A_BAND_MASK) {
+					band |=
+#ifdef WL_6G_BAND
+						WLC_BAND_6G |
+#endif /* WL_6G_BAND */
+						WLC_BAND_5G;
+				}
+
+				err = _dhd_pno_get_channels(dhd, ch_list, &nchan,
+				                          (band & GSCAN_ABG_BAND_MASK),
+				                          !(*gscan_band & GSCAN_DFS_MASK));
+
+				if (err < 0) {
+					DHD_ERROR(("%s: failed to get valid channel list\n",
+						__FUNCTION__));
+					*len = 0;
+				} else {
+					mem_needed = sizeof(uint32) * nchan;
+					p = (uint32 *)MALLOC(dhd->osh, mem_needed);
+					if (!p) {
+						DHD_ERROR(("%s: Unable to malloc %d bytes\n",
+							__FUNCTION__, mem_needed));
+						break;
+					}
+					for (i = 0; i < nchan; i++) {
+						p[i] = wl_channel_to_frequency(
+							(ch_list[i]),
+							(ch_list[i] <= CH_MAX_2G_CHANNEL?
+							WL_CHANSPEC_BAND_2G : WL_CHANSPEC_BAND_5G));
+					}
+					ret = p;
+					*len = mem_needed;
+				}
+			} else {
+				*len = 0;
+				DHD_ERROR(("%s: info buffer is NULL\n", __FUNCTION__));
+			}
+			break;
+		case DHD_PNO_GET_NEW_EPNO_SSID_ELEM:
+			epno_cfg = &_params->params_gscan.epno_cfg;
+			if (epno_cfg->num_epno_ssid >=
+					MAX_EPNO_SSID_NUM) {
+				DHD_ERROR(("Excessive number of ePNO SSIDs programmed %d\n",
+					epno_cfg->num_epno_ssid));
+				return NULL;
+			}
+			if (!epno_cfg->num_epno_ssid) {
+				INIT_LIST_HEAD(&epno_cfg->epno_ssid_list);
+			}
+			ssid_elem = MALLOCZ(dhd->osh, sizeof(dhd_pno_ssid_t));
+			if (!ssid_elem) {
+				DHD_ERROR(("EPNO ssid: cannot alloc %zd bytes",
+					sizeof(dhd_pno_ssid_t)));
+				return NULL;
+			}
+			epno_cfg->num_epno_ssid++;
+			list_add_tail(&ssid_elem->list, &epno_cfg->epno_ssid_list);
+			ret = ssid_elem;
+			break;
+		default:
+			DHD_ERROR(("%s: Unrecognized cmd type - %d\n", __FUNCTION__, type));
+			break;
+	}
+
+	return ret;
+
+}
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+
+static int
+_dhd_pno_get_for_batch(dhd_pub_t *dhd, char *buf, int bufsize, int reason)
+{
+	int err = BCME_OK;
+	int i, j;
+	uint32 timestamp = 0;
+	dhd_pno_params_t *_params = NULL;
+	dhd_pno_status_info_t *_pno_state = NULL;
+	wl_pfn_lscanresults_v1_t *plbestnet_v1 = NULL;
+	wl_pfn_lscanresults_v2_t *plbestnet_v2 = NULL;
+	wl_pfn_lnet_info_v1_t *plnetinfo;
+	wl_pfn_lnet_info_v2_t *plnetinfo_v2;
+	dhd_pno_bestnet_entry_t *pbestnet_entry;
+	dhd_pno_best_header_t *pbestnetheader = NULL;
+	dhd_pno_scan_results_t *pscan_results = NULL, *siter, *snext;
+	bool allocate_header = FALSE;
+	uint16 fwstatus = PFN_INCOMPLETE;
+	uint16 fwcount;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+
+	/* The static asserts below guarantee the v1 and v2 net_info and subnet_info
+	 * structures are compatible in size and SSID offset, allowing v1 to be safely
+	 * used in the code below except for lscanresults fields themselves
+	 * (status, count, offset to netinfo).
+	 */
+	STATIC_ASSERT(sizeof(wl_pfn_net_info_v1_t) == sizeof(wl_pfn_net_info_v2_t));
+	STATIC_ASSERT(sizeof(wl_pfn_lnet_info_v1_t) == sizeof(wl_pfn_lnet_info_v2_t));
+	STATIC_ASSERT(sizeof(wl_pfn_subnet_info_v1_t) == sizeof(wl_pfn_subnet_info_v2_t));
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit_no_unlock;
+	}
+
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit_no_unlock;
+	}
+
+	if (!(_pno_state->pno_mode & DHD_PNO_BATCH_MODE)) {
+		DHD_ERROR(("%s: Batching SCAN mode is not enabled\n", __FUNCTION__));
+		goto exit_no_unlock;
+	}
+	mutex_lock(&_pno_state->pno_mutex);
+	_params = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS];
+	if (buf && bufsize) {
+		if (!list_empty(&_params->params_batch.get_batch.expired_scan_results_list)) {
+			/* need to check whether we have cashed data or not */
+			DHD_PNO(("%s: have cashed batching data in Driver\n",
+				__FUNCTION__));
+			/* convert to results format */
+			goto convert_format;
+		} else {
+			/* this is a first try to get batching results */
+			if (!list_empty(&_params->params_batch.get_batch.scan_results_list)) {
+				/* move the scan_results_list to expired_scan_results_lists */
+				GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+				list_for_each_entry_safe(siter, snext,
+					&_params->params_batch.get_batch.scan_results_list, list) {
+					GCC_DIAGNOSTIC_POP();
+					list_move_tail(&siter->list,
+					&_params->params_batch.get_batch.expired_scan_results_list);
+				}
+				_params->params_batch.get_batch.top_node_cnt = 0;
+				_params->params_batch.get_batch.expired_tot_scan_cnt =
+					_params->params_batch.get_batch.tot_scan_cnt;
+				_params->params_batch.get_batch.tot_scan_cnt = 0;
+				goto convert_format;
+			}
+		}
+	}
+	/* create dhd_pno_scan_results_t whenever we got event WLC_E_PFN_BEST_BATCHING */
+	pscan_results = (dhd_pno_scan_results_t *)MALLOC(dhd->osh, SCAN_RESULTS_SIZE);
+	if (pscan_results == NULL) {
+		err = BCME_NOMEM;
+		DHD_ERROR(("failed to allocate dhd_pno_scan_results_t\n"));
+		goto exit;
+	}
+	pscan_results->bestnetheader = NULL;
+	pscan_results->cnt_header = 0;
+	/* add the element into list unless total node cnt is less than MAX_NODE_ CNT */
+	if (_params->params_batch.get_batch.top_node_cnt < MAX_NODE_CNT) {
+		list_add(&pscan_results->list, &_params->params_batch.get_batch.scan_results_list);
+		_params->params_batch.get_batch.top_node_cnt++;
+	} else {
+		int _removed_scan_cnt;
+		/* remove oldest one and add new one */
+		DHD_PNO(("%s : Remove oldest node and add new one\n", __FUNCTION__));
+		_removed_scan_cnt = _dhd_pno_clear_all_batch_results(dhd,
+			&_params->params_batch.get_batch.scan_results_list, TRUE);
+		_params->params_batch.get_batch.tot_scan_cnt -= _removed_scan_cnt;
+		list_add(&pscan_results->list, &_params->params_batch.get_batch.scan_results_list);
+
+	}
+
+	plbestnet_v1 = (wl_pfn_lscanresults_v1_t *)MALLOC(dhd->osh, PNO_BESTNET_LEN);
+	if (!plbestnet_v1) {
+		err = BCME_NOMEM;
+		DHD_ERROR(("%s: failed to allocate buffer for bestnet", __FUNCTION__));
+		goto exit;
+	}
+
+	plbestnet_v2 = (wl_pfn_lscanresults_v2_t*)plbestnet_v1;
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	do {
+		err = dhd_iovar(dhd, 0, "pfnlbest", NULL, 0, (char *)plbestnet_v1, PNO_BESTNET_LEN,
+				FALSE);
+		if (err < 0) {
+			if (err == BCME_EPERM) {
+				DHD_ERROR(("we cannot get the batching data "
+					"during scanning in firmware, try again\n,"));
+				msleep(500);
+				continue;
+			} else {
+				DHD_ERROR(("%s : failed to execute pfnlbest (err :%d)\n",
+					__FUNCTION__, err));
+				goto exit;
+			}
+		}
+
+		if (plbestnet_v1->version == PFN_LBEST_SCAN_RESULT_VERSION_V1) {
+			fwstatus = plbestnet_v1->status;
+			fwcount = plbestnet_v1->count;
+			plnetinfo = &plbestnet_v1->netinfo[0];
+			if (fwcount == 0) {
+				DHD_PNO(("No more batch results\n"));
+				goto exit;
+			}
+			if (fwcount > BESTN_MAX) {
+				DHD_ERROR(("%s :fwcount %d is greater than BESTN_MAX %d \n",
+					__FUNCTION__, fwcount, (int)BESTN_MAX));
+				/* Process only BESTN_MAX number of results per batch */
+				fwcount = BESTN_MAX;
+			}
+			for (i = 0; i < fwcount; i++) {
+				pbestnet_entry = (dhd_pno_bestnet_entry_t *)
+					MALLOC(dhd->osh, BESTNET_ENTRY_SIZE);
+				if (pbestnet_entry == NULL) {
+					err = BCME_NOMEM;
+					DHD_ERROR(("failed to allocate dhd_pno_bestnet_entry\n"));
+					goto exit;
+				}
+				memset(pbestnet_entry, 0, BESTNET_ENTRY_SIZE);
+				/* record the current time */
+				pbestnet_entry->recorded_time = jiffies;
+				/* create header for the first entry */
+				allocate_header = (i == 0)? TRUE : FALSE;
+				/* check whether the new generation is started or not */
+				if (timestamp && (TIME_DIFF(timestamp, plnetinfo->timestamp)
+					> TIME_MIN_DIFF))
+					allocate_header = TRUE;
+				timestamp = plnetinfo->timestamp;
+				if (allocate_header) {
+					pbestnetheader = (dhd_pno_best_header_t *)
+						MALLOC(dhd->osh, BEST_HEADER_SIZE);
+					if (pbestnetheader == NULL) {
+						err = BCME_NOMEM;
+						if (pbestnet_entry)
+							MFREE(dhd->osh, pbestnet_entry,
+								BESTNET_ENTRY_SIZE);
+						DHD_ERROR(("failed to allocate"
+							" dhd_pno_bestnet_entry\n"));
+						goto exit;
+					}
+					/* increase total cnt of bestnet header */
+					pscan_results->cnt_header++;
+					/* need to record the reason to call dhd_pno_get_for_bach */
+					if (reason)
+						pbestnetheader->reason = (ENABLE << reason);
+					memset(pbestnetheader, 0, BEST_HEADER_SIZE);
+					/* initialize the head of linked list */
+					INIT_LIST_HEAD(&(pbestnetheader->entry_list));
+					/* link the pbestnet heaer into existed list */
+					if (pscan_results->bestnetheader == NULL)
+						/* In case of header */
+						pscan_results->bestnetheader = pbestnetheader;
+					else {
+						dhd_pno_best_header_t *head =
+							pscan_results->bestnetheader;
+						pscan_results->bestnetheader = pbestnetheader;
+						pbestnetheader->next = head;
+					}
+				}
+				pbestnet_entry->channel = plnetinfo->pfnsubnet.channel;
+				pbestnet_entry->RSSI = plnetinfo->RSSI;
+				if (plnetinfo->flags & PFN_PARTIAL_SCAN_MASK) {
+					/* if RSSI is positive value, we assume that
+					 * this scan is aborted by other scan
+					 */
+					DHD_PNO(("This scan is aborted\n"));
+					pbestnetheader->reason = (ENABLE << PNO_STATUS_ABORT);
+				}
+				pbestnet_entry->rtt0 = plnetinfo->rtt0;
+				pbestnet_entry->rtt1 = plnetinfo->rtt1;
+				pbestnet_entry->timestamp = plnetinfo->timestamp;
+				if (plnetinfo->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+					DHD_ERROR(("%s: Invalid SSID length"
+						" %d: trimming it to max\n",
+						__FUNCTION__, plnetinfo->pfnsubnet.SSID_len));
+					plnetinfo->pfnsubnet.SSID_len = DOT11_MAX_SSID_LEN;
+				}
+				pbestnet_entry->SSID_len = plnetinfo->pfnsubnet.SSID_len;
+				memcpy(pbestnet_entry->SSID, plnetinfo->pfnsubnet.SSID,
+						pbestnet_entry->SSID_len);
+				memcpy(&pbestnet_entry->BSSID, &plnetinfo->pfnsubnet.BSSID,
+						ETHER_ADDR_LEN);
+				/* add the element into list */
+				list_add_tail(&pbestnet_entry->list, &pbestnetheader->entry_list);
+				/* increase best entry count */
+				pbestnetheader->tot_cnt++;
+				pbestnetheader->tot_size += BESTNET_ENTRY_SIZE;
+				DHD_PNO(("Header %d\n", pscan_results->cnt_header - 1));
+				DHD_PNO(("\tSSID : "));
+				for (j = 0; j < plnetinfo->pfnsubnet.SSID_len; j++)
+					DHD_PNO(("%c", plnetinfo->pfnsubnet.SSID[j]));
+				DHD_PNO(("\n"));
+				DHD_PNO(("\tBSSID: "MACDBG"\n",
+					MAC2STRDBG(plnetinfo->pfnsubnet.BSSID.octet)));
+				DHD_PNO(("\tchannel: %d, RSSI: %d, timestamp: %d ms\n",
+					plnetinfo->pfnsubnet.channel,
+					plnetinfo->RSSI, plnetinfo->timestamp));
+				DHD_PNO(("\tRTT0 : %d, RTT1: %d\n", plnetinfo->rtt0,
+					plnetinfo->rtt1));
+				plnetinfo++;
+			}
+		} else if (plbestnet_v2->version == PFN_LBEST_SCAN_RESULT_VERSION_V2) {
+			fwstatus = plbestnet_v2->status;
+			fwcount = plbestnet_v2->count;
+			plnetinfo_v2 = (wl_pfn_lnet_info_v2_t*)&plbestnet_v2->netinfo[0];
+			if (fwcount == 0) {
+				DHD_PNO(("No more batch results\n"));
+				goto exit;
+			}
+			if (fwcount > BESTN_MAX) {
+				DHD_ERROR(("%s :fwcount %d is greater than BESTN_MAX %d \n",
+					__FUNCTION__, fwcount, (int)BESTN_MAX));
+				/* Process only BESTN_MAX number of results per batch */
+				fwcount = BESTN_MAX;
+			}
+			DHD_PNO(("ver %d, status : %d, count %d\n",
+				plbestnet_v2->version, fwstatus, fwcount));
+
+			for (i = 0; i < fwcount; i++) {
+				pbestnet_entry = (dhd_pno_bestnet_entry_t *)
+					MALLOC(dhd->osh, BESTNET_ENTRY_SIZE);
+				if (pbestnet_entry == NULL) {
+					err = BCME_NOMEM;
+					DHD_ERROR(("failed to allocate dhd_pno_bestnet_entry\n"));
+					goto exit;
+				}
+				memset(pbestnet_entry, 0, BESTNET_ENTRY_SIZE);
+				/* record the current time */
+				pbestnet_entry->recorded_time = jiffies;
+				/* create header for the first entry */
+				allocate_header = (i == 0)? TRUE : FALSE;
+				/* check whether the new generation is started or not */
+				if (timestamp && (TIME_DIFF(timestamp, plnetinfo_v2->timestamp)
+					> TIME_MIN_DIFF))
+					allocate_header = TRUE;
+				timestamp = plnetinfo_v2->timestamp;
+				if (allocate_header) {
+					pbestnetheader = (dhd_pno_best_header_t *)
+						MALLOC(dhd->osh, BEST_HEADER_SIZE);
+					if (pbestnetheader == NULL) {
+						err = BCME_NOMEM;
+						if (pbestnet_entry)
+							MFREE(dhd->osh, pbestnet_entry,
+								BESTNET_ENTRY_SIZE);
+						DHD_ERROR(("failed to allocate"
+							" dhd_pno_bestnet_entry\n"));
+						goto exit;
+					}
+					/* increase total cnt of bestnet header */
+					pscan_results->cnt_header++;
+					/* need to record the reason to call dhd_pno_get_for_bach */
+					if (reason)
+						pbestnetheader->reason = (ENABLE << reason);
+					memset(pbestnetheader, 0, BEST_HEADER_SIZE);
+					/* initialize the head of linked list */
+					INIT_LIST_HEAD(&(pbestnetheader->entry_list));
+					/* link the pbestnet heaer into existed list */
+					if (pscan_results->bestnetheader == NULL)
+						/* In case of header */
+						pscan_results->bestnetheader = pbestnetheader;
+					else {
+						dhd_pno_best_header_t *head =
+							pscan_results->bestnetheader;
+						pscan_results->bestnetheader = pbestnetheader;
+						pbestnetheader->next = head;
+					}
+				}
+				/* fills the best network info */
+				pbestnet_entry->channel = plnetinfo_v2->pfnsubnet.channel;
+				pbestnet_entry->RSSI = plnetinfo_v2->RSSI;
+				if (plnetinfo_v2->flags & PFN_PARTIAL_SCAN_MASK) {
+					/* if RSSI is positive value, we assume that
+					 * this scan is aborted by other scan
+					 */
+					DHD_PNO(("This scan is aborted\n"));
+					pbestnetheader->reason = (ENABLE << PNO_STATUS_ABORT);
+				}
+				pbestnet_entry->rtt0 = plnetinfo_v2->rtt0;
+				pbestnet_entry->rtt1 = plnetinfo_v2->rtt1;
+				pbestnet_entry->timestamp = plnetinfo_v2->timestamp;
+				if (plnetinfo_v2->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+					DHD_ERROR(("%s: Invalid SSID length"
+						" %d: trimming it to max\n",
+						__FUNCTION__, plnetinfo_v2->pfnsubnet.SSID_len));
+					plnetinfo_v2->pfnsubnet.SSID_len = DOT11_MAX_SSID_LEN;
+				}
+				pbestnet_entry->SSID_len = plnetinfo_v2->pfnsubnet.SSID_len;
+				memcpy(pbestnet_entry->SSID, plnetinfo_v2->pfnsubnet.u.SSID,
+					pbestnet_entry->SSID_len);
+				memcpy(&pbestnet_entry->BSSID, &plnetinfo_v2->pfnsubnet.BSSID,
+					ETHER_ADDR_LEN);
+				/* add the element into list */
+				list_add_tail(&pbestnet_entry->list, &pbestnetheader->entry_list);
+				/* increase best entry count */
+				pbestnetheader->tot_cnt++;
+				pbestnetheader->tot_size += BESTNET_ENTRY_SIZE;
+				DHD_PNO(("Header %d\n", pscan_results->cnt_header - 1));
+				DHD_PNO(("\tSSID : "));
+				for (j = 0; j < plnetinfo_v2->pfnsubnet.SSID_len; j++)
+					DHD_PNO(("%c", plnetinfo_v2->pfnsubnet.u.SSID[j]));
+				DHD_PNO(("\n"));
+				DHD_PNO(("\tBSSID: "MACDBG"\n",
+					MAC2STRDBG(plnetinfo_v2->pfnsubnet.BSSID.octet)));
+				DHD_PNO(("\tchannel: %d, RSSI: %d, timestamp: %d ms\n",
+					plnetinfo_v2->pfnsubnet.channel,
+					plnetinfo_v2->RSSI, plnetinfo_v2->timestamp));
+				DHD_PNO(("\tRTT0 : %d, RTT1: %d\n", plnetinfo_v2->rtt0,
+					plnetinfo_v2->rtt1));
+				plnetinfo_v2++;
+			}
+		} else {
+			err = BCME_VERSION;
+			DHD_ERROR(("bestnet fw version %d not supported\n",
+				plbestnet_v1->version));
+			goto exit;
+		}
+	} while (fwstatus != PFN_COMPLETE);
+
+	if (pscan_results->cnt_header == 0) {
+		/* In case that we didn't get any data from the firmware
+		 * Remove the current scan_result list from get_bach.scan_results_list.
+		 */
+		DHD_PNO(("NO BATCH DATA from Firmware, Delete current SCAN RESULT LIST\n"));
+		list_del(&pscan_results->list);
+		MFREE(dhd->osh, pscan_results, SCAN_RESULTS_SIZE);
+		_params->params_batch.get_batch.top_node_cnt--;
+	} else {
+		/* increase total scan count using current scan count */
+		_params->params_batch.get_batch.tot_scan_cnt += pscan_results->cnt_header;
+	}
+
+	if (buf && bufsize) {
+		/* This is a first try to get batching results */
+		if (!list_empty(&_params->params_batch.get_batch.scan_results_list)) {
+			/* move the scan_results_list to expired_scan_results_lists */
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(siter, snext,
+				&_params->params_batch.get_batch.scan_results_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				list_move_tail(&siter->list,
+					&_params->params_batch.get_batch.expired_scan_results_list);
+			}
+			/* reset gloval values after  moving to expired list */
+			_params->params_batch.get_batch.top_node_cnt = 0;
+			_params->params_batch.get_batch.expired_tot_scan_cnt =
+				_params->params_batch.get_batch.tot_scan_cnt;
+			_params->params_batch.get_batch.tot_scan_cnt = 0;
+		}
+convert_format:
+		err = _dhd_pno_convert_format(dhd, &_params->params_batch, buf, bufsize);
+		if (err < 0) {
+			DHD_ERROR(("failed to convert the data into upper layer format\n"));
+			goto exit;
+		}
+	}
+exit:
+	if (plbestnet_v1)
+		MFREE(dhd->osh, plbestnet_v1, PNO_BESTNET_LEN);
+	if (_params) {
+		_params->params_batch.get_batch.buf = NULL;
+		_params->params_batch.get_batch.bufsize = 0;
+		_params->params_batch.get_batch.bytes_written = err;
+	}
+	mutex_unlock(&_pno_state->pno_mutex);
+exit_no_unlock:
+	if (COMPLETION_WAIT_QUEUE_ACTIVE(&_pno_state->get_batch_done.wait))
+		complete(&_pno_state->get_batch_done);
+	return err;
+}
+
+static void
+_dhd_pno_get_batch_handler(struct work_struct *work)
+{
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pub_t *dhd;
+	struct dhd_pno_batch_params *params_batch;
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	_pno_state = container_of(work, struct dhd_pno_status_info, work);
+	GCC_DIAGNOSTIC_POP();
+
+	dhd = _pno_state->dhd;
+	if (dhd == NULL) {
+		DHD_ERROR(("%s : dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+#ifdef GSCAN_SUPPORT
+	_dhd_pno_get_gscan_batch_from_fw(dhd);
+#endif /* GSCAN_SUPPORT */
+	if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+		params_batch = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS].params_batch;
+
+		_dhd_pno_get_for_batch(dhd, params_batch->get_batch.buf,
+			params_batch->get_batch.bufsize, params_batch->get_batch.reason);
+	}
+}
+
+int
+dhd_pno_get_for_batch(dhd_pub_t *dhd, char *buf, int bufsize, int reason)
+{
+	int err = BCME_OK;
+	char *pbuf = buf;
+	dhd_pno_status_info_t *_pno_state;
+	struct dhd_pno_batch_params *params_batch;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+	params_batch = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS].params_batch;
+#ifdef GSCAN_SUPPORT
+	if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+		struct dhd_pno_gscan_params *gscan_params;
+		gscan_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan;
+		gscan_params->reason = reason;
+		err = dhd_retreive_batch_scan_results(dhd);
+		if (err == BCME_OK) {
+			wait_event_interruptible_timeout(_pno_state->batch_get_wait,
+			     is_batch_retrieval_complete(gscan_params),
+			     msecs_to_jiffies(GSCAN_BATCH_GET_MAX_WAIT));
+		}
+	} else
+#endif
+	{
+		if (!(_pno_state->pno_mode & DHD_PNO_BATCH_MODE)) {
+			DHD_ERROR(("%s: Batching SCAN mode is not enabled\n", __FUNCTION__));
+			memset(pbuf, 0, bufsize);
+			pbuf += snprintf(pbuf, bufsize, "scancount=%d\n", 0);
+			snprintf(pbuf, bufsize, "%s", RESULTS_END_MARKER);
+			err = strlen(buf);
+			goto exit;
+		}
+		params_batch->get_batch.buf = buf;
+		params_batch->get_batch.bufsize = bufsize;
+		params_batch->get_batch.reason = reason;
+		params_batch->get_batch.bytes_written = 0;
+		schedule_work(&_pno_state->work);
+		wait_for_completion(&_pno_state->get_batch_done);
+	}
+
+#ifdef GSCAN_SUPPORT
+	if (!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE))
+#endif
+	err = params_batch->get_batch.bytes_written;
+exit:
+	return err;
+}
+
+int
+dhd_pno_stop_for_batch(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	int mode = 0;
+	int i = 0;
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	wl_pfn_bssid_t *p_pfn_bssid = NULL;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n",
+			__FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+#ifdef GSCAN_SUPPORT
+	if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+		DHD_PNO(("Gscan is ongoing, nothing to stop here\n"));
+		return err;
+	}
+#endif
+
+	if (!(_pno_state->pno_mode & DHD_PNO_BATCH_MODE)) {
+		DHD_ERROR(("%s : PNO BATCH MODE is not enabled\n", __FUNCTION__));
+		goto exit;
+	}
+	_pno_state->pno_mode &= ~DHD_PNO_BATCH_MODE;
+	if (_pno_state->pno_mode & (DHD_PNO_LEGACY_MODE | DHD_PNO_HOTLIST_MODE)) {
+		mode = _pno_state->pno_mode;
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+
+		_pno_state->pno_mode = mode;
+		/* restart Legacy PNO if the Legacy PNO is on */
+		if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+			struct dhd_pno_legacy_params *_params_legacy;
+			_params_legacy =
+				&(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS].params_legacy);
+			err = dhd_pno_set_legacy_pno(dhd, _params_legacy->scan_fr,
+				_params_legacy->pno_repeat,
+				_params_legacy->pno_freq_expo_max,
+				_params_legacy->chan_list, _params_legacy->nchan);
+			if (err < 0) {
+				DHD_ERROR(("%s : failed to restart legacy PNO scan(err: %d)\n",
+					__FUNCTION__, err));
+				goto exit;
+			}
+		} else if (_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE) {
+			struct dhd_pno_bssid *iter, *next;
+			_params = &(_pno_state->pno_params_arr[INDEX_OF_HOTLIST_PARAMS]);
+			p_pfn_bssid = (wl_pfn_bssid_t *)MALLOCZ(dhd->osh,
+				sizeof(wl_pfn_bssid_t) * _params->params_hotlist.nbssid);
+			if (p_pfn_bssid == NULL) {
+				DHD_ERROR(("%s : failed to allocate wl_pfn_bssid_t array"
+					" (count: %d)",
+					__FUNCTION__, _params->params_hotlist.nbssid));
+				err = BCME_ERROR;
+				_pno_state->pno_mode &= ~DHD_PNO_HOTLIST_MODE;
+				goto exit;
+			}
+			i = 0;
+			/* convert dhd_pno_bssid to wl_pfn_bssid */
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(iter, next,
+				&_params->params_hotlist.bssid_list, list) {
+				GCC_DIAGNOSTIC_POP();
+				memcpy(&p_pfn_bssid[i].macaddr, &iter->macaddr, ETHER_ADDR_LEN);
+				p_pfn_bssid[i].flags = iter->flags;
+				i++;
+			}
+			err = dhd_pno_set_for_hotlist(dhd, p_pfn_bssid, &_params->params_hotlist);
+			if (err < 0) {
+				_pno_state->pno_mode &= ~DHD_PNO_HOTLIST_MODE;
+				DHD_ERROR(("%s : failed to restart hotlist scan(err: %d)\n",
+					__FUNCTION__, err));
+				goto exit;
+			}
+		}
+	} else {
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+	}
+exit:
+	_params = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS];
+	_dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_BATCH_MODE);
+	MFREE(dhd->osh, p_pfn_bssid,
+		sizeof(wl_pfn_bssid_t) * _params->params_hotlist.nbssid);
+	return err;
+}
+
+int
+dhd_pno_set_for_hotlist(dhd_pub_t *dhd, wl_pfn_bssid_t *p_pfn_bssid,
+	struct dhd_pno_hotlist_params *hotlist_params)
+{
+	int err = BCME_OK;
+	int i;
+	uint16 _chan_list[WL_NUMCHANNELS];
+	int rem_nchan = 0;
+	int tot_nchan = 0;
+	int mode = 0;
+	dhd_pno_params_t *_params;
+	dhd_pno_params_t *_params2;
+	struct dhd_pno_bssid *_pno_bssid;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	NULL_CHECK(hotlist_params, "hotlist_params is NULL", err);
+	NULL_CHECK(p_pfn_bssid, "p_pfn_bssid is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+
+	if (!dhd_support_sta_mode(dhd)) {
+		err = BCME_BADOPTION;
+		goto exit;
+	}
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+	_params = &_pno_state->pno_params_arr[INDEX_OF_HOTLIST_PARAMS];
+	if (!(_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE)) {
+		_pno_state->pno_mode |= DHD_PNO_HOTLIST_MODE;
+		err = _dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_HOTLIST_MODE);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call _dhd_pno_reinitialize_prof\n",
+				__FUNCTION__));
+			goto exit;
+		}
+	}
+	_params->params_batch.nchan = hotlist_params->nchan;
+	_params->params_batch.scan_fr = hotlist_params->scan_fr;
+	if (hotlist_params->nchan)
+		memcpy(_params->params_hotlist.chan_list, hotlist_params->chan_list,
+			sizeof(_params->params_hotlist.chan_list));
+	memset(_chan_list, 0, sizeof(_chan_list));
+
+	rem_nchan = ARRAYSIZE(hotlist_params->chan_list) - hotlist_params->nchan;
+	if (hotlist_params->band == WLC_BAND_2G ||
+#ifdef WL_6G_BAND
+		hotlist_params->band == WLC_BAND_6G ||
+#endif /* WL_6G_BAND */
+		hotlist_params->band == WLC_BAND_5G) {
+		/* get a valid channel list based on band B or A */
+		err = _dhd_pno_get_channels(dhd,
+		&_params->params_hotlist.chan_list[hotlist_params->nchan],
+		&rem_nchan, hotlist_params->band, FALSE);
+		if (err < 0) {
+			DHD_ERROR(("%s: failed to get valid channel list(band : %d)\n",
+				__FUNCTION__, hotlist_params->band));
+			goto exit;
+		}
+		/* now we need to update nchan because rem_chan has valid channel count */
+		_params->params_hotlist.nchan += rem_nchan;
+		/* need to sort channel list */
+		sort(_params->params_hotlist.chan_list, _params->params_hotlist.nchan,
+			sizeof(_params->params_hotlist.chan_list[0]), _dhd_pno_cmpfunc, NULL);
+	}
+#ifdef PNO_DEBUG
+	{
+		int i;
+		DHD_PNO(("Channel list : "));
+		for (i = 0; i < _params->params_batch.nchan; i++) {
+			DHD_PNO(("%d ", _params->params_batch.chan_list[i]));
+		}
+		DHD_PNO(("\n"));
+	}
+#endif
+	if (_params->params_hotlist.nchan) {
+		/* copy the channel list into local array */
+		memcpy(_chan_list, _params->params_hotlist.chan_list,
+			sizeof(_chan_list));
+		tot_nchan = _params->params_hotlist.nchan;
+	}
+	if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+			DHD_PNO(("PNO SSID is on progress in firmware\n"));
+			/* store current pno_mode before disabling pno */
+			mode = _pno_state->pno_mode;
+			err = _dhd_pno_enable(dhd, PNO_OFF);
+			if (err < 0) {
+				DHD_ERROR(("%s : failed to disable PNO\n", __FUNCTION__));
+				goto exit;
+			}
+			/* restore the previous mode */
+			_pno_state->pno_mode = mode;
+			/* Use the superset for channelist between two mode */
+			_params2 = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]);
+			if (_params2->params_legacy.nchan > 0 &&
+				_params->params_hotlist.nchan > 0) {
+				err = _dhd_pno_chan_merge(_chan_list, &tot_nchan,
+					&_params2->params_legacy.chan_list[0],
+					_params2->params_legacy.nchan,
+					&_params->params_hotlist.chan_list[0],
+					_params->params_hotlist.nchan);
+				if (err < 0) {
+					DHD_ERROR(("%s : failed to merge channel list"
+						"between legacy and hotlist\n",
+						__FUNCTION__));
+					goto exit;
+				}
+			}
+
+	}
+
+	INIT_LIST_HEAD(&(_params->params_hotlist.bssid_list));
+
+	err = _dhd_pno_add_bssid(dhd, p_pfn_bssid, hotlist_params->nbssid);
+	if (err < 0) {
+		DHD_ERROR(("%s : failed to call _dhd_pno_add_bssid(err :%d)\n",
+			__FUNCTION__, err));
+		goto exit;
+	}
+	if ((err = _dhd_pno_set(dhd, _params, DHD_PNO_HOTLIST_MODE)) < 0) {
+		DHD_ERROR(("%s : failed to set call pno_set (err %d) in firmware\n",
+			__FUNCTION__, err));
+		goto exit;
+	}
+	if (tot_nchan > 0) {
+		if ((err = _dhd_pno_cfg(dhd, _chan_list, tot_nchan)) < 0) {
+			DHD_ERROR(("%s : failed to set call pno_cfg (err %d) in firmware\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+	}
+	for (i = 0; i < hotlist_params->nbssid; i++) {
+		_pno_bssid = (struct dhd_pno_bssid *)MALLOCZ(dhd->osh,
+			sizeof(struct dhd_pno_bssid));
+		NULL_CHECK(_pno_bssid, "_pfn_bssid is NULL", err);
+		memcpy(&_pno_bssid->macaddr, &p_pfn_bssid[i].macaddr, ETHER_ADDR_LEN);
+		_pno_bssid->flags = p_pfn_bssid[i].flags;
+		list_add_tail(&_pno_bssid->list, &_params->params_hotlist.bssid_list);
+	}
+	_params->params_hotlist.nbssid = hotlist_params->nbssid;
+	if (_pno_state->pno_status == DHD_PNO_DISABLED) {
+		if ((err = _dhd_pno_enable(dhd, PNO_ON)) < 0)
+			DHD_ERROR(("%s : failed to enable PNO\n", __FUNCTION__));
+	}
+exit:
+	/* clear mode in case of error */
+	if (err < 0)
+		_pno_state->pno_mode &= ~DHD_PNO_HOTLIST_MODE;
+	return err;
+}
+
+int
+dhd_pno_stop_for_hotlist(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	uint32 mode = 0;
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n",
+			__FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	if (!(_pno_state->pno_mode & DHD_PNO_HOTLIST_MODE)) {
+		DHD_ERROR(("%s : Hotlist MODE is not enabled\n",
+			__FUNCTION__));
+		goto exit;
+	}
+	_pno_state->pno_mode &= ~DHD_PNO_BATCH_MODE;
+
+	if (_pno_state->pno_mode & (DHD_PNO_LEGACY_MODE | DHD_PNO_BATCH_MODE)) {
+		/* retrieve the batching data from firmware into host */
+		dhd_pno_get_for_batch(dhd, NULL, 0, PNO_STATUS_DISABLE);
+		/* save current pno_mode before calling dhd_pno_clean */
+		mode = _pno_state->pno_mode;
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+		/* restore previos pno mode */
+		_pno_state->pno_mode = mode;
+		if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+			/* restart Legacy PNO Scan */
+			struct dhd_pno_legacy_params *_params_legacy;
+			_params_legacy =
+			&(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS].params_legacy);
+			err = dhd_pno_set_legacy_pno(dhd, _params_legacy->scan_fr,
+				_params_legacy->pno_repeat, _params_legacy->pno_freq_expo_max,
+				_params_legacy->chan_list, _params_legacy->nchan);
+			if (err < 0) {
+				DHD_ERROR(("%s : failed to restart legacy PNO scan(err: %d)\n",
+					__FUNCTION__, err));
+				goto exit;
+			}
+		} else if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+			/* restart Batching Scan */
+			_params = &(_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS]);
+			/* restart BATCH SCAN */
+			err = dhd_pno_set_for_batch(dhd, &_params->params_batch);
+			if (err < 0) {
+				_pno_state->pno_mode &= ~DHD_PNO_BATCH_MODE;
+				DHD_ERROR(("%s : failed to restart batch scan(err: %d)\n",
+					__FUNCTION__,  err));
+				goto exit;
+			}
+		}
+	} else {
+		err = dhd_pno_clean(dhd);
+		if (err < 0) {
+			DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n",
+				__FUNCTION__, err));
+			goto exit;
+		}
+	}
+exit:
+	return err;
+}
+
+#ifdef GSCAN_SUPPORT
+int
+dhd_retreive_batch_scan_results(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	struct dhd_pno_batch_params *params_batch;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+
+	params_batch = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS].params_batch;
+	if (_params->params_gscan.get_batch_flag == GSCAN_BATCH_RETRIEVAL_COMPLETE) {
+		DHD_PNO(("Retreive batch results\n"));
+		params_batch->get_batch.buf = NULL;
+		params_batch->get_batch.bufsize = 0;
+		params_batch->get_batch.reason = PNO_STATUS_EVENT;
+		_params->params_gscan.get_batch_flag = GSCAN_BATCH_RETRIEVAL_IN_PROGRESS;
+		smp_wmb();
+		schedule_work(&_pno_state->work);
+	} else {
+		DHD_PNO(("%s : WLC_E_PFN_BEST_BATCHING retrieval"
+			"already in progress, will skip\n", __FUNCTION__));
+		err = BCME_ERROR;
+	}
+
+	return err;
+}
+
+void
+dhd_gscan_hotlist_cache_cleanup(dhd_pub_t *dhd, hotlist_type_t type)
+{
+	dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd);
+	struct dhd_pno_gscan_params *gscan_params;
+	gscan_results_cache_t *iter, *tmp;
+
+	if (!_pno_state) {
+		return;
+	}
+	gscan_params = &(_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan);
+
+	if (type == HOTLIST_FOUND) {
+		iter = gscan_params->gscan_hotlist_found;
+		gscan_params->gscan_hotlist_found = NULL;
+	} else {
+		iter = gscan_params->gscan_hotlist_lost;
+		gscan_params->gscan_hotlist_lost = NULL;
+	}
+
+	while (iter) {
+		tmp = iter->next;
+		MFREE(dhd->osh, iter,
+				((iter->tot_count - 1) * sizeof(wifi_gscan_result_t))
+				+ sizeof(gscan_results_cache_t));
+		iter = tmp;
+	}
+
+	return;
+}
+
+void *
+dhd_process_full_gscan_result(dhd_pub_t *dhd, const void *data, uint32 len, int *size)
+{
+	wl_bss_info_v109_t *bi = NULL;
+	wl_gscan_result_v2_t *gscan_result;
+	wifi_gscan_full_result_t *result = NULL;
+	u32 bi_length = 0;
+	uint8 channel;
+	uint32 mem_needed;
+	struct osl_timespec ts;
+	u32 bi_ie_length = 0;
+	u32 bi_ie_offset = 0;
+
+	*size = 0;
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	gscan_result = (wl_gscan_result_v2_t *)data;
+	GCC_DIAGNOSTIC_POP();
+	if (!gscan_result) {
+		DHD_ERROR(("Invalid gscan result (NULL pointer)\n"));
+		goto exit;
+	}
+
+	if ((len < sizeof(*gscan_result)) ||
+	    (len < dtoh32(gscan_result->buflen)) ||
+	    (dtoh32(gscan_result->buflen) >
+	    (sizeof(*gscan_result) + WL_SCAN_IE_LEN_MAX))) {
+		DHD_ERROR(("%s: invalid gscan buflen:%u\n", __FUNCTION__,
+			dtoh32(gscan_result->buflen)));
+		goto exit;
+	}
+
+	bi = &gscan_result->bss_info[0].info;
+	bi_length = dtoh32(bi->length);
+	if (bi_length != (dtoh32(gscan_result->buflen) -
+	       OFFSETOF(wl_gscan_result_v2_t, bss_info) -
+	       OFFSETOF(wl_gscan_bss_info_v2_t, info))) {
+		DHD_ERROR(("Invalid bss_info length %d: ignoring\n", bi_length));
+		goto exit;
+	}
+	bi_ie_offset = dtoh32(bi->ie_offset);
+	bi_ie_length = dtoh32(bi->ie_length);
+	if ((bi_ie_offset + bi_ie_length) > bi_length) {
+		DHD_ERROR(("%s: Invalid ie_length:%u or ie_offset:%u\n",
+			__FUNCTION__, bi_ie_length, bi_ie_offset));
+		goto exit;
+	}
+	if (bi->SSID_len > DOT11_MAX_SSID_LEN) {
+		DHD_ERROR(("%s: Invalid SSID length:%u\n", __FUNCTION__, bi->SSID_len));
+		goto exit;
+	}
+
+	mem_needed = OFFSETOF(wifi_gscan_full_result_t, ie_data) + bi->ie_length;
+	result = (wifi_gscan_full_result_t *)MALLOC(dhd->osh, mem_needed);
+	if (!result) {
+		DHD_ERROR(("%s Cannot malloc scan result buffer %d bytes\n",
+		  __FUNCTION__, mem_needed));
+		goto exit;
+	}
+
+	result->scan_ch_bucket = gscan_result->scan_ch_bucket;
+	memcpy(result->fixed.ssid, bi->SSID, bi->SSID_len);
+	result->fixed.ssid[bi->SSID_len] = '\0';
+	channel = wf_chspec_ctlchspec(bi->chanspec);
+	result->fixed.channel = wl_channel_to_frequency(channel, CHSPEC_BAND(channel));
+	result->fixed.rssi = (int32) bi->RSSI;
+	result->fixed.rtt = 0;
+	result->fixed.rtt_sd = 0;
+	osl_get_monotonic_boottime(&ts);
+	result->fixed.ts = (uint64) TIMESPEC64_TO_US(ts);
+	result->fixed.beacon_period = dtoh16(bi->beacon_period);
+	result->fixed.capability = dtoh16(bi->capability);
+	result->ie_length = bi_ie_length;
+	memcpy(&result->fixed.macaddr, &bi->BSSID, ETHER_ADDR_LEN);
+	memcpy(result->ie_data, ((uint8 *)bi + bi_ie_offset), bi_ie_length);
+	*size = mem_needed;
+exit:
+	return result;
+}
+
+static void *
+dhd_pno_update_pfn_v3_results(dhd_pub_t *dhd, wl_pfn_scanresults_v3_t *pfn_result,
+	uint32 *mem_needed, struct dhd_pno_gscan_params *gscan_params, uint32 event)
+{
+	uint32 i;
+	uint8 ssid[DOT11_MAX_SSID_LEN + 1];
+	struct ether_addr *bssid;
+	wl_pfn_net_info_v3_t *net_info = NULL;
+	dhd_epno_results_t *results = NULL;
+
+	if ((pfn_result->count == 0) || (pfn_result->count > EVENT_MAX_NETCNT_V3)) {
+		DHD_ERROR(("%s event %d: wrong pfn v3 results count %d\n",
+				__FUNCTION__, event, pfn_result->count));
+		return NULL;
+	}
+
+	*mem_needed = sizeof(dhd_epno_results_t) * pfn_result->count;
+	results = (dhd_epno_results_t *)MALLOC(dhd->osh, (*mem_needed));
+	if (!results) {
+		DHD_ERROR(("%s: Can't malloc %d bytes for results\n", __FUNCTION__,
+			*mem_needed));
+		return NULL;
+	}
+	for (i = 0; i < pfn_result->count; i++) {
+		net_info = &pfn_result->netinfo[i];
+		results[i].rssi = net_info->RSSI;
+		results[i].channel =  wl_channel_to_frequency(
+			CHSPEC_CHANNEL(net_info->pfnsubnet.chanspec),
+			CHSPEC_BAND(net_info->pfnsubnet.chanspec));
+		results[i].flags = (event == WLC_E_PFN_NET_FOUND) ?
+			WL_PFN_SSID_EXT_FOUND: WL_PFN_SSID_EXT_LOST;
+		results[i].ssid_len = min(net_info->pfnsubnet.SSID_len,
+			(uint8)DOT11_MAX_SSID_LEN);
+		bssid = &results[i].bssid;
+		(void)memcpy_s(bssid, ETHER_ADDR_LEN,
+			&net_info->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+		if (!net_info->pfnsubnet.SSID_len) {
+			dhd_pno_idx_to_ssid(gscan_params, &results[i],
+				net_info->pfnsubnet.u.index);
+		} else {
+			(void)memcpy_s(results[i].ssid,	DOT11_MAX_SSID_LEN,
+				net_info->pfnsubnet.u.SSID, results[i].ssid_len);
+		}
+		(void)memcpy_s(ssid, DOT11_MAX_SSID_LEN, results[i].ssid, results[i].ssid_len);
+		ssid[results[i].ssid_len] = '\0';
+		DHD_PNO(("ssid - %s bssid "MACDBG" ch %d rssi %d flags %d\n",
+			ssid, MAC2STRDBG(bssid->octet),	results[i].channel,
+			results[i].rssi, results[i].flags));
+	}
+
+	return results;
+}
+
+void *
+dhd_pno_process_epno_result(dhd_pub_t *dhd, const void *data, uint32 event, int *size)
+{
+	dhd_epno_results_t *results = NULL;
+	dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd);
+	struct dhd_pno_gscan_params *gscan_params;
+	uint32 count, mem_needed = 0, i;
+	uint8 ssid[DOT11_MAX_SSID_LEN + 1];
+	struct ether_addr *bssid;
+
+	*size = 0;
+	if (!_pno_state)
+		return NULL;
+	gscan_params = &(_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan);
+
+	if (event == WLC_E_PFN_NET_FOUND || event == WLC_E_PFN_NET_LOST) {
+		wl_pfn_scanresults_v1_t *pfn_result = (wl_pfn_scanresults_v1_t *)data;
+		wl_pfn_scanresults_v2_t *pfn_result_v2 = (wl_pfn_scanresults_v2_t *)data;
+		wl_pfn_scanresults_v3_t *pfn_result_v3 = (wl_pfn_scanresults_v3_t *)data;
+		wl_pfn_net_info_v1_t *net;
+		wl_pfn_net_info_v2_t *net_v2;
+
+		if (pfn_result->version == PFN_SCANRESULT_VERSION_V1) {
+			if ((pfn_result->count == 0) || (pfn_result->count > EVENT_MAX_NETCNT_V1)) {
+				DHD_ERROR(("%s event %d: wrong pfn v1 results count %d\n",
+						__FUNCTION__, event, pfn_result->count));
+				return NULL;
+			}
+			count = pfn_result->count;
+			mem_needed = sizeof(dhd_epno_results_t) * count;
+			results = (dhd_epno_results_t *)MALLOC(dhd->osh, mem_needed);
+			if (!results) {
+				DHD_ERROR(("%s: Can't malloc %d bytes for results\n", __FUNCTION__,
+					mem_needed));
+				return NULL;
+			}
+			for (i = 0; i < count; i++) {
+				net = &pfn_result->netinfo[i];
+				results[i].rssi = net->RSSI;
+				results[i].channel =  wf_channel2mhz(net->pfnsubnet.channel,
+					(net->pfnsubnet.channel <= CH_MAX_2G_CHANNEL ?
+					WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G));
+				results[i].flags = (event == WLC_E_PFN_NET_FOUND) ?
+					WL_PFN_SSID_EXT_FOUND: WL_PFN_SSID_EXT_LOST;
+				results[i].ssid_len = min(net->pfnsubnet.SSID_len,
+					(uint8)DOT11_MAX_SSID_LEN);
+				bssid = &results[i].bssid;
+				(void)memcpy_s(bssid, ETHER_ADDR_LEN,
+					&net->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+				if (!net->pfnsubnet.SSID_len) {
+					DHD_ERROR(("%s: Gscan results indexing is not"
+						" supported in version 1 \n", __FUNCTION__));
+					MFREE(dhd->osh, results, mem_needed);
+					return NULL;
+				} else {
+					(void)memcpy_s(results[i].ssid,	DOT11_MAX_SSID_LEN,
+						net->pfnsubnet.SSID, results[i].ssid_len);
+				}
+				(void)memcpy_s(ssid, DOT11_MAX_SSID_LEN,
+					results[i].ssid, results[i].ssid_len);
+				ssid[results[i].ssid_len] = '\0';
+				DHD_PNO(("ssid - %s bssid "MACDBG" ch %d rssi %d flags %d\n",
+					ssid, MAC2STRDBG(bssid->octet), results[i].channel,
+					results[i].rssi, results[i].flags));
+			}
+		} else if (pfn_result_v2->version == PFN_SCANRESULT_VERSION_V2) {
+			if ((pfn_result->count == 0) || (pfn_result->count > EVENT_MAX_NETCNT_V2)) {
+				DHD_ERROR(("%s event %d: wrong pfn v2 results count %d\n",
+						__FUNCTION__, event, pfn_result->count));
+				return NULL;
+			}
+			count = pfn_result_v2->count;
+			mem_needed = sizeof(dhd_epno_results_t) * count;
+			results = (dhd_epno_results_t *)MALLOC(dhd->osh, mem_needed);
+			if (!results) {
+				DHD_ERROR(("%s: Can't malloc %d bytes for results\n", __FUNCTION__,
+					mem_needed));
+				return NULL;
+			}
+			for (i = 0; i < count; i++) {
+				net_v2 = &pfn_result_v2->netinfo[i];
+				results[i].rssi = net_v2->RSSI;
+				results[i].channel =  wf_channel2mhz(net_v2->pfnsubnet.channel,
+					(net_v2->pfnsubnet.channel <= CH_MAX_2G_CHANNEL ?
+				WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G));
+				results[i].flags = (event == WLC_E_PFN_NET_FOUND) ?
+					WL_PFN_SSID_EXT_FOUND: WL_PFN_SSID_EXT_LOST;
+				results[i].ssid_len = min(net_v2->pfnsubnet.SSID_len,
+					(uint8)DOT11_MAX_SSID_LEN);
+				bssid = &results[i].bssid;
+				(void)memcpy_s(bssid, ETHER_ADDR_LEN,
+					&net_v2->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+				if (!net_v2->pfnsubnet.SSID_len) {
+					dhd_pno_idx_to_ssid(gscan_params, &results[i],
+						net_v2->pfnsubnet.u.index);
+				} else {
+					(void)memcpy_s(results[i].ssid,	DOT11_MAX_SSID_LEN,
+						net_v2->pfnsubnet.u.SSID, results[i].ssid_len);
+				}
+				(void)memcpy_s(ssid, DOT11_MAX_SSID_LEN,
+					results[i].ssid, results[i].ssid_len);
+				ssid[results[i].ssid_len] = '\0';
+				DHD_PNO(("ssid - %s bssid "MACDBG" ch %d rssi %d flags %d\n",
+					ssid, MAC2STRDBG(bssid->octet),	results[i].channel,
+					results[i].rssi, results[i].flags));
+			}
+		} else if (pfn_result_v3->version == PFN_SCANRESULT_VERSION_V3) {
+			results = dhd_pno_update_pfn_v3_results(dhd, pfn_result_v3, &mem_needed,
+				gscan_params, event);
+			if (results == NULL) {
+				return results;
+			}
+		} else {
+			DHD_ERROR(("%s event %d: Incorrect version %d , not supported\n",
+				__FUNCTION__, event, pfn_result->version));
+			return NULL;
+		}
+	}
+	*size = mem_needed;
+	return results;
+}
+
+static void *
+dhd_pno_update_hotlist_v3_results(dhd_pub_t *dhd, wl_pfn_scanresults_v3_t *pfn_result,
+	int *send_evt_bytes, hotlist_type_t type,  u32 *buf_len)
+{
+	u32 malloc_size = 0, i;
+	struct osl_timespec tm_spec;
+	struct dhd_pno_gscan_params *gscan_params;
+	gscan_results_cache_t *gscan_hotlist_cache;
+	wifi_gscan_result_t *hotlist_found_array;
+	dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd);
+	wl_pfn_net_info_v3_t *pnetinfo = (wl_pfn_net_info_v3_t*)&pfn_result->netinfo[0];
+
+	gscan_params = &(_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan);
+
+	if (!pfn_result->count || (pfn_result->count > EVENT_MAX_NETCNT_V3)) {
+		DHD_ERROR(("%s: wrong v3 fwcount:%d\n", __FUNCTION__, pfn_result->count));
+		*send_evt_bytes = 0;
+		return NULL;
+	}
+
+	osl_get_monotonic_boottime(&tm_spec);
+	malloc_size = sizeof(gscan_results_cache_t) +
+		((pfn_result->count - 1) * sizeof(wifi_gscan_result_t));
+	gscan_hotlist_cache =
+		(gscan_results_cache_t *)MALLOC(dhd->osh, malloc_size);
+	if (!gscan_hotlist_cache) {
+		DHD_ERROR(("%s Cannot Malloc %d bytes!!\n", __FUNCTION__, malloc_size));
+		*send_evt_bytes = 0;
+		return NULL;
+	}
+	*buf_len = malloc_size;
+	if (type == HOTLIST_FOUND) {
+		gscan_hotlist_cache->next = gscan_params->gscan_hotlist_found;
+		gscan_params->gscan_hotlist_found = gscan_hotlist_cache;
+		DHD_PNO(("%s enter, FOUND results count %d\n", __FUNCTION__, pfn_result->count));
+	} else {
+		gscan_hotlist_cache->next = gscan_params->gscan_hotlist_lost;
+		gscan_params->gscan_hotlist_lost = gscan_hotlist_cache;
+		DHD_PNO(("%s enter, LOST results count %d\n", __FUNCTION__, pfn_result->count));
+	}
+
+	gscan_hotlist_cache->tot_count = pfn_result->count;
+	gscan_hotlist_cache->tot_consumed = 0;
+	gscan_hotlist_cache->scan_ch_bucket = pfn_result->scan_ch_bucket;
+
+	for (i = 0; i < pfn_result->count; i++, pnetinfo++) {
+		hotlist_found_array = &gscan_hotlist_cache->results[i];
+		(void)memset_s(hotlist_found_array, sizeof(wifi_gscan_result_t),
+				0, sizeof(wifi_gscan_result_t));
+		hotlist_found_array->channel = wl_channel_to_frequency(
+			CHSPEC_CHANNEL(pnetinfo->pfnsubnet.chanspec),
+			CHSPEC_BAND(pnetinfo->pfnsubnet.chanspec));
+		hotlist_found_array->rssi = (int32) pnetinfo->RSSI;
+
+		hotlist_found_array->ts =
+			convert_fw_rel_time_to_systime(&tm_spec,
+			(pnetinfo->timestamp * 1000));
+		if (pnetinfo->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+			DHD_ERROR(("Invalid SSID length %d: trimming it to max\n",
+				pnetinfo->pfnsubnet.SSID_len));
+			pnetinfo->pfnsubnet.SSID_len = DOT11_MAX_SSID_LEN;
+		}
+		(void)memcpy_s(hotlist_found_array->ssid, DOT11_MAX_SSID_LEN,
+			pnetinfo->pfnsubnet.u.SSID, pnetinfo->pfnsubnet.SSID_len);
+		hotlist_found_array->ssid[pnetinfo->pfnsubnet.SSID_len] = '\0';
+
+		(void)memcpy_s(&hotlist_found_array->macaddr, ETHER_ADDR_LEN,
+			&pnetinfo->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+		DHD_PNO(("\t%s "MACDBG" rssi %d\n",
+			hotlist_found_array->ssid,
+			MAC2STRDBG(hotlist_found_array->macaddr.octet),
+			hotlist_found_array->rssi));
+	}
+
+	return gscan_hotlist_cache;
+}
+
+void *
+dhd_handle_hotlist_scan_evt(dhd_pub_t *dhd, const void *event_data,
+        int *send_evt_bytes, hotlist_type_t type, u32 *buf_len)
+{
+	void *ptr = NULL;
+	dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd);
+	struct dhd_pno_gscan_params *gscan_params;
+	wl_pfn_scanresults_v1_t *results_v1 = (wl_pfn_scanresults_v1_t *)event_data;
+	wl_pfn_scanresults_v2_t *results_v2 = (wl_pfn_scanresults_v2_t *)event_data;
+	wl_pfn_scanresults_v3_t *results_v3 = (wl_pfn_scanresults_v3_t *)event_data;
+	wifi_gscan_result_t *hotlist_found_array;
+	wl_pfn_net_info_v1_t *pnetinfo;
+	wl_pfn_net_info_v2_t *pnetinfo_v2;
+	gscan_results_cache_t *gscan_hotlist_cache;
+	u32 malloc_size = 0, i, total = 0;
+	struct osl_timespec tm_spec;
+	uint16 fwstatus;
+	uint16 fwcount;
+
+	/* Static asserts in _dhd_pno_get_for_batch() above guarantee the v1 and v2
+	 * net_info and subnet_info structures are compatible in size and SSID offset,
+	 * allowing v1 to be safely used in the code below except for lscanresults
+	 * fields themselves (status, count, offset to netinfo).
+	 */
+
+	*buf_len = 0;
+	if (results_v1->version == PFN_SCANRESULTS_VERSION_V1) {
+		fwstatus = results_v1->status;
+		fwcount = results_v1->count;
+		pnetinfo = &results_v1->netinfo[0];
+
+		gscan_params = &(_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan);
+
+		if (!fwcount || (fwcount > EVENT_MAX_NETCNT_V1)) {
+			DHD_ERROR(("%s: wrong v1 fwcount:%d\n", __FUNCTION__, fwcount));
+			*send_evt_bytes = 0;
+			return ptr;
+		}
+
+		osl_get_monotonic_boottime(&tm_spec);
+		malloc_size = sizeof(gscan_results_cache_t) +
+			((fwcount - 1) * sizeof(wifi_gscan_result_t));
+		gscan_hotlist_cache = (gscan_results_cache_t *)MALLOC(dhd->osh, malloc_size);
+		if (!gscan_hotlist_cache) {
+			DHD_ERROR(("%s Cannot Malloc %d bytes!!\n", __FUNCTION__, malloc_size));
+			*send_evt_bytes = 0;
+			return ptr;
+		}
+
+		*buf_len = malloc_size;
+		if (type == HOTLIST_FOUND) {
+			gscan_hotlist_cache->next = gscan_params->gscan_hotlist_found;
+			gscan_params->gscan_hotlist_found = gscan_hotlist_cache;
+			DHD_PNO(("%s enter, FOUND results count %d\n", __FUNCTION__, fwcount));
+		} else {
+			gscan_hotlist_cache->next = gscan_params->gscan_hotlist_lost;
+			gscan_params->gscan_hotlist_lost = gscan_hotlist_cache;
+			DHD_PNO(("%s enter, LOST results count %d\n", __FUNCTION__, fwcount));
+		}
+
+		gscan_hotlist_cache->tot_count = fwcount;
+		gscan_hotlist_cache->tot_consumed = 0;
+
+		for (i = 0; i < fwcount; i++, pnetinfo++) {
+			hotlist_found_array = &gscan_hotlist_cache->results[i];
+			memset(hotlist_found_array, 0, sizeof(wifi_gscan_result_t));
+			hotlist_found_array->channel = wf_channel2mhz(pnetinfo->pfnsubnet.channel,
+				(pnetinfo->pfnsubnet.channel <= CH_MAX_2G_CHANNEL?
+				WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G));
+			hotlist_found_array->rssi = (int32) pnetinfo->RSSI;
+
+			hotlist_found_array->ts =
+				convert_fw_rel_time_to_systime(&tm_spec,
+					(pnetinfo->timestamp * 1000));
+			if (pnetinfo->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+				DHD_ERROR(("Invalid SSID length %d: trimming it to max\n",
+					pnetinfo->pfnsubnet.SSID_len));
+				pnetinfo->pfnsubnet.SSID_len = DOT11_MAX_SSID_LEN;
+			}
+			(void)memcpy_s(hotlist_found_array->ssid, DOT11_MAX_SSID_LEN,
+				pnetinfo->pfnsubnet.SSID, pnetinfo->pfnsubnet.SSID_len);
+			hotlist_found_array->ssid[pnetinfo->pfnsubnet.SSID_len] = '\0';
+
+			(void)memcpy_s(&hotlist_found_array->macaddr, ETHER_ADDR_LEN,
+				&pnetinfo->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+			DHD_PNO(("\t%s "MACDBG" rssi %d\n",
+				hotlist_found_array->ssid,
+				MAC2STRDBG(hotlist_found_array->macaddr.octet),
+				hotlist_found_array->rssi));
+		}
+	} else if (results_v2->version == PFN_SCANRESULTS_VERSION_V2) {
+		fwstatus = results_v2->status;
+		fwcount = results_v2->count;
+		pnetinfo_v2 = (wl_pfn_net_info_v2_t*)&results_v2->netinfo[0];
+
+		gscan_params = &(_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan);
+
+		if (!fwcount || (fwcount > EVENT_MAX_NETCNT_V2)) {
+			DHD_ERROR(("%s: wrong v2 fwcount:%d\n", __FUNCTION__, fwcount));
+			*send_evt_bytes = 0;
+			return ptr;
+		}
+
+		osl_get_monotonic_boottime(&tm_spec);
+		malloc_size = sizeof(gscan_results_cache_t) +
+			((fwcount - 1) * sizeof(wifi_gscan_result_t));
+		gscan_hotlist_cache =
+			(gscan_results_cache_t *)MALLOC(dhd->osh, malloc_size);
+		if (!gscan_hotlist_cache) {
+			DHD_ERROR(("%s Cannot Malloc %d bytes!!\n", __FUNCTION__, malloc_size));
+			*send_evt_bytes = 0;
+			return ptr;
+		}
+		*buf_len = malloc_size;
+		if (type == HOTLIST_FOUND) {
+			gscan_hotlist_cache->next = gscan_params->gscan_hotlist_found;
+			gscan_params->gscan_hotlist_found = gscan_hotlist_cache;
+			DHD_PNO(("%s enter, FOUND results count %d\n", __FUNCTION__, fwcount));
+		} else {
+			gscan_hotlist_cache->next = gscan_params->gscan_hotlist_lost;
+			gscan_params->gscan_hotlist_lost = gscan_hotlist_cache;
+			DHD_PNO(("%s enter, LOST results count %d\n", __FUNCTION__, fwcount));
+		}
+
+		gscan_hotlist_cache->tot_count = fwcount;
+		gscan_hotlist_cache->tot_consumed = 0;
+		gscan_hotlist_cache->scan_ch_bucket = results_v2->scan_ch_bucket;
+
+		for (i = 0; i < fwcount; i++, pnetinfo_v2++) {
+			hotlist_found_array = &gscan_hotlist_cache->results[i];
+			memset(hotlist_found_array, 0, sizeof(wifi_gscan_result_t));
+			hotlist_found_array->channel =
+				wf_channel2mhz(pnetinfo_v2->pfnsubnet.channel,
+				(pnetinfo_v2->pfnsubnet.channel <= CH_MAX_2G_CHANNEL?
+				WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G));
+			hotlist_found_array->rssi = (int32) pnetinfo_v2->RSSI;
+
+			hotlist_found_array->ts =
+				convert_fw_rel_time_to_systime(&tm_spec,
+				(pnetinfo_v2->timestamp * 1000));
+			if (pnetinfo_v2->pfnsubnet.SSID_len > DOT11_MAX_SSID_LEN) {
+				DHD_ERROR(("Invalid SSID length %d: trimming it to max\n",
+					pnetinfo_v2->pfnsubnet.SSID_len));
+				pnetinfo_v2->pfnsubnet.SSID_len = DOT11_MAX_SSID_LEN;
+			}
+			(void)memcpy_s(hotlist_found_array->ssid, DOT11_MAX_SSID_LEN,
+				pnetinfo_v2->pfnsubnet.u.SSID, pnetinfo_v2->pfnsubnet.SSID_len);
+			hotlist_found_array->ssid[pnetinfo_v2->pfnsubnet.SSID_len] = '\0';
+
+			(void)memcpy_s(&hotlist_found_array->macaddr, ETHER_ADDR_LEN,
+				&pnetinfo_v2->pfnsubnet.BSSID, ETHER_ADDR_LEN);
+			DHD_PNO(("\t%s "MACDBG" rssi %d\n",
+				hotlist_found_array->ssid,
+				MAC2STRDBG(hotlist_found_array->macaddr.octet),
+				hotlist_found_array->rssi));
+		}
+	} else if (results_v3->version == PFN_SCANRESULTS_VERSION_V3) {
+		fwstatus = results_v3->status;
+		gscan_hotlist_cache = (gscan_results_cache_t *)dhd_pno_update_hotlist_v3_results(
+			dhd, results_v3, send_evt_bytes, type, buf_len);
+	} else {
+		DHD_ERROR(("%s: event version %d not supported\n",
+			__FUNCTION__, results_v1->version));
+		*send_evt_bytes = 0;
+		return ptr;
+	}
+	if (fwstatus == PFN_COMPLETE) {
+		ptr = (void *) gscan_hotlist_cache;
+		while (gscan_hotlist_cache) {
+			total += gscan_hotlist_cache->tot_count;
+			gscan_hotlist_cache = gscan_hotlist_cache->next;
+		}
+		*send_evt_bytes =  total * sizeof(wifi_gscan_result_t);
+	}
+
+	return ptr;
+}
+#endif /* GSCAN_SUPPORT */
+
+int
+dhd_pno_event_handler(dhd_pub_t *dhd, wl_event_msg_t *event, void *event_data)
+{
+	int err = BCME_OK;
+	uint event_type;
+	dhd_pno_status_info_t *_pno_state;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(dhd->pno_state, "pno_state is NULL", err);
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	if (!WLS_SUPPORTED(_pno_state)) {
+		DHD_ERROR(("%s : wifi location service is not supported\n", __FUNCTION__));
+		err = BCME_UNSUPPORTED;
+		goto exit;
+	}
+	event_type = ntoh32(event->event_type);
+	DHD_PNO(("%s enter : event_type :%d\n", __FUNCTION__, event_type));
+	switch (event_type) {
+	case WLC_E_PFN_BSSID_NET_FOUND:
+	case WLC_E_PFN_BSSID_NET_LOST:
+		/* how can we inform this to framework ? */
+		/* TODO : need to implement event logic using generic netlink */
+		break;
+	case WLC_E_PFN_BEST_BATCHING:
+#ifndef GSCAN_SUPPORT
+	{
+		struct dhd_pno_batch_params *params_batch;
+		params_batch = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS].params_batch;
+		if (!COMPLETION_WAIT_QUEUE_ACTIVE(&_pno_state->get_batch_done.wait)) {
+			DHD_PNO(("%s : WLC_E_PFN_BEST_BATCHING\n", __FUNCTION__));
+			params_batch->get_batch.buf = NULL;
+			params_batch->get_batch.bufsize = 0;
+			params_batch->get_batch.reason = PNO_STATUS_EVENT;
+			schedule_work(&_pno_state->work);
+		} else
+			DHD_PNO(("%s : WLC_E_PFN_BEST_BATCHING"
+				"will skip this event\n", __FUNCTION__));
+		break;
+	}
+#else
+		break;
+#endif /* !GSCAN_SUPPORT */
+	default:
+		DHD_ERROR(("unknown event : %d\n", event_type));
+	}
+exit:
+	return err;
+}
+
+int dhd_pno_init(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	dhd_pno_status_info_t *_pno_state;
+	char *buf = NULL;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	UNUSED_PARAMETER(_dhd_pno_suspend);
+	if (dhd->pno_state)
+		goto exit;
+	dhd->pno_state = MALLOC(dhd->osh, sizeof(dhd_pno_status_info_t));
+	NULL_CHECK(dhd->pno_state, "failed to create dhd_pno_state", err);
+	memset(dhd->pno_state, 0, sizeof(dhd_pno_status_info_t));
+	/* need to check whether current firmware support batching and hotlist scan */
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	_pno_state->wls_supported = TRUE;
+	_pno_state->dhd = dhd;
+	mutex_init(&_pno_state->pno_mutex);
+	INIT_WORK(&_pno_state->work, _dhd_pno_get_batch_handler);
+	init_completion(&_pno_state->get_batch_done);
+#ifdef GSCAN_SUPPORT
+	init_waitqueue_head(&_pno_state->batch_get_wait);
+#endif /* GSCAN_SUPPORT */
+	buf = MALLOC(dhd->osh, WLC_IOCTL_SMLEN);
+	if (!buf) {
+		DHD_ERROR((":%s buf alloc err.\n", __FUNCTION__));
+		return BCME_NOMEM;
+	}
+	err = dhd_iovar(dhd, 0, "pfnlbest", NULL, 0, buf, WLC_IOCTL_SMLEN,
+			FALSE);
+	if (err == BCME_UNSUPPORTED) {
+		_pno_state->wls_supported = FALSE;
+		DHD_ERROR(("Android Location Service, UNSUPPORTED\n"));
+		DHD_INFO(("Current firmware doesn't support"
+			" Android Location Service\n"));
+	} else {
+		DHD_ERROR(("%s: Support Android Location Service\n",
+			__FUNCTION__));
+	}
+exit:
+	MFREE(dhd->osh, buf, WLC_IOCTL_SMLEN);
+	return err;
+}
+
+int dhd_pno_deinit(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	dhd_pno_status_info_t *_pno_state;
+	dhd_pno_params_t *_params;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+
+	DHD_PNO(("%s enter\n", __FUNCTION__));
+	_pno_state = PNO_GET_PNOSTATE(dhd);
+	NULL_CHECK(_pno_state, "pno_state is NULL", err);
+	/* may need to free legacy ssid_list */
+	if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) {
+		_params = &_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS];
+		_dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_LEGACY_MODE);
+	}
+
+#ifdef GSCAN_SUPPORT
+	if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) {
+		_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS];
+		mutex_lock(&_pno_state->pno_mutex);
+		dhd_pno_reset_cfg_gscan(dhd, _params, _pno_state, GSCAN_FLUSH_ALL_CFG);
+		mutex_unlock(&_pno_state->pno_mutex);
+	}
+#endif /* GSCAN_SUPPORT */
+
+	if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) {
+		_params = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS];
+		/* clear resource if the BATCH MODE is on */
+		_dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_BATCH_MODE);
+	}
+	cancel_work_sync(&_pno_state->work);
+	MFREE(dhd->osh, _pno_state, sizeof(dhd_pno_status_info_t));
+	dhd->pno_state = NULL;
+	return err;
+}
+
+#ifndef OEM_ANDROID
+#if defined(NDIS)
+#define DHD_IOVAR_BUF_SIZE	128
+int
+dhd_pno_cfg(dhd_pub_t *dhd, wl_pfn_cfg_t *pcfg)
+{
+	int ret = -1;
+	uint len = 0;
+	char iovbuf[2 * DHD_IOVAR_BUF_SIZE];
+
+	if (!dhd)
+		return ret;
+	memset(iovbuf, 0, sizeof(iovbuf));
+	if ((len =
+	     bcm_mkiovar("pfn_cfg", (char *)pcfg,
+	     sizeof(wl_pfn_cfg_t), iovbuf, sizeof(iovbuf))) > 0) {
+		if ((ret =
+		dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, len, TRUE, 0)) < 0)
+			DHD_ERROR(("%s failed for error=%d\n",
+			           __FUNCTION__, ret));
+		else
+			DHD_ERROR(("%s set OK\n", __FUNCTION__));
+	} else {
+		DHD_ERROR(("%s iovar failed\n", __FUNCTION__));
+		ret = -1;
+	}
+
+	return ret;
+}
+
+int
+dhd_pno_suspend(dhd_pub_t *dhd, int pfn_suspend)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int ret = -1;
+
+	if ((!dhd) || ((pfn_suspend != 0) && (pfn_suspend != 1))) {
+		DHD_ERROR(("%s error exit\n", __FUNCTION__));
+		return ret;
+	}
+
+	memset(iovbuf, 0, sizeof(iovbuf));
+	/* suspend/resume PNO */
+	if ((ret = bcm_mkiovar("pfn_suspend", (char *)&pfn_suspend, 4, iovbuf,
+		sizeof(iovbuf))) > 0) {
+		if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf,
+		                            sizeof(iovbuf), TRUE, 0)) < 0)
+			DHD_ERROR(("%s failed for error=%d\n", __FUNCTION__, ret));
+		else  {
+			DHD_TRACE(("%s set pno to %s\n", __FUNCTION__,
+				(pfn_suspend? "suspend" : "resume")));
+			dhd->pno_suspend = pfn_suspend;
+		}
+	}
+	else {
+		DHD_ERROR(("%s failed at mkiovar, err=%d\n", __FUNCTION__, ret));
+		ret = -1;
+	}
+
+	return ret;
+}
+
+int
+dhd_pno_set_add(dhd_pub_t *dhd, wl_pfn_t *netinfo, int nssid, ushort scan_fr, ushort slowscan_fr,
+	uint8 pno_repeat, uint8 pno_freq_expo_max, int16 flags)
+{
+	int err = -1;
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int k, i;
+	wl_pfn_param_t pfn_param;
+	wl_pfn_t	pfn_element;
+	uint len = 0;
+
+	DHD_TRACE(("%s nssid=%d scan_fr=%d\n", __FUNCTION__, nssid, scan_fr));
+
+	if ((!dhd) || (!netinfo) ||
+		(nssid > MAX_PFN_LIST_COUNT) || (nssid <= 0)) {
+		DHD_ERROR(("%s error exit\n", __FUNCTION__));
+		return err;
+	}
+
+	/* Check for broadcast ssid */
+	for (k = 0; k < nssid; k++) {
+		if (!netinfo[k].ssid.SSID_len) {
+			DHD_ERROR(("%d: Broadcast SSID is ilegal for PNO setting\n", k));
+			return err;
+		}
+	}
+
+	/* clean up everything */
+	if  (dhd_pno_clean(dhd) < 0) {
+		DHD_ERROR(("%s failed\n", __FUNCTION__));
+		return err;
+	}
+	memset(&pfn_param, 0, sizeof(pfn_param));
+	memset(&pfn_element, 0, sizeof(pfn_element));
+
+	/* set pfn parameters */
+	pfn_param.version = htod32(PFN_VERSION);
+	pfn_param.flags = htod16(flags |(PFN_LIST_ORDER << SORT_CRITERIA_BIT));
+
+	/* set extra pno params */
+	pfn_param.repeat = pno_repeat;
+	pfn_param.exp = pno_freq_expo_max;
+	pfn_param.slow_freq = slowscan_fr;
+
+	/* set up pno scan fr */
+	if (scan_fr > PNO_SCAN_MAX_FW_SEC) {
+		DHD_ERROR(("%s pno freq above %d sec\n", __FUNCTION__, PNO_SCAN_MAX_FW_SEC));
+		return err;
+	}
+	if (scan_fr < PNO_SCAN_MIN_FW_SEC) {
+		DHD_ERROR(("%s pno freq less %d sec\n", __FUNCTION__, PNO_SCAN_MIN_FW_SEC));
+		return err;
+	}
+	pfn_param.scan_freq = htod32(scan_fr);
+	if (slowscan_fr)
+		pfn_param.lost_network_timeout = -1; /* so no aging out */
+	memset(iovbuf, 0, sizeof(iovbuf));
+	len = bcm_mkiovar("pfn_set", (char *)&pfn_param, sizeof(pfn_param), iovbuf, sizeof(iovbuf));
+	if (!len)
+		return err;
+
+	if (dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, len, TRUE, 0) < 0)
+		return err;
+
+	/* set all pfn ssid */
+	for (i = 0; i < nssid; i++) {
+		pfn_element.infra = htod32(1);
+		pfn_element.auth = htod32(netinfo[i].auth);
+		pfn_element.wpa_auth = htod32(netinfo[i].wpa_auth);
+		pfn_element.wsec = htod32(netinfo[i].wsec);
+		pfn_element.flags = htod32(netinfo[i].flags);
+
+		memcpy((char *)pfn_element.ssid.SSID, netinfo[i].ssid.SSID,
+			netinfo[i].ssid.SSID_len);
+		pfn_element.ssid.SSID_len = netinfo[i].ssid.SSID_len;
+
+		if ((len =
+		bcm_mkiovar("pfn_add", (char *)&pfn_element,
+			sizeof(pfn_element), iovbuf, sizeof(iovbuf))) > 0) {
+			if ((err =
+			dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, len, TRUE, 0)) < 0) {
+				DHD_ERROR(("%s failed for i=%d error=%d\n",
+					__FUNCTION__, i, err));
+				return err;
+			}
+			else
+				DHD_ERROR(("%s set ssid %s\n",
+					__FUNCTION__, netinfo[i].ssid.SSID));
+		}
+		else
+			DHD_ERROR(("%s: mkiovar pfn_add failed\n", __FUNCTION__));
+
+		memset(&pfn_element, 0, sizeof(pfn_element));
+	}
+
+	return err;
+}
+
+int
+dhd_pno_enable(dhd_pub_t *dhd, int pfn_enabled)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int ret = -1;
+
+	if ((!dhd) || ((pfn_enabled != 0) && (pfn_enabled != 1))) {
+		DHD_ERROR(("%s error exit\n", __FUNCTION__));
+		return ret;
+	}
+
+#ifndef WL_SCHED_SCAN
+	memset(iovbuf, 0, sizeof(iovbuf));
+
+	if ((pfn_enabled) && (dhd_is_associated(dhd, 0, NULL) == TRUE)) {
+		DHD_ERROR(("%s pno is NOT enable : called in assoc mode , ignore\n", __FUNCTION__));
+		return ret;
+	}
+#endif /* !WL_SCHED_SCAN */
+
+	/* make sure PNO is not suspended when it is going to be enabled */
+	if (pfn_enabled) {
+		int pfn_suspend = 0;
+		memset(iovbuf, 0, sizeof(iovbuf));
+		if ((ret = bcm_mkiovar("pfn_suspend", (char *)&pfn_suspend, 4, iovbuf,
+		        sizeof(iovbuf))) > 0) {
+			if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf,
+				sizeof(iovbuf), TRUE, 0)) < 0) {
+				DHD_ERROR(("pfn_suspend failed for error=%d\n", __FUNCTION__, ret));
+				return ret;
+			} else {
+				DHD_TRACE(("pno resumed\n"));
+			}
+		} else {
+			return -1;
+		}
+	}
+
+	/* Enable/disable PNO */
+	if ((ret = bcm_mkiovar("pfn", (char *)&pfn_enabled, 4, iovbuf, sizeof(iovbuf))) > 0) {
+		if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR,
+		        iovbuf, sizeof(iovbuf), TRUE, 0)) < 0) {
+			DHD_ERROR(("%s failed for error=%d\n", __FUNCTION__, ret));
+			return ret;
+		} else {
+			dhd->pno_enable = pfn_enabled;
+			DHD_TRACE(("%s set pno as %s\n",
+			        __FUNCTION__, dhd->pno_enable ? "Enable" : "Disable"));
+		}
+	}
+	else DHD_ERROR(("%s failed err=%d\n", __FUNCTION__, ret));
+
+	return ret;
+}
+
+int
+dhd_pno_clean(dhd_pub_t *dhd)
+{
+	char iovbuf[DHD_IOVAR_BUF_SIZE];
+	int pfn_enabled = 0;
+	int iov_len = 0;
+	int ret;
+
+	/* Disable pfn */
+	iov_len = bcm_mkiovar("pfn", (char *)&pfn_enabled, 4, iovbuf, sizeof(iovbuf));
+	if (!iov_len) {
+		DHD_ERROR(("%s: Insufficient iovar buffer size %d \n",
+		        __FUNCTION__, sizeof(iovbuf)));
+		return -1;
+	}
+	if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, iov_len, TRUE, 0)) >= 0) {
+		/* clear pfn */
+		iov_len = bcm_mkiovar("pfnclear", 0, 0, iovbuf, sizeof(iovbuf));
+		if (iov_len) {
+			if ((ret = dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf,
+			                            iov_len, TRUE, 0)) < 0) {
+				DHD_ERROR(("%s failed code %d\n", __FUNCTION__, ret));
+			}
+		} else {
+			ret = -1;
+			DHD_ERROR(("%s failed code %d\n", __FUNCTION__, iov_len));
+		}
+	} else
+		DHD_ERROR(("%s failed code %d\n", __FUNCTION__, ret));
+
+	return ret;
+}
+#endif /* defined(NDIS) */
+#endif /* OEM_ANDROID */
+#endif /* PNO_SUPPORT */
diff --git a/drivers/net/wireless/ap12275/dhd_pno.h b/drivers/net/wireless/ap12275/dhd_pno.h
new file mode 100644
index 000000000000..1846b2eb2929
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_pno.h
@@ -0,0 +1,585 @@
+/*
+ * Header file of Broadcom Dongle Host Driver (DHD)
+ * Prefered Network Offload code and Wi-Fi Location Service(WLS) code.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+
+#ifndef __DHD_PNO_H__
+#define __DHD_PNO_H__
+
+#if defined(PNO_SUPPORT)
+#define PNO_TLV_PREFIX			'S'
+#define PNO_TLV_VERSION			'1'
+#define PNO_TLV_SUBTYPE_LEGACY_PNO '2'
+#define PNO_TLV_RESERVED		'0'
+
+#define PNO_BATCHING_SET "SET"
+#define PNO_BATCHING_GET "GET"
+#define PNO_BATCHING_STOP "STOP"
+
+#define PNO_PARAMS_DELIMETER " "
+#define PNO_PARAM_CHANNEL_DELIMETER ","
+#define PNO_PARAM_VALUE_DELLIMETER '='
+#define PNO_PARAM_SCANFREQ "SCANFREQ"
+#define PNO_PARAM_BESTN	"BESTN"
+#define PNO_PARAM_MSCAN "MSCAN"
+#define PNO_PARAM_CHANNEL "CHANNEL"
+#define PNO_PARAM_RTT "RTT"
+
+#define PNO_TLV_TYPE_SSID_IE		'S'
+#define PNO_TLV_TYPE_TIME		'T'
+#define PNO_TLV_FREQ_REPEAT		'R'
+#define PNO_TLV_FREQ_EXPO_MAX		'M'
+
+#define MAXNUM_SSID_PER_ADD	16
+#define MAXNUM_PNO_PARAMS 2
+#define PNO_TLV_COMMON_LENGTH	1
+#define DEFAULT_BATCH_MSCAN 16
+
+#define RESULTS_END_MARKER "----\n"
+#define SCAN_END_MARKER "####\n"
+#define AP_END_MARKER "====\n"
+#define PNO_RSSI_MARGIN_DBM          30
+
+#define CSCAN_COMMAND			"CSCAN "
+#define CSCAN_TLV_PREFIX		'S'
+#define CSCAN_TLV_VERSION		1
+#define CSCAN_TLV_SUBVERSION		0
+#define CSCAN_TLV_TYPE_SSID_IE		'S'
+#define CSCAN_TLV_TYPE_CHANNEL_IE	'C'
+#define CSCAN_TLV_TYPE_NPROBE_IE	'N'
+#define CSCAN_TLV_TYPE_ACTIVE_IE	'A'
+#define CSCAN_TLV_TYPE_PASSIVE_IE	'P'
+#define CSCAN_TLV_TYPE_HOME_IE		'H'
+#define CSCAN_TLV_TYPE_STYPE_IE		'T'
+
+#define WL_SCAN_PARAMS_SSID_MAX         10
+#define GET_SSID                        "SSID="
+#define GET_CHANNEL                     "CH="
+#define GET_NPROBE                      "NPROBE="
+#define GET_ACTIVE_ASSOC_DWELL          "ACTIVE="
+#define GET_PASSIVE_ASSOC_DWELL         "PASSIVE="
+#define GET_HOME_DWELL                  "HOME="
+#define GET_SCAN_TYPE                   "TYPE="
+
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+#define GSCAN_MAX_CH_BUCKETS             8
+#define GSCAN_MAX_CHANNELS_IN_BUCKET     32
+#define GSCAN_MAX_AP_CACHE_PER_SCAN      32
+#define GSCAN_MAX_AP_CACHE               320
+#define GSCAN_BG_BAND_MASK             (1 << 0)
+#define GSCAN_A_BAND_MASK              (1 << 1)
+#define GSCAN_DFS_MASK                 (1 << 2)
+#define GSCAN_ABG_BAND_MASK            (GSCAN_A_BAND_MASK | GSCAN_BG_BAND_MASK)
+#define GSCAN_BAND_MASK                (GSCAN_ABG_BAND_MASK | GSCAN_DFS_MASK)
+
+#define GSCAN_FLUSH_HOTLIST_CFG      (1 << 0)
+#define GSCAN_FLUSH_SIGNIFICANT_CFG  (1 << 1)
+#define GSCAN_FLUSH_SCAN_CFG         (1 << 2)
+#define GSCAN_FLUSH_EPNO_CFG         (1 << 3)
+#define GSCAN_FLUSH_ALL_CFG     (GSCAN_FLUSH_SCAN_CFG | \
+								GSCAN_FLUSH_SIGNIFICANT_CFG | \
+								GSCAN_FLUSH_HOTLIST_CFG  | \
+								GSCAN_FLUSH_EPNO_CFG)
+#define DHD_EPNO_HIDDEN_SSID          (1 << 0)
+#define DHD_EPNO_A_BAND_TRIG          (1 << 1)
+#define DHD_EPNO_BG_BAND_TRIG         (1 << 2)
+#define DHD_EPNO_STRICT_MATCH         (1 << 3)
+#define DHD_EPNO_SAME_NETWORK         (1 << 4)
+#define DHD_PNO_USE_SSID              (DHD_EPNO_HIDDEN_SSID | DHD_EPNO_STRICT_MATCH)
+
+/* Do not change GSCAN_BATCH_RETRIEVAL_COMPLETE */
+#define GSCAN_BATCH_RETRIEVAL_COMPLETE      0
+#define GSCAN_BATCH_RETRIEVAL_IN_PROGRESS   1
+#define GSCAN_BATCH_NO_THR_SET              101
+#define GSCAN_LOST_AP_WINDOW_DEFAULT        4
+#define GSCAN_MIN_BSSID_TIMEOUT             90
+#define GSCAN_BATCH_GET_MAX_WAIT            500
+#define CHANNEL_BUCKET_EMPTY_INDEX                      0xFFFF
+#define GSCAN_RETRY_THRESHOLD              3
+
+#define MAX_EPNO_SSID_NUM                   64
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+
+enum scan_status {
+	/* SCAN ABORT by other scan */
+	PNO_STATUS_ABORT,
+	/* RTT is presence or not */
+	PNO_STATUS_RTT_PRESENCE,
+	/* Disable PNO by Driver */
+	PNO_STATUS_DISABLE,
+	/* NORMAL BATCHING GET */
+	PNO_STATUS_NORMAL,
+	/* WLC_E_PFN_BEST_BATCHING */
+	PNO_STATUS_EVENT,
+	PNO_STATUS_MAX
+};
+#define PNO_STATUS_ABORT_MASK 0x0001
+#define PNO_STATUS_RTT_MASK 0x0002
+#define PNO_STATUS_DISABLE_MASK 0x0004
+#define PNO_STATUS_OOM_MASK 0x0010
+
+enum index_mode {
+	INDEX_OF_LEGACY_PARAMS,
+	INDEX_OF_BATCH_PARAMS,
+	INDEX_OF_HOTLIST_PARAMS,
+	/* GSCAN includes hotlist scan and they do not run
+	 * independent of each other
+	 */
+	INDEX_OF_GSCAN_PARAMS = INDEX_OF_HOTLIST_PARAMS,
+	INDEX_MODE_MAX
+};
+enum dhd_pno_status {
+	DHD_PNO_DISABLED,
+	DHD_PNO_ENABLED,
+	DHD_PNO_SUSPEND
+};
+typedef struct cmd_tlv {
+	char prefix;
+	char version;
+	char subtype;
+	char reserved;
+} cmd_tlv_t;
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+typedef enum {
+	HOTLIST_LOST,
+	HOTLIST_FOUND
+} hotlist_type_t;
+
+typedef enum dhd_pno_gscan_cmd_cfg {
+	DHD_PNO_BATCH_SCAN_CFG_ID = 0,
+	DHD_PNO_GEOFENCE_SCAN_CFG_ID,
+	DHD_PNO_SIGNIFICANT_SCAN_CFG_ID,
+	DHD_PNO_SCAN_CFG_ID,
+	DHD_PNO_GET_CAPABILITIES,
+	DHD_PNO_GET_BATCH_RESULTS,
+	DHD_PNO_GET_CHANNEL_LIST,
+	DHD_PNO_GET_NEW_EPNO_SSID_ELEM,
+	DHD_PNO_EPNO_CFG_ID,
+	DHD_PNO_GET_AUTOJOIN_CAPABILITIES,
+	DHD_PNO_EPNO_PARAMS_ID
+} dhd_pno_gscan_cmd_cfg_t;
+
+typedef enum dhd_pno_mode {
+	/* Wi-Fi Legacy PNO Mode */
+	DHD_PNO_NONE_MODE   = 0,
+	DHD_PNO_LEGACY_MODE = (1 << (0)),
+	/* Wi-Fi Android BATCH SCAN Mode */
+	DHD_PNO_BATCH_MODE = (1 << (1)),
+	/* Wi-Fi Android Hotlist SCAN Mode */
+	DHD_PNO_HOTLIST_MODE = (1 << (2)),
+	/* Wi-Fi Google Android SCAN Mode */
+	DHD_PNO_GSCAN_MODE = (1 << (3))
+} dhd_pno_mode_t;
+#else
+typedef enum dhd_pno_mode {
+	/* Wi-Fi Legacy PNO Mode */
+	DHD_PNO_NONE_MODE   = 0,
+	DHD_PNO_LEGACY_MODE = (1 << (0)),
+	/* Wi-Fi Android BATCH SCAN Mode */
+	DHD_PNO_BATCH_MODE = (1 << (1)),
+	/* Wi-Fi Android Hotlist SCAN Mode */
+	DHD_PNO_HOTLIST_MODE = (1 << (2))
+} dhd_pno_mode_t;
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+
+typedef struct dhd_pno_ssid {
+	bool		hidden;
+	int8		rssi_thresh;
+	uint8		dummy;
+	uint16		SSID_len;
+	uint32		flags;
+	int32		wpa_auth;
+	uchar		SSID[DOT11_MAX_SSID_LEN];
+	struct list_head list;
+} dhd_pno_ssid_t;
+
+struct dhd_pno_bssid {
+	struct ether_addr	macaddr;
+	/* Bit4: suppress_lost, Bit3: suppress_found */
+	uint16			flags;
+	struct list_head list;
+};
+
+typedef struct dhd_pno_bestnet_entry {
+	struct ether_addr BSSID;
+	uint8	SSID_len;
+	uint8	SSID[DOT11_MAX_SSID_LEN];
+	int8	RSSI;
+	uint8	channel;
+	uint32	timestamp;
+	uint16	rtt0; /* distance_cm based on RTT */
+	uint16	rtt1; /* distance_cm based on sample standard deviation */
+	unsigned long recorded_time;
+	struct list_head list;
+} dhd_pno_bestnet_entry_t;
+#define BESTNET_ENTRY_SIZE (sizeof(dhd_pno_bestnet_entry_t))
+
+typedef struct dhd_pno_bestnet_header {
+	struct dhd_pno_bestnet_header *next;
+	uint8 reason;
+	uint32 tot_cnt;
+	uint32 tot_size;
+	struct list_head entry_list;
+} dhd_pno_best_header_t;
+#define BEST_HEADER_SIZE (sizeof(dhd_pno_best_header_t))
+
+typedef struct dhd_pno_scan_results {
+	dhd_pno_best_header_t *bestnetheader;
+	uint8 cnt_header;
+	struct list_head list;
+} dhd_pno_scan_results_t;
+#define SCAN_RESULTS_SIZE (sizeof(dhd_pno_scan_results_t))
+
+struct dhd_pno_get_batch_info {
+	/* info related to get batch */
+	char *buf;
+	bool batch_started;
+	uint32 tot_scan_cnt;
+	uint32 expired_tot_scan_cnt;
+	uint32 top_node_cnt;
+	uint32 bufsize;
+	uint32 bytes_written;
+	int reason;
+	struct list_head scan_results_list;
+	struct list_head expired_scan_results_list;
+};
+struct dhd_pno_legacy_params {
+	uint16 scan_fr;
+	uint16 chan_list[WL_NUMCHANNELS];
+	uint16 nchan;
+	int pno_repeat;
+	int pno_freq_expo_max;
+	int nssid;
+	struct list_head ssid_list;
+};
+struct dhd_pno_batch_params {
+	int32 scan_fr;
+	uint8 bestn;
+	uint8 mscan;
+	uint8 band;
+	uint16 chan_list[WL_NUMCHANNELS];
+	uint16 nchan;
+	uint16 rtt;
+	struct dhd_pno_get_batch_info get_batch;
+};
+struct dhd_pno_hotlist_params {
+	uint8 band;
+	int32 scan_fr;
+	uint16 chan_list[WL_NUMCHANNELS];
+	uint16 nchan;
+	uint16 nbssid;
+	struct list_head bssid_list;
+};
+
+#define DHD_PNO_CHSPEC_SUPPORT_VER	14
+
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+#define DHD_PNO_REPORT_NO_BATCH      (1 << 2)
+
+typedef struct dhd_pno_gscan_channel_bucket {
+	uint16 bucket_freq_multiple;
+	/* band = 1 All bg band channels,
+	 * band = 2 All a band channels,
+	 * band = 0 chan_list channels
+	 */
+	uint16 band;
+	uint8 report_flag;
+	uint8 num_channels;
+	uint16 repeat;
+	uint16 bucket_max_multiple;
+	uint16 chan_list[GSCAN_MAX_CHANNELS_IN_BUCKET];
+} dhd_pno_gscan_channel_bucket_t;
+
+#define DHD_PNO_AUTH_CODE_OPEN  1 /*  Open   */
+#define DHD_PNO_AUTH_CODE_PSK   2 /* WPA_PSK or WPA2PSK */
+#define DHD_PNO_AUTH_CODE_EAPOL 4 /* any EAPOL    */
+
+#define DHD_EPNO_DEFAULT_INDEX     0xFFFFFFFF
+
+typedef struct dhd_epno_params {
+	uint8 ssid[DOT11_MAX_SSID_LEN];
+	uint8 ssid_len;
+	int8 rssi_thresh;
+	uint8 flags;
+	uint8 auth;
+	/* index required only for visble ssid */
+	uint32 index;
+	struct list_head list;
+} dhd_epno_params_t;
+
+typedef struct dhd_epno_results {
+	uint8 ssid[DOT11_MAX_SSID_LEN];
+	uint8 ssid_len;
+	int8 rssi;
+	uint16 channel;
+	uint16 flags;
+	struct ether_addr bssid;
+} dhd_epno_results_t;
+
+typedef struct dhd_pno_swc_evt_param {
+	uint16 results_rxed_so_far;
+	wl_pfn_significant_net_t *change_array;
+} dhd_pno_swc_evt_param_t;
+
+typedef struct wifi_gscan_result {
+	uint64 ts;			/* Time of discovery           */
+	char ssid[DOT11_MAX_SSID_LEN+1]; /* null terminated		*/
+	struct ether_addr macaddr;	/* BSSID                      */
+	uint32 channel;			/* channel frequency in MHz    */
+	int32 rssi;			/* in db                       */
+	uint64 rtt;			/* in nanoseconds              */
+	uint64 rtt_sd;			/* standard deviation in rtt   */
+	uint16 beacon_period;		/* units are Kusec             */
+	uint16 capability;		/* Capability information       */
+	uint32 pad;
+} wifi_gscan_result_t;
+
+typedef struct wifi_gscan_full_result {
+    wifi_gscan_result_t fixed;
+    uint32 scan_ch_bucket;
+    uint32 ie_length;		/* byte length of Information Elements */
+    char  ie_data[1];		/* IE  data to follow       */
+} wifi_gscan_full_result_t;
+
+typedef struct gscan_results_cache {
+	struct gscan_results_cache *next;
+	uint8  scan_id;
+	uint8  flag;
+	uint8  tot_count;
+	uint8  tot_consumed;
+	uint32 scan_ch_bucket;
+	wifi_gscan_result_t results[1];
+} gscan_results_cache_t;
+
+typedef struct dhd_pno_gscan_capabilities {
+	int max_scan_cache_size;
+	int max_scan_buckets;
+	int max_ap_cache_per_scan;
+	int max_rssi_sample_size;
+	int max_scan_reporting_threshold;
+	int max_hotlist_bssids;
+	int max_hotlist_ssids;
+	int max_significant_wifi_change_aps;
+	int max_bssid_history_entries;
+	int max_epno_ssid_crc32;
+	int max_epno_hidden_ssid;
+	int max_white_list_ssid;
+} dhd_pno_gscan_capabilities_t;
+
+typedef struct dhd_epno_ssid_cfg {
+	wl_ssid_ext_params_t params;
+	uint32 num_epno_ssid;
+	struct list_head epno_ssid_list;
+} dhd_epno_ssid_cfg_t;
+
+struct dhd_pno_gscan_params {
+	int32 scan_fr;
+	uint8 bestn;
+	uint8 mscan;
+	uint8 buffer_threshold;
+	uint8 swc_nbssid_threshold;
+	uint8 swc_rssi_window_size;
+	uint8 lost_ap_window;
+	uint8 nchannel_buckets;
+	uint8 reason;
+	uint8 get_batch_flag;
+	uint8 send_all_results_flag;
+	uint16 max_ch_bucket_freq;
+	gscan_results_cache_t *gscan_batch_cache;
+	gscan_results_cache_t *gscan_hotlist_found;
+	gscan_results_cache_t*gscan_hotlist_lost;
+	uint16 nbssid_significant_change;
+	uint16 nbssid_hotlist;
+	struct dhd_pno_swc_evt_param param_significant;
+	struct dhd_pno_gscan_channel_bucket channel_bucket[GSCAN_MAX_CH_BUCKETS];
+	struct list_head hotlist_bssid_list;
+	struct list_head significant_bssid_list;
+	dhd_epno_ssid_cfg_t epno_cfg;
+	uint32 scan_id;
+};
+
+typedef struct gscan_scan_params {
+	int32 scan_fr;
+	uint16 nchannel_buckets;
+	struct dhd_pno_gscan_channel_bucket channel_bucket[GSCAN_MAX_CH_BUCKETS];
+} gscan_scan_params_t;
+
+typedef struct gscan_batch_params {
+	uint8 bestn;
+	uint8 mscan;
+	uint8 buffer_threshold;
+} gscan_batch_params_t;
+
+struct bssid_t {
+	struct ether_addr	macaddr;
+	int16 rssi_reporting_threshold;  /* 0 -> no reporting threshold */
+};
+
+typedef struct gscan_hotlist_scan_params {
+	uint16 lost_ap_window; /* number of scans to declare LOST */
+	uint16 nbssid;   /* number of bssids  */
+	struct bssid_t bssid[1];  /* n bssids to follow */
+} gscan_hotlist_scan_params_t;
+
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+
+typedef union dhd_pno_params {
+	struct dhd_pno_legacy_params params_legacy;
+	struct dhd_pno_batch_params params_batch;
+	struct dhd_pno_hotlist_params params_hotlist;
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+	struct dhd_pno_gscan_params params_gscan;
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+} dhd_pno_params_t;
+
+typedef struct dhd_pno_status_info {
+	dhd_pub_t *dhd;
+	struct work_struct work;
+	struct mutex pno_mutex;
+#ifdef GSCAN_SUPPORT
+	wait_queue_head_t batch_get_wait;
+#endif /* GSCAN_SUPPORT */
+	struct completion get_batch_done;
+	bool wls_supported; /* wifi location service supported or not */
+	enum dhd_pno_status pno_status;
+	enum dhd_pno_mode pno_mode;
+	dhd_pno_params_t pno_params_arr[INDEX_MODE_MAX];
+	struct list_head head_list;
+} dhd_pno_status_info_t;
+
+/* wrapper functions */
+extern int
+dhd_dev_pno_enable(struct net_device *dev, int enable);
+
+extern int
+dhd_dev_pno_stop_for_ssid(struct net_device *dev);
+
+extern int
+dhd_dev_pno_set_for_ssid(struct net_device *dev, wlc_ssid_ext_t* ssids_local, int nssid,
+	uint16 scan_fr, int pno_repeat, int pno_freq_expo_max, uint16 *channel_list, int nchan);
+
+extern int
+dhd_dev_pno_set_for_batch(struct net_device *dev,
+	struct dhd_pno_batch_params *batch_params);
+
+extern int
+dhd_dev_pno_get_for_batch(struct net_device *dev, char *buf, int bufsize);
+
+extern int
+dhd_dev_pno_stop_for_batch(struct net_device *dev);
+
+extern int
+dhd_dev_pno_set_for_hotlist(struct net_device *dev, wl_pfn_bssid_t *p_pfn_bssid,
+	struct dhd_pno_hotlist_params *hotlist_params);
+extern bool dhd_dev_is_legacy_pno_enabled(struct net_device *dev);
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+extern void *
+dhd_dev_pno_get_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type, void *info,
+        uint32 *len);
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+#ifdef GSCAN_SUPPORT
+extern int
+dhd_dev_pno_set_cfg_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type,
+              void *buf, bool flush);
+int dhd_dev_pno_lock_access_batch_results(struct net_device *dev);
+void dhd_dev_pno_unlock_access_batch_results(struct net_device *dev);
+extern int dhd_dev_pno_run_gscan(struct net_device *dev, bool run, bool flush);
+extern int dhd_dev_pno_enable_full_scan_result(struct net_device *dev, bool real_time);
+int dhd_retreive_batch_scan_results(dhd_pub_t *dhd);
+extern void * dhd_dev_hotlist_scan_event(struct net_device *dev,
+            const void  *data, int *send_evt_bytes, hotlist_type_t type, u32 *buf_len);
+void * dhd_dev_process_full_gscan_result(struct net_device *dev,
+            const void  *data, uint32 len, int *send_evt_bytes);
+extern int dhd_dev_gscan_batch_cache_cleanup(struct net_device *dev);
+extern void dhd_dev_gscan_hotlist_cache_cleanup(struct net_device *dev, hotlist_type_t type);
+extern int dhd_dev_wait_batch_results_complete(struct net_device *dev);
+extern void * dhd_dev_process_epno_result(struct net_device *dev,
+		const void  *data, uint32 event, int *send_evt_bytes);
+extern int dhd_dev_set_epno(struct net_device *dev);
+extern int dhd_dev_flush_fw_epno(struct net_device *dev);
+#endif /* GSCAN_SUPPORT */
+/* dhd pno fuctions */
+extern int dhd_pno_stop_for_ssid(dhd_pub_t *dhd);
+extern int dhd_pno_enable(dhd_pub_t *dhd, int enable);
+extern int dhd_pno_set_for_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssid_list, int nssid,
+	uint16  scan_fr, int pno_repeat, int pno_freq_expo_max, uint16 *channel_list, int nchan);
+
+extern int dhd_pno_set_for_batch(dhd_pub_t *dhd, struct dhd_pno_batch_params *batch_params);
+
+extern int dhd_pno_get_for_batch(dhd_pub_t *dhd, char *buf, int bufsize, int reason);
+
+extern int dhd_pno_stop_for_batch(dhd_pub_t *dhd);
+
+extern int dhd_pno_set_for_hotlist(dhd_pub_t *dhd, wl_pfn_bssid_t *p_pfn_bssid,
+	struct dhd_pno_hotlist_params *hotlist_params);
+
+extern int dhd_pno_stop_for_hotlist(dhd_pub_t *dhd);
+
+extern int dhd_pno_event_handler(dhd_pub_t *dhd, wl_event_msg_t *event, void *event_data);
+extern int dhd_pno_init(dhd_pub_t *dhd);
+extern int dhd_pno_deinit(dhd_pub_t *dhd);
+extern bool dhd_is_pno_supported(dhd_pub_t *dhd);
+extern bool dhd_is_legacy_pno_enabled(dhd_pub_t *dhd);
+#if defined(GSCAN_SUPPORT) || defined(DHD_GET_VALID_CHANNELS)
+extern void * dhd_pno_get_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, void *info,
+                       uint32 *len);
+#endif /* GSCAN_SUPPORT || DHD_GET_VALID_CHANNELS */
+#ifdef GSCAN_SUPPORT
+extern int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type,
+                       void *buf, bool flush);
+extern int dhd_pno_lock_batch_results(dhd_pub_t *dhd);
+extern void dhd_pno_unlock_batch_results(dhd_pub_t *dhd);
+extern int dhd_pno_initiate_gscan_request(dhd_pub_t *dhd, bool run, bool flush);
+extern int dhd_pno_enable_full_scan_result(dhd_pub_t *dhd, bool real_time_flag);
+extern int dhd_pno_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, void *buf);
+extern int dhd_dev_retrieve_batch_scan(struct net_device *dev);
+extern void *dhd_handle_hotlist_scan_evt(dhd_pub_t *dhd, const void *event_data,
+                       int *send_evt_bytes, hotlist_type_t type, u32 *buf_len);
+extern void *dhd_process_full_gscan_result(dhd_pub_t *dhd, const void *event_data,
+                       uint32 len, int *send_evt_bytes);
+extern int dhd_gscan_batch_cache_cleanup(dhd_pub_t *dhd);
+extern void dhd_gscan_hotlist_cache_cleanup(dhd_pub_t *dhd, hotlist_type_t type);
+extern int dhd_wait_batch_results_complete(dhd_pub_t *dhd);
+extern void * dhd_pno_process_epno_result(dhd_pub_t *dhd, const void *data,
+         uint32 event, int *size);
+extern void dhd_pno_translate_epno_fw_flags(uint32 *flags);
+extern int dhd_pno_set_epno(dhd_pub_t *dhd);
+extern int dhd_pno_flush_fw_epno(dhd_pub_t *dhd);
+extern void dhd_pno_set_epno_auth_flag(uint32 *wpa_auth);
+#endif /* GSCAN_SUPPORT */
+#endif /* #if defined(PNO_SUPPORT) */
+
+#if defined(NDIS)
+#if defined(PNO_SUPPORT)
+extern int dhd_pno_cfg(dhd_pub_t *dhd, wl_pfn_cfg_t *pcfg);
+extern int dhd_pno_suspend(dhd_pub_t *dhd, int pfn_suspend);
+extern int dhd_pno_set_add(dhd_pub_t *dhd, wl_pfn_t *netinfo, int nssid, ushort scan_fr,
+	ushort slowscan_fr, uint8 pno_repeat, uint8 pno_freq_expo_max, int16 flags);
+extern int dhd_pno_enable(dhd_pub_t *dhd, int pfn_enabled);
+extern int dhd_pno_clean(dhd_pub_t *dhd);
+#endif /* #if defined(PNO_SUPPORT) */
+#endif /* #if defined(NDIS) */
+
+#define FW_MAJOR_VER_PNO_CHSPEC_BACK_PORTED(ver) \
+    ((ver.wlc_ver_major >= 12) && (ver.wlc_ver_minor >= 3))
+
+#endif /* __DHD_PNO_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_proto.h b/drivers/net/wireless/ap12275/dhd_proto.h
new file mode 100644
index 000000000000..6241da7dc067
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_proto.h
@@ -0,0 +1,272 @@
+/*
+ * Header file describing the internal (inter-module) DHD interfaces.
+ *
+ * Provides type definitions and function prototypes used to link the
+ * DHD OS, bus, and protocol modules.
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#ifndef _dhd_proto_h_
+#define _dhd_proto_h_
+
+#include <dhdioctl.h>
+#include <wlioctl.h>
+#ifdef BCMPCIE
+#include <dhd_flowring.h>
+#endif
+
+#define DEFAULT_IOCTL_RESP_TIMEOUT	(5 * 1000) /* 5 seconds */
+#ifndef IOCTL_RESP_TIMEOUT
+#if defined(BCMQT_HW)
+#define IOCTL_RESP_TIMEOUT  (600 * 1000) /* 600 sec in real time */
+#elif defined(BCMFPGA_HW)
+#define IOCTL_RESP_TIMEOUT  (60 * 1000) /* 60 sec in real time */
+#else
+/* In milli second default value for Production FW */
+#define IOCTL_RESP_TIMEOUT  DEFAULT_IOCTL_RESP_TIMEOUT
+#endif /* BCMQT */
+#endif /* IOCTL_RESP_TIMEOUT */
+
+#if defined(BCMQT_HW)
+#define IOCTL_DMAXFER_TIMEOUT  (260 * 1000) /* 260 seconds second */
+#elif defined(BCMFPGA_HW)
+#define IOCTL_DMAXFER_TIMEOUT  (120 * 1000) /* 120 seconds */
+#else
+/* In milli second default value for Production FW */
+#define IOCTL_DMAXFER_TIMEOUT  (15 * 1000) /* 15 seconds for Production FW */
+#endif /* BCMQT */
+
+#ifndef MFG_IOCTL_RESP_TIMEOUT
+#define MFG_IOCTL_RESP_TIMEOUT  20000  /* In milli second default value for MFG FW */
+#endif /* MFG_IOCTL_RESP_TIMEOUT */
+
+#define DEFAULT_D3_ACK_RESP_TIMEOUT	2000
+#ifndef D3_ACK_RESP_TIMEOUT
+#define D3_ACK_RESP_TIMEOUT		DEFAULT_D3_ACK_RESP_TIMEOUT
+#endif /* D3_ACK_RESP_TIMEOUT */
+
+#define DEFAULT_DHD_BUS_BUSY_TIMEOUT	(IOCTL_RESP_TIMEOUT + 1000)
+#ifndef DHD_BUS_BUSY_TIMEOUT
+#define DHD_BUS_BUSY_TIMEOUT	DEFAULT_DHD_BUS_BUSY_TIMEOUT
+#endif /* DEFAULT_DHD_BUS_BUSY_TIMEOUT */
+
+#define DS_EXIT_TIMEOUT	1000 /* In ms */
+#define DS_ENTER_TIMEOUT 1000 /* In ms */
+
+#define IOCTL_DISABLE_TIMEOUT 0
+
+/*
+ * Exported from the dhd protocol module (dhd_cdc, dhd_rndis)
+ */
+
+/* Linkage, sets prot link and updates hdrlen in pub */
+extern int dhd_prot_attach(dhd_pub_t *dhdp);
+
+/* Initilizes the index block for dma'ing indices */
+extern int dhd_prot_dma_indx_init(dhd_pub_t *dhdp, uint32 rw_index_sz,
+	uint8 type, uint32 length);
+#ifdef DHD_DMA_INDICES_SEQNUM
+extern int dhd_prot_dma_indx_copybuf_init(dhd_pub_t *dhd, uint32 buf_sz,
+	uint8 type);
+extern uint32 dhd_prot_read_seqnum(dhd_pub_t *dhd, bool host);
+extern void dhd_prot_write_host_seqnum(dhd_pub_t *dhd, uint32 seq_num);
+extern void dhd_prot_save_dmaidx(dhd_pub_t *dhd);
+#endif /* DHD_DMA_INDICES_SEQNUM */
+/* Unlink, frees allocated protocol memory (including dhd_prot) */
+extern void dhd_prot_detach(dhd_pub_t *dhdp);
+
+/* Initialize protocol: sync w/dongle state.
+ * Sets dongle media info (iswl, drv_version, mac address).
+ */
+extern int dhd_sync_with_dongle(dhd_pub_t *dhdp);
+
+/* Protocol initialization needed for IOCTL/IOVAR path */
+extern int dhd_prot_init(dhd_pub_t *dhd);
+
+/* Stop protocol: sync w/dongle state. */
+extern void dhd_prot_stop(dhd_pub_t *dhdp);
+
+/* Add any protocol-specific data header.
+ * Caller must reserve prot_hdrlen prepend space.
+ */
+extern void dhd_prot_hdrpush(dhd_pub_t *, int ifidx, void *txp);
+extern uint dhd_prot_hdrlen(dhd_pub_t *, void *txp);
+
+/* Remove any protocol-specific data header. */
+extern int dhd_prot_hdrpull(dhd_pub_t *, int *ifidx, void *rxp, uchar *buf, uint *len);
+
+#ifdef DHD_LOSSLESS_ROAMING
+extern int dhd_update_sdio_data_prio_map(dhd_pub_t *dhdp);
+#endif // DHD_LOSSLESS_ROAMING
+
+/* Use protocol to issue ioctl to dongle */
+extern int dhd_prot_ioctl(dhd_pub_t *dhd, int ifidx, wl_ioctl_t * ioc, void * buf, int len);
+
+/* Handles a protocol control response asynchronously */
+extern int dhd_prot_ctl_complete(dhd_pub_t *dhd);
+
+/* Check for and handle local prot-specific iovar commands */
+extern int dhd_prot_iovar_op(dhd_pub_t *dhdp, const char *name,
+                             void *params, int plen, void *arg, int len, bool set);
+
+/* Add prot dump output to a buffer */
+extern void dhd_prot_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf);
+extern void dhd_prot_counters(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf,
+	bool print_ringinfo, bool print_pktidinfo);
+
+/* Dump extended trap data */
+extern int dhd_prot_dump_extended_trap(dhd_pub_t *dhdp, struct bcmstrbuf *b, bool raw);
+
+/* Update local copy of dongle statistics */
+extern void dhd_prot_dstats(dhd_pub_t *dhdp);
+
+extern int dhd_ioctl(dhd_pub_t * dhd_pub, dhd_ioctl_t *ioc, void * buf, uint buflen);
+
+extern int dhd_preinit_ioctls(dhd_pub_t *dhd);
+
+extern int dhd_process_pkt_reorder_info(dhd_pub_t *dhd, uchar *reorder_info_buf,
+	uint reorder_info_len, void **pkt, uint32 *free_buf_count);
+
+#ifdef BCMPCIE
+extern bool dhd_prot_process_msgbuf_txcpl(dhd_pub_t *dhd, int ringtype, uint32 *txcpl_items);
+extern bool dhd_prot_process_msgbuf_rxcpl(dhd_pub_t *dhd, int ringtype,	uint32 *rxcpl_items);
+extern bool dhd_prot_process_msgbuf_infocpl(dhd_pub_t *dhd, uint bound,
+	uint32 *evtlog_items);
+uint32 dhd_prot_get_tx_post_bound(dhd_pub_t *dhd);
+uint32 dhd_prot_get_ctrl_cpl_post_bound(dhd_pub_t *dhd);
+uint32 dhd_prot_get_tx_cpl_bound(dhd_pub_t *dhd);
+uint32 dhd_prot_get_rx_cpl_post_bound(dhd_pub_t *dhd);
+void dhd_prot_set_tx_cpl_bound(dhd_pub_t *dhd, uint32 val);
+void dhd_prot_set_rx_cpl_post_bound(dhd_pub_t *dhd, uint32 val);
+void dhd_prot_set_tx_post_bound(dhd_pub_t *dhd, uint32 val);
+void dhd_prot_set_ctrl_cpl_post_bound(dhd_pub_t *dhd, uint32 val);
+extern bool dhd_prot_process_ctrlbuf(dhd_pub_t * dhd, uint32 *ctrlcpl_items);
+extern int dhd_prot_process_trapbuf(dhd_pub_t * dhd);
+extern bool dhd_prot_dtohsplit(dhd_pub_t * dhd);
+extern int dhd_post_dummy_msg(dhd_pub_t *dhd);
+extern int dhdmsgbuf_lpbk_req(dhd_pub_t *dhd, uint len);
+extern void dhd_prot_rx_dataoffset(dhd_pub_t *dhd, uint32 offset);
+extern int dhd_prot_txdata(dhd_pub_t *dhd, void *p, uint8 ifidx);
+extern void dhd_prot_schedule_aggregate_h2d_db(dhd_pub_t *dhd, uint16 flow_id);
+extern int dhdmsgbuf_dmaxfer_req(dhd_pub_t *dhd,
+	uint len, uint srcdelay, uint destdelay, uint d11_lpbk, uint core_num,
+	uint32 mem_addr);
+extern int dhdmsgbuf_dmaxfer_status(dhd_pub_t *dhd, dma_xfer_info_t *result);
+
+extern void dhd_dma_buf_init(dhd_pub_t *dhd, void *dma_buf,
+	void *va, uint32 len, dmaaddr_t pa, void *dmah, void *secdma);
+extern void dhd_prot_flowrings_pool_release(dhd_pub_t *dhd,
+	uint16 flowid, void *msgbuf_ring);
+extern int dhd_prot_flow_ring_create(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node);
+extern int dhd_post_tx_ring_item(dhd_pub_t *dhd, void *PKTBUF, uint8 ifindex);
+extern int dhd_prot_flow_ring_delete(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node);
+extern int dhd_prot_flow_ring_flush(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node);
+extern int dhd_prot_ringupd_dump(dhd_pub_t *dhd, struct bcmstrbuf *b);
+extern uint32 dhd_prot_metadata_dbg_set(dhd_pub_t *dhd, bool val);
+extern uint32 dhd_prot_metadata_dbg_get(dhd_pub_t *dhd);
+extern uint32 dhd_prot_metadatalen_set(dhd_pub_t *dhd, uint32 val, bool rx);
+extern uint32 dhd_prot_metadatalen_get(dhd_pub_t *dhd, bool rx);
+extern void dhd_prot_print_flow_ring(dhd_pub_t *dhd, void *msgbuf_flow_info, bool h2d,
+	struct bcmstrbuf *strbuf, const char * fmt);
+extern void dhd_prot_print_info(dhd_pub_t *dhd, struct bcmstrbuf *strbuf);
+extern bool dhd_prot_update_txflowring(dhd_pub_t *dhdp, uint16 flow_id, void *msgring_info);
+extern void dhd_prot_txdata_write_flush(dhd_pub_t *dhd, uint16 flow_id);
+extern uint32 dhd_prot_txp_threshold(dhd_pub_t *dhd, bool set, uint32 val);
+extern void dhd_prot_reset(dhd_pub_t *dhd);
+extern uint16 dhd_get_max_flow_rings(dhd_pub_t *dhd);
+
+#ifdef IDLE_TX_FLOW_MGMT
+extern int dhd_prot_flow_ring_batch_suspend_request(dhd_pub_t *dhd, uint16 *ringid, uint16 count);
+extern int dhd_prot_flow_ring_resume(dhd_pub_t *dhd, flow_ring_node_t *flow_ring_node);
+#endif /* IDLE_TX_FLOW_MGMT */
+extern int dhd_prot_init_info_rings(dhd_pub_t *dhd);
+extern int dhd_prot_init_md_rings(dhd_pub_t *dhd);
+extern int dhd_prot_check_tx_resource(dhd_pub_t *dhd);
+#else /* BCMPCIE */
+static INLINE uint32 dhd_prot_get_tx_post_bound(dhd_pub_t *dhd) { return 0; }
+static INLINE uint32 dhd_prot_get_ctrl_cpl_post_bound(dhd_pub_t *dhd) { return 0; }
+static INLINE uint32 dhd_prot_get_tx_cpl_bound(dhd_pub_t *dhd) { return 0; }
+static INLINE uint32 dhd_prot_get_rx_cpl_post_bound(dhd_pub_t *dhd) { return 0; }
+static INLINE void dhd_prot_set_tx_cpl_bound(dhd_pub_t *dhd, uint32 val) { }
+static INLINE void dhd_prot_set_rx_cpl_post_bound(dhd_pub_t *dhd, uint32 val) { }
+static INLINE void dhd_prot_set_tx_post_bound(dhd_pub_t *dhd, uint32 val) { }
+static INLINE void dhd_prot_set_ctrl_cpl_post_bound(dhd_pub_t *dhd, uint32 val) { }
+#endif /* BCMPCIE */
+
+#ifdef DHD_LB
+extern void dhd_lb_tx_compl_handler(unsigned long data);
+extern void dhd_lb_rx_compl_handler(unsigned long data);
+extern void dhd_lb_rx_process_handler(unsigned long data);
+#endif /* DHD_LB */
+extern int dhd_prot_h2d_mbdata_send_ctrlmsg(dhd_pub_t *dhd, uint32 mb_data);
+
+#ifdef BCMPCIE
+extern int dhd_prot_send_host_timestamp(dhd_pub_t *dhdp, uchar *tlv, uint16 tlv_len,
+	uint16 seq, uint16 xt_id);
+extern bool dhd_prot_data_path_tx_timestamp_logging(dhd_pub_t *dhd,  bool enable, bool set);
+extern bool dhd_prot_data_path_rx_timestamp_logging(dhd_pub_t *dhd,  bool enable, bool set);
+extern bool dhd_prot_pkt_noretry(dhd_pub_t *dhd, bool enable, bool set);
+extern bool dhd_prot_pkt_noaggr(dhd_pub_t *dhd, bool enable, bool set);
+extern bool dhd_prot_pkt_fixed_rate(dhd_pub_t *dhd, bool enable, bool set);
+#else /* BCMPCIE */
+#define dhd_prot_send_host_timestamp(a, b, c, d, e)		0
+#define dhd_prot_data_path_tx_timestamp_logging(a, b, c)	0
+#define dhd_prot_data_path_rx_timestamp_logging(a, b, c)	0
+#endif /* BCMPCIE */
+
+extern void dhd_prot_dma_indx_free(dhd_pub_t *dhd);
+
+#ifdef EWP_EDL
+int dhd_prot_init_edl_rings(dhd_pub_t *dhd);
+bool dhd_prot_process_msgbuf_edl(dhd_pub_t *dhd, uint32 *evtlog_items);
+int dhd_prot_process_edl_complete(dhd_pub_t *dhd, void *evt_decode_data);
+#endif /* EWP_EDL  */
+
+/* APIs for managing a DMA-able buffer */
+int  dhd_dma_buf_alloc(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf, uint32 buf_len);
+void dhd_dma_buf_free(dhd_pub_t *dhd, dhd_dma_buf_t *dma_buf);
+void dhd_local_buf_reset(char *buf, uint32 len);
+
+/********************************
+ * For version-string expansion *
+ */
+#if defined(BDC)
+#define DHD_PROTOCOL "bdc"
+#elif defined(CDC)
+#define DHD_PROTOCOL "cdc"
+#else
+#define DHD_PROTOCOL "unknown"
+#endif /* proto */
+
+int dhd_get_hscb_info(dhd_pub_t *dhd, void ** va, uint32 *len);
+int dhd_get_hscb_buff(dhd_pub_t *dhd, uint32 offset, uint32 length, void * buff);
+
+extern int dhd_prot_mdring_link_unlink(dhd_pub_t *dhd, int idx, bool link);
+extern int dhd_prot_mdring_linked_ring(dhd_pub_t *dhd);
+
+#ifdef DHD_MAP_LOGGING
+extern void dhd_prot_smmu_fault_dump(dhd_pub_t *dhdp);
+#endif /* DHD_MAP_LOGGING */
+
+void dhd_prot_set_ring_size_ver(dhd_pub_t *dhd, int version);
+#endif /* _dhd_proto_h_ */
diff --git a/drivers/net/wireless/ap12275/dhd_rtt.c b/drivers/net/wireless/ap12275/dhd_rtt.c
new file mode 100644
index 000000000000..3d393b7e4fa6
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_rtt.c
@@ -0,0 +1,5562 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), RTT
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+#include <typedefs.h>
+#include <osl.h>
+
+#include <epivers.h>
+#include <bcmutils.h>
+
+#include <bcmendian.h>
+#include <linuxver.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/sort.h>
+#include <dngl_stats.h>
+#include <wlioctl.h>
+#include <bcmwifi_rspec.h>
+
+#include <bcmevent.h>
+#include <dhd.h>
+#include <dhd_rtt.h>
+#include <dhd_dbg.h>
+#include <dhd_bus.h>
+#include <wldev_common.h>
+#ifdef WL_CFG80211
+#include <wl_cfg80211.h>
+#endif /* WL_CFG80211 */
+#ifdef WL_NAN
+#include <wl_cfgnan.h>
+#endif /* WL_NAN */
+
+#ifdef CUSTOM_PREFIX
+#define RTT_PRINT_PREFIX "[%s]"CUSTOM_PREFIX, OSL_GET_RTCTIME()
+#define RTT_PRINT_SYSTEM_TIME pr_cont(RTT_PRINT_PREFIX)
+#define RTT_CONS_ONLY(args)     \
+do {    \
+	RTT_PRINT_SYSTEM_TIME;  \
+	pr_cont args;           \
+} while (0)
+#else
+#define RTT_PRINT_SYSTEM_TIME
+#define RTT_CONS_ONLY(args) do { printf args;} while (0)
+#endif /* CUSTOM_PREFIX */
+
+static DEFINE_SPINLOCK(noti_list_lock);
+#define NULL_CHECK(p, s, err)  \
+	do { \
+		if (!(p)) { \
+			RTT_CONS_ONLY(("NULL POINTER (%s) : %s\n", __FUNCTION__, (s))); \
+			err = BCME_ERROR; \
+			return err; \
+		} \
+	} while (0)
+#define DHD_RTT_CHK_SET_PARAM(param, param_cnt, targets, tlvid)  \
+	do { \
+		if ((param_cnt) >= FTM_MAX_PARAMS) { \
+			DHD_RTT_ERR(("Param cnt exceeded for FTM cfg iovar\n")); \
+			err = BCME_ERROR; \
+			goto exit; \
+		} else { \
+			dhd_rtt_set_ftm_config_param((param), &(param_cnt), \
+				(targets), (tlvid)); \
+		}\
+	} while (0)
+
+#undef DHD_RTT_MEM
+#undef DHD_RTT_ERR
+#define DHD_RTT_MEM DHD_LOG_MEM
+#define DHD_RTT_ERR DHD_ERROR
+
+#define FTM_IOC_BUFSZ  2048	/* ioc buffsize for our module (> BCM_XTLV_HDR_SIZE) */
+#define FTM_AVAIL_MAX_SLOTS		32
+#define FTM_MAX_CONFIGS 10
+#define FTM_MAX_PARAMS 20
+#define FTM_DEFAULT_SESSION 1
+#define FTM_BURST_TIMEOUT_UNIT 250 /* 250 ns */
+#define FTM_INVALID -1
+#define	FTM_DEFAULT_CNT_20M		24u
+#define FTM_DEFAULT_CNT_40M		16u
+#define FTM_DEFAULT_CNT_80M		11u
+#define FTM_DEFAULT_CNT_160M		5u
+/* To handle congestion env, set max dur/timeout */
+#define FTM_MAX_BURST_DUR_TMO_MS	128u
+
+/* convenience macros */
+#define FTM_TU2MICRO(_tu) ((uint64)(_tu) << 10)
+#define FTM_MICRO2TU(_tu) ((uint64)(_tu) >> 10)
+#define FTM_TU2MILLI(_tu) ((uint32)FTM_TU2MICRO(_tu) / 1000)
+#define FTM_MICRO2MILLI(_x) ((uint32)(_x) / 1000)
+#define FTM_MICRO2SEC(_x) ((uint32)(_x) / 1000000)
+#define FTM_INTVL2NSEC(_intvl) ((uint32)ftm_intvl2nsec(_intvl))
+#define FTM_INTVL2USEC(_intvl) ((uint32)ftm_intvl2usec(_intvl))
+#define FTM_INTVL2MSEC(_intvl) (FTM_INTVL2USEC(_intvl) / 1000)
+#define FTM_INTVL2SEC(_intvl) (FTM_INTVL2USEC(_intvl) / 1000000)
+#define FTM_USECIN100MILLI(_usec) ((_usec) / 100000)
+
+/* broadcom specific set to have more accurate data */
+#define ENABLE_VHT_ACK
+
+/* CUR ETH became obsolete with this major version onwards */
+#define RTT_IOV_CUR_ETH_OBSOLETE 12
+
+/*
+ * Parallel RTT Sessions are supported
+ * with this major and minor verion onwards
+ */
+#define RTT_PARALLEL_SSNS_SUPPORTED_MAJ_VER	12
+#define RTT_PARALLEL_SSNS_SUPPORTED_MIN_VER	2
+
+/* PROXD TIMEOUT */
+#define DHD_RTT_TIMER_INTERVAL_MS		5000u
+#define DHD_NAN_RTT_TIMER_INTERVAL_MS		20000u
+
+#define DHD_NAN_RTT_MAX_SESSIONS		4u
+#define DHD_NAN_RTT_MAX_SESSIONS_LEGACY		1u
+
+struct rtt_noti_callback {
+	struct list_head list;
+	void *ctx;
+	dhd_rtt_compl_noti_fn noti_fn;
+};
+
+/* bitmask indicating which command groups; */
+typedef enum {
+	FTM_SUBCMD_FLAG_METHOD	= 0x01,	/* FTM method command */
+	FTM_SUBCMD_FLAG_SESSION = 0x02,	/* FTM session command */
+	FTM_SUBCMD_FLAG_ALL = FTM_SUBCMD_FLAG_METHOD | FTM_SUBCMD_FLAG_SESSION
+} ftm_subcmd_flag_t;
+
+/* proxd ftm config-category definition */
+typedef enum {
+	FTM_CONFIG_CAT_GENERAL = 1,	/* generial configuration */
+	FTM_CONFIG_CAT_OPTIONS = 2,	/* 'config options' */
+	FTM_CONFIG_CAT_AVAIL = 3,	/* 'config avail' */
+} ftm_config_category_t;
+
+typedef struct ftm_subcmd_info {
+	int16				version;    /* FTM version (optional) */
+	char				*name;		/* cmd-name string as cmdline input */
+	wl_proxd_cmd_t		cmdid;		/* cmd-id */
+	bcm_xtlv_unpack_cbfn_t *handler;  /* cmd response handler (optional) */
+	ftm_subcmd_flag_t	cmdflag; /* CMD flag (optional)  */
+} ftm_subcmd_info_t;
+
+typedef struct ftm_config_options_info {
+	uint32 flags;				/* wl_proxd_flags_t/wl_proxd_session_flags_t */
+	bool enable;
+} ftm_config_options_info_t;
+
+typedef struct ftm_config_param_info {
+	uint16		tlvid;	/* mapping TLV id for the item */
+	union {
+		uint32  chanspec;
+		struct ether_addr mac_addr;
+		wl_proxd_intvl_t data_intvl;
+		uint32 data32;
+		uint16 data16;
+		uint8 data8;
+		uint32 event_mask;
+	};
+} ftm_config_param_info_t;
+
+/*
+* definition for id-string mapping.
+*   This is used to map an id (can be cmd-id, tlv-id, ....) to a text-string
+*   for debug-display or cmd-log-display
+*/
+typedef struct ftm_strmap_entry {
+	int32		id;
+	char		*text;
+} ftm_strmap_entry_t;
+
+typedef struct ftm_status_map_host_entry {
+	wl_proxd_status_t proxd_status;
+	rtt_reason_t rtt_reason;
+} ftm_status_map_host_entry_t;
+
+static uint16
+rtt_result_ver(uint16 tlvid, const uint8 *p_data);
+
+static int
+dhd_rtt_convert_results_to_host_v1(rtt_result_t *rtt_result, const uint8 *p_data,
+	uint16 tlvid, uint16 len);
+
+static int
+dhd_rtt_convert_results_to_host_v2(rtt_result_t *rtt_result, const uint8 *p_data,
+	uint16 tlvid, uint16 len);
+
+static int
+dhd_rtt_convert_results_to_host_v3(rtt_result_t *rtt_result, const uint8 *p_data,
+	uint16 tlvid, uint16 len);
+
+static wifi_rate_v1
+dhd_rtt_convert_rate_to_host(uint32 ratespec);
+
+#if defined(WL_CFG80211) && defined(RTT_DEBUG)
+const char *
+ftm_cmdid_to_str(uint16 cmdid);
+#endif /* WL_CFG80211 && RTT_DEBUG */
+
+#ifdef WL_CFG80211
+static int
+dhd_rtt_start(dhd_pub_t *dhd);
+static int dhd_rtt_create_failure_result(rtt_status_info_t *rtt_status,
+	struct ether_addr *addr);
+static void dhd_rtt_handle_rtt_session_end(dhd_pub_t *dhd);
+static void dhd_rtt_timeout_work(struct work_struct *work);
+static bool dhd_rtt_get_report_header(rtt_status_info_t *rtt_status,
+	rtt_results_header_t **rtt_results_header, struct ether_addr *addr);
+static void dhd_rtt_set_ftm_config_param(ftm_config_param_info_t *ftm_params,
+	int *ftm_param_cnt, rtt_target_info_t *rtt_target, uint16 tlvid);
+static int dhd_rtt_ftm_config(dhd_pub_t *dhd, wl_proxd_session_id_t session_id,
+	void *ftm_cfg_opt, int ftm_cfg_opt_cnt, void *ftm_cfg_gen, int ftm_cfg_gen_cnt);
+#ifdef WL_NAN
+static void dhd_rtt_trigger_pending_targets_on_session_end(dhd_pub_t *dhd);
+#endif /* WL_NAN */
+#endif /* WL_CFG80211 */
+static const int burst_duration_idx[]  = {0, 0, 1, 2, 4, 8, 16, 32, 64, 128, 0, 0};
+
+/* ftm status mapping to host status */
+static const ftm_status_map_host_entry_t ftm_status_map_info[] = {
+	{WL_PROXD_E_INCOMPLETE, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_OVERRIDDEN, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_ASAP_FAILED, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_NOTSTARTED, RTT_STATUS_FAIL_NOT_SCHEDULED_YET},
+	{WL_PROXD_E_INVALIDMEAS, RTT_STATUS_FAIL_INVALID_TS},
+	{WL_PROXD_E_INCAPABLE, RTT_STATUS_FAIL_NO_CAPABILITY},
+	{WL_PROXD_E_MISMATCH, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_DUP_SESSION, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_REMOTE_FAIL, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_REMOTE_INCAPABLE, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_SCHED_FAIL, RTT_STATUS_FAIL_SCHEDULE},
+	{WL_PROXD_E_PROTO, RTT_STATUS_FAIL_PROTOCOL},
+	{WL_PROXD_E_EXPIRED, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_TIMEOUT, RTT_STATUS_FAIL_TM_TIMEOUT},
+	{WL_PROXD_E_NOACK, RTT_STATUS_FAIL_NO_RSP},
+	{WL_PROXD_E_DEFERRED, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_INVALID_SID, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_REMOTE_CANCEL, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_CANCELED, RTT_STATUS_ABORTED},
+	{WL_PROXD_E_INVALID_SESSION, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_BAD_STATE, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_ERROR, RTT_STATUS_FAILURE},
+	{WL_PROXD_E_OK, RTT_STATUS_SUCCESS}
+};
+
+static const ftm_strmap_entry_t ftm_event_type_loginfo[] = {
+	/* wl_proxd_event_type_t,		text-string */
+	{ WL_PROXD_EVENT_NONE,			"none" },
+	{ WL_PROXD_EVENT_SESSION_CREATE,	"session create" },
+	{ WL_PROXD_EVENT_SESSION_START,		"session start" },
+	{ WL_PROXD_EVENT_FTM_REQ,		"FTM req" },
+	{ WL_PROXD_EVENT_BURST_START,		"burst start" },
+	{ WL_PROXD_EVENT_BURST_END,		"burst end" },
+	{ WL_PROXD_EVENT_SESSION_END,		"session end" },
+	{ WL_PROXD_EVENT_SESSION_RESTART,	"session restart" },
+	{ WL_PROXD_EVENT_BURST_RESCHED,		"burst rescheduled" },
+	{ WL_PROXD_EVENT_SESSION_DESTROY,	"session destroy" },
+	{ WL_PROXD_EVENT_RANGE_REQ,		"range request" },
+	{ WL_PROXD_EVENT_FTM_FRAME,		"FTM frame" },
+	{ WL_PROXD_EVENT_DELAY,			"delay" },
+	{ WL_PROXD_EVENT_VS_INITIATOR_RPT,	"initiator-report " }, /* rx initiator-rpt */
+	{ WL_PROXD_EVENT_RANGING,		"ranging " },
+#ifdef WL_RTT_LCI
+	{ WL_PROXD_EVENT_LCI_MEAS_REP,		"lci report" },
+	{ WL_PROXD_EVENT_CIVIC_MEAS_REP,	"civic report" },
+#endif /* WL_RTT_LCI */
+	{ WL_PROXD_EVENT_COLLECT,		"collect" },
+	{ WL_PROXD_EVENT_MF_STATS,		"mf_stats" },
+	{ WL_PROXD_EVENT_START_WAIT,		"start-wait"}
+};
+
+/*
+* session-state --> text string mapping
+*/
+static const ftm_strmap_entry_t ftm_session_state_value_loginfo[] = {
+	/* wl_proxd_session_state_t,		text string */
+	{ WL_PROXD_SESSION_STATE_CREATED,	"created" },
+	{ WL_PROXD_SESSION_STATE_CONFIGURED,	"configured" },
+	{ WL_PROXD_SESSION_STATE_STARTED,	"started" },
+	{ WL_PROXD_SESSION_STATE_DELAY,		"delay" },
+	{ WL_PROXD_SESSION_STATE_USER_WAIT,	"user-wait" },
+	{ WL_PROXD_SESSION_STATE_SCHED_WAIT,	"sched-wait" },
+	{ WL_PROXD_SESSION_STATE_BURST,		"burst" },
+	{ WL_PROXD_SESSION_STATE_STOPPING,	"stopping" },
+	{ WL_PROXD_SESSION_STATE_ENDED,		"ended" },
+	{ WL_PROXD_SESSION_STATE_DESTROYING,	"destroying" },
+	{ WL_PROXD_SESSION_STATE_NONE,		"none" }
+};
+
+/*
+* status --> text string mapping
+*/
+static const ftm_strmap_entry_t ftm_status_value_loginfo[] = {
+	/* wl_proxd_status_t,			text-string */
+	{ WL_PROXD_E_OVERRIDDEN,		"overridden" },
+	{ WL_PROXD_E_ASAP_FAILED,		"ASAP failed" },
+	{ WL_PROXD_E_NOTSTARTED,		"not started" },
+	{ WL_PROXD_E_INVALIDMEAS,		"invalid measurement" },
+	{ WL_PROXD_E_INCAPABLE,			"incapable" },
+	{ WL_PROXD_E_MISMATCH,			"mismatch"},
+	{ WL_PROXD_E_DUP_SESSION,		"dup session" },
+	{ WL_PROXD_E_REMOTE_FAIL,		"remote fail" },
+	{ WL_PROXD_E_REMOTE_INCAPABLE,		"remote incapable" },
+	{ WL_PROXD_E_SCHED_FAIL,		"sched failure" },
+	{ WL_PROXD_E_PROTO,			"protocol error" },
+	{ WL_PROXD_E_EXPIRED,			"expired" },
+	{ WL_PROXD_E_TIMEOUT,			"timeout" },
+	{ WL_PROXD_E_NOACK,			"no ack" },
+	{ WL_PROXD_E_DEFERRED,			"deferred" },
+	{ WL_PROXD_E_INVALID_SID,		"invalid session id" },
+	{ WL_PROXD_E_REMOTE_CANCEL,		"remote cancel" },
+	{ WL_PROXD_E_CANCELED,			"canceled" },
+	{ WL_PROXD_E_INVALID_SESSION,		"invalid session" },
+	{ WL_PROXD_E_BAD_STATE,			"bad state" },
+	{ WL_PROXD_E_ERROR,			"error" },
+	{ WL_PROXD_E_OK,			"OK" }
+};
+
+/*
+* time interval unit --> text string mapping
+*/
+static const ftm_strmap_entry_t ftm_tmu_value_loginfo[] = {
+	/* wl_proxd_tmu_t,		text-string */
+	{ WL_PROXD_TMU_TU,		"TU" },
+	{ WL_PROXD_TMU_SEC,		"sec" },
+	{ WL_PROXD_TMU_MILLI_SEC,	"ms" },
+	{ WL_PROXD_TMU_MICRO_SEC,	"us" },
+	{ WL_PROXD_TMU_NANO_SEC,	"ns" },
+	{ WL_PROXD_TMU_PICO_SEC,	"ps" }
+};
+
+struct ieee_80211_mcs_rate_info {
+	uint8 constellation_bits;
+	uint8 coding_q;
+	uint8 coding_d;
+};
+
+static const struct ieee_80211_mcs_rate_info wl_mcs_info[] = {
+	{ 1, 1, 2 }, /* MCS  0: MOD: BPSK,   CR 1/2 */
+	{ 2, 1, 2 }, /* MCS  1: MOD: QPSK,   CR 1/2 */
+	{ 2, 3, 4 }, /* MCS  2: MOD: QPSK,   CR 3/4 */
+	{ 4, 1, 2 }, /* MCS  3: MOD: 16QAM,  CR 1/2 */
+	{ 4, 3, 4 }, /* MCS  4: MOD: 16QAM,  CR 3/4 */
+	{ 6, 2, 3 }, /* MCS  5: MOD: 64QAM,  CR 2/3 */
+	{ 6, 3, 4 }, /* MCS  6: MOD: 64QAM,  CR 3/4 */
+	{ 6, 5, 6 }, /* MCS  7: MOD: 64QAM,  CR 5/6 */
+	{ 8, 3, 4 }, /* MCS  8: MOD: 256QAM, CR 3/4 */
+	{ 8, 5, 6 }  /* MCS  9: MOD: 256QAM, CR 5/6 */
+};
+
+/**
+ * Returns the rate in [Kbps] units for a caller supplied MCS/bandwidth/Nss/Sgi combination.
+ *     'mcs' : a *single* spatial stream MCS (11n or 11ac)
+ */
+uint
+rate_mcs2rate(uint mcs, uint nss, uint bw, int sgi)
+{
+	const int ksps = 250; /* kilo symbols per sec, 4 us sym */
+	const int Nsd_20MHz = 52;
+	const int Nsd_40MHz = 108;
+	const int Nsd_80MHz = 234;
+	const int Nsd_160MHz = 468;
+	uint rate;
+
+	if (mcs == 32) {
+		/* just return fixed values for mcs32 instead of trying to parametrize */
+		rate = (sgi == 0) ? 6000 : 6778;
+	} else if (mcs <= 9) {
+		/* This calculation works for 11n HT and 11ac VHT if the HT mcs values
+		 * are decomposed into a base MCS = MCS % 8, and Nss = 1 + MCS / 8.
+		 * That is, HT MCS 23 is a base MCS = 7, Nss = 3
+		 */
+
+		/* find the number of complex numbers per symbol */
+		if (RSPEC_IS20MHZ(bw)) {
+			/* 4360 TODO: eliminate Phy const in rspec bw, then just compare
+			 * as in 80 and 160 case below instead of RSPEC_IS20MHZ(bw)
+			 */
+			rate = Nsd_20MHz;
+		} else if (RSPEC_IS40MHZ(bw)) {
+			/* 4360 TODO: eliminate Phy const in rspec bw, then just compare
+			 * as in 80 and 160 case below instead of RSPEC_IS40MHZ(bw)
+			 */
+			rate = Nsd_40MHz;
+		} else if (bw == WL_RSPEC_BW_80MHZ) {
+			rate = Nsd_80MHz;
+		} else if (bw == WL_RSPEC_BW_160MHZ) {
+			rate = Nsd_160MHz;
+		} else {
+			rate = 0;
+		}
+
+		/* multiply by bits per number from the constellation in use */
+		rate = rate * wl_mcs_info[mcs].constellation_bits;
+
+		/* adjust for the number of spatial streams */
+		rate = rate * nss;
+
+		/* adjust for the coding rate given as a quotient and divisor */
+		rate = (rate * wl_mcs_info[mcs].coding_q) / wl_mcs_info[mcs].coding_d;
+
+		/* multiply by Kilo symbols per sec to get Kbps */
+		rate = rate * ksps;
+
+		/* adjust the symbols per sec for SGI
+		 * symbol duration is 4 us without SGI, and 3.6 us with SGI,
+		 * so ratio is 10 / 9
+		 */
+		if (sgi) {
+			/* add 4 for rounding of division by 9 */
+			rate = ((rate * 10) + 4) / 9;
+		}
+	} else {
+		rate = 0;
+	}
+
+	return rate;
+} /* wlc_rate_mcs2rate */
+
+/** take a well formed ratespec_t arg and return phy rate in [Kbps] units */
+static uint32
+rate_rspec2rate(uint32 rspec)
+{
+	int rate = 0;
+
+	if (RSPEC_ISLEGACY(rspec)) {
+		rate = 500 * (rspec & WL_RSPEC_RATE_MASK);
+	} else if (RSPEC_ISHT(rspec)) {
+		uint mcs = (rspec & WL_RSPEC_RATE_MASK);
+
+		if (mcs == 32) {
+			rate = rate_mcs2rate(mcs, 1, WL_RSPEC_BW_40MHZ, RSPEC_ISSGI(rspec));
+		} else {
+			uint nss = 1 + (mcs / 8);
+			mcs = mcs % 8;
+			rate = rate_mcs2rate(mcs, nss, RSPEC_BW(rspec), RSPEC_ISSGI(rspec));
+		}
+	} else if (RSPEC_ISVHT(rspec)) {
+		uint mcs = (rspec & WL_RSPEC_VHT_MCS_MASK);
+		uint nss = (rspec & WL_RSPEC_VHT_NSS_MASK) >> WL_RSPEC_VHT_NSS_SHIFT;
+		if (mcs > 9 || nss > 8) {
+			DHD_RTT(("%s: Invalid mcs %d or nss %d\n", __FUNCTION__, mcs, nss));
+			goto exit;
+		}
+
+		rate = rate_mcs2rate(mcs, nss, RSPEC_BW(rspec), RSPEC_ISSGI(rspec));
+	} else {
+		DHD_RTT(("%s: wrong rspec:%d\n", __FUNCTION__, rspec));
+	}
+exit:
+	return rate;
+}
+
+char resp_buf[WLC_IOCTL_SMLEN];
+
+static uint64
+ftm_intvl2nsec(const wl_proxd_intvl_t *intvl)
+{
+	uint64 ret;
+	ret = intvl->intvl;
+	switch (intvl->tmu) {
+	case WL_PROXD_TMU_TU:			ret = FTM_TU2MICRO(ret) * 1000; break;
+	case WL_PROXD_TMU_SEC:			ret *= 1000000000; break;
+	case WL_PROXD_TMU_MILLI_SEC:	ret *= 1000000; break;
+	case WL_PROXD_TMU_MICRO_SEC:	ret *= 1000; break;
+	case WL_PROXD_TMU_PICO_SEC:		ret = intvl->intvl / 1000; break;
+	case WL_PROXD_TMU_NANO_SEC:		/* fall through */
+	default:						break;
+	}
+	return ret;
+}
+
+uint64
+ftm_intvl2usec(const wl_proxd_intvl_t *intvl)
+{
+	uint64 ret;
+	ret = intvl->intvl;
+	switch (intvl->tmu) {
+	case WL_PROXD_TMU_TU:			ret = FTM_TU2MICRO(ret); break;
+	case WL_PROXD_TMU_SEC:			ret *= 1000000; break;
+	case WL_PROXD_TMU_NANO_SEC:		ret = intvl->intvl / 1000; break;
+	case WL_PROXD_TMU_PICO_SEC:		ret = intvl->intvl / 1000000; break;
+	case WL_PROXD_TMU_MILLI_SEC:	ret *= 1000; break;
+	case WL_PROXD_TMU_MICRO_SEC:	/* fall through */
+	default:						break;
+	}
+	return ret;
+}
+
+/*
+* lookup 'id' (as a key) from a fw status to host map table
+* if found, return the corresponding reason code
+*/
+
+static rtt_reason_t
+ftm_get_statusmap_info(wl_proxd_status_t id, const ftm_status_map_host_entry_t *p_table,
+	uint32 num_entries)
+{
+	int i;
+	const ftm_status_map_host_entry_t *p_entry;
+	/* scan thru the table till end */
+	p_entry = p_table;
+	for (i = 0; i < (int) num_entries; i++)
+	{
+		if (p_entry->proxd_status == id) {
+			return p_entry->rtt_reason;
+		}
+		p_entry++;		/* next entry */
+	}
+	return RTT_STATUS_FAILURE; /* not found */
+}
+
+/*
+* lookup 'id' (as a key) from a table
+* if found, return the entry pointer, otherwise return NULL
+*/
+static const ftm_strmap_entry_t*
+ftm_get_strmap_info(int32 id, const ftm_strmap_entry_t *p_table, uint32 num_entries)
+{
+	int i;
+	const ftm_strmap_entry_t *p_entry;
+
+	/* scan thru the table till end */
+	p_entry = p_table;
+	for (i = 0; i < (int) num_entries; i++)
+	{
+		if (p_entry->id == id)
+			return p_entry;
+		p_entry++;		/* next entry */
+	}
+	return NULL;			/* not found */
+}
+
+/*
+* map enum to a text-string for display, this function is called by the following:
+* For debug/trace:
+*     ftm_[cmdid|tlvid]_to_str()
+* For TLV-output log for 'get' commands
+*     ftm_[method|tmu|caps|status|state]_value_to_logstr()
+* Input:
+*     pTable -- point to a 'enum to string' table.
+*/
+static const char *
+ftm_map_id_to_str(int32 id, const ftm_strmap_entry_t *p_table, uint32 num_entries)
+{
+	const ftm_strmap_entry_t*p_entry = ftm_get_strmap_info(id, p_table, num_entries);
+	if (p_entry)
+		return (p_entry->text);
+
+	return "invalid";
+}
+
+#if defined(WL_CFG80211) && defined(RTT_DEBUG)
+/* define entry, e.g. { WL_PROXD_CMD_xxx, "WL_PROXD_CMD_xxx" } */
+#define DEF_STRMAP_ENTRY(id) { (id), #id }
+
+/* ftm cmd-id mapping */
+static const ftm_strmap_entry_t ftm_cmdid_map[] = {
+	/* {wl_proxd_cmd_t(WL_PROXD_CMD_xxx), "WL_PROXD_CMD_xxx" }, */
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_NONE),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_VERSION),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_ENABLE),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_DISABLE),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_CONFIG),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_START_SESSION),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_BURST_REQUEST),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_STOP_SESSION),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_DELETE_SESSION),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_RESULT),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_INFO),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_STATUS),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_SESSIONS),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_COUNTERS),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_CLEAR_COUNTERS),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_COLLECT),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_TUNE),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_DUMP),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_START_RANGING),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_STOP_RANGING),
+	DEF_STRMAP_ENTRY(WL_PROXD_CMD_GET_RANGING_INFO),
+};
+
+/*
+* map a ftm cmd-id to a text-string for display
+*/
+const char *
+ftm_cmdid_to_str(uint16 cmdid)
+{
+	return ftm_map_id_to_str((int32) cmdid, &ftm_cmdid_map[0], ARRAYSIZE(ftm_cmdid_map));
+}
+#endif /* WL_CFG80211 && RTT_DEBUG */
+
+/*
+* convert BCME_xxx error codes into related error strings
+* note, bcmerrorstr() defined in bcmutils is for BCMDRIVER only,
+*       this duplicate copy is for WL access and may need to clean up later
+*/
+static const char *ftm_bcmerrorstrtable[] = BCMERRSTRINGTABLE;
+static const char *
+ftm_status_value_to_logstr(wl_proxd_status_t status)
+{
+	static char ftm_msgbuf_status_undef[32];
+	const ftm_strmap_entry_t *p_loginfo;
+	int bcmerror;
+
+	/* check if within BCME_xxx error range */
+	bcmerror = (int) status;
+	if (VALID_BCMERROR(bcmerror))
+		return ftm_bcmerrorstrtable[-bcmerror];
+
+	/* otherwise, look for 'proxd ftm status' range */
+	p_loginfo = ftm_get_strmap_info((int32) status,
+		&ftm_status_value_loginfo[0], ARRAYSIZE(ftm_status_value_loginfo));
+	if (p_loginfo)
+		return p_loginfo->text;
+
+	/* report for 'out of range' FTM-status error code */
+	memset(ftm_msgbuf_status_undef, 0, sizeof(ftm_msgbuf_status_undef));
+	snprintf(ftm_msgbuf_status_undef, sizeof(ftm_msgbuf_status_undef),
+		"Undefined status %d", status);
+	return &ftm_msgbuf_status_undef[0];
+}
+
+static const char *
+ftm_tmu_value_to_logstr(wl_proxd_tmu_t tmu)
+{
+	return ftm_map_id_to_str((int32)tmu,
+		&ftm_tmu_value_loginfo[0], ARRAYSIZE(ftm_tmu_value_loginfo));
+}
+
+static const ftm_strmap_entry_t*
+ftm_get_event_type_loginfo(wl_proxd_event_type_t	event_type)
+{
+	/* look up 'event-type' from a predefined table  */
+	return ftm_get_strmap_info((int32) event_type,
+		ftm_event_type_loginfo, ARRAYSIZE(ftm_event_type_loginfo));
+}
+
+static const char *
+ftm_session_state_value_to_logstr(wl_proxd_session_state_t state)
+{
+	return ftm_map_id_to_str((int32)state, &ftm_session_state_value_loginfo[0],
+		ARRAYSIZE(ftm_session_state_value_loginfo));
+}
+
+#ifdef WL_CFG80211
+/*
+* send 'proxd' iovar for all ftm get-related commands
+*/
+static int
+rtt_do_get_ioctl(dhd_pub_t *dhd, wl_proxd_iov_t *p_proxd_iov, uint16 proxd_iovsize,
+		ftm_subcmd_info_t *p_subcmd_info)
+{
+
+	wl_proxd_iov_t *p_iovresp = (wl_proxd_iov_t *)resp_buf;
+	int status;
+	int tlvs_len;
+	/*  send getbuf proxd iovar */
+	status = dhd_getiovar(dhd, 0, "proxd", (char *)p_proxd_iov,
+			proxd_iovsize, (char **)&p_iovresp, WLC_IOCTL_SMLEN);
+	if (status != BCME_OK) {
+		DHD_RTT_ERR(("%s: failed to send getbuf proxd iovar (CMD ID : %d), status=%d\n",
+			__FUNCTION__, p_subcmd_info->cmdid, status));
+		return status;
+	}
+	if (p_subcmd_info->cmdid == WL_PROXD_CMD_GET_VERSION) {
+		p_subcmd_info->version = ltoh16(p_iovresp->version);
+		DHD_RTT(("ftm version: 0x%x\n", ltoh16(p_iovresp->version)));
+		goto exit;
+	}
+
+	tlvs_len = ltoh16(p_iovresp->len) - WL_PROXD_IOV_HDR_SIZE;
+	if (tlvs_len < 0) {
+		DHD_RTT_ERR(("%s: alert, p_iovresp->len(%d) should not be smaller than %d\n",
+			__FUNCTION__, ltoh16(p_iovresp->len), (int) WL_PROXD_IOV_HDR_SIZE));
+		tlvs_len = 0;
+	}
+
+	if (tlvs_len > 0 && p_subcmd_info->handler) {
+		/* unpack TLVs and invokes the cbfn for processing */
+		status = bcm_unpack_xtlv_buf(p_proxd_iov, (uint8 *)p_iovresp->tlvs,
+				tlvs_len, BCM_XTLV_OPTION_ALIGN32, p_subcmd_info->handler);
+	}
+exit:
+	return status;
+}
+
+static wl_proxd_iov_t *
+rtt_alloc_getset_buf(dhd_pub_t *dhd, wl_proxd_method_t method, wl_proxd_session_id_t session_id,
+	wl_proxd_cmd_t cmdid, uint16 tlvs_bufsize, uint16 *p_out_bufsize)
+{
+	uint16 proxd_iovsize;
+	wl_proxd_tlv_t *p_tlv;
+	wl_proxd_iov_t *p_proxd_iov = (wl_proxd_iov_t *) NULL;
+
+	*p_out_bufsize = 0;	/* init */
+	/* calculate the whole buffer size, including one reserve-tlv entry in the header */
+	proxd_iovsize = sizeof(wl_proxd_iov_t) + tlvs_bufsize;
+
+	p_proxd_iov = (wl_proxd_iov_t *)MALLOCZ(dhd->osh, proxd_iovsize);
+	if (p_proxd_iov == NULL) {
+		DHD_RTT_ERR(("error: failed to allocate %d bytes of memory\n", proxd_iovsize));
+		return NULL;
+	}
+
+	/* setup proxd-FTM-method iovar header */
+	p_proxd_iov->version = htol16(WL_PROXD_API_VERSION_3);
+	p_proxd_iov->len = htol16(proxd_iovsize); /* caller may adjust it based on #of TLVs */
+	p_proxd_iov->cmd = htol16(cmdid);
+	p_proxd_iov->method = htol16(method);
+	p_proxd_iov->sid = htol16(session_id);
+
+	/* initialize the reserved/dummy-TLV in iovar header */
+	p_tlv = p_proxd_iov->tlvs;
+	p_tlv->id = htol16(WL_PROXD_TLV_ID_NONE);
+	p_tlv->len = htol16(0);
+
+	*p_out_bufsize = proxd_iovsize;	/* for caller's reference */
+
+	return p_proxd_iov;
+}
+
+static int
+dhd_rtt_common_get_handler(dhd_pub_t *dhd, ftm_subcmd_info_t *p_subcmd_info,
+		wl_proxd_method_t method,
+		wl_proxd_session_id_t session_id)
+{
+	int status = BCME_OK;
+	uint16 proxd_iovsize = 0;
+	wl_proxd_iov_t *p_proxd_iov;
+#ifdef RTT_DEBUG
+	DHD_RTT(("enter %s: method=%d, session_id=%d, cmdid=%d(%s)\n",
+		__FUNCTION__, method, session_id, p_subcmd_info->cmdid,
+		ftm_cmdid_to_str(p_subcmd_info->cmdid)));
+#endif
+	/* alloc mem for ioctl headr + reserved 0 bufsize for tlvs (initialize to zero) */
+	p_proxd_iov = rtt_alloc_getset_buf(dhd, method, session_id, p_subcmd_info->cmdid,
+		0, &proxd_iovsize);
+
+	if (p_proxd_iov == NULL)
+		return BCME_NOMEM;
+
+	status = rtt_do_get_ioctl(dhd, p_proxd_iov, proxd_iovsize, p_subcmd_info);
+
+	if (status != BCME_OK) {
+		DHD_RTT(("%s failed: status=%d\n", __FUNCTION__, status));
+	}
+	MFREE(dhd->osh, p_proxd_iov, proxd_iovsize);
+	return status;
+}
+
+/*
+* common handler for set-related proxd method commands which require no TLV as input
+*   wl proxd ftm [session-id] <set-subcmd>
+* e.g.
+*   wl proxd ftm enable -- to enable ftm
+*   wl proxd ftm disable -- to disable ftm
+*   wl proxd ftm <session-id> start -- to start a specified session
+*   wl proxd ftm <session-id> stop  -- to cancel a specified session;
+*                                    state is maintained till session is delete.
+*   wl proxd ftm <session-id> delete -- to delete a specified session
+*   wl proxd ftm [<session-id>] clear-counters -- to clear counters
+*   wl proxd ftm <session-id> burst-request -- on initiator: to send burst request;
+*                                              on target: send FTM frame
+*   wl proxd ftm <session-id> collect
+*   wl proxd ftm tune     (TBD)
+*/
+static int
+dhd_rtt_common_set_handler(dhd_pub_t *dhd, const ftm_subcmd_info_t *p_subcmd_info,
+	wl_proxd_method_t method, wl_proxd_session_id_t session_id)
+{
+	uint16 proxd_iovsize;
+	wl_proxd_iov_t *p_proxd_iov;
+	int ret;
+
+#ifdef RTT_DEBUG
+	DHD_RTT(("enter %s: method=%d, session_id=%d, cmdid=%d(%s)\n",
+		__FUNCTION__, method, session_id, p_subcmd_info->cmdid,
+		ftm_cmdid_to_str(p_subcmd_info->cmdid)));
+#endif
+
+	/* allocate and initialize a temp buffer for 'set proxd' iovar */
+	proxd_iovsize = 0;
+	p_proxd_iov = rtt_alloc_getset_buf(dhd, method, session_id, p_subcmd_info->cmdid,
+			0, &proxd_iovsize);		/* no TLV */
+	if (p_proxd_iov == NULL)
+		return BCME_NOMEM;
+
+	/* no TLV to pack, simply issue a set-proxd iovar */
+	ret = dhd_iovar(dhd, 0, "proxd", (char *)p_proxd_iov, proxd_iovsize, NULL, 0, TRUE);
+#ifdef RTT_DEBUG
+	if (ret != BCME_OK) {
+		DHD_RTT(("error: IOVAR failed, status=%d\n", ret));
+	}
+#endif
+	/* clean up */
+	MFREE(dhd->osh, p_proxd_iov, proxd_iovsize);
+
+	return ret;
+}
+#endif /* WL_CFG80211 */
+
+/* gets the length and returns the version
+ * of the wl_proxd_collect_event_t version
+ */
+static uint
+rtt_collect_data_event_ver(uint16 len)
+{
+	if (len > sizeof(wl_proxd_collect_event_data_v3_t)) {
+		return WL_PROXD_COLLECT_EVENT_DATA_VERSION_MAX;
+	} else if (len == sizeof(wl_proxd_collect_event_data_v4_t)) {
+		return WL_PROXD_COLLECT_EVENT_DATA_VERSION_4;
+	} else if (len == sizeof(wl_proxd_collect_event_data_v3_t)) {
+		return WL_PROXD_COLLECT_EVENT_DATA_VERSION_3;
+	} else if (len == sizeof(wl_proxd_collect_event_data_v2_t)) {
+		return WL_PROXD_COLLECT_EVENT_DATA_VERSION_2;
+	} else {
+		return WL_PROXD_COLLECT_EVENT_DATA_VERSION_1;
+	}
+}
+
+static void
+rtt_collect_event_data_display(uint8 ver, void *ctx, const uint8 *p_data, uint16 len)
+{
+	int i;
+	wl_proxd_collect_event_data_v1_t *p_collect_data_v1 = NULL;
+	wl_proxd_collect_event_data_v2_t *p_collect_data_v2 = NULL;
+	wl_proxd_collect_event_data_v3_t *p_collect_data_v3 = NULL;
+	wl_proxd_collect_event_data_v4_t *p_collect_data_v4 = NULL;
+
+	if (!ctx || !p_data) {
+		return;
+	}
+
+	switch (ver) {
+	case WL_PROXD_COLLECT_EVENT_DATA_VERSION_1:
+		DHD_RTT(("\tVERSION_1\n"));
+		memcpy(ctx, p_data, sizeof(wl_proxd_collect_event_data_v1_t));
+		p_collect_data_v1 = (wl_proxd_collect_event_data_v1_t *)ctx;
+		DHD_RTT(("\tH_RX\n"));
+		for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+			p_collect_data_v1->H_RX[i] = ltoh32_ua(&p_collect_data_v1->H_RX[i]);
+			DHD_RTT(("\t%u\n", p_collect_data_v1->H_RX[i]));
+		}
+		DHD_RTT(("\n"));
+		DHD_RTT(("\tH_LB\n"));
+		for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+			p_collect_data_v1->H_LB[i] = ltoh32_ua(&p_collect_data_v1->H_LB[i]);
+			DHD_RTT(("\t%u\n", p_collect_data_v1->H_LB[i]));
+		}
+		DHD_RTT(("\n"));
+		DHD_RTT(("\tri_rr\n"));
+		for (i = 0; i < FTM_TPK_RI_RR_LEN; i++) {
+			DHD_RTT(("\t%u\n", p_collect_data_v1->ri_rr[i]));
+		}
+		p_collect_data_v1->phy_err_mask = ltoh32_ua(&p_collect_data_v1->phy_err_mask);
+		DHD_RTT(("\tphy_err_mask=0x%x\n", p_collect_data_v1->phy_err_mask));
+		break;
+	case WL_PROXD_COLLECT_EVENT_DATA_VERSION_2:
+		memcpy(ctx, p_data, sizeof(wl_proxd_collect_event_data_v2_t));
+		p_collect_data_v2 = (wl_proxd_collect_event_data_v2_t *)ctx;
+		DHD_RTT(("\tH_RX\n"));
+		for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+			p_collect_data_v2->H_RX[i] = ltoh32_ua(&p_collect_data_v2->H_RX[i]);
+			DHD_RTT(("\t%u\n", p_collect_data_v2->H_RX[i]));
+		}
+		DHD_RTT(("\n"));
+		DHD_RTT(("\tH_LB\n"));
+		for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+			p_collect_data_v2->H_LB[i] = ltoh32_ua(&p_collect_data_v2->H_LB[i]);
+			DHD_RTT(("\t%u\n", p_collect_data_v2->H_LB[i]));
+		}
+		DHD_RTT(("\n"));
+		DHD_RTT(("\tri_rr\n"));
+		for (i = 0; i < FTM_TPK_RI_RR_LEN_SECURE_2_0; i++) {
+			DHD_RTT(("\t%u\n", p_collect_data_v2->ri_rr[i]));
+		}
+		p_collect_data_v2->phy_err_mask = ltoh32_ua(&p_collect_data_v2->phy_err_mask);
+		DHD_RTT(("\tphy_err_mask=0x%x\n", p_collect_data_v2->phy_err_mask));
+		break;
+	case WL_PROXD_COLLECT_EVENT_DATA_VERSION_3:
+		memcpy(ctx, p_data, sizeof(wl_proxd_collect_event_data_v3_t));
+		p_collect_data_v3 = (wl_proxd_collect_event_data_v3_t *)ctx;
+		switch (p_collect_data_v3->version) {
+		case WL_PROXD_COLLECT_EVENT_DATA_VERSION_3:
+			if (p_collect_data_v3->length !=
+				(len - OFFSETOF(wl_proxd_collect_event_data_v3_t, H_LB))) {
+				DHD_RTT(("\tversion/length mismatch\n"));
+				break;
+			}
+			DHD_RTT(("\tH_RX\n"));
+			for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+				p_collect_data_v3->H_RX[i] =
+					ltoh32_ua(&p_collect_data_v3->H_RX[i]);
+				DHD_RTT(("\t%u\n", p_collect_data_v3->H_RX[i]));
+			}
+			DHD_RTT(("\n"));
+			DHD_RTT(("\tH_LB\n"));
+			for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+				p_collect_data_v3->H_LB[i] =
+					ltoh32_ua(&p_collect_data_v3->H_LB[i]);
+				DHD_RTT(("\t%u\n", p_collect_data_v3->H_LB[i]));
+			}
+			DHD_RTT(("\n"));
+			DHD_RTT(("\tri_rr\n"));
+			for (i = 0; i < FTM_TPK_RI_RR_LEN_SECURE_2_0; i++) {
+				DHD_RTT(("\t%u\n", p_collect_data_v3->ri_rr[i]));
+			}
+			p_collect_data_v3->phy_err_mask =
+				ltoh32_ua(&p_collect_data_v3->phy_err_mask);
+			DHD_RTT(("\tphy_err_mask=0x%x\n", p_collect_data_v3->phy_err_mask));
+			break;
+		/* future case */
+		}
+		break;
+	case WL_PROXD_COLLECT_EVENT_DATA_VERSION_4:
+		memcpy(ctx, p_data, sizeof(wl_proxd_collect_event_data_v4_t));
+		p_collect_data_v4 = (wl_proxd_collect_event_data_v4_t *)ctx;
+		switch (p_collect_data_v4->version) {
+		case WL_PROXD_COLLECT_EVENT_DATA_VERSION_4:
+			if (p_collect_data_v4->length !=
+				(len - OFFSETOF(wl_proxd_collect_event_data_v4_t, H_LB))) {
+				DHD_RTT(("\tversion/length mismatch\n"));
+				break;
+			}
+			DHD_RTT(("\tH_RX\n"));
+			for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+				p_collect_data_v4->H_RX[i] =
+					ltoh32_ua(&p_collect_data_v4->H_RX[i]);
+				DHD_RTT(("\t%u\n", p_collect_data_v4->H_RX[i]));
+			}
+			DHD_RTT(("\n"));
+			DHD_RTT(("\tH_LB\n"));
+			for (i = 0; i < K_TOF_COLLECT_H_SIZE_20MHZ; i++) {
+				p_collect_data_v4->H_LB[i] =
+					ltoh32_ua(&p_collect_data_v4->H_LB[i]);
+				DHD_RTT(("\t%u\n", p_collect_data_v4->H_LB[i]));
+			}
+			DHD_RTT(("\n"));
+			DHD_RTT(("\tri_rr\n"));
+			for (i = 0; i < FTM_TPK_RI_RR_LEN_SECURE_2_0_5G; i++) {
+				DHD_RTT(("\t%u\n", p_collect_data_v4->ri_rr[i]));
+			}
+			p_collect_data_v4->phy_err_mask =
+				ltoh32_ua(&p_collect_data_v4->phy_err_mask);
+			DHD_RTT(("\tphy_err_mask=0x%x\n", p_collect_data_v4->phy_err_mask));
+			break;
+		/* future case */
+		}
+		break;
+	}
+}
+
+static uint16
+rtt_result_ver(uint16 tlvid, const uint8 *p_data)
+{
+	uint16 ret = BCME_OK;
+	const wl_proxd_rtt_result_v2_t *r_v2 = NULL;
+	const wl_proxd_rtt_result_v3_t *r_v3 = NULL;
+
+	switch (tlvid) {
+	case WL_PROXD_TLV_ID_RTT_RESULT:
+		BCM_REFERENCE(p_data);
+		ret = WL_PROXD_RTT_RESULT_VERSION_1;
+		break;
+	case WL_PROXD_TLV_ID_RTT_RESULT_V2:
+		if (p_data) {
+			r_v2 = (const wl_proxd_rtt_result_v2_t *)p_data;
+			if (r_v2->version == WL_PROXD_RTT_RESULT_VERSION_2) {
+				ret = WL_PROXD_RTT_RESULT_VERSION_2;
+			}
+		}
+		break;
+	case WL_PROXD_TLV_ID_RTT_RESULT_V3:
+		if (p_data) {
+			r_v3 = (const wl_proxd_rtt_result_v3_t *)p_data;
+			if (r_v3->version == WL_PROXD_RTT_RESULT_VERSION_3) {
+				ret = WL_PROXD_RTT_RESULT_VERSION_3;
+			}
+		}
+		break;
+	default:
+		DHD_RTT_ERR(("%s: > Unsupported TLV ID %d\n",
+			__FUNCTION__, tlvid));
+		break;
+	}
+	return ret;
+}
+
+/* pretty hex print a contiguous buffer */
+static void
+rtt_prhex(const char *msg, const uint8 *buf, uint nbytes)
+{
+	char line[128], *p;
+	int len = sizeof(line);
+	int nchar;
+	uint i;
+
+	if (msg && (msg[0] != '\0'))
+		DHD_RTT(("%s:\n", msg));
+
+	p = line;
+	for (i = 0; i < nbytes; i++) {
+		if (i % 16 == 0) {
+			nchar = snprintf(p, len, "  %04d: ", i);	/* line prefix */
+			p += nchar;
+			len -= nchar;
+		}
+		if (len > 0) {
+			nchar = snprintf(p, len, "%02x ", buf[i]);
+			p += nchar;
+			len -= nchar;
+		}
+
+		if (i % 16 == 15) {
+			DHD_RTT(("%s\n", line));	/* flush line */
+			p = line;
+			len = sizeof(line);
+		}
+	}
+
+	/* flush last partial line */
+	if (p != line)
+		DHD_RTT(("%s\n", line));
+}
+
+static int
+rtt_unpack_xtlv_cbfn(void *ctx, const uint8 *p_data, uint16 tlvid, uint16 len)
+{
+	int ret = BCME_OK;
+	int i;
+	wl_proxd_ftm_session_status_t *p_data_info = NULL;
+	uint32 chan_data_entry = 0;
+	uint16 expected_rtt_result_ver = 0;
+#ifdef WL_RTT_LCI
+	bcm_xtlv_t *tlv = NULL;
+#endif /* WL_RTT_LCI */
+
+	BCM_REFERENCE(p_data_info);
+
+	switch (tlvid) {
+	case WL_PROXD_TLV_ID_RTT_RESULT:
+	case WL_PROXD_TLV_ID_RTT_RESULT_V2:
+	case WL_PROXD_TLV_ID_RTT_RESULT_V3:
+		DHD_RTT(("WL_PROXD_TLV_ID_RTT_RESULT\n"));
+		expected_rtt_result_ver = rtt_result_ver(tlvid, p_data);
+		switch (expected_rtt_result_ver) {
+		case WL_PROXD_RTT_RESULT_VERSION_1:
+			ret = dhd_rtt_convert_results_to_host_v1((rtt_result_t *)ctx,
+					p_data, tlvid, len);
+			break;
+		case WL_PROXD_RTT_RESULT_VERSION_2:
+			ret = dhd_rtt_convert_results_to_host_v2((rtt_result_t *)ctx,
+					p_data, tlvid, len);
+			break;
+		case WL_PROXD_RTT_RESULT_VERSION_3:
+			ret = dhd_rtt_convert_results_to_host_v3((rtt_result_t *)ctx,
+					p_data, tlvid, len);
+			break;
+		default:
+			DHD_RTT_ERR((" > Unsupported RTT_RESULT version\n"));
+			ret = BCME_UNSUPPORTED;
+			break;
+		}
+		break;
+	case WL_PROXD_TLV_ID_SESSION_STATUS:
+		DHD_RTT(("WL_PROXD_TLV_ID_SESSION_STATUS\n"));
+		memcpy(ctx, p_data, sizeof(wl_proxd_ftm_session_status_t));
+		p_data_info = (wl_proxd_ftm_session_status_t *)ctx;
+		p_data_info->sid = ltoh16_ua(&p_data_info->sid);
+		p_data_info->state = ltoh16_ua(&p_data_info->state);
+		p_data_info->status = ltoh32_ua(&p_data_info->status);
+		p_data_info->burst_num = ltoh16_ua(&p_data_info->burst_num);
+		p_data_info->core_info = ltoh16_ua(&p_data_info->core_info);
+		DHD_RTT(("\tsid=%u, state=%d, status=%d, burst_num=%u\n",
+			p_data_info->sid, p_data_info->state,
+			p_data_info->status, p_data_info->burst_num));
+		DHD_RTT(("\tnum_cores=%u, core=%u\n", (p_data_info->core_info & 0xFF),
+			(p_data_info->core_info >> 8u & 0xFF)));
+
+		break;
+	case WL_PROXD_TLV_ID_COLLECT_DATA:
+		DHD_RTT(("WL_PROXD_TLV_ID_COLLECT_DATA\n"));
+		/* we do not have handle to wl in the context of
+		 * xtlv callback without changing the xtlv API.
+		 */
+		rtt_collect_event_data_display(
+			rtt_collect_data_event_ver(len),
+			ctx, p_data, len);
+		break;
+	case WL_PROXD_TLV_ID_COLLECT_CHAN_DATA:
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		DHD_RTT(("WL_PROXD_TLV_ID_COLLECT_CHAN_DATA\n"));
+		DHD_RTT(("\tchan est %u\n", (uint32) (len / sizeof(uint32))));
+		for (i = 0; (uint16)i < (uint16)(len/sizeof(chan_data_entry)); i++) {
+			uint32 *p = (uint32*)p_data;
+			chan_data_entry = ltoh32_ua(p + i);
+			DHD_RTT(("\t%u\n", chan_data_entry));
+		}
+		GCC_DIAGNOSTIC_POP();
+		break;
+	case WL_PROXD_TLV_ID_MF_STATS_DATA:
+		DHD_RTT(("WL_PROXD_TLV_ID_MF_STATS_DATA\n"));
+		DHD_RTT(("\tmf stats len=%u\n", len));
+		rtt_prhex("", p_data, len);
+		break;
+#ifdef WL_RTT_LCI
+	case WL_PROXD_TLV_ID_LCI:
+		tlv = (bcm_xtlv_t *)ctx;
+		DHD_RTT(("WL_PROXD_TLV_ID_LCI, IE data=%lx, len=%d\n",
+			(unsigned long)p_data, len));
+		rtt_prhex("", p_data, len);
+		if (tlv) {
+			tlv->id = WL_PROXD_TLV_ID_LCI;
+			tlv->len = len;
+			(void)memcpy_s(tlv->data, len, p_data, len);
+		}
+		break;
+	case WL_PROXD_TLV_ID_CIVIC:
+		tlv = (bcm_xtlv_t *)ctx;
+		DHD_RTT(("WL_PROXD_TLV_ID_CIVIC, IE data=%lx, len=%d\n",
+			(unsigned long)p_data, len));
+		rtt_prhex("", p_data, len);
+		if (tlv) {
+			tlv->id = WL_PROXD_TLV_ID_CIVIC;
+			tlv->len = len;
+			(void)memcpy_s(tlv->data, len, p_data, len);
+		}
+		break;
+#endif /* WL_RTT_LCI */
+	default:
+		DHD_RTT_ERR(("> Unsupported TLV ID %d\n", tlvid));
+		ret = BCME_ERROR;
+		break;
+	}
+
+	return ret;
+}
+
+#ifdef WL_CFG80211
+static int
+rtt_handle_config_options(wl_proxd_session_id_t session_id, wl_proxd_tlv_t **p_tlv,
+	uint16 *p_buf_space_left, ftm_config_options_info_t *ftm_configs, int ftm_cfg_cnt)
+{
+	int ret = BCME_OK;
+	int cfg_idx = 0;
+	uint32 flags = WL_PROXD_FLAG_NONE;
+	uint32 flags_mask = WL_PROXD_FLAG_NONE;
+	uint32 new_mask;		/* cmdline input */
+	ftm_config_options_info_t *p_option_info;
+	uint16 type = (session_id == WL_PROXD_SESSION_ID_GLOBAL) ?
+			WL_PROXD_TLV_ID_FLAGS_MASK : WL_PROXD_TLV_ID_SESSION_FLAGS_MASK;
+	for (cfg_idx = 0; cfg_idx < ftm_cfg_cnt; cfg_idx++) {
+		p_option_info = (ftm_configs + cfg_idx);
+		if (p_option_info != NULL) {
+			new_mask = p_option_info->flags;
+			/* update flags mask */
+			flags_mask |= new_mask;
+			if (p_option_info->enable) {
+				flags |= new_mask;	/* set the bit on */
+			} else {
+				flags &= ~new_mask;	/* set the bit off */
+			}
+		}
+	}
+	flags = htol32(flags);
+	flags_mask = htol32(flags_mask);
+	/* setup flags_mask TLV */
+	ret = bcm_pack_xtlv_entry((uint8 **)p_tlv, p_buf_space_left,
+		type, sizeof(uint32), (uint8 *)&flags_mask, BCM_XTLV_OPTION_ALIGN32);
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("%s : bcm_pack_xltv_entry() for mask flags failed, status=%d\n",
+			__FUNCTION__, ret));
+		goto exit;
+	}
+
+	type = (session_id == WL_PROXD_SESSION_ID_GLOBAL)?
+		WL_PROXD_TLV_ID_FLAGS : WL_PROXD_TLV_ID_SESSION_FLAGS;
+	/* setup flags TLV */
+	ret = bcm_pack_xtlv_entry((uint8 **)p_tlv, p_buf_space_left,
+			type, sizeof(uint32), (uint8 *)&flags, BCM_XTLV_OPTION_ALIGN32);
+		if (ret != BCME_OK) {
+#ifdef RTT_DEBUG
+			DHD_RTT(("%s: bcm_pack_xltv_entry() for flags failed, status=%d\n",
+				__FUNCTION__, ret));
+#endif
+		}
+exit:
+	return ret;
+}
+
+static int
+rtt_handle_config_general(wl_proxd_session_id_t session_id, wl_proxd_tlv_t **p_tlv,
+	uint16 *p_buf_space_left, ftm_config_param_info_t *ftm_configs, int ftm_cfg_cnt)
+{
+	int ret = BCME_OK;
+	int cfg_idx = 0;
+	uint32 chanspec;
+	ftm_config_param_info_t *p_config_param_info;
+	void		*p_src_data;
+	uint16	src_data_size;	/* size of data pointed by p_src_data as 'source' */
+	for (cfg_idx = 0; cfg_idx < ftm_cfg_cnt; cfg_idx++) {
+		p_config_param_info = (ftm_configs + cfg_idx);
+		if (p_config_param_info != NULL) {
+			switch (p_config_param_info->tlvid)	{
+			case WL_PROXD_TLV_ID_BSS_INDEX:
+			case WL_PROXD_TLV_ID_FTM_RETRIES:
+			case WL_PROXD_TLV_ID_FTM_REQ_RETRIES:
+				p_src_data = &p_config_param_info->data8;
+				src_data_size = sizeof(uint8);
+				break;
+			case WL_PROXD_TLV_ID_BURST_NUM_FTM: /* uint16 */
+			case WL_PROXD_TLV_ID_NUM_BURST:
+			case WL_PROXD_TLV_ID_RX_MAX_BURST:
+				p_src_data = &p_config_param_info->data16;
+				src_data_size = sizeof(uint16);
+				break;
+			case WL_PROXD_TLV_ID_TX_POWER:		/* uint32 */
+			case WL_PROXD_TLV_ID_RATESPEC:
+			case WL_PROXD_TLV_ID_EVENT_MASK: /* wl_proxd_event_mask_t/uint32 */
+			case WL_PROXD_TLV_ID_DEBUG_MASK:
+				p_src_data = &p_config_param_info->data32;
+				src_data_size = sizeof(uint32);
+				break;
+			case WL_PROXD_TLV_ID_CHANSPEC:		/* chanspec_t --> 32bit */
+				chanspec = p_config_param_info->chanspec;
+				p_src_data = (void *) &chanspec;
+				src_data_size = sizeof(uint32);
+				break;
+			case WL_PROXD_TLV_ID_BSSID: /* mac address */
+			case WL_PROXD_TLV_ID_PEER_MAC:
+			case WL_PROXD_TLV_ID_CUR_ETHER_ADDR:
+				p_src_data = &p_config_param_info->mac_addr;
+				src_data_size = sizeof(struct ether_addr);
+				break;
+			case WL_PROXD_TLV_ID_BURST_DURATION:	/* wl_proxd_intvl_t */
+			case WL_PROXD_TLV_ID_BURST_PERIOD:
+			case WL_PROXD_TLV_ID_BURST_FTM_SEP:
+			case WL_PROXD_TLV_ID_BURST_TIMEOUT:
+			case WL_PROXD_TLV_ID_INIT_DELAY:
+				p_src_data = &p_config_param_info->data_intvl;
+				src_data_size = sizeof(wl_proxd_intvl_t);
+				break;
+			default:
+				ret = BCME_BADARG;
+				break;
+			}
+			if (ret != BCME_OK) {
+				DHD_RTT_ERR(("%s bad TLV ID : %d\n",
+					__FUNCTION__, p_config_param_info->tlvid));
+				break;
+			}
+
+			ret = bcm_pack_xtlv_entry((uint8 **) p_tlv, p_buf_space_left,
+				p_config_param_info->tlvid, src_data_size, (uint8 *)p_src_data,
+				BCM_XTLV_OPTION_ALIGN32);
+			if (ret != BCME_OK) {
+				DHD_RTT_ERR(("%s: bcm_pack_xltv_entry() failed,"
+					" status=%d\n", __FUNCTION__, ret));
+				break;
+			}
+
+		}
+	}
+	return ret;
+}
+
+static int
+dhd_rtt_ftm_enable(dhd_pub_t *dhd, bool enable)
+{
+	ftm_subcmd_info_t subcmd_info;
+	subcmd_info.name = (enable)? "enable" : "disable";
+	subcmd_info.cmdid = (enable)? WL_PROXD_CMD_ENABLE: WL_PROXD_CMD_DISABLE;
+	subcmd_info.handler = NULL;
+	return dhd_rtt_common_set_handler(dhd, &subcmd_info,
+			WL_PROXD_METHOD_FTM, WL_PROXD_SESSION_ID_GLOBAL);
+}
+
+static int
+dhd_rtt_start_session(dhd_pub_t *dhd, wl_proxd_session_id_t session_id, bool start)
+{
+	ftm_subcmd_info_t subcmd_info;
+	subcmd_info.name = (start)? "start session" : "stop session";
+	subcmd_info.cmdid = (start)? WL_PROXD_CMD_START_SESSION: WL_PROXD_CMD_STOP_SESSION;
+	subcmd_info.handler = NULL;
+	return dhd_rtt_common_set_handler(dhd, &subcmd_info,
+			WL_PROXD_METHOD_FTM, session_id);
+}
+
+static int
+dhd_rtt_delete_session(dhd_pub_t *dhd, wl_proxd_session_id_t session_id)
+{
+	ftm_subcmd_info_t subcmd_info;
+	subcmd_info.name = "delete session";
+	subcmd_info.cmdid = WL_PROXD_CMD_DELETE_SESSION;
+	subcmd_info.handler = NULL;
+	return dhd_rtt_common_set_handler(dhd, &subcmd_info,
+			WL_PROXD_METHOD_FTM, session_id);
+}
+
+#ifdef WL_NAN
+int
+dhd_rtt_delete_nan_session(dhd_pub_t *dhd)
+{
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	struct wireless_dev *wdev = ndev_to_wdev(dev);
+	struct wiphy *wiphy = wdev->wiphy;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	wl_cfgnan_terminate_directed_rtt_sessions(dev, cfg);
+	return BCME_OK;
+}
+
+/* API to find out if the given Peer Mac from FTM events
+* is nan-peer. Based on this we will handle the SESSION_END
+* event. For nan-peer FTM_SESSION_END event is ignored and handled in
+* nan-ranging-cancel or nan-ranging-end event.
+*/
+static bool
+dhd_rtt_is_nan_peer(dhd_pub_t *dhd, struct ether_addr *peer_mac)
+{
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	struct wireless_dev *wdev = ndev_to_wdev(dev);
+	struct wiphy *wiphy = wdev->wiphy;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	nan_ranging_inst_t *ranging_inst = NULL;
+	bool ret = FALSE;
+
+	if ((wl_cfgnan_is_enabled(cfg) == FALSE) || ETHER_ISNULLADDR(peer_mac)) {
+		goto exit;
+	}
+
+	ranging_inst = wl_cfgnan_check_for_ranging(cfg, peer_mac);
+	if (ranging_inst) {
+		DHD_RTT((" RTT peer is of type NAN\n"));
+		ret = TRUE;
+		goto exit;
+	}
+exit:
+	return ret;
+}
+
+bool
+dhd_rtt_nan_is_directed_setup_in_prog(dhd_pub_t *dhd)
+{
+	bool setup_in_prog;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	setup_in_prog = rtt_status->directed_cfg.directed_setup_status.directed_na_setup_inprog;
+
+	return setup_in_prog;
+}
+
+bool
+dhd_rtt_nan_is_directed_setup_in_prog_with_peer(dhd_pub_t *dhd,
+		struct ether_addr *peer)
+{
+	bool setup_in_prog = TRUE;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+	nan_ranging_inst_t *ranging_inst = NULL;
+
+	if (!dhd_rtt_nan_is_directed_setup_in_prog(dhd)) {
+		setup_in_prog = FALSE;
+		goto exit;
+	}
+
+	ranging_inst = wl_cfgnan_check_for_ranging(cfg, peer);
+
+	if ((ranging_inst == NULL) ||
+			(ranging_inst != rtt_status->directed_cfg.directed_setup_status.rng_inst)) {
+		setup_in_prog = FALSE;
+	}
+
+exit:
+	return setup_in_prog;
+}
+
+void
+dhd_rtt_nan_update_directed_setup_inprog(dhd_pub_t *dhd,
+	nan_ranging_inst_t *rng_inst, bool inprog)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	rtt_status->directed_cfg.directed_setup_status.directed_na_setup_inprog = inprog;
+	if (inprog) {
+		ASSERT(rng_inst);
+		rtt_status->directed_cfg.directed_setup_status.rng_inst = rng_inst;
+	} else {
+		rtt_status->directed_cfg.directed_setup_status.rng_inst = NULL;
+	}
+}
+
+bool
+dhd_rtt_nan_directed_sessions_allowed(dhd_pub_t *dhd)
+{
+	int max_sessions = 0;
+	bool allowed = TRUE;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	max_sessions = rtt_status->max_nan_rtt_sessions;
+
+	if (dhd_rtt_nan_is_directed_setup_in_prog(dhd)) {
+		max_sessions--;
+	}
+
+	if (rtt_status->directed_cfg.directed_sessions_cnt >= max_sessions) {
+		allowed = FALSE;
+	}
+
+	return allowed;
+}
+
+bool
+dhd_rtt_nan_all_directed_sessions_triggered(dhd_pub_t *dhd)
+{
+	bool done;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	if ((rtt_status->cur_idx + 1) >= rtt_status->rtt_config.rtt_target_cnt) {
+		done = TRUE;
+	} else {
+		done = FALSE;
+	}
+
+	return done;
+}
+
+void
+dhd_rtt_nan_update_directed_sessions_cnt(dhd_pub_t *dhd, bool incr)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	if (incr) {
+		rtt_status->directed_cfg.directed_sessions_cnt++;
+	} else {
+		rtt_status->directed_cfg.directed_sessions_cnt--;
+	}
+}
+
+static void
+dhd_rtt_event_trigger_failure(dhd_pub_t *dhd, rtt_target_info_t *rtt_target)
+{
+	wl_event_msg_t msg;
+
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	wl_proxd_event_t p_event;
+
+	bzero(&msg, sizeof(wl_event_msg_t));
+	bzero(&p_event, sizeof(p_event));
+
+	msg.event_type =  hton32(WLC_E_PROXD);
+	msg.bsscfgidx  =  0;
+	msg.datalen = hton32(sizeof(p_event));
+	msg.addr = rtt_target->addr;
+
+	p_event.version = htol16(WL_PROXD_API_VERSION_3);
+	p_event.type = htol16(WL_PROXD_EVENT_SESSION_END);
+	p_event.len = htol16(OFFSETOF(wl_proxd_event_t, tlvs));
+
+	wl_cfg80211_event(dev, &msg, &p_event);
+}
+
+static int
+dhd_rtt_nan_start_session(dhd_pub_t *dhd, rtt_target_info_t *rtt_target)
+{
+	s32 err = BCME_OK;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	struct wireless_dev *wdev = ndev_to_wdev(dev);
+	struct wiphy *wiphy = wdev->wiphy;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	nan_ranging_inst_t *ranging_inst = NULL;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	ftm_config_param_info_t ftm_params[FTM_MAX_PARAMS];
+	int ftm_param_cnt = 0;
+
+	memset(ftm_params, 0, sizeof(ftm_params));
+
+	NAN_MUTEX_LOCK();
+
+	if (!rtt_status) {
+		err = BCME_NOTENABLED;
+		goto done;
+	}
+
+	if (NAN_RTT_ENABLED(cfg) != TRUE) {
+		/* If nan is not enabled or nan ranging is not enabled report error */
+		err = BCME_NOTENABLED;
+		goto done;
+	}
+
+	/* Below Scenarios should be avoided by callers/schedulers */
+	if (dhd_rtt_nan_is_directed_setup_in_prog(dhd)) {
+		DHD_RTT_ERR(("dhd_rtt_nan_start_session failed, setup already in prog\n"));
+		err = BCME_ERROR;
+		ASSERT(0);
+		goto done;
+	}
+
+	if (!dhd_rtt_nan_directed_sessions_allowed(dhd)) {
+		DHD_RTT_ERR(("dhd_rtt_nan_start_session failed, already max sessions running\n"));
+		err = BCME_ERROR;
+		ASSERT(0);
+		goto done;
+	}
+
+	ranging_inst = wl_cfgnan_get_ranging_inst(cfg,
+			&rtt_target->addr, NAN_RANGING_ROLE_INITIATOR);
+	if (!ranging_inst) {
+		err = BCME_NORESOURCE;
+		goto done;
+	}
+
+	/* apply event mask */
+	dhd_rtt_set_ftm_config_param(ftm_params, &ftm_param_cnt, rtt_target,
+		WL_PROXD_TLV_ID_EVENT_MASK);
+	dhd_rtt_ftm_config(dhd, 0, NULL, 0, ftm_params, ftm_param_cnt);
+
+	DHD_RTT(("Trigger nan based range request\n"));
+	err = wl_cfgnan_trigger_ranging(bcmcfg_to_prmry_ndev(cfg),
+			cfg, ranging_inst, NULL, NAN_RANGE_REQ_CMD, TRUE);
+	if (unlikely(err)) {
+		goto done;
+	}
+	ranging_inst->range_type = RTT_TYPE_NAN_DIRECTED;
+	ranging_inst->range_role = NAN_RANGING_ROLE_INITIATOR;
+	dhd_rtt_nan_update_directed_setup_inprog(dhd, ranging_inst, TRUE);
+
+done:
+	if (err) {
+		DHD_RTT_ERR(("Failed to issue Nan Ranging Request err %d\n", err));
+		/* Fake session end event which will help in
+		* scheduling the new target in the deffered context instead of here
+		* i.e, avoid scheduling work from itself
+		*/
+		dhd_rtt_event_trigger_failure(dhd, rtt_target);
+	}
+	NAN_MUTEX_UNLOCK();
+	return err;
+}
+#endif /* WL_NAN */
+
+static int
+dhd_rtt_ftm_config(dhd_pub_t *dhd, wl_proxd_session_id_t session_id,
+	void *ftm_cfg_opt, int ftm_cfg_opt_cnt, void *ftm_cfg_gen, int ftm_cfg_gen_cnt)
+{
+	ftm_subcmd_info_t subcmd_info;
+	wl_proxd_tlv_t *p_tlv;
+	/* alloc mem for ioctl headr + reserved 0 bufsize for tlvs (initialize to zero) */
+	wl_proxd_iov_t *p_proxd_iov;
+	uint16 proxd_iovsize = 0;
+	uint16 bufsize;
+	uint16 buf_space_left;
+	uint16 all_tlvsize;
+	int ret = BCME_OK;
+
+	subcmd_info.name = "config";
+	subcmd_info.cmdid = WL_PROXD_CMD_CONFIG;
+
+	p_proxd_iov = rtt_alloc_getset_buf(dhd, WL_PROXD_METHOD_FTM, session_id, subcmd_info.cmdid,
+			FTM_IOC_BUFSZ, &proxd_iovsize);
+
+	if (p_proxd_iov == NULL) {
+		DHD_RTT_ERR(("%s : failed to allocate the iovar (size :%d)\n",
+			__FUNCTION__, FTM_IOC_BUFSZ));
+		return BCME_NOMEM;
+	}
+	/* setup TLVs */
+	bufsize = proxd_iovsize - WL_PROXD_IOV_HDR_SIZE; /* adjust available size for TLVs */
+	p_tlv = &p_proxd_iov->tlvs[0];
+	/* TLV buffer starts with a full size, will decrement for each packed TLV */
+	buf_space_left = bufsize;
+	if (ftm_cfg_opt) {
+		ret = rtt_handle_config_options(session_id, &p_tlv, &buf_space_left,
+				(ftm_config_options_info_t *)ftm_cfg_opt, ftm_cfg_opt_cnt);
+	}
+	if (ftm_cfg_gen) {
+		ret = rtt_handle_config_general(session_id, &p_tlv, &buf_space_left,
+				(ftm_config_param_info_t *)ftm_cfg_gen, ftm_cfg_gen_cnt);
+	}
+	if (ret == BCME_OK) {
+		/* update the iov header, set len to include all TLVs + header */
+		all_tlvsize = (bufsize - buf_space_left);
+		p_proxd_iov->len = htol16(all_tlvsize + WL_PROXD_IOV_HDR_SIZE);
+		ret = dhd_iovar(dhd, 0, "proxd", (char *)p_proxd_iov,
+				all_tlvsize + WL_PROXD_IOV_HDR_SIZE, NULL, 0, TRUE);
+		if (ret != BCME_OK) {
+			DHD_RTT_ERR(("%s : failed to set config err %d\n", __FUNCTION__, ret));
+		}
+	}
+	/* clean up */
+	MFREE(dhd->osh, p_proxd_iov, proxd_iovsize);
+	return ret;
+}
+
+static int
+dhd_rtt_get_version(dhd_pub_t *dhd, int *out_version)
+{
+	int ret;
+	ftm_subcmd_info_t subcmd_info;
+	subcmd_info.name = "ver";
+	subcmd_info.cmdid = WL_PROXD_CMD_GET_VERSION;
+	subcmd_info.handler = NULL;
+	ret = dhd_rtt_common_get_handler(dhd, &subcmd_info,
+			WL_PROXD_METHOD_FTM, WL_PROXD_SESSION_ID_GLOBAL);
+	*out_version = (ret == BCME_OK) ? subcmd_info.version : 0;
+	return ret;
+}
+#endif /* WL_CFG80211 */
+
+chanspec_t
+dhd_rtt_convert_to_chspec(wifi_channel_info channel)
+{
+	chanspec_bw_t bw = INVCHANSPEC;
+	chanspec_t chanspec = INVCHANSPEC;
+	int primary_chan;
+	chanspec_band_t band;
+	uint32 center_freq = channel.center_freq;
+
+	band = wf_mhz2chanspec_band(center_freq);
+	if (band == INVCHANSPEC) {
+		DHD_RTT_ERR(("Invalid centre frequency %x\n", center_freq));
+		goto done;
+	}
+
+	primary_chan = wf_mhz2channel(center_freq, 0);
+	if (primary_chan == -1) {
+		DHD_RTT_ERR(("Failed to get primary channel for frequency %x\n",
+				center_freq));
+		goto done;
+	}
+
+	switch (channel.width) {
+	case WIFI_CHAN_WIDTH_20:
+		bw = WL_CHANSPEC_BW_20;
+		break;
+	case WIFI_CHAN_WIDTH_40:
+		bw = WL_CHANSPEC_BW_40;
+		break;
+	case WIFI_CHAN_WIDTH_80:
+		bw = WL_CHANSPEC_BW_80;
+		break;
+#ifdef WL_RTT_BW160
+	case WIFI_CHAN_WIDTH_160:
+		bw = WL_CHANSPEC_BW_160;
+		break;
+#endif /* WL_RTT_BW160 */
+	default:
+		DHD_RTT_ERR(("doesn't support this bandwith : %d\n", channel.width));
+		bw = INVCHANSPEC;
+		break;
+	}
+
+	if ((bw != INVCHANSPEC)) {
+#ifdef WL_6G_320_SUPPORT
+		chanspec = wf_create_chspec_from_primary(primary_chan, bw, band, 0);
+#else
+		chanspec = wf_create_chspec_from_primary(primary_chan, bw, band);
+#endif /* WL_6G_320_SUPPORT */
+	}
+
+done:
+	return chanspec;
+}
+
+int
+dhd_rtt_idx_to_burst_duration(uint idx)
+{
+	if (idx >= ARRAY_SIZE(burst_duration_idx)) {
+		return -1;
+	}
+	return burst_duration_idx[idx];
+}
+
+int8
+dhd_rtt_get_cur_target_idx(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	return rtt_status->cur_idx;
+}
+
+int8
+dhd_rtt_set_next_target_idx(dhd_pub_t *dhd, int start_idx)
+{
+	int idx;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	for (idx = start_idx; idx < rtt_status->rtt_config.rtt_target_cnt; idx++) {
+		/* skip the disabled device */
+		if (rtt_status->rtt_config.target_info[idx].disable) {
+			continue;
+		} else {
+			/* set the idx to cur_idx */
+			rtt_status->cur_idx = idx;
+			break;
+		}
+	}
+
+	if (idx == rtt_status->rtt_config.rtt_target_cnt) {
+		/* All targets trigerred */
+		rtt_status->cur_idx = rtt_status->rtt_config.rtt_target_cnt;
+	}
+
+	return (int8)rtt_status->cur_idx;
+}
+
+/* API to know if the all the targets provided in
+ * startRTT call from framework are of type NAN
+ */
+bool
+dhd_rtt_is_taget_list_mode_nan(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	bool nan_mode = FALSE;
+
+	if (!rtt_status) {
+		goto exit;
+	}
+
+	if (rtt_status->rtt_config.target_list_mode ==
+		RNG_TARGET_LIST_MODE_NAN) {
+		nan_mode = TRUE;
+		goto exit;
+	}
+
+exit:
+	return nan_mode;
+}
+
+void
+dhd_rtt_set_target_list_mode(dhd_pub_t *dhd)
+{
+	int8 idx;
+	bool legacy = FALSE, nan = FALSE;
+	rtt_target_info_t *rtt_target;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	rtt_status->rtt_config.target_list_mode = RNG_TARGET_LIST_MODE_INVALID;
+	for (idx = rtt_status->start_idx; idx < rtt_status->rtt_config.rtt_target_cnt; idx++) {
+		rtt_target = &rtt_status->rtt_config.target_info[idx];
+		/* skip the disabled device */
+		if (rtt_target->disable) {
+			continue;
+		} else {
+			if (rtt_target->peer == RTT_PEER_NAN) {
+				nan = TRUE;
+			} else {
+				legacy = TRUE;
+			}
+		}
+	}
+
+	if ((nan == TRUE) && (legacy == TRUE)) {
+		rtt_status->rtt_config.target_list_mode = RNG_TARGET_LIST_MODE_MIX;
+	} else if (nan == TRUE) {
+		rtt_status->rtt_config.target_list_mode = RNG_TARGET_LIST_MODE_NAN;
+	} else if (legacy == TRUE) {
+		rtt_status->rtt_config.target_list_mode = RNG_TARGET_LIST_MODE_LEGACY;
+	}
+}
+
+int
+dhd_rtt_set_cfg(dhd_pub_t *dhd, rtt_config_params_t *params)
+{
+	int err = BCME_OK;
+	rtt_status_info_t *rtt_status = NULL;
+	struct net_device *dev = NULL;
+
+	NULL_CHECK(params, "params is NULL", err);
+	NULL_CHECK(dhd, "dhd is NULL", err);
+
+	dev = dhd_linux_get_primary_netdev(dhd);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	NULL_CHECK(dev, "dev is NULL", err);
+
+	DHD_RTT(("%s enter\n", __FUNCTION__));
+
+	if (!HAS_11MC_CAP(rtt_status->rtt_capa.proto)) {
+		DHD_RTT_ERR(("doesn't support RTT \n"));
+		err = BCME_ERROR;
+		goto exit;
+	}
+
+	/* check if work is being scheduled ...cancel/sync if so
+	* Host REQ has more priority , so we to have cancel any
+	* geofence sessions in progress...for that we need to make sure
+	* work queue is IDLE & then cancel the geofence sessions
+	*/
+	cancel_work_sync(&rtt_status->work);
+
+	mutex_lock(&rtt_status->rtt_mutex);
+
+	if (rtt_status->status != RTT_STOPPED) {
+		DHD_RTT_ERR(("rtt is already started, status : %d\n", rtt_status->status));
+		err = BCME_BUSY;
+		goto exit;
+	}
+	if (params->rtt_target_cnt > 0) {
+#ifdef WL_NAN
+		/* cancel ongoing geofence RTT both initiators and responders */
+		wl_cfgnan_suspend_all_geofence_rng_sessions(dev,
+			RTT_GEO_SUSPN_HOST_DIR_RTT_TRIG, 0);
+#endif /* WL_NAN */
+	} else {
+		err = BCME_BADARG;
+		goto exit;
+	}
+
+	memset(rtt_status->rtt_config.target_info, 0, TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT));
+	rtt_status->rtt_config.rtt_target_cnt = params->rtt_target_cnt;
+	memcpy(rtt_status->rtt_config.target_info,
+		params->target_info, TARGET_INFO_SIZE(params->rtt_target_cnt));
+	rtt_status->status = RTT_STARTED;
+	DHD_RTT_MEM(("dhd_rtt_set_cfg: RTT Started, target_cnt = %d\n", params->rtt_target_cnt));
+
+	/* This is the starting Directed RTT index */
+	rtt_status->start_idx = dhd_rtt_set_next_target_idx(dhd, 0);
+
+	dhd_rtt_set_target_list_mode(dhd);
+
+	if (rtt_status->cur_idx < rtt_status->rtt_config.rtt_target_cnt) {
+#ifdef WL_NAN
+		if (rtt_status->rtt_config.target_list_mode == RNG_TARGET_LIST_MODE_NAN) {
+			/* reset directed cfg params */
+			dhd_rtt_nan_update_directed_setup_inprog(dhd, NULL, FALSE);
+			rtt_status->directed_cfg.directed_sessions_cnt = 0;
+
+			/*
+			 * schedule proxd timeout
+			 * Proxd timeout for NAN target list is scheduled as a whole,
+			 * and not per target, unlike for legacy target list
+			 */
+			schedule_delayed_work(&rtt_status->proxd_timeout,
+				msecs_to_jiffies(DHD_NAN_RTT_TIMER_INTERVAL_MS));
+		}
+#endif /* WL_NAN */
+		/* schedule RTT */
+		dhd_rtt_schedule_rtt_work_thread(dhd, RTT_SCHED_HOST_TRIGGER);
+	}
+exit:
+	mutex_unlock(&rtt_status->rtt_mutex);
+	return err;
+}
+
+#ifdef WL_NAN
+void
+dhd_rtt_initialize_geofence_cfg(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	if (!rtt_status) {
+		return;
+	}
+
+	GEOFENCE_RTT_LOCK(rtt_status);
+	memset_s(&rtt_status->geofence_cfg, sizeof(rtt_status->geofence_cfg),
+		0, sizeof(rtt_status->geofence_cfg));
+
+	/* initialize non zero params of geofence cfg */
+	rtt_status->geofence_cfg.cur_target_idx = DHD_RTT_INVALID_TARGET_INDEX;
+	rtt_status->geofence_cfg.geofence_rtt_interval = DHD_RTT_RETRY_TIMER_INTERVAL_MS;
+	rtt_status->geofence_cfg.geofence_sessions_cnt = 0;
+
+	rtt_status->geofence_cfg.max_geofence_sessions =
+		dhd_rtt_get_max_nan_rtt_sessions_supported(dhd);
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+	return;
+}
+
+#ifdef RTT_GEOFENCE_CONT
+void
+dhd_rtt_get_geofence_cont_ind(dhd_pub_t *dhd, bool* geofence_cont)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	if (!rtt_status) {
+		return;
+	}
+	GEOFENCE_RTT_LOCK(rtt_status);
+	*geofence_cont = rtt_status->geofence_cfg.geofence_cont;
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+}
+
+void
+dhd_rtt_set_geofence_cont_ind(dhd_pub_t *dhd, bool geofence_cont)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	if (!rtt_status) {
+		return;
+	}
+	GEOFENCE_RTT_LOCK(rtt_status);
+	rtt_status->geofence_cfg.geofence_cont = geofence_cont;
+	DHD_RTT(("dhd_rtt_set_geofence_cont_override, geofence_cont = %d\n",
+		rtt_status->geofence_cfg.geofence_cont));
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+}
+#endif /* RTT_GEOFENCE_CONT */
+
+#ifdef RTT_GEOFENCE_INTERVAL
+void
+dhd_rtt_set_geofence_rtt_interval(dhd_pub_t *dhd, int interval)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	if (!rtt_status) {
+		return;
+	}
+	GEOFENCE_RTT_LOCK(rtt_status);
+	rtt_status->geofence_cfg.geofence_rtt_interval = interval;
+	DHD_RTT(("dhd_rtt_set_geofence_rtt_interval: geofence interval = %d\n",
+		rtt_status->geofence_cfg.geofence_rtt_interval));
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+}
+#endif /* RTT_GEOFENCE_INTERVAL */
+
+int
+dhd_rtt_get_geofence_max_sessions(dhd_pub_t *dhd)
+{
+	int max_sessions;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_cfg_t* geofence_cfg = &rtt_status->geofence_cfg;
+
+	max_sessions = geofence_cfg->max_geofence_sessions;
+	if (dhd_rtt_is_geofence_setup_inprog(dhd)) {
+		/* One slot busy with setup in prog */
+		max_sessions -= 1;
+	}
+
+	return max_sessions;
+}
+
+/*
+ * Return True, if geofence
+ * session count maxed out
+ */
+bool
+dhd_rtt_geofence_sessions_maxed_out(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_cfg_t* geofence_cfg = &rtt_status->geofence_cfg;
+	bool ret = TRUE;
+
+	if (geofence_cfg->geofence_sessions_cnt <
+		dhd_rtt_get_geofence_max_sessions(dhd)) {
+		ret = FALSE;
+	}
+
+	return ret;
+}
+
+int
+dhd_rtt_get_geofence_sessions_cnt(dhd_pub_t *dhd)
+{
+
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_cfg_t* geofence_cfg = &rtt_status->geofence_cfg;
+
+	return geofence_cfg->geofence_sessions_cnt;
+}
+
+int
+dhd_rtt_update_geofence_sessions_cnt(dhd_pub_t *dhd, bool incr,
+	struct ether_addr *peer_addr)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_cfg_t* geofence_cfg = &rtt_status->geofence_cfg;
+	int ret = BCME_OK;
+	int geofence_ssn_cnt_before_upd = geofence_cfg->geofence_sessions_cnt;
+
+	if (incr) {
+		//ASSERT(!dhd_rtt_geofence_sessions_maxed_out(dhd));
+		if (dhd_rtt_geofence_sessions_maxed_out(dhd)) {
+			ret = BCME_ERROR;
+			goto exit;
+		}
+		geofence_cfg->geofence_sessions_cnt++;
+	} else {
+		if (peer_addr && dhd_rtt_is_geofence_setup_inprog_with_peer(dhd,
+				peer_addr)) {
+			/* Set geofence RTT in progress state to false */
+			dhd_rtt_set_geofence_setup_status(dhd, FALSE, NULL);
+		} else {
+			//ASSERT(geofence_cfg->geofence_sessions_cnt > 0);
+			if (geofence_cfg->geofence_sessions_cnt <= 0) {
+				ret = BCME_ERROR;
+				goto exit;
+			}
+			/* Decrement session count */
+			geofence_cfg->geofence_sessions_cnt--;
+		}
+	}
+
+exit:
+	if (peer_addr) {
+		WL_INFORM_MEM(("session cnt update, upd = %d, cnt = %d, cnt_bef_upd = %d, "
+			" peer : "MACDBG", ret = %d\n", incr, geofence_cfg->geofence_sessions_cnt,
+			geofence_ssn_cnt_before_upd, MAC2STRDBG(peer_addr), ret));
+	} else {
+		WL_INFORM_MEM(("session cnt update, upd = %d, cnt = %d, cnt_bef_upd = %d, "
+			"ret = %d\n", incr, geofence_cfg->geofence_sessions_cnt,
+			 geofence_ssn_cnt_before_upd, ret));
+	}
+
+	return ret;
+}
+
+int8
+dhd_rtt_get_geofence_target_cnt(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	if (!rtt_status) {
+		return 0;
+	}
+	return rtt_status->geofence_cfg.geofence_target_cnt;
+}
+
+/* returns geofence RTT target list Head */
+rtt_geofence_target_info_t*
+dhd_rtt_get_geofence_target_head(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_target_info_t* head = NULL;
+
+	if (!rtt_status) {
+		return NULL;
+	}
+
+	if (rtt_status->geofence_cfg.geofence_target_cnt) {
+		head = &rtt_status->geofence_cfg.geofence_target_info[0];
+	}
+
+	return head;
+}
+
+int8
+dhd_rtt_get_geofence_cur_target_idx(dhd_pub_t *dhd)
+{
+	int8 target_cnt = 0, cur_idx = DHD_RTT_INVALID_TARGET_INDEX;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	if (!rtt_status) {
+		goto exit;
+	}
+
+	target_cnt = rtt_status->geofence_cfg.geofence_target_cnt;
+	if (target_cnt == 0) {
+		goto exit;
+	}
+
+	cur_idx = rtt_status->geofence_cfg.cur_target_idx;
+	if (cur_idx >= target_cnt) {
+		WL_INFORM_MEM(("dhd_rtt_get_geofence_cur_target_idx: "
+			"cur_index exceeded (>=) target_cnt, cur_idx = %d, "
+			"target_cnt = %d\n", cur_idx, target_cnt));
+		ASSERT(cur_idx < target_cnt);
+		cur_idx = DHD_RTT_INVALID_TARGET_INDEX;
+	}
+
+exit:
+	return cur_idx;
+}
+
+void
+dhd_rtt_set_geofence_cur_target_idx(dhd_pub_t *dhd, int8 idx)
+{
+	int8 target_cnt = 0;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	UNUSED_PARAMETER(target_cnt);
+
+	target_cnt = rtt_status->geofence_cfg.geofence_target_cnt;
+	ASSERT(idx < target_cnt);
+	rtt_status->geofence_cfg.cur_target_idx = idx;
+	return;
+}
+
+void
+dhd_rtt_move_geofence_cur_target_idx_to_next(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	if (!rtt_status) {
+		return;
+	}
+
+	if (rtt_status->geofence_cfg.geofence_target_cnt == 0) {
+		/* Invalidate current idx if no targets */
+		rtt_status->geofence_cfg.cur_target_idx =
+			DHD_RTT_INVALID_TARGET_INDEX;
+		/* Cancel pending retry timer if any */
+		if (delayed_work_pending(&rtt_status->rtt_retry_timer)) {
+			cancel_delayed_work(&rtt_status->rtt_retry_timer);
+		}
+		return;
+	}
+	rtt_status->geofence_cfg.cur_target_idx++;
+
+	if (rtt_status->geofence_cfg.cur_target_idx >=
+		rtt_status->geofence_cfg.geofence_target_cnt) {
+		/* Reset once all targets done */
+		rtt_status->geofence_cfg.cur_target_idx = 0;
+	}
+}
+
+/* returns geofence current RTT target */
+rtt_geofence_target_info_t*
+dhd_rtt_get_geofence_current_target(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_target_info_t* cur_target = NULL;
+	int cur_idx = 0;
+
+	if (!rtt_status) {
+		return NULL;
+	}
+
+	cur_idx = dhd_rtt_get_geofence_cur_target_idx(dhd);
+	if (cur_idx >= 0) {
+		cur_target = &rtt_status->geofence_cfg.geofence_target_info[cur_idx];
+	}
+
+	return cur_target;
+}
+
+/* returns geofence target from list for the peer */
+rtt_geofence_target_info_t*
+dhd_rtt_get_geofence_target(dhd_pub_t *dhd, struct ether_addr* peer_addr, int8 *index)
+{
+	int8 i;
+	rtt_status_info_t *rtt_status;
+	int target_cnt;
+	rtt_geofence_target_info_t *geofence_target_info, *tgt = NULL;
+
+	rtt_status = GET_RTTSTATE(dhd);
+
+	if (!rtt_status) {
+		return NULL;
+	}
+
+	target_cnt = rtt_status->geofence_cfg.geofence_target_cnt;
+	geofence_target_info = rtt_status->geofence_cfg.geofence_target_info;
+
+	/* Loop through to find target */
+	for (i = 0; i < target_cnt; i++) {
+		if (geofence_target_info[i].valid == FALSE) {
+			break;
+		}
+		if (!memcmp(peer_addr, &geofence_target_info[i].peer_addr,
+				ETHER_ADDR_LEN)) {
+			*index = i;
+			tgt = &geofence_target_info[i];
+		}
+	}
+	if (!tgt) {
+		DHD_RTT(("dhd_rtt_get_geofence_target: Target not found in list,"
+			" MAC ADDR: "MACDBG" \n", MAC2STRDBG(peer_addr)));
+	}
+	return tgt;
+}
+
+/* add geofence target to the target list */
+int
+dhd_rtt_add_geofence_target(dhd_pub_t *dhd, rtt_geofence_target_info_t *target)
+{
+	int err = BCME_OK;
+	rtt_status_info_t *rtt_status;
+	rtt_geofence_target_info_t  *geofence_target_info;
+	int8 geofence_target_cnt, index;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+
+	GEOFENCE_RTT_LOCK(rtt_status);
+
+	/* Get the geofence_target via peer addr, index param is dumm here */
+	geofence_target_info = dhd_rtt_get_geofence_target(dhd, &target->peer_addr, &index);
+	if (geofence_target_info) {
+		DHD_RTT(("Duplicate geofencing RTT add request dropped\n"));
+		err = BCME_OK;
+		goto exit;
+	}
+
+	geofence_target_cnt = rtt_status->geofence_cfg.geofence_target_cnt;
+	if (geofence_target_cnt >= RTT_MAX_GEOFENCE_TARGET_CNT) {
+		DHD_RTT(("Queue full, Geofencing RTT add request dropped\n"));
+		err = BCME_NORESOURCE;
+		goto exit;
+	}
+
+	/* Add Geofence RTT request and increment target count */
+	geofence_target_info = rtt_status->geofence_cfg.geofence_target_info;
+	/* src and dest buffer len same, pointers of same DS statically allocated */
+	(void)memcpy_s(&geofence_target_info[geofence_target_cnt],
+		sizeof(geofence_target_info[geofence_target_cnt]), target,
+		sizeof(*target));
+	geofence_target_info[geofence_target_cnt].valid = TRUE;
+	rtt_status->geofence_cfg.geofence_target_cnt++;
+	if (rtt_status->geofence_cfg.geofence_target_cnt == 1) {
+		/* Adding first target */
+		rtt_status->geofence_cfg.cur_target_idx = 0;
+	}
+
+	WL_INFORM_MEM(("dhd_rtt_add_geofence_target: " MACDBG
+		", cur_idx = %d, total cnt = %d\n", MAC2STRDBG(&target->peer_addr),
+		rtt_status->geofence_cfg.cur_target_idx,
+		rtt_status->geofence_cfg.geofence_target_cnt));
+
+exit:
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+	return err;
+}
+
+/* removes geofence target from the target list */
+int
+dhd_rtt_remove_geofence_target(dhd_pub_t *dhd, struct ether_addr *peer_addr)
+{
+	int err = BCME_OK;
+	rtt_status_info_t *rtt_status;
+	rtt_geofence_target_info_t  *geofence_target_info;
+	int8 geofence_target_cnt, j, index = DHD_RTT_INVALID_TARGET_INDEX;
+	struct net_device *dev;
+	struct bcm_cfg80211 *cfg;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+
+	dev = dhd_linux_get_primary_netdev(dhd);
+	cfg = wl_get_cfg(dev);
+
+	GEOFENCE_RTT_LOCK(rtt_status);
+
+	geofence_target_cnt = dhd_rtt_get_geofence_target_cnt(dhd);
+	if (geofence_target_cnt == 0) {
+		DHD_RTT(("Queue Empty, Geofencing RTT remove request dropped\n"));
+		goto exit;
+	}
+
+	/* Get the geofence_target index via peer addr */
+	dhd_rtt_get_geofence_target(dhd, peer_addr, &index);
+	if (index == DHD_RTT_INVALID_TARGET_INDEX) {
+		DHD_RTT(("Geofencing RTT target not found, remove request dropped\n"));
+		err = BCME_NOTFOUND;
+		goto exit;
+	}
+
+	geofence_target_info = rtt_status->geofence_cfg.geofence_target_info;
+	/* left shift all the valid entries, as we dont keep holes in list */
+	for (j = index; j < geofence_target_cnt; j++) {
+		/*
+		 * src and dest buffer len same, pointers of same DS
+		 * statically allocated
+		 */
+		if ((j + 1) < geofence_target_cnt) {
+			(void)memcpy_s(&geofence_target_info[j], sizeof(rtt_geofence_target_info_t),
+				&geofence_target_info[j + 1], sizeof(rtt_geofence_target_info_t));
+		} else {
+			/* reset the last target info */
+			bzero(&geofence_target_info[j], sizeof(rtt_geofence_target_info_t));
+		}
+	}
+
+	rtt_status->geofence_cfg.geofence_target_cnt--;
+	if (rtt_status->geofence_cfg.geofence_target_cnt == 0) {
+		rtt_status->geofence_cfg.cur_target_idx =
+			DHD_RTT_INVALID_TARGET_INDEX;
+	} else {
+		if (rtt_status->geofence_cfg.geofence_target_cnt ==
+				rtt_status->geofence_cfg.cur_target_idx) {
+			/*
+			 * Wrap to first (next here) target again,
+			 * as the last target, got removed,
+			 * which was the current target (idx) also
+			 */
+			rtt_status->geofence_cfg.cur_target_idx = 0;
+		}
+		wl_cfgnan_update_geofence_target_idx(cfg);
+	}
+
+	WL_INFORM_MEM(("dhd_rtt_remove_geofence_target: " MACDBG
+		", cur_idx = %d, target_cnt = %d\n", MAC2STRDBG(peer_addr),
+		rtt_status->geofence_cfg.cur_target_idx,
+		rtt_status->geofence_cfg.geofence_target_cnt));
+
+exit:
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+	return err;
+}
+
+/* deletes/empty geofence target list */
+int
+dhd_rtt_delete_geofence_target_list(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status;
+
+	int err = BCME_OK;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	GEOFENCE_RTT_LOCK(rtt_status);
+	memset_s(&rtt_status->geofence_cfg, sizeof(rtt_geofence_cfg_t),
+		0, sizeof(rtt_geofence_cfg_t));
+	GEOFENCE_RTT_UNLOCK(rtt_status);
+	return err;
+}
+
+rtt_geofence_setup_status_t*
+dhd_rtt_get_geofence_setup_status(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_cfg_t* geofence_cfg = &rtt_status->geofence_cfg;
+	rtt_geofence_setup_status_t* rng_setup_status;
+
+	rng_setup_status = &geofence_cfg->geofence_setup_status;
+	return rng_setup_status;
+}
+
+bool
+dhd_rtt_is_geofence_setup_inprog(dhd_pub_t *dhd)
+{
+	rtt_geofence_setup_status_t* rng_setup_status;
+
+	rng_setup_status = dhd_rtt_get_geofence_setup_status(dhd);
+
+	return rng_setup_status->geofence_setup_inprog;
+}
+
+bool
+dhd_rtt_is_geofence_setup_inprog_with_peer(dhd_pub_t *dhd,
+		struct ether_addr *peer_addr)
+{
+	rtt_geofence_setup_status_t* rng_setup_status;
+	struct nan_ranging_inst *rng_inst = NULL;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+	bool ret = FALSE;
+
+	rng_setup_status = dhd_rtt_get_geofence_setup_status(dhd);
+	if (rng_setup_status->geofence_setup_inprog == FALSE) {
+		goto exit;
+	}
+	rng_inst = wl_cfgnan_check_for_ranging(cfg, peer_addr);
+	if (rng_inst && (rng_inst == rng_setup_status->rng_inst)) {
+		ret = TRUE;
+	}
+
+exit:
+	return ret;
+}
+
+/*
+ * Call with inprog as true and corresponding
+ * rng_inst, to take setup lock,
+ * call with inprog as False and rng_inst as NULL
+ * to unlock setup lock
+ */
+void
+dhd_rtt_set_geofence_setup_status(dhd_pub_t *dhd, bool inprog,
+		struct ether_addr *peer_addr)
+{
+	struct nan_ranging_inst *rng_inst = NULL;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+	rtt_geofence_setup_status_t* rng_setup_status;
+
+	rng_setup_status = dhd_rtt_get_geofence_setup_status(dhd);
+	rng_setup_status->geofence_setup_inprog = inprog;
+	if (inprog) {
+		ASSERT(peer_addr);
+		rng_inst = wl_cfgnan_check_for_ranging(cfg, peer_addr);
+		ASSERT(rng_inst);
+		if (rng_inst) {
+			rng_setup_status->rng_inst = rng_inst;
+		}
+	} else {
+		rng_setup_status->rng_inst = NULL;
+	}
+}
+
+int
+dhd_rtt_sched_geofencing_target(dhd_pub_t *dhd)
+{
+	rtt_geofence_target_info_t  *geofence_target_info;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	int ret = BCME_OK;
+	u8 rtt_invalid_reason = RTT_STATE_VALID;
+	struct bcm_cfg80211 *cfg = wl_get_cfg(dev);
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	rtt_geofence_cfg_t* geofence_cfg = &rtt_status->geofence_cfg;
+	int8 target_cnt = 0, cur_idx = DHD_RTT_INVALID_TARGET_INDEX;
+
+	NAN_MUTEX_LOCK();
+
+	if (wl_cfgnan_is_enabled(cfg) == FALSE) {
+		ret = BCME_NOTENABLED;
+		goto done;
+	}
+
+	DHD_RTT_ERR(("dhd_rtt_sched_geofencing_target: "
+		" sched_reason = %d, sessions cnt = %d, cur target idx = %d\n",
+		rtt_status->rtt_sched_reason, geofence_cfg->geofence_sessions_cnt,
+		rtt_status->geofence_cfg.cur_target_idx));
+
+	//ASSERT(!dhd_rtt_geofence_sessions_maxed_out(dhd));
+	if (dhd_rtt_geofence_sessions_maxed_out(dhd)) {
+		ret = BCME_NORESOURCE;
+		goto done;
+	}
+
+	target_cnt = dhd_rtt_get_geofence_target_cnt(dhd);
+
+	if (target_cnt == 0) {
+		DHD_RTT_MEM(("dhd_rtt_sched_geofencing_target: "
+			"No geofence targets to schedule\n"));
+		goto done;
+	}
+
+	cur_idx = dhd_rtt_get_geofence_cur_target_idx(dhd);
+	if (cur_idx == DHD_RTT_INVALID_TARGET_INDEX) {
+		/*
+		 * This Can be valid scenario, as cur_idx might
+		 * get invalidated, after rtt thread sched and
+		 * thread actually executing
+		 */
+		DHD_RTT_MEM(("dhd_rtt_sched_geofencing_target: "
+			"cur idx is invalid, bail out\n"));
+		goto done;
+	}
+
+	/* Get current geofencing target */
+	geofence_target_info = dhd_rtt_get_geofence_current_target(dhd);
+	//ASSERT(geofence_target_info);
+
+	/* call cfg API for trigerring geofencing RTT */
+	if (geofence_target_info) {
+		/* check for dp/others concurrency */
+		rtt_invalid_reason = dhd_rtt_invalid_states(dev,
+				&geofence_target_info->peer_addr);
+		if ((rtt_invalid_reason != RTT_STATE_VALID) ||
+			wl_cfgnan_check_role_concurrency(cfg,
+				&geofence_target_info->peer_addr)) {
+			/* TODO: see if we can move to next target..
+			* i.e, if invalid state is due to DP with same peer
+			*/
+			ret = BCME_BUSY;
+			DHD_RTT_ERR(("DRV State is not valid for RTT, "
+				"invalid_state = %d\n", rtt_invalid_reason));
+			goto done;
+		}
+
+		ret = wl_cfgnan_trigger_geofencing_ranging(dev,
+				&geofence_target_info->peer_addr);
+		if (ret == BCME_OK) {
+			dhd_rtt_set_geofence_setup_status(dhd, TRUE,
+				&geofence_target_info->peer_addr);
+		}
+	} else {
+		DHD_RTT(("No RTT target to schedule\n"));
+		ret = BCME_NOTFOUND;
+	}
+
+done:
+	NAN_MUTEX_UNLOCK();
+	return ret;
+}
+#endif /* WL_NAN */
+
+#ifdef WL_CFG80211
+#ifdef WL_NAN
+static void
+dhd_rtt_retry(dhd_pub_t *dhd)
+{
+
+	/* Attempt RTT for current geofence target */
+	wl_cfgnan_reset_geofence_ranging_for_cur_target(dhd,
+		RTT_SCHED_RTT_RETRY_GEOFENCE);
+
+}
+
+static void
+dhd_rtt_retry_work(struct work_struct *work)
+{
+	rtt_status_info_t *rtt_status = NULL;
+	dhd_pub_t *dhd = NULL;
+	struct net_device *dev = NULL;
+	struct bcm_cfg80211 *cfg = NULL;
+
+	if (!work) {
+		goto exit;
+	}
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+	rtt_status = container_of(work, rtt_status_info_t, rtt_retry_timer.work);
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+	dhd = rtt_status->dhd;
+	if (dhd == NULL) {
+		DHD_RTT_ERR(("%s : dhd is NULL\n", __FUNCTION__));
+		goto exit;
+	}
+	dev = dhd_linux_get_primary_netdev(dhd);
+	cfg = wl_get_cfg(dev);
+
+	NAN_MUTEX_LOCK();
+	(void) dhd_rtt_retry(dhd);
+	NAN_MUTEX_UNLOCK();
+
+exit:
+	return;
+}
+#endif /* WL_NAN */
+
+/*
+ * Return zero (0)
+ * for valid RTT state
+ * means if RTT is applicable
+ */
+uint8
+dhd_rtt_invalid_states(struct net_device *ndev, struct ether_addr *peer_addr)
+{
+	uint8 invalid_reason = RTT_STATE_VALID;
+	struct bcm_cfg80211 *cfg = wl_get_cfg(ndev);
+
+	UNUSED_PARAMETER(cfg);
+	UNUSED_PARAMETER(invalid_reason);
+
+	/* Make sure peer addr is not NULL in caller */
+	ASSERT(peer_addr);
+	/*
+	 * Keep adding prohibited drv states here
+	 * Only generic conditions which block
+	 * All RTTs like NDP connection
+	 */
+
+#ifdef WL_NAN
+	if (wl_cfgnan_data_dp_exists_with_peer(cfg, peer_addr)) {
+		invalid_reason = RTT_STATE_INV_REASON_NDP_EXIST;
+		DHD_RTT(("NDP in progress/connected, RTT prohibited\n"));
+		goto exit;
+	}
+#endif /* WL_NAN */
+
+	/* Remove below #defines once more exit calls come */
+#ifdef WL_NAN
+exit:
+#endif /* WL_NAN */
+	return invalid_reason;
+}
+#endif /* WL_CFG80211 */
+
+void
+dhd_rtt_schedule_rtt_work_thread(dhd_pub_t *dhd, int sched_reason)
+{
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	if (rtt_status == NULL) {
+		ASSERT(0);
+	} else {
+		rtt_status->rtt_sched_reason = sched_reason;
+		rtt_status->rtt_sched = TRUE;
+		schedule_work(&rtt_status->work);
+	}
+	return;
+}
+
+int
+dhd_rtt_stop(dhd_pub_t *dhd, struct ether_addr *mac_list, int mac_cnt)
+{
+	int err = BCME_OK;
+#ifdef WL_CFG80211
+	int i = 0, j = 0;
+	rtt_status_info_t *rtt_status;
+	rtt_results_header_t *entry, *next;
+	rtt_result_t *rtt_result, *next2;
+	struct rtt_noti_callback *iter;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	if (rtt_status->status == RTT_STOPPED) {
+		DHD_RTT_ERR(("rtt is not started\n"));
+		return BCME_OK;
+	}
+	DHD_RTT_ERR(("dhd_rtt_stop tgt_count %d\n", mac_cnt));
+	mutex_lock(&rtt_status->rtt_mutex);
+	for (i = 0; i < mac_cnt; i++) {
+		for (j = 0; j < rtt_status->rtt_config.rtt_target_cnt; j++) {
+			if (!bcmp(&mac_list[i], &rtt_status->rtt_config.target_info[j].addr,
+				ETHER_ADDR_LEN)) {
+				rtt_status->rtt_config.target_info[j].disable = TRUE;
+				dhd_rtt_delete_session(dhd,
+					rtt_status->rtt_config.target_info[j].sid);
+#ifdef WL_RTT_LCI
+				if (rtt_status->rtt_config.target_info[j].LCI) {
+					MFREE(dhd->osh, rtt_status->rtt_config.target_info[j].LCI,
+						rtt_status->rtt_config.target_info[j].LCI->len +
+						BCM_XTLV_HDR_SIZE);
+					rtt_status->rtt_config.target_info[j].LCI = NULL;
+				}
+				if (rtt_status->rtt_config.target_info[j].LCR) {
+					MFREE(dhd->osh, rtt_status->rtt_config.target_info[j].LCR,
+						rtt_status->rtt_config.target_info[j].LCR->len +
+						BCM_XTLV_HDR_SIZE);
+					rtt_status->rtt_config.target_info[j].LCR = NULL;
+				}
+#endif /* WL_RTT_LCI */
+			}
+		}
+	}
+	if (rtt_status->all_cancel) {
+		/* cancel all of request */
+		rtt_status->status = RTT_STOPPED;
+		for (i = 0; i < rtt_status->rtt_config.rtt_target_cnt; i++) {
+			dhd_rtt_delete_session(dhd,
+				rtt_status->rtt_config.target_info[i].sid);
+		}
+		DHD_RTT(("current RTT process is cancelled\n"));
+		/* remove the rtt results in cache */
+		if (!list_empty(&rtt_status->rtt_results_cache)) {
+			/* Iterate rtt_results_header list */
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			list_for_each_entry_safe(entry, next,
+				&rtt_status->rtt_results_cache, list) {
+				list_del(&entry->list);
+				/* Iterate rtt_result list */
+				list_for_each_entry_safe(rtt_result, next2,
+					&entry->result_list, list) {
+					list_del(&rtt_result->list);
+					MFREE(dhd->osh, rtt_result,
+						sizeof(rtt_result_t));
+				}
+				MFREE(dhd->osh, entry, sizeof(rtt_results_header_t));
+			}
+			GCC_DIAGNOSTIC_POP();
+		}
+		/* send the rtt complete event to wake up the user process */
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		list_for_each_entry(iter, &rtt_status->noti_fn_list, list) {
+			GCC_DIAGNOSTIC_POP();
+			iter->noti_fn(iter->ctx, &rtt_status->rtt_results_cache);
+		}
+		/* reinitialize the HEAD */
+		INIT_LIST_HEAD(&rtt_status->rtt_results_cache);
+		/* clear information for rtt_config */
+		rtt_status->rtt_config.rtt_target_cnt = 0;
+		memset(rtt_status->rtt_config.target_info, 0,
+			TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT));
+		rtt_status->cur_idx = 0;
+		/* Cancel pending proxd timeout work if any */
+		if (delayed_work_pending(&rtt_status->proxd_timeout)) {
+			cancel_delayed_work(&rtt_status->proxd_timeout);
+		}
+		dhd_rtt_delete_session(dhd, FTM_DEFAULT_SESSION);
+#ifdef WL_NAN
+		dhd_rtt_delete_nan_session(dhd);
+#endif /* WL_NAN */
+		dhd_rtt_ftm_enable(dhd, FALSE);
+	}
+	mutex_unlock(&rtt_status->rtt_mutex);
+#endif /* WL_CFG80211 */
+	return err;
+}
+
+#ifdef WL_CFG80211
+static void
+dhd_rtt_timeout(dhd_pub_t *dhd)
+{
+	rtt_status_info_t *rtt_status;
+#ifndef DHD_DUMP_ON_RTT_TIMEOUT
+	rtt_target_info_t *rtt_target = NULL;
+	rtt_target_info_t *rtt_target_info = NULL;
+#ifdef WL_NAN
+	int8 idx;
+	nan_ranging_inst_t *ranging_inst = NULL;
+	int ret = BCME_OK;
+	uint32 status;
+	struct net_device *ndev = dhd_linux_get_primary_netdev(dhd);
+	struct bcm_cfg80211 *cfg =  wiphy_priv(ndev->ieee80211_ptr->wiphy);
+#endif /* WL_NAN */
+#endif /* !DHD_DUMP_ON_RTT_TIMEOUT */
+
+	rtt_status = GET_RTTSTATE(dhd);
+	if (!rtt_status) {
+		DHD_RTT_ERR(("Proxd timer expired but no RTT status\n"));
+		goto exit;
+	}
+
+	if (RTT_IS_STOPPED(rtt_status)) {
+		DHD_RTT_ERR(("Proxd timer expired but no RTT Request\n"));
+		goto exit;
+	}
+
+#ifdef DHD_DUMP_ON_RTT_TIMEOUT
+	/* Dump, and Panic depending on memdump.info */
+#ifdef BCMDONGLEHOST
+	if (dhd_query_bus_erros(dhd)) {
+		goto exit;
+	}
+#ifdef DHD_FW_COREDUMP
+	if (dhd->memdump_enabled) {
+		/* Behave based on user memdump info */
+		dhd->memdump_type = DUMP_TYPE_PROXD_TIMEOUT;
+		dhd_bus_mem_dump(dhd);
+	}
+#endif /* DHD_FW_COREDUMP */
+#endif /* BCMDONGLEHOST */
+#else /* DHD_DUMP_ON_RTT_TIMEOUT */
+#ifdef WL_NAN
+	if (rtt_status->rtt_config.target_list_mode == RNG_TARGET_LIST_MODE_NAN) {
+		for (idx = rtt_status->start_idx;
+			idx < rtt_status->rtt_config.rtt_target_cnt; idx++) {
+			rtt_target = &rtt_status->rtt_config.target_info[idx];
+			if ((!rtt_target->disable) &&
+				(!dhd_rtt_get_report_header(rtt_status,
+				NULL, &rtt_target->addr))) {
+				if (wl_cfgnan_ranging_is_in_prog_for_peer(cfg, &rtt_target->addr)) {
+					ranging_inst = wl_cfgnan_check_for_ranging(cfg,
+						&rtt_target->addr);
+					if (ranging_inst) {
+						ret =  wl_cfgnan_cancel_ranging(ndev, cfg,
+							&ranging_inst->range_id,
+							NAN_RNG_TERM_FLAG_IMMEDIATE, &status);
+						if (unlikely(ret) || unlikely(status)) {
+							WL_ERR(("%s:nan range cancel failed "
+								"ret = %d status = %d\n",
+								__FUNCTION__, ret, status));
+						}
+					} else {
+						WL_ERR(("%s: Ranging Instance get failed\n",
+							__FUNCTION__));
+					}
+				}
+				dhd_rtt_create_failure_result(rtt_status, &rtt_target->addr);
+			}
+		}
+		dhd_rtt_handle_rtt_session_end(dhd);
+		/* reset directed cfg params */
+		rtt_status->directed_cfg.directed_setup_status.rng_inst = NULL;
+		rtt_status->directed_cfg.directed_setup_status.directed_na_setup_inprog = FALSE;
+		rtt_status->directed_cfg.directed_sessions_cnt = 0;
+	} else
+#endif /* WL_NAN */
+	{
+		/* Cancel RTT for target and proceed to next target */
+		rtt_target_info = rtt_status->rtt_config.target_info;
+		if ((!rtt_target_info) ||
+				(rtt_status->cur_idx >= rtt_status->rtt_config.rtt_target_cnt)) {
+			goto exit;
+		}
+		rtt_target = &rtt_target_info[rtt_status->cur_idx];
+		WL_ERR(("Proxd timer expired for Target: "MACDBG" \n",
+			MAC2STRDBG(&rtt_target->addr)));
+		/* For Legacy RTT */
+		dhd_rtt_delete_session(dhd, FTM_DEFAULT_SESSION);
+		dhd_rtt_create_failure_result(rtt_status, &rtt_target->addr);
+		dhd_rtt_handle_rtt_session_end(dhd);
+	}
+#endif /* DHD_DUMP_ON_RTT_TIMEOUT */
+exit:
+	return;
+}
+
+static void
+dhd_rtt_timeout_work(struct work_struct *work)
+{
+	rtt_status_info_t *rtt_status = NULL;
+	dhd_pub_t *dhd = NULL;
+
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+	rtt_status = container_of(work, rtt_status_info_t, proxd_timeout.work);
+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+	dhd = rtt_status->dhd;
+	if (dhd == NULL) {
+		DHD_RTT_ERR(("%s : dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+
+	mutex_lock(&rtt_status->rtt_mutex);
+	(void) dhd_rtt_timeout(dhd);
+	mutex_unlock(&rtt_status->rtt_mutex);
+}
+
+static void
+dhd_rtt_set_ftm_config_ratespec(ftm_config_param_info_t *ftm_params,
+		int *ftm_param_cnt, rtt_target_info_t *rtt_target)
+{
+	bool use_default = FALSE;
+	int nss;
+	int mcs;
+	uint32 rspec = 0;
+
+	if (!(rtt_target->bw && rtt_target->preamble)) {
+		goto exit;
+	}
+	switch (rtt_target->preamble) {
+		case RTT_PREAMBLE_LEGACY:
+			rspec |= WL_RSPEC_ENCODE_RATE; /* 11abg */
+			rspec |= WL_RATE_6M;
+			break;
+		case RTT_PREAMBLE_HT:
+			rspec |= WL_RSPEC_ENCODE_HT; /* 11n HT */
+			mcs = 0; /* default MCS 0 */
+			rspec |= mcs;
+			break;
+		case RTT_PREAMBLE_VHT:
+			rspec |= WL_RSPEC_ENCODE_VHT; /* 11ac VHT */
+			mcs = 0; /* default MCS 0 */
+			nss = 1; /* default Nss = 1  */
+			rspec |= (nss << WL_RSPEC_VHT_NSS_SHIFT) | mcs;
+			break;
+		default:
+			DHD_RTT(("doesn't support this preamble : %d\n",
+				rtt_target->preamble));
+			use_default = TRUE;
+			break;
+	}
+	switch (rtt_target->bw) {
+		case RTT_BW_20:
+			rspec |= WL_RSPEC_BW_20MHZ;
+			break;
+		case RTT_BW_40:
+			rspec |= WL_RSPEC_BW_40MHZ;
+			break;
+		case RTT_BW_80:
+			rspec |= WL_RSPEC_BW_80MHZ;
+			break;
+		default:
+			DHD_RTT(("doesn't support this BW : %d\n",
+				rtt_target->bw));
+			use_default = TRUE;
+			break;
+	}
+	if (!use_default) {
+		ftm_params[*ftm_param_cnt].data32 = htol32(rspec);
+		ftm_params[*ftm_param_cnt].tlvid =
+			WL_PROXD_TLV_ID_RATESPEC;
+		*ftm_param_cnt = *ftm_param_cnt + 1;
+		DHD_RTT((">\t ratespec : %d\n", rspec));
+	}
+
+exit:
+	return;
+
+}
+
+static void
+dhd_rtt_set_ftm_config_param(ftm_config_param_info_t *ftm_params,
+	int *ftm_param_cnt, rtt_target_info_t *rtt_target, uint16 tlvid)
+{
+	char eabuf[ETHER_ADDR_STR_LEN];
+	char chanbuf[CHANSPEC_STR_LEN];
+	uint32 num_ftm_pref = rtt_target->num_frames_per_burst;
+
+	switch (tlvid) {
+		case WL_PROXD_TLV_ID_CUR_ETHER_ADDR:
+			/* local mac address */
+			if (!ETHER_ISNULLADDR(rtt_target->local_addr.octet)) {
+				ftm_params[*ftm_param_cnt].mac_addr = rtt_target->local_addr;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_CUR_ETHER_ADDR;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				bcm_ether_ntoa(&rtt_target->local_addr, eabuf);
+				DHD_RTT((">\t local %s\n", eabuf));
+			}
+			break;
+		case WL_PROXD_TLV_ID_PEER_MAC:
+			/* target's mac address */
+			if (!ETHER_ISNULLADDR(rtt_target->addr.octet)) {
+				ftm_params[*ftm_param_cnt].mac_addr = rtt_target->addr;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_PEER_MAC;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				bcm_ether_ntoa(&rtt_target->addr, eabuf);
+				DHD_RTT((">\t target %s\n", eabuf));
+			}
+			break;
+		case WL_PROXD_TLV_ID_CHANSPEC:
+			/* target's chanspec */
+			if (rtt_target->chanspec) {
+				ftm_params[*ftm_param_cnt].chanspec =
+					htol32((uint32)rtt_target->chanspec);
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_CHANSPEC;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				wf_chspec_ntoa(rtt_target->chanspec, chanbuf);
+				DHD_RTT((">\t chanspec : %s\n", chanbuf));
+			}
+			break;
+		case WL_PROXD_TLV_ID_NUM_BURST:
+			/* num-burst */
+			if (rtt_target->num_burst) {
+				ftm_params[*ftm_param_cnt].data16 = htol16(rtt_target->num_burst);
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_NUM_BURST;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				DHD_RTT((">\t num of burst : %d\n", rtt_target->num_burst));
+			}
+			break;
+		case WL_PROXD_TLV_ID_BURST_NUM_FTM:
+			/* number of frame per burst */
+			/* get pref num_ftm based on chanspec */
+			ftm_params[*ftm_param_cnt].data16 = htol16(num_ftm_pref);
+			ftm_params[*ftm_param_cnt].tlvid =
+				WL_PROXD_TLV_ID_BURST_NUM_FTM;
+			*ftm_param_cnt = *ftm_param_cnt + 1;
+			DHD_RTT((">\t number of frame per burst : %d\n",
+				rtt_target->num_frames_per_burst));
+			break;
+		case WL_PROXD_TLV_ID_FTM_RETRIES:
+			/* FTM retry count */
+			if (rtt_target->num_retries_per_ftm) {
+				ftm_params[*ftm_param_cnt].data8 = rtt_target->num_retries_per_ftm;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_FTM_RETRIES;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				DHD_RTT((">\t retry count of FTM  : %d\n",
+					rtt_target->num_retries_per_ftm));
+			}
+			break;
+		case WL_PROXD_TLV_ID_FTM_REQ_RETRIES:
+			/* FTM Request retry count */
+			if (rtt_target->num_retries_per_ftmr) {
+				ftm_params[*ftm_param_cnt].data8 = rtt_target->num_retries_per_ftmr;
+				ftm_params[*ftm_param_cnt].tlvid =
+					WL_PROXD_TLV_ID_FTM_REQ_RETRIES;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				DHD_RTT((">\t retry count of FTM Req : %d\n",
+					rtt_target->num_retries_per_ftmr));
+			}
+			break;
+		case WL_PROXD_TLV_ID_BURST_PERIOD:
+			/* burst-period */
+			if (rtt_target->burst_period) {
+				ftm_params[*ftm_param_cnt].data_intvl.intvl =
+					htol32(rtt_target->burst_period); /* ms */
+				ftm_params[*ftm_param_cnt].data_intvl.tmu = WL_PROXD_TMU_MILLI_SEC;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_BURST_PERIOD;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				DHD_RTT((">\t burst period : %d ms\n", rtt_target->burst_period));
+			}
+			break;
+		case WL_PROXD_TLV_ID_BURST_DURATION:
+			/* burst-duration */
+			rtt_target->burst_duration = FTM_MAX_BURST_DUR_TMO_MS;
+			if (rtt_target->burst_duration) {
+				ftm_params[*ftm_param_cnt].data_intvl.intvl =
+					htol32(rtt_target->burst_duration); /* ms */
+				ftm_params[*ftm_param_cnt].data_intvl.tmu = WL_PROXD_TMU_MILLI_SEC;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_BURST_DURATION;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				DHD_RTT((">\t burst duration : %d ms\n",
+					rtt_target->burst_duration));
+			}
+			break;
+		case WL_PROXD_TLV_ID_BURST_TIMEOUT:
+			/* burst-timeout */
+			rtt_target->burst_timeout = FTM_MAX_BURST_DUR_TMO_MS;
+			if (rtt_target->burst_timeout) {
+				ftm_params[*ftm_param_cnt].data_intvl.intvl =
+					htol32(rtt_target->burst_timeout); /* ms */
+				ftm_params[*ftm_param_cnt].data_intvl.tmu = WL_PROXD_TMU_MILLI_SEC;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_BURST_TIMEOUT;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+				DHD_RTT((">\t burst timeout : %d ms\n",
+					rtt_target->burst_timeout));
+			}
+			break;
+		case WL_PROXD_TLV_ID_RATESPEC:
+			dhd_rtt_set_ftm_config_ratespec(ftm_params,
+				ftm_param_cnt, rtt_target);
+			break;
+		case WL_PROXD_TLV_ID_EVENT_MASK:
+			{
+				/* set burst end and session end in ev mask by def */
+				uint32 event_mask = ((1 << WL_PROXD_EVENT_BURST_END) |
+						(1 << WL_PROXD_EVENT_SESSION_END));
+#ifdef WL_RTT_LCI
+				/* LCI request */
+				if (rtt_target && rtt_target->LCI_request) {
+					event_mask |= (1 << WL_PROXD_EVENT_LCI_MEAS_REP);
+				}
+				/* LCR request (CIVIC) */
+				if (rtt_target && rtt_target->LCR_request) {
+					event_mask |= (1 << WL_PROXD_EVENT_CIVIC_MEAS_REP);
+				}
+#endif /* WL_RTT_LCI */
+				/* only burst end for directed nan-rtt target */
+				if (rtt_target && (rtt_target->peer == RTT_PEER_NAN)) {
+					event_mask = (1 << WL_PROXD_EVENT_BURST_END);
+				}
+				ftm_params[*ftm_param_cnt].event_mask = event_mask;
+				ftm_params[*ftm_param_cnt].tlvid = WL_PROXD_TLV_ID_EVENT_MASK;
+				*ftm_param_cnt = *ftm_param_cnt + 1;
+			}
+			break;
+		default:
+			DHD_RTT_ERR(("Invalid FTM Param Config, tlvid = %d\n", tlvid));
+			break;
+	}
+
+	return;
+}
+
+/* API to configure a legacy/sta RTT session.
+ * @rtt_target will be having the user configuration for RTT session
+ */
+static int
+dhd_rtt_config_sta_rtt(dhd_pub_t *dhd, struct net_device *dev,
+	rtt_target_info_t *rtt_target)
+{
+	int ftm_cfg_cnt = 0;
+	ftm_config_options_info_t ftm_configs[FTM_MAX_CONFIGS];
+	ftm_config_param_info_t ftm_params[FTM_MAX_PARAMS];
+	int ftm_param_cnt = 0;
+	int err = BCME_OK;
+	uint8 err_at = 0;
+	u8 ioctl_buf[WLC_IOCTL_SMLEN];
+
+	memset(ftm_configs, 0, sizeof(ftm_configs));
+	memset(ftm_params, 0, sizeof(ftm_params));
+
+	/* configure the session 1 as initiator */
+	if  (ftm_cfg_cnt < FTM_MAX_CONFIGS) {
+		ftm_configs[ftm_cfg_cnt].enable = TRUE;
+		ftm_configs[ftm_cfg_cnt].flags =
+			WL_PROXD_SESSION_FLAG_INITIATOR | WL_PROXD_SESSION_FLAG_RANDMAC;
+		if (rtt_target->type == RTT_ONE_WAY) {
+			ftm_configs[ftm_cfg_cnt].flags |= WL_PROXD_SESSION_FLAG_ONE_WAY;
+		}
+#ifdef WL_RTT_LCI
+		/* LCI request */
+		if (rtt_target->LCI_request) {
+			ftm_configs[ftm_cfg_cnt].flags |= WL_PROXD_SESSION_FLAG_REQ_LCI;
+		}
+		/* LCR request (CIVIC) */
+		if (rtt_target->LCR_request) {
+			ftm_configs[ftm_cfg_cnt].flags |= WL_PROXD_SESSION_FLAG_REQ_CIV;
+		}
+#endif /* WL_RTT_LCI */
+		DHD_RTT_ERR(("RTT flags for the session %x sid %d\n",
+			ftm_configs[ftm_cfg_cnt].flags, rtt_target->sid));
+		ftm_cfg_cnt++;
+	} else {
+		DHD_RTT_ERR(("Max FTM Config Options exceeded\n"));
+		err = BCME_ERROR;
+		err_at = 1;
+		goto exit;
+	}
+
+	memset(ioctl_buf, 0, WLC_IOCTL_SMLEN);
+	/* Rand Mac for newer version in place of cur_eth */
+	if (dhd->wlc_ver_major < RTT_IOV_CUR_ETH_OBSOLETE) {
+		err = wldev_iovar_getbuf(dev, "cur_etheraddr", NULL, 0,
+				ioctl_buf, WLC_IOCTL_SMLEN, NULL);
+		if (err) {
+			DHD_RTT_ERR(("WLC_GET_CUR_ETHERADDR failed, error %d\n", err));
+			err_at = 2;
+			goto exit;
+		}
+		memcpy(rtt_target->local_addr.octet, ioctl_buf, ETHER_ADDR_LEN);
+
+		/* local mac address */
+		DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+			rtt_target, WL_PROXD_TLV_ID_CUR_ETHER_ADDR);
+	}
+	/* target's mac address */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_PEER_MAC);
+
+	/* target's chanspec */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_CHANSPEC);
+
+	/* num-burst */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_NUM_BURST);
+
+	/* number of frame per burst */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_BURST_NUM_FTM);
+
+	/* FTM retry count */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_FTM_RETRIES);
+
+	/* FTM Request retry count */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_FTM_REQ_RETRIES);
+
+	/* burst-period */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_BURST_PERIOD);
+
+	/* Setting both duration and timeout to MAX duration
+	 * to handle the congestion environments.
+	 * Hence ignoring the user config.
+	 */
+	/* burst-duration */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_BURST_DURATION);
+
+	/* burst-timeout */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_BURST_TIMEOUT);
+
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_RATESPEC);
+
+	/* event_mask..applicable for only Legacy RTT.
+	 * For nan-rtt config happens from firmware
+	 */
+	DHD_RTT_CHK_SET_PARAM(ftm_params, ftm_param_cnt,
+		rtt_target, WL_PROXD_TLV_ID_EVENT_MASK);
+
+	err = dhd_rtt_ftm_config(dhd, rtt_target->sid, ftm_configs,
+			ftm_cfg_cnt, ftm_params, ftm_param_cnt);
+	if (err != BCME_OK) {
+		err_at = 3;
+		goto exit;
+	}
+
+exit:
+	if (err != BCME_OK) {
+		DHD_RTT_ERR(("dhd_rtt_config_sta_rtt: err %d err_at %d\n",
+			err, err_at));
+	}
+	return err;
+}
+
+#ifdef DHD_RTT_USE_FTM_RANGE
+
+static int
+dhd_rtt_stop_ranging(dhd_pub_t *dhd)
+{
+	ftm_subcmd_info_t subcmd_info;
+	subcmd_info.name = "stop-ranging";
+	subcmd_info.cmdid = WL_PROXD_CMD_STOP_RANGING;
+	subcmd_info.handler = NULL;
+	return dhd_rtt_common_set_handler(dhd, &subcmd_info,
+			WL_PROXD_METHOD_FTM, WL_PROXD_SESSION_ID_GLOBAL);
+}
+
+static int
+dhd_rtt_start_ranging(dhd_pub_t *dhd, rtt_config_params_t *config)
+{
+	wl_proxd_tlv_t *p_tlv;
+	wl_proxd_iov_t *p_proxd_iov;
+	uint16 proxd_iovsize = 0;
+	uint16 bufsize;
+	uint16 buf_space_left;
+	uint16 all_tlvsize;
+	wl_proxd_ranging_flags_t flags;
+	wl_proxd_ranging_flags_t flags_mask;
+	uint16 ranging_sids_size = 0;
+	wl_proxd_session_id_list_t *ranging_sids = NULL;
+	uint8 i = 0;
+	int ret = BCME_OK;
+
+	p_proxd_iov = rtt_alloc_getset_buf(dhd, WL_PROXD_METHOD_FTM,
+		WL_PROXD_SESSION_ID_GLOBAL, WL_PROXD_CMD_START_RANGING, FTM_IOC_BUFSZ,
+		&proxd_iovsize);
+
+	if (p_proxd_iov == NULL) {
+		DHD_RTT_ERR(("dhd_rtt_start_ranging: failed to allocate the iovar (size :%d)\n",
+			FTM_IOC_BUFSZ));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+
+	/* setup TLVs */
+	bufsize = proxd_iovsize - WL_PROXD_IOV_HDR_SIZE; /* adjust available size for TLVs */
+	p_tlv = &p_proxd_iov->tlvs[0];
+	buf_space_left = bufsize;
+
+	/* flags & mask */
+	flags = WL_PROXD_RANGING_FLAG_DEL_SESSIONS_ON_STOP;
+	flags_mask = WL_PROXD_RANGING_FLAG_ALL;
+
+	ret = bcm_pack_xtlv_entry((uint8 **) &p_tlv, &buf_space_left,
+		WL_PROXD_TLV_ID_RANGING_FLAGS,
+		sizeof(uint16), (uint8 *)&flags, BCM_XTLV_OPTION_ALIGN32);
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("dhd_rtt_start_ranging: failed to pack ranging-flags in xtlv,"
+			" err=%d\n", ret));
+		goto done;
+	}
+
+	ret = bcm_pack_xtlv_entry((uint8 **) &p_tlv, &buf_space_left,
+		WL_PROXD_TLV_ID_RANGING_FLAGS_MASK,
+		sizeof(uint16), (uint8 *)&flags_mask, BCM_XTLV_OPTION_ALIGN32);
+
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("dhd_rtt_start_ranging: failed to pack ranging flags_mask in xtlv,"
+			" err=%d\n", ret));
+		goto done;	/* abort */
+	}
+
+	/* allocate a temp buffer for parsing cmd-args */
+	ranging_sids_size = OFFSETOF(wl_proxd_session_id_list_t, ids) +
+		(config->rtt_target_cnt) * sizeof(wl_proxd_session_id_t);
+	ranging_sids = (wl_proxd_session_id_list_t *)MALLOCZ(dhd->osh,
+		ranging_sids_size);
+	if (ranging_sids == NULL) {
+		DHD_RTT_ERR(("dhd_rtt_start_ranging: failed to allocate %d bytes of"
+			" memory for ranging\n", ranging_sids_size));
+		ret = BCME_NOMEM;
+		goto done;
+	}
+
+	for (i = 0; i < config->rtt_target_cnt; i++) {
+		ranging_sids->ids[i] = config->target_info[i].sid;
+	}
+
+	ranging_sids->num_ids = config->rtt_target_cnt;
+
+	ret = bcm_pack_xtlv_entry((uint8 **) &p_tlv, &buf_space_left,
+		WL_PROXD_TLV_ID_SESSION_ID_LIST, ranging_sids_size,
+		(uint8 *)ranging_sids, BCM_XTLV_OPTION_ALIGN32);
+
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("dhd_rtt_start_ranging: failed to pack ranging-ids in xtlv, err=%d\n",
+			ret));
+		goto done;
+	}
+
+	/* update the iov header, set len to include all TLVs + header */
+	all_tlvsize = (bufsize - buf_space_left);
+	p_proxd_iov->len = htol16(all_tlvsize + WL_PROXD_IOV_HDR_SIZE);
+	ret = dhd_iovar(dhd, 0, "proxd", (char *)p_proxd_iov,
+			all_tlvsize + WL_PROXD_IOV_HDR_SIZE, NULL, 0, TRUE);
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("%s : failed to set config err %d\n", __FUNCTION__, ret));
+		goto done;
+	}
+
+done:
+	MFREE(dhd->osh, p_proxd_iov, proxd_iovsize);
+	MFREE(dhd->osh, ranging_sids, ranging_sids_size);
+	return ret;
+}
+#endif /* DHD_RTT_USE_FTM_RANGE */
+
+/* Work thread API to start the RTT.
+ * If all targets are AP only, then this API will confgure all sessions
+ * and start(FW) the RTT for first session.
+ * Else, this API will configure and start a single RTT session.
+ * If no host provided targets present, then this API will check for
+ * geofence RTT targets and trigger the same.
+ */
+static int
+dhd_rtt_start(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	int err_at = 0;
+	rtt_target_info_t *rtt_target;
+	rtt_status_info_t *rtt_status;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	u8 rtt_invalid_reason = RTT_STATE_VALID;
+	int rtt_sched_type = RTT_TYPE_INVALID;
+	int i = 0;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+
+	DHD_RTT(("Enter %s\n", __FUNCTION__));
+
+	if (RTT_IS_STOPPED(rtt_status)) {
+		DHD_RTT(("No Directed RTT target to process, check for geofence\n"));
+		goto geofence;
+	}
+
+	if (rtt_status->cur_idx >= rtt_status->rtt_config.rtt_target_cnt) {
+		err = BCME_RANGE;
+		err_at = 1;
+		DHD_RTT(("%s : idx %d is out of range\n", __FUNCTION__, rtt_status->cur_idx));
+		if (rtt_status->flags == WL_PROXD_SESSION_FLAG_TARGET) {
+			DHD_RTT_ERR(("STA is set as Target/Responder \n"));
+			err = BCME_ERROR;
+		}
+		goto exit;
+	}
+
+	/* Get a target information */
+	rtt_target = &rtt_status->rtt_config.target_info[rtt_status->cur_idx];
+
+	DHD_RTT(("%s enter\n", __FUNCTION__));
+
+	if (ETHER_ISNULLADDR(rtt_target->addr.octet)) {
+		err = BCME_BADADDR;
+		err_at = 2;
+		DHD_RTT(("RTT Target addr is NULL\n"));
+		goto exit;
+	}
+
+	/* check for dp/others concurrency */
+	rtt_invalid_reason = dhd_rtt_invalid_states(dev, &rtt_target->addr);
+	if (rtt_invalid_reason != RTT_STATE_VALID) {
+		err = BCME_BUSY;
+		err_at = 3;
+		DHD_RTT(("DRV State is not valid for RTT\n"));
+		goto exit;
+	}
+
+	/* enable ftm */
+	DHD_RTT_MEM(("Enabling FTM \n"));
+	err = dhd_rtt_ftm_enable(dhd, TRUE);
+	if (err) {
+		DHD_RTT_ERR(("failed to enable FTM (%d)\n", err));
+		err_at = 4;
+		goto exit;
+	}
+	rtt_status->status = RTT_ENABLED;
+
+#ifdef WL_NAN
+	if (rtt_target->peer == RTT_PEER_NAN) {
+		rtt_sched_type = RTT_TYPE_NAN_DIRECTED;
+		/* Ignore return value..failure taken care inside the API */
+		dhd_rtt_nan_start_session(dhd, rtt_target);
+		goto exit;
+	}
+#endif /* WL_NAN */
+
+	rtt_sched_type = RTT_TYPE_LEGACY;
+	mutex_lock(&rtt_status->rtt_mutex);
+	if (rtt_status->rtt_config.target_list_mode ==
+		RNG_TARGET_LIST_MODE_LEGACY) {
+		uint16 sid = WL_PROXD_SID_HOST_START;
+		DHD_RTT_MEM(("Configuring RTT sessions, count %d\n",
+			rtt_status->rtt_config.rtt_target_cnt));
+		for (i = 0; i < rtt_status->rtt_config.rtt_target_cnt; i++) {
+			rtt_target = &rtt_status->rtt_config.target_info[i];
+			rtt_target->sid = sid++;
+			err = dhd_rtt_config_sta_rtt(dhd, dev, rtt_target);
+			if (err) {
+				goto exit;
+			}
+		}
+		rtt_target = &rtt_status->rtt_config.target_info[0];
+#ifdef DHD_RTT_USE_FTM_RANGE
+		err = dhd_rtt_start_ranging(dhd, &rtt_status->rtt_config);
+#else
+		err = dhd_rtt_start_session(dhd, rtt_target->sid, TRUE);
+#endif /* DHD_RTT_USE_FTM_RANGE */
+		if (err) {
+			goto exit;
+		}
+	} else {
+		rtt_target->sid = FTM_DEFAULT_SESSION;
+		dhd_rtt_delete_session(dhd, FTM_DEFAULT_SESSION);
+		err = dhd_rtt_config_sta_rtt(dhd, dev, rtt_target);
+		if (err) {
+			goto exit;
+		}
+		err = dhd_rtt_start_session(dhd, FTM_DEFAULT_SESSION, TRUE);
+	}
+	mutex_unlock(&rtt_status->rtt_mutex);
+
+	if (err) {
+		DHD_RTT_ERR(("failed to start session of FTM : error %d\n", err));
+		err_at = 5;
+		goto exit;
+	} else {
+		/* schedule proxd timeout */
+		schedule_delayed_work(&rtt_status->proxd_timeout,
+			msecs_to_jiffies(DHD_NAN_RTT_TIMER_INTERVAL_MS));
+
+	}
+
+	goto exit;
+geofence:
+#ifdef WL_NAN
+	/* sched geofencing rtt */
+	rtt_sched_type = RTT_TYPE_NAN_GEOFENCE;
+	if ((err = dhd_rtt_sched_geofencing_target(dhd)) != BCME_OK) {
+		DHD_RTT_ERR(("geofencing sched failed, err = %d\n", err));
+		err_at = 6;
+	}
+#endif /* WL_NAN */
+
+exit:
+	if (err) {
+		/* RTT Failed */
+		DHD_RTT_ERR(("dhd_rtt_start: Failed & RTT_STOPPED, err = %d,"
+			" err_at = %d, rtt_sched_type = %d, rtt_invalid_reason = %d\n"
+			" sched_reason = %d",
+			err, err_at, rtt_sched_type, rtt_invalid_reason,
+			rtt_status->rtt_sched_reason));
+		rtt_status->status = RTT_STOPPED;
+		/* disable FTM */
+		dhd_rtt_ftm_enable(dhd, FALSE);
+	}
+	rtt_status->rtt_sched = FALSE;
+	return err;
+}
+#endif /* WL_CFG80211 */
+
+int
+dhd_rtt_register_noti_callback(dhd_pub_t *dhd, void *ctx, dhd_rtt_compl_noti_fn noti_fn)
+{
+	int err = BCME_OK;
+	struct rtt_noti_callback *cb = NULL, *iter;
+	rtt_status_info_t *rtt_status;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(noti_fn, "noti_fn is NULL", err);
+
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	spin_lock_bh(&noti_list_lock);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry(iter, &rtt_status->noti_fn_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		if (iter->noti_fn == noti_fn) {
+			goto exit;
+		}
+	}
+	cb = (struct rtt_noti_callback *)MALLOCZ(dhd->osh, sizeof(struct rtt_noti_callback));
+	if (!cb) {
+		err = -ENOMEM;
+		goto exit;
+	}
+	cb->noti_fn = noti_fn;
+	cb->ctx = ctx;
+	list_add(&cb->list, &rtt_status->noti_fn_list);
+exit:
+	spin_unlock_bh(&noti_list_lock);
+	return err;
+}
+
+int
+dhd_rtt_unregister_noti_callback(dhd_pub_t *dhd, dhd_rtt_compl_noti_fn noti_fn)
+{
+	int err = BCME_OK;
+	struct rtt_noti_callback *cb = NULL, *iter;
+	rtt_status_info_t *rtt_status;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	NULL_CHECK(noti_fn, "noti_fn is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	spin_lock_bh(&noti_list_lock);
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	list_for_each_entry(iter, &rtt_status->noti_fn_list, list) {
+		GCC_DIAGNOSTIC_POP();
+		if (iter->noti_fn == noti_fn) {
+			cb = iter;
+			list_del(&cb->list);
+			break;
+		}
+	}
+
+	spin_unlock_bh(&noti_list_lock);
+	if (cb) {
+		MFREE(dhd->osh, cb, sizeof(struct rtt_noti_callback));
+	}
+	return err;
+}
+
+static wifi_rate_v1
+dhd_rtt_convert_rate_to_host(uint32 rspec)
+{
+	wifi_rate_v1 host_rate;
+	uint32 bandwidth;
+	memset(&host_rate, 0, sizeof(wifi_rate_v1));
+	if (RSPEC_ISLEGACY(rspec)) {
+		host_rate.preamble = 0;
+	} else if (RSPEC_ISHT(rspec)) {
+		host_rate.preamble = 2;
+		host_rate.rateMcsIdx = rspec & WL_RSPEC_RATE_MASK;
+	} else if (RSPEC_ISVHT(rspec)) {
+		host_rate.preamble = 3;
+		host_rate.rateMcsIdx = rspec & WL_RSPEC_VHT_MCS_MASK;
+		host_rate.nss = (rspec & WL_RSPEC_VHT_NSS_MASK) >> WL_RSPEC_VHT_NSS_SHIFT;
+	}
+
+	bandwidth = RSPEC_BW(rspec);
+	switch (bandwidth) {
+	case WL_RSPEC_BW_20MHZ:
+		host_rate.bw = RTT_RATE_20M;
+		break;
+	case WL_RSPEC_BW_40MHZ:
+		host_rate.bw = RTT_RATE_40M;
+		break;
+	case WL_RSPEC_BW_80MHZ:
+		host_rate.bw = RTT_RATE_80M;
+		break;
+	case WL_RSPEC_BW_160MHZ:
+		host_rate.bw = RTT_RATE_160M;
+		break;
+	default:
+		host_rate.bw = RTT_RATE_20M;
+		break;
+	}
+
+	host_rate.bitrate = rate_rspec2rate(rspec) / 100; /* 100kbps */
+	DHD_RTT(("bit rate : %d\n", host_rate.bitrate));
+	return host_rate;
+}
+
+#define FTM_FRAME_TYPES	{"SETUP", "TRIGGER", "TIMESTAMP"}
+static int
+dhd_rtt_convert_results_to_host_v1(rtt_result_t *rtt_result, const uint8 *p_data,
+	uint16 tlvid, uint16 len)
+{
+	int i;
+	int err = BCME_OK;
+	char eabuf[ETHER_ADDR_STR_LEN];
+	wl_proxd_result_flags_t flags;
+	wl_proxd_session_state_t session_state;
+	wl_proxd_status_t proxd_status;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	struct osl_timespec ts;
+#endif /* LINUX_VER >= 2.6.39 */
+	uint32 ratespec;
+	int32 avg_dist;
+	const wl_proxd_rtt_result_v1_t *p_data_info = NULL;
+	const wl_proxd_rtt_sample_v1_t *p_sample_avg = NULL;
+	const wl_proxd_rtt_sample_v1_t *p_sample = NULL;
+	wl_proxd_intvl_t rtt;
+	wl_proxd_intvl_t p_time;
+	uint16 num_rtt = 0, snr = 0, bitflips = 0;
+	wl_proxd_phy_error_t tof_phy_error = 0;
+	wl_proxd_phy_error_t tof_phy_tgt_error = 0;
+	wl_proxd_snr_t tof_target_snr = 0;
+	wl_proxd_bitflips_t tof_target_bitflips = 0;
+	int16 rssi = 0;
+	int32 dist = 0;
+	uint8 num_ftm = 0;
+	char *ftm_frame_types[] = FTM_FRAME_TYPES;
+	rtt_report_t *rtt_report = &(rtt_result->report);
+
+	BCM_REFERENCE(ftm_frame_types);
+	BCM_REFERENCE(dist);
+	BCM_REFERENCE(rssi);
+	BCM_REFERENCE(tof_target_bitflips);
+	BCM_REFERENCE(tof_target_snr);
+	BCM_REFERENCE(tof_phy_tgt_error);
+	BCM_REFERENCE(tof_phy_error);
+	BCM_REFERENCE(bitflips);
+	BCM_REFERENCE(snr);
+	BCM_REFERENCE(session_state);
+	BCM_REFERENCE(ftm_session_state_value_to_logstr);
+
+	NULL_CHECK(rtt_report, "rtt_report is NULL", err);
+	NULL_CHECK(p_data, "p_data is NULL", err);
+	DHD_RTT(("%s enter\n", __FUNCTION__));
+	p_data_info = (const wl_proxd_rtt_result_v1_t *) p_data;
+	/* unpack and format 'flags' for display */
+	flags = ltoh16_ua(&p_data_info->flags);
+
+	/* session state and status */
+	session_state = ltoh16_ua(&p_data_info->state);
+	proxd_status = ltoh32_ua(&p_data_info->status);
+	bcm_ether_ntoa((&(p_data_info->peer)), eabuf);
+	DHD_RTT((">\tTarget(%s) session state=%d(%s), status=%d(%s)\n",
+		eabuf,
+		session_state,
+		ftm_session_state_value_to_logstr(session_state),
+		proxd_status,
+		ftm_status_value_to_logstr(proxd_status)));
+
+	/* show avg_dist (1/256m units), burst_num */
+	avg_dist = ltoh32_ua(&p_data_info->avg_dist);
+	if (avg_dist == 0xffffffff) {	/* report 'failure' case */
+		DHD_RTT((">\tavg_dist=-1m, burst_num=%d, valid_measure_cnt=%d\n",
+		ltoh16_ua(&p_data_info->burst_num),
+		p_data_info->num_valid_rtt)); /* in a session */
+		avg_dist = FTM_INVALID;
+	}
+	else {
+		DHD_RTT((">\tavg_dist=%d.%04dm, burst_num=%d, valid_measure_cnt=%d num_ftm=%d\n",
+			avg_dist >> 8, /* 1/256m units */
+			((avg_dist & 0xff) * 625) >> 4,
+			ltoh16_ua(&p_data_info->burst_num),
+			p_data_info->num_valid_rtt,
+			p_data_info->num_ftm)); /* in a session */
+	}
+	/* show 'avg_rtt' sample */
+	p_sample_avg = &p_data_info->avg_rtt;
+	ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample_avg->rtt.tmu));
+	DHD_RTT((">\tavg_rtt sample: rssi=%d rtt=%d%s std_deviation =%d.%d ratespec=0x%08x\n",
+		(int16) ltoh16_ua(&p_sample_avg->rssi),
+		ltoh32_ua(&p_sample_avg->rtt.intvl),
+		ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample_avg->rtt.tmu)),
+		ltoh16_ua(&p_data_info->sd_rtt)/10, ltoh16_ua(&p_data_info->sd_rtt)%10,
+		ltoh32_ua(&p_sample_avg->ratespec)));
+
+	/* set peer address */
+	rtt_report->addr = p_data_info->peer;
+	/* burst num */
+	rtt_report->burst_num = ltoh16_ua(&p_data_info->burst_num);
+	/* success num */
+	rtt_report->success_num = p_data_info->num_valid_rtt;
+	/* actual number of FTM supported by peer */
+	rtt_report->num_per_burst_peer = p_data_info->num_ftm;
+	rtt_report->negotiated_burst_num = p_data_info->num_ftm;
+	/* status */
+	rtt_report->status = ftm_get_statusmap_info(proxd_status,
+			&ftm_status_map_info[0], ARRAYSIZE(ftm_status_map_info));
+
+	/* rssi (0.5db) */
+	rtt_report->rssi = ABS((wl_proxd_rssi_t)ltoh16_ua(&p_data_info->avg_rtt.rssi)) * 2;
+
+	/* rx rate */
+	ratespec = ltoh32_ua(&p_data_info->avg_rtt.ratespec);
+	rtt_report->rx_rate = dhd_rtt_convert_rate_to_host(ratespec);
+	/* tx rate */
+	if (flags & WL_PROXD_RESULT_FLAG_VHTACK) {
+		rtt_report->tx_rate = dhd_rtt_convert_rate_to_host(0x2010010);
+	} else {
+		rtt_report->tx_rate = dhd_rtt_convert_rate_to_host(0xc);
+	}
+	/* rtt_sd */
+	rtt.tmu = ltoh16_ua(&p_data_info->avg_rtt.rtt.tmu);
+	rtt.intvl = ltoh32_ua(&p_data_info->avg_rtt.rtt.intvl);
+	rtt_report->rtt = (wifi_timespan)FTM_INTVL2NSEC(&rtt) * 1000; /* nano -> pico seconds */
+	rtt_report->rtt_sd = ltoh16_ua(&p_data_info->sd_rtt); /* nano -> 0.1 nano */
+	DHD_RTT(("rtt_report->rtt : %llu\n", rtt_report->rtt));
+	DHD_RTT(("rtt_report->rssi : %d (0.5db)\n", rtt_report->rssi));
+
+	/* average distance */
+	if (avg_dist != FTM_INVALID) {
+		rtt_report->distance = (avg_dist >> 8) * 1000; /* meter -> mm */
+		rtt_report->distance += (avg_dist & 0xff) * 1000 / 256;
+	} else {
+		rtt_report->distance = FTM_INVALID;
+	}
+	/* time stamp */
+	/* get the time elapsed from boot time */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	osl_get_monotonic_boottime(&ts);
+	rtt_report->ts = (uint64)TIMESPEC64_TO_US(ts);
+#endif /* LINUX_VER >= 2.6.39 */
+
+	if (proxd_status == WL_PROXD_E_REMOTE_FAIL) {
+		/* retry time  after failure */
+		p_time.intvl = ltoh32_ua(&p_data_info->u.retry_after.intvl);
+		p_time.tmu = ltoh16_ua(&p_data_info->u.retry_after.tmu);
+		rtt_report->retry_after_duration = FTM_INTVL2SEC(&p_time); /* s -> s */
+		DHD_RTT((">\tretry_after: %d%s\n",
+			ltoh32_ua(&p_data_info->u.retry_after.intvl),
+			ftm_tmu_value_to_logstr(ltoh16_ua(&p_data_info->u.retry_after.tmu))));
+	} else {
+		/* burst duration */
+		p_time.intvl = ltoh32_ua(&p_data_info->u.retry_after.intvl);
+		p_time.tmu = ltoh16_ua(&p_data_info->u.retry_after.tmu);
+		rtt_report->burst_duration =  FTM_INTVL2MSEC(&p_time); /* s -> ms */
+		DHD_RTT((">\tburst_duration: %d%s\n",
+			ltoh32_ua(&p_data_info->u.burst_duration.intvl),
+			ftm_tmu_value_to_logstr(ltoh16_ua(&p_data_info->u.burst_duration.tmu))));
+		DHD_RTT(("rtt_report->burst_duration : %d\n", rtt_report->burst_duration));
+	}
+
+	/* display detail if available */
+	num_rtt = ltoh16_ua(&p_data_info->num_rtt);
+	if (num_rtt > 0) {
+		DHD_RTT((">\tnum rtt: %d samples\n", num_rtt));
+		p_sample = &p_data_info->rtt[0];
+		for (i = 0; i < num_rtt; i++) {
+			snr = 0;
+			bitflips = 0;
+			tof_phy_error = 0;
+			tof_phy_tgt_error = 0;
+			tof_target_snr = 0;
+			tof_target_bitflips = 0;
+			rssi = 0;
+			dist = 0;
+			num_ftm = p_data_info->num_ftm;
+			/* FTM frames 1,4,7,11 have valid snr, rssi and bitflips */
+			if ((i % num_ftm) == 1) {
+				rssi = (wl_proxd_rssi_t) ltoh16_ua(&p_sample->rssi);
+				snr = (wl_proxd_snr_t) ltoh16_ua(&p_sample->snr);
+				bitflips = (wl_proxd_bitflips_t) ltoh16_ua(&p_sample->bitflips);
+				tof_phy_error =
+					(wl_proxd_phy_error_t)
+					ltoh32_ua(&p_sample->tof_phy_error);
+				tof_phy_tgt_error =
+					(wl_proxd_phy_error_t)
+					ltoh32_ua(&p_sample->tof_tgt_phy_error);
+				tof_target_snr =
+					(wl_proxd_snr_t)
+					ltoh16_ua(&p_sample->tof_tgt_snr);
+				tof_target_bitflips =
+					(wl_proxd_bitflips_t)
+					ltoh16_ua(&p_sample->tof_tgt_bitflips);
+				dist = ltoh32_ua(&p_sample->distance);
+			} else {
+				rssi = -1;
+				snr = 0;
+				bitflips = 0;
+				dist = 0;
+				tof_target_bitflips = 0;
+				tof_target_snr = 0;
+				tof_phy_tgt_error = 0;
+			}
+			DHD_RTT((">\t sample[%d]: id=%d rssi=%d snr=0x%x bitflips=%d"
+				" tof_phy_error %x tof_phy_tgt_error %x target_snr=0x%x"
+				" target_bitflips=%d dist=%d rtt=%d%s status %s"
+				" Type %s coreid=%d\n",
+				i, p_sample->id, rssi, snr,
+				bitflips, tof_phy_error, tof_phy_tgt_error,
+				tof_target_snr,
+				tof_target_bitflips, dist,
+				ltoh32_ua(&p_sample->rtt.intvl),
+				ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample->rtt.tmu)),
+				ftm_status_value_to_logstr(ltoh32_ua(&p_sample->status)),
+				ftm_frame_types[i % num_ftm], p_sample->coreid));
+			p_sample++;
+		}
+	}
+	return err;
+}
+
+static int
+dhd_rtt_convert_results_to_host_v2(rtt_result_t *rtt_result, const uint8 *p_data,
+	uint16 tlvid, uint16 len)
+{
+	int i;
+	int err = BCME_OK;
+	char eabuf[ETHER_ADDR_STR_LEN];
+	wl_proxd_result_flags_t flags;
+	wl_proxd_session_state_t session_state;
+	wl_proxd_status_t proxd_status;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	struct osl_timespec ts;
+#endif /* LINUX_VER >= 2.6.39 */
+	uint32 ratespec;
+	int32 avg_dist;
+	const wl_proxd_rtt_result_v2_t *p_data_info = NULL;
+	const wl_proxd_rtt_sample_v2_t *p_sample_avg = NULL;
+	const wl_proxd_rtt_sample_v2_t *p_sample = NULL;
+	uint16 num_rtt = 0;
+	wl_proxd_intvl_t rtt;
+	wl_proxd_intvl_t p_time;
+	uint16 snr = 0, bitflips = 0;
+	wl_proxd_phy_error_t tof_phy_error = 0;
+	wl_proxd_phy_error_t tof_phy_tgt_error = 0;
+	wl_proxd_snr_t tof_target_snr = 0;
+	wl_proxd_bitflips_t tof_target_bitflips = 0;
+	int16 rssi = 0;
+	int32 dist = 0;
+	uint32 chanspec = 0;
+	uint8 num_ftm = 0;
+	char *ftm_frame_types[] =  FTM_FRAME_TYPES;
+	rtt_report_t *rtt_report = &(rtt_result->report);
+
+	BCM_REFERENCE(ftm_frame_types);
+	BCM_REFERENCE(dist);
+	BCM_REFERENCE(rssi);
+	BCM_REFERENCE(tof_target_bitflips);
+	BCM_REFERENCE(tof_target_snr);
+	BCM_REFERENCE(tof_phy_tgt_error);
+	BCM_REFERENCE(tof_phy_error);
+	BCM_REFERENCE(bitflips);
+	BCM_REFERENCE(snr);
+	BCM_REFERENCE(chanspec);
+	BCM_REFERENCE(session_state);
+	BCM_REFERENCE(ftm_session_state_value_to_logstr);
+
+	NULL_CHECK(rtt_report, "rtt_report is NULL", err);
+	NULL_CHECK(p_data, "p_data is NULL", err);
+	DHD_RTT(("%s enter\n", __FUNCTION__));
+	p_data_info = (const wl_proxd_rtt_result_v2_t *) p_data;
+	/* unpack and format 'flags' for display */
+	flags = ltoh16_ua(&p_data_info->flags);
+	/* session state and status */
+	session_state = ltoh16_ua(&p_data_info->state);
+	proxd_status = ltoh32_ua(&p_data_info->status);
+	bcm_ether_ntoa((&(p_data_info->peer)), eabuf);
+
+	if ((proxd_status != BCME_OK) || (p_data_info->num_meas == 0)) {
+		DHD_RTT_ERR((">\tTarget(%s) session state=%d(%s), status=%d(%s) "
+			"num_meas_ota %d num_valid_rtt %d result_flags %x\n",
+			eabuf, session_state,
+			ftm_session_state_value_to_logstr(session_state),
+			proxd_status, ftm_status_value_to_logstr(proxd_status),
+			p_data_info->num_meas, p_data_info->num_valid_rtt,
+			p_data_info->flags));
+	} else {
+		DHD_RTT((">\tTarget(%s) session state=%d(%s), status=%d(%s)\n",
+		eabuf, session_state,
+		ftm_session_state_value_to_logstr(session_state),
+		proxd_status, ftm_status_value_to_logstr(proxd_status)));
+	}
+	/* show avg_dist (1/256m units), burst_num */
+	avg_dist = ltoh32_ua(&p_data_info->avg_dist);
+	if (avg_dist == 0xffffffff) {	/* report 'failure' case */
+		DHD_RTT((">\tavg_dist=-1m, burst_num=%d, valid_measure_cnt=%d\n",
+		ltoh16_ua(&p_data_info->burst_num),
+		p_data_info->num_valid_rtt)); /* in a session */
+		avg_dist = FTM_INVALID;
+	} else {
+		DHD_RTT((">\tavg_dist=%d.%04dm, burst_num=%d, valid_measure_cnt=%d num_ftm=%d "
+			"num_meas_ota=%d, result_flags=%x\n", avg_dist >> 8, /* 1/256m units */
+			((avg_dist & 0xff) * 625) >> 4,
+			ltoh16_ua(&p_data_info->burst_num),
+			p_data_info->num_valid_rtt,
+			p_data_info->num_ftm, p_data_info->num_meas,
+			p_data_info->flags)); /* in a session */
+	}
+	rtt_result->rtt_detail.num_ota_meas = p_data_info->num_meas;
+	rtt_result->rtt_detail.result_flags = p_data_info->flags;
+	/* show 'avg_rtt' sample */
+	/* in v2, avg_rtt is the first element of the variable rtt[] */
+	p_sample_avg = &p_data_info->rtt[0];
+	ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample_avg->rtt.tmu));
+	DHD_RTT((">\tavg_rtt sample: rssi=%d rtt=%d%s std_deviation =%d.%d"
+		"ratespec=0x%08x chanspec=0x%08x\n",
+		(int16) ltoh16_ua(&p_sample_avg->rssi),
+		ltoh32_ua(&p_sample_avg->rtt.intvl),
+		ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample_avg->rtt.tmu)),
+		ltoh16_ua(&p_data_info->sd_rtt)/10, ltoh16_ua(&p_data_info->sd_rtt)%10,
+		ltoh32_ua(&p_sample_avg->ratespec),
+		ltoh32_ua(&p_sample_avg->chanspec)));
+
+	/* set peer address */
+	rtt_report->addr = p_data_info->peer;
+
+	/* burst num */
+	rtt_report->burst_num = ltoh16_ua(&p_data_info->burst_num);
+
+	/* success num */
+	rtt_report->success_num = p_data_info->num_valid_rtt;
+
+	/* num-ftm configured */
+	rtt_report->ftm_num = p_data_info->num_ftm;
+
+	/* actual number of FTM supported by peer */
+	rtt_report->num_per_burst_peer = p_data_info->num_ftm;
+	rtt_report->negotiated_burst_num = p_data_info->num_ftm;
+
+	/* status */
+	rtt_report->status = ftm_get_statusmap_info(proxd_status,
+			&ftm_status_map_info[0], ARRAYSIZE(ftm_status_map_info));
+
+	/* Framework expects status as SUCCESS else all results will be
+	* set to zero even if we have partial valid result.
+	* So setting status as SUCCESS if we have a valid_rtt
+	* On burst timeout we stop burst with "timeout" reason and
+	* on msch end we set status as "cancel"
+	*/
+	if ((proxd_status == WL_PROXD_E_TIMEOUT ||
+		proxd_status == WL_PROXD_E_CANCELED) &&
+		rtt_report->success_num) {
+		rtt_report->status = RTT_STATUS_SUCCESS;
+	}
+
+	/* rssi (0.5db) */
+	rtt_report->rssi = ABS((wl_proxd_rssi_t)ltoh16_ua(&p_sample_avg->rssi)) * 2;
+
+	/* rx rate */
+	ratespec = ltoh32_ua(&p_sample_avg->ratespec);
+	rtt_report->rx_rate = dhd_rtt_convert_rate_to_host(ratespec);
+
+	/* tx rate */
+	if (flags & WL_PROXD_RESULT_FLAG_VHTACK) {
+		rtt_report->tx_rate = dhd_rtt_convert_rate_to_host(0x2010010);
+	} else {
+		rtt_report->tx_rate = dhd_rtt_convert_rate_to_host(0xc);
+	}
+
+	/* rtt_sd */
+	rtt.tmu = ltoh16_ua(&p_sample_avg->rtt.tmu);
+	rtt.intvl = ltoh32_ua(&p_sample_avg->rtt.intvl);
+	if (rtt.tmu == WL_PROXD_TMU_PICO_SEC) {
+		rtt_report->rtt = (wifi_timespan)rtt.intvl;
+	} else {
+		 /* nano -> pico seconds */
+		rtt_report->rtt = (wifi_timespan)(FTM_INTVL2NSEC(&rtt) * 1000);
+	}
+
+	rtt_report->rtt_sd = ltoh16_ua(&p_data_info->sd_rtt); /* nano -> 0.1 nano */
+	DHD_RTT(("rtt_report->rtt : %lld\n", rtt_report->rtt));
+	DHD_RTT(("rtt_report->rssi : %d (0.5db)\n", rtt_report->rssi));
+
+	/* average distance */
+	if (avg_dist != FTM_INVALID) {
+		rtt_report->distance = (avg_dist >> 8) * 1000; /* meter -> mm */
+		rtt_report->distance += (avg_dist & 0xff) * 1000 / 256;
+		/* rtt_sd is in 0.1 ns.
+		* host needs distance_sd in milli mtrs
+		* (0.1 * rtt_sd/2 * 10^-9) * C * 1000
+		*/
+		rtt_report->distance_sd = rtt_report->rtt_sd * 15; /* mm */
+	} else {
+		rtt_report->distance = FTM_INVALID;
+	}
+
+	if (p_data_info->flags & WL_PROXD_RESULT_SIGNED) {
+		rtt_report->rtt *= -1;
+		rtt_report->distance *= -1;
+	}
+
+	/* time stamp */
+	/* get the time elapsed from boot time */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	osl_get_monotonic_boottime(&ts);
+	rtt_report->ts = (uint64)TIMESPEC64_TO_US(ts);
+#endif /* LINUX_VER >= 2.6.39 */
+
+	if (proxd_status == WL_PROXD_E_REMOTE_FAIL) {
+		/* retry time  after failure */
+		p_time.intvl = ltoh32_ua(&p_data_info->u.retry_after.intvl);
+		p_time.tmu = ltoh16_ua(&p_data_info->u.retry_after.tmu);
+		rtt_report->retry_after_duration = FTM_INTVL2SEC(&p_time); /* s -> s */
+		DHD_RTT((">\tretry_after: %d%s\n",
+			ltoh32_ua(&p_data_info->u.retry_after.intvl),
+			ftm_tmu_value_to_logstr(ltoh16_ua(&p_data_info->u.retry_after.tmu))));
+	} else {
+		/* burst duration */
+		p_time.intvl = ltoh32_ua(&p_data_info->u.retry_after.intvl);
+		p_time.tmu = ltoh16_ua(&p_data_info->u.retry_after.tmu);
+		rtt_report->burst_duration =  FTM_INTVL2MSEC(&p_time); /* s -> ms */
+		DHD_RTT((">\tburst_duration: %d%s\n",
+			ltoh32_ua(&p_data_info->u.burst_duration.intvl),
+			ftm_tmu_value_to_logstr(ltoh16_ua(&p_data_info->u.burst_duration.tmu))));
+		DHD_RTT(("rtt_report->burst_duration : %d\n", rtt_report->burst_duration));
+	}
+	/* display detail if available */
+	num_rtt = ltoh16_ua(&p_data_info->num_rtt);
+	if (num_rtt > 0) {
+		DHD_RTT((">\tnum rtt: %d samples\n", num_rtt));
+		p_sample = &p_data_info->rtt[1];
+		for (i = 0; i < num_rtt; i++) {
+			snr = 0;
+			bitflips = 0;
+			tof_phy_error = 0;
+			tof_phy_tgt_error = 0;
+			tof_target_snr = 0;
+			tof_target_bitflips = 0;
+			rssi = 0;
+			dist = 0;
+			num_ftm = p_data_info->num_ftm;
+			/* FTM frames 1,4,7,11 have valid snr, rssi and bitflips */
+			if ((i % num_ftm) == 1) {
+				rssi = (wl_proxd_rssi_t) ltoh16_ua(&p_sample->rssi);
+				snr = (wl_proxd_snr_t) ltoh16_ua(&p_sample->snr);
+				bitflips = (wl_proxd_bitflips_t) ltoh16_ua(&p_sample->bitflips);
+				tof_phy_error =
+					(wl_proxd_phy_error_t)
+					ltoh32_ua(&p_sample->tof_phy_error);
+				tof_phy_tgt_error =
+					(wl_proxd_phy_error_t)
+					ltoh32_ua(&p_sample->tof_tgt_phy_error);
+				tof_target_snr =
+					(wl_proxd_snr_t)
+					ltoh16_ua(&p_sample->tof_tgt_snr);
+				tof_target_bitflips =
+					(wl_proxd_bitflips_t)
+					ltoh16_ua(&p_sample->tof_tgt_bitflips);
+				dist = ltoh32_ua(&p_sample->distance);
+				chanspec = ltoh32_ua(&p_sample->chanspec);
+			} else {
+				rssi = -1;
+				snr = 0;
+				bitflips = 0;
+				dist = 0;
+				tof_target_bitflips = 0;
+				tof_target_snr = 0;
+				tof_phy_tgt_error = 0;
+			}
+			DHD_RTT((">\t sample[%d]: id=%d rssi=%d snr=0x%x bitflips=%d"
+				" tof_phy_error %x tof_phy_tgt_error %x target_snr=0x%x"
+				" target_bitflips=%d dist=%d rtt=%d%s status %s Type %s"
+				" coreid=%d chanspec=0x%08x\n",
+				i, p_sample->id, rssi, snr,
+				bitflips, tof_phy_error, tof_phy_tgt_error,
+				tof_target_snr,
+				tof_target_bitflips, dist,
+				ltoh32_ua(&p_sample->rtt.intvl),
+				ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample->rtt.tmu)),
+				ftm_status_value_to_logstr(ltoh32_ua(&p_sample->status)),
+				ftm_frame_types[i % num_ftm], p_sample->coreid,
+				chanspec));
+			p_sample++;
+		}
+	}
+	return err;
+}
+
+static int
+dhd_rtt_convert_results_to_host_v3(rtt_result_t *rtt_result, const uint8 *p_data,
+	uint16 tlvid, uint16 len)
+{
+	int i;
+	int err = BCME_OK;
+	char eabuf[ETHER_ADDR_STR_LEN];
+	wl_proxd_result_flags_t flags;
+	wl_proxd_session_state_t session_state;
+	wl_proxd_status_t proxd_status;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	struct timespec64 ts;
+#endif /* LINUX_VER >= 2.6.39 */
+	uint32 ratespec;
+	int32 avg_dist;
+	const wl_proxd_rtt_result_v3_t *p_data_info = NULL;
+	const wl_proxd_rtt_sample_v3_t *p_sample_avg = NULL;
+	const wl_proxd_rtt_sample_v3_t *p_sample = NULL;
+	uint16 num_rtt = 0;
+	wl_proxd_intvl_t rtt;
+	wl_proxd_intvl_t p_time;
+	uint16 snr = 0, bitflips = 0;
+	wl_proxd_phy_error_t tof_phy_error = 0;
+	wl_proxd_phy_error_t tof_phy_tgt_error = 0;
+	wl_proxd_snr_t tof_target_snr = 0;
+	wl_proxd_bitflips_t tof_target_bitflips = 0;
+	int16 rssi = 0;
+	int32 dist = 0;
+	uint32 chanspec = 0, gd_variance = 0;
+	uint8 num_ftm = 0;
+	char *ftm_frame_types[] =  FTM_FRAME_TYPES;
+	rtt_report_t *rtt_report = &(rtt_result->report);
+
+	BCM_REFERENCE(ftm_frame_types);
+	BCM_REFERENCE(dist);
+	BCM_REFERENCE(rssi);
+	BCM_REFERENCE(tof_target_bitflips);
+	BCM_REFERENCE(tof_target_snr);
+	BCM_REFERENCE(tof_phy_tgt_error);
+	BCM_REFERENCE(tof_phy_error);
+	BCM_REFERENCE(bitflips);
+	BCM_REFERENCE(snr);
+	BCM_REFERENCE(chanspec);
+	BCM_REFERENCE(session_state);
+	BCM_REFERENCE(ftm_session_state_value_to_logstr);
+	BCM_REFERENCE(gd_variance);
+
+	NULL_CHECK(rtt_report, "rtt_report is NULL", err);
+	NULL_CHECK(p_data, "p_data is NULL", err);
+	DHD_RTT(("%s enter\n", __FUNCTION__));
+	p_data_info = (const wl_proxd_rtt_result_v3_t *) p_data;
+	/* unpack and format 'flags' for display */
+	flags = ltoh16_ua(&p_data_info->flags);
+	/* session state and status */
+	session_state = ltoh16_ua(&p_data_info->state);
+	proxd_status = ltoh32_ua(&p_data_info->status);
+	bcm_ether_ntoa((&(p_data_info->peer)), eabuf);
+
+	if ((proxd_status != BCME_OK) || (p_data_info->num_meas == 0)) {
+		DHD_RTT_ERR((">\tTarget(%s) session state=%d(%s), status=%d(%s) "
+			"num_meas_ota %d num_valid_rtt %d result_flags %x\n",
+			eabuf, session_state,
+			ftm_session_state_value_to_logstr(session_state),
+			proxd_status, ftm_status_value_to_logstr(proxd_status),
+			p_data_info->num_meas, p_data_info->num_valid_rtt,
+			p_data_info->flags));
+	} else {
+		DHD_RTT((">\tTarget(%s) session state=%d(%s), status=%d(%s)\n",
+		eabuf, session_state,
+		ftm_session_state_value_to_logstr(session_state),
+		proxd_status, ftm_status_value_to_logstr(proxd_status)));
+	}
+	/* show avg_dist (1/256m units), burst_num */
+	avg_dist = ltoh32_ua(&p_data_info->avg_dist);
+	if (avg_dist == 0xffffffff) {	/* report 'failure' case */
+		DHD_RTT((">\tavg_dist=-1m, burst_num=%d, valid_measure_cnt=%d\n",
+		ltoh16_ua(&p_data_info->burst_num),
+		p_data_info->num_valid_rtt)); /* in a session */
+		avg_dist = FTM_INVALID;
+	} else {
+		DHD_RTT((">\tavg_dist=%d.%04dm, burst_num=%d, valid_measure_cnt=%d num_ftm=%d "
+			"num_meas_ota=%d, result_flags=%x\n", avg_dist >> 8, /* 1/256m units */
+			((avg_dist & 0xff) * 625) >> 4,
+			ltoh16_ua(&p_data_info->burst_num),
+			p_data_info->num_valid_rtt,
+			p_data_info->num_ftm, p_data_info->num_meas,
+			p_data_info->flags)); /* in a session */
+	}
+	rtt_result->rtt_detail.num_ota_meas = p_data_info->num_meas;
+	rtt_result->rtt_detail.result_flags = p_data_info->flags;
+	/* show 'avg_rtt' sample */
+	/* in v2, avg_rtt is the first element of the variable rtt[] */
+	p_sample_avg = &p_data_info->rtt[0];
+	ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample_avg->rtt.tmu));
+	DHD_RTT((">\tavg_rtt sample: rssi=%d rtt=%d%s std_deviation =%d.%d"
+		"ratespec=0x%08x chanspec=0x%08x gd_variance %u\n",
+		(int16) ltoh16_ua(&p_sample_avg->rssi),
+		ltoh32_ua(&p_sample_avg->rtt.intvl),
+		ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample_avg->rtt.tmu)),
+		ltoh16_ua(&p_data_info->sd_rtt)/10, ltoh16_ua(&p_data_info->sd_rtt)%10,
+		ltoh32_ua(&p_sample_avg->ratespec),
+		ltoh32_ua(&p_sample_avg->chanspec),
+		ltoh32_ua(&p_sample_avg->gd_variance)));
+
+	/* set peer address */
+	rtt_report->addr = p_data_info->peer;
+
+	/* burst num */
+	rtt_report->burst_num = ltoh16_ua(&p_data_info->burst_num);
+
+	/* success num */
+	rtt_report->success_num = p_data_info->num_valid_rtt;
+
+	/* num-ftm configured */
+	rtt_report->ftm_num = p_data_info->num_ftm;
+
+	/* actual number of FTM supported by peer */
+	rtt_report->num_per_burst_peer = p_data_info->num_ftm;
+	rtt_report->negotiated_burst_num = p_data_info->num_ftm;
+
+	/* status */
+	rtt_report->status = ftm_get_statusmap_info(proxd_status,
+			&ftm_status_map_info[0], ARRAYSIZE(ftm_status_map_info));
+
+	/* Framework expects status as SUCCESS else all results will be
+	* set to zero even if we have partial valid result.
+	* So setting status as SUCCESS if we have a valid_rtt
+	* On burst timeout we stop burst with "timeout" reason and
+	* on msch end we set status as "cancel"
+	*/
+	if ((proxd_status == WL_PROXD_E_TIMEOUT ||
+		proxd_status == WL_PROXD_E_CANCELED) &&
+		rtt_report->success_num) {
+		rtt_report->status = RTT_STATUS_SUCCESS;
+	}
+
+	/* rssi (0.5db) */
+	rtt_report->rssi = ABS((wl_proxd_rssi_t)ltoh16_ua(&p_sample_avg->rssi)) * 2;
+
+	/* rx rate */
+	ratespec = ltoh32_ua(&p_sample_avg->ratespec);
+	rtt_report->rx_rate = dhd_rtt_convert_rate_to_host(ratespec);
+
+	/* tx rate */
+	if (flags & WL_PROXD_RESULT_FLAG_VHTACK) {
+		rtt_report->tx_rate = dhd_rtt_convert_rate_to_host(0x2010010);
+	} else {
+		rtt_report->tx_rate = dhd_rtt_convert_rate_to_host(0xc);
+	}
+
+	/* rtt_sd */
+	rtt.tmu = ltoh16_ua(&p_sample_avg->rtt.tmu);
+	rtt.intvl = ltoh32_ua(&p_sample_avg->rtt.intvl);
+	rtt_report->rtt = (wifi_timespan)FTM_INTVL2NSEC(&rtt) * 1000; /* nano -> pico seconds */
+	rtt_report->rtt_sd = ltoh16_ua(&p_data_info->sd_rtt); /* nano -> 0.1 nano */
+	DHD_RTT(("rtt_report->rtt : %llu\n", rtt_report->rtt));
+	DHD_RTT(("rtt_report->rssi : %d (0.5db)\n", rtt_report->rssi));
+
+	/* average distance */
+	if (avg_dist != FTM_INVALID) {
+		rtt_report->distance = (avg_dist >> 8) * 1000; /* meter -> mm */
+		rtt_report->distance += (avg_dist & 0xff) * 1000 / 256;
+		/* rtt_sd is in 0.1 ns.
+		* host needs distance_sd in milli mtrs
+		* (0.1 * rtt_sd/2 * 10^-9) * C * 1000
+		*/
+		rtt_report->distance_sd = rtt_report->rtt_sd * 15; /* mm */
+	} else {
+		rtt_report->distance = FTM_INVALID;
+	}
+	/* time stamp */
+	/* get the time elapsed from boot time */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	ts = ktime_to_timespec64(ktime_get_boottime());
+	rtt_report->ts = (uint64)TIMESPEC64_TO_US(ts);
+#endif /* LINUX_VER >= 2.6.39 */
+
+	if (proxd_status == WL_PROXD_E_REMOTE_FAIL) {
+		/* retry time  after failure */
+		p_time.intvl = ltoh32_ua(&p_data_info->u.retry_after.intvl);
+		p_time.tmu = ltoh16_ua(&p_data_info->u.retry_after.tmu);
+		rtt_report->retry_after_duration = FTM_INTVL2SEC(&p_time); /* s -> s */
+		DHD_RTT((">\tretry_after: %d%s\n",
+			ltoh32_ua(&p_data_info->u.retry_after.intvl),
+			ftm_tmu_value_to_logstr(ltoh16_ua(&p_data_info->u.retry_after.tmu))));
+	} else {
+		/* burst duration */
+		p_time.intvl = ltoh32_ua(&p_data_info->u.retry_after.intvl);
+		p_time.tmu = ltoh16_ua(&p_data_info->u.retry_after.tmu);
+		rtt_report->burst_duration =  FTM_INTVL2MSEC(&p_time); /* s -> ms */
+		DHD_RTT((">\tburst_duration: %d%s\n",
+			ltoh32_ua(&p_data_info->u.burst_duration.intvl),
+			ftm_tmu_value_to_logstr(ltoh16_ua(&p_data_info->u.burst_duration.tmu))));
+		DHD_RTT(("rtt_report->burst_duration : %d\n", rtt_report->burst_duration));
+	}
+	/* display detail if available */
+	num_rtt = ltoh16_ua(&p_data_info->num_rtt);
+	if (num_rtt > 0) {
+		DHD_RTT((">\tnum rtt: %d samples\n", num_rtt));
+		p_sample = &p_data_info->rtt[1];
+		for (i = 0; i < num_rtt; i++) {
+			snr = 0;
+			bitflips = 0;
+			tof_phy_error = 0;
+			tof_phy_tgt_error = 0;
+			tof_target_snr = 0;
+			tof_target_bitflips = 0;
+			rssi = 0;
+			dist = 0;
+			num_ftm = p_data_info->num_ftm;
+			/* FTM frames 1,4,7,11 have valid snr, rssi and bitflips */
+			if ((i % num_ftm) == 1) {
+				rssi = (wl_proxd_rssi_t) ltoh16_ua(&p_sample->rssi);
+				snr = (wl_proxd_snr_t) ltoh16_ua(&p_sample->snr);
+				bitflips = (wl_proxd_bitflips_t) ltoh16_ua(&p_sample->bitflips);
+				tof_phy_error =
+					(wl_proxd_phy_error_t)
+					ltoh32_ua(&p_sample->tof_phy_error);
+				tof_phy_tgt_error =
+					(wl_proxd_phy_error_t)
+					ltoh32_ua(&p_sample->tof_tgt_phy_error);
+				tof_target_snr =
+					(wl_proxd_snr_t)
+					ltoh16_ua(&p_sample->tof_tgt_snr);
+				tof_target_bitflips =
+					(wl_proxd_bitflips_t)
+					ltoh16_ua(&p_sample->tof_tgt_bitflips);
+				dist = ltoh32_ua(&p_sample->distance);
+				chanspec = ltoh32_ua(&p_sample->chanspec);
+				gd_variance = ltoh32_ua(&p_sample->gd_variance);
+			} else {
+				rssi = -1;
+				snr = 0;
+				bitflips = 0;
+				dist = 0;
+				tof_target_bitflips = 0;
+				tof_target_snr = 0;
+				tof_phy_tgt_error = 0;
+			}
+			DHD_RTT((">\t sample[%d]: id=%d rssi=%d snr=0x%x bitflips=%d"
+				" tof_phy_error %x tof_phy_tgt_error %x target_snr=0x%x"
+				" target_bitflips=%d dist=%d rtt=%d%s status %s Type %s"
+				" coreid=%d chanspec=0x%08x gd_variance=%u\n",
+				i, p_sample->id, rssi, snr,
+				bitflips, tof_phy_error, tof_phy_tgt_error,
+				tof_target_snr,
+				tof_target_bitflips, dist,
+				ltoh32_ua(&p_sample->rtt.intvl),
+				ftm_tmu_value_to_logstr(ltoh16_ua(&p_sample->rtt.tmu)),
+				ftm_status_value_to_logstr(ltoh32_ua(&p_sample->status)),
+				ftm_frame_types[i % num_ftm], p_sample->coreid,
+				chanspec, gd_variance));
+			p_sample++;
+		}
+	}
+	return err;
+}
+
+#ifdef WL_CFG80211
+/* Common API for handling Session End.
+* This API will flush out the results for a peer MAC.
+*
+* @For legacy FTM session, this API will be called
+* when legacy FTM_SESSION_END event is received.
+* @For legacy Nan-RTT , this API will be called when
+* we are cancelling the nan-ranging session or on
+* nan-ranging-end event.
+*/
+
+static bool
+dhd_rtt_all_directed_targets_done(dhd_pub_t *dhd)
+{
+	int8 idx;
+	bool done = TRUE;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+
+	for (idx = rtt_status->start_idx; idx < rtt_status->rtt_config.rtt_target_cnt; idx++) {
+		if (!rtt_status->rtt_config.target_info[idx].disable) {
+			if (!dhd_rtt_get_report_header(rtt_status,
+					NULL, &rtt_status->rtt_config.target_info[idx].addr)) {
+				done = FALSE;
+				break;
+			}
+		}
+	}
+	return done;
+}
+
+static void
+dhd_rtt_handle_rtt_session_end(dhd_pub_t *dhd)
+{
+
+	struct rtt_noti_callback *iter;
+	rtt_results_header_t *entry, *next;
+	rtt_result_t *next2;
+	rtt_result_t *rtt_result;
+	bool all_targets_done = FALSE;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+#ifdef WL_NAN
+	struct net_device *ndev = dhd_linux_get_primary_netdev(dhd);
+	struct bcm_cfg80211 *cfg = wiphy_priv(ndev->ieee80211_ptr->wiphy);
+#endif /* WL_NAN */
+	int i;
+	rtt_target_info_t *rtt_target = NULL;
+	rtt_config_params_t *rtt_config = &rtt_status->rtt_config;
+
+	/* Cancel pending proxd timeout work if any */
+	if (delayed_work_pending(&rtt_status->proxd_timeout)) {
+		cancel_delayed_work(&rtt_status->proxd_timeout);
+	}
+
+	/* check if all targets results received */
+	all_targets_done = dhd_rtt_all_directed_targets_done(dhd);
+	if (all_targets_done) {
+		DHD_RTT_MEM(("RTT_STOPPED\n"));
+#ifdef DHD_RTT_USE_FTM_RANGE
+		dhd_rtt_stop_ranging(dhd);
+#endif
+		rtt_status->status = RTT_STOPPED;
+		/* notify the completed information to others */
+		GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+		list_for_each_entry(iter, &rtt_status->noti_fn_list, list) {
+			iter->noti_fn(iter->ctx, &rtt_status->rtt_results_cache);
+		}
+		/* remove the rtt results in cache */
+		if (!list_empty(&rtt_status->rtt_results_cache)) {
+			/* Iterate rtt_results_header list */
+			list_for_each_entry_safe(entry, next,
+				&rtt_status->rtt_results_cache, list) {
+				list_del(&entry->list);
+				/* Iterate rtt_result list */
+				list_for_each_entry_safe(rtt_result, next2,
+					&entry->result_list, list) {
+					list_del(&rtt_result->list);
+					MFREE(dhd->osh, rtt_result,
+						sizeof(rtt_result_t));
+				}
+				MFREE(dhd->osh, entry, sizeof(rtt_results_header_t));
+			}
+		}
+		GCC_DIAGNOSTIC_POP();
+		/* reinitialize the HEAD */
+		INIT_LIST_HEAD(&rtt_status->rtt_results_cache);
+		/* clear information for rtt_config */
+		if (rtt_config->target_list_mode == RNG_TARGET_LIST_MODE_LEGACY) {
+			for (i = 0; i < rtt_config->rtt_target_cnt; i++) {
+				rtt_target = &rtt_status->rtt_config.target_info[i];
+				dhd_rtt_delete_session(dhd, rtt_target->sid);
+			}
+			dhd_rtt_ftm_enable(dhd, FALSE);
+		}
+		rtt_config->rtt_target_cnt = 0;
+		(void)memset_s(rtt_config->target_info, TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT),
+			0, TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT));
+		rtt_status->cur_idx = 0;
+#ifdef WL_NAN
+		/* Reset for Geofence */
+		wl_cfgnan_reset_geofence_ranging(cfg, NULL,
+			RTT_SCHED_RNG_RPT_DIRECTED, FALSE);
+#endif /* WL_NAN */
+	} else {
+		/* Targets still pending */
+#ifdef WL_NAN
+		if (rtt_config->target_list_mode == RNG_TARGET_LIST_MODE_NAN) {
+			/* Pure NAN target list */
+			dhd_rtt_trigger_pending_targets_on_session_end(dhd);
+		} else
+#endif /* WL_NAN */
+		{
+			/* Pure legacy target list or mixed list */
+			dhd_rtt_set_next_target_idx(dhd, (rtt_status->cur_idx + 1));
+			if (rtt_status->cur_idx < rtt_config->rtt_target_cnt) {
+				rtt_target = &rtt_config->target_info[rtt_status->cur_idx];
+				/* restart to measure RTT from next device */
+				if (rtt_config->target_list_mode == RNG_TARGET_LIST_MODE_LEGACY) {
+#ifndef DHD_RTT_USE_FTM_RANGE
+					dhd_rtt_start_session(dhd, rtt_target->sid, TRUE);
+#endif
+				} else {
+					dhd_rtt_schedule_rtt_work_thread(dhd,
+						rtt_status->rtt_sched_reason);
+				}
+			}
+		}
+		DHD_RTT_MEM(("Triggered/schedlued next target idx %d mode %d\n",
+			rtt_status->cur_idx, rtt_config->target_list_mode));
+	}
+}
+#endif /* WL_CFG80211 */
+
+#ifdef WL_CFG80211
+static int
+dhd_rtt_create_failure_result(rtt_status_info_t *rtt_status,
+	struct ether_addr *addr)
+{
+	rtt_results_header_t *rtt_results_header = NULL;
+	rtt_target_info_t *rtt_target_info;
+	int ret = BCME_OK;
+	rtt_result_t *rtt_result;
+
+	/* allocate new header for rtt_results */
+	rtt_results_header = (rtt_results_header_t *)MALLOCZ(rtt_status->dhd->osh,
+		sizeof(rtt_results_header_t));
+	if (!rtt_results_header) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+	rtt_target_info = &rtt_status->rtt_config.target_info[rtt_status->cur_idx];
+	/* Initialize the head of list for rtt result */
+	INIT_LIST_HEAD(&rtt_results_header->result_list);
+	/* same src and dest len */
+	(void)memcpy_s(&rtt_results_header->peer_mac,
+		ETHER_ADDR_LEN, addr, ETHER_ADDR_LEN);
+	list_add_tail(&rtt_results_header->list, &rtt_status->rtt_results_cache);
+
+	/* allocate rtt_results for new results */
+	rtt_result = (rtt_result_t *)MALLOCZ(rtt_status->dhd->osh,
+		sizeof(rtt_result_t));
+	if (!rtt_result) {
+		ret = -ENOMEM;
+		/* Free rtt result header */
+		MFREE(rtt_status->dhd->osh, rtt_results_header, sizeof(rtt_results_header_t));
+		goto exit;
+	}
+	/* fill out the results from the configuration param */
+	rtt_result->report.ftm_num = rtt_target_info->num_frames_per_burst;
+	rtt_result->report.type = rtt_target_info->type;
+	DHD_RTT(("report->ftm_num : %d\n", rtt_result->report.ftm_num));
+	rtt_result->report_len = RTT_REPORT_SIZE;
+	rtt_result->report.status = RTT_STATUS_FAIL_NO_RSP;
+	/* same src and dest len */
+	(void)memcpy_s(&rtt_result->report.addr, ETHER_ADDR_LEN,
+		&rtt_target_info->addr, ETHER_ADDR_LEN);
+	rtt_result->report.distance = FTM_INVALID;
+	list_add_tail(&rtt_result->list, &rtt_results_header->result_list);
+	rtt_results_header->result_cnt++;
+	rtt_results_header->result_tot_len += rtt_result->report_len;
+exit:
+	return ret;
+}
+
+static bool
+dhd_rtt_get_report_header(rtt_status_info_t *rtt_status,
+	rtt_results_header_t **rtt_results_header, struct ether_addr *addr)
+{
+	rtt_results_header_t *entry;
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	/* find a rtt_report_header for this mac address */
+	list_for_each_entry(entry, &rtt_status->rtt_results_cache, list) {
+		GCC_DIAGNOSTIC_POP();
+		if (!memcmp(&entry->peer_mac, addr, ETHER_ADDR_LEN))  {
+			/* found a rtt_report_header for peer_mac in the list */
+			if (rtt_results_header) {
+				*rtt_results_header = entry;
+			}
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+#ifdef WL_NAN
+int
+dhd_rtt_handle_nan_rtt_session_end(dhd_pub_t *dhd, struct ether_addr *peer)
+{
+	bool is_new = TRUE;
+	rtt_status_info_t *rtt_status = GET_RTTSTATE(dhd);
+	mutex_lock(&rtt_status->rtt_mutex);
+	is_new = !dhd_rtt_get_report_header(rtt_status, NULL, peer);
+
+	if (is_new) { /* no FTM result..create failure result */
+		dhd_rtt_create_failure_result(rtt_status, peer);
+	}
+	DHD_RTT_MEM(("RTT Session End for NAN peer "MACDBG"\n", MAC2STRDBG(peer)));
+	dhd_rtt_handle_rtt_session_end(dhd);
+	mutex_unlock(&rtt_status->rtt_mutex);
+	return BCME_OK;
+}
+
+static bool
+dhd_rtt_is_valid_measurement(rtt_result_t *rtt_result)
+{
+	bool ret = FALSE;
+
+	if (rtt_result && (rtt_result->report.success_num != 0)) {
+		ret = TRUE;
+	}
+	return ret;
+}
+
+static void
+dhd_rtt_trigger_pending_targets_on_session_end(dhd_pub_t *dhd)
+{
+	if (!(dhd_rtt_nan_is_directed_setup_in_prog(dhd)) &&
+			(!dhd_rtt_nan_all_directed_sessions_triggered(dhd)) &&
+			(!dhd_rtt_nan_directed_sessions_allowed(dhd))) {
+		/* Trigger next target from here */
+		dhd_rtt_set_next_target_idx(dhd,
+				(dhd_rtt_get_cur_target_idx(dhd) + 1));
+		dhd_rtt_schedule_rtt_work_thread(dhd, RTT_SCHED_RNG_DIR_EXCESS_TARGET);
+	}
+}
+#endif /* WL_NAN */
+#endif /* WL_CFG80211 */
+
+static int
+dhd_rtt_parse_result_event(wl_proxd_event_t *proxd_ev_data,
+	int tlvs_len, rtt_result_t *rtt_result)
+{
+	int ret = BCME_OK;
+
+	/* unpack TLVs and invokes the cbfn to print the event content TLVs */
+	ret = bcm_unpack_xtlv_buf((void *) rtt_result,
+			(uint8 *)&proxd_ev_data->tlvs[0], tlvs_len,
+			BCM_XTLV_OPTION_ALIGN32, rtt_unpack_xtlv_cbfn);
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("%s : Failed to unpack xtlv for an event\n",
+			__FUNCTION__));
+		goto exit;
+	}
+	/* fill out the results from the configuration param */
+	rtt_result->report.type = RTT_TWO_WAY;
+	DHD_RTT(("report->ftm_num : %d\n", rtt_result->report.ftm_num));
+	rtt_result->report_len = RTT_REPORT_SIZE;
+	rtt_result->detail_len = sizeof(rtt_result->rtt_detail);
+
+exit:
+	return ret;
+
+}
+
+static int
+dhd_rtt_handle_directed_rtt_burst_end(dhd_pub_t *dhd, struct ether_addr *peer_addr,
+        wl_proxd_event_t *proxd_ev_data, int tlvs_len, rtt_result_t *rtt_result, bool is_nan)
+{
+	int ret = BCME_OK;
+
+#ifdef WL_CFG80211
+	int err_at = 0;
+	rtt_status_info_t *rtt_status;
+	bool is_new = TRUE;
+	rtt_results_header_t *rtt_results_header = NULL;
+#endif /* WL_CFG80211 */
+
+#ifdef WL_CFG80211
+	rtt_status = GET_RTTSTATE(dhd);
+	is_new = !dhd_rtt_get_report_header(rtt_status,
+		&rtt_results_header, peer_addr);
+
+	if (tlvs_len > 0) {
+		if (is_new) {
+			/* allocate new header for rtt_results */
+			rtt_results_header = (rtt_results_header_t *)MALLOCZ(rtt_status->dhd->osh,
+				sizeof(rtt_results_header_t));
+			if (!rtt_results_header) {
+				ret = BCME_NORESOURCE;
+				err_at = 1;
+				goto exit;
+			}
+			/* Initialize the head of list for rtt result */
+			INIT_LIST_HEAD(&rtt_results_header->result_list);
+			/* same src and header len */
+			(void)memcpy_s(&rtt_results_header->peer_mac, ETHER_ADDR_LEN,
+				peer_addr, ETHER_ADDR_LEN);
+			list_add_tail(&rtt_results_header->list, &rtt_status->rtt_results_cache);
+		}
+#endif /* WL_CFG80211 */
+
+		ret = dhd_rtt_parse_result_event(proxd_ev_data, tlvs_len, rtt_result);
+#ifdef WL_CFG80211
+		if (ret == BCME_OK) {
+			rtt_target_info_t *target =
+				&rtt_status->rtt_config.target_info[rtt_status->cur_idx];
+#ifdef WL_RTT_LCI
+			/* assign the cached location info */
+			if (target->LCI) {
+				rtt_result->report.LCI = (bcm_tlv_t*)target->LCI->data;
+				rtt_result->report_len += target->LCI->len;
+			}
+			if (target->LCR) {
+				rtt_result->report.LCR = (bcm_tlv_t*)target->LCR->data;
+				rtt_result->report_len += target->LCR->len;
+			}
+#endif /* WL_RTT_LCI */
+			rtt_result->report.type = target->type;
+
+			list_add_tail(&rtt_result->list, &rtt_results_header->result_list);
+			rtt_results_header->result_cnt++;
+			rtt_results_header->result_tot_len += rtt_result->report_len +
+				rtt_result->detail_len;
+		} else {
+			err_at = 2;
+			goto exit;
+		}
+	} else {
+		ret = BCME_ERROR;
+		err_at = 4;
+		goto exit;
+	}
+
+exit:
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("dhd_rtt_handle_directed_rtt_burst_end: failed, "
+			" ret = %d, err_at = %d\n", ret, err_at));
+		if (rtt_results_header) {
+			list_del(&rtt_results_header->list);
+			MFREE(dhd->osh, rtt_results_header,
+				sizeof(rtt_results_header_t));
+		}
+	}
+#endif /* WL_CFG80211 */
+	return ret;
+}
+
+#ifdef WL_NAN
+static	void
+dhd_rtt_nan_range_report(struct bcm_cfg80211 *cfg,
+		rtt_result_t *rtt_result, bool is_geofence)
+{
+	wl_nan_ev_rng_rpt_ind_t range_res;
+	int rtt_status;
+
+	UNUSED_PARAMETER(range_res);
+
+	if (!rtt_result)
+		return;
+
+	rtt_status = rtt_result->report.status;
+	bzero(&range_res, sizeof(range_res));
+	/* RTT can be negative(for GG req).. for geofence make it zero */
+	range_res.dist_mm = (rtt_result->report.distance < 0) ?
+		0 : rtt_result->report.distance;
+	/* same src and header len, ignoring ret val here */
+	(void)memcpy_s(&range_res.peer_m_addr, ETHER_ADDR_LEN,
+		&rtt_result->report.addr, ETHER_ADDR_LEN);
+	wl_cfgnan_process_range_report(cfg, &range_res, rtt_status);
+
+	return;
+}
+
+static int
+dhd_rtt_handle_nan_burst_end(dhd_pub_t *dhd, struct ether_addr *peer_addr,
+	wl_proxd_event_t *proxd_ev_data, int tlvs_len)
+{
+	struct net_device *ndev = NULL;
+	struct bcm_cfg80211 *cfg = NULL;
+	nan_ranging_inst_t *rng_inst = NULL;
+	rtt_status_info_t *rtt_status = NULL;
+	rtt_result_t *rtt_result = NULL;
+	bool geofence_rtt = FALSE;
+	int ret = BCME_OK;
+	rtt_result_t nan_rtt_res;
+	uint8 ftm_retry_cnt = 0;
+	int burst_status = -1;
+
+	ndev = dhd_linux_get_primary_netdev(dhd);
+	cfg =  wiphy_priv(ndev->ieee80211_ptr->wiphy);
+
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", ret);
+	NAN_MUTEX_LOCK();
+	mutex_lock(&rtt_status->rtt_mutex);
+
+	if ((wl_cfgnan_is_enabled(cfg) == FALSE) ||
+		ETHER_ISNULLADDR(peer_addr)) {
+		DHD_RTT_ERR(("Received Burst End with NULL ether addr, "
+			"or nan disable, nan_enable = %d\n", wl_cfgnan_is_enabled(cfg)));
+		ret = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	rng_inst = wl_cfgnan_check_for_ranging(cfg, peer_addr);
+	if (rng_inst) {
+		geofence_rtt = (rng_inst->range_type
+			== RTT_TYPE_NAN_GEOFENCE);
+	} else {
+		DHD_RTT_ERR(("Received Burst End without Ranging Instance\n"));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	if (rng_inst->range_role == NAN_RANGING_ROLE_RESPONDER) {
+		ret = BCME_OK;
+		goto exit;
+	}
+
+	bzero(&nan_rtt_res, sizeof(nan_rtt_res));
+	ret = dhd_rtt_parse_result_event(proxd_ev_data, tlvs_len, &nan_rtt_res);
+	if (ret != BCME_OK) {
+		DHD_RTT_ERR(("Failed to parse RTT result %d\n", ret));
+		goto exit;
+	}
+
+	burst_status = nan_rtt_res.report.status;
+	if (nan_rtt_res.rtt_detail.num_ota_meas <= 1) {
+		/* Wait for some time(CRBs) for ftm protocol to go through */
+		if (rng_inst->ftm_ssn_retry_count < NAN_RTT_FTM_SSN_RETRIES) {
+			rng_inst->ftm_ssn_retry_count++;
+			ftm_retry_cnt = rng_inst->ftm_ssn_retry_count;
+			ret = BCME_ERROR;
+			goto exit;
+		}
+		/* retries over...report the result as is to host */
+	}
+
+	BCM_REFERENCE(dhd_rtt_is_valid_measurement);
+
+	if (geofence_rtt) {
+		rtt_result = &nan_rtt_res;
+	} else {
+		if (RTT_IS_STOPPED(rtt_status)) {
+			/* Ignore the Proxd event */
+			DHD_RTT((" event handler rtt is stopped \n"));
+			if (rtt_status->flags == WL_PROXD_SESSION_FLAG_TARGET) {
+				DHD_RTT(("Device is target/Responder. Recv the event. \n"));
+			} else {
+				ret = BCME_UNSUPPORTED;
+				goto exit;
+			}
+		}
+		/* allocate rtt_results for new results */
+		rtt_result = (rtt_result_t *)MALLOCZ(dhd->osh, sizeof(rtt_result_t));
+		if (!rtt_result) {
+			ret = BCME_NORESOURCE;
+			goto exit;
+		}
+		ret = dhd_rtt_handle_directed_rtt_burst_end(dhd, peer_addr,
+			proxd_ev_data, tlvs_len, rtt_result, TRUE);
+		if (ret != BCME_OK) {
+			goto exit;
+		}
+
+	}
+
+exit:
+	mutex_unlock(&rtt_status->rtt_mutex);
+	if (ret == BCME_OK) {
+		/* Nothing to do for Responder */
+		if (rng_inst->range_role == NAN_RANGING_ROLE_INITIATOR) {
+			dhd_rtt_nan_range_report(cfg, rtt_result, geofence_rtt);
+		}
+	} else {
+		DHD_RTT_ERR(("nan-rtt: Burst End handling failed err %d is_geofence %d "
+			"retry cnt %d burst status %d", ret, geofence_rtt,
+			ftm_retry_cnt, burst_status));
+		if (rtt_result && !geofence_rtt) {
+			MFREE(dhd->osh, rtt_result,
+				sizeof(rtt_result_t));
+		}
+	}
+	NAN_MUTEX_UNLOCK();
+	return ret;
+}
+#endif /* WL_NAN */
+
+int
+dhd_rtt_event_handler(dhd_pub_t *dhd, wl_event_msg_t *event, void *event_data)
+{
+	int ret = BCME_OK;
+	int tlvs_len;
+	uint16 version;
+	wl_proxd_event_t *p_event;
+	wl_proxd_event_type_t event_type;
+	wl_proxd_ftm_session_status_t session_status;
+	const ftm_strmap_entry_t *p_loginfo;
+	rtt_result_t *rtt_result;
+#ifdef WL_CFG80211
+	rtt_status_info_t *rtt_status;
+	rtt_results_header_t *rtt_results_header = NULL;
+	bool is_new = TRUE;
+	rtt_target_info_t *target = NULL;
+#endif /* WL_CFG80211 */
+
+	DHD_RTT(("Enter %s \n", __FUNCTION__));
+	NULL_CHECK(dhd, "dhd is NULL", ret);
+
+	if (ntoh32_ua((void *)&event->datalen) < OFFSETOF(wl_proxd_event_t, tlvs)) {
+		DHD_RTT(("%s: wrong datalen:%d\n", __FUNCTION__,
+			ntoh32_ua((void *)&event->datalen)));
+		return -EINVAL;
+	}
+	event_type = ntoh32_ua((void *)&event->event_type);
+	if (event_type != WLC_E_PROXD) {
+		DHD_RTT_ERR((" failed event \n"));
+		return -EINVAL;
+	}
+
+	if (!event_data) {
+		DHD_RTT_ERR(("%s: event_data:NULL\n", __FUNCTION__));
+		return -EINVAL;
+	}
+	p_event = (wl_proxd_event_t *) event_data;
+	version = ltoh16(p_event->version);
+	if (version < WL_PROXD_API_VERSION_3) {
+		DHD_RTT_ERR(("ignore non-ftm event version = 0x%0x < WL_PROXD_API_VERSION (0x%x)\n",
+			version, WL_PROXD_API_VERSION_3));
+		return ret;
+	}
+
+	event_type = (wl_proxd_event_type_t) ltoh16(p_event->type);
+
+	DHD_RTT(("event_type=0x%x, ntoh16()=0x%x, ltoh16()=0x%x\n",
+		p_event->type, ntoh16(p_event->type), ltoh16(p_event->type)));
+	p_loginfo = ftm_get_event_type_loginfo(event_type);
+	if (p_loginfo == NULL) {
+		DHD_RTT_ERR(("receive an invalid FTM event %d\n", event_type));
+		ret = -EINVAL;
+		return ret;	/* ignore this event */
+	}
+	/* get TLVs len, skip over event header */
+	if (ltoh16(p_event->len) < OFFSETOF(wl_proxd_event_t, tlvs)) {
+		DHD_RTT_ERR(("invalid FTM event length:%d\n", ltoh16(p_event->len)));
+		ret = -EINVAL;
+		return ret;
+	}
+	tlvs_len = ltoh16(p_event->len) - OFFSETOF(wl_proxd_event_t, tlvs);
+	DHD_RTT(("receive '%s' event: version=0x%x len=%d method=%d sid=%d tlvs_len=%d\n",
+		p_loginfo->text,
+		version,
+		ltoh16(p_event->len),
+		ltoh16(p_event->method),
+		ltoh16(p_event->sid),
+		tlvs_len));
+#ifdef WL_CFG80211
+#ifdef WL_NAN
+	if ((event_type == WL_PROXD_EVENT_BURST_END) &&
+			dhd_rtt_is_nan_peer(dhd, &event->addr)) {
+		DHD_RTT(("WL_PROXD_EVENT_BURST_END for NAN RTT\n"));
+		ret = dhd_rtt_handle_nan_burst_end(dhd, &event->addr, p_event, tlvs_len);
+		return ret;
+	}
+#endif /* WL_NAN */
+
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", ret);
+	mutex_lock(&rtt_status->rtt_mutex);
+
+	if (RTT_IS_STOPPED(rtt_status)) {
+		/* Ignore the Proxd event */
+		DHD_RTT((" event handler rtt is stopped \n"));
+		if (rtt_status->flags == WL_PROXD_SESSION_FLAG_TARGET) {
+			DHD_RTT(("Device is target/Responder. Recv the event. \n"));
+		} else {
+			ret = BCME_NOTREADY;
+			goto exit;
+		}
+	}
+
+	/* check current target_mac and event_mac are matching */
+	target = &rtt_status->rtt_config.target_info[rtt_status->cur_idx];
+	if (memcmp(&target->addr, &event->addr, ETHER_ADDR_LEN)) {
+		DHD_RTT(("Ignore Proxd event for the unexpected peer "MACDBG
+			" expected peer "MACDBG"\n", MAC2STRDBG(&event->addr),
+			MAC2STRDBG(&target->addr)));
+		goto exit;
+	}
+
+#endif /* WL_CFG80211 */
+
+#ifdef WL_CFG80211
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	is_new = !dhd_rtt_get_report_header(rtt_status,
+		&rtt_results_header, &event->addr);
+	GCC_DIAGNOSTIC_POP();
+#endif /* WL_CFG80211 */
+	switch (event_type) {
+	case WL_PROXD_EVENT_SESSION_CREATE:
+		DHD_RTT(("WL_PROXD_EVENT_SESSION_CREATE\n"));
+		break;
+	case WL_PROXD_EVENT_SESSION_START:
+		DHD_RTT(("WL_PROXD_EVENT_SESSION_START\n"));
+		break;
+	case WL_PROXD_EVENT_BURST_START:
+		DHD_RTT(("WL_PROXD_EVENT_BURST_START\n"));
+		break;
+	case WL_PROXD_EVENT_BURST_END:
+		DHD_RTT(("WL_PROXD_EVENT_BURST_END for Legacy RTT\n"));
+		/* allocate rtt_results for new legacy rtt results */
+		rtt_result = (rtt_result_t *)MALLOCZ(dhd->osh, sizeof(rtt_result_t));
+		if (!rtt_result) {
+			ret = -ENOMEM;
+			goto exit;
+		}
+		ret = dhd_rtt_handle_directed_rtt_burst_end(dhd, &event->addr,
+			p_event, tlvs_len, rtt_result, FALSE);
+		if (rtt_result &&
+#ifdef WL_CFG80211
+		(ret != BCME_OK) &&
+#endif /* WL_CFG80211 */
+		TRUE) {
+			/*
+			 * Free rtt_result irrespectively, for non-cfg,
+			 * as it would not be needed any further
+			 */
+			MFREE(dhd->osh, rtt_result,
+					sizeof(rtt_result_t));
+			goto exit;
+		}
+		break;
+	case WL_PROXD_EVENT_SESSION_END:
+		DHD_RTT(("WL_PROXD_EVENT_SESSION_END\n"));
+#ifdef WL_CFG80211
+		if (!RTT_IS_ENABLED(rtt_status)) {
+			DHD_RTT(("Ignore the session end evt\n"));
+			goto exit;
+		}
+#endif /* WL_CFG80211 */
+		if (tlvs_len > 0) {
+			/* unpack TLVs and invokes the cbfn to print the event content TLVs */
+			ret = bcm_unpack_xtlv_buf((void *) &session_status,
+				(uint8 *)&p_event->tlvs[0], tlvs_len,
+				BCM_XTLV_OPTION_ALIGN32, rtt_unpack_xtlv_cbfn);
+			if (ret != BCME_OK) {
+				DHD_RTT_ERR(("%s : Failed to unpack xtlv for an event\n",
+					__FUNCTION__));
+				goto exit;
+			}
+		}
+#ifdef WL_CFG80211
+		/* In case of no result for the peer device, make fake result for error case */
+		if (is_new) {
+			dhd_rtt_create_failure_result(rtt_status, &event->addr);
+		}
+		DHD_RTT_MEM(("RTT Session End for Legacy peer "MACDBG"\n",
+			MAC2STRDBG(&event->addr)));
+		dhd_rtt_handle_rtt_session_end(dhd);
+#endif /* WL_CFG80211 */
+		break;
+	case WL_PROXD_EVENT_SESSION_RESTART:
+		DHD_RTT(("WL_PROXD_EVENT_SESSION_RESTART\n"));
+		break;
+	case WL_PROXD_EVENT_BURST_RESCHED:
+		DHD_RTT(("WL_PROXD_EVENT_BURST_RESCHED\n"));
+		break;
+	case WL_PROXD_EVENT_SESSION_DESTROY:
+		DHD_RTT(("WL_PROXD_EVENT_SESSION_DESTROY\n"));
+		break;
+	case WL_PROXD_EVENT_FTM_FRAME:
+		DHD_RTT(("WL_PROXD_EVENT_FTM_FRAME\n"));
+		break;
+	case WL_PROXD_EVENT_DELAY:
+		DHD_RTT(("WL_PROXD_EVENT_DELAY\n"));
+		break;
+	case WL_PROXD_EVENT_VS_INITIATOR_RPT:
+		DHD_RTT(("WL_PROXD_EVENT_VS_INITIATOR_RPT\n "));
+		break;
+#ifdef WL_RTT_LCI
+	case WL_PROXD_EVENT_LCI_MEAS_REP:
+	case WL_PROXD_EVENT_CIVIC_MEAS_REP:
+		DHD_RTT(("WL_PROXD_EVENT_LCI/CIVIC_MEAS_REP\n"));
+		if (tlvs_len > 0) {
+			void *buffer = NULL;
+			if (!(buffer = (void *)MALLOCZ(dhd->osh, tlvs_len))) {
+				ret = -ENOMEM;
+				goto exit;
+			}
+			/* unpack TLVs and invokes the cbfn to print the event content TLVs */
+			ret = bcm_unpack_xtlv_buf(buffer,
+				(uint8 *)&p_event->tlvs[0], tlvs_len,
+				BCM_XTLV_OPTION_NONE, rtt_unpack_xtlv_cbfn);
+			if (ret != BCME_OK) {
+				DHD_RTT_ERR(("%s : Failed to unpack xtlv for event %d\n",
+					__FUNCTION__, event_type));
+				MFREE(dhd->osh, buffer, tlvs_len);
+				goto exit;
+			}
+#ifdef WL_CFG80211
+			if (event_type == WL_PROXD_EVENT_LCI_MEAS_REP) {
+				/* free previous one and update it */
+				if (target->LCI) {
+					MFREE(dhd->osh, target->LCI,
+						target->LCI->len + BCM_XTLV_HDR_SIZE);
+				}
+				DHD_RTT(("WL_PROXD_EVENT_LCI_MEAS_REP: cache the LCI tlv\n"));
+				target->LCI = (bcm_xtlv_t *)buffer;
+			} else {
+				/* free previous one and update it */
+				if (target->LCR) {
+					MFREE(dhd->osh, target->LCR,
+						target->LCR->len + BCM_XTLV_HDR_SIZE);
+				}
+				DHD_RTT(("WL_PROXD_EVENT_CIVIC_MEAS_REP: cache the LCR tlv\n"));
+				target->LCR = (bcm_xtlv_t *)buffer;
+			}
+#endif /* WL_CFG80211 */
+		}
+		break;
+#endif /* WL_RTT_LCI */
+	case WL_PROXD_EVENT_RANGING:
+		DHD_RTT(("WL_PROXD_EVENT_RANGING\n"));
+		break;
+	case WL_PROXD_EVENT_COLLECT:
+		DHD_RTT(("WL_PROXD_EVENT_COLLECT\n"));
+		if (tlvs_len > 0) {
+			void *buffer = NULL;
+			if (!(buffer = (void *)MALLOCZ(dhd->osh, tlvs_len))) {
+				ret = -ENOMEM;
+				goto exit;
+			}
+			/* unpack TLVs and invokes the cbfn to print the event content TLVs */
+			ret = bcm_unpack_xtlv_buf(buffer,
+				(uint8 *)&p_event->tlvs[0], tlvs_len,
+				BCM_XTLV_OPTION_NONE, rtt_unpack_xtlv_cbfn);
+			MFREE(dhd->osh, buffer, tlvs_len);
+			if (ret != BCME_OK) {
+				DHD_RTT_ERR(("%s : Failed to unpack xtlv for event %d\n",
+					__FUNCTION__, event_type));
+				goto exit;
+			}
+		}
+		break;
+	case WL_PROXD_EVENT_MF_STATS:
+		DHD_RTT(("WL_PROXD_EVENT_MF_STATS\n"));
+		if (tlvs_len > 0) {
+			void *buffer = NULL;
+			if (!(buffer = (void *)MALLOCZ(dhd->osh, tlvs_len))) {
+				ret = -ENOMEM;
+				goto exit;
+			}
+			/* unpack TLVs and invokes the cbfn to print the event content TLVs */
+			ret = bcm_unpack_xtlv_buf(buffer,
+				(uint8 *)&p_event->tlvs[0], tlvs_len,
+				BCM_XTLV_OPTION_NONE, rtt_unpack_xtlv_cbfn);
+			MFREE(dhd->osh, buffer, tlvs_len);
+			if (ret != BCME_OK) {
+				DHD_RTT_ERR(("%s : Failed to unpack xtlv for event %d\n",
+					__FUNCTION__, event_type));
+				goto exit;
+			}
+		}
+		break;
+
+	default:
+		DHD_RTT_ERR(("WLC_E_PROXD: not supported EVENT Type:%d\n", event_type));
+		break;
+	}
+exit:
+#ifdef WL_CFG80211
+	mutex_unlock(&rtt_status->rtt_mutex);
+#endif /* WL_CFG80211 */
+
+	return ret;
+}
+
+/*
+ * RTT Event Handler
+ * for builds needing
+ * only debug purpose
+ * and not functional
+ */
+
+#ifdef WL_CFG80211
+static void
+dhd_rtt_work(struct work_struct *work)
+{
+	rtt_status_info_t *rtt_status;
+	dhd_pub_t *dhd;
+
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	rtt_status = container_of(work, rtt_status_info_t, work);
+	GCC_DIAGNOSTIC_POP();
+
+	dhd = rtt_status->dhd;
+	if (dhd == NULL) {
+		DHD_RTT_ERR(("%s : dhd is NULL\n", __FUNCTION__));
+		return;
+	}
+	(void) dhd_rtt_start(dhd);
+}
+#endif /* WL_CFG80211 */
+
+int
+dhd_rtt_capability(dhd_pub_t *dhd, rtt_capabilities_t *capa)
+{
+	rtt_status_info_t *rtt_status;
+	int err = BCME_OK;
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	NULL_CHECK(capa, "capa is NULL", err);
+	bzero(capa, sizeof(rtt_capabilities_t));
+
+	/* set rtt capabilities */
+	if (rtt_status->rtt_capa.proto & RTT_CAP_ONE_WAY)
+		capa->rtt_one_sided_supported = 1;
+	if (rtt_status->rtt_capa.proto & RTT_CAP_FTM_WAY)
+		capa->rtt_ftm_supported = 1;
+
+	if (rtt_status->rtt_capa.feature & RTT_FEATURE_LCI)
+		capa->lci_support = 1;
+	if (rtt_status->rtt_capa.feature & RTT_FEATURE_LCR)
+		capa->lcr_support = 1;
+	if (rtt_status->rtt_capa.feature & RTT_FEATURE_PREAMBLE)
+		capa->preamble_support = 1;
+	if (rtt_status->rtt_capa.feature & RTT_FEATURE_BW)
+		capa->bw_support = 1;
+
+	/* bit mask */
+	capa->preamble_support = rtt_status->rtt_capa.preamble;
+	capa->bw_support = rtt_status->rtt_capa.bw;
+
+	return err;
+}
+
+#ifdef WL_CFG80211
+int
+dhd_rtt_avail_channel(dhd_pub_t *dhd, wifi_channel_info *channel_info)
+{
+	u32 chanspec = 0;
+	int err = BCME_OK;
+	chanspec_t c = 0;
+	u32 channel;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+
+	if ((err = wldev_iovar_getint(dev, "chanspec",
+		(s32 *)&chanspec)) == BCME_OK) {
+		c = (chanspec_t)dtoh32(chanspec);
+		c = wl_chspec_driver_to_host(c);
+		channel  = wf_chspec_ctlchan(c);
+		DHD_RTT((" control channel is %d \n", channel));
+		if (CHSPEC_IS20(c)) {
+			channel_info->width = WIFI_CHAN_WIDTH_20;
+			DHD_RTT((" band is 20 \n"));
+		} else if (CHSPEC_IS40(c)) {
+			channel_info->width = WIFI_CHAN_WIDTH_40;
+			DHD_RTT(("band is 40 \n"));
+		} else {
+			channel_info->width = WIFI_CHAN_WIDTH_80;
+			DHD_RTT(("band is 80 \n"));
+		}
+		if (CHSPEC_IS2G(c) && (channel >= CH_MIN_2G_CHANNEL) &&
+			(channel <= CH_MAX_2G_CHANNEL)) {
+			channel_info->center_freq =
+				ieee80211_channel_to_frequency(channel, IEEE80211_BAND_2GHZ);
+		} else if (CHSPEC_IS5G(c) && channel >= CH_MIN_5G_CHANNEL) {
+			channel_info->center_freq =
+				ieee80211_channel_to_frequency(channel, IEEE80211_BAND_5GHZ);
+		}
+		if ((channel_info->width == WIFI_CHAN_WIDTH_80) ||
+			(channel_info->width == WIFI_CHAN_WIDTH_40)) {
+			channel = CHSPEC_CHANNEL(c);
+			channel_info->center_freq0 =
+				ieee80211_channel_to_frequency(channel, IEEE80211_BAND_5GHZ);
+		}
+	} else {
+		DHD_RTT_ERR(("Failed to get the chanspec \n"));
+	}
+	return err;
+}
+
+int
+dhd_rtt_enable_responder(dhd_pub_t *dhd, wifi_channel_info *channel_info)
+{
+	int err = BCME_OK;
+	char chanbuf[CHANSPEC_STR_LEN];
+	int pm = PM_OFF;
+	int ftm_cfg_cnt = 0;
+	chanspec_t chanspec;
+	wifi_channel_info channel;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+	ftm_config_options_info_t ftm_configs[FTM_MAX_CONFIGS];
+	ftm_config_param_info_t ftm_params[FTM_MAX_PARAMS];
+	rtt_status_info_t *rtt_status;
+
+	memset(&channel, 0, sizeof(channel));
+	BCM_REFERENCE(chanbuf);
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	if (RTT_IS_STOPPED(rtt_status)) {
+		DHD_RTT(("STA responder/Target. \n"));
+	}
+	DHD_RTT(("Enter %s \n", __FUNCTION__));
+	if (!dhd_is_associated(dhd, 0, NULL)) {
+		if (channel_info) {
+			channel.width = channel_info->width;
+			channel.center_freq = channel_info->center_freq;
+			channel.center_freq0 = channel_info->center_freq;
+		}
+		else {
+			channel.width = WIFI_CHAN_WIDTH_80;
+			channel.center_freq = DEFAULT_FTM_FREQ;
+			channel.center_freq0 = DEFAULT_FTM_CNTR_FREQ0;
+		}
+		chanspec =  dhd_rtt_convert_to_chspec(channel);
+		DHD_RTT(("chanspec/channel set as %s for rtt.\n",
+			wf_chspec_ntoa(chanspec, chanbuf)));
+		if (chanspec == INVCHANSPEC) {
+			DHD_RTT_ERR(("Channel is not valid \n"));
+			goto exit;
+		}
+		err = wldev_iovar_setint(dev, "chanspec", chanspec);
+		if (err) {
+			DHD_RTT_ERR(("Failed to set the chanspec \n"));
+		}
+	}
+
+	err = wldev_ioctl_get(dev, WLC_GET_PM, &rtt_status->pm, sizeof(rtt_status->pm));
+	DHD_RTT(("Current PM value read %d\n", rtt_status->pm));
+	if (err) {
+		DHD_RTT_ERR(("Failed to get the PM value \n"));
+	} else {
+		err = wldev_ioctl_set(dev, WLC_SET_PM, &pm, sizeof(pm));
+		if (err) {
+			DHD_RTT_ERR(("Failed to set the PM \n"));
+			rtt_status->pm_restore = FALSE;
+		} else {
+			rtt_status->pm_restore = TRUE;
+		}
+	}
+	if (!RTT_IS_ENABLED(rtt_status)) {
+		err = dhd_rtt_ftm_enable(dhd, TRUE);
+		if (err) {
+			DHD_RTT_ERR(("Failed to enable FTM (%d)\n", err));
+			goto exit;
+		}
+		DHD_RTT(("FTM enabled \n"));
+	}
+	rtt_status->status = RTT_ENABLED;
+	DHD_RTT(("Responder enabled \n"));
+	memset(ftm_configs, 0, sizeof(ftm_configs));
+	memset(ftm_params, 0, sizeof(ftm_params));
+	ftm_configs[ftm_cfg_cnt].enable = TRUE;
+	ftm_configs[ftm_cfg_cnt++].flags = WL_PROXD_SESSION_FLAG_TARGET;
+	rtt_status->flags = WL_PROXD_SESSION_FLAG_TARGET;
+	DHD_RTT(("Set the device as responder \n"));
+	err = dhd_rtt_ftm_config(dhd, FTM_DEFAULT_SESSION,
+		ftm_configs, ftm_cfg_cnt, NULL, 0);
+exit:
+	if (err) {
+		rtt_status->status = RTT_STOPPED;
+		DHD_RTT_ERR(("rtt is stopped  %s \n", __FUNCTION__));
+		dhd_rtt_ftm_enable(dhd, FALSE);
+		DHD_RTT(("restoring the PM value \n"));
+		if (rtt_status->pm_restore) {
+			pm = rtt_status->pm;
+			err = wldev_ioctl_set(dev, WLC_SET_PM, &pm, sizeof(pm));
+			if (err) {
+				DHD_RTT_ERR(("Failed to restore PM \n"));
+			} else {
+				rtt_status->pm_restore = FALSE;
+			}
+		}
+	}
+	return err;
+}
+
+int
+dhd_rtt_cancel_responder(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+	rtt_status_info_t *rtt_status;
+	int pm = 0;
+	struct net_device *dev = dhd_linux_get_primary_netdev(dhd);
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+	DHD_RTT(("Enter %s \n", __FUNCTION__));
+	err = dhd_rtt_ftm_enable(dhd, FALSE);
+	if (err) {
+		DHD_RTT_ERR(("failed to disable FTM (%d)\n", err));
+	}
+	rtt_status->status = RTT_STOPPED;
+	if (rtt_status->pm_restore) {
+		pm = rtt_status->pm;
+		DHD_RTT(("pm_restore =%d \n", rtt_status->pm_restore));
+		err = wldev_ioctl_set(dev, WLC_SET_PM, &pm, sizeof(pm));
+		if (err) {
+			DHD_RTT_ERR(("Failed to restore PM \n"));
+		} else {
+			rtt_status->pm_restore = FALSE;
+		}
+	}
+	return err;
+}
+
+#ifdef WL_NAN
+static bool
+dhd_rtt_parallel_nan_rtt_sessions_supported(dhd_pub_t *dhd)
+{
+	bool supported = FALSE;
+
+	if ((dhd->wlc_ver_major > RTT_PARALLEL_SSNS_SUPPORTED_MAJ_VER) ||
+		((dhd->wlc_ver_major == RTT_PARALLEL_SSNS_SUPPORTED_MAJ_VER) &&
+		(dhd->wlc_ver_minor >= RTT_PARALLEL_SSNS_SUPPORTED_MIN_VER))) {
+		supported = TRUE;
+	}
+
+	return supported;
+}
+
+int
+dhd_rtt_get_max_nan_rtt_sessions_supported(dhd_pub_t *dhd)
+{
+	int max_sessions = 0;
+
+	/* Older fw branches does not support parallel rtt sessions */
+	if (dhd_rtt_parallel_nan_rtt_sessions_supported(dhd)) {
+		max_sessions = DHD_NAN_RTT_MAX_SESSIONS;
+	} else {
+		max_sessions = DHD_NAN_RTT_MAX_SESSIONS_LEGACY;
+	}
+
+	return max_sessions;
+}
+#endif /* WL_NAN */
+#endif /* WL_CFG80211 */
+
+/*
+ * DHD Attach Context
+ */
+int
+dhd_rtt_attach(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+#ifdef WL_CFG80211
+	rtt_status_info_t *rtt_status = NULL;
+
+	dhd->rtt_supported = FALSE;
+	if (dhd->rtt_state) {
+		return err;
+	}
+
+	dhd->rtt_state = (rtt_status_info_t *)MALLOCZ(dhd->osh,
+		sizeof(rtt_status_info_t));
+	if (dhd->rtt_state == NULL) {
+		err = BCME_NOMEM;
+		DHD_RTT_ERR(("%s : failed to create rtt_state\n", __FUNCTION__));
+		return err;
+	}
+	bzero(dhd->rtt_state, sizeof(rtt_status_info_t));
+	rtt_status = GET_RTTSTATE(dhd);
+	rtt_status->rtt_config.target_info =
+		(rtt_target_info_t *)MALLOCZ(dhd->osh,
+		TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT));
+	if (rtt_status->rtt_config.target_info == NULL) {
+		DHD_RTT_ERR(("%s failed to allocate the target info for %d\n",
+			__FUNCTION__, RTT_MAX_TARGET_CNT));
+		err = BCME_NOMEM;
+		goto exit;
+	}
+	rtt_status->dhd = dhd;
+	rtt_status->pm = PM_FAST;
+	mutex_init(&rtt_status->rtt_mutex);
+	mutex_init(&rtt_status->geofence_mutex);
+	INIT_LIST_HEAD(&rtt_status->noti_fn_list);
+	INIT_LIST_HEAD(&rtt_status->rtt_results_cache);
+	INIT_WORK(&rtt_status->work, dhd_rtt_work);
+	/* initialize proxd timer */
+	INIT_DELAYED_WORK(&rtt_status->proxd_timeout, dhd_rtt_timeout_work);
+#ifdef WL_NAN
+	/* initialize proxd retry timer */
+	INIT_DELAYED_WORK(&rtt_status->rtt_retry_timer, dhd_rtt_retry_work);
+#endif /* WL_NAN */
+exit:
+	if (err < 0) {
+		MFREE(dhd->osh, rtt_status->rtt_config.target_info,
+			TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT));
+		MFREE(dhd->osh, dhd->rtt_state, sizeof(rtt_status_info_t));
+	}
+#endif /* WL_CFG80211 */
+	return err;
+
+}
+
+/*
+ * DHD Detach Context
+ */
+int
+dhd_rtt_detach(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+
+#ifdef WL_CFG80211
+	rtt_status_info_t *rtt_status;
+#ifdef WL_RTT_LCI
+	rtt_config_params_t *rtt_config;
+	int i;
+#endif /* WL_RTT_LCI */
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+
+	err = dhd_rtt_deinit(dhd);
+	if (err != BCME_OK) {
+		DHD_RTT_ERR(("dhd_rtt_deinit failed while detaching"
+			" err = %d\n", err));
+		goto exit;
+	}
+
+exit:
+#ifdef WL_RTT_LCI
+	/* free the cache of LCI/CIVIC IE */
+	rtt_config = &(rtt_status->rtt_config);
+	for (i = 0; i < rtt_config->rtt_target_cnt; i++) {
+		if (rtt_config->target_info[i].LCI) {
+			MFREE(dhd->osh, rtt_config->target_info[i].LCI,
+				rtt_config->target_info[i].LCI->len +
+				BCM_XTLV_HDR_SIZE);
+		}
+		if (rtt_config->target_info[i].LCR) {
+			MFREE(dhd->osh, rtt_config->target_info[i].LCR,
+				rtt_config->target_info[i].LCR->len +
+				BCM_XTLV_HDR_SIZE);
+		}
+	}
+#endif /* WL_RTT_LCI */
+	MFREE(dhd->osh, rtt_status->rtt_config.target_info,
+		TARGET_INFO_SIZE(RTT_MAX_TARGET_CNT));
+	MFREE(dhd->osh, dhd->rtt_state, sizeof(rtt_status_info_t));
+
+#endif /* WL_CFG80211 */
+
+	return err;
+}
+
+/*
+ * If Up context
+ */
+int
+dhd_rtt_init(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+#ifdef WL_CFG80211
+	int ret;
+	int32 version;
+	rtt_status_info_t *rtt_status;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+
+	DHD_RTT_MEM(("dhd_rtt_init ENTRY\n"));
+
+	ret = dhd_rtt_get_version(dhd, &version);
+	if (ret == BCME_OK && (version == WL_PROXD_API_VERSION_3)) {
+		DHD_RTT_ERR(("%s : FTM is supported\n", __FUNCTION__));
+		dhd->rtt_supported = TRUE;
+#ifdef WL_RTT_ONE_WAY
+		rtt_status->rtt_capa.proto |= RTT_CAP_ONE_WAY;
+#endif /* WIFI_RTT_ONE_WAY */
+		rtt_status->rtt_capa.proto |= RTT_CAP_FTM_WAY;
+
+		/* indicate to set tx rate */
+		rtt_status->rtt_capa.feature |= RTT_FEATURE_LCI;
+		rtt_status->rtt_capa.feature |= RTT_FEATURE_LCR;
+		rtt_status->rtt_capa.feature |= RTT_FEATURE_PREAMBLE;
+		rtt_status->rtt_capa.preamble |= RTT_PREAMBLE_VHT;
+		rtt_status->rtt_capa.preamble |= RTT_PREAMBLE_HT;
+
+		/* indicate to set bandwith */
+		rtt_status->rtt_capa.feature |= RTT_FEATURE_BW;
+		rtt_status->rtt_capa.bw |= RTT_BW_20;
+		rtt_status->rtt_capa.bw |= RTT_BW_40;
+		rtt_status->rtt_capa.bw |= RTT_BW_80;
+#ifdef WL_RTT_BW160
+		rtt_status->rtt_capa.bw |= RTT_BW_160;
+#endif /* WL_RTT_BW160 */
+	} else {
+		if ((ret != BCME_OK) || (version == 0)) {
+			DHD_RTT_ERR(("%s : FTM is not supported\n", __FUNCTION__));
+		} else {
+			DHD_RTT_ERR(("%s : FTM version mismatch between HOST (%d) and FW (%d)\n",
+				__FUNCTION__, WL_PROXD_API_VERSION_3, version));
+		}
+		goto exit;
+	}
+
+#ifdef WL_NAN
+	rtt_status->max_nan_rtt_sessions = dhd_rtt_get_max_nan_rtt_sessions_supported(dhd);
+#endif /* WL_NAN */
+	/* cancel all of RTT request once we got the cancel request */
+	rtt_status->all_cancel = TRUE;
+
+exit:
+	DHD_ERROR(("dhd_rtt_init EXIT, err = %d\n", err));
+#endif /* WL_CFG80211 */
+
+	return err;
+}
+
+/*
+ * If Down context
+ */
+int
+dhd_rtt_deinit(dhd_pub_t *dhd)
+{
+	int err = BCME_OK;
+#ifdef WL_CFG80211
+	rtt_status_info_t *rtt_status;
+	rtt_results_header_t *rtt_header, *next;
+	rtt_result_t *rtt_result, *next2;
+	struct rtt_noti_callback *iter, *iter2;
+	rtt_target_info_t *rtt_target = NULL;
+
+	NULL_CHECK(dhd, "dhd is NULL", err);
+	rtt_status = GET_RTTSTATE(dhd);
+	NULL_CHECK(rtt_status, "rtt_status is NULL", err);
+
+	DHD_RTT_MEM(("dhd_rtt_deinit: ENTER\n"));
+
+#ifdef WL_NAN
+	cancel_delayed_work_sync(&rtt_status->rtt_retry_timer);
+#endif /* WL_NAN */
+
+	cancel_work_sync(&rtt_status->work);
+	rtt_status->rtt_sched = FALSE;
+
+	cancel_delayed_work_sync(&rtt_status->proxd_timeout);
+
+	mutex_lock(&rtt_status->rtt_mutex);
+	/*
+	 * Cleanup attempt is required,
+	 * if legacy RTT session is in progress
+	 */
+	if ((!RTT_IS_STOPPED(rtt_status)) &&
+			rtt_status->rtt_config.rtt_target_cnt &&
+			(rtt_status->cur_idx < rtt_status->rtt_config.rtt_target_cnt)) {
+		/* if dhd is started and there is a target cnt */
+		rtt_target = &rtt_status->rtt_config.target_info[rtt_status->cur_idx];
+		if (rtt_target->peer == RTT_PEER_AP) {
+			DHD_RTT_MEM(("dhd_rtt_deinit: Deleting Default FTM Session\n"));
+			dhd_rtt_delete_session(dhd, FTM_DEFAULT_SESSION);
+		}
+	}
+
+	rtt_status->status = RTT_STOPPED;
+	DHD_RTT(("rtt is stopped %s \n", __FUNCTION__));
+	/* clear evt callback list */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	if (!list_empty(&rtt_status->noti_fn_list)) {
+		list_for_each_entry_safe(iter, iter2, &rtt_status->noti_fn_list, list) {
+			list_del(&iter->list);
+			MFREE(dhd->osh, iter, sizeof(struct rtt_noti_callback));
+		}
+	}
+	/* remove the rtt results */
+	if (!list_empty(&rtt_status->rtt_results_cache)) {
+		list_for_each_entry_safe(rtt_header, next, &rtt_status->rtt_results_cache, list) {
+			list_del(&rtt_header->list);
+			list_for_each_entry_safe(rtt_result, next2,
+					&rtt_header->result_list, list) {
+				list_del(&rtt_result->list);
+				MFREE(dhd->osh, rtt_result, sizeof(rtt_result_t));
+			}
+			MFREE(dhd->osh, rtt_header, sizeof(rtt_results_header_t));
+		}
+	}
+	mutex_unlock(&rtt_status->rtt_mutex);
+	GCC_DIAGNOSTIC_POP();
+	DHD_RTT_MEM(("dhd_rtt_deinit: EXIT, err = %d\n", err));
+#endif /* WL_CFG80211 */
+	return err;
+}
diff --git a/drivers/net/wireless/ap12275/dhd_rtt.h b/drivers/net/wireless/ap12275/dhd_rtt.h
new file mode 100644
index 000000000000..6fb3500771d7
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_rtt.h
@@ -0,0 +1,563 @@
+/*
+ * Broadcom Dongle Host Driver (DHD), RTT
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Dual:>>
+ */
+#ifndef __DHD_RTT_H__
+#define __DHD_RTT_H__
+
+#include <dngl_stats.h>
+#include "wifi_stats.h"
+
+#define RTT_MAX_TARGET_CNT 50
+#define RTT_MAX_FRAME_CNT 25
+#define RTT_MAX_RETRY_CNT 10
+#define DEFAULT_FTM_CNT 6
+#define DEFAULT_RETRY_CNT 6
+#define DEFAULT_FTM_FREQ 5180
+#define DEFAULT_FTM_CNTR_FREQ0 5210
+#define RTT_MAX_GEOFENCE_TARGET_CNT 8
+
+#define TARGET_INFO_SIZE(count) (sizeof(rtt_target_info_t) * count)
+
+#define TARGET_TYPE(target) (target->type)
+
+#define RTT_IS_ENABLED(rtt_status) (rtt_status->status == RTT_ENABLED)
+#define RTT_IS_STOPPED(rtt_status) (rtt_status->status == RTT_STOPPED)
+
+#define GEOFENCE_RTT_LOCK(rtt_status) mutex_lock(&(rtt_status)->geofence_mutex)
+#define GEOFENCE_RTT_UNLOCK(rtt_status) mutex_unlock(&(rtt_status)->geofence_mutex)
+
+#ifndef BIT
+#define BIT(x) (1 << (x))
+#endif
+
+/* DSSS, CCK and 802.11n rates in [500kbps] units */
+#define WL_MAXRATE	108	/* in 500kbps units */
+#define WL_RATE_1M	2	/* in 500kbps units */
+#define WL_RATE_2M	4	/* in 500kbps units */
+#define WL_RATE_5M5	11	/* in 500kbps units */
+#define WL_RATE_11M	22	/* in 500kbps units */
+#define WL_RATE_6M	12	/* in 500kbps units */
+#define WL_RATE_9M	18	/* in 500kbps units */
+#define WL_RATE_12M	24	/* in 500kbps units */
+#define WL_RATE_18M	36	/* in 500kbps units */
+#define WL_RATE_24M	48	/* in 500kbps units */
+#define WL_RATE_36M	72	/* in 500kbps units */
+#define WL_RATE_48M	96	/* in 500kbps units */
+#define WL_RATE_54M	108	/* in 500kbps units */
+#define GET_RTTSTATE(dhd) ((rtt_status_info_t *)dhd->rtt_state)
+
+#ifdef WL_NAN
+/* RTT Retry Timer Interval */
+/* Fix Me: Revert back once retry logic is back in place */
+#define DHD_RTT_RETRY_TIMER_INTERVAL_MS			-1
+#endif /* WL_NAN */
+
+#define DHD_RTT_INVALID_TARGET_INDEX		-1
+
+enum rtt_role {
+	RTT_INITIATOR = 0,
+	RTT_TARGET = 1
+};
+enum rtt_status {
+	RTT_STOPPED = 0,
+	RTT_STARTED = 1,
+	RTT_ENABLED = 2
+};
+typedef int64_t wifi_timestamp; /* In microseconds (us) */
+typedef int64_t wifi_timespan;
+typedef int32 wifi_rssi_rtt;
+
+typedef enum {
+	RTT_INVALID,
+	RTT_ONE_WAY,
+	RTT_TWO_WAY,
+	RTT_AUTO
+} rtt_type_t;
+
+/* RTT peer type */
+typedef enum {
+	RTT_PEER_AP         = 0x1,
+	RTT_PEER_STA        = 0x2,
+	RTT_PEER_P2P_GO     = 0x3,
+	RTT_PEER_P2P_CLIENT = 0x4,
+	RTT_PEER_NAN        = 0x5,
+	RTT_PEER_INVALID    = 0x6
+} rtt_peer_type_t;
+
+/* Ranging status */
+typedef enum rtt_reason {
+	RTT_STATUS_SUCCESS       = 0,
+	RTT_STATUS_FAILURE       = 1,           // general failure status
+	RTT_STATUS_FAIL_NO_RSP   = 2,           // target STA does not respond to request
+	RTT_STATUS_FAIL_REJECTED = 3,           // request rejected. Applies to 2-sided RTT only
+	RTT_STATUS_FAIL_NOT_SCHEDULED_YET  = 4,
+	RTT_STATUS_FAIL_TM_TIMEOUT         = 5, // timing measurement times out
+	RTT_STATUS_FAIL_AP_ON_DIFF_CHANNEL = 6, // Target on different channel, cannot range
+	RTT_STATUS_FAIL_NO_CAPABILITY  = 7,     // ranging not supported
+	RTT_STATUS_ABORTED             = 8,     // request aborted for unknown reason
+	RTT_STATUS_FAIL_INVALID_TS     = 9,     // Invalid T1-T4 timestamp
+	RTT_STATUS_FAIL_PROTOCOL       = 10,    // 11mc protocol failed
+	RTT_STATUS_FAIL_SCHEDULE       = 11,    // request could not be scheduled
+	RTT_STATUS_FAIL_BUSY_TRY_LATER = 12,    // responder cannot collaborate at time of request
+	RTT_STATUS_INVALID_REQ         = 13,    // bad request args
+	RTT_STATUS_NO_WIFI             = 14,    // WiFi not enabled Responder overrides param info
+						// cannot range with new params
+	RTT_STATUS_FAIL_FTM_PARAM_OVERRIDE = 15
+} rtt_reason_t;
+
+enum {
+	RTT_CAP_ONE_WAY	 = BIT(0),
+	/* IEEE802.11mc */
+	RTT_CAP_FTM_WAY  = BIT(1)
+};
+
+enum {
+	RTT_FEATURE_LCI = BIT(0),
+	RTT_FEATURE_LCR = BIT(1),
+	RTT_FEATURE_PREAMBLE = BIT(2),
+	RTT_FEATURE_BW = BIT(3)
+};
+
+enum {
+	RTT_PREAMBLE_LEGACY = BIT(0),
+	RTT_PREAMBLE_HT = BIT(1),
+	RTT_PREAMBLE_VHT = BIT(2)
+};
+
+enum {
+	RTT_BW_5 = BIT(0),
+	RTT_BW_10 = BIT(1),
+	RTT_BW_20 = BIT(2),
+	RTT_BW_40 = BIT(3),
+	RTT_BW_80 = BIT(4),
+	RTT_BW_160 = BIT(5)
+};
+
+enum rtt_rate_bw {
+	RTT_RATE_20M,
+	RTT_RATE_40M,
+	RTT_RATE_80M,
+	RTT_RATE_160M
+};
+
+typedef enum ranging_type {
+	RTT_TYPE_INVALID	=	0,
+	RTT_TYPE_LEGACY		=	1,
+	RTT_TYPE_NAN_DIRECTED	=	2,
+	RTT_TYPE_NAN_GEOFENCE	=	3
+} ranging_type_t;
+
+typedef enum ranging_target_list_mode {
+	RNG_TARGET_LIST_MODE_INVALID	=	0,
+	RNG_TARGET_LIST_MODE_LEGACY	=	1,
+	RNG_TARGET_LIST_MODE_NAN	=	2,
+	RNG_TARGET_LIST_MODE_MIX	=	3
+} ranging_target_list_mode_t;
+
+#define FTM_MAX_NUM_BURST_EXP	14
+#define HAS_11MC_CAP(cap) (cap & RTT_CAP_FTM_WAY)
+#define HAS_ONEWAY_CAP(cap) (cap & RTT_CAP_ONE_WAY)
+#define HAS_RTT_CAP(cap) (HAS_ONEWAY_CAP(cap) || HAS_11MC_CAP(cap))
+
+typedef struct rtt_target_info {
+	struct ether_addr addr;
+	struct ether_addr local_addr;
+	rtt_type_t type; /* rtt_type */
+	rtt_peer_type_t peer; /* peer type */
+	wifi_channel_info channel; /* channel information */
+	chanspec_t chanspec; /* chanspec for channel */
+	bool	disable; /* disable for RTT measurement */
+	/*
+	* Time interval between bursts (units: 100 ms).
+	* Applies to 1-sided and 2-sided RTT multi-burst requests.
+	* Range: 0-31, 0: no preference by initiator (2-sided RTT)
+	*/
+	uint32	burst_period;
+	/*
+	* Total number of RTT bursts to be executed. It will be
+	* specified in the same way as the parameter "Number of
+	* Burst Exponent" found in the FTM frame format. It
+	* applies to both: 1-sided RTT and 2-sided RTT. Valid
+	* values are 0 to 15 as defined in 802.11mc std.
+	* 0 means single shot
+	* The implication of this parameter on the maximum
+	* number of RTT results is the following:
+	* for 1-sided RTT: max num of RTT results = (2^num_burst)*(num_frames_per_burst)
+	* for 2-sided RTT: max num of RTT results = (2^num_burst)*(num_frames_per_burst - 1)
+	*/
+	uint16 num_burst;
+	/*
+	* num of frames per burst.
+	* Minimum value = 1, Maximum value = 31
+	* For 2-sided this equals the number of FTM frames
+	* to be attempted in a single burst. This also
+	* equals the number of FTM frames that the
+	* initiator will request that the responder send
+	* in a single frame
+	*/
+	uint32 num_frames_per_burst;
+	/*
+	 * num of frames in each RTT burst
+	 * for single side, measurement result num = frame number
+	 * for 2 side RTT, measurement result num  = frame number - 1
+	 */
+	uint32 num_retries_per_ftm; /* retry time for RTT measurment frame */
+	/* following fields are only valid for 2 side RTT */
+	uint32 num_retries_per_ftmr;
+	uint8  LCI_request;
+	uint8  LCR_request;
+#ifdef WL_RTT_LCI
+	bcm_xtlv_t *LCI; /* LCI Report */
+	bcm_xtlv_t *LCR; /* Location Civic Report */
+#endif /* WL_RTT_LCI */
+	/*
+	* Applies to 1-sided and 2-sided RTT. Valid values will
+	* be 2-11 and 15 as specified by the 802.11mc std for
+	* the FTM parameter burst duration. In a multi-burst
+	* request, if responder overrides with larger value,
+	* the initiator will return failure. In a single-burst
+	* request if responder overrides with larger value,
+	* the initiator will sent TMR_STOP to terminate RTT
+	* at the end of the burst_duration it requested.
+	*/
+	uint32 burst_duration;
+	uint32 burst_timeout;
+	uint8  preamble;	/* 1 - Legacy, 2 - HT, 4 - VHT */
+	uint8  bw;		/* 5, 10, 20, 40, 80, 160 */
+	uint16 sid;		/* session ID for the target */
+} rtt_target_info_t;
+
+typedef struct rtt_goefence_target_info {
+	bool valid;
+	struct ether_addr peer_addr;
+} rtt_geofence_target_info_t;
+
+typedef struct rtt_config_params {
+	int8 rtt_target_cnt;
+	uint8 target_list_mode;
+	rtt_target_info_t *target_info;
+} rtt_config_params_t;
+
+typedef struct rtt_geofence_setup_status {
+	bool geofence_setup_inprog; /* Lock to serialize geofence setup */
+	struct nan_ranging_inst *rng_inst; /* Locked for this ranging instance */
+} rtt_geofence_setup_status_t;
+
+typedef struct rtt_geofence_cfg {
+	int8 geofence_target_cnt;
+	int8 cur_target_idx;
+	rtt_geofence_target_info_t geofence_target_info[RTT_MAX_GEOFENCE_TARGET_CNT];
+	int geofence_rtt_interval;
+	int max_geofence_sessions; /* Polled from FW via IOVAR Query */
+	int geofence_sessions_cnt; /* No. of Geofence/Resp Sessions running currently */
+	rtt_geofence_setup_status_t geofence_setup_status;
+#ifdef RTT_GEOFENCE_CONT
+	bool geofence_cont;
+#endif /* RTT_GEOFENCE_CONT */
+} rtt_geofence_cfg_t;
+
+typedef struct rtt_directed_setup_status {
+	bool directed_na_setup_inprog; /* Lock to serialize directed setup */
+	struct nan_ranging_inst *rng_inst; /* Locked for this ranging instance */
+} rtt_directed_setup_status_t;
+
+typedef struct rtt_directed_cfg {
+	int directed_sessions_cnt; /* No. of Geofence/Resp Sessions running currently */
+	rtt_directed_setup_status_t directed_setup_status;
+} rtt_directed_cfg_t;
+
+/*
+ * Keep Adding more reasons
+ * going forward if needed
+ */
+enum rtt_schedule_reason {
+	RTT_SCHED_HOST_TRIGGER			= 1, /* On host command for directed RTT */
+	RTT_SCHED_SUB_MATCH			= 2, /* on Sub Match for svc with range req */
+	RTT_SCHED_DIR_TRIGGER_FAIL		= 3, /* On failure of Directed RTT Trigger */
+	RTT_SCHED_DP_END			= 4, /* ON NDP End event from fw */
+	RTT_SCHED_DP_REJECTED			= 5, /* On receving reject dp event from fw */
+	RTT_SCHED_RNG_RPT_DIRECTED		= 6, /* On Ranging report for directed RTT */
+	RTT_SCHED_RNG_TERM			= 7, /* On Range Term Indicator */
+	RTT_SHCED_HOST_DIRECTED_TERM		= 8, /* On host terminating directed RTT sessions */
+	RTT_SCHED_RNG_RPT_GEOFENCE		= 9, /* On Ranging report for geofence RTT */
+	RTT_SCHED_RTT_RETRY_GEOFENCE		= 10, /* On Geofence Retry */
+	RTT_SCHED_RNG_TERM_PEND_ROLE_CHANGE	= 11, /* On Rng Term, while pending role change */
+	RTT_SCHED_RNG_TERM_SUB_SVC_CANCEL	= 12, /* Due rng canc attempt, on sub cancel */
+	RTT_SCHED_RNG_TERM_SUB_SVC_UPD		= 13, /* Due rng canc attempt, on sub update */
+	RTT_SCHED_RNG_TERM_PUB_RNG_CLEAR	= 14, /* Due rng canc attempt, on pub upd/timeout */
+	RTT_SCHED_RNG_RESP_IND			= 15, /* Due to rng resp ind */
+	RTT_SCHED_RNG_DIR_EXCESS_TARGET		= 16  /* On ssn end, if excess dir tgt pending */
+};
+
+/*
+ * Keep Adding more invalid RTT states
+ * going forward if needed
+ */
+enum rtt_invalid_state {
+	RTT_STATE_VALID			= 0, /* RTT state is valid */
+	RTT_STATE_INV_REASON_NDP_EXIST	= 1 /* RTT state invalid as ndp exists */
+};
+
+typedef struct rtt_status_info {
+	dhd_pub_t	*dhd;
+	int8		status;   /* current status for the current entry */
+	int8		txchain; /* current device tx chain */
+	int		pm; /* to save current value of pm */
+	int8		pm_restore; /* flag to reset the old value of pm */
+	int8		cur_idx; /* current entry to do RTT */
+	int8		start_idx; /* start index for RTT */
+	bool		all_cancel; /* cancel all request once we got the cancel requet */
+	uint32		flags; /* indicate whether device is configured as initiator or target */
+	struct capability {
+		int32 proto     :8;
+		int32 feature   :8;
+		int32 preamble  :8;
+		int32 bw        :8;
+	} rtt_capa; /* rtt capability */
+	struct			mutex rtt_mutex;
+	struct			mutex geofence_mutex;
+	rtt_config_params_t	rtt_config;
+	rtt_geofence_cfg_t	geofence_cfg;
+	rtt_directed_cfg_t	directed_cfg;
+	struct work_struct	work;
+	struct list_head	noti_fn_list;
+	struct list_head	rtt_results_cache; /* store results for RTT */
+	int			rtt_sched_reason; /* rtt_schedule_reason: what scheduled RTT */
+	struct delayed_work	proxd_timeout; /* Proxd Timeout work */
+	struct delayed_work	rtt_retry_timer;   /* Timer for retry RTT after all targets done */
+	bool			rtt_sched; /* TO serialize rtt thread */
+	int			max_nan_rtt_sessions; /* To be Polled from FW via IOVAR Query */
+} rtt_status_info_t;
+
+typedef struct rtt_report {
+	struct ether_addr addr;
+	unsigned int burst_num; /* # of burst inside a multi-burst request */
+	unsigned int ftm_num; /* total RTT measurement frames attempted */
+	unsigned int success_num; /* total successful RTT measurement frames */
+	uint8  num_per_burst_peer; /* max number of FTM number per burst the peer support */
+	rtt_reason_t status; /* raging status */
+	/* in s, 11mc only, only for RTT_REASON_FAIL_BUSY_TRY_LATER, 1- 31s */
+	uint8 retry_after_duration;
+	rtt_type_t type; /* rtt type */
+	wifi_rssi_rtt  rssi; /* average rssi in 0.5 dB steps e.g. 143 implies -71.5 dB */
+	wifi_rssi_rtt  rssi_spread; /* rssi spread in 0.5 db steps e.g. 5 implies 2.5 spread */
+	/*
+	* 1-sided RTT: TX rate of RTT frame.
+	* 2-sided RTT: TX rate of initiator's Ack in response to FTM frame.
+	*/
+	wifi_rate_v1 tx_rate;
+	/*
+	* 1-sided RTT: TX rate of Ack from other side.
+	* 2-sided RTT: TX rate of FTM frame coming from responder.
+	*/
+	wifi_rate_v1 rx_rate;
+	wifi_timespan rtt;	/*  round trip time in 0.1 nanoseconds */
+	wifi_timespan rtt_sd;	/* rtt standard deviation in 0.1 nanoseconds */
+	wifi_timespan rtt_spread; /* difference between max and min rtt times recorded */
+	int distance; /* distance in cm (optional) */
+	int distance_sd; /* standard deviation in cm (optional) */
+	int distance_spread; /* difference between max and min distance recorded (optional) */
+	wifi_timestamp ts; /* time of the measurement (in microseconds since boot) */
+	int burst_duration; /* in ms, how long the FW time is to fininish one burst measurement */
+	int negotiated_burst_num; /* Number of bursts allowed by the responder */
+	bcm_tlv_t *LCI; /* LCI Report */
+	bcm_tlv_t *LCR; /* Location Civic Report */
+} rtt_report_t;
+#define RTT_REPORT_SIZE (sizeof(rtt_report_t))
+
+/* rtt_results_header to maintain rtt result list per mac address */
+typedef struct rtt_results_header {
+	struct ether_addr peer_mac;
+	uint32 result_cnt;
+	uint32 result_tot_len; /* sum of report_len of rtt_result */
+	struct list_head list;
+	struct list_head result_list;
+} rtt_results_header_t;
+struct rtt_result_detail {
+	uint8 num_ota_meas;
+	uint32 result_flags;
+};
+/* rtt_result to link all of rtt_report */
+typedef struct rtt_result {
+	struct list_head list;
+	struct rtt_report report;
+	int32 report_len; /* total length of rtt_report */
+	struct rtt_result_detail rtt_detail;
+	int32 detail_len;
+} rtt_result_t;
+
+/* RTT Capabilities */
+typedef struct rtt_capabilities {
+	uint8 rtt_one_sided_supported;  /* if 1-sided rtt data collection is supported */
+	uint8 rtt_ftm_supported;        /* if ftm rtt data collection is supported */
+	uint8 lci_support;		/* location configuration information */
+	uint8 lcr_support;		/* Civic Location */
+	uint8 preamble_support;         /* bit mask indicate what preamble is supported */
+	uint8 bw_support;               /* bit mask indicate what BW is supported */
+} rtt_capabilities_t;
+
+/* RTT responder information */
+typedef struct wifi_rtt_responder {
+	wifi_channel_info channel;   /* channel of responder */
+	uint8 preamble;             /* preamble supported by responder */
+} wifi_rtt_responder_t;
+
+typedef void (*dhd_rtt_compl_noti_fn)(void *ctx, void *rtt_data);
+/* Linux wrapper to call common dhd_rtt_set_cfg */
+int dhd_dev_rtt_set_cfg(struct net_device *dev, void *buf);
+
+int dhd_dev_rtt_cancel_cfg(struct net_device *dev, struct ether_addr *mac_list, int mac_cnt);
+
+int dhd_dev_rtt_register_noti_callback(struct net_device *dev, void *ctx,
+	dhd_rtt_compl_noti_fn noti_fn);
+
+int dhd_dev_rtt_unregister_noti_callback(struct net_device *dev, dhd_rtt_compl_noti_fn noti_fn);
+
+int dhd_dev_rtt_capability(struct net_device *dev, rtt_capabilities_t *capa);
+
+int dhd_dev_rtt_avail_channel(struct net_device *dev, wifi_channel_info *channel_info);
+
+int dhd_dev_rtt_enable_responder(struct net_device *dev, wifi_channel_info *channel_info);
+
+int dhd_dev_rtt_cancel_responder(struct net_device *dev);
+/* export to upper layer */
+chanspec_t dhd_rtt_convert_to_chspec(wifi_channel_info channel);
+
+int dhd_rtt_idx_to_burst_duration(uint idx);
+
+int dhd_rtt_set_cfg(dhd_pub_t *dhd, rtt_config_params_t *params);
+
+#ifdef WL_NAN
+void dhd_rtt_initialize_geofence_cfg(dhd_pub_t *dhd);
+#ifdef RTT_GEOFENCE_CONT
+void dhd_rtt_set_geofence_cont_ind(dhd_pub_t *dhd, bool geofence_cont);
+
+void dhd_rtt_get_geofence_cont_ind(dhd_pub_t *dhd, bool* geofence_cont);
+#endif /* RTT_GEOFENCE_CONT */
+
+#ifdef RTT_GEOFENCE_INTERVAL
+void dhd_rtt_set_geofence_rtt_interval(dhd_pub_t *dhd, int interval);
+#endif /* RTT_GEOFENCE_INTERVAL */
+
+int dhd_rtt_get_geofence_max_sessions(dhd_pub_t *dhd);
+
+bool dhd_rtt_geofence_sessions_maxed_out(dhd_pub_t *dhd);
+
+int dhd_rtt_get_geofence_sessions_cnt(dhd_pub_t *dhd);
+
+int dhd_rtt_update_geofence_sessions_cnt(dhd_pub_t *dhd, bool incr,
+	struct ether_addr *peer_addr);
+
+int8 dhd_rtt_get_geofence_target_cnt(dhd_pub_t *dhd);
+
+rtt_geofence_target_info_t* dhd_rtt_get_geofence_target_head(dhd_pub_t *dhd);
+
+rtt_geofence_target_info_t* dhd_rtt_get_geofence_current_target(dhd_pub_t *dhd);
+
+rtt_geofence_target_info_t*
+dhd_rtt_get_geofence_target(dhd_pub_t *dhd, struct ether_addr* peer_addr,
+	int8 *index);
+
+int dhd_rtt_add_geofence_target(dhd_pub_t *dhd, rtt_geofence_target_info_t  *target);
+
+int dhd_rtt_remove_geofence_target(dhd_pub_t *dhd, struct ether_addr *peer_addr);
+
+int dhd_rtt_delete_geofence_target_list(dhd_pub_t *dhd);
+
+int dhd_rtt_delete_nan_session(dhd_pub_t *dhd);
+
+bool dhd_rtt_nan_is_directed_setup_in_prog(dhd_pub_t *dhd);
+
+bool dhd_rtt_nan_is_directed_setup_in_prog_with_peer(dhd_pub_t *dhd,
+	struct ether_addr *peer);
+
+void dhd_rtt_nan_update_directed_setup_inprog(dhd_pub_t *dhd,
+	struct nan_ranging_inst *rng_inst, bool inprog);
+
+bool dhd_rtt_nan_directed_sessions_allowed(dhd_pub_t *dhd);
+
+bool dhd_rtt_nan_all_directed_sessions_triggered(dhd_pub_t *dhd);
+
+void dhd_rtt_nan_update_directed_sessions_cnt(dhd_pub_t *dhd, bool incr);
+#endif /* WL_NAN */
+
+uint8 dhd_rtt_invalid_states(struct net_device *ndev, struct ether_addr *peer_addr);
+
+int8 dhd_rtt_get_cur_target_idx(dhd_pub_t *dhd);
+
+int8 dhd_rtt_set_next_target_idx(dhd_pub_t *dhd, int start_idx);
+
+void dhd_rtt_schedule_rtt_work_thread(dhd_pub_t *dhd, int sched_reason);
+
+int dhd_rtt_stop(dhd_pub_t *dhd, struct ether_addr *mac_list, int mac_cnt);
+
+int dhd_rtt_register_noti_callback(dhd_pub_t *dhd, void *ctx, dhd_rtt_compl_noti_fn noti_fn);
+
+int dhd_rtt_unregister_noti_callback(dhd_pub_t *dhd, dhd_rtt_compl_noti_fn noti_fn);
+
+int dhd_rtt_event_handler(dhd_pub_t *dhd, wl_event_msg_t *event, void *event_data);
+
+int dhd_rtt_capability(dhd_pub_t *dhd, rtt_capabilities_t *capa);
+
+int dhd_rtt_avail_channel(dhd_pub_t *dhd, wifi_channel_info *channel_info);
+
+int dhd_rtt_enable_responder(dhd_pub_t *dhd, wifi_channel_info *channel_info);
+
+int dhd_rtt_cancel_responder(dhd_pub_t *dhd);
+
+int dhd_rtt_attach(dhd_pub_t *dhd);
+
+int dhd_rtt_detach(dhd_pub_t *dhd);
+
+int dhd_rtt_init(dhd_pub_t *dhd);
+
+int dhd_rtt_deinit(dhd_pub_t *dhd);
+
+#ifdef WL_CFG80211
+#ifdef WL_NAN
+int dhd_rtt_handle_nan_rtt_session_end(dhd_pub_t *dhd,
+	struct ether_addr *peer);
+
+void dhd_rtt_move_geofence_cur_target_idx_to_next(dhd_pub_t *dhd);
+
+int8 dhd_rtt_get_geofence_cur_target_idx(dhd_pub_t *dhd);
+
+void dhd_rtt_set_geofence_cur_target_idx(dhd_pub_t *dhd, int8 idx);
+
+rtt_geofence_setup_status_t* dhd_rtt_get_geofence_setup_status(dhd_pub_t *dhd);
+
+bool dhd_rtt_is_geofence_setup_inprog(dhd_pub_t *dhd);
+
+bool dhd_rtt_is_geofence_setup_inprog_with_peer(dhd_pub_t *dhd,
+	struct ether_addr *peer_addr);
+
+void dhd_rtt_set_geofence_setup_status(dhd_pub_t *dhd, bool inprog,
+	struct ether_addr *peer_addr);
+
+int dhd_rtt_get_max_nan_rtt_sessions_supported(dhd_pub_t *dhd);
+
+bool dhd_rtt_is_taget_list_mode_nan(dhd_pub_t *dhd);
+
+#endif /* WL_NAN */
+#endif /* WL_CFG80211 */
+
+#endif /* __DHD_RTT_H__ */
diff --git a/drivers/net/wireless/ap12275/dhd_sdio.c b/drivers/net/wireless/ap12275/dhd_sdio.c
new file mode 100644
index 000000000000..84daa46d1269
--- /dev/null
+++ b/drivers/net/wireless/ap12275/dhd_sdio.c
@@ -0,0 +1,12552 @@
+/*
+ * DHD Bus Module for SDIO
+ *
+ * Copyright (C) 2022, Broadcom.
+ *
+ *      Unless you and Broadcom execute a separate written software license
+ * agreement governing use of this software, this software is licensed to you
+ * under the terms of the GNU General Public License version 2 (the "GPL"),
+ * available at http://www.broadcom.com/licenses/GPLv2.php, with the
+ * following added to such license:
+ *
+ *      As a special exception, the copyright holders of this software give you
+ * permission to link this software with independent modules, and to copy and
+ * distribute the resulting executable under terms of your choice, provided that
+ * you also meet, for each linked independent module, the terms and conditions of
+ * the license of that module.  An independent module is a module which is not
+ * derived from this software.  The special exception does not apply to any
+ * modifications of the software.
+ *
+ *
+ * <<Broadcom-WL-IPTag/Open:>>
+ *
+ * $Id$
+ */
+
+#include <typedefs.h>
+#include <osl.h>
+#include <bcmsdh.h>
+
+#ifdef BCMEMBEDIMAGE
+#include BCMEMBEDIMAGE
+#endif /* BCMEMBEDIMAGE */
+
+#include <bcmdefs.h>
+#include <bcmutils.h>
+#include <bcmendian.h>
+#include <bcmdevs.h>
+#include <bcmdevs_legacy.h>    /* need to still support chips no longer in trunk firmware */
+
+#include <siutils.h>
+#include <hndpmu.h>
+#include <hndsoc.h>
+#include <bcmsdpcm.h>
+#include <hnd_armtrap.h>
+#include <hnd_cons.h>
+#include <sbchipc.h>
+#include <sbhnddma.h>
+#if defined(DHD_SPROM)
+#include <bcmsrom.h>
+#endif /* defined(DHD_SPROM) */
+
+#include <sdio.h>
+#ifdef BCMSPI
+#include <spid.h>
+#endif /* BCMSPI */
+#include <sbsdio.h>
+#include <sbsdpcmdev.h>
+#include <bcmsdpcm.h>
+#include <bcmsdbus.h>
+
+#include <ethernet.h>
+#include <802.1d.h>
+#include <802.11.h>
+
+#include <dngl_stats.h>
+#include <dhd.h>
+#include <dhd_bus.h>
+#include <dhd_proto.h>
+#include <dhd_dbg.h>
+#include <dhdioctl.h>
+#include <sdiovar.h>
+#include <wl_android.h>
+#include <dhd_config.h>
+#ifdef DHD_PKTDUMP_TOFW
+#include <dhd_linux_pktdump.h>
+#endif
+#include <linux/mmc/sdio_func.h>
+#include <dhd_linux.h>
+
+#ifdef PROP_TXSTATUS
+#include <dhd_wlfc.h>
+#endif
+#ifdef DHDTCPACK_SUPPRESS
+#include <dhd_ip.h>
+#endif /* DHDTCPACK_SUPPRESS */
+
+#ifdef BT_OVER_SDIO
+#include <dhd_bt_interface.h>
+#endif /* BT_OVER_SDIO */
+
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+#include <debugger.h>
+#endif /* DEBUGGER || DHD_DSCOPE */
+
+#include <dngl_rtlv.h>
+#if defined(FW_SIGNATURE)
+#include <bootrommem.h>
+#endif /* FW_SIGNATURE */
+#include <fwpkg_utils.h>
+
+bool dhd_mp_halting(dhd_pub_t *dhdp);
+extern void bcmsdh_waitfor_iodrain(void *sdh);
+extern void bcmsdh_reject_ioreqs(void *sdh, bool reject);
+extern bool  bcmsdh_fatal_error(void *sdh);
+static int dhdsdio_suspend(void *context);
+static int dhdsdio_resume(void *context);
+
+#ifndef DHDSDIO_MEM_DUMP_FNAME
+#define DHDSDIO_MEM_DUMP_FNAME         "mem_dump"
+#endif
+
+#define QLEN		(1024) /* bulk rx and tx queue lengths */
+#define FCHI		(QLEN - 10)
+#define FCLOW		(FCHI / 2)
+#define PRIOMASK	7 	/* XXX FixMe: should come from elsewhere...
+				 * MAXPRIO?  PKTQ_MAX_PREC? WLC? Other?
+				 */
+
+#define TXRETRIES	2	/* # of retries for tx frames */
+#define READ_FRM_CNT_RETRIES	3
+#ifndef DHD_RXBOUND
+#define DHD_RXBOUND	50	/* Default for max rx frames in one scheduling */
+#endif
+
+#ifndef DHD_TXBOUND
+#define DHD_TXBOUND	20	/* Default for max tx frames in one scheduling */
+#endif
+
+#define DHD_TXMINMAX	1	/* Max tx frames if rx still pending */
+
+#define MEMBLOCK	2048		/* Block size used for downloading of dongle image */
+#define MAX_MEMBLOCK  (32 * 1024)	/* Block size used for downloading of dongle image */
+
+#define MAX_DATA_BUF	(64 * 1024)	/* Must be large enough to hold biggest possible glom */
+#define MAX_MEM_BUF	4096
+
+#ifndef DHD_FIRSTREAD
+#define DHD_FIRSTREAD   32
+#endif
+#if !ISPOWEROF2(DHD_FIRSTREAD)
+#error DHD_FIRSTREAD is not a power of 2!
+#endif
+
+/* Total length of frame header for dongle protocol */
+#define SDPCM_HDRLEN	(SDPCM_FRAMETAG_LEN + SDPCM_SWHEADER_LEN)
+#define SDPCM_HDRLEN_TXGLOM	(SDPCM_HDRLEN + SDPCM_HWEXT_LEN)
+#define MAX_TX_PKTCHAIN_CNT	SDPCM_MAXGLOM_SIZE
+
+#ifdef SDTEST
+#define SDPCM_RESERVE	(SDPCM_HDRLEN + SDPCM_TEST_HDRLEN + DHD_SDALIGN)
+#else
+#define SDPCM_RESERVE	(SDPCM_HDRLEN + DHD_SDALIGN)
+#endif
+
+/* Space for header read, limit for data packets */
+#ifndef MAX_HDR_READ
+#define MAX_HDR_READ	32
+#endif
+#if !ISPOWEROF2(MAX_HDR_READ)
+#error MAX_HDR_READ is not a power of 2!
+#endif
+
+#define MAX_RX_DATASZ	2048	/* XXX Should be based on PKTGET limits? */
+
+/* Maximum milliseconds to wait for firmware to come up */
+#ifdef BCMQT
+#define DHD_WAIT_READSHARED  30000
+#endif /* BCMQT */
+
+/* Maximum milliseconds to wait for F2 to come up */
+#ifdef BCMQT
+#define DHD_WAIT_F2RDY  30000
+#define CONS_ADDR_SIGNATURE  0xac0ffee
+#else
+#define DHD_WAIT_F2RDY	3000
+#endif /* BCMQT */
+
+/* Maximum usec to wait for HTAVAIL to come up */
+#ifdef BCMQT
+#define DHD_WAIT_HTAVAIL	10000000
+#else
+#define DHD_WAIT_HTAVAIL	10000
+#endif /* BCMQT */
+
+/* Bump up limit on waiting for HT to account for first startup;
+ * if the image is doing a CRC calculation before programming the PMU
+ * for HT availability, it could take a couple hundred ms more, so
+ * max out at a 1 second (1000000us).
+ */
+#if (PMU_MAX_TRANSITION_DLY <= 1000000)
+#undef PMU_MAX_TRANSITION_DLY
+#ifdef NO_EXT32K
+#define PMU_MAX_TRANSITION_DLY (1000000*5)
+#else
+#define PMU_MAX_TRANSITION_DLY 1000000
+#endif
+#endif // endif
+
+/* hooks for limiting threshold custom tx num in rx processing */
+#define DEFAULT_TXINRX_THRES    0
+#ifndef CUSTOM_TXINRX_THRES
+#define CUSTOM_TXINRX_THRES     DEFAULT_TXINRX_THRES
+#endif
+
+/* Value for ChipClockCSR during initial setup */
+#define DHD_INIT_CLKCTL1	(SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_ALP_AVAIL_REQ)
+#define DHD_INIT_CLKCTL2	(SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_FORCE_ALP)
+
+/* Flags for SDH calls */
+#define F2SYNC	(SDIO_REQ_4BYTE | SDIO_REQ_FIXED)
+/* XXX #define F2ASYNC	(SDIO_REQ_4BYTE | SDIO_REQ_FIXED | SDIO_REQ_ASYNC) */
+
+/* Packet free applicable unconditionally for sdio and sdspi.  Conditional if
+ * bufpool was present for gspi bus.
+ */
+#define PKTFREE2()		if ((bus->bus != SPI_BUS) || bus->usebufpool) \
+					PKTFREE(bus->dhd->osh, pkt, FALSE);
+DHD_SPINWAIT_SLEEP_INIT(sdioh_spinwait_sleep);
+
+#ifdef SUPPORT_MULTIPLE_BOARD_REV_FROM_HW
+extern unsigned int system_hw_rev;
+#endif /* SUPPORT_MULTIPLE_BOARD_REV_FROM_HW */
+
+/* Device console log buffer state */
+#define CONSOLE_LINE_MAX	192
+#define CONSOLE_BUFFER_MAX	8192
+typedef struct dhd_console {
+	uint		count;			/* Poll interval msec counter */
+	uint		log_addr;		/* Log struct address (fixed) */
+	hnd_log_t	log;			/* Log struct (host copy) */
+	uint		bufsize;		/* Size of log buffer */
+	uint8		*buf;			/* Log buffer (host copy) */
+	uint		last;			/* Last buffer read index */
+} dhd_console_t;
+
+#define	REMAP_ENAB(bus)			((bus)->remap)
+#define	REMAP_ISADDR(bus, a)		(((a) >= ((bus)->orig_ramsize)) && ((a) < ((bus)->ramsize)))
+#define	KSO_ENAB(bus)			((bus)->kso)
+#define	SR_ENAB(bus)			((bus)->_srenab)
+#define	SLPAUTO_ENAB(bus)		((SR_ENAB(bus)) && ((bus)->_slpauto))
+
+#define	MIN_RSRC_SR			0x3
+#define	CORE_CAPEXT_ADDR_OFFSET		(0x64c)
+#define	CORE_CAPEXT_SR_SUPPORTED_MASK	(1 << 1)
+#define RCTL_MACPHY_DISABLE_MASK	(1 << 26)
+#define RCTL_LOGIC_DISABLE_MASK		(1 << 27)
+
+#define	OOB_WAKEUP_ENAB(bus)		((bus)->_oobwakeup)
+#define	GPIO_DEV_SRSTATE		16	/* Host gpio17 mapped to device gpio0 SR state */
+#define	GPIO_DEV_SRSTATE_TIMEOUT	320000	/* 320ms */
+#define	GPIO_DEV_WAKEUP			17	/* Host gpio17 mapped to device gpio1 wakeup */
+#define	CC_CHIPCTRL2_GPIO1_WAKEUP	(1  << 0)
+#define	CC_CHIPCTRL3_SR_ENG_ENABLE	(1  << 2)
+#define OVERFLOW_BLKSZ512_WM		96
+#define OVERFLOW_BLKSZ512_MES		80
+
+#define CC_PMUCC3	(0x3)
+
+#ifdef DHD_UCODE_DOWNLOAD
+/* Ucode host download related macros */
+#define UCODE_DOWNLOAD_REQUEST  0xCAFECAFE
+#define UCODE_DOWNLOAD_COMPLETE 0xABCDABCD
+#endif /* DHD_UCODE_DOWNLOAD */
+
+#if defined(BT_OVER_SDIO)
+#define BTMEM_OFFSET			0x19000000
+/* BIT0 => WLAN Power UP and BIT1=> WLAN Wake */
+#define BT2WLAN_PWRUP_WAKE		0x03
+#define BT2WLAN_PWRUP_ADDR		0x640894	/* This address is specific to 43012B0 */
+
+#define BTFW_MAX_STR_LEN		600
+#define BTFW_DOWNLOAD_BLK_SIZE		(BTFW_MAX_STR_LEN/2 + 8)
+
+#define BTFW_ADDR_MODE_UNKNOWN		0
+#define BTFW_ADDR_MODE_EXTENDED		1
+#define BTFW_ADDR_MODE_SEGMENT		2
+#define BTFW_ADDR_MODE_LINEAR32		3
+
+#define BTFW_HEX_LINE_TYPE_DATA				0
+#define BTFW_HEX_LINE_TYPE_END_OF_DATA			1
+#define BTFW_HEX_LINE_TYPE_EXTENDED_SEGMENT_ADDRESS	2
+#define BTFW_HEX_LINE_TYPE_EXTENDED_ADDRESS		4
+#define BTFW_HEX_LINE_TYPE_ABSOLUTE_32BIT_ADDRESS	5
+
+#endif /* defined (BT_OVER_SDIO) */
+
+/* Max length of filename in IOVAR or in module parameter */
+#define DHD_MAX_PATH    2048u
+
+/* Private data for SDIO bus interaction */
+typedef struct dhd_bus {
+	dhd_pub_t	*dhd;
+
+	bcmsdh_info_t	*sdh;			/* Handle for BCMSDH calls */
+	si_t		*sih;			/* Handle for SI calls */
+	char		*vars;			/* Variables (from CIS and/or other) */
+	uint		varsz;			/* Size of variables buffer */
+	uint32		sbaddr;			/* Current SB window pointer (-1, invalid) */
+
+	sdpcmd_regs_t	*regs;			/* Registers for SDIO core */
+	uint		sdpcmrev;		/* SDIO core revision */
+	uint		armrev;			/* CPU core revision */
+	uint		ramrev;			/* SOCRAM core revision */
+	uint32		ramsize;		/* Size of RAM in SOCRAM (bytes) */
+	uint32		orig_ramsize;		/* Size of RAM in SOCRAM (bytes) */
+	uint32		srmemsize;		/* Size of SRMEM */
+
+	uint32		bus;			/* gSPI or SDIO bus */
+	uint32		bus_num;		/* bus number */
+	uint32		slot_num;		/* slot ID */
+	uint32		hostintmask;	/* Copy of Host Interrupt Mask */
+	uint32		intstatus;		/* Intstatus bits (events) pending */
+	bool		dpc_sched;		/* Indicates DPC schedule (intrpt rcvd) */
+	bool		fcstate;		/* State of dongle flow-control */
+
+	uint16		cl_devid;		/* cached devid for dhdsdio_probe_attach() */
+	char		*fw_path;		/* module_param: path to firmware image */
+	char		*nv_path;		/* module_param: path to nvram vars file */
+
+	uint		blocksize;		/* Block size of SDIO transfers */
+	uint		roundup;		/* Max roundup limit */
+
+	struct pktq	txq;			/* Queue length used for flow-control */
+	uint8		flowcontrol;		/* per prio flow control bitmask */
+	uint8		tx_seq;			/* Transmit sequence number (next) */
+	uint8		tx_max;			/* Maximum transmit sequence allowed */
+
+#ifdef DYNAMIC_MAX_HDR_READ
+	uint8		*hdrbufp;
+#else
+	uint8		hdrbuf[MAX_HDR_READ + DHD_SDALIGN];
+#endif
+	uint8		*rxhdr;			/* Header of current rx frame (in hdrbuf) */
+	uint16		nextlen;		/* Next Read Len from last header */
+	uint8		rx_seq;			/* Receive sequence number (expected) */
+	bool		rxskip;			/* Skip receive (awaiting NAK ACK) */
+
+	void		*glomd;			/* Packet containing glomming descriptor */
+	void		*glom;			/* Packet chain for glommed superframe */
+	uint		glomerr;		/* Glom packet read errors */
+
+	uint8		*rxbuf;			/* Buffer for receiving control packets */
+	uint		rxblen;			/* Allocated length of rxbuf */
+	uint8		*rxctl;			/* Aligned pointer into rxbuf */
+	uint8		*databuf;		/* Buffer for receiving big glom packet */
+	uint8		*dataptr;		/* Aligned pointer into databuf */
+	uint		rxlen;			/* Length of valid data in buffer */
+
+	uint8		sdpcm_ver;		/* Bus protocol reported by dongle */
+
+	bool		intr;			/* Use interrupts */
+	bool		poll;			/* Use polling */
+	bool		ipend;			/* Device interrupt is pending */
+	bool		intdis;			/* Interrupts disabled by isr */
+	uint 		intrcount;		/* Count of device interrupt callbacks */
+	uint		lastintrs;		/* Count as of last watchdog timer */
+	uint		spurious;		/* Count of spurious interrupts */
+	uint		pollrate;		/* Ticks between device polls */
+	uint		polltick;		/* Tick counter */
+	uint		pollcnt;		/* Count of active polls */
+
+	dhd_console_t	console;		/* Console output polling support */
+	uint		console_addr;		/* Console address from shared struct */
+
+	uint		regfails;		/* Count of R_REG/W_REG failures */
+
+	uint		clkstate;		/* State of sd and backplane clock(s) */
+	bool		activity;		/* Activity flag for clock down */
+	int32		idletime;		/* Control for activity timeout */
+	int32		idlecount;		/* Activity timeout counter */
+	int32		idleclock;		/* How to set bus driver when idle */
+	int32		sd_divisor;		/* Speed control to bus driver */
+	int32		sd_mode;		/* Mode control to bus driver */
+	int32		sd_rxchain;		/* If bcmsdh api accepts PKT chains */
+	bool		use_rxchain;		/* If dhd should use PKT chains */
+	bool		sleeping;		/* Is SDIO bus sleeping? */
+#if defined(LINUX) && defined(SUPPORT_P2P_GO_PS)
+	wait_queue_head_t bus_sleep;
+#endif /* LINUX && SUPPORT_P2P_GO_PS */
+	bool		ctrl_wait;
+	wait_queue_head_t ctrl_tx_wait;
+	uint		rxflow_mode;		/* Rx flow control mode */
+	bool		rxflow;			/* Is rx flow control on */
+	uint		prev_rxlim_hit;		/* Is prev rx limit exceeded (per dpc schedule) */
+	bool		alp_only;		/* Don't use HT clock (ALP only) */
+	/* Field to decide if rx of control frames happen in rxbuf or lb-pool */
+	bool		usebufpool;
+	int32		txinrx_thres;	/* num of in-queued pkts */
+	int32		dotxinrx;	/* tx first in dhdsdio_readframes */
+#ifdef BCMSDIO_RXLIM_POST
+	bool		rxlim_en;
+	uint32		rxlim_addr;
+#endif /* BCMSDIO_RXLIM_POST */
+#ifdef SDTEST
+	/* external loopback */
+	bool		ext_loop;
+	uint8		loopid;
+
+	/* pktgen configuration */
+	uint		pktgen_freq;		/* Ticks between bursts */
+	uint		pktgen_count;		/* Packets to send each burst */
+	uint		pktgen_print;		/* Bursts between count displays */
+	uint		pktgen_total;		/* Stop after this many */
+	uint		pktgen_minlen;		/* Minimum packet data len */
+	uint		pktgen_maxlen;		/* Maximum packet data len */
+	uint		pktgen_mode;		/* Configured mode: tx, rx, or echo */
+	uint		pktgen_stop;		/* Number of tx failures causing stop */
+
+	/* active pktgen fields */
+	uint		pktgen_tick;		/* Tick counter for bursts */
+	uint		pktgen_ptick;		/* Burst counter for printing */
+	uint		pktgen_sent;		/* Number of test packets generated */
+	uint		pktgen_rcvd;		/* Number of test packets received */
+	uint		pktgen_prev_time;	/* Time at which previous stats where printed */
+	uint		pktgen_prev_sent;	/* Number of test packets generated when
+						 * previous stats were printed
+						 */
+	uint		pktgen_prev_rcvd;	/* Number of test packets received when
+						 * previous stats were printed
+						 */
+	uint		pktgen_fail;		/* Number of failed send attempts */
+	uint16		pktgen_len;		/* Length of next packet to send */
+#define PKTGEN_RCV_IDLE     (0)
+#define PKTGEN_RCV_ONGOING  (1)
+	uint16		pktgen_rcv_state;		/* receive state */
+	uint		pktgen_rcvd_rcvsession;	/* test pkts rcvd per rcv session. */
+#endif /* SDTEST */
+
+	/* Some additional counters */
+	uint		tx_sderrs;		/* Count of tx attempts with sd errors */
+	uint		fcqueued;		/* Tx packets that got queued */
+	uint		rxrtx;			/* Count of rtx requests (NAK to dongle) */
+	uint		rx_toolong;		/* Receive frames too long to receive */
+	uint		rxc_errors;		/* SDIO errors when reading control frames */
+	uint		rx_hdrfail;		/* SDIO errors on header reads */
+	uint		rx_badhdr;		/* Bad received headers (roosync?) */
+	uint		rx_badseq;		/* Mismatched rx sequence number */
+	uint		fc_rcvd;		/* Number of flow-control events received */
+	uint		fc_xoff;		/* Number which turned on flow-control */
+	uint		fc_xon;			/* Number which turned off flow-control */
+	uint		rxglomfail;		/* Failed deglom attempts */
+	uint		rxglomframes;		/* Number of glom frames (superframes) */
+	uint		rxglompkts;		/* Number of packets from glom frames */
+	uint		f2rxhdrs;		/* Number of header reads */
+	uint		f2rxdata;		/* Number of frame data reads */
+	uint		f2txdata;		/* Number of f2 frame writes */
+	uint		f1regdata;		/* Number of f1 register accesses */
+	wake_counts_t	wake_counts;		/* Wake up counter */
+#ifdef BCMSPI
+	bool		dwordmode;
+#endif /* BCMSPI */
+#ifdef DHDENABLE_TAILPAD
+	uint		tx_tailpad_chain;	/* Number of tail padding by chaining pad_pkt */
+	uint		tx_tailpad_pktget;	/* Number of tail padding by new PKTGET */
+#endif /* DHDENABLE_TAILPAD */
+	uint8		*ctrl_frame_buf;
+	uint32		ctrl_frame_len;
+	bool		ctrl_frame_stat;
+#ifndef BCMSPI
+	uint32		rxint_mode;	/* rx interrupt mode */
+#endif /* BCMSPI */
+	bool		remap;		/* Contiguous 1MB RAM: 512K socram + 512K devram
+					 * Available with socram rev 16
+					 * Remap region not DMA-able
+					 */
+	bool		kso;
+	bool		_slpauto;
+	bool		_oobwakeup;
+	bool		_srenab;
+	bool        readframes;
+	bool        reqbussleep;
+	uint32		resetinstr;
+	uint32		dongle_ram_base;
+
+	void		*glom_pkt_arr[SDPCM_MAXGLOM_SIZE];	/* Array of pkts for glomming */
+	uint32		txglom_cnt;	/* Number of pkts in the glom array */
+	uint32		txglom_total_len;	/* Total length of pkts in glom array */
+	bool		txglom_enable;	/* Flag to indicate whether tx glom is enabled/disabled */
+	uint32		txglomsize;	/* Glom size limitation */
+#ifdef DHDENABLE_TAILPAD
+	void		*pad_pkt;
+#endif /* DHDENABLE_TAILPAD */
+	uint32		dongle_trap_addr; /* device trap addr location in device memory */
+#if defined(BT_OVER_SDIO)
+	char		*btfw_path;	/* module_param: path to BT firmware image */
+	uint32		bt_use_count; /* Counter that tracks whether BT is using the bus */
+#endif /* defined (BT_OVER_SDIO) */
+	uint64		last_suspend_end_time;
+
+	/* Information used to compose the memory map and to write the memory map,
+	 * FW, and FW signature to dongle RAM.
+	 * This information is used by the bootloader.
+	 */
+	uint32 ramtop_addr;             /* Dongle address of unused space at top of RAM */
+	uint32 fw_download_addr;        /* Dongle address of FW download */
+	uint32 fw_download_len;         /* Length in bytes of FW download */
+	uint32 fwsig_download_addr;     /* Dongle address of FW signature download */
+	uint32 fwsig_download_len;      /* Length in bytes of FW signature download */
+	uint32 fwstat_download_addr;    /* Dongle address of FWS status download */
+	uint32 fwstat_download_len;     /* Length in bytes of FWS status download */
+	uint32 fw_memmap_download_addr; /* Dongle address of FWS memory-info download */
+	uint32 fw_memmap_download_len;  /* Length in bytes of FWS memory-info download */
+
+	char fwsig_filename[DHD_MAX_PATH];              /* Name of FW signature file */
+	char bootloader_filename[DHD_FILENAME_MAX];     /* Name of bootloader image file */
+	uint32 bootloader_addr;         /* Dongle address of bootloader download */
+	fwpkg_info_t fwpkg;     /* combined fw package info structure */
+	uint		txglomframes;	/* Number of tx glom frames (superframes) */
+	uint		txglompkts;		/* Number of packets from tx glom frames */
+#ifdef PKT_STATICS
+	struct pkt_statics tx_statics;
+#endif
+	uint8		*membuf;		/* Buffer for dhdsdio_membytes */
+#ifdef CONSOLE_DPC
+	char		cons_cmd[16];
+#endif
+} dhd_bus_t;
+
+/*
+ * Whenever DHD_IDLE_IMMEDIATE condition is handled, we have to now check if
+ * BT is active too. Instead of adding #ifdef code in all the places, we thought
+ * of adding one macro check as part of the if condition that checks for DHD_IDLE_IMMEDIATE
+ * In case of non BT over SDIO builds, this macro will always return TRUE. In case
+ * of the builds where BT_OVER_SDIO is enabled, it will expand to a condition check
+ * that checks if bt_use_count is zero. So this macro will return equate to 1 if
+ * bt_use_count is 0, indicating that there are no active users and if bt_use_count
+ * is non zero it would return 0 there by preventing the caller from executing the
+ * sleep calls.
+ */
+#ifdef BT_OVER_SDIO
+#define NO_OTHER_ACTIVE_BUS_USER(bus)		(bus->bt_use_count == 0)
+#else
+#define NO_OTHER_ACTIVE_BUS_USER(bus)		(1)
+#endif /* BT_OVER_SDIO */
+
+/* clkstate */
+#define CLK_NONE	0
+#define CLK_SDONLY	1
+#define CLK_PENDING	2	/* Not used yet */
+#define CLK_AVAIL	3
+
+#define DHD_NOPMU(dhd)	(FALSE)
+
+#if defined(BCMSDIOH_STD)
+#define BLK_64_MAXTXGLOM 20
+#endif /* BCMSDIOH_STD */
+
+#ifdef DHD_DEBUG
+static int qcount[NUMPRIO];
+static int tx_packets[NUMPRIO];
+#endif /* DHD_DEBUG */
+
+/* Deferred transmit */
+const uint dhd_deferred_tx = 1;
+
+extern uint dhd_watchdog_ms;
+#ifndef NDIS
+extern uint sd_f1_blocksize;
+#endif /* !NDIS */
+
+#ifdef BCMSPI_ANDROID
+extern uint *dhd_spi_lockcount;
+#endif /* BCMSPI_ANDROID */
+
+extern void dhd_os_wd_timer(void *bus, uint wdtick);
+int dhd_enableOOB(dhd_pub_t *dhd, bool sleep);
+#ifdef DHD_DEBUG
+static int dhdsdio_readconsole(dhd_bus_t *bus);
+#ifdef BCMQT
+static bool dhdsdio_force_console_dump(dhd_bus_t *bus);
+#endif /* BCMQT */
+#endif /* DHD_DEBUG */
+
+#ifdef DHD_PM_CONTROL_FROM_FILE
+extern bool g_pm_control;
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+
+/* Tx/Rx bounds */
+uint dhd_txbound;
+uint dhd_rxbound;
+uint dhd_txminmax = DHD_TXMINMAX;
+
+/* override the RAM size if possible */
+#define DONGLE_MIN_RAMSIZE (128 *1024)
+int dhd_dongle_ramsize;
+
+uint dhd_doflow = TRUE;
+uint dhd_dpcpoll = FALSE;
+/* SR memory size */
+uint dhd_srmem = 0;
+
+#ifdef linux
+module_param(dhd_doflow, uint, 0644);
+module_param(dhd_dpcpoll, uint, 0644);
+module_param(dhd_srmem, uint, 0644);
+#endif
+
+static bool dhd_alignctl;
+
+static bool sd1idle;
+
+static bool retrydata;
+#define RETRYCHAN(chan) (((chan) == SDPCM_EVENT_CHANNEL) || retrydata)
+
+#ifdef BCMSPI
+/* At a watermark around 8 the spid hits underflow error. */
+static uint watermark = 32;
+static uint mesbusyctrl = 0;
+#else
+static uint watermark = 8;
+static uint mesbusyctrl = 0;
+#endif /* BCMSPI */
+#ifdef DYNAMIC_MAX_HDR_READ
+uint firstread = DHD_FIRSTREAD;
+#else
+static const uint firstread = DHD_FIRSTREAD;
+#endif
+
+/* Retry count for register access failures */
+static const uint retry_limit = 2;
+
+/* Force even SD lengths (some host controllers mess up on odd bytes) */
+static bool forcealign;
+
+#if defined(DEBUGGER)
+static uint32 dhd_sdio_reg_read(struct dhd_bus *bus, ulong addr);
+static void dhd_sdio_reg_write(struct dhd_bus *bus, ulong addr, uint32 val);
+
+/** the debugger layer will call back into this (bus) layer to read/write dongle memory */
+static struct dhd_dbg_bus_ops_s  bus_ops = {
+	.read_u16 = NULL,
+	.read_u32 = dhd_sdio_reg_read,
+	.write_u32 = dhd_sdio_reg_write,
+};
+#endif /* DEBUGGER */
+
+#define ALIGNMENT  4
+
+#if (defined(OOB_INTR_ONLY) && defined(HW_OOB)) || defined(FORCE_WOWLAN)
+extern void bcmsdh_enable_hw_oob_intr(void *sdh, bool enable);
+#endif
+
+#if defined(OOB_INTR_ONLY) && defined(SDIO_ISR_THREAD)
+#error OOB_INTR_ONLY is NOT working with SDIO_ISR_THREAD
+#endif /* defined(OOB_INTR_ONLY) && defined(SDIO_ISR_THREAD) */
+#define PKTALIGN(osh, p, len, align)					\
+	do {								\
+		uintptr datalign;						\
+		datalign = (uintptr)PKTDATA((osh), (p));		\
+		datalign = ROUNDUP(datalign, (align)) - datalign;	\
+		ASSERT(datalign < (align));				\
+		ASSERT(PKTLEN((osh), (p)) >= ((len) + datalign));	\
+		if (datalign)						\
+			PKTPULL((osh), (p), (uint)datalign);			\
+		PKTSETLEN((osh), (p), (len));				\
+	} while (0)
+
+/* Limit on rounding up frames */
+static const uint max_roundup = 512;
+
+/* Try doing readahead */
+static bool dhd_readahead;
+
+#if defined(BCMSDIOH_TXGLOM_EXT)
+bool
+dhdsdio_is_dataok(dhd_bus_t *bus) {
+	return (((uint8)(bus->tx_max - bus->tx_seq) - bus->dhd->conf->tx_max_offset > 1) && \
+	(((uint8)(bus->tx_max - bus->tx_seq) & 0x80) == 0));
+}
+
+uint8
+dhdsdio_get_databufcnt(dhd_bus_t *bus) {
+	return ((uint8)(bus->tx_max - bus->tx_seq) - 1 - bus->dhd->conf->tx_max_offset);
+}
+#endif
+
+/* To check if there's window offered */
+#if defined(BCMSDIOH_TXGLOM_EXT)
+#define DATAOK(bus) dhdsdio_is_dataok(bus)
+#else
+#define DATAOK(bus) \
+	(((uint8)(bus->tx_max - bus->tx_seq) > 1) && \
+	(((uint8)(bus->tx_max - bus->tx_seq) & 0x80) == 0))
+#endif
+
+/* To check if there's window offered for ctrl frame */
+#define TXCTLOK(bus) \
+	(((uint8)(bus->tx_max - bus->tx_seq) != 0) && \
+	(((uint8)(bus->tx_max - bus->tx_seq) & 0x80) == 0))
+
+/* Number of pkts available in dongle for data RX */
+#if defined(BCMSDIOH_TXGLOM_EXT)
+#define DATABUFCNT(bus) dhdsdio_get_databufcnt(bus)
+#else
+#define DATABUFCNT(bus) \
+	((uint8)(bus->tx_max - bus->tx_seq) - 1)
+#endif
+
+/* Macros to get register read/write status */
+/* NOTE: these assume a local dhdsdio_bus_t *bus! */
+/* XXX Need to replace these with something better. */
+#define R_SDREG(regvar, regaddr, retryvar) \
+do { \
+	retryvar = 0; \
+	do { \
+		regvar = R_REG(bus->dhd->osh, regaddr); \
+	} while (bcmsdh_regfail(bus->sdh) && (++retryvar <= retry_limit)); \
+	if (retryvar) { \
+		bus->regfails += (retryvar-1); \
+		if (retryvar > retry_limit) { \
+			DHD_ERROR(("%s: FAILED" #regvar "READ, LINE %d\n", \
+			           __FUNCTION__, __LINE__)); \
+			regvar = 0; \
+		} \
+	} \
+} while (0)
+
+#define W_SDREG(regval, regaddr, retryvar) \
+do { \
+	retryvar = 0; \
+	do { \
+		W_REG(bus->dhd->osh, regaddr, regval); \
+	} while (bcmsdh_regfail(bus->sdh) && (++retryvar <= retry_limit)); \
+	if (retryvar) { \
+		bus->regfails += (retryvar-1); \
+		if (retryvar > retry_limit) \
+			DHD_ERROR(("%s: FAILED REGISTER WRITE, LINE %d\n", \
+			           __FUNCTION__, __LINE__)); \
+	} \
+} while (0)
+
+#define BUS_WAKE(bus) \
+	do { \
+		bus->idlecount = 0; \
+		if ((bus)->sleeping) \
+			dhdsdio_bussleep((bus), FALSE); \
+	} while (0);
+
+/*
+ * pktavail interrupts from dongle to host can be managed in 3 different ways
+ * whenever there is a packet available in dongle to transmit to host.
+ *
+ * Mode 0:	Dongle writes the software host mailbox and host is interrupted.
+ * Mode 1:	(sdiod core rev >= 4)
+ *		Device sets a new bit in the intstatus whenever there is a packet
+ *		available in fifo.  Host can't clear this specific status bit until all the
+ *		packets are read from the FIFO.  No need to ack dongle intstatus.
+ * Mode 2:	(sdiod core rev >= 4)
+ *		Device sets a bit in the intstatus, and host acks this by writing
+ *		one to this bit.  Dongle won't generate anymore packet interrupts
+ *		until host reads all the packets from the dongle and reads a zero to
+ *		figure that there are no more packets.  No need to disable host ints.
+ *		Need to ack the intstatus.
+ */
+
+#define SDIO_DEVICE_HMB_RXINT		0	/* default old way */
+#define SDIO_DEVICE_RXDATAINT_MODE_0	1	/* from sdiod rev 4 */
+#define SDIO_DEVICE_RXDATAINT_MODE_1	2	/* from sdiod rev 4 */
+
+#ifdef BCMSPI
+
+#define FRAME_AVAIL_MASK(bus) I_HMB_FRAME_IND
+
+#define DHD_BUS			SPI_BUS
+
+/* check packet-available-interrupt in piggybacked dstatus */
+#define PKT_AVAILABLE(bus, intstatus)	(bcmsdh_get_dstatus(bus->sdh) & STATUS_F2_PKT_AVAILABLE)
+
+#define HOSTINTMASK		(I_HMB_FC_CHANGE | I_HMB_HOST_INT)
+
+#define GSPI_PR55150_BAILOUT									\
+do {												\
+	uint32 dstatussw = bcmsdh_get_dstatus((void *)bus->sdh);				\
+	uint32 dstatushw = bcmsdh_cfg_read_word(bus->sdh, SDIO_FUNC_0, SPID_STATUS_REG, NULL);	\
+	uint32 intstatuserr = 0;								\
+	uint retries = 0;									\
+												\
+	R_SDREG(intstatuserr, &bus->regs->intstatus, retries);					\
+	printf("dstatussw = 0x%x, dstatushw = 0x%x, intstatus = 0x%x\n",			\
+	        dstatussw, dstatushw, intstatuserr); 						\
+												\
+	bus->nextlen = 0;									\
+	*finished = TRUE;									\
+} while (0)
+
+#else /* BCMSDIO */
+
+#define FRAME_AVAIL_MASK(bus) 	\
+	((bus->rxint_mode == SDIO_DEVICE_HMB_RXINT) ? I_HMB_FRAME_IND : I_XMTDATA_AVAIL)
+
+#define DHD_BUS			SDIO_BUS
+
+#define PKT_AVAILABLE(bus, intstatus)	((intstatus) & (FRAME_AVAIL_MASK(bus)))
+
+#define HOSTINTMASK		(I_HMB_SW_MASK | I_CHIPACTIVE)
+
+#define GSPI_PR55150_BAILOUT
+
+#endif /* BCMSPI */
+
+#ifdef SDTEST
+static void dhdsdio_testrcv(dhd_bus_t *bus, void *pkt, uint seq);
+static void dhdsdio_sdtest_set(dhd_bus_t *bus, uint count);
+#endif
+
+static int dhdsdio_checkdied(dhd_bus_t *bus, char *data, uint size);
+#ifdef DHD_DEBUG
+static int dhd_serialconsole(dhd_bus_t *bus, bool get, bool enable, int *bcmerror);
+#endif /* DHD_DEBUG */
+
+#if defined(DHD_FW_COREDUMP)
+static int dhdsdio_mem_dump(dhd_bus_t *bus);
+static int dhdsdio_get_mem_dump(dhd_bus_t *bus);
+#endif /* DHD_FW_COREDUMP */
+static int dhdsdio_devcap_set(dhd_bus_t *bus, uint8 cap);
+static int dhdsdio_download_state(dhd_bus_t *bus, bool enter);
+
+static void dhdsdio_release(dhd_bus_t *bus, osl_t *osh);
+static void dhdsdio_release_malloc(dhd_bus_t *bus, osl_t *osh);
+static void dhdsdio_disconnect(void *ptr);
+static bool dhdsdio_chipmatch(uint16 chipid);
+static bool dhdsdio_probe_attach(dhd_bus_t *bus, osl_t *osh, void *sdh,
+                                 void * regsva, uint16  devid);
+static bool dhdsdio_probe_malloc(dhd_bus_t *bus, osl_t *osh, void *sdh);
+static bool dhdsdio_probe_init(dhd_bus_t *bus, osl_t *osh, void *sdh);
+static void dhdsdio_release_dongle(dhd_bus_t *bus, osl_t *osh, bool dongle_isolation,
+	bool reset_flag);
+
+static void dhd_dongle_setramsize(struct dhd_bus *bus, int mem_size);
+static int dhd_bcmsdh_recv_buf(dhd_bus_t *bus, uint32 addr, uint fn, uint flags,
+	uint8 *buf, uint nbytes,
+	void *pkt, bcmsdh_cmplt_fn_t complete, void *handle);
+static int dhd_bcmsdh_send_buf(dhd_bus_t *bus, uint32 addr, uint fn, uint flags,
+	uint8 *buf, uint nbytes,
+	void *pkt, bcmsdh_cmplt_fn_t complete, void *handle, int max_retry);
+static int dhdsdio_txpkt(dhd_bus_t *bus, uint chan, void** pkts, int num_pkt, bool free_pkt);
+static int dhdsdio_txpkt_preprocess(dhd_bus_t *bus, void *pkt, int chan, int txseq,
+	int prev_chain_total_len, bool last_chained_pkt,
+	int *pad_pkt_len, void **new_pkt
+#if defined(BCMSDIOH_TXGLOM_EXT)
+	, int first_frame
+#endif
+);
+static int dhdsdio_txpkt_postprocess(dhd_bus_t *bus, void *pkt);
+
+static int dhdsdio_download_firmware(dhd_bus_t *bus, osl_t *osh, void *sdh);
+static int _dhdsdio_download_firmware(dhd_bus_t *bus);
+
+#if defined(FW_SIGNATURE)
+static int dhdsdio_bus_download_fw_signature(dhd_bus_t *bus, bool *do_write);
+static int dhdsdio_bus_write_fw_signature(dhd_bus_t *bus);
+static int dhdsdio_bus_download_ram_bootloader(dhd_bus_t *bus);
+static int dhdsdio_bus_write_fws_status(dhd_bus_t *bus);
+static int dhdsdio_bus_write_fws_mem_info(dhd_bus_t *bus);
+static int dhdsdio_bus_write_fwsig(dhd_bus_t *bus, char *fwsig_path, char *nvsig_path);
+static int dhdsdio_download_rtlv_end(dhd_bus_t *bus);
+static int dhdsdio_bus_save_download_info(dhd_bus_t *bus, uint32 download_addr,
+	uint32 download_size, const char *signature_fname,
+	const char *bloader_fname, uint32 bloader_download_addr);
+static int dhdsdio_read_fwstatus(dhd_bus_t *bus, bl_verif_status_t *status);
+static int dhd_bus_dump_fws(dhd_bus_t *bus, struct bcmstrbuf *strbuf);
+#endif /* FW_SIGNATURE */
+
+#ifdef DHD_UCODE_DOWNLOAD
+static int dhdsdio_download_ucode_file(struct dhd_bus *bus, char *ucode_path);
+#endif /* DHD_UCODE_DOWNLOAD */
+static int dhdsdio_download_code_file(dhd_bus_t *bus, char *image_path);
+static int dhdsdio_download_nvram(dhd_bus_t *bus);
+#ifdef BCMEMBEDIMAGE
+static int dhdsdio_download_code_array(dhd_bus_t *bus);
+#endif
+static int dhdsdio_bussleep(dhd_bus_t *bus, bool sleep);
+static int dhdsdio_clkctl(dhd_bus_t *bus, uint target, bool pendok);
+static uint8 dhdsdio_sleepcsr_get(dhd_bus_t *bus);
+static bool dhdsdio_dpc(dhd_bus_t *bus);
+static int dhd_bcmsdh_send_buffer(void *bus, uint8 *frame, uint16 len);
+static int dhdsdio_set_sdmode(dhd_bus_t *bus, int32 sd_mode);
+static int dhdsdio_sdclk(dhd_bus_t *bus, bool on);
+static void dhdsdio_advertise_bus_cleanup(dhd_pub_t *dhdp);
+static void dhdsdio_advertise_bus_remove(dhd_pub_t *dhdp);
+
+#if defined(BT_OVER_SDIO)
+static int extract_hex_field(char * line, uint16 start_pos, uint16 num_chars, uint16 * value);
+static int read_more_btbytes(struct dhd_bus *bus, void * file, char *line, int * addr_mode,
+	uint16 * hi_addr, uint32 * dest_addr, uint8 *data_bytes, uint32 * num_bytes);
+static int dhdsdio_download_btfw(struct dhd_bus *bus, osl_t *osh, void *sdh);
+static int _dhdsdio_download_btfw(struct dhd_bus *bus);
+#endif /* defined (BT_OVER_SDIO) */
+
+/*
+ * PR 114233: [4335] Sdio 3.0 overflow due to spur mode PLL change
+ */
+static void
+dhdsdio_tune_fifoparam(struct dhd_bus *bus)
+{
+	int err;
+	uint8 devctl, wm, mes;
+
+	if (bus->sih->buscorerev >= 15) {
+		/* See .ppt in PR for these recommended values */
+		if (bus->blocksize == 512) {
+			wm = OVERFLOW_BLKSZ512_WM;
+			mes = OVERFLOW_BLKSZ512_MES;
+		} else {
+			mes = bus->blocksize/4;
+			wm = bus->blocksize/4;
+		}
+
+		/* XXX: Need to set watermark since SBSDIO_WATERMARK could be overwritten
+		 based on watermark value in other place. Refer to SWDHD-17.
+		*/
+		watermark = wm;
+		mesbusyctrl = mes;
+	} else {
+		DHD_INFO(("skip fifotune: SdioRev(%d) is lower than minimal requested ver\n",
+			bus->sih->buscorerev));
+		return;
+	}
+
+	/* Update watermark */
+	if (wm > 0) {
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_WATERMARK, wm, &err);
+
+		devctl = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err);
+		devctl |= SBSDIO_DEVCTL_F2WM_ENAB;
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, devctl, &err);
+	}
+
+	/* Update MES */
+	if (mes > 0) {
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_MESBUSYCTRL,
+			(mes | SBSDIO_MESBUSYCTRL_ENAB), &err);
+	}
+
+	DHD_INFO(("Apply overflow WAR: 0x%02x 0x%02x 0x%02x\n",
+		bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err),
+		bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_WATERMARK, &err),
+		bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_MESBUSYCTRL, &err)));
+}
+
+static void
+dhd_dongle_setramsize(struct dhd_bus *bus, int mem_size)
+{
+	int32 min_size =  DONGLE_MIN_RAMSIZE;
+	/* Restrict the ramsize to user specified limit */
+	DHD_ERROR(("user: Restrict the dongle ram size to %d, min accepted %d\n",
+		dhd_dongle_ramsize, min_size));
+	if ((dhd_dongle_ramsize > min_size) &&
+		(dhd_dongle_ramsize < (int32)bus->orig_ramsize))
+		bus->ramsize = dhd_dongle_ramsize;
+}
+
+static int
+dhdsdio_set_siaddr_window(dhd_bus_t *bus, uint32 address)
+{
+	int err = 0;
+	bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SBADDRLOW,
+	                 (address >> 8) & SBSDIO_SBADDRLOW_MASK, &err);
+	if (!err)
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SBADDRMID,
+		                 (address >> 16) & SBSDIO_SBADDRMID_MASK, &err);
+	if (!err)
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SBADDRHIGH,
+		                 (address >> 24) & SBSDIO_SBADDRHIGH_MASK, &err);
+	return err;
+}
+
+#ifdef BCMSPI
+static void
+dhdsdio_wkwlan(dhd_bus_t *bus, bool on)
+{
+	int err;
+	uint32 regdata;
+	bcmsdh_info_t *sdh = bus->sdh;
+
+	/* XXX: sdiod cores have SPI as a block, PCMCIA doesn't have the gspi core */
+	/* XXX: may be we don't even need this check at all */
+	if (bus->sih->buscoretype == SDIOD_CORE_ID) {
+		/* wake up wlan function :WAKE_UP goes as ht_avail_request and alp_avail_request */
+		regdata = bcmsdh_cfg_read_word(sdh, SDIO_FUNC_0, SPID_CONFIG, NULL);
+		DHD_INFO(("F0 REG0 rd = 0x%x\n", regdata));
+
+		if (on == TRUE)
+			regdata |= WAKE_UP;
+		else
+			regdata &= ~WAKE_UP;
+
+		bcmsdh_cfg_write_word(sdh, SDIO_FUNC_0, SPID_CONFIG, regdata, &err);
+	}
+}
+#endif /* BCMSPI */
+
+#ifdef USE_OOB_GPIO1
+static int
+dhdsdio_oobwakeup_init(dhd_bus_t *bus)
+{
+	uint32 val, addr, data;
+
+	bcmsdh_gpioouten(bus->sdh, GPIO_DEV_WAKEUP);
+
+	addr = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, chipcontrol_addr);
+	data = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, chipcontrol_data);
+
+	/* Set device for gpio1 wakeup */
+	bcmsdh_reg_write(bus->sdh, addr, 4, 2);
+	val = bcmsdh_reg_read(bus->sdh, data, 4);
+	val |= CC_CHIPCTRL2_GPIO1_WAKEUP;
+	bcmsdh_reg_write(bus->sdh, data, 4, val);
+
+	bus->_oobwakeup = TRUE;
+
+	return 0;
+}
+#endif /* USE_OOB_GPIO1 */
+
+#ifndef BCMSPI
+/*
+ * Query if FW is in SR mode
+ */
+static bool
+dhdsdio_sr_cap(dhd_bus_t *bus)
+{
+	bool cap = FALSE;
+	uint32  core_capext, addr, data;
+
+	if (bus->sih->chip == BCM43430_CHIP_ID ||
+		bus->sih->chip == BCM43018_CHIP_ID) {
+		/* check if fw initialized sr engine */
+		addr = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, sr_control1);
+		if (bcmsdh_reg_read(bus->sdh, addr, 4) != 0)
+			cap = TRUE;
+
+		return cap;
+	}
+	if (
+		0) {
+			core_capext = FALSE;
+	} else if ((bus->sih->chip == BCM4330_CHIP_ID) ||
+		(bus->sih->chip == BCM43362_CHIP_ID) ||
+		(BCM4347_CHIP(bus->sih->chip))) {
+			core_capext = FALSE;
+	} else if ((bus->sih->chip == BCM4335_CHIP_ID) ||
+		(bus->sih->chip == BCM4339_CHIP_ID) ||
+		BCM4345_CHIP(bus->sih->chip) ||
+		(bus->sih->chip == BCM4354_CHIP_ID) ||
+		(bus->sih->chip == BCM4356_CHIP_ID) ||
+		(bus->sih->chip == BCM4358_CHIP_ID) ||
+		(bus->sih->chip == BCM43569_CHIP_ID) ||
+		(bus->sih->chip == BCM4371_CHIP_ID) ||
+		(BCM4349_CHIP(bus->sih->chip))		||
+		(bus->sih->chip == BCM4350_CHIP_ID) ||
+		(bus->sih->chip == BCM4362_CHIP_ID) ||
+		(bus->sih->chip == BCM4381_CHIP_ID) ||
+		(bus->sih->chip == BCM4382_CHIP_ID) ||
+		(bus->sih->chip == BCM43012_CHIP_ID) ||
+		(bus->sih->chip == BCM43013_CHIP_ID) ||
+		(bus->sih->chip == BCM43014_CHIP_ID) ||
+		(bus->sih->chip == BCM43751_CHIP_ID) ||
+		(bus->sih->chip == BCM43752_CHIP_ID) ||
+		(bus->sih->chip == BCM43756_CHIP_ID) ||
+		(bus->sih->chip == BCM43711_CHIP_ID)) {
+		core_capext = TRUE;
+	} else {
+		/* XXX: For AOB, CORE_CAPEXT_ADDR is moved to PMU core */
+		core_capext = bcmsdh_reg_read(bus->sdh,
+			si_get_pmu_reg_addr(bus->sih, OFFSETOF(chipcregs_t, core_cap_ext)),
+			4);
+
+		core_capext = (core_capext & CORE_CAPEXT_SR_SUPPORTED_MASK);
+	}
+	if (!(core_capext))
+		return FALSE;
+
+	if ((bus->sih->chip == BCM4335_CHIP_ID) ||
+		(bus->sih->chip == BCM4339_CHIP_ID) ||
+		BCM4345_CHIP(bus->sih->chip) ||
+		(bus->sih->chip == BCM4354_CHIP_ID) ||
+		(bus->sih->chip == BCM4356_CHIP_ID) ||
+		(bus->sih->chip == BCM4358_CHIP_ID) ||
+		(bus->sih->chip == BCM43569_CHIP_ID) ||
+		(bus->sih->chip == BCM4371_CHIP_ID) ||
+		(bus->sih->chip == BCM4350_CHIP_ID)) {
+		uint32 enabval = 0;
+		addr = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, chipcontrol_addr);
+		data = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, chipcontrol_data);
+		/* XXX: assuming the dongle doesn't change chipcontrol_addr, because
+		* if that happens, the chipcontrol_data read will be wrong. So we need
+		* to make sure the dongle and host will not access chipcontrol_addr
+		* simultaneously at this point.
+		*/
+		bcmsdh_reg_write(bus->sdh, addr, 4, CC_PMUCC3);
+		enabval = bcmsdh_reg_read(bus->sdh, data, 4);
+
+		if ((bus->sih->chip == BCM4350_CHIP_ID) ||
+			BCM4345_CHIP(bus->sih->chip) ||
+			(bus->sih->chip == BCM4354_CHIP_ID) ||
+			(bus->sih->chip == BCM4356_CHIP_ID) ||
+			(bus->sih->chip == BCM4358_CHIP_ID) ||
+			(bus->sih->chip == BCM43569_CHIP_ID) ||
+			(bus->sih->chip == BCM4371_CHIP_ID))
+			enabval &= CC_CHIPCTRL3_SR_ENG_ENABLE;
+
+		/* XXX: not checking the CC_PMUCC3_SRCC_SR_ENG_ENAB bit [val 4], but
+		* will check the whole register to be non-zero so that, sleep sequence
+		* can be also checked without enabling SR.
+		*/
+		if (enabval)
+			cap = TRUE;
+	} else {
+		data = bcmsdh_reg_read(bus->sdh,
+			si_get_pmu_reg_addr(bus->sih, OFFSETOF(chipcregs_t, retention_ctl)),
+			4);
+		if ((data & (RCTL_MACPHY_DISABLE_MASK | RCTL_LOGIC_DISABLE_MASK)) == 0)
+			cap = TRUE;
+	}
+
+	return cap;
+}
+
+static int
+dhdsdio_sr_init(dhd_bus_t *bus)
+{
+	uint8 val, wkctl;
+	int err = 0;
+
+	if (bus->sih->chip == BCM43012_CHIP_ID ||
+		bus->sih->chip == BCM43013_CHIP_ID ||
+		bus->sih->chip == BCM43014_CHIP_ID) {
+		val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL, NULL);
+		val |= 1 << SBSDIO_FUNC1_WCTRL_ALPWAIT_SHIFT;
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL,
+			1 << SBSDIO_FUNC1_WCTRL_ALPWAIT_SHIFT, &err);
+		wkctl = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL, NULL);
+	} else {
+		val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL, NULL);
+		val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT;
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL,
+			1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT, &err);
+		wkctl = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL, NULL);
+	}
+
+	if (!(wkctl & val)) {
+		DHD_ERROR(("%s:Set wakeup ctrl failed. SBSDIO_FUNC1_WAKEUPCTRL %08x, expect %08x\n",
+			__FUNCTION__, wkctl, val));
+	}
+
+#ifdef USE_CMD14
+	/* Add CMD14 Support */
+	dhdsdio_devcap_set(bus,
+		(SDIOD_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | SDIOD_CCCR_BRCM_CARDCAP_CMD14_EXT));
+#endif /* USE_CMD14 */
+
+	if (CHIPID(bus->sih->chip) == BCM43430_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43018_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM4339_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM4362_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM4381_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43012_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43013_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43014_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43751_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43752_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43756_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43711_CHIP_ID)
+			dhdsdio_devcap_set(bus, SDIOD_CCCR_BRCM_CARDCAP_CMD_NODEC);
+
+	if (bus->sih->chip == BCM43012_CHIP_ID ||
+		bus->sih->chip == BCM43013_CHIP_ID ||
+		bus->sih->chip == BCM43014_CHIP_ID) {
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1,
+			SBSDIO_FUNC1_CHIPCLKCSR, SBSDIO_HT_AVAIL_REQ, &err);
+	} else {
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1,
+			SBSDIO_FUNC1_CHIPCLKCSR, SBSDIO_FORCE_HT, &err);
+	}
+	bus->_slpauto = dhd_slpauto ? TRUE : FALSE;
+
+	bus->_srenab = TRUE;
+
+	return 0;
+}
+#endif /* BCMSPI */
+
+/*
+ * FIX: Be sure KSO bit is enabled
+ * Currently, it's defaulting to 0 which should be 1.
+ */
+static int
+dhdsdio_clk_kso_init(dhd_bus_t *bus)
+{
+	uint8 val;
+	int err = 0;
+
+	/* set flag */
+	bus->kso = TRUE;
+
+	/*
+	 * Enable KeepSdioOn (KSO) bit for normal operation
+	 * Default is 0 (4334A0) so set it. Fixed in B0.
+	 */
+	val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, NULL);
+	if (!(val & SBSDIO_FUNC1_SLEEPCSR_KSO_MASK)) {
+		val |= (SBSDIO_FUNC1_SLEEPCSR_KSO_EN << SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT);
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, val, &err);
+		if (err)
+			DHD_ERROR(("%s: SBSDIO_FUNC1_SLEEPCSR err: 0x%x\n", __FUNCTION__, err));
+	}
+
+	return 0;
+}
+
+static void
+dhdsdio_set_wakeupctrl(dhd_bus_t *bus)
+{
+	uint8 val;
+	int err = 0;
+	UNUSED_PARAMETER(val);
+
+	/* programme the wakeup wait */
+	val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL, NULL);
+	val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT;
+	bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL,
+			1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT, &err);
+	val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WAKEUPCTRL, NULL);
+}
+
+#define KSO_DBG(x)
+/* XXX KSO set typically takes depending on resource up & number of
+* resources which were down. Max value is PMU_MAX_TRANSITION_DLY usec.
+* Currently the KSO attempt loop is such that, it waits
+* (KSO_WAIT_US [50usec] time + 2 SDIO operations) * MAX_KSO_ATTEMPTS.
+* So setting a value of maximum PMU_MAX_TRANSITION_DLY as wait time.,
+* to calculate MAX_KSO_ATTEMPTS.
+*/
+#define KSO_WAIT_US 50
+#define KSO_WAIT_MS 1
+#define KSO_SLEEP_RETRY_COUNT 20
+#define KSO_WAKE_RETRY_COUNT 100
+#define ERROR_BCME_NODEVICE_MAX 1
+
+#define DEFAULT_MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US)
+#ifndef CUSTOM_MAX_KSO_ATTEMPTS
+#define CUSTOM_MAX_KSO_ATTEMPTS DEFAULT_MAX_KSO_ATTEMPTS
+#endif
+
+static int
+dhdsdio_clk_kso_enab(dhd_bus_t *bus, bool on)
+{
+	uint8 wr_val = 0, rd_val, cmp_val, bmask;
+	int err = 0;
+	int try_cnt = 0, try_max = CUSTOM_MAX_KSO_ATTEMPTS;
+	struct dhd_conf *conf = bus->dhd->conf;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)) && !defined(ANDROID13_KERNEL515_BKPORT)
+	wifi_adapter_info_t *adapter = NULL;
+	uint32 bus_type = -1, bus_num = -1, slot_num = -1;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0) */
+
+	KSO_DBG(("%s> op:%s\n", __FUNCTION__, (on ? "KSO_SET" : "KSO_CLR")));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)) && !defined(ANDROID13_KERNEL515_BKPORT)
+	dhd_bus_get_ids(bus, &bus_type, &bus_num, &slot_num);
+	adapter = dhd_wifi_platform_get_adapter(bus_type, bus_num, slot_num);
+	sdio_retune_crc_disable(adapter->sdio_func);
+	if (on)
+		sdio_retune_hold_now(adapter->sdio_func);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0) */
+
+	wr_val |= (on << SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT);
+
+	/* XXX 1st KSO write goes to AOS wake up core if device is asleep  */
+	bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err);
+
+	/* In case of 43012 chip, the chip could go down immediately after KSO bit is cleared.
+	 * So the further reads of KSO register could fail. Thereby just bailing out immediately
+	 * after clearing KSO bit, to avoid polling of KSO bit.
+	 */
+	if ((!on) && (bus->sih->chip == BCM43012_CHIP_ID ||
+		(bus->sih->chip == BCM4381_CHIP_ID) ||
+		(bus->sih->chip == BCM4382_CHIP_ID) ||
+		bus->sih->chip == BCM43013_CHIP_ID ||
+		bus->sih->chip == BCM43014_CHIP_ID)) {
+		goto exit;
+	}
+
+	if (on) {
+		/* XXX
+		 *  device WAKEUP through KSO:
+		 *  write bit 0 & read back until
+		 *  both bits 0(kso bit) & 1 (dev on status) are set
+		 */
+		cmp_val = SBSDIO_FUNC1_SLEEPCSR_KSO_MASK |  SBSDIO_FUNC1_SLEEPCSR_DEVON_MASK;
+		bmask = cmp_val;
+
+#if defined(NDIS)
+		/* XXX Windows Host controller hangs if chip still sleeps before read.
+		 * So during a wake we write 0x1 for 5 msec to guarantee that chip is a wake.
+		 */
+		for (int i = 0; i < KSO_WAKE_RETRY_COUNT; i++) {
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR,
+				wr_val, &err);
+			OSL_DELAY(KSO_WAIT_US);
+		}
+
+		rd_val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, &err);
+		if (((rd_val & bmask) != cmp_val) || err) {
+			/* Sdio Bus Failure - Bus hang */
+			DHD_ERROR(("%s> op:%s, ERROR: SDIO Bus Hang, rd_val:%x, ERR:%x \n",
+				__FUNCTION__, "KSO_SET", rd_val, err));
+		}
+#else
+		OSL_SLEEP(3);
+#endif /* defined(NDIS) */
+
+	} else {
+		/* Put device to sleep, turn off  KSO  */
+		cmp_val = 0;
+		/* XXX only check for bit0, bit1(devon status) may not get cleared right away */
+		bmask = SBSDIO_FUNC1_SLEEPCSR_KSO_MASK;
+	}
+#if !defined(NDIS)
+	/* XXX We can't use polling in Windows since Windows Host controller
+	 * hangs if chip is a sleep during read or write.
+	 */
+
+	if (conf->kso_try_max)
+		try_max = conf->kso_try_max;
+	do {
+		/*
+		 *  XXX reliable KSO bit set/clr:
+		 *  the sdiod sleep write access appears to be is synced to PMU 32khz clk
+		 *  just one write attempt may fail,(same is with read ?)
+		 *  in any case, read it back until it matches written value
+		 */
+		rd_val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, &err);
+		if (((rd_val & bmask) == cmp_val) && !err)
+			break;
+
+		KSO_DBG(("%s> KSO wr/rd retry:%d, ERR:%x \n", __FUNCTION__, try_cnt, err));
+
+		if (((try_cnt + 1) % KSO_SLEEP_RETRY_COUNT) == 0) {
+			OSL_SLEEP(KSO_WAIT_MS);
+		} else
+			OSL_DELAY(KSO_WAIT_US);
+
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err);
+	} while (try_cnt++ < try_max);
+
+#ifdef KSO_DEBUG
+	if (try_cnt > 0 && try_cnt <= 10)
+		conf->kso_try_array[0] += 1;
+	else if (try_cnt <= 50)
+		conf->kso_try_array[1] += 1;
+	else if (try_cnt <= 100)
+		conf->kso_try_array[2] += 1;
+	else if (try_cnt <= 200)
+		conf->kso_try_array[3] += 1;
+	else if (try_cnt <= 500)
+		conf->kso_try_array[4] += 1;
+	else if (try_cnt <= 1000)
+		conf->kso_try_array[5] += 1;
+	else if (try_cnt <= 2000)
+		conf->kso_try_array[6] += 1;
+	else if (try_cnt <= 5000)
+		conf->kso_try_array[7] += 1;
+	else if (try_cnt <= 10000)
+		conf->kso_try_array[8] += 1;
+	else
+		conf->kso_try_array[9] += 1;
+#endif
+	if (try_cnt > 2)
+		KSO_DBG(("%s> op:%s, try_cnt:%d, rd_val:%x, ERR:%x \n",
+			__FUNCTION__, (on ? "KSO_SET" : "KSO_CLR"), try_cnt, rd_val, err));
+
+	if (try_cnt > try_max)  {
+		DHD_ERROR(("%s> op:%s, ERROR: try_cnt:%d, rd_val:%x, ERR:%x \n",
+			__FUNCTION__, (on ? "KSO_SET" : "KSO_CLR"), try_cnt, rd_val, err));
+#ifdef KSO_DEBUG
+		{
+			int i;
+			printk(KERN_CONT DHD_LOG_PREFIXS);
+			for (i=0; i<10; i++) {
+				printk(KERN_CONT "[%d]: %d, ", i, conf->kso_try_array[i]);
+		 	}
+			printk("\n");
+		}
+#endif
+	}
+#endif /* !defined(NDIS) */
+
+exit:
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)) && !defined(ANDROID13_KERNEL515_BKPORT)
+	if (on)
+		sdio_retune_release(adapter->sdio_func);
+	sdio_retune_crc_enable(adapter->sdio_func);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0) */
+
+	return err;
+}
+
+static int
+dhdsdio_clk_kso_iovar(dhd_bus_t *bus, bool on)
+{
+	int err = 0;
+
+	if (on == FALSE) {
+
+		BUS_WAKE(bus);
+		dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+		DHD_ERROR(("%s: KSO disable clk: 0x%x\n", __FUNCTION__,
+			bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+			SBSDIO_FUNC1_CHIPCLKCSR, &err)));
+		dhdsdio_clk_kso_enab(bus, FALSE);
+	} else {
+		DHD_ERROR(("%s: KSO enable\n", __FUNCTION__));
+
+		/* Make sure we have SD bus access */
+		if (bus->clkstate == CLK_NONE) {
+			DHD_ERROR(("%s: Request SD clk\n", __FUNCTION__));
+			dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+		}
+
+		dhdsdio_clk_kso_enab(bus, TRUE);
+
+		DHD_ERROR(("%s: sleepcsr: 0x%x\n", __FUNCTION__,
+			dhdsdio_sleepcsr_get(bus)));
+	}
+
+	bus->kso = on;
+	BCM_REFERENCE(err);
+
+	return 0;
+}
+
+static uint8
+dhdsdio_sleepcsr_get(dhd_bus_t *bus)
+{
+	int err = 0;
+	uint8 val = 0;
+
+	val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_SLEEPCSR, &err);
+	/* XXX: Propagate error */
+	if (err)
+		DHD_TRACE(("Failed to read SLEEPCSR: %d\n", err));
+
+	return val;
+}
+
+uint8
+dhdsdio_devcap_get(dhd_bus_t *bus)
+{
+	return bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_BRCM_CARDCAP, NULL);
+}
+
+static int
+dhdsdio_devcap_set(dhd_bus_t *bus, uint8 cap)
+{
+	int err = 0;
+
+	bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_BRCM_CARDCAP, cap, &err);
+	if (err)
+		DHD_ERROR(("%s: devcap set err: 0x%x\n", __FUNCTION__, err));
+
+	return 0;
+}
+
+static int
+dhdsdio_clk_devsleep_iovar(dhd_bus_t *bus, bool on)
+{
+	int err = 0, retry;
+	uint8 val;
+
+	retry = 0;
+	if (on == TRUE) {
+		/* Enter Sleep */
+
+		/* Be sure we request clk before going to sleep
+		 * so we can wake-up with clk request already set
+		 * else device can go back to sleep immediately
+		 */
+		if (!SLPAUTO_ENAB(bus))
+			dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+		else {
+			/* XXX: Check if Host cleared clock request
+			 * XXX: With CMD14, Host does not need to explicitly toggle clock requests
+			 * XXX: Just keep clock request active and use CMD14 to enter/exit sleep
+			 */
+			val = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+			if ((val & SBSDIO_CSR_MASK) == 0) {
+				DHD_ERROR(("%s: No clock before enter sleep:0x%x\n",
+					__FUNCTION__, val));
+
+				/* Reset clock request */
+				bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR,
+					SBSDIO_ALP_AVAIL_REQ, &err);
+				DHD_ERROR(("%s: clock before sleep:0x%x\n", __FUNCTION__,
+					bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+					SBSDIO_FUNC1_CHIPCLKCSR, &err)));
+			}
+		}
+
+		DHD_TRACE(("%s: clk before sleep: 0x%x\n", __FUNCTION__,
+			bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+			SBSDIO_FUNC1_CHIPCLKCSR, &err)));
+#ifdef USE_CMD14
+		err = bcmsdh_sleep(bus->sdh, TRUE);
+#else
+		if ((SLPAUTO_ENAB(bus)) && (bus->idleclock == DHD_IDLE_STOP)) {
+			if (sd1idle) {
+				/* Change to SD1 mode */
+				dhdsdio_set_sdmode(bus, 1);
+			}
+		}
+
+		err = dhdsdio_clk_kso_enab(bus, FALSE);
+		if (OOB_WAKEUP_ENAB(bus))
+		{
+#if !defined(NDIS)
+			err = bcmsdh_gpioout(bus->sdh, GPIO_DEV_WAKEUP, FALSE);  /* GPIO_1 is off */
+#endif /* !defined(NDIS) */
+		}
+#endif /* USE_CMD14 */
+
+		if ((SLPAUTO_ENAB(bus)) && (bus->idleclock != DHD_IDLE_ACTIVE)) {
+			DHD_TRACE(("%s: Turnoff SD clk\n", __FUNCTION__));
+			/* Now remove the SD clock */
+			err = dhdsdio_sdclk(bus, FALSE);
+		}
+	} else {
+		/* Exit Sleep */
+		/* Make sure we have SD bus access */
+		if (bus->clkstate == CLK_NONE) {
+			DHD_TRACE(("%s: Request SD clk\n", __FUNCTION__));
+			dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+		}
+#ifdef USE_CMD14
+		err = bcmsdh_sleep(bus->sdh, FALSE);
+		if (SLPAUTO_ENAB(bus) && (err != 0)) {
+			/* XXX: CMD14 exit sleep is failing somehow
+			 * XXX: Is Host out of sync with device?
+			 * XXX: Try toggling the reverse
+			 */
+			OSL_DELAY(10000);
+			DHD_TRACE(("%s: Resync device sleep\n", __FUNCTION__));
+
+			/* Toggle sleep to resync with host and device */
+			err = bcmsdh_sleep(bus->sdh, TRUE);
+			OSL_DELAY(10000);
+			err = bcmsdh_sleep(bus->sdh, FALSE);
+
+			/* XXX: Ugly hack for host-device out-of-sync while testing
+			 * XXX: Need to root-cause
+			 */
+			if (err) {
+				/* XXX: Host and device out-of-sync */
+				OSL_DELAY(10000);
+				DHD_ERROR(("%s: CMD14 exit failed again!\n", __FUNCTION__));
+
+				/* Toggle sleep to resync with host and device */
+				err = bcmsdh_sleep(bus->sdh, TRUE);
+				OSL_DELAY(10000);
+				err = bcmsdh_sleep(bus->sdh, FALSE);
+				if (err) {
+					/* XXX: Give up and assumed it has exited sleep
+					 * XXX: Device probably dead at this point
+					 * XXX: So far only happens with SR
+					 */
+					DHD_ERROR(("%s: CMD14 exit failed twice!\n", __FUNCTION__));
+					DHD_ERROR(("%s: FATAL: Device non-response!\n",
+						__FUNCTION__));
+					err = 0;
+				}
+			}
+		}
+#else
+		if (OOB_WAKEUP_ENAB(bus))
+		{
+#if !defined(NDIS)
+			err = bcmsdh_gpioout(bus->sdh, GPIO_DEV_WAKEUP, TRUE);  /* GPIO_1 is on */
+#endif /* !defined(NDIS) */
+		}
+		/* PR 101351: sdiod_aos sleep followed by immediate wakeup
+		 *    before sdiod_aos takes over has a problem.
+		 * Set KSO after ExitSleep.
+		 */
+		do {
+			err = dhdsdio_clk_kso_enab(bus, TRUE);
+			if (err)
+				OSL_SLEEP(10);
+		} while ((err != 0) && (++retry < 3));
+
+		if (err != 0) {
+			DHD_ERROR(("ERROR: kso set failed retry: %d\n", retry));
+#ifndef BT_OVER_SDIO
+			err = 0; /* continue anyway */
+#endif /* BT_OVER_SDIO */
+		}
+
+		if ((SLPAUTO_ENAB(bus)) && (bus->idleclock == DHD_IDLE_STOP)) {
+			dhdsdio_set_sdmode(bus, bus->sd_mode);
+		}
+#endif /* !USE_CMD14 */
+
+		if (err == 0) {
+			uint8 csr;
+
+			/* Wait for device ready during transition to wake-up */
+			SPINWAIT_SLEEP(sdioh_spinwait_sleep,
+				(((csr = dhdsdio_sleepcsr_get(bus)) &
+				SBSDIO_FUNC1_SLEEPCSR_DEVON_MASK) !=
+				(SBSDIO_FUNC1_SLEEPCSR_DEVON_MASK)), (20000));
+
+			DHD_TRACE(("%s: ExitSleep sleepcsr: 0x%x\n", __FUNCTION__, csr));
+
+			if (!(csr & SBSDIO_FUNC1_SLEEPCSR_DEVON_MASK)) {
+				DHD_ERROR(("%s:ERROR: ExitSleep device NOT Ready! 0x%x\n",
+					__FUNCTION__, csr));
+				err = BCME_NODEVICE;
+			}
+
+			/* PR 101351: sdiod_aos sleep followed by immediate wakeup
+			 *    before sdiod_aos takes over has a problem.
+			 */
+			SPINWAIT_SLEEP(sdioh_spinwait_sleep,
+				(((csr = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+				SBSDIO_FUNC1_CHIPCLKCSR, &err)) & SBSDIO_HT_AVAIL) !=
+				(SBSDIO_HT_AVAIL)), (DHD_WAIT_HTAVAIL));
+
+			DHD_TRACE(("%s: SBSDIO_FUNC1_CHIPCLKCSR : 0x%x\n", __FUNCTION__, csr));
+			if (!err && ((csr & SBSDIO_HT_AVAIL) != SBSDIO_HT_AVAIL)) {
+				DHD_ERROR(("%s:ERROR: device NOT Ready! 0x%x\n",
+					__FUNCTION__, csr));
+				err = BCME_NODEVICE;
+			}
+		}
+	}
+
+	/* Update if successful */
+	if (err == 0)
+		bus->kso = on ? FALSE : TRUE;
+	else {
+		DHD_ERROR(("%s: Sleep request failed: kso:%d on:%d err:%d\n",
+			__FUNCTION__, bus->kso, on, err));
+		if (!on && retry > 2)
+			bus->kso = FALSE;
+	}
+
+	return err;
+}
+
+/* Turn backplane clock on or off */
+static int
+dhdsdio_htclk(dhd_bus_t *bus, bool on, bool pendok)
+{
+#define HT_AVAIL_ERROR_MAX 10
+	static int ht_avail_error = 0;
+	int err;
+	uint8 clkctl, clkreq, devctl;
+	bcmsdh_info_t *sdh;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	clkctl = 0;
+	sdh = bus->sdh;
+
+	if (!KSO_ENAB(bus))
+		return BCME_OK;
+
+	if (SLPAUTO_ENAB(bus)) {
+		bus->clkstate = (on ? CLK_AVAIL : CLK_SDONLY);
+		return BCME_OK;
+	}
+
+	if (on) {
+		/* Request HT Avail */
+		clkreq = bus->alp_only ? SBSDIO_ALP_AVAIL_REQ : SBSDIO_HT_AVAIL_REQ;
+
+#ifdef BCMSPI
+		dhdsdio_wkwlan(bus, TRUE);
+#endif /* BCMSPI */
+
+		/* XXX Should be able to early-exit if pendok && PENDING */
+
+		bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, clkreq, &err);
+		if (err) {
+			ht_avail_error++;
+			if (ht_avail_error < HT_AVAIL_ERROR_MAX) {
+				DHD_ERROR(("%s: HT Avail request error: %d\n", __FUNCTION__, err));
+			}
+
+#ifdef OEM_ANDROID
+			else if (ht_avail_error == HT_AVAIL_ERROR_MAX) {
+				bus->dhd->hang_reason = HANG_REASON_HT_AVAIL_ERROR;
+				dhd_os_send_hang_message(bus->dhd);
+			}
+#endif /* OEM_ANDROID */
+			return BCME_ERROR;
+		} else {
+			ht_avail_error = 0;
+		}
+
+		/* Check current status */
+		clkctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+		if (err) {
+			DHD_ERROR(("%s: HT Avail read error: %d\n", __FUNCTION__, err));
+			return BCME_ERROR;
+		}
+
+#if !defined(OOB_INTR_ONLY)
+		/* Go to pending and await interrupt if appropriate */
+		if (!SBSDIO_CLKAV(clkctl, bus->alp_only) && pendok) {
+			/* Allow only clock-available interrupt */
+			devctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err);
+			if (err) {
+				DHD_ERROR(("%s: Devctl access error setting CA: %d\n",
+				           __FUNCTION__, err));
+				return BCME_ERROR;
+			}
+
+			devctl |= SBSDIO_DEVCTL_CA_INT_ONLY;
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, devctl, &err);
+			DHD_INFO(("CLKCTL: set PENDING\n"));
+			bus->clkstate = CLK_PENDING;
+			return BCME_OK;
+		} else
+#endif /* !defined (OOB_INTR_ONLY) */
+		{
+			if (bus->clkstate == CLK_PENDING) {
+				/* Cancel CA-only interrupt filter */
+				devctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err);
+				devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
+				bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, devctl, &err);
+			}
+		}
+#ifndef BCMSDIOLITE
+		/* Otherwise, wait here (polling) for HT Avail */
+		if (!SBSDIO_CLKAV(clkctl, bus->alp_only)) {
+			SPINWAIT_SLEEP(sdioh_spinwait_sleep,
+				((clkctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1,
+			                                    SBSDIO_FUNC1_CHIPCLKCSR, &err)),
+			          !SBSDIO_CLKAV(clkctl, bus->alp_only)), PMU_MAX_TRANSITION_DLY);
+		}
+		if (err) {
+			DHD_ERROR(("%s: HT Avail request error: %d\n", __FUNCTION__, err));
+			return BCME_ERROR;
+		}
+		if (!SBSDIO_CLKAV(clkctl, bus->alp_only)) {
+			DHD_ERROR(("%s: HT Avail timeout (%d): clkctl 0x%02x\n",
+			           __FUNCTION__, PMU_MAX_TRANSITION_DLY, clkctl));
+			return BCME_ERROR;
+		}
+#endif /* BCMSDIOLITE */
+		/* Mark clock available */
+		bus->clkstate = CLK_AVAIL;
+		DHD_INFO(("CLKCTL: turned ON\n"));
+
+#if defined(DHD_DEBUG)
+		if (bus->alp_only == TRUE) {
+#if !defined(BCMLXSDMMC)
+			/* XXX For the SDMMC Driver stack, if DHD was unloaded,
+			 * the chip is not completely reset, so in this case,
+			 * the PMU may already be programmed to allow HT clock.
+			 */
+			if (!SBSDIO_ALPONLY(clkctl)) {
+				DHD_ERROR(("%s: HT Clock, when ALP Only\n", __FUNCTION__));
+			}
+#endif /* !defined(BCMLXSDMMC) */
+		} else {
+			if (SBSDIO_ALPONLY(clkctl)) {
+				DHD_ERROR(("%s: HT Clock should be on.\n", __FUNCTION__));
+			}
+		}
+#endif /* defined (DHD_DEBUG) */
+
+		bus->activity = TRUE;
+#ifdef DHD_USE_IDLECOUNT
+		bus->idlecount = 0;
+#endif /* DHD_USE_IDLECOUNT */
+	} else {
+		clkreq = 0;
+
+		if (bus->clkstate == CLK_PENDING) {
+			/* Cancel CA-only interrupt filter */
+			devctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err);
+			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, devctl, &err);
+		}
+
+		bus->clkstate = CLK_SDONLY;
+		if (!SR_ENAB(bus)) {
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, clkreq, &err);
+			DHD_INFO(("CLKCTL: turned OFF\n"));
+			if (err) {
+				DHD_ERROR(("%s: Failed access turning clock off: %d\n",
+				           __FUNCTION__, err));
+				return BCME_ERROR;
+			}
+		}
+#ifdef BCMSPI
+			dhdsdio_wkwlan(bus, FALSE);
+#endif /* BCMSPI */
+	}
+	return BCME_OK;
+}
+
+/* Change SD1/SD4 bus mode */
+static int
+dhdsdio_set_sdmode(dhd_bus_t *bus, int32 sd_mode)
+{
+	int err;
+
+	err = bcmsdh_iovar_op(bus->sdh, "sd_mode", NULL, 0,
+		&sd_mode, sizeof(sd_mode), TRUE);
+	if (err) {
+		DHD_ERROR(("%s: error changing sd_mode: %d\n",
+			__FUNCTION__, err));
+		return BCME_ERROR;
+	}
+	return BCME_OK;
+}
+
+/* Change idle/active SD state */
+static int
+dhdsdio_sdclk(dhd_bus_t *bus, bool on)
+{
+#ifndef BCMSPI
+	int err;
+	int32 iovalue;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (on) {
+		if (bus->idleclock == DHD_IDLE_STOP) {
+			/* Turn on clock and restore mode */
+			iovalue = 1;
+			err = bcmsdh_iovar_op(bus->sdh, "sd_clock", NULL, 0,
+			                      &iovalue, sizeof(iovalue), TRUE);
+			if (err) {
+				DHD_ERROR(("%s: error enabling sd_clock: %d\n",
+				           __FUNCTION__, err));
+				return BCME_ERROR;
+			}
+
+		} else if (bus->idleclock != DHD_IDLE_ACTIVE) {
+			/* Restore clock speed */
+			iovalue = bus->sd_divisor;
+			err = bcmsdh_iovar_op(bus->sdh, "sd_divisor", NULL, 0,
+			                      &iovalue, sizeof(iovalue), TRUE);
+			if (err) {
+				DHD_ERROR(("%s: error restoring sd_divisor: %d\n",
+				           __FUNCTION__, err));
+				return BCME_ERROR;
+			}
+		}
+		bus->clkstate = CLK_SDONLY;
+	} else {
+		/* Stop or slow the SD clock itself */
+		if ((bus->sd_divisor == -1) || (bus->sd_mode == -1)) {
+			DHD_TRACE(("%s: can't idle clock, divisor %d mode %d\n",
+			           __FUNCTION__, bus->sd_divisor, bus->sd_mode));
+			return BCME_ERROR;
+		}
+		if (bus->idleclock == DHD_IDLE_STOP) {
+			iovalue = 0;
+			err = bcmsdh_iovar_op(bus->sdh, "sd_clock", NULL, 0,
+			                      &iovalue, sizeof(iovalue), TRUE);
+			if (err) {
+				DHD_ERROR(("%s: error disabling sd_clock: %d\n",
+				           __FUNCTION__, err));
+				return BCME_ERROR;
+			}
+		} else if (bus->idleclock != DHD_IDLE_ACTIVE) {
+			/* Set divisor to idle value */
+			iovalue = bus->idleclock;
+			err = bcmsdh_iovar_op(bus->sdh, "sd_divisor", NULL, 0,
+			                      &iovalue, sizeof(iovalue), TRUE);
+			if (err) {
+				DHD_ERROR(("%s: error changing sd_divisor: %d\n",
+				           __FUNCTION__, err));
+				return BCME_ERROR;
+			}
+		}
+		bus->clkstate = CLK_NONE;
+	}
+#endif /* BCMSPI */
+
+	return BCME_OK;
+}
+
+/* Transition SD and backplane clock readiness */
+static int
+dhdsdio_clkctl(dhd_bus_t *bus, uint target, bool pendok)
+{
+	int ret = BCME_OK;
+#ifdef DHD_DEBUG
+	uint oldstate = bus->clkstate;
+#endif /* DHD_DEBUG */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	/* Early exit if we're already there */
+	if (bus->clkstate == target) {
+		if (target == CLK_AVAIL) {
+			dhd_os_wd_timer(bus->dhd, dhd_watchdog_ms);
+			bus->activity = TRUE;
+#ifdef DHD_USE_IDLECOUNT
+			bus->idlecount = 0;
+#endif /* DHD_USE_IDLECOUNT */
+		}
+		return ret;
+	}
+
+	switch (target) {
+	case CLK_AVAIL:
+		/* Make sure SD clock is available */
+		if (bus->clkstate == CLK_NONE)
+			dhdsdio_sdclk(bus, TRUE);
+		/* Now request HT Avail on the backplane */
+		ret = dhdsdio_htclk(bus, TRUE, pendok);
+		if (ret == BCME_OK) {
+			dhd_os_wd_timer(bus->dhd, dhd_watchdog_ms);
+		bus->activity = TRUE;
+#ifdef DHD_USE_IDLECOUNT
+			bus->idlecount = 0;
+#endif /* DHD_USE_IDLECOUNT */
+		}
+		break;
+
+	case CLK_SDONLY:
+
+#ifdef BT_OVER_SDIO
+		/*
+		 * If the request is to switch off Back plane clock,
+		 * confirm that BT is inactive before doing so.
+		 * If this call had come from Non Watchdog context any way
+		 * the Watchdog would switch off the clock again when
+		 * nothing is to be done & Bt has finished using the bus.
+		 */
+		if (bus->bt_use_count != 0) {
+			DHD_INFO(("%s(): Req CLK_SDONLY, BT is active %d not switching off \r\n",
+				__FUNCTION__, bus->bt_use_count));
+			ret = BCME_OK;
+			dhd_os_wd_timer(bus->dhd, dhd_watchdog_ms);
+			break;
+		}
+
+		DHD_INFO(("%s(): Request CLK_NONE BT is NOT active switching off \r\n",
+			__FUNCTION__));
+#endif /* BT_OVER_SDIO */
+
+		/* Remove HT request, or bring up SD clock */
+		if (bus->clkstate == CLK_NONE)
+			ret = dhdsdio_sdclk(bus, TRUE);
+		else if (bus->clkstate == CLK_AVAIL)
+			ret = dhdsdio_htclk(bus, FALSE, FALSE);
+		else
+			DHD_ERROR(("dhdsdio_clkctl: request for %d -> %d\n",
+			           bus->clkstate, target));
+		if (ret == BCME_OK) {
+			dhd_os_wd_timer(bus->dhd, dhd_watchdog_ms);
+		}
+		break;
+
+	case CLK_NONE:
+
+#ifdef BT_OVER_SDIO
+		/*
+		 * If the request is to switch off Back plane clock,
+		 * confirm that BT is inactive before doing so.
+		 * If this call had come from Non Watchdog context any way
+		 * the Watchdog would switch off the clock again when
+		 * nothing is to be done & Bt has finished using the bus.
+		 */
+		if (bus->bt_use_count != 0) {
+			DHD_INFO(("%s(): Request CLK_NONE BT is active %d not switching off \r\n",
+				__FUNCTION__, bus->bt_use_count));
+			ret = BCME_OK;
+			break;
+		}
+
+		DHD_INFO(("%s(): Request CLK_NONE BT is NOT active switching off \r\n",
+			__FUNCTION__));
+#endif /* BT_OVER_SDIO */
+
+		/* Make sure to remove HT request */
+		if (bus->clkstate == CLK_AVAIL)
+			ret = dhdsdio_htclk(bus, FALSE, FALSE);
+		/* Now remove the SD clock */
+		ret = dhdsdio_sdclk(bus, FALSE);
+		if (bus->poll == 0)
+			dhd_os_wd_timer(bus->dhd, 0);
+		break;
+	}
+#ifdef DHD_DEBUG
+	DHD_INFO(("dhdsdio_clkctl: %d -> %d\n", oldstate, bus->clkstate));
+#endif /* DHD_DEBUG */
+
+	return ret;
+}
+
+static int
+dhdsdio_bussleep(dhd_bus_t *bus, bool sleep)
+{
+	int err = 0;
+	bcmsdh_info_t *sdh = bus->sdh;
+	sdpcmd_regs_t *regs = bus->regs;
+	uint retries = 0;
+#if defined(BCMSDIOH_STD)
+	uint32 sd3_tuning_disable = FALSE;
+#endif /* BCMSDIOH_STD */
+#if defined(WL_EXT_IAPSTA) && defined(DHD_LOSSLESS_ROAMING)
+	int state;
+#endif /* WL_EXT_IAPSTA && DHD_LOSSLESS_ROAMING */
+
+	DHD_INFO(("dhdsdio_bussleep: request %s (currently %s)\n",
+	         (sleep ? "SLEEP" : "WAKE"),
+	          (bus->sleeping ? "SLEEP" : "WAKE")));
+
+	if (bus->dhd->hang_was_sent)
+		return BCME_ERROR;
+
+	/* Done if we're already in the requested state */
+	if (sleep == bus->sleeping)
+		return BCME_OK;
+
+	/* Going to sleep: set the alarm and turn off the lights... */
+	if (sleep) {
+		/* Don't sleep if something is pending */
+#ifdef DHD_USE_IDLECOUNT
+		if (bus->dpc_sched || bus->rxskip || pktq_n_pkts_tot(&bus->txq) ||
+			bus->readframes || bus->ctrl_frame_stat)
+#else
+		if (bus->dpc_sched || bus->rxskip || pktq_n_pkts_tot(&bus->txq))
+#endif /* DHD_USE_IDLECOUNT */
+			return BCME_BUSY;
+#if defined(WL_EXT_IAPSTA) && defined(DHD_LOSSLESS_ROAMING)
+		state = wl_ext_any_sta_handshaking(bus->dhd);
+		if (state) {
+			if (dump_msg_level & DUMP_EAPOL_VAL)
+				DHD_ERROR(("handshaking %d\n", state));
+			return BCME_BUSY;
+		}
+#endif /* WL_EXT_IAPSTA && DHD_LOSSLESS_ROAMING */
+
+#ifdef BT_OVER_SDIO
+		/*
+		 * The following is the assumption based on which the hook is placed.
+		 * From WLAN driver, either from the active contexts OR from the Watchdog contexts
+		 * we will be attempting to Go to Sleep. AT that moment if we see that BT is still
+		 * actively using the bus, we will return BCME_BUSY from here, but the bus->sleeping
+		 * state would not have changed. So the caller can then schedule the Watchdog again
+		 * which will come and attempt to sleep at a later point.
+		 *
+		 * In case if BT is the only one and is the last user, we don't switch off the clock
+		 * immediately, we allow the WLAN to decide when to sleep i.e from the watchdog.
+		 * Now if the watchdog becomes active and attempts to switch off the clock and if
+		 * another WLAN context is active they are any way serialized with sdlock.
+		 */
+		if (bus->bt_use_count != 0) {
+			DHD_INFO(("%s(): Cannot sleep BT is active \r\n", __FUNCTION__));
+			return BCME_BUSY;
+		}
+#endif /* !BT_OVER_SDIO */
+
+		/* XXX Is it an error to sleep when not in data state? */
+
+		if (!SLPAUTO_ENAB(bus)) {
+			/* Disable SDIO interrupts (no longer interested) */
+			bcmsdh_intr_disable(bus->sdh);
+
+			/* Make sure the controller has the bus up */
+			dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+			/* Tell device to start using OOB wakeup */
+			W_SDREG(SMB_USE_OOB, &regs->tosbmailbox, retries);
+			if (retries > retry_limit)
+				DHD_ERROR(("CANNOT SIGNAL CHIP, WILL NOT WAKE UP!!\n"));
+
+			/* Turn off our contribution to the HT clock request */
+			dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR,
+				SBSDIO_FORCE_HW_CLKREQ_OFF, NULL);
+
+			/* Isolate the bus */
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL,
+					SBSDIO_DEVCTL_PADS_ISO, NULL);
+		} else {
+#ifdef FORCE_SWOOB_ENABLE
+			/* Tell device to start using OOB wakeup */
+			W_SDREG(SMB_USE_OOB, &regs->tosbmailbox, retries);
+			if (retries > retry_limit)
+				DHD_ERROR(("CANNOT SIGNAL CHIP, WILL NOT WAKE UP!!\n"));
+#endif
+			/* Leave interrupts enabled since device can exit sleep and
+			 * interrupt host
+			 */
+			err = dhdsdio_clk_devsleep_iovar(bus, TRUE /* sleep */);
+		}
+
+		/* Change state */
+		bus->sleeping = TRUE;
+#if defined(BCMSDIOH_STD)
+		sd3_tuning_disable = TRUE;
+		err = bcmsdh_iovar_op(bus->sdh, "sd3_tuning_disable", NULL, 0,
+			&sd3_tuning_disable, sizeof(sd3_tuning_disable), TRUE);
+#endif /* BCMSDIOH_STD */
+#if defined(LINUX) && defined(SUPPORT_P2P_GO_PS)
+		wake_up(&bus->bus_sleep);
+#endif /* LINUX && SUPPORT_P2P_GO_PS */
+		/* XXX Should be able to turn off clock and power */
+		/* XXX Make sure GPIO interrupt input is enabled */
+	} else {
+		/* Waking up: bus power up is ok, set local state */
+
+		if (!SLPAUTO_ENAB(bus)) {
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, 0, &err);
+
+			/* Force pad isolation off if possible (in case power never toggled) */
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, 0, NULL);
+
+			/* XXX Make sure GPIO interrupt input is disabled */
+			/* XXX Should be able to turn on power and clock */
+
+			/* Make sure the controller has the bus up */
+			dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+			/* Send misc interrupt to indicate OOB not needed */
+			W_SDREG(0, &regs->tosbmailboxdata, retries);
+			if (retries <= retry_limit)
+				W_SDREG(SMB_DEV_INT, &regs->tosbmailbox, retries);
+
+			if (retries > retry_limit)
+				DHD_ERROR(("CANNOT SIGNAL CHIP TO CLEAR OOB!!\n"));
+
+			/* Make sure we have SD bus access */
+			dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+
+			/* Enable interrupts again */
+			if (bus->intr && (bus->dhd->busstate == DHD_BUS_DATA)) {
+				bus->intdis = FALSE;
+				bcmsdh_intr_enable(bus->sdh);
+			}
+		} else {
+			err = dhdsdio_clk_devsleep_iovar(bus, FALSE /* wake */);
+#ifdef FORCE_SWOOB_ENABLE
+			/* Send misc interrupt to indicate OOB not needed */
+			W_SDREG(0, &regs->tosbmailboxdata, retries);
+			if (retries <= retry_limit)
+				W_SDREG(SMB_DEV_INT, &regs->tosbmailbox, retries);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && defined(OEM_ANDROID)
+			if (err < 0) {
+				struct net_device *net = NULL;
+				dhd_pub_t *dhd = bus->dhd;
+				net = dhd_idx2net(dhd, 0);
+				if (net != NULL) {
+					DHD_ERROR(("<< WIFI HANG by KSO Enabled failure\n"));
+					dhd_os_sdunlock(dhd);
+					net_os_send_hang_message(net);
+					dhd_os_sdlock(dhd);
+				} else {
+					DHD_ERROR(("<< WIFI HANG Fail because net is NULL\n"));
+				}
+			}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27) && OEM_ANDROID */
+		}
+
+		if (err == 0) {
+			/* Change state */
+			bus->sleeping = FALSE;
+#if defined(BCMSDIOH_STD)
+			sd3_tuning_disable = FALSE;
+			err = bcmsdh_iovar_op(bus->sdh, "sd3_tuning_disable", NULL, 0,
+				&sd3_tuning_disable, sizeof(sd3_tuning_disable), TRUE);
+#endif /* BCMSDIOH_STD */
+		}
+	}
+
+	return err;
+}
+
+#ifdef BT_OVER_SDIO
+/*
+ * Call this function to Get the Clock running.
+ * Assumes that the caller holds the sdlock.
+ * bus - Pointer to the dhd_bus handle
+ * can_wait - TRUE if the caller can wait until the clock becomes ready
+ *            FALSE if the caller cannot wait
+ */
+int __dhdsdio_clk_enable(struct dhd_bus *bus, bus_owner_t owner, int can_wait)
+{
+	int ret = BCME_ERROR;
+
+	BCM_REFERENCE(owner);
+
+	bus->bt_use_count++;
+
+	/*
+	 * We can call BUS_WAKE, clkctl multiple times, both of the items
+	 * have states and if its already ON, no new configuration is done
+	 */
+
+	/* Wake up the Dongle FW from SR */
+	BUS_WAKE(bus);
+
+	/*
+	 * Make sure back plane ht clk is on
+	 * CLK_AVAIL - Turn On both SD & HT clock
+	 */
+	ret = dhdsdio_clkctl(bus, CLK_AVAIL, can_wait);
+
+	DHD_INFO(("%s():bt_use_count %d \r\n", __FUNCTION__,
+		bus->bt_use_count));
+	return ret;
+}
+
+/*
+ * Call this function to relinquish the Clock.
+ * Assumes that the caller holds the sdlock.
+ * bus - Pointer to the dhd_bus handle
+ * can_wait - TRUE if the caller can wait until the clock becomes ready
+ *            FALSE if the caller cannot wait
+ */
+int __dhdsdio_clk_disable(struct dhd_bus *bus, bus_owner_t owner, int can_wait)
+{
+	int ret = BCME_ERROR;
+
+	BCM_REFERENCE(owner);
+	BCM_REFERENCE(can_wait);
+
+	if (bus->bt_use_count == 0) {
+		DHD_ERROR(("%s(): Clocks are already turned off \r\n",
+			__FUNCTION__));
+		return ret;
+	}
+
+	bus->bt_use_count--;
+
+	/*
+	 * When the SDIO Bus is shared between BT & WLAN, we turn Off the clock
+	 * once the last user has relinqushed the same. But there are two schemes
+	 * in that too. We consider WLAN as the  bus master (even if its not
+	 * active). Even when the WLAN is OFF the DHD Watchdog is active.
+	 * So this Bus Watchdog is the context whill put the Bus to sleep.
+	 * Refer dhd_bus_watchdog function
+	 */
+
+	ret = BCME_OK;
+	DHD_INFO(("%s():bt_use_count %d \r\n", __FUNCTION__,
+		bus->bt_use_count));
+	return ret;
+}
+
+void dhdsdio_reset_bt_use_count(struct dhd_bus *bus)
+{
+	/* reset bt use count */
+	bus->bt_use_count = 0;
+}
+#endif /* BT_OVER_SDIO */
+
+#ifdef USE_DYNAMIC_F2_BLKSIZE
+int dhdsdio_func_blocksize(dhd_pub_t *dhd, int function_num, int block_size)
+{
+	int func_blk_size = function_num;
+	int bcmerr = 0;
+	int result;
+
+	bcmerr = dhd_bus_iovar_op(dhd, "sd_blocksize", &func_blk_size,
+		sizeof(int), &result, sizeof(int), IOV_GET);
+
+	if (bcmerr != BCME_OK) {
+		DHD_ERROR(("%s: Get F%d Block size error\n", __FUNCTION__, function_num));
+		return BCME_ERROR;
+	}
+
+	if (result != block_size) {
+		DHD_TRACE_HW4(("%s: F%d Block size set from %d to %d\n",
+			__FUNCTION__, function_num, result, block_size));
+		func_blk_size = function_num << 16 | block_size;
+		bcmerr = dhd_bus_iovar_op(dhd, "sd_blocksize", NULL,
+			0, &func_blk_size, sizeof(int32), IOV_SET);
+		if (bcmerr != BCME_OK) {
+			DHD_ERROR(("%s: Set F2 Block size error\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+	}
+
+	return BCME_OK;
+}
+#endif /* USE_DYNAMIC_F2_BLKSIZE */
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) || defined(FORCE_WOWLAN)
+void
+dhd_enable_oob_intr(struct dhd_bus *bus, bool enable)
+{
+#if defined(BCMSPI_ANDROID)
+	bcmsdh_intr_enable(bus->sdh);
+#elif defined(HW_OOB) || defined(FORCE_WOWLAN)
+	bcmsdh_enable_hw_oob_intr(bus->sdh, enable);
+#else
+	sdpcmd_regs_t *regs = bus->regs;
+	uint retries = 0;
+
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+	if (enable == TRUE) {
+
+		/* Tell device to start using OOB wakeup */
+		W_SDREG(SMB_USE_OOB, &regs->tosbmailbox, retries);
+		if (retries > retry_limit)
+			DHD_ERROR(("CANNOT SIGNAL CHIP, WILL NOT WAKE UP!!\n"));
+
+	} else {
+		/* Send misc interrupt to indicate OOB not needed */
+		W_SDREG(0, &regs->tosbmailboxdata, retries);
+		if (retries <= retry_limit)
+			W_SDREG(SMB_DEV_INT, &regs->tosbmailbox, retries);
+	}
+
+	/* Turn off our contribution to the HT clock request */
+	dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+#endif /* !defined(HW_OOB) */
+}
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+
+int
+dhd_bus_txdata(struct dhd_bus *bus, void *pkt)
+{
+	int ret = BCME_ERROR;
+	osl_t *osh;
+	uint datalen, prec;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	osh = bus->dhd->osh;
+	datalen = PKTLEN(osh, pkt);
+
+#ifdef SDTEST
+	/* Push the test header if doing loopback */
+	if (bus->ext_loop) {
+		uint8* data;
+		PKTPUSH(osh, pkt, SDPCM_TEST_HDRLEN);
+		data = PKTDATA(osh, pkt);
+		*data++ = SDPCM_TEST_ECHOREQ;
+		*data++ = (uint8)bus->loopid++;
+		*data++ = (datalen >> 0);
+		*data++ = (datalen >> 8);
+		datalen += SDPCM_TEST_HDRLEN;
+	}
+#else /* SDTEST */
+	BCM_REFERENCE(datalen);
+#endif /* SDTEST */
+
+	prec = PRIO2PREC((PKTPRIO(pkt) & PRIOMASK));
+
+	/* move from dhdsdio_sendfromq(), try to orphan skb early */
+	if (bus->dhd->conf->orphan_move == 1)
+		PKTORPHAN(pkt, bus->dhd->conf->tsq);
+
+	/* Check for existing queue, current flow-control, pending event, or pending clock */
+	if (dhd_deferred_tx || bus->fcstate || pktq_n_pkts_tot(&bus->txq) || bus->dpc_sched ||
+	    (!DATAOK(bus)) || (bus->flowcontrol & NBITVAL(prec)) ||
+	    (bus->clkstate != CLK_AVAIL)) {
+		bool deq_ret;
+		int pkq_len = 0;
+
+		DHD_TRACE(("%s: deferring pktq len %d\n", __FUNCTION__,
+			pktq_n_pkts_tot(&bus->txq)));
+		bus->fcqueued++;
+
+		/* Priority based enq */
+		dhd_os_sdlock_txq(bus->dhd);
+		deq_ret = dhd_prec_enq(bus->dhd, &bus->txq, pkt, prec);
+		dhd_os_sdunlock_txq(bus->dhd);
+
+		if (!deq_ret) {
+#ifdef PROP_TXSTATUS
+			if (DHD_PKTTAG_WLFCPKT(PKTTAG(pkt)) == 0)
+#endif /* PROP_TXSTATUS */
+			{
+#ifdef DHDTCPACK_SUPPRESS
+				if (dhd_tcpack_check_xmit(bus->dhd, pkt) == BCME_ERROR) {
+					DHD_ERROR(("%s %d: tcpack_suppress ERROR!!! Stop using\n",
+						__FUNCTION__, __LINE__));
+					dhd_tcpack_suppress_set(bus->dhd, TCPACK_SUP_OFF);
+				}
+#endif /* DHDTCPACK_SUPPRESS */
+				dhd_txcomplete(bus->dhd, pkt, FALSE);
+				PKTFREE(osh, pkt, TRUE);	/* XXX update counter */
+			}
+			ret = BCME_NORESOURCE;
+		} else
+			ret = BCME_OK;
+
+		/* XXX Possible race since check and action are not locked? */
+		if (dhd_doflow) {
+			dhd_os_sdlock_txq(bus->dhd);
+			pkq_len = pktq_n_pkts_tot(&bus->txq);
+			dhd_os_sdunlock_txq(bus->dhd);
+		}
+		if (dhd_doflow && pkq_len >= FCHI) {
+			bool wlfc_enabled = FALSE;
+#ifdef PROP_TXSTATUS
+			wlfc_enabled = (dhd_wlfc_flowcontrol(bus->dhd, ON, FALSE) !=
+				WLFC_UNSUPPORTED);
+#endif
+			if (!wlfc_enabled && dhd_doflow) {
+				dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, ON);
+			}
+		}
+
+#ifdef DHD_DEBUG
+		dhd_os_sdlock_txq(bus->dhd);
+		if (pktqprec_n_pkts(&bus->txq, prec) > qcount[prec])
+			qcount[prec] = pktqprec_n_pkts(&bus->txq, prec);
+		dhd_os_sdunlock_txq(bus->dhd);
+#endif
+
+		/* Schedule DPC if needed to send queued packet(s) */
+		/* XXX Also here, since other deferral conditions may no longer hold? */
+		if (dhd_deferred_tx && !bus->dpc_sched) {
+			if (bus->dhd->conf->deferred_tx_len) {
+				if(dhd_os_wd_timer_enabled(bus->dhd) == FALSE) {
+					bus->dpc_sched = TRUE;
+					dhd_sched_dpc(bus->dhd);
+				}
+				if(pktq_n_pkts_tot(&bus->txq) >= bus->dhd->conf->deferred_tx_len &&
+						dhd_os_wd_timer_enabled(bus->dhd) == FALSE) {
+					bus->dpc_sched = TRUE;
+					dhd_sched_dpc(bus->dhd);
+				}
+			} else {
+				bus->dpc_sched = TRUE;
+				dhd_sched_dpc(bus->dhd);
+			}
+		}
+	} else {
+		int chan = SDPCM_DATA_CHANNEL;
+
+#ifdef SDTEST
+		chan = (bus->ext_loop ? SDPCM_TEST_CHANNEL : SDPCM_DATA_CHANNEL);
+#endif
+		/* Lock: we're about to use shared data/code (and SDIO) */
+		dhd_os_sdlock(bus->dhd);
+
+		/* Otherwise, send it now */
+		BUS_WAKE(bus);
+		/* Make sure back plane ht clk is on, no pending allowed */
+		dhdsdio_clkctl(bus, CLK_AVAIL, TRUE);
+
+		ret = dhdsdio_txpkt(bus, chan, &pkt, 1, TRUE);
+
+		if (ret != BCME_OK)
+			bus->dhd->tx_errors++;
+		else
+			bus->dhd->dstats.tx_bytes += datalen;
+
+		if ((bus->idletime == DHD_IDLE_IMMEDIATE) && !bus->dpc_sched &&
+				NO_OTHER_ACTIVE_BUS_USER(bus)) {
+			bus->activity = FALSE;
+			dhdsdio_bussleep(bus, TRUE);
+			dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+		}
+
+		dhd_os_sdunlock(bus->dhd);
+	}
+
+	return ret;
+}
+
+/* align packet data pointer and packet length to n-byte boundary, process packet headers,
+ * a new packet may be allocated if there is not enough head and/or tail from for padding.
+ * the caller is responsible for updating the glom size in the head packet (when glom is
+ * used)
+ *
+ * pad_pkt_len: returns the length of extra padding needed from the padding packet, this parameter
+ * is taken in tx glom mode only
+ *
+ * new_pkt: out, pointer of the new packet allocated due to insufficient head room for alignment
+ * padding, NULL if not needed, the caller is responsible for freeing the new packet
+ *
+ * return: positive value - length of the packet, including head and tail padding
+ *		   negative value - errors
+ */
+static int dhdsdio_txpkt_preprocess(dhd_bus_t *bus, void *pkt, int chan, int txseq,
+	int prev_chain_total_len, bool last_chained_pkt,
+	int *pad_pkt_len, void **new_pkt
+#if defined(BCMSDIOH_TXGLOM_EXT)
+	, int first_frame
+#endif
+)
+{
+	osl_t *osh;
+	uint8 *frame;
+	int pkt_len;
+	int modulo;
+	int head_padding;
+	int tail_padding = 0;
+	uint32 swheader;
+	uint32 swhdr_offset;
+	bool alloc_new_pkt = FALSE;
+	uint8 sdpcm_hdrlen = bus->txglom_enable ? SDPCM_HDRLEN_TXGLOM : SDPCM_HDRLEN;
+#ifdef PKT_STATICS
+	uint16 len;
+#endif
+
+	*new_pkt = NULL;
+	osh = bus->dhd->osh;
+
+#ifdef DHDTCPACK_SUPPRESS
+	if (dhd_tcpack_check_xmit(bus->dhd, pkt) == BCME_ERROR) {
+		DHD_ERROR(("%s %d: tcpack_suppress ERROR!!! Stop using it\n",
+			__FUNCTION__, __LINE__));
+		dhd_tcpack_suppress_set(bus->dhd, TCPACK_SUP_OFF);
+	}
+#endif /* DHDTCPACK_SUPPRESS */
+
+	/* Add space for the SDPCM hardware/software headers */
+	PKTPUSH(osh, pkt, sdpcm_hdrlen);
+	ASSERT(ISALIGNED((uintptr)PKTDATA(osh, pkt), 2));
+
+	frame = (uint8*)PKTDATA(osh, pkt);
+	pkt_len = (uint16)PKTLEN(osh, pkt);
+
+#ifdef PKT_STATICS
+	len = (uint16)PKTLEN(osh, pkt);
+	switch(chan) {
+		case SDPCM_CONTROL_CHANNEL:
+			bus->tx_statics.ctrl_count++;
+			bus->tx_statics.ctrl_size += len;
+			break;
+		case SDPCM_DATA_CHANNEL:
+			bus->tx_statics.data_count++;
+			bus->tx_statics.data_size += len;
+			break;
+		case SDPCM_GLOM_CHANNEL:
+			bus->tx_statics.glom_count++;
+			bus->tx_statics.glom_size += len;
+			break;
+		case SDPCM_EVENT_CHANNEL:
+			bus->tx_statics.event_count++;
+			bus->tx_statics.event_size += len;
+			break;
+		case SDPCM_TEST_CHANNEL:
+			bus->tx_statics.test_count++;
+			bus->tx_statics.test_size += len;
+			break;
+
+		default:
+			break;
+	}
+#endif /* PKT_STATICS */
+#ifdef DHD_DEBUG
+	if (PKTPRIO(pkt) < ARRAYSIZE(tx_packets))
+		tx_packets[PKTPRIO(pkt)]++;
+#endif /* DHD_DEBUG */
+
+	/* align the data pointer, allocate a new packet if there is not enough space (new
+	 * packet data pointer will be aligned thus no padding will be needed)
+	 */
+	head_padding = (uintptr)frame % DHD_SDALIGN;
+	if (PKTHEADROOM(osh, pkt) < head_padding) {
+		head_padding = 0;
+		alloc_new_pkt = TRUE;
+	} else {
+		uint cur_chain_total_len;
+		int chain_tail_padding = 0;
+
+		/* All packets need to be aligned by DHD_SDALIGN */
+		modulo = (pkt_len + head_padding) % DHD_SDALIGN;
+		tail_padding = modulo > 0 ? (DHD_SDALIGN - modulo) : 0;
+
+		/* Total pkt chain length needs to be aligned by block size,
+		 * unless it is a single pkt chain with total length less than one block size,
+		 * which we prefer sending by byte mode.
+		 *
+		 * Do the chain alignment here if
+		 * 1. This is the last pkt of the chain of multiple pkts or a single pkt.
+		 * 2-1. This chain is of multiple pkts, or
+		 * 2-2. This is a single pkt whose size is longer than one block size.
+		 */
+		cur_chain_total_len = prev_chain_total_len +
+			(head_padding + pkt_len + tail_padding);
+		if (last_chained_pkt && bus->blocksize != 0 &&
+			(cur_chain_total_len > (int)bus->blocksize || prev_chain_total_len > 0)) {
+			modulo = cur_chain_total_len % bus->blocksize;
+			chain_tail_padding = modulo > 0 ? (bus->blocksize - modulo) : 0;
+		}
+
+#ifdef DHDENABLE_TAILPAD
+		if (PKTTAILROOM(osh, pkt) < tail_padding) {
+			/* We don't have tail room to align by DHD_SDALIGN */
+			alloc_new_pkt = TRUE;
+			bus->tx_tailpad_pktget++;
+		} else if (PKTTAILROOM(osh, pkt) < tail_padding + chain_tail_padding) {
+			/* We have tail room for tail_padding of this pkt itself, but not for
+			 * total pkt chain alignment by block size.
+			 * Use the padding packet to avoid memory copy if applicable,
+			 * otherwise, just allocate a new pkt.
+			 */
+			if (bus->pad_pkt) {
+				*pad_pkt_len = chain_tail_padding;
+				bus->tx_tailpad_chain++;
+			} else {
+				alloc_new_pkt = TRUE;
+				bus->tx_tailpad_pktget++;
+			}
+		} else
+		/* This last pkt's tailroom is sufficient to hold both tail_padding
+		 * of the pkt itself and chain_tail_padding of total pkt chain
+		 */
+#endif /* DHDENABLE_TAILPAD */
+		tail_padding += chain_tail_padding;
+	}
+
+	DHD_INFO(("%s sdhdr len + orig_pkt_len %d h_pad %d t_pad %d pad_pkt_len %d\n",
+		__FUNCTION__, pkt_len, head_padding, tail_padding, *pad_pkt_len));
+
+	if (alloc_new_pkt) {
+		void *tmp_pkt;
+		int newpkt_size;
+		int cur_total_len;
+
+		ASSERT(*pad_pkt_len == 0);
+
+		DHD_INFO(("%s allocating new packet for padding\n", __FUNCTION__));
+
+		/* head pointer is aligned now, no padding needed */
+		head_padding = 0;
+
+		/* update the tail padding as it depends on the head padding, since a new packet is
+		 * allocated, the head padding is non longer needed and packet length is chagned
+		 */
+
+		cur_total_len = prev_chain_total_len + pkt_len;
+		if (last_chained_pkt && bus->blocksize != 0 &&
+			(cur_total_len > (int)bus->blocksize || prev_chain_total_len > 0)) {
+			modulo = cur_total_len % bus->blocksize;
+			tail_padding = modulo > 0 ? (bus->blocksize - modulo) : 0;
+		} else {
+			modulo = pkt_len % DHD_SDALIGN;
+			tail_padding = modulo > 0 ? (DHD_SDALIGN - modulo) : 0;
+		}
+
+		newpkt_size = PKTLEN(osh, pkt) + bus->blocksize + DHD_SDALIGN;
+		bus->dhd->tx_realloc++;
+		tmp_pkt = PKTGET(osh, newpkt_size, TRUE);
+		if (tmp_pkt == NULL) {
+			DHD_ERROR(("failed to alloc new %d byte packet\n", newpkt_size));
+			return BCME_NOMEM;
+		}
+		PKTALIGN(osh, tmp_pkt, PKTLEN(osh, pkt), DHD_SDALIGN);
+		bcopy(PKTDATA(osh, pkt), PKTDATA(osh, tmp_pkt), PKTLEN(osh, pkt));
+		*new_pkt = tmp_pkt;
+		pkt = tmp_pkt;
+	}
+
+	if (head_padding)
+		PKTPUSH(osh, pkt, head_padding);
+
+	frame = (uint8*)PKTDATA(osh, pkt);
+	bzero(frame, head_padding + sdpcm_hdrlen);
+	pkt_len = (uint16)PKTLEN(osh, pkt);
+
+	/* the header has the followming format
+	 * 4-byte HW frame tag: length, ~length (for glom this is the total length)
+	 *
+	 * 8-byte HW extesion flags (glom mode only) as the following:
+	 *			2-byte packet length, excluding HW tag and padding
+	 *			2-byte frame channel and frame flags (e.g. next frame following)
+	 *			2-byte header length
+	 *			2-byte tail padding size
+	 *
+	 * 8-byte SW frame tags as the following
+	 *			4-byte flags: host tx seq, channel, data offset
+	 *			4-byte flags: TBD
+	 */
+
+	swhdr_offset = SDPCM_FRAMETAG_LEN;
+
+	/* hardware frame tag:
+	 *
+	 * in tx-glom mode, dongle only checks the hardware frame tag in the first
+	 * packet and sees it as the total lenght of the glom (including tail padding),
+	 * for each packet in the glom, the packet length needs to be updated, (see
+	 * below PKTSETLEN)
+	 *
+	 * in non tx-glom mode, PKTLEN still need to include tail padding as to be
+	 * referred to in sdioh_request_buffer(). The tail length will be excluded in
+	 * dhdsdio_txpkt_postprocess().
+	 */
+#if defined(BCMSDIOH_TXGLOM_EXT)
+	if (bus->dhd->conf->txglom_bucket_size)
+		tail_padding = 0;
+#endif
+	*(uint16*)frame = (uint16)htol16(pkt_len);
+	*(((uint16*)frame) + 1) = (uint16)htol16(~pkt_len);
+	pkt_len += tail_padding;
+
+	/* hardware extesion flags */
+	if (bus->txglom_enable) {
+		uint32 hwheader1;
+		uint32 hwheader2;
+#ifdef BCMSDIOH_TXGLOM_EXT
+		uint32 act_len = pkt_len - tail_padding;
+		uint32 real_pad = 0;
+		if(bus->dhd->conf->txglom_ext && !last_chained_pkt) {
+			tail_padding = 0;
+			if(first_frame == 0) {
+				// first pkt, add pad to bucket size - recv offset
+				pkt_len = bus->dhd->conf->txglom_bucket_size - TXGLOM_RECV_OFFSET;
+			} else {
+				// add pad to bucket size
+				pkt_len = bus->dhd->conf->txglom_bucket_size;
+			}
+			swhdr_offset += SDPCM_HWEXT_LEN;
+			hwheader1 = (act_len - SDPCM_FRAMETAG_LEN) | (last_chained_pkt << 24);
+			hwheader2 = (pkt_len - act_len) << 16;
+			htol32_ua_store(hwheader1, frame + SDPCM_FRAMETAG_LEN);
+			htol32_ua_store(hwheader2, frame + SDPCM_FRAMETAG_LEN + 4);
+			real_pad = pkt_len - act_len;
+
+			if (PKTTAILROOM(osh, pkt) < real_pad) {
+				DHD_INFO(("%s : insufficient tailroom %d for %d real_pad\n",
+					__func__, (int)PKTTAILROOM(osh, pkt), real_pad));
+				if (PKTPADTAILROOM(osh, pkt, real_pad)) {
+					DHD_ERROR(("CHK1: padding error size %d\n", real_pad));
+				} else
+					frame = (uint8 *)PKTDATA(osh, pkt);
+			}
+		} else
+#endif
+		{
+			swhdr_offset += SDPCM_HWEXT_LEN;
+			hwheader1 = (pkt_len - SDPCM_FRAMETAG_LEN - tail_padding) |
+				(last_chained_pkt << 24);
+			hwheader2 = (tail_padding) << 16;
+			htol32_ua_store(hwheader1, frame + SDPCM_FRAMETAG_LEN);
+			htol32_ua_store(hwheader2, frame + SDPCM_FRAMETAG_LEN + 4);
+		}
+	}
+	PKTSETLEN((osh), (pkt), (pkt_len));
+
+	/* software frame tags */
+	swheader = ((chan << SDPCM_CHANNEL_SHIFT) & SDPCM_CHANNEL_MASK)
+		| (txseq % SDPCM_SEQUENCE_WRAP) |
+		(((head_padding + sdpcm_hdrlen) << SDPCM_DOFFSET_SHIFT) & SDPCM_DOFFSET_MASK);
+	htol32_ua_store(swheader, frame + swhdr_offset);
+	htol32_ua_store(0, frame + swhdr_offset + sizeof(swheader));
+
+	return pkt_len;
+}
+
+static int dhdsdio_txpkt_postprocess(dhd_bus_t *bus, void *pkt)
+{
+	osl_t *osh;
+	uint8 *frame;
+	int data_offset;
+	int tail_padding;
+	int swhdr_offset = SDPCM_FRAMETAG_LEN + (bus->txglom_enable ? SDPCM_HWEXT_LEN : 0);
+
+	(void)osh;
+	osh = bus->dhd->osh;
+
+	/* restore pkt buffer pointer, but keeps the header pushed by dhd_prot_hdrpush */
+	frame = (uint8*)PKTDATA(osh, pkt);
+
+	DHD_INFO(("%s PKTLEN before postprocess %d\n",
+		__FUNCTION__, PKTLEN(osh, pkt)));
+
+	/* PKTLEN still includes tail_padding, so exclude it.
+	 * We shall have head_padding + original pkt_len for PKTLEN afterwards.
+	 */
+	if (bus->txglom_enable) {
+		/* txglom pkts have tail_padding length in HW ext header */
+		tail_padding = ltoh32_ua(frame + SDPCM_FRAMETAG_LEN + 4) >> 16;
+		PKTSETLEN(osh, pkt, PKTLEN(osh, pkt) - tail_padding);
+		DHD_INFO((" txglom pkt: tail_padding %d PKTLEN %d\n",
+			tail_padding, PKTLEN(osh, pkt)));
+	} else {
+		/* non-txglom pkts have head_padding + original pkt length in HW frame tag.
+		 * We cannot refer to this field for txglom pkts as the first pkt of the chain will
+		 * have the field for the total length of the chain.
+		 */
+		PKTSETLEN(osh, pkt, *(uint16*)frame);
+		DHD_INFO((" non-txglom pkt: HW frame tag len %d after PKTLEN %d\n",
+			*(uint16*)frame, PKTLEN(osh, pkt)));
+	}
+
+	data_offset = ltoh32_ua(frame + swhdr_offset);
+	data_offset = (data_offset & SDPCM_DOFFSET_MASK) >> SDPCM_DOFFSET_SHIFT;
+	/* Get rid of sdpcm header + head_padding */
+	PKTPULL(osh, pkt, data_offset);
+
+	DHD_INFO(("%s data_offset %d, PKTLEN %d\n",
+		__FUNCTION__, data_offset, PKTLEN(osh, pkt)));
+
+	return BCME_OK;
+}
+
+static int dhdsdio_txpkt(dhd_bus_t *bus, uint chan, void** pkts, int num_pkt, bool free_pkt)
+{
+	int i;
+	int ret = 0;
+	osl_t *osh;
+	bcmsdh_info_t *sdh;
+	void *pkt = NULL;
+	void *pkt_chain;
+	int total_len = 0;
+	void *head_pkt = NULL;
+	void *prev_pkt = NULL;
+	int pad_pkt_len = 0;
+	int new_pkt_num = 0;
+	void *new_pkts[MAX_TX_PKTCHAIN_CNT];
+	bool wlfc_enabled = FALSE;
+
+	if (bus->dhd->dongle_reset)
+		return BCME_NOTREADY;
+
+	if (num_pkt <= 0)
+		return BCME_BADARG;
+
+	sdh = bus->sdh;
+	osh = bus->dhd->osh;
+	/* init new_pkts[0] to make some compiler happy, not necessary as we check new_pkt_num */
+	new_pkts[0] = NULL;
+
+	for (i = 0; i < num_pkt; i++) {
+		int pkt_len;
+		bool last_pkt;
+		void *new_pkt = NULL;
+
+		pkt = pkts[i];
+		ASSERT(pkt);
+		last_pkt = (i == num_pkt - 1);
+		pkt_len = dhdsdio_txpkt_preprocess(bus, pkt, chan, bus->tx_seq + i,
+			total_len, last_pkt, &pad_pkt_len, &new_pkt
+#if defined(BCMSDIOH_TXGLOM_EXT)
+			, i
+#endif
+		);
+		if (pkt_len <= 0) {
+			if (new_pkt)
+				PKTFREE(osh, new_pkt, TRUE);
+			goto done;
+		}
+		if (new_pkt) {
+			pkt = new_pkt;
+			new_pkts[new_pkt_num++] = new_pkt;
+		}
+		total_len += pkt_len;
+
+		PKTSETNEXT(osh, pkt, NULL);
+		/* insert the packet into the list */
+		head_pkt ? PKTSETNEXT(osh, prev_pkt, pkt) : (head_pkt = pkt);
+		prev_pkt = pkt;
+
+	}
+
+	/* Update the HW frame tag (total length) in the first pkt of the glom */
+	if (bus->txglom_enable) {
+		uint8 *frame;
+
+		total_len += pad_pkt_len;
+		frame = (uint8*)PKTDATA(osh, head_pkt);
+		*(uint16*)frame = (uint16)htol16(total_len);
+		*(((uint16*)frame) + 1) = (uint16)htol16(~total_len);
+
+	}
+
+#ifdef DHDENABLE_TAILPAD
+	/* if a padding packet if needed, insert it to the end of the link list */
+	if (pad_pkt_len) {
+		PKTSETLEN(osh, bus->pad_pkt, pad_pkt_len);
+		PKTSETNEXT(osh, pkt, bus->pad_pkt);
+	}
+#endif /* DHDENABLE_TAILPAD */
+
+	/* dhd_bcmsdh_send_buf ignores the buffer pointer if he packet
+	 * parameter is not NULL, for non packet chian we pass NULL pkt pointer
+	 * so it will take the aligned length and buffer pointer.
+	 */
+	pkt_chain = PKTNEXT(osh, head_pkt) ? head_pkt : NULL;
+#ifdef TPUT_MONITOR
+	if ((bus->dhd->conf->data_drop_mode == TXPKT_DROP) && (total_len > 500))
+		ret = BCME_OK;
+	else
+#endif
+	ret = dhd_bcmsdh_send_buf(bus, bcmsdh_cur_sbwad(sdh), SDIO_FUNC_2, F2SYNC,
+		PKTDATA(osh, head_pkt), total_len, pkt_chain, NULL, NULL, TXRETRIES);
+	if (ret == BCME_OK)
+		bus->tx_seq = (bus->tx_seq + num_pkt) % SDPCM_SEQUENCE_WRAP;
+
+	/* if a padding packet was needed, remove it from the link list as it not a data pkt */
+	if (pad_pkt_len && pkt)
+		PKTSETNEXT(osh, pkt, NULL);
+
+done:
+	pkt = head_pkt;
+	while (pkt) {
+		void *pkt_next = PKTNEXT(osh, pkt);
+		PKTSETNEXT(osh, pkt, NULL);
+		dhdsdio_txpkt_postprocess(bus, pkt);
+		pkt = pkt_next;
+	}
+
+	/* new packets might be allocated due to insufficient room for padding, but we
+	 * still have to indicate the original packets to upper layer
+	 */
+	for (i = 0; i < num_pkt; i++) {
+		pkt = pkts[i];
+		wlfc_enabled = FALSE;
+#ifdef PROP_TXSTATUS
+		if (DHD_PKTTAG_WLFCPKT(PKTTAG(pkt))) {
+			wlfc_enabled = (dhd_wlfc_txcomplete(bus->dhd, pkt, ret == 0) !=
+				WLFC_UNSUPPORTED);
+		}
+#endif /* PROP_TXSTATUS */
+		if (!wlfc_enabled) {
+			PKTSETNEXT(osh, pkt, NULL);
+			dhd_txcomplete(bus->dhd, pkt, ret != 0);
+			if (free_pkt)
+				PKTFREE(osh, pkt, TRUE);
+		}
+	}
+
+	for (i = 0; i < new_pkt_num; i++)
+		PKTFREE(osh, new_pkts[i], TRUE);
+
+	return ret;
+}
+
+static uint
+dhdsdio_sendfromq(dhd_bus_t *bus, uint maxframes)
+{
+	uint cnt = 0;
+	uint8 tx_prec_map;
+	uint16 txpktqlen = 0;
+	uint32 intstatus = 0;
+	uint retries = 0;
+	osl_t *osh;
+	dhd_pub_t *dhd = bus->dhd;
+	sdpcmd_regs_t *regs = bus->regs;
+#if defined(DHD_LOSSLESS_ROAMING) || defined(DHD_PKTDUMP_TOFW)
+	uint8 *pktdata;
+	struct ether_header *eh;
+#ifdef BDC
+	struct bdc_header *bdc_header;
+	uint8 data_offset;
+#endif
+#endif /* DHD_LOSSLESS_ROAMING */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (!KSO_ENAB(bus)) {
+		DHD_ERROR(("%s: Device asleep\n", __FUNCTION__));
+		return BCME_NODEVICE;
+	}
+
+	osh = dhd->osh;
+	tx_prec_map = ~bus->flowcontrol;
+#ifdef DHD_LOSSLESS_ROAMING
+	tx_prec_map &= dhd->dequeue_prec_map;
+#endif /* DHD_LOSSLESS_ROAMING */
+	for (cnt = 0; (cnt < maxframes) && DATAOK(bus);) {
+		int i;
+		int num_pkt = 1;
+		void *pkts[MAX_TX_PKTCHAIN_CNT];
+		int prec_out;
+		uint datalen = 0;
+
+		dhd_os_sdlock_txq(bus->dhd);
+		if (bus->txglom_enable) {
+			uint32 glomlimit = (uint32)bus->txglomsize;
+#if defined(BCMSDIOH_STD)
+			if (bus->blocksize == 64) {
+				glomlimit = MIN((uint32)bus->txglomsize, BLK_64_MAXTXGLOM);
+			}
+#endif /* BCMSDIOH_STD */
+			num_pkt = MIN((uint32)DATABUFCNT(bus), glomlimit);
+			num_pkt = MIN(num_pkt, ARRAYSIZE(pkts));
+		}
+		num_pkt = MIN(num_pkt, pktq_mlen(&bus->txq, tx_prec_map));
+		for (i = 0; i < num_pkt; i++) {
+			pkts[i] = pktq_mdeq(&bus->txq, tx_prec_map, &prec_out);
+			if (!pkts[i]) {
+				DHD_ERROR(("%s: pktq_mlen non-zero when no pkt\n",
+					__FUNCTION__));
+				ASSERT(0);
+				break;
+			}
+#if defined(DHD_LOSSLESS_ROAMING) || defined(DHD_PKTDUMP_TOFW)
+			pktdata = (uint8 *)PKTDATA(osh, pkts[i]);
+#ifdef BDC
+			/* Skip BDC header */
+			bdc_header = (struct bdc_header *)pktdata;
+			data_offset = bdc_header->dataOffset;
+			pktdata += BDC_HEADER_LEN + (data_offset << 2);
+#endif
+			eh = (struct ether_header *)pktdata;
+#ifdef DHD_LOSSLESS_ROAMING
+			if (eh->ether_type == hton16(ETHER_TYPE_802_1X)) {
+				uint8 prio = (uint8)PKTPRIO(pkts[i]);
+
+				/* Restore to original priority for 802.1X packet */
+				if (prio == PRIO_8021D_NC) {
+					PKTSETPRIO(pkts[i], dhd->prio_8021x);
+#ifdef BDC
+					/* Restore to original priority in BDC header */
+					bdc_header->priority =
+						(dhd->prio_8021x & BDC_PRIORITY_MASK);
+#endif
+				}
+			}
+#endif /* DHD_LOSSLESS_ROAMING */
+#ifdef DHD_PKTDUMP_TOFW
+			dhd_dump_pkt(bus->dhd, BDC_GET_IF_IDX(bdc_header), pktdata,
+				(uint32)PKTLEN(bus->dhd->osh, pkts[i]), TRUE, NULL, NULL);
+#endif /* DHD_PKTDUMP_TOFW */
+#endif /* DHD_LOSSLESS_ROAMING || DHD_PKTDUMP_TOFW */
+			if (!bus->dhd->conf->orphan_move)
+				PKTORPHAN(pkts[i], bus->dhd->conf->tsq);
+			datalen += PKTLEN(osh, pkts[i]);
+		}
+		dhd_os_sdunlock_txq(bus->dhd);
+
+		if (i == 0)
+			break;
+		if (dhdsdio_txpkt(bus, SDPCM_DATA_CHANNEL, pkts, i, TRUE) != BCME_OK)
+			dhd->tx_errors++;
+		else {
+			dhd->dstats.tx_bytes += datalen;
+			bus->txglomframes++;
+			bus->txglompkts += num_pkt;
+#ifdef PKT_STATICS
+			bus->tx_statics.glom_cnt_us[num_pkt-1] =
+				(bus->tx_statics.glom_cnt[num_pkt-1]*bus->tx_statics.glom_cnt_us[num_pkt-1]
+				+ bcmsdh_get_spend_time(bus->sdh))/(bus->tx_statics.glom_cnt[num_pkt-1] + 1);
+#endif
+		}
+		cnt += i;
+#ifdef PKT_STATICS
+		if (num_pkt) {
+			bus->tx_statics.glom_cnt[num_pkt-1]++;
+			if (num_pkt > bus->tx_statics.glom_max)
+				bus->tx_statics.glom_max = num_pkt;
+		}
+#endif
+
+		/* In poll mode, need to check for other events */
+		if (!bus->intr && cnt)
+		{
+			/* Check device status, signal pending interrupt */
+			R_SDREG(intstatus, &regs->intstatus, retries);
+			bus->f2txdata++;
+			if (bcmsdh_regfail(bus->sdh))
+				break;
+			if (intstatus & bus->hostintmask)
+				bus->ipend = TRUE;
+		}
+
+	}
+
+	if (dhd_doflow) {
+		dhd_os_sdlock_txq(bus->dhd);
+		txpktqlen = pktq_n_pkts_tot(&bus->txq);
+		dhd_os_sdunlock_txq(bus->dhd);
+	}
+
+	/* Do flow-control if needed */
+	if (dhd->up && (dhd->busstate == DHD_BUS_DATA) && (txpktqlen < FCLOW)) {
+		bool wlfc_enabled = FALSE;
+#ifdef PROP_TXSTATUS
+		wlfc_enabled = (dhd_wlfc_flowcontrol(dhd, OFF, TRUE) != WLFC_UNSUPPORTED);
+#endif
+		if (!wlfc_enabled && dhd_doflow && dhd->txoff) {
+			dhd_txflowcontrol(dhd, ALL_INTERFACES, OFF);
+		}
+	}
+
+	return cnt;
+}
+
+static void
+dhdsdio_sendpendctl(dhd_bus_t *bus)
+{
+	bcmsdh_info_t *sdh = bus->sdh;
+	int ret;
+	uint8* frame_seq = bus->ctrl_frame_buf + SDPCM_FRAMETAG_LEN;
+
+	if (bus->txglom_enable)
+		frame_seq += SDPCM_HWEXT_LEN;
+
+	if (*frame_seq != bus->tx_seq) {
+		DHD_INFO(("%s IOCTL frame seq lag detected!"
+			" frm_seq:%d != bus->tx_seq:%d, corrected\n",
+			__FUNCTION__, *frame_seq, bus->tx_seq));
+		*frame_seq = bus->tx_seq;
+	}
+
+	ret = dhd_bcmsdh_send_buf(bus, bcmsdh_cur_sbwad(sdh), SDIO_FUNC_2, F2SYNC,
+		(uint8 *)bus->ctrl_frame_buf, (uint32)bus->ctrl_frame_len,
+		NULL, NULL, NULL, 1);
+	if (ret == BCME_OK)
+		bus->tx_seq = (bus->tx_seq + 1) % SDPCM_SEQUENCE_WRAP;
+
+	bus->ctrl_frame_stat = FALSE;
+	dhd_wait_event_wakeup(bus->dhd);
+}
+
+int
+dhd_bus_txctl(struct dhd_bus *bus, uchar *msg, uint msglen)
+{
+	static int err_nodevice = 0;
+	uint8 *frame;
+	uint16 len;
+	uint32 swheader;
+	uint8 doff = 0;
+	int ret = -1;
+	uint8 sdpcm_hdrlen = bus->txglom_enable ? SDPCM_HDRLEN_TXGLOM : SDPCM_HDRLEN;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd->dongle_reset)
+		return -EIO;
+
+	/* Back the pointer to make a room for bus header */
+	frame = msg - sdpcm_hdrlen;
+	len = (msglen += sdpcm_hdrlen);
+
+	/* Add alignment padding (optional for ctl frames) */
+	if (dhd_alignctl) {
+		if ((doff = ((uintptr)frame % DHD_SDALIGN))) {
+			frame -= doff;
+			len += doff;
+			msglen += doff;
+			bzero(frame, doff + sdpcm_hdrlen);
+		}
+		ASSERT(doff < DHD_SDALIGN);
+	}
+	doff += sdpcm_hdrlen;
+
+#ifndef BCMSPI
+	/* Round send length to next SDIO block */
+	if (bus->roundup && bus->blocksize && (len > bus->blocksize)) {
+		uint16 pad = bus->blocksize - (len % bus->blocksize);
+		if ((pad <= bus->roundup) && (pad < bus->blocksize))
+			len += pad;
+	} else if (len % DHD_SDALIGN) {
+		len += DHD_SDALIGN - (len % DHD_SDALIGN);
+	}
+#endif /* BCMSPI */
+
+	/* Satisfy length-alignment requirements */
+	if (forcealign && (len & (ALIGNMENT - 1)))
+		len = ROUNDUP(len, ALIGNMENT);
+
+	ASSERT(ISALIGNED((uintptr)frame, 2));
+
+	/* Need to lock here to protect txseq and SDIO tx calls */
+	dhd_os_sdlock(bus->dhd);
+	if (bus->dhd->conf->txctl_tmo_fix > 0 && !TXCTLOK(bus)) {
+		bus->ctrl_wait = TRUE;
+		dhd_os_sdunlock(bus->dhd);
+		wait_event_interruptible_timeout(bus->ctrl_tx_wait, TXCTLOK(bus),
+			msecs_to_jiffies(bus->dhd->conf->txctl_tmo_fix));
+		dhd_os_sdlock(bus->dhd);
+		bus->ctrl_wait = FALSE;
+	}
+
+	BUS_WAKE(bus);
+
+#ifdef BCMQT
+	/* Make sure firmware is ready before request HT clock for PMU maximum resource setting */
+	dhd_timeout_start(&tmo, DHD_WAIT_READSHARED * 1000);
+	ready = 0;
+	while (!ready && !dhd_timeout_expired(&tmo)) {
+		sdpcm_shared_t shared;
+		ready = (dhdsdio_readshared(bus, &shared) == 0);
+	}
+
+	/* Readshared fail, shared memory wasn't update by firmware initialization */
+	if (!ready) {
+		DHD_ERROR(("%s: Readshared timeout, firmware up fail.\n", __FUNCTION__));
+		ret = -1;
+		goto exit;
+	}
+#endif /* BCMQT */
+
+	/* Make sure backplane clock is on */
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+	/* Hardware tag: 2 byte len followed by 2 byte ~len check (all LE) */
+	*(uint16*)frame = htol16((uint16)msglen);
+	*(((uint16*)frame) + 1) = htol16(~msglen);
+
+	if (bus->txglom_enable) {
+		uint32 hwheader1, hwheader2;
+		/* Software tag: channel, sequence number, data offset */
+		swheader = ((SDPCM_CONTROL_CHANNEL << SDPCM_CHANNEL_SHIFT) & SDPCM_CHANNEL_MASK)
+				| bus->tx_seq
+				| ((doff << SDPCM_DOFFSET_SHIFT) & SDPCM_DOFFSET_MASK);
+		htol32_ua_store(swheader, frame + SDPCM_FRAMETAG_LEN + SDPCM_HWEXT_LEN);
+		htol32_ua_store(0, frame + SDPCM_FRAMETAG_LEN
+			+ SDPCM_HWEXT_LEN + sizeof(swheader));
+
+		hwheader1 = (msglen - SDPCM_FRAMETAG_LEN) | (1 << 24);
+		hwheader2 = (len - (msglen)) << 16;
+		htol32_ua_store(hwheader1, frame + SDPCM_FRAMETAG_LEN);
+		htol32_ua_store(hwheader2, frame + SDPCM_FRAMETAG_LEN + 4);
+
+		*(uint16*)frame = htol16(len);
+		*(((uint16*)frame) + 1) = htol16(~(len));
+	} else {
+		/* Software tag: channel, sequence number, data offset */
+		swheader = ((SDPCM_CONTROL_CHANNEL << SDPCM_CHANNEL_SHIFT) & SDPCM_CHANNEL_MASK)
+		        | bus->tx_seq | ((doff << SDPCM_DOFFSET_SHIFT) & SDPCM_DOFFSET_MASK);
+		htol32_ua_store(swheader, frame + SDPCM_FRAMETAG_LEN);
+		htol32_ua_store(0, frame + SDPCM_FRAMETAG_LEN + sizeof(swheader));
+	}
+
+	if (!TXCTLOK(bus))
+	{
+		DHD_INFO(("%s: No bus credit bus->tx_max %d, bus->tx_seq %d\n",
+			__FUNCTION__, bus->tx_max, bus->tx_seq));
+		bus->ctrl_frame_stat = TRUE;
+		/* Send from dpc */
+		bus->ctrl_frame_buf = frame;
+		bus->ctrl_frame_len = len;
+
+#if defined(NDIS)
+		dhd_os_sdunlock(bus->dhd);
+		dhd_wait_for_event(bus->dhd, &bus->ctrl_frame_stat);
+		dhd_os_sdlock(bus->dhd);
+#else
+		if (!bus->dpc_sched) {
+			bus->dpc_sched = TRUE;
+			dhd_sched_dpc(bus->dhd);
+		}
+		if (bus->ctrl_frame_stat) {
+			dhd_wait_for_event(bus->dhd, &bus->ctrl_frame_stat);
+		}
+#endif /* NDIS */
+
+		if (bus->ctrl_frame_stat == FALSE) {
+			DHD_INFO(("%s: ctrl_frame_stat == FALSE\n", __FUNCTION__));
+			ret = 0;
+		} else {
+			bus->dhd->txcnt_timeout++;
+			if (!bus->dhd->hang_was_sent) {
+				DHD_ERROR(("%s: ctrl_frame_stat == TRUE txcnt_timeout=%d\n",
+					__FUNCTION__, bus->dhd->txcnt_timeout));
+#ifdef BCMSDIO_RXLIM_POST
+				DHD_ERROR(("%s: rxlim_en=%d, rxlim enable=%d, rxlim_addr=0x%x\n",
+					__FUNCTION__,
+					bus->dhd->conf->rxlim_en, bus->rxlim_en, bus->rxlim_addr));
+#endif /* BCMSDIO_RXLIM_POST */
+			}
+#ifdef DHD_FW_COREDUMP
+			/* Collect socram dump */
+			if ((bus->dhd->memdump_enabled) &&
+				(bus->dhd->txcnt_timeout >= MAX_CNTL_TX_TIMEOUT)) {
+				/* collect core dump */
+				bus->dhd->memdump_type = DUMP_TYPE_RESUMED_ON_TIMEOUT_TX;
+				dhd_os_sdunlock(bus->dhd);
+				dhd_bus_mem_dump(bus->dhd);
+				dhd_os_sdlock(bus->dhd);
+			}
+#endif /* DHD_FW_COREDUMP */
+			ret = -1;
+			bus->ctrl_frame_stat = FALSE;
+			goto done;
+		}
+	}
+
+	bus->dhd->txcnt_timeout = 0;
+	bus->ctrl_frame_stat = TRUE;
+
+	if (ret == -1) {
+#ifdef DHD_DEBUG
+		if (DHD_BYTES_ON() && DHD_CTL_ON()) {
+			prhex("Tx Frame", frame, len);
+		} else if (DHD_HDRS_ON()) {
+			prhex("TxHdr", frame, MIN(len, 16));
+		}
+#endif
+#ifdef PKT_STATICS
+		bus->tx_statics.ctrl_count++;
+		bus->tx_statics.ctrl_size += len;
+#endif
+		ret = dhd_bcmsdh_send_buffer(bus, frame, len);
+	}
+	bus->ctrl_frame_stat = FALSE;
+
+done:
+	if ((bus->idletime == DHD_IDLE_IMMEDIATE) && !bus->dpc_sched &&
+		NO_OTHER_ACTIVE_BUS_USER(bus)) {
+		bus->activity = FALSE;
+		dhdsdio_bussleep(bus, TRUE);
+		dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+	}
+
+	dhd_os_sdunlock(bus->dhd);
+
+	/* XXX Need to validate return code (ranges) */
+	if (ret)
+		bus->dhd->tx_ctlerrs++;
+	else
+		bus->dhd->tx_ctlpkts++;
+
+	if (bus->dhd->txcnt_timeout >= MAX_CNTL_TX_TIMEOUT) {
+#ifdef DHD_PM_CONTROL_FROM_FILE
+		if (g_pm_control == TRUE) {
+			return -BCME_ERROR;
+		} else {
+		return -ETIMEDOUT;
+		}
+#else
+		return -ETIMEDOUT;
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+	}
+	if (ret == BCME_NODEVICE)
+		err_nodevice++;
+	else
+		err_nodevice = 0;
+
+	return ret ? err_nodevice >= ERROR_BCME_NODEVICE_MAX ? -ETIMEDOUT : -EIO : 0;
+}
+
+int
+dhd_bus_rxctl(struct dhd_bus *bus, uchar *msg, uint msglen)
+{
+	int timeleft;
+	uint rxlen = 0;
+	static uint cnt = 0;
+	uint max_rxcnt;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd->dongle_reset)
+		return -EIO;
+
+	/* Wait until control frame is available */
+	timeleft = dhd_os_ioctl_resp_wait(bus->dhd, &bus->rxlen);
+
+	dhd_os_sdlock(bus->dhd);
+	rxlen = bus->rxlen;
+	bcopy(bus->rxctl, msg, MIN(msglen, rxlen));
+	bus->rxlen = 0;
+	dhd_os_sdunlock(bus->dhd);
+
+	if (bus->dhd->conf->ctrl_resched > 0 && !rxlen && timeleft == 0) {
+		cnt++;
+		if (cnt <= bus->dhd->conf->ctrl_resched) {
+			uint32 status, retry = 0;
+			R_SDREG(status, &bus->regs->intstatus, retry);
+			if ((status & I_HMB_HOST_INT) || PKT_AVAILABLE(bus, status)) {
+				DHD_ERROR(("%s: reschedule dhd_dpc, cnt=%d, status=0x%x\n",
+					__FUNCTION__, cnt, status));
+				bus->ipend = TRUE;
+				bus->dpc_sched = TRUE;
+				dhd_sched_dpc(bus->dhd);
+
+				/* Wait until control frame is available */
+				timeleft = dhd_os_ioctl_resp_wait(bus->dhd, &bus->rxlen);
+
+				dhd_os_sdlock(bus->dhd);
+				rxlen = bus->rxlen;
+				bcopy(bus->rxctl, msg, MIN(msglen, rxlen));
+				bus->rxlen = 0;
+				dhd_os_sdunlock(bus->dhd);
+			}
+		}
+	} else {
+		cnt = 0;
+	}
+
+	if (rxlen) {
+		DHD_CTL(("%s: resumed on rxctl frame, got %d expected %d\n",
+			__FUNCTION__, rxlen, msglen));
+	} else {
+		if (timeleft == 0) {
+#ifdef DHD_DEBUG
+			uint32 status, retry = 0;
+			R_SDREG(status, &bus->regs->intstatus, retry);
+			DHD_ERROR(("%s: resumed on timeout, INT status=0x%08X\n",
+				__FUNCTION__, status));
+#else
+			DHD_ERROR(("%s: resumed on timeout\n", __FUNCTION__));
+#endif /* DHD_DEBUG */
+			if (!bus->dhd->dongle_trap_occured) {
+#ifdef DHD_FW_COREDUMP
+				bus->dhd->memdump_type = DUMP_TYPE_RESUMED_ON_TIMEOUT;
+#endif /* DHD_FW_COREDUMP */
+				dhd_os_sdlock(bus->dhd);
+				dhdsdio_checkdied(bus, NULL, 0);
+				dhd_os_sdunlock(bus->dhd);
+			}
+		} else {
+			DHD_CTL(("%s: resumed for unknown reason?\n", __FUNCTION__));
+			if (!bus->dhd->dongle_trap_occured) {
+#ifdef DHD_FW_COREDUMP
+				bus->dhd->memdump_type = DUMP_TYPE_RESUMED_UNKNOWN;
+#endif /* DHD_FW_COREDUMP */
+				dhd_os_sdlock(bus->dhd);
+				dhdsdio_checkdied(bus, NULL, 0);
+				dhd_os_sdunlock(bus->dhd);
+			}
+		}
+#ifdef DHD_FW_COREDUMP
+		/* Dump the ram image */
+		if (bus->dhd->memdump_enabled && !bus->dhd->dongle_trap_occured)
+			dhdsdio_mem_dump(bus);
+#endif /* DHD_FW_COREDUMP */
+	}
+	if (timeleft == 0) {
+		if (rxlen == 0)
+			bus->dhd->rxcnt_timeout++;
+		DHD_ERROR(("%s: rxcnt_timeout=%d, rxlen=%d\n", __FUNCTION__,
+			bus->dhd->rxcnt_timeout, rxlen));
+#ifdef DHD_FW_COREDUMP
+		/* collect socram dump */
+		if (bus->dhd->memdump_enabled) {
+			bus->dhd->memdump_type = DUMP_TYPE_RESUMED_ON_TIMEOUT_RX;
+			dhd_bus_mem_dump(bus->dhd);
+		}
+#endif /* DHD_FW_COREDUMP */
+	} else {
+		bus->dhd->rxcnt_timeout = 0;
+	}
+
+	if (rxlen)
+		bus->dhd->rx_ctlpkts++;
+	else
+		bus->dhd->rx_ctlerrs++;
+
+	if (bus->dhd->conf->rxcnt_timeout)
+		max_rxcnt = bus->dhd->conf->rxcnt_timeout;
+	else
+		max_rxcnt = MAX_CNTL_RX_TIMEOUT;
+	if (bus->dhd->rxcnt_timeout >= max_rxcnt) {
+#ifdef DHD_PM_CONTROL_FROM_FILE
+		if (g_pm_control == TRUE) {
+			return -BCME_ERROR;
+		} else {
+			return -ETIMEDOUT;
+		}
+#else
+		return -ETIMEDOUT;
+#endif /* DHD_PM_CONTROL_FROM_FILE */
+	}
+	if (bus->dhd->dongle_trap_occured)
+		return -EREMOTEIO;
+
+	return rxlen ? (int)rxlen : -EIO; /* XXX Returns EIO error  */
+}
+
+/* IOVar table */
+enum {
+	IOV_INTR = 1,
+	IOV_POLLRATE,
+	IOV_SDREG,
+	IOV_SBREG,
+	IOV_SDCIS,
+#ifdef DHD_BUS_MEM_ACCESS
+	IOV_MEMBYTES,
+#endif /* DHD_BUS_MEM_ACCESS */
+	IOV_RAMSIZE,
+	IOV_RAMSTART,
+#ifdef DHD_DEBUG
+	IOV_CHECKDIED,
+	IOV_SERIALCONS,
+#endif /* DHD_DEBUG */
+	IOV_SET_DOWNLOAD_STATE,
+	IOV_SET_DOWNLOAD_INFO,
+	IOV_SOCRAM_STATE,
+	IOV_FORCEEVEN,
+	IOV_SDIOD_DRIVE,
+	IOV_READAHEAD,
+	IOV_SDRXCHAIN,
+	IOV_ALIGNCTL,
+	IOV_SDALIGN,
+	IOV_DEVRESET,
+	IOV_CPU,
+#if defined(USE_SDIOFIFO_IOVAR)
+	IOV_WATERMARK,
+	IOV_MESBUSYCTRL,
+#endif /* USE_SDIOFIFO_IOVAR */
+#ifdef SDTEST
+	IOV_PKTGEN,
+	IOV_EXTLOOP,
+#endif /* SDTEST */
+	IOV_SPROM,
+	IOV_TXBOUND,
+	IOV_RXBOUND,
+	IOV_TXMINMAX,
+	IOV_IDLETIME,
+	IOV_IDLECLOCK,
+	IOV_SD1IDLE,
+	IOV_SLEEP,
+	IOV_DONGLEISOLATION,
+	IOV_KSO,
+	IOV_DEVSLEEP,
+	IOV_DEVCAP,
+	IOV_VARS,
+#ifdef SOFTAP
+	IOV_FWPATH,
+#endif
+	IOV_TXGLOMSIZE,
+	IOV_TXGLOMMODE,
+	IOV_HANGREPORT,
+	IOV_TXINRX_THRES,
+	IOV_SDIO_SUSPEND
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+	IOV_GDB_SERVER,  /**< starts gdb server on given interface */
+#endif /* DEBUGGER || DHD_DSCOPE */
+};
+
+const bcm_iovar_t dhdsdio_iovars[] = {
+	{"intr",	IOV_INTR,	0, 0,	IOVT_BOOL,	0 },
+	{"sleep",	IOV_SLEEP,	0, 0,	IOVT_BOOL,	0 },
+	{"pollrate",	IOV_POLLRATE,	0, 0,	IOVT_UINT32,	0 },
+	{"idletime",	IOV_IDLETIME,	0, 0,	IOVT_INT32,	0 },
+	{"idleclock",	IOV_IDLECLOCK,	0, 0,	IOVT_INT32,	0 },
+	{"sd1idle",	IOV_SD1IDLE,	0, 0,	IOVT_BOOL,	0 },
+#ifdef DHD_BUS_MEM_ACCESS
+	{"membytes",	IOV_MEMBYTES,	0, 0,	IOVT_BUFFER,	2 * sizeof(int) },
+#endif /* DHD_BUS_MEM_ACCESS */
+	{"ramsize",	IOV_RAMSIZE,	0, 0,	IOVT_UINT32,	0 },
+	{"ramstart",	IOV_RAMSTART,	0, 0,	IOVT_UINT32,	0 },
+	{"dwnldstate",	IOV_SET_DOWNLOAD_STATE,	0, 0,	IOVT_BOOL,	0 },
+	{"dwnldinfo",   IOV_SET_DOWNLOAD_INFO,  0,      0, IOVT_BUFFER,
+	sizeof(fw_download_info_t) },
+	{"socram_state",	IOV_SOCRAM_STATE,	0, 0,	IOVT_BOOL,	0 },
+	{"vars",	IOV_VARS,	0, 0,	IOVT_BUFFER,	0 },
+	{"sdiod_drive",	IOV_SDIOD_DRIVE, 0, 0,	IOVT_UINT32,	0 },
+	{"readahead",	IOV_READAHEAD,	0, 0,	IOVT_BOOL,	0 },
+	{"sdrxchain",	IOV_SDRXCHAIN,	0, 0,	IOVT_BOOL,	0 },
+	{"alignctl",	IOV_ALIGNCTL,	0, 0,	IOVT_BOOL,	0 },
+	{"sdalign",	IOV_SDALIGN,	0, 0,	IOVT_BOOL,	0 },
+	{"devreset",	IOV_DEVRESET,	0, 0,	IOVT_BOOL,	0 },
+#ifdef DHD_DEBUG
+	{"sdreg",	IOV_SDREG,	0, 0,	IOVT_BUFFER,	sizeof(sdreg_t) },
+	{"sbreg",	IOV_SBREG,	0, 0,	IOVT_BUFFER,	sizeof(sdreg_t) },
+	{"sd_cis",	IOV_SDCIS,	0, 0,	IOVT_BUFFER,	DHD_IOCTL_MAXLEN },
+	{"forcealign",	IOV_FORCEEVEN,	0, 0,	IOVT_BOOL,	0 },
+	{"txbound",	IOV_TXBOUND,	0, 0,	IOVT_UINT32,	0 },
+	{"rxbound",	IOV_RXBOUND,	0, 0,	IOVT_UINT32,	0 },
+	{"txminmax",	IOV_TXMINMAX,	0, 0,	IOVT_UINT32,	0 },
+	{"cpu",		IOV_CPU,	0, 0,	IOVT_BOOL,	0 },
+#ifdef DHD_DEBUG
+	{"checkdied",	IOV_CHECKDIED,	0, 0,	IOVT_BUFFER,	0 },
+	{"serial",	IOV_SERIALCONS,	0, 0,	IOVT_UINT32,	0 },
+#endif /* DHD_DEBUG  */
+#endif /* DHD_DEBUG */
+#if defined(DHD_SPROM)
+	{"sprom",	IOV_SPROM,	0, 0,	IOVT_BUFFER,	2 * sizeof(int) },
+#endif
+#ifdef SDTEST
+	{"extloop",	IOV_EXTLOOP,	0, 0,	IOVT_BOOL,	0 },
+	{"pktgen",	IOV_PKTGEN,	0, 0,	IOVT_BUFFER,	sizeof(dhd_pktgen_t) },
+#endif /* SDTEST */
+#if defined(USE_SDIOFIFO_IOVAR)
+	{"watermark",	IOV_WATERMARK,	0, 0,	IOVT_UINT32,	0 },
+	{"mesbusyctrl",	IOV_MESBUSYCTRL,	0, 0,	IOVT_UINT32,	0 },
+#endif /* USE_SDIOFIFO_IOVAR */
+	{"devcap", IOV_DEVCAP,	0, 0,	IOVT_UINT32,	0 },
+	{"dngl_isolation", IOV_DONGLEISOLATION,	0, 0,	IOVT_UINT32,	0 },
+	{"kso",	IOV_KSO,	0, 0,	IOVT_UINT32,	0 },
+	{"devsleep", IOV_DEVSLEEP,	0, 0,	IOVT_UINT32,	0 },
+#ifdef SOFTAP
+	{"fwpath", IOV_FWPATH, 0, 0, IOVT_BUFFER, 0 },
+#endif
+	{"txglomsize", IOV_TXGLOMSIZE, 0, 0, IOVT_UINT32, 0 },
+	{"fw_hang_report", IOV_HANGREPORT, 0, 0, IOVT_BOOL, 0 },
+	{"txinrx_thres", IOV_TXINRX_THRES, 0, 0, IOVT_INT32, 0 },
+	{"sdio_suspend", IOV_SDIO_SUSPEND, 0, 0, IOVT_UINT32, 0 },
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+	{"gdb_server", IOV_GDB_SERVER,    0, 0,      IOVT_UINT32,    0 },
+#endif /* DEBUGGER || DHD_DSCOPE */
+	{NULL, 0, 0, 0, 0, 0 }
+};
+
+static void
+dhd_dump_pct(struct bcmstrbuf *strbuf, char *desc, uint num, uint div)
+{
+	uint q1, q2;
+
+	if (!div) {
+		bcm_bprintf(strbuf, "%s N/A", desc);
+	} else {
+		q1 = num / div;
+		q2 = (100 * (num - (q1 * div))) / div;
+		bcm_bprintf(strbuf, "%s %d.%02d", desc, q1, q2);
+	}
+}
+
+void
+dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf)
+{
+	dhd_bus_t *bus = dhdp->bus;
+#if defined(DHD_WAKE_STATUS) && defined(DHD_WAKE_EVENT_STATUS)
+	int i;
+#endif
+
+#if defined(FW_SIGNATURE)
+	/* Dump secure firmware status. */
+	if (dhdp->busstate <= DHD_BUS_LOAD) {
+		dhd_bus_dump_fws(dhdp->bus, strbuf);
+	}
+#endif /* FW_SIGNATURE */
+	bcm_bprintf(strbuf, "Bus SDIO structure:\n");
+	bcm_bprintf(strbuf, "hostintmask 0x%08x intstatus 0x%08x sdpcm_ver %d\n",
+	            bus->hostintmask, bus->intstatus, bus->sdpcm_ver);
+	bcm_bprintf(strbuf, "fcstate %d qlen %u tx_seq %d, max %d, rxskip %d rxlen %u rx_seq %d\n",
+	            bus->fcstate, pktq_n_pkts_tot(&bus->txq), bus->tx_seq, bus->tx_max, bus->rxskip,
+	            bus->rxlen, bus->rx_seq);
+	bcm_bprintf(strbuf, "intr %d intrcount %u lastintrs %u spurious %u\n",
+	            bus->intr, bus->intrcount, bus->lastintrs, bus->spurious);
+
+#ifdef DHD_WAKE_STATUS
+	bcm_bprintf(strbuf, "wake %u rxwake %u readctrlwake %u\n",
+		bcmsdh_get_total_wake(bus->sdh), bus->wake_counts.rxwake,
+		bus->wake_counts.rcwake);
+#ifdef DHD_WAKE_RX_STATUS
+	bcm_bprintf(strbuf, " unicast %u multicast %u broadcast %u arp %u\n",
+		bus->wake_counts.rx_ucast, bus->wake_counts.rx_mcast,
+		bus->wake_counts.rx_bcast, bus->wake_counts.rx_arp);
+	bcm_bprintf(strbuf, " multi4 %u multi6 %u icmp %u icmp6 %u multiother %u\n",
+		bus->wake_counts.rx_multi_ipv4, bus->wake_counts.rx_multi_ipv6,
+		bus->wake_counts.rx_icmp, bus->wake_counts.rx_icmpv6,
+		bus->wake_counts.rx_multi_other);
+	bcm_bprintf(strbuf, " icmp6_ra %u, icmp6_na %u, icmp6_ns %u\n",
+		bus->wake_counts.rx_icmpv6_ra, bus->wake_counts.rx_icmpv6_na,
+		bus->wake_counts.rx_icmpv6_ns);
+#endif /* DHD_WAKE_RX_STATUS */
+#ifdef DHD_WAKE_EVENT_STATUS
+#ifdef CUSTOM_WAKE_REASON_STATS
+	bcm_bprintf(strbuf, "rc_event_idx = %d, which indicates queue head\n",
+		bus->wake_counts.rc_event_idx);
+	for (i = 0; i < MAX_WAKE_REASON_STATS; i++)
+		if (bus->wake_counts.rc_event[i] != -1)
+			bcm_bprintf(strbuf, " %u = %s\n", i,
+				bcmevent_get_name(bus->wake_counts.rc_event[i]));
+#else
+	for (i = 0; i < WLC_E_LAST; i++)
+		if (bus->wake_counts.rc_event[i] != 0)
+			bcm_bprintf(strbuf, " %s = %u\n", bcmevent_get_name(i),
+				bus->wake_counts.rc_event[i]);
+#endif /* CUSTOM_WAKE_REASON_STATS */
+	bcm_bprintf(strbuf, "\n");
+#endif /* DHD_WAKE_EVENT_STATUS */
+#endif /* DHD_WAKE_STATUS */
+
+	bcm_bprintf(strbuf, "pollrate %u pollcnt %u regfails %u\n",
+	            bus->pollrate, bus->pollcnt, bus->regfails);
+
+	bcm_bprintf(strbuf, "\nAdditional counters:\n");
+#ifdef DHDENABLE_TAILPAD
+	bcm_bprintf(strbuf, "tx_tailpad_chain %u tx_tailpad_pktget %u\n",
+	            bus->tx_tailpad_chain, bus->tx_tailpad_pktget);
+#endif /* DHDENABLE_TAILPAD */
+	bcm_bprintf(strbuf, "tx_sderrs %u fcqueued %u rxrtx %u rx_toolong %u rxc_errors %u\n",
+	            bus->tx_sderrs, bus->fcqueued, bus->rxrtx, bus->rx_toolong,
+	            bus->rxc_errors);
+	bcm_bprintf(strbuf, "rx_hdrfail %u badhdr %u badseq %u\n",
+	            bus->rx_hdrfail, bus->rx_badhdr, bus->rx_badseq);
+	bcm_bprintf(strbuf, "fc_rcvd %u, fc_xoff %u, fc_xon %u\n",
+	            bus->fc_rcvd, bus->fc_xoff, bus->fc_xon);
+	bcm_bprintf(strbuf, "rxglomfail %u, rxglomframes %u, rxglompkts %u\n",
+	            bus->rxglomfail, bus->rxglomframes, bus->rxglompkts);
+	bcm_bprintf(strbuf, "f2rx (hdrs/data) %u (%u/%u), f2tx %u f1regs %u\n",
+	            (bus->f2rxhdrs + bus->f2rxdata), bus->f2rxhdrs, bus->f2rxdata,
+	            bus->f2txdata, bus->f1regdata);
+	{
+		dhd_dump_pct(strbuf, "\nRx: pkts/f2rd", bus->dhd->rx_packets,
+		             (bus->f2rxhdrs + bus->f2rxdata));
+		dhd_dump_pct(strbuf, ", pkts/f1sd", bus->dhd->rx_packets, bus->f1regdata);
+		dhd_dump_pct(strbuf, ", pkts/sd", bus->dhd->rx_packets,
+		             (bus->f2rxhdrs + bus->f2rxdata + bus->f1regdata));
+		dhd_dump_pct(strbuf, ", pkts/int", bus->dhd->rx_packets, bus->intrcount);
+		bcm_bprintf(strbuf, "\n");
+
+		dhd_dump_pct(strbuf, "Rx: glom pct", (100 * bus->rxglompkts),
+		             bus->dhd->rx_packets);
+		dhd_dump_pct(strbuf, ", pkts/glom", bus->rxglompkts, bus->rxglomframes);
+		bcm_bprintf(strbuf, "\n");
+
+		dhd_dump_pct(strbuf, "Tx: pkts/f2wr", bus->dhd->tx_packets, bus->f2txdata);
+		dhd_dump_pct(strbuf, ", pkts/f1sd", bus->dhd->tx_packets, bus->f1regdata);
+		dhd_dump_pct(strbuf, ", pkts/sd", bus->dhd->tx_packets,
+		             (bus->f2txdata + bus->f1regdata));
+		dhd_dump_pct(strbuf, ", pkts/int", bus->dhd->tx_packets, bus->intrcount);
+		bcm_bprintf(strbuf, "\n");
+
+		dhd_dump_pct(strbuf, "Total: pkts/f2rw",
+		             (bus->dhd->tx_packets + bus->dhd->rx_packets),
+		             (bus->f2txdata + bus->f2rxhdrs + bus->f2rxdata));
+		dhd_dump_pct(strbuf, ", pkts/f1sd",
+		             (bus->dhd->tx_packets + bus->dhd->rx_packets), bus->f1regdata);
+		dhd_dump_pct(strbuf, ", pkts/sd",
+		             (bus->dhd->tx_packets + bus->dhd->rx_packets),
+		             (bus->f2txdata + bus->f2rxhdrs + bus->f2rxdata + bus->f1regdata));
+		dhd_dump_pct(strbuf, ", pkts/int",
+		             (bus->dhd->tx_packets + bus->dhd->rx_packets), bus->intrcount);
+		bcm_bprintf(strbuf, "\n\n");
+	}
+
+#ifdef SDTEST
+	/* XXX Add new stats, include pktq len */
+	if (bus->pktgen_count) {
+		bcm_bprintf(strbuf, "pktgen config and count:\n");
+		bcm_bprintf(strbuf, "freq %u count %u print %u total %u min %u len %u\n",
+		            bus->pktgen_freq, bus->pktgen_count, bus->pktgen_print,
+		            bus->pktgen_total, bus->pktgen_minlen, bus->pktgen_maxlen);
+		bcm_bprintf(strbuf, "send attempts %u rcvd %u fail %u\n",
+		            bus->pktgen_sent, bus->pktgen_rcvd, bus->pktgen_fail);
+	}
+#endif /* SDTEST */
+#ifdef DHD_DEBUG
+	bcm_bprintf(strbuf, "dpc_sched %d host interrupt%spending\n",
+	            bus->dpc_sched, (bcmsdh_intr_pending(bus->sdh) ? " " : " not "));
+	bcm_bprintf(strbuf, "blocksize %u roundup %u\n", bus->blocksize, bus->roundup);
+#endif /* DHD_DEBUG */
+	bcm_bprintf(strbuf, "clkstate %d activity %d idletime %d idlecount %d sleeping %d\n",
+	            bus->clkstate, bus->activity, bus->idletime, bus->idlecount, bus->sleeping);
+	dhd_dump_pct(strbuf, "Tx: glom pct", (100 * bus->txglompkts), bus->dhd->tx_packets);
+	dhd_dump_pct(strbuf, ", pkts/glom", bus->txglompkts, bus->txglomframes);
+	bcm_bprintf(strbuf, "\n");
+	bcm_bprintf(strbuf, "txglomframes %u, txglompkts %u\n", bus->txglomframes, bus->txglompkts);
+	bcm_bprintf(strbuf, "\n");
+}
+
+void
+dhd_bus_clearcounts(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = (dhd_bus_t *)dhdp->bus;
+
+	bus->intrcount = bus->lastintrs = bus->spurious = bus->regfails = 0;
+	bus->rxrtx = bus->rx_toolong = bus->rxc_errors = 0;
+	bus->rx_hdrfail = bus->rx_badhdr = bus->rx_badseq = 0;
+#ifdef DHDENABLE_TAILPAD
+	bus->tx_tailpad_chain = bus->tx_tailpad_pktget = 0;
+#endif /* DHDENABLE_TAILPAD */
+	bus->tx_sderrs = bus->fc_rcvd = bus->fc_xoff = bus->fc_xon = 0;
+	bus->rxglomfail = bus->rxglomframes = bus->rxglompkts = 0;
+	bus->f2rxhdrs = bus->f2rxdata = bus->f2txdata = bus->f1regdata = 0;
+	bus->txglomframes = bus->txglompkts = 0;
+}
+
+#ifdef SDTEST
+static int
+dhdsdio_pktgen_get(dhd_bus_t *bus, uint8 *arg)
+{
+	dhd_pktgen_t pktgen;
+
+	pktgen.version = DHD_PKTGEN_VERSION;
+	pktgen.freq = bus->pktgen_freq;
+	pktgen.count = bus->pktgen_count;
+	pktgen.print = bus->pktgen_print;
+	pktgen.total = bus->pktgen_total;
+	pktgen.minlen = bus->pktgen_minlen;
+	pktgen.maxlen = bus->pktgen_maxlen;
+	pktgen.numsent = bus->pktgen_sent;
+	pktgen.numrcvd = bus->pktgen_rcvd;
+	pktgen.numfail = bus->pktgen_fail;
+	pktgen.mode = bus->pktgen_mode;
+	pktgen.stop = bus->pktgen_stop;
+
+	bcopy(&pktgen, arg, sizeof(pktgen));
+
+	return 0;
+}
+
+static int
+dhdsdio_pktgen_set(dhd_bus_t *bus, uint8 *arg)
+{
+	dhd_pktgen_t pktgen;
+	uint oldcnt, oldmode;
+
+	bcopy(arg, &pktgen, sizeof(pktgen));
+	if (pktgen.version != DHD_PKTGEN_VERSION)
+		return BCME_BADARG;
+
+	oldcnt = bus->pktgen_count;
+	oldmode = bus->pktgen_mode;
+
+	bus->pktgen_freq = pktgen.freq;
+	bus->pktgen_count = pktgen.count;
+	bus->pktgen_print = pktgen.print;
+	bus->pktgen_total = pktgen.total;
+	bus->pktgen_minlen = pktgen.minlen;
+	bus->pktgen_maxlen = pktgen.maxlen;
+	bus->pktgen_mode = pktgen.mode;
+	bus->pktgen_stop = pktgen.stop;
+
+	bus->pktgen_tick = bus->pktgen_ptick = 0;
+#if defined(LINUX)
+	bus->pktgen_prev_time = jiffies;
+#endif /* LINUX */
+	bus->pktgen_len = MAX(bus->pktgen_len, bus->pktgen_minlen);
+	bus->pktgen_len = MIN(bus->pktgen_len, bus->pktgen_maxlen);
+
+	/* Clear counts for a new pktgen (mode change, or was stopped) */
+	if (bus->pktgen_count && (!oldcnt || oldmode != bus->pktgen_mode)) {
+		bus->pktgen_sent = bus->pktgen_prev_sent = bus->pktgen_rcvd = 0;
+		bus->pktgen_prev_rcvd = bus->pktgen_fail = 0;
+	}
+
+	return 0;
+}
+#endif /* SDTEST */
+
+static int
+dhdsdio_membytes(dhd_bus_t *bus, bool write, uint32 address, uint8 *data, uint size)
+{
+	int bcmerror = 0;
+	uint32 sdaddr;
+	uint dsize;
+	uint8 *pdata;
+
+	/* In remap mode, adjust address beyond socram and redirect
+	 * to devram at SOCDEVRAM_BP_ADDR since remap address > orig_ramsize
+	 * is not backplane accessible
+	 */
+	if (REMAP_ENAB(bus) && REMAP_ISADDR(bus, address)) {
+		address -= bus->orig_ramsize;
+		address += SOCDEVRAM_BP_ADDR;
+	}
+
+	/* Determine initial transfer parameters */
+	sdaddr = address & SBSDIO_SB_OFT_ADDR_MASK;
+	if ((sdaddr + size) & SBSDIO_SBWINDOW_MASK)
+		dsize = (SBSDIO_SB_OFT_ADDR_LIMIT - sdaddr);
+	else
+		dsize = size;
+
+	/* Set the backplane window to include the start address */
+	if ((bcmerror = dhdsdio_set_siaddr_window(bus, address))) {
+		DHD_ERROR(("%s: window change failed\n", __FUNCTION__));
+		goto xfer_done;
+	}
+
+	/* Do the transfer(s) */
+	while (size) {
+		DHD_INFO(("%s: %s %d bytes at offset 0x%08x in window 0x%08x\n",
+		          __FUNCTION__, (write ? "write" : "read"), dsize, sdaddr,
+		          (address & SBSDIO_SBWINDOW_MASK)));
+		if (dsize <= MAX_MEM_BUF) {
+			pdata = bus->membuf;
+			if (write)
+				memcpy(bus->membuf, data, dsize);
+		} else {
+			pdata = data;
+		}
+		if ((bcmerror = bcmsdh_rwdata(bus->sdh, write, sdaddr, pdata, dsize))) {
+			DHD_ERROR(("%s: membytes transfer failed\n", __FUNCTION__));
+			break;
+		}
+		if (dsize <= MAX_MEM_BUF && !write)
+			memcpy(data, bus->membuf, dsize);
+
+		/* Adjust for next transfer (if any) */
+		if ((size -= dsize)) {
+			data += dsize;
+			address += dsize;
+			if ((bcmerror = dhdsdio_set_siaddr_window(bus, address))) {
+				DHD_ERROR(("%s: window change failed\n", __FUNCTION__));
+				break;
+			}
+			sdaddr = 0;
+			dsize = MIN(SBSDIO_SB_OFT_ADDR_LIMIT, size);
+		}
+
+	}
+
+xfer_done:
+	/* Return the window to backplane enumeration space for core access */
+	if (dhdsdio_set_siaddr_window(bus, bcmsdh_cur_sbwad(bus->sdh))) {
+		DHD_ERROR(("%s: FAILED to set window back to 0x%x\n", __FUNCTION__,
+			bcmsdh_cur_sbwad(bus->sdh)));
+	}
+
+	return bcmerror;
+}
+
+#if defined(BCMSDIO_RXLIM_POST) || defined(BCMSDIO_TXSEQ_SYNC)
+static void
+dhdsdio_txseq_sync(dhd_bus_t *bus, sdpcm_shared_t *sh)
+{
+	struct dhd_conf *conf = bus->dhd->conf;
+
+	if (sh->flags & SDPCM_SHARED_RXLIM_POST) {
+#ifdef BCMSDIO_RXLIM_POST
+		if (conf->rxlim_en) {
+			if (sh->msgtrace_addr) {
+				bus->rxlim_en = TRUE;
+				bus->rxlim_addr = sh->msgtrace_addr;
+				DHD_INFO(("%s: RXLIM_POST enabled with rxlim_addr=0x%x\n",
+					__FUNCTION__, bus->rxlim_addr));
+			} else {
+				DHD_INFO(("%s: RXLIM_POST not enabled in fw\n", __FUNCTION__));
+			}
+		} else
+#endif /* BCMSDIO_RXLIM_POST */
+#ifdef BCMSDIO_TXSEQ_SYNC
+		if (conf->txseq_sync) {
+			uint8 val = 0;
+			sh->txseq_sync_addr = ltoh32(sh->txseq_sync_addr);
+			DHD_INFO(("%s: TXSEQ_SYNC enabled\n", __FUNCTION__));
+			if (0 == dhdsdio_membytes(bus, FALSE, sh->txseq_sync_addr, (uint8 *)&val, 1)) {
+				if (bus->tx_seq != val) {
+					DHD_INFO(("%s: Sync tx_seq from %d to %d\n",
+						__FUNCTION__, bus->tx_seq, val));
+					bus->tx_seq = val;
+					bus->tx_max = bus->tx_seq + 4;
+				}
+			}
+		} else
+#endif /* BCMSDIO_TXSEQ_SYNC */
+		{
+			DHD_INFO(("%s: rxlim_en and txseq_sync not enabled in config.txt\n", __FUNCTION__));
+		}
+		sh->flags &= ~SDPCM_SHARED_RXLIM_POST;
+	}
+	else {
+#ifdef BCMSDIO_RXLIM_POST
+		bus->rxlim_en = 0;
+#endif /* BCMSDIO_RXLIM_POST */
+#ifdef BCMSDIO_TXSEQ_SYNC
+		conf->txseq_sync = FALSE;
+#endif /* BCMSDIO_TXSEQ_SYNC */
+		DHD_INFO(("%s: TXSEQ_SYNC and RXLIM_POST not supported in fw\n", __FUNCTION__));
+	}
+}
+#endif /* BCMSDIO_RXLIM_POST || BCMSDIO_TXSEQ_SYNC */
+
+static int
+dhdsdio_readshared(dhd_bus_t *bus, sdpcm_shared_t *sh)
+{
+	uint32 addr;
+	int rv, i;
+	uint32 shaddr = 0;
+
+	if (bus->sih == NULL) {
+		if (bus->dhd && bus->dhd->dongle_reset) {
+			DHD_ERROR(("%s: Dongle is in reset state\n", __FUNCTION__));
+			return BCME_NOTREADY;
+		} else {
+			ASSERT(bus->dhd);
+			ASSERT(bus->sih);
+			DHD_ERROR(("%s: The address of sih is invalid\n", __FUNCTION__));
+			return BCME_ERROR;
+		}
+	}
+	/*
+	 * If SR is not implemented in 43430 FW we should not adjust shaddr
+	 * XXX Should be REMOVED after SR will be implemented in 43430 FW
+	 */
+	if ((CHIPID(bus->sih->chip) == BCM43430_CHIP_ID ||
+		CHIPID(bus->sih->chip) == BCM43018_CHIP_ID) && !dhdsdio_sr_cap(bus))
+		bus->srmemsize = 0;
+
+	/* Fix first time get console address failed issue */
+	if ((CHIPID(bus->sih->chip) == BCM43430_CHIP_ID) && dhdsdio_sr_cap(bus) &&
+		(bus->srmemsize == 0)) {
+		if (dhd_srmem) {
+			bus->srmemsize = dhd_srmem;
+		} else {
+			/* 43436/8 default sr size is 64K */
+			bus->srmemsize = 0x10000;
+		}
+	}
+
+	shaddr = bus->dongle_ram_base + bus->ramsize - 4;
+	i = 0;
+	do {
+		/* Read last word in memory to determine address of sdpcm_shared structure */
+		if ((rv = dhdsdio_membytes(bus, FALSE, shaddr, (uint8 *)&addr, 4)) < 0)
+			return rv;
+
+		addr = ltoh32(addr);
+
+		DHD_INFO(("sdpcm_shared shaddr %x addr 0x%08X\n", shaddr, addr));
+
+		/*
+		 * Check if addr is valid.
+		 * NVRAM length at the end of memory should have been overwritten.
+		 */
+		if (addr == 0 || ((~addr >> 16) & 0xffff) == (addr & 0xffff)) {
+			if ((bus->srmemsize > 0) && (i++ == 0)) {
+				shaddr -= bus->srmemsize;
+			} else {
+#ifdef BCMQT
+				if (bus->console_addr) {
+					dhdsdio_readconsole(bus);
+				} else {
+					if (dhdsdio_force_console_dump(bus))
+						DHD_ERROR(("%s: FW console has been initialized\n",
+							__func__));
+				}
+#endif /* BCMQT */
+				DHD_ERROR(("%s: address (0x%08x) of sdpcm_shared invalid\n",
+					__FUNCTION__, addr));
+				return BCME_ERROR;
+			}
+		} else
+			break;
+	} while (i < 2);
+
+	/* Read hndrte_shared structure */
+	if ((rv = dhdsdio_membytes(bus, FALSE, addr, (uint8 *)sh, sizeof(sdpcm_shared_t))) < 0)
+		return rv;
+
+	/* Endianness */
+	sh->flags = ltoh32(sh->flags);
+	sh->trap_addr = ltoh32(sh->trap_addr);
+	sh->assert_exp_addr = ltoh32(sh->assert_exp_addr);
+	sh->assert_file_addr = ltoh32(sh->assert_file_addr);
+	sh->assert_line = ltoh32(sh->assert_line);
+	sh->console_addr = ltoh32(sh->console_addr);
+	sh->msgtrace_addr = ltoh32(sh->msgtrace_addr);
+
+#if defined(BCMSDIO_RXLIM_POST) || defined(BCMSDIO_TXSEQ_SYNC)
+	dhdsdio_txseq_sync(bus, sh);
+#endif
+
+	/*
+	 * XXX - Allow a sdpcm_shared_t version mismatch between dhd structure
+	 * version 1 and firmware structure version 3.
+	 * The sdpcm_shared_t stucture fields used in this function are in the
+	 * same positions in these two structure versions.
+	 * For some chips in the FALCON release, the dhd driver is from the
+	 * FALCON branch (sdpcm_shared_t structure version 1) and the firmware
+	 * comes from the ROMTERM3 branch (sdpcm_shared_t structure version 1).
+	*/
+	if ((sh->flags & SDPCM_SHARED_VERSION_MASK) == 3 && SDPCM_SHARED_VERSION == 1)
+		return BCME_OK;
+
+	if ((sh->flags & SDPCM_SHARED_VERSION_MASK) != SDPCM_SHARED_VERSION) {
+		DHD_ERROR(("%s: sdpcm_shared version %d in dhd "
+		           "is different than sdpcm_shared version %d in dongle\n",
+		           __FUNCTION__, SDPCM_SHARED_VERSION,
+		           sh->flags & SDPCM_SHARED_VERSION_MASK));
+		return BCME_ERROR;
+	}
+
+#if defined(FW_SIGNATURE)
+	if ((bus->fwsig_filename[0] != 0)) {
+		bl_verif_status_t status;
+
+		(void)dhdsdio_read_fwstatus(bus, &status);
+		DHD_ERROR(("Verification status: (%08x)\n"
+			"\tstatus: %d\n"
+			"\tstate: %u\n"
+			"\talloc_bytes: %u\n"
+			"\tmax_alloc_bytes: %u\n"
+			"\ttotal_alloc_bytes: %u\n"
+			"\ttotal_freed_bytes: %u\n"
+			"\tnum_allocs: %u\n"
+			"\tmax_allocs: %u\n"
+			"\tmax_alloc_size: %u\n"
+			"\talloc_failures: %u\n",
+			bus->fwstat_download_addr,
+			status.status,
+			status.state,
+			status.alloc_bytes,
+			status.max_alloc_bytes,
+			status.total_alloc_bytes,
+			status.total_freed_bytes,
+			status.num_allocs,
+			status.max_allocs,
+			status.max_alloc_size,
+			status.alloc_failures));
+	}
+#endif /* FW_SIGNATURE */
+	return BCME_OK;
+}
+
+void
+dhd_bus_check_srmemsize(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	uint32 srmem_size = 0;
+	int err = BCME_OK;
+
+	if ((CHIPID(bus->sih->chip) == BCM43430_CHIP_ID) && dhdsdio_sr_cap(bus)) {
+		if (dhd_srmem) {
+			bus->srmemsize = dhd_srmem;
+		} else {
+			/* 43436/8 default sr size is 64K */
+			bus->srmemsize = 0x10000;
+		}
+		DHD_INFO(("%s srmem size is set %x\n", __func__, bus->srmemsize));
+	}
+
+	err = dhd_iovar(dhdp, 0, "bus:srmem_size", NULL, 0,
+		(char *)&srmem_size, sizeof(srmem_size), FALSE);
+	if (err) {
+		DHD_INFO(("%s : srmem_size no need to change.\n", __FUNCTION__));
+		return;
+	}
+
+	if (srmem_size != bus->srmemsize) {
+		sdpcm_shared_t shared;
+		if (err == BCME_OK)
+			bus->srmemsize = srmem_size;
+		dhd_os_sdlock(dhdp);
+		if (dhdsdio_readshared(bus, &shared) == 0)
+			bus->console_addr = shared.console_addr;
+		dhd_os_sdunlock(dhdp);
+	}
+
+	return;
+}
+
+#define CONSOLE_LINE_MAX	192
+
+#ifdef DHD_DEBUG
+#ifdef BCMQT
+static bool dhdsdio_force_console_dump(dhd_bus_t *bus)
+{
+	uint32 cons_sign = 0;
+	uint32 cons_addr = 0;
+	uint32 val = 0;
+
+	cons_sign = bus->dongle_ram_base + bus->ramsize - 12;
+
+	/* Read last word in memory to determine address of sdpcm_shared structure */
+	if (dhdsdio_membytes(bus, FALSE, cons_sign, (uint8 *)&val, 4) < 0)
+		return FALSE;
+
+	val = ltoh32(val);
+
+	/* try again if srmemsize is set */
+	if (val != CONS_ADDR_SIGNATURE) {
+		if (bus->srmemsize > 0) {
+			cons_sign -= bus->srmemsize;
+			if (dhdsdio_membytes(bus, FALSE, cons_sign, (uint8 *)&val, 4) < 0) {
+				return FALSE;
+			}
+			val = ltoh32(val);
+		}
+	}
+
+	if (val == CONS_ADDR_SIGNATURE) {
+		cons_addr = cons_sign + 4;
+		DHD_ERROR(("%s: Got correct console address signature\n", __func__));
+
+		/* Read last word in memory to determine address of sdpcm_shared structure */
+		if (dhdsdio_membytes(bus, FALSE, cons_addr, (uint8 *)&val, 4) < 0)
+			return FALSE;
+
+		bus->console_addr = ltoh32(val);
+		dhd_msg_level |= DHD_FWLOG_VAL;
+		dhdsdio_readconsole(bus);
+		return TRUE;
+	}
+	return FALSE;
+}
+#endif /* BCMQT */
+
+static int
+dhdsdio_readconsole(dhd_bus_t *bus)
+{
+	dhd_console_t *c = &bus->console;
+	uint8 line[CONSOLE_LINE_MAX], ch;
+	uint32 n, idx, addr;
+	int rv;
+
+	/* Don't do anything until FWREADY updates console address */
+	if (bus->console_addr == 0)
+		return 0;
+
+	if (!KSO_ENAB(bus))
+		return 0;
+
+	/* Read console log struct */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, log);
+	if ((rv = dhdsdio_membytes(bus, FALSE, addr, (uint8 *)&c->log, sizeof(c->log))) < 0)
+		return rv;
+
+	/* Allocate console buffer (one time only) */
+	if (c->buf == NULL) {
+		c->bufsize = ltoh32(c->log.buf_size);
+		if ((c->buf = MALLOC(bus->dhd->osh, c->bufsize)) == NULL)
+			return BCME_NOMEM;
+	}
+
+	idx = ltoh32(c->log.idx);
+
+	/* Protect against corrupt value */
+	if (idx > c->bufsize)
+		return BCME_ERROR;
+
+	/* Skip reading the console buffer if the index pointer has not moved */
+	if (idx == c->last)
+		return BCME_OK;
+
+	/* Read the console buffer */
+	/* xxx this could optimize and read only the portion of the buffer needed, but
+	 * it would also have to handle wrap-around.
+	 */
+	addr = ltoh32(c->log.buf);
+	if ((rv = dhdsdio_membytes(bus, FALSE, addr, c->buf, c->bufsize)) < 0)
+		return rv;
+
+	while (c->last != idx) {
+		for (n = 0; n < CONSOLE_LINE_MAX - 2; n++) {
+			if (c->last == idx) {
+				/* This would output a partial line.  Instead, back up
+				 * the buffer pointer and output this line next time around.
+				 */
+				if (c->last >= n)
+					c->last -= n;
+				else
+					c->last = c->bufsize - n;
+				goto break2;
+			}
+			ch = c->buf[c->last];
+			c->last = (c->last + 1) % c->bufsize;
+			if (ch == '\n')
+				break;
+			line[n] = ch;
+		}
+
+		if (n > 0) {
+			if (line[n - 1] == '\r')
+				n--;
+			line[n] = 0;
+			printf("CONSOLE: %s\n", line);
+#ifdef LOG_INTO_TCPDUMP
+			dhd_sendup_log(bus->dhd, line, n);
+#endif /* LOG_INTO_TCPDUMP */
+		}
+	}
+break2:
+
+	return BCME_OK;
+}
+#endif /* DHD_DEBUG */
+
+static int
+dhdsdio_checkdied(dhd_bus_t *bus, char *data, uint size)
+{
+	int bcmerror = 0;
+	uint msize = 512;
+	char *mbuffer = NULL;
+	char *console_buffer = NULL;
+	uint maxstrlen = 256;
+	char *str = NULL;
+	sdpcm_shared_t l_sdpcm_shared;
+	struct bcmstrbuf strbuf;
+	uint32 console_ptr, console_size, console_index;
+	uint8 line[CONSOLE_LINE_MAX], ch;
+	uint32 n, i, addr;
+	int rv;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (DHD_NOCHECKDIED_ON())
+		return 0;
+
+	if (data == NULL) {
+		/*
+		 * Called after a rx ctrl timeout. "data" is NULL.
+		 * allocate memory to trace the trap or assert.
+		 */
+		size = msize;
+		mbuffer = data = MALLOC(bus->dhd->osh, msize);
+		if (mbuffer == NULL) {
+			DHD_ERROR(("%s: MALLOC(%d) failed \n", __FUNCTION__, msize));
+			bcmerror = BCME_NOMEM;
+			goto done;
+		}
+	}
+
+	if ((str = MALLOC(bus->dhd->osh, maxstrlen)) == NULL) {
+		DHD_ERROR(("%s: MALLOC(%d) failed \n", __FUNCTION__, maxstrlen));
+		bcmerror = BCME_NOMEM;
+		goto done;
+	}
+
+	if ((bcmerror = dhdsdio_readshared(bus, &l_sdpcm_shared)) < 0)
+		goto done;
+
+	bcm_binit(&strbuf, data, size);
+
+	bcm_bprintf(&strbuf, "msgtrace address : 0x%08X\nconsole address  : 0x%08X\n",
+	            l_sdpcm_shared.msgtrace_addr, l_sdpcm_shared.console_addr);
+
+	if ((l_sdpcm_shared.flags & SDPCM_SHARED_ASSERT_BUILT) == 0) {
+		/* NOTE: Misspelled assert is intentional - DO NOT FIX.
+		 * (Avoids conflict with real asserts for programmatic parsing of output.)
+		 */
+		bcm_bprintf(&strbuf, "Assrt not built in dongle\n");
+	}
+
+	if ((l_sdpcm_shared.flags & (SDPCM_SHARED_ASSERT|SDPCM_SHARED_TRAP)) == 0) {
+		/* NOTE: Misspelled assert is intentional - DO NOT FIX.
+		 * (Avoids conflict with real asserts for programmatic parsing of output.)
+		 */
+		bcm_bprintf(&strbuf, "No trap%s in dongle",
+		          (l_sdpcm_shared.flags & SDPCM_SHARED_ASSERT_BUILT)
+		          ?"/assrt" :"");
+	} else {
+		if (l_sdpcm_shared.flags & SDPCM_SHARED_ASSERT) {
+			/* Download assert */
+			bcm_bprintf(&strbuf, "Dongle assert");
+			if (l_sdpcm_shared.assert_exp_addr != 0) {
+				str[0] = '\0';
+				if ((bcmerror = dhdsdio_membytes(bus, FALSE,
+				                                 l_sdpcm_shared.assert_exp_addr,
+				                                 (uint8 *)str, maxstrlen)) < 0)
+					goto done;
+
+				str[maxstrlen - 1] = '\0';
+				bcm_bprintf(&strbuf, " expr \"%s\"", str);
+			}
+
+			if (l_sdpcm_shared.assert_file_addr != 0) {
+				str[0] = '\0';
+				if ((bcmerror = dhdsdio_membytes(bus, FALSE,
+				                   l_sdpcm_shared.assert_file_addr,
+				                                 (uint8 *)str, maxstrlen)) < 0)
+					goto done;
+
+				str[maxstrlen - 1] = '\0';
+				bcm_bprintf(&strbuf, " file \"%s\"", str);
+			}
+
+			bcm_bprintf(&strbuf, " line %d ", l_sdpcm_shared.assert_line);
+		}
+
+		if (l_sdpcm_shared.flags & SDPCM_SHARED_TRAP) {
+			trap_t *tr = &bus->dhd->last_trap_info;
+			bus->dhd->dongle_trap_occured = TRUE;
+			if ((bcmerror = dhdsdio_membytes(bus, FALSE,
+			                                 l_sdpcm_shared.trap_addr,
+			                                 (uint8*)tr, sizeof(trap_t))) < 0)
+				goto done;
+
+			bus->dongle_trap_addr = ltoh32(l_sdpcm_shared.trap_addr);
+
+			dhd_bus_dump_trap_info(bus, &strbuf);
+
+			addr = l_sdpcm_shared.console_addr + OFFSETOF(hnd_cons_t, log);
+			if ((rv = dhdsdio_membytes(bus, FALSE, addr,
+				(uint8 *)&console_ptr, sizeof(console_ptr))) < 0)
+				goto printbuf;
+
+			addr = l_sdpcm_shared.console_addr + OFFSETOF(hnd_cons_t, log.buf_size);
+			if ((rv = dhdsdio_membytes(bus, FALSE, addr,
+				(uint8 *)&console_size, sizeof(console_size))) < 0)
+				goto printbuf;
+
+			addr = l_sdpcm_shared.console_addr + OFFSETOF(hnd_cons_t, log.idx);
+			if ((rv = dhdsdio_membytes(bus, FALSE, addr,
+				(uint8 *)&console_index, sizeof(console_index))) < 0)
+				goto printbuf;
+
+			console_ptr = ltoh32(console_ptr);
+			console_size = ltoh32(console_size);
+			console_index = ltoh32(console_index);
+
+			if (console_size > CONSOLE_BUFFER_MAX ||
+				!(console_buffer = MALLOC(bus->dhd->osh, console_size)))
+				goto printbuf;
+
+			if ((rv = dhdsdio_membytes(bus, FALSE, console_ptr,
+				(uint8 *)console_buffer, console_size)) < 0)
+				goto printbuf;
+
+			for (i = 0, n = 0; i < console_size; i += n + 1) {
+				for (n = 0; n < CONSOLE_LINE_MAX - 2; n++) {
+					ch = console_buffer[(console_index + i + n) % console_size];
+					if (ch == '\n')
+						break;
+					line[n] = ch;
+				}
+
+				if (n > 0) {
+					if (line[n - 1] == '\r')
+						n--;
+					line[n] = 0;
+					/* Don't use DHD_ERROR macro since we print
+					 * a lot of information quickly. The macro
+					 * will truncate a lot of the printfs
+					 */
+
+					if (dhd_msg_level & DHD_ERROR_VAL)
+						printf("CONSOLE: %s\n", line);
+				}
+			}
+		}
+	}
+
+printbuf:
+	if (l_sdpcm_shared.flags & (SDPCM_SHARED_ASSERT | SDPCM_SHARED_TRAP)) {
+		DHD_ERROR(("%s: %s\n", __FUNCTION__, strbuf.origbuf));
+	}
+
+#if defined(DHD_FW_COREDUMP)
+	if (bus->dhd->memdump_enabled && (l_sdpcm_shared.flags & SDPCM_SHARED_TRAP)) {
+		/* Mem dump to a file on device */
+		bus->dhd->memdump_type = DUMP_TYPE_DONGLE_TRAP;
+		/* xxx this sdunlock has been put as a WAR here. We tried to come up
+		  * with a better solution but with the current structure of sdlocks it is very
+		  * unlikely to have a better fix for now. The better Rearch of sdio bus
+		  * locking has been put up as a cleanup activity and a thorough
+		  * code walkthrough is needed.
+		  */
+		dhd_os_sdunlock(bus->dhd);
+		dhdsdio_mem_dump(bus);
+		dhd_os_sdlock(bus->dhd);
+#ifdef NDIS
+		/* Windows would like to crash and collect memory dump for analysis */
+		ASSERT(0 && "Dongle firmware died.");
+
+		/* For free drivers ASSERT will not bugcheck */
+		KeBugCheckEx(__LINE__, 0, 0, 0, 0);
+#endif
+	}
+#endif /* #if defined(DHD_FW_COREDUMP) */
+
+done:
+	if (mbuffer)
+		MFREE(bus->dhd->osh, mbuffer, msize);
+	if (str)
+		MFREE(bus->dhd->osh, str, maxstrlen);
+	if (console_buffer)
+		MFREE(bus->dhd->osh, console_buffer, console_size);
+
+	return bcmerror;
+}
+
+#if defined(DHD_FW_COREDUMP)
+int
+dhd_bus_mem_dump(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	if (dhdp->busstate == DHD_BUS_SUSPEND) {
+		DHD_ERROR(("%s: Bus is suspend so skip\n", __FUNCTION__));
+		return 0;
+	}
+	return dhdsdio_mem_dump(bus);
+}
+
+int
+dhd_bus_get_mem_dump(dhd_pub_t *dhdp)
+{
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return BCME_ERROR;
+	}
+
+	return dhdsdio_get_mem_dump(dhdp->bus);
+}
+
+static int
+dhdsdio_get_mem_dump(dhd_bus_t *bus)
+{
+	int ret = BCME_ERROR;
+	int size = bus->ramsize;		/* Full mem size */
+	uint32 start = bus->dongle_ram_base;	/* Start address */
+	uint read_size = 0;			/* Read size of each iteration */
+	uint8 *p_buf = NULL, *databuf = NULL;
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	dhd_dongledump_status_t dump_status;
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	/* Get full mem size */
+	p_buf = dhd_get_fwdump_buf(bus->dhd, size);
+	if (!p_buf) {
+		DHD_ERROR(("%s: Out of memory (%d bytes)\n",
+			__FUNCTION__, size));
+		ret = BCME_ERROR;
+		goto exit;
+	}
+
+	dhd_os_sdlock(bus->dhd);
+	BUS_WAKE(bus);
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	dump_status = dhd_get_dump_status(bus->dhd);
+	if (dump_status != DUMP_IN_PROGRESS) {
+		dhd_set_dump_status(bus->dhd, DUMP_IN_PROGRESS);
+	}
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	/* Read mem content */
+	DHD_ERROR(("Dump dongle memory\n"));
+	databuf = p_buf;
+	while (size) {
+		read_size = MIN(MEMBLOCK, size);
+		ret = dhdsdio_membytes(bus, FALSE, start, databuf, read_size);
+		if (ret) {
+			DHD_ERROR(("%s: Error membytes %d\n", __FUNCTION__, ret));
+			ret = BCME_ERROR;
+			break;
+		}
+		/* Decrement size and increment start address */
+		size -= read_size;
+		start += read_size;
+		databuf += read_size;
+	}
+
+	if ((bus->idletime == DHD_IDLE_IMMEDIATE) && !bus->dpc_sched &&
+		NO_OTHER_ACTIVE_BUS_USER(bus)) {
+		bus->activity = FALSE;
+		dhdsdio_clkctl(bus, CLK_NONE, TRUE);
+	}
+
+exit:
+#if defined(DHD_FILE_DUMP_EVENT) && defined(DHD_FW_COREDUMP)
+	if (ret != BCME_OK) {
+		dhd_set_dump_status(bus->dhd, DUMP_FAILURE);
+	}
+#endif /* DHD_FILE_DUMP_EVENT && DHD_FW_COREDUMP */
+
+	dhd_os_sdunlock(bus->dhd);
+
+	return ret;
+}
+
+static int
+dhdsdio_mem_dump(dhd_bus_t *bus)
+{
+	dhd_pub_t *dhdp;
+	int ret = BCME_ERROR;
+
+	dhdp = bus->dhd;
+	if (!dhdp) {
+		DHD_ERROR(("%s: dhdp is NULL\n", __FUNCTION__));
+		return ret;
+	}
+
+	ret = dhdsdio_get_mem_dump(bus);
+	if (ret) {
+		DHD_ERROR(("%s: failed to get mem dump, err=%d\n",
+			__FUNCTION__, ret));
+	} else {
+		/* schedule a work queue to perform actual memdump.
+		 * dhd_mem_dump() performs the job
+		 */
+		dhd_schedule_memdump(dhdp, dhdp->soc_ram, dhdp->soc_ram_length);
+		/* soc_ram free handled in dhd_{free,clear} */
+	}
+
+	return ret;
+}
+#endif /* DHD_FW_COREDUMP */
+
+int
+dhd_socram_dump(dhd_bus_t * bus)
+{
+#if defined(DHD_FW_COREDUMP)
+	return (dhdsdio_mem_dump(bus));
+#else
+	return -1;
+#endif
+}
+
+int
+dhdsdio_downloadvars(dhd_bus_t *bus, void *arg, int len)
+{
+	int bcmerror = BCME_OK;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd->up &&
+		1) {
+		bcmerror = BCME_NOTDOWN;
+		goto err;
+	}
+	if (!len) {
+		bcmerror = BCME_BUFTOOSHORT;
+		goto err;
+	}
+
+	/* Free the old ones and replace with passed variables */
+	if (bus->vars)
+		MFREE(bus->dhd->osh, bus->vars, bus->varsz);
+
+	bus->vars = MALLOC(bus->dhd->osh, len);
+	bus->varsz = bus->vars ? len : 0;
+	if (bus->vars == NULL) {
+		bcmerror = BCME_NOMEM;
+		goto err;
+	}
+
+	/* Copy the passed variables, which should include the terminating double-null */
+	bcopy(arg, bus->vars, bus->varsz);
+err:
+	return bcmerror;
+}
+
+#ifdef DHD_DEBUG
+static int
+dhd_serialconsole(dhd_bus_t *bus, bool set, bool enable, int *bcmerror)
+{
+	int int_val;
+	uint32 addr, data, uart_enab = 0;
+
+	addr = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, chipcontrol_addr);
+	data = SI_ENUM_BASE(bus->sih) + OFFSETOF(chipcregs_t, chipcontrol_data);
+	*bcmerror = 0;
+
+	bcmsdh_reg_write(bus->sdh, addr, 4, 1);
+	if (bcmsdh_regfail(bus->sdh)) {
+		*bcmerror = BCME_SDIO_ERROR;
+		return -1;
+	}
+	int_val = bcmsdh_reg_read(bus->sdh, data, 4);
+	if (bcmsdh_regfail(bus->sdh)) {
+		*bcmerror = BCME_SDIO_ERROR;
+		return -1;
+	}
+
+	if (!set)
+		return (int_val & uart_enab);
+	if (enable)
+		int_val |= uart_enab;
+	else
+		int_val &= ~uart_enab;
+	bcmsdh_reg_write(bus->sdh, data, 4, int_val);
+	if (bcmsdh_regfail(bus->sdh)) {
+		*bcmerror = BCME_SDIO_ERROR;
+		return -1;
+	}
+
+	return (int_val & uart_enab);
+}
+#endif /* DHD_DEBUG */
+
+static int
+dhdsdio_doiovar(dhd_bus_t *bus, const bcm_iovar_t *vi, uint32 actionid, const char *name,
+                void *params, uint plen, void *arg, uint len, uint val_size)
+{
+	int bcmerror = 0;
+	int32 int_val = 0;
+	bool bool_val = 0;
+
+	DHD_TRACE(("%s: Enter, action %d name %s params %p plen %d arg %p len %d val_size %d\n",
+	           __FUNCTION__, actionid, name, params, plen, arg, len, val_size));
+
+	if ((bcmerror = bcm_iovar_lencheck(vi, arg, len, IOV_ISSET(actionid))) != 0)
+		goto exit;
+
+	if (plen >= sizeof(int_val))
+		bcopy(params, &int_val, sizeof(int_val));
+
+	bool_val = (int_val != 0) ? TRUE : FALSE;
+
+	/* Some ioctls use the bus */
+	dhd_os_sdlock(bus->dhd);
+
+	/* Check if dongle is in reset. If so, only allow DEVRESET iovars */
+	if (bus->dhd->dongle_reset && !(actionid == IOV_SVAL(IOV_DEVRESET) ||
+	                                actionid == IOV_GVAL(IOV_DEVRESET))) {
+		bcmerror = BCME_NOTREADY;
+		goto exit;
+	}
+
+	/*
+	 * Special handling for keepSdioOn: New SDIO Wake-up Mechanism
+	 */
+	if ((vi->varid == IOV_KSO) && (IOV_ISSET(actionid))) {
+		dhdsdio_clk_kso_iovar(bus, bool_val);
+		goto exit;
+	} else if ((vi->varid == IOV_DEVSLEEP) && (IOV_ISSET(actionid))) {
+		{
+			dhdsdio_clk_devsleep_iovar(bus, bool_val);
+			if (!SLPAUTO_ENAB(bus) && (bool_val == FALSE) && (bus->ipend)) {
+				DHD_ERROR(("INT pending in devsleep 1, dpc_sched: %d\n",
+					bus->dpc_sched));
+				if (!bus->dpc_sched) {
+					bus->dpc_sched = TRUE;
+					dhd_sched_dpc(bus->dhd);
+				}
+			}
+		}
+		goto exit;
+	}
+
+	/* Handle sleep stuff before any clock mucking */
+	if (vi->varid == IOV_SLEEP) {
+		if (IOV_ISSET(actionid)) {
+			bcmerror = dhdsdio_bussleep(bus, bool_val);
+		} else {
+			int_val = (int32)bus->sleeping;
+			bcopy(&int_val, arg, val_size);
+		}
+		goto exit;
+	}
+
+	/* Request clock to allow SDIO accesses */
+	if (!bus->dhd->dongle_reset) {
+		BUS_WAKE(bus);
+		dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+	}
+
+	switch (actionid) {
+	case IOV_GVAL(IOV_INTR):
+		int_val = (int32)bus->intr;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_INTR):
+		bus->intr = bool_val;
+		bus->intdis = FALSE;
+		if (bus->dhd->up) {
+			if (bus->intr) {
+				DHD_INTR(("%s: enable SDIO device interrupts\n", __FUNCTION__));
+				// terence 20141207: enbale intdis
+				bus->intdis = TRUE;
+				bcmsdh_intr_enable(bus->sdh);
+			} else {
+				DHD_INTR(("%s: disable SDIO interrupts\n", __FUNCTION__));
+				bcmsdh_intr_disable(bus->sdh);
+			}
+		}
+		break;
+
+	case IOV_GVAL(IOV_POLLRATE):
+		int_val = (int32)bus->pollrate;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_POLLRATE):
+		bus->pollrate = (uint)int_val;
+		bus->poll = (bus->pollrate != 0);
+		break;
+
+	case IOV_GVAL(IOV_IDLETIME):
+		int_val = bus->idletime;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_IDLETIME):
+		if ((int_val < 0) && (int_val != DHD_IDLE_IMMEDIATE)) {
+			bcmerror = BCME_BADARG;
+		} else {
+			bus->idletime = int_val;
+		}
+		break;
+
+	case IOV_GVAL(IOV_IDLECLOCK):
+		int_val = (int32)bus->idleclock;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_IDLECLOCK):
+		bus->idleclock = int_val;
+		break;
+
+	case IOV_GVAL(IOV_SD1IDLE):
+		int_val = (int32)sd1idle;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SD1IDLE):
+		sd1idle = bool_val;
+		break;
+
+#ifdef DHD_DEBUG
+	case IOV_GVAL(IOV_CHECKDIED):
+		bcmerror = dhdsdio_checkdied(bus, arg, len);
+		break;
+#endif /* DHD_DEBUG */
+
+#ifdef DHD_BUS_MEM_ACCESS
+	case IOV_SVAL(IOV_MEMBYTES):
+	case IOV_GVAL(IOV_MEMBYTES):
+	{
+		uint32 address;
+		uint size, dsize;
+		uint8 *data;
+
+		bool set = (actionid == IOV_SVAL(IOV_MEMBYTES));
+
+		ASSERT(plen >= 2*sizeof(int));
+
+		address = (uint32)int_val;
+		bcopy((char *)params + sizeof(int_val), &int_val, sizeof(int_val));
+		size = (uint)int_val;
+
+		/* Do some validation */
+		dsize = set ? plen - (2 * sizeof(int)) : len;
+		if (dsize < size) {
+			DHD_ERROR(("%s: error on %s membytes, addr 0x%08x size %d dsize %d\n",
+			           __FUNCTION__, (set ? "set" : "get"), address, size, dsize));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		DHD_INFO(("%s: Request to %s %d bytes at address 0x%08x\n", __FUNCTION__,
+		          (set ? "write" : "read"), size, address));
+
+		/* check if CR4 */
+		if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			/*
+			 * If address is start of RAM (i.e. a downloaded image),
+			 * store the reset instruction to be written in 0
+			 */
+			if (set && address == bus->dongle_ram_base) {
+				bus->resetinstr = *(((uint32*)params) + 2);
+			}
+		}
+
+		/* Generate the actual data pointer */
+		data = set ? (uint8*)params + 2 * sizeof(int): (uint8*)arg;
+
+		/* Call to do the transfer */
+		bcmerror = dhdsdio_membytes(bus, set, address, data, size);
+
+		break;
+	}
+#endif /* DHD_BUS_MEM_ACCESS */
+
+#if defined(FW_SIGNATURE)
+	case IOV_SVAL(IOV_SET_DOWNLOAD_INFO):
+	{
+		fw_download_info_t *info = (fw_download_info_t*)params;
+		bcmerror = dhdsdio_bus_save_download_info(bus,
+				info->fw_start_addr, info->fw_size, info->fw_signature_fname,
+				info->bootloader_fname, info->bootloader_start_addr);
+		break;
+	}
+#endif /* FW_SIGNATURE */
+
+	case IOV_GVAL(IOV_RAMSIZE):
+		int_val = (int32)bus->ramsize;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_RAMSTART):
+		int_val = (int32)bus->dongle_ram_base;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_SDIOD_DRIVE):
+		int_val = (int32)dhd_sdiod_drive_strength;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SDIOD_DRIVE):
+		dhd_sdiod_drive_strength = int_val;
+		si_sdiod_drive_strength_init(bus->sih, bus->dhd->osh, dhd_sdiod_drive_strength);
+		break;
+
+	case IOV_SVAL(IOV_SET_DOWNLOAD_STATE):
+		bcmerror = dhdsdio_download_state(bus, bool_val);
+		break;
+
+	case IOV_SVAL(IOV_SOCRAM_STATE):
+		bcmerror = dhdsdio_download_state(bus, bool_val);
+		break;
+
+	case IOV_SVAL(IOV_VARS):
+		bcmerror = dhdsdio_downloadvars(bus, arg, len);
+		break;
+
+	case IOV_GVAL(IOV_READAHEAD):
+		int_val = (int32)dhd_readahead;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_READAHEAD):
+		if (bool_val && !dhd_readahead)
+			bus->nextlen = 0;
+		dhd_readahead = bool_val;
+		break;
+
+	case IOV_GVAL(IOV_SDRXCHAIN):
+		int_val = (int32)bus->use_rxchain;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SDRXCHAIN):
+		if (bool_val && !bus->sd_rxchain)
+			bcmerror = BCME_UNSUPPORTED;
+		else
+			bus->use_rxchain = bool_val;
+		break;
+#ifndef BCMSPI
+	case IOV_GVAL(IOV_ALIGNCTL):
+		int_val = (int32)dhd_alignctl;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_ALIGNCTL):
+		dhd_alignctl = bool_val;
+		break;
+#endif /* BCMSPI */
+
+	case IOV_GVAL(IOV_SDALIGN):
+		int_val = DHD_SDALIGN;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+#ifdef DHD_DEBUG
+	case IOV_GVAL(IOV_VARS):
+		if (bus->varsz < (uint)len)
+			bcopy(bus->vars, arg, bus->varsz);
+		else
+			bcmerror = BCME_BUFTOOSHORT;
+		break;
+#endif /* DHD_DEBUG */
+
+#ifdef DHD_DEBUG
+	/* XXX Until these return BCME ranges, make assumptions here */
+	case IOV_GVAL(IOV_SDREG):
+	{
+		sdreg_t *sd_ptr;
+		uintptr addr;
+		uint size;
+
+		sd_ptr = (sdreg_t *)params;
+
+		addr = ((uintptr)bus->regs + sd_ptr->offset);
+		size = sd_ptr->func;
+		int_val = (int32)bcmsdh_reg_read(bus->sdh, addr, size);
+		if (bcmsdh_regfail(bus->sdh))
+			bcmerror = BCME_SDIO_ERROR;
+		bcopy(&int_val, arg, sizeof(int32));
+		break;
+	}
+
+	case IOV_SVAL(IOV_SDREG):
+	{
+		sdreg_t *sd_ptr;
+		uintptr addr;
+		uint size;
+
+		sd_ptr = (sdreg_t *)params;
+
+		addr = ((uintptr)bus->regs + sd_ptr->offset);
+		size = sd_ptr->func;
+		bcmsdh_reg_write(bus->sdh, addr, size, sd_ptr->value);
+		if (bcmsdh_regfail(bus->sdh))
+			bcmerror = BCME_SDIO_ERROR;
+		break;
+	}
+
+	/* XXX Same as above */
+	/* Same as above, but offset is not backplane (not SDIO core) */
+	case IOV_GVAL(IOV_SBREG):
+	{
+		sdreg_t sdreg;
+		uint32 addr, size;
+
+		bcopy(params, &sdreg, sizeof(sdreg));
+
+		addr = SI_ENUM_BASE(bus->sih) + sdreg.offset;
+		size = sdreg.func;
+		int_val = (int32)bcmsdh_reg_read(bus->sdh, addr, size);
+		if (bcmsdh_regfail(bus->sdh))
+			bcmerror = BCME_SDIO_ERROR;
+		bcopy(&int_val, arg, sizeof(int32));
+		break;
+	}
+
+	case IOV_SVAL(IOV_SBREG):
+	{
+		sdreg_t sdreg;
+		uint32 addr, size;
+
+		bcopy(params, &sdreg, sizeof(sdreg));
+
+		addr = SI_ENUM_BASE(bus->sih) + sdreg.offset;
+		size = sdreg.func;
+		bcmsdh_reg_write(bus->sdh, addr, size, sdreg.value);
+		if (bcmsdh_regfail(bus->sdh))
+			bcmerror = BCME_SDIO_ERROR;
+		break;
+	}
+
+	case IOV_GVAL(IOV_SDCIS):
+	{
+		*(char *)arg = 0;
+
+		/* XXX Ignoring return codes, should be evident from printed results */
+		bcmstrcat(arg, "\nFunc 0\n");
+		bcmsdh_cis_read(bus->sdh, 0x10, (uint8 *)arg + strlen(arg), SBSDIO_CIS_SIZE_LIMIT);
+		bcmstrcat(arg, "\nFunc 1\n");
+		bcmsdh_cis_read(bus->sdh, 0x11, (uint8 *)arg + strlen(arg), SBSDIO_CIS_SIZE_LIMIT);
+		bcmstrcat(arg, "\nFunc 2\n");
+		bcmsdh_cis_read(bus->sdh, 0x12, (uint8 *)arg + strlen(arg), SBSDIO_CIS_SIZE_LIMIT);
+		break;
+	}
+
+	case IOV_GVAL(IOV_FORCEEVEN):
+		int_val = (int32)forcealign;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_FORCEEVEN):
+		forcealign = bool_val;
+		break;
+
+	case IOV_GVAL(IOV_TXBOUND):
+		int_val = (int32)dhd_txbound;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_TXBOUND):
+		dhd_txbound = (uint)int_val;
+		break;
+
+	case IOV_GVAL(IOV_RXBOUND):
+		int_val = (int32)dhd_rxbound;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_RXBOUND):
+		dhd_rxbound = (uint)int_val;
+		break;
+
+	case IOV_GVAL(IOV_TXMINMAX):
+		int_val = (int32)dhd_txminmax;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_TXMINMAX):
+		dhd_txminmax = (uint)int_val;
+		break;
+
+#ifdef DHD_DEBUG
+	case IOV_GVAL(IOV_SERIALCONS):
+		int_val = dhd_serialconsole(bus, FALSE, 0, &bcmerror);
+		if (bcmerror != 0)
+			break;
+
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SERIALCONS):
+		dhd_serialconsole(bus, TRUE, bool_val, &bcmerror);
+		break;
+#endif /* DHD_DEBUG */
+
+#endif /* DHD_DEBUG */
+
+#if defined(DHD_SPROM)
+	case IOV_SVAL(IOV_SPROM):
+	case IOV_GVAL(IOV_SPROM):
+	{
+		uint32 offset;
+		uint size, dsize;
+
+		bool set = (actionid == IOV_SVAL(IOV_SPROM));
+
+		ASSERT(plen >= 2*sizeof(int));
+
+		offset = (uint32)int_val;
+		bcopy((char *)params + sizeof(int_val), &int_val, sizeof(int_val));
+		size = (uint)int_val;
+
+		/* Avoid bigger size of srom reads that may be requested from app.
+		 * gSPI has only F1 OTP visible from CC.  There is no CIS in gSPI.
+		 */
+		if (bus->bus == SPI_BUS)
+			size = SBSDIO_CIS_SIZE_LIMIT;
+
+		/* Do some validation */
+		dsize = set ? plen - (2 * sizeof(int)) : len;
+		if (dsize < size) {
+			DHD_ERROR(("%s: error on srom %s, addr 0x%08x size %d dsize %d\n",
+			           __FUNCTION__, (set ? "write" : "read"), offset, size, dsize));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		if ((offset > SROM_MAX) || ((offset + size) > SROM_MAX)) {
+			DHD_ERROR(("%s: error on srom %s, offset %d size %d exceeds limit %d\n",
+			           __FUNCTION__, (set ? "write" : "read"), offset, size, SROM_MAX));
+			bcmerror = BCME_BADARG;
+			break;
+		}
+
+		if (!set) {
+			if (!ISALIGNED((uintptr)arg, sizeof(uint16))) {
+				DHD_ERROR(("%s: srom data pointer %p not word-aligned\n",
+				           __FUNCTION__, arg));
+				bcmerror = BCME_BADARG;
+				break;
+			}
+
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			bcmerror = srom_read(bus->sih, DHD_BUS, (void*)bus->regs, bus->dhd->osh,
+			                     offset, size, (uint16*)arg, FALSE);
+			GCC_DIAGNOSTIC_POP();
+
+		} else {
+			arg = (void*)((uintptr)arg + 2 * sizeof(int));
+			if (!ISALIGNED((uintptr)arg, sizeof(uint16))) {
+				DHD_ERROR(("%s: srom data pointer %p not word-aligned\n",
+				           __FUNCTION__, arg));
+				bcmerror = BCME_BADARG;
+				break;
+			}
+			GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+			bcmerror = srom_write(bus->sih, DHD_BUS, (void*)bus->regs, bus->dhd->osh,
+			                      offset, size, (uint16*)arg);
+			GCC_DIAGNOSTIC_POP();
+		}
+		break;
+	}
+#endif /* DHD_SPROM */
+
+#ifdef SDTEST
+	case IOV_GVAL(IOV_EXTLOOP):
+		int_val = (int32)bus->ext_loop;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_EXTLOOP):
+		bus->ext_loop = bool_val;
+		break;
+
+	case IOV_GVAL(IOV_PKTGEN):
+		bcmerror = dhdsdio_pktgen_get(bus, arg);
+		break;
+
+	case IOV_SVAL(IOV_PKTGEN):
+		bcmerror = dhdsdio_pktgen_set(bus, arg);
+		break;
+#endif /* SDTEST */
+
+#if defined(USE_SDIOFIFO_IOVAR)
+	case IOV_GVAL(IOV_WATERMARK):
+		int_val = (int32)watermark;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_WATERMARK):
+		watermark = (uint)int_val;
+		watermark = (watermark > SBSDIO_WATERMARK_MASK) ? SBSDIO_WATERMARK_MASK : watermark;
+		DHD_ERROR(("Setting watermark as 0x%x.\n", watermark));
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_WATERMARK, (uint8)watermark, NULL);
+		break;
+
+	case IOV_GVAL(IOV_MESBUSYCTRL):
+		int_val = (int32)mesbusyctrl;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_MESBUSYCTRL):
+		mesbusyctrl = (uint)int_val;
+		mesbusyctrl = (mesbusyctrl > SBSDIO_MESBUSYCTRL_MASK)
+			? SBSDIO_MESBUSYCTRL_MASK : mesbusyctrl;
+		DHD_ERROR(("Setting mesbusyctrl as 0x%x.\n", mesbusyctrl));
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_MESBUSYCTRL,
+			((uint8)mesbusyctrl | 0x80), NULL);
+		break;
+#endif /* USE_SDIOFIFO_IOVAR */
+
+	case IOV_GVAL(IOV_DONGLEISOLATION):
+		int_val = bus->dhd->dongle_isolation;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DONGLEISOLATION):
+		bus->dhd->dongle_isolation = bool_val;
+		break;
+
+	case IOV_SVAL(IOV_DEVRESET):
+		DHD_TRACE(("%s: Called set IOV_DEVRESET=%d dongle_reset=%d busstate=%d\n",
+		           __FUNCTION__, bool_val, bus->dhd->dongle_reset,
+		           bus->dhd->busstate));
+
+		ASSERT(bus->dhd->osh);
+		/* ASSERT(bus->cl_devid); */
+
+		/* must release sdlock, since devreset also acquires it */
+		dhd_os_sdunlock(bus->dhd);
+		dhd_bus_devreset(bus->dhd, (uint8)bool_val);
+		dhd_os_sdlock(bus->dhd);
+		break;
+	/*
+	 * softap firmware is updated through module parameter or android private command
+	 */
+
+	case IOV_GVAL(IOV_DEVRESET):
+		DHD_TRACE(("%s: Called get IOV_DEVRESET\n", __FUNCTION__));
+
+		/* Get its status */
+		int_val = (bool) bus->dhd->dongle_reset;
+		bcopy(&int_val, arg, val_size);
+
+		break;
+
+	case IOV_GVAL(IOV_KSO):
+		int_val = dhdsdio_sleepcsr_get(bus);
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_DEVCAP):
+		int_val = dhdsdio_devcap_get(bus);
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_DEVCAP):
+		dhdsdio_devcap_set(bus, (uint8) int_val);
+		break;
+	case IOV_GVAL(IOV_TXGLOMSIZE):
+		int_val = (int32)bus->txglomsize;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_TXGLOMSIZE):
+		if (int_val > SDPCM_MAXGLOM_SIZE) {
+			bcmerror = BCME_ERROR;
+		} else {
+			bus->txglomsize = (uint)int_val;
+		}
+		break;
+	case IOV_SVAL(IOV_HANGREPORT):
+		bus->dhd->hang_report = bool_val;
+		DHD_ERROR(("%s: Set hang_report as %d\n", __FUNCTION__, bus->dhd->hang_report));
+		break;
+
+	case IOV_GVAL(IOV_HANGREPORT):
+		int_val = (int32)bus->dhd->hang_report;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_GVAL(IOV_TXINRX_THRES):
+		int_val = bus->txinrx_thres;
+		bcopy(&int_val, arg, val_size);
+		break;
+	case IOV_SVAL(IOV_TXINRX_THRES):
+		if (int_val < 0) {
+			bcmerror = BCME_BADARG;
+		} else {
+			bus->txinrx_thres = int_val;
+		}
+		break;
+
+	case IOV_GVAL(IOV_SDIO_SUSPEND):
+		int_val = (bus->dhd->busstate == DHD_BUS_SUSPEND) ? 1 : 0;
+		bcopy(&int_val, arg, val_size);
+		break;
+
+	case IOV_SVAL(IOV_SDIO_SUSPEND):
+		if (bool_val) { /* Suspend */
+			dhdsdio_suspend(bus);
+		}
+		else { /* Resume */
+			dhdsdio_resume(bus);
+		}
+		break;
+
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+	case IOV_SVAL(IOV_GDB_SERVER):
+		if (bool_val == TRUE) {
+			debugger_init((void *) bus, &bus_ops, int_val, SI_ENUM_BASE(bus->sih));
+		} else {
+			debugger_close();
+		}
+		break;
+#endif /* DEBUGGER || DHD_DSCOPE */
+
+	default:
+		bcmerror = BCME_UNSUPPORTED;
+		break;
+	}
+
+exit:
+	if ((bus->idletime == DHD_IDLE_IMMEDIATE) && !bus->dpc_sched &&
+		NO_OTHER_ACTIVE_BUS_USER(bus)) {
+		bus->activity = FALSE;
+		dhdsdio_bussleep(bus, TRUE);
+		dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+	}
+
+	dhd_os_sdunlock(bus->dhd);
+
+	return bcmerror;
+}
+
+static int
+dhdsdio_write_vars(dhd_bus_t *bus)
+{
+	int bcmerror = 0;
+	uint32 varsize, phys_size;
+	uint32 varaddr;
+	uint8 *vbuffer;
+	uint32 varsizew;
+#ifdef DHD_DEBUG
+	uint8 *nvram_ularray;
+#endif /* DHD_DEBUG */
+
+	/* Even if there are no vars are to be written, we still need to set the ramsize. */
+	varsize = bus->varsz ? ROUNDUP(bus->varsz, 4) : 0;
+	varaddr = (bus->ramsize - 4) - varsize;
+
+	// terence 20150412: fix for nvram failed to download
+	if (bus->dhd->conf->chip == BCM43340_CHIP_ID ||
+			bus->dhd->conf->chip == BCM43341_CHIP_ID) {
+		varsize = varsize ? ROUNDUP(varsize, 64) : 0;
+		varaddr = (bus->ramsize - 64) - varsize;
+	}
+
+	varaddr += bus->dongle_ram_base;
+	bus->ramtop_addr = varaddr;
+
+	if (bus->vars) {
+		/* XXX: WAR for PR85623 */
+		if ((bus->sih->buscoretype == SDIOD_CORE_ID) && (bus->sdpcmrev == 7)) {
+			if (((varaddr & 0x3C) == 0x3C) && (varsize > 4)) {
+				DHD_INFO(("PR85623WAR in place\n"));
+				varsize += 4;
+				varaddr -= 4;
+			}
+		}
+
+		/* XXX In case the controller has trouble with odd bytes... */
+		vbuffer = (uint8 *)MALLOC(bus->dhd->osh, varsize);
+		if (!vbuffer)
+			return BCME_NOMEM;
+
+		bzero(vbuffer, varsize);
+		bcopy(bus->vars, vbuffer, bus->varsz);
+
+		/* Write the vars list */
+		bcmerror = dhdsdio_membytes(bus, TRUE, varaddr, vbuffer, varsize);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+				__FUNCTION__, bcmerror, varsize, varaddr));
+			return bcmerror;
+		}
+
+#ifdef DHD_DEBUG
+		/* Verify NVRAM bytes */
+		DHD_INFO(("Compare NVRAM dl & ul; varsize=%d\n", varsize));
+		nvram_ularray = (uint8*)MALLOC(bus->dhd->osh, varsize);
+		if (!nvram_ularray) {
+			MFREE(bus->dhd->osh, vbuffer, varsize);
+			return BCME_NOMEM;
+		}
+
+		/* Upload image to verify downloaded contents. */
+		memset(nvram_ularray, 0xaa, varsize);
+
+		/* Read the vars list to temp buffer for comparison */
+		bcmerror = dhdsdio_membytes(bus, FALSE, varaddr, nvram_ularray, varsize);
+		if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d nvram bytes at 0x%08x\n",
+					__FUNCTION__, bcmerror, varsize, varaddr));
+		}
+		/* Compare the org NVRAM with the one read from RAM */
+		if (memcmp(vbuffer, nvram_ularray, varsize)) {
+			DHD_INFO(("%s: Downloaded NVRAM image is corrupted.\n", __FUNCTION__));
+		} else
+			DHD_INFO(("%s: Download, Upload and compare of NVRAM succeeded.\n",
+			__FUNCTION__));
+
+		MFREE(bus->dhd->osh, nvram_ularray, varsize);
+#endif /* DHD_DEBUG */
+
+		MFREE(bus->dhd->osh, vbuffer, varsize);
+	}
+
+#ifdef MINIME
+	phys_size = bus->ramsize;
+#else
+	phys_size = REMAP_ENAB(bus) ? bus->ramsize : bus->orig_ramsize;
+#endif
+
+	phys_size += bus->dongle_ram_base;
+
+	/* adjust to the user specified RAM */
+	DHD_INFO(("Physical memory size: %d, usable memory size: %d\n",
+		phys_size, bus->ramsize));
+	DHD_INFO(("Vars are at %d, orig varsize is %d\n",
+		varaddr, varsize));
+	varsize = ((phys_size - 4) - varaddr);
+
+	/*
+	 * Determine the length token:
+	 * Varsize, converted to words, in lower 16-bits, checksum in upper 16-bits.
+	 */
+#ifdef DHD_DEBUG
+	if (bcmerror) {
+		varsizew = 0;
+	} else
+#endif /* DHD_DEBUG */
+	{
+		varsizew = varsize / 4;
+		varsizew = (~varsizew << 16) | (varsizew & 0x0000FFFF);
+		varsizew = htol32(varsizew);
+	}
+
+	DHD_INFO(("New varsize is %d, length token=0x%08x\n", varsize, varsizew));
+
+	/* Write the length token to the last word */
+	bcmerror = dhdsdio_membytes(bus, TRUE, (phys_size - 4),
+		(uint8*)&varsizew, 4);
+
+	return bcmerror;
+}
+
+bool
+dhd_bus_is_multibp_capable(struct dhd_bus *bus)
+{
+	return MULTIBP_CAP(bus->sih);
+}
+
+static int
+dhdsdio_download_state(dhd_bus_t *bus, bool enter)
+{
+	uint retries;
+	int bcmerror = 0;
+	int foundcr4 = 0;
+	bool do_wr_flops = TRUE;
+
+	if (!bus->sih)
+		return BCME_ERROR;
+	/* To enter download state, disable ARM and reset SOCRAM.
+	 * To exit download state, simply reset ARM (default is RAM boot).
+	 */
+	if (enter) {
+		bus->alp_only = TRUE;
+
+		if (!(si_setcore(bus->sih, ARM7S_CORE_ID, 0)) &&
+		    !(si_setcore(bus->sih, ARMCM3_CORE_ID, 0))) {
+			if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+				foundcr4 = 1;
+			} else {
+				DHD_ERROR(("%s: Failed to find ARM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+		}
+
+		if (!foundcr4) {
+			si_core_disable(bus->sih, 0);
+			if (bcmsdh_regfail(bus->sdh)) {
+				bcmerror = BCME_SDIO_ERROR;
+				goto fail;
+			}
+
+			if (!(si_setcore(bus->sih, SOCRAM_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find SOCRAM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			si_core_reset(bus->sih, 0, 0);
+			if (bcmsdh_regfail(bus->sdh)) {
+				DHD_ERROR(("%s: Failure trying reset SOCRAM core?\n",
+				           __FUNCTION__));
+				bcmerror = BCME_SDIO_ERROR;
+				goto fail;
+			}
+
+			if (CHIPID(bus->sih->chip) == BCM43430_CHIP_ID ||
+				CHIPID(bus->sih->chip) == BCM43018_CHIP_ID) {
+				/* Disabling Remap for SRAM_3 */
+				si_socram_set_bankpda(bus->sih, 0x3, 0x0);
+			}
+
+			/* Clear the top bit of memory */
+			if (bus->ramsize) {
+				uint32 zeros = 0;
+				if (dhdsdio_membytes(bus, TRUE, bus->ramsize - 4,
+				                     (uint8*)&zeros, 4) < 0) {
+					bcmerror = BCME_SDIO_ERROR;
+					goto fail;
+				}
+			}
+		} else {
+			/* For CR4,
+			 * Halt ARM
+			 * Remove ARM reset
+			 * Read RAM base address [0x18_0000]
+			 * [next] Download firmware
+			 * [done at else] Populate the reset vector
+			 * [done at else] Remove ARM halt
+			*/
+			/* Halt ARM & remove reset */
+			si_core_reset(bus->sih, SICF_CPUHALT, SICF_CPUHALT);
+		}
+	} else {
+		if (!si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			if (!(si_setcore(bus->sih, SOCRAM_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find SOCRAM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			if (!si_iscoreup(bus->sih)) {
+				DHD_ERROR(("%s: SOCRAM core is down after reset?\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+
+			if ((bcmerror = dhdsdio_write_vars(bus))) {
+				DHD_ERROR(("%s: could not write vars to RAM\n", __FUNCTION__));
+				goto fail;
+			}
+
+#ifdef BCMSDIOLITE
+			if (!si_setcore(bus->sih, CC_CORE_ID, 0)) {
+				DHD_ERROR(("%s: Can't set to Chip Common core?\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+#else
+			if (!si_setcore(bus->sih, PCMCIA_CORE_ID, 0) &&
+			    !si_setcore(bus->sih, SDIOD_CORE_ID, 0)) {
+				DHD_ERROR(("%s: Can't change back to SDIO core?\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+#endif
+			W_SDREG(0xFFFFFFFF, &bus->regs->intstatus, retries);
+
+			/* XXX Change standby configuration here if necessary */
+
+			if (!(si_setcore(bus->sih, ARM7S_CORE_ID, 0)) &&
+			    !(si_setcore(bus->sih, ARMCM3_CORE_ID, 0))) {
+				DHD_ERROR(("%s: Failed to find ARM core!\n", __FUNCTION__));
+				bcmerror = BCME_ERROR;
+				goto fail;
+			}
+		} else {
+			/* cr4 has no socram, but tcm's */
+			/* write vars */
+			if ((bcmerror = dhdsdio_write_vars(bus))) {
+				DHD_ERROR(("%s: could not write vars to RAM\n", __FUNCTION__));
+				goto fail;
+			}
+#if defined(FW_SIGNATURE)
+			if ((bcmerror = dhdsdio_bus_download_fw_signature(bus, &do_wr_flops))
+					!= BCME_OK) {
+				goto fail;
+			}
+#endif /* FW_SIGNATURE */
+			if (do_wr_flops) {
+#ifdef BCMSDIOLITE
+				if (!si_setcore(bus->sih, CC_CORE_ID, 0)) {
+					DHD_ERROR(("%s: Can't set to Chip Common core?\n",
+						__FUNCTION__));
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+#else
+				if (!si_setcore(bus->sih, PCMCIA_CORE_ID, 0) &&
+				    !si_setcore(bus->sih, SDIOD_CORE_ID, 0)) {
+					DHD_ERROR(("%s: Can't change back to SDIO core?\n",
+						__FUNCTION__));
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+#endif
+				W_SDREG(0xFFFFFFFF, &bus->regs->intstatus, retries);
+
+				/* switch back to arm core again */
+				if (!(si_setcore(bus->sih, ARMCR4_CORE_ID, 0))) {
+					DHD_ERROR(("%s: Failed to find ARM CR4 core!\n",
+						__FUNCTION__));
+					bcmerror = BCME_ERROR;
+					goto fail;
+				}
+				/* write address 0 with reset instruction */
+				bcmerror = dhdsdio_membytes(bus, TRUE, 0,
+					(uint8 *)&bus->resetinstr, sizeof(bus->resetinstr));
+
+				if (bcmerror == BCME_OK) {
+					uint32 tmp;
+
+					/* verify write */
+					bcmerror = dhdsdio_membytes(bus, FALSE, 0,
+						(uint8 *)&tmp, sizeof(tmp));
+
+					if (bcmerror == BCME_OK && tmp != bus->resetinstr) {
+						DHD_ERROR(("%s: Failed to write 0x%08x to addr 0\n",
+						          __FUNCTION__, bus->resetinstr));
+						DHD_ERROR(("%s: contents of addr 0 is 0x%08x\n",
+						          __FUNCTION__, tmp));
+						bcmerror = BCME_SDIO_ERROR;
+						goto fail;
+					}
+				}
+
+				/* now remove reset and halt and continue to run CR4 */
+			}
+		}
+
+		si_core_reset(bus->sih, 0, 0);
+		if (bcmsdh_regfail(bus->sdh)) {
+			DHD_ERROR(("%s: Failure trying to reset ARM core?\n", __FUNCTION__));
+			bcmerror = BCME_SDIO_ERROR;
+			goto fail;
+		}
+
+		/* Allow HT Clock now that the ARM is running. */
+		bus->alp_only = FALSE;
+
+		bus->dhd->busstate = DHD_BUS_LOAD;
+	}
+
+fail:
+	/* Always return to SDIOD core */
+	if (!si_setcore(bus->sih, PCMCIA_CORE_ID, 0))
+		si_setcore(bus->sih, SDIOD_CORE_ID, 0);
+
+	return bcmerror;
+}
+
+#if defined(FW_SIGNATURE)
+static int
+dhdsdio_bus_download_fw_signature(dhd_bus_t *bus, bool *do_write)
+{
+	int bcmerror = BCME_OK;
+
+	DHD_ERROR(("FWSIG: bl=%s,%x fw=%x,%u sig=%s,%x,%u"
+		" stat=%x,%u ram=%x,%x\n",
+		bus->bootloader_filename, bus->bootloader_addr,
+		bus->fw_download_addr, bus->fw_download_len,
+		bus->fwsig_filename, bus->fwsig_download_addr,
+		bus->fwsig_download_len,
+		bus->fwstat_download_addr, bus->fwstat_download_len,
+		bus->dongle_ram_base, bus->ramtop_addr));
+
+	if (bus->fwsig_filename[0] == 0) {
+		DHD_INFO(("%s: missing signature file\n", __FUNCTION__));
+		goto exit;
+	}
+
+	/* Write RAM Bootloader to TCM if requested */
+	if ((bcmerror = dhdsdio_bus_download_ram_bootloader(bus))
+		!= BCME_OK) {
+		DHD_ERROR(("%s: could not write RAM BL to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write FW signature to memory */
+	if ((bcmerror = dhdsdio_bus_write_fw_signature(bus))) {
+		DHD_ERROR(("%s: could not write FWsig , err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* In case of BL RAM, do write flops */
+	if (bus->bootloader_filename[0] != 0) {
+		*do_write = TRUE;
+	} else {
+		*do_write = FALSE;
+	}
+
+exit:
+	return bcmerror;
+}
+
+/* Complete RAM structure, write the followings
+ * signature image
+ * signature verification status
+ * FW memory map
+ * end-of-TLVs marker
+ */
+static int
+dhdsdio_bus_write_fw_signature(dhd_bus_t *bus)
+{
+	int bcmerror = BCME_OK;
+
+	/* Write FW signature rTLV to TCM */
+	if ((bcmerror = dhdsdio_bus_write_fwsig(bus, bus->fwsig_filename,
+		NULL))) {
+		DHD_ERROR(("%s: could not write FWsig to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write FW signature verification status rTLV to TCM */
+	if ((bcmerror = dhdsdio_bus_write_fws_status(bus)) != BCME_OK) {
+		DHD_ERROR(("%s: could not write FWinfo to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write FW memory map rTLV to TCM */
+	if ((bcmerror = dhdsdio_bus_write_fws_mem_info(bus)) != BCME_OK) {
+		DHD_ERROR(("%s: could not write FWinfo to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Write a end-of-TLVs marker to TCM */
+	if ((bcmerror = dhdsdio_download_rtlv_end(bus)) != BCME_OK) {
+		DHD_ERROR(("%s: could not write rTLV-end marker to TCM, err %d\n",
+			__FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+exit:
+	return bcmerror;
+}
+
+/* Download a reversed-TLV to the top of dongle RAM without overlapping any existing rTLVs */
+static int
+dhdsdio_download_rtlv(dhd_bus_t *bus, dngl_rtlv_type_t type, dngl_rtlv_len_t len, uint8 *value)
+{
+	int bcmerror = BCME_OK;
+#ifdef DHD_DEBUG
+	uint8 *readback_buf = NULL;
+	uint32 readback_val = 0;
+#endif /* DHD_DEBUG */
+	uint32 dest_addr = 0;           /* dongle RAM dest address */
+	uint32 dest_size = 0;           /* dongle RAM dest size */
+	uint32 dest_raw_size = 0;       /* dest size with added checksum */
+
+	/* Calculate the destination dongle RAM address and size */
+	dest_size = ROUNDUP(len, 4);
+	dest_addr = bus->ramtop_addr - sizeof(dngl_rtlv_type_t) - sizeof(dngl_rtlv_len_t)
+		- dest_size;
+	bus->ramtop_addr = dest_addr;
+
+	/* Create the rTLV size field.  This consists of 2 16-bit fields:
+	 * The lower 16 bits is the size.  The higher 16 bits is a checksum
+	 * consisting of the size with all bits reversed.
+	 *     +-------------+-------------+
+	 *     |   checksum  |   size      |
+	 *     +-------------+-------------+
+	 *      High 16 bits    Low 16 bits
+	 */
+	if (type == DNGL_RTLV_TYPE_RNG_SIGNATURE) {
+		/* Random number seed TLV (DNGL_RTLV_TYPE_RNG_SIGNATURE) was the very
+		 * first TLV, its length does not have checksum, firmware crashes if
+		 * checksum is present
+		 */
+		dest_raw_size = dest_size & 0x0000FFFF;
+	} else {
+		dest_raw_size = (~dest_size << 16) | (dest_size & 0x0000FFFF);
+	}
+	/* Write the value block */
+	if (dest_size > 0) {
+		bcmerror = dhdsdio_membytes(bus, TRUE, dest_addr, value, dest_size);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes to 0x%08x\n",
+				__FUNCTION__, bcmerror, dest_size, dest_addr));
+			goto exit;
+		}
+	}
+
+	/* Write the length word */
+	bcmerror = dhdsdio_membytes(bus, TRUE, dest_addr + dest_size,
+		(uint8*)&dest_raw_size, sizeof(dngl_rtlv_len_t));
+
+	/* Write the type word */
+	bcmerror = dhdsdio_membytes(bus, TRUE,
+		dest_addr + dest_size + sizeof(dngl_rtlv_len_t),
+		(uint8*)&type, sizeof(dngl_rtlv_type_t));
+
+#ifdef DHD_DEBUG
+	/* Read back and compare the downloaded data */
+	if (dest_size > 0) {
+		readback_buf = (uint8*)MALLOC(bus->dhd->osh, dest_size);
+		if (!readback_buf) {
+			bcmerror = BCME_NOMEM;
+			goto exit;
+		}
+		memset(readback_buf, 0xaa, dest_size);
+		bcmerror = dhdsdio_membytes(bus, FALSE, dest_addr, readback_buf, dest_size);
+		if (bcmerror) {
+			DHD_ERROR(("%s: readback error %d, %d bytes from 0x%08x\n",
+				__FUNCTION__, bcmerror, dest_size, dest_addr));
+			goto exit;
+		}
+		if (memcmp(value, readback_buf, dest_size) != 0) {
+			DHD_ERROR(("%s: Downloaded data mismatch.\n", __FUNCTION__));
+			bcmerror = BCME_ERROR;
+			goto exit;
+		} else {
+			DHD_ERROR(("Download and compare of TLV 0x%x succeeded"
+				" (size %u, addr %x).\n", type, dest_size, dest_addr));
+		}
+	}
+
+	/* Read back and compare the downloaded len field */
+	bcmerror = dhdsdio_membytes(bus, FALSE, dest_addr + dest_size,
+		(uint8*)&readback_val, sizeof(dngl_rtlv_len_t));
+	if (!bcmerror) {
+		if (readback_val != dest_raw_size) {
+			bcmerror = BCME_BADLEN;
+		}
+	}
+	if (bcmerror) {
+		DHD_ERROR(("%s: Downloaded len error %d\n", __FUNCTION__, bcmerror));
+		goto exit;
+	}
+
+	/* Read back and compare the downloaded type field */
+	bcmerror = dhdsdio_membytes(bus, FALSE,
+		dest_addr + dest_size + sizeof(dngl_rtlv_len_t),
+		(uint8*)&readback_val, sizeof(dngl_rtlv_type_t));
+	if (!bcmerror) {
+		if (readback_val != type) {
+			bcmerror = BCME_BADOPTION;
+		}
+	}
+	if (bcmerror) {
+		DHD_ERROR(("%s: Downloaded type error %d\n", __FUNCTION__, bcmerror));
+		goto exit;
+	}
+#endif /* DHD_DEBUG */
+
+	bus->ramtop_addr = dest_addr;
+
+exit:
+#ifdef DHD_DEBUG
+	if (readback_buf) {
+		MFREE(bus->dhd->osh, readback_buf, dest_size);
+	}
+#endif /* DHD_DEBUG */
+
+	return bcmerror;
+} /* dhdsdio_download_rtlv */
+
+/* Download a reversed-TLV END marker to the top of dongle RAM */
+static int
+dhdsdio_download_rtlv_end(dhd_bus_t *bus)
+{
+	return dhdsdio_download_rtlv(bus, DNGL_RTLV_TYPE_END_MARKER, 0, NULL);
+}
+
+/* Write the FW signature verification status to dongle memory */
+static int
+dhdsdio_bus_write_fws_status(dhd_bus_t *bus)
+{
+	bl_verif_status_t vstatus;
+	int ret;
+
+	bzero(&vstatus, sizeof(vstatus));
+
+	ret = dhdsdio_download_rtlv(bus, DNGL_RTLV_TYPE_FWSIGN_STATUS, sizeof(vstatus),
+		(uint8*)&vstatus);
+	bus->fwstat_download_addr = bus->ramtop_addr;
+	bus->fwstat_download_len = sizeof(vstatus);
+
+	return ret;
+} /* dhdsdio_bus_write_fws_status */
+
+/* Write the FW signature verification memory map to dongle memory */
+static int
+dhdsdio_bus_write_fws_mem_info(dhd_bus_t *bus)
+{
+	bl_mem_info_t memmap;
+	int ret;
+
+	bzero(&memmap, sizeof(memmap));
+	memmap.firmware.start = bus->fw_download_addr;
+	memmap.firmware.end = memmap.firmware.start + bus->fw_download_len;
+	memmap.heap.start = ROUNDUP(memmap.firmware.end + BL_HEAP_START_GAP_SIZE, 4);
+	memmap.heap.end = memmap.heap.start + BL_HEAP_SIZE;
+	memmap.signature.start = bus->fwsig_download_addr;
+	memmap.signature.end = memmap.signature.start + bus->fwsig_download_len;
+	memmap.vstatus.start = bus->fwstat_download_addr;
+	memmap.vstatus.end = memmap.vstatus.start + bus->fwstat_download_len;
+	DHD_ERROR(("%s: mem_info: fw=%x-%x heap=%x-%x sig=%x-%x vst=%x-%x res=%x\n",
+		__FUNCTION__,
+		memmap.firmware.start, memmap.firmware.end,
+		memmap.heap.start, memmap.heap.end,
+		memmap.signature.start, memmap.signature.end,
+		memmap.vstatus.start, memmap.vstatus.end,
+		memmap.reset_vec.start));
+
+	ret = dhdsdio_download_rtlv(bus, DNGL_RTLV_TYPE_FWSIGN_MEM_MAP, sizeof(memmap),
+		(uint8*)&memmap);
+	bus->fw_memmap_download_addr = bus->ramtop_addr;
+	bus->fw_memmap_download_len = sizeof(memmap);
+
+	return ret;
+} /* dhdsdio_bus_write_fws_mem_info */
+
+/* Download a bootloader image to dongle RAM */
+static int
+dhdsdio_bus_download_ram_bootloader(dhd_bus_t *bus)
+{
+	int ret = BCME_OK;
+	uint32 dongle_ram_base_save;
+
+	DHD_INFO(("download_bloader: %s,0x%x. ramtop=0x%x\n",
+		bus->bootloader_filename, bus->bootloader_addr, bus->ramtop_addr));
+	if (bus->bootloader_filename[0] == '\0') {
+		return ret;
+	}
+
+	/* Save ram base */
+	dongle_ram_base_save = bus->dongle_ram_base;
+
+	/* Set ram base to bootloader download start address */
+	bus->dongle_ram_base = bus->bootloader_addr;
+
+	/* Download the bootloader image to TCM */
+	ret = dhdsdio_download_code_file(bus, bus->bootloader_filename);
+
+	/* Restore ram base */
+	bus->dongle_ram_base = dongle_ram_base_save;
+
+	return ret;
+} /* dhdsdio_bus_download_ram_bootloader */
+
+/* Save the FW download address and size */
+static int
+dhdsdio_bus_save_download_info(dhd_bus_t *bus, uint32 download_addr,
+	uint32 download_size, const char *signature_fname,
+	const char *bloader_fname, uint32 bloader_download_addr)
+{
+	/* DHD EXE may not have size of the downloaded FW image in case of swpaging binary */
+	if (download_size) {
+		bus->fw_download_len = download_size;
+	}
+	bus->fw_download_addr = download_addr;
+	strlcpy(bus->fwsig_filename, signature_fname, sizeof(bus->fwsig_filename));
+	strlcpy(bus->bootloader_filename, bloader_fname, sizeof(bus->bootloader_filename));
+	bus->bootloader_addr = bloader_download_addr;
+#ifdef GDB_PROXY
+	/* GDB proxy bootloader mode - if signature file specified (i.e.
+	 * bootloader is used), but bootloader is not specified (i.e. ROM
+	 * bootloader is uses).
+	 * Bootloader mode is significant only for for preattachment debugging
+	 * of chips, in which debug cell can't be initialized before ARM CPU
+	 * start
+	 */
+	bus->gdb_proxy_bootloader_mode = bus->fwsig_filename[0] != 0;
+#endif /* GDB_PROXY */
+	return BCME_OK;
+} /* dhdsdio_bus_save_download_info */
+
+/* Read a binary file and write it to the specified socram dest address */
+#ifdef DHD_LINUX_STD_FW_API
+static int
+dhdsdio_download_sig_file(dhd_bus_t *bus, char *path, uint32 type)
+{
+	int bcmerror = BCME_OK;
+	int srcsize = 0;
+	uint32 dest_size = 0;   /* dongle RAM dest size */
+	const struct firmware *sig = NULL;
+
+	if (path == NULL || path[0] == '\0') {
+		DHD_ERROR(("%s: no file\n", __FUNCTION__));
+		bcmerror = BCME_NOTFOUND;
+		goto exit;
+	}
+
+	bcmerror = dhd_os_get_img_fwreq(&sig, bus->fwsig_filename);
+	if (bcmerror < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			bcmerror));
+		goto exit;
+	}
+	DHD_ERROR(("%s: dhd_os_get_img(Request Firmware API) success. size %d.\n",
+		__FUNCTION__, (int)sig->size));
+
+	srcsize = sig->size;
+	if (srcsize <= 0 || srcsize > MEMBLOCK) {
+		DHD_ERROR(("%s: invalid fwsig size %u\n", __FUNCTION__, srcsize));
+		bcmerror = BCME_BUFTOOSHORT;
+		goto exit;
+	}
+
+	dest_size = ROUNDUP(srcsize, 4);
+	/* Write the src buffer as a rTLV to the dongle */
+	GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST();
+	bcmerror = dhdsdio_download_rtlv(bus, type, dest_size, (uint8 *)sig->data);
+	GCC_DIAGNOSTIC_POP();
+	if (bcmerror) {
+		DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			__FUNCTION__, bcmerror, srcsize, bus->ramtop_addr));
+		goto exit;
+	}
+
+	bus->fwsig_download_addr = bus->ramtop_addr;
+	bus->fwsig_download_len = dest_size;
+
+exit:
+	if (sig)
+	dhd_os_close_img_fwreq(sig);
+
+	return bcmerror;
+}
+#else
+static int
+dhdsdio_download_sig_file(dhd_bus_t *bus, char *path, uint32 type)
+{
+	int bcmerror = BCME_OK;
+	void *filep = NULL;
+	uint8 *srcbuf = NULL;
+	int srcsize = 0;
+	int len;
+	uint32 dest_size = 0;   /* dongle RAM dest size */
+	fwpkg_info_t *fwpkg = NULL;
+
+	if (path == NULL || path[0] == '\0') {
+		DHD_ERROR(("%s: no file\n", __FUNCTION__));
+		bcmerror = BCME_NOTFOUND;
+		goto exit;
+	}
+
+	bcmerror = fwpkg_init(&bus->fwpkg, path);
+	if (bcmerror == BCME_ERROR) {
+		printf("%s: fwpkg_init failed %s\n", __FUNCTION__, path);
+		goto exit;
+	}
+	fwpkg = &bus->fwpkg;
+
+	/* Open file, get size */
+	bcmerror = fwpkg_open_signature_img(fwpkg, path, &filep);
+	if (bcmerror == BCME_ERROR) {
+		DHD_ERROR(("%s: error opening file %s\n", __FUNCTION__, path));
+		goto exit;
+	}
+
+	srcsize = fwpkg_get_signature_img_size(fwpkg);
+
+	if (srcsize <= 0 || srcsize > MEMBLOCK) {
+		DHD_ERROR(("%s: invalid fwsig size %u\n", __FUNCTION__, srcsize));
+		bcmerror = BCME_BUFTOOSHORT;
+		goto exit;
+	}
+	dest_size = ROUNDUP(srcsize, 4);
+
+	/* Allocate src buffer, read in the entire file */
+	srcbuf = (uint8 *)MALLOCZ(bus->dhd->osh, dest_size);
+	if (!srcbuf) {
+		bcmerror = BCME_NOMEM;
+		goto exit;
+	}
+	len = dhd_os_get_image_block(srcbuf, srcsize, filep);
+	if (len != srcsize) {
+		DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+		bcmerror = BCME_BADLEN;
+		goto exit;
+	}
+
+	/* Write the src buffer as a rTLV to the dongle */
+	bcmerror = dhdsdio_download_rtlv(bus, type, dest_size, srcbuf);
+
+	bus->fwsig_download_addr = bus->ramtop_addr;
+	bus->fwsig_download_len = dest_size;
+
+exit:
+	if (filep) {
+		dhd_os_close_image1(bus->dhd, filep);
+	}
+	if (srcbuf) {
+		MFREE(bus->dhd->osh, srcbuf, dest_size);
+	}
+
+	return bcmerror;
+} /* dhdsdio_download_sig_file */
+#endif /* DHD_LINUX_STD_FW_API */
+
+static int
+dhdsdio_bus_write_fwsig(dhd_bus_t *bus, char *fwsig_path, char *nvsig_path)
+{
+	int bcmerror = BCME_OK;
+
+	/* Download the FW signature file to the chip */
+	bcmerror = dhdsdio_download_sig_file(bus, fwsig_path, DNGL_RTLV_TYPE_FW_SIGNATURE);
+	if (bcmerror) {
+		goto exit;
+	}
+
+exit:
+	if (bcmerror) {
+		DHD_ERROR(("%s: error %d\n", __FUNCTION__, bcmerror));
+	}
+	return bcmerror;
+} /* dhdsdio_bus_write_fwsig */
+
+static int
+dhdsdio_read_fwstatus(dhd_bus_t *bus, bl_verif_status_t *status)
+{
+	int ret = BCME_OK;
+
+	bzero(status, sizeof(*status));
+	if (bus->fwstat_download_addr != 0) {
+		ret = dhdsdio_membytes(bus, FALSE, bus->fwstat_download_addr,
+			(uint8 *)status, sizeof(*status));
+		if (ret != BCME_OK) {
+			DHD_ERROR(("%s: error %d on reading %zu membytes at 0x%08x\n",
+				__FUNCTION__, ret, sizeof(*status), bus->fwstat_download_addr));
+		}
+	}
+
+	return ret;
+}
+
+/* Dump secure firmware status. */
+static int
+dhd_bus_dump_fws(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	bl_verif_status_t status;
+	bl_mem_info_t     meminfo;
+	int               err = BCME_OK;
+
+	err = dhdsdio_read_fwstatus(bus, &status);
+	if (err != BCME_OK) {
+		return (err);
+	}
+
+	bzero(&meminfo, sizeof(meminfo));
+	if (bus->fw_memmap_download_addr != 0) {
+		err = dhdsdio_membytes(bus, FALSE, bus->fw_memmap_download_addr,
+			(uint8 *)&meminfo, sizeof(meminfo));
+		if (err != BCME_OK) {
+			DHD_ERROR(("%s: error %d on reading %zu membytes at 0x%08x\n",
+				__FUNCTION__, err, sizeof(meminfo), bus->fw_memmap_download_addr));
+			return (err);
+		}
+	}
+
+	bcm_bprintf(strbuf, "Firmware signing\nSignature: (%08x) len (%d)\n",
+		bus->fwsig_download_addr, bus->fwsig_download_len);
+
+	bcm_bprintf(strbuf,
+		"Verification status: (%08x)\n"
+		"\tstatus: %d\n"
+		"\tstate: %u\n"
+		"\talloc_bytes: %u\n"
+		"\tmax_alloc_bytes: %u\n"
+		"\ttotal_alloc_bytes: %u\n"
+		"\ttotal_freed_bytes: %u\n"
+		"\tnum_allocs: %u\n"
+		"\tmax_allocs: %u\n"
+		"\tmax_alloc_size: %u\n"
+		"\talloc_failures: %u\n",
+		bus->fwstat_download_addr,
+		status.status,
+		status.state,
+		status.alloc_bytes,
+		status.max_alloc_bytes,
+		status.total_alloc_bytes,
+		status.total_freed_bytes,
+		status.num_allocs,
+		status.max_allocs,
+		status.max_alloc_size,
+		status.alloc_failures);
+
+	bcm_bprintf(strbuf,
+		"Memory info: (%08x)\n"
+		"\tfw   %08x-%08x\n\theap %08x-%08x\n\tsig  %08x-%08x\n\tvst  %08x-%08x\n",
+		bus->fw_memmap_download_addr,
+		meminfo.firmware.start,  meminfo.firmware.end,
+		meminfo.heap.start,      meminfo.heap.end,
+		meminfo.signature.start, meminfo.signature.end,
+		meminfo.vstatus.start,   meminfo.vstatus.end);
+
+	return (err);
+}
+#endif /* FW_SIGNATURE */
+
+int
+dhd_bus_iovar_op(dhd_pub_t *dhdp, const char *name,
+                 void *params, uint plen, void *arg, uint len, bool set)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	const bcm_iovar_t *vi = NULL;
+	int bcmerror = 0;
+	uint val_size;
+	uint32 actionid;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(name);
+
+	/* Get MUST have return space */
+	ASSERT(set || (arg && len));
+
+	/* Set does NOT take qualifiers */
+	ASSERT(!set || (!params && !plen));
+
+	/* Look up var locally; if not found pass to host driver */
+	if ((vi = bcm_iovar_lookup(dhdsdio_iovars, name)) == NULL) {
+		dhd_os_sdlock(bus->dhd);
+
+		BUS_WAKE(bus);
+
+		/* Turn on clock in case SD command needs backplane */
+		dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+		bcmerror = bcmsdh_iovar_op(bus->sdh, name, params, plen, arg, len, set);
+
+		/* Check for bus configuration changes of interest */
+
+		/* If it was divisor change, read the new one */
+		if (set && strcmp(name, "sd_divisor") == 0) {
+			if (bcmsdh_iovar_op(bus->sdh, "sd_divisor", NULL, 0,
+			                    &bus->sd_divisor, sizeof(int32), FALSE) != BCME_OK) {
+				bus->sd_divisor = -1;
+				DHD_ERROR(("%s: fail on %s get\n", __FUNCTION__, name));
+			} else {
+				DHD_INFO(("%s: noted %s update, value now %d\n",
+				          __FUNCTION__, name, bus->sd_divisor));
+			}
+		}
+		/* If it was a mode change, read the new one */
+		if (set && strcmp(name, "sd_mode") == 0) {
+			if (bcmsdh_iovar_op(bus->sdh, "sd_mode", NULL, 0,
+			                    &bus->sd_mode, sizeof(int32), FALSE) != BCME_OK) {
+				bus->sd_mode = -1;
+				DHD_ERROR(("%s: fail on %s get\n", __FUNCTION__, name));
+			} else {
+				DHD_INFO(("%s: noted %s update, value now %d\n",
+				          __FUNCTION__, name, bus->sd_mode));
+			}
+		}
+		/* Similar check for blocksize change */
+		if (set && strcmp(name, "sd_blocksize") == 0) {
+			int32 fnum = 2;
+			if (bcmsdh_iovar_op(bus->sdh, "sd_blocksize", &fnum, sizeof(int32),
+			                    &bus->blocksize, sizeof(int32), FALSE) != BCME_OK) {
+				bus->blocksize = 0;
+				DHD_ERROR(("%s: fail on %s get\n", __FUNCTION__, "sd_blocksize"));
+			} else {
+				DHD_INFO(("%s: noted %s update, value now %d\n",
+				          __FUNCTION__, "sd_blocksize", bus->blocksize));
+
+				dhdsdio_tune_fifoparam(bus);
+			}
+		}
+		bus->roundup = MIN(max_roundup, bus->blocksize);
+
+		if ((bus->idletime == DHD_IDLE_IMMEDIATE) && !bus->dpc_sched &&
+			NO_OTHER_ACTIVE_BUS_USER(bus)) {
+			bus->activity = FALSE;
+			dhdsdio_bussleep(bus, TRUE);
+			dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+		}
+
+		dhd_os_sdunlock(bus->dhd);
+		goto exit;
+	}
+
+	DHD_CTL(("%s: %s %s, len %d plen %d\n", __FUNCTION__,
+	         name, (set ? "set" : "get"), len, plen));
+
+	/* set up 'params' pointer in case this is a set command so that
+	 * the convenience int and bool code can be common to set and get
+	 */
+	if (params == NULL) {
+		params = arg;
+		plen = len;
+	}
+
+	if (vi->type == IOVT_VOID)
+		val_size = 0;
+	else if (vi->type == IOVT_BUFFER)
+		val_size = len;
+	else
+		/* all other types are integer sized */
+		val_size = sizeof(int);
+
+	actionid = set ? IOV_SVAL(vi->varid) : IOV_GVAL(vi->varid);
+	bcmerror = dhdsdio_doiovar(bus, vi, actionid, name, params, plen, arg, len, val_size);
+
+exit:
+	return bcmerror;
+}
+
+void
+dhd_bus_stop(struct dhd_bus *bus, bool enforce_mutex)
+{
+	osl_t *osh;
+	uint32 local_hostintmask;
+	uint8 saveclk;
+	uint retries;
+	int err;
+	bool wlfc_enabled = FALSE;
+	unsigned long flags;
+
+	if (!bus->dhd)
+		return;
+
+	osh = bus->dhd->osh;
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	bcmsdh_waitlockfree(bus->sdh);
+
+	if (enforce_mutex)
+		dhd_os_sdlock(bus->dhd);
+
+	if ((bus->dhd->busstate == DHD_BUS_DOWN) || bus->dhd->hang_was_sent) {
+		/* if Firmware already hangs disbale any interrupt */
+		DHD_INFO(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+		bus->dhd->busstate = DHD_BUS_DOWN;
+		bus->hostintmask = 0;
+		bcmsdh_intr_disable(bus->sdh);
+	} else {
+
+		BUS_WAKE(bus);
+		if (KSO_ENAB(bus)) {
+
+		/* Enable clock for device interrupts */
+		dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+		/* Disable and clear interrupts at the chip level also */
+#ifdef DHD_SI_WD_RESET
+		if (!bus->dhd->si_wd)
+#endif
+			W_SDREG(0, &bus->regs->hostintmask, retries);
+
+		local_hostintmask = bus->hostintmask;
+		bus->hostintmask = 0;
+
+		/* Force clocks on backplane to be sure F2 interrupt propagates */
+#ifdef DHD_SI_WD_RESET
+		if (!bus->dhd->si_wd) {
+#else
+		if (1) {
+#endif
+			saveclk = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+			if (!err) {
+				bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR,
+						(saveclk | SBSDIO_FORCE_HT), &err);
+			}
+			if (err) {
+				DHD_ERROR(("%s: Failed to force clock for F2: err %d\n",
+						__FUNCTION__, err));
+			}
+		}
+		/* Turn off the bus (F2), free any pending packets */
+		/* XXX How to wake up any waiting processes? */
+		/* XXX New API: bcmsdh_fn_set(bus->sdh, SDIO_FUNC_2, FALSE); */
+		DHD_INTR(("%s: disable SDIO interrupts\n", __FUNCTION__));
+#if !defined(NDIS)
+		bcmsdh_intr_disable(bus->sdh); /* XXX bcmsdh_intr_mask(bus->sdh); */
+#endif /* !defined(NDIS) */
+#ifndef BCMSPI
+#ifdef DHD_SI_WD_RESET
+		if (!bus->dhd->si_wd)
+#endif
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_IOEN, SDIO_FUNC_ENABLE_1, NULL);
+#endif /* !BCMSPI */
+
+		/* Clear any pending interrupts now that F2 is disabled */
+#ifdef DHD_SI_WD_RESET
+		if (!bus->dhd->si_wd)
+#endif
+			W_SDREG(local_hostintmask, &bus->regs->intstatus, retries);
+		}
+		/* Turn off the backplane clock (only) */
+#ifdef DHD_SI_WD_RESET
+		if (!bus->dhd->si_wd)
+#endif
+			dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+
+		/* Change our idea of bus state */
+		DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+		DHD_INFO(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+		bus->dhd->busstate = DHD_BUS_DOWN;
+		DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+	}
+
+#ifdef PROP_TXSTATUS
+	wlfc_enabled = (dhd_wlfc_cleanup_txq(bus->dhd, NULL, 0) != WLFC_UNSUPPORTED);
+#endif
+	if (!wlfc_enabled) {
+#ifdef DHDTCPACK_SUPPRESS
+		/* Clean tcp_ack_info_tbl in order to prevent access to flushed pkt,
+		 * when there is a newly coming packet from network stack.
+		 */
+		dhd_tcpack_info_tbl_clean(bus->dhd);
+#endif /* DHDTCPACK_SUPPRESS */
+		dhd_os_sdlock_txq(bus->dhd);
+		/* Clear the data packet queues */
+		pktq_flush(osh, &bus->txq, TRUE);
+		dhd_os_sdunlock_txq(bus->dhd);
+	}
+
+	/* Clear any held glomming stuff */
+	if (bus->glomd)
+		PKTFREE(osh, bus->glomd, FALSE);
+
+	if (bus->glom)
+		PKTFREE(osh, bus->glom, FALSE);
+
+	bus->glom = bus->glomd = NULL;
+
+	/* Clear rx control and wake any waiters */
+	/* XXX More important in disconnect, but no context? */
+	bus->rxlen = 0;
+	dhd_os_ioctl_resp_wake(bus->dhd);
+
+	/* Reset some F2 state stuff */
+	bus->rxskip = FALSE;
+	bus->tx_seq = bus->rx_seq = 0;
+
+	/* Initializing tx_max to a reasonable value to start xfer
+	 * Gets updated to correct value after receving the first
+	 * packet from firmware
+	 * XXX - Need to find a right mechanism to querry from
+	 * firmware when the device is coming up
+	 */
+	bus->tx_max = 4;
+
+	if (enforce_mutex)
+		dhd_os_sdunlock(bus->dhd);
+}
+
+#if defined(BCMSDIOH_TXGLOM) && defined(BCMSDIOH_STD)
+extern uint sd_txglom;
+#endif
+void
+dhd_txglom_enable(dhd_pub_t *dhdp, bool enable)
+{
+	/* can't enable host txglom by default, some platforms have no
+	 * (or crappy) ADMA support and txglom will cause kernel assertions (e.g.
+	 * panda board)
+	 */
+	dhd_bus_t *bus = dhdp->bus;
+#ifdef BCMSDIOH_TXGLOM
+	uint32 rxglom;
+	int32 ret;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+#ifdef BCMSDIOH_STD
+	if (enable)
+		enable = sd_txglom;
+#endif /* BCMSDIOH_STD */
+
+	if (enable) {
+		rxglom = 1;
+		ret = dhd_iovar(dhdp, 0, "bus:rxglom", (char *)&rxglom, sizeof(rxglom), NULL, 0,
+				TRUE);
+		if (ret >= 0)
+			bus->txglom_enable = TRUE;
+		else {
+#ifdef BCMSDIOH_STD
+			sd_txglom = 0;
+#endif /* BCMSDIOH_STD */
+			bus->txglom_enable = FALSE;
+		}
+	} else
+#endif /* BCMSDIOH_TXGLOM */
+		bus->txglom_enable = FALSE;
+	DHD_TRACE(("%s: enable %d\n",  __FUNCTION__, bus->txglom_enable));
+	dhd_conf_set_txglom_params(bus->dhd, bus->txglom_enable);
+	bcmsdh_set_mode(bus->sdh, bus->dhd->conf->txglom_mode);
+}
+
+int
+dhd_bus_init(dhd_pub_t *dhdp, bool enforce_mutex)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	dhd_timeout_t tmo;
+	uint retries = 0;
+	uint8 ready, enable;
+	int err, ret = 0;
+#ifdef BCMSPI
+	uint32 dstatus = 0;	/* gSPI device-status bits */
+#else /* BCMSPI */
+	uint8 saveclk;
+#endif /* BCMSPI */
+#if defined(SDIO_ISR_THREAD)
+	int intr_extn;
+#endif
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	ASSERT(bus->dhd);
+	if (!bus->dhd)
+		return 0;
+
+	if (enforce_mutex)
+		dhd_os_sdlock(bus->dhd);
+
+	if (bus->sih->chip == BCM43362_CHIP_ID) {
+		DHD_INFO(("%s: delay 100ms for BCM43362\n", __FUNCTION__));
+		OSL_DELAY(100000); // terence 20131209: delay for 43362
+	}
+
+	/* Make sure backplane clock is on, needed to generate F2 interrupt */
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+	if (bus->clkstate != CLK_AVAIL) {
+		DHD_ERROR(("%s: clock state is wrong. state = %d\n", __FUNCTION__, bus->clkstate));
+		ret = -1;
+		goto exit;
+	}
+
+#ifdef BCMSPI
+	/* fake "ready" for spi, wake-wlan would have already enabled F1 and F2 */
+	ready = (SDIO_FUNC_ENABLE_1 | SDIO_FUNC_ENABLE_2);
+	enable = 0;
+
+	/* Give the dongle some time to do its thing and set IOR2 */
+	dhd_timeout_start(&tmo, WAIT_F2RXFIFORDY * WAIT_F2RXFIFORDY_DELAY * 1000);
+	while (!enable && !dhd_timeout_expired(&tmo)) {
+		dstatus = bcmsdh_cfg_read_word(bus->sdh, SDIO_FUNC_0, SPID_STATUS_REG, NULL);
+		if (dstatus & STATUS_F2_RX_READY)
+			enable = TRUE;
+	}
+
+	if (enable) {
+		DHD_INFO(("Took %u usec before dongle is ready\n", tmo.elapsed));
+		enable = ready;
+	} else {
+		DHD_ERROR(("dstatus when timed out on f2-fifo not ready = 0x%x\n", dstatus));
+		DHD_ERROR(("Waited %u usec, dongle is not ready\n", tmo.elapsed));
+		ret = -1;
+		goto exit;
+	}
+
+#else /* !BCMSPI */
+	/* Force clocks on backplane to be sure F2 interrupt propagates */
+	saveclk = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+
+	if (!err) {
+#ifndef BCMQT
+		/* QT requires HT clock */
+		if (bus->sih->chip == BCM43012_CHIP_ID ||
+			bus->sih->chip == BCM43013_CHIP_ID ||
+			bus->sih->chip == BCM43014_CHIP_ID) {
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR,
+				(saveclk | SBSDIO_HT_AVAIL_REQ), &err);
+		} else
+#endif /* BCMQT */
+		{
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR,
+				(saveclk | SBSDIO_FORCE_HT), &err);
+		}
+	}
+
+	if (err) {
+		DHD_ERROR(("%s: Failed to force clock for F2: err %d\n", __FUNCTION__, err));
+		ret = -1;
+		goto exit;
+	}
+
+	/* Enable function 2 (frame transfers) */
+	/* XXX New API: change to bcmsdh_fn_set(sdh, SDIO_FUNC_2, TRUE); */
+	W_SDREG((SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT),
+	        &bus->regs->tosbmailboxdata, retries);
+	enable = (SDIO_FUNC_ENABLE_1 | SDIO_FUNC_ENABLE_2);
+
+	bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_IOEN, enable, NULL);
+
+	/* Give the dongle some time to do its thing and set IOR2 */
+#ifdef BCMSLTGT
+	dhd_timeout_start(&tmo, DHD_WAIT_F2RDY * 1000 * htclkratio);
+#else
+	dhd_timeout_start(&tmo, DHD_WAIT_F2RDY * 1000);
+#endif /* BCMSLTGT */
+
+	ready = 0;
+	while (ready != enable && !dhd_timeout_expired(&tmo))
+	        ready = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_IORDY, NULL);
+
+#endif /* !BCMSPI */
+
+	DHD_INFO(("%s: enable 0x%02x, ready 0x%02x (waited %uus)\n",
+	          __FUNCTION__, enable, ready, tmo.elapsed));
+
+#if defined(SDIO_ISR_THREAD)
+	if (dhdp->conf->intr_extn) {
+		intr_extn = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_INTR_EXTN, NULL);
+		if (intr_extn & 0x1) {
+			intr_extn |= 0x2;
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_INTR_EXTN, intr_extn, NULL);
+		}
+	}
+#endif
+
+	/* XXX For simplicity, fail and await next call if F2 not ready.
+	 * XXX Should really set timer to poll, and/or enable interrupt;
+	 * XXX then put this process in wait until a result...
+	 */
+
+	/* If F2 successfully enabled, set core and enable interrupts */
+	if (ready == enable) {
+		/* Make sure we're talking to the core. */
+#ifdef BCMSDIOLITE
+		bus->regs = si_setcore(bus->sih, CC_CORE_ID, 0);
+		ASSERT(bus->regs != NULL);
+#else
+		if (!(bus->regs = si_setcore(bus->sih, PCMCIA_CORE_ID, 0)))
+			bus->regs = si_setcore(bus->sih, SDIOD_CORE_ID, 0);
+		ASSERT(bus->regs != NULL);
+#endif
+		/* Set up the interrupt mask and enable interrupts */
+		bus->hostintmask = HOSTINTMASK;
+		/* corerev 4 could use the newer interrupt logic to detect the frames */
+#ifndef BCMSPI
+		if ((bus->sih->buscoretype == SDIOD_CORE_ID) && (bus->sdpcmrev == 4) &&
+			(bus->rxint_mode != SDIO_DEVICE_HMB_RXINT)) {
+			bus->hostintmask &= ~I_HMB_FRAME_IND;
+			bus->hostintmask |= I_XMTDATA_AVAIL;
+		}
+#endif /* BCMSPI */
+		W_SDREG(bus->hostintmask, &bus->regs->hostintmask, retries);
+
+		/* PR47410 - Lower F2 Watermark to avoid DMA Hang
+		 * in F2 when SD Clock is stopped.
+		 */
+		if (bus->sih->buscorerev < 15) {
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1, SBSDIO_WATERMARK,
+				(uint8)watermark, &err);
+		}
+
+		/* Set bus state according to enable result */
+		dhdp->busstate = DHD_BUS_DATA;
+
+		/* Need to set fn2 block size to match fn1 block size.
+		 * Requests to fn2 go thru fn1. *
+		 * faltwig has this code contitioned with #if !BCMSPI_ANDROID.
+		 * It would be cleaner to use the ->sdh->block_sz[fno] instead of
+		 * 64, but this layer has no access to sdh types.
+		 */
+#if defined(NDIS)
+	{
+		uint8 *ptr = NULL;
+		uint16 block_sz = 64;
+		ptr = (uint8*) &block_sz;
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0,
+			(SDIOD_FBR_BASE(SDIO_FUNC_2) + SDIOD_CCCR_BLKSIZE_0),
+			*ptr++, &err);
+		if (err == BCME_OK)
+			bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0,
+			(SDIOD_FBR_BASE(SDIO_FUNC_2) + SDIOD_CCCR_BLKSIZE_1),
+			*ptr++, &err);
+		if (err != BCME_OK) {
+			printf("%s: set block size for func 2 failed\n",
+				__FUNCTION__);
+			ret = -1;
+			goto exit;
+		}
+	}
+#endif /* NDIS */
+
+		/* XXX These need to change w/API updates */
+		/* bcmsdh_intr_unmask(bus->sdh); */
+
+		bus->intdis = FALSE;
+		if (bus->intr) {
+			DHD_INTR(("%s: enable SDIO device interrupts\n", __FUNCTION__));
+#ifndef BCMSPI_ANDROID
+			bcmsdh_intr_enable(bus->sdh);
+#endif /* !BCMSPI_ANDROID */
+		} else {
+			DHD_INTR(("%s: disable SDIO interrupts\n", __FUNCTION__));
+			bcmsdh_intr_disable(bus->sdh);
+		}
+
+#ifdef DEBUG_LOST_INTERRUPTS
+		{
+			uint32 intstatus;
+			bool hostpending;
+			uint8 devena, devpend;
+			uint sdr_retries = 0;
+
+			hostpending = bcmsdh_intr_pending(bus->sdh);
+			devena = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_INTEN, NULL);
+			devpend = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_INTPEND, NULL);
+
+			R_SDREG(intstatus, &bus->regs->intstatus, sdr_retries);
+			intstatus &= bus->hostintmask;
+
+			DHD_INFO(("%s: interrupts -- host %s device ena/pend 0x%02x/0x%02x\n"
+			           "intstatus 0x%08x, hostmask 0x%08x\n", __FUNCTION__,
+			           (hostpending ? "PENDING" : "NOT PENDING"),
+			           devena, devpend, intstatus, bus->hostintmask));
+		}
+#endif /* DEBUG_LOST_INTERRUPTS */
+	}
+
+#ifndef BCMSPI
+
+	else {
+		/* Disable F2 again */
+		enable = SDIO_FUNC_ENABLE_1;
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_IOEN, enable, NULL);
+	}
+
+	if (dhdsdio_sr_cap(bus)) {
+		dhdsdio_sr_init(bus);
+		/* Masking the chip active interrupt  permanantly */
+		bus->hostintmask &= ~I_CHIPACTIVE;
+		W_SDREG(bus->hostintmask, &bus->regs->hostintmask, retries);
+		DHD_INFO(("%s: disable I_CHIPACTIVE in hostintmask[0x%08x]\n",
+		__FUNCTION__, bus->hostintmask));
+	} else {
+		bcmsdh_cfg_write(bus->sdh, SDIO_FUNC_1,
+			SBSDIO_FUNC1_CHIPCLKCSR, saveclk, &err);
+	}
+#endif /* !BCMSPI */
+
+	/* If we didn't come up, turn off backplane clock */
+	if (dhdp->busstate != DHD_BUS_DATA)
+		dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+
+exit:
+	if (enforce_mutex)
+		dhd_os_sdunlock(bus->dhd);
+#if defined(FW_SIGNATURE)
+	if ((ret == BCME_ERROR) && (bus->fwsig_filename[0] != 0)) {
+		bl_verif_status_t status;
+
+		(void)dhdsdio_read_fwstatus(bus, &status);
+		DHD_INFO(("Verification status: (%08x)\n"
+			"\tstatus: %d\n"
+			"\tstate: %u\n"
+			"\talloc_bytes: %u\n"
+			"\tmax_alloc_bytes: %u\n"
+			"\ttotal_alloc_bytes: %u\n"
+			"\ttotal_freed_bytes: %u\n"
+			"\tnum_allocs: %u\n"
+			"\tmax_allocs: %u\n"
+			"\tmax_alloc_size: %u\n"
+			"\talloc_failures: %u\n",
+			bus->fwstat_download_addr,
+			status.status,
+			status.state,
+			status.alloc_bytes,
+			status.max_alloc_bytes,
+			status.total_alloc_bytes,
+			status.total_freed_bytes,
+			status.num_allocs,
+			status.max_allocs,
+			status.max_alloc_size,
+			status.alloc_failures));
+	}
+#endif /* FW_SIGNATURE */
+	/* XXX Temp errnum workaround: return ok, caller checks bus state */
+	return ret;
+}
+
+static void
+dhdsdio_rxfail(dhd_bus_t *bus, bool abort, bool rtx)
+{
+	bcmsdh_info_t *sdh = bus->sdh;
+	sdpcmd_regs_t *regs = bus->regs;
+	uint retries = 0;
+	uint16 lastrbc;
+	uint8 hi, lo;
+	int err;
+
+	DHD_ERROR(("%s: %sterminate frame%s\n", __FUNCTION__,
+	           (abort ? "abort command, " : ""), (rtx ? ", send NAK" : "")));
+
+	if (!KSO_ENAB(bus)) {
+		DHD_ERROR(("%s: Device asleep\n", __FUNCTION__));
+		return;
+	}
+
+	if (abort) {
+		bcmsdh_abort(sdh, SDIO_FUNC_2);
+	}
+
+	bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_FRAMECTRL, SFC_RF_TERM, &err);
+	if (err) {
+		DHD_ERROR(("%s: SBSDIO_FUNC1_FRAMECTRL cmd err\n", __FUNCTION__));
+		goto fail;
+	}
+	bus->f1regdata++;
+
+	/* Wait until the packet has been flushed (device/FIFO stable) */
+	for (lastrbc = retries = 0xffff; retries > 0; retries--) {
+		hi = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_RFRAMEBCHI, NULL);
+		lo = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_RFRAMEBCLO, &err);
+		if (err) {
+			DHD_ERROR(("%s: SBSDIO_FUNC1_RFAMEBCLO cmd err\n", __FUNCTION__));
+			goto fail;
+		}
+
+		bus->f1regdata += 2;
+
+		if ((hi == 0) && (lo == 0))
+			break;
+
+		if ((hi > (lastrbc >> 8)) && (lo > (lastrbc & 0x00ff))) {
+			DHD_ERROR(("%s: count growing: last 0x%04x now 0x%04x\n",
+			           __FUNCTION__, lastrbc, ((hi << 8) + lo)));
+		}
+		lastrbc = (hi << 8) + lo;
+	}
+
+	if (!retries) {
+		DHD_ERROR(("%s: count never zeroed: last 0x%04x\n", __FUNCTION__, lastrbc));
+	} else {
+		DHD_INFO(("%s: flush took %d iterations\n", __FUNCTION__, (0xffff - retries)));
+	}
+
+	if (rtx) {
+		bus->rxrtx++;
+		W_SDREG(SMB_NAK, &regs->tosbmailbox, retries);
+		bus->f1regdata++;
+		if (retries <= retry_limit) {
+			bus->rxskip = TRUE;
+		}
+	}
+
+	/* Clear partial in any case */
+	bus->nextlen = 0;
+
+fail:
+	/* If we can't reach the device, signal failure */
+	if (err || bcmsdh_regfail(sdh)) {
+		DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+		bus->dhd->busstate = DHD_BUS_DOWN;
+	}
+}
+
+static void
+dhdsdio_read_control(dhd_bus_t *bus, uint8 *hdr, uint len, uint doff)
+{
+	bcmsdh_info_t *sdh = bus->sdh;
+	uint rdlen, pad;
+
+	int sdret;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	/* Control data already received in aligned rxctl */
+	if ((bus->bus == SPI_BUS) && (!bus->usebufpool))
+		goto gotpkt;
+
+	ASSERT(bus->rxbuf);
+	/* Set rxctl for frame (w/optional alignment) */
+	bus->rxctl = bus->rxbuf;
+	if (dhd_alignctl) {
+		bus->rxctl += firstread;
+		if ((pad = ((uintptr)bus->rxctl % DHD_SDALIGN)))
+			bus->rxctl += (DHD_SDALIGN - pad);
+		bus->rxctl -= firstread;
+	}
+	ASSERT(bus->rxctl >= bus->rxbuf);
+
+	/* Copy the already-read portion over */
+	bcopy(hdr, bus->rxctl, firstread);
+	if (len <= firstread)
+		goto gotpkt;
+
+	/* Copy the full data pkt in gSPI case and process ioctl. */
+	if (bus->bus == SPI_BUS) {
+		bcopy(hdr, bus->rxctl, len);
+		goto gotpkt;
+	}
+
+	/* Raise rdlen to next SDIO block to avoid tail command */
+	rdlen = len - firstread;
+	if (bus->roundup && bus->blocksize && (rdlen > bus->blocksize)) {
+		pad = bus->blocksize - (rdlen % bus->blocksize);
+		if ((pad <= bus->roundup) && (pad < bus->blocksize) &&
+		    ((len + pad) < bus->dhd->maxctl))
+			rdlen += pad;
+	} else if (rdlen % DHD_SDALIGN) {
+		rdlen += DHD_SDALIGN - (rdlen % DHD_SDALIGN);
+	}
+
+	/* Satisfy length-alignment requirements */
+	if (forcealign && (rdlen & (ALIGNMENT - 1)))
+		rdlen = ROUNDUP(rdlen, ALIGNMENT);
+
+	/* Drop if the read is too big or it exceeds our maximum */
+	if ((rdlen + firstread) > bus->dhd->maxctl) {
+		DHD_ERROR(("%s: %d-byte control read exceeds %d-byte buffer\n",
+		           __FUNCTION__, rdlen, bus->dhd->maxctl));
+		bus->dhd->rx_errors++;
+		dhdsdio_rxfail(bus, FALSE, FALSE);
+		goto done;
+	}
+
+	if ((len - doff) > bus->dhd->maxctl) {
+		DHD_ERROR(("%s: %d-byte ctl frame (%d-byte ctl data) exceeds %d-byte limit\n",
+		           __FUNCTION__, len, (len - doff), bus->dhd->maxctl));
+		bus->dhd->rx_errors++; bus->rx_toolong++;
+		dhdsdio_rxfail(bus, FALSE, FALSE);
+		goto done;
+	}
+
+	/* XXX Could block readers with rxlen=0? */
+
+	/* Read remainder of frame body into the rxctl buffer */
+	sdret = dhd_bcmsdh_recv_buf(bus, bcmsdh_cur_sbwad(sdh), SDIO_FUNC_2, F2SYNC,
+	                            (bus->rxctl + firstread), rdlen, NULL, NULL, NULL);
+	bus->f2rxdata++;
+	ASSERT(sdret != BCME_PENDING);
+
+	/* Control frame failures need retransmission */
+	if (sdret < 0) {
+		DHD_ERROR(("%s: read %d control bytes failed: %d\n", __FUNCTION__, rdlen, sdret));
+		bus->rxc_errors++; /* dhd.rx_ctlerrs is higher level */
+		dhdsdio_rxfail(bus, TRUE, TRUE);
+		goto done;
+	}
+
+gotpkt:
+
+#ifdef DHD_DEBUG
+	if (DHD_BYTES_ON() && DHD_CTL_ON()) {
+		prhex("RxCtrl", bus->rxctl, len);
+	}
+#endif
+
+	/* Point to valid data and indicate its length */
+	bus->rxctl += doff;
+	bus->rxlen = len - doff;
+
+done:
+	/* Awake any waiters */
+	dhd_os_ioctl_resp_wake(bus->dhd);
+}
+int
+dhd_process_pkt_reorder_info(dhd_pub_t *dhd, uchar *reorder_info_buf, uint reorder_info_len,
+	void **pkt, uint32 *pkt_count);
+
+static uint8
+dhdsdio_rxglom(dhd_bus_t *bus, uint8 rxseq)
+{
+	uint16 dlen, totlen;
+	uint8 *dptr, num = 0;
+
+	uint16 sublen, check;
+	void *pfirst, *plast, *pnext;
+	void * list_tail[DHD_MAX_IFS] = { NULL };
+	void * list_head[DHD_MAX_IFS] = { NULL };
+	uint8 idx;
+	osl_t *osh = bus->dhd->osh;
+
+	int errcode;
+	uint8 chan, seq, doff, sfdoff;
+	uint8 txmax;
+	uchar reorder_info_buf[WLHOST_REORDERDATA_TOTLEN];
+	uint reorder_info_len;
+
+	int ifidx = 0;
+	bool usechain = bus->use_rxchain;
+
+	/* If packets, issue read(s) and send up packet chain */
+	/* Return sequence numbers consumed? */
+
+	DHD_TRACE(("dhdsdio_rxglom: start: glomd %p glom %p\n", bus->glomd, bus->glom));
+
+	/* If there's a descriptor, generate the packet chain */
+	if (bus->glomd) {
+		dhd_os_sdlock_rxq(bus->dhd);
+
+		pfirst = plast = pnext = NULL;
+		dlen = (uint16)PKTLEN(osh, bus->glomd);
+		dptr = PKTDATA(osh, bus->glomd);
+		if (!dlen || (dlen & 1)) {
+			DHD_ERROR(("%s: bad glomd len (%d), ignore descriptor\n",
+			           __FUNCTION__, dlen));
+			dlen = 0;
+		}
+
+		for (totlen = num = 0; dlen; num++) {
+			/* Get (and move past) next length */
+			sublen = ltoh16_ua(dptr);
+			dlen -= sizeof(uint16);
+			dptr += sizeof(uint16);
+			if ((sublen < SDPCM_HDRLEN) ||
+			    ((num == 0) && (sublen < (2 * SDPCM_HDRLEN)))) {
+				DHD_ERROR(("%s: descriptor len %d bad: %d\n",
+				           __FUNCTION__, num, sublen));
+				pnext = NULL;
+				break;
+			}
+			if (sublen % DHD_SDALIGN) {
+				DHD_ERROR(("%s: sublen %d not a multiple of %d\n",
+				           __FUNCTION__, sublen, DHD_SDALIGN));
+				usechain = FALSE;
+			}
+			totlen += sublen;
+
+			/* For last frame, adjust read len so total is a block multiple */
+			if (!dlen) {
+				sublen += (ROUNDUP(totlen, bus->blocksize) - totlen);
+				totlen = ROUNDUP(totlen, bus->blocksize);
+			}
+
+			/* Allocate/chain packet for next subframe */
+			if ((pnext = PKTGET(osh, sublen + DHD_SDALIGN, FALSE)) == NULL) {
+				DHD_ERROR(("%s: PKTGET failed, num %d len %d\n",
+				           __FUNCTION__, num, sublen));
+				break;
+			}
+			ASSERT(!PKTLINK(pnext));
+			if (!pfirst) {
+				ASSERT(!plast);
+				pfirst = plast = pnext;
+			} else {
+				ASSERT(plast);
+				PKTSETNEXT(osh, plast, pnext);
+				plast = pnext;
+			}
+
+			/* Adhere to start alignment requirements */
+			PKTALIGN(osh, pnext, sublen, DHD_SDALIGN);
+		}
+
+		/* If all allocations succeeded, save packet chain in bus structure */
+		if (pnext) {
+			DHD_GLOM(("%s: allocated %d-byte packet chain for %d subframes\n",
+			          __FUNCTION__, totlen, num));
+			if (DHD_GLOM_ON() && bus->nextlen) {
+				if (totlen != bus->nextlen) {
+					DHD_GLOM(("%s: glomdesc mismatch: nextlen %d glomdesc %d "
+					          "rxseq %d\n", __FUNCTION__, bus->nextlen,
+					          totlen, rxseq));
+				}
+			}
+			bus->glom = pfirst;
+			pfirst = pnext = NULL;
+		} else {
+			if (pfirst)
+				PKTFREE(osh, pfirst, FALSE);
+			bus->glom = NULL;
+			num = 0;
+		}
+
+		/* Done with descriptor packet */
+		PKTFREE(osh, bus->glomd, FALSE);
+		bus->glomd = NULL;
+		bus->nextlen = 0;
+
+		dhd_os_sdunlock_rxq(bus->dhd);
+	}
+
+	/* Ok -- either we just generated a packet chain, or had one from before */
+	if (bus->glom) {
+		if (DHD_GLOM_ON()) {
+			DHD_GLOM(("%s: attempt superframe read, packet chain:\n", __FUNCTION__));
+			for (pnext = bus->glom; pnext; pnext = PKTNEXT(osh, pnext)) {
+				DHD_GLOM(("    %p: %p len 0x%04x (%d)\n",
+				          pnext, (uint8*)PKTDATA(osh, pnext),
+				          PKTLEN(osh, pnext), PKTLEN(osh, pnext)));
+			}
+		}
+
+		pfirst = bus->glom;
+		dlen = (uint16)pkttotlen(osh, pfirst);
+
+		/* Do an SDIO read for the superframe.  Configurable iovar to
+		 * read directly into the chained packet, or allocate a large
+		 * packet and and copy into the chain.
+		 */
+		if (usechain) {
+			errcode = dhd_bcmsdh_recv_buf(bus,
+			                              bcmsdh_cur_sbwad(bus->sdh), SDIO_FUNC_2,
+			                              F2SYNC, (uint8*)PKTDATA(osh, pfirst),
+			                              dlen, pfirst, NULL, NULL);
+		} else if (bus->dataptr) {
+			errcode = dhd_bcmsdh_recv_buf(bus,
+			                              bcmsdh_cur_sbwad(bus->sdh), SDIO_FUNC_2,
+			                              F2SYNC, bus->dataptr,
+			                              dlen, NULL, NULL, NULL);
+			sublen = (uint16)pktfrombuf(osh, pfirst, 0, dlen, bus->dataptr);
+			if (sublen != dlen) {
+				DHD_ERROR(("%s: FAILED TO COPY, dlen %d sublen %d\n",
+				           __FUNCTION__, dlen, sublen));
+				errcode = -1;
+			}
+			pnext = NULL;
+			BCM_REFERENCE(pnext);
+		} else {
+			DHD_ERROR(("COULDN'T ALLOC %d-BYTE GLOM, FORCE FAILURE\n", dlen));
+			errcode = -1;
+		}
+		bus->f2rxdata++;
+		ASSERT(errcode != BCME_PENDING);
+
+		/* On failure, kill the superframe, allow a couple retries */
+		if (errcode < 0) {
+			DHD_ERROR(("%s: glom read of %d bytes failed: %d\n",
+			           __FUNCTION__, dlen, errcode));
+			bus->dhd->rx_errors++; /* XXX Account for rtx?? */
+
+			if (bus->glomerr++ < 3) {
+				dhdsdio_rxfail(bus, TRUE, TRUE);
+			} else {
+				bus->glomerr = 0;
+				dhdsdio_rxfail(bus, TRUE, FALSE);
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE(osh, bus->glom, FALSE);
+				dhd_os_sdunlock_rxq(bus->dhd);
+				bus->rxglomfail++;
+				bus->glom = NULL;
+			}
+			return 0;
+		}
+
+#ifdef DHD_DEBUG
+		if (DHD_GLOM_ON()) {
+			prhex("SUPERFRAME", PKTDATA(osh, pfirst),
+			      MIN(PKTLEN(osh, pfirst), 48));
+		}
+#endif
+
+		/* Validate the superframe header */
+		dptr = (uint8 *)PKTDATA(osh, pfirst);
+		sublen = ltoh16_ua(dptr);
+		check = ltoh16_ua(dptr + sizeof(uint16));
+
+		chan = SDPCM_PACKET_CHANNEL(&dptr[SDPCM_FRAMETAG_LEN]);
+		seq = SDPCM_PACKET_SEQUENCE(&dptr[SDPCM_FRAMETAG_LEN]);
+		bus->nextlen = dptr[SDPCM_FRAMETAG_LEN + SDPCM_NEXTLEN_OFFSET];
+		if ((bus->nextlen << 4) > MAX_RX_DATASZ) {
+			DHD_INFO(("%s: got frame w/nextlen too large (%d) seq %d\n",
+			          __FUNCTION__, bus->nextlen, seq));
+			bus->nextlen = 0;
+		}
+		doff = SDPCM_DOFFSET_VALUE(&dptr[SDPCM_FRAMETAG_LEN]);
+		txmax = SDPCM_WINDOW_VALUE(&dptr[SDPCM_FRAMETAG_LEN]);
+
+		errcode = 0;
+		if ((uint16)~(sublen^check)) {
+			DHD_ERROR(("%s (superframe): HW hdr error: len/check 0x%04x/0x%04x\n",
+			           __FUNCTION__, sublen, check));
+			errcode = -1;
+		} else if (ROUNDUP(sublen, bus->blocksize) != dlen) {
+			DHD_ERROR(("%s (superframe): len 0x%04x, rounded 0x%04x, expect 0x%04x\n",
+			           __FUNCTION__, sublen, ROUNDUP(sublen, bus->blocksize), dlen));
+			errcode = -1;
+		} else if (SDPCM_PACKET_CHANNEL(&dptr[SDPCM_FRAMETAG_LEN]) != SDPCM_GLOM_CHANNEL) {
+			DHD_ERROR(("%s (superframe): bad channel %d\n", __FUNCTION__,
+			           SDPCM_PACKET_CHANNEL(&dptr[SDPCM_FRAMETAG_LEN])));
+			errcode = -1;
+		} else if (SDPCM_GLOMDESC(&dptr[SDPCM_FRAMETAG_LEN])) {
+			DHD_ERROR(("%s (superframe): got second descriptor?\n", __FUNCTION__));
+			errcode = -1;
+		} else if ((doff < SDPCM_HDRLEN) ||
+		           (doff > (PKTLEN(osh, pfirst) - SDPCM_HDRLEN))) {
+			DHD_ERROR(("%s (superframe): Bad data offset %d: HW %d pkt %d min %d\n",
+				__FUNCTION__, doff, sublen, PKTLEN(osh, pfirst),
+				SDPCM_HDRLEN));
+			errcode = -1;
+		}
+
+		/* Check sequence number of superframe SW header */
+		if (rxseq != seq) {
+			DHD_INFO(("%s: (superframe) rx_seq %d, expected %d\n",
+			          __FUNCTION__, seq, rxseq));
+			bus->rx_badseq++;
+			rxseq = seq;
+		}
+
+		/* Check window for sanity */
+		if ((uint8)(txmax - bus->tx_seq) > 0x70) {
+			DHD_INFO(("%s: got unlikely tx max %d with tx_seq %d\n",
+			           __FUNCTION__, txmax, bus->tx_seq));
+			txmax = bus->tx_max;
+		}
+		bus->tx_max = txmax;
+
+		/* Remove superframe header, remember offset */
+		PKTPULL(osh, pfirst, doff);
+		sfdoff = doff;
+
+		/* Validate all the subframe headers */
+		for (num = 0, pnext = pfirst; pnext && !errcode;
+		     num++, pnext = PKTNEXT(osh, pnext)) {
+			dptr = (uint8 *)PKTDATA(osh, pnext);
+			dlen = (uint16)PKTLEN(osh, pnext);
+			sublen = ltoh16_ua(dptr);
+			check = ltoh16_ua(dptr + sizeof(uint16));
+			chan = SDPCM_PACKET_CHANNEL(&dptr[SDPCM_FRAMETAG_LEN]);
+			doff = SDPCM_DOFFSET_VALUE(&dptr[SDPCM_FRAMETAG_LEN]);
+#ifdef DHD_DEBUG
+			if (DHD_GLOM_ON()) {
+				prhex("subframe", dptr, 32);
+			}
+#endif
+
+			if ((uint16)~(sublen^check)) {
+				DHD_ERROR(("%s (subframe %d): HW hdr error: "
+				           "len/check 0x%04x/0x%04x\n",
+				           __FUNCTION__, num, sublen, check));
+				errcode = -1;
+			} else if ((sublen > dlen) || (sublen < SDPCM_HDRLEN)) {
+				DHD_ERROR(("%s (subframe %d): length mismatch: "
+				           "len 0x%04x, expect 0x%04x\n",
+				           __FUNCTION__, num, sublen, dlen));
+				errcode = -1;
+			} else if ((chan != SDPCM_DATA_CHANNEL) &&
+			           (chan != SDPCM_EVENT_CHANNEL)) {
+				DHD_ERROR(("%s (subframe %d): bad channel %d\n",
+				           __FUNCTION__, num, chan));
+				errcode = -1;
+			} else if ((doff < SDPCM_HDRLEN) || (doff > sublen)) {
+				DHD_ERROR(("%s (subframe %d): Bad data offset %d: HW %d min %d\n",
+				           __FUNCTION__, num, doff, sublen, SDPCM_HDRLEN));
+				errcode = -1;
+			}
+		}
+
+		if (errcode) {
+			/* Terminate frame on error, request a couple retries */
+			if (bus->glomerr++ < 3) {
+				/* Restore superframe header space */
+				PKTPUSH(osh, pfirst, sfdoff);
+				dhdsdio_rxfail(bus, TRUE, TRUE);
+			} else {
+				bus->glomerr = 0;
+				dhdsdio_rxfail(bus, TRUE, FALSE);
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE(osh, bus->glom, FALSE);
+				dhd_os_sdunlock_rxq(bus->dhd);
+				bus->rxglomfail++;
+				bus->glom = NULL;
+			}
+			bus->nextlen = 0;
+			return 0;
+		}
+
+		/* Basic SD framing looks ok - process each packet (header) */
+		bus->glom = NULL;
+		plast = NULL;
+
+		dhd_os_sdlock_rxq(bus->dhd);
+		for (num = 0; pfirst; rxseq++, pfirst = pnext) {
+			pnext = PKTNEXT(osh, pfirst);
+			PKTSETNEXT(osh, pfirst, NULL);
+
+			dptr = (uint8 *)PKTDATA(osh, pfirst);
+			sublen = ltoh16_ua(dptr);
+			chan = SDPCM_PACKET_CHANNEL(&dptr[SDPCM_FRAMETAG_LEN]);
+			seq = SDPCM_PACKET_SEQUENCE(&dptr[SDPCM_FRAMETAG_LEN]);
+			doff = SDPCM_DOFFSET_VALUE(&dptr[SDPCM_FRAMETAG_LEN]);
+
+			DHD_GLOM(("%s: Get subframe %d, %p(%p/%d), sublen %d chan %d seq %d\n",
+			          __FUNCTION__, num, pfirst, PKTDATA(osh, pfirst),
+			          PKTLEN(osh, pfirst), sublen, chan, seq));
+
+			ASSERT((chan == SDPCM_DATA_CHANNEL) || (chan == SDPCM_EVENT_CHANNEL));
+
+			if (rxseq != seq) {
+				DHD_GLOM(("%s: rx_seq %d, expected %d\n",
+				          __FUNCTION__, seq, rxseq));
+				bus->rx_badseq++;
+				rxseq = seq;
+			}
+
+#ifdef DHD_DEBUG
+			if (DHD_BYTES_ON() && DHD_DATA_ON()) {
+				prhex("Rx Subframe Data", dptr, dlen);
+			}
+#endif
+
+			PKTSETLEN(osh, pfirst, sublen);
+			PKTPULL(osh, pfirst, doff);
+
+			reorder_info_len = sizeof(reorder_info_buf);
+
+			if (PKTLEN(osh, pfirst) == 0) {
+				PKTFREE(bus->dhd->osh, pfirst, FALSE);
+				continue;
+			} else if (dhd_prot_hdrpull(bus->dhd, &ifidx, pfirst, reorder_info_buf,
+				&reorder_info_len) != 0) {
+				DHD_ERROR(("%s: rx protocol error\n", __FUNCTION__));
+				bus->dhd->rx_errors++;
+				PKTFREE(osh, pfirst, FALSE);
+				continue;
+			}
+			if (reorder_info_len) {
+				uint32 free_buf_count;
+				void *ppfirst;
+
+				ppfirst = pfirst;
+				/* Reordering info from the firmware */
+				dhd_process_pkt_reorder_info(bus->dhd, reorder_info_buf,
+					reorder_info_len, &ppfirst, &free_buf_count);
+
+				if (free_buf_count == 0) {
+					continue;
+				} else {
+					void *temp;
+
+					/*  go to the end of the chain and attach the pnext there */
+					temp = ppfirst;
+					while (PKTNEXT(osh, temp) != NULL) {
+						temp = PKTNEXT(osh, temp);
+					}
+					pfirst = temp;
+					if (list_tail[ifidx] == NULL)
+						list_head[ifidx] = ppfirst;
+					else
+						PKTSETNEXT(osh, list_tail[ifidx], ppfirst);
+					list_tail[ifidx] = pfirst;
+				}
+
+				num += (uint8)free_buf_count;
+			} else {
+				/* this packet will go up, link back into chain and count it */
+
+				if (list_tail[ifidx] == NULL) {
+					list_head[ifidx] = list_tail[ifidx] = pfirst;
+				} else {
+					PKTSETNEXT(osh, list_tail[ifidx], pfirst);
+					list_tail[ifidx] = pfirst;
+				}
+				num++;
+			}
+#ifdef DHD_DEBUG
+			if (DHD_GLOM_ON()) {
+				DHD_GLOM(("%s subframe %d to stack, %p(%p/%d) nxt/lnk %p/%p\n",
+				          __FUNCTION__, num, pfirst,
+				          PKTDATA(osh, pfirst), PKTLEN(osh, pfirst),
+				          PKTNEXT(osh, pfirst), PKTLINK(pfirst)));
+				prhex("", (uint8 *)PKTDATA(osh, pfirst),
+				      MIN(PKTLEN(osh, pfirst), 32));
+			}
+#endif /* DHD_DEBUG */
+		}
+		dhd_os_sdunlock_rxq(bus->dhd);
+
+		for (idx = 0; idx < DHD_MAX_IFS; idx++) {
+			if (list_head[idx]) {
+				void *temp;
+				uint8 cnt = 0;
+				temp = list_head[idx];
+				do {
+					temp = PKTNEXT(osh, temp);
+					cnt++;
+				} while (temp);
+				if (cnt) {
+					dhd_os_sdunlock(bus->dhd);
+					dhd_rx_frame(bus->dhd, idx, list_head[idx], cnt, 0);
+					dhd_os_sdlock(bus->dhd);
+				}
+			}
+		}
+		bus->rxglomframes++;
+		bus->rxglompkts += num;
+	}
+	return num;
+}
+
+/* Return TRUE if there may be more frames to read */
+static uint
+dhdsdio_readframes(dhd_bus_t *bus, uint maxframes, bool *finished)
+{
+	osl_t *osh = bus->dhd->osh;
+	bcmsdh_info_t *sdh = bus->sdh;
+
+	uint16 len, check;	/* Extracted hardware header fields */
+	uint8 chan, seq, doff;	/* Extracted software header fields */
+	uint8 fcbits;		/* Extracted fcbits from software header */
+	uint8 delta;
+
+	void *pkt;	/* Packet for event or data frames */
+	uint16 pad;	/* Number of pad bytes to read */
+	uint16 rdlen;	/* Total number of bytes to read */
+	uint8 rxseq;	/* Next sequence number to expect */
+	uint rxleft = 0;	/* Remaining number of frames allowed */
+	int sdret;	/* Return code from bcmsdh calls */
+	uint8 txmax;	/* Maximum tx sequence offered */
+#ifdef BCMSPI
+	uint32 dstatus = 0;	/* gSPI device status bits of */
+#endif /* BCMSPI */
+	bool len_consistent; /* Result of comparing readahead len and len from hw-hdr */
+	uint8 *rxbuf;
+	int ifidx = 0;
+	uint rxcount = 0; /* Total frames read */
+	uchar reorder_info_buf[WLHOST_REORDERDATA_TOTLEN];
+	uint reorder_info_len;
+	uint pkt_count;
+
+#if defined(DHD_DEBUG) || defined(SDTEST)
+	bool sdtest = FALSE;	/* To limit message spew from test mode */
+#endif
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	bus->readframes = TRUE;
+
+	if (!KSO_ENAB(bus)) {
+		DHD_ERROR(("%s: KSO off\n", __FUNCTION__));
+		bus->readframes = FALSE;
+		return 0;
+	}
+
+	ASSERT(maxframes);
+
+#ifdef SDTEST
+	/* Allow pktgen to override maxframes */
+	if (bus->pktgen_count && (bus->pktgen_mode == DHD_PKTGEN_RECV)) {
+		maxframes = bus->pktgen_count;
+		sdtest = TRUE;
+		UNUSED_PARAMETER(sdtest);
+	}
+#endif
+
+	/* Not finished unless we encounter no more frames indication */
+	*finished = FALSE;
+
+#ifdef BCMSPI
+	/* Get pktlen from gSPI device F0 reg. */
+	if (bus->bus == SPI_BUS) {
+		/* Peek in dstatus bits and find out size to do rx-read. */
+		dstatus = bcmsdh_get_dstatus(bus->sdh);
+		if (dstatus == 0)
+			DHD_ERROR(("%s:ZERO spi dstatus, a case observed in PR61352 hit !!!\n",
+			           __FUNCTION__));
+
+		DHD_TRACE(("Device status from regread = 0x%x\n", dstatus));
+		DHD_TRACE(("Device status from bit-reconstruction = 0x%x\n",
+		          bcmsdh_get_dstatus((void *)bus->sdh)));
+
+		/* Check underflow also, WAR for PR55150 */
+		if ((dstatus & STATUS_F2_PKT_AVAILABLE) && (((dstatus & STATUS_UNDERFLOW)) == 0)) {
+			bus->nextlen = ((dstatus & STATUS_F2_PKT_LEN_MASK) >>
+			                STATUS_F2_PKT_LEN_SHIFT);
+			/* '0' size with pkt-available interrupt is eqvt to 2048 bytes */
+			bus->nextlen = (bus->nextlen == 0) ? SPI_MAX_PKT_LEN : bus->nextlen;
+			if (bus->dwordmode)
+				bus->nextlen = bus->nextlen << 2;
+			DHD_TRACE(("Entering %s: length to be read from gSPI = %d\n",
+			          __FUNCTION__, bus->nextlen));
+		} else {
+			if (dstatus & STATUS_F2_PKT_AVAILABLE)
+				DHD_ERROR(("Underflow during %s.\n", __FUNCTION__));
+			else
+				DHD_ERROR(("False pkt-available intr.\n"));
+			*finished = TRUE;
+			return (maxframes - rxleft);
+		}
+	}
+#endif /* BCMSPI */
+
+	for (rxseq = bus->rx_seq, rxleft = maxframes;
+	     !bus->rxskip && rxleft && bus->dhd->busstate != DHD_BUS_DOWN;
+	     rxseq++, rxleft--) {
+#ifdef DHDTCPACK_SUP_DBG
+		if (bus->dhd->tcpack_sup_mode != TCPACK_SUP_DELAYTX) {
+			if (bus->dotxinrx == FALSE)
+				DHD_ERROR(("%s %d: dotxinrx FALSE with tcpack_sub_mode %d\n",
+					__FUNCTION__, __LINE__, bus->dhd->tcpack_sup_mode));
+		}
+#ifdef DEBUG_COUNTER
+		else if (pktq_mlen(&bus->txq, ~bus->flowcontrol) > 0) {
+			tack_tbl.cnt[bus->dotxinrx ? 6 : 7]++;
+		}
+#endif /* DEBUG_COUNTER */
+#endif /* DHDTCPACK_SUP_DBG */
+		/* tx more to improve rx performance */
+		if (TXCTLOK(bus) && bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL)) {
+			dhdsdio_sendpendctl(bus);
+		} else if (bus->dotxinrx && (bus->clkstate == CLK_AVAIL) &&
+			!bus->fcstate && DATAOK(bus) &&
+			(pktq_mlen(&bus->txq, ~bus->flowcontrol) > bus->txinrx_thres)) {
+			dhdsdio_sendfromq(bus, dhd_txbound);
+#ifdef DHDTCPACK_SUPPRESS
+			/* In TCPACK_SUP_DELAYTX mode, do txinrx only if
+			 * 1. Any DATA packet to TX
+			 * 2. TCPACK to TCPDATA PSH packets.
+			 * in bus txq.
+			 */
+			bus->dotxinrx = (bus->dhd->tcpack_sup_mode == TCPACK_SUP_DELAYTX) ?
+				FALSE : TRUE;
+#endif
+		}
+
+		/* Handle glomming separately */
+		if (bus->glom || bus->glomd) {
+			uint8 cnt;
+			DHD_GLOM(("%s: calling rxglom: glomd %p, glom %p\n",
+			          __FUNCTION__, bus->glomd, bus->glom));
+			cnt = dhdsdio_rxglom(bus, rxseq);
+			DHD_GLOM(("%s: rxglom returned %d\n", __FUNCTION__, cnt));
+			rxseq += cnt - 1;
+			rxleft = (rxleft > cnt) ? (rxleft - cnt) : 1;
+			continue;
+		}
+
+		/* Try doing single read if we can */
+		if (dhd_readahead && bus->nextlen) {
+			uint16 nextlen = bus->nextlen;
+			bus->nextlen = 0;
+
+			if (bus->bus == SPI_BUS) {
+				rdlen = len = nextlen;
+			} else {
+				rdlen = len = nextlen << 4;
+
+				/* Pad read to blocksize for efficiency */
+				if (bus->roundup && bus->blocksize) {
+					pad = bus->blocksize - (rdlen % bus->blocksize);
+					if ((pad <= bus->roundup) && (pad < bus->blocksize) &&
+						((rdlen + pad + firstread) < MAX_RX_DATASZ))
+						rdlen += pad;
+				} else if (rdlen % DHD_SDALIGN) {
+					rdlen += DHD_SDALIGN - (rdlen % DHD_SDALIGN);
+				}
+			}
+
+			/* We use bus->rxctl buffer in WinXP for initial control pkt receives.
+			 * Later we use buffer-poll for data as well as control packets.
+			 * This is required because dhd receives full frame in gSPI unlike SDIO.
+			 * After the frame is received we have to distinguish whether it is data
+			 * or non-data frame.
+			 */
+			/* Allocate a packet buffer */
+			dhd_os_sdlock_rxq(bus->dhd);
+			if (!(pkt = PKTGET(osh, rdlen + DHD_SDALIGN, FALSE))) {
+				if (bus->bus == SPI_BUS) {
+					bus->usebufpool = FALSE;
+					bus->rxctl = bus->rxbuf;
+					if (dhd_alignctl) {
+						bus->rxctl += firstread;
+						if ((pad = ((uintptr)bus->rxctl % DHD_SDALIGN)))
+							bus->rxctl += (DHD_SDALIGN - pad);
+						bus->rxctl -= firstread;
+					}
+					ASSERT(bus->rxctl >= bus->rxbuf);
+					rxbuf = bus->rxctl;
+					/* Read the entire frame */
+					sdret = dhd_bcmsdh_recv_buf(bus,
+					                            bcmsdh_cur_sbwad(sdh),
+					                            SDIO_FUNC_2,
+					                            F2SYNC, rxbuf, rdlen,
+					                            NULL, NULL, NULL);
+					bus->f2rxdata++;
+					ASSERT(sdret != BCME_PENDING);
+
+#ifdef BCMSPI
+					/* PR55150 WAR: Wait for next pkt-available interrupt for
+					 * further processing
+					 */
+					if (bcmsdh_get_dstatus((void *)bus->sdh) &
+					                STATUS_UNDERFLOW) {
+						bus->nextlen = 0;
+						*finished = TRUE;
+						DHD_ERROR(("%s: read %d control bytes failed "
+						           "due to spi underflow\n",
+						           __FUNCTION__, rdlen));
+						/* dhd.rx_ctlerrs is higher level */
+						bus->rxc_errors++;
+						dhd_os_sdunlock_rxq(bus->dhd);
+						continue;
+					}
+#endif /* BCMSPI */
+
+					/* Control frame failures need retransmission */
+					if (sdret < 0) {
+						DHD_ERROR(("%s: read %d control bytes failed: %d\n",
+						   __FUNCTION__, rdlen, sdret));
+						/* dhd.rx_ctlerrs is higher level */
+						bus->rxc_errors++;
+						dhd_os_sdunlock_rxq(bus->dhd);
+						dhdsdio_rxfail(bus, TRUE,
+						    (bus->bus == SPI_BUS) ? FALSE : TRUE);
+						continue;
+					}
+				} else {
+					/* Give up on data, request rtx of events */
+					DHD_ERROR(("%s (nextlen): PKTGET failed: len %d rdlen %d "
+					           "expected rxseq %d\n",
+					           __FUNCTION__, len, rdlen, rxseq));
+					/* XXX Can't issue retry (NAK), frame not started. */
+					/* Just go try again w/normal header read */
+					dhd_os_sdunlock_rxq(bus->dhd);
+					continue;
+				}
+			} else {
+				if (bus->bus == SPI_BUS)
+					bus->usebufpool = TRUE;
+
+				ASSERT(!PKTLINK(pkt));
+				PKTALIGN(osh, pkt, rdlen, DHD_SDALIGN);
+				rxbuf = (uint8 *)PKTDATA(osh, pkt);
+				/* Read the entire frame */
+				sdret = dhd_bcmsdh_recv_buf(bus, bcmsdh_cur_sbwad(sdh),
+				                            SDIO_FUNC_2,
+				                            F2SYNC, rxbuf, rdlen,
+				                            pkt, NULL, NULL);
+				bus->f2rxdata++;
+				ASSERT(sdret != BCME_PENDING);
+#ifdef BCMSPI
+				/* PR55150 WAR: Wait for next pkt-available interrupt for further
+				 * processing
+				 */
+				if (bcmsdh_get_dstatus((void *)bus->sdh) & STATUS_UNDERFLOW) {
+					bus->nextlen = 0;
+					*finished = TRUE;
+					DHD_ERROR(("%s (nextlen): read %d bytes failed due "
+					           "to spi underflow\n",
+					           __FUNCTION__, rdlen));
+					PKTFREE(bus->dhd->osh, pkt, FALSE);
+					bus->dhd->rx_errors++;
+					dhd_os_sdunlock_rxq(bus->dhd);
+					continue;
+				}
+#endif /* BCMSPI */
+
+				if (sdret < 0) {
+					DHD_ERROR(("%s (nextlen): read %d bytes failed: %d\n",
+					   __FUNCTION__, rdlen, sdret));
+					PKTFREE(bus->dhd->osh, pkt, FALSE);
+					bus->dhd->rx_errors++; /* XXX Account for rtx?? */
+					dhd_os_sdunlock_rxq(bus->dhd);
+					/* Force retry w/normal header read.  Don't attempt NAK for
+					 * gSPI
+					 */
+					dhdsdio_rxfail(bus, TRUE,
+					      (bus->bus == SPI_BUS) ? FALSE : TRUE);
+					continue;
+				}
+			}
+			dhd_os_sdunlock_rxq(bus->dhd);
+
+			/* Now check the header */
+			bcopy(rxbuf, bus->rxhdr, SDPCM_HDRLEN);
+
+			/* Extract hardware header fields */
+			len = ltoh16_ua(bus->rxhdr);
+			check = ltoh16_ua(bus->rxhdr + sizeof(uint16));
+
+			/* All zeros means readahead info was bad */
+			if (!(len|check)) {
+				DHD_INFO(("%s (nextlen): read zeros in HW header???\n",
+				           __FUNCTION__));
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE2();
+				dhd_os_sdunlock_rxq(bus->dhd);
+				GSPI_PR55150_BAILOUT;
+				continue;
+			}
+
+			/* Validate check bytes */
+			if ((uint16)~(len^check)) {
+				DHD_ERROR(("%s (nextlen): HW hdr error: nextlen/len/check"
+				           " 0x%04x/0x%04x/0x%04x\n", __FUNCTION__, nextlen,
+				           len, check));
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE2();
+				dhd_os_sdunlock_rxq(bus->dhd);
+				bus->rx_badhdr++;
+				dhdsdio_rxfail(bus, FALSE, FALSE);
+				GSPI_PR55150_BAILOUT;
+				continue;
+			}
+
+			/* Validate frame length */
+			if (len < SDPCM_HDRLEN) {
+				/* XXX Might choose to allow length 4 for signaling */
+				DHD_ERROR(("%s (nextlen): HW hdr length invalid: %d\n",
+				           __FUNCTION__, len));
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE2();
+				dhd_os_sdunlock_rxq(bus->dhd);
+				GSPI_PR55150_BAILOUT;
+				continue;
+			}
+
+			/* Check for consistency with readahead info */
+#ifdef BCMSPI
+			if (bus->bus == SPI_BUS) {
+				if (bus->dwordmode) {
+					uint16 spilen;
+					spilen = ROUNDUP(len, 4);
+					len_consistent = (nextlen != spilen);
+				} else
+					len_consistent = (nextlen != len);
+			} else
+#endif  /* BCMSPI */
+				len_consistent = (nextlen != (ROUNDUP(len, 16) >> 4));
+			if (len_consistent) {
+				/* Mismatch, force retry w/normal header (may be >4K) */
+				DHD_ERROR(("%s (nextlen): mismatch, nextlen %d len %d rnd %d; "
+				           "expected rxseq %d\n",
+				           __FUNCTION__, nextlen, len, ROUNDUP(len, 16), rxseq));
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE2();
+				dhd_os_sdunlock_rxq(bus->dhd);
+				dhdsdio_rxfail(bus, TRUE, (bus->bus == SPI_BUS) ? FALSE : TRUE);
+				GSPI_PR55150_BAILOUT;
+				continue;
+			}
+
+			/* Extract software header fields */
+			chan = SDPCM_PACKET_CHANNEL(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+			seq = SDPCM_PACKET_SEQUENCE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+			doff = SDPCM_DOFFSET_VALUE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+			txmax = SDPCM_WINDOW_VALUE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+
+#ifdef BCMSPI
+			/* Save the readahead length if there is one */
+			if (bus->bus == SPI_BUS) {
+				/* Use reconstructed dstatus bits and find out readahead size */
+				dstatus = bcmsdh_get_dstatus((void *)bus->sdh);
+				DHD_INFO(("Device status from bit-reconstruction = 0x%x\n",
+				bcmsdh_get_dstatus((void *)bus->sdh)));
+				if (dstatus & STATUS_F2_PKT_AVAILABLE) {
+					bus->nextlen = ((dstatus & STATUS_F2_PKT_LEN_MASK) >>
+					                STATUS_F2_PKT_LEN_SHIFT);
+					bus->nextlen = (bus->nextlen == 0) ?
+					           SPI_MAX_PKT_LEN : bus->nextlen;
+					if (bus->dwordmode)
+						bus->nextlen = bus->nextlen << 2;
+					DHD_INFO(("readahead len from gSPI = %d \n",
+					           bus->nextlen));
+					bus->dhd->rx_readahead_cnt ++;
+				} else {
+					bus->nextlen = 0;
+					*finished = TRUE;
+				}
+			} else {
+#endif /* BCMSPI */
+				bus->nextlen =
+				         bus->rxhdr[SDPCM_FRAMETAG_LEN + SDPCM_NEXTLEN_OFFSET];
+				if ((bus->nextlen << 4) > MAX_RX_DATASZ) {
+					DHD_INFO(("%s (nextlen): got frame w/nextlen too large"
+					          " (%d), seq %d\n", __FUNCTION__, bus->nextlen,
+					          seq));
+					bus->nextlen = 0;
+				}
+
+				bus->dhd->rx_readahead_cnt ++;
+#ifdef BCMSPI
+			}
+#endif /* BCMSPI */
+			/* Handle Flow Control */
+			fcbits = SDPCM_FCMASK_VALUE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+
+			delta = 0;
+			if (~bus->flowcontrol & fcbits) {
+				bus->fc_xoff++;
+				delta = 1;
+			}
+			if (bus->flowcontrol & ~fcbits) {
+				bus->fc_xon++;
+				delta = 1;
+			}
+
+			if (delta) {
+				bus->fc_rcvd++;
+				bus->flowcontrol = fcbits;
+			}
+
+			/* Check and update sequence number */
+			if (rxseq != seq) {
+				DHD_INFO(("%s (nextlen): rx_seq %d, expected %d\n",
+				          __FUNCTION__, seq, rxseq));
+				bus->rx_badseq++;
+				rxseq = seq;
+			}
+
+			/* Check window for sanity */
+			if ((uint8)(txmax - bus->tx_seq) > 0x70) {
+#ifdef BCMSPI
+				if ((bus->bus == SPI_BUS) && !(dstatus & STATUS_F2_RX_READY)) {
+					DHD_ERROR(("%s: got unlikely tx max %d with tx_seq %d\n",
+						__FUNCTION__, txmax, bus->tx_seq));
+					txmax = bus->tx_seq + 2;
+				} else {
+#endif /* BCMSPI */
+					DHD_INFO(("%s: got unlikely tx max %d with tx_seq %d\n",
+						__FUNCTION__, txmax, bus->tx_seq));
+					txmax = bus->tx_max;
+#ifdef BCMSPI
+				}
+#endif /* BCMSPI */
+			}
+			bus->tx_max = txmax;
+
+#ifdef DHD_DEBUG
+			if (DHD_BYTES_ON() && DHD_DATA_ON()) {
+				prhex("Rx Data", rxbuf, len);
+			} else if (DHD_HDRS_ON()) {
+				prhex("RxHdr", bus->rxhdr, SDPCM_HDRLEN);
+			}
+#endif
+
+			if (chan == SDPCM_CONTROL_CHANNEL) {
+				if (bus->bus == SPI_BUS) {
+					dhdsdio_read_control(bus, rxbuf, len, doff);
+					if (bus->usebufpool) {
+						dhd_os_sdlock_rxq(bus->dhd);
+						PKTFREE(bus->dhd->osh, pkt, FALSE);
+						dhd_os_sdunlock_rxq(bus->dhd);
+					}
+					continue;
+				} else {
+					DHD_ERROR(("%s (nextlen): readahead on control"
+					           " packet %d?\n", __FUNCTION__, seq));
+					/* Force retry w/normal header read */
+					bus->nextlen = 0;
+					dhdsdio_rxfail(bus, FALSE, TRUE);
+					dhd_os_sdlock_rxq(bus->dhd);
+					PKTFREE2();
+					dhd_os_sdunlock_rxq(bus->dhd);
+					continue;
+				}
+			}
+
+			if ((bus->bus == SPI_BUS) && !bus->usebufpool) {
+				DHD_ERROR(("Received %d bytes on %d channel. Running out of "
+				           "rx pktbuf's or not yet malloced.\n", len, chan));
+				continue;
+			}
+
+			/* Validate data offset */
+			if ((doff < SDPCM_HDRLEN) || (doff > len)) {
+				DHD_ERROR(("%s (nextlen): bad data offset %d: HW len %d min %d\n",
+				           __FUNCTION__, doff, len, SDPCM_HDRLEN));
+				dhd_os_sdlock_rxq(bus->dhd);
+				PKTFREE2();
+				dhd_os_sdunlock_rxq(bus->dhd);
+				ASSERT(0);
+				dhdsdio_rxfail(bus, FALSE, FALSE);
+				continue;
+			}
+
+			/* All done with this one -- now deliver the packet */
+			goto deliver;
+		}
+		/* gSPI frames should not be handled in fractions */
+		if (bus->bus == SPI_BUS) {
+			break;
+		}
+
+		/* Read frame header (hardware and software) */
+		sdret = dhd_bcmsdh_recv_buf(bus, bcmsdh_cur_sbwad(sdh), SDIO_FUNC_2, F2SYNC,
+		                            bus->rxhdr, firstread, NULL, NULL, NULL);
+		bus->f2rxhdrs++;
+		ASSERT(sdret != BCME_PENDING);
+
+		if (sdret < 0) {
+			DHD_ERROR(("%s: RXHEADER FAILED: %d\n", __FUNCTION__, sdret));
+			bus->rx_hdrfail++;
+			dhdsdio_rxfail(bus, TRUE, TRUE);
+			continue;
+		}
+
+#ifdef DHD_DEBUG
+		if (DHD_BYTES_ON() || DHD_HDRS_ON()) {
+			prhex("RxHdr", bus->rxhdr, SDPCM_HDRLEN);
+		}
+#endif
+
+		/* Extract hardware header fields */
+		len = ltoh16_ua(bus->rxhdr);
+		check = ltoh16_ua(bus->rxhdr + sizeof(uint16));
+
+		/* All zeros means no more frames */
+		if (!(len|check)) {
+			*finished = TRUE;
+			break;
+		}
+
+		/* Validate check bytes */
+		if ((uint16)~(len^check)) {
+			DHD_ERROR(("%s: HW hdr error: len/check 0x%04x/0x%04x\n",
+			           __FUNCTION__, len, check));
+			bus->rx_badhdr++;
+			dhdsdio_rxfail(bus, FALSE, FALSE);
+			continue;
+		}
+
+		/* Validate frame length */
+		if (len < SDPCM_HDRLEN) {
+			/* XXX Might choose to allow length 4 for signaling */
+			DHD_ERROR(("%s: HW hdr length invalid: %d\n", __FUNCTION__, len));
+			continue;
+		}
+
+		/* Extract software header fields */
+		chan = SDPCM_PACKET_CHANNEL(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+		seq = SDPCM_PACKET_SEQUENCE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+		doff = SDPCM_DOFFSET_VALUE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+		txmax = SDPCM_WINDOW_VALUE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+
+		/* Validate data offset */
+		if ((doff < SDPCM_HDRLEN) || (doff > len)) {
+			DHD_ERROR(("%s: Bad data offset %d: HW len %d, min %d seq %d\n",
+			           __FUNCTION__, doff, len, SDPCM_HDRLEN, seq));
+			bus->rx_badhdr++;
+			ASSERT(0);
+			dhdsdio_rxfail(bus, FALSE, FALSE);
+			continue;
+		}
+
+		/* Save the readahead length if there is one */
+		bus->nextlen = bus->rxhdr[SDPCM_FRAMETAG_LEN + SDPCM_NEXTLEN_OFFSET];
+		if ((bus->nextlen << 4) > MAX_RX_DATASZ) {
+			DHD_INFO(("%s (nextlen): got frame w/nextlen too large (%d), seq %d\n",
+			          __FUNCTION__, bus->nextlen, seq));
+			bus->nextlen = 0;
+		}
+
+		/* Handle Flow Control */
+		fcbits = SDPCM_FCMASK_VALUE(&bus->rxhdr[SDPCM_FRAMETAG_LEN]);
+
+		delta = 0;
+		if (~bus->flowcontrol & fcbits) {
+			bus->fc_xoff++;
+			delta = 1;
+		}
+		if (bus->flowcontrol & ~fcbits) {
+			bus->fc_xon++;
+			delta = 1;
+		}
+
+		if (delta) {
+			bus->fc_rcvd++;
+			bus->flowcontrol = fcbits;
+		}
+
+		/* Check and update sequence number */
+		if (rxseq != seq) {
+			DHD_INFO(("%s: rx_seq %d, expected %d\n", __FUNCTION__, seq, rxseq));
+			bus->rx_badseq++;
+			rxseq = seq;
+		}
+
+		/* Check window for sanity */
+		if ((uint8)(txmax - bus->tx_seq) > 0x70) {
+			DHD_INFO(("%s: got unlikely tx max %d with tx_seq %d\n",
+			           __FUNCTION__, txmax, bus->tx_seq));
+			txmax = bus->tx_max;
+		}
+		bus->tx_max = txmax;
+
+		/* Call a separate function for control frames */
+		if (chan == SDPCM_CONTROL_CHANNEL) {
+			dhdsdio_read_control(bus, bus->rxhdr, len, doff);
+			continue;
+		}
+
+		ASSERT((chan == SDPCM_DATA_CHANNEL) || (chan == SDPCM_EVENT_CHANNEL) ||
+		       (chan == SDPCM_TEST_CHANNEL) || (chan == SDPCM_GLOM_CHANNEL));
+
+		/* Length to read */
+		rdlen = (len > firstread) ? (len - firstread) : 0;
+
+		/* May pad read to blocksize for efficiency */
+		if (bus->roundup && bus->blocksize) {
+			pad = bus->blocksize - (rdlen % bus->blocksize);
+			if ((pad <= bus->roundup) && (pad < bus->blocksize) &&
+			    ((rdlen + pad + firstread) < MAX_RX_DATASZ))
+				rdlen += pad;
+		} else if (rdlen % DHD_SDALIGN) {
+			rdlen += DHD_SDALIGN - (rdlen % DHD_SDALIGN);
+		}
+
+		/* Satisfy length-alignment requirements */
+		if (forcealign && (rdlen & (ALIGNMENT - 1)))
+			rdlen = ROUNDUP(rdlen, ALIGNMENT);
+
+		if ((rdlen + firstread) > MAX_RX_DATASZ) {
+			/* Too long -- skip this frame */
+			DHD_ERROR(("%s: too long: len %d rdlen %d\n", __FUNCTION__, len, rdlen));
+			bus->dhd->rx_errors++; bus->rx_toolong++;
+			dhdsdio_rxfail(bus, FALSE, FALSE);
+			continue;
+		}
+
+		dhd_os_sdlock_rxq(bus->dhd);
+		if (!(pkt = PKTGET(osh, (rdlen + firstread + DHD_SDALIGN), FALSE))) {
+			/* Give up on data, request rtx of events */
+			DHD_ERROR(("%s: PKTGET failed: rdlen %d chan %d\n",
+			           __FUNCTION__, rdlen, chan));
+			bus->dhd->rx_dropped++;
+			dhd_os_sdunlock_rxq(bus->dhd);
+			dhdsdio_rxfail(bus, FALSE, RETRYCHAN(chan));
+			continue;
+		}
+		dhd_os_sdunlock_rxq(bus->dhd);
+
+		ASSERT(!PKTLINK(pkt));
+
+		/* XXX Should check len for small packets in case we're done? */
+		/* Leave room for what we already read, and align remainder */
+		ASSERT(firstread < (PKTLEN(osh, pkt)));
+		PKTPULL(osh, pkt, firstread);
+		PKTALIGN(osh, pkt, rdlen, DHD_SDALIGN);
+
+		/* Read the remaining frame data */
+		sdret = dhd_bcmsdh_recv_buf(bus, bcmsdh_cur_sbwad(sdh), SDIO_FUNC_2, F2SYNC,
+		                            ((uint8 *)PKTDATA(osh, pkt)), rdlen, pkt, NULL, NULL);
+		bus->f2rxdata++;
+		ASSERT(sdret != BCME_PENDING);
+
+		if (sdret < 0) {
+			DHD_ERROR(("%s: read %d %s bytes failed: %d\n", __FUNCTION__, rdlen,
+			           ((chan == SDPCM_EVENT_CHANNEL) ? "event" :
+			            ((chan == SDPCM_DATA_CHANNEL) ? "data" : "test")), sdret));
+			dhd_os_sdlock_rxq(bus->dhd);
+			PKTFREE(bus->dhd->osh, pkt, FALSE);
+			dhd_os_sdunlock_rxq(bus->dhd);
+			bus->dhd->rx_errors++; /* XXX Account for rtx?? */
+			dhdsdio_rxfail(bus, TRUE, RETRYCHAN(chan));
+			continue;
+		}
+
+		/* Copy the already-read portion */
+		PKTPUSH(osh, pkt, firstread);
+		bcopy(bus->rxhdr, PKTDATA(osh, pkt), firstread);
+
+#ifdef DHD_DEBUG
+		if (DHD_BYTES_ON() && DHD_DATA_ON()) {
+			prhex("Rx Data", PKTDATA(osh, pkt), len);
+		}
+#endif
+
+deliver:
+		/* Save superframe descriptor and allocate packet frame */
+		if (chan == SDPCM_GLOM_CHANNEL) {
+			if (SDPCM_GLOMDESC(&bus->rxhdr[SDPCM_FRAMETAG_LEN])) {
+				DHD_GLOM(("%s: got glom descriptor, %d bytes:\n",
+				          __FUNCTION__, len));
+#ifdef DHD_DEBUG
+				if (DHD_GLOM_ON()) {
+					prhex("Glom Data", PKTDATA(osh, pkt), len);
+				}
+#endif
+				PKTSETLEN(osh, pkt, len);
+				ASSERT(doff == SDPCM_HDRLEN);
+				PKTPULL(osh, pkt, SDPCM_HDRLEN);
+				bus->glomd = pkt;
+			} else {
+				DHD_ERROR(("%s: glom superframe w/o descriptor!\n", __FUNCTION__));
+				dhdsdio_rxfail(bus, FALSE, FALSE);
+			}
+			continue;
+		}
+
+		/* Fill in packet len and prio, deliver upward */
+		PKTSETLEN(osh, pkt, len);
+		PKTPULL(osh, pkt, doff);
+
+#ifdef SDTEST
+		/* Test channel packets are processed separately */
+		if (chan == SDPCM_TEST_CHANNEL) {
+			dhdsdio_testrcv(bus, pkt, seq);
+			continue;
+		}
+#endif /* SDTEST */
+
+		if (PKTLEN(osh, pkt) == 0) {
+			dhd_os_sdlock_rxq(bus->dhd);
+			PKTFREE(bus->dhd->osh, pkt, FALSE);
+			dhd_os_sdunlock_rxq(bus->dhd);
+			continue;
+		} else if (dhd_prot_hdrpull(bus->dhd, &ifidx, pkt, reorder_info_buf,
+			&reorder_info_len) != 0) {
+			DHD_ERROR(("%s: rx protocol error\n", __FUNCTION__));
+			dhd_os_sdlock_rxq(bus->dhd);
+			PKTFREE(bus->dhd->osh, pkt, FALSE);
+			dhd_os_sdunlock_rxq(bus->dhd);
+			bus->dhd->rx_errors++;
+			continue;
+		}
+
+		if (reorder_info_len) {
+			/* Reordering info from the firmware */
+			dhd_process_pkt_reorder_info(bus->dhd, reorder_info_buf, reorder_info_len,
+				&pkt, &pkt_count);
+			if (pkt_count == 0)
+				continue;
+		} else {
+			pkt_count = 1;
+		}
+
+		/* XXX Release the lock around the rx delivery: an OS (like Windows)
+		 * might call tx in the same thread context, resulting in deadlock.
+		 */
+		/* Unlock during rx call */
+		dhd_os_sdunlock(bus->dhd);
+		dhd_rx_frame(bus->dhd, ifidx, pkt, pkt_count, chan);
+		dhd_os_sdlock(bus->dhd);
+#if defined(SDIO_ISR_THREAD)
+		/* terence 20150615: fix for below error due to bussleep in watchdog after dhd_os_sdunlock here,
+		  * so call BUS_WAKE to wake up bus again
+		  * dhd_bcmsdh_recv_buf: Device asleep
+		  * dhdsdio_readframes: RXHEADER FAILED: -40
+		  * dhdsdio_rxfail: abort command, terminate frame, send NAK
+		*/
+		BUS_WAKE(bus);
+#endif
+	}
+	rxcount = maxframes - rxleft;
+#ifdef DHD_DEBUG
+	/* Message if we hit the limit */
+	if (!rxleft && !sdtest)
+		DHD_DATA(("%s: hit rx limit of %d frames\n", __FUNCTION__, maxframes));
+	else
+#endif /* DHD_DEBUG */
+	DHD_DATA(("%s: processed %d frames\n", __FUNCTION__, rxcount));
+	/* Back off rxseq if awaiting rtx, update rx_seq */
+	if (bus->rxskip)
+		rxseq--;
+	bus->rx_seq = rxseq;
+
+	if (bus->reqbussleep)
+	{
+		dhdsdio_bussleep(bus, TRUE);
+		bus->reqbussleep = FALSE;
+	}
+	bus->readframes = FALSE;
+
+	return rxcount;
+}
+
+static uint32
+dhdsdio_hostmail(dhd_bus_t *bus, uint32 *hmbd)
+{
+	sdpcmd_regs_t *regs = bus->regs;
+	uint32 intstatus = 0;
+	uint32 hmb_data;
+	uint8 fcbits;
+	uint retries = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	/* Read mailbox data and ack that we did so */
+	R_SDREG(hmb_data, &regs->tohostmailboxdata, retries);
+	if (retries <= retry_limit)
+		W_SDREG(SMB_INT_ACK, &regs->tosbmailbox, retries);
+	bus->f1regdata += 2;
+
+	/* Dongle recomposed rx frames, accept them again */
+	if (hmb_data & HMB_DATA_NAKHANDLED) {
+		DHD_INFO(("Dongle reports NAK handled, expect rtx of %d\n", bus->rx_seq));
+		/* XXX ASSERT(bus->rxskip); */
+		if (!bus->rxskip) {
+			DHD_ERROR(("%s: unexpected NAKHANDLED!\n", __FUNCTION__));
+		}
+		bus->rxskip = FALSE;
+		intstatus |= FRAME_AVAIL_MASK(bus);
+	}
+
+	/*
+	 * DEVREADY does not occur with gSPI.
+	 */
+	if (hmb_data & (HMB_DATA_DEVREADY | HMB_DATA_FWREADY)) {
+		bus->sdpcm_ver = (hmb_data & HMB_DATA_VERSION_MASK) >> HMB_DATA_VERSION_SHIFT;
+		if (bus->sdpcm_ver != SDPCM_PROT_VERSION)
+			DHD_ERROR(("Version mismatch, dongle reports %d, expecting %d\n",
+			           bus->sdpcm_ver, SDPCM_PROT_VERSION));
+		else
+			DHD_INFO(("Dongle ready, protocol version %d\n", bus->sdpcm_ver));
+#ifndef BCMSPI
+		/* make sure for the SDIO_DEVICE_RXDATAINT_MODE_1 corecontrol is proper */
+		if ((bus->sih->buscoretype == SDIOD_CORE_ID) && (bus->sdpcmrev >= 4) &&
+		    (bus->rxint_mode  == SDIO_DEVICE_RXDATAINT_MODE_1)) {
+			uint32 val;
+
+			val = R_REG(bus->dhd->osh, &bus->regs->corecontrol);
+			val &= ~CC_XMTDATAAVAIL_MODE;
+			val |= CC_XMTDATAAVAIL_CTRL;
+			W_REG(bus->dhd->osh, &bus->regs->corecontrol, val);
+
+			val = R_REG(bus->dhd->osh, &bus->regs->corecontrol);
+		}
+#endif /* BCMSPI */
+
+// open FW cons and log by default
+#if defined(DHD_DEBUG) || defined(DHD_FW_LOG_SUPPORT) || defined(DHD_FW_CONS_SUPPORT)
+		/* Retrieve console state address now that firmware should have updated it */
+		{
+			sdpcm_shared_t shared;
+			if (dhdsdio_readshared(bus, &shared) == 0)
+				bus->console_addr = shared.console_addr;
+		}
+#endif /* DHD_DEBUG || DHD_FW_LOG_SUPPORT || DHD_FW_CONS_SUPPORT ) */
+	}
+
+	/*
+	 * Flow Control has been moved into the RX headers and this out of band
+	 * method isn't used any more.  Leave this here for possibly remaining backward
+	 * compatible with older dongles
+	 */
+	if (hmb_data & HMB_DATA_FC) {
+		fcbits = (hmb_data & HMB_DATA_FCDATA_MASK) >> HMB_DATA_FCDATA_SHIFT;
+
+		if (fcbits & ~bus->flowcontrol)
+			bus->fc_xoff++;
+		if (bus->flowcontrol & ~fcbits)
+			bus->fc_xon++;
+
+		bus->fc_rcvd++;
+		bus->flowcontrol = fcbits;
+	}
+
+	/* At least print a message if FW halted */
+	if (hmb_data & HMB_DATA_FWHALT) {
+		DHD_ERROR(("FIRMWARE HALTED\n"));
+		dhdsdio_checkdied(bus, NULL, 0);
+	}
+
+	/* Shouldn't be any others */
+	if (hmb_data & ~(HMB_DATA_DEVREADY |
+	                 HMB_DATA_FWHALT |
+	                 HMB_DATA_NAKHANDLED |
+	                 HMB_DATA_FC |
+	                 HMB_DATA_FWREADY |
+	                 HMB_DATA_FCDATA_MASK |
+	                 HMB_DATA_VERSION_MASK)) {
+		DHD_ERROR(("Unknown mailbox data content: 0x%02x\n", hmb_data));
+	}
+
+	if (hmbd) {
+		*hmbd = hmb_data;
+	}
+
+	return intstatus;
+}
+
+#ifdef BCMSDIO_INTSTATUS_WAR
+static uint32
+dhdsdio_read_intstatus_byte(dhd_bus_t *bus)
+{
+	bcmsdh_info_t *sdh = bus->sdh;
+	sdpcmd_regs_t *regs = bus->regs;
+	uint32 newstatus = 0, intstatus_byte = 0;
+	uint retries = 0;
+	int err1 = 0, err2 = 0, err3 = 0, err4 = 0;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	/* read_intr_mode:
+	  * 0: word mode only (default)
+	  * 1: byte mode after read word failed
+	  * 2: byte mode only
+	*/
+	if (bus->dhd->conf->read_intr_mode) {
+		if (bus->dhd->conf->read_intr_mode == 1) {
+			R_SDREG(newstatus, &regs->intstatus, retries);
+			if (!bcmsdh_regfail(bus->sdh)) {
+				goto exit;
+			}
+		}
+		intstatus_byte = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+			((unsigned long)&regs->intstatus & 0xffff) + 0, &err1);
+		if (!err1)
+			newstatus |= intstatus_byte;
+		intstatus_byte = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+			((unsigned long)&regs->intstatus & 0xffff) + 1, &err2) << 8;
+		if (!err2)
+			newstatus |= intstatus_byte;
+		intstatus_byte |= bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+			((unsigned long)&regs->intstatus & 0xffff) + 2, &err3) << 16;
+		if (!err3)
+			newstatus |= intstatus_byte;
+		intstatus_byte |= bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_1,
+			((unsigned long)&regs->intstatus & 0xffff) + 3, &err4) << 24;
+		if (!err4)
+			newstatus |= intstatus_byte;
+
+		if (!err1 || !err2 || !err3 || !err4)
+			sdh->regfail = FALSE;
+	}
+	else {
+		R_SDREG(newstatus, &regs->intstatus, retries);
+	}
+
+exit:
+	return newstatus;
+}
+#endif
+
+static bool
+dhdsdio_dpc(dhd_bus_t *bus)
+{
+	bcmsdh_info_t *sdh = bus->sdh;
+	sdpcmd_regs_t *regs = bus->regs;
+	uint32 intstatus, newstatus = 0;
+	uint retries = 0;
+	uint rxlimit = dhd_rxbound; /* Rx frames to read before resched */
+	uint txlimit = dhd_txbound; /* Tx frames to send before resched */
+	uint framecnt = 0;		  /* Temporary counter of tx/rx frames */
+	bool rxdone = TRUE;		  /* Flag for no more read data */
+	bool resched = FALSE;	  /* Flag indicating resched wanted */
+	unsigned long flags;
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+	bool is_resched_by_readframe = FALSE;
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	dhd_os_sdlock(bus->dhd);
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	if (bus->dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s: Bus down, ret\n", __FUNCTION__));
+		bus->intstatus = 0;
+		DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+		dhd_os_sdunlock(bus->dhd);
+		return 0;
+	}
+
+	DHD_BUS_BUSY_SET_IN_DPC(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+	/* Start with leftover status bits */
+	intstatus = bus->intstatus;
+
+	if (!SLPAUTO_ENAB(bus) && !KSO_ENAB(bus)) {
+		DHD_ERROR(("%s: Device asleep\n", __FUNCTION__));
+		goto exit;
+	}
+
+	/* If waiting for HTAVAIL, check status */
+	if (!SLPAUTO_ENAB(bus) && (bus->clkstate == CLK_PENDING)) {
+		int err;
+		uint8 clkctl, devctl = 0;
+
+#ifdef DHD_DEBUG
+		/* Check for inconsistent device control */
+		devctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err);
+		if (err) {
+			DHD_ERROR(("%s: error reading DEVCTL: %d\n", __FUNCTION__, err));
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+		} else {
+			ASSERT(devctl & SBSDIO_DEVCTL_CA_INT_ONLY);
+		}
+#endif /* DHD_DEBUG */
+
+		/* Read CSR, if clock on switch to AVAIL, else ignore */
+		clkctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+		if (err) {
+			DHD_ERROR(("%s: error reading CSR: %d\n", __FUNCTION__, err));
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+		}
+
+		DHD_INFO(("DPC: PENDING, devctl 0x%02x clkctl 0x%02x\n", devctl, clkctl));
+
+		if (SBSDIO_HTAV(clkctl)) {
+			devctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, &err);
+			if (err) {
+				DHD_ERROR(("%s: error reading DEVCTL: %d\n",
+				           __FUNCTION__, err));
+				DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+				bus->dhd->busstate = DHD_BUS_DOWN;
+			}
+			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_DEVICE_CTL, devctl, &err);
+			if (err) {
+				DHD_ERROR(("%s: error writing DEVCTL: %d\n",
+				           __FUNCTION__, err));
+				DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+				bus->dhd->busstate = DHD_BUS_DOWN;
+			}
+			bus->clkstate = CLK_AVAIL;
+		} else {
+			goto clkwait;
+		}
+	}
+
+	BUS_WAKE(bus);
+
+	/* Make sure backplane clock is on */
+	dhdsdio_clkctl(bus, CLK_AVAIL, TRUE);
+	if (bus->clkstate != CLK_AVAIL)
+		goto clkwait;
+
+	/* Pending interrupt indicates new device status */
+	if (bus->ipend) {
+		bus->ipend = FALSE;
+#if defined(BT_OVER_SDIO)
+	bcmsdh_btsdio_process_f3_intr();
+#endif /* defined (BT_OVER_SDIO) */
+
+#ifdef BCMSDIO_INTSTATUS_WAR
+		newstatus = dhdsdio_read_intstatus_byte(bus);
+#else
+		R_SDREG(newstatus, &regs->intstatus, retries);
+#endif
+		bus->f1regdata++;
+		if (bcmsdh_regfail(bus->sdh))
+			newstatus = 0;
+		newstatus &= bus->hostintmask;
+		bus->fcstate = !!(newstatus & I_HMB_FC_STATE);
+		if (newstatus) {
+			bus->f1regdata++;
+#ifndef BCMSPI
+			if ((bus->rxint_mode == SDIO_DEVICE_RXDATAINT_MODE_0) &&
+				(newstatus == I_XMTDATA_AVAIL)) {
+			} else
+#endif /* BCMSPI */
+				W_SDREG(newstatus, &regs->intstatus, retries);
+		}
+	}
+
+	/* Merge new bits with previous */
+	intstatus |= newstatus;
+	bus->intstatus = 0;
+
+	/* Handle flow-control change: read new state in case our ack
+	 * crossed another change interrupt.  If change still set, assume
+	 * FC ON for safety, let next loop through do the debounce.
+	 */
+	if (intstatus & I_HMB_FC_CHANGE) {
+		intstatus &= ~I_HMB_FC_CHANGE;
+		W_SDREG(I_HMB_FC_CHANGE, &regs->intstatus, retries);
+		R_SDREG(newstatus, &regs->intstatus, retries);
+		bus->f1regdata += 2;
+		bus->fcstate = !!(newstatus & (I_HMB_FC_STATE | I_HMB_FC_CHANGE));
+		intstatus |= (newstatus & bus->hostintmask);
+	}
+
+	/* Handle host mailbox indication */
+	if (intstatus & I_HMB_HOST_INT) {
+		uint32 hmbdata = 0;
+
+		intstatus &= ~I_HMB_HOST_INT;
+		intstatus |= dhdsdio_hostmail(bus, &hmbdata);
+
+	}
+
+#ifdef DHD_UCODE_DOWNLOAD
+exit_ucode:
+#endif /* DHD_UCODE_DOWNLOAD */
+
+	/* Just being here means nothing more to do for chipactive */
+	if (intstatus & I_CHIPACTIVE) {
+		/* ASSERT(bus->clkstate == CLK_AVAIL); */
+		intstatus &= ~I_CHIPACTIVE;
+	}
+
+	/* Handle host mailbox indication */
+	if (intstatus & I_HMB_HOST_INT) {
+		intstatus &= ~I_HMB_HOST_INT;
+		intstatus |= dhdsdio_hostmail(bus, NULL);
+	}
+
+	/* Generally don't ask for these, can get CRC errors... */
+	/* XXX Besides noting the error, should we ABORT/TERM? */
+	if (intstatus & I_WR_OOSYNC) {
+		DHD_ERROR(("Dongle reports WR_OOSYNC\n"));
+		intstatus &= ~I_WR_OOSYNC;
+	}
+
+	if (intstatus & I_RD_OOSYNC) {
+		DHD_ERROR(("Dongle reports RD_OOSYNC\n"));
+		intstatus &= ~I_RD_OOSYNC;
+	}
+
+	/* XXX Should reset or something here... */
+	if (intstatus & I_SBINT) {
+		DHD_ERROR(("Dongle reports SBINT\n"));
+		intstatus &= ~I_SBINT;
+	}
+
+	/* Would be active due to wake-wlan in gSPI */
+	if (intstatus & I_CHIPACTIVE) {
+		DHD_INFO(("Dongle reports CHIPACTIVE\n"));
+		intstatus &= ~I_CHIPACTIVE;
+	}
+
+	if (intstatus & I_HMB_FC_STATE) {
+		DHD_INFO(("Dongle reports HMB_FC_STATE\n"));
+		intstatus &= ~I_HMB_FC_STATE;
+	}
+
+	/* Ignore frame indications if rxskip is set */
+	if (bus->rxskip) {
+		intstatus &= ~FRAME_AVAIL_MASK(bus);
+	}
+
+	/* On frame indication, read available frames */
+	if (PKT_AVAILABLE(bus, intstatus)) {
+
+		framecnt = dhdsdio_readframes(bus, rxlimit, &rxdone);
+		if (rxdone || bus->rxskip)
+			intstatus  &= ~FRAME_AVAIL_MASK(bus);
+		rxlimit -= MIN(framecnt, rxlimit);
+	}
+
+	/* Keep still-pending events for next scheduling */
+	bus->intstatus = intstatus;
+
+clkwait:
+	/* Re-enable interrupts to detect new device events (mailbox, rx frame)
+	 * or clock availability.  (Allows tx loop to check ipend if desired.)
+	 * (Unless register access seems hosed, as we may not be able to ACK...)
+	 */
+	if (bus->intr && bus->intdis && !bcmsdh_regfail(sdh) &&
+			!(bus->dhd->conf->oob_enabled_later && !bus->ctrl_frame_stat)) {
+		DHD_INTR(("%s: enable SDIO interrupts, rxdone %d framecnt %d\n",
+		          __FUNCTION__, rxdone, framecnt));
+		bus->intdis = FALSE;
+#if defined(OOB_INTR_ONLY)
+		bcmsdh_oob_intr_set(bus->sdh, TRUE);
+#endif /* defined(OOB_INTR_ONLY) */
+#if !defined(NDIS)
+		bcmsdh_intr_enable(sdh);
+#endif /* !defined(NDIS) */
+#ifdef BCMSPI_ANDROID
+		if (*dhd_spi_lockcount == 0)
+			bcmsdh_oob_intr_set(bus->sdh, TRUE);
+#endif /* BCMSPI_ANDROID */
+	}
+
+#if defined(OOB_INTR_ONLY) && !defined(HW_OOB)
+	/* In case of SW-OOB(using edge trigger),
+	 * Check interrupt status in the dongle again after enable irq on the host.
+	 * and rechedule dpc if interrupt is pended in the dongle.
+	 * There is a chance to miss OOB interrupt while irq is disabled on the host.
+	 * No need to do this with HW-OOB(level trigger)
+	 */
+	R_SDREG(newstatus, &regs->intstatus, retries);
+	if (bcmsdh_regfail(bus->sdh))
+		newstatus = 0;
+	if (newstatus & bus->hostintmask) {
+		bus->ipend = TRUE;
+		resched = TRUE;
+	}
+#endif /* defined(OOB_INTR_ONLY) && !defined(HW_OOB) */
+
+#ifdef BCMSDIO_RXLIM_POST
+	if (!DATAOK(bus) && bus->rxlim_en) {
+		uint8 rxlim = 0;
+		if (0 == dhdsdio_membytes(bus, FALSE, bus->rxlim_addr, (uint8 *)&rxlim, 1)) {
+			if (bus->tx_max != rxlim) {
+				DHD_INFO(("%s: bus->tx_max/rxlim=%d/%d\n", __FUNCTION__,
+					bus->tx_max, rxlim));
+				bus->tx_max = rxlim;
+			}
+		}
+	}
+#endif /* BCMSDIO_RXLIM_POST */
+
+#ifdef PROP_TXSTATUS
+	dhd_wlfc_commit_packets(bus->dhd, (f_commitpkt_t)dhd_bus_txdata, (void *)bus, NULL, FALSE);
+#endif
+
+	if (TXCTLOK(bus) && bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL))
+		dhdsdio_sendpendctl(bus);
+#ifdef CONSOLE_DPC
+	else if (DATAOK(bus) && strlen(bus->cons_cmd) && (bus->clkstate == CLK_AVAIL) &&
+			!bus->fcstate) {
+		dhd_bus_console_in(bus->dhd, bus->cons_cmd, strlen(bus->cons_cmd));
+	}
+#endif
+
+	/* Send queued frames (limit 1 if rx may still be pending) */
+	else if ((bus->clkstate == CLK_AVAIL) && !bus->fcstate &&
+	    pktq_mlen(&bus->txq, ~bus->flowcontrol) && txlimit && DATAOK(bus)) {
+
+			if (bus->dhd->conf->dhd_txminmax < 0)
+				framecnt = rxdone ? txlimit : MIN(txlimit, DATABUFCNT(bus));
+			else
+				framecnt = rxdone ? txlimit : MIN(txlimit, bus->dhd->conf->dhd_txminmax);
+			framecnt = dhdsdio_sendfromq(bus, framecnt);
+			txlimit -= framecnt;
+	}
+	/* Resched the DPC if ctrl cmd is pending on bus credit */
+	if (bus->ctrl_frame_stat) {
+		if (bus->dhd->conf->txctl_tmo_fix) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			if (!kthread_should_stop())
+				schedule_timeout(1);
+			set_current_state(TASK_RUNNING);
+		}
+		resched = TRUE;
+	}
+
+	/* Resched if events or tx frames are pending, else await next interrupt */
+	/* On failed register access, all bets are off: no resched or interrupts */
+	if ((bus->dhd->busstate == DHD_BUS_DOWN) || bcmsdh_regfail(sdh)) {
+		if ((bus->sih && bus->sih->buscorerev >= 12) && !(dhdsdio_sleepcsr_get(bus) &
+			SBSDIO_FUNC1_SLEEPCSR_KSO_MASK)) {
+			/* Bus failed because of KSO */
+			DHD_ERROR(("%s: Bus failed due to KSO\n", __FUNCTION__));
+			bus->kso = FALSE;
+		} else {
+			DHD_ERROR(("%s: failed backplane access over SDIO, halting operation\n",
+				__FUNCTION__));
+			DHD_ERROR(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+			bus->intstatus = 0;
+			/* XXX Under certain conditions it may be reasonable to enable interrupts.
+			 * E.g. if we get occasional 'bcmsdh_regfail' we should be able to continue
+			 * operation. May want to make the decision to enable or not based on count
+			 * of failures, so in case of bus lock up we avoid continuous interrupt.
+			 */
+		}
+	} else if (bus->clkstate == CLK_PENDING) {
+		/* Awaiting I_CHIPACTIVE; don't resched */
+	} else if (bus->intstatus || bus->ipend ||
+			(!bus->fcstate && (pktq_mlen(&bus->txq, ((~bus->flowcontrol)
+#ifdef DHD_LOSSLESS_ROAMING
+			& bus->dhd->dequeue_prec_map
+#endif /* DHD_LOSSLESS_ROAMING */
+			))) && DATAOK(bus)) ||
+			PKT_AVAILABLE(bus, bus->intstatus)) {  /* Read multiple frames */
+		resched = TRUE;
+	}
+
+	bus->dpc_sched = resched;
+
+	/* If we're done for now, turn off clock request. */
+	/* XXX Leave request on if just waiting for new credit? */
+	if ((bus->idletime == DHD_IDLE_IMMEDIATE) && (bus->clkstate != CLK_PENDING) &&
+		NO_OTHER_ACTIVE_BUS_USER(bus)) {
+		bus->activity = FALSE;
+		dhdsdio_bussleep(bus, TRUE);
+		dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+	}
+
+exit:
+
+	if (!resched) {
+		/* Re-enable interrupts to detect new device events (mailbox, rx frame)
+		 * or clock availability.  (Allows tx loop to check ipend if desired.)
+		 * (Unless register access seems hosed, as we may not be able to ACK...)
+		 */
+		if (bus->intr && bus->intdis && !bcmsdh_regfail(sdh) &&
+				(bus->dhd->conf->oob_enabled_later && !bus->ctrl_frame_stat)) {
+			DHD_INTR(("%s: enable SDIO interrupts, rxdone %d framecnt %d\n",
+					  __FUNCTION__, rxdone, framecnt));
+			bus->intdis = FALSE;
+#if defined(OOB_INTR_ONLY)
+			bcmsdh_oob_intr_set(bus->sdh, TRUE);
+#endif /* defined(OOB_INTR_ONLY) */
+			bcmsdh_intr_enable(sdh);
+		}
+		if (dhd_dpcpoll) {
+			if (dhdsdio_readframes(bus, dhd_rxbound, &rxdone) != 0) {
+				resched = TRUE;
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+				is_resched_by_readframe = TRUE;
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+			}
+		}
+	}
+
+#ifdef TPUT_MONITOR
+	dhd_conf_tput_monitor(bus->dhd);
+#endif
+
+	if (bus->ctrl_wait && TXCTLOK(bus))
+		wake_up_interruptible(&bus->ctrl_tx_wait);
+	dhd_os_sdunlock(bus->dhd);
+#ifdef DEBUG_DPC_THREAD_WATCHDOG
+	if (bus->dhd->dhd_bug_on) {
+		DHD_INFO(("%s: resched = %d ctrl_frame_stat = %d intstatus 0x%08x"
+			" ipend = %d pktq_mlen = %d is_resched_by_readframe = %d \n",
+			__FUNCTION__, resched, bus->ctrl_frame_stat,
+			bus->intstatus, bus->ipend,
+			pktq_mlen(&bus->txq, ~bus->flowcontrol), is_resched_by_readframe));
+
+			bus->dhd->dhd_bug_on = FALSE;
+	}
+#endif /* DEBUG_DPC_THREAD_WATCHDOG */
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_IN_DPC(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+	return resched;
+}
+
+bool
+dhd_bus_dpc(struct dhd_bus *bus)
+{
+	bool resched;
+#ifdef DHD_WAKE_STATUS
+	dhd_pub_t  *dhdp = bus->dhd;
+#endif
+
+	/* Call the DPC directly. */
+	DHD_TRACE(("Calling dhdsdio_dpc() from %s\n", __FUNCTION__));
+#ifdef DHD_WAKE_STATUS
+	if (dhdp && dhdp->in_suspend)
+		bcmsdh_set_get_wake(bus->sdh, 1);
+#endif
+	resched = dhdsdio_dpc(bus);
+
+	return resched;
+}
+
+void
+dhdsdio_isr(void *arg)
+{
+	dhd_bus_t *bus = (dhd_bus_t*)arg;
+	bcmsdh_info_t *sdh;
+
+	if (!bus) {
+		DHD_ERROR(("%s : bus is null pointer , exit \n", __FUNCTION__));
+		return;
+	}
+	sdh = bus->sdh;
+
+	if (bus->dhd->busstate == DHD_BUS_DOWN) {
+		DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
+		return;
+	}
+	/* XXX Overall operation:
+	 * XXX   - Mask further interrupts
+	 * XXX   - Read/ack intstatus
+	 * XXX   - Take action based on bits and state
+	 * XXX   - Reenable interrupts (as per state)
+	 */
+
+	DHD_INTR(("%s: Enter\n", __FUNCTION__));
+
+	/* Count the interrupt call */
+	bus->intrcount++;
+	bus->ipend = TRUE;
+
+	/* Shouldn't get this interrupt if we're sleeping? */
+	if (!SLPAUTO_ENAB(bus)) {
+		if (bus->sleeping) {
+			DHD_ERROR(("INTERRUPT WHILE SLEEPING??\n"));
+			return;
+		} else if (!KSO_ENAB(bus)) {
+			DHD_ERROR(("ISR in devsleep 1\n"));
+		}
+	}
+
+	/* Disable additional interrupts (is this needed now)? */
+	if (bus->intr) {
+		DHD_INTR(("%s: disable SDIO interrupts\n", __FUNCTION__));
+	} else {
+		DHD_ERROR(("dhdsdio_isr() w/o interrupt configured!\n"));
+	}
+
+#ifdef BCMSPI_ANDROID
+	bcmsdh_oob_intr_set(bus->sdh, FALSE);
+#endif /* BCMSPI_ANDROID */
+#if !defined(NDIS)
+	bcmsdh_intr_disable(sdh); /* XXX New API: bcmsdh_intr_mask()? */
+#endif /* !defined(NDIS) */
+	bus->intdis = TRUE;
+
+#if defined(SDIO_ISR_THREAD)
+	DHD_TRACE(("Calling dhdsdio_dpc() from %s\n", __FUNCTION__));
+	DHD_OS_WAKE_LOCK(bus->dhd);
+	/* terence 20150209: dpc should be scheded again if dpc_sched is TRUE or dhd_bus_txdata can
+	    not schedule anymore because dpc_sched is TRUE now.
+	 */
+	if (dhdsdio_dpc(bus)) {
+		bus->dpc_sched = TRUE;
+		dhd_sched_dpc(bus->dhd);
+	}
+	DHD_OS_WAKE_UNLOCK(bus->dhd);
+#else // SDIO_ISR_THREAD
+#if !defined(NDIS)
+	bus->dpc_sched = TRUE;
+	dhd_sched_dpc(bus->dhd);
+#endif /* !defined(NDIS) */
+#endif /* defined(SDIO_ISR_THREAD) */
+
+}
+
+#ifdef PKT_STATICS
+void
+dhd_bus_dump_txpktstatics(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	uint32 total = 0;
+	uint i;
+
+	printf("%s: TYPE EVENT: %d pkts (size=%d) transfered\n",
+		__FUNCTION__, bus->tx_statics.event_count, bus->tx_statics.event_size);
+	printf("%s: TYPE CTRL:  %d pkts (size=%d) transfered\n",
+		__FUNCTION__, bus->tx_statics.ctrl_count, bus->tx_statics.ctrl_size);
+	printf("%s: TYPE DATA:  %d pkts (size=%d) transfered\n",
+		__FUNCTION__, bus->tx_statics.data_count, bus->tx_statics.data_size);
+	printf("%s: Glom size distribution:\n", __FUNCTION__);
+	for (i=0;i<bus->tx_statics.glom_max;i++) {
+		total += bus->tx_statics.glom_cnt[i];
+	}
+	printk(KERN_CONT DHD_LOG_PREFIXS);
+	for (i=0;i<bus->tx_statics.glom_max;i++) {
+		printk(KERN_CONT "%02d: %5d", i+1, bus->tx_statics.glom_cnt[i]);
+		if ((i+1)%8)
+			printk(KERN_CONT ", ");
+		else {
+			printk("\n");
+			printk(KERN_CONT DHD_LOG_PREFIXS);
+		}
+	}
+	printk("\n");
+	printk(KERN_CONT DHD_LOG_PREFIXS);
+	for (i=0;i<bus->tx_statics.glom_max;i++) {
+		printk(KERN_CONT "%02d:%5d%%", i+1, (bus->tx_statics.glom_cnt[i]*100)/total);
+		if ((i+1)%8)
+			printk(KERN_CONT ", ");
+		else {
+			printk("\n");
+			printk(KERN_CONT DHD_LOG_PREFIXS);
+		}
+	}
+	printk("\n");
+	printf("%s: Glom spend time distribution(us):\n", __FUNCTION__);
+	printk(KERN_CONT DHD_LOG_PREFIXS);
+	for (i=0;i<bus->tx_statics.glom_max;i++) {
+		printk(KERN_CONT "%02d: %5u", i+1, bus->tx_statics.glom_cnt_us[i]);
+		if ((i+1)%8)
+			printk(KERN_CONT ", ");
+		else {
+			printk("\n");
+			printk(KERN_CONT DHD_LOG_PREFIXS);
+		}
+	}
+	printk("\n");
+	if (total) {
+		printf("%s: data(%d)/glom(%d)=%d, glom_max=%d\n",
+			__FUNCTION__, bus->tx_statics.data_count, total,
+			bus->tx_statics.data_count/total, bus->tx_statics.glom_max);
+	}
+	printf("%s: TYPE RX GLOM: %d pkts (size=%d) transfered\n",
+		__FUNCTION__, bus->tx_statics.glom_count, bus->tx_statics.glom_size);
+	printf("%s: TYPE TEST: %d pkts (size=%d) transfered\n",
+		__FUNCTION__, bus->tx_statics.test_count, bus->tx_statics.test_size);
+
+#ifdef KSO_DEBUG
+	printf("%s: kso try distribution(us):\n", __FUNCTION__);
+	printk(KERN_CONT DHD_LOG_PREFIXS);
+	for (i=0; i<10; i++) {
+		printk(KERN_CONT "[%d]: %d, ", i, dhdp->conf->kso_try_array[i]);
+	}
+	printk("\n");
+#endif
+}
+
+void
+dhd_bus_clear_txpktstatics(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	memset((uint8*) &bus->tx_statics, 0, sizeof(pkt_statics_t));
+}
+#endif
+
+#ifdef SDTEST
+static void
+dhdsdio_pktgen_init(dhd_bus_t *bus)
+{
+	/* Default to specified length, or full range */
+	if (dhd_pktgen_len) {
+		bus->pktgen_maxlen = MIN(dhd_pktgen_len, MAX_PKTGEN_LEN);
+		bus->pktgen_minlen = bus->pktgen_maxlen;
+	} else {
+		bus->pktgen_maxlen = MAX_PKTGEN_LEN;
+		bus->pktgen_minlen = 0;
+	}
+	bus->pktgen_len = (uint16)bus->pktgen_minlen;
+
+	/* Default to per-watchdog burst with 10s print time */
+	bus->pktgen_freq = 1;
+	bus->pktgen_print = dhd_watchdog_ms ? (10000 / dhd_watchdog_ms) : 0;
+	bus->pktgen_count = (dhd_pktgen * dhd_watchdog_ms + 999) / 1000;
+
+	/* Default to echo mode */
+	bus->pktgen_mode = DHD_PKTGEN_ECHO;
+	bus->pktgen_stop = 1;
+}
+
+static void
+dhdsdio_pktgen(dhd_bus_t *bus)
+{
+	void *pkt;
+	uint8 *data;
+	uint pktcount;
+	uint fillbyte;
+	osl_t *osh = bus->dhd->osh;
+	uint16 len;
+#if defined(LINUX)
+	ulong time_lapse;
+	uint sent_pkts;
+	uint rcvd_pkts;
+#endif /* LINUX */
+
+	/* Display current count if appropriate */
+	if (bus->pktgen_print && (++bus->pktgen_ptick >= bus->pktgen_print)) {
+		bus->pktgen_ptick = 0;
+		printf("%s: send attempts %d, rcvd %d, errors %d\n",
+		       __FUNCTION__, bus->pktgen_sent, bus->pktgen_rcvd, bus->pktgen_fail);
+
+#if defined(LINUX)
+		/* Print throughput stats only for constant length packet runs */
+		if (bus->pktgen_minlen == bus->pktgen_maxlen) {
+			time_lapse = jiffies - bus->pktgen_prev_time;
+			bus->pktgen_prev_time = jiffies;
+			sent_pkts = bus->pktgen_sent - bus->pktgen_prev_sent;
+			bus->pktgen_prev_sent = bus->pktgen_sent;
+			rcvd_pkts = bus->pktgen_rcvd - bus->pktgen_prev_rcvd;
+			bus->pktgen_prev_rcvd = bus->pktgen_rcvd;
+
+			printf("%s: Tx Throughput %d kbps, Rx Throughput %d kbps\n",
+			  __FUNCTION__,
+			  (sent_pkts * bus->pktgen_len / jiffies_to_msecs(time_lapse)) * 8,
+			  (rcvd_pkts * bus->pktgen_len  / jiffies_to_msecs(time_lapse)) * 8);
+		}
+#endif /* LINUX */
+	}
+
+	/* For recv mode, just make sure dongle has started sending */
+	if (bus->pktgen_mode == DHD_PKTGEN_RECV) {
+		if (bus->pktgen_rcv_state == PKTGEN_RCV_IDLE) {
+			bus->pktgen_rcv_state = PKTGEN_RCV_ONGOING;
+			dhdsdio_sdtest_set(bus, bus->pktgen_total);
+		}
+		return;
+	}
+
+	/* Otherwise, generate or request the specified number of packets */
+	for (pktcount = 0; pktcount < bus->pktgen_count; pktcount++) {
+		/* Stop if total has been reached */
+		if (bus->pktgen_total && (bus->pktgen_sent >= bus->pktgen_total)) {
+			bus->pktgen_count = 0;
+			break;
+		}
+
+		/* Allocate an appropriate-sized packet */
+		if (bus->pktgen_mode == DHD_PKTGEN_RXBURST) {
+			len = SDPCM_TEST_PKT_CNT_FLD_LEN;
+		} else {
+			len = bus->pktgen_len;
+		}
+		if (!(pkt = PKTGET(osh, (len + SDPCM_HDRLEN + SDPCM_TEST_HDRLEN + DHD_SDALIGN),
+		                   TRUE))) {;
+			DHD_ERROR(("%s: PKTGET failed!\n", __FUNCTION__));
+			break;
+		}
+		PKTALIGN(osh, pkt, (len + SDPCM_HDRLEN + SDPCM_TEST_HDRLEN), DHD_SDALIGN);
+		data = (uint8*)PKTDATA(osh, pkt) + SDPCM_HDRLEN;
+
+		/* Write test header cmd and extra based on mode */
+		switch (bus->pktgen_mode) {
+		case DHD_PKTGEN_ECHO:
+			*data++ = SDPCM_TEST_ECHOREQ;
+			*data++ = (uint8)bus->pktgen_sent;
+			break;
+
+		case DHD_PKTGEN_SEND:
+			*data++ = SDPCM_TEST_DISCARD;
+			*data++ = (uint8)bus->pktgen_sent;
+			break;
+
+		case DHD_PKTGEN_RXBURST:
+			*data++ = SDPCM_TEST_BURST;
+			*data++ = (uint8)bus->pktgen_count; /* Just for backward compatability */
+			break;
+
+		default:
+			DHD_ERROR(("Unrecognized pktgen mode %d\n", bus->pktgen_mode));
+			PKTFREE(osh, pkt, TRUE);
+			bus->pktgen_count = 0;
+			return;
+		}
+
+		/* Write test header length field */
+		*data++ = (bus->pktgen_len >> 0);
+		*data++ = (bus->pktgen_len >> 8);
+
+		/* Write frame count in a 4 byte field adjucent to SDPCM test header for
+		 * burst mode
+		 */
+		if (bus->pktgen_mode == DHD_PKTGEN_RXBURST) {
+			*data++ = (uint8)(bus->pktgen_count >> 0);
+			*data++ = (uint8)(bus->pktgen_count >> 8);
+			*data++ = (uint8)(bus->pktgen_count >> 16);
+			*data++ = (uint8)(bus->pktgen_count >> 24);
+		} else {
+
+			/* Then fill in the remainder -- N/A for burst */
+			for (fillbyte = 0; fillbyte < len; fillbyte++)
+				*data++ = SDPCM_TEST_FILL(fillbyte, (uint8)bus->pktgen_sent);
+		}
+
+#ifdef DHD_DEBUG
+		if (DHD_BYTES_ON() && DHD_DATA_ON()) {
+			data = (uint8*)PKTDATA(osh, pkt) + SDPCM_HDRLEN;
+			prhex("dhdsdio_pktgen: Tx Data", data, PKTLEN(osh, pkt) - SDPCM_HDRLEN);
+		}
+#endif
+
+		/* Send it */
+		if (dhdsdio_txpkt(bus, SDPCM_TEST_CHANNEL, &pkt, 1, TRUE) != BCME_OK) {
+			bus->pktgen_fail++;
+			if (bus->pktgen_stop && bus->pktgen_stop == bus->pktgen_fail)
+				bus->pktgen_count = 0;
+		}
+		bus->pktgen_sent++;
+
+		/* Bump length if not fixed, wrap at max */
+		if (++bus->pktgen_len > bus->pktgen_maxlen)
+			bus->pktgen_len = (uint16)bus->pktgen_minlen;
+
+		/* Special case for burst mode: just send one request! */
+		if (bus->pktgen_mode == DHD_PKTGEN_RXBURST)
+			break;
+	}
+}
+
+static void
+dhdsdio_sdtest_set(dhd_bus_t *bus, uint count)
+{
+	void *pkt;
+	uint8 *data;
+	osl_t *osh = bus->dhd->osh;
+
+	/* Allocate the packet */
+	if (!(pkt = PKTGET(osh, SDPCM_HDRLEN + SDPCM_TEST_HDRLEN +
+		SDPCM_TEST_PKT_CNT_FLD_LEN + DHD_SDALIGN, TRUE))) {
+		DHD_ERROR(("%s: PKTGET failed!\n", __FUNCTION__));
+		return;
+	}
+	PKTALIGN(osh, pkt, (SDPCM_HDRLEN + SDPCM_TEST_HDRLEN +
+		SDPCM_TEST_PKT_CNT_FLD_LEN), DHD_SDALIGN);
+	data = (uint8*)PKTDATA(osh, pkt) + SDPCM_HDRLEN;
+
+	/* Fill in the test header */
+	*data++ = SDPCM_TEST_SEND;
+	*data++ = (count > 0)?TRUE:FALSE;
+	*data++ = (bus->pktgen_maxlen >> 0);
+	*data++ = (bus->pktgen_maxlen >> 8);
+	*data++ = (uint8)(count >> 0);
+	*data++ = (uint8)(count >> 8);
+	*data++ = (uint8)(count >> 16);
+	*data++ = (uint8)(count >> 24);
+
+	/* Send it */
+	if (dhdsdio_txpkt(bus, SDPCM_TEST_CHANNEL, &pkt, 1, TRUE) != BCME_OK)
+		bus->pktgen_fail++;
+}
+
+static void
+dhdsdio_testrcv(dhd_bus_t *bus, void *pkt, uint seq)
+{
+	osl_t *osh = bus->dhd->osh;
+	uint8 *data;
+	uint pktlen;
+
+	uint8 cmd;
+	uint8 extra;
+	uint16 len;
+	uint16 offset;
+
+	/* Check for min length */
+	if ((pktlen = PKTLEN(osh, pkt)) < SDPCM_TEST_HDRLEN) {
+		DHD_ERROR(("dhdsdio_restrcv: toss runt frame, pktlen %d\n", pktlen));
+		PKTFREE(osh, pkt, FALSE);
+		return;
+	}
+
+	/* Extract header fields */
+	data = PKTDATA(osh, pkt);
+	cmd = *data++;
+	extra = *data++;
+	len = *data++; len += *data++ << 8;
+	DHD_TRACE(("%s:cmd:%d, xtra:%d,len:%d\n", __FUNCTION__, cmd, extra, len));
+	/* Check length for relevant commands */
+	if (cmd == SDPCM_TEST_DISCARD || cmd == SDPCM_TEST_ECHOREQ || cmd == SDPCM_TEST_ECHORSP) {
+		if (pktlen != len + SDPCM_TEST_HDRLEN) {
+			DHD_ERROR(("dhdsdio_testrcv: frame length mismatch, pktlen %d seq %d"
+			           " cmd %d extra %d len %d\n", pktlen, seq, cmd, extra, len));
+			PKTFREE(osh, pkt, FALSE);
+			return;
+		}
+	}
+
+	/* Process as per command */
+	switch (cmd) {
+	case SDPCM_TEST_ECHOREQ:
+		/* Rx->Tx turnaround ok (even on NDIS w/current implementation) */
+		*(uint8 *)(PKTDATA(osh, pkt)) = SDPCM_TEST_ECHORSP;
+		if (dhdsdio_txpkt(bus, SDPCM_TEST_CHANNEL, &pkt, 1, TRUE) == BCME_OK) {
+			bus->pktgen_sent++;
+		} else {
+			bus->pktgen_fail++;
+			PKTFREE(osh, pkt, FALSE);
+		}
+		bus->pktgen_rcvd++;
+		break;
+
+	case SDPCM_TEST_ECHORSP:
+		if (bus->ext_loop) {
+			PKTFREE(osh, pkt, FALSE);
+			bus->pktgen_rcvd++;
+			break;
+		}
+
+		for (offset = 0; offset < len; offset++, data++) {
+			if (*data != SDPCM_TEST_FILL(offset, extra)) {
+				DHD_ERROR(("dhdsdio_testrcv: echo data mismatch: "
+				           "offset %d (len %d) expect 0x%02x rcvd 0x%02x\n",
+				           offset, len, SDPCM_TEST_FILL(offset, extra), *data));
+				break;
+			}
+		}
+		PKTFREE(osh, pkt, FALSE);
+		bus->pktgen_rcvd++;
+		break;
+
+	case SDPCM_TEST_DISCARD:
+		{
+			int i = 0;
+			uint8 *prn = data;
+			uint8 testval = extra;
+			for (i = 0; i < len; i++) {
+				if (*prn != testval) {
+					DHD_ERROR(("DIErr@Pkt#:%d,Ix:%d, expected:0x%x, got:0x%x\n",
+						i, bus->pktgen_rcvd_rcvsession, testval, *prn));
+					prn++; testval++;
+				}
+			}
+		}
+		PKTFREE(osh, pkt, FALSE);
+		bus->pktgen_rcvd++;
+		break;
+
+	case SDPCM_TEST_BURST:
+	case SDPCM_TEST_SEND:
+	default:
+		DHD_INFO(("dhdsdio_testrcv: unsupported or unknown command, pktlen %d seq %d"
+		          " cmd %d extra %d len %d\n", pktlen, seq, cmd, extra, len));
+		PKTFREE(osh, pkt, FALSE);
+		break;
+	}
+
+	/* For recv mode, stop at limit (and tell dongle to stop sending) */
+	if (bus->pktgen_mode == DHD_PKTGEN_RECV) {
+		if (bus->pktgen_rcv_state != PKTGEN_RCV_IDLE) {
+			bus->pktgen_rcvd_rcvsession++;
+
+			if (bus->pktgen_total &&
+				(bus->pktgen_rcvd_rcvsession >= bus->pktgen_total)) {
+			bus->pktgen_count = 0;
+			DHD_ERROR(("Pktgen:rcv test complete!\n"));
+			bus->pktgen_rcv_state = PKTGEN_RCV_IDLE;
+			dhdsdio_sdtest_set(bus, FALSE);
+				bus->pktgen_rcvd_rcvsession = 0;
+			}
+		}
+	}
+}
+#endif /* SDTEST */
+
+int dhd_bus_oob_intr_register(dhd_pub_t *dhdp)
+{
+	int err = 0;
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+	err = bcmsdh_oob_intr_register(dhdp->bus->sdh, dhdsdio_isr, dhdp->bus);
+#endif
+	return err;
+}
+
+void dhd_bus_oob_intr_unregister(dhd_pub_t *dhdp)
+{
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+	bcmsdh_oob_intr_unregister(dhdp->bus->sdh);
+#endif
+}
+
+void dhd_bus_oob_intr_set(dhd_pub_t *dhdp, bool enable)
+{
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+	bcmsdh_oob_intr_set(dhdp->bus->sdh, enable);
+#endif
+}
+
+int dhd_bus_get_oob_irq_num(dhd_pub_t *dhdp)
+{
+	int irq_num = 0;
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+	irq_num = bcmsdh_get_oob_intr_num(dhdp->bus->sdh);
+#endif /* OOB_INTR_ONLY || BCMSPI_ANDROID */
+	return irq_num;
+}
+
+#ifdef LINUX
+struct device *dhd_bus_to_dev(struct dhd_bus *bus)
+{
+	return (struct device *)bcmsdh_get_dev(bus->sdh);
+}
+#endif /* LINUX */
+
+void dhd_bus_dev_pm_stay_awake(dhd_pub_t *dhdpub)
+{
+#ifdef LINUX
+	bcmsdh_dev_pm_stay_awake(dhdpub->bus->sdh);
+#endif
+}
+
+void dhd_bus_dev_pm_relax(dhd_pub_t *dhdpub)
+{
+#ifdef LINUX
+	bcmsdh_dev_relax(dhdpub->bus->sdh);
+#endif
+}
+
+bool dhd_bus_dev_pm_enabled(dhd_pub_t *dhdpub)
+{
+	bool enabled = FALSE;
+
+#ifdef LINUX
+	enabled = bcmsdh_dev_pm_enabled(dhdpub->bus->sdh);
+#endif
+	return enabled;
+}
+
+extern bool
+dhd_bus_watchdog(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus;
+	unsigned long flags;
+
+	DHD_TIMER(("%s: Enter\n", __FUNCTION__));
+
+	bus = dhdp->bus;
+
+	if (bus->dhd->dongle_reset)
+		return FALSE;
+
+	if (bus->dhd->hang_was_sent) {
+		dhd_os_wd_timer(bus->dhd, 0);
+		return FALSE;
+	}
+
+	/* Ignore the timer if simulating bus down */
+	if (!SLPAUTO_ENAB(bus) && bus->sleeping)
+		return FALSE;
+
+	DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+	if (DHD_BUS_CHECK_DOWN_OR_DOWN_IN_PROGRESS(dhdp) ||
+			DHD_BUS_CHECK_SUSPEND_OR_SUSPEND_IN_PROGRESS(dhdp)) {
+		DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+		return FALSE;
+	}
+	DHD_BUS_BUSY_SET_IN_WD(dhdp);
+	DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+
+	dhd_os_sdlock(bus->dhd);
+
+	/* Poll period: check device if appropriate. */
+	// terence 20160615: remove !SLPAUTO_ENAB(bus) to fix not able to polling if sr supported
+	if (1 && (bus->poll && (++bus->polltick >= bus->pollrate))) {
+		uint32 intstatus = 0;
+
+		/* Reset poll tick */
+		bus->polltick = 0;
+
+		/* Check device if no interrupts */
+		if (!bus->intr || (bus->intrcount == bus->lastintrs)) {
+#ifdef DEBUG_LOST_INTERRUPTS
+			uint retries = 0;
+			bool hostpending;
+			uint8 devena, devpend;
+
+			/* Make sure backplane clock is on */
+			dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+			hostpending = bcmsdh_intr_pending(bus->sdh);
+			devena = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_INTEN, NULL);
+			devpend = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0, SDIOD_CCCR_INTPEND, NULL);
+
+			R_SDREG(intstatus, &bus->regs->intstatus, retries);
+			intstatus &= bus->hostintmask;
+
+			if (intstatus && !hostpending) {
+				DHD_ERROR(("%s: !hpend: ena 0x%02x pend 0x%02x intstatus 0x%08x\n",
+				           __FUNCTION__, devena, devpend, intstatus));
+			}
+#endif /* DEBUG_LOST_INTERRUPTS */
+
+#ifndef BCMSPI
+			/* XXX Needs to be fixed for polling operation (in CE) */
+			if (!bus->dpc_sched) {
+				uint8 devpend;
+				devpend = bcmsdh_cfg_read(bus->sdh, SDIO_FUNC_0,
+				                          SDIOD_CCCR_INTPEND, NULL);
+				intstatus = devpend & (INTR_STATUS_FUNC1 | INTR_STATUS_FUNC2);
+			}
+#else
+			if (!bus->dpc_sched) {
+				uint32 devpend;
+				devpend = bcmsdh_cfg_read_word(bus->sdh, SDIO_FUNC_0,
+					SPID_STATUS_REG, NULL);
+				intstatus = devpend & STATUS_F2_PKT_AVAILABLE;
+			}
+#endif /* !BCMSPI */
+
+			/* If there is something, make like the ISR and schedule the DPC */
+			if (intstatus) {
+				bus->pollcnt++;
+				bus->ipend = TRUE;
+				if (bus->intr) {
+					bcmsdh_intr_disable(bus->sdh);
+				}
+				bus->dpc_sched = TRUE;
+				dhd_sched_dpc(bus->dhd);
+			}
+		}
+
+		/* Update interrupt tracking */
+		bus->lastintrs = bus->intrcount;
+	}
+
+	if ((!bus->dpc_sched) && pktq_n_pkts_tot(&bus->txq)) {
+		bus->dpc_sched = TRUE;
+		dhd_sched_dpc(bus->dhd);
+	}
+
+#ifdef DHD_DEBUG
+	/* Poll for console output periodically */
+	if (dhdp->busstate == DHD_BUS_DATA && dhdp->dhd_console_ms != 0) {
+		bus->console.count += dhd_watchdog_ms;
+		if (bus->console.count >= dhdp->dhd_console_ms) {
+			bus->console.count -= dhdp->dhd_console_ms;
+			/* Make sure backplane clock is on */
+			if (SLPAUTO_ENAB(bus))
+				dhdsdio_bussleep(bus, FALSE);
+			else
+			dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+			if (dhdsdio_readconsole(bus) < 0)
+				dhdp->dhd_console_ms = 0;	/* On error, stop trying */
+		}
+	}
+#endif /* DHD_DEBUG */
+
+#ifdef SDTEST
+	/* Generate packets if configured */
+	if (bus->pktgen_count && (++bus->pktgen_tick >= bus->pktgen_freq)) {
+		/* Make sure backplane clock is on */
+		if (SLPAUTO_ENAB(bus))
+			dhdsdio_bussleep(bus, FALSE);
+		else
+			dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+		bus->pktgen_tick = 0;
+		dhdsdio_pktgen(bus);
+	}
+#endif
+
+	/* On idle timeout clear activity flag and/or turn off clock */
+#ifdef DHD_USE_IDLECOUNT
+	if (bus->activity)
+		bus->activity = FALSE;
+	else {
+		bus->idlecount++;
+
+		/*
+		 * If the condition to switch off the clock is reached And if
+		 * BT is inactive (in case of BT_OVER_SDIO build) turn off clk.
+		 *
+		 * Consider the following case, DHD is configured with
+		 * 1) idletime == DHD_IDLE_IMMEDIATE
+		 * 2) BT is the last user of the clock
+		 * We cannot disable the clock from __dhdsdio_clk_disable
+		 * since WLAN might be using it. If WLAN is active then
+		 * from the respective function/context after doing the job
+		 * the clk is turned off.
+		 * But if WLAN is actually inactive then the watchdog should
+		 * disable the clock. So the condition check below should be
+		 * bus->idletime != 0 instead of idletime == 0
+		 */
+		if ((bus->idletime != 0) && (bus->idlecount >= bus->idletime) &&
+			NO_OTHER_ACTIVE_BUS_USER(bus)) {
+			DHD_TIMER(("%s: DHD Idle state!!\n", __FUNCTION__));
+			if (!bus->poll && SLPAUTO_ENAB(bus)) {
+				if (dhdsdio_bussleep(bus, TRUE) != BCME_BUSY)
+					dhd_os_wd_timer(bus->dhd, 0);
+			} else
+				dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+
+			bus->idlecount = 0;
+		}
+	}
+#else
+	if ((bus->idletime != 0) && (bus->clkstate == CLK_AVAIL) &&
+		NO_OTHER_ACTIVE_BUS_USER(bus)) {
+		if (++bus->idlecount >= bus->idletime) {
+			bus->idlecount = 0;
+			if (bus->activity) {
+				bus->activity = FALSE;
+#if !defined(OEM_ANDROID) && !defined(NDIS)
+/* XXX
+ * For Android turn off clocks as soon as possible, to improve power
+ * efficiency. For non-android, extend clock-active period for voice
+ * quality reasons (see PR84690/Jira:SWWLAN-7650).
+ */
+			} else {
+#endif /* !defined(OEM_ANDROID) && !defined(NDIS) */
+				if (!bus->poll && SLPAUTO_ENAB(bus)) {
+					if (!bus->readframes)
+						dhdsdio_bussleep(bus, TRUE);
+					else
+						bus->reqbussleep = TRUE;
+				} else {
+					dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+				}
+			}
+		}
+	}
+#endif /* DHD_USE_IDLECOUNT */
+
+	dhd_os_sdunlock(bus->dhd);
+
+	DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+	DHD_BUS_BUSY_CLEAR_IN_WD(dhdp);
+	dhd_os_busbusy_wake(dhdp);
+	DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+
+	return bus->ipend;
+}
+
+extern int
+dhd_bus_console_in(dhd_pub_t *dhdp, uchar *msg, uint msglen)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	uint32 addr, val;
+	int rv;
+	void *pkt;
+
+#ifndef CONSOLE_DPC
+	/* Exclusive bus access */
+	dhd_os_sdlock(bus->dhd);
+#endif
+
+	/* Address could be zero if CONSOLE := 0 in dongle Makefile */
+	if (bus->console_addr == 0) {
+		rv = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	/* Don't allow input if dongle is in reset */
+	if (bus->dhd->dongle_reset) {
+		rv = BCME_NOTREADY;
+		goto exit;
+	}
+
+#ifndef CONSOLE_DPC
+	if (!DATAOK(bus)) {
+		DHD_CTL(("%s: No bus credit bus->tx_max %d, bus->tx_seq %d, pktq_len %d\n",
+			__FUNCTION__, bus->tx_max, bus->tx_seq, pktq_n_pkts_tot(&bus->txq)));
+		rv = BCME_NOTREADY;
+		goto exit;
+	}
+
+	/* Request clock to allow SDIO accesses */
+	BUS_WAKE(bus);
+	/* No pend allowed since txpkt is called later, ht clk has to be on */
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+#endif
+
+	/* Zero cbuf_index */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, cbuf_idx);
+	/* handle difference in definition of hnd_log_t in certain branches */
+	if (dhdp->wlc_ver_major < 14) {
+		addr -= sizeof(uint32);
+	}
+	val = htol32(0);
+	if ((rv = dhdsdio_membytes(bus, TRUE, addr, (uint8 *)&val, sizeof(val))) < 0)
+		goto done;
+
+	/* Write message into cbuf */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, cbuf);
+	/* handle difference in definition of hnd_log_t in certain branches */
+	if (dhdp->wlc_ver_major < 14) {
+		addr -= sizeof(uint32);
+	}
+	if ((rv = dhdsdio_membytes(bus, TRUE, addr, (uint8 *)msg, msglen)) < 0)
+		goto done;
+
+	/* Write length into vcons_in */
+	addr = bus->console_addr + OFFSETOF(hnd_cons_t, vcons_in);
+	val = htol32(msglen);
+	if ((rv = dhdsdio_membytes(bus, TRUE, addr, (uint8 *)&val, sizeof(val))) < 0)
+		goto done;
+
+	/* Bump dongle by sending an empty packet on the event channel.
+	 * sdpcm_sendup (RX) checks for virtual console input.
+	 */
+	if ((pkt = PKTGET(bus->dhd->osh, 4 + SDPCM_RESERVE, TRUE)) != NULL)
+		rv = dhdsdio_txpkt(bus, SDPCM_EVENT_CHANNEL, &pkt, 1, TRUE);
+
+done:
+#ifndef CONSOLE_DPC
+	if ((bus->idletime == DHD_IDLE_IMMEDIATE) && !bus->dpc_sched &&
+		NO_OTHER_ACTIVE_BUS_USER(bus)) {
+		bus->activity = FALSE;
+		dhdsdio_bussleep(bus, TRUE);
+		dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+	}
+#endif
+
+exit:
+#ifdef CONSOLE_DPC
+	memset(bus->cons_cmd, 0, sizeof(bus->cons_cmd));
+#else
+	dhd_os_sdunlock(bus->dhd);
+#endif
+	return rv;
+}
+
+#ifdef CONSOLE_DPC
+extern int
+dhd_bus_txcons(dhd_pub_t *dhdp, uchar *msg, uint msglen)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	int ret = BCME_OK;
+
+	dhd_os_sdlock(bus->dhd);
+
+	/* Address could be zero if CONSOLE := 0 in dongle Makefile */
+	if (bus->console_addr == 0) {
+		ret = BCME_UNSUPPORTED;
+		goto exit;
+	}
+
+	/* Don't allow input if dongle is in reset */
+	if (bus->dhd->dongle_reset) {
+		ret = BCME_NOTREADY;
+		goto exit;
+	}
+
+	if (msglen >= sizeof(bus->cons_cmd)) {
+		DHD_ERROR(("%s: \"%s\"(%d) too long\n", __FUNCTION__, msg, msglen));
+		ret = BCME_BADARG;
+		goto exit;
+	}
+
+	if (!strlen(bus->cons_cmd)) {
+		strncpy(bus->cons_cmd, msg, sizeof(bus->cons_cmd));
+		DHD_CTL(("%s: \"%s\" delay send, tx_max %d, tx_seq %d, pktq_len %d\n",
+			__FUNCTION__, bus->cons_cmd, bus->tx_max, bus->tx_seq, pktq_n_pkts_tot(&bus->txq)));
+		if (!bus->dpc_sched) {
+			bus->dpc_sched = TRUE;
+			dhd_sched_dpc(bus->dhd);
+		}
+	} else {
+		DHD_CTL(("%s: \"%s\" is pending, tx_max %d, tx_seq %d, pktq_len %d\n",
+			__FUNCTION__, bus->cons_cmd, bus->tx_max, bus->tx_seq, pktq_n_pkts_tot(&bus->txq)));
+		ret = BCME_NOTREADY;
+	}
+
+exit:
+	dhd_os_sdunlock(bus->dhd);
+
+	return ret;
+}
+#endif
+
+#if defined(DHD_DEBUG) && !defined(BCMSDIOLITE)
+static void
+dhd_dump_cis(uint fn, uint8 *cis)
+{
+	uint byte, tag, tdata;
+	DHD_INFO(("Function %d CIS:\n", fn));
+
+	for (tdata = byte = 0; byte < SBSDIO_CIS_SIZE_LIMIT; byte++) {
+		if (!tdata--) {
+			tag = cis[byte];
+			if (tag == 0xff)
+				break;
+			else if (!tag)
+				tdata = 0;
+			else if ((byte + 1) < SBSDIO_CIS_SIZE_LIMIT)
+				tdata = cis[byte + 1] + 1;
+		}
+	}
+	prhex(NULL, (const u8 *) cis, byte+1);
+}
+#endif /* DHD_DEBUG */
+
+static bool
+dhdsdio_chipmatch(uint16 chipid)
+{
+	if (chipid == BCM4330_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43362_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43340_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43341_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4334_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4324_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4335_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4339_CHIP_ID)
+		return TRUE;
+	if (BCM4345_CHIP(chipid))
+		return TRUE;
+	if (chipid == BCM4350_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4354_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4356_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4358_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43569_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM4371_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43430_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43018_CHIP_ID)
+		return TRUE;
+	if (BCM4349_CHIP(chipid))
+		return TRUE;
+	if (chipid == BCM4364_CHIP_ID)
+			return TRUE;
+
+	if (chipid == BCM4381_CHIP_ID) {
+		return TRUE;
+	}
+
+	if (chipid == BCM4382_CHIP_ID) {
+		return TRUE;
+	}
+
+	if (chipid == BCM43012_CHIP_ID)
+		return TRUE;
+
+	if (chipid == BCM43014_CHIP_ID)
+		return TRUE;
+
+	if (chipid == BCM43013_CHIP_ID)
+		return TRUE;
+
+	if (chipid == BCM4369_CHIP_ID)
+		return TRUE;
+
+	if (BCM4378_CHIP(chipid)) {
+		return TRUE;
+	}
+
+	if (chipid == BCM4362_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43751_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43752_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43756_CHIP_ID)
+		return TRUE;
+	if (chipid == BCM43711_CHIP_ID)
+		return TRUE;
+	return FALSE;
+}
+
+static void *
+dhdsdio_probe(uint16 venid, uint16 devid, uint16 bus_no, uint16 slot,
+	uint16 func, uint bustype, void *regsva, osl_t * osh, void *sdh)
+{
+	int ret;
+	dhd_bus_t *bus;
+
+	DHD_MUTEX_LOCK();
+
+	/* Init global variables at run-time, not as part of the declaration.
+	 * This is required to support init/de-init of the driver. Initialization
+	 * of globals as part of the declaration results in non-deterministic
+	 * behavior since the value of the globals may be different on the
+	 * first time that the driver is initialized vs subsequent initializations.
+	 */
+	dhd_txbound = DHD_TXBOUND;
+	dhd_rxbound = DHD_RXBOUND;
+#ifdef BCMSPI
+	dhd_alignctl = FALSE;
+#else
+	dhd_alignctl = TRUE;
+#endif /* BCMSPI */
+	sd1idle = TRUE;
+	dhd_readahead = TRUE;
+	retrydata = FALSE;
+
+#ifdef DISABLE_FLOW_CONTROL
+	dhd_doflow = FALSE;
+#endif /* DISABLE_FLOW_CONTROL */
+	dhd_dongle_ramsize = 0;
+	dhd_txminmax = DHD_TXMINMAX;
+
+#ifdef BCMSPI
+	forcealign = FALSE;
+#else
+	forcealign = TRUE;
+#endif /* !BCMSPI */
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+	DHD_INFO(("%s: venid 0x%04x devid 0x%04x\n", __FUNCTION__, venid, devid));
+
+	/* We make assumptions about address window mappings */
+	ASSERT((uintptr)regsva == si_enum_base(devid));
+
+	/* BCMSDH passes venid and devid based on CIS parsing -- but low-power start
+	 * means early parse could fail, so here we should get either an ID
+	 * we recognize OR (-1) indicating we must request power first.
+	 */
+	/* Check the Vendor ID */
+	switch (venid) {
+		case 0x0000:
+		case VENDOR_BROADCOM:
+			break;
+		default:
+			DHD_ERROR(("%s: unknown vendor: 0x%04x\n",
+			           __FUNCTION__, venid));
+			goto forcereturn;
+	}
+
+	/* Check the Device ID and make sure it's one that we support */
+	switch (devid) {
+		case 0:
+			DHD_INFO(("%s: allow device id 0, will check chip internals\n",
+			          __FUNCTION__));
+			break;
+
+		default:
+			DHD_ERROR(("%s: skipping 0x%04x/0x%04x, not a dongle\n",
+			           __FUNCTION__, venid, devid));
+			goto forcereturn;
+	}
+
+	if (osh == NULL) {
+		DHD_ERROR(("%s: osh is NULL!\n", __FUNCTION__));
+		goto forcereturn;
+	}
+
+	/* Allocate private bus interface state */
+	if (!(bus = MALLOC(osh, sizeof(dhd_bus_t)))) {
+		DHD_ERROR(("%s: MALLOC of dhd_bus_t failed\n", __FUNCTION__));
+		goto fail;
+	}
+	bzero(bus, sizeof(dhd_bus_t));
+	bus->sdh = sdh;
+	bus->cl_devid = (uint16)devid;
+	bus->bus = DHD_BUS;
+	bus->bus_num = bus_no;
+	bus->slot_num = slot;
+	bus->tx_seq = SDPCM_SEQUENCE_WRAP - 1;
+	bus->usebufpool = FALSE; /* Use bufpool if allocated, else use locally malloced rxbuf */
+#ifdef BT_OVER_SDIO
+	bus->bt_use_count = 0;
+#endif
+
+#if defined(LINUX) && defined(SUPPORT_P2P_GO_PS)
+	init_waitqueue_head(&bus->bus_sleep);
+#endif /* LINUX && SUPPORT_P2P_GO_PS */
+	init_waitqueue_head(&bus->ctrl_tx_wait);
+
+	/* attempt to attach to the dongle */
+	if (!(dhdsdio_probe_attach(bus, osh, sdh, regsva, devid))) {
+		DHD_ERROR(("%s: dhdsdio_probe_attach failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	/* Attach to the dhd/OS/network interface */
+	if (!(bus->dhd = dhd_attach(osh, bus, SDPCM_RESERVE))) {
+		DHD_ERROR(("%s: dhd_attach failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+#if defined(GET_OTP_MAC_ENABLE) || defined(GET_OTP_MODULE_NAME)
+	dhd_conf_get_otp(bus->dhd, sdh, bus->sih);
+#endif
+
+	/* Allocate buffers */
+	if (!(dhdsdio_probe_malloc(bus, osh, sdh))) {
+		DHD_ERROR(("%s: dhdsdio_probe_malloc failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	if (!(dhdsdio_probe_init(bus, osh, sdh))) {
+		DHD_ERROR(("%s: dhdsdio_probe_init failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+	if (bus->intr) {
+		/* Register interrupt callback, but mask it (not operational yet). */
+		DHD_INTR(("%s: disable SDIO interrupts (not interested yet)\n", __FUNCTION__));
+		bcmsdh_intr_disable(sdh); /* XXX New API: bcmsdh_intr_mask()? */
+		if ((ret = bcmsdh_intr_reg(sdh, dhdsdio_isr, bus)) != 0) {
+			DHD_ERROR(("%s: FAILED: bcmsdh_intr_reg returned %d\n",
+			           __FUNCTION__, ret));
+			goto fail;
+		}
+		DHD_INTR(("%s: registered SDIO interrupt function ok\n", __FUNCTION__));
+	} else {
+		DHD_INFO(("%s: SDIO interrupt function is NOT registered due to polling mode\n",
+		           __FUNCTION__));
+	}
+
+	DHD_INFO(("%s: completed!!\n", __FUNCTION__));
+
+	/* if firmware path present try to download and bring up bus */
+	bus->dhd->hang_report  = TRUE;
+
+#if defined(BCMDHD_MODULAR) && defined(INSMOD_FW_LOAD)
+	if (1)
+#else
+#if defined(LINUX) || defined(linux)
+	if (!dhd_conf_legacy_otp_chip(bus->dhd))
+#endif /* LINUX || linux */
+#endif
+	{
+		if ((ret = dhd_bus_start(bus->dhd)) != 0) {
+			DHD_ERROR(("%s: dhd_bus_start failed\n", __FUNCTION__));
+#if !defined(OEM_ANDROID)
+			if (ret == BCME_NOTUP)
+#endif /* !OEM_ANDROID */
+				goto fail;
+		}
+#if defined(LINUX) || defined(linux)
+	}
+	else {
+		/* Set random MAC address during boot time */
+		get_random_bytes(&bus->dhd->mac.octet[3], 3);
+		/* Adding BRCM OUI */
+		bus->dhd->mac.octet[0] = 0;
+		bus->dhd->mac.octet[1] = 0x90;
+		bus->dhd->mac.octet[2] = 0x4C;
+	}
+#endif /* LINUX || linux */
+#if defined(BT_OVER_SDIO)
+	/* At this point Regulators are turned on and iconditionaly sdio bus is started
+	 * based upon dhd_download_fw_on_driverload check, so
+	 * increase the bus user count, this count will only be disabled inside
+	 * dhd_register_if() function if flag dhd_download_fw_on_driverload is set to false,
+	 * i.e FW download during insmod is not needed, otherwise it will not be decremented
+	 * so that WALN will always hold the bus untill rmmod is done.
+	 */
+	dhdsdio_bus_usr_cnt_inc(bus->dhd);
+#endif /* BT_OVER_SDIO */
+
+	/* Ok, have the per-port tell the stack we're open for business */
+	DHD_TRACE(("%s(): Calling dhd_attach_net() \n", __FUNCTION__));
+	if (dhd_attach_net(bus->dhd, TRUE)) {
+		DHD_ERROR(("%s(): ERROR.. dhd_attach_net() failed\n", __FUNCTION__));
+		goto fail;
+	}
+
+#ifdef BCMHOST_XTAL_PU_TIME_MOD
+	bcmsdh_reg_write(bus->sdh, 0x18000620, 2, 11);
+	bcmsdh_reg_write(bus->sdh, 0x18000628, 4, 0x00F80001);
+#endif /* BCMHOST_XTAL_PU_TIME_MOD */
+
+#if defined(MULTIPLE_SUPPLICANT)
+	wl_android_post_init(); // terence 20120530: fix critical section in dhd_open and dhdsdio_probe
+#endif /* MULTIPLE_SUPPLICANT */
+	DHD_MUTEX_UNLOCK();
+
+	return bus;
+
+fail:
+	dhdsdio_release(bus, osh);
+
+forcereturn:
+	DHD_MUTEX_UNLOCK();
+
+	return NULL;
+}
+
+static bool
+dhdsdio_probe_attach(struct dhd_bus *bus, osl_t *osh, void *sdh, void *regsva,
+                     uint16 devid)
+{
+#ifndef BCMSPI
+	int err = 0;
+	uint8 clkctl = 0;
+#endif /* !BCMSPI */
+
+	bus->alp_only = TRUE;
+	bus->sih = NULL;
+
+	/* Return the window to backplane enumeration space for core access */
+	if (dhdsdio_set_siaddr_window(bus, si_enum_base(devid))) {
+		DHD_INFO(("%s: FAILED to return to SI_ENUM_BASE\n", __FUNCTION__));
+	}
+
+#if defined(DHD_DEBUG)
+	DHD_INFO(("F1 signature read @0x18000000=0x%4x\n",
+		bcmsdh_reg_read(bus->sdh, si_enum_base(devid), 4)));
+#endif
+
+#ifndef BCMSPI	/* wake-wlan in gSPI will bring up the htavail/alpavail clocks. */
+
+	/* Force PLL off until si_attach() programs PLL control regs */
+
+	/* XXX Ideally should not access F1 power control regs before
+	 * reading CIS and confirming device.  But strapping option for
+	 * low-power start requires turning on ALP before reading CIS,
+	 * and at some point bcmsdh should read the CIS for the ID and
+	 * not even tell us if it's some other device.  At this point
+	 * (see above) we should know it's us (powered on) or can't read
+	 * CIS so we need to power on and try.
+	 */
+
+	/* WAR for PR 39902: must force HT off until PLL programmed. */
+	/* WAR for PR43618, PR44891: don't do ALPReq until ALPAvail set */
+
+	/* XXX Replace write/read sequence with single bcmsdh_cfg_raw() call */
+	bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, DHD_INIT_CLKCTL1, &err);
+	if (!err)
+		clkctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+
+	if (err || ((clkctl & ~SBSDIO_AVBITS) != DHD_INIT_CLKCTL1)) {
+		DHD_ERROR(("dhdsdio_probe: ChipClkCSR access: err %d wrote 0x%02x read 0x%02x\n",
+		           err, DHD_INIT_CLKCTL1, clkctl));
+		goto fail;
+	}
+
+#endif /* !BCMSPI */
+
+#ifdef DHD_DEBUG
+	if (DHD_INFO_ON()) {
+		uint fn, numfn;
+		uint8 *cis = NULL;
+		int local_err = 0;
+
+#ifndef BCMSPI
+		numfn = bcmsdh_query_iofnum(sdh);
+		ASSERT(numfn <= SDIOD_MAX_IOFUNCS);
+
+		/* Make sure ALP is available before trying to read CIS */
+		SPINWAIT(((clkctl = bcmsdh_cfg_read(sdh, SDIO_FUNC_1,
+		                                    SBSDIO_FUNC1_CHIPCLKCSR, NULL)),
+		          !SBSDIO_ALPAV(clkctl)), PMU_MAX_TRANSITION_DLY);
+
+		/* Now request ALP be put on the bus */
+		bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR,
+		                 DHD_INIT_CLKCTL2, &local_err);
+		/* XXX Account for possible delay between ALP available and on active */
+		OSL_DELAY(65);
+#else
+		numfn = 0; /* internally func is hardcoded to 1 as gSPI has cis on F1 only */
+#endif /* !BCMSPI */
+#ifndef BCMSDIOLITE
+		if (!(cis = MALLOC(osh, SBSDIO_CIS_SIZE_LIMIT))) {
+			DHD_INFO(("dhdsdio_probe: cis malloc failed\n"));
+			goto fail;
+		}
+
+		for (fn = 0; fn <= numfn; fn++) {
+			bzero(cis, SBSDIO_CIS_SIZE_LIMIT);
+			if ((err = bcmsdh_cis_read(sdh, fn, cis,
+				SBSDIO_CIS_SIZE_LIMIT))) {
+				DHD_INFO(("dhdsdio_probe: fn %d cis read err %d\n",
+					fn, err));
+				break;
+			}
+			dhd_dump_cis(fn, cis);
+		}
+		MFREE(osh, cis, SBSDIO_CIS_SIZE_LIMIT);
+#else
+	BCM_REFERENCE(cis);
+	BCM_REFERENCE(fn);
+#endif /* DHD_DEBUG */
+		if (local_err) {
+			DHD_ERROR(("dhdsdio_probe: failure reading or parsing CIS\n"));
+			goto fail;
+		}
+	}
+#endif /* DHD_DEBUG */
+
+	/* si_attach() will provide an SI handle and scan the backplane */
+	if (!(bus->sih = si_attach((uint)devid, osh, regsva, DHD_BUS, sdh,
+	                           &bus->vars, &bus->varsz))) {
+		DHD_ERROR(("%s: si_attach failed!\n", __FUNCTION__));
+		goto fail;
+	}
+
+#ifdef DHD_DEBUG
+	DHD_INFO(("F1 signature OK, socitype:0x%x chip:0x%4x rev:0x%x pkg:0x%x\n",
+		bus->sih->socitype, bus->sih->chip, bus->sih->chiprev, bus->sih->chippkg));
+#endif /* DHD_DEBUG */
+
+	/* XXX Let the layers below dhd know the chipid and chiprev for
+	 * controlling sw WARs for hw PRs
+	 */
+	bcmsdh_chipinfo(sdh, bus->sih->chip, bus->sih->chiprev);
+
+	if (!dhdsdio_chipmatch((uint16)bus->sih->chip)) {
+		DHD_ERROR(("%s: unsupported chip: 0x%04x\n",
+		           __FUNCTION__, bus->sih->chip));
+		goto fail;
+	}
+
+	if (bus->sih->buscorerev >= 12) {
+		dhdsdio_clk_kso_init(bus);
+	} else {
+		bus->kso = TRUE;
+	}
+
+	/* to wake up completely incase sleep is triggered before bus start */
+	dhdsdio_set_wakeupctrl(bus);
+
+	si_sdiod_drive_strength_init(bus->sih, osh, dhd_sdiod_drive_strength);
+
+	/* Get info on the ARM and SOCRAM cores... */
+	/* XXX Should really be qualified by device id */
+	if (!DHD_NOPMU(bus)) {
+		if ((si_setcore(bus->sih, ARM7S_CORE_ID, 0)) ||
+		    (si_setcore(bus->sih, ARMCM3_CORE_ID, 0)) ||
+		    (si_setcore(bus->sih, ARMCR4_CORE_ID, 0))) {
+			bus->armrev = si_corerev(bus->sih);
+		} else {
+			DHD_ERROR(("%s: failed to find ARM core!\n", __FUNCTION__));
+			goto fail;
+		}
+
+		if (!si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			if (!(bus->orig_ramsize = si_socram_size(bus->sih))) {
+				DHD_ERROR(("%s: failed to find SOCRAM memory!\n", __FUNCTION__));
+				goto fail;
+			}
+		} else {
+			/* cr4 has a different way to find the RAM size from TCM's */
+			if (!(bus->orig_ramsize = si_tcm_size(bus->sih))) {
+				DHD_ERROR(("%s: failed to find CR4-TCM memory!\n", __FUNCTION__));
+				goto fail;
+			}
+			/* also populate base address */
+			switch ((uint16)bus->sih->chip) {
+			case BCM4335_CHIP_ID:
+			case BCM4339_CHIP_ID:
+				bus->dongle_ram_base = CR4_4335_RAM_BASE;
+				break;
+			case BCM4350_CHIP_ID:
+			case BCM4354_CHIP_ID:
+			case BCM4356_CHIP_ID:
+			case BCM4358_CHIP_ID:
+			case BCM43569_CHIP_ID:
+			case BCM4371_CHIP_ID:
+				bus->dongle_ram_base = CR4_4350_RAM_BASE;
+				break;
+			case BCM4360_CHIP_ID:
+				bus->dongle_ram_base = CR4_4360_RAM_BASE;
+				break;
+			CASE_BCM4345_CHIP:
+				bus->dongle_ram_base = (bus->sih->chiprev < 6)  /* from 4345C0 */
+					? CR4_4345_LT_C0_RAM_BASE : CR4_4345_GE_C0_RAM_BASE;
+				break;
+			case BCM4349_CHIP_GRPID:
+				/* RAM based changed from 4349c0(revid=9) onwards */
+				bus->dongle_ram_base = ((bus->sih->chiprev < 9) ?
+					CR4_4349_RAM_BASE: CR4_4349_RAM_BASE_FROM_REV_9);
+				break;
+			case BCM4364_CHIP_ID:
+				bus->dongle_ram_base = CR4_4364_RAM_BASE;
+				break;
+			case BCM4381_CHIP_ID:
+				bus->dongle_ram_base = CR4_4381_RAM_BASE;
+				break;
+			case BCM4382_CHIP_ID:
+				bus->dongle_ram_base = CR4_4382_RAM_BASE;
+				break;
+			case BCM4362_CHIP_ID:
+				bus->dongle_ram_base = CR4_4362_RAM_BASE;
+				break;
+			case BCM43751_CHIP_ID:
+				bus->dongle_ram_base = CR4_43751_RAM_BASE;
+				break;
+			case BCM43752_CHIP_ID:
+				bus->dongle_ram_base = CR4_43752_RAM_BASE;
+				break;
+			case BCM43756_CHIP_ID:
+				bus->dongle_ram_base = CR4_43756_RAM_BASE;
+				break;
+			case BCM43711_CHIP_ID:
+				bus->dongle_ram_base = CR4_43711_RAM_BASE;
+				break;
+			case BCM4369_CHIP_ID:
+				bus->dongle_ram_base = CR4_4369_RAM_BASE;
+				break;
+			case BCM4378_CHIP_GRPID:
+				bus->dongle_ram_base = CR4_4378_RAM_BASE;
+				break;
+			default:
+				bus->dongle_ram_base = 0;
+				DHD_INFO(("%s: WARNING: Using default ram base at 0x%x\n",
+				           __FUNCTION__, bus->dongle_ram_base));
+			}
+		}
+		bus->ramsize = bus->orig_ramsize;
+		if (dhd_dongle_ramsize)
+			dhd_dongle_setramsize(bus, dhd_dongle_ramsize);
+
+		DHD_INFO(("DHD: dongle ram size is set to %d(orig %d) at 0x%x\n",
+		           bus->ramsize, bus->orig_ramsize, bus->dongle_ram_base));
+
+		bus->srmemsize = si_socram_srmem_size(bus->sih);
+	}
+
+	/* ...but normally deal with the SDPCMDEV core */
+#ifdef BCMSDIOLITE
+	if (!(bus->regs = si_setcore(bus->sih, CC_CORE_ID, 0))) {
+		DHD_ERROR(("%s: failed to find Chip Common core!\n", __FUNCTION__));
+		goto fail;
+	}
+#else
+	if (!(bus->regs = si_setcore(bus->sih, PCMCIA_CORE_ID, 0)) &&
+	    !(bus->regs = si_setcore(bus->sih, SDIOD_CORE_ID, 0))) {
+		DHD_ERROR(("%s: failed to find SDIODEV core!\n", __FUNCTION__));
+		goto fail;
+	}
+#endif
+	bus->sdpcmrev = si_corerev(bus->sih);
+
+	/* Set core control so an SDIO reset does a backplane reset */
+	OR_REG(osh, &bus->regs->corecontrol, CC_BPRESEN);
+#ifndef BCMSPI
+	bus->rxint_mode = SDIO_DEVICE_HMB_RXINT;
+
+	if ((bus->sih->buscoretype == SDIOD_CORE_ID) && (bus->sdpcmrev >= 4) &&
+		(bus->rxint_mode  == SDIO_DEVICE_RXDATAINT_MODE_1))
+	{
+		uint32 val;
+
+		val = R_REG(osh, &bus->regs->corecontrol);
+		val &= ~CC_XMTDATAAVAIL_MODE;
+		val |= CC_XMTDATAAVAIL_CTRL;
+		W_REG(osh, &bus->regs->corecontrol, val);
+	}
+#endif /* BCMSPI */
+
+	/* XXX Tx needs priority queue, where to determine levels? */
+	/* XXX Should it try to do WLC mapping, or just pass through? */
+	pktq_init(&bus->txq, (PRIOMASK + 1), QLEN);
+
+	/* Locate an appropriately-aligned portion of hdrbuf */
+#ifndef DYNAMIC_MAX_HDR_READ
+	bus->rxhdr = (uint8 *)ROUNDUP((uintptr)&bus->hdrbuf[0], DHD_SDALIGN);
+#endif
+
+	/* Set the poll and/or interrupt flags */
+	bus->intr = (bool)dhd_intr;
+	if ((bus->poll = (bool)dhd_poll))
+		bus->pollrate = 1;
+
+	/* Setting default Glom size */
+	bus->txglomsize = SDPCM_DEFGLOM_SIZE;
+
+	return TRUE;
+
+fail:
+	if (bus->sih != NULL) {
+		si_detach(bus->sih);
+		bus->sih = NULL;
+	}
+	return FALSE;
+}
+
+static bool
+dhdsdio_probe_malloc(dhd_bus_t *bus, osl_t *osh, void *sdh)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd->maxctl) {
+		bus->rxblen = ROUNDUP((bus->dhd->maxctl+SDPCM_HDRLEN), ALIGNMENT) + DHD_SDALIGN;
+		if (!(bus->rxbuf = DHD_OS_PREALLOC(bus->dhd, DHD_PREALLOC_RXBUF, bus->rxblen))) {
+			DHD_ERROR(("%s: MALLOC of %d-byte rxbuf failed\n",
+			           __FUNCTION__, bus->rxblen));
+			goto fail;
+		}
+	}
+	/* Allocate buffer to receive glomed packet */
+	if (!(bus->databuf = DHD_OS_PREALLOC(bus->dhd, DHD_PREALLOC_DATABUF, MAX_DATA_BUF))) {
+		DHD_ERROR(("%s: MALLOC of %d-byte databuf failed\n",
+			__FUNCTION__, MAX_DATA_BUF));
+		/* release rxbuf which was already located as above */
+		if (!bus->rxblen)
+			DHD_OS_PREFREE(bus->dhd, bus->rxbuf, bus->rxblen);
+		goto fail;
+	}
+	/* Allocate buffer to membuf */
+	bus->membuf = MALLOC(osh, MAX_MEM_BUF);
+	if (bus->membuf == NULL) {
+		DHD_ERROR(("%s: MALLOC of %d-byte membuf failed\n",
+			__FUNCTION__, MAX_MEM_BUF));
+		if (bus->databuf) {
+#ifndef CONFIG_DHD_USE_STATIC_BUF
+			MFREE(osh, bus->databuf, MAX_DATA_BUF);
+#endif
+			bus->databuf = NULL;
+		}
+		/* release rxbuf which was already located as above */
+		if (!bus->rxblen)
+			DHD_OS_PREFREE(bus->dhd, bus->rxbuf, bus->rxblen);
+		goto fail;
+	}
+	memset(bus->membuf, 0, MAX_MEM_BUF);
+
+	/* Align the buffer */
+	if ((uintptr)bus->databuf % DHD_SDALIGN)
+		bus->dataptr = bus->databuf + (DHD_SDALIGN - ((uintptr)bus->databuf % DHD_SDALIGN));
+	else
+		bus->dataptr = bus->databuf;
+
+	return TRUE;
+
+fail:
+	return FALSE;
+}
+
+static bool
+dhdsdio_probe_init(dhd_bus_t *bus, osl_t *osh, void *sdh)
+{
+	int32 fnum;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	bus->_srenab = FALSE;
+
+#ifdef SDTEST
+	dhdsdio_pktgen_init(bus);
+#endif /* SDTEST */
+
+#ifndef BCMSPI
+	/* Disable F2 to clear any intermediate frame state on the dongle */
+	/* XXX New API: change to bcmsdh_fn_set(sdh, SDIO_FUNC_2, FALSE); */
+	/* XXX Might write SRES instead, or reset ARM (download prep)? */
+	bcmsdh_cfg_write(sdh, SDIO_FUNC_0, SDIOD_CCCR_IOEN, SDIO_FUNC_ENABLE_1, NULL);
+#endif /* !BCMSPI */
+
+	DHD_INFO(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+	bus->dhd->busstate = DHD_BUS_DOWN;
+	bus->sleeping = FALSE;
+	bus->rxflow = FALSE;
+	bus->prev_rxlim_hit = 0;
+
+#ifndef BCMSPI
+	/* Done with backplane-dependent accesses, can drop clock... */
+	bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL);
+#endif /* !BCMSPI */
+
+	/* ...and initialize clock/power states */
+	bus->clkstate = CLK_SDONLY;
+	bus->idletime = (int32)dhd_idletime;
+	bus->idleclock = DHD_IDLE_ACTIVE;
+
+	/* Query the SD clock speed */
+	if (bcmsdh_iovar_op(sdh, "sd_divisor", NULL, 0,
+	                    &bus->sd_divisor, sizeof(int32), FALSE) != BCME_OK) {
+		DHD_ERROR(("%s: fail on %s get\n", __FUNCTION__, "sd_divisor"));
+		bus->sd_divisor = -1;
+	} else {
+		DHD_INFO(("%s: Initial value for %s is %d\n",
+		          __FUNCTION__, "sd_divisor", bus->sd_divisor));
+	}
+
+	/* Query the SD bus mode */
+	if (bcmsdh_iovar_op(sdh, "sd_mode", NULL, 0,
+	                    &bus->sd_mode, sizeof(int32), FALSE) != BCME_OK) {
+		DHD_ERROR(("%s: fail on %s get\n", __FUNCTION__, "sd_mode"));
+		bus->sd_mode = -1;
+	} else {
+		DHD_INFO(("%s: Initial value for %s is %d\n",
+		          __FUNCTION__, "sd_mode", bus->sd_mode));
+	}
+
+	/* Query the F2 block size, set roundup accordingly */
+	fnum = 2;
+	if (bcmsdh_iovar_op(sdh, "sd_blocksize", &fnum, sizeof(int32),
+	                    &bus->blocksize, sizeof(int32), FALSE) != BCME_OK) {
+		bus->blocksize = 0;
+		DHD_ERROR(("%s: fail on %s get\n", __FUNCTION__, "sd_blocksize"));
+	} else {
+		DHD_INFO(("%s: Initial value for %s is %d\n",
+		          __FUNCTION__, "sd_blocksize", bus->blocksize));
+
+		dhdsdio_tune_fifoparam(bus);
+	}
+	bus->roundup = MIN(max_roundup, bus->blocksize);
+
+#ifdef DHDENABLE_TAILPAD
+	if (bus->pad_pkt)
+		PKTFREE(osh, bus->pad_pkt, FALSE);
+	bus->pad_pkt = PKTGET(osh, SDIO_MAX_BLOCK_SIZE, FALSE);
+	if (bus->pad_pkt == NULL)
+		DHD_ERROR(("failed to allocate padding packet\n"));
+	else {
+		int alignment_offset = 0;
+		uintptr pktprt = (uintptr)PKTDATA(osh, bus->pad_pkt);
+		if (!(pktprt&1) && (pktprt = (pktprt % DHD_SDALIGN)))
+			PKTPUSH(osh, bus->pad_pkt, alignment_offset);
+		PKTSETNEXT(osh, bus->pad_pkt, NULL);
+	}
+#endif /* DHDENABLE_TAILPAD */
+
+	/* Query if bus module supports packet chaining, default to use if supported */
+	if (bcmsdh_iovar_op(sdh, "sd_rxchain", NULL, 0,
+	                    &bus->sd_rxchain, sizeof(int32), FALSE) != BCME_OK) {
+		bus->sd_rxchain = FALSE;
+	} else {
+		DHD_INFO(("%s: bus module (through bcmsdh API) %s chaining\n",
+		          __FUNCTION__, (bus->sd_rxchain ? "supports" : "does not support")));
+	}
+	bus->use_rxchain = (bool)bus->sd_rxchain;
+	bus->txinrx_thres = CUSTOM_TXINRX_THRES;
+	/* TX first in dhdsdio_readframes() */
+	bus->dotxinrx = TRUE;
+
+#ifdef PKT_STATICS
+	dhd_bus_clear_txpktstatics(bus->dhd);
+#endif
+
+	return TRUE;
+}
+
+int
+dhd_bus_download_firmware(struct dhd_bus *bus, osl_t *osh,
+                          char *pfw_path, char *pnv_path,
+                          char *pclm_path, char *pconf_path)
+{
+	int ret;
+
+	bus->fw_path = pfw_path;
+	bus->nv_path = pnv_path;
+	bus->dhd->clm_path = pclm_path;
+	bus->dhd->conf_path = pconf_path;
+
+	ret = dhdsdio_download_firmware(bus, osh, bus->sdh);
+
+	return ret;
+}
+
+int
+dhd_set_bus_params(struct dhd_bus *bus)
+{
+	int ret = 0;
+
+	if (bus->dhd->conf->dhd_poll >= 0) {
+		bus->poll = bus->dhd->conf->dhd_poll;
+		if (!bus->pollrate)
+			bus->pollrate = 1;
+		printf("%s: set polling mode %d\n", __FUNCTION__, bus->dhd->conf->dhd_poll);
+	}
+	if (bus->dhd->conf->use_rxchain >= 0) {
+		bus->use_rxchain = (bool)bus->dhd->conf->use_rxchain;
+	}
+	if (bus->dhd->conf->txinrx_thres >= 0) {
+		bus->txinrx_thres = bus->dhd->conf->txinrx_thres;
+	}
+	if (bus->dhd->conf->txglomsize >= 0) {
+		bus->txglomsize = bus->dhd->conf->txglomsize;
+	}
+#ifdef MINIME
+	if (bus->dhd->conf->fw_type == FW_TYPE_MINIME) {
+		bus->ramsize = bus->dhd->conf->ramsize;
+		printf("%s: set ramsize 0x%x\n", __FUNCTION__, bus->ramsize);
+	}
+#endif
+#ifdef DYNAMIC_MAX_HDR_READ
+	if (bus->dhd->conf->max_hdr_read <= 0) {
+		bus->dhd->conf->max_hdr_read = MAX_HDR_READ;
+	}
+	if (bus->hdrbufp) {
+		MFREE(bus->dhd->osh, bus->hdrbufp, bus->dhd->conf->max_hdr_read + DHD_SDALIGN);
+	}
+	bus->hdrbufp = MALLOC(bus->dhd->osh, bus->dhd->conf->max_hdr_read + DHD_SDALIGN);
+	if (bus->hdrbufp == NULL) {
+		DHD_ERROR(("%s: MALLOC of %d-byte hdrbufp failed\n",
+			__FUNCTION__, bus->dhd->conf->max_hdr_read + DHD_SDALIGN));
+		ret = -1;
+		goto exit;
+	}
+	bus->rxhdr = (uint8 *)ROUNDUP((uintptr)bus->hdrbufp, DHD_SDALIGN);
+
+exit:
+#endif
+	return ret;
+}
+
+#ifdef BCM_REQUEST_FW
+extern char clm_path[MOD_PARAM_PATHLEN];
+#endif /* BCM_REQUEST_FW */
+static int
+dhdsdio_download_firmware(struct dhd_bus *bus, osl_t *osh, void *sdh)
+{
+	int ret;
+
+#if defined(SUPPORT_MULTIPLE_REVISION)
+	if (concate_revision(bus, bus->fw_path, bus->nv_path) != 0) {
+		DHD_ERROR(("%s: fail to concatnate revison \n",
+			__FUNCTION__));
+		return BCME_BADARG;
+	}
+#endif /* SUPPORT_MULTIPLE_REVISION */
+
+#if defined(DHD_BLOB_EXISTENCE_CHECK)
+	dhd_set_blob_support(bus->dhd, bus->fw_path);
+#endif /* DHD_BLOB_EXISTENCE_CHECK */
+
+	DHD_TRACE_HW4(("%s: firmware path=%s, nvram path=%s\n",
+		__FUNCTION__, bus->fw_path, bus->nv_path));
+	DHD_OS_WAKE_LOCK(bus->dhd);
+
+	dhd_conf_set_path_params(bus->dhd, bus->fw_path, bus->nv_path);
+	ret = dhd_set_bus_params(bus);
+	if (ret) {
+		goto exit;
+	}
+
+	/* Download the firmware */
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+	ret = _dhdsdio_download_firmware(bus);
+
+	dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+
+exit:
+	DHD_OS_WAKE_UNLOCK(bus->dhd);
+	return ret;
+}
+
+/* Detach and free everything */
+static void
+dhdsdio_release(dhd_bus_t *bus, osl_t *osh)
+{
+	bool dongle_isolation = FALSE;
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus) {
+		ASSERT(osh);
+
+		if (bus->dhd) {
+#if defined(DEBUGGER) || defined(DHD_DSCOPE)
+			debugger_close();
+#endif /* DEBUGGER || DHD_DSCOPE */
+			dongle_isolation = bus->dhd->dongle_isolation;
+			dhd_detach(bus->dhd);
+		}
+
+		/* De-register interrupt handler */
+		bcmsdh_intr_disable(bus->sdh);
+		bcmsdh_intr_dereg(bus->sdh);
+
+		if (bus->dhd) {
+			dhdsdio_release_dongle(bus, osh, dongle_isolation, TRUE);
+			dhd_free(bus->dhd);
+			bus->dhd = NULL;
+		}
+
+		dhdsdio_release_malloc(bus, osh);
+
+#ifdef DHD_DEBUG
+		if (bus->console.buf != NULL)
+			MFREE(osh, bus->console.buf, bus->console.bufsize);
+#endif
+
+#ifdef DHDENABLE_TAILPAD
+		if (bus->pad_pkt)
+			PKTFREE(osh, bus->pad_pkt, FALSE);
+#endif /* DHDENABLE_TAILPAD */
+#ifdef DYNAMIC_MAX_HDR_READ
+	if (bus->hdrbufp) {
+		MFREE(osh, bus->hdrbufp, MAX_HDR_READ + DHD_SDALIGN);
+	}
+#endif
+
+		MFREE(osh, bus, sizeof(dhd_bus_t));
+	}
+
+	DHD_TRACE(("%s: Disconnected\n", __FUNCTION__));
+}
+
+static void
+dhdsdio_release_malloc(dhd_bus_t *bus, osl_t *osh)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	if (bus->dhd && bus->dhd->dongle_reset)
+		return;
+
+	if (bus->rxbuf) {
+#ifndef CONFIG_DHD_USE_STATIC_BUF
+		MFREE(osh, bus->rxbuf, bus->rxblen);
+#endif
+		bus->rxctl = NULL;
+		bus->rxlen = 0;
+	}
+
+	if (bus->databuf) {
+#ifndef CONFIG_DHD_USE_STATIC_BUF
+		MFREE(osh, bus->databuf, MAX_DATA_BUF);
+#endif
+	}
+
+	if (bus->membuf) {
+		MFREE(osh, bus->membuf, MAX_MEM_BUF);
+		bus->membuf = NULL;
+	}
+
+	if (bus->vars && bus->varsz) {
+		MFREE(osh, bus->vars, bus->varsz);
+	}
+
+}
+
+static void
+dhdsdio_release_dongle(dhd_bus_t *bus, osl_t *osh, bool dongle_isolation, bool reset_flag)
+{
+	DHD_TRACE(("%s: Enter bus->dhd %p bus->dhd->dongle_reset %d \n", __FUNCTION__,
+		bus->dhd, bus->dhd->dongle_reset));
+
+	if ((bus->dhd && bus->dhd->dongle_reset) && reset_flag)
+		return;
+
+	if (bus->sih) {
+		/* In Win10, system will be BSOD if using "sysprep" to do OS image */
+		/* Skip this will not cause the BSOD. */
+#if defined(DHD_SI_WD_RESET) || (!defined(BCMLXSDMMC) && !defined(NDIS))
+		if (CHIPID(bus->sih->chip) == BCM4381_CHIP_GRPID) {
+			/* XXX - Using the watchdog to reset the chip does not allow
+			 * further SDIO communication.  For the SDMMC Driver, this
+			 * causes interrupt to not be de-registered properly.
+			 */
+			/* XXX: dongle isolation mode is on don't reset the chip */
+			if (bus->dhd) {
+				dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+			}
+			if (KSO_ENAB(bus) && (dongle_isolation == FALSE)) {
+				si_watchdog(bus->sih, 4);
+				DHD_ERROR(("%s: after si_watchdog, dongle is going to be released\n", __FUNCTION__));
+#if defined(DHD_SI_WD_RESET)
+				DHD_ERROR(("%s: set si_wd TRUE\n", __FUNCTION__));
+				bus->dhd->si_wd = TRUE;
+#endif
+			}
+		}
+#endif /* !defined(BCMLXSDMMC) */
+		if (bus->dhd) {
+#ifdef DHD_SI_WD_RESET
+			if (!bus->dhd->si_wd)
+#endif
+				dhdsdio_clkctl(bus, CLK_NONE, FALSE);
+		}
+		si_detach(bus->sih);
+		bus->sih = NULL;
+		if (bus->vars && bus->varsz)
+			MFREE(osh, bus->vars, bus->varsz);
+		bus->vars = NULL;
+	}
+
+	DHD_TRACE(("%s: Disconnected\n", __FUNCTION__));
+}
+
+static void
+dhdsdio_disconnect(void *ptr)
+{
+	dhd_bus_t *bus = (dhd_bus_t *)ptr;
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	DHD_MUTEX_LOCK();
+	if (bus) {
+		ASSERT(bus->dhd);
+		/* Advertise bus remove during rmmod */
+		dhdsdio_advertise_bus_remove(bus->dhd);
+		dhdsdio_release(bus, bus->dhd->osh);
+	}
+	DHD_MUTEX_UNLOCK();
+
+	DHD_TRACE(("%s: Disconnected\n", __FUNCTION__));
+}
+
+#ifdef PWRSTATS_SYSFS
+uint64
+dhdsdio_get_last_suspend_time(dhd_pub_t *dhdp)
+{
+	return dhdp->bus->last_suspend_end_time;
+}
+#endif /* PWRSTATS_SYSFS */
+
+static int
+dhdsdio_suspend(void *context)
+{
+	int ret = 0;
+#ifdef SUPPORT_P2P_GO_PS
+	int wait_time = 0;
+#endif /* SUPPORT_P2P_GO_PS */
+
+#if defined(LINUX)
+	dhd_bus_t *bus = (dhd_bus_t*)context;
+	unsigned long flags;
+
+	DHD_ERROR(("%s Enter\n", __FUNCTION__));
+	if (bus->dhd == NULL) {
+		DHD_ERROR(("bus not inited\n"));
+		return BCME_ERROR;
+	}
+	if (bus->dhd->prot == NULL) {
+		DHD_ERROR(("prot is not inited\n"));
+		return BCME_ERROR;
+	}
+
+	if (bus->dhd->up == FALSE) {
+		return BCME_OK;
+	}
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	if (bus->dhd->busstate != DHD_BUS_DATA && bus->dhd->busstate != DHD_BUS_SUSPEND) {
+		DHD_ERROR(("not in a readystate to LPBK  is not inited\n"));
+		DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+		return BCME_ERROR;
+	}
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+	if (bus->dhd->dongle_reset) {
+		DHD_ERROR(("Dongle is in reset state.\n"));
+		return -EIO;
+	}
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	/* stop all interface network queue. */
+	dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, ON);
+	bus->dhd->busstate = DHD_BUS_SUSPEND;
+#if defined(LINUX) || defined(linux)
+	if (DHD_BUS_BUSY_CHECK_IN_TX(bus->dhd)) {
+		DHD_ERROR(("Tx Request is not ended\n"));
+		bus->dhd->busstate = DHD_BUS_DATA;
+		/* resume all interface network queue. */
+		dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+		DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+		return -EBUSY;
+	}
+#endif /* LINUX || linux */
+	DHD_BUS_BUSY_SET_SUSPEND_IN_PROGRESS(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+#ifdef SUPPORT_P2P_GO_PS
+	if (bus->idletime > 0) {
+		wait_time = msecs_to_jiffies(bus->idletime * dhd_watchdog_ms);
+	}
+#endif /* SUPPORT_P2P_GO_PS */
+	ret = dhd_os_check_wakelock(bus->dhd);
+#ifdef SUPPORT_P2P_GO_PS
+	// terence 20141124: fix for suspend issue
+	if (SLPAUTO_ENAB(bus) && (!ret) && (bus->dhd->up) && (bus->dhd->op_mode != DHD_FLAG_HOSTAP_MODE)) {
+		if (wait_event_timeout(bus->bus_sleep, bus->sleeping, wait_time) == 0) {
+			if (!bus->sleeping) {
+				ret = 1;
+			}
+		}
+	}
+#endif /* SUPPORT_P2P_GO_PS */
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	if (ret) {
+		bus->dhd->busstate = DHD_BUS_DATA;
+		/* resume all interface network queue. */
+		dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+	} else {
+		bus->last_suspend_end_time = OSL_LOCALTIME_NS();
+	}
+	bus->dhd->hostsleep = 2;
+	DHD_BUS_BUSY_CLEAR_SUSPEND_IN_PROGRESS(bus->dhd);
+	dhd_os_busbusy_wake(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+#endif /* LINUX */
+	return ret;
+}
+
+static int
+dhdsdio_resume(void *context)
+{
+	dhd_bus_t *bus = (dhd_bus_t*)context;
+	ulong flags;
+
+	DHD_ERROR(("%s Enter\n", __FUNCTION__));
+
+	if (bus->dhd->up == FALSE) {
+		return BCME_OK;
+	}
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_SET_RESUME_IN_PROGRESS(bus->dhd);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+	if (dhd_os_check_if_up(bus->dhd))
+		bcmsdh_oob_intr_set(bus->sdh, TRUE);
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+
+	DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+	DHD_BUS_BUSY_CLEAR_RESUME_IN_PROGRESS(bus->dhd);
+	bus->dhd->hostsleep = 0;
+	bus->dhd->busstate = DHD_BUS_DATA;
+	dhd_os_busbusy_wake(bus->dhd);
+	/* resume all interface network queue. */
+	dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+	DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+	return 0;
+}
+
+/* Register/Unregister functions are called by the main DHD entry
+ * point (e.g. module insertion) to link with the bus driver, in
+ * order to look for or await the device.
+ */
+
+static bcmsdh_driver_t dhd_sdio = {
+	dhdsdio_probe,
+	dhdsdio_disconnect,
+	dhdsdio_suspend,
+	dhdsdio_resume
+};
+
+int
+dhd_bus_register(void)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	return bcmsdh_register(&dhd_sdio);
+}
+
+void
+dhd_bus_unregister(void)
+{
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	bcmsdh_unregister();
+}
+
+#if defined(BCMLXSDMMC)
+/* Register a dummy SDIO client driver in order to be notified of new SDIO device */
+int dhd_bus_reg_sdio_notify(void* semaphore)
+{
+	return bcmsdh_reg_sdio_notify(semaphore);
+}
+
+void dhd_bus_unreg_sdio_notify(void)
+{
+	bcmsdh_unreg_sdio_notify();
+}
+#endif /* defined(BCMLXSDMMC) */
+
+#ifdef BCMEMBEDIMAGE
+static int
+dhdsdio_download_code_array(struct dhd_bus *bus)
+{
+	int bcmerror = -1;
+	int offset = 0;
+	unsigned char *ularray = NULL;
+
+	DHD_INFO(("%s: download embedded firmware...\n", __FUNCTION__));
+
+	/* Download image */
+	while ((offset + MEMBLOCK) < sizeof(dlarray)) {
+		/* check if CR4 */
+		if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			/* if address is 0, store the reset instruction to be written in 0 */
+
+			if (offset == 0) {
+				bus->resetinstr = *(((uint32*)dlarray));
+				/* Add start of RAM address to the address given by user */
+				offset += bus->dongle_ram_base;
+			}
+		}
+
+		bcmerror = dhdsdio_membytes(bus, TRUE, offset,
+			(uint8 *) (dlarray + offset), MEMBLOCK);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			        __FUNCTION__, bcmerror, MEMBLOCK, offset));
+			goto err;
+		}
+
+		offset += MEMBLOCK;
+	}
+
+	if (offset < sizeof(dlarray)) {
+		bcmerror = dhdsdio_membytes(bus, TRUE, offset,
+			(uint8 *) (dlarray + offset), sizeof(dlarray) - offset);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			        __FUNCTION__, bcmerror, sizeof(dlarray) - offset, offset));
+			goto err;
+		}
+	}
+
+#ifdef DHD_DEBUG
+	/* Upload and compare the downloaded code */
+	{
+		ularray = MALLOC(bus->dhd->osh, bus->ramsize);
+		/* Upload image to verify downloaded contents. */
+		offset = 0;
+		memset(ularray, 0xaa, bus->ramsize);
+		while ((offset + MEMBLOCK) < sizeof(dlarray)) {
+			bcmerror = dhdsdio_membytes(bus, FALSE, offset, ularray + offset, MEMBLOCK);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+					__FUNCTION__, bcmerror, MEMBLOCK, offset));
+				goto err;
+			}
+
+			offset += MEMBLOCK;
+		}
+
+		if (offset < sizeof(dlarray)) {
+			bcmerror = dhdsdio_membytes(bus, FALSE, offset,
+				ularray + offset, sizeof(dlarray) - offset);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+					__FUNCTION__, bcmerror, sizeof(dlarray) - offset, offset));
+				goto err;
+			}
+		}
+
+		if (memcmp(dlarray, ularray, sizeof(dlarray))) {
+			DHD_ERROR(("%s: Downloaded image is corrupted (%s, %s, %s).\n",
+			           __FUNCTION__, dlimagename, dlimagever, dlimagedate));
+			goto err;
+		} else
+			DHD_ERROR(("%s: Download, Upload and compare succeeded (%s, %s, %s).\n",
+			           __FUNCTION__, dlimagename, dlimagever, dlimagedate));
+
+	}
+#endif /* DHD_DEBUG */
+
+err:
+	if (ularray)
+		MFREE(bus->dhd->osh, ularray, bus->ramsize);
+	return bcmerror;
+}
+#endif /* BCMEMBEDIMAGE */
+
+#ifdef DHD_LINUX_STD_FW_API
+static int
+dhdsdio_download_code_file(struct dhd_bus *bus, char *pfw_path)
+{
+	int bcmerror = -1;
+	int offset = 0;
+	int len;
+	uint8 *memblock = NULL, *memptr;
+#ifdef CHECK_DOWNLOAD_FW
+	uint8 *memptr_tmp = NULL; // terence: check downloaded firmware is correct
+#endif
+	uint memblock_size = MEMBLOCK;
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	unsigned long initial_jiffies = 0;
+	uint firmware_sz = 0;
+#endif
+	int offset_end = bus->ramsize;
+	const struct firmware *fw = NULL;
+	int buf_offset = 0, residual_len = 0;
+
+	DHD_INFO(("%s: download firmware %s\n", __FUNCTION__, pfw_path));
+
+	/* XXX: Should succeed in opening image if it is actually given through registry
+	 * entry or in module param.
+	 */
+	bcmerror = dhd_os_get_img_fwreq(&fw, bus->fw_path);
+	if (bcmerror < 0) {
+		DHD_ERROR(("dhd_os_get_img(Request Firmware API) error : %d\n",
+			bcmerror));
+		goto err;
+	}
+	bus->fw_download_len = fw->size;
+	bus->fw_download_addr = bus->dongle_ram_base;
+	residual_len = fw->size;
+
+	/* Update the dongle image download block size depending on the F1 block size */
+#ifndef NDIS
+	if (sd_f1_blocksize == 512)
+		memblock_size = MAX_MEMBLOCK;
+#endif /* !NDIS */
+
+	memptr = memblock = MALLOC(bus->dhd->osh, memblock_size + DHD_SDALIGN);
+	if (memblock == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__,
+			memblock_size));
+		goto err;
+	}
+	if ((uint32)(uintptr)memblock % DHD_SDALIGN)
+		memptr += (DHD_SDALIGN - ((uint32)(uintptr)memblock % DHD_SDALIGN));
+
+#ifdef CHECK_DOWNLOAD_FW
+	if (bus->dhd->conf->fwchk) {
+		memptr_tmp = MALLOC(bus->dhd->osh, MEMBLOCK + DHD_SDALIGN);
+		if (memptr_tmp == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__, MEMBLOCK));
+			goto err;
+		}
+	}
+#endif
+
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	initial_jiffies = jiffies;
+#endif
+
+	/* Download image */
+	while (residual_len) {
+		len = MIN(residual_len, memblock_size);
+		bcopy((const uint8 *)fw->data + buf_offset, (uint8 *)memptr, len);
+		/* check if CR4 */
+		if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			/* if address is 0, store the reset instruction to be written in 0 */
+
+			if (offset == 0) {
+				bus->resetinstr = *(((uint32*)memptr));
+				/* Add start of RAM address to the address given by user */
+				offset += bus->dongle_ram_base;
+				offset_end += offset;
+			}
+		}
+
+		bcmerror = dhdsdio_membytes(bus, TRUE, offset, (uint8 *)memptr, len);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			        __FUNCTION__, bcmerror, memblock_size, offset));
+			goto err;
+		}
+
+#ifdef CHECK_DOWNLOAD_FW
+		if (bus->dhd->conf->fwchk) {
+			bcmerror = dhdsdio_membytes(bus, FALSE, offset, memptr_tmp, len);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+				        __FUNCTION__, bcmerror, MEMBLOCK, offset));
+				goto err;
+			}
+			if (memcmp(memptr_tmp, memptr, len)) {
+				DHD_ERROR(("%s: Downloaded image is corrupted at 0x%08x\n", __FUNCTION__, offset));
+				bcmerror = BCME_ERROR;
+				goto err;
+			} else
+				DHD_INFO(("%s: Download, Upload and compare succeeded.\n", __FUNCTION__));
+		}
+#endif
+
+		offset += memblock_size;
+#ifdef DHD_DEBUG_DOWNLOADTIME
+		firmware_sz += len;
+#endif
+		if (offset >= offset_end) {
+			DHD_ERROR(("%s: invalid address access to %x (offset end: %x)\n",
+				__FUNCTION__, offset, offset_end));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+		residual_len -= len;
+		buf_offset += len;
+	}
+
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	DHD_ERROR(("Firmware download time for %u bytes: %u ms\n",
+			firmware_sz, jiffies_to_msecs(jiffies - initial_jiffies)));
+#endif
+
+err:
+	if (memblock)
+		MFREE(bus->dhd->osh, memblock, memblock_size + DHD_SDALIGN);
+#ifdef CHECK_DOWNLOAD_FW
+	if (bus->dhd->conf->fwchk) {
+		if (memptr_tmp)
+			MFREE(bus->dhd->osh, memptr_tmp, MEMBLOCK + DHD_SDALIGN);
+	}
+#endif
+
+	if (fw) {
+		dhd_os_close_img_fwreq(fw);
+	}
+
+	return bcmerror;
+}
+#else
+static int
+dhdsdio_download_code_file(struct dhd_bus *bus, char *pfw_path)
+{
+	int bcmerror = -1;
+	int offset = 0;
+	int len;
+	void *image = NULL;
+	uint8 *memblock = NULL, *memptr;
+#ifdef CHECK_DOWNLOAD_FW
+	uint8 *memptr_tmp = NULL; // terence: check downloaded firmware is correct
+#endif
+	uint memblock_size = MEMBLOCK;
+	uint32 file_size = 0;
+	fwpkg_info_t *fwpkg;
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	unsigned long initial_jiffies = 0;
+	uint firmware_sz = 0;
+#endif
+
+	DHD_INFO(("%s: download firmware %s\n", __FUNCTION__, pfw_path));
+
+	/* XXX: Should succeed in opening image if it is actually given through registry
+	 * entry or in module param.
+	 */
+	bcmerror = fwpkg_init(&bus->fwpkg, pfw_path);
+	if (bcmerror == BCME_ERROR) {
+		printf("%s: fwpkg_init failed %s\n", __FUNCTION__, pfw_path);
+		goto err;
+	}
+	fwpkg = &bus->fwpkg;
+	/* Should succeed in opening image if it is actually given through registry
+	 * entry or in module param.
+	 */
+	bcmerror = fwpkg_open_firmware_img(fwpkg, pfw_path, &image);
+	if (bcmerror == BCME_ERROR) {
+		printf("%s: Open firmware file failed %s\n", __FUNCTION__, pfw_path);
+		goto err;
+	}
+
+	if (bcmerror == BCME_UNSUPPORTED) {
+		file_size = fwpkg->file_size;
+		DHD_INFO(("%s Using SINGLE image (size %d)\n",
+			__FUNCTION__, file_size));
+	} else {
+		file_size = fwpkg_get_firmware_img_size(fwpkg);
+		strlcpy(bus->fwsig_filename, pfw_path, sizeof(bus->fwsig_filename));
+		DHD_INFO(("%s Using COMBINED image (size %d)\n",
+			__FUNCTION__, file_size));
+	}
+	bus->fw_download_len = file_size;
+	bus->fw_download_addr = bus->dongle_ram_base;
+
+	/* Update the dongle image download block size depending on the F1 block size */
+#ifndef NDIS
+	if (sd_f1_blocksize == 512)
+		memblock_size = MAX_MEMBLOCK;
+#endif /* !NDIS */
+
+	memptr = memblock = MALLOC(bus->dhd->osh, memblock_size + DHD_SDALIGN);
+	if (memblock == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__,
+			memblock_size));
+		goto err;
+	}
+	if ((uint32)(uintptr)memblock % DHD_SDALIGN)
+		memptr += (DHD_SDALIGN - ((uint32)(uintptr)memblock % DHD_SDALIGN));
+
+#ifdef CHECK_DOWNLOAD_FW
+	if (bus->dhd->conf->fwchk) {
+		memptr_tmp = MALLOC(bus->dhd->osh, MEMBLOCK + DHD_SDALIGN);
+		if (memptr_tmp == NULL) {
+			DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__, MEMBLOCK));
+			goto err;
+		}
+	}
+#endif
+
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	initial_jiffies = jiffies;
+#endif
+
+	/* Download image */
+	while ((len = dhd_os_get_image_block((char*)memptr, memblock_size, image))) {
+		// terence 20150412: fix for firmware failed to download
+		if (bus->dhd->conf->chip == BCM43340_CHIP_ID ||
+				bus->dhd->conf->chip == BCM43341_CHIP_ID) {
+			if (len % 64 != 0) {
+				memset(memptr+len, 0, len%64);
+				len += (64 - len%64);
+			}
+		}
+		if (len < 0) {
+			DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+		/* check if CR4 */
+		if (si_setcore(bus->sih, ARMCR4_CORE_ID, 0)) {
+			/* if address is 0, store the reset instruction to be written in 0 */
+
+			if (offset == 0) {
+				bus->resetinstr = *(((uint32*)memptr));
+				/* Add start of RAM address to the address given by user */
+				offset += bus->dongle_ram_base;
+			}
+		}
+
+		bcmerror = dhdsdio_membytes(bus, TRUE, offset, memptr, len);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			        __FUNCTION__, bcmerror, memblock_size, offset));
+			goto err;
+		}
+
+#ifdef CHECK_DOWNLOAD_FW
+		if (bus->dhd->conf->fwchk) {
+			bcmerror = dhdsdio_membytes(bus, FALSE, offset, memptr_tmp, len);
+			if (bcmerror) {
+				DHD_ERROR(("%s: error %d on reading %d membytes at 0x%08x\n",
+				        __FUNCTION__, bcmerror, MEMBLOCK, offset));
+				goto err;
+			}
+			if (memcmp(memptr_tmp, memptr, len)) {
+				DHD_ERROR(("%s: Downloaded image is corrupted at 0x%08x\n", __FUNCTION__, offset));
+				bcmerror = BCME_ERROR;
+				goto err;
+			} else
+				DHD_INFO(("%s: Download, Upload and compare succeeded.\n", __FUNCTION__));
+		}
+#endif
+
+		offset += memblock_size;
+#ifdef DHD_DEBUG_DOWNLOADTIME
+		firmware_sz += len;
+#endif
+	}
+
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	DHD_INFO(("Firmware download time for %u bytes: %u ms\n",
+			firmware_sz, jiffies_to_msecs(jiffies - initial_jiffies)));
+#endif
+
+err:
+	if (memblock)
+		MFREE(bus->dhd->osh, memblock, memblock_size + DHD_SDALIGN);
+#ifdef CHECK_DOWNLOAD_FW
+	if (bus->dhd->conf->fwchk) {
+		if (memptr_tmp)
+			MFREE(bus->dhd->osh, memptr_tmp, MEMBLOCK + DHD_SDALIGN);
+	}
+#endif
+
+	if (image)
+		dhd_os_close_image1(bus->dhd, image);
+
+	return bcmerror;
+}
+#endif /* DHD_LINUX_STD_FW_API */
+
+#ifdef DHD_UCODE_DOWNLOAD
+/* Currently supported only for the chips in which ucode RAM is AXI addressable */
+static uint32
+dhdsdio_ucode_base(struct dhd_bus *bus)
+{
+	uint32 ucode_base = 0;
+
+	switch ((uint16)bus->sih->chip) {
+	case BCM43012_CHIP_ID:
+	case BCM43013_CHIP_ID:
+	case BCM43014_CHIP_ID:
+		ucode_base = 0xE8020000;
+		break;
+	default:
+		DHD_ERROR(("%s: Unsupported!\n", __func__));
+		break;
+	}
+
+	return ucode_base;
+}
+
+static int
+dhdsdio_download_ucode_file(struct dhd_bus *bus, char *ucode_path)
+{
+	int bcmerror = -1;
+	int offset = 0;
+	int len;
+	uint32 ucode_base;
+	void *image = NULL;
+	uint8 *memblock = NULL, *memptr;
+	uint memblock_size = MEMBLOCK;
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	unsigned long initial_jiffies = 0;
+	uint firmware_sz = 0;
+#endif
+
+	DHD_INFO(("%s: download firmware %s\n", __FUNCTION__, ucode_path));
+
+	ucode_base = dhdsdio_ucode_base(bus);
+
+	/* XXX: Should succeed in opening image if it is actually given through registry
+	 * entry or in module param.
+	 */
+	image = dhd_os_open_image1(bus->dhd, ucode_path);
+	if (image == NULL)
+		goto err;
+
+	/* Update the dongle image download block size depending on the F1 block size */
+	if (sd_f1_blocksize == 512)
+		memblock_size = MAX_MEMBLOCK;
+
+	memptr = memblock = MALLOC(bus->dhd->osh, memblock_size + DHD_SDALIGN);
+	if (memblock == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__,
+			memblock_size));
+		goto err;
+	}
+	if ((uint32)(uintptr)memblock % DHD_SDALIGN)
+		memptr += (DHD_SDALIGN - ((uint32)(uintptr)memblock % DHD_SDALIGN));
+
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	initial_jiffies = jiffies;
+#endif
+
+	/* Download image */
+	while ((len = dhd_os_get_image_block((char*)memptr, memblock_size, image))) {
+		if (len < 0) {
+			DHD_ERROR(("%s: dhd_os_get_image_block failed (%d)\n", __FUNCTION__, len));
+			bcmerror = BCME_ERROR;
+			goto err;
+		}
+
+		bcmerror = dhdsdio_membytes(bus, TRUE, (ucode_base + offset), memptr, len);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+			        __FUNCTION__, bcmerror, memblock_size, offset));
+			goto err;
+		}
+
+		offset += memblock_size;
+#ifdef DHD_DEBUG_DOWNLOADTIME
+		firmware_sz += len;
+#endif
+	}
+
+#ifdef DHD_DEBUG_DOWNLOADTIME
+	DHD_ERROR(("ucode download time for %u bytes: %u ms\n",
+			firmware_sz, jiffies_to_msecs(jiffies - initial_jiffies)));
+#endif
+
+err:
+	if (memblock)
+		MFREE(bus->dhd->osh, memblock, memblock_size + DHD_SDALIGN);
+
+	if (image)
+		dhd_os_close_image1(bus->dhd, image);
+
+	return bcmerror;
+} /* dhdsdio_download_ucode_file */
+
+void
+dhd_bus_ucode_download(struct dhd_bus *bus)
+{
+	uint32 shaddr = 0, shdata = 0;
+
+	shaddr = bus->dongle_ram_base + bus->ramsize - 4;
+	dhdsdio_membytes(bus, FALSE, shaddr, (uint8 *)&shdata, 4);
+
+	DHD_TRACE(("%s: shdata:[0x%08x :0x%08x]\n", __func__, shaddr, shdata));
+
+	if (shdata == UCODE_DOWNLOAD_REQUEST)
+	{
+		DHD_ERROR(("%s: Received ucode download request!\n", __func__));
+
+		/* Download the ucode */
+		if (!dhd_get_ucode_path(bus->dhd)) {
+			DHD_ERROR(("%s: bus->uc_path not set!\n", __func__));
+			return;
+		}
+		dhdsdio_download_ucode_file(bus, dhd_get_ucode_path(bus->dhd));
+
+		DHD_ERROR(("%s: Ucode downloaded successfully!\n", __func__));
+
+		shdata = UCODE_DOWNLOAD_COMPLETE;
+		dhdsdio_membytes(bus, TRUE, shaddr, (uint8 *)&shdata, 4);
+	}
+}
+
+#endif /* DHD_UCODE_DOWNLOAD */
+
+static int
+dhdsdio_download_nvram(struct dhd_bus *bus)
+{
+	int bcmerror = -1;
+	uint len, memblock_len = 0;
+	char * memblock = NULL;
+	char *bufp;
+	char *pnv_path;
+	bool nvram_file_exists;
+
+	pnv_path = bus->nv_path;
+
+	nvram_file_exists = ((pnv_path != NULL) && (pnv_path[0] != '\0'));
+
+	len = MAX_NVRAMBUF_SIZE;
+	if (nvram_file_exists)
+		bcmerror = dhd_get_download_buffer(bus->dhd, pnv_path, NVRAM, &memblock,
+			(int *)&len);
+	else
+		bcmerror = dhd_get_download_buffer(bus->dhd, NULL, NVRAM, &memblock, (int *)&len);
+
+	if (bcmerror != BCME_OK)
+		goto err;
+
+	memblock_len = MAX_NVRAMBUF_SIZE;
+
+	if (len > 0 && len < MAX_NVRAMBUF_SIZE) {
+		bufp = (char *)memblock;
+		bufp[len] = 0;
+		len = process_nvram_vars(bufp, len);
+		if (len % 4) {
+			len += 4 - (len % 4);
+		}
+		bufp += len;
+		*bufp++ = 0;
+		if (len)
+			bcmerror = dhdsdio_downloadvars(bus, memblock, len + 1);
+		if (bcmerror) {
+			DHD_ERROR(("%s: error downloading vars: %d\n",
+			           __FUNCTION__, bcmerror));
+		}
+	} else {
+		DHD_ERROR(("%s: error reading nvram file: %s %d\n",
+		           __FUNCTION__, pnv_path, len));
+		bcmerror = BCME_SDIO_ERROR;
+	}
+
+err:
+	if (memblock)
+		dhd_free_download_buffer(bus->dhd, memblock, memblock_len);
+	return bcmerror;
+}
+
+static int
+_dhdsdio_download_firmware(struct dhd_bus *bus)
+{
+	int bcmerror = -1;
+
+	bool embed = FALSE;	/* download embedded firmware */
+	bool dlok = FALSE;	/* download firmware succeeded */
+
+	/* Out immediately if no image to download */
+	if ((bus->fw_path == NULL) || (bus->fw_path[0] == '\0')) {
+#ifdef BCMEMBEDIMAGE
+		embed = TRUE;
+#else
+		return bcmerror;
+#endif
+	}
+
+	/* Keep arm in reset */
+	if (dhdsdio_download_state(bus, TRUE)) {
+		DHD_ERROR(("%s: error placing ARM core in reset\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* External image takes precedence if specified */
+	if ((bus->fw_path != NULL) && (bus->fw_path[0] != '\0')) {
+		if (dhdsdio_download_code_file(bus, bus->fw_path)) {
+			DHD_ERROR(("%s: dongle image file %s download failed\n",
+				__FUNCTION__, bus->fw_path));
+#ifdef BCMEMBEDIMAGE
+			embed = TRUE;
+#else
+			goto err;
+#endif
+		} else {
+			embed = FALSE;
+			dlok = TRUE;
+		}
+	}
+
+#ifdef BCMEMBEDIMAGE
+	if (embed) {
+		if (dhdsdio_download_code_array(bus)) {
+			DHD_ERROR(("%s: dongle image array download failed\n", __FUNCTION__));
+			goto err;
+		} else {
+			dlok = TRUE;
+		}
+	}
+#else
+	BCM_REFERENCE(embed);
+#endif
+	if (!dlok) {
+		DHD_ERROR(("%s: dongle image download failed\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* External nvram takes precedence if specified */
+	if (dhdsdio_download_nvram(bus)) {
+		DHD_ERROR(("%s: dongle nvram file download failed\n", __FUNCTION__));
+		goto err;
+	}
+
+	/* Take arm out of reset */
+	if (dhdsdio_download_state(bus, FALSE)) {
+		DHD_ERROR(("%s: error getting out of ARM core reset\n", __FUNCTION__));
+		goto err;
+	}
+
+	bcmerror = 0;
+
+err:
+	return bcmerror;
+}
+
+static int
+dhd_bcmsdh_recv_buf(dhd_bus_t *bus, uint32 addr, uint fn, uint flags, uint8 *buf, uint nbytes,
+	void *pkt, bcmsdh_cmplt_fn_t complete_fn, void *handle)
+{
+	int status;
+
+	if (!KSO_ENAB(bus)) {
+		DHD_ERROR(("%s: Device asleep\n", __FUNCTION__));
+		return BCME_NODEVICE;
+	}
+
+	status = bcmsdh_recv_buf(bus->sdh, addr, fn, flags, buf, nbytes, pkt, complete_fn, handle);
+
+	return status;
+}
+
+static int
+dhd_bcmsdh_send_buf(dhd_bus_t *bus, uint32 addr, uint fn, uint flags, uint8 *buf, uint nbytes,
+	void *pkt, bcmsdh_cmplt_fn_t complete_fn, void *handle, int max_retry)
+{
+	int ret;
+	int i = 0;
+	int retries = 0;
+	bcmsdh_info_t *sdh;
+
+	if (!KSO_ENAB(bus)) {
+		DHD_ERROR(("%s: Device asleep\n", __FUNCTION__));
+		return BCME_NODEVICE;
+	}
+
+	sdh = bus->sdh;
+	do {
+		ret = bcmsdh_send_buf(bus->sdh, addr, fn, flags, buf, nbytes,
+			pkt, complete_fn, handle);
+
+		bus->f2txdata++;
+		ASSERT(ret != BCME_PENDING);
+
+		if (ret == BCME_NODEVICE) {
+			DHD_ERROR(("%s: Device asleep already\n", __FUNCTION__));
+		} else if (ret < 0) {
+			/* On failure, abort the command and terminate the frame */
+			DHD_ERROR(("%s: sdio error %d, abort command and terminate frame.\n",
+				__FUNCTION__, ret));
+			bus->tx_sderrs++;
+			bus->f1regdata++;
+			bus->dhd->tx_errors++;
+			bcmsdh_abort(sdh, SDIO_FUNC_2);
+			bcmsdh_cfg_write(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_FRAMECTRL,
+				SFC_WF_TERM, NULL);
+			for (i = 0; i < READ_FRM_CNT_RETRIES; i++) {
+				uint8 hi, lo;
+				hi = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WFRAMEBCHI,
+					NULL);
+				lo = bcmsdh_cfg_read(sdh, SDIO_FUNC_1, SBSDIO_FUNC1_WFRAMEBCLO,
+					NULL);
+				bus->f1regdata += 2;
+				if ((hi == 0) && (lo == 0))
+					break;
+			}
+		}
+	} while ((ret < 0) && retrydata && ++retries < max_retry);
+
+	return ret;
+}
+
+uint8
+dhd_bus_is_ioready(struct dhd_bus *bus)
+{
+	uint8 enable;
+	bcmsdh_info_t *sdh;
+	ASSERT(bus);
+	ASSERT(bus->sih != NULL);
+	enable = (SDIO_FUNC_ENABLE_1 | SDIO_FUNC_ENABLE_2);
+	sdh = bus->sdh;
+	return (enable == bcmsdh_cfg_read(sdh, SDIO_FUNC_0, SDIOD_CCCR_IORDY, NULL));
+}
+
+uint
+dhd_bus_chip(struct dhd_bus *bus)
+{
+	ASSERT(bus->sih != NULL);
+	return bus->sih->chip;
+}
+
+uint
+dhd_bus_chiprev(struct dhd_bus *bus)
+{
+	ASSERT(bus);
+	ASSERT(bus->sih != NULL);
+	return bus->sih->chiprev;
+}
+
+void *
+dhd_bus_pub(struct dhd_bus *bus)
+{
+	return bus->dhd;
+}
+
+void *
+dhd_bus_sih(struct dhd_bus *bus)
+{
+	return (void *)bus->sih;
+}
+
+void *
+dhd_bus_txq(struct dhd_bus *bus)
+{
+	return &bus->txq;
+}
+
+uint
+dhd_bus_hdrlen(struct dhd_bus *bus)
+{
+	return (bus->txglom_enable) ? SDPCM_HDRLEN_TXGLOM : SDPCM_HDRLEN;
+}
+
+void
+dhd_bus_set_dotxinrx(struct dhd_bus *bus, bool val)
+{
+	bus->dotxinrx = val;
+}
+
+/*
+ *  dhdsdio_advertise_bus_cleanup advertises that clean up is under progress
+ * to other bus user contexts like Tx, Rx, IOVAR, WD etc and it waits for other contexts
+ * to gracefully exit. All the bus usage contexts before marking busstate as busy, will check for
+ * whether the busstate is DHD_BUS_DOWN or DHD_BUS_DOWN_IN_PROGRESS, if so
+ * they will exit from there itself without marking dhd_bus_busy_state as BUSY.
+ */
+static void
+dhdsdio_advertise_bus_cleanup(dhd_pub_t	 *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+	dhdp->busstate = DHD_BUS_DOWN_IN_PROGRESS;
+	DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+#ifdef LINUX
+	if ((timeleft == 0) || (timeleft == 1))
+#else
+	if (timeleft == 0)
+#endif
+	{
+		/* XXX This condition ideally should not occur, this means some
+		 * bus usage context is not clearing the respective usage bit, print
+		 * dhd_bus_busy_state and crash the host for further debugging.
+		 */
+		DHD_ERROR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+		ASSERT(0);
+	}
+
+	return;
+}
+
+static void
+dhdsdio_advertise_bus_remove(dhd_pub_t	 *dhdp)
+{
+	unsigned long flags;
+	int timeleft;
+
+	DHD_LINUX_GENERAL_LOCK(dhdp, flags);
+	dhdp->busstate = DHD_BUS_REMOVE;
+	DHD_LINUX_GENERAL_UNLOCK(dhdp, flags);
+
+	timeleft = dhd_os_busbusy_wait_negation(dhdp, &dhdp->dhd_bus_busy_state);
+	if ((timeleft == 0) || (timeleft == 1)) {
+		DHD_ERROR(("%s : Timeout due to dhd_bus_busy_state=0x%x\n",
+				__FUNCTION__, dhdp->dhd_bus_busy_state));
+		ASSERT(0);
+	}
+
+	return;
+}
+
+int
+dhd_bus_devreset(dhd_pub_t *dhdp, uint8 flag)
+{
+	int bcmerror = 0;
+	dhd_bus_t *bus;
+	unsigned long flags;
+
+	bus = dhdp->bus;
+
+	if (flag == TRUE) {
+		if (!bus->dhd->dongle_reset) {
+			DHD_INFO(("%s: == Power OFF ==\n", __FUNCTION__));
+#ifdef DHD_SI_WD_RESET
+			if (CHIPID(bus->sih->chip) == BCM4381_CHIP_GRPID) {
+				DHD_ERROR(("%s: RESET PMU status\n", __FUNCTION__));
+				bcmsdh_reg_write(bus->sdh, 0x18012618, 4, 0x64fffff);
+				OSL_DELAY(100);
+			}
+#endif
+			dhdsdio_advertise_bus_cleanup(bus->dhd);
+			dhd_os_sdlock(dhdp);
+			dhd_os_wd_timer(dhdp, 0);
+#if defined(OEM_ANDROID)
+#if !defined(IGNORE_ETH0_DOWN)
+			/* Force flow control as protection when stop come before ifconfig_down */
+			dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, ON);
+#endif /* !defined(IGNORE_ETH0_DOWN) */
+#endif /* OEM_ANDROID */
+			/* Expect app to have torn down any connection before calling */
+			/* Stop the bus, disable F2 */
+			dhd_bus_stop(bus, FALSE);
+
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+			/* Clean up any pending IRQ */
+			dhd_enable_oob_intr(bus, FALSE);
+			bcmsdh_oob_intr_set(bus->sdh, FALSE);
+			bcmsdh_oob_intr_unregister(bus->sdh);
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+
+			/* Clean tx/rx buffer pointers, detach from the dongle */
+#ifdef DHD_SI_WD_RESET
+			dhdsdio_release_dongle(bus, bus->dhd->osh, FALSE, TRUE);
+#else
+			dhdsdio_release_dongle(bus, bus->dhd->osh, TRUE, TRUE);
+#endif
+			bus->dhd->dongle_reset = TRUE;
+			DHD_INFO(("%s: making dhdpub up FALSE\n", __FUNCTION__));
+			bus->dhd->up = FALSE;
+			dhd_txglom_enable(dhdp, FALSE);
+			dhd_os_sdunlock(dhdp);
+
+			DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+			DHD_INFO(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+			bus->dhd->busstate = DHD_BUS_DOWN;
+			DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+
+			DHD_TRACE(("%s:  WLAN OFF DONE\n", __FUNCTION__));
+			/* App can now remove power from device */
+		} else
+			bcmerror = BCME_SDIO_ERROR;
+	} else {
+		/* App must have restored power to device before calling */
+
+		DHD_INFO(("%s: == Power ON ==\n", __FUNCTION__));
+
+#ifdef DHD_SI_WD_RESET
+		DHD_INFO(("%s: set si_wd FALSE\n", __FUNCTION__));
+		bus->dhd->si_wd = FALSE;
+#endif
+		if (bus->dhd->dongle_reset) {
+			/* Turn on WLAN */
+			dhd_os_sdlock(dhdp);
+			/* Reset SD client */
+			bcmsdh_reset(bus->sdh);
+
+			/* Attempt to re-attach & download */
+			if (dhdsdio_probe_attach(bus, bus->dhd->osh, bus->sdh,
+				(uint32 *)(uintptr)si_enum_base(bus->cl_devid),
+				bus->cl_devid)) {
+
+				DHD_LINUX_GENERAL_LOCK(bus->dhd, flags);
+				DHD_INFO(("%s: making DHD_BUS_DOWN\n", __FUNCTION__));
+				bus->dhd->busstate = DHD_BUS_DOWN;
+				DHD_LINUX_GENERAL_UNLOCK(bus->dhd, flags);
+				/* Attempt to download binary to the dongle */
+				if (dhdsdio_probe_init(bus, bus->dhd->osh, bus->sdh) &&
+				    dhdsdio_download_firmware(bus, bus->dhd->osh, bus->sdh) >= 0) {
+
+					/* Re-init bus, enable F2 transfer */
+					bcmerror = dhd_bus_init((dhd_pub_t *) bus->dhd, FALSE);
+					if (bcmerror == BCME_OK) {
+#if defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID)
+						dhd_enable_oob_intr(bus, TRUE);
+						bcmsdh_oob_intr_register(bus->sdh,
+							dhdsdio_isr, bus);
+						bcmsdh_oob_intr_set(bus->sdh, TRUE);
+#elif defined(FORCE_WOWLAN)
+						dhd_enable_oob_intr(bus, TRUE);
+#endif /* defined(OOB_INTR_ONLY) || defined(BCMSPI_ANDROID) */
+
+						bus->dhd->dongle_reset = FALSE;
+						bus->dhd->up = TRUE;
+
+#if defined(OEM_ANDROID) && !defined(IGNORE_ETH0_DOWN)
+						/* Restore flow control  */
+						dhd_txflowcontrol(bus->dhd, ALL_INTERFACES, OFF);
+#endif /* defined(OEM_ANDROID) &&  (!defined(IGNORE_ETH0_DOWN)) */
+						dhd_os_wd_timer(dhdp, dhd_watchdog_ms);
+
+						DHD_TRACE(("%s: WLAN ON DONE\n", __FUNCTION__));
+					} else {
+						dhd_bus_stop(bus, FALSE);
+						dhdsdio_release_dongle(bus, bus->dhd->osh,
+							TRUE, FALSE);
+					}
+				} else {
+					DHD_ERROR(("%s Failed to download binary to the dongle\n",
+						__FUNCTION__));
+					if (bus->sih != NULL) {
+						si_detach(bus->sih);
+						bus->sih = NULL;
+					}
+					bcmerror = BCME_SDIO_ERROR;
+				}
+			} else
+				bcmerror = BCME_SDIO_ERROR;
+
+			dhd_os_sdunlock(dhdp);
+		} else {
+			DHD_INFO(("%s called when dongle is not in reset\n",
+				__FUNCTION__));
+#if defined(OEM_ANDROID)
+			DHD_INFO(("Will call dhd_bus_start instead\n"));
+			bcmerror = dhd_bus_resume(dhdp, 1);
+			if (bcmerror)
+				return bcmerror;
+#if defined(HW_OOB) || defined(FORCE_WOWLAN)
+			dhd_conf_set_hw_oob_intr(bus->sdh, bus->sih); // terence 20120615: fix for OOB initial issue
+#endif
+			if ((bcmerror = dhd_bus_start(dhdp)) != 0)
+				DHD_ERROR(("%s: dhd_bus_start fail with %d\n",
+					__FUNCTION__, bcmerror));
+#endif /* defined(OEM_ANDROID) */
+		}
+	}
+
+#ifdef PKT_STATICS
+	dhd_bus_clear_txpktstatics(dhdp);
+#endif
+	return bcmerror;
+}
+
+#if defined(LINUX)
+int dhd_bus_suspend(dhd_pub_t *dhdpub)
+{
+	return bcmsdh_stop(dhdpub->bus->sdh);
+}
+
+int dhd_bus_resume(dhd_pub_t *dhdpub, int stage)
+{
+	return bcmsdh_start(dhdpub->bus->sdh, stage);
+}
+#endif /* defined(LINUX) */
+
+/* Get Chip ID version */
+uint dhd_bus_chip_id(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+
+	if (bus && bus->sih)
+		return bus->sih->chip;
+	else
+		return 0;
+}
+
+/* Get Chip Rev ID version */
+uint dhd_bus_chiprev_id(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+
+	if (bus && bus->sih)
+		return bus->sih->chiprev;
+	else
+		return 0;
+}
+
+/* Get Chip Pkg ID version */
+uint dhd_bus_chippkg_id(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+
+	return bus->sih->chippkg;
+}
+
+int dhd_bus_get_ids(struct dhd_bus *bus, uint32 *bus_type, uint32 *bus_num, uint32 *slot_num)
+{
+	*bus_type = bus->bus;
+	*bus_num = bus->bus_num;
+	*slot_num = bus->slot_num;
+	return 0;
+}
+
+int
+dhd_bus_membytes(dhd_pub_t *dhdp, bool set, uint32 address, uint8 *data, uint size)
+{
+	dhd_bus_t *bus;
+
+	bus = dhdp->bus;
+	return dhdsdio_membytes(bus, set, address, data, size);
+}
+
+#if defined(SUPPORT_MULTIPLE_REVISION)
+static int
+concate_revision_bcm4335(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+
+	uint chipver;
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+	char chipver_tag[10] = "_4335";
+#else
+	char chipver_tag[4] = {0, };
+#endif /* defined(SUPPORT_MULTIPLE_CHIPS) */
+
+	DHD_TRACE(("%s: BCM4335 Multiple Revision Check\n", __FUNCTION__));
+	if (bus->sih->chip != BCM4335_CHIP_ID) {
+		DHD_ERROR(("%s:Chip is not BCM4335\n", __FUNCTION__));
+		return -1;
+	}
+	chipver = bus->sih->chiprev;
+	DHD_ERROR(("CHIP VER = [0x%x]\n", chipver));
+	if (chipver == 0x0) {
+		DHD_ERROR(("----- CHIP bcm4335_A0 -----\n"));
+		strcat(chipver_tag, "_a0");
+	} else if (chipver == 0x1) {
+		DHD_ERROR(("----- CHIP bcm4335_B0 -----\n"));
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+		strcat(chipver_tag, "_b0");
+#endif /* defined(SUPPORT_MULTIPLE_CHIPS) */
+	}
+
+	strcat(fw_path, chipver_tag);
+	strcat(nv_path, chipver_tag);
+	return 0;
+}
+
+static int
+concate_revision_bcm4339(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+
+	uint chipver;
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+	char chipver_tag[10] = "_4339";
+#else
+	char chipver_tag[4] = {0, };
+#endif /* defined(SUPPORT_MULTIPLE_CHIPS) */
+
+	DHD_TRACE(("%s: BCM4339 Multiple Revision Check\n", __FUNCTION__));
+	if (bus->sih->chip != BCM4339_CHIP_ID) {
+		DHD_ERROR(("%s:Chip is not BCM4339\n", __FUNCTION__));
+		return -1;
+	}
+	chipver = bus->sih->chiprev;
+	DHD_ERROR(("CHIP VER = [0x%x]\n", chipver));
+	if (chipver == 0x1) {
+		DHD_ERROR(("----- CHIP bcm4339_A0 -----\n"));
+		strcat(chipver_tag, "_a0");
+	} else {
+		DHD_ERROR(("----- CHIP bcm4339 unknown revision %d -----\n",
+			chipver));
+	}
+
+	strcat(fw_path, chipver_tag);
+	strcat(nv_path, chipver_tag);
+	return 0;
+}
+
+static int concate_revision_bcm4350(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	uint32 chip_ver;
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+	char chipver_tag[10] = {0, };
+#else
+	char chipver_tag[4] = {0, };
+#endif /* defined(SUPPORT_MULTIPLE_CHIPS) */
+	chip_ver = bus->sih->chiprev;
+
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+	if (chip_ver == 3)
+		strcat(chipver_tag, "_4354");
+	else
+		strcat(chipver_tag, "_4350");
+#endif
+
+	if (chip_ver == 3) {
+		DHD_ERROR(("----- CHIP 4354 A0 -----\n"));
+		strcat(chipver_tag, "_a0");
+	} else {
+		DHD_ERROR(("----- Unknown chip version, ver=%x -----\n", chip_ver));
+	}
+
+	strcat(fw_path, chipver_tag);
+	strcat(nv_path, chipver_tag);
+	return 0;
+}
+
+static int concate_revision_bcm4354(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	uint32 chip_ver;
+#if defined(SUPPORT_MULTIPLE_CHIPS)
+	char chipver_tag[10] = "_4354";
+#else
+	char chipver_tag[4] = {0, };
+#endif /* SUPPORT_MULTIPLE_CHIPS */
+
+	chip_ver = bus->sih->chiprev;
+	if (chip_ver == 1) {
+		DHD_ERROR(("----- CHIP 4354 A1 -----\n"));
+		strcat(chipver_tag, "_a1");
+	} else {
+		DHD_ERROR(("----- Unknown chip version, ver=%x -----\n", chip_ver));
+	}
+
+	strcat(fw_path, chipver_tag);
+	strcat(nv_path, chipver_tag);
+
+	return 0;
+}
+
+static int
+concate_revision_bcm43454(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	char chipver_tag[10] = {0, };
+#ifdef SUPPORT_MULTIPLE_BOARD_REV_FROM_DT
+	int base_system_rev_for_nv = 0;
+#endif /* SUPPORT_MULTIPLE_BOARD_REV_FROM_DT */
+
+	DHD_TRACE(("%s: BCM43454 Multiple Revision Check\n", __FUNCTION__));
+	if (bus->sih->chip != BCM43454_CHIP_ID) {
+		DHD_ERROR(("%s:Chip is not BCM43454!\n", __FUNCTION__));
+		return -1;
+	}
+#ifdef SUPPORT_MULTIPLE_BOARD_REV_FROM_DT
+	base_system_rev_for_nv = dhd_get_system_rev();
+	if (base_system_rev_for_nv > 0) {
+		DHD_ERROR(("----- Board Rev  [%d] -----\n", base_system_rev_for_nv));
+		sprintf(chipver_tag, "_r%02d", base_system_rev_for_nv);
+	}
+#endif /* SUPPORT_MULTIPLE_BOARD_REV_FROM_DT */
+#ifdef SUPPORT_MULTIPLE_BOARD_REV_FROM_HW
+	DHD_ERROR(("----- Rev [%d] Fot MULTIPLE Board. -----\n", system_hw_rev));
+	if ((system_hw_rev >= 8) && (system_hw_rev <= 11)) {
+		DHD_ERROR(("This HW is Rev 08 ~ 11. this is For FD-HW\n"));
+		strcat(chipver_tag, "_FD");
+	}
+#endif /* SUPPORT_MULTIPLE_BOARD_REV_FROM_HW */
+
+	strcat(nv_path, chipver_tag);
+	return 0;
+}
+
+int
+concate_revision(dhd_bus_t *bus, char *fw_path, char *nv_path)
+{
+	int res = 0;
+
+	if (!bus || !bus->sih) {
+		DHD_ERROR(("%s:Bus is Invalid\n", __FUNCTION__));
+		return -1;
+	}
+
+	switch (bus->sih->chip) {
+	case BCM4335_CHIP_ID:
+		res = concate_revision_bcm4335(bus, fw_path, nv_path);
+
+		break;
+	case BCM4339_CHIP_ID:
+		res = concate_revision_bcm4339(bus, fw_path, nv_path);
+		break;
+	case BCM4350_CHIP_ID:
+		res = concate_revision_bcm4350(bus, fw_path, nv_path);
+		break;
+	case BCM4354_CHIP_ID:
+		res = concate_revision_bcm4354(bus, fw_path, nv_path);
+		break;
+	case BCM43454_CHIP_ID:
+		res = concate_revision_bcm43454(bus, fw_path, nv_path);
+		break;
+
+		/* XXX: Add New Multiple CHIP ID */
+	default:
+		DHD_ERROR(("REVISION SPECIFIC feature is not required\n"));
+		/* XXX: if revision specific feature is not required then return true always */
+		return res;
+	}
+
+	if (res == 0) {
+#ifdef BCMDBG
+		printf("dhd concatenated fw & nv:\n   fw_path:%s\n"
+			"   nv_path:%s\n", fw_path, nv_path);
+		printf("make sure they exist\n");
+#endif
+	}
+	return res;
+}
+#endif /* SUPPORT_MULTIPLE_REVISION */
+
+#if defined(NDIS)
+void
+dhd_bus_reject_ioreqs(dhd_pub_t *dhdp, bool reject)
+{
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	bcmsdh_reject_ioreqs(dhdp->bus->sdh, reject);
+}
+
+void
+dhd_bus_waitfor_iodrain(dhd_pub_t *dhdp)
+{
+
+	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
+
+	bcmsdh_waitfor_iodrain(dhdp->bus->sdh);
+}
+#endif /* (NDIS) */
+
+void
+dhd_bus_update_fw_nv_path(struct dhd_bus *bus, char *pfw_path, char *pnv_path,
+									char *pclm_path, char *pconf_path)
+{
+	bus->fw_path = pfw_path;
+	bus->nv_path = pnv_path;
+	bus->dhd->clm_path = pclm_path;
+	bus->dhd->conf_path = pconf_path;
+}
+
+int
+dhd_enableOOB(dhd_pub_t *dhd, bool sleep)
+{
+	dhd_bus_t *bus = dhd->bus;
+	sdpcmd_regs_t *regs = bus->regs;
+	uint retries = 0;
+
+	if (sleep) {
+		dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+		/* Tell device to start using OOB wakeup */
+		W_SDREG(SMB_USE_OOB, &regs->tosbmailbox, retries);
+		if (retries > retry_limit) {
+			DHD_ERROR(("CANNOT SIGNAL CHIP, WILL NOT WAKE UP!!\n"));
+			return BCME_BUSY;
+		}
+		/* Turn off our contribution to the HT clock request */
+		dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+	} else {
+		/* Make sure the controller has the bus up */
+		dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+		/* Send misc interrupt to indicate OOB not needed */
+		W_SDREG(0, &regs->tosbmailboxdata, retries);
+		if (retries <= retry_limit)
+			W_SDREG(SMB_DEV_INT, &regs->tosbmailbox, retries);
+
+		if (retries > retry_limit)
+			DHD_ERROR(("CANNOT SIGNAL CHIP TO CLEAR OOB!!\n"));
+
+		/* Make sure we have SD bus access */
+		dhdsdio_clkctl(bus, CLK_SDONLY, FALSE);
+	}
+	return BCME_OK;
+}
+
+void
+dhd_bus_pktq_flush(dhd_pub_t *dhdp)
+{
+	dhd_bus_t *bus = dhdp->bus;
+	bool wlfc_enabled = FALSE;
+
+#ifdef PROP_TXSTATUS
+	wlfc_enabled = (dhd_wlfc_cleanup_txq(dhdp, NULL, 0) != WLFC_UNSUPPORTED);
+#endif
+	if (!wlfc_enabled) {
+#ifdef DHDTCPACK_SUPPRESS
+		/* Clean tcp_ack_info_tbl in order to prevent access to flushed pkt,
+		 * when there is a newly coming packet from network stack.
+		 */
+		dhd_tcpack_info_tbl_clean(bus->dhd);
+#endif /* DHDTCPACK_SUPPRESS */
+		/* Clear the data packet queues */
+		pktq_flush(dhdp->osh, &bus->txq, TRUE);
+	}
+}
+
+#ifdef BCMSDIO
+int
+dhd_sr_config(dhd_pub_t *dhd, bool on)
+{
+	dhd_bus_t *bus = dhd->bus;
+
+	if (!bus->_srenab)
+		return -1;
+
+	return dhdsdio_clk_devsleep_iovar(bus, on);
+}
+
+uint16
+dhd_get_chipid(struct dhd_bus *bus)
+{
+	if (bus && bus->sih)
+		return (uint16)bus->sih->chip;
+	else
+		return 0;
+}
+#endif /* BCMSDIO */
+
+#ifdef DEBUGGER
+static uint32
+dhd_sdio_reg_read(struct dhd_bus *bus, ulong addr)
+{
+	uint32 rval;
+
+	dhd_os_sdlock(bus->dhd);
+
+	BUS_WAKE(bus);
+
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+	rval = bcmsdh_reg_read(bus->sdh, addr, 4);
+
+	dhd_os_sdunlock(bus->dhd);
+
+	return rval;
+}
+
+static void
+dhd_sdio_reg_write(struct dhd_bus *bus, ulong addr, uint32 val)
+{
+	dhd_os_sdlock(bus->dhd);
+
+	BUS_WAKE(bus);
+
+	dhdsdio_clkctl(bus, CLK_AVAIL, FALSE);
+
+	bcmsdh_reg_write(bus->sdh, addr, 4, val);
+
+	dhd_os_sdunlock(bus->dhd);
+}
+
+#endif /* DEBUGGER */
+
+#if defined(SOFTAP_TPUT_ENHANCE)
+void dhd_bus_setidletime(dhd_pub_t *dhdp, int idle_time)
+{
+	if (!dhdp || !dhdp->bus) {
+		DHD_ERROR(("%s:Bus is Invalid\n", __FUNCTION__));
+		return;
+	}
+	dhdp->bus->idletime = idle_time;
+}
+
+void dhd_bus_getidletime(dhd_pub_t *dhdp, int* idle_time)
+{
+	if (!dhdp || !dhdp->bus) {
+		DHD_ERROR(("%s:Bus is Invalid\n", __FUNCTION__));
+		return;
+	}
+
+	if (!idle_time) {
+		DHD_ERROR(("%s:Arg idle_time is NULL\n", __FUNCTION__));
+		return;
+	}
+	*idle_time = dhdp->bus->idletime;
+}
+#endif /* SOFTAP_TPUT_ENHANCE */
+
+#if defined(BT_OVER_SDIO)
+uint8 dhd_bus_cfg_read(void *h, uint fun_num, uint32 addr, int *err)
+{
+	uint8 intrd;
+	dhd_pub_t *dhdp = (dhd_pub_t *)h;
+	dhd_bus_t *bus = (dhd_bus_t *)dhdp->bus;
+
+	dhd_os_sdlock(bus->dhd);
+
+	intrd = bcmsdh_cfg_read(bus->sdh, fun_num, addr, err);
+
+	dhd_os_sdunlock(bus->dhd);
+
+	return intrd;
+} EXPORT_SYMBOL(dhd_bus_cfg_read);
+
+void dhd_bus_cfg_write(void *h, uint fun_num, uint32 addr, uint8 val, int *err)
+{
+	dhd_pub_t *dhdp = (dhd_pub_t *)h;
+	dhd_bus_t *bus = (dhd_bus_t *)dhdp->bus;
+
+	dhd_os_sdlock(bus->dhd);
+
+	bcmsdh_cfg_write(bus->sdh, fun_num, addr, val, err);
+
+	dhd_os_sdunlock(bus->dhd);
+
+} EXPORT_SYMBOL(dhd_bus_cfg_write);
+
+static int
+extract_hex_field(char * line, uint16 start_pos, uint16 num_chars, uint16 * value)
+{
+	char field [8];
+
+	strlcpy(field, line + start_pos, sizeof(field));
+
+	return (sscanf (field, "%hX", value) == 1);
+}
+
+static int
+read_more_btbytes(struct dhd_bus *bus, void * file, char *line, int * addr_mode, uint16 * hi_addr,
+	uint32 * dest_addr, uint8 *data_bytes, uint32 * num_bytes)
+{
+	int		str_len;
+	uint16	num_data_bytes, addr, data_pos, type, w, i;
+	uint32	abs_base_addr32 = 0;
+	*num_bytes = 0;
+
+	while (!*num_bytes)
+	{
+		str_len = dhd_os_gets_image(bus->dhd, line, BTFW_MAX_STR_LEN, file);
+
+		DHD_TRACE(("%s: Len :0x%x  %s\n", __FUNCTION__, str_len, line));
+
+		if (str_len == 0) {
+			break;
+		} else if (str_len > 9) {
+			extract_hex_field(line, 1, 2, &num_data_bytes);
+			extract_hex_field(line, 3, 4, &addr);
+			extract_hex_field(line, 7, 2, &type);
+
+			data_pos = 9;
+			for (i = 0; i < num_data_bytes; i++) {
+				extract_hex_field(line, data_pos, 2, &w);
+				data_bytes [i] = (uint8)(w & 0x00FF);
+				data_pos += 2;
+			}
+
+			if (type == BTFW_HEX_LINE_TYPE_EXTENDED_ADDRESS) {
+				*hi_addr = (data_bytes [0] << 8) | data_bytes [1];
+				*addr_mode = BTFW_ADDR_MODE_EXTENDED;
+			} else if (type == BTFW_HEX_LINE_TYPE_EXTENDED_SEGMENT_ADDRESS) {
+				*hi_addr = (data_bytes [0] << 8) | data_bytes [1];
+				*addr_mode = BTFW_ADDR_MODE_SEGMENT;
+			} else if (type == BTFW_HEX_LINE_TYPE_ABSOLUTE_32BIT_ADDRESS) {
+				abs_base_addr32 = (data_bytes [0] << 24) | (data_bytes [1] << 16) |
+					(data_bytes [2] << 8) | data_bytes [3];
+				*addr_mode = BTFW_ADDR_MODE_LINEAR32;
+			} else if (type == BTFW_HEX_LINE_TYPE_DATA) {
+				*dest_addr = addr;
+				if (*addr_mode == BTFW_ADDR_MODE_EXTENDED)
+					*dest_addr += (*hi_addr << 16);
+				else if (*addr_mode == BTFW_ADDR_MODE_SEGMENT)
+					*dest_addr += (*hi_addr << 4);
+				else if (*addr_mode == BTFW_ADDR_MODE_LINEAR32)
+					*dest_addr += abs_base_addr32;
+				*num_bytes = num_data_bytes;
+			}
+		}
+	}
+	return (*num_bytes > 0);
+}
+
+static int
+_dhdsdio_download_btfw(struct dhd_bus *bus)
+{
+	int	bcm_error = -1;
+	void	*image = NULL;
+	uint8	*mem_blk = NULL, *mem_ptr = NULL, *data_ptr = NULL;
+
+	uint32 offset_addr = 0, offset_len = 0, bytes_to_write = 0;
+
+	char	*line = NULL;
+	uint32	dest_addr = 0, num_bytes;
+	uint16	hiAddress = 0;
+	uint32	start_addr, start_data, end_addr, end_data, i, index, pad,
+	bt2wlan_pwrup_adr;
+
+	int	addr_mode		= BTFW_ADDR_MODE_EXTENDED;
+
+	/* Out immediately if no image to download */
+	if ((bus->btfw_path == NULL) || (bus->btfw_path[0] == '\0')) {
+		return 0;
+	}
+
+	/* XXX: Should succeed in opening image if it is actually given through registry
+	 * entry or in module param.
+	 */
+	image = dhd_os_open_image1(bus->dhd, bus->btfw_path);
+	if (image == NULL)
+		goto err;
+
+	mem_ptr = mem_blk = MALLOC(bus->dhd->osh, BTFW_DOWNLOAD_BLK_SIZE + DHD_SDALIGN);
+	if (mem_blk == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__,
+			BTFW_DOWNLOAD_BLK_SIZE + DHD_SDALIGN));
+		goto err;
+	}
+	if ((uint32)(uintptr)mem_blk % DHD_SDALIGN)
+		mem_ptr += (DHD_SDALIGN - ((uint32)(uintptr)mem_blk % DHD_SDALIGN));
+
+	data_ptr = MALLOC(bus->dhd->osh, BTFW_DOWNLOAD_BLK_SIZE - 8);
+	if (data_ptr == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n", __FUNCTION__,
+			BTFW_DOWNLOAD_BLK_SIZE - 8));
+		goto err;
+	}
+	/* Write to BT register to hold WLAN wake high during BT FW download */
+	bt2wlan_pwrup_adr = BTMEM_OFFSET + BT2WLAN_PWRUP_ADDR;
+	bcmsdh_reg_write(bus->sdh, bt2wlan_pwrup_adr, 4, BT2WLAN_PWRUP_WAKE);
+	/*
+	  * Wait for at least 2msec for the clock to be ready/Available.
+	  */
+	OSL_DELAY(2000);
+
+	line = MALLOC(bus->dhd->osh, BTFW_MAX_STR_LEN);
+	if (line == NULL) {
+		DHD_ERROR(("%s: Failed to allocate memory %d bytes\n",
+			__FUNCTION__, BTFW_MAX_STR_LEN));
+		goto err;
+	}
+	memset(line, 0, BTFW_MAX_STR_LEN);
+
+	while (read_more_btbytes (bus, image, line, &addr_mode, &hiAddress, &dest_addr,
+		data_ptr, &num_bytes)) {
+
+		DHD_TRACE(("read %d bytes at address %08X\n", num_bytes, dest_addr));
+
+		start_addr = BTMEM_OFFSET + dest_addr;
+		index = 0;
+
+		/* Make sure the start address is 4 byte aligned to avoid alignment issues
+		 * with SD host controllers
+		 */
+		if (!ISALIGNED(start_addr, 4)) {
+			pad = start_addr % 4;
+			start_addr = ROUNDDN(start_addr, 4);
+			start_data = bcmsdh_reg_read(bus->sdh, start_addr, 4);
+			for (i = 0; i < pad; i++, index++) {
+			    mem_ptr[index] = (uint8)((uint8 *)&start_data)[i];
+			}
+		}
+		bcopy(data_ptr, &(mem_ptr[index]), num_bytes);
+		index += num_bytes;
+
+		/* Make sure the length is multiple of 4bytes to avoid alignment issues
+		 * with SD host controllers
+		 */
+		end_addr = start_addr + index;
+		if (!ISALIGNED(end_addr, 4)) {
+			end_addr = ROUNDDN(end_addr, 4);
+			end_data = bcmsdh_reg_read(bus->sdh, end_addr, 4);
+			for (i = (index % 4); i < 4; i++, index++) {
+				mem_ptr[index] = (uint8)((uint8 *)&end_data)[i];
+			}
+		}
+
+		offset_addr = start_addr & 0xFFF;
+		offset_len =  offset_addr + index;
+		if (offset_len <= 0x1000) {
+			bcm_error = dhdsdio_membytes(bus, TRUE, start_addr, mem_ptr, index);
+			if (bcm_error) {
+				DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+				__FUNCTION__, bcm_error, num_bytes, start_addr));
+				goto err;
+			}
+		}
+		else {
+			bytes_to_write = 0x1000 - offset_addr;
+			bcm_error = dhdsdio_membytes(bus, TRUE, start_addr, mem_ptr,
+				bytes_to_write);
+			if (bcm_error) {
+				DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+				__FUNCTION__, bcm_error, num_bytes, start_addr));
+				goto err;
+			}
+
+			OSL_DELAY(10000);
+
+			bcm_error = dhdsdio_membytes(bus, TRUE, (start_addr + bytes_to_write),
+				(mem_ptr + bytes_to_write), (index - bytes_to_write));
+			if (bcm_error) {
+				DHD_ERROR(("%s: error %d on writing %d membytes at 0x%08x\n",
+					__FUNCTION__, bcm_error, num_bytes, start_addr));
+				goto err;
+			}
+		}
+		memset(line, 0, BTFW_MAX_STR_LEN);
+	}
+
+	bcm_error = 0;
+err:
+	if (mem_blk)
+		MFREE(bus->dhd->osh, mem_blk, BTFW_DOWNLOAD_BLK_SIZE + DHD_SDALIGN);
+
+	if (data_ptr)
+		MFREE(bus->dhd->osh, data_ptr, BTFW_DOWNLOAD_BLK_SIZE - 8);
+
+	if (line)
+		MFREE(bus->dhd->osh, line, BTFW_MAX_STR_LEN);
+
+	if (image)
+		dhd_os_close_image1(bus->dhd, image);
+
+	return bcm_error;
+}
+
+static int
+dhdsdio_download_btfw(struct dhd_bus *bus, osl_t *osh, void *sdh)
+{
+	int ret;
+
+	DHD_TRACE(("%s: btfw path=%s\n",
+		__FUNCTION__, bus->btfw_path));
+	DHD_OS_WAKE_LOCK(bus->dhd);
+	dhd_os_sdlock(bus->dhd);
+
+	/* Download the firmware */
+	ret = _dhdsdio_download_btfw(bus);
+
+	dhd_os_sdunlock(bus->dhd);
+	DHD_OS_WAKE_UNLOCK(bus->dhd);
+
+	return ret;
+}
+
+int
+dhd_bus_download_btfw(struct dhd_bus *bus, osl_t *osh,
+                          char *pbtfw_path)
+{
+	int ret;
+
+	bus->btfw_path = pbtfw_path;
+
+	ret = dhdsdio_download_btfw(bus, osh, bus->sdh);
+
+	return ret;
+}
+#endif /* defined (BT_OVER_SDIO) */
+
+void
+dhd_bus_dump_trap_info(dhd_bus_t *bus, struct bcmstrbuf *strbuf)
+{
+	trap_t *tr = &bus->dhd->last_trap_info;
+
+	bcm_bprintf(strbuf,
+		"Dongle trap type 0x%x @ epc 0x%x, cpsr 0x%x, spsr 0x%x, sp 0x%x,"
+		"lp 0x%x, rpc 0x%x Trap offset 0x%x, "
+		"r0 0x%x, r1 0x%x, r2 0x%x, r3 0x%x, "
+		"r4 0x%x, 